Repository: delta-io/delta Branch: master Commit: cd215cb69353 Files: 3757 Total size: 22.5 MB Directory structure: gitextract_9jbpsnjv/ ├── .gitattributes ├── .github/ │ ├── CODEOWNERS │ ├── ISSUE_TEMPLATE/ │ │ ├── bug-issue.md │ │ ├── feature-request.md │ │ └── protocol-rfc.md │ ├── PULL_REQUEST_TEMPLATE.md │ └── workflows/ │ ├── build.yaml │ ├── flink_test.yaml │ ├── iceberg_test.yaml │ ├── kernel_docs.yaml │ ├── kernel_test.yaml │ ├── kernel_unitycatalog_test.yaml │ ├── publish_docs.yaml │ ├── spark_examples_test.yaml │ ├── spark_python_test.yaml │ ├── spark_test.yaml │ └── unidoc.yaml ├── .gitignore ├── .sbtopts ├── .scalafmt.conf ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE.txt ├── NOTICE.txt ├── PROTOCOL.md ├── README.md ├── benchmarks/ │ ├── README.md │ ├── build/ │ │ ├── sbt │ │ └── sbt-launch-lib.bash │ ├── build.sbt │ ├── infrastructure/ │ │ ├── aws/ │ │ │ └── terraform/ │ │ │ ├── .terraform.lock.hcl │ │ │ ├── README.md │ │ │ ├── main.tf │ │ │ ├── modules/ │ │ │ │ ├── networking/ │ │ │ │ │ ├── main.tf │ │ │ │ │ ├── outputs.tf │ │ │ │ │ └── variables.tf │ │ │ │ ├── processing/ │ │ │ │ │ ├── main.tf │ │ │ │ │ ├── outputs.tf │ │ │ │ │ └── variables.tf │ │ │ │ └── storage/ │ │ │ │ ├── main.tf │ │ │ │ └── variables.tf │ │ │ ├── outputs.tf │ │ │ ├── providers.tf │ │ │ ├── variables.tf │ │ │ └── versions.tf │ │ └── gcp/ │ │ └── terraform/ │ │ ├── .terraform.lock.hcl │ │ ├── README.md │ │ ├── main.tf │ │ ├── modules/ │ │ │ ├── processing/ │ │ │ │ ├── data.tf │ │ │ │ ├── main.tf │ │ │ │ ├── outputs.tf │ │ │ │ └── variables.tf │ │ │ └── storage/ │ │ │ ├── main.tf │ │ │ └── variables.tf │ │ ├── outputs.tf │ │ ├── providers.tf │ │ ├── variables.tf │ │ └── versions.tf │ ├── project/ │ │ ├── build.properties │ │ └── plugins.sbt │ ├── run-benchmark.py │ ├── scripts/ │ │ ├── benchmarks.py │ │ └── utils.py │ └── src/ │ └── main/ │ └── scala/ │ ├── benchmark/ │ │ ├── Benchmark.scala │ │ ├── MergeBenchmark.scala │ │ ├── MergeDataLoad.scala │ │ ├── MergeTestCases.scala │ │ ├── TPCDSBenchmark.scala │ │ ├── TPCDSBenchmarkQueries.scala │ │ ├── TPCDSDataLoad.scala │ │ └── TestBenchmark.scala │ └── org/ │ └── apache/ │ └── spark/ │ └── SparkUtils.scala ├── build/ │ ├── sbt │ ├── sbt-config/ │ │ └── repositories │ └── sbt-launch-lib.bash ├── build.sbt ├── connectors/ │ ├── .gitignore │ ├── README.md │ ├── golden-tables/ │ │ └── src/ │ │ ├── main/ │ │ │ ├── resources/ │ │ │ │ └── golden/ │ │ │ │ ├── 124-decimal-decode-bug/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-2abbde89-2d0f-465e-a2f0-3e84f1b84654-c000.snappy.parquet │ │ │ │ │ └── part-00001-5419c9a2-bb44-454f-a109-6e6c6f000a24-c000.snappy.parquet │ │ │ │ ├── 125-iterator-bug/ │ │ │ │ │ ├── .part-00000-15088d9b-5348-490b-933d-5bf9b7d0b223-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-1b8ea57e-424b-4068-8d0e-707edf853376-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-223768c3-2e58-4e8a-9d15-54fa113e8c21-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-2a248db5-8f96-423c-a0f7-c503fe640c6a-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-3f0f0396-41aa-4fa7-954a-c5b22f5b157a-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-66d18d0c-8cab-4cfa-a2c6-7e90df860b5a-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-93beced9-3a9d-4519-b31a-5602a972ffa4-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-c4738537-d851-4caa-9596-d543afa47196-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-c855206c-f42a-4b53-a526-08a9a957ad58-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-d8e947c6-4f26-455b-a25f-84acb1240f3a-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-f0b12818-15f5-4476-8ebc-9235c74408d2-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-f9490ff6-f374-4b40-9d76-22addae085d1-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000010.checkpoint.parquet.crc │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ │ ├── 00000000000000000004.json │ │ │ │ │ │ ├── 00000000000000000005.json │ │ │ │ │ │ ├── 00000000000000000006.json │ │ │ │ │ │ ├── 00000000000000000007.json │ │ │ │ │ │ ├── 00000000000000000008.json │ │ │ │ │ │ ├── 00000000000000000009.json │ │ │ │ │ │ ├── 00000000000000000010.checkpoint.parquet │ │ │ │ │ │ ├── 00000000000000000010.json │ │ │ │ │ │ ├── 00000000000000000011.json │ │ │ │ │ │ └── _last_checkpoint │ │ │ │ │ ├── part-00000-15088d9b-5348-490b-933d-5bf9b7d0b223-c000.snappy.parquet │ │ │ │ │ ├── part-00000-1b8ea57e-424b-4068-8d0e-707edf853376-c000.snappy.parquet │ │ │ │ │ ├── part-00000-223768c3-2e58-4e8a-9d15-54fa113e8c21-c000.snappy.parquet │ │ │ │ │ ├── part-00000-2a248db5-8f96-423c-a0f7-c503fe640c6a-c000.snappy.parquet │ │ │ │ │ ├── part-00000-3f0f0396-41aa-4fa7-954a-c5b22f5b157a-c000.snappy.parquet │ │ │ │ │ ├── part-00000-66d18d0c-8cab-4cfa-a2c6-7e90df860b5a-c000.snappy.parquet │ │ │ │ │ ├── part-00000-93beced9-3a9d-4519-b31a-5602a972ffa4-c000.snappy.parquet │ │ │ │ │ ├── part-00000-c4738537-d851-4caa-9596-d543afa47196-c000.snappy.parquet │ │ │ │ │ ├── part-00000-c855206c-f42a-4b53-a526-08a9a957ad58-c000.snappy.parquet │ │ │ │ │ ├── part-00000-d8e947c6-4f26-455b-a25f-84acb1240f3a-c000.snappy.parquet │ │ │ │ │ ├── part-00000-f0b12818-15f5-4476-8ebc-9235c74408d2-c000.snappy.parquet │ │ │ │ │ └── part-00000-f9490ff6-f374-4b40-9d76-22addae085d1-c000.snappy.parquet │ │ │ │ ├── basic-decimal-table/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part=-2342342.23423/ │ │ │ │ │ │ ├── .part-00000-8f850371-9b03-42c4-9d22-f83bc81c9b68.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00000-8f850371-9b03-42c4-9d22-f83bc81c9b68.c000.snappy.parquet │ │ │ │ │ ├── part=0.00004/ │ │ │ │ │ │ ├── .part-00000-1cb60e36-6cd4-4191-a318-ae9355f877c3.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00000-1cb60e36-6cd4-4191-a318-ae9355f877c3.c000.snappy.parquet │ │ │ │ │ ├── part=234.00000/ │ │ │ │ │ │ ├── .part-00000-ac109189-97e5-49af-947f-335a5e46ee5c.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00000-ac109189-97e5-49af-947f-335a5e46ee5c.c000.snappy.parquet │ │ │ │ │ └── part=2342222.23454/ │ │ │ │ │ ├── .part-00000-d5a0c70f-7cd3-4d32-a9c0-7171a06547c6.c000.snappy.parquet.crc │ │ │ │ │ └── part-00000-d5a0c70f-7cd3-4d32-a9c0-7171a06547c6.c000.snappy.parquet │ │ │ │ ├── basic-decimal-table-legacy/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part=-2342342.23423/ │ │ │ │ │ │ ├── .part-00000-ba2f74ac-7b9b-47b9-a287-97d92bd20efc.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00000-ba2f74ac-7b9b-47b9-a287-97d92bd20efc.c000.snappy.parquet │ │ │ │ │ ├── part=0.00004/ │ │ │ │ │ │ ├── .part-00000-3de65390-7061-47d6-8995-cbb632b4b203.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00000-3de65390-7061-47d6-8995-cbb632b4b203.c000.snappy.parquet │ │ │ │ │ ├── part=234.00000/ │ │ │ │ │ │ ├── .part-00000-654d80b0-611a-4ff3-a8e6-2328dd21cf11.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00000-654d80b0-611a-4ff3-a8e6-2328dd21cf11.c000.snappy.parquet │ │ │ │ │ └── part=2342222.23454/ │ │ │ │ │ ├── .part-00000-fe848a88-0465-4b4f-8414-25e6da7062f8.c000.snappy.parquet.crc │ │ │ │ │ └── part-00000-fe848a88-0465-4b4f-8414-25e6da7062f8.c000.snappy.parquet │ │ │ │ ├── basic-with-inserts-deletes-checkpoint/ │ │ │ │ │ ├── .part-00000-0869ab64-e69d-407f-80d4-1a2ea1f69d11-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-1b0098ea-c696-4470-84cc-d43bb7afb833-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-26da113c-2e45-4aba-b1ce-6eb5e46c53f7-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-4b448490-06f4-4c74-9f65-9f36ae68e3b2-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-60f14460-c8e0-41b4-a33f-1a83bb59f13c-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-7d1a368c-74ea-42df-9527-2c9a7c8292b9-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-b09fdf65-0ae3-44d0-96d0-1d85a121b76a-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-b326e43b-3e01-4cf1-b8ff-c73c8abd1616-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-bd8763c3-45e4-435e-acd6-8e599aa840bc-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-c92cba9e-6c07-4a93-916a-0a6e115e39b3-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-c967edfa-f104-44ea-b0da-8bc1f5402af4-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-ca2d0b26-c15c-454f-a933-fc724e15e5f1-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-da82aeb5-4edb-4cc1-91ef-970c75c965cc-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-f80053c6-2b0d-41ed-ab5f-61ef1503cae6-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ │ ├── .00000000000000000002.json.crc │ │ │ │ │ │ ├── .00000000000000000003.json.crc │ │ │ │ │ │ ├── .00000000000000000004.json.crc │ │ │ │ │ │ ├── .00000000000000000005.json.crc │ │ │ │ │ │ ├── .00000000000000000006.json.crc │ │ │ │ │ │ ├── .00000000000000000007.json.crc │ │ │ │ │ │ ├── .00000000000000000008.json.crc │ │ │ │ │ │ ├── .00000000000000000009.json.crc │ │ │ │ │ │ ├── .00000000000000000010.checkpoint.parquet.crc │ │ │ │ │ │ ├── .00000000000000000010.json.crc │ │ │ │ │ │ ├── .00000000000000000011.json.crc │ │ │ │ │ │ ├── .00000000000000000012.json.crc │ │ │ │ │ │ ├── .00000000000000000013.json.crc │ │ │ │ │ │ ├── ._last_checkpoint.crc │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ │ ├── 00000000000000000004.json │ │ │ │ │ │ ├── 00000000000000000005.json │ │ │ │ │ │ ├── 00000000000000000006.json │ │ │ │ │ │ ├── 00000000000000000007.json │ │ │ │ │ │ ├── 00000000000000000008.json │ │ │ │ │ │ ├── 00000000000000000009.json │ │ │ │ │ │ ├── 00000000000000000010.checkpoint.parquet │ │ │ │ │ │ ├── 00000000000000000010.json │ │ │ │ │ │ ├── 00000000000000000011.json │ │ │ │ │ │ ├── 00000000000000000012.json │ │ │ │ │ │ ├── 00000000000000000013.json │ │ │ │ │ │ └── _last_checkpoint │ │ │ │ │ ├── part-00000-0869ab64-e69d-407f-80d4-1a2ea1f69d11-c000.snappy.parquet │ │ │ │ │ ├── part-00000-1b0098ea-c696-4470-84cc-d43bb7afb833-c000.snappy.parquet │ │ │ │ │ ├── part-00000-26da113c-2e45-4aba-b1ce-6eb5e46c53f7-c000.snappy.parquet │ │ │ │ │ ├── part-00000-4b448490-06f4-4c74-9f65-9f36ae68e3b2-c000.snappy.parquet │ │ │ │ │ ├── part-00000-60f14460-c8e0-41b4-a33f-1a83bb59f13c-c000.snappy.parquet │ │ │ │ │ ├── part-00000-7d1a368c-74ea-42df-9527-2c9a7c8292b9-c000.snappy.parquet │ │ │ │ │ ├── part-00000-b09fdf65-0ae3-44d0-96d0-1d85a121b76a-c000.snappy.parquet │ │ │ │ │ ├── part-00000-b326e43b-3e01-4cf1-b8ff-c73c8abd1616-c000.snappy.parquet │ │ │ │ │ ├── part-00000-bd8763c3-45e4-435e-acd6-8e599aa840bc-c000.snappy.parquet │ │ │ │ │ ├── part-00000-c92cba9e-6c07-4a93-916a-0a6e115e39b3-c000.snappy.parquet │ │ │ │ │ ├── part-00000-c967edfa-f104-44ea-b0da-8bc1f5402af4-c000.snappy.parquet │ │ │ │ │ ├── part-00000-ca2d0b26-c15c-454f-a933-fc724e15e5f1-c000.snappy.parquet │ │ │ │ │ ├── part-00000-da82aeb5-4edb-4cc1-91ef-970c75c965cc-c000.snappy.parquet │ │ │ │ │ └── part-00000-f80053c6-2b0d-41ed-ab5f-61ef1503cae6-c000.snappy.parquet │ │ │ │ ├── basic-with-inserts-merge/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ └── 00000000000000000001.json │ │ │ │ │ ├── part-00000-992247c6-6cf4-45f8-8367-11a5e14b8ea9-c000.snappy.parquet │ │ │ │ │ ├── part-00000-b4335bad-f5f0-4426-9ec4-14ed854f862b-c000.snappy.parquet │ │ │ │ │ └── part-00001-b80a2dea-5a83-4580-96d5-4977d14195ab-c000.snappy.parquet │ │ │ │ ├── basic-with-inserts-overwrite-restore/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ ├── 00000000000000000003.checkpoint.parquet │ │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ │ └── _last_checkpoint │ │ │ │ │ ├── part-00000-5e752668-638c-4e95-9521-5e88926e3169-c000.snappy.parquet │ │ │ │ │ ├── part-00000-79fa68ed-3d70-4f61-95da-9eb676b24a98-c000.snappy.parquet │ │ │ │ │ ├── part-00000-d24c9b15-187d-4542-90ef-7834bfaa4971-c000.snappy.parquet │ │ │ │ │ ├── part-00001-180c081a-f358-4bf9-8daa-4d04a5aa7f51-c000.snappy.parquet │ │ │ │ │ ├── part-00001-66f56273-e583-4a88-9da6-2c199bdaf665-c000.snappy.parquet │ │ │ │ │ └── part-00001-bc9b37c2-a201-499d-b604-93623e2de1d6-c000.snappy.parquet │ │ │ │ ├── basic-with-inserts-updates/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ └── 00000000000000000001.json │ │ │ │ │ ├── part-00000-6dfaec75-bd45-4fd6-b20f-7d58c9341479-c000.snappy.parquet │ │ │ │ │ ├── part-00000-f9886fc2-20a0-42fe-8b30-c3abb5e3c720-c000.snappy.parquet │ │ │ │ │ └── part-00001-13a6bfd9-3835-44dd-b4f1-465aa95b2bf4-c000.snappy.parquet │ │ │ │ ├── basic-with-vacuum-protocol-check-feature/ │ │ │ │ │ ├── .part-00000-e719b63b-4142-4bad-9776-45642d5858ae-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-fd905e0a-6d0c-4ce3-bb41-147517448b3b-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ └── 00000000000000000001.json │ │ │ │ │ ├── part-00000-e719b63b-4142-4bad-9776-45642d5858ae-c000.snappy.parquet │ │ │ │ │ └── part-00001-fd905e0a-6d0c-4ce3-bb41-147517448b3b-c000.snappy.parquet │ │ │ │ ├── canonicalized-paths-normal-a/ │ │ │ │ │ └── _delta_log/ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ └── 00000000000000000001.json │ │ │ │ ├── canonicalized-paths-normal-b/ │ │ │ │ │ └── _delta_log/ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ └── 00000000000000000001.json │ │ │ │ ├── canonicalized-paths-special-a/ │ │ │ │ │ └── _delta_log/ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ └── 00000000000000000001.json │ │ │ │ ├── canonicalized-paths-special-b/ │ │ │ │ │ └── _delta_log/ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ └── 00000000000000000001.json │ │ │ │ ├── commit-info-containing-arbitrary-operationParams-types/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.crc.crc │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ ├── .00000000000000000001.crc.crc │ │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ │ ├── .00000000000000000002.crc.crc │ │ │ │ │ │ ├── .00000000000000000002.json.crc │ │ │ │ │ │ ├── 00000000000000000000.crc │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.crc │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.crc │ │ │ │ │ │ └── 00000000000000000002.json │ │ │ │ │ ├── month=1/ │ │ │ │ │ │ ├── .part-00000-22d25ea7-a383-44df-ad22-6b06d871b547.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00000-c5babbd8-6013-484c-818f-22d546976866.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-22d25ea7-a383-44df-ad22-6b06d871b547.c000.snappy.parquet │ │ │ │ │ │ └── part-00000-c5babbd8-6013-484c-818f-22d546976866.c000.snappy.parquet │ │ │ │ │ └── month=2/ │ │ │ │ │ ├── .part-00000-129a0441-5f41-4e46-be33-fd0289e53614.c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-cc2a9650-0450-4879-9757-873b7f544510.c000.snappy.parquet.crc │ │ │ │ │ ├── part-00000-129a0441-5f41-4e46-be33-fd0289e53614.c000.snappy.parquet │ │ │ │ │ └── part-00000-cc2a9650-0450-4879-9757-873b7f544510.c000.snappy.parquet │ │ │ │ ├── corrupted-last-checkpoint/ │ │ │ │ │ └── _delta_log/ │ │ │ │ │ ├── .00000000000000000010.checkpoint.parquet.crc │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ ├── 00000000000000000004.json │ │ │ │ │ ├── 00000000000000000005.json │ │ │ │ │ ├── 00000000000000000006.json │ │ │ │ │ ├── 00000000000000000007.json │ │ │ │ │ ├── 00000000000000000008.json │ │ │ │ │ ├── 00000000000000000009.json │ │ │ │ │ ├── 00000000000000000010.checkpoint.parquet │ │ │ │ │ ├── 00000000000000000010.json │ │ │ │ │ └── _last_checkpoint │ │ │ │ ├── corrupted-last-checkpoint-kernel/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ │ ├── 00000000000000000004.json │ │ │ │ │ │ ├── 00000000000000000005.json │ │ │ │ │ │ ├── 00000000000000000006.json │ │ │ │ │ │ ├── 00000000000000000007.json │ │ │ │ │ │ ├── 00000000000000000008.json │ │ │ │ │ │ ├── 00000000000000000009.json │ │ │ │ │ │ ├── 00000000000000000010.checkpoint.parquet │ │ │ │ │ │ ├── 00000000000000000010.json │ │ │ │ │ │ ├── 00000000000000000011.json │ │ │ │ │ │ └── _last_checkpoint │ │ │ │ │ ├── part-00000-45318b19-5a29-4bb9-b273-1738e817d63e-c000.snappy.parquet │ │ │ │ │ ├── part-00000-45ddfb64-1797-4618-a4e4-58d687ae9d21-c000.snappy.parquet │ │ │ │ │ ├── part-00000-51f8ff2c-8e81-4031-94c9-93eae615d3e3-c000.snappy.parquet │ │ │ │ │ ├── part-00000-59a396e0-b0f4-4685-80f1-f58e07601862-c000.snappy.parquet │ │ │ │ │ ├── part-00000-69f4e384-139f-4b75-b51f-09213866a62a-c000.snappy.parquet │ │ │ │ │ ├── part-00000-82c1686f-287a-4e6f-8a7a-0099d54d7738-c000.snappy.parquet │ │ │ │ │ ├── part-00000-99f8ecc2-cc99-4e3e-866e-07135df25e52-c000.snappy.parquet │ │ │ │ │ ├── part-00000-a57ecbd0-7dad-4b6c-a3fe-8ab4f7e73f5a-c000.snappy.parquet │ │ │ │ │ ├── part-00000-bca1b163-25a1-4130-b74c-b905c61018ca-c000.snappy.parquet │ │ │ │ │ ├── part-00000-cbc535a8-3499-4339-be3f-9df89091871e-c000.snappy.parquet │ │ │ │ │ ├── part-00000-cd63e6e7-227f-4bae-8ffc-fad3bfea242c-c000.snappy.parquet │ │ │ │ │ ├── part-00000-d9d02879-5155-46d4-84a8-41c83c5df9e4-c000.snappy.parquet │ │ │ │ │ ├── part-00001-400931d7-721c-4dbc-82e6-5c29f1dfcde1-c000.snappy.parquet │ │ │ │ │ ├── part-00001-4eeaf77f-87b7-45bb-8e1f-1faf9c957918-c000.snappy.parquet │ │ │ │ │ ├── part-00001-71b04841-d4e6-4cd6-930a-5e33fd1bd7a0-c000.snappy.parquet │ │ │ │ │ ├── part-00001-81d22bd7-311e-4934-839e-f635ea6f364f-c000.snappy.parquet │ │ │ │ │ ├── part-00001-84978e4c-0e36-40d7-a3e0-c69204409c28-c000.snappy.parquet │ │ │ │ │ ├── part-00001-89b7b3e6-d076-43af-963f-3a4055a1eca6-c000.snappy.parquet │ │ │ │ │ ├── part-00001-8e839ba6-38f3-4093-8eb4-bc894159348c-c000.snappy.parquet │ │ │ │ │ ├── part-00001-a68acb2a-ac4f-46c2-940b-f962480a6517-c000.snappy.parquet │ │ │ │ │ ├── part-00001-c1199313-5eb1-4d9d-9cec-a43245621024-c000.snappy.parquet │ │ │ │ │ ├── part-00001-c4cbb8cf-9c18-4bab-bfa7-967faa14e15d-c000.snappy.parquet │ │ │ │ │ ├── part-00001-e471a872-a1ee-4610-9454-062854327ad6-c000.snappy.parquet │ │ │ │ │ └── part-00001-ef16b167-3dda-4681-bdd0-cd6bb9f07c30-c000.snappy.parquet │ │ │ │ ├── data-reader-absolute-paths-escaped-chars/ │ │ │ │ │ └── _delta_log/ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ └── 00000000000000000001.json │ │ │ │ ├── data-reader-array-complex-objects/ │ │ │ │ │ ├── .part-00000-a7d58b1a-7743-4bb0-b208-438bbe179c93-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-7b211746-0a31-4e77-9822-b0985158cd66-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-a7d58b1a-7743-4bb0-b208-438bbe179c93-c000.snappy.parquet │ │ │ │ │ └── part-00001-7b211746-0a31-4e77-9822-b0985158cd66-c000.snappy.parquet │ │ │ │ ├── data-reader-array-primitives/ │ │ │ │ │ ├── .part-00000-182665f0-30df-470d-a5cb-8d9d483ed390-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-2e274fe7-eb75-4b73-8c72-423ee747abc0-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-182665f0-30df-470d-a5cb-8d9d483ed390-c000.snappy.parquet │ │ │ │ │ └── part-00001-2e274fe7-eb75-4b73-8c72-423ee747abc0-c000.snappy.parquet │ │ │ │ ├── data-reader-date-types-America/ │ │ │ │ │ └── Los_Angeles/ │ │ │ │ │ ├── .part-00000-e85ca549-604b-4340-b56d-868e9acc78e8-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-1e808610-ee7f-44e7-be9b-be02c2bc5895-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-e85ca549-604b-4340-b56d-868e9acc78e8-c000.snappy.parquet │ │ │ │ │ └── part-00001-1e808610-ee7f-44e7-be9b-be02c2bc5895-c000.snappy.parquet │ │ │ │ ├── data-reader-date-types-Asia/ │ │ │ │ │ └── Beirut/ │ │ │ │ │ ├── .part-00000-58828e3c-041e-47b4-80dd-196ae1b1d1a6-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-8590d66f-6907-40a9-9e97-a4a098321340-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-58828e3c-041e-47b4-80dd-196ae1b1d1a6-c000.snappy.parquet │ │ │ │ │ └── part-00001-8590d66f-6907-40a9-9e97-a4a098321340-c000.snappy.parquet │ │ │ │ ├── data-reader-date-types-Etc/ │ │ │ │ │ └── GMT+9/ │ │ │ │ │ ├── .part-00000-23e032bb-e586-4573-9fc0-1c9a4c9a5081-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-d91bf3dd-78c9-4abf-aa54-e89228e8316c-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-23e032bb-e586-4573-9fc0-1c9a4c9a5081-c000.snappy.parquet │ │ │ │ │ └── part-00001-d91bf3dd-78c9-4abf-aa54-e89228e8316c-c000.snappy.parquet │ │ │ │ ├── data-reader-date-types-Iceland/ │ │ │ │ │ ├── .part-00000-8be8ec9f-d9af-474e-8ec9-35ec76debc6a-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-56f07a95-04d4-4c12-bf08-fd89cedc8559-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-8be8ec9f-d9af-474e-8ec9-35ec76debc6a-c000.snappy.parquet │ │ │ │ │ └── part-00001-56f07a95-04d4-4c12-bf08-fd89cedc8559-c000.snappy.parquet │ │ │ │ ├── data-reader-date-types-JST/ │ │ │ │ │ ├── .part-00000-3f9100ce-0b94-43cb-bb23-f0e36dc7af2b-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-dc211b29-0c30-41e8-8700-f8bb374964e1-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-3f9100ce-0b94-43cb-bb23-f0e36dc7af2b-c000.snappy.parquet │ │ │ │ │ └── part-00001-dc211b29-0c30-41e8-8700-f8bb374964e1-c000.snappy.parquet │ │ │ │ ├── data-reader-date-types-PST/ │ │ │ │ │ ├── .part-00000-0a103e9a-6236-470c-94f7-5f60926f01da-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-980a117f-027e-4396-81ce-3a5a8ac70815-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-0a103e9a-6236-470c-94f7-5f60926f01da-c000.snappy.parquet │ │ │ │ │ └── part-00001-980a117f-027e-4396-81ce-3a5a8ac70815-c000.snappy.parquet │ │ │ │ ├── data-reader-date-types-UTC/ │ │ │ │ │ ├── .part-00000-803e1cfa-c859-4ce7-977b-ff150d6e138c-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-0108113a-2933-41b3-b9a6-e68bb9ed25cc-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-803e1cfa-c859-4ce7-977b-ff150d6e138c-c000.snappy.parquet │ │ │ │ │ └── part-00001-0108113a-2933-41b3-b9a6-e68bb9ed25cc-c000.snappy.parquet │ │ │ │ ├── data-reader-escaped-chars/ │ │ │ │ │ ├── _2=bar+%2521/ │ │ │ │ │ │ ├── .part-00000-af08f887-922f-4c31-82a7-8e142c4280a6.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00000-af08f887-922f-4c31-82a7-8e142c4280a6.c000.snappy.parquet │ │ │ │ │ ├── _2=bar+%2522/ │ │ │ │ │ │ ├── .part-00000-c1bfd944-5e0d-4133-af16-7851061e37aa.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00000-c1bfd944-5e0d-4133-af16-7851061e37aa.c000.snappy.parquet │ │ │ │ │ ├── _2=bar+%2523/ │ │ │ │ │ │ ├── .part-00000-92352854-5503-4ba5-8c29-b11777034eb7.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00000-92352854-5503-4ba5-8c29-b11777034eb7.c000.snappy.parquet │ │ │ │ │ └── _delta_log/ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ └── 00000000000000000002.json │ │ │ │ ├── data-reader-map/ │ │ │ │ │ ├── .part-00000-d9004e55-077b-4728-9ee6-b3401faa46ba-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-3d30d085-4cde-471e-a396-12af34a70812-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-d9004e55-077b-4728-9ee6-b3401faa46ba-c000.snappy.parquet │ │ │ │ │ └── part-00001-3d30d085-4cde-471e-a396-12af34a70812-c000.snappy.parquet │ │ │ │ ├── data-reader-nested-struct/ │ │ │ │ │ ├── .part-00000-f2547b28-9219-4628-8462-cc9c56edfebb-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-0f755735-3b5b-449a-8f93-92a40d9f065d-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-f2547b28-9219-4628-8462-cc9c56edfebb-c000.snappy.parquet │ │ │ │ │ └── part-00001-0f755735-3b5b-449a-8f93-92a40d9f065d-c000.snappy.parquet │ │ │ │ ├── data-reader-nullable-field-invalid-schema-key/ │ │ │ │ │ ├── .part-00000-d1f74401-ecb8-494e-96d6-adb95ec7e1c2-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-d6454547-1a50-4f43-910d-2f84c5aedae1-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-d1f74401-ecb8-494e-96d6-adb95ec7e1c2-c000.snappy.parquet │ │ │ │ │ └── part-00001-d6454547-1a50-4f43-910d-2f84c5aedae1-c000.snappy.parquet │ │ │ │ ├── data-reader-partition-values/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── as_int=0/ │ │ │ │ │ │ └── as_long=0/ │ │ │ │ │ │ └── as_byte=0/ │ │ │ │ │ │ └── as_short=0/ │ │ │ │ │ │ └── as_boolean=true/ │ │ │ │ │ │ └── as_float=0.0/ │ │ │ │ │ │ └── as_double=0.0/ │ │ │ │ │ │ └── as_string=0/ │ │ │ │ │ │ └── as_string_lit_null=null/ │ │ │ │ │ │ └── as_date=2021-09-08/ │ │ │ │ │ │ └── as_timestamp=2021-09-08 11%3A11%3A11/ │ │ │ │ │ │ └── as_big_decimal=0/ │ │ │ │ │ │ ├── .part-00000-b9dc86ae-0134-4363-bd87-19cfb3403e9a.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00000-b9dc86ae-0134-4363-bd87-19cfb3403e9a.c000.snappy.parquet │ │ │ │ │ ├── as_int=1/ │ │ │ │ │ │ └── as_long=1/ │ │ │ │ │ │ └── as_byte=1/ │ │ │ │ │ │ └── as_short=1/ │ │ │ │ │ │ └── as_boolean=false/ │ │ │ │ │ │ └── as_float=1.0/ │ │ │ │ │ │ └── as_double=1.0/ │ │ │ │ │ │ └── as_string=1/ │ │ │ │ │ │ └── as_string_lit_null=null/ │ │ │ │ │ │ └── as_date=2021-09-08/ │ │ │ │ │ │ └── as_timestamp=2021-09-08 11%3A11%3A11/ │ │ │ │ │ │ └── as_big_decimal=1/ │ │ │ │ │ │ ├── .part-00001-cb007d48-a9f5-40e7-adbe-60920680770f.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00001-cb007d48-a9f5-40e7-adbe-60920680770f.c000.snappy.parquet │ │ │ │ │ └── as_int=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ │ └── as_long=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ │ └── as_byte=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ │ └── as_short=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ │ └── as_boolean=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ │ └── as_float=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ │ └── as_double=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ │ └── as_string=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ │ └── as_string_lit_null=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ │ └── as_date=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ │ └── as_timestamp=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ │ └── as_big_decimal=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ │ ├── .part-00001-9ee474eb-385b-43cf-9acb-0fbed63e011c.c000.snappy.parquet.crc │ │ │ │ │ └── part-00001-9ee474eb-385b-43cf-9acb-0fbed63e011c.c000.snappy.parquet │ │ │ │ ├── data-reader-primitives/ │ │ │ │ │ ├── .part-00000-4f2f0b9f-50b3-4e7b-96a1-e2bb0f246b06-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-09e47b80-36c2-4475-a810-fbd8e7994971-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-4f2f0b9f-50b3-4e7b-96a1-e2bb0f246b06-c000.snappy.parquet │ │ │ │ │ └── part-00001-09e47b80-36c2-4475-a810-fbd8e7994971-c000.snappy.parquet │ │ │ │ ├── data-reader-timestamp_ntz/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ └── 00000000000000000001.json │ │ │ │ │ ├── tsNtzPartition=2013-07-05 17%3A01%3A00.123456/ │ │ │ │ │ │ ├── .part-00000-6240e68e-2304-449a-a1e6-0e24866d3508.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-336e3e5f-a202-4bd9-b117-28d871bbb639.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-6240e68e-2304-449a-a1e6-0e24866d3508.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-336e3e5f-a202-4bd9-b117-28d871bbb639.c000.snappy.parquet │ │ │ │ │ ├── tsNtzPartition=2021-11-18 02%3A30%3A00.123456/ │ │ │ │ │ │ ├── .part-00000-65fcd5cb-f2f3-44f4-96ef-f43825143ba9.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00000-65fcd5cb-f2f3-44f4-96ef-f43825143ba9.c000.snappy.parquet │ │ │ │ │ └── tsNtzPartition=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ │ ├── .part-00001-53fd3b3b-7773-459a-921c-bb64bf0bbd03.c000.snappy.parquet.crc │ │ │ │ │ └── part-00001-53fd3b3b-7773-459a-921c-bb64bf0bbd03.c000.snappy.parquet │ │ │ │ ├── data-reader-timestamp_ntz-id-mode/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ └── 00000000000000000001.json │ │ │ │ │ ├── col-31f31113-4fc7-437c-b8e8-b7bca8a2f698=2013-07-05 17%3A01%3A00.123456/ │ │ │ │ │ │ ├── .part-00000-468b79b5-ef3e-40ee-b077-8d7b48ef8385.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-94a2fe48-a4c5-4d3e-823c-d76b59b9f597.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-468b79b5-ef3e-40ee-b077-8d7b48ef8385.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-94a2fe48-a4c5-4d3e-823c-d76b59b9f597.c000.snappy.parquet │ │ │ │ │ ├── col-31f31113-4fc7-437c-b8e8-b7bca8a2f698=2021-11-18 02%3A30%3A00.123456/ │ │ │ │ │ │ ├── .part-00000-80e4d2e9-69f2-420e-8152-8d5bb810b259.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00000-80e4d2e9-69f2-420e-8152-8d5bb810b259.c000.snappy.parquet │ │ │ │ │ └── col-31f31113-4fc7-437c-b8e8-b7bca8a2f698=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ │ ├── .part-00001-047834e2-8a38-47ff-9f1c-01f94a618369.c000.snappy.parquet.crc │ │ │ │ │ └── part-00001-047834e2-8a38-47ff-9f1c-01f94a618369.c000.snappy.parquet │ │ │ │ ├── data-reader-timestamp_ntz-name-mode/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ └── 00000000000000000001.json │ │ │ │ │ ├── col-805808af-d12a-42e5-a7ec-f1a99abb82ee=2013-07-05 17%3A01%3A00.123456/ │ │ │ │ │ │ ├── .part-00000-19009b69-d0d2-4c9c-9994-770c77ce5c1e.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-459a6750-6f78-44ff-9706-03448c1dde8b.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-19009b69-d0d2-4c9c-9994-770c77ce5c1e.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-459a6750-6f78-44ff-9706-03448c1dde8b.c000.snappy.parquet │ │ │ │ │ ├── col-805808af-d12a-42e5-a7ec-f1a99abb82ee=2021-11-18 02%3A30%3A00.123456/ │ │ │ │ │ │ ├── .part-00000-55eb3e92-fedb-4a0e-a327-d44ee8e356b2.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00000-55eb3e92-fedb-4a0e-a327-d44ee8e356b2.c000.snappy.parquet │ │ │ │ │ └── col-805808af-d12a-42e5-a7ec-f1a99abb82ee=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ │ ├── .part-00001-4325cf1b-146e-4e85-b36f-ab9c4a9d8125.c000.snappy.parquet.crc │ │ │ │ │ └── part-00001-4325cf1b-146e-4e85-b36f-ab9c4a9d8125.c000.snappy.parquet │ │ │ │ ├── data-skipping-basic-stats-all-types/ │ │ │ │ │ ├── .part-00000-087fe319-d261-41b4-91b4-0e8412e60b9e-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-93fc8b78-4b92-45c7-ad3f-bb766e6d2e28-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-087fe319-d261-41b4-91b4-0e8412e60b9e-c000.snappy.parquet │ │ │ │ │ └── part-00001-93fc8b78-4b92-45c7-ad3f-bb766e6d2e28-c000.snappy.parquet │ │ │ │ ├── data-skipping-basic-stats-all-types-checkpoint/ │ │ │ │ │ ├── .part-00000-56a3869e-5a30-4765-9a7d-702537d70c3d-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-ed0f17f3-dab5-4131-8ff8-5a5f4399d0ef-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-56a3869e-5a30-4765-9a7d-702537d70c3d-c000.snappy.parquet │ │ │ │ │ └── part-00001-ed0f17f3-dab5-4131-8ff8-5a5f4399d0ef-c000.snappy.parquet │ │ │ │ ├── data-skipping-basic-stats-all-types-columnmapping-id/ │ │ │ │ │ ├── .part-00000-45eafd89-d2ac-43ee-8ac3-c400d2bc828e-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-4596bea2-786f-404e-bc15-5adc99f00e30-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-45eafd89-d2ac-43ee-8ac3-c400d2bc828e-c000.snappy.parquet │ │ │ │ │ └── part-00001-4596bea2-786f-404e-bc15-5adc99f00e30-c000.snappy.parquet │ │ │ │ ├── data-skipping-basic-stats-all-types-columnmapping-name/ │ │ │ │ │ ├── .part-00000-23579e01-a3ed-4d10-b208-c34d51bdcd50-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-97ba0cfd-25fe-4911-a28f-29d37288fdd0-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-23579e01-a3ed-4d10-b208-c34d51bdcd50-c000.snappy.parquet │ │ │ │ │ └── part-00001-97ba0cfd-25fe-4911-a28f-29d37288fdd0-c000.snappy.parquet │ │ │ │ ├── data-skipping-change-stats-collected-across-versions/ │ │ │ │ │ ├── .part-00000-4deb5922-56af-43f6-9f20-75634a766a96-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-749a71d6-ff8e-4397-a7b0-8d33df259f58-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-fffd95ce-0d67-442e-b3d5-8fb90da5e1dd-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-c09e5ddb-2337-4e49-b8be-83fd96008375-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-cb335794-98b0-43c3-a3a1-a4c86e3da38d-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-e5d736b6-2ecd-457a-8bb2-947b61f9c67e-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ │ ├── .00000000000000000002.json.crc │ │ │ │ │ │ ├── .00000000000000000003.json.crc │ │ │ │ │ │ ├── .00000000000000000004.json.crc │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ │ └── 00000000000000000004.json │ │ │ │ │ ├── part-00000-4deb5922-56af-43f6-9f20-75634a766a96-c000.snappy.parquet │ │ │ │ │ ├── part-00000-749a71d6-ff8e-4397-a7b0-8d33df259f58-c000.snappy.parquet │ │ │ │ │ ├── part-00000-fffd95ce-0d67-442e-b3d5-8fb90da5e1dd-c000.snappy.parquet │ │ │ │ │ ├── part-00001-c09e5ddb-2337-4e49-b8be-83fd96008375-c000.snappy.parquet │ │ │ │ │ ├── part-00001-cb335794-98b0-43c3-a3a1-a4c86e3da38d-c000.snappy.parquet │ │ │ │ │ └── part-00001-e5d736b6-2ecd-457a-8bb2-947b61f9c67e-c000.snappy.parquet │ │ │ │ ├── data-skipping-partition-and-data-column/ │ │ │ │ │ ├── .part-00000-37b9e82e-6e87-4a9b-bc63-bd0bf3681e6e-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-3fbd7548-fc00-4946-bc27-6255b52ef227-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-8a1edea4-0555-4250-a795-8d3bc2d9e2da-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-f610738e-af5e-442a-8f5e-e806354ed14a-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-2822cff2-34ab-4b93-9cbb-4e751084a422-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-2c0ee02a-8591-4026-a5ab-952bdb347fc5-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-442a6473-8d9a-41d3-8172-e2248e8be169-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-6dedd756-e903-46d7-9e6c-01b3c4ebeab3-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ │ ├── .00000000000000000002.json.crc │ │ │ │ │ │ ├── .00000000000000000003.json.crc │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ └── 00000000000000000003.json │ │ │ │ │ ├── part-00000-37b9e82e-6e87-4a9b-bc63-bd0bf3681e6e-c000.snappy.parquet │ │ │ │ │ ├── part-00000-3fbd7548-fc00-4946-bc27-6255b52ef227-c000.snappy.parquet │ │ │ │ │ ├── part-00000-8a1edea4-0555-4250-a795-8d3bc2d9e2da-c000.snappy.parquet │ │ │ │ │ ├── part-00000-f610738e-af5e-442a-8f5e-e806354ed14a-c000.snappy.parquet │ │ │ │ │ ├── part-00001-2822cff2-34ab-4b93-9cbb-4e751084a422-c000.snappy.parquet │ │ │ │ │ ├── part-00001-2c0ee02a-8591-4026-a5ab-952bdb347fc5-c000.snappy.parquet │ │ │ │ │ ├── part-00001-442a6473-8d9a-41d3-8172-e2248e8be169-c000.snappy.parquet │ │ │ │ │ └── part-00001-6dedd756-e903-46d7-9e6c-01b3c4ebeab3-c000.snappy.parquet │ │ │ │ ├── decimal-various-scale-precision/ │ │ │ │ │ ├── .part-00000-bb4b3e59-ddb9-4d26-beaf-de9554e14517-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ └── part-00000-bb4b3e59-ddb9-4d26-beaf-de9554e14517-c000.snappy.parquet │ │ │ │ ├── delete-re-add-same-file-different-transactions/ │ │ │ │ │ └── _delta_log/ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ └── 00000000000000000003.json │ │ │ │ ├── deltalog-commit-info/ │ │ │ │ │ └── _delta_log/ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ ├── deltalog-getChanges/ │ │ │ │ │ └── _delta_log/ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ └── 00000000000000000002.json │ │ │ │ ├── deltalog-invalid-protocol-version/ │ │ │ │ │ └── _delta_log/ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ ├── deltalog-state-reconstruction-from-checkpoint-missing-metadata/ │ │ │ │ │ └── _delta_log/ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ ├── 00000000000000000004.json │ │ │ │ │ ├── 00000000000000000005.json │ │ │ │ │ ├── 00000000000000000006.json │ │ │ │ │ ├── 00000000000000000007.json │ │ │ │ │ ├── 00000000000000000008.json │ │ │ │ │ ├── 00000000000000000009.json │ │ │ │ │ ├── 00000000000000000010.checkpoint.parquet │ │ │ │ │ ├── 00000000000000000010.json │ │ │ │ │ └── _last_checkpoint │ │ │ │ ├── deltalog-state-reconstruction-from-checkpoint-missing-protocol/ │ │ │ │ │ └── _delta_log/ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ ├── 00000000000000000004.json │ │ │ │ │ ├── 00000000000000000005.json │ │ │ │ │ ├── 00000000000000000006.json │ │ │ │ │ ├── 00000000000000000007.json │ │ │ │ │ ├── 00000000000000000008.json │ │ │ │ │ ├── 00000000000000000009.json │ │ │ │ │ ├── 00000000000000000010.checkpoint.parquet │ │ │ │ │ ├── 00000000000000000010.json │ │ │ │ │ └── _last_checkpoint │ │ │ │ ├── deltalog-state-reconstruction-without-metadata/ │ │ │ │ │ └── _delta_log/ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ ├── deltalog-state-reconstruction-without-protocol/ │ │ │ │ │ └── _delta_log/ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ ├── dv-partitioned-with-checkpoint/ │ │ │ │ │ ├── .deletion_vector_0229ec3d-5100-44e1-8e53-702d448da8da.bin.crc │ │ │ │ │ ├── .deletion_vector_0237686e-c424-4e4c-8337-e8bd1b02ea48.bin.crc │ │ │ │ │ ├── .deletion_vector_179561f5-946c-40d8-b088-0d890e8c854c.bin.crc │ │ │ │ │ ├── .deletion_vector_2fc6c93f-f217-47db-8582-b9732a18de04.bin.crc │ │ │ │ │ ├── .deletion_vector_3cf682dd-5194-4fe8-98ed-d860be48ef78.bin.crc │ │ │ │ │ ├── .deletion_vector_57eabe30-1981-4c70-85b0-343c24650691.bin.crc │ │ │ │ │ ├── .deletion_vector_6f6c4302-fd0b-49e9-8877-cf9056f4b3cb.bin.crc │ │ │ │ │ ├── .deletion_vector_9ab6d39e-2b86-4282-919f-c0813c228da9.bin.crc │ │ │ │ │ ├── .deletion_vector_aa46415b-60bb-4096-a0c7-de47449cb72e.bin.crc │ │ │ │ │ ├── .deletion_vector_acb1fa71-86e9-445d-833c-5cda9a64f221.bin.crc │ │ │ │ │ ├── .deletion_vector_be3f06b1-59b3-4537-b5de-554c3bb2fad9.bin.crc │ │ │ │ │ ├── .deletion_vector_cc54e26b-ec32-4705-ab80-eccffa88e1e6.bin.crc │ │ │ │ │ ├── .deletion_vector_db0b9397-6055-4aa6-a8f8-de723719d996.bin.crc │ │ │ │ │ ├── .deletion_vector_f34fad76-197a-4fd7-9382-f7773fc8eff9.bin.crc │ │ │ │ │ ├── .deletion_vector_f37f3d8e-af4f-40c0-a5b4-5b3c02c1bdd8.bin.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ │ ├── .00000000000000000002.json.crc │ │ │ │ │ │ ├── .00000000000000000003.json.crc │ │ │ │ │ │ ├── .00000000000000000004.json.crc │ │ │ │ │ │ ├── .00000000000000000005.json.crc │ │ │ │ │ │ ├── .00000000000000000006.json.crc │ │ │ │ │ │ ├── .00000000000000000007.json.crc │ │ │ │ │ │ ├── .00000000000000000008.json.crc │ │ │ │ │ │ ├── .00000000000000000009.json.crc │ │ │ │ │ │ ├── .00000000000000000010.checkpoint.parquet.crc │ │ │ │ │ │ ├── .00000000000000000010.json.crc │ │ │ │ │ │ ├── .00000000000000000011.json.crc │ │ │ │ │ │ ├── .00000000000000000012.json.crc │ │ │ │ │ │ ├── .00000000000000000013.json.crc │ │ │ │ │ │ ├── .00000000000000000014.json.crc │ │ │ │ │ │ ├── .00000000000000000015.json.crc │ │ │ │ │ │ ├── ._last_checkpoint.crc │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ │ ├── 00000000000000000004.json │ │ │ │ │ │ ├── 00000000000000000005.json │ │ │ │ │ │ ├── 00000000000000000006.json │ │ │ │ │ │ ├── 00000000000000000007.json │ │ │ │ │ │ ├── 00000000000000000008.json │ │ │ │ │ │ ├── 00000000000000000009.json │ │ │ │ │ │ ├── 00000000000000000010.checkpoint.parquet │ │ │ │ │ │ ├── 00000000000000000010.json │ │ │ │ │ │ ├── 00000000000000000011.json │ │ │ │ │ │ ├── 00000000000000000012.json │ │ │ │ │ │ ├── 00000000000000000013.json │ │ │ │ │ │ ├── 00000000000000000014.json │ │ │ │ │ │ ├── 00000000000000000015.json │ │ │ │ │ │ └── _last_checkpoint │ │ │ │ │ ├── part=0/ │ │ │ │ │ │ ├── .part-00000-8387c699-30b1-4734-a791-9278d560ec19.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-24cdbe06-d3dc-449f-bd38-575228ca42a7.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-8387c699-30b1-4734-a791-9278d560ec19.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-24cdbe06-d3dc-449f-bd38-575228ca42a7.c000.snappy.parquet │ │ │ │ │ ├── part=1/ │ │ │ │ │ │ ├── .part-00000-a1586fa1-50e8-4f06-858a-b43b2e83010b.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-d7e5d32a-55fa-410a-afee-adcdf46bc859.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-a1586fa1-50e8-4f06-858a-b43b2e83010b.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-d7e5d32a-55fa-410a-afee-adcdf46bc859.c000.snappy.parquet │ │ │ │ │ ├── part=2/ │ │ │ │ │ │ ├── .part-00000-ad58cb56-93db-4374-91ba-e65e7fa68e76.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-ab1247be-1f77-41e6-a392-50a99b2db864.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-ad58cb56-93db-4374-91ba-e65e7fa68e76.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-ab1247be-1f77-41e6-a392-50a99b2db864.c000.snappy.parquet │ │ │ │ │ ├── part=3/ │ │ │ │ │ │ ├── .part-00000-319bea86-657f-4431-9b26-949dba99cf2c.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-afeef1dd-2517-49b9-873e-e9e6e8a74b19.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-319bea86-657f-4431-9b26-949dba99cf2c.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-afeef1dd-2517-49b9-873e-e9e6e8a74b19.c000.snappy.parquet │ │ │ │ │ ├── part=4/ │ │ │ │ │ │ ├── .part-00000-69ec928d-3737-4eb3-a3d8-9555a6b55ff5.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-e63d3db6-9e97-4472-aacc-6af9fa44e73d.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-69ec928d-3737-4eb3-a3d8-9555a6b55ff5.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-e63d3db6-9e97-4472-aacc-6af9fa44e73d.c000.snappy.parquet │ │ │ │ │ ├── part=5/ │ │ │ │ │ │ ├── .part-00000-5c963f16-d5b8-4f8b-8d8a-0e3403228be2.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-f344b457-fbd0-4bc4-9502-2c07025e5bb1.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-5c963f16-d5b8-4f8b-8d8a-0e3403228be2.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-f344b457-fbd0-4bc4-9502-2c07025e5bb1.c000.snappy.parquet │ │ │ │ │ ├── part=6/ │ │ │ │ │ │ ├── .part-00000-be524334-115d-4d01-8614-e1bc8c630926.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-6fc16401-ac51-4b89-bf08-bb86cecb5cc2.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-be524334-115d-4d01-8614-e1bc8c630926.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-6fc16401-ac51-4b89-bf08-bb86cecb5cc2.c000.snappy.parquet │ │ │ │ │ ├── part=7/ │ │ │ │ │ │ ├── .part-00000-33cc19fc-3607-4ea7-ab6d-af4e3ebf62c4.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-986abb06-e672-4134-83d4-261752b236b8.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-33cc19fc-3607-4ea7-ab6d-af4e3ebf62c4.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-986abb06-e672-4134-83d4-261752b236b8.c000.snappy.parquet │ │ │ │ │ ├── part=8/ │ │ │ │ │ │ ├── .part-00000-02c66988-3465-4483-9f85-7155e6aee1f4.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-7c58de64-d72f-4373-8d86-dfdc00fb264e.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-02c66988-3465-4483-9f85-7155e6aee1f4.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-7c58de64-d72f-4373-8d86-dfdc00fb264e.c000.snappy.parquet │ │ │ │ │ └── part=9/ │ │ │ │ │ ├── .part-00000-e4012c8c-cc60-44c0-babb-8c5d264a3a31.c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-c0430af8-a8e0-4b23-8776-b2fc549b3e4e.c000.snappy.parquet.crc │ │ │ │ │ ├── part-00000-e4012c8c-cc60-44c0-babb-8c5d264a3a31.c000.snappy.parquet │ │ │ │ │ └── part-00001-c0430af8-a8e0-4b23-8776-b2fc549b3e4e.c000.snappy.parquet │ │ │ │ ├── dv-with-columnmapping/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ │ ├── 00000000000000000004.json │ │ │ │ │ │ ├── 00000000000000000005.json │ │ │ │ │ │ ├── 00000000000000000006.json │ │ │ │ │ │ ├── 00000000000000000007.json │ │ │ │ │ │ ├── 00000000000000000008.json │ │ │ │ │ │ ├── 00000000000000000009.json │ │ │ │ │ │ ├── 00000000000000000010.checkpoint.parquet │ │ │ │ │ │ ├── 00000000000000000010.json │ │ │ │ │ │ ├── 00000000000000000011.json │ │ │ │ │ │ ├── 00000000000000000012.json │ │ │ │ │ │ ├── 00000000000000000013.json │ │ │ │ │ │ ├── 00000000000000000014.json │ │ │ │ │ │ ├── 00000000000000000015.json │ │ │ │ │ │ └── _last_checkpoint │ │ │ │ │ ├── col-60c949ca-b8bc-4330-b931-b73fb4c60037=0/ │ │ │ │ │ │ ├── part-00000-d1888b8a-150e-4fe3-a397-1514739499b4.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-0e48fbde-daec-44ff-b579-d5c49b6c827f.c000.snappy.parquet │ │ │ │ │ ├── col-60c949ca-b8bc-4330-b931-b73fb4c60037=1/ │ │ │ │ │ │ ├── part-00000-19513938-badc-4bd4-9513-3d043d1491dc.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-fb5e7c74-75ab-4bee-8234-400040ae127a.c000.snappy.parquet │ │ │ │ │ ├── col-60c949ca-b8bc-4330-b931-b73fb4c60037=2/ │ │ │ │ │ │ ├── part-00000-3f054c46-7f8f-45f3-a541-25525787b631.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-4825f848-06bb-4b91-94e8-deb40f05feca.c000.snappy.parquet │ │ │ │ │ ├── col-60c949ca-b8bc-4330-b931-b73fb4c60037=3/ │ │ │ │ │ │ ├── part-00000-1a0ac64e-0ce2-493e-b1b0-6cf15c1988f5.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-d1fc7b93-6ec3-4c75-8363-ffd8f1f43420.c000.snappy.parquet │ │ │ │ │ ├── col-60c949ca-b8bc-4330-b931-b73fb4c60037=4/ │ │ │ │ │ │ ├── part-00000-dbfb557b-b778-454e-bef4-bab9481bcea7.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-ee7c50c6-4119-41ff-9c0a-285f844e7c31.c000.snappy.parquet │ │ │ │ │ ├── col-60c949ca-b8bc-4330-b931-b73fb4c60037=5/ │ │ │ │ │ │ ├── part-00000-6d057276-2da0-45c3-86eb-aed7fd3429b8.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-ee535eb0-972e-470f-b705-61884acbbe39.c000.snappy.parquet │ │ │ │ │ ├── col-60c949ca-b8bc-4330-b931-b73fb4c60037=6/ │ │ │ │ │ │ ├── part-00000-c0ca807e-59eb-4c84-a67d-c65a2e03c3c5.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-8c38a718-ea0d-4ac1-9515-3a6ec23cc86b.c000.snappy.parquet │ │ │ │ │ ├── col-60c949ca-b8bc-4330-b931-b73fb4c60037=7/ │ │ │ │ │ │ ├── part-00000-12e816f9-daa3-4197-98f2-217a983bdafd.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-e2c8fd65-f478-4738-89f2-b4f63bdc166f.c000.snappy.parquet │ │ │ │ │ ├── col-60c949ca-b8bc-4330-b931-b73fb4c60037=8/ │ │ │ │ │ │ ├── part-00000-ee6122b8-1474-4764-8bdf-8f8b95c734af.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-0878dadb-c875-4347-92a3-8739c303d7bd.c000.snappy.parquet │ │ │ │ │ └── col-60c949ca-b8bc-4330-b931-b73fb4c60037=9/ │ │ │ │ │ ├── part-00000-f2e5dc2f-b7c6-4772-85d7-23b273a9e54d.c000.snappy.parquet │ │ │ │ │ └── part-00001-8bbcb266-0863-4b31-adc0-e1c4d1194cec.c000.snappy.parquet │ │ │ │ ├── hive/ │ │ │ │ │ ├── deltatbl-column-names-case-insensitive/ │ │ │ │ │ │ ├── BarFoo=foo0/ │ │ │ │ │ │ │ ├── .part-00000-36c1f69c-21dc-4374-a89e-1c4468eff784.c000.snappy.parquet.crc │ │ │ │ │ │ │ ├── .part-00001-27f5c1f6-2393-4021-9a0f-44d143761f88.c000.snappy.parquet.crc │ │ │ │ │ │ │ ├── part-00000-36c1f69c-21dc-4374-a89e-1c4468eff784.c000.snappy.parquet │ │ │ │ │ │ │ └── part-00001-27f5c1f6-2393-4021-9a0f-44d143761f88.c000.snappy.parquet │ │ │ │ │ │ ├── BarFoo=foo1/ │ │ │ │ │ │ │ ├── .part-00000-5c80a439-70eb-435a-92eb-04549d3f220e.c000.snappy.parquet.crc │ │ │ │ │ │ │ ├── .part-00001-b6134dd2-aa40-4868-a708-bec69fc562a2.c000.snappy.parquet.crc │ │ │ │ │ │ │ ├── part-00000-5c80a439-70eb-435a-92eb-04549d3f220e.c000.snappy.parquet │ │ │ │ │ │ │ └── part-00001-b6134dd2-aa40-4868-a708-bec69fc562a2.c000.snappy.parquet │ │ │ │ │ │ └── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── deltatbl-deleted-path/ │ │ │ │ │ │ ├── .part-00000-377b2930-7ed7-41e6-bab2-d565a7ca5bfb-c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-6537e97d-662a-430d-9ad9-f6d087ae7cb8-c000.snappy.parquet.crc │ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ │ ├── part-00000-377b2930-7ed7-41e6-bab2-d565a7ca5bfb-c000.snappy.parquet │ │ │ │ │ │ └── part-00001-6537e97d-662a-430d-9ad9-f6d087ae7cb8-c000.snappy.parquet │ │ │ │ │ ├── deltatbl-incorrect-format-config/ │ │ │ │ │ │ ├── .part-00000-7b3124df-d8a4-4a4a-9d99-e98cfde281cf-c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-e8582398-602e-4697-a508-fc046c1c57cf-c000.snappy.parquet.crc │ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ │ ├── part-00000-7b3124df-d8a4-4a4a-9d99-e98cfde281cf-c000.snappy.parquet │ │ │ │ │ │ └── part-00001-e8582398-602e-4697-a508-fc046c1c57cf-c000.snappy.parquet │ │ │ │ │ ├── deltatbl-map-types-correctly/ │ │ │ │ │ │ ├── .part-00000-c9259a22-ce39-45df-8d76-768bd813c3ff-c000.snappy.parquet.crc │ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ │ └── part-00000-c9259a22-ce39-45df-8d76-768bd813c3ff-c000.snappy.parquet │ │ │ │ │ ├── deltatbl-non-partitioned/ │ │ │ │ │ │ ├── .part-00000-e24c5388-1621-46bd-94eb-fea5209018d0-c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-f2126b8d-1594-451b-9c89-c4c2481bfd93-c000.snappy.parquet.crc │ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ │ ├── part-00000-e24c5388-1621-46bd-94eb-fea5209018d0-c000.snappy.parquet │ │ │ │ │ │ └── part-00001-f2126b8d-1594-451b-9c89-c4c2481bfd93-c000.snappy.parquet │ │ │ │ │ ├── deltatbl-not-allow-write/ │ │ │ │ │ │ ├── .part-00000-fab61bc4-5175-46ea-ac35-249c0f5750ff-c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-6eb569ba-9300-49e7-9b5a-d064e8c5be2d-c000.snappy.parquet.crc │ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ │ ├── part-00000-fab61bc4-5175-46ea-ac35-249c0f5750ff-c000.snappy.parquet │ │ │ │ │ │ └── part-00001-6eb569ba-9300-49e7-9b5a-d064e8c5be2d-c000.snappy.parquet │ │ │ │ │ ├── deltatbl-partition-prune/ │ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ │ ├── date=20180512/ │ │ │ │ │ │ │ └── city=sh/ │ │ │ │ │ │ │ ├── .part-00001-c87aeb63-6d9c-4511-b8b3-71d02178554f.c000.snappy.parquet.crc │ │ │ │ │ │ │ └── part-00001-c87aeb63-6d9c-4511-b8b3-71d02178554f.c000.snappy.parquet │ │ │ │ │ │ ├── date=20180520/ │ │ │ │ │ │ │ ├── city=bj/ │ │ │ │ │ │ │ │ ├── .part-00001-4c732f0f-a473-400a-8ba3-1499f599b8f1.c000.snappy.parquet.crc │ │ │ │ │ │ │ │ └── part-00001-4c732f0f-a473-400a-8ba3-1499f599b8f1.c000.snappy.parquet │ │ │ │ │ │ │ └── city=hz/ │ │ │ │ │ │ │ ├── .part-00000-de1d5bcd-ad7e-4b88-ba9b-31fb8aeb8093.c000.snappy.parquet.crc │ │ │ │ │ │ │ └── part-00000-de1d5bcd-ad7e-4b88-ba9b-31fb8aeb8093.c000.snappy.parquet │ │ │ │ │ │ ├── date=20180718/ │ │ │ │ │ │ │ └── city=hz/ │ │ │ │ │ │ │ ├── .part-00000-f888e95b-c831-43fe-bba8-3dbf43b4eb86.c000.snappy.parquet.crc │ │ │ │ │ │ │ └── part-00000-f888e95b-c831-43fe-bba8-3dbf43b4eb86.c000.snappy.parquet │ │ │ │ │ │ └── date=20181212/ │ │ │ │ │ │ └── city=sz/ │ │ │ │ │ │ ├── .part-00001-529ff89b-55c6-4405-a6cc-04759d5f692b.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00001-529ff89b-55c6-4405-a6cc-04759d5f692b.c000.snappy.parquet │ │ │ │ │ ├── deltatbl-partitioned/ │ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ │ ├── c2=foo0/ │ │ │ │ │ │ │ ├── .part-00000-2bcc9ff6-0551-4401-bd22-d361a60627e3.c000.snappy.parquet.crc │ │ │ │ │ │ │ ├── .part-00001-ca647ee7-f1ad-4d70-bf02-5d1872324d6f.c000.snappy.parquet.crc │ │ │ │ │ │ │ ├── part-00000-2bcc9ff6-0551-4401-bd22-d361a60627e3.c000.snappy.parquet │ │ │ │ │ │ │ └── part-00001-ca647ee7-f1ad-4d70-bf02-5d1872324d6f.c000.snappy.parquet │ │ │ │ │ │ └── c2=foo1/ │ │ │ │ │ │ ├── .part-00000-786c7455-9587-454f-9a4c-de0b22b62bbd.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-1c702e73-89b5-465a-9c6a-25f7559cd150.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-786c7455-9587-454f-9a4c-de0b22b62bbd.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-1c702e73-89b5-465a-9c6a-25f7559cd150.c000.snappy.parquet │ │ │ │ │ ├── deltatbl-schema-match/ │ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ │ ├── b=foo0/ │ │ │ │ │ │ │ ├── .part-00000-531fe778-e359-44c9-8c35-7ed2416c5ff5.c000.snappy.parquet.crc │ │ │ │ │ │ │ ├── .part-00001-923b258c-b34c-4cb9-8da9-622005e49f2c.c000.snappy.parquet.crc │ │ │ │ │ │ │ ├── part-00000-531fe778-e359-44c9-8c35-7ed2416c5ff5.c000.snappy.parquet │ │ │ │ │ │ │ └── part-00001-923b258c-b34c-4cb9-8da9-622005e49f2c.c000.snappy.parquet │ │ │ │ │ │ └── b=foo1/ │ │ │ │ │ │ ├── .part-00000-7dad1d59-f42c-46c1-992e-35c2fb4d9c09.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-e44bca08-b26b-4f4d-8a22-5bb45a598dcf.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-7dad1d59-f42c-46c1-992e-35c2fb4d9c09.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-e44bca08-b26b-4f4d-8a22-5bb45a598dcf.c000.snappy.parquet │ │ │ │ │ ├── deltatbl-special-chars-in-partition-column/ │ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ │ ├── c2=+ %3D%250/ │ │ │ │ │ │ │ ├── .part-00000-88ad45a3-9b80-4e66-b474-1748ba085060.c000.snappy.parquet.crc │ │ │ │ │ │ │ ├── .part-00001-aff2b410-c566-4e51-a968-acfa96d6f1e9.c000.snappy.parquet.crc │ │ │ │ │ │ │ ├── part-00000-88ad45a3-9b80-4e66-b474-1748ba085060.c000.snappy.parquet │ │ │ │ │ │ │ └── part-00001-aff2b410-c566-4e51-a968-acfa96d6f1e9.c000.snappy.parquet │ │ │ │ │ │ └── c2=+ %3D%251/ │ │ │ │ │ │ ├── .part-00000-180d1a36-4ba9-4321-8145-1e0d73406b02.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-3379bbbf-1ab8-4781-8b7e-29038d983f83.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-180d1a36-4ba9-4321-8145-1e0d73406b02.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-3379bbbf-1ab8-4781-8b7e-29038d983f83.c000.snappy.parquet │ │ │ │ │ └── deltatbl-touch-files-needed-for-partitioned/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── c2=foo0/ │ │ │ │ │ │ ├── .part-00000-f1acd078-4e44-4d47-91b2-6568396e2ec3.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-e7f40ed6-fefa-41f5-b8a6-c6e9b78a1448.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-f1acd078-4e44-4d47-91b2-6568396e2ec3.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-e7f40ed6-fefa-41f5-b8a6-c6e9b78a1448.c000.snappy.parquet │ │ │ │ │ └── c2=foo1/ │ │ │ │ │ ├── .part-00000-1bb7c99b-be0e-4c49-ae73-9baf5a8a08d0.c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-c357f264-a317-4e93-a530-a8b1360ca9f6.c000.snappy.parquet.crc │ │ │ │ │ ├── part-00000-1bb7c99b-be0e-4c49-ae73-9baf5a8a08d0.c000.snappy.parquet │ │ │ │ │ └── part-00001-c357f264-a317-4e93-a530-a8b1360ca9f6.c000.snappy.parquet │ │ │ │ ├── kernel-timestamp-INT96/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part=1969-01-01 00%3A00%3A00/ │ │ │ │ │ │ ├── .part-00001-75ac07ae-d2e8-4030-be59-c490d47c4496.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00001-75ac07ae-d2e8-4030-be59-c490d47c4496.c000.snappy.parquet │ │ │ │ │ ├── part=2020-01-01 08%3A09%3A10.001/ │ │ │ │ │ │ ├── .part-00000-bd889aef-417c-4493-b5f7-a9884ba4b247.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00000-bd889aef-417c-4493-b5f7-a9884ba4b247.c000.snappy.parquet │ │ │ │ │ ├── part=2021-10-01 08%3A09%3A20/ │ │ │ │ │ │ ├── .part-00000-57e97070-8fc8-485a-95c6-af55daf5e09b.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-bd0c6fb8-aafd-48dc-9bba-331c1c6f137b.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-57e97070-8fc8-485a-95c6-af55daf5e09b.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-bd0c6fb8-aafd-48dc-9bba-331c1c6f137b.c000.snappy.parquet │ │ │ │ │ └── part=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ │ ├── .part-00001-7cb5f53e-936c-4d24-bca1-9fa0fc7a66e4.c000.snappy.parquet.crc │ │ │ │ │ └── part-00001-7cb5f53e-936c-4d24-bca1-9fa0fc7a66e4.c000.snappy.parquet │ │ │ │ ├── kernel-timestamp-PST/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part=1969-01-01 00%3A00%3A00/ │ │ │ │ │ │ ├── .part-00001-48d8c27a-3661-4e1e-95cb-02ef244c1cf4.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00001-48d8c27a-3661-4e1e-95cb-02ef244c1cf4.c000.snappy.parquet │ │ │ │ │ ├── part=2020-01-01 08%3A09%3A10.001/ │ │ │ │ │ │ ├── .part-00000-a8be3fd2-1fd5-4dd7-84d2-6899a62d99e8.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00000-a8be3fd2-1fd5-4dd7-84d2-6899a62d99e8.c000.snappy.parquet │ │ │ │ │ ├── part=2021-10-01 08%3A09%3A20/ │ │ │ │ │ │ ├── .part-00000-321ea6ca-841e-4654-9844-2d4041b6d0d6.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-b223f8fd-9d33-465b-b139-36c41abb10e8.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-321ea6ca-841e-4654-9844-2d4041b6d0d6.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-b223f8fd-9d33-465b-b139-36c41abb10e8.c000.snappy.parquet │ │ │ │ │ └── part=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ │ ├── .part-00001-18484b3d-01e6-48bc-9e8b-2a75d36d9f7a.c000.snappy.parquet.crc │ │ │ │ │ └── part-00001-18484b3d-01e6-48bc-9e8b-2a75d36d9f7a.c000.snappy.parquet │ │ │ │ ├── kernel-timestamp-TIMESTAMP_MICROS/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part=1969-01-01 00%3A00%3A00/ │ │ │ │ │ │ ├── .part-00001-2b5694f1-b839-4037-b264-353b31af6e7b.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00001-2b5694f1-b839-4037-b264-353b31af6e7b.c000.snappy.parquet │ │ │ │ │ ├── part=2020-01-01 08%3A09%3A10.001/ │ │ │ │ │ │ ├── .part-00000-3cac2575-d0b4-4647-a7a3-b4a9d910cb32.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00000-3cac2575-d0b4-4647-a7a3-b4a9d910cb32.c000.snappy.parquet │ │ │ │ │ ├── part=2021-10-01 08%3A09%3A20/ │ │ │ │ │ │ ├── .part-00000-038fb25c-ca6b-43b6-b0dc-d987f38d0ab9.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-226faf2a-427a-40ee-bfb5-5d53c8642c8a.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-038fb25c-ca6b-43b6-b0dc-d987f38d0ab9.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-226faf2a-427a-40ee-bfb5-5d53c8642c8a.c000.snappy.parquet │ │ │ │ │ └── part=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ │ ├── .part-00001-107828e6-a4b9-42b1-9f1f-244c0efc1b08.c000.snappy.parquet.crc │ │ │ │ │ └── part-00001-107828e6-a4b9-42b1-9f1f-244c0efc1b08.c000.snappy.parquet │ │ │ │ ├── kernel-timestamp-TIMESTAMP_MILLIS/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part=1969-01-01 00%3A00%3A00/ │ │ │ │ │ │ ├── .part-00001-4c527a95-ca90-4aeb-a61c-8d89b6330772.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00001-4c527a95-ca90-4aeb-a61c-8d89b6330772.c000.snappy.parquet │ │ │ │ │ ├── part=2020-01-01 08%3A09%3A10.001/ │ │ │ │ │ │ ├── .part-00000-4b5188f5-4784-47ce-b4ad-1d3eae80710e.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00000-4b5188f5-4784-47ce-b4ad-1d3eae80710e.c000.snappy.parquet │ │ │ │ │ ├── part=2021-10-01 08%3A09%3A20/ │ │ │ │ │ │ ├── .part-00000-086f164a-4d32-4631-b9f9-8aeab485f19c.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-94d3f0af-754c-4cde-bc6e-08338a03a32e.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-086f164a-4d32-4631-b9f9-8aeab485f19c.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-94d3f0af-754c-4cde-bc6e-08338a03a32e.c000.snappy.parquet │ │ │ │ │ └── part=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ │ ├── .part-00001-f81daebf-3993-4686-bf72-470e1fe078d9.c000.snappy.parquet.crc │ │ │ │ │ └── part-00001-f81daebf-3993-4686-bf72-470e1fe078d9.c000.snappy.parquet │ │ │ │ ├── kernel-timestamp-partition-col-ISO8601/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.crc.crc │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ ├── 00000000000000000000.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── ts=2024-01-01 10%3A00%3A00/ │ │ │ │ │ │ ├── .part-00000-9630b3f5-7ab4-4688-9822-3ef93a9d0559.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00000-9630b3f5-7ab4-4688-9822-3ef93a9d0559.c000.snappy.parquet │ │ │ │ │ └── ts=2024-01-02 12%3A30%3A00/ │ │ │ │ │ ├── .part-00000-17b5fc05-b487-4b8b-82ff-9ef4352767a5.c000.snappy.parquet.crc │ │ │ │ │ └── part-00000-17b5fc05-b487-4b8b-82ff-9ef4352767a5.c000.snappy.parquet │ │ │ │ ├── log-replay-dv-key-cases/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ └── 00000000000000000003.json │ │ │ │ │ └── part-00000-90177277-75c2-48db-92a2-20dcba39fd06-c000.snappy.parquet │ │ │ │ ├── log-replay-latest-metadata-protocol/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ └── 00000000000000000002.json │ │ │ │ │ ├── part-00000-66f9221e-0720-45f9-910a-0e81885c93e7-c000.snappy.parquet │ │ │ │ │ ├── part-00000-fc7f7936-944d-472b-9e1e-2cb7464e668a-c000.snappy.parquet │ │ │ │ │ ├── part-00001-9624cca6-2238-4f36-a6c1-707b86b81b81-c000.snappy.parquet │ │ │ │ │ └── part-00001-a54b97f9-bd3c-4724-917b-2730fd9b6c3a-c000.snappy.parquet │ │ │ │ ├── log-replay-special-characters/ │ │ │ │ │ └── _delta_log/ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ └── 00000000000000000001.json │ │ │ │ ├── log-replay-special-characters-a/ │ │ │ │ │ └── _delta_log/ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ └── 00000000000000000001.json │ │ │ │ ├── log-replay-special-characters-b/ │ │ │ │ │ └── _delta_log/ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ ├── log-store-listFrom/ │ │ │ │ │ ├── 1 │ │ │ │ │ ├── 2 │ │ │ │ │ └── 3 │ │ │ │ ├── log-store-read/ │ │ │ │ │ ├── 0 │ │ │ │ │ └── 1 │ │ │ │ ├── multi-part-checkpoint/ │ │ │ │ │ ├── .part-00000-7f49f4e3-2c9c-4ea7-b6c3-42c9a6fc6070-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-e3cd9d97-2f4e-40c4-825f-8ecf456540b0-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-a5222079-1b7e-4bab-a747-ccc4f88b9915-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00002-3a12664c-2859-4236-b718-6c9e03f6496f-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00003-1bb5a769-f4c6-4672-a94a-68ed6788ca78-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00004-b7080e6d-bc43-43da-becf-7c9bedffee68-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00005-a0ce8d21-d9b6-44b9-803b-a4085a4b43cd-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00006-9093b02c-22e2-4505-bb37-104a4825137f-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00007-8cea4b0f-450b-444f-936d-e2695b1adca6-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00008-470c16a2-bd1d-45e5-9cfc-5741ba5b57e1-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ ├── .00000000000000000001.checkpoint.0000000001.0000000002.parquet.crc │ │ │ │ │ │ ├── .00000000000000000001.checkpoint.0000000002.0000000002.parquet.crc │ │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ │ ├── ._last_checkpoint.crc │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.checkpoint.0000000001.0000000002.parquet │ │ │ │ │ │ ├── 00000000000000000001.checkpoint.0000000002.0000000002.parquet │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ └── _last_checkpoint │ │ │ │ │ ├── part-00000-7f49f4e3-2c9c-4ea7-b6c3-42c9a6fc6070-c000.snappy.parquet │ │ │ │ │ ├── part-00000-e3cd9d97-2f4e-40c4-825f-8ecf456540b0-c000.snappy.parquet │ │ │ │ │ ├── part-00001-a5222079-1b7e-4bab-a747-ccc4f88b9915-c000.snappy.parquet │ │ │ │ │ ├── part-00002-3a12664c-2859-4236-b718-6c9e03f6496f-c000.snappy.parquet │ │ │ │ │ ├── part-00003-1bb5a769-f4c6-4672-a94a-68ed6788ca78-c000.snappy.parquet │ │ │ │ │ ├── part-00004-b7080e6d-bc43-43da-becf-7c9bedffee68-c000.snappy.parquet │ │ │ │ │ ├── part-00005-a0ce8d21-d9b6-44b9-803b-a4085a4b43cd-c000.snappy.parquet │ │ │ │ │ ├── part-00006-9093b02c-22e2-4505-bb37-104a4825137f-c000.snappy.parquet │ │ │ │ │ ├── part-00007-8cea4b0f-450b-444f-936d-e2695b1adca6-c000.snappy.parquet │ │ │ │ │ └── part-00008-470c16a2-bd1d-45e5-9cfc-5741ba5b57e1-c000.snappy.parquet │ │ │ │ ├── no-delta-log-folder/ │ │ │ │ │ ├── part-00000-d064d3e2-ed60-4836-a9dc-e09964b59c22-c000.snappy.parquet │ │ │ │ │ └── part-00001-d064d3e2-ed60-4836-a9dc-e09964b59c22-c000.snappy.parquet │ │ │ │ ├── only-checkpoint-files/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.checkpoint.parquet │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.checkpoint.parquet │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ └── _last_checkpoint │ │ │ │ │ ├── part-00000-04e29928-9102-44df-8974-365c864ebd9e-c000.snappy.parquet │ │ │ │ │ ├── part-00000-b4e80ee6-4cbd-4cc6-b565-d2c625d0731a-c000.snappy.parquet │ │ │ │ │ ├── part-00000-dfc22a82-c022-4a82-86e5-1893449a9ac9-c000.snappy.parquet │ │ │ │ │ ├── part-00001-43c03a40-f6fe-4cc7-80d5-b1273adab930-c000.snappy.parquet │ │ │ │ │ ├── part-00001-b50e4584-a496-4a37-a227-f7b3e9705aee-c000.snappy.parquet │ │ │ │ │ ├── part-00002-896896b5-bff2-4d67-a74a-46dbb9730710-c000.snappy.parquet │ │ │ │ │ ├── part-00003-885e84c4-be75-485c-8459-257a0a552a2d-c000.snappy.parquet │ │ │ │ │ ├── part-00004-dcf3f384-2139-4406-af81-89aff10b612d-c000.snappy.parquet │ │ │ │ │ ├── part-00005-e3a5e9cc-e036-41cb-952d-fd3e374af794-c000.snappy.parquet │ │ │ │ │ ├── part-00006-c048e558-898c-43b3-b144-47efbbab72d1-c000.snappy.parquet │ │ │ │ │ ├── part-00007-50c77d47-31b2-4e0a-a43d-2f6be5ff15ee-c000.snappy.parquet │ │ │ │ │ ├── part-00008-9fcc1da0-688a-4be6-a296-21da16557267-c000.snappy.parquet │ │ │ │ │ └── part-00009-ac03e5b4-bd86-48e0-a9a0-094180656170-c000.snappy.parquet │ │ │ │ ├── parquet-all-types/ │ │ │ │ │ ├── .part-00000-bf6680d4-5e83-4fce-8ebb-d2b60d7e69c9-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ └── part-00000-bf6680d4-5e83-4fce-8ebb-d2b60d7e69c9-c000.snappy.parquet │ │ │ │ ├── parquet-all-types-legacy-format/ │ │ │ │ │ ├── .part-00000-5afb67f1-094a-4a15-922e-c1eb96683964-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ └── part-00000-5afb67f1-094a-4a15-922e-c1eb96683964-c000.snappy.parquet │ │ │ │ ├── parquet-decimal-dictionaries/ │ │ │ │ │ ├── .part-00000-60b8c840-c0d4-428e-9005-89f02233be85-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ └── part-00000-60b8c840-c0d4-428e-9005-89f02233be85-c000.snappy.parquet │ │ │ │ ├── parquet-decimal-dictionaries-v1/ │ │ │ │ │ ├── .part-00000-92f97f0b-304f-4587-9d25-088cb386fa64-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ └── part-00000-92f97f0b-304f-4587-9d25-088cb386fa64-c000.snappy.parquet │ │ │ │ ├── parquet-decimal-dictionaries-v2/ │ │ │ │ │ ├── .part-00000-2509b8ef-98ac-42da-98ee-9d2c58ac6031-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ └── part-00000-2509b8ef-98ac-42da-98ee-9d2c58ac6031-c000.snappy.parquet │ │ │ │ ├── parquet-decimal-type/ │ │ │ │ │ ├── .part-00000-8c8ffc0f-9259-478b-9b1b-ea6d37ce5889-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ └── part-00000-8c8ffc0f-9259-478b-9b1b-ea6d37ce5889-c000.snappy.parquet │ │ │ │ ├── snapshot-data0/ │ │ │ │ │ ├── .part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet │ │ │ │ │ └── part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet │ │ │ │ ├── snapshot-data1/ │ │ │ │ │ ├── .part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-64680d94-9e18-4fa1-9ca9-f0cd8a9cfd11-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-b8249b87-0b7a-4461-8a8a-fa958802b523-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ └── 00000000000000000001.json │ │ │ │ │ ├── part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet │ │ │ │ │ ├── part-00000-64680d94-9e18-4fa1-9ca9-f0cd8a9cfd11-c000.snappy.parquet │ │ │ │ │ ├── part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet │ │ │ │ │ └── part-00001-b8249b87-0b7a-4461-8a8a-fa958802b523-c000.snappy.parquet │ │ │ │ ├── snapshot-data2/ │ │ │ │ │ ├── .part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-64680d94-9e18-4fa1-9ca9-f0cd8a9cfd11-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-842017c2-3e02-44b5-a3d6-5b9ae1745045-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-b8249b87-0b7a-4461-8a8a-fa958802b523-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-e62ca5a1-923c-4ee6-998b-c61d1cfb0b1c-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ └── 00000000000000000002.json │ │ │ │ │ ├── part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet │ │ │ │ │ ├── part-00000-64680d94-9e18-4fa1-9ca9-f0cd8a9cfd11-c000.snappy.parquet │ │ │ │ │ ├── part-00000-842017c2-3e02-44b5-a3d6-5b9ae1745045-c000.snappy.parquet │ │ │ │ │ ├── part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet │ │ │ │ │ ├── part-00001-b8249b87-0b7a-4461-8a8a-fa958802b523-c000.snappy.parquet │ │ │ │ │ └── part-00001-e62ca5a1-923c-4ee6-998b-c61d1cfb0b1c-c000.snappy.parquet │ │ │ │ ├── snapshot-data2-deleted/ │ │ │ │ │ ├── .part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-64680d94-9e18-4fa1-9ca9-f0cd8a9cfd11-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-842017c2-3e02-44b5-a3d6-5b9ae1745045-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-cb078bc1-0aeb-46ed-9cf8-74a843b32c8c-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-d83dafd8-c344-49f0-ab1c-acd944e32493-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-9bf4b8f8-1b95-411b-bf10-28dc03aa9d2f-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-b8249b87-0b7a-4461-8a8a-fa958802b523-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-e62ca5a1-923c-4ee6-998b-c61d1cfb0b1c-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ │ └── 00000000000000000004.json │ │ │ │ │ ├── part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet │ │ │ │ │ ├── part-00000-64680d94-9e18-4fa1-9ca9-f0cd8a9cfd11-c000.snappy.parquet │ │ │ │ │ ├── part-00000-842017c2-3e02-44b5-a3d6-5b9ae1745045-c000.snappy.parquet │ │ │ │ │ ├── part-00000-cb078bc1-0aeb-46ed-9cf8-74a843b32c8c-c000.snappy.parquet │ │ │ │ │ ├── part-00000-d83dafd8-c344-49f0-ab1c-acd944e32493-c000.snappy.parquet │ │ │ │ │ ├── part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet │ │ │ │ │ ├── part-00001-9bf4b8f8-1b95-411b-bf10-28dc03aa9d2f-c000.snappy.parquet │ │ │ │ │ ├── part-00001-b8249b87-0b7a-4461-8a8a-fa958802b523-c000.snappy.parquet │ │ │ │ │ └── part-00001-e62ca5a1-923c-4ee6-998b-c61d1cfb0b1c-c000.snappy.parquet │ │ │ │ ├── snapshot-data3/ │ │ │ │ │ ├── .part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-64680d94-9e18-4fa1-9ca9-f0cd8a9cfd11-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-842017c2-3e02-44b5-a3d6-5b9ae1745045-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-cb078bc1-0aeb-46ed-9cf8-74a843b32c8c-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-9bf4b8f8-1b95-411b-bf10-28dc03aa9d2f-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-b8249b87-0b7a-4461-8a8a-fa958802b523-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-e62ca5a1-923c-4ee6-998b-c61d1cfb0b1c-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ └── 00000000000000000003.json │ │ │ │ │ ├── part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet │ │ │ │ │ ├── part-00000-64680d94-9e18-4fa1-9ca9-f0cd8a9cfd11-c000.snappy.parquet │ │ │ │ │ ├── part-00000-842017c2-3e02-44b5-a3d6-5b9ae1745045-c000.snappy.parquet │ │ │ │ │ ├── part-00000-cb078bc1-0aeb-46ed-9cf8-74a843b32c8c-c000.snappy.parquet │ │ │ │ │ ├── part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet │ │ │ │ │ ├── part-00001-9bf4b8f8-1b95-411b-bf10-28dc03aa9d2f-c000.snappy.parquet │ │ │ │ │ ├── part-00001-b8249b87-0b7a-4461-8a8a-fa958802b523-c000.snappy.parquet │ │ │ │ │ └── part-00001-e62ca5a1-923c-4ee6-998b-c61d1cfb0b1c-c000.snappy.parquet │ │ │ │ ├── snapshot-repartitioned/ │ │ │ │ │ ├── .part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-64680d94-9e18-4fa1-9ca9-f0cd8a9cfd11-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-842017c2-3e02-44b5-a3d6-5b9ae1745045-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-cb078bc1-0aeb-46ed-9cf8-74a843b32c8c-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-d83dafd8-c344-49f0-ab1c-acd944e32493-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-f95c1140-7256-4bfa-b651-e7a7eb6208bb-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-0b5675f1-d9b2-4240-914f-250ae37e8fa4-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-9bf4b8f8-1b95-411b-bf10-28dc03aa9d2f-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-b8249b87-0b7a-4461-8a8a-fa958802b523-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-e62ca5a1-923c-4ee6-998b-c61d1cfb0b1c-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ │ ├── 00000000000000000004.json │ │ │ │ │ │ └── 00000000000000000005.json │ │ │ │ │ ├── part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet │ │ │ │ │ ├── part-00000-64680d94-9e18-4fa1-9ca9-f0cd8a9cfd11-c000.snappy.parquet │ │ │ │ │ ├── part-00000-842017c2-3e02-44b5-a3d6-5b9ae1745045-c000.snappy.parquet │ │ │ │ │ ├── part-00000-cb078bc1-0aeb-46ed-9cf8-74a843b32c8c-c000.snappy.parquet │ │ │ │ │ ├── part-00000-d83dafd8-c344-49f0-ab1c-acd944e32493-c000.snappy.parquet │ │ │ │ │ ├── part-00000-f95c1140-7256-4bfa-b651-e7a7eb6208bb-c000.snappy.parquet │ │ │ │ │ ├── part-00001-0b5675f1-d9b2-4240-914f-250ae37e8fa4-c000.snappy.parquet │ │ │ │ │ ├── part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet │ │ │ │ │ ├── part-00001-9bf4b8f8-1b95-411b-bf10-28dc03aa9d2f-c000.snappy.parquet │ │ │ │ │ ├── part-00001-b8249b87-0b7a-4461-8a8a-fa958802b523-c000.snappy.parquet │ │ │ │ │ └── part-00001-e62ca5a1-923c-4ee6-998b-c61d1cfb0b1c-c000.snappy.parquet │ │ │ │ ├── snapshot-vacuumed/ │ │ │ │ │ ├── .part-00000-f95c1140-7256-4bfa-b651-e7a7eb6208bb-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-0b5675f1-d9b2-4240-914f-250ae37e8fa4-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ │ ├── 00000000000000000004.json │ │ │ │ │ │ └── 00000000000000000005.json │ │ │ │ │ ├── part-00000-f95c1140-7256-4bfa-b651-e7a7eb6208bb-c000.snappy.parquet │ │ │ │ │ └── part-00001-0b5675f1-d9b2-4240-914f-250ae37e8fa4-c000.snappy.parquet │ │ │ │ ├── spark-variant-checkpoint/ │ │ │ │ │ ├── .part-00000-16c852df-ba66-4080-be25-530a05922422-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-1e14ba22-3114-46d1-96fb-48b4912507ce-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-9a9c570c-ee32-4322-ad2f-8c837a77d398-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-664313d3-14b4-4dbf-8110-77001b877182-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ │ ├── .00000000000000000002.checkpoint.parquet.crc │ │ │ │ │ │ ├── .00000000000000000002.json.crc │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.checkpoint.parquet │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ └── _last_checkpoint │ │ │ │ │ ├── info.txt │ │ │ │ │ ├── part-00000-16c852df-ba66-4080-be25-530a05922422-c000.snappy.parquet │ │ │ │ │ ├── part-00000-1e14ba22-3114-46d1-96fb-48b4912507ce-c000.snappy.parquet │ │ │ │ │ ├── part-00000-9a9c570c-ee32-4322-ad2f-8c837a77d398-c000.snappy.parquet │ │ │ │ │ └── part-00001-664313d3-14b4-4dbf-8110-77001b877182-c000.snappy.parquet │ │ │ │ ├── table-with-columnmapping-mode-id/ │ │ │ │ │ ├── .part-00000-37fc7686-b5a9-432d-8cdc-8caa8cf999e5-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-0321adc4-f601-4c9d-bb7c-a0ddf759c7b2-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-37fc7686-b5a9-432d-8cdc-8caa8cf999e5-c000.snappy.parquet │ │ │ │ │ └── part-00001-0321adc4-f601-4c9d-bb7c-a0ddf759c7b2-c000.snappy.parquet │ │ │ │ ├── table-with-columnmapping-mode-name/ │ │ │ │ │ ├── .part-00000-2887cf52-61be-4009-afba-00b218602665-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-b664b3db-62d8-4e02-9dc5-26dbce3abfc1-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-2887cf52-61be-4009-afba-00b218602665-c000.snappy.parquet │ │ │ │ │ └── part-00001-b664b3db-62d8-4e02-9dc5-26dbce3abfc1-c000.snappy.parquet │ │ │ │ ├── table-with-icebegCompatV2Enabled/ │ │ │ │ │ ├── .part-00000-cbb3f19e-57e0-4922-a6c3-f211a65d918f-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-5bf41539-fbc6-4b96-9f42-946d36a7f4c9-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-cbb3f19e-57e0-4922-a6c3-f211a65d918f-c000.snappy.parquet │ │ │ │ │ └── part-00001-5bf41539-fbc6-4b96-9f42-946d36a7f4c9-c000.snappy.parquet │ │ │ │ ├── time-travel-partition-changes-a/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part5=0/ │ │ │ │ │ │ ├── .part-00000-67b6882e-f49f-4df5-9850-b5e8a72f4917.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-4f02a740-31dc-46c6-bc0e-c19d164ac82d.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-67b6882e-f49f-4df5-9850-b5e8a72f4917.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-4f02a740-31dc-46c6-bc0e-c19d164ac82d.c000.snappy.parquet │ │ │ │ │ ├── part5=1/ │ │ │ │ │ │ ├── .part-00000-8a40c3d2-f658-4131-a17f-388265ab04b7.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-3dcad520-b001-4829-a6e5-3d578b0964f4.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-8a40c3d2-f658-4131-a17f-388265ab04b7.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-3dcad520-b001-4829-a6e5-3d578b0964f4.c000.snappy.parquet │ │ │ │ │ ├── part5=2/ │ │ │ │ │ │ ├── .part-00000-ec6e3a2e-ecbf-4d39-9076-37e523cd62f1.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-e20bae81-3f27-4c5c-aeca-5cfa6b38615c.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-ec6e3a2e-ecbf-4d39-9076-37e523cd62f1.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-e20bae81-3f27-4c5c-aeca-5cfa6b38615c.c000.snappy.parquet │ │ │ │ │ ├── part5=3/ │ │ │ │ │ │ ├── .part-00000-eaf1edf4-b9da-4df8-b957-08583e2a1d1b.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-b9c6b926-a274-4d8e-b882-31c4aac05038.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-eaf1edf4-b9da-4df8-b957-08583e2a1d1b.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-b9c6b926-a274-4d8e-b882-31c4aac05038.c000.snappy.parquet │ │ │ │ │ └── part5=4/ │ │ │ │ │ ├── .part-00000-ce66c2ca-8fdf-48d3-a6e7-5980a370461a.c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-5705917d-d837-4d7f-b8c4-f0ada8cf9663.c000.snappy.parquet.crc │ │ │ │ │ ├── part-00000-ce66c2ca-8fdf-48d3-a6e7-5980a370461a.c000.snappy.parquet │ │ │ │ │ └── part-00001-5705917d-d837-4d7f-b8c4-f0ada8cf9663.c000.snappy.parquet │ │ │ │ ├── time-travel-partition-changes-b/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ └── 00000000000000000001.json │ │ │ │ │ ├── part2=0/ │ │ │ │ │ │ ├── .part-00000-7bce012e-f358-4a97-91da-55c4d3266fbe.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-2a830e69-78f3-4d09-9b2c-3bfd9debc2f0.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-7bce012e-f358-4a97-91da-55c4d3266fbe.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-2a830e69-78f3-4d09-9b2c-3bfd9debc2f0.c000.snappy.parquet │ │ │ │ │ ├── part2=1/ │ │ │ │ │ │ ├── .part-00000-82368d1d-588b-487a-be01-16dc85260296.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-0a72544a-fb83-4eaa-8d62-9e6ab59afa8b.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-82368d1d-588b-487a-be01-16dc85260296.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-0a72544a-fb83-4eaa-8d62-9e6ab59afa8b.c000.snappy.parquet │ │ │ │ │ ├── part5=0/ │ │ │ │ │ │ ├── .part-00000-67b6882e-f49f-4df5-9850-b5e8a72f4917.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-4f02a740-31dc-46c6-bc0e-c19d164ac82d.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-67b6882e-f49f-4df5-9850-b5e8a72f4917.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-4f02a740-31dc-46c6-bc0e-c19d164ac82d.c000.snappy.parquet │ │ │ │ │ ├── part5=1/ │ │ │ │ │ │ ├── .part-00000-8a40c3d2-f658-4131-a17f-388265ab04b7.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-3dcad520-b001-4829-a6e5-3d578b0964f4.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-8a40c3d2-f658-4131-a17f-388265ab04b7.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-3dcad520-b001-4829-a6e5-3d578b0964f4.c000.snappy.parquet │ │ │ │ │ ├── part5=2/ │ │ │ │ │ │ ├── .part-00000-ec6e3a2e-ecbf-4d39-9076-37e523cd62f1.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-e20bae81-3f27-4c5c-aeca-5cfa6b38615c.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-ec6e3a2e-ecbf-4d39-9076-37e523cd62f1.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-e20bae81-3f27-4c5c-aeca-5cfa6b38615c.c000.snappy.parquet │ │ │ │ │ ├── part5=3/ │ │ │ │ │ │ ├── .part-00000-eaf1edf4-b9da-4df8-b957-08583e2a1d1b.c000.snappy.parquet.crc │ │ │ │ │ │ ├── .part-00001-b9c6b926-a274-4d8e-b882-31c4aac05038.c000.snappy.parquet.crc │ │ │ │ │ │ ├── part-00000-eaf1edf4-b9da-4df8-b957-08583e2a1d1b.c000.snappy.parquet │ │ │ │ │ │ └── part-00001-b9c6b926-a274-4d8e-b882-31c4aac05038.c000.snappy.parquet │ │ │ │ │ └── part5=4/ │ │ │ │ │ ├── .part-00000-ce66c2ca-8fdf-48d3-a6e7-5980a370461a.c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-5705917d-d837-4d7f-b8c4-f0ada8cf9663.c000.snappy.parquet.crc │ │ │ │ │ ├── part-00000-ce66c2ca-8fdf-48d3-a6e7-5980a370461a.c000.snappy.parquet │ │ │ │ │ └── part-00001-5705917d-d837-4d7f-b8c4-f0ada8cf9663.c000.snappy.parquet │ │ │ │ ├── time-travel-schema-changes-a/ │ │ │ │ │ ├── .part-00000-83680aa8-547c-40bc-8ca9-5c10997e307b-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-3c1f89ce-a996-4d44-a79c-21a6f3d53138-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-83680aa8-547c-40bc-8ca9-5c10997e307b-c000.snappy.parquet │ │ │ │ │ └── part-00001-3c1f89ce-a996-4d44-a79c-21a6f3d53138-c000.snappy.parquet │ │ │ │ ├── time-travel-schema-changes-b/ │ │ │ │ │ ├── .part-00000-83680aa8-547c-40bc-8ca9-5c10997e307b-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-a830a49c-6cc8-4caf-80a5-7ff8a959bd53-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-3c1f89ce-a996-4d44-a79c-21a6f3d53138-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-5fdfd303-d5e8-4e77-9b5d-4e831fa723e1-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ └── 00000000000000000001.json │ │ │ │ │ ├── part-00000-83680aa8-547c-40bc-8ca9-5c10997e307b-c000.snappy.parquet │ │ │ │ │ ├── part-00000-a830a49c-6cc8-4caf-80a5-7ff8a959bd53-c000.snappy.parquet │ │ │ │ │ ├── part-00001-3c1f89ce-a996-4d44-a79c-21a6f3d53138-c000.snappy.parquet │ │ │ │ │ └── part-00001-5fdfd303-d5e8-4e77-9b5d-4e831fa723e1-c000.snappy.parquet │ │ │ │ ├── time-travel-start/ │ │ │ │ │ ├── .part-00000-c6271e23-2077-455c-94f9-52866f930213-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-e6177404-aaf5-4e07-8dc0-543a90f4657f-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-c6271e23-2077-455c-94f9-52866f930213-c000.snappy.parquet │ │ │ │ │ └── part-00001-e6177404-aaf5-4e07-8dc0-543a90f4657f-c000.snappy.parquet │ │ │ │ ├── time-travel-start-start20/ │ │ │ │ │ ├── .part-00000-632e29c6-fedf-4822-9223-233d6d8d9086-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-c6271e23-2077-455c-94f9-52866f930213-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-90fee26a-1483-44e3-b239-805343fec254-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-e6177404-aaf5-4e07-8dc0-543a90f4657f-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ └── 00000000000000000001.json │ │ │ │ │ ├── part-00000-632e29c6-fedf-4822-9223-233d6d8d9086-c000.snappy.parquet │ │ │ │ │ ├── part-00000-c6271e23-2077-455c-94f9-52866f930213-c000.snappy.parquet │ │ │ │ │ ├── part-00001-90fee26a-1483-44e3-b239-805343fec254-c000.snappy.parquet │ │ │ │ │ └── part-00001-e6177404-aaf5-4e07-8dc0-543a90f4657f-c000.snappy.parquet │ │ │ │ ├── time-travel-start-start20-start40/ │ │ │ │ │ ├── .part-00000-632e29c6-fedf-4822-9223-233d6d8d9086-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-aef3cbc1-92ef-43b1-8258-284d13163fbb-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-c6271e23-2077-455c-94f9-52866f930213-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-2b364e64-4212-4a35-a95f-ab64504f7c5c-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-90fee26a-1483-44e3-b239-805343fec254-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-e6177404-aaf5-4e07-8dc0-543a90f4657f-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ └── 00000000000000000002.json │ │ │ │ │ ├── part-00000-632e29c6-fedf-4822-9223-233d6d8d9086-c000.snappy.parquet │ │ │ │ │ ├── part-00000-aef3cbc1-92ef-43b1-8258-284d13163fbb-c000.snappy.parquet │ │ │ │ │ ├── part-00000-c6271e23-2077-455c-94f9-52866f930213-c000.snappy.parquet │ │ │ │ │ ├── part-00001-2b364e64-4212-4a35-a95f-ab64504f7c5c-c000.snappy.parquet │ │ │ │ │ ├── part-00001-90fee26a-1483-44e3-b239-805343fec254-c000.snappy.parquet │ │ │ │ │ └── part-00001-e6177404-aaf5-4e07-8dc0-543a90f4657f-c000.snappy.parquet │ │ │ │ ├── type-widening/ │ │ │ │ │ ├── .part-00000-1045efe0-45bb-4b99-9f83-5ffa04a63ab2-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-cd317895-4ae0-4292-b918-62d4ca832bd7-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ │ ├── .00000000000000000002.json.crc │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ └── 00000000000000000002.json │ │ │ │ │ ├── part-00000-1045efe0-45bb-4b99-9f83-5ffa04a63ab2-c000.snappy.parquet │ │ │ │ │ └── part-00000-cd317895-4ae0-4292-b918-62d4ca832bd7-c000.snappy.parquet │ │ │ │ ├── type-widening-nested/ │ │ │ │ │ ├── .part-00000-138244f1-b939-40db-a4bd-d57cf3d214d2-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-1f777f86-350c-4181-b7ef-73df70847eac-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ │ ├── .00000000000000000002.json.crc │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ └── 00000000000000000002.json │ │ │ │ │ ├── part-00000-138244f1-b939-40db-a4bd-d57cf3d214d2-c000.snappy.parquet │ │ │ │ │ └── part-00000-1f777f86-350c-4181-b7ef-73df70847eac-c000.snappy.parquet │ │ │ │ ├── update-deleted-directory/ │ │ │ │ │ └── _delta_log/ │ │ │ │ │ ├── .00000000000000000000.checkpoint.parquet.crc │ │ │ │ │ ├── 00000000000000000000.checkpoint.parquet │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ └── _last_checkpoint │ │ │ │ ├── v2-checkpoint-json/ │ │ │ │ │ ├── .part-00000-240b5dd6-323b-4f74-b6bc-ab9fdcacc630-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-534ea355-2edd-4046-8d49-d932469170c7-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00002-4438bc9d-9c60-4dd2-9343-574743ea4ca8-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00003-ae431d66-23d5-4dc7-b961-136ce33e63da-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ │ ├── .00000000000000000002.checkpoint.6374b053-df23-479b-b2cf-c9c550132b49.json.crc │ │ │ │ │ │ ├── .00000000000000000002.json.crc │ │ │ │ │ │ ├── ._last_checkpoint.crc │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.checkpoint.6374b053-df23-479b-b2cf-c9c550132b49.json │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ ├── _last_checkpoint │ │ │ │ │ │ └── _sidecars/ │ │ │ │ │ │ ├── .00000000000000000002.checkpoint.0000000001.0000000002.bd1885fd-6ec0-4370-b0f5-43b5162fd4de.parquet.crc │ │ │ │ │ │ ├── .00000000000000000002.checkpoint.0000000002.0000000002.0a8d73ee-aa83-49d0-9583-c99db75b89b2.parquet.crc │ │ │ │ │ │ ├── 00000000000000000002.checkpoint.0000000001.0000000002.bd1885fd-6ec0-4370-b0f5-43b5162fd4de.parquet │ │ │ │ │ │ └── 00000000000000000002.checkpoint.0000000002.0000000002.0a8d73ee-aa83-49d0-9583-c99db75b89b2.parquet │ │ │ │ │ ├── part-00000-240b5dd6-323b-4f74-b6bc-ab9fdcacc630-c000.snappy.parquet │ │ │ │ │ ├── part-00001-534ea355-2edd-4046-8d49-d932469170c7-c000.snappy.parquet │ │ │ │ │ ├── part-00002-4438bc9d-9c60-4dd2-9343-574743ea4ca8-c000.snappy.parquet │ │ │ │ │ └── part-00003-ae431d66-23d5-4dc7-b961-136ce33e63da-c000.snappy.parquet │ │ │ │ ├── v2-checkpoint-parquet/ │ │ │ │ │ ├── .part-00000-485b0fff-1c7b-4f14-92e9-a72300fcdf88-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-f7a80035-0622-431e-832e-a756c65cb2a5-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00002-5754df9c-5a25-43a6-947b-f27840fddb1a-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00003-6ab7bbbb-e14d-4fa3-8767-06b509e0a666-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ │ ├── .00000000000000000002.checkpoint.e8fa2696-9728-4e9c-b285-634743fdd4fb.parquet.crc │ │ │ │ │ │ ├── .00000000000000000002.json.crc │ │ │ │ │ │ ├── ._last_checkpoint.crc │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.checkpoint.e8fa2696-9728-4e9c-b285-634743fdd4fb.parquet │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ ├── _last_checkpoint │ │ │ │ │ │ └── _sidecars/ │ │ │ │ │ │ ├── .00000000000000000002.checkpoint.0000000001.0000000002.055454d8-329c-4e0e-864d-7f867075af33.parquet.crc │ │ │ │ │ │ ├── .00000000000000000002.checkpoint.0000000002.0000000002.33321cc1-9c55-4d1f-8511-fafe6d2e1133.parquet.crc │ │ │ │ │ │ ├── 00000000000000000002.checkpoint.0000000001.0000000002.055454d8-329c-4e0e-864d-7f867075af33.parquet │ │ │ │ │ │ └── 00000000000000000002.checkpoint.0000000002.0000000002.33321cc1-9c55-4d1f-8511-fafe6d2e1133.parquet │ │ │ │ │ ├── part-00000-485b0fff-1c7b-4f14-92e9-a72300fcdf88-c000.snappy.parquet │ │ │ │ │ ├── part-00001-f7a80035-0622-431e-832e-a756c65cb2a5-c000.snappy.parquet │ │ │ │ │ ├── part-00002-5754df9c-5a25-43a6-947b-f27840fddb1a-c000.snappy.parquet │ │ │ │ │ └── part-00003-6ab7bbbb-e14d-4fa3-8767-06b509e0a666-c000.snappy.parquet │ │ │ │ └── versions-not-contiguous/ │ │ │ │ └── _delta_log/ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ └── 00000000000000000002.json │ │ │ └── scala/ │ │ │ └── io/ │ │ │ └── delta/ │ │ │ └── golden/ │ │ │ └── GoldenTableUtils.scala │ │ └── test/ │ │ └── scala/ │ │ └── io/ │ │ └── delta/ │ │ └── golden/ │ │ └── GoldenTables.scala │ └── licenses/ │ ├── LICENSE-apache-spark.txt │ └── LICENSE-parquet4s.txt ├── contribs/ │ └── src/ │ ├── main/ │ │ └── scala/ │ │ └── io/ │ │ └── delta/ │ │ └── storage/ │ │ ├── IBMCOSLogStore.scala │ │ └── OracleCloudLogStore.scala │ └── test/ │ └── scala/ │ └── io/ │ └── delta/ │ └── storage/ │ ├── IBMCOSLogStoreSuite.scala │ └── OracleCloudLogStoreSuite.scala ├── dev/ │ ├── check-delta-connect-codegen-python.py │ ├── checkstyle-suppressions.xml │ ├── connectors-checkstyle.xml │ ├── copyrightHeader │ ├── delta-connect-gen-protos.sh │ ├── kernel-checkstyle.xml │ ├── lint-python │ ├── pyproject.toml │ ├── requirements.txt │ ├── spark_structured_logging_style.py │ └── tox.ini ├── docs/ │ ├── .gitignore │ ├── .nvmrc │ ├── .prettierignore │ ├── .prettierrc.json │ ├── README.md │ ├── apis/ │ │ ├── api-docs.css │ │ ├── api-docs.js │ │ ├── api-javadocs.css │ │ ├── api-javadocs.js │ │ ├── generate_api_docs.py │ │ └── python/ │ │ ├── Makefile │ │ ├── conf.py │ │ └── index.rst │ ├── astro.config.mjs │ ├── environment.yml │ ├── eslint.config.mjs │ ├── generate_docs.py │ ├── package.json │ ├── scripts/ │ │ ├── download-api-docs │ │ └── upgrade-dependencies │ ├── src/ │ │ ├── content/ │ │ │ └── docs/ │ │ │ ├── best-practices.mdx │ │ │ ├── bigquery-integration.mdx │ │ │ ├── concurrency-control.mdx │ │ │ ├── delta-apidoc.mdx │ │ │ ├── delta-athena-integration.mdx │ │ │ ├── delta-batch.mdx │ │ │ ├── delta-catalog-managed-tables.mdx │ │ │ ├── delta-change-data-feed.mdx │ │ │ ├── delta-clustering.mdx │ │ │ ├── delta-column-mapping.mdx │ │ │ ├── delta-constraints.mdx │ │ │ ├── delta-default-columns.mdx │ │ │ ├── delta-deletion-vectors.mdx │ │ │ ├── delta-drop-feature.mdx │ │ │ ├── delta-faq.mdx │ │ │ ├── delta-kernel-java.mdx │ │ │ ├── delta-kernel-rust.mdx │ │ │ ├── delta-kernel.mdx │ │ │ ├── delta-more-connectors.mdx │ │ │ ├── delta-presto-integration.mdx │ │ │ ├── delta-resources.mdx │ │ │ ├── delta-row-tracking.mdx │ │ │ ├── delta-sharing.mdx │ │ │ ├── delta-spark-connect.mdx │ │ │ ├── delta-standalone.mdx │ │ │ ├── delta-starburst-integration.mdx │ │ │ ├── delta-storage.mdx │ │ │ ├── delta-streaming/ │ │ │ │ └── index.mdx │ │ │ ├── delta-trino-integration.mdx │ │ │ ├── delta-type-widening.mdx │ │ │ ├── delta-uniform.mdx │ │ │ ├── delta-update.mdx │ │ │ ├── delta-utility/ │ │ │ │ └── index.mdx │ │ │ ├── flink-integration.mdx │ │ │ ├── hive-integration.mdx │ │ │ ├── index.md │ │ │ ├── integrations.mdx │ │ │ ├── optimizations-oss/ │ │ │ │ └── index.mdx │ │ │ ├── porting.mdx │ │ │ ├── presto-integration.mdx │ │ │ ├── quick-start.mdx │ │ │ ├── redshift-spectrum-integration.mdx │ │ │ ├── releases.mdx │ │ │ ├── snowflake-integration.mdx │ │ │ ├── table-properties.mdx │ │ │ └── versioning.mdx │ │ ├── content.config.ts │ │ ├── env.d.ts │ │ ├── pages/ │ │ │ └── robots.txt.ts │ │ └── styles/ │ │ └── custom.css │ └── tsconfig.json ├── examples/ │ ├── README.md │ ├── python/ │ │ ├── change_data_feed.py │ │ ├── delta_connect.py │ │ ├── image_storage.py │ │ ├── missing_delta_storage_jar.py │ │ ├── quickstart.py │ │ ├── quickstart_sql.py │ │ ├── quickstart_sql_on_paths.py │ │ ├── streaming.py │ │ ├── table_exists.py │ │ ├── using_with_pip.py │ │ └── utilities.py │ └── scala/ │ ├── .scalafmt.conf │ ├── README.md │ ├── build/ │ │ ├── sbt │ │ ├── sbt-config/ │ │ │ └── repositories │ │ └── sbt-launch-lib.bash │ ├── build.sbt │ ├── project/ │ │ └── build.properties │ └── src/ │ └── main/ │ ├── resources/ │ │ └── log4j2.properties │ └── scala/ │ └── example/ │ ├── ChangeDataFeed.scala │ ├── Clustering.scala │ ├── EvolutionWithMap.scala │ ├── IcebergCompatV2.scala │ ├── Quickstart.scala │ ├── QuickstartSQL.scala │ ├── QuickstartSQLOnPaths.scala │ ├── Streaming.scala │ ├── UniForm.scala │ ├── UnityCatalogQuickstart.scala │ ├── Utilities.scala │ └── Variant.scala ├── flink/ │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ ├── .placeholder │ │ │ └── io/ │ │ │ └── delta/ │ │ │ └── flink/ │ │ │ ├── Conf.java │ │ │ ├── kernel/ │ │ │ │ ├── CheckpointActionRow.java │ │ │ │ ├── CheckpointWriter.java │ │ │ │ └── ColumnVectorUtils.java │ │ │ └── table/ │ │ │ ├── AbstractKernelTable.java │ │ │ ├── CredentialManager.java │ │ │ ├── DeltaCatalog.java │ │ │ ├── DeltaTable.java │ │ │ ├── ExceptionUtils.java │ │ │ ├── MetricListener.java │ │ │ ├── SnapshotCacheManager.java │ │ │ ├── TableConf.java │ │ │ ├── TableEventListener.java │ │ │ └── postcommit/ │ │ │ ├── ChecksumListener.java │ │ │ └── MaintenanceListener.java │ │ └── resources/ │ │ └── delta-flink.properties │ └── test/ │ ├── java/ │ │ └── io/ │ │ └── delta/ │ │ └── flink/ │ │ ├── DummyHttp.java │ │ ├── TestHelper.java │ │ ├── kernel/ │ │ │ └── CheckpointWriterTest.java │ │ └── table/ │ │ ├── AbstractKernelTableTest.java │ │ ├── CredentialManagerTest.java │ │ ├── DataColumnVectorView.java │ │ ├── LocalFileSystemCatalog.java │ │ └── LocalFileSystemTable.java │ └── resources/ │ └── log4j2-test.properties ├── hudi/ │ ├── README.md │ ├── integration_tests/ │ │ └── write_uniform_hudi.py │ └── src/ │ ├── main/ │ │ └── scala/ │ │ └── org/ │ │ └── apache/ │ │ └── spark/ │ │ └── sql/ │ │ └── delta/ │ │ └── hudi/ │ │ ├── HudiConversionTransaction.scala │ │ ├── HudiConverter.scala │ │ ├── HudiSchemaUtils.scala │ │ └── HudiTransactionUtils.scala │ └── test/ │ └── scala/ │ └── org/ │ └── apache/ │ └── spark/ │ └── sql/ │ └── delta/ │ └── hudi/ │ └── ConvertToHudiSuite.scala ├── iceberg/ │ ├── integration_tests/ │ │ └── iceberg_converter.py │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── org/ │ │ │ └── apache/ │ │ │ └── spark/ │ │ │ └── sql/ │ │ │ └── delta/ │ │ │ └── serverSidePlanning/ │ │ │ └── FixedGcsAccessTokenProvider.java │ │ └── scala/ │ │ └── org/ │ │ └── apache/ │ │ ├── iceberg/ │ │ │ └── transforms/ │ │ │ └── IcebergPartitionUtil.scala │ │ └── spark/ │ │ └── sql/ │ │ ├── catalyst/ │ │ │ └── analysis/ │ │ │ └── NoSuchProcedureException.scala │ │ └── delta/ │ │ ├── IcebergFileManifest.scala │ │ ├── IcebergPartitionConverter.scala │ │ ├── IcebergSchemaUtils.scala │ │ ├── IcebergSparkWrappers.scala │ │ ├── IcebergStatsUtils.scala │ │ ├── IcebergTable.scala │ │ ├── TypeToSparkTypeWithCustomCast.scala │ │ ├── icebergShaded/ │ │ │ ├── DeltaToIcebergConvert.scala │ │ │ ├── IcebergConversionTransaction.scala │ │ │ ├── IcebergConverter.scala │ │ │ ├── IcebergSchemaUtils.scala │ │ │ ├── IcebergStatsConverter.scala │ │ │ └── IcebergTransactionUtils.scala │ │ └── serverSidePlanning/ │ │ ├── IcebergRESTCatalogPlanningClient.scala │ │ ├── IcebergRESTCatalogPlanningClientFactory.scala │ │ └── SparkToIcebergExpressionConverter.scala │ └── test/ │ ├── java/ │ │ └── shadedForDelta/ │ │ └── org/ │ │ └── apache/ │ │ └── iceberg/ │ │ └── rest/ │ │ ├── IcebergRESTCatalogAdapterWithPlanSupport.java │ │ ├── IcebergRESTServer.java │ │ └── IcebergRESTServletWithPlanSupport.java │ ├── resources/ │ │ └── META-INF/ │ │ └── services/ │ │ └── org.apache.spark.sql.sources.DataSourceRegister │ └── scala/ │ └── org/ │ └── apache/ │ └── spark/ │ └── sql/ │ └── delta/ │ ├── CloneIcebergSuite.scala │ ├── ConvertIcebergToDeltaPartitionSuite.scala │ ├── ConvertIcebergToDeltaSuite.scala │ ├── ConvertToIcebergSuite.scala │ ├── NonSparkIcebergTestUtils.scala │ ├── commands/ │ │ └── convert/ │ │ ├── IcebergPartitionConverterSuite.scala │ │ └── IcebergStatsUtilsSuite.scala │ ├── serverSidePlanning/ │ │ ├── IcebergRESTCatalogPlanningClientSuite.scala │ │ ├── IcebergRESTServerTestUtils.scala │ │ ├── ServerSidePlanningCredentialsSuite.scala │ │ ├── SparkToIcebergExpressionConverterSuite.scala │ │ └── TestSchemas.scala │ └── uniform/ │ ├── IcebergCompatV2EnableUniformByAlterTableSuite.scala │ ├── TypeWideningUniformSuite.scala │ ├── UniFormConverterSuite.scala │ ├── UniFormE2EIcebergSuite.scala │ └── UniversalFormatSuite.scala ├── icebergShaded/ │ └── src/ │ └── main/ │ └── java/ │ └── org/ │ └── apache/ │ └── iceberg/ │ ├── MetadataUpdate.java │ ├── PartitionSpec.java │ ├── TableMetadata.java │ ├── hive/ │ │ ├── HiveCatalog.java │ │ └── HiveTableOperations.java │ ├── rest/ │ │ └── RESTFileScanTaskParser.java │ └── unityCatalog/ │ ├── UnityCatalog.java │ └── UnityCatalogTableOperations.java ├── kernel/ │ ├── EXCEPTION_PRINCIPLES.md │ ├── README.md │ ├── USER_GUIDE.md │ ├── build/ │ │ ├── sbt │ │ ├── sbt-config/ │ │ │ └── repositories │ │ └── sbt-launch-lib.bash │ ├── examples/ │ │ ├── kernel-examples/ │ │ │ ├── pom.xml │ │ │ └── src/ │ │ │ └── main/ │ │ │ └── java/ │ │ │ └── io/ │ │ │ └── delta/ │ │ │ └── kernel/ │ │ │ ├── examples/ │ │ │ │ ├── BaseTableReader.java │ │ │ │ ├── BaseTableWriter.java │ │ │ │ ├── CreateTable.java │ │ │ │ ├── CreateTableAndInsertData.java │ │ │ │ ├── MultiThreadedTableReader.java │ │ │ │ ├── SingleThreadedTableReader.java │ │ │ │ └── utils/ │ │ │ │ ├── RowSerDe.java │ │ │ │ └── Utils.java │ │ │ └── integration/ │ │ │ ├── ReadIntegrationTestSuite.java │ │ │ └── WriteIntegrationTestSuite.java │ │ └── run-kernel-examples.py │ ├── kernel-api/ │ │ └── src/ │ │ ├── main/ │ │ │ └── java/ │ │ │ └── io/ │ │ │ └── delta/ │ │ │ └── kernel/ │ │ │ ├── CommitActions.java │ │ │ ├── CommitRange.java │ │ │ ├── CommitRangeBuilder.java │ │ │ ├── DataWriteContext.java │ │ │ ├── Operation.java │ │ │ ├── PaginatedScan.java │ │ │ ├── PaginatedScanFilesIterator.java │ │ │ ├── Scan.java │ │ │ ├── ScanBuilder.java │ │ │ ├── Snapshot.java │ │ │ ├── SnapshotBuilder.java │ │ │ ├── Table.java │ │ │ ├── TableManager.java │ │ │ ├── Transaction.java │ │ │ ├── TransactionBuilder.java │ │ │ ├── TransactionCommitResult.java │ │ │ ├── annotation/ │ │ │ │ ├── Evolving.java │ │ │ │ └── Experimental.java │ │ │ ├── commit/ │ │ │ │ ├── CatalogCommitter.java │ │ │ │ ├── CatalogCommitterUtils.java │ │ │ │ ├── CommitFailedException.java │ │ │ │ ├── CommitMetadata.java │ │ │ │ ├── CommitResponse.java │ │ │ │ ├── Committer.java │ │ │ │ ├── PublishFailedException.java │ │ │ │ └── PublishMetadata.java │ │ │ ├── data/ │ │ │ │ ├── ArrayValue.java │ │ │ │ ├── ColumnVector.java │ │ │ │ ├── ColumnarBatch.java │ │ │ │ ├── FilteredColumnarBatch.java │ │ │ │ ├── MapValue.java │ │ │ │ ├── Row.java │ │ │ │ └── package-info.java │ │ │ ├── engine/ │ │ │ │ ├── Engine.java │ │ │ │ ├── ExpressionHandler.java │ │ │ │ ├── FileReadRequest.java │ │ │ │ ├── FileReadResult.java │ │ │ │ ├── FileSystemClient.java │ │ │ │ ├── JsonHandler.java │ │ │ │ ├── MetricsReporter.java │ │ │ │ ├── ParquetHandler.java │ │ │ │ └── package-info.java │ │ │ ├── exceptions/ │ │ │ │ ├── CheckpointAlreadyExistsException.java │ │ │ │ ├── CommitRangeNotFoundException.java │ │ │ │ ├── CommitStateUnknownException.java │ │ │ │ ├── ConcurrentTransactionException.java │ │ │ │ ├── ConcurrentWriteException.java │ │ │ │ ├── DomainDoesNotExistException.java │ │ │ │ ├── InvalidConfigurationValueException.java │ │ │ │ ├── InvalidTableException.java │ │ │ │ ├── KernelEngineException.java │ │ │ │ ├── KernelException.java │ │ │ │ ├── MaxCommitRetryLimitReachedException.java │ │ │ │ ├── MetadataChangedException.java │ │ │ │ ├── ProtocolChangedException.java │ │ │ │ ├── TableAlreadyExistsException.java │ │ │ │ ├── TableNotFoundException.java │ │ │ │ ├── UnknownConfigurationException.java │ │ │ │ ├── UnsupportedProtocolVersionException.java │ │ │ │ └── UnsupportedTableFeatureException.java │ │ │ ├── expressions/ │ │ │ │ ├── AlwaysFalse.java │ │ │ │ ├── AlwaysTrue.java │ │ │ │ ├── And.java │ │ │ │ ├── Column.java │ │ │ │ ├── Expression.java │ │ │ │ ├── ExpressionEvaluator.java │ │ │ │ ├── In.java │ │ │ │ ├── Literal.java │ │ │ │ ├── Or.java │ │ │ │ ├── PartitionValueExpression.java │ │ │ │ ├── Predicate.java │ │ │ │ ├── PredicateEvaluator.java │ │ │ │ ├── ScalarExpression.java │ │ │ │ └── package-info.java │ │ │ ├── hook/ │ │ │ │ └── PostCommitHook.java │ │ │ ├── internal/ │ │ │ │ ├── CommitActionsImpl.java │ │ │ │ ├── CreateTableTransactionBuilderImpl.java │ │ │ │ ├── DataWriteContextImpl.java │ │ │ │ ├── DeltaErrors.java │ │ │ │ ├── DeltaErrorsInternal.java │ │ │ │ ├── DeltaHistoryManager.java │ │ │ │ ├── DeltaLogActionUtils.java │ │ │ │ ├── InternalScanFileUtils.java │ │ │ │ ├── PaginatedScanImpl.java │ │ │ │ ├── ReplaceTableTransactionBuilderImpl.java │ │ │ │ ├── ReplaceTableTransactionBuilderV2Impl.java │ │ │ │ ├── ScanBuilderImpl.java │ │ │ │ ├── ScanImpl.java │ │ │ │ ├── SnapshotImpl.java │ │ │ │ ├── TableChangesUtils.java │ │ │ │ ├── TableConfig.java │ │ │ │ ├── TableImpl.java │ │ │ │ ├── TransactionBuilderImpl.java │ │ │ │ ├── TransactionImpl.java │ │ │ │ ├── TransactionMetadataFactory.java │ │ │ │ ├── UpdateTableTransactionBuilderImpl.java │ │ │ │ ├── actions/ │ │ │ │ │ ├── AddCDCFile.java │ │ │ │ │ ├── AddFile.java │ │ │ │ │ ├── CommitInfo.java │ │ │ │ │ ├── DeletionVectorDescriptor.java │ │ │ │ │ ├── DomainMetadata.java │ │ │ │ │ ├── Format.java │ │ │ │ │ ├── GenerateIcebergCompatActionUtils.java │ │ │ │ │ ├── Metadata.java │ │ │ │ │ ├── Protocol.java │ │ │ │ │ ├── RemoveFile.java │ │ │ │ │ ├── RowBackedAction.java │ │ │ │ │ ├── SetTransaction.java │ │ │ │ │ └── SingleAction.java │ │ │ │ ├── annotation/ │ │ │ │ │ └── VisibleForTesting.java │ │ │ │ ├── checkpoints/ │ │ │ │ │ ├── CheckpointInstance.java │ │ │ │ │ ├── CheckpointMetaData.java │ │ │ │ │ ├── CheckpointMetadataAction.java │ │ │ │ │ ├── Checkpointer.java │ │ │ │ │ └── SidecarFile.java │ │ │ │ ├── checksum/ │ │ │ │ │ ├── CRCInfo.java │ │ │ │ │ ├── ChecksumReader.java │ │ │ │ │ ├── ChecksumUtils.java │ │ │ │ │ └── ChecksumWriter.java │ │ │ │ ├── clustering/ │ │ │ │ │ ├── ClusteringMetadataDomain.java │ │ │ │ │ └── ClusteringUtils.java │ │ │ │ ├── columndefaults/ │ │ │ │ │ └── ColumnDefaults.java │ │ │ │ ├── commit/ │ │ │ │ │ └── DefaultFileSystemManagedTableOnlyCommitter.java │ │ │ │ ├── commitrange/ │ │ │ │ │ ├── CommitRangeBuilderImpl.java │ │ │ │ │ ├── CommitRangeFactory.java │ │ │ │ │ └── CommitRangeImpl.java │ │ │ │ ├── compaction/ │ │ │ │ │ └── LogCompactionWriter.java │ │ │ │ ├── data/ │ │ │ │ │ ├── ChildVectorBasedRow.java │ │ │ │ │ ├── ColumnarBatchRow.java │ │ │ │ │ ├── DelegateRow.java │ │ │ │ │ ├── GenericColumnVector.java │ │ │ │ │ ├── GenericRow.java │ │ │ │ │ ├── ScanStateRow.java │ │ │ │ │ ├── SelectionColumnVector.java │ │ │ │ │ ├── StructRow.java │ │ │ │ │ └── TransactionStateRow.java │ │ │ │ ├── deletionvectors/ │ │ │ │ │ ├── Base85Codec.java │ │ │ │ │ ├── DeletionVectorStoredBitmap.java │ │ │ │ │ ├── DeletionVectorUtils.java │ │ │ │ │ └── RoaringBitmapArray.java │ │ │ │ ├── files/ │ │ │ │ │ ├── LogDataUtils.java │ │ │ │ │ ├── ParsedCatalogCommitData.java │ │ │ │ │ ├── ParsedCheckpointData.java │ │ │ │ │ ├── ParsedChecksumData.java │ │ │ │ │ ├── ParsedClassicCheckpointData.java │ │ │ │ │ ├── ParsedDeltaData.java │ │ │ │ │ ├── ParsedLogCompactionData.java │ │ │ │ │ ├── ParsedLogData.java │ │ │ │ │ ├── ParsedMultiPartCheckpointData.java │ │ │ │ │ ├── ParsedPublishedDeltaData.java │ │ │ │ │ └── ParsedV2CheckpointData.java │ │ │ │ ├── fs/ │ │ │ │ │ └── Path.java │ │ │ │ ├── hook/ │ │ │ │ │ ├── CheckpointHook.java │ │ │ │ │ ├── ChecksumFullHook.java │ │ │ │ │ ├── ChecksumSimpleHook.java │ │ │ │ │ └── LogCompactionHook.java │ │ │ │ ├── icebergcompat/ │ │ │ │ │ ├── IcebergCompatMetadataValidatorAndUpdater.java │ │ │ │ │ ├── IcebergCompatV2MetadataValidatorAndUpdater.java │ │ │ │ │ ├── IcebergCompatV3MetadataValidatorAndUpdater.java │ │ │ │ │ ├── IcebergUniversalFormatMetadataValidatorAndUpdater.java │ │ │ │ │ ├── IcebergWriterCompatMetadataValidatorAndUpdater.java │ │ │ │ │ ├── IcebergWriterCompatV1MetadataValidatorAndUpdater.java │ │ │ │ │ └── IcebergWriterCompatV3MetadataValidatorAndUpdater.java │ │ │ │ ├── lang/ │ │ │ │ │ ├── Lazy.java │ │ │ │ │ └── ListUtils.java │ │ │ │ ├── metadatadomain/ │ │ │ │ │ └── JsonMetadataDomain.java │ │ │ │ ├── metrics/ │ │ │ │ │ ├── Counter.java │ │ │ │ │ ├── DeltaOperationReportImpl.java │ │ │ │ │ ├── MetricsReportSerializer.java │ │ │ │ │ ├── ScanMetrics.java │ │ │ │ │ ├── ScanReportImpl.java │ │ │ │ │ ├── SnapshotMetrics.java │ │ │ │ │ ├── SnapshotQueryContext.java │ │ │ │ │ ├── SnapshotReportImpl.java │ │ │ │ │ ├── Timer.java │ │ │ │ │ ├── TransactionMetrics.java │ │ │ │ │ └── TransactionReportImpl.java │ │ │ │ ├── replay/ │ │ │ │ │ ├── ActionWrapper.java │ │ │ │ │ ├── ActionsIterator.java │ │ │ │ │ ├── ActiveAddFilesIterator.java │ │ │ │ │ ├── ConflictChecker.java │ │ │ │ │ ├── CreateCheckpointIterator.java │ │ │ │ │ ├── DeltaLogFile.java │ │ │ │ │ ├── LogReplay.java │ │ │ │ │ ├── LogReplayUtils.java │ │ │ │ │ ├── PageToken.java │ │ │ │ │ ├── PaginatedScanFilesIteratorImpl.java │ │ │ │ │ ├── PaginationContext.java │ │ │ │ │ └── ProtocolMetadataLogReplay.java │ │ │ │ ├── rowtracking/ │ │ │ │ │ ├── MaterializedRowTrackingColumn.java │ │ │ │ │ ├── RowTracking.java │ │ │ │ │ └── RowTrackingMetadataDomain.java │ │ │ │ ├── skipping/ │ │ │ │ │ ├── DataSkippingPredicate.java │ │ │ │ │ ├── DataSkippingUtils.java │ │ │ │ │ └── StatsSchemaHelper.java │ │ │ │ ├── snapshot/ │ │ │ │ │ ├── LogSegment.java │ │ │ │ │ ├── MetadataCleanup.java │ │ │ │ │ └── SnapshotManager.java │ │ │ │ ├── stats/ │ │ │ │ │ └── FileSizeHistogram.java │ │ │ │ ├── table/ │ │ │ │ │ ├── SnapshotBuilderImpl.java │ │ │ │ │ └── SnapshotFactory.java │ │ │ │ ├── tablefeatures/ │ │ │ │ │ ├── FeatureAutoEnabledByMetadata.java │ │ │ │ │ ├── TableFeature.java │ │ │ │ │ └── TableFeatures.java │ │ │ │ ├── types/ │ │ │ │ │ ├── DataTypeJsonSerDe.java │ │ │ │ │ └── TypeWideningChecker.java │ │ │ │ └── util/ │ │ │ │ ├── CaseInsensitiveMap.java │ │ │ │ ├── Clock.java │ │ │ │ ├── ColumnMapping.java │ │ │ │ ├── DateTimeConstants.java │ │ │ │ ├── DirectoryCreationUtils.java │ │ │ │ ├── DomainMetadataUtils.java │ │ │ │ ├── ExpressionUtils.java │ │ │ │ ├── FileNames.java │ │ │ │ ├── InCommitTimestampUtils.java │ │ │ │ ├── InternalUtils.java │ │ │ │ ├── IntervalParserUtils.java │ │ │ │ ├── JsonUtils.java │ │ │ │ ├── ManualClock.java │ │ │ │ ├── PartitionUtils.java │ │ │ │ ├── Preconditions.java │ │ │ │ ├── SchemaChanges.java │ │ │ │ ├── SchemaIterable.java │ │ │ │ ├── SchemaUtils.java │ │ │ │ ├── TimestampUtils.java │ │ │ │ ├── Tuple2.java │ │ │ │ ├── Utils.java │ │ │ │ └── VectorUtils.java │ │ │ ├── metrics/ │ │ │ │ ├── DeltaOperationReport.java │ │ │ │ ├── FileSizeHistogramResult.java │ │ │ │ ├── MetricsReport.java │ │ │ │ ├── ScanMetricsResult.java │ │ │ │ ├── ScanReport.java │ │ │ │ ├── SnapshotMetricsResult.java │ │ │ │ ├── SnapshotReport.java │ │ │ │ ├── TransactionMetricsResult.java │ │ │ │ └── TransactionReport.java │ │ │ ├── package-info.java │ │ │ ├── statistics/ │ │ │ │ ├── DataFileStatistics.java │ │ │ │ └── SnapshotStatistics.java │ │ │ ├── transaction/ │ │ │ │ ├── CreateTableTransactionBuilder.java │ │ │ │ ├── DataLayoutSpec.java │ │ │ │ ├── ReplaceTableTransactionBuilder.java │ │ │ │ └── UpdateTableTransactionBuilder.java │ │ │ ├── types/ │ │ │ │ ├── ArrayType.java │ │ │ │ ├── BasePrimitiveType.java │ │ │ │ ├── BinaryType.java │ │ │ │ ├── BooleanType.java │ │ │ │ ├── ByteType.java │ │ │ │ ├── CollationIdentifier.java │ │ │ │ ├── DataType.java │ │ │ │ ├── DateType.java │ │ │ │ ├── DecimalType.java │ │ │ │ ├── DoubleType.java │ │ │ │ ├── FieldMetadata.java │ │ │ │ ├── FloatType.java │ │ │ │ ├── GeographyType.java │ │ │ │ ├── GeometryType.java │ │ │ │ ├── IntegerType.java │ │ │ │ ├── LongType.java │ │ │ │ ├── MapType.java │ │ │ │ ├── MetadataColumnSpec.java │ │ │ │ ├── ShortType.java │ │ │ │ ├── StringType.java │ │ │ │ ├── StructField.java │ │ │ │ ├── StructType.java │ │ │ │ ├── TimestampNTZType.java │ │ │ │ ├── TimestampType.java │ │ │ │ ├── TypeChange.java │ │ │ │ ├── VariantType.java │ │ │ │ └── package-info.java │ │ │ └── utils/ │ │ │ ├── CloseableIterable.java │ │ │ ├── CloseableIterator.java │ │ │ ├── DataFileStatus.java │ │ │ ├── FileStatus.java │ │ │ ├── PartitionUtils.java │ │ │ └── package-info.java │ │ └── test/ │ │ ├── resources/ │ │ │ └── log4j2.properties │ │ └── scala/ │ │ └── io/ │ │ └── delta/ │ │ └── kernel/ │ │ ├── CloseableIteratorSuite.scala │ │ ├── TransactionSuite.scala │ │ ├── commit/ │ │ │ └── CatalogCommitterUtilsSuite.scala │ │ ├── deletionvectors/ │ │ │ ├── Base85CodecSuite.scala │ │ │ └── RoaringBitmapArraySuite.scala │ │ ├── exceptions/ │ │ │ └── ExceptionSuite.scala │ │ ├── expressions/ │ │ │ ├── ExpressionsSuite.scala │ │ │ └── PredicateSuite.scala │ │ ├── internal/ │ │ │ ├── CommitRangeBuilderSuite.scala │ │ │ ├── DeltaHistoryManagerSuite.scala │ │ │ ├── DeltaLogActionUtilsSuite.scala │ │ │ ├── FilteredColumnarBatchSuite.scala │ │ │ ├── PageTokenSuite.scala │ │ │ ├── PaginationContextSuite.scala │ │ │ ├── SnapshotManagerSuite.scala │ │ │ ├── TableConfigSuite.scala │ │ │ ├── TableImplSuite.scala │ │ │ ├── TransactionBuilderImplSuite.scala │ │ │ ├── TransactionMetadataFactorySuite.scala │ │ │ ├── actions/ │ │ │ │ ├── AddFileSuite.scala │ │ │ │ ├── DeletionVectorDescriptorSuite.scala │ │ │ │ ├── GenerateIcebergCompatActionUtilsSuite.scala │ │ │ │ ├── MetadataSuite.scala │ │ │ │ ├── ProtocolSuite.scala │ │ │ │ └── RemoveFileSuite.scala │ │ │ ├── catalogManaged/ │ │ │ │ ├── CatalogManagedLogSegmentSuite.scala │ │ │ │ └── SnapshotBuilderSuite.scala │ │ │ ├── checkpoints/ │ │ │ │ ├── CheckpointInstanceSuite.scala │ │ │ │ └── CheckpointerSuite.scala │ │ │ ├── checksum/ │ │ │ │ ├── CRCInfoReadCompatSuite.scala │ │ │ │ └── ChecksumWriterSuite.scala │ │ │ ├── clustering/ │ │ │ │ └── ClusteringMetadataDomainSuite.scala │ │ │ ├── columndefaults/ │ │ │ │ └── ColumnDefaultsSuite.scala │ │ │ ├── commit/ │ │ │ │ ├── CommitMetadataSuite.scala │ │ │ │ ├── DefaultCommitterSuite.scala │ │ │ │ └── PublishMetadataSuite.scala │ │ │ ├── files/ │ │ │ │ ├── LogDataUtilsSuite.scala │ │ │ │ └── ParsedLogDataSuite.scala │ │ │ ├── fs/ │ │ │ │ ├── PathSuite.scala │ │ │ │ └── benchmarks/ │ │ │ │ └── PathNormalizationBenchmarks.java │ │ │ ├── icebergcompat/ │ │ │ │ ├── IcebergCompatMetadataValidatorAndUpdaterSuiteBase.scala │ │ │ │ ├── IcebergCompatV2MetadataValidatorAndUpdaterSuite.scala │ │ │ │ ├── IcebergCompatV3MetadataValidatorAndUpdateSuite.scala │ │ │ │ ├── IcebergUniversalFormatMetadataValidatorAndUpdaterSuite.scala │ │ │ │ ├── IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite.scala │ │ │ │ └── IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite.scala │ │ │ ├── metadatadomain/ │ │ │ │ ├── JsonMetadataDomainSuite.scala │ │ │ │ └── TestJsonMetadataDomain.java │ │ │ ├── metrics/ │ │ │ │ ├── CounterSuite.scala │ │ │ │ ├── MetricsReportSerializerSuite.scala │ │ │ │ └── TimerSuite.scala │ │ │ ├── replay/ │ │ │ │ └── ActionsIteratorSuite.scala │ │ │ ├── skipping/ │ │ │ │ ├── DataSkippingUtilsSuite.scala │ │ │ │ └── StatsSchemaHelperSuite.scala │ │ │ ├── snapshot/ │ │ │ │ ├── LogSegmentSuite.scala │ │ │ │ └── MetadataCleanupSuite.scala │ │ │ ├── stats/ │ │ │ │ └── FileSizeHistogramSuite.scala │ │ │ ├── tablefeatures/ │ │ │ │ └── TableFeaturesSuite.scala │ │ │ ├── types/ │ │ │ │ ├── DataTypeJsonSerDeSuite.scala │ │ │ │ └── TypeWideningCheckerSuite.scala │ │ │ └── util/ │ │ │ ├── ColumnMappingSuite.scala │ │ │ ├── ColumnMappingSuiteBase.scala │ │ │ ├── DataFileStatisticsSuite.scala │ │ │ ├── FileNamesSuite.scala │ │ │ ├── IntervalParserUtilsSuite.scala │ │ │ ├── JsonUtilsSuite.scala │ │ │ ├── PartitionUtilsSuite.scala │ │ │ ├── SchemaIterableSuite.scala │ │ │ ├── SchemaUtilsSuite.scala │ │ │ ├── TimestampUtilsSuite.scala │ │ │ └── VectorUtilsSuite.scala │ │ ├── test/ │ │ │ ├── ActionUtils.scala │ │ │ ├── MockEngineUtils.scala │ │ │ ├── MockFileSystemClientUtils.scala │ │ │ ├── MockSnapshotUtils.scala │ │ │ ├── TestFixtures.scala │ │ │ ├── TestUtils.scala │ │ │ └── VectorTestUtils.scala │ │ ├── transaction/ │ │ │ └── DataLayoutSpecSuite.scala │ │ ├── types/ │ │ │ ├── CollationIdentifierSuite.scala │ │ │ ├── DataTypeSuite.scala │ │ │ ├── FieldMetadataSuite.scala │ │ │ ├── MetadataColumnSuite.scala │ │ │ ├── StringTypeSuite.scala │ │ │ ├── StructFieldSuite.scala │ │ │ └── TypesSuite.scala │ │ └── utils/ │ │ └── MetadataColumnTestUtils.scala │ ├── kernel-benchmarks/ │ │ └── src/ │ │ └── test/ │ │ ├── java/ │ │ │ └── io/ │ │ │ └── delta/ │ │ │ └── kernel/ │ │ │ └── benchmarks/ │ │ │ ├── AbstractBenchmarkState.java │ │ │ ├── BenchmarkParallelCheckpointReading.java │ │ │ ├── BenchmarkUtils.java │ │ │ ├── KernelMetricsProfiler.java │ │ │ ├── WorkloadBenchmark.java │ │ │ ├── WorkloadOutputFormat.java │ │ │ ├── models/ │ │ │ │ ├── ReadSpec.java │ │ │ │ ├── SnapshotConstructionSpec.java │ │ │ │ ├── TableInfo.java │ │ │ │ ├── UcCatalogInfo.java │ │ │ │ ├── WorkloadSpec.java │ │ │ │ └── WriteSpec.java │ │ │ └── workloadrunners/ │ │ │ ├── ReadMetadataRunner.java │ │ │ ├── SnapshotConstructionRunner.java │ │ │ ├── WorkloadRunner.java │ │ │ └── WriteRunner.java │ │ └── resources/ │ │ └── workload_specs/ │ │ ├── basic_append/ │ │ │ ├── delta/ │ │ │ │ ├── .part-00000-a9daef62-5a40-43c5-ac63-3ad4a7d749ae-c000.snappy.parquet.crc │ │ │ │ ├── .part-00000-c9f44819-b06d-45dd-b33d-ae9aa1b96909-c000.snappy.parquet.crc │ │ │ │ ├── _delta_log/ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ └── 00000000000000000001.json │ │ │ │ ├── part-00000-a9daef62-5a40-43c5-ac63-3ad4a7d749ae-c000.snappy.parquet │ │ │ │ └── part-00000-c9f44819-b06d-45dd-b33d-ae9aa1b96909-c000.snappy.parquet │ │ │ ├── specs/ │ │ │ │ ├── read_latest/ │ │ │ │ │ └── spec.json │ │ │ │ ├── read_v0/ │ │ │ │ │ └── spec.json │ │ │ │ ├── snapshot_latest/ │ │ │ │ │ └── spec.json │ │ │ │ ├── snapshot_v0/ │ │ │ │ │ └── spec.json │ │ │ │ └── write_appends/ │ │ │ │ ├── commit_2_adds.json │ │ │ │ ├── commit_add.json │ │ │ │ └── spec.json │ │ │ └── table_info.json │ │ └── basic_catalog_managed/ │ │ ├── catalog_managed_info.json │ │ ├── delta/ │ │ │ ├── _delta_log/ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ ├── 00000000000000000001.json │ │ │ │ └── _staged_commits/ │ │ │ │ ├── 00000000000000000002.a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d.json │ │ │ │ └── 00000000000000000003.f7e8d9c0-b1a2-4536-9748-5a6b7c8d9e0f.json │ │ │ ├── part-00000-a9daef62-5a40-43c5-ac63-3ad4a7d749ae-c000.snappy.parquet │ │ │ └── part-00000-c9f44819-b06d-45dd-b33d-ae9aa1b96909-c000.snappy.parquet │ │ ├── specs/ │ │ │ ├── read_with_staged/ │ │ │ │ └── spec.json │ │ │ └── write_with_staged/ │ │ │ ├── commit_2.json │ │ │ └── spec.json │ │ └── table_info.json │ ├── kernel-defaults/ │ │ └── src/ │ │ ├── main/ │ │ │ └── java/ │ │ │ └── io/ │ │ │ └── delta/ │ │ │ └── kernel/ │ │ │ └── defaults/ │ │ │ ├── engine/ │ │ │ │ ├── DefaultEngine.java │ │ │ │ ├── DefaultExpressionHandler.java │ │ │ │ ├── DefaultFileSystemClient.java │ │ │ │ ├── DefaultJsonHandler.java │ │ │ │ ├── DefaultParquetHandler.java │ │ │ │ ├── LoggingMetricsReporter.java │ │ │ │ ├── fileio/ │ │ │ │ │ ├── FileIO.java │ │ │ │ │ ├── InputFile.java │ │ │ │ │ ├── OutputFile.java │ │ │ │ │ ├── PositionOutputStream.java │ │ │ │ │ └── SeekableInputStream.java │ │ │ │ ├── hadoopio/ │ │ │ │ │ ├── HadoopFileIO.java │ │ │ │ │ ├── HadoopInputFile.java │ │ │ │ │ ├── HadoopOutputFile.java │ │ │ │ │ ├── HadoopPositionOutputStream.java │ │ │ │ │ └── HadoopSeekableInputStream.java │ │ │ │ └── package-info.java │ │ │ └── internal/ │ │ │ ├── DefaultEngineErrors.java │ │ │ ├── DefaultKernelUtils.java │ │ │ ├── data/ │ │ │ │ ├── DefaultColumnarBatch.java │ │ │ │ ├── DefaultJsonRow.java │ │ │ │ ├── DefaultRowBasedColumnarBatch.java │ │ │ │ └── vector/ │ │ │ │ ├── AbstractColumnVector.java │ │ │ │ ├── DefaultArrayVector.java │ │ │ │ ├── DefaultBinaryVector.java │ │ │ │ ├── DefaultBooleanVector.java │ │ │ │ ├── DefaultByteVector.java │ │ │ │ ├── DefaultConstantVector.java │ │ │ │ ├── DefaultDecimalVector.java │ │ │ │ ├── DefaultDoubleVector.java │ │ │ │ ├── DefaultFloatVector.java │ │ │ │ ├── DefaultGenericVector.java │ │ │ │ ├── DefaultIntVector.java │ │ │ │ ├── DefaultLongVector.java │ │ │ │ ├── DefaultMapVector.java │ │ │ │ ├── DefaultShortVector.java │ │ │ │ ├── DefaultStructVector.java │ │ │ │ ├── DefaultSubFieldVector.java │ │ │ │ └── DefaultViewVector.java │ │ │ ├── expressions/ │ │ │ │ ├── DefaultExpressionEvaluator.java │ │ │ │ ├── DefaultExpressionUtils.java │ │ │ │ ├── DefaultPredicateEvaluator.java │ │ │ │ ├── ElementAtEvaluator.java │ │ │ │ ├── ExpressionVisitor.java │ │ │ │ ├── ImplicitCastExpression.java │ │ │ │ ├── InExpressionEvaluator.java │ │ │ │ ├── LikeExpressionEvaluator.java │ │ │ │ ├── PartitionValueEvaluator.java │ │ │ │ ├── StartsWithExpressionEvaluator.java │ │ │ │ └── SubstringEvaluator.java │ │ │ ├── json/ │ │ │ │ └── JsonUtils.java │ │ │ ├── logstore/ │ │ │ │ └── LogStoreProvider.java │ │ │ └── parquet/ │ │ │ ├── ArrayColumnReader.java │ │ │ ├── DecimalColumnReader.java │ │ │ ├── MapColumnReader.java │ │ │ ├── ParquetColumnReaders.java │ │ │ ├── ParquetColumnWriters.java │ │ │ ├── ParquetFileReader.java │ │ │ ├── ParquetFileWriter.java │ │ │ ├── ParquetFilterUtils.java │ │ │ ├── ParquetIOUtils.java │ │ │ ├── ParquetSchemaUtils.java │ │ │ ├── ParquetStatsReader.java │ │ │ ├── RepeatedValueConverter.java │ │ │ ├── RowColumnReader.java │ │ │ └── TimestampConverters.java │ │ └── test/ │ │ ├── java/ │ │ │ └── io/ │ │ │ └── delta/ │ │ │ └── kernel/ │ │ │ └── defaults/ │ │ │ ├── integration/ │ │ │ │ └── DataBuilderUtils.java │ │ │ └── utils/ │ │ │ └── DefaultKernelTestUtils.java │ │ ├── resources/ │ │ │ ├── basic-dv-no-checkpoint/ │ │ │ │ ├── .deletion_vector_899cef78-06b3-4c14-b024-03860e62cd40.bin.crc │ │ │ │ ├── .part-00000-a489737f-d477-4d9a-8b4a-bd6a6536df5b-c000.snappy.parquet.crc │ │ │ │ ├── .part-00001-1c9b5e60-ab86-4017-9ec9-a6fe4150cdd5-c000.snappy.parquet.crc │ │ │ │ ├── _delta_log/ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ └── 00000000000000000001.json │ │ │ │ ├── part-00000-a489737f-d477-4d9a-8b4a-bd6a6536df5b-c000.snappy.parquet │ │ │ │ └── part-00001-1c9b5e60-ab86-4017-9ec9-a6fe4150cdd5-c000.snappy.parquet │ │ │ ├── basic-dv-with-checkpoint/ │ │ │ │ ├── .deletion_vector_03686710-1009-4aba-a2a6-59d866288a1c.bin.crc │ │ │ │ ├── .deletion_vector_03e6283c-5c18-431e-b43b-5ab5a15b74e5.bin.crc │ │ │ │ ├── .deletion_vector_0498d0e5-d192-44fc-b2f4-a17f228043e6.bin.crc │ │ │ │ ├── .deletion_vector_07bc7f84-8d75-48f2-b39c-15632a2fcf2c.bin.crc │ │ │ │ ├── .deletion_vector_08fa8257-1841-44a7-9660-673f8c92b0ba.bin.crc │ │ │ │ ├── .deletion_vector_11973cae-328a-4116-acdb-9b9b539c4404.bin.crc │ │ │ │ ├── .deletion_vector_11e9779e-4890-4d45-a68b-9ee0bc00fb86.bin.crc │ │ │ │ ├── .deletion_vector_1883ce16-093d-4e9a-8a02-d6d141eab667.bin.crc │ │ │ │ ├── .deletion_vector_1ad67768-c04d-44df-8107-119c6a4b497c.bin.crc │ │ │ │ ├── .deletion_vector_1d371e78-ca14-424f-9e01-0d0e4c50759c.bin.crc │ │ │ │ ├── .deletion_vector_2a78be13-b634-4fc4-9994-fde7d90f8753.bin.crc │ │ │ │ ├── .deletion_vector_2c2af44e-3021-496d-8596-ae5b3a74cb83.bin.crc │ │ │ │ ├── .deletion_vector_2cdf4f8c-5b3f-4f55-8047-be1d1c9bb6b5.bin.crc │ │ │ │ ├── .deletion_vector_44632fee-319a-426f-991d-ad78241e4a3e.bin.crc │ │ │ │ ├── .deletion_vector_4567487b-ca26-4bee-9f7b-f07af32e83c3.bin.crc │ │ │ │ ├── .deletion_vector_45768b99-d5f1-4b0c-8f06-9acc55a85928.bin.crc │ │ │ │ ├── .deletion_vector_4ab2a9ff-9cbd-4391-aebe-5ac8423192e9.bin.crc │ │ │ │ ├── .deletion_vector_4b2dfb3e-4544-4612-9990-3d956e1d06ee.bin.crc │ │ │ │ ├── .deletion_vector_55afff88-4865-45d7-ba5f-05ef95ffa35c.bin.crc │ │ │ │ ├── .deletion_vector_5fb8f71b-f727-494f-8629-6abadbbd4805.bin.crc │ │ │ │ ├── .deletion_vector_609a831c-6ead-4397-8727-02a6a15de803.bin.crc │ │ │ │ ├── .deletion_vector_71b8b79f-6604-431b-ad10-437d1c1cc62c.bin.crc │ │ │ │ ├── .deletion_vector_71f26dca-c66c-41c6-a06f-76fe3809e048.bin.crc │ │ │ │ ├── .deletion_vector_75df38a2-329f-4f07-b9cb-eef2664bffb2.bin.crc │ │ │ │ ├── .deletion_vector_78f381a5-4ab5-4d93-8eaa-5906c98550ea.bin.crc │ │ │ │ ├── .deletion_vector_828143d0-e473-4e79-81c3-7150fc854627.bin.crc │ │ │ │ ├── .deletion_vector_8e35761b-7c0b-4a9a-8c59-bf5875550854.bin.crc │ │ │ │ ├── .deletion_vector_a3e8af8a-e7a4-4f52-8eb5-e23a19e856fe.bin.crc │ │ │ │ ├── .deletion_vector_a73e626d-43fd-4f8b-b448-87bf23ce685e.bin.crc │ │ │ │ ├── .deletion_vector_abed2f64-7626-4c84-8cb1-d412cd3c6cc9.bin.crc │ │ │ │ ├── .deletion_vector_b3a6359b-f50b-4db5-a4a5-e05e4641b76f.bin.crc │ │ │ │ ├── .deletion_vector_bbabaab9-5b28-4320-acb4-e92b34672b54.bin.crc │ │ │ │ ├── .deletion_vector_bc65fecb-20bf-42f8-b5e3-365ecc988ddd.bin.crc │ │ │ │ ├── .deletion_vector_be1d9221-e478-4af5-9988-b5aa6bb57b79.bin.crc │ │ │ │ ├── .deletion_vector_cc322f0c-38e3-4464-945c-ec4e62369941.bin.crc │ │ │ │ ├── .deletion_vector_ce462c97-7338-4c8a-ba30-542e303ba8b9.bin.crc │ │ │ │ ├── .deletion_vector_d1a467a4-cd86-4999-8a32-bcd85257602d.bin.crc │ │ │ │ ├── .deletion_vector_d8940b51-9c96-44bd-8e45-325be665de99.bin.crc │ │ │ │ ├── .deletion_vector_dd9a754e-63f5-4c5f-a551-ccbda05b02c1.bin.crc │ │ │ │ ├── .deletion_vector_e3d66788-4fc0-40c6-83a1-5a1bb7ad59c2.bin.crc │ │ │ │ ├── .deletion_vector_e6e73c4c-2fd7-4c41-b3b8-67517926951e.bin.crc │ │ │ │ ├── .deletion_vector_ea8f857f-676a-4358-b40f-52220677ba42.bin.crc │ │ │ │ ├── .deletion_vector_edbe9d1e-bcb3-4b99-adc3-9dc716574ece.bin.crc │ │ │ │ ├── .deletion_vector_f16b78af-8245-4846-90c0-07e3664b35c1.bin.crc │ │ │ │ ├── .deletion_vector_f1b41214-fae0-4aab-bdd6-9e8a8a0730a7.bin.crc │ │ │ │ ├── .deletion_vector_fb01af59-2712-4a87-aa00-4a72099e2a8e.bin.crc │ │ │ │ ├── .part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet.crc │ │ │ │ ├── .part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet.crc │ │ │ │ ├── _delta_log/ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ ├── .00000000000000000002.json.crc │ │ │ │ │ ├── .00000000000000000003.json.crc │ │ │ │ │ ├── .00000000000000000004.json.crc │ │ │ │ │ ├── .00000000000000000005.json.crc │ │ │ │ │ ├── .00000000000000000006.json.crc │ │ │ │ │ ├── .00000000000000000007.json.crc │ │ │ │ │ ├── .00000000000000000008.json.crc │ │ │ │ │ ├── .00000000000000000009.json.crc │ │ │ │ │ ├── .00000000000000000010.checkpoint.parquet.crc │ │ │ │ │ ├── .00000000000000000010.json.crc │ │ │ │ │ ├── .00000000000000000011.json.crc │ │ │ │ │ ├── .00000000000000000012.json.crc │ │ │ │ │ ├── .00000000000000000013.json.crc │ │ │ │ │ ├── .00000000000000000014.json.crc │ │ │ │ │ ├── .00000000000000000015.json.crc │ │ │ │ │ ├── .00000000000000000016.json.crc │ │ │ │ │ ├── .00000000000000000017.json.crc │ │ │ │ │ ├── .00000000000000000018.json.crc │ │ │ │ │ ├── .00000000000000000019.json.crc │ │ │ │ │ ├── .00000000000000000020.checkpoint.parquet.crc │ │ │ │ │ ├── .00000000000000000020.json.crc │ │ │ │ │ ├── .00000000000000000021.json.crc │ │ │ │ │ ├── .00000000000000000022.json.crc │ │ │ │ │ ├── .00000000000000000023.json.crc │ │ │ │ │ ├── .00000000000000000024.json.crc │ │ │ │ │ ├── .00000000000000000025.json.crc │ │ │ │ │ ├── .00000000000000000026.json.crc │ │ │ │ │ ├── .00000000000000000027.json.crc │ │ │ │ │ ├── .00000000000000000028.json.crc │ │ │ │ │ ├── .00000000000000000029.json.crc │ │ │ │ │ ├── .00000000000000000030.checkpoint.parquet.crc │ │ │ │ │ ├── .00000000000000000030.json.crc │ │ │ │ │ ├── .00000000000000000031.json.crc │ │ │ │ │ ├── .00000000000000000032.json.crc │ │ │ │ │ ├── .00000000000000000033.json.crc │ │ │ │ │ ├── .00000000000000000034.json.crc │ │ │ │ │ ├── .00000000000000000035.json.crc │ │ │ │ │ ├── .00000000000000000036.json.crc │ │ │ │ │ ├── .00000000000000000037.json.crc │ │ │ │ │ ├── .00000000000000000038.json.crc │ │ │ │ │ ├── .00000000000000000039.json.crc │ │ │ │ │ ├── .00000000000000000040.checkpoint.parquet.crc │ │ │ │ │ ├── .00000000000000000040.json.crc │ │ │ │ │ ├── .00000000000000000041.json.crc │ │ │ │ │ ├── .00000000000000000042.json.crc │ │ │ │ │ ├── .00000000000000000043.json.crc │ │ │ │ │ ├── .00000000000000000044.json.crc │ │ │ │ │ ├── .00000000000000000045.json.crc │ │ │ │ │ ├── .00000000000000000046.json.crc │ │ │ │ │ ├── ._last_checkpoint.crc │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ ├── 00000000000000000004.json │ │ │ │ │ ├── 00000000000000000005.json │ │ │ │ │ ├── 00000000000000000006.json │ │ │ │ │ ├── 00000000000000000007.json │ │ │ │ │ ├── 00000000000000000008.json │ │ │ │ │ ├── 00000000000000000009.json │ │ │ │ │ ├── 00000000000000000010.checkpoint.parquet │ │ │ │ │ ├── 00000000000000000010.json │ │ │ │ │ ├── 00000000000000000011.json │ │ │ │ │ ├── 00000000000000000012.json │ │ │ │ │ ├── 00000000000000000013.json │ │ │ │ │ ├── 00000000000000000014.json │ │ │ │ │ ├── 00000000000000000015.json │ │ │ │ │ ├── 00000000000000000016.json │ │ │ │ │ ├── 00000000000000000017.json │ │ │ │ │ ├── 00000000000000000018.json │ │ │ │ │ ├── 00000000000000000019.json │ │ │ │ │ ├── 00000000000000000020.checkpoint.parquet │ │ │ │ │ ├── 00000000000000000020.json │ │ │ │ │ ├── 00000000000000000021.json │ │ │ │ │ ├── 00000000000000000022.json │ │ │ │ │ ├── 00000000000000000023.json │ │ │ │ │ ├── 00000000000000000024.json │ │ │ │ │ ├── 00000000000000000025.json │ │ │ │ │ ├── 00000000000000000026.json │ │ │ │ │ ├── 00000000000000000027.json │ │ │ │ │ ├── 00000000000000000028.json │ │ │ │ │ ├── 00000000000000000029.json │ │ │ │ │ ├── 00000000000000000030.checkpoint.parquet │ │ │ │ │ ├── 00000000000000000030.json │ │ │ │ │ ├── 00000000000000000031.json │ │ │ │ │ ├── 00000000000000000032.json │ │ │ │ │ ├── 00000000000000000033.json │ │ │ │ │ ├── 00000000000000000034.json │ │ │ │ │ ├── 00000000000000000035.json │ │ │ │ │ ├── 00000000000000000036.json │ │ │ │ │ ├── 00000000000000000037.json │ │ │ │ │ ├── 00000000000000000038.json │ │ │ │ │ ├── 00000000000000000039.json │ │ │ │ │ ├── 00000000000000000040.checkpoint.parquet │ │ │ │ │ ├── 00000000000000000040.json │ │ │ │ │ ├── 00000000000000000041.json │ │ │ │ │ ├── 00000000000000000042.json │ │ │ │ │ ├── 00000000000000000043.json │ │ │ │ │ ├── 00000000000000000044.json │ │ │ │ │ ├── 00000000000000000045.json │ │ │ │ │ ├── 00000000000000000046.json │ │ │ │ │ └── _last_checkpoint │ │ │ │ ├── part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet │ │ │ │ └── part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet │ │ │ ├── basic-with-checkpoint/ │ │ │ │ ├── .part-00000-0d9c05f4-8afc-4325-b1e0-ea32e4eff918-c000.snappy.parquet.crc │ │ │ │ ├── .part-00000-100e4547-5ff3-4735-9550-7757ca23c61d-c000.snappy.parquet.crc │ │ │ │ ├── .part-00000-14b8a37a-107b-455f-ab94-8f55e44c004b-c000.snappy.parquet.crc │ │ │ │ ├── .part-00000-1bbb3853-04b4-4539-a112-be7140314153-c000.snappy.parquet.crc │ │ │ │ ├── .part-00000-2a210d80-a4e6-4a1c-8716-ee0b542aee08-c000.snappy.parquet.crc │ │ │ │ ├── .part-00000-326010e2-01f4-4dfb-90a7-98cbc04a60d1-c000.snappy.parquet.crc │ │ │ │ ├── .part-00000-3317387d-183d-4db7-ac3e-596901d90de0-c000.snappy.parquet.crc │ │ │ │ ├── .part-00000-5c99dc53-38c0-420f-a91b-6df7a4c27a2b-c000.snappy.parquet.crc │ │ │ │ ├── .part-00000-5e9186c7-c7b0-4c4d-9f22-1c0cd403142c-c000.snappy.parquet.crc │ │ │ │ ├── .part-00000-6e367682-7cd1-48e6-bc2f-bc94aa94d1a3-c000.snappy.parquet.crc │ │ │ │ ├── .part-00000-8d5f08ff-261b-4315-99cb-d289a3191368-c000.snappy.parquet.crc │ │ │ │ ├── .part-00000-a65ab59f-72fd-44c9-a73e-e2d09459f836-c000.snappy.parquet.crc │ │ │ │ ├── .part-00000-ce6aed75-3e85-4d7c-90de-9d465e9acc04-c000.snappy.parquet.crc │ │ │ │ ├── .part-00000-e51a2d2a-d1a3-437e-a428-f5afe93d4619-c000.snappy.parquet.crc │ │ │ │ ├── .part-00000-eb1dae3f-8c89-46c3-818f-491cc673936a-c000.snappy.parquet.crc │ │ │ │ ├── .part-00001-30432f6b-710f-440c-8145-adbaed187c63-c000.snappy.parquet.crc │ │ │ │ ├── .part-00001-39e6196f-2259-4ba4-b1d6-005712cd7784-c000.snappy.parquet.crc │ │ │ │ ├── .part-00001-3cd0b397-0ac3-48ac-88ab-3cc21542e303-c000.snappy.parquet.crc │ │ │ │ ├── .part-00001-43e387db-3e56-44f3-8965-9187a80fce9a-c000.snappy.parquet.crc │ │ │ │ ├── .part-00001-4707caa4-d293-4b4a-aeef-fa4d5815e732-c000.snappy.parquet.crc │ │ │ │ ├── .part-00001-4e30e0a7-63d2-4d2f-a028-b92058c3c8cf-c000.snappy.parquet.crc │ │ │ │ ├── .part-00001-51925029-c591-4366-b3e9-aeea97594037-c000.snappy.parquet.crc │ │ │ │ ├── .part-00001-63b224e2-ba72-4b95-af02-5af2367d4130-c000.snappy.parquet.crc │ │ │ │ ├── .part-00001-8be0e82d-ce51-43a6-92eb-df71a9088173-c000.snappy.parquet.crc │ │ │ │ ├── .part-00001-a65a81c6-292a-49f2-8eea-82c0299cdfb3-c000.snappy.parquet.crc │ │ │ │ ├── .part-00001-a9a33a7f-26fa-447d-8b34-863b5f695f06-c000.snappy.parquet.crc │ │ │ │ ├── .part-00001-ba1ceb1e-6a37-4e2e-8e97-a17b9b1bb33d-c000.snappy.parquet.crc │ │ │ │ ├── .part-00001-d64b05c7-d80d-4eff-8c58-d209003ee4c0-c000.snappy.parquet.crc │ │ │ │ ├── .part-00001-ed427b16-f597-432a-a49e-135b126d38a8-c000.snappy.parquet.crc │ │ │ │ ├── .part-00001-fdaa71cc-84b2-43b1-b049-7cd36dbaa0de-c000.snappy.parquet.crc │ │ │ │ ├── _delta_log/ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ ├── .00000000000000000002.json.crc │ │ │ │ │ ├── .00000000000000000003.json.crc │ │ │ │ │ ├── .00000000000000000004.json.crc │ │ │ │ │ ├── .00000000000000000005.json.crc │ │ │ │ │ ├── .00000000000000000006.json.crc │ │ │ │ │ ├── .00000000000000000007.json.crc │ │ │ │ │ ├── .00000000000000000008.json.crc │ │ │ │ │ ├── .00000000000000000009.json.crc │ │ │ │ │ ├── .00000000000000000010.checkpoint.parquet.crc │ │ │ │ │ ├── .00000000000000000010.json.crc │ │ │ │ │ ├── .00000000000000000011.json.crc │ │ │ │ │ ├── .00000000000000000012.json.crc │ │ │ │ │ ├── .00000000000000000013.json.crc │ │ │ │ │ ├── .00000000000000000014.json.crc │ │ │ │ │ ├── ._last_checkpoint.crc │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ ├── 00000000000000000004.json │ │ │ │ │ ├── 00000000000000000005.json │ │ │ │ │ ├── 00000000000000000006.json │ │ │ │ │ ├── 00000000000000000007.json │ │ │ │ │ ├── 00000000000000000008.json │ │ │ │ │ ├── 00000000000000000009.json │ │ │ │ │ ├── 00000000000000000010.checkpoint.parquet │ │ │ │ │ ├── 00000000000000000010.json │ │ │ │ │ ├── 00000000000000000011.json │ │ │ │ │ ├── 00000000000000000012.json │ │ │ │ │ ├── 00000000000000000013.json │ │ │ │ │ ├── 00000000000000000014.json │ │ │ │ │ └── _last_checkpoint │ │ │ │ ├── part-00000-0d9c05f4-8afc-4325-b1e0-ea32e4eff918-c000.snappy.parquet │ │ │ │ ├── part-00000-100e4547-5ff3-4735-9550-7757ca23c61d-c000.snappy.parquet │ │ │ │ ├── part-00000-14b8a37a-107b-455f-ab94-8f55e44c004b-c000.snappy.parquet │ │ │ │ ├── part-00000-1bbb3853-04b4-4539-a112-be7140314153-c000.snappy.parquet │ │ │ │ ├── part-00000-2a210d80-a4e6-4a1c-8716-ee0b542aee08-c000.snappy.parquet │ │ │ │ ├── part-00000-326010e2-01f4-4dfb-90a7-98cbc04a60d1-c000.snappy.parquet │ │ │ │ ├── part-00000-3317387d-183d-4db7-ac3e-596901d90de0-c000.snappy.parquet │ │ │ │ ├── part-00000-5c99dc53-38c0-420f-a91b-6df7a4c27a2b-c000.snappy.parquet │ │ │ │ ├── part-00000-5e9186c7-c7b0-4c4d-9f22-1c0cd403142c-c000.snappy.parquet │ │ │ │ ├── part-00000-6e367682-7cd1-48e6-bc2f-bc94aa94d1a3-c000.snappy.parquet │ │ │ │ ├── part-00000-8d5f08ff-261b-4315-99cb-d289a3191368-c000.snappy.parquet │ │ │ │ ├── part-00000-a65ab59f-72fd-44c9-a73e-e2d09459f836-c000.snappy.parquet │ │ │ │ ├── part-00000-ce6aed75-3e85-4d7c-90de-9d465e9acc04-c000.snappy.parquet │ │ │ │ ├── part-00000-e51a2d2a-d1a3-437e-a428-f5afe93d4619-c000.snappy.parquet │ │ │ │ ├── part-00000-eb1dae3f-8c89-46c3-818f-491cc673936a-c000.snappy.parquet │ │ │ │ ├── part-00001-30432f6b-710f-440c-8145-adbaed187c63-c000.snappy.parquet │ │ │ │ ├── part-00001-39e6196f-2259-4ba4-b1d6-005712cd7784-c000.snappy.parquet │ │ │ │ ├── part-00001-3cd0b397-0ac3-48ac-88ab-3cc21542e303-c000.snappy.parquet │ │ │ │ ├── part-00001-43e387db-3e56-44f3-8965-9187a80fce9a-c000.snappy.parquet │ │ │ │ ├── part-00001-4707caa4-d293-4b4a-aeef-fa4d5815e732-c000.snappy.parquet │ │ │ │ ├── part-00001-4e30e0a7-63d2-4d2f-a028-b92058c3c8cf-c000.snappy.parquet │ │ │ │ ├── part-00001-51925029-c591-4366-b3e9-aeea97594037-c000.snappy.parquet │ │ │ │ ├── part-00001-63b224e2-ba72-4b95-af02-5af2367d4130-c000.snappy.parquet │ │ │ │ ├── part-00001-8be0e82d-ce51-43a6-92eb-df71a9088173-c000.snappy.parquet │ │ │ │ ├── part-00001-a65a81c6-292a-49f2-8eea-82c0299cdfb3-c000.snappy.parquet │ │ │ │ ├── part-00001-a9a33a7f-26fa-447d-8b34-863b5f695f06-c000.snappy.parquet │ │ │ │ ├── part-00001-ba1ceb1e-6a37-4e2e-8e97-a17b9b1bb33d-c000.snappy.parquet │ │ │ │ ├── part-00001-d64b05c7-d80d-4eff-8c58-d209003ee4c0-c000.snappy.parquet │ │ │ │ ├── part-00001-ed427b16-f597-432a-a49e-135b126d38a8-c000.snappy.parquet │ │ │ │ └── part-00001-fdaa71cc-84b2-43b1-b049-7cd36dbaa0de-c000.snappy.parquet │ │ │ ├── catalog-owned-preview/ │ │ │ │ ├── _delta_log/ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ └── _staged_commits/ │ │ │ │ │ ├── 00000000000000000001.4cb9708e-b478-44de-b203-53f9ba9b2876.json │ │ │ │ │ └── 00000000000000000002.5b9bba4a-0085-430d-a65e-b0d38c1afbe9.json │ │ │ │ ├── info.txt │ │ │ │ ├── part1=0/ │ │ │ │ │ └── part-00000-13fefaba-8ec2-4762-b17e-aeda657451c5.c000.snappy.parquet │ │ │ │ └── part1=1/ │ │ │ │ └── part-00000-8afb1c56-2018-4af2-aa4f-4336c1b39efd.c000.snappy.parquet │ │ │ ├── column-mapping-id/ │ │ │ │ ├── .part-00000-7f7d554f-a8f2-459f-aaca-9a3b7e8af2dc-c000.snappy.parquet.crc │ │ │ │ ├── .part-00001-85082a62-baeb-46c5-8970-c9c6c23dc33c-c000.snappy.parquet.crc │ │ │ │ ├── _delta_log/ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ ├── part-00000-7f7d554f-a8f2-459f-aaca-9a3b7e8af2dc-c000.snappy.parquet │ │ │ │ └── part-00001-85082a62-baeb-46c5-8970-c9c6c23dc33c-c000.snappy.parquet │ │ │ ├── data-reader-partition-values-column-mapping-name/ │ │ │ │ ├── _delta_log/ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ ├── col-25948c99-9f51-4e05-9f9e-b4f7042f75ed=0/ │ │ │ │ │ └── col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa=0/ │ │ │ │ │ └── col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08=0/ │ │ │ │ │ └── col-29f826c0-7fff-4e5f-bc11-44a6975c7708=0/ │ │ │ │ │ └── col-7781d665-6951-4244-b9bc-a28e477e2d57=true/ │ │ │ │ │ └── col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae=0.0/ │ │ │ │ │ └── col-3463c48b-4b94-4500-b14f-4a554284b94f=0.0/ │ │ │ │ │ └── col-05f332c4-ebdb-4437-9e80-e23f92bee4a2=0/ │ │ │ │ │ └── col-c025b8f8-481c-4db2-8932-f37129146ceb=2021-09-08/ │ │ │ │ │ └── col-bd963d5f-2199-4700-b5d6-0759bd7a9d90=2021-09-08 11%3A11%3A11/ │ │ │ │ │ └── col-01ec4063-ed54-41db-805e-ebfd9b9a6e67=0/ │ │ │ │ │ ├── .part-00002-e0842c02-93d2-4c38-b041-fc88b581688b.c000.snappy.parquet.crc │ │ │ │ │ └── part-00002-e0842c02-93d2-4c38-b041-fc88b581688b.c000.snappy.parquet │ │ │ │ ├── col-25948c99-9f51-4e05-9f9e-b4f7042f75ed=1/ │ │ │ │ │ └── col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa=1/ │ │ │ │ │ └── col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08=1/ │ │ │ │ │ └── col-29f826c0-7fff-4e5f-bc11-44a6975c7708=1/ │ │ │ │ │ └── col-7781d665-6951-4244-b9bc-a28e477e2d57=false/ │ │ │ │ │ └── col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae=1.0/ │ │ │ │ │ └── col-3463c48b-4b94-4500-b14f-4a554284b94f=1.0/ │ │ │ │ │ └── col-05f332c4-ebdb-4437-9e80-e23f92bee4a2=1/ │ │ │ │ │ └── col-c025b8f8-481c-4db2-8932-f37129146ceb=2021-09-08/ │ │ │ │ │ └── col-bd963d5f-2199-4700-b5d6-0759bd7a9d90=2021-09-08 11%3A11%3A11/ │ │ │ │ │ └── col-01ec4063-ed54-41db-805e-ebfd9b9a6e67=1/ │ │ │ │ │ ├── .part-00000-c9d9ab23-0f5c-4a12-837f-b709c5037905.c000.snappy.parquet.crc │ │ │ │ │ └── part-00000-c9d9ab23-0f5c-4a12-837f-b709c5037905.c000.snappy.parquet │ │ │ │ └── col-25948c99-9f51-4e05-9f9e-b4f7042f75ed=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ └── col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ └── col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ └── col-29f826c0-7fff-4e5f-bc11-44a6975c7708=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ └── col-7781d665-6951-4244-b9bc-a28e477e2d57=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ └── col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ └── col-3463c48b-4b94-4500-b14f-4a554284b94f=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ └── col-05f332c4-ebdb-4437-9e80-e23f92bee4a2=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ └── col-c025b8f8-481c-4db2-8932-f37129146ceb=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ └── col-bd963d5f-2199-4700-b5d6-0759bd7a9d90=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ └── col-01ec4063-ed54-41db-805e-ebfd9b9a6e67=__HIVE_DEFAULT_PARTITION__/ │ │ │ │ ├── .part-00001-dac9e981-94cc-4dc5-8d01-2cbae8ef69c6.c000.snappy.parquet.crc │ │ │ │ └── part-00001-dac9e981-94cc-4dc5-8d01-2cbae8ef69c6.c000.snappy.parquet │ │ │ ├── data-reader-primitives-column-mapping-name/ │ │ │ │ ├── .part-00000-dedd3195-6cd1-451d-83b8-fe0028f9b2b6-c000.snappy.parquet.crc │ │ │ │ ├── .part-00001-d8bdfc55-29fe-40bc-bfe4-f7732d559aa9-c000.snappy.parquet.crc │ │ │ │ ├── _delta_log/ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ ├── part-00000-dedd3195-6cd1-451d-83b8-fe0028f9b2b6-c000.snappy.parquet │ │ │ │ └── part-00001-d8bdfc55-29fe-40bc-bfe4-f7732d559aa9-c000.snappy.parquet │ │ │ ├── json-files/ │ │ │ │ ├── 1.json │ │ │ │ ├── 2.json │ │ │ │ └── 3.json │ │ │ ├── json-files-all-empty/ │ │ │ │ ├── 1.json │ │ │ │ ├── 2.json │ │ │ │ └── 3.json │ │ │ ├── json-files-with-empty/ │ │ │ │ ├── 1.json │ │ │ │ ├── 2.json │ │ │ │ ├── 3.json │ │ │ │ ├── 4.json │ │ │ │ ├── 5.json │ │ │ │ ├── 6.json │ │ │ │ ├── 7.json │ │ │ │ └── 8.json │ │ │ ├── kernel-pagination-all-jsons/ │ │ │ │ ├── _delta_log/ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ └── 00000000000000000002.json │ │ │ │ ├── info.txt │ │ │ │ ├── part-00000-520b63c8-7004-4fb3-9a7d-6b1ee913d1ac-c000.snappy.parquet │ │ │ │ ├── part-00000-5aead402-bb3a-4c76-9d78-67c09cfcfa8a-c000.snappy.parquet │ │ │ │ ├── part-00000-8dc5e78e-a25c-47ed-8025-e171c85ace7a-c000.snappy.parquet │ │ │ │ ├── part-00001-5109896f-2a5e-4e79-b445-8966c6c82ef0-c000.snappy.parquet │ │ │ │ ├── part-00001-5236d99c-44d0-4eb3-b9a6-c84066e73742-c000.snappy.parquet │ │ │ │ ├── part-00001-f7ad6879-5e8e-4ac2-817f-2dc95477d7d7-c000.snappy.parquet │ │ │ │ ├── part-00002-021db285-9b02-4973-a282-afcfd8afd007-c000.snappy.parquet │ │ │ │ ├── part-00002-113a09fa-8d92-4285-a192-75d9ef68ccdf-c000.snappy.parquet │ │ │ │ ├── part-00002-38299dc0-eb52-4e16-a1df-f3c6bb37eaf4-c000.snappy.parquet │ │ │ │ ├── part-00003-90bfacb7-34aa-4dab-bc3d-9367d9045103-c000.snappy.parquet │ │ │ │ ├── part-00003-a2307510-6cac-4772-8ad1-94e874534491-c000.snappy.parquet │ │ │ │ ├── part-00003-d6736222-7d56-4b90-ad5e-abea2a47353d-c000.snappy.parquet │ │ │ │ ├── part-00004-39753157-19bf-4a6d-b65d-04505516762d-c000.snappy.parquet │ │ │ │ ├── part-00004-b0a130bc-96cd-4b89-8d24-253f9fea21cb-c000.snappy.parquet │ │ │ │ └── part-00004-cdc9fd11-6e89-45d0-8941-a74a104fde75-c000.snappy.parquet │ │ │ ├── kernel-pagination-multi-part-checkpoints/ │ │ │ │ ├── _delta_log/ │ │ │ │ │ ├── 00000000000000000000.checkpoint.0000000001.0000000003.parquet │ │ │ │ │ ├── 00000000000000000000.checkpoint.0000000002.0000000003.parquet │ │ │ │ │ ├── 00000000000000000000.checkpoint.0000000003.0000000003.parquet │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ └── _last_checkpoint │ │ │ │ ├── info.txt │ │ │ │ ├── part-00000-3ae77526-4e2f-4408-b0fd-769dfed78395-c000.snappy.parquet │ │ │ │ ├── part-00000-78e9541a-f565-4548-a474-a675a4aab37f-c000.snappy.parquet │ │ │ │ ├── part-00000-99790b5f-5683-4194-8d08-b3c87c854589-c000.snappy.parquet │ │ │ │ ├── part-00001-54a44075-ed30-4dc5-b4a2-59a91f28fd37-c000.snappy.parquet │ │ │ │ ├── part-00002-9f4f85c0-5694-4b0c-9cfb-3b616460786c-c000.snappy.parquet │ │ │ │ ├── part-00003-e33eff7b-6300-4cc5-8612-27f4959d10eb-c000.snappy.parquet │ │ │ │ ├── part-00004-a5f332bd-bf7f-4552-a0ae-011cc6888787-c000.snappy.parquet │ │ │ │ ├── part-00005-4d98fede-302c-4db9-817e-ab226e024e63-c000.snappy.parquet │ │ │ │ ├── part-00006-06a29b13-6f35-42fa-a937-1403c16c5d9f-c000.snappy.parquet │ │ │ │ ├── part-00007-6c15fb5f-ece5-4067-bff3-fe2b4481066e-c000.snappy.parquet │ │ │ │ ├── part-00008-2e216ef2-8209-4c29-a775-d1ff656485a1-c000.snappy.parquet │ │ │ │ ├── part-00009-ba5132c6-d400-4260-9b1e-59d575d34e45-c000.snappy.parquet │ │ │ │ ├── part-00010-1b506159-7065-4f32-886b-b7ac88b9f25a-c000.snappy.parquet │ │ │ │ ├── part-00011-679b09b3-3872-4ead-98a3-97ad55c83560-c000.snappy.parquet │ │ │ │ ├── part-00012-7773a168-219f-46fe-a977-807219b9facb-c000.snappy.parquet │ │ │ │ ├── part-00013-340640ec-f6d0-4ca8-a015-8b6be2045627-c000.snappy.parquet │ │ │ │ ├── part-00014-61a01309-3035-43dd-ab57-722dc7a1d6c7-c000.snappy.parquet │ │ │ │ ├── part-00015-c4272dab-2228-43fc-9a33-f778a584e0f8-c000.snappy.parquet │ │ │ │ ├── part-00016-e5846569-bace-41ec-9652-81e1a5b01b31-c000.snappy.parquet │ │ │ │ └── part-00017-2057bacd-70e6-43b9-be0b-eb6787dfb990-c000.snappy.parquet │ │ │ ├── kernel-pagination-single-checkpoint/ │ │ │ │ ├── _delta_log/ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ ├── 00000000000000000004.json │ │ │ │ │ ├── 00000000000000000005.json │ │ │ │ │ ├── 00000000000000000006.json │ │ │ │ │ ├── 00000000000000000007.json │ │ │ │ │ ├── 00000000000000000008.json │ │ │ │ │ ├── 00000000000000000009.checkpoint.parquet │ │ │ │ │ ├── 00000000000000000009.json │ │ │ │ │ ├── 00000000000000000010.checkpoint.parquet │ │ │ │ │ ├── 00000000000000000010.json │ │ │ │ │ ├── 00000000000000000011.json │ │ │ │ │ ├── 00000000000000000012.json │ │ │ │ │ └── _last_checkpoint │ │ │ │ ├── info.txt │ │ │ │ ├── part-00000-016be37f-0c8d-470d-b7b1-160515f14dc2-c000.snappy.parquet │ │ │ │ ├── part-00000-0c720b4a-672d-4bcc-a267-d79c3c6dbc8f-c000.snappy.parquet │ │ │ │ ├── part-00000-24d7f48b-c0c4-4363-a208-2a6417e19022-c000.snappy.parquet │ │ │ │ ├── part-00000-2ae23645-7b78-491f-aa29-01ba885bba82-c000.snappy.parquet │ │ │ │ ├── part-00000-39319b4a-4a46-4698-a3f0-0b9f42ff9fad-c000.snappy.parquet │ │ │ │ ├── part-00000-3e44afef-2600-426f-9f47-771f7ace1868-c000.snappy.parquet │ │ │ │ ├── part-00000-677492ad-40aa-40c6-a1f0-bf9c7dd641e2-c000.snappy.parquet │ │ │ │ ├── part-00000-70b4a723-351c-4495-b4ea-088d5637d901-c000.snappy.parquet │ │ │ │ ├── part-00000-852814f7-34c8-43da-9c22-8e5f8bd0d571-c000.snappy.parquet │ │ │ │ ├── part-00000-b29b2c0c-a58e-43d8-880b-9a113c27035e-c000.snappy.parquet │ │ │ │ ├── part-00000-c5b4b509-af7c-42b7-bbf5-8797cdb5eeaa-c000.snappy.parquet │ │ │ │ ├── part-00000-ed129b33-0211-4af7-bdce-432cf823c5b7-c000.snappy.parquet │ │ │ │ ├── part-00000-f54d5bbb-fe06-4c91-a1ed-500abce87546-c000.snappy.parquet │ │ │ │ ├── part-00001-0cea8033-06b0-4cbf-b32b-f14cc03fae77-c000.snappy.parquet │ │ │ │ ├── part-00001-2605f1b1-7042-411e-850a-a969e55275ae-c000.snappy.parquet │ │ │ │ ├── part-00001-4014c89a-e2c8-44ea-8421-86d80922ed3d-c000.snappy.parquet │ │ │ │ ├── part-00001-7c0e1634-1491-4f2d-9eec-15950d9d900d-c000.snappy.parquet │ │ │ │ ├── part-00001-7d8b44fb-445c-4e16-a1ff-75da579203db-c000.snappy.parquet │ │ │ │ ├── part-00001-81d82b44-4bce-405c-b483-ff741b45d3ef-c000.snappy.parquet │ │ │ │ ├── part-00001-827b6732-c283-437a-a9e0-95458ded5340-c000.snappy.parquet │ │ │ │ ├── part-00001-8410fa8f-df52-445b-82ea-6d03366945d0-c000.snappy.parquet │ │ │ │ ├── part-00001-af89282e-3e0f-4589-9862-274a3e343245-c000.snappy.parquet │ │ │ │ ├── part-00001-c8077b5a-da61-4f74-a958-fa885076c35b-c000.snappy.parquet │ │ │ │ ├── part-00001-d25a786d-5ecc-4d53-98e3-b82dae5627da-c000.snappy.parquet │ │ │ │ ├── part-00001-d723c7f5-e27d-4a89-b4a9-586a6c960721-c000.snappy.parquet │ │ │ │ └── part-00001-eec9d516-0606-439e-9067-4edd54ce97c6-c000.snappy.parquet │ │ │ ├── kernel-pagination-v2-checkpoint-json/ │ │ │ │ ├── _delta_log/ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ ├── 00000000000000000002.checkpoint.6374b053-df23-479b-b2cf-c9c550132b49.json │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ ├── 00000000000000000004.checkpoint.a2670232-dd52-4e21-8ba7-1f70fe762bce.json │ │ │ │ │ ├── 00000000000000000004.json │ │ │ │ │ ├── _last_checkpoint │ │ │ │ │ └── _sidecars/ │ │ │ │ │ ├── 00000000000000000002.checkpoint.0000000001.0000000002.bd1885fd-6ec0-4370-b0f5-43b5162fd4de.parquet │ │ │ │ │ ├── 00000000000000000002.checkpoint.0000000002.0000000002.0a8d73ee-aa83-49d0-9583-c99db75b89b2.parquet │ │ │ │ │ └── 00000000000000000004.checkpoint.0000000001.0000000001.019924f2-3318-4cca-a460-b7d0b75f0d0f.parquet │ │ │ │ ├── part-00000-240b5dd6-323b-4f74-b6bc-ab9fdcacc630-c000.snappy.parquet │ │ │ │ ├── part-00000-813a3813-84c9-4251-bbe6-f6502a32b833-c000.snappy.parquet │ │ │ │ ├── part-00000-9a247ca4-22bf-4173-bd69-66401dad2178-c000.snappy.parquet │ │ │ │ ├── part-00001-534ea355-2edd-4046-8d49-d932469170c7-c000.snappy.parquet │ │ │ │ ├── part-00002-4438bc9d-9c60-4dd2-9343-574743ea4ca8-c000.snappy.parquet │ │ │ │ └── part-00003-ae431d66-23d5-4dc7-b961-136ce33e63da-c000.snappy.parquet │ │ │ ├── kernel-pagination-v2-checkpoint-parquet/ │ │ │ │ ├── _delta_log/ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ ├── 00000000000000000002.checkpoint.e8fa2696-9728-4e9c-b285-634743fdd4fb.parquet │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ ├── 00000000000000000004.checkpoint.1391a262-4df6-494d-8166-dcd139a6ba46.json │ │ │ │ │ ├── 00000000000000000004.json │ │ │ │ │ ├── _last_checkpoint │ │ │ │ │ └── _sidecars/ │ │ │ │ │ ├── 00000000000000000002.checkpoint.0000000001.0000000002.055454d8-329c-4e0e-864d-7f867075af33.parquet │ │ │ │ │ ├── 00000000000000000002.checkpoint.0000000002.0000000002.33321cc1-9c55-4d1f-8511-fafe6d2e1133.parquet │ │ │ │ │ └── 00000000000000000004.checkpoint.0000000001.0000000001.87b3aafd-6627-401d-b9aa-83f6b2450f0a.parquet │ │ │ │ ├── part-00000-38f3b7ca-0e92-449a-a2ff-0d4b7c7908f3-c000.snappy.parquet │ │ │ │ ├── part-00000-485b0fff-1c7b-4f14-92e9-a72300fcdf88-c000.snappy.parquet │ │ │ │ ├── part-00000-4f78beda-ea4d-4ab3-95fc-ab68e40b3fce-c000.snappy.parquet │ │ │ │ ├── part-00001-f7a80035-0622-431e-832e-a756c65cb2a5-c000.snappy.parquet │ │ │ │ ├── part-00002-5754df9c-5a25-43a6-947b-f27840fddb1a-c000.snappy.parquet │ │ │ │ └── part-00003-6ab7bbbb-e14d-4fa3-8767-06b509e0a666-c000.snappy.parquet │ │ │ ├── log4j2.properties │ │ │ ├── parquet/ │ │ │ │ ├── parquet-thrift-compat.snappy.parquet │ │ │ │ ├── parquet-timestamp_ntz_int96.parquet │ │ │ │ └── row_index_multiple_row_groups.parquet │ │ │ ├── parquet-basic-row-indexes/ │ │ │ │ ├── part-00000-4c461819-c86f-4a4b-8efe-8cb90239fa87-c000.snappy.parquet │ │ │ │ ├── part-00000-5a2c496f-7402-4feb-a0e5-e5af2e795ec6-c000.snappy.parquet │ │ │ │ └── part-00000-5cd71227-8d18-48e8-9d66-269a9c4eae10-c000.snappy.parquet │ │ │ ├── spark-shredded-variant-preview-delta/ │ │ │ │ ├── .test%file%prefix-part-00000-24104cdb-691b-4410-a2a1-afc84fe2ea18-c000.snappy.parquet.crc │ │ │ │ ├── .test%file%prefix-part-00000-5ed80cd3-35e4-419e-bf56-e685f8634cbf-c000.snappy.parquet.crc │ │ │ │ ├── .test%file%prefix-part-00000-bda6fee1-d8d4-4a8b-a1fb-eb171758ef40-c000.snappy.parquet.crc │ │ │ │ ├── .test%file%prefix-part-00001-0ffdbb7b-af76-4d0b-9cbe-08bb2091c1c9-c000.snappy.parquet.crc │ │ │ │ ├── _delta_log/ │ │ │ │ │ ├── .00000000000000000000.crc.crc │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ ├── .00000000000000000001.crc.crc │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ ├── .00000000000000000002.checkpoint.parquet.crc │ │ │ │ │ ├── .00000000000000000002.crc.crc │ │ │ │ │ ├── .00000000000000000002.json.crc │ │ │ │ │ ├── 00000000000000000000.crc │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ ├── 00000000000000000001.crc │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ ├── 00000000000000000002.checkpoint.parquet │ │ │ │ │ ├── 00000000000000000002.crc │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ └── _last_checkpoint │ │ │ │ ├── info.txt │ │ │ │ ├── test%file%prefix-part-00000-24104cdb-691b-4410-a2a1-afc84fe2ea18-c000.snappy.parquet │ │ │ │ ├── test%file%prefix-part-00000-5ed80cd3-35e4-419e-bf56-e685f8634cbf-c000.snappy.parquet │ │ │ │ ├── test%file%prefix-part-00000-bda6fee1-d8d4-4a8b-a1fb-eb171758ef40-c000.snappy.parquet │ │ │ │ └── test%file%prefix-part-00001-0ffdbb7b-af76-4d0b-9cbe-08bb2091c1c9-c000.snappy.parquet │ │ │ ├── spark-variant-checkpoint/ │ │ │ │ ├── .part-00000-16c852df-ba66-4080-be25-530a05922422-c000.snappy.parquet.crc │ │ │ │ ├── .part-00000-1e14ba22-3114-46d1-96fb-48b4912507ce-c000.snappy.parquet.crc │ │ │ │ ├── .part-00000-9a9c570c-ee32-4322-ad2f-8c837a77d398-c000.snappy.parquet.crc │ │ │ │ ├── .part-00001-664313d3-14b4-4dbf-8110-77001b877182-c000.snappy.parquet.crc │ │ │ │ ├── _delta_log/ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ ├── .00000000000000000002.checkpoint.parquet.crc │ │ │ │ │ ├── .00000000000000000002.json.crc │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ ├── 00000000000000000002.checkpoint.parquet │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ └── _last_checkpoint │ │ │ │ ├── info.txt │ │ │ │ ├── part-00000-16c852df-ba66-4080-be25-530a05922422-c000.snappy.parquet │ │ │ │ ├── part-00000-1e14ba22-3114-46d1-96fb-48b4912507ce-c000.snappy.parquet │ │ │ │ ├── part-00000-9a9c570c-ee32-4322-ad2f-8c837a77d398-c000.snappy.parquet │ │ │ │ └── part-00001-664313d3-14b4-4dbf-8110-77001b877182-c000.snappy.parquet │ │ │ └── spark-variant-stable-feature-checkpoint/ │ │ │ ├── .test%file%prefix-part-00000-5f6f82ed-28c5-4f4e-b358-93904826c84d-c000.snappy.parquet.crc │ │ │ ├── .test%file%prefix-part-00000-c98a0433-2bfc-4903-9b2e-0fb34243f552-c000.snappy.parquet.crc │ │ │ ├── .test%file%prefix-part-00001-95062e44-13fa-4917-b169-d289cd21c717-c000.snappy.parquet.crc │ │ │ ├── .test%file%prefix-part-00001-c7ee7ba3-625c-495b-95df-06f44ffb72c9-c000.snappy.parquet.crc │ │ │ ├── _delta_log/ │ │ │ │ ├── .00000000000000000000.crc.crc │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ ├── .00000000000000000001.crc.crc │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ ├── 00000000000000000000.crc │ │ │ │ ├── 00000000000000000000.json │ │ │ │ ├── 00000000000000000001.crc │ │ │ │ ├── 00000000000000000001.json │ │ │ │ └── info.txt │ │ │ ├── part-00000-5f6f82ed-28c5-4f4e-b358-93904826c84d-c000.snappy.parquet │ │ │ ├── part-00000-c98a0433-2bfc-4903-9b2e-0fb34243f552-c000.snappy.parquet │ │ │ ├── part-00001-95062e44-13fa-4917-b169-d289cd21c717-c000.snappy.parquet │ │ │ └── part-00001-c7ee7ba3-625c-495b-95df-06f44ffb72c9-c000.snappy.parquet │ │ └── scala/ │ │ └── io/ │ │ └── delta/ │ │ └── kernel/ │ │ └── defaults/ │ │ ├── CheckpointV2ReadSuite.scala │ │ ├── ChecksumLogReplayMetricsTestBase.scala │ │ ├── ChecksumSimpleComparisonSuite.scala │ │ ├── ChecksumStatsSuite.scala │ │ ├── ChecksumUtilsSuite.scala │ │ ├── ColumnDefaultsSuite.scala │ │ ├── CommitIcebergActionSuite.scala │ │ ├── CommitMetadataE2ESuite.scala │ │ ├── CreateCheckpointSuite.scala │ │ ├── DataSkippingDeltaTestsUtils.scala │ │ ├── DeletionVectorSuite.scala │ │ ├── DeltaColumnMappingSuite.scala │ │ ├── DeltaIcebergCompatBaseSuite.scala │ │ ├── DeltaIcebergCompatV2Suite.scala │ │ ├── DeltaIcebergCompatV3Suite.scala │ │ ├── DeltaLogActionUtilsE2ESuite.scala │ │ ├── DeltaReplaceTableColumnMappingSuite.scala │ │ ├── DeltaReplaceTableSuite.scala │ │ ├── DeltaReplaceTableSuiteBase.scala │ │ ├── DeltaTableClusteringSuite.scala │ │ ├── DeltaTableFeaturesSuite.scala │ │ ├── DeltaTableReadsSuite.scala │ │ ├── DeltaTableSchemaEvolutionSuite.scala │ │ ├── DeltaTableWriteWithCrcSuite.scala │ │ ├── DeltaTableWritesSuite.scala │ │ ├── DeltaTableWritesTransactionBuilderV2Suite.scala │ │ ├── DirectoryCreationSuite.scala │ │ ├── DomainMetadataCheckSumReplayMetricsSuite.scala │ │ ├── DomainMetadataSuite.scala │ │ ├── IcebergWriterCompatV1Suite.scala │ │ ├── InCommitTimestampSuite.scala │ │ ├── LogCompactionSuite.scala │ │ ├── LogCompactionWriterSuite.scala │ │ ├── LogReplayBaseSuite.scala │ │ ├── LogReplayEngineMetricsSuite.scala │ │ ├── LogReplaySuite.scala │ │ ├── PaginatedScanSuite.scala │ │ ├── PartitionPruningSuite.scala │ │ ├── PartitionUtilsSuite.scala │ │ ├── PostCommitSnapshotSuite.scala │ │ ├── RowTrackingSuite.scala │ │ ├── ScanSuite.scala │ │ ├── SnapshotChecksumStatisticsAndWriteSuite.scala │ │ ├── SnapshotSuite.scala │ │ ├── TableChangesSuite.scala │ │ ├── TablePropertiesSuite.scala │ │ ├── TimestampStatsAndDataSkippingSuite.scala │ │ ├── TransactionCommitLoopSuite.scala │ │ ├── catalogManaged/ │ │ │ ├── CatalogManagedE2EReadSuite.scala │ │ │ └── CatalogManagedPropertyValidationSuite.scala │ │ ├── engine/ │ │ │ ├── DefaultExpressionHandlerSuite.scala │ │ │ ├── DefaultFileSystemClientSuite.scala │ │ │ ├── DefaultJsonHandlerSuite.scala │ │ │ └── DefaultParquetHandlerSuite.scala │ │ ├── internal/ │ │ │ ├── expressions/ │ │ │ │ ├── DefaultExpressionEvaluatorSuite.scala │ │ │ │ ├── DefaultPredicateEvaluatorSuite.scala │ │ │ │ ├── ExpressionSuiteBase.scala │ │ │ │ └── ImplicitCastExpressionSuite.scala │ │ │ ├── json/ │ │ │ │ └── JsonUtilsSuite.scala │ │ │ ├── logstore/ │ │ │ │ └── LogStoreProviderSuite.scala │ │ │ └── parquet/ │ │ │ ├── ParquetFileReaderSuite.scala │ │ │ ├── ParquetFileWriterSuite.scala │ │ │ ├── ParquetReaderPredicatePushdownSuite.scala │ │ │ ├── ParquetSchemaUtilsSuite.scala │ │ │ └── ParquetSuiteBase.scala │ │ ├── metrics/ │ │ │ ├── LoggingMetricsReporterSuite.scala │ │ │ ├── MetricsReportTestUtils.scala │ │ │ ├── ScanReportSuite.scala │ │ │ ├── SnapshotReportSuite.scala │ │ │ └── TransactionReportSuite.scala │ │ ├── test/ │ │ │ └── AbstractTableManagerAdapter.scala │ │ └── utils/ │ │ ├── DefaultVectorTestUtils.scala │ │ ├── ExpressionTestUtils.scala │ │ ├── TestCommitterUtils.scala │ │ ├── TestRow.scala │ │ ├── TestUtils.scala │ │ ├── TransactionBuilderSupport.scala │ │ └── WriteUtils.scala │ ├── project/ │ │ └── plugins.sbt │ ├── scalastyle-config.xml │ ├── unitycatalog/ │ │ └── src/ │ │ ├── main/ │ │ │ └── java/ │ │ │ └── io/ │ │ │ └── delta/ │ │ │ └── kernel/ │ │ │ └── unitycatalog/ │ │ │ ├── UCCatalogManagedClient.java │ │ │ ├── UCCatalogManagedCommitter.java │ │ │ ├── UnityCatalogUtils.java │ │ │ ├── adapters/ │ │ │ │ ├── MetadataAdapter.java │ │ │ │ ├── ProtocolAdapter.java │ │ │ │ └── UniformAdapter.java │ │ │ ├── metrics/ │ │ │ │ ├── UcCommitTelemetry.java │ │ │ │ ├── UcLoadSnapshotTelemetry.java │ │ │ │ └── UcPublishTelemetry.java │ │ │ └── utils/ │ │ │ └── OperationTimer.java │ │ └── test/ │ │ ├── resources/ │ │ │ └── log4j2.properties │ │ └── scala/ │ │ └── io/ │ │ └── delta/ │ │ └── kernel/ │ │ └── unitycatalog/ │ │ ├── InMemoryUCClient.scala │ │ ├── InMemoryUCClientSuite.scala │ │ ├── UCCatalogManagedClientCommitRangeSuite.scala │ │ ├── UCCatalogManagedClientSuite.scala │ │ ├── UCCatalogManagedCommitterSuite.scala │ │ ├── UCCatalogManagedTestUtils.scala │ │ ├── UCE2ESuite.scala │ │ ├── UCPublishingSuite.scala │ │ ├── UcCommitTelemetrySuite.scala │ │ ├── UcLoadSnapshotTelemetrySuite.scala │ │ ├── UcPublishTelemetrySuite.scala │ │ ├── UnityCatalogUtilsSuite.scala │ │ └── adapters/ │ │ └── ActionAdaptersSuite.scala │ └── version.sbt ├── project/ │ ├── Checkstyle.scala │ ├── CrossSparkVersions.scala │ ├── FlinkMimaExcludes.scala │ ├── Mima.scala │ ├── MultiShardMultiJVMTestParallelization.scala │ ├── README.md │ ├── ShadedIcebergBuild.scala │ ├── SparkMimaExcludes.scala │ ├── StandaloneMimaExcludes.scala │ ├── TestParallelization.scala │ ├── Unidoc.scala │ ├── build.properties │ ├── plugins.sbt │ ├── project/ │ │ └── plugins.sbt │ ├── scripts/ │ │ ├── collect_test_durations.py │ │ └── get_spark_version_info.py │ ├── test-durations.csv │ └── tests/ │ └── test_cross_spark_publish.py ├── protocol_rfcs/ │ ├── README.md │ ├── accepted/ │ │ ├── catalog-managed.md │ │ ├── in-commit-timestamps.md │ │ ├── type-widening.md │ │ ├── vacuum-protocol-check.md │ │ └── variant-type.md │ ├── checkpoint-protection.md │ ├── collated-string-type.md │ ├── column-mapping-usage-tracking.md │ ├── iceberg-compat-v3.md │ ├── iceberg-writer-compat-v1.md │ ├── materialize-partition-columns.md │ ├── rejected/ │ │ └── managed-commits.md │ ├── template.md │ └── variant-shredding.md ├── python/ │ ├── README.md │ ├── delta/ │ │ ├── __init__.py │ │ ├── _typing.py │ │ ├── connect/ │ │ │ ├── __init__.py │ │ │ ├── _typing.py │ │ │ ├── exceptions.py │ │ │ ├── plan.py │ │ │ ├── proto/ │ │ │ │ ├── __init__.py │ │ │ │ ├── base_pb2.py │ │ │ │ ├── base_pb2.pyi │ │ │ │ ├── commands_pb2.py │ │ │ │ ├── commands_pb2.pyi │ │ │ │ ├── relations_pb2.py │ │ │ │ └── relations_pb2.pyi │ │ │ ├── tables.py │ │ │ ├── testing/ │ │ │ │ ├── __init__.py │ │ │ │ └── utils.py │ │ │ └── tests/ │ │ │ ├── __init__.py │ │ │ └── test_deltatable.py │ │ ├── exceptions/ │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ └── captured.py │ │ ├── integration_tests/ │ │ │ └── unity-catalog-commit-coordinator-integration-tests.py │ │ ├── pip_utils.py │ │ ├── py.typed │ │ ├── tables.py │ │ ├── testing/ │ │ │ ├── __init__.py │ │ │ ├── log4j2.properties │ │ │ └── utils.py │ │ ├── tests/ │ │ │ ├── __init__.py │ │ │ ├── test_deltatable.py │ │ │ ├── test_exceptions.py │ │ │ ├── test_pip_utils.py │ │ │ ├── test_sql.py │ │ │ └── test_version.py │ │ └── version.py │ ├── environment.yml │ ├── mypy.ini │ └── run-tests.py ├── run-integration-tests.py ├── run-tests.py ├── scalastyle-config.xml ├── setup.py ├── sharing/ │ └── src/ │ ├── main/ │ │ ├── resources/ │ │ │ └── META-INF/ │ │ │ └── services/ │ │ │ └── org.apache.spark.sql.sources.DataSourceRegister │ │ └── scala/ │ │ └── io/ │ │ └── delta/ │ │ └── sharing/ │ │ └── spark/ │ │ ├── DeltaFormatSharingLimitPushDown.scala │ │ ├── DeltaFormatSharingSource.scala │ │ ├── DeltaSharingCDFUtils.scala │ │ ├── DeltaSharingDataSource.scala │ │ ├── DeltaSharingFileIndex.scala │ │ ├── DeltaSharingLogFileSystem.scala │ │ ├── DeltaSharingUtils.scala │ │ ├── PrepareDeltaSharingScan.scala │ │ └── model.scala │ └── test/ │ ├── scala/ │ │ └── io/ │ │ └── delta/ │ │ └── sharing/ │ │ └── spark/ │ │ ├── DeltaFormatSharingSourceSuite.scala │ │ ├── DeltaSharingCDFUtilsSuite.scala │ │ ├── DeltaSharingDataSourceCMSuite.scala │ │ ├── DeltaSharingDataSourceDeltaSuite.scala │ │ ├── DeltaSharingDataSourceDeltaTestUtils.scala │ │ ├── DeltaSharingDataSourceTypeWideningSuite.scala │ │ ├── DeltaSharingFileIndexSuite.scala │ │ ├── DeltaSharingLogFileSystemSuite.scala │ │ ├── DeltaSharingTestSparkUtils.scala │ │ ├── DeltaSharingUtilsSuite.scala │ │ ├── TestClientForDeltaFormatSharing.scala │ │ └── TestDeltaSharingFileSystem.scala │ └── scala-shims/ │ ├── spark-4.0/ │ │ └── SharingStreamingTestShims.scala │ ├── spark-4.1/ │ │ └── SharingStreamingTestShims.scala │ └── spark-4.2/ │ └── SharingStreamingTestShims.scala ├── spark/ │ ├── delta-suite-generator/ │ │ └── src/ │ │ ├── main/ │ │ │ ├── resources/ │ │ │ │ └── scalafmt.conf │ │ │ └── scala/ │ │ │ └── io/ │ │ │ └── delta/ │ │ │ └── suitegenerator/ │ │ │ ├── ModularSuiteGenerator.scala │ │ │ ├── SuiteGeneratorConfig.scala │ │ │ └── SuitesWriter.scala │ │ └── test/ │ │ └── scala/ │ │ └── io/ │ │ └── delta/ │ │ └── suitegenerator/ │ │ └── ValidateGeneratedSuites.scala │ ├── src/ │ │ ├── main/ │ │ │ ├── antlr4/ │ │ │ │ └── io/ │ │ │ │ └── delta/ │ │ │ │ └── sql/ │ │ │ │ └── parser/ │ │ │ │ └── DeltaSqlBase.g4 │ │ │ ├── java/ │ │ │ │ ├── io/ │ │ │ │ │ └── delta/ │ │ │ │ │ └── dynamodbcommitcoordinator/ │ │ │ │ │ ├── DynamoDBCommitCoordinatorClient.java │ │ │ │ │ ├── DynamoDBCommitCoordinatorClientBuilder.java │ │ │ │ │ ├── DynamoDBTableEntryConstants.java │ │ │ │ │ ├── ReflectionUtils.java │ │ │ │ │ └── integration_tests/ │ │ │ │ │ └── dynamodb_commitcoordinator_integration_test.py │ │ │ │ └── org/ │ │ │ │ └── apache/ │ │ │ │ └── spark/ │ │ │ │ └── sql/ │ │ │ │ └── delta/ │ │ │ │ ├── DeltaV2Mode.java │ │ │ │ ├── RowIndexFilter.java │ │ │ │ ├── RowIndexFilterType.java │ │ │ │ ├── sources/ │ │ │ │ │ └── AdmittableFile.java │ │ │ │ └── util/ │ │ │ │ └── CatalogTableUtils.java │ │ │ ├── java-shims/ │ │ │ │ ├── spark-4.0/ │ │ │ │ │ └── org/ │ │ │ │ │ └── apache/ │ │ │ │ │ └── spark/ │ │ │ │ │ └── sql/ │ │ │ │ │ └── delta/ │ │ │ │ │ └── shims/ │ │ │ │ │ └── VariantStatsShims.java │ │ │ │ ├── spark-4.1/ │ │ │ │ │ └── org/ │ │ │ │ │ └── apache/ │ │ │ │ │ └── spark/ │ │ │ │ │ └── sql/ │ │ │ │ │ └── delta/ │ │ │ │ │ └── shims/ │ │ │ │ │ └── VariantStatsShims.java │ │ │ │ └── spark-4.2/ │ │ │ │ └── org/ │ │ │ │ └── apache/ │ │ │ │ └── spark/ │ │ │ │ └── sql/ │ │ │ │ └── delta/ │ │ │ │ └── shims/ │ │ │ │ └── VariantStatsShims.java │ │ │ ├── resources/ │ │ │ │ ├── META-INF/ │ │ │ │ │ └── services/ │ │ │ │ │ └── org.apache.spark.sql.sources.DataSourceRegister │ │ │ │ ├── error/ │ │ │ │ │ └── delta-error-classes.json │ │ │ │ └── org/ │ │ │ │ └── apache/ │ │ │ │ └── spark/ │ │ │ │ └── SparkLayout.json │ │ │ ├── scala/ │ │ │ │ ├── com/ │ │ │ │ │ └── databricks/ │ │ │ │ │ └── spark/ │ │ │ │ │ └── util/ │ │ │ │ │ └── DatabricksLogging.scala │ │ │ │ ├── io/ │ │ │ │ │ └── delta/ │ │ │ │ │ ├── exceptions/ │ │ │ │ │ │ └── DeltaConcurrentExceptions.scala │ │ │ │ │ ├── implicits/ │ │ │ │ │ │ └── package.scala │ │ │ │ │ ├── sql/ │ │ │ │ │ │ ├── AbstractDeltaSparkSessionExtension.scala │ │ │ │ │ │ └── parser/ │ │ │ │ │ │ └── DeltaSqlParser.scala │ │ │ │ │ └── tables/ │ │ │ │ │ ├── DeltaColumnBuilder.scala │ │ │ │ │ ├── DeltaMergeBuilder.scala │ │ │ │ │ ├── DeltaOptimizeBuilder.scala │ │ │ │ │ ├── DeltaTable.scala │ │ │ │ │ ├── DeltaTableBuilder.scala │ │ │ │ │ └── execution/ │ │ │ │ │ ├── DeltaConvert.scala │ │ │ │ │ ├── DeltaTableBuilderOptions.scala │ │ │ │ │ ├── DeltaTableOperations.scala │ │ │ │ │ └── VacuumTableCommand.scala │ │ │ │ └── org/ │ │ │ │ └── apache/ │ │ │ │ └── spark/ │ │ │ │ └── sql/ │ │ │ │ ├── catalyst/ │ │ │ │ │ ├── TimeTravel.scala │ │ │ │ │ ├── expressions/ │ │ │ │ │ │ └── aggregation/ │ │ │ │ │ │ └── BitmapAggregator.scala │ │ │ │ │ └── plans/ │ │ │ │ │ └── logical/ │ │ │ │ │ ├── CloneTableStatement.scala │ │ │ │ │ ├── DeltaDelete.scala │ │ │ │ │ ├── DeltaUpdateTable.scala │ │ │ │ │ ├── RestoreTableStatement.scala │ │ │ │ │ ├── SyncIdentity.scala │ │ │ │ │ ├── deltaConstraints.scala │ │ │ │ │ ├── deltaMerge.scala │ │ │ │ │ └── deltaTableFeatures.scala │ │ │ │ ├── delta/ │ │ │ │ │ ├── AllowedUserProvidedExpressions.scala │ │ │ │ │ ├── CheckUnresolvedRelationTimeTravel.scala │ │ │ │ │ ├── CheckpointProvider.scala │ │ │ │ │ ├── Checkpoints.scala │ │ │ │ │ ├── Checksum.scala │ │ │ │ │ ├── ClassicColumnConversions.scala │ │ │ │ │ ├── ColumnWithDefaultExprUtils.scala │ │ │ │ │ ├── CommittedTransaction.scala │ │ │ │ │ ├── ConcurrencyHelpers.scala │ │ │ │ │ ├── ConflictChecker.scala │ │ │ │ │ ├── ConflictCheckerPredicateElimination.scala │ │ │ │ │ ├── DataFrameUtils.scala │ │ │ │ │ ├── DefaultRowCommitVersion.scala │ │ │ │ │ ├── DeltaAnalysis.scala │ │ │ │ │ ├── DeltaColumnMapping.scala │ │ │ │ │ ├── DeltaCommitTag.scala │ │ │ │ │ ├── DeltaConfig.scala │ │ │ │ │ ├── DeltaErrors.scala │ │ │ │ │ ├── DeltaFileFormat.scala │ │ │ │ │ ├── DeltaFileProviderUtils.scala │ │ │ │ │ ├── DeltaHistoryManager.scala │ │ │ │ │ ├── DeltaLog.scala │ │ │ │ │ ├── DeltaLogFileIndex.scala │ │ │ │ │ ├── DeltaMergeActionResolver.scala │ │ │ │ │ ├── DeltaOperations.scala │ │ │ │ │ ├── DeltaOptions.scala │ │ │ │ │ ├── DeltaParquetFileFormat.scala │ │ │ │ │ ├── DeltaParquetWriteSupport.scala │ │ │ │ │ ├── DeltaSharedExceptions.scala │ │ │ │ │ ├── DeltaTable.scala │ │ │ │ │ ├── DeltaTableIdentifier.scala │ │ │ │ │ ├── DeltaTableValueFunctions.scala │ │ │ │ │ ├── DeltaThrowable.scala │ │ │ │ │ ├── DeltaThrowableHelper.scala │ │ │ │ │ ├── DeltaTimeTravelSpec.scala │ │ │ │ │ ├── DeltaUDF.scala │ │ │ │ │ ├── DeltaUnsupportedOperationsCheck.scala │ │ │ │ │ ├── DeltaViewHelper.scala │ │ │ │ │ ├── DomainMetadataUtils.scala │ │ │ │ │ ├── FallbackToV1Relations.scala │ │ │ │ │ ├── FileMetadataMaterializationTracker.scala │ │ │ │ │ ├── GenerateIdentityValues.scala │ │ │ │ │ ├── GenerateRowIDs.scala │ │ │ │ │ ├── GeneratedColumn.scala │ │ │ │ │ ├── IcebergCompat.scala │ │ │ │ │ ├── IdentityColumn.scala │ │ │ │ │ ├── JsonMetadataDomain.scala │ │ │ │ │ ├── LastCheckpointInfo.scala │ │ │ │ │ ├── MaterializedRowTrackingColumn.scala │ │ │ │ │ ├── MetadataCleanup.scala │ │ │ │ │ ├── NumRecordsStats.scala │ │ │ │ │ ├── OptimisticTransaction.scala │ │ │ │ │ ├── PostHocResolveUpCast.scala │ │ │ │ │ ├── PreDowngradeTableFeatureCommand.scala │ │ │ │ │ ├── PreprocessTableDelete.scala │ │ │ │ │ ├── PreprocessTableMerge.scala │ │ │ │ │ ├── PreprocessTableUpdate.scala │ │ │ │ │ ├── PreprocessTableWithDVs.scala │ │ │ │ │ ├── PreprocessTableWithDVsStrategy.scala │ │ │ │ │ ├── PreprocessTimeTravel.scala │ │ │ │ │ ├── ProtocolMetadataAdapter.scala │ │ │ │ │ ├── ProtocolMetadataAdapterV1.scala │ │ │ │ │ ├── ProvidesUniFormConverters.scala │ │ │ │ │ ├── ResolveDeltaMergeInto.scala │ │ │ │ │ ├── ResolveDeltaPathTable.scala │ │ │ │ │ ├── ResolveDeltaTableWithPartitionFilters.scala │ │ │ │ │ ├── RowCommitVersion.scala │ │ │ │ │ ├── RowId.scala │ │ │ │ │ ├── RowTracking.scala │ │ │ │ │ ├── Snapshot.scala │ │ │ │ │ ├── SnapshotManagement.scala │ │ │ │ │ ├── SnapshotState.scala │ │ │ │ │ ├── SubqueryTransformerHelper.scala │ │ │ │ │ ├── TableFeature.scala │ │ │ │ │ ├── ThreadStorageExecutionObserver.scala │ │ │ │ │ ├── TransactionExecutionObserver.scala │ │ │ │ │ ├── TypeWidening.scala │ │ │ │ │ ├── TypeWideningMetadata.scala │ │ │ │ │ ├── TypeWideningMode.scala │ │ │ │ │ ├── UniversalFormat.scala │ │ │ │ │ ├── UpdateExpressionsSupport.scala │ │ │ │ │ ├── actions/ │ │ │ │ │ │ ├── DeletionVectorDescriptor.scala │ │ │ │ │ │ ├── InMemoryLogReplay.scala │ │ │ │ │ │ ├── LogReplay.scala │ │ │ │ │ │ ├── TableFeatureSupport.scala │ │ │ │ │ │ └── actions.scala │ │ │ │ │ ├── catalog/ │ │ │ │ │ │ ├── AbstractDeltaCatalog.scala │ │ │ │ │ │ ├── CatalogResolver.scala │ │ │ │ │ │ ├── DeltaTableV2.scala │ │ │ │ │ │ └── IcebergTablePlaceHolder.scala │ │ │ │ │ ├── clustering/ │ │ │ │ │ │ └── ClusteringMetadataDomain.scala │ │ │ │ │ ├── commands/ │ │ │ │ │ │ ├── CloneTableBase.scala │ │ │ │ │ │ ├── CloneTableCommand.scala │ │ │ │ │ │ ├── ConvertToDeltaCommand.scala │ │ │ │ │ │ ├── CreateDeltaTableCommand.scala │ │ │ │ │ │ ├── CreateDeltaTableLike.scala │ │ │ │ │ │ ├── DMLUtils.scala │ │ │ │ │ │ ├── DMLWithDeletionVectorsHelper.scala │ │ │ │ │ │ ├── DeleteCommand.scala │ │ │ │ │ │ ├── DeletionVectorUtils.scala │ │ │ │ │ │ ├── DeltaCommand.scala │ │ │ │ │ │ ├── DeltaCommandInvariants.scala │ │ │ │ │ │ ├── DeltaGenerateCommand.scala │ │ │ │ │ │ ├── DeltaReorgTableCommand.scala │ │ │ │ │ │ ├── DescribeDeltaDetailsCommand.scala │ │ │ │ │ │ ├── DescribeDeltaHistoryCommand.scala │ │ │ │ │ │ ├── MergeIntoCommand.scala │ │ │ │ │ │ ├── MergeIntoCommandBase.scala │ │ │ │ │ │ ├── OptimizeTableCommand.scala │ │ │ │ │ │ ├── OptimizeTableStrategy.scala │ │ │ │ │ │ ├── ReorgTableForUpgradeUniformHelper.scala │ │ │ │ │ │ ├── ReorgTableHelper.scala │ │ │ │ │ │ ├── RestoreTableCommand.scala │ │ │ │ │ │ ├── ShowDeltaTableColumnsCommand.scala │ │ │ │ │ │ ├── UpdateCommand.scala │ │ │ │ │ │ ├── VacuumCommand.scala │ │ │ │ │ │ ├── WriteIntoDelta.scala │ │ │ │ │ │ ├── WriteIntoDeltaLike.scala │ │ │ │ │ │ ├── alterDeltaTableCommands.scala │ │ │ │ │ │ ├── backfill/ │ │ │ │ │ │ │ ├── BackfillBatch.scala │ │ │ │ │ │ │ ├── BackfillBatchStats.scala │ │ │ │ │ │ │ ├── BackfillCommand.scala │ │ │ │ │ │ │ ├── BackfillCommandStats.scala │ │ │ │ │ │ │ ├── BackfillExecutionObserver.scala │ │ │ │ │ │ │ ├── BackfillExecutor.scala │ │ │ │ │ │ │ ├── RowTrackingBackfillBatch.scala │ │ │ │ │ │ │ ├── RowTrackingBackfillCommand.scala │ │ │ │ │ │ │ ├── RowTrackingBackfillExecutor.scala │ │ │ │ │ │ │ ├── RowTrackingUnBackfillBatch.scala │ │ │ │ │ │ │ ├── RowTrackingUnBackfillCommand.scala │ │ │ │ │ │ │ └── RowTrackingUnBackfillExecutor.scala │ │ │ │ │ │ ├── cdc/ │ │ │ │ │ │ │ ├── CDCReader.scala │ │ │ │ │ │ │ └── CDCReaderBase.scala │ │ │ │ │ │ ├── columnmapping/ │ │ │ │ │ │ │ └── RemoveColumnMappingCommand.scala │ │ │ │ │ │ ├── convert/ │ │ │ │ │ │ │ ├── ConvertUtils.scala │ │ │ │ │ │ │ ├── ParquetFileManifest.scala │ │ │ │ │ │ │ ├── ParquetTable.scala │ │ │ │ │ │ │ └── interfaces.scala │ │ │ │ │ │ ├── merge/ │ │ │ │ │ │ │ ├── ClassicMergeExecutor.scala │ │ │ │ │ │ │ ├── InsertOnlyMergeExecutor.scala │ │ │ │ │ │ │ ├── MergeIntoMaterializeSource.scala │ │ │ │ │ │ │ ├── MergeOutputGeneration.scala │ │ │ │ │ │ │ └── MergeStats.scala │ │ │ │ │ │ └── optimize/ │ │ │ │ │ │ ├── AddFileWithNumRecords.scala │ │ │ │ │ │ ├── OptimizeStats.scala │ │ │ │ │ │ ├── ZCubeFileStatsCollector.scala │ │ │ │ │ │ └── ZOrderMetrics.scala │ │ │ │ │ ├── constraints/ │ │ │ │ │ │ ├── CharVarcharConstraint.scala │ │ │ │ │ │ ├── CheckDeltaInvariant.scala │ │ │ │ │ │ ├── Constraints.scala │ │ │ │ │ │ ├── DeltaInvariantCheckerExec.scala │ │ │ │ │ │ ├── ExpressionLogicalPlanWrapper.scala │ │ │ │ │ │ ├── Invariants.scala │ │ │ │ │ │ └── tableChanges.scala │ │ │ │ │ ├── coordinatedcommits/ │ │ │ │ │ │ ├── AbstractBatchBackfillingCommitCoordinatorClient.scala │ │ │ │ │ │ ├── CommitCoordinatorClient.scala │ │ │ │ │ │ ├── CoordinatedCommitsUsageLogs.scala │ │ │ │ │ │ ├── CoordinatedCommitsUtils.scala │ │ │ │ │ │ ├── InMemoryCommitCoordinator.scala │ │ │ │ │ │ ├── InMemoryUCClient.scala │ │ │ │ │ │ ├── InMemoryUCCommitCoordinator.scala │ │ │ │ │ │ ├── TableCommitCoordinatorClient.scala │ │ │ │ │ │ └── UCCommitCoordinatorBuilder.scala │ │ │ │ │ ├── deletionvectors/ │ │ │ │ │ │ ├── RoaringBitmapArray.scala │ │ │ │ │ │ ├── RowIndexMarkingFilters.scala │ │ │ │ │ │ └── StoredBitmap.scala │ │ │ │ │ ├── expressions/ │ │ │ │ │ │ ├── DecodeNestedZ85EncodedVariant.scala │ │ │ │ │ │ ├── EncodeNestedVariantAsZ85String.scala │ │ │ │ │ │ ├── HilbertIndex.scala │ │ │ │ │ │ ├── HilbertStates.java │ │ │ │ │ │ ├── HilbertUtils.scala │ │ │ │ │ │ ├── InterleaveBits.scala │ │ │ │ │ │ ├── JoinedProjection.scala │ │ │ │ │ │ └── RangePartitionId.scala │ │ │ │ │ ├── files/ │ │ │ │ │ │ ├── CdcAddFileIndex.scala │ │ │ │ │ │ ├── DelayedCommitProtocol.scala │ │ │ │ │ │ ├── DeltaFileFormatWriter.scala │ │ │ │ │ │ ├── DeltaSourceSnapshot.scala │ │ │ │ │ │ ├── SQLMetricsReporting.scala │ │ │ │ │ │ ├── TahoeChangeFileIndex.scala │ │ │ │ │ │ ├── TahoeFileIndex.scala │ │ │ │ │ │ ├── TahoeRemoveFileIndex.scala │ │ │ │ │ │ └── TransactionalWrite.scala │ │ │ │ │ ├── fuzzer/ │ │ │ │ │ │ ├── AtomicBarrier.scala │ │ │ │ │ │ ├── ExecutionPhaseLock.scala │ │ │ │ │ │ ├── OptimisticTransactionPhases.scala │ │ │ │ │ │ ├── PhaseLockingExecutionObserver.scala │ │ │ │ │ │ └── PhaseLockingTransactionExecutionObserver.scala │ │ │ │ │ ├── hooks/ │ │ │ │ │ │ ├── AutoCompact.scala │ │ │ │ │ │ ├── AutoCompactUtils.scala │ │ │ │ │ │ ├── CheckpointHook.scala │ │ │ │ │ │ ├── ChecksumHook.scala │ │ │ │ │ │ ├── GenerateSymlinkManifest.scala │ │ │ │ │ │ ├── HudiConverterHook.scala │ │ │ │ │ │ ├── IcebergConverterHook.scala │ │ │ │ │ │ ├── PostCommitHook.scala │ │ │ │ │ │ └── UpdateCatalog.scala │ │ │ │ │ ├── implicits/ │ │ │ │ │ │ ├── RichSparkClasses.scala │ │ │ │ │ │ └── package.scala │ │ │ │ │ ├── isolationLevels.scala │ │ │ │ │ ├── logging/ │ │ │ │ │ │ └── DeltaLogKeys.scala │ │ │ │ │ ├── metering/ │ │ │ │ │ │ ├── DeltaLogging.scala │ │ │ │ │ │ └── ScanReport.scala │ │ │ │ │ ├── metric/ │ │ │ │ │ │ └── IncrementMetric.scala │ │ │ │ │ ├── optimizablePartitionExpressions.scala │ │ │ │ │ ├── optimizer/ │ │ │ │ │ │ └── RangePartitionIdRewrite.scala │ │ │ │ │ ├── perf/ │ │ │ │ │ │ ├── DeltaOptimizedWriterExec.scala │ │ │ │ │ │ └── OptimizeMetadataOnlyDeltaQuery.scala │ │ │ │ │ ├── redirect/ │ │ │ │ │ │ └── TableRedirect.scala │ │ │ │ │ ├── schema/ │ │ │ │ │ │ ├── ImplicitMetadataOperation.scala │ │ │ │ │ │ ├── InvariantViolationException.scala │ │ │ │ │ │ ├── SchemaMergingUtils.scala │ │ │ │ │ │ └── SchemaUtils.scala │ │ │ │ │ ├── serverSidePlanning/ │ │ │ │ │ │ ├── ServerSidePlannedTable.scala │ │ │ │ │ │ ├── ServerSidePlanningClient.scala │ │ │ │ │ │ ├── ServerSidePlanningMetadata.scala │ │ │ │ │ │ └── UnityCatalogMetadata.scala │ │ │ │ │ ├── skipping/ │ │ │ │ │ │ ├── MultiDimClustering.scala │ │ │ │ │ │ ├── MultiDimClusteringFunctions.scala │ │ │ │ │ │ └── clustering/ │ │ │ │ │ │ ├── ClusteredTableUtils.scala │ │ │ │ │ │ ├── ClusteringColumn.scala │ │ │ │ │ │ ├── ClusteringStats.scala │ │ │ │ │ │ ├── ZCube.scala │ │ │ │ │ │ └── temp/ │ │ │ │ │ │ ├── AlterTableClusterBy.scala │ │ │ │ │ │ ├── ClusterBySpec.scala │ │ │ │ │ │ └── ClusterByTransform.scala │ │ │ │ │ ├── sources/ │ │ │ │ │ │ ├── DeltaDataSource.scala │ │ │ │ │ │ ├── DeltaSQLConf.scala │ │ │ │ │ │ ├── DeltaSink.scala │ │ │ │ │ │ ├── DeltaSource.scala │ │ │ │ │ │ ├── DeltaSourceCDCSupport.scala │ │ │ │ │ │ ├── DeltaSourceMetadataEvolutionSupport.scala │ │ │ │ │ │ ├── DeltaSourceMetadataTrackingLog.scala │ │ │ │ │ │ ├── DeltaSourceOffset.scala │ │ │ │ │ │ ├── DeltaSourceUtils.scala │ │ │ │ │ │ ├── DeltaStreamUtils.scala │ │ │ │ │ │ └── limits.scala │ │ │ │ │ ├── stats/ │ │ │ │ │ │ ├── ArrayAccumulator.scala │ │ │ │ │ │ ├── AutoCompactPartitionStats.scala │ │ │ │ │ │ ├── DataSkippingPredicateBuilder.scala │ │ │ │ │ │ ├── DataSkippingReader.scala │ │ │ │ │ │ ├── DataSkippingStatsTracker.scala │ │ │ │ │ │ ├── DeletedRecordCountsHistogram.scala │ │ │ │ │ │ ├── DeletedRecordCountsHistogramUtils.scala │ │ │ │ │ │ ├── DeltaScan.scala │ │ │ │ │ │ ├── DeltaScanGenerator.scala │ │ │ │ │ │ ├── FileSizeHistogram.scala │ │ │ │ │ │ ├── FileStatsHistogram.scala │ │ │ │ │ │ ├── PrepareDeltaScan.scala │ │ │ │ │ │ ├── ReadsMetadataFields.scala │ │ │ │ │ │ ├── StatisticsCollection.scala │ │ │ │ │ │ ├── StatsCollectionUtils.scala │ │ │ │ │ │ └── StatsProvider.scala │ │ │ │ │ ├── storage/ │ │ │ │ │ │ ├── AzureLogStore.scala │ │ │ │ │ │ ├── ClosableIterator.scala │ │ │ │ │ │ ├── DelegatingLogStore.scala │ │ │ │ │ │ ├── HDFSLogStore.scala │ │ │ │ │ │ ├── HadoopFileSystemLogStore.scala │ │ │ │ │ │ ├── LineClosableIterator.scala │ │ │ │ │ │ ├── LocalLogStore.scala │ │ │ │ │ │ ├── LogStore.scala │ │ │ │ │ │ ├── S3SingleDriverLogStore.scala │ │ │ │ │ │ └── dv/ │ │ │ │ │ │ └── DeletionVectorStore.scala │ │ │ │ │ ├── streaming/ │ │ │ │ │ │ └── SchemaTrackingLog.scala │ │ │ │ │ ├── tablefeatures/ │ │ │ │ │ │ └── tableChanges.scala │ │ │ │ │ ├── uniform/ │ │ │ │ │ │ └── ParquetIcebergCompatV2Utils.scala │ │ │ │ │ ├── util/ │ │ │ │ │ │ ├── AnalysisHelper.scala │ │ │ │ │ │ ├── BinPackingIterator.scala │ │ │ │ │ │ ├── BinPackingUtils.scala │ │ │ │ │ │ ├── Codec.scala │ │ │ │ │ │ ├── DatasetRefCache.scala │ │ │ │ │ │ ├── DateFormatter.scala │ │ │ │ │ │ ├── DateTimeFormatterHelper.scala │ │ │ │ │ │ ├── DateTimeUtils.scala │ │ │ │ │ │ ├── DeltaCommitFileProvider.scala │ │ │ │ │ │ ├── DeltaEncoders.scala │ │ │ │ │ │ ├── DeltaFileOperations.scala │ │ │ │ │ │ ├── DeltaLogGroupingIterator.scala │ │ │ │ │ │ ├── DeltaProgressReporter.scala │ │ │ │ │ │ ├── DeltaSparkPlanUtils.scala │ │ │ │ │ │ ├── DeltaSqlParserUtils.scala │ │ │ │ │ │ ├── DeltaStatsJsonUtils.scala │ │ │ │ │ │ ├── FileNames.scala │ │ │ │ │ │ ├── InCommitTimestampUtils.scala │ │ │ │ │ │ ├── JsonUtils.scala │ │ │ │ │ │ ├── PartitionUtils.scala │ │ │ │ │ │ ├── PathWithFileSystem.scala │ │ │ │ │ │ ├── SetAccumulator.scala │ │ │ │ │ │ ├── StateCache.scala │ │ │ │ │ │ ├── TimestampFormatter.scala │ │ │ │ │ │ ├── TransactionHelper.scala │ │ │ │ │ │ ├── Utils.scala │ │ │ │ │ │ └── threads/ │ │ │ │ │ │ ├── DeltaThreadPool.scala │ │ │ │ │ │ └── SparkThreadLocalForwardingThreadPoolExecutor.scala │ │ │ │ │ ├── v2/ │ │ │ │ │ │ └── interop/ │ │ │ │ │ │ ├── AbstractCommitInfo.scala │ │ │ │ │ │ ├── AbstractMetadata.scala │ │ │ │ │ │ ├── AbstractProtocol.scala │ │ │ │ │ │ └── README.md │ │ │ │ │ └── zorder/ │ │ │ │ │ └── ZCubeInfo.scala │ │ │ │ └── util/ │ │ │ │ └── ScalaExtensions.scala │ │ │ └── scala-shims/ │ │ │ ├── spark-4.0/ │ │ │ │ ├── CreateDeltaTableLikeShims.scala │ │ │ │ ├── DataSourceV2RelationShims.scala │ │ │ │ ├── DateTimeExpressionShims.scala │ │ │ │ ├── LogKeyShims.scala │ │ │ │ ├── ParquetFooterReaderShims.scala │ │ │ │ ├── ParseExceptionShims.scala │ │ │ │ ├── QualifiedColTypeShims.scala │ │ │ │ ├── RelocatedStreamingClassesShims.scala │ │ │ │ ├── SupportsV1OverwriteWithSaveAsTable.scala │ │ │ │ ├── VariantShreddingShims.scala │ │ │ │ └── ViewShims.scala │ │ │ ├── spark-4.1/ │ │ │ │ ├── CreateDeltaTableLikeShims.scala │ │ │ │ ├── DataSourceV2RelationShims.scala │ │ │ │ ├── DateTimeExpressionShims.scala │ │ │ │ ├── LogKeyShims.scala │ │ │ │ ├── ParquetFooterReaderShims.scala │ │ │ │ ├── ParseExceptionShims.scala │ │ │ │ ├── QualifiedColTypeShims.scala │ │ │ │ ├── RelocatedStreamingClassesShims.scala │ │ │ │ ├── VariantShreddingShims.scala │ │ │ │ └── ViewShims.scala │ │ │ └── spark-4.2/ │ │ │ ├── CreateDeltaTableLikeShims.scala │ │ │ ├── DataSourceV2RelationShims.scala │ │ │ ├── DateTimeExpressionShims.scala │ │ │ ├── LogKeyShims.scala │ │ │ ├── ParquetFooterReaderShims.scala │ │ │ ├── ParseExceptionShims.scala │ │ │ ├── QualifiedColTypeShims.scala │ │ │ ├── RelocatedStreamingClassesShims.scala │ │ │ ├── VariantShreddingShims.scala │ │ │ └── ViewShims.scala │ │ └── test/ │ │ ├── java/ │ │ │ ├── io/ │ │ │ │ └── delta/ │ │ │ │ ├── sql/ │ │ │ │ │ └── JavaDeltaSparkSessionExtensionSuite.java │ │ │ │ └── tables/ │ │ │ │ ├── JavaDeltaTableBuilderSuite.java │ │ │ │ └── JavaDeltaTableSuite.java │ │ │ └── org/ │ │ │ └── apache/ │ │ │ └── spark/ │ │ │ └── sql/ │ │ │ └── delta/ │ │ │ ├── DeleteJavaSuite.java │ │ │ ├── DeltaSQLCommandJavaTest.java │ │ │ ├── MergeIntoJavaSuite.java │ │ │ ├── UpdateJavaSuite.java │ │ │ └── util/ │ │ │ └── CatalogTableUtilsTest.java │ │ ├── resources/ │ │ │ ├── delta/ │ │ │ │ ├── dbr_8_0_non_generated_columns/ │ │ │ │ │ ├── .part-00000-74e02f0d-e727-46e5-8d74-779d2abd616e-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ ├── 00000000000000000000.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ └── part-00000-74e02f0d-e727-46e5-8d74-779d2abd616e-c000.snappy.parquet │ │ │ │ ├── dbr_8_1_generated_columns/ │ │ │ │ │ └── _delta_log/ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ ├── 00000000000000000000.crc │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ ├── delta-0.1.0/ │ │ │ │ │ ├── .part-00000-348d7f43-38f6-4778-88c7-45f379471c49-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-f4aeebd0-a689-4e1b-bc7a-bbb0ec59dce5-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-6d252218-2632-416e-9e46-f32316ec314a-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-f1cb1cf9-7a73-439c-b0ea-dcba5c2280a6-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ │ ├── .00000000000000000002.json.crc │ │ │ │ │ │ ├── .00000000000000000003.checkpoint.parquet.crc │ │ │ │ │ │ ├── .00000000000000000003.json.crc │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ ├── 00000000000000000003.checkpoint.parquet │ │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ │ └── _last_checkpoint │ │ │ │ │ ├── id=4/ │ │ │ │ │ │ ├── .part-00001-36c738bf-7836-479b-9cc1-7a4934207856.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00001-36c738bf-7836-479b-9cc1-7a4934207856.c000.snappy.parquet │ │ │ │ │ ├── id=5/ │ │ │ │ │ │ ├── .part-00000-f1e0b560-ca00-409e-a274-f1ab264bc412.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00000-f1e0b560-ca00-409e-a274-f1ab264bc412.c000.snappy.parquet │ │ │ │ │ ├── id=6/ │ │ │ │ │ │ ├── .part-00000-adb59f54-6b8f-4bfd-9915-ae26bd0f0e2c.c000.snappy.parquet.crc │ │ │ │ │ │ └── part-00000-adb59f54-6b8f-4bfd-9915-ae26bd0f0e2c.c000.snappy.parquet │ │ │ │ │ ├── part-00000-348d7f43-38f6-4778-88c7-45f379471c49-c000.snappy.parquet │ │ │ │ │ ├── part-00000-f4aeebd0-a689-4e1b-bc7a-bbb0ec59dce5-c000.snappy.parquet │ │ │ │ │ ├── part-00001-6d252218-2632-416e-9e46-f32316ec314a-c000.snappy.parquet │ │ │ │ │ └── part-00001-f1cb1cf9-7a73-439c-b0ea-dcba5c2280a6-c000.snappy.parquet │ │ │ │ ├── delta-1.2.1/ │ │ │ │ │ ├── .part-00000-59316e80-0f6c-491a-9716-5e0419434e46-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-635b7994-d3f9-4623-b032-8a9c8a7ca5b9-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-87624dd4-c6dc-4163-a4e6-0e50caa28760-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-e107d259-11d5-4e5b-b472-62daa676743b-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-91d10124-a73d-42c2-9ef0-75ed41ca73d8-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00002-dca394a5-9d0a-4630-a90a-a8f7f675e4e4-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ │ ├── .00000000000000000002.checkpoint.parquet.crc │ │ │ │ │ │ ├── .00000000000000000002.json.crc │ │ │ │ │ │ ├── .00000000000000000003.json.crc │ │ │ │ │ │ ├── .00000000000000000004.checkpoint.parquet.crc │ │ │ │ │ │ ├── .00000000000000000004.json.crc │ │ │ │ │ │ ├── ._last_checkpoint.crc │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.checkpoint.parquet │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ │ ├── 00000000000000000004.checkpoint.parquet │ │ │ │ │ │ ├── 00000000000000000004.json │ │ │ │ │ │ └── _last_checkpoint │ │ │ │ │ ├── part-00000-59316e80-0f6c-491a-9716-5e0419434e46-c000.snappy.parquet │ │ │ │ │ ├── part-00000-635b7994-d3f9-4623-b032-8a9c8a7ca5b9-c000.snappy.parquet │ │ │ │ │ ├── part-00000-87624dd4-c6dc-4163-a4e6-0e50caa28760-c000.snappy.parquet │ │ │ │ │ ├── part-00000-e107d259-11d5-4e5b-b472-62daa676743b-c000.snappy.parquet │ │ │ │ │ ├── part-00001-91d10124-a73d-42c2-9ef0-75ed41ca73d8-c000.snappy.parquet │ │ │ │ │ └── part-00002-dca394a5-9d0a-4630-a90a-a8f7f675e4e4-c000.snappy.parquet │ │ │ │ ├── history/ │ │ │ │ │ └── delta-0.2.0/ │ │ │ │ │ ├── .part-00000-512e1537-8aaa-4193-b8b4-bef3de0de409-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-7c2deba3-1994-4fb8-bc07-d46c948aa415-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-b44fcdb0-8b06-4f3a-8606-f8311a96f6dc-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-cb6b150b-30b8-4662-ad28-ff32ddab96d2-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-185eca06-e017-4dea-ae49-fc48b973e37e-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-4327c977-2734-4477-9507-7ccf67924649-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-c373a5bd-85f0-4758-815e-7eb62007a15c-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── ..00000000000000000000.json.c6b312ca-665d-46ab-93a9-9f87ad2baa92.tmp.crc │ │ │ │ │ │ ├── ..00000000000000000001.json.641a776e-6e56-4423-a9b0-7efc9e58826a.tmp.crc │ │ │ │ │ │ ├── ..00000000000000000002.json.e64807e6-437c-44c9-abd2-50e6514d236e.tmp.crc │ │ │ │ │ │ ├── ..00000000000000000003.json.b374eda7-fa09-48ce-b06c-56025163f6ae.tmp.crc │ │ │ │ │ │ ├── .._last_checkpoint.477ba875-7a14-4e57-9973-1349c21a152c.tmp.crc │ │ │ │ │ │ ├── .00000000000000000003.checkpoint.parquet.crc │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ ├── 00000000000000000003.checkpoint.parquet │ │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ │ └── _last_checkpoint │ │ │ │ │ ├── part-00000-512e1537-8aaa-4193-b8b4-bef3de0de409-c000.snappy.parquet │ │ │ │ │ ├── part-00000-7c2deba3-1994-4fb8-bc07-d46c948aa415-c000.snappy.parquet │ │ │ │ │ ├── part-00000-b44fcdb0-8b06-4f3a-8606-f8311a96f6dc-c000.snappy.parquet │ │ │ │ │ ├── part-00000-cb6b150b-30b8-4662-ad28-ff32ddab96d2-c000.snappy.parquet │ │ │ │ │ ├── part-00001-185eca06-e017-4dea-ae49-fc48b973e37e-c000.snappy.parquet │ │ │ │ │ ├── part-00001-4327c977-2734-4477-9507-7ccf67924649-c000.snappy.parquet │ │ │ │ │ └── part-00001-c373a5bd-85f0-4758-815e-7eb62007a15c-c000.snappy.parquet │ │ │ │ ├── identity_test_written_by_version_5/ │ │ │ │ │ ├── .part-00000-1ec4087c-3109-48b4-9e1c-c44cad50f3d8-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-77d98c61-0299-4a5a-b68d-305cab1a46f6-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.crc │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ ├── part-00000-1ec4087c-3109-48b4-9e1c-c44cad50f3d8-c000.snappy.parquet │ │ │ │ │ └── part-00001-77d98c61-0299-4a5a-b68d-305cab1a46f6-c000.snappy.parquet │ │ │ │ ├── partitioned-table-with-dv-large/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ │ └── 00000000000000000004.json │ │ │ │ │ ├── partCol=0/ │ │ │ │ │ │ └── part-00000-757a3870-38dd-41ac-86f1-e1e6826df6bc.c000.snappy.parquet │ │ │ │ │ ├── partCol=1/ │ │ │ │ │ │ └── part-00000-ffe81e1a-1a1f-4803-bc2a-e68f7b2ea122.c000.snappy.parquet │ │ │ │ │ ├── partCol=2/ │ │ │ │ │ │ └── part-00000-5963000f-3e52-4c43-a106-d7e527f5722a.c000.snappy.parquet │ │ │ │ │ ├── partCol=3/ │ │ │ │ │ │ ├── part-00000-068d9a17-0362-43f9-ad68-6bfcbd27448d.c000.snappy.parquet │ │ │ │ │ │ └── part-00000-8775b518-3470-41d4-8d7e-27596c48053e.c000.snappy.parquet │ │ │ │ │ ├── partCol=4/ │ │ │ │ │ │ └── part-00000-c66868e5-d1e0-4f22-ae89-9cc4d2a133fa.c000.snappy.parquet │ │ │ │ │ ├── partCol=5/ │ │ │ │ │ │ └── part-00000-70dbcf83-e5c0-4c91-8e1a-be86f08b98f4.c000.snappy.parquet │ │ │ │ │ ├── partCol=6/ │ │ │ │ │ │ ├── part-00000-2dee959e-3d92-4c43-ac01-24d888ba82fd.c000.snappy.parquet │ │ │ │ │ │ └── part-00000-34e763ec-3291-4cd0-9b90-fd2d24c68098.c000.snappy.parquet │ │ │ │ │ ├── partCol=7/ │ │ │ │ │ │ ├── part-00000-156df4a5-759c-4b9f-82b1-9727a62b7990.c000.snappy.parquet │ │ │ │ │ │ └── part-00000-f43c32e8-3996-43ae-9b14-9b7f8fec6221.c000.snappy.parquet │ │ │ │ │ ├── partCol=8/ │ │ │ │ │ │ ├── part-00000-a1137e9e-5425-4589-b039-84378f061fc4.c000.snappy.parquet │ │ │ │ │ │ └── part-00000-fe120a67-87dc-4997-8811-3ad9d8dc3743.c000.snappy.parquet │ │ │ │ │ └── partCol=9/ │ │ │ │ │ └── part-00000-6bcf7302-8e23-4613-aec2-02856f8f1d05.c000.snappy.parquet │ │ │ │ ├── table-with-dv-gigantic/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ └── part-00000-2bc940f0-dd3f-461d-8581-136026bf6f95-c000.snappy.parquet │ │ │ │ ├── table-with-dv-large/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ │ └── 00000000000000000004.json │ │ │ │ │ ├── part-00000-51219d56-88a7-41cc-be5d-eada75aceb4f-c000.snappy.parquet │ │ │ │ │ ├── part-00000-7c52eadd-8da7-4782-a5d5-621cd92cab11-c000.snappy.parquet │ │ │ │ │ ├── part-00000-f5c18e7b-d1bf-4ba5-85dd-e63ddc5931bf-c000.snappy.parquet │ │ │ │ │ ├── part-00001-5dbf0ba2-220a-4770-8e26-18a77cf875f0-c000.snappy.parquet │ │ │ │ │ ├── part-00002-5459a52f-3fd3-4b79-83a6-e7f57db28650-c000.snappy.parquet │ │ │ │ │ ├── part-00003-0e842060-9e04-4896-ba21-029309ab8736-c000.snappy.parquet │ │ │ │ │ ├── part-00004-a72dbdec-2d0e-43d8-a756-4d0d63ef9fcb-c000.snappy.parquet │ │ │ │ │ ├── part-00005-0972979f-852d-4f3e-8f64-bf0bf072de5f-c000.snappy.parquet │ │ │ │ │ ├── part-00006-227c6a1e-0180-4feb-8816-19eccf7939f5-c000.snappy.parquet │ │ │ │ │ ├── part-00007-7c37e5e3-abb2-419e-8cba-eba4eeb3b11a-c000.snappy.parquet │ │ │ │ │ ├── part-00008-1a0b4375-bbcc-4f3c-8e51-ecb551c89430-c000.snappy.parquet │ │ │ │ │ ├── part-00009-52689115-1770-4f15-b98d-b942db5b7359-c000.snappy.parquet │ │ │ │ │ ├── part-00010-7f35fa1b-7993-4aff-8f60-2b76f1eb3f2c-c000.snappy.parquet │ │ │ │ │ ├── part-00011-fce7841f-be9a-43b8-b283-9e2308ef5487-c000.snappy.parquet │ │ │ │ │ ├── part-00012-9b83c213-31ff-4b2c-a5d9-be1a2bc2431d-c000.snappy.parquet │ │ │ │ │ ├── part-00013-c6b05dd2-0143-4e9f-a231-1a2d08a83a0e-c000.snappy.parquet │ │ │ │ │ ├── part-00014-41a4f51e-62cd-41f5-bb03-afba1e70ea29-c000.snappy.parquet │ │ │ │ │ ├── part-00015-f2f141bb-fa8f-4553-a5db-d1b8d682153b-c000.snappy.parquet │ │ │ │ │ ├── part-00016-d8f58ffc-8bff-4e12-b709-e628f9bf2553-c000.snappy.parquet │ │ │ │ │ ├── part-00017-45bac3c9-7eb8-42cb-bb51-fc5b4dd0be10-c000.snappy.parquet │ │ │ │ │ ├── part-00018-9d74a51b-b800-4e4d-a258-738e585a78a5-c000.snappy.parquet │ │ │ │ │ └── part-00019-a9bb3ce8-afba-47ec-8451-13edcd855b15-c000.snappy.parquet │ │ │ │ ├── table-with-dv-small/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ └── 00000000000000000001.json │ │ │ │ │ └── r4/ │ │ │ │ │ └── part-00000-5521fc5e-6e49-4437-8b2d-ce6a1a94a34a-c000.snappy.parquet │ │ │ │ ├── table-with-dv-special-char/ │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ │ └── part-00000-8d24f407-08d3-49ab-9d1c-f7f6c129e882-c000.snappy.parquet │ │ │ │ ├── transaction_log_schema_evolvability/ │ │ │ │ │ ├── .part-00000-9f483b95-3ea3-44f0-b54d-73199574be15-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-dfb1dd9a-0fe2-420e-81d5-a84004aebcee-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00000-f654b1f4-e1ea-40e5-a8cd-452f7c3359d8-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-bfb08fc5-c967-40e4-a646-c8178d8b5e21-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-d1030238-b55d-48f8-a4d6-89ef12e9d501-c000.snappy.parquet.crc │ │ │ │ │ ├── .part-00001-d5da9c60-a615-4065-a3cb-4796d86fc797-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000003.json.crc │ │ │ │ │ │ ├── .00000000000000000004.json.crc │ │ │ │ │ │ ├── .00000000000000000005.json.crc │ │ │ │ │ │ ├── ._last_checkpoint.crc │ │ │ │ │ │ ├── 00000000000000000000.crc │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.crc │ │ │ │ │ │ ├── 00000000000000000001.json │ │ │ │ │ │ ├── 00000000000000000002.checkpoint.parquet │ │ │ │ │ │ ├── 00000000000000000002.crc │ │ │ │ │ │ ├── 00000000000000000002.json │ │ │ │ │ │ ├── 00000000000000000003.crc │ │ │ │ │ │ ├── 00000000000000000003.json │ │ │ │ │ │ ├── 00000000000000000004.json │ │ │ │ │ │ ├── 00000000000000000005.json │ │ │ │ │ │ └── _last_checkpoint │ │ │ │ │ ├── part-00000-9f483b95-3ea3-44f0-b54d-73199574be15-c000.snappy.parquet │ │ │ │ │ ├── part-00000-dfb1dd9a-0fe2-420e-81d5-a84004aebcee-c000.snappy.parquet │ │ │ │ │ ├── part-00000-f654b1f4-e1ea-40e5-a8cd-452f7c3359d8-c000.snappy.parquet │ │ │ │ │ ├── part-00001-bfb08fc5-c967-40e4-a646-c8178d8b5e21-c000.snappy.parquet │ │ │ │ │ ├── part-00001-d1030238-b55d-48f8-a4d6-89ef12e9d501-c000.snappy.parquet │ │ │ │ │ └── part-00001-d5da9c60-a615-4065-a3cb-4796d86fc797-c000.snappy.parquet │ │ │ │ ├── variant-stats-no-checkpoint/ │ │ │ │ │ ├── .part-00000-20135f43-a68e-4348-9a46-e6eeed704c0e-c000.snappy.parquet.crc │ │ │ │ │ ├── _delta_log/ │ │ │ │ │ │ ├── .00000000000000000000.crc.crc │ │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ │ ├── .00000000000000000001.crc.crc │ │ │ │ │ │ ├── .00000000000000000001.json.crc │ │ │ │ │ │ ├── 00000000000000000000.crc │ │ │ │ │ │ ├── 00000000000000000000.json │ │ │ │ │ │ ├── 00000000000000000001.crc │ │ │ │ │ │ └── 00000000000000000001.json │ │ │ │ │ └── part-00000-20135f43-a68e-4348-9a46-e6eeed704c0e-c000.snappy.parquet │ │ │ │ └── variant-stats-state-reconstruction/ │ │ │ │ ├── .part-00031-b7a56bf1-1672-47dc-b8e2-255d62f630ee-c000.snappy.parquet.crc │ │ │ │ ├── _delta_log/ │ │ │ │ │ ├── .00000000000000000000.crc.crc │ │ │ │ │ ├── .00000000000000000000.json.crc │ │ │ │ │ ├── 00000000000000000000.crc │ │ │ │ │ └── 00000000000000000000.json │ │ │ │ └── part-00031-b7a56bf1-1672-47dc-b8e2-255d62f630ee-c000.snappy.parquet │ │ │ ├── hms/ │ │ │ │ ├── README.md │ │ │ │ └── hive-schema-3.1.0.derby.sql │ │ │ └── log4j2.properties │ │ ├── scala/ │ │ │ ├── io/ │ │ │ │ └── delta/ │ │ │ │ ├── exceptions/ │ │ │ │ │ └── DeltaConcurrentExceptionsSuite.scala │ │ │ │ ├── sql/ │ │ │ │ │ ├── DeltaExtensionAndCatalogSuite.scala │ │ │ │ │ └── parser/ │ │ │ │ │ └── DeltaSqlParserSuite.scala │ │ │ │ └── tables/ │ │ │ │ ├── DeltaTableBuilderSuite.scala │ │ │ │ ├── DeltaTableForNameSuite.scala │ │ │ │ ├── DeltaTableSuite.scala │ │ │ │ └── DeltaTableTestUtils.scala │ │ │ └── org/ │ │ │ └── apache/ │ │ │ └── spark/ │ │ │ └── sql/ │ │ │ └── delta/ │ │ │ ├── ActionSerializerSuite.scala │ │ │ ├── AutoCompactSuite.scala │ │ │ ├── BlockWritesLocalFileSystem.scala │ │ │ ├── CheckCDCAnswer.scala │ │ │ ├── CheckpointInstanceSuite.scala │ │ │ ├── CheckpointProtectionTestUtilsMixin.scala │ │ │ ├── CheckpointProviderSuite.scala │ │ │ ├── CheckpointsSuite.scala │ │ │ ├── ChecksumDVMetricsSuite.scala │ │ │ ├── ChecksumSuite.scala │ │ │ ├── CloneParquetSuite.scala │ │ │ ├── CloneParquetSuiteBase.scala │ │ │ ├── CloneTableSQLSuite.scala │ │ │ ├── CloneTableScalaSuite.scala │ │ │ ├── CloneTableSuiteBase.scala │ │ │ ├── CloneTableTestMixin.scala │ │ │ ├── CommitInfoSerializerSuite.scala │ │ │ ├── CommitSanityCheckSuite.scala │ │ │ ├── ConflictCheckerPredicateEliminationUnitSuite.scala │ │ │ ├── ConflictResolutionTestUtils.scala │ │ │ ├── ConvertToDeltaSQLSuite.scala │ │ │ ├── ConvertToDeltaScalaSuite.scala │ │ │ ├── ConvertToDeltaSuiteBase.scala │ │ │ ├── CustomCatalogSuite.scala │ │ │ ├── DDLTestUtils.scala │ │ │ ├── DelegatingLogStoreSuite.scala │ │ │ ├── DeleteMetricsSuite.scala │ │ │ ├── DeleteSQLSuite.scala │ │ │ ├── DeleteScalaSuite.scala │ │ │ ├── DeleteSuiteBase.scala │ │ │ ├── DeletionVectorsTestUtils.scala │ │ │ ├── DeltaAllFilesInCrcSuite.scala │ │ │ ├── DeltaAlterTableReplaceTests.scala │ │ │ ├── DeltaAlterTableTests.scala │ │ │ ├── DeltaArbitraryColumnNameSuite.scala │ │ │ ├── DeltaCDCColumnMappingSuite.scala │ │ │ ├── DeltaCDCSQLSuite.scala │ │ │ ├── DeltaCDCStreamSuite.scala │ │ │ ├── DeltaCDCSuite.scala │ │ │ ├── DeltaCheckpointWithStructColsSuite.scala │ │ │ ├── DeltaColumnMappingSuite.scala │ │ │ ├── DeltaColumnMappingTestUtils.scala │ │ │ ├── DeltaColumnRenameSuite.scala │ │ │ ├── DeltaCommitLockSuite.scala │ │ │ ├── DeltaConfigSuite.scala │ │ │ ├── DeltaCreateTableLikeSuite.scala │ │ │ ├── DeltaDDLSuite.scala │ │ │ ├── DeltaDDLUsingPathSuite.scala │ │ │ ├── DeltaDataFrameHadoopOptionsSuite.scala │ │ │ ├── DeltaDataFrameWriterV2Suite.scala │ │ │ ├── DeltaDropColumnSuite.scala │ │ │ ├── DeltaErrorsSuite.scala │ │ │ ├── DeltaFastDropFeatureSuite.scala │ │ │ ├── DeltaGenerateSymlinkManifestSuite.scala │ │ │ ├── DeltaHistoryManagerSuite.scala │ │ │ ├── DeltaImplicitsSuite.scala │ │ │ ├── DeltaIncrementalSetTransactionsSuite.scala │ │ │ ├── DeltaInsertIntoColumnOrderSuite.scala │ │ │ ├── DeltaInsertIntoImplicitCastSuite.scala │ │ │ ├── DeltaInsertIntoMissingColumnSuite.scala │ │ │ ├── DeltaInsertIntoSchemaEvolutionSuite.scala │ │ │ ├── DeltaInsertIntoTableSuite.scala │ │ │ ├── DeltaInsertIntoTest.scala │ │ │ ├── DeltaLimitPushDownSuite.scala │ │ │ ├── DeltaLogMinorCompactionSuite.scala │ │ │ ├── DeltaLogSuite.scala │ │ │ ├── DeltaMetricsUtils.scala │ │ │ ├── DeltaNotSupportedDDLSuite.scala │ │ │ ├── DeltaOptionSuite.scala │ │ │ ├── DeltaParquetFileFormatSuite.scala │ │ │ ├── DeltaProtocolTransitionsSuite.scala │ │ │ ├── DeltaProtocolVersionSuite.scala │ │ │ ├── DeltaRestartSessionSuite.scala │ │ │ ├── DeltaRetentionSuite.scala │ │ │ ├── DeltaRetentionSuiteBase.scala │ │ │ ├── DeltaSinkImplicitCastSuite.scala │ │ │ ├── DeltaSinkSuite.scala │ │ │ ├── DeltaSourceColumnMappingSuite.scala │ │ │ ├── DeltaSourceDeletionVectorsSuite.scala │ │ │ ├── DeltaSourceFastDropFeatureSuite.scala │ │ │ ├── DeltaSourceLargeLogSuite.scala │ │ │ ├── DeltaSourceOffsetSuite.scala │ │ │ ├── DeltaSourceSchemaEvolutionSuite.scala │ │ │ ├── DeltaSourceSuite.scala │ │ │ ├── DeltaSourceSuiteBase.scala │ │ │ ├── DeltaSourceTableAPISuite.scala │ │ │ ├── DeltaStreamUtilsSuite.scala │ │ │ ├── DeltaSuite.scala │ │ │ ├── DeltaTableCreationTests.scala │ │ │ ├── DeltaTableFeatureSuite.scala │ │ │ ├── DeltaTableUtilsSuite.scala │ │ │ ├── DeltaTestUtils.scala │ │ │ ├── DeltaThrowableSuite.scala │ │ │ ├── DeltaTimeTravelSuite.scala │ │ │ ├── DeltaTimestampNTZSuite.scala │ │ │ ├── DeltaUDFSuite.scala │ │ │ ├── DeltaUpdateCatalogSuite.scala │ │ │ ├── DeltaUpdateCatalogSuiteBase.scala │ │ │ ├── DeltaUsageLogsOpsTypes.scala │ │ │ ├── DeltaVacuumSuite.scala │ │ │ ├── DeltaVariantShreddingSuite.scala │ │ │ ├── DeltaVariantSuite.scala │ │ │ ├── DeltaWithNewTransactionSuite.scala │ │ │ ├── DeltaWriteConfigsSuite.scala │ │ │ ├── DescribeDeltaDetailSuite.scala │ │ │ ├── DescribeDeltaHistorySuite.scala │ │ │ ├── DomainMetadataRemovalSuite.scala │ │ │ ├── DomainMetadataSuite.scala │ │ │ ├── DuplicatingListLogStoreSuite.scala │ │ │ ├── EvolvabilitySuite.scala │ │ │ ├── EvolvabilitySuiteBase.scala │ │ │ ├── FakeFileSystem.scala │ │ │ ├── FeatureEnablementConcurrencySuite.scala │ │ │ ├── FileMetadataMaterializationTrackerSuite.scala │ │ │ ├── FileNamesSuite.scala │ │ │ ├── FindLastCompleteCheckpointSuite.scala │ │ │ ├── GenerateIdentityValuesSuite.scala │ │ │ ├── GeneratedColumnCompatibilitySuite.scala │ │ │ ├── GeneratedColumnSuite.scala │ │ │ ├── GeneratedColumnTest.scala │ │ │ ├── HiveConvertToDeltaSuite.scala │ │ │ ├── HiveDeltaDDLSuite.scala │ │ │ ├── HiveDeltaNotSupportedDDLSuite.scala │ │ │ ├── IcebergCompatUtilsBase.scala │ │ │ ├── IdentityColumnAdmissionSuite.scala │ │ │ ├── IdentityColumnConflictSuite.scala │ │ │ ├── IdentityColumnDMLSuite.scala │ │ │ ├── IdentityColumnIngestionSuite.scala │ │ │ ├── IdentityColumnSuite.scala │ │ │ ├── IdentityColumnSyncSuite.scala │ │ │ ├── IdentityColumnTestUtils.scala │ │ │ ├── ImplicitDMLCastingSuite.scala │ │ │ ├── InCommitTimestampSuite.scala │ │ │ ├── InCommitTimestampTestUtils.scala │ │ │ ├── LastCheckpointInfoSuite.scala │ │ │ ├── LogStoreProviderSuite.scala │ │ │ ├── LogStoreSuite.scala │ │ │ ├── LogStoreSuiteBase.scala │ │ │ ├── MaterializePartitionColumnsFeatureSuite.scala │ │ │ ├── MergeIntoAccumulatorSuite.scala │ │ │ ├── MergeIntoDVsSuite.scala │ │ │ ├── MergeIntoMaterializeSourceSuite.scala │ │ │ ├── MergeIntoMetricsBase.scala │ │ │ ├── MergeIntoNotMatchedBySourceSuite.scala │ │ │ ├── MergeIntoSQLSuite.scala │ │ │ ├── MergeIntoScalaSuite.scala │ │ │ ├── MergeIntoSchemaEvolutionSuite.scala │ │ │ ├── MergeIntoStructEvolutionNullnessSuite.scala │ │ │ ├── MergeIntoSuiteBase.scala │ │ │ ├── MergeIntoTestUtils.scala │ │ │ ├── MergeIntoTimestampConsistencySuite.scala │ │ │ ├── NonFateSharingFutureSuite.scala │ │ │ ├── OptimisticTransactionLegacyTests.scala │ │ │ ├── OptimisticTransactionSuite.scala │ │ │ ├── OptimisticTransactionSuiteBase.scala │ │ │ ├── ProtocolMetadataAdapterSuite.scala │ │ │ ├── RestoreTableSQLSuite.scala │ │ │ ├── RestoreTableScalaSuite.scala │ │ │ ├── RestoreTableSuiteBase.scala │ │ │ ├── S3LikeLocalFileSystem.scala │ │ │ ├── S3SingleDriverLogStoreSuite.scala │ │ │ ├── SchemaValidationSuite.scala │ │ │ ├── ShowDeltaTableColumnsSuite.scala │ │ │ ├── SnapshotManagementSuite.scala │ │ │ ├── TableRedirectSuite.scala │ │ │ ├── TightBoundsSuite.scala │ │ │ ├── TimestampLocalFileSystem.scala │ │ │ ├── UCManagedTableKillSwitchSuite.scala │ │ │ ├── UniversalFormatSuiteBase.scala │ │ │ ├── UpdateMetricsSuite.scala │ │ │ ├── UpdateSQLSuite.scala │ │ │ ├── UpdateScalaSuite.scala │ │ │ ├── UpdateSuiteBase.scala │ │ │ ├── VersionChecksumHistogramCompatSuite.scala │ │ │ ├── actions/ │ │ │ │ ├── AddFileSuite.scala │ │ │ │ └── DeletionVectorDescriptorSuite.scala │ │ │ ├── cdc/ │ │ │ │ ├── CDCReaderSuite.scala │ │ │ │ ├── CDCWorkloadSuite.scala │ │ │ │ ├── DeleteCDCSuite.scala │ │ │ │ ├── MergeCDCSuite.scala │ │ │ │ └── UpdateCDCSuite.scala │ │ │ ├── clustering/ │ │ │ │ ├── ClusteredTableClusteringSuite.scala │ │ │ │ ├── ClusteringMetadataDomainSuite.scala │ │ │ │ └── ClusteringTableFeatureSuite.scala │ │ │ ├── columnmapping/ │ │ │ │ ├── DropColumnMappingFeatureSuite.scala │ │ │ │ ├── RemoveColumnMappingCDCSuite.scala │ │ │ │ ├── RemoveColumnMappingRowTrackingSuite.scala │ │ │ │ ├── RemoveColumnMappingStreamingReadSuite.scala │ │ │ │ ├── RemoveColumnMappingSuite.scala │ │ │ │ └── RemoveColumnMappingSuiteUtils.scala │ │ │ ├── commands/ │ │ │ │ ├── DeltaCommandInvariantsSuite.scala │ │ │ │ └── backfill/ │ │ │ │ ├── RowTrackingBackfillBackfillConflictsSuite.scala │ │ │ │ ├── RowTrackingBackfillCloneConflictsSuite.scala │ │ │ │ └── RowTrackingBackfillConflictsSuite.scala │ │ │ ├── concurrency/ │ │ │ │ ├── PhaseLockingTestMixin.scala │ │ │ │ ├── TransactionExecutionObserverSuite.scala │ │ │ │ └── TransactionExecutionTestMixin.scala │ │ │ ├── coordinatedcommits/ │ │ │ │ ├── CatalogManagedStreamingSuite.scala │ │ │ │ ├── CatalogOwnedEnablementSuite.scala │ │ │ │ ├── CatalogOwnedPropertySuite.scala │ │ │ │ ├── CommitCoordinatorClientImplSuiteBase.scala │ │ │ │ ├── CommitCoordinatorClientSuite.scala │ │ │ │ ├── CoordinatedCommitsEnablementSuite.scala │ │ │ │ ├── CoordinatedCommitsPropertySuiteBase.scala │ │ │ │ ├── CoordinatedCommitsSuite.scala │ │ │ │ ├── CoordinatedCommitsTestUtils.scala │ │ │ │ ├── CoordinatedCommitsUtilsSuite.scala │ │ │ │ ├── DynamoDBCommitCoordinatorClientSuite.scala │ │ │ │ ├── InMemoryCommitCoordinatorSuite.scala │ │ │ │ ├── UCCommitCoordinatorBuilderSuite.scala │ │ │ │ ├── UCCommitCoordinatorClientSuite.scala │ │ │ │ └── UCCommitCoordinatorClientSuiteBase.scala │ │ │ ├── deletionvectors/ │ │ │ │ ├── DeletionVectorsSuite.scala │ │ │ │ ├── RoaringBitmapArraySuite.scala │ │ │ │ └── RowIndexMarkingFiltersSuite.scala │ │ │ ├── expressions/ │ │ │ │ ├── DecodeNestedZ85EncodedVariantSuite.scala │ │ │ │ ├── HilbertIndexSuite.scala │ │ │ │ ├── HilbertUtilsSuite.scala │ │ │ │ ├── InterleaveBitsBenchmark.scala │ │ │ │ ├── InterleaveBitsSuite.scala │ │ │ │ ├── RangePartitionIdSuite.scala │ │ │ │ └── aggregation/ │ │ │ │ └── BitmapAggregatorSuite.scala │ │ │ ├── files/ │ │ │ │ └── TransactionalWriteSuite.scala │ │ │ ├── fuzzer/ │ │ │ │ └── AtomicBarrierSuite.scala │ │ │ ├── generatedsuites/ │ │ │ │ ├── DeleteSuitesDeleteBaseTests.scala │ │ │ │ ├── DeleteSuitesDeleteCDCTests.scala │ │ │ │ ├── DeleteSuitesDeleteSQLTests.scala │ │ │ │ ├── DeleteSuitesDeleteScalaTests.scala │ │ │ │ ├── DeleteSuitesDeleteTempViewTests.scala │ │ │ │ ├── DeleteSuitesRowTrackingDeleteDvBase.scala │ │ │ │ ├── DeleteSuitesRowTrackingDeleteSuiteBase.scala │ │ │ │ ├── InsertSuitesDeltaInsertIntoImplicitCastStreamingWriteTests.scala │ │ │ │ ├── InsertSuitesDeltaInsertIntoImplicitCastTests.scala │ │ │ │ ├── MergeSuitesMergeCDCTests.scala │ │ │ │ ├── MergeSuitesMergeIntoAnalysisExceptionTests.scala │ │ │ │ ├── MergeSuitesMergeIntoBasicTests.scala │ │ │ │ ├── MergeSuitesMergeIntoDVsTests.scala │ │ │ │ ├── MergeSuitesMergeIntoExtendedSyntaxTests.scala │ │ │ │ ├── MergeSuitesMergeIntoMaterializeSourceErrorTests.scala │ │ │ │ ├── MergeSuitesMergeIntoMaterializeSourceTests.scala │ │ │ │ ├── MergeSuitesMergeIntoNestedArrayStructEvolutionNullnessTests.scala │ │ │ │ ├── MergeSuitesMergeIntoNestedDataTests.scala │ │ │ │ ├── MergeSuitesMergeIntoNestedMapStructEvolutionNullnessTests.scala │ │ │ │ ├── MergeSuitesMergeIntoNestedStructEvolutionInsertTests.scala │ │ │ │ ├── MergeSuitesMergeIntoNestedStructEvolutionNullnessTests.scala │ │ │ │ ├── MergeSuitesMergeIntoNestedStructEvolutionUpdateOnlyTests.scala │ │ │ │ ├── MergeSuitesMergeIntoNestedStructInMapEvolutionTests.scala │ │ │ │ ├── MergeSuitesMergeIntoNotMatchedBySourceCDCPart1Tests.scala │ │ │ │ ├── MergeSuitesMergeIntoNotMatchedBySourceCDCPart2Tests.scala │ │ │ │ ├── MergeSuitesMergeIntoNotMatchedBySourceSuite.scala │ │ │ │ ├── MergeSuitesMergeIntoSQLNondeterministicOrderTests.scala │ │ │ │ ├── MergeSuitesMergeIntoSQLTests.scala │ │ │ │ ├── MergeSuitesMergeIntoScalaTests.scala │ │ │ │ ├── MergeSuitesMergeIntoSchemaEvoStoreAssignmentPolicyTests.scala │ │ │ │ ├── MergeSuitesMergeIntoSchemaEvolutionBaseExistingColumnTests.scala │ │ │ │ ├── MergeSuitesMergeIntoSchemaEvolutionBaseNewColumnTests.scala │ │ │ │ ├── MergeSuitesMergeIntoSchemaEvolutionCoreTests.scala │ │ │ │ ├── MergeSuitesMergeIntoSchemaEvolutionNotMatchedBySourceTests.scala │ │ │ │ ├── MergeSuitesMergeIntoStructEvolutionNullnessMultiClauseTests.scala │ │ │ │ ├── MergeSuitesMergeIntoSuiteBaseMiscTests.scala │ │ │ │ ├── MergeSuitesMergeIntoTempViewsTests.scala │ │ │ │ ├── MergeSuitesMergeIntoTopLevelArrayStructEvolutionNullnessTests.scala │ │ │ │ ├── MergeSuitesMergeIntoTopLevelMapStructEvolutionNullnessTests.scala │ │ │ │ ├── MergeSuitesMergeIntoTopLevelStructEvolutionNullnessTests.scala │ │ │ │ ├── MergeSuitesMergeIntoUnlimitedMergeClausesTests.scala │ │ │ │ ├── MergeSuitesRowTrackingMergeCommonTests.scala │ │ │ │ ├── UpdateSuitesRowTrackingUpdateCommonTests.scala │ │ │ │ ├── UpdateSuitesUpdateBaseMiscTests.scala │ │ │ │ ├── UpdateSuitesUpdateBaseTempViewTests.scala │ │ │ │ ├── UpdateSuitesUpdateCDCTests.scala │ │ │ │ ├── UpdateSuitesUpdateCDCWithDeletionVectorsTests.scala │ │ │ │ ├── UpdateSuitesUpdateSQLTests.scala │ │ │ │ ├── UpdateSuitesUpdateSQLWithDeletionVectorsTests.scala │ │ │ │ └── UpdateSuitesUpdateScalaTests.scala │ │ │ ├── logging/ │ │ │ │ ├── DeltaStructuredLoggingSuite.scala │ │ │ │ └── LogThrottlingSuite.scala │ │ │ ├── metric/ │ │ │ │ └── IncrementMetricSuite.scala │ │ │ ├── optimize/ │ │ │ │ ├── CompactionTestHelper.scala │ │ │ │ ├── DeltaReorgSuite.scala │ │ │ │ ├── OptimizeCompactionSuite.scala │ │ │ │ ├── OptimizeConflictSuite.scala │ │ │ │ ├── OptimizeMetricsSuite.scala │ │ │ │ └── OptimizeZOrderSuite.scala │ │ │ ├── perf/ │ │ │ │ ├── OptimizeGeneratedColumnSuite.scala │ │ │ │ ├── OptimizeMetadataOnlyDeltaQuerySuite.scala │ │ │ │ └── OptimizedWritesSuite.scala │ │ │ ├── rowid/ │ │ │ │ ├── ConflictCheckerRowIdSuite.scala │ │ │ │ ├── GenerateRowIDsSuite.scala │ │ │ │ ├── RowIdCloneSuite.scala │ │ │ │ ├── RowIdCreateReplaceTableSuite.scala │ │ │ │ ├── RowIdSuite.scala │ │ │ │ ├── RowIdTestUtils.scala │ │ │ │ ├── RowTrackingBackfillSuite.scala │ │ │ │ ├── RowTrackingCompactionSuite.scala │ │ │ │ ├── RowTrackingDeleteSuite.scala │ │ │ │ ├── RowTrackingMergeSuite.scala │ │ │ │ ├── RowTrackingRemovalConcurrencySuite.scala │ │ │ │ ├── RowTrackingRemovalSuite.scala │ │ │ │ └── RowTrackingUpdateSuite.scala │ │ │ ├── rowtracking/ │ │ │ │ ├── DefaultRowCommitVersionSuite.scala │ │ │ │ ├── MaterializedColumnSuite.scala │ │ │ │ ├── RowTrackingConflictResolutionSuite.scala │ │ │ │ ├── RowTrackingReadWriteSuite.scala │ │ │ │ └── RowTrackingTestUtils.scala │ │ │ ├── schema/ │ │ │ │ ├── CaseSensitivitySuite.scala │ │ │ │ ├── CheckConstraintsSuite.scala │ │ │ │ ├── InvariantEnforcementSuite.scala │ │ │ │ ├── SchemaEnforcementSuite.scala │ │ │ │ └── SchemaUtilsSuite.scala │ │ │ ├── serverSidePlanning/ │ │ │ │ ├── ServerSidePlannedTableSuite.scala │ │ │ │ ├── ServerSidePlanningClientFactorySuite.scala │ │ │ │ └── TestServerSidePlanningClient.scala │ │ │ ├── skipping/ │ │ │ │ ├── ClusteredTableTestUtils.scala │ │ │ │ ├── MultiDimClusteringFunctionsSuite.scala │ │ │ │ ├── MultiDimClusteringSuite.scala │ │ │ │ └── clustering/ │ │ │ │ ├── ClusteredTableDDLSuite.scala │ │ │ │ ├── ClusteringColumnSuite.scala │ │ │ │ ├── ClusteringProviderSuite.scala │ │ │ │ └── IncrementalZCubeClusteringSuite.scala │ │ │ ├── sources/ │ │ │ │ └── DeltaSourceMetadataEvolutionSupportSuite.scala │ │ │ ├── stats/ │ │ │ │ ├── DataSkippingDeltaConstructDataFiltersSuite.scala │ │ │ │ ├── DataSkippingDeltaTests.scala │ │ │ │ ├── PartitionLikeDataSkippingSuite.scala │ │ │ │ ├── PreparedDeltaFileIndexRowCountSuite.scala │ │ │ │ ├── StatsCollectionSuite.scala │ │ │ │ └── StatsUtils.scala │ │ │ ├── storage/ │ │ │ │ ├── LineClosableIteratorSuite.scala │ │ │ │ └── dv/ │ │ │ │ ├── DeletionVectorFileSizeSuite.scala │ │ │ │ └── DeletionVectorStoreSuite.scala │ │ │ ├── test/ │ │ │ │ ├── CustomCatalogs.scala │ │ │ │ ├── DeltaColumnMappingSelectedTestMixin.scala │ │ │ │ ├── DeltaExceptionTestUtils.scala │ │ │ │ ├── DeltaExcludedTestMixin.scala │ │ │ │ ├── DeltaHiveTest.scala │ │ │ │ ├── DeltaSQLCommandTest.scala │ │ │ │ ├── DeltaSQLTestUtils.scala │ │ │ │ ├── DeltaTestImplicits.scala │ │ │ │ ├── ScanReportHelper.scala │ │ │ │ └── TestsStatistics.scala │ │ │ ├── typewidening/ │ │ │ │ ├── TypeWideningAlterTableNestedSuite.scala │ │ │ │ ├── TypeWideningAlterTableSuite.scala │ │ │ │ ├── TypeWideningConstraintsSuite.scala │ │ │ │ ├── TypeWideningFeatureCompatibilitySuite.scala │ │ │ │ ├── TypeWideningGeneratedColumnsSuite.scala │ │ │ │ ├── TypeWideningInsertSchemaEvolutionBasicSuite.scala │ │ │ │ ├── TypeWideningInsertSchemaEvolutionExtendedSuite.scala │ │ │ │ ├── TypeWideningMergeIntoSchemaEvolutionSuite.scala │ │ │ │ ├── TypeWideningMetadataSuite.scala │ │ │ │ ├── TypeWideningStatsSuite.scala │ │ │ │ ├── TypeWideningStreamingSinkSuite.scala │ │ │ │ ├── TypeWideningStreamingSourceSuite.scala │ │ │ │ ├── TypeWideningTableFeatureSuite.scala │ │ │ │ ├── TypeWideningTestCases.scala │ │ │ │ ├── TypeWideningTestMixin.scala │ │ │ │ └── TypeWideningUniformTests.scala │ │ │ ├── uniform/ │ │ │ │ ├── IcebergCompatV2EnableUniformByAlterTableSuiteBase.scala │ │ │ │ ├── SparkSessionSwitch.scala │ │ │ │ ├── UniFormE2EIcebergSuiteBase.scala │ │ │ │ ├── UniFormE2ETest.scala │ │ │ │ └── hms/ │ │ │ │ ├── EmbeddedHMS.scala │ │ │ │ ├── HMSServer.scala │ │ │ │ └── HMSTest.scala │ │ │ └── util/ │ │ │ ├── AnalysisHelperSuite.scala │ │ │ ├── BinPackingIteratorSuite.scala │ │ │ ├── BinPackingUtilsSuite.scala │ │ │ ├── BitmapAggregatorE2ESuite.scala │ │ │ ├── CatalogTableTestUtils.scala │ │ │ ├── CodecSuite.scala │ │ │ ├── DatasetRefCacheSuite.scala │ │ │ ├── DeltaLogGroupingIteratorSuite.scala │ │ │ ├── JsonUtilsSuite.scala │ │ │ └── threads/ │ │ │ ├── DeltaThreadPoolSuite.scala │ │ │ └── SparkThreadLocalForwardingSuite.scala │ │ └── scala-shims/ │ │ ├── spark-4.0/ │ │ │ ├── GridTestShim.scala │ │ │ ├── InvalidDefaultValueErrorShims.scala │ │ │ ├── StreamingTestShims.scala │ │ │ ├── UnsupportedTableOperationErrorShims.scala │ │ │ └── VariantShreddingTestShims.scala │ │ ├── spark-4.1/ │ │ │ ├── GridTestShim.scala │ │ │ ├── InvalidDefaultValueErrorShims.scala │ │ │ ├── StreamingTestShims.scala │ │ │ ├── UnsupportedTableOperationErrorShims.scala │ │ │ └── VariantShreddingTestShims.scala │ │ └── spark-4.2/ │ │ ├── GridTestShim.scala │ │ ├── InvalidDefaultValueErrorShims.scala │ │ ├── StreamingTestShims.scala │ │ ├── UnsupportedTableOperationErrorShims.scala │ │ └── VariantShreddingTestShims.scala │ ├── unitycatalog/ │ │ └── src/ │ │ └── test/ │ │ └── java/ │ │ └── io/ │ │ └── sparkuctest/ │ │ ├── S3CredentialFileSystem.java │ │ ├── UCDeltaStreamingTest.java │ │ ├── UCDeltaTableBlockMetadataUpdateTest.java │ │ ├── UCDeltaTableCreationTest.java │ │ ├── UCDeltaTableDMLTest.java │ │ ├── UCDeltaTableDataFrameReadTest.java │ │ ├── UCDeltaTableDataFrameStreamingTest.java │ │ ├── UCDeltaTableDataFrameWriteTest.java │ │ ├── UCDeltaTableDeltaAPITest.java │ │ ├── UCDeltaTableIntegrationBaseTest.java │ │ ├── UCDeltaTableReadTest.java │ │ ├── UCDeltaUtilityTest.java │ │ ├── UnityCatalogSupport.java │ │ └── UnityCatalogSupportTest.java │ └── v2/ │ ├── README.md │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── io/ │ │ └── delta/ │ │ └── spark/ │ │ └── internal/ │ │ └── v2/ │ │ ├── catalog/ │ │ │ └── SparkTable.java │ │ ├── exception/ │ │ │ └── VersionNotFoundException.java │ │ ├── read/ │ │ │ ├── DeltaInputPartition.java │ │ │ ├── DeltaParquetFileFormatV2.java │ │ │ ├── IndexedFile.java │ │ │ ├── ProtocolMetadataAdapterV2.java │ │ │ ├── SparkBatch.java │ │ │ ├── SparkMicroBatchStream.java │ │ │ ├── SparkPartitionReader.java │ │ │ ├── SparkReaderFactory.java │ │ │ ├── SparkScan.java │ │ │ ├── SparkScanBuilder.java │ │ │ └── deletionvector/ │ │ │ ├── ColumnVectorWithFilter.java │ │ │ ├── DeletionVectorReadFunction.java │ │ │ └── DeletionVectorSchemaContext.java │ │ ├── snapshot/ │ │ │ ├── DeltaSnapshotManager.java │ │ │ ├── PathBasedSnapshotManager.java │ │ │ ├── SnapshotManagerFactory.java │ │ │ └── unitycatalog/ │ │ │ ├── UCManagedTableSnapshotManager.java │ │ │ ├── UCTableInfo.java │ │ │ └── UCUtils.java │ │ └── utils/ │ │ ├── CloseableIterator.java │ │ ├── ExpressionUtils.java │ │ ├── PartitionUtils.java │ │ ├── RowTrackingUtils.java │ │ ├── ScalaUtils.java │ │ ├── SchemaUtils.java │ │ ├── SerializableKernelRowWrapper.java │ │ ├── StatsUtils.java │ │ └── StreamingHelper.java │ └── test/ │ ├── java/ │ │ └── io/ │ │ └── delta/ │ │ └── spark/ │ │ └── internal/ │ │ └── v2/ │ │ ├── DeltaV2TestBase.java │ │ ├── InternalRowTestUtils.java │ │ ├── V2DDLTest.java │ │ ├── V2ReadTest.java │ │ ├── V2StreamingReadTest.java │ │ ├── V2TestBase.java │ │ ├── catalog/ │ │ │ ├── SparkTableTest.java │ │ │ └── TestCatalog.java │ │ ├── read/ │ │ │ ├── SparkGoldenTableTest.java │ │ │ ├── SparkMicroBatchStreamTest.java │ │ │ ├── SparkScanBuilderTest.java │ │ │ ├── SparkScanTest.java │ │ │ └── deletionvector/ │ │ │ ├── ColumnVectorWithFilterTest.java │ │ │ ├── DeletionVectorReadFunctionTest.java │ │ │ └── DeletionVectorSchemaContextTest.java │ │ ├── snapshot/ │ │ │ ├── PathBasedSnapshotManagerTest.java │ │ │ └── unitycatalog/ │ │ │ └── UCTableInfoTest.java │ │ └── utils/ │ │ ├── CloseableIteratorTest.java │ │ ├── ExpressionUtilsTest.java │ │ ├── PartitionUtilsTest.java │ │ ├── RowTrackingUtilsTest.java │ │ ├── ScalaUtilsTest.java │ │ ├── SchemaUtilsTest.java │ │ ├── StatsUtilsTest.java │ │ └── StreamingHelperTest.java │ └── scala/ │ └── io/ │ └── delta/ │ └── spark/ │ └── internal/ │ └── v2/ │ ├── snapshot/ │ │ └── unitycatalog/ │ │ ├── UCManagedTableSnapshotManagerSuite.scala │ │ └── UCUtilsSuite.scala │ └── utils/ │ └── CatalogTableTestUtils.scala ├── spark-connect/ │ ├── client/ │ │ └── src/ │ │ ├── main/ │ │ │ ├── scala/ │ │ │ │ └── io/ │ │ │ │ └── delta/ │ │ │ │ └── connect/ │ │ │ │ └── tables/ │ │ │ │ ├── DeltaColumnBuilder.scala │ │ │ │ ├── DeltaMergeBuilder.scala │ │ │ │ ├── DeltaOptimizeBuilder.scala │ │ │ │ ├── DeltaTable.scala │ │ │ │ ├── DeltaTableBuilder.scala │ │ │ │ └── execution/ │ │ │ │ └── DeltaTableBuilderOptions.scala │ │ │ └── scala-shims/ │ │ │ ├── spark-4.0/ │ │ │ │ └── SparkStringUtilsShims.scala │ │ │ ├── spark-4.1/ │ │ │ │ └── SparkStringUtilsShims.scala │ │ │ └── spark-4.2/ │ │ │ └── SparkStringUtilsShims.scala │ │ └── test/ │ │ └── scala/ │ │ └── io/ │ │ └── delta/ │ │ └── connect/ │ │ └── tables/ │ │ ├── DeltaMergeBuilderSuite.scala │ │ ├── DeltaQueryTest.scala │ │ ├── DeltaTableBuilderSuite.scala │ │ ├── DeltaTableSuite.scala │ │ └── RemoteSparkSession.scala │ ├── common/ │ │ └── src/ │ │ └── main/ │ │ ├── buf.gen.yaml │ │ ├── buf.work.yaml │ │ ├── protobuf/ │ │ │ ├── buf.yaml │ │ │ ├── delta/ │ │ │ │ └── connect/ │ │ │ │ ├── base.proto │ │ │ │ ├── commands.proto │ │ │ │ └── relations.proto │ │ │ └── spark/ │ │ │ └── connect/ │ │ │ ├── base.proto │ │ │ ├── catalog.proto │ │ │ ├── commands.proto │ │ │ ├── common.proto │ │ │ ├── example_plugins.proto │ │ │ ├── expressions.proto │ │ │ ├── ml.proto │ │ │ ├── ml_common.proto │ │ │ ├── pipelines.proto │ │ │ ├── relations.proto │ │ │ └── types.proto │ │ └── scala/ │ │ └── org/ │ │ └── apache/ │ │ └── spark/ │ │ └── sql/ │ │ └── connect/ │ │ └── delta/ │ │ └── ImplicitProtoConversions.scala │ └── server/ │ └── src/ │ ├── main/ │ │ └── scala/ │ │ └── io/ │ │ └── delta/ │ │ └── connect/ │ │ ├── DeltaCommandPlugin.scala │ │ ├── DeltaPlannerBase.scala │ │ ├── DeltaRelationPlugin.scala │ │ └── SimpleDeltaConnectService.scala │ └── test/ │ └── scala/ │ └── io/ │ └── delta/ │ └── connect/ │ └── DeltaConnectPlannerSuite.scala ├── spark-unified/ │ ├── README.md │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── org/ │ │ │ └── apache/ │ │ │ └── spark/ │ │ │ └── sql/ │ │ │ └── delta/ │ │ │ └── catalog/ │ │ │ └── DeltaCatalog.java │ │ └── scala/ │ │ └── io/ │ │ └── delta/ │ │ ├── internal/ │ │ │ └── ApplyV2Streaming.scala │ │ └── sql/ │ │ └── DeltaSparkSessionExtension.scala │ └── test/ │ └── scala/ │ ├── io/ │ │ └── delta/ │ │ └── internal/ │ │ └── ApplyV2StreamingSuite.scala │ └── org/ │ └── apache/ │ └── spark/ │ └── sql/ │ └── delta/ │ ├── DataFrameWriterV2WithV2ConnectorSuite.scala │ ├── ProtocolMetadataAdapterV2Suite.scala │ ├── catalog/ │ │ └── DeltaCatalogSuite.scala │ └── test/ │ ├── DeltaV2SourceDeletionVectorsSuite.scala │ ├── DeltaV2SourceSuite.scala │ └── V2ForceTest.scala ├── storage/ │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── io/ │ │ └── delta/ │ │ └── storage/ │ │ ├── AzureLogStore.java │ │ ├── CloseableIterator.java │ │ ├── GCSLogStore.java │ │ ├── HDFSLogStore.java │ │ ├── HadoopFileSystemLogStore.java │ │ ├── LineCloseableIterator.java │ │ ├── LocalLogStore.java │ │ ├── LogStore.java │ │ ├── S3SingleDriverLogStore.java │ │ ├── commit/ │ │ │ ├── Commit.java │ │ │ ├── CommitCoordinatorClient.java │ │ │ ├── CommitFailedException.java │ │ │ ├── CommitResponse.java │ │ │ ├── CoordinatedCommitsUtils.java │ │ │ ├── GetCommitsResponse.java │ │ │ ├── TableDescriptor.java │ │ │ ├── TableIdentifier.java │ │ │ ├── UpdatedActions.java │ │ │ ├── actions/ │ │ │ │ ├── AbstractCommitInfo.java │ │ │ │ ├── AbstractMetadata.java │ │ │ │ └── AbstractProtocol.java │ │ │ ├── uccommitcoordinator/ │ │ │ │ ├── CommitLimitReachedException.java │ │ │ │ ├── InvalidTargetTableException.java │ │ │ │ ├── UCClient.java │ │ │ │ ├── UCCommitCoordinatorClient.java │ │ │ │ ├── UCCommitCoordinatorException.java │ │ │ │ ├── UCCoordinatedCommitsUsageLogs.java │ │ │ │ ├── UCRestClientPayload.java │ │ │ │ ├── UCTokenBasedRestClient.java │ │ │ │ └── UpgradeNotAllowedException.java │ │ │ └── uniform/ │ │ │ ├── IcebergMetadata.java │ │ │ └── UniformMetadata.java │ │ └── internal/ │ │ ├── FileNameUtils.java │ │ ├── LogStoreErrors.java │ │ ├── PathLock.java │ │ ├── S3LogStoreUtil.java │ │ └── ThreadUtils.java │ └── test/ │ └── scala/ │ └── io/ │ └── delta/ │ └── storage/ │ ├── ThreadUtilsSuite.scala │ ├── commit/ │ │ ├── InMemoryCommitCoordinator.scala │ │ └── uccommitcoordinator/ │ │ └── UCTokenBasedRestClientSuite.scala │ ├── integration/ │ │ └── S3LogStoreUtilIntegrationTest.scala │ └── internal/ │ └── S3LogStoreUtilTest.scala ├── storage-s3-dynamodb/ │ ├── integration_tests/ │ │ └── dynamodb_logstore.py │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── io/ │ │ └── delta/ │ │ └── storage/ │ │ ├── BaseExternalLogStore.java │ │ ├── ExternalCommitEntry.java │ │ ├── RetryableCloseableIterator.java │ │ ├── S3DynamoDBLogStore.java │ │ └── utils/ │ │ ├── ReflectionUtils.java │ │ └── ThrowingSupplier.java │ └── test/ │ ├── java/ │ │ └── io/ │ │ └── delta/ │ │ └── storage/ │ │ ├── FailingS3DynamoDBLogStore.java │ │ ├── MemoryLogStore.java │ │ └── utils/ │ │ └── ReflectionsUtilsSuiteHelper.java │ └── scala/ │ └── io/ │ └── delta/ │ └── storage/ │ ├── ExternalLogStoreSuite.scala │ ├── RetryableCloseableIteratorSuite.scala │ └── utils/ │ └── ReflectionsUtilsSuite.scala ├── testDeltaIcebergJar/ │ └── src/ │ └── test/ │ └── scala/ │ └── JarSuite.scala └── version.sbt ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitattributes ================================================ *.bat text eol=crlf *.cmd text eol=crlf *.bin binary ================================================ FILE: .github/CODEOWNERS ================================================ # CODEOWNERS file for Delta Lake # This file defines code owners who must approve changes to specific files/directories. # See: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners # Build configuration files and directories /build/ @tdas /build.sbt @tdas /project/ @tdas /version.sbt @tdas # Spark V2 and Unified modules /spark/v2/ @tdas @huan233usc @TimothyW553 @raveeram-db @murali-db /spark-unified/ @tdas @huan233usc @TimothyW553 @raveeram-db @murali-db # All files in the root directory /* @tdas ================================================ FILE: .github/ISSUE_TEMPLATE/bug-issue.md ================================================ --- name: Bug Issue about: Use this template for reporting a bug labels: 'bug' title: '[BUG]' --- ## Bug #### Which Delta project/connector is this regarding? - [ ] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ### Describe the problem #### Steps to reproduce #### Observed results #### Expected results #### Further details ### Environment information * Delta Lake version: * Spark version: * Scala version: ### Willingness to contribute The Delta Lake Community encourages bug fix contributions. Would you or another member of your organization be willing to contribute a fix for this bug to the Delta Lake code base? - [ ] Yes. I can contribute a fix for this bug independently. - [ ] Yes. I would be willing to contribute a fix for this bug with guidance from the Delta Lake community. - [ ] No. I cannot contribute a bug fix at this time. ================================================ FILE: .github/ISSUE_TEMPLATE/feature-request.md ================================================ --- name: Feature Request about: Use this template for raising a feature request labels: 'enhancement' title: '[Feature Request]' --- ## Feature request #### Which Delta project/connector is this regarding? - [ ] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ### Overview ### Motivation ### Further details ### Willingness to contribute The Delta Lake Community encourages new feature contributions. Would you or another member of your organization be willing to contribute an implementation of this feature? - [ ] Yes. I can contribute this feature independently. - [ ] Yes. I would be willing to contribute this feature with guidance from the Delta Lake community. - [ ] No. I cannot contribute this feature at this time. ================================================ FILE: .github/ISSUE_TEMPLATE/protocol-rfc.md ================================================ --- name: Protocol Change Request about: Use this template to propose a new feature that impacts the Delta protocol specification labels: 'protocol' title: '[PROTOCOL RFC]' --- ## Protocol Change Request ### Description of the protocol change ### Willingness to contribute The Delta Lake Community encourages protocol innovations. Would you or another member of your organization be willing to contribute this feature to the Delta Lake code base? - [ ] Yes. I can contribute. - [ ] Yes. I would be willing to contribute with guidance from the Delta Lake community. - [ ] No. I cannot contribute at this time. ================================================ FILE: .github/PULL_REQUEST_TEMPLATE.md ================================================ #### Which Delta project/connector is this regarding? - [ ] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description ## How was this patch tested? ## Does this PR introduce _any_ user-facing changes? ================================================ FILE: .github/workflows/build.yaml ================================================ name: "Delta Build" on: push: paths-ignore: - '**.md' - '**.txt' pull_request: paths-ignore: - '**.md' - '**.txt' jobs: test: name: "Build Test" runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v3 - name: Install Java 17 uses: actions/setup-java@v3 with: distribution: "zulu" java-version: "17" - name: Set up Python uses: actions/setup-python@v4 with: python-version: '3.9' - name: Cache Scala, SBT uses: actions/cache@v3 with: path: | ~/.sbt ~/.ivy2 ~/.cache/coursier key: delta-sbt-cache-cross-spark - name: Run cross-Spark build test run: python project/tests/test_cross_spark_publish.py ================================================ FILE: .github/workflows/flink_test.yaml ================================================ name: "Delta Flink" on: [push, pull_request] # Cancel previous runs when new commits are pushed concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true env: # Point SBT to our cache directories for consistency SBT_OPTS: "-Dsbt.coursier.home-dir=/home/runner/.cache/coursier -Dsbt.ivy.home=/home/runner/.ivy2" jobs: test: name: "DF" runs-on: ubuntu-24.04 steps: - name: Show runner specs run: | echo "=== GitHub Runner Specs ===" echo "CPU cores: $(nproc)" echo "CPU info: $(lscpu | grep 'Model name' | cut -d':' -f2 | xargs)" echo "Total RAM: $(free -h | grep '^Mem:' | awk '{print $2}')" echo "Available RAM: $(free -h | grep '^Mem:' | awk '{print $7}')" echo "Disk space: $(df -h / | tail -1 | awk '{print $2 " total, " $4 " available"}')" echo "Runner OS: ${{ runner.os }}" echo "Runner arch: ${{ runner.arch }}" - name: Checkout code uses: actions/checkout@v4 # Run unit tests with JDK 17. These unit tests depend on Spark, and Spark 4.0+ is JDK 17. - name: install java uses: actions/setup-java@v4 with: distribution: "zulu" java-version: "17" - name: Cache SBT and dependencies id: cache-sbt uses: actions/cache@v4 with: path: | ~/.sbt ~/.ivy2/cache ~/.coursier/cache ~/.cache/coursier key: sbt-flink - name: Check cache status run: | if [ "${{ steps.cache-sbt.outputs.cache-hit }}" == "true" ]; then echo "✅ Cache HIT - using cached dependencies" else echo "❌ Cache MISS - will download dependencies" fi - name: Run unit tests run: | build/sbt flinkGroup/test ================================================ FILE: .github/workflows/iceberg_test.yaml ================================================ name: "Delta Iceberg Latest" on: push: paths-ignore: - '**.md' - '**.txt' pull_request: paths-ignore: - '**.md' - '**.txt' jobs: test: name: "DIL: Scala ${{ matrix.scala }}" runs-on: ubuntu-24.04 strategy: matrix: # These Scala versions must match those in the build.sbt scala: [2.13.16] env: SCALA_VERSION: ${{ matrix.scala }} steps: - uses: actions/checkout@v3 - name: install java uses: actions/setup-java@v3 with: distribution: "zulu" java-version: "17" - name: Cache Scala, SBT uses: actions/cache@v3 with: path: | ~/.sbt ~/.ivy2 ~/.cache/coursier # Change the key if dependencies are changed. For each key, GitHub Actions will cache the # the above directories when we use the key for the first time. After that, each run will # just use the cache. The cache is immutable so we need to use a new key when trying to # cache new stuff. key: delta-sbt-cache-spark4.0-scala${{ matrix.scala }} - name: Install Job dependencies run: | sudo apt-get update sudo apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python3-openssl git sudo apt install libedit-dev curl -LO https://github.com/bufbuild/buf/releases/download/v1.28.1/buf-Linux-x86_64.tar.gz mkdir -p ~/buf tar -xvzf buf-Linux-x86_64.tar.gz -C ~/buf --strip-components 1 rm buf-Linux-x86_64.tar.gz sudo apt install python3-pip --fix-missing sudo pip3 install pipenv==2024.4.1 curl https://pyenv.run | bash export PATH="~/.pyenv/bin:$PATH" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" pyenv install 3.8.18 pyenv global system 3.8.18 pipenv --python 3.8.18 install - name: Run Scala/Java and Python tests # when changing TEST_PARALLELISM_COUNT make sure to also change it in spark_master_test.yaml run: | TEST_PARALLELISM_COUNT=4 pipenv run python run-tests.py --group iceberg --spark-version 4.0 ================================================ FILE: .github/workflows/kernel_docs.yaml ================================================ # Simple workflow for deploying static content to GitHub Pages name: Deploy static content to Pages on: # Allows you to run this workflow manually from the Actions tab workflow_dispatch: # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages permissions: contents: read pages: write id-token: write # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. concurrency: group: "pages" cancel-in-progress: false jobs: # Single deploy job since we're just deploying deploy_docs: environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 - name: install java uses: actions/setup-java@v3 with: distribution: "zulu" java-version: "11" - name: Generate docs run: | build/sbt kernelGroup/unidoc mkdir -p kernel/docs/snapshot/kernel-api/java mkdir -p kernel/docs/snapshot/kernel-defaults/java cp -r kernel/kernel-api/target/javaunidoc/. kernel/docs/snapshot/kernel-api/java/ cp -r kernel/kernel-defaults/target/javaunidoc/. kernel/docs/snapshot/kernel-defaults/java/ - name: Setup Pages uses: actions/configure-pages@v3 - name: Upload artifact uses: actions/upload-pages-artifact@v1 with: # Upload kernel docs path: kernel/docs - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@v2 ================================================ FILE: .github/workflows/kernel_test.yaml ================================================ name: "Delta Kernel" on: push: paths-ignore: - '**.md' - '**.txt' pull_request: paths-ignore: - '**.md' - '**.txt' # Cancel previous runs when new commits are pushed concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true env: # Point SBT to our cache directories for consistency SBT_OPTS: "-Dsbt.coursier.home-dir=/home/runner/.cache/coursier -Dsbt.ivy.home=/home/runner/.ivy2" jobs: test: name: "DK: Shard ${{ matrix.shard }}" runs-on: ubuntu-24.04 strategy: fail-fast: false # Allow all shards to run even if one fails matrix: shard: [0, 1, 2, 3] env: SCALA_VERSION: 2.13.16 NUM_SHARDS: 4 DISABLE_UNIDOC: true # Another unidoc workflow will test unidoc. TEST_PARALLELISM_COUNT: 4 steps: - name: Show runner specs run: | echo "=== GitHub Runner Specs ===" echo "CPU cores: $(nproc)" echo "CPU info: $(lscpu | grep 'Model name' | cut -d':' -f2 | xargs)" echo "Total RAM: $(free -h | grep '^Mem:' | awk '{print $2}')" echo "Available RAM: $(free -h | grep '^Mem:' | awk '{print $7}')" echo "Disk space: $(df -h / | tail -1 | awk '{print $2 " total, " $4 " available"}')" echo "Runner OS: ${{ runner.os }}" echo "Runner arch: ${{ runner.arch }}" - name: Checkout code uses: actions/checkout@v4 # Run unit tests with JDK 17. These unit tests depend on Spark, and Spark 4.0+ is JDK 17. - name: install java uses: actions/setup-java@v4 with: distribution: "zulu" java-version: "17" - name: Cache SBT and dependencies id: cache-sbt uses: actions/cache@v4 with: path: | ~/.sbt ~/.ivy2/cache ~/.coursier/cache ~/.cache/coursier key: sbt-kernel-${{ runner.os }}-scala${{ env.SCALA_VERSION }} - name: Check cache status run: | if [ "${{ steps.cache-sbt.outputs.cache-hit }}" == "true" ]; then echo "✅ Cache HIT - using cached dependencies" else echo "❌ Cache MISS - will download dependencies" fi - name: Run unit tests run: | python run-tests.py --group kernel --coverage --shard ${{ matrix.shard }} integration-test: name: "DK: Integration" runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v3 # Run integration tests with JDK 11, as they have no Spark dependency - name: install java uses: actions/setup-java@v3 with: distribution: "zulu" java-version: "11" - name: Run integration tests run: | cd kernel/examples && python run-kernel-examples.py --use-local ================================================ FILE: .github/workflows/kernel_unitycatalog_test.yaml ================================================ name: "Kernel Unity Catalog" on: push: paths: - 'build.sbt' - 'version.sbt' - 'kernel/**/*.scala' - 'kernel/**/*.java' - 'storage/**/*.scala' - 'storage/**/*.java' - '.github/workflows/kernel_unitycatalog_test.yaml' pull_request: paths: - 'build.sbt' - 'version.sbt' - 'kernel/**/*.scala' - 'kernel/**/*.java' - 'storage/**/*.scala' - 'storage/**/*.java' - '.github/workflows/kernel_unitycatalog_test.yaml' jobs: test: name: "Kernel Unity Catalog Tests" runs-on: ubuntu-24.04 env: SCALA_VERSION: 2.13.16 steps: - uses: actions/checkout@v3 - name: install java uses: actions/setup-java@v3 with: distribution: "zulu" java-version: "17" - name: Run Unity tests with coverage run: | ./build/sbt "++ ${{ env.SCALA_VERSION }}" clean coverage kernelUnityCatalog/test coverageAggregate coverageOff -v ================================================ FILE: .github/workflows/publish_docs.yaml ================================================ name: Publish Docs on: push: branches: - master paths: - docs/** release: types: - published workflow_dispatch: jobs: build_api_docs: name: Build API docs (${{ matrix.version.name }}) runs-on: ubuntu-latest strategy: matrix: version: - name: latest ref: v4.0.1 java: 17 out_dir: docs/apis/_site/api - name: 3.3.2 ref: v3.3.2 java: 8 out_dir: docs/apis/_site/api steps: - uses: actions/checkout@v4 with: repository: delta-io/delta ref: ${{ matrix.version.ref }} - uses: actions/setup-java@v3 with: distribution: "zulu" java-version: ${{ matrix.version.java }} - name: Setup python environment uses: conda-incubator/setup-miniconda@v3 with: activate-environment: delta_docs environment-file: docs/environment.yml - name: Fix generate_api_docs script if: contains(matrix.version.ref, 'v4') run: | sed -i 's|scala-2\.12|scala-2.13|g' docs/apis/generate_api_docs.py sed -i '/standalone_javadoc_gen_dir,/d' docs/apis/generate_api_docs.py sed -i '/flink_javadoc_gen_dir,/d' docs/apis/generate_api_docs.py - name: Generate API docs shell: bash -el {0} run: python3 docs/generate_docs.py --api-docs env: _DELTA_LAKE_RELEASE_VERSION_: ${{ matrix.version.name }} - name: Move doc contents up one level run: | find docs/apis/_site/api -type d \( -name "unidoc" -o -name "javaunidoc" -o -name "html" \) | while read dir; do echo "Processing $dir" parent="$(dirname "$dir")" # Move all files (including hidden ones) using find find "$dir" -maxdepth 1 -type f -exec mv {} "$parent"/ \; # Move all subdirectories find "$dir" -maxdepth 1 -type d ! -path "$dir" -exec mv {} "$parent"/ \; rmdir "$dir" done - name: Upload artifact uses: actions/upload-artifact@v4 with: name: ${{ github.run_id }}-apidocs-${{ matrix.version.name }} path: ${{ matrix.version.out_dir }} build_site: name: Build site needs: build_api_docs runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Install Node.js uses: actions/setup-node@v4 with: node-version-file: docs/.nvmrc - uses: pnpm/action-setup@v4 name: Install pnpm with: package_json_file: docs/package.json - name: Install Node.js dependencies run: pnpm install working-directory: docs - name: Download API docs artifacts uses: actions/download-artifact@v4 with: pattern: ${{ github.run_id }}-apidocs-* path: docs/public/api - name: Rename API docs artifact folders run: | for d in docs/public/api/${{ github.run_id }}-apidocs-*; do [ -d "$d" ] || continue new_name="$(echo "$d" | sed "s|docs/public/api/${{ github.run_id }}-apidocs-||")" mv "$d" "docs/public/api/$new_name" done - name: Generate docs site run: python3 docs/generate_docs.py - name: Install Netlify CLI run: pnpm i -g netlify-cli - name: Publish site to Netlify run: netlify deploy --dir=docs/dist --prod env: NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} NETLIFY_SITE_ID: ${{ vars.NETLIFY_SITE_ID }} ================================================ FILE: .github/workflows/spark_examples_test.yaml ================================================ name: "Delta Spark Publishing and Examples" on: push: paths-ignore: - '**.md' - '**.txt' pull_request: paths-ignore: - '**.md' - '**.txt' jobs: # Generate Spark versions matrix from CrossSparkVersions.scala # This workflow tests against released versions only (no snapshots) generate-matrix: name: "Generate Released Spark Versions Matrix" runs-on: ubuntu-24.04 outputs: spark_versions: ${{ steps.generate.outputs.spark_versions }} steps: - uses: actions/checkout@v3 - name: install java uses: actions/setup-java@v3 with: distribution: "zulu" java-version: "17" - name: Generate released Spark versions matrix id: generate run: | # Get only released versions (exclude snapshots) SPARK_VERSIONS=$(python3 project/scripts/get_spark_version_info.py --released-spark-versions) echo "spark_versions=$SPARK_VERSIONS" >> $GITHUB_OUTPUT echo "Generated released Spark versions: $SPARK_VERSIONS" test: name: "DSP&E: Spark ${{ matrix.spark_version }}, Scala ${{ matrix.scala }}" runs-on: ubuntu-24.04 needs: generate-matrix strategy: matrix: # Spark versions are dynamically generated - released versions only spark_version: ${{ fromJson(needs.generate-matrix.outputs.spark_versions) }} # These Scala versions must match those in the build.sbt scala: [2.13.17] env: SCALA_VERSION: ${{ matrix.scala }} steps: - uses: actions/checkout@v3 - name: Get Spark version details id: spark-details run: | # Get JVM version, package suffix, iceberg support, and full version for this Spark version JVM_VERSION=$(python3 project/scripts/get_spark_version_info.py --get-field "${{ matrix.spark_version }}" targetJvm | jq -r) SPARK_PACKAGE_SUFFIX=$(python3 project/scripts/get_spark_version_info.py --get-field "${{ matrix.spark_version }}" packageSuffix | jq -r) SUPPORT_ICEBERG=$(python3 project/scripts/get_spark_version_info.py --get-field "${{ matrix.spark_version }}" supportIceberg | jq -r) SPARK_FULL_VERSION=$(python3 project/scripts/get_spark_version_info.py --get-field "${{ matrix.spark_version }}" fullVersion | jq -r) echo "jvm_version=$JVM_VERSION" >> $GITHUB_OUTPUT echo "spark_package_suffix=$SPARK_PACKAGE_SUFFIX" >> $GITHUB_OUTPUT echo "support_iceberg=$SUPPORT_ICEBERG" >> $GITHUB_OUTPUT echo "spark_full_version=$SPARK_FULL_VERSION" >> $GITHUB_OUTPUT echo "Using JVM $JVM_VERSION for Spark $SPARK_FULL_VERSION, package suffix: '$SPARK_PACKAGE_SUFFIX', support iceberg: '$SUPPORT_ICEBERG'" - name: install java uses: actions/setup-java@v3 with: distribution: "zulu" java-version: ${{ steps.spark-details.outputs.jvm_version }} - name: Cache Scala, SBT uses: actions/cache@v3 with: path: | ~/.sbt ~/.ivy2 ~/.cache/coursier # Change the key if dependencies are changed. For each key, GitHub Actions will cache the # the above directories when we use the key for the first time. After that, each run will # just use the cache. The cache is immutable so we need to use a new key when trying to # cache new stuff. key: delta-sbt-cache-spark${{ matrix.spark_version }}-scala${{ matrix.scala }} - name: Install Job dependencies run: | sudo apt-get update sudo apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python3-openssl git sudo apt install libedit-dev - name: Run Delta Spark Local Publishing and Examples Compilation # examples/scala/build.sbt will compile against the local Delta release version (e.g. 3.2.0-SNAPSHOT). # Thus, we need to publishM2 first so those jars are locally accessible. # -DsparkVersion is for the Delta project's publishM2 (which Spark version to compile Delta against). # SPARK_VERSION/SPARK_PACKAGE_SUFFIX/SUPPORT_ICEBERG are for examples/scala/build.sbt (dependency resolution). env: SPARK_PACKAGE_SUFFIX: ${{ steps.spark-details.outputs.spark_package_suffix }} SUPPORT_ICEBERG: ${{ steps.spark-details.outputs.support_iceberg }} SPARK_VERSION: ${{ steps.spark-details.outputs.spark_full_version }} run: | build/sbt clean build/sbt -DsparkVersion=${{ steps.spark-details.outputs.spark_full_version }} publishM2 cd examples/scala && build/sbt "++ $SCALA_VERSION compile" - name: Run UC Delta Integration Test # Verifies that delta-spark resolved from Maven local includes all kernel module # dependencies transitively by running a real UC-backed Delta workload. env: SPARK_PACKAGE_SUFFIX: ${{ steps.spark-details.outputs.spark_package_suffix }} SPARK_VERSION: ${{ steps.spark-details.outputs.spark_full_version }} run: | cd examples/scala && build/sbt "++ $SCALA_VERSION runMain example.UnityCatalogQuickstart" ================================================ FILE: .github/workflows/spark_python_test.yaml ================================================ name: "Delta Spark Python" on: push: paths-ignore: - '**.md' - '**.txt' pull_request: paths-ignore: - '**.md' - '**.txt' jobs: # Generate Spark versions matrix from CrossSparkVersions.scala # This workflow tests against released versions only (no snapshots) generate-matrix: name: "Generate Released Spark Versions Matrix" runs-on: ubuntu-24.04 outputs: spark_versions: ${{ steps.generate.outputs.spark_versions }} steps: - uses: actions/checkout@v3 - name: install java uses: actions/setup-java@v3 with: distribution: "zulu" java-version: "17" - name: Generate released Spark versions matrix id: generate run: | # Get only released versions (exclude snapshots) SPARK_VERSIONS=$(python3 project/scripts/get_spark_version_info.py --released-spark-versions) echo "spark_versions=$SPARK_VERSIONS" >> $GITHUB_OUTPUT echo "Generated released Spark versions: $SPARK_VERSIONS" test: name: "DSP (${{ matrix.spark_version }})" runs-on: ubuntu-24.04 needs: generate-matrix strategy: matrix: # Spark versions are dynamically generated - released versions only spark_version: ${{ fromJson(needs.generate-matrix.outputs.spark_versions) }} # These Scala versions must match those in the build.sbt scala: [2.13.16] env: SCALA_VERSION: ${{ matrix.scala }} SPARK_VERSION: ${{ matrix.spark_version }} steps: - uses: actions/checkout@v3 - name: Get Spark version details id: spark-details run: | # Get JVM version and full Spark version for this matrix entry JVM_VERSION=$(python3 project/scripts/get_spark_version_info.py --get-field "${{ matrix.spark_version }}" targetJvm | jq -r) FULL_VERSION=$(python3 project/scripts/get_spark_version_info.py --get-field "${{ matrix.spark_version }}" fullVersion | jq -r) echo "jvm_version=$JVM_VERSION" >> $GITHUB_OUTPUT echo "spark_full_version=$FULL_VERSION" >> $GITHUB_OUTPUT echo "Using JVM $JVM_VERSION for Spark ${{ matrix.spark_version }} (full: $FULL_VERSION)" - name: install java uses: actions/setup-java@v3 with: distribution: "zulu" java-version: ${{ steps.spark-details.outputs.jvm_version }} - name: Cache Scala, SBT uses: actions/cache@v3 with: path: | ~/.sbt ~/.ivy2 ~/.cache/coursier # Change the key if dependencies are changed. For each key, GitHub Actions will cache the # the above directories when we use the key for the first time. After that, each run will # just use the cache. The cache is immutable so we need to use a new key when trying to # cache new stuff. key: delta-sbt-cache-spark${{ matrix.spark_version }}-scala${{ matrix.scala }} - name: Install Job dependencies run: | sudo apt-get update sudo apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python3-openssl git sudo apt install libedit-dev curl -LO https://github.com/bufbuild/buf/releases/download/v1.28.1/buf-Linux-x86_64.tar.gz mkdir -p ~/buf tar -xvzf buf-Linux-x86_64.tar.gz -C ~/buf --strip-components 1 rm buf-Linux-x86_64.tar.gz sudo apt install python3-pip --fix-missing sudo pip3 install pipenv==2024.4.1 curl https://pyenv.run | bash export PATH="~/.pyenv/bin:$PATH" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" pyenv install 3.10 pyenv global system 3.10 pipenv --python 3.10 install # Update the pip version to 24.0. By default `pyenv.run` installs the latest pip version # available. From version 24.1, `pip` doesn't allow installing python packages # with version string containing `-`. In Delta-Spark case, the pypi package generated has # `-SNAPSHOT` in version (e.g. `3.3.0-SNAPSHOT`) as the version is picked up from # the`version.sbt` file. pipenv run pip install pip==24.0 setuptools==69.5.1 wheel==0.43.0 # Install pyspark matching the full spark version pipenv run pip install pyspark==${{ steps.spark-details.outputs.spark_full_version }} pipenv run pip install flake8==3.9.0 pipenv run pip install black==23.12.1 pipenv run pip install importlib_metadata==3.10.0 pipenv run pip install mypy==1.8.0 pipenv run pip install mypy-protobuf==3.3.0 pipenv run pip install cryptography==37.0.4 pipenv run pip install twine==4.0.1 pipenv run pip install wheel==0.33.4 pipenv run pip install setuptools==41.1.0 pipenv run pip install pydocstyle==3.0.0 pipenv run pip install pandas==2.2.0 pipenv run pip install pyarrow==15.0.0 pipenv run pip install pypandoc==1.3.3 pipenv run pip install numpy==1.22.4 pipenv run pip install googleapis-common-protos-stubs==2.2.0 pipenv run pip install grpc-stubs==1.24.11 # Version-specific dependencies for Spark Connect compatibility if [[ "${{ matrix.spark_version }}" == "4.0" ]]; then pipenv run pip install grpcio==1.67.0 pipenv run pip install grpcio-status==1.67.0 pipenv run pip install googleapis-common-protos==1.65.0 pipenv run pip install protobuf==5.29.1 else # Spark 4.1+ requirements from https://github.com/apache/spark/blob/branch-4.1/dev/requirements.txt pipenv run pip install grpcio==1.76.0 pipenv run pip install grpcio-status==1.76.0 pipenv run pip install googleapis-common-protos==1.71.0 pipenv run pip install protobuf==6.33.0 pipenv run pip install zstandard==0.25.0 fi - name: Run Python tests # when changing TEST_PARALLELISM_COUNT make sure to also change it in spark_test.yaml run: | TEST_PARALLELISM_COUNT=4 pipenv run python run-tests.py --group spark-python --spark-version ${{ matrix.spark_version }} ================================================ FILE: .github/workflows/spark_test.yaml ================================================ name: "Delta Spark" on: push: paths-ignore: - '**.md' - '**.txt' pull_request: paths-ignore: - '**.md' - '**.txt' jobs: # Generate Spark versions matrix from CrossSparkVersions.scala # This ensures the workflow always uses the versions defined in the build generate-matrix: name: "Generate Spark Versions Matrix" runs-on: ubuntu-24.04 outputs: spark_versions: ${{ steps.generate.outputs.spark_versions }} steps: - uses: actions/checkout@v3 - name: install java uses: actions/setup-java@v3 with: distribution: "zulu" java-version: "17" - name: Generate Spark versions matrix id: generate run: | # The script automatically generates spark-versions.json from CrossSparkVersions.scala SPARK_VERSIONS=$(python3 project/scripts/get_spark_version_info.py --all-spark-versions) echo "spark_versions=$SPARK_VERSIONS" >> $GITHUB_OUTPUT echo "Generated Spark versions: $SPARK_VERSIONS" test: name: "DS: Spark ${{ matrix.spark_version }}, Scala ${{ matrix.scala }}, Shard ${{ matrix.shard }}" runs-on: ubuntu-24.04 needs: generate-matrix strategy: fail-fast: false matrix: # Spark versions are dynamically generated from CrossSparkVersions.scala # DO NOT hardcode versions here - they are automatically loaded from the build configuration spark_version: ${{ fromJson(needs.generate-matrix.outputs.spark_versions) }} # These Scala versions must match those in the build.sbt scala: [2.13.16] # Important: This list of shards must be [0..NUM_SHARDS - 1] shard: [0, 1, 2, 3, 4, 5, 6, 7] env: SCALA_VERSION: ${{ matrix.scala }} SPARK_VERSION: ${{ matrix.spark_version }} # Important: This must be the same as the length of shards in matrix NUM_SHARDS: 8 steps: - uses: actions/checkout@v3 - name: Get Spark version details id: spark-details run: | # The script automatically generates spark-versions.json if needed JVM_VERSION=$(python3 project/scripts/get_spark_version_info.py --get-field "${{ matrix.spark_version }}" targetJvm | jq -r) echo "jvm_version=$JVM_VERSION" >> $GITHUB_OUTPUT echo "Using JVM version: $JVM_VERSION for Spark ${{ matrix.spark_version }}" - name: install java uses: actions/setup-java@v3 with: distribution: "zulu" java-version: ${{ steps.spark-details.outputs.jvm_version }} - name: Cache Scala, SBT uses: actions/cache@v3 with: path: | ~/.sbt ~/.ivy2 ~/.cache/coursier # Change the key if dependencies are changed. For each key, GitHub Actions will cache the # the above directories when we use the key for the first time. After that, each run will # just use the cache. The cache is immutable so we need to use a new key when trying to # cache new stuff. key: delta-sbt-cache-spark${{ matrix.spark_version }}-scala${{ matrix.scala }} - name: Install Job dependencies run: | sudo apt-get update sudo apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python3-openssl git sudo apt install libedit-dev curl -LO https://github.com/bufbuild/buf/releases/download/v1.28.1/buf-Linux-x86_64.tar.gz mkdir -p ~/buf tar -xvzf buf-Linux-x86_64.tar.gz -C ~/buf --strip-components 1 rm buf-Linux-x86_64.tar.gz sudo apt install python3-pip --fix-missing sudo pip3 install pipenv==2024.4.1 curl https://pyenv.run | bash export PATH="~/.pyenv/bin:$PATH" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" pyenv install 3.9 pyenv global system 3.9 pipenv --python 3.9 install # Update the pip version to 24.0. By default `pyenv.run` installs the latest pip version # available. From version 24.1, `pip` doesn't allow installing python packages # with version string containing `-`. In Delta-Spark case, the pypi package generated has # `-SNAPSHOT` in version (e.g. `3.3.0-SNAPSHOT`) as the version is picked up from # the`version.sbt` file. pipenv run pip install pip==24.0 setuptools==69.5.1 wheel==0.43.0 pipenv run pip install flake8==3.9.0 pipenv run pip install black==23.12.1 pipenv run pip install importlib_metadata==3.10.0 pipenv run pip install mypy==1.8.0 pipenv run pip install mypy-protobuf==3.3.0 pipenv run pip install cryptography==37.0.4 pipenv run pip install twine==4.0.1 pipenv run pip install wheel==0.33.4 pipenv run pip install setuptools==41.1.0 pipenv run pip install pydocstyle==3.0.0 pipenv run pip install pandas==2.2.0 pipenv run pip install pyarrow==11.0.0 pipenv run pip install pypandoc==1.3.3 pipenv run pip install numpy==1.22.4 pipenv run pip install grpcio==1.67.0 pipenv run pip install grpcio-status==1.67.0 pipenv run pip install googleapis-common-protos==1.65.0 pipenv run pip install protobuf==5.29.1 pipenv run pip install googleapis-common-protos-stubs==2.2.0 pipenv run pip install grpc-stubs==1.24.11 - name: Scala structured logging style check run: | if [ -f ./dev/spark_structured_logging_style.py ]; then python3 ./dev/spark_structured_logging_style.py fi - name: Run Scala/Java tests # when changing TEST_PARALLELISM_COUNT make sure to also change it in spark_python_test.yaml run: | TEST_PARALLELISM_COUNT=4 pipenv run python run-tests.py --group spark --shard ${{ matrix.shard }} --spark-version ${{ matrix.spark_version }} - name: Upload test reports if: always() uses: actions/upload-artifact@v4 with: name: test-reports-spark${{ matrix.spark_version }}-shard${{ matrix.shard }} path: "**/target/test-reports/*.xml" retention-days: 7 ================================================ FILE: .github/workflows/unidoc.yaml ================================================ name: "Unidoc" on: [push, pull_request] jobs: build: name: "U: Scala ${{ matrix.scala }}" runs-on: ubuntu-24.04 strategy: matrix: # These Scala versions must match those in the build.sbt scala: [2.13.16] steps: - name: install java uses: actions/setup-java@v3 with: distribution: "zulu" java-version: "17" - uses: actions/checkout@v3 - name: generate unidoc run: build/sbt "++ ${{ matrix.scala }}" unidoc ================================================ FILE: .gitignore ================================================ *#*# *.#* *.iml *.ipr *.iws *.pyc *.pyo *.swp *~ .DS_Store .ammonite .bloop .bsp .cache .classpath .ensime .ensime_cache/ .ensime_lucene .generated-mima* .idea/ .idea_modules/ .metals .project .pydevproject .scala_dependencies .settings /lib/ R-unit-tests.log R/unit-tests.out R/cran-check.out R/pkg/vignettes/sparkr-vignettes.html R/pkg/tests/fulltests/Rplots.pdf build/*.jar build/apache-maven* build/scala* build/zinc* cache checkpoint conf/*.cmd conf/*.conf conf/*.properties conf/*.sh conf/*.xml conf/java-opts dependency-reduced-pom.xml derby.log dev/create-release/*final dev/create-release/*txt dev/pr-deps/ dist/ docs/_site docs/api sql/docs sql/site lib_managed/ lint-r-report.log log/ logs/ metals.sbt out/ project/boot/ project/build/target/ project/plugins/lib_managed/ project/plugins/project/build.properties project/plugins/src_managed/ project/plugins/target/ python/lib/pyspark.zip python/deps docs/python/_static/ docs/python/_templates/ docs/python/_build/ python/test_coverage/coverage_data python/test_coverage/htmlcov python/pyspark/python reports/ scalastyle-on-compile.generated.xml scalastyle-output.xml scalastyle.txt spark-*-bin-*.tgz spark-tests.log src_managed/ streaming-tests.log target/ unit-tests.log work/ docs/.jekyll-metadata # For Hive TempStatsStore/ metastore/ metastore_db/ sql/hive-thriftserver/test_warehouses warehouse/ spark-warehouse/ # For R session data .RData .RHistory .Rhistory *.Rproj *.Rproj.* .Rproj.user **/src/main/resources/js # For SBT .jvmopts sbt-launch-*.jar # For Python linting pep8*.py pycodestyle*.py # For IDE settings .vscode # For Terraform **/.terraform/* *.tfstate *.tfstate.* crash.log crash.*.log *.tfvars *.tfvars.json override.tf override.tf.json *_override.tf *_override.tf.json .terraformrc .terraform.rc # Local Netlify folder .netlify # Ignore kernel benchmark report kernel/kernel-benchmarks/benchmark_report.json # Unity Catalog test artifacts spark/unitycatalog/etc/ .scala-build/ ================================================ FILE: .sbtopts ================================================ -J-Xmx4G ================================================ FILE: .scalafmt.conf ================================================ # Copyright (2025) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. align = none align.openParenDefnSite = false align.openParenCallSite = false align.tokens = [] importSelectors = "singleLine" optIn.configStyleArguments = false continuationIndent { callSite = 2 defnSite = 4 } danglingParentheses { defnSite = false callSite = false } docstrings { style = Asterisk wrap = no } literals.hexDigits = upper maxColumn = 100 newlines { beforeCurlyLambdaParams = false source = keep } rewrite.rules = [Imports] rewrite.imports.sort = scalastyle rewrite.imports.groups = [ ["java\\..*"], ["scala\\..*"], ["io\\.delta\\..*"], ["org\\.apache\\.spark\\.sql\\.delta.*"] ] runner.dialect = scala212 version = 3.8.6 ================================================ FILE: CODE_OF_CONDUCT.md ================================================ # Delta Lake Code of Conduct ## Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. ## Our Standards Examples of behavior that contributes to creating a positive environment include: * Using welcoming and inclusive language * Being respectful of differing viewpoints and experiences * Gracefully accepting constructive criticism * Focusing on what is best for the community * Showing empathy towards other community members Examples of unacceptable behavior by participants include: shipit * The use of sexualized language or imagery and unwelcome sexual attention or advances * Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or electronic address, without explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting shipit ## Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. ## Scope This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the Technical Steering Committee defined [here](https://github.com/delta-io/delta/blob/master/CONTRIBUTING.md#governance). All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html [homepage]: https://www.contributor-covenant.org For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq ## Linux Foundation Code of Conduct Your use is additionally subject to the [Linux Foundation Code of Conduct](https://lfprojects.org/policies/code-of-conduct/) ================================================ FILE: CONTRIBUTING.md ================================================ We happily welcome contributions to Delta Lake. We use [GitHub Issues](/../../issues/) to track community reported issues and [GitHub Pull Requests ](/../../pulls/) for accepting changes. # Governance Delta Lake is an independent open-source project and not controlled by any single company. To emphasize this we joined the [Delta Lake Project](https://community.linuxfoundation.org/delta-lake/) in 2019, which is a sub-project of the Linux Foundation Projects. Within the project, we make decisions based on [these rules](https://delta.io/pdfs/delta-charter.pdf). Delta Lake is supported by a wide set of developers from over 50 organizations across multiple repositories. Since 2019, more than 190 developers have contributed to Delta Lake! The Delta Lake community is growing by leaps and bounds with more than 6000 members in the [Delta Users slack](https://go.delta.io/slack)). For more information, please refer to the [founding technical charter](https://delta.io/pdfs/delta-charter.pdf). # Communication - Before starting work on a major feature, please reach out to us via [GitHub](https://github.com/delta-io/delta/issues), [Slack](https://go.delta.io/slack), [email](https://groups.google.com/g/delta-users), etc. We will make sure no one else is already working on it and ask you to open a GitHub issue. - A "major feature" is defined as any change that is > 100 LOC altered (not including tests), or changes any user-facing behavior. - We will use the GitHub issue to discuss the feature and come to agreement. - This is to prevent your time being wasted, as well as ours. - The GitHub review process for major features is also important so that organizations with commit access can come to agreement on design. - If it is appropriate to write a design document, the document must be hosted either in the GitHub tracking issue, or linked to from the issue and hosted in a world-readable location. Examples of design documents include [sample 1](https://docs.google.com/document/d/16S7xoAmXpSax7W1OWYYHo5nZ71t5NvrQ-F79pZF6yb8), [sample 2](https://docs.google.com/document/d/1MJhmW_H7doGWY2oty-I78vciziPzBy_nzuuB-Wv5XQ8), and [sample 3](https://docs.google.com/document/d/19CU4eJuBXOwW7FC58uSqyCbcLTsgvQ5P1zoPOPgUSpI). - Specifically, if the goal is to add a new extension, please read the extension policy. - Small patches and bug fixes don't need prior communication. If you have identified a bug and have ways to solve it, please create an [issue](https://github.com/delta-io/delta/issues) or create a [pull request](https://github.com/delta-io/delta/pulls). - If you have an example code that explains a use case or a feature, create a pull request to post under [examples](https://github.com/delta-io/delta/tree/master/examples). # Coding style We generally follow the [Apache Spark Scala Style Guide](https://spark.apache.org/contributing.html). # Sign your work The sign-off is a simple line at the end of the explanation for the patch. Your signature certifies that you wrote the patch or otherwise have the right to pass it on as an open-source patch. The rules are pretty simple: if you can certify the below (from developercertificate.org): ``` Developer Certificate of Origin Version 1.1 Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 1 Letterman Drive Suite D4700 San Francisco, CA, 94129 Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Developer's Certificate of Origin 1.1 By making a contribution to this project, I certify that: (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. ``` Then you just add a line to every git commit message: ``` Signed-off-by: Jane Smith Use your real name (sorry, no pseudonyms or anonymous contributions.) ``` If you set your `user.name` and `user.email` git configs, you can sign your commit automatically with `git commit -s`. ================================================ FILE: Dockerfile ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # FROM ubuntu:focal-20221019 ENV DEBIAN_FRONTEND noninteractive ENV DEBCONF_NONINTERACTIVE_SEEN true RUN apt-get update RUN apt-get install -y software-properties-common RUN apt-get install -y curl RUN apt-get install -y wget RUN apt-get install -y openjdk-8-jdk RUN apt-get install -y python3.8 RUN apt-get install -y python3-pip RUN apt-get install -y git # Upgrade pip. This is needed to use prebuilt wheels for packages cffi (dep of cryptography) and # cryptography. Otherwise, building wheels for these packages fails. RUN pip3 install --upgrade pip # Update the pip version to 24.0. By default `pyenv.run` installs the latest pip version # available. From version 24.1, `pip` doesn't allow installing python packages # with version string containing `-`. In Delta-Spark case, the pypi package generated has # `-SNAPSHOT` in version (e.g. `3.3.0-SNAPSHOT`) as the version is picked up from # the`version.sbt` file. RUN pip install pip==24.0 setuptools==69.5.1 wheel==0.43.0 RUN pip3 install pyspark==3.5.3 RUN pip3 install mypy==0.982 RUN pip3 install pydocstyle==3.0.0 RUN pip3 install pandas==1.0.5 RUN pip3 install pyarrow==8.0.0 RUN pip3 install numpy==1.20.3 RUN pip3 install importlib_metadata==3.10.0 RUN pip3 install cryptography==37.0.4 # We must install cryptography before twine. Else, twine will pull a newer version of # cryptography that requires a newer version of Rust and may break tests. RUN pip3 install twine==4.0.1 RUN pip3 install wheel==0.33.4 RUN pip3 install setuptools==41.0.1 # Do not add any non-deterministic changes (e.g., copy from files # from repo) in this Dockerfile, so that the docker image # generated from this can be reused across builds. ================================================ FILE: LICENSE.txt ================================================ Copyright (2021) The Delta Lake Project Authors. All rights reserved. Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS ------------------------------------------------------------------------- This project includes code derived from the Apache Spark project. The individual files containing this code carry the original Apache Spark license, which is reproduced here as well: Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: NOTICE.txt ================================================ Delta Lake Copyright (2021) The Delta Lake Project Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. This project includes software licensed by the Apache Software Foundation (Apache 2.0) from the Apache Spark project (www.github.com/apache/spark) ---------------------------------------------------------- Apache Spark Copyright 2014 and onwards The Apache Software Foundation. This product includes software developed at The Apache Software Foundation (http://www.apache.org/). ================================================ FILE: PROTOCOL.md ================================================ # Delta Transaction Log Protocol - [Overview](#overview) - [Delta Table Specification](#delta-table-specification) - [File Types](#file-types) - [Data Files](#data-files) - [Deletion Vector Files](#deletion-vector-files) - [Change Data Files](#change-data-files) - [Delta Log Entries](#delta-log-entries) - [Checkpoints](#checkpoints) - [Sidecar Files](#sidecar-files) - [Log Compaction Files](#log-compaction-files) - [Last Checkpoint File](#last-checkpoint-file) - [Version Checksum File](#version-checksum-file) - [File Size Histogram Schema](#file-size-histogram-schema) - [Actions](#actions) - [Change Metadata](#change-metadata) - [Format Specification](#format-specification) - [Add File and Remove File](#add-file-and-remove-file) - [Add CDC File](#add-cdc-file) - [Writer Requirements for AddCDCFile](#writer-requirements-for-addcdcfile) - [Reader Requirements for AddCDCFile](#reader-requirements-for-addcdcfile) - [Transaction Identifiers](#transaction-identifiers) - [Protocol Evolution](#protocol-evolution) - [Commit Provenance Information](#commit-provenance-information) - [Domain Metadata](#domain-metadata) - [Reader Requirements for Domain Metadata](#reader-requirements-for-domain-metadata) - [Writer Requirements for Domain Metadata](#writer-requirements-for-domain-metadata) - [Sidecar File Information](#sidecar-file-information) - [Checkpoint Metadata](#checkpoint-metadata) - [Action Reconciliation](#action-reconciliation) - [Table Features](#table-features) - [Table Features for New and Existing Tables](#table-features-for-new-and-existing-tables) - [Supported Features](#supported-features) - [Active Features](#active-features) - [Column Mapping](#column-mapping) - [Writer Requirements for Column Mapping](#writer-requirements-for-column-mapping) - [Reader Requirements for Column Mapping](#reader-requirements-for-column-mapping) - [Deletion Vectors](#deletion-vectors) - [Deletion Vector Descriptor Schema](#deletion-vector-descriptor-schema) - [Derived Fields](#derived-fields) - [JSON Example 1 — On Disk with Relative Path (with Random Prefix)](#json-example-1--on-disk-with-relative-path-with-random-prefix) - [JSON Example 2 — On Disk with Absolute Path](#json-example-2--on-disk-with-absolute-path) - [JSON Example 3 — Inline](#json-example-3--inline) - [Reader Requirements for Deletion Vectors](#reader-requirements-for-deletion-vectors) - [Writer Requirement for Deletion Vectors](#writer-requirement-for-deletion-vectors) - [Iceberg Compatibility V1](#iceberg-compatibility-v1) - [Writer Requirements for IcebergCompatV1](#writer-requirements-for-icebergcompatv1) - [Iceberg Compatibility V2](#iceberg-compatibility-v2) - [Writer Requirement for IcebergCompatV2](#iceberg-compatibility-v2) - [Timestamp without timezone (TimestampNtz)](#timestamp-without-timezone-timestampntz) - [V2 Checkpoint Table Feature](#v2-checkpoint-table-feature) - [Row Tracking](#row-tracking) - [Row IDs](#row-ids) - [Row Commit Versions](#row-commit-versions) - [Reader Requirements for Row Tracking](#reader-requirements-for-row-tracking) - [Writer Requirements for Row Tracking](#writer-requirements-for-row-tracking) - [VACUUM Protocol Check](#vacuum-protocol-check) - [Writer Requirements for Vacuum Protocol Check](#writer-requirements-for-vacuum-protocol-check) - [Reader Requirements for Vacuum Protocol Check](#reader-requirements-for-vacuum-protocol-check) - [Clustered Table](#clustered-table) - [Writer Requirements for Clustered Table](#writer-requirements-for-clustered-table) - [Variant Data Type](#variant-data-type) - [Variant data in Parquet](#variant-data-in-parquet) - [Writer Requirements for Variant Type](#writer-requirements-for-variant-type) - [Reader Requirements for Variant Data Type](#reader-requirements-for-variant-data-type) - [Compatibility with other Delta Features](#compatibility-with-other-delta-features) - [Catalog-managed tables](#catalog-managed-tables) - [Terminology: Commits](#terminology-commits) - [Terminology: Delta Client](#terminology-delta-client) - [Terminology: Catalogs](#terminology-catalogs) - [Catalog Responsibilities](#catalog-responsibilities) - [Reading Catalog-managed Tables](#reading-catalog-managed-tables) - [Commit Protocol](#commit-protocol) - [Getting Ratified Commits from the Catalog](#getting-ratified-commits-from-the-catalog) - [Publishing Commits](#publishing-commits) - [Maintenance Operations on Catalog-managed Tables](#maintenance-operations-on-catalog-managed-tables) - [Creating and Dropping Catalog-managed Tables](#creating-and-dropping-catalog-managed-tables) - [Catalog-managed Table Enablement](#catalog-managed-table-enablement) - [Writer Requirements for Catalog-managed tables](#writer-requirements-for-catalog-managed-tables) - [Reader Requirements for Catalog-managed tables](#reader-requirements-for-catalog-managed-tables) - [Table Discovery](#table-discovery) - [Sample Catalog Client API](#sample-catalog-client-api) - [Requirements for Writers](#requirements-for-writers) - [Creation of New Log Entries](#creation-of-new-log-entries) - [Consistency Between Table Metadata and Data Files](#consistency-between-table-metadata-and-data-files) - [Delta Log Entries](#delta-log-entries-1) - [Checkpoints](#checkpoints-1) - [Checkpoint Specs](#checkpoint-specs) - [V2 Spec](#v2-spec) - [V1 Spec](#v1-spec) - [Checkpoint Naming Scheme](#checkpoint-naming-scheme) - [UUID-named checkpoint](#uuid-named-checkpoint) - [Classic checkpoint](#classic-checkpoint) - [Multi-part checkpoint](#multi-part-checkpoint) - [Problems with multi-part checkpoints](#problems-with-multi-part-checkpoints) - [Handling Backward compatibility while moving to UUID-named v2 Checkpoints](#handling-backward-compatibility-while-moving-to-uuid-named-v2-checkpoints) - [Allowed combinations for `checkpoint spec` <-> `checkpoint file naming`](#allowed-combinations-for-checkpoint-spec---checkpoint-file-naming) - [Metadata Cleanup](#metadata-cleanup) - [Data Files](#data-files-1) - [Append-only Tables](#append-only-tables) - [Column Invariants](#column-invariants) - [CHECK Constraints](#check-constraints) - [Generated Columns](#generated-columns) - [Default Columns](#default-columns) - [Identity Columns](#identity-columns) - [Writer Version Requirements](#writer-version-requirements) - [Requirements for Readers](#requirements-for-readers) - [Reader Version Requirements](#reader-version-requirements) - [Appendix](#appendix) - [Valid Feature Names in Table Features](#valid-feature-names-in-table-features) - [Deletion Vector Format](#deletion-vector-format) - [Deletion Vector File Storage Format](#deletion-vector-file-storage-format) - [Per-file Statistics](#per-file-statistics) - [Partition Value Serialization](#partition-value-serialization) - [Schema Serialization Format](#schema-serialization-format) - [Primitive Types](#primitive-types) - [Struct Type](#struct-type) - [Struct Field](#struct-field) - [Array Type](#array-type) - [Map Type](#map-type) - [Variant Type](#variant-type) - [Column Metadata](#column-metadata) - [Example](#example) - [Checkpoint Schema](#checkpoint-schema) - [Last Checkpoint File Schema](#last-checkpoint-file-schema) - [JSON checksum](#json-checksum) - [How to URL encode keys and string values](#how-to-url-encode-keys-and-string-values) - [Delta Data Type to Parquet Type Mappings](#delta-data-type-to-parquet-type-mappings) # Overview This document is a specification for the Delta Transaction Protocol, which brings [ACID](https://en.wikipedia.org/wiki/ACID) properties to large collections of data, stored as files, in a distributed file system or object store. The protocol was designed with the following goals in mind: - **Serializable ACID Writes** - multiple writers can concurrently modify a Delta table while maintaining ACID semantics. - **Snapshot Isolation for Reads** - readers can read a consistent snapshot of a Delta table, even in the face of concurrent writes. - **Scalability to billions of partitions or files** - queries against a Delta table can be planned on a single machine or in parallel. - **Self describing** - all metadata for a Delta table is stored alongside the data. This design eliminates the need to maintain a separate metastore just to read the data and also allows static tables to be copied or moved using standard filesystem tools. - **Support for incremental processing** - readers can tail the Delta log to determine what data has been added in a given period of time, allowing for efficient streaming. Delta's transactions are implemented using multi-version concurrency control (MVCC). As a table changes, Delta's MVCC algorithm keeps multiple copies of the data around rather than immediately replacing files that contain records that are being updated or removed. Readers of the table ensure that they only see one consistent _snapshot_ of a table at time by using the _transaction log_ to selectively choose which _data files_ to process. Writers modify the table in two phases: First, they optimistically write out new data files or updated copies of existing ones. Then, they _commit_, creating the latest _atomic version_ of the table by adding a new entry to the log. In this log entry they record which data files to logically add and remove, along with changes to other metadata about the table. Data files that are no longer present in the latest version of the table can be lazily deleted by the vacuum command after a user-specified retention period (default 7 days). # Delta Table Specification A table has a single serial history of atomic versions, which are named using contiguous, monotonically-increasing integers. The state of a table at a given version is called a _snapshot_ and is defined by the following properties: - **Delta log protocol** consists of two **protocol versions**, and if applicable, corresponding **table features**, that are required to correctly read or write the table - **Reader features** only exists when Reader Version is 3 - **Writer features** only exists when Writer Version is 7 - **Metadata** of the table (e.g., the schema, a unique identifier, partition columns, and other configuration properties) - **Set of files** present in the table, along with metadata about those files - **Set of tombstones** for files that were recently deleted - **Set of applications-specific transactions** that have been successfully committed to the table ## File Types A Delta table is stored within a directory and is composed of the following different types of files. Here is an example of a Delta table with four entries in the commit log, stored in the directory `mytable`. ``` /mytable/_delta_log/00000000000000000042.json /mytable/_delta_log/00000000000000000042.checkpoint.parquet /mytable/_delta_log/00000000000000000043.json /mytable/_delta_log/00000000000000000044.json /mytable/_delta_log/00000000000000000045.json /mytable/_delta_log/_last_checkpoint /mytable/_change_data/cdc-00000-924d9ac7-21a9-4121-b067-a0a6517aa8ed.c000.snappy.parquet /mytable/part-00000-3935a07c-416b-4344-ad97-2a38342ee2fc.c000.snappy.parquet /mytable/deletion_vector-0c6cbaaf-5e04-4c9d-8959-1088814f58ef.bin ``` This example represents a table after [metadata cleanup](#metadata-cleanup) has removed older log entries. The checkpoint at version 42 contains the complete table state, while versions 43-45 are subsequent commits. Each file type is described in the sections below. ### Data Files Data files can be stored in the root directory of the table or in any non-hidden subdirectory (i.e., one whose name does not start with an `_`). By default, the reference implementation stores data files in directories that are named based on the partition values for data in that file (i.e. `part1=value1/part2=value2/...`). This directory format is only used to follow existing conventions and is not required by the protocol. Actual partition values for a file must be read from the transaction log. ### Deletion Vector Files Deletion Vector (DV) files are stored in the root directory of the table alongside the data files. A DV file contains one or more serialised DV, each describing the set of *invalidated* (or "soft deleted") rows for a particular data file it is associated with. For data with partition values, DV files are *not* kept in the same directory hierarchy as data files, as each one can contain DVs for files from multiple partitions. DV files store DVs in a [binary format](#deletion-vector-format). ### Change Data Files Change data files are stored in a directory at the root of the table named `_change_data`, and represent the changes for the table version they are in. For data with partition values, it is recommended that the change data files are stored within the `_change_data` directory in their respective partitions (i.e. `_change_data/part1=value1/...`). Writers can _optionally_ produce these change data files as a consequence of operations that change underlying data, like `UPDATE`, `DELETE`, and `MERGE` operations to a Delta Lake table. If an operation only adds new data or removes existing data without updating any existing rows, a writer can write only data files and commit them in `add` or `remove` actions without duplicating the data into change data files. When available, change data readers should use the change data files instead of computing changes from the underlying data files. In addition to the data columns, change data files contain additional columns that identify the type of change event: Field Name | Data Type | Description -|-|- _change_type|`String`| `insert`, `update_preimage` , `update_postimage`, `delete` __(1)__ __(1)__ `preimage` is the value before the update, `postimage` is the value after the update. ### Delta Log Entries Delta Log Entries, also known as Delta files, are JSON files stored in the `_delta_log` directory at the root of the table. Together with checkpoints, they make up the log of all changes that have occurred to a table. Delta files are the unit of atomicity for a table, and are named using the next available version number, zero-padded to 20 digits. For example: ``` ./_delta_log/00000000000000000000.json ``` Delta files use newline-delimited JSON format, where every action is stored as a single-line JSON document. A Delta file, corresponding to version `v`, contains an atomic set of [_actions_](#actions) that should be applied to the previous table state corresponding to version `v-1`, in order to construct the `v`th snapshot of the table. An action changes one aspect of the table's state, for example, adding or removing a file. **Note:** If the [catalogManaged table feature](#catalog-managed-tables) is enabled on the table, recently [ratified commits](#ratified-commit) may not yet be published to the `_delta_log` directory as normal Delta files - they may be stored directly by the catalog or reside in the `_delta_log/_staged_commits` directory. Delta clients must contact the table's managing catalog in order to find the information about these [ratified, potentially-unpublished commits](#publishing-commits). The `_delta_log/_staged_commits` directory is the staging area for [staged](#staged-commit) commits. Delta files in this directory have a UUID embedded into them and follow the pattern `..json`, where the version corresponds to the proposed commit version, zero-padded to 20 digits. For example: ``` ./_delta_log/_staged_commits/00000000000000000000.3a0d65cd-4056-49b8-937b-95f9e3ee90e5.json ./_delta_log/_staged_commits/00000000000000000001.7d17ac10-5cc3-401b-bd1a-9c82dd2ea032.json ./_delta_log/_staged_commits/00000000000000000001.016ae953-37a9-438e-8683-9a9a4a79a395.json ./_delta_log/_staged_commits/00000000000000000002.3ae45b72-24e1-865a-a211-34987ae02f2a.json ``` NOTE: The (proposed) version number of a staged commit is authoritative - file `00000000000000000100..json` always corresponds to a commit attempt for version 100. Besides simplifying implementations, it also acknowledges the fact that commit files cannot safely be reused for multiple commit attempts. For example, resolving conflicts in a table with [row tracking](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#row-tracking) enabled requires rewriting all file actions to update their `baseRowId` field. The [catalog](#terminology-catalogs) is the source of truth about which staged commit files in the `_delta_log/_staged_commits` directory correspond to ratified versions, and Delta clients should not attempt to directly interpret the contents of that directory. Refer to [catalog-managed tables](#catalog-managed-tables) for more details. ### Checkpoints Checkpoints are also stored in the `_delta_log` directory, and can be created at any time, for any committed version of the table. For performance reasons, readers should prefer to use the newest complete checkpoint possible. For time travel, the checkpoint used must not be newer than the time travel version. A checkpoint contains the complete replay of all actions, up to and including the checkpointed table version, with invalid actions removed. Invalid actions are those that have been canceled out by subsequent ones (for example removing a file that has been added), using the [rules for reconciliation](#Action-Reconciliation). In addition to above, checkpoint also contains the [_remove tombstones_](#add-file-and-remove-file) until they are expired. Checkpoints allow readers to short-cut the cost of reading the log up-to a given point in order to reconstruct a snapshot, and they also allow [Metadata cleanup](#metadata-cleanup) to delete expired JSON Delta log entries. Readers SHOULD NOT make any assumptions about the existence or frequency of checkpoints, with one exception: [Metadata cleanup](#metadata-cleanup) MUST provide a checkpoint for the oldest kept table version, to cover all deleted [Delta log entries](#delta-log-entries). That said, writers are encouraged to checkpoint reasonably frequently, so that readers do not pay excessive log replay costs due to reading large numbers of delta files. The checkpoint file name is based on the version of the table that the checkpoint contains. Delta supports three kinds of checkpoints: 1. UUID-named Checkpoints: These follow [V2 spec](#v2-spec) which uses the following file name: `n.checkpoint.u.{json/parquet}`, where `u` is a UUID and `n` is the snapshot version that this checkpoint represents. Here `n` must be zero padded to have length 20. The UUID-named V2 Checkpoint may be in json or parquet format, and references zero or more checkpoint sidecars in the `_delta_log/_sidecars` directory. A checkpoint sidecar is a uniquely-named parquet file: `{unique}.parquet` where `unique` is some unique string such as a UUID. For example: ``` 00000000000000000010.checkpoint.80a083e8-7026-4e79-81be-64bd76c43a11.json _sidecars/3a0d65cd-4056-49b8-937b-95f9e3ee90e5.parquet _sidecars/016ae953-37a9-438e-8683-9a9a4a79a395.parquet _sidecars/7d17ac10-5cc3-401b-bd1a-9c82dd2ea032.parquet ``` 2. A [classic checkpoint](#classic-checkpoint) for version `n` of the table consists of a file named `n.checkpoint.parquet`. Here `n` must be zero padded to have length 20. These could follow either [V1 spec](#v1-spec) or [V2 spec](#v2-spec). For example: ``` 00000000000000000010.checkpoint.parquet ``` 3. A [multi-part checkpoint](#multi-part-checkpoint) for version `n` consists of `p` "part" files (`p > 1`), where part `o` of `p` is named `n.checkpoint.o.p.parquet`. Here `n` must be zero padded to have length 20, while `o` and `p` must be zero padded to have length 10. These are always [V1 checkpoints](#v1-spec). For example: ``` 00000000000000000010.checkpoint.0000000001.0000000003.parquet 00000000000000000010.checkpoint.0000000002.0000000003.parquet 00000000000000000010.checkpoint.0000000003.0000000003.parquet ``` A writer can choose to write checkpoints with following constraints: - Writers are always allowed create a [classic checkpoint](#classic-checkpoint) following [v1 spec](#v1-spec). - Writers are forbidden to create [multi-part checkpoints](#multi-part-checkpoint) if [v2 checkpoints](#v2-checkpoint-table-feature) are enabled. - Writers are allowed to create v2 spec checkpoints (either [classic](#classic-checkpoint) or [uuid-named](#uuid-named-checkpoint)) if [v2 checkpoint table feature](#v2-checkpoint-table-feature) is enabled. Multi-part checkpoints are [deprecated](#problems-with-multi-part-checkpoints), and writers should avoid creating them. Use uuid-named [V2 spec](#v2-spec) checkpoints instead of these. Multiple checkpoints could exist for the same table version, e.g. if two clients race to create checkpoints at the same time, but with different formats. In such cases, a client can choose which checkpoint to use. Because a multi-part checkpoint cannot be created atomically (e.g. vulnerable to slow and/or failed writes), readers must ignore multi-part checkpoints with missing parts. Checkpoints for a given version must only be created after the associated delta file has been successfully written. #### Sidecar Files A sidecar file contains file actions. These files are in parquet format and they must have unique names. These are then [linked](#sidecar-file-information) to checkpoints. Refer to [V2 checkpoint spec](#v2-spec) for more detail. The sidecar files can have only [add file and remove file](#Add-File-and-Remove-File) entries as of now. The add and remove file actions are stored as their individual columns in parquet as struct fields. These files reside in the `_delta_log/_sidecars` directory. ### Log Compaction Files Log compaction files reside in the `_delta_log` directory. A log compaction file from a start version `x` to an end version `y` (`y` must be _greater_ than `x`) will have the following name: `..compacted.json`. This contains the aggregated actions for commit range `[x, y]`. Similar to commits, each row in the log compaction file represents an [action](#actions). The commit files for a given range are created by doing [Action Reconciliation](#action-reconciliation) of the corresponding commits. Instead of reading the individual commit files in range `[x, y]`, an implementation could choose to read the log compaction file `..compacted.json` to speed up the snapshot construction. Example: Suppose we have `00000000000000000004.json` as: ``` {"commitInfo":{...}} {"add":{"path":"f2",...}} {"remove":{"path":"f1",...}} ``` `00000000000000000005.json` as: ``` {"commitInfo":{...}} {"add":{"path":"f3",...}} {"add":{"path":"f4",...}} {"txn":{"appId":"3ae45b72-24e1-865a-a211-34987ae02f2a","version":4389}} ``` `00000000000000000006.json` as: ``` {"commitInfo":{...}} {"remove":{"path":"f3",...}} {"txn":{"appId":"3ae45b72-24e1-865a-a211-34987ae02f2a","version":4390}} ``` Then `00000000000000000004.00000000000000000006.compacted.json` will have the following content: ``` {"add":{"path":"f2",...}} {"add":{"path":"f4",...}} {"remove":{"path":"f1",...}} {"remove":{"path":"f3",...}} {"txn":{"appId":"3ae45b72-24e1-865a-a211-34987ae02f2a","version":4390}} ``` Writers: - Can optionally produce log compactions for any given commit range Readers: - Can optionally consume log compactions, if available - The compaction replaces the corresponding commits during action reconciliation ### Last Checkpoint File The Delta transaction log will often contain many (e.g. 10,000+) files. Listing such a large directory can be prohibitively expensive. The last checkpoint file can help reduce the cost of constructing the latest snapshot of the table by providing a pointer to near the end of the log. Rather than list the entire directory, readers can locate a recent checkpoint by looking at the `_delta_log/_last_checkpoint` file. Due to the zero-padded encoding of the files in the log, the version id of this recent checkpoint can be used on storage systems that support lexicographically-sorted, paginated directory listing to enumerate any delta files or newer checkpoints that comprise more recent versions of the table. ### Version Checksum File The Delta transaction log must remain an append-only log. To enable the detection of non-compliant modifications to Delta files, writers can optionally emit an auxiliary file with every commit, which contains important information about the state of the table as of that version. This file is referred to as the **Version Checksum** and can be used to validate the integrity of the table. ### Version Checksum File Schema A Version Checksum file must have the following properties: - Be named `{version}.crc` where `version` is zero-padded to 20 digits (e.g., `00000000000000000001.crc`) - Be stored directly in the `_delta_log` directory alongside Delta log files - Contain exactly one JSON object with the following schema: Field Name | Data Type | Description | optional/required -|-|-|- txnId | String | A unique identifier for the transaction that produced this commit. | optional tableSizeBytes | Long | Total size of the table in bytes, calculated as the sum of the `size` field of all live `add` actions. | required numFiles | Long | Number of live `add` actions in this table version after Action Reconciliation. | required numMetadata | Long | Number of `metaData` actions. Must be 1. | required numProtocol | Long | Number of `protocol` actions. Must be 1. | required inCommitTimestampOpt | Long | The in-commit timestamp of this version. Present if and only if [In-Commit Timestamps](#in-commit-timestamps) are enabled. | optional setTransactions | Array[`txn`] | Live [Transaction Identifier](#transaction-identifiers) actions at this version. | optional domainMetadata | Array[`domainMetadata`] | Live [Domain Metadata](#domain-metadata) actions at this version, excluding tombstones. | optional metadata | Metadata | The table [metadata](#change-metadata) at this version. | required protocol | Protocol | The table [protocol](#protocol-evolution) at this version. | required fileSizeHistogram | FileSizeHistogram | Size distribution information of files remaining after [Action Reconciliation](#action-reconciliation). See [FileSizeHistogram](#file-size-histogram-schema) for more details. | optional allFiles | Array[`add`] | All live [Add File](#add-file-and-remove-file) actions at this version. | optional numDeletedRecordsOpt | Long | Number of records deleted through Deletion Vectors in this table version. | optional numDeletionVectorsOpt | Long | Number of Deletion Vectors active in this table version. | optional deletedRecordCountsHistogramOpt | DeletedRecordCountsHistogram | Distribution of deleted record counts across files. See [this](#deleted-record-counts-histogram-schema) section for more details. | optional ##### File Size Histogram Schema The `FileSizeHistogram` object represents a histogram tracking file counts and total bytes across different size ranges. It has the following schema: Field Name | Data Type | Description | optional/required -|-|-|- sortedBinBoundaries | Array[Long] | A sorted array of bin boundaries where each element represents the start of a bin (inclusive) and the next element represents the end of the bin (exclusive). The first element must be 0. | required fileCounts | Array[Long] | Count of files in each bin. Length must match `sortedBinBoundaries`. | required totalBytes | Array[Long] | Total bytes of files in each bin. Length must match `sortedBinBoundaries`. | required Each index `i` in these arrays corresponds to a size range from `sortedBinBoundaries[i]` (inclusive) up to but not including `sortedBinBoundaries[i+1]`. The last bin ends at positive infinity. For example, given boundaries `[0, 1024, 4096]`: - Bin 0 contains files of size [0, 1024) bytes - Bin 1 contains files of size [1024, 4096) bytes - Bin 2 contains files of size [4096, ∞) bytes The arrays `fileCounts` and `totalBytes` store the number of files and their total size respectively that fall into each bin. This data structure enables efficient analysis of file size distributions in Delta tables. ### Deleted Record Counts Histogram Schema The `DeletedRecordCountsHistogram` object represents a histogram tracking the distribution of deleted record counts across files in the table. Each bin in the histogram represents a range of deletion counts and stores the number of files having that many deleted records. Field Name | Data Type | Description | optional/required -|-|-|- deletedRecordCounts | Array[Long] | Array of size 10 where each element represents the count of files falling into a specific deletion count range. | required The histogram bins correspond to the following ranges: - Bin 0: [0, 0] (files with no deletions) - Bin 1: [1, 9] (files with 1-9 deleted records) - Bin 2: [10, 99] (files with 10-99 deleted records) - Bin 3: [100, 999] (files with 100-999 deleted records) - Bin 4: [1000, 9999] (files with 1,000-9,999 deleted records) - Bin 5: [10000, 99999] (files with 10,000-99,999 deleted records) - Bin 6: [100000, 999999] (files with 100,000-999,999 deleted records) - Bin 7: [1000000, 9999999] (files with 1,000,000-9,999,999 deleted records) - Bin 8: [10000000, 2147483646] (files with 10,000,000 to 2147483646 (i.e. Int.MaxValue-1 in Java) deleted records) - Bin 9: [2147483647, ∞) (files with 2147483647 or more deleted records) This histogram allows analyzing the distribution of deleted records across files in a Delta table, which can be useful for monitoring and optimizing deletion patterns. #### State Validation Readers can validate table state integrity at a particular version by: 1. Reading the Version Checksum file for that version 2. Independently computing the same metrics by performing [Action Reconciliation](#action-reconciliation) on the table state 3. Comparing the computed values against those recorded in the Version Checksum If any discrepancy is found between computed and recorded values, the table state at that version should be considered potentially corrupted. ### Writer Requirements - Writers SHOULD produce a Version Checksum file for each commit - Writers MUST ensure all metrics in the Version Checksum accurately reflect table state after Action Reconciliation - Writers MUST write the Version Checksum file only after successfully writing the corresponding Delta log entry - Writers MUST NOT overwrite existing Version Checksum files ### Reader Requirements - Readers MAY use Version Checksums to validate table state integrity - If performing validation, readers SHOULD verify all required fields match computed values - If validation fails, readers SHOULD surface the discrepancy to users via error messaging - Readers MUST continue functioning if Version Checksum files are missing ## Actions Actions modify the state of the table and they are stored both in delta files and in checkpoints. This section lists the space of available actions as well as their schema. ### Change Metadata The `metaData` action changes the current metadata of the table. The first version of a table must contain a `metaData` action. Subsequent` metaData` actions completely overwrite the current metadata of the table. There can be at most one metadata action in a given version of the table. Every metadata action **must** include required fields at a minimum. The schema of the `metaData` action is as follows: Field Name | Data Type | Description | optional/required -|-|-|- id|`GUID`|Unique identifier for this table | required name|`String`| User-provided identifier for this table | optional description|`String`| User-provided description for this table | optional format|[Format Struct](#Format-Specification)| Specification of the encoding for the files stored in the table | required schemaString|[Schema Struct](#Schema-Serialization-Format)| Schema of the table | required partitionColumns|`Array[String]`| An array containing the names of columns by which the data should be partitioned | required createdTime|`Option[Long]`| The time when this metadata action is created, in milliseconds since the Unix epoch | optional configuration|`Map[String, String]`| A map containing configuration options for the metadata action | required #### Format Specification Field Name | Data Type | Description -|-|- provider|`String`|Name of the encoding for files in this table options|`Map[String, String]`|A map containing configuration options for the format In the reference implementation, the provider field is used to instantiate a Spark SQL [`FileFormat`](https://github.com/apache/spark/blob/master/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormat.scala). As of Spark 2.4.3 there is built-in `FileFormat` support for `parquet`, `csv`, `orc`, `json`, and `text`. As of Delta Lake 0.3.0, user-facing APIs only allow the creation of tables where `format = 'parquet'` and `options = {}`. Support for reading other formats is present both for legacy reasons and to enable possible support for other formats in the future (See [#87](https://github.com/delta-io/delta/issues/87)). The following is an example `metaData` action: ```json { "metaData":{ "id":"af23c9d7-fff1-4a5a-a2c8-55c59bd782aa", "format":{"provider":"parquet","options":{}}, "schemaString":"...", "partitionColumns":[], "configuration":{ "appendOnly": "true" } } } ``` ### Add File and Remove File The `add` and `remove` actions are used to modify the data in a table by adding or removing individual _logical files_ respectively. Every _logical file_ of the table is represented by a path to a data file, combined with an optional Deletion Vector (DV) that indicates which rows of the data file are no longer in the table. Deletion Vectors are an optional feature, see their [reader requirements](#deletion-vectors) for details. When an `add` action is encountered for a logical file that is already present in the table, statistics and other information from the latest version should replace that from any previous version. The primary key for the entry of a logical file in the set of files is a tuple of the data file's `path` and a unique id describing the DV. If no DV is part of this logical file, then its primary key is `(path, NULL)` instead. The `remove` action includes a timestamp that indicates when the removal occurred. Physical deletion of physical files can happen lazily after some user-specified expiration time threshold. This delay allows concurrent readers to continue to execute against a stale snapshot of the data. A `remove` action should remain in the state of the table as a _tombstone_ until it has expired. A tombstone expires when *current time* (according to the node performing the cleanup) exceeds the expiration threshold added to the `remove` action timestamp. In the following statements, `dvId` can refer to either the unique id of a specific Deletion Vector (`deletionVector.uniqueId`) or to `NULL`, indicating that no rows are invalidated. Since actions within a given Delta commit are not guaranteed to be applied in order, a **valid** version is restricted to contain at most one file action *of the same type* (i.e. `add`/`remove`) for any one combination of `path` and `dvId`. Moreover, for simplicity it is required that there is at most one file action of the same type for any `path` (regardless of `dvId`). That means specifically that for any commit… - it is **legal** for the same `path` to occur in an `add` action and a `remove` action, but with two different `dvId`s. - it is **legal** for the same `path` to be added and/or removed and also occur in a `cdc` action. - it is **illegal** for the same `path` to occur twice with different `dvId`s within each set of `add` or `remove` actions. - it is **illegal** for a `path` to occur in an `add` action that already occurs with a different `dvId` in the list of `add` actions from the snapshot of the version immediately preceeding the commit, unless the commit also contains a remove for the later combination. - it is **legal** to commit an existing `path` and `dvId` combination again (this allows metadata updates). The `dataChange` flag on either an `add` or a `remove` can be set to `false` to indicate that an action when combined with other actions in the same atomic version only rearranges existing data or adds new statistics. For example, streaming queries that are tailing the transaction log can use this flag to skip actions that would not affect the final results. The schema of the `add` action is as follows: Field Name | Data Type | Description | optional/required -|-|-|- path| String | A relative path to a data file from the root of the table or an absolute path to a file that should be added to the table. The path is a URI as specified by [RFC 2396 URI Generic Syntax](https://www.ietf.org/rfc/rfc2396.txt), which needs to be decoded to get the data file path. | required partitionValues| Map[String, String] | A map from partition column to value for this logical file. See also [Partition Value Serialization](#Partition-Value-Serialization) | required size| Long | The size of this data file in bytes | required modificationTime | Long | The time this logical file was created, as milliseconds since the epoch | required dataChange | Boolean | When `false` the logical file must already be present in the table or the records in the added file must be contained in one or more `remove` actions in the same version | required stats | [Statistics Struct](#Per-file-Statistics) | Contains statistics (e.g., count, min/max values for columns) about the data in this logical file | optional tags | Map[String, String] | Map containing metadata about this logical file | optional deletionVector | [DeletionVectorDescriptor Struct](#Deletion-Vectors) | Either null (or absent in JSON) when no DV is associated with this data file, or a struct (described below) that contains necessary information about the DV that is part of this logical file. | optional baseRowId | Long | Default generated Row ID of the first row in the file. The default generated Row IDs of the other rows in the file can be reconstructed by adding the physical index of the row within the file to the base Row ID. See also [Row IDs](#row-ids) | optional defaultRowCommitVersion | Long | First commit version in which an `add` action with the same `path` was committed to the table. | optional clusteringProvider | String | The name of the clustering implementation. See also [Clustered Table](#clustered-table)| optional The following is an example `add` action for a partitioned table: ```json { "add": { "path": "date=2017-12-10/part-000...c000.gz.parquet", "partitionValues": {"date": "2017-12-10"}, "size": 841454, "modificationTime": 1512909768000, "dataChange": true, "baseRowId": 4071, "defaultRowCommitVersion": 41, "stats": "{\"numRecords\":1,\"minValues\":{\"val..." } } ``` The following is an example `add` action for a clustered table: ```json { "add": { "path": "date=2017-12-10/part-000...c000.gz.parquet", "partitionValues": {}, "size": 841454, "modificationTime": 1512909768000, "dataChange": true, "baseRowId": 4071, "defaultRowCommitVersion": 41, "clusteringProvider": "liquid", "stats": "{\"numRecords\":1,\"minValues\":{\"val..." } } ``` The schema of the `remove` action is as follows: Field Name | Data Type | Description | optional/required -|-|-|- path| String | A relative path to a file from the root of the table or an absolute path to a file that should be removed from the table. The path is a URI as specified by [RFC 2396 URI Generic Syntax](https://www.ietf.org/rfc/rfc2396.txt), which needs to be decoded to get the data file path. | required deletionTimestamp | Option[Long] | The time the deletion occurred, represented as milliseconds since the epoch | optional dataChange | Boolean | When `false` the records in the removed file must be contained in one or more `add` file actions in the same version | required extendedFileMetadata | Boolean | When `true` the fields `partitionValues`, `size`, and `tags` are present | optional partitionValues| Map[String, String] | A map from partition column to value for this file. See also [Partition Value Serialization](#Partition-Value-Serialization) | optional size| Long | The size of this data file in bytes | optional stats | [Statistics Struct](#Per-file-Statistics) | Contains statistics (e.g., count, min/max values for columns) about the data in this logical file | optional tags | Map[String, String] | Map containing metadata about this file | optional deletionVector | [DeletionVectorDescriptor Struct](#Deletion-Vectors) | Either null (or absent in JSON) when no DV is associated with this data file, or a struct (described below) that contains necessary information about the DV that is part of this logical file. | optional baseRowId | Long | Default generated Row ID of the first row in the file. The default generated Row IDs of the other rows in the file can be reconstructed by adding the physical index of the row within the file to the base Row ID. See also [Row IDs](#row-ids) | optional defaultRowCommitVersion | Long | First commit version in which an `add` action with the same `path` was committed to the table | optional The following is an example `remove` action. ```json { "remove": { "path": "part-00001-9…..snappy.parquet", "deletionTimestamp": 1515488792485, "baseRowId": 4071, "defaultRowCommitVersion": 41, "dataChange": true } } ``` ### Add CDC File The `cdc` action is used to add a [file](#change-data-files) containing only the data that was changed as part of the transaction. The `cdc` action is allowed to add a [Data File](#data-files) that is also added by an `add` action, when it does not contain any copied rows and the `_change_type` column is filled for all rows. When change data readers encounter a `cdc` action in a particular Delta table version, they must read the changes made in that version exclusively using the `cdc` files. If a version has no `cdc` action, then the data in `add` and `remove` actions are read as inserted and deleted rows, respectively. The schema of the `cdc` action is as follows: Field Name | Data Type | Description -|-|- path| String | A relative path to a change data file from the root of the table or an absolute path to a change data file that should be added to the table. The path is a URI as specified by [RFC 2396 URI Generic Syntax](https://www.ietf.org/rfc/rfc2396.txt), which needs to be decoded to get the file path. partitionValues| Map[String, String] | A map from partition column to value for this file. See also [Partition Value Serialization](#Partition-Value-Serialization) size| Long | The size of this file in bytes dataChange | Boolean | Should always be set to `false` for `cdc` actions because they _do not_ change the underlying data of the table tags | Map[String, String] | Map containing metadata about this file The following is an example of `cdc` action. ```json { "cdc": { "path": "_change_data/cdc-00001-c…..snappy.parquet", "partitionValues": {}, "size": 1213, "dataChange": false } } ``` #### Writer Requirements for AddCDCFile For [Writer Versions 4 up to 6](#Writer-Version-Requirements), all writers must respect the `delta.enableChangeDataFeed` configuration flag in the metadata of the table. When `delta.enableChangeDataFeed` is `true`, writers must produce the relevant `AddCDCFile`'s for any operation that changes data, as specified in [Change Data Files](#change-data-files). For Writer Version 7, all writers must respect the `delta.enableChangeDataFeed` configuration flag in the metadata of the table only if the feature `changeDataFeed` exists in the table `protocol`'s `writerFeatures`. #### Reader Requirements for AddCDCFile When available, change data readers should use the `cdc` actions in a given table version instead of computing changes from the underlying data files referenced by the `add` and `remove` actions. Specifically, to read the row-level changes made in a version, the following strategy should be used: 1. If there are `cdc` actions in this version, then read only those to get the row-level changes, and skip the remaining `add` and `remove` actions in this version. 2. Otherwise, if there are no `cdc` actions in this version, read and treat all the rows in the `add` and `remove` actions as inserted and deleted rows, respectively. 3. Change data readers should return the following extra columns: Field Name | Data Type | Description -|-|- _commit_version|`Long`| The table version containing the change. This can be derived from the name of the Delta log file that contains actions. _commit_timestamp|`Timestamp`| The timestamp associated when the commit was created. Depending on whether [In-Commit Timestamps](#in-commit-timestamps) are enabled, this is derived from either the `inCommitTimestamp` field of the `commitInfo` action of the version's Delta log file, or from the Delta log file's modification time. ##### Note for non-change data readers In a table with Change Data Feed enabled, the data Parquet files referenced by `add` and `remove` actions are allowed to contain an extra column `_change_type`. This column is not present in the table's schema. When accessing these files, readers should disregard this column and only process columns defined within the table's schema. ### Transaction Identifiers Incremental processing systems (e.g., streaming systems) that track progress using their own application-specific versions need to record what progress has been made, in order to avoid duplicating data in the face of failures and retries during a write. Transaction identifiers allow this information to be recorded atomically in the transaction log of a delta table along with the other actions that modify the contents of the table. Transaction identifiers are stored in the form of `appId` `version` pairs, where `appId` is a unique identifier for the process that is modifying the table and `version` is an indication of how much progress has been made by that application. The atomic recording of this information along with modifications to the table enables these external system to make their writes into a Delta table _idempotent_. For example, the [Delta Sink for Apache Spark's Structured Streaming](https://github.com/delta-io/delta/blob/master/core/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSink.scala) ensures exactly-once semantics when writing a stream into a table using the following process: 1. Record in a write-ahead-log the data that will be written, along with a monotonically increasing identifier for this batch. 2. Check the current version of the transaction with `appId = streamId` in the target table. If this value is greater than or equal to the batch being written, then this data has already been added to the table and processing can skip to the next batch. 3. Write the data optimistically into the table. 4. Attempt to commit the transaction containing both the addition of the data written out and an updated `appId` `version` pair. The semantics of the application-specific `version` are left up to the external system. Delta only ensures that the latest `version` for a given `appId` is available in the table snapshot. The Delta transaction protocol does not, for example, assume monotonicity of the `version` and it would be valid for the `version` to decrease, possibly representing a "rollback" of an earlier transaction. The schema of the `txn` action is as follows: Field Name | Data Type | Description | optional/required -|-|-|- appId | String | A unique identifier for the application performing the transaction | required version | Long | An application-specific numeric identifier for this transaction | required lastUpdated | Option[Long] | The time when this transaction action is created, in milliseconds since the Unix epoch | optional The following is an example `txn` action: ```json { "txn": { "appId":"3ba13872-2d47-4e17-86a0-21afd2a22395", "version":364475 } } ``` ### Protocol Evolution The `protocol` action is used to increase the version of the Delta protocol that is required to read or write a given table. Protocol versioning allows a newer client to exclude older readers and/or writers that are missing features required to correctly interpret the transaction log. The _protocol version_ will be increased whenever non-forward-compatible changes are made to this specification. In the case where a client is running an invalid protocol version, an error should be thrown instructing the user to upgrade to a newer protocol version of their Delta client library. Since breaking changes must be accompanied by an increase in the protocol version recorded in a table or by the addition of a table feature, clients can assume that unrecognized actions, fields, and/or metadata domains are never required in order to correctly interpret the transaction log. Clients must ignore such unrecognized fields, and should not produce an error when reading a table that contains unrecognized fields. Reader Version 3 and Writer Version 7 add two lists of table features to the protocol action. The capability for readers and writers to operate on such a table is not only dependent on their supported protocol versions, but also on whether they support all features listed in `readerFeatures` and `writerFeatures`. See [Table Features](#table-features) section for more information. The schema of the `protocol` action is as follows: Field Name | Data Type | Description | optional/required -|-|-|- minReaderVersion | Int | The minimum version of the Delta read protocol that a client must implement in order to correctly *read* this table | required minWriterVersion | Int | The minimum version of the Delta write protocol that a client must implement in order to correctly *write* this table | required readerFeatures | Array[String] | A collection of features that a client must implement in order to correctly read this table (exist only when `minReaderVersion` is set to `3`) | optional writerFeatures | Array[String] | A collection of features that a client must implement in order to correctly write this table (exist only when `minWriterVersion` is set to `7`) | optional Some example Delta protocols: ```json { "protocol":{ "minReaderVersion":1, "minWriterVersion":2 } } ``` A table that is using table features only for writers: ```json { "protocol":{ "readerVersion":2, "writerVersion":7, "writerFeatures":["columnMapping","identityColumns"] } } ``` Reader version 2 in the above example does not support listing reader features but supports Column Mapping. This example is equivalent to the next one, where Column Mapping is represented as a reader table feature. A table that is using table features for both readers and writers: ```json { "protocol": { "readerVersion":3, "writerVersion":7, "readerFeatures":["columnMapping"], "writerFeatures":["columnMapping","identityColumns"] } } ``` ### Commit Provenance Information A delta file can optionally contain additional provenance information about what higher-level operation was being performed as well as who executed it. When the `catalogManaged` table feature is enabled, the `commitInfo` action must have a field `txnId` that stores a unique transaction identifier string. Implementations are free to store any valid JSON-formatted data via the `commitInfo` action. When [In-Commit Timestamps](#in-commit-timestamps) are enabled, writers are required to include a `commitInfo` action with every commit, which must include the `inCommitTimestamp` field. Also, the `commitInfo` action must be first action in the commit. An example of storing provenance information related to an `INSERT` operation: ```json { "commitInfo":{ "timestamp":1515491537026, "userId":"100121", "userName":"michael@databricks.com", "operation":"INSERT", "operationParameters":{"mode":"Append","partitionBy":"[]"}, "notebook":{ "notebookId":"4443029", "notebookPath":"Users/michael@databricks.com/actions"}, "clusterId":"1027-202406-pooh991" } } ``` ### Domain Metadata The domain metadata action contains a configuration (string) for a named metadata domain. Two overlapping transactions conflict if they both contain a domain metadata action for the same metadata domain. There are two types of metadata domains: 1. **User-controlled metadata domains** have names that start with anything other than the `delta.` prefix. Any Delta client implementation or user application can modify these metadata domains, and can allow users to modify them arbitrarily. Delta clients and user applications are encouraged to use a naming convention designed to avoid conflicts with other clients' or users' metadata domains (e.g. `com.databricks.*` or `org.apache.*`). 2. **System-controlled metadata domains** have names that start with the `delta.` prefix. This prefix is reserved for metadata domains defined by the Delta spec, and Delta client implementations must not allow users to modify the metadata for system-controlled domains. A Delta client implementation should only update metadata for system-controlled domains that it knows about and understands. System-controlled metadata domains are used by various table features and each table feature may impose additional semantics on the metadata domains it uses. The schema of the `domainMetadata` action is as follows: Field Name | Data Type | Description -|-|- domain | String | Identifier for this domain (system- or user-provided) configuration | String | String containing configuration for the metadata domain removed | Boolean | When `true`, the action serves as a tombstone to logically delete a metadata domain. Writers should preserve an accurate pre-image of the configuration. To support this feature: - The table must be on Writer Version 7. - A feature name `domainMetadata` must exist in the table's `writerFeatures`. #### Reader Requirements for Domain Metadata - Readers are not required to support domain metadata. - Readers who choose not to support domain metadata should ignore metadata domain actions as unrecognized (see [Protocol Evolution](#protocol-evolution)) and snapshots should not include any metadata domains. - Readers who choose to support domain metadata must apply [Action Reconciliation](#action-reconciliation) to all metadata domains and snapshots must include them -- even if the reader does not understand them. - Any system-controlled domain that imposes any requirements on readers is a [breaking change](#protocol-evolution), and must be part of a reader-writer table feature that specifies the desired behavior. #### Writer Requirements for Domain Metadata - Writers must preserve all domains even if they don't understand them. - Writers must not allow users to modify or delete system-controlled domains. - Writers must only modify or delete system-controlled domains they understand. - Any system-controlled domain that imposes additional requirements on the writer is a [breaking change](#protocol-evolution), and must be part of a writer table feature that specifies the desired behavior. The following is an example `domainMetadata` action: ```json { "domainMetadata": { "domain": "delta.deltaTableFeatureX", "configuration": "{\"key1\":\"value1\"}", "removed": false } } ``` ### Sidecar File Information The `sidecar` action references a [sidecar file](#sidecar-files) which provides some of the checkpoint's file actions. This action is only allowed in checkpoints following [V2 spec](#v2-spec). The schema of `sidecar` action is as follows: Field Name | Data Type | Description | optional/required -|-|-|- path | String | URI-encoded path to the sidecar file. Because sidecar files must always reside in the table's own _delta_log/_sidecars directory, implementations are encouraged to store only the file's name (without scheme or parent directories). | required sizeInBytes | Long | Size of the sidecar file. | required modificationTime | Long | The time this logical file was created, as milliseconds since the epoch. | required tags|`Map[String, String]`|Map containing any additional metadata about the checkpoint sidecar file. | optional The following is an example `sidecar` action: ```json { "sidecar":{ "path": "016ae953-37a9-438e-8683-9a9a4a79a395.parquet", "sizeInBytes": 2304522, "modificationTime": 1512909768000, "tags": {} } } ``` #### Checkpoint Metadata This action is only allowed in checkpoints following [V2 spec](#v2-spec). It describes the details about the checkpoint. It has the following schema: Field Name | Data Type | Description | optional/required -|-|-|- version|`Long`|The checkpoint version.| required tags|`Map[String, String]`|Map containing any additional metadata about the v2 spec checkpoint.| optional E.g. ```json { "checkpointMetadata":{ "version":1, "tags":{} } } ``` # Action Reconciliation A given snapshot of the table can be computed by replaying the events committed to the table in ascending order by commit version. A given snapshot of a Delta table consists of: - A single `protocol` action - A single `metaData` action - A collection of `txn` actions with unique `appId`s - A collection of `domainMetadata` actions with unique `domain`s. - A collection of `add` actions with unique path keys, corresponding to the newest (path, deletionVector.uniqueId) pair encountered for each path. - A collection of `remove` actions with unique `(path, deletionVector.uniqueId)` keys. The intersection of the primary keys in the `add` collection and `remove` collection must be empty. That means a logical file cannot exist in both the `remove` and `add` collections at the same time; however, the same *data file* can exist with *different* DVs in the `remove` collection, as logically they represent different content. The `remove` actions act as _tombstones_, and only exist for the benefit of the VACUUM command. Snapshot reads only return `add` actions on the read path. To achieve the requirements above, related actions from different delta files need to be reconciled with each other: - The latest `protocol` action seen wins - The latest `metaData` action seen wins - For `txn` actions, the latest `version` seen for a given `appId` wins - For `domainMetadata`, the latest `domainMetadata` seen for a given `domain` wins. The actions with `removed=true` act as tombstones to suppress earlier versions. Snapshot reads do _not_ return removed `domainMetadata` actions. - For `commitInfo` actions, only the `commitInfo` from the commit at the snapshot version is included in the snapshot. [Checkpoints](#checkpoints) and [log compaction files](#log-compaction-files) do not preserve `commitInfo` actions, so this information must be read from the JSON commit file at the snapshot version. - Logical files in a table are identified by their `(path, deletionVector.uniqueId)` primary key. File actions (`add` or `remove`) reference logical files, and a log can contain any number of references to a single file. - To replay the log, scan all file actions and keep only the newest reference for each logical file. - `add` actions in the result identify logical files currently present in the table (for queries). `remove` actions in the result identify tombstones of logical files no longer present in the table (for VACUUM). - [v2 checkpoint spec](#v2-spec) actions are not allowed in normal commit files, and do not participate in log replay. # Table Features Table features must only exist on tables that have a supported protocol version. When the table's Reader Version is 3, `readerFeatures` must exist in the `protocol` action, and when the Writer Version is 7, `writerFeatures` must exist in the `protocol` action. `readerFeatures` and `writerFeatures` define the features that readers and writers must implement in order to read and write this table. Readers and writers must not ignore table features when they are present: - to read a table, readers must implement and respect all features listed in `readerFeatures`; - to write a table, writers must implement and respect all features listed in `writerFeatures`. Because writers have to read the table (or only the Delta log) before write, they must implement and respect all reader features as well. ## Table Features for New and Existing Tables It is possible to create a new table or upgrade an existing table to the protocol versions that supports the use of table features. A table must support either the use of writer features or both reader and writer features. It is illegal to support reader but not writer features. For new tables, when a new table is created with a Reader Version up to 2 and Writer Version 7, its `protocol` action must only contain `writerFeatures`. When a new table is created with Reader Version 3 and Writer Version 7, its `protocol` action must contain both `readerFeatures` and `writerFeatures`. Creating a table with a Reader Version 3 and Writer Version less than 7 is not allowed. When upgrading an existing table to Reader Version 3 and/or Writer Version 7, the client should, on a best effort basis, determine which features supported by the original protocol version are used in any historical version of the table, and add only used features to reader and/or writer feature sets. The client must assume a feature has been used, unless it can prove that the feature is *definitely* not used in any historical version of the table that is reachable by time travel. For example, given a table on Reader Version 1 and Writer Version 4, along with four versions: 1. Table property change: set `delta.enableChangeDataFeed` to `true`. 2. Data change: three rows updated. 3. Table property change: unset `delta.enableChangeDataFeed`. 4. Table protocol change: upgrade protocol to Reader Version 3 and Writer Version 7. To produce Version 4, a writer could look at only Version 3 and discover that Change Data Feed has not been used. But in fact, this feature has been used and the table does contain some Change Data Files for Version 2. This means that, to determine all features that have ever been used by the table, a writer must either scan the whole history (which is very time-consuming) or assume the worst case: all features supported by protocol `(1, 4)` has been used. ## Supported Features A feature is supported by a table when its name is in the `protocol` action’s `readerFeatures` and/or `writerFeatures`. Subsequent read and/or write operations on this table must respect the feature. Clients must not remove the feature from the `protocol` action. Writers are allowed to add support of a feature to the table by adding its name to `readerFeatures` or `writerFeatures`. Reader features should be listed in both `readerFeatures` and `writerFeatures` simultaneously, while writer features should be listed only in `writerFeatures`. It is not allowed to list a feature only in `readerFeatures` but not in `writerFeatures`. A feature being supported does not imply that it is active. For example, a table may have the [Append-only Tables](#append-only-tables) feature (feature name `appendOnly`) listed in `writerFeatures`, but it does not have a table property `delta.appendOnly` that is set to `true`. In such a case the table is not append-only, and writers are allowed to change, remove, and rearrange data. However, writers must know that the table property `delta.appendOnly` should be checked before writing the table. ## Active Features A feature is active on a table when it is supported *and* its metadata requirements are satisfied. Each feature defines its own metadata requirements, as stated in the corresponding sections of this document. For example, the Append-only feature is active when the `appendOnly` feature name is present in a `protocol`'s `writerFeatures` *and* a table property `delta.appendOnly` set to `true`. # Column Mapping Delta can use column mapping to avoid any column naming restrictions, and to support the renaming and dropping of columns without having to rewrite all the data. There are two modes of column mapping, by `name` and by `id`. In both modes, every column - nested or leaf - is assigned a unique _physical_ name, and a unique 32-bit integer as an id. The physical name is stored as part of the column metadata with the key `delta.columnMapping.physicalName`. The column id is stored within the metadata with the key `delta.columnMapping.id`. The column mapping is governed by the table property `delta.columnMapping.mode` being one of `none`, `id`, and `name`. The table property should only be honored if the table's protocol has reader and writer versions and/or table features that support the `columnMapping` table feature. For readers this is Reader Version 2, or Reader Version 3 with the `columnMapping` table feature listed as supported. For writers this is Writer Version 5 or 6, or Writer Version 7 with the `columnMapping` table feature supported. The following is an example for the column definition of a table that leverages column mapping. See the [appendix](#schema-serialization-format) for a more complete schema definition. ```json { "name" : "e", "type" : { "type" : "array", "elementType" : { "type" : "struct", "fields" : [ { "name" : "d", "type" : "integer", "nullable" : false, "metadata" : { "delta.columnMapping.id": 5, "delta.columnMapping.physicalName": "col-a7f4159c-53be-4cb0-b81a-f7e5240cfc49" } } ] }, "containsNull" : true }, "nullable" : true, "metadata" : { "delta.columnMapping.id": 4, "delta.columnMapping.physicalName": "col-5f422f40-de70-45b2-88ab-1d5c90e94db1" } } ``` ## Writer Requirements for Column Mapping In order to support column mapping, writers must: - Write `protocol` and `metaData` actions when Column Mapping is turned on for the first time: - If the table is on Writer Version 5 or 6: write a `metaData` action to add the `delta.columnMapping.mode` table property; - If the table is on Writer Version 7: - write a `protocol` action to add the feature `columnMapping` to both `readerFeatures` and `writerFeatures`, and - write a `metaData` action to add the `delta.columnMapping.mode` table property. - Write data files by using the _physical name_ that is chosen for each column. The physical name of the column is static and can be different than the _display name_ of the column, which is changeable. - Write the 32 bit integer column identifier as part of the `field_id` field of the `SchemaElement` struct in the [Parquet Thrift specification](https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift). - Track partition values, column level statistics, and [clustering column](#clustered-table) names with the physical name of the column in the transaction log. - Assign a globally unique identifier as the physical name for each new column that is added to the schema. This is especially important for supporting cheap column deletions in `name` mode. In addition, column identifiers need to be assigned to each column. The maximum id that is assigned to a column is tracked as the table property `delta.columnMapping.maxColumnId`. This is an internal table property that cannot be configured by users. This value must increase monotonically as new columns are introduced and committed to the table alongside the introduction of the new columns to the schema. ## Reader Requirements for Column Mapping If the table is on Reader Version 2, or if the table is on Reader Version 3 and the feature `columnMapping` is present in `readerFeatures`, readers and writers must read the table property `delta.columnMapping.mode` and do one of the following. In `none` mode, or if the table property is not present, readers must read the parquet files by using the display names (the `name` field of the column definition) of the columns in the schema. In `id ` mode, readers must resolve columns by using the `field_id` in the parquet metadata for each file, as given by the column metadata property `delta.columnMapping.id` in the Delta schema. Partition values and column level statistics must be resolved by their *physical names* for each `add` entry in the transaction log. If a data file does not contain field ids, readers must refuse to read that file or return nulls for each column. For ids that cannot be found in a file, readers must return `null` values for those columns. In `name` mode, readers must resolve columns in the data files by their physical names as given by the column metadata property `delta.columnMapping.physicalName` in the Delta schema. Partition values and column level statistics will also be resolved by their physical names. For columns that are not found in the files, `null`s need to be returned. Column ids are not used in this mode for resolution purposes. # Deletion Vectors To support this feature: - To support Deletion Vectors, a table must have Reader Version 3 and Writer Version 7. A feature name `deletionVectors` must exist in the table's `readerFeatures` and `writerFeatures`. When supported: - A table may have a metadata property `delta.enableDeletionVectors` in the Delta schema set to `true`. Writers must only write new Deletion Vectors (DVs) when this property is set to `true`. - A table's `add` and `remove` actions can optionally include a DV that provides information about logically deleted rows, that are however still physically present in the underlying data file and must thus be skipped during processing. Readers must read the table considering the existence of DVs, even when the `delta.enableDeletionVectors` table property is not set. DVs can be stored and accessed in different ways, indicated by the `storageType` field. The Delta protocol currently supports inline or on-disk storage, where the latter can be accessed either by a relative path derived from a UUID or an absolute path. ## Deletion Vector Descriptor Schema The schema of the `DeletionVectorDescriptor` struct is as follows: Field Name | Data Type | Description -|-|- storageType | String | A single character to indicate how to access the DV. Legal options are: `['u', 'i', 'p']`. pathOrInlineDv | String | Three format options are currently proposed:
  • If `storageType = 'u'` then ``: The deletion vector is stored in a file with a path relative to the data directory of this Delta table, and the file name can be reconstructed from the UUID. See Derived Fields for how to reconstruct the file name. The random prefix is recovered as the extra characters before the (20 characters fixed length) uuid.
  • If `storageType = 'i'` then ``: The deletion vector is stored inline in the log. The format used is the `RoaringBitmapArray` format also used when the DV is stored on disk and described in [Deletion Vector Format](#Deletion-Vector-Format).
  • If `storageType = 'p'` then ``: The DV is stored in a file with an absolute path given by this path, which has the same format as the `path` field in the `add`/`remove` actions.
offset | Option[Int] | Start of the data for this DV in number of bytes from the beginning of the file it is stored in. Always `None` (absent in JSON) when `storageType = 'i'`. sizeInBytes | Int | Size of the serialized DV in bytes (raw data size, i.e. before base85 encoding, if inline). cardinality | Long | Number of rows the given DV logically removes from the file. The concrete Base85 variant used is [Z85](https://rfc.zeromq.org/spec/32/), because it is JSON-friendly. ### Derived Fields Some fields that are necessary to use the DV are not stored explicitly but can be derived in code from the stored fields. Field Name | Data Type | Description | Computed As -|-|-|- uniqueId | String | Uniquely identifies a DV for a given file. This is used for snapshot reconstruction to differentiate the same file with different DVs in successive versions. | If `offset` is `None` then ``.
Otherwise `@`. absolutePath | String/URI/Path | The absolute path of the DV file. Can be calculated for relative path DVs by providing a parent directory path. | If `storageType='p'`, just use the already absolute path. If `storageType='u'`, the DV is stored at `//deletion_vector_.bin`. This is not a legal field if `storageType='i'`, as an inline DV has no absolute path. ### JSON Example 1 — On Disk with Relative Path (with Random Prefix) ```json { "storageType" : "u", "pathOrInlineDv" : "ab^-aqEH.-t@S}K{vb[*k^", "offset" : 4, "sizeInBytes" : 40, "cardinality" : 6 } ``` Assuming that this DV is stored relative to an `s3://mytable/` directory, the absolute path to be resolved here would be: `s3://mytable/ab/deletion_vector_d2c639aa-8816-431a-aaf6-d3fe2512ff61.bin`. ### JSON Example 2 — On Disk with Absolute Path ```json { "storageType" : "p", "pathOrInlineDv" : "s3://mytable/deletion_vector_d2c639aa-8816-431a-aaf6-d3fe2512ff61.bin", "offset" : 4, "sizeInBytes" : 40, "cardinality" : 6 } ``` ### JSON Example 3 — Inline ```json { "storageType" : "i", "pathOrInlineDv" : "wi5b=000010000siXQKl0rr91000f55c8Xg0@@D72lkbi5=-{L", "sizeInBytes" : 40, "cardinality" : 6 } ``` The row indexes encoded in this DV are: 3, 4, 7, 11, 18, 29. ## Reader Requirements for Deletion Vectors If a snapshot contains logical files with records that are invalidated by a DV, then these records *must not* be returned in the output. ## Writer Requirement for Deletion Vectors When adding a logical file with a deletion vector, then that logical file must have correct `numRecords` information for the data file in the `stats` field. # Catalog-managed tables With this feature enabled, the [catalog](#terminology-catalogs) that manages the table becomes the source of truth for whether a given commit attempt succeeded. The table feature defines the parts of the [commit protocol](#commit-protocol) that directly impact the Delta table (e.g. atomicity requirements, publishing, etc). The Delta client and catalog together are responsible for implementing the Delta-specific aspects of commit as defined by this spec, but are otherwise free to define their own APIs and protocols for communication with each other. **NOTE**: Filesystem-based access to catalog-managed tables is not supported. Delta clients are expected to discover and access catalog-managed tables through the managing catalog, not by direct listing in the filesystem. This feature is primarily designed to warn filesystem-based readers that might attempt to access a catalog-managed table's storage location without going through the catalog first, and to block filesystem-based writers who could otherwise corrupt both the table and the catalog by failing to commit through the catalog. Before we can go into details of this protocol feature, we must first align our terminology. ## Terminology: Commits A commit is a set of [actions](#actions) that transform a Delta table from version `v - 1` to `v`. It contains the same kind of content as is stored in a [Delta file](#delta-log-entries). A commit may be stored in the file system as a Delta file - either _published_ or _staged_ - or stored _inline_ in the managing catalog, using whatever format the catalog prefers. There are several types of commits: 1. **Proposed commit**: A commit that a Delta client has proposed for the next version of the table. It could be _staged_ or _inline_. It will either become _ratified_ or be rejected. 2. **Staged commit**: A commit that is written to disk at `_delta_log/_staged_commits/..json`. It has the same content and format as a published Delta file. - Here, the `uuid` is a random UUID that is generated for each commit and `v` is the version which is proposed to be committed, zero-padded to 20 digits. - The mere existence of a staged commit does not mean that the file has been ratified or even proposed. It might correspond to a failed or in-progress commit attempt. - The catalog is the source of truth around which staged commits are ratified. - The catalog stores only the location, not the content, of a staged (and ratified) commit. 3. **Inline commit**: A proposed commit that is not written to disk but rather has its content sent to the catalog for the catalog to store directly. 4. **Ratified commit**: A proposed commit that a catalog has determined has won the commit at the desired version of the table. - The catalog must store ratified commits (that is, the staged commit's location or the inline commit's content) until they are published to the `_delta_log` directory. - A ratified commit may or may not yet be published. - A ratified commit may or may not even be stored by the catalog at all - the catalog may have just atomically published it to the filesystem directly, relying on PUT-if-absent primitives to facilitate the ratification and publication all in one step. 5. **Published commit**: A ratified commit that has been copied into the `_delta_log` as a normal Delta file, i.e. `_delta_log/.json`. - Here, the `v` is the version which is being committed, zero-padded to 20 digits. - The existence of a `.json` file proves that the corresponding version `v` is ratified, regardless of whether the table is catalog-managed or filesystem-based. The catalog is allowed to return information about published commits, but Delta clients can also use filesystem listing operations to directly discover them. - Published commits do not need to be stored by the catalog. ## Terminology: Delta Client This is the component that implements support for reading and writing Delta tables, and implements the logic required by the `catalogManaged` table feature. Among other things, it - triggers the filesystem listing, if needed, to discover published commits - generates the commit content (the set of [actions](#actions)) - works together with the query engine to trigger the commit process and invoke the client-side catalog component with the commit content The Delta client is also responsible for defining the client-side API that catalogs should target. That is, there must be _some_ API that the [catalog client](#catalog-client) can use to communicate to the Delta client the subset of catalog-managed information that the Delta client cares about. This protocol feature is concerned with what information Delta cares about, but leaves to Delta clients the design of the API they use to obtain that information from catalog clients. ## Terminology: Catalogs 1. **Catalog**: A catalog is an entity which manages a Delta table, including its creation, writes, reads, and eventual deletion. - It could be backed by a database, a filesystem, or any other persistence mechanism. - Each catalog has its own spec around how catalog clients should interact with them, and how they perform a commit. 2. **Catalog Client**: The catalog always has a client-side component which the Delta client interacts with directly. This client-side component has two primary responsibilities: - implement any client-side catalog-specific logic (such as staging or [publishing](#publishing-commits) commits) - communicate with the Catalog Server, if any 3. **Catalog Server**: The catalog may also involve a server-side component which the client-side component would be responsible to communicate with. - This server is responsible for coordinating commits and potentially persisting table metadata and enforcing authorization policies. - Not all catalogs require a server; some may be entirely client-side, e.g. filesystem-backed catalogs, or they may make use of a generic database server and implement all of the catalog's business logic client-side. **NOTE**: This specification outlines the responsibilities and actions that catalogs must implement. This spec does its best not to assume any specific catalog _implementation_, though it does call out likely client-side and server-side responsibilities. Nonetheless, what a given catalog does client-side or server-side is up to each catalog implementation to decide for itself. ## Catalog Responsibilities When the `catalogManaged` table feature is enabled, a catalog performs commits to the table on behalf of the Delta client. As stated above, the Delta spec does not mandate any particular client-server design or API for catalogs that manage Delta tables. However, the catalog does need to provide certain capabilities for reading and writing Delta tables: - Atomically commit a version `v` with a given set of `actions`. This is explained in detail in the [commit protocol](#commit-protocol) section. - Retrieve information about recent ratified commits and the latest ratified version on the table. This is explained in detail in the [Getting Ratified Commits from the Catalog](#getting-ratified-commits-from-the-catalog) section. - Though not required, it is encouraged that catalogs also return the latest table-level metadata, such as the latest Protocol and Metadata actions, for the table. This can provide significant performance advantages to conforming Delta clients, who may forgo log replay and instead trust the information provided by the catalog during query planning. ## Reading Catalog-managed Tables A catalog-managed table can have a mix of (a) published and (b) ratified but non-published commits. The catalog is the source of truth for ratified commits. Also recall that ratified commits can be [staged commits](#staged-commit) that are persisted to the `_delta_log/_staged_commits` directory, or [inline commits](#inline-commit) whose content the catalog stores directly. For example, suppose the `_delta_log` directory contains the following files: ``` 00000000000000000000.json 00000000000000000001.json 00000000000000000002.checkpoint.parquet 00000000000000000002.json 00000000000000000003.00000000000000000005.compacted.json 00000000000000000003.json 00000000000000000004.json 00000000000000000005.json 00000000000000000006.json 00000000000000000007.json _staged_commits/00000000000000000007.016ae953-37a9-438e-8683-9a9a4a79a395.json // ratified and published _staged_commits/00000000000000000008.7d17ac10-5cc3-401b-bd1a-9c82dd2ea032.json // ratified _staged_commits/00000000000000000008.b91807ba-fe18-488c-a15e-c4807dbd2174.json // rejected _staged_commits/00000000000000000010.0f707846-cd18-4e01-b40e-84ee0ae987b0.json // not yet ratified _staged_commits/00000000000000000010.7a980438-cb67-4b89-82d2-86f73239b6d6.json // partial file ``` Further, suppose the catalog stores the following ratified commits: ``` { 7 -> "00000000000000000007.016ae953-37a9-438e-8683-9a9a4a79a395.json", 8 -> "00000000000000000008.7d17ac10-5cc3-401b-bd1a-9c82dd2ea032.json", 9 -> } ``` Some things to note are: - the catalog isn't aware that commit 7 was already published - perhaps the response from the filesystem was dropped - commit 9 is an inline commit - neither of the two staged commits for version 10 have been ratified To read such tables, Delta clients must first contact the catalog to get the ratified commits. This informs the Delta client of commits [7, 9] as well as the latest ratified version, 9. If this information is insufficient to construct a complete snapshot of the table, Delta clients must LIST the `_delta_log` directory to get information about the published commits. For commits that are both returned by the catalog and already published, Delta clients must treat the catalog's version as authoritative and read the commit returned by the catalog. Additionally, Delta clients must ignore any files with versions greater than the latest ratified commit version returned by the catalog. Combining these two sets of files and commits enables Delta clients to generate a snapshot at the latest version of the table. **NOTE**: This spec prescribes the _minimum_ required interactions between Delta clients and catalogs for commits. Catalogs may very well expose APIs and work with Delta clients to be informed of other non-commit [file types](#file-types), such as checkpoint, log compaction, and version checksum files. This would allow catalogs to return additional information to Delta clients during query and scan planning, potentially allowing Delta clients to avoid LISTing the filesystem altogether. ## Commit Protocol To start, Delta Clients send the desired actions to be committed to the client-side component of the catalog. This component then has several options for proposing, ratifying, and publishing the commit, detailed below. - Option 1: Write the actions (likely client-side) to a [staged commit file](#staged-commit) in the `_delta_log/_staged_commits` directory and then ratify the staged commit (likely server-side) by atomically recording (in persistent storage of some kind) that the file corresponds to version `v`. - Option 2: Treat this as an [inline commit](#inline-commit) (i.e. likely that the client-side component sends the contents to the server-side component) and atomically record (in persistent storage of some kind) the content of the commit as version `v` of the table. - Option 3: Catalog implementations that use PUT-if-absent (client- or server-side) can ratify and publish all-in-one by atomically writing a [published commit file](#published-commit) in the `_delta_log` directory. Note that this commit will be considered to have succeeded as soon as the file becomes visible in the filesystem, regardless of when or whether the catalog is made aware of the successful publish. The catalog does not need to store these files. A catalog must not ratify version `v` until it has ratified version `v - 1`, and it must ratify version `v` at most once. The catalog must store both flavors of ratified commits (staged or inline) and make them available to readers until they are [published](#publishing-commits). For performance reasons, Delta clients are encouraged to establish an API contract where the catalog provides the latest ratified commit information whenever a commit fails due to version conflict. ## Getting Ratified Commits from the Catalog Even after a commit is ratified, it is not discoverable through filesystem operations until it is [published](#publishing-commits). The catalog-client is responsible to implement an API (defined by the Delta client) that Delta clients can use to retrieve the latest ratified commit version (authoritative), as well as the set of ratified commits the catalog is still storing for the table. If some commits needed to complete the snapshot are not stored by the catalog, as they are already published, Delta clients can issue a filesystem LIST operation to retrieve them. Delta clients must establish an API contract where the catalog provides ratified commit information as part of the standard table resolution process performed at query planning time. ## Publishing Commits Publishing is the process of copying the ratified commit with version `` to `_delta_log/.json`. The ratified commit may be a staged commit located in `_delta_log/_staged_commits/..json`, or it may be an inline commit whose content the catalog stores itself. Because the content of a ratified commit is immutable, it does not matter whether the client-side, server-side, or both catalog components initiate publishing. Implementations are strongly encouraged to publish commits promptly. This reduces the number of commits the catalog needs to store internally (and serve up to readers). Commits must be published _in order_. That is, version `v - 1` must be published _before_ version `v`. **NOTE**: Because commit publishing can happen at any time after the commit succeeds, the file modification timestamp of the published file will not accurately reflect the original commit time. For this reason, catalog-managed tables must use [in-commit-timestamps](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#in-commit-timestamps) to ensure stability of time travel reads. Refer to [Writer Requirements for Catalog-managed Tables](#writer-requirements-for-catalog-managed-tables) section for more details. ## Maintenance Operations on Catalog-managed Tables [Checkpoints](#checkpoints-1) and [Log Compaction Files](#log-compaction-files) can only be created for versions that are already published in the `_delta_log`. In other words, in order to checkpoint version `v` or produce a log compaction file for commit range `x <= v <= y`, `_delta_log/.json` must exist. Notably, the [Version Checksum File](#version-checksum-file) for version `v` _can_ be created in the `_delta_log` even if the commit for version `v` is not published. By default, maintenance operations are prohibited unless the managing catalog explicitly permits the client to run them. The only exceptions are checkpoints, log compaction, and version checksum, as they are essential for all basic table operations (e.g. reads and writes) to operate reliably. All other maintenance operations such as the following are not allowed by default. - [Log and other metadata files clean up](#metadata-cleanup). - Data files cleanup, for example VACUUM. - Data layout changes, for example OPTIMIZE and REORG. ## Creating and Dropping Catalog-managed Tables The catalog and query engine ultimately dictate how to create and drop catalog-managed tables. As one example, table creation often works in three phases: 1. An initial catalog operation to obtain a unique storage location which serves as an unnamed "staging" table 2. A table operation that physically initializes a new `catalogManaged`-enabled table at the staging location. 3. A final catalog operation that registers the new table with its intended name. Delta clients would primarily be involved with the second step, but an implementation could choose to combine the second and third steps so that a single catalog call registers the table as part of the table's first commit. As another example, dropping a table can be as simple as removing its name from the catalog (a "soft delete"), followed at some later point by a "hard delete" that physically purges the data. The Delta client would not be involved at all in this process, because no commits are made to the table. ## Catalog-managed Table Enablement The `catalogManaged` table feature is supported and active when: - The table is on Reader Version 3 and Writer Version 7. - The table has a `protocol` action with `readerFeatures` and `writerFeatures` both containing the feature `catalogManaged`. ## Writer Requirements for Catalog-managed tables When supported and active: - Writers must discover and access the table using catalog calls, which happens _before_ the table's protocol is known. See [Table Discovery](#table-discovery) for more details. - The [in-commit-timestamps](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#in-commit-timestamps) table feature must be supported and active. - The `commitInfo` action must also contain a field `txnId` that stores a unique transaction identifier string - Writers must follow the catalog's [commit protocol](#commit-protocol) and must not perform ordinary filesystem-based commits against the table. - Writers must follow the catalog's [maintenance operation protocol](#maintenance-operations-on-catalog-managed-tables) ## Reader Requirements for Catalog-managed tables When supported and active: - Readers must discover the table using catalog calls, which happens before the table's protocol is known. See [Table Discovery](#table-discovery) for more details. - Readers must contact the catalog for information about unpublished ratified commits. - Readers must follow the rules described in the [Reading Catalog-managed Tables](#reading-catalog-managed-tables) section above. Notably - If the catalog said `v` is the latest version, clients must ignore any later versions that may have been published - When the catalog returns a ratified commit for version `v`, readers must use that catalog-supplied commit and ignore any published Delta file for version `v` that might also be present. ## Table Discovery The requirements above state that readers and writers must discover and access the table using catalog calls, which occurs _before_ the table's protocol is known. This raises an important question: how can a client discover a `catalogManaged` Delta table without first knowing that it _is_, in fact, `catalogManaged` (according to the protocol)? To solve this, first note that, in practice, catalog-integrated engines already ask the catalog to resolve a table name to its storage location during the name resolution step. This protocol therefore encourages that the same name resolution step also indicate whether the table is catalog-managed. Surfacing this at the very moment the catalog returns the path imposes no extra round-trips, yet it lets the client decide — early and unambiguously — whether to follow the `catalogManaged` read and write rules. ## Sample Catalog Client API The following is an example of a possible API which a Java-based Delta client might require catalog implementations to target: ```scala interface CatalogManagedTable { /** * Commits the given set of `actions` to the given commit `version`. * * @param version The version we want to commit. * @param actions Actions that need to be committed. * * @return CommitResponse which has details around the new committed delta file. */ def commit( version: Long, actions: Iterator[String]): CommitResponse /** * Retrieves a (possibly empty) suffix of ratified commits in the range [startVersion, * endVersion] for this table. * * Some of these ratified commits may already have been published. Some of them may be staged, * in which case the staged commit file path is returned; others may be inline, in which case * the inline commit content is returned. * * The returned commits are sorted in ascending version number and are contiguous. * * If neither start nor end version is specified, the catalog will return all available ratified * commits (possibly empty, if all commits have been published). * * In all cases, the response also includes the table's latest ratified commit version. * * @return GetCommitsResponse which contains an ordered list of ratified commits * stored by the catalog, as well as table's latest commit version. */ def getRatifiedCommits( startVersion: Option[Long], endVersion: Option[Long]): GetCommitsResponse } ``` Note that the above is only one example of a possible Catalog Client API. It is also _NOT_ a catalog API (no table discovery, ACL, create/drop, etc). The Delta protocol is agnostic to API details, and the API surface Delta clients define should only cover the specific catalog capabilities that Delta client needs to correctly read and write catalog-managed tables. # Iceberg Compatibility V1 This table feature (`icebergCompatV1`) ensures that Delta tables can be converted to Apache Iceberg™ format, though this table feature does not implement or specify that conversion. To support this feature: - Since this table feature depends on Column Mapping, the table must be on Reader Version = 2, or it must be on Reader Version >= 3 and the feature `columnMapping` must exist in the `protocol`'s `readerFeatures`. - The table must be on Writer Version 7. - The feature `icebergCompatV1` must exist in the table `protocol`'s `writerFeatures`. This table feature is enabled when the table property `delta.enableIcebergCompatV1` is set to `true`. ## Writer Requirements for IcebergCompatV1 When supported and active, writers must: - Require that Column Mapping be enabled and set to either `name` or `id` mode - Require that Deletion Vectors are not supported (and, consequently, not active, either). i.e., the `deletionVectors` table feature is not present in the table `protocol`. - Require that partition column values are materialized into any Parquet data file that is present in the table, placed *after* the data columns in the parquet schema - Require that all `AddFile`s committed to the table have the `numRecords` statistic populated in their `stats` field - Block adding `Map`/`Array`/`Void` types to the table schema (and, thus, block writing them, too) - Block replacing partitioned tables with a differently-named partition spec - e.g. replacing a table partitioned by `part_a INT` with partition spec `part_b INT` must be blocked - e.g. replacing a table partitioned by `part_a INT` with partition spec `part_a LONG` is allowed - When the [Type Widening](#type-widening) table feature is supported, require that all type changes applied on the table are supported by [Iceberg V2](https://iceberg.apache.org/spec/#schema-evolution), based on the [Type Change Metadata](#type-change-metadata) recorded in the table schema. # Iceberg Compatibility V2 This table feature (`icebergCompatV2`) ensures that Delta tables can be converted to Apache Iceberg™ format, though this table feature does not implement or specify that conversion. To support this feature: - Since this table feature depends on Column Mapping, the table must be on Reader Version = 2, or it must be on Reader Version >= 3 and the feature `columnMapping` must exist in the `protocol`'s `readerFeatures`. - The table must be on Writer Version 7. - The feature `icebergCompatV2` must exist in the table protocol's `writerFeatures`. This table feature is enabled when the table property `delta.enableIcebergCompatV2` is set to `true`. ## Writer Requirements for IcebergCompatV2 When this feature is supported and enabled, writers must: - Require that Column Mapping be enabled and set to either `name` or `id` mode - Require that the nested `element` field of ArrayTypes and the nested `key` and `value` fields of MapTypes be assigned 32 bit integer identifiers. These identifiers must be unique and different from those used in [Column Mapping](#column-mapping), and must be stored in the metadata of their nearest ancestor [StructField](#struct-field) of the Delta table schema. Identifiers belonging to the same `StructField` must be organized as a `Map[String, Long]` and stored in metadata with key `parquet.field.nested.ids`. The keys of the map are "element", "key", or "value", prefixed by the name of the nearest ancestor StructField, separated by dots. The values are the identifiers. The keys for fields in nested arrays or nested maps are prefixed by their parents' key, separated by dots. An [example](#example-of-storing-identifiers-for-nested-fields-in-arraytype-and-maptype) is provided below to demonstrate how the identifiers are stored. These identifiers must be also written to the `field_id` field of the `SchemaElement` struct in the [Parquet Thrift specification](https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift) when writing parquet files. - Require that IcebergCompatV1 is not active, which means either the `icebergCompatV1` table feature is not present in the table protocol or the table property `delta.enableIcebergCompatV1` is not set to `true` - Require that Deletion Vectors are not active, which means either the `deletionVectors` table feature is not present in the table protocol or the table property `delta.enableDeletionVectors` is not set to `true` - Require that partition column values be materialized when writing Parquet data files - Require that all new `AddFile`s committed to the table have the `numRecords` statistic populated in their `stats` field - Require writing timestamp columns as int64 - Require that the table schema contains only data types in the following allow-list: [`byte`, `short`, `integer`, `long`, `float`, `double`, `decimal`, `string`, `binary`, `boolean`, `timestamp`, `timestampNTZ`, `date`, `array`, `map`, `struct`]. - Block replacing partitioned tables with a differently-named partition spec - e.g. replacing a table partitioned by `part_a INT` with partition spec `part_b INT` must be blocked - e.g. replacing a table partitioned by `part_a INT` with partition spec `part_a LONG` is allowed - When the [Type Widening](#type-widening) table feature is supported, require that all type changes applied on the table are supported by [Iceberg V2](https://iceberg.apache.org/spec/#schema-evolution), based on the [Type Change Metadata](#type-change-metadata) recorded in the table schema. ### Example of storing identifiers for nested fields in ArrayType and MapType The following is an example of storing the identifiers for nested fields in `ArrayType` and `MapType`, of a table with the following schema, ``` |-- col1: array[array[int]] |-- col2: map[int, array[int]] |-- col3: map[int, struct] |-- subcol1: array[int] ``` The identifiers for the nested fields are stored in the metadata as follows: ```json [ { "name": "col1", "type": { "type": "array", "elementType": { "type": "array", "elementType": "int" } }, "metadata": { "parquet.field.nested.ids": { "col1.element": 100, "col1.element.element": 101 } } }, { "name": "col2", "type": { "type": "map", "keyType": "int", "valueType": { "type": "array", "elementType": "int" } }, "metadata": { "parquet.field.nested.ids": { "col2.key": 102, "col2.value": 103, "col2.value.element": 104 } } }, { "name": "col3", "type": { "type": "map", "keyType": "int", "valueType": { "type": "struct", "fields": [ { "name": "subcol1", "type": { "type": "array", "elementType": "int" }, "metadata": { "parquet.field.nested.ids": { "subcol1.element": 107 } } } ] } }, "metadata": { "parquet.field.nested.ids": { "col3.key": 105, "col3.value": 106 } } } ] ``` # Timestamp without timezone (TimestampNtz) This feature introduces a new data type to support timestamps without timezone information. For example: `1970-01-01 00:00:00`, or `1970-01-01 00:00:00.123456`. The serialization method is described in Sections [Partition Value Serialization](#partition-value-serialization) and [Schema Serialization Format](#schema-serialization-format). To support this feature: - To have a column of TimestampNtz type in a table, the table must have Reader Version 3 and Writer Version 7. A feature name `timestampNtz` must exist in the table's `readerFeatures` and `writerFeatures`. # V2 Checkpoint Table Feature To support this feature: - To add [V2 Checkpoints](#v2-spec) support to a table, the table must have Reader Version 3 and Writer Version 7. A feature name `v2Checkpoint` must exist in the table's `readerFeatures` and `writerFeatures`. When supported: - A table could use [uuid-named](#uuid-named-checkpoint) [V2 spec Checkpoints](#v2-spec) which must have [checkpoint metadata](#checkpoint-metadata) and may have [sidecar files](#sidecar-files) OR - A table could use [classic](#classic-checkpoint) checkpoints which can follow [V1](#v1-spec) or [V2](#v2-spec) spec. - A table must not use [multi-part checkpoints](#multi-part-checkpoint) # Row Tracking Row Tracking is a feature that allows the tracking of rows across multiple versions of a Delta table. It enables this by exposing two metadata columns: Row IDs, which uniquely identify a row across multiple versions of a table, and Row Commit Versions, which make it possible to check whether two rows with the same ID in two different versions of the table represent the same version of the row. Row Tracking is defined to be **supported** or **enabled** on a table as follows: - When the feature `rowTracking` exists in the table `protocol`'s `writerFeatures`, then we say that Row Tracking is **supported**. In this situation, writers must assign Row IDs and Commit Versions as long as `delta.rowTrackingSuspended` table property is absent or set to false. However, they cannot yet be relied upon to be present in the table. When Row Tracking is supported but not yet enabled writers cannot preserve Row IDs and Commit Versions. - When additionally the table property `delta.enableRowTracking` is set to `true`, then we say that Row Tracking is **enabled**. In this situation, Row IDs and Row Commit versions can be relied upon to be present in the table for all rows. When Row Tracking is enabled writers are expected to preserve Row IDs and Commit Versions. - When the table property `delta.rowTrackingSuspended` is set to true, writers should suspend the assignment of Row IDs and Commit Versions. Table property `delta.rowTrackingSuspended` should not be enabled together with table property `delta.enableRowTracking`. Enablement: - The table must be on Writer Version 7. - The feature `rowTracking` must exist in the table `protocol`'s `writerFeatures`. The feature `domainMetadata` is required in the table `protocol`'s `writerFeatures`. - The table property `delta.enableRowTracking` must be set to `true`. - The table property `delta.rowTrackingSuspended` should be absent or set to `false`. ## Row IDs Delta provides Row IDs. Row IDs are integers that are used to uniquely identify rows within a table. Every row has two Row IDs: - A **fresh** or unstable **Row ID**. This ID uniquely identifies the row within one version of the table. The fresh ID of a row may change every time the table is updated, even for rows that are not modified. E.g. when a row is copied unchanged during an update operation, it will get a new fresh ID. Fresh IDs can be used to identify rows within one version of the table, e.g. for identifying matching rows in self joins. - A **stable Row ID**. This ID uniquely identifies the row across versions of the table and across updates. When a row is inserted, it is assigned a new stable Row ID that is equal to the fresh Row ID. When a row is updated or copied, the stable Row ID for this row is preserved. When a row is restored (i.e. the table is restored to an earlier version), its stable Row ID is restored as well. The fresh and stable Row IDs are not required to be equal. Row IDs are stored in two ways: - **Default generated Row IDs** use the `baseRowId` field stored in `add` and `remove` actions to generate fresh Row IDs. The default generated Row IDs for data files are calculated by adding the `baseRowId` of the file in which a row is contained to the (physical) position (index) of the row within the file. Default generated Row IDs require little storage overhead but are reassigned every time a row is updated or moved to a different file (for instance when a row is contained in a file that is compacted by OPTIMIZE). - **Materialized Row IDs** are stored in a column in the data files. This column is hidden from readers and writers, i.e. it is not part of the `schemaString` in the table's `metaData`. Instead, the name of this column can be found in the value for the `delta.rowTracking.materializedRowIdColumnName` key in the `configuration` of the table's `metaData` action. This column may contain `null` values meaning that the corresponding row has no materialized Row ID. This column may be omitted if all its values are `null` in the file. Materialized Row IDs provide a mechanism for writers to preserve stable Row IDs for rows that are updated or copied. The fresh Row ID of a row is equal to the default generated Row ID. The stable Row ID of a row is equal to the materialized Row ID of the row when that column is present and the value is not NULL, otherwise it is equal to the default generated Row ID. When Row Tracking is enabled: - Default generated Row IDs must be assigned to all existing rows. This means in particular that all files that are part of the table version that sets the table property `delta.enableRowTracking` to `true` must have `baseRowId` set. A backfill operation may be required to commit `add` and `remove` actions with the `baseRowId` field set for all data files before the table property `delta.enableRowTracking` can be set to `true`. ## Row Commit Versions Row Commit Versions provide versioning of rows. - **Fresh** or unstable **Row Commit Versions** can be used to identify the first commit version in which the `add` action containing the row was committed. The fresh Commit Version of a row may change every time the table is updated, even for rows that are not modified. E.g. when a row is copied unchanged during an update operation, it will get a new fresh Commit Version. - **Stable Row Commit Versions** identify the last commit version in which the row (with the same ID) was either inserted or updated. When a row is inserted or updated, it is assigned the commit version number of the log entry containing the `add` entry with the new row. When a row is copied, the stable Row Commit Version for this row is preserved. When a row is restored (i.e. the table is restored to an earlier version), its stable Row Commit Version is restored as well. The fresh and stable Row Commit Versions are not required to be equal. Commit Versions are stored in two ways: - **Default generated Row Commit Versions** use the `defaultRowCommitVersion` field in `add` and `remove` actions. Default generated Row Commit Versions require little storage overhead but are reassigned every time a row is updated or moved to a different file (for instance when a row is contained in a file that is compacted by OPTIMIZE). - **Materialized Row Commit Versions** are stored in a column in the data files. This column is hidden from readers and writers, i.e. it is not part of the `schemaString` in the table's `metaData`. Instead, the name of this column can be found in the value for the `delta.rowTracking.materializedRowCommitVersionColumnName` key in the `configuration` of the table's `metaData` action. This column may contain `null` values meaning that the corresponding row has no materialized Row Commit Version. This column may be omitted if all its values are `null` in the file. Materialized Row Commit Versions provide a mechanism for writers to preserve Row Commit Versions for rows that are copied. The fresh Row Commit Version of a row is equal to the default generated Row Commit version. The stable Row Commit Version of a row is equal to the materialized Row Commit Version of the row when that column is present and the value is not NULL, otherwise it is equal to the default generated Commit Version. ## Reader Requirements for Row Tracking When Row Tracking is enabled (when the table property `delta.enableRowTracking` is set to `true`), then: - When Row IDs are requested, readers must reconstruct stable Row IDs as follows: 1. Readers must use the materialized Row ID if the column determined by `delta.rowTracking.materializedRowIdColumnName` is present in the data file and the column contains a non `null` value for a row. 2. Otherwise, readers must use the default generated Row ID of the `add` or `remove` action containing the row in all other cases. I.e. readers must add the index of the row in the file to the `baseRowId` of the `add` or `remove` action for the file containing the row. - When Row Commit Versions are requested, readers must reconstruct them as follows: 1. Readers must use the materialized Row Commit Versions if the column determined by `delta.rowTracking.materializedRowCommitVersionColumnName` is present in the data file and the column contains a non `null` value for a row. 2. Otherwise, Readers must use the default generated Row Commit Versions of the `add` or `remove` action containing the row in all other cases. I.e. readers must use the `defaultRowCommitVersion` of the `add` or `remove` action for the file containing the row. - Readers cannot read Row IDs and Row Commit Versions while reading change data files from `cdc` actions. ## Writer Requirements for Row Tracking When Row Tracking is supported (when the `writerFeatures` field of a table's `protocol` action contains `rowTracking`) and Row Tracking is not suspended (when `delta.rowTrackingSuspended` table property is absent or set to false), then: - Writers must assign unique fresh Row IDs to all rows that they commit. - Writers must set the `baseRowId` field in all `add` actions that they commit so that all default generated Row IDs are unique in the table version. Writers must never commit duplicate Row IDs in the table in any version. - Writers must set the `baseRowId` field in recommitted and checkpointed `add` actions and `remove` actions to the `baseRowId` value (if present) of the last committed `add` action with the same `path`. - Writers must track the high water mark, i.e. the highest fresh row id assigned. - The high water mark must be stored in a `domainMetadata` action with `delta.rowTracking` as the `domain` and a `configuration` containing a single key-value pair with `rowIdHighWaterMark` as the key and the highest assigned fresh row id as the value. - Writers must include a `domainMetadata` for `delta.rowTracking` whenever they assign new fresh Row IDs that are higher than `rowIdHighWaterMark` value of the current `domainMetadata` for `delta.rowTracking`. The `rowIdHighWaterMark` value in the `configuration` of this `domainMetadata` action must always be equal to or greater than the highest fresh Row ID committed so far. Writers can either commit this `domainMetadata` in the same commit, or they can reserve the fresh Row IDs in an earlier commit. - Writers must set the `baseRowId` field to a value that is higher than the row id high water mark. - Writer must assign fresh Row Commit Versions to all rows that they commit. - Writers must set the `defaultRowCommitVersion` field in new `add` actions to the version number of the log enty containing the `add` action. - Writers must set the `defaultRowCommitVersion` field in recommitted and checkpointed `add` actions and `remove` actions to the `defaultRowCommitVersion` of the last committed `add` action with the same `path`. On the other hand, when Row Tracking is supported but suspended (table property `delta.rowTrackingSuspended` is set to `true`), writers should not assign the `baseRowId` or the `defaultRowCommitVersion`. Writers can enable Row Tracking by setting `delta.enableRowTracking` to `true` in the `configuration` of the table's `metaData`. This is only allowed if the following requirements are satisfied: - The feature `rowTracking` has been added to the `writerFeatures` field of a table's `protocol` action either in the same version of the table or in an earlier version of the table. - The column name for the materialized Row IDs and Row Commit Versions have been assigned and added to the `configuration` in the table's `metaData` action using the keys `delta.rowTracking.materializedRowIdColumnName` and `delta.rowTracking.materializedRowCommitVersionColumnName` respectively. - The assigned column names must be unique. They must not be equal to the name of any other column in the table's schema. The assigned column names must remain unique in all future versions of the table. If [Column Mapping](#column-mapping) is enabled, then the assigned column name must be distinct from the physical column names of the table. - The `baseRowId` and `defaultRowCommitVersion` fields are set for all active `add` actions in the version of the table in which `delta.enableRowTracking` is set to `true`. - If the `baseRowId` and `defaultRowCommitVersion` fields are not set in some active `add` action in the table, then writers must first commit new `add` actions that set these fields to replace the `add` actions that do not have these fields set. This can be done in the commit that sets `delta.enableRowTracking` to `true` or in an earlier commit. The assigned `baseRowId` and `defaultRowCommitVersion` values must satisfy the same requirements as when assigning fresh Row IDs and fresh Row Commit Versions respectively. Furthermore, writers should also verify table property `delta.rowTrackingSuspended` is absent or set to false before enabling Row Tracking. When Row Tracking is enabled (when the table property `delta.enableRowTracking` is set to `true`), then: - Writers must assign stable Row IDs to all rows. - Stable Row IDs must be unique within a version of the table and must not be equal to the fresh Row IDs of other rows in the same version of the table. - Writers should preserve the stable Row IDs of rows that are updated or copied using materialized Row IDs. - The preserved stable Row ID (i.e. a stable Row ID that is not equal to the fresh Row ID of the same physical row) should be equal to the stable Row ID of the same logical row before it was updated or copied. - Materialized Row IDs must be written to the column determined by `delta.rowTracking.materializedRowIdColumnName` in the `configuration` of the table's `metaData` action. The value in this column must be set to `NULL` for stable Row IDs that are not preserved. - Writers must assign stable Row Commit Versions to all rows. - Writers should preserve the stable Row Commit Versions of rows that are copied (but not updated) using materialized Row Commit Versions. - The preserved stable Row Commit Version (i.e. a stable Row Commit Version that is not equal to the fresh Row Commit Version of the same physical row) should be equal to the stable Commit Version of the same logical row before it was copied. - Materialized Row Commit Versions must be written to the column determined by `delta.rowTracking.materializedRowCommitVersionColumnName` in the `configuration` of the table's `metaData` action. The value in this column must be set to `NULL` for stable Row Commit Versions that are not preserved (i.e. that are equal to the fresh Row Commit Version). - Writers should set `delta.rowTracking.preserved` in the `tags` of the `commitInfo` action to `true` whenever all the stable Row IDs of rows that are updated or copied and all the stable Row Commit Versions of rows that are copied were preserved. In particular, writers should set `delta.rowTracking.preserved` in the `tags` of the `commitInfo` action to `true` if no rows are updated or copied. Writers should set that flag to false otherwise. # VACUUM Protocol Check The `vacuumProtocolCheck` ReaderWriter feature ensures consistent application of reader and writer protocol checks during `VACUUM` operations, addressing potential protocol discrepancies and mitigating the risk of data corruption due to skipped writer checks. Enablement: - The table must be on Writer Version 7 and Reader Version 3. - The feature `vacuumProtocolCheck` must exist in the table `protocol`'s `writerFeatures` and `readerFeatures`. ## Writer Requirements for Vacuum Protocol Check This feature affects only the VACUUM operations; standard commits remain unaffected. Before performing a VACUUM operation, writers must ensure that they check the table's write protocol. This is most easily implemented by adding an unconditional write protocol check for all tables, which removes the need to examine individual table properties. Writers that do not implement VACUUM do not need to change anything and can safely write to tables that enable the feature. ## Reader Requirements for Vacuum Protocol Check For tables with Vacuum Protocol Check enabled, readers don’t need to understand or change anything new; they just need to acknowledge the feature exists. Making this feature a ReaderWriter feature (rather than solely a Writer feature) ensures that: - Older vacuum implementations, which only performed the Reader protocol check and lacked the Writer protocol check, will begin to fail if the table has `vacuumProtocolCheck` enabled. This change allows future writer features to have greater flexibility and safety in managing files within the table directory, eliminating the risk of older Vacuum implementations (that lack the Writer protocol check) accidentally deleting relevant files. # Clustered Table The Clustered Table feature facilitates the physical clustering of rows that share similar values on a predefined set of clustering columns. This enhances query performance when selective filters are applied to these clustering columns through data skipping. Clustering columns can be specified during the initial creation of a table, or they can be added later, provided that the table doesn't have partition columns. A table is defined as a clustered table through the following criteria: - When the feature `clustering` exists in the table `protocol`'s `writerFeatures`, then we say that the table is a clustered table. The feature `domainMetadata` is required in the table `protocol`'s `writerFeatures`. Enablement: - The table must be on Writer Version 7. - The feature `clustering` must exist in the table `protocol`'s `writerFeatures`, either during its creation or at a later stage, provided the table does not have partition columns. ## Writer Requirements for Clustered Table When the Clustered Table is supported (when the `writerFeatures` field of a table's `protocol` action contains `clustering`), then: - Writers must track clustering column names in a `domainMetadata` action with `delta.clustering` as the `domain` and a `configuration` containing all clustering column names. If [Column Mapping](#column-mapping) is enabled, the physical column names should be used. - Writers must write out [per-file statistics](#per-file-statistics) and per-column statistics for clustering columns in `add` action. If a new column is included in the clustering columns list, it is required for all table files to have statistics for these added columns. - When a clustering implementation clusters files, writers must set the name of the clustering implementation in the `clusteringProvider` field when adding `add` actions for clustered files. - By default, a clustering implementation must only recluster files that have the field `clusteringProvider` set to the name of the same clustering implementation, or to the names of other clustering implementations that are superseded by the current clustering implementation. In addition, a clustering implementation may cluster any files with an unset `clusteringProvider` field (i.e., unclustered files). - Writer is not required to cluster a specific file at any specific moment. - A clustering implementation is free to add additional information such as adding a new user-controlled metadata domain to keep track of its metadata. - Writers must not define clustered and partitioned table at the same time. The following is an example for the `domainMetadata` action definition of a table that leverages column mapping. ```json { "domainMetadata": { "domain": "delta.clustering", "configuration": "{\"clusteringColumns\":[\"col-daadafd7-7c20-4697-98f8-bff70199b1f9\", \"col-5abe0e80-cf57-47ac-9ffc-a861a3d1077e\"]}", "removed": false } } ``` The example above converts `configuration` field into JSON format, including escaping characters. Here's how it looks in plain JSON for better understanding. ```json { "clusteringColumns": [ "col-daadafd7-7c20-4697-98f8-bff70199b1f9", "col-5abe0e80-cf57-47ac-9ffc-a861a3d1077e" ] } ``` # Variant Data Type This feature enables support for the `variant` data type, which stores semi-structured data. The schema serialization method is described in [Schema Serialization Format](#schema-serialization-format). To support this feature: - The table must be on Reader Version 3 and Writer Version 7 - The feature `variantType` must exist in the table `protocol`'s `readerFeatures` and `writerFeatures`. ## Example JSON-Encoded Delta Table Schema with Variant types ``` { "type" : "struct", "fields" : [ { "name" : "raw_data", "type" : "variant", "nullable" : true, "metadata" : { } }, { "name" : "variant_array", "type" : { "type" : "array", "elementType" : { "type" : "variant" }, "containsNull" : false }, "nullable" : false, "metadata" : { } } ] } ``` ## Variant data in Parquet The Variant data type is represented as two binary encoded values, according to the [Spark Variant binary encoding specification](https://github.com/apache/spark/blob/master/common/variant/README.md). The two binary values are named `value` and `metadata`. When writing Variant data to parquet files, the Variant data is written as a single Parquet struct, with the following fields: Struct field name | Parquet primitive type | Description -|-|- value | binary | The binary-encoded Variant value, as described in [Variant binary encoding](https://github.com/apache/spark/blob/master/common/variant/README.md) metadata | binary | The binary-encoded Variant metadata, as described in [Variant binary encoding](https://github.com/apache/spark/blob/master/common/variant/README.md) The parquet struct must include the two struct fields `value` and `metadata`. Supported writers must write the two binary fields, and supported readers must read the two binary fields. [Variant shredding](https://github.com/apache/parquet-format/blob/master/VariantShredding.md) will be introduced in a separate `variantShredding` table feature. will be introduced later, as a separate `variantShredding` table feature. ## Writer Requirements for Variant Data Type When Variant type is supported (`writerFeatures` field of a table's `protocol` action contains `variantType`), writers: - must write a column of type `variant` to parquet as a struct containing the fields `value` and `metadata` and storing values that conform to the [Variant binary encoding specification](https://github.com/apache/spark/blob/master/common/variant/README.md) - must not write a parquet struct field named `typed_value` to avoid confusion with the field required by [Variant shredding](https://github.com/apache/parquet-format/blob/master/VariantShredding.md) with the same name. ## Reader Requirements for Variant Data Type When Variant type is supported (`readerFeatures` field of a table's `protocol` action contains `variantType`), readers: - must recognize and tolerate a `variant` data type in a Delta schema - must use the correct physical schema (struct-of-binary, with fields `value` and `metadata`) when reading a Variant data type from file - must make the column available to the engine: - [Recommended] Expose and interpret the struct-of-binary as a single Variant field in accordance with the [Spark Variant binary encoding specification](https://github.com/apache/spark/blob/master/common/variant/README.md). - [Alternate] Expose the raw physical struct-of-binary, e.g. if the engine does not support Variant. - [Alternate] Convert the struct-of-binary to a string, and expose the string representation, e.g. if the engine does not support Variant. ## Compatibility with other Delta Features Feature | Support for Variant Data Type -|- Partition Columns | **Supported:** A Variant column is allowed to be a non-partitioned column of a partitioned table.
**Unsupported:** Variant is not a comparable data type, so it cannot be included in a partition column. Clustered Tables | **Supported:** A Variant column is allowed to be a non-clustering column of a clustered table.
**Unsupported:** Variant is not a comparable data type, so it cannot be included in a clustering column. Delta Column Statistics | **Supported:** A Variant column supports the `nullCount` statistic.
**Unsupported:** Variant is not a comparable data type, so a Variant column does not support the `minValues` and `maxValues` statistics. Generated Columns | **Supported:** A Variant column is allowed to be used as a source in a generated column expression, as long as the Variant type is not the result type of the generated column expression.
**Unsupported:** The Variant data type is not allowed to be the result type of a generated column expression. Delta CHECK Constraints | **Supported:** A Variant column is allowed to be used for a CHECK constraint expression. Default Column Values | **Supported:** A Variant column is allowed to have a default column value. Change Data Feed | **Supported:** A table using the Variant data type is allowed to enable the Delta Change Data Feed. # In-Commit Timestamps The In-Commit Timestamps writer feature strongly associates a monotonically increasing timestamp with each commit by storing it in the commit's metadata. Enablement: - The table must be on Writer Version 7. - The feature `inCommitTimestamp` must exist in the table `protocol`'s `writerFeatures`. - The table property `delta.enableInCommitTimestamps` must be set to `true`. ## Writer Requirements for In-Commit Timestamps When In-Commit Timestamps is enabled, then: 1. Writers must write the `commitInfo` (see [Commit Provenance Information](#commit-provenance-information)) action in the commit. 2. The `commitInfo` action must be the first action in the commit. 3. The `commitInfo` action must include a field named `inCommitTimestamp`, of type `long` (see [Primitive Types](#primitive-types)), which represents the time (in milliseconds since the Unix epoch) when the commit is considered to have succeeded. It is the larger of two values: - The time, in milliseconds since the Unix epoch, at which the writer attempted the commit - One millisecond later than the previous commit's `inCommitTimestamp` 4. If the table has commits from a period when this feature was not enabled, provenance information around when this feature was enabled must be tracked in table properties: - The property `delta.inCommitTimestampEnablementVersion` must be used to track the version of the table when this feature was enabled. - The property `delta.inCommitTimestampEnablementTimestamp` must be the same as the `inCommitTimestamp` of the commit when this feature was enabled. 5. The `inCommitTimestamp` of the commit that enables this feature must be greater than the file modification time of the immediately preceding commit. ## Recommendations for Readers of Tables with In-Commit Timestamps For tables with In-Commit timestamps enabled, readers should use the `inCommitTimestamp` as the commit timestamp for operations like time travel and [`DESCRIBE HISTORY`](https://docs.delta.io/latest/delta-utility.html#retrieve-delta-table-history). If a table has commits from a period before In-Commit timestamps were enabled, the table properties `delta.inCommitTimestampEnablementVersion` and `delta.inCommitTimestampEnablementTimestamp` would be set and can be used to identify commits that don't have `inCommitTimestamp`. To correctly determine the commit timestamp for these tables, readers can use the following rules: 1. For commits with version >= `delta.inCommitTimestampEnablementVersion`, readers should use the `inCommitTimestamp` field of the `commitInfo` action. 2. For commits with version < `delta.inCommitTimestampEnablementVersion`, readers should use the file modification timestamp. Furthermore, when attempting timestamp-based time travel where table state must be fetched as of `timestamp X`, readers should use the following rules: 1. If `timestamp X` >= `delta.inCommitTimestampEnablementTimestamp`, only table versions >= `delta.inCommitTimestampEnablementVersion` should be considered for the query. 2. Otherwise, only table versions less than `delta.inCommitTimestampEnablementVersion` should be considered for the query. # Type Widening The Type Widening feature enables changing the type of a column or field in an existing Delta table to a wider type. The supported type changes are: - Integer widening: - `Byte` -> `Short` -> `Int` -> `Long` - Floating-point widening: - `Float` -> `Double` - `Byte`, `Short` or `Int` -> `Double` - Date widening: - `Date` -> `Timestamp without timezone` - Decimal widening - `p` and `s` denote the decimal precision and scale respectively. - `Decimal(p, s)` -> `Decimal(p + k1, s + k2)` where `k1 >= k2 >= 0`. - `Byte`, `Short` or `Int` -> `Decimal(10 + k1, k2)` where `k1 >= k2 >= 0`. - `Long` -> `Decimal(20 + k1, k2)` where `k1 >= k2 >= 0`. To support this feature: - The table must be on Reader version 3 and Writer Version 7. - The feature `typeWidening` must exist in the table `protocol`'s `readerFeatures` and `writerFeatures`, either during its creation or at a later stage. When supported: - A table may have a metadata property `delta.enableTypeWidening` in the Delta schema set to `true`. Writers must reject widening type changes when this property isn't set to `true`. - The `metadata` for a column or field in the table schema may contain the key `delta.typeChanges` storing a history of type changes for that column or field. ### Type Change Metadata Type changes applied to a table are recorded in the table schema and stored in the `metadata` of their nearest ancestor [StructField](#struct-field) using the key `delta.typeChanges`. The value for the key `delta.typeChanges` must be a JSON list of objects, where each object contains the following fields: Field Name | optional/required | Description -|-|- `fromType`| required | The type of the column or field before the type change. `toType`| required | The type of the column or field after the type change. `fieldPath`| optional | When updating the type of a map key/value or array element only: the path from the struct field holding the metadata to the map key/value or array element that was updated. The `fieldPath` value is "key", "value" and "element" when updating resp. the type of a map key, map value and array element. The `fieldPath` value for nested maps and nested arrays are prefixed by their parents' path, separated by dots. The following is an example for the definition of a column that went through two type changes: ```json { "name" : "e", "type" : "long", "nullable" : true, "metadata" : { "delta.typeChanges": [ { "fromType": "short", "toType": "integer" }, { "fromType": "integer", "toType": "long" } ] } } ``` The following is an example for the definition of a column after changing the type of a map key: ```json { "name" : "e", "type" : { "type": "map", "keyType": "double", "valueType": "integer", "valueContainsNull": true }, "nullable" : true, "metadata" : { "delta.typeChanges": [ { "fromType": "float", "toType": "double", "fieldPath": "key" } ] } } ``` The following is an example for the definition of a column after changing the type of a map value nested in an array: ```json { "name" : "e", "type" : { "type": "array", "elementType": { "type": "map", "keyType": "string", "valueType": "decimal(10, 4)", "valueContainsNull": true }, "containsNull": true }, "nullable" : true, "metadata" : { "delta.typeChanges": [ { "fromType": "decimal(6, 2)", "toType": "decimal(10, 4)", "fieldPath": "element.value" } ] } } ``` ## Writer Requirements for Type Widening When Type Widening is supported (when the `writerFeatures` field of a table's `protocol` action contains `typeWidening`), then: - Writers must reject applying any unsupported type change. - Writers must reject applying type changes not supported by [Iceberg V2](https://iceberg.apache.org/spec/#schema-evolution) when either the [Iceberg Compatibility V1](#iceberg-compatibility-v1) or [Iceberg Compatibility V2](#iceberg-compatibility-v2) table feature is supported: - `Byte`, `Short` or `Int` -> `Double` - `Date` -> `Timestamp without timezone` - Decimal scale increase - `Byte`, `Short`, `Int` or `Long` -> `Decimal` - Writers must record type change information in the `metadata` of the nearest ancestor [StructField](#struct-field). See [Type Change Metadata](#type-change-metadata). - Writers must preserve the `delta.typeChanges` field in the metadata fields in the schema when the table schema is updated. - Writers may remove the `delta.typeChanges` metadata in the table schema if all data files use the same field types as the table schema. When Type Widening is enabled (when the table property `delta.enableTypeWidening` is set to `true`), then: - Writers should allow updating the table schema to apply a supported type change to a column, struct field, map key/value or array element. When removing the Type Widening table feature from the table, in the version that removes `typeWidening` from the `writerFeatures` and `readerFeatures` fields of the table's `protocol` action: - Writers must ensure no `delta.typeChanges` metadata key is present in the table schema. This may require rewriting existing data files to ensure that all data files use the same field types as the table schema in order to fulfill the requirement to remove type widening metadata. - Writers must ensure that the table property `delta.enableTypeWidening` is not set. ## Reader Requirements for Type Widening When Type Widening is supported (when the `readerFeatures` field of a table's `protocol` action contains `typeWidening`), then: - Readers must allow reading data files written before the table underwent any supported type change, and must convert such values to the current, wider type. - Readers must validate that they support all type changes in the `delta.typeChanges` field in the table schema for the table version they are reading and fail when finding any unsupported type change. # Requirements for Writers This section documents additional requirements that writers must follow in order to preserve some of the higher level guarantees that Delta provides. ## Creation of New Log Entries - Writers MUST never overwrite an existing log entry. When ever possible they should use atomic primitives of the underlying filesystem to ensure concurrent writers do not overwrite each other's entries. ## Consistency Between Table Metadata and Data Files - Any column that exists in a data file present in the table MUST also be present in the metadata of the table. - Values for all partition columns present in the schema MUST be present for all files in the table. - Columns present in the schema of the table MAY be missing from data files. Readers SHOULD fill these missing columns in with `null`. ## Delta Log Entries - A single log entry MUST NOT include more than one action that reconciles with each other. - Add / Remove actions with the same `(path, DV)` tuple. - More than one Metadata action - More than one protocol action - More than one SetTransaction with the same `appId` ## Checkpoints Each row in the checkpoint corresponds to a single action. The checkpoint **must** contain all information regarding the following actions: * The [protocol version](#Protocol-Evolution) * The [metadata](#Change-Metadata) of the table * Files that have been [added](#Add-File-and-Remove-File) and not yet removed * Files that were recently [removed](#Add-File-and-Remove-File) and have not yet expired * [Transaction identifiers](#Transaction-Identifiers) * [Domain Metadata](#Domain-Metadata) * [Checkpoint Metadata](#checkpoint-metadata) - Requires [V2 checkpoints](#v2-spec) * [Sidecar File](#sidecar-files) - Requires [V2 checkpoints](#v2-spec) All of these actions are stored as their individual columns in parquet as struct fields. Any missing column should be treated as null. Checkpoints must not preserve [commit provenance information](#commit-provenance-information) nor [change data](#add-cdc-file) actions. Within the checkpoint, the `add` struct may or may not contain the following columns based on the configuration of the table: - partitionValues_parsed: In this struct, the column names correspond to the partition columns and the values are stored in their corresponding data type. This is a required field when the table is partitioned and the table property `delta.checkpoint.writeStatsAsStruct` is set to `true`. If the table is not partitioned, this column can be omitted. For example, for partition columns `year`, `month` and `event` with data types `int`, `int` and `string` respectively, the schema for this field will look like: ``` |-- add: struct | |-- partitionValues_parsed: struct | | |-- year: int | | |-- month: int | | |-- event: string ``` - stats: Column level statistics can be stored as a JSON string in the checkpoint. This field needs to be written when statistics are available and the table property: `delta.checkpoint.writeStatsAsJson` is set to `true` (which is the default). When this property is set to `false`, this field should be omitted from the checkpoint. - stats_parsed: The stats can be stored in their [original format](#Per-file-Statistics). This field needs to be written when statistics are available and the table property: `delta.checkpoint.writeStatsAsStruct` is set to `true`. When this property is set to `false` (which is the default), this field should be omitted from the checkpoint. Within the checkpoint, the `remove` struct does not contain the `stats` and `tags` fields because the `remove` actions stored in checkpoints act only as tombstones for VACUUM operations, and VACUUM tombstones do not require `stats` or `tags`. These fields are only stored in Delta JSON commit files. Refer to the [appendix](#checkpoint-schema) for an example on the schema of the checkpoint. Delta supports two checkpoint specs and three kind of checkpoint naming schemes. ### Checkpoint Specs Delta supports following two checkpoint specs: #### V2 Spec This checkpoint spec allows putting [add and remove file](#Add-File-and-Remove-File) in the [sidecar files](#sidecar-files). This spec can be used only when [v2 checkpoint table feature](#v2-checkpoint-table-feature) is enabled. Checkpoints following V2 spec have the following structure: - Each v2 spec checkpoint includes exactly one [Checkpoint Metadata](#checkpoint-metadata) action. - Remaining rows in the V2 spec checkpoint refer to the other actions mentioned [here](#checkpoints-1) - All the non-file actions i.e. all actions except [add and remove file](#Add-File-and-Remove-File) must be part of the v2 spec checkpoint itself. - A writer could choose to include the [add and remove file](#Add-File-and-Remove-File) action in the V2 spec Checkpoint or they could write the [add and remove file](#Add-File-and-Remove-File) actions in separate [sidecar files](#sidecar-files). These sidecar files will then be referenced in the V2 spec checkpoint. All sidecar files reside in the `_delta_log/_sidecars` directory. - A V2 spec Checkpoint could reference zero or more [sidecar file actions](#sidecar-file-information). Note: A V2 spec Checkpoint can either have all the [add and remove file](#Add-File-and-Remove-File) actions embedded inside itself or all of them should be in [sidecar files](#sidecar-files). Having partial add and remove file actions in V2 Checkpoint and partial entries in sidecar files is not allowed. After producing a V2 spec checkpoint, a writer can choose to embed some or all of the V2 spec checkpoint in the `_last_checkpoint` file, so that readers don't have to read the V2 Checkpoint. E.g. showing the content of V2 spec checkpoint: ``` {"checkpointMetadata":{"version":364475,"tags":{}}} {"metaData":{...}} {"protocol":{...}} {"txn":{"appId":"3ba13872-2d47-4e17-86a0-21afd2a22395","version":364475}} {"txn":{"appId":"3ae45b72-24e1-865a-a211-34987ae02f2a","version":4389}} {"sidecar":{"path":"3a0d65cd-4056-49b8-937b-95f9e3ee90e5.parquet","sizeInBytes":2341330,"modificationTime":1512909768000,"tags":{}} {"sidecar":{"path":"016ae953-37a9-438e-8683-9a9a4a79a395.parquet","sizeInBytes":8468120,"modificationTime":1512909848000,"tags":{}} ``` Another example of a v2 spec checkpoint without sidecars: ``` {"checkpointMetadata":{"version":364475,"tags":{}}} {"metaData":{...}} {"protocol":{...}} {"txn":{"appId":"3ba13872-2d47-4e17-86a0-21afd2a22395","version":364475}} {"add":{"path":"date=2017-12-10/part-000...c000.gz.parquet",...} {"add":{"path":"date=2017-12-09/part-000...c000.gz.parquet",...} {"remove":{"path":"date=2017-12-08/part-000...c000.gz.parquet",...} ``` #### V1 Spec The V1 Spec does not support [sidecar files](#sidecar-files) and [checkpoint metadata](#checkpoint-metadata). These are flat checkpoints which contains all actions mentioned [here](#checkpoints-1). ### Checkpoint Naming Scheme Delta supports three checkpoint naming schemes: UUID-named, classic, and multi-part. #### UUID-named checkpoint This naming scheme represents a [V2 spec checkpoint](#v2-spec) with following file name: `n.checkpoint.u.{json/parquet}`, where `u` is a UUID and `n` is the snapshot version that this checkpoint represents. The UUID-named checkpoints may be in JSON or parquet format. Since these are following [V2 spec](#v2-spec), they must have a [checkpoint metadata](#checkpoint-metadata) action and may reference zero or more checkpoint [sidecar files](#sidecar-files). Example-1: Json UUID-named checkpoint with sidecars ``` 00000000000000000010.checkpoint.80a083e8-7026-4e79-81be-64bd76c43a11.json _sidecars/016ae953-37a9-438e-8683-9a9a4a79a395.parquet _sidecars/3a0d65cd-4056-49b8-937b-95f9e3ee90e5.parquet _sidecars/7d17ac10-5cc3-401b-bd1a-9c82dd2ea032.parquet ``` Example-2: Parquet UUID-named checkpoint with sidecars ``` 00000000000000000020.checkpoint.80a083e8-7026-4e79-81be-64bd76c43a11.parquet _sidecars/016ae953-37a9-438e-8683-9a9a4a79a395.parquet _sidecars/3a0d65cd-4056-49b8-937b-95f9e3ee90e5.parquet ``` Example-3: Json UUID-named checkpoint without sidecars ``` 00000000000000000112.checkpoint.80a083e8-7026-4e79-81be-64bd76c43a11.json ``` #### Classic checkpoint A classic checkpoint for version `n` uses the file name `n.checkpoint.parquet`. For example: ``` 00000000000000000010.checkpoint.parquet ``` If two checkpoint writers race to create the same classic checkpoint, the latest writer wins. However, this should not matter because both checkpoints should contain the same information and a reader could safely use either one. A classic checkpoint could: 1. Either follow [V1 spec](#v1-spec) or 2. Could follow [V2 spec](#v2-spec). This is possible only when [V2 Checkpoint table feature](#v2-checkpoint-table-feature) is enabled. In this case it must include [checkpoint metadata](#checkpoint-metadata) and may or may not have [sidecar files](#sidecar-file-information). #### Multi-part checkpoint Multi-part checkpoint uses parquet format. This checkpoint type is [deprecated](#problems-with-multi-part-checkpoints) and writers should avoid using it. A multi-part checkpoint for version `n` consists of `p` "part" files (`p > 1`), where part `o` of `p` is named `n.checkpoint.o.p.parquet`. For example: ``` 00000000000000000010.checkpoint.0000000001.0000000003.parquet 00000000000000000010.checkpoint.0000000002.0000000003.parquet 00000000000000000010.checkpoint.0000000003.0000000003.parquet ``` For [safety reasons](#problems-with-multi-part-checkpoints), multi-part checkpoints MUST be clustered by spark-style hash partitioning. If the table supports [Deletion Vectors](#deletion-vectors), the partitioning key is the logical file identifier `(path, dvId)`; otherwise the key is just `path` (not `(path, NULL)`). This ensures deterministic content in each part file in case of multiple attempts to write the files -- even when older and newer Delta clients race. ##### Problems with multi-part checkpoints Because they cannot be written atomically, multi-part checkpoints have several weaknesses: 1. A writer cannot validate the content of the just-written checkpoint before readers could start using it. 2. Two writers who race to produce the same checkpoint (same version, same number of parts) can overwrite each other, producing an arbitrary mix of checkpoint part files. If an overwrite changes the content of a file in any way, the resulting checkpoint may not produce an accurate snapshot. 3. Not amenable to performance and scalability optimizations. For example, there is no way to store skipping stats for checkpoint parts, nor to reuse checkpoint part files across multiple checkpoints. 4. Multi-part checkpoints also bloat the _delta_log dir and slow down LIST operations. The [UUID-named](#uuid-named-checkpoint) checkpoint (which follows [V2 spec](#v2-spec)) solves all of these problems and should be preferred over multi-part checkpoints. For this reason, Multi-part checkpoints are forbidden when [V2 Checkpoints table feature](#v2-checkpoint-table-feature) is enabled. ### Handling Backward compatibility while moving to UUID-named v2 Checkpoints A UUID-named v2 Checkpoint should only be created by clients if the [v2 checkpoint table feature](#v2-checkpoint-table-feature) is enabled. When UUID-named v2 checkpoints are enabled, Writers should occasionally create a v2 [Classic Checkpoint](#classic-checkpoint) to maintain compatibility with older clients which do not support [v2 checkpoint table feature](#v2-checkpoint-table-feature) and so do not recognize UUID-named checkpoints. These classic checkpoints have the same content as the UUID-named v2 checkpoint, but older clients will recognize the classic file name, allowing them to extract [Protocol](#protocol-evolution) and fail gracefully with an invalid protocol version error on v2-checkpoint-enabled tables. Writers should create classic checkpoints often enough to allow older clients to discover them and fail gracefully. ### Allowed combinations for `checkpoint spec` <-> `checkpoint file naming` Checkpoint Spec | [UUID-named](#uuid-named-checkpoint) | [classic](#classic-checkpoint) | [multi-part](#multi-part-checkpoint) -|-|-|- [V1](#v1-spec) | Invalid | Valid | Valid [V2](#v2-spec) | Valid | Valid | Invalid ### Metadata Cleanup The _delta_log directory grows over time as more and more commits and checkpoints are accumulated. Implementations are recommended to delete expired commits and checkpoints in order to reduce the directory size. The following steps could be used to do cleanup of the DeltaLog directory: 1. Identify a threshold (in days) uptil which we want to preserve the deltaLog. Let's refer to midnight UTC of that day as `cutOffTimestamp`. The newest commit not newer than the `cutOffTimestamp` is the `cutoffCommit`, because a commit exactly at midnight is an acceptable cutoff. We want to retain everything including and after the `cutoffCommit`. 2. Identify the newest checkpoint that is not newer than the `cutOffCommit`. A checkpoint at the `cutOffCommit` is ideal, but an older one will do. Let's call it `cutOffCheckpoint`. We need to preserve the `cutOffCheckpoint` (both the checkpoint file and the JSON commit file at that version) and all published commits after it. The JSON commit file at the `cutOffCheckpoint` version must be preserved because checkpoints do not preserve [commit provenance information](#commit-provenance-information) (e.g., `commitInfo` actions), which may be required by table features such as [In-Commit Timestamps](#in-commit-timestamps). All published commits after `cutOffCheckpoint` must be preserved to enable time travel for commits between `cutOffCheckpoint` and the next available checkpoint. - If no `cutOffCheckpoint` can be found, do not proceed with metadata cleanup as there is nothing to cleanup. 3. Delete all [delta log entries](#delta-log-entries), [checkpoint files](#checkpoints), and [version checksum files](#version-checksum-file) before the `cutOffCheckpoint` checkpoint. Also delete all the [log compaction files](#log-compaction-files) having startVersion <= `cutOffCheckpoint`'s version. - Also delete all the [staged commit files](#staged-commit) having version <= `cutOffCheckpoint`'s version from the `_delta_log/_staged_commits` directory. 4. Now read all the available [checkpoints](#checkpoints-1) in the _delta_log directory and identify the corresponding [sidecar files](#sidecar-files). These sidecar files need to be protected. 5. List all the files in `_delta_log/_sidecars` directory, preserve files that are less than a day old (as of midnight UTC), to not break in-progress checkpoints. Also preserve the referenced sidecar files identified in Step-4 above. Delete everything else. ## Data Files - Data files MUST be uniquely named and MUST NOT be overwritten. The reference implementation uses a GUID in the name to ensure this property. ## Append-only Tables To support this feature: - The table must be on a Writer Version starting from 2 up to 7. - If the table is on Writer Version 7, the feature `appendOnly` must exist in the table `protocol`'s `writerFeatures`. When supported, and if the table has a property `delta.appendOnly` set to `true`: - New log entries MUST NOT change or remove data from the table. - New log entries may rearrange data (i.e. `add` and `remove` actions where `dataChange=false`). To remove the append-only restriction, the table property `delta.appendOnly` must be set to `false`, or it must be removed. ## Column Invariants To support this feature - If the table is on a Writer Version starting from 2 up to 6, Column Invariants are always enabled. - If the table is on Writer Version 7, the feature `invariants` must exist in the table `protocol`'s `writerFeatures`. When supported: - The `metadata` for a column in the table schema MAY contain the key `delta.invariants`. - The value of `delta.invariants` SHOULD be parsed as a JSON string containing a boolean SQL expression at the key `expression.expression` (that is, `{"expression": {"expression": ""}}`). - Writers MUST abort any transaction that adds a row to the table, where an invariant evaluates to `false` or `null`. For example, given the schema string (pretty printed for readability. The entire schema string in the log should be a single JSON line): ```json { "type": "struct", "fields": [ { "name": "x", "type": "integer", "nullable": true, "metadata": { "delta.invariants": "{\"expression\": { \"expression\": \"x > 3\"} }" } } ] } ``` Writers should reject any transaction that contains data where the expression `x > 3` returns `false` or `null`. ## CHECK Constraints To support this feature: - If the table is on a Writer Version starting from 3 up to 6, CHECK Constraints are always supported. - If the table is on Writer Version 7, a feature name `checkConstraints` must exist in the table `protocol`'s `writerFeatures`. CHECK constraints are stored in the map of the `configuration` field in [Metadata](#change-metadata). Each CHECK constraint has a name and is stored as a key value pair. The key format is `delta.constraints.{name}`, and the value is a SQL expression string whose return type must be `Boolean`. Columns referred by the SQL expression must exist in the table schema. Rows in a table must satisfy CHECK constraints. In other words, evaluating the SQL expressions of CHECK constraints must return `true` for each row in a table. For example, a key value pair (`delta.constraints.birthDateCheck`, `birthDate > '1900-01-01'`) means there is a CHECK constraint called `birthDateCheck` in the table and the value of the `birthDate` column in each row must be greater than `1900-01-01`. Hence, a writer must follow the rules below: - CHECK Constraints may not be added to a table unless the above "to support this feature" rules are satisfied. When adding a CHECK Constraint to a table for the first time, writers are allowed to submit a `protocol` change in the same commit to add support of this feature. - When adding a CHECK constraint to a table, a writer must validate the existing data in the table and ensure every row satisfies the new CHECK constraint before committing the change. Otherwise, the write operation must fail and the table must stay unchanged. - When writing to a table that contains CHECK constraints, every new row being written to the table must satisfy CHECK constraints in the table. Otherwise, the write operation must fail and the table must stay unchanged. ## Generated Columns To support this feature: - If the table is on a Writer Version starting from 4 up to 6, Generated Columns are always supported. - If the table is on Writer Version 7, a feature name `generatedColumns` must exist in the table `protocol`'s `writerFeatures`. When supported: - The `metadata` for a column in the table schema MAY contain the key `delta.generationExpression`. - The value of `delta.generationExpression` SHOULD be parsed as a SQL expression. - Writers MUST enforce that any data writing to the table satisfy the condition `( <=> ) IS TRUE`. `<=>` is the NULL-safe equal operator which performs an equality comparison like the `=` operator but returns `TRUE` rather than NULL if both operands are `NULL` ## Default Columns Delta supports defining default expressions for columns on Delta tables. Delta will generate default values for columns when users do not explicitly provide values for them when writing to such tables, or when the user explicitly specifies the `DEFAULT` SQL keyword for any such column. Semantics for write and read operations: - Note that this metadata only applies for write operations, not read operations. - Table write operations (such as SQL INSERT, UPDATE, and MERGE commands) will use the default values. For example, this SQL command will use default values: `INSERT INTO t VALUES (42, DEFAULT);` - Table operations that add new columns (such as SQL ALTER TABLE ... ADD COLUMN commands) MUST not specify a default value for any column in the same command that the column is created. For example, this SQL command is not supported in Delta Lake: `ALTER TABLE t ADD COLUMN c INT DEFAULT 42;` - Note that it is acceptable to assign or update default values for columns that were already created in previous commands, however. For example, this SQL command is valid: `ALTER TABLE t ALTER COLUMN c SET DEFAULT 42;` Enablement: - The table must be on Writer Version 7, and a feature name `allowColumnDefaults` must exist in the table `protocol`'s `writerFeatures`. When enabled: - The `metadata` for the column in the table schema MAY contain the key `CURRENT_DEFAULT`. - The value of `CURRENT_DEFAULT` SHOULD be parsed as a SQL expression. - Writers MUST enforce that before writing any rows to the table, for each such requested row that lacks any explicit value (including NULL) for columns with default values, the writing system will assign the result of evaluating the default value expression for each such column as the value for that column in the row. By the same token, if the engine specified the explicit `DEFAULT` SQL keyword for any column, the expression result must be substituted in the same way. - All columns of `variant` type must default to null. ## Identity Columns Delta supports defining Identity columns on Delta tables. Delta will generate unique values for Identity columns when users do not explicitly provide values for them when writing to such tables. To support Identity Columns: - The table must be on Writer Version 6, or - The table must be on Writer Version 7, and a feature name `identityColumns` must exist in the table `protocol`'s `writerFeatures`. When supported, the `metadata` for a column in the table schema MAY contain the following keys for Identity Column properties: - `delta.identity.start`: Starting value for the Identity column. This is a long type value. It should not be changed after table creation. - `delta.identity.step`: Increment to the next Identity value. This is a long type value. It cannot be set to 0. It should not be changed after table creation. - `delta.identity.highWaterMark`: The highest value generated for the Identity column. This is a long type value. When `delta.identity.step` is positive (negative), this should be the largest (smallest) value in the column. - `delta.identity.allowExplicitInsert`: True if this column allows explicitly inserted values. This is a boolean type value. It should not be changed after table creation. When `delta.identity.allowExplicitInsert` is true, writers should meet the following requirements: - Users should be allowed to provide their own values for Identity columns. When `delta.identity.allowExplicitInsert` is false, writers should meet the following requirements: - Users should not be allowed to provide their own values for Identity columns. - Delta should generate values that satisfy the following requirements - The new value does not already exist in the column. - The new value should satisfy `value = start + k * step` where k is a non-negative integer. - The new value should be higher than `delta.identity.highWaterMark`. When `delta.identity.step` is positive (negative), the new value should be the greater (smaller) than `delta.identity.highWaterMark`. - Overflow when calculating generated Identity values should be detected and such writes should not be allowed. - `delta.identity.highWaterMark` should be updated to the new highest value when the write operation commits. ## Writer Version Requirements The requirements of the writers according to the protocol versions are summarized in the table below. Each row inherits the requirements from the preceding row.
| Requirements -|- Writer Version 2 | - Respect [Append-only Tables](#append-only-tables)
- Respect [Column Invariants](#column-invariants) Writer Version 3 | - Enforce `delta.checkpoint.writeStatsAsJson`
- Enforce `delta.checkpoint.writeStatsAsStruct`
- Respect [`CHECK` constraints](#check-constraints) Writer Version 4 | - Respect [Change Data Feed](#add-cdc-file)
- Respect [Generated Columns](#generated-columns) Writer Version 5 | Respect [Column Mapping](#column-mapping) Writer Version 6 | Respect [Identity Columns](#identity-columns) Writer Version 7 | Respect [Table Features](#table-features) for writers # Requirements for Readers This section documents additional requirements that readers must respect in order to produce correct scans of a Delta table. ## Reader Version Requirements The requirements of the readers according to the protocol versions are summarized in the table below. Each row inherits the requirements from the preceding row.
| Requirements -|- Reader Version 2 | Respect [Column Mapping](#column-mapping) Reader Version 3 | Respect [Table Features](#table-features) for readers
- Writer Version must be 7 # Appendix ## Valid Feature Names in Table Features Feature | Name | Readers or Writers? -|-|- [Append-only Tables](#append-only-tables) | `appendOnly` | Writers only [Column Invariants](#column-invariants) | `invariants` | Writers only [`CHECK` constraints](#check-constraints) | `checkConstraints` | Writers only [Generated Columns](#generated-columns) | `generatedColumns` | Writers only [Default Columns](#default-columns) | `allowColumnDefaults` | Writers only [Change Data Feed](#add-cdc-file) | `changeDataFeed` | Writers only [Column Mapping](#column-mapping) | `columnMapping` | Readers and writers [Identity Columns](#identity-columns) | `identityColumns` | Writers only [Deletion Vectors](#deletion-vectors) | `deletionVectors` | Readers and writers [Row Tracking](#row-tracking) | `rowTracking` | Writers only [Timestamp without Timezone](#timestamp-without-timezone-timestampNtz) | `timestampNtz` | Readers and writers [Domain Metadata](#domain-metadata) | `domainMetadata` | Writers only [V2 Checkpoint](#v2-checkpoint-table-feature) | `v2Checkpoint` | Readers and writers [Catalog-managed Tables](#catalog-managed-tables) | `catalogManaged` | Readers and writers [Iceberg Compatibility V1](#iceberg-compatibility-v1) | `icebergCompatV1` | Writers only [Iceberg Compatibility V2](#iceberg-compatibility-v2) | `icebergCompatV2` | Writers only [Clustered Table](#clustered-table) | `clustering` | Writers only [VACUUM Protocol Check](#vacuum-protocol-check) | `vacuumProtocolCheck` | Readers and Writers [In-Commit Timestamps](#in-commit-timestamps) | `inCommitTimestamp` | Writers only ## Deletion Vector Format Deletion Vectors are basically sets of row indexes, that is 64-bit integers that describe the position (index) of a row in a parquet file starting from zero. We store these sets in a compressed format. The fundamental building block for this is the open source [RoaringBitmap](https://roaringbitmap.org/) library. RoaringBitmap is a flexible format for storing 32-bit integers that automatically switches between three different encodings at the granularity of a 16-bit block (64K values): - Simple integer array, when the number of values in the block is small. - Bitmap-compressed, when the number of values in the block is large and scattered. - Run-length encoded, when the number of values in the block is large, but clustered. The serialization format is [standardized](https://github.com/RoaringBitmap/RoaringFormatSpec), and both [Java](https://github.com/lemire/RoaringBitmap/) and [C/C++](https://github.com/RoaringBitmap/CRoaring) implementations are available (among others). The above description only applies to 32-bit bitmaps, but Deletion Vectors use 64-bit integers. In order to extend coverage from 32 to 64 bits, RoaringBitmaps defines a "portable" serialization format in the [RoaringBitmaps Specification](https://github.com/RoaringBitmap/RoaringFormatSpec#extention-for-64-bit-implementations). This format essentially splits the space into an outer part with the most significant 32-bit "keys" indexing the least significant 32-bit RoaringBitmaps in ascending sequence. The spec calls these least signficant 32-bit RoaringBitmaps "buckets". Bytes | Name | Description -|-|- 0 – 7 | numBuckets | The number of distinct 32-bit buckets in this bitmap. `repeat for each bucket b` | | For each bucket in ascending order of keys. `` – ` + 3` | key | The most significant 32-bit of all the values in this bucket. ` + 4` – `` | bucketData | A serialized 32-bit RoaringBitmap with all the least signficant 32-bit entries in this bucket. The 32-bit serialization format then consists of a header that describes all the (least signficant) 16-bit containers, their types (s. above), and their key (most significant 16-bits). This is followed by the data for each individual container in a container-specific format. Reference Implementations of the Roaring format: - [32-bit Java RoaringBitmap](https://github.com/RoaringBitmap/RoaringBitmap/blob/c7993318d7224cd3cc0244dcc99c8bbc5ddb0c87/RoaringBitmap/src/main/java/org/roaringbitmap/RoaringArray.java#L905-L949) - [64-bit Java RoaringNavigableBitmap](https://github.com/RoaringBitmap/RoaringBitmap/blob/c7993318d7224cd3cc0244dcc99c8bbc5ddb0c87/RoaringBitmap/src/main/java/org/roaringbitmap/longlong/Roaring64NavigableMap.java#L1253-L1260) Delta uses the format described above as a black box, but with two additions: 1. We prepend a "magic number", which can be used to make sure we are reading the correct format and also retains the ability to evolve the format in the future. 2. We require that every "key" (s. above) in the bitmap has a 0 as its most significant bit. This ensures that in Java, where values are read signed, we never read negative keys. The concrete serialization format is as follows (all numerical values are written in little endian byte order): Bytes | Name | Description -|-|- 0 — 3 | magicNumber | 1681511377; Indicates that the following bytes are serialized in this exact format. Future alternative—but related—formats must have a different magic number, for example by incrementing this one. 4 — end | bitmap | A serialized 64-bit bitmap in the portable standard format as defined in the [RoaringBitmaps Specification](https://github.com/RoaringBitmap/RoaringFormatSpec#extention-for-64-bit-implementations). This can be treated as a black box by any Delta implementation that has a native, standard-compliant RoaringBitmap library available to pass these bytes to. ### Deletion Vector File Storage Format Deletion Vectors can be stored in files in cloud storage or inline in the Delta log. The format for storing DVs in file storage is one (or more) DV, using the 64-bit RoaringBitmaps described in the previous section, per file, together with a checksum for each DV. The concrete format is as follows, with all numerical values written in big endian byte order: Bytes | Name | Description -|-|- 0 | version | The format version of this file: `1` for the format described here. `repeat for each DV i` | | For each DV `` — ` + 3` | dataSize | Size of this DV’s data (without the checksum) ` + 4` — ` + 4 + dataSize - 1` | bitmapData | One 64-bit RoaringBitmap serialised as described above. ` + 4 + dataSize` — ` + 4 + dataSize + 3` | checksum | CRC-32 checksum of `bitmapData` ## Per-file Statistics `add` and `remove` actions can optionally contain statistics about the data in the file being added or removed from the table. These statistics can be used for eliminating files based on query predicates or as inputs to query optimization. Global statistics record information about the entire file. The following global statistic is currently supported: Name | Description -|- numRecords | The number of records in this data file. tightBounds | Whether per-column statistics are currently **tight** or **wide** (see below). For any logical file where `deletionVector` is not `null`, the `numRecords` statistic *must* be present and accurate. That is, it must equal the number of records in the data file, not the valid records in the logical file. In the presence of [Deletion Vectors](#Deletion-Vectors) the statistics may be somewhat outdated, i.e. not reflecting deleted rows yet. The flag `stats.tightBounds` indicates whether we have **tight bounds** (i.e. the min/maxValue exists[^1] in the valid state of the file) or **wide bounds** (i.e. the minValue is <= all valid values in the file, and the maxValue >= all valid values in the file). These upper/lower bounds are sufficient information for data skipping. Note, `stats.tightBounds` should be treated as `true` when it is not explicitly present in the statistics. Per-column statistics record information for each column in the file and they are encoded, mirroring the schema of the actual data. For example, given the following data schema: ``` |-- a: struct | |-- b: struct | | |-- c: long ``` Statistics could be stored with the following schema: ``` |-- stats: struct | |-- numRecords: long | |-- tightBounds: boolean | |-- minValues: struct | | |-- a: struct | | | |-- b: struct | | | | |-- c: long | |-- maxValues: struct | | |-- a: struct | | | |-- b: struct | | | | |-- c: long ``` The following per-column statistics are currently supported: Name | Description (`stats.tightBounds=true`) | Description (`stats.tightBounds=false`) -|-|- nullCount | The number of `null` values for this column |

If the `nullCount` for a column equals the physical number of records (`stats.numRecords`) then **all** valid rows for this column must have `null` values (the reverse is not necessarily true).

If the `nullCount` for a column equals 0 then **all** valid rows are non-`null` in this column (the reverse is not necessarily true).

If the `nullCount` for a column is any value other than these two special cases, the value carries no information and should be treated as if absent.

minValues | A value that is equal to the smallest valid value[^1] present in the file for this column. If all valid rows are null, this carries no information. | A value that is less than or equal to all valid values[^1] present in this file for this column. If all valid rows are null, this carries no information. maxValues | A value that is equal to the largest valid value[^1] present in the file for this column. If all valid rows are null, this carries no information. | A value that is greater than or equal to all valid values[^1] present in this file for this column. If all valid rows are null, this carries no information. [^1]: String columns are cut off at a fixed prefix length. Timestamp columns are truncated down to milliseconds. ## Partition Value Serialization Partition values are stored as strings, using the following formats. An empty string for any type translates to a `null` partition value. Type | Serialization Format -|- string | No translation required numeric types | The string representation of the number date | Encoded as `{year}-{month}-{day}`. For example, `1970-01-01` timestamp | Encoded as `{year}-{month}-{day} {hour}:{minute}:{second}` or `{year}-{month}-{day} {hour}:{minute}:{second}.{microsecond}`. For example: `1970-01-01 00:00:00`, or `1970-01-01 00:00:00.123456`. Timestamps may also be encoded as an ISO8601 formatted timestamp adjusted to UTC timestamp such as `1970-01-01T00:00:00.123456Z` timestamp without timezone | Encoded as `{year}-{month}-{day} {hour}:{minute}:{second}` or `{year}-{month}-{day} {hour}:{minute}:{second}.{microsecond}` For example: `1970-01-01 00:00:00`, or `1970-01-01 00:00:00.123456` To use this type, a table must support a feature `timestampNtz`. See section [Timestamp without timezone (TimestampNtz)](#timestamp-without-timezone-timestampNtz) for more information. boolean | Encoded as the string "true" or "false" binary | Encoded as a string of escaped binary values. For example, `"\u0001\u0002\u0003"` Note: A timestamp value in a partition value may be stored in one of the following ways: 1. Without a timezone, where the timestamp should be interpreted using the time zone of the system which wrote to the table. 2. Adjusted to UTC and stored in ISO8601 format. It is highly recommended that modern writers adjust the timestamp to UTC and store the timestamp in ISO8601 format as outlined in 2. ## Schema Serialization Format Delta uses a subset of Spark SQL's JSON Schema representation to record the schema of a table in the transaction log. All column names must be unique regardless of casing. A reference implementation can be found in [the catalyst package of the Apache Spark repository](https://github.com/apache/spark/tree/master/sql/catalyst/src/main/scala/org/apache/spark/sql/types). ### Primitive Types Type Name | Description -|- string| UTF-8 encoded string of characters long| 8-byte signed integer. Range: -9223372036854775808 to 9223372036854775807 integer|4-byte signed integer. Range: -2147483648 to 2147483647 short| 2-byte signed integer numbers. Range: -32768 to 32767 byte| 1-byte signed integer number. Range: -128 to 127 float| 4-byte single-precision floating-point numbers double| 8-byte double-precision floating-point numbers decimal| signed decimal number with fixed precision (maximum number of digits) and scale (number of digits on right side of dot). The precision and scale can be up to 38. boolean| `true` or `false` binary| A sequence of binary data. date| A calendar date, represented as a year-month-day triple without a timezone. timestamp| Microsecond precision timestamp elapsed since the Unix epoch, 1970-01-01 00:00:00 UTC. When this is stored in a parquet file, its `isAdjustedToUTC` must be set to `true`. timestamp without time zone | Microsecond precision timestamp in a local timezone elapsed since the Unix epoch, 1970-01-01 00:00:00. It doesn't have the timezone information, and a value of this type can map to multiple physical time instants. It should always be displayed in the same way, regardless of the local time zone in effect. When this is stored in a parquet file, its `isAdjustedToUTC` must be set to `false`. To use this type, a table must support a feature `timestampNtz`. See section [Timestamp without timezone (TimestampNtz)](#timestamp-without-timezone-timestampNtz) for more information. See Parquet [timestamp type](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#timestamp) for more details about timestamp and `isAdjustedToUTC`. Note: Existing tables may have `void` data type columns. Behavior is undefined for `void` data type columns but it is recommended to drop any `void` data type columns on reads (as is implemented by the Spark connector). ### Struct Type A struct is used to represent both the top-level schema of the table as well as struct columns that contain nested columns. A struct is encoded as a JSON object with the following fields: Field Name | Description -|- type | Always the string "struct" fields | An array of fields ### Struct Field A struct field represents a top-level or nested column. Field Name | Description -|- name| Name of this (possibly nested) column type| String containing the name of a primitive type, a struct definition, an array definition or a map definition nullable| Boolean denoting whether this field can be null metadata| A JSON map containing information about this column. Keys prefixed with `Delta` are reserved for the implementation. See [Column Metadata](#column-metadata) for more information on column level metadata that clients must handle when writing to a table. ### Array Type An array stores a variable length collection of items of some type. Field Name | Description -|- type| Always the string "array" elementType| The type of element stored in this array, represented as a string containing the name of a primitive type, a struct definition, an array definition or a map definition containsNull| Boolean denoting whether this array can contain one or more null values ### Map Type A map stores an arbitrary length collection of key-value pairs with a single `keyType` and a single `valueType`. Field Name | Description -|- type| Always the string "map". keyType| The type of element used for the key of this map, represented as a string containing the name of a primitive type, a struct definition, an array definition or a map definition valueType| The type of element used for the key of this map, represented as a string containing the name of a primitive type, a struct definition, an array definition or a map definition ### Variant Type Variant data uses the Delta type name `variant` for Delta schema serialization. Field Name | Description -|- type | Always the string "variant" ### Column Metadata A column metadata stores various information about the column. For example, this MAY contain some keys like [`delta.columnMapping`](#column-mapping) or [`delta.generationExpression`](#generated-columns) or [`CURRENT_DEFAULT`](#default-columns). Field Name | Description -|- delta.columnMapping.*| These keys are used to store information about the mapping between the logical column name to the physical name. See [Column Mapping](#column-mapping) for details. delta.identity.*| These keys are for defining identity columns. See [Identity Columns](#identity-columns) for details. delta.invariants| JSON string contains SQL expression information. See [Column Invariants](#column-invariants) for details. delta.generationExpression| SQL expression string. See [Generated Columns](#generated-columns) for details. delta.typeChanges| JSON string containing information about previous type changes applied to this column. See [Type Change Metadata](#type-change-metadata) for details. ### Example Example Table Schema: ``` |-- a: integer (nullable = false) |-- b: struct (nullable = true) | |-- d: integer (nullable = false) |-- c: array (nullable = true) | |-- element: integer (containsNull = false) |-- e: array (nullable = true) | |-- element: struct (containsNull = true) | | |-- d: integer (nullable = false) |-- f: map (nullable = true) | |-- key: string | |-- value: string (valueContainsNull = true) ``` JSON Encoded Table Schema: ``` { "type" : "struct", "fields" : [ { "name" : "a", "type" : "integer", "nullable" : false, "metadata" : { } }, { "name" : "b", "type" : { "type" : "struct", "fields" : [ { "name" : "d", "type" : "integer", "nullable" : false, "metadata" : { } } ] }, "nullable" : true, "metadata" : { } }, { "name" : "c", "type" : { "type" : "array", "elementType" : "integer", "containsNull" : false }, "nullable" : true, "metadata" : { } }, { "name" : "e", "type" : { "type" : "array", "elementType" : { "type" : "struct", "fields" : [ { "name" : "d", "type" : "integer", "nullable" : false, "metadata" : { } } ] }, "containsNull" : true }, "nullable" : true, "metadata" : { } }, { "name" : "f", "type" : { "type" : "map", "keyType" : "string", "valueType" : "string", "valueContainsNull" : true }, "nullable" : true, "metadata" : { } } ] } ``` ## Checkpoint Schema The following examples uses a table with two partition columns: "date" and "region" of types date and string, respectively, and three data columns: "asset", "quantity", and "is_available" with data types string, double, and boolean. The checkpoint schema will look as follows: ``` |-- metaData: struct | |-- id: string | |-- name: string | |-- description: string | |-- format: struct | | |-- provider: string | | |-- options: map | |-- schemaString: string | |-- partitionColumns: array | |-- createdTime: long | |-- configuration: map |-- protocol: struct | |-- minReaderVersion: int | |-- minWriterVersion: int | |-- readerFeatures: array[string] | |-- writerFeatures: array[string] |-- txn: struct | |-- appId: string | |-- version: long |-- add: struct | |-- path: string | |-- partitionValues: map | |-- size: long | |-- modificationTime: long | |-- dataChange: boolean | |-- stats: string | |-- tags: map | |-- baseRowId: long | |-- defaultRowCommitVersion: long | |-- partitionValues_parsed: struct | | |-- date: date | | |-- region: string | |-- stats_parsed: struct | | |-- numRecords: long | | |-- minValues: struct | | | |-- asset: string | | | |-- quantity: double | | |-- maxValues: struct | | | |-- asset: string | | | |-- quantity: double | | |-- nullCount: struct | | | |-- asset: long | | | |-- quantity: long |-- remove: struct | |-- path: string | |-- deletionTimestamp: long | |-- dataChange: boolean |-- checkpointMetadata: struct | |-- version: long | |-- tags: map |-- sidecar: struct | |-- path: string | |-- sizeInBytes: long | |-- modificationTime: long | |-- tags: map ``` Observe that `readerFeatures` and `writerFeatures` fields should comply with: - If a table has Reader Version 3, then a writer must write checkpoints with a not-null `readerFeatures` in the schema. - If a table has Writer Version 7, then a writer must write checkpoints with a not-null `writerFeatures` in the schema. - If a table has neither of the above, then a writer chooses whether to write `readerFeatures` and/or `writerFeatures` into the checkpoint schema. But if it does, their values must be null. Note that `remove` actions in the checkpoint are tombstones used only by VACUUM, and do not contain the `stats` and `tags` fields. For a table that uses column mapping, whether in `id` or `name` mode, the schema of the `add` column will look as follows. Schema definition: ``` { "type" : "struct", "fields" : [ { "name" : "asset", "type" : "string", "nullable" : true, "metadata" : { "delta.columnMapping.id": 1, "delta.columnMapping.physicalName": "col-b96921f0-2329-4cb3-8d79-184b2bdab23b" } }, { "name" : "quantity", "type" : "double", "nullable" : true, "metadata" : { "delta.columnMapping.id": 2, "delta.columnMapping.physicalName": "col-04ee4877-ee53-4cb9-b1fb-1a4eb74b508c" } }, { "name" : "date", "type" : "date", "nullable" : true, "metadata" : { "delta.columnMapping.id": 3, "delta.columnMapping.physicalName": "col-798f4abc-c63f-444c-9a04-e2cf1ecba115" } }, { "name" : "region", "type" : "string", "nullable" : true, "metadata" : { "delta.columnMapping.id": 4, "delta.columnMapping.physicalName": "col-19034dc3-8e3d-4156-82fc-8e05533c088e" } } ] } ``` Checkpoint schema (just the `add` column): ``` |-- add: struct | |-- path: string | |-- partitionValues: map | |-- size: long | |-- modificationTime: long | |-- dataChange: boolean | |-- stats: string | |-- tags: map | |-- baseRowId: long | |-- defaultRowCommitVersion: long | |-- partitionValues_parsed: struct | | |-- col-798f4abc-c63f-444c-9a04-e2cf1ecba115: date | | |-- col-19034dc3-8e3d-4156-82fc-8e05533c088e: string | |-- stats_parsed: struct | | |-- numRecords: long | | |-- minValues: struct | | | |-- col-b96921f0-2329-4cb3-8d79-184b2bdab23b: string | | | |-- col-04ee4877-ee53-4cb9-b1fb-1a4eb74b508c: double | | |-- maxValues: struct | | | |-- col-b96921f0-2329-4cb3-8d79-184b2bdab23b: string | | | |-- col-04ee4877-ee53-4cb9-b1fb-1a4eb74b508c: double | | |-- nullCount: struct | | | |-- col-b96921f0-2329-4cb3-8d79-184b2bdab23b: long | | | |-- col-04ee4877-ee53-4cb9-b1fb-1a4eb74b508c: long ``` ## Last Checkpoint File Schema This last checkpoint file is encoded as JSON and contains the following information: Field | Description -|- version | The version of the table when the last checkpoint was made. size | The number of actions that are stored in the checkpoint. parts | The number of fragments if the last checkpoint was written in multiple parts. This field is optional. sizeInBytes | The number of bytes of the checkpoint. This field is optional. numOfAddFiles | The number of AddFile actions in the checkpoint. This field is optional. checkpointSchema | The schema of the checkpoint file. This field is optional. tags | String-string map containing any additional metadata about the last checkpoint. This field is optional. checksum | The checksum of the last checkpoint JSON. This field is optional. The checksum field is an optional field which contains the MD5 checksum for fields of the last checkpoint json file. Last checkpoint file readers are encouraged to validate the checksum, if present, and writers are encouraged to write the checksum while overwriting the file. Refer to [this section](#json-checksum) for rules around calculating the checksum field for the last checkpoint JSON. ### JSON checksum To generate the checksum for the last checkpoint JSON, firstly, the checksum JSON is canonicalized and converted to a string. Then the 32 character MD5 digest is calculated on the resultant string to get the checksum. Rules for [JSON](https://datatracker.ietf.org/doc/html/rfc8259) canonicalization are: 1. Literal values (`true`, `false`, and `null`) are their own canonical form 2. Numeric values (e.g. `42` or `3.14`) are their own canonical form 3. String values (e.g. `"hello world"`) are canonicalized by preserving the surrounding quotes and [URL-encoding](#how-to-url-encode-keys-and-string-values) their content, e.g. `"hello%20world"` 4. Object values (e.g. `{"a": 10, "b": {"y": null, "x": "https://delta.io"} }` are canonicalized by: * Canonicalize each scalar (leaf) value following the rule for its type (literal, numeric, string) * Canonicalize each (string) name along the path to that value * Connect path segments by `+`, e.g. `"b"+"y"` * Connect path and value pairs by `=`, e.g. `"b"+"y"=null` * Sort canonicalized path/value pairs using a byte-order sort on paths. The byte-order sort can be done by converting paths to byte array using UTF-8 charset\ and then comparing them, e.g. `"a" < "b"+"x" < "b"+"y"` * Separate ordered pairs by `,`, e.g. `"a"=10,"b"+"x"="https%3A%2F%2Fdelta.io","b"+"y"=null` 5. Array values (e.g. `[null, "hi ho", 2.71]`) are canonicalized as if they were objects, except the "name" has numeric type instead of string type, and gives the (0-based) position of the corresponding array element, e.g. `0=null,1="hi%20ho",2=2.71` 6. Top level `checksum` key is ignored in the canonicalization process. e.g. `{"k1": "v1", "checksum": "", "k3": 23}` is canonicalized to `"k1"="v1","k3"=23` 7. Duplicate keys are not allowed in the last checkpoint JSON and such JSON is considered invalid. Given the following test sample JSON, a correct implementation of JSON canonicalization should produce the corresponding canonicalized form and checksum value: e.g. Json: `{"k0":"'v 0'", "checksum": "adsaskfljadfkjadfkj", "k1":{"k2": 2, "k3": ["v3", [1, 2], {"k4": "v4", "k5": ["v5", "v6", "v7"]}]}}`\ Canonicalized form: `"k0"="%27v%200%27","k1"+"k2"=2,"k1"+"k3"+0="v3","k1"+"k3"+1+0=1,"k1"+"k3"+1+1=2,"k1"+"k3"+2+"k4"="v4","k1"+"k3"+2+"k5"+0="v5","k1"+"k3"+2+"k5"+1="v6","k1"+"k3"+2+"k5"+2="v7"`\ Checksum is `6a92d155a59bf2eecbd4b4ec7fd1f875` #### How to URL encode keys and string values The [URL Encoding](https://datatracker.ietf.org/doc/html/rfc3986) spec is a bit flexible to give a reliable encoding. e.g. the spec allows both uppercase and lowercase as part of percent-encoding. Thus, we require a stricter set of rules for encoding: 1. The string to be encoded must be represented as octets according to the UTF-8 character encoding 2. All octets except a-z / A-Z / 0-9 / "-" / "." / "_" / "~" are reserved 3. Always [percent-encode](https://datatracker.ietf.org/doc/html/rfc3986#section-2) reserved octets 4. Never percent-encode non-reserved octets 5. A percent-encoded octet consists of three characters: `%` followed by its 2-digit hexadecimal value in uppercase letters, e.g. `>` encodes to `%3E` ## Delta Data Type to Parquet Type Mappings Below table captures how each Delta data type is stored physically in Parquet files. Parquet files are used for storing the table data or metadata ([checkpoints](#checkpoints)). Parquet has a limited number of [physical types](https://parquet.apache.org/docs/file-format/types/). Parquet [logical types](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md) are used to extend the types by specifying how the physical types should be interpreted. For some of the Delta data types, there are multiple ways store the values physically in Parquet file. For example, `timestamp` can be stored either as `int96` or `int64`. The exact physical type depends on the engine that is writing the Parquet file and/or engine specific configuration options. For a Delta lake table reader, it is recommended that the Parquet file reader support at least the Parquet physical and logical types mentioned in the below table. Delta Type Name | Parquet Physical Type | Parquet Logical Type -|-|- boolean| `boolean` | byte| `int32` | `INT(bitwidth = 8, signed = true)` short| `int32` | `INT(bitwidth = 16, signed = true)` int| `int32` | `INT(bitwidth = 32, signed = true)` long| `int64` | `INT(bitwidth = 64, signed = true)` date| `int32` | `DATE` timestamp| `int96` or `int64` | `TIMESTAMP(isAdjustedToUTC = true, units = microseconds)` timestamp without time zone| `int96` or `int64` | `TIMESTAMP(isAdjustedToUTC = false, units = microseconds)` float| `float` | double| `double` | decimal| `int32`, `int64` or `fixed_length_binary` | `DECIMAL(scale, precision)` string| `binary` | `string (UTF-8)` binary| `binary` | array| either as `2-level` or `3-level` representation. Refer to [Parquet documentation](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists) for further details | `LIST` map| either as `2-level` or `3-level` representation. Refer to [Parquet documentation](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#maps) for further details | `MAP` struct| `group` | ================================================ FILE: README.md ================================================ Delta Lake Logo [![Test](https://github.com/delta-io/delta/actions/workflows/test.yaml/badge.svg)](https://github.com/delta-io/delta/actions/workflows/test.yaml) [![License](https://img.shields.io/badge/license-Apache%202-brightgreen.svg)](https://github.com/delta-io/delta/blob/master/LICENSE.txt) [![PyPI](https://img.shields.io/pypi/v/delta-spark.svg)](https://pypi.org/project/delta-spark/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/delta-spark)](https://pypistats.org/packages/delta-spark) Delta Lake is an open-source storage framework that enables building a [Lakehouse architecture](http://cidrdb.org/cidr2021/papers/cidr2021_paper17.pdf) with compute engines including Spark, PrestoDB, Flink, Trino, and Hive and APIs for Scala, Java, Rust, Ruby, and Python. * See the [Delta Lake Documentation](https://docs.delta.io) for details. * See the [Quick Start Guide](https://docs.delta.io/latest/quick-start.html) to get started with Scala, Java and Python. * Note, this repo is one of many Delta Lake repositories in the [delta.io](https://github.com/delta-io) organizations including [delta](https://github.com/delta-io/delta), [delta-rs](https://github.com/delta-io/delta-rs), [delta-sharing](https://github.com/delta-io/delta-sharing), [kafka-delta-ingest](https://github.com/delta-io/kafka-delta-ingest), and [website](https://github.com/delta-io/website). The following are some of the more popular Delta Lake integrations, refer to [delta.io/integrations](https://delta.io/integrations/) for the complete list: * [Apache Spark™](https://docs.delta.io/): This connector allows Apache Spark™ to read from and write to Delta Lake. * [Apache Flink (Preview)](https://github.com/delta-io/delta/tree/master/connectors/flink): This connector allows Apache Flink to write to Delta Lake. * [PrestoDB](https://prestodb.io/docs/current/connector/deltalake.html): This connector allows PrestoDB to read from Delta Lake. * [Trino](https://trino.io/docs/current/connector/delta-lake.html): This connector allows Trino to read from and write to Delta Lake. * [Delta Standalone](https://docs.delta.io/latest/delta-standalone.html): This library allows Scala and Java-based projects (including Apache Flink, Apache Hive, Apache Beam, and PrestoDB) to read from and write to Delta Lake. * [Apache Hive](https://docs.delta.io/latest/hive-integration.html): This connector allows Apache Hive to read from Delta Lake. * [Delta Rust API](https://docs.rs/deltalake/latest/deltalake/): This library allows Rust (with Python and Ruby bindings) low level access to Delta tables and is intended to be used with data processing frameworks like datafusion, ballista, rust-dataframe, vega, etc.
Table of Contents * [Latest binaries](#latest-binaries) * [API Documentation](#api-documentation) * [Compatibility](#compatibility) * [API Compatibility](#api-compatibility) * [Data Storage Compatibility](#data-storage-compatibility) * [Roadmap](#roadmap) * [Building](#building) * [Transaction Protocol](#transaction-protocol) * [Requirements for Underlying Storage Systems](#requirements-for-underlying-storage-systems) * [Concurrency Control](#concurrency-control) * [Reporting issues](#reporting-issues) * [Contributing](#contributing) * [License](#license) * [Community](#community)
## Latest Binaries See the [online documentation](https://docs.delta.io/latest/) for the latest release. ## API Documentation * [Scala API docs](https://docs.delta.io/latest/delta-apidoc.html) * [Java API docs](https://docs.delta.io/latest/api/java/index.html) * [Python API docs](https://docs.delta.io/latest/api/python/index.html) ## Compatibility [Delta Standalone](https://docs.delta.io/latest/delta-standalone.html) library is a single-node Java library that can be used to read from and write to Delta tables. Specifically, this library provides APIs to interact with a table’s metadata in the transaction log, implementing the Delta Transaction Log Protocol to achieve the transactional guarantees of the Delta Lake format. ### API Compatibility There are two types of APIs provided by the Delta Lake project. - Direct Java/Scala/Python APIs - The classes and methods documented in the [API docs](https://docs.delta.io/latest/delta-apidoc.html) are considered as stable public APIs. All other classes, interfaces, methods that may be directly accessible in code are considered internal, and they are subject to change across releases. - Spark-based APIs - You can read Delta tables through the `DataFrameReader`/`Writer` (i.e. `spark.read`, `df.write`, `spark.readStream` and `df.writeStream`). Options to these APIs will remain stable within a major release of Delta Lake (e.g., 1.x.x). - See the [online documentation](https://docs.delta.io/latest/releases.html) for the releases and their compatibility with Apache Spark versions. ### Data Storage Compatibility Delta Lake guarantees backward compatibility for all Delta Lake tables (i.e., newer versions of Delta Lake will always be able to read tables written by older versions of Delta Lake). However, we reserve the right to break forward compatibility as new features are introduced to the transaction protocol (i.e., an older version of Delta Lake may not be able to read a table produced by a newer version). Breaking changes in the protocol are indicated by incrementing the minimum reader/writer version in the `Protocol` [action](https://github.com/delta-io/delta/blob/master/spark/src/test/scala/org/apache/spark/sql/delta/ActionSerializerSuite.scala). ## Roadmap * For the high-level Delta Lake roadmap, see [Delta Lake 2022H1 roadmap](http://delta.io/roadmap). * For the detailed timeline, see the [project roadmap](https://github.com/delta-io/delta/milestones). ## Transaction Protocol [Delta Transaction Log Protocol](PROTOCOL.md) document provides a specification of the transaction protocol. ## Requirements for Underlying Storage Systems Delta Lake ACID guarantees are predicated on the atomicity and durability guarantees of the storage system. Specifically, we require the storage system to provide the following. 1. **Atomic visibility**: There must be a way for a file to be visible in its entirety or not visible at all. 2. **Mutual exclusion**: Only one writer must be able to create (or rename) a file at the final destination. 3. **Consistent listing**: Once a file has been written in a directory, all future listings for that directory must return that file. See the [online documentation on Storage Configuration](https://docs.delta.io/latest/delta-storage.html) for details. ## Concurrency Control Delta Lake ensures _serializability_ for concurrent reads and writes. Please see [Delta Lake Concurrency Control](https://docs.delta.io/concurrency-control/) for more details. ## Reporting issues We use [GitHub Issues](https://github.com/delta-io/delta/issues) to track community reported issues. You can also [contact](#community) the community for getting answers. ## Contributing We welcome contributions to Delta Lake. See our [CONTRIBUTING.md](https://github.com/delta-io/delta/blob/master/CONTRIBUTING.md) for more details. We also adhere to the [Delta Lake Code of Conduct](https://github.com/delta-io/delta/blob/master/CODE_OF_CONDUCT.md). ## Building Delta Lake is compiled using [SBT](https://www.scala-sbt.org/1.x/docs/Command-Line-Reference.html). Ensure that your Java version is at least 17 (you can verify with `java -version`). To compile, run build/sbt compile To generate artifacts, run build/sbt package To execute tests, run build/sbt test To execute a single test suite, run build/sbt spark/'testOnly org.apache.spark.sql.delta.optimize.OptimizeCompactionSQLSuite' To execute a single test within and a single test suite, run build/sbt spark/'testOnly *.OptimizeCompactionSQLSuite -- -z "optimize command: on partitioned table - all partitions"' Refer to [SBT docs](https://www.scala-sbt.org/1.x/docs/Command-Line-Reference.html) for more commands. ## Running python tests locally ### Setup Environment #### Install Conda (Skip if you already installed it) Follow [Conda Download](https://www.anaconda.com/download/) to install Anaconda. #### Create an environment from environment file Follow [Create Environment From Environment file](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#create-env-from-file) to create a Conda environment from `/python/environment.yml` and activate the newly created `delta_python_tests` environment. ``` # Note the `--file` argument should be a fully qualified path. Using `~` in file # path doesn't work. Example valid path: `/Users/macuser/delta/python/environment.yml` conda env create --name delta_python_tests --file=/python/environment.yml` ``` #### JDK Setup Build needs JDK 11. Make sure to setup `JAVA_HOME` that points to JDK 11. #### Running tests ``` conda activate delta_python_tests python3 /python/run-tests.py ``` ## IntelliJ Setup IntelliJ is the recommended IDE to use when developing Delta Lake. To import Delta Lake as a new project: 1. Clone Delta Lake into, for example, `~/delta`. 2. In IntelliJ, select `File` > `New Project` > `Project from Existing Sources...` and select `~/delta`. 3. Under `Import project from external model` select `sbt`. Click `Next`. 4. Under `Project JDK` specify a valid Java `11` JDK and opt to use SBT shell for `project reload` and `builds`. 5. Click `Finish`. 6. In your terminal, run `build/sbt clean package`. Make sure you use Java `11`. The build will generate files that are necessary for Intellij to index the repository. ### Setup Verification After waiting for IntelliJ to index, verify your setup by running a test suite in IntelliJ. 1. Search for and open `DeltaLogSuite` 2. Next to the class declaration, right click on the two green arrows and select `Run 'DeltaLogSuite'` ### Troubleshooting If you see errors of the form ``` Error:(46, 28) object DeltaSqlBaseParser is not a member of package io.delta.sql.parser import io.delta.sql.parser.DeltaSqlBaseParser._ ... Error:(91, 22) not found: type DeltaSqlBaseParser val parser = new DeltaSqlBaseParser(tokenStream) ``` then follow these steps: 1. Ensure you are using Java `11`. You can set this using ``` export JAVA_HOME=`/usr/libexec/java_home -v 11` ``` 2. Compile using the SBT CLI: `build/sbt clean compile`. 2. Go to `File` > `Project Structure...` > `Modules` > `delta-spark`. 3. In the right panel under `Source Folders` remove any `target` folders, e.g. `target/scala-2.12/src_managed/main [generated]` 4. Click `Apply` and then re-run your test. ## License Apache License 2.0, see [LICENSE](https://github.com/delta-io/delta/blob/master/LICENSE.txt). ## Community There are two mediums of communication within the Delta Lake community. * Public Slack Channel - [Register here](https://go.delta.io/slack) - [Login here](https://delta-users.slack.com/) * [Linkedin page](https://www.linkedin.com/company/deltalake) * [Youtube channel](https://www.youtube.com/c/deltalake) * Public [Mailing list](https://groups.google.com/forum/#!forum/delta-users) ================================================ FILE: benchmarks/README.md ================================================ # Benchmarks ## Overview This is a basic framework for writing benchmarks to measure Delta's performance. It is currently designed to run benchmark on Spark running in an EMR or a Dataproc cluster. However, it can be easily extended for other Spark-based benchmarks. To get started, first download/clone this repository in your local machine. Then you have to set up a cluster and run the benchmark scripts in this directory. See the next section for more details. ## Running TPC-DS benchmark This TPC-DS benchmark is constructed such that you have to run the following two steps. 1. *Load data*: You have to create the TPC-DS database with all the Delta tables. To do that, the raw TPC-DS data has been provided as Apache Parquet files. In this step you will have to use your EMR or a Dataproc cluster to read the parquet files and rewrite them as Delta tables. 2. *Query data*: Then, using the tables definitions in the Hive Metatore, you can run the 99 benchmark queries. The next section will provide the detailed steps of how to setup the necessary Hive Metastore and a cluster, how to test the setup with small-scale data, and then finally run the full scale benchmark. ### Configure cluster with Amazon Web Services #### Prerequisites - An AWS account with necessary permissions to do the following: - Manage RDS instances for creating an external Hive Metastore - Manage EMR clusters for running the benchmark - Read and write to an S3 bucket from the EMR cluster - A S3 bucket which will be used to generate the TPC-DS data. - A machine which has access to the AWS setup and where this repository has been downloaded or cloned. There are two ways to create infrastructure required for benchmarks - using provided [Terraform template](infrastructure/aws/terraform/README.md) or manually (described below). #### Create external Hive Metastore using Amazon RDS Create an external Hive Metastore in a MySQL database using Amazon RDS with the following specifications: - MySQL 8.x on a `db.m5.large`. - General purpose SSDs, and no Autoscaling storage. - Non-empty password for admin - Same region, VPC, subnet as those you will run the EMR cluster. See AWS docs for more guidance. - *Note:* Region us-west-2 since that is what this benchmark has been most tested with. After the database is ready, note the JDBC connection details, the username and password. We will need them for the next step. Note that this step needs to be done just once. All EMR clusters can connect and reused this Hive Metastsore. #### Create EMR cluster Create an EMR cluster that connects to the external Hive Metastore. Here are the specifications of the EMR cluster required for running benchmarks. - EMR with Spark and Hive (needed for writing to Hive Metastore). Choose the EMR version based on the Spark version compatible with the format. For example: - For Delta 2.0 on Spark 3.2 - EMR 6.6.0 - For Delta 1.0 on Spark 3.1 - EMR 6.5.0 - Master - i3.2xlarge - Workers - 16 x i3.2xlarge (or just 1 worker if you are just testing by running the 1GB benchmark). - Hive-site configuration to connect to the Hive Metastore. See [Using an external MySQL database or Amazon Aurora](https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-hive-metastore-external.html) for more details. - Same region, VPC, subnet as those of the Hive Metastore. - *Note:* Region us-west-2 since that is what this benchmark has been most tested with. - No autoscaling, and default EBS storage. Once the EMR cluster is ready, note the following: - Hostname of the EMR cluster master node. - PEM file for SSH into the master node. These will be needed to run the workloads in this framework. #### Prepare S3 bucket Create a new S3 bucket (or use an existing one) which is in the same region as your EMR cluster. _________________ ### Configure cluster with Google Cloud Platform #### Prerequisites - A GCP account with necessary permissions to do the following: - Manage Dataproc clusters for running the benchmark - Manage Dataproc Metastore instances - Read and write to a GCS bucket from the Dataproc cluster - A GCS bucket which will be used to generate the TPC-DS data. - A machine which has access to the GCP setup and where this repository has been downloaded or cloned. - SSH keys for a user which will be used to access the master node. The user's SSH key can be either [a project-wide key](https://cloud.google.com/compute/docs/connect/add-ssh-keys#add_ssh_keys_to_project_metadata) or assigned to the [master node](https://cloud.google.com/compute/docs/connect/add-ssh-keys#after-vm-creation) only. - Ideally, all GCP components used in benchmark should be in the same location (Storage bucket, Dataproc Metastore service and Dataproc cluster). There are two ways to create infrastructure required for benchmarks - using provided [Terraform template](infrastructure/gcp/terraform/README.md) or manually (described below). #### Prepare GCS bucket Create a new GCS bucket (or use an existing one) which is in the same region as your Dataproc cluster. #### Create Dataproc Metastore You can create [Dataproc metastore](https://cloud.google.com/dataproc-metastore/docs/create-service) either via Web Console or gcloud command. Sample create command: ```bash gcloud metastore services create dataproc-metastore-for-benchmarks \ --location= \ --tier=enterprise ``` #### Create Dataproc cluster Here are the specifications of the Dataproc cluster required for running benchmarks. - Image version >= 2.0 having Apache Spark 3.1 - Master - n2-highmem-8 (8 vCPU, 64 GB memory) - Workers - 16 x n2-highmem-8 (or just 2 workers if you are just testing by running the 1GB benchmark). - The cluster connects to the Dataproc Metastore. - Same region and subnet as those of the Dataproc Metastore and GCS bucket. - No autoscaling. Sample create command: ```bash gcloud dataproc clusters create delta-performance-benchmarks-cluster \ --project \ --enable-component-gateway \ --region \ --zone \ --subnet default \ --master-machine-type n2-highmem-8 \ --master-boot-disk-type pd-ssd \ --master-boot-disk-size 100 \ --num-master-local-ssds 4 \ --master-local-ssd-interface NVME \ --num-workers 16 \ --worker-machine-type n2-highmem-8 \ --worker-boot-disk-type pd-ssd \ --worker-boot-disk-size 100 \ --num-worker-local-ssds 4 \ --worker-local-ssd-interface NVME \ --dataproc-metastore projects//locations//services/dataproc-metastore-for-benchmarks \ --enable-component-gateway \ --image-version 2.0-debian10 ``` #### Input data The benchmark is run using the raw TPC-DS data which has been provided as Apache Parquet files. There are two predefined datasets of different size, 1GB and 3TB, located in `s3://devrel-delta-datasets/tpcds-2.13/tpcds_sf1_parquet/` and `s3://devrel-delta-datasets/tpcds-2.13/tpcds_sf3000_parquet/`, respectively. Please keep in mind that `devrel-delta-datasets` bucket is configured as [Requester Pays](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ObjectsinRequesterPaysBuckets.html) bucket, so [access requests have to be configured properly](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ObjectsinRequesterPaysBuckets.html). Unfortunately, Hadoop in versions available in Dataproc does not support *Requester Pays* feature. It will be available as of Hadoop 3.3.4 ([HADOOP-14661](https://issues.apache.org/jira/browse/HADOOP-14661)). In consequence, one need to copy the datasets to Google Storage manually before running benchmarks. The simplest solution is to copy the data in two steps: first to a S3 bucket with *Requester Pays* disabled, then copy the data using [Cloud Storage Transfer Service](https://cloud.google.com/storage-transfer/docs/how-to). _________________ ### Test the cluster setup Navigate to your local copy of this repository and this benchmark directory. Then run the following steps. #### Run simple test workload Verify that you have the following information - : Cluster master node host name - : Local path to your PEM file for SSH into the master node. - : The username that will be used to SSH into the master node. The username is tied to the SSH key you have imported into the cloud. It defaults to `hadoop`. - : Path where tables will be created. Make sure your credentials have read/write permission to that path. - : Currently either `gcp` or `aws`. For each storage type, different Delta properties might be added. Then run a simple table write-read test: Run the following in your shell. ```sh ./run-benchmark.py \ --cluster-hostname \ -i \ --ssh-user \ --benchmark-path \ --cloud-provider \ --benchmark test ``` If this works correctly, then you should see an output that look like this. ```text >>> Benchmark script generated and uploaded ... There is a screen on: 12001..ip-172-31-21-247 (Detached) Files for this benchmark: 20220126-191336-test-benchmarks.jar 20220126-191336-test-cmd.sh 20220126-191336-test-out.txt >>> Benchmark script started in a screen. Stdout piped into 20220126-191336-test-out.txt.Final report will be generated on completion in 20220126-191336-test-report.json. ``` The test workload launched in a `screen` is going to run the following: - Spark jobs to run a simple SQL query - Create a Delta table in the given location - Read it back To see whether they worked correctly, SSH into the node and check the output of 20220126-191336-test-out.txt. Once the workload terminates, the last few lines should be something like the following: ```text RESULT: { "benchmarkSpecs" : { "benchmarkPath" : ..., "benchmarkId" : "20220126-191336-test" }, "queryResults" : [ { "name" : "sql-test", "durationMs" : 11075 }, { "name" : "db-list-test", "durationMs" : 208 }, { "name" : "db-create-test", "durationMs" : 4070 }, { "name" : "db-use-test", "durationMs" : 41 }, { "name" : "table-drop-test", "durationMs" : 74 }, { "name" : "table-create-test", "durationMs" : 33812 }, { "name" : "table-query-test", "durationMs" : 4795 } ] } FILE UPLOAD: Uploaded /home/hadoop/20220126-191336-test-report.json to s3:// ... SUCCESS ``` The above metrics are also written to a json file and uploaded to the given path. Please verify that both the table and report are generated in that path. #### Run 1GB TPC-DS Now that you are familiar with how the framework runs the workload, you can try running the small scale TPC-DS benchmark. 1. Load data as Delta tables: ```bash ./run-benchmark.py \ --cluster-hostname \ -i \ --ssh-user \ --benchmark-path \ --cloud-provider \ --benchmark tpcds-1gb-delta-load ``` If you run the benchmark in GCP you should provide `--source-path ` parameter, where `` is the location of the raw parquet input data files (see *Input data* section). ```bash ./run-benchmark.py \ --cluster-hostname \ -i \ --ssh-user \ --benchmark-path \ --source-path \ --cloud-provider gcp \ --benchmark tpcds-1gb-delta-load ``` 3. Run queries on Delta tables: ```bash ./run-benchmark.py \ --cluster-hostname \ -i \ --ssh-user \ --benchmark-path \ --cloud-provider \ --benchmark tpcds-1gb-delta ``` ### Run 3TB TPC-DS Finally, you are all set up to run the full scale benchmark. Similar to the 1GB benchmark, run the following 1. Load data as Delta tables: ```bash ./run-benchmark.py \ --cluster-hostname \ -i \ --ssh-user \ --benchmark-path \ --cloud-provider \ --benchmark tpcds-3tb-delta-load ``` If you run the benchmark in GCP you should provide `--source-path ` parameter, where `` is the location of the raw parquet input data files (see *Input data* section). ```bash ./run-benchmark.py \ --cluster-hostname \ -i \ --ssh-user \ --benchmark-path \ --source-path \ --cloud-provider gcp \ --benchmark tpcds-3tb-delta-load ``` 2. Run queries on Delta tables: ```bash ./run-benchmark.py \ --cluster-hostname \ -i \ --ssh-user \ --benchmark-path \ --cloud-provider \ --benchmark tpcds-3tb-delta ``` Compare the results using the generated JSON files. _________________ ## Internals of the framework Structure of this framework's code - `build.sbt`, `project/`, `src/` form the SBT project which contains the Scala code that define the benchmark workload. - `Benchmark.scala` is the basic interface, and `TestBenchmark.scala` is a sample implementation. - `run-benchmark.py` contains the specification of the benchmarks defined by name (e.g. `tpcds-3tb-delta`). Each benchmark specification is defined by the following: - Fully qualified name of the main Scala class to be started. - Command line argument for the main function. - Additional Maven artifact to load (example `io.delta:delta-core_2.12:1.0.0`). - Spark configurations to use. - `scripts` has the core python scripts that are called by `run-benchmark.py` The script `run-benchmark.py` does the following: - Compile the Scala code into a uber jar. - Upload it to the given hostname. - Using ssh to the hostname, it will launch a screen and start the main class with spark-submit. ================================================ FILE: benchmarks/build/sbt ================================================ #!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # # This file contains code from the Apache Spark project (original license above). # It contains modifications, which are licensed as follows: # # # Copyright (2021) The Delta Lake Project Authors. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # When creating new tests for Spark SQL Hive, the HADOOP_CLASSPATH must contain the hive jars so # that we can run Hive to generate the golden answer. This is not required for normal development # or testing. if [ -n "$HIVE_HOME" ]; then for i in "$HIVE_HOME"/lib/* do HADOOP_CLASSPATH="$HADOOP_CLASSPATH:$i" done export HADOOP_CLASSPATH fi realpath () { ( TARGET_FILE="$1" cd "$(dirname "$TARGET_FILE")" TARGET_FILE="$(basename "$TARGET_FILE")" COUNT=0 while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] do TARGET_FILE="$(readlink "$TARGET_FILE")" cd $(dirname "$TARGET_FILE") TARGET_FILE="$(basename $TARGET_FILE)" COUNT=$(($COUNT + 1)) done echo "$(pwd -P)/"$TARGET_FILE"" ) } if [[ "$JENKINS_URL" != "" ]]; then # Make Jenkins use Google Mirror first as Maven Central may ban us SBT_REPOSITORIES_CONFIG="$(dirname "$(realpath "$0")")/sbt-config/repositories" export SBT_OPTS="-Dsbt.override.build.repos=true -Dsbt.repository.config=$SBT_REPOSITORIES_CONFIG" fi . "$(dirname "$(realpath "$0")")"/sbt-launch-lib.bash declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy" declare -r sbt_opts_file=".sbtopts" declare -r etc_sbt_opts_file="/etc/sbt/sbtopts" usage() { cat < path to global settings/plugins directory (default: ~/.sbt) -sbt-boot path to shared boot directory (default: ~/.sbt/boot in 0.11 series) -ivy path to local Ivy repository (default: ~/.ivy2) -mem set memory options (default: $sbt_mem, which is $(get_mem_opts $sbt_mem)) -no-share use all local caches; no sharing -no-global uses global caches, but does not use global ~/.sbt directory. -jvm-debug Turn on JVM debugging, open at the given port. -batch Disable interactive mode # sbt version (default: from project/build.properties if present, else latest release) -sbt-version use the specified version of sbt -sbt-jar use the specified jar as the sbt launcher -sbt-rc use an RC version of sbt -sbt-snapshot use a snapshot version of sbt # java version (default: java from PATH, currently $(java -version 2>&1 | grep version)) -java-home alternate JAVA_HOME # jvm options and output control JAVA_OPTS environment variable, if unset uses "$java_opts" SBT_OPTS environment variable, if unset uses "$default_sbt_opts" .sbtopts if this file exists in the current directory, it is prepended to the runner args /etc/sbt/sbtopts if this file exists, it is prepended to the runner args -Dkey=val pass -Dkey=val directly to the java runtime -J-X pass option -X directly to the java runtime (-J is stripped) -S-X add -X to sbt's scalacOptions (-S is stripped) -PmavenProfiles Enable a maven profile for the build. In the case of duplicated or conflicting options, the order above shows precedence: JAVA_OPTS lowest, command line options highest. EOM } process_my_args () { while [[ $# -gt 0 ]]; do case "$1" in -no-colors) addJava "-Dsbt.log.noformat=true" && shift ;; -no-share) addJava "$noshare_opts" && shift ;; -no-global) addJava "-Dsbt.global.base=$(pwd)/project/.sbtboot" && shift ;; -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;; -sbt-dir) require_arg path "$1" "$2" && addJava "-Dsbt.global.base=$2" && shift 2 ;; -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;; -batch) exec /dev/null) if [[ ! $? ]]; then saved_stty="" fi } saveSttySettings trap onExit INT run "$@" exit_status=$? onExit ================================================ FILE: benchmarks/build/sbt-launch-lib.bash ================================================ #!/usr/bin/env bash # # A library to simplify using the SBT launcher from other packages. # Note: This should be used by tools like giter8/conscript etc. # TODO - Should we merge the main SBT script with this library? if test -z "$HOME"; then declare -r script_dir="$(dirname "$script_path")" else declare -r script_dir="$HOME/.sbt" fi declare -a residual_args declare -a java_args declare -a scalac_args declare -a sbt_commands declare -a maven_profiles if test -x "$JAVA_HOME/bin/java"; then echo -e "Using $JAVA_HOME as default JAVA_HOME." echo "Note, this will be overridden by -java-home if it is set." declare java_cmd="$JAVA_HOME/bin/java" else declare java_cmd=java fi echoerr () { echo 1>&2 "$@" } vlog () { [[ $verbose || $debug ]] && echoerr "$@" } dlog () { [[ $debug ]] && echoerr "$@" } acquire_sbt_jar () { SBT_VERSION=`awk -F "=" '/sbt\.version/ {print $2}' ./project/build.properties` URL1=${DEFAULT_ARTIFACT_REPOSITORY:-https://repo1.maven.org/maven2/}org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar JAR=build/sbt-launch-${SBT_VERSION}.jar sbt_jar=$JAR if [[ ! -f "$sbt_jar" ]]; then # Download sbt launch jar if it hasn't been downloaded yet if [ ! -f "${JAR}" ]; then # Download printf "Attempting to fetch sbt\n" JAR_DL="${JAR}.part" if [ $(command -v curl) ]; then curl --fail --location --silent ${URL1} > "${JAR_DL}" &&\ mv "${JAR_DL}" "${JAR}" elif [ $(command -v wget) ]; then wget --quiet ${URL1} -O "${JAR_DL}" &&\ mv "${JAR_DL}" "${JAR}" else printf "You do not have curl or wget installed, please install sbt manually from http://www.scala-sbt.org/\n" exit -1 fi fi if [ ! -f "${JAR}" ]; then # We failed to download printf "Our attempt to download sbt locally to ${JAR} failed. Please install sbt manually from http://www.scala-sbt.org/\n" exit -1 fi printf "Launching sbt from ${JAR}\n" fi } execRunner () { # print the arguments one to a line, quoting any containing spaces [[ $verbose || $debug ]] && echo "# Executing command line:" && { for arg; do if printf "%s\n" "$arg" | grep -q ' '; then printf "\"%s\"\n" "$arg" else printf "%s\n" "$arg" fi done echo "" } "$@" } addJava () { dlog "[addJava] arg = '$1'" java_args=( "${java_args[@]}" "$1" ) } enableProfile () { dlog "[enableProfile] arg = '$1'" maven_profiles=( "${maven_profiles[@]}" "$1" ) export SBT_MAVEN_PROFILES="${maven_profiles[@]}" } addSbt () { dlog "[addSbt] arg = '$1'" sbt_commands=( "${sbt_commands[@]}" "$1" ) } addResidual () { dlog "[residual] arg = '$1'" residual_args=( "${residual_args[@]}" "$1" ) } addDebugger () { addJava "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$1" } # a ham-fisted attempt to move some memory settings in concert # so they need not be dicked around with individually. get_mem_opts () { local mem=${1:-1000} local perm=$(( $mem / 4 )) (( $perm > 256 )) || perm=256 (( $perm < 4096 )) || perm=4096 local codecache=$(( $perm / 2 )) echo "-Xms${mem}m -Xmx${mem}m -XX:ReservedCodeCacheSize=${codecache}m" } require_arg () { local type="$1" local opt="$2" local arg="$3" if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then echo "$opt requires <$type> argument" 1>&2 exit 1 fi } is_function_defined() { declare -f "$1" > /dev/null } process_args () { while [[ $# -gt 0 ]]; do case "$1" in -h|-help) usage; exit 1 ;; -v|-verbose) verbose=1 && shift ;; -d|-debug) debug=1 && shift ;; -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;; -mem) require_arg integer "$1" "$2" && sbt_mem="$2" && shift 2 ;; -jvm-debug) require_arg port "$1" "$2" && addDebugger $2 && shift 2 ;; -batch) exec MergeStrategy.discard case x => MergeStrategy.first } ) ================================================ FILE: benchmarks/infrastructure/aws/terraform/.terraform.lock.hcl ================================================ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/aws" { version = "4.15.1" constraints = "~> 4.15.1" hashes = [ "h1:KPu3MdNXCScye05Sp4JlE9WwhS9k3yD9KRoRDHg5sDE=", "zh:1d944144f8d613b8090c0c8391e4b205ca036086d70aceb4cdf664856fa8410c", "zh:2a0ca16a6b12c0ac509f64512f80bd2ed6e7ea0ec369212efd4be3fa65e9773d", "zh:3f9efdce4f1c320ffd061e8715e1d031deac1be0b959eaa60c25a274925653e4", "zh:4cf82f3267b0c3e08be29b0345f711ab84ea1ea75f0e8ce81f5a2fe635ba67b4", "zh:58474a0b7da438e1bcd53e87f10e28830836ff9b46cce5f09413c90952ae4f78", "zh:6eb1be8afb0314b6b8424fe212b13beeb04f3f24692f0f3ee86c5153c7eb2e63", "zh:8022da7d3b050d452ce6c679844e13729bdb4e1b3e75dcf68931af17a06b9277", "zh:8e2683d00fff1df43440d6e7c04a2c1eb432c7d5dacff32fe8ce9045bc948fe6", "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", "zh:b0c22d9a306e8ac2de57b5291a3d0a7a2c1713e33b7d076005662451afdc4d29", "zh:ba6b7d7d91388b636145b133da6b4e32620cdc8046352e2dc8f3f0f81ff5d2e2", "zh:d38a816eb60f4419d99303136a3bb61a0d2df3ca8a1dce2ced9b99bf23efa9f7", ] } ================================================ FILE: benchmarks/infrastructure/aws/terraform/README.md ================================================ # Create infrastructure with Terraform 1. Install [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli?in=terraform/aws-get-started). 2. Create an IAM user which will be used to create benchmarks infrastructure. Ensure that your AWS CLI is configured. You should either have valid credentials in shared credentials file (e.g. `~/.aws/credentials`) ``` [default] aws_access_key_id = anaccesskey aws_secret_access_key = asecretkey ``` or export keys as environment variables: ```bash export AWS_ACCESS_KEY_ID="anaccesskey" export AWS_SECRET_ACCESS_KEY="asecretkey" ``` 3. Add permissions for the IAM user. You can either assign `AdministratorAccess` AWS managed policy (discouraged) or assign AWS managed policies in a more granular way: * `IAMFullAccess` * `AmazonVPCFullAccess` * `AmazonEMRFullAccessPolicy_v2` * `AmazonElasticMapReduceFullAccess` * `AmazonRDSFullAccess` * `AmazonS3FullAccess` * a custom policy for EC2 key pairs management ```json { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "ec2:ImportKeyPair", "ec2:CreateKeyPair", "ec2:DeleteKeyPair" ], "Resource": "arn:aws:ec2:*:*:key-pair/benchmarks_key_pair" } ] } ``` 4. Create Terraform variable file `benchmarks/infrastructure/aws/terraform/terraform.tfvars` and fill in variable values. ```tf region = "" availability_zone1 = "" availability_zone2 = "" benchmarks_bucket_name = "" source_bucket_name = "" mysql_user = "" mysql_password = "" emr_public_key_path = "" user_ip_address = "" emr_workers = WORKERS_COUNT tags = { key1 = "value1" key2 = "value2" } ``` Please check `variables.tf` to learn more about each parameter. 5. Run: ```bash terraform init terraform validate terraform apply ``` As a result, a new VPC, a S3 bucket, a MySQL instance (metastore) and a EMR cluster will be created. The `apply` command returns `master_node_address` that will be used when running benchmarks. ``` Apply complete! Resources: 16 added, 0 changed, 0 destroyed. Outputs: master_node_address = "35.165.163.250" ``` 6. Once the benchmarks are finished, destroy the resources. ```bash terraform destroy ``` If the S3 bucket contains any objects, it will not be destroyed automatically. One need to do that manually to avoid any accidental data loss. ``` Error: deleting S3 Bucket (my-bucket): BucketNotEmpty: The bucket you tried to delete is not empty status code: 409, request id: Q11TYZ5E0B23QGQ2, host id: WdeFY88km5IBhy+bi2hqXzgjBxjrn1+OPtCstsWDjkwGNCyEhXYjq330DZq1jbfNXojBEejH6Wg= ``` ================================================ FILE: benchmarks/infrastructure/aws/terraform/main.tf ================================================ module "networking" { source = "./modules/networking" availability_zone1 = var.availability_zone1 availability_zone2 = var.availability_zone2 } module "storage" { source = "./modules/storage" benchmarks_bucket_name = var.benchmarks_bucket_name } module "processing" { source = "./modules/processing" vpc_id = module.networking.vpc_id subnet1_id = module.networking.subnet1_id subnet2_id = module.networking.subnet2_id availability_zone1 = var.availability_zone1 benchmarks_bucket_name = var.benchmarks_bucket_name source_bucket_name = var.source_bucket_name mysql_user = var.mysql_user mysql_password = var.mysql_password emr_public_key_path = var.emr_public_key_path emr_workers = var.emr_workers user_ip_address = var.user_ip_address depends_on = [module.networking, module.storage] } ================================================ FILE: benchmarks/infrastructure/aws/terraform/modules/networking/main.tf ================================================ resource "aws_vpc" "this" { cidr_block = "10.0.0.0/16" } resource "aws_subnet" "benchmarks_subnet1" { vpc_id = aws_vpc.this.id availability_zone = var.availability_zone1 cidr_block = "10.0.0.0/17" } # There are two subnets needed to create an RDS subnet group. In fact this one is unused. # If DB subnet group is built using only one AZ, the following error is thrown: # The DB subnet group doesn't meet Availability Zone (AZ) coverage requirement. # Current AZ coverage: us-west-2a. Add subnets to cover at least 2 AZs. resource "aws_subnet" "benchmarks_subnet2" { vpc_id = aws_vpc.this.id availability_zone = var.availability_zone2 cidr_block = "10.0.128.0/17" } resource "aws_internet_gateway" "this" { vpc_id = aws_vpc.this.id } resource "aws_default_route_table" "public" { default_route_table_id = aws_vpc.this.default_route_table_id route { cidr_block = "0.0.0.0/0" gateway_id = aws_internet_gateway.this.id } } ================================================ FILE: benchmarks/infrastructure/aws/terraform/modules/networking/outputs.tf ================================================ output "vpc_id" { value = aws_vpc.this.id } output "subnet1_id" { value = aws_subnet.benchmarks_subnet1.id } output "subnet2_id" { value = aws_subnet.benchmarks_subnet2.id } ================================================ FILE: benchmarks/infrastructure/aws/terraform/modules/networking/variables.tf ================================================ variable "availability_zone1" { type = string } variable "availability_zone2" { type = string } ================================================ FILE: benchmarks/infrastructure/aws/terraform/modules/processing/main.tf ================================================ resource "aws_db_instance" "metastore_service" { engine = "mysql" engine_version = "8.0.28" instance_class = "db.m5.large" db_name = "hive" username = var.mysql_user password = var.mysql_password availability_zone = var.availability_zone1 skip_final_snapshot = true allocated_storage = 50 db_subnet_group_name = aws_db_subnet_group.metastore_service.name vpc_security_group_ids = [aws_security_group.metastore_service.id] } resource "aws_db_subnet_group" "metastore_service" { name = "benchmarks_subnet_group_for_metastore_service" subnet_ids = [var.subnet1_id, var.subnet2_id] } /* EC2 key used to SSH to EMR cluster nodes. */ resource "aws_key_pair" "benchmarks" { key_name = "benchmarks_key_pair" public_key = file(var.emr_public_key_path) } resource "aws_emr_cluster" "benchmarks" { name = "delta_performance_benchmarks_cluster" release_label = "emr-6.5.0" applications = ["Spark", "Hive"] termination_protection = false keep_job_flow_alive_when_no_steps = true ec2_attributes { instance_profile = aws_iam_instance_profile.benchmarks_emr_profile.arn key_name = aws_key_pair.benchmarks.key_name subnet_id = var.subnet1_id emr_managed_master_security_group = aws_security_group.emr.id emr_managed_slave_security_group = aws_security_group.emr.id } master_instance_group { instance_type = "i3.2xlarge" } core_instance_group { instance_type = "i3.2xlarge" instance_count = var.emr_workers } configurations_json = < -i --ssh-user --cloud-provider --benchmark test """ def parse_args(): # Parse cmd line arguments parser = argparse.ArgumentParser() parser.add_argument( "--benchmark", "-b", required=True, help="Run the given benchmark. See this " + "python file for the list of predefined benchmark names and definitions.") parser.add_argument( "--cluster-hostname", required=True, help="Hostname or public IP of the cluster driver") parser.add_argument( "--ssh-id-file", "-i", required=True, help="SSH identity file") parser.add_argument( "--spark-conf", action="append", help="Run benchmark with given spark conf. Use separate --spark-conf for multiple confs.") parser.add_argument( "--resume-benchmark", help="Resume waiting for the given running benchmark.") parser.add_argument( "--use-local-delta-dir", help="Local path to delta repository which will be used for running the benchmark " + "instead of the version specified in the specification. Make sure that new delta" + " version is compatible with version in the spec.") parser.add_argument( "--cloud-provider", choices=delta_log_store_classes.keys(), help="Cloud where the benchmark will be executed.") parser.add_argument( "--ssh-user", default="hadoop", help="The user which is used to communicate with the master via SSH.") parsed_args, parsed_passthru_args = parser.parse_known_args() return parsed_args, parsed_passthru_args def run_single_benchmark(benchmark_name, benchmark_spec, other_args): benchmark_spec.append_spark_confs(other_args.spark_conf) benchmark_spec.append_spark_conf(delta_log_store_classes.get(other_args.cloud_provider)) benchmark_spec.append_main_class_args(passthru_args) print("------") print("Benchmark spec to run:\n" + str(vars(benchmark_spec))) print("------") benchmark = Benchmark(benchmark_name, benchmark_spec, use_spark_shell=True, local_delta_dir=other_args.use_local_delta_dir) benchmark_dir = os.path.dirname(os.path.abspath(__file__)) with WorkingDirectory(benchmark_dir): benchmark.run(other_args.cluster_hostname, other_args.ssh_id_file, other_args.ssh_user) if __name__ == "__main__": """ Run benchmark on a cluster using ssh. Example usage: ./run-benchmark.py --cluster-hostname -i --ssh-user --cloud-provider --benchmark test """ args, passthru_args = parse_args() if args.resume_benchmark is not None: Benchmark.wait_for_completion( args.cluster_hostname, args.ssh_id_file, args.resume_benchmark, args.ssh_user) exit(0) benchmark_names = args.benchmark.split(",") for benchmark_name in benchmark_names: # Create and run the benchmark if benchmark_name in benchmarks: run_single_benchmark(benchmark_name, benchmarks[benchmark_name], args) else: raise Exception("Could not find benchmark spec for '" + benchmark_name + "'." + "Must provide one of the predefined benchmark names:\n" + "\n".join(benchmarks.keys()) + "\nSee this python file for more details.") ================================================ FILE: benchmarks/scripts/benchmarks.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from scripts.utils import * from datetime import datetime import time class BenchmarkSpec: """ Specifications of a benchmark. :param format_name: Spark format name :param maven_artifacts: Maven artifact name in x:y:z format :param spark_confs: list of spark conf strings in key=value format :param benchmark_main_class: Name of main Scala class from the JAR to run :param main_class_args: command line args for the main class """ def __init__( self, format_name, maven_artifacts, spark_confs, benchmark_main_class, main_class_args, extra_spark_shell_args=None, **kwargs): if main_class_args is None: main_class_args = [] if extra_spark_shell_args is None: extra_spark_shell_args = [] self.format_name = format_name self.maven_artifacts = maven_artifacts self.spark_confs = spark_confs self.benchmark_main_class = benchmark_main_class self.benchmark_main_class_args = main_class_args self.extra_spark_shell_args = extra_spark_shell_args def append_spark_conf(self, new_conf): if isinstance(new_conf, str): self.spark_confs.append(new_conf) def append_spark_confs(self, new_confs): if new_confs is not None and isinstance(new_confs, list): self.spark_confs.extend(new_confs) def append_main_class_args(self, new_args): if new_args is not None and isinstance(new_args, list): self.benchmark_main_class_args.extend(new_args) def get_sparksubmit_cmd(self, benchmark_jar_path): spark_conf_str = "" for conf in self.spark_confs: print(f"conf={conf}") spark_conf_str += f"""--conf "{conf}" """ main_class_args = ' '.join(self.benchmark_main_class_args) spark_shell_args_str = ' '.join(self.extra_spark_shell_args) spark_submit_cmd = ( f"spark-submit {spark_shell_args_str} " + (f"--packages {self.maven_artifacts} " if self.maven_artifacts else "") + f"{spark_conf_str} --class {self.benchmark_main_class} " + f"{benchmark_jar_path} {main_class_args}" ) print(spark_submit_cmd) return spark_submit_cmd def get_sparkshell_cmd(self, benchmark_jar_path, benchmark_init_file_path): spark_conf_str = "" for conf in self.spark_confs: print(f"conf={conf}") spark_conf_str += f"""--conf "{conf}" """ spark_shell_args_str = ' '.join(self.extra_spark_shell_args) spark_shell_cmd = ( f"spark-shell {spark_shell_args_str} " + (f"--packages {self.maven_artifacts} " if self.maven_artifacts else "") + f"{spark_conf_str} --jars {benchmark_jar_path} -I {benchmark_init_file_path}" ) print(spark_shell_cmd) return spark_shell_cmd class TPCDSDataLoadSpec(BenchmarkSpec): """ Specifications of TPC-DS data load process. Always mixin in this first before the base benchmark class. """ def __init__(self, scale_in_gb, exclude_nulls=True, **kwargs): # forward all keyword args to next constructor super().__init__(benchmark_main_class="benchmark.TPCDSDataLoad", **kwargs) self.benchmark_main_class_args.extend([ "--format", self.format_name, "--scale-in-gb", str(scale_in_gb), "--exclude-nulls", str(exclude_nulls), ]) # To access the public TPCDS parquet files on S3 self.spark_confs.extend(["spark.hadoop.fs.s3.useRequesterPaysHeader=true"]) class TPCDSBenchmarkSpec(BenchmarkSpec): """ Specifications of TPC-DS benchmark. """ def __init__(self, scale_in_gb, **kwargs): # forward all keyword args to next constructor super().__init__(benchmark_main_class="benchmark.TPCDSBenchmark", **kwargs) # after init of super class, use the format to add main class args self.benchmark_main_class_args.extend([ "--format", self.format_name, "--scale-in-gb", str(scale_in_gb) ]) class MergeDataLoadSpec(BenchmarkSpec): """ Specifications of Merge data load process. Always mixin in this first before the base benchmark class. """ def __init__(self, scale_in_gb, exclude_nulls=True, **kwargs): # forward all keyword args to next constructor super().__init__(benchmark_main_class="benchmark.MergeDataLoad", **kwargs) self.benchmark_main_class_args.extend([ "--scale-in-gb", str(scale_in_gb), ]) # To access the public TPCDS parquet files on S3 self.spark_confs.extend(["spark.hadoop.fs.s3.useRequesterPaysHeader=true"]) class MergeBenchmarkSpec(BenchmarkSpec): """ Specifications of Merge benchmark. """ def __init__(self, scale_in_gb, **kwargs): # forward all keyword args to next constructor super().__init__(benchmark_main_class="benchmark.MergeBenchmark", **kwargs) # after init of super class, use the format to add main class args self.benchmark_main_class_args.extend([ "--scale-in-gb", str(scale_in_gb) ]) # ============== Delta benchmark specifications ============== class DeltaBenchmarkSpec(BenchmarkSpec): """ Specification of a benchmark using the Delta format. """ def __init__(self, delta_version, benchmark_main_class, main_class_args=None, scala_version="2.12", **kwargs): delta_spark_confs = [ "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension", "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" ] self.scala_version = scala_version if "spark_confs" in kwargs and isinstance(kwargs["spark_confs"], list): kwargs["spark_confs"].extend(delta_spark_confs) else: kwargs["spark_confs"] = delta_spark_confs super().__init__( format_name="delta", maven_artifacts=self.delta_maven_artifacts(delta_version, self.scala_version), benchmark_main_class=benchmark_main_class, main_class_args=main_class_args, **kwargs ) def update_delta_version(self, new_delta_version): self.maven_artifacts = \ DeltaBenchmarkSpec.delta_maven_artifacts(new_delta_version, self.scala_version) @staticmethod def delta_maven_artifacts(delta_version, scala_version): return f"io.delta:delta-core_{scala_version}:{delta_version},io.delta:delta-contribs_{scala_version}:{delta_version},io.delta:delta-hive_{scala_version}:0.2.0" class DeltaTPCDSDataLoadSpec(TPCDSDataLoadSpec, DeltaBenchmarkSpec): def __init__(self, delta_version, scale_in_gb=1): super().__init__(delta_version=delta_version, scale_in_gb=scale_in_gb) class DeltaTPCDSBenchmarkSpec(TPCDSBenchmarkSpec, DeltaBenchmarkSpec): def __init__(self, delta_version, scale_in_gb=1): super().__init__(delta_version=delta_version, scale_in_gb=scale_in_gb) class DeltaMergeDataLoadSpec(MergeDataLoadSpec, DeltaBenchmarkSpec): def __init__(self, delta_version, scale_in_gb=1): super().__init__(delta_version=delta_version, scale_in_gb=scale_in_gb) class DeltaMergeBenchmarkSpec(MergeBenchmarkSpec, DeltaBenchmarkSpec): def __init__(self, delta_version, scale_in_gb=1): super().__init__(delta_version=delta_version, scale_in_gb=scale_in_gb) # ============== Parquet benchmark specifications ============== class ParquetBenchmarkSpec(BenchmarkSpec): """ Specification of a benchmark using the Parquet format. """ def __init__(self, benchmark_main_class, main_class_args=None, **kwargs): super().__init__( format_name="parquet", maven_artifacts=None, spark_confs=[], benchmark_main_class=benchmark_main_class, main_class_args=main_class_args, **kwargs ) class ParquetTPCDSDataLoadSpec(TPCDSDataLoadSpec, ParquetBenchmarkSpec): def __init__(self, scale_in_gb=1): super().__init__(scale_in_gb=scale_in_gb) class ParquetTPCDSBenchmarkSpec(TPCDSBenchmarkSpec, ParquetBenchmarkSpec): def __init__(self, scale_in_gb=1): super().__init__(scale_in_gb=scale_in_gb) # ============== General benchmark execution ============== class Benchmark: """ Represents a benchmark that can be run on a remote Spark cluster :param benchmark_name: A name to be used for uniquely identifying this benchmark. Added to file names generated by this benchmark. :param benchmark_spec: Specification of the benchmark. See BenchmarkSpec. """ def __init__(self, benchmark_name, benchmark_spec, use_spark_shell, local_delta_dir=None): now = datetime.now() self.benchmark_id = now.strftime("%Y%m%d-%H%M%S") + "-" + benchmark_name self.benchmark_spec = benchmark_spec # Add benchmark id as a spark conf so that it get transferred automatically to scala code self.benchmark_spec.append_spark_confs([f"spark.benchmarkId={self.benchmark_id}"]) self.output_file = Benchmark.output_file(self.benchmark_id) self.json_report_file = Benchmark.json_report_file(self.benchmark_id) self.completed_file = Benchmark.completed_file(self.benchmark_id) self.use_spark_shell = use_spark_shell self.local_delta_dir = local_delta_dir def run(self, cluster_hostname, ssh_id_file, ssh_user): if self.local_delta_dir and isinstance(self.benchmark_spec, DeltaBenchmarkSpec): # Upload new Delta jar to cluster and update spec to use the jar's version delta_version_to_use = \ self.upload_delta_jars_to_cluster_and_get_version(cluster_hostname, ssh_id_file, ssh_user) self.benchmark_spec.update_delta_version(delta_version_to_use) jar_path_in_cluster = self.upload_jar_to_cluster(cluster_hostname, ssh_id_file, ssh_user) self.install_dependencies_via_ssh(cluster_hostname, ssh_id_file, ssh_user) self.start_benchmark_via_ssh(cluster_hostname, ssh_id_file, jar_path_in_cluster, ssh_user) Benchmark.wait_for_completion(cluster_hostname, ssh_id_file, self.benchmark_id, ssh_user) def spark_submit_script_content(self, jar_path): return f""" #!/bin/bash jps | grep "Spark" | cut -f 1 -d ' ' | xargs kill -9 set -e {self.benchmark_spec.get_sparksubmit_cmd(jar_path)} 2>&1 | tee {self.output_file} """.strip() def spark_shell_script_content(self, jar_path): shell_init_file_name = f"{self.benchmark_id}_shell_init.scala" benchmark_cmd_line_params_str = \ ', '.join(f'"{w}"' for w in self.benchmark_spec.benchmark_main_class_args) call_main_with_args = \ f"{self.benchmark_spec.benchmark_main_class}.main(Array[String]({benchmark_cmd_line_params_str}))" shell_init_file_content = \ "try { %s } catch { case t => println(t); println(\"FAILED\"); System.exit(1) } ; System.exit(0)" % call_main_with_args shell_cmd = self.benchmark_spec.get_sparkshell_cmd(jar_path, shell_init_file_name) return f""" #!/bin/bash jps | grep "Spark" | cut -f 1 -d ' ' | xargs kill -9 echo '{shell_init_file_content}' > {shell_init_file_name} {shell_cmd} 2>&1 | tee {self.output_file} touch {self.completed_file} """.strip() def upload_jar_to_cluster(self, cluster_hostname, ssh_id_file, ssh_user, delta_version_to_use=None): # Compile JAR # Note: Deleting existing JARs instead of sbt clean is faster if os.path.exists("target"): run_cmd("""find target -name "*.jar" -type f -delete""", stream_output=True) run_cmd("build/sbt assembly", stream_output=True) (_, out, _) = run_cmd("find target -name *.jar") print(">>> Benchmark JAR compiled\n") # Upload JAR jar_local_path = out.decode("utf-8").strip() jar_remote_path = f"{self.benchmark_id}-benchmarks.jar" scp_cmd = \ f"scp -C -i {ssh_id_file} {jar_local_path} {ssh_user}@{cluster_hostname}:{jar_remote_path}" print(scp_cmd) run_cmd(scp_cmd, stream_output=True) print(">>> Benchmark JAR uploaded to cluster\n") return f"~/{jar_remote_path}" def install_dependencies_via_ssh(self, cluster_hostname, ssh_id_file, ssh_user): script_file_name = f"{self.benchmark_id}-install-deps.sh" script_file_text = """ #!/bin/bash package='screen' if [ -x "$(command -v yum)" ]; then if rpm -q $package; then echo "$package has already been installed" else sudo yum -y install $package fi elif [ -x "$(command -v apt)" ]; then if dpkg -s $package; then echo "$package has already been installed" else sudo apt install $package fi else echo "Failed to install packages: Package manager not found. You must manually install: $package">&2; exit 1; fi """.strip() self.copy_script_via_ssh(cluster_hostname, ssh_id_file, ssh_user, script_file_name, script_file_text) print(">>> Install dependencies script generated and uploaded\n") job_cmd = ( f"ssh -i {ssh_id_file} {ssh_user}@{cluster_hostname} " + f"bash {script_file_name}" ) print(job_cmd) run_cmd(job_cmd, stream_output=True) print(">>> Dependencies have been installed\n") def start_benchmark_via_ssh(self, cluster_hostname, ssh_id_file, jar_path, ssh_user): # Generate and upload the script to run the benchmark script_file_name = f"{self.benchmark_id}-cmd.sh" if self.use_spark_shell: script_file_text = self.spark_shell_script_content(jar_path) else: script_file_text = self.spark_submit_script_content(jar_path) self.copy_script_via_ssh(cluster_hostname, ssh_id_file, ssh_user, script_file_name, script_file_text) print(">>> Benchmark script generated and uploaded\n") # Start the script job_cmd = ( f"ssh -i {ssh_id_file} {ssh_user}@{cluster_hostname} " + f"screen -d -m bash {script_file_name}" ) print(job_cmd) run_cmd(job_cmd, stream_output=True) # Print the screen where it is running run_cmd(f"ssh -i {ssh_id_file} {ssh_user}@{cluster_hostname}" + f""" "screen -ls ; sleep 2; echo Files for this benchmark: ; ls {self.benchmark_id}*" """, stream_output=True, throw_on_error=False) print(f">>> Benchmark id {self.benchmark_id} started in a screen. Stdout piped into {self.output_file}. " f"Final report will be generated on completion in {self.json_report_file}.\n") @staticmethod def copy_script_via_ssh(cluster_hostname, ssh_id_file, ssh_user, script_file_name, script_file_text): try: script_file = open(script_file_name, "w") script_file.write(script_file_text) script_file.close() scp_cmd = ( f"scp -i {ssh_id_file} {script_file_name}" + f" {ssh_user}@{cluster_hostname}:{script_file_name}" ) print(scp_cmd) run_cmd(scp_cmd, stream_output=True) run_cmd_over_ssh(f"chmod +x {script_file_name}", cluster_hostname, ssh_id_file, ssh_user, throw_on_error=False) finally: if os.path.exists(script_file_name): os.remove(script_file_name) @staticmethod def output_file(benchmark_id): return f"{benchmark_id}-out.txt" @staticmethod def json_report_file(benchmark_id): return f"{benchmark_id}-report.json" @staticmethod def csv_report_file(benchmark_id): return f"{benchmark_id}-report.csv" @staticmethod def completed_file(benchmark_id): return f"{benchmark_id}-completed.txt" @staticmethod def wait_for_completion(cluster_hostname, ssh_id_file, benchmark_id, ssh_user, copy_report=True): completed = False succeeded = False output_file = Benchmark.output_file(benchmark_id) completed_file = Benchmark.completed_file(benchmark_id) json_report_file = Benchmark.json_report_file(benchmark_id) csv_report_file = Benchmark.csv_report_file(benchmark_id) print(f"\nWaiting for completion of benchmark id {benchmark_id}") while not completed: # Print the size of the output file to show progress (_, out, _) = run_cmd_over_ssh(f"stat -c '%n: [%y] [%s bytes]' {output_file}", cluster_hostname, ssh_id_file, ssh_user, throw_on_error=False) out = out.decode("utf-8").strip() print(out) if "No such file" in out: print(">>> Benchmark failed to start") return # Check for the existence of the completed file (_, out, _) = run_cmd_over_ssh(f"ls {completed_file}", cluster_hostname, ssh_id_file, ssh_user, throw_on_error=False) if completed_file in out.decode("utf-8"): completed = True else: time.sleep(60) # Check the last few lines of output files to identify success (_, out, _) = run_cmd_over_ssh(f"tail {output_file}", cluster_hostname, ssh_id_file, ssh_user, throw_on_error=False) if "SUCCESS" in out.decode("utf-8"): succeeded = True print(">>> Benchmark completed with success\n") else: print(">>> Benchmark completed with failure\n") # Download reports if copy_report: Benchmark.download_file(output_file, cluster_hostname, ssh_id_file, ssh_user) if succeeded: report_files = [json_report_file, csv_report_file] for report_file in report_files: Benchmark.download_file(report_file, cluster_hostname, ssh_id_file, ssh_user) print(">>> Downloaded reports to local directory") @staticmethod def download_file(file, cluster_hostname, ssh_id_file, ssh_user): run_cmd(f"scp -C -i {ssh_id_file} " + f"{ssh_user}@{cluster_hostname}:{file} {file}", stream_output=True) def upload_delta_jars_to_cluster_and_get_version(self, cluster_hostname, ssh_id_file, ssh_user): if not self.local_delta_dir: raise Exception("Path to delta repo not specified") delta_repo_dir = os.path.abspath(self.local_delta_dir) with WorkingDirectory(delta_repo_dir): # Compile Delta JARs by publishing to local maven cache print(f"Compiling Delta to local dir {delta_repo_dir}") local_maven_delta_dir = os.path.expanduser("~/.ivy2/local/io.delta/") if os.path.exists(local_maven_delta_dir): run_cmd(f"rm -rf {local_maven_delta_dir}", stream_output=True) print(f"Cleared local maven cache at {local_maven_delta_dir}") run_cmd("build/sbt publishLocal", stream_output=False, throw_on_error=True) # Get the new version (_, out, _) = run_cmd("""build/sbt "show version" """) version = out.decode("utf-8").strip().rsplit("\n", 1)[-1].rsplit(" ", 1)[-1].strip() if not version: raise Exception(f"Could not find the version from the sbt output:\n--\n{out}\n-") # Upload JARs to cluster's local maven cache remote_maven_dir = ".ivy2/local/" # must have "/" at the end run_cmd_over_ssh( f"rm -rf {remote_maven_dir}/* .ivy2/cache/io.delta .ivy2/jars/io.delta*", cluster_hostname, ssh_id_file, ssh_user, stream_output=True, throw_on_error=False) run_cmd_over_ssh(f"mkdir -p {remote_maven_dir}", cluster_hostname, ssh_id_file, ssh_user, stream_output=True) scp_cmd = f"""scp -r -C -i {ssh_id_file} {local_maven_delta_dir.rstrip("/")} """ +\ f"{ssh_user}@{cluster_hostname}:{remote_maven_dir}" print(scp_cmd) run_cmd(scp_cmd, stream_output=True) print(f">>> Delta {version} JAR uploaded to cluster\n") return version ================================================ FILE: benchmarks/scripts/utils.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os import shlex import subprocess def run_cmd(cmd, throw_on_error=True, env=None, stream_output=False, **kwargs): if isinstance(cmd, str): cmd = shlex.split(cmd) cmd_env = os.environ.copy() if env: cmd_env.update(env) if stream_output: child = subprocess.Popen(cmd, env=cmd_env, **kwargs) exit_code = child.wait() if throw_on_error and exit_code != 0: raise Exception("Non-zero exitcode: %s" % (exit_code)) return exit_code else: child = subprocess.Popen( cmd, env=cmd_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) (stdout, stderr) = child.communicate() exit_code = child.wait() if throw_on_error and exit_code != 0: raise Exception( "Non-zero exitcode: %s\n\nSTDOUT:\n%s\n\nSTDERR:%s" % (exit_code, stdout, stderr)) return exit_code, stdout, stderr def run_cmd_over_ssh(cmd, host, ssh_id_file, user, **kwargs): full_cmd = f"""ssh -i {ssh_id_file} {user}@{host} "{cmd}" """ return run_cmd(full_cmd, **kwargs) # pylint: disable=too-few-public-methods class WorkingDirectory(object): def __init__(self, working_directory): self.working_directory = working_directory self.old_workdir = os.getcwd() def __enter__(self): os.chdir(self.working_directory) def __exit__(self, tpe, value, traceback): os.chdir(self.old_workdir) ================================================ FILE: benchmarks/src/main/scala/benchmark/Benchmark.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package benchmark import java.net.URI import java.nio.file.{Files, Paths} import java.nio.charset.StandardCharsets import scala.collection.mutable import scala.language.postfixOps import scala.sys.process._ import scala.util.control.NonFatal import com.fasterxml.jackson.annotation.JsonInclude.Include import com.fasterxml.jackson.annotation.JsonPropertyOrder import com.fasterxml.jackson.databind.{DeserializationFeature, MapperFeature, ObjectMapper} import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper} import org.apache.spark.SparkUtils import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.functions.col import org.apache.spark.sql.internal.SQLConf trait BenchmarkConf extends Product { /** Cloud path where benchmark data is going to be written. */ def benchmarkPath: Option[String] /** Get the database location given the database name and the benchmark path. */ def dbLocation(dbName: String, suffix: String = ""): String = { benchmarkPath.map(p => s"$p/databases/${dbName}_${suffix}").getOrElse { throw new IllegalArgumentException("Benchmark path must be specified") } } /** Cloud path where benchmark reports will be uploaded. */ def reportUploadPath: String = { benchmarkPath.map(p => s"$p/reports/").getOrElse { throw new IllegalArgumentException("Benchmark path must be specified") } } def jsonReportUploadPath: String = s"$reportUploadPath/json/" def csvReportUploadPath: String = s"$reportUploadPath/csv/" /** Get the benchmark conf details as a map. */ def asMap: Map[String, String] = SparkUtils.caseClassToMap(this) } @JsonPropertyOrder(alphabetic=true) case class QueryResult( name: String, iteration: Option[Int], durationMs: Option[Long], errorMsg: Option[String]) @JsonPropertyOrder(alphabetic=true) case class SparkEnvironmentInfo( @JsonPropertyOrder(alphabetic=true) sparkBuildInfo: Map[String, String], @JsonPropertyOrder(alphabetic=true) runtimeInfo: Map[String, String], @JsonPropertyOrder(alphabetic=true) sparkProps: Map[String, String], @JsonPropertyOrder(alphabetic=true) hadoopProps: Map[String, String], @JsonPropertyOrder(alphabetic=true) systemProps: Map[String, String], @JsonPropertyOrder(alphabetic=true) classpathEntries: Map[String, String]) @JsonPropertyOrder(alphabetic=true) case class BenchmarkReport( @JsonPropertyOrder(alphabetic=true) benchmarkSpecs: Map[String, String], queryResults: Array[QueryResult], extraMetrics: Map[String, Double], sparkEnvInfo: SparkEnvironmentInfo) /** * Base class for any benchmark with the core functionality of measuring SQL query durations * and printing the details as json in a report file. */ abstract class Benchmark(private val conf: BenchmarkConf) { /* Methods that implementations should override. */ protected def runInternal(): Unit /* Fields and methods that implementations should not have to override */ final protected lazy val spark = { val s = SparkSession.builder() .config("spark.ui.proxyBase", "") .getOrCreate() log("Spark started with configuration:\n" + s.conf.getAll.toSeq.sortBy(_._1).map(x => x._1 + ": " + x._2).mkString("\t", "\n\t", "\n")) s.sparkContext.setLogLevel("WARN") sys.props.update("spark.ui.proxyBase", "") s } val extraConfs: Map[String, String] = Map( SQLConf.BROADCAST_TIMEOUT.key -> "7200", SQLConf.CROSS_JOINS_ENABLED.key -> "true" ) private val queryResults = new mutable.ArrayBuffer[QueryResult] private val extraMetrics = new mutable.HashMap[String, Double] protected def run(): Unit = { try { log("=" * 80) log("=" * 80) runInternal() log("=" * 80) } finally { generateReport() } println(s"SUCCESS") } protected def runQuery( sqlCmd: String, queryName: String = "", iteration: Option[Int] = None, printRows: Boolean = false, ignoreError: Boolean = true): Seq[Row] = synchronized { val iterationStr = iteration.map(i => s" - iteration $i").getOrElse("") var banner = s"$queryName$iterationStr" if (banner.trim.isEmpty) { banner = sqlCmd.split("\n")(0).trim + (if (sqlCmd.split("\n").size > 1) "..." else "") } log("=" * 80) log(s"START: $banner") log("SQL: " + sqlCmd.replaceAll("\n\\s*", " ")) spark.sparkContext.setJobGroup(banner, banner, interruptOnCancel = true) try { val before = System.nanoTime() val df = spark.sql(sqlCmd) val r = df.collect() val after = System.nanoTime() if (printRows) df.show(false) val durationMs = (after - before) / (1000 * 1000) queryResults += QueryResult(queryName, iteration, Some(durationMs), errorMsg = None) log(s"END took $durationMs ms: $banner") log("=" * 80) r } catch { case NonFatal(e) => log(s"ERROR: $banner\n${e.getMessage}") queryResults += QueryResult(queryName, iteration, durationMs = None, errorMsg = Some(e.getMessage)) if (!ignoreError) throw e else Nil } } protected def runFunc( queryName: String = "", iteration: Option[Int] = None, ignoreError: Boolean = true)(f: => Unit): Unit = synchronized { val iterationStr = iteration.map(i => s" - iteration $i").getOrElse("") var banner = s"$queryName$iterationStr" log("=" * 80) log(s"START: $banner") spark.sparkContext.setJobGroup(banner, banner, interruptOnCancel = true) try { val before = System.nanoTime() f val after = System.nanoTime() val durationMs = (after - before) / (1000 * 1000) queryResults += QueryResult(queryName, iteration, Some(durationMs), errorMsg = None) log(s"END took $durationMs ms: $banner") log("=" * 80) } catch { case NonFatal(e) => log(s"ERROR: $banner\n${e.getMessage}") queryResults += QueryResult(queryName, iteration, durationMs = None, errorMsg = Some(e.getMessage)) if (!ignoreError) throw e else spark.emptyDataFrame } } protected def reportExtraMetric(name: String, value: Double): Unit = synchronized { extraMetrics += (name -> value) } protected def getQueryResults(): Array[QueryResult] = synchronized { queryResults.toArray } private def generateJSONReport(report: BenchmarkReport): Unit = synchronized { import Benchmark._ val resultJson = toPrettyJson(report) val resultFileName = if (benchmarkId.trim.isEmpty) "report.json" else s"$benchmarkId-report.json" val reportLocalPath = Paths.get(resultFileName).toAbsolutePath() Files.write(reportLocalPath, resultJson.getBytes(StandardCharsets.UTF_8)) println(s"RESULT:\n$resultJson") uploadFile(reportLocalPath.toString, conf.jsonReportUploadPath) } private def generateCSVReport(): Unit = synchronized { val csvHeader = "name,iteration,durationMs" val csvRows = queryResults.map { r => s"${r.name},${r.iteration.getOrElse(1)},${r.durationMs.getOrElse(-1)}" } val csvText = (Seq(csvHeader) ++ csvRows).mkString("\n") val resultFileName = if (benchmarkId.trim.isEmpty) "report.csv" else s"$benchmarkId-report.csv" val reportLocalPath = Paths.get(resultFileName).toAbsolutePath() Files.write(reportLocalPath, csvText.getBytes(StandardCharsets.UTF_8)) uploadFile(reportLocalPath.toString, conf.csvReportUploadPath) } private def generateReport(): Unit = synchronized { val report = BenchmarkReport( benchmarkSpecs = conf.asMap + ("benchmarkId" -> benchmarkId), queryResults = queryResults.toArray, extraMetrics = extraMetrics.toMap, sparkEnvInfo = SparkUtils.getEnvironmentInfo(spark.sparkContext) ) generateJSONReport(report) generateCSVReport() } private def uploadFile(localPath: String, targetPath: String): Unit = { val targetUri = new URI(targetPath) val sanitizedTargetPath = targetUri.normalize().toString val scheme = new URI(targetPath).getScheme try { if (scheme.equals("s3")) s"aws s3 cp $localPath $sanitizedTargetPath/" ! else if (scheme.equals("gs")) s"gsutil cp $localPath $sanitizedTargetPath/" ! else throw new IllegalArgumentException(String.format("Unsupported scheme %s.", scheme)) println(s"FILE UPLOAD: Uploaded $localPath to $sanitizedTargetPath") } catch { case NonFatal(e) => log(s"FILE UPLOAD: Failed to upload $localPath to $sanitizedTargetPath: $e") } } protected def benchmarkId: String = sys.env.getOrElse("BENCHMARK_ID", spark.conf.getOption("spark.benchmarkId").getOrElse("")) protected def log(str: => String): Unit = { println(s"${java.time.LocalDateTime.now} $str") } } object Benchmark { private lazy val mapper = { val _mapper = new ObjectMapper with ScalaObjectMapper _mapper.setSerializationInclusion(Include.NON_ABSENT) _mapper.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY) _mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) _mapper.registerModule(DefaultScalaModule) _mapper } def toJson[T: Manifest](obj: T): String = { mapper.writeValueAsString(obj) } def toPrettyJson[T: Manifest](obj: T): String = { mapper.writerWithDefaultPrettyPrinter().writeValueAsString(obj) } } ================================================ FILE: benchmarks/src/main/scala/benchmark/MergeBenchmark.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package benchmark import java.util.UUID import org.apache.spark.SparkUtils import org.apache.spark.sql.Row trait MergeConf extends BenchmarkConf { def scaleInGB: Int def tableName: String = "web_returns" def userDefinedDbName: Option[String] def dbName: String = userDefinedDbName.getOrElse(s"merge_sf${scaleInGB}") def dbLocation: String = dbLocation(dbName) } case class MergeBenchmarkConf( scaleInGB: Int = 0, userDefinedDbName: Option[String] = None, iterations: Int = 3, benchmarkPath: Option[String] = None) extends MergeConf { } object MergeBenchmarkConf { import scopt.OParser private val builder = OParser.builder[MergeBenchmarkConf] private val argParser = { import builder._ OParser.sequence( programName("Merge Benchmark"), opt[String]("scale-in-gb") .required() .valueName("") .action((x, c) => c.copy(scaleInGB = x.toInt)) .text("Scale factor in GBs of the TPCDS benchmark"), opt[String]("benchmark-path") .required() .valueName("") .action((x, c) => c.copy(benchmarkPath = Some(x))) .text("Cloud path to be used for creating table and generating reports"), opt[String]("iterations") .optional() .valueName("") .action((x, c) => c.copy(iterations = x.toInt)) .text("Number of times to run the queries")) } def parse(args: Array[String]): Option[MergeBenchmarkConf] = { OParser.parse(argParser, args, MergeBenchmarkConf()) } } class MergeBenchmark(conf: MergeBenchmarkConf) extends Benchmark(conf) { /** * Runs every merge test case multiple times and records the duration. */ override def runInternal(): Unit = { for ((k, v) <- extraConfs) spark.conf.set(k, v) spark.sparkContext.setLogLevel("WARN") log("All configs:\n\t" + spark.conf.getAll.toSeq.sortBy(_._1).mkString("\n\t")) spark.sql(s"USE ${conf.dbName}") val targetRowCount = spark.read.table(s"`${conf.dbName}`.`target_${conf.tableName}`").count for (iteration <- 1 to conf.iterations) { MergeTestCases.testCases.foreach { runMerge(_, targetRowCount, iteration = Some(iteration)) } } val results = getQueryResults().filter(_.name.startsWith("q")) if (results.forall(x => x.errorMsg.isEmpty && x.durationMs.nonEmpty) ) { val medianDurationSecPerQuery = results.groupBy(_.name).map { case (q, results) => assert(results.length == conf.iterations) val medianMs = SparkUtils.median(results.map(_.durationMs.get), alreadySorted = false) (q, medianMs / 1000.0) } val sumOfMedians = medianDurationSecPerQuery.values.sum reportExtraMetric("merge-result-seconds", sumOfMedians) } } /** * Merge test runner performing the following steps: * - Clone a fresh target table. * - Run the merge test case. * - Check invariants. * - Drop the cloned table. */ protected def runMerge( testCase: MergeTestCase, targetRowCount: Long, iteration: Option[Int] = None, printRows: Boolean = false, ignoreError: Boolean = true): Seq[Row] = synchronized { withCloneTargetTable(testCase.name) { targetTable => val result = super.runQuery( testCase.sqlCmd(targetTable), testCase.name, iteration, printRows, ignoreError) testCase.validate(result, targetRowCount) result } } /** * Clones the target table before each test case to use a fresh target table and drops the clone * afterwards. */ protected def withCloneTargetTable[T](testCaseName: String)(f: String => T): T = { val target = s"`${conf.dbName}`.`target_${conf.tableName}`" val clonedTableName = s"`${conf.dbName}`.`${conf.tableName}_${generateShortUUID()}`" runQuery(s"CREATE TABLE $clonedTableName SHALLOW CLONE $target", s"clone-target-$testCaseName") try { f(clonedTableName) } finally { runQuery(s"DROP TABLE IF EXISTS $clonedTableName", s"drop-target-clone-$testCaseName") } } protected def generateShortUUID(): String = UUID.randomUUID.toString.replace("-", "_").take(8) } object MergeBenchmark { def main(args: Array[String]): Unit = { MergeBenchmarkConf.parse(args).foreach { conf => new MergeBenchmark(conf).run() } } } ================================================ FILE: benchmarks/src/main/scala/benchmark/MergeDataLoad.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package benchmark import java.util.Locale import org.apache.spark.sql.functions.{col, countDistinct, hash, isnull, max, rand} case class MergeDataLoadConf( scaleInGB: Int = 0, userDefinedDbName: Option[String] = None, loadFromPath: Option[String] = None, benchmarkPath: Option[String] = None, excludeNulls: Boolean = true) extends MergeConf { } /** * Represents a table configuration used as a source in merge test cases. Each [[MergeTestCase]] has * one [[MergeSourceTable]] associated with it, the data loader will collect all source table * configurations for all tests and create the required source tables. * @param filesMatchedFraction Fraction of files from the base table that will get sampled to * create the source table. * @param rowsMatchedFraction Fraction of rows from the selected files that will get sampled to form * the part of the source table that matches the merge condition. * @param rowsNotMatchedFraction Fraction of rows from the selected files that will get sampled to * form the part of the source table that doesn't match the merge * condition. */ case class MergeSourceTable( filesMatchedFraction: Double, rowsMatchedFraction: Double, rowsNotMatchedFraction: Double) { def name: String = formatTableName(s"source_" + s"_filesMatchedFraction_$filesMatchedFraction" + s"_rowsMatchedFraction_$rowsMatchedFraction" + s"_rowsNotMatchedFraction_$rowsNotMatchedFraction") protected def formatTableName(s: String): String = { s.toLowerCase(Locale.ROOT).replaceAll("\\s+", "_").replaceAll("[-,.]", "_") } } object MergeDataLoadConf { import scopt.OParser private val builder = OParser.builder[MergeDataLoadConf] private val argParser = { import builder._ OParser.sequence( programName("Merge Data Load"), opt[String]("scale-in-gb") .required() .valueName("") .action((x, c) => c.copy(scaleInGB = x.toInt)) .text("Scale factor of the Merge benchmark"), opt[String]("benchmark-path") .required() .valueName("") .action((x, c) => c.copy(benchmarkPath = Some(x))) .text("Cloud storage path to be used for creating table and generating reports"), opt[String]("db-name") .optional() .valueName("") .action((x, c) => c.copy(userDefinedDbName = Some(x))) .text("Name of the target database to create with TPC-DS tables in necessary format"), opt[String]("load-from-path") .optional() .valueName("") .action((x, c) => c.copy(loadFromPath = Some(x))) .text("The location of the TPC-DS raw input data"), opt[String]("exclude-nulls") .optional() .valueName("true/false") .action((x, c) => c.copy(excludeNulls = x.toBoolean)) .text("Whether to remove null primary keys when loading data, default = false")) } def parse(args: Array[String]): Option[MergeDataLoadConf] = { OParser.parse(argParser, args, MergeDataLoadConf()) } } class MergeDataLoad(conf: MergeDataLoadConf) extends Benchmark(conf) { protected def targetTableFullName = s"`${conf.dbName}`.`target_${conf.tableName}`" protected def dataLoadFromPath: String = conf.loadFromPath.getOrElse { s"s3://devrel-delta-datasets/tpcds-2.13/tpcds_sf${conf.scaleInGB}_parquet/${conf.tableName}/" } /** * Creates the target table and all source table configuration used in merge test cases. */ def runInternal(): Unit = { val dbName = conf.dbName val dbLocation = conf.dbLocation(dbName, suffix = benchmarkId.replace("-", "_")) val dbCatalog = "spark_catalog" require(Seq(1, 3000).contains(conf.scaleInGB), "") log(s"====== Creating database =======") runQuery(s"DROP DATABASE IF EXISTS ${dbName} CASCADE", s"drop-database") runQuery(s"CREATE DATABASE IF NOT EXISTS ${dbName}", s"create-database") log(s"====== Creating merge target table =======") loadMergeTargetTable() log(s"====== Creating merge source tables =======") MergeTestCases.testCases.map(_.sourceTable).distinct.foreach(loadMergeSourceTable) log(s"====== Created all tables in database ${dbName} at '${dbLocation}' =======") runQuery(s"USE $dbCatalog.$dbName;") runQuery("SHOW TABLES", printRows = true) } /** * Creates the target Delta table and performs sanity checks. This table will be cloned before * each merge test case and the clone serves as a single-use merge target table. */ protected def loadMergeTargetTable(): Unit = { val dbLocation = conf.dbLocation(conf.dbName, suffix = benchmarkId.replace("-", "_")) val location = s"${dbLocation}/${conf.tableName}/" val format = "parquet" runQuery(s"DROP TABLE IF EXISTS $targetTableFullName", s"drop-table-$targetTableFullName") runQuery( s"""CREATE TABLE $targetTableFullName USING DELTA LOCATION '$location' SELECT * FROM `${format}`.`$dataLoadFromPath` """, s"create-table-$targetTableFullName", ignoreError = true) val sourceRowCount = spark.sql(s"SELECT * FROM `${format}`.`$dataLoadFromPath`").count() val targetRowCount = spark.table(targetTableFullName).count() val targetFileCount = spark.table(targetTableFullName).select(countDistinct("_metadata.file_path")) log(s"Target file count: $targetFileCount") log(s"Target row count: $targetRowCount") assert(targetRowCount == sourceRowCount, s"Row count mismatch: source table = $sourceRowCount, " + s"target $targetTableFullName = $targetRowCount") } /** * Creates a table that will be used as a merge source table in the merge test cases. The table is * created by sampling the merge target table created by [[loadMergeTargetTable]]. The merge test * cases don't modify the source table and a single source table is reused across different test * cases if the same source table configuration is used. */ protected def loadMergeSourceTable(sourceTableConf: MergeSourceTable): Unit = { val fullTableName = s"`${conf.dbName}`.`${sourceTableConf.name}`" val dbLocation = conf.dbLocation(conf.dbName, suffix = benchmarkId.replace("-", "_")) runQuery(s"DROP TABLE IF EXISTS $fullTableName", s"drop-table-${sourceTableConf.name}") val fullTableDF = spark.read.format("delta") .load(s"${dbLocation}/${conf.tableName}/") // Sample files based on their file path. val sampledFilesDF = fullTableDF .select("_metadata.file_path") .distinct .sample(sourceTableConf.filesMatchedFraction) // Read the data from the sampled files and sample two sets of rows for MATCHED clauses and // NOT MATCHED clauses respectively. val sampledDataDF = fullTableDF .withColumn("file_path", col("_metadata.file_path")) .join(sampledFilesDF, "file_path") log(s"Matching files row count: ${sampledDataDF.count}") val numberOfNulls = sampledDataDF.filter(isnull(col("wr_order_number"))).count log(s"wr_order_number contains $numberOfNulls null values") val matchedData = sampledDataDF.sample(sourceTableConf.rowsMatchedFraction) val notMatchedData = sampledDataDF.sample(sourceTableConf.rowsNotMatchedFraction) .withColumn("wr_order_number", rand()) .withColumn("wr_item_sk", rand()) val data = matchedData.union(notMatchedData) val dupes = data.groupBy("wr_order_number", "wr_item_sk").count.filter("count > 1") log(s"Duplicates: ${dupes.collect().mkString("Array(", ",\n", ")")}") data.write.format("delta").saveAsTable(fullTableName) } } object MergeDataLoad { def main(args: Array[String]): Unit = { MergeDataLoadConf.parse(args).foreach { conf => new MergeDataLoad(conf).run() } } } ================================================ FILE: benchmarks/src/main/scala/benchmark/MergeTestCases.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package benchmark import org.apache.spark.sql.Row trait MergeTestCase { /** * Name of the test case used e.p. in the test results. */ def name: String /** * The source table configuration to use for the test case. When a test case is defined, * [[MergeDataLoad]] will collect all source table configuration and create the source tables * required by all tests. */ def sourceTable: MergeSourceTable /** * The merge command to execute as a SQL string. */ def sqlCmd(targetTable: String): String /** * Each test case can define invariants to check after the merge command runs to ensure that the * benchmark results are valid. */ def validate(mergeStats: Seq[Row], targetRowCount: Long): Unit } /** * Trait shared by all insert-only merge test cases. */ trait InsertOnlyTestCase extends MergeTestCase { val filesMatchedFraction: Double val rowsNotMatchedFraction: Double override def sourceTable: MergeSourceTable = MergeSourceTable( filesMatchedFraction, rowsMatchedFraction = 0, rowsNotMatchedFraction) override def validate(mergeStats: Seq[Row], targetRowCount: Long): Unit = { assert(mergeStats.length == 1) assert(mergeStats.head.getAs[Long]("num_updated_rows") == 0) assert(mergeStats.head.getAs[Long]("num_deleted_rows") == 0) } } /** * A merge test case with a single WHEN NOT MATCHED THEN INSERT * clause. */ case class SingleInsertOnlyTestCase( filesMatchedFraction: Double, rowsNotMatchedFraction: Double) extends InsertOnlyTestCase { override val name: String = "single_insert_only" + s"_filesMatchedFraction_$filesMatchedFraction" + s"_rowsNotMatchedFraction_$rowsNotMatchedFraction" override def sqlCmd(targetTable: String): String = { s"""MERGE INTO $targetTable t |USING ${sourceTable.name} s |ON t.wr_order_number = s.wr_order_number AND t.wr_item_sk = s.wr_item_sk |WHEN NOT MATCHED THEN INSERT *""".stripMargin } } /** * A merge test case with two WHEN NOT MATCHED (AND condition) THEN INSERT * clauses. */ case class MultipleInsertOnlyTestCase( filesMatchedFraction: Double, rowsNotMatchedFraction: Double) extends InsertOnlyTestCase { override val name: String = "multiple_insert_only" + s"_filesMatchedFraction_$filesMatchedFraction" + s"_rowsNotMatchedFraction_$rowsNotMatchedFraction" override def sqlCmd(targetTable: String): String = { s"""MERGE INTO $targetTable t |USING ${sourceTable.name} s |ON t.wr_order_number = s.wr_order_number AND t.wr_item_sk = s.wr_item_sk |WHEN NOT MATCHED AND s.wr_item_sk % 2 = 0 THEN INSERT * |WHEN NOT MATCHED THEN INSERT *""".stripMargin } } /** * A merge test case with a single WHEN MATCHED THEN DELETED clause. */ case class DeleteOnlyTestCase( filesMatchedFraction: Double, rowsMatchedFraction: Double) extends MergeTestCase { override val name: String = "delete_only" + s"_filesMatchedFraction_$filesMatchedFraction" + s"_rowsMatchedFraction_$rowsMatchedFraction" override def sourceTable: MergeSourceTable = MergeSourceTable( filesMatchedFraction, rowsMatchedFraction, rowsNotMatchedFraction = 0) override def sqlCmd(targetTable: String): String = { s"""MERGE INTO $targetTable t |USING ${sourceTable.name} s |ON t.wr_order_number = s.wr_order_number AND t.wr_item_sk = s.wr_item_sk |WHEN MATCHED THEN DELETE""".stripMargin } override def validate(mergeStats: Seq[Row], targetRowCount: Long): Unit = { assert(mergeStats.length == 1) assert(mergeStats.head.getAs[Long]("num_updated_rows") == 0) assert(mergeStats.head.getAs[Long]("num_inserted_rows") == 0) } } /** * A merge test case with a MATCHED UPDATE and a NOT MATCHED INSERT clause. */ case class UpsertTestCase( filesMatchedFraction: Double, rowsMatchedFraction: Double, rowsNotMatchedFraction: Double) extends MergeTestCase { override val name: String = "upsert" + s"_filesMatchedFraction_$filesMatchedFraction" + s"_rowsMatchedFraction_$rowsMatchedFraction" + s"_rowsNotMatchedFraction_$rowsNotMatchedFraction" override def sourceTable: MergeSourceTable = MergeSourceTable( filesMatchedFraction, rowsMatchedFraction, rowsNotMatchedFraction) override def sqlCmd(targetTable: String): String = { s"""MERGE INTO $targetTable t |USING ${sourceTable.name} s |ON t.wr_order_number = s.wr_order_number AND t.wr_item_sk = s.wr_item_sk |WHEN MATCHED THEN UPDATE SET * |WHEN NOT MATCHED THEN INSERT *""".stripMargin } override def validate(mergeStats: Seq[Row], targetRowCount: Long): Unit = { assert(mergeStats.length == 1) assert(mergeStats.head.getAs[Long]("num_deleted_rows") == 0) } } object MergeTestCases { def testCases: Seq[MergeTestCase] = insertOnlyTestCases ++ deleteOnlyTestCases ++ upsertTestCases def insertOnlyTestCases: Seq[MergeTestCase] = Seq(0.05, 0.5, 1.0).flatMap { rowsNotMatchedFraction => Seq( SingleInsertOnlyTestCase( filesMatchedFraction = 0.05, rowsNotMatchedFraction), MultipleInsertOnlyTestCase( filesMatchedFraction = 0.05, rowsNotMatchedFraction) ) } def deleteOnlyTestCases: Seq[MergeTestCase] = Seq( DeleteOnlyTestCase( filesMatchedFraction = 0.05, rowsMatchedFraction = 0.05)) def upsertTestCases: Seq[MergeTestCase] = Seq( Seq(0.0, 0.01, 0.1).map { rowsMatchedFraction => UpsertTestCase( filesMatchedFraction = 0.05, rowsMatchedFraction, rowsNotMatchedFraction = 0.1) }, Seq(0.5, 0.99, 1.0).map { rowsMatchedFraction => UpsertTestCase( filesMatchedFraction = 0.05, rowsMatchedFraction, rowsNotMatchedFraction = 0.001) }, Seq( UpsertTestCase( filesMatchedFraction = 0.05, rowsMatchedFraction = 0.1, rowsNotMatchedFraction = 0.0), UpsertTestCase( filesMatchedFraction = 0.5, rowsMatchedFraction = 0.01, rowsNotMatchedFraction = 0.001), UpsertTestCase( filesMatchedFraction = 1.0, rowsMatchedFraction = 0.01, rowsNotMatchedFraction = 0.001) ) ).flatten } ================================================ FILE: benchmarks/src/main/scala/benchmark/TPCDSBenchmark.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package benchmark import benchmark.TPCDSBenchmarkQueries._ trait TPCDSConf extends BenchmarkConf { protected def format: Option[String] def scaleInGB: Int def userDefinedDbName: Option[String] def formatName: String = format.getOrElse { throw new IllegalArgumentException("format must be specified") } def dbName: String = userDefinedDbName.getOrElse(s"tpcds_sf${scaleInGB}_${formatName}") def dbLocation: String = dbLocation(dbName) } case class TPCDSBenchmarkConf( protected val format: Option[String] = None, scaleInGB: Int = 0, userDefinedDbName: Option[String] = None, iterations: Int = 3, benchmarkPath: Option[String] = None) extends TPCDSConf object TPCDSBenchmarkConf { import scopt.OParser private val builder = OParser.builder[TPCDSBenchmarkConf] private val argParser = { import builder._ OParser.sequence( programName("TPC-DS Benchmark"), opt[String]("format") .required() .action((x, c) => c.copy(format = Some(x))) .text("Spark's short name for the file format to use"), opt[String]("scale-in-gb") .required() .valueName("") .action((x, c) => c.copy(scaleInGB = x.toInt)) .text("Scale factor of the TPCDS benchmark"), opt[String]("benchmark-path") .required() .valueName("") .action((x, c) => c.copy(benchmarkPath = Some(x))) .text("Cloud path to be used for creating table and generating reports"), opt[String]("iterations") .optional() .valueName("") .action((x, c) => c.copy(iterations = x.toInt)) .text("Number of times to run the queries"), ) } def parse(args: Array[String]): Option[TPCDSBenchmarkConf] = { OParser.parse(argParser, args, TPCDSBenchmarkConf()) } } class TPCDSBenchmark(conf: TPCDSBenchmarkConf) extends Benchmark(conf) { val queries: Map[String, String] = { if (conf.scaleInGB <= 3000) TPCDSQueries3TB else if (conf.scaleInGB == 10) TPCDSQueries10TB else throw new IllegalArgumentException( s"Unsupported scale factor of ${conf.scaleInGB} GB") } val dbName = conf.dbName def runInternal(): Unit = { for ((k, v) <- extraConfs) spark.conf.set(k, v) spark.sparkContext.setLogLevel("WARN") log("All configs:\n\t" + spark.conf.getAll.toSeq.sortBy(_._1).mkString("\n\t")) spark.sql(s"USE $dbName") for (iteration <- 1 to conf.iterations) { queries.toSeq.sortBy(_._1).foreach { case (name, sql) => runQuery(sql, iteration = Some(iteration), queryName = name) } } val results = getQueryResults().filter(_.name.startsWith("q")) if (results.forall(x => x.errorMsg.isEmpty && x.durationMs.nonEmpty) ) { val medianDurationSecPerQuery = results.groupBy(_.name).map { case (q, results) => assert(results.size == conf.iterations) val medianMs = results.map(_.durationMs.get).sorted .drop(math.floor(conf.iterations / 2.0).toInt).head (q, medianMs / 1000.0) } val sumOfMedians = medianDurationSecPerQuery.map(_._2).sum reportExtraMetric("tpcds-result-seconds", sumOfMedians) } } } object TPCDSBenchmark { def main(args: Array[String]): Unit = { TPCDSBenchmarkConf.parse(args).foreach { conf => new TPCDSBenchmark(conf).run() } } } ================================================ FILE: benchmarks/src/main/scala/benchmark/TPCDSBenchmarkQueries.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package benchmark object TPCDSBenchmarkQueries { val TPCDSQueries3TB = Map( "q1" -> """ with customer_total_return as (select sr_customer_sk as ctr_customer_sk ,sr_store_sk as ctr_store_sk ,sum(SR_FEE) as ctr_total_return from store_returns ,date_dim where sr_returned_date_sk = d_date_sk and d_year =2000 group by sr_customer_sk ,sr_store_sk) select c_customer_id from customer_total_return ctr1 ,store ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_store_sk = ctr2.ctr_store_sk) and s_store_sk = ctr1.ctr_store_sk and s_state = 'TN' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id limit 100""", "q2" -> """ with wscs as (select sold_date_sk ,sales_price from (select ws_sold_date_sk sold_date_sk ,ws_ext_sales_price sales_price from web_sales union all select cs_sold_date_sk sold_date_sk ,cs_ext_sales_price sales_price from catalog_sales)), wswscs as (select d_week_seq, sum(case when (d_day_name='Sunday') then sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then sales_price else null end) sat_sales from wscs ,date_dim where d_date_sk = sold_date_sk group by d_week_seq) select d_week_seq1 ,round(sun_sales1/sun_sales2,2) ,round(mon_sales1/mon_sales2,2) ,round(tue_sales1/tue_sales2,2) ,round(wed_sales1/wed_sales2,2) ,round(thu_sales1/thu_sales2,2) ,round(fri_sales1/fri_sales2,2) ,round(sat_sales1/sat_sales2,2) from (select wswscs.d_week_seq d_week_seq1 ,sun_sales sun_sales1 ,mon_sales mon_sales1 ,tue_sales tue_sales1 ,wed_sales wed_sales1 ,thu_sales thu_sales1 ,fri_sales fri_sales1 ,sat_sales sat_sales1 from wswscs,date_dim where date_dim.d_week_seq = wswscs.d_week_seq and d_year = 1998) y, (select wswscs.d_week_seq d_week_seq2 ,sun_sales sun_sales2 ,mon_sales mon_sales2 ,tue_sales tue_sales2 ,wed_sales wed_sales2 ,thu_sales thu_sales2 ,fri_sales fri_sales2 ,sat_sales sat_sales2 from wswscs ,date_dim where date_dim.d_week_seq = wswscs.d_week_seq and d_year = 1998+1) z where d_week_seq1=d_week_seq2-53 order by d_week_seq1""", "q3" -> """ select dt.d_year ,item.i_brand_id brand_id ,item.i_brand brand ,sum(ss_sales_price) sum_agg from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manufact_id = 816 and dt.d_moy=11 group by dt.d_year ,item.i_brand ,item.i_brand_id order by dt.d_year ,sum_agg desc ,brand_id limit 100""", "q4" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum((((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2) ) year_total ,'c' sale_type from customer ,catalog_sales ,date_dim where c_customer_sk = cs_bill_customer_sk and cs_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum((((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2) ) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year ) select t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_birth_country from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_c_firstyear ,year_total t_c_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_c_secyear.customer_id and t_s_firstyear.customer_id = t_c_firstyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.sale_type = 's' and t_c_firstyear.sale_type = 'c' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_c_secyear.sale_type = 'c' and t_w_secyear.sale_type = 'w' and t_s_firstyear.dyear = 1999 and t_s_secyear.dyear = 1999+1 and t_c_firstyear.dyear = 1999 and t_c_secyear.dyear = 1999+1 and t_w_firstyear.dyear = 1999 and t_w_secyear.dyear = 1999+1 and t_s_firstyear.year_total > 0 and t_c_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end > case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end order by t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_birth_country limit 100""", "q5" -> """ with ssr as (select s_store_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select ss_store_sk as store_sk, ss_sold_date_sk as date_sk, ss_ext_sales_price as sales_price, ss_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from store_sales union all select sr_store_sk as store_sk, sr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, sr_return_amt as return_amt, sr_net_loss as net_loss from store_returns ) salesreturns, date_dim, store where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and store_sk = s_store_sk group by s_store_id) , csr as (select cp_catalog_page_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select cs_catalog_page_sk as page_sk, cs_sold_date_sk as date_sk, cs_ext_sales_price as sales_price, cs_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from catalog_sales union all select cr_catalog_page_sk as page_sk, cr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, cr_return_amount as return_amt, cr_net_loss as net_loss from catalog_returns ) salesreturns, date_dim, catalog_page where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and page_sk = cp_catalog_page_sk group by cp_catalog_page_id) , wsr as (select web_site_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select ws_web_site_sk as wsr_web_site_sk, ws_sold_date_sk as date_sk, ws_ext_sales_price as sales_price, ws_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from web_sales union all select ws_web_site_sk as wsr_web_site_sk, wr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, wr_return_amt as return_amt, wr_net_loss as net_loss from web_returns left outer join web_sales on ( wr_item_sk = ws_item_sk and wr_order_number = ws_order_number) ) salesreturns, date_dim, web_site where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and wsr_web_site_sk = web_site_sk group by web_site_id) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , 'store' || s_store_id as id , sales , returns , (profit - profit_loss) as profit from ssr union all select 'catalog channel' as channel , 'catalog_page' || cp_catalog_page_id as id , sales , returns , (profit - profit_loss) as profit from csr union all select 'web channel' as channel , 'web_site' || web_site_id as id , sales , returns , (profit - profit_loss) as profit from wsr ) x group by rollup (channel, id) order by channel ,id limit 100""", "q6" -> """ select a.ca_state state, count(*) cnt from customer_address a ,customer c ,store_sales s ,date_dim d ,item i where a.ca_address_sk = c.c_current_addr_sk and c.c_customer_sk = s.ss_customer_sk and s.ss_sold_date_sk = d.d_date_sk and s.ss_item_sk = i.i_item_sk and d.d_month_seq = (select distinct (d_month_seq) from date_dim where d_year = 2002 and d_moy = 3 ) and i.i_current_price > 1.2 * (select avg(j.i_current_price) from item j where j.i_category = i.i_category) group by a.ca_state having count(*) >= 10 order by cnt, a.ca_state limit 100""", "q7" -> """ select i_item_id, avg(ss_quantity) agg1, avg(ss_list_price) agg2, avg(ss_coupon_amt) agg3, avg(ss_sales_price) agg4 from store_sales, customer_demographics, date_dim, item, promotion where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and ss_cdemo_sk = cd_demo_sk and ss_promo_sk = p_promo_sk and cd_gender = 'F' and cd_marital_status = 'W' and cd_education_status = 'College' and (p_channel_email = 'N' or p_channel_event = 'N') and d_year = 2001 group by i_item_id order by i_item_id limit 100""", "q8" -> """ select s_store_name ,sum(ss_net_profit) from store_sales ,date_dim ,store, (select ca_zip from ( SELECT substr(ca_zip,1,5) ca_zip FROM customer_address WHERE substr(ca_zip,1,5) IN ( '47602','16704','35863','28577','83910','36201', '58412','48162','28055','41419','80332', '38607','77817','24891','16226','18410', '21231','59345','13918','51089','20317', '17167','54585','67881','78366','47770', '18360','51717','73108','14440','21800', '89338','45859','65501','34948','25973', '73219','25333','17291','10374','18829', '60736','82620','41351','52094','19326', '25214','54207','40936','21814','79077', '25178','75742','77454','30621','89193', '27369','41232','48567','83041','71948', '37119','68341','14073','16891','62878', '49130','19833','24286','27700','40979', '50412','81504','94835','84844','71954', '39503','57649','18434','24987','12350', '86379','27413','44529','98569','16515', '27287','24255','21094','16005','56436', '91110','68293','56455','54558','10298', '83647','32754','27052','51766','19444', '13869','45645','94791','57631','20712', '37788','41807','46507','21727','71836', '81070','50632','88086','63991','20244', '31655','51782','29818','63792','68605', '94898','36430','57025','20601','82080', '33869','22728','35834','29086','92645', '98584','98072','11652','78093','57553', '43830','71144','53565','18700','90209', '71256','38353','54364','28571','96560', '57839','56355','50679','45266','84680', '34306','34972','48530','30106','15371', '92380','84247','92292','68852','13338', '34594','82602','70073','98069','85066', '47289','11686','98862','26217','47529', '63294','51793','35926','24227','14196', '24594','32489','99060','49472','43432', '49211','14312','88137','47369','56877', '20534','81755','15794','12318','21060', '73134','41255','63073','81003','73873', '66057','51184','51195','45676','92696', '70450','90669','98338','25264','38919', '59226','58581','60298','17895','19489', '52301','80846','95464','68770','51634', '19988','18367','18421','11618','67975', '25494','41352','95430','15734','62585', '97173','33773','10425','75675','53535', '17879','41967','12197','67998','79658', '59130','72592','14851','43933','68101', '50636','25717','71286','24660','58058', '72991','95042','15543','33122','69280', '11912','59386','27642','65177','17672', '33467','64592','36335','54010','18767', '63193','42361','49254','33113','33159', '36479','59080','11855','81963','31016', '49140','29392','41836','32958','53163', '13844','73146','23952','65148','93498', '14530','46131','58454','13376','13378', '83986','12320','17193','59852','46081', '98533','52389','13086','68843','31013', '13261','60560','13443','45533','83583', '11489','58218','19753','22911','25115', '86709','27156','32669','13123','51933', '39214','41331','66943','14155','69998', '49101','70070','35076','14242','73021', '59494','15782','29752','37914','74686', '83086','34473','15751','81084','49230', '91894','60624','17819','28810','63180', '56224','39459','55233','75752','43639', '55349','86057','62361','50788','31830', '58062','18218','85761','60083','45484', '21204','90229','70041','41162','35390', '16364','39500','68908','26689','52868', '81335','40146','11340','61527','61794', '71997','30415','59004','29450','58117', '69952','33562','83833','27385','61860', '96435','48333','23065','32961','84919', '61997','99132','22815','56600','68730', '48017','95694','32919','88217','27116', '28239','58032','18884','16791','21343', '97462','18569','75660','15475') intersect select ca_zip from (SELECT substr(ca_zip,1,5) ca_zip,count(*) cnt FROM customer_address, customer WHERE ca_address_sk = c_current_addr_sk and c_preferred_cust_flag='Y' group by ca_zip having count(*) > 10)A1)A2) V1 where ss_store_sk = s_store_sk and ss_sold_date_sk = d_date_sk and d_qoy = 2 and d_year = 1998 and (substr(s_zip,1,2) = substr(V1.ca_zip,1,2)) group by s_store_name order by s_store_name limit 100""", "q9" -> """ select case when (select count(*) from store_sales where ss_quantity between 1 and 20) > 2972190 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 1 and 20) else (select avg(ss_net_profit) from store_sales where ss_quantity between 1 and 20) end bucket1 , case when (select count(*) from store_sales where ss_quantity between 21 and 40) > 111711138 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 21 and 40) else (select avg(ss_net_profit) from store_sales where ss_quantity between 21 and 40) end bucket2, case when (select count(*) from store_sales where ss_quantity between 41 and 60) > 127958920 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 41 and 60) else (select avg(ss_net_profit) from store_sales where ss_quantity between 41 and 60) end bucket3, case when (select count(*) from store_sales where ss_quantity between 61 and 80) > 41162107 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 61 and 80) else (select avg(ss_net_profit) from store_sales where ss_quantity between 61 and 80) end bucket4, case when (select count(*) from store_sales where ss_quantity between 81 and 100) > 25211875 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 81 and 100) else (select avg(ss_net_profit) from store_sales where ss_quantity between 81 and 100) end bucket5 from reason where r_reason_sk = 1""", "q10" -> """ select cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3, cd_dep_count, count(*) cnt4, cd_dep_employed_count, count(*) cnt5, cd_dep_college_count, count(*) cnt6 from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and ca_county in ('Allen County','Jefferson County','Lamar County','Dakota County','Park County') and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2001 and d_moy between 4 and 4+3) and (exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2001 and d_moy between 4 ANd 4+3) or exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2001 and d_moy between 4 and 4+3)) group by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count order by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count limit 100""", "q11" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(ss_ext_list_price-ss_ext_discount_amt) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(ws_ext_list_price-ws_ext_discount_amt) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year ) select t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_login from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.sale_type = 's' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_w_secyear.sale_type = 'w' and t_s_firstyear.dyear = 1998 and t_s_secyear.dyear = 1998+1 and t_w_firstyear.dyear = 1998 and t_w_secyear.dyear = 1998+1 and t_s_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else 0.0 end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else 0.0 end order by t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_login limit 100""", "q12" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(ws_ext_sales_price) as itemrevenue ,sum(ws_ext_sales_price)*100/sum(sum(ws_ext_sales_price)) over (partition by i_class) as revenueratio from web_sales ,item ,date_dim where ws_item_sk = i_item_sk and i_category in ('Men', 'Books', 'Children') and ws_sold_date_sk = d_date_sk and d_date between cast('1998-03-28' as date) and (cast('1998-03-28' as date) + INTERVAL 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio limit 100""", "q13" -> """ select avg(ss_quantity) ,avg(ss_ext_sales_price) ,avg(ss_ext_wholesale_cost) ,sum(ss_ext_wholesale_cost) from store_sales ,store ,customer_demographics ,household_demographics ,customer_address ,date_dim where s_store_sk = ss_store_sk and ss_sold_date_sk = d_date_sk and d_year = 2001 and((ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'U' and cd_education_status = 'Unknown' and ss_sales_price between 100.00 and 150.00 and hd_dep_count = 3 )or (ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'W' and cd_education_status = '2 yr Degree' and ss_sales_price between 50.00 and 100.00 and hd_dep_count = 1 ) or (ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'S' and cd_education_status = 'College' and ss_sales_price between 150.00 and 200.00 and hd_dep_count = 1 )) and((ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('WV', 'GA', 'TX') and ss_net_profit between 100 and 200 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('TN', 'KY', 'SC') and ss_net_profit between 150 and 300 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('OK', 'NE', 'CA') and ss_net_profit between 50 and 250 ))""", "q14a" -> """ with cross_items as (select i_item_sk ss_item_sk from item, (select iss.i_brand_id brand_id ,iss.i_class_id class_id ,iss.i_category_id category_id from store_sales ,item iss ,date_dim d1 where ss_item_sk = iss.i_item_sk and ss_sold_date_sk = d1.d_date_sk and d1.d_year between 1998 AND 1998 + 2 intersect select ics.i_brand_id ,ics.i_class_id ,ics.i_category_id from catalog_sales ,item ics ,date_dim d2 where cs_item_sk = ics.i_item_sk and cs_sold_date_sk = d2.d_date_sk and d2.d_year between 1998 AND 1998 + 2 intersect select iws.i_brand_id ,iws.i_class_id ,iws.i_category_id from web_sales ,item iws ,date_dim d3 where ws_item_sk = iws.i_item_sk and ws_sold_date_sk = d3.d_date_sk and d3.d_year between 1998 AND 1998 + 2) where i_brand_id = brand_id and i_class_id = class_id and i_category_id = category_id ), avg_sales as (select avg(quantity*list_price) average_sales from (select ss_quantity quantity ,ss_list_price list_price from store_sales ,date_dim where ss_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2 union all select cs_quantity quantity ,cs_list_price list_price from catalog_sales ,date_dim where cs_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2 union all select ws_quantity quantity ,ws_list_price list_price from web_sales ,date_dim where ws_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2) x) select channel, i_brand_id,i_class_id,i_category_id,sum(sales), sum(number_sales) from( select 'store' channel, i_brand_id,i_class_id ,i_category_id,sum(ss_quantity*ss_list_price) sales , count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1998+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales) union all select 'catalog' channel, i_brand_id,i_class_id,i_category_id, sum(cs_quantity*cs_list_price) sales, count(*) number_sales from catalog_sales ,item ,date_dim where cs_item_sk in (select ss_item_sk from cross_items) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1998+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(cs_quantity*cs_list_price) > (select average_sales from avg_sales) union all select 'web' channel, i_brand_id,i_class_id,i_category_id, sum(ws_quantity*ws_list_price) sales , count(*) number_sales from web_sales ,item ,date_dim where ws_item_sk in (select ss_item_sk from cross_items) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1998+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(ws_quantity*ws_list_price) > (select average_sales from avg_sales) ) y group by rollup (channel, i_brand_id,i_class_id,i_category_id) order by channel,i_brand_id,i_class_id,i_category_id limit 100""", "q14b" -> """ with cross_items as (select i_item_sk ss_item_sk from item, (select iss.i_brand_id brand_id ,iss.i_class_id class_id ,iss.i_category_id category_id from store_sales ,item iss ,date_dim d1 where ss_item_sk = iss.i_item_sk and ss_sold_date_sk = d1.d_date_sk and d1.d_year between 1998 AND 1998 + 2 intersect select ics.i_brand_id ,ics.i_class_id ,ics.i_category_id from catalog_sales ,item ics ,date_dim d2 where cs_item_sk = ics.i_item_sk and cs_sold_date_sk = d2.d_date_sk and d2.d_year between 1998 AND 1998 + 2 intersect select iws.i_brand_id ,iws.i_class_id ,iws.i_category_id from web_sales ,item iws ,date_dim d3 where ws_item_sk = iws.i_item_sk and ws_sold_date_sk = d3.d_date_sk and d3.d_year between 1998 AND 1998 + 2) x where i_brand_id = brand_id and i_class_id = class_id and i_category_id = category_id ), avg_sales as (select avg(quantity*list_price) average_sales from (select ss_quantity quantity ,ss_list_price list_price from store_sales ,date_dim where ss_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2 union all select cs_quantity quantity ,cs_list_price list_price from catalog_sales ,date_dim where cs_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2 union all select ws_quantity quantity ,ws_list_price list_price from web_sales ,date_dim where ws_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2) x) select this_year.channel ty_channel ,this_year.i_brand_id ty_brand ,this_year.i_class_id ty_class ,this_year.i_category_id ty_category ,this_year.sales ty_sales ,this_year.number_sales ty_number_sales ,last_year.channel ly_channel ,last_year.i_brand_id ly_brand ,last_year.i_class_id ly_class ,last_year.i_category_id ly_category ,last_year.sales ly_sales ,last_year.number_sales ly_number_sales from (select 'store' channel, i_brand_id,i_class_id,i_category_id ,sum(ss_quantity*ss_list_price) sales, count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_week_seq = (select d_week_seq from date_dim where d_year = 1998 + 1 and d_moy = 12 and d_dom = 20) group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) this_year, (select 'store' channel, i_brand_id,i_class_id ,i_category_id, sum(ss_quantity*ss_list_price) sales, count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_week_seq = (select d_week_seq from date_dim where d_year = 1998 and d_moy = 12 and d_dom = 20) group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) last_year where this_year.i_brand_id= last_year.i_brand_id and this_year.i_class_id = last_year.i_class_id and this_year.i_category_id = last_year.i_category_id order by this_year.channel, this_year.i_brand_id, this_year.i_class_id, this_year.i_category_id limit 100""", "q15" -> """ select ca_zip ,sum(cs_sales_price) from catalog_sales ,customer ,customer_address ,date_dim where cs_bill_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') or ca_state in ('CA','WA','GA') or cs_sales_price > 500) and cs_sold_date_sk = d_date_sk and d_qoy = 1 and d_year = 2000 group by ca_zip order by ca_zip limit 100""", "q16" -> """ select count(distinct cs_order_number) as `order count` ,sum(cs_ext_ship_cost) as `total shipping cost` ,sum(cs_net_profit) as `total net profit` from catalog_sales cs1 ,date_dim ,customer_address ,call_center where d_date between '2001-2-01' and (cast('2001-2-01' as date) + INTERVAL 60 days) and cs1.cs_ship_date_sk = d_date_skq and cs1.cs_ship_addr_sk = ca_address_sk and ca_state = 'MS' and cs1.cs_call_center_sk = cc_call_center_sk and cc_county in ('Jackson County','Daviess County','Walker County','Dauphin County', 'Mobile County' ) and exists (select * from catalog_sales cs2 where cs1.cs_order_number = cs2.cs_order_number and cs1.cs_warehouse_sk <> cs2.cs_warehouse_sk) and not exists(select * from catalog_returns cr1 where cs1.cs_order_number = cr1.cr_order_number) order by count(distinct cs_order_number) limit 100""", "q17" -> """ select i_item_id ,i_item_desc ,s_state ,count(ss_quantity) as store_sales_quantitycount ,avg(ss_quantity) as store_sales_quantityave ,stddev_samp(ss_quantity) as store_sales_quantitystdev ,stddev_samp(ss_quantity)/avg(ss_quantity) as store_sales_quantitycov ,count(sr_return_quantity) as store_returns_quantitycount ,avg(sr_return_quantity) as store_returns_quantityave ,stddev_samp(sr_return_quantity) as store_returns_quantitystdev ,stddev_samp(sr_return_quantity)/avg(sr_return_quantity) as store_returns_quantitycov ,count(cs_quantity) as catalog_sales_quantitycount ,avg(cs_quantity) as catalog_sales_quantityave ,stddev_samp(cs_quantity) as catalog_sales_quantitystdev ,stddev_samp(cs_quantity)/avg(cs_quantity) as catalog_sales_quantitycov from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_quarter_name = '1999Q1' and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_quarter_name in ('1999Q1','1999Q2','1999Q3') and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_quarter_name in ('1999Q1','1999Q2','1999Q3') group by i_item_id ,i_item_desc ,s_state order by i_item_id ,i_item_desc ,s_state limit 100""", "q18" -> """ select i_item_id, ca_country, ca_state, ca_county, avg( cast(cs_quantity as decimal(12,2))) agg1, avg( cast(cs_list_price as decimal(12,2))) agg2, avg( cast(cs_coupon_amt as decimal(12,2))) agg3, avg( cast(cs_sales_price as decimal(12,2))) agg4, avg( cast(cs_net_profit as decimal(12,2))) agg5, avg( cast(c_birth_year as decimal(12,2))) agg6, avg( cast(cd1.cd_dep_count as decimal(12,2))) agg7 from catalog_sales, customer_demographics cd1, customer_demographics cd2, customer, customer_address, date_dim, item where cs_sold_date_sk = d_date_sk and cs_item_sk = i_item_sk and cs_bill_cdemo_sk = cd1.cd_demo_sk and cs_bill_customer_sk = c_customer_sk and cd1.cd_gender = 'F' and cd1.cd_education_status = 'Primary' and c_current_cdemo_sk = cd2.cd_demo_sk and c_current_addr_sk = ca_address_sk and c_birth_month in (6,7,3,11,12,8) and d_year = 1999 and ca_state in ('IL','WV','KS' ,'GA','LA','PA','TX') group by rollup (i_item_id, ca_country, ca_state, ca_county) order by ca_country, ca_state, ca_county, i_item_id limit 100""", "q19" -> """ select i_brand_id brand_id, i_brand brand, i_manufact_id, i_manufact, sum(ss_ext_sales_price) ext_price from date_dim, store_sales, item,customer,customer_address,store where d_date_sk = ss_sold_date_sk and ss_item_sk = i_item_sk and i_manager_id=26 and d_moy=12 and d_year=2000 and ss_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and substr(ca_zip,1,5) <> substr(s_zip,1,5) and ss_store_sk = s_store_sk group by i_brand ,i_brand_id ,i_manufact_id ,i_manufact order by ext_price desc ,i_brand ,i_brand_id ,i_manufact_id ,i_manufact limit 100 """, "q20" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(cs_ext_sales_price) as itemrevenue ,sum(cs_ext_sales_price)*100/sum(sum(cs_ext_sales_price)) over (partition by i_class) as revenueratio from catalog_sales ,item ,date_dim where cs_item_sk = i_item_sk and i_category in ('Books', 'Home', 'Jewelry') and cs_sold_date_sk = d_date_sk and d_date between cast('1998-05-08' as date) and (cast('1998-05-08' as date) + INTERVAL 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio limit 100""", "q21" -> """ select * from(select w_warehouse_name ,i_item_id ,sum(case when (cast(d_date as date) < cast ('2000-05-22' as date)) then inv_quantity_on_hand else 0 end) as inv_before ,sum(case when (cast(d_date as date) >= cast ('2000-05-22' as date)) then inv_quantity_on_hand else 0 end) as inv_after from inventory ,warehouse ,item ,date_dim where i_current_price between 0.99 and 1.49 and i_item_sk = inv_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_date between (cast ('2000-05-22' as date) - INTERVAL 30 days) and (cast ('2000-05-22' as date) + INTERVAL 30 days) group by w_warehouse_name, i_item_id) x where (case when inv_before > 0 then inv_after / inv_before else null end) between 2.0/3.0 and 3.0/2.0 order by w_warehouse_name ,i_item_id limit 100""", "q22" -> """ select i_product_name ,i_brand ,i_class ,i_category ,avg(inv_quantity_on_hand) qoh from inventory ,date_dim ,item where inv_date_sk=d_date_sk and inv_item_sk=i_item_sk and d_month_seq between 1199 and 1199 + 11 group by rollup(i_product_name ,i_brand ,i_class ,i_category) order by qoh, i_product_name, i_brand, i_class, i_category limit 100""", "q23a" -> """ with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt from store_sales ,date_dim ,item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and d_year in (2000,2000+1,2000+2,2000+3) group by substr(i_item_desc,1,30),i_item_sk,d_date having count(*) >4), max_store_sales as (select max(csales) tpcds_cmax from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales from store_sales ,customer ,date_dim where ss_customer_sk = c_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (2000,2000+1,2000+2,2000+3) group by c_customer_sk)), best_ss_customer as (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales from store_sales ,customer where ss_customer_sk = c_customer_sk group by c_customer_sk having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select * from max_store_sales)) select sum(sales) from (select cs_quantity*cs_list_price sales from catalog_sales ,date_dim where d_year = 2000 and d_moy = 5 and cs_sold_date_sk = d_date_sk and cs_item_sk in (select item_sk from frequent_ss_items) and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) union all select ws_quantity*ws_list_price sales from web_sales ,date_dim where d_year = 2000 and d_moy = 5 and ws_sold_date_sk = d_date_sk and ws_item_sk in (select item_sk from frequent_ss_items) and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer)) limit 100""", "q23b" -> """ with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt from store_sales ,date_dim ,item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and d_year in (2000,2000 + 1,2000 + 2,2000 + 3) group by substr(i_item_desc,1,30),i_item_sk,d_date having count(*) >4), max_store_sales as (select max(csales) tpcds_cmax from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales from store_sales ,customer ,date_dim where ss_customer_sk = c_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (2000,2000+1,2000+2,2000+3) group by c_customer_sk)), best_ss_customer as (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales from store_sales ,customer where ss_customer_sk = c_customer_sk group by c_customer_sk having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select * from max_store_sales)) select c_last_name,c_first_name,sales from (select c_last_name,c_first_name,sum(cs_quantity*cs_list_price) sales from catalog_sales ,customer ,date_dim where d_year = 2000 and d_moy = 5 and cs_sold_date_sk = d_date_sk and cs_item_sk in (select item_sk from frequent_ss_items) and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) and cs_bill_customer_sk = c_customer_sk group by c_last_name,c_first_name union all select c_last_name,c_first_name,sum(ws_quantity*ws_list_price) sales from web_sales ,customer ,date_dim where d_year = 2000 and d_moy = 5 and ws_sold_date_sk = d_date_sk and ws_item_sk in (select item_sk from frequent_ss_items) and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer) and ws_bill_customer_sk = c_customer_sk group by c_last_name,c_first_name) order by c_last_name,c_first_name,sales limit 100""", "q24a" -> """ with ssales as (select c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size ,sum(ss_net_paid_inc_tax) netpaid from store_sales ,store_returns ,store ,item ,customer ,customer_address where ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_customer_sk = c_customer_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and c_current_addr_sk = ca_address_sk and c_birth_country <> upper(ca_country) and s_zip = ca_zip and s_market_id=10 group by c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size) select c_last_name ,c_first_name ,s_store_name ,sum(netpaid) paid from ssales where i_color = 'navy' group by c_last_name ,c_first_name ,s_store_name having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) order by c_last_name ,c_first_name ,s_store_name""", "q24b" -> """ with ssales as (select c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size ,sum(ss_net_paid_inc_tax) netpaid from store_sales ,store_returns ,store ,item ,customer ,customer_address where ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_customer_sk = c_customer_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and c_current_addr_sk = ca_address_sk and c_birth_country <> upper(ca_country) and s_zip = ca_zip and s_market_id = 10 group by c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size) select c_last_name ,c_first_name ,s_store_name ,sum(netpaid) paid from ssales where i_color = 'beige' group by c_last_name ,c_first_name ,s_store_name having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) order by c_last_name ,c_first_name ,s_store_name""", "q25" -> """ select i_item_id ,i_item_desc ,s_store_id ,s_store_name ,sum(ss_net_profit) as store_sales_profit ,sum(sr_net_loss) as store_returns_loss ,sum(cs_net_profit) as catalog_sales_profit from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_moy = 4 and d1.d_year = 2002 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_moy between 4 and 10 and d2.d_year = 2002 and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_moy between 4 and 10 and d3.d_year = 2002 group by i_item_id ,i_item_desc ,s_store_id ,s_store_name order by i_item_id ,i_item_desc ,s_store_id ,s_store_name limit 100""", "q26" -> """ select i_item_id, avg(cs_quantity) agg1, avg(cs_list_price) agg2, avg(cs_coupon_amt) agg3, avg(cs_sales_price) agg4 from catalog_sales, customer_demographics, date_dim, item, promotion where cs_sold_date_sk = d_date_sk and cs_item_sk = i_item_sk and cs_bill_cdemo_sk = cd_demo_sk and cs_promo_sk = p_promo_sk and cd_gender = 'F' and cd_marital_status = 'M' and cd_education_status = '2 yr Degree' and (p_channel_email = 'N' or p_channel_event = 'N') and d_year = 2002 group by i_item_id order by i_item_id limit 100""", "q27" -> """ select i_item_id, s_state, grouping(s_state) g_state, avg(ss_quantity) agg1, avg(ss_list_price) agg2, avg(ss_coupon_amt) agg3, avg(ss_sales_price) agg4 from store_sales, customer_demographics, date_dim, store, item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and ss_cdemo_sk = cd_demo_sk and cd_gender = 'F' and cd_marital_status = 'S' and cd_education_status = 'Advanced Degree' and d_year = 2000 and s_state in ('WA','LA', 'LA', 'TX', 'AL', 'PA') group by rollup (i_item_id, s_state) order by i_item_id ,s_state limit 100""", "q28" -> """ select * from (select avg(ss_list_price) B1_LP ,count(ss_list_price) B1_CNT ,count(distinct ss_list_price) B1_CNTD from store_sales where ss_quantity between 0 and 5 and (ss_list_price between 189 and 189+10 or ss_coupon_amt between 4483 and 4483+1000 or ss_wholesale_cost between 24 and 24+20)) B1, (select avg(ss_list_price) B2_LP ,count(ss_list_price) B2_CNT ,count(distinct ss_list_price) B2_CNTD from store_sales where ss_quantity between 6 and 10 and (ss_list_price between 71 and 71+10 or ss_coupon_amt between 14775 and 14775+1000 or ss_wholesale_cost between 38 and 38+20)) B2, (select avg(ss_list_price) B3_LP ,count(ss_list_price) B3_CNT ,count(distinct ss_list_price) B3_CNTD from store_sales where ss_quantity between 11 and 15 and (ss_list_price between 183 and 183+10 or ss_coupon_amt between 13456 and 13456+1000 or ss_wholesale_cost between 31 and 31+20)) B3, (select avg(ss_list_price) B4_LP ,count(ss_list_price) B4_CNT ,count(distinct ss_list_price) B4_CNTD from store_sales where ss_quantity between 16 and 20 and (ss_list_price between 135 and 135+10 or ss_coupon_amt between 4905 and 4905+1000 or ss_wholesale_cost between 27 and 27+20)) B4, (select avg(ss_list_price) B5_LP ,count(ss_list_price) B5_CNT ,count(distinct ss_list_price) B5_CNTD from store_sales where ss_quantity between 21 and 25 and (ss_list_price between 180 and 180+10 or ss_coupon_amt between 17430 and 17430+1000 or ss_wholesale_cost between 57 and 57+20)) B5, (select avg(ss_list_price) B6_LP ,count(ss_list_price) B6_CNT ,count(distinct ss_list_price) B6_CNTD from store_sales where ss_quantity between 26 and 30 and (ss_list_price between 49 and 49+10 or ss_coupon_amt between 2950 and 2950+1000 or ss_wholesale_cost between 52 and 52+20)) B6 limit 100""", "q29" -> """ select i_item_id ,i_item_desc ,s_store_id ,s_store_name ,stddev_samp(ss_quantity) as store_sales_quantity ,stddev_samp(sr_return_quantity) as store_returns_quantity ,stddev_samp(cs_quantity) as catalog_sales_quantity from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_moy = 4 and d1.d_year = 1998 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_moy between 4 and 4 + 3 and d2.d_year = 1998 and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_year in (1998,1998+1,1998+2) group by i_item_id ,i_item_desc ,s_store_id ,s_store_name order by i_item_id ,i_item_desc ,s_store_id ,s_store_name limit 100""", "q30" -> """ with customer_total_return as (select wr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, sum(wr_return_amt) as ctr_total_return from web_returns ,date_dim ,customer_address where wr_returned_date_sk = d_date_sk and d_year =2000 and wr_returning_addr_sk = ca_address_sk group by wr_returning_customer_sk ,ca_state) select c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address ,c_last_review_date,ctr_total_return from customer_total_return ctr1 ,customer_address ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_state = ctr2.ctr_state) and ca_address_sk = c_current_addr_sk and ca_state = 'GA' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address ,c_last_review_date,ctr_total_return limit 100""", "q31" -> """ with ss as (select ca_county,d_qoy, d_year,sum(ss_ext_sales_price) as store_sales from store_sales,date_dim,customer_address where ss_sold_date_sk = d_date_sk and ss_addr_sk=ca_address_sk group by ca_county,d_qoy, d_year), ws as (select ca_county,d_qoy, d_year,sum(ws_ext_sales_price) as web_sales from web_sales,date_dim,customer_address where ws_sold_date_sk = d_date_sk and ws_bill_addr_sk=ca_address_sk group by ca_county,d_qoy, d_year) select ss1.ca_county ,ss1.d_year ,ws2.web_sales/ws1.web_sales web_q1_q2_increase ,ss2.store_sales/ss1.store_sales store_q1_q2_increase ,ws3.web_sales/ws2.web_sales web_q2_q3_increase ,ss3.store_sales/ss2.store_sales store_q2_q3_increase from ss ss1 ,ss ss2 ,ss ss3 ,ws ws1 ,ws ws2 ,ws ws3 where ss1.d_qoy = 1 and ss1.d_year = 1998 and ss1.ca_county = ss2.ca_county and ss2.d_qoy = 2 and ss2.d_year = 1998 and ss2.ca_county = ss3.ca_county and ss3.d_qoy = 3 and ss3.d_year = 1998 and ss1.ca_county = ws1.ca_county and ws1.d_qoy = 1 and ws1.d_year = 1998 and ws1.ca_county = ws2.ca_county and ws2.d_qoy = 2 and ws2.d_year = 1998 and ws1.ca_county = ws3.ca_county and ws3.d_qoy = 3 and ws3.d_year =1998 and case when ws1.web_sales > 0 then ws2.web_sales/ws1.web_sales else null end > case when ss1.store_sales > 0 then ss2.store_sales/ss1.store_sales else null end and case when ws2.web_sales > 0 then ws3.web_sales/ws2.web_sales else null end > case when ss2.store_sales > 0 then ss3.store_sales/ss2.store_sales else null end order by ss1.ca_county""", "q32" -> """ select sum(cs_ext_discount_amt) as `excess discount amount` from catalog_sales ,item ,date_dim where i_manufact_id = 948 and i_item_sk = cs_item_sk and d_date between '1998-02-03' and (cast('1998-02-03' as date) + INTERVAL 90 days) and d_date_sk = cs_sold_date_sk and cs_ext_discount_amt > ( select 1.3 * avg(cs_ext_discount_amt) from catalog_sales ,date_dim where cs_item_sk = i_item_sk and d_date between '1998-02-03' and (cast('1998-02-03' as date) + INTERVAL 90 days) and d_date_sk = cs_sold_date_sk ) limit 100""", "q33" -> """ with ss as ( select i_manufact_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Electronics')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 2 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_manufact_id), cs as ( select i_manufact_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Electronics')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 2 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_manufact_id), ws as ( select i_manufact_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Electronics')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 2 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_manufact_id) select i_manufact_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_manufact_id order by total_sales limit 100""", "q34" -> """ select c_last_name ,c_first_name ,c_salutation ,c_preferred_cust_flag ,ss_ticket_number ,cnt from (select ss_ticket_number ,ss_customer_sk ,count(*) cnt from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and (date_dim.d_dom between 1 and 3 or date_dim.d_dom between 25 and 28) and (household_demographics.hd_buy_potential = '>10000' or household_demographics.hd_buy_potential = '5001-10000') and household_demographics.hd_vehicle_count > 0 and (case when household_demographics.hd_vehicle_count > 0 then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end) > 1.2 and date_dim.d_year in (1999,1999+1,1999+2) and store.s_county in ('Jefferson Davis Parish','Levy County','Coal County','Oglethorpe County', 'Mobile County','Gage County','Richland County','Gogebic County') group by ss_ticket_number,ss_customer_sk) dn,customer where ss_customer_sk = c_customer_sk and cnt between 15 and 20 order by c_last_name,c_first_name,c_salutation,c_preferred_cust_flag desc, ss_ticket_number""", "q35" -> """ select ca_state, cd_gender, cd_marital_status, cd_dep_count, count(*) cnt1, stddev_samp(cd_dep_count), stddev_samp(cd_dep_count), min(cd_dep_count), cd_dep_employed_count, count(*) cnt2, stddev_samp(cd_dep_employed_count), stddev_samp(cd_dep_employed_count), min(cd_dep_employed_count), cd_dep_college_count, count(*) cnt3, stddev_samp(cd_dep_college_count), stddev_samp(cd_dep_college_count), min(cd_dep_college_count) from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2002 and d_qoy < 4) and (exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2002 and d_qoy < 4) or exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2002 and d_qoy < 4)) group by ca_state, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count order by ca_state, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count limit 100""", "q36" -> """ select sum(ss_net_profit)/sum(ss_ext_sales_price) as gross_margin ,i_category ,i_class ,grouping(i_category)+grouping(i_class) as lochierarchy ,rank() over ( partition by grouping(i_category)+grouping(i_class), case when grouping(i_class) = 0 then i_category end order by sum(ss_net_profit)/sum(ss_ext_sales_price) asc) as rank_within_parent from store_sales ,date_dim d1 ,item ,store where d1.d_year = 1998 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and s_state in ('OH','WV','PA','TN', 'MN','MO','NM','MI') group by rollup(i_category,i_class) order by lochierarchy desc ,case when lochierarchy = 0 then i_category end ,rank_within_parent limit 100""", "q37" -> """ select i_item_id ,i_item_desc ,i_current_price from item, inventory, date_dim, catalog_sales where i_current_price between 35 and 35 + 30 and inv_item_sk = i_item_sk and d_date_sk=inv_date_sk and d_date between cast('2001-01-20' as date) and (cast('2001-01-20' as date) + interval 60 days) and i_manufact_id in (928,715,942,861) and inv_quantity_on_hand between 100 and 500 and cs_item_sk = i_item_sk group by i_item_id,i_item_desc,i_current_price order by i_item_id limit 100""", "q38" -> """ select count(*) from ( select distinct c_last_name, c_first_name, d_date from store_sales, date_dim, customer where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_customer_sk = customer.c_customer_sk and d_month_seq between 1222 and 1222 + 11 intersect select distinct c_last_name, c_first_name, d_date from catalog_sales, date_dim, customer where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1222 and 1222 + 11 intersect select distinct c_last_name, c_first_name, d_date from web_sales, date_dim, customer where web_sales.ws_sold_date_sk = date_dim.d_date_sk and web_sales.ws_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1222 and 1222 + 11 ) hot_cust limit 100""", "q39a" -> """ with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stdev,mean, case mean when 0 then null else stdev/mean end cov from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean from inventory ,item ,warehouse ,date_dim where inv_item_sk = i_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_year =1998 group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo where case mean when 0 then 0 else stdev/mean end > 1) select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov from inv inv1,inv inv2 where inv1.i_item_sk = inv2.i_item_sk and inv1.w_warehouse_sk = inv2.w_warehouse_sk and inv1.d_moy=4 and inv2.d_moy=4+1 order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov ,inv2.d_moy,inv2.mean, inv2.cov""", "q39b" -> """ with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stdev,mean, case mean when 0 then null else stdev/mean end cov from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean from inventory ,item ,warehouse ,date_dim where inv_item_sk = i_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_year =1998 group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo where case mean when 0 then 0 else stdev/mean end > 1) select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov from inv inv1,inv inv2 where inv1.i_item_sk = inv2.i_item_sk and inv1.w_warehouse_sk = inv2.w_warehouse_sk and inv1.d_moy=4 and inv2.d_moy=4+1 and inv1.cov > 1.5 order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov ,inv2.d_moy,inv2.mean, inv2.cov""", "q40" -> """ select w_state ,i_item_id ,sum(case when (cast(d_date as date) < cast ('1999-02-02' as date)) then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_before ,sum(case when (cast(d_date as date) >= cast ('1999-02-02' as date)) then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_after from catalog_sales left outer join catalog_returns on (cs_order_number = cr_order_number and cs_item_sk = cr_item_sk) ,warehouse ,item ,date_dim where i_current_price between 0.99 and 1.49 and i_item_sk = cs_item_sk and cs_warehouse_sk = w_warehouse_sk and cs_sold_date_sk = d_date_sk and d_date between (cast ('1999-02-02' as date) - INTERVAL 30 days) and (cast ('1999-02-02' as date) + INTERVAL 30 days) group by w_state,i_item_id order by w_state,i_item_id limit 100""", "q41" -> """ select distinct(i_product_name) from item i1 where i_manufact_id between 732 and 732+40 and (select count(*) as item_cnt from item where (i_manufact = i1.i_manufact and ((i_category = 'Women' and (i_color = 'beige' or i_color = 'spring') and (i_units = 'Tsp' or i_units = 'Ton') and (i_size = 'petite' or i_size = 'extra large') ) or (i_category = 'Women' and (i_color = 'white' or i_color = 'pale') and (i_units = 'Box' or i_units = 'Dram') and (i_size = 'large' or i_size = 'economy') ) or (i_category = 'Men' and (i_color = 'midnight' or i_color = 'frosted') and (i_units = 'Bunch' or i_units = 'Carton') and (i_size = 'small' or i_size = 'N/A') ) or (i_category = 'Men' and (i_color = 'azure' or i_color = 'goldenrod') and (i_units = 'Pallet' or i_units = 'Gross') and (i_size = 'petite' or i_size = 'extra large') ))) or (i_manufact = i1.i_manufact and ((i_category = 'Women' and (i_color = 'brown' or i_color = 'hot') and (i_units = 'Tbl' or i_units = 'Cup') and (i_size = 'petite' or i_size = 'extra large') ) or (i_category = 'Women' and (i_color = 'powder' or i_color = 'honeydew') and (i_units = 'Bundle' or i_units = 'Unknown') and (i_size = 'large' or i_size = 'economy') ) or (i_category = 'Men' and (i_color = 'antique' or i_color = 'purple') and (i_units = 'N/A' or i_units = 'Dozen') and (i_size = 'small' or i_size = 'N/A') ) or (i_category = 'Men' and (i_color = 'lavender' or i_color = 'tomato') and (i_units = 'Lb' or i_units = 'Oz') and (i_size = 'petite' or i_size = 'extra large') )))) > 0 order by i_product_name limit 100""", "q42" -> """ select dt.d_year ,item.i_category_id ,item.i_category ,sum(ss_ext_sales_price) from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manager_id = 1 and dt.d_moy=11 and dt.d_year=2002 group by dt.d_year ,item.i_category_id ,item.i_category order by sum(ss_ext_sales_price) desc,dt.d_year ,item.i_category_id ,item.i_category limit 100 """, "q43" -> """ select s_store_name, s_store_id, sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales from date_dim, store_sales, store where d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk and s_gmt_offset = -6 and d_year = 1999 group by s_store_name, s_store_id order by s_store_name, s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales limit 100""", "q44" -> """ select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing from(select * from (select item_sk,rank() over (order by rank_col asc) rnk from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col from store_sales ss1 where ss_store_sk = 321 group by ss_item_sk having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col from store_sales where ss_store_sk = 321 and ss_addr_sk is null group by ss_store_sk))V1)V11 where rnk < 11) asceding, (select * from (select item_sk,rank() over (order by rank_col desc) rnk from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col from store_sales ss1 where ss_store_sk = 321 group by ss_item_sk having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col from store_sales where ss_store_sk = 321 and ss_addr_sk is null group by ss_store_sk))V2)V21 where rnk < 11) descending, item i1, item i2 where asceding.rnk = descending.rnk and i1.i_item_sk=asceding.item_sk and i2.i_item_sk=descending.item_sk order by asceding.rnk limit 100""", "q45" -> """ select ca_zip, ca_county, sum(ws_sales_price) from web_sales, customer, customer_address, date_dim, item where ws_bill_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and ws_item_sk = i_item_sk and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') or i_item_id in (select i_item_id from item where i_item_sk in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) ) ) and ws_sold_date_sk = d_date_sk and d_qoy = 2 and d_year = 1999 group by ca_zip, ca_county order by ca_zip, ca_county limit 100""", "q46" -> """ select c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number ,amt,profit from (select ss_ticket_number ,ss_customer_sk ,ca_city bought_city ,sum(ss_coupon_amt) amt ,sum(ss_net_profit) profit from store_sales,date_dim,store,household_demographics,customer_address where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and store_sales.ss_addr_sk = customer_address.ca_address_sk and (household_demographics.hd_dep_count = 2 or household_demographics.hd_vehicle_count= 2) and date_dim.d_dow in (6,0) and date_dim.d_year in (1998,1998+1,1998+2) and store.s_city in ('Antioch','Mount Vernon','Jamestown','Wilson','Farmington') group by ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city) dn,customer,customer_address current_addr where ss_customer_sk = c_customer_sk and customer.c_current_addr_sk = current_addr.ca_address_sk and current_addr.ca_city <> bought_city order by c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number limit 100""", "q47" -> """ with v1 as( select i_category, i_brand, s_store_name, s_company_name, d_year, d_moy, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_category, i_brand, s_store_name, s_company_name, d_year) avg_monthly_sales, rank() over (partition by i_category, i_brand, s_store_name, s_company_name order by d_year, d_moy) rn from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ( d_year = 2001 or ( d_year = 2001-1 and d_moy =12) or ( d_year = 2001+1 and d_moy =1) ) group by i_category, i_brand, s_store_name, s_company_name, d_year, d_moy), v2 as( select v1.s_company_name ,v1.d_year, v1.d_moy ,v1.avg_monthly_sales ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum from v1, v1 v1_lag, v1 v1_lead where v1.i_category = v1_lag.i_category and v1.i_category = v1_lead.i_category and v1.i_brand = v1_lag.i_brand and v1.i_brand = v1_lead.i_brand and v1.s_store_name = v1_lag.s_store_name and v1.s_store_name = v1_lead.s_store_name and v1.s_company_name = v1_lag.s_company_name and v1.s_company_name = v1_lead.s_company_name and v1.rn = v1_lag.rn + 1 and v1.rn = v1_lead.rn - 1) select * from v2 where d_year = 2001 and avg_monthly_sales > 0 and case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by sum_sales - avg_monthly_sales, avg_monthly_sales limit 100""", "q48" -> """ select sum (ss_quantity) from store_sales, store, customer_demographics, customer_address, date_dim where s_store_sk = ss_store_sk and ss_sold_date_sk = d_date_sk and d_year = 1999 and ( ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'D' and cd_education_status = 'College' and ss_sales_price between 100.00 and 150.00 ) or ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'W' and cd_education_status = 'Secondary' and ss_sales_price between 50.00 and 100.00 ) or ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'M' and cd_education_status = '2 yr Degree' and ss_sales_price between 150.00 and 200.00 ) ) and ( ( ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('NE', 'IA', 'NY') and ss_net_profit between 0 and 2000 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('IN', 'TN', 'OH') and ss_net_profit between 150 and 3000 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('KS', 'CA', 'CO') and ss_net_profit between 50 and 25000 ) )""", "q49" -> """ select channel, item, return_ratio, return_rank, currency_rank from (select 'web' as channel ,web.item ,web.return_ratio ,web.return_rank ,web.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select ws.ws_item_sk as item ,(cast(sum(coalesce(wr.wr_return_quantity,0)) as decimal(15,4))/ cast(sum(coalesce(ws.ws_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(wr.wr_return_amt,0)) as decimal(15,4))/ cast(sum(coalesce(ws.ws_net_paid,0)) as decimal(15,4) )) as currency_ratio from web_sales ws left outer join web_returns wr on (ws.ws_order_number = wr.wr_order_number and ws.ws_item_sk = wr.wr_item_sk) ,date_dim where wr.wr_return_amt > 10000 and ws.ws_net_profit > 1 and ws.ws_net_paid > 0 and ws.ws_quantity > 0 and ws_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 11 group by ws.ws_item_sk ) in_web ) web where ( web.return_rank <= 10 or web.currency_rank <= 10 ) union select 'catalog' as channel ,catalog.item ,catalog.return_ratio ,catalog.return_rank ,catalog.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select cs.cs_item_sk as item ,(cast(sum(coalesce(cr.cr_return_quantity,0)) as decimal(15,4))/ cast(sum(coalesce(cs.cs_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(cr.cr_return_amount,0)) as decimal(15,4))/ cast(sum(coalesce(cs.cs_net_paid,0)) as decimal(15,4) )) as currency_ratio from catalog_sales cs left outer join catalog_returns cr on (cs.cs_order_number = cr.cr_order_number and cs.cs_item_sk = cr.cr_item_sk) ,date_dim where cr.cr_return_amount > 10000 and cs.cs_net_profit > 1 and cs.cs_net_paid > 0 and cs.cs_quantity > 0 and cs_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 11 group by cs.cs_item_sk ) in_cat ) catalog where ( catalog.return_rank <= 10 or catalog.currency_rank <=10 ) union select 'store' as channel ,store.item ,store.return_ratio ,store.return_rank ,store.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select sts.ss_item_sk as item ,(cast(sum(coalesce(sr.sr_return_quantity,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(sr.sr_return_amt,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_net_paid,0)) as decimal(15,4) )) as currency_ratio from store_sales sts left outer join store_returns sr on (sts.ss_ticket_number = sr.sr_ticket_number and sts.ss_item_sk = sr.sr_item_sk) ,date_dim where sr.sr_return_amt > 10000 and sts.ss_net_profit > 1 and sts.ss_net_paid > 0 and sts.ss_quantity > 0 and ss_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 11 group by sts.ss_item_sk ) in_store ) store where ( store.return_rank <= 10 or store.currency_rank <= 10 ) ) order by 1,4,5,2 limit 100""", "q50" -> """ select s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip ,sum(case when (sr_returned_date_sk - ss_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 30) and (sr_returned_date_sk - ss_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 60) and (sr_returned_date_sk - ss_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 90) and (sr_returned_date_sk - ss_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from store_sales ,store_returns ,store ,date_dim d1 ,date_dim d2 where d2.d_year = 1999 and d2.d_moy = 9 and ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_sold_date_sk = d1.d_date_sk and sr_returned_date_sk = d2.d_date_sk and ss_customer_sk = sr_customer_sk and ss_store_sk = s_store_sk group by s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip order by s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip limit 100""", "q51" -> """ WITH web_v1 as ( select ws_item_sk item_sk, d_date, sum(sum(ws_sales_price)) over (partition by ws_item_sk order by d_date rows between unbounded preceding and current row) cume_sales from web_sales ,date_dim where ws_sold_date_sk=d_date_sk and d_month_seq between 1176 and 1176+11 and ws_item_sk is not NULL group by ws_item_sk, d_date), store_v1 as ( select ss_item_sk item_sk, d_date, sum(sum(ss_sales_price)) over (partition by ss_item_sk order by d_date rows between unbounded preceding and current row) cume_sales from store_sales ,date_dim where ss_sold_date_sk=d_date_sk and d_month_seq between 1176 and 1176+11 and ss_item_sk is not NULL group by ss_item_sk, d_date) select * from (select item_sk ,d_date ,web_sales ,store_sales ,max(web_sales) over (partition by item_sk order by d_date rows between unbounded preceding and current row) web_cumulative ,max(store_sales) over (partition by item_sk order by d_date rows between unbounded preceding and current row) store_cumulative from (select case when web.item_sk is not null then web.item_sk else store.item_sk end item_sk ,case when web.d_date is not null then web.d_date else store.d_date end d_date ,web.cume_sales web_sales ,store.cume_sales store_sales from web_v1 web full outer join store_v1 store on (web.item_sk = store.item_sk and web.d_date = store.d_date) )x )y where web_cumulative > store_cumulative order by item_sk ,d_date limit 100""", "q52" -> """ select dt.d_year ,item.i_brand_id brand_id ,item.i_brand brand ,sum(ss_ext_sales_price) ext_price from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manager_id = 1 and dt.d_moy=11 and dt.d_year=2001 group by dt.d_year ,item.i_brand ,item.i_brand_id order by dt.d_year ,ext_price desc ,brand_id limit 100 """, "q53" -> """ select * from (select i_manufact_id, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_manufact_id) avg_quarterly_sales from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_month_seq in (1218,1218+1,1218+2,1218+3,1218+4,1218+5,1218+6,1218+7,1218+8,1218+9,1218+10,1218+11) and ((i_category in ('Books','Children','Electronics') and i_class in ('personal','portable','reference','self-help') and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', 'exportiunivamalg #9','scholaramalgamalg #9')) or(i_category in ('Women','Music','Men') and i_class in ('accessories','classical','fragrances','pants') and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 'importoamalg #1'))) group by i_manufact_id, d_qoy ) tmp1 where case when avg_quarterly_sales > 0 then abs (sum_sales - avg_quarterly_sales)/ avg_quarterly_sales else null end > 0.1 order by avg_quarterly_sales, sum_sales, i_manufact_id limit 100""", "q54" -> """ with my_customers as ( select distinct c_customer_sk , c_current_addr_sk from ( select cs_sold_date_sk sold_date_sk, cs_bill_customer_sk customer_sk, cs_item_sk item_sk from catalog_sales union all select ws_sold_date_sk sold_date_sk, ws_bill_customer_sk customer_sk, ws_item_sk item_sk from web_sales ) cs_or_ws_sales, item, date_dim, customer where sold_date_sk = d_date_sk and item_sk = i_item_sk and i_category = 'Music' and i_class = 'country' and c_customer_sk = cs_or_ws_sales.customer_sk and d_moy = 7 and d_year = 2001 ) , my_revenue as ( select c_customer_sk, sum(ss_ext_sales_price) as revenue from my_customers, store_sales, customer_address, store, date_dim where c_current_addr_sk = ca_address_sk and ca_county = s_county and ca_state = s_state and ss_sold_date_sk = d_date_sk and c_customer_sk = ss_customer_sk and d_month_seq between (select distinct d_month_seq+1 from date_dim where d_year = 2001 and d_moy = 7) and (select distinct d_month_seq+3 from date_dim where d_year = 2001 and d_moy = 7) group by c_customer_sk ) , segments as (select cast((revenue/50) as int) as segment from my_revenue ) select segment, count(*) as num_customers, segment*50 as segment_base from segments group by segment order by segment, num_customers limit 100""", "q55" -> """ select i_brand_id brand_id, i_brand brand, sum(ss_ext_sales_price) ext_price from date_dim, store_sales, item where d_date_sk = ss_sold_date_sk and ss_item_sk = i_item_sk and i_manager_id=87 and d_moy=11 and d_year=2001 group by i_brand, i_brand_id order by ext_price desc, i_brand_id limit 100 """, "q56" -> """ with ss as ( select i_item_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('tan','lace','gainsboro')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 3 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id), cs as ( select i_item_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('tan','lace','gainsboro')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 3 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id), ws as ( select i_item_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('tan','lace','gainsboro')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 3 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id) select i_item_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_item_id order by total_sales, i_item_id limit 100""", "q57" -> """ with v1 as( select i_category, i_brand, cc_name, d_year, d_moy, sum(cs_sales_price) sum_sales, avg(sum(cs_sales_price)) over (partition by i_category, i_brand, cc_name, d_year) avg_monthly_sales, rank() over (partition by i_category, i_brand, cc_name order by d_year, d_moy) rn from item, catalog_sales, date_dim, call_center where cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and cc_call_center_sk= cs_call_center_sk and ( d_year = 2001 or ( d_year = 2001-1 and d_moy =12) or ( d_year = 2001+1 and d_moy =1) ) group by i_category, i_brand, cc_name , d_year, d_moy), v2 as( select v1.i_category, v1.i_brand, v1.cc_name ,v1.d_year, v1.d_moy ,v1.avg_monthly_sales ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum from v1, v1 v1_lag, v1 v1_lead where v1.i_category = v1_lag.i_category and v1.i_category = v1_lead.i_category and v1.i_brand = v1_lag.i_brand and v1.i_brand = v1_lead.i_brand and v1. cc_name = v1_lag. cc_name and v1. cc_name = v1_lead. cc_name and v1.rn = v1_lag.rn + 1 and v1.rn = v1_lead.rn - 1) select * from v2 where d_year = 2001 and avg_monthly_sales > 0 and case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by sum_sales - avg_monthly_sales, avg_monthly_sales limit 100""", "q58" -> """ with ss_items as (select i_item_id item_id ,sum(ss_ext_sales_price) ss_item_rev from store_sales ,item ,date_dim where ss_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq = (select d_week_seq from date_dim where d_date = '2000-03-26')) and ss_sold_date_sk = d_date_sk group by i_item_id), cs_items as (select i_item_id item_id ,sum(cs_ext_sales_price) cs_item_rev from catalog_sales ,item ,date_dim where cs_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq = (select d_week_seq from date_dim where d_date = '2000-03-26')) and cs_sold_date_sk = d_date_sk group by i_item_id), ws_items as (select i_item_id item_id ,sum(ws_ext_sales_price) ws_item_rev from web_sales ,item ,date_dim where ws_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq =(select d_week_seq from date_dim where d_date = '2000-03-26')) and ws_sold_date_sk = d_date_sk group by i_item_id) select ss_items.item_id ,ss_item_rev ,ss_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ss_dev ,cs_item_rev ,cs_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 cs_dev ,ws_item_rev ,ws_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ws_dev ,(ss_item_rev+cs_item_rev+ws_item_rev)/3 average from ss_items,cs_items,ws_items where ss_items.item_id=cs_items.item_id and ss_items.item_id=ws_items.item_id and ss_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev and ss_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev and cs_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev and cs_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev and ws_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev and ws_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev order by item_id ,ss_item_rev limit 100""", "q59" -> """ with wss as (select d_week_seq, ss_store_sk, sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales from store_sales,date_dim where d_date_sk = ss_sold_date_sk group by d_week_seq,ss_store_sk ) select s_store_name1,s_store_id1,d_week_seq1 ,sun_sales1/sun_sales2,mon_sales1/mon_sales2 ,tue_sales1/tue_sales2,wed_sales1/wed_sales2,thu_sales1/thu_sales2 ,fri_sales1/fri_sales2,sat_sales1/sat_sales2 from (select s_store_name s_store_name1,wss.d_week_seq d_week_seq1 ,s_store_id s_store_id1,sun_sales sun_sales1 ,mon_sales mon_sales1,tue_sales tue_sales1 ,wed_sales wed_sales1,thu_sales thu_sales1 ,fri_sales fri_sales1,sat_sales sat_sales1 from wss,store,date_dim d where d.d_week_seq = wss.d_week_seq and ss_store_sk = s_store_sk and d_month_seq between 1199 and 1199 + 11) y, (select s_store_name s_store_name2,wss.d_week_seq d_week_seq2 ,s_store_id s_store_id2,sun_sales sun_sales2 ,mon_sales mon_sales2,tue_sales tue_sales2 ,wed_sales wed_sales2,thu_sales thu_sales2 ,fri_sales fri_sales2,sat_sales sat_sales2 from wss,store,date_dim d where d.d_week_seq = wss.d_week_seq and ss_store_sk = s_store_sk and d_month_seq between 1199+ 12 and 1199 + 23) x where s_store_id1=s_store_id2 and d_week_seq1=d_week_seq2-52 order by s_store_name1,s_store_id1,d_week_seq1 limit 100""", "q60" -> """ with ss as ( select i_item_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Men')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 9 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id), cs as ( select i_item_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Men')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 9 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id), ws as ( select i_item_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Men')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 9 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id) select i_item_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_item_id order by i_item_id ,total_sales limit 100""", "q61" -> """ select promotions,total,cast(promotions as decimal(15,4))/cast(total as decimal(15,4))*100 from (select sum(ss_ext_sales_price) promotions from store_sales ,store ,promotion ,date_dim ,customer ,customer_address ,item where ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ss_promo_sk = p_promo_sk and ss_customer_sk= c_customer_sk and ca_address_sk = c_current_addr_sk and ss_item_sk = i_item_sk and ca_gmt_offset = -7 and i_category = 'Electronics' and (p_channel_dmail = 'Y' or p_channel_email = 'Y' or p_channel_tv = 'Y') and s_gmt_offset = -7 and d_year = 2001 and d_moy = 11) promotional_sales, (select sum(ss_ext_sales_price) total from store_sales ,store ,date_dim ,customer ,customer_address ,item where ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ss_customer_sk= c_customer_sk and ca_address_sk = c_current_addr_sk and ss_item_sk = i_item_sk and ca_gmt_offset = -7 and i_category = 'Electronics' and s_gmt_offset = -7 and d_year = 2001 and d_moy = 11) all_sales order by promotions, total limit 100""", "q62" -> """ select substr(w_warehouse_name,1,20) ,sm_type ,web_name ,sum(case when (ws_ship_date_sk - ws_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 30) and (ws_ship_date_sk - ws_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 60) and (ws_ship_date_sk - ws_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 90) and (ws_ship_date_sk - ws_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from web_sales ,warehouse ,ship_mode ,web_site ,date_dim where d_month_seq between 1194 and 1194 + 11 and ws_ship_date_sk = d_date_sk and ws_warehouse_sk = w_warehouse_sk and ws_ship_mode_sk = sm_ship_mode_sk and ws_web_site_sk = web_site_sk group by substr(w_warehouse_name,1,20) ,sm_type ,web_name order by substr(w_warehouse_name,1,20) ,sm_type ,web_name limit 100""", "q63" -> """ select * from (select i_manager_id ,sum(ss_sales_price) sum_sales ,avg(sum(ss_sales_price)) over (partition by i_manager_id) avg_monthly_sales from item ,store_sales ,date_dim ,store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_month_seq in (1205,1205+1,1205+2,1205+3,1205+4,1205+5,1205+6,1205+7,1205+8,1205+9,1205+10,1205+11) and (( i_category in ('Books','Children','Electronics') and i_class in ('personal','portable','reference','self-help') and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', 'exportiunivamalg #9','scholaramalgamalg #9')) or( i_category in ('Women','Music','Men') and i_class in ('accessories','classical','fragrances','pants') and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 'importoamalg #1'))) group by i_manager_id, d_moy) tmp1 where case when avg_monthly_sales > 0 then abs (sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by i_manager_id ,avg_monthly_sales ,sum_sales limit 100""", "q64" -> """ with cs_ui as (select cs_item_sk ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund from catalog_sales ,catalog_returns where cs_item_sk = cr_item_sk and cs_order_number = cr_order_number group by cs_item_sk having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), cross_sales as (select i_product_name product_name ,i_item_sk item_sk ,s_store_name store_name ,s_zip store_zip ,ad1.ca_street_number b_street_number ,ad1.ca_street_name b_street_name ,ad1.ca_city b_city ,ad1.ca_zip b_zip ,ad2.ca_street_number c_street_number ,ad2.ca_street_name c_street_name ,ad2.ca_city c_city ,ad2.ca_zip c_zip ,d1.d_year as syear ,d2.d_year as fsyear ,d3.d_year s2year ,count(*) cnt ,sum(ss_wholesale_cost) s1 ,sum(ss_list_price) s2 ,sum(ss_coupon_amt) s3 FROM store_sales ,store_returns ,cs_ui ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,customer ,customer_demographics cd1 ,customer_demographics cd2 ,promotion ,household_demographics hd1 ,household_demographics hd2 ,customer_address ad1 ,customer_address ad2 ,income_band ib1 ,income_band ib2 ,item WHERE ss_store_sk = s_store_sk AND ss_sold_date_sk = d1.d_date_sk AND ss_customer_sk = c_customer_sk AND ss_cdemo_sk= cd1.cd_demo_sk AND ss_hdemo_sk = hd1.hd_demo_sk AND ss_addr_sk = ad1.ca_address_sk and ss_item_sk = i_item_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and ss_item_sk = cs_ui.cs_item_sk and c_current_cdemo_sk = cd2.cd_demo_sk AND c_current_hdemo_sk = hd2.hd_demo_sk AND c_current_addr_sk = ad2.ca_address_sk and c_first_sales_date_sk = d2.d_date_sk and c_first_shipto_date_sk = d3.d_date_sk and ss_promo_sk = p_promo_sk and hd1.hd_income_band_sk = ib1.ib_income_band_sk and hd2.hd_income_band_sk = ib2.ib_income_band_sk and cd1.cd_marital_status <> cd2.cd_marital_status and i_color in ('peach','misty','drab','chocolate','almond','saddle') and i_current_price between 75 and 75 + 10 and i_current_price between 75 + 1 and 75 + 15 group by i_product_name ,i_item_sk ,s_store_name ,s_zip ,ad1.ca_street_number ,ad1.ca_street_name ,ad1.ca_city ,ad1.ca_zip ,ad2.ca_street_number ,ad2.ca_street_name ,ad2.ca_city ,ad2.ca_zip ,d1.d_year ,d2.d_year ,d3.d_year ) select cs1.product_name ,cs1.store_name ,cs1.store_zip ,cs1.b_street_number ,cs1.b_street_name ,cs1.b_city ,cs1.b_zip ,cs1.c_street_number ,cs1.c_street_name ,cs1.c_city ,cs1.c_zip ,cs1.syear ,cs1.cnt ,cs1.s1 as s11 ,cs1.s2 as s21 ,cs1.s3 as s31 ,cs2.s1 as s12 ,cs2.s2 as s22 ,cs2.s3 as s32 ,cs2.syear ,cs2.cnt from cross_sales cs1,cross_sales cs2 where cs1.item_sk=cs2.item_sk and cs1.syear = 2000 and cs2.syear = 2000 + 1 and cs2.cnt <= cs1.cnt and cs1.store_name = cs2.store_name and cs1.store_zip = cs2.store_zip order by cs1.product_name ,cs1.store_name ,cs2.cnt ,cs1.s1 ,cs2.s1""", "q65" -> """ select s_store_name, i_item_desc, sc.revenue, i_current_price, i_wholesale_cost, i_brand from store, item, (select ss_store_sk, avg(revenue) as ave from (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue from store_sales, date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1208 and 1208+11 group by ss_store_sk, ss_item_sk) sa group by ss_store_sk) sb, (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue from store_sales, date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1208 and 1208+11 group by ss_store_sk, ss_item_sk) sc where sb.ss_store_sk = sc.ss_store_sk and sc.revenue <= 0.1 * sb.ave and s_store_sk = sc.ss_store_sk and i_item_sk = sc.ss_item_sk order by s_store_name, i_item_desc limit 100""", "q66" -> """ select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,ship_carriers ,year ,sum(jan_sales) as jan_sales ,sum(feb_sales) as feb_sales ,sum(mar_sales) as mar_sales ,sum(apr_sales) as apr_sales ,sum(may_sales) as may_sales ,sum(jun_sales) as jun_sales ,sum(jul_sales) as jul_sales ,sum(aug_sales) as aug_sales ,sum(sep_sales) as sep_sales ,sum(oct_sales) as oct_sales ,sum(nov_sales) as nov_sales ,sum(dec_sales) as dec_sales ,sum(jan_sales/w_warehouse_sq_ft) as jan_sales_per_sq_foot ,sum(feb_sales/w_warehouse_sq_ft) as feb_sales_per_sq_foot ,sum(mar_sales/w_warehouse_sq_ft) as mar_sales_per_sq_foot ,sum(apr_sales/w_warehouse_sq_ft) as apr_sales_per_sq_foot ,sum(may_sales/w_warehouse_sq_ft) as may_sales_per_sq_foot ,sum(jun_sales/w_warehouse_sq_ft) as jun_sales_per_sq_foot ,sum(jul_sales/w_warehouse_sq_ft) as jul_sales_per_sq_foot ,sum(aug_sales/w_warehouse_sq_ft) as aug_sales_per_sq_foot ,sum(sep_sales/w_warehouse_sq_ft) as sep_sales_per_sq_foot ,sum(oct_sales/w_warehouse_sq_ft) as oct_sales_per_sq_foot ,sum(nov_sales/w_warehouse_sq_ft) as nov_sales_per_sq_foot ,sum(dec_sales/w_warehouse_sq_ft) as dec_sales_per_sq_foot ,sum(jan_net) as jan_net ,sum(feb_net) as feb_net ,sum(mar_net) as mar_net ,sum(apr_net) as apr_net ,sum(may_net) as may_net ,sum(jun_net) as jun_net ,sum(jul_net) as jul_net ,sum(aug_net) as aug_net ,sum(sep_net) as sep_net ,sum(oct_net) as oct_net ,sum(nov_net) as nov_net ,sum(dec_net) as dec_net from ( select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,'HARMSTORF' || ',' || 'USPS' as ship_carriers ,d_year as year ,sum(case when d_moy = 1 then ws_sales_price* ws_quantity else 0 end) as jan_sales ,sum(case when d_moy = 2 then ws_sales_price* ws_quantity else 0 end) as feb_sales ,sum(case when d_moy = 3 then ws_sales_price* ws_quantity else 0 end) as mar_sales ,sum(case when d_moy = 4 then ws_sales_price* ws_quantity else 0 end) as apr_sales ,sum(case when d_moy = 5 then ws_sales_price* ws_quantity else 0 end) as may_sales ,sum(case when d_moy = 6 then ws_sales_price* ws_quantity else 0 end) as jun_sales ,sum(case when d_moy = 7 then ws_sales_price* ws_quantity else 0 end) as jul_sales ,sum(case when d_moy = 8 then ws_sales_price* ws_quantity else 0 end) as aug_sales ,sum(case when d_moy = 9 then ws_sales_price* ws_quantity else 0 end) as sep_sales ,sum(case when d_moy = 10 then ws_sales_price* ws_quantity else 0 end) as oct_sales ,sum(case when d_moy = 11 then ws_sales_price* ws_quantity else 0 end) as nov_sales ,sum(case when d_moy = 12 then ws_sales_price* ws_quantity else 0 end) as dec_sales ,sum(case when d_moy = 1 then ws_net_paid_inc_tax * ws_quantity else 0 end) as jan_net ,sum(case when d_moy = 2 then ws_net_paid_inc_tax * ws_quantity else 0 end) as feb_net ,sum(case when d_moy = 3 then ws_net_paid_inc_tax * ws_quantity else 0 end) as mar_net ,sum(case when d_moy = 4 then ws_net_paid_inc_tax * ws_quantity else 0 end) as apr_net ,sum(case when d_moy = 5 then ws_net_paid_inc_tax * ws_quantity else 0 end) as may_net ,sum(case when d_moy = 6 then ws_net_paid_inc_tax * ws_quantity else 0 end) as jun_net ,sum(case when d_moy = 7 then ws_net_paid_inc_tax * ws_quantity else 0 end) as jul_net ,sum(case when d_moy = 8 then ws_net_paid_inc_tax * ws_quantity else 0 end) as aug_net ,sum(case when d_moy = 9 then ws_net_paid_inc_tax * ws_quantity else 0 end) as sep_net ,sum(case when d_moy = 10 then ws_net_paid_inc_tax * ws_quantity else 0 end) as oct_net ,sum(case when d_moy = 11 then ws_net_paid_inc_tax * ws_quantity else 0 end) as nov_net ,sum(case when d_moy = 12 then ws_net_paid_inc_tax * ws_quantity else 0 end) as dec_net from web_sales ,warehouse ,date_dim ,time_dim ,ship_mode where ws_warehouse_sk = w_warehouse_sk and ws_sold_date_sk = d_date_sk and ws_sold_time_sk = t_time_sk and ws_ship_mode_sk = sm_ship_mode_sk and d_year = 2002 and t_time between 24285 and 24285+28800 and sm_carrier in ('HARMSTORF','USPS') group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,d_year union all select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,'HARMSTORF' || ',' || 'USPS' as ship_carriers ,d_year as year ,sum(case when d_moy = 1 then cs_ext_list_price* cs_quantity else 0 end) as jan_sales ,sum(case when d_moy = 2 then cs_ext_list_price* cs_quantity else 0 end) as feb_sales ,sum(case when d_moy = 3 then cs_ext_list_price* cs_quantity else 0 end) as mar_sales ,sum(case when d_moy = 4 then cs_ext_list_price* cs_quantity else 0 end) as apr_sales ,sum(case when d_moy = 5 then cs_ext_list_price* cs_quantity else 0 end) as may_sales ,sum(case when d_moy = 6 then cs_ext_list_price* cs_quantity else 0 end) as jun_sales ,sum(case when d_moy = 7 then cs_ext_list_price* cs_quantity else 0 end) as jul_sales ,sum(case when d_moy = 8 then cs_ext_list_price* cs_quantity else 0 end) as aug_sales ,sum(case when d_moy = 9 then cs_ext_list_price* cs_quantity else 0 end) as sep_sales ,sum(case when d_moy = 10 then cs_ext_list_price* cs_quantity else 0 end) as oct_sales ,sum(case when d_moy = 11 then cs_ext_list_price* cs_quantity else 0 end) as nov_sales ,sum(case when d_moy = 12 then cs_ext_list_price* cs_quantity else 0 end) as dec_sales ,sum(case when d_moy = 1 then cs_net_paid * cs_quantity else 0 end) as jan_net ,sum(case when d_moy = 2 then cs_net_paid * cs_quantity else 0 end) as feb_net ,sum(case when d_moy = 3 then cs_net_paid * cs_quantity else 0 end) as mar_net ,sum(case when d_moy = 4 then cs_net_paid * cs_quantity else 0 end) as apr_net ,sum(case when d_moy = 5 then cs_net_paid * cs_quantity else 0 end) as may_net ,sum(case when d_moy = 6 then cs_net_paid * cs_quantity else 0 end) as jun_net ,sum(case when d_moy = 7 then cs_net_paid * cs_quantity else 0 end) as jul_net ,sum(case when d_moy = 8 then cs_net_paid * cs_quantity else 0 end) as aug_net ,sum(case when d_moy = 9 then cs_net_paid * cs_quantity else 0 end) as sep_net ,sum(case when d_moy = 10 then cs_net_paid * cs_quantity else 0 end) as oct_net ,sum(case when d_moy = 11 then cs_net_paid * cs_quantity else 0 end) as nov_net ,sum(case when d_moy = 12 then cs_net_paid * cs_quantity else 0 end) as dec_net from catalog_sales ,warehouse ,date_dim ,time_dim ,ship_mode where cs_warehouse_sk = w_warehouse_sk and cs_sold_date_sk = d_date_sk and cs_sold_time_sk = t_time_sk and cs_ship_mode_sk = sm_ship_mode_sk and d_year = 2002 and t_time between 24285 AND 24285+28800 and sm_carrier in ('HARMSTORF','USPS') group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,d_year ) x group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,ship_carriers ,year order by w_warehouse_name limit 100""", "q67" -> """ select * from (select i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sumsales ,rank() over (partition by i_category order by sumsales desc) rk from (select i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sum(coalesce(ss_sales_price*ss_quantity,0)) sumsales from store_sales ,date_dim ,store ,item where ss_sold_date_sk=d_date_sk and ss_item_sk=i_item_sk and ss_store_sk = s_store_sk and d_month_seq between 1196 and 1196+11 group by rollup(i_category, i_class, i_brand, i_product_name, d_year, d_qoy, d_moy,s_store_id))dw1) dw2 where rk <= 100 order by i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sumsales ,rk limit 100""", "q68" -> """ select c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number ,extended_price ,extended_tax ,list_price from (select ss_ticket_number ,ss_customer_sk ,ca_city bought_city ,sum(ss_ext_sales_price) extended_price ,sum(ss_ext_list_price) list_price ,sum(ss_ext_tax) extended_tax from store_sales ,date_dim ,store ,household_demographics ,customer_address where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and store_sales.ss_addr_sk = customer_address.ca_address_sk and date_dim.d_dom between 1 and 2 and (household_demographics.hd_dep_count = 1 or household_demographics.hd_vehicle_count= -1) and date_dim.d_year in (1998,1998+1,1998+2) and store.s_city in ('Bethel','Summit') group by ss_ticket_number ,ss_customer_sk ,ss_addr_sk,ca_city) dn ,customer ,customer_address current_addr where ss_customer_sk = c_customer_sk and customer.c_current_addr_sk = current_addr.ca_address_sk and current_addr.ca_city <> bought_city order by c_last_name ,ss_ticket_number limit 100""", "q69" -> """ select cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3 from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and ca_state in ('OK','GA','VA') and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2004 and d_moy between 4 and 4+2) and (not exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2004 and d_moy between 4 and 4+2) and not exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2004 and d_moy between 4 and 4+2)) group by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating order by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating limit 100""", "q70" -> """ select sum(ss_net_profit) as total_sum ,s_state ,s_county ,grouping(s_state)+grouping(s_county) as lochierarchy ,rank() over ( partition by grouping(s_state)+grouping(s_county), case when grouping(s_county) = 0 then s_state end order by sum(ss_net_profit) desc) as rank_within_parent from store_sales ,date_dim d1 ,store where d1.d_month_seq between 1197 and 1197+11 and d1.d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk and s_state in ( select s_state from (select s_state as s_state, rank() over ( partition by s_state order by sum(ss_net_profit) desc) as ranking from store_sales, store, date_dim where d_month_seq between 1197 and 1197+11 and d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk group by s_state ) tmp1 where ranking <= 5 ) group by rollup(s_state,s_county) order by lochierarchy desc ,case when lochierarchy = 0 then s_state end ,rank_within_parent limit 100""", "q71" -> """ select i_brand_id brand_id, i_brand brand,t_hour,t_minute, sum(ext_price) ext_price from item, (select ws_ext_sales_price as ext_price, ws_sold_date_sk as sold_date_sk, ws_item_sk as sold_item_sk, ws_sold_time_sk as time_sk from web_sales,date_dim where d_date_sk = ws_sold_date_sk and d_moy=12 and d_year=1999 union all select cs_ext_sales_price as ext_price, cs_sold_date_sk as sold_date_sk, cs_item_sk as sold_item_sk, cs_sold_time_sk as time_sk from catalog_sales,date_dim where d_date_sk = cs_sold_date_sk and d_moy=12 and d_year=1999 union all select ss_ext_sales_price as ext_price, ss_sold_date_sk as sold_date_sk, ss_item_sk as sold_item_sk, ss_sold_time_sk as time_sk from store_sales,date_dim where d_date_sk = ss_sold_date_sk and d_moy=12 and d_year=1999 ) tmp,time_dim where sold_item_sk = i_item_sk and i_manager_id=1 and time_sk = t_time_sk and (t_meal_time = 'breakfast' or t_meal_time = 'dinner') group by i_brand, i_brand_id,t_hour,t_minute order by ext_price desc, i_brand_id """, "q72" -> """ select i_item_desc ,w_warehouse_name ,d1.d_week_seq ,sum(case when p_promo_sk is null then 1 else 0 end) no_promo ,sum(case when p_promo_sk is not null then 1 else 0 end) promo ,count(*) total_cnt from catalog_sales join inventory on (cs_item_sk = inv_item_sk) join warehouse on (w_warehouse_sk=inv_warehouse_sk) join item on (i_item_sk = cs_item_sk) join customer_demographics on (cs_bill_cdemo_sk = cd_demo_sk) join household_demographics on (cs_bill_hdemo_sk = hd_demo_sk) join date_dim d1 on (cs_sold_date_sk = d1.d_date_sk) join date_dim d2 on (inv_date_sk = d2.d_date_sk) join date_dim d3 on (cs_ship_date_sk = d3.d_date_sk) left outer join promotion on (cs_promo_sk=p_promo_sk) left outer join catalog_returns on (cr_item_sk = cs_item_sk and cr_order_number = cs_order_number) where d1.d_week_seq = d2.d_week_seq and inv_quantity_on_hand < cs_quantity and d3.d_date > d1.d_date + interval 5 days and hd_buy_potential = '>10000' and d1.d_year = 2002 and cd_marital_status = 'D' group by i_item_desc,w_warehouse_name,d1.d_week_seq order by total_cnt desc, i_item_desc, w_warehouse_name, d_week_seq limit 100""", "q73" -> """ select c_last_name ,c_first_name ,c_salutation ,c_preferred_cust_flag ,ss_ticket_number ,cnt from (select ss_ticket_number ,ss_customer_sk ,count(*) cnt from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and date_dim.d_dom between 1 and 2 and (household_demographics.hd_buy_potential = '501-1000' or household_demographics.hd_buy_potential = 'Unknown') and household_demographics.hd_vehicle_count > 0 and case when household_demographics.hd_vehicle_count > 0 then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end > 1 and date_dim.d_year in (1999,1999+1,1999+2) and store.s_county in ('Franklin Parish','Ziebach County','Luce County','Williamson County') group by ss_ticket_number,ss_customer_sk) dj,customer where ss_customer_sk = c_customer_sk and cnt between 1 and 5 order by cnt desc, c_last_name asc""", "q74" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,d_year as year ,max(ss_net_paid) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (2001,2001+1) group by c_customer_id ,c_first_name ,c_last_name ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,d_year as year ,max(ws_net_paid) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year in (2001,2001+1) group by c_customer_id ,c_first_name ,c_last_name ,d_year ) select t_s_secyear.customer_id, t_s_secyear.customer_first_name, t_s_secyear.customer_last_name from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.sale_type = 's' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_w_secyear.sale_type = 'w' and t_s_firstyear.year = 2001 and t_s_secyear.year = 2001+1 and t_w_firstyear.year = 2001 and t_w_secyear.year = 2001+1 and t_s_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end order by 3,1,2 limit 100""", "q75" -> """ WITH all_sales AS ( SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,SUM(sales_cnt) AS sales_cnt ,SUM(sales_amt) AS sales_amt FROM (SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,cs_quantity - COALESCE(cr_return_quantity,0) AS sales_cnt ,cs_ext_sales_price - COALESCE(cr_return_amount,0.0) AS sales_amt FROM catalog_sales JOIN item ON i_item_sk=cs_item_sk JOIN date_dim ON d_date_sk=cs_sold_date_sk LEFT JOIN catalog_returns ON (cs_order_number=cr_order_number AND cs_item_sk=cr_item_sk) WHERE i_category='Sports' UNION SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,ss_quantity - COALESCE(sr_return_quantity,0) AS sales_cnt ,ss_ext_sales_price - COALESCE(sr_return_amt,0.0) AS sales_amt FROM store_sales JOIN item ON i_item_sk=ss_item_sk JOIN date_dim ON d_date_sk=ss_sold_date_sk LEFT JOIN store_returns ON (ss_ticket_number=sr_ticket_number AND ss_item_sk=sr_item_sk) WHERE i_category='Sports' UNION SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,ws_quantity - COALESCE(wr_return_quantity,0) AS sales_cnt ,ws_ext_sales_price - COALESCE(wr_return_amt,0.0) AS sales_amt FROM web_sales JOIN item ON i_item_sk=ws_item_sk JOIN date_dim ON d_date_sk=ws_sold_date_sk LEFT JOIN web_returns ON (ws_order_number=wr_order_number AND ws_item_sk=wr_item_sk) WHERE i_category='Sports') sales_detail GROUP BY d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id) SELECT prev_yr.d_year AS prev_year ,curr_yr.d_year AS year ,curr_yr.i_brand_id ,curr_yr.i_class_id ,curr_yr.i_category_id ,curr_yr.i_manufact_id ,prev_yr.sales_cnt AS prev_yr_cnt ,curr_yr.sales_cnt AS curr_yr_cnt ,curr_yr.sales_cnt-prev_yr.sales_cnt AS sales_cnt_diff ,curr_yr.sales_amt-prev_yr.sales_amt AS sales_amt_diff FROM all_sales curr_yr, all_sales prev_yr WHERE curr_yr.i_brand_id=prev_yr.i_brand_id AND curr_yr.i_class_id=prev_yr.i_class_id AND curr_yr.i_category_id=prev_yr.i_category_id AND curr_yr.i_manufact_id=prev_yr.i_manufact_id AND curr_yr.d_year=2001 AND prev_yr.d_year=2001-1 AND CAST(curr_yr.sales_cnt AS DECIMAL(17,2))/CAST(prev_yr.sales_cnt AS DECIMAL(17,2))<0.9 ORDER BY sales_cnt_diff,sales_amt_diff limit 100""", "q76" -> """ select channel, col_name, d_year, d_qoy, i_category, COUNT(*) sales_cnt, SUM(ext_sales_price) sales_amt FROM ( SELECT 'store' as channel, 'ss_cdemo_sk' col_name, d_year, d_qoy, i_category, ss_ext_sales_price ext_sales_price FROM store_sales, item, date_dim WHERE ss_cdemo_sk IS NULL AND ss_sold_date_sk=d_date_sk AND ss_item_sk=i_item_sk UNION ALL SELECT 'web' as channel, 'ws_ship_hdemo_sk' col_name, d_year, d_qoy, i_category, ws_ext_sales_price ext_sales_price FROM web_sales, item, date_dim WHERE ws_ship_hdemo_sk IS NULL AND ws_sold_date_sk=d_date_sk AND ws_item_sk=i_item_sk UNION ALL SELECT 'catalog' as channel, 'cs_ship_customer_sk' col_name, d_year, d_qoy, i_category, cs_ext_sales_price ext_sales_price FROM catalog_sales, item, date_dim WHERE cs_ship_customer_sk IS NULL AND cs_sold_date_sk=d_date_sk AND cs_item_sk=i_item_sk) foo GROUP BY channel, col_name, d_year, d_qoy, i_category ORDER BY channel, col_name, d_year, d_qoy, i_category limit 100""", "q77" -> """ with ss as (select s_store_sk, sum(ss_ext_sales_price) as sales, sum(ss_net_profit) as profit from store_sales, date_dim, store where ss_sold_date_sk = d_date_sk and d_date between cast('2001-08-27' as date) and (cast('2001-08-27' as date) + INTERVAL 30 days) and ss_store_sk = s_store_sk group by s_store_sk) , sr as (select s_store_sk, sum(sr_return_amt) as returns, sum(sr_net_loss) as profit_loss from store_returns, date_dim, store where sr_returned_date_sk = d_date_sk and d_date between cast('2001-08-27' as date) and (cast('2001-08-27' as date) + INTERVAL 30 days) and sr_store_sk = s_store_sk group by s_store_sk), cs as (select cs_call_center_sk, sum(cs_ext_sales_price) as sales, sum(cs_net_profit) as profit from catalog_sales, date_dim where cs_sold_date_sk = d_date_sk and d_date between cast('2001-08-27' as date) and (cast('2001-08-27' as date) + INTERVAL 30 days) group by cs_call_center_sk ), cr as (select cr_call_center_sk, sum(cr_return_amount) as returns, sum(cr_net_loss) as profit_loss from catalog_returns, date_dim where cr_returned_date_sk = d_date_sk and d_date between cast('2001-08-27' as date) and (cast('2001-08-27' as date) + INTERVAL 30 days) group by cr_call_center_sk ), ws as ( select wp_web_page_sk, sum(ws_ext_sales_price) as sales, sum(ws_net_profit) as profit from web_sales, date_dim, web_page where ws_sold_date_sk = d_date_sk and d_date between cast('2001-08-27' as date) and (cast('2001-08-27' as date) + INTERVAL 30 days) and ws_web_page_sk = wp_web_page_sk group by wp_web_page_sk), wr as (select wp_web_page_sk, sum(wr_return_amt) as returns, sum(wr_net_loss) as profit_loss from web_returns, date_dim, web_page where wr_returned_date_sk = d_date_sk and d_date between cast('2001-08-27' as date) and (cast('2001-08-27' as date) + INTERVAL 30 days) and wr_web_page_sk = wp_web_page_sk group by wp_web_page_sk) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , ss.s_store_sk as id , sales , coalesce(returns, 0) as returns , (profit - coalesce(profit_loss,0)) as profit from ss left join sr on ss.s_store_sk = sr.s_store_sk union all select 'catalog channel' as channel , cs_call_center_sk as id , sales , returns , (profit - profit_loss) as profit from cs , cr union all select 'web channel' as channel , ws.wp_web_page_sk as id , sales , coalesce(returns, 0) returns , (profit - coalesce(profit_loss,0)) as profit from ws left join wr on ws.wp_web_page_sk = wr.wp_web_page_sk ) x group by rollup (channel, id) order by channel ,id limit 100""", "q78" -> """ with ws as (select d_year AS ws_sold_year, ws_item_sk, ws_bill_customer_sk ws_customer_sk, sum(ws_quantity) ws_qty, sum(ws_wholesale_cost) ws_wc, sum(ws_sales_price) ws_sp from web_sales left join web_returns on wr_order_number=ws_order_number and ws_item_sk=wr_item_sk join date_dim on ws_sold_date_sk = d_date_sk where wr_order_number is null group by d_year, ws_item_sk, ws_bill_customer_sk ), cs as (select d_year AS cs_sold_year, cs_item_sk, cs_bill_customer_sk cs_customer_sk, sum(cs_quantity) cs_qty, sum(cs_wholesale_cost) cs_wc, sum(cs_sales_price) cs_sp from catalog_sales left join catalog_returns on cr_order_number=cs_order_number and cs_item_sk=cr_item_sk join date_dim on cs_sold_date_sk = d_date_sk where cr_order_number is null group by d_year, cs_item_sk, cs_bill_customer_sk ), ss as (select d_year AS ss_sold_year, ss_item_sk, ss_customer_sk, sum(ss_quantity) ss_qty, sum(ss_wholesale_cost) ss_wc, sum(ss_sales_price) ss_sp from store_sales left join store_returns on sr_ticket_number=ss_ticket_number and ss_item_sk=sr_item_sk join date_dim on ss_sold_date_sk = d_date_sk where sr_ticket_number is null group by d_year, ss_item_sk, ss_customer_sk ) select ss_sold_year, ss_item_sk, ss_customer_sk, round(ss_qty/(coalesce(ws_qty,0)+coalesce(cs_qty,0)),2) ratio, ss_qty store_qty, ss_wc store_wholesale_cost, ss_sp store_sales_price, coalesce(ws_qty,0)+coalesce(cs_qty,0) other_chan_qty, coalesce(ws_wc,0)+coalesce(cs_wc,0) other_chan_wholesale_cost, coalesce(ws_sp,0)+coalesce(cs_sp,0) other_chan_sales_price from ss left join ws on (ws_sold_year=ss_sold_year and ws_item_sk=ss_item_sk and ws_customer_sk=ss_customer_sk) left join cs on (cs_sold_year=ss_sold_year and cs_item_sk=ss_item_sk and cs_customer_sk=ss_customer_sk) where (coalesce(ws_qty,0)>0 or coalesce(cs_qty, 0)>0) and ss_sold_year=2002 order by ss_sold_year, ss_item_sk, ss_customer_sk, ss_qty desc, ss_wc desc, ss_sp desc, other_chan_qty, other_chan_wholesale_cost, other_chan_sales_price, ratio limit 100""", "q79" -> """ select c_last_name,c_first_name,substr(s_city,1,30),ss_ticket_number,amt,profit from (select ss_ticket_number ,ss_customer_sk ,store.s_city ,sum(ss_coupon_amt) amt ,sum(ss_net_profit) profit from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and (household_demographics.hd_dep_count = 0 or household_demographics.hd_vehicle_count > 0) and date_dim.d_dow = 1 and date_dim.d_year in (2000,2000+1,2000+2) and store.s_number_employees between 200 and 295 group by ss_ticket_number,ss_customer_sk,ss_addr_sk,store.s_city) ms,customer where ss_customer_sk = c_customer_sk order by c_last_name,c_first_name,substr(s_city,1,30), profit limit 100""", "q80" -> """ with ssr as (select s_store_id as store_id, sum(ss_ext_sales_price) as sales, sum(coalesce(sr_return_amt, 0)) as returns, sum(ss_net_profit - coalesce(sr_net_loss, 0)) as profit from store_sales left outer join store_returns on (ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number), date_dim, store, item, promotion where ss_sold_date_sk = d_date_sk and d_date between cast('1999-08-12' as date) and (cast('1999-08-12' as date) + INTERVAL 60 days) and ss_store_sk = s_store_sk and ss_item_sk = i_item_sk and i_current_price > 50 and ss_promo_sk = p_promo_sk and p_channel_tv = 'N' group by s_store_id) , csr as (select cp_catalog_page_id as catalog_page_id, sum(cs_ext_sales_price) as sales, sum(coalesce(cr_return_amount, 0)) as returns, sum(cs_net_profit - coalesce(cr_net_loss, 0)) as profit from catalog_sales left outer join catalog_returns on (cs_item_sk = cr_item_sk and cs_order_number = cr_order_number), date_dim, catalog_page, item, promotion where cs_sold_date_sk = d_date_sk and d_date between cast('1999-08-12' as date) and (cast('1999-08-12' as date) + INTERVAL 60 days) and cs_catalog_page_sk = cp_catalog_page_sk and cs_item_sk = i_item_sk and i_current_price > 50 and cs_promo_sk = p_promo_sk and p_channel_tv = 'N' group by cp_catalog_page_id) , wsr as (select web_site_id, sum(ws_ext_sales_price) as sales, sum(coalesce(wr_return_amt, 0)) as returns, sum(ws_net_profit - coalesce(wr_net_loss, 0)) as profit from web_sales left outer join web_returns on (ws_item_sk = wr_item_sk and ws_order_number = wr_order_number), date_dim, web_site, item, promotion where ws_sold_date_sk = d_date_sk and d_date between cast('1999-08-12' as date) and (cast('1999-08-12' as date) + INTERVAL 60 days) and ws_web_site_sk = web_site_sk and ws_item_sk = i_item_sk and i_current_price > 50 and ws_promo_sk = p_promo_sk and p_channel_tv = 'N' group by web_site_id) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , 'store' || store_id as id , sales , returns , profit from ssr union all select 'catalog channel' as channel , 'catalog_page' || catalog_page_id as id , sales , returns , profit from csr union all select 'web channel' as channel , 'web_site' || web_site_id as id , sales , returns , profit from wsr ) x group by rollup (channel, id) order by channel ,id limit 100""", "q81" -> """ with customer_total_return as (select cr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, sum(cr_return_amt_inc_tax) as ctr_total_return from catalog_returns ,date_dim ,customer_address where cr_returned_date_sk = d_date_sk and d_year =2001 and cr_returning_addr_sk = ca_address_sk group by cr_returning_customer_sk ,ca_state ) select c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset ,ca_location_type,ctr_total_return from customer_total_return ctr1 ,customer_address ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_state = ctr2.ctr_state) and ca_address_sk = c_current_addr_sk and ca_state = 'NC' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset ,ca_location_type,ctr_total_return limit 100""", "q82" -> """ select i_item_id ,i_item_desc ,i_current_price from item, inventory, date_dim, store_sales where i_current_price between 82 and 82+30 and inv_item_sk = i_item_sk and d_date_sk=inv_date_sk and d_date between cast('2002-03-10' as date) and (cast('2002-03-10' as date) + INTERVAL 60 days) and i_manufact_id in (941,920,105,693) and inv_quantity_on_hand between 100 and 500 and ss_item_sk = i_item_sk group by i_item_id,i_item_desc,i_current_price order by i_item_id limit 100""", "q83" -> """ with sr_items as (select i_item_id item_id, sum(sr_return_quantity) sr_item_qty from store_returns, item, date_dim where sr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('1999-04-14','1999-09-28','1999-11-12'))) and sr_returned_date_sk = d_date_sk group by i_item_id), cr_items as (select i_item_id item_id, sum(cr_return_quantity) cr_item_qty from catalog_returns, item, date_dim where cr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('1999-04-14','1999-09-28','1999-11-12'))) and cr_returned_date_sk = d_date_sk group by i_item_id), wr_items as (select i_item_id item_id, sum(wr_return_quantity) wr_item_qty from web_returns, item, date_dim where wr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('1999-04-14','1999-09-28','1999-11-12'))) and wr_returned_date_sk = d_date_sk group by i_item_id) select sr_items.item_id ,sr_item_qty ,sr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 sr_dev ,cr_item_qty ,cr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 cr_dev ,wr_item_qty ,wr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 wr_dev ,(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 average from sr_items ,cr_items ,wr_items where sr_items.item_id=cr_items.item_id and sr_items.item_id=wr_items.item_id order by sr_items.item_id ,sr_item_qty limit 100""", "q84" -> """ select c_customer_id as customer_id , coalesce(c_last_name,'') || ', ' || coalesce(c_first_name,'') as customername from customer ,customer_address ,customer_demographics ,household_demographics ,income_band ,store_returns where ca_city = 'Antioch' and c_current_addr_sk = ca_address_sk and ib_lower_bound >= 55019 and ib_upper_bound <= 55019 + 50000 and ib_income_band_sk = hd_income_band_sk and cd_demo_sk = c_current_cdemo_sk and hd_demo_sk = c_current_hdemo_sk and sr_cdemo_sk = cd_demo_sk order by c_customer_id limit 100""", "q85" -> """ select substr(r_reason_desc,1,20) ,avg(ws_quantity) ,avg(wr_refunded_cash) ,avg(wr_fee) from web_sales, web_returns, web_page, customer_demographics cd1, customer_demographics cd2, customer_address, date_dim, reason where ws_web_page_sk = wp_web_page_sk and ws_item_sk = wr_item_sk and ws_order_number = wr_order_number and ws_sold_date_sk = d_date_sk and d_year = 2001 and cd1.cd_demo_sk = wr_refunded_cdemo_sk and cd2.cd_demo_sk = wr_returning_cdemo_sk and ca_address_sk = wr_refunded_addr_sk and r_reason_sk = wr_reason_sk and ( ( cd1.cd_marital_status = 'S' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = '2 yr Degree' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 100.00 and 150.00 ) or ( cd1.cd_marital_status = 'D' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = 'Advanced Degree' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 50.00 and 100.00 ) or ( cd1.cd_marital_status = 'W' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = '4 yr Degree' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 150.00 and 200.00 ) ) and ( ( ca_country = 'United States' and ca_state in ('OK', 'TX', 'MO') and ws_net_profit between 100 and 200 ) or ( ca_country = 'United States' and ca_state in ('GA', 'KS', 'NC') and ws_net_profit between 150 and 300 ) or ( ca_country = 'United States' and ca_state in ('VA', 'WI', 'WV') and ws_net_profit between 50 and 250 ) ) group by r_reason_desc order by substr(r_reason_desc,1,20) ,avg(ws_quantity) ,avg(wr_refunded_cash) ,avg(wr_fee) limit 100""", "q86" -> """ select sum(ws_net_paid) as total_sum ,i_category ,i_class ,grouping(i_category)+grouping(i_class) as lochierarchy ,rank() over ( partition by grouping(i_category)+grouping(i_class), case when grouping(i_class) = 0 then i_category end order by sum(ws_net_paid) desc) as rank_within_parent from web_sales ,date_dim d1 ,item where d1.d_month_seq between 1180 and 1180+11 and d1.d_date_sk = ws_sold_date_sk and i_item_sk = ws_item_sk group by rollup(i_category,i_class) order by lochierarchy desc, case when lochierarchy = 0 then i_category end, rank_within_parent limit 100""", "q87" -> """ select count(*) from ((select distinct c_last_name, c_first_name, d_date from store_sales, date_dim, customer where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_customer_sk = customer.c_customer_sk and d_month_seq between 1204 and 1204+11) except (select distinct c_last_name, c_first_name, d_date from catalog_sales, date_dim, customer where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1204 and 1204+11) except (select distinct c_last_name, c_first_name, d_date from web_sales, date_dim, customer where web_sales.ws_sold_date_sk = date_dim.d_date_sk and web_sales.ws_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1204 and 1204+11) ) cool_cust""", "q88" -> """ select * from (select count(*) h8_30_to_9 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 8 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) and store.s_store_name = 'ese') s1, (select count(*) h9_to_9_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 9 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) and store.s_store_name = 'ese') s2, (select count(*) h9_30_to_10 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 9 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) and store.s_store_name = 'ese') s3, (select count(*) h10_to_10_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 10 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) and store.s_store_name = 'ese') s4, (select count(*) h10_30_to_11 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 10 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) and store.s_store_name = 'ese') s5, (select count(*) h11_to_11_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 11 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) and store.s_store_name = 'ese') s6, (select count(*) h11_30_to_12 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 11 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) and store.s_store_name = 'ese') s7, (select count(*) h12_to_12_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 12 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) and store.s_store_name = 'ese') s8""", "q89" -> """ select * from( select i_category, i_class, i_brand, s_store_name, s_company_name, d_moy, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_category, i_brand, s_store_name, s_company_name) avg_monthly_sales from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_year in (2001) and ((i_category in ('Women','Music','Home') and i_class in ('fragrances','pop','bedding') ) or (i_category in ('Books','Men','Children') and i_class in ('home repair','sports-apparel','infants') )) group by i_category, i_class, i_brand, s_store_name, s_company_name, d_moy) tmp1 where case when (avg_monthly_sales <> 0) then (abs(sum_sales - avg_monthly_sales) / avg_monthly_sales) else null end > 0.1 order by sum_sales - avg_monthly_sales, s_store_name limit 100""", "q90" -> """ select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio from ( select count(*) amc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 8 and 8+1 and household_demographics.hd_dep_count = 4 and web_page.wp_char_count between 5000 and 5200) at, ( select count(*) pmc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 19 and 19+1 and household_demographics.hd_dep_count = 4 and web_page.wp_char_count between 5000 and 5200) pt order by am_pm_ratio limit 100""", "q91" -> """ select cc_call_center_id Call_Center, cc_name Call_Center_Name, cc_manager Manager, sum(cr_net_loss) Returns_Loss from call_center, catalog_returns, date_dim, customer, customer_address, customer_demographics, household_demographics where cr_call_center_sk = cc_call_center_sk and cr_returned_date_sk = d_date_sk and cr_returning_customer_sk= c_customer_sk and cd_demo_sk = c_current_cdemo_sk and hd_demo_sk = c_current_hdemo_sk and ca_address_sk = c_current_addr_sk and d_year = 2002 and d_moy = 11 and ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') or(cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) and hd_buy_potential like '5001-10000%' and ca_gmt_offset = -6 group by cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status order by sum(cr_net_loss) desc""", "q92" -> """ select sum(ws_ext_discount_amt) as `Excess Discount Amount` from web_sales ,item ,date_dim where i_manufact_id = 561 and i_item_sk = ws_item_sk and d_date between '2001-03-13' and (cast('2001-03-13' as date) + INTERVAL 90 days) and d_date_sk = ws_sold_date_sk and ws_ext_discount_amt > ( SELECT 1.3 * avg(ws_ext_discount_amt) FROM web_sales ,date_dim WHERE ws_item_sk = i_item_sk and d_date between '2001-03-13' and (cast('2001-03-13' as date) + INTERVAL 90 days) and d_date_sk = ws_sold_date_sk ) order by sum(ws_ext_discount_amt) limit 100""", "q93" -> """ select ss_customer_sk ,sum(act_sales) sumsales from (select ss_item_sk ,ss_ticket_number ,ss_customer_sk ,case when sr_return_quantity is not null then (ss_quantity-sr_return_quantity)*ss_sales_price else (ss_quantity*ss_sales_price) end act_sales from store_sales left outer join store_returns on (sr_item_sk = ss_item_sk and sr_ticket_number = ss_ticket_number) ,reason where sr_reason_sk = r_reason_sk and r_reason_desc = 'reason 64') t group by ss_customer_sk order by sumsales, ss_customer_sk limit 100""", "q94" -> """ select count(distinct ws_order_number) as `order count` ,sum(ws_ext_ship_cost) as `total shipping cost` ,sum(ws_net_profit) as `total net profit` from web_sales ws1 ,date_dim ,customer_address ,web_site where d_date between '2001-5-01' and (cast('2001-5-01' as date) + INTERVAL 60 days) and ws1.ws_ship_date_sk = d_date_sk and ws1.ws_ship_addr_sk = ca_address_sk and ca_state = 'TX' and ws1.ws_web_site_sk = web_site_sk and web_company_name = 'pri' and exists (select * from web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) and not exists(select * from web_returns wr1 where ws1.ws_order_number = wr1.wr_order_number) order by count(distinct ws_order_number) limit 100""", "q95" -> """ with ws_wh as (select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 from web_sales ws1,web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) select count(distinct ws_order_number) as `order count` ,sum(ws_ext_ship_cost) as `total shipping cost` ,sum(ws_net_profit) as `total net profit` from web_sales ws1 ,date_dim ,customer_address ,web_site where d_date between '2000-3-01' and (cast('2000-3-01' as date) + INTERVAL 60 days) and ws1.ws_ship_date_sk = d_date_sk and ws1.ws_ship_addr_sk = ca_address_sk and ca_state = 'TN' and ws1.ws_web_site_sk = web_site_sk and web_company_name = 'pri' and ws1.ws_order_number in (select ws_order_number from ws_wh) and ws1.ws_order_number in (select wr_order_number from web_returns,ws_wh where wr_order_number = ws_wh.ws_order_number) order by count(distinct ws_order_number) limit 100""", "q96" -> """ select count(*) from store_sales ,household_demographics ,time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 16 and time_dim.t_minute >= 30 and household_demographics.hd_dep_count = 4 and store.s_store_name = 'ese' order by count(*) limit 100""", "q97" -> """ with ssci as ( select ss_customer_sk customer_sk ,ss_item_sk item_sk from store_sales,date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1209 and 1209 + 11 group by ss_customer_sk ,ss_item_sk), csci as( select cs_bill_customer_sk customer_sk ,cs_item_sk item_sk from catalog_sales,date_dim where cs_sold_date_sk = d_date_sk and d_month_seq between 1209 and 1209 + 11 group by cs_bill_customer_sk ,cs_item_sk) select sum(case when ssci.customer_sk is not null and csci.customer_sk is null then 1 else 0 end) store_only ,sum(case when ssci.customer_sk is null and csci.customer_sk is not null then 1 else 0 end) catalog_only ,sum(case when ssci.customer_sk is not null and csci.customer_sk is not null then 1 else 0 end) store_and_catalog from ssci full outer join csci on (ssci.customer_sk=csci.customer_sk and ssci.item_sk = csci.item_sk) limit 100""", "q98" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(ss_ext_sales_price) as itemrevenue ,sum(ss_ext_sales_price)*100/sum(sum(ss_ext_sales_price)) over (partition by i_class) as revenueratio from store_sales ,item ,date_dim where ss_item_sk = i_item_sk and i_category in ('Jewelry', 'Home', 'Shoes') and ss_sold_date_sk = d_date_sk and d_date between cast('2001-04-12' as date) and (cast('2001-04-12' as date) + interval 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio""", "q99" -> """ select substr(w_warehouse_name,1,20) ,sm_type ,cc_name ,sum(case when (cs_ship_date_sk - cs_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 30) and (cs_ship_date_sk - cs_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 60) and (cs_ship_date_sk - cs_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 90) and (cs_ship_date_sk - cs_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from catalog_sales ,warehouse ,ship_mode ,call_center ,date_dim where d_month_seq between 1203 and 1203 + 11 and cs_ship_date_sk = d_date_sk and cs_warehouse_sk = w_warehouse_sk and cs_ship_mode_sk = sm_ship_mode_sk and cs_call_center_sk = cc_call_center_sk group by substr(w_warehouse_name,1,20) ,sm_type ,cc_name order by substr(w_warehouse_name,1,20) ,sm_type ,cc_name limit 100""", "q1" -> """ with customer_total_return as (select sr_customer_sk as ctr_customer_sk ,sr_store_sk as ctr_store_sk ,sum(SR_FEE) as ctr_total_return from store_returns ,date_dim where sr_returned_date_sk = d_date_sk and d_year =2000 group by sr_customer_sk ,sr_store_sk) select c_customer_id from customer_total_return ctr1 ,store ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_store_sk = ctr2.ctr_store_sk) and s_store_sk = ctr1.ctr_store_sk and s_state = 'NM' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id limit 100""", "q2" -> """ with wscs as (select sold_date_sk ,sales_price from (select ws_sold_date_sk sold_date_sk ,ws_ext_sales_price sales_price from web_sales union all select cs_sold_date_sk sold_date_sk ,cs_ext_sales_price sales_price from catalog_sales)), wswscs as (select d_week_seq, sum(case when (d_day_name='Sunday') then sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then sales_price else null end) sat_sales from wscs ,date_dim where d_date_sk = sold_date_sk group by d_week_seq) select d_week_seq1 ,round(sun_sales1/sun_sales2,2) ,round(mon_sales1/mon_sales2,2) ,round(tue_sales1/tue_sales2,2) ,round(wed_sales1/wed_sales2,2) ,round(thu_sales1/thu_sales2,2) ,round(fri_sales1/fri_sales2,2) ,round(sat_sales1/sat_sales2,2) from (select wswscs.d_week_seq d_week_seq1 ,sun_sales sun_sales1 ,mon_sales mon_sales1 ,tue_sales tue_sales1 ,wed_sales wed_sales1 ,thu_sales thu_sales1 ,fri_sales fri_sales1 ,sat_sales sat_sales1 from wswscs,date_dim where date_dim.d_week_seq = wswscs.d_week_seq and d_year = 1998) y, (select wswscs.d_week_seq d_week_seq2 ,sun_sales sun_sales2 ,mon_sales mon_sales2 ,tue_sales tue_sales2 ,wed_sales wed_sales2 ,thu_sales thu_sales2 ,fri_sales fri_sales2 ,sat_sales sat_sales2 from wswscs ,date_dim where date_dim.d_week_seq = wswscs.d_week_seq and d_year = 1998+1) z where d_week_seq1=d_week_seq2-53 order by d_week_seq1""", "q3" -> """ select dt.d_year ,item.i_brand_id brand_id ,item.i_brand brand ,sum(ss_sales_price) sum_agg from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manufact_id = 816 and dt.d_moy=11 group by dt.d_year ,item.i_brand ,item.i_brand_id order by dt.d_year ,sum_agg desc ,brand_id limit 100""", "q4" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum((((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2) ) year_total ,'c' sale_type from customer ,catalog_sales ,date_dim where c_customer_sk = cs_bill_customer_sk and cs_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum((((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2) ) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year ) select t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_birth_country from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_c_firstyear ,year_total t_c_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_c_secyear.customer_id and t_s_firstyear.customer_id = t_c_firstyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.sale_type = 's' and t_c_firstyear.sale_type = 'c' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_c_secyear.sale_type = 'c' and t_w_secyear.sale_type = 'w' and t_s_firstyear.dyear = 1999 and t_s_secyear.dyear = 1999+1 and t_c_firstyear.dyear = 1999 and t_c_secyear.dyear = 1999+1 and t_w_firstyear.dyear = 1999 and t_w_secyear.dyear = 1999+1 and t_s_firstyear.year_total > 0 and t_c_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end > case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end order by t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_birth_country limit 100""", "q5" -> """ with ssr as (select s_store_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select ss_store_sk as store_sk, ss_sold_date_sk as date_sk, ss_ext_sales_price as sales_price, ss_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from store_sales union all select sr_store_sk as store_sk, sr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, sr_return_amt as return_amt, sr_net_loss as net_loss from store_returns ) salesreturns, date_dim, store where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and store_sk = s_store_sk group by s_store_id) , csr as (select cp_catalog_page_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select cs_catalog_page_sk as page_sk, cs_sold_date_sk as date_sk, cs_ext_sales_price as sales_price, cs_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from catalog_sales union all select cr_catalog_page_sk as page_sk, cr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, cr_return_amount as return_amt, cr_net_loss as net_loss from catalog_returns ) salesreturns, date_dim, catalog_page where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and page_sk = cp_catalog_page_sk group by cp_catalog_page_id) , wsr as (select web_site_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select ws_web_site_sk as wsr_web_site_sk, ws_sold_date_sk as date_sk, ws_ext_sales_price as sales_price, ws_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from web_sales union all select ws_web_site_sk as wsr_web_site_sk, wr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, wr_return_amt as return_amt, wr_net_loss as net_loss from web_returns left outer join web_sales on ( wr_item_sk = ws_item_sk and wr_order_number = ws_order_number) ) salesreturns, date_dim, web_site where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and wsr_web_site_sk = web_site_sk group by web_site_id) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , 'store' || s_store_id as id , sales , returns , (profit - profit_loss) as profit from ssr union all select 'catalog channel' as channel , 'catalog_page' || cp_catalog_page_id as id , sales , returns , (profit - profit_loss) as profit from csr union all select 'web channel' as channel , 'web_site' || web_site_id as id , sales , returns , (profit - profit_loss) as profit from wsr ) x group by rollup (channel, id) order by channel ,id limit 100""", "q6" -> """ select a.ca_state state, count(*) cnt from customer_address a ,customer c ,store_sales s ,date_dim d ,item i where a.ca_address_sk = c.c_current_addr_sk and c.c_customer_sk = s.ss_customer_sk and s.ss_sold_date_sk = d.d_date_sk and s.ss_item_sk = i.i_item_sk and d.d_month_seq = (select distinct (d_month_seq) from date_dim where d_year = 2002 and d_moy = 3 ) and i.i_current_price > 1.2 * (select avg(j.i_current_price) from item j where j.i_category = i.i_category) group by a.ca_state having count(*) >= 10 order by cnt, a.ca_state limit 100""", "q7" -> """ select i_item_id, avg(ss_quantity) agg1, avg(ss_list_price) agg2, avg(ss_coupon_amt) agg3, avg(ss_sales_price) agg4 from store_sales, customer_demographics, date_dim, item, promotion where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and ss_cdemo_sk = cd_demo_sk and ss_promo_sk = p_promo_sk and cd_gender = 'F' and cd_marital_status = 'W' and cd_education_status = 'College' and (p_channel_email = 'N' or p_channel_event = 'N') and d_year = 2001 group by i_item_id order by i_item_id limit 100""", "q8" -> """ select s_store_name ,sum(ss_net_profit) from store_sales ,date_dim ,store, (select ca_zip from ( SELECT substr(ca_zip,1,5) ca_zip FROM customer_address WHERE substr(ca_zip,1,5) IN ( '47602','16704','35863','28577','83910','36201', '58412','48162','28055','41419','80332', '38607','77817','24891','16226','18410', '21231','59345','13918','51089','20317', '17167','54585','67881','78366','47770', '18360','51717','73108','14440','21800', '89338','45859','65501','34948','25973', '73219','25333','17291','10374','18829', '60736','82620','41351','52094','19326', '25214','54207','40936','21814','79077', '25178','75742','77454','30621','89193', '27369','41232','48567','83041','71948', '37119','68341','14073','16891','62878', '49130','19833','24286','27700','40979', '50412','81504','94835','84844','71954', '39503','57649','18434','24987','12350', '86379','27413','44529','98569','16515', '27287','24255','21094','16005','56436', '91110','68293','56455','54558','10298', '83647','32754','27052','51766','19444', '13869','45645','94791','57631','20712', '37788','41807','46507','21727','71836', '81070','50632','88086','63991','20244', '31655','51782','29818','63792','68605', '94898','36430','57025','20601','82080', '33869','22728','35834','29086','92645', '98584','98072','11652','78093','57553', '43830','71144','53565','18700','90209', '71256','38353','54364','28571','96560', '57839','56355','50679','45266','84680', '34306','34972','48530','30106','15371', '92380','84247','92292','68852','13338', '34594','82602','70073','98069','85066', '47289','11686','98862','26217','47529', '63294','51793','35926','24227','14196', '24594','32489','99060','49472','43432', '49211','14312','88137','47369','56877', '20534','81755','15794','12318','21060', '73134','41255','63073','81003','73873', '66057','51184','51195','45676','92696', '70450','90669','98338','25264','38919', '59226','58581','60298','17895','19489', '52301','80846','95464','68770','51634', '19988','18367','18421','11618','67975', '25494','41352','95430','15734','62585', '97173','33773','10425','75675','53535', '17879','41967','12197','67998','79658', '59130','72592','14851','43933','68101', '50636','25717','71286','24660','58058', '72991','95042','15543','33122','69280', '11912','59386','27642','65177','17672', '33467','64592','36335','54010','18767', '63193','42361','49254','33113','33159', '36479','59080','11855','81963','31016', '49140','29392','41836','32958','53163', '13844','73146','23952','65148','93498', '14530','46131','58454','13376','13378', '83986','12320','17193','59852','46081', '98533','52389','13086','68843','31013', '13261','60560','13443','45533','83583', '11489','58218','19753','22911','25115', '86709','27156','32669','13123','51933', '39214','41331','66943','14155','69998', '49101','70070','35076','14242','73021', '59494','15782','29752','37914','74686', '83086','34473','15751','81084','49230', '91894','60624','17819','28810','63180', '56224','39459','55233','75752','43639', '55349','86057','62361','50788','31830', '58062','18218','85761','60083','45484', '21204','90229','70041','41162','35390', '16364','39500','68908','26689','52868', '81335','40146','11340','61527','61794', '71997','30415','59004','29450','58117', '69952','33562','83833','27385','61860', '96435','48333','23065','32961','84919', '61997','99132','22815','56600','68730', '48017','95694','32919','88217','27116', '28239','58032','18884','16791','21343', '97462','18569','75660','15475') intersect select ca_zip from (SELECT substr(ca_zip,1,5) ca_zip,count(*) cnt FROM customer_address, customer WHERE ca_address_sk = c_current_addr_sk and c_preferred_cust_flag='Y' group by ca_zip having count(*) > 10)A1)A2) V1 where ss_store_sk = s_store_sk and ss_sold_date_sk = d_date_sk and d_qoy = 2 and d_year = 1998 and (substr(s_zip,1,2) = substr(V1.ca_zip,1,2)) group by s_store_name order by s_store_name limit 100""", "q9" -> """ select case when (select count(*) from store_sales where ss_quantity between 1 and 20) > 98972190 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 1 and 20) else (select avg(ss_net_profit) from store_sales where ss_quantity between 1 and 20) end bucket1 , case when (select count(*) from store_sales where ss_quantity between 21 and 40) > 160856845 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 21 and 40) else (select avg(ss_net_profit) from store_sales where ss_quantity between 21 and 40) end bucket2, case when (select count(*) from store_sales where ss_quantity between 41 and 60) > 12733327 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 41 and 60) else (select avg(ss_net_profit) from store_sales where ss_quantity between 41 and 60) end bucket3, case when (select count(*) from store_sales where ss_quantity between 61 and 80) > 96251173 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 61 and 80) else (select avg(ss_net_profit) from store_sales where ss_quantity between 61 and 80) end bucket4, case when (select count(*) from store_sales where ss_quantity between 81 and 100) > 80049606 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 81 and 100) else (select avg(ss_net_profit) from store_sales where ss_quantity between 81 and 100) end bucket5 from reason where r_reason_sk = 1""", "q10" -> """ select cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3, cd_dep_count, count(*) cnt4, cd_dep_employed_count, count(*) cnt5, cd_dep_college_count, count(*) cnt6 from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and ca_county in ('Fillmore County','McPherson County','Bonneville County','Boone County','Brown County') and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2000 and d_moy between 3 and 3+3) and (exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2000 and d_moy between 3 ANd 3+3) or exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2000 and d_moy between 3 and 3+3)) group by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count order by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count limit 100""", "q11" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(ss_ext_list_price-ss_ext_discount_amt) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(ws_ext_list_price-ws_ext_discount_amt) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year ) select t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_birth_country from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.sale_type = 's' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_w_secyear.sale_type = 'w' and t_s_firstyear.dyear = 1999 and t_s_secyear.dyear = 1999+1 and t_w_firstyear.dyear = 1999 and t_w_secyear.dyear = 1999+1 and t_s_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else 0.0 end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else 0.0 end order by t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_birth_country limit 100""", "q12" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(ws_ext_sales_price) as itemrevenue ,sum(ws_ext_sales_price)*100/sum(sum(ws_ext_sales_price)) over (partition by i_class) as revenueratio from web_sales ,item ,date_dim where ws_item_sk = i_item_sk and i_category in ('Electronics', 'Books', 'Women') and ws_sold_date_sk = d_date_sk and d_date between cast('1998-01-06' as date) and (cast('1998-01-06' as date) + INTERVAL 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio limit 100""", "q13" -> """ select avg(ss_quantity) ,avg(ss_ext_sales_price) ,avg(ss_ext_wholesale_cost) ,sum(ss_ext_wholesale_cost) from store_sales ,store ,customer_demographics ,household_demographics ,customer_address ,date_dim where s_store_sk = ss_store_sk and ss_sold_date_sk = d_date_sk and d_year = 2001 and((ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'U' and cd_education_status = 'Secondary' and ss_sales_price between 100.00 and 150.00 and hd_dep_count = 3 )or (ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'W' and cd_education_status = 'College' and ss_sales_price between 50.00 and 100.00 and hd_dep_count = 1 ) or (ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'D' and cd_education_status = 'Primary' and ss_sales_price between 150.00 and 200.00 and hd_dep_count = 1 )) and((ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('TX', 'OK', 'MI') and ss_net_profit between 100 and 200 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('WA', 'NC', 'OH') and ss_net_profit between 150 and 300 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('MT', 'FL', 'GA') and ss_net_profit between 50 and 250 ))""", "q14a" -> """ with cross_items as (select i_item_sk ss_item_sk from item, (select iss.i_brand_id brand_id ,iss.i_class_id class_id ,iss.i_category_id category_id from store_sales ,item iss ,date_dim d1 where ss_item_sk = iss.i_item_sk and ss_sold_date_sk = d1.d_date_sk and d1.d_year between 2000 AND 2000 + 2 intersect select ics.i_brand_id ,ics.i_class_id ,ics.i_category_id from catalog_sales ,item ics ,date_dim d2 where cs_item_sk = ics.i_item_sk and cs_sold_date_sk = d2.d_date_sk and d2.d_year between 2000 AND 2000 + 2 intersect select iws.i_brand_id ,iws.i_class_id ,iws.i_category_id from web_sales ,item iws ,date_dim d3 where ws_item_sk = iws.i_item_sk and ws_sold_date_sk = d3.d_date_sk and d3.d_year between 2000 AND 2000 + 2) where i_brand_id = brand_id and i_class_id = class_id and i_category_id = category_id ), avg_sales as (select avg(quantity*list_price) average_sales from (select ss_quantity quantity ,ss_list_price list_price from store_sales ,date_dim where ss_sold_date_sk = d_date_sk and d_year between 2000 and 2000 + 2 union all select cs_quantity quantity ,cs_list_price list_price from catalog_sales ,date_dim where cs_sold_date_sk = d_date_sk and d_year between 2000 and 2000 + 2 union all select ws_quantity quantity ,ws_list_price list_price from web_sales ,date_dim where ws_sold_date_sk = d_date_sk and d_year between 2000 and 2000 + 2) x) select channel, i_brand_id,i_class_id,i_category_id,sum(sales), sum(number_sales) from( select 'store' channel, i_brand_id,i_class_id ,i_category_id,sum(ss_quantity*ss_list_price) sales , count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 2000+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales) union all select 'catalog' channel, i_brand_id,i_class_id,i_category_id, sum(cs_quantity*cs_list_price) sales, count(*) number_sales from catalog_sales ,item ,date_dim where cs_item_sk in (select ss_item_sk from cross_items) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 2000+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(cs_quantity*cs_list_price) > (select average_sales from avg_sales) union all select 'web' channel, i_brand_id,i_class_id,i_category_id, sum(ws_quantity*ws_list_price) sales , count(*) number_sales from web_sales ,item ,date_dim where ws_item_sk in (select ss_item_sk from cross_items) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 2000+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(ws_quantity*ws_list_price) > (select average_sales from avg_sales) ) y group by rollup (channel, i_brand_id,i_class_id,i_category_id) order by channel,i_brand_id,i_class_id,i_category_id limit 100""", "q14b" -> """ with cross_items as (select i_item_sk ss_item_sk from item, (select iss.i_brand_id brand_id ,iss.i_class_id class_id ,iss.i_category_id category_id from store_sales ,item iss ,date_dim d1 where ss_item_sk = iss.i_item_sk and ss_sold_date_sk = d1.d_date_sk and d1.d_year between 2000 AND 2000 + 2 intersect select ics.i_brand_id ,ics.i_class_id ,ics.i_category_id from catalog_sales ,item ics ,date_dim d2 where cs_item_sk = ics.i_item_sk and cs_sold_date_sk = d2.d_date_sk and d2.d_year between 2000 AND 2000 + 2 intersect select iws.i_brand_id ,iws.i_class_id ,iws.i_category_id from web_sales ,item iws ,date_dim d3 where ws_item_sk = iws.i_item_sk and ws_sold_date_sk = d3.d_date_sk and d3.d_year between 2000 AND 2000 + 2) x where i_brand_id = brand_id and i_class_id = class_id and i_category_id = category_id ), avg_sales as (select avg(quantity*list_price) average_sales from (select ss_quantity quantity ,ss_list_price list_price from store_sales ,date_dim where ss_sold_date_sk = d_date_sk and d_year between 2000 and 2000 + 2 union all select cs_quantity quantity ,cs_list_price list_price from catalog_sales ,date_dim where cs_sold_date_sk = d_date_sk and d_year between 2000 and 2000 + 2 union all select ws_quantity quantity ,ws_list_price list_price from web_sales ,date_dim where ws_sold_date_sk = d_date_sk and d_year between 2000 and 2000 + 2) x) select this_year.channel ty_channel ,this_year.i_brand_id ty_brand ,this_year.i_class_id ty_class ,this_year.i_category_id ty_category ,this_year.sales ty_sales ,this_year.number_sales ty_number_sales ,last_year.channel ly_channel ,last_year.i_brand_id ly_brand ,last_year.i_class_id ly_class ,last_year.i_category_id ly_category ,last_year.sales ly_sales ,last_year.number_sales ly_number_sales from (select 'store' channel, i_brand_id,i_class_id,i_category_id ,sum(ss_quantity*ss_list_price) sales, count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_week_seq = (select d_week_seq from date_dim where d_year = 2000 + 1 and d_moy = 12 and d_dom = 15) group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) this_year, (select 'store' channel, i_brand_id,i_class_id ,i_category_id, sum(ss_quantity*ss_list_price) sales, count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_week_seq = (select d_week_seq from date_dim where d_year = 2000 and d_moy = 12 and d_dom = 15) group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) last_year where this_year.i_brand_id= last_year.i_brand_id and this_year.i_class_id = last_year.i_class_id and this_year.i_category_id = last_year.i_category_id order by this_year.channel, this_year.i_brand_id, this_year.i_class_id, this_year.i_category_id limit 100""", "q15" -> """ select ca_zip ,sum(cs_sales_price) from catalog_sales ,customer ,customer_address ,date_dim where cs_bill_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') or ca_state in ('CA','WA','GA') or cs_sales_price > 500) and cs_sold_date_sk = d_date_sk and d_qoy = 2 and d_year = 1998 group by ca_zip order by ca_zip limit 100""", "q16" -> """ select count(distinct cs_order_number) as `order count` ,sum(cs_ext_ship_cost) as `total shipping cost` ,sum(cs_net_profit) as `total net profit` from catalog_sales cs1 ,date_dim ,customer_address ,call_center where d_date between '1999-4-01' and (cast('1999-4-01' as date) + INTERVAL 60 days) and cs1.cs_ship_date_sk = d_date_sk and cs1.cs_ship_addr_sk = ca_address_sk and ca_state = 'IL' and cs1.cs_call_center_sk = cc_call_center_sk and cc_county in ('Richland County','Bronx County','Maverick County','Mesa County', 'Raleigh County' ) and exists (select * from catalog_sales cs2 where cs1.cs_order_number = cs2.cs_order_number and cs1.cs_warehouse_sk <> cs2.cs_warehouse_sk) and not exists(select * from catalog_returns cr1 where cs1.cs_order_number = cr1.cr_order_number) order by count(distinct cs_order_number) limit 100""", "q17" -> """ select i_item_id ,i_item_desc ,s_state ,count(ss_quantity) as store_sales_quantitycount ,avg(ss_quantity) as store_sales_quantityave ,stddev_samp(ss_quantity) as store_sales_quantitystdev ,stddev_samp(ss_quantity)/avg(ss_quantity) as store_sales_quantitycov ,count(sr_return_quantity) as store_returns_quantitycount ,avg(sr_return_quantity) as store_returns_quantityave ,stddev_samp(sr_return_quantity) as store_returns_quantitystdev ,stddev_samp(sr_return_quantity)/avg(sr_return_quantity) as store_returns_quantitycov ,count(cs_quantity) as catalog_sales_quantitycount ,avg(cs_quantity) as catalog_sales_quantityave ,stddev_samp(cs_quantity) as catalog_sales_quantitystdev ,stddev_samp(cs_quantity)/avg(cs_quantity) as catalog_sales_quantitycov from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_quarter_name = '2000Q1' and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_quarter_name in ('2000Q1','2000Q2','2000Q3') and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_quarter_name in ('2000Q1','2000Q2','2000Q3') group by i_item_id ,i_item_desc ,s_state order by i_item_id ,i_item_desc ,s_state limit 100""", "q18" -> """ select i_item_id, ca_country, ca_state, ca_county, avg( cast(cs_quantity as decimal(12,2))) agg1, avg( cast(cs_list_price as decimal(12,2))) agg2, avg( cast(cs_coupon_amt as decimal(12,2))) agg3, avg( cast(cs_sales_price as decimal(12,2))) agg4, avg( cast(cs_net_profit as decimal(12,2))) agg5, avg( cast(c_birth_year as decimal(12,2))) agg6, avg( cast(cd1.cd_dep_count as decimal(12,2))) agg7 from catalog_sales, customer_demographics cd1, customer_demographics cd2, customer, customer_address, date_dim, item where cs_sold_date_sk = d_date_sk and cs_item_sk = i_item_sk and cs_bill_cdemo_sk = cd1.cd_demo_sk and cs_bill_customer_sk = c_customer_sk and cd1.cd_gender = 'M' and cd1.cd_education_status = 'Unknown' and c_current_cdemo_sk = cd2.cd_demo_sk and c_current_addr_sk = ca_address_sk and c_birth_month in (5,1,4,7,8,9) and d_year = 2002 and ca_state in ('AR','TX','NC' ,'GA','MS','WV','AL') group by rollup (i_item_id, ca_country, ca_state, ca_county) order by ca_country, ca_state, ca_county, i_item_id limit 100""", "q19" -> """ select i_brand_id brand_id, i_brand brand, i_manufact_id, i_manufact, sum(ss_ext_sales_price) ext_price from date_dim, store_sales, item,customer,customer_address,store where d_date_sk = ss_sold_date_sk and ss_item_sk = i_item_sk and i_manager_id=16 and d_moy=12 and d_year=1998 and ss_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and substr(ca_zip,1,5) <> substr(s_zip,1,5) and ss_store_sk = s_store_sk group by i_brand ,i_brand_id ,i_manufact_id ,i_manufact order by ext_price desc ,i_brand ,i_brand_id ,i_manufact_id ,i_manufact limit 100 """, "q20" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(cs_ext_sales_price) as itemrevenue ,sum(cs_ext_sales_price)*100/sum(sum(cs_ext_sales_price)) over (partition by i_class) as revenueratio from catalog_sales ,item ,date_dim where cs_item_sk = i_item_sk and i_category in ('Shoes', 'Electronics', 'Children') and cs_sold_date_sk = d_date_sk and d_date between cast('2001-03-14' as date) and (cast('2001-03-14' as date) + INTERVAL 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio limit 100""", "q21" -> """ select * from(select w_warehouse_name ,i_item_id ,sum(case when (cast(d_date as date) < cast ('1999-03-20' as date)) then inv_quantity_on_hand else 0 end) as inv_before ,sum(case when (cast(d_date as date) >= cast ('1999-03-20' as date)) then inv_quantity_on_hand else 0 end) as inv_after from inventory ,warehouse ,item ,date_dim where i_current_price between 0.99 and 1.49 and i_item_sk = inv_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_date between (cast ('1999-03-20' as date) - INTERVAL 30 days) and (cast ('1999-03-20' as date) + INTERVAL 30 days) group by w_warehouse_name, i_item_id) x where (case when inv_before > 0 then inv_after / inv_before else null end) between 2.0/3.0 and 3.0/2.0 order by w_warehouse_name ,i_item_id limit 100""", "q22" -> """ select i_product_name ,i_brand ,i_class ,i_category ,avg(inv_quantity_on_hand) qoh from inventory ,date_dim ,item where inv_date_sk=d_date_sk and inv_item_sk=i_item_sk and d_month_seq between 1186 and 1186 + 11 group by rollup(i_product_name ,i_brand ,i_class ,i_category) order by qoh, i_product_name, i_brand, i_class, i_category limit 100""", "q23a" -> """ with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt from store_sales ,date_dim ,item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and d_year in (2000,2000+1,2000+2,2000+3) group by substr(i_item_desc,1,30),i_item_sk,d_date having count(*) >4), max_store_sales as (select max(csales) tpcds_cmax from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales from store_sales ,customer ,date_dim where ss_customer_sk = c_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (2000,2000+1,2000+2,2000+3) group by c_customer_sk)), best_ss_customer as (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales from store_sales ,customer where ss_customer_sk = c_customer_sk group by c_customer_sk having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select * from max_store_sales)) select sum(sales) from (select cs_quantity*cs_list_price sales from catalog_sales ,date_dim where d_year = 2000 and d_moy = 3 and cs_sold_date_sk = d_date_sk and cs_item_sk in (select item_sk from frequent_ss_items) and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) union all select ws_quantity*ws_list_price sales from web_sales ,date_dim where d_year = 2000 and d_moy = 3 and ws_sold_date_sk = d_date_sk and ws_item_sk in (select item_sk from frequent_ss_items) and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer)) limit 100""", "q23b" -> """ with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt from store_sales ,date_dim ,item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and d_year in (2000,2000 + 1,2000 + 2,2000 + 3) group by substr(i_item_desc,1,30),i_item_sk,d_date having count(*) >4), max_store_sales as (select max(csales) tpcds_cmax from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales from store_sales ,customer ,date_dim where ss_customer_sk = c_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (2000,2000+1,2000+2,2000+3) group by c_customer_sk)), best_ss_customer as (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales from store_sales ,customer where ss_customer_sk = c_customer_sk group by c_customer_sk having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select * from max_store_sales)) select c_last_name,c_first_name,sales from (select c_last_name,c_first_name,sum(cs_quantity*cs_list_price) sales from catalog_sales ,customer ,date_dim where d_year = 2000 and d_moy = 3 and cs_sold_date_sk = d_date_sk and cs_item_sk in (select item_sk from frequent_ss_items) and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) and cs_bill_customer_sk = c_customer_sk group by c_last_name,c_first_name union all select c_last_name,c_first_name,sum(ws_quantity*ws_list_price) sales from web_sales ,customer ,date_dim where d_year = 2000 and d_moy = 3 and ws_sold_date_sk = d_date_sk and ws_item_sk in (select item_sk from frequent_ss_items) and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer) and ws_bill_customer_sk = c_customer_sk group by c_last_name,c_first_name) order by c_last_name,c_first_name,sales limit 100""", "q24a" -> """ with ssales as (select c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size ,sum(ss_sales_price) netpaid from store_sales ,store_returns ,store ,item ,customer ,customer_address where ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_customer_sk = c_customer_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and c_current_addr_sk = ca_address_sk and c_birth_country <> upper(ca_country) and s_zip = ca_zip and s_market_id=10 group by c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size) select c_last_name ,c_first_name ,s_store_name ,sum(netpaid) paid from ssales where i_color = 'snow' group by c_last_name ,c_first_name ,s_store_name having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) order by c_last_name ,c_first_name ,s_store_name""", "q24b" -> """ with ssales as (select c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size ,sum(ss_sales_price) netpaid from store_sales ,store_returns ,store ,item ,customer ,customer_address where ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_customer_sk = c_customer_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and c_current_addr_sk = ca_address_sk and c_birth_country <> upper(ca_country) and s_zip = ca_zip and s_market_id = 10 group by c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size) select c_last_name ,c_first_name ,s_store_name ,sum(netpaid) paid from ssales where i_color = 'chiffon' group by c_last_name ,c_first_name ,s_store_name having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) order by c_last_name ,c_first_name ,s_store_name""", "q25" -> """ select i_item_id ,i_item_desc ,s_store_id ,s_store_name ,sum(ss_net_profit) as store_sales_profit ,sum(sr_net_loss) as store_returns_loss ,sum(cs_net_profit) as catalog_sales_profit from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_moy = 4 and d1.d_year = 2000 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_moy between 4 and 10 and d2.d_year = 2000 and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_moy between 4 and 10 and d3.d_year = 2000 group by i_item_id ,i_item_desc ,s_store_id ,s_store_name order by i_item_id ,i_item_desc ,s_store_id ,s_store_name limit 100""", "q26" -> """ select i_item_id, avg(cs_quantity) agg1, avg(cs_list_price) agg2, avg(cs_coupon_amt) agg3, avg(cs_sales_price) agg4 from catalog_sales, customer_demographics, date_dim, item, promotion where cs_sold_date_sk = d_date_sk and cs_item_sk = i_item_sk and cs_bill_cdemo_sk = cd_demo_sk and cs_promo_sk = p_promo_sk and cd_gender = 'F' and cd_marital_status = 'S' and cd_education_status = 'College' and (p_channel_email = 'N' or p_channel_event = 'N') and d_year = 1998 group by i_item_id order by i_item_id limit 100""", "q27" -> """ select i_item_id, s_state, grouping(s_state) g_state, avg(ss_quantity) agg1, avg(ss_list_price) agg2, avg(ss_coupon_amt) agg3, avg(ss_sales_price) agg4 from store_sales, customer_demographics, date_dim, store, item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and ss_cdemo_sk = cd_demo_sk and cd_gender = 'F' and cd_marital_status = 'U' and cd_education_status = '2 yr Degree' and d_year = 2000 and s_state in ('AL','IN', 'SC', 'NY', 'OH', 'FL') group by rollup (i_item_id, s_state) order by i_item_id ,s_state limit 100""", "q28" -> """ select * from (select avg(ss_list_price) B1_LP ,count(ss_list_price) B1_CNT ,count(distinct ss_list_price) B1_CNTD from store_sales where ss_quantity between 0 and 5 and (ss_list_price between 73 and 73+10 or ss_coupon_amt between 7826 and 7826+1000 or ss_wholesale_cost between 70 and 70+20)) B1, (select avg(ss_list_price) B2_LP ,count(ss_list_price) B2_CNT ,count(distinct ss_list_price) B2_CNTD from store_sales where ss_quantity between 6 and 10 and (ss_list_price between 152 and 152+10 or ss_coupon_amt between 2196 and 2196+1000 or ss_wholesale_cost between 56 and 56+20)) B2, (select avg(ss_list_price) B3_LP ,count(ss_list_price) B3_CNT ,count(distinct ss_list_price) B3_CNTD from store_sales where ss_quantity between 11 and 15 and (ss_list_price between 53 and 53+10 or ss_coupon_amt between 3430 and 3430+1000 or ss_wholesale_cost between 13 and 13+20)) B3, (select avg(ss_list_price) B4_LP ,count(ss_list_price) B4_CNT ,count(distinct ss_list_price) B4_CNTD from store_sales where ss_quantity between 16 and 20 and (ss_list_price between 182 and 182+10 or ss_coupon_amt between 3262 and 3262+1000 or ss_wholesale_cost between 20 and 20+20)) B4, (select avg(ss_list_price) B5_LP ,count(ss_list_price) B5_CNT ,count(distinct ss_list_price) B5_CNTD from store_sales where ss_quantity between 21 and 25 and (ss_list_price between 85 and 85+10 or ss_coupon_amt between 3310 and 3310+1000 or ss_wholesale_cost between 37 and 37+20)) B5, (select avg(ss_list_price) B6_LP ,count(ss_list_price) B6_CNT ,count(distinct ss_list_price) B6_CNTD from store_sales where ss_quantity between 26 and 30 and (ss_list_price between 180 and 180+10 or ss_coupon_amt between 12592 and 12592+1000 or ss_wholesale_cost between 22 and 22+20)) B6 limit 100""", "q29" -> """ select i_item_id ,i_item_desc ,s_store_id ,s_store_name ,stddev_samp(ss_quantity) as store_sales_quantity ,stddev_samp(sr_return_quantity) as store_returns_quantity ,stddev_samp(cs_quantity) as catalog_sales_quantity from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_moy = 4 and d1.d_year = 1998 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_moy between 4 and 4 + 3 and d2.d_year = 1998 and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_year in (1998,1998+1,1998+2) group by i_item_id ,i_item_desc ,s_store_id ,s_store_name order by i_item_id ,i_item_desc ,s_store_id ,s_store_name limit 100""", "q30" -> """ with customer_total_return as (select wr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, sum(wr_return_amt) as ctr_total_return from web_returns ,date_dim ,customer_address where wr_returned_date_sk = d_date_sk and d_year =2000 and wr_returning_addr_sk = ca_address_sk group by wr_returning_customer_sk ,ca_state) select c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address ,c_last_review_date,ctr_total_return from customer_total_return ctr1 ,customer_address ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_state = ctr2.ctr_state) and ca_address_sk = c_current_addr_sk and ca_state = 'GA' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address ,c_last_review_date,ctr_total_return limit 100""", "q31" -> """ with ss as (select ca_county,d_qoy, d_year,sum(ss_ext_sales_price) as store_sales from store_sales,date_dim,customer_address where ss_sold_date_sk = d_date_sk and ss_addr_sk=ca_address_sk group by ca_county,d_qoy, d_year), ws as (select ca_county,d_qoy, d_year,sum(ws_ext_sales_price) as web_sales from web_sales,date_dim,customer_address where ws_sold_date_sk = d_date_sk and ws_bill_addr_sk=ca_address_sk group by ca_county,d_qoy, d_year) select ss1.ca_county ,ss1.d_year ,ws2.web_sales/ws1.web_sales web_q1_q2_increase ,ss2.store_sales/ss1.store_sales store_q1_q2_increase ,ws3.web_sales/ws2.web_sales web_q2_q3_increase ,ss3.store_sales/ss2.store_sales store_q2_q3_increase from ss ss1 ,ss ss2 ,ss ss3 ,ws ws1 ,ws ws2 ,ws ws3 where ss1.d_qoy = 1 and ss1.d_year = 1999 and ss1.ca_county = ss2.ca_county and ss2.d_qoy = 2 and ss2.d_year = 1999 and ss2.ca_county = ss3.ca_county and ss3.d_qoy = 3 and ss3.d_year = 1999 and ss1.ca_county = ws1.ca_county and ws1.d_qoy = 1 and ws1.d_year = 1999 and ws1.ca_county = ws2.ca_county and ws2.d_qoy = 2 and ws2.d_year = 1999 and ws1.ca_county = ws3.ca_county and ws3.d_qoy = 3 and ws3.d_year =1999 and case when ws1.web_sales > 0 then ws2.web_sales/ws1.web_sales else null end > case when ss1.store_sales > 0 then ss2.store_sales/ss1.store_sales else null end and case when ws2.web_sales > 0 then ws3.web_sales/ws2.web_sales else null end > case when ss2.store_sales > 0 then ss3.store_sales/ss2.store_sales else null end order by ss1.d_year""", "q32" -> """ select sum(cs_ext_discount_amt) as `excess discount amount` from catalog_sales ,item ,date_dim where i_manufact_id = 66 and i_item_sk = cs_item_sk and d_date between '2002-03-29' and (cast('2002-03-29' as date) + INTERVAL 90 days) and d_date_sk = cs_sold_date_sk and cs_ext_discount_amt > ( select 1.3 * avg(cs_ext_discount_amt) from catalog_sales ,date_dim where cs_item_sk = i_item_sk and d_date between '2002-03-29' and (cast('2002-03-29' as date) + INTERVAL 90 days) and d_date_sk = cs_sold_date_sk ) limit 100""", "q33" -> """ with ss as ( select i_manufact_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Home')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 5 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_manufact_id), cs as ( select i_manufact_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Home')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 5 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_manufact_id), ws as ( select i_manufact_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Home')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 5 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_manufact_id) select i_manufact_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_manufact_id order by total_sales limit 100""", "q34" -> """ select c_last_name ,c_first_name ,c_salutation ,c_preferred_cust_flag ,ss_ticket_number ,cnt from (select ss_ticket_number ,ss_customer_sk ,count(*) cnt from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and (date_dim.d_dom between 1 and 3 or date_dim.d_dom between 25 and 28) and (household_demographics.hd_buy_potential = '>10000' or household_demographics.hd_buy_potential = 'Unknown') and household_demographics.hd_vehicle_count > 0 and (case when household_demographics.hd_vehicle_count > 0 then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end) > 1.2 and date_dim.d_year in (2000,2000+1,2000+2) and store.s_county in ('Salem County','Terrell County','Arthur County','Oglethorpe County', 'Lunenburg County','Perry County','Halifax County','Sumner County') group by ss_ticket_number,ss_customer_sk) dn,customer where ss_customer_sk = c_customer_sk and cnt between 15 and 20 order by c_last_name,c_first_name,c_salutation,c_preferred_cust_flag desc, ss_ticket_number""", "q35" -> """ select ca_state, cd_gender, cd_marital_status, cd_dep_count, count(*) cnt1, avg(cd_dep_count), min(cd_dep_count), stddev_samp(cd_dep_count), cd_dep_employed_count, count(*) cnt2, avg(cd_dep_employed_count), min(cd_dep_employed_count), stddev_samp(cd_dep_employed_count), cd_dep_college_count, count(*) cnt3, avg(cd_dep_college_count), min(cd_dep_college_count), stddev_samp(cd_dep_college_count) from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2001 and d_qoy < 4) and (exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2001 and d_qoy < 4) or exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2001 and d_qoy < 4)) group by ca_state, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count order by ca_state, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count limit 100""", "q36" -> """ select sum(ss_net_profit)/sum(ss_ext_sales_price) as gross_margin ,i_category ,i_class ,grouping(i_category)+grouping(i_class) as lochierarchy ,rank() over ( partition by grouping(i_category)+grouping(i_class), case when grouping(i_class) = 0 then i_category end order by sum(ss_net_profit)/sum(ss_ext_sales_price) asc) as rank_within_parent from store_sales ,date_dim d1 ,item ,store where d1.d_year = 1999 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and s_state in ('IN','AL','MI','MN', 'TN','LA','FL','NM') group by rollup(i_category,i_class) order by lochierarchy desc ,case when lochierarchy = 0 then i_category end ,rank_within_parent limit 100""", "q37" -> """ select i_item_id ,i_item_desc ,i_current_price from item, inventory, date_dim, catalog_sales where i_current_price between 39 and 39 + 30 and inv_item_sk = i_item_sk and d_date_sk=inv_date_sk and d_date between cast('2001-01-16' as date) and (cast('2001-01-16' as date) + interval 60 days) and i_manufact_id in (765,886,889,728) and inv_quantity_on_hand between 100 and 500 and cs_item_sk = i_item_sk group by i_item_id,i_item_desc,i_current_price order by i_item_id limit 100""", "q38" -> """ select count(*) from ( select distinct c_last_name, c_first_name, d_date from store_sales, date_dim, customer where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_customer_sk = customer.c_customer_sk and d_month_seq between 1186 and 1186 + 11 intersect select distinct c_last_name, c_first_name, d_date from catalog_sales, date_dim, customer where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1186 and 1186 + 11 intersect select distinct c_last_name, c_first_name, d_date from web_sales, date_dim, customer where web_sales.ws_sold_date_sk = date_dim.d_date_sk and web_sales.ws_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1186 and 1186 + 11 ) hot_cust limit 100""", "q39a" -> """ with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stdev,mean, case mean when 0 then null else stdev/mean end cov from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean from inventory ,item ,warehouse ,date_dim where inv_item_sk = i_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_year =2000 group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo where case mean when 0 then 0 else stdev/mean end > 1) select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov from inv inv1,inv inv2 where inv1.i_item_sk = inv2.i_item_sk and inv1.w_warehouse_sk = inv2.w_warehouse_sk and inv1.d_moy=2 and inv2.d_moy=2+1 order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov ,inv2.d_moy,inv2.mean, inv2.cov""", "q39b" -> """ with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stdev,mean, case mean when 0 then null else stdev/mean end cov from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean from inventory ,item ,warehouse ,date_dim where inv_item_sk = i_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_year =2000 group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo where case mean when 0 then 0 else stdev/mean end > 1) select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov from inv inv1,inv inv2 where inv1.i_item_sk = inv2.i_item_sk and inv1.w_warehouse_sk = inv2.w_warehouse_sk and inv1.d_moy=2 and inv2.d_moy=2+1 and inv1.cov > 1.5 order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov ,inv2.d_moy,inv2.mean, inv2.cov""", "q40" -> """ select w_state ,i_item_id ,sum(case when (cast(d_date as date) < cast ('2000-03-18' as date)) then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_before ,sum(case when (cast(d_date as date) >= cast ('2000-03-18' as date)) then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_after from catalog_sales left outer join catalog_returns on (cs_order_number = cr_order_number and cs_item_sk = cr_item_sk) ,warehouse ,item ,date_dim where i_current_price between 0.99 and 1.49 and i_item_sk = cs_item_sk and cs_warehouse_sk = w_warehouse_sk and cs_sold_date_sk = d_date_sk and d_date between (cast ('2000-03-18' as date) - INTERVAL 30 days) and (cast ('2000-03-18' as date) + INTERVAL 30 days) group by w_state,i_item_id order by w_state,i_item_id limit 100""", "q41" -> """ select distinct(i_product_name) from item i1 where i_manufact_id between 970 and 970+40 and (select count(*) as item_cnt from item where (i_manufact = i1.i_manufact and ((i_category = 'Women' and (i_color = 'frosted' or i_color = 'rose') and (i_units = 'Lb' or i_units = 'Gross') and (i_size = 'medium' or i_size = 'large') ) or (i_category = 'Women' and (i_color = 'chocolate' or i_color = 'black') and (i_units = 'Box' or i_units = 'Dram') and (i_size = 'economy' or i_size = 'petite') ) or (i_category = 'Men' and (i_color = 'slate' or i_color = 'magenta') and (i_units = 'Carton' or i_units = 'Bundle') and (i_size = 'N/A' or i_size = 'small') ) or (i_category = 'Men' and (i_color = 'cornflower' or i_color = 'firebrick') and (i_units = 'Pound' or i_units = 'Oz') and (i_size = 'medium' or i_size = 'large') ))) or (i_manufact = i1.i_manufact and ((i_category = 'Women' and (i_color = 'almond' or i_color = 'steel') and (i_units = 'Tsp' or i_units = 'Case') and (i_size = 'medium' or i_size = 'large') ) or (i_category = 'Women' and (i_color = 'purple' or i_color = 'aquamarine') and (i_units = 'Bunch' or i_units = 'Gram') and (i_size = 'economy' or i_size = 'petite') ) or (i_category = 'Men' and (i_color = 'lavender' or i_color = 'papaya') and (i_units = 'Pallet' or i_units = 'Cup') and (i_size = 'N/A' or i_size = 'small') ) or (i_category = 'Men' and (i_color = 'maroon' or i_color = 'cyan') and (i_units = 'Each' or i_units = 'N/A') and (i_size = 'medium' or i_size = 'large') )))) > 0 order by i_product_name limit 100""", "q42" -> """ select dt.d_year ,item.i_category_id ,item.i_category ,sum(ss_ext_sales_price) from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manager_id = 1 and dt.d_moy=12 and dt.d_year=1998 group by dt.d_year ,item.i_category_id ,item.i_category order by sum(ss_ext_sales_price) desc,dt.d_year ,item.i_category_id ,item.i_category limit 100 """, "q43" -> """ select s_store_name, s_store_id, sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales from date_dim, store_sales, store where d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk and s_gmt_offset = -6 and d_year = 2001 group by s_store_name, s_store_id order by s_store_name, s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales limit 100""", "q44" -> """ select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing from(select * from (select item_sk,rank() over (order by rank_col asc) rnk from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col from store_sales ss1 where ss_store_sk = 366 group by ss_item_sk having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col from store_sales where ss_store_sk = 366 and ss_cdemo_sk is null group by ss_store_sk))V1)V11 where rnk < 11) asceding, (select * from (select item_sk,rank() over (order by rank_col desc) rnk from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col from store_sales ss1 where ss_store_sk = 366 group by ss_item_sk having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col from store_sales where ss_store_sk = 366 and ss_cdemo_sk is null group by ss_store_sk))V2)V21 where rnk < 11) descending, item i1, item i2 where asceding.rnk = descending.rnk and i1.i_item_sk=asceding.item_sk and i2.i_item_sk=descending.item_sk order by asceding.rnk limit 100""", "q45" -> """ select ca_zip, ca_county, sum(ws_sales_price) from web_sales, customer, customer_address, date_dim, item where ws_bill_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and ws_item_sk = i_item_sk and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') or i_item_id in (select i_item_id from item where i_item_sk in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) ) ) and ws_sold_date_sk = d_date_sk and d_qoy = 1 and d_year = 1998 group by ca_zip, ca_county order by ca_zip, ca_county limit 100""", "q46" -> """ select c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number ,amt,profit from (select ss_ticket_number ,ss_customer_sk ,ca_city bought_city ,sum(ss_coupon_amt) amt ,sum(ss_net_profit) profit from store_sales,date_dim,store,household_demographics,customer_address where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and store_sales.ss_addr_sk = customer_address.ca_address_sk and (household_demographics.hd_dep_count = 0 or household_demographics.hd_vehicle_count= 1) and date_dim.d_dow in (6,0) and date_dim.d_year in (2000,2000+1,2000+2) and store.s_city in ('Five Forks','Oakland','Fairview','Winchester','Farmington') group by ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city) dn,customer,customer_address current_addr where ss_customer_sk = c_customer_sk and customer.c_current_addr_sk = current_addr.ca_address_sk and current_addr.ca_city <> bought_city order by c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number limit 100""", "q47" -> """ with v1 as( select i_category, i_brand, s_store_name, s_company_name, d_year, d_moy, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_category, i_brand, s_store_name, s_company_name, d_year) avg_monthly_sales, rank() over (partition by i_category, i_brand, s_store_name, s_company_name order by d_year, d_moy) rn from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ( d_year = 1999 or ( d_year = 1999-1 and d_moy =12) or ( d_year = 1999+1 and d_moy =1) ) group by i_category, i_brand, s_store_name, s_company_name, d_year, d_moy), v2 as( select v1.s_store_name ,v1.d_year, v1.d_moy ,v1.avg_monthly_sales ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum from v1, v1 v1_lag, v1 v1_lead where v1.i_category = v1_lag.i_category and v1.i_category = v1_lead.i_category and v1.i_brand = v1_lag.i_brand and v1.i_brand = v1_lead.i_brand and v1.s_store_name = v1_lag.s_store_name and v1.s_store_name = v1_lead.s_store_name and v1.s_company_name = v1_lag.s_company_name and v1.s_company_name = v1_lead.s_company_name and v1.rn = v1_lag.rn + 1 and v1.rn = v1_lead.rn - 1) select * from v2 where d_year = 1999 and avg_monthly_sales > 0 and case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by sum_sales - avg_monthly_sales, sum_sales limit 100""", "q48" -> """ select sum (ss_quantity) from store_sales, store, customer_demographics, customer_address, date_dim where s_store_sk = ss_store_sk and ss_sold_date_sk = d_date_sk and d_year = 1998 and ( ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'M' and cd_education_status = 'Unknown' and ss_sales_price between 100.00 and 150.00 ) or ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'W' and cd_education_status = 'College' and ss_sales_price between 50.00 and 100.00 ) or ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'D' and cd_education_status = 'Primary' and ss_sales_price between 150.00 and 200.00 ) ) and ( ( ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('MI', 'GA', 'NH') and ss_net_profit between 0 and 2000 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('TX', 'KY', 'SD') and ss_net_profit between 150 and 3000 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('NY', 'OH', 'FL') and ss_net_profit between 50 and 25000 ) )""", "q49" -> """ select channel, item, return_ratio, return_rank, currency_rank from (select 'web' as channel ,web.item ,web.return_ratio ,web.return_rank ,web.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select ws.ws_item_sk as item ,(cast(sum(coalesce(wr.wr_return_quantity,0)) as decimal(15,4))/ cast(sum(coalesce(ws.ws_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(wr.wr_return_amt,0)) as decimal(15,4))/ cast(sum(coalesce(ws.ws_net_paid,0)) as decimal(15,4) )) as currency_ratio from web_sales ws left outer join web_returns wr on (ws.ws_order_number = wr.wr_order_number and ws.ws_item_sk = wr.wr_item_sk) ,date_dim where wr.wr_return_amt > 10000 and ws.ws_net_profit > 1 and ws.ws_net_paid > 0 and ws.ws_quantity > 0 and ws_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 12 group by ws.ws_item_sk ) in_web ) web where ( web.return_rank <= 10 or web.currency_rank <= 10 ) union select 'catalog' as channel ,catalog.item ,catalog.return_ratio ,catalog.return_rank ,catalog.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select cs.cs_item_sk as item ,(cast(sum(coalesce(cr.cr_return_quantity,0)) as decimal(15,4))/ cast(sum(coalesce(cs.cs_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(cr.cr_return_amount,0)) as decimal(15,4))/ cast(sum(coalesce(cs.cs_net_paid,0)) as decimal(15,4) )) as currency_ratio from catalog_sales cs left outer join catalog_returns cr on (cs.cs_order_number = cr.cr_order_number and cs.cs_item_sk = cr.cr_item_sk) ,date_dim where cr.cr_return_amount > 10000 and cs.cs_net_profit > 1 and cs.cs_net_paid > 0 and cs.cs_quantity > 0 and cs_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 12 group by cs.cs_item_sk ) in_cat ) catalog where ( catalog.return_rank <= 10 or catalog.currency_rank <=10 ) union select 'store' as channel ,store.item ,store.return_ratio ,store.return_rank ,store.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select sts.ss_item_sk as item ,(cast(sum(coalesce(sr.sr_return_quantity,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(sr.sr_return_amt,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_net_paid,0)) as decimal(15,4) )) as currency_ratio from store_sales sts left outer join store_returns sr on (sts.ss_ticket_number = sr.sr_ticket_number and sts.ss_item_sk = sr.sr_item_sk) ,date_dim where sr.sr_return_amt > 10000 and sts.ss_net_profit > 1 and sts.ss_net_paid > 0 and sts.ss_quantity > 0 and ss_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 12 group by sts.ss_item_sk ) in_store ) store where ( store.return_rank <= 10 or store.currency_rank <= 10 ) ) order by 1,4,5,2 limit 100""", "q50" -> """ select s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip ,sum(case when (sr_returned_date_sk - ss_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 30) and (sr_returned_date_sk - ss_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 60) and (sr_returned_date_sk - ss_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 90) and (sr_returned_date_sk - ss_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from store_sales ,store_returns ,store ,date_dim d1 ,date_dim d2 where d2.d_year = 1998 and d2.d_moy = 9 and ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_sold_date_sk = d1.d_date_sk and sr_returned_date_sk = d2.d_date_sk and ss_customer_sk = sr_customer_sk and ss_store_sk = s_store_sk group by s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip order by s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip limit 100""", "q51" -> """ WITH web_v1 as ( select ws_item_sk item_sk, d_date, sum(sum(ws_sales_price)) over (partition by ws_item_sk order by d_date rows between unbounded preceding and current row) cume_sales from web_sales ,date_dim where ws_sold_date_sk=d_date_sk and d_month_seq between 1214 and 1214+11 and ws_item_sk is not NULL group by ws_item_sk, d_date), store_v1 as ( select ss_item_sk item_sk, d_date, sum(sum(ss_sales_price)) over (partition by ss_item_sk order by d_date rows between unbounded preceding and current row) cume_sales from store_sales ,date_dim where ss_sold_date_sk=d_date_sk and d_month_seq between 1214 and 1214+11 and ss_item_sk is not NULL group by ss_item_sk, d_date) select * from (select item_sk ,d_date ,web_sales ,store_sales ,max(web_sales) over (partition by item_sk order by d_date rows between unbounded preceding and current row) web_cumulative ,max(store_sales) over (partition by item_sk order by d_date rows between unbounded preceding and current row) store_cumulative from (select case when web.item_sk is not null then web.item_sk else store.item_sk end item_sk ,case when web.d_date is not null then web.d_date else store.d_date end d_date ,web.cume_sales web_sales ,store.cume_sales store_sales from web_v1 web full outer join store_v1 store on (web.item_sk = store.item_sk and web.d_date = store.d_date) )x )y where web_cumulative > store_cumulative order by item_sk ,d_date limit 100""", "q52" -> """ select dt.d_year ,item.i_brand_id brand_id ,item.i_brand brand ,sum(ss_ext_sales_price) ext_price from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manager_id = 1 and dt.d_moy=12 and dt.d_year=2000 group by dt.d_year ,item.i_brand ,item.i_brand_id order by dt.d_year ,ext_price desc ,brand_id limit 100 """, "q53" -> """ select * from (select i_manufact_id, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_manufact_id) avg_quarterly_sales from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_month_seq in (1212,1212+1,1212+2,1212+3,1212+4,1212+5,1212+6,1212+7,1212+8,1212+9,1212+10,1212+11) and ((i_category in ('Books','Children','Electronics') and i_class in ('personal','portable','reference','self-help') and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', 'exportiunivamalg #9','scholaramalgamalg #9')) or(i_category in ('Women','Music','Men') and i_class in ('accessories','classical','fragrances','pants') and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 'importoamalg #1'))) group by i_manufact_id, d_qoy ) tmp1 where case when avg_quarterly_sales > 0 then abs (sum_sales - avg_quarterly_sales)/ avg_quarterly_sales else null end > 0.1 order by avg_quarterly_sales, sum_sales, i_manufact_id limit 100""", "q54" -> """ with my_customers as ( select distinct c_customer_sk , c_current_addr_sk from ( select cs_sold_date_sk sold_date_sk, cs_bill_customer_sk customer_sk, cs_item_sk item_sk from catalog_sales union all select ws_sold_date_sk sold_date_sk, ws_bill_customer_sk customer_sk, ws_item_sk item_sk from web_sales ) cs_or_ws_sales, item, date_dim, customer where sold_date_sk = d_date_sk and item_sk = i_item_sk and i_category = 'Books' and i_class = 'business' and c_customer_sk = cs_or_ws_sales.customer_sk and d_moy = 2 and d_year = 2000 ) , my_revenue as ( select c_customer_sk, sum(ss_ext_sales_price) as revenue from my_customers, store_sales, customer_address, store, date_dim where c_current_addr_sk = ca_address_sk and ca_county = s_county and ca_state = s_state and ss_sold_date_sk = d_date_sk and c_customer_sk = ss_customer_sk and d_month_seq between (select distinct d_month_seq+1 from date_dim where d_year = 2000 and d_moy = 2) and (select distinct d_month_seq+3 from date_dim where d_year = 2000 and d_moy = 2) group by c_customer_sk ) , segments as (select cast((revenue/50) as int) as segment from my_revenue ) select segment, count(*) as num_customers, segment*50 as segment_base from segments group by segment order by segment, num_customers limit 100""", "q55" -> """ select i_brand_id brand_id, i_brand brand, sum(ss_ext_sales_price) ext_price from date_dim, store_sales, item where d_date_sk = ss_sold_date_sk and ss_item_sk = i_item_sk and i_manager_id=13 and d_moy=11 and d_year=1999 group by i_brand, i_brand_id order by ext_price desc, i_brand_id limit 100 """, "q56" -> """ with ss as ( select i_item_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('chiffon','smoke','lace')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 2001 and d_moy = 5 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_item_id), cs as ( select i_item_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('chiffon','smoke','lace')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 2001 and d_moy = 5 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_item_id), ws as ( select i_item_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('chiffon','smoke','lace')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 2001 and d_moy = 5 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_item_id) select i_item_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_item_id order by total_sales, i_item_id limit 100""", "q57" -> """ with v1 as( select i_category, i_brand, cc_name, d_year, d_moy, sum(cs_sales_price) sum_sales, avg(sum(cs_sales_price)) over (partition by i_category, i_brand, cc_name, d_year) avg_monthly_sales, rank() over (partition by i_category, i_brand, cc_name order by d_year, d_moy) rn from item, catalog_sales, date_dim, call_center where cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and cc_call_center_sk= cs_call_center_sk and ( d_year = 1999 or ( d_year = 1999-1 and d_moy =12) or ( d_year = 1999+1 and d_moy =1) ) group by i_category, i_brand, cc_name , d_year, d_moy), v2 as( select v1.i_category, v1.i_brand ,v1.d_year, v1.d_moy ,v1.avg_monthly_sales ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum from v1, v1 v1_lag, v1 v1_lead where v1.i_category = v1_lag.i_category and v1.i_category = v1_lead.i_category and v1.i_brand = v1_lag.i_brand and v1.i_brand = v1_lead.i_brand and v1. cc_name = v1_lag. cc_name and v1. cc_name = v1_lead. cc_name and v1.rn = v1_lag.rn + 1 and v1.rn = v1_lead.rn - 1) select * from v2 where d_year = 1999 and avg_monthly_sales > 0 and case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by sum_sales - avg_monthly_sales, avg_monthly_sales limit 100""", "q58" -> """ with ss_items as (select i_item_id item_id ,sum(ss_ext_sales_price) ss_item_rev from store_sales ,item ,date_dim where ss_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq = (select d_week_seq from date_dim where d_date = '1998-02-21')) and ss_sold_date_sk = d_date_sk group by i_item_id), cs_items as (select i_item_id item_id ,sum(cs_ext_sales_price) cs_item_rev from catalog_sales ,item ,date_dim where cs_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq = (select d_week_seq from date_dim where d_date = '1998-02-21')) and cs_sold_date_sk = d_date_sk group by i_item_id), ws_items as (select i_item_id item_id ,sum(ws_ext_sales_price) ws_item_rev from web_sales ,item ,date_dim where ws_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq =(select d_week_seq from date_dim where d_date = '1998-02-21')) and ws_sold_date_sk = d_date_sk group by i_item_id) select ss_items.item_id ,ss_item_rev ,ss_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ss_dev ,cs_item_rev ,cs_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 cs_dev ,ws_item_rev ,ws_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ws_dev ,(ss_item_rev+cs_item_rev+ws_item_rev)/3 average from ss_items,cs_items,ws_items where ss_items.item_id=cs_items.item_id and ss_items.item_id=ws_items.item_id and ss_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev and ss_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev and cs_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev and cs_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev and ws_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev and ws_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev order by item_id ,ss_item_rev limit 100""", "q59" -> """ with wss as (select d_week_seq, ss_store_sk, sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales from store_sales,date_dim where d_date_sk = ss_sold_date_sk group by d_week_seq,ss_store_sk ) select s_store_name1,s_store_id1,d_week_seq1 ,sun_sales1/sun_sales2,mon_sales1/mon_sales2 ,tue_sales1/tue_sales2,wed_sales1/wed_sales2,thu_sales1/thu_sales2 ,fri_sales1/fri_sales2,sat_sales1/sat_sales2 from (select s_store_name s_store_name1,wss.d_week_seq d_week_seq1 ,s_store_id s_store_id1,sun_sales sun_sales1 ,mon_sales mon_sales1,tue_sales tue_sales1 ,wed_sales wed_sales1,thu_sales thu_sales1 ,fri_sales fri_sales1,sat_sales sat_sales1 from wss,store,date_dim d where d.d_week_seq = wss.d_week_seq and ss_store_sk = s_store_sk and d_month_seq between 1205 and 1205 + 11) y, (select s_store_name s_store_name2,wss.d_week_seq d_week_seq2 ,s_store_id s_store_id2,sun_sales sun_sales2 ,mon_sales mon_sales2,tue_sales tue_sales2 ,wed_sales wed_sales2,thu_sales thu_sales2 ,fri_sales fri_sales2,sat_sales sat_sales2 from wss,store,date_dim d where d.d_week_seq = wss.d_week_seq and ss_store_sk = s_store_sk and d_month_seq between 1205+ 12 and 1205 + 23) x where s_store_id1=s_store_id2 and d_week_seq1=d_week_seq2-52 order by s_store_name1,s_store_id1,d_week_seq1 limit 100""", "q60" -> """ with ss as ( select i_item_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Children')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 10 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id), cs as ( select i_item_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Children')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 10 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id), ws as ( select i_item_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Children')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 10 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id) select i_item_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_item_id order by i_item_id ,total_sales limit 100""", "q61" -> """ select promotions,total,cast(promotions as decimal(15,4))/cast(total as decimal(15,4))*100 from (select sum(ss_ext_sales_price) promotions from store_sales ,store ,promotion ,date_dim ,customer ,customer_address ,item where ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ss_promo_sk = p_promo_sk and ss_customer_sk= c_customer_sk and ca_address_sk = c_current_addr_sk and ss_item_sk = i_item_sk and ca_gmt_offset = -6 and i_category = 'Sports' and (p_channel_dmail = 'Y' or p_channel_email = 'Y' or p_channel_tv = 'Y') and s_gmt_offset = -6 and d_year = 2001 and d_moy = 12) promotional_sales, (select sum(ss_ext_sales_price) total from store_sales ,store ,date_dim ,customer ,customer_address ,item where ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ss_customer_sk= c_customer_sk and ca_address_sk = c_current_addr_sk and ss_item_sk = i_item_sk and ca_gmt_offset = -6 and i_category = 'Sports' and s_gmt_offset = -6 and d_year = 2001 and d_moy = 12) all_sales order by promotions, total limit 100""", "q62" -> """ select substr(w_warehouse_name,1,20) ,sm_type ,web_name ,sum(case when (ws_ship_date_sk - ws_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 30) and (ws_ship_date_sk - ws_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 60) and (ws_ship_date_sk - ws_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 90) and (ws_ship_date_sk - ws_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from web_sales ,warehouse ,ship_mode ,web_site ,date_dim where d_month_seq between 1215 and 1215 + 11 and ws_ship_date_sk = d_date_sk and ws_warehouse_sk = w_warehouse_sk and ws_ship_mode_sk = sm_ship_mode_sk and ws_web_site_sk = web_site_sk group by substr(w_warehouse_name,1,20) ,sm_type ,web_name order by substr(w_warehouse_name,1,20) ,sm_type ,web_name limit 100""", "q63" -> """ select * from (select i_manager_id ,sum(ss_sales_price) sum_sales ,avg(sum(ss_sales_price)) over (partition by i_manager_id) avg_monthly_sales from item ,store_sales ,date_dim ,store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_month_seq in (1211,1211+1,1211+2,1211+3,1211+4,1211+5,1211+6,1211+7,1211+8,1211+9,1211+10,1211+11) and (( i_category in ('Books','Children','Electronics') and i_class in ('personal','portable','reference','self-help') and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', 'exportiunivamalg #9','scholaramalgamalg #9')) or( i_category in ('Women','Music','Men') and i_class in ('accessories','classical','fragrances','pants') and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 'importoamalg #1'))) group by i_manager_id, d_moy) tmp1 where case when avg_monthly_sales > 0 then abs (sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by i_manager_id ,avg_monthly_sales ,sum_sales limit 100""", "q64" -> """ with cs_ui as (select cs_item_sk ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund from catalog_sales ,catalog_returns where cs_item_sk = cr_item_sk and cs_order_number = cr_order_number group by cs_item_sk having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), cross_sales as (select i_product_name product_name ,i_item_sk item_sk ,s_store_name store_name ,s_zip store_zip ,ad1.ca_street_number b_street_number ,ad1.ca_street_name b_street_name ,ad1.ca_city b_city ,ad1.ca_zip b_zip ,ad2.ca_street_number c_street_number ,ad2.ca_street_name c_street_name ,ad2.ca_city c_city ,ad2.ca_zip c_zip ,d1.d_year as syear ,d2.d_year as fsyear ,d3.d_year s2year ,count(*) cnt ,sum(ss_wholesale_cost) s1 ,sum(ss_list_price) s2 ,sum(ss_coupon_amt) s3 FROM store_sales ,store_returns ,cs_ui ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,customer ,customer_demographics cd1 ,customer_demographics cd2 ,promotion ,household_demographics hd1 ,household_demographics hd2 ,customer_address ad1 ,customer_address ad2 ,income_band ib1 ,income_band ib2 ,item WHERE ss_store_sk = s_store_sk AND ss_sold_date_sk = d1.d_date_sk AND ss_customer_sk = c_customer_sk AND ss_cdemo_sk= cd1.cd_demo_sk AND ss_hdemo_sk = hd1.hd_demo_sk AND ss_addr_sk = ad1.ca_address_sk and ss_item_sk = i_item_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and ss_item_sk = cs_ui.cs_item_sk and c_current_cdemo_sk = cd2.cd_demo_sk AND c_current_hdemo_sk = hd2.hd_demo_sk AND c_current_addr_sk = ad2.ca_address_sk and c_first_sales_date_sk = d2.d_date_sk and c_first_shipto_date_sk = d3.d_date_sk and ss_promo_sk = p_promo_sk and hd1.hd_income_band_sk = ib1.ib_income_band_sk and hd2.hd_income_band_sk = ib2.ib_income_band_sk and cd1.cd_marital_status <> cd2.cd_marital_status and i_color in ('azure','gainsboro','misty','blush','hot','lemon') and i_current_price between 80 and 80 + 10 and i_current_price between 80 + 1 and 80 + 15 group by i_product_name ,i_item_sk ,s_store_name ,s_zip ,ad1.ca_street_number ,ad1.ca_street_name ,ad1.ca_city ,ad1.ca_zip ,ad2.ca_street_number ,ad2.ca_street_name ,ad2.ca_city ,ad2.ca_zip ,d1.d_year ,d2.d_year ,d3.d_year ) select cs1.product_name ,cs1.store_name ,cs1.store_zip ,cs1.b_street_number ,cs1.b_street_name ,cs1.b_city ,cs1.b_zip ,cs1.c_street_number ,cs1.c_street_name ,cs1.c_city ,cs1.c_zip ,cs1.syear ,cs1.cnt ,cs1.s1 as s11 ,cs1.s2 as s21 ,cs1.s3 as s31 ,cs2.s1 as s12 ,cs2.s2 as s22 ,cs2.s3 as s32 ,cs2.syear ,cs2.cnt from cross_sales cs1,cross_sales cs2 where cs1.item_sk=cs2.item_sk and cs1.syear = 1999 and cs2.syear = 1999 + 1 and cs2.cnt <= cs1.cnt and cs1.store_name = cs2.store_name and cs1.store_zip = cs2.store_zip order by cs1.product_name ,cs1.store_name ,cs2.cnt ,cs1.s1 ,cs2.s1""", "q65" -> """ select s_store_name, i_item_desc, sc.revenue, i_current_price, i_wholesale_cost, i_brand from store, item, (select ss_store_sk, avg(revenue) as ave from (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue from store_sales, date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1186 and 1186+11 group by ss_store_sk, ss_item_sk) sa group by ss_store_sk) sb, (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue from store_sales, date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1186 and 1186+11 group by ss_store_sk, ss_item_sk) sc where sb.ss_store_sk = sc.ss_store_sk and sc.revenue <= 0.1 * sb.ave and s_store_sk = sc.ss_store_sk and i_item_sk = sc.ss_item_sk order by s_store_name, i_item_desc limit 100""", "q66" -> """ select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,ship_carriers ,year ,sum(jan_sales) as jan_sales ,sum(feb_sales) as feb_sales ,sum(mar_sales) as mar_sales ,sum(apr_sales) as apr_sales ,sum(may_sales) as may_sales ,sum(jun_sales) as jun_sales ,sum(jul_sales) as jul_sales ,sum(aug_sales) as aug_sales ,sum(sep_sales) as sep_sales ,sum(oct_sales) as oct_sales ,sum(nov_sales) as nov_sales ,sum(dec_sales) as dec_sales ,sum(jan_sales/w_warehouse_sq_ft) as jan_sales_per_sq_foot ,sum(feb_sales/w_warehouse_sq_ft) as feb_sales_per_sq_foot ,sum(mar_sales/w_warehouse_sq_ft) as mar_sales_per_sq_foot ,sum(apr_sales/w_warehouse_sq_ft) as apr_sales_per_sq_foot ,sum(may_sales/w_warehouse_sq_ft) as may_sales_per_sq_foot ,sum(jun_sales/w_warehouse_sq_ft) as jun_sales_per_sq_foot ,sum(jul_sales/w_warehouse_sq_ft) as jul_sales_per_sq_foot ,sum(aug_sales/w_warehouse_sq_ft) as aug_sales_per_sq_foot ,sum(sep_sales/w_warehouse_sq_ft) as sep_sales_per_sq_foot ,sum(oct_sales/w_warehouse_sq_ft) as oct_sales_per_sq_foot ,sum(nov_sales/w_warehouse_sq_ft) as nov_sales_per_sq_foot ,sum(dec_sales/w_warehouse_sq_ft) as dec_sales_per_sq_foot ,sum(jan_net) as jan_net ,sum(feb_net) as feb_net ,sum(mar_net) as mar_net ,sum(apr_net) as apr_net ,sum(may_net) as may_net ,sum(jun_net) as jun_net ,sum(jul_net) as jul_net ,sum(aug_net) as aug_net ,sum(sep_net) as sep_net ,sum(oct_net) as oct_net ,sum(nov_net) as nov_net ,sum(dec_net) as dec_net from ( select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,'MSC' || ',' || 'GERMA' as ship_carriers ,d_year as year ,sum(case when d_moy = 1 then ws_sales_price* ws_quantity else 0 end) as jan_sales ,sum(case when d_moy = 2 then ws_sales_price* ws_quantity else 0 end) as feb_sales ,sum(case when d_moy = 3 then ws_sales_price* ws_quantity else 0 end) as mar_sales ,sum(case when d_moy = 4 then ws_sales_price* ws_quantity else 0 end) as apr_sales ,sum(case when d_moy = 5 then ws_sales_price* ws_quantity else 0 end) as may_sales ,sum(case when d_moy = 6 then ws_sales_price* ws_quantity else 0 end) as jun_sales ,sum(case when d_moy = 7 then ws_sales_price* ws_quantity else 0 end) as jul_sales ,sum(case when d_moy = 8 then ws_sales_price* ws_quantity else 0 end) as aug_sales ,sum(case when d_moy = 9 then ws_sales_price* ws_quantity else 0 end) as sep_sales ,sum(case when d_moy = 10 then ws_sales_price* ws_quantity else 0 end) as oct_sales ,sum(case when d_moy = 11 then ws_sales_price* ws_quantity else 0 end) as nov_sales ,sum(case when d_moy = 12 then ws_sales_price* ws_quantity else 0 end) as dec_sales ,sum(case when d_moy = 1 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as jan_net ,sum(case when d_moy = 2 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as feb_net ,sum(case when d_moy = 3 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as mar_net ,sum(case when d_moy = 4 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as apr_net ,sum(case when d_moy = 5 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as may_net ,sum(case when d_moy = 6 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as jun_net ,sum(case when d_moy = 7 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as jul_net ,sum(case when d_moy = 8 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as aug_net ,sum(case when d_moy = 9 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as sep_net ,sum(case when d_moy = 10 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as oct_net ,sum(case when d_moy = 11 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as nov_net ,sum(case when d_moy = 12 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as dec_net from web_sales ,warehouse ,date_dim ,time_dim ,ship_mode where ws_warehouse_sk = w_warehouse_sk and ws_sold_date_sk = d_date_sk and ws_sold_time_sk = t_time_sk and ws_ship_mode_sk = sm_ship_mode_sk and d_year = 2001 and t_time between 9453 and 9453+28800 and sm_carrier in ('MSC','GERMA') group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,d_year union all select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,'MSC' || ',' || 'GERMA' as ship_carriers ,d_year as year ,sum(case when d_moy = 1 then cs_ext_list_price* cs_quantity else 0 end) as jan_sales ,sum(case when d_moy = 2 then cs_ext_list_price* cs_quantity else 0 end) as feb_sales ,sum(case when d_moy = 3 then cs_ext_list_price* cs_quantity else 0 end) as mar_sales ,sum(case when d_moy = 4 then cs_ext_list_price* cs_quantity else 0 end) as apr_sales ,sum(case when d_moy = 5 then cs_ext_list_price* cs_quantity else 0 end) as may_sales ,sum(case when d_moy = 6 then cs_ext_list_price* cs_quantity else 0 end) as jun_sales ,sum(case when d_moy = 7 then cs_ext_list_price* cs_quantity else 0 end) as jul_sales ,sum(case when d_moy = 8 then cs_ext_list_price* cs_quantity else 0 end) as aug_sales ,sum(case when d_moy = 9 then cs_ext_list_price* cs_quantity else 0 end) as sep_sales ,sum(case when d_moy = 10 then cs_ext_list_price* cs_quantity else 0 end) as oct_sales ,sum(case when d_moy = 11 then cs_ext_list_price* cs_quantity else 0 end) as nov_sales ,sum(case when d_moy = 12 then cs_ext_list_price* cs_quantity else 0 end) as dec_sales ,sum(case when d_moy = 1 then cs_net_paid_inc_ship * cs_quantity else 0 end) as jan_net ,sum(case when d_moy = 2 then cs_net_paid_inc_ship * cs_quantity else 0 end) as feb_net ,sum(case when d_moy = 3 then cs_net_paid_inc_ship * cs_quantity else 0 end) as mar_net ,sum(case when d_moy = 4 then cs_net_paid_inc_ship * cs_quantity else 0 end) as apr_net ,sum(case when d_moy = 5 then cs_net_paid_inc_ship * cs_quantity else 0 end) as may_net ,sum(case when d_moy = 6 then cs_net_paid_inc_ship * cs_quantity else 0 end) as jun_net ,sum(case when d_moy = 7 then cs_net_paid_inc_ship * cs_quantity else 0 end) as jul_net ,sum(case when d_moy = 8 then cs_net_paid_inc_ship * cs_quantity else 0 end) as aug_net ,sum(case when d_moy = 9 then cs_net_paid_inc_ship * cs_quantity else 0 end) as sep_net ,sum(case when d_moy = 10 then cs_net_paid_inc_ship * cs_quantity else 0 end) as oct_net ,sum(case when d_moy = 11 then cs_net_paid_inc_ship * cs_quantity else 0 end) as nov_net ,sum(case when d_moy = 12 then cs_net_paid_inc_ship * cs_quantity else 0 end) as dec_net from catalog_sales ,warehouse ,date_dim ,time_dim ,ship_mode where cs_warehouse_sk = w_warehouse_sk and cs_sold_date_sk = d_date_sk and cs_sold_time_sk = t_time_sk and cs_ship_mode_sk = sm_ship_mode_sk and d_year = 2001 and t_time between 9453 AND 9453+28800 and sm_carrier in ('MSC','GERMA') group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,d_year ) x group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,ship_carriers ,year order by w_warehouse_name limit 100""", "q67" -> """ select * from (select i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sumsales ,rank() over (partition by i_category order by sumsales desc) rk from (select i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sum(coalesce(ss_sales_price*ss_quantity,0)) sumsales from store_sales ,date_dim ,store ,item where ss_sold_date_sk=d_date_sk and ss_item_sk=i_item_sk and ss_store_sk = s_store_sk and d_month_seq between 1185 and 1185+11 group by rollup(i_category, i_class, i_brand, i_product_name, d_year, d_qoy, d_moy,s_store_id))dw1) dw2 where rk <= 100 order by i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sumsales ,rk limit 100""", "q68" -> """ select c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number ,extended_price ,extended_tax ,list_price from (select ss_ticket_number ,ss_customer_sk ,ca_city bought_city ,sum(ss_ext_sales_price) extended_price ,sum(ss_ext_list_price) list_price ,sum(ss_ext_tax) extended_tax from store_sales ,date_dim ,store ,household_demographics ,customer_address where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and store_sales.ss_addr_sk = customer_address.ca_address_sk and date_dim.d_dom between 1 and 2 and (household_demographics.hd_dep_count = 4 or household_demographics.hd_vehicle_count= 0) and date_dim.d_year in (1999,1999+1,1999+2) and store.s_city in ('Pleasant Hill','Bethel') group by ss_ticket_number ,ss_customer_sk ,ss_addr_sk,ca_city) dn ,customer ,customer_address current_addr where ss_customer_sk = c_customer_sk and customer.c_current_addr_sk = current_addr.ca_address_sk and current_addr.ca_city <> bought_city order by c_last_name ,ss_ticket_number limit 100""", "q69" -> """ select cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3 from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and ca_state in ('MO','MN','AZ') and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2003 and d_moy between 2 and 2+2) and (not exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2003 and d_moy between 2 and 2+2) and not exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2003 and d_moy between 2 and 2+2)) group by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating order by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating limit 100""", "q70" -> """ select sum(ss_net_profit) as total_sum ,s_state ,s_county ,grouping(s_state)+grouping(s_county) as lochierarchy ,rank() over ( partition by grouping(s_state)+grouping(s_county), case when grouping(s_county) = 0 then s_state end order by sum(ss_net_profit) desc) as rank_within_parent from store_sales ,date_dim d1 ,store where d1.d_month_seq between 1218 and 1218+11 and d1.d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk and s_state in ( select s_state from (select s_state as s_state, rank() over ( partition by s_state order by sum(ss_net_profit) desc) as ranking from store_sales, store, date_dim where d_month_seq between 1218 and 1218+11 and d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk group by s_state ) tmp1 where ranking <= 5 ) group by rollup(s_state,s_county) order by lochierarchy desc ,case when lochierarchy = 0 then s_state end ,rank_within_parent limit 100""", "q71" -> """ select i_brand_id brand_id, i_brand brand,t_hour,t_minute, sum(ext_price) ext_price from item, (select ws_ext_sales_price as ext_price, ws_sold_date_sk as sold_date_sk, ws_item_sk as sold_item_sk, ws_sold_time_sk as time_sk from web_sales,date_dim where d_date_sk = ws_sold_date_sk and d_moy=12 and d_year=2000 union all select cs_ext_sales_price as ext_price, cs_sold_date_sk as sold_date_sk, cs_item_sk as sold_item_sk, cs_sold_time_sk as time_sk from catalog_sales,date_dim where d_date_sk = cs_sold_date_sk and d_moy=12 and d_year=2000 union all select ss_ext_sales_price as ext_price, ss_sold_date_sk as sold_date_sk, ss_item_sk as sold_item_sk, ss_sold_time_sk as time_sk from store_sales,date_dim where d_date_sk = ss_sold_date_sk and d_moy=12 and d_year=2000 ) tmp,time_dim where sold_item_sk = i_item_sk and i_manager_id=1 and time_sk = t_time_sk and (t_meal_time = 'breakfast' or t_meal_time = 'dinner') group by i_brand, i_brand_id,t_hour,t_minute order by ext_price desc, i_brand_id """, "q72" -> """ select i_item_desc ,w_warehouse_name ,d1.d_week_seq ,sum(case when p_promo_sk is null then 1 else 0 end) no_promo ,sum(case when p_promo_sk is not null then 1 else 0 end) promo ,count(*) total_cnt from catalog_sales join inventory on (cs_item_sk = inv_item_sk) join warehouse on (w_warehouse_sk=inv_warehouse_sk) join item on (i_item_sk = cs_item_sk) join customer_demographics on (cs_bill_cdemo_sk = cd_demo_sk) join household_demographics on (cs_bill_hdemo_sk = hd_demo_sk) join date_dim d1 on (cs_sold_date_sk = d1.d_date_sk) join date_dim d2 on (inv_date_sk = d2.d_date_sk) join date_dim d3 on (cs_ship_date_sk = d3.d_date_sk) left outer join promotion on (cs_promo_sk=p_promo_sk) left outer join catalog_returns on (cr_item_sk = cs_item_sk and cr_order_number = cs_order_number) where d1.d_week_seq = d2.d_week_seq and inv_quantity_on_hand < cs_quantity and d3.d_date > d1.d_date + interval 5 days and hd_buy_potential = '1001-5000' and d1.d_year = 2000 and cd_marital_status = 'D' group by i_item_desc,w_warehouse_name,d1.d_week_seq order by total_cnt desc, i_item_desc, w_warehouse_name, d_week_seq limit 100""", "q73" -> """ select c_last_name ,c_first_name ,c_salutation ,c_preferred_cust_flag ,ss_ticket_number ,cnt from (select ss_ticket_number ,ss_customer_sk ,count(*) cnt from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and date_dim.d_dom between 1 and 2 and (household_demographics.hd_buy_potential = '>10000' or household_demographics.hd_buy_potential = '5001-10000') and household_demographics.hd_vehicle_count > 0 and case when household_demographics.hd_vehicle_count > 0 then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end > 1 and date_dim.d_year in (2000,2000+1,2000+2) and store.s_county in ('Lea County','Furnas County','Pennington County','Bronx County') group by ss_ticket_number,ss_customer_sk) dj,customer where ss_customer_sk = c_customer_sk and cnt between 1 and 5 order by cnt desc, c_last_name asc""", "q74" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,d_year as year ,sum(ss_net_paid) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (1998,1998+1) group by c_customer_id ,c_first_name ,c_last_name ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,d_year as year ,sum(ws_net_paid) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year in (1998,1998+1) group by c_customer_id ,c_first_name ,c_last_name ,d_year ) select t_s_secyear.customer_id, t_s_secyear.customer_first_name, t_s_secyear.customer_last_name from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.sale_type = 's' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_w_secyear.sale_type = 'w' and t_s_firstyear.year = 1998 and t_s_secyear.year = 1998+1 and t_w_firstyear.year = 1998 and t_w_secyear.year = 1998+1 and t_s_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end order by 3,1,2 limit 100""", "q75" -> """ WITH all_sales AS ( SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,SUM(sales_cnt) AS sales_cnt ,SUM(sales_amt) AS sales_amt FROM (SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,cs_quantity - COALESCE(cr_return_quantity,0) AS sales_cnt ,cs_ext_sales_price - COALESCE(cr_return_amount,0.0) AS sales_amt FROM catalog_sales JOIN item ON i_item_sk=cs_item_sk JOIN date_dim ON d_date_sk=cs_sold_date_sk LEFT JOIN catalog_returns ON (cs_order_number=cr_order_number AND cs_item_sk=cr_item_sk) WHERE i_category='Sports' UNION SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,ss_quantity - COALESCE(sr_return_quantity,0) AS sales_cnt ,ss_ext_sales_price - COALESCE(sr_return_amt,0.0) AS sales_amt FROM store_sales JOIN item ON i_item_sk=ss_item_sk JOIN date_dim ON d_date_sk=ss_sold_date_sk LEFT JOIN store_returns ON (ss_ticket_number=sr_ticket_number AND ss_item_sk=sr_item_sk) WHERE i_category='Sports' UNION SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,ws_quantity - COALESCE(wr_return_quantity,0) AS sales_cnt ,ws_ext_sales_price - COALESCE(wr_return_amt,0.0) AS sales_amt FROM web_sales JOIN item ON i_item_sk=ws_item_sk JOIN date_dim ON d_date_sk=ws_sold_date_sk LEFT JOIN web_returns ON (ws_order_number=wr_order_number AND ws_item_sk=wr_item_sk) WHERE i_category='Sports') sales_detail GROUP BY d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id) SELECT prev_yr.d_year AS prev_year ,curr_yr.d_year AS year ,curr_yr.i_brand_id ,curr_yr.i_class_id ,curr_yr.i_category_id ,curr_yr.i_manufact_id ,prev_yr.sales_cnt AS prev_yr_cnt ,curr_yr.sales_cnt AS curr_yr_cnt ,curr_yr.sales_cnt-prev_yr.sales_cnt AS sales_cnt_diff ,curr_yr.sales_amt-prev_yr.sales_amt AS sales_amt_diff FROM all_sales curr_yr, all_sales prev_yr WHERE curr_yr.i_brand_id=prev_yr.i_brand_id AND curr_yr.i_class_id=prev_yr.i_class_id AND curr_yr.i_category_id=prev_yr.i_category_id AND curr_yr.i_manufact_id=prev_yr.i_manufact_id AND curr_yr.d_year=2001 AND prev_yr.d_year=2001-1 AND CAST(curr_yr.sales_cnt AS DECIMAL(17,2))/CAST(prev_yr.sales_cnt AS DECIMAL(17,2))<0.9 ORDER BY sales_cnt_diff,sales_amt_diff limit 100""", "q76" -> """ select channel, col_name, d_year, d_qoy, i_category, COUNT(*) sales_cnt, SUM(ext_sales_price) sales_amt FROM ( SELECT 'store' as channel, 'ss_customer_sk' col_name, d_year, d_qoy, i_category, ss_ext_sales_price ext_sales_price FROM store_sales, item, date_dim WHERE ss_customer_sk IS NULL AND ss_sold_date_sk=d_date_sk AND ss_item_sk=i_item_sk UNION ALL SELECT 'web' as channel, 'ws_ship_addr_sk' col_name, d_year, d_qoy, i_category, ws_ext_sales_price ext_sales_price FROM web_sales, item, date_dim WHERE ws_ship_addr_sk IS NULL AND ws_sold_date_sk=d_date_sk AND ws_item_sk=i_item_sk UNION ALL SELECT 'catalog' as channel, 'cs_ship_mode_sk' col_name, d_year, d_qoy, i_category, cs_ext_sales_price ext_sales_price FROM catalog_sales, item, date_dim WHERE cs_ship_mode_sk IS NULL AND cs_sold_date_sk=d_date_sk AND cs_item_sk=i_item_sk) foo GROUP BY channel, col_name, d_year, d_qoy, i_category ORDER BY channel, col_name, d_year, d_qoy, i_category limit 100""", "q77" -> """ with ss as (select s_store_sk, sum(ss_ext_sales_price) as sales, sum(ss_net_profit) as profit from store_sales, date_dim, store where ss_sold_date_sk = d_date_sk and d_date between cast('2000-08-16' as date) and (cast('2000-08-16' as date) + INTERVAL 30 days) and ss_store_sk = s_store_sk group by s_store_sk) , sr as (select s_store_sk, sum(sr_return_amt) as returns, sum(sr_net_loss) as profit_loss from store_returns, date_dim, store where sr_returned_date_sk = d_date_sk and d_date between cast('2000-08-16' as date) and (cast('2000-08-16' as date) + INTERVAL 30 days) and sr_store_sk = s_store_sk group by s_store_sk), cs as (select cs_call_center_sk, sum(cs_ext_sales_price) as sales, sum(cs_net_profit) as profit from catalog_sales, date_dim where cs_sold_date_sk = d_date_sk and d_date between cast('2000-08-16' as date) and (cast('2000-08-16' as date) + INTERVAL 30 days) group by cs_call_center_sk ), cr as (select cr_call_center_sk, sum(cr_return_amount) as returns, sum(cr_net_loss) as profit_loss from catalog_returns, date_dim where cr_returned_date_sk = d_date_sk and d_date between cast('2000-08-16' as date) and (cast('2000-08-16' as date) + INTERVAL 30 days) group by cr_call_center_sk ), ws as ( select wp_web_page_sk, sum(ws_ext_sales_price) as sales, sum(ws_net_profit) as profit from web_sales, date_dim, web_page where ws_sold_date_sk = d_date_sk and d_date between cast('2000-08-16' as date) and (cast('2000-08-16' as date) + INTERVAL 30 days) and ws_web_page_sk = wp_web_page_sk group by wp_web_page_sk), wr as (select wp_web_page_sk, sum(wr_return_amt) as returns, sum(wr_net_loss) as profit_loss from web_returns, date_dim, web_page where wr_returned_date_sk = d_date_sk and d_date between cast('2000-08-16' as date) and (cast('2000-08-16' as date) + INTERVAL 30 days) and wr_web_page_sk = wp_web_page_sk group by wp_web_page_sk) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , ss.s_store_sk as id , sales , coalesce(returns, 0) as returns , (profit - coalesce(profit_loss,0)) as profit from ss left join sr on ss.s_store_sk = sr.s_store_sk union all select 'catalog channel' as channel , cs_call_center_sk as id , sales , returns , (profit - profit_loss) as profit from cs , cr union all select 'web channel' as channel , ws.wp_web_page_sk as id , sales , coalesce(returns, 0) returns , (profit - coalesce(profit_loss,0)) as profit from ws left join wr on ws.wp_web_page_sk = wr.wp_web_page_sk ) x group by rollup (channel, id) order by channel ,id limit 100""", "q78" -> """ with ws as (select d_year AS ws_sold_year, ws_item_sk, ws_bill_customer_sk ws_customer_sk, sum(ws_quantity) ws_qty, sum(ws_wholesale_cost) ws_wc, sum(ws_sales_price) ws_sp from web_sales left join web_returns on wr_order_number=ws_order_number and ws_item_sk=wr_item_sk join date_dim on ws_sold_date_sk = d_date_sk where wr_order_number is null group by d_year, ws_item_sk, ws_bill_customer_sk ), cs as (select d_year AS cs_sold_year, cs_item_sk, cs_bill_customer_sk cs_customer_sk, sum(cs_quantity) cs_qty, sum(cs_wholesale_cost) cs_wc, sum(cs_sales_price) cs_sp from catalog_sales left join catalog_returns on cr_order_number=cs_order_number and cs_item_sk=cr_item_sk join date_dim on cs_sold_date_sk = d_date_sk where cr_order_number is null group by d_year, cs_item_sk, cs_bill_customer_sk ), ss as (select d_year AS ss_sold_year, ss_item_sk, ss_customer_sk, sum(ss_quantity) ss_qty, sum(ss_wholesale_cost) ss_wc, sum(ss_sales_price) ss_sp from store_sales left join store_returns on sr_ticket_number=ss_ticket_number and ss_item_sk=sr_item_sk join date_dim on ss_sold_date_sk = d_date_sk where sr_ticket_number is null group by d_year, ss_item_sk, ss_customer_sk ) select ss_customer_sk, round(ss_qty/(coalesce(ws_qty,0)+coalesce(cs_qty,0)),2) ratio, ss_qty store_qty, ss_wc store_wholesale_cost, ss_sp store_sales_price, coalesce(ws_qty,0)+coalesce(cs_qty,0) other_chan_qty, coalesce(ws_wc,0)+coalesce(cs_wc,0) other_chan_wholesale_cost, coalesce(ws_sp,0)+coalesce(cs_sp,0) other_chan_sales_price from ss left join ws on (ws_sold_year=ss_sold_year and ws_item_sk=ss_item_sk and ws_customer_sk=ss_customer_sk) left join cs on (cs_sold_year=ss_sold_year and cs_item_sk=ss_item_sk and cs_customer_sk=ss_customer_sk) where (coalesce(ws_qty,0)>0 or coalesce(cs_qty, 0)>0) and ss_sold_year=2001 order by ss_customer_sk, ss_qty desc, ss_wc desc, ss_sp desc, other_chan_qty, other_chan_wholesale_cost, other_chan_sales_price, ratio limit 100""", "q79" -> """ select c_last_name,c_first_name,substr(s_city,1,30),ss_ticket_number,amt,profit from (select ss_ticket_number ,ss_customer_sk ,store.s_city ,sum(ss_coupon_amt) amt ,sum(ss_net_profit) profit from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and (household_demographics.hd_dep_count = 0 or household_demographics.hd_vehicle_count > 3) and date_dim.d_dow = 1 and date_dim.d_year in (1998,1998+1,1998+2) and store.s_number_employees between 200 and 295 group by ss_ticket_number,ss_customer_sk,ss_addr_sk,store.s_city) ms,customer where ss_customer_sk = c_customer_sk order by c_last_name,c_first_name,substr(s_city,1,30), profit limit 100""", "q80" -> """ with ssr as (select s_store_id as store_id, sum(ss_ext_sales_price) as sales, sum(coalesce(sr_return_amt, 0)) as returns, sum(ss_net_profit - coalesce(sr_net_loss, 0)) as profit from store_sales left outer join store_returns on (ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number), date_dim, store, item, promotion where ss_sold_date_sk = d_date_sk and d_date between cast('2002-08-06' as date) and (cast('2002-08-06' as date) + INTERVAL 60 days) and ss_store_sk = s_store_sk and ss_item_sk = i_item_sk and i_current_price > 50 and ss_promo_sk = p_promo_sk and p_channel_tv = 'N' group by s_store_id) , csr as (select cp_catalog_page_id as catalog_page_id, sum(cs_ext_sales_price) as sales, sum(coalesce(cr_return_amount, 0)) as returns, sum(cs_net_profit - coalesce(cr_net_loss, 0)) as profit from catalog_sales left outer join catalog_returns on (cs_item_sk = cr_item_sk and cs_order_number = cr_order_number), date_dim, catalog_page, item, promotion where cs_sold_date_sk = d_date_sk and d_date between cast('2002-08-06' as date) and (cast('2002-08-06' as date) + INTERVAL 60 days) and cs_catalog_page_sk = cp_catalog_page_sk and cs_item_sk = i_item_sk and i_current_price > 50 and cs_promo_sk = p_promo_sk and p_channel_tv = 'N' group by cp_catalog_page_id) , wsr as (select web_site_id, sum(ws_ext_sales_price) as sales, sum(coalesce(wr_return_amt, 0)) as returns, sum(ws_net_profit - coalesce(wr_net_loss, 0)) as profit from web_sales left outer join web_returns on (ws_item_sk = wr_item_sk and ws_order_number = wr_order_number), date_dim, web_site, item, promotion where ws_sold_date_sk = d_date_sk and d_date between cast('2002-08-06' as date) and (cast('2002-08-06' as date) + INTERVAL 60 days) and ws_web_site_sk = web_site_sk and ws_item_sk = i_item_sk and i_current_price > 50 and ws_promo_sk = p_promo_sk and p_channel_tv = 'N' group by web_site_id) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , 'store' || store_id as id , sales , returns , profit from ssr union all select 'catalog channel' as channel , 'catalog_page' || catalog_page_id as id , sales , returns , profit from csr union all select 'web channel' as channel , 'web_site' || web_site_id as id , sales , returns , profit from wsr ) x group by rollup (channel, id) order by channel ,id limit 100""", "q81" -> """ with customer_total_return as (select cr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, sum(cr_return_amt_inc_tax) as ctr_total_return from catalog_returns ,date_dim ,customer_address where cr_returned_date_sk = d_date_sk and d_year =1998 and cr_returning_addr_sk = ca_address_sk group by cr_returning_customer_sk ,ca_state ) select c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset ,ca_location_type,ctr_total_return from customer_total_return ctr1 ,customer_address ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_state = ctr2.ctr_state) and ca_address_sk = c_current_addr_sk and ca_state = 'TX' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset ,ca_location_type,ctr_total_return limit 100""", "q82" -> """ select i_item_id ,i_item_desc ,i_current_price from item, inventory, date_dim, store_sales where i_current_price between 49 and 49+30 and inv_item_sk = i_item_sk and d_date_sk=inv_date_sk and d_date between cast('2001-01-28' as date) and (cast('2001-01-28' as date) + INTERVAL 60 days) and i_manufact_id in (80,675,292,17) and inv_quantity_on_hand between 100 and 500 and ss_item_sk = i_item_sk group by i_item_id,i_item_desc,i_current_price order by i_item_id limit 100""", "q83" -> """ with sr_items as (select i_item_id item_id, sum(sr_return_quantity) sr_item_qty from store_returns, item, date_dim where sr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('2000-06-17','2000-08-22','2000-11-17'))) and sr_returned_date_sk = d_date_sk group by i_item_id), cr_items as (select i_item_id item_id, sum(cr_return_quantity) cr_item_qty from catalog_returns, item, date_dim where cr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('2000-06-17','2000-08-22','2000-11-17'))) and cr_returned_date_sk = d_date_sk group by i_item_id), wr_items as (select i_item_id item_id, sum(wr_return_quantity) wr_item_qty from web_returns, item, date_dim where wr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('2000-06-17','2000-08-22','2000-11-17'))) and wr_returned_date_sk = d_date_sk group by i_item_id) select sr_items.item_id ,sr_item_qty ,sr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 sr_dev ,cr_item_qty ,cr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 cr_dev ,wr_item_qty ,wr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 wr_dev ,(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 average from sr_items ,cr_items ,wr_items where sr_items.item_id=cr_items.item_id and sr_items.item_id=wr_items.item_id order by sr_items.item_id ,sr_item_qty limit 100""", "q84" -> """ select c_customer_id as customer_id , coalesce(c_last_name,'') || ', ' || coalesce(c_first_name,'') as customername from customer ,customer_address ,customer_demographics ,household_demographics ,income_band ,store_returns where ca_city = 'Hopewell' and c_current_addr_sk = ca_address_sk and ib_lower_bound >= 37855 and ib_upper_bound <= 37855 + 50000 and ib_income_band_sk = hd_income_band_sk and cd_demo_sk = c_current_cdemo_sk and hd_demo_sk = c_current_hdemo_sk and sr_cdemo_sk = cd_demo_sk order by c_customer_id limit 100""", "q85" -> """ select substr(r_reason_desc,1,20) ,avg(ws_quantity) ,avg(wr_refunded_cash) ,avg(wr_fee) from web_sales, web_returns, web_page, customer_demographics cd1, customer_demographics cd2, customer_address, date_dim, reason where ws_web_page_sk = wp_web_page_sk and ws_item_sk = wr_item_sk and ws_order_number = wr_order_number and ws_sold_date_sk = d_date_sk and d_year = 2001 and cd1.cd_demo_sk = wr_refunded_cdemo_sk and cd2.cd_demo_sk = wr_returning_cdemo_sk and ca_address_sk = wr_refunded_addr_sk and r_reason_sk = wr_reason_sk and ( ( cd1.cd_marital_status = 'M' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = '4 yr Degree' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 100.00 and 150.00 ) or ( cd1.cd_marital_status = 'S' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = 'College' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 50.00 and 100.00 ) or ( cd1.cd_marital_status = 'D' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = 'Secondary' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 150.00 and 200.00 ) ) and ( ( ca_country = 'United States' and ca_state in ('TX', 'VA', 'CA') and ws_net_profit between 100 and 200 ) or ( ca_country = 'United States' and ca_state in ('AR', 'NE', 'MO') and ws_net_profit between 150 and 300 ) or ( ca_country = 'United States' and ca_state in ('IA', 'MS', 'WA') and ws_net_profit between 50 and 250 ) ) group by r_reason_desc order by substr(r_reason_desc,1,20) ,avg(ws_quantity) ,avg(wr_refunded_cash) ,avg(wr_fee) limit 100""", "q86" -> """ select sum(ws_net_paid) as total_sum ,i_category ,i_class ,grouping(i_category)+grouping(i_class) as lochierarchy ,rank() over ( partition by grouping(i_category)+grouping(i_class), case when grouping(i_class) = 0 then i_category end order by sum(ws_net_paid) desc) as rank_within_parent from web_sales ,date_dim d1 ,item where d1.d_month_seq between 1215 and 1215+11 and d1.d_date_sk = ws_sold_date_sk and i_item_sk = ws_item_sk group by rollup(i_category,i_class) order by lochierarchy desc, case when lochierarchy = 0 then i_category end, rank_within_parent limit 100""", "q87" -> """ select count(*) from ((select distinct c_last_name, c_first_name, d_date from store_sales, date_dim, customer where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_customer_sk = customer.c_customer_sk and d_month_seq between 1221 and 1221+11) except (select distinct c_last_name, c_first_name, d_date from catalog_sales, date_dim, customer where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1221 and 1221+11) except (select distinct c_last_name, c_first_name, d_date from web_sales, date_dim, customer where web_sales.ws_sold_date_sk = date_dim.d_date_sk and web_sales.ws_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1221 and 1221+11) ) cool_cust""", "q88" -> """ select * from (select count(*) h8_30_to_9 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 8 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s1, (select count(*) h9_to_9_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 9 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s2, (select count(*) h9_30_to_10 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 9 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s3, (select count(*) h10_to_10_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 10 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s4, (select count(*) h10_30_to_11 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 10 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s5, (select count(*) h11_to_11_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 11 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s6, (select count(*) h11_30_to_12 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 11 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s7, (select count(*) h12_to_12_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 12 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s8""", "q89" -> """ select * from( select i_category, i_class, i_brand, s_store_name, s_company_name, d_moy, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_category, i_brand, s_store_name, s_company_name) avg_monthly_sales from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_year in (2000) and ((i_category in ('Home','Music','Books') and i_class in ('glassware','classical','fiction') ) or (i_category in ('Jewelry','Sports','Women') and i_class in ('semi-precious','baseball','dresses') )) group by i_category, i_class, i_brand, s_store_name, s_company_name, d_moy) tmp1 where case when (avg_monthly_sales <> 0) then (abs(sum_sales - avg_monthly_sales) / avg_monthly_sales) else null end > 0.1 order by sum_sales - avg_monthly_sales, s_store_name limit 100""", "q90" -> """ select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio from ( select count(*) amc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 9 and 9+1 and household_demographics.hd_dep_count = 3 and web_page.wp_char_count between 5000 and 5200) at, ( select count(*) pmc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 16 and 16+1 and household_demographics.hd_dep_count = 3 and web_page.wp_char_count between 5000 and 5200) pt order by am_pm_ratio limit 100""", "q91" -> """ select cc_call_center_id Call_Center, cc_name Call_Center_Name, cc_manager Manager, sum(cr_net_loss) Returns_Loss from call_center, catalog_returns, date_dim, customer, customer_address, customer_demographics, household_demographics where cr_call_center_sk = cc_call_center_sk and cr_returned_date_sk = d_date_sk and cr_returning_customer_sk= c_customer_sk and cd_demo_sk = c_current_cdemo_sk and hd_demo_sk = c_current_hdemo_sk and ca_address_sk = c_current_addr_sk and d_year = 2000 and d_moy = 12 and ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') or(cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) and hd_buy_potential like 'Unknown%' and ca_gmt_offset = -7 group by cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status order by sum(cr_net_loss) desc""", "q92" -> """ select sum(ws_ext_discount_amt) as `Excess Discount Amount` from web_sales ,item ,date_dim where i_manufact_id = 356 and i_item_sk = ws_item_sk and d_date between '2001-03-12' and (cast('2001-03-12' as date) + INTERVAL 90 days) and d_date_sk = ws_sold_date_sk and ws_ext_discount_amt > ( SELECT 1.3 * avg(ws_ext_discount_amt) FROM web_sales ,date_dim WHERE ws_item_sk = i_item_sk and d_date between '2001-03-12' and (cast('2001-03-12' as date) + INTERVAL 90 days) and d_date_sk = ws_sold_date_sk ) order by sum(ws_ext_discount_amt) limit 100""", "q93" -> """ select ss_customer_sk ,sum(act_sales) sumsales from (select ss_item_sk ,ss_ticket_number ,ss_customer_sk ,case when sr_return_quantity is not null then (ss_quantity-sr_return_quantity)*ss_sales_price else (ss_quantity*ss_sales_price) end act_sales from store_sales left outer join store_returns on (sr_item_sk = ss_item_sk and sr_ticket_number = ss_ticket_number) ,reason where sr_reason_sk = r_reason_sk and r_reason_desc = 'reason 66') t group by ss_customer_sk order by sumsales, ss_customer_sk limit 100""", "q94" -> """ select count(distinct ws_order_number) as `order count` ,sum(ws_ext_ship_cost) as `total shipping cost` ,sum(ws_net_profit) as `total net profit` from web_sales ws1 ,date_dim ,customer_address ,web_site where d_date between '1999-4-01' and (cast('1999-4-01' as date) + INTERVAL 60 days) and ws1.ws_ship_date_sk = d_date_sk and ws1.ws_ship_addr_sk = ca_address_sk and ca_state = 'NE' and ws1.ws_web_site_sk = web_site_sk and web_company_name = 'pri' and exists (select * from web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) and not exists(select * from web_returns wr1 where ws1.ws_order_number = wr1.wr_order_number) order by count(distinct ws_order_number) limit 100""", "q95" -> """ with ws_wh as (select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 from web_sales ws1,web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) select count(distinct ws_order_number) as `order count` ,sum(ws_ext_ship_cost) as `total shipping cost` ,sum(ws_net_profit) as `total net profit` from web_sales ws1 ,date_dim ,customer_address ,web_site where d_date between '2002-4-01' and (cast('2002-4-01' as date) + INTERVAL 60 days) and ws1.ws_ship_date_sk = d_date_sk and ws1.ws_ship_addr_sk = ca_address_sk and ca_state = 'AL' and ws1.ws_web_site_sk = web_site_sk and web_company_name = 'pri' and ws1.ws_order_number in (select ws_order_number from ws_wh) and ws1.ws_order_number in (select wr_order_number from web_returns,ws_wh where wr_order_number = ws_wh.ws_order_number) order by count(distinct ws_order_number) limit 100""", "q96" -> """ select count(*) from store_sales ,household_demographics ,time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 16 and time_dim.t_minute >= 30 and household_demographics.hd_dep_count = 6 and store.s_store_name = 'ese' order by count(*) limit 100""", "q97" -> """ with ssci as ( select ss_customer_sk customer_sk ,ss_item_sk item_sk from store_sales,date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1190 and 1190 + 11 group by ss_customer_sk ,ss_item_sk), csci as( select cs_bill_customer_sk customer_sk ,cs_item_sk item_sk from catalog_sales,date_dim where cs_sold_date_sk = d_date_sk and d_month_seq between 1190 and 1190 + 11 group by cs_bill_customer_sk ,cs_item_sk) select sum(case when ssci.customer_sk is not null and csci.customer_sk is null then 1 else 0 end) store_only ,sum(case when ssci.customer_sk is null and csci.customer_sk is not null then 1 else 0 end) catalog_only ,sum(case when ssci.customer_sk is not null and csci.customer_sk is not null then 1 else 0 end) store_and_catalog from ssci full outer join csci on (ssci.customer_sk=csci.customer_sk and ssci.item_sk = csci.item_sk) limit 100""", "q98" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(ss_ext_sales_price) as itemrevenue ,sum(ss_ext_sales_price)*100/sum(sum(ss_ext_sales_price)) over (partition by i_class) as revenueratio from store_sales ,item ,date_dim where ss_item_sk = i_item_sk and i_category in ('Home', 'Sports', 'Men') and ss_sold_date_sk = d_date_sk and d_date between cast('2002-01-05' as date) and (cast('2002-01-05' as date) + interval 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio""", "q99" -> """ select substr(w_warehouse_name,1,20) ,sm_type ,cc_name ,sum(case when (cs_ship_date_sk - cs_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 30) and (cs_ship_date_sk - cs_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 60) and (cs_ship_date_sk - cs_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 90) and (cs_ship_date_sk - cs_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from catalog_sales ,warehouse ,ship_mode ,call_center ,date_dim where d_month_seq between 1178 and 1178 + 11 and cs_ship_date_sk = d_date_sk and cs_warehouse_sk = w_warehouse_sk and cs_ship_mode_sk = sm_ship_mode_sk and cs_call_center_sk = cc_call_center_sk group by substr(w_warehouse_name,1,20) ,sm_type ,cc_name order by substr(w_warehouse_name,1,20) ,sm_type ,cc_name limit 100""", "q1" -> """ with customer_total_return as (select sr_customer_sk as ctr_customer_sk ,sr_store_sk as ctr_store_sk ,sum(SR_FEE) as ctr_total_return from store_returns ,date_dim where sr_returned_date_sk = d_date_sk and d_year =2000 group by sr_customer_sk ,sr_store_sk) select c_customer_id from customer_total_return ctr1 ,store ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_store_sk = ctr2.ctr_store_sk) and s_store_sk = ctr1.ctr_store_sk and s_state = 'NY' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id limit 100""", "q2" -> """ with wscs as (select sold_date_sk ,sales_price from (select ws_sold_date_sk sold_date_sk ,ws_ext_sales_price sales_price from web_sales union all select cs_sold_date_sk sold_date_sk ,cs_ext_sales_price sales_price from catalog_sales)), wswscs as (select d_week_seq, sum(case when (d_day_name='Sunday') then sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then sales_price else null end) sat_sales from wscs ,date_dim where d_date_sk = sold_date_sk group by d_week_seq) select d_week_seq1 ,round(sun_sales1/sun_sales2,2) ,round(mon_sales1/mon_sales2,2) ,round(tue_sales1/tue_sales2,2) ,round(wed_sales1/wed_sales2,2) ,round(thu_sales1/thu_sales2,2) ,round(fri_sales1/fri_sales2,2) ,round(sat_sales1/sat_sales2,2) from (select wswscs.d_week_seq d_week_seq1 ,sun_sales sun_sales1 ,mon_sales mon_sales1 ,tue_sales tue_sales1 ,wed_sales wed_sales1 ,thu_sales thu_sales1 ,fri_sales fri_sales1 ,sat_sales sat_sales1 from wswscs,date_dim where date_dim.d_week_seq = wswscs.d_week_seq and d_year = 1998) y, (select wswscs.d_week_seq d_week_seq2 ,sun_sales sun_sales2 ,mon_sales mon_sales2 ,tue_sales tue_sales2 ,wed_sales wed_sales2 ,thu_sales thu_sales2 ,fri_sales fri_sales2 ,sat_sales sat_sales2 from wswscs ,date_dim where date_dim.d_week_seq = wswscs.d_week_seq and d_year = 1998+1) z where d_week_seq1=d_week_seq2-53 order by d_week_seq1""", "q3" -> """ select dt.d_year ,item.i_brand_id brand_id ,item.i_brand brand ,sum(ss_sales_price) sum_agg from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manufact_id = 816 and dt.d_moy=11 group by dt.d_year ,item.i_brand ,item.i_brand_id order by dt.d_year ,sum_agg desc ,brand_id limit 100""", "q4" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum((((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2) ) year_total ,'c' sale_type from customer ,catalog_sales ,date_dim where c_customer_sk = cs_bill_customer_sk and cs_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum((((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2) ) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year ) select t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_birth_country from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_c_firstyear ,year_total t_c_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_c_secyear.customer_id and t_s_firstyear.customer_id = t_c_firstyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.sale_type = 's' and t_c_firstyear.sale_type = 'c' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_c_secyear.sale_type = 'c' and t_w_secyear.sale_type = 'w' and t_s_firstyear.dyear = 1999 and t_s_secyear.dyear = 1999+1 and t_c_firstyear.dyear = 1999 and t_c_secyear.dyear = 1999+1 and t_w_firstyear.dyear = 1999 and t_w_secyear.dyear = 1999+1 and t_s_firstyear.year_total > 0 and t_c_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end > case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end order by t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_birth_country limit 100""", "q5" -> """ with ssr as (select s_store_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select ss_store_sk as store_sk, ss_sold_date_sk as date_sk, ss_ext_sales_price as sales_price, ss_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from store_sales union all select sr_store_sk as store_sk, sr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, sr_return_amt as return_amt, sr_net_loss as net_loss from store_returns ) salesreturns, date_dim, store where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and store_sk = s_store_sk group by s_store_id) , csr as (select cp_catalog_page_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select cs_catalog_page_sk as page_sk, cs_sold_date_sk as date_sk, cs_ext_sales_price as sales_price, cs_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from catalog_sales union all select cr_catalog_page_sk as page_sk, cr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, cr_return_amount as return_amt, cr_net_loss as net_loss from catalog_returns ) salesreturns, date_dim, catalog_page where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and page_sk = cp_catalog_page_sk group by cp_catalog_page_id) , wsr as (select web_site_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select ws_web_site_sk as wsr_web_site_sk, ws_sold_date_sk as date_sk, ws_ext_sales_price as sales_price, ws_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from web_sales union all select ws_web_site_sk as wsr_web_site_sk, wr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, wr_return_amt as return_amt, wr_net_loss as net_loss from web_returns left outer join web_sales on ( wr_item_sk = ws_item_sk and wr_order_number = ws_order_number) ) salesreturns, date_dim, web_site where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and wsr_web_site_sk = web_site_sk group by web_site_id) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , 'store' || s_store_id as id , sales , returns , (profit - profit_loss) as profit from ssr union all select 'catalog channel' as channel , 'catalog_page' || cp_catalog_page_id as id , sales , returns , (profit - profit_loss) as profit from csr union all select 'web channel' as channel , 'web_site' || web_site_id as id , sales , returns , (profit - profit_loss) as profit from wsr ) x group by rollup (channel, id) order by channel ,id limit 100""", "q6" -> """ select a.ca_state state, count(*) cnt from customer_address a ,customer c ,store_sales s ,date_dim d ,item i where a.ca_address_sk = c.c_current_addr_sk and c.c_customer_sk = s.ss_customer_sk and s.ss_sold_date_sk = d.d_date_sk and s.ss_item_sk = i.i_item_sk and d.d_month_seq = (select distinct (d_month_seq) from date_dim where d_year = 2002 and d_moy = 3 ) and i.i_current_price > 1.2 * (select avg(j.i_current_price) from item j where j.i_category = i.i_category) group by a.ca_state having count(*) >= 10 order by cnt, a.ca_state limit 100""", "q7" -> """ select i_item_id, avg(ss_quantity) agg1, avg(ss_list_price) agg2, avg(ss_coupon_amt) agg3, avg(ss_sales_price) agg4 from store_sales, customer_demographics, date_dim, item, promotion where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and ss_cdemo_sk = cd_demo_sk and ss_promo_sk = p_promo_sk and cd_gender = 'F' and cd_marital_status = 'W' and cd_education_status = 'College' and (p_channel_email = 'N' or p_channel_event = 'N') and d_year = 2001 group by i_item_id order by i_item_id limit 100""", "q8" -> """ select s_store_name ,sum(ss_net_profit) from store_sales ,date_dim ,store, (select ca_zip from ( SELECT substr(ca_zip,1,5) ca_zip FROM customer_address WHERE substr(ca_zip,1,5) IN ( '47602','16704','35863','28577','83910','36201', '58412','48162','28055','41419','80332', '38607','77817','24891','16226','18410', '21231','59345','13918','51089','20317', '17167','54585','67881','78366','47770', '18360','51717','73108','14440','21800', '89338','45859','65501','34948','25973', '73219','25333','17291','10374','18829', '60736','82620','41351','52094','19326', '25214','54207','40936','21814','79077', '25178','75742','77454','30621','89193', '27369','41232','48567','83041','71948', '37119','68341','14073','16891','62878', '49130','19833','24286','27700','40979', '50412','81504','94835','84844','71954', '39503','57649','18434','24987','12350', '86379','27413','44529','98569','16515', '27287','24255','21094','16005','56436', '91110','68293','56455','54558','10298', '83647','32754','27052','51766','19444', '13869','45645','94791','57631','20712', '37788','41807','46507','21727','71836', '81070','50632','88086','63991','20244', '31655','51782','29818','63792','68605', '94898','36430','57025','20601','82080', '33869','22728','35834','29086','92645', '98584','98072','11652','78093','57553', '43830','71144','53565','18700','90209', '71256','38353','54364','28571','96560', '57839','56355','50679','45266','84680', '34306','34972','48530','30106','15371', '92380','84247','92292','68852','13338', '34594','82602','70073','98069','85066', '47289','11686','98862','26217','47529', '63294','51793','35926','24227','14196', '24594','32489','99060','49472','43432', '49211','14312','88137','47369','56877', '20534','81755','15794','12318','21060', '73134','41255','63073','81003','73873', '66057','51184','51195','45676','92696', '70450','90669','98338','25264','38919', '59226','58581','60298','17895','19489', '52301','80846','95464','68770','51634', '19988','18367','18421','11618','67975', '25494','41352','95430','15734','62585', '97173','33773','10425','75675','53535', '17879','41967','12197','67998','79658', '59130','72592','14851','43933','68101', '50636','25717','71286','24660','58058', '72991','95042','15543','33122','69280', '11912','59386','27642','65177','17672', '33467','64592','36335','54010','18767', '63193','42361','49254','33113','33159', '36479','59080','11855','81963','31016', '49140','29392','41836','32958','53163', '13844','73146','23952','65148','93498', '14530','46131','58454','13376','13378', '83986','12320','17193','59852','46081', '98533','52389','13086','68843','31013', '13261','60560','13443','45533','83583', '11489','58218','19753','22911','25115', '86709','27156','32669','13123','51933', '39214','41331','66943','14155','69998', '49101','70070','35076','14242','73021', '59494','15782','29752','37914','74686', '83086','34473','15751','81084','49230', '91894','60624','17819','28810','63180', '56224','39459','55233','75752','43639', '55349','86057','62361','50788','31830', '58062','18218','85761','60083','45484', '21204','90229','70041','41162','35390', '16364','39500','68908','26689','52868', '81335','40146','11340','61527','61794', '71997','30415','59004','29450','58117', '69952','33562','83833','27385','61860', '96435','48333','23065','32961','84919', '61997','99132','22815','56600','68730', '48017','95694','32919','88217','27116', '28239','58032','18884','16791','21343', '97462','18569','75660','15475') intersect select ca_zip from (SELECT substr(ca_zip,1,5) ca_zip,count(*) cnt FROM customer_address, customer WHERE ca_address_sk = c_current_addr_sk and c_preferred_cust_flag='Y' group by ca_zip having count(*) > 10)A1)A2) V1 where ss_store_sk = s_store_sk and ss_sold_date_sk = d_date_sk and d_qoy = 2 and d_year = 1998 and (substr(s_zip,1,2) = substr(V1.ca_zip,1,2)) group by s_store_name order by s_store_name limit 100""", "q9" -> """ select case when (select count(*) from store_sales where ss_quantity between 1 and 20) > 578972190 then (select avg(ss_ext_list_price) from store_sales where ss_quantity between 1 and 20) else (select avg(ss_net_paid_inc_tax) from store_sales where ss_quantity between 1 and 20) end bucket1 , case when (select count(*) from store_sales where ss_quantity between 21 and 40) > 536856786 then (select avg(ss_ext_list_price) from store_sales where ss_quantity between 21 and 40) else (select avg(ss_net_paid_inc_tax) from store_sales where ss_quantity between 21 and 40) end bucket2, case when (select count(*) from store_sales where ss_quantity between 41 and 60) > 12733327 then (select avg(ss_ext_list_price) from store_sales where ss_quantity between 41 and 60) else (select avg(ss_net_paid_inc_tax) from store_sales where ss_quantity between 41 and 60) end bucket3, case when (select count(*) from store_sales where ss_quantity between 61 and 80) > 205136171 then (select avg(ss_ext_list_price) from store_sales where ss_quantity between 61 and 80) else (select avg(ss_net_paid_inc_tax) from store_sales where ss_quantity between 61 and 80) end bucket4, case when (select count(*) from store_sales where ss_quantity between 81 and 100) > 1192341092 then (select avg(ss_ext_list_price) from store_sales where ss_quantity between 81 and 100) else (select avg(ss_net_paid_inc_tax) from store_sales where ss_quantity between 81 and 100) end bucket5 from reason where r_reason_sk = 1""", "q10" -> """ select cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3, cd_dep_count, count(*) cnt4, cd_dep_employed_count, count(*) cnt5, cd_dep_college_count, count(*) cnt6 from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and ca_county in ('Baltimore city','Stafford County','Greene County','Ballard County','Franklin County') and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2000 and d_moy between 1 and 1+3) and (exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2000 and d_moy between 1 ANd 1+3) or exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2000 and d_moy between 1 and 1+3)) group by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count order by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count limit 100""", "q11" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(ss_ext_list_price-ss_ext_discount_amt) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(ws_ext_list_price-ws_ext_discount_amt) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year ) select t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_preferred_cust_flag from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.sale_type = 's' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_w_secyear.sale_type = 'w' and t_s_firstyear.dyear = 2001 and t_s_secyear.dyear = 2001+1 and t_w_firstyear.dyear = 2001 and t_w_secyear.dyear = 2001+1 and t_s_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else 0.0 end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else 0.0 end order by t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_preferred_cust_flag limit 100""", "q12" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(ws_ext_sales_price) as itemrevenue ,sum(ws_ext_sales_price)*100/sum(sum(ws_ext_sales_price)) over (partition by i_class) as revenueratio from web_sales ,item ,date_dim where ws_item_sk = i_item_sk and i_category in ('Children', 'Shoes', 'Women') and ws_sold_date_sk = d_date_sk and d_date between cast('1998-06-19' as date) and (cast('1998-06-19' as date) + INTERVAL 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio limit 100""", "q13" -> """ select avg(ss_quantity) ,avg(ss_ext_sales_price) ,avg(ss_ext_wholesale_cost) ,sum(ss_ext_wholesale_cost) from store_sales ,store ,customer_demographics ,household_demographics ,customer_address ,date_dim where s_store_sk = ss_store_sk and ss_sold_date_sk = d_date_sk and d_year = 2001 and((ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'D' and cd_education_status = '2 yr Degree' and ss_sales_price between 100.00 and 150.00 and hd_dep_count = 3 )or (ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'U' and cd_education_status = 'College' and ss_sales_price between 50.00 and 100.00 and hd_dep_count = 1 ) or (ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'S' and cd_education_status = 'Primary' and ss_sales_price between 150.00 and 200.00 and hd_dep_count = 1 )) and((ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('GA', 'IN', 'NY') and ss_net_profit between 100 and 200 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('ND', 'WV', 'TX') and ss_net_profit between 150 and 300 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('KS', 'NC', 'NM') and ss_net_profit between 50 and 250 ))""", "q14a" -> """ with cross_items as (select i_item_sk ss_item_sk from item, (select iss.i_brand_id brand_id ,iss.i_class_id class_id ,iss.i_category_id category_id from store_sales ,item iss ,date_dim d1 where ss_item_sk = iss.i_item_sk and ss_sold_date_sk = d1.d_date_sk and d1.d_year between 1998 AND 1998 + 2 intersect select ics.i_brand_id ,ics.i_class_id ,ics.i_category_id from catalog_sales ,item ics ,date_dim d2 where cs_item_sk = ics.i_item_sk and cs_sold_date_sk = d2.d_date_sk and d2.d_year between 1998 AND 1998 + 2 intersect select iws.i_brand_id ,iws.i_class_id ,iws.i_category_id from web_sales ,item iws ,date_dim d3 where ws_item_sk = iws.i_item_sk and ws_sold_date_sk = d3.d_date_sk and d3.d_year between 1998 AND 1998 + 2) where i_brand_id = brand_id and i_class_id = class_id and i_category_id = category_id ), avg_sales as (select avg(quantity*list_price) average_sales from (select ss_quantity quantity ,ss_list_price list_price from store_sales ,date_dim where ss_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2 union all select cs_quantity quantity ,cs_list_price list_price from catalog_sales ,date_dim where cs_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2 union all select ws_quantity quantity ,ws_list_price list_price from web_sales ,date_dim where ws_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2) x) select channel, i_brand_id,i_class_id,i_category_id,sum(sales), sum(number_sales) from( select 'store' channel, i_brand_id,i_class_id ,i_category_id,sum(ss_quantity*ss_list_price) sales , count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1998+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales) union all select 'catalog' channel, i_brand_id,i_class_id,i_category_id, sum(cs_quantity*cs_list_price) sales, count(*) number_sales from catalog_sales ,item ,date_dim where cs_item_sk in (select ss_item_sk from cross_items) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1998+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(cs_quantity*cs_list_price) > (select average_sales from avg_sales) union all select 'web' channel, i_brand_id,i_class_id,i_category_id, sum(ws_quantity*ws_list_price) sales , count(*) number_sales from web_sales ,item ,date_dim where ws_item_sk in (select ss_item_sk from cross_items) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1998+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(ws_quantity*ws_list_price) > (select average_sales from avg_sales) ) y group by rollup (channel, i_brand_id,i_class_id,i_category_id) order by channel,i_brand_id,i_class_id,i_category_id limit 100""", "q14b" -> """ with cross_items as (select i_item_sk ss_item_sk from item, (select iss.i_brand_id brand_id ,iss.i_class_id class_id ,iss.i_category_id category_id from store_sales ,item iss ,date_dim d1 where ss_item_sk = iss.i_item_sk and ss_sold_date_sk = d1.d_date_sk and d1.d_year between 1998 AND 1998 + 2 intersect select ics.i_brand_id ,ics.i_class_id ,ics.i_category_id from catalog_sales ,item ics ,date_dim d2 where cs_item_sk = ics.i_item_sk and cs_sold_date_sk = d2.d_date_sk and d2.d_year between 1998 AND 1998 + 2 intersect select iws.i_brand_id ,iws.i_class_id ,iws.i_category_id from web_sales ,item iws ,date_dim d3 where ws_item_sk = iws.i_item_sk and ws_sold_date_sk = d3.d_date_sk and d3.d_year between 1998 AND 1998 + 2) x where i_brand_id = brand_id and i_class_id = class_id and i_category_id = category_id ), avg_sales as (select avg(quantity*list_price) average_sales from (select ss_quantity quantity ,ss_list_price list_price from store_sales ,date_dim where ss_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2 union all select cs_quantity quantity ,cs_list_price list_price from catalog_sales ,date_dim where cs_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2 union all select ws_quantity quantity ,ws_list_price list_price from web_sales ,date_dim where ws_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2) x) select this_year.channel ty_channel ,this_year.i_brand_id ty_brand ,this_year.i_class_id ty_class ,this_year.i_category_id ty_category ,this_year.sales ty_sales ,this_year.number_sales ty_number_sales ,last_year.channel ly_channel ,last_year.i_brand_id ly_brand ,last_year.i_class_id ly_class ,last_year.i_category_id ly_category ,last_year.sales ly_sales ,last_year.number_sales ly_number_sales from (select 'store' channel, i_brand_id,i_class_id,i_category_id ,sum(ss_quantity*ss_list_price) sales, count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_week_seq = (select d_week_seq from date_dim where d_year = 1998 + 1 and d_moy = 12 and d_dom = 17) group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) this_year, (select 'store' channel, i_brand_id,i_class_id ,i_category_id, sum(ss_quantity*ss_list_price) sales, count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_week_seq = (select d_week_seq from date_dim where d_year = 1998 and d_moy = 12 and d_dom = 17) group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) last_year where this_year.i_brand_id= last_year.i_brand_id and this_year.i_class_id = last_year.i_class_id and this_year.i_category_id = last_year.i_category_id order by this_year.channel, this_year.i_brand_id, this_year.i_class_id, this_year.i_category_id limit 100""", "q15" -> """ select ca_zip ,sum(cs_sales_price) from catalog_sales ,customer ,customer_address ,date_dim where cs_bill_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') or ca_state in ('CA','WA','GA') or cs_sales_price > 500) and cs_sold_date_sk = d_date_sk and d_qoy = 1 and d_year = 2002 group by ca_zip order by ca_zip limit 100""", "q16" -> """ select count(distinct cs_order_number) as `order count` ,sum(cs_ext_ship_cost) as `total shipping cost` ,sum(cs_net_profit) as `total net profit` from catalog_sales cs1 ,date_dim ,customer_address ,call_center where d_date between '2001-3-01' and (cast('2001-3-01' as date) + INTERVAL 60 days) and cs1.cs_ship_date_sk = d_date_sk and cs1.cs_ship_addr_sk = ca_address_sk and ca_state = 'PA' and cs1.cs_call_center_sk = cc_call_center_sk and cc_county in ('Luce County','Franklin Parish','Sierra County','Williamson County', 'Kittitas County' ) and exists (select * from catalog_sales cs2 where cs1.cs_order_number = cs2.cs_order_number and cs1.cs_warehouse_sk <> cs2.cs_warehouse_sk) and not exists(select * from catalog_returns cr1 where cs1.cs_order_number = cr1.cr_order_number) order by count(distinct cs_order_number) limit 100""", "q17" -> """ select i_item_id ,i_item_desc ,s_state ,count(ss_quantity) as store_sales_quantitycount ,avg(ss_quantity) as store_sales_quantityave ,stddev_samp(ss_quantity) as store_sales_quantitystdev ,stddev_samp(ss_quantity)/avg(ss_quantity) as store_sales_quantitycov ,count(sr_return_quantity) as store_returns_quantitycount ,avg(sr_return_quantity) as store_returns_quantityave ,stddev_samp(sr_return_quantity) as store_returns_quantitystdev ,stddev_samp(sr_return_quantity)/avg(sr_return_quantity) as store_returns_quantitycov ,count(cs_quantity) as catalog_sales_quantitycount ,avg(cs_quantity) as catalog_sales_quantityave ,stddev_samp(cs_quantity) as catalog_sales_quantitystdev ,stddev_samp(cs_quantity)/avg(cs_quantity) as catalog_sales_quantitycov from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_quarter_name = '2001Q1' and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_quarter_name in ('2001Q1','2001Q2','2001Q3') and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_quarter_name in ('2001Q1','2001Q2','2001Q3') group by i_item_id ,i_item_desc ,s_state order by i_item_id ,i_item_desc ,s_state limit 100""", "q18" -> """ select i_item_id, ca_country, ca_state, ca_county, avg( cast(cs_quantity as decimal(12,2))) agg1, avg( cast(cs_list_price as decimal(12,2))) agg2, avg( cast(cs_coupon_amt as decimal(12,2))) agg3, avg( cast(cs_sales_price as decimal(12,2))) agg4, avg( cast(cs_net_profit as decimal(12,2))) agg5, avg( cast(c_birth_year as decimal(12,2))) agg6, avg( cast(cd1.cd_dep_count as decimal(12,2))) agg7 from catalog_sales, customer_demographics cd1, customer_demographics cd2, customer, customer_address, date_dim, item where cs_sold_date_sk = d_date_sk and cs_item_sk = i_item_sk and cs_bill_cdemo_sk = cd1.cd_demo_sk and cs_bill_customer_sk = c_customer_sk and cd1.cd_gender = 'M' and cd1.cd_education_status = 'Unknown' and c_current_cdemo_sk = cd2.cd_demo_sk and c_current_addr_sk = ca_address_sk and c_birth_month in (5,7,8,6,12,4) and d_year = 2000 and ca_state in ('MO','NY','ME' ,'MI','IA','OH','MS') group by rollup (i_item_id, ca_country, ca_state, ca_county) order by ca_country, ca_state, ca_county, i_item_id limit 100""", "q19" -> """ select i_brand_id brand_id, i_brand brand, i_manufact_id, i_manufact, sum(ss_ext_sales_price) ext_price from date_dim, store_sales, item,customer,customer_address,store where d_date_sk = ss_sold_date_sk and ss_item_sk = i_item_sk and i_manager_id=55 and d_moy=11 and d_year=1998 and ss_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and substr(ca_zip,1,5) <> substr(s_zip,1,5) and ss_store_sk = s_store_sk group by i_brand ,i_brand_id ,i_manufact_id ,i_manufact order by ext_price desc ,i_brand ,i_brand_id ,i_manufact_id ,i_manufact limit 100 """, "q20" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(cs_ext_sales_price) as itemrevenue ,sum(cs_ext_sales_price)*100/sum(sum(cs_ext_sales_price)) over (partition by i_class) as revenueratio from catalog_sales ,item ,date_dim where cs_item_sk = i_item_sk and i_category in ('Shoes', 'Electronics', 'Home') and cs_sold_date_sk = d_date_sk and d_date between cast('2000-05-15' as date) and (cast('2000-05-15' as date) + INTERVAL 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio limit 100""", "q21" -> """ select * from(select w_warehouse_name ,i_item_id ,sum(case when (cast(d_date as date) < cast ('2002-02-15' as date)) then inv_quantity_on_hand else 0 end) as inv_before ,sum(case when (cast(d_date as date) >= cast ('2002-02-15' as date)) then inv_quantity_on_hand else 0 end) as inv_after from inventory ,warehouse ,item ,date_dim where i_current_price between 0.99 and 1.49 and i_item_sk = inv_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_date between (cast ('2002-02-15' as date) - INTERVAL 30 days) and (cast ('2002-02-15' as date) + INTERVAL 30 days) group by w_warehouse_name, i_item_id) x where (case when inv_before > 0 then inv_after / inv_before else null end) between 2.0/3.0 and 3.0/2.0 order by w_warehouse_name ,i_item_id limit 100""", "q22" -> """ select i_product_name ,i_brand ,i_class ,i_category ,avg(inv_quantity_on_hand) qoh from inventory ,date_dim ,item where inv_date_sk=d_date_sk and inv_item_sk=i_item_sk and d_month_seq between 1202 and 1202 + 11 group by rollup(i_product_name ,i_brand ,i_class ,i_category) order by qoh, i_product_name, i_brand, i_class, i_category limit 100""", "q23a" -> """ with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt from store_sales ,date_dim ,item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and d_year in (2000,2000+1,2000+2,2000+3) group by substr(i_item_desc,1,30),i_item_sk,d_date having count(*) >4), max_store_sales as (select max(csales) tpcds_cmax from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales from store_sales ,customer ,date_dim where ss_customer_sk = c_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (2000,2000+1,2000+2,2000+3) group by c_customer_sk)), best_ss_customer as (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales from store_sales ,customer where ss_customer_sk = c_customer_sk group by c_customer_sk having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select * from max_store_sales)) select sum(sales) from (select cs_quantity*cs_list_price sales from catalog_sales ,date_dim where d_year = 2000 and d_moy = 4 and cs_sold_date_sk = d_date_sk and cs_item_sk in (select item_sk from frequent_ss_items) and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) union all select ws_quantity*ws_list_price sales from web_sales ,date_dim where d_year = 2000 and d_moy = 4 and ws_sold_date_sk = d_date_sk and ws_item_sk in (select item_sk from frequent_ss_items) and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer)) limit 100""", "q23b" -> """ with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt from store_sales ,date_dim ,item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and d_year in (2000,2000 + 1,2000 + 2,2000 + 3) group by substr(i_item_desc,1,30),i_item_sk,d_date having count(*) >4), max_store_sales as (select max(csales) tpcds_cmax from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales from store_sales ,customer ,date_dim where ss_customer_sk = c_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (2000,2000+1,2000+2,2000+3) group by c_customer_sk)), best_ss_customer as (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales from store_sales ,customer where ss_customer_sk = c_customer_sk group by c_customer_sk having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select * from max_store_sales)) select c_last_name,c_first_name,sales from (select c_last_name,c_first_name,sum(cs_quantity*cs_list_price) sales from catalog_sales ,customer ,date_dim where d_year = 2000 and d_moy = 4 and cs_sold_date_sk = d_date_sk and cs_item_sk in (select item_sk from frequent_ss_items) and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) and cs_bill_customer_sk = c_customer_sk group by c_last_name,c_first_name union all select c_last_name,c_first_name,sum(ws_quantity*ws_list_price) sales from web_sales ,customer ,date_dim where d_year = 2000 and d_moy = 4 and ws_sold_date_sk = d_date_sk and ws_item_sk in (select item_sk from frequent_ss_items) and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer) and ws_bill_customer_sk = c_customer_sk group by c_last_name,c_first_name) order by c_last_name,c_first_name,sales limit 100""", "q24a" -> """ with ssales as (select c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size ,sum(ss_net_paid_inc_tax) netpaid from store_sales ,store_returns ,store ,item ,customer ,customer_address where ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_customer_sk = c_customer_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and c_current_addr_sk = ca_address_sk and c_birth_country <> upper(ca_country) and s_zip = ca_zip and s_market_id=5 group by c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size) select c_last_name ,c_first_name ,s_store_name ,sum(netpaid) paid from ssales where i_color = 'cyan' group by c_last_name ,c_first_name ,s_store_name having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) order by c_last_name ,c_first_name ,s_store_name""", "q24b" -> """ with ssales as (select c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size ,sum(ss_net_paid_inc_tax) netpaid from store_sales ,store_returns ,store ,item ,customer ,customer_address where ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_customer_sk = c_customer_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and c_current_addr_sk = ca_address_sk and c_birth_country <> upper(ca_country) and s_zip = ca_zip and s_market_id = 5 group by c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size) select c_last_name ,c_first_name ,s_store_name ,sum(netpaid) paid from ssales where i_color = 'ivory' group by c_last_name ,c_first_name ,s_store_name having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) order by c_last_name ,c_first_name ,s_store_name""", "q25" -> """ select i_item_id ,i_item_desc ,s_store_id ,s_store_name ,stddev_samp(ss_net_profit) as store_sales_profit ,stddev_samp(sr_net_loss) as store_returns_loss ,stddev_samp(cs_net_profit) as catalog_sales_profit from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_moy = 4 and d1.d_year = 2000 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_moy between 4 and 10 and d2.d_year = 2000 and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_moy between 4 and 10 and d3.d_year = 2000 group by i_item_id ,i_item_desc ,s_store_id ,s_store_name order by i_item_id ,i_item_desc ,s_store_id ,s_store_name limit 100""", "q26" -> """ select i_item_id, avg(cs_quantity) agg1, avg(cs_list_price) agg2, avg(cs_coupon_amt) agg3, avg(cs_sales_price) agg4 from catalog_sales, customer_demographics, date_dim, item, promotion where cs_sold_date_sk = d_date_sk and cs_item_sk = i_item_sk and cs_bill_cdemo_sk = cd_demo_sk and cs_promo_sk = p_promo_sk and cd_gender = 'F' and cd_marital_status = 'M' and cd_education_status = 'Unknown' and (p_channel_email = 'N' or p_channel_event = 'N') and d_year = 2001 group by i_item_id order by i_item_id limit 100""", "q27" -> """ select i_item_id, s_state, grouping(s_state) g_state, avg(ss_quantity) agg1, avg(ss_list_price) agg2, avg(ss_coupon_amt) agg3, avg(ss_sales_price) agg4 from store_sales, customer_demographics, date_dim, store, item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and ss_cdemo_sk = cd_demo_sk and cd_gender = 'F' and cd_marital_status = 'D' and cd_education_status = '2 yr Degree' and d_year = 1999 and s_state in ('MI','WV', 'MI', 'NY', 'TN', 'MI') group by rollup (i_item_id, s_state) order by i_item_id ,s_state limit 100""", "q28" -> """ select * from (select avg(ss_list_price) B1_LP ,count(ss_list_price) B1_CNT ,count(distinct ss_list_price) B1_CNTD from store_sales where ss_quantity between 0 and 5 and (ss_list_price between 151 and 151+10 or ss_coupon_amt between 4349 and 4349+1000 or ss_wholesale_cost between 75 and 75+20)) B1, (select avg(ss_list_price) B2_LP ,count(ss_list_price) B2_CNT ,count(distinct ss_list_price) B2_CNTD from store_sales where ss_quantity between 6 and 10 and (ss_list_price between 45 and 45+10 or ss_coupon_amt between 12490 and 12490+1000 or ss_wholesale_cost between 37 and 37+20)) B2, (select avg(ss_list_price) B3_LP ,count(ss_list_price) B3_CNT ,count(distinct ss_list_price) B3_CNTD from store_sales where ss_quantity between 11 and 15 and (ss_list_price between 54 and 54+10 or ss_coupon_amt between 13038 and 13038+1000 or ss_wholesale_cost between 17 and 17+20)) B3, (select avg(ss_list_price) B4_LP ,count(ss_list_price) B4_CNT ,count(distinct ss_list_price) B4_CNTD from store_sales where ss_quantity between 16 and 20 and (ss_list_price between 178 and 178+10 or ss_coupon_amt between 10744 and 10744+1000 or ss_wholesale_cost between 51 and 51+20)) B4, (select avg(ss_list_price) B5_LP ,count(ss_list_price) B5_CNT ,count(distinct ss_list_price) B5_CNTD from store_sales where ss_quantity between 21 and 25 and (ss_list_price between 49 and 49+10 or ss_coupon_amt between 8494 and 8494+1000 or ss_wholesale_cost between 56 and 56+20)) B5, (select avg(ss_list_price) B6_LP ,count(ss_list_price) B6_CNT ,count(distinct ss_list_price) B6_CNTD from store_sales where ss_quantity between 26 and 30 and (ss_list_price between 0 and 0+10 or ss_coupon_amt between 17854 and 17854+1000 or ss_wholesale_cost between 31 and 31+20)) B6 limit 100""", "q29" -> """ select i_item_id ,i_item_desc ,s_store_id ,s_store_name ,max(ss_quantity) as store_sales_quantity ,max(sr_return_quantity) as store_returns_quantity ,max(cs_quantity) as catalog_sales_quantity from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_moy = 4 and d1.d_year = 1999 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_moy between 4 and 4 + 3 and d2.d_year = 1999 and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_year in (1999,1999+1,1999+2) group by i_item_id ,i_item_desc ,s_store_id ,s_store_name order by i_item_id ,i_item_desc ,s_store_id ,s_store_name limit 100""", "q30" -> """ with customer_total_return as (select wr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, sum(wr_return_amt) as ctr_total_return from web_returns ,date_dim ,customer_address where wr_returned_date_sk = d_date_sk and d_year =2000 and wr_returning_addr_sk = ca_address_sk group by wr_returning_customer_sk ,ca_state) select c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address ,c_last_review_date,ctr_total_return from customer_total_return ctr1 ,customer_address ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_state = ctr2.ctr_state) and ca_address_sk = c_current_addr_sk and ca_state = 'MD' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address ,c_last_review_date,ctr_total_return limit 100""", "q31" -> """ with ss as (select ca_county,d_qoy, d_year,sum(ss_ext_sales_price) as store_sales from store_sales,date_dim,customer_address where ss_sold_date_sk = d_date_sk and ss_addr_sk=ca_address_sk group by ca_county,d_qoy, d_year), ws as (select ca_county,d_qoy, d_year,sum(ws_ext_sales_price) as web_sales from web_sales,date_dim,customer_address where ws_sold_date_sk = d_date_sk and ws_bill_addr_sk=ca_address_sk group by ca_county,d_qoy, d_year) select ss1.ca_county ,ss1.d_year ,ws2.web_sales/ws1.web_sales web_q1_q2_increase ,ss2.store_sales/ss1.store_sales store_q1_q2_increase ,ws3.web_sales/ws2.web_sales web_q2_q3_increase ,ss3.store_sales/ss2.store_sales store_q2_q3_increase from ss ss1 ,ss ss2 ,ss ss3 ,ws ws1 ,ws ws2 ,ws ws3 where ss1.d_qoy = 1 and ss1.d_year = 1999 and ss1.ca_county = ss2.ca_county and ss2.d_qoy = 2 and ss2.d_year = 1999 and ss2.ca_county = ss3.ca_county and ss3.d_qoy = 3 and ss3.d_year = 1999 and ss1.ca_county = ws1.ca_county and ws1.d_qoy = 1 and ws1.d_year = 1999 and ws1.ca_county = ws2.ca_county and ws2.d_qoy = 2 and ws2.d_year = 1999 and ws1.ca_county = ws3.ca_county and ws3.d_qoy = 3 and ws3.d_year =1999 and case when ws1.web_sales > 0 then ws2.web_sales/ws1.web_sales else null end > case when ss1.store_sales > 0 then ss2.store_sales/ss1.store_sales else null end and case when ws2.web_sales > 0 then ws3.web_sales/ws2.web_sales else null end > case when ss2.store_sales > 0 then ss3.store_sales/ss2.store_sales else null end order by store_q2_q3_increase""", "q32" -> """ select sum(cs_ext_discount_amt) as `excess discount amount` from catalog_sales ,item ,date_dim where i_manufact_id = 7 and i_item_sk = cs_item_sk and d_date between '2000-01-21' and (cast('2000-01-21' as date) + INTERVAL 90 days) and d_date_sk = cs_sold_date_sk and cs_ext_discount_amt > ( select 1.3 * avg(cs_ext_discount_amt) from catalog_sales ,date_dim where cs_item_sk = i_item_sk and d_date between '2000-01-21' and (cast('2000-01-21' as date) + INTERVAL 90 days) and d_date_sk = cs_sold_date_sk ) limit 100""", "q33" -> """ with ss as ( select i_manufact_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Books')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 6 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -7 group by i_manufact_id), cs as ( select i_manufact_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Books')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 6 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -7 group by i_manufact_id), ws as ( select i_manufact_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Books')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 6 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -7 group by i_manufact_id) select i_manufact_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_manufact_id order by total_sales limit 100""", "q34" -> """ select c_last_name ,c_first_name ,c_salutation ,c_preferred_cust_flag ,ss_ticket_number ,cnt from (select ss_ticket_number ,ss_customer_sk ,count(*) cnt from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and (date_dim.d_dom between 1 and 3 or date_dim.d_dom between 25 and 28) and (household_demographics.hd_buy_potential = '501-1000' or household_demographics.hd_buy_potential = '5001-10000') and household_demographics.hd_vehicle_count > 0 and (case when household_demographics.hd_vehicle_count > 0 then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end) > 1.2 and date_dim.d_year in (1999,1999+1,1999+2) and store.s_county in ('Levy County','Val Verde County','Porter County','Nowata County', 'Lincoln County','Brazos County','Franklin Parish','Pipestone County') group by ss_ticket_number,ss_customer_sk) dn,customer where ss_customer_sk = c_customer_sk and cnt between 15 and 20 order by c_last_name,c_first_name,c_salutation,c_preferred_cust_flag desc, ss_ticket_number""", "q35" -> """ select ca_state, cd_gender, cd_marital_status, cd_dep_count, count(*) cnt1, sum(cd_dep_count), sum(cd_dep_count), sum(cd_dep_count), cd_dep_employed_count, count(*) cnt2, sum(cd_dep_employed_count), sum(cd_dep_employed_count), sum(cd_dep_employed_count), cd_dep_college_count, count(*) cnt3, sum(cd_dep_college_count), sum(cd_dep_college_count), sum(cd_dep_college_count) from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2001 and d_qoy < 4) and (exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2001 and d_qoy < 4) or exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2001 and d_qoy < 4)) group by ca_state, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count order by ca_state, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count limit 100""", "q36" -> """ select sum(ss_net_profit)/sum(ss_ext_sales_price) as gross_margin ,i_category ,i_class ,grouping(i_category)+grouping(i_class) as lochierarchy ,rank() over ( partition by grouping(i_category)+grouping(i_class), case when grouping(i_class) = 0 then i_category end order by sum(ss_net_profit)/sum(ss_ext_sales_price) asc) as rank_within_parent from store_sales ,date_dim d1 ,item ,store where d1.d_year = 1999 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and s_state in ('MO','AL','OH','WV', 'AL','MN','TN','WA') group by rollup(i_category,i_class) order by lochierarchy desc ,case when lochierarchy = 0 then i_category end ,rank_within_parent limit 100""", "q37" -> """ select i_item_id ,i_item_desc ,i_current_price from item, inventory, date_dim, catalog_sales where i_current_price between 57 and 57 + 30 and inv_item_sk = i_item_sk and d_date_sk=inv_date_sk and d_date between cast('2001-04-19' as date) and (cast('2001-04-19' as date) + interval 60 days) and i_manufact_id in (804,916,707,680) and inv_quantity_on_hand between 100 and 500 and cs_item_sk = i_item_sk group by i_item_id,i_item_desc,i_current_price order by i_item_id limit 100""", "q38" -> """ select count(*) from ( select distinct c_last_name, c_first_name, d_date from store_sales, date_dim, customer where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_customer_sk = customer.c_customer_sk and d_month_seq between 1189 and 1189 + 11 intersect select distinct c_last_name, c_first_name, d_date from catalog_sales, date_dim, customer where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1189 and 1189 + 11 intersect select distinct c_last_name, c_first_name, d_date from web_sales, date_dim, customer where web_sales.ws_sold_date_sk = date_dim.d_date_sk and web_sales.ws_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1189 and 1189 + 11 ) hot_cust limit 100""", "q39a" -> """ with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stdev,mean, case mean when 0 then null else stdev/mean end cov from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean from inventory ,item ,warehouse ,date_dim where inv_item_sk = i_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_year =2000 group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo where case mean when 0 then 0 else stdev/mean end > 1) select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov from inv inv1,inv inv2 where inv1.i_item_sk = inv2.i_item_sk and inv1.w_warehouse_sk = inv2.w_warehouse_sk and inv1.d_moy=3 and inv2.d_moy=3+1 order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov ,inv2.d_moy,inv2.mean, inv2.cov""", "q39b" -> """ with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stdev,mean, case mean when 0 then null else stdev/mean end cov from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean from inventory ,item ,warehouse ,date_dim where inv_item_sk = i_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_year =2000 group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo where case mean when 0 then 0 else stdev/mean end > 1) select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov from inv inv1,inv inv2 where inv1.i_item_sk = inv2.i_item_sk and inv1.w_warehouse_sk = inv2.w_warehouse_sk and inv1.d_moy=3 and inv2.d_moy=3+1 and inv1.cov > 1.5 order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov ,inv2.d_moy,inv2.mean, inv2.cov""", "q40" -> """ select w_state ,i_item_id ,sum(case when (cast(d_date as date) < cast ('2000-04-09' as date)) then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_before ,sum(case when (cast(d_date as date) >= cast ('2000-04-09' as date)) then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_after from catalog_sales left outer join catalog_returns on (cs_order_number = cr_order_number and cs_item_sk = cr_item_sk) ,warehouse ,item ,date_dim where i_current_price between 0.99 and 1.49 and i_item_sk = cs_item_sk and cs_warehouse_sk = w_warehouse_sk and cs_sold_date_sk = d_date_sk and d_date between (cast ('2000-04-09' as date) - INTERVAL 30 days) and (cast ('2000-04-09' as date) + INTERVAL 30 days) group by w_state,i_item_id order by w_state,i_item_id limit 100""", "q41" -> """ select distinct(i_product_name) from item i1 where i_manufact_id between 917 and 917+40 and (select count(*) as item_cnt from item where (i_manufact = i1.i_manufact and ((i_category = 'Women' and (i_color = 'antique' or i_color = 'pale') and (i_units = 'Tbl' or i_units = 'Case') and (i_size = 'small' or i_size = 'extra large') ) or (i_category = 'Women' and (i_color = 'snow' or i_color = 'lemon') and (i_units = 'Box' or i_units = 'Ounce') and (i_size = 'economy' or i_size = 'N/A') ) or (i_category = 'Men' and (i_color = 'green' or i_color = 'blue') and (i_units = 'Gross' or i_units = 'Ton') and (i_size = 'large' or i_size = 'petite') ) or (i_category = 'Men' and (i_color = 'cream' or i_color = 'frosted') and (i_units = 'Bundle' or i_units = 'Gram') and (i_size = 'small' or i_size = 'extra large') ))) or (i_manufact = i1.i_manufact and ((i_category = 'Women' and (i_color = 'orange' or i_color = 'spring') and (i_units = 'Lb' or i_units = 'Carton') and (i_size = 'small' or i_size = 'extra large') ) or (i_category = 'Women' and (i_color = 'lawn' or i_color = 'violet') and (i_units = 'Oz' or i_units = 'Cup') and (i_size = 'economy' or i_size = 'N/A') ) or (i_category = 'Men' and (i_color = 'navy' or i_color = 'linen') and (i_units = 'Pound' or i_units = 'Unknown') and (i_size = 'large' or i_size = 'petite') ) or (i_category = 'Men' and (i_color = 'almond' or i_color = 'olive') and (i_units = 'Pallet' or i_units = 'Bunch') and (i_size = 'small' or i_size = 'extra large') )))) > 0 order by i_product_name limit 100""", "q42" -> """ select dt.d_year ,item.i_category_id ,item.i_category ,sum(ss_ext_sales_price) from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manager_id = 1 and dt.d_moy=11 and dt.d_year=1998 group by dt.d_year ,item.i_category_id ,item.i_category order by sum(ss_ext_sales_price) desc,dt.d_year ,item.i_category_id ,item.i_category limit 100 """, "q43" -> """ select s_store_name, s_store_id, sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales from date_dim, store_sales, store where d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk and s_gmt_offset = -6 and d_year = 2000 group by s_store_name, s_store_id order by s_store_name, s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales limit 100""", "q44" -> """ select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing from(select * from (select item_sk,rank() over (order by rank_col asc) rnk from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col from store_sales ss1 where ss_store_sk = 731 group by ss_item_sk having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col from store_sales where ss_store_sk = 731 and ss_promo_sk is null group by ss_store_sk))V1)V11 where rnk < 11) asceding, (select * from (select item_sk,rank() over (order by rank_col desc) rnk from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col from store_sales ss1 where ss_store_sk = 731 group by ss_item_sk having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col from store_sales where ss_store_sk = 731 and ss_promo_sk is null group by ss_store_sk))V2)V21 where rnk < 11) descending, item i1, item i2 where asceding.rnk = descending.rnk and i1.i_item_sk=asceding.item_sk and i2.i_item_sk=descending.item_sk order by asceding.rnk limit 100""", "q45" -> """ select ca_zip, ca_city, sum(ws_sales_price) from web_sales, customer, customer_address, date_dim, item where ws_bill_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and ws_item_sk = i_item_sk and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') or i_item_id in (select i_item_id from item where i_item_sk in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) ) ) and ws_sold_date_sk = d_date_sk and d_qoy = 1 and d_year = 2000 group by ca_zip, ca_city order by ca_zip, ca_city limit 100""", "q46" -> """ select c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number ,amt,profit from (select ss_ticket_number ,ss_customer_sk ,ca_city bought_city ,sum(ss_coupon_amt) amt ,sum(ss_net_profit) profit from store_sales,date_dim,store,household_demographics,customer_address where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and store_sales.ss_addr_sk = customer_address.ca_address_sk and (household_demographics.hd_dep_count = 1 or household_demographics.hd_vehicle_count= 2) and date_dim.d_dow in (6,0) and date_dim.d_year in (2000,2000+1,2000+2) and store.s_city in ('Buena Vista','Friendship','Monroe','Oak Hill','Randolph') group by ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city) dn,customer,customer_address current_addr where ss_customer_sk = c_customer_sk and customer.c_current_addr_sk = current_addr.ca_address_sk and current_addr.ca_city <> bought_city order by c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number limit 100""", "q47" -> """ with v1 as( select i_category, i_brand, s_store_name, s_company_name, d_year, d_moy, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_category, i_brand, s_store_name, s_company_name, d_year) avg_monthly_sales, rank() over (partition by i_category, i_brand, s_store_name, s_company_name order by d_year, d_moy) rn from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ( d_year = 1999 or ( d_year = 1999-1 and d_moy =12) or ( d_year = 1999+1 and d_moy =1) ) group by i_category, i_brand, s_store_name, s_company_name, d_year, d_moy), v2 as( select v1.i_category ,v1.d_year, v1.d_moy ,v1.avg_monthly_sales ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum from v1, v1 v1_lag, v1 v1_lead where v1.i_category = v1_lag.i_category and v1.i_category = v1_lead.i_category and v1.i_brand = v1_lag.i_brand and v1.i_brand = v1_lead.i_brand and v1.s_store_name = v1_lag.s_store_name and v1.s_store_name = v1_lead.s_store_name and v1.s_company_name = v1_lag.s_company_name and v1.s_company_name = v1_lead.s_company_name and v1.rn = v1_lag.rn + 1 and v1.rn = v1_lead.rn - 1) select * from v2 where d_year = 1999 and avg_monthly_sales > 0 and case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by sum_sales - avg_monthly_sales, sum_sales limit 100""", "q48" -> """ select sum (ss_quantity) from store_sales, store, customer_demographics, customer_address, date_dim where s_store_sk = ss_store_sk and ss_sold_date_sk = d_date_sk and d_year = 1999 and ( ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'S' and cd_education_status = 'Primary' and ss_sales_price between 100.00 and 150.00 ) or ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'D' and cd_education_status = 'College' and ss_sales_price between 50.00 and 100.00 ) or ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'U' and cd_education_status = '2 yr Degree' and ss_sales_price between 150.00 and 200.00 ) ) and ( ( ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('ND', 'NC', 'TX') and ss_net_profit between 0 and 2000 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('VA', 'IA', 'AR') and ss_net_profit between 150 and 3000 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('MA', 'FL', 'TN') and ss_net_profit between 50 and 25000 ) )""", "q49" -> """ select channel, item, return_ratio, return_rank, currency_rank from (select 'web' as channel ,web.item ,web.return_ratio ,web.return_rank ,web.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select ws.ws_item_sk as item ,(cast(sum(coalesce(wr.wr_return_quantity,0)) as decimal(15,4))/ cast(sum(coalesce(ws.ws_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(wr.wr_return_amt,0)) as decimal(15,4))/ cast(sum(coalesce(ws.ws_net_paid,0)) as decimal(15,4) )) as currency_ratio from web_sales ws left outer join web_returns wr on (ws.ws_order_number = wr.wr_order_number and ws.ws_item_sk = wr.wr_item_sk) ,date_dim where wr.wr_return_amt > 10000 and ws.ws_net_profit > 1 and ws.ws_net_paid > 0 and ws.ws_quantity > 0 and ws_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 11 group by ws.ws_item_sk ) in_web ) web where ( web.return_rank <= 10 or web.currency_rank <= 10 ) union select 'catalog' as channel ,catalog.item ,catalog.return_ratio ,catalog.return_rank ,catalog.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select cs.cs_item_sk as item ,(cast(sum(coalesce(cr.cr_return_quantity,0)) as decimal(15,4))/ cast(sum(coalesce(cs.cs_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(cr.cr_return_amount,0)) as decimal(15,4))/ cast(sum(coalesce(cs.cs_net_paid,0)) as decimal(15,4) )) as currency_ratio from catalog_sales cs left outer join catalog_returns cr on (cs.cs_order_number = cr.cr_order_number and cs.cs_item_sk = cr.cr_item_sk) ,date_dim where cr.cr_return_amount > 10000 and cs.cs_net_profit > 1 and cs.cs_net_paid > 0 and cs.cs_quantity > 0 and cs_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 11 group by cs.cs_item_sk ) in_cat ) catalog where ( catalog.return_rank <= 10 or catalog.currency_rank <=10 ) union select 'store' as channel ,store.item ,store.return_ratio ,store.return_rank ,store.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select sts.ss_item_sk as item ,(cast(sum(coalesce(sr.sr_return_quantity,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(sr.sr_return_amt,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_net_paid,0)) as decimal(15,4) )) as currency_ratio from store_sales sts left outer join store_returns sr on (sts.ss_ticket_number = sr.sr_ticket_number and sts.ss_item_sk = sr.sr_item_sk) ,date_dim where sr.sr_return_amt > 10000 and sts.ss_net_profit > 1 and sts.ss_net_paid > 0 and sts.ss_quantity > 0 and ss_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 11 group by sts.ss_item_sk ) in_store ) store where ( store.return_rank <= 10 or store.currency_rank <= 10 ) ) order by 1,4,5,2 limit 100""", "q50" -> """ select s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip ,sum(case when (sr_returned_date_sk - ss_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 30) and (sr_returned_date_sk - ss_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 60) and (sr_returned_date_sk - ss_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 90) and (sr_returned_date_sk - ss_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from store_sales ,store_returns ,store ,date_dim d1 ,date_dim d2 where d2.d_year = 2002 and d2.d_moy = 10 and ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_sold_date_sk = d1.d_date_sk and sr_returned_date_sk = d2.d_date_sk and ss_customer_sk = sr_customer_sk and ss_store_sk = s_store_sk group by s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip order by s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip limit 100""", "q51" -> """ WITH web_v1 as ( select ws_item_sk item_sk, d_date, sum(sum(ws_sales_price)) over (partition by ws_item_sk order by d_date rows between unbounded preceding and current row) cume_sales from web_sales ,date_dim where ws_sold_date_sk=d_date_sk and d_month_seq between 1213 and 1213+11 and ws_item_sk is not NULL group by ws_item_sk, d_date), store_v1 as ( select ss_item_sk item_sk, d_date, sum(sum(ss_sales_price)) over (partition by ss_item_sk order by d_date rows between unbounded preceding and current row) cume_sales from store_sales ,date_dim where ss_sold_date_sk=d_date_sk and d_month_seq between 1213 and 1213+11 and ss_item_sk is not NULL group by ss_item_sk, d_date) select * from (select item_sk ,d_date ,web_sales ,store_sales ,max(web_sales) over (partition by item_sk order by d_date rows between unbounded preceding and current row) web_cumulative ,max(store_sales) over (partition by item_sk order by d_date rows between unbounded preceding and current row) store_cumulative from (select case when web.item_sk is not null then web.item_sk else store.item_sk end item_sk ,case when web.d_date is not null then web.d_date else store.d_date end d_date ,web.cume_sales web_sales ,store.cume_sales store_sales from web_v1 web full outer join store_v1 store on (web.item_sk = store.item_sk and web.d_date = store.d_date) )x )y where web_cumulative > store_cumulative order by item_sk ,d_date limit 100""", "q52" -> """ select dt.d_year ,item.i_brand_id brand_id ,item.i_brand brand ,sum(ss_ext_sales_price) ext_price from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manager_id = 1 and dt.d_moy=11 and dt.d_year=1998 group by dt.d_year ,item.i_brand ,item.i_brand_id order by dt.d_year ,ext_price desc ,brand_id limit 100 """, "q53" -> """ select * from (select i_manufact_id, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_manufact_id) avg_quarterly_sales from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_month_seq in (1219,1219+1,1219+2,1219+3,1219+4,1219+5,1219+6,1219+7,1219+8,1219+9,1219+10,1219+11) and ((i_category in ('Books','Children','Electronics') and i_class in ('personal','portable','reference','self-help') and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', 'exportiunivamalg #9','scholaramalgamalg #9')) or(i_category in ('Women','Music','Men') and i_class in ('accessories','classical','fragrances','pants') and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 'importoamalg #1'))) group by i_manufact_id, d_qoy ) tmp1 where case when avg_quarterly_sales > 0 then abs (sum_sales - avg_quarterly_sales)/ avg_quarterly_sales else null end > 0.1 order by avg_quarterly_sales, sum_sales, i_manufact_id limit 100""", "q54" -> """ with my_customers as ( select distinct c_customer_sk , c_current_addr_sk from ( select cs_sold_date_sk sold_date_sk, cs_bill_customer_sk customer_sk, cs_item_sk item_sk from catalog_sales union all select ws_sold_date_sk sold_date_sk, ws_bill_customer_sk customer_sk, ws_item_sk item_sk from web_sales ) cs_or_ws_sales, item, date_dim, customer where sold_date_sk = d_date_sk and item_sk = i_item_sk and i_category = 'Men' and i_class = 'shirts' and c_customer_sk = cs_or_ws_sales.customer_sk and d_moy = 2 and d_year = 1999 ) , my_revenue as ( select c_customer_sk, sum(ss_ext_sales_price) as revenue from my_customers, store_sales, customer_address, store, date_dim where c_current_addr_sk = ca_address_sk and ca_county = s_county and ca_state = s_state and ss_sold_date_sk = d_date_sk and c_customer_sk = ss_customer_sk and d_month_seq between (select distinct d_month_seq+1 from date_dim where d_year = 1999 and d_moy = 2) and (select distinct d_month_seq+3 from date_dim where d_year = 1999 and d_moy = 2) group by c_customer_sk ) , segments as (select cast((revenue/50) as int) as segment from my_revenue ) select segment, count(*) as num_customers, segment*50 as segment_base from segments group by segment order by segment, num_customers limit 100""", "q55" -> """ select i_brand_id brand_id, i_brand brand, sum(ss_ext_sales_price) ext_price from date_dim, store_sales, item where d_date_sk = ss_sold_date_sk and ss_item_sk = i_item_sk and i_manager_id=96 and d_moy=11 and d_year=2000 group by i_brand, i_brand_id order by ext_price desc, i_brand_id limit 100 """, "q56" -> """ with ss as ( select i_item_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('antique','white','smoke')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 6 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_item_id), cs as ( select i_item_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('antique','white','smoke')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 6 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_item_id), ws as ( select i_item_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('antique','white','smoke')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 6 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_item_id) select i_item_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_item_id order by total_sales, i_item_id limit 100""", "q57" -> """ with v1 as( select i_category, i_brand, cc_name, d_year, d_moy, sum(cs_sales_price) sum_sales, avg(sum(cs_sales_price)) over (partition by i_category, i_brand, cc_name, d_year) avg_monthly_sales, rank() over (partition by i_category, i_brand, cc_name order by d_year, d_moy) rn from item, catalog_sales, date_dim, call_center where cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and cc_call_center_sk= cs_call_center_sk and ( d_year = 2000 or ( d_year = 2000-1 and d_moy =12) or ( d_year = 2000+1 and d_moy =1) ) group by i_category, i_brand, cc_name , d_year, d_moy), v2 as( select v1.i_category, v1.i_brand, v1.cc_name ,v1.d_year ,v1.avg_monthly_sales ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum from v1, v1 v1_lag, v1 v1_lead where v1.i_category = v1_lag.i_category and v1.i_category = v1_lead.i_category and v1.i_brand = v1_lag.i_brand and v1.i_brand = v1_lead.i_brand and v1. cc_name = v1_lag. cc_name and v1. cc_name = v1_lead. cc_name and v1.rn = v1_lag.rn + 1 and v1.rn = v1_lead.rn - 1) select * from v2 where d_year = 2000 and avg_monthly_sales > 0 and case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by sum_sales - avg_monthly_sales, sum_sales limit 100""", "q58" -> """ with ss_items as (select i_item_id item_id ,sum(ss_ext_sales_price) ss_item_rev from store_sales ,item ,date_dim where ss_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq = (select d_week_seq from date_dim where d_date = '2001-01-27')) and ss_sold_date_sk = d_date_sk group by i_item_id), cs_items as (select i_item_id item_id ,sum(cs_ext_sales_price) cs_item_rev from catalog_sales ,item ,date_dim where cs_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq = (select d_week_seq from date_dim where d_date = '2001-01-27')) and cs_sold_date_sk = d_date_sk group by i_item_id), ws_items as (select i_item_id item_id ,sum(ws_ext_sales_price) ws_item_rev from web_sales ,item ,date_dim where ws_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq =(select d_week_seq from date_dim where d_date = '2001-01-27')) and ws_sold_date_sk = d_date_sk group by i_item_id) select ss_items.item_id ,ss_item_rev ,ss_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ss_dev ,cs_item_rev ,cs_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 cs_dev ,ws_item_rev ,ws_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ws_dev ,(ss_item_rev+cs_item_rev+ws_item_rev)/3 average from ss_items,cs_items,ws_items where ss_items.item_id=cs_items.item_id and ss_items.item_id=ws_items.item_id and ss_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev and ss_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev and cs_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev and cs_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev and ws_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev and ws_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev order by item_id ,ss_item_rev limit 100""", "q59" -> """ with wss as (select d_week_seq, ss_store_sk, sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales from store_sales,date_dim where d_date_sk = ss_sold_date_sk group by d_week_seq,ss_store_sk ) select s_store_name1,s_store_id1,d_week_seq1 ,sun_sales1/sun_sales2,mon_sales1/mon_sales2 ,tue_sales1/tue_sales2,wed_sales1/wed_sales2,thu_sales1/thu_sales2 ,fri_sales1/fri_sales2,sat_sales1/sat_sales2 from (select s_store_name s_store_name1,wss.d_week_seq d_week_seq1 ,s_store_id s_store_id1,sun_sales sun_sales1 ,mon_sales mon_sales1,tue_sales tue_sales1 ,wed_sales wed_sales1,thu_sales thu_sales1 ,fri_sales fri_sales1,sat_sales sat_sales1 from wss,store,date_dim d where d.d_week_seq = wss.d_week_seq and ss_store_sk = s_store_sk and d_month_seq between 1177 and 1177 + 11) y, (select s_store_name s_store_name2,wss.d_week_seq d_week_seq2 ,s_store_id s_store_id2,sun_sales sun_sales2 ,mon_sales mon_sales2,tue_sales tue_sales2 ,wed_sales wed_sales2,thu_sales thu_sales2 ,fri_sales fri_sales2,sat_sales sat_sales2 from wss,store,date_dim d where d.d_week_seq = wss.d_week_seq and ss_store_sk = s_store_sk and d_month_seq between 1177+ 12 and 1177 + 23) x where s_store_id1=s_store_id2 and d_week_seq1=d_week_seq2-52 order by s_store_name1,s_store_id1,d_week_seq1 limit 100""", "q60" -> """ with ss as ( select i_item_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Shoes')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 2002 and d_moy = 8 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -7 group by i_item_id), cs as ( select i_item_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Shoes')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 2002 and d_moy = 8 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -7 group by i_item_id), ws as ( select i_item_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Shoes')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 2002 and d_moy = 8 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -7 group by i_item_id) select i_item_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_item_id order by i_item_id ,total_sales limit 100""", "q61" -> """ select promotions,total,cast(promotions as decimal(15,4))/cast(total as decimal(15,4))*100 from (select sum(ss_ext_sales_price) promotions from store_sales ,store ,promotion ,date_dim ,customer ,customer_address ,item where ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ss_promo_sk = p_promo_sk and ss_customer_sk= c_customer_sk and ca_address_sk = c_current_addr_sk and ss_item_sk = i_item_sk and ca_gmt_offset = -6 and i_category = 'Home' and (p_channel_dmail = 'Y' or p_channel_email = 'Y' or p_channel_tv = 'Y') and s_gmt_offset = -6 and d_year = 1999 and d_moy = 12) promotional_sales, (select sum(ss_ext_sales_price) total from store_sales ,store ,date_dim ,customer ,customer_address ,item where ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ss_customer_sk= c_customer_sk and ca_address_sk = c_current_addr_sk and ss_item_sk = i_item_sk and ca_gmt_offset = -6 and i_category = 'Home' and s_gmt_offset = -6 and d_year = 1999 and d_moy = 12) all_sales order by promotions, total limit 100""", "q62" -> """ select substr(w_warehouse_name,1,20) ,sm_type ,web_name ,sum(case when (ws_ship_date_sk - ws_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 30) and (ws_ship_date_sk - ws_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 60) and (ws_ship_date_sk - ws_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 90) and (ws_ship_date_sk - ws_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from web_sales ,warehouse ,ship_mode ,web_site ,date_dim where d_month_seq between 1191 and 1191 + 11 and ws_ship_date_sk = d_date_sk and ws_warehouse_sk = w_warehouse_sk and ws_ship_mode_sk = sm_ship_mode_sk and ws_web_site_sk = web_site_sk group by substr(w_warehouse_name,1,20) ,sm_type ,web_name order by substr(w_warehouse_name,1,20) ,sm_type ,web_name limit 100""", "q63" -> """ select * from (select i_manager_id ,sum(ss_sales_price) sum_sales ,avg(sum(ss_sales_price)) over (partition by i_manager_id) avg_monthly_sales from item ,store_sales ,date_dim ,store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_month_seq in (1193,1193+1,1193+2,1193+3,1193+4,1193+5,1193+6,1193+7,1193+8,1193+9,1193+10,1193+11) and (( i_category in ('Books','Children','Electronics') and i_class in ('personal','portable','reference','self-help') and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', 'exportiunivamalg #9','scholaramalgamalg #9')) or( i_category in ('Women','Music','Men') and i_class in ('accessories','classical','fragrances','pants') and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 'importoamalg #1'))) group by i_manager_id, d_moy) tmp1 where case when avg_monthly_sales > 0 then abs (sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by i_manager_id ,avg_monthly_sales ,sum_sales limit 100""", "q64" -> """ with cs_ui as (select cs_item_sk ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund from catalog_sales ,catalog_returns where cs_item_sk = cr_item_sk and cs_order_number = cr_order_number group by cs_item_sk having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), cross_sales as (select i_product_name product_name ,i_item_sk item_sk ,s_store_name store_name ,s_zip store_zip ,ad1.ca_street_number b_street_number ,ad1.ca_street_name b_street_name ,ad1.ca_city b_city ,ad1.ca_zip b_zip ,ad2.ca_street_number c_street_number ,ad2.ca_street_name c_street_name ,ad2.ca_city c_city ,ad2.ca_zip c_zip ,d1.d_year as syear ,d2.d_year as fsyear ,d3.d_year s2year ,count(*) cnt ,sum(ss_wholesale_cost) s1 ,sum(ss_list_price) s2 ,sum(ss_coupon_amt) s3 FROM store_sales ,store_returns ,cs_ui ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,customer ,customer_demographics cd1 ,customer_demographics cd2 ,promotion ,household_demographics hd1 ,household_demographics hd2 ,customer_address ad1 ,customer_address ad2 ,income_band ib1 ,income_band ib2 ,item WHERE ss_store_sk = s_store_sk AND ss_sold_date_sk = d1.d_date_sk AND ss_customer_sk = c_customer_sk AND ss_cdemo_sk= cd1.cd_demo_sk AND ss_hdemo_sk = hd1.hd_demo_sk AND ss_addr_sk = ad1.ca_address_sk and ss_item_sk = i_item_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and ss_item_sk = cs_ui.cs_item_sk and c_current_cdemo_sk = cd2.cd_demo_sk AND c_current_hdemo_sk = hd2.hd_demo_sk AND c_current_addr_sk = ad2.ca_address_sk and c_first_sales_date_sk = d2.d_date_sk and c_first_shipto_date_sk = d3.d_date_sk and ss_promo_sk = p_promo_sk and hd1.hd_income_band_sk = ib1.ib_income_band_sk and hd2.hd_income_band_sk = ib2.ib_income_band_sk and cd1.cd_marital_status <> cd2.cd_marital_status and i_color in ('orange','aquamarine','olive','linen','smoke','coral') and i_current_price between 74 and 74 + 10 and i_current_price between 74 + 1 and 74 + 15 group by i_product_name ,i_item_sk ,s_store_name ,s_zip ,ad1.ca_street_number ,ad1.ca_street_name ,ad1.ca_city ,ad1.ca_zip ,ad2.ca_street_number ,ad2.ca_street_name ,ad2.ca_city ,ad2.ca_zip ,d1.d_year ,d2.d_year ,d3.d_year ) select cs1.product_name ,cs1.store_name ,cs1.store_zip ,cs1.b_street_number ,cs1.b_street_name ,cs1.b_city ,cs1.b_zip ,cs1.c_street_number ,cs1.c_street_name ,cs1.c_city ,cs1.c_zip ,cs1.syear ,cs1.cnt ,cs1.s1 as s11 ,cs1.s2 as s21 ,cs1.s3 as s31 ,cs2.s1 as s12 ,cs2.s2 as s22 ,cs2.s3 as s32 ,cs2.syear ,cs2.cnt from cross_sales cs1,cross_sales cs2 where cs1.item_sk=cs2.item_sk and cs1.syear = 2001 and cs2.syear = 2001 + 1 and cs2.cnt <= cs1.cnt and cs1.store_name = cs2.store_name and cs1.store_zip = cs2.store_zip order by cs1.product_name ,cs1.store_name ,cs2.cnt ,cs1.s1 ,cs2.s1""", "q65" -> """ select s_store_name, i_item_desc, sc.revenue, i_current_price, i_wholesale_cost, i_brand from store, item, (select ss_store_sk, avg(revenue) as ave from (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue from store_sales, date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1195 and 1195+11 group by ss_store_sk, ss_item_sk) sa group by ss_store_sk) sb, (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue from store_sales, date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1195 and 1195+11 group by ss_store_sk, ss_item_sk) sc where sb.ss_store_sk = sc.ss_store_sk and sc.revenue <= 0.1 * sb.ave and s_store_sk = sc.ss_store_sk and i_item_sk = sc.ss_item_sk order by s_store_name, i_item_desc limit 100""", "q66" -> """ select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,ship_carriers ,year ,sum(jan_sales) as jan_sales ,sum(feb_sales) as feb_sales ,sum(mar_sales) as mar_sales ,sum(apr_sales) as apr_sales ,sum(may_sales) as may_sales ,sum(jun_sales) as jun_sales ,sum(jul_sales) as jul_sales ,sum(aug_sales) as aug_sales ,sum(sep_sales) as sep_sales ,sum(oct_sales) as oct_sales ,sum(nov_sales) as nov_sales ,sum(dec_sales) as dec_sales ,sum(jan_sales/w_warehouse_sq_ft) as jan_sales_per_sq_foot ,sum(feb_sales/w_warehouse_sq_ft) as feb_sales_per_sq_foot ,sum(mar_sales/w_warehouse_sq_ft) as mar_sales_per_sq_foot ,sum(apr_sales/w_warehouse_sq_ft) as apr_sales_per_sq_foot ,sum(may_sales/w_warehouse_sq_ft) as may_sales_per_sq_foot ,sum(jun_sales/w_warehouse_sq_ft) as jun_sales_per_sq_foot ,sum(jul_sales/w_warehouse_sq_ft) as jul_sales_per_sq_foot ,sum(aug_sales/w_warehouse_sq_ft) as aug_sales_per_sq_foot ,sum(sep_sales/w_warehouse_sq_ft) as sep_sales_per_sq_foot ,sum(oct_sales/w_warehouse_sq_ft) as oct_sales_per_sq_foot ,sum(nov_sales/w_warehouse_sq_ft) as nov_sales_per_sq_foot ,sum(dec_sales/w_warehouse_sq_ft) as dec_sales_per_sq_foot ,sum(jan_net) as jan_net ,sum(feb_net) as feb_net ,sum(mar_net) as mar_net ,sum(apr_net) as apr_net ,sum(may_net) as may_net ,sum(jun_net) as jun_net ,sum(jul_net) as jul_net ,sum(aug_net) as aug_net ,sum(sep_net) as sep_net ,sum(oct_net) as oct_net ,sum(nov_net) as nov_net ,sum(dec_net) as dec_net from ( select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,'LATVIAN' || ',' || 'ALLIANCE' as ship_carriers ,d_year as year ,sum(case when d_moy = 1 then ws_ext_list_price* ws_quantity else 0 end) as jan_sales ,sum(case when d_moy = 2 then ws_ext_list_price* ws_quantity else 0 end) as feb_sales ,sum(case when d_moy = 3 then ws_ext_list_price* ws_quantity else 0 end) as mar_sales ,sum(case when d_moy = 4 then ws_ext_list_price* ws_quantity else 0 end) as apr_sales ,sum(case when d_moy = 5 then ws_ext_list_price* ws_quantity else 0 end) as may_sales ,sum(case when d_moy = 6 then ws_ext_list_price* ws_quantity else 0 end) as jun_sales ,sum(case when d_moy = 7 then ws_ext_list_price* ws_quantity else 0 end) as jul_sales ,sum(case when d_moy = 8 then ws_ext_list_price* ws_quantity else 0 end) as aug_sales ,sum(case when d_moy = 9 then ws_ext_list_price* ws_quantity else 0 end) as sep_sales ,sum(case when d_moy = 10 then ws_ext_list_price* ws_quantity else 0 end) as oct_sales ,sum(case when d_moy = 11 then ws_ext_list_price* ws_quantity else 0 end) as nov_sales ,sum(case when d_moy = 12 then ws_ext_list_price* ws_quantity else 0 end) as dec_sales ,sum(case when d_moy = 1 then ws_net_paid_inc_ship * ws_quantity else 0 end) as jan_net ,sum(case when d_moy = 2 then ws_net_paid_inc_ship * ws_quantity else 0 end) as feb_net ,sum(case when d_moy = 3 then ws_net_paid_inc_ship * ws_quantity else 0 end) as mar_net ,sum(case when d_moy = 4 then ws_net_paid_inc_ship * ws_quantity else 0 end) as apr_net ,sum(case when d_moy = 5 then ws_net_paid_inc_ship * ws_quantity else 0 end) as may_net ,sum(case when d_moy = 6 then ws_net_paid_inc_ship * ws_quantity else 0 end) as jun_net ,sum(case when d_moy = 7 then ws_net_paid_inc_ship * ws_quantity else 0 end) as jul_net ,sum(case when d_moy = 8 then ws_net_paid_inc_ship * ws_quantity else 0 end) as aug_net ,sum(case when d_moy = 9 then ws_net_paid_inc_ship * ws_quantity else 0 end) as sep_net ,sum(case when d_moy = 10 then ws_net_paid_inc_ship * ws_quantity else 0 end) as oct_net ,sum(case when d_moy = 11 then ws_net_paid_inc_ship * ws_quantity else 0 end) as nov_net ,sum(case when d_moy = 12 then ws_net_paid_inc_ship * ws_quantity else 0 end) as dec_net from web_sales ,warehouse ,date_dim ,time_dim ,ship_mode where ws_warehouse_sk = w_warehouse_sk and ws_sold_date_sk = d_date_sk and ws_sold_time_sk = t_time_sk and ws_ship_mode_sk = sm_ship_mode_sk and d_year = 1998 and t_time between 16224 and 16224+28800 and sm_carrier in ('LATVIAN','ALLIANCE') group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,d_year union all select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,'LATVIAN' || ',' || 'ALLIANCE' as ship_carriers ,d_year as year ,sum(case when d_moy = 1 then cs_ext_sales_price* cs_quantity else 0 end) as jan_sales ,sum(case when d_moy = 2 then cs_ext_sales_price* cs_quantity else 0 end) as feb_sales ,sum(case when d_moy = 3 then cs_ext_sales_price* cs_quantity else 0 end) as mar_sales ,sum(case when d_moy = 4 then cs_ext_sales_price* cs_quantity else 0 end) as apr_sales ,sum(case when d_moy = 5 then cs_ext_sales_price* cs_quantity else 0 end) as may_sales ,sum(case when d_moy = 6 then cs_ext_sales_price* cs_quantity else 0 end) as jun_sales ,sum(case when d_moy = 7 then cs_ext_sales_price* cs_quantity else 0 end) as jul_sales ,sum(case when d_moy = 8 then cs_ext_sales_price* cs_quantity else 0 end) as aug_sales ,sum(case when d_moy = 9 then cs_ext_sales_price* cs_quantity else 0 end) as sep_sales ,sum(case when d_moy = 10 then cs_ext_sales_price* cs_quantity else 0 end) as oct_sales ,sum(case when d_moy = 11 then cs_ext_sales_price* cs_quantity else 0 end) as nov_sales ,sum(case when d_moy = 12 then cs_ext_sales_price* cs_quantity else 0 end) as dec_sales ,sum(case when d_moy = 1 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as jan_net ,sum(case when d_moy = 2 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as feb_net ,sum(case when d_moy = 3 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as mar_net ,sum(case when d_moy = 4 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as apr_net ,sum(case when d_moy = 5 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as may_net ,sum(case when d_moy = 6 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as jun_net ,sum(case when d_moy = 7 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as jul_net ,sum(case when d_moy = 8 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as aug_net ,sum(case when d_moy = 9 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as sep_net ,sum(case when d_moy = 10 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as oct_net ,sum(case when d_moy = 11 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as nov_net ,sum(case when d_moy = 12 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as dec_net from catalog_sales ,warehouse ,date_dim ,time_dim ,ship_mode where cs_warehouse_sk = w_warehouse_sk and cs_sold_date_sk = d_date_sk and cs_sold_time_sk = t_time_sk and cs_ship_mode_sk = sm_ship_mode_sk and d_year = 1998 and t_time between 16224 AND 16224+28800 and sm_carrier in ('LATVIAN','ALLIANCE') group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,d_year ) x group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,ship_carriers ,year order by w_warehouse_name limit 100""", "q67" -> """ select * from (select i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sumsales ,rank() over (partition by i_category order by sumsales desc) rk from (select i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sum(coalesce(ss_sales_price*ss_quantity,0)) sumsales from store_sales ,date_dim ,store ,item where ss_sold_date_sk=d_date_sk and ss_item_sk=i_item_sk and ss_store_sk = s_store_sk and d_month_seq between 1203 and 1203+11 group by rollup(i_category, i_class, i_brand, i_product_name, d_year, d_qoy, d_moy,s_store_id))dw1) dw2 where rk <= 100 order by i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sumsales ,rk limit 100""", "q68" -> """ select c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number ,extended_price ,extended_tax ,list_price from (select ss_ticket_number ,ss_customer_sk ,ca_city bought_city ,sum(ss_ext_sales_price) extended_price ,sum(ss_ext_list_price) list_price ,sum(ss_ext_tax) extended_tax from store_sales ,date_dim ,store ,household_demographics ,customer_address where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and store_sales.ss_addr_sk = customer_address.ca_address_sk and date_dim.d_dom between 1 and 2 and (household_demographics.hd_dep_count = 3 or household_demographics.hd_vehicle_count= -1) and date_dim.d_year in (1999,1999+1,1999+2) and store.s_city in ('Jamestown','Pine Hill') group by ss_ticket_number ,ss_customer_sk ,ss_addr_sk,ca_city) dn ,customer ,customer_address current_addr where ss_customer_sk = c_customer_sk and customer.c_current_addr_sk = current_addr.ca_address_sk and current_addr.ca_city <> bought_city order by c_last_name ,ss_ticket_number limit 100""", "q69" -> """ select cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3 from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and ca_state in ('CA','MT','SD') and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2003 and d_moy between 2 and 2+2) and (not exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2003 and d_moy between 2 and 2+2) and not exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2003 and d_moy between 2 and 2+2)) group by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating order by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating limit 100""", "q70" -> """ select sum(ss_net_profit) as total_sum ,s_state ,s_county ,grouping(s_state)+grouping(s_county) as lochierarchy ,rank() over ( partition by grouping(s_state)+grouping(s_county), case when grouping(s_county) = 0 then s_state end order by sum(ss_net_profit) desc) as rank_within_parent from store_sales ,date_dim d1 ,store where d1.d_month_seq between 1215 and 1215+11 and d1.d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk and s_state in ( select s_state from (select s_state as s_state, rank() over ( partition by s_state order by sum(ss_net_profit) desc) as ranking from store_sales, store, date_dim where d_month_seq between 1215 and 1215+11 and d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk group by s_state ) tmp1 where ranking <= 5 ) group by rollup(s_state,s_county) order by lochierarchy desc ,case when lochierarchy = 0 then s_state end ,rank_within_parent limit 100""", "q71" -> """ select i_brand_id brand_id, i_brand brand,t_hour,t_minute, sum(ext_price) ext_price from item, (select ws_ext_sales_price as ext_price, ws_sold_date_sk as sold_date_sk, ws_item_sk as sold_item_sk, ws_sold_time_sk as time_sk from web_sales,date_dim where d_date_sk = ws_sold_date_sk and d_moy=11 and d_year=1998 union all select cs_ext_sales_price as ext_price, cs_sold_date_sk as sold_date_sk, cs_item_sk as sold_item_sk, cs_sold_time_sk as time_sk from catalog_sales,date_dim where d_date_sk = cs_sold_date_sk and d_moy=11 and d_year=1998 union all select ss_ext_sales_price as ext_price, ss_sold_date_sk as sold_date_sk, ss_item_sk as sold_item_sk, ss_sold_time_sk as time_sk from store_sales,date_dim where d_date_sk = ss_sold_date_sk and d_moy=11 and d_year=1998 ) tmp,time_dim where sold_item_sk = i_item_sk and i_manager_id=1 and time_sk = t_time_sk and (t_meal_time = 'breakfast' or t_meal_time = 'dinner') group by i_brand, i_brand_id,t_hour,t_minute order by ext_price desc, i_brand_id """, "q72" -> """ select i_item_desc ,w_warehouse_name ,d1.d_week_seq ,sum(case when p_promo_sk is null then 1 else 0 end) no_promo ,sum(case when p_promo_sk is not null then 1 else 0 end) promo ,count(*) total_cnt from catalog_sales join inventory on (cs_item_sk = inv_item_sk) join warehouse on (w_warehouse_sk=inv_warehouse_sk) join item on (i_item_sk = cs_item_sk) join customer_demographics on (cs_bill_cdemo_sk = cd_demo_sk) join household_demographics on (cs_bill_hdemo_sk = hd_demo_sk) join date_dim d1 on (cs_sold_date_sk = d1.d_date_sk) join date_dim d2 on (inv_date_sk = d2.d_date_sk) join date_dim d3 on (cs_ship_date_sk = d3.d_date_sk) left outer join promotion on (cs_promo_sk=p_promo_sk) left outer join catalog_returns on (cr_item_sk = cs_item_sk and cr_order_number = cs_order_number) where d1.d_week_seq = d2.d_week_seq and inv_quantity_on_hand < cs_quantity and d3.d_date > d1.d_date + interval 5 days and hd_buy_potential = '1001-5000' and d1.d_year = 1998 and cd_marital_status = 'S' group by i_item_desc,w_warehouse_name,d1.d_week_seq order by total_cnt desc, i_item_desc, w_warehouse_name, d_week_seq limit 100""", "q73" -> """ select c_last_name ,c_first_name ,c_salutation ,c_preferred_cust_flag ,ss_ticket_number ,cnt from (select ss_ticket_number ,ss_customer_sk ,count(*) cnt from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and date_dim.d_dom between 1 and 2 and (household_demographics.hd_buy_potential = '>10000' or household_demographics.hd_buy_potential = 'Unknown') and household_demographics.hd_vehicle_count > 0 and case when household_demographics.hd_vehicle_count > 0 then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end > 1 and date_dim.d_year in (1998,1998+1,1998+2) and store.s_county in ('Van Buren County','Terrell County','Belknap County','Kootenai County') group by ss_ticket_number,ss_customer_sk) dj,customer where ss_customer_sk = c_customer_sk and cnt between 1 and 5 order by cnt desc, c_last_name asc""", "q74" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,d_year as year ,max(ss_net_paid) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (2001,2001+1) group by c_customer_id ,c_first_name ,c_last_name ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,d_year as year ,max(ws_net_paid) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year in (2001,2001+1) group by c_customer_id ,c_first_name ,c_last_name ,d_year ) select t_s_secyear.customer_id, t_s_secyear.customer_first_name, t_s_secyear.customer_last_name from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.sale_type = 's' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_w_secyear.sale_type = 'w' and t_s_firstyear.year = 2001 and t_s_secyear.year = 2001+1 and t_w_firstyear.year = 2001 and t_w_secyear.year = 2001+1 and t_s_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end order by 2,3,1 limit 100""", "q75" -> """ WITH all_sales AS ( SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,SUM(sales_cnt) AS sales_cnt ,SUM(sales_amt) AS sales_amt FROM (SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,cs_quantity - COALESCE(cr_return_quantity,0) AS sales_cnt ,cs_ext_sales_price - COALESCE(cr_return_amount,0.0) AS sales_amt FROM catalog_sales JOIN item ON i_item_sk=cs_item_sk JOIN date_dim ON d_date_sk=cs_sold_date_sk LEFT JOIN catalog_returns ON (cs_order_number=cr_order_number AND cs_item_sk=cr_item_sk) WHERE i_category='Music' UNION SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,ss_quantity - COALESCE(sr_return_quantity,0) AS sales_cnt ,ss_ext_sales_price - COALESCE(sr_return_amt,0.0) AS sales_amt FROM store_sales JOIN item ON i_item_sk=ss_item_sk JOIN date_dim ON d_date_sk=ss_sold_date_sk LEFT JOIN store_returns ON (ss_ticket_number=sr_ticket_number AND ss_item_sk=sr_item_sk) WHERE i_category='Music' UNION SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,ws_quantity - COALESCE(wr_return_quantity,0) AS sales_cnt ,ws_ext_sales_price - COALESCE(wr_return_amt,0.0) AS sales_amt FROM web_sales JOIN item ON i_item_sk=ws_item_sk JOIN date_dim ON d_date_sk=ws_sold_date_sk LEFT JOIN web_returns ON (ws_order_number=wr_order_number AND ws_item_sk=wr_item_sk) WHERE i_category='Music') sales_detail GROUP BY d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id) SELECT prev_yr.d_year AS prev_year ,curr_yr.d_year AS year ,curr_yr.i_brand_id ,curr_yr.i_class_id ,curr_yr.i_category_id ,curr_yr.i_manufact_id ,prev_yr.sales_cnt AS prev_yr_cnt ,curr_yr.sales_cnt AS curr_yr_cnt ,curr_yr.sales_cnt-prev_yr.sales_cnt AS sales_cnt_diff ,curr_yr.sales_amt-prev_yr.sales_amt AS sales_amt_diff FROM all_sales curr_yr, all_sales prev_yr WHERE curr_yr.i_brand_id=prev_yr.i_brand_id AND curr_yr.i_class_id=prev_yr.i_class_id AND curr_yr.i_category_id=prev_yr.i_category_id AND curr_yr.i_manufact_id=prev_yr.i_manufact_id AND curr_yr.d_year=2001 AND prev_yr.d_year=2001-1 AND CAST(curr_yr.sales_cnt AS DECIMAL(17,2))/CAST(prev_yr.sales_cnt AS DECIMAL(17,2))<0.9 ORDER BY sales_cnt_diff,sales_amt_diff limit 100""", "q76" -> """ select channel, col_name, d_year, d_qoy, i_category, COUNT(*) sales_cnt, SUM(ext_sales_price) sales_amt FROM ( SELECT 'store' as channel, 'ss_promo_sk' col_name, d_year, d_qoy, i_category, ss_ext_sales_price ext_sales_price FROM store_sales, item, date_dim WHERE ss_promo_sk IS NULL AND ss_sold_date_sk=d_date_sk AND ss_item_sk=i_item_sk UNION ALL SELECT 'web' as channel, 'ws_web_site_sk' col_name, d_year, d_qoy, i_category, ws_ext_sales_price ext_sales_price FROM web_sales, item, date_dim WHERE ws_web_site_sk IS NULL AND ws_sold_date_sk=d_date_sk AND ws_item_sk=i_item_sk UNION ALL SELECT 'catalog' as channel, 'cs_bill_addr_sk' col_name, d_year, d_qoy, i_category, cs_ext_sales_price ext_sales_price FROM catalog_sales, item, date_dim WHERE cs_bill_addr_sk IS NULL AND cs_sold_date_sk=d_date_sk AND cs_item_sk=i_item_sk) foo GROUP BY channel, col_name, d_year, d_qoy, i_category ORDER BY channel, col_name, d_year, d_qoy, i_category limit 100""", "q77" -> """ with ss as (select s_store_sk, sum(ss_ext_sales_price) as sales, sum(ss_net_profit) as profit from store_sales, date_dim, store where ss_sold_date_sk = d_date_sk and d_date between cast('2001-08-22' as date) and (cast('2001-08-22' as date) + INTERVAL 30 days) and ss_store_sk = s_store_sk group by s_store_sk) , sr as (select s_store_sk, sum(sr_return_amt) as returns, sum(sr_net_loss) as profit_loss from store_returns, date_dim, store where sr_returned_date_sk = d_date_sk and d_date between cast('2001-08-22' as date) and (cast('2001-08-22' as date) + INTERVAL 30 days) and sr_store_sk = s_store_sk group by s_store_sk), cs as (select cs_call_center_sk, sum(cs_ext_sales_price) as sales, sum(cs_net_profit) as profit from catalog_sales, date_dim where cs_sold_date_sk = d_date_sk and d_date between cast('2001-08-22' as date) and (cast('2001-08-22' as date) + INTERVAL 30 days) group by cs_call_center_sk ), cr as (select cr_call_center_sk, sum(cr_return_amount) as returns, sum(cr_net_loss) as profit_loss from catalog_returns, date_dim where cr_returned_date_sk = d_date_sk and d_date between cast('2001-08-22' as date) and (cast('2001-08-22' as date) + INTERVAL 30 days) group by cr_call_center_sk ), ws as ( select wp_web_page_sk, sum(ws_ext_sales_price) as sales, sum(ws_net_profit) as profit from web_sales, date_dim, web_page where ws_sold_date_sk = d_date_sk and d_date between cast('2001-08-22' as date) and (cast('2001-08-22' as date) + INTERVAL 30 days) and ws_web_page_sk = wp_web_page_sk group by wp_web_page_sk), wr as (select wp_web_page_sk, sum(wr_return_amt) as returns, sum(wr_net_loss) as profit_loss from web_returns, date_dim, web_page where wr_returned_date_sk = d_date_sk and d_date between cast('2001-08-22' as date) and (cast('2001-08-22' as date) + INTERVAL 30 days) and wr_web_page_sk = wp_web_page_sk group by wp_web_page_sk) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , ss.s_store_sk as id , sales , coalesce(returns, 0) as returns , (profit - coalesce(profit_loss,0)) as profit from ss left join sr on ss.s_store_sk = sr.s_store_sk union all select 'catalog channel' as channel , cs_call_center_sk as id , sales , returns , (profit - profit_loss) as profit from cs , cr union all select 'web channel' as channel , ws.wp_web_page_sk as id , sales , coalesce(returns, 0) returns , (profit - coalesce(profit_loss,0)) as profit from ws left join wr on ws.wp_web_page_sk = wr.wp_web_page_sk ) x group by rollup (channel, id) order by channel ,id limit 100""", "q78" -> """ with ws as (select d_year AS ws_sold_year, ws_item_sk, ws_bill_customer_sk ws_customer_sk, sum(ws_quantity) ws_qty, sum(ws_wholesale_cost) ws_wc, sum(ws_sales_price) ws_sp from web_sales left join web_returns on wr_order_number=ws_order_number and ws_item_sk=wr_item_sk join date_dim on ws_sold_date_sk = d_date_sk where wr_order_number is null group by d_year, ws_item_sk, ws_bill_customer_sk ), cs as (select d_year AS cs_sold_year, cs_item_sk, cs_bill_customer_sk cs_customer_sk, sum(cs_quantity) cs_qty, sum(cs_wholesale_cost) cs_wc, sum(cs_sales_price) cs_sp from catalog_sales left join catalog_returns on cr_order_number=cs_order_number and cs_item_sk=cr_item_sk join date_dim on cs_sold_date_sk = d_date_sk where cr_order_number is null group by d_year, cs_item_sk, cs_bill_customer_sk ), ss as (select d_year AS ss_sold_year, ss_item_sk, ss_customer_sk, sum(ss_quantity) ss_qty, sum(ss_wholesale_cost) ss_wc, sum(ss_sales_price) ss_sp from store_sales left join store_returns on sr_ticket_number=ss_ticket_number and ss_item_sk=sr_item_sk join date_dim on ss_sold_date_sk = d_date_sk where sr_ticket_number is null group by d_year, ss_item_sk, ss_customer_sk ) select ss_sold_year, ss_item_sk, ss_customer_sk, round(ss_qty/(coalesce(ws_qty,0)+coalesce(cs_qty,0)),2) ratio, ss_qty store_qty, ss_wc store_wholesale_cost, ss_sp store_sales_price, coalesce(ws_qty,0)+coalesce(cs_qty,0) other_chan_qty, coalesce(ws_wc,0)+coalesce(cs_wc,0) other_chan_wholesale_cost, coalesce(ws_sp,0)+coalesce(cs_sp,0) other_chan_sales_price from ss left join ws on (ws_sold_year=ss_sold_year and ws_item_sk=ss_item_sk and ws_customer_sk=ss_customer_sk) left join cs on (cs_sold_year=ss_sold_year and cs_item_sk=ss_item_sk and cs_customer_sk=ss_customer_sk) where (coalesce(ws_qty,0)>0 or coalesce(cs_qty, 0)>0) and ss_sold_year=2001 order by ss_sold_year, ss_item_sk, ss_customer_sk, ss_qty desc, ss_wc desc, ss_sp desc, other_chan_qty, other_chan_wholesale_cost, other_chan_sales_price, ratio limit 100""", "q79" -> """ select c_last_name,c_first_name,substr(s_city,1,30),ss_ticket_number,amt,profit from (select ss_ticket_number ,ss_customer_sk ,store.s_city ,sum(ss_coupon_amt) amt ,sum(ss_net_profit) profit from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and (household_demographics.hd_dep_count = 6 or household_demographics.hd_vehicle_count > -1) and date_dim.d_dow = 1 and date_dim.d_year in (1998,1998+1,1998+2) and store.s_number_employees between 200 and 295 group by ss_ticket_number,ss_customer_sk,ss_addr_sk,store.s_city) ms,customer where ss_customer_sk = c_customer_sk order by c_last_name,c_first_name,substr(s_city,1,30), profit limit 100""", "q80" -> """ with ssr as (select s_store_id as store_id, sum(ss_ext_sales_price) as sales, sum(coalesce(sr_return_amt, 0)) as returns, sum(ss_net_profit - coalesce(sr_net_loss, 0)) as profit from store_sales left outer join store_returns on (ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number), date_dim, store, item, promotion where ss_sold_date_sk = d_date_sk and d_date between cast('2000-08-25' as date) and (cast('2000-08-25' as date) + INTERVAL 60 days) and ss_store_sk = s_store_sk and ss_item_sk = i_item_sk and i_current_price > 50 and ss_promo_sk = p_promo_sk and p_channel_tv = 'N' group by s_store_id) , csr as (select cp_catalog_page_id as catalog_page_id, sum(cs_ext_sales_price) as sales, sum(coalesce(cr_return_amount, 0)) as returns, sum(cs_net_profit - coalesce(cr_net_loss, 0)) as profit from catalog_sales left outer join catalog_returns on (cs_item_sk = cr_item_sk and cs_order_number = cr_order_number), date_dim, catalog_page, item, promotion where cs_sold_date_sk = d_date_sk and d_date between cast('2000-08-25' as date) and (cast('2000-08-25' as date) + INTERVAL 60 days) and cs_catalog_page_sk = cp_catalog_page_sk and cs_item_sk = i_item_sk and i_current_price > 50 and cs_promo_sk = p_promo_sk and p_channel_tv = 'N' group by cp_catalog_page_id) , wsr as (select web_site_id, sum(ws_ext_sales_price) as sales, sum(coalesce(wr_return_amt, 0)) as returns, sum(ws_net_profit - coalesce(wr_net_loss, 0)) as profit from web_sales left outer join web_returns on (ws_item_sk = wr_item_sk and ws_order_number = wr_order_number), date_dim, web_site, item, promotion where ws_sold_date_sk = d_date_sk and d_date between cast('2000-08-25' as date) and (cast('2000-08-25' as date) + INTERVAL 60 days) and ws_web_site_sk = web_site_sk and ws_item_sk = i_item_sk and i_current_price > 50 and ws_promo_sk = p_promo_sk and p_channel_tv = 'N' group by web_site_id) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , 'store' || store_id as id , sales , returns , profit from ssr union all select 'catalog channel' as channel , 'catalog_page' || catalog_page_id as id , sales , returns , profit from csr union all select 'web channel' as channel , 'web_site' || web_site_id as id , sales , returns , profit from wsr ) x group by rollup (channel, id) order by channel ,id limit 100""", "q81" -> """ with customer_total_return as (select cr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, sum(cr_return_amt_inc_tax) as ctr_total_return from catalog_returns ,date_dim ,customer_address where cr_returned_date_sk = d_date_sk and d_year =2000 and cr_returning_addr_sk = ca_address_sk group by cr_returning_customer_sk ,ca_state ) select c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset ,ca_location_type,ctr_total_return from customer_total_return ctr1 ,customer_address ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_state = ctr2.ctr_state) and ca_address_sk = c_current_addr_sk and ca_state = 'SC' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset ,ca_location_type,ctr_total_return limit 100""", "q82" -> """ select i_item_id ,i_item_desc ,i_current_price from item, inventory, date_dim, store_sales where i_current_price between 6 and 6+30 and inv_item_sk = i_item_sk and d_date_sk=inv_date_sk and d_date between cast('2001-02-23' as date) and (cast('2001-02-23' as date) + INTERVAL 60 days) and i_manufact_id in (669,623,578,379) and inv_quantity_on_hand between 100 and 500 and ss_item_sk = i_item_sk group by i_item_id,i_item_desc,i_current_price order by i_item_id limit 100""", "q83" -> """ with sr_items as (select i_item_id item_id, sum(sr_return_quantity) sr_item_qty from store_returns, item, date_dim where sr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('2001-01-15','2001-09-03','2001-11-17'))) and sr_returned_date_sk = d_date_sk group by i_item_id), cr_items as (select i_item_id item_id, sum(cr_return_quantity) cr_item_qty from catalog_returns, item, date_dim where cr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('2001-01-15','2001-09-03','2001-11-17'))) and cr_returned_date_sk = d_date_sk group by i_item_id), wr_items as (select i_item_id item_id, sum(wr_return_quantity) wr_item_qty from web_returns, item, date_dim where wr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('2001-01-15','2001-09-03','2001-11-17'))) and wr_returned_date_sk = d_date_sk group by i_item_id) select sr_items.item_id ,sr_item_qty ,sr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 sr_dev ,cr_item_qty ,cr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 cr_dev ,wr_item_qty ,wr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 wr_dev ,(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 average from sr_items ,cr_items ,wr_items where sr_items.item_id=cr_items.item_id and sr_items.item_id=wr_items.item_id order by sr_items.item_id ,sr_item_qty limit 100""", "q84" -> """ select c_customer_id as customer_id , coalesce(c_last_name,'') || ', ' || coalesce(c_first_name,'') as customername from customer ,customer_address ,customer_demographics ,household_demographics ,income_band ,store_returns where ca_city = 'Walnut Grove' and c_current_addr_sk = ca_address_sk and ib_lower_bound >= 53669 and ib_upper_bound <= 53669 + 50000 and ib_income_band_sk = hd_income_band_sk and cd_demo_sk = c_current_cdemo_sk and hd_demo_sk = c_current_hdemo_sk and sr_cdemo_sk = cd_demo_sk order by c_customer_id limit 100""", "q85" -> """ select substr(r_reason_desc,1,20) ,avg(ws_quantity) ,avg(wr_refunded_cash) ,avg(wr_fee) from web_sales, web_returns, web_page, customer_demographics cd1, customer_demographics cd2, customer_address, date_dim, reason where ws_web_page_sk = wp_web_page_sk and ws_item_sk = wr_item_sk and ws_order_number = wr_order_number and ws_sold_date_sk = d_date_sk and d_year = 2001 and cd1.cd_demo_sk = wr_refunded_cdemo_sk and cd2.cd_demo_sk = wr_returning_cdemo_sk and ca_address_sk = wr_refunded_addr_sk and r_reason_sk = wr_reason_sk and ( ( cd1.cd_marital_status = 'S' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = 'Secondary' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 100.00 and 150.00 ) or ( cd1.cd_marital_status = 'D' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = 'Advanced Degree' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 50.00 and 100.00 ) or ( cd1.cd_marital_status = 'W' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = 'Primary' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 150.00 and 200.00 ) ) and ( ( ca_country = 'United States' and ca_state in ('AZ', 'SD', 'TN') and ws_net_profit between 100 and 200 ) or ( ca_country = 'United States' and ca_state in ('TX', 'GA', 'IA') and ws_net_profit between 150 and 300 ) or ( ca_country = 'United States' and ca_state in ('WI', 'VT', 'AL') and ws_net_profit between 50 and 250 ) ) group by r_reason_desc order by substr(r_reason_desc,1,20) ,avg(ws_quantity) ,avg(wr_refunded_cash) ,avg(wr_fee) limit 100""", "q86" -> """ select sum(ws_net_paid) as total_sum ,i_category ,i_class ,grouping(i_category)+grouping(i_class) as lochierarchy ,rank() over ( partition by grouping(i_category)+grouping(i_class), case when grouping(i_class) = 0 then i_category end order by sum(ws_net_paid) desc) as rank_within_parent from web_sales ,date_dim d1 ,item where d1.d_month_seq between 1195 and 1195+11 and d1.d_date_sk = ws_sold_date_sk and i_item_sk = ws_item_sk group by rollup(i_category,i_class) order by lochierarchy desc, case when lochierarchy = 0 then i_category end, rank_within_parent limit 100""", "q87" -> """ select count(*) from ((select distinct c_last_name, c_first_name, d_date from store_sales, date_dim, customer where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_customer_sk = customer.c_customer_sk and d_month_seq between 1194 and 1194+11) except (select distinct c_last_name, c_first_name, d_date from catalog_sales, date_dim, customer where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1194 and 1194+11) except (select distinct c_last_name, c_first_name, d_date from web_sales, date_dim, customer where web_sales.ws_sold_date_sk = date_dim.d_date_sk and web_sales.ws_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1194 and 1194+11) ) cool_cust""", "q88" -> """ select * from (select count(*) h8_30_to_9 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 8 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2)) and store.s_store_name = 'ese') s1, (select count(*) h9_to_9_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 9 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2)) and store.s_store_name = 'ese') s2, (select count(*) h9_30_to_10 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 9 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2)) and store.s_store_name = 'ese') s3, (select count(*) h10_to_10_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 10 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2)) and store.s_store_name = 'ese') s4, (select count(*) h10_30_to_11 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 10 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2)) and store.s_store_name = 'ese') s5, (select count(*) h11_to_11_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 11 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2)) and store.s_store_name = 'ese') s6, (select count(*) h11_30_to_12 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 11 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2)) and store.s_store_name = 'ese') s7, (select count(*) h12_to_12_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 12 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2)) and store.s_store_name = 'ese') s8""", "q89" -> """ select * from( select i_category, i_class, i_brand, s_store_name, s_company_name, d_moy, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_category, i_brand, s_store_name, s_company_name) avg_monthly_sales from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_year in (2000) and ((i_category in ('Home','Shoes','Electronics') and i_class in ('flatware','mens','televisions') ) or (i_category in ('Women','Sports','Music') and i_class in ('maternity','camping','rock') )) group by i_category, i_class, i_brand, s_store_name, s_company_name, d_moy) tmp1 where case when (avg_monthly_sales <> 0) then (abs(sum_sales - avg_monthly_sales) / avg_monthly_sales) else null end > 0.1 order by sum_sales - avg_monthly_sales, s_store_name limit 100""", "q90" -> """ select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio from ( select count(*) amc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 8 and 8+1 and household_demographics.hd_dep_count = 4 and web_page.wp_char_count between 5000 and 5200) at, ( select count(*) pmc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 20 and 20+1 and household_demographics.hd_dep_count = 4 and web_page.wp_char_count between 5000 and 5200) pt order by am_pm_ratio limit 100""", "q91" -> """ select cc_call_center_id Call_Center, cc_name Call_Center_Name, cc_manager Manager, sum(cr_net_loss) Returns_Loss from call_center, catalog_returns, date_dim, customer, customer_address, customer_demographics, household_demographics where cr_call_center_sk = cc_call_center_sk and cr_returned_date_sk = d_date_sk and cr_returning_customer_sk= c_customer_sk and cd_demo_sk = c_current_cdemo_sk and hd_demo_sk = c_current_hdemo_sk and ca_address_sk = c_current_addr_sk and d_year = 2001 and d_moy = 12 and ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') or(cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) and hd_buy_potential like 'Unknown%' and ca_gmt_offset = -6 group by cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status order by sum(cr_net_loss) desc""", "q92" -> """ select sum(ws_ext_discount_amt) as `Excess Discount Amount` from web_sales ,item ,date_dim where i_manufact_id = 7 and i_item_sk = ws_item_sk and d_date between '2000-01-16' and (cast('2000-01-16' as date) + INTERVAL 90 days) and d_date_sk = ws_sold_date_sk and ws_ext_discount_amt > ( SELECT 1.3 * avg(ws_ext_discount_amt) FROM web_sales ,date_dim WHERE ws_item_sk = i_item_sk and d_date between '2000-01-16' and (cast('2000-01-16' as date) + INTERVAL 90 days) and d_date_sk = ws_sold_date_sk ) order by sum(ws_ext_discount_amt) limit 100""", "q93" -> """ select ss_customer_sk ,sum(act_sales) sumsales from (select ss_item_sk ,ss_ticket_number ,ss_customer_sk ,case when sr_return_quantity is not null then (ss_quantity-sr_return_quantity)*ss_sales_price else (ss_quantity*ss_sales_price) end act_sales from store_sales left outer join store_returns on (sr_item_sk = ss_item_sk and sr_ticket_number = ss_ticket_number) ,reason where sr_reason_sk = r_reason_sk and r_reason_desc = 'reason 24') t group by ss_customer_sk order by sumsales, ss_customer_sk limit 100""", "q94" -> """ select count(distinct ws_order_number) as `order count` ,sum(ws_ext_ship_cost) as `total shipping cost` ,sum(ws_net_profit) as `total net profit` from web_sales ws1 ,date_dim ,customer_address ,web_site where d_date between '2001-2-01' and (cast('2001-2-01' as date) + INTERVAL 60 days) and ws1.ws_ship_date_sk = d_date_sk and ws1.ws_ship_addr_sk = ca_address_sk and ca_state = 'VT' and ws1.ws_web_site_sk = web_site_sk and web_company_name = 'pri' and exists (select * from web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) and not exists(select * from web_returns wr1 where ws1.ws_order_number = wr1.wr_order_number) order by count(distinct ws_order_number) limit 100""", "q95" -> """ with ws_wh as (select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 from web_sales ws1,web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) select count(distinct ws_order_number) as `order count` ,sum(ws_ext_ship_cost) as `total shipping cost` ,sum(ws_net_profit) as `total net profit` from web_sales ws1 ,date_dim ,customer_address ,web_site where d_date between '2001-3-01' and (cast('2001-3-01' as date) + INTERVAL 60 days) and ws1.ws_ship_date_sk = d_date_sk and ws1.ws_ship_addr_sk = ca_address_sk and ca_state = 'TN' and ws1.ws_web_site_sk = web_site_sk and web_company_name = 'pri' and ws1.ws_order_number in (select ws_order_number from ws_wh) and ws1.ws_order_number in (select wr_order_number from web_returns,ws_wh where wr_order_number = ws_wh.ws_order_number) order by count(distinct ws_order_number) limit 100""", "q96" -> """ select count(*) from store_sales ,household_demographics ,time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 20 and time_dim.t_minute >= 30 and household_demographics.hd_dep_count = 6 and store.s_store_name = 'ese' order by count(*) limit 100""", "q97" -> """ with ssci as ( select ss_customer_sk customer_sk ,ss_item_sk item_sk from store_sales,date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1206 and 1206 + 11 group by ss_customer_sk ,ss_item_sk), csci as( select cs_bill_customer_sk customer_sk ,cs_item_sk item_sk from catalog_sales,date_dim where cs_sold_date_sk = d_date_sk and d_month_seq between 1206 and 1206 + 11 group by cs_bill_customer_sk ,cs_item_sk) select sum(case when ssci.customer_sk is not null and csci.customer_sk is null then 1 else 0 end) store_only ,sum(case when ssci.customer_sk is null and csci.customer_sk is not null then 1 else 0 end) catalog_only ,sum(case when ssci.customer_sk is not null and csci.customer_sk is not null then 1 else 0 end) store_and_catalog from ssci full outer join csci on (ssci.customer_sk=csci.customer_sk and ssci.item_sk = csci.item_sk) limit 100""", "q98" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(ss_ext_sales_price) as itemrevenue ,sum(ss_ext_sales_price)*100/sum(sum(ss_ext_sales_price)) over (partition by i_class) as revenueratio from store_sales ,item ,date_dim where ss_item_sk = i_item_sk and i_category in ('Sports', 'Books', 'Electronics') and ss_sold_date_sk = d_date_sk and d_date between cast('2002-06-29' as date) and (cast('2002-06-29' as date) + interval 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio""", "q99" -> """ select substr(w_warehouse_name,1,20) ,sm_type ,cc_name ,sum(case when (cs_ship_date_sk - cs_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 30) and (cs_ship_date_sk - cs_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 60) and (cs_ship_date_sk - cs_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 90) and (cs_ship_date_sk - cs_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from catalog_sales ,warehouse ,ship_mode ,call_center ,date_dim where d_month_seq between 1199 and 1199 + 11 and cs_ship_date_sk = d_date_sk and cs_warehouse_sk = w_warehouse_sk and cs_ship_mode_sk = sm_ship_mode_sk and cs_call_center_sk = cc_call_center_sk group by substr(w_warehouse_name,1,20) ,sm_type ,cc_name order by substr(w_warehouse_name,1,20) ,sm_type ,cc_name limit 100""", "q1" -> """ with customer_total_return as (select sr_customer_sk as ctr_customer_sk ,sr_store_sk as ctr_store_sk ,sum(SR_FEE) as ctr_total_return from store_returns ,date_dim where sr_returned_date_sk = d_date_sk and d_year =2000 group by sr_customer_sk ,sr_store_sk) select c_customer_id from customer_total_return ctr1 ,store ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_store_sk = ctr2.ctr_store_sk) and s_store_sk = ctr1.ctr_store_sk and s_state = 'MO' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id limit 100""", "q2" -> """ with wscs as (select sold_date_sk ,sales_price from (select ws_sold_date_sk sold_date_sk ,ws_ext_sales_price sales_price from web_sales union all select cs_sold_date_sk sold_date_sk ,cs_ext_sales_price sales_price from catalog_sales)), wswscs as (select d_week_seq, sum(case when (d_day_name='Sunday') then sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then sales_price else null end) sat_sales from wscs ,date_dim where d_date_sk = sold_date_sk group by d_week_seq) select d_week_seq1 ,round(sun_sales1/sun_sales2,2) ,round(mon_sales1/mon_sales2,2) ,round(tue_sales1/tue_sales2,2) ,round(wed_sales1/wed_sales2,2) ,round(thu_sales1/thu_sales2,2) ,round(fri_sales1/fri_sales2,2) ,round(sat_sales1/sat_sales2,2) from (select wswscs.d_week_seq d_week_seq1 ,sun_sales sun_sales1 ,mon_sales mon_sales1 ,tue_sales tue_sales1 ,wed_sales wed_sales1 ,thu_sales thu_sales1 ,fri_sales fri_sales1 ,sat_sales sat_sales1 from wswscs,date_dim where date_dim.d_week_seq = wswscs.d_week_seq and d_year = 1998) y, (select wswscs.d_week_seq d_week_seq2 ,sun_sales sun_sales2 ,mon_sales mon_sales2 ,tue_sales tue_sales2 ,wed_sales wed_sales2 ,thu_sales thu_sales2 ,fri_sales fri_sales2 ,sat_sales sat_sales2 from wswscs ,date_dim where date_dim.d_week_seq = wswscs.d_week_seq and d_year = 1998+1) z where d_week_seq1=d_week_seq2-53 order by d_week_seq1""", "q3" -> """ select dt.d_year ,item.i_brand_id brand_id ,item.i_brand brand ,sum(ss_sales_price) sum_agg from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manufact_id = 816 and dt.d_moy=11 group by dt.d_year ,item.i_brand ,item.i_brand_id order by dt.d_year ,sum_agg desc ,brand_id limit 100""", "q4" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum((((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2) ) year_total ,'c' sale_type from customer ,catalog_sales ,date_dim where c_customer_sk = cs_bill_customer_sk and cs_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum((((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2) ) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year ) select t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_birth_country from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_c_firstyear ,year_total t_c_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_c_secyear.customer_id and t_s_firstyear.customer_id = t_c_firstyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.sale_type = 's' and t_c_firstyear.sale_type = 'c' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_c_secyear.sale_type = 'c' and t_w_secyear.sale_type = 'w' and t_s_firstyear.dyear = 1999 and t_s_secyear.dyear = 1999+1 and t_c_firstyear.dyear = 1999 and t_c_secyear.dyear = 1999+1 and t_w_firstyear.dyear = 1999 and t_w_secyear.dyear = 1999+1 and t_s_firstyear.year_total > 0 and t_c_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end > case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end order by t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_birth_country limit 100""", "q5" -> """ with ssr as (select s_store_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select ss_store_sk as store_sk, ss_sold_date_sk as date_sk, ss_ext_sales_price as sales_price, ss_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from store_sales union all select sr_store_sk as store_sk, sr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, sr_return_amt as return_amt, sr_net_loss as net_loss from store_returns ) salesreturns, date_dim, store where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and store_sk = s_store_sk group by s_store_id) , csr as (select cp_catalog_page_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select cs_catalog_page_sk as page_sk, cs_sold_date_sk as date_sk, cs_ext_sales_price as sales_price, cs_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from catalog_sales union all select cr_catalog_page_sk as page_sk, cr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, cr_return_amount as return_amt, cr_net_loss as net_loss from catalog_returns ) salesreturns, date_dim, catalog_page where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and page_sk = cp_catalog_page_sk group by cp_catalog_page_id) , wsr as (select web_site_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select ws_web_site_sk as wsr_web_site_sk, ws_sold_date_sk as date_sk, ws_ext_sales_price as sales_price, ws_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from web_sales union all select ws_web_site_sk as wsr_web_site_sk, wr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, wr_return_amt as return_amt, wr_net_loss as net_loss from web_returns left outer join web_sales on ( wr_item_sk = ws_item_sk and wr_order_number = ws_order_number) ) salesreturns, date_dim, web_site where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and wsr_web_site_sk = web_site_sk group by web_site_id) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , 'store' || s_store_id as id , sales , returns , (profit - profit_loss) as profit from ssr union all select 'catalog channel' as channel , 'catalog_page' || cp_catalog_page_id as id , sales , returns , (profit - profit_loss) as profit from csr union all select 'web channel' as channel , 'web_site' || web_site_id as id , sales , returns , (profit - profit_loss) as profit from wsr ) x group by rollup (channel, id) order by channel ,id limit 100""", "q6" -> """ select a.ca_state state, count(*) cnt from customer_address a ,customer c ,store_sales s ,date_dim d ,item i where a.ca_address_sk = c.c_current_addr_sk and c.c_customer_sk = s.ss_customer_sk and s.ss_sold_date_sk = d.d_date_sk and s.ss_item_sk = i.i_item_sk and d.d_month_seq = (select distinct (d_month_seq) from date_dim where d_year = 2002 and d_moy = 3 ) and i.i_current_price > 1.2 * (select avg(j.i_current_price) from item j where j.i_category = i.i_category) group by a.ca_state having count(*) >= 10 order by cnt, a.ca_state limit 100""", "q7" -> """ select i_item_id, avg(ss_quantity) agg1, avg(ss_list_price) agg2, avg(ss_coupon_amt) agg3, avg(ss_sales_price) agg4 from store_sales, customer_demographics, date_dim, item, promotion where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and ss_cdemo_sk = cd_demo_sk and ss_promo_sk = p_promo_sk and cd_gender = 'F' and cd_marital_status = 'W' and cd_education_status = 'College' and (p_channel_email = 'N' or p_channel_event = 'N') and d_year = 2001 group by i_item_id order by i_item_id limit 100""", "q8" -> """ select s_store_name ,sum(ss_net_profit) from store_sales ,date_dim ,store, (select ca_zip from ( SELECT substr(ca_zip,1,5) ca_zip FROM customer_address WHERE substr(ca_zip,1,5) IN ( '47602','16704','35863','28577','83910','36201', '58412','48162','28055','41419','80332', '38607','77817','24891','16226','18410', '21231','59345','13918','51089','20317', '17167','54585','67881','78366','47770', '18360','51717','73108','14440','21800', '89338','45859','65501','34948','25973', '73219','25333','17291','10374','18829', '60736','82620','41351','52094','19326', '25214','54207','40936','21814','79077', '25178','75742','77454','30621','89193', '27369','41232','48567','83041','71948', '37119','68341','14073','16891','62878', '49130','19833','24286','27700','40979', '50412','81504','94835','84844','71954', '39503','57649','18434','24987','12350', '86379','27413','44529','98569','16515', '27287','24255','21094','16005','56436', '91110','68293','56455','54558','10298', '83647','32754','27052','51766','19444', '13869','45645','94791','57631','20712', '37788','41807','46507','21727','71836', '81070','50632','88086','63991','20244', '31655','51782','29818','63792','68605', '94898','36430','57025','20601','82080', '33869','22728','35834','29086','92645', '98584','98072','11652','78093','57553', '43830','71144','53565','18700','90209', '71256','38353','54364','28571','96560', '57839','56355','50679','45266','84680', '34306','34972','48530','30106','15371', '92380','84247','92292','68852','13338', '34594','82602','70073','98069','85066', '47289','11686','98862','26217','47529', '63294','51793','35926','24227','14196', '24594','32489','99060','49472','43432', '49211','14312','88137','47369','56877', '20534','81755','15794','12318','21060', '73134','41255','63073','81003','73873', '66057','51184','51195','45676','92696', '70450','90669','98338','25264','38919', '59226','58581','60298','17895','19489', '52301','80846','95464','68770','51634', '19988','18367','18421','11618','67975', '25494','41352','95430','15734','62585', '97173','33773','10425','75675','53535', '17879','41967','12197','67998','79658', '59130','72592','14851','43933','68101', '50636','25717','71286','24660','58058', '72991','95042','15543','33122','69280', '11912','59386','27642','65177','17672', '33467','64592','36335','54010','18767', '63193','42361','49254','33113','33159', '36479','59080','11855','81963','31016', '49140','29392','41836','32958','53163', '13844','73146','23952','65148','93498', '14530','46131','58454','13376','13378', '83986','12320','17193','59852','46081', '98533','52389','13086','68843','31013', '13261','60560','13443','45533','83583', '11489','58218','19753','22911','25115', '86709','27156','32669','13123','51933', '39214','41331','66943','14155','69998', '49101','70070','35076','14242','73021', '59494','15782','29752','37914','74686', '83086','34473','15751','81084','49230', '91894','60624','17819','28810','63180', '56224','39459','55233','75752','43639', '55349','86057','62361','50788','31830', '58062','18218','85761','60083','45484', '21204','90229','70041','41162','35390', '16364','39500','68908','26689','52868', '81335','40146','11340','61527','61794', '71997','30415','59004','29450','58117', '69952','33562','83833','27385','61860', '96435','48333','23065','32961','84919', '61997','99132','22815','56600','68730', '48017','95694','32919','88217','27116', '28239','58032','18884','16791','21343', '97462','18569','75660','15475') intersect select ca_zip from (SELECT substr(ca_zip,1,5) ca_zip,count(*) cnt FROM customer_address, customer WHERE ca_address_sk = c_current_addr_sk and c_preferred_cust_flag='Y' group by ca_zip having count(*) > 10)A1)A2) V1 where ss_store_sk = s_store_sk and ss_sold_date_sk = d_date_sk and d_qoy = 2 and d_year = 1998 and (substr(s_zip,1,2) = substr(V1.ca_zip,1,2)) group by s_store_name order by s_store_name limit 100""", "q9" -> """ select case when (select count(*) from store_sales where ss_quantity between 1 and 20) > 4502397049 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 1 and 20) else (select avg(ss_net_profit) from store_sales where ss_quantity between 1 and 20) end bucket1 , case when (select count(*) from store_sales where ss_quantity between 21 and 40) > 4756228269 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 21 and 40) else (select avg(ss_net_profit) from store_sales where ss_quantity between 21 and 40) end bucket2, case when (select count(*) from store_sales where ss_quantity between 41 and 60) > 4101835064 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 41 and 60) else (select avg(ss_net_profit) from store_sales where ss_quantity between 41 and 60) end bucket3, case when (select count(*) from store_sales where ss_quantity between 61 and 80) > 4583261513 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 61 and 80) else (select avg(ss_net_profit) from store_sales where ss_quantity between 61 and 80) end bucket4, case when (select count(*) from store_sales where ss_quantity between 81 and 100) > 4208819283 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 81 and 100) else (select avg(ss_net_profit) from store_sales where ss_quantity between 81 and 100) end bucket5 from reason where r_reason_sk = 1""", "q10" -> """ select cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3, cd_dep_count, count(*) cnt4, cd_dep_employed_count, count(*) cnt5, cd_dep_college_count, count(*) cnt6 from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and ca_county in ('Grady County','Marion County','Decatur County','Lyman County','Beaver County') and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 1999 and d_moy between 2 and 2+3) and (exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 1999 and d_moy between 2 ANd 2+3) or exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 1999 and d_moy between 2 and 2+3)) group by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count order by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count limit 100""", "q11" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(ss_ext_list_price-ss_ext_discount_amt) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(ws_ext_list_price-ws_ext_discount_amt) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year ) select t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_preferred_cust_flag from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.sale_type = 's' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_w_secyear.sale_type = 'w' and t_s_firstyear.dyear = 2001 and t_s_secyear.dyear = 2001+1 and t_w_firstyear.dyear = 2001 and t_w_secyear.dyear = 2001+1 and t_s_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else 0.0 end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else 0.0 end order by t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_preferred_cust_flag limit 100""", "q12" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(ws_ext_sales_price) as itemrevenue ,sum(ws_ext_sales_price)*100/sum(sum(ws_ext_sales_price)) over (partition by i_class) as revenueratio from web_sales ,item ,date_dim where ws_item_sk = i_item_sk and i_category in ('Children', 'Jewelry', 'Music') and ws_sold_date_sk = d_date_sk and d_date between cast('2001-05-11' as date) and (cast('2001-05-11' as date) + INTERVAL 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio limit 100""", "q13" -> """ select avg(ss_quantity) ,avg(ss_ext_sales_price) ,avg(ss_ext_wholesale_cost) ,sum(ss_ext_wholesale_cost) from store_sales ,store ,customer_demographics ,household_demographics ,customer_address ,date_dim where s_store_sk = ss_store_sk and ss_sold_date_sk = d_date_sk and d_year = 2001 and((ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'M' and cd_education_status = 'Primary' and ss_sales_price between 100.00 and 150.00 and hd_dep_count = 3 )or (ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'S' and cd_education_status = '4 yr Degree' and ss_sales_price between 50.00 and 100.00 and hd_dep_count = 1 ) or (ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'W' and cd_education_status = '2 yr Degree' and ss_sales_price between 150.00 and 200.00 and hd_dep_count = 1 )) and((ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('SC', 'WY', 'TX') and ss_net_profit between 100 and 200 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('NY', 'NE', 'GA') and ss_net_profit between 150 and 300 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('AL', 'AR', 'MI') and ss_net_profit between 50 and 250 ))""", "q14a" -> """ with cross_items as (select i_item_sk ss_item_sk from item, (select iss.i_brand_id brand_id ,iss.i_class_id class_id ,iss.i_category_id category_id from store_sales ,item iss ,date_dim d1 where ss_item_sk = iss.i_item_sk and ss_sold_date_sk = d1.d_date_sk and d1.d_year between 1999 AND 1999 + 2 intersect select ics.i_brand_id ,ics.i_class_id ,ics.i_category_id from catalog_sales ,item ics ,date_dim d2 where cs_item_sk = ics.i_item_sk and cs_sold_date_sk = d2.d_date_sk and d2.d_year between 1999 AND 1999 + 2 intersect select iws.i_brand_id ,iws.i_class_id ,iws.i_category_id from web_sales ,item iws ,date_dim d3 where ws_item_sk = iws.i_item_sk and ws_sold_date_sk = d3.d_date_sk and d3.d_year between 1999 AND 1999 + 2) where i_brand_id = brand_id and i_class_id = class_id and i_category_id = category_id ), avg_sales as (select avg(quantity*list_price) average_sales from (select ss_quantity quantity ,ss_list_price list_price from store_sales ,date_dim where ss_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2 union all select cs_quantity quantity ,cs_list_price list_price from catalog_sales ,date_dim where cs_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2 union all select ws_quantity quantity ,ws_list_price list_price from web_sales ,date_dim where ws_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2) x) select channel, i_brand_id,i_class_id,i_category_id,sum(sales), sum(number_sales) from( select 'store' channel, i_brand_id,i_class_id ,i_category_id,sum(ss_quantity*ss_list_price) sales , count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1999+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales) union all select 'catalog' channel, i_brand_id,i_class_id,i_category_id, sum(cs_quantity*cs_list_price) sales, count(*) number_sales from catalog_sales ,item ,date_dim where cs_item_sk in (select ss_item_sk from cross_items) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1999+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(cs_quantity*cs_list_price) > (select average_sales from avg_sales) union all select 'web' channel, i_brand_id,i_class_id,i_category_id, sum(ws_quantity*ws_list_price) sales , count(*) number_sales from web_sales ,item ,date_dim where ws_item_sk in (select ss_item_sk from cross_items) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1999+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(ws_quantity*ws_list_price) > (select average_sales from avg_sales) ) y group by rollup (channel, i_brand_id,i_class_id,i_category_id) order by channel,i_brand_id,i_class_id,i_category_id limit 100""", "q14b" -> """ with cross_items as (select i_item_sk ss_item_sk from item, (select iss.i_brand_id brand_id ,iss.i_class_id class_id ,iss.i_category_id category_id from store_sales ,item iss ,date_dim d1 where ss_item_sk = iss.i_item_sk and ss_sold_date_sk = d1.d_date_sk and d1.d_year between 1999 AND 1999 + 2 intersect select ics.i_brand_id ,ics.i_class_id ,ics.i_category_id from catalog_sales ,item ics ,date_dim d2 where cs_item_sk = ics.i_item_sk and cs_sold_date_sk = d2.d_date_sk and d2.d_year between 1999 AND 1999 + 2 intersect select iws.i_brand_id ,iws.i_class_id ,iws.i_category_id from web_sales ,item iws ,date_dim d3 where ws_item_sk = iws.i_item_sk and ws_sold_date_sk = d3.d_date_sk and d3.d_year between 1999 AND 1999 + 2) x where i_brand_id = brand_id and i_class_id = class_id and i_category_id = category_id ), avg_sales as (select avg(quantity*list_price) average_sales from (select ss_quantity quantity ,ss_list_price list_price from store_sales ,date_dim where ss_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2 union all select cs_quantity quantity ,cs_list_price list_price from catalog_sales ,date_dim where cs_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2 union all select ws_quantity quantity ,ws_list_price list_price from web_sales ,date_dim where ws_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2) x) select this_year.channel ty_channel ,this_year.i_brand_id ty_brand ,this_year.i_class_id ty_class ,this_year.i_category_id ty_category ,this_year.sales ty_sales ,this_year.number_sales ty_number_sales ,last_year.channel ly_channel ,last_year.i_brand_id ly_brand ,last_year.i_class_id ly_class ,last_year.i_category_id ly_category ,last_year.sales ly_sales ,last_year.number_sales ly_number_sales from (select 'store' channel, i_brand_id,i_class_id,i_category_id ,sum(ss_quantity*ss_list_price) sales, count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_week_seq = (select d_week_seq from date_dim where d_year = 1999 + 1 and d_moy = 12 and d_dom = 5) group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) this_year, (select 'store' channel, i_brand_id,i_class_id ,i_category_id, sum(ss_quantity*ss_list_price) sales, count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_week_seq = (select d_week_seq from date_dim where d_year = 1999 and d_moy = 12 and d_dom = 5) group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) last_year where this_year.i_brand_id= last_year.i_brand_id and this_year.i_class_id = last_year.i_class_id and this_year.i_category_id = last_year.i_category_id order by this_year.channel, this_year.i_brand_id, this_year.i_class_id, this_year.i_category_id limit 100""", "q15" -> """ select ca_zip ,sum(cs_sales_price) from catalog_sales ,customer ,customer_address ,date_dim where cs_bill_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') or ca_state in ('CA','WA','GA') or cs_sales_price > 500) and cs_sold_date_sk = d_date_sk and d_qoy = 1 and d_year = 1998 group by ca_zip order by ca_zip limit 100""", "q16" -> """ select count(distinct cs_order_number) as `order count` ,sum(cs_ext_ship_cost) as `total shipping cost` ,sum(cs_net_profit) as `total net profit` from catalog_sales cs1 ,date_dim ,customer_address ,call_center where d_date between '2000-3-01' and (cast('2000-3-01' as date) + INTERVAL 60 days) and cs1.cs_ship_date_sk = d_date_sk and cs1.cs_ship_addr_sk = ca_address_sk and ca_state = 'IA' and cs1.cs_call_center_sk = cc_call_center_sk and cc_county in ('Luce County','Wadena County','Jefferson Davis Parish','Daviess County', 'Williamson County' ) and exists (select * from catalog_sales cs2 where cs1.cs_order_number = cs2.cs_order_number and cs1.cs_warehouse_sk <> cs2.cs_warehouse_sk) and not exists(select * from catalog_returns cr1 where cs1.cs_order_number = cr1.cr_order_number) order by count(distinct cs_order_number) limit 100""", "q17" -> """ select i_item_id ,i_item_desc ,s_state ,count(ss_quantity) as store_sales_quantitycount ,avg(ss_quantity) as store_sales_quantityave ,stddev_samp(ss_quantity) as store_sales_quantitystdev ,stddev_samp(ss_quantity)/avg(ss_quantity) as store_sales_quantitycov ,count(sr_return_quantity) as store_returns_quantitycount ,avg(sr_return_quantity) as store_returns_quantityave ,stddev_samp(sr_return_quantity) as store_returns_quantitystdev ,stddev_samp(sr_return_quantity)/avg(sr_return_quantity) as store_returns_quantitycov ,count(cs_quantity) as catalog_sales_quantitycount ,avg(cs_quantity) as catalog_sales_quantityave ,stddev_samp(cs_quantity) as catalog_sales_quantitystdev ,stddev_samp(cs_quantity)/avg(cs_quantity) as catalog_sales_quantitycov from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_quarter_name = '1999Q1' and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_quarter_name in ('1999Q1','1999Q2','1999Q3') and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_quarter_name in ('1999Q1','1999Q2','1999Q3') group by i_item_id ,i_item_desc ,s_state order by i_item_id ,i_item_desc ,s_state limit 100""", "q18" -> """ select i_item_id, ca_country, ca_state, ca_county, avg( cast(cs_quantity as decimal(12,2))) agg1, avg( cast(cs_list_price as decimal(12,2))) agg2, avg( cast(cs_coupon_amt as decimal(12,2))) agg3, avg( cast(cs_sales_price as decimal(12,2))) agg4, avg( cast(cs_net_profit as decimal(12,2))) agg5, avg( cast(c_birth_year as decimal(12,2))) agg6, avg( cast(cd1.cd_dep_count as decimal(12,2))) agg7 from catalog_sales, customer_demographics cd1, customer_demographics cd2, customer, customer_address, date_dim, item where cs_sold_date_sk = d_date_sk and cs_item_sk = i_item_sk and cs_bill_cdemo_sk = cd1.cd_demo_sk and cs_bill_customer_sk = c_customer_sk and cd1.cd_gender = 'F' and cd1.cd_education_status = 'Unknown' and c_current_cdemo_sk = cd2.cd_demo_sk and c_current_addr_sk = ca_address_sk and c_birth_month in (4,8,12,10,11,9) and d_year = 2001 and ca_state in ('AR','IA','TX' ,'KS','LA','NC','SD') group by rollup (i_item_id, ca_country, ca_state, ca_county) order by ca_country, ca_state, ca_county, i_item_id limit 100""", "q19" -> """ select i_brand_id brand_id, i_brand brand, i_manufact_id, i_manufact, sum(ss_ext_sales_price) ext_price from date_dim, store_sales, item,customer,customer_address,store where d_date_sk = ss_sold_date_sk and ss_item_sk = i_item_sk and i_manager_id=63 and d_moy=11 and d_year=2002 and ss_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and substr(ca_zip,1,5) <> substr(s_zip,1,5) and ss_store_sk = s_store_sk group by i_brand ,i_brand_id ,i_manufact_id ,i_manufact order by ext_price desc ,i_brand ,i_brand_id ,i_manufact_id ,i_manufact limit 100 """, "q20" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(cs_ext_sales_price) as itemrevenue ,sum(cs_ext_sales_price)*100/sum(sum(cs_ext_sales_price)) over (partition by i_class) as revenueratio from catalog_sales ,item ,date_dim where cs_item_sk = i_item_sk and i_category in ('Electronics', 'Children', 'Home') and cs_sold_date_sk = d_date_sk and d_date between cast('2002-03-19' as date) and (cast('2002-03-19' as date) + INTERVAL 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio limit 100""", "q21" -> """ select * from(select w_warehouse_name ,i_item_id ,sum(case when (cast(d_date as date) < cast ('1999-04-12' as date)) then inv_quantity_on_hand else 0 end) as inv_before ,sum(case when (cast(d_date as date) >= cast ('1999-04-12' as date)) then inv_quantity_on_hand else 0 end) as inv_after from inventory ,warehouse ,item ,date_dim where i_current_price between 0.99 and 1.49 and i_item_sk = inv_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_date between (cast ('1999-04-12' as date) - INTERVAL 30 days) and (cast ('1999-04-12' as date) + INTERVAL 30 days) group by w_warehouse_name, i_item_id) x where (case when inv_before > 0 then inv_after / inv_before else null end) between 2.0/3.0 and 3.0/2.0 order by w_warehouse_name ,i_item_id limit 100""", "q22" -> """ select i_product_name ,i_brand ,i_class ,i_category ,avg(inv_quantity_on_hand) qoh from inventory ,date_dim ,item where inv_date_sk=d_date_sk and inv_item_sk=i_item_sk and d_month_seq between 1188 and 1188 + 11 group by rollup(i_product_name ,i_brand ,i_class ,i_category) order by qoh, i_product_name, i_brand, i_class, i_category limit 100""", "q23a" -> """ with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt from store_sales ,date_dim ,item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and d_year in (1998,1998+1,1998+2,1998+3) group by substr(i_item_desc,1,30),i_item_sk,d_date having count(*) >4), max_store_sales as (select max(csales) tpcds_cmax from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales from store_sales ,customer ,date_dim where ss_customer_sk = c_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (1998,1998+1,1998+2,1998+3) group by c_customer_sk)), best_ss_customer as (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales from store_sales ,customer where ss_customer_sk = c_customer_sk group by c_customer_sk having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select * from max_store_sales)) select sum(sales) from (select cs_quantity*cs_list_price sales from catalog_sales ,date_dim where d_year = 1998 and d_moy = 7 and cs_sold_date_sk = d_date_sk and cs_item_sk in (select item_sk from frequent_ss_items) and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) union all select ws_quantity*ws_list_price sales from web_sales ,date_dim where d_year = 1998 and d_moy = 7 and ws_sold_date_sk = d_date_sk and ws_item_sk in (select item_sk from frequent_ss_items) and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer)) limit 100""", "q23b" -> """ with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt from store_sales ,date_dim ,item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and d_year in (1998,1998 + 1,1998 + 2,1998 + 3) group by substr(i_item_desc,1,30),i_item_sk,d_date having count(*) >4), max_store_sales as (select max(csales) tpcds_cmax from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales from store_sales ,customer ,date_dim where ss_customer_sk = c_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (1998,1998+1,1998+2,1998+3) group by c_customer_sk)), best_ss_customer as (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales from store_sales ,customer where ss_customer_sk = c_customer_sk group by c_customer_sk having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select * from max_store_sales)) select c_last_name,c_first_name,sales from (select c_last_name,c_first_name,sum(cs_quantity*cs_list_price) sales from catalog_sales ,customer ,date_dim where d_year = 1998 and d_moy = 7 and cs_sold_date_sk = d_date_sk and cs_item_sk in (select item_sk from frequent_ss_items) and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) and cs_bill_customer_sk = c_customer_sk group by c_last_name,c_first_name union all select c_last_name,c_first_name,sum(ws_quantity*ws_list_price) sales from web_sales ,customer ,date_dim where d_year = 1998 and d_moy = 7 and ws_sold_date_sk = d_date_sk and ws_item_sk in (select item_sk from frequent_ss_items) and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer) and ws_bill_customer_sk = c_customer_sk group by c_last_name,c_first_name) order by c_last_name,c_first_name,sales limit 100""", "q24a" -> """ with ssales as (select c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size ,sum(ss_sales_price) netpaid from store_sales ,store_returns ,store ,item ,customer ,customer_address where ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_customer_sk = c_customer_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and c_current_addr_sk = ca_address_sk and c_birth_country <> upper(ca_country) and s_zip = ca_zip and s_market_id=7 group by c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size) select c_last_name ,c_first_name ,s_store_name ,sum(netpaid) paid from ssales where i_color = 'goldenrod' group by c_last_name ,c_first_name ,s_store_name having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) order by c_last_name ,c_first_name ,s_store_name""", "q24b" -> """ with ssales as (select c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size ,sum(ss_sales_price) netpaid from store_sales ,store_returns ,store ,item ,customer ,customer_address where ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_customer_sk = c_customer_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and c_current_addr_sk = ca_address_sk and c_birth_country <> upper(ca_country) and s_zip = ca_zip and s_market_id = 7 group by c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size) select c_last_name ,c_first_name ,s_store_name ,sum(netpaid) paid from ssales where i_color = 'magenta' group by c_last_name ,c_first_name ,s_store_name having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) order by c_last_name ,c_first_name ,s_store_name""", "q25" -> """ select i_item_id ,i_item_desc ,s_store_id ,s_store_name ,min(ss_net_profit) as store_sales_profit ,min(sr_net_loss) as store_returns_loss ,min(cs_net_profit) as catalog_sales_profit from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_moy = 4 and d1.d_year = 2002 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_moy between 4 and 10 and d2.d_year = 2002 and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_moy between 4 and 10 and d3.d_year = 2002 group by i_item_id ,i_item_desc ,s_store_id ,s_store_name order by i_item_id ,i_item_desc ,s_store_id ,s_store_name limit 100""", "q26" -> """ select i_item_id, avg(cs_quantity) agg1, avg(cs_list_price) agg2, avg(cs_coupon_amt) agg3, avg(cs_sales_price) agg4 from catalog_sales, customer_demographics, date_dim, item, promotion where cs_sold_date_sk = d_date_sk and cs_item_sk = i_item_sk and cs_bill_cdemo_sk = cd_demo_sk and cs_promo_sk = p_promo_sk and cd_gender = 'F' and cd_marital_status = 'M' and cd_education_status = '4 yr Degree' and (p_channel_email = 'N' or p_channel_event = 'N') and d_year = 1998 group by i_item_id order by i_item_id limit 100""", "q27" -> """ select i_item_id, s_state, grouping(s_state) g_state, avg(ss_quantity) agg1, avg(ss_list_price) agg2, avg(ss_coupon_amt) agg3, avg(ss_sales_price) agg4 from store_sales, customer_demographics, date_dim, store, item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and ss_cdemo_sk = cd_demo_sk and cd_gender = 'M' and cd_marital_status = 'M' and cd_education_status = 'Secondary' and d_year = 1999 and s_state in ('AL','FL', 'TX', 'NM', 'MI', 'GA') group by rollup (i_item_id, s_state) order by i_item_id ,s_state limit 100""", "q28" -> """ select * from (select avg(ss_list_price) B1_LP ,count(ss_list_price) B1_CNT ,count(distinct ss_list_price) B1_CNTD from store_sales where ss_quantity between 0 and 5 and (ss_list_price between 74 and 74+10 or ss_coupon_amt between 2949 and 2949+1000 or ss_wholesale_cost between 49 and 49+20)) B1, (select avg(ss_list_price) B2_LP ,count(ss_list_price) B2_CNT ,count(distinct ss_list_price) B2_CNTD from store_sales where ss_quantity between 6 and 10 and (ss_list_price between 136 and 136+10 or ss_coupon_amt between 10027 and 10027+1000 or ss_wholesale_cost between 53 and 53+20)) B2, (select avg(ss_list_price) B3_LP ,count(ss_list_price) B3_CNT ,count(distinct ss_list_price) B3_CNTD from store_sales where ss_quantity between 11 and 15 and (ss_list_price between 73 and 73+10 or ss_coupon_amt between 1451 and 1451+1000 or ss_wholesale_cost between 78 and 78+20)) B3, (select avg(ss_list_price) B4_LP ,count(ss_list_price) B4_CNT ,count(distinct ss_list_price) B4_CNTD from store_sales where ss_quantity between 16 and 20 and (ss_list_price between 87 and 87+10 or ss_coupon_amt between 17007 and 17007+1000 or ss_wholesale_cost between 55 and 55+20)) B4, (select avg(ss_list_price) B5_LP ,count(ss_list_price) B5_CNT ,count(distinct ss_list_price) B5_CNTD from store_sales where ss_quantity between 21 and 25 and (ss_list_price between 112 and 112+10 or ss_coupon_amt between 17243 and 17243+1000 or ss_wholesale_cost between 2 and 2+20)) B5, (select avg(ss_list_price) B6_LP ,count(ss_list_price) B6_CNT ,count(distinct ss_list_price) B6_CNTD from store_sales where ss_quantity between 26 and 30 and (ss_list_price between 119 and 119+10 or ss_coupon_amt between 4954 and 4954+1000 or ss_wholesale_cost between 22 and 22+20)) B6 limit 100""", "q29" -> """ select i_item_id ,i_item_desc ,s_store_id ,s_store_name ,stddev_samp(ss_quantity) as store_sales_quantity ,stddev_samp(sr_return_quantity) as store_returns_quantity ,stddev_samp(cs_quantity) as catalog_sales_quantity from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_moy = 4 and d1.d_year = 2000 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_moy between 4 and 4 + 3 and d2.d_year = 2000 and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_year in (2000,2000+1,2000+2) group by i_item_id ,i_item_desc ,s_store_id ,s_store_name order by i_item_id ,i_item_desc ,s_store_id ,s_store_name limit 100""", "q30" -> """ with customer_total_return as (select wr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, sum(wr_return_amt) as ctr_total_return from web_returns ,date_dim ,customer_address where wr_returned_date_sk = d_date_sk and d_year =2001 and wr_returning_addr_sk = ca_address_sk group by wr_returning_customer_sk ,ca_state) select c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address ,c_last_review_date,ctr_total_return from customer_total_return ctr1 ,customer_address ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_state = ctr2.ctr_state) and ca_address_sk = c_current_addr_sk and ca_state = 'MI' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address ,c_last_review_date,ctr_total_return limit 100""", "q31" -> """ with ss as (select ca_county,d_qoy, d_year,sum(ss_ext_sales_price) as store_sales from store_sales,date_dim,customer_address where ss_sold_date_sk = d_date_sk and ss_addr_sk=ca_address_sk group by ca_county,d_qoy, d_year), ws as (select ca_county,d_qoy, d_year,sum(ws_ext_sales_price) as web_sales from web_sales,date_dim,customer_address where ws_sold_date_sk = d_date_sk and ws_bill_addr_sk=ca_address_sk group by ca_county,d_qoy, d_year) select ss1.ca_county ,ss1.d_year ,ws2.web_sales/ws1.web_sales web_q1_q2_increase ,ss2.store_sales/ss1.store_sales store_q1_q2_increase ,ws3.web_sales/ws2.web_sales web_q2_q3_increase ,ss3.store_sales/ss2.store_sales store_q2_q3_increase from ss ss1 ,ss ss2 ,ss ss3 ,ws ws1 ,ws ws2 ,ws ws3 where ss1.d_qoy = 1 and ss1.d_year = 2000 and ss1.ca_county = ss2.ca_county and ss2.d_qoy = 2 and ss2.d_year = 2000 and ss2.ca_county = ss3.ca_county and ss3.d_qoy = 3 and ss3.d_year = 2000 and ss1.ca_county = ws1.ca_county and ws1.d_qoy = 1 and ws1.d_year = 2000 and ws1.ca_county = ws2.ca_county and ws2.d_qoy = 2 and ws2.d_year = 2000 and ws1.ca_county = ws3.ca_county and ws3.d_qoy = 3 and ws3.d_year =2000 and case when ws1.web_sales > 0 then ws2.web_sales/ws1.web_sales else null end > case when ss1.store_sales > 0 then ss2.store_sales/ss1.store_sales else null end and case when ws2.web_sales > 0 then ws3.web_sales/ws2.web_sales else null end > case when ss2.store_sales > 0 then ss3.store_sales/ss2.store_sales else null end order by store_q1_q2_increase""", "q32" -> """ select sum(cs_ext_discount_amt) as `excess discount amount` from catalog_sales ,item ,date_dim where i_manufact_id = 490 and i_item_sk = cs_item_sk and d_date between '1999-01-27' and (cast('1999-01-27' as date) + INTERVAL 90 days) and d_date_sk = cs_sold_date_sk and cs_ext_discount_amt > ( select 1.3 * avg(cs_ext_discount_amt) from catalog_sales ,date_dim where cs_item_sk = i_item_sk and d_date between '1999-01-27' and (cast('1999-01-27' as date) + INTERVAL 90 days) and d_date_sk = cs_sold_date_sk ) limit 100""", "q33" -> """ with ss as ( select i_manufact_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Electronics')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 2001 and d_moy = 1 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_manufact_id), cs as ( select i_manufact_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Electronics')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 2001 and d_moy = 1 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_manufact_id), ws as ( select i_manufact_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Electronics')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 2001 and d_moy = 1 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_manufact_id) select i_manufact_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_manufact_id order by total_sales limit 100""", "q34" -> """ select c_last_name ,c_first_name ,c_salutation ,c_preferred_cust_flag ,ss_ticket_number ,cnt from (select ss_ticket_number ,ss_customer_sk ,count(*) cnt from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and (date_dim.d_dom between 1 and 3 or date_dim.d_dom between 25 and 28) and (household_demographics.hd_buy_potential = '1001-5000' or household_demographics.hd_buy_potential = 'Unknown') and household_demographics.hd_vehicle_count > 0 and (case when household_demographics.hd_vehicle_count > 0 then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end) > 1.2 and date_dim.d_year in (1999,1999+1,1999+2) and store.s_county in ('Nez Perce County','Murray County','Surry County','Calhoun County', 'Wilkinson County','Brown County','Wallace County','Carter County') group by ss_ticket_number,ss_customer_sk) dn,customer where ss_customer_sk = c_customer_sk and cnt between 15 and 20 order by c_last_name,c_first_name,c_salutation,c_preferred_cust_flag desc, ss_ticket_number""", "q35" -> """ select ca_state, cd_gender, cd_marital_status, cd_dep_count, count(*) cnt1, stddev_samp(cd_dep_count), sum(cd_dep_count), min(cd_dep_count), cd_dep_employed_count, count(*) cnt2, stddev_samp(cd_dep_employed_count), sum(cd_dep_employed_count), min(cd_dep_employed_count), cd_dep_college_count, count(*) cnt3, stddev_samp(cd_dep_college_count), sum(cd_dep_college_count), min(cd_dep_college_count) from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2002 and d_qoy < 4) and (exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2002 and d_qoy < 4) or exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2002 and d_qoy < 4)) group by ca_state, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count order by ca_state, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count limit 100""", "q36" -> """ select sum(ss_net_profit)/sum(ss_ext_sales_price) as gross_margin ,i_category ,i_class ,grouping(i_category)+grouping(i_class) as lochierarchy ,rank() over ( partition by grouping(i_category)+grouping(i_class), case when grouping(i_class) = 0 then i_category end order by sum(ss_net_profit)/sum(ss_ext_sales_price) asc) as rank_within_parent from store_sales ,date_dim d1 ,item ,store where d1.d_year = 2000 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and s_state in ('MN','TX','TX','IN', 'CA','LA','NM','TX') group by rollup(i_category,i_class) order by lochierarchy desc ,case when lochierarchy = 0 then i_category end ,rank_within_parent limit 100""", "q37" -> """ select i_item_id ,i_item_desc ,i_current_price from item, inventory, date_dim, catalog_sales where i_current_price between 16 and 16 + 30 and inv_item_sk = i_item_sk and d_date_sk=inv_date_sk and d_date between cast('2002-06-05' as date) and (cast('2002-06-05' as date) + interval 60 days) and i_manufact_id in (841,790,796,739) and inv_quantity_on_hand between 100 and 500 and cs_item_sk = i_item_sk group by i_item_id,i_item_desc,i_current_price order by i_item_id limit 100""", "q38" -> """ select count(*) from ( select distinct c_last_name, c_first_name, d_date from store_sales, date_dim, customer where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_customer_sk = customer.c_customer_sk and d_month_seq between 1203 and 1203 + 11 intersect select distinct c_last_name, c_first_name, d_date from catalog_sales, date_dim, customer where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1203 and 1203 + 11 intersect select distinct c_last_name, c_first_name, d_date from web_sales, date_dim, customer where web_sales.ws_sold_date_sk = date_dim.d_date_sk and web_sales.ws_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1203 and 1203 + 11 ) hot_cust limit 100""", "q39a" -> """ with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stdev,mean, case mean when 0 then null else stdev/mean end cov from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean from inventory ,item ,warehouse ,date_dim where inv_item_sk = i_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_year =1999 group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo where case mean when 0 then 0 else stdev/mean end > 1) select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov from inv inv1,inv inv2 where inv1.i_item_sk = inv2.i_item_sk and inv1.w_warehouse_sk = inv2.w_warehouse_sk and inv1.d_moy=3 and inv2.d_moy=3+1 order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov ,inv2.d_moy,inv2.mean, inv2.cov""", "q39b" -> """ with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stdev,mean, case mean when 0 then null else stdev/mean end cov from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean from inventory ,item ,warehouse ,date_dim where inv_item_sk = i_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_year =1999 group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo where case mean when 0 then 0 else stdev/mean end > 1) select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov from inv inv1,inv inv2 where inv1.i_item_sk = inv2.i_item_sk and inv1.w_warehouse_sk = inv2.w_warehouse_sk and inv1.d_moy=3 and inv2.d_moy=3+1 and inv1.cov > 1.5 order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov ,inv2.d_moy,inv2.mean, inv2.cov""", "q40" -> """ select w_state ,i_item_id ,sum(case when (cast(d_date as date) < cast ('1999-04-27' as date)) then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_before ,sum(case when (cast(d_date as date) >= cast ('1999-04-27' as date)) then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_after from catalog_sales left outer join catalog_returns on (cs_order_number = cr_order_number and cs_item_sk = cr_item_sk) ,warehouse ,item ,date_dim where i_current_price between 0.99 and 1.49 and i_item_sk = cs_item_sk and cs_warehouse_sk = w_warehouse_sk and cs_sold_date_sk = d_date_sk and d_date between (cast ('1999-04-27' as date) - INTERVAL 30 days) and (cast ('1999-04-27' as date) + INTERVAL 30 days) group by w_state,i_item_id order by w_state,i_item_id limit 100""", "q41" -> """ select distinct(i_product_name) from item i1 where i_manufact_id between 841 and 841+40 and (select count(*) as item_cnt from item where (i_manufact = i1.i_manufact and ((i_category = 'Women' and (i_color = 'bisque' or i_color = 'khaki') and (i_units = 'Carton' or i_units = 'Box') and (i_size = 'large' or i_size = 'extra large') ) or (i_category = 'Women' and (i_color = 'antique' or i_color = 'sandy') and (i_units = 'Pallet' or i_units = 'Cup') and (i_size = 'petite' or i_size = 'small') ) or (i_category = 'Men' and (i_color = 'forest' or i_color = 'brown') and (i_units = 'Dram' or i_units = 'Ton') and (i_size = 'economy' or i_size = 'medium') ) or (i_category = 'Men' and (i_color = 'chartreuse' or i_color = 'light') and (i_units = 'Pound' or i_units = 'Dozen') and (i_size = 'large' or i_size = 'extra large') ))) or (i_manufact = i1.i_manufact and ((i_category = 'Women' and (i_color = 'turquoise' or i_color = 'chocolate') and (i_units = 'Bundle' or i_units = 'Unknown') and (i_size = 'large' or i_size = 'extra large') ) or (i_category = 'Women' and (i_color = 'maroon' or i_color = 'pale') and (i_units = 'Each' or i_units = 'Tbl') and (i_size = 'petite' or i_size = 'small') ) or (i_category = 'Men' and (i_color = 'almond' or i_color = 'floral') and (i_units = 'Gross' or i_units = 'N/A') and (i_size = 'economy' or i_size = 'medium') ) or (i_category = 'Men' and (i_color = 'drab' or i_color = 'plum') and (i_units = 'Bunch' or i_units = 'Case') and (i_size = 'large' or i_size = 'extra large') )))) > 0 order by i_product_name limit 100""", "q42" -> """ select dt.d_year ,item.i_category_id ,item.i_category ,sum(ss_ext_sales_price) from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manager_id = 1 and dt.d_moy=11 and dt.d_year=2002 group by dt.d_year ,item.i_category_id ,item.i_category order by sum(ss_ext_sales_price) desc,dt.d_year ,item.i_category_id ,item.i_category limit 100 """, "q43" -> """ select s_store_name, s_store_id, sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales from date_dim, store_sales, store where d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk and s_gmt_offset = -5 and d_year = 2002 group by s_store_name, s_store_id order by s_store_name, s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales limit 100""", "q44" -> """ select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing from(select * from (select item_sk,rank() over (order by rank_col asc) rnk from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col from store_sales ss1 where ss_store_sk = 709 group by ss_item_sk having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col from store_sales where ss_store_sk = 709 and ss_addr_sk is null group by ss_store_sk))V1)V11 where rnk < 11) asceding, (select * from (select item_sk,rank() over (order by rank_col desc) rnk from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col from store_sales ss1 where ss_store_sk = 709 group by ss_item_sk having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col from store_sales where ss_store_sk = 709 and ss_addr_sk is null group by ss_store_sk))V2)V21 where rnk < 11) descending, item i1, item i2 where asceding.rnk = descending.rnk and i1.i_item_sk=asceding.item_sk and i2.i_item_sk=descending.item_sk order by asceding.rnk limit 100""", "q45" -> """ select ca_zip, ca_state, sum(ws_sales_price) from web_sales, customer, customer_address, date_dim, item where ws_bill_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and ws_item_sk = i_item_sk and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') or i_item_id in (select i_item_id from item where i_item_sk in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) ) ) and ws_sold_date_sk = d_date_sk and d_qoy = 2 and d_year = 2002 group by ca_zip, ca_state order by ca_zip, ca_state limit 100""", "q46" -> """ select c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number ,amt,profit from (select ss_ticket_number ,ss_customer_sk ,ca_city bought_city ,sum(ss_coupon_amt) amt ,sum(ss_net_profit) profit from store_sales,date_dim,store,household_demographics,customer_address where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and store_sales.ss_addr_sk = customer_address.ca_address_sk and (household_demographics.hd_dep_count = 0 or household_demographics.hd_vehicle_count= 1) and date_dim.d_dow in (6,0) and date_dim.d_year in (1999,1999+1,1999+2) and store.s_city in ('Johnson','Norwood','Cambridge','Klondike','Rock Hill') group by ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city) dn,customer,customer_address current_addr where ss_customer_sk = c_customer_sk and customer.c_current_addr_sk = current_addr.ca_address_sk and current_addr.ca_city <> bought_city order by c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number limit 100""", "q47" -> """ with v1 as( select i_category, i_brand, s_store_name, s_company_name, d_year, d_moy, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_category, i_brand, s_store_name, s_company_name, d_year) avg_monthly_sales, rank() over (partition by i_category, i_brand, s_store_name, s_company_name order by d_year, d_moy) rn from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ( d_year = 2001 or ( d_year = 2001-1 and d_moy =12) or ( d_year = 2001+1 and d_moy =1) ) group by i_category, i_brand, s_store_name, s_company_name, d_year, d_moy), v2 as( select v1.i_category, v1.i_brand, v1.s_store_name, v1.s_company_name ,v1.d_year, v1.d_moy ,v1.avg_monthly_sales ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum from v1, v1 v1_lag, v1 v1_lead where v1.i_category = v1_lag.i_category and v1.i_category = v1_lead.i_category and v1.i_brand = v1_lag.i_brand and v1.i_brand = v1_lead.i_brand and v1.s_store_name = v1_lag.s_store_name and v1.s_store_name = v1_lead.s_store_name and v1.s_company_name = v1_lag.s_company_name and v1.s_company_name = v1_lead.s_company_name and v1.rn = v1_lag.rn + 1 and v1.rn = v1_lead.rn - 1) select * from v2 where d_year = 2001 and avg_monthly_sales > 0 and case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by sum_sales - avg_monthly_sales, psum limit 100""", "q48" -> """ select sum (ss_quantity) from store_sales, store, customer_demographics, customer_address, date_dim where s_store_sk = ss_store_sk and ss_sold_date_sk = d_date_sk and d_year = 2000 and ( ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'U' and cd_education_status = '2 yr Degree' and ss_sales_price between 100.00 and 150.00 ) or ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'S' and cd_education_status = 'Primary' and ss_sales_price between 50.00 and 100.00 ) or ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'W' and cd_education_status = '4 yr Degree' and ss_sales_price between 150.00 and 200.00 ) ) and ( ( ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('MT', 'OH', 'GA') and ss_net_profit between 0 and 2000 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('WV', 'AZ', 'NM') and ss_net_profit between 150 and 3000 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('NY', 'PA', 'KY') and ss_net_profit between 50 and 25000 ) )""", "q49" -> """ select channel, item, return_ratio, return_rank, currency_rank from (select 'web' as channel ,web.item ,web.return_ratio ,web.return_rank ,web.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select ws.ws_item_sk as item ,(cast(sum(coalesce(wr.wr_return_quantity,0)) as decimal(15,4))/ cast(sum(coalesce(ws.ws_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(wr.wr_return_amt,0)) as decimal(15,4))/ cast(sum(coalesce(ws.ws_net_paid,0)) as decimal(15,4) )) as currency_ratio from web_sales ws left outer join web_returns wr on (ws.ws_order_number = wr.wr_order_number and ws.ws_item_sk = wr.wr_item_sk) ,date_dim where wr.wr_return_amt > 10000 and ws.ws_net_profit > 1 and ws.ws_net_paid > 0 and ws.ws_quantity > 0 and ws_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 11 group by ws.ws_item_sk ) in_web ) web where ( web.return_rank <= 10 or web.currency_rank <= 10 ) union select 'catalog' as channel ,catalog.item ,catalog.return_ratio ,catalog.return_rank ,catalog.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select cs.cs_item_sk as item ,(cast(sum(coalesce(cr.cr_return_quantity,0)) as decimal(15,4))/ cast(sum(coalesce(cs.cs_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(cr.cr_return_amount,0)) as decimal(15,4))/ cast(sum(coalesce(cs.cs_net_paid,0)) as decimal(15,4) )) as currency_ratio from catalog_sales cs left outer join catalog_returns cr on (cs.cs_order_number = cr.cr_order_number and cs.cs_item_sk = cr.cr_item_sk) ,date_dim where cr.cr_return_amount > 10000 and cs.cs_net_profit > 1 and cs.cs_net_paid > 0 and cs.cs_quantity > 0 and cs_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 11 group by cs.cs_item_sk ) in_cat ) catalog where ( catalog.return_rank <= 10 or catalog.currency_rank <=10 ) union select 'store' as channel ,store.item ,store.return_ratio ,store.return_rank ,store.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select sts.ss_item_sk as item ,(cast(sum(coalesce(sr.sr_return_quantity,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(sr.sr_return_amt,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_net_paid,0)) as decimal(15,4) )) as currency_ratio from store_sales sts left outer join store_returns sr on (sts.ss_ticket_number = sr.sr_ticket_number and sts.ss_item_sk = sr.sr_item_sk) ,date_dim where sr.sr_return_amt > 10000 and sts.ss_net_profit > 1 and sts.ss_net_paid > 0 and sts.ss_quantity > 0 and ss_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 11 group by sts.ss_item_sk ) in_store ) store where ( store.return_rank <= 10 or store.currency_rank <= 10 ) ) order by 1,4,5,2 limit 100""", "q50" -> """ select s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip ,sum(case when (sr_returned_date_sk - ss_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 30) and (sr_returned_date_sk - ss_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 60) and (sr_returned_date_sk - ss_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 90) and (sr_returned_date_sk - ss_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from store_sales ,store_returns ,store ,date_dim d1 ,date_dim d2 where d2.d_year = 2000 and d2.d_moy = 9 and ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_sold_date_sk = d1.d_date_sk and sr_returned_date_sk = d2.d_date_sk and ss_customer_sk = sr_customer_sk and ss_store_sk = s_store_sk group by s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip order by s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip limit 100""", "q51" -> """ WITH web_v1 as ( select ws_item_sk item_sk, d_date, sum(sum(ws_sales_price)) over (partition by ws_item_sk order by d_date rows between unbounded preceding and current row) cume_sales from web_sales ,date_dim where ws_sold_date_sk=d_date_sk and d_month_seq between 1177 and 1177+11 and ws_item_sk is not NULL group by ws_item_sk, d_date), store_v1 as ( select ss_item_sk item_sk, d_date, sum(sum(ss_sales_price)) over (partition by ss_item_sk order by d_date rows between unbounded preceding and current row) cume_sales from store_sales ,date_dim where ss_sold_date_sk=d_date_sk and d_month_seq between 1177 and 1177+11 and ss_item_sk is not NULL group by ss_item_sk, d_date) select * from (select item_sk ,d_date ,web_sales ,store_sales ,max(web_sales) over (partition by item_sk order by d_date rows between unbounded preceding and current row) web_cumulative ,max(store_sales) over (partition by item_sk order by d_date rows between unbounded preceding and current row) store_cumulative from (select case when web.item_sk is not null then web.item_sk else store.item_sk end item_sk ,case when web.d_date is not null then web.d_date else store.d_date end d_date ,web.cume_sales web_sales ,store.cume_sales store_sales from web_v1 web full outer join store_v1 store on (web.item_sk = store.item_sk and web.d_date = store.d_date) )x )y where web_cumulative > store_cumulative order by item_sk ,d_date limit 100""", "q52" -> """ select dt.d_year ,item.i_brand_id brand_id ,item.i_brand brand ,sum(ss_ext_sales_price) ext_price from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manager_id = 1 and dt.d_moy=12 and dt.d_year=2001 group by dt.d_year ,item.i_brand ,item.i_brand_id order by dt.d_year ,ext_price desc ,brand_id limit 100 """, "q53" -> """ select * from (select i_manufact_id, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_manufact_id) avg_quarterly_sales from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_month_seq in (1188,1188+1,1188+2,1188+3,1188+4,1188+5,1188+6,1188+7,1188+8,1188+9,1188+10,1188+11) and ((i_category in ('Books','Children','Electronics') and i_class in ('personal','portable','reference','self-help') and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', 'exportiunivamalg #9','scholaramalgamalg #9')) or(i_category in ('Women','Music','Men') and i_class in ('accessories','classical','fragrances','pants') and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 'importoamalg #1'))) group by i_manufact_id, d_qoy ) tmp1 where case when avg_quarterly_sales > 0 then abs (sum_sales - avg_quarterly_sales)/ avg_quarterly_sales else null end > 0.1 order by avg_quarterly_sales, sum_sales, i_manufact_id limit 100""", "q54" -> """ with my_customers as ( select distinct c_customer_sk , c_current_addr_sk from ( select cs_sold_date_sk sold_date_sk, cs_bill_customer_sk customer_sk, cs_item_sk item_sk from catalog_sales union all select ws_sold_date_sk sold_date_sk, ws_bill_customer_sk customer_sk, ws_item_sk item_sk from web_sales ) cs_or_ws_sales, item, date_dim, customer where sold_date_sk = d_date_sk and item_sk = i_item_sk and i_category = 'Men' and i_class = 'pants' and c_customer_sk = cs_or_ws_sales.customer_sk and d_moy = 5 and d_year = 2002 ) , my_revenue as ( select c_customer_sk, sum(ss_ext_sales_price) as revenue from my_customers, store_sales, customer_address, store, date_dim where c_current_addr_sk = ca_address_sk and ca_county = s_county and ca_state = s_state and ss_sold_date_sk = d_date_sk and c_customer_sk = ss_customer_sk and d_month_seq between (select distinct d_month_seq+1 from date_dim where d_year = 2002 and d_moy = 5) and (select distinct d_month_seq+3 from date_dim where d_year = 2002 and d_moy = 5) group by c_customer_sk ) , segments as (select cast((revenue/50) as int) as segment from my_revenue ) select segment, count(*) as num_customers, segment*50 as segment_base from segments group by segment order by segment, num_customers limit 100""", "q55" -> """ select i_brand_id brand_id, i_brand brand, sum(ss_ext_sales_price) ext_price from date_dim, store_sales, item where d_date_sk = ss_sold_date_sk and ss_item_sk = i_item_sk and i_manager_id=67 and d_moy=11 and d_year=2001 group by i_brand, i_brand_id order by ext_price desc, i_brand_id limit 100 """, "q56" -> """ with ss as ( select i_item_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('blanched','spring','seashell')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 6 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -7 group by i_item_id), cs as ( select i_item_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('blanched','spring','seashell')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 6 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -7 group by i_item_id), ws as ( select i_item_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('blanched','spring','seashell')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 6 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -7 group by i_item_id) select i_item_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_item_id order by total_sales, i_item_id limit 100""", "q57" -> """ with v1 as( select i_category, i_brand, cc_name, d_year, d_moy, sum(cs_sales_price) sum_sales, avg(sum(cs_sales_price)) over (partition by i_category, i_brand, cc_name, d_year) avg_monthly_sales, rank() over (partition by i_category, i_brand, cc_name order by d_year, d_moy) rn from item, catalog_sales, date_dim, call_center where cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and cc_call_center_sk= cs_call_center_sk and ( d_year = 2000 or ( d_year = 2000-1 and d_moy =12) or ( d_year = 2000+1 and d_moy =1) ) group by i_category, i_brand, cc_name , d_year, d_moy), v2 as( select v1.i_category, v1.i_brand ,v1.d_year ,v1.avg_monthly_sales ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum from v1, v1 v1_lag, v1 v1_lead where v1.i_category = v1_lag.i_category and v1.i_category = v1_lead.i_category and v1.i_brand = v1_lag.i_brand and v1.i_brand = v1_lead.i_brand and v1. cc_name = v1_lag. cc_name and v1. cc_name = v1_lead. cc_name and v1.rn = v1_lag.rn + 1 and v1.rn = v1_lead.rn - 1) select * from v2 where d_year = 2000 and avg_monthly_sales > 0 and case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by sum_sales - avg_monthly_sales, sum_sales limit 100""", "q58" -> """ with ss_items as (select i_item_id item_id ,sum(ss_ext_sales_price) ss_item_rev from store_sales ,item ,date_dim where ss_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq = (select d_week_seq from date_dim where d_date = '2000-05-24')) and ss_sold_date_sk = d_date_sk group by i_item_id), cs_items as (select i_item_id item_id ,sum(cs_ext_sales_price) cs_item_rev from catalog_sales ,item ,date_dim where cs_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq = (select d_week_seq from date_dim where d_date = '2000-05-24')) and cs_sold_date_sk = d_date_sk group by i_item_id), ws_items as (select i_item_id item_id ,sum(ws_ext_sales_price) ws_item_rev from web_sales ,item ,date_dim where ws_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq =(select d_week_seq from date_dim where d_date = '2000-05-24')) and ws_sold_date_sk = d_date_sk group by i_item_id) select ss_items.item_id ,ss_item_rev ,ss_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ss_dev ,cs_item_rev ,cs_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 cs_dev ,ws_item_rev ,ws_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ws_dev ,(ss_item_rev+cs_item_rev+ws_item_rev)/3 average from ss_items,cs_items,ws_items where ss_items.item_id=cs_items.item_id and ss_items.item_id=ws_items.item_id and ss_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev and ss_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev and cs_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev and cs_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev and ws_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev and ws_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev order by item_id ,ss_item_rev limit 100""", "q59" -> """ with wss as (select d_week_seq, ss_store_sk, sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales from store_sales,date_dim where d_date_sk = ss_sold_date_sk group by d_week_seq,ss_store_sk ) select s_store_name1,s_store_id1,d_week_seq1 ,sun_sales1/sun_sales2,mon_sales1/mon_sales2 ,tue_sales1/tue_sales2,wed_sales1/wed_sales2,thu_sales1/thu_sales2 ,fri_sales1/fri_sales2,sat_sales1/sat_sales2 from (select s_store_name s_store_name1,wss.d_week_seq d_week_seq1 ,s_store_id s_store_id1,sun_sales sun_sales1 ,mon_sales mon_sales1,tue_sales tue_sales1 ,wed_sales wed_sales1,thu_sales thu_sales1 ,fri_sales fri_sales1,sat_sales sat_sales1 from wss,store,date_dim d where d.d_week_seq = wss.d_week_seq and ss_store_sk = s_store_sk and d_month_seq between 1197 and 1197 + 11) y, (select s_store_name s_store_name2,wss.d_week_seq d_week_seq2 ,s_store_id s_store_id2,sun_sales sun_sales2 ,mon_sales mon_sales2,tue_sales tue_sales2 ,wed_sales wed_sales2,thu_sales thu_sales2 ,fri_sales fri_sales2,sat_sales sat_sales2 from wss,store,date_dim d where d.d_week_seq = wss.d_week_seq and ss_store_sk = s_store_sk and d_month_seq between 1197+ 12 and 1197 + 23) x where s_store_id1=s_store_id2 and d_week_seq1=d_week_seq2-52 order by s_store_name1,s_store_id1,d_week_seq1 limit 100""", "q60" -> """ with ss as ( select i_item_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Shoes')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 10 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id), cs as ( select i_item_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Shoes')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 10 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id), ws as ( select i_item_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Shoes')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 10 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id) select i_item_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_item_id order by i_item_id ,total_sales limit 100""", "q61" -> """ select promotions,total,cast(promotions as decimal(15,4))/cast(total as decimal(15,4))*100 from (select sum(ss_ext_sales_price) promotions from store_sales ,store ,promotion ,date_dim ,customer ,customer_address ,item where ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ss_promo_sk = p_promo_sk and ss_customer_sk= c_customer_sk and ca_address_sk = c_current_addr_sk and ss_item_sk = i_item_sk and ca_gmt_offset = -7 and i_category = 'Jewelry' and (p_channel_dmail = 'Y' or p_channel_email = 'Y' or p_channel_tv = 'Y') and s_gmt_offset = -7 and d_year = 2002 and d_moy = 11) promotional_sales, (select sum(ss_ext_sales_price) total from store_sales ,store ,date_dim ,customer ,customer_address ,item where ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ss_customer_sk= c_customer_sk and ca_address_sk = c_current_addr_sk and ss_item_sk = i_item_sk and ca_gmt_offset = -7 and i_category = 'Jewelry' and s_gmt_offset = -7 and d_year = 2002 and d_moy = 11) all_sales order by promotions, total limit 100""", "q62" -> """ select substr(w_warehouse_name,1,20) ,sm_type ,web_name ,sum(case when (ws_ship_date_sk - ws_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 30) and (ws_ship_date_sk - ws_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 60) and (ws_ship_date_sk - ws_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 90) and (ws_ship_date_sk - ws_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from web_sales ,warehouse ,ship_mode ,web_site ,date_dim where d_month_seq between 1194 and 1194 + 11 and ws_ship_date_sk = d_date_sk and ws_warehouse_sk = w_warehouse_sk and ws_ship_mode_sk = sm_ship_mode_sk and ws_web_site_sk = web_site_sk group by substr(w_warehouse_name,1,20) ,sm_type ,web_name order by substr(w_warehouse_name,1,20) ,sm_type ,web_name limit 100""", "q63" -> """ select * from (select i_manager_id ,sum(ss_sales_price) sum_sales ,avg(sum(ss_sales_price)) over (partition by i_manager_id) avg_monthly_sales from item ,store_sales ,date_dim ,store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_month_seq in (1222,1222+1,1222+2,1222+3,1222+4,1222+5,1222+6,1222+7,1222+8,1222+9,1222+10,1222+11) and (( i_category in ('Books','Children','Electronics') and i_class in ('personal','portable','reference','self-help') and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', 'exportiunivamalg #9','scholaramalgamalg #9')) or( i_category in ('Women','Music','Men') and i_class in ('accessories','classical','fragrances','pants') and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 'importoamalg #1'))) group by i_manager_id, d_moy) tmp1 where case when avg_monthly_sales > 0 then abs (sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by i_manager_id ,avg_monthly_sales ,sum_sales limit 100""", "q64" -> """ with cs_ui as (select cs_item_sk ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund from catalog_sales ,catalog_returns where cs_item_sk = cr_item_sk and cs_order_number = cr_order_number group by cs_item_sk having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), cross_sales as (select i_product_name product_name ,i_item_sk item_sk ,s_store_name store_name ,s_zip store_zip ,ad1.ca_street_number b_street_number ,ad1.ca_street_name b_street_name ,ad1.ca_city b_city ,ad1.ca_zip b_zip ,ad2.ca_street_number c_street_number ,ad2.ca_street_name c_street_name ,ad2.ca_city c_city ,ad2.ca_zip c_zip ,d1.d_year as syear ,d2.d_year as fsyear ,d3.d_year s2year ,count(*) cnt ,sum(ss_wholesale_cost) s1 ,sum(ss_list_price) s2 ,sum(ss_coupon_amt) s3 FROM store_sales ,store_returns ,cs_ui ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,customer ,customer_demographics cd1 ,customer_demographics cd2 ,promotion ,household_demographics hd1 ,household_demographics hd2 ,customer_address ad1 ,customer_address ad2 ,income_band ib1 ,income_band ib2 ,item WHERE ss_store_sk = s_store_sk AND ss_sold_date_sk = d1.d_date_sk AND ss_customer_sk = c_customer_sk AND ss_cdemo_sk= cd1.cd_demo_sk AND ss_hdemo_sk = hd1.hd_demo_sk AND ss_addr_sk = ad1.ca_address_sk and ss_item_sk = i_item_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and ss_item_sk = cs_ui.cs_item_sk and c_current_cdemo_sk = cd2.cd_demo_sk AND c_current_hdemo_sk = hd2.hd_demo_sk AND c_current_addr_sk = ad2.ca_address_sk and c_first_sales_date_sk = d2.d_date_sk and c_first_shipto_date_sk = d3.d_date_sk and ss_promo_sk = p_promo_sk and hd1.hd_income_band_sk = ib1.ib_income_band_sk and hd2.hd_income_band_sk = ib2.ib_income_band_sk and cd1.cd_marital_status <> cd2.cd_marital_status and i_color in ('ivory','purple','almond','bisque','lawn','azure') and i_current_price between 60 and 60 + 10 and i_current_price between 60 + 1 and 60 + 15 group by i_product_name ,i_item_sk ,s_store_name ,s_zip ,ad1.ca_street_number ,ad1.ca_street_name ,ad1.ca_city ,ad1.ca_zip ,ad2.ca_street_number ,ad2.ca_street_name ,ad2.ca_city ,ad2.ca_zip ,d1.d_year ,d2.d_year ,d3.d_year ) select cs1.product_name ,cs1.store_name ,cs1.store_zip ,cs1.b_street_number ,cs1.b_street_name ,cs1.b_city ,cs1.b_zip ,cs1.c_street_number ,cs1.c_street_name ,cs1.c_city ,cs1.c_zip ,cs1.syear ,cs1.cnt ,cs1.s1 as s11 ,cs1.s2 as s21 ,cs1.s3 as s31 ,cs2.s1 as s12 ,cs2.s2 as s22 ,cs2.s3 as s32 ,cs2.syear ,cs2.cnt from cross_sales cs1,cross_sales cs2 where cs1.item_sk=cs2.item_sk and cs1.syear = 2001 and cs2.syear = 2001 + 1 and cs2.cnt <= cs1.cnt and cs1.store_name = cs2.store_name and cs1.store_zip = cs2.store_zip order by cs1.product_name ,cs1.store_name ,cs2.cnt ,cs1.s1 ,cs2.s1""", "q65" -> """ select s_store_name, i_item_desc, sc.revenue, i_current_price, i_wholesale_cost, i_brand from store, item, (select ss_store_sk, avg(revenue) as ave from (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue from store_sales, date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1185 and 1185+11 group by ss_store_sk, ss_item_sk) sa group by ss_store_sk) sb, (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue from store_sales, date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1185 and 1185+11 group by ss_store_sk, ss_item_sk) sc where sb.ss_store_sk = sc.ss_store_sk and sc.revenue <= 0.1 * sb.ave and s_store_sk = sc.ss_store_sk and i_item_sk = sc.ss_item_sk order by s_store_name, i_item_desc limit 100""", "q66" -> """ select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,ship_carriers ,year ,sum(jan_sales) as jan_sales ,sum(feb_sales) as feb_sales ,sum(mar_sales) as mar_sales ,sum(apr_sales) as apr_sales ,sum(may_sales) as may_sales ,sum(jun_sales) as jun_sales ,sum(jul_sales) as jul_sales ,sum(aug_sales) as aug_sales ,sum(sep_sales) as sep_sales ,sum(oct_sales) as oct_sales ,sum(nov_sales) as nov_sales ,sum(dec_sales) as dec_sales ,sum(jan_sales/w_warehouse_sq_ft) as jan_sales_per_sq_foot ,sum(feb_sales/w_warehouse_sq_ft) as feb_sales_per_sq_foot ,sum(mar_sales/w_warehouse_sq_ft) as mar_sales_per_sq_foot ,sum(apr_sales/w_warehouse_sq_ft) as apr_sales_per_sq_foot ,sum(may_sales/w_warehouse_sq_ft) as may_sales_per_sq_foot ,sum(jun_sales/w_warehouse_sq_ft) as jun_sales_per_sq_foot ,sum(jul_sales/w_warehouse_sq_ft) as jul_sales_per_sq_foot ,sum(aug_sales/w_warehouse_sq_ft) as aug_sales_per_sq_foot ,sum(sep_sales/w_warehouse_sq_ft) as sep_sales_per_sq_foot ,sum(oct_sales/w_warehouse_sq_ft) as oct_sales_per_sq_foot ,sum(nov_sales/w_warehouse_sq_ft) as nov_sales_per_sq_foot ,sum(dec_sales/w_warehouse_sq_ft) as dec_sales_per_sq_foot ,sum(jan_net) as jan_net ,sum(feb_net) as feb_net ,sum(mar_net) as mar_net ,sum(apr_net) as apr_net ,sum(may_net) as may_net ,sum(jun_net) as jun_net ,sum(jul_net) as jul_net ,sum(aug_net) as aug_net ,sum(sep_net) as sep_net ,sum(oct_net) as oct_net ,sum(nov_net) as nov_net ,sum(dec_net) as dec_net from ( select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,'FEDEX' || ',' || 'MSC' as ship_carriers ,d_year as year ,sum(case when d_moy = 1 then ws_ext_list_price* ws_quantity else 0 end) as jan_sales ,sum(case when d_moy = 2 then ws_ext_list_price* ws_quantity else 0 end) as feb_sales ,sum(case when d_moy = 3 then ws_ext_list_price* ws_quantity else 0 end) as mar_sales ,sum(case when d_moy = 4 then ws_ext_list_price* ws_quantity else 0 end) as apr_sales ,sum(case when d_moy = 5 then ws_ext_list_price* ws_quantity else 0 end) as may_sales ,sum(case when d_moy = 6 then ws_ext_list_price* ws_quantity else 0 end) as jun_sales ,sum(case when d_moy = 7 then ws_ext_list_price* ws_quantity else 0 end) as jul_sales ,sum(case when d_moy = 8 then ws_ext_list_price* ws_quantity else 0 end) as aug_sales ,sum(case when d_moy = 9 then ws_ext_list_price* ws_quantity else 0 end) as sep_sales ,sum(case when d_moy = 10 then ws_ext_list_price* ws_quantity else 0 end) as oct_sales ,sum(case when d_moy = 11 then ws_ext_list_price* ws_quantity else 0 end) as nov_sales ,sum(case when d_moy = 12 then ws_ext_list_price* ws_quantity else 0 end) as dec_sales ,sum(case when d_moy = 1 then ws_net_profit * ws_quantity else 0 end) as jan_net ,sum(case when d_moy = 2 then ws_net_profit * ws_quantity else 0 end) as feb_net ,sum(case when d_moy = 3 then ws_net_profit * ws_quantity else 0 end) as mar_net ,sum(case when d_moy = 4 then ws_net_profit * ws_quantity else 0 end) as apr_net ,sum(case when d_moy = 5 then ws_net_profit * ws_quantity else 0 end) as may_net ,sum(case when d_moy = 6 then ws_net_profit * ws_quantity else 0 end) as jun_net ,sum(case when d_moy = 7 then ws_net_profit * ws_quantity else 0 end) as jul_net ,sum(case when d_moy = 8 then ws_net_profit * ws_quantity else 0 end) as aug_net ,sum(case when d_moy = 9 then ws_net_profit * ws_quantity else 0 end) as sep_net ,sum(case when d_moy = 10 then ws_net_profit * ws_quantity else 0 end) as oct_net ,sum(case when d_moy = 11 then ws_net_profit * ws_quantity else 0 end) as nov_net ,sum(case when d_moy = 12 then ws_net_profit * ws_quantity else 0 end) as dec_net from web_sales ,warehouse ,date_dim ,time_dim ,ship_mode where ws_warehouse_sk = w_warehouse_sk and ws_sold_date_sk = d_date_sk and ws_sold_time_sk = t_time_sk and ws_ship_mode_sk = sm_ship_mode_sk and d_year = 2002 and t_time between 2662 and 2662+28800 and sm_carrier in ('FEDEX','MSC') group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,d_year union all select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,'FEDEX' || ',' || 'MSC' as ship_carriers ,d_year as year ,sum(case when d_moy = 1 then cs_ext_list_price* cs_quantity else 0 end) as jan_sales ,sum(case when d_moy = 2 then cs_ext_list_price* cs_quantity else 0 end) as feb_sales ,sum(case when d_moy = 3 then cs_ext_list_price* cs_quantity else 0 end) as mar_sales ,sum(case when d_moy = 4 then cs_ext_list_price* cs_quantity else 0 end) as apr_sales ,sum(case when d_moy = 5 then cs_ext_list_price* cs_quantity else 0 end) as may_sales ,sum(case when d_moy = 6 then cs_ext_list_price* cs_quantity else 0 end) as jun_sales ,sum(case when d_moy = 7 then cs_ext_list_price* cs_quantity else 0 end) as jul_sales ,sum(case when d_moy = 8 then cs_ext_list_price* cs_quantity else 0 end) as aug_sales ,sum(case when d_moy = 9 then cs_ext_list_price* cs_quantity else 0 end) as sep_sales ,sum(case when d_moy = 10 then cs_ext_list_price* cs_quantity else 0 end) as oct_sales ,sum(case when d_moy = 11 then cs_ext_list_price* cs_quantity else 0 end) as nov_sales ,sum(case when d_moy = 12 then cs_ext_list_price* cs_quantity else 0 end) as dec_sales ,sum(case when d_moy = 1 then cs_net_profit * cs_quantity else 0 end) as jan_net ,sum(case when d_moy = 2 then cs_net_profit * cs_quantity else 0 end) as feb_net ,sum(case when d_moy = 3 then cs_net_profit * cs_quantity else 0 end) as mar_net ,sum(case when d_moy = 4 then cs_net_profit * cs_quantity else 0 end) as apr_net ,sum(case when d_moy = 5 then cs_net_profit * cs_quantity else 0 end) as may_net ,sum(case when d_moy = 6 then cs_net_profit * cs_quantity else 0 end) as jun_net ,sum(case when d_moy = 7 then cs_net_profit * cs_quantity else 0 end) as jul_net ,sum(case when d_moy = 8 then cs_net_profit * cs_quantity else 0 end) as aug_net ,sum(case when d_moy = 9 then cs_net_profit * cs_quantity else 0 end) as sep_net ,sum(case when d_moy = 10 then cs_net_profit * cs_quantity else 0 end) as oct_net ,sum(case when d_moy = 11 then cs_net_profit * cs_quantity else 0 end) as nov_net ,sum(case when d_moy = 12 then cs_net_profit * cs_quantity else 0 end) as dec_net from catalog_sales ,warehouse ,date_dim ,time_dim ,ship_mode where cs_warehouse_sk = w_warehouse_sk and cs_sold_date_sk = d_date_sk and cs_sold_time_sk = t_time_sk and cs_ship_mode_sk = sm_ship_mode_sk and d_year = 2002 and t_time between 2662 AND 2662+28800 and sm_carrier in ('FEDEX','MSC') group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,d_year ) x group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,ship_carriers ,year order by w_warehouse_name limit 100""", "q67" -> """ select * from (select i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sumsales ,rank() over (partition by i_category order by sumsales desc) rk from (select i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sum(coalesce(ss_sales_price*ss_quantity,0)) sumsales from store_sales ,date_dim ,store ,item where ss_sold_date_sk=d_date_sk and ss_item_sk=i_item_sk and ss_store_sk = s_store_sk and d_month_seq between 1177 and 1177+11 group by rollup(i_category, i_class, i_brand, i_product_name, d_year, d_qoy, d_moy,s_store_id))dw1) dw2 where rk <= 100 order by i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sumsales ,rk limit 100""", "q68" -> """ select c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number ,extended_price ,extended_tax ,list_price from (select ss_ticket_number ,ss_customer_sk ,ca_city bought_city ,sum(ss_ext_sales_price) extended_price ,sum(ss_ext_list_price) list_price ,sum(ss_ext_tax) extended_tax from store_sales ,date_dim ,store ,household_demographics ,customer_address where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and store_sales.ss_addr_sk = customer_address.ca_address_sk and date_dim.d_dom between 1 and 2 and (household_demographics.hd_dep_count = 5 or household_demographics.hd_vehicle_count= 4) and date_dim.d_year in (1999,1999+1,1999+2) and store.s_city in ('Lodi','Richmond') group by ss_ticket_number ,ss_customer_sk ,ss_addr_sk,ca_city) dn ,customer ,customer_address current_addr where ss_customer_sk = c_customer_sk and customer.c_current_addr_sk = current_addr.ca_address_sk and current_addr.ca_city <> bought_city order by c_last_name ,ss_ticket_number limit 100""", "q69" -> """ select cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3 from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and ca_state in ('IL','FL','SD') and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 1999 and d_moy between 1 and 1+2) and (not exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 1999 and d_moy between 1 and 1+2) and not exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 1999 and d_moy between 1 and 1+2)) group by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating order by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating limit 100""", "q70" -> """ select sum(ss_net_profit) as total_sum ,s_state ,s_county ,grouping(s_state)+grouping(s_county) as lochierarchy ,rank() over ( partition by grouping(s_state)+grouping(s_county), case when grouping(s_county) = 0 then s_state end order by sum(ss_net_profit) desc) as rank_within_parent from store_sales ,date_dim d1 ,store where d1.d_month_seq between 1206 and 1206+11 and d1.d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk and s_state in ( select s_state from (select s_state as s_state, rank() over ( partition by s_state order by sum(ss_net_profit) desc) as ranking from store_sales, store, date_dim where d_month_seq between 1206 and 1206+11 and d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk group by s_state ) tmp1 where ranking <= 5 ) group by rollup(s_state,s_county) order by lochierarchy desc ,case when lochierarchy = 0 then s_state end ,rank_within_parent limit 100""", "q71" -> """ select i_brand_id brand_id, i_brand brand,t_hour,t_minute, sum(ext_price) ext_price from item, (select ws_ext_sales_price as ext_price, ws_sold_date_sk as sold_date_sk, ws_item_sk as sold_item_sk, ws_sold_time_sk as time_sk from web_sales,date_dim where d_date_sk = ws_sold_date_sk and d_moy=11 and d_year=1999 union all select cs_ext_sales_price as ext_price, cs_sold_date_sk as sold_date_sk, cs_item_sk as sold_item_sk, cs_sold_time_sk as time_sk from catalog_sales,date_dim where d_date_sk = cs_sold_date_sk and d_moy=11 and d_year=1999 union all select ss_ext_sales_price as ext_price, ss_sold_date_sk as sold_date_sk, ss_item_sk as sold_item_sk, ss_sold_time_sk as time_sk from store_sales,date_dim where d_date_sk = ss_sold_date_sk and d_moy=11 and d_year=1999 ) tmp,time_dim where sold_item_sk = i_item_sk and i_manager_id=1 and time_sk = t_time_sk and (t_meal_time = 'breakfast' or t_meal_time = 'dinner') group by i_brand, i_brand_id,t_hour,t_minute order by ext_price desc, i_brand_id """, "q72" -> """ select i_item_desc ,w_warehouse_name ,d1.d_week_seq ,sum(case when p_promo_sk is null then 1 else 0 end) no_promo ,sum(case when p_promo_sk is not null then 1 else 0 end) promo ,count(*) total_cnt from catalog_sales join inventory on (cs_item_sk = inv_item_sk) join warehouse on (w_warehouse_sk=inv_warehouse_sk) join item on (i_item_sk = cs_item_sk) join customer_demographics on (cs_bill_cdemo_sk = cd_demo_sk) join household_demographics on (cs_bill_hdemo_sk = hd_demo_sk) join date_dim d1 on (cs_sold_date_sk = d1.d_date_sk) join date_dim d2 on (inv_date_sk = d2.d_date_sk) join date_dim d3 on (cs_ship_date_sk = d3.d_date_sk) left outer join promotion on (cs_promo_sk=p_promo_sk) left outer join catalog_returns on (cr_item_sk = cs_item_sk and cr_order_number = cs_order_number) where d1.d_week_seq = d2.d_week_seq and inv_quantity_on_hand < cs_quantity and d3.d_date > d1.d_date + interval 5 days and hd_buy_potential = '1001-5000' and d1.d_year = 2000 and cd_marital_status = 'S' group by i_item_desc,w_warehouse_name,d1.d_week_seq order by total_cnt desc, i_item_desc, w_warehouse_name, d_week_seq limit 100""", "q73" -> """ select c_last_name ,c_first_name ,c_salutation ,c_preferred_cust_flag ,ss_ticket_number ,cnt from (select ss_ticket_number ,ss_customer_sk ,count(*) cnt from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and date_dim.d_dom between 1 and 2 and (household_demographics.hd_buy_potential = '1001-5000' or household_demographics.hd_buy_potential = 'Unknown') and household_demographics.hd_vehicle_count > 0 and case when household_demographics.hd_vehicle_count > 0 then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end > 1 and date_dim.d_year in (1999,1999+1,1999+2) and store.s_county in ('Humboldt County','Hickman County','Galax city','Abbeville County') group by ss_ticket_number,ss_customer_sk) dj,customer where ss_customer_sk = c_customer_sk and cnt between 1 and 5 order by cnt desc, c_last_name asc""", "q74" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,d_year as year ,max(ss_net_paid) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (2001,2001+1) group by c_customer_id ,c_first_name ,c_last_name ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,d_year as year ,max(ws_net_paid) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year in (2001,2001+1) group by c_customer_id ,c_first_name ,c_last_name ,d_year ) select t_s_secyear.customer_id, t_s_secyear.customer_first_name, t_s_secyear.customer_last_name from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.sale_type = 's' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_w_secyear.sale_type = 'w' and t_s_firstyear.year = 2001 and t_s_secyear.year = 2001+1 and t_w_firstyear.year = 2001 and t_w_secyear.year = 2001+1 and t_s_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end order by 3,1,2 limit 100""", "q75" -> """ WITH all_sales AS ( SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,SUM(sales_cnt) AS sales_cnt ,SUM(sales_amt) AS sales_amt FROM (SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,cs_quantity - COALESCE(cr_return_quantity,0) AS sales_cnt ,cs_ext_sales_price - COALESCE(cr_return_amount,0.0) AS sales_amt FROM catalog_sales JOIN item ON i_item_sk=cs_item_sk JOIN date_dim ON d_date_sk=cs_sold_date_sk LEFT JOIN catalog_returns ON (cs_order_number=cr_order_number AND cs_item_sk=cr_item_sk) WHERE i_category='Books' UNION SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,ss_quantity - COALESCE(sr_return_quantity,0) AS sales_cnt ,ss_ext_sales_price - COALESCE(sr_return_amt,0.0) AS sales_amt FROM store_sales JOIN item ON i_item_sk=ss_item_sk JOIN date_dim ON d_date_sk=ss_sold_date_sk LEFT JOIN store_returns ON (ss_ticket_number=sr_ticket_number AND ss_item_sk=sr_item_sk) WHERE i_category='Books' UNION SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,ws_quantity - COALESCE(wr_return_quantity,0) AS sales_cnt ,ws_ext_sales_price - COALESCE(wr_return_amt,0.0) AS sales_amt FROM web_sales JOIN item ON i_item_sk=ws_item_sk JOIN date_dim ON d_date_sk=ws_sold_date_sk LEFT JOIN web_returns ON (ws_order_number=wr_order_number AND ws_item_sk=wr_item_sk) WHERE i_category='Books') sales_detail GROUP BY d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id) SELECT prev_yr.d_year AS prev_year ,curr_yr.d_year AS year ,curr_yr.i_brand_id ,curr_yr.i_class_id ,curr_yr.i_category_id ,curr_yr.i_manufact_id ,prev_yr.sales_cnt AS prev_yr_cnt ,curr_yr.sales_cnt AS curr_yr_cnt ,curr_yr.sales_cnt-prev_yr.sales_cnt AS sales_cnt_diff ,curr_yr.sales_amt-prev_yr.sales_amt AS sales_amt_diff FROM all_sales curr_yr, all_sales prev_yr WHERE curr_yr.i_brand_id=prev_yr.i_brand_id AND curr_yr.i_class_id=prev_yr.i_class_id AND curr_yr.i_category_id=prev_yr.i_category_id AND curr_yr.i_manufact_id=prev_yr.i_manufact_id AND curr_yr.d_year=2001 AND prev_yr.d_year=2001-1 AND CAST(curr_yr.sales_cnt AS DECIMAL(17,2))/CAST(prev_yr.sales_cnt AS DECIMAL(17,2))<0.9 ORDER BY sales_cnt_diff,sales_amt_diff limit 100""", "q76" -> """ select channel, col_name, d_year, d_qoy, i_category, COUNT(*) sales_cnt, SUM(ext_sales_price) sales_amt FROM ( SELECT 'store' as channel, 'ss_promo_sk' col_name, d_year, d_qoy, i_category, ss_ext_sales_price ext_sales_price FROM store_sales, item, date_dim WHERE ss_promo_sk IS NULL AND ss_sold_date_sk=d_date_sk AND ss_item_sk=i_item_sk UNION ALL SELECT 'web' as channel, 'ws_ship_addr_sk' col_name, d_year, d_qoy, i_category, ws_ext_sales_price ext_sales_price FROM web_sales, item, date_dim WHERE ws_ship_addr_sk IS NULL AND ws_sold_date_sk=d_date_sk AND ws_item_sk=i_item_sk UNION ALL SELECT 'catalog' as channel, 'cs_ship_customer_sk' col_name, d_year, d_qoy, i_category, cs_ext_sales_price ext_sales_price FROM catalog_sales, item, date_dim WHERE cs_ship_customer_sk IS NULL AND cs_sold_date_sk=d_date_sk AND cs_item_sk=i_item_sk) foo GROUP BY channel, col_name, d_year, d_qoy, i_category ORDER BY channel, col_name, d_year, d_qoy, i_category limit 100""", "q77" -> """ with ss as (select s_store_sk, sum(ss_ext_sales_price) as sales, sum(ss_net_profit) as profit from store_sales, date_dim, store where ss_sold_date_sk = d_date_sk and d_date between cast('2001-08-16' as date) and (cast('2001-08-16' as date) + INTERVAL 30 days) and ss_store_sk = s_store_sk group by s_store_sk) , sr as (select s_store_sk, sum(sr_return_amt) as returns, sum(sr_net_loss) as profit_loss from store_returns, date_dim, store where sr_returned_date_sk = d_date_sk and d_date between cast('2001-08-16' as date) and (cast('2001-08-16' as date) + INTERVAL 30 days) and sr_store_sk = s_store_sk group by s_store_sk), cs as (select cs_call_center_sk, sum(cs_ext_sales_price) as sales, sum(cs_net_profit) as profit from catalog_sales, date_dim where cs_sold_date_sk = d_date_sk and d_date between cast('2001-08-16' as date) and (cast('2001-08-16' as date) + INTERVAL 30 days) group by cs_call_center_sk ), cr as (select cr_call_center_sk, sum(cr_return_amount) as returns, sum(cr_net_loss) as profit_loss from catalog_returns, date_dim where cr_returned_date_sk = d_date_sk and d_date between cast('2001-08-16' as date) and (cast('2001-08-16' as date) + INTERVAL 30 days) group by cr_call_center_sk ), ws as ( select wp_web_page_sk, sum(ws_ext_sales_price) as sales, sum(ws_net_profit) as profit from web_sales, date_dim, web_page where ws_sold_date_sk = d_date_sk and d_date between cast('2001-08-16' as date) and (cast('2001-08-16' as date) + INTERVAL 30 days) and ws_web_page_sk = wp_web_page_sk group by wp_web_page_sk), wr as (select wp_web_page_sk, sum(wr_return_amt) as returns, sum(wr_net_loss) as profit_loss from web_returns, date_dim, web_page where wr_returned_date_sk = d_date_sk and d_date between cast('2001-08-16' as date) and (cast('2001-08-16' as date) + INTERVAL 30 days) and wr_web_page_sk = wp_web_page_sk group by wp_web_page_sk) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , ss.s_store_sk as id , sales , coalesce(returns, 0) as returns , (profit - coalesce(profit_loss,0)) as profit from ss left join sr on ss.s_store_sk = sr.s_store_sk union all select 'catalog channel' as channel , cs_call_center_sk as id , sales , returns , (profit - profit_loss) as profit from cs , cr union all select 'web channel' as channel , ws.wp_web_page_sk as id , sales , coalesce(returns, 0) returns , (profit - coalesce(profit_loss,0)) as profit from ws left join wr on ws.wp_web_page_sk = wr.wp_web_page_sk ) x group by rollup (channel, id) order by channel ,id limit 100""", "q78" -> """ with ws as (select d_year AS ws_sold_year, ws_item_sk, ws_bill_customer_sk ws_customer_sk, sum(ws_quantity) ws_qty, sum(ws_wholesale_cost) ws_wc, sum(ws_sales_price) ws_sp from web_sales left join web_returns on wr_order_number=ws_order_number and ws_item_sk=wr_item_sk join date_dim on ws_sold_date_sk = d_date_sk where wr_order_number is null group by d_year, ws_item_sk, ws_bill_customer_sk ), cs as (select d_year AS cs_sold_year, cs_item_sk, cs_bill_customer_sk cs_customer_sk, sum(cs_quantity) cs_qty, sum(cs_wholesale_cost) cs_wc, sum(cs_sales_price) cs_sp from catalog_sales left join catalog_returns on cr_order_number=cs_order_number and cs_item_sk=cr_item_sk join date_dim on cs_sold_date_sk = d_date_sk where cr_order_number is null group by d_year, cs_item_sk, cs_bill_customer_sk ), ss as (select d_year AS ss_sold_year, ss_item_sk, ss_customer_sk, sum(ss_quantity) ss_qty, sum(ss_wholesale_cost) ss_wc, sum(ss_sales_price) ss_sp from store_sales left join store_returns on sr_ticket_number=ss_ticket_number and ss_item_sk=sr_item_sk join date_dim on ss_sold_date_sk = d_date_sk where sr_ticket_number is null group by d_year, ss_item_sk, ss_customer_sk ) select ss_item_sk, round(ss_qty/(coalesce(ws_qty,0)+coalesce(cs_qty,0)),2) ratio, ss_qty store_qty, ss_wc store_wholesale_cost, ss_sp store_sales_price, coalesce(ws_qty,0)+coalesce(cs_qty,0) other_chan_qty, coalesce(ws_wc,0)+coalesce(cs_wc,0) other_chan_wholesale_cost, coalesce(ws_sp,0)+coalesce(cs_sp,0) other_chan_sales_price from ss left join ws on (ws_sold_year=ss_sold_year and ws_item_sk=ss_item_sk and ws_customer_sk=ss_customer_sk) left join cs on (cs_sold_year=ss_sold_year and cs_item_sk=ss_item_sk and cs_customer_sk=ss_customer_sk) where (coalesce(ws_qty,0)>0 or coalesce(cs_qty, 0)>0) and ss_sold_year=2000 order by ss_item_sk, ss_qty desc, ss_wc desc, ss_sp desc, other_chan_qty, other_chan_wholesale_cost, other_chan_sales_price, ratio limit 100""", "q79" -> """ select c_last_name,c_first_name,substr(s_city,1,30),ss_ticket_number,amt,profit from (select ss_ticket_number ,ss_customer_sk ,store.s_city ,sum(ss_coupon_amt) amt ,sum(ss_net_profit) profit from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and (household_demographics.hd_dep_count = 5 or household_demographics.hd_vehicle_count > -1) and date_dim.d_dow = 1 and date_dim.d_year in (1999,1999+1,1999+2) and store.s_number_employees between 200 and 295 group by ss_ticket_number,ss_customer_sk,ss_addr_sk,store.s_city) ms,customer where ss_customer_sk = c_customer_sk order by c_last_name,c_first_name,substr(s_city,1,30), profit limit 100""", "q80" -> """ with ssr as (select s_store_id as store_id, sum(ss_ext_sales_price) as sales, sum(coalesce(sr_return_amt, 0)) as returns, sum(ss_net_profit - coalesce(sr_net_loss, 0)) as profit from store_sales left outer join store_returns on (ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number), date_dim, store, item, promotion where ss_sold_date_sk = d_date_sk and d_date between cast('2001-08-19' as date) and (cast('2001-08-19' as date) + INTERVAL 60 days) and ss_store_sk = s_store_sk and ss_item_sk = i_item_sk and i_current_price > 50 and ss_promo_sk = p_promo_sk and p_channel_tv = 'N' group by s_store_id) , csr as (select cp_catalog_page_id as catalog_page_id, sum(cs_ext_sales_price) as sales, sum(coalesce(cr_return_amount, 0)) as returns, sum(cs_net_profit - coalesce(cr_net_loss, 0)) as profit from catalog_sales left outer join catalog_returns on (cs_item_sk = cr_item_sk and cs_order_number = cr_order_number), date_dim, catalog_page, item, promotion where cs_sold_date_sk = d_date_sk and d_date between cast('2001-08-19' as date) and (cast('2001-08-19' as date) + INTERVAL 60 days) and cs_catalog_page_sk = cp_catalog_page_sk and cs_item_sk = i_item_sk and i_current_price > 50 and cs_promo_sk = p_promo_sk and p_channel_tv = 'N' group by cp_catalog_page_id) , wsr as (select web_site_id, sum(ws_ext_sales_price) as sales, sum(coalesce(wr_return_amt, 0)) as returns, sum(ws_net_profit - coalesce(wr_net_loss, 0)) as profit from web_sales left outer join web_returns on (ws_item_sk = wr_item_sk and ws_order_number = wr_order_number), date_dim, web_site, item, promotion where ws_sold_date_sk = d_date_sk and d_date between cast('2001-08-19' as date) and (cast('2001-08-19' as date) + INTERVAL 60 days) and ws_web_site_sk = web_site_sk and ws_item_sk = i_item_sk and i_current_price > 50 and ws_promo_sk = p_promo_sk and p_channel_tv = 'N' group by web_site_id) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , 'store' || store_id as id , sales , returns , profit from ssr union all select 'catalog channel' as channel , 'catalog_page' || catalog_page_id as id , sales , returns , profit from csr union all select 'web channel' as channel , 'web_site' || web_site_id as id , sales , returns , profit from wsr ) x group by rollup (channel, id) order by channel ,id limit 100""", "q81" -> """ with customer_total_return as (select cr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, sum(cr_return_amt_inc_tax) as ctr_total_return from catalog_returns ,date_dim ,customer_address where cr_returned_date_sk = d_date_sk and d_year =1999 and cr_returning_addr_sk = ca_address_sk group by cr_returning_customer_sk ,ca_state ) select c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset ,ca_location_type,ctr_total_return from customer_total_return ctr1 ,customer_address ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_state = ctr2.ctr_state) and ca_address_sk = c_current_addr_sk and ca_state = 'MO' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset ,ca_location_type,ctr_total_return limit 100""", "q82" -> """ select i_item_id ,i_item_desc ,i_current_price from item, inventory, date_dim, store_sales where i_current_price between 68 and 68+30 and inv_item_sk = i_item_sk and d_date_sk=inv_date_sk and d_date between cast('2002-05-08' as date) and (cast('2002-05-08' as date) + INTERVAL 60 days) and i_manufact_id in (562,370,230,182) and inv_quantity_on_hand between 100 and 500 and ss_item_sk = i_item_sk group by i_item_id,i_item_desc,i_current_price order by i_item_id limit 100""", "q83" -> """ with sr_items as (select i_item_id item_id, sum(sr_return_quantity) sr_item_qty from store_returns, item, date_dim where sr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('2000-02-20','2000-10-08','2000-11-04'))) and sr_returned_date_sk = d_date_sk group by i_item_id), cr_items as (select i_item_id item_id, sum(cr_return_quantity) cr_item_qty from catalog_returns, item, date_dim where cr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('2000-02-20','2000-10-08','2000-11-04'))) and cr_returned_date_sk = d_date_sk group by i_item_id), wr_items as (select i_item_id item_id, sum(wr_return_quantity) wr_item_qty from web_returns, item, date_dim where wr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('2000-02-20','2000-10-08','2000-11-04'))) and wr_returned_date_sk = d_date_sk group by i_item_id) select sr_items.item_id ,sr_item_qty ,sr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 sr_dev ,cr_item_qty ,cr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 cr_dev ,wr_item_qty ,wr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 wr_dev ,(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 average from sr_items ,cr_items ,wr_items where sr_items.item_id=cr_items.item_id and sr_items.item_id=wr_items.item_id order by sr_items.item_id ,sr_item_qty limit 100""", "q84" -> """ select c_customer_id as customer_id , coalesce(c_last_name,'') || ', ' || coalesce(c_first_name,'') as customername from customer ,customer_address ,customer_demographics ,household_demographics ,income_band ,store_returns where ca_city = 'Buena Vista' and c_current_addr_sk = ca_address_sk and ib_lower_bound >= 49786 and ib_upper_bound <= 49786 + 50000 and ib_income_band_sk = hd_income_band_sk and cd_demo_sk = c_current_cdemo_sk and hd_demo_sk = c_current_hdemo_sk and sr_cdemo_sk = cd_demo_sk order by c_customer_id limit 100""", "q85" -> """ select substr(r_reason_desc,1,20) ,avg(ws_quantity) ,avg(wr_refunded_cash) ,avg(wr_fee) from web_sales, web_returns, web_page, customer_demographics cd1, customer_demographics cd2, customer_address, date_dim, reason where ws_web_page_sk = wp_web_page_sk and ws_item_sk = wr_item_sk and ws_order_number = wr_order_number and ws_sold_date_sk = d_date_sk and d_year = 2001 and cd1.cd_demo_sk = wr_refunded_cdemo_sk and cd2.cd_demo_sk = wr_returning_cdemo_sk and ca_address_sk = wr_refunded_addr_sk and r_reason_sk = wr_reason_sk and ( ( cd1.cd_marital_status = 'D' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = '4 yr Degree' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 100.00 and 150.00 ) or ( cd1.cd_marital_status = 'M' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = 'Primary' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 50.00 and 100.00 ) or ( cd1.cd_marital_status = 'U' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = '2 yr Degree' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 150.00 and 200.00 ) ) and ( ( ca_country = 'United States' and ca_state in ('IA', 'ND', 'FL') and ws_net_profit between 100 and 200 ) or ( ca_country = 'United States' and ca_state in ('OH', 'MS', 'VA') and ws_net_profit between 150 and 300 ) or ( ca_country = 'United States' and ca_state in ('MN', 'LA', 'TX') and ws_net_profit between 50 and 250 ) ) group by r_reason_desc order by substr(r_reason_desc,1,20) ,avg(ws_quantity) ,avg(wr_refunded_cash) ,avg(wr_fee) limit 100""", "q86" -> """ select sum(ws_net_paid) as total_sum ,i_category ,i_class ,grouping(i_category)+grouping(i_class) as lochierarchy ,rank() over ( partition by grouping(i_category)+grouping(i_class), case when grouping(i_class) = 0 then i_category end order by sum(ws_net_paid) desc) as rank_within_parent from web_sales ,date_dim d1 ,item where d1.d_month_seq between 1217 and 1217+11 and d1.d_date_sk = ws_sold_date_sk and i_item_sk = ws_item_sk group by rollup(i_category,i_class) order by lochierarchy desc, case when lochierarchy = 0 then i_category end, rank_within_parent limit 100""", "q87" -> """ select count(*) from ((select distinct c_last_name, c_first_name, d_date from store_sales, date_dim, customer where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_customer_sk = customer.c_customer_sk and d_month_seq between 1224 and 1224+11) except (select distinct c_last_name, c_first_name, d_date from catalog_sales, date_dim, customer where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1224 and 1224+11) except (select distinct c_last_name, c_first_name, d_date from web_sales, date_dim, customer where web_sales.ws_sold_date_sk = date_dim.d_date_sk and web_sales.ws_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1224 and 1224+11) ) cool_cust""", "q88" -> """ select * from (select count(*) h8_30_to_9 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 8 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s1, (select count(*) h9_to_9_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 9 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s2, (select count(*) h9_30_to_10 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 9 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s3, (select count(*) h10_to_10_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 10 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s4, (select count(*) h10_30_to_11 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 10 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s5, (select count(*) h11_to_11_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 11 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s6, (select count(*) h11_30_to_12 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 11 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s7, (select count(*) h12_to_12_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 12 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s8""", "q89" -> """ select * from( select i_category, i_class, i_brand, s_store_name, s_company_name, d_moy, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_category, i_brand, s_store_name, s_company_name) avg_monthly_sales from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_year in (2001) and ((i_category in ('Children','Home','Women') and i_class in ('toddlers','flatware','fragrances') ) or (i_category in ('Music','Electronics','Shoes') and i_class in ('country','dvd/vcr players','mens') )) group by i_category, i_class, i_brand, s_store_name, s_company_name, d_moy) tmp1 where case when (avg_monthly_sales <> 0) then (abs(sum_sales - avg_monthly_sales) / avg_monthly_sales) else null end > 0.1 order by sum_sales - avg_monthly_sales, s_store_name limit 100""", "q90" -> """ select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio from ( select count(*) amc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 7 and 7+1 and household_demographics.hd_dep_count = 1 and web_page.wp_char_count between 5000 and 5200) at, ( select count(*) pmc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 20 and 20+1 and household_demographics.hd_dep_count = 1 and web_page.wp_char_count between 5000 and 5200) pt order by am_pm_ratio limit 100""", "q91" -> """ select cc_call_center_id Call_Center, cc_name Call_Center_Name, cc_manager Manager, sum(cr_net_loss) Returns_Loss from call_center, catalog_returns, date_dim, customer, customer_address, customer_demographics, household_demographics where cr_call_center_sk = cc_call_center_sk and cr_returned_date_sk = d_date_sk and cr_returning_customer_sk= c_customer_sk and cd_demo_sk = c_current_cdemo_sk and hd_demo_sk = c_current_hdemo_sk and ca_address_sk = c_current_addr_sk and d_year = 1998 and d_moy = 12 and ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') or(cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) and hd_buy_potential like 'Unknown%' and ca_gmt_offset = -6 group by cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status order by sum(cr_net_loss) desc""", "q92" -> """ select sum(ws_ext_discount_amt) as `Excess Discount Amount` from web_sales ,item ,date_dim where i_manufact_id = 172 and i_item_sk = ws_item_sk and d_date between '1999-01-12' and (cast('1999-01-12' as date) + INTERVAL 90 days) and d_date_sk = ws_sold_date_sk and ws_ext_discount_amt > ( SELECT 1.3 * avg(ws_ext_discount_amt) FROM web_sales ,date_dim WHERE ws_item_sk = i_item_sk and d_date between '1999-01-12' and (cast('1999-01-12' as date) + INTERVAL 90 days) and d_date_sk = ws_sold_date_sk ) order by sum(ws_ext_discount_amt) limit 100""", "q93" -> """ select ss_customer_sk ,sum(act_sales) sumsales from (select ss_item_sk ,ss_ticket_number ,ss_customer_sk ,case when sr_return_quantity is not null then (ss_quantity-sr_return_quantity)*ss_sales_price else (ss_quantity*ss_sales_price) end act_sales from store_sales left outer join store_returns on (sr_item_sk = ss_item_sk and sr_ticket_number = ss_ticket_number) ,reason where sr_reason_sk = r_reason_sk and r_reason_desc = 'reason 58') t group by ss_customer_sk order by sumsales, ss_customer_sk limit 100""", "q94" -> """ select count(distinct ws_order_number) as `order count` ,sum(ws_ext_ship_cost) as `total shipping cost` ,sum(ws_net_profit) as `total net profit` from web_sales ws1 ,date_dim ,customer_address ,web_site where d_date between '2002-3-01' and (cast('2002-3-01' as date) + INTERVAL 60 days) and ws1.ws_ship_date_sk = d_date_sk and ws1.ws_ship_addr_sk = ca_address_sk and ca_state = 'GA' and ws1.ws_web_site_sk = web_site_sk and web_company_name = 'pri' and exists (select * from web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) and not exists(select * from web_returns wr1 where ws1.ws_order_number = wr1.wr_order_number) order by count(distinct ws_order_number) limit 100""", "q95" -> """ with ws_wh as (select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 from web_sales ws1,web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) select count(distinct ws_order_number) as `order count` ,sum(ws_ext_ship_cost) as `total shipping cost` ,sum(ws_net_profit) as `total net profit` from web_sales ws1 ,date_dim ,customer_address ,web_site where d_date between '2001-3-01' and (cast('2001-3-01' as date) + INTERVAL 60 days) and ws1.ws_ship_date_sk = d_date_sk and ws1.ws_ship_addr_sk = ca_address_sk and ca_state = 'NE' and ws1.ws_web_site_sk = web_site_sk and web_company_name = 'pri' and ws1.ws_order_number in (select ws_order_number from ws_wh) and ws1.ws_order_number in (select wr_order_number from web_returns,ws_wh where wr_order_number = ws_wh.ws_order_number) order by count(distinct ws_order_number) limit 100""", "q96" -> """ select count(*) from store_sales ,household_demographics ,time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 16 and time_dim.t_minute >= 30 and household_demographics.hd_dep_count = 0 and store.s_store_name = 'ese' order by count(*) limit 100""", "q97" -> """ with ssci as ( select ss_customer_sk customer_sk ,ss_item_sk item_sk from store_sales,date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1219 and 1219 + 11 group by ss_customer_sk ,ss_item_sk), csci as( select cs_bill_customer_sk customer_sk ,cs_item_sk item_sk from catalog_sales,date_dim where cs_sold_date_sk = d_date_sk and d_month_seq between 1219 and 1219 + 11 group by cs_bill_customer_sk ,cs_item_sk) select sum(case when ssci.customer_sk is not null and csci.customer_sk is null then 1 else 0 end) store_only ,sum(case when ssci.customer_sk is null and csci.customer_sk is not null then 1 else 0 end) catalog_only ,sum(case when ssci.customer_sk is not null and csci.customer_sk is not null then 1 else 0 end) store_and_catalog from ssci full outer join csci on (ssci.customer_sk=csci.customer_sk and ssci.item_sk = csci.item_sk) limit 100""", "q98" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(ss_ext_sales_price) as itemrevenue ,sum(ss_ext_sales_price)*100/sum(sum(ss_ext_sales_price)) over (partition by i_class) as revenueratio from store_sales ,item ,date_dim where ss_item_sk = i_item_sk and i_category in ('Books', 'Children', 'Sports') and ss_sold_date_sk = d_date_sk and d_date between cast('2001-03-10' as date) and (cast('2001-03-10' as date) + interval 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio""", "q99" -> """ select substr(w_warehouse_name,1,20) ,sm_type ,cc_name ,sum(case when (cs_ship_date_sk - cs_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 30) and (cs_ship_date_sk - cs_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 60) and (cs_ship_date_sk - cs_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 90) and (cs_ship_date_sk - cs_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from catalog_sales ,warehouse ,ship_mode ,call_center ,date_dim where d_month_seq between 1205 and 1205 + 11 and cs_ship_date_sk = d_date_sk and cs_warehouse_sk = w_warehouse_sk and cs_ship_mode_sk = sm_ship_mode_sk and cs_call_center_sk = cc_call_center_sk group by substr(w_warehouse_name,1,20) ,sm_type ,cc_name order by substr(w_warehouse_name,1,20) ,sm_type ,cc_name limit 100""" ) val TPCDSQueries10TB = Map( "q1" -> """ with customer_total_return as (select sr_customer_sk as ctr_customer_sk ,sr_store_sk as ctr_store_sk ,sum(SR_FEE) as ctr_total_return from store_returns ,date_dim where sr_returned_date_sk = d_date_sk and d_year =2000 group by sr_customer_sk ,sr_store_sk) select c_customer_id from customer_total_return ctr1 ,store ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_store_sk = ctr2.ctr_store_sk) and s_store_sk = ctr1.ctr_store_sk and s_state = 'TN' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id limit 100""", "q2" -> """ with wscs as (select sold_date_sk ,sales_price from (select ws_sold_date_sk sold_date_sk ,ws_ext_sales_price sales_price from web_sales union all select cs_sold_date_sk sold_date_sk ,cs_ext_sales_price sales_price from catalog_sales)), wswscs as (select d_week_seq, sum(case when (d_day_name='Sunday') then sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then sales_price else null end) sat_sales from wscs ,date_dim where d_date_sk = sold_date_sk group by d_week_seq) select d_week_seq1 ,round(sun_sales1/sun_sales2,2) ,round(mon_sales1/mon_sales2,2) ,round(tue_sales1/tue_sales2,2) ,round(wed_sales1/wed_sales2,2) ,round(thu_sales1/thu_sales2,2) ,round(fri_sales1/fri_sales2,2) ,round(sat_sales1/sat_sales2,2) from (select wswscs.d_week_seq d_week_seq1 ,sun_sales sun_sales1 ,mon_sales mon_sales1 ,tue_sales tue_sales1 ,wed_sales wed_sales1 ,thu_sales thu_sales1 ,fri_sales fri_sales1 ,sat_sales sat_sales1 from wswscs,date_dim where date_dim.d_week_seq = wswscs.d_week_seq and d_year = 1998) y, (select wswscs.d_week_seq d_week_seq2 ,sun_sales sun_sales2 ,mon_sales mon_sales2 ,tue_sales tue_sales2 ,wed_sales wed_sales2 ,thu_sales thu_sales2 ,fri_sales fri_sales2 ,sat_sales sat_sales2 from wswscs ,date_dim where date_dim.d_week_seq = wswscs.d_week_seq and d_year = 1998+1) z where d_week_seq1=d_week_seq2-53 order by d_week_seq1""", "q3" -> """ select dt.d_year ,item.i_brand_id brand_id ,item.i_brand brand ,sum(ss_sales_price) sum_agg from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manufact_id = 816 and dt.d_moy=11 group by dt.d_year ,item.i_brand ,item.i_brand_id order by dt.d_year ,sum_agg desc ,brand_id limit 100""", "q4" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum((((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2) ) year_total ,'c' sale_type from customer ,catalog_sales ,date_dim where c_customer_sk = cs_bill_customer_sk and cs_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum((((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2) ) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year ) select t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_birth_country from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_c_firstyear ,year_total t_c_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_c_secyear.customer_id and t_s_firstyear.customer_id = t_c_firstyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.sale_type = 's' and t_c_firstyear.sale_type = 'c' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_c_secyear.sale_type = 'c' and t_w_secyear.sale_type = 'w' and t_s_firstyear.dyear = 1999 and t_s_secyear.dyear = 1999+1 and t_c_firstyear.dyear = 1999 and t_c_secyear.dyear = 1999+1 and t_w_firstyear.dyear = 1999 and t_w_secyear.dyear = 1999+1 and t_s_firstyear.year_total > 0 and t_c_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end > case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end order by t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_birth_country limit 100""", "q5" -> """ with ssr as (select s_store_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select ss_store_sk as store_sk, ss_sold_date_sk as date_sk, ss_ext_sales_price as sales_price, ss_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from store_sales union all select sr_store_sk as store_sk, sr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, sr_return_amt as return_amt, sr_net_loss as net_loss from store_returns ) salesreturns, date_dim, store where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and store_sk = s_store_sk group by s_store_id) , csr as (select cp_catalog_page_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select cs_catalog_page_sk as page_sk, cs_sold_date_sk as date_sk, cs_ext_sales_price as sales_price, cs_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from catalog_sales union all select cr_catalog_page_sk as page_sk, cr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, cr_return_amount as return_amt, cr_net_loss as net_loss from catalog_returns ) salesreturns, date_dim, catalog_page where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and page_sk = cp_catalog_page_sk group by cp_catalog_page_id) , wsr as (select web_site_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select ws_web_site_sk as wsr_web_site_sk, ws_sold_date_sk as date_sk, ws_ext_sales_price as sales_price, ws_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from web_sales union all select ws_web_site_sk as wsr_web_site_sk, wr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, wr_return_amt as return_amt, wr_net_loss as net_loss from web_returns left outer join web_sales on ( wr_item_sk = ws_item_sk and wr_order_number = ws_order_number) ) salesreturns, date_dim, web_site where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and wsr_web_site_sk = web_site_sk group by web_site_id) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , 'store' || s_store_id as id , sales , returns , (profit - profit_loss) as profit from ssr union all select 'catalog channel' as channel , 'catalog_page' || cp_catalog_page_id as id , sales , returns , (profit - profit_loss) as profit from csr union all select 'web channel' as channel , 'web_site' || web_site_id as id , sales , returns , (profit - profit_loss) as profit from wsr ) x group by rollup (channel, id) order by channel ,id limit 100""", "q6" -> """ select a.ca_state state, count(*) cnt from customer_address a ,customer c ,store_sales s ,date_dim d ,item i where a.ca_address_sk = c.c_current_addr_sk and c.c_customer_sk = s.ss_customer_sk and s.ss_sold_date_sk = d.d_date_sk and s.ss_item_sk = i.i_item_sk and d.d_month_seq = (select distinct (d_month_seq) from date_dim where d_year = 2002 and d_moy = 3 ) and i.i_current_price > 1.2 * (select avg(j.i_current_price) from item j where j.i_category = i.i_category) group by a.ca_state having count(*) >= 10 order by cnt, a.ca_state limit 100""", "q7" -> """ select i_item_id, avg(ss_quantity) agg1, avg(ss_list_price) agg2, avg(ss_coupon_amt) agg3, avg(ss_sales_price) agg4 from store_sales, customer_demographics, date_dim, item, promotion where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and ss_cdemo_sk = cd_demo_sk and ss_promo_sk = p_promo_sk and cd_gender = 'F' and cd_marital_status = 'W' and cd_education_status = 'College' and (p_channel_email = 'N' or p_channel_event = 'N') and d_year = 2001 group by i_item_id order by i_item_id limit 100""", "q8" -> """ select s_store_name ,sum(ss_net_profit) from store_sales ,date_dim ,store, (select ca_zip from ( SELECT substr(ca_zip,1,5) ca_zip FROM customer_address WHERE substr(ca_zip,1,5) IN ( '47602','16704','35863','28577','83910','36201', '58412','48162','28055','41419','80332', '38607','77817','24891','16226','18410', '21231','59345','13918','51089','20317', '17167','54585','67881','78366','47770', '18360','51717','73108','14440','21800', '89338','45859','65501','34948','25973', '73219','25333','17291','10374','18829', '60736','82620','41351','52094','19326', '25214','54207','40936','21814','79077', '25178','75742','77454','30621','89193', '27369','41232','48567','83041','71948', '37119','68341','14073','16891','62878', '49130','19833','24286','27700','40979', '50412','81504','94835','84844','71954', '39503','57649','18434','24987','12350', '86379','27413','44529','98569','16515', '27287','24255','21094','16005','56436', '91110','68293','56455','54558','10298', '83647','32754','27052','51766','19444', '13869','45645','94791','57631','20712', '37788','41807','46507','21727','71836', '81070','50632','88086','63991','20244', '31655','51782','29818','63792','68605', '94898','36430','57025','20601','82080', '33869','22728','35834','29086','92645', '98584','98072','11652','78093','57553', '43830','71144','53565','18700','90209', '71256','38353','54364','28571','96560', '57839','56355','50679','45266','84680', '34306','34972','48530','30106','15371', '92380','84247','92292','68852','13338', '34594','82602','70073','98069','85066', '47289','11686','98862','26217','47529', '63294','51793','35926','24227','14196', '24594','32489','99060','49472','43432', '49211','14312','88137','47369','56877', '20534','81755','15794','12318','21060', '73134','41255','63073','81003','73873', '66057','51184','51195','45676','92696', '70450','90669','98338','25264','38919', '59226','58581','60298','17895','19489', '52301','80846','95464','68770','51634', '19988','18367','18421','11618','67975', '25494','41352','95430','15734','62585', '97173','33773','10425','75675','53535', '17879','41967','12197','67998','79658', '59130','72592','14851','43933','68101', '50636','25717','71286','24660','58058', '72991','95042','15543','33122','69280', '11912','59386','27642','65177','17672', '33467','64592','36335','54010','18767', '63193','42361','49254','33113','33159', '36479','59080','11855','81963','31016', '49140','29392','41836','32958','53163', '13844','73146','23952','65148','93498', '14530','46131','58454','13376','13378', '83986','12320','17193','59852','46081', '98533','52389','13086','68843','31013', '13261','60560','13443','45533','83583', '11489','58218','19753','22911','25115', '86709','27156','32669','13123','51933', '39214','41331','66943','14155','69998', '49101','70070','35076','14242','73021', '59494','15782','29752','37914','74686', '83086','34473','15751','81084','49230', '91894','60624','17819','28810','63180', '56224','39459','55233','75752','43639', '55349','86057','62361','50788','31830', '58062','18218','85761','60083','45484', '21204','90229','70041','41162','35390', '16364','39500','68908','26689','52868', '81335','40146','11340','61527','61794', '71997','30415','59004','29450','58117', '69952','33562','83833','27385','61860', '96435','48333','23065','32961','84919', '61997','99132','22815','56600','68730', '48017','95694','32919','88217','27116', '28239','58032','18884','16791','21343', '97462','18569','75660','15475') intersect select ca_zip from (SELECT substr(ca_zip,1,5) ca_zip,count(*) cnt FROM customer_address, customer WHERE ca_address_sk = c_current_addr_sk and c_preferred_cust_flag='Y' group by ca_zip having count(*) > 10)A1)A2) V1 where ss_store_sk = s_store_sk and ss_sold_date_sk = d_date_sk and d_qoy = 2 and d_year = 1998 and (substr(s_zip,1,2) = substr(V1.ca_zip,1,2)) group by s_store_name order by s_store_name limit 100""", "q9" -> """ select case when (select count(*) from store_sales where ss_quantity between 1 and 20) > 2972190 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 1 and 20) else (select avg(ss_net_profit) from store_sales where ss_quantity between 1 and 20) end bucket1 , case when (select count(*) from store_sales where ss_quantity between 21 and 40) > 111711138 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 21 and 40) else (select avg(ss_net_profit) from store_sales where ss_quantity between 21 and 40) end bucket2, case when (select count(*) from store_sales where ss_quantity between 41 and 60) > 127958920 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 41 and 60) else (select avg(ss_net_profit) from store_sales where ss_quantity between 41 and 60) end bucket3, case when (select count(*) from store_sales where ss_quantity between 61 and 80) > 41162107 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 61 and 80) else (select avg(ss_net_profit) from store_sales where ss_quantity between 61 and 80) end bucket4, case when (select count(*) from store_sales where ss_quantity between 81 and 100) > 25211875 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 81 and 100) else (select avg(ss_net_profit) from store_sales where ss_quantity between 81 and 100) end bucket5 from reason where r_reason_sk = 1""", "q10" -> """ select cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3, cd_dep_count, count(*) cnt4, cd_dep_employed_count, count(*) cnt5, cd_dep_college_count, count(*) cnt6 from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and ca_county in ('Allen County','Jefferson County','Lamar County','Dakota County','Park County') and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2001 and d_moy between 4 and 4+3) and (exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2001 and d_moy between 4 ANd 4+3) or exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2001 and d_moy between 4 and 4+3)) group by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count order by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count limit 100""", "q11" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(ss_ext_list_price-ss_ext_discount_amt) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(ws_ext_list_price-ws_ext_discount_amt) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year ) select t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_login from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.sale_type = 's' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_w_secyear.sale_type = 'w' and t_s_firstyear.dyear = 1998 and t_s_secyear.dyear = 1998+1 and t_w_firstyear.dyear = 1998 and t_w_secyear.dyear = 1998+1 and t_s_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else 0.0 end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else 0.0 end order by t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_login limit 100""", "q12" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(ws_ext_sales_price) as itemrevenue ,sum(ws_ext_sales_price)*100/sum(sum(ws_ext_sales_price)) over (partition by i_class) as revenueratio from web_sales ,item ,date_dim where ws_item_sk = i_item_sk and i_category in ('Men', 'Books', 'Children') and ws_sold_date_sk = d_date_sk and d_date between cast('1998-03-28' as date) and (cast('1998-03-28' as date) + INTERVAL 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio limit 100""", "q13" -> """ select avg(ss_quantity) ,avg(ss_ext_sales_price) ,avg(ss_ext_wholesale_cost) ,sum(ss_ext_wholesale_cost) from store_sales ,store ,customer_demographics ,household_demographics ,customer_address ,date_dim where s_store_sk = ss_store_sk and ss_sold_date_sk = d_date_sk and d_year = 2001 and((ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'U' and cd_education_status = 'Unknown' and ss_sales_price between 100.00 and 150.00 and hd_dep_count = 3 )or (ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'W' and cd_education_status = '2 yr Degree' and ss_sales_price between 50.00 and 100.00 and hd_dep_count = 1 ) or (ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'S' and cd_education_status = 'College' and ss_sales_price between 150.00 and 200.00 and hd_dep_count = 1 )) and((ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('WV', 'GA', 'TX') and ss_net_profit between 100 and 200 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('TN', 'KY', 'SC') and ss_net_profit between 150 and 300 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('OK', 'NE', 'CA') and ss_net_profit between 50 and 250 ))""", "q14a" -> """ with cross_items as (select i_item_sk ss_item_sk from item, (select iss.i_brand_id brand_id ,iss.i_class_id class_id ,iss.i_category_id category_id from store_sales ,item iss ,date_dim d1 where ss_item_sk = iss.i_item_sk and ss_sold_date_sk = d1.d_date_sk and d1.d_year between 1998 AND 1998 + 2 intersect select ics.i_brand_id ,ics.i_class_id ,ics.i_category_id from catalog_sales ,item ics ,date_dim d2 where cs_item_sk = ics.i_item_sk and cs_sold_date_sk = d2.d_date_sk and d2.d_year between 1998 AND 1998 + 2 intersect select iws.i_brand_id ,iws.i_class_id ,iws.i_category_id from web_sales ,item iws ,date_dim d3 where ws_item_sk = iws.i_item_sk and ws_sold_date_sk = d3.d_date_sk and d3.d_year between 1998 AND 1998 + 2) where i_brand_id = brand_id and i_class_id = class_id and i_category_id = category_id ), avg_sales as (select avg(quantity*list_price) average_sales from (select ss_quantity quantity ,ss_list_price list_price from store_sales ,date_dim where ss_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2 union all select cs_quantity quantity ,cs_list_price list_price from catalog_sales ,date_dim where cs_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2 union all select ws_quantity quantity ,ws_list_price list_price from web_sales ,date_dim where ws_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2) x) select channel, i_brand_id,i_class_id,i_category_id,sum(sales), sum(number_sales) from( select 'store' channel, i_brand_id,i_class_id ,i_category_id,sum(ss_quantity*ss_list_price) sales , count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1998+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales) union all select 'catalog' channel, i_brand_id,i_class_id,i_category_id, sum(cs_quantity*cs_list_price) sales, count(*) number_sales from catalog_sales ,item ,date_dim where cs_item_sk in (select ss_item_sk from cross_items) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1998+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(cs_quantity*cs_list_price) > (select average_sales from avg_sales) union all select 'web' channel, i_brand_id,i_class_id,i_category_id, sum(ws_quantity*ws_list_price) sales , count(*) number_sales from web_sales ,item ,date_dim where ws_item_sk in (select ss_item_sk from cross_items) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1998+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(ws_quantity*ws_list_price) > (select average_sales from avg_sales) ) y group by rollup (channel, i_brand_id,i_class_id,i_category_id) order by channel,i_brand_id,i_class_id,i_category_id limit 100""", "q14b" -> """ with cross_items as (select i_item_sk ss_item_sk from item, (select iss.i_brand_id brand_id ,iss.i_class_id class_id ,iss.i_category_id category_id from store_sales ,item iss ,date_dim d1 where ss_item_sk = iss.i_item_sk and ss_sold_date_sk = d1.d_date_sk and d1.d_year between 1998 AND 1998 + 2 intersect select ics.i_brand_id ,ics.i_class_id ,ics.i_category_id from catalog_sales ,item ics ,date_dim d2 where cs_item_sk = ics.i_item_sk and cs_sold_date_sk = d2.d_date_sk and d2.d_year between 1998 AND 1998 + 2 intersect select iws.i_brand_id ,iws.i_class_id ,iws.i_category_id from web_sales ,item iws ,date_dim d3 where ws_item_sk = iws.i_item_sk and ws_sold_date_sk = d3.d_date_sk and d3.d_year between 1998 AND 1998 + 2) x where i_brand_id = brand_id and i_class_id = class_id and i_category_id = category_id ), avg_sales as (select avg(quantity*list_price) average_sales from (select ss_quantity quantity ,ss_list_price list_price from store_sales ,date_dim where ss_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2 union all select cs_quantity quantity ,cs_list_price list_price from catalog_sales ,date_dim where cs_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2 union all select ws_quantity quantity ,ws_list_price list_price from web_sales ,date_dim where ws_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2) x) select this_year.channel ty_channel ,this_year.i_brand_id ty_brand ,this_year.i_class_id ty_class ,this_year.i_category_id ty_category ,this_year.sales ty_sales ,this_year.number_sales ty_number_sales ,last_year.channel ly_channel ,last_year.i_brand_id ly_brand ,last_year.i_class_id ly_class ,last_year.i_category_id ly_category ,last_year.sales ly_sales ,last_year.number_sales ly_number_sales from (select 'store' channel, i_brand_id,i_class_id,i_category_id ,sum(ss_quantity*ss_list_price) sales, count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_week_seq = (select d_week_seq from date_dim where d_year = 1998 + 1 and d_moy = 12 and d_dom = 20) group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) this_year, (select 'store' channel, i_brand_id,i_class_id ,i_category_id, sum(ss_quantity*ss_list_price) sales, count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_week_seq = (select d_week_seq from date_dim where d_year = 1998 and d_moy = 12 and d_dom = 20) group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) last_year where this_year.i_brand_id= last_year.i_brand_id and this_year.i_class_id = last_year.i_class_id and this_year.i_category_id = last_year.i_category_id order by this_year.channel, this_year.i_brand_id, this_year.i_class_id, this_year.i_category_id limit 100""", "q15" -> """ select ca_zip ,sum(cs_sales_price) from catalog_sales ,customer ,customer_address ,date_dim where cs_bill_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') or ca_state in ('CA','WA','GA') or cs_sales_price > 500) and cs_sold_date_sk = d_date_sk and d_qoy = 1 and d_year = 2000 group by ca_zip order by ca_zip limit 100""", "q16" -> """ select count(distinct cs_order_number) as `order count` ,sum(cs_ext_ship_cost) as `total shipping cost` ,sum(cs_net_profit) as `total net profit` from catalog_sales cs1 ,date_dim ,customer_address ,call_center where d_date between '2001-2-01' and (cast('2001-2-01' as date) + INTERVAL 60 days) and cs1.cs_ship_date_sk = d_date_sk and cs1.cs_ship_addr_sk = ca_address_sk and ca_state = 'MS' and cs1.cs_call_center_sk = cc_call_center_sk and cc_county in ('Jackson County','Daviess County','Walker County','Dauphin County', 'Mobile County' ) and exists (select * from catalog_sales cs2 where cs1.cs_order_number = cs2.cs_order_number and cs1.cs_warehouse_sk <> cs2.cs_warehouse_sk) and not exists(select * from catalog_returns cr1 where cs1.cs_order_number = cr1.cr_order_number) order by count(distinct cs_order_number) limit 100""", "q17" -> """ select i_item_id ,i_item_desc ,s_state ,count(ss_quantity) as store_sales_quantitycount ,avg(ss_quantity) as store_sales_quantityave ,stddev_samp(ss_quantity) as store_sales_quantitystdev ,stddev_samp(ss_quantity)/avg(ss_quantity) as store_sales_quantitycov ,count(sr_return_quantity) as store_returns_quantitycount ,avg(sr_return_quantity) as store_returns_quantityave ,stddev_samp(sr_return_quantity) as store_returns_quantitystdev ,stddev_samp(sr_return_quantity)/avg(sr_return_quantity) as store_returns_quantitycov ,count(cs_quantity) as catalog_sales_quantitycount ,avg(cs_quantity) as catalog_sales_quantityave ,stddev_samp(cs_quantity) as catalog_sales_quantitystdev ,stddev_samp(cs_quantity)/avg(cs_quantity) as catalog_sales_quantitycov from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_quarter_name = '1999Q1' and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_quarter_name in ('1999Q1','1999Q2','1999Q3') and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_quarter_name in ('1999Q1','1999Q2','1999Q3') group by i_item_id ,i_item_desc ,s_state order by i_item_id ,i_item_desc ,s_state limit 100""", "q18" -> """ select i_item_id, ca_country, ca_state, ca_county, avg( cast(cs_quantity as decimal(12,2))) agg1, avg( cast(cs_list_price as decimal(12,2))) agg2, avg( cast(cs_coupon_amt as decimal(12,2))) agg3, avg( cast(cs_sales_price as decimal(12,2))) agg4, avg( cast(cs_net_profit as decimal(12,2))) agg5, avg( cast(c_birth_year as decimal(12,2))) agg6, avg( cast(cd1.cd_dep_count as decimal(12,2))) agg7 from catalog_sales, customer_demographics cd1, customer_demographics cd2, customer, customer_address, date_dim, item where cs_sold_date_sk = d_date_sk and cs_item_sk = i_item_sk and cs_bill_cdemo_sk = cd1.cd_demo_sk and cs_bill_customer_sk = c_customer_sk and cd1.cd_gender = 'F' and cd1.cd_education_status = 'Primary' and c_current_cdemo_sk = cd2.cd_demo_sk and c_current_addr_sk = ca_address_sk and c_birth_month in (6,7,3,11,12,8) and d_year = 1999 and ca_state in ('IL','WV','KS' ,'GA','LA','PA','TX') group by rollup (i_item_id, ca_country, ca_state, ca_county) order by ca_country, ca_state, ca_county, i_item_id limit 100""", "q19" -> """ select i_brand_id brand_id, i_brand brand, i_manufact_id, i_manufact, sum(ss_ext_sales_price) ext_price from date_dim, store_sales, item,customer,customer_address,store where d_date_sk = ss_sold_date_sk and ss_item_sk = i_item_sk and i_manager_id=26 and d_moy=12 and d_year=2000 and ss_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and substr(ca_zip,1,5) <> substr(s_zip,1,5) and ss_store_sk = s_store_sk group by i_brand ,i_brand_id ,i_manufact_id ,i_manufact order by ext_price desc ,i_brand ,i_brand_id ,i_manufact_id ,i_manufact limit 100 """, "q20" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(cs_ext_sales_price) as itemrevenue ,sum(cs_ext_sales_price)*100/sum(sum(cs_ext_sales_price)) over (partition by i_class) as revenueratio from catalog_sales ,item ,date_dim where cs_item_sk = i_item_sk and i_category in ('Books', 'Home', 'Jewelry') and cs_sold_date_sk = d_date_sk and d_date between cast('1998-05-08' as date) and (cast('1998-05-08' as date) + INTERVAL 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio limit 100""", "q21" -> """ select * from(select w_warehouse_name ,i_item_id ,sum(case when (cast(d_date as date) < cast ('2000-05-22' as date)) then inv_quantity_on_hand else 0 end) as inv_before ,sum(case when (cast(d_date as date) >= cast ('2000-05-22' as date)) then inv_quantity_on_hand else 0 end) as inv_after from inventory ,warehouse ,item ,date_dim where i_current_price between 0.99 and 1.49 and i_item_sk = inv_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_date between (cast ('2000-05-22' as date) - INTERVAL 30 days) and (cast ('2000-05-22' as date) + INTERVAL 30 days) group by w_warehouse_name, i_item_id) x where (case when inv_before > 0 then inv_after / inv_before else null end) between 2.0/3.0 and 3.0/2.0 order by w_warehouse_name ,i_item_id limit 100""", "q22" -> """ select i_product_name ,i_brand ,i_class ,i_category ,avg(inv_quantity_on_hand) qoh from inventory ,date_dim ,item where inv_date_sk=d_date_sk and inv_item_sk=i_item_sk and d_month_seq between 1199 and 1199 + 11 group by rollup(i_product_name ,i_brand ,i_class ,i_category) order by qoh, i_product_name, i_brand, i_class, i_category limit 100""", "q23a" -> """ with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt from store_sales ,date_dim ,item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and d_year in (2000,2000+1,2000+2,2000+3) group by substr(i_item_desc,1,30),i_item_sk,d_date having count(*) >4), max_store_sales as (select max(csales) tpcds_cmax from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales from store_sales ,customer ,date_dim where ss_customer_sk = c_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (2000,2000+1,2000+2,2000+3) group by c_customer_sk)), best_ss_customer as (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales from store_sales ,customer where ss_customer_sk = c_customer_sk group by c_customer_sk having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select * from max_store_sales)) select sum(sales) from (select cs_quantity*cs_list_price sales from catalog_sales ,date_dim where d_year = 2000 and d_moy = 5 and cs_sold_date_sk = d_date_sk and cs_item_sk in (select item_sk from frequent_ss_items) and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) union all select ws_quantity*ws_list_price sales from web_sales ,date_dim where d_year = 2000 and d_moy = 5 and ws_sold_date_sk = d_date_sk and ws_item_sk in (select item_sk from frequent_ss_items) and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer)) limit 100""", "q23b" -> """ with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt from store_sales ,date_dim ,item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and d_year in (2000,2000 + 1,2000 + 2,2000 + 3) group by substr(i_item_desc,1,30),i_item_sk,d_date having count(*) >4), max_store_sales as (select max(csales) tpcds_cmax from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales from store_sales ,customer ,date_dim where ss_customer_sk = c_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (2000,2000+1,2000+2,2000+3) group by c_customer_sk)), best_ss_customer as (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales from store_sales ,customer where ss_customer_sk = c_customer_sk group by c_customer_sk having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select * from max_store_sales)) select c_last_name,c_first_name,sales from (select c_last_name,c_first_name,sum(cs_quantity*cs_list_price) sales from catalog_sales ,customer ,date_dim where d_year = 2000 and d_moy = 5 and cs_sold_date_sk = d_date_sk and cs_item_sk in (select item_sk from frequent_ss_items) and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) and cs_bill_customer_sk = c_customer_sk group by c_last_name,c_first_name union all select c_last_name,c_first_name,sum(ws_quantity*ws_list_price) sales from web_sales ,customer ,date_dim where d_year = 2000 and d_moy = 5 and ws_sold_date_sk = d_date_sk and ws_item_sk in (select item_sk from frequent_ss_items) and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer) and ws_bill_customer_sk = c_customer_sk group by c_last_name,c_first_name) order by c_last_name,c_first_name,sales limit 100""", "q24a" -> """ with ssales as (select c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size ,sum(ss_net_paid_inc_tax) netpaid from store_sales ,store_returns ,store ,item ,customer ,customer_address where ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_customer_sk = c_customer_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and c_current_addr_sk = ca_address_sk and c_birth_country <> upper(ca_country) and s_zip = ca_zip and s_market_id=10 group by c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size) select c_last_name ,c_first_name ,s_store_name ,sum(netpaid) paid from ssales where i_color = 'navy' group by c_last_name ,c_first_name ,s_store_name having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) order by c_last_name ,c_first_name ,s_store_name""", "q24b" -> """ with ssales as (select c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size ,sum(ss_net_paid_inc_tax) netpaid from store_sales ,store_returns ,store ,item ,customer ,customer_address where ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_customer_sk = c_customer_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and c_current_addr_sk = ca_address_sk and c_birth_country <> upper(ca_country) and s_zip = ca_zip and s_market_id = 10 group by c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size) select c_last_name ,c_first_name ,s_store_name ,sum(netpaid) paid from ssales where i_color = 'beige' group by c_last_name ,c_first_name ,s_store_name having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) order by c_last_name ,c_first_name ,s_store_name""", "q25" -> """ select i_item_id ,i_item_desc ,s_store_id ,s_store_name ,sum(ss_net_profit) as store_sales_profit ,sum(sr_net_loss) as store_returns_loss ,sum(cs_net_profit) as catalog_sales_profit from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_moy = 4 and d1.d_year = 2002 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_moy between 4 and 10 and d2.d_year = 2002 and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_moy between 4 and 10 and d3.d_year = 2002 group by i_item_id ,i_item_desc ,s_store_id ,s_store_name order by i_item_id ,i_item_desc ,s_store_id ,s_store_name limit 100""", "q26" -> """ select i_item_id, avg(cs_quantity) agg1, avg(cs_list_price) agg2, avg(cs_coupon_amt) agg3, avg(cs_sales_price) agg4 from catalog_sales, customer_demographics, date_dim, item, promotion where cs_sold_date_sk = d_date_sk and cs_item_sk = i_item_sk and cs_bill_cdemo_sk = cd_demo_sk and cs_promo_sk = p_promo_sk and cd_gender = 'F' and cd_marital_status = 'M' and cd_education_status = '2 yr Degree' and (p_channel_email = 'N' or p_channel_event = 'N') and d_year = 2002 group by i_item_id order by i_item_id limit 100""", "q27" -> """ select i_item_id, s_state, grouping(s_state) g_state, avg(ss_quantity) agg1, avg(ss_list_price) agg2, avg(ss_coupon_amt) agg3, avg(ss_sales_price) agg4 from store_sales, customer_demographics, date_dim, store, item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and ss_cdemo_sk = cd_demo_sk and cd_gender = 'F' and cd_marital_status = 'S' and cd_education_status = 'Advanced Degree' and d_year = 2000 and s_state in ('WA','LA', 'LA', 'TX', 'AL', 'PA') group by rollup (i_item_id, s_state) order by i_item_id ,s_state limit 100""", "q28" -> """ select * from (select avg(ss_list_price) B1_LP ,count(ss_list_price) B1_CNT ,count(distinct ss_list_price) B1_CNTD from store_sales where ss_quantity between 0 and 5 and (ss_list_price between 189 and 189+10 or ss_coupon_amt between 4483 and 4483+1000 or ss_wholesale_cost between 24 and 24+20)) B1, (select avg(ss_list_price) B2_LP ,count(ss_list_price) B2_CNT ,count(distinct ss_list_price) B2_CNTD from store_sales where ss_quantity between 6 and 10 and (ss_list_price between 71 and 71+10 or ss_coupon_amt between 14775 and 14775+1000 or ss_wholesale_cost between 38 and 38+20)) B2, (select avg(ss_list_price) B3_LP ,count(ss_list_price) B3_CNT ,count(distinct ss_list_price) B3_CNTD from store_sales where ss_quantity between 11 and 15 and (ss_list_price between 183 and 183+10 or ss_coupon_amt between 13456 and 13456+1000 or ss_wholesale_cost between 31 and 31+20)) B3, (select avg(ss_list_price) B4_LP ,count(ss_list_price) B4_CNT ,count(distinct ss_list_price) B4_CNTD from store_sales where ss_quantity between 16 and 20 and (ss_list_price between 135 and 135+10 or ss_coupon_amt between 4905 and 4905+1000 or ss_wholesale_cost between 27 and 27+20)) B4, (select avg(ss_list_price) B5_LP ,count(ss_list_price) B5_CNT ,count(distinct ss_list_price) B5_CNTD from store_sales where ss_quantity between 21 and 25 and (ss_list_price between 180 and 180+10 or ss_coupon_amt between 17430 and 17430+1000 or ss_wholesale_cost between 57 and 57+20)) B5, (select avg(ss_list_price) B6_LP ,count(ss_list_price) B6_CNT ,count(distinct ss_list_price) B6_CNTD from store_sales where ss_quantity between 26 and 30 and (ss_list_price between 49 and 49+10 or ss_coupon_amt between 2950 and 2950+1000 or ss_wholesale_cost between 52 and 52+20)) B6 limit 100""", "q29" -> """ select i_item_id ,i_item_desc ,s_store_id ,s_store_name ,stddev_samp(ss_quantity) as store_sales_quantity ,stddev_samp(sr_return_quantity) as store_returns_quantity ,stddev_samp(cs_quantity) as catalog_sales_quantity from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_moy = 4 and d1.d_year = 1998 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_moy between 4 and 4 + 3 and d2.d_year = 1998 and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_year in (1998,1998+1,1998+2) group by i_item_id ,i_item_desc ,s_store_id ,s_store_name order by i_item_id ,i_item_desc ,s_store_id ,s_store_name limit 100""", "q30" -> """ with customer_total_return as (select wr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, sum(wr_return_amt) as ctr_total_return from web_returns ,date_dim ,customer_address where wr_returned_date_sk = d_date_sk and d_year =2000 and wr_returning_addr_sk = ca_address_sk group by wr_returning_customer_sk ,ca_state) select c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address ,c_last_review_date,ctr_total_return from customer_total_return ctr1 ,customer_address ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_state = ctr2.ctr_state) and ca_address_sk = c_current_addr_sk and ca_state = 'GA' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address ,c_last_review_date,ctr_total_return limit 100""", "q31" -> """ with ss as (select ca_county,d_qoy, d_year,sum(ss_ext_sales_price) as store_sales from store_sales,date_dim,customer_address where ss_sold_date_sk = d_date_sk and ss_addr_sk=ca_address_sk group by ca_county,d_qoy, d_year), ws as (select ca_county,d_qoy, d_year,sum(ws_ext_sales_price) as web_sales from web_sales,date_dim,customer_address where ws_sold_date_sk = d_date_sk and ws_bill_addr_sk=ca_address_sk group by ca_county,d_qoy, d_year) select ss1.ca_county ,ss1.d_year ,ws2.web_sales/ws1.web_sales web_q1_q2_increase ,ss2.store_sales/ss1.store_sales store_q1_q2_increase ,ws3.web_sales/ws2.web_sales web_q2_q3_increase ,ss3.store_sales/ss2.store_sales store_q2_q3_increase from ss ss1 ,ss ss2 ,ss ss3 ,ws ws1 ,ws ws2 ,ws ws3 where ss1.d_qoy = 1 and ss1.d_year = 1998 and ss1.ca_county = ss2.ca_county and ss2.d_qoy = 2 and ss2.d_year = 1998 and ss2.ca_county = ss3.ca_county and ss3.d_qoy = 3 and ss3.d_year = 1998 and ss1.ca_county = ws1.ca_county and ws1.d_qoy = 1 and ws1.d_year = 1998 and ws1.ca_county = ws2.ca_county and ws2.d_qoy = 2 and ws2.d_year = 1998 and ws1.ca_county = ws3.ca_county and ws3.d_qoy = 3 and ws3.d_year =1998 and case when ws1.web_sales > 0 then ws2.web_sales/ws1.web_sales else null end > case when ss1.store_sales > 0 then ss2.store_sales/ss1.store_sales else null end and case when ws2.web_sales > 0 then ws3.web_sales/ws2.web_sales else null end > case when ss2.store_sales > 0 then ss3.store_sales/ss2.store_sales else null end order by ss1.ca_county""", "q32" -> """ select sum(cs_ext_discount_amt) as `excess discount amount` from catalog_sales ,item ,date_dim where i_manufact_id = 948 and i_item_sk = cs_item_sk and d_date between '1998-02-03' and (cast('1998-02-03' as date) + INTERVAL 90 days) and d_date_sk = cs_sold_date_sk and cs_ext_discount_amt > ( select 1.3 * avg(cs_ext_discount_amt) from catalog_sales ,date_dim where cs_item_sk = i_item_sk and d_date between '1998-02-03' and (cast('1998-02-03' as date) + INTERVAL 90 days) and d_date_sk = cs_sold_date_sk ) limit 100""", "q33" -> """ with ss as ( select i_manufact_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Electronics')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 2 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_manufact_id), cs as ( select i_manufact_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Electronics')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 2 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_manufact_id), ws as ( select i_manufact_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Electronics')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 2 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_manufact_id) select i_manufact_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_manufact_id order by total_sales limit 100""", "q34" -> """ select c_last_name ,c_first_name ,c_salutation ,c_preferred_cust_flag ,ss_ticket_number ,cnt from (select ss_ticket_number ,ss_customer_sk ,count(*) cnt from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and (date_dim.d_dom between 1 and 3 or date_dim.d_dom between 25 and 28) and (household_demographics.hd_buy_potential = '>10000' or household_demographics.hd_buy_potential = '5001-10000') and household_demographics.hd_vehicle_count > 0 and (case when household_demographics.hd_vehicle_count > 0 then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end) > 1.2 and date_dim.d_year in (1999,1999+1,1999+2) and store.s_county in ('Jefferson Davis Parish','Levy County','Coal County','Oglethorpe County', 'Mobile County','Gage County','Richland County','Gogebic County') group by ss_ticket_number,ss_customer_sk) dn,customer where ss_customer_sk = c_customer_sk and cnt between 15 and 20 order by c_last_name,c_first_name,c_salutation,c_preferred_cust_flag desc, ss_ticket_number""", "q35" -> """ select ca_state, cd_gender, cd_marital_status, cd_dep_count, count(*) cnt1, stddev_samp(cd_dep_count), stddev_samp(cd_dep_count), min(cd_dep_count), cd_dep_employed_count, count(*) cnt2, stddev_samp(cd_dep_employed_count), stddev_samp(cd_dep_employed_count), min(cd_dep_employed_count), cd_dep_college_count, count(*) cnt3, stddev_samp(cd_dep_college_count), stddev_samp(cd_dep_college_count), min(cd_dep_college_count) from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2002 and d_qoy < 4) and (exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2002 and d_qoy < 4) or exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2002 and d_qoy < 4)) group by ca_state, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count order by ca_state, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count limit 100""", "q36" -> """ select sum(ss_net_profit)/sum(ss_ext_sales_price) as gross_margin ,i_category ,i_class ,grouping(i_category)+grouping(i_class) as lochierarchy ,rank() over ( partition by grouping(i_category)+grouping(i_class), case when grouping(i_class) = 0 then i_category end order by sum(ss_net_profit)/sum(ss_ext_sales_price) asc) as rank_within_parent from store_sales ,date_dim d1 ,item ,store where d1.d_year = 1998 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and s_state in ('OH','WV','PA','TN', 'MN','MO','NM','MI') group by rollup(i_category,i_class) order by lochierarchy desc ,case when lochierarchy = 0 then i_category end ,rank_within_parent limit 100""", "q37" -> """ select i_item_id ,i_item_desc ,i_current_price from item, inventory, date_dim, catalog_sales where i_current_price between 35 and 35 + 30 and inv_item_sk = i_item_sk and d_date_sk=inv_date_sk and d_date between cast('2001-01-20' as date) and (cast('2001-01-20' as date) + interval 60 days) and i_manufact_id in (928,715,942,861) and inv_quantity_on_hand between 100 and 500 and cs_item_sk = i_item_sk group by i_item_id,i_item_desc,i_current_price order by i_item_id limit 100""", "q38" -> """ select count(*) from ( select distinct c_last_name, c_first_name, d_date from store_sales, date_dim, customer where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_customer_sk = customer.c_customer_sk and d_month_seq between 1222 and 1222 + 11 intersect select distinct c_last_name, c_first_name, d_date from catalog_sales, date_dim, customer where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1222 and 1222 + 11 intersect select distinct c_last_name, c_first_name, d_date from web_sales, date_dim, customer where web_sales.ws_sold_date_sk = date_dim.d_date_sk and web_sales.ws_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1222 and 1222 + 11 ) hot_cust limit 100""", "q39a" -> """ with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stdev,mean, case mean when 0 then null else stdev/mean end cov from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean from inventory ,item ,warehouse ,date_dim where inv_item_sk = i_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_year =1998 group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo where case mean when 0 then 0 else stdev/mean end > 1) select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov from inv inv1,inv inv2 where inv1.i_item_sk = inv2.i_item_sk and inv1.w_warehouse_sk = inv2.w_warehouse_sk and inv1.d_moy=4 and inv2.d_moy=4+1 order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov ,inv2.d_moy,inv2.mean, inv2.cov""", "q39b" -> """ with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stdev,mean, case mean when 0 then null else stdev/mean end cov from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean from inventory ,item ,warehouse ,date_dim where inv_item_sk = i_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_year =1998 group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo where case mean when 0 then 0 else stdev/mean end > 1) select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov from inv inv1,inv inv2 where inv1.i_item_sk = inv2.i_item_sk and inv1.w_warehouse_sk = inv2.w_warehouse_sk and inv1.d_moy=4 and inv2.d_moy=4+1 and inv1.cov > 1.5 order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov ,inv2.d_moy,inv2.mean, inv2.cov""", "q40" -> """ select w_state ,i_item_id ,sum(case when (cast(d_date as date) < cast ('1999-02-02' as date)) then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_before ,sum(case when (cast(d_date as date) >= cast ('1999-02-02' as date)) then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_after from catalog_sales left outer join catalog_returns on (cs_order_number = cr_order_number and cs_item_sk = cr_item_sk) ,warehouse ,item ,date_dim where i_current_price between 0.99 and 1.49 and i_item_sk = cs_item_sk and cs_warehouse_sk = w_warehouse_sk and cs_sold_date_sk = d_date_sk and d_date between (cast ('1999-02-02' as date) - INTERVAL 30 days) and (cast ('1999-02-02' as date) + INTERVAL 30 days) group by w_state,i_item_id order by w_state,i_item_id limit 100""", "q41" -> """ select distinct(i_product_name) from item i1 where i_manufact_id between 732 and 732+40 and (select count(*) as item_cnt from item where (i_manufact = i1.i_manufact and ((i_category = 'Women' and (i_color = 'beige' or i_color = 'spring') and (i_units = 'Tsp' or i_units = 'Ton') and (i_size = 'petite' or i_size = 'extra large') ) or (i_category = 'Women' and (i_color = 'white' or i_color = 'pale') and (i_units = 'Box' or i_units = 'Dram') and (i_size = 'large' or i_size = 'economy') ) or (i_category = 'Men' and (i_color = 'midnight' or i_color = 'frosted') and (i_units = 'Bunch' or i_units = 'Carton') and (i_size = 'small' or i_size = 'N/A') ) or (i_category = 'Men' and (i_color = 'azure' or i_color = 'goldenrod') and (i_units = 'Pallet' or i_units = 'Gross') and (i_size = 'petite' or i_size = 'extra large') ))) or (i_manufact = i1.i_manufact and ((i_category = 'Women' and (i_color = 'brown' or i_color = 'hot') and (i_units = 'Tbl' or i_units = 'Cup') and (i_size = 'petite' or i_size = 'extra large') ) or (i_category = 'Women' and (i_color = 'powder' or i_color = 'honeydew') and (i_units = 'Bundle' or i_units = 'Unknown') and (i_size = 'large' or i_size = 'economy') ) or (i_category = 'Men' and (i_color = 'antique' or i_color = 'purple') and (i_units = 'N/A' or i_units = 'Dozen') and (i_size = 'small' or i_size = 'N/A') ) or (i_category = 'Men' and (i_color = 'lavender' or i_color = 'tomato') and (i_units = 'Lb' or i_units = 'Oz') and (i_size = 'petite' or i_size = 'extra large') )))) > 0 order by i_product_name limit 100""", "q42" -> """ select dt.d_year ,item.i_category_id ,item.i_category ,sum(ss_ext_sales_price) from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manager_id = 1 and dt.d_moy=11 and dt.d_year=2002 group by dt.d_year ,item.i_category_id ,item.i_category order by sum(ss_ext_sales_price) desc,dt.d_year ,item.i_category_id ,item.i_category limit 100 """, "q43" -> """ select s_store_name, s_store_id, sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales from date_dim, store_sales, store where d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk and s_gmt_offset = -6 and d_year = 1999 group by s_store_name, s_store_id order by s_store_name, s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales limit 100""", "q44" -> """ select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing from(select * from (select item_sk,rank() over (order by rank_col asc) rnk from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col from store_sales ss1 where ss_store_sk = 321 group by ss_item_sk having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col from store_sales where ss_store_sk = 321 and ss_addr_sk is null group by ss_store_sk))V1)V11 where rnk < 11) asceding, (select * from (select item_sk,rank() over (order by rank_col desc) rnk from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col from store_sales ss1 where ss_store_sk = 321 group by ss_item_sk having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col from store_sales where ss_store_sk = 321 and ss_addr_sk is null group by ss_store_sk))V2)V21 where rnk < 11) descending, item i1, item i2 where asceding.rnk = descending.rnk and i1.i_item_sk=asceding.item_sk and i2.i_item_sk=descending.item_sk order by asceding.rnk limit 100""", "q45" -> """ select ca_zip, ca_county, sum(ws_sales_price) from web_sales, customer, customer_address, date_dim, item where ws_bill_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and ws_item_sk = i_item_sk and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') or i_item_id in (select i_item_id from item where i_item_sk in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) ) ) and ws_sold_date_sk = d_date_sk and d_qoy = 2 and d_year = 1999 group by ca_zip, ca_county order by ca_zip, ca_county limit 100""", "q46" -> """ select c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number ,amt,profit from (select ss_ticket_number ,ss_customer_sk ,ca_city bought_city ,sum(ss_coupon_amt) amt ,sum(ss_net_profit) profit from store_sales,date_dim,store,household_demographics,customer_address where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and store_sales.ss_addr_sk = customer_address.ca_address_sk and (household_demographics.hd_dep_count = 2 or household_demographics.hd_vehicle_count= 2) and date_dim.d_dow in (6,0) and date_dim.d_year in (1998,1998+1,1998+2) and store.s_city in ('Antioch','Mount Vernon','Jamestown','Wilson','Farmington') group by ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city) dn,customer,customer_address current_addr where ss_customer_sk = c_customer_sk and customer.c_current_addr_sk = current_addr.ca_address_sk and current_addr.ca_city <> bought_city order by c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number limit 100""", "q47" -> """ with v1 as( select i_category, i_brand, s_store_name, s_company_name, d_year, d_moy, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_category, i_brand, s_store_name, s_company_name, d_year) avg_monthly_sales, rank() over (partition by i_category, i_brand, s_store_name, s_company_name order by d_year, d_moy) rn from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ( d_year = 2001 or ( d_year = 2001-1 and d_moy =12) or ( d_year = 2001+1 and d_moy =1) ) group by i_category, i_brand, s_store_name, s_company_name, d_year, d_moy), v2 as( select v1.s_company_name ,v1.d_year, v1.d_moy ,v1.avg_monthly_sales ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum from v1, v1 v1_lag, v1 v1_lead where v1.i_category = v1_lag.i_category and v1.i_category = v1_lead.i_category and v1.i_brand = v1_lag.i_brand and v1.i_brand = v1_lead.i_brand and v1.s_store_name = v1_lag.s_store_name and v1.s_store_name = v1_lead.s_store_name and v1.s_company_name = v1_lag.s_company_name and v1.s_company_name = v1_lead.s_company_name and v1.rn = v1_lag.rn + 1 and v1.rn = v1_lead.rn - 1) select * from v2 where d_year = 2001 and avg_monthly_sales > 0 and case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by sum_sales - avg_monthly_sales, avg_monthly_sales limit 100""", "q48" -> """ select sum (ss_quantity) from store_sales, store, customer_demographics, customer_address, date_dim where s_store_sk = ss_store_sk and ss_sold_date_sk = d_date_sk and d_year = 1999 and ( ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'D' and cd_education_status = 'College' and ss_sales_price between 100.00 and 150.00 ) or ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'W' and cd_education_status = 'Secondary' and ss_sales_price between 50.00 and 100.00 ) or ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'M' and cd_education_status = '2 yr Degree' and ss_sales_price between 150.00 and 200.00 ) ) and ( ( ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('NE', 'IA', 'NY') and ss_net_profit between 0 and 2000 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('IN', 'TN', 'OH') and ss_net_profit between 150 and 3000 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('KS', 'CA', 'CO') and ss_net_profit between 50 and 25000 ) )""", "q49" -> """ select channel, item, return_ratio, return_rank, currency_rank from (select 'web' as channel ,web.item ,web.return_ratio ,web.return_rank ,web.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select ws.ws_item_sk as item ,(cast(sum(coalesce(wr.wr_return_quantity,0)) as decimal(15,4))/ cast(sum(coalesce(ws.ws_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(wr.wr_return_amt,0)) as decimal(15,4))/ cast(sum(coalesce(ws.ws_net_paid,0)) as decimal(15,4) )) as currency_ratio from web_sales ws left outer join web_returns wr on (ws.ws_order_number = wr.wr_order_number and ws.ws_item_sk = wr.wr_item_sk) ,date_dim where wr.wr_return_amt > 10000 and ws.ws_net_profit > 1 and ws.ws_net_paid > 0 and ws.ws_quantity > 0 and ws_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 11 group by ws.ws_item_sk ) in_web ) web where ( web.return_rank <= 10 or web.currency_rank <= 10 ) union select 'catalog' as channel ,catalog.item ,catalog.return_ratio ,catalog.return_rank ,catalog.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select cs.cs_item_sk as item ,(cast(sum(coalesce(cr.cr_return_quantity,0)) as decimal(15,4))/ cast(sum(coalesce(cs.cs_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(cr.cr_return_amount,0)) as decimal(15,4))/ cast(sum(coalesce(cs.cs_net_paid,0)) as decimal(15,4) )) as currency_ratio from catalog_sales cs left outer join catalog_returns cr on (cs.cs_order_number = cr.cr_order_number and cs.cs_item_sk = cr.cr_item_sk) ,date_dim where cr.cr_return_amount > 10000 and cs.cs_net_profit > 1 and cs.cs_net_paid > 0 and cs.cs_quantity > 0 and cs_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 11 group by cs.cs_item_sk ) in_cat ) catalog where ( catalog.return_rank <= 10 or catalog.currency_rank <=10 ) union select 'store' as channel ,store.item ,store.return_ratio ,store.return_rank ,store.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select sts.ss_item_sk as item ,(cast(sum(coalesce(sr.sr_return_quantity,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(sr.sr_return_amt,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_net_paid,0)) as decimal(15,4) )) as currency_ratio from store_sales sts left outer join store_returns sr on (sts.ss_ticket_number = sr.sr_ticket_number and sts.ss_item_sk = sr.sr_item_sk) ,date_dim where sr.sr_return_amt > 10000 and sts.ss_net_profit > 1 and sts.ss_net_paid > 0 and sts.ss_quantity > 0 and ss_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 11 group by sts.ss_item_sk ) in_store ) store where ( store.return_rank <= 10 or store.currency_rank <= 10 ) ) order by 1,4,5,2 limit 100""", "q50" -> """ select s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip ,sum(case when (sr_returned_date_sk - ss_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 30) and (sr_returned_date_sk - ss_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 60) and (sr_returned_date_sk - ss_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 90) and (sr_returned_date_sk - ss_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from store_sales ,store_returns ,store ,date_dim d1 ,date_dim d2 where d2.d_year = 1999 and d2.d_moy = 9 and ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_sold_date_sk = d1.d_date_sk and sr_returned_date_sk = d2.d_date_sk and ss_customer_sk = sr_customer_sk and ss_store_sk = s_store_sk group by s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip order by s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip limit 100""", "q51" -> """ WITH web_v1 as ( select ws_item_sk item_sk, d_date, sum(sum(ws_sales_price)) over (partition by ws_item_sk order by d_date rows between unbounded preceding and current row) cume_sales from web_sales ,date_dim where ws_sold_date_sk=d_date_sk and d_month_seq between 1176 and 1176+11 and ws_item_sk is not NULL group by ws_item_sk, d_date), store_v1 as ( select ss_item_sk item_sk, d_date, sum(sum(ss_sales_price)) over (partition by ss_item_sk order by d_date rows between unbounded preceding and current row) cume_sales from store_sales ,date_dim where ss_sold_date_sk=d_date_sk and d_month_seq between 1176 and 1176+11 and ss_item_sk is not NULL group by ss_item_sk, d_date) select * from (select item_sk ,d_date ,web_sales ,store_sales ,max(web_sales) over (partition by item_sk order by d_date rows between unbounded preceding and current row) web_cumulative ,max(store_sales) over (partition by item_sk order by d_date rows between unbounded preceding and current row) store_cumulative from (select case when web.item_sk is not null then web.item_sk else store.item_sk end item_sk ,case when web.d_date is not null then web.d_date else store.d_date end d_date ,web.cume_sales web_sales ,store.cume_sales store_sales from web_v1 web full outer join store_v1 store on (web.item_sk = store.item_sk and web.d_date = store.d_date) )x )y where web_cumulative > store_cumulative order by item_sk ,d_date limit 100""", "q52" -> """ select dt.d_year ,item.i_brand_id brand_id ,item.i_brand brand ,sum(ss_ext_sales_price) ext_price from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manager_id = 1 and dt.d_moy=11 and dt.d_year=2001 group by dt.d_year ,item.i_brand ,item.i_brand_id order by dt.d_year ,ext_price desc ,brand_id limit 100 """, "q53" -> """ select * from (select i_manufact_id, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_manufact_id) avg_quarterly_sales from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_month_seq in (1218,1218+1,1218+2,1218+3,1218+4,1218+5,1218+6,1218+7,1218+8,1218+9,1218+10,1218+11) and ((i_category in ('Books','Children','Electronics') and i_class in ('personal','portable','reference','self-help') and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', 'exportiunivamalg #9','scholaramalgamalg #9')) or(i_category in ('Women','Music','Men') and i_class in ('accessories','classical','fragrances','pants') and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 'importoamalg #1'))) group by i_manufact_id, d_qoy ) tmp1 where case when avg_quarterly_sales > 0 then abs (sum_sales - avg_quarterly_sales)/ avg_quarterly_sales else null end > 0.1 order by avg_quarterly_sales, sum_sales, i_manufact_id limit 100""", "q54" -> """ with my_customers as ( select distinct c_customer_sk , c_current_addr_sk from ( select cs_sold_date_sk sold_date_sk, cs_bill_customer_sk customer_sk, cs_item_sk item_sk from catalog_sales union all select ws_sold_date_sk sold_date_sk, ws_bill_customer_sk customer_sk, ws_item_sk item_sk from web_sales ) cs_or_ws_sales, item, date_dim, customer where sold_date_sk = d_date_sk and item_sk = i_item_sk and i_category = 'Music' and i_class = 'country' and c_customer_sk = cs_or_ws_sales.customer_sk and d_moy = 7 and d_year = 2001 ) , my_revenue as ( select c_customer_sk, sum(ss_ext_sales_price) as revenue from my_customers, store_sales, customer_address, store, date_dim where c_current_addr_sk = ca_address_sk and ca_county = s_county and ca_state = s_state and ss_sold_date_sk = d_date_sk and c_customer_sk = ss_customer_sk and d_month_seq between (select distinct d_month_seq+1 from date_dim where d_year = 2001 and d_moy = 7) and (select distinct d_month_seq+3 from date_dim where d_year = 2001 and d_moy = 7) group by c_customer_sk ) , segments as (select cast((revenue/50) as int) as segment from my_revenue ) select segment, count(*) as num_customers, segment*50 as segment_base from segments group by segment order by segment, num_customers limit 100""", "q55" -> """ select i_brand_id brand_id, i_brand brand, sum(ss_ext_sales_price) ext_price from date_dim, store_sales, item where d_date_sk = ss_sold_date_sk and ss_item_sk = i_item_sk and i_manager_id=87 and d_moy=11 and d_year=2001 group by i_brand, i_brand_id order by ext_price desc, i_brand_id limit 100 """, "q56" -> """ with ss as ( select i_item_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('tan','lace','gainsboro')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 3 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id), cs as ( select i_item_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('tan','lace','gainsboro')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 3 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id), ws as ( select i_item_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('tan','lace','gainsboro')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 3 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id) select i_item_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_item_id order by total_sales, i_item_id limit 100""", "q57" -> """ with v1 as( select i_category, i_brand, cc_name, d_year, d_moy, sum(cs_sales_price) sum_sales, avg(sum(cs_sales_price)) over (partition by i_category, i_brand, cc_name, d_year) avg_monthly_sales, rank() over (partition by i_category, i_brand, cc_name order by d_year, d_moy) rn from item, catalog_sales, date_dim, call_center where cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and cc_call_center_sk= cs_call_center_sk and ( d_year = 2001 or ( d_year = 2001-1 and d_moy =12) or ( d_year = 2001+1 and d_moy =1) ) group by i_category, i_brand, cc_name , d_year, d_moy), v2 as( select v1.i_category, v1.i_brand, v1.cc_name ,v1.d_year, v1.d_moy ,v1.avg_monthly_sales ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum from v1, v1 v1_lag, v1 v1_lead where v1.i_category = v1_lag.i_category and v1.i_category = v1_lead.i_category and v1.i_brand = v1_lag.i_brand and v1.i_brand = v1_lead.i_brand and v1. cc_name = v1_lag. cc_name and v1. cc_name = v1_lead. cc_name and v1.rn = v1_lag.rn + 1 and v1.rn = v1_lead.rn - 1) select * from v2 where d_year = 2001 and avg_monthly_sales > 0 and case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by sum_sales - avg_monthly_sales, avg_monthly_sales limit 100""", "q58" -> """ with ss_items as (select i_item_id item_id ,sum(ss_ext_sales_price) ss_item_rev from store_sales ,item ,date_dim where ss_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq = (select d_week_seq from date_dim where d_date = '2000-03-26')) and ss_sold_date_sk = d_date_sk group by i_item_id), cs_items as (select i_item_id item_id ,sum(cs_ext_sales_price) cs_item_rev from catalog_sales ,item ,date_dim where cs_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq = (select d_week_seq from date_dim where d_date = '2000-03-26')) and cs_sold_date_sk = d_date_sk group by i_item_id), ws_items as (select i_item_id item_id ,sum(ws_ext_sales_price) ws_item_rev from web_sales ,item ,date_dim where ws_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq =(select d_week_seq from date_dim where d_date = '2000-03-26')) and ws_sold_date_sk = d_date_sk group by i_item_id) select ss_items.item_id ,ss_item_rev ,ss_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ss_dev ,cs_item_rev ,cs_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 cs_dev ,ws_item_rev ,ws_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ws_dev ,(ss_item_rev+cs_item_rev+ws_item_rev)/3 average from ss_items,cs_items,ws_items where ss_items.item_id=cs_items.item_id and ss_items.item_id=ws_items.item_id and ss_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev and ss_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev and cs_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev and cs_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev and ws_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev and ws_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev order by item_id ,ss_item_rev limit 100""", "q59" -> """ with wss as (select d_week_seq, ss_store_sk, sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales from store_sales,date_dim where d_date_sk = ss_sold_date_sk group by d_week_seq,ss_store_sk ) select s_store_name1,s_store_id1,d_week_seq1 ,sun_sales1/sun_sales2,mon_sales1/mon_sales2 ,tue_sales1/tue_sales2,wed_sales1/wed_sales2,thu_sales1/thu_sales2 ,fri_sales1/fri_sales2,sat_sales1/sat_sales2 from (select s_store_name s_store_name1,wss.d_week_seq d_week_seq1 ,s_store_id s_store_id1,sun_sales sun_sales1 ,mon_sales mon_sales1,tue_sales tue_sales1 ,wed_sales wed_sales1,thu_sales thu_sales1 ,fri_sales fri_sales1,sat_sales sat_sales1 from wss,store,date_dim d where d.d_week_seq = wss.d_week_seq and ss_store_sk = s_store_sk and d_month_seq between 1199 and 1199 + 11) y, (select s_store_name s_store_name2,wss.d_week_seq d_week_seq2 ,s_store_id s_store_id2,sun_sales sun_sales2 ,mon_sales mon_sales2,tue_sales tue_sales2 ,wed_sales wed_sales2,thu_sales thu_sales2 ,fri_sales fri_sales2,sat_sales sat_sales2 from wss,store,date_dim d where d.d_week_seq = wss.d_week_seq and ss_store_sk = s_store_sk and d_month_seq between 1199+ 12 and 1199 + 23) x where s_store_id1=s_store_id2 and d_week_seq1=d_week_seq2-52 order by s_store_name1,s_store_id1,d_week_seq1 limit 100""", "q60" -> """ with ss as ( select i_item_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Men')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 9 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id), cs as ( select i_item_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Men')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 9 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id), ws as ( select i_item_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Men')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 9 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id) select i_item_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_item_id order by i_item_id ,total_sales limit 100""", "q61" -> """ select promotions,total,cast(promotions as decimal(15,4))/cast(total as decimal(15,4))*100 from (select sum(ss_ext_sales_price) promotions from store_sales ,store ,promotion ,date_dim ,customer ,customer_address ,item where ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ss_promo_sk = p_promo_sk and ss_customer_sk= c_customer_sk and ca_address_sk = c_current_addr_sk and ss_item_sk = i_item_sk and ca_gmt_offset = -7 and i_category = 'Electronics' and (p_channel_dmail = 'Y' or p_channel_email = 'Y' or p_channel_tv = 'Y') and s_gmt_offset = -7 and d_year = 2001 and d_moy = 11) promotional_sales, (select sum(ss_ext_sales_price) total from store_sales ,store ,date_dim ,customer ,customer_address ,item where ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ss_customer_sk= c_customer_sk and ca_address_sk = c_current_addr_sk and ss_item_sk = i_item_sk and ca_gmt_offset = -7 and i_category = 'Electronics' and s_gmt_offset = -7 and d_year = 2001 and d_moy = 11) all_sales order by promotions, total limit 100""", "q62" -> """ select substr(w_warehouse_name,1,20) ,sm_type ,web_name ,sum(case when (ws_ship_date_sk - ws_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 30) and (ws_ship_date_sk - ws_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 60) and (ws_ship_date_sk - ws_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 90) and (ws_ship_date_sk - ws_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from web_sales ,warehouse ,ship_mode ,web_site ,date_dim where d_month_seq between 1194 and 1194 + 11 and ws_ship_date_sk = d_date_sk and ws_warehouse_sk = w_warehouse_sk and ws_ship_mode_sk = sm_ship_mode_sk and ws_web_site_sk = web_site_sk group by substr(w_warehouse_name,1,20) ,sm_type ,web_name order by substr(w_warehouse_name,1,20) ,sm_type ,web_name limit 100""", "q63" -> """ select * from (select i_manager_id ,sum(ss_sales_price) sum_sales ,avg(sum(ss_sales_price)) over (partition by i_manager_id) avg_monthly_sales from item ,store_sales ,date_dim ,store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_month_seq in (1205,1205+1,1205+2,1205+3,1205+4,1205+5,1205+6,1205+7,1205+8,1205+9,1205+10,1205+11) and (( i_category in ('Books','Children','Electronics') and i_class in ('personal','portable','reference','self-help') and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', 'exportiunivamalg #9','scholaramalgamalg #9')) or( i_category in ('Women','Music','Men') and i_class in ('accessories','classical','fragrances','pants') and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 'importoamalg #1'))) group by i_manager_id, d_moy) tmp1 where case when avg_monthly_sales > 0 then abs (sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by i_manager_id ,avg_monthly_sales ,sum_sales limit 100""", "q64" -> """ with cs_ui as (select cs_item_sk ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund from catalog_sales ,catalog_returns where cs_item_sk = cr_item_sk and cs_order_number = cr_order_number group by cs_item_sk having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), cross_sales as (select i_product_name product_name ,i_item_sk item_sk ,s_store_name store_name ,s_zip store_zip ,ad1.ca_street_number b_street_number ,ad1.ca_street_name b_street_name ,ad1.ca_city b_city ,ad1.ca_zip b_zip ,ad2.ca_street_number c_street_number ,ad2.ca_street_name c_street_name ,ad2.ca_city c_city ,ad2.ca_zip c_zip ,d1.d_year as syear ,d2.d_year as fsyear ,d3.d_year s2year ,count(*) cnt ,sum(ss_wholesale_cost) s1 ,sum(ss_list_price) s2 ,sum(ss_coupon_amt) s3 FROM store_sales ,store_returns ,cs_ui ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,customer ,customer_demographics cd1 ,customer_demographics cd2 ,promotion ,household_demographics hd1 ,household_demographics hd2 ,customer_address ad1 ,customer_address ad2 ,income_band ib1 ,income_band ib2 ,item WHERE ss_store_sk = s_store_sk AND ss_sold_date_sk = d1.d_date_sk AND ss_customer_sk = c_customer_sk AND ss_cdemo_sk= cd1.cd_demo_sk AND ss_hdemo_sk = hd1.hd_demo_sk AND ss_addr_sk = ad1.ca_address_sk and ss_item_sk = i_item_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and ss_item_sk = cs_ui.cs_item_sk and c_current_cdemo_sk = cd2.cd_demo_sk AND c_current_hdemo_sk = hd2.hd_demo_sk AND c_current_addr_sk = ad2.ca_address_sk and c_first_sales_date_sk = d2.d_date_sk and c_first_shipto_date_sk = d3.d_date_sk and ss_promo_sk = p_promo_sk and hd1.hd_income_band_sk = ib1.ib_income_band_sk and hd2.hd_income_band_sk = ib2.ib_income_band_sk and cd1.cd_marital_status <> cd2.cd_marital_status and i_color in ('peach','misty','drab','chocolate','almond','saddle') and i_current_price between 75 and 75 + 10 and i_current_price between 75 + 1 and 75 + 15 group by i_product_name ,i_item_sk ,s_store_name ,s_zip ,ad1.ca_street_number ,ad1.ca_street_name ,ad1.ca_city ,ad1.ca_zip ,ad2.ca_street_number ,ad2.ca_street_name ,ad2.ca_city ,ad2.ca_zip ,d1.d_year ,d2.d_year ,d3.d_year ) select cs1.product_name ,cs1.store_name ,cs1.store_zip ,cs1.b_street_number ,cs1.b_street_name ,cs1.b_city ,cs1.b_zip ,cs1.c_street_number ,cs1.c_street_name ,cs1.c_city ,cs1.c_zip ,cs1.syear ,cs1.cnt ,cs1.s1 as s11 ,cs1.s2 as s21 ,cs1.s3 as s31 ,cs2.s1 as s12 ,cs2.s2 as s22 ,cs2.s3 as s32 ,cs2.syear ,cs2.cnt from cross_sales cs1,cross_sales cs2 where cs1.item_sk=cs2.item_sk and cs1.syear = 2000 and cs2.syear = 2000 + 1 and cs2.cnt <= cs1.cnt and cs1.store_name = cs2.store_name and cs1.store_zip = cs2.store_zip order by cs1.product_name ,cs1.store_name ,cs2.cnt ,cs1.s1 ,cs2.s1""", "q65" -> """ select s_store_name, i_item_desc, sc.revenue, i_current_price, i_wholesale_cost, i_brand from store, item, (select ss_store_sk, avg(revenue) as ave from (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue from store_sales, date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1208 and 1208+11 group by ss_store_sk, ss_item_sk) sa group by ss_store_sk) sb, (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue from store_sales, date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1208 and 1208+11 group by ss_store_sk, ss_item_sk) sc where sb.ss_store_sk = sc.ss_store_sk and sc.revenue <= 0.1 * sb.ave and s_store_sk = sc.ss_store_sk and i_item_sk = sc.ss_item_sk order by s_store_name, i_item_desc limit 100""", "q66" -> """ select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,ship_carriers ,year ,sum(jan_sales) as jan_sales ,sum(feb_sales) as feb_sales ,sum(mar_sales) as mar_sales ,sum(apr_sales) as apr_sales ,sum(may_sales) as may_sales ,sum(jun_sales) as jun_sales ,sum(jul_sales) as jul_sales ,sum(aug_sales) as aug_sales ,sum(sep_sales) as sep_sales ,sum(oct_sales) as oct_sales ,sum(nov_sales) as nov_sales ,sum(dec_sales) as dec_sales ,sum(jan_sales/w_warehouse_sq_ft) as jan_sales_per_sq_foot ,sum(feb_sales/w_warehouse_sq_ft) as feb_sales_per_sq_foot ,sum(mar_sales/w_warehouse_sq_ft) as mar_sales_per_sq_foot ,sum(apr_sales/w_warehouse_sq_ft) as apr_sales_per_sq_foot ,sum(may_sales/w_warehouse_sq_ft) as may_sales_per_sq_foot ,sum(jun_sales/w_warehouse_sq_ft) as jun_sales_per_sq_foot ,sum(jul_sales/w_warehouse_sq_ft) as jul_sales_per_sq_foot ,sum(aug_sales/w_warehouse_sq_ft) as aug_sales_per_sq_foot ,sum(sep_sales/w_warehouse_sq_ft) as sep_sales_per_sq_foot ,sum(oct_sales/w_warehouse_sq_ft) as oct_sales_per_sq_foot ,sum(nov_sales/w_warehouse_sq_ft) as nov_sales_per_sq_foot ,sum(dec_sales/w_warehouse_sq_ft) as dec_sales_per_sq_foot ,sum(jan_net) as jan_net ,sum(feb_net) as feb_net ,sum(mar_net) as mar_net ,sum(apr_net) as apr_net ,sum(may_net) as may_net ,sum(jun_net) as jun_net ,sum(jul_net) as jul_net ,sum(aug_net) as aug_net ,sum(sep_net) as sep_net ,sum(oct_net) as oct_net ,sum(nov_net) as nov_net ,sum(dec_net) as dec_net from ( select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,'HARMSTORF' || ',' || 'USPS' as ship_carriers ,d_year as year ,sum(case when d_moy = 1 then ws_sales_price* ws_quantity else 0 end) as jan_sales ,sum(case when d_moy = 2 then ws_sales_price* ws_quantity else 0 end) as feb_sales ,sum(case when d_moy = 3 then ws_sales_price* ws_quantity else 0 end) as mar_sales ,sum(case when d_moy = 4 then ws_sales_price* ws_quantity else 0 end) as apr_sales ,sum(case when d_moy = 5 then ws_sales_price* ws_quantity else 0 end) as may_sales ,sum(case when d_moy = 6 then ws_sales_price* ws_quantity else 0 end) as jun_sales ,sum(case when d_moy = 7 then ws_sales_price* ws_quantity else 0 end) as jul_sales ,sum(case when d_moy = 8 then ws_sales_price* ws_quantity else 0 end) as aug_sales ,sum(case when d_moy = 9 then ws_sales_price* ws_quantity else 0 end) as sep_sales ,sum(case when d_moy = 10 then ws_sales_price* ws_quantity else 0 end) as oct_sales ,sum(case when d_moy = 11 then ws_sales_price* ws_quantity else 0 end) as nov_sales ,sum(case when d_moy = 12 then ws_sales_price* ws_quantity else 0 end) as dec_sales ,sum(case when d_moy = 1 then ws_net_paid_inc_tax * ws_quantity else 0 end) as jan_net ,sum(case when d_moy = 2 then ws_net_paid_inc_tax * ws_quantity else 0 end) as feb_net ,sum(case when d_moy = 3 then ws_net_paid_inc_tax * ws_quantity else 0 end) as mar_net ,sum(case when d_moy = 4 then ws_net_paid_inc_tax * ws_quantity else 0 end) as apr_net ,sum(case when d_moy = 5 then ws_net_paid_inc_tax * ws_quantity else 0 end) as may_net ,sum(case when d_moy = 6 then ws_net_paid_inc_tax * ws_quantity else 0 end) as jun_net ,sum(case when d_moy = 7 then ws_net_paid_inc_tax * ws_quantity else 0 end) as jul_net ,sum(case when d_moy = 8 then ws_net_paid_inc_tax * ws_quantity else 0 end) as aug_net ,sum(case when d_moy = 9 then ws_net_paid_inc_tax * ws_quantity else 0 end) as sep_net ,sum(case when d_moy = 10 then ws_net_paid_inc_tax * ws_quantity else 0 end) as oct_net ,sum(case when d_moy = 11 then ws_net_paid_inc_tax * ws_quantity else 0 end) as nov_net ,sum(case when d_moy = 12 then ws_net_paid_inc_tax * ws_quantity else 0 end) as dec_net from web_sales ,warehouse ,date_dim ,time_dim ,ship_mode where ws_warehouse_sk = w_warehouse_sk and ws_sold_date_sk = d_date_sk and ws_sold_time_sk = t_time_sk and ws_ship_mode_sk = sm_ship_mode_sk and d_year = 2002 and t_time between 24285 and 24285+28800 and sm_carrier in ('HARMSTORF','USPS') group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,d_year union all select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,'HARMSTORF' || ',' || 'USPS' as ship_carriers ,d_year as year ,sum(case when d_moy = 1 then cs_ext_list_price* cs_quantity else 0 end) as jan_sales ,sum(case when d_moy = 2 then cs_ext_list_price* cs_quantity else 0 end) as feb_sales ,sum(case when d_moy = 3 then cs_ext_list_price* cs_quantity else 0 end) as mar_sales ,sum(case when d_moy = 4 then cs_ext_list_price* cs_quantity else 0 end) as apr_sales ,sum(case when d_moy = 5 then cs_ext_list_price* cs_quantity else 0 end) as may_sales ,sum(case when d_moy = 6 then cs_ext_list_price* cs_quantity else 0 end) as jun_sales ,sum(case when d_moy = 7 then cs_ext_list_price* cs_quantity else 0 end) as jul_sales ,sum(case when d_moy = 8 then cs_ext_list_price* cs_quantity else 0 end) as aug_sales ,sum(case when d_moy = 9 then cs_ext_list_price* cs_quantity else 0 end) as sep_sales ,sum(case when d_moy = 10 then cs_ext_list_price* cs_quantity else 0 end) as oct_sales ,sum(case when d_moy = 11 then cs_ext_list_price* cs_quantity else 0 end) as nov_sales ,sum(case when d_moy = 12 then cs_ext_list_price* cs_quantity else 0 end) as dec_sales ,sum(case when d_moy = 1 then cs_net_paid * cs_quantity else 0 end) as jan_net ,sum(case when d_moy = 2 then cs_net_paid * cs_quantity else 0 end) as feb_net ,sum(case when d_moy = 3 then cs_net_paid * cs_quantity else 0 end) as mar_net ,sum(case when d_moy = 4 then cs_net_paid * cs_quantity else 0 end) as apr_net ,sum(case when d_moy = 5 then cs_net_paid * cs_quantity else 0 end) as may_net ,sum(case when d_moy = 6 then cs_net_paid * cs_quantity else 0 end) as jun_net ,sum(case when d_moy = 7 then cs_net_paid * cs_quantity else 0 end) as jul_net ,sum(case when d_moy = 8 then cs_net_paid * cs_quantity else 0 end) as aug_net ,sum(case when d_moy = 9 then cs_net_paid * cs_quantity else 0 end) as sep_net ,sum(case when d_moy = 10 then cs_net_paid * cs_quantity else 0 end) as oct_net ,sum(case when d_moy = 11 then cs_net_paid * cs_quantity else 0 end) as nov_net ,sum(case when d_moy = 12 then cs_net_paid * cs_quantity else 0 end) as dec_net from catalog_sales ,warehouse ,date_dim ,time_dim ,ship_mode where cs_warehouse_sk = w_warehouse_sk and cs_sold_date_sk = d_date_sk and cs_sold_time_sk = t_time_sk and cs_ship_mode_sk = sm_ship_mode_sk and d_year = 2002 and t_time between 24285 AND 24285+28800 and sm_carrier in ('HARMSTORF','USPS') group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,d_year ) x group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,ship_carriers ,year order by w_warehouse_name limit 100""", "q67" -> """ select * from (select i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sumsales ,rank() over (partition by i_category order by sumsales desc) rk from (select i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sum(coalesce(ss_sales_price*ss_quantity,0)) sumsales from store_sales ,date_dim ,store ,item where ss_sold_date_sk=d_date_sk and ss_item_sk=i_item_sk and ss_store_sk = s_store_sk and d_month_seq between 1196 and 1196+11 group by rollup(i_category, i_class, i_brand, i_product_name, d_year, d_qoy, d_moy,s_store_id))dw1) dw2 where rk <= 100 order by i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sumsales ,rk limit 100""", "q68" -> """ select c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number ,extended_price ,extended_tax ,list_price from (select ss_ticket_number ,ss_customer_sk ,ca_city bought_city ,sum(ss_ext_sales_price) extended_price ,sum(ss_ext_list_price) list_price ,sum(ss_ext_tax) extended_tax from store_sales ,date_dim ,store ,household_demographics ,customer_address where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and store_sales.ss_addr_sk = customer_address.ca_address_sk and date_dim.d_dom between 1 and 2 and (household_demographics.hd_dep_count = 1 or household_demographics.hd_vehicle_count= -1) and date_dim.d_year in (1998,1998+1,1998+2) and store.s_city in ('Bethel','Summit') group by ss_ticket_number ,ss_customer_sk ,ss_addr_sk,ca_city) dn ,customer ,customer_address current_addr where ss_customer_sk = c_customer_sk and customer.c_current_addr_sk = current_addr.ca_address_sk and current_addr.ca_city <> bought_city order by c_last_name ,ss_ticket_number limit 100""", "q69" -> """ select cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3 from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and ca_state in ('OK','GA','VA') and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2004 and d_moy between 4 and 4+2) and (not exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2004 and d_moy between 4 and 4+2) and not exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2004 and d_moy between 4 and 4+2)) group by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating order by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating limit 100""", "q70" -> """ select sum(ss_net_profit) as total_sum ,s_state ,s_county ,grouping(s_state)+grouping(s_county) as lochierarchy ,rank() over ( partition by grouping(s_state)+grouping(s_county), case when grouping(s_county) = 0 then s_state end order by sum(ss_net_profit) desc) as rank_within_parent from store_sales ,date_dim d1 ,store where d1.d_month_seq between 1197 and 1197+11 and d1.d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk and s_state in ( select s_state from (select s_state as s_state, rank() over ( partition by s_state order by sum(ss_net_profit) desc) as ranking from store_sales, store, date_dim where d_month_seq between 1197 and 1197+11 and d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk group by s_state ) tmp1 where ranking <= 5 ) group by rollup(s_state,s_county) order by lochierarchy desc ,case when lochierarchy = 0 then s_state end ,rank_within_parent limit 100""", "q71" -> """ select i_brand_id brand_id, i_brand brand,t_hour,t_minute, sum(ext_price) ext_price from item, (select ws_ext_sales_price as ext_price, ws_sold_date_sk as sold_date_sk, ws_item_sk as sold_item_sk, ws_sold_time_sk as time_sk from web_sales,date_dim where d_date_sk = ws_sold_date_sk and d_moy=12 and d_year=1999 union all select cs_ext_sales_price as ext_price, cs_sold_date_sk as sold_date_sk, cs_item_sk as sold_item_sk, cs_sold_time_sk as time_sk from catalog_sales,date_dim where d_date_sk = cs_sold_date_sk and d_moy=12 and d_year=1999 union all select ss_ext_sales_price as ext_price, ss_sold_date_sk as sold_date_sk, ss_item_sk as sold_item_sk, ss_sold_time_sk as time_sk from store_sales,date_dim where d_date_sk = ss_sold_date_sk and d_moy=12 and d_year=1999 ) tmp,time_dim where sold_item_sk = i_item_sk and i_manager_id=1 and time_sk = t_time_sk and (t_meal_time = 'breakfast' or t_meal_time = 'dinner') group by i_brand, i_brand_id,t_hour,t_minute order by ext_price desc, i_brand_id """, "q72" -> """ select i_item_desc ,w_warehouse_name ,d1.d_week_seq ,sum(case when p_promo_sk is null then 1 else 0 end) no_promo ,sum(case when p_promo_sk is not null then 1 else 0 end) promo ,count(*) total_cnt from catalog_sales join inventory on (cs_item_sk = inv_item_sk) join warehouse on (w_warehouse_sk=inv_warehouse_sk) join item on (i_item_sk = cs_item_sk) join customer_demographics on (cs_bill_cdemo_sk = cd_demo_sk) join household_demographics on (cs_bill_hdemo_sk = hd_demo_sk) join date_dim d1 on (cs_sold_date_sk = d1.d_date_sk) join date_dim d2 on (inv_date_sk = d2.d_date_sk) join date_dim d3 on (cs_ship_date_sk = d3.d_date_sk) left outer join promotion on (cs_promo_sk=p_promo_sk) left outer join catalog_returns on (cr_item_sk = cs_item_sk and cr_order_number = cs_order_number) where d1.d_week_seq = d2.d_week_seq and inv_quantity_on_hand < cs_quantity and d3.d_date > d1.d_date + interval 5 days and hd_buy_potential = '>10000' and d1.d_year = 2002 and cd_marital_status = 'D' group by i_item_desc,w_warehouse_name,d1.d_week_seq order by total_cnt desc, i_item_desc, w_warehouse_name, d_week_seq limit 100""", "q73" -> """ select c_last_name ,c_first_name ,c_salutation ,c_preferred_cust_flag ,ss_ticket_number ,cnt from (select ss_ticket_number ,ss_customer_sk ,count(*) cnt from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and date_dim.d_dom between 1 and 2 and (household_demographics.hd_buy_potential = '501-1000' or household_demographics.hd_buy_potential = 'Unknown') and household_demographics.hd_vehicle_count > 0 and case when household_demographics.hd_vehicle_count > 0 then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end > 1 and date_dim.d_year in (1999,1999+1,1999+2) and store.s_county in ('Franklin Parish','Ziebach County','Luce County','Williamson County') group by ss_ticket_number,ss_customer_sk) dj,customer where ss_customer_sk = c_customer_sk and cnt between 1 and 5 order by cnt desc, c_last_name asc""", "q74" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,d_year as year ,max(ss_net_paid) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (2001,2001+1) group by c_customer_id ,c_first_name ,c_last_name ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,d_year as year ,max(ws_net_paid) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year in (2001,2001+1) group by c_customer_id ,c_first_name ,c_last_name ,d_year ) select t_s_secyear.customer_id, t_s_secyear.customer_first_name, t_s_secyear.customer_last_name from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.sale_type = 's' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_w_secyear.sale_type = 'w' and t_s_firstyear.year = 2001 and t_s_secyear.year = 2001+1 and t_w_firstyear.year = 2001 and t_w_secyear.year = 2001+1 and t_s_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end order by 3,1,2 limit 100""", "q75" -> """ WITH all_sales AS ( SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,SUM(sales_cnt) AS sales_cnt ,SUM(sales_amt) AS sales_amt FROM (SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,cs_quantity - COALESCE(cr_return_quantity,0) AS sales_cnt ,cs_ext_sales_price - COALESCE(cr_return_amount,0.0) AS sales_amt FROM catalog_sales JOIN item ON i_item_sk=cs_item_sk JOIN date_dim ON d_date_sk=cs_sold_date_sk LEFT JOIN catalog_returns ON (cs_order_number=cr_order_number AND cs_item_sk=cr_item_sk) WHERE i_category='Sports' UNION SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,ss_quantity - COALESCE(sr_return_quantity,0) AS sales_cnt ,ss_ext_sales_price - COALESCE(sr_return_amt,0.0) AS sales_amt FROM store_sales JOIN item ON i_item_sk=ss_item_sk JOIN date_dim ON d_date_sk=ss_sold_date_sk LEFT JOIN store_returns ON (ss_ticket_number=sr_ticket_number AND ss_item_sk=sr_item_sk) WHERE i_category='Sports' UNION SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,ws_quantity - COALESCE(wr_return_quantity,0) AS sales_cnt ,ws_ext_sales_price - COALESCE(wr_return_amt,0.0) AS sales_amt FROM web_sales JOIN item ON i_item_sk=ws_item_sk JOIN date_dim ON d_date_sk=ws_sold_date_sk LEFT JOIN web_returns ON (ws_order_number=wr_order_number AND ws_item_sk=wr_item_sk) WHERE i_category='Sports') sales_detail GROUP BY d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id) SELECT prev_yr.d_year AS prev_year ,curr_yr.d_year AS year ,curr_yr.i_brand_id ,curr_yr.i_class_id ,curr_yr.i_category_id ,curr_yr.i_manufact_id ,prev_yr.sales_cnt AS prev_yr_cnt ,curr_yr.sales_cnt AS curr_yr_cnt ,curr_yr.sales_cnt-prev_yr.sales_cnt AS sales_cnt_diff ,curr_yr.sales_amt-prev_yr.sales_amt AS sales_amt_diff FROM all_sales curr_yr, all_sales prev_yr WHERE curr_yr.i_brand_id=prev_yr.i_brand_id AND curr_yr.i_class_id=prev_yr.i_class_id AND curr_yr.i_category_id=prev_yr.i_category_id AND curr_yr.i_manufact_id=prev_yr.i_manufact_id AND curr_yr.d_year=2001 AND prev_yr.d_year=2001-1 AND CAST(curr_yr.sales_cnt AS DECIMAL(17,2))/CAST(prev_yr.sales_cnt AS DECIMAL(17,2))<0.9 ORDER BY sales_cnt_diff,sales_amt_diff limit 100""", "q76" -> """ select channel, col_name, d_year, d_qoy, i_category, COUNT(*) sales_cnt, SUM(ext_sales_price) sales_amt FROM ( SELECT 'store' as channel, 'ss_cdemo_sk' col_name, d_year, d_qoy, i_category, ss_ext_sales_price ext_sales_price FROM store_sales, item, date_dim WHERE ss_cdemo_sk IS NULL AND ss_sold_date_sk=d_date_sk AND ss_item_sk=i_item_sk UNION ALL SELECT 'web' as channel, 'ws_ship_hdemo_sk' col_name, d_year, d_qoy, i_category, ws_ext_sales_price ext_sales_price FROM web_sales, item, date_dim WHERE ws_ship_hdemo_sk IS NULL AND ws_sold_date_sk=d_date_sk AND ws_item_sk=i_item_sk UNION ALL SELECT 'catalog' as channel, 'cs_ship_customer_sk' col_name, d_year, d_qoy, i_category, cs_ext_sales_price ext_sales_price FROM catalog_sales, item, date_dim WHERE cs_ship_customer_sk IS NULL AND cs_sold_date_sk=d_date_sk AND cs_item_sk=i_item_sk) foo GROUP BY channel, col_name, d_year, d_qoy, i_category ORDER BY channel, col_name, d_year, d_qoy, i_category limit 100""", "q77" -> """ with ss as (select s_store_sk, sum(ss_ext_sales_price) as sales, sum(ss_net_profit) as profit from store_sales, date_dim, store where ss_sold_date_sk = d_date_sk and d_date between cast('2001-08-27' as date) and (cast('2001-08-27' as date) + INTERVAL 30 days) and ss_store_sk = s_store_sk group by s_store_sk) , sr as (select s_store_sk, sum(sr_return_amt) as returns, sum(sr_net_loss) as profit_loss from store_returns, date_dim, store where sr_returned_date_sk = d_date_sk and d_date between cast('2001-08-27' as date) and (cast('2001-08-27' as date) + INTERVAL 30 days) and sr_store_sk = s_store_sk group by s_store_sk), cs as (select cs_call_center_sk, sum(cs_ext_sales_price) as sales, sum(cs_net_profit) as profit from catalog_sales, date_dim where cs_sold_date_sk = d_date_sk and d_date between cast('2001-08-27' as date) and (cast('2001-08-27' as date) + INTERVAL 30 days) group by cs_call_center_sk ), cr as (select cr_call_center_sk, sum(cr_return_amount) as returns, sum(cr_net_loss) as profit_loss from catalog_returns, date_dim where cr_returned_date_sk = d_date_sk and d_date between cast('2001-08-27' as date) and (cast('2001-08-27' as date) + INTERVAL 30 days) group by cr_call_center_sk ), ws as ( select wp_web_page_sk, sum(ws_ext_sales_price) as sales, sum(ws_net_profit) as profit from web_sales, date_dim, web_page where ws_sold_date_sk = d_date_sk and d_date between cast('2001-08-27' as date) and (cast('2001-08-27' as date) + INTERVAL 30 days) and ws_web_page_sk = wp_web_page_sk group by wp_web_page_sk), wr as (select wp_web_page_sk, sum(wr_return_amt) as returns, sum(wr_net_loss) as profit_loss from web_returns, date_dim, web_page where wr_returned_date_sk = d_date_sk and d_date between cast('2001-08-27' as date) and (cast('2001-08-27' as date) + INTERVAL 30 days) and wr_web_page_sk = wp_web_page_sk group by wp_web_page_sk) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , ss.s_store_sk as id , sales , coalesce(returns, 0) as returns , (profit - coalesce(profit_loss,0)) as profit from ss left join sr on ss.s_store_sk = sr.s_store_sk union all select 'catalog channel' as channel , cs_call_center_sk as id , sales , returns , (profit - profit_loss) as profit from cs , cr union all select 'web channel' as channel , ws.wp_web_page_sk as id , sales , coalesce(returns, 0) returns , (profit - coalesce(profit_loss,0)) as profit from ws left join wr on ws.wp_web_page_sk = wr.wp_web_page_sk ) x group by rollup (channel, id) order by channel ,id limit 100""", "q78" -> """ with ws as (select d_year AS ws_sold_year, ws_item_sk, ws_bill_customer_sk ws_customer_sk, sum(ws_quantity) ws_qty, sum(ws_wholesale_cost) ws_wc, sum(ws_sales_price) ws_sp from web_sales left join web_returns on wr_order_number=ws_order_number and ws_item_sk=wr_item_sk join date_dim on ws_sold_date_sk = d_date_sk where wr_order_number is null group by d_year, ws_item_sk, ws_bill_customer_sk ), cs as (select d_year AS cs_sold_year, cs_item_sk, cs_bill_customer_sk cs_customer_sk, sum(cs_quantity) cs_qty, sum(cs_wholesale_cost) cs_wc, sum(cs_sales_price) cs_sp from catalog_sales left join catalog_returns on cr_order_number=cs_order_number and cs_item_sk=cr_item_sk join date_dim on cs_sold_date_sk = d_date_sk where cr_order_number is null group by d_year, cs_item_sk, cs_bill_customer_sk ), ss as (select d_year AS ss_sold_year, ss_item_sk, ss_customer_sk, sum(ss_quantity) ss_qty, sum(ss_wholesale_cost) ss_wc, sum(ss_sales_price) ss_sp from store_sales left join store_returns on sr_ticket_number=ss_ticket_number and ss_item_sk=sr_item_sk join date_dim on ss_sold_date_sk = d_date_sk where sr_ticket_number is null group by d_year, ss_item_sk, ss_customer_sk ) select ss_sold_year, ss_item_sk, ss_customer_sk, round(ss_qty/(coalesce(ws_qty,0)+coalesce(cs_qty,0)),2) ratio, ss_qty store_qty, ss_wc store_wholesale_cost, ss_sp store_sales_price, coalesce(ws_qty,0)+coalesce(cs_qty,0) other_chan_qty, coalesce(ws_wc,0)+coalesce(cs_wc,0) other_chan_wholesale_cost, coalesce(ws_sp,0)+coalesce(cs_sp,0) other_chan_sales_price from ss left join ws on (ws_sold_year=ss_sold_year and ws_item_sk=ss_item_sk and ws_customer_sk=ss_customer_sk) left join cs on (cs_sold_year=ss_sold_year and cs_item_sk=ss_item_sk and cs_customer_sk=ss_customer_sk) where (coalesce(ws_qty,0)>0 or coalesce(cs_qty, 0)>0) and ss_sold_year=2002 order by ss_sold_year, ss_item_sk, ss_customer_sk, ss_qty desc, ss_wc desc, ss_sp desc, other_chan_qty, other_chan_wholesale_cost, other_chan_sales_price, ratio limit 100""", "q79" -> """ select c_last_name,c_first_name,substr(s_city,1,30),ss_ticket_number,amt,profit from (select ss_ticket_number ,ss_customer_sk ,store.s_city ,sum(ss_coupon_amt) amt ,sum(ss_net_profit) profit from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and (household_demographics.hd_dep_count = 0 or household_demographics.hd_vehicle_count > 0) and date_dim.d_dow = 1 and date_dim.d_year in (2000,2000+1,2000+2) and store.s_number_employees between 200 and 295 group by ss_ticket_number,ss_customer_sk,ss_addr_sk,store.s_city) ms,customer where ss_customer_sk = c_customer_sk order by c_last_name,c_first_name,substr(s_city,1,30), profit limit 100""", "q80" -> """ with ssr as (select s_store_id as store_id, sum(ss_ext_sales_price) as sales, sum(coalesce(sr_return_amt, 0)) as returns, sum(ss_net_profit - coalesce(sr_net_loss, 0)) as profit from store_sales left outer join store_returns on (ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number), date_dim, store, item, promotion where ss_sold_date_sk = d_date_sk and d_date between cast('1999-08-12' as date) and (cast('1999-08-12' as date) + INTERVAL 60 days) and ss_store_sk = s_store_sk and ss_item_sk = i_item_sk and i_current_price > 50 and ss_promo_sk = p_promo_sk and p_channel_tv = 'N' group by s_store_id) , csr as (select cp_catalog_page_id as catalog_page_id, sum(cs_ext_sales_price) as sales, sum(coalesce(cr_return_amount, 0)) as returns, sum(cs_net_profit - coalesce(cr_net_loss, 0)) as profit from catalog_sales left outer join catalog_returns on (cs_item_sk = cr_item_sk and cs_order_number = cr_order_number), date_dim, catalog_page, item, promotion where cs_sold_date_sk = d_date_sk and d_date between cast('1999-08-12' as date) and (cast('1999-08-12' as date) + INTERVAL 60 days) and cs_catalog_page_sk = cp_catalog_page_sk and cs_item_sk = i_item_sk and i_current_price > 50 and cs_promo_sk = p_promo_sk and p_channel_tv = 'N' group by cp_catalog_page_id) , wsr as (select web_site_id, sum(ws_ext_sales_price) as sales, sum(coalesce(wr_return_amt, 0)) as returns, sum(ws_net_profit - coalesce(wr_net_loss, 0)) as profit from web_sales left outer join web_returns on (ws_item_sk = wr_item_sk and ws_order_number = wr_order_number), date_dim, web_site, item, promotion where ws_sold_date_sk = d_date_sk and d_date between cast('1999-08-12' as date) and (cast('1999-08-12' as date) + INTERVAL 60 days) and ws_web_site_sk = web_site_sk and ws_item_sk = i_item_sk and i_current_price > 50 and ws_promo_sk = p_promo_sk and p_channel_tv = 'N' group by web_site_id) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , 'store' || store_id as id , sales , returns , profit from ssr union all select 'catalog channel' as channel , 'catalog_page' || catalog_page_id as id , sales , returns , profit from csr union all select 'web channel' as channel , 'web_site' || web_site_id as id , sales , returns , profit from wsr ) x group by rollup (channel, id) order by channel ,id limit 100""", "q81" -> """ with customer_total_return as (select cr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, sum(cr_return_amt_inc_tax) as ctr_total_return from catalog_returns ,date_dim ,customer_address where cr_returned_date_sk = d_date_sk and d_year =2001 and cr_returning_addr_sk = ca_address_sk group by cr_returning_customer_sk ,ca_state ) select c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset ,ca_location_type,ctr_total_return from customer_total_return ctr1 ,customer_address ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_state = ctr2.ctr_state) and ca_address_sk = c_current_addr_sk and ca_state = 'NC' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset ,ca_location_type,ctr_total_return limit 100""", "q82" -> """ select i_item_id ,i_item_desc ,i_current_price from item, inventory, date_dim, store_sales where i_current_price between 82 and 82+30 and inv_item_sk = i_item_sk and d_date_sk=inv_date_sk and d_date between cast('2002-03-10' as date) and (cast('2002-03-10' as date) + INTERVAL 60 days) and i_manufact_id in (941,920,105,693) and inv_quantity_on_hand between 100 and 500 and ss_item_sk = i_item_sk group by i_item_id,i_item_desc,i_current_price order by i_item_id limit 100""", "q83" -> """ with sr_items as (select i_item_id item_id, sum(sr_return_quantity) sr_item_qty from store_returns, item, date_dim where sr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('1999-04-14','1999-09-28','1999-11-12'))) and sr_returned_date_sk = d_date_sk group by i_item_id), cr_items as (select i_item_id item_id, sum(cr_return_quantity) cr_item_qty from catalog_returns, item, date_dim where cr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('1999-04-14','1999-09-28','1999-11-12'))) and cr_returned_date_sk = d_date_sk group by i_item_id), wr_items as (select i_item_id item_id, sum(wr_return_quantity) wr_item_qty from web_returns, item, date_dim where wr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('1999-04-14','1999-09-28','1999-11-12'))) and wr_returned_date_sk = d_date_sk group by i_item_id) select sr_items.item_id ,sr_item_qty ,sr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 sr_dev ,cr_item_qty ,cr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 cr_dev ,wr_item_qty ,wr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 wr_dev ,(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 average from sr_items ,cr_items ,wr_items where sr_items.item_id=cr_items.item_id and sr_items.item_id=wr_items.item_id order by sr_items.item_id ,sr_item_qty limit 100""", "q84" -> """ select c_customer_id as customer_id , coalesce(c_last_name,'') || ', ' || coalesce(c_first_name,'') as customername from customer ,customer_address ,customer_demographics ,household_demographics ,income_band ,store_returns where ca_city = 'Antioch' and c_current_addr_sk = ca_address_sk and ib_lower_bound >= 55019 and ib_upper_bound <= 55019 + 50000 and ib_income_band_sk = hd_income_band_sk and cd_demo_sk = c_current_cdemo_sk and hd_demo_sk = c_current_hdemo_sk and sr_cdemo_sk = cd_demo_sk order by c_customer_id limit 100""", "q85" -> """ select substr(r_reason_desc,1,20) ,avg(ws_quantity) ,avg(wr_refunded_cash) ,avg(wr_fee) from web_sales, web_returns, web_page, customer_demographics cd1, customer_demographics cd2, customer_address, date_dim, reason where ws_web_page_sk = wp_web_page_sk and ws_item_sk = wr_item_sk and ws_order_number = wr_order_number and ws_sold_date_sk = d_date_sk and d_year = 2001 and cd1.cd_demo_sk = wr_refunded_cdemo_sk and cd2.cd_demo_sk = wr_returning_cdemo_sk and ca_address_sk = wr_refunded_addr_sk and r_reason_sk = wr_reason_sk and ( ( cd1.cd_marital_status = 'S' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = '2 yr Degree' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 100.00 and 150.00 ) or ( cd1.cd_marital_status = 'D' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = 'Advanced Degree' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 50.00 and 100.00 ) or ( cd1.cd_marital_status = 'W' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = '4 yr Degree' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 150.00 and 200.00 ) ) and ( ( ca_country = 'United States' and ca_state in ('OK', 'TX', 'MO') and ws_net_profit between 100 and 200 ) or ( ca_country = 'United States' and ca_state in ('GA', 'KS', 'NC') and ws_net_profit between 150 and 300 ) or ( ca_country = 'United States' and ca_state in ('VA', 'WI', 'WV') and ws_net_profit between 50 and 250 ) ) group by r_reason_desc order by substr(r_reason_desc,1,20) ,avg(ws_quantity) ,avg(wr_refunded_cash) ,avg(wr_fee) limit 100""", "q86" -> """ select sum(ws_net_paid) as total_sum ,i_category ,i_class ,grouping(i_category)+grouping(i_class) as lochierarchy ,rank() over ( partition by grouping(i_category)+grouping(i_class), case when grouping(i_class) = 0 then i_category end order by sum(ws_net_paid) desc) as rank_within_parent from web_sales ,date_dim d1 ,item where d1.d_month_seq between 1180 and 1180+11 and d1.d_date_sk = ws_sold_date_sk and i_item_sk = ws_item_sk group by rollup(i_category,i_class) order by lochierarchy desc, case when lochierarchy = 0 then i_category end, rank_within_parent limit 100""", "q87" -> """ select count(*) from ((select distinct c_last_name, c_first_name, d_date from store_sales, date_dim, customer where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_customer_sk = customer.c_customer_sk and d_month_seq between 1204 and 1204+11) except (select distinct c_last_name, c_first_name, d_date from catalog_sales, date_dim, customer where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1204 and 1204+11) except (select distinct c_last_name, c_first_name, d_date from web_sales, date_dim, customer where web_sales.ws_sold_date_sk = date_dim.d_date_sk and web_sales.ws_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1204 and 1204+11) ) cool_cust""", "q88" -> """ select * from (select count(*) h8_30_to_9 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 8 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) and store.s_store_name = 'ese') s1, (select count(*) h9_to_9_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 9 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) and store.s_store_name = 'ese') s2, (select count(*) h9_30_to_10 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 9 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) and store.s_store_name = 'ese') s3, (select count(*) h10_to_10_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 10 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) and store.s_store_name = 'ese') s4, (select count(*) h10_30_to_11 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 10 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) and store.s_store_name = 'ese') s5, (select count(*) h11_to_11_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 11 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) and store.s_store_name = 'ese') s6, (select count(*) h11_30_to_12 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 11 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) and store.s_store_name = 'ese') s7, (select count(*) h12_to_12_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 12 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2)) and store.s_store_name = 'ese') s8""", "q89" -> """ select * from( select i_category, i_class, i_brand, s_store_name, s_company_name, d_moy, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_category, i_brand, s_store_name, s_company_name) avg_monthly_sales from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_year in (2001) and ((i_category in ('Women','Music','Home') and i_class in ('fragrances','pop','bedding') ) or (i_category in ('Books','Men','Children') and i_class in ('home repair','sports-apparel','infants') )) group by i_category, i_class, i_brand, s_store_name, s_company_name, d_moy) tmp1 where case when (avg_monthly_sales <> 0) then (abs(sum_sales - avg_monthly_sales) / avg_monthly_sales) else null end > 0.1 order by sum_sales - avg_monthly_sales, s_store_name limit 100""", "q90" -> """ select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio from ( select count(*) amc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 8 and 8+1 and household_demographics.hd_dep_count = 4 and web_page.wp_char_count between 5000 and 5200) at, ( select count(*) pmc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 19 and 19+1 and household_demographics.hd_dep_count = 4 and web_page.wp_char_count between 5000 and 5200) pt order by am_pm_ratio limit 100""", "q91" -> """ select cc_call_center_id Call_Center, cc_name Call_Center_Name, cc_manager Manager, sum(cr_net_loss) Returns_Loss from call_center, catalog_returns, date_dim, customer, customer_address, customer_demographics, household_demographics where cr_call_center_sk = cc_call_center_sk and cr_returned_date_sk = d_date_sk and cr_returning_customer_sk= c_customer_sk and cd_demo_sk = c_current_cdemo_sk and hd_demo_sk = c_current_hdemo_sk and ca_address_sk = c_current_addr_sk and d_year = 2002 and d_moy = 11 and ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') or(cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) and hd_buy_potential like '5001-10000%' and ca_gmt_offset = -6 group by cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status order by sum(cr_net_loss) desc""", "q92" -> """ select sum(ws_ext_discount_amt) as `Excess Discount Amount` from web_sales ,item ,date_dim where i_manufact_id = 561 and i_item_sk = ws_item_sk and d_date between '2001-03-13' and (cast('2001-03-13' as date) + INTERVAL 90 days) and d_date_sk = ws_sold_date_sk and ws_ext_discount_amt > ( SELECT 1.3 * avg(ws_ext_discount_amt) FROM web_sales ,date_dim WHERE ws_item_sk = i_item_sk and d_date between '2001-03-13' and (cast('2001-03-13' as date) + INTERVAL 90 days) and d_date_sk = ws_sold_date_sk ) order by sum(ws_ext_discount_amt) limit 100""", "q93" -> """ select ss_customer_sk ,sum(act_sales) sumsales from (select ss_item_sk ,ss_ticket_number ,ss_customer_sk ,case when sr_return_quantity is not null then (ss_quantity-sr_return_quantity)*ss_sales_price else (ss_quantity*ss_sales_price) end act_sales from store_sales left outer join store_returns on (sr_item_sk = ss_item_sk and sr_ticket_number = ss_ticket_number) ,reason where sr_reason_sk = r_reason_sk and r_reason_desc = 'reason 64') t group by ss_customer_sk order by sumsales, ss_customer_sk limit 100""", "q94" -> """ select count(distinct ws_order_number) as `order count` ,sum(ws_ext_ship_cost) as `total shipping cost` ,sum(ws_net_profit) as `total net profit` from web_sales ws1 ,date_dim ,customer_address ,web_site where d_date between '2001-5-01' and (cast('2001-5-01' as date) + INTERVAL 60 days) and ws1.ws_ship_date_sk = d_date_sk and ws1.ws_ship_addr_sk = ca_address_sk and ca_state = 'TX' and ws1.ws_web_site_sk = web_site_sk and web_company_name = 'pri' and exists (select * from web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) and not exists(select * from web_returns wr1 where ws1.ws_order_number = wr1.wr_order_number) order by count(distinct ws_order_number) limit 100""", "q95" -> """ with ws_wh as (select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 from web_sales ws1,web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) select count(distinct ws_order_number) as `order count` ,sum(ws_ext_ship_cost) as `total shipping cost` ,sum(ws_net_profit) as `total net profit` from web_sales ws1 ,date_dim ,customer_address ,web_site where d_date between '2000-3-01' and (cast('2000-3-01' as date) + INTERVAL 60 days) and ws1.ws_ship_date_sk = d_date_sk and ws1.ws_ship_addr_sk = ca_address_sk and ca_state = 'TN' and ws1.ws_web_site_sk = web_site_sk and web_company_name = 'pri' and ws1.ws_order_number in (select ws_order_number from ws_wh) and ws1.ws_order_number in (select wr_order_number from web_returns,ws_wh where wr_order_number = ws_wh.ws_order_number) order by count(distinct ws_order_number) limit 100""", "q96" -> """ select count(*) from store_sales ,household_demographics ,time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 16 and time_dim.t_minute >= 30 and household_demographics.hd_dep_count = 4 and store.s_store_name = 'ese' order by count(*) limit 100""", "q97" -> """ with ssci as ( select ss_customer_sk customer_sk ,ss_item_sk item_sk from store_sales,date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1209 and 1209 + 11 group by ss_customer_sk ,ss_item_sk), csci as( select cs_bill_customer_sk customer_sk ,cs_item_sk item_sk from catalog_sales,date_dim where cs_sold_date_sk = d_date_sk and d_month_seq between 1209 and 1209 + 11 group by cs_bill_customer_sk ,cs_item_sk) select sum(case when ssci.customer_sk is not null and csci.customer_sk is null then 1 else 0 end) store_only ,sum(case when ssci.customer_sk is null and csci.customer_sk is not null then 1 else 0 end) catalog_only ,sum(case when ssci.customer_sk is not null and csci.customer_sk is not null then 1 else 0 end) store_and_catalog from ssci full outer join csci on (ssci.customer_sk=csci.customer_sk and ssci.item_sk = csci.item_sk) limit 100""", "q98" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(ss_ext_sales_price) as itemrevenue ,sum(ss_ext_sales_price)*100/sum(sum(ss_ext_sales_price)) over (partition by i_class) as revenueratio from store_sales ,item ,date_dim where ss_item_sk = i_item_sk and i_category in ('Jewelry', 'Home', 'Shoes') and ss_sold_date_sk = d_date_sk and d_date between cast('2001-04-12' as date) and (cast('2001-04-12' as date) + interval 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio""", "q99" -> """ select substr(w_warehouse_name,1,20) ,sm_type ,cc_name ,sum(case when (cs_ship_date_sk - cs_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 30) and (cs_ship_date_sk - cs_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 60) and (cs_ship_date_sk - cs_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 90) and (cs_ship_date_sk - cs_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from catalog_sales ,warehouse ,ship_mode ,call_center ,date_dim where d_month_seq between 1203 and 1203 + 11 and cs_ship_date_sk = d_date_sk and cs_warehouse_sk = w_warehouse_sk and cs_ship_mode_sk = sm_ship_mode_sk and cs_call_center_sk = cc_call_center_sk group by substr(w_warehouse_name,1,20) ,sm_type ,cc_name order by substr(w_warehouse_name,1,20) ,sm_type ,cc_name limit 100""", "q1" -> """ with customer_total_return as (select sr_customer_sk as ctr_customer_sk ,sr_store_sk as ctr_store_sk ,sum(SR_FEE) as ctr_total_return from store_returns ,date_dim where sr_returned_date_sk = d_date_sk and d_year =2000 group by sr_customer_sk ,sr_store_sk) select c_customer_id from customer_total_return ctr1 ,store ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_store_sk = ctr2.ctr_store_sk) and s_store_sk = ctr1.ctr_store_sk and s_state = 'NM' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id limit 100""", "q2" -> """ with wscs as (select sold_date_sk ,sales_price from (select ws_sold_date_sk sold_date_sk ,ws_ext_sales_price sales_price from web_sales union all select cs_sold_date_sk sold_date_sk ,cs_ext_sales_price sales_price from catalog_sales)), wswscs as (select d_week_seq, sum(case when (d_day_name='Sunday') then sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then sales_price else null end) sat_sales from wscs ,date_dim where d_date_sk = sold_date_sk group by d_week_seq) select d_week_seq1 ,round(sun_sales1/sun_sales2,2) ,round(mon_sales1/mon_sales2,2) ,round(tue_sales1/tue_sales2,2) ,round(wed_sales1/wed_sales2,2) ,round(thu_sales1/thu_sales2,2) ,round(fri_sales1/fri_sales2,2) ,round(sat_sales1/sat_sales2,2) from (select wswscs.d_week_seq d_week_seq1 ,sun_sales sun_sales1 ,mon_sales mon_sales1 ,tue_sales tue_sales1 ,wed_sales wed_sales1 ,thu_sales thu_sales1 ,fri_sales fri_sales1 ,sat_sales sat_sales1 from wswscs,date_dim where date_dim.d_week_seq = wswscs.d_week_seq and d_year = 1998) y, (select wswscs.d_week_seq d_week_seq2 ,sun_sales sun_sales2 ,mon_sales mon_sales2 ,tue_sales tue_sales2 ,wed_sales wed_sales2 ,thu_sales thu_sales2 ,fri_sales fri_sales2 ,sat_sales sat_sales2 from wswscs ,date_dim where date_dim.d_week_seq = wswscs.d_week_seq and d_year = 1998+1) z where d_week_seq1=d_week_seq2-53 order by d_week_seq1""", "q3" -> """ select dt.d_year ,item.i_brand_id brand_id ,item.i_brand brand ,sum(ss_sales_price) sum_agg from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manufact_id = 816 and dt.d_moy=11 group by dt.d_year ,item.i_brand ,item.i_brand_id order by dt.d_year ,sum_agg desc ,brand_id limit 100""", "q4" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum((((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2) ) year_total ,'c' sale_type from customer ,catalog_sales ,date_dim where c_customer_sk = cs_bill_customer_sk and cs_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum((((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2) ) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year ) select t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_birth_country from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_c_firstyear ,year_total t_c_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_c_secyear.customer_id and t_s_firstyear.customer_id = t_c_firstyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.sale_type = 's' and t_c_firstyear.sale_type = 'c' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_c_secyear.sale_type = 'c' and t_w_secyear.sale_type = 'w' and t_s_firstyear.dyear = 1999 and t_s_secyear.dyear = 1999+1 and t_c_firstyear.dyear = 1999 and t_c_secyear.dyear = 1999+1 and t_w_firstyear.dyear = 1999 and t_w_secyear.dyear = 1999+1 and t_s_firstyear.year_total > 0 and t_c_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end > case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end order by t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_birth_country limit 100""", "q5" -> """ with ssr as (select s_store_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select ss_store_sk as store_sk, ss_sold_date_sk as date_sk, ss_ext_sales_price as sales_price, ss_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from store_sales union all select sr_store_sk as store_sk, sr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, sr_return_amt as return_amt, sr_net_loss as net_loss from store_returns ) salesreturns, date_dim, store where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and store_sk = s_store_sk group by s_store_id) , csr as (select cp_catalog_page_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select cs_catalog_page_sk as page_sk, cs_sold_date_sk as date_sk, cs_ext_sales_price as sales_price, cs_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from catalog_sales union all select cr_catalog_page_sk as page_sk, cr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, cr_return_amount as return_amt, cr_net_loss as net_loss from catalog_returns ) salesreturns, date_dim, catalog_page where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and page_sk = cp_catalog_page_sk group by cp_catalog_page_id) , wsr as (select web_site_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select ws_web_site_sk as wsr_web_site_sk, ws_sold_date_sk as date_sk, ws_ext_sales_price as sales_price, ws_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from web_sales union all select ws_web_site_sk as wsr_web_site_sk, wr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, wr_return_amt as return_amt, wr_net_loss as net_loss from web_returns left outer join web_sales on ( wr_item_sk = ws_item_sk and wr_order_number = ws_order_number) ) salesreturns, date_dim, web_site where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and wsr_web_site_sk = web_site_sk group by web_site_id) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , 'store' || s_store_id as id , sales , returns , (profit - profit_loss) as profit from ssr union all select 'catalog channel' as channel , 'catalog_page' || cp_catalog_page_id as id , sales , returns , (profit - profit_loss) as profit from csr union all select 'web channel' as channel , 'web_site' || web_site_id as id , sales , returns , (profit - profit_loss) as profit from wsr ) x group by rollup (channel, id) order by channel ,id limit 100""", "q6" -> """ select a.ca_state state, count(*) cnt from customer_address a ,customer c ,store_sales s ,date_dim d ,item i where a.ca_address_sk = c.c_current_addr_sk and c.c_customer_sk = s.ss_customer_sk and s.ss_sold_date_sk = d.d_date_sk and s.ss_item_sk = i.i_item_sk and d.d_month_seq = (select distinct (d_month_seq) from date_dim where d_year = 2002 and d_moy = 3 ) and i.i_current_price > 1.2 * (select avg(j.i_current_price) from item j where j.i_category = i.i_category) group by a.ca_state having count(*) >= 10 order by cnt, a.ca_state limit 100""", "q7" -> """ select i_item_id, avg(ss_quantity) agg1, avg(ss_list_price) agg2, avg(ss_coupon_amt) agg3, avg(ss_sales_price) agg4 from store_sales, customer_demographics, date_dim, item, promotion where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and ss_cdemo_sk = cd_demo_sk and ss_promo_sk = p_promo_sk and cd_gender = 'F' and cd_marital_status = 'W' and cd_education_status = 'College' and (p_channel_email = 'N' or p_channel_event = 'N') and d_year = 2001 group by i_item_id order by i_item_id limit 100""", "q8" -> """ select s_store_name ,sum(ss_net_profit) from store_sales ,date_dim ,store, (select ca_zip from ( SELECT substr(ca_zip,1,5) ca_zip FROM customer_address WHERE substr(ca_zip,1,5) IN ( '47602','16704','35863','28577','83910','36201', '58412','48162','28055','41419','80332', '38607','77817','24891','16226','18410', '21231','59345','13918','51089','20317', '17167','54585','67881','78366','47770', '18360','51717','73108','14440','21800', '89338','45859','65501','34948','25973', '73219','25333','17291','10374','18829', '60736','82620','41351','52094','19326', '25214','54207','40936','21814','79077', '25178','75742','77454','30621','89193', '27369','41232','48567','83041','71948', '37119','68341','14073','16891','62878', '49130','19833','24286','27700','40979', '50412','81504','94835','84844','71954', '39503','57649','18434','24987','12350', '86379','27413','44529','98569','16515', '27287','24255','21094','16005','56436', '91110','68293','56455','54558','10298', '83647','32754','27052','51766','19444', '13869','45645','94791','57631','20712', '37788','41807','46507','21727','71836', '81070','50632','88086','63991','20244', '31655','51782','29818','63792','68605', '94898','36430','57025','20601','82080', '33869','22728','35834','29086','92645', '98584','98072','11652','78093','57553', '43830','71144','53565','18700','90209', '71256','38353','54364','28571','96560', '57839','56355','50679','45266','84680', '34306','34972','48530','30106','15371', '92380','84247','92292','68852','13338', '34594','82602','70073','98069','85066', '47289','11686','98862','26217','47529', '63294','51793','35926','24227','14196', '24594','32489','99060','49472','43432', '49211','14312','88137','47369','56877', '20534','81755','15794','12318','21060', '73134','41255','63073','81003','73873', '66057','51184','51195','45676','92696', '70450','90669','98338','25264','38919', '59226','58581','60298','17895','19489', '52301','80846','95464','68770','51634', '19988','18367','18421','11618','67975', '25494','41352','95430','15734','62585', '97173','33773','10425','75675','53535', '17879','41967','12197','67998','79658', '59130','72592','14851','43933','68101', '50636','25717','71286','24660','58058', '72991','95042','15543','33122','69280', '11912','59386','27642','65177','17672', '33467','64592','36335','54010','18767', '63193','42361','49254','33113','33159', '36479','59080','11855','81963','31016', '49140','29392','41836','32958','53163', '13844','73146','23952','65148','93498', '14530','46131','58454','13376','13378', '83986','12320','17193','59852','46081', '98533','52389','13086','68843','31013', '13261','60560','13443','45533','83583', '11489','58218','19753','22911','25115', '86709','27156','32669','13123','51933', '39214','41331','66943','14155','69998', '49101','70070','35076','14242','73021', '59494','15782','29752','37914','74686', '83086','34473','15751','81084','49230', '91894','60624','17819','28810','63180', '56224','39459','55233','75752','43639', '55349','86057','62361','50788','31830', '58062','18218','85761','60083','45484', '21204','90229','70041','41162','35390', '16364','39500','68908','26689','52868', '81335','40146','11340','61527','61794', '71997','30415','59004','29450','58117', '69952','33562','83833','27385','61860', '96435','48333','23065','32961','84919', '61997','99132','22815','56600','68730', '48017','95694','32919','88217','27116', '28239','58032','18884','16791','21343', '97462','18569','75660','15475') intersect select ca_zip from (SELECT substr(ca_zip,1,5) ca_zip,count(*) cnt FROM customer_address, customer WHERE ca_address_sk = c_current_addr_sk and c_preferred_cust_flag='Y' group by ca_zip having count(*) > 10)A1)A2) V1 where ss_store_sk = s_store_sk and ss_sold_date_sk = d_date_sk and d_qoy = 2 and d_year = 1998 and (substr(s_zip,1,2) = substr(V1.ca_zip,1,2)) group by s_store_name order by s_store_name limit 100""", "q9" -> """ select case when (select count(*) from store_sales where ss_quantity between 1 and 20) > 98972190 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 1 and 20) else (select avg(ss_net_profit) from store_sales where ss_quantity between 1 and 20) end bucket1 , case when (select count(*) from store_sales where ss_quantity between 21 and 40) > 160856845 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 21 and 40) else (select avg(ss_net_profit) from store_sales where ss_quantity between 21 and 40) end bucket2, case when (select count(*) from store_sales where ss_quantity between 41 and 60) > 12733327 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 41 and 60) else (select avg(ss_net_profit) from store_sales where ss_quantity between 41 and 60) end bucket3, case when (select count(*) from store_sales where ss_quantity between 61 and 80) > 96251173 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 61 and 80) else (select avg(ss_net_profit) from store_sales where ss_quantity between 61 and 80) end bucket4, case when (select count(*) from store_sales where ss_quantity between 81 and 100) > 80049606 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 81 and 100) else (select avg(ss_net_profit) from store_sales where ss_quantity between 81 and 100) end bucket5 from reason where r_reason_sk = 1""", "q10" -> """ select cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3, cd_dep_count, count(*) cnt4, cd_dep_employed_count, count(*) cnt5, cd_dep_college_count, count(*) cnt6 from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and ca_county in ('Fillmore County','McPherson County','Bonneville County','Boone County','Brown County') and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2000 and d_moy between 3 and 3+3) and (exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2000 and d_moy between 3 ANd 3+3) or exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2000 and d_moy between 3 and 3+3)) group by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count order by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count limit 100""", "q11" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(ss_ext_list_price-ss_ext_discount_amt) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(ws_ext_list_price-ws_ext_discount_amt) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year ) select t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_birth_country from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.sale_type = 's' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_w_secyear.sale_type = 'w' and t_s_firstyear.dyear = 1999 and t_s_secyear.dyear = 1999+1 and t_w_firstyear.dyear = 1999 and t_w_secyear.dyear = 1999+1 and t_s_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else 0.0 end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else 0.0 end order by t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_birth_country limit 100""", "q12" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(ws_ext_sales_price) as itemrevenue ,sum(ws_ext_sales_price)*100/sum(sum(ws_ext_sales_price)) over (partition by i_class) as revenueratio from web_sales ,item ,date_dim where ws_item_sk = i_item_sk and i_category in ('Electronics', 'Books', 'Women') and ws_sold_date_sk = d_date_sk and d_date between cast('1998-01-06' as date) and (cast('1998-01-06' as date) + INTERVAL 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio limit 100""", "q13" -> """ select avg(ss_quantity) ,avg(ss_ext_sales_price) ,avg(ss_ext_wholesale_cost) ,sum(ss_ext_wholesale_cost) from store_sales ,store ,customer_demographics ,household_demographics ,customer_address ,date_dim where s_store_sk = ss_store_sk and ss_sold_date_sk = d_date_sk and d_year = 2001 and((ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'U' and cd_education_status = 'Secondary' and ss_sales_price between 100.00 and 150.00 and hd_dep_count = 3 )or (ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'W' and cd_education_status = 'College' and ss_sales_price between 50.00 and 100.00 and hd_dep_count = 1 ) or (ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'D' and cd_education_status = 'Primary' and ss_sales_price between 150.00 and 200.00 and hd_dep_count = 1 )) and((ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('TX', 'OK', 'MI') and ss_net_profit between 100 and 200 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('WA', 'NC', 'OH') and ss_net_profit between 150 and 300 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('MT', 'FL', 'GA') and ss_net_profit between 50 and 250 ))""", "q14a" -> """ with cross_items as (select i_item_sk ss_item_sk from item, (select iss.i_brand_id brand_id ,iss.i_class_id class_id ,iss.i_category_id category_id from store_sales ,item iss ,date_dim d1 where ss_item_sk = iss.i_item_sk and ss_sold_date_sk = d1.d_date_sk and d1.d_year between 2000 AND 2000 + 2 intersect select ics.i_brand_id ,ics.i_class_id ,ics.i_category_id from catalog_sales ,item ics ,date_dim d2 where cs_item_sk = ics.i_item_sk and cs_sold_date_sk = d2.d_date_sk and d2.d_year between 2000 AND 2000 + 2 intersect select iws.i_brand_id ,iws.i_class_id ,iws.i_category_id from web_sales ,item iws ,date_dim d3 where ws_item_sk = iws.i_item_sk and ws_sold_date_sk = d3.d_date_sk and d3.d_year between 2000 AND 2000 + 2) where i_brand_id = brand_id and i_class_id = class_id and i_category_id = category_id ), avg_sales as (select avg(quantity*list_price) average_sales from (select ss_quantity quantity ,ss_list_price list_price from store_sales ,date_dim where ss_sold_date_sk = d_date_sk and d_year between 2000 and 2000 + 2 union all select cs_quantity quantity ,cs_list_price list_price from catalog_sales ,date_dim where cs_sold_date_sk = d_date_sk and d_year between 2000 and 2000 + 2 union all select ws_quantity quantity ,ws_list_price list_price from web_sales ,date_dim where ws_sold_date_sk = d_date_sk and d_year between 2000 and 2000 + 2) x) select channel, i_brand_id,i_class_id,i_category_id,sum(sales), sum(number_sales) from( select 'store' channel, i_brand_id,i_class_id ,i_category_id,sum(ss_quantity*ss_list_price) sales , count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 2000+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales) union all select 'catalog' channel, i_brand_id,i_class_id,i_category_id, sum(cs_quantity*cs_list_price) sales, count(*) number_sales from catalog_sales ,item ,date_dim where cs_item_sk in (select ss_item_sk from cross_items) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 2000+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(cs_quantity*cs_list_price) > (select average_sales from avg_sales) union all select 'web' channel, i_brand_id,i_class_id,i_category_id, sum(ws_quantity*ws_list_price) sales , count(*) number_sales from web_sales ,item ,date_dim where ws_item_sk in (select ss_item_sk from cross_items) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 2000+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(ws_quantity*ws_list_price) > (select average_sales from avg_sales) ) y group by rollup (channel, i_brand_id,i_class_id,i_category_id) order by channel,i_brand_id,i_class_id,i_category_id limit 100""", "q14b" -> """ with cross_items as (select i_item_sk ss_item_sk from item, (select iss.i_brand_id brand_id ,iss.i_class_id class_id ,iss.i_category_id category_id from store_sales ,item iss ,date_dim d1 where ss_item_sk = iss.i_item_sk and ss_sold_date_sk = d1.d_date_sk and d1.d_year between 2000 AND 2000 + 2 intersect select ics.i_brand_id ,ics.i_class_id ,ics.i_category_id from catalog_sales ,item ics ,date_dim d2 where cs_item_sk = ics.i_item_sk and cs_sold_date_sk = d2.d_date_sk and d2.d_year between 2000 AND 2000 + 2 intersect select iws.i_brand_id ,iws.i_class_id ,iws.i_category_id from web_sales ,item iws ,date_dim d3 where ws_item_sk = iws.i_item_sk and ws_sold_date_sk = d3.d_date_sk and d3.d_year between 2000 AND 2000 + 2) x where i_brand_id = brand_id and i_class_id = class_id and i_category_id = category_id ), avg_sales as (select avg(quantity*list_price) average_sales from (select ss_quantity quantity ,ss_list_price list_price from store_sales ,date_dim where ss_sold_date_sk = d_date_sk and d_year between 2000 and 2000 + 2 union all select cs_quantity quantity ,cs_list_price list_price from catalog_sales ,date_dim where cs_sold_date_sk = d_date_sk and d_year between 2000 and 2000 + 2 union all select ws_quantity quantity ,ws_list_price list_price from web_sales ,date_dim where ws_sold_date_sk = d_date_sk and d_year between 2000 and 2000 + 2) x) select this_year.channel ty_channel ,this_year.i_brand_id ty_brand ,this_year.i_class_id ty_class ,this_year.i_category_id ty_category ,this_year.sales ty_sales ,this_year.number_sales ty_number_sales ,last_year.channel ly_channel ,last_year.i_brand_id ly_brand ,last_year.i_class_id ly_class ,last_year.i_category_id ly_category ,last_year.sales ly_sales ,last_year.number_sales ly_number_sales from (select 'store' channel, i_brand_id,i_class_id,i_category_id ,sum(ss_quantity*ss_list_price) sales, count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_week_seq = (select d_week_seq from date_dim where d_year = 2000 + 1 and d_moy = 12 and d_dom = 15) group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) this_year, (select 'store' channel, i_brand_id,i_class_id ,i_category_id, sum(ss_quantity*ss_list_price) sales, count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_week_seq = (select d_week_seq from date_dim where d_year = 2000 and d_moy = 12 and d_dom = 15) group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) last_year where this_year.i_brand_id= last_year.i_brand_id and this_year.i_class_id = last_year.i_class_id and this_year.i_category_id = last_year.i_category_id order by this_year.channel, this_year.i_brand_id, this_year.i_class_id, this_year.i_category_id limit 100""", "q15" -> """ select ca_zip ,sum(cs_sales_price) from catalog_sales ,customer ,customer_address ,date_dim where cs_bill_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') or ca_state in ('CA','WA','GA') or cs_sales_price > 500) and cs_sold_date_sk = d_date_sk and d_qoy = 2 and d_year = 1998 group by ca_zip order by ca_zip limit 100""", "q16" -> """ select count(distinct cs_order_number) as `order count` ,sum(cs_ext_ship_cost) as `total shipping cost` ,sum(cs_net_profit) as `total net profit` from catalog_sales cs1 ,date_dim ,customer_address ,call_center where d_date between '1999-4-01' and (cast('1999-4-01' as date) + INTERVAL 60 days) and cs1.cs_ship_date_sk = d_date_sk and cs1.cs_ship_addr_sk = ca_address_sk and ca_state = 'IL' and cs1.cs_call_center_sk = cc_call_center_sk and cc_county in ('Richland County','Bronx County','Maverick County','Mesa County', 'Raleigh County' ) and exists (select * from catalog_sales cs2 where cs1.cs_order_number = cs2.cs_order_number and cs1.cs_warehouse_sk <> cs2.cs_warehouse_sk) and not exists(select * from catalog_returns cr1 where cs1.cs_order_number = cr1.cr_order_number) order by count(distinct cs_order_number) limit 100""", "q17" -> """ select i_item_id ,i_item_desc ,s_state ,count(ss_quantity) as store_sales_quantitycount ,avg(ss_quantity) as store_sales_quantityave ,stddev_samp(ss_quantity) as store_sales_quantitystdev ,stddev_samp(ss_quantity)/avg(ss_quantity) as store_sales_quantitycov ,count(sr_return_quantity) as store_returns_quantitycount ,avg(sr_return_quantity) as store_returns_quantityave ,stddev_samp(sr_return_quantity) as store_returns_quantitystdev ,stddev_samp(sr_return_quantity)/avg(sr_return_quantity) as store_returns_quantitycov ,count(cs_quantity) as catalog_sales_quantitycount ,avg(cs_quantity) as catalog_sales_quantityave ,stddev_samp(cs_quantity) as catalog_sales_quantitystdev ,stddev_samp(cs_quantity)/avg(cs_quantity) as catalog_sales_quantitycov from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_quarter_name = '2000Q1' and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_quarter_name in ('2000Q1','2000Q2','2000Q3') and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_quarter_name in ('2000Q1','2000Q2','2000Q3') group by i_item_id ,i_item_desc ,s_state order by i_item_id ,i_item_desc ,s_state limit 100""", "q18" -> """ select i_item_id, ca_country, ca_state, ca_county, avg( cast(cs_quantity as decimal(12,2))) agg1, avg( cast(cs_list_price as decimal(12,2))) agg2, avg( cast(cs_coupon_amt as decimal(12,2))) agg3, avg( cast(cs_sales_price as decimal(12,2))) agg4, avg( cast(cs_net_profit as decimal(12,2))) agg5, avg( cast(c_birth_year as decimal(12,2))) agg6, avg( cast(cd1.cd_dep_count as decimal(12,2))) agg7 from catalog_sales, customer_demographics cd1, customer_demographics cd2, customer, customer_address, date_dim, item where cs_sold_date_sk = d_date_sk and cs_item_sk = i_item_sk and cs_bill_cdemo_sk = cd1.cd_demo_sk and cs_bill_customer_sk = c_customer_sk and cd1.cd_gender = 'M' and cd1.cd_education_status = 'Unknown' and c_current_cdemo_sk = cd2.cd_demo_sk and c_current_addr_sk = ca_address_sk and c_birth_month in (5,1,4,7,8,9) and d_year = 2002 and ca_state in ('AR','TX','NC' ,'GA','MS','WV','AL') group by rollup (i_item_id, ca_country, ca_state, ca_county) order by ca_country, ca_state, ca_county, i_item_id limit 100""", "q19" -> """ select i_brand_id brand_id, i_brand brand, i_manufact_id, i_manufact, sum(ss_ext_sales_price) ext_price from date_dim, store_sales, item,customer,customer_address,store where d_date_sk = ss_sold_date_sk and ss_item_sk = i_item_sk and i_manager_id=16 and d_moy=12 and d_year=1998 and ss_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and substr(ca_zip,1,5) <> substr(s_zip,1,5) and ss_store_sk = s_store_sk group by i_brand ,i_brand_id ,i_manufact_id ,i_manufact order by ext_price desc ,i_brand ,i_brand_id ,i_manufact_id ,i_manufact limit 100 """, "q20" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(cs_ext_sales_price) as itemrevenue ,sum(cs_ext_sales_price)*100/sum(sum(cs_ext_sales_price)) over (partition by i_class) as revenueratio from catalog_sales ,item ,date_dim where cs_item_sk = i_item_sk and i_category in ('Shoes', 'Electronics', 'Children') and cs_sold_date_sk = d_date_sk and d_date between cast('2001-03-14' as date) and (cast('2001-03-14' as date) + INTERVAL 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio limit 100""", "q21" -> """ select * from(select w_warehouse_name ,i_item_id ,sum(case when (cast(d_date as date) < cast ('1999-03-20' as date)) then inv_quantity_on_hand else 0 end) as inv_before ,sum(case when (cast(d_date as date) >= cast ('1999-03-20' as date)) then inv_quantity_on_hand else 0 end) as inv_after from inventory ,warehouse ,item ,date_dim where i_current_price between 0.99 and 1.49 and i_item_sk = inv_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_date between (cast ('1999-03-20' as date) - INTERVAL 30 days) and (cast ('1999-03-20' as date) + INTERVAL 30 days) group by w_warehouse_name, i_item_id) x where (case when inv_before > 0 then inv_after / inv_before else null end) between 2.0/3.0 and 3.0/2.0 order by w_warehouse_name ,i_item_id limit 100""", "q22" -> """ select i_product_name ,i_brand ,i_class ,i_category ,avg(inv_quantity_on_hand) qoh from inventory ,date_dim ,item where inv_date_sk=d_date_sk and inv_item_sk=i_item_sk and d_month_seq between 1186 and 1186 + 11 group by rollup(i_product_name ,i_brand ,i_class ,i_category) order by qoh, i_product_name, i_brand, i_class, i_category limit 100""", "q23a" -> """ with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt from store_sales ,date_dim ,item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and d_year in (2000,2000+1,2000+2,2000+3) group by substr(i_item_desc,1,30),i_item_sk,d_date having count(*) >4), max_store_sales as (select max(csales) tpcds_cmax from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales from store_sales ,customer ,date_dim where ss_customer_sk = c_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (2000,2000+1,2000+2,2000+3) group by c_customer_sk)), best_ss_customer as (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales from store_sales ,customer where ss_customer_sk = c_customer_sk group by c_customer_sk having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select * from max_store_sales)) select sum(sales) from (select cs_quantity*cs_list_price sales from catalog_sales ,date_dim where d_year = 2000 and d_moy = 3 and cs_sold_date_sk = d_date_sk and cs_item_sk in (select item_sk from frequent_ss_items) and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) union all select ws_quantity*ws_list_price sales from web_sales ,date_dim where d_year = 2000 and d_moy = 3 and ws_sold_date_sk = d_date_sk and ws_item_sk in (select item_sk from frequent_ss_items) and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer)) limit 100""", "q23b" -> """ with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt from store_sales ,date_dim ,item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and d_year in (2000,2000 + 1,2000 + 2,2000 + 3) group by substr(i_item_desc,1,30),i_item_sk,d_date having count(*) >4), max_store_sales as (select max(csales) tpcds_cmax from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales from store_sales ,customer ,date_dim where ss_customer_sk = c_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (2000,2000+1,2000+2,2000+3) group by c_customer_sk)), best_ss_customer as (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales from store_sales ,customer where ss_customer_sk = c_customer_sk group by c_customer_sk having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select * from max_store_sales)) select c_last_name,c_first_name,sales from (select c_last_name,c_first_name,sum(cs_quantity*cs_list_price) sales from catalog_sales ,customer ,date_dim where d_year = 2000 and d_moy = 3 and cs_sold_date_sk = d_date_sk and cs_item_sk in (select item_sk from frequent_ss_items) and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) and cs_bill_customer_sk = c_customer_sk group by c_last_name,c_first_name union all select c_last_name,c_first_name,sum(ws_quantity*ws_list_price) sales from web_sales ,customer ,date_dim where d_year = 2000 and d_moy = 3 and ws_sold_date_sk = d_date_sk and ws_item_sk in (select item_sk from frequent_ss_items) and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer) and ws_bill_customer_sk = c_customer_sk group by c_last_name,c_first_name) order by c_last_name,c_first_name,sales limit 100""", "q24a" -> """ with ssales as (select c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size ,sum(ss_sales_price) netpaid from store_sales ,store_returns ,store ,item ,customer ,customer_address where ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_customer_sk = c_customer_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and c_current_addr_sk = ca_address_sk and c_birth_country <> upper(ca_country) and s_zip = ca_zip and s_market_id=10 group by c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size) select c_last_name ,c_first_name ,s_store_name ,sum(netpaid) paid from ssales where i_color = 'snow' group by c_last_name ,c_first_name ,s_store_name having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) order by c_last_name ,c_first_name ,s_store_name""", "q24b" -> """ with ssales as (select c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size ,sum(ss_sales_price) netpaid from store_sales ,store_returns ,store ,item ,customer ,customer_address where ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_customer_sk = c_customer_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and c_current_addr_sk = ca_address_sk and c_birth_country <> upper(ca_country) and s_zip = ca_zip and s_market_id = 10 group by c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size) select c_last_name ,c_first_name ,s_store_name ,sum(netpaid) paid from ssales where i_color = 'chiffon' group by c_last_name ,c_first_name ,s_store_name having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) order by c_last_name ,c_first_name ,s_store_name""", "q25" -> """ select i_item_id ,i_item_desc ,s_store_id ,s_store_name ,sum(ss_net_profit) as store_sales_profit ,sum(sr_net_loss) as store_returns_loss ,sum(cs_net_profit) as catalog_sales_profit from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_moy = 4 and d1.d_year = 2000 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_moy between 4 and 10 and d2.d_year = 2000 and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_moy between 4 and 10 and d3.d_year = 2000 group by i_item_id ,i_item_desc ,s_store_id ,s_store_name order by i_item_id ,i_item_desc ,s_store_id ,s_store_name limit 100""", "q26" -> """ select i_item_id, avg(cs_quantity) agg1, avg(cs_list_price) agg2, avg(cs_coupon_amt) agg3, avg(cs_sales_price) agg4 from catalog_sales, customer_demographics, date_dim, item, promotion where cs_sold_date_sk = d_date_sk and cs_item_sk = i_item_sk and cs_bill_cdemo_sk = cd_demo_sk and cs_promo_sk = p_promo_sk and cd_gender = 'F' and cd_marital_status = 'S' and cd_education_status = 'College' and (p_channel_email = 'N' or p_channel_event = 'N') and d_year = 1998 group by i_item_id order by i_item_id limit 100""", "q27" -> """ select i_item_id, s_state, grouping(s_state) g_state, avg(ss_quantity) agg1, avg(ss_list_price) agg2, avg(ss_coupon_amt) agg3, avg(ss_sales_price) agg4 from store_sales, customer_demographics, date_dim, store, item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and ss_cdemo_sk = cd_demo_sk and cd_gender = 'F' and cd_marital_status = 'U' and cd_education_status = '2 yr Degree' and d_year = 2000 and s_state in ('AL','IN', 'SC', 'NY', 'OH', 'FL') group by rollup (i_item_id, s_state) order by i_item_id ,s_state limit 100""", "q28" -> """ select * from (select avg(ss_list_price) B1_LP ,count(ss_list_price) B1_CNT ,count(distinct ss_list_price) B1_CNTD from store_sales where ss_quantity between 0 and 5 and (ss_list_price between 73 and 73+10 or ss_coupon_amt between 7826 and 7826+1000 or ss_wholesale_cost between 70 and 70+20)) B1, (select avg(ss_list_price) B2_LP ,count(ss_list_price) B2_CNT ,count(distinct ss_list_price) B2_CNTD from store_sales where ss_quantity between 6 and 10 and (ss_list_price between 152 and 152+10 or ss_coupon_amt between 2196 and 2196+1000 or ss_wholesale_cost between 56 and 56+20)) B2, (select avg(ss_list_price) B3_LP ,count(ss_list_price) B3_CNT ,count(distinct ss_list_price) B3_CNTD from store_sales where ss_quantity between 11 and 15 and (ss_list_price between 53 and 53+10 or ss_coupon_amt between 3430 and 3430+1000 or ss_wholesale_cost between 13 and 13+20)) B3, (select avg(ss_list_price) B4_LP ,count(ss_list_price) B4_CNT ,count(distinct ss_list_price) B4_CNTD from store_sales where ss_quantity between 16 and 20 and (ss_list_price between 182 and 182+10 or ss_coupon_amt between 3262 and 3262+1000 or ss_wholesale_cost between 20 and 20+20)) B4, (select avg(ss_list_price) B5_LP ,count(ss_list_price) B5_CNT ,count(distinct ss_list_price) B5_CNTD from store_sales where ss_quantity between 21 and 25 and (ss_list_price between 85 and 85+10 or ss_coupon_amt between 3310 and 3310+1000 or ss_wholesale_cost between 37 and 37+20)) B5, (select avg(ss_list_price) B6_LP ,count(ss_list_price) B6_CNT ,count(distinct ss_list_price) B6_CNTD from store_sales where ss_quantity between 26 and 30 and (ss_list_price between 180 and 180+10 or ss_coupon_amt between 12592 and 12592+1000 or ss_wholesale_cost between 22 and 22+20)) B6 limit 100""", "q29" -> """ select i_item_id ,i_item_desc ,s_store_id ,s_store_name ,stddev_samp(ss_quantity) as store_sales_quantity ,stddev_samp(sr_return_quantity) as store_returns_quantity ,stddev_samp(cs_quantity) as catalog_sales_quantity from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_moy = 4 and d1.d_year = 1998 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_moy between 4 and 4 + 3 and d2.d_year = 1998 and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_year in (1998,1998+1,1998+2) group by i_item_id ,i_item_desc ,s_store_id ,s_store_name order by i_item_id ,i_item_desc ,s_store_id ,s_store_name limit 100""", "q30" -> """ with customer_total_return as (select wr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, sum(wr_return_amt) as ctr_total_return from web_returns ,date_dim ,customer_address where wr_returned_date_sk = d_date_sk and d_year =2000 and wr_returning_addr_sk = ca_address_sk group by wr_returning_customer_sk ,ca_state) select c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address ,c_last_review_date,ctr_total_return from customer_total_return ctr1 ,customer_address ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_state = ctr2.ctr_state) and ca_address_sk = c_current_addr_sk and ca_state = 'GA' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address ,c_last_review_date,ctr_total_return limit 100""", "q31" -> """ with ss as (select ca_county,d_qoy, d_year,sum(ss_ext_sales_price) as store_sales from store_sales,date_dim,customer_address where ss_sold_date_sk = d_date_sk and ss_addr_sk=ca_address_sk group by ca_county,d_qoy, d_year), ws as (select ca_county,d_qoy, d_year,sum(ws_ext_sales_price) as web_sales from web_sales,date_dim,customer_address where ws_sold_date_sk = d_date_sk and ws_bill_addr_sk=ca_address_sk group by ca_county,d_qoy, d_year) select ss1.ca_county ,ss1.d_year ,ws2.web_sales/ws1.web_sales web_q1_q2_increase ,ss2.store_sales/ss1.store_sales store_q1_q2_increase ,ws3.web_sales/ws2.web_sales web_q2_q3_increase ,ss3.store_sales/ss2.store_sales store_q2_q3_increase from ss ss1 ,ss ss2 ,ss ss3 ,ws ws1 ,ws ws2 ,ws ws3 where ss1.d_qoy = 1 and ss1.d_year = 1999 and ss1.ca_county = ss2.ca_county and ss2.d_qoy = 2 and ss2.d_year = 1999 and ss2.ca_county = ss3.ca_county and ss3.d_qoy = 3 and ss3.d_year = 1999 and ss1.ca_county = ws1.ca_county and ws1.d_qoy = 1 and ws1.d_year = 1999 and ws1.ca_county = ws2.ca_county and ws2.d_qoy = 2 and ws2.d_year = 1999 and ws1.ca_county = ws3.ca_county and ws3.d_qoy = 3 and ws3.d_year =1999 and case when ws1.web_sales > 0 then ws2.web_sales/ws1.web_sales else null end > case when ss1.store_sales > 0 then ss2.store_sales/ss1.store_sales else null end and case when ws2.web_sales > 0 then ws3.web_sales/ws2.web_sales else null end > case when ss2.store_sales > 0 then ss3.store_sales/ss2.store_sales else null end order by ss1.d_year""", "q32" -> """ select sum(cs_ext_discount_amt) as `excess discount amount` from catalog_sales ,item ,date_dim where i_manufact_id = 66 and i_item_sk = cs_item_sk and d_date between '2002-03-29' and (cast('2002-03-29' as date) + INTERVAL 90 days) and d_date_sk = cs_sold_date_sk and cs_ext_discount_amt > ( select 1.3 * avg(cs_ext_discount_amt) from catalog_sales ,date_dim where cs_item_sk = i_item_sk and d_date between '2002-03-29' and (cast('2002-03-29' as date) + INTERVAL 90 days) and d_date_sk = cs_sold_date_sk ) limit 100""", "q33" -> """ with ss as ( select i_manufact_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Home')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 5 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_manufact_id), cs as ( select i_manufact_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Home')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 5 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_manufact_id), ws as ( select i_manufact_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Home')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 5 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_manufact_id) select i_manufact_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_manufact_id order by total_sales limit 100""", "q34" -> """ select c_last_name ,c_first_name ,c_salutation ,c_preferred_cust_flag ,ss_ticket_number ,cnt from (select ss_ticket_number ,ss_customer_sk ,count(*) cnt from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and (date_dim.d_dom between 1 and 3 or date_dim.d_dom between 25 and 28) and (household_demographics.hd_buy_potential = '>10000' or household_demographics.hd_buy_potential = 'Unknown') and household_demographics.hd_vehicle_count > 0 and (case when household_demographics.hd_vehicle_count > 0 then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end) > 1.2 and date_dim.d_year in (2000,2000+1,2000+2) and store.s_county in ('Salem County','Terrell County','Arthur County','Oglethorpe County', 'Lunenburg County','Perry County','Halifax County','Sumner County') group by ss_ticket_number,ss_customer_sk) dn,customer where ss_customer_sk = c_customer_sk and cnt between 15 and 20 order by c_last_name,c_first_name,c_salutation,c_preferred_cust_flag desc, ss_ticket_number""", "q35" -> """ select ca_state, cd_gender, cd_marital_status, cd_dep_count, count(*) cnt1, avg(cd_dep_count), min(cd_dep_count), stddev_samp(cd_dep_count), cd_dep_employed_count, count(*) cnt2, avg(cd_dep_employed_count), min(cd_dep_employed_count), stddev_samp(cd_dep_employed_count), cd_dep_college_count, count(*) cnt3, avg(cd_dep_college_count), min(cd_dep_college_count), stddev_samp(cd_dep_college_count) from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2001 and d_qoy < 4) and (exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2001 and d_qoy < 4) or exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2001 and d_qoy < 4)) group by ca_state, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count order by ca_state, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count limit 100""", "q36" -> """ select sum(ss_net_profit)/sum(ss_ext_sales_price) as gross_margin ,i_category ,i_class ,grouping(i_category)+grouping(i_class) as lochierarchy ,rank() over ( partition by grouping(i_category)+grouping(i_class), case when grouping(i_class) = 0 then i_category end order by sum(ss_net_profit)/sum(ss_ext_sales_price) asc) as rank_within_parent from store_sales ,date_dim d1 ,item ,store where d1.d_year = 1999 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and s_state in ('IN','AL','MI','MN', 'TN','LA','FL','NM') group by rollup(i_category,i_class) order by lochierarchy desc ,case when lochierarchy = 0 then i_category end ,rank_within_parent limit 100""", "q37" -> """ select i_item_id ,i_item_desc ,i_current_price from item, inventory, date_dim, catalog_sales where i_current_price between 39 and 39 + 30 and inv_item_sk = i_item_sk and d_date_sk=inv_date_sk and d_date between cast('2001-01-16' as date) and (cast('2001-01-16' as date) + interval 60 days) and i_manufact_id in (765,886,889,728) and inv_quantity_on_hand between 100 and 500 and cs_item_sk = i_item_sk group by i_item_id,i_item_desc,i_current_price order by i_item_id limit 100""", "q38" -> """ select count(*) from ( select distinct c_last_name, c_first_name, d_date from store_sales, date_dim, customer where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_customer_sk = customer.c_customer_sk and d_month_seq between 1186 and 1186 + 11 intersect select distinct c_last_name, c_first_name, d_date from catalog_sales, date_dim, customer where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1186 and 1186 + 11 intersect select distinct c_last_name, c_first_name, d_date from web_sales, date_dim, customer where web_sales.ws_sold_date_sk = date_dim.d_date_sk and web_sales.ws_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1186 and 1186 + 11 ) hot_cust limit 100""", "q39a" -> """ with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stdev,mean, case mean when 0 then null else stdev/mean end cov from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean from inventory ,item ,warehouse ,date_dim where inv_item_sk = i_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_year =2000 group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo where case mean when 0 then 0 else stdev/mean end > 1) select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov from inv inv1,inv inv2 where inv1.i_item_sk = inv2.i_item_sk and inv1.w_warehouse_sk = inv2.w_warehouse_sk and inv1.d_moy=2 and inv2.d_moy=2+1 order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov ,inv2.d_moy,inv2.mean, inv2.cov""", "q39b" -> """ with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stdev,mean, case mean when 0 then null else stdev/mean end cov from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean from inventory ,item ,warehouse ,date_dim where inv_item_sk = i_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_year =2000 group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo where case mean when 0 then 0 else stdev/mean end > 1) select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov from inv inv1,inv inv2 where inv1.i_item_sk = inv2.i_item_sk and inv1.w_warehouse_sk = inv2.w_warehouse_sk and inv1.d_moy=2 and inv2.d_moy=2+1 and inv1.cov > 1.5 order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov ,inv2.d_moy,inv2.mean, inv2.cov""", "q40" -> """ select w_state ,i_item_id ,sum(case when (cast(d_date as date) < cast ('2000-03-18' as date)) then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_before ,sum(case when (cast(d_date as date) >= cast ('2000-03-18' as date)) then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_after from catalog_sales left outer join catalog_returns on (cs_order_number = cr_order_number and cs_item_sk = cr_item_sk) ,warehouse ,item ,date_dim where i_current_price between 0.99 and 1.49 and i_item_sk = cs_item_sk and cs_warehouse_sk = w_warehouse_sk and cs_sold_date_sk = d_date_sk and d_date between (cast ('2000-03-18' as date) - INTERVAL 30 days) and (cast ('2000-03-18' as date) + INTERVAL 30 days) group by w_state,i_item_id order by w_state,i_item_id limit 100""", "q41" -> """ select distinct(i_product_name) from item i1 where i_manufact_id between 970 and 970+40 and (select count(*) as item_cnt from item where (i_manufact = i1.i_manufact and ((i_category = 'Women' and (i_color = 'frosted' or i_color = 'rose') and (i_units = 'Lb' or i_units = 'Gross') and (i_size = 'medium' or i_size = 'large') ) or (i_category = 'Women' and (i_color = 'chocolate' or i_color = 'black') and (i_units = 'Box' or i_units = 'Dram') and (i_size = 'economy' or i_size = 'petite') ) or (i_category = 'Men' and (i_color = 'slate' or i_color = 'magenta') and (i_units = 'Carton' or i_units = 'Bundle') and (i_size = 'N/A' or i_size = 'small') ) or (i_category = 'Men' and (i_color = 'cornflower' or i_color = 'firebrick') and (i_units = 'Pound' or i_units = 'Oz') and (i_size = 'medium' or i_size = 'large') ))) or (i_manufact = i1.i_manufact and ((i_category = 'Women' and (i_color = 'almond' or i_color = 'steel') and (i_units = 'Tsp' or i_units = 'Case') and (i_size = 'medium' or i_size = 'large') ) or (i_category = 'Women' and (i_color = 'purple' or i_color = 'aquamarine') and (i_units = 'Bunch' or i_units = 'Gram') and (i_size = 'economy' or i_size = 'petite') ) or (i_category = 'Men' and (i_color = 'lavender' or i_color = 'papaya') and (i_units = 'Pallet' or i_units = 'Cup') and (i_size = 'N/A' or i_size = 'small') ) or (i_category = 'Men' and (i_color = 'maroon' or i_color = 'cyan') and (i_units = 'Each' or i_units = 'N/A') and (i_size = 'medium' or i_size = 'large') )))) > 0 order by i_product_name limit 100""", "q42" -> """ select dt.d_year ,item.i_category_id ,item.i_category ,sum(ss_ext_sales_price) from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manager_id = 1 and dt.d_moy=12 and dt.d_year=1998 group by dt.d_year ,item.i_category_id ,item.i_category order by sum(ss_ext_sales_price) desc,dt.d_year ,item.i_category_id ,item.i_category limit 100 """, "q43" -> """ select s_store_name, s_store_id, sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales from date_dim, store_sales, store where d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk and s_gmt_offset = -6 and d_year = 2001 group by s_store_name, s_store_id order by s_store_name, s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales limit 100""", "q44" -> """ select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing from(select * from (select item_sk,rank() over (order by rank_col asc) rnk from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col from store_sales ss1 where ss_store_sk = 366 group by ss_item_sk having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col from store_sales where ss_store_sk = 366 and ss_cdemo_sk is null group by ss_store_sk))V1)V11 where rnk < 11) asceding, (select * from (select item_sk,rank() over (order by rank_col desc) rnk from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col from store_sales ss1 where ss_store_sk = 366 group by ss_item_sk having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col from store_sales where ss_store_sk = 366 and ss_cdemo_sk is null group by ss_store_sk))V2)V21 where rnk < 11) descending, item i1, item i2 where asceding.rnk = descending.rnk and i1.i_item_sk=asceding.item_sk and i2.i_item_sk=descending.item_sk order by asceding.rnk limit 100""", "q45" -> """ select ca_zip, ca_county, sum(ws_sales_price) from web_sales, customer, customer_address, date_dim, item where ws_bill_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and ws_item_sk = i_item_sk and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') or i_item_id in (select i_item_id from item where i_item_sk in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) ) ) and ws_sold_date_sk = d_date_sk and d_qoy = 1 and d_year = 1998 group by ca_zip, ca_county order by ca_zip, ca_county limit 100""", "q46" -> """ select c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number ,amt,profit from (select ss_ticket_number ,ss_customer_sk ,ca_city bought_city ,sum(ss_coupon_amt) amt ,sum(ss_net_profit) profit from store_sales,date_dim,store,household_demographics,customer_address where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and store_sales.ss_addr_sk = customer_address.ca_address_sk and (household_demographics.hd_dep_count = 0 or household_demographics.hd_vehicle_count= 1) and date_dim.d_dow in (6,0) and date_dim.d_year in (2000,2000+1,2000+2) and store.s_city in ('Five Forks','Oakland','Fairview','Winchester','Farmington') group by ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city) dn,customer,customer_address current_addr where ss_customer_sk = c_customer_sk and customer.c_current_addr_sk = current_addr.ca_address_sk and current_addr.ca_city <> bought_city order by c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number limit 100""", "q47" -> """ with v1 as( select i_category, i_brand, s_store_name, s_company_name, d_year, d_moy, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_category, i_brand, s_store_name, s_company_name, d_year) avg_monthly_sales, rank() over (partition by i_category, i_brand, s_store_name, s_company_name order by d_year, d_moy) rn from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ( d_year = 1999 or ( d_year = 1999-1 and d_moy =12) or ( d_year = 1999+1 and d_moy =1) ) group by i_category, i_brand, s_store_name, s_company_name, d_year, d_moy), v2 as( select v1.s_store_name ,v1.d_year, v1.d_moy ,v1.avg_monthly_sales ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum from v1, v1 v1_lag, v1 v1_lead where v1.i_category = v1_lag.i_category and v1.i_category = v1_lead.i_category and v1.i_brand = v1_lag.i_brand and v1.i_brand = v1_lead.i_brand and v1.s_store_name = v1_lag.s_store_name and v1.s_store_name = v1_lead.s_store_name and v1.s_company_name = v1_lag.s_company_name and v1.s_company_name = v1_lead.s_company_name and v1.rn = v1_lag.rn + 1 and v1.rn = v1_lead.rn - 1) select * from v2 where d_year = 1999 and avg_monthly_sales > 0 and case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by sum_sales - avg_monthly_sales, sum_sales limit 100""", "q48" -> """ select sum (ss_quantity) from store_sales, store, customer_demographics, customer_address, date_dim where s_store_sk = ss_store_sk and ss_sold_date_sk = d_date_sk and d_year = 1998 and ( ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'M' and cd_education_status = 'Unknown' and ss_sales_price between 100.00 and 150.00 ) or ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'W' and cd_education_status = 'College' and ss_sales_price between 50.00 and 100.00 ) or ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'D' and cd_education_status = 'Primary' and ss_sales_price between 150.00 and 200.00 ) ) and ( ( ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('MI', 'GA', 'NH') and ss_net_profit between 0 and 2000 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('TX', 'KY', 'SD') and ss_net_profit between 150 and 3000 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('NY', 'OH', 'FL') and ss_net_profit between 50 and 25000 ) )""", "q49" -> """ select channel, item, return_ratio, return_rank, currency_rank from (select 'web' as channel ,web.item ,web.return_ratio ,web.return_rank ,web.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select ws.ws_item_sk as item ,(cast(sum(coalesce(wr.wr_return_quantity,0)) as decimal(15,4))/ cast(sum(coalesce(ws.ws_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(wr.wr_return_amt,0)) as decimal(15,4))/ cast(sum(coalesce(ws.ws_net_paid,0)) as decimal(15,4) )) as currency_ratio from web_sales ws left outer join web_returns wr on (ws.ws_order_number = wr.wr_order_number and ws.ws_item_sk = wr.wr_item_sk) ,date_dim where wr.wr_return_amt > 10000 and ws.ws_net_profit > 1 and ws.ws_net_paid > 0 and ws.ws_quantity > 0 and ws_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 12 group by ws.ws_item_sk ) in_web ) web where ( web.return_rank <= 10 or web.currency_rank <= 10 ) union select 'catalog' as channel ,catalog.item ,catalog.return_ratio ,catalog.return_rank ,catalog.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select cs.cs_item_sk as item ,(cast(sum(coalesce(cr.cr_return_quantity,0)) as decimal(15,4))/ cast(sum(coalesce(cs.cs_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(cr.cr_return_amount,0)) as decimal(15,4))/ cast(sum(coalesce(cs.cs_net_paid,0)) as decimal(15,4) )) as currency_ratio from catalog_sales cs left outer join catalog_returns cr on (cs.cs_order_number = cr.cr_order_number and cs.cs_item_sk = cr.cr_item_sk) ,date_dim where cr.cr_return_amount > 10000 and cs.cs_net_profit > 1 and cs.cs_net_paid > 0 and cs.cs_quantity > 0 and cs_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 12 group by cs.cs_item_sk ) in_cat ) catalog where ( catalog.return_rank <= 10 or catalog.currency_rank <=10 ) union select 'store' as channel ,store.item ,store.return_ratio ,store.return_rank ,store.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select sts.ss_item_sk as item ,(cast(sum(coalesce(sr.sr_return_quantity,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(sr.sr_return_amt,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_net_paid,0)) as decimal(15,4) )) as currency_ratio from store_sales sts left outer join store_returns sr on (sts.ss_ticket_number = sr.sr_ticket_number and sts.ss_item_sk = sr.sr_item_sk) ,date_dim where sr.sr_return_amt > 10000 and sts.ss_net_profit > 1 and sts.ss_net_paid > 0 and sts.ss_quantity > 0 and ss_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 12 group by sts.ss_item_sk ) in_store ) store where ( store.return_rank <= 10 or store.currency_rank <= 10 ) ) order by 1,4,5,2 limit 100""", "q50" -> """ select s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip ,sum(case when (sr_returned_date_sk - ss_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 30) and (sr_returned_date_sk - ss_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 60) and (sr_returned_date_sk - ss_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 90) and (sr_returned_date_sk - ss_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from store_sales ,store_returns ,store ,date_dim d1 ,date_dim d2 where d2.d_year = 1998 and d2.d_moy = 9 and ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_sold_date_sk = d1.d_date_sk and sr_returned_date_sk = d2.d_date_sk and ss_customer_sk = sr_customer_sk and ss_store_sk = s_store_sk group by s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip order by s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip limit 100""", "q51" -> """ WITH web_v1 as ( select ws_item_sk item_sk, d_date, sum(sum(ws_sales_price)) over (partition by ws_item_sk order by d_date rows between unbounded preceding and current row) cume_sales from web_sales ,date_dim where ws_sold_date_sk=d_date_sk and d_month_seq between 1214 and 1214+11 and ws_item_sk is not NULL group by ws_item_sk, d_date), store_v1 as ( select ss_item_sk item_sk, d_date, sum(sum(ss_sales_price)) over (partition by ss_item_sk order by d_date rows between unbounded preceding and current row) cume_sales from store_sales ,date_dim where ss_sold_date_sk=d_date_sk and d_month_seq between 1214 and 1214+11 and ss_item_sk is not NULL group by ss_item_sk, d_date) select * from (select item_sk ,d_date ,web_sales ,store_sales ,max(web_sales) over (partition by item_sk order by d_date rows between unbounded preceding and current row) web_cumulative ,max(store_sales) over (partition by item_sk order by d_date rows between unbounded preceding and current row) store_cumulative from (select case when web.item_sk is not null then web.item_sk else store.item_sk end item_sk ,case when web.d_date is not null then web.d_date else store.d_date end d_date ,web.cume_sales web_sales ,store.cume_sales store_sales from web_v1 web full outer join store_v1 store on (web.item_sk = store.item_sk and web.d_date = store.d_date) )x )y where web_cumulative > store_cumulative order by item_sk ,d_date limit 100""", "q52" -> """ select dt.d_year ,item.i_brand_id brand_id ,item.i_brand brand ,sum(ss_ext_sales_price) ext_price from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manager_id = 1 and dt.d_moy=12 and dt.d_year=2000 group by dt.d_year ,item.i_brand ,item.i_brand_id order by dt.d_year ,ext_price desc ,brand_id limit 100 """, "q53" -> """ select * from (select i_manufact_id, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_manufact_id) avg_quarterly_sales from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_month_seq in (1212,1212+1,1212+2,1212+3,1212+4,1212+5,1212+6,1212+7,1212+8,1212+9,1212+10,1212+11) and ((i_category in ('Books','Children','Electronics') and i_class in ('personal','portable','reference','self-help') and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', 'exportiunivamalg #9','scholaramalgamalg #9')) or(i_category in ('Women','Music','Men') and i_class in ('accessories','classical','fragrances','pants') and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 'importoamalg #1'))) group by i_manufact_id, d_qoy ) tmp1 where case when avg_quarterly_sales > 0 then abs (sum_sales - avg_quarterly_sales)/ avg_quarterly_sales else null end > 0.1 order by avg_quarterly_sales, sum_sales, i_manufact_id limit 100""", "q54" -> """ with my_customers as ( select distinct c_customer_sk , c_current_addr_sk from ( select cs_sold_date_sk sold_date_sk, cs_bill_customer_sk customer_sk, cs_item_sk item_sk from catalog_sales union all select ws_sold_date_sk sold_date_sk, ws_bill_customer_sk customer_sk, ws_item_sk item_sk from web_sales ) cs_or_ws_sales, item, date_dim, customer where sold_date_sk = d_date_sk and item_sk = i_item_sk and i_category = 'Books' and i_class = 'business' and c_customer_sk = cs_or_ws_sales.customer_sk and d_moy = 2 and d_year = 2000 ) , my_revenue as ( select c_customer_sk, sum(ss_ext_sales_price) as revenue from my_customers, store_sales, customer_address, store, date_dim where c_current_addr_sk = ca_address_sk and ca_county = s_county and ca_state = s_state and ss_sold_date_sk = d_date_sk and c_customer_sk = ss_customer_sk and d_month_seq between (select distinct d_month_seq+1 from date_dim where d_year = 2000 and d_moy = 2) and (select distinct d_month_seq+3 from date_dim where d_year = 2000 and d_moy = 2) group by c_customer_sk ) , segments as (select cast((revenue/50) as int) as segment from my_revenue ) select segment, count(*) as num_customers, segment*50 as segment_base from segments group by segment order by segment, num_customers limit 100""", "q55" -> """ select i_brand_id brand_id, i_brand brand, sum(ss_ext_sales_price) ext_price from date_dim, store_sales, item where d_date_sk = ss_sold_date_sk and ss_item_sk = i_item_sk and i_manager_id=13 and d_moy=11 and d_year=1999 group by i_brand, i_brand_id order by ext_price desc, i_brand_id limit 100 """, "q56" -> """ with ss as ( select i_item_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('chiffon','smoke','lace')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 2001 and d_moy = 5 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_item_id), cs as ( select i_item_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('chiffon','smoke','lace')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 2001 and d_moy = 5 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_item_id), ws as ( select i_item_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('chiffon','smoke','lace')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 2001 and d_moy = 5 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_item_id) select i_item_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_item_id order by total_sales, i_item_id limit 100""", "q57" -> """ with v1 as( select i_category, i_brand, cc_name, d_year, d_moy, sum(cs_sales_price) sum_sales, avg(sum(cs_sales_price)) over (partition by i_category, i_brand, cc_name, d_year) avg_monthly_sales, rank() over (partition by i_category, i_brand, cc_name order by d_year, d_moy) rn from item, catalog_sales, date_dim, call_center where cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and cc_call_center_sk= cs_call_center_sk and ( d_year = 1999 or ( d_year = 1999-1 and d_moy =12) or ( d_year = 1999+1 and d_moy =1) ) group by i_category, i_brand, cc_name , d_year, d_moy), v2 as( select v1.i_category, v1.i_brand ,v1.d_year, v1.d_moy ,v1.avg_monthly_sales ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum from v1, v1 v1_lag, v1 v1_lead where v1.i_category = v1_lag.i_category and v1.i_category = v1_lead.i_category and v1.i_brand = v1_lag.i_brand and v1.i_brand = v1_lead.i_brand and v1. cc_name = v1_lag. cc_name and v1. cc_name = v1_lead. cc_name and v1.rn = v1_lag.rn + 1 and v1.rn = v1_lead.rn - 1) select * from v2 where d_year = 1999 and avg_monthly_sales > 0 and case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by sum_sales - avg_monthly_sales, avg_monthly_sales limit 100""", "q58" -> """ with ss_items as (select i_item_id item_id ,sum(ss_ext_sales_price) ss_item_rev from store_sales ,item ,date_dim where ss_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq = (select d_week_seq from date_dim where d_date = '1998-02-21')) and ss_sold_date_sk = d_date_sk group by i_item_id), cs_items as (select i_item_id item_id ,sum(cs_ext_sales_price) cs_item_rev from catalog_sales ,item ,date_dim where cs_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq = (select d_week_seq from date_dim where d_date = '1998-02-21')) and cs_sold_date_sk = d_date_sk group by i_item_id), ws_items as (select i_item_id item_id ,sum(ws_ext_sales_price) ws_item_rev from web_sales ,item ,date_dim where ws_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq =(select d_week_seq from date_dim where d_date = '1998-02-21')) and ws_sold_date_sk = d_date_sk group by i_item_id) select ss_items.item_id ,ss_item_rev ,ss_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ss_dev ,cs_item_rev ,cs_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 cs_dev ,ws_item_rev ,ws_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ws_dev ,(ss_item_rev+cs_item_rev+ws_item_rev)/3 average from ss_items,cs_items,ws_items where ss_items.item_id=cs_items.item_id and ss_items.item_id=ws_items.item_id and ss_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev and ss_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev and cs_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev and cs_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev and ws_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev and ws_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev order by item_id ,ss_item_rev limit 100""", "q59" -> """ with wss as (select d_week_seq, ss_store_sk, sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales from store_sales,date_dim where d_date_sk = ss_sold_date_sk group by d_week_seq,ss_store_sk ) select s_store_name1,s_store_id1,d_week_seq1 ,sun_sales1/sun_sales2,mon_sales1/mon_sales2 ,tue_sales1/tue_sales2,wed_sales1/wed_sales2,thu_sales1/thu_sales2 ,fri_sales1/fri_sales2,sat_sales1/sat_sales2 from (select s_store_name s_store_name1,wss.d_week_seq d_week_seq1 ,s_store_id s_store_id1,sun_sales sun_sales1 ,mon_sales mon_sales1,tue_sales tue_sales1 ,wed_sales wed_sales1,thu_sales thu_sales1 ,fri_sales fri_sales1,sat_sales sat_sales1 from wss,store,date_dim d where d.d_week_seq = wss.d_week_seq and ss_store_sk = s_store_sk and d_month_seq between 1205 and 1205 + 11) y, (select s_store_name s_store_name2,wss.d_week_seq d_week_seq2 ,s_store_id s_store_id2,sun_sales sun_sales2 ,mon_sales mon_sales2,tue_sales tue_sales2 ,wed_sales wed_sales2,thu_sales thu_sales2 ,fri_sales fri_sales2,sat_sales sat_sales2 from wss,store,date_dim d where d.d_week_seq = wss.d_week_seq and ss_store_sk = s_store_sk and d_month_seq between 1205+ 12 and 1205 + 23) x where s_store_id1=s_store_id2 and d_week_seq1=d_week_seq2-52 order by s_store_name1,s_store_id1,d_week_seq1 limit 100""", "q60" -> """ with ss as ( select i_item_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Children')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 10 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id), cs as ( select i_item_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Children')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 10 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id), ws as ( select i_item_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Children')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 10 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id) select i_item_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_item_id order by i_item_id ,total_sales limit 100""", "q61" -> """ select promotions,total,cast(promotions as decimal(15,4))/cast(total as decimal(15,4))*100 from (select sum(ss_ext_sales_price) promotions from store_sales ,store ,promotion ,date_dim ,customer ,customer_address ,item where ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ss_promo_sk = p_promo_sk and ss_customer_sk= c_customer_sk and ca_address_sk = c_current_addr_sk and ss_item_sk = i_item_sk and ca_gmt_offset = -6 and i_category = 'Sports' and (p_channel_dmail = 'Y' or p_channel_email = 'Y' or p_channel_tv = 'Y') and s_gmt_offset = -6 and d_year = 2001 and d_moy = 12) promotional_sales, (select sum(ss_ext_sales_price) total from store_sales ,store ,date_dim ,customer ,customer_address ,item where ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ss_customer_sk= c_customer_sk and ca_address_sk = c_current_addr_sk and ss_item_sk = i_item_sk and ca_gmt_offset = -6 and i_category = 'Sports' and s_gmt_offset = -6 and d_year = 2001 and d_moy = 12) all_sales order by promotions, total limit 100""", "q62" -> """ select substr(w_warehouse_name,1,20) ,sm_type ,web_name ,sum(case when (ws_ship_date_sk - ws_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 30) and (ws_ship_date_sk - ws_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 60) and (ws_ship_date_sk - ws_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 90) and (ws_ship_date_sk - ws_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from web_sales ,warehouse ,ship_mode ,web_site ,date_dim where d_month_seq between 1215 and 1215 + 11 and ws_ship_date_sk = d_date_sk and ws_warehouse_sk = w_warehouse_sk and ws_ship_mode_sk = sm_ship_mode_sk and ws_web_site_sk = web_site_sk group by substr(w_warehouse_name,1,20) ,sm_type ,web_name order by substr(w_warehouse_name,1,20) ,sm_type ,web_name limit 100""", "q63" -> """ select * from (select i_manager_id ,sum(ss_sales_price) sum_sales ,avg(sum(ss_sales_price)) over (partition by i_manager_id) avg_monthly_sales from item ,store_sales ,date_dim ,store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_month_seq in (1211,1211+1,1211+2,1211+3,1211+4,1211+5,1211+6,1211+7,1211+8,1211+9,1211+10,1211+11) and (( i_category in ('Books','Children','Electronics') and i_class in ('personal','portable','reference','self-help') and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', 'exportiunivamalg #9','scholaramalgamalg #9')) or( i_category in ('Women','Music','Men') and i_class in ('accessories','classical','fragrances','pants') and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 'importoamalg #1'))) group by i_manager_id, d_moy) tmp1 where case when avg_monthly_sales > 0 then abs (sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by i_manager_id ,avg_monthly_sales ,sum_sales limit 100""", "q64" -> """ with cs_ui as (select cs_item_sk ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund from catalog_sales ,catalog_returns where cs_item_sk = cr_item_sk and cs_order_number = cr_order_number group by cs_item_sk having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), cross_sales as (select i_product_name product_name ,i_item_sk item_sk ,s_store_name store_name ,s_zip store_zip ,ad1.ca_street_number b_street_number ,ad1.ca_street_name b_street_name ,ad1.ca_city b_city ,ad1.ca_zip b_zip ,ad2.ca_street_number c_street_number ,ad2.ca_street_name c_street_name ,ad2.ca_city c_city ,ad2.ca_zip c_zip ,d1.d_year as syear ,d2.d_year as fsyear ,d3.d_year s2year ,count(*) cnt ,sum(ss_wholesale_cost) s1 ,sum(ss_list_price) s2 ,sum(ss_coupon_amt) s3 FROM store_sales ,store_returns ,cs_ui ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,customer ,customer_demographics cd1 ,customer_demographics cd2 ,promotion ,household_demographics hd1 ,household_demographics hd2 ,customer_address ad1 ,customer_address ad2 ,income_band ib1 ,income_band ib2 ,item WHERE ss_store_sk = s_store_sk AND ss_sold_date_sk = d1.d_date_sk AND ss_customer_sk = c_customer_sk AND ss_cdemo_sk= cd1.cd_demo_sk AND ss_hdemo_sk = hd1.hd_demo_sk AND ss_addr_sk = ad1.ca_address_sk and ss_item_sk = i_item_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and ss_item_sk = cs_ui.cs_item_sk and c_current_cdemo_sk = cd2.cd_demo_sk AND c_current_hdemo_sk = hd2.hd_demo_sk AND c_current_addr_sk = ad2.ca_address_sk and c_first_sales_date_sk = d2.d_date_sk and c_first_shipto_date_sk = d3.d_date_sk and ss_promo_sk = p_promo_sk and hd1.hd_income_band_sk = ib1.ib_income_band_sk and hd2.hd_income_band_sk = ib2.ib_income_band_sk and cd1.cd_marital_status <> cd2.cd_marital_status and i_color in ('azure','gainsboro','misty','blush','hot','lemon') and i_current_price between 80 and 80 + 10 and i_current_price between 80 + 1 and 80 + 15 group by i_product_name ,i_item_sk ,s_store_name ,s_zip ,ad1.ca_street_number ,ad1.ca_street_name ,ad1.ca_city ,ad1.ca_zip ,ad2.ca_street_number ,ad2.ca_street_name ,ad2.ca_city ,ad2.ca_zip ,d1.d_year ,d2.d_year ,d3.d_year ) select cs1.product_name ,cs1.store_name ,cs1.store_zip ,cs1.b_street_number ,cs1.b_street_name ,cs1.b_city ,cs1.b_zip ,cs1.c_street_number ,cs1.c_street_name ,cs1.c_city ,cs1.c_zip ,cs1.syear ,cs1.cnt ,cs1.s1 as s11 ,cs1.s2 as s21 ,cs1.s3 as s31 ,cs2.s1 as s12 ,cs2.s2 as s22 ,cs2.s3 as s32 ,cs2.syear ,cs2.cnt from cross_sales cs1,cross_sales cs2 where cs1.item_sk=cs2.item_sk and cs1.syear = 1999 and cs2.syear = 1999 + 1 and cs2.cnt <= cs1.cnt and cs1.store_name = cs2.store_name and cs1.store_zip = cs2.store_zip order by cs1.product_name ,cs1.store_name ,cs2.cnt ,cs1.s1 ,cs2.s1""", "q65" -> """ select s_store_name, i_item_desc, sc.revenue, i_current_price, i_wholesale_cost, i_brand from store, item, (select ss_store_sk, avg(revenue) as ave from (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue from store_sales, date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1186 and 1186+11 group by ss_store_sk, ss_item_sk) sa group by ss_store_sk) sb, (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue from store_sales, date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1186 and 1186+11 group by ss_store_sk, ss_item_sk) sc where sb.ss_store_sk = sc.ss_store_sk and sc.revenue <= 0.1 * sb.ave and s_store_sk = sc.ss_store_sk and i_item_sk = sc.ss_item_sk order by s_store_name, i_item_desc limit 100""", "q66" -> """ select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,ship_carriers ,year ,sum(jan_sales) as jan_sales ,sum(feb_sales) as feb_sales ,sum(mar_sales) as mar_sales ,sum(apr_sales) as apr_sales ,sum(may_sales) as may_sales ,sum(jun_sales) as jun_sales ,sum(jul_sales) as jul_sales ,sum(aug_sales) as aug_sales ,sum(sep_sales) as sep_sales ,sum(oct_sales) as oct_sales ,sum(nov_sales) as nov_sales ,sum(dec_sales) as dec_sales ,sum(jan_sales/w_warehouse_sq_ft) as jan_sales_per_sq_foot ,sum(feb_sales/w_warehouse_sq_ft) as feb_sales_per_sq_foot ,sum(mar_sales/w_warehouse_sq_ft) as mar_sales_per_sq_foot ,sum(apr_sales/w_warehouse_sq_ft) as apr_sales_per_sq_foot ,sum(may_sales/w_warehouse_sq_ft) as may_sales_per_sq_foot ,sum(jun_sales/w_warehouse_sq_ft) as jun_sales_per_sq_foot ,sum(jul_sales/w_warehouse_sq_ft) as jul_sales_per_sq_foot ,sum(aug_sales/w_warehouse_sq_ft) as aug_sales_per_sq_foot ,sum(sep_sales/w_warehouse_sq_ft) as sep_sales_per_sq_foot ,sum(oct_sales/w_warehouse_sq_ft) as oct_sales_per_sq_foot ,sum(nov_sales/w_warehouse_sq_ft) as nov_sales_per_sq_foot ,sum(dec_sales/w_warehouse_sq_ft) as dec_sales_per_sq_foot ,sum(jan_net) as jan_net ,sum(feb_net) as feb_net ,sum(mar_net) as mar_net ,sum(apr_net) as apr_net ,sum(may_net) as may_net ,sum(jun_net) as jun_net ,sum(jul_net) as jul_net ,sum(aug_net) as aug_net ,sum(sep_net) as sep_net ,sum(oct_net) as oct_net ,sum(nov_net) as nov_net ,sum(dec_net) as dec_net from ( select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,'MSC' || ',' || 'GERMA' as ship_carriers ,d_year as year ,sum(case when d_moy = 1 then ws_sales_price* ws_quantity else 0 end) as jan_sales ,sum(case when d_moy = 2 then ws_sales_price* ws_quantity else 0 end) as feb_sales ,sum(case when d_moy = 3 then ws_sales_price* ws_quantity else 0 end) as mar_sales ,sum(case when d_moy = 4 then ws_sales_price* ws_quantity else 0 end) as apr_sales ,sum(case when d_moy = 5 then ws_sales_price* ws_quantity else 0 end) as may_sales ,sum(case when d_moy = 6 then ws_sales_price* ws_quantity else 0 end) as jun_sales ,sum(case when d_moy = 7 then ws_sales_price* ws_quantity else 0 end) as jul_sales ,sum(case when d_moy = 8 then ws_sales_price* ws_quantity else 0 end) as aug_sales ,sum(case when d_moy = 9 then ws_sales_price* ws_quantity else 0 end) as sep_sales ,sum(case when d_moy = 10 then ws_sales_price* ws_quantity else 0 end) as oct_sales ,sum(case when d_moy = 11 then ws_sales_price* ws_quantity else 0 end) as nov_sales ,sum(case when d_moy = 12 then ws_sales_price* ws_quantity else 0 end) as dec_sales ,sum(case when d_moy = 1 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as jan_net ,sum(case when d_moy = 2 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as feb_net ,sum(case when d_moy = 3 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as mar_net ,sum(case when d_moy = 4 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as apr_net ,sum(case when d_moy = 5 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as may_net ,sum(case when d_moy = 6 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as jun_net ,sum(case when d_moy = 7 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as jul_net ,sum(case when d_moy = 8 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as aug_net ,sum(case when d_moy = 9 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as sep_net ,sum(case when d_moy = 10 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as oct_net ,sum(case when d_moy = 11 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as nov_net ,sum(case when d_moy = 12 then ws_net_paid_inc_ship_tax * ws_quantity else 0 end) as dec_net from web_sales ,warehouse ,date_dim ,time_dim ,ship_mode where ws_warehouse_sk = w_warehouse_sk and ws_sold_date_sk = d_date_sk and ws_sold_time_sk = t_time_sk and ws_ship_mode_sk = sm_ship_mode_sk and d_year = 2001 and t_time between 9453 and 9453+28800 and sm_carrier in ('MSC','GERMA') group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,d_year union all select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,'MSC' || ',' || 'GERMA' as ship_carriers ,d_year as year ,sum(case when d_moy = 1 then cs_ext_list_price* cs_quantity else 0 end) as jan_sales ,sum(case when d_moy = 2 then cs_ext_list_price* cs_quantity else 0 end) as feb_sales ,sum(case when d_moy = 3 then cs_ext_list_price* cs_quantity else 0 end) as mar_sales ,sum(case when d_moy = 4 then cs_ext_list_price* cs_quantity else 0 end) as apr_sales ,sum(case when d_moy = 5 then cs_ext_list_price* cs_quantity else 0 end) as may_sales ,sum(case when d_moy = 6 then cs_ext_list_price* cs_quantity else 0 end) as jun_sales ,sum(case when d_moy = 7 then cs_ext_list_price* cs_quantity else 0 end) as jul_sales ,sum(case when d_moy = 8 then cs_ext_list_price* cs_quantity else 0 end) as aug_sales ,sum(case when d_moy = 9 then cs_ext_list_price* cs_quantity else 0 end) as sep_sales ,sum(case when d_moy = 10 then cs_ext_list_price* cs_quantity else 0 end) as oct_sales ,sum(case when d_moy = 11 then cs_ext_list_price* cs_quantity else 0 end) as nov_sales ,sum(case when d_moy = 12 then cs_ext_list_price* cs_quantity else 0 end) as dec_sales ,sum(case when d_moy = 1 then cs_net_paid_inc_ship * cs_quantity else 0 end) as jan_net ,sum(case when d_moy = 2 then cs_net_paid_inc_ship * cs_quantity else 0 end) as feb_net ,sum(case when d_moy = 3 then cs_net_paid_inc_ship * cs_quantity else 0 end) as mar_net ,sum(case when d_moy = 4 then cs_net_paid_inc_ship * cs_quantity else 0 end) as apr_net ,sum(case when d_moy = 5 then cs_net_paid_inc_ship * cs_quantity else 0 end) as may_net ,sum(case when d_moy = 6 then cs_net_paid_inc_ship * cs_quantity else 0 end) as jun_net ,sum(case when d_moy = 7 then cs_net_paid_inc_ship * cs_quantity else 0 end) as jul_net ,sum(case when d_moy = 8 then cs_net_paid_inc_ship * cs_quantity else 0 end) as aug_net ,sum(case when d_moy = 9 then cs_net_paid_inc_ship * cs_quantity else 0 end) as sep_net ,sum(case when d_moy = 10 then cs_net_paid_inc_ship * cs_quantity else 0 end) as oct_net ,sum(case when d_moy = 11 then cs_net_paid_inc_ship * cs_quantity else 0 end) as nov_net ,sum(case when d_moy = 12 then cs_net_paid_inc_ship * cs_quantity else 0 end) as dec_net from catalog_sales ,warehouse ,date_dim ,time_dim ,ship_mode where cs_warehouse_sk = w_warehouse_sk and cs_sold_date_sk = d_date_sk and cs_sold_time_sk = t_time_sk and cs_ship_mode_sk = sm_ship_mode_sk and d_year = 2001 and t_time between 9453 AND 9453+28800 and sm_carrier in ('MSC','GERMA') group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,d_year ) x group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,ship_carriers ,year order by w_warehouse_name limit 100""", "q67" -> """ select * from (select i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sumsales ,rank() over (partition by i_category order by sumsales desc) rk from (select i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sum(coalesce(ss_sales_price*ss_quantity,0)) sumsales from store_sales ,date_dim ,store ,item where ss_sold_date_sk=d_date_sk and ss_item_sk=i_item_sk and ss_store_sk = s_store_sk and d_month_seq between 1185 and 1185+11 group by rollup(i_category, i_class, i_brand, i_product_name, d_year, d_qoy, d_moy,s_store_id))dw1) dw2 where rk <= 100 order by i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sumsales ,rk limit 100""", "q68" -> """ select c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number ,extended_price ,extended_tax ,list_price from (select ss_ticket_number ,ss_customer_sk ,ca_city bought_city ,sum(ss_ext_sales_price) extended_price ,sum(ss_ext_list_price) list_price ,sum(ss_ext_tax) extended_tax from store_sales ,date_dim ,store ,household_demographics ,customer_address where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and store_sales.ss_addr_sk = customer_address.ca_address_sk and date_dim.d_dom between 1 and 2 and (household_demographics.hd_dep_count = 4 or household_demographics.hd_vehicle_count= 0) and date_dim.d_year in (1999,1999+1,1999+2) and store.s_city in ('Pleasant Hill','Bethel') group by ss_ticket_number ,ss_customer_sk ,ss_addr_sk,ca_city) dn ,customer ,customer_address current_addr where ss_customer_sk = c_customer_sk and customer.c_current_addr_sk = current_addr.ca_address_sk and current_addr.ca_city <> bought_city order by c_last_name ,ss_ticket_number limit 100""", "q69" -> """ select cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3 from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and ca_state in ('MO','MN','AZ') and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2003 and d_moy between 2 and 2+2) and (not exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2003 and d_moy between 2 and 2+2) and not exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2003 and d_moy between 2 and 2+2)) group by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating order by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating limit 100""", "q70" -> """ select sum(ss_net_profit) as total_sum ,s_state ,s_county ,grouping(s_state)+grouping(s_county) as lochierarchy ,rank() over ( partition by grouping(s_state)+grouping(s_county), case when grouping(s_county) = 0 then s_state end order by sum(ss_net_profit) desc) as rank_within_parent from store_sales ,date_dim d1 ,store where d1.d_month_seq between 1218 and 1218+11 and d1.d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk and s_state in ( select s_state from (select s_state as s_state, rank() over ( partition by s_state order by sum(ss_net_profit) desc) as ranking from store_sales, store, date_dim where d_month_seq between 1218 and 1218+11 and d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk group by s_state ) tmp1 where ranking <= 5 ) group by rollup(s_state,s_county) order by lochierarchy desc ,case when lochierarchy = 0 then s_state end ,rank_within_parent limit 100""", "q71" -> """ select i_brand_id brand_id, i_brand brand,t_hour,t_minute, sum(ext_price) ext_price from item, (select ws_ext_sales_price as ext_price, ws_sold_date_sk as sold_date_sk, ws_item_sk as sold_item_sk, ws_sold_time_sk as time_sk from web_sales,date_dim where d_date_sk = ws_sold_date_sk and d_moy=12 and d_year=2000 union all select cs_ext_sales_price as ext_price, cs_sold_date_sk as sold_date_sk, cs_item_sk as sold_item_sk, cs_sold_time_sk as time_sk from catalog_sales,date_dim where d_date_sk = cs_sold_date_sk and d_moy=12 and d_year=2000 union all select ss_ext_sales_price as ext_price, ss_sold_date_sk as sold_date_sk, ss_item_sk as sold_item_sk, ss_sold_time_sk as time_sk from store_sales,date_dim where d_date_sk = ss_sold_date_sk and d_moy=12 and d_year=2000 ) tmp,time_dim where sold_item_sk = i_item_sk and i_manager_id=1 and time_sk = t_time_sk and (t_meal_time = 'breakfast' or t_meal_time = 'dinner') group by i_brand, i_brand_id,t_hour,t_minute order by ext_price desc, i_brand_id """, "q72" -> """ select i_item_desc ,w_warehouse_name ,d1.d_week_seq ,sum(case when p_promo_sk is null then 1 else 0 end) no_promo ,sum(case when p_promo_sk is not null then 1 else 0 end) promo ,count(*) total_cnt from catalog_sales join inventory on (cs_item_sk = inv_item_sk) join warehouse on (w_warehouse_sk=inv_warehouse_sk) join item on (i_item_sk = cs_item_sk) join customer_demographics on (cs_bill_cdemo_sk = cd_demo_sk) join household_demographics on (cs_bill_hdemo_sk = hd_demo_sk) join date_dim d1 on (cs_sold_date_sk = d1.d_date_sk) join date_dim d2 on (inv_date_sk = d2.d_date_sk) join date_dim d3 on (cs_ship_date_sk = d3.d_date_sk) left outer join promotion on (cs_promo_sk=p_promo_sk) left outer join catalog_returns on (cr_item_sk = cs_item_sk and cr_order_number = cs_order_number) where d1.d_week_seq = d2.d_week_seq and inv_quantity_on_hand < cs_quantity and d3.d_date > d1.d_date + interval 5 days and hd_buy_potential = '1001-5000' and d1.d_year = 2000 and cd_marital_status = 'D' group by i_item_desc,w_warehouse_name,d1.d_week_seq order by total_cnt desc, i_item_desc, w_warehouse_name, d_week_seq limit 100""", "q73" -> """ select c_last_name ,c_first_name ,c_salutation ,c_preferred_cust_flag ,ss_ticket_number ,cnt from (select ss_ticket_number ,ss_customer_sk ,count(*) cnt from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and date_dim.d_dom between 1 and 2 and (household_demographics.hd_buy_potential = '>10000' or household_demographics.hd_buy_potential = '5001-10000') and household_demographics.hd_vehicle_count > 0 and case when household_demographics.hd_vehicle_count > 0 then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end > 1 and date_dim.d_year in (2000,2000+1,2000+2) and store.s_county in ('Lea County','Furnas County','Pennington County','Bronx County') group by ss_ticket_number,ss_customer_sk) dj,customer where ss_customer_sk = c_customer_sk and cnt between 1 and 5 order by cnt desc, c_last_name asc""", "q74" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,d_year as year ,sum(ss_net_paid) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (1998,1998+1) group by c_customer_id ,c_first_name ,c_last_name ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,d_year as year ,sum(ws_net_paid) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year in (1998,1998+1) group by c_customer_id ,c_first_name ,c_last_name ,d_year ) select t_s_secyear.customer_id, t_s_secyear.customer_first_name, t_s_secyear.customer_last_name from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.sale_type = 's' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_w_secyear.sale_type = 'w' and t_s_firstyear.year = 1998 and t_s_secyear.year = 1998+1 and t_w_firstyear.year = 1998 and t_w_secyear.year = 1998+1 and t_s_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end order by 3,1,2 limit 100""", "q75" -> """ WITH all_sales AS ( SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,SUM(sales_cnt) AS sales_cnt ,SUM(sales_amt) AS sales_amt FROM (SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,cs_quantity - COALESCE(cr_return_quantity,0) AS sales_cnt ,cs_ext_sales_price - COALESCE(cr_return_amount,0.0) AS sales_amt FROM catalog_sales JOIN item ON i_item_sk=cs_item_sk JOIN date_dim ON d_date_sk=cs_sold_date_sk LEFT JOIN catalog_returns ON (cs_order_number=cr_order_number AND cs_item_sk=cr_item_sk) WHERE i_category='Sports' UNION SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,ss_quantity - COALESCE(sr_return_quantity,0) AS sales_cnt ,ss_ext_sales_price - COALESCE(sr_return_amt,0.0) AS sales_amt FROM store_sales JOIN item ON i_item_sk=ss_item_sk JOIN date_dim ON d_date_sk=ss_sold_date_sk LEFT JOIN store_returns ON (ss_ticket_number=sr_ticket_number AND ss_item_sk=sr_item_sk) WHERE i_category='Sports' UNION SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,ws_quantity - COALESCE(wr_return_quantity,0) AS sales_cnt ,ws_ext_sales_price - COALESCE(wr_return_amt,0.0) AS sales_amt FROM web_sales JOIN item ON i_item_sk=ws_item_sk JOIN date_dim ON d_date_sk=ws_sold_date_sk LEFT JOIN web_returns ON (ws_order_number=wr_order_number AND ws_item_sk=wr_item_sk) WHERE i_category='Sports') sales_detail GROUP BY d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id) SELECT prev_yr.d_year AS prev_year ,curr_yr.d_year AS year ,curr_yr.i_brand_id ,curr_yr.i_class_id ,curr_yr.i_category_id ,curr_yr.i_manufact_id ,prev_yr.sales_cnt AS prev_yr_cnt ,curr_yr.sales_cnt AS curr_yr_cnt ,curr_yr.sales_cnt-prev_yr.sales_cnt AS sales_cnt_diff ,curr_yr.sales_amt-prev_yr.sales_amt AS sales_amt_diff FROM all_sales curr_yr, all_sales prev_yr WHERE curr_yr.i_brand_id=prev_yr.i_brand_id AND curr_yr.i_class_id=prev_yr.i_class_id AND curr_yr.i_category_id=prev_yr.i_category_id AND curr_yr.i_manufact_id=prev_yr.i_manufact_id AND curr_yr.d_year=2001 AND prev_yr.d_year=2001-1 AND CAST(curr_yr.sales_cnt AS DECIMAL(17,2))/CAST(prev_yr.sales_cnt AS DECIMAL(17,2))<0.9 ORDER BY sales_cnt_diff,sales_amt_diff limit 100""", "q76" -> """ select channel, col_name, d_year, d_qoy, i_category, COUNT(*) sales_cnt, SUM(ext_sales_price) sales_amt FROM ( SELECT 'store' as channel, 'ss_customer_sk' col_name, d_year, d_qoy, i_category, ss_ext_sales_price ext_sales_price FROM store_sales, item, date_dim WHERE ss_customer_sk IS NULL AND ss_sold_date_sk=d_date_sk AND ss_item_sk=i_item_sk UNION ALL SELECT 'web' as channel, 'ws_ship_addr_sk' col_name, d_year, d_qoy, i_category, ws_ext_sales_price ext_sales_price FROM web_sales, item, date_dim WHERE ws_ship_addr_sk IS NULL AND ws_sold_date_sk=d_date_sk AND ws_item_sk=i_item_sk UNION ALL SELECT 'catalog' as channel, 'cs_ship_mode_sk' col_name, d_year, d_qoy, i_category, cs_ext_sales_price ext_sales_price FROM catalog_sales, item, date_dim WHERE cs_ship_mode_sk IS NULL AND cs_sold_date_sk=d_date_sk AND cs_item_sk=i_item_sk) foo GROUP BY channel, col_name, d_year, d_qoy, i_category ORDER BY channel, col_name, d_year, d_qoy, i_category limit 100""", "q77" -> """ with ss as (select s_store_sk, sum(ss_ext_sales_price) as sales, sum(ss_net_profit) as profit from store_sales, date_dim, store where ss_sold_date_sk = d_date_sk and d_date between cast('2000-08-16' as date) and (cast('2000-08-16' as date) + INTERVAL 30 days) and ss_store_sk = s_store_sk group by s_store_sk) , sr as (select s_store_sk, sum(sr_return_amt) as returns, sum(sr_net_loss) as profit_loss from store_returns, date_dim, store where sr_returned_date_sk = d_date_sk and d_date between cast('2000-08-16' as date) and (cast('2000-08-16' as date) + INTERVAL 30 days) and sr_store_sk = s_store_sk group by s_store_sk), cs as (select cs_call_center_sk, sum(cs_ext_sales_price) as sales, sum(cs_net_profit) as profit from catalog_sales, date_dim where cs_sold_date_sk = d_date_sk and d_date between cast('2000-08-16' as date) and (cast('2000-08-16' as date) + INTERVAL 30 days) group by cs_call_center_sk ), cr as (select cr_call_center_sk, sum(cr_return_amount) as returns, sum(cr_net_loss) as profit_loss from catalog_returns, date_dim where cr_returned_date_sk = d_date_sk and d_date between cast('2000-08-16' as date) and (cast('2000-08-16' as date) + INTERVAL 30 days) group by cr_call_center_sk ), ws as ( select wp_web_page_sk, sum(ws_ext_sales_price) as sales, sum(ws_net_profit) as profit from web_sales, date_dim, web_page where ws_sold_date_sk = d_date_sk and d_date between cast('2000-08-16' as date) and (cast('2000-08-16' as date) + INTERVAL 30 days) and ws_web_page_sk = wp_web_page_sk group by wp_web_page_sk), wr as (select wp_web_page_sk, sum(wr_return_amt) as returns, sum(wr_net_loss) as profit_loss from web_returns, date_dim, web_page where wr_returned_date_sk = d_date_sk and d_date between cast('2000-08-16' as date) and (cast('2000-08-16' as date) + INTERVAL 30 days) and wr_web_page_sk = wp_web_page_sk group by wp_web_page_sk) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , ss.s_store_sk as id , sales , coalesce(returns, 0) as returns , (profit - coalesce(profit_loss,0)) as profit from ss left join sr on ss.s_store_sk = sr.s_store_sk union all select 'catalog channel' as channel , cs_call_center_sk as id , sales , returns , (profit - profit_loss) as profit from cs , cr union all select 'web channel' as channel , ws.wp_web_page_sk as id , sales , coalesce(returns, 0) returns , (profit - coalesce(profit_loss,0)) as profit from ws left join wr on ws.wp_web_page_sk = wr.wp_web_page_sk ) x group by rollup (channel, id) order by channel ,id limit 100""", "q78" -> """ with ws as (select d_year AS ws_sold_year, ws_item_sk, ws_bill_customer_sk ws_customer_sk, sum(ws_quantity) ws_qty, sum(ws_wholesale_cost) ws_wc, sum(ws_sales_price) ws_sp from web_sales left join web_returns on wr_order_number=ws_order_number and ws_item_sk=wr_item_sk join date_dim on ws_sold_date_sk = d_date_sk where wr_order_number is null group by d_year, ws_item_sk, ws_bill_customer_sk ), cs as (select d_year AS cs_sold_year, cs_item_sk, cs_bill_customer_sk cs_customer_sk, sum(cs_quantity) cs_qty, sum(cs_wholesale_cost) cs_wc, sum(cs_sales_price) cs_sp from catalog_sales left join catalog_returns on cr_order_number=cs_order_number and cs_item_sk=cr_item_sk join date_dim on cs_sold_date_sk = d_date_sk where cr_order_number is null group by d_year, cs_item_sk, cs_bill_customer_sk ), ss as (select d_year AS ss_sold_year, ss_item_sk, ss_customer_sk, sum(ss_quantity) ss_qty, sum(ss_wholesale_cost) ss_wc, sum(ss_sales_price) ss_sp from store_sales left join store_returns on sr_ticket_number=ss_ticket_number and ss_item_sk=sr_item_sk join date_dim on ss_sold_date_sk = d_date_sk where sr_ticket_number is null group by d_year, ss_item_sk, ss_customer_sk ) select ss_customer_sk, round(ss_qty/(coalesce(ws_qty,0)+coalesce(cs_qty,0)),2) ratio, ss_qty store_qty, ss_wc store_wholesale_cost, ss_sp store_sales_price, coalesce(ws_qty,0)+coalesce(cs_qty,0) other_chan_qty, coalesce(ws_wc,0)+coalesce(cs_wc,0) other_chan_wholesale_cost, coalesce(ws_sp,0)+coalesce(cs_sp,0) other_chan_sales_price from ss left join ws on (ws_sold_year=ss_sold_year and ws_item_sk=ss_item_sk and ws_customer_sk=ss_customer_sk) left join cs on (cs_sold_year=ss_sold_year and cs_item_sk=ss_item_sk and cs_customer_sk=ss_customer_sk) where (coalesce(ws_qty,0)>0 or coalesce(cs_qty, 0)>0) and ss_sold_year=2001 order by ss_customer_sk, ss_qty desc, ss_wc desc, ss_sp desc, other_chan_qty, other_chan_wholesale_cost, other_chan_sales_price, ratio limit 100""", "q79" -> """ select c_last_name,c_first_name,substr(s_city,1,30),ss_ticket_number,amt,profit from (select ss_ticket_number ,ss_customer_sk ,store.s_city ,sum(ss_coupon_amt) amt ,sum(ss_net_profit) profit from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and (household_demographics.hd_dep_count = 0 or household_demographics.hd_vehicle_count > 3) and date_dim.d_dow = 1 and date_dim.d_year in (1998,1998+1,1998+2) and store.s_number_employees between 200 and 295 group by ss_ticket_number,ss_customer_sk,ss_addr_sk,store.s_city) ms,customer where ss_customer_sk = c_customer_sk order by c_last_name,c_first_name,substr(s_city,1,30), profit limit 100""", "q80" -> """ with ssr as (select s_store_id as store_id, sum(ss_ext_sales_price) as sales, sum(coalesce(sr_return_amt, 0)) as returns, sum(ss_net_profit - coalesce(sr_net_loss, 0)) as profit from store_sales left outer join store_returns on (ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number), date_dim, store, item, promotion where ss_sold_date_sk = d_date_sk and d_date between cast('2002-08-06' as date) and (cast('2002-08-06' as date) + INTERVAL 60 days) and ss_store_sk = s_store_sk and ss_item_sk = i_item_sk and i_current_price > 50 and ss_promo_sk = p_promo_sk and p_channel_tv = 'N' group by s_store_id) , csr as (select cp_catalog_page_id as catalog_page_id, sum(cs_ext_sales_price) as sales, sum(coalesce(cr_return_amount, 0)) as returns, sum(cs_net_profit - coalesce(cr_net_loss, 0)) as profit from catalog_sales left outer join catalog_returns on (cs_item_sk = cr_item_sk and cs_order_number = cr_order_number), date_dim, catalog_page, item, promotion where cs_sold_date_sk = d_date_sk and d_date between cast('2002-08-06' as date) and (cast('2002-08-06' as date) + INTERVAL 60 days) and cs_catalog_page_sk = cp_catalog_page_sk and cs_item_sk = i_item_sk and i_current_price > 50 and cs_promo_sk = p_promo_sk and p_channel_tv = 'N' group by cp_catalog_page_id) , wsr as (select web_site_id, sum(ws_ext_sales_price) as sales, sum(coalesce(wr_return_amt, 0)) as returns, sum(ws_net_profit - coalesce(wr_net_loss, 0)) as profit from web_sales left outer join web_returns on (ws_item_sk = wr_item_sk and ws_order_number = wr_order_number), date_dim, web_site, item, promotion where ws_sold_date_sk = d_date_sk and d_date between cast('2002-08-06' as date) and (cast('2002-08-06' as date) + INTERVAL 60 days) and ws_web_site_sk = web_site_sk and ws_item_sk = i_item_sk and i_current_price > 50 and ws_promo_sk = p_promo_sk and p_channel_tv = 'N' group by web_site_id) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , 'store' || store_id as id , sales , returns , profit from ssr union all select 'catalog channel' as channel , 'catalog_page' || catalog_page_id as id , sales , returns , profit from csr union all select 'web channel' as channel , 'web_site' || web_site_id as id , sales , returns , profit from wsr ) x group by rollup (channel, id) order by channel ,id limit 100""", "q81" -> """ with customer_total_return as (select cr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, sum(cr_return_amt_inc_tax) as ctr_total_return from catalog_returns ,date_dim ,customer_address where cr_returned_date_sk = d_date_sk and d_year =1998 and cr_returning_addr_sk = ca_address_sk group by cr_returning_customer_sk ,ca_state ) select c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset ,ca_location_type,ctr_total_return from customer_total_return ctr1 ,customer_address ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_state = ctr2.ctr_state) and ca_address_sk = c_current_addr_sk and ca_state = 'TX' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset ,ca_location_type,ctr_total_return limit 100""", "q82" -> """ select i_item_id ,i_item_desc ,i_current_price from item, inventory, date_dim, store_sales where i_current_price between 49 and 49+30 and inv_item_sk = i_item_sk and d_date_sk=inv_date_sk and d_date between cast('2001-01-28' as date) and (cast('2001-01-28' as date) + INTERVAL 60 days) and i_manufact_id in (80,675,292,17) and inv_quantity_on_hand between 100 and 500 and ss_item_sk = i_item_sk group by i_item_id,i_item_desc,i_current_price order by i_item_id limit 100""", "q83" -> """ with sr_items as (select i_item_id item_id, sum(sr_return_quantity) sr_item_qty from store_returns, item, date_dim where sr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('2000-06-17','2000-08-22','2000-11-17'))) and sr_returned_date_sk = d_date_sk group by i_item_id), cr_items as (select i_item_id item_id, sum(cr_return_quantity) cr_item_qty from catalog_returns, item, date_dim where cr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('2000-06-17','2000-08-22','2000-11-17'))) and cr_returned_date_sk = d_date_sk group by i_item_id), wr_items as (select i_item_id item_id, sum(wr_return_quantity) wr_item_qty from web_returns, item, date_dim where wr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('2000-06-17','2000-08-22','2000-11-17'))) and wr_returned_date_sk = d_date_sk group by i_item_id) select sr_items.item_id ,sr_item_qty ,sr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 sr_dev ,cr_item_qty ,cr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 cr_dev ,wr_item_qty ,wr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 wr_dev ,(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 average from sr_items ,cr_items ,wr_items where sr_items.item_id=cr_items.item_id and sr_items.item_id=wr_items.item_id order by sr_items.item_id ,sr_item_qty limit 100""", "q84" -> """ select c_customer_id as customer_id , coalesce(c_last_name,'') || ', ' || coalesce(c_first_name,'') as customername from customer ,customer_address ,customer_demographics ,household_demographics ,income_band ,store_returns where ca_city = 'Hopewell' and c_current_addr_sk = ca_address_sk and ib_lower_bound >= 37855 and ib_upper_bound <= 37855 + 50000 and ib_income_band_sk = hd_income_band_sk and cd_demo_sk = c_current_cdemo_sk and hd_demo_sk = c_current_hdemo_sk and sr_cdemo_sk = cd_demo_sk order by c_customer_id limit 100""", "q85" -> """ select substr(r_reason_desc,1,20) ,avg(ws_quantity) ,avg(wr_refunded_cash) ,avg(wr_fee) from web_sales, web_returns, web_page, customer_demographics cd1, customer_demographics cd2, customer_address, date_dim, reason where ws_web_page_sk = wp_web_page_sk and ws_item_sk = wr_item_sk and ws_order_number = wr_order_number and ws_sold_date_sk = d_date_sk and d_year = 2001 and cd1.cd_demo_sk = wr_refunded_cdemo_sk and cd2.cd_demo_sk = wr_returning_cdemo_sk and ca_address_sk = wr_refunded_addr_sk and r_reason_sk = wr_reason_sk and ( ( cd1.cd_marital_status = 'M' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = '4 yr Degree' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 100.00 and 150.00 ) or ( cd1.cd_marital_status = 'S' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = 'College' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 50.00 and 100.00 ) or ( cd1.cd_marital_status = 'D' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = 'Secondary' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 150.00 and 200.00 ) ) and ( ( ca_country = 'United States' and ca_state in ('TX', 'VA', 'CA') and ws_net_profit between 100 and 200 ) or ( ca_country = 'United States' and ca_state in ('AR', 'NE', 'MO') and ws_net_profit between 150 and 300 ) or ( ca_country = 'United States' and ca_state in ('IA', 'MS', 'WA') and ws_net_profit between 50 and 250 ) ) group by r_reason_desc order by substr(r_reason_desc,1,20) ,avg(ws_quantity) ,avg(wr_refunded_cash) ,avg(wr_fee) limit 100""", "q86" -> """ select sum(ws_net_paid) as total_sum ,i_category ,i_class ,grouping(i_category)+grouping(i_class) as lochierarchy ,rank() over ( partition by grouping(i_category)+grouping(i_class), case when grouping(i_class) = 0 then i_category end order by sum(ws_net_paid) desc) as rank_within_parent from web_sales ,date_dim d1 ,item where d1.d_month_seq between 1215 and 1215+11 and d1.d_date_sk = ws_sold_date_sk and i_item_sk = ws_item_sk group by rollup(i_category,i_class) order by lochierarchy desc, case when lochierarchy = 0 then i_category end, rank_within_parent limit 100""", "q87" -> """ select count(*) from ((select distinct c_last_name, c_first_name, d_date from store_sales, date_dim, customer where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_customer_sk = customer.c_customer_sk and d_month_seq between 1221 and 1221+11) except (select distinct c_last_name, c_first_name, d_date from catalog_sales, date_dim, customer where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1221 and 1221+11) except (select distinct c_last_name, c_first_name, d_date from web_sales, date_dim, customer where web_sales.ws_sold_date_sk = date_dim.d_date_sk and web_sales.ws_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1221 and 1221+11) ) cool_cust""", "q88" -> """ select * from (select count(*) h8_30_to_9 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 8 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s1, (select count(*) h9_to_9_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 9 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s2, (select count(*) h9_30_to_10 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 9 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s3, (select count(*) h10_to_10_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 10 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s4, (select count(*) h10_30_to_11 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 10 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s5, (select count(*) h11_to_11_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 11 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s6, (select count(*) h11_30_to_12 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 11 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s7, (select count(*) h12_to_12_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 12 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s8""", "q89" -> """ select * from( select i_category, i_class, i_brand, s_store_name, s_company_name, d_moy, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_category, i_brand, s_store_name, s_company_name) avg_monthly_sales from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_year in (2000) and ((i_category in ('Home','Music','Books') and i_class in ('glassware','classical','fiction') ) or (i_category in ('Jewelry','Sports','Women') and i_class in ('semi-precious','baseball','dresses') )) group by i_category, i_class, i_brand, s_store_name, s_company_name, d_moy) tmp1 where case when (avg_monthly_sales <> 0) then (abs(sum_sales - avg_monthly_sales) / avg_monthly_sales) else null end > 0.1 order by sum_sales - avg_monthly_sales, s_store_name limit 100""", "q90" -> """ select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio from ( select count(*) amc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 9 and 9+1 and household_demographics.hd_dep_count = 3 and web_page.wp_char_count between 5000 and 5200) at, ( select count(*) pmc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 16 and 16+1 and household_demographics.hd_dep_count = 3 and web_page.wp_char_count between 5000 and 5200) pt order by am_pm_ratio limit 100""", "q91" -> """ select cc_call_center_id Call_Center, cc_name Call_Center_Name, cc_manager Manager, sum(cr_net_loss) Returns_Loss from call_center, catalog_returns, date_dim, customer, customer_address, customer_demographics, household_demographics where cr_call_center_sk = cc_call_center_sk and cr_returned_date_sk = d_date_sk and cr_returning_customer_sk= c_customer_sk and cd_demo_sk = c_current_cdemo_sk and hd_demo_sk = c_current_hdemo_sk and ca_address_sk = c_current_addr_sk and d_year = 2000 and d_moy = 12 and ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') or(cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) and hd_buy_potential like 'Unknown%' and ca_gmt_offset = -7 group by cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status order by sum(cr_net_loss) desc""", "q92" -> """ select sum(ws_ext_discount_amt) as `Excess Discount Amount` from web_sales ,item ,date_dim where i_manufact_id = 356 and i_item_sk = ws_item_sk and d_date between '2001-03-12' and (cast('2001-03-12' as date) + INTERVAL 90 days) and d_date_sk = ws_sold_date_sk and ws_ext_discount_amt > ( SELECT 1.3 * avg(ws_ext_discount_amt) FROM web_sales ,date_dim WHERE ws_item_sk = i_item_sk and d_date between '2001-03-12' and (cast('2001-03-12' as date) + INTERVAL 90 days) and d_date_sk = ws_sold_date_sk ) order by sum(ws_ext_discount_amt) limit 100""", "q93" -> """ select ss_customer_sk ,sum(act_sales) sumsales from (select ss_item_sk ,ss_ticket_number ,ss_customer_sk ,case when sr_return_quantity is not null then (ss_quantity-sr_return_quantity)*ss_sales_price else (ss_quantity*ss_sales_price) end act_sales from store_sales left outer join store_returns on (sr_item_sk = ss_item_sk and sr_ticket_number = ss_ticket_number) ,reason where sr_reason_sk = r_reason_sk and r_reason_desc = 'reason 66') t group by ss_customer_sk order by sumsales, ss_customer_sk limit 100""", "q94" -> """ select count(distinct ws_order_number) as `order count` ,sum(ws_ext_ship_cost) as `total shipping cost` ,sum(ws_net_profit) as `total net profit` from web_sales ws1 ,date_dim ,customer_address ,web_site where d_date between '1999-4-01' and (cast('1999-4-01' as date) + INTERVAL 60 days) and ws1.ws_ship_date_sk = d_date_sk and ws1.ws_ship_addr_sk = ca_address_sk and ca_state = 'NE' and ws1.ws_web_site_sk = web_site_sk and web_company_name = 'pri' and exists (select * from web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) and not exists(select * from web_returns wr1 where ws1.ws_order_number = wr1.wr_order_number) order by count(distinct ws_order_number) limit 100""", "q95" -> """ with ws_wh as (select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 from web_sales ws1,web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) select count(distinct ws_order_number) as `order count` ,sum(ws_ext_ship_cost) as `total shipping cost` ,sum(ws_net_profit) as `total net profit` from web_sales ws1 ,date_dim ,customer_address ,web_site where d_date between '2002-4-01' and (cast('2002-4-01' as date) + INTERVAL 60 days) and ws1.ws_ship_date_sk = d_date_sk and ws1.ws_ship_addr_sk = ca_address_sk and ca_state = 'AL' and ws1.ws_web_site_sk = web_site_sk and web_company_name = 'pri' and ws1.ws_order_number in (select ws_order_number from ws_wh) and ws1.ws_order_number in (select wr_order_number from web_returns,ws_wh where wr_order_number = ws_wh.ws_order_number) order by count(distinct ws_order_number) limit 100""", "q96" -> """ select count(*) from store_sales ,household_demographics ,time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 16 and time_dim.t_minute >= 30 and household_demographics.hd_dep_count = 6 and store.s_store_name = 'ese' order by count(*) limit 100""", "q97" -> """ with ssci as ( select ss_customer_sk customer_sk ,ss_item_sk item_sk from store_sales,date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1190 and 1190 + 11 group by ss_customer_sk ,ss_item_sk), csci as( select cs_bill_customer_sk customer_sk ,cs_item_sk item_sk from catalog_sales,date_dim where cs_sold_date_sk = d_date_sk and d_month_seq between 1190 and 1190 + 11 group by cs_bill_customer_sk ,cs_item_sk) select sum(case when ssci.customer_sk is not null and csci.customer_sk is null then 1 else 0 end) store_only ,sum(case when ssci.customer_sk is null and csci.customer_sk is not null then 1 else 0 end) catalog_only ,sum(case when ssci.customer_sk is not null and csci.customer_sk is not null then 1 else 0 end) store_and_catalog from ssci full outer join csci on (ssci.customer_sk=csci.customer_sk and ssci.item_sk = csci.item_sk) limit 100""", "q98" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(ss_ext_sales_price) as itemrevenue ,sum(ss_ext_sales_price)*100/sum(sum(ss_ext_sales_price)) over (partition by i_class) as revenueratio from store_sales ,item ,date_dim where ss_item_sk = i_item_sk and i_category in ('Home', 'Sports', 'Men') and ss_sold_date_sk = d_date_sk and d_date between cast('2002-01-05' as date) and (cast('2002-01-05' as date) + interval 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio""", "q99" -> """ select substr(w_warehouse_name,1,20) ,sm_type ,cc_name ,sum(case when (cs_ship_date_sk - cs_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 30) and (cs_ship_date_sk - cs_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 60) and (cs_ship_date_sk - cs_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 90) and (cs_ship_date_sk - cs_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from catalog_sales ,warehouse ,ship_mode ,call_center ,date_dim where d_month_seq between 1178 and 1178 + 11 and cs_ship_date_sk = d_date_sk and cs_warehouse_sk = w_warehouse_sk and cs_ship_mode_sk = sm_ship_mode_sk and cs_call_center_sk = cc_call_center_sk group by substr(w_warehouse_name,1,20) ,sm_type ,cc_name order by substr(w_warehouse_name,1,20) ,sm_type ,cc_name limit 100""", "q1" -> """ with customer_total_return as (select sr_customer_sk as ctr_customer_sk ,sr_store_sk as ctr_store_sk ,sum(SR_FEE) as ctr_total_return from store_returns ,date_dim where sr_returned_date_sk = d_date_sk and d_year =2000 group by sr_customer_sk ,sr_store_sk) select c_customer_id from customer_total_return ctr1 ,store ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_store_sk = ctr2.ctr_store_sk) and s_store_sk = ctr1.ctr_store_sk and s_state = 'NY' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id limit 100""", "q2" -> """ with wscs as (select sold_date_sk ,sales_price from (select ws_sold_date_sk sold_date_sk ,ws_ext_sales_price sales_price from web_sales union all select cs_sold_date_sk sold_date_sk ,cs_ext_sales_price sales_price from catalog_sales)), wswscs as (select d_week_seq, sum(case when (d_day_name='Sunday') then sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then sales_price else null end) sat_sales from wscs ,date_dim where d_date_sk = sold_date_sk group by d_week_seq) select d_week_seq1 ,round(sun_sales1/sun_sales2,2) ,round(mon_sales1/mon_sales2,2) ,round(tue_sales1/tue_sales2,2) ,round(wed_sales1/wed_sales2,2) ,round(thu_sales1/thu_sales2,2) ,round(fri_sales1/fri_sales2,2) ,round(sat_sales1/sat_sales2,2) from (select wswscs.d_week_seq d_week_seq1 ,sun_sales sun_sales1 ,mon_sales mon_sales1 ,tue_sales tue_sales1 ,wed_sales wed_sales1 ,thu_sales thu_sales1 ,fri_sales fri_sales1 ,sat_sales sat_sales1 from wswscs,date_dim where date_dim.d_week_seq = wswscs.d_week_seq and d_year = 1998) y, (select wswscs.d_week_seq d_week_seq2 ,sun_sales sun_sales2 ,mon_sales mon_sales2 ,tue_sales tue_sales2 ,wed_sales wed_sales2 ,thu_sales thu_sales2 ,fri_sales fri_sales2 ,sat_sales sat_sales2 from wswscs ,date_dim where date_dim.d_week_seq = wswscs.d_week_seq and d_year = 1998+1) z where d_week_seq1=d_week_seq2-53 order by d_week_seq1""", "q3" -> """ select dt.d_year ,item.i_brand_id brand_id ,item.i_brand brand ,sum(ss_sales_price) sum_agg from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manufact_id = 816 and dt.d_moy=11 group by dt.d_year ,item.i_brand ,item.i_brand_id order by dt.d_year ,sum_agg desc ,brand_id limit 100""", "q4" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum((((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2) ) year_total ,'c' sale_type from customer ,catalog_sales ,date_dim where c_customer_sk = cs_bill_customer_sk and cs_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum((((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2) ) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year ) select t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_birth_country from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_c_firstyear ,year_total t_c_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_c_secyear.customer_id and t_s_firstyear.customer_id = t_c_firstyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.sale_type = 's' and t_c_firstyear.sale_type = 'c' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_c_secyear.sale_type = 'c' and t_w_secyear.sale_type = 'w' and t_s_firstyear.dyear = 1999 and t_s_secyear.dyear = 1999+1 and t_c_firstyear.dyear = 1999 and t_c_secyear.dyear = 1999+1 and t_w_firstyear.dyear = 1999 and t_w_secyear.dyear = 1999+1 and t_s_firstyear.year_total > 0 and t_c_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end > case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end order by t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_birth_country limit 100""", "q5" -> """ with ssr as (select s_store_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select ss_store_sk as store_sk, ss_sold_date_sk as date_sk, ss_ext_sales_price as sales_price, ss_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from store_sales union all select sr_store_sk as store_sk, sr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, sr_return_amt as return_amt, sr_net_loss as net_loss from store_returns ) salesreturns, date_dim, store where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and store_sk = s_store_sk group by s_store_id) , csr as (select cp_catalog_page_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select cs_catalog_page_sk as page_sk, cs_sold_date_sk as date_sk, cs_ext_sales_price as sales_price, cs_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from catalog_sales union all select cr_catalog_page_sk as page_sk, cr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, cr_return_amount as return_amt, cr_net_loss as net_loss from catalog_returns ) salesreturns, date_dim, catalog_page where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and page_sk = cp_catalog_page_sk group by cp_catalog_page_id) , wsr as (select web_site_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select ws_web_site_sk as wsr_web_site_sk, ws_sold_date_sk as date_sk, ws_ext_sales_price as sales_price, ws_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from web_sales union all select ws_web_site_sk as wsr_web_site_sk, wr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, wr_return_amt as return_amt, wr_net_loss as net_loss from web_returns left outer join web_sales on ( wr_item_sk = ws_item_sk and wr_order_number = ws_order_number) ) salesreturns, date_dim, web_site where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and wsr_web_site_sk = web_site_sk group by web_site_id) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , 'store' || s_store_id as id , sales , returns , (profit - profit_loss) as profit from ssr union all select 'catalog channel' as channel , 'catalog_page' || cp_catalog_page_id as id , sales , returns , (profit - profit_loss) as profit from csr union all select 'web channel' as channel , 'web_site' || web_site_id as id , sales , returns , (profit - profit_loss) as profit from wsr ) x group by rollup (channel, id) order by channel ,id limit 100""", "q6" -> """ select a.ca_state state, count(*) cnt from customer_address a ,customer c ,store_sales s ,date_dim d ,item i where a.ca_address_sk = c.c_current_addr_sk and c.c_customer_sk = s.ss_customer_sk and s.ss_sold_date_sk = d.d_date_sk and s.ss_item_sk = i.i_item_sk and d.d_month_seq = (select distinct (d_month_seq) from date_dim where d_year = 2002 and d_moy = 3 ) and i.i_current_price > 1.2 * (select avg(j.i_current_price) from item j where j.i_category = i.i_category) group by a.ca_state having count(*) >= 10 order by cnt, a.ca_state limit 100""", "q7" -> """ select i_item_id, avg(ss_quantity) agg1, avg(ss_list_price) agg2, avg(ss_coupon_amt) agg3, avg(ss_sales_price) agg4 from store_sales, customer_demographics, date_dim, item, promotion where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and ss_cdemo_sk = cd_demo_sk and ss_promo_sk = p_promo_sk and cd_gender = 'F' and cd_marital_status = 'W' and cd_education_status = 'College' and (p_channel_email = 'N' or p_channel_event = 'N') and d_year = 2001 group by i_item_id order by i_item_id limit 100""", "q8" -> """ select s_store_name ,sum(ss_net_profit) from store_sales ,date_dim ,store, (select ca_zip from ( SELECT substr(ca_zip,1,5) ca_zip FROM customer_address WHERE substr(ca_zip,1,5) IN ( '47602','16704','35863','28577','83910','36201', '58412','48162','28055','41419','80332', '38607','77817','24891','16226','18410', '21231','59345','13918','51089','20317', '17167','54585','67881','78366','47770', '18360','51717','73108','14440','21800', '89338','45859','65501','34948','25973', '73219','25333','17291','10374','18829', '60736','82620','41351','52094','19326', '25214','54207','40936','21814','79077', '25178','75742','77454','30621','89193', '27369','41232','48567','83041','71948', '37119','68341','14073','16891','62878', '49130','19833','24286','27700','40979', '50412','81504','94835','84844','71954', '39503','57649','18434','24987','12350', '86379','27413','44529','98569','16515', '27287','24255','21094','16005','56436', '91110','68293','56455','54558','10298', '83647','32754','27052','51766','19444', '13869','45645','94791','57631','20712', '37788','41807','46507','21727','71836', '81070','50632','88086','63991','20244', '31655','51782','29818','63792','68605', '94898','36430','57025','20601','82080', '33869','22728','35834','29086','92645', '98584','98072','11652','78093','57553', '43830','71144','53565','18700','90209', '71256','38353','54364','28571','96560', '57839','56355','50679','45266','84680', '34306','34972','48530','30106','15371', '92380','84247','92292','68852','13338', '34594','82602','70073','98069','85066', '47289','11686','98862','26217','47529', '63294','51793','35926','24227','14196', '24594','32489','99060','49472','43432', '49211','14312','88137','47369','56877', '20534','81755','15794','12318','21060', '73134','41255','63073','81003','73873', '66057','51184','51195','45676','92696', '70450','90669','98338','25264','38919', '59226','58581','60298','17895','19489', '52301','80846','95464','68770','51634', '19988','18367','18421','11618','67975', '25494','41352','95430','15734','62585', '97173','33773','10425','75675','53535', '17879','41967','12197','67998','79658', '59130','72592','14851','43933','68101', '50636','25717','71286','24660','58058', '72991','95042','15543','33122','69280', '11912','59386','27642','65177','17672', '33467','64592','36335','54010','18767', '63193','42361','49254','33113','33159', '36479','59080','11855','81963','31016', '49140','29392','41836','32958','53163', '13844','73146','23952','65148','93498', '14530','46131','58454','13376','13378', '83986','12320','17193','59852','46081', '98533','52389','13086','68843','31013', '13261','60560','13443','45533','83583', '11489','58218','19753','22911','25115', '86709','27156','32669','13123','51933', '39214','41331','66943','14155','69998', '49101','70070','35076','14242','73021', '59494','15782','29752','37914','74686', '83086','34473','15751','81084','49230', '91894','60624','17819','28810','63180', '56224','39459','55233','75752','43639', '55349','86057','62361','50788','31830', '58062','18218','85761','60083','45484', '21204','90229','70041','41162','35390', '16364','39500','68908','26689','52868', '81335','40146','11340','61527','61794', '71997','30415','59004','29450','58117', '69952','33562','83833','27385','61860', '96435','48333','23065','32961','84919', '61997','99132','22815','56600','68730', '48017','95694','32919','88217','27116', '28239','58032','18884','16791','21343', '97462','18569','75660','15475') intersect select ca_zip from (SELECT substr(ca_zip,1,5) ca_zip,count(*) cnt FROM customer_address, customer WHERE ca_address_sk = c_current_addr_sk and c_preferred_cust_flag='Y' group by ca_zip having count(*) > 10)A1)A2) V1 where ss_store_sk = s_store_sk and ss_sold_date_sk = d_date_sk and d_qoy = 2 and d_year = 1998 and (substr(s_zip,1,2) = substr(V1.ca_zip,1,2)) group by s_store_name order by s_store_name limit 100""", "q9" -> """ select case when (select count(*) from store_sales where ss_quantity between 1 and 20) > 578972190 then (select avg(ss_ext_list_price) from store_sales where ss_quantity between 1 and 20) else (select avg(ss_net_paid_inc_tax) from store_sales where ss_quantity between 1 and 20) end bucket1 , case when (select count(*) from store_sales where ss_quantity between 21 and 40) > 536856786 then (select avg(ss_ext_list_price) from store_sales where ss_quantity between 21 and 40) else (select avg(ss_net_paid_inc_tax) from store_sales where ss_quantity between 21 and 40) end bucket2, case when (select count(*) from store_sales where ss_quantity between 41 and 60) > 12733327 then (select avg(ss_ext_list_price) from store_sales where ss_quantity between 41 and 60) else (select avg(ss_net_paid_inc_tax) from store_sales where ss_quantity between 41 and 60) end bucket3, case when (select count(*) from store_sales where ss_quantity between 61 and 80) > 205136171 then (select avg(ss_ext_list_price) from store_sales where ss_quantity between 61 and 80) else (select avg(ss_net_paid_inc_tax) from store_sales where ss_quantity between 61 and 80) end bucket4, case when (select count(*) from store_sales where ss_quantity between 81 and 100) > 1192341092 then (select avg(ss_ext_list_price) from store_sales where ss_quantity between 81 and 100) else (select avg(ss_net_paid_inc_tax) from store_sales where ss_quantity between 81 and 100) end bucket5 from reason where r_reason_sk = 1""", "q10" -> """ select cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3, cd_dep_count, count(*) cnt4, cd_dep_employed_count, count(*) cnt5, cd_dep_college_count, count(*) cnt6 from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and ca_county in ('Baltimore city','Stafford County','Greene County','Ballard County','Franklin County') and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2000 and d_moy between 1 and 1+3) and (exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2000 and d_moy between 1 ANd 1+3) or exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2000 and d_moy between 1 and 1+3)) group by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count order by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count limit 100""", "q11" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(ss_ext_list_price-ss_ext_discount_amt) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(ws_ext_list_price-ws_ext_discount_amt) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year ) select t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_preferred_cust_flag from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.sale_type = 's' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_w_secyear.sale_type = 'w' and t_s_firstyear.dyear = 2001 and t_s_secyear.dyear = 2001+1 and t_w_firstyear.dyear = 2001 and t_w_secyear.dyear = 2001+1 and t_s_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else 0.0 end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else 0.0 end order by t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_preferred_cust_flag limit 100""", "q12" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(ws_ext_sales_price) as itemrevenue ,sum(ws_ext_sales_price)*100/sum(sum(ws_ext_sales_price)) over (partition by i_class) as revenueratio from web_sales ,item ,date_dim where ws_item_sk = i_item_sk and i_category in ('Children', 'Shoes', 'Women') and ws_sold_date_sk = d_date_sk and d_date between cast('1998-06-19' as date) and (cast('1998-06-19' as date) + INTERVAL 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio limit 100""", "q13" -> """ select avg(ss_quantity) ,avg(ss_ext_sales_price) ,avg(ss_ext_wholesale_cost) ,sum(ss_ext_wholesale_cost) from store_sales ,store ,customer_demographics ,household_demographics ,customer_address ,date_dim where s_store_sk = ss_store_sk and ss_sold_date_sk = d_date_sk and d_year = 2001 and((ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'D' and cd_education_status = '2 yr Degree' and ss_sales_price between 100.00 and 150.00 and hd_dep_count = 3 )or (ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'U' and cd_education_status = 'College' and ss_sales_price between 50.00 and 100.00 and hd_dep_count = 1 ) or (ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'S' and cd_education_status = 'Primary' and ss_sales_price between 150.00 and 200.00 and hd_dep_count = 1 )) and((ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('GA', 'IN', 'NY') and ss_net_profit between 100 and 200 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('ND', 'WV', 'TX') and ss_net_profit between 150 and 300 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('KS', 'NC', 'NM') and ss_net_profit between 50 and 250 ))""", "q14a" -> """ with cross_items as (select i_item_sk ss_item_sk from item, (select iss.i_brand_id brand_id ,iss.i_class_id class_id ,iss.i_category_id category_id from store_sales ,item iss ,date_dim d1 where ss_item_sk = iss.i_item_sk and ss_sold_date_sk = d1.d_date_sk and d1.d_year between 1998 AND 1998 + 2 intersect select ics.i_brand_id ,ics.i_class_id ,ics.i_category_id from catalog_sales ,item ics ,date_dim d2 where cs_item_sk = ics.i_item_sk and cs_sold_date_sk = d2.d_date_sk and d2.d_year between 1998 AND 1998 + 2 intersect select iws.i_brand_id ,iws.i_class_id ,iws.i_category_id from web_sales ,item iws ,date_dim d3 where ws_item_sk = iws.i_item_sk and ws_sold_date_sk = d3.d_date_sk and d3.d_year between 1998 AND 1998 + 2) where i_brand_id = brand_id and i_class_id = class_id and i_category_id = category_id ), avg_sales as (select avg(quantity*list_price) average_sales from (select ss_quantity quantity ,ss_list_price list_price from store_sales ,date_dim where ss_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2 union all select cs_quantity quantity ,cs_list_price list_price from catalog_sales ,date_dim where cs_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2 union all select ws_quantity quantity ,ws_list_price list_price from web_sales ,date_dim where ws_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2) x) select channel, i_brand_id,i_class_id,i_category_id,sum(sales), sum(number_sales) from( select 'store' channel, i_brand_id,i_class_id ,i_category_id,sum(ss_quantity*ss_list_price) sales , count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1998+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales) union all select 'catalog' channel, i_brand_id,i_class_id,i_category_id, sum(cs_quantity*cs_list_price) sales, count(*) number_sales from catalog_sales ,item ,date_dim where cs_item_sk in (select ss_item_sk from cross_items) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1998+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(cs_quantity*cs_list_price) > (select average_sales from avg_sales) union all select 'web' channel, i_brand_id,i_class_id,i_category_id, sum(ws_quantity*ws_list_price) sales , count(*) number_sales from web_sales ,item ,date_dim where ws_item_sk in (select ss_item_sk from cross_items) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1998+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(ws_quantity*ws_list_price) > (select average_sales from avg_sales) ) y group by rollup (channel, i_brand_id,i_class_id,i_category_id) order by channel,i_brand_id,i_class_id,i_category_id limit 100""", "q14b" -> """ with cross_items as (select i_item_sk ss_item_sk from item, (select iss.i_brand_id brand_id ,iss.i_class_id class_id ,iss.i_category_id category_id from store_sales ,item iss ,date_dim d1 where ss_item_sk = iss.i_item_sk and ss_sold_date_sk = d1.d_date_sk and d1.d_year between 1998 AND 1998 + 2 intersect select ics.i_brand_id ,ics.i_class_id ,ics.i_category_id from catalog_sales ,item ics ,date_dim d2 where cs_item_sk = ics.i_item_sk and cs_sold_date_sk = d2.d_date_sk and d2.d_year between 1998 AND 1998 + 2 intersect select iws.i_brand_id ,iws.i_class_id ,iws.i_category_id from web_sales ,item iws ,date_dim d3 where ws_item_sk = iws.i_item_sk and ws_sold_date_sk = d3.d_date_sk and d3.d_year between 1998 AND 1998 + 2) x where i_brand_id = brand_id and i_class_id = class_id and i_category_id = category_id ), avg_sales as (select avg(quantity*list_price) average_sales from (select ss_quantity quantity ,ss_list_price list_price from store_sales ,date_dim where ss_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2 union all select cs_quantity quantity ,cs_list_price list_price from catalog_sales ,date_dim where cs_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2 union all select ws_quantity quantity ,ws_list_price list_price from web_sales ,date_dim where ws_sold_date_sk = d_date_sk and d_year between 1998 and 1998 + 2) x) select this_year.channel ty_channel ,this_year.i_brand_id ty_brand ,this_year.i_class_id ty_class ,this_year.i_category_id ty_category ,this_year.sales ty_sales ,this_year.number_sales ty_number_sales ,last_year.channel ly_channel ,last_year.i_brand_id ly_brand ,last_year.i_class_id ly_class ,last_year.i_category_id ly_category ,last_year.sales ly_sales ,last_year.number_sales ly_number_sales from (select 'store' channel, i_brand_id,i_class_id,i_category_id ,sum(ss_quantity*ss_list_price) sales, count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_week_seq = (select d_week_seq from date_dim where d_year = 1998 + 1 and d_moy = 12 and d_dom = 17) group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) this_year, (select 'store' channel, i_brand_id,i_class_id ,i_category_id, sum(ss_quantity*ss_list_price) sales, count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_week_seq = (select d_week_seq from date_dim where d_year = 1998 and d_moy = 12 and d_dom = 17) group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) last_year where this_year.i_brand_id= last_year.i_brand_id and this_year.i_class_id = last_year.i_class_id and this_year.i_category_id = last_year.i_category_id order by this_year.channel, this_year.i_brand_id, this_year.i_class_id, this_year.i_category_id limit 100""", "q15" -> """ select ca_zip ,sum(cs_sales_price) from catalog_sales ,customer ,customer_address ,date_dim where cs_bill_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') or ca_state in ('CA','WA','GA') or cs_sales_price > 500) and cs_sold_date_sk = d_date_sk and d_qoy = 1 and d_year = 2002 group by ca_zip order by ca_zip limit 100""", "q16" -> """ select count(distinct cs_order_number) as `order count` ,sum(cs_ext_ship_cost) as `total shipping cost` ,sum(cs_net_profit) as `total net profit` from catalog_sales cs1 ,date_dim ,customer_address ,call_center where d_date between '2001-3-01' and (cast('2001-3-01' as date) + INTERVAL 60 days) and cs1.cs_ship_date_sk = d_date_sk and cs1.cs_ship_addr_sk = ca_address_sk and ca_state = 'PA' and cs1.cs_call_center_sk = cc_call_center_sk and cc_county in ('Luce County','Franklin Parish','Sierra County','Williamson County', 'Kittitas County' ) and exists (select * from catalog_sales cs2 where cs1.cs_order_number = cs2.cs_order_number and cs1.cs_warehouse_sk <> cs2.cs_warehouse_sk) and not exists(select * from catalog_returns cr1 where cs1.cs_order_number = cr1.cr_order_number) order by count(distinct cs_order_number) limit 100""", "q17" -> """ select i_item_id ,i_item_desc ,s_state ,count(ss_quantity) as store_sales_quantitycount ,avg(ss_quantity) as store_sales_quantityave ,stddev_samp(ss_quantity) as store_sales_quantitystdev ,stddev_samp(ss_quantity)/avg(ss_quantity) as store_sales_quantitycov ,count(sr_return_quantity) as store_returns_quantitycount ,avg(sr_return_quantity) as store_returns_quantityave ,stddev_samp(sr_return_quantity) as store_returns_quantitystdev ,stddev_samp(sr_return_quantity)/avg(sr_return_quantity) as store_returns_quantitycov ,count(cs_quantity) as catalog_sales_quantitycount ,avg(cs_quantity) as catalog_sales_quantityave ,stddev_samp(cs_quantity) as catalog_sales_quantitystdev ,stddev_samp(cs_quantity)/avg(cs_quantity) as catalog_sales_quantitycov from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_quarter_name = '2001Q1' and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_quarter_name in ('2001Q1','2001Q2','2001Q3') and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_quarter_name in ('2001Q1','2001Q2','2001Q3') group by i_item_id ,i_item_desc ,s_state order by i_item_id ,i_item_desc ,s_state limit 100""", "q18" -> """ select i_item_id, ca_country, ca_state, ca_county, avg( cast(cs_quantity as decimal(12,2))) agg1, avg( cast(cs_list_price as decimal(12,2))) agg2, avg( cast(cs_coupon_amt as decimal(12,2))) agg3, avg( cast(cs_sales_price as decimal(12,2))) agg4, avg( cast(cs_net_profit as decimal(12,2))) agg5, avg( cast(c_birth_year as decimal(12,2))) agg6, avg( cast(cd1.cd_dep_count as decimal(12,2))) agg7 from catalog_sales, customer_demographics cd1, customer_demographics cd2, customer, customer_address, date_dim, item where cs_sold_date_sk = d_date_sk and cs_item_sk = i_item_sk and cs_bill_cdemo_sk = cd1.cd_demo_sk and cs_bill_customer_sk = c_customer_sk and cd1.cd_gender = 'M' and cd1.cd_education_status = 'Unknown' and c_current_cdemo_sk = cd2.cd_demo_sk and c_current_addr_sk = ca_address_sk and c_birth_month in (5,7,8,6,12,4) and d_year = 2000 and ca_state in ('MO','NY','ME' ,'MI','IA','OH','MS') group by rollup (i_item_id, ca_country, ca_state, ca_county) order by ca_country, ca_state, ca_county, i_item_id limit 100""", "q19" -> """ select i_brand_id brand_id, i_brand brand, i_manufact_id, i_manufact, sum(ss_ext_sales_price) ext_price from date_dim, store_sales, item,customer,customer_address,store where d_date_sk = ss_sold_date_sk and ss_item_sk = i_item_sk and i_manager_id=55 and d_moy=11 and d_year=1998 and ss_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and substr(ca_zip,1,5) <> substr(s_zip,1,5) and ss_store_sk = s_store_sk group by i_brand ,i_brand_id ,i_manufact_id ,i_manufact order by ext_price desc ,i_brand ,i_brand_id ,i_manufact_id ,i_manufact limit 100 """, "q20" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(cs_ext_sales_price) as itemrevenue ,sum(cs_ext_sales_price)*100/sum(sum(cs_ext_sales_price)) over (partition by i_class) as revenueratio from catalog_sales ,item ,date_dim where cs_item_sk = i_item_sk and i_category in ('Shoes', 'Electronics', 'Home') and cs_sold_date_sk = d_date_sk and d_date between cast('2000-05-15' as date) and (cast('2000-05-15' as date) + INTERVAL 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio limit 100""", "q21" -> """ select * from(select w_warehouse_name ,i_item_id ,sum(case when (cast(d_date as date) < cast ('2002-02-15' as date)) then inv_quantity_on_hand else 0 end) as inv_before ,sum(case when (cast(d_date as date) >= cast ('2002-02-15' as date)) then inv_quantity_on_hand else 0 end) as inv_after from inventory ,warehouse ,item ,date_dim where i_current_price between 0.99 and 1.49 and i_item_sk = inv_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_date between (cast ('2002-02-15' as date) - INTERVAL 30 days) and (cast ('2002-02-15' as date) + INTERVAL 30 days) group by w_warehouse_name, i_item_id) x where (case when inv_before > 0 then inv_after / inv_before else null end) between 2.0/3.0 and 3.0/2.0 order by w_warehouse_name ,i_item_id limit 100""", "q22" -> """ select i_product_name ,i_brand ,i_class ,i_category ,avg(inv_quantity_on_hand) qoh from inventory ,date_dim ,item where inv_date_sk=d_date_sk and inv_item_sk=i_item_sk and d_month_seq between 1202 and 1202 + 11 group by rollup(i_product_name ,i_brand ,i_class ,i_category) order by qoh, i_product_name, i_brand, i_class, i_category limit 100""", "q23a" -> """ with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt from store_sales ,date_dim ,item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and d_year in (2000,2000+1,2000+2,2000+3) group by substr(i_item_desc,1,30),i_item_sk,d_date having count(*) >4), max_store_sales as (select max(csales) tpcds_cmax from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales from store_sales ,customer ,date_dim where ss_customer_sk = c_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (2000,2000+1,2000+2,2000+3) group by c_customer_sk)), best_ss_customer as (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales from store_sales ,customer where ss_customer_sk = c_customer_sk group by c_customer_sk having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select * from max_store_sales)) select sum(sales) from (select cs_quantity*cs_list_price sales from catalog_sales ,date_dim where d_year = 2000 and d_moy = 4 and cs_sold_date_sk = d_date_sk and cs_item_sk in (select item_sk from frequent_ss_items) and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) union all select ws_quantity*ws_list_price sales from web_sales ,date_dim where d_year = 2000 and d_moy = 4 and ws_sold_date_sk = d_date_sk and ws_item_sk in (select item_sk from frequent_ss_items) and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer)) limit 100""", "q23b" -> """ with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt from store_sales ,date_dim ,item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and d_year in (2000,2000 + 1,2000 + 2,2000 + 3) group by substr(i_item_desc,1,30),i_item_sk,d_date having count(*) >4), max_store_sales as (select max(csales) tpcds_cmax from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales from store_sales ,customer ,date_dim where ss_customer_sk = c_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (2000,2000+1,2000+2,2000+3) group by c_customer_sk)), best_ss_customer as (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales from store_sales ,customer where ss_customer_sk = c_customer_sk group by c_customer_sk having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select * from max_store_sales)) select c_last_name,c_first_name,sales from (select c_last_name,c_first_name,sum(cs_quantity*cs_list_price) sales from catalog_sales ,customer ,date_dim where d_year = 2000 and d_moy = 4 and cs_sold_date_sk = d_date_sk and cs_item_sk in (select item_sk from frequent_ss_items) and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) and cs_bill_customer_sk = c_customer_sk group by c_last_name,c_first_name union all select c_last_name,c_first_name,sum(ws_quantity*ws_list_price) sales from web_sales ,customer ,date_dim where d_year = 2000 and d_moy = 4 and ws_sold_date_sk = d_date_sk and ws_item_sk in (select item_sk from frequent_ss_items) and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer) and ws_bill_customer_sk = c_customer_sk group by c_last_name,c_first_name) order by c_last_name,c_first_name,sales limit 100""", "q24a" -> """ with ssales as (select c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size ,sum(ss_net_paid_inc_tax) netpaid from store_sales ,store_returns ,store ,item ,customer ,customer_address where ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_customer_sk = c_customer_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and c_current_addr_sk = ca_address_sk and c_birth_country <> upper(ca_country) and s_zip = ca_zip and s_market_id=5 group by c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size) select c_last_name ,c_first_name ,s_store_name ,sum(netpaid) paid from ssales where i_color = 'cyan' group by c_last_name ,c_first_name ,s_store_name having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) order by c_last_name ,c_first_name ,s_store_name""", "q24b" -> """ with ssales as (select c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size ,sum(ss_net_paid_inc_tax) netpaid from store_sales ,store_returns ,store ,item ,customer ,customer_address where ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_customer_sk = c_customer_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and c_current_addr_sk = ca_address_sk and c_birth_country <> upper(ca_country) and s_zip = ca_zip and s_market_id = 5 group by c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size) select c_last_name ,c_first_name ,s_store_name ,sum(netpaid) paid from ssales where i_color = 'ivory' group by c_last_name ,c_first_name ,s_store_name having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) order by c_last_name ,c_first_name ,s_store_name""", "q25" -> """ select i_item_id ,i_item_desc ,s_store_id ,s_store_name ,stddev_samp(ss_net_profit) as store_sales_profit ,stddev_samp(sr_net_loss) as store_returns_loss ,stddev_samp(cs_net_profit) as catalog_sales_profit from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_moy = 4 and d1.d_year = 2000 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_moy between 4 and 10 and d2.d_year = 2000 and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_moy between 4 and 10 and d3.d_year = 2000 group by i_item_id ,i_item_desc ,s_store_id ,s_store_name order by i_item_id ,i_item_desc ,s_store_id ,s_store_name limit 100""", "q26" -> """ select i_item_id, avg(cs_quantity) agg1, avg(cs_list_price) agg2, avg(cs_coupon_amt) agg3, avg(cs_sales_price) agg4 from catalog_sales, customer_demographics, date_dim, item, promotion where cs_sold_date_sk = d_date_sk and cs_item_sk = i_item_sk and cs_bill_cdemo_sk = cd_demo_sk and cs_promo_sk = p_promo_sk and cd_gender = 'F' and cd_marital_status = 'M' and cd_education_status = 'Unknown' and (p_channel_email = 'N' or p_channel_event = 'N') and d_year = 2001 group by i_item_id order by i_item_id limit 100""", "q27" -> """ select i_item_id, s_state, grouping(s_state) g_state, avg(ss_quantity) agg1, avg(ss_list_price) agg2, avg(ss_coupon_amt) agg3, avg(ss_sales_price) agg4 from store_sales, customer_demographics, date_dim, store, item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and ss_cdemo_sk = cd_demo_sk and cd_gender = 'F' and cd_marital_status = 'D' and cd_education_status = '2 yr Degree' and d_year = 1999 and s_state in ('MI','WV', 'MI', 'NY', 'TN', 'MI') group by rollup (i_item_id, s_state) order by i_item_id ,s_state limit 100""", "q28" -> """ select * from (select avg(ss_list_price) B1_LP ,count(ss_list_price) B1_CNT ,count(distinct ss_list_price) B1_CNTD from store_sales where ss_quantity between 0 and 5 and (ss_list_price between 151 and 151+10 or ss_coupon_amt between 4349 and 4349+1000 or ss_wholesale_cost between 75 and 75+20)) B1, (select avg(ss_list_price) B2_LP ,count(ss_list_price) B2_CNT ,count(distinct ss_list_price) B2_CNTD from store_sales where ss_quantity between 6 and 10 and (ss_list_price between 45 and 45+10 or ss_coupon_amt between 12490 and 12490+1000 or ss_wholesale_cost between 37 and 37+20)) B2, (select avg(ss_list_price) B3_LP ,count(ss_list_price) B3_CNT ,count(distinct ss_list_price) B3_CNTD from store_sales where ss_quantity between 11 and 15 and (ss_list_price between 54 and 54+10 or ss_coupon_amt between 13038 and 13038+1000 or ss_wholesale_cost between 17 and 17+20)) B3, (select avg(ss_list_price) B4_LP ,count(ss_list_price) B4_CNT ,count(distinct ss_list_price) B4_CNTD from store_sales where ss_quantity between 16 and 20 and (ss_list_price between 178 and 178+10 or ss_coupon_amt between 10744 and 10744+1000 or ss_wholesale_cost between 51 and 51+20)) B4, (select avg(ss_list_price) B5_LP ,count(ss_list_price) B5_CNT ,count(distinct ss_list_price) B5_CNTD from store_sales where ss_quantity between 21 and 25 and (ss_list_price between 49 and 49+10 or ss_coupon_amt between 8494 and 8494+1000 or ss_wholesale_cost between 56 and 56+20)) B5, (select avg(ss_list_price) B6_LP ,count(ss_list_price) B6_CNT ,count(distinct ss_list_price) B6_CNTD from store_sales where ss_quantity between 26 and 30 and (ss_list_price between 0 and 0+10 or ss_coupon_amt between 17854 and 17854+1000 or ss_wholesale_cost between 31 and 31+20)) B6 limit 100""", "q29" -> """ select i_item_id ,i_item_desc ,s_store_id ,s_store_name ,max(ss_quantity) as store_sales_quantity ,max(sr_return_quantity) as store_returns_quantity ,max(cs_quantity) as catalog_sales_quantity from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_moy = 4 and d1.d_year = 1999 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_moy between 4 and 4 + 3 and d2.d_year = 1999 and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_year in (1999,1999+1,1999+2) group by i_item_id ,i_item_desc ,s_store_id ,s_store_name order by i_item_id ,i_item_desc ,s_store_id ,s_store_name limit 100""", "q30" -> """ with customer_total_return as (select wr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, sum(wr_return_amt) as ctr_total_return from web_returns ,date_dim ,customer_address where wr_returned_date_sk = d_date_sk and d_year =2000 and wr_returning_addr_sk = ca_address_sk group by wr_returning_customer_sk ,ca_state) select c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address ,c_last_review_date,ctr_total_return from customer_total_return ctr1 ,customer_address ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_state = ctr2.ctr_state) and ca_address_sk = c_current_addr_sk and ca_state = 'MD' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address ,c_last_review_date,ctr_total_return limit 100""", "q31" -> """ with ss as (select ca_county,d_qoy, d_year,sum(ss_ext_sales_price) as store_sales from store_sales,date_dim,customer_address where ss_sold_date_sk = d_date_sk and ss_addr_sk=ca_address_sk group by ca_county,d_qoy, d_year), ws as (select ca_county,d_qoy, d_year,sum(ws_ext_sales_price) as web_sales from web_sales,date_dim,customer_address where ws_sold_date_sk = d_date_sk and ws_bill_addr_sk=ca_address_sk group by ca_county,d_qoy, d_year) select ss1.ca_county ,ss1.d_year ,ws2.web_sales/ws1.web_sales web_q1_q2_increase ,ss2.store_sales/ss1.store_sales store_q1_q2_increase ,ws3.web_sales/ws2.web_sales web_q2_q3_increase ,ss3.store_sales/ss2.store_sales store_q2_q3_increase from ss ss1 ,ss ss2 ,ss ss3 ,ws ws1 ,ws ws2 ,ws ws3 where ss1.d_qoy = 1 and ss1.d_year = 1999 and ss1.ca_county = ss2.ca_county and ss2.d_qoy = 2 and ss2.d_year = 1999 and ss2.ca_county = ss3.ca_county and ss3.d_qoy = 3 and ss3.d_year = 1999 and ss1.ca_county = ws1.ca_county and ws1.d_qoy = 1 and ws1.d_year = 1999 and ws1.ca_county = ws2.ca_county and ws2.d_qoy = 2 and ws2.d_year = 1999 and ws1.ca_county = ws3.ca_county and ws3.d_qoy = 3 and ws3.d_year =1999 and case when ws1.web_sales > 0 then ws2.web_sales/ws1.web_sales else null end > case when ss1.store_sales > 0 then ss2.store_sales/ss1.store_sales else null end and case when ws2.web_sales > 0 then ws3.web_sales/ws2.web_sales else null end > case when ss2.store_sales > 0 then ss3.store_sales/ss2.store_sales else null end order by store_q2_q3_increase""", "q32" -> """ select sum(cs_ext_discount_amt) as `excess discount amount` from catalog_sales ,item ,date_dim where i_manufact_id = 7 and i_item_sk = cs_item_sk and d_date between '2000-01-21' and (cast('2000-01-21' as date) + INTERVAL 90 days) and d_date_sk = cs_sold_date_sk and cs_ext_discount_amt > ( select 1.3 * avg(cs_ext_discount_amt) from catalog_sales ,date_dim where cs_item_sk = i_item_sk and d_date between '2000-01-21' and (cast('2000-01-21' as date) + INTERVAL 90 days) and d_date_sk = cs_sold_date_sk ) limit 100""", "q33" -> """ with ss as ( select i_manufact_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Books')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 6 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -7 group by i_manufact_id), cs as ( select i_manufact_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Books')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 6 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -7 group by i_manufact_id), ws as ( select i_manufact_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Books')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 6 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -7 group by i_manufact_id) select i_manufact_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_manufact_id order by total_sales limit 100""", "q34" -> """ select c_last_name ,c_first_name ,c_salutation ,c_preferred_cust_flag ,ss_ticket_number ,cnt from (select ss_ticket_number ,ss_customer_sk ,count(*) cnt from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and (date_dim.d_dom between 1 and 3 or date_dim.d_dom between 25 and 28) and (household_demographics.hd_buy_potential = '501-1000' or household_demographics.hd_buy_potential = '5001-10000') and household_demographics.hd_vehicle_count > 0 and (case when household_demographics.hd_vehicle_count > 0 then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end) > 1.2 and date_dim.d_year in (1999,1999+1,1999+2) and store.s_county in ('Levy County','Val Verde County','Porter County','Nowata County', 'Lincoln County','Brazos County','Franklin Parish','Pipestone County') group by ss_ticket_number,ss_customer_sk) dn,customer where ss_customer_sk = c_customer_sk and cnt between 15 and 20 order by c_last_name,c_first_name,c_salutation,c_preferred_cust_flag desc, ss_ticket_number""", "q35" -> """ select ca_state, cd_gender, cd_marital_status, cd_dep_count, count(*) cnt1, sum(cd_dep_count), sum(cd_dep_count), sum(cd_dep_count), cd_dep_employed_count, count(*) cnt2, sum(cd_dep_employed_count), sum(cd_dep_employed_count), sum(cd_dep_employed_count), cd_dep_college_count, count(*) cnt3, sum(cd_dep_college_count), sum(cd_dep_college_count), sum(cd_dep_college_count) from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2001 and d_qoy < 4) and (exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2001 and d_qoy < 4) or exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2001 and d_qoy < 4)) group by ca_state, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count order by ca_state, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count limit 100""", "q36" -> """ select sum(ss_net_profit)/sum(ss_ext_sales_price) as gross_margin ,i_category ,i_class ,grouping(i_category)+grouping(i_class) as lochierarchy ,rank() over ( partition by grouping(i_category)+grouping(i_class), case when grouping(i_class) = 0 then i_category end order by sum(ss_net_profit)/sum(ss_ext_sales_price) asc) as rank_within_parent from store_sales ,date_dim d1 ,item ,store where d1.d_year = 1999 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and s_state in ('MO','AL','OH','WV', 'AL','MN','TN','WA') group by rollup(i_category,i_class) order by lochierarchy desc ,case when lochierarchy = 0 then i_category end ,rank_within_parent limit 100""", "q37" -> """ select i_item_id ,i_item_desc ,i_current_price from item, inventory, date_dim, catalog_sales where i_current_price between 57 and 57 + 30 and inv_item_sk = i_item_sk and d_date_sk=inv_date_sk and d_date between cast('2001-04-19' as date) and (cast('2001-04-19' as date) + interval 60 days) and i_manufact_id in (804,916,707,680) and inv_quantity_on_hand between 100 and 500 and cs_item_sk = i_item_sk group by i_item_id,i_item_desc,i_current_price order by i_item_id limit 100""", "q38" -> """ select count(*) from ( select distinct c_last_name, c_first_name, d_date from store_sales, date_dim, customer where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_customer_sk = customer.c_customer_sk and d_month_seq between 1189 and 1189 + 11 intersect select distinct c_last_name, c_first_name, d_date from catalog_sales, date_dim, customer where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1189 and 1189 + 11 intersect select distinct c_last_name, c_first_name, d_date from web_sales, date_dim, customer where web_sales.ws_sold_date_sk = date_dim.d_date_sk and web_sales.ws_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1189 and 1189 + 11 ) hot_cust limit 100""", "q39a" -> """ with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stdev,mean, case mean when 0 then null else stdev/mean end cov from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean from inventory ,item ,warehouse ,date_dim where inv_item_sk = i_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_year =2000 group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo where case mean when 0 then 0 else stdev/mean end > 1) select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov from inv inv1,inv inv2 where inv1.i_item_sk = inv2.i_item_sk and inv1.w_warehouse_sk = inv2.w_warehouse_sk and inv1.d_moy=3 and inv2.d_moy=3+1 order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov ,inv2.d_moy,inv2.mean, inv2.cov""", "q39b" -> """ with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stdev,mean, case mean when 0 then null else stdev/mean end cov from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean from inventory ,item ,warehouse ,date_dim where inv_item_sk = i_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_year =2000 group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo where case mean when 0 then 0 else stdev/mean end > 1) select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov from inv inv1,inv inv2 where inv1.i_item_sk = inv2.i_item_sk and inv1.w_warehouse_sk = inv2.w_warehouse_sk and inv1.d_moy=3 and inv2.d_moy=3+1 and inv1.cov > 1.5 order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov ,inv2.d_moy,inv2.mean, inv2.cov""", "q40" -> """ select w_state ,i_item_id ,sum(case when (cast(d_date as date) < cast ('2000-04-09' as date)) then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_before ,sum(case when (cast(d_date as date) >= cast ('2000-04-09' as date)) then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_after from catalog_sales left outer join catalog_returns on (cs_order_number = cr_order_number and cs_item_sk = cr_item_sk) ,warehouse ,item ,date_dim where i_current_price between 0.99 and 1.49 and i_item_sk = cs_item_sk and cs_warehouse_sk = w_warehouse_sk and cs_sold_date_sk = d_date_sk and d_date between (cast ('2000-04-09' as date) - INTERVAL 30 days) and (cast ('2000-04-09' as date) + INTERVAL 30 days) group by w_state,i_item_id order by w_state,i_item_id limit 100""", "q41" -> """ select distinct(i_product_name) from item i1 where i_manufact_id between 917 and 917+40 and (select count(*) as item_cnt from item where (i_manufact = i1.i_manufact and ((i_category = 'Women' and (i_color = 'antique' or i_color = 'pale') and (i_units = 'Tbl' or i_units = 'Case') and (i_size = 'small' or i_size = 'extra large') ) or (i_category = 'Women' and (i_color = 'snow' or i_color = 'lemon') and (i_units = 'Box' or i_units = 'Ounce') and (i_size = 'economy' or i_size = 'N/A') ) or (i_category = 'Men' and (i_color = 'green' or i_color = 'blue') and (i_units = 'Gross' or i_units = 'Ton') and (i_size = 'large' or i_size = 'petite') ) or (i_category = 'Men' and (i_color = 'cream' or i_color = 'frosted') and (i_units = 'Bundle' or i_units = 'Gram') and (i_size = 'small' or i_size = 'extra large') ))) or (i_manufact = i1.i_manufact and ((i_category = 'Women' and (i_color = 'orange' or i_color = 'spring') and (i_units = 'Lb' or i_units = 'Carton') and (i_size = 'small' or i_size = 'extra large') ) or (i_category = 'Women' and (i_color = 'lawn' or i_color = 'violet') and (i_units = 'Oz' or i_units = 'Cup') and (i_size = 'economy' or i_size = 'N/A') ) or (i_category = 'Men' and (i_color = 'navy' or i_color = 'linen') and (i_units = 'Pound' or i_units = 'Unknown') and (i_size = 'large' or i_size = 'petite') ) or (i_category = 'Men' and (i_color = 'almond' or i_color = 'olive') and (i_units = 'Pallet' or i_units = 'Bunch') and (i_size = 'small' or i_size = 'extra large') )))) > 0 order by i_product_name limit 100""", "q42" -> """ select dt.d_year ,item.i_category_id ,item.i_category ,sum(ss_ext_sales_price) from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manager_id = 1 and dt.d_moy=11 and dt.d_year=1998 group by dt.d_year ,item.i_category_id ,item.i_category order by sum(ss_ext_sales_price) desc,dt.d_year ,item.i_category_id ,item.i_category limit 100 """, "q43" -> """ select s_store_name, s_store_id, sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales from date_dim, store_sales, store where d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk and s_gmt_offset = -6 and d_year = 2000 group by s_store_name, s_store_id order by s_store_name, s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales limit 100""", "q44" -> """ select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing from(select * from (select item_sk,rank() over (order by rank_col asc) rnk from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col from store_sales ss1 where ss_store_sk = 731 group by ss_item_sk having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col from store_sales where ss_store_sk = 731 and ss_promo_sk is null group by ss_store_sk))V1)V11 where rnk < 11) asceding, (select * from (select item_sk,rank() over (order by rank_col desc) rnk from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col from store_sales ss1 where ss_store_sk = 731 group by ss_item_sk having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col from store_sales where ss_store_sk = 731 and ss_promo_sk is null group by ss_store_sk))V2)V21 where rnk < 11) descending, item i1, item i2 where asceding.rnk = descending.rnk and i1.i_item_sk=asceding.item_sk and i2.i_item_sk=descending.item_sk order by asceding.rnk limit 100""", "q45" -> """ select ca_zip, ca_city, sum(ws_sales_price) from web_sales, customer, customer_address, date_dim, item where ws_bill_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and ws_item_sk = i_item_sk and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') or i_item_id in (select i_item_id from item where i_item_sk in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) ) ) and ws_sold_date_sk = d_date_sk and d_qoy = 1 and d_year = 2000 group by ca_zip, ca_city order by ca_zip, ca_city limit 100""", "q46" -> """ select c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number ,amt,profit from (select ss_ticket_number ,ss_customer_sk ,ca_city bought_city ,sum(ss_coupon_amt) amt ,sum(ss_net_profit) profit from store_sales,date_dim,store,household_demographics,customer_address where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and store_sales.ss_addr_sk = customer_address.ca_address_sk and (household_demographics.hd_dep_count = 1 or household_demographics.hd_vehicle_count= 2) and date_dim.d_dow in (6,0) and date_dim.d_year in (2000,2000+1,2000+2) and store.s_city in ('Buena Vista','Friendship','Monroe','Oak Hill','Randolph') group by ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city) dn,customer,customer_address current_addr where ss_customer_sk = c_customer_sk and customer.c_current_addr_sk = current_addr.ca_address_sk and current_addr.ca_city <> bought_city order by c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number limit 100""", "q47" -> """ with v1 as( select i_category, i_brand, s_store_name, s_company_name, d_year, d_moy, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_category, i_brand, s_store_name, s_company_name, d_year) avg_monthly_sales, rank() over (partition by i_category, i_brand, s_store_name, s_company_name order by d_year, d_moy) rn from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ( d_year = 1999 or ( d_year = 1999-1 and d_moy =12) or ( d_year = 1999+1 and d_moy =1) ) group by i_category, i_brand, s_store_name, s_company_name, d_year, d_moy), v2 as( select v1.i_category ,v1.d_year, v1.d_moy ,v1.avg_monthly_sales ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum from v1, v1 v1_lag, v1 v1_lead where v1.i_category = v1_lag.i_category and v1.i_category = v1_lead.i_category and v1.i_brand = v1_lag.i_brand and v1.i_brand = v1_lead.i_brand and v1.s_store_name = v1_lag.s_store_name and v1.s_store_name = v1_lead.s_store_name and v1.s_company_name = v1_lag.s_company_name and v1.s_company_name = v1_lead.s_company_name and v1.rn = v1_lag.rn + 1 and v1.rn = v1_lead.rn - 1) select * from v2 where d_year = 1999 and avg_monthly_sales > 0 and case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by sum_sales - avg_monthly_sales, sum_sales limit 100""", "q48" -> """ select sum (ss_quantity) from store_sales, store, customer_demographics, customer_address, date_dim where s_store_sk = ss_store_sk and ss_sold_date_sk = d_date_sk and d_year = 1999 and ( ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'S' and cd_education_status = 'Primary' and ss_sales_price between 100.00 and 150.00 ) or ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'D' and cd_education_status = 'College' and ss_sales_price between 50.00 and 100.00 ) or ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'U' and cd_education_status = '2 yr Degree' and ss_sales_price between 150.00 and 200.00 ) ) and ( ( ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('ND', 'NC', 'TX') and ss_net_profit between 0 and 2000 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('VA', 'IA', 'AR') and ss_net_profit between 150 and 3000 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('MA', 'FL', 'TN') and ss_net_profit between 50 and 25000 ) )""", "q49" -> """ select channel, item, return_ratio, return_rank, currency_rank from (select 'web' as channel ,web.item ,web.return_ratio ,web.return_rank ,web.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select ws.ws_item_sk as item ,(cast(sum(coalesce(wr.wr_return_quantity,0)) as decimal(15,4))/ cast(sum(coalesce(ws.ws_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(wr.wr_return_amt,0)) as decimal(15,4))/ cast(sum(coalesce(ws.ws_net_paid,0)) as decimal(15,4) )) as currency_ratio from web_sales ws left outer join web_returns wr on (ws.ws_order_number = wr.wr_order_number and ws.ws_item_sk = wr.wr_item_sk) ,date_dim where wr.wr_return_amt > 10000 and ws.ws_net_profit > 1 and ws.ws_net_paid > 0 and ws.ws_quantity > 0 and ws_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 11 group by ws.ws_item_sk ) in_web ) web where ( web.return_rank <= 10 or web.currency_rank <= 10 ) union select 'catalog' as channel ,catalog.item ,catalog.return_ratio ,catalog.return_rank ,catalog.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select cs.cs_item_sk as item ,(cast(sum(coalesce(cr.cr_return_quantity,0)) as decimal(15,4))/ cast(sum(coalesce(cs.cs_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(cr.cr_return_amount,0)) as decimal(15,4))/ cast(sum(coalesce(cs.cs_net_paid,0)) as decimal(15,4) )) as currency_ratio from catalog_sales cs left outer join catalog_returns cr on (cs.cs_order_number = cr.cr_order_number and cs.cs_item_sk = cr.cr_item_sk) ,date_dim where cr.cr_return_amount > 10000 and cs.cs_net_profit > 1 and cs.cs_net_paid > 0 and cs.cs_quantity > 0 and cs_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 11 group by cs.cs_item_sk ) in_cat ) catalog where ( catalog.return_rank <= 10 or catalog.currency_rank <=10 ) union select 'store' as channel ,store.item ,store.return_ratio ,store.return_rank ,store.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select sts.ss_item_sk as item ,(cast(sum(coalesce(sr.sr_return_quantity,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(sr.sr_return_amt,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_net_paid,0)) as decimal(15,4) )) as currency_ratio from store_sales sts left outer join store_returns sr on (sts.ss_ticket_number = sr.sr_ticket_number and sts.ss_item_sk = sr.sr_item_sk) ,date_dim where sr.sr_return_amt > 10000 and sts.ss_net_profit > 1 and sts.ss_net_paid > 0 and sts.ss_quantity > 0 and ss_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 11 group by sts.ss_item_sk ) in_store ) store where ( store.return_rank <= 10 or store.currency_rank <= 10 ) ) order by 1,4,5,2 limit 100""", "q50" -> """ select s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip ,sum(case when (sr_returned_date_sk - ss_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 30) and (sr_returned_date_sk - ss_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 60) and (sr_returned_date_sk - ss_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 90) and (sr_returned_date_sk - ss_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from store_sales ,store_returns ,store ,date_dim d1 ,date_dim d2 where d2.d_year = 2002 and d2.d_moy = 10 and ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_sold_date_sk = d1.d_date_sk and sr_returned_date_sk = d2.d_date_sk and ss_customer_sk = sr_customer_sk and ss_store_sk = s_store_sk group by s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip order by s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip limit 100""", "q51" -> """ WITH web_v1 as ( select ws_item_sk item_sk, d_date, sum(sum(ws_sales_price)) over (partition by ws_item_sk order by d_date rows between unbounded preceding and current row) cume_sales from web_sales ,date_dim where ws_sold_date_sk=d_date_sk and d_month_seq between 1213 and 1213+11 and ws_item_sk is not NULL group by ws_item_sk, d_date), store_v1 as ( select ss_item_sk item_sk, d_date, sum(sum(ss_sales_price)) over (partition by ss_item_sk order by d_date rows between unbounded preceding and current row) cume_sales from store_sales ,date_dim where ss_sold_date_sk=d_date_sk and d_month_seq between 1213 and 1213+11 and ss_item_sk is not NULL group by ss_item_sk, d_date) select * from (select item_sk ,d_date ,web_sales ,store_sales ,max(web_sales) over (partition by item_sk order by d_date rows between unbounded preceding and current row) web_cumulative ,max(store_sales) over (partition by item_sk order by d_date rows between unbounded preceding and current row) store_cumulative from (select case when web.item_sk is not null then web.item_sk else store.item_sk end item_sk ,case when web.d_date is not null then web.d_date else store.d_date end d_date ,web.cume_sales web_sales ,store.cume_sales store_sales from web_v1 web full outer join store_v1 store on (web.item_sk = store.item_sk and web.d_date = store.d_date) )x )y where web_cumulative > store_cumulative order by item_sk ,d_date limit 100""", "q52" -> """ select dt.d_year ,item.i_brand_id brand_id ,item.i_brand brand ,sum(ss_ext_sales_price) ext_price from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manager_id = 1 and dt.d_moy=11 and dt.d_year=1998 group by dt.d_year ,item.i_brand ,item.i_brand_id order by dt.d_year ,ext_price desc ,brand_id limit 100 """, "q53" -> """ select * from (select i_manufact_id, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_manufact_id) avg_quarterly_sales from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_month_seq in (1219,1219+1,1219+2,1219+3,1219+4,1219+5,1219+6,1219+7,1219+8,1219+9,1219+10,1219+11) and ((i_category in ('Books','Children','Electronics') and i_class in ('personal','portable','reference','self-help') and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', 'exportiunivamalg #9','scholaramalgamalg #9')) or(i_category in ('Women','Music','Men') and i_class in ('accessories','classical','fragrances','pants') and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 'importoamalg #1'))) group by i_manufact_id, d_qoy ) tmp1 where case when avg_quarterly_sales > 0 then abs (sum_sales - avg_quarterly_sales)/ avg_quarterly_sales else null end > 0.1 order by avg_quarterly_sales, sum_sales, i_manufact_id limit 100""", "q54" -> """ with my_customers as ( select distinct c_customer_sk , c_current_addr_sk from ( select cs_sold_date_sk sold_date_sk, cs_bill_customer_sk customer_sk, cs_item_sk item_sk from catalog_sales union all select ws_sold_date_sk sold_date_sk, ws_bill_customer_sk customer_sk, ws_item_sk item_sk from web_sales ) cs_or_ws_sales, item, date_dim, customer where sold_date_sk = d_date_sk and item_sk = i_item_sk and i_category = 'Men' and i_class = 'shirts' and c_customer_sk = cs_or_ws_sales.customer_sk and d_moy = 2 and d_year = 1999 ) , my_revenue as ( select c_customer_sk, sum(ss_ext_sales_price) as revenue from my_customers, store_sales, customer_address, store, date_dim where c_current_addr_sk = ca_address_sk and ca_county = s_county and ca_state = s_state and ss_sold_date_sk = d_date_sk and c_customer_sk = ss_customer_sk and d_month_seq between (select distinct d_month_seq+1 from date_dim where d_year = 1999 and d_moy = 2) and (select distinct d_month_seq+3 from date_dim where d_year = 1999 and d_moy = 2) group by c_customer_sk ) , segments as (select cast((revenue/50) as int) as segment from my_revenue ) select segment, count(*) as num_customers, segment*50 as segment_base from segments group by segment order by segment, num_customers limit 100""", "q55" -> """ select i_brand_id brand_id, i_brand brand, sum(ss_ext_sales_price) ext_price from date_dim, store_sales, item where d_date_sk = ss_sold_date_sk and ss_item_sk = i_item_sk and i_manager_id=96 and d_moy=11 and d_year=2000 group by i_brand, i_brand_id order by ext_price desc, i_brand_id limit 100 """, "q56" -> """ with ss as ( select i_item_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('antique','white','smoke')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 6 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_item_id), cs as ( select i_item_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('antique','white','smoke')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 6 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_item_id), ws as ( select i_item_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('antique','white','smoke')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 6 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_item_id) select i_item_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_item_id order by total_sales, i_item_id limit 100""", "q57" -> """ with v1 as( select i_category, i_brand, cc_name, d_year, d_moy, sum(cs_sales_price) sum_sales, avg(sum(cs_sales_price)) over (partition by i_category, i_brand, cc_name, d_year) avg_monthly_sales, rank() over (partition by i_category, i_brand, cc_name order by d_year, d_moy) rn from item, catalog_sales, date_dim, call_center where cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and cc_call_center_sk= cs_call_center_sk and ( d_year = 2000 or ( d_year = 2000-1 and d_moy =12) or ( d_year = 2000+1 and d_moy =1) ) group by i_category, i_brand, cc_name , d_year, d_moy), v2 as( select v1.i_category, v1.i_brand, v1.cc_name ,v1.d_year ,v1.avg_monthly_sales ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum from v1, v1 v1_lag, v1 v1_lead where v1.i_category = v1_lag.i_category and v1.i_category = v1_lead.i_category and v1.i_brand = v1_lag.i_brand and v1.i_brand = v1_lead.i_brand and v1. cc_name = v1_lag. cc_name and v1. cc_name = v1_lead. cc_name and v1.rn = v1_lag.rn + 1 and v1.rn = v1_lead.rn - 1) select * from v2 where d_year = 2000 and avg_monthly_sales > 0 and case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by sum_sales - avg_monthly_sales, sum_sales limit 100""", "q58" -> """ with ss_items as (select i_item_id item_id ,sum(ss_ext_sales_price) ss_item_rev from store_sales ,item ,date_dim where ss_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq = (select d_week_seq from date_dim where d_date = '2001-01-27')) and ss_sold_date_sk = d_date_sk group by i_item_id), cs_items as (select i_item_id item_id ,sum(cs_ext_sales_price) cs_item_rev from catalog_sales ,item ,date_dim where cs_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq = (select d_week_seq from date_dim where d_date = '2001-01-27')) and cs_sold_date_sk = d_date_sk group by i_item_id), ws_items as (select i_item_id item_id ,sum(ws_ext_sales_price) ws_item_rev from web_sales ,item ,date_dim where ws_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq =(select d_week_seq from date_dim where d_date = '2001-01-27')) and ws_sold_date_sk = d_date_sk group by i_item_id) select ss_items.item_id ,ss_item_rev ,ss_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ss_dev ,cs_item_rev ,cs_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 cs_dev ,ws_item_rev ,ws_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ws_dev ,(ss_item_rev+cs_item_rev+ws_item_rev)/3 average from ss_items,cs_items,ws_items where ss_items.item_id=cs_items.item_id and ss_items.item_id=ws_items.item_id and ss_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev and ss_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev and cs_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev and cs_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev and ws_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev and ws_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev order by item_id ,ss_item_rev limit 100""", "q59" -> """ with wss as (select d_week_seq, ss_store_sk, sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales from store_sales,date_dim where d_date_sk = ss_sold_date_sk group by d_week_seq,ss_store_sk ) select s_store_name1,s_store_id1,d_week_seq1 ,sun_sales1/sun_sales2,mon_sales1/mon_sales2 ,tue_sales1/tue_sales2,wed_sales1/wed_sales2,thu_sales1/thu_sales2 ,fri_sales1/fri_sales2,sat_sales1/sat_sales2 from (select s_store_name s_store_name1,wss.d_week_seq d_week_seq1 ,s_store_id s_store_id1,sun_sales sun_sales1 ,mon_sales mon_sales1,tue_sales tue_sales1 ,wed_sales wed_sales1,thu_sales thu_sales1 ,fri_sales fri_sales1,sat_sales sat_sales1 from wss,store,date_dim d where d.d_week_seq = wss.d_week_seq and ss_store_sk = s_store_sk and d_month_seq between 1177 and 1177 + 11) y, (select s_store_name s_store_name2,wss.d_week_seq d_week_seq2 ,s_store_id s_store_id2,sun_sales sun_sales2 ,mon_sales mon_sales2,tue_sales tue_sales2 ,wed_sales wed_sales2,thu_sales thu_sales2 ,fri_sales fri_sales2,sat_sales sat_sales2 from wss,store,date_dim d where d.d_week_seq = wss.d_week_seq and ss_store_sk = s_store_sk and d_month_seq between 1177+ 12 and 1177 + 23) x where s_store_id1=s_store_id2 and d_week_seq1=d_week_seq2-52 order by s_store_name1,s_store_id1,d_week_seq1 limit 100""", "q60" -> """ with ss as ( select i_item_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Shoes')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 2002 and d_moy = 8 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -7 group by i_item_id), cs as ( select i_item_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Shoes')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 2002 and d_moy = 8 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -7 group by i_item_id), ws as ( select i_item_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Shoes')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 2002 and d_moy = 8 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -7 group by i_item_id) select i_item_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_item_id order by i_item_id ,total_sales limit 100""", "q61" -> """ select promotions,total,cast(promotions as decimal(15,4))/cast(total as decimal(15,4))*100 from (select sum(ss_ext_sales_price) promotions from store_sales ,store ,promotion ,date_dim ,customer ,customer_address ,item where ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ss_promo_sk = p_promo_sk and ss_customer_sk= c_customer_sk and ca_address_sk = c_current_addr_sk and ss_item_sk = i_item_sk and ca_gmt_offset = -6 and i_category = 'Home' and (p_channel_dmail = 'Y' or p_channel_email = 'Y' or p_channel_tv = 'Y') and s_gmt_offset = -6 and d_year = 1999 and d_moy = 12) promotional_sales, (select sum(ss_ext_sales_price) total from store_sales ,store ,date_dim ,customer ,customer_address ,item where ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ss_customer_sk= c_customer_sk and ca_address_sk = c_current_addr_sk and ss_item_sk = i_item_sk and ca_gmt_offset = -6 and i_category = 'Home' and s_gmt_offset = -6 and d_year = 1999 and d_moy = 12) all_sales order by promotions, total limit 100""", "q62" -> """ select substr(w_warehouse_name,1,20) ,sm_type ,web_name ,sum(case when (ws_ship_date_sk - ws_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 30) and (ws_ship_date_sk - ws_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 60) and (ws_ship_date_sk - ws_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 90) and (ws_ship_date_sk - ws_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from web_sales ,warehouse ,ship_mode ,web_site ,date_dim where d_month_seq between 1191 and 1191 + 11 and ws_ship_date_sk = d_date_sk and ws_warehouse_sk = w_warehouse_sk and ws_ship_mode_sk = sm_ship_mode_sk and ws_web_site_sk = web_site_sk group by substr(w_warehouse_name,1,20) ,sm_type ,web_name order by substr(w_warehouse_name,1,20) ,sm_type ,web_name limit 100""", "q63" -> """ select * from (select i_manager_id ,sum(ss_sales_price) sum_sales ,avg(sum(ss_sales_price)) over (partition by i_manager_id) avg_monthly_sales from item ,store_sales ,date_dim ,store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_month_seq in (1193,1193+1,1193+2,1193+3,1193+4,1193+5,1193+6,1193+7,1193+8,1193+9,1193+10,1193+11) and (( i_category in ('Books','Children','Electronics') and i_class in ('personal','portable','reference','self-help') and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', 'exportiunivamalg #9','scholaramalgamalg #9')) or( i_category in ('Women','Music','Men') and i_class in ('accessories','classical','fragrances','pants') and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 'importoamalg #1'))) group by i_manager_id, d_moy) tmp1 where case when avg_monthly_sales > 0 then abs (sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by i_manager_id ,avg_monthly_sales ,sum_sales limit 100""", "q64" -> """ with cs_ui as (select cs_item_sk ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund from catalog_sales ,catalog_returns where cs_item_sk = cr_item_sk and cs_order_number = cr_order_number group by cs_item_sk having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), cross_sales as (select i_product_name product_name ,i_item_sk item_sk ,s_store_name store_name ,s_zip store_zip ,ad1.ca_street_number b_street_number ,ad1.ca_street_name b_street_name ,ad1.ca_city b_city ,ad1.ca_zip b_zip ,ad2.ca_street_number c_street_number ,ad2.ca_street_name c_street_name ,ad2.ca_city c_city ,ad2.ca_zip c_zip ,d1.d_year as syear ,d2.d_year as fsyear ,d3.d_year s2year ,count(*) cnt ,sum(ss_wholesale_cost) s1 ,sum(ss_list_price) s2 ,sum(ss_coupon_amt) s3 FROM store_sales ,store_returns ,cs_ui ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,customer ,customer_demographics cd1 ,customer_demographics cd2 ,promotion ,household_demographics hd1 ,household_demographics hd2 ,customer_address ad1 ,customer_address ad2 ,income_band ib1 ,income_band ib2 ,item WHERE ss_store_sk = s_store_sk AND ss_sold_date_sk = d1.d_date_sk AND ss_customer_sk = c_customer_sk AND ss_cdemo_sk= cd1.cd_demo_sk AND ss_hdemo_sk = hd1.hd_demo_sk AND ss_addr_sk = ad1.ca_address_sk and ss_item_sk = i_item_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and ss_item_sk = cs_ui.cs_item_sk and c_current_cdemo_sk = cd2.cd_demo_sk AND c_current_hdemo_sk = hd2.hd_demo_sk AND c_current_addr_sk = ad2.ca_address_sk and c_first_sales_date_sk = d2.d_date_sk and c_first_shipto_date_sk = d3.d_date_sk and ss_promo_sk = p_promo_sk and hd1.hd_income_band_sk = ib1.ib_income_band_sk and hd2.hd_income_band_sk = ib2.ib_income_band_sk and cd1.cd_marital_status <> cd2.cd_marital_status and i_color in ('orange','aquamarine','olive','linen','smoke','coral') and i_current_price between 74 and 74 + 10 and i_current_price between 74 + 1 and 74 + 15 group by i_product_name ,i_item_sk ,s_store_name ,s_zip ,ad1.ca_street_number ,ad1.ca_street_name ,ad1.ca_city ,ad1.ca_zip ,ad2.ca_street_number ,ad2.ca_street_name ,ad2.ca_city ,ad2.ca_zip ,d1.d_year ,d2.d_year ,d3.d_year ) select cs1.product_name ,cs1.store_name ,cs1.store_zip ,cs1.b_street_number ,cs1.b_street_name ,cs1.b_city ,cs1.b_zip ,cs1.c_street_number ,cs1.c_street_name ,cs1.c_city ,cs1.c_zip ,cs1.syear ,cs1.cnt ,cs1.s1 as s11 ,cs1.s2 as s21 ,cs1.s3 as s31 ,cs2.s1 as s12 ,cs2.s2 as s22 ,cs2.s3 as s32 ,cs2.syear ,cs2.cnt from cross_sales cs1,cross_sales cs2 where cs1.item_sk=cs2.item_sk and cs1.syear = 2001 and cs2.syear = 2001 + 1 and cs2.cnt <= cs1.cnt and cs1.store_name = cs2.store_name and cs1.store_zip = cs2.store_zip order by cs1.product_name ,cs1.store_name ,cs2.cnt ,cs1.s1 ,cs2.s1""", "q65" -> """ select s_store_name, i_item_desc, sc.revenue, i_current_price, i_wholesale_cost, i_brand from store, item, (select ss_store_sk, avg(revenue) as ave from (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue from store_sales, date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1195 and 1195+11 group by ss_store_sk, ss_item_sk) sa group by ss_store_sk) sb, (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue from store_sales, date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1195 and 1195+11 group by ss_store_sk, ss_item_sk) sc where sb.ss_store_sk = sc.ss_store_sk and sc.revenue <= 0.1 * sb.ave and s_store_sk = sc.ss_store_sk and i_item_sk = sc.ss_item_sk order by s_store_name, i_item_desc limit 100""", "q66" -> """ select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,ship_carriers ,year ,sum(jan_sales) as jan_sales ,sum(feb_sales) as feb_sales ,sum(mar_sales) as mar_sales ,sum(apr_sales) as apr_sales ,sum(may_sales) as may_sales ,sum(jun_sales) as jun_sales ,sum(jul_sales) as jul_sales ,sum(aug_sales) as aug_sales ,sum(sep_sales) as sep_sales ,sum(oct_sales) as oct_sales ,sum(nov_sales) as nov_sales ,sum(dec_sales) as dec_sales ,sum(jan_sales/w_warehouse_sq_ft) as jan_sales_per_sq_foot ,sum(feb_sales/w_warehouse_sq_ft) as feb_sales_per_sq_foot ,sum(mar_sales/w_warehouse_sq_ft) as mar_sales_per_sq_foot ,sum(apr_sales/w_warehouse_sq_ft) as apr_sales_per_sq_foot ,sum(may_sales/w_warehouse_sq_ft) as may_sales_per_sq_foot ,sum(jun_sales/w_warehouse_sq_ft) as jun_sales_per_sq_foot ,sum(jul_sales/w_warehouse_sq_ft) as jul_sales_per_sq_foot ,sum(aug_sales/w_warehouse_sq_ft) as aug_sales_per_sq_foot ,sum(sep_sales/w_warehouse_sq_ft) as sep_sales_per_sq_foot ,sum(oct_sales/w_warehouse_sq_ft) as oct_sales_per_sq_foot ,sum(nov_sales/w_warehouse_sq_ft) as nov_sales_per_sq_foot ,sum(dec_sales/w_warehouse_sq_ft) as dec_sales_per_sq_foot ,sum(jan_net) as jan_net ,sum(feb_net) as feb_net ,sum(mar_net) as mar_net ,sum(apr_net) as apr_net ,sum(may_net) as may_net ,sum(jun_net) as jun_net ,sum(jul_net) as jul_net ,sum(aug_net) as aug_net ,sum(sep_net) as sep_net ,sum(oct_net) as oct_net ,sum(nov_net) as nov_net ,sum(dec_net) as dec_net from ( select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,'LATVIAN' || ',' || 'ALLIANCE' as ship_carriers ,d_year as year ,sum(case when d_moy = 1 then ws_ext_list_price* ws_quantity else 0 end) as jan_sales ,sum(case when d_moy = 2 then ws_ext_list_price* ws_quantity else 0 end) as feb_sales ,sum(case when d_moy = 3 then ws_ext_list_price* ws_quantity else 0 end) as mar_sales ,sum(case when d_moy = 4 then ws_ext_list_price* ws_quantity else 0 end) as apr_sales ,sum(case when d_moy = 5 then ws_ext_list_price* ws_quantity else 0 end) as may_sales ,sum(case when d_moy = 6 then ws_ext_list_price* ws_quantity else 0 end) as jun_sales ,sum(case when d_moy = 7 then ws_ext_list_price* ws_quantity else 0 end) as jul_sales ,sum(case when d_moy = 8 then ws_ext_list_price* ws_quantity else 0 end) as aug_sales ,sum(case when d_moy = 9 then ws_ext_list_price* ws_quantity else 0 end) as sep_sales ,sum(case when d_moy = 10 then ws_ext_list_price* ws_quantity else 0 end) as oct_sales ,sum(case when d_moy = 11 then ws_ext_list_price* ws_quantity else 0 end) as nov_sales ,sum(case when d_moy = 12 then ws_ext_list_price* ws_quantity else 0 end) as dec_sales ,sum(case when d_moy = 1 then ws_net_paid_inc_ship * ws_quantity else 0 end) as jan_net ,sum(case when d_moy = 2 then ws_net_paid_inc_ship * ws_quantity else 0 end) as feb_net ,sum(case when d_moy = 3 then ws_net_paid_inc_ship * ws_quantity else 0 end) as mar_net ,sum(case when d_moy = 4 then ws_net_paid_inc_ship * ws_quantity else 0 end) as apr_net ,sum(case when d_moy = 5 then ws_net_paid_inc_ship * ws_quantity else 0 end) as may_net ,sum(case when d_moy = 6 then ws_net_paid_inc_ship * ws_quantity else 0 end) as jun_net ,sum(case when d_moy = 7 then ws_net_paid_inc_ship * ws_quantity else 0 end) as jul_net ,sum(case when d_moy = 8 then ws_net_paid_inc_ship * ws_quantity else 0 end) as aug_net ,sum(case when d_moy = 9 then ws_net_paid_inc_ship * ws_quantity else 0 end) as sep_net ,sum(case when d_moy = 10 then ws_net_paid_inc_ship * ws_quantity else 0 end) as oct_net ,sum(case when d_moy = 11 then ws_net_paid_inc_ship * ws_quantity else 0 end) as nov_net ,sum(case when d_moy = 12 then ws_net_paid_inc_ship * ws_quantity else 0 end) as dec_net from web_sales ,warehouse ,date_dim ,time_dim ,ship_mode where ws_warehouse_sk = w_warehouse_sk and ws_sold_date_sk = d_date_sk and ws_sold_time_sk = t_time_sk and ws_ship_mode_sk = sm_ship_mode_sk and d_year = 1998 and t_time between 16224 and 16224+28800 and sm_carrier in ('LATVIAN','ALLIANCE') group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,d_year union all select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,'LATVIAN' || ',' || 'ALLIANCE' as ship_carriers ,d_year as year ,sum(case when d_moy = 1 then cs_ext_sales_price* cs_quantity else 0 end) as jan_sales ,sum(case when d_moy = 2 then cs_ext_sales_price* cs_quantity else 0 end) as feb_sales ,sum(case when d_moy = 3 then cs_ext_sales_price* cs_quantity else 0 end) as mar_sales ,sum(case when d_moy = 4 then cs_ext_sales_price* cs_quantity else 0 end) as apr_sales ,sum(case when d_moy = 5 then cs_ext_sales_price* cs_quantity else 0 end) as may_sales ,sum(case when d_moy = 6 then cs_ext_sales_price* cs_quantity else 0 end) as jun_sales ,sum(case when d_moy = 7 then cs_ext_sales_price* cs_quantity else 0 end) as jul_sales ,sum(case when d_moy = 8 then cs_ext_sales_price* cs_quantity else 0 end) as aug_sales ,sum(case when d_moy = 9 then cs_ext_sales_price* cs_quantity else 0 end) as sep_sales ,sum(case when d_moy = 10 then cs_ext_sales_price* cs_quantity else 0 end) as oct_sales ,sum(case when d_moy = 11 then cs_ext_sales_price* cs_quantity else 0 end) as nov_sales ,sum(case when d_moy = 12 then cs_ext_sales_price* cs_quantity else 0 end) as dec_sales ,sum(case when d_moy = 1 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as jan_net ,sum(case when d_moy = 2 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as feb_net ,sum(case when d_moy = 3 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as mar_net ,sum(case when d_moy = 4 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as apr_net ,sum(case when d_moy = 5 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as may_net ,sum(case when d_moy = 6 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as jun_net ,sum(case when d_moy = 7 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as jul_net ,sum(case when d_moy = 8 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as aug_net ,sum(case when d_moy = 9 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as sep_net ,sum(case when d_moy = 10 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as oct_net ,sum(case when d_moy = 11 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as nov_net ,sum(case when d_moy = 12 then cs_net_paid_inc_ship_tax * cs_quantity else 0 end) as dec_net from catalog_sales ,warehouse ,date_dim ,time_dim ,ship_mode where cs_warehouse_sk = w_warehouse_sk and cs_sold_date_sk = d_date_sk and cs_sold_time_sk = t_time_sk and cs_ship_mode_sk = sm_ship_mode_sk and d_year = 1998 and t_time between 16224 AND 16224+28800 and sm_carrier in ('LATVIAN','ALLIANCE') group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,d_year ) x group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,ship_carriers ,year order by w_warehouse_name limit 100""", "q67" -> """ select * from (select i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sumsales ,rank() over (partition by i_category order by sumsales desc) rk from (select i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sum(coalesce(ss_sales_price*ss_quantity,0)) sumsales from store_sales ,date_dim ,store ,item where ss_sold_date_sk=d_date_sk and ss_item_sk=i_item_sk and ss_store_sk = s_store_sk and d_month_seq between 1203 and 1203+11 group by rollup(i_category, i_class, i_brand, i_product_name, d_year, d_qoy, d_moy,s_store_id))dw1) dw2 where rk <= 100 order by i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sumsales ,rk limit 100""", "q68" -> """ select c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number ,extended_price ,extended_tax ,list_price from (select ss_ticket_number ,ss_customer_sk ,ca_city bought_city ,sum(ss_ext_sales_price) extended_price ,sum(ss_ext_list_price) list_price ,sum(ss_ext_tax) extended_tax from store_sales ,date_dim ,store ,household_demographics ,customer_address where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and store_sales.ss_addr_sk = customer_address.ca_address_sk and date_dim.d_dom between 1 and 2 and (household_demographics.hd_dep_count = 3 or household_demographics.hd_vehicle_count= -1) and date_dim.d_year in (1999,1999+1,1999+2) and store.s_city in ('Jamestown','Pine Hill') group by ss_ticket_number ,ss_customer_sk ,ss_addr_sk,ca_city) dn ,customer ,customer_address current_addr where ss_customer_sk = c_customer_sk and customer.c_current_addr_sk = current_addr.ca_address_sk and current_addr.ca_city <> bought_city order by c_last_name ,ss_ticket_number limit 100""", "q69" -> """ select cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3 from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and ca_state in ('CA','MT','SD') and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2003 and d_moy between 2 and 2+2) and (not exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2003 and d_moy between 2 and 2+2) and not exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2003 and d_moy between 2 and 2+2)) group by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating order by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating limit 100""", "q70" -> """ select sum(ss_net_profit) as total_sum ,s_state ,s_county ,grouping(s_state)+grouping(s_county) as lochierarchy ,rank() over ( partition by grouping(s_state)+grouping(s_county), case when grouping(s_county) = 0 then s_state end order by sum(ss_net_profit) desc) as rank_within_parent from store_sales ,date_dim d1 ,store where d1.d_month_seq between 1215 and 1215+11 and d1.d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk and s_state in ( select s_state from (select s_state as s_state, rank() over ( partition by s_state order by sum(ss_net_profit) desc) as ranking from store_sales, store, date_dim where d_month_seq between 1215 and 1215+11 and d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk group by s_state ) tmp1 where ranking <= 5 ) group by rollup(s_state,s_county) order by lochierarchy desc ,case when lochierarchy = 0 then s_state end ,rank_within_parent limit 100""", "q71" -> """ select i_brand_id brand_id, i_brand brand,t_hour,t_minute, sum(ext_price) ext_price from item, (select ws_ext_sales_price as ext_price, ws_sold_date_sk as sold_date_sk, ws_item_sk as sold_item_sk, ws_sold_time_sk as time_sk from web_sales,date_dim where d_date_sk = ws_sold_date_sk and d_moy=11 and d_year=1998 union all select cs_ext_sales_price as ext_price, cs_sold_date_sk as sold_date_sk, cs_item_sk as sold_item_sk, cs_sold_time_sk as time_sk from catalog_sales,date_dim where d_date_sk = cs_sold_date_sk and d_moy=11 and d_year=1998 union all select ss_ext_sales_price as ext_price, ss_sold_date_sk as sold_date_sk, ss_item_sk as sold_item_sk, ss_sold_time_sk as time_sk from store_sales,date_dim where d_date_sk = ss_sold_date_sk and d_moy=11 and d_year=1998 ) tmp,time_dim where sold_item_sk = i_item_sk and i_manager_id=1 and time_sk = t_time_sk and (t_meal_time = 'breakfast' or t_meal_time = 'dinner') group by i_brand, i_brand_id,t_hour,t_minute order by ext_price desc, i_brand_id """, "q72" -> """ select i_item_desc ,w_warehouse_name ,d1.d_week_seq ,sum(case when p_promo_sk is null then 1 else 0 end) no_promo ,sum(case when p_promo_sk is not null then 1 else 0 end) promo ,count(*) total_cnt from catalog_sales join inventory on (cs_item_sk = inv_item_sk) join warehouse on (w_warehouse_sk=inv_warehouse_sk) join item on (i_item_sk = cs_item_sk) join customer_demographics on (cs_bill_cdemo_sk = cd_demo_sk) join household_demographics on (cs_bill_hdemo_sk = hd_demo_sk) join date_dim d1 on (cs_sold_date_sk = d1.d_date_sk) join date_dim d2 on (inv_date_sk = d2.d_date_sk) join date_dim d3 on (cs_ship_date_sk = d3.d_date_sk) left outer join promotion on (cs_promo_sk=p_promo_sk) left outer join catalog_returns on (cr_item_sk = cs_item_sk and cr_order_number = cs_order_number) where d1.d_week_seq = d2.d_week_seq and inv_quantity_on_hand < cs_quantity and d3.d_date > d1.d_date + interval 5 days and hd_buy_potential = '1001-5000' and d1.d_year = 1998 and cd_marital_status = 'S' group by i_item_desc,w_warehouse_name,d1.d_week_seq order by total_cnt desc, i_item_desc, w_warehouse_name, d_week_seq limit 100""", "q73" -> """ select c_last_name ,c_first_name ,c_salutation ,c_preferred_cust_flag ,ss_ticket_number ,cnt from (select ss_ticket_number ,ss_customer_sk ,count(*) cnt from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and date_dim.d_dom between 1 and 2 and (household_demographics.hd_buy_potential = '>10000' or household_demographics.hd_buy_potential = 'Unknown') and household_demographics.hd_vehicle_count > 0 and case when household_demographics.hd_vehicle_count > 0 then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end > 1 and date_dim.d_year in (1998,1998+1,1998+2) and store.s_county in ('Van Buren County','Terrell County','Belknap County','Kootenai County') group by ss_ticket_number,ss_customer_sk) dj,customer where ss_customer_sk = c_customer_sk and cnt between 1 and 5 order by cnt desc, c_last_name asc""", "q74" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,d_year as year ,max(ss_net_paid) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (2001,2001+1) group by c_customer_id ,c_first_name ,c_last_name ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,d_year as year ,max(ws_net_paid) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year in (2001,2001+1) group by c_customer_id ,c_first_name ,c_last_name ,d_year ) select t_s_secyear.customer_id, t_s_secyear.customer_first_name, t_s_secyear.customer_last_name from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.sale_type = 's' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_w_secyear.sale_type = 'w' and t_s_firstyear.year = 2001 and t_s_secyear.year = 2001+1 and t_w_firstyear.year = 2001 and t_w_secyear.year = 2001+1 and t_s_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end order by 2,3,1 limit 100""", "q75" -> """ WITH all_sales AS ( SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,SUM(sales_cnt) AS sales_cnt ,SUM(sales_amt) AS sales_amt FROM (SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,cs_quantity - COALESCE(cr_return_quantity,0) AS sales_cnt ,cs_ext_sales_price - COALESCE(cr_return_amount,0.0) AS sales_amt FROM catalog_sales JOIN item ON i_item_sk=cs_item_sk JOIN date_dim ON d_date_sk=cs_sold_date_sk LEFT JOIN catalog_returns ON (cs_order_number=cr_order_number AND cs_item_sk=cr_item_sk) WHERE i_category='Music' UNION SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,ss_quantity - COALESCE(sr_return_quantity,0) AS sales_cnt ,ss_ext_sales_price - COALESCE(sr_return_amt,0.0) AS sales_amt FROM store_sales JOIN item ON i_item_sk=ss_item_sk JOIN date_dim ON d_date_sk=ss_sold_date_sk LEFT JOIN store_returns ON (ss_ticket_number=sr_ticket_number AND ss_item_sk=sr_item_sk) WHERE i_category='Music' UNION SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,ws_quantity - COALESCE(wr_return_quantity,0) AS sales_cnt ,ws_ext_sales_price - COALESCE(wr_return_amt,0.0) AS sales_amt FROM web_sales JOIN item ON i_item_sk=ws_item_sk JOIN date_dim ON d_date_sk=ws_sold_date_sk LEFT JOIN web_returns ON (ws_order_number=wr_order_number AND ws_item_sk=wr_item_sk) WHERE i_category='Music') sales_detail GROUP BY d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id) SELECT prev_yr.d_year AS prev_year ,curr_yr.d_year AS year ,curr_yr.i_brand_id ,curr_yr.i_class_id ,curr_yr.i_category_id ,curr_yr.i_manufact_id ,prev_yr.sales_cnt AS prev_yr_cnt ,curr_yr.sales_cnt AS curr_yr_cnt ,curr_yr.sales_cnt-prev_yr.sales_cnt AS sales_cnt_diff ,curr_yr.sales_amt-prev_yr.sales_amt AS sales_amt_diff FROM all_sales curr_yr, all_sales prev_yr WHERE curr_yr.i_brand_id=prev_yr.i_brand_id AND curr_yr.i_class_id=prev_yr.i_class_id AND curr_yr.i_category_id=prev_yr.i_category_id AND curr_yr.i_manufact_id=prev_yr.i_manufact_id AND curr_yr.d_year=2001 AND prev_yr.d_year=2001-1 AND CAST(curr_yr.sales_cnt AS DECIMAL(17,2))/CAST(prev_yr.sales_cnt AS DECIMAL(17,2))<0.9 ORDER BY sales_cnt_diff,sales_amt_diff limit 100""", "q76" -> """ select channel, col_name, d_year, d_qoy, i_category, COUNT(*) sales_cnt, SUM(ext_sales_price) sales_amt FROM ( SELECT 'store' as channel, 'ss_promo_sk' col_name, d_year, d_qoy, i_category, ss_ext_sales_price ext_sales_price FROM store_sales, item, date_dim WHERE ss_promo_sk IS NULL AND ss_sold_date_sk=d_date_sk AND ss_item_sk=i_item_sk UNION ALL SELECT 'web' as channel, 'ws_web_site_sk' col_name, d_year, d_qoy, i_category, ws_ext_sales_price ext_sales_price FROM web_sales, item, date_dim WHERE ws_web_site_sk IS NULL AND ws_sold_date_sk=d_date_sk AND ws_item_sk=i_item_sk UNION ALL SELECT 'catalog' as channel, 'cs_bill_addr_sk' col_name, d_year, d_qoy, i_category, cs_ext_sales_price ext_sales_price FROM catalog_sales, item, date_dim WHERE cs_bill_addr_sk IS NULL AND cs_sold_date_sk=d_date_sk AND cs_item_sk=i_item_sk) foo GROUP BY channel, col_name, d_year, d_qoy, i_category ORDER BY channel, col_name, d_year, d_qoy, i_category limit 100""", "q77" -> """ with ss as (select s_store_sk, sum(ss_ext_sales_price) as sales, sum(ss_net_profit) as profit from store_sales, date_dim, store where ss_sold_date_sk = d_date_sk and d_date between cast('2001-08-22' as date) and (cast('2001-08-22' as date) + INTERVAL 30 days) and ss_store_sk = s_store_sk group by s_store_sk) , sr as (select s_store_sk, sum(sr_return_amt) as returns, sum(sr_net_loss) as profit_loss from store_returns, date_dim, store where sr_returned_date_sk = d_date_sk and d_date between cast('2001-08-22' as date) and (cast('2001-08-22' as date) + INTERVAL 30 days) and sr_store_sk = s_store_sk group by s_store_sk), cs as (select cs_call_center_sk, sum(cs_ext_sales_price) as sales, sum(cs_net_profit) as profit from catalog_sales, date_dim where cs_sold_date_sk = d_date_sk and d_date between cast('2001-08-22' as date) and (cast('2001-08-22' as date) + INTERVAL 30 days) group by cs_call_center_sk ), cr as (select cr_call_center_sk, sum(cr_return_amount) as returns, sum(cr_net_loss) as profit_loss from catalog_returns, date_dim where cr_returned_date_sk = d_date_sk and d_date between cast('2001-08-22' as date) and (cast('2001-08-22' as date) + INTERVAL 30 days) group by cr_call_center_sk ), ws as ( select wp_web_page_sk, sum(ws_ext_sales_price) as sales, sum(ws_net_profit) as profit from web_sales, date_dim, web_page where ws_sold_date_sk = d_date_sk and d_date between cast('2001-08-22' as date) and (cast('2001-08-22' as date) + INTERVAL 30 days) and ws_web_page_sk = wp_web_page_sk group by wp_web_page_sk), wr as (select wp_web_page_sk, sum(wr_return_amt) as returns, sum(wr_net_loss) as profit_loss from web_returns, date_dim, web_page where wr_returned_date_sk = d_date_sk and d_date between cast('2001-08-22' as date) and (cast('2001-08-22' as date) + INTERVAL 30 days) and wr_web_page_sk = wp_web_page_sk group by wp_web_page_sk) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , ss.s_store_sk as id , sales , coalesce(returns, 0) as returns , (profit - coalesce(profit_loss,0)) as profit from ss left join sr on ss.s_store_sk = sr.s_store_sk union all select 'catalog channel' as channel , cs_call_center_sk as id , sales , returns , (profit - profit_loss) as profit from cs , cr union all select 'web channel' as channel , ws.wp_web_page_sk as id , sales , coalesce(returns, 0) returns , (profit - coalesce(profit_loss,0)) as profit from ws left join wr on ws.wp_web_page_sk = wr.wp_web_page_sk ) x group by rollup (channel, id) order by channel ,id limit 100""", "q78" -> """ with ws as (select d_year AS ws_sold_year, ws_item_sk, ws_bill_customer_sk ws_customer_sk, sum(ws_quantity) ws_qty, sum(ws_wholesale_cost) ws_wc, sum(ws_sales_price) ws_sp from web_sales left join web_returns on wr_order_number=ws_order_number and ws_item_sk=wr_item_sk join date_dim on ws_sold_date_sk = d_date_sk where wr_order_number is null group by d_year, ws_item_sk, ws_bill_customer_sk ), cs as (select d_year AS cs_sold_year, cs_item_sk, cs_bill_customer_sk cs_customer_sk, sum(cs_quantity) cs_qty, sum(cs_wholesale_cost) cs_wc, sum(cs_sales_price) cs_sp from catalog_sales left join catalog_returns on cr_order_number=cs_order_number and cs_item_sk=cr_item_sk join date_dim on cs_sold_date_sk = d_date_sk where cr_order_number is null group by d_year, cs_item_sk, cs_bill_customer_sk ), ss as (select d_year AS ss_sold_year, ss_item_sk, ss_customer_sk, sum(ss_quantity) ss_qty, sum(ss_wholesale_cost) ss_wc, sum(ss_sales_price) ss_sp from store_sales left join store_returns on sr_ticket_number=ss_ticket_number and ss_item_sk=sr_item_sk join date_dim on ss_sold_date_sk = d_date_sk where sr_ticket_number is null group by d_year, ss_item_sk, ss_customer_sk ) select ss_sold_year, ss_item_sk, ss_customer_sk, round(ss_qty/(coalesce(ws_qty,0)+coalesce(cs_qty,0)),2) ratio, ss_qty store_qty, ss_wc store_wholesale_cost, ss_sp store_sales_price, coalesce(ws_qty,0)+coalesce(cs_qty,0) other_chan_qty, coalesce(ws_wc,0)+coalesce(cs_wc,0) other_chan_wholesale_cost, coalesce(ws_sp,0)+coalesce(cs_sp,0) other_chan_sales_price from ss left join ws on (ws_sold_year=ss_sold_year and ws_item_sk=ss_item_sk and ws_customer_sk=ss_customer_sk) left join cs on (cs_sold_year=ss_sold_year and cs_item_sk=ss_item_sk and cs_customer_sk=ss_customer_sk) where (coalesce(ws_qty,0)>0 or coalesce(cs_qty, 0)>0) and ss_sold_year=2001 order by ss_sold_year, ss_item_sk, ss_customer_sk, ss_qty desc, ss_wc desc, ss_sp desc, other_chan_qty, other_chan_wholesale_cost, other_chan_sales_price, ratio limit 100""", "q79" -> """ select c_last_name,c_first_name,substr(s_city,1,30),ss_ticket_number,amt,profit from (select ss_ticket_number ,ss_customer_sk ,store.s_city ,sum(ss_coupon_amt) amt ,sum(ss_net_profit) profit from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and (household_demographics.hd_dep_count = 6 or household_demographics.hd_vehicle_count > -1) and date_dim.d_dow = 1 and date_dim.d_year in (1998,1998+1,1998+2) and store.s_number_employees between 200 and 295 group by ss_ticket_number,ss_customer_sk,ss_addr_sk,store.s_city) ms,customer where ss_customer_sk = c_customer_sk order by c_last_name,c_first_name,substr(s_city,1,30), profit limit 100""", "q80" -> """ with ssr as (select s_store_id as store_id, sum(ss_ext_sales_price) as sales, sum(coalesce(sr_return_amt, 0)) as returns, sum(ss_net_profit - coalesce(sr_net_loss, 0)) as profit from store_sales left outer join store_returns on (ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number), date_dim, store, item, promotion where ss_sold_date_sk = d_date_sk and d_date between cast('2000-08-25' as date) and (cast('2000-08-25' as date) + INTERVAL 60 days) and ss_store_sk = s_store_sk and ss_item_sk = i_item_sk and i_current_price > 50 and ss_promo_sk = p_promo_sk and p_channel_tv = 'N' group by s_store_id) , csr as (select cp_catalog_page_id as catalog_page_id, sum(cs_ext_sales_price) as sales, sum(coalesce(cr_return_amount, 0)) as returns, sum(cs_net_profit - coalesce(cr_net_loss, 0)) as profit from catalog_sales left outer join catalog_returns on (cs_item_sk = cr_item_sk and cs_order_number = cr_order_number), date_dim, catalog_page, item, promotion where cs_sold_date_sk = d_date_sk and d_date between cast('2000-08-25' as date) and (cast('2000-08-25' as date) + INTERVAL 60 days) and cs_catalog_page_sk = cp_catalog_page_sk and cs_item_sk = i_item_sk and i_current_price > 50 and cs_promo_sk = p_promo_sk and p_channel_tv = 'N' group by cp_catalog_page_id) , wsr as (select web_site_id, sum(ws_ext_sales_price) as sales, sum(coalesce(wr_return_amt, 0)) as returns, sum(ws_net_profit - coalesce(wr_net_loss, 0)) as profit from web_sales left outer join web_returns on (ws_item_sk = wr_item_sk and ws_order_number = wr_order_number), date_dim, web_site, item, promotion where ws_sold_date_sk = d_date_sk and d_date between cast('2000-08-25' as date) and (cast('2000-08-25' as date) + INTERVAL 60 days) and ws_web_site_sk = web_site_sk and ws_item_sk = i_item_sk and i_current_price > 50 and ws_promo_sk = p_promo_sk and p_channel_tv = 'N' group by web_site_id) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , 'store' || store_id as id , sales , returns , profit from ssr union all select 'catalog channel' as channel , 'catalog_page' || catalog_page_id as id , sales , returns , profit from csr union all select 'web channel' as channel , 'web_site' || web_site_id as id , sales , returns , profit from wsr ) x group by rollup (channel, id) order by channel ,id limit 100""", "q81" -> """ with customer_total_return as (select cr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, sum(cr_return_amt_inc_tax) as ctr_total_return from catalog_returns ,date_dim ,customer_address where cr_returned_date_sk = d_date_sk and d_year =2000 and cr_returning_addr_sk = ca_address_sk group by cr_returning_customer_sk ,ca_state ) select c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset ,ca_location_type,ctr_total_return from customer_total_return ctr1 ,customer_address ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_state = ctr2.ctr_state) and ca_address_sk = c_current_addr_sk and ca_state = 'SC' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset ,ca_location_type,ctr_total_return limit 100""", "q82" -> """ select i_item_id ,i_item_desc ,i_current_price from item, inventory, date_dim, store_sales where i_current_price between 6 and 6+30 and inv_item_sk = i_item_sk and d_date_sk=inv_date_sk and d_date between cast('2001-02-23' as date) and (cast('2001-02-23' as date) + INTERVAL 60 days) and i_manufact_id in (669,623,578,379) and inv_quantity_on_hand between 100 and 500 and ss_item_sk = i_item_sk group by i_item_id,i_item_desc,i_current_price order by i_item_id limit 100""", "q83" -> """ with sr_items as (select i_item_id item_id, sum(sr_return_quantity) sr_item_qty from store_returns, item, date_dim where sr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('2001-01-15','2001-09-03','2001-11-17'))) and sr_returned_date_sk = d_date_sk group by i_item_id), cr_items as (select i_item_id item_id, sum(cr_return_quantity) cr_item_qty from catalog_returns, item, date_dim where cr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('2001-01-15','2001-09-03','2001-11-17'))) and cr_returned_date_sk = d_date_sk group by i_item_id), wr_items as (select i_item_id item_id, sum(wr_return_quantity) wr_item_qty from web_returns, item, date_dim where wr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('2001-01-15','2001-09-03','2001-11-17'))) and wr_returned_date_sk = d_date_sk group by i_item_id) select sr_items.item_id ,sr_item_qty ,sr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 sr_dev ,cr_item_qty ,cr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 cr_dev ,wr_item_qty ,wr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 wr_dev ,(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 average from sr_items ,cr_items ,wr_items where sr_items.item_id=cr_items.item_id and sr_items.item_id=wr_items.item_id order by sr_items.item_id ,sr_item_qty limit 100""", "q84" -> """ select c_customer_id as customer_id , coalesce(c_last_name,'') || ', ' || coalesce(c_first_name,'') as customername from customer ,customer_address ,customer_demographics ,household_demographics ,income_band ,store_returns where ca_city = 'Walnut Grove' and c_current_addr_sk = ca_address_sk and ib_lower_bound >= 53669 and ib_upper_bound <= 53669 + 50000 and ib_income_band_sk = hd_income_band_sk and cd_demo_sk = c_current_cdemo_sk and hd_demo_sk = c_current_hdemo_sk and sr_cdemo_sk = cd_demo_sk order by c_customer_id limit 100""", "q85" -> """ select substr(r_reason_desc,1,20) ,avg(ws_quantity) ,avg(wr_refunded_cash) ,avg(wr_fee) from web_sales, web_returns, web_page, customer_demographics cd1, customer_demographics cd2, customer_address, date_dim, reason where ws_web_page_sk = wp_web_page_sk and ws_item_sk = wr_item_sk and ws_order_number = wr_order_number and ws_sold_date_sk = d_date_sk and d_year = 2001 and cd1.cd_demo_sk = wr_refunded_cdemo_sk and cd2.cd_demo_sk = wr_returning_cdemo_sk and ca_address_sk = wr_refunded_addr_sk and r_reason_sk = wr_reason_sk and ( ( cd1.cd_marital_status = 'S' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = 'Secondary' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 100.00 and 150.00 ) or ( cd1.cd_marital_status = 'D' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = 'Advanced Degree' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 50.00 and 100.00 ) or ( cd1.cd_marital_status = 'W' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = 'Primary' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 150.00 and 200.00 ) ) and ( ( ca_country = 'United States' and ca_state in ('AZ', 'SD', 'TN') and ws_net_profit between 100 and 200 ) or ( ca_country = 'United States' and ca_state in ('TX', 'GA', 'IA') and ws_net_profit between 150 and 300 ) or ( ca_country = 'United States' and ca_state in ('WI', 'VT', 'AL') and ws_net_profit between 50 and 250 ) ) group by r_reason_desc order by substr(r_reason_desc,1,20) ,avg(ws_quantity) ,avg(wr_refunded_cash) ,avg(wr_fee) limit 100""", "q86" -> """ select sum(ws_net_paid) as total_sum ,i_category ,i_class ,grouping(i_category)+grouping(i_class) as lochierarchy ,rank() over ( partition by grouping(i_category)+grouping(i_class), case when grouping(i_class) = 0 then i_category end order by sum(ws_net_paid) desc) as rank_within_parent from web_sales ,date_dim d1 ,item where d1.d_month_seq between 1195 and 1195+11 and d1.d_date_sk = ws_sold_date_sk and i_item_sk = ws_item_sk group by rollup(i_category,i_class) order by lochierarchy desc, case when lochierarchy = 0 then i_category end, rank_within_parent limit 100""", "q87" -> """ select count(*) from ((select distinct c_last_name, c_first_name, d_date from store_sales, date_dim, customer where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_customer_sk = customer.c_customer_sk and d_month_seq between 1194 and 1194+11) except (select distinct c_last_name, c_first_name, d_date from catalog_sales, date_dim, customer where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1194 and 1194+11) except (select distinct c_last_name, c_first_name, d_date from web_sales, date_dim, customer where web_sales.ws_sold_date_sk = date_dim.d_date_sk and web_sales.ws_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1194 and 1194+11) ) cool_cust""", "q88" -> """ select * from (select count(*) h8_30_to_9 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 8 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2)) and store.s_store_name = 'ese') s1, (select count(*) h9_to_9_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 9 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2)) and store.s_store_name = 'ese') s2, (select count(*) h9_30_to_10 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 9 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2)) and store.s_store_name = 'ese') s3, (select count(*) h10_to_10_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 10 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2)) and store.s_store_name = 'ese') s4, (select count(*) h10_30_to_11 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 10 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2)) and store.s_store_name = 'ese') s5, (select count(*) h11_to_11_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 11 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2)) and store.s_store_name = 'ese') s6, (select count(*) h11_30_to_12 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 11 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2)) and store.s_store_name = 'ese') s7, (select count(*) h12_to_12_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 12 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2)) and store.s_store_name = 'ese') s8""", "q89" -> """ select * from( select i_category, i_class, i_brand, s_store_name, s_company_name, d_moy, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_category, i_brand, s_store_name, s_company_name) avg_monthly_sales from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_year in (2000) and ((i_category in ('Home','Shoes','Electronics') and i_class in ('flatware','mens','televisions') ) or (i_category in ('Women','Sports','Music') and i_class in ('maternity','camping','rock') )) group by i_category, i_class, i_brand, s_store_name, s_company_name, d_moy) tmp1 where case when (avg_monthly_sales <> 0) then (abs(sum_sales - avg_monthly_sales) / avg_monthly_sales) else null end > 0.1 order by sum_sales - avg_monthly_sales, s_store_name limit 100""", "q90" -> """ select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio from ( select count(*) amc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 8 and 8+1 and household_demographics.hd_dep_count = 4 and web_page.wp_char_count between 5000 and 5200) at, ( select count(*) pmc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 20 and 20+1 and household_demographics.hd_dep_count = 4 and web_page.wp_char_count between 5000 and 5200) pt order by am_pm_ratio limit 100""", "q91" -> """ select cc_call_center_id Call_Center, cc_name Call_Center_Name, cc_manager Manager, sum(cr_net_loss) Returns_Loss from call_center, catalog_returns, date_dim, customer, customer_address, customer_demographics, household_demographics where cr_call_center_sk = cc_call_center_sk and cr_returned_date_sk = d_date_sk and cr_returning_customer_sk= c_customer_sk and cd_demo_sk = c_current_cdemo_sk and hd_demo_sk = c_current_hdemo_sk and ca_address_sk = c_current_addr_sk and d_year = 2001 and d_moy = 12 and ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') or(cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) and hd_buy_potential like 'Unknown%' and ca_gmt_offset = -6 group by cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status order by sum(cr_net_loss) desc""", "q92" -> """ select sum(ws_ext_discount_amt) as `Excess Discount Amount` from web_sales ,item ,date_dim where i_manufact_id = 7 and i_item_sk = ws_item_sk and d_date between '2000-01-16' and (cast('2000-01-16' as date) + INTERVAL 90 days) and d_date_sk = ws_sold_date_sk and ws_ext_discount_amt > ( SELECT 1.3 * avg(ws_ext_discount_amt) FROM web_sales ,date_dim WHERE ws_item_sk = i_item_sk and d_date between '2000-01-16' and (cast('2000-01-16' as date) + INTERVAL 90 days) and d_date_sk = ws_sold_date_sk ) order by sum(ws_ext_discount_amt) limit 100""", "q93" -> """ select ss_customer_sk ,sum(act_sales) sumsales from (select ss_item_sk ,ss_ticket_number ,ss_customer_sk ,case when sr_return_quantity is not null then (ss_quantity-sr_return_quantity)*ss_sales_price else (ss_quantity*ss_sales_price) end act_sales from store_sales left outer join store_returns on (sr_item_sk = ss_item_sk and sr_ticket_number = ss_ticket_number) ,reason where sr_reason_sk = r_reason_sk and r_reason_desc = 'reason 24') t group by ss_customer_sk order by sumsales, ss_customer_sk limit 100""", "q94" -> """ select count(distinct ws_order_number) as `order count` ,sum(ws_ext_ship_cost) as `total shipping cost` ,sum(ws_net_profit) as `total net profit` from web_sales ws1 ,date_dim ,customer_address ,web_site where d_date between '2001-2-01' and (cast('2001-2-01' as date) + INTERVAL 60 days) and ws1.ws_ship_date_sk = d_date_sk and ws1.ws_ship_addr_sk = ca_address_sk and ca_state = 'VT' and ws1.ws_web_site_sk = web_site_sk and web_company_name = 'pri' and exists (select * from web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) and not exists(select * from web_returns wr1 where ws1.ws_order_number = wr1.wr_order_number) order by count(distinct ws_order_number) limit 100""", "q95" -> """ with ws_wh as (select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 from web_sales ws1,web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) select count(distinct ws_order_number) as `order count` ,sum(ws_ext_ship_cost) as `total shipping cost` ,sum(ws_net_profit) as `total net profit` from web_sales ws1 ,date_dim ,customer_address ,web_site where d_date between '2001-3-01' and (cast('2001-3-01' as date) + INTERVAL 60 days) and ws1.ws_ship_date_sk = d_date_sk and ws1.ws_ship_addr_sk = ca_address_sk and ca_state = 'TN' and ws1.ws_web_site_sk = web_site_sk and web_company_name = 'pri' and ws1.ws_order_number in (select ws_order_number from ws_wh) and ws1.ws_order_number in (select wr_order_number from web_returns,ws_wh where wr_order_number = ws_wh.ws_order_number) order by count(distinct ws_order_number) limit 100""", "q96" -> """ select count(*) from store_sales ,household_demographics ,time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 20 and time_dim.t_minute >= 30 and household_demographics.hd_dep_count = 6 and store.s_store_name = 'ese' order by count(*) limit 100""", "q97" -> """ with ssci as ( select ss_customer_sk customer_sk ,ss_item_sk item_sk from store_sales,date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1206 and 1206 + 11 group by ss_customer_sk ,ss_item_sk), csci as( select cs_bill_customer_sk customer_sk ,cs_item_sk item_sk from catalog_sales,date_dim where cs_sold_date_sk = d_date_sk and d_month_seq between 1206 and 1206 + 11 group by cs_bill_customer_sk ,cs_item_sk) select sum(case when ssci.customer_sk is not null and csci.customer_sk is null then 1 else 0 end) store_only ,sum(case when ssci.customer_sk is null and csci.customer_sk is not null then 1 else 0 end) catalog_only ,sum(case when ssci.customer_sk is not null and csci.customer_sk is not null then 1 else 0 end) store_and_catalog from ssci full outer join csci on (ssci.customer_sk=csci.customer_sk and ssci.item_sk = csci.item_sk) limit 100""", "q98" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(ss_ext_sales_price) as itemrevenue ,sum(ss_ext_sales_price)*100/sum(sum(ss_ext_sales_price)) over (partition by i_class) as revenueratio from store_sales ,item ,date_dim where ss_item_sk = i_item_sk and i_category in ('Sports', 'Books', 'Electronics') and ss_sold_date_sk = d_date_sk and d_date between cast('2002-06-29' as date) and (cast('2002-06-29' as date) + interval 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio""", "q99" -> """ select substr(w_warehouse_name,1,20) ,sm_type ,cc_name ,sum(case when (cs_ship_date_sk - cs_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 30) and (cs_ship_date_sk - cs_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 60) and (cs_ship_date_sk - cs_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 90) and (cs_ship_date_sk - cs_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from catalog_sales ,warehouse ,ship_mode ,call_center ,date_dim where d_month_seq between 1199 and 1199 + 11 and cs_ship_date_sk = d_date_sk and cs_warehouse_sk = w_warehouse_sk and cs_ship_mode_sk = sm_ship_mode_sk and cs_call_center_sk = cc_call_center_sk group by substr(w_warehouse_name,1,20) ,sm_type ,cc_name order by substr(w_warehouse_name,1,20) ,sm_type ,cc_name limit 100""", "q1" -> """ with customer_total_return as (select sr_customer_sk as ctr_customer_sk ,sr_store_sk as ctr_store_sk ,sum(SR_FEE) as ctr_total_return from store_returns ,date_dim where sr_returned_date_sk = d_date_sk and d_year =2000 group by sr_customer_sk ,sr_store_sk) select c_customer_id from customer_total_return ctr1 ,store ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_store_sk = ctr2.ctr_store_sk) and s_store_sk = ctr1.ctr_store_sk and s_state = 'MO' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id limit 100""", "q2" -> """ with wscs as (select sold_date_sk ,sales_price from (select ws_sold_date_sk sold_date_sk ,ws_ext_sales_price sales_price from web_sales union all select cs_sold_date_sk sold_date_sk ,cs_ext_sales_price sales_price from catalog_sales)), wswscs as (select d_week_seq, sum(case when (d_day_name='Sunday') then sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then sales_price else null end) sat_sales from wscs ,date_dim where d_date_sk = sold_date_sk group by d_week_seq) select d_week_seq1 ,round(sun_sales1/sun_sales2,2) ,round(mon_sales1/mon_sales2,2) ,round(tue_sales1/tue_sales2,2) ,round(wed_sales1/wed_sales2,2) ,round(thu_sales1/thu_sales2,2) ,round(fri_sales1/fri_sales2,2) ,round(sat_sales1/sat_sales2,2) from (select wswscs.d_week_seq d_week_seq1 ,sun_sales sun_sales1 ,mon_sales mon_sales1 ,tue_sales tue_sales1 ,wed_sales wed_sales1 ,thu_sales thu_sales1 ,fri_sales fri_sales1 ,sat_sales sat_sales1 from wswscs,date_dim where date_dim.d_week_seq = wswscs.d_week_seq and d_year = 1998) y, (select wswscs.d_week_seq d_week_seq2 ,sun_sales sun_sales2 ,mon_sales mon_sales2 ,tue_sales tue_sales2 ,wed_sales wed_sales2 ,thu_sales thu_sales2 ,fri_sales fri_sales2 ,sat_sales sat_sales2 from wswscs ,date_dim where date_dim.d_week_seq = wswscs.d_week_seq and d_year = 1998+1) z where d_week_seq1=d_week_seq2-53 order by d_week_seq1""", "q3" -> """ select dt.d_year ,item.i_brand_id brand_id ,item.i_brand brand ,sum(ss_sales_price) sum_agg from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manufact_id = 816 and dt.d_moy=11 group by dt.d_year ,item.i_brand ,item.i_brand_id order by dt.d_year ,sum_agg desc ,brand_id limit 100""", "q4" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum((((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2) ) year_total ,'c' sale_type from customer ,catalog_sales ,date_dim where c_customer_sk = cs_bill_customer_sk and cs_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum((((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2) ) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year ) select t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_birth_country from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_c_firstyear ,year_total t_c_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_c_secyear.customer_id and t_s_firstyear.customer_id = t_c_firstyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.sale_type = 's' and t_c_firstyear.sale_type = 'c' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_c_secyear.sale_type = 'c' and t_w_secyear.sale_type = 'w' and t_s_firstyear.dyear = 1999 and t_s_secyear.dyear = 1999+1 and t_c_firstyear.dyear = 1999 and t_c_secyear.dyear = 1999+1 and t_w_firstyear.dyear = 1999 and t_w_secyear.dyear = 1999+1 and t_s_firstyear.year_total > 0 and t_c_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end > case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end order by t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_birth_country limit 100""", "q5" -> """ with ssr as (select s_store_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select ss_store_sk as store_sk, ss_sold_date_sk as date_sk, ss_ext_sales_price as sales_price, ss_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from store_sales union all select sr_store_sk as store_sk, sr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, sr_return_amt as return_amt, sr_net_loss as net_loss from store_returns ) salesreturns, date_dim, store where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and store_sk = s_store_sk group by s_store_id) , csr as (select cp_catalog_page_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select cs_catalog_page_sk as page_sk, cs_sold_date_sk as date_sk, cs_ext_sales_price as sales_price, cs_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from catalog_sales union all select cr_catalog_page_sk as page_sk, cr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, cr_return_amount as return_amt, cr_net_loss as net_loss from catalog_returns ) salesreturns, date_dim, catalog_page where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and page_sk = cp_catalog_page_sk group by cp_catalog_page_id) , wsr as (select web_site_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select ws_web_site_sk as wsr_web_site_sk, ws_sold_date_sk as date_sk, ws_ext_sales_price as sales_price, ws_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from web_sales union all select ws_web_site_sk as wsr_web_site_sk, wr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, wr_return_amt as return_amt, wr_net_loss as net_loss from web_returns left outer join web_sales on ( wr_item_sk = ws_item_sk and wr_order_number = ws_order_number) ) salesreturns, date_dim, web_site where date_sk = d_date_sk and d_date between cast('2000-08-19' as date) and (cast('2000-08-19' as date) + INTERVAL 14 days) and wsr_web_site_sk = web_site_sk group by web_site_id) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , 'store' || s_store_id as id , sales , returns , (profit - profit_loss) as profit from ssr union all select 'catalog channel' as channel , 'catalog_page' || cp_catalog_page_id as id , sales , returns , (profit - profit_loss) as profit from csr union all select 'web channel' as channel , 'web_site' || web_site_id as id , sales , returns , (profit - profit_loss) as profit from wsr ) x group by rollup (channel, id) order by channel ,id limit 100""", "q6" -> """ select a.ca_state state, count(*) cnt from customer_address a ,customer c ,store_sales s ,date_dim d ,item i where a.ca_address_sk = c.c_current_addr_sk and c.c_customer_sk = s.ss_customer_sk and s.ss_sold_date_sk = d.d_date_sk and s.ss_item_sk = i.i_item_sk and d.d_month_seq = (select distinct (d_month_seq) from date_dim where d_year = 2002 and d_moy = 3 ) and i.i_current_price > 1.2 * (select avg(j.i_current_price) from item j where j.i_category = i.i_category) group by a.ca_state having count(*) >= 10 order by cnt, a.ca_state limit 100""", "q7" -> """ select i_item_id, avg(ss_quantity) agg1, avg(ss_list_price) agg2, avg(ss_coupon_amt) agg3, avg(ss_sales_price) agg4 from store_sales, customer_demographics, date_dim, item, promotion where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and ss_cdemo_sk = cd_demo_sk and ss_promo_sk = p_promo_sk and cd_gender = 'F' and cd_marital_status = 'W' and cd_education_status = 'College' and (p_channel_email = 'N' or p_channel_event = 'N') and d_year = 2001 group by i_item_id order by i_item_id limit 100""", "q8" -> """ select s_store_name ,sum(ss_net_profit) from store_sales ,date_dim ,store, (select ca_zip from ( SELECT substr(ca_zip,1,5) ca_zip FROM customer_address WHERE substr(ca_zip,1,5) IN ( '47602','16704','35863','28577','83910','36201', '58412','48162','28055','41419','80332', '38607','77817','24891','16226','18410', '21231','59345','13918','51089','20317', '17167','54585','67881','78366','47770', '18360','51717','73108','14440','21800', '89338','45859','65501','34948','25973', '73219','25333','17291','10374','18829', '60736','82620','41351','52094','19326', '25214','54207','40936','21814','79077', '25178','75742','77454','30621','89193', '27369','41232','48567','83041','71948', '37119','68341','14073','16891','62878', '49130','19833','24286','27700','40979', '50412','81504','94835','84844','71954', '39503','57649','18434','24987','12350', '86379','27413','44529','98569','16515', '27287','24255','21094','16005','56436', '91110','68293','56455','54558','10298', '83647','32754','27052','51766','19444', '13869','45645','94791','57631','20712', '37788','41807','46507','21727','71836', '81070','50632','88086','63991','20244', '31655','51782','29818','63792','68605', '94898','36430','57025','20601','82080', '33869','22728','35834','29086','92645', '98584','98072','11652','78093','57553', '43830','71144','53565','18700','90209', '71256','38353','54364','28571','96560', '57839','56355','50679','45266','84680', '34306','34972','48530','30106','15371', '92380','84247','92292','68852','13338', '34594','82602','70073','98069','85066', '47289','11686','98862','26217','47529', '63294','51793','35926','24227','14196', '24594','32489','99060','49472','43432', '49211','14312','88137','47369','56877', '20534','81755','15794','12318','21060', '73134','41255','63073','81003','73873', '66057','51184','51195','45676','92696', '70450','90669','98338','25264','38919', '59226','58581','60298','17895','19489', '52301','80846','95464','68770','51634', '19988','18367','18421','11618','67975', '25494','41352','95430','15734','62585', '97173','33773','10425','75675','53535', '17879','41967','12197','67998','79658', '59130','72592','14851','43933','68101', '50636','25717','71286','24660','58058', '72991','95042','15543','33122','69280', '11912','59386','27642','65177','17672', '33467','64592','36335','54010','18767', '63193','42361','49254','33113','33159', '36479','59080','11855','81963','31016', '49140','29392','41836','32958','53163', '13844','73146','23952','65148','93498', '14530','46131','58454','13376','13378', '83986','12320','17193','59852','46081', '98533','52389','13086','68843','31013', '13261','60560','13443','45533','83583', '11489','58218','19753','22911','25115', '86709','27156','32669','13123','51933', '39214','41331','66943','14155','69998', '49101','70070','35076','14242','73021', '59494','15782','29752','37914','74686', '83086','34473','15751','81084','49230', '91894','60624','17819','28810','63180', '56224','39459','55233','75752','43639', '55349','86057','62361','50788','31830', '58062','18218','85761','60083','45484', '21204','90229','70041','41162','35390', '16364','39500','68908','26689','52868', '81335','40146','11340','61527','61794', '71997','30415','59004','29450','58117', '69952','33562','83833','27385','61860', '96435','48333','23065','32961','84919', '61997','99132','22815','56600','68730', '48017','95694','32919','88217','27116', '28239','58032','18884','16791','21343', '97462','18569','75660','15475') intersect select ca_zip from (SELECT substr(ca_zip,1,5) ca_zip,count(*) cnt FROM customer_address, customer WHERE ca_address_sk = c_current_addr_sk and c_preferred_cust_flag='Y' group by ca_zip having count(*) > 10)A1)A2) V1 where ss_store_sk = s_store_sk and ss_sold_date_sk = d_date_sk and d_qoy = 2 and d_year = 1998 and (substr(s_zip,1,2) = substr(V1.ca_zip,1,2)) group by s_store_name order by s_store_name limit 100""", "q9" -> """ select case when (select count(*) from store_sales where ss_quantity between 1 and 20) > 4502397049 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 1 and 20) else (select avg(ss_net_profit) from store_sales where ss_quantity between 1 and 20) end bucket1 , case when (select count(*) from store_sales where ss_quantity between 21 and 40) > 4756228269 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 21 and 40) else (select avg(ss_net_profit) from store_sales where ss_quantity between 21 and 40) end bucket2, case when (select count(*) from store_sales where ss_quantity between 41 and 60) > 4101835064 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 41 and 60) else (select avg(ss_net_profit) from store_sales where ss_quantity between 41 and 60) end bucket3, case when (select count(*) from store_sales where ss_quantity between 61 and 80) > 4583261513 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 61 and 80) else (select avg(ss_net_profit) from store_sales where ss_quantity between 61 and 80) end bucket4, case when (select count(*) from store_sales where ss_quantity between 81 and 100) > 4208819283 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 81 and 100) else (select avg(ss_net_profit) from store_sales where ss_quantity between 81 and 100) end bucket5 from reason where r_reason_sk = 1""", "q10" -> """ select cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3, cd_dep_count, count(*) cnt4, cd_dep_employed_count, count(*) cnt5, cd_dep_college_count, count(*) cnt6 from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and ca_county in ('Grady County','Marion County','Decatur County','Lyman County','Beaver County') and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 1999 and d_moy between 2 and 2+3) and (exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 1999 and d_moy between 2 ANd 2+3) or exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 1999 and d_moy between 2 and 2+3)) group by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count order by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count limit 100""", "q11" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(ss_ext_list_price-ss_ext_discount_amt) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(ws_ext_list_price-ws_ext_discount_amt) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year ) select t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_preferred_cust_flag from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.sale_type = 's' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_w_secyear.sale_type = 'w' and t_s_firstyear.dyear = 2001 and t_s_secyear.dyear = 2001+1 and t_w_firstyear.dyear = 2001 and t_w_secyear.dyear = 2001+1 and t_s_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else 0.0 end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else 0.0 end order by t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_preferred_cust_flag limit 100""", "q12" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(ws_ext_sales_price) as itemrevenue ,sum(ws_ext_sales_price)*100/sum(sum(ws_ext_sales_price)) over (partition by i_class) as revenueratio from web_sales ,item ,date_dim where ws_item_sk = i_item_sk and i_category in ('Children', 'Jewelry', 'Music') and ws_sold_date_sk = d_date_sk and d_date between cast('2001-05-11' as date) and (cast('2001-05-11' as date) + INTERVAL 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio limit 100""", "q13" -> """ select avg(ss_quantity) ,avg(ss_ext_sales_price) ,avg(ss_ext_wholesale_cost) ,sum(ss_ext_wholesale_cost) from store_sales ,store ,customer_demographics ,household_demographics ,customer_address ,date_dim where s_store_sk = ss_store_sk and ss_sold_date_sk = d_date_sk and d_year = 2001 and((ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'M' and cd_education_status = 'Primary' and ss_sales_price between 100.00 and 150.00 and hd_dep_count = 3 )or (ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'S' and cd_education_status = '4 yr Degree' and ss_sales_price between 50.00 and 100.00 and hd_dep_count = 1 ) or (ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'W' and cd_education_status = '2 yr Degree' and ss_sales_price between 150.00 and 200.00 and hd_dep_count = 1 )) and((ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('SC', 'WY', 'TX') and ss_net_profit between 100 and 200 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('NY', 'NE', 'GA') and ss_net_profit between 150 and 300 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('AL', 'AR', 'MI') and ss_net_profit between 50 and 250 ))""", "q14a" -> """ with cross_items as (select i_item_sk ss_item_sk from item, (select iss.i_brand_id brand_id ,iss.i_class_id class_id ,iss.i_category_id category_id from store_sales ,item iss ,date_dim d1 where ss_item_sk = iss.i_item_sk and ss_sold_date_sk = d1.d_date_sk and d1.d_year between 1999 AND 1999 + 2 intersect select ics.i_brand_id ,ics.i_class_id ,ics.i_category_id from catalog_sales ,item ics ,date_dim d2 where cs_item_sk = ics.i_item_sk and cs_sold_date_sk = d2.d_date_sk and d2.d_year between 1999 AND 1999 + 2 intersect select iws.i_brand_id ,iws.i_class_id ,iws.i_category_id from web_sales ,item iws ,date_dim d3 where ws_item_sk = iws.i_item_sk and ws_sold_date_sk = d3.d_date_sk and d3.d_year between 1999 AND 1999 + 2) where i_brand_id = brand_id and i_class_id = class_id and i_category_id = category_id ), avg_sales as (select avg(quantity*list_price) average_sales from (select ss_quantity quantity ,ss_list_price list_price from store_sales ,date_dim where ss_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2 union all select cs_quantity quantity ,cs_list_price list_price from catalog_sales ,date_dim where cs_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2 union all select ws_quantity quantity ,ws_list_price list_price from web_sales ,date_dim where ws_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2) x) select channel, i_brand_id,i_class_id,i_category_id,sum(sales), sum(number_sales) from( select 'store' channel, i_brand_id,i_class_id ,i_category_id,sum(ss_quantity*ss_list_price) sales , count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1999+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales) union all select 'catalog' channel, i_brand_id,i_class_id,i_category_id, sum(cs_quantity*cs_list_price) sales, count(*) number_sales from catalog_sales ,item ,date_dim where cs_item_sk in (select ss_item_sk from cross_items) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1999+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(cs_quantity*cs_list_price) > (select average_sales from avg_sales) union all select 'web' channel, i_brand_id,i_class_id,i_category_id, sum(ws_quantity*ws_list_price) sales , count(*) number_sales from web_sales ,item ,date_dim where ws_item_sk in (select ss_item_sk from cross_items) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1999+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(ws_quantity*ws_list_price) > (select average_sales from avg_sales) ) y group by rollup (channel, i_brand_id,i_class_id,i_category_id) order by channel,i_brand_id,i_class_id,i_category_id limit 100""", "q14b" -> """ with cross_items as (select i_item_sk ss_item_sk from item, (select iss.i_brand_id brand_id ,iss.i_class_id class_id ,iss.i_category_id category_id from store_sales ,item iss ,date_dim d1 where ss_item_sk = iss.i_item_sk and ss_sold_date_sk = d1.d_date_sk and d1.d_year between 1999 AND 1999 + 2 intersect select ics.i_brand_id ,ics.i_class_id ,ics.i_category_id from catalog_sales ,item ics ,date_dim d2 where cs_item_sk = ics.i_item_sk and cs_sold_date_sk = d2.d_date_sk and d2.d_year between 1999 AND 1999 + 2 intersect select iws.i_brand_id ,iws.i_class_id ,iws.i_category_id from web_sales ,item iws ,date_dim d3 where ws_item_sk = iws.i_item_sk and ws_sold_date_sk = d3.d_date_sk and d3.d_year between 1999 AND 1999 + 2) x where i_brand_id = brand_id and i_class_id = class_id and i_category_id = category_id ), avg_sales as (select avg(quantity*list_price) average_sales from (select ss_quantity quantity ,ss_list_price list_price from store_sales ,date_dim where ss_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2 union all select cs_quantity quantity ,cs_list_price list_price from catalog_sales ,date_dim where cs_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2 union all select ws_quantity quantity ,ws_list_price list_price from web_sales ,date_dim where ws_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2) x) select this_year.channel ty_channel ,this_year.i_brand_id ty_brand ,this_year.i_class_id ty_class ,this_year.i_category_id ty_category ,this_year.sales ty_sales ,this_year.number_sales ty_number_sales ,last_year.channel ly_channel ,last_year.i_brand_id ly_brand ,last_year.i_class_id ly_class ,last_year.i_category_id ly_category ,last_year.sales ly_sales ,last_year.number_sales ly_number_sales from (select 'store' channel, i_brand_id,i_class_id,i_category_id ,sum(ss_quantity*ss_list_price) sales, count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_week_seq = (select d_week_seq from date_dim where d_year = 1999 + 1 and d_moy = 12 and d_dom = 5) group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) this_year, (select 'store' channel, i_brand_id,i_class_id ,i_category_id, sum(ss_quantity*ss_list_price) sales, count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_week_seq = (select d_week_seq from date_dim where d_year = 1999 and d_moy = 12 and d_dom = 5) group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) last_year where this_year.i_brand_id= last_year.i_brand_id and this_year.i_class_id = last_year.i_class_id and this_year.i_category_id = last_year.i_category_id order by this_year.channel, this_year.i_brand_id, this_year.i_class_id, this_year.i_category_id limit 100""", "q15" -> """ select ca_zip ,sum(cs_sales_price) from catalog_sales ,customer ,customer_address ,date_dim where cs_bill_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') or ca_state in ('CA','WA','GA') or cs_sales_price > 500) and cs_sold_date_sk = d_date_sk and d_qoy = 1 and d_year = 1998 group by ca_zip order by ca_zip limit 100""", "q16" -> """ select count(distinct cs_order_number) as `order count` ,sum(cs_ext_ship_cost) as `total shipping cost` ,sum(cs_net_profit) as `total net profit` from catalog_sales cs1 ,date_dim ,customer_address ,call_center where d_date between '2000-3-01' and (cast('2000-3-01' as date) + INTERVAL 60 days) and cs1.cs_ship_date_sk = d_date_sk and cs1.cs_ship_addr_sk = ca_address_sk and ca_state = 'IA' and cs1.cs_call_center_sk = cc_call_center_sk and cc_county in ('Luce County','Wadena County','Jefferson Davis Parish','Daviess County', 'Williamson County' ) and exists (select * from catalog_sales cs2 where cs1.cs_order_number = cs2.cs_order_number and cs1.cs_warehouse_sk <> cs2.cs_warehouse_sk) and not exists(select * from catalog_returns cr1 where cs1.cs_order_number = cr1.cr_order_number) order by count(distinct cs_order_number) limit 100""", "q17" -> """ select i_item_id ,i_item_desc ,s_state ,count(ss_quantity) as store_sales_quantitycount ,avg(ss_quantity) as store_sales_quantityave ,stddev_samp(ss_quantity) as store_sales_quantitystdev ,stddev_samp(ss_quantity)/avg(ss_quantity) as store_sales_quantitycov ,count(sr_return_quantity) as store_returns_quantitycount ,avg(sr_return_quantity) as store_returns_quantityave ,stddev_samp(sr_return_quantity) as store_returns_quantitystdev ,stddev_samp(sr_return_quantity)/avg(sr_return_quantity) as store_returns_quantitycov ,count(cs_quantity) as catalog_sales_quantitycount ,avg(cs_quantity) as catalog_sales_quantityave ,stddev_samp(cs_quantity) as catalog_sales_quantitystdev ,stddev_samp(cs_quantity)/avg(cs_quantity) as catalog_sales_quantitycov from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_quarter_name = '1999Q1' and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_quarter_name in ('1999Q1','1999Q2','1999Q3') and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_quarter_name in ('1999Q1','1999Q2','1999Q3') group by i_item_id ,i_item_desc ,s_state order by i_item_id ,i_item_desc ,s_state limit 100""", "q18" -> """ select i_item_id, ca_country, ca_state, ca_county, avg( cast(cs_quantity as decimal(12,2))) agg1, avg( cast(cs_list_price as decimal(12,2))) agg2, avg( cast(cs_coupon_amt as decimal(12,2))) agg3, avg( cast(cs_sales_price as decimal(12,2))) agg4, avg( cast(cs_net_profit as decimal(12,2))) agg5, avg( cast(c_birth_year as decimal(12,2))) agg6, avg( cast(cd1.cd_dep_count as decimal(12,2))) agg7 from catalog_sales, customer_demographics cd1, customer_demographics cd2, customer, customer_address, date_dim, item where cs_sold_date_sk = d_date_sk and cs_item_sk = i_item_sk and cs_bill_cdemo_sk = cd1.cd_demo_sk and cs_bill_customer_sk = c_customer_sk and cd1.cd_gender = 'F' and cd1.cd_education_status = 'Unknown' and c_current_cdemo_sk = cd2.cd_demo_sk and c_current_addr_sk = ca_address_sk and c_birth_month in (4,8,12,10,11,9) and d_year = 2001 and ca_state in ('AR','IA','TX' ,'KS','LA','NC','SD') group by rollup (i_item_id, ca_country, ca_state, ca_county) order by ca_country, ca_state, ca_county, i_item_id limit 100""", "q19" -> """ select i_brand_id brand_id, i_brand brand, i_manufact_id, i_manufact, sum(ss_ext_sales_price) ext_price from date_dim, store_sales, item,customer,customer_address,store where d_date_sk = ss_sold_date_sk and ss_item_sk = i_item_sk and i_manager_id=63 and d_moy=11 and d_year=2002 and ss_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and substr(ca_zip,1,5) <> substr(s_zip,1,5) and ss_store_sk = s_store_sk group by i_brand ,i_brand_id ,i_manufact_id ,i_manufact order by ext_price desc ,i_brand ,i_brand_id ,i_manufact_id ,i_manufact limit 100 """, "q20" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(cs_ext_sales_price) as itemrevenue ,sum(cs_ext_sales_price)*100/sum(sum(cs_ext_sales_price)) over (partition by i_class) as revenueratio from catalog_sales ,item ,date_dim where cs_item_sk = i_item_sk and i_category in ('Electronics', 'Children', 'Home') and cs_sold_date_sk = d_date_sk and d_date between cast('2002-03-19' as date) and (cast('2002-03-19' as date) + INTERVAL 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio limit 100""", "q21" -> """ select * from(select w_warehouse_name ,i_item_id ,sum(case when (cast(d_date as date) < cast ('1999-04-12' as date)) then inv_quantity_on_hand else 0 end) as inv_before ,sum(case when (cast(d_date as date) >= cast ('1999-04-12' as date)) then inv_quantity_on_hand else 0 end) as inv_after from inventory ,warehouse ,item ,date_dim where i_current_price between 0.99 and 1.49 and i_item_sk = inv_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_date between (cast ('1999-04-12' as date) - INTERVAL 30 days) and (cast ('1999-04-12' as date) + INTERVAL 30 days) group by w_warehouse_name, i_item_id) x where (case when inv_before > 0 then inv_after / inv_before else null end) between 2.0/3.0 and 3.0/2.0 order by w_warehouse_name ,i_item_id limit 100""", "q22" -> """ select i_product_name ,i_brand ,i_class ,i_category ,avg(inv_quantity_on_hand) qoh from inventory ,date_dim ,item where inv_date_sk=d_date_sk and inv_item_sk=i_item_sk and d_month_seq between 1188 and 1188 + 11 group by rollup(i_product_name ,i_brand ,i_class ,i_category) order by qoh, i_product_name, i_brand, i_class, i_category limit 100""", "q23a" -> """ with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt from store_sales ,date_dim ,item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and d_year in (1998,1998+1,1998+2,1998+3) group by substr(i_item_desc,1,30),i_item_sk,d_date having count(*) >4), max_store_sales as (select max(csales) tpcds_cmax from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales from store_sales ,customer ,date_dim where ss_customer_sk = c_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (1998,1998+1,1998+2,1998+3) group by c_customer_sk)), best_ss_customer as (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales from store_sales ,customer where ss_customer_sk = c_customer_sk group by c_customer_sk having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select * from max_store_sales)) select sum(sales) from (select cs_quantity*cs_list_price sales from catalog_sales ,date_dim where d_year = 1998 and d_moy = 7 and cs_sold_date_sk = d_date_sk and cs_item_sk in (select item_sk from frequent_ss_items) and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) union all select ws_quantity*ws_list_price sales from web_sales ,date_dim where d_year = 1998 and d_moy = 7 and ws_sold_date_sk = d_date_sk and ws_item_sk in (select item_sk from frequent_ss_items) and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer)) limit 100""", "q23b" -> """ with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt from store_sales ,date_dim ,item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and d_year in (1998,1998 + 1,1998 + 2,1998 + 3) group by substr(i_item_desc,1,30),i_item_sk,d_date having count(*) >4), max_store_sales as (select max(csales) tpcds_cmax from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales from store_sales ,customer ,date_dim where ss_customer_sk = c_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (1998,1998+1,1998+2,1998+3) group by c_customer_sk)), best_ss_customer as (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales from store_sales ,customer where ss_customer_sk = c_customer_sk group by c_customer_sk having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select * from max_store_sales)) select c_last_name,c_first_name,sales from (select c_last_name,c_first_name,sum(cs_quantity*cs_list_price) sales from catalog_sales ,customer ,date_dim where d_year = 1998 and d_moy = 7 and cs_sold_date_sk = d_date_sk and cs_item_sk in (select item_sk from frequent_ss_items) and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) and cs_bill_customer_sk = c_customer_sk group by c_last_name,c_first_name union all select c_last_name,c_first_name,sum(ws_quantity*ws_list_price) sales from web_sales ,customer ,date_dim where d_year = 1998 and d_moy = 7 and ws_sold_date_sk = d_date_sk and ws_item_sk in (select item_sk from frequent_ss_items) and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer) and ws_bill_customer_sk = c_customer_sk group by c_last_name,c_first_name) order by c_last_name,c_first_name,sales limit 100""", "q24a" -> """ with ssales as (select c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size ,sum(ss_sales_price) netpaid from store_sales ,store_returns ,store ,item ,customer ,customer_address where ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_customer_sk = c_customer_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and c_current_addr_sk = ca_address_sk and c_birth_country <> upper(ca_country) and s_zip = ca_zip and s_market_id=7 group by c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size) select c_last_name ,c_first_name ,s_store_name ,sum(netpaid) paid from ssales where i_color = 'goldenrod' group by c_last_name ,c_first_name ,s_store_name having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) order by c_last_name ,c_first_name ,s_store_name""", "q24b" -> """ with ssales as (select c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size ,sum(ss_sales_price) netpaid from store_sales ,store_returns ,store ,item ,customer ,customer_address where ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_customer_sk = c_customer_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and c_current_addr_sk = ca_address_sk and c_birth_country <> upper(ca_country) and s_zip = ca_zip and s_market_id = 7 group by c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size) select c_last_name ,c_first_name ,s_store_name ,sum(netpaid) paid from ssales where i_color = 'magenta' group by c_last_name ,c_first_name ,s_store_name having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) order by c_last_name ,c_first_name ,s_store_name""", "q25" -> """ select i_item_id ,i_item_desc ,s_store_id ,s_store_name ,min(ss_net_profit) as store_sales_profit ,min(sr_net_loss) as store_returns_loss ,min(cs_net_profit) as catalog_sales_profit from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_moy = 4 and d1.d_year = 2002 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_moy between 4 and 10 and d2.d_year = 2002 and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_moy between 4 and 10 and d3.d_year = 2002 group by i_item_id ,i_item_desc ,s_store_id ,s_store_name order by i_item_id ,i_item_desc ,s_store_id ,s_store_name limit 100""", "q26" -> """ select i_item_id, avg(cs_quantity) agg1, avg(cs_list_price) agg2, avg(cs_coupon_amt) agg3, avg(cs_sales_price) agg4 from catalog_sales, customer_demographics, date_dim, item, promotion where cs_sold_date_sk = d_date_sk and cs_item_sk = i_item_sk and cs_bill_cdemo_sk = cd_demo_sk and cs_promo_sk = p_promo_sk and cd_gender = 'F' and cd_marital_status = 'M' and cd_education_status = '4 yr Degree' and (p_channel_email = 'N' or p_channel_event = 'N') and d_year = 1998 group by i_item_id order by i_item_id limit 100""", "q27" -> """ select i_item_id, s_state, grouping(s_state) g_state, avg(ss_quantity) agg1, avg(ss_list_price) agg2, avg(ss_coupon_amt) agg3, avg(ss_sales_price) agg4 from store_sales, customer_demographics, date_dim, store, item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and ss_cdemo_sk = cd_demo_sk and cd_gender = 'M' and cd_marital_status = 'M' and cd_education_status = 'Secondary' and d_year = 1999 and s_state in ('AL','FL', 'TX', 'NM', 'MI', 'GA') group by rollup (i_item_id, s_state) order by i_item_id ,s_state limit 100""", "q28" -> """ select * from (select avg(ss_list_price) B1_LP ,count(ss_list_price) B1_CNT ,count(distinct ss_list_price) B1_CNTD from store_sales where ss_quantity between 0 and 5 and (ss_list_price between 74 and 74+10 or ss_coupon_amt between 2949 and 2949+1000 or ss_wholesale_cost between 49 and 49+20)) B1, (select avg(ss_list_price) B2_LP ,count(ss_list_price) B2_CNT ,count(distinct ss_list_price) B2_CNTD from store_sales where ss_quantity between 6 and 10 and (ss_list_price between 136 and 136+10 or ss_coupon_amt between 10027 and 10027+1000 or ss_wholesale_cost between 53 and 53+20)) B2, (select avg(ss_list_price) B3_LP ,count(ss_list_price) B3_CNT ,count(distinct ss_list_price) B3_CNTD from store_sales where ss_quantity between 11 and 15 and (ss_list_price between 73 and 73+10 or ss_coupon_amt between 1451 and 1451+1000 or ss_wholesale_cost between 78 and 78+20)) B3, (select avg(ss_list_price) B4_LP ,count(ss_list_price) B4_CNT ,count(distinct ss_list_price) B4_CNTD from store_sales where ss_quantity between 16 and 20 and (ss_list_price between 87 and 87+10 or ss_coupon_amt between 17007 and 17007+1000 or ss_wholesale_cost between 55 and 55+20)) B4, (select avg(ss_list_price) B5_LP ,count(ss_list_price) B5_CNT ,count(distinct ss_list_price) B5_CNTD from store_sales where ss_quantity between 21 and 25 and (ss_list_price between 112 and 112+10 or ss_coupon_amt between 17243 and 17243+1000 or ss_wholesale_cost between 2 and 2+20)) B5, (select avg(ss_list_price) B6_LP ,count(ss_list_price) B6_CNT ,count(distinct ss_list_price) B6_CNTD from store_sales where ss_quantity between 26 and 30 and (ss_list_price between 119 and 119+10 or ss_coupon_amt between 4954 and 4954+1000 or ss_wholesale_cost between 22 and 22+20)) B6 limit 100""", "q29" -> """ select i_item_id ,i_item_desc ,s_store_id ,s_store_name ,stddev_samp(ss_quantity) as store_sales_quantity ,stddev_samp(sr_return_quantity) as store_returns_quantity ,stddev_samp(cs_quantity) as catalog_sales_quantity from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_moy = 4 and d1.d_year = 2000 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_moy between 4 and 4 + 3 and d2.d_year = 2000 and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_year in (2000,2000+1,2000+2) group by i_item_id ,i_item_desc ,s_store_id ,s_store_name order by i_item_id ,i_item_desc ,s_store_id ,s_store_name limit 100""", "q30" -> """ with customer_total_return as (select wr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, sum(wr_return_amt) as ctr_total_return from web_returns ,date_dim ,customer_address where wr_returned_date_sk = d_date_sk and d_year =2001 and wr_returning_addr_sk = ca_address_sk group by wr_returning_customer_sk ,ca_state) select c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address ,c_last_review_date,ctr_total_return from customer_total_return ctr1 ,customer_address ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_state = ctr2.ctr_state) and ca_address_sk = c_current_addr_sk and ca_state = 'MI' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address ,c_last_review_date,ctr_total_return limit 100""", "q31" -> """ with ss as (select ca_county,d_qoy, d_year,sum(ss_ext_sales_price) as store_sales from store_sales,date_dim,customer_address where ss_sold_date_sk = d_date_sk and ss_addr_sk=ca_address_sk group by ca_county,d_qoy, d_year), ws as (select ca_county,d_qoy, d_year,sum(ws_ext_sales_price) as web_sales from web_sales,date_dim,customer_address where ws_sold_date_sk = d_date_sk and ws_bill_addr_sk=ca_address_sk group by ca_county,d_qoy, d_year) select ss1.ca_county ,ss1.d_year ,ws2.web_sales/ws1.web_sales web_q1_q2_increase ,ss2.store_sales/ss1.store_sales store_q1_q2_increase ,ws3.web_sales/ws2.web_sales web_q2_q3_increase ,ss3.store_sales/ss2.store_sales store_q2_q3_increase from ss ss1 ,ss ss2 ,ss ss3 ,ws ws1 ,ws ws2 ,ws ws3 where ss1.d_qoy = 1 and ss1.d_year = 2000 and ss1.ca_county = ss2.ca_county and ss2.d_qoy = 2 and ss2.d_year = 2000 and ss2.ca_county = ss3.ca_county and ss3.d_qoy = 3 and ss3.d_year = 2000 and ss1.ca_county = ws1.ca_county and ws1.d_qoy = 1 and ws1.d_year = 2000 and ws1.ca_county = ws2.ca_county and ws2.d_qoy = 2 and ws2.d_year = 2000 and ws1.ca_county = ws3.ca_county and ws3.d_qoy = 3 and ws3.d_year =2000 and case when ws1.web_sales > 0 then ws2.web_sales/ws1.web_sales else null end > case when ss1.store_sales > 0 then ss2.store_sales/ss1.store_sales else null end and case when ws2.web_sales > 0 then ws3.web_sales/ws2.web_sales else null end > case when ss2.store_sales > 0 then ss3.store_sales/ss2.store_sales else null end order by store_q1_q2_increase""", "q32" -> """ select sum(cs_ext_discount_amt) as `excess discount amount` from catalog_sales ,item ,date_dim where i_manufact_id = 490 and i_item_sk = cs_item_sk and d_date between '1999-01-27' and (cast('1999-01-27' as date) + INTERVAL 90 days) and d_date_sk = cs_sold_date_sk and cs_ext_discount_amt > ( select 1.3 * avg(cs_ext_discount_amt) from catalog_sales ,date_dim where cs_item_sk = i_item_sk and d_date between '1999-01-27' and (cast('1999-01-27' as date) + INTERVAL 90 days) and d_date_sk = cs_sold_date_sk ) limit 100""", "q33" -> """ with ss as ( select i_manufact_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Electronics')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 2001 and d_moy = 1 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_manufact_id), cs as ( select i_manufact_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Electronics')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 2001 and d_moy = 1 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_manufact_id), ws as ( select i_manufact_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Electronics')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 2001 and d_moy = 1 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_manufact_id) select i_manufact_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_manufact_id order by total_sales limit 100""", "q34" -> """ select c_last_name ,c_first_name ,c_salutation ,c_preferred_cust_flag ,ss_ticket_number ,cnt from (select ss_ticket_number ,ss_customer_sk ,count(*) cnt from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and (date_dim.d_dom between 1 and 3 or date_dim.d_dom between 25 and 28) and (household_demographics.hd_buy_potential = '1001-5000' or household_demographics.hd_buy_potential = 'Unknown') and household_demographics.hd_vehicle_count > 0 and (case when household_demographics.hd_vehicle_count > 0 then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end) > 1.2 and date_dim.d_year in (1999,1999+1,1999+2) and store.s_county in ('Nez Perce County','Murray County','Surry County','Calhoun County', 'Wilkinson County','Brown County','Wallace County','Carter County') group by ss_ticket_number,ss_customer_sk) dn,customer where ss_customer_sk = c_customer_sk and cnt between 15 and 20 order by c_last_name,c_first_name,c_salutation,c_preferred_cust_flag desc, ss_ticket_number""", "q35" -> """ select ca_state, cd_gender, cd_marital_status, cd_dep_count, count(*) cnt1, stddev_samp(cd_dep_count), sum(cd_dep_count), min(cd_dep_count), cd_dep_employed_count, count(*) cnt2, stddev_samp(cd_dep_employed_count), sum(cd_dep_employed_count), min(cd_dep_employed_count), cd_dep_college_count, count(*) cnt3, stddev_samp(cd_dep_college_count), sum(cd_dep_college_count), min(cd_dep_college_count) from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2002 and d_qoy < 4) and (exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2002 and d_qoy < 4) or exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2002 and d_qoy < 4)) group by ca_state, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count order by ca_state, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count limit 100""", "q36" -> """ select sum(ss_net_profit)/sum(ss_ext_sales_price) as gross_margin ,i_category ,i_class ,grouping(i_category)+grouping(i_class) as lochierarchy ,rank() over ( partition by grouping(i_category)+grouping(i_class), case when grouping(i_class) = 0 then i_category end order by sum(ss_net_profit)/sum(ss_ext_sales_price) asc) as rank_within_parent from store_sales ,date_dim d1 ,item ,store where d1.d_year = 2000 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and s_state in ('MN','TX','TX','IN', 'CA','LA','NM','TX') group by rollup(i_category,i_class) order by lochierarchy desc ,case when lochierarchy = 0 then i_category end ,rank_within_parent limit 100""", "q37" -> """ select i_item_id ,i_item_desc ,i_current_price from item, inventory, date_dim, catalog_sales where i_current_price between 16 and 16 + 30 and inv_item_sk = i_item_sk and d_date_sk=inv_date_sk and d_date between cast('2002-06-05' as date) and (cast('2002-06-05' as date) + interval 60 days) and i_manufact_id in (841,790,796,739) and inv_quantity_on_hand between 100 and 500 and cs_item_sk = i_item_sk group by i_item_id,i_item_desc,i_current_price order by i_item_id limit 100""", "q38" -> """ select count(*) from ( select distinct c_last_name, c_first_name, d_date from store_sales, date_dim, customer where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_customer_sk = customer.c_customer_sk and d_month_seq between 1203 and 1203 + 11 intersect select distinct c_last_name, c_first_name, d_date from catalog_sales, date_dim, customer where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1203 and 1203 + 11 intersect select distinct c_last_name, c_first_name, d_date from web_sales, date_dim, customer where web_sales.ws_sold_date_sk = date_dim.d_date_sk and web_sales.ws_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1203 and 1203 + 11 ) hot_cust limit 100""", "q39a" -> """ with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stdev,mean, case mean when 0 then null else stdev/mean end cov from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean from inventory ,item ,warehouse ,date_dim where inv_item_sk = i_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_year =1999 group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo where case mean when 0 then 0 else stdev/mean end > 1) select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov from inv inv1,inv inv2 where inv1.i_item_sk = inv2.i_item_sk and inv1.w_warehouse_sk = inv2.w_warehouse_sk and inv1.d_moy=3 and inv2.d_moy=3+1 order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov ,inv2.d_moy,inv2.mean, inv2.cov""", "q39b" -> """ with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stdev,mean, case mean when 0 then null else stdev/mean end cov from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean from inventory ,item ,warehouse ,date_dim where inv_item_sk = i_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_year =1999 group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo where case mean when 0 then 0 else stdev/mean end > 1) select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov from inv inv1,inv inv2 where inv1.i_item_sk = inv2.i_item_sk and inv1.w_warehouse_sk = inv2.w_warehouse_sk and inv1.d_moy=3 and inv2.d_moy=3+1 and inv1.cov > 1.5 order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov ,inv2.d_moy,inv2.mean, inv2.cov""", "q40" -> """ select w_state ,i_item_id ,sum(case when (cast(d_date as date) < cast ('1999-04-27' as date)) then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_before ,sum(case when (cast(d_date as date) >= cast ('1999-04-27' as date)) then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_after from catalog_sales left outer join catalog_returns on (cs_order_number = cr_order_number and cs_item_sk = cr_item_sk) ,warehouse ,item ,date_dim where i_current_price between 0.99 and 1.49 and i_item_sk = cs_item_sk and cs_warehouse_sk = w_warehouse_sk and cs_sold_date_sk = d_date_sk and d_date between (cast ('1999-04-27' as date) - INTERVAL 30 days) and (cast ('1999-04-27' as date) + INTERVAL 30 days) group by w_state,i_item_id order by w_state,i_item_id limit 100""", "q41" -> """ select distinct(i_product_name) from item i1 where i_manufact_id between 841 and 841+40 and (select count(*) as item_cnt from item where (i_manufact = i1.i_manufact and ((i_category = 'Women' and (i_color = 'bisque' or i_color = 'khaki') and (i_units = 'Carton' or i_units = 'Box') and (i_size = 'large' or i_size = 'extra large') ) or (i_category = 'Women' and (i_color = 'antique' or i_color = 'sandy') and (i_units = 'Pallet' or i_units = 'Cup') and (i_size = 'petite' or i_size = 'small') ) or (i_category = 'Men' and (i_color = 'forest' or i_color = 'brown') and (i_units = 'Dram' or i_units = 'Ton') and (i_size = 'economy' or i_size = 'medium') ) or (i_category = 'Men' and (i_color = 'chartreuse' or i_color = 'light') and (i_units = 'Pound' or i_units = 'Dozen') and (i_size = 'large' or i_size = 'extra large') ))) or (i_manufact = i1.i_manufact and ((i_category = 'Women' and (i_color = 'turquoise' or i_color = 'chocolate') and (i_units = 'Bundle' or i_units = 'Unknown') and (i_size = 'large' or i_size = 'extra large') ) or (i_category = 'Women' and (i_color = 'maroon' or i_color = 'pale') and (i_units = 'Each' or i_units = 'Tbl') and (i_size = 'petite' or i_size = 'small') ) or (i_category = 'Men' and (i_color = 'almond' or i_color = 'floral') and (i_units = 'Gross' or i_units = 'N/A') and (i_size = 'economy' or i_size = 'medium') ) or (i_category = 'Men' and (i_color = 'drab' or i_color = 'plum') and (i_units = 'Bunch' or i_units = 'Case') and (i_size = 'large' or i_size = 'extra large') )))) > 0 order by i_product_name limit 100""", "q42" -> """ select dt.d_year ,item.i_category_id ,item.i_category ,sum(ss_ext_sales_price) from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manager_id = 1 and dt.d_moy=11 and dt.d_year=2002 group by dt.d_year ,item.i_category_id ,item.i_category order by sum(ss_ext_sales_price) desc,dt.d_year ,item.i_category_id ,item.i_category limit 100 """, "q43" -> """ select s_store_name, s_store_id, sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales from date_dim, store_sales, store where d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk and s_gmt_offset = -5 and d_year = 2002 group by s_store_name, s_store_id order by s_store_name, s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales limit 100""", "q44" -> """ select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing from(select * from (select item_sk,rank() over (order by rank_col asc) rnk from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col from store_sales ss1 where ss_store_sk = 709 group by ss_item_sk having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col from store_sales where ss_store_sk = 709 and ss_addr_sk is null group by ss_store_sk))V1)V11 where rnk < 11) asceding, (select * from (select item_sk,rank() over (order by rank_col desc) rnk from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col from store_sales ss1 where ss_store_sk = 709 group by ss_item_sk having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col from store_sales where ss_store_sk = 709 and ss_addr_sk is null group by ss_store_sk))V2)V21 where rnk < 11) descending, item i1, item i2 where asceding.rnk = descending.rnk and i1.i_item_sk=asceding.item_sk and i2.i_item_sk=descending.item_sk order by asceding.rnk limit 100""", "q45" -> """ select ca_zip, ca_state, sum(ws_sales_price) from web_sales, customer, customer_address, date_dim, item where ws_bill_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and ws_item_sk = i_item_sk and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') or i_item_id in (select i_item_id from item where i_item_sk in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) ) ) and ws_sold_date_sk = d_date_sk and d_qoy = 2 and d_year = 2002 group by ca_zip, ca_state order by ca_zip, ca_state limit 100""", "q46" -> """ select c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number ,amt,profit from (select ss_ticket_number ,ss_customer_sk ,ca_city bought_city ,sum(ss_coupon_amt) amt ,sum(ss_net_profit) profit from store_sales,date_dim,store,household_demographics,customer_address where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and store_sales.ss_addr_sk = customer_address.ca_address_sk and (household_demographics.hd_dep_count = 0 or household_demographics.hd_vehicle_count= 1) and date_dim.d_dow in (6,0) and date_dim.d_year in (1999,1999+1,1999+2) and store.s_city in ('Johnson','Norwood','Cambridge','Klondike','Rock Hill') group by ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city) dn,customer,customer_address current_addr where ss_customer_sk = c_customer_sk and customer.c_current_addr_sk = current_addr.ca_address_sk and current_addr.ca_city <> bought_city order by c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number limit 100""", "q47" -> """ with v1 as( select i_category, i_brand, s_store_name, s_company_name, d_year, d_moy, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_category, i_brand, s_store_name, s_company_name, d_year) avg_monthly_sales, rank() over (partition by i_category, i_brand, s_store_name, s_company_name order by d_year, d_moy) rn from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ( d_year = 2001 or ( d_year = 2001-1 and d_moy =12) or ( d_year = 2001+1 and d_moy =1) ) group by i_category, i_brand, s_store_name, s_company_name, d_year, d_moy), v2 as( select v1.i_category, v1.i_brand, v1.s_store_name, v1.s_company_name ,v1.d_year, v1.d_moy ,v1.avg_monthly_sales ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum from v1, v1 v1_lag, v1 v1_lead where v1.i_category = v1_lag.i_category and v1.i_category = v1_lead.i_category and v1.i_brand = v1_lag.i_brand and v1.i_brand = v1_lead.i_brand and v1.s_store_name = v1_lag.s_store_name and v1.s_store_name = v1_lead.s_store_name and v1.s_company_name = v1_lag.s_company_name and v1.s_company_name = v1_lead.s_company_name and v1.rn = v1_lag.rn + 1 and v1.rn = v1_lead.rn - 1) select * from v2 where d_year = 2001 and avg_monthly_sales > 0 and case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by sum_sales - avg_monthly_sales, psum limit 100""", "q48" -> """ select sum (ss_quantity) from store_sales, store, customer_demographics, customer_address, date_dim where s_store_sk = ss_store_sk and ss_sold_date_sk = d_date_sk and d_year = 2000 and ( ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'U' and cd_education_status = '2 yr Degree' and ss_sales_price between 100.00 and 150.00 ) or ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'S' and cd_education_status = 'Primary' and ss_sales_price between 50.00 and 100.00 ) or ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'W' and cd_education_status = '4 yr Degree' and ss_sales_price between 150.00 and 200.00 ) ) and ( ( ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('MT', 'OH', 'GA') and ss_net_profit between 0 and 2000 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('WV', 'AZ', 'NM') and ss_net_profit between 150 and 3000 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('NY', 'PA', 'KY') and ss_net_profit between 50 and 25000 ) )""", "q49" -> """ select channel, item, return_ratio, return_rank, currency_rank from (select 'web' as channel ,web.item ,web.return_ratio ,web.return_rank ,web.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select ws.ws_item_sk as item ,(cast(sum(coalesce(wr.wr_return_quantity,0)) as decimal(15,4))/ cast(sum(coalesce(ws.ws_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(wr.wr_return_amt,0)) as decimal(15,4))/ cast(sum(coalesce(ws.ws_net_paid,0)) as decimal(15,4) )) as currency_ratio from web_sales ws left outer join web_returns wr on (ws.ws_order_number = wr.wr_order_number and ws.ws_item_sk = wr.wr_item_sk) ,date_dim where wr.wr_return_amt > 10000 and ws.ws_net_profit > 1 and ws.ws_net_paid > 0 and ws.ws_quantity > 0 and ws_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 11 group by ws.ws_item_sk ) in_web ) web where ( web.return_rank <= 10 or web.currency_rank <= 10 ) union select 'catalog' as channel ,catalog.item ,catalog.return_ratio ,catalog.return_rank ,catalog.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select cs.cs_item_sk as item ,(cast(sum(coalesce(cr.cr_return_quantity,0)) as decimal(15,4))/ cast(sum(coalesce(cs.cs_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(cr.cr_return_amount,0)) as decimal(15,4))/ cast(sum(coalesce(cs.cs_net_paid,0)) as decimal(15,4) )) as currency_ratio from catalog_sales cs left outer join catalog_returns cr on (cs.cs_order_number = cr.cr_order_number and cs.cs_item_sk = cr.cr_item_sk) ,date_dim where cr.cr_return_amount > 10000 and cs.cs_net_profit > 1 and cs.cs_net_paid > 0 and cs.cs_quantity > 0 and cs_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 11 group by cs.cs_item_sk ) in_cat ) catalog where ( catalog.return_rank <= 10 or catalog.currency_rank <=10 ) union select 'store' as channel ,store.item ,store.return_ratio ,store.return_rank ,store.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select sts.ss_item_sk as item ,(cast(sum(coalesce(sr.sr_return_quantity,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(sr.sr_return_amt,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_net_paid,0)) as decimal(15,4) )) as currency_ratio from store_sales sts left outer join store_returns sr on (sts.ss_ticket_number = sr.sr_ticket_number and sts.ss_item_sk = sr.sr_item_sk) ,date_dim where sr.sr_return_amt > 10000 and sts.ss_net_profit > 1 and sts.ss_net_paid > 0 and sts.ss_quantity > 0 and ss_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 11 group by sts.ss_item_sk ) in_store ) store where ( store.return_rank <= 10 or store.currency_rank <= 10 ) ) order by 1,4,5,2 limit 100""", "q50" -> """ select s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip ,sum(case when (sr_returned_date_sk - ss_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 30) and (sr_returned_date_sk - ss_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 60) and (sr_returned_date_sk - ss_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 90) and (sr_returned_date_sk - ss_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from store_sales ,store_returns ,store ,date_dim d1 ,date_dim d2 where d2.d_year = 2000 and d2.d_moy = 9 and ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_sold_date_sk = d1.d_date_sk and sr_returned_date_sk = d2.d_date_sk and ss_customer_sk = sr_customer_sk and ss_store_sk = s_store_sk group by s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip order by s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip limit 100""", "q51" -> """ WITH web_v1 as ( select ws_item_sk item_sk, d_date, sum(sum(ws_sales_price)) over (partition by ws_item_sk order by d_date rows between unbounded preceding and current row) cume_sales from web_sales ,date_dim where ws_sold_date_sk=d_date_sk and d_month_seq between 1177 and 1177+11 and ws_item_sk is not NULL group by ws_item_sk, d_date), store_v1 as ( select ss_item_sk item_sk, d_date, sum(sum(ss_sales_price)) over (partition by ss_item_sk order by d_date rows between unbounded preceding and current row) cume_sales from store_sales ,date_dim where ss_sold_date_sk=d_date_sk and d_month_seq between 1177 and 1177+11 and ss_item_sk is not NULL group by ss_item_sk, d_date) select * from (select item_sk ,d_date ,web_sales ,store_sales ,max(web_sales) over (partition by item_sk order by d_date rows between unbounded preceding and current row) web_cumulative ,max(store_sales) over (partition by item_sk order by d_date rows between unbounded preceding and current row) store_cumulative from (select case when web.item_sk is not null then web.item_sk else store.item_sk end item_sk ,case when web.d_date is not null then web.d_date else store.d_date end d_date ,web.cume_sales web_sales ,store.cume_sales store_sales from web_v1 web full outer join store_v1 store on (web.item_sk = store.item_sk and web.d_date = store.d_date) )x )y where web_cumulative > store_cumulative order by item_sk ,d_date limit 100""", "q52" -> """ select dt.d_year ,item.i_brand_id brand_id ,item.i_brand brand ,sum(ss_ext_sales_price) ext_price from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manager_id = 1 and dt.d_moy=12 and dt.d_year=2001 group by dt.d_year ,item.i_brand ,item.i_brand_id order by dt.d_year ,ext_price desc ,brand_id limit 100 """, "q53" -> """ select * from (select i_manufact_id, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_manufact_id) avg_quarterly_sales from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_month_seq in (1188,1188+1,1188+2,1188+3,1188+4,1188+5,1188+6,1188+7,1188+8,1188+9,1188+10,1188+11) and ((i_category in ('Books','Children','Electronics') and i_class in ('personal','portable','reference','self-help') and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', 'exportiunivamalg #9','scholaramalgamalg #9')) or(i_category in ('Women','Music','Men') and i_class in ('accessories','classical','fragrances','pants') and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 'importoamalg #1'))) group by i_manufact_id, d_qoy ) tmp1 where case when avg_quarterly_sales > 0 then abs (sum_sales - avg_quarterly_sales)/ avg_quarterly_sales else null end > 0.1 order by avg_quarterly_sales, sum_sales, i_manufact_id limit 100""", "q54" -> """ with my_customers as ( select distinct c_customer_sk , c_current_addr_sk from ( select cs_sold_date_sk sold_date_sk, cs_bill_customer_sk customer_sk, cs_item_sk item_sk from catalog_sales union all select ws_sold_date_sk sold_date_sk, ws_bill_customer_sk customer_sk, ws_item_sk item_sk from web_sales ) cs_or_ws_sales, item, date_dim, customer where sold_date_sk = d_date_sk and item_sk = i_item_sk and i_category = 'Men' and i_class = 'pants' and c_customer_sk = cs_or_ws_sales.customer_sk and d_moy = 5 and d_year = 2002 ) , my_revenue as ( select c_customer_sk, sum(ss_ext_sales_price) as revenue from my_customers, store_sales, customer_address, store, date_dim where c_current_addr_sk = ca_address_sk and ca_county = s_county and ca_state = s_state and ss_sold_date_sk = d_date_sk and c_customer_sk = ss_customer_sk and d_month_seq between (select distinct d_month_seq+1 from date_dim where d_year = 2002 and d_moy = 5) and (select distinct d_month_seq+3 from date_dim where d_year = 2002 and d_moy = 5) group by c_customer_sk ) , segments as (select cast((revenue/50) as int) as segment from my_revenue ) select segment, count(*) as num_customers, segment*50 as segment_base from segments group by segment order by segment, num_customers limit 100""", "q55" -> """ select i_brand_id brand_id, i_brand brand, sum(ss_ext_sales_price) ext_price from date_dim, store_sales, item where d_date_sk = ss_sold_date_sk and ss_item_sk = i_item_sk and i_manager_id=67 and d_moy=11 and d_year=2001 group by i_brand, i_brand_id order by ext_price desc, i_brand_id limit 100 """, "q56" -> """ with ss as ( select i_item_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('blanched','spring','seashell')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 6 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -7 group by i_item_id), cs as ( select i_item_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('blanched','spring','seashell')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 6 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -7 group by i_item_id), ws as ( select i_item_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('blanched','spring','seashell')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 6 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -7 group by i_item_id) select i_item_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_item_id order by total_sales, i_item_id limit 100""", "q57" -> """ with v1 as( select i_category, i_brand, cc_name, d_year, d_moy, sum(cs_sales_price) sum_sales, avg(sum(cs_sales_price)) over (partition by i_category, i_brand, cc_name, d_year) avg_monthly_sales, rank() over (partition by i_category, i_brand, cc_name order by d_year, d_moy) rn from item, catalog_sales, date_dim, call_center where cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and cc_call_center_sk= cs_call_center_sk and ( d_year = 2000 or ( d_year = 2000-1 and d_moy =12) or ( d_year = 2000+1 and d_moy =1) ) group by i_category, i_brand, cc_name , d_year, d_moy), v2 as( select v1.i_category, v1.i_brand ,v1.d_year ,v1.avg_monthly_sales ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum from v1, v1 v1_lag, v1 v1_lead where v1.i_category = v1_lag.i_category and v1.i_category = v1_lead.i_category and v1.i_brand = v1_lag.i_brand and v1.i_brand = v1_lead.i_brand and v1. cc_name = v1_lag. cc_name and v1. cc_name = v1_lead. cc_name and v1.rn = v1_lag.rn + 1 and v1.rn = v1_lead.rn - 1) select * from v2 where d_year = 2000 and avg_monthly_sales > 0 and case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by sum_sales - avg_monthly_sales, sum_sales limit 100""", "q58" -> """ with ss_items as (select i_item_id item_id ,sum(ss_ext_sales_price) ss_item_rev from store_sales ,item ,date_dim where ss_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq = (select d_week_seq from date_dim where d_date = '2000-05-24')) and ss_sold_date_sk = d_date_sk group by i_item_id), cs_items as (select i_item_id item_id ,sum(cs_ext_sales_price) cs_item_rev from catalog_sales ,item ,date_dim where cs_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq = (select d_week_seq from date_dim where d_date = '2000-05-24')) and cs_sold_date_sk = d_date_sk group by i_item_id), ws_items as (select i_item_id item_id ,sum(ws_ext_sales_price) ws_item_rev from web_sales ,item ,date_dim where ws_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq =(select d_week_seq from date_dim where d_date = '2000-05-24')) and ws_sold_date_sk = d_date_sk group by i_item_id) select ss_items.item_id ,ss_item_rev ,ss_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ss_dev ,cs_item_rev ,cs_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 cs_dev ,ws_item_rev ,ws_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ws_dev ,(ss_item_rev+cs_item_rev+ws_item_rev)/3 average from ss_items,cs_items,ws_items where ss_items.item_id=cs_items.item_id and ss_items.item_id=ws_items.item_id and ss_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev and ss_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev and cs_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev and cs_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev and ws_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev and ws_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev order by item_id ,ss_item_rev limit 100""", "q59" -> """ with wss as (select d_week_seq, ss_store_sk, sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales from store_sales,date_dim where d_date_sk = ss_sold_date_sk group by d_week_seq,ss_store_sk ) select s_store_name1,s_store_id1,d_week_seq1 ,sun_sales1/sun_sales2,mon_sales1/mon_sales2 ,tue_sales1/tue_sales2,wed_sales1/wed_sales2,thu_sales1/thu_sales2 ,fri_sales1/fri_sales2,sat_sales1/sat_sales2 from (select s_store_name s_store_name1,wss.d_week_seq d_week_seq1 ,s_store_id s_store_id1,sun_sales sun_sales1 ,mon_sales mon_sales1,tue_sales tue_sales1 ,wed_sales wed_sales1,thu_sales thu_sales1 ,fri_sales fri_sales1,sat_sales sat_sales1 from wss,store,date_dim d where d.d_week_seq = wss.d_week_seq and ss_store_sk = s_store_sk and d_month_seq between 1197 and 1197 + 11) y, (select s_store_name s_store_name2,wss.d_week_seq d_week_seq2 ,s_store_id s_store_id2,sun_sales sun_sales2 ,mon_sales mon_sales2,tue_sales tue_sales2 ,wed_sales wed_sales2,thu_sales thu_sales2 ,fri_sales fri_sales2,sat_sales sat_sales2 from wss,store,date_dim d where d.d_week_seq = wss.d_week_seq and ss_store_sk = s_store_sk and d_month_seq between 1197+ 12 and 1197 + 23) x where s_store_id1=s_store_id2 and d_week_seq1=d_week_seq2-52 order by s_store_name1,s_store_id1,d_week_seq1 limit 100""", "q60" -> """ with ss as ( select i_item_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Shoes')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 10 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id), cs as ( select i_item_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Shoes')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 10 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id), ws as ( select i_item_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Shoes')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 10 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id) select i_item_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_item_id order by i_item_id ,total_sales limit 100""", "q61" -> """ select promotions,total,cast(promotions as decimal(15,4))/cast(total as decimal(15,4))*100 from (select sum(ss_ext_sales_price) promotions from store_sales ,store ,promotion ,date_dim ,customer ,customer_address ,item where ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ss_promo_sk = p_promo_sk and ss_customer_sk= c_customer_sk and ca_address_sk = c_current_addr_sk and ss_item_sk = i_item_sk and ca_gmt_offset = -7 and i_category = 'Jewelry' and (p_channel_dmail = 'Y' or p_channel_email = 'Y' or p_channel_tv = 'Y') and s_gmt_offset = -7 and d_year = 2002 and d_moy = 11) promotional_sales, (select sum(ss_ext_sales_price) total from store_sales ,store ,date_dim ,customer ,customer_address ,item where ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ss_customer_sk= c_customer_sk and ca_address_sk = c_current_addr_sk and ss_item_sk = i_item_sk and ca_gmt_offset = -7 and i_category = 'Jewelry' and s_gmt_offset = -7 and d_year = 2002 and d_moy = 11) all_sales order by promotions, total limit 100""", "q62" -> """ select substr(w_warehouse_name,1,20) ,sm_type ,web_name ,sum(case when (ws_ship_date_sk - ws_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 30) and (ws_ship_date_sk - ws_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 60) and (ws_ship_date_sk - ws_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 90) and (ws_ship_date_sk - ws_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from web_sales ,warehouse ,ship_mode ,web_site ,date_dim where d_month_seq between 1194 and 1194 + 11 and ws_ship_date_sk = d_date_sk and ws_warehouse_sk = w_warehouse_sk and ws_ship_mode_sk = sm_ship_mode_sk and ws_web_site_sk = web_site_sk group by substr(w_warehouse_name,1,20) ,sm_type ,web_name order by substr(w_warehouse_name,1,20) ,sm_type ,web_name limit 100""", "q63" -> """ select * from (select i_manager_id ,sum(ss_sales_price) sum_sales ,avg(sum(ss_sales_price)) over (partition by i_manager_id) avg_monthly_sales from item ,store_sales ,date_dim ,store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_month_seq in (1222,1222+1,1222+2,1222+3,1222+4,1222+5,1222+6,1222+7,1222+8,1222+9,1222+10,1222+11) and (( i_category in ('Books','Children','Electronics') and i_class in ('personal','portable','reference','self-help') and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', 'exportiunivamalg #9','scholaramalgamalg #9')) or( i_category in ('Women','Music','Men') and i_class in ('accessories','classical','fragrances','pants') and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 'importoamalg #1'))) group by i_manager_id, d_moy) tmp1 where case when avg_monthly_sales > 0 then abs (sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by i_manager_id ,avg_monthly_sales ,sum_sales limit 100""", "q64" -> """ with cs_ui as (select cs_item_sk ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund from catalog_sales ,catalog_returns where cs_item_sk = cr_item_sk and cs_order_number = cr_order_number group by cs_item_sk having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), cross_sales as (select i_product_name product_name ,i_item_sk item_sk ,s_store_name store_name ,s_zip store_zip ,ad1.ca_street_number b_street_number ,ad1.ca_street_name b_street_name ,ad1.ca_city b_city ,ad1.ca_zip b_zip ,ad2.ca_street_number c_street_number ,ad2.ca_street_name c_street_name ,ad2.ca_city c_city ,ad2.ca_zip c_zip ,d1.d_year as syear ,d2.d_year as fsyear ,d3.d_year s2year ,count(*) cnt ,sum(ss_wholesale_cost) s1 ,sum(ss_list_price) s2 ,sum(ss_coupon_amt) s3 FROM store_sales ,store_returns ,cs_ui ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,customer ,customer_demographics cd1 ,customer_demographics cd2 ,promotion ,household_demographics hd1 ,household_demographics hd2 ,customer_address ad1 ,customer_address ad2 ,income_band ib1 ,income_band ib2 ,item WHERE ss_store_sk = s_store_sk AND ss_sold_date_sk = d1.d_date_sk AND ss_customer_sk = c_customer_sk AND ss_cdemo_sk= cd1.cd_demo_sk AND ss_hdemo_sk = hd1.hd_demo_sk AND ss_addr_sk = ad1.ca_address_sk and ss_item_sk = i_item_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and ss_item_sk = cs_ui.cs_item_sk and c_current_cdemo_sk = cd2.cd_demo_sk AND c_current_hdemo_sk = hd2.hd_demo_sk AND c_current_addr_sk = ad2.ca_address_sk and c_first_sales_date_sk = d2.d_date_sk and c_first_shipto_date_sk = d3.d_date_sk and ss_promo_sk = p_promo_sk and hd1.hd_income_band_sk = ib1.ib_income_band_sk and hd2.hd_income_band_sk = ib2.ib_income_band_sk and cd1.cd_marital_status <> cd2.cd_marital_status and i_color in ('ivory','purple','almond','bisque','lawn','azure') and i_current_price between 60 and 60 + 10 and i_current_price between 60 + 1 and 60 + 15 group by i_product_name ,i_item_sk ,s_store_name ,s_zip ,ad1.ca_street_number ,ad1.ca_street_name ,ad1.ca_city ,ad1.ca_zip ,ad2.ca_street_number ,ad2.ca_street_name ,ad2.ca_city ,ad2.ca_zip ,d1.d_year ,d2.d_year ,d3.d_year ) select cs1.product_name ,cs1.store_name ,cs1.store_zip ,cs1.b_street_number ,cs1.b_street_name ,cs1.b_city ,cs1.b_zip ,cs1.c_street_number ,cs1.c_street_name ,cs1.c_city ,cs1.c_zip ,cs1.syear ,cs1.cnt ,cs1.s1 as s11 ,cs1.s2 as s21 ,cs1.s3 as s31 ,cs2.s1 as s12 ,cs2.s2 as s22 ,cs2.s3 as s32 ,cs2.syear ,cs2.cnt from cross_sales cs1,cross_sales cs2 where cs1.item_sk=cs2.item_sk and cs1.syear = 2001 and cs2.syear = 2001 + 1 and cs2.cnt <= cs1.cnt and cs1.store_name = cs2.store_name and cs1.store_zip = cs2.store_zip order by cs1.product_name ,cs1.store_name ,cs2.cnt ,cs1.s1 ,cs2.s1""", "q65" -> """ select s_store_name, i_item_desc, sc.revenue, i_current_price, i_wholesale_cost, i_brand from store, item, (select ss_store_sk, avg(revenue) as ave from (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue from store_sales, date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1185 and 1185+11 group by ss_store_sk, ss_item_sk) sa group by ss_store_sk) sb, (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue from store_sales, date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1185 and 1185+11 group by ss_store_sk, ss_item_sk) sc where sb.ss_store_sk = sc.ss_store_sk and sc.revenue <= 0.1 * sb.ave and s_store_sk = sc.ss_store_sk and i_item_sk = sc.ss_item_sk order by s_store_name, i_item_desc limit 100""", "q66" -> """ select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,ship_carriers ,year ,sum(jan_sales) as jan_sales ,sum(feb_sales) as feb_sales ,sum(mar_sales) as mar_sales ,sum(apr_sales) as apr_sales ,sum(may_sales) as may_sales ,sum(jun_sales) as jun_sales ,sum(jul_sales) as jul_sales ,sum(aug_sales) as aug_sales ,sum(sep_sales) as sep_sales ,sum(oct_sales) as oct_sales ,sum(nov_sales) as nov_sales ,sum(dec_sales) as dec_sales ,sum(jan_sales/w_warehouse_sq_ft) as jan_sales_per_sq_foot ,sum(feb_sales/w_warehouse_sq_ft) as feb_sales_per_sq_foot ,sum(mar_sales/w_warehouse_sq_ft) as mar_sales_per_sq_foot ,sum(apr_sales/w_warehouse_sq_ft) as apr_sales_per_sq_foot ,sum(may_sales/w_warehouse_sq_ft) as may_sales_per_sq_foot ,sum(jun_sales/w_warehouse_sq_ft) as jun_sales_per_sq_foot ,sum(jul_sales/w_warehouse_sq_ft) as jul_sales_per_sq_foot ,sum(aug_sales/w_warehouse_sq_ft) as aug_sales_per_sq_foot ,sum(sep_sales/w_warehouse_sq_ft) as sep_sales_per_sq_foot ,sum(oct_sales/w_warehouse_sq_ft) as oct_sales_per_sq_foot ,sum(nov_sales/w_warehouse_sq_ft) as nov_sales_per_sq_foot ,sum(dec_sales/w_warehouse_sq_ft) as dec_sales_per_sq_foot ,sum(jan_net) as jan_net ,sum(feb_net) as feb_net ,sum(mar_net) as mar_net ,sum(apr_net) as apr_net ,sum(may_net) as may_net ,sum(jun_net) as jun_net ,sum(jul_net) as jul_net ,sum(aug_net) as aug_net ,sum(sep_net) as sep_net ,sum(oct_net) as oct_net ,sum(nov_net) as nov_net ,sum(dec_net) as dec_net from ( select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,'FEDEX' || ',' || 'MSC' as ship_carriers ,d_year as year ,sum(case when d_moy = 1 then ws_ext_list_price* ws_quantity else 0 end) as jan_sales ,sum(case when d_moy = 2 then ws_ext_list_price* ws_quantity else 0 end) as feb_sales ,sum(case when d_moy = 3 then ws_ext_list_price* ws_quantity else 0 end) as mar_sales ,sum(case when d_moy = 4 then ws_ext_list_price* ws_quantity else 0 end) as apr_sales ,sum(case when d_moy = 5 then ws_ext_list_price* ws_quantity else 0 end) as may_sales ,sum(case when d_moy = 6 then ws_ext_list_price* ws_quantity else 0 end) as jun_sales ,sum(case when d_moy = 7 then ws_ext_list_price* ws_quantity else 0 end) as jul_sales ,sum(case when d_moy = 8 then ws_ext_list_price* ws_quantity else 0 end) as aug_sales ,sum(case when d_moy = 9 then ws_ext_list_price* ws_quantity else 0 end) as sep_sales ,sum(case when d_moy = 10 then ws_ext_list_price* ws_quantity else 0 end) as oct_sales ,sum(case when d_moy = 11 then ws_ext_list_price* ws_quantity else 0 end) as nov_sales ,sum(case when d_moy = 12 then ws_ext_list_price* ws_quantity else 0 end) as dec_sales ,sum(case when d_moy = 1 then ws_net_profit * ws_quantity else 0 end) as jan_net ,sum(case when d_moy = 2 then ws_net_profit * ws_quantity else 0 end) as feb_net ,sum(case when d_moy = 3 then ws_net_profit * ws_quantity else 0 end) as mar_net ,sum(case when d_moy = 4 then ws_net_profit * ws_quantity else 0 end) as apr_net ,sum(case when d_moy = 5 then ws_net_profit * ws_quantity else 0 end) as may_net ,sum(case when d_moy = 6 then ws_net_profit * ws_quantity else 0 end) as jun_net ,sum(case when d_moy = 7 then ws_net_profit * ws_quantity else 0 end) as jul_net ,sum(case when d_moy = 8 then ws_net_profit * ws_quantity else 0 end) as aug_net ,sum(case when d_moy = 9 then ws_net_profit * ws_quantity else 0 end) as sep_net ,sum(case when d_moy = 10 then ws_net_profit * ws_quantity else 0 end) as oct_net ,sum(case when d_moy = 11 then ws_net_profit * ws_quantity else 0 end) as nov_net ,sum(case when d_moy = 12 then ws_net_profit * ws_quantity else 0 end) as dec_net from web_sales ,warehouse ,date_dim ,time_dim ,ship_mode where ws_warehouse_sk = w_warehouse_sk and ws_sold_date_sk = d_date_sk and ws_sold_time_sk = t_time_sk and ws_ship_mode_sk = sm_ship_mode_sk and d_year = 2002 and t_time between 2662 and 2662+28800 and sm_carrier in ('FEDEX','MSC') group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,d_year union all select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,'FEDEX' || ',' || 'MSC' as ship_carriers ,d_year as year ,sum(case when d_moy = 1 then cs_ext_list_price* cs_quantity else 0 end) as jan_sales ,sum(case when d_moy = 2 then cs_ext_list_price* cs_quantity else 0 end) as feb_sales ,sum(case when d_moy = 3 then cs_ext_list_price* cs_quantity else 0 end) as mar_sales ,sum(case when d_moy = 4 then cs_ext_list_price* cs_quantity else 0 end) as apr_sales ,sum(case when d_moy = 5 then cs_ext_list_price* cs_quantity else 0 end) as may_sales ,sum(case when d_moy = 6 then cs_ext_list_price* cs_quantity else 0 end) as jun_sales ,sum(case when d_moy = 7 then cs_ext_list_price* cs_quantity else 0 end) as jul_sales ,sum(case when d_moy = 8 then cs_ext_list_price* cs_quantity else 0 end) as aug_sales ,sum(case when d_moy = 9 then cs_ext_list_price* cs_quantity else 0 end) as sep_sales ,sum(case when d_moy = 10 then cs_ext_list_price* cs_quantity else 0 end) as oct_sales ,sum(case when d_moy = 11 then cs_ext_list_price* cs_quantity else 0 end) as nov_sales ,sum(case when d_moy = 12 then cs_ext_list_price* cs_quantity else 0 end) as dec_sales ,sum(case when d_moy = 1 then cs_net_profit * cs_quantity else 0 end) as jan_net ,sum(case when d_moy = 2 then cs_net_profit * cs_quantity else 0 end) as feb_net ,sum(case when d_moy = 3 then cs_net_profit * cs_quantity else 0 end) as mar_net ,sum(case when d_moy = 4 then cs_net_profit * cs_quantity else 0 end) as apr_net ,sum(case when d_moy = 5 then cs_net_profit * cs_quantity else 0 end) as may_net ,sum(case when d_moy = 6 then cs_net_profit * cs_quantity else 0 end) as jun_net ,sum(case when d_moy = 7 then cs_net_profit * cs_quantity else 0 end) as jul_net ,sum(case when d_moy = 8 then cs_net_profit * cs_quantity else 0 end) as aug_net ,sum(case when d_moy = 9 then cs_net_profit * cs_quantity else 0 end) as sep_net ,sum(case when d_moy = 10 then cs_net_profit * cs_quantity else 0 end) as oct_net ,sum(case when d_moy = 11 then cs_net_profit * cs_quantity else 0 end) as nov_net ,sum(case when d_moy = 12 then cs_net_profit * cs_quantity else 0 end) as dec_net from catalog_sales ,warehouse ,date_dim ,time_dim ,ship_mode where cs_warehouse_sk = w_warehouse_sk and cs_sold_date_sk = d_date_sk and cs_sold_time_sk = t_time_sk and cs_ship_mode_sk = sm_ship_mode_sk and d_year = 2002 and t_time between 2662 AND 2662+28800 and sm_carrier in ('FEDEX','MSC') group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,d_year ) x group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,ship_carriers ,year order by w_warehouse_name limit 100""", "q67" -> """ select * from (select i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sumsales ,rank() over (partition by i_category order by sumsales desc) rk from (select i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sum(coalesce(ss_sales_price*ss_quantity,0)) sumsales from store_sales ,date_dim ,store ,item where ss_sold_date_sk=d_date_sk and ss_item_sk=i_item_sk and ss_store_sk = s_store_sk and d_month_seq between 1177 and 1177+11 group by rollup(i_category, i_class, i_brand, i_product_name, d_year, d_qoy, d_moy,s_store_id))dw1) dw2 where rk <= 100 order by i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sumsales ,rk limit 100""", "q68" -> """ select c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number ,extended_price ,extended_tax ,list_price from (select ss_ticket_number ,ss_customer_sk ,ca_city bought_city ,sum(ss_ext_sales_price) extended_price ,sum(ss_ext_list_price) list_price ,sum(ss_ext_tax) extended_tax from store_sales ,date_dim ,store ,household_demographics ,customer_address where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and store_sales.ss_addr_sk = customer_address.ca_address_sk and date_dim.d_dom between 1 and 2 and (household_demographics.hd_dep_count = 5 or household_demographics.hd_vehicle_count= 4) and date_dim.d_year in (1999,1999+1,1999+2) and store.s_city in ('Lodi','Richmond') group by ss_ticket_number ,ss_customer_sk ,ss_addr_sk,ca_city) dn ,customer ,customer_address current_addr where ss_customer_sk = c_customer_sk and customer.c_current_addr_sk = current_addr.ca_address_sk and current_addr.ca_city <> bought_city order by c_last_name ,ss_ticket_number limit 100""", "q69" -> """ select cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3 from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and ca_state in ('IL','FL','SD') and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 1999 and d_moy between 1 and 1+2) and (not exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 1999 and d_moy between 1 and 1+2) and not exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 1999 and d_moy between 1 and 1+2)) group by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating order by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating limit 100""", "q70" -> """ select sum(ss_net_profit) as total_sum ,s_state ,s_county ,grouping(s_state)+grouping(s_county) as lochierarchy ,rank() over ( partition by grouping(s_state)+grouping(s_county), case when grouping(s_county) = 0 then s_state end order by sum(ss_net_profit) desc) as rank_within_parent from store_sales ,date_dim d1 ,store where d1.d_month_seq between 1206 and 1206+11 and d1.d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk and s_state in ( select s_state from (select s_state as s_state, rank() over ( partition by s_state order by sum(ss_net_profit) desc) as ranking from store_sales, store, date_dim where d_month_seq between 1206 and 1206+11 and d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk group by s_state ) tmp1 where ranking <= 5 ) group by rollup(s_state,s_county) order by lochierarchy desc ,case when lochierarchy = 0 then s_state end ,rank_within_parent limit 100""", "q71" -> """ select i_brand_id brand_id, i_brand brand,t_hour,t_minute, sum(ext_price) ext_price from item, (select ws_ext_sales_price as ext_price, ws_sold_date_sk as sold_date_sk, ws_item_sk as sold_item_sk, ws_sold_time_sk as time_sk from web_sales,date_dim where d_date_sk = ws_sold_date_sk and d_moy=11 and d_year=1999 union all select cs_ext_sales_price as ext_price, cs_sold_date_sk as sold_date_sk, cs_item_sk as sold_item_sk, cs_sold_time_sk as time_sk from catalog_sales,date_dim where d_date_sk = cs_sold_date_sk and d_moy=11 and d_year=1999 union all select ss_ext_sales_price as ext_price, ss_sold_date_sk as sold_date_sk, ss_item_sk as sold_item_sk, ss_sold_time_sk as time_sk from store_sales,date_dim where d_date_sk = ss_sold_date_sk and d_moy=11 and d_year=1999 ) tmp,time_dim where sold_item_sk = i_item_sk and i_manager_id=1 and time_sk = t_time_sk and (t_meal_time = 'breakfast' or t_meal_time = 'dinner') group by i_brand, i_brand_id,t_hour,t_minute order by ext_price desc, i_brand_id """, "q72" -> """ select i_item_desc ,w_warehouse_name ,d1.d_week_seq ,sum(case when p_promo_sk is null then 1 else 0 end) no_promo ,sum(case when p_promo_sk is not null then 1 else 0 end) promo ,count(*) total_cnt from catalog_sales join inventory on (cs_item_sk = inv_item_sk) join warehouse on (w_warehouse_sk=inv_warehouse_sk) join item on (i_item_sk = cs_item_sk) join customer_demographics on (cs_bill_cdemo_sk = cd_demo_sk) join household_demographics on (cs_bill_hdemo_sk = hd_demo_sk) join date_dim d1 on (cs_sold_date_sk = d1.d_date_sk) join date_dim d2 on (inv_date_sk = d2.d_date_sk) join date_dim d3 on (cs_ship_date_sk = d3.d_date_sk) left outer join promotion on (cs_promo_sk=p_promo_sk) left outer join catalog_returns on (cr_item_sk = cs_item_sk and cr_order_number = cs_order_number) where d1.d_week_seq = d2.d_week_seq and inv_quantity_on_hand < cs_quantity and d3.d_date > d1.d_date + interval 5 days and hd_buy_potential = '1001-5000' and d1.d_year = 2000 and cd_marital_status = 'S' group by i_item_desc,w_warehouse_name,d1.d_week_seq order by total_cnt desc, i_item_desc, w_warehouse_name, d_week_seq limit 100""", "q73" -> """ select c_last_name ,c_first_name ,c_salutation ,c_preferred_cust_flag ,ss_ticket_number ,cnt from (select ss_ticket_number ,ss_customer_sk ,count(*) cnt from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and date_dim.d_dom between 1 and 2 and (household_demographics.hd_buy_potential = '1001-5000' or household_demographics.hd_buy_potential = 'Unknown') and household_demographics.hd_vehicle_count > 0 and case when household_demographics.hd_vehicle_count > 0 then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end > 1 and date_dim.d_year in (1999,1999+1,1999+2) and store.s_county in ('Humboldt County','Hickman County','Galax city','Abbeville County') group by ss_ticket_number,ss_customer_sk) dj,customer where ss_customer_sk = c_customer_sk and cnt between 1 and 5 order by cnt desc, c_last_name asc""", "q74" -> """ with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,d_year as year ,max(ss_net_paid) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (2001,2001+1) group by c_customer_id ,c_first_name ,c_last_name ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,d_year as year ,max(ws_net_paid) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year in (2001,2001+1) group by c_customer_id ,c_first_name ,c_last_name ,d_year ) select t_s_secyear.customer_id, t_s_secyear.customer_first_name, t_s_secyear.customer_last_name from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.sale_type = 's' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_w_secyear.sale_type = 'w' and t_s_firstyear.year = 2001 and t_s_secyear.year = 2001+1 and t_w_firstyear.year = 2001 and t_w_secyear.year = 2001+1 and t_s_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end order by 3,1,2 limit 100""", "q75" -> """ WITH all_sales AS ( SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,SUM(sales_cnt) AS sales_cnt ,SUM(sales_amt) AS sales_amt FROM (SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,cs_quantity - COALESCE(cr_return_quantity,0) AS sales_cnt ,cs_ext_sales_price - COALESCE(cr_return_amount,0.0) AS sales_amt FROM catalog_sales JOIN item ON i_item_sk=cs_item_sk JOIN date_dim ON d_date_sk=cs_sold_date_sk LEFT JOIN catalog_returns ON (cs_order_number=cr_order_number AND cs_item_sk=cr_item_sk) WHERE i_category='Books' UNION SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,ss_quantity - COALESCE(sr_return_quantity,0) AS sales_cnt ,ss_ext_sales_price - COALESCE(sr_return_amt,0.0) AS sales_amt FROM store_sales JOIN item ON i_item_sk=ss_item_sk JOIN date_dim ON d_date_sk=ss_sold_date_sk LEFT JOIN store_returns ON (ss_ticket_number=sr_ticket_number AND ss_item_sk=sr_item_sk) WHERE i_category='Books' UNION SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,ws_quantity - COALESCE(wr_return_quantity,0) AS sales_cnt ,ws_ext_sales_price - COALESCE(wr_return_amt,0.0) AS sales_amt FROM web_sales JOIN item ON i_item_sk=ws_item_sk JOIN date_dim ON d_date_sk=ws_sold_date_sk LEFT JOIN web_returns ON (ws_order_number=wr_order_number AND ws_item_sk=wr_item_sk) WHERE i_category='Books') sales_detail GROUP BY d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id) SELECT prev_yr.d_year AS prev_year ,curr_yr.d_year AS year ,curr_yr.i_brand_id ,curr_yr.i_class_id ,curr_yr.i_category_id ,curr_yr.i_manufact_id ,prev_yr.sales_cnt AS prev_yr_cnt ,curr_yr.sales_cnt AS curr_yr_cnt ,curr_yr.sales_cnt-prev_yr.sales_cnt AS sales_cnt_diff ,curr_yr.sales_amt-prev_yr.sales_amt AS sales_amt_diff FROM all_sales curr_yr, all_sales prev_yr WHERE curr_yr.i_brand_id=prev_yr.i_brand_id AND curr_yr.i_class_id=prev_yr.i_class_id AND curr_yr.i_category_id=prev_yr.i_category_id AND curr_yr.i_manufact_id=prev_yr.i_manufact_id AND curr_yr.d_year=2001 AND prev_yr.d_year=2001-1 AND CAST(curr_yr.sales_cnt AS DECIMAL(17,2))/CAST(prev_yr.sales_cnt AS DECIMAL(17,2))<0.9 ORDER BY sales_cnt_diff,sales_amt_diff limit 100""", "q76" -> """ select channel, col_name, d_year, d_qoy, i_category, COUNT(*) sales_cnt, SUM(ext_sales_price) sales_amt FROM ( SELECT 'store' as channel, 'ss_promo_sk' col_name, d_year, d_qoy, i_category, ss_ext_sales_price ext_sales_price FROM store_sales, item, date_dim WHERE ss_promo_sk IS NULL AND ss_sold_date_sk=d_date_sk AND ss_item_sk=i_item_sk UNION ALL SELECT 'web' as channel, 'ws_ship_addr_sk' col_name, d_year, d_qoy, i_category, ws_ext_sales_price ext_sales_price FROM web_sales, item, date_dim WHERE ws_ship_addr_sk IS NULL AND ws_sold_date_sk=d_date_sk AND ws_item_sk=i_item_sk UNION ALL SELECT 'catalog' as channel, 'cs_ship_customer_sk' col_name, d_year, d_qoy, i_category, cs_ext_sales_price ext_sales_price FROM catalog_sales, item, date_dim WHERE cs_ship_customer_sk IS NULL AND cs_sold_date_sk=d_date_sk AND cs_item_sk=i_item_sk) foo GROUP BY channel, col_name, d_year, d_qoy, i_category ORDER BY channel, col_name, d_year, d_qoy, i_category limit 100""", "q77" -> """ with ss as (select s_store_sk, sum(ss_ext_sales_price) as sales, sum(ss_net_profit) as profit from store_sales, date_dim, store where ss_sold_date_sk = d_date_sk and d_date between cast('2001-08-16' as date) and (cast('2001-08-16' as date) + INTERVAL 30 days) and ss_store_sk = s_store_sk group by s_store_sk) , sr as (select s_store_sk, sum(sr_return_amt) as returns, sum(sr_net_loss) as profit_loss from store_returns, date_dim, store where sr_returned_date_sk = d_date_sk and d_date between cast('2001-08-16' as date) and (cast('2001-08-16' as date) + INTERVAL 30 days) and sr_store_sk = s_store_sk group by s_store_sk), cs as (select cs_call_center_sk, sum(cs_ext_sales_price) as sales, sum(cs_net_profit) as profit from catalog_sales, date_dim where cs_sold_date_sk = d_date_sk and d_date between cast('2001-08-16' as date) and (cast('2001-08-16' as date) + INTERVAL 30 days) group by cs_call_center_sk ), cr as (select cr_call_center_sk, sum(cr_return_amount) as returns, sum(cr_net_loss) as profit_loss from catalog_returns, date_dim where cr_returned_date_sk = d_date_sk and d_date between cast('2001-08-16' as date) and (cast('2001-08-16' as date) + INTERVAL 30 days) group by cr_call_center_sk ), ws as ( select wp_web_page_sk, sum(ws_ext_sales_price) as sales, sum(ws_net_profit) as profit from web_sales, date_dim, web_page where ws_sold_date_sk = d_date_sk and d_date between cast('2001-08-16' as date) and (cast('2001-08-16' as date) + INTERVAL 30 days) and ws_web_page_sk = wp_web_page_sk group by wp_web_page_sk), wr as (select wp_web_page_sk, sum(wr_return_amt) as returns, sum(wr_net_loss) as profit_loss from web_returns, date_dim, web_page where wr_returned_date_sk = d_date_sk and d_date between cast('2001-08-16' as date) and (cast('2001-08-16' as date) + INTERVAL 30 days) and wr_web_page_sk = wp_web_page_sk group by wp_web_page_sk) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , ss.s_store_sk as id , sales , coalesce(returns, 0) as returns , (profit - coalesce(profit_loss,0)) as profit from ss left join sr on ss.s_store_sk = sr.s_store_sk union all select 'catalog channel' as channel , cs_call_center_sk as id , sales , returns , (profit - profit_loss) as profit from cs , cr union all select 'web channel' as channel , ws.wp_web_page_sk as id , sales , coalesce(returns, 0) returns , (profit - coalesce(profit_loss,0)) as profit from ws left join wr on ws.wp_web_page_sk = wr.wp_web_page_sk ) x group by rollup (channel, id) order by channel ,id limit 100""", "q78" -> """ with ws as (select d_year AS ws_sold_year, ws_item_sk, ws_bill_customer_sk ws_customer_sk, sum(ws_quantity) ws_qty, sum(ws_wholesale_cost) ws_wc, sum(ws_sales_price) ws_sp from web_sales left join web_returns on wr_order_number=ws_order_number and ws_item_sk=wr_item_sk join date_dim on ws_sold_date_sk = d_date_sk where wr_order_number is null group by d_year, ws_item_sk, ws_bill_customer_sk ), cs as (select d_year AS cs_sold_year, cs_item_sk, cs_bill_customer_sk cs_customer_sk, sum(cs_quantity) cs_qty, sum(cs_wholesale_cost) cs_wc, sum(cs_sales_price) cs_sp from catalog_sales left join catalog_returns on cr_order_number=cs_order_number and cs_item_sk=cr_item_sk join date_dim on cs_sold_date_sk = d_date_sk where cr_order_number is null group by d_year, cs_item_sk, cs_bill_customer_sk ), ss as (select d_year AS ss_sold_year, ss_item_sk, ss_customer_sk, sum(ss_quantity) ss_qty, sum(ss_wholesale_cost) ss_wc, sum(ss_sales_price) ss_sp from store_sales left join store_returns on sr_ticket_number=ss_ticket_number and ss_item_sk=sr_item_sk join date_dim on ss_sold_date_sk = d_date_sk where sr_ticket_number is null group by d_year, ss_item_sk, ss_customer_sk ) select ss_item_sk, round(ss_qty/(coalesce(ws_qty,0)+coalesce(cs_qty,0)),2) ratio, ss_qty store_qty, ss_wc store_wholesale_cost, ss_sp store_sales_price, coalesce(ws_qty,0)+coalesce(cs_qty,0) other_chan_qty, coalesce(ws_wc,0)+coalesce(cs_wc,0) other_chan_wholesale_cost, coalesce(ws_sp,0)+coalesce(cs_sp,0) other_chan_sales_price from ss left join ws on (ws_sold_year=ss_sold_year and ws_item_sk=ss_item_sk and ws_customer_sk=ss_customer_sk) left join cs on (cs_sold_year=ss_sold_year and cs_item_sk=ss_item_sk and cs_customer_sk=ss_customer_sk) where (coalesce(ws_qty,0)>0 or coalesce(cs_qty, 0)>0) and ss_sold_year=2000 order by ss_item_sk, ss_qty desc, ss_wc desc, ss_sp desc, other_chan_qty, other_chan_wholesale_cost, other_chan_sales_price, ratio limit 100""", "q79" -> """ select c_last_name,c_first_name,substr(s_city,1,30),ss_ticket_number,amt,profit from (select ss_ticket_number ,ss_customer_sk ,store.s_city ,sum(ss_coupon_amt) amt ,sum(ss_net_profit) profit from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and (household_demographics.hd_dep_count = 5 or household_demographics.hd_vehicle_count > -1) and date_dim.d_dow = 1 and date_dim.d_year in (1999,1999+1,1999+2) and store.s_number_employees between 200 and 295 group by ss_ticket_number,ss_customer_sk,ss_addr_sk,store.s_city) ms,customer where ss_customer_sk = c_customer_sk order by c_last_name,c_first_name,substr(s_city,1,30), profit limit 100""", "q80" -> """ with ssr as (select s_store_id as store_id, sum(ss_ext_sales_price) as sales, sum(coalesce(sr_return_amt, 0)) as returns, sum(ss_net_profit - coalesce(sr_net_loss, 0)) as profit from store_sales left outer join store_returns on (ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number), date_dim, store, item, promotion where ss_sold_date_sk = d_date_sk and d_date between cast('2001-08-19' as date) and (cast('2001-08-19' as date) + INTERVAL 60 days) and ss_store_sk = s_store_sk and ss_item_sk = i_item_sk and i_current_price > 50 and ss_promo_sk = p_promo_sk and p_channel_tv = 'N' group by s_store_id) , csr as (select cp_catalog_page_id as catalog_page_id, sum(cs_ext_sales_price) as sales, sum(coalesce(cr_return_amount, 0)) as returns, sum(cs_net_profit - coalesce(cr_net_loss, 0)) as profit from catalog_sales left outer join catalog_returns on (cs_item_sk = cr_item_sk and cs_order_number = cr_order_number), date_dim, catalog_page, item, promotion where cs_sold_date_sk = d_date_sk and d_date between cast('2001-08-19' as date) and (cast('2001-08-19' as date) + INTERVAL 60 days) and cs_catalog_page_sk = cp_catalog_page_sk and cs_item_sk = i_item_sk and i_current_price > 50 and cs_promo_sk = p_promo_sk and p_channel_tv = 'N' group by cp_catalog_page_id) , wsr as (select web_site_id, sum(ws_ext_sales_price) as sales, sum(coalesce(wr_return_amt, 0)) as returns, sum(ws_net_profit - coalesce(wr_net_loss, 0)) as profit from web_sales left outer join web_returns on (ws_item_sk = wr_item_sk and ws_order_number = wr_order_number), date_dim, web_site, item, promotion where ws_sold_date_sk = d_date_sk and d_date between cast('2001-08-19' as date) and (cast('2001-08-19' as date) + INTERVAL 60 days) and ws_web_site_sk = web_site_sk and ws_item_sk = i_item_sk and i_current_price > 50 and ws_promo_sk = p_promo_sk and p_channel_tv = 'N' group by web_site_id) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , 'store' || store_id as id , sales , returns , profit from ssr union all select 'catalog channel' as channel , 'catalog_page' || catalog_page_id as id , sales , returns , profit from csr union all select 'web channel' as channel , 'web_site' || web_site_id as id , sales , returns , profit from wsr ) x group by rollup (channel, id) order by channel ,id limit 100""", "q81" -> """ with customer_total_return as (select cr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, sum(cr_return_amt_inc_tax) as ctr_total_return from catalog_returns ,date_dim ,customer_address where cr_returned_date_sk = d_date_sk and d_year =1999 and cr_returning_addr_sk = ca_address_sk group by cr_returning_customer_sk ,ca_state ) select c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset ,ca_location_type,ctr_total_return from customer_total_return ctr1 ,customer_address ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_state = ctr2.ctr_state) and ca_address_sk = c_current_addr_sk and ca_state = 'MO' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset ,ca_location_type,ctr_total_return limit 100""", "q82" -> """ select i_item_id ,i_item_desc ,i_current_price from item, inventory, date_dim, store_sales where i_current_price between 68 and 68+30 and inv_item_sk = i_item_sk and d_date_sk=inv_date_sk and d_date between cast('2002-05-08' as date) and (cast('2002-05-08' as date) + INTERVAL 60 days) and i_manufact_id in (562,370,230,182) and inv_quantity_on_hand between 100 and 500 and ss_item_sk = i_item_sk group by i_item_id,i_item_desc,i_current_price order by i_item_id limit 100""", "q83" -> """ with sr_items as (select i_item_id item_id, sum(sr_return_quantity) sr_item_qty from store_returns, item, date_dim where sr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('2000-02-20','2000-10-08','2000-11-04'))) and sr_returned_date_sk = d_date_sk group by i_item_id), cr_items as (select i_item_id item_id, sum(cr_return_quantity) cr_item_qty from catalog_returns, item, date_dim where cr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('2000-02-20','2000-10-08','2000-11-04'))) and cr_returned_date_sk = d_date_sk group by i_item_id), wr_items as (select i_item_id item_id, sum(wr_return_quantity) wr_item_qty from web_returns, item, date_dim where wr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('2000-02-20','2000-10-08','2000-11-04'))) and wr_returned_date_sk = d_date_sk group by i_item_id) select sr_items.item_id ,sr_item_qty ,sr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 sr_dev ,cr_item_qty ,cr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 cr_dev ,wr_item_qty ,wr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 wr_dev ,(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 average from sr_items ,cr_items ,wr_items where sr_items.item_id=cr_items.item_id and sr_items.item_id=wr_items.item_id order by sr_items.item_id ,sr_item_qty limit 100""", "q84" -> """ select c_customer_id as customer_id , coalesce(c_last_name,'') || ', ' || coalesce(c_first_name,'') as customername from customer ,customer_address ,customer_demographics ,household_demographics ,income_band ,store_returns where ca_city = 'Buena Vista' and c_current_addr_sk = ca_address_sk and ib_lower_bound >= 49786 and ib_upper_bound <= 49786 + 50000 and ib_income_band_sk = hd_income_band_sk and cd_demo_sk = c_current_cdemo_sk and hd_demo_sk = c_current_hdemo_sk and sr_cdemo_sk = cd_demo_sk order by c_customer_id limit 100""", "q85" -> """ select substr(r_reason_desc,1,20) ,avg(ws_quantity) ,avg(wr_refunded_cash) ,avg(wr_fee) from web_sales, web_returns, web_page, customer_demographics cd1, customer_demographics cd2, customer_address, date_dim, reason where ws_web_page_sk = wp_web_page_sk and ws_item_sk = wr_item_sk and ws_order_number = wr_order_number and ws_sold_date_sk = d_date_sk and d_year = 2001 and cd1.cd_demo_sk = wr_refunded_cdemo_sk and cd2.cd_demo_sk = wr_returning_cdemo_sk and ca_address_sk = wr_refunded_addr_sk and r_reason_sk = wr_reason_sk and ( ( cd1.cd_marital_status = 'D' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = '4 yr Degree' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 100.00 and 150.00 ) or ( cd1.cd_marital_status = 'M' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = 'Primary' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 50.00 and 100.00 ) or ( cd1.cd_marital_status = 'U' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = '2 yr Degree' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 150.00 and 200.00 ) ) and ( ( ca_country = 'United States' and ca_state in ('IA', 'ND', 'FL') and ws_net_profit between 100 and 200 ) or ( ca_country = 'United States' and ca_state in ('OH', 'MS', 'VA') and ws_net_profit between 150 and 300 ) or ( ca_country = 'United States' and ca_state in ('MN', 'LA', 'TX') and ws_net_profit between 50 and 250 ) ) group by r_reason_desc order by substr(r_reason_desc,1,20) ,avg(ws_quantity) ,avg(wr_refunded_cash) ,avg(wr_fee) limit 100""", "q86" -> """ select sum(ws_net_paid) as total_sum ,i_category ,i_class ,grouping(i_category)+grouping(i_class) as lochierarchy ,rank() over ( partition by grouping(i_category)+grouping(i_class), case when grouping(i_class) = 0 then i_category end order by sum(ws_net_paid) desc) as rank_within_parent from web_sales ,date_dim d1 ,item where d1.d_month_seq between 1217 and 1217+11 and d1.d_date_sk = ws_sold_date_sk and i_item_sk = ws_item_sk group by rollup(i_category,i_class) order by lochierarchy desc, case when lochierarchy = 0 then i_category end, rank_within_parent limit 100""", "q87" -> """ select count(*) from ((select distinct c_last_name, c_first_name, d_date from store_sales, date_dim, customer where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_customer_sk = customer.c_customer_sk and d_month_seq between 1224 and 1224+11) except (select distinct c_last_name, c_first_name, d_date from catalog_sales, date_dim, customer where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1224 and 1224+11) except (select distinct c_last_name, c_first_name, d_date from web_sales, date_dim, customer where web_sales.ws_sold_date_sk = date_dim.d_date_sk and web_sales.ws_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1224 and 1224+11) ) cool_cust""", "q88" -> """ select * from (select count(*) h8_30_to_9 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 8 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s1, (select count(*) h9_to_9_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 9 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s2, (select count(*) h9_30_to_10 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 9 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s3, (select count(*) h10_to_10_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 10 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s4, (select count(*) h10_30_to_11 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 10 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s5, (select count(*) h11_to_11_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 11 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s6, (select count(*) h11_30_to_12 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 11 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s7, (select count(*) h12_to_12_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 12 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or (household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) and store.s_store_name = 'ese') s8""", "q89" -> """ select * from( select i_category, i_class, i_brand, s_store_name, s_company_name, d_moy, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_category, i_brand, s_store_name, s_company_name) avg_monthly_sales from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_year in (2001) and ((i_category in ('Children','Home','Women') and i_class in ('toddlers','flatware','fragrances') ) or (i_category in ('Music','Electronics','Shoes') and i_class in ('country','dvd/vcr players','mens') )) group by i_category, i_class, i_brand, s_store_name, s_company_name, d_moy) tmp1 where case when (avg_monthly_sales <> 0) then (abs(sum_sales - avg_monthly_sales) / avg_monthly_sales) else null end > 0.1 order by sum_sales - avg_monthly_sales, s_store_name limit 100""", "q90" -> """ select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio from ( select count(*) amc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 7 and 7+1 and household_demographics.hd_dep_count = 1 and web_page.wp_char_count between 5000 and 5200) at, ( select count(*) pmc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 20 and 20+1 and household_demographics.hd_dep_count = 1 and web_page.wp_char_count between 5000 and 5200) pt order by am_pm_ratio limit 100""", "q91" -> """ select cc_call_center_id Call_Center, cc_name Call_Center_Name, cc_manager Manager, sum(cr_net_loss) Returns_Loss from call_center, catalog_returns, date_dim, customer, customer_address, customer_demographics, household_demographics where cr_call_center_sk = cc_call_center_sk and cr_returned_date_sk = d_date_sk and cr_returning_customer_sk= c_customer_sk and cd_demo_sk = c_current_cdemo_sk and hd_demo_sk = c_current_hdemo_sk and ca_address_sk = c_current_addr_sk and d_year = 1998 and d_moy = 12 and ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') or(cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) and hd_buy_potential like 'Unknown%' and ca_gmt_offset = -6 group by cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status order by sum(cr_net_loss) desc""", "q92" -> """ select sum(ws_ext_discount_amt) as `Excess Discount Amount` from web_sales ,item ,date_dim where i_manufact_id = 172 and i_item_sk = ws_item_sk and d_date between '1999-01-12' and (cast('1999-01-12' as date) + INTERVAL 90 days) and d_date_sk = ws_sold_date_sk and ws_ext_discount_amt > ( SELECT 1.3 * avg(ws_ext_discount_amt) FROM web_sales ,date_dim WHERE ws_item_sk = i_item_sk and d_date between '1999-01-12' and (cast('1999-01-12' as date) + INTERVAL 90 days) and d_date_sk = ws_sold_date_sk ) order by sum(ws_ext_discount_amt) limit 100""", "q93" -> """ select ss_customer_sk ,sum(act_sales) sumsales from (select ss_item_sk ,ss_ticket_number ,ss_customer_sk ,case when sr_return_quantity is not null then (ss_quantity-sr_return_quantity)*ss_sales_price else (ss_quantity*ss_sales_price) end act_sales from store_sales left outer join store_returns on (sr_item_sk = ss_item_sk and sr_ticket_number = ss_ticket_number) ,reason where sr_reason_sk = r_reason_sk and r_reason_desc = 'reason 58') t group by ss_customer_sk order by sumsales, ss_customer_sk limit 100""", "q94" -> """ select count(distinct ws_order_number) as `order count` ,sum(ws_ext_ship_cost) as `total shipping cost` ,sum(ws_net_profit) as `total net profit` from web_sales ws1 ,date_dim ,customer_address ,web_site where d_date between '2002-3-01' and (cast('2002-3-01' as date) + INTERVAL 60 days) and ws1.ws_ship_date_sk = d_date_sk and ws1.ws_ship_addr_sk = ca_address_sk and ca_state = 'GA' and ws1.ws_web_site_sk = web_site_sk and web_company_name = 'pri' and exists (select * from web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) and not exists(select * from web_returns wr1 where ws1.ws_order_number = wr1.wr_order_number) order by count(distinct ws_order_number) limit 100""", "q95" -> """ with ws_wh as (select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 from web_sales ws1,web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) select count(distinct ws_order_number) as `order count` ,sum(ws_ext_ship_cost) as `total shipping cost` ,sum(ws_net_profit) as `total net profit` from web_sales ws1 ,date_dim ,customer_address ,web_site where d_date between '2001-3-01' and (cast('2001-3-01' as date) + INTERVAL 60 days) and ws1.ws_ship_date_sk = d_date_sk and ws1.ws_ship_addr_sk = ca_address_sk and ca_state = 'NE' and ws1.ws_web_site_sk = web_site_sk and web_company_name = 'pri' and ws1.ws_order_number in (select ws_order_number from ws_wh) and ws1.ws_order_number in (select wr_order_number from web_returns,ws_wh where wr_order_number = ws_wh.ws_order_number) order by count(distinct ws_order_number) limit 100""", "q96" -> """ select count(*) from store_sales ,household_demographics ,time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 16 and time_dim.t_minute >= 30 and household_demographics.hd_dep_count = 0 and store.s_store_name = 'ese' order by count(*) limit 100""", "q97" -> """ with ssci as ( select ss_customer_sk customer_sk ,ss_item_sk item_sk from store_sales,date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1219 and 1219 + 11 group by ss_customer_sk ,ss_item_sk), csci as( select cs_bill_customer_sk customer_sk ,cs_item_sk item_sk from catalog_sales,date_dim where cs_sold_date_sk = d_date_sk and d_month_seq between 1219 and 1219 + 11 group by cs_bill_customer_sk ,cs_item_sk) select sum(case when ssci.customer_sk is not null and csci.customer_sk is null then 1 else 0 end) store_only ,sum(case when ssci.customer_sk is null and csci.customer_sk is not null then 1 else 0 end) catalog_only ,sum(case when ssci.customer_sk is not null and csci.customer_sk is not null then 1 else 0 end) store_and_catalog from ssci full outer join csci on (ssci.customer_sk=csci.customer_sk and ssci.item_sk = csci.item_sk) limit 100""", "q98" -> """ select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(ss_ext_sales_price) as itemrevenue ,sum(ss_ext_sales_price)*100/sum(sum(ss_ext_sales_price)) over (partition by i_class) as revenueratio from store_sales ,item ,date_dim where ss_item_sk = i_item_sk and i_category in ('Books', 'Children', 'Sports') and ss_sold_date_sk = d_date_sk and d_date between cast('2001-03-10' as date) and (cast('2001-03-10' as date) + interval 30 days) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio""", "q99" -> """ select substr(w_warehouse_name,1,20) ,sm_type ,cc_name ,sum(case when (cs_ship_date_sk - cs_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 30) and (cs_ship_date_sk - cs_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 60) and (cs_ship_date_sk - cs_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 90) and (cs_ship_date_sk - cs_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from catalog_sales ,warehouse ,ship_mode ,call_center ,date_dim where d_month_seq between 1205 and 1205 + 11 and cs_ship_date_sk = d_date_sk and cs_warehouse_sk = w_warehouse_sk and cs_ship_mode_sk = sm_ship_mode_sk and cs_call_center_sk = cc_call_center_sk group by substr(w_warehouse_name,1,20) ,sm_type ,cc_name order by substr(w_warehouse_name,1,20) ,sm_type ,cc_name limit 100""" ) } ================================================ FILE: benchmarks/src/main/scala/benchmark/TPCDSDataLoad.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package benchmark case class TPCDSDataLoadConf( protected val format: Option[String] = None, scaleInGB: Int = 0, userDefinedDbName: Option[String] = None, sourcePath: Option[String] = None, benchmarkPath: Option[String] = None, excludeNulls: Boolean = true) extends TPCDSConf object TPCDSDataLoadConf { import scopt.OParser private val builder = OParser.builder[TPCDSDataLoadConf] private val argParser = { import builder._ OParser.sequence( programName("TPC-DS Data Load"), opt[String]("format") .required() .action((x, c) => c.copy(format = Some(x))) .text("file format to use"), opt[String]("scale-in-gb") .required() .valueName("") .action((x, c) => c.copy(scaleInGB = x.toInt)) .text("Scale factor of the TPCDS benchmark"), opt[String]("benchmark-path") .required() .valueName("") .action((x, c) => c.copy(benchmarkPath = Some(x))) .text("Cloud storage path to be used for creating table and generating reports"), opt[String]("db-name") .optional() .valueName("") .action((x, c) => c.copy(userDefinedDbName = Some(x))) .text("Name of the target database to create with TPC-DS tables in necessary format"), opt[String]("source-path") .optional() .valueName("") .action((x, c) => c.copy(sourcePath = Some(x))) .text("The location of the TPC-DS raw input data"), opt[String]("exclude-nulls") .optional() .valueName("true/false") .action((x, c) => c.copy(excludeNulls = x.toBoolean)) .text("Whether to remove null primary keys when loading data, default = false"), ) } def parse(args: Array[String]): Option[TPCDSDataLoadConf] = { OParser.parse(argParser, args, TPCDSDataLoadConf()) } } class TPCDSDataLoad(conf: TPCDSDataLoadConf) extends Benchmark(conf) { import TPCDSDataLoad._ def runInternal(): Unit = { val dbName = conf.dbName val dbLocation = conf.dbLocation(dbName, suffix=benchmarkId.replace("-", "_")) val dbCatalog = "spark_catalog" val partitionTables = true val primaryKeys = true val sourceFormat = "parquet" require(conf.scaleInGB > 0) require(Seq(1, 3000).contains(conf.scaleInGB), "") val sourceLocation = conf.sourcePath.getOrElse { s"s3://devrel-delta-datasets/tpcds-2.13/tpcds_sf${conf.scaleInGB}_parquet/" } runQuery(s"DROP DATABASE IF EXISTS ${dbName} CASCADE", s"drop-database") runQuery(s"CREATE DATABASE IF NOT EXISTS ${dbName}", s"create-database") // Iterate through all the source tables tableNamesTpcds.foreach { tableName => val sourceTableLocation = s"${sourceLocation}/${tableName}/" val targetLocation = s"${dbLocation}/${tableName}/" val fullTableName = s"`$dbName`.`$tableName`" log(s"Generating $tableName at $dbLocation/$tableName") val partitionedBy = if (!partitionTables || tablePartitionKeys(tableName)(0).isEmpty) "" else "PARTITIONED BY " + tablePartitionKeys(tableName).mkString("(", ", ", ")") // Excluding nulls automatically when n val excludeNulls = if (!partitionTables || tablePartitionKeys(tableName)(0).isEmpty) "" else "WHERE " + tablePartitionKeys(tableName)(0) + " IS NOT NULL" var tableOptions = "" runQuery(s"DROP TABLE IF EXISTS $fullTableName", s"drop-table-$tableName") runQuery(s"""CREATE TABLE $fullTableName USING ${conf.formatName} $partitionedBy $tableOptions LOCATION '$targetLocation' SELECT * FROM `${sourceFormat}`.`$sourceTableLocation` $excludeNulls """, s"create-table-$tableName", ignoreError = true) val sourceCount = spark.sql(s"SELECT * FROM `${sourceFormat}`.`$sourceTableLocation` ${excludeNulls}").count() val targetCount = spark.table(fullTableName).count() assert(targetCount == sourceCount, s"Row count mismatch: source table = $sourceCount, target $fullTableName = $targetCount") } log(s"====== Created all tables in database ${dbName} at '${dbLocation}' =======") runQuery(s"USE ${dbCatalog}.${dbName};") runQuery("SHOW TABLES", printRows = true) } } object TPCDSDataLoad { def main(args: Array[String]): Unit = { TPCDSDataLoadConf.parse(args).foreach { conf => new TPCDSDataLoad(conf).run() } } val tableNamesTpcds = Seq( // with partitions "inventory", "catalog_returns", "catalog_sales", "store_returns", "web_returns", "web_sales", "store_sales", // no partitions "call_center", "catalog_page", "customer_address", "customer_demographics", "customer", "date_dim", "household_demographics", "income_band", "item", "promotion", "reason", "ship_mode", "store", "time_dim", "warehouse", "web_page", "web_site" ).sorted val tableColumnSchemas = Map( "dbgen_version" -> """ dv_version varchar(16) , dv_create_date date , dv_create_time time , dv_cmdline_args varchar(200) """, "call_center" -> """ cc_call_center_sk integer not null, cc_call_center_id char(16) not null, cc_rec_start_date date , cc_rec_end_date date , cc_closed_date_sk integer , cc_open_date_sk integer , cc_name varchar(50) , cc_class varchar(50) , cc_employees integer , cc_sq_ft integer , cc_hours char(20) , cc_manager varchar(40) , cc_mkt_id integer , cc_mkt_class char(50) , cc_mkt_desc varchar(100) , cc_market_manager varchar(40) , cc_division integer , cc_division_name varchar(50) , cc_company integer , cc_company_name char(50) , cc_street_number char(10) , cc_street_name varchar(60) , cc_street_type char(15) , cc_suite_number char(10) , cc_city varchar(60) , cc_county varchar(30) , cc_state char(2) , cc_zip char(10) , cc_country varchar(20) , cc_gmt_offset decimal(5,2) , cc_tax_percentage decimal(5,2) """, "catalog_page" -> """ cp_catalog_page_sk integer not null, cp_catalog_page_id char(16) not null, cp_start_date_sk integer , cp_end_date_sk integer , cp_department varchar(50) , cp_catalog_number integer , cp_catalog_page_number integer , cp_description varchar(100) , cp_type varchar(100) """, "catalog_returns" -> """ cr_returned_date_sk integer , cr_returned_time_sk integer , cr_item_sk integer not null, cr_refunded_customer_sk integer , cr_refunded_cdemo_sk integer , cr_refunded_hdemo_sk integer , cr_refunded_addr_sk integer , cr_returning_customer_sk integer , cr_returning_cdemo_sk integer , cr_returning_hdemo_sk integer , cr_returning_addr_sk integer , cr_call_center_sk integer , cr_catalog_page_sk integer , cr_ship_mode_sk integer , cr_warehouse_sk integer , cr_reason_sk integer , cr_order_number bigint not null, cr_return_quantity integer , cr_return_amount decimal(7,2) , cr_return_tax decimal(7,2) , cr_return_amt_inc_tax decimal(7,2) , cr_fee decimal(7,2) , cr_return_ship_cost decimal(7,2) , cr_refunded_cash decimal(7,2) , cr_reversed_charge decimal(7,2) , cr_store_credit decimal(7,2) , cr_net_loss decimal(7,2) """, "catalog_sales" -> """ cs_sold_date_sk integer , cs_sold_time_sk integer , cs_ship_date_sk integer , cs_bill_customer_sk integer , cs_bill_cdemo_sk integer , cs_bill_hdemo_sk integer , cs_bill_addr_sk integer , cs_ship_customer_sk integer , cs_ship_cdemo_sk integer , cs_ship_hdemo_sk integer , cs_ship_addr_sk integer , cs_call_center_sk integer , cs_catalog_page_sk integer , cs_ship_mode_sk integer , cs_warehouse_sk integer , cs_item_sk integer not null, cs_promo_sk integer , cs_order_number bigint not null, cs_quantity integer , cs_wholesale_cost decimal(7,2) , cs_list_price decimal(7,2) , cs_sales_price decimal(7,2) , cs_ext_discount_amt decimal(7,2) , cs_ext_sales_price decimal(7,2) , cs_ext_wholesale_cost decimal(7,2) , cs_ext_list_price decimal(7,2) , cs_ext_tax decimal(7,2) , cs_coupon_amt decimal(7,2) , cs_ext_ship_cost decimal(7,2) , cs_net_paid decimal(7,2) , cs_net_paid_inc_tax decimal(7,2) , cs_net_paid_inc_ship decimal(7,2) , cs_net_paid_inc_ship_tax decimal(7,2) , cs_net_profit decimal(7,2) """, "customer" -> """ c_customer_sk integer not null, c_customer_id char(16) not null, c_current_cdemo_sk integer , c_current_hdemo_sk integer , c_current_addr_sk integer , c_first_shipto_date_sk integer , c_first_sales_date_sk integer , c_salutation char(10) , c_first_name char(20) , c_last_name char(30) , c_preferred_cust_flag char(1) , c_birth_day integer , c_birth_month integer , c_birth_year integer , c_birth_country varchar(20) , c_login char(13) , c_email_address char(50) , c_last_review_date_sk integer """, "customer_address" -> """ ca_address_sk integer not null, ca_address_id char(16) not null, ca_street_number char(10) , ca_street_name varchar(60) , ca_street_type char(15) , ca_suite_number char(10) , ca_city varchar(60) , ca_county varchar(30) , ca_state char(2) , ca_zip char(10) , ca_country varchar(20) , ca_gmt_offset decimal(5,2) , ca_location_type char(20) """, "customer_demographics" -> """ cd_demo_sk integer not null, cd_gender char(1) , cd_marital_status char(1) , cd_education_status char(20) , cd_purchase_estimate integer , cd_credit_rating char(10) , cd_dep_count integer , cd_dep_employed_count integer , cd_dep_college_count integer """, "date_dim" -> """ d_date_sk integer not null, d_date_id char(16) not null, d_date date , d_month_seq integer , d_week_seq integer , d_quarter_seq integer , d_year integer , d_dow integer , d_moy integer , d_dom integer , d_qoy integer , d_fy_year integer , d_fy_quarter_seq integer , d_fy_week_seq integer , d_day_name char(9) , d_quarter_name char(6) , d_holiday char(1) , d_weekend char(1) , d_following_holiday char(1) , d_first_dom integer , d_last_dom integer , d_same_day_ly integer , d_same_day_lq integer , d_current_day char(1) , d_current_week char(1) , d_current_month char(1) , d_current_quarter char(1) , d_current_year char(1) """, "household_demographics" -> """ hd_demo_sk integer not null, hd_income_band_sk integer , hd_buy_potential char(15) , hd_dep_count integer , hd_vehicle_count integer """, "income_band" -> """ ib_income_band_sk integer not null, ib_lower_bound integer , ib_upper_bound integer """, "inventory" -> """ inv_date_sk integer not null, inv_item_sk integer not null, inv_warehouse_sk integer not null, inv_quantity_on_hand integer """, "item" -> """ i_item_sk integer not null, i_item_id char(16) not null, i_rec_start_date date , i_rec_end_date date , i_item_desc varchar(200) , i_current_price decimal(7,2) , i_wholesale_cost decimal(7,2) , i_brand_id integer , i_brand char(50) , i_class_id integer , i_class char(50) , i_category_id integer , i_category char(50) , i_manufact_id integer , i_manufact char(50) , i_size char(20) , i_formulation char(20) , i_color char(20) , i_units char(10) , i_container char(10) , i_manager_id integer , i_product_name char(50) """, "promotion" -> """ p_promo_sk integer not null, p_promo_id char(16) not null, p_start_date_sk integer , p_end_date_sk integer , p_item_sk integer , p_cost decimal(15,2) , p_response_target integer , p_promo_name char(50) , p_channel_dmail char(1) , p_channel_email char(1) , p_channel_catalog char(1) , p_channel_tv char(1) , p_channel_radio char(1) , p_channel_press char(1) , p_channel_event char(1) , p_channel_demo char(1) , p_channel_details varchar(100) , p_purpose char(15) , p_discount_active char(1) """, "reason" -> """ r_reason_sk integer not null, r_reason_id char(16) not null, r_reason_desc char(100) """, "ship_mode" -> """ sm_ship_mode_sk integer not null, sm_ship_mode_id char(16) not null, sm_type char(30) , sm_code char(10) , sm_carrier char(20) , sm_contract char(20) """, "store" -> """ s_store_sk integer not null, s_store_id char(16) not null, s_rec_start_date date , s_rec_end_date date , s_closed_date_sk integer , s_store_name varchar(50) , s_number_employees integer , s_floor_space integer , s_hours char(20) , s_manager varchar(40) , s_market_id integer , s_geography_class varchar(100) , s_market_desc varchar(100) , s_market_manager varchar(40) , s_division_id integer , s_division_name varchar(50) , s_company_id integer , s_company_name varchar(50) , s_street_number varchar(10) , s_street_name varchar(60) , s_street_type char(15) , s_suite_number char(10) , s_city varchar(60) , s_county varchar(30) , s_state char(2) , s_zip char(10) , s_country varchar(20) , s_gmt_offset decimal(5,2) , s_tax_precentage decimal(5,2) """, "store_returns" -> """ sr_returned_date_sk integer , sr_return_time_sk integer , sr_item_sk integer not null, sr_customer_sk integer , sr_cdemo_sk integer , sr_hdemo_sk integer , sr_addr_sk integer , sr_store_sk integer , sr_reason_sk integer , sr_ticket_number bigint not null, sr_return_quantity integer , sr_return_amt decimal(7,2) , sr_return_tax decimal(7,2) , sr_return_amt_inc_tax decimal(7,2) , sr_fee decimal(7,2) , sr_return_ship_cost decimal(7,2) , sr_refunded_cash decimal(7,2) , sr_reversed_charge decimal(7,2) , sr_store_credit decimal(7,2) , sr_net_loss decimal(7,2) """, "store_sales" -> """ ss_sold_date_sk integer , ss_sold_time_sk integer , ss_item_sk integer not null, ss_customer_sk integer , ss_cdemo_sk integer , ss_hdemo_sk integer , ss_addr_sk integer , ss_store_sk integer , ss_promo_sk integer , ss_ticket_number bigint not null, ss_quantity integer , ss_wholesale_cost decimal(7,2) , ss_list_price decimal(7,2) , ss_sales_price decimal(7,2) , ss_ext_discount_amt decimal(7,2) , ss_ext_sales_price decimal(7,2) , ss_ext_wholesale_cost decimal(7,2) , ss_ext_list_price decimal(7,2) , ss_ext_tax decimal(7,2) , ss_coupon_amt decimal(7,2) , ss_net_paid decimal(7,2) , ss_net_paid_inc_tax decimal(7,2) , ss_net_profit decimal(7,2) """, "time_dim" -> """ t_time_sk integer not null, t_time_id char(16) not null, t_time integer , t_hour integer , t_minute integer , t_second integer , t_am_pm char(2) , t_shift char(20) , t_sub_shift char(20) , t_meal_time char(20) """, "warehouse" -> """ w_warehouse_sk integer not null, w_warehouse_id char(16) not null, w_warehouse_name varchar(20) , w_warehouse_sq_ft integer , w_street_number char(10) , w_street_name varchar(60) , w_street_type char(15) , w_suite_number char(10) , w_city varchar(60) , w_county varchar(30) , w_state char(2) , w_zip char(10) , w_country varchar(20) , w_gmt_offset decimal(5,2) """, "web_page" -> """ wp_web_page_sk integer not null, wp_web_page_id char(16) not null, wp_rec_start_date date , wp_rec_end_date date , wp_creation_date_sk integer , wp_access_date_sk integer , wp_autogen_flag char(1) , wp_customer_sk integer , wp_url varchar(100) , wp_type char(50) , wp_char_count integer , wp_link_count integer , wp_image_count integer , wp_max_ad_count integer """, "web_returns" -> """ wr_returned_date_sk integer , wr_returned_time_sk integer , wr_item_sk integer not null, wr_refunded_customer_sk integer , wr_refunded_cdemo_sk integer , wr_refunded_hdemo_sk integer , wr_refunded_addr_sk integer , wr_returning_customer_sk integer , wr_returning_cdemo_sk integer , wr_returning_hdemo_sk integer , wr_returning_addr_sk integer , wr_web_page_sk integer , wr_reason_sk integer , wr_order_number bigint not null, wr_return_quantity integer , wr_return_amt decimal(7,2) , wr_return_tax decimal(7,2) , wr_return_amt_inc_tax decimal(7,2) , wr_fee decimal(7,2) , wr_return_ship_cost decimal(7,2) , wr_refunded_cash decimal(7,2) , wr_reversed_charge decimal(7,2) , wr_account_credit decimal(7,2) , wr_net_loss decimal(7,2) """, "web_sales" -> """ ws_sold_date_sk integer , ws_sold_time_sk integer , ws_ship_date_sk integer , ws_item_sk integer not null, ws_bill_customer_sk integer , ws_bill_cdemo_sk integer , ws_bill_hdemo_sk integer , ws_bill_addr_sk integer , ws_ship_customer_sk integer , ws_ship_cdemo_sk integer , ws_ship_hdemo_sk integer , ws_ship_addr_sk integer , ws_web_page_sk integer , ws_web_site_sk integer , ws_ship_mode_sk integer , ws_warehouse_sk integer , ws_promo_sk integer , ws_order_number bigint not null, ws_quantity integer , ws_wholesale_cost decimal(7,2) , ws_list_price decimal(7,2) , ws_sales_price decimal(7,2) , ws_ext_discount_amt decimal(7,2) , ws_ext_sales_price decimal(7,2) , ws_ext_wholesale_cost decimal(7,2) , ws_ext_list_price decimal(7,2) , ws_ext_tax decimal(7,2) , ws_coupon_amt decimal(7,2) , ws_ext_ship_cost decimal(7,2) , ws_net_paid decimal(7,2) , ws_net_paid_inc_tax decimal(7,2) , ws_net_paid_inc_ship decimal(7,2) , ws_net_paid_inc_ship_tax decimal(7,2) , ws_net_profit decimal(7,2) """, "web_site" -> """ web_site_sk integer not null, web_site_id char(16) not null, web_rec_start_date date , web_rec_end_date date , web_name varchar(50) , web_open_date_sk integer , web_close_date_sk integer , web_class varchar(50) , web_manager varchar(40) , web_mkt_id integer , web_mkt_class varchar(50) , web_mkt_desc varchar(100) , web_market_manager varchar(40) , web_company_id integer , web_company_name char(50) , web_street_number char(10) , web_street_name varchar(60) , web_street_type char(15) , web_suite_number char(10) , web_city varchar(60) , web_county varchar(30) , web_state char(2) , web_zip char(10) , web_country varchar(20) , web_gmt_offset decimal(5,2) , web_tax_percentage decimal(5,2) """ ) val tablePrimaryKeys = Map( "dbgen_version" -> Seq(""), "call_center" -> Seq("cc_call_center_sk"), "catalog_page" -> Seq("cp_catalog_page_sk"), "catalog_returns" -> Seq("cr_item_sk", "cr_order_number"), "catalog_sales" -> Seq("cs_item_sk", "cs_order_number"), "customer" -> Seq("c_customer_sk"), "customer_address" -> Seq("ca_address_sk"), "customer_demographics" -> Seq("cd_demo_sk"), "date_dim" -> Seq("d_date_sk"), "household_demographics" -> Seq("hd_demo_sk"), "income_band" -> Seq("ib_income_band_sk"), "inventory" -> Seq("inv_date_sk", "inv_item_sk", "inv_warehouse_sk"), "item" -> Seq("i_item_sk"), "promotion" -> Seq("p_promo_sk"), "reason" -> Seq("r_reason_sk"), "ship_mode" -> Seq("sm_ship_mode_sk"), "store" -> Seq("s_store_sk"), "store_returns" -> Seq("sr_item_sk", "sr_ticket_number"), "store_sales" -> Seq("ss_item_sk", "ss_ticket_number"), "time_dim" -> Seq("t_time_sk"), "warehouse" -> Seq("w_warehouse_sk"), "web_page" -> Seq("wp_web_page_sk"), "web_returns" -> Seq("wr_item_sk", "wr_order_number"), "web_sales" -> Seq("ws_item_sk", "ws_order_number"), "web_site" -> Seq("web_site_sk") ) val tablePartitionKeys = Map( "dbgen_version" -> Seq(""), "call_center" -> Seq(""), "catalog_page" -> Seq(""), "catalog_returns" -> Seq("cr_returned_date_sk"), "catalog_sales" -> Seq("cs_sold_date_sk"), "customer" -> Seq(""), "customer_address" -> Seq(""), "customer_demographics" -> Seq(""), "date_dim" -> Seq(""), "household_demographics" -> Seq(""), "income_band" -> Seq(""), "inventory" -> Seq("inv_date_sk"), "item" -> Seq(""), "promotion" -> Seq(""), "reason" -> Seq(""), "ship_mode" -> Seq(""), "store" -> Seq(""), "store_returns" -> Seq("sr_returned_date_sk"), "store_sales" -> Seq("ss_sold_date_sk"), "time_dim" -> Seq(""), "warehouse" -> Seq(""), "web_page" -> Seq(""), "web_returns" -> Seq("wr_returned_date_sk"), "web_sales" -> Seq("ws_sold_date_sk"), "web_site" -> Seq("") ) } ================================================ FILE: benchmarks/src/main/scala/benchmark/TestBenchmark.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package benchmark case class TestBenchmarkConf( dbName: Option[String] = None, benchmarkPath: Option[String] = None) extends BenchmarkConf object TestBenchmarkConf { import scopt.OParser private val builder = OParser.builder[TestBenchmarkConf] private val argParser = { import builder._ OParser.sequence( programName("Test Benchmark"), opt[String]("test-param") .required() .action((x, c) => c) // ignore .text("Name of the target database to create with TPC-DS tables in necessary format"), opt[String]("benchmark-path") .optional() .action((x, c) => c.copy(benchmarkPath = Some(x))) .text("Cloud path to be used for creating table and generating reports"), opt[String]("db-name") .optional() .action((x, c) => c.copy(dbName = Some(x))) .text("Name of the test database to create") ) } def parse(args: Array[String]): Option[TestBenchmarkConf] = { OParser.parse(argParser, args, TestBenchmarkConf()) } } class TestBenchmark(conf: TestBenchmarkConf) extends Benchmark(conf) { def runInternal(): Unit = { // Test Spark SQL runQuery("SELECT 1 AS X", "sql-test") if (conf.benchmarkPath.isEmpty) { log("Skipping the delta read / write test as benchmark path has not been provided") return } val dbName = conf.dbName.getOrElse(benchmarkId.replaceAll("-", "_")) val dbLocation = conf.dbLocation(dbName) // Run database management tests runQuery("SHOW DATABASES", "db-list-test") runQuery(s"""CREATE DATABASE IF NOT EXISTS $dbName LOCATION "$dbLocation" """, "db-create-test") runQuery(s"USE $dbName", "db-use-test") // Run table tests val tableName = "test" runQuery(s"DROP TABLE IF EXISTS $tableName", "table-drop-test") runQuery(s"CREATE TABLE $tableName USING delta SELECT 1 AS x", "table-create-test") runQuery(s"SELECT * FROM $tableName", "table-query-test") } } object TestBenchmark { def main(args: Array[String]): Unit = { println("All command line args = " + args.toSeq) TestBenchmarkConf.parse(args).foreach { conf => new TestBenchmark(conf).run() } } } ================================================ FILE: benchmarks/src/main/scala/org/apache/spark/SparkUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark import benchmark.SparkEnvironmentInfo import org.apache.spark.util.Utils object SparkUtils { def getEnvironmentInfo(sc: SparkContext): SparkEnvironmentInfo = { val info = sc.statusStore.environmentInfo() val sparkBuildInfo = Map( "sparkBuildBranch" -> SPARK_BRANCH, "sparkBuildVersion" -> SPARK_VERSION, "sparkBuildDate" -> SPARK_BUILD_DATE, "sparkBuildUser" -> SPARK_BUILD_USER, "sparkBuildRevision" -> SPARK_REVISION ) SparkEnvironmentInfo( sparkBuildInfo = sparkBuildInfo, runtimeInfo = caseClassToMap(info.runtime), sparkProps = Utils.redact(sc.conf, info.sparkProperties).toMap, hadoopProps = Utils.redact(sc.conf, info.hadoopProperties).toMap .filterKeys(k => !k.startsWith("mapred") && !k.startsWith("yarn")), systemProps = Utils.redact(sc.conf, info.systemProperties).toMap, classpathEntries = info.classpathEntries.toMap ) } def caseClassToMap(obj: Object): Map[String, String] = { obj.getClass.getDeclaredFields.flatMap { f => f.setAccessible(true) val valueOption = f.get(obj) match { case o: Option[_] => o.map(_.toString) case s => Some(s.toString) } valueOption.map(value => f.getName -> value) }.toMap } def median(sizes: Array[Long], alreadySorted: Boolean): Long = Utils.median(sizes, alreadySorted) } ================================================ FILE: build/sbt ================================================ #!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # # This file contains code from the Apache Spark project (original license above). # It contains modifications, which are licensed as follows: # # # Copyright (2021) The Delta Lake Project Authors. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # When creating new tests for Spark SQL Hive, the HADOOP_CLASSPATH must contain the hive jars so # that we can run Hive to generate the golden answer. This is not required for normal development # or testing. if [ -n "$HIVE_HOME" ]; then for i in "$HIVE_HOME"/lib/* do HADOOP_CLASSPATH="$HADOOP_CLASSPATH:$i" done export HADOOP_CLASSPATH fi realpath () { ( TARGET_FILE="$1" cd "$(dirname "$TARGET_FILE")" TARGET_FILE="$(basename "$TARGET_FILE")" COUNT=0 while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] do TARGET_FILE="$(readlink "$TARGET_FILE")" cd $(dirname "$TARGET_FILE") TARGET_FILE="$(basename $TARGET_FILE)" COUNT=$(($COUNT + 1)) done echo "$(pwd -P)/"$TARGET_FILE"" ) } # Make Jenkins use Google Mirror first as Maven Central may ban us SBT_REPOSITORIES_CONFIG="$(dirname "$(realpath "$0")")/sbt-config/repositories" export SBT_OPTS="-Dsbt.override.build.repos=true -Dsbt.repository.config=$SBT_REPOSITORIES_CONFIG" . "$(dirname "$(realpath "$0")")"/sbt-launch-lib.bash declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy" declare -r sbt_opts_file=".sbtopts" declare -r etc_sbt_opts_file="/etc/sbt/sbtopts" usage() { cat < path to global settings/plugins directory (default: ~/.sbt) -sbt-boot path to shared boot directory (default: ~/.sbt/boot in 0.11 series) -ivy path to local Ivy repository (default: ~/.ivy2) -mem set memory options (default: $sbt_mem, which is $(get_mem_opts $sbt_mem)) -no-share use all local caches; no sharing -no-global uses global caches, but does not use global ~/.sbt directory. -jvm-debug Turn on JVM debugging, open at the given port. -batch Disable interactive mode # sbt version (default: from project/build.properties if present, else latest release) -sbt-version use the specified version of sbt -sbt-jar use the specified jar as the sbt launcher -sbt-rc use an RC version of sbt -sbt-snapshot use a snapshot version of sbt # java version (default: java from PATH, currently $(java -version 2>&1 | grep version)) -java-home alternate JAVA_HOME # jvm options and output control JAVA_OPTS environment variable, if unset uses "$java_opts" SBT_OPTS environment variable, if unset uses "$default_sbt_opts" .sbtopts if this file exists in the current directory, it is prepended to the runner args /etc/sbt/sbtopts if this file exists, it is prepended to the runner args -Dkey=val pass -Dkey=val directly to the java runtime -J-X pass option -X directly to the java runtime (-J is stripped) -S-X add -X to sbt's scalacOptions (-S is stripped) -PmavenProfiles Enable a maven profile for the build. In the case of duplicated or conflicting options, the order above shows precedence: JAVA_OPTS lowest, command line options highest. EOM } process_my_args () { while [[ $# -gt 0 ]]; do case "$1" in -no-colors) addJava "-Dsbt.log.noformat=true" && shift ;; -no-share) addJava "$noshare_opts" && shift ;; -no-global) addJava "-Dsbt.global.base=$(pwd)/project/.sbtboot" && shift ;; -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;; -sbt-dir) require_arg path "$1" "$2" && addJava "-Dsbt.global.base=$2" && shift 2 ;; -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;; -batch) exec /dev/null) if [[ ! $? ]]; then saved_stty="" fi } saveSttySettings trap onExit INT run "$@" exit_status=$? onExit ================================================ FILE: build/sbt-config/repositories ================================================ [repositories] local local-preloaded-ivy: file:///${sbt.preloaded-${sbt.global.base-${user.home}/.sbt}/preloaded/}, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext] local-preloaded: file:///${sbt.preloaded-${sbt.global.base-${user.home}/.sbt}/preloaded/} mulesoft: https://repository.mulesoft.org/nexus/content/groups/public/ gcs-maven-central-mirror: https://maven-central.storage-download.googleapis.com/maven2/ maven-central typesafe-ivy-releases: https://repo.typesafe.com/typesafe/ivy-releases/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly sbt-ivy-snapshots: https://repo.scala-sbt.org/scalasbt/ivy-snapshots/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly sbt-plugin-releases: https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/, [organization]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] bintray-typesafe-sbt-plugin-releases: https://dl.bintray.com/typesafe/sbt-plugins/, [organization]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] repos-spark-packages: https://repos.spark-packages.org typesafe-releases: https://repo.typesafe.com/typesafe/releases/ apache-snapshot: https://repository.apache.org/content/groups/snapshots/ jitpack: https://jitpack.io ================================================ FILE: build/sbt-launch-lib.bash ================================================ #!/usr/bin/env bash # # A library to simplify using the SBT launcher from other packages. # Note: This should be used by tools like giter8/conscript etc. # TODO - Should we merge the main SBT script with this library? if test -z "$HOME"; then declare -r script_dir="$(dirname "$script_path")" else declare -r script_dir="$HOME/.sbt" fi declare -a residual_args declare -a java_args declare -a scalac_args declare -a sbt_commands declare -a maven_profiles if test -x "$JAVA_HOME/bin/java"; then echo -e "Using $JAVA_HOME as default JAVA_HOME." echo "Note, this will be overridden by -java-home if it is set." declare java_cmd="$JAVA_HOME/bin/java" else declare java_cmd=java fi echoerr () { echo 1>&2 "$@" } vlog () { [[ $verbose || $debug ]] && echoerr "$@" } dlog () { [[ $debug ]] && echoerr "$@" } download_sbt () { local url=$1 local output=$2 local temp_file="${output}.part" if [ $(command -v curl) ]; then curl --fail --location --silent ${url} > "${temp_file}" &&\ mv "${temp_file}" "${output}" elif [ $(command -v wget) ]; then wget --quiet ${url} -O "${temp_file}" &&\ mv "${temp_file}" "${output}" else printf "You do not have curl or wget installed, unable to downlaod ${url}\n" exit -1 fi } acquire_sbt_jar () { SBT_VERSION=`awk -F "=" '/sbt\.version/ {print $2}' ./project/build.properties` # Set primary and fallback URLs if [[ "${SBT_VERSION}" == "0.13.18" ]] && [[ -n "${SBT_MIRROR_JAR_URL}" ]]; then URL1="${SBT_MIRROR_JAR_URL}" elif [[ "${SBT_VERSION}" == "1.5.5" ]] && [[ -n "${SBT_1_5_5_MIRROR_JAR_URL}" ]]; then URL1="${SBT_1_5_5_MIRROR_JAR_URL}" else URL1=${DEFAULT_ARTIFACT_REPOSITORY:-https://maven-central.storage-download.googleapis.com/maven2/}org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar fi BACKUP_URL="https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar" JAR=build/sbt-launch-${SBT_VERSION}.jar sbt_jar=$JAR if [[ ! -f "$sbt_jar" ]]; then printf 'Attempting to fetch sbt from %s\n' "${URL1}" download_sbt "${URL1}" "${JAR}" if [[ ! -f "${JAR}" ]]; then printf 'Download from %s failed. Retrying from %s\n' "${URL1}" "${BACKUP_URL}" download_sbt "${BACKUP_URL}" "${JAR}" fi if [[ ! -f "${JAR}" ]]; then printf "Failed to download sbt. Please install sbt manually from https://www.scala-sbt.org/\n" exit 1 fi printf "Launching sbt from ${JAR}\n" fi } execRunner () { # print the arguments one to a line, quoting any containing spaces [[ $verbose || $debug ]] && echo "# Executing command line:" && { for arg; do if printf "%s\n" "$arg" | grep -q ' '; then printf "\"%s\"\n" "$arg" else printf "%s\n" "$arg" fi done echo "" } "$@" } addJava () { dlog "[addJava] arg = '$1'" java_args=( "${java_args[@]}" "$1" ) } enableProfile () { dlog "[enableProfile] arg = '$1'" maven_profiles=( "${maven_profiles[@]}" "$1" ) export SBT_MAVEN_PROFILES="${maven_profiles[@]}" } addSbt () { dlog "[addSbt] arg = '$1'" sbt_commands=( "${sbt_commands[@]}" "$1" ) } addResidual () { dlog "[residual] arg = '$1'" residual_args=( "${residual_args[@]}" "$1" ) } addDebugger () { addJava "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$1" } # a ham-fisted attempt to move some memory settings in concert # so they need not be dicked around with individually. get_mem_opts () { local mem=${1:-1000} local perm=$(( $mem / 4 )) (( $perm > 256 )) || perm=256 (( $perm < 4096 )) || perm=4096 local codecache=$(( $perm / 2 )) echo "-Xms${mem}m -Xmx${mem}m -XX:ReservedCodeCacheSize=${codecache}m" } require_arg () { local type="$1" local opt="$2" local arg="$3" if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then echo "$opt requires <$type> argument" 1>&2 exit 1 fi } is_function_defined() { declare -f "$1" > /dev/null } process_args () { while [[ $# -gt 0 ]]; do case "$1" in -h|-help) usage; exit 1 ;; -v|-verbose) verbose=1 && shift ;; -d|-debug) debug=1 && shift ;; -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;; -mem) require_arg integer "$1" "$2" && sbt_mem="$2" && shift 2 ;; -jvm-debug) require_arg port "$1" "$2" && addDebugger $2 && shift 2 ;; -batch) exec = 17) { Seq( // For Java 17 + "--add-opens=java.base/java.nio=ALL-UNNAMED", "--add-opens=java.base/java.lang=ALL-UNNAMED", "--add-opens=java.base/java.net=ALL-UNNAMED", "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED" ) } else { Seq.empty } }, testOptions += Tests.Argument("-oF"), // Unidoc settings: by default dont document any source file unidocSourceFilePatterns := Nil, ) //////////////////////////// // START: Code Formatting // //////////////////////////// /** Enforce java code style on compile. */ def javafmtCheckSettings(): Seq[Def.Setting[Task[CompileAnalysis]]] = Seq( (Compile / compile) := ((Compile / compile) dependsOn (Compile / javafmtCheckAll)).value ) /** Enforce scala code style on compile. */ def scalafmtCheckSettings(): Seq[Def.Setting[Task[CompileAnalysis]]] = Seq( (Compile / compile) := ((Compile / compile) dependsOn (Compile / scalafmtCheckAll)).value, ) // TODO: define fmtAll and fmtCheckAll tasks that run both scala and java fmts/checks ////////////////////////// // END: Code Formatting // ////////////////////////// /** * Note: we cannot access sparkVersion.value here, since that can only be used within a task or * setting macro. */ def runTaskOnlyOnSparkMaster[T]( task: sbt.TaskKey[T], taskName: String, projectName: String, emptyValue: => T): Def.Initialize[Task[T]] = { if (CrossSparkVersions.getSparkVersionSpec().isMaster) { Def.task(task.value) } else { Def.task { // scalastyle:off println val masterVersion = SparkVersionSpec.MASTER.map(_.fullVersion).getOrElse("(no master version configured)") println(s"Project $projectName: Skipping `$taskName` as Spark version " + s"${CrossSparkVersions.getSparkVersion()} does not equal $masterVersion.") // scalastyle:on println emptyValue } } } lazy val connectCommon = (project in file("spark-connect/common")) .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin) .settings( name := "delta-connect-common", commonSettings, CrossSparkVersions.sparkDependentSettings(sparkVersion), releaseSettings, // Export as JAR instead of classes directory. This ensures protobuf-generated classes // (e.g., io.delta.connect.proto.DeltaCommand) are available as a JAR file in fullClasspath, // which can be symlinked and picked up by Spark Submit's jars/* wildcard in connectClient tests. exportJars := true, libraryDependencies ++= Seq( "io.grpc" % "protoc-gen-grpc-java" % grpcVersion asProtocPlugin(), "io.grpc" % "grpc-protobuf" % grpcVersion, "io.grpc" % "grpc-stub" % grpcVersion, "com.google.protobuf" % "protobuf-java" % protoVersion % "protobuf", "javax.annotation" % "javax.annotation-api" % "1.3.2", "org.apache.spark" %% "spark-connect-common" % sparkVersion.value % "provided", ), PB.protocVersion := protoVersion, Compile / PB.targets := Seq( PB.gens.java -> (Compile / sourceManaged).value, PB.gens.plugin("grpc-java") -> (Compile / sourceManaged).value ) ) lazy val connectClient = (project in file("spark-connect/client")) .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin) .dependsOn(connectCommon % "compile->compile;test->test;provided->provided") .settings( name := "delta-connect-client", commonSettings, releaseSettings, CrossSparkVersions.sparkDependentSettings(sparkVersion), libraryDependencies ++= Seq( "com.google.protobuf" % "protobuf-java" % protoVersion % "protobuf", "org.apache.spark" %% "spark-connect-client-jvm" % sparkVersion.value % "provided", // Test deps "org.scalatest" %% "scalatest" % scalaTestVersion % "test", "org.apache.spark" %% "spark-connect-client-jvm" % sparkVersion.value % "test" classifier "tests" ), (Test / javaOptions) += { // Create a (mini) Spark Distribution based on the server classpath. val serverClassPath = (connectServer / Compile / fullClasspath).value val distributionDir = crossTarget.value / "test-dist" val jarsDir = distributionDir / "jars" if (!distributionDir.exists()) { IO.createDirectory(jarsDir) // Create symlinks for all dependencies (filter to only JAR files) serverClassPath.distinct.filter(_.data.isFile).foreach { entry => val jarFile = entry.data.toPath val linkedJarFile = jarsDir / entry.data.getName if (!java.nio.file.Files.exists(linkedJarFile.toPath)) { Files.createSymbolicLink(linkedJarFile.toPath, jarFile) } } // Create a symlink for the log4j properties val confDir = distributionDir / "conf" IO.createDirectory(confDir) val log4jProps = (sparkV1 / Test / resourceDirectory).value / "log4j2.properties" val linkedLog4jProps = confDir / "log4j2.properties" if (!java.nio.file.Files.exists(linkedLog4jProps.toPath)) { Files.createSymbolicLink(linkedLog4jProps.toPath, log4jProps.toPath) } } // Return the location of the distribution directory. "-Ddelta.spark.home=" + distributionDir }, // Required for testing addFeatureSupport/dropFeatureSupport. Test / envVars += ("DELTA_TESTING", "1") ) lazy val connectServer = (project in file("spark-connect/server")) .dependsOn(connectCommon % "compile->compile;test->test;provided->provided") .dependsOn(spark % "compile->compile;test->test;provided->provided") .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin) .settings( name := "delta-connect-server", commonSettings, releaseSettings, CrossSparkVersions.sparkDependentSettings(sparkVersion), // Export as JAR instead of classes directory. Required for connectClient test setup so that // classes like SimpleDeltaConnectService are available as a JAR file that can be symlinked // and picked up by Spark Submit's jars/* wildcard. Also prevents classpath conflicts. exportJars := true, assembly / assemblyMergeStrategy := { // Discard module-info.class files from Java 9+ modules and multi-release JARs case "module-info.class" => MergeStrategy.discard case PathList("META-INF", "versions", _, "module-info.class") => MergeStrategy.discard case x => val oldStrategy = (assembly / assemblyMergeStrategy).value oldStrategy(x) }, libraryDependencies ++= Seq( "com.google.protobuf" % "protobuf-java" % protoVersion % "protobuf", "org.apache.spark" %% "spark-hive" % sparkVersion.value % "provided", "org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided", "org.apache.spark" %% "spark-core" % sparkVersion.value % "provided", "org.apache.spark" %% "spark-catalyst" % sparkVersion.value % "provided", "org.apache.spark" %% "spark-connect" % sparkVersion.value % "provided", "org.apache.spark" %% "spark-catalyst" % sparkVersion.value % "test" classifier "tests", "org.apache.spark" %% "spark-core" % sparkVersion.value % "test" classifier "tests", "org.apache.spark" %% "spark-sql" % sparkVersion.value % "test" classifier "tests", "org.apache.spark" %% "spark-hive" % sparkVersion.value % "test" classifier "tests", "org.apache.spark" %% "spark-connect" % sparkVersion.value % "test" classifier "tests", ), excludeDependencies ++= Seq( // Exclude connect common because a properly shaded version of it is included in the // spark-connect jar. Including it causes classpath problems. ExclusionRule("org.apache.spark", "spark-connect-common_2.13"), // Exclude connect shims because we have spark-core on the classpath. The shims are only // needed for the client. Including it causes classpath problems. ExclusionRule("org.apache.spark", "spark-connect-shims_2.13") ), // Required for testing addFeatureSupport/dropFeatureSupport. Test / envVars += ("DELTA_TESTING", "1"), // Force Spark to bind to localhost to avoid network issues Test / envVars += ("SPARK_LOCAL_IP", "127.0.0.1") ) lazy val deltaSuiteGenerator = (project in file("spark/delta-suite-generator")) .disablePlugins(ScalafmtPlugin) .settings ( name := "delta-suite-generator", commonSettings, scalaStyleSettings, skipReleaseSettings, // Internal module - not published to Maven libraryDependencies ++= Seq( "org.scala-lang.modules" %% "scala-collection-compat" % "2.11.0", "org.scalameta" %% "scalameta" % "4.13.5", "org.scalameta" %% "scalafmt-core" % "3.9.6", "commons-cli" % "commons-cli" % "1.9.0", "commons-codec" % "commons-codec" % "1.17.2", "org.scalatest" %% "scalatest" % scalaTestVersion % "test", ), Compile / mainClass := Some("io.delta.suitegenerator.ModularSuiteGenerator"), Test / baseDirectory := (ThisBuild / baseDirectory).value, ) // ============================================================ // Spark Module 1: sparkV1 (prod code only, no tests) // ============================================================ lazy val sparkV1 = (project in file("spark")) .dependsOn(storage) .enablePlugins(Antlr4Plugin) .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin) .settings ( name := "delta-spark-v1", commonSettings, scalaStyleSettings, skipReleaseSettings, // Internal module - not published to Maven CrossSparkVersions.sparkDependentSettings(sparkVersion), // Export as JAR instead of classes directory. This prevents dependent projects // (e.g., connectServer) from seeing multiple 'classes' directories with the same // name in their classpath, which would cause FileAlreadyExistsException. exportJars := true, // Tests are compiled in the final 'spark' module to avoid circular dependencies Test / sources := Seq.empty, Test / resources := Seq.empty, libraryDependencies ++= Seq( // Adding test classifier seems to break transitive resolution of the core dependencies "org.apache.spark" %% "spark-hive" % sparkVersion.value % "provided", "org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided", "org.apache.spark" %% "spark-core" % sparkVersion.value % "provided", "org.apache.spark" %% "spark-catalyst" % sparkVersion.value % "provided", // For DynamoDBCommitStore "com.amazonaws" % "aws-java-sdk" % "1.12.262" % "provided", // Test deps "org.scalatest" %% "scalatest" % scalaTestVersion % "test", "org.scalatestplus" %% "scalacheck-1-15" % "3.2.9.0" % "test", "junit" % "junit" % "4.13.2" % "test", "com.novocode" % "junit-interface" % "0.11" % "test", "org.apache.spark" %% "spark-catalyst" % sparkVersion.value % "test" classifier "tests", "org.apache.spark" %% "spark-core" % sparkVersion.value % "test" classifier "tests", "org.apache.spark" %% "spark-sql" % sparkVersion.value % "test" classifier "tests", "org.apache.spark" %% "spark-hive" % sparkVersion.value % "test" classifier "tests", "org.mockito" % "mockito-inline" % "4.11.0" % "test", ), Compile / packageBin / mappings := (Compile / packageBin / mappings).value ++ listPythonFiles(baseDirectory.value.getParentFile / "python"), Antlr4 / antlr4PackageName := Some("io.delta.sql.parser"), Antlr4 / antlr4GenListener := true, Antlr4 / antlr4GenVisitor := true, // Introduced in https://github.com/delta-io/delta/commit/d2990624d34b6b86fa5cf230e00a89b095fde254 // // Hack to avoid errors related to missing repo-root/target/scala-2.13/classes/ // In multi-module sbt projects, some dependencies may attempt to locate this directory // at the repository root, causing build failures if it doesn't exist. createTargetClassesDir := { val dir = baseDirectory.value.getParentFile / "target" / "scala-2.13" / "classes" Files.createDirectories(dir.toPath) }, // Generate Python version.py file with hardcoded version. // This file is committed to git and auto-updated during build. generatePythonVersion := { val versionValue = version.value // Trim -SNAPSHOT suffix to get PyPI-compatible version (like setup.py does) val trimmedVersion = versionValue.split("-SNAPSHOT")(0) val versionFile = baseDirectory.value.getParentFile / "python" / "delta" / "version.py" val content = s"""# |# Copyright (2026) The Delta Lake Project Authors. |# |# Licensed under the Apache License, Version 2.0 (the "License"); |# you may not use this file except in compliance with the License. |# You may obtain a copy of the License at |# |# http://www.apache.org/licenses/LICENSE-2.0 |# |# Unless required by applicable law or agreed to in writing, software |# distributed under the License is distributed on an "AS IS" BASIS, |# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |# See the License for the specific language governing permissions and |# limitations under the License. |# | |# This file is auto-generated by the build.sbt generatePythonVersion task. |# Do not edit manually - edit version.sbt instead and run: |# build/sbt sparkV1/generatePythonVersion | |__version__ = "$trimmedVersion" |""".stripMargin IO.write(versionFile, content) versionFile }, // Hook both createTargetClassesDir and generatePythonVersion into compile task Compile / compile := ((Compile / compile) dependsOn createTargetClassesDir dependsOn generatePythonVersion).value, // Generate the package object to provide the version information in runtime. Compile / sourceGenerators += Def.task { val file = (Compile / sourceManaged).value / "io" / "delta" / "package.scala" IO.write(file, s"""package io | |package object delta { | val VERSION = "${version.value}" |} |""".stripMargin) Seq(file) }, ) // ============================================================ // Spark Module 2: sparkV1Filtered (v1 without DeltaLog for v2 dependency) // This filtered version of sparkV1 is needed because sparkV2 (spark/v2) depends on some // V1 classes for utilities and common functionality, but must NOT have access to DeltaLog, // Snapshot, OptimisticTransaction, or actions that belongs to core V1 delta libraries. // We should use Kernel as the Delta implementation. // ============================================================ lazy val sparkV1Filtered = (project in file("spark-v1-filtered")) .dependsOn(sparkV1) .dependsOn(storage) .settings( name := "delta-spark-v1-filtered", commonSettings, skipReleaseSettings, // Internal module - not published to Maven exportJars := true, // Export as JAR to avoid classpath conflicts // No source code - just repackage sparkV1 without DeltaLog classes Compile / sources := Seq.empty, Test / sources := Seq.empty, // Repackage sparkV1 jar but exclude DeltaLog and related classes Compile / packageBin / mappings := { val v1Mappings = (sparkV1 / Compile / packageBin / mappings).value // Filter out DeltaLog, Snapshot, OptimisticTransaction, and actions.scala classes v1Mappings.filterNot { case (file, path) => path.contains("org/apache/spark/sql/delta/DeltaLog") || path.contains("org/apache/spark/sql/delta/Snapshot") || path.contains("org/apache/spark/sql/delta/OptimisticTransaction") || path.contains("org/apache/spark/sql/delta/actions/actions") } }, ) // ============================================================ // Spark Module 3: sparkV2 (Kernel-based DSv2 connector, depends on v1-filtered) // ============================================================ lazy val sparkV2 = (project in file("spark/v2")) .dependsOn(sparkV1Filtered) .dependsOn(kernelDefaults) .dependsOn(kernelUnityCatalog % "compile->compile;test->test") .dependsOn(goldenTables % "test") .settings( name := "delta-spark-v2", commonSettings, javafmtCheckSettings, skipReleaseSettings, // Internal module - not published to Maven CrossSparkVersions.sparkDependentSettings(sparkVersion), exportJars := true, // Export as JAR to avoid classpath conflicts Test / javaOptions ++= Seq("-ea"), // make sure shaded kernel-api jar exists before compiling/testing Compile / compile := (Compile / compile) .dependsOn(kernelApi / Compile / packageBin).value, Test / test := (Test / test) .dependsOn(kernelApi / Compile / packageBin).value, Test / unmanagedJars += (kernelApi / Test / packageBin).value, Compile / unmanagedJars ++= Seq( (kernelApi / Compile / packageBin).value ), libraryDependencies ++= Seq( "org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided", "org.apache.spark" %% "spark-core" % sparkVersion.value % "provided", "org.apache.spark" %% "spark-catalyst" % sparkVersion.value % "provided", // Test dependencies "org.junit.jupiter" % "junit-jupiter-api" % "5.11.4" % "test", "org.junit.jupiter" % "junit-jupiter-engine" % "5.11.4" % "test", "org.junit.jupiter" % "junit-jupiter-params" % "5.11.4" % "test", "com.github.sbt.junit" % "jupiter-interface" % "0.17.0" % "test", // Spark test classes for Scala/Java test utilities "org.apache.spark" %% "spark-catalyst" % sparkVersion.value % "test" classifier "tests", "org.apache.spark" %% "spark-core" % sparkVersion.value % "test" classifier "tests", "org.apache.spark" %% "spark-sql" % sparkVersion.value % "test" classifier "tests", // ScalaTest for test utilities (needed by Spark test classes) "org.scalatest" %% "scalatest" % scalaTestVersion % "test" ), Test / testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"), TestParallelization.settings ) // ============================================================ // Spark Module 4: delta-spark (final published module - unified v1+v2) // ============================================================ lazy val spark = (project in file("spark-unified")) .dependsOn(sparkV1) .dependsOn(sparkV2) .dependsOn(storage) .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin) .settings ( name := "delta-spark", commonSettings, scalaStyleSettings, sparkMimaSettings, releaseSettings, // Published to Maven as delta-spark.jar // Set Test baseDirectory before sparkDependentSettings() so it uses the correct directory Test / baseDirectory := (sparkV1 / baseDirectory).value, // Test sources from spark/ directory (sparkV1's directory) AND spark-unified's own directory // MUST be set BEFORE crossSparkSettings() to avoid overwriting version-specific directories Test / unmanagedSourceDirectories := { val sparkDir = (sparkV1 / baseDirectory).value val unifiedDir = baseDirectory.value Seq( sparkDir / "src" / "test" / "scala", sparkDir / "src" / "test" / "java", unifiedDir / "src" / "test" / "scala", unifiedDir / "src" / "test" / "java" ) }, Test / unmanagedResourceDirectories := Seq( (sparkV1 / baseDirectory).value / "src" / "test" / "resources", baseDirectory.value / "src" / "test" / "resources" ), CrossSparkVersions.sparkDependentSettings(sparkVersion), // MiMa should use the generated JAR (not classDirectory) because we merge classes at package time mimaCurrentClassfiles := (Compile / packageBin).value, // Export as JAR to dependent projects (e.g., connectServer, connectClient). // This prevents classpath conflicts from internal module 'classes' directories. exportJars := true, // Internal module artifact names to exclude from published POM internalModuleNames := Set("delta-spark-v1", "delta-spark-v1-shaded", "delta-spark-v2"), // Merge classes from internal modules (v1, v2) into final JAR // kernel modules are kept as separate JARs and listed as dependencies in POM Compile / packageBin / mappings ++= { val log = streams.value.log // Collect mappings from internal modules val v1Mappings = (sparkV1 / Compile / packageBin / mappings).value val v2Mappings = (sparkV2 / Compile / packageBin / mappings).value // Include Python files (from spark/ directory) val pythonMappings = listPythonFiles(baseDirectory.value.getParentFile / "python") // Combine all mappings val allMappings = v1Mappings ++ v2Mappings ++ pythonMappings // Detect duplicate class files val classFiles = allMappings.filter(_._2.endsWith(".class")) val duplicates = classFiles.groupBy(_._2).filter(_._2.size > 1) if (duplicates.nonEmpty) { log.error(s"Found ${duplicates.size} duplicate class(es) in packageBin mappings:") duplicates.foreach { case (className, entries) => log.error(s" - $className:") entries.foreach { case (file, path) => log.error(s" from: $file") } } sys.error("Duplicate classes found. This indicates overlapping code between sparkV1, sparkV2, and storage modules.") } allMappings.distinct }, // Exclude internal modules from published POM and add kernel dependencies. // Kernel modules are transitive through sparkV2 (an internal module), so they // are lost when sparkV2 is filtered out. We re-add them explicitly here. pomPostProcess := { node => val internalModules = internalModuleNames.value val ver = version.value import scala.xml._ import scala.xml.transform._ def kernelDependencyNode(artifactId: String): Elem = { io.delta {artifactId} {ver} } val kernelDeps = Seq( kernelDependencyNode("delta-kernel-api"), kernelDependencyNode("delta-kernel-defaults"), kernelDependencyNode("delta-kernel-unitycatalog") ) new RuleTransformer(new RewriteRule { override def transform(n: Node): Seq[Node] = n match { case e: Elem if e.label == "dependencies" => val filtered = e.child.filter { case child: Elem if child.label == "dependency" => val artifactId = (child \ "artifactId").text !internalModules.exists(module => artifactId.startsWith(module)) case _ => true } Seq(e.copy(child = filtered ++ kernelDeps)) case _ => Seq(n) } }).transform(node).head }, pomIncludeRepository := { _ => false }, // Filter internal modules from project dependencies // This works together with pomPostProcess to ensure internal modules // (sparkV1, sparkV2, sparkV1Filtered) are not listed as dependencies in POM projectDependencies := { val internalModules = internalModuleNames.value projectDependencies.value.filterNot(dep => internalModules.contains(dep.name)) }, libraryDependencies ++= Seq( "org.apache.spark" %% "spark-hive" % sparkVersion.value % "provided", "org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided", "org.apache.spark" %% "spark-core" % sparkVersion.value % "provided", "org.apache.spark" %% "spark-catalyst" % sparkVersion.value % "provided", "com.amazonaws" % "aws-java-sdk" % "1.12.262" % "provided", "org.scalatest" %% "scalatest" % scalaTestVersion % "test", "org.scalatestplus" %% "scalacheck-1-15" % "3.2.9.0" % "test", "junit" % "junit" % "4.13.2" % "test", "com.novocode" % "junit-interface" % "0.11" % "test", "org.apache.spark" %% "spark-catalyst" % sparkVersion.value % "test" classifier "tests", "org.apache.spark" %% "spark-core" % sparkVersion.value % "test" classifier "tests", "org.apache.spark" %% "spark-sql" % sparkVersion.value % "test" classifier "tests", "org.apache.spark" %% "spark-hive" % sparkVersion.value % "test" classifier "tests", "org.mockito" % "mockito-inline" % "4.11.0" % "test", ), Test / testOptions += Tests.Argument("-oDF"), Test / testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"), // Don't execute in parallel since we can't have multiple Sparks in the same JVM Test / parallelExecution := false, javaOptions += "-Xmx1024m", // Configurations to speed up tests and reduce memory footprint Test / javaOptions ++= Seq( "-Dspark.ui.enabled=false", "-Dspark.ui.showConsoleProgress=false", "-Dspark.databricks.delta.snapshotPartitions=2", "-Dspark.sql.shuffle.partitions=5", "-Ddelta.log.cacheSize=3", "-Dspark.databricks.delta.delta.log.cacheSize=3", "-Dspark.sql.sources.parallelPartitionDiscovery.parallelism=5", "-Xmx1024m" ), // Required for testing table features see https://github.com/delta-io/delta/issues/1602 Test / envVars += ("DELTA_TESTING", "1"), TestParallelization.settings, ) .configureUnidoc( generatedJavaDoc = CrossSparkVersions.getSparkVersionSpec().generateDocs, generateScalaDoc = CrossSparkVersions.getSparkVersionSpec().generateDocs, // spark-connect has classes with the same name as spark-core, this causes compilation issues // with unidoc since it concatenates the classpaths from all modules // ==> thus we exclude such sources // (mostly) relevant github issue: https://github.com/sbt/sbt-unidoc/issues/77 classPathToSkip = "spark-connect" ) lazy val contribs = (project in file("contribs")) .dependsOn(spark % "compile->compile;test->test;provided->provided") .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin) .settings ( name := "delta-contribs", commonSettings, scalaStyleSettings, releaseSettings, // Set sparkVersion directly (not sparkDependentModuleName) so that // runOnlyForReleasableSparkModules discovers this module, but without adding a Spark // suffix to the artifact name. delta-contribs is only published as delta-contribs_2.13. sparkVersion := CrossSparkVersions.getSparkVersion(), Compile / packageBin / mappings := (Compile / packageBin / mappings).value ++ listPythonFiles(baseDirectory.value.getParentFile / "python"), Test / testOptions += Tests.Argument("-oDF"), Test / testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"), // Don't execute in parallel since we can't have multiple Sparks in the same JVM Test / parallelExecution := false, javaOptions += "-Xmx1024m", // Configurations to speed up tests and reduce memory footprint Test / javaOptions ++= Seq( "-Dspark.ui.enabled=false", "-Dspark.ui.showConsoleProgress=false", "-Dspark.databricks.delta.snapshotPartitions=2", "-Dspark.sql.shuffle.partitions=5", "-Ddelta.log.cacheSize=3", "-Dspark.databricks.delta.delta.log.cacheSize=3", "-Dspark.sql.sources.parallelPartitionDiscovery.parallelism=5", "-Xmx1024m" ), // Introduced in https://github.com/delta-io/delta/commit/d2990624d34b6b86fa5cf230e00a89b095fde254 // // Hack to avoid errors related to missing repo-root/target/scala-2.13/classes/ // In multi-module sbt projects, some dependencies may attempt to locate this directory // at the repository root, causing build failures if it doesn't exist. createTargetClassesDir := { val dir = baseDirectory.value.getParentFile / "target" / "scala-2.13" / "classes" Files.createDirectories(dir.toPath) }, Compile / compile := ((Compile / compile) dependsOn createTargetClassesDir).value, TestParallelization.settings ).configureUnidoc() val unityCatalogVersion = "0.4.0" val sparkUnityCatalogJacksonVersion = "2.15.4" // We are using Spark 4.0's Jackson version 2.15.x, to override Unity Catalog 0.3.0's version 2.18.x lazy val sparkUnityCatalog = (project in file("spark/unitycatalog")) .dependsOn(spark % "compile->compile;test->test;provided->provided") .disablePlugins(ScalafmtPlugin) .settings( name := "delta-spark-unitycatalog", commonSettings, skipReleaseSettings, javafmtCheckSettings(), CrossSparkVersions.sparkDependentSettings(sparkVersion), // This is a test-only module - no production sources Compile / sources := Seq.empty, // Ensure Java sources are picked up Test / unmanagedSourceDirectories += baseDirectory.value / "src" / "test" / "java", Test / javaOptions ++= Seq("-ea"), // Don't execute in parallel since we can't have multiple Sparks in the same JVM Test / parallelExecution := false, // Force ALL Jackson dependencies to match Spark's Jackson version // This overrides Jackson from Unity Catalog's transitive dependencies (e.g., Armeria) dependencyOverrides ++= Seq( "com.fasterxml.jackson.core" % "jackson-core" % sparkUnityCatalogJacksonVersion, "com.fasterxml.jackson.core" % "jackson-annotations" % sparkUnityCatalogJacksonVersion, "com.fasterxml.jackson.core" % "jackson-databind" % sparkUnityCatalogJacksonVersion, "com.fasterxml.jackson.module" %% "jackson-module-scala" % sparkUnityCatalogJacksonVersion, "com.fasterxml.jackson.dataformat" % "jackson-dataformat-yaml" % sparkUnityCatalogJacksonVersion, "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % sparkUnityCatalogJacksonVersion, "com.fasterxml.jackson.datatype" % "jackson-datatype-jdk8" % sparkUnityCatalogJacksonVersion ), libraryDependencies ++= Seq( "org.assertj" % "assertj-core" % "3.26.3" % "test", // JUnit 5 test dependencies "org.junit.jupiter" % "junit-jupiter-api" % "5.11.4" % "test", "org.junit.jupiter" % "junit-jupiter-engine" % "5.11.4" % "test", "org.junit.jupiter" % "junit-jupiter-params" % "5.11.4" % "test", "com.github.sbt.junit" % "jupiter-interface" % "0.17.0" % "test", // Lombok for generating boilerplate code "org.projectlombok" % "lombok" % "1.18.34" % "test", // Unity Catalog dependencies - exclude Jackson to use Spark's Jackson 2.15.x "io.unitycatalog" %% "unitycatalog-spark" % unityCatalogVersion % "test" excludeAll( ExclusionRule(organization = "com.fasterxml.jackson.core"), ExclusionRule(organization = "com.fasterxml.jackson.module"), ExclusionRule(organization = "com.fasterxml.jackson.datatype"), ExclusionRule(organization = "com.fasterxml.jackson.dataformat") ), "io.unitycatalog" % "unitycatalog-server" % unityCatalogVersion % "test" excludeAll( ExclusionRule(organization = "com.fasterxml.jackson.core"), ExclusionRule(organization = "com.fasterxml.jackson.module"), ExclusionRule(organization = "com.fasterxml.jackson.datatype"), ExclusionRule(organization = "com.fasterxml.jackson.dataformat") ), // Spark test dependencies "org.apache.spark" %% "spark-sql" % sparkVersion.value % "test", "org.apache.spark" %% "spark-catalyst" % sparkVersion.value % "test", "org.apache.spark" %% "spark-core" % sparkVersion.value % "test", ), // Conditionally add hadoop-aws dependency only when UC_REMOTE=true // Please see: https://github.com/delta-io/delta/issues/5624#issuecomment-3673383736 // Once we release the relocated unitycatalog-server, we can remove this. libraryDependencies ++= { if (sys.env.get("UC_REMOTE").contains("true")) { Seq( "org.apache.hadoop" % "hadoop-aws" % hadoopVersion % "test", "org.apache.hadoop" % "hadoop-common" % hadoopVersion % "test", "org.apache.hadoop" % "hadoop-client-api" % hadoopVersion % "test", "org.apache.hadoop" % "hadoop-client-runtime" % hadoopVersion % "test" ) } else { Seq.empty } }, Test / testOptions += Tests.Argument("-oDF"), Test / testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a") ) lazy val sharing = (project in file("sharing")) .dependsOn(spark % "compile->compile;test->test;provided->provided") .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin) .settings( name := "delta-sharing-spark", commonSettings, scalaStyleSettings, releaseSettings, CrossSparkVersions.sparkDependentSettings(sparkVersion), Test / javaOptions ++= Seq("-ea"), libraryDependencies ++= Seq( "org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided", "io.delta" %% "delta-sharing-client" % "1.3.10", // Test deps "org.scalatest" %% "scalatest" % scalaTestVersion % "test", "org.scalatestplus" %% "scalacheck-1-15" % "3.2.9.0" % "test", "junit" % "junit" % "4.13.2" % "test", "com.novocode" % "junit-interface" % "0.11" % "test", "org.apache.spark" %% "spark-catalyst" % sparkVersion.value % "test" classifier "tests", "org.apache.spark" %% "spark-core" % sparkVersion.value % "test" classifier "tests", "org.apache.spark" %% "spark-sql" % sparkVersion.value % "test" classifier "tests", "org.apache.spark" %% "spark-hive" % sparkVersion.value % "test" classifier "tests", ), TestParallelization.settings ).configureUnidoc() lazy val kernelApi = (project in file("kernel/kernel-api")) .enablePlugins(ScalafmtPlugin) .settings( name := "delta-kernel-api", commonSettings, scalaStyleSettings, javaOnlyReleaseSettings, javafmtCheckSettings, scalafmtCheckSettings, // Use unique classDirectory name to avoid conflicts in connectClient test setup // This allows connectClient to create symlinks without FileAlreadyExistsException Compile / classDirectory := target.value / "scala-2.13" / "kernel-api-classes", Test / javaOptions ++= Seq("-ea"), // Also publish a test-jar (classifier = "tests") so consumers (e.g. kernelDefault) // can depend on test utilities via a published artifact instead of depending on raw class directories. Test / publishArtifact := true, Test / packageBin / artifactClassifier := Some("tests"), libraryDependencies ++= Seq( "org.roaringbitmap" % "RoaringBitmap" % "0.9.25", "org.slf4j" % "slf4j-api" % "1.7.36", "com.fasterxml.jackson.core" % "jackson-databind" % "2.13.5", "com.fasterxml.jackson.core" % "jackson-core" % "2.13.5", "com.fasterxml.jackson.core" % "jackson-annotations" % "2.13.5", "com.fasterxml.jackson.datatype" % "jackson-datatype-jdk8" % "2.13.5", // JSR-305 annotations for @Nullable "com.google.code.findbugs" % "jsr305" % "3.0.2", "org.scalatest" %% "scalatest" % scalaTestVersion % "test", "junit" % "junit" % "4.13.2" % "test", "com.novocode" % "junit-interface" % "0.11" % "test", "org.apache.logging.log4j" % "log4j-slf4j-impl" % "2.25.3" % "test", "org.apache.logging.log4j" % "log4j-core" % "2.25.3" % "test", "org.assertj" % "assertj-core" % "3.26.3" % "test", // JMH dependencies allow writing micro-benchmarks for testing performance of components. // JMH has framework to define benchmarks and takes care of many common functionalities // such as warm runs, cold runs, defining benchmark parameter variables etc. "org.openjdk.jmh" % "jmh-core" % "1.37" % "test", "org.openjdk.jmh" % "jmh-generator-annprocess" % "1.37" % "test" ), // Shade jackson libraries so that connector developers don't have to worry // about jackson version conflicts. Compile / packageBin := assembly.value, assembly / assemblyJarName := s"${name.value}-${version.value}.jar", assembly / logLevel := Level.Info, assembly / test := {}, assembly / assemblyExcludedJars := { val cp = (assembly / fullClasspath).value val allowedPrefixes = Set("META_INF", "io", "jackson") cp.filter { f => !allowedPrefixes.exists(prefix => f.data.getName.startsWith(prefix)) } }, assembly / assemblyShadeRules := Seq( ShadeRule.rename("com.fasterxml.jackson.**" -> "io.delta.kernel.shaded.com.fasterxml.jackson.@1").inAll ), assembly / assemblyMergeStrategy := { // Discard `module-info.class` to fix the `different file contents found` error. // TODO Upgrade SBT to 1.5 which will do this automatically case "module-info.class" => MergeStrategy.discard case PathList("META-INF", "services", xs @ _*) => MergeStrategy.discard case x => val oldStrategy = (assembly / assemblyMergeStrategy).value oldStrategy(x) }, // Generate the package object to provide the version information in runtime. Compile / sourceGenerators += Def.task { val file = (Compile / sourceManaged).value / "io" / "delta" / "kernel" / "Meta.java" IO.write(file, s"""/* | * Copyright (2024) The Delta Lake Project Authors. | * | * Licensed under the Apache License, Version 2.0 (the "License"); | * you may not use this file except in compliance with the License. | * You may obtain a copy of the License at | * | * http://www.apache.org/licenses/LICENSE-2.0 | * | * Unless required by applicable law or agreed to in writing, software | * distributed under the License is distributed on an "AS IS" BASIS, | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | * See the License for the specific language governing permissions and | * limitations under the License. | */ |package io.delta.kernel; | |public final class Meta { | public static final String KERNEL_VERSION = "${version.value}"; |} |""".stripMargin) Seq(file) }, MultiShardMultiJVMTestParallelization.settings, javaCheckstyleSettings("dev/kernel-checkstyle.xml"), // Unidoc settings unidocSourceFilePatterns := Seq(SourceFilePattern("io/delta/kernel/")), ).configureUnidoc(docTitle = "Delta Kernel") lazy val kernelDefaults = (project in file("kernel/kernel-defaults")) .enablePlugins(ScalafmtPlugin) .dependsOn(storage) .dependsOn(storage % "test->test") // Required for InMemoryCommitCoordinator for tests .dependsOn(goldenTables % "test") .settings( name := "delta-kernel-defaults", commonSettings, scalaStyleSettings, javaOnlyReleaseSettings, javafmtCheckSettings, scalafmtCheckSettings, // Use unique classDirectory name to avoid conflicts in connectClient test setup // This allows connectClient to create symlinks without FileAlreadyExistsException Compile / classDirectory := target.value / "scala-2.13" / "kernel-defaults-classes", Test / javaOptions ++= Seq("-ea"), // This allows generating tables with unsupported test table features in delta-spark Test / envVars += ("DELTA_TESTING", "1"), // Put the shaded kernel-api JAR on the classpath (compile & test) Compile / unmanagedJars += (kernelApi / Compile / packageBin).value, Test / unmanagedJars += (kernelApi / Compile / packageBin).value, // Make sure the shaded JAR is produced before we compile/run tests Compile / compile := (Compile / compile).dependsOn(kernelApi / Compile / packageBin).value, Test / test := (Test / test).dependsOn(kernelApi / Compile / packageBin).value, Test / unmanagedJars += (kernelApi / Test / packageBin).value, libraryDependencies ++= Seq( "org.assertj" % "assertj-core" % "3.26.3" % Test, "org.apache.hadoop" % "hadoop-client-runtime" % hadoopVersion, "com.fasterxml.jackson.core" % "jackson-databind" % "2.13.5", "com.fasterxml.jackson.datatype" % "jackson-datatype-jdk8" % "2.13.5", "org.apache.parquet" % "parquet-hadoop" % "1.12.3", "org.scalatest" %% "scalatest" % scalaTestVersion % "test", "junit" % "junit" % "4.13.2" % "test", "commons-io" % "commons-io" % "2.8.0" % "test", "com.novocode" % "junit-interface" % "0.11" % "test", "org.apache.logging.log4j" % "log4j-slf4j-impl" % "2.25.3" % "test", "org.apache.logging.log4j" % "log4j-core" % "2.25.3" % "test", // JMH dependencies allow writing micro-benchmarks for testing performance of components. // JMH has framework to define benchmarks and takes care of many common functionalities // such as warm runs, cold runs, defining benchmark parameter variables etc. "org.openjdk.jmh" % "jmh-core" % "1.37" % "test", "org.openjdk.jmh" % "jmh-generator-annprocess" % "1.37" % "test", // The delta-spark and spark dependencies are mainly used for catalog-based table creation. // Instead of using the latest snapshot, those are fine to use the released 4.0.0. "io.delta" %% "delta-spark" % "4.0.0" % "test", "org.apache.spark" %% "spark-hive" % sparkVersionForKernelTest % "test" classifier "tests", "org.apache.spark" %% "spark-sql" % sparkVersionForKernelTest % "test" classifier "tests", "org.apache.spark" %% "spark-core" % sparkVersionForKernelTest % "test" classifier "tests", "org.apache.spark" %% "spark-catalyst" % sparkVersionForKernelTest % "test" classifier "tests", ), MultiShardMultiJVMTestParallelization.settings, javaCheckstyleSettings("dev/kernel-checkstyle.xml"), // Unidoc settings unidocSourceFilePatterns += SourceFilePattern("io/delta/kernel/"), ).configureUnidoc(docTitle = "Delta Kernel Defaults") lazy val kernelBenchmarks = (project in file("kernel/kernel-benchmarks")) .enablePlugins(ScalafmtPlugin) .dependsOn(kernelDefaults % "test->test") .dependsOn(kernelApi % "test->test") .dependsOn(storage % "test->test") .dependsOn(kernelUnityCatalog % "test->test") .settings( name := "delta-kernel-benchmarks", commonSettings, skipReleaseSettings, exportJars := false, javafmtCheckSettings, scalafmtCheckSettings, libraryDependencies ++= Seq( "org.openjdk.jmh" % "jmh-core" % "1.37" % "test", "org.openjdk.jmh" % "jmh-generator-annprocess" % "1.37" % "test", ), ) lazy val kernelUnityCatalog = (project in file("kernel/unitycatalog")) .enablePlugins(ScalafmtPlugin) .dependsOn(kernelDefaults % "test->test") .dependsOn(storage) .settings ( name := "delta-kernel-unitycatalog", commonSettings, javaOnlyReleaseSettings, javafmtCheckSettings, javaCheckstyleSettings("dev/kernel-checkstyle.xml"), scalaStyleSettings, scalafmtCheckSettings, // Put the shaded kernel-api JAR on the classpath (compile & test) Compile / unmanagedJars += (kernelApi / Compile / packageBin).value, Test / unmanagedJars += (kernelApi / Compile / packageBin).value, // Make sure the shaded JAR is produced before we compile/run tests Compile / compile := (Compile / compile).dependsOn(kernelApi / Compile / packageBin).value, Test / test := (Test / test).dependsOn(kernelApi / Compile / packageBin).value, Test / unmanagedJars += (kernelApi / Test / packageBin).value, libraryDependencies ++= Seq( "org.apache.hadoop" % "hadoop-common" % hadoopVersion % "provided", "org.scalatest" %% "scalatest" % scalaTestVersion % "test", "org.apache.logging.log4j" % "log4j-slf4j-impl" % "2.25.3" % "test", "org.apache.logging.log4j" % "log4j-core" % "2.25.3" % "test", ), unidocSourceFilePatterns += SourceFilePattern("src/main/java/io/delta/unity/"), ).configureUnidoc() // TODO javastyle tests // TODO unidoc // TODO(scott): figure out a better way to include tests in this project lazy val storage = (project in file("storage")) .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin) .settings ( name := "delta-storage", commonSettings, exportJars := true, javaOnlyReleaseSettings, libraryDependencies ++= Seq( // User can provide any 2.x or 3.x version. We don't use any new fancy APIs. Watch out for // versions with known vulnerabilities. "org.apache.hadoop" % "hadoop-common" % hadoopVersion % "provided", // Note that the org.apache.hadoop.fs.s3a.Listing::createFileStatusListingIterator 3.3.1 API // is not compatible with 3.3.2. "org.apache.hadoop" % "hadoop-aws" % hadoopVersion % "provided", "io.unitycatalog" % "unitycatalog-client" % unityCatalogVersion excludeAll( ExclusionRule(organization = "org.openapitools"), ExclusionRule(organization = "com.fasterxml.jackson.core"), ExclusionRule(organization = "com.fasterxml.jackson.module"), ExclusionRule(organization = "com.fasterxml.jackson.datatype"), ExclusionRule(organization = "com.fasterxml.jackson.dataformat") ), // Test Deps "org.scalatest" %% "scalatest" % scalaTestVersion % "test", // Jackson datatype module needed for UC SDK tests (excluded from main compile scope) "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % "2.15.4" % "test", ), // Unidoc settings unidocSourceFilePatterns += SourceFilePattern("/LogStore.java", "/CloseableIterator.java"), TestParallelization.settings ).configureUnidoc() lazy val storageS3DynamoDB = (project in file("storage-s3-dynamodb")) .dependsOn(storage % "compile->compile;test->test;provided->provided") .dependsOn(spark % "test->test") .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin) .settings ( name := "delta-storage-s3-dynamodb", commonSettings, javaOnlyReleaseSettings, // uncomment only when testing FailingS3DynamoDBLogStore. this will include test sources in // a separate test jar. // Test / publishArtifact := true, libraryDependencies ++= Seq( "com.amazonaws" % "aws-java-sdk" % "1.12.262" % "provided", // Test Deps "org.apache.hadoop" % "hadoop-aws" % hadoopVersion % "test", // RemoteFileChangedException ), TestParallelization.settings ).configureUnidoc() val icebergSparkRuntimeArtifactName = { val currentSparkVersion = CrossSparkVersions.getSparkVersion() val (expMaj, expMin, _) = getMajorMinorPatch(currentSparkVersion) s"iceberg-spark-runtime-$expMaj.$expMin" } lazy val testDeltaIcebergJar = (project in file("testDeltaIcebergJar")) // delta-iceberg depends on delta-spark! So, we need to include it during our test. .dependsOn(spark % "test") .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin) .settings( name := "test-delta-iceberg-jar", commonSettings, skipReleaseSettings, exportJars := true, Compile / unmanagedJars += (iceberg / assembly).value, libraryDependencies ++= Seq( "org.apache.hadoop" % "hadoop-client" % hadoopVersion, "org.scalatest" %% "scalatest" % scalaTestVersion % "test", "org.apache.spark" %% "spark-core" % defaultSparkVersion % "test" ) ) val deltaIcebergSparkIncludePrefixes = Seq( // We want everything from this package "org/apache/spark/sql/delta/icebergShaded", // Server-side planning support "org/apache/spark/sql/delta/serverSidePlanning", // We only want the files in this project from this package. e.g. we want to exclude // org/apache/spark/sql/delta/commands/convert/ConvertTargetFile.class (from delta-spark project). "org/apache/spark/sql/delta/commands/convert/IcebergFileManifest", "org/apache/spark/sql/delta/commands/convert/IcebergSchemaUtils", "org/apache/spark/sql/delta/commands/convert/IcebergTable" ) // Build using: build/sbt clean icebergShaded/compile iceberg/compile // It will fail the first time, just re-run it. // scalastyle:off println lazy val iceberg = (project in file("iceberg")) .dependsOn(spark % "compile->compile;test->test;provided->provided") .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin) .settings ( name := "delta-iceberg", commonSettings, scalaStyleSettings, releaseSettings, // Set sparkVersion directly (not sparkDependentModuleName) so that // runOnlyForReleasableSparkModules discovers this module, but without adding a Spark // suffix to the artifact name. delta-iceberg is only published as delta-iceberg_2.13. sparkVersion := CrossSparkVersions.getSparkVersion(), libraryDependencies ++= { if (supportIceberg) { Seq( // Fix Iceberg's legacy java.lang.NoClassDefFoundError: scala/jdk/CollectionConverters$ error // due to legacy scala. "org.scala-lang.modules" %% "scala-collection-compat" % "2.1.1", "com.github.ben-manes.caffeine" % "caffeine" % "2.9.3", "com.jolbox" % "bonecp" % "0.8.0.RELEASE" % "test", "org.eclipse.jetty" % "jetty-server" % "11.0.26" % "test", "org.eclipse.jetty" % "jetty-servlet" % "11.0.26" % "test", "org.xerial" % "sqlite-jdbc" % "3.45.0.0" % "test", "org.apache.httpcomponents.core5" % "httpcore5" % "5.2.4" % "test", "org.apache.httpcomponents.client5" % "httpclient5" % "5.3.1" % "test", "org.apache.iceberg" %% icebergSparkRuntimeArtifactName % "1.10.0" % "provided", // For FixedGcsAccessTokenProvider (GCS server-side planning credentials) "com.google.cloud.bigdataoss" % "util-hadoop" % "hadoop3-2.2.26" % "provided" ) } else { Seq.empty } }, // Skip compilation and publishing when supportIceberg is false Compile / skip := !supportIceberg, Test / skip := !supportIceberg, publish / skip := !supportIceberg, publishLocal / skip := !supportIceberg, publishM2 / skip := !supportIceberg, Compile / unmanagedJars += (icebergShaded / assembly).value, // Generate the assembly JAR as the package JAR Compile / packageBin := assembly.value, Compile / scalacOptions += "-nowarn", Test / unmanagedJars += (icebergTestsShaded / assembly).value, Test / scalacOptions += "-nowarn", assembly / assemblyJarName := { s"${moduleName.value}_${scalaBinaryVersion.value}-${version.value}.jar" }, assembly / logLevel := Level.Info, assembly / test := {}, assembly / assemblyExcludedJars := { // Note: the input here is only `libraryDependencies` jars, not `.dependsOn(_)` jars. val allowedJars = Seq( s"iceberg-shaded_${scalaBinaryVersion.value}-${version.value}.jar", s"scala-library-${scala213}.jar", s"scala-collection-compat_${scalaBinaryVersion.value}-2.1.1.jar", "caffeine-2.9.3.jar", // Note: We are excluding // - antlr4-runtime-4.9.3.jar // - checker-qual-3.19.0.jar // - error_prone_annotations-2.10.0.jar ) val cp = (assembly / fullClasspath).value // Return `true` when we want the jar `f` to be excluded from the assembly jar cp.filter { f => val doExclude = !allowedJars.contains(f.data.getName) println(s"Excluding jar: ${f.data.getName} ? $doExclude") doExclude } }, assembly / assemblyMergeStrategy := { // Project iceberg `dependsOn` spark and accidentally brings in it, along with its // compile-time dependencies (like delta-storage). We want these excluded from the // delta-iceberg jar. case PathList("io", "delta", xs @ _*) => // - delta-storage will bring in classes: io/delta/storage // - delta-spark will bring in classes: io/delta/exceptions/, io/delta/implicits, // io/delta/package, io/delta/sql, io/delta/tables, MergeStrategy.discard case PathList("com", "databricks", xs @ _*) => // delta-spark will bring in com/databricks/spark/util MergeStrategy.discard case PathList("org", "apache", "spark", xs @ _*) if !deltaIcebergSparkIncludePrefixes.exists { prefix => s"org/apache/spark/${xs.mkString("/")}".startsWith(prefix) } => MergeStrategy.discard case PathList("scoverage", xs @ _*) => MergeStrategy.discard case x => (assembly / assemblyMergeStrategy).value(x) }, assemblyPackageScala / assembleArtifact := false ) // scalastyle:on println val icebergShadedVersion = "1.10.1" lazy val icebergShaded = (project in file("icebergShaded")) .dependsOn(spark % "provided") .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin) .settings ( name := "iceberg-shaded", commonSettings, skipReleaseSettings, // must exclude all dependencies from Iceberg that delta-spark includes libraryDependencies ++= Seq( // Fix Iceberg's legacy java.lang.NoClassDefFoundError: scala/jdk/CollectionConverters$ error // due to legacy scala. "org.scala-lang.modules" %% "scala-collection-compat" % "2.1.1" % "provided", "org.apache.iceberg" % "iceberg-core" % icebergShadedVersion excludeAll ( icebergExclusionRules: _* ), "org.apache.iceberg" % "iceberg-hive-metastore" % icebergShadedVersion excludeAll ( icebergExclusionRules: _* ), // the hadoop client and hive metastore versions come from this file in the // iceberg repo of icebergShadedVersion: iceberg/gradle/libs.versions.toml "org.apache.hadoop" % "hadoop-client" % "2.7.3" % "provided" excludeAll ( hadoopClientExclusionRules: _* ), "org.apache.hive" % "hive-metastore" % "2.3.8" % "provided" excludeAll ( hiveMetastoreExclusionRules: _* ) ), // Generated shaded Iceberg JARs Compile / packageBin := assembly.value, assembly / assemblyJarName := s"${name.value}_${scalaBinaryVersion.value}-${version.value}.jar", assembly / logLevel := Level.Info, assembly / test := {}, assembly / assemblyShadeRules := Seq( ShadeRule.rename("org.apache.iceberg.**" -> "shadedForDelta.@0").inAll ), assembly / assemblyExcludedJars := { val cp = (assembly / fullClasspath).value cp.filter { jar => val doExclude = jar.data.getName.contains("jackson-annotations") || jar.data.getName.contains("RoaringBitmap") doExclude } }, // all following clases have Delta customized implementation under icebergShaded/src and thus // require them to be 'first' to replace the class from iceberg jar assembly / assemblyMergeStrategy := updateMergeStrategy((assembly / assemblyMergeStrategy).value), assemblyPackageScala / assembleArtifact := false, ) lazy val icebergTestsShaded = (project in file("icebergTestsShaded")) .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin) .settings ( name := "iceberg-tests-shaded", commonSettings, skipReleaseSettings, // must exclude all dependencies from Iceberg that delta-spark includes libraryDependencies ++= Seq( "org.apache.iceberg" % "iceberg-core" % icebergShadedVersion classifier "tests" excludeAll ( icebergExclusionRules: _* ), ), // Generated shaded Iceberg JARs Compile / packageBin := assembly.value, assembly / assemblyJarName := s"${name.value}_${scalaBinaryVersion.value}-${version.value}.jar", assembly / logLevel := Level.Info, assembly / test := {}, assembly / assemblyShadeRules := Seq( ShadeRule.rename("org.apache.iceberg.**" -> "shadedForDelta.@0").inAll ), assembly / assemblyExcludedJars := { val cp = (fullClasspath in assembly).value cp.filter { jar => val doExclude = jar.data.getName.contains("jackson-annotations") || jar.data.getName.contains("RoaringBitmap") doExclude } }, assembly / assemblyMergeStrategy := updateMergeStrategy((assembly / assemblyMergeStrategy).value), assemblyPackageScala / assembleArtifact := false, ) lazy val hudi = (project in file("hudi")) .dependsOn(spark % "compile->compile;test->test;provided->provided") .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin) .settings ( name := "delta-hudi", commonSettings, scalaStyleSettings, releaseSettings, // Set sparkVersion directly (not sparkDependentModuleName) so that // runOnlyForReleasableSparkModules discovers this module, but without adding a Spark // suffix to the artifact name. delta-hudi is only published as delta-hudi_2.13. sparkVersion := CrossSparkVersions.getSparkVersion(), libraryDependencies ++= { if (supportHudi) { Seq( "org.apache.hudi" % "hudi-java-client" % "0.15.0" % "compile" excludeAll( ExclusionRule(organization = "org.apache.hadoop"), ExclusionRule(organization = "org.apache.zookeeper"), ), "org.apache.spark" %% "spark-avro" % sparkVersion.value % "test" excludeAll ExclusionRule(organization = "org.apache.hadoop"), "org.apache.parquet" % "parquet-avro" % "1.12.3" % "compile" ) } else { Seq.empty } }, // Skip compilation and publishing when supportHudi is false Compile / skip := !supportHudi, Test / skip := !supportHudi, publish / skip := !supportHudi, publishLocal / skip := !supportHudi, publishM2 / skip := !supportHudi, assembly / assemblyJarName := s"${name.value}-assembly_${scalaBinaryVersion.value}-${version.value}.jar", assembly / logLevel := Level.Info, assembly / test := {}, assembly / assemblyMergeStrategy := { // Project hudi `dependsOn` spark and accidentally brings in it, along with its // compile-time dependencies (like delta-storage). We want these excluded from the // delta-hudi jar. case PathList("io", "delta", xs @ _*) => // - delta-storage will bring in classes: io/delta/storage // - delta-spark will bring in classes: io/delta/exceptions/, io/delta/implicits, // io/delta/package, io/delta/sql, io/delta/tables, MergeStrategy.discard case PathList("com", "databricks", xs @ _*) => // delta-spark will bring in com/databricks/spark/util MergeStrategy.discard case PathList("org", "apache", "spark", "sql", "delta", "hudi", xs @ _*) => MergeStrategy.first case PathList("org", "apache", "spark", xs @ _*) => MergeStrategy.discard // Discard `module-info.class` to fix the `different file contents found` error. // TODO Upgrade SBT to 1.5 which will do this automatically case "module-info.class" => MergeStrategy.discard // Discard unused `parquet.thrift` so that we don't conflict the file used by the user case "parquet.thrift" => MergeStrategy.discard // Hudi metadata writer requires this service file to be present on the classpath case "META-INF/services/org.apache.hadoop.hbase.regionserver.MetricsRegionServerSourceFactory" => MergeStrategy.first // Discard the jackson service configs that we don't need. These files are not shaded so // adding them may conflict with other jackson version used by the user. case PathList("META-INF", "services", xs @ _*) => MergeStrategy.discard case x => MergeStrategy.first }, // Make the 'compile' invoke the 'assembly' task to generate the uber jar. Compile / packageBin := assembly.value, TestParallelization.settings ) lazy val flink = (project in file("flink")) // .dependsOn(kernelApi) .dependsOn(kernelDefaults) .dependsOn(kernelUnityCatalog) .settings( name := "delta-flink", commonSettings, skipReleaseSettings, javafmtCheckSettings(), publishArtifact := scalaBinaryVersion.value == "2.12", // only publish once autoScalaLibrary := false, // exclude scala-library from dependencies assembly / assemblyJarName := s"delta-flink-$flinkVersion-${version.value}.jar", assembly / assemblyMergeStrategy := { // Discard module-info.class files from Java 9+ modules and multi-release JARs case "module-info.class" => MergeStrategy.discard case "parquet.thrift" => MergeStrategy.discard case PathList("META-INF", "versions", _, "module-info.class") => MergeStrategy.discard case PathList("mozilla", "public-suffix-list.txt") => MergeStrategy.discard case x => MergeStrategy.first }, assembly / assemblyExcludedJars := { val cp = (assembly / fullClasspath).value cp.filter { entry => entry.data.getName.startsWith("bundle-") && entry.data.getName.endsWith(".jar") } }, Compile / unmanagedJars += (kernelApi / Compile / packageBin).value, Test / unmanagedJars += (kernelApi / Compile / packageBin).value, // Make sure the shaded JAR is produced before we compile/run tests Compile / compile := (Compile / compile).dependsOn(kernelApi / Compile / packageBin).value, Test / test := (Test / test).dependsOn(kernelApi / Compile / packageBin).value, Test / unmanagedJars += (kernelApi / Test / packageBin).value, Test / publishArtifact := false, Test / javaOptions ++= Seq( "--add-opens=java.base/java.util=ALL-UNNAMED" // for Flink with Java 17. ), crossPaths := false, libraryDependencies ++= Seq( "org.apache.flink" % "flink-core" % flinkVersion % "provided", "org.apache.flink" % "flink-table-common" % flinkVersion % "provided", "org.apache.flink" % "flink-streaming-java" % flinkVersion % "provided", "org.apache.flink" % "flink-table-api-java-bridge" % flinkVersion % "provided", "io.unitycatalog" % "unitycatalog-client" % "0.3.1", "org.apache.httpcomponents" % "httpclient" % "4.5.14" % Runtime, "dev.failsafe" % "failsafe" % "3.2.0", "com.github.ben-manes.caffeine" % "caffeine" % "3.1.8", "org.apache.hadoop" % "hadoop-aws" % hadoopVersion, // Test dependencies "org.junit.jupiter" % "junit-jupiter-api" % "5.11.4" % "test", "org.junit.jupiter" % "junit-jupiter-engine" % "5.11.4" % "test", "org.junit.jupiter" % "junit-jupiter-params" % "5.11.4" % "test", "com.github.sbt.junit" % "jupiter-interface" % "0.17.0" % "test", "org.apache.flink" % "flink-test-utils" % flinkVersion % "test", "org.apache.flink" % "flink-clients" % flinkVersion % "test", "org.apache.flink" % "flink-table-api-java-bridge" % flinkVersion % Test, "org.apache.flink" % "flink-table-planner-loader" % flinkVersion % Test, "org.apache.flink" % "flink-table-runtime" % flinkVersion % Test, "org.apache.flink" % "flink-test-utils-junit" % flinkVersion % Test, "org.slf4j" % "slf4j-log4j12" % "2.0.17" % "test", "com.github.tomakehurst" % "wiremock-jre8" % "2.35.0" % Test ), // Use jupiter excludeDependencies ++= Seq( ExclusionRule("junit", "junit"), ExclusionRule("org.junit.vintage", "junit-vintage-engine") ) ) lazy val goldenTables = (project in file("connectors/golden-tables")) .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin) .settings( name := "golden-tables", commonSettings, skipReleaseSettings, libraryDependencies ++= Seq( // Test Dependencies "org.scalatest" %% "scalatest" % scalaTestVersion % "test", "commons-io" % "commons-io" % "2.8.0" % "test", "io.delta" %% "delta-spark" % "3.3.2" % "test", "org.apache.spark" %% "spark-sql" % defaultSparkVersion % "test", "org.apache.spark" %% "spark-catalyst" % defaultSparkVersion % "test" classifier "tests", "org.apache.spark" %% "spark-core" % defaultSparkVersion % "test" classifier "tests", "org.apache.spark" %% "spark-sql" % defaultSparkVersion % "test" classifier "tests" ) ) /** * Get list of python files and return the mapping between source files and target paths * in the generated package JAR. */ def listPythonFiles(pythonBase: File): Seq[(File, String)] = { val pythonExcludeDirs = pythonBase / "lib" :: pythonBase / "doc" :: pythonBase / "bin" :: Nil import scala.collection.JavaConverters._ val pythonFiles = Files.walk(pythonBase.toPath).iterator().asScala .map { path => path.toFile() } .filter { file => file.getName.endsWith(".py") && ! file.getName.contains("test") } .filter { file => ! pythonExcludeDirs.exists { base => IO.relativize(base, file).nonEmpty} } .toSeq pythonFiles pair Path.relativeTo(pythonBase) } ThisBuild / parallelExecution := false val createTargetClassesDir = taskKey[Unit]("create target classes dir") val generatePythonVersion = taskKey[File]("Generate Python version.py file") /* ****************** * Project groups * ****************** */ // Don't use these groups for any other projects lazy val sparkGroup = { val baseProjects = Seq(spark, sparkV1, sparkV1Filtered, sparkV2, contribs, sparkUnityCatalog, storage, storageS3DynamoDB, sharing, connectCommon, connectClient, connectServer) val allProjects = if (supportHudi) { baseProjects :+ hudi } else { baseProjects } Project("sparkGroup", file("sparkGroup")) .aggregate(allProjects.map(_.project): _*) .settings( // crossScalaVersions must be set to Nil on the aggregating project crossScalaVersions := Nil, publishArtifact := false, publish / skip := true, ) } lazy val icebergGroup = { val allProjects = if (supportIceberg) { Seq(iceberg, testDeltaIcebergJar) } else { Seq.empty } Project("icebergGroup", file("icebergGroup")) .aggregate(allProjects.map(_.project): _*) .settings( // crossScalaVersions must be set to Nil on the aggregating project crossScalaVersions := Nil, publishArtifact := false, publish / skip := true, ) } lazy val kernelGroup = project .aggregate(kernelApi, kernelDefaults, kernelBenchmarks) .settings( // crossScalaVersions must be set to Nil on the aggregating project crossScalaVersions := Nil, publishArtifact := false, publish / skip := true, unidocSourceFilePatterns := { (kernelApi / unidocSourceFilePatterns).value.scopeToProject(kernelApi) ++ (kernelDefaults / unidocSourceFilePatterns).value.scopeToProject(kernelDefaults) } ).configureUnidoc(docTitle = "Delta Kernel") lazy val flinkGroup = project .aggregate(flink) .settings( // crossScalaVersions must be set to Nil on the aggregating project crossScalaVersions := Nil, publishArtifact := false, publish / skip := true, ) /* ******************** * Release settings * ******************** */ import ReleaseTransformations._ lazy val skipReleaseSettings = Seq( publishArtifact := false, publish / skip := true ) // Release settings for artifact that contains only Java source code lazy val javaOnlyReleaseSettings = releaseSettings ++ Seq( // drop off Scala suffix from artifact names crossPaths := false, // we publish jars for each scalaVersion in crossScalaVersions. however, we only need to publish // one java jar. thus, only do so when the current scala version == default scala version publishArtifact := { val (expMaj, expMin, _) = getMajorMinorPatch(default_scala_version.value) s"$expMaj.$expMin" == scalaBinaryVersion.value }, // exclude scala-library from dependencies in generated pom.xml autoScalaLibrary := false, ) lazy val releaseSettings = Seq( publishMavenStyle := true, publishArtifact := true, Test / publishArtifact := false, releasePublishArtifactsAction := PgpKeys.publishSigned.value, releaseCrossBuild := true, pgpPassphrase := sys.env.get("PGP_PASSPHRASE").map(_.toArray), // TODO: This isn't working yet ... sonatypeProfileName := "io.delta", // sonatype account domain name prefix / group ID credentials += Credentials( "OSSRH Staging API Service", "ossrh-staging-api.central.sonatype.com", sys.env.getOrElse("SONATYPE_USERNAME", ""), sys.env.getOrElse("SONATYPE_PASSWORD", "") ), credentials += Credentials( "Sonatype Nexus Repository Manager", "central.sonatype.com", sys.env.getOrElse("SONATYPE_USERNAME", ""), sys.env.getOrElse("SONATYPE_PASSWORD", "") ), publishTo := { val ossrhBase = "https://ossrh-staging-api.central.sonatype.com/" val centralSnapshots = "https://central.sonatype.com/repository/maven-snapshots/" if (isSnapshot.value) { Some("snapshots" at centralSnapshots) } else { Some("releases" at ossrhBase + "service/local/staging/deploy/maven2") } }, licenses += ("Apache-2.0", url("http://www.apache.org/licenses/LICENSE-2.0")), pomExtra := https://delta.io/ git@github.com:delta-io/delta.git scm:git:git@github.com:delta-io/delta.git marmbrus Michael Armbrust https://github.com/marmbrus brkyvz Burak Yavuz https://github.com/brkyvz jose-torres Jose Torres https://github.com/jose-torres liwensun Liwen Sun https://github.com/liwensun mukulmurthy Mukul Murthy https://github.com/mukulmurthy tdas Tathagata Das https://github.com/tdas zsxwing Shixiong Zhu https://github.com/zsxwing scottsand-db Scott Sandre https://github.com/scottsand-db windpiger Jun Song https://github.com/windpiger ) // Looks like some of release settings should be set for the root project as well. publishArtifact := false // Don't release the root project publish / skip := true publishTo := Some("snapshots" at "https://central.sonatype.com/repository/maven-snapshots/") releaseCrossBuild := false // Don't use sbt-release's cross facility releaseProcess := Seq[ReleaseStep]( checkSnapshotDependencies, inquireVersions, runTest, setReleaseVersion, commitReleaseVersion, tagRelease ) ++ CrossSparkVersions.crossSparkReleaseSteps("publishSigned") ++ Seq[ReleaseStep]( // Do NOT use `sonatypeBundleRelease` - it will actually release to Maven! We want to do that // manually. // // Do NOT use `sonatypePromote` - it will promote the closed staging repository (i.e. sync to // Maven central) // // See https://github.com/xerial/sbt-sonatype#publishing-your-artifact. // // - sonatypePrepare: Drop the existing staging repositories (if exist) and create a new staging // repository using sonatypeSessionName as a unique key // - sonatypeBundleUpload: Upload your local staging folder contents to a remote Sonatype // repository // - sonatypeClose: closes your staging repository at Sonatype. This step verifies Maven central // sync requirement, GPG-signature, javadoc and source code presence, pom.xml // settings, etc // TODO: this isn't working yet // releaseStepCommand("sonatypePrepare; sonatypeBundleUpload; sonatypeClose"), setNextVersion, commitNextVersion ) ================================================ FILE: connectors/.gitignore ================================================ *#*# *.#* *.iml *.ipr *.iws *.pyc *.pyo *.swp *~ .DS_Store .bsp .cache .classpath .ensime .ensime_cache/ .ensime_lucene .generated-mima* .idea/ .idea_modules/ .project .pydevproject .scala_dependencies .settings *.pbix /lib/ R-unit-tests.log R/unit-tests.out R/cran-check.out R/pkg/vignettes/sparkr-vignettes.html R/pkg/tests/fulltests/Rplots.pdf build/*.jar build/apache-maven* build/scala* build/zinc* cache conf/*.cmd conf/*.conf conf/*.properties conf/*.sh conf/*.xml conf/java-opts conf/slaves dependency-reduced-pom.xml derby.log dev/create-release/*final dev/create-release/*txt dev/pr-deps/ dist/ docs/_site docs/api sql/docs sql/site lib_managed/ lint-r-report.log log/ logs/ out/ project/boot/ project/build/target/ project/plugins/lib_managed/ project/plugins/project/build.properties project/plugins/src_managed/ project/plugins/target/ python/lib/pyspark.zip python/deps docs/python/_static/ docs/python/_templates/ docs/python/_build/ python/test_coverage/coverage_data python/test_coverage/htmlcov python/pyspark/python reports/ scalastyle-on-compile.generated.xml scalastyle-output.xml scalastyle.txt spark-*-bin-*.tgz spark-tests.log src_managed/ streaming-tests.log target/ unit-tests.log work/ docs/.jekyll-metadata # For Hive TempStatsStore/ metastore/ metastore_db/ sql/hive-thriftserver/test_warehouses warehouse/ spark-warehouse/ # For R session data .RData .RHistory .Rhistory *.Rproj *.Rproj.* .Rproj.user **/src/main/resources/js # For SBT .jvmopts # For VS /.vs /obj /bin ================================================ FILE: connectors/README.md ================================================ Connectors projects are no longer maintained in the `master` branch and new releases due to migration to the Delta Kernel project. Projects will continue to be supported in maintanence mode from the `spark-3.5-support` branch. ================================================ FILE: connectors/golden-tables/src/main/resources/golden/124-decimal-decode-bug/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1636689272898,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputBytes":"844","numOutputRows":"1"}}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"large_decimal\",\"type\":\"decimal(10,0)\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1636689270919}} {"add":{"path":"part-00000-2abbde89-2d0f-465e-a2f0-3e84f1b84654-c000.snappy.parquet","partitionValues":{},"size":333,"modificationTime":1636689272000,"dataChange":true}} {"add":{"path":"part-00001-5419c9a2-bb44-454f-a109-6e6c6f000a24-c000.snappy.parquet","partitionValues":{},"size":511,"modificationTime":1636689272000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/125-iterator-bug/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1633728454095,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputBytes":"303","numOutputRows":"0"}}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"col1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1633728453099}} {"add":{"path":"part-00000-2a248db5-8f96-423c-a0f7-c503fe640c6a-c000.snappy.parquet","partitionValues":{},"size":303,"modificationTime":1633728454000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/125-iterator-bug/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1633728458439,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputBytes":"433","numOutputRows":"1"}}} {"add":{"path":"part-00000-15088d9b-5348-490b-933d-5bf9b7d0b223-c000.snappy.parquet","partitionValues":{},"size":433,"modificationTime":1633728458000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/125-iterator-bug/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1633728459288,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputBytes":"433","numOutputRows":"1"}}} {"add":{"path":"part-00000-c855206c-f42a-4b53-a526-08a9a957ad58-c000.snappy.parquet","partitionValues":{},"size":433,"modificationTime":1633728459000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/125-iterator-bug/_delta_log/00000000000000000003.json ================================================ {"commitInfo":{"timestamp":1633728460020,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":2,"isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputBytes":"303","numOutputRows":"0"}}} {"add":{"path":"part-00000-3f0f0396-41aa-4fa7-954a-c5b22f5b157a-c000.snappy.parquet","partitionValues":{},"size":303,"modificationTime":1633728460000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/125-iterator-bug/_delta_log/00000000000000000004.json ================================================ {"commitInfo":{"timestamp":1633728460726,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":3,"isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputBytes":"433","numOutputRows":"1"}}} {"add":{"path":"part-00000-c4738537-d851-4caa-9596-d543afa47196-c000.snappy.parquet","partitionValues":{},"size":433,"modificationTime":1633728460000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/125-iterator-bug/_delta_log/00000000000000000005.json ================================================ {"commitInfo":{"timestamp":1633728461405,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":4,"isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputBytes":"303","numOutputRows":"0"}}} {"add":{"path":"part-00000-f9490ff6-f374-4b40-9d76-22addae085d1-c000.snappy.parquet","partitionValues":{},"size":303,"modificationTime":1633728461000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/125-iterator-bug/_delta_log/00000000000000000006.json ================================================ {"commitInfo":{"timestamp":1633728462063,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":5,"isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputBytes":"303","numOutputRows":"0"}}} {"add":{"path":"part-00000-66d18d0c-8cab-4cfa-a2c6-7e90df860b5a-c000.snappy.parquet","partitionValues":{},"size":303,"modificationTime":1633728462000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/125-iterator-bug/_delta_log/00000000000000000007.json ================================================ {"commitInfo":{"timestamp":1633728462739,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":6,"isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputBytes":"433","numOutputRows":"1"}}} {"add":{"path":"part-00000-1b8ea57e-424b-4068-8d0e-707edf853376-c000.snappy.parquet","partitionValues":{},"size":433,"modificationTime":1633728462000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/125-iterator-bug/_delta_log/00000000000000000008.json ================================================ {"commitInfo":{"timestamp":1633728463394,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":7,"isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputBytes":"303","numOutputRows":"0"}}} {"add":{"path":"part-00000-93beced9-3a9d-4519-b31a-5602a972ffa4-c000.snappy.parquet","partitionValues":{},"size":303,"modificationTime":1633728463000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/125-iterator-bug/_delta_log/00000000000000000009.json ================================================ {"commitInfo":{"timestamp":1633728464026,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":8,"isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputBytes":"303","numOutputRows":"0"}}} {"add":{"path":"part-00000-d8e947c6-4f26-455b-a25f-84acb1240f3a-c000.snappy.parquet","partitionValues":{},"size":303,"modificationTime":1633728464000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/125-iterator-bug/_delta_log/00000000000000000010.json ================================================ {"commitInfo":{"timestamp":1633728464667,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":9,"isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputBytes":"303","numOutputRows":"0"}}} {"add":{"path":"part-00000-f0b12818-15f5-4476-8ebc-9235c74408d2-c000.snappy.parquet","partitionValues":{},"size":303,"modificationTime":1633728464000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/125-iterator-bug/_delta_log/00000000000000000011.json ================================================ {"commitInfo":{"timestamp":1633728465909,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":10,"isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputBytes":"433","numOutputRows":"1"}}} {"add":{"path":"part-00000-223768c3-2e58-4e8a-9d15-54fa113e8c21-c000.snappy.parquet","partitionValues":{},"size":433,"modificationTime":1633728465000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/125-iterator-bug/_delta_log/_last_checkpoint ================================================ {"version":10,"size":13} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-decimal-table/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1690853005164,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"part\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"4","numOutputRows":"4","numOutputBytes":"4131"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"451ba03f-e80c-4fda-9bba-8fdfda856925"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"part\",\"type\":\"decimal(12,5)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"col1\",\"type\":\"decimal(5,2)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"col2\",\"type\":\"decimal(10,5)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"col3\",\"type\":\"decimal(20,10)\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["part"],"configuration":{},"createdTime":1690852998865}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part=-2342342.23423/part-00000-8f850371-9b03-42c4-9d22-f83bc81c9b68.c000.snappy.parquet","partitionValues":{"part":"-2342342.23423"},"size":1032,"modificationTime":1690853004000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col1\":-999.99,\"col2\":-99999.99999,\"col3\":-9999999999.9999999999},\"maxValues\":{\"col1\":-999.99,\"col2\":-99999.99999,\"col3\":-9999999999.9999999999},\"nullCount\":{\"col1\":0,\"col2\":0,\"col3\":0}}"}} {"add":{"path":"part=0.00004/part-00000-1cb60e36-6cd4-4191-a318-ae9355f877c3.c000.snappy.parquet","partitionValues":{"part":"0.00004"},"size":1033,"modificationTime":1690853004000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col1\":0.00,\"col2\":0.00000,\"col3\":0E-10},\"maxValues\":{\"col1\":0.00,\"col2\":0.00000,\"col3\":0E-10},\"nullCount\":{\"col1\":0,\"col2\":0,\"col3\":0}}"}} {"add":{"path":"part=234.00000/part-00000-ac109189-97e5-49af-947f-335a5e46ee5c.c000.snappy.parquet","partitionValues":{"part":"234.00000"},"size":1033,"modificationTime":1690853004000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col1\":1.00,\"col2\":2.00000,\"col3\":3.0000000000},\"maxValues\":{\"col1\":1.00,\"col2\":2.00000,\"col3\":3.0000000000},\"nullCount\":{\"col1\":0,\"col2\":0,\"col3\":0}}"}} {"add":{"path":"part=2342222.23454/part-00000-d5a0c70f-7cd3-4d32-a9c0-7171a06547c6.c000.snappy.parquet","partitionValues":{"part":"2342222.23454"},"size":1033,"modificationTime":1690853004000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col1\":111.11,\"col2\":22222.22222,\"col3\":3333333333.3333333333},\"maxValues\":{\"col1\":111.11,\"col2\":22222.22222,\"col3\":3333333333.3333333333},\"nullCount\":{\"col1\":0,\"col2\":0,\"col3\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-decimal-table-legacy/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1690853019754,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"part\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"4","numOutputRows":"4","numOutputBytes":"4036"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"01c3f3ee-9b79-4245-93f1-7f43ce7afa9c"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"part\",\"type\":\"decimal(12,5)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"col1\",\"type\":\"decimal(5,2)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"col2\",\"type\":\"decimal(10,5)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"col3\",\"type\":\"decimal(20,10)\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["part"],"configuration":{},"createdTime":1690853018509}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part=-2342342.23423/part-00000-ba2f74ac-7b9b-47b9-a287-97d92bd20efc.c000.snappy.parquet","partitionValues":{"part":"-2342342.23423"},"size":1009,"modificationTime":1690853019000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col1\":-999.99,\"col2\":-99999.99999,\"col3\":-9999999999.9999999999},\"maxValues\":{\"col1\":-999.99,\"col2\":-99999.99999,\"col3\":-9999999999.9999999999},\"nullCount\":{\"col1\":0,\"col2\":0,\"col3\":0}}"}} {"add":{"path":"part=0.00004/part-00000-3de65390-7061-47d6-8995-cbb632b4b203.c000.snappy.parquet","partitionValues":{"part":"0.00004"},"size":1009,"modificationTime":1690853019000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col1\":0.00,\"col2\":0.00000,\"col3\":0E-10},\"maxValues\":{\"col1\":0.00,\"col2\":0.00000,\"col3\":0E-10},\"nullCount\":{\"col1\":0,\"col2\":0,\"col3\":0}}"}} {"add":{"path":"part=234.00000/part-00000-654d80b0-611a-4ff3-a8e6-2328dd21cf11.c000.snappy.parquet","partitionValues":{"part":"234.00000"},"size":1009,"modificationTime":1690853019000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col1\":1.00,\"col2\":2.00000,\"col3\":3.0000000000},\"maxValues\":{\"col1\":1.00,\"col2\":2.00000,\"col3\":3.0000000000},\"nullCount\":{\"col1\":0,\"col2\":0,\"col3\":0}}"}} {"add":{"path":"part=2342222.23454/part-00000-fe848a88-0465-4b4f-8414-25e6da7062f8.c000.snappy.parquet","partitionValues":{"part":"2342222.23454"},"size":1009,"modificationTime":1690853019000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col1\":111.11,\"col2\":22222.22222,\"col3\":3333333333.3333333333},\"maxValues\":{\"col1\":111.11,\"col2\":22222.22222,\"col3\":3333333333.3333333333},\"nullCount\":{\"col1\":0,\"col2\":0,\"col3\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-deletes-checkpoint/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1691426732135,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"10","numOutputBytes":"539"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"8c9f4d2d-645f-4f02-b13b-aa6da098716c"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1691426730560}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part-00000-b09fdf65-0ae3-44d0-96d0-1d85a121b76a-c000.snappy.parquet","partitionValues":{},"size":539,"modificationTime":1691426732072,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-deletes-checkpoint/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1691426734180,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"10","numOutputBytes":"527"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"1529cdd3-5178-47ce-b13d-962cbfdcb028"}} {"add":{"path":"part-00000-0869ab64-e69d-407f-80d4-1a2ea1f69d11-c000.snappy.parquet","partitionValues":{},"size":527,"modificationTime":1691426734175,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":10},\"maxValues\":{\"id\":19},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-deletes-checkpoint/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1691426734787,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"10","numOutputBytes":"527"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"d1bc1416-99e2-4aba-9e4b-1b157d9d2b49"}} {"add":{"path":"part-00000-bd8763c3-45e4-435e-acd6-8e599aa840bc-c000.snappy.parquet","partitionValues":{},"size":527,"modificationTime":1691426734784,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":20},\"maxValues\":{\"id\":29},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-deletes-checkpoint/_delta_log/00000000000000000003.json ================================================ {"commitInfo":{"timestamp":1691426735371,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"10","numOutputBytes":"527"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"3dfdeb21-169d-4197-a49b-411fc956153a"}} {"add":{"path":"part-00000-60f14460-c8e0-41b4-a33f-1a83bb59f13c-c000.snappy.parquet","partitionValues":{},"size":527,"modificationTime":1691426735367,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":30},\"maxValues\":{\"id\":39},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-deletes-checkpoint/_delta_log/00000000000000000004.json ================================================ {"commitInfo":{"timestamp":1691426735942,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":3,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"10","numOutputBytes":"527"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"56a9553b-8f9d-44e0-a236-25081a0083ad"}} {"add":{"path":"part-00000-b326e43b-3e01-4cf1-b8ff-c73c8abd1616-c000.snappy.parquet","partitionValues":{},"size":527,"modificationTime":1691426735939,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":40},\"maxValues\":{\"id\":49},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-deletes-checkpoint/_delta_log/00000000000000000005.json ================================================ {"commitInfo":{"timestamp":1691426737153,"operation":"DELETE","operationParameters":{"predicate":"[\"((id#1904L >= 5) AND (id#1904L <= 9))\"]"},"readVersion":4,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"539","numCopiedRows":"5","numAddedChangeFiles":"0","executionTimeMs":"681","numDeletedRows":"5","scanTimeMs":"508","numAddedFiles":"1","numAddedBytes":"500","rewriteTimeMs":"172"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"8deec26c-c828-46dc-962d-e324783081c0"}} {"remove":{"path":"part-00000-b09fdf65-0ae3-44d0-96d0-1d85a121b76a-c000.snappy.parquet","deletionTimestamp":1691426737143,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":539}} {"add":{"path":"part-00000-ca2d0b26-c15c-454f-a933-fc724e15e5f1-c000.snappy.parquet","partitionValues":{},"size":500,"modificationTime":1691426737139,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-deletes-checkpoint/_delta_log/00000000000000000006.json ================================================ {"commitInfo":{"timestamp":1691426737814,"operation":"DELETE","operationParameters":{"predicate":"[\"((id#2606L >= 15) AND (id#2606L <= 19))\"]"},"readVersion":5,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"527","numCopiedRows":"5","numAddedChangeFiles":"0","executionTimeMs":"284","numDeletedRows":"5","scanTimeMs":"168","numAddedFiles":"1","numAddedBytes":"503","rewriteTimeMs":"116"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"dc2ad0e6-7337-4501-a47b-c6af0138b7dc"}} {"remove":{"path":"part-00000-0869ab64-e69d-407f-80d4-1a2ea1f69d11-c000.snappy.parquet","deletionTimestamp":1691426737813,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":527}} {"add":{"path":"part-00000-c92cba9e-6c07-4a93-916a-0a6e115e39b3-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1691426737811,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":10},\"maxValues\":{\"id\":14},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-deletes-checkpoint/_delta_log/00000000000000000007.json ================================================ {"commitInfo":{"timestamp":1691426738561,"operation":"DELETE","operationParameters":{"predicate":"[\"((id#3297L >= 25) AND (id#3297L <= 29))\"]"},"readVersion":6,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"527","numCopiedRows":"5","numAddedChangeFiles":"0","executionTimeMs":"293","numDeletedRows":"5","scanTimeMs":"158","numAddedFiles":"1","numAddedBytes":"503","rewriteTimeMs":"135"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"dcdfb390-1424-44a8-a1ef-9713726bc6cb"}} {"remove":{"path":"part-00000-bd8763c3-45e4-435e-acd6-8e599aa840bc-c000.snappy.parquet","deletionTimestamp":1691426738560,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":527}} {"add":{"path":"part-00000-1b0098ea-c696-4470-84cc-d43bb7afb833-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1691426738558,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":20},\"maxValues\":{\"id\":24},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-deletes-checkpoint/_delta_log/00000000000000000008.json ================================================ {"commitInfo":{"timestamp":1691426739285,"operation":"DELETE","operationParameters":{"predicate":"[\"((id#3988L >= 35) AND (id#3988L <= 39))\"]"},"readVersion":7,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"527","numCopiedRows":"5","numAddedChangeFiles":"0","executionTimeMs":"345","numDeletedRows":"5","scanTimeMs":"175","numAddedFiles":"1","numAddedBytes":"503","rewriteTimeMs":"170"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"8e184ae5-6338-43db-96d1-c54619c2e20f"}} {"remove":{"path":"part-00000-60f14460-c8e0-41b4-a33f-1a83bb59f13c-c000.snappy.parquet","deletionTimestamp":1691426739284,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":527}} {"add":{"path":"part-00000-f80053c6-2b0d-41ed-ab5f-61ef1503cae6-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1691426739280,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":30},\"maxValues\":{\"id\":34},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-deletes-checkpoint/_delta_log/00000000000000000009.json ================================================ {"commitInfo":{"timestamp":1691426740025,"operation":"DELETE","operationParameters":{"predicate":"[\"((id#4679L >= 45) AND (id#4679L <= 49))\"]"},"readVersion":8,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"527","numCopiedRows":"5","numAddedChangeFiles":"0","executionTimeMs":"324","numDeletedRows":"5","scanTimeMs":"158","numAddedFiles":"1","numAddedBytes":"503","rewriteTimeMs":"166"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"85e2749a-937c-4c55-8234-0342ae82a54f"}} {"remove":{"path":"part-00000-b326e43b-3e01-4cf1-b8ff-c73c8abd1616-c000.snappy.parquet","deletionTimestamp":1691426740025,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":527}} {"add":{"path":"part-00000-4b448490-06f4-4c74-9f65-9f36ae68e3b2-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1691426740023,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":40},\"maxValues\":{\"id\":44},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-deletes-checkpoint/_delta_log/00000000000000000010.json ================================================ {"commitInfo":{"timestamp":1691426740500,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":9,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"10","numOutputBytes":"527"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"5f7d481e-9635-43e1-a4a1-584bf0ae63f9"}} {"add":{"path":"part-00000-da82aeb5-4edb-4cc1-91ef-970c75c965cc-c000.snappy.parquet","partitionValues":{},"size":527,"modificationTime":1691426740498,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":50},\"maxValues\":{\"id\":59},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-deletes-checkpoint/_delta_log/00000000000000000011.json ================================================ {"commitInfo":{"timestamp":1691426741681,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":10,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"10","numOutputBytes":"527"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"a99db5a3-bc5b-44f8-abc5-ce30bda4c0b0"}} {"add":{"path":"part-00000-26da113c-2e45-4aba-b1ce-6eb5e46c53f7-c000.snappy.parquet","partitionValues":{},"size":527,"modificationTime":1691426741411,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":60},\"maxValues\":{\"id\":69},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-deletes-checkpoint/_delta_log/00000000000000000012.json ================================================ {"commitInfo":{"timestamp":1691426742288,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":11,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"10","numOutputBytes":"527"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"6c4dae3d-b3b4-4359-9abc-34883703faba"}} {"add":{"path":"part-00000-c967edfa-f104-44ea-b0da-8bc1f5402af4-c000.snappy.parquet","partitionValues":{},"size":527,"modificationTime":1691426742286,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":70},\"maxValues\":{\"id\":79},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-deletes-checkpoint/_delta_log/00000000000000000013.json ================================================ {"commitInfo":{"timestamp":1691426743015,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#6991L >= 66)\"]"},"readVersion":12,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"2","numRemovedBytes":"1054","numCopiedRows":"6","numAddedChangeFiles":"0","executionTimeMs":"270","numDeletedRows":"14","scanTimeMs":"151","numAddedFiles":"1","numAddedBytes":"510","rewriteTimeMs":"119"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"153f81dc-ba06-46e8-81c7-75c26e500a57"}} {"remove":{"path":"part-00000-26da113c-2e45-4aba-b1ce-6eb5e46c53f7-c000.snappy.parquet","deletionTimestamp":1691426743014,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":527}} {"remove":{"path":"part-00000-c967edfa-f104-44ea-b0da-8bc1f5402af4-c000.snappy.parquet","deletionTimestamp":1691426743014,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":527}} {"add":{"path":"part-00000-7d1a368c-74ea-42df-9527-2c9a7c8292b9-c000.snappy.parquet","partitionValues":{},"size":510,"modificationTime":1691426743013,"dataChange":true,"stats":"{\"numRecords\":6,\"minValues\":{\"id\":60},\"maxValues\":{\"id\":65},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-deletes-checkpoint/_delta_log/_last_checkpoint ================================================ {"version":10,"size":13,"sizeInBytes":16479,"numOfAddFiles":6,"checkpointSchema":{"type":"struct","fields":[{"name":"txn","type":{"type":"struct","fields":[{"name":"appId","type":"string","nullable":true,"metadata":{}},{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"lastUpdated","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"add","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}},{"name":"stats","type":"string","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"remove","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"deletionTimestamp","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"extendedFileMetadata","type":"boolean","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"metaData","type":{"type":"struct","fields":[{"name":"id","type":"string","nullable":true,"metadata":{}},{"name":"name","type":"string","nullable":true,"metadata":{}},{"name":"description","type":"string","nullable":true,"metadata":{}},{"name":"format","type":{"type":"struct","fields":[{"name":"provider","type":"string","nullable":true,"metadata":{}},{"name":"options","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"schemaString","type":"string","nullable":true,"metadata":{}},{"name":"partitionColumns","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"configuration","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"createdTime","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"protocol","type":{"type":"struct","fields":[{"name":"minReaderVersion","type":"integer","nullable":true,"metadata":{}},{"name":"minWriterVersion","type":"integer","nullable":true,"metadata":{}},{"name":"readerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"writerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"domainMetadata","type":{"type":"struct","fields":[{"name":"domain","type":"string","nullable":true,"metadata":{}},{"name":"configuration","type":"string","nullable":true,"metadata":{}},{"name":"removed","type":"boolean","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"checksum":"6872b3692f168925bdd80e3f92163949"} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-merge/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1697587480772,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"100","numOutputBytes":"2191"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"f7ec08aa-a91e-49c6-97e8-a1d13f4e20af"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"str\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1697587476380}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part-00000-b4335bad-f5f0-4426-9ec4-14ed854f862b-c000.snappy.parquet","partitionValues":{},"size":1091,"modificationTime":1697587480000,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":0,\"str\":\"val=0\"},\"maxValues\":{\"id\":49,\"str\":\"val=9\"},\"nullCount\":{\"id\":0,\"str\":0}}"}} {"add":{"path":"part-00001-b80a2dea-5a83-4580-96d5-4977d14195ab-c000.snappy.parquet","partitionValues":{},"size":1100,"modificationTime":1697587480000,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":50,\"str\":\"val=50\"},\"maxValues\":{\"id\":99,\"str\":\"val=99\"},\"nullCount\":{\"id\":0,\"str\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-merge/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1697587497062,"operation":"MERGE","operationParameters":{"predicate":"[\"(id#480L = cast(id#482 as bigint))\"]","matchedPredicates":"[{\"actionType\":\"update\"}]","notMatchedPredicates":"[{\"actionType\":\"insert\"}]","notMatchedBySourcePredicates":"[{\"predicate\":\"(id#482 < 10)\",\"actionType\":\"delete\"}]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numTargetRowsCopied":"40","numTargetRowsDeleted":"10","numTargetFilesAdded":"1","numTargetBytesAdded":"1495","numTargetBytesRemoved":"2191","numTargetRowsMatchedUpdated":"50","executionTimeMs":"5573","numTargetRowsInserted":"50","numTargetRowsMatchedDeleted":"0","scanTimeMs":"3600","numTargetRowsUpdated":"50","numOutputRows":"140","numTargetRowsNotMatchedBySourceUpdated":"0","numTargetChangeFilesAdded":"0","numSourceRows":"100","numTargetFilesRemoved":"2","numTargetRowsNotMatchedBySourceDeleted":"10","rewriteTimeMs":"1284"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"70de6669-546f-4f21-884c-96762f8bb154"}} {"remove":{"path":"part-00001-b80a2dea-5a83-4580-96d5-4977d14195ab-c000.snappy.parquet","deletionTimestamp":1697587496997,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1100,"stats":"{\"numRecords\":50}"}} {"remove":{"path":"part-00000-b4335bad-f5f0-4426-9ec4-14ed854f862b-c000.snappy.parquet","deletionTimestamp":1697587496998,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1091,"stats":"{\"numRecords\":50}"}} {"add":{"path":"part-00000-992247c6-6cf4-45f8-8367-11a5e14b8ea9-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1697587496000,"dataChange":true,"stats":"{\"numRecords\":140,\"minValues\":{\"id\":10,\"str\":\"EXT\"},\"maxValues\":{\"id\":149,\"str\":\"val=49\"},\"nullCount\":{\"id\":0,\"str\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-overwrite-restore/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1697588664008,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"100","numOutputBytes":"1454"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"0a02e21a-7f3f-4945-92c7-5761336283de"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1697588658132}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part-00000-d24c9b15-187d-4542-90ef-7834bfaa4971-c000.snappy.parquet","partitionValues":{},"size":765,"modificationTime":1697588663000,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":49},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-66f56273-e583-4a88-9da6-2c199bdaf665-c000.snappy.parquet","partitionValues":{},"size":689,"modificationTime":1697588663000,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":50},\"maxValues\":{\"id\":99},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-overwrite-restore/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1697588676444,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"100","numOutputBytes":"1380"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"64533fbc-52a0-4d99-a4dc-cf7e90c4f8f4"}} {"add":{"path":"part-00000-5e752668-638c-4e95-9521-5e88926e3169-c000.snappy.parquet","partitionValues":{},"size":689,"modificationTime":1697588676000,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":100},\"maxValues\":{\"id\":149},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-180c081a-f358-4bf9-8daa-4d04a5aa7f51-c000.snappy.parquet","partitionValues":{},"size":691,"modificationTime":1697588676000,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":150},\"maxValues\":{\"id\":199},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-overwrite-restore/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1697588681190,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"2","numOutputRows":"500","numOutputBytes":"2995"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"9ad1c5a7-c402-416b-90a9-dc22d4fa2b81"}} {"add":{"path":"part-00000-79fa68ed-3d70-4f61-95da-9eb676b24a98-c000.snappy.parquet","partitionValues":{},"size":1496,"modificationTime":1697588679000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":500},\"maxValues\":{\"id\":749},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-bc9b37c2-a201-499d-b604-93623e2de1d6-c000.snappy.parquet","partitionValues":{},"size":1499,"modificationTime":1697588679000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":750},\"maxValues\":{\"id\":999},\"nullCount\":{\"id\":0}}"}} {"remove":{"path":"part-00001-180c081a-f358-4bf9-8daa-4d04a5aa7f51-c000.snappy.parquet","deletionTimestamp":1697588681179,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":691}} {"remove":{"path":"part-00001-66f56273-e583-4a88-9da6-2c199bdaf665-c000.snappy.parquet","deletionTimestamp":1697588681180,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":689}} {"remove":{"path":"part-00000-5e752668-638c-4e95-9521-5e88926e3169-c000.snappy.parquet","deletionTimestamp":1697588681180,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":689}} {"remove":{"path":"part-00000-d24c9b15-187d-4542-90ef-7834bfaa4971-c000.snappy.parquet","deletionTimestamp":1697588681180,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":765}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-overwrite-restore/_delta_log/00000000000000000003.json ================================================ {"commitInfo":{"timestamp":1697588691204,"operation":"RESTORE","operationParameters":{"version":1,"timestamp":null},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRestoredFiles":"4","removedFilesSize":"2995","numRemovedFiles":"2","restoredFilesSize":"2834","numOfFilesAfterRestore":"4","tableSizeAfterRestore":"2834"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"ec6a69f9-8721-4043-ae13-84e1c26c1e1d"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1697588658132}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part-00001-66f56273-e583-4a88-9da6-2c199bdaf665-c000.snappy.parquet","partitionValues":{},"size":689,"modificationTime":1697588663000,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":50},\"maxValues\":{\"id\":99},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-180c081a-f358-4bf9-8daa-4d04a5aa7f51-c000.snappy.parquet","partitionValues":{},"size":691,"modificationTime":1697588676000,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":150},\"maxValues\":{\"id\":199},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00000-5e752668-638c-4e95-9521-5e88926e3169-c000.snappy.parquet","partitionValues":{},"size":689,"modificationTime":1697588676000,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":100},\"maxValues\":{\"id\":149},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00000-d24c9b15-187d-4542-90ef-7834bfaa4971-c000.snappy.parquet","partitionValues":{},"size":765,"modificationTime":1697588663000,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":49},\"nullCount\":{\"id\":0}}"}} {"remove":{"path":"part-00001-bc9b37c2-a201-499d-b604-93623e2de1d6-c000.snappy.parquet","deletionTimestamp":1697588691416,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1499,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":750},\"maxValues\":{\"id\":999},\"nullCount\":{\"id\":0}}"}} {"remove":{"path":"part-00000-79fa68ed-3d70-4f61-95da-9eb676b24a98-c000.snappy.parquet","deletionTimestamp":1697588691418,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1496,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":500},\"maxValues\":{\"id\":749},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-overwrite-restore/_delta_log/_last_checkpoint ================================================ {"version":3,"size":8,"sizeInBytes":16053,"numOfAddFiles":4,"checkpointSchema":{"type":"struct","fields":[{"name":"txn","type":{"type":"struct","fields":[{"name":"appId","type":"string","nullable":true,"metadata":{}},{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"lastUpdated","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"add","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}},{"name":"stats","type":"string","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"remove","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"deletionTimestamp","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"extendedFileMetadata","type":"boolean","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"metaData","type":{"type":"struct","fields":[{"name":"id","type":"string","nullable":true,"metadata":{}},{"name":"name","type":"string","nullable":true,"metadata":{}},{"name":"description","type":"string","nullable":true,"metadata":{}},{"name":"format","type":{"type":"struct","fields":[{"name":"provider","type":"string","nullable":true,"metadata":{}},{"name":"options","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"schemaString","type":"string","nullable":true,"metadata":{}},{"name":"partitionColumns","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"configuration","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"createdTime","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"protocol","type":{"type":"struct","fields":[{"name":"minReaderVersion","type":"integer","nullable":true,"metadata":{}},{"name":"minWriterVersion","type":"integer","nullable":true,"metadata":{}},{"name":"readerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"writerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"domainMetadata","type":{"type":"struct","fields":[{"name":"domain","type":"string","nullable":true,"metadata":{}},{"name":"configuration","type":"string","nullable":true,"metadata":{}},{"name":"removed","type":"boolean","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"checksum":"a68904381fcbd9efd0ac76b9756166c7"} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-updates/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1697587748016,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"100","numOutputBytes":"2191"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"1a8d27d3-4e3d-4d34-af7d-88cdd71e1b99"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"str\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1697587743472}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part-00000-f9886fc2-20a0-42fe-8b30-c3abb5e3c720-c000.snappy.parquet","partitionValues":{},"size":1091,"modificationTime":1697587747000,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":0,\"str\":\"val=0\"},\"maxValues\":{\"id\":49,\"str\":\"val=9\"},\"nullCount\":{\"id\":0,\"str\":0}}"}} {"add":{"path":"part-00001-13a6bfd9-3835-44dd-b4f1-465aa95b2bf4-c000.snappy.parquet","partitionValues":{},"size":1100,"modificationTime":1697587747000,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":50,\"str\":\"val=50\"},\"maxValues\":{\"id\":99,\"str\":\"val=99\"},\"nullCount\":{\"id\":0,\"str\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-inserts-updates/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1697587762295,"operation":"UPDATE","operationParameters":{"predicate":"[\"(id#480 < 50)\"]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"1091","numCopiedRows":"0","numDeletionVectorsAdded":"0","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"0","executionTimeMs":"4190","numDeletionVectorsUpdated":"0","scanTimeMs":"3438","numAddedFiles":"1","numUpdatedRows":"50","numAddedBytes":"912","rewriteTimeMs":"750"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"2a6cf901-2357-430d-a046-aaf95aa527de"}} {"add":{"path":"part-00000-6dfaec75-bd45-4fd6-b20f-7d58c9341479-c000.snappy.parquet","partitionValues":{},"size":912,"modificationTime":1697587762000,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":0,\"str\":\"N/A\"},\"maxValues\":{\"id\":49,\"str\":\"N/A\"},\"nullCount\":{\"id\":0,\"str\":0}}"}} {"remove":{"path":"part-00000-f9886fc2-20a0-42fe-8b30-c3abb5e3c720-c000.snappy.parquet","deletionTimestamp":1697587762273,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1091}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-vacuum-protocol-check-feature/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1711484004670,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"100","numOutputBytes":"2191"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"aa418762-a98f-4d46-af25-30c4ac4b18a1"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"str\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1711484003431}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part-00000-e719b63b-4142-4bad-9776-45642d5858ae-c000.snappy.parquet","partitionValues":{},"size":1091,"modificationTime":1711484004548,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":0,\"str\":\"val=0\"},\"maxValues\":{\"id\":49,\"str\":\"val=9\"},\"nullCount\":{\"id\":0,\"str\":0}}"}} {"add":{"path":"part-00001-fd905e0a-6d0c-4ce3-bb41-147517448b3b-c000.snappy.parquet","partitionValues":{},"size":1100,"modificationTime":1711484004548,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":50,\"str\":\"val=50\"},\"maxValues\":{\"id\":99,\"str\":\"val=99\"},\"nullCount\":{\"id\":0,\"str\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/basic-with-vacuum-protocol-check-feature/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1711484006901,"operation":"SET TBLPROPERTIES","operationParameters":{"properties":"{\"delta.feature.vacuumprotocolcheck\":\"supported\"}"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"b1e2fa88-d44e-4fd6-9e2a-a1fa020f0ba6"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"str\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1711484003431}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["vacuumProtocolCheck"],"writerFeatures":["appendOnly","invariants","vacuumProtocolCheck"]}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/canonicalized-paths-normal-a/_delta_log/00000000000000000000.json ================================================ {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"7afc4b76-09fb-4b06-836d-f9972b9c1f91","format":{"provider":"parquet","options":{}},"partitionColumns":[],"configuration":{},"createdTime":1603723990637}} {"add":{"path":"/some/unqualified/absolute/path","partitionValues":{},"size":100,"modificationTime":10,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/canonicalized-paths-normal-a/_delta_log/00000000000000000001.json ================================================ {"remove":{"path":"file:/some/unqualified/absolute/path","deletionTimestamp":200,"dataChange":false}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/canonicalized-paths-normal-b/_delta_log/00000000000000000000.json ================================================ {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"6b8e62a0-dd56-4453-b00b-9f9669076189","format":{"provider":"parquet","options":{}},"partitionColumns":[],"configuration":{},"createdTime":1603723991085}} {"add":{"path":"/some/unqualified/absolute/path","partitionValues":{},"size":100,"modificationTime":10,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/canonicalized-paths-normal-b/_delta_log/00000000000000000001.json ================================================ {"remove":{"path":"file:///some/unqualified/absolute/path","deletionTimestamp":200,"dataChange":false}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/canonicalized-paths-special-a/_delta_log/00000000000000000000.json ================================================ {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"b2facba1-1669-43f3-9b1d-7580c207873e","format":{"provider":"parquet","options":{}},"partitionColumns":[],"configuration":{},"createdTime":1603723991525}} {"add":{"path":"/some/unqualified/with%20space/p@%23h","partitionValues":{},"size":100,"modificationTime":10,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/canonicalized-paths-special-a/_delta_log/00000000000000000001.json ================================================ {"remove":{"path":"file:/some/unqualified/with%20space/p@%23h","deletionTimestamp":200,"dataChange":false}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/canonicalized-paths-special-b/_delta_log/00000000000000000000.json ================================================ {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"c23cd784-bc31-46e5-a95b-73bcbe1111a5","format":{"provider":"parquet","options":{}},"partitionColumns":[],"configuration":{},"createdTime":1603723991975}} {"add":{"path":"/some/unqualified/with%20space/p@%23h","partitionValues":{},"size":100,"modificationTime":10,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/canonicalized-paths-special-b/_delta_log/00000000000000000001.json ================================================ {"remove":{"path":"file:///some/unqualified/with%20space/p@%23h","deletionTimestamp":200,"dataChange":false}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/commit-info-containing-arbitrary-operationParams-types/_delta_log/00000000000000000000.crc ================================================ {"txnId":"bda32d72-442d-4705-9a8c-16093eb31744","tableSizeBytes":452,"numFiles":1,"numMetadata":1,"numProtocol":1,"setTransactions":[],"domainMetadata":[],"metadata":{"id":"da00fe29-8b6e-4f3b-b91f-a3729283bc1a","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"month\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["month"],"configuration":{"delta.enableChangeDataFeed":"true"},"createdTime":1740185389028},"protocol":{"minReaderVersion":1,"minWriterVersion":7,"writerFeatures":["changeDataFeed","appendOnly","invariants"]},"allFiles":[{"path":"month=1/part-00000-22d25ea7-a383-44df-ad22-6b06d871b547.c000.snappy.parquet","partitionValues":{"month":"1"},"size":452,"modificationTime":1740185390672,"dataChange":false,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":1},\"nullCount\":{\"id\":0}}"}]} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/commit-info-containing-arbitrary-operationParams-types/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1740185390903,"operation":"CREATE TABLE AS SELECT","operationParameters":{"partitionBy":"[\"month\"]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{\"delta.enableChangeDataFeed\":\"true\"}"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"452"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"bda32d72-442d-4705-9a8c-16093eb31744"}} {"metaData":{"id":"da00fe29-8b6e-4f3b-b91f-a3729283bc1a","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"month\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["month"],"configuration":{"delta.enableChangeDataFeed":"true"},"createdTime":1740185389028}} {"protocol":{"minReaderVersion":1,"minWriterVersion":7,"writerFeatures":["changeDataFeed","appendOnly","invariants"]}} {"add":{"path":"month=1/part-00000-22d25ea7-a383-44df-ad22-6b06d871b547.c000.snappy.parquet","partitionValues":{"month":"1"},"size":452,"modificationTime":1740185390672,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":1},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/commit-info-containing-arbitrary-operationParams-types/_delta_log/00000000000000000001.crc ================================================ {"txnId":"0d7d28b8-55c2-4d8b-b48e-88b22c90aed1","tableSizeBytes":904,"numFiles":2,"numMetadata":1,"numProtocol":1,"setTransactions":[],"domainMetadata":[],"metadata":{"id":"da00fe29-8b6e-4f3b-b91f-a3729283bc1a","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"month\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["month"],"configuration":{"delta.enableChangeDataFeed":"true"},"createdTime":1740185389028},"protocol":{"minReaderVersion":1,"minWriterVersion":7,"writerFeatures":["changeDataFeed","appendOnly","invariants"]},"allFiles":[{"path":"month=2/part-00000-cc2a9650-0450-4879-9757-873b7f544510.c000.snappy.parquet","partitionValues":{"month":"2"},"size":452,"modificationTime":1740185395663,"dataChange":false,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":2},\"maxValues\":{\"id\":2},\"nullCount\":{\"id\":0}}"},{"path":"month=1/part-00000-22d25ea7-a383-44df-ad22-6b06d871b547.c000.snappy.parquet","partitionValues":{"month":"1"},"size":452,"modificationTime":1740185390672,"dataChange":false,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":1},\"nullCount\":{\"id\":0}}"}]} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/commit-info-containing-arbitrary-operationParams-types/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1740185395669,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"452"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"0d7d28b8-55c2-4d8b-b48e-88b22c90aed1"}} {"add":{"path":"month=2/part-00000-cc2a9650-0450-4879-9757-873b7f544510.c000.snappy.parquet","partitionValues":{"month":"2"},"size":452,"modificationTime":1740185395663,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":2},\"maxValues\":{\"id\":2},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/commit-info-containing-arbitrary-operationParams-types/_delta_log/00000000000000000002.crc ================================================ {"txnId":"79b3e3aa-82dc-4c18-b95e-8b50089b55c7","tableSizeBytes":904,"numFiles":2,"numMetadata":1,"numProtocol":1,"setTransactions":[],"domainMetadata":[],"metadata":{"id":"da00fe29-8b6e-4f3b-b91f-a3729283bc1a","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"month\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["month"],"configuration":{"delta.enableChangeDataFeed":"true"},"createdTime":1740185389028},"protocol":{"minReaderVersion":1,"minWriterVersion":7,"writerFeatures":["changeDataFeed","appendOnly","invariants"]},"allFiles":[{"path":"month=2/part-00000-129a0441-5f41-4e46-be33-fd0289e53614.c000.snappy.parquet","partitionValues":{"month":"2"},"size":452,"modificationTime":1740185397380,"dataChange":false,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":2},\"maxValues\":{\"id\":2},\"nullCount\":{\"id\":0}}"},{"path":"month=1/part-00000-c5babbd8-6013-484c-818f-22d546976866.c000.snappy.parquet","partitionValues":{"month":"1"},"size":452,"modificationTime":1740185397384,"dataChange":false,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":1},\"nullCount\":{\"id\":0}}"}]} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/commit-info-containing-arbitrary-operationParams-types/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1740185397394,"operation":"OPTIMIZE","operationParameters":{"predicate":"[]","zOrderBy":"[\"id\"]","clusterBy":"[]","auto":false},"readVersion":1,"isolationLevel":"SnapshotIsolation","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"2","numRemovedBytes":"904","p25FileSize":"452","numDeletionVectorsRemoved":"0","minFileSize":"452","numAddedFiles":"2","maxFileSize":"452","p75FileSize":"452","p50FileSize":"452","numAddedBytes":"904"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"79b3e3aa-82dc-4c18-b95e-8b50089b55c7"}} {"add":{"path":"month=1/part-00000-c5babbd8-6013-484c-818f-22d546976866.c000.snappy.parquet","partitionValues":{"month":"1"},"size":452,"modificationTime":1740185397384,"dataChange":false,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":1},\"nullCount\":{\"id\":0}}"}} {"remove":{"path":"month=1/part-00000-22d25ea7-a383-44df-ad22-6b06d871b547.c000.snappy.parquet","deletionTimestamp":1740185396708,"dataChange":false,"extendedFileMetadata":true,"partitionValues":{"month":"1"},"size":452,"stats":"{\"numRecords\":1}"}} {"add":{"path":"month=2/part-00000-129a0441-5f41-4e46-be33-fd0289e53614.c000.snappy.parquet","partitionValues":{"month":"2"},"size":452,"modificationTime":1740185397380,"dataChange":false,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":2},\"maxValues\":{\"id\":2},\"nullCount\":{\"id\":0}}"}} {"remove":{"path":"month=2/part-00000-cc2a9650-0450-4879-9757-873b7f544510.c000.snappy.parquet","deletionTimestamp":1740185396708,"dataChange":false,"extendedFileMetadata":true,"partitionValues":{"month":"2"},"size":452,"stats":"{\"numRecords\":1}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603723979876,"operation":"Manual Update","operationParameters":{},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"5756e7b1-4b09-4c4e-a3b8-da3c214613d0","format":{"provider":"parquet","options":{}},"partitionColumns":[],"configuration":{},"createdTime":1603723979876}} {"add":{"path":"0","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1603723980484,"operation":"Manual Update","operationParameters":{},"readVersion":0,"isBlindAppend":true}} {"add":{"path":"1","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1603723981300,"operation":"Manual Update","operationParameters":{},"readVersion":1,"isBlindAppend":true}} {"add":{"path":"2","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint/_delta_log/00000000000000000003.json ================================================ {"commitInfo":{"timestamp":1603723982125,"operation":"Manual Update","operationParameters":{},"readVersion":2,"isBlindAppend":true}} {"add":{"path":"3","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint/_delta_log/00000000000000000004.json ================================================ {"commitInfo":{"timestamp":1603723982971,"operation":"Manual Update","operationParameters":{},"readVersion":3,"isBlindAppend":true}} {"add":{"path":"4","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint/_delta_log/00000000000000000005.json ================================================ {"commitInfo":{"timestamp":1603723984006,"operation":"Manual Update","operationParameters":{},"readVersion":4,"isBlindAppend":true}} {"add":{"path":"5","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint/_delta_log/00000000000000000006.json ================================================ {"commitInfo":{"timestamp":1603723985117,"operation":"Manual Update","operationParameters":{},"readVersion":5,"isBlindAppend":true}} {"add":{"path":"6","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint/_delta_log/00000000000000000007.json ================================================ {"commitInfo":{"timestamp":1603723986119,"operation":"Manual Update","operationParameters":{},"readVersion":6,"isBlindAppend":true}} {"add":{"path":"7","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint/_delta_log/00000000000000000008.json ================================================ {"commitInfo":{"timestamp":1603723987024,"operation":"Manual Update","operationParameters":{},"readVersion":7,"isBlindAppend":true}} {"add":{"path":"8","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint/_delta_log/00000000000000000009.json ================================================ {"commitInfo":{"timestamp":1603723987921,"operation":"Manual Update","operationParameters":{},"readVersion":8,"isBlindAppend":true}} {"add":{"path":"9","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint/_delta_log/00000000000000000010.json ================================================ {"commitInfo":{"timestamp":1603723988863,"operation":"Manual Update","operationParameters":{},"readVersion":9,"isBlindAppend":true}} {"add":{"path":"10","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint/_delta_log/_last_checkpoint ================================================ {"version":10,"size":13} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint-kernel/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1697654175172,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1003"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"68e91815-2b2a-4bc9-9d5c-0c88abe5bb49"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1697654170283}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part-00000-bca1b163-25a1-4130-b74c-b905c61018ca-c000.snappy.parquet","partitionValues":{},"size":500,"modificationTime":1697654174000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-c1199313-5eb1-4d9d-9cec-a43245621024-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1697654174000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":5},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint-kernel/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1697654186351,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1003"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"a9dc1765-ff32-4c44-9df0-0716157df530"}} {"add":{"path":"part-00000-cd63e6e7-227f-4bae-8ffc-fad3bfea242c-c000.snappy.parquet","partitionValues":{},"size":500,"modificationTime":1697654186000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-89b7b3e6-d076-43af-963f-3a4055a1eca6-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1697654186000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":5},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint-kernel/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1697654189454,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1003"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"c2955e78-34d2-48d8-a438-9e3d94240fef"}} {"add":{"path":"part-00000-51f8ff2c-8e81-4031-94c9-93eae615d3e3-c000.snappy.parquet","partitionValues":{},"size":500,"modificationTime":1697654189000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-81d22bd7-311e-4934-839e-f635ea6f364f-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1697654189000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":5},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint-kernel/_delta_log/00000000000000000003.json ================================================ {"commitInfo":{"timestamp":1697654191877,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1003"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"ff507eb6-5061-4d9b-9e05-1f4a52d19bac"}} {"add":{"path":"part-00000-59a396e0-b0f4-4685-80f1-f58e07601862-c000.snappy.parquet","partitionValues":{},"size":500,"modificationTime":1697654191000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-a68acb2a-ac4f-46c2-940b-f962480a6517-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1697654191000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":5},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint-kernel/_delta_log/00000000000000000004.json ================================================ {"commitInfo":{"timestamp":1697654194175,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":3,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1003"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"1df7a487-0697-4474-a8ec-0f0911216e68"}} {"add":{"path":"part-00000-99f8ecc2-cc99-4e3e-866e-07135df25e52-c000.snappy.parquet","partitionValues":{},"size":500,"modificationTime":1697654194000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-8e839ba6-38f3-4093-8eb4-bc894159348c-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1697654194000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":5},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint-kernel/_delta_log/00000000000000000005.json ================================================ {"commitInfo":{"timestamp":1697654195998,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":4,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1003"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"49c01705-e0fc-419c-ac2c-6962d449438b"}} {"add":{"path":"part-00000-a57ecbd0-7dad-4b6c-a3fe-8ab4f7e73f5a-c000.snappy.parquet","partitionValues":{},"size":500,"modificationTime":1697654195000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-ef16b167-3dda-4681-bdd0-cd6bb9f07c30-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1697654195000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":5},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint-kernel/_delta_log/00000000000000000006.json ================================================ {"commitInfo":{"timestamp":1697654197786,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":5,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1003"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"43fdd9d9-cfef-4cd7-982c-e0bb41114f62"}} {"add":{"path":"part-00000-d9d02879-5155-46d4-84a8-41c83c5df9e4-c000.snappy.parquet","partitionValues":{},"size":500,"modificationTime":1697654197000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-84978e4c-0e36-40d7-a3e0-c69204409c28-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1697654197000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":5},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint-kernel/_delta_log/00000000000000000007.json ================================================ {"commitInfo":{"timestamp":1697654200341,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":6,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1003"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"a845999b-ec65-424a-a62a-ace1b2d7336c"}} {"add":{"path":"part-00000-45ddfb64-1797-4618-a4e4-58d687ae9d21-c000.snappy.parquet","partitionValues":{},"size":500,"modificationTime":1697654200000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-e471a872-a1ee-4610-9454-062854327ad6-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1697654200000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":5},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint-kernel/_delta_log/00000000000000000008.json ================================================ {"commitInfo":{"timestamp":1697654202124,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":7,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1003"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"fe722b89-1814-46ad-84dd-3d0b3148dd90"}} {"add":{"path":"part-00000-69f4e384-139f-4b75-b51f-09213866a62a-c000.snappy.parquet","partitionValues":{},"size":500,"modificationTime":1697654202000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-400931d7-721c-4dbc-82e6-5c29f1dfcde1-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1697654202000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":5},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint-kernel/_delta_log/00000000000000000009.json ================================================ {"commitInfo":{"timestamp":1697654203783,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":8,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1003"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"40c60253-5a25-4e4c-a4a1-47795ea78217"}} {"add":{"path":"part-00000-82c1686f-287a-4e6f-8a7a-0099d54d7738-c000.snappy.parquet","partitionValues":{},"size":500,"modificationTime":1697654203000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-71b04841-d4e6-4cd6-930a-5e33fd1bd7a0-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1697654203000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":5},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint-kernel/_delta_log/00000000000000000010.json ================================================ {"commitInfo":{"timestamp":1697654205382,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":9,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1003"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"b6376238-762c-4644-90ad-6fee36425722"}} {"add":{"path":"part-00000-cbc535a8-3499-4339-be3f-9df89091871e-c000.snappy.parquet","partitionValues":{},"size":500,"modificationTime":1697654205000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-c4cbb8cf-9c18-4bab-bfa7-967faa14e15d-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1697654205000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":5},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint-kernel/_delta_log/00000000000000000011.json ================================================ {"commitInfo":{"timestamp":1697654211950,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":10,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"2","numOutputRows":"100","numOutputBytes":"1454"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"6f33f474-6422-4edf-83fb-d97f276fd8d2"}} {"add":{"path":"part-00000-45318b19-5a29-4bb9-b273-1738e817d63e-c000.snappy.parquet","partitionValues":{},"size":765,"modificationTime":1697654210000,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":49},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-4eeaf77f-87b7-45bb-8e1f-1faf9c957918-c000.snappy.parquet","partitionValues":{},"size":689,"modificationTime":1697654210000,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":50},\"maxValues\":{\"id\":99},\"nullCount\":{\"id\":0}}"}} {"remove":{"path":"part-00000-45ddfb64-1797-4618-a4e4-58d687ae9d21-c000.snappy.parquet","deletionTimestamp":1697654211945,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":500}} {"remove":{"path":"part-00000-69f4e384-139f-4b75-b51f-09213866a62a-c000.snappy.parquet","deletionTimestamp":1697654211946,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":500}} {"remove":{"path":"part-00000-82c1686f-287a-4e6f-8a7a-0099d54d7738-c000.snappy.parquet","deletionTimestamp":1697654211946,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":500}} {"remove":{"path":"part-00000-99f8ecc2-cc99-4e3e-866e-07135df25e52-c000.snappy.parquet","deletionTimestamp":1697654211946,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":500}} {"remove":{"path":"part-00000-a57ecbd0-7dad-4b6c-a3fe-8ab4f7e73f5a-c000.snappy.parquet","deletionTimestamp":1697654211946,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":500}} {"remove":{"path":"part-00000-bca1b163-25a1-4130-b74c-b905c61018ca-c000.snappy.parquet","deletionTimestamp":1697654211946,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":500}} {"remove":{"path":"part-00000-cbc535a8-3499-4339-be3f-9df89091871e-c000.snappy.parquet","deletionTimestamp":1697654211946,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":500}} {"remove":{"path":"part-00000-cd63e6e7-227f-4bae-8ffc-fad3bfea242c-c000.snappy.parquet","deletionTimestamp":1697654211946,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":500}} {"remove":{"path":"part-00001-400931d7-721c-4dbc-82e6-5c29f1dfcde1-c000.snappy.parquet","deletionTimestamp":1697654211946,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":503}} {"remove":{"path":"part-00001-71b04841-d4e6-4cd6-930a-5e33fd1bd7a0-c000.snappy.parquet","deletionTimestamp":1697654211946,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":503}} {"remove":{"path":"part-00001-a68acb2a-ac4f-46c2-940b-f962480a6517-c000.snappy.parquet","deletionTimestamp":1697654211946,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":503}} {"remove":{"path":"part-00001-c1199313-5eb1-4d9d-9cec-a43245621024-c000.snappy.parquet","deletionTimestamp":1697654211946,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":503}} {"remove":{"path":"part-00001-c4cbb8cf-9c18-4bab-bfa7-967faa14e15d-c000.snappy.parquet","deletionTimestamp":1697654211946,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":503}} {"remove":{"path":"part-00001-e471a872-a1ee-4610-9454-062854327ad6-c000.snappy.parquet","deletionTimestamp":1697654211946,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":503}} {"remove":{"path":"part-00001-ef16b167-3dda-4681-bdd0-cd6bb9f07c30-c000.snappy.parquet","deletionTimestamp":1697654211946,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":503}} {"remove":{"path":"part-00000-51f8ff2c-8e81-4031-94c9-93eae615d3e3-c000.snappy.parquet","deletionTimestamp":1697654211946,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":500}} {"remove":{"path":"part-00000-59a396e0-b0f4-4685-80f1-f58e07601862-c000.snappy.parquet","deletionTimestamp":1697654211946,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":500}} {"remove":{"path":"part-00000-d9d02879-5155-46d4-84a8-41c83c5df9e4-c000.snappy.parquet","deletionTimestamp":1697654211946,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":500}} {"remove":{"path":"part-00001-81d22bd7-311e-4934-839e-f635ea6f364f-c000.snappy.parquet","deletionTimestamp":1697654211946,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":503}} {"remove":{"path":"part-00001-84978e4c-0e36-40d7-a3e0-c69204409c28-c000.snappy.parquet","deletionTimestamp":1697654211946,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":503}} {"remove":{"path":"part-00001-89b7b3e6-d076-43af-963f-3a4055a1eca6-c000.snappy.parquet","deletionTimestamp":1697654211946,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":503}} {"remove":{"path":"part-00001-8e839ba6-38f3-4093-8eb4-bc894159348c-c000.snappy.parquet","deletionTimestamp":1697654211946,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":503}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/corrupted-last-checkpoint-kernel/_delta_log/_last_checkpoint ================================================ ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-absolute-paths-escaped-chars/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603387084639,"operation":"Manual Update","operationParameters":{},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"..//Users/scott.sandre/connectors/golden-tables/src/test/resources/golden/data-reader-absolute-paths-escaped-chars/foo.snappy.parquet","partitionValues":{},"size":1,"modificationTime":1603387084631,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-absolute-paths-escaped-chars/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1603387085189,"operation":"Manual Update","operationParameters":{},"readVersion":0,"isBlindAppend":true}} {"add":{"path":"bar%2Dbar.snappy.parquet","partitionValues":{},"size":1,"modificationTime":1603387085181,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-array-complex-objects/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603724039052,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"38be1738-32ad-448f-9e29-912a7536d4ca","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"i\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"3d_int_list\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"containsNull\":true},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"4d_int_list\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"containsNull\":true},\"containsNull\":true},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"list_of_maps\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"long\",\"valueContainsNull\":true},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"list_of_records\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"val\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724038935}} {"add":{"path":"part-00000-a7d58b1a-7743-4bb0-b208-438bbe179c93-c000.snappy.parquet","partitionValues":{},"size":2830,"modificationTime":1603724039000,"dataChange":true}} {"add":{"path":"part-00001-7b211746-0a31-4e77-9822-b0985158cd66-c000.snappy.parquet","partitionValues":{},"size":2832,"modificationTime":1603724039000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-array-primitives/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603724038064,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"caaa1362-3717-449b-ab9b-f7d8d536018d","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"as_array_int\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"as_array_long\",\"type\":{\"type\":\"array\",\"elementType\":\"long\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"as_array_byte\",\"type\":{\"type\":\"array\",\"elementType\":\"byte\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"as_array_short\",\"type\":{\"type\":\"array\",\"elementType\":\"short\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"as_array_boolean\",\"type\":{\"type\":\"array\",\"elementType\":\"boolean\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"as_array_float\",\"type\":{\"type\":\"array\",\"elementType\":\"float\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"as_array_double\",\"type\":{\"type\":\"array\",\"elementType\":\"double\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"as_array_string\",\"type\":{\"type\":\"array\",\"elementType\":\"string\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"as_array_binary\",\"type\":{\"type\":\"array\",\"elementType\":\"binary\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"as_array_big_decimal\",\"type\":{\"type\":\"array\",\"elementType\":\"decimal(1,0)\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724037970}} {"add":{"path":"part-00000-182665f0-30df-470d-a5cb-8d9d483ed390-c000.snappy.parquet","partitionValues":{},"size":3627,"modificationTime":1603724038000,"dataChange":true}} {"add":{"path":"part-00001-2e274fe7-eb75-4b73-8c72-423ee747abc0-c000.snappy.parquet","partitionValues":{},"size":3644,"modificationTime":1603724038000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-date-types-America/Los_Angeles/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603724034349,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"475dbe77-c782-43a9-830b-d1777f3a7244","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"timestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724034283}} {"add":{"path":"part-00000-e85ca549-604b-4340-b56d-868e9acc78e8-c000.snappy.parquet","partitionValues":{},"size":358,"modificationTime":1603724034000,"dataChange":true}} {"add":{"path":"part-00001-1e808610-ee7f-44e7-be9b-be02c2bc5895-c000.snappy.parquet","partitionValues":{},"size":717,"modificationTime":1603724034000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-date-types-Asia/Beirut/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603724036152,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"7575fa96-acd9-4e2b-9f29-ce44fac98c60","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"timestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724036096}} {"add":{"path":"part-00000-58828e3c-041e-47b4-80dd-196ae1b1d1a6-c000.snappy.parquet","partitionValues":{},"size":358,"modificationTime":1603724036000,"dataChange":true}} {"add":{"path":"part-00001-8590d66f-6907-40a9-9e97-a4a098321340-c000.snappy.parquet","partitionValues":{},"size":717,"modificationTime":1603724036000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-date-types-Etc/GMT+9/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603724035263,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"8684eda0-16ef-4527-a298-798fef1e87f4","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"timestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724035206}} {"add":{"path":"part-00000-23e032bb-e586-4573-9fc0-1c9a4c9a5081-c000.snappy.parquet","partitionValues":{},"size":358,"modificationTime":1603724035000,"dataChange":true}} {"add":{"path":"part-00001-d91bf3dd-78c9-4abf-aa54-e89228e8316c-c000.snappy.parquet","partitionValues":{},"size":717,"modificationTime":1603724035000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-date-types-Iceland/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603724032094,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"fac6661d-d03f-4dca-954d-f3546571c198","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"timestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724032022}} {"add":{"path":"part-00000-8be8ec9f-d9af-474e-8ec9-35ec76debc6a-c000.snappy.parquet","partitionValues":{},"size":358,"modificationTime":1603724032000,"dataChange":true}} {"add":{"path":"part-00001-56f07a95-04d4-4c12-bf08-fd89cedc8559-c000.snappy.parquet","partitionValues":{},"size":717,"modificationTime":1603724032000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-date-types-JST/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603724037072,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"b6b84722-3b6d-4f69-8870-48baebf70fe7","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"timestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724037013}} {"add":{"path":"part-00000-3f9100ce-0b94-43cb-bb23-f0e36dc7af2b-c000.snappy.parquet","partitionValues":{},"size":358,"modificationTime":1603724037000,"dataChange":true}} {"add":{"path":"part-00001-dc211b29-0c30-41e8-8700-f8bb374964e1-c000.snappy.parquet","partitionValues":{},"size":717,"modificationTime":1603724037000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-date-types-PST/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603724033415,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"6b978932-93d9-431e-a7a2-b572472b09c6","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"timestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724033355}} {"add":{"path":"part-00000-0a103e9a-6236-470c-94f7-5f60926f01da-c000.snappy.parquet","partitionValues":{},"size":358,"modificationTime":1603724033000,"dataChange":true}} {"add":{"path":"part-00001-980a117f-027e-4396-81ce-3a5a8ac70815-c000.snappy.parquet","partitionValues":{},"size":717,"modificationTime":1603724033000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-date-types-UTC/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603724030655,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"d33c8691-c845-46c4-bb93-1ae64db706b5","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"timestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724030532}} {"add":{"path":"part-00000-803e1cfa-c859-4ce7-977b-ff150d6e138c-c000.snappy.parquet","partitionValues":{},"size":358,"modificationTime":1603724030000,"dataChange":true}} {"add":{"path":"part-00001-0108113a-2933-41b3-b9a6-e68bb9ed25cc-c000.snappy.parquet","partitionValues":{},"size":717,"modificationTime":1603724030000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-escaped-chars/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603724042582,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_2\"]"},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"ccdc1b2a-f27e-47a6-aadb-dab6b88ac899","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["_2"],"configuration":{},"createdTime":1603724042500}} {"add":{"path":"_2=bar+%252521/part-00000-af08f887-922f-4c31-82a7-8e142c4280a6.c000.snappy.parquet","partitionValues":{"_2":"bar+%21"},"size":398,"modificationTime":1603724042000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-escaped-chars/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1603724043128,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_2\"]"},"readVersion":0,"isBlindAppend":true}} {"add":{"path":"_2=bar+%252522/part-00000-c1bfd944-5e0d-4133-af16-7851061e37aa.c000.snappy.parquet","partitionValues":{"_2":"bar+%22"},"size":398,"modificationTime":1603724043000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-escaped-chars/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1603724043721,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_2\"]"},"readVersion":1,"isBlindAppend":true}} {"add":{"path":"_2=bar+%252523/part-00000-92352854-5503-4ba5-8c29-b11777034eb7.c000.snappy.parquet","partitionValues":{"_2":"bar+%23"},"size":398,"modificationTime":1603724043000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-map/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603724039953,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"e52f2c3e-fac0-4b28-9627-2e33e6b85dc0","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"i\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"a\",\"type\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":\"integer\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"b\",\"type\":{\"type\":\"map\",\"keyType\":\"long\",\"valueType\":\"byte\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"c\",\"type\":{\"type\":\"map\",\"keyType\":\"short\",\"valueType\":\"boolean\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"d\",\"type\":{\"type\":\"map\",\"keyType\":\"float\",\"valueType\":\"double\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"e\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"decimal(1,0)\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"f\",\"type\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"val\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724039866}} {"add":{"path":"part-00000-d9004e55-077b-4728-9ee6-b3401faa46ba-c000.snappy.parquet","partitionValues":{},"size":3638,"modificationTime":1603724039000,"dataChange":true}} {"add":{"path":"part-00001-3d30d085-4cde-471e-a396-12af34a70812-c000.snappy.parquet","partitionValues":{},"size":3655,"modificationTime":1603724039000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-nested-struct/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603724040818,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"975ef365-8dec-4bbf-ab88-264c10987001","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"a\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aa\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ab\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ac\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aca\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"acb\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"b\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724040747}} {"add":{"path":"part-00000-f2547b28-9219-4628-8462-cc9c56edfebb-c000.snappy.parquet","partitionValues":{},"size":1432,"modificationTime":1603724040000,"dataChange":true}} {"add":{"path":"part-00001-0f755735-3b5b-449a-8f93-92a40d9f065d-c000.snappy.parquet","partitionValues":{},"size":1439,"modificationTime":1603724040000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-nullable-field-invalid-schema-key/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603724041694,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"ab05c2c1-6f1c-421b-815b-0f04dbf34814","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"array_can_contain_null\",\"type\":{\"type\":\"array\",\"elementType\":\"string\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724041628}} {"add":{"path":"part-00000-d1f74401-ecb8-494e-96d6-adb95ec7e1c2-c000.snappy.parquet","partitionValues":{},"size":385,"modificationTime":1603724041000,"dataChange":true}} {"add":{"path":"part-00001-d6454547-1a50-4f43-910d-2f84c5aedae1-c000.snappy.parquet","partitionValues":{},"size":500,"modificationTime":1603724041000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-partition-values/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1636147668568,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"as_int\",\"as_long\",\"as_byte\",\"as_short\",\"as_boolean\",\"as_float\",\"as_double\",\"as_string\",\"as_string_lit_null\",\"as_date\",\"as_timestamp\",\"as_big_decimal\"]"},"isBlindAppend":true,"operationMetrics":{"numFiles":"3","numOutputBytes":"5832","numOutputRows":"3"}}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"as_int\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_long\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_byte\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_short\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_boolean\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_float\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_string\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_string_lit_null\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_timestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_big_decimal\",\"type\":\"decimal(1,0)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_list_of_records\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"val\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"as_nested_struct\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aa\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ab\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ac\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aca\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"acb\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"value\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["as_int","as_long","as_byte","as_short","as_boolean","as_float","as_double","as_string","as_string_lit_null","as_date","as_timestamp","as_big_decimal"],"configuration":{},"createdTime":1636147666386}} {"add":{"path":"as_int=0/as_long=0/as_byte=0/as_short=0/as_boolean=true/as_float=0.0/as_double=0.0/as_string=0/as_string_lit_null=null/as_date=2021-09-08/as_timestamp=2021-09-08%2011%253A11%253A11/as_big_decimal=0/part-00000-b9dc86ae-0134-4363-bd87-19cfb3403e9a.c000.snappy.parquet","partitionValues":{"as_big_decimal":"0","as_int":"0","as_byte":"0","as_long":"0","as_date":"2021-09-08","as_string":"0","as_timestamp":"2021-09-08 11:11:11","as_float":"0.0","as_short":"0","as_boolean":"true","as_string_lit_null":"null","as_double":"0.0"},"size":1944,"modificationTime":1636147668000,"dataChange":true}} {"add":{"path":"as_int=__HIVE_DEFAULT_PARTITION__/as_long=__HIVE_DEFAULT_PARTITION__/as_byte=__HIVE_DEFAULT_PARTITION__/as_short=__HIVE_DEFAULT_PARTITION__/as_boolean=__HIVE_DEFAULT_PARTITION__/as_float=__HIVE_DEFAULT_PARTITION__/as_double=__HIVE_DEFAULT_PARTITION__/as_string=__HIVE_DEFAULT_PARTITION__/as_string_lit_null=__HIVE_DEFAULT_PARTITION__/as_date=__HIVE_DEFAULT_PARTITION__/as_timestamp=__HIVE_DEFAULT_PARTITION__/as_big_decimal=__HIVE_DEFAULT_PARTITION__/part-00001-9ee474eb-385b-43cf-9acb-0fbed63e011c.c000.snappy.parquet","partitionValues":{"as_big_decimal":null,"as_int":null,"as_byte":null,"as_long":null,"as_date":null,"as_string":null,"as_timestamp":null,"as_float":null,"as_short":null,"as_boolean":null,"as_string_lit_null":null,"as_double":null},"size":1944,"modificationTime":1636147668000,"dataChange":true}} {"add":{"path":"as_int=1/as_long=1/as_byte=1/as_short=1/as_boolean=false/as_float=1.0/as_double=1.0/as_string=1/as_string_lit_null=null/as_date=2021-09-08/as_timestamp=2021-09-08%2011%253A11%253A11/as_big_decimal=1/part-00001-cb007d48-a9f5-40e7-adbe-60920680770f.c000.snappy.parquet","partitionValues":{"as_big_decimal":"1","as_int":"1","as_byte":"1","as_long":"1","as_date":"2021-09-08","as_string":"1","as_timestamp":"2021-09-08 11:11:11","as_float":"1.0","as_short":"1","as_boolean":"false","as_string_lit_null":"null","as_double":"1.0"},"size":1944,"modificationTime":1636147668000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-primitives/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1607520163636,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputBytes":"5050","numOutputRows":"11"}}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"as_int\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_long\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_byte\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_short\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_boolean\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_float\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_string\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_binary\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_big_decimal\",\"type\":\"decimal(1,0)\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1607520161353}} {"add":{"path":"part-00000-4f2f0b9f-50b3-4e7b-96a1-e2bb0f246b06-c000.snappy.parquet","partitionValues":{},"size":2482,"modificationTime":1607520163000,"dataChange":true}} {"add":{"path":"part-00001-09e47b80-36c2-4475-a810-fbd8e7994971-c000.snappy.parquet","partitionValues":{},"size":2568,"modificationTime":1607520163000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-timestamp_ntz/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1712333988110,"operation":"CREATE TABLE","operationParameters":{"partitionBy":"[\"tsNtzPartition\"]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{}"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"fecbfd56-6849-421b-8439-070f0d694787"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"tsNtz\",\"type\":\"timestamp_ntz\",\"nullable\":true,\"metadata\":{}},{\"name\":\"tsNtzPartition\",\"type\":\"timestamp_ntz\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["tsNtzPartition"],"configuration":{},"createdTime":1712333987987}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["timestampNtz"],"writerFeatures":["timestampNtz"]}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-timestamp_ntz/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1712333992682,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"4","numOutputRows":"9","numOutputBytes":"2940"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"39f277cb-1414-419a-b634-f6a983ed9b37"}} {"add":{"path":"tsNtzPartition=2013-07-05%2017%253A01%253A00.123456/part-00000-6240e68e-2304-449a-a1e6-0e24866d3508.c000.snappy.parquet","partitionValues":{"tsNtzPartition":"2013-07-05 17:01:00.123456"},"size":726,"modificationTime":1712333992612,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":3,\"tsNtz\":\"2021-11-18T02:30:00.123\"},\"maxValues\":{\"id\":3,\"tsNtz\":\"2021-11-18T02:30:00.123\"},\"nullCount\":{\"id\":0,\"tsNtz\":0}}"}} {"add":{"path":"tsNtzPartition=2021-11-18%2002%253A30%253A00.123456/part-00000-65fcd5cb-f2f3-44f4-96ef-f43825143ba9.c000.snappy.parquet","partitionValues":{"tsNtzPartition":"2021-11-18 02:30:00.123456"},"size":742,"modificationTime":1712333992666,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"id\":0,\"tsNtz\":\"2013-07-05T17:01:00.123\"},\"maxValues\":{\"id\":2,\"tsNtz\":\"2021-11-18T02:30:00.123\"},\"nullCount\":{\"id\":0,\"tsNtz\":1}}"}} {"add":{"path":"tsNtzPartition=__HIVE_DEFAULT_PARTITION__/part-00001-53fd3b3b-7773-459a-921c-bb64bf0bbd03.c000.snappy.parquet","partitionValues":{"tsNtzPartition":null},"size":742,"modificationTime":1712333992612,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"id\":6,\"tsNtz\":\"2013-07-05T17:01:00.123\"},\"maxValues\":{\"id\":8,\"tsNtz\":\"2021-11-18T02:30:00.123\"},\"nullCount\":{\"id\":0,\"tsNtz\":1}}"}} {"add":{"path":"tsNtzPartition=2013-07-05%2017%253A01%253A00.123456/part-00001-336e3e5f-a202-4bd9-b117-28d871bbb639.c000.snappy.parquet","partitionValues":{"tsNtzPartition":"2013-07-05 17:01:00.123456"},"size":730,"modificationTime":1712333992659,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"id\":4,\"tsNtz\":\"2013-07-05T17:01:00.123\"},\"maxValues\":{\"id\":5,\"tsNtz\":\"2013-07-05T17:01:00.123\"},\"nullCount\":{\"id\":0,\"tsNtz\":1}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-timestamp_ntz-id-mode/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1712333993464,"operation":"CREATE TABLE","operationParameters":{"partitionBy":"[\"tsNtzPartition\"]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{\"delta.columnMapping.mode\":\"id\",\"delta.columnMapping.maxColumnId\":\"3\"}"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"1b8ef756-4197-4263-b9a2-4da05afa76af"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":1,\"delta.columnMapping.physicalName\":\"col-10623d2b-e242-4e70-bda5-826469ad0d2a\"}},{\"name\":\"tsNtz\",\"type\":\"timestamp_ntz\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":2,\"delta.columnMapping.physicalName\":\"col-3095b00d-efaa-493e-b85d-8db894dffffc\"}},{\"name\":\"tsNtzPartition\",\"type\":\"timestamp_ntz\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":3,\"delta.columnMapping.physicalName\":\"col-31f31113-4fc7-437c-b8e8-b7bca8a2f698\"}}]}","partitionColumns":["tsNtzPartition"],"configuration":{"delta.columnMapping.mode":"id","delta.columnMapping.maxColumnId":"3"},"createdTime":1712333993412}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["timestampNtz","columnMapping"],"writerFeatures":["timestampNtz","columnMapping"]}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-timestamp_ntz-id-mode/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1712333994313,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"4","numOutputRows":"9","numOutputBytes":"4832"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"a4cb2faa-bc24-4374-a1c6-99764812a400"}} {"add":{"path":"col-31f31113-4fc7-437c-b8e8-b7bca8a2f698=2013-07-05%2017%253A01%253A00.123456/part-00000-468b79b5-ef3e-40ee-b077-8d7b48ef8385.c000.snappy.parquet","partitionValues":{"col-31f31113-4fc7-437c-b8e8-b7bca8a2f698":"2013-07-05 17:01:00.123456"},"size":1199,"modificationTime":1712333994274,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col-10623d2b-e242-4e70-bda5-826469ad0d2a\":3,\"col-3095b00d-efaa-493e-b85d-8db894dffffc\":\"2021-11-18T02:30:00.123\"},\"maxValues\":{\"col-10623d2b-e242-4e70-bda5-826469ad0d2a\":3,\"col-3095b00d-efaa-493e-b85d-8db894dffffc\":\"2021-11-18T02:30:00.123\"},\"nullCount\":{\"col-10623d2b-e242-4e70-bda5-826469ad0d2a\":0,\"col-3095b00d-efaa-493e-b85d-8db894dffffc\":0}}"}} {"add":{"path":"col-31f31113-4fc7-437c-b8e8-b7bca8a2f698=2021-11-18%2002%253A30%253A00.123456/part-00000-80e4d2e9-69f2-420e-8152-8d5bb810b259.c000.snappy.parquet","partitionValues":{"col-31f31113-4fc7-437c-b8e8-b7bca8a2f698":"2021-11-18 02:30:00.123456"},"size":1215,"modificationTime":1712333994310,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-10623d2b-e242-4e70-bda5-826469ad0d2a\":0,\"col-3095b00d-efaa-493e-b85d-8db894dffffc\":\"2013-07-05T17:01:00.123\"},\"maxValues\":{\"col-10623d2b-e242-4e70-bda5-826469ad0d2a\":2,\"col-3095b00d-efaa-493e-b85d-8db894dffffc\":\"2021-11-18T02:30:00.123\"},\"nullCount\":{\"col-10623d2b-e242-4e70-bda5-826469ad0d2a\":0,\"col-3095b00d-efaa-493e-b85d-8db894dffffc\":1}}"}} {"add":{"path":"col-31f31113-4fc7-437c-b8e8-b7bca8a2f698=__HIVE_DEFAULT_PARTITION__/part-00001-047834e2-8a38-47ff-9f1c-01f94a618369.c000.snappy.parquet","partitionValues":{"col-31f31113-4fc7-437c-b8e8-b7bca8a2f698":null},"size":1215,"modificationTime":1712333994276,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-10623d2b-e242-4e70-bda5-826469ad0d2a\":6,\"col-3095b00d-efaa-493e-b85d-8db894dffffc\":\"2013-07-05T17:01:00.123\"},\"maxValues\":{\"col-10623d2b-e242-4e70-bda5-826469ad0d2a\":8,\"col-3095b00d-efaa-493e-b85d-8db894dffffc\":\"2021-11-18T02:30:00.123\"},\"nullCount\":{\"col-10623d2b-e242-4e70-bda5-826469ad0d2a\":0,\"col-3095b00d-efaa-493e-b85d-8db894dffffc\":1}}"}} {"add":{"path":"col-31f31113-4fc7-437c-b8e8-b7bca8a2f698=2013-07-05%2017%253A01%253A00.123456/part-00001-94a2fe48-a4c5-4d3e-823c-d76b59b9f597.c000.snappy.parquet","partitionValues":{"col-31f31113-4fc7-437c-b8e8-b7bca8a2f698":"2013-07-05 17:01:00.123456"},"size":1203,"modificationTime":1712333994303,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col-10623d2b-e242-4e70-bda5-826469ad0d2a\":4,\"col-3095b00d-efaa-493e-b85d-8db894dffffc\":\"2013-07-05T17:01:00.123\"},\"maxValues\":{\"col-10623d2b-e242-4e70-bda5-826469ad0d2a\":5,\"col-3095b00d-efaa-493e-b85d-8db894dffffc\":\"2013-07-05T17:01:00.123\"},\"nullCount\":{\"col-10623d2b-e242-4e70-bda5-826469ad0d2a\":0,\"col-3095b00d-efaa-493e-b85d-8db894dffffc\":1}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-timestamp_ntz-name-mode/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1712333994816,"operation":"CREATE TABLE","operationParameters":{"partitionBy":"[\"tsNtzPartition\"]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{\"delta.columnMapping.mode\":\"name\",\"delta.columnMapping.maxColumnId\":\"3\"}"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"a70ffe6d-47af-4356-9571-3b4a42168511"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":1,\"delta.columnMapping.physicalName\":\"col-70450211-7268-473d-95c1-6d05710dfafa\"}},{\"name\":\"tsNtz\",\"type\":\"timestamp_ntz\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":2,\"delta.columnMapping.physicalName\":\"col-c14bb714-7e9e-4229-8f81-5d395c21ce7f\"}},{\"name\":\"tsNtzPartition\",\"type\":\"timestamp_ntz\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":3,\"delta.columnMapping.physicalName\":\"col-805808af-d12a-42e5-a7ec-f1a99abb82ee\"}}]}","partitionColumns":["tsNtzPartition"],"configuration":{"delta.columnMapping.mode":"name","delta.columnMapping.maxColumnId":"3"},"createdTime":1712333994784}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["timestampNtz","columnMapping"],"writerFeatures":["timestampNtz","columnMapping"]}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-reader-timestamp_ntz-name-mode/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1712333995537,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"4","numOutputRows":"9","numOutputBytes":"4832"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"e8276040-535a-4261-826f-8d71cae6773d"}} {"add":{"path":"col-805808af-d12a-42e5-a7ec-f1a99abb82ee=2013-07-05%2017%253A01%253A00.123456/part-00000-19009b69-d0d2-4c9c-9994-770c77ce5c1e.c000.snappy.parquet","partitionValues":{"col-805808af-d12a-42e5-a7ec-f1a99abb82ee":"2013-07-05 17:01:00.123456"},"size":1199,"modificationTime":1712333995499,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col-70450211-7268-473d-95c1-6d05710dfafa\":3,\"col-c14bb714-7e9e-4229-8f81-5d395c21ce7f\":\"2021-11-18T02:30:00.123\"},\"maxValues\":{\"col-70450211-7268-473d-95c1-6d05710dfafa\":3,\"col-c14bb714-7e9e-4229-8f81-5d395c21ce7f\":\"2021-11-18T02:30:00.123\"},\"nullCount\":{\"col-70450211-7268-473d-95c1-6d05710dfafa\":0,\"col-c14bb714-7e9e-4229-8f81-5d395c21ce7f\":0}}"}} {"add":{"path":"col-805808af-d12a-42e5-a7ec-f1a99abb82ee=2021-11-18%2002%253A30%253A00.123456/part-00000-55eb3e92-fedb-4a0e-a327-d44ee8e356b2.c000.snappy.parquet","partitionValues":{"col-805808af-d12a-42e5-a7ec-f1a99abb82ee":"2021-11-18 02:30:00.123456"},"size":1215,"modificationTime":1712333995535,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-70450211-7268-473d-95c1-6d05710dfafa\":0,\"col-c14bb714-7e9e-4229-8f81-5d395c21ce7f\":\"2013-07-05T17:01:00.123\"},\"maxValues\":{\"col-70450211-7268-473d-95c1-6d05710dfafa\":2,\"col-c14bb714-7e9e-4229-8f81-5d395c21ce7f\":\"2021-11-18T02:30:00.123\"},\"nullCount\":{\"col-70450211-7268-473d-95c1-6d05710dfafa\":0,\"col-c14bb714-7e9e-4229-8f81-5d395c21ce7f\":1}}"}} {"add":{"path":"col-805808af-d12a-42e5-a7ec-f1a99abb82ee=__HIVE_DEFAULT_PARTITION__/part-00001-4325cf1b-146e-4e85-b36f-ab9c4a9d8125.c000.snappy.parquet","partitionValues":{"col-805808af-d12a-42e5-a7ec-f1a99abb82ee":null},"size":1215,"modificationTime":1712333995502,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-70450211-7268-473d-95c1-6d05710dfafa\":6,\"col-c14bb714-7e9e-4229-8f81-5d395c21ce7f\":\"2013-07-05T17:01:00.123\"},\"maxValues\":{\"col-70450211-7268-473d-95c1-6d05710dfafa\":8,\"col-c14bb714-7e9e-4229-8f81-5d395c21ce7f\":\"2021-11-18T02:30:00.123\"},\"nullCount\":{\"col-70450211-7268-473d-95c1-6d05710dfafa\":0,\"col-c14bb714-7e9e-4229-8f81-5d395c21ce7f\":1}}"}} {"add":{"path":"col-805808af-d12a-42e5-a7ec-f1a99abb82ee=2013-07-05%2017%253A01%253A00.123456/part-00001-459a6750-6f78-44ff-9706-03448c1dde8b.c000.snappy.parquet","partitionValues":{"col-805808af-d12a-42e5-a7ec-f1a99abb82ee":"2013-07-05 17:01:00.123456"},"size":1203,"modificationTime":1712333995527,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col-70450211-7268-473d-95c1-6d05710dfafa\":4,\"col-c14bb714-7e9e-4229-8f81-5d395c21ce7f\":\"2013-07-05T17:01:00.123\"},\"maxValues\":{\"col-70450211-7268-473d-95c1-6d05710dfafa\":5,\"col-c14bb714-7e9e-4229-8f81-5d395c21ce7f\":\"2013-07-05T17:01:00.123\"},\"nullCount\":{\"col-70450211-7268-473d-95c1-6d05710dfafa\":0,\"col-c14bb714-7e9e-4229-8f81-5d395c21ce7f\":1}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-skipping-basic-stats-all-types/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1704847172421,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"1","numOutputBytes":"3865"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0-SNAPSHOT","txnId":"a56746fa-7289-4e45-8f2d-83c31b34d932"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"as_int\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_long\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_byte\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_short\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_float\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_string\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_timestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_big_decimal\",\"type\":\"decimal(1,0)\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1704847166710}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part-00001-93fc8b78-4b92-45c7-ad3f-bb766e6d2e28-c000.snappy.parquet","partitionValues":{},"size":2738,"modificationTime":1704847171000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"as_int\":0,\"as_long\":0,\"as_byte\":0,\"as_short\":0,\"as_float\":0.0,\"as_double\":0.0,\"as_string\":\"0\",\"as_date\":\"2000-01-01\",\"as_timestamp\":\"2000-01-01T00:00:00.000-08:00\",\"as_big_decimal\":0},\"maxValues\":{\"as_int\":0,\"as_long\":0,\"as_byte\":0,\"as_short\":0,\"as_float\":0.0,\"as_double\":0.0,\"as_string\":\"0\",\"as_date\":\"2000-01-01\",\"as_timestamp\":\"2000-01-01T00:00:00.000-08:00\",\"as_big_decimal\":0},\"nullCount\":{\"as_int\":0,\"as_long\":0,\"as_byte\":0,\"as_short\":0,\"as_float\":0,\"as_double\":0,\"as_string\":0,\"as_date\":0,\"as_timestamp\":0,\"as_big_decimal\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-skipping-basic-stats-all-types-checkpoint/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1704847197221,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"1","numOutputBytes":"3865"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0-SNAPSHOT","txnId":"101278f9-eaad-4623-9341-e7a0441e6f56"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"as_int\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_long\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_byte\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_short\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_float\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_string\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_timestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"as_big_decimal\",\"type\":\"decimal(1,0)\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"1"},"createdTime":1704847196699}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part-00001-ed0f17f3-dab5-4131-8ff8-5a5f4399d0ef-c000.snappy.parquet","partitionValues":{},"size":2738,"modificationTime":1704847197000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"as_int\":0,\"as_long\":0,\"as_byte\":0,\"as_short\":0,\"as_float\":0.0,\"as_double\":0.0,\"as_string\":\"0\",\"as_date\":\"2000-01-01\",\"as_timestamp\":\"2000-01-01T00:00:00.000-08:00\",\"as_big_decimal\":0},\"maxValues\":{\"as_int\":0,\"as_long\":0,\"as_byte\":0,\"as_short\":0,\"as_float\":0.0,\"as_double\":0.0,\"as_string\":\"0\",\"as_date\":\"2000-01-01\",\"as_timestamp\":\"2000-01-01T00:00:00.000-08:00\",\"as_big_decimal\":0},\"nullCount\":{\"as_int\":0,\"as_long\":0,\"as_byte\":0,\"as_short\":0,\"as_float\":0,\"as_double\":0,\"as_string\":0,\"as_date\":0,\"as_timestamp\":0,\"as_big_decimal\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-skipping-basic-stats-all-types-columnmapping-id/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1704847194834,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"1","numOutputBytes":"7974"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0-SNAPSHOT","txnId":"6ee73132-d762-4f51-92f2-b21dd90d75be"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"as_int\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":1,\"delta.columnMapping.physicalName\":\"col-57160bcb-7e76-4076-bfcf-bd8f51835098\"}},{\"name\":\"as_long\",\"type\":\"long\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":2,\"delta.columnMapping.physicalName\":\"col-176e8b51-c3b4-411a-b4ba-7e9e47683d42\"}},{\"name\":\"as_byte\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":3,\"delta.columnMapping.physicalName\":\"col-dc5127f8-f533-44b2-b17b-c50e16f7d83f\"}},{\"name\":\"as_short\",\"type\":\"short\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":4,\"delta.columnMapping.physicalName\":\"col-79b0e87b-98d5-443a-86e4-1ae4a81bba3d\"}},{\"name\":\"as_float\",\"type\":\"float\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":5,\"delta.columnMapping.physicalName\":\"col-2cfa4761-5937-41fb-a388-d60cdfb38987\"}},{\"name\":\"as_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":6,\"delta.columnMapping.physicalName\":\"col-7a5ff7b6-b289-4935-8939-01cdd5f1d011\"}},{\"name\":\"as_string\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":7,\"delta.columnMapping.physicalName\":\"col-c1358712-7cd8-4069-b0d0-c8f32297fbc9\"}},{\"name\":\"as_date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":8,\"delta.columnMapping.physicalName\":\"col-01ea1e58-bb43-4b64-afbf-7b9b1a08b232\"}},{\"name\":\"as_timestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":9,\"delta.columnMapping.physicalName\":\"col-b396c7aa-feb5-451a-a7ce-9123986b7723\"}},{\"name\":\"as_big_decimal\",\"type\":\"decimal(1,0)\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":10,\"delta.columnMapping.physicalName\":\"col-e1cb6194-e2f7-4014-adb6-1821881014dc\"}}]}","partitionColumns":[],"configuration":{"delta.columnMapping.mode":"id","delta.columnMapping.maxColumnId":"10"},"createdTime":1704847193977}} {"protocol":{"minReaderVersion":2,"minWriterVersion":5}} {"add":{"path":"part-00001-4596bea2-786f-404e-bc15-5adc99f00e30-c000.snappy.parquet","partitionValues":{},"size":4949,"modificationTime":1704847194000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col-57160bcb-7e76-4076-bfcf-bd8f51835098\":0,\"col-176e8b51-c3b4-411a-b4ba-7e9e47683d42\":0,\"col-dc5127f8-f533-44b2-b17b-c50e16f7d83f\":0,\"col-79b0e87b-98d5-443a-86e4-1ae4a81bba3d\":0,\"col-2cfa4761-5937-41fb-a388-d60cdfb38987\":0.0,\"col-7a5ff7b6-b289-4935-8939-01cdd5f1d011\":0.0,\"col-c1358712-7cd8-4069-b0d0-c8f32297fbc9\":\"0\",\"col-01ea1e58-bb43-4b64-afbf-7b9b1a08b232\":\"2000-01-01\",\"col-b396c7aa-feb5-451a-a7ce-9123986b7723\":\"2000-01-01T00:00:00.000-08:00\",\"col-e1cb6194-e2f7-4014-adb6-1821881014dc\":0},\"maxValues\":{\"col-57160bcb-7e76-4076-bfcf-bd8f51835098\":0,\"col-176e8b51-c3b4-411a-b4ba-7e9e47683d42\":0,\"col-dc5127f8-f533-44b2-b17b-c50e16f7d83f\":0,\"col-79b0e87b-98d5-443a-86e4-1ae4a81bba3d\":0,\"col-2cfa4761-5937-41fb-a388-d60cdfb38987\":0.0,\"col-7a5ff7b6-b289-4935-8939-01cdd5f1d011\":0.0,\"col-c1358712-7cd8-4069-b0d0-c8f32297fbc9\":\"0\",\"col-01ea1e58-bb43-4b64-afbf-7b9b1a08b232\":\"2000-01-01\",\"col-b396c7aa-feb5-451a-a7ce-9123986b7723\":\"2000-01-01T00:00:00.000-08:00\",\"col-e1cb6194-e2f7-4014-adb6-1821881014dc\":0},\"nullCount\":{\"col-57160bcb-7e76-4076-bfcf-bd8f51835098\":0,\"col-176e8b51-c3b4-411a-b4ba-7e9e47683d42\":0,\"col-dc5127f8-f533-44b2-b17b-c50e16f7d83f\":0,\"col-79b0e87b-98d5-443a-86e4-1ae4a81bba3d\":0,\"col-2cfa4761-5937-41fb-a388-d60cdfb38987\":0,\"col-7a5ff7b6-b289-4935-8939-01cdd5f1d011\":0,\"col-c1358712-7cd8-4069-b0d0-c8f32297fbc9\":0,\"col-01ea1e58-bb43-4b64-afbf-7b9b1a08b232\":0,\"col-b396c7aa-feb5-451a-a7ce-9123986b7723\":0,\"col-e1cb6194-e2f7-4014-adb6-1821881014dc\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-skipping-basic-stats-all-types-columnmapping-name/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1704847190800,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"1","numOutputBytes":"7974"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0-SNAPSHOT","txnId":"0ab95e02-ce9c-463b-9bfd-640d92be62c0"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"as_int\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":1,\"delta.columnMapping.physicalName\":\"col-51c5f1e8-f3d9-4504-86fb-003ea4d1b703\"}},{\"name\":\"as_long\",\"type\":\"long\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":2,\"delta.columnMapping.physicalName\":\"col-e0a075f2-1847-4b21-a0f7-3974f2442f08\"}},{\"name\":\"as_byte\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":3,\"delta.columnMapping.physicalName\":\"col-3101512b-834d-41e9-83b6-e7a2c0ef318a\"}},{\"name\":\"as_short\",\"type\":\"short\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":4,\"delta.columnMapping.physicalName\":\"col-d89267e7-eaa8-492b-81c6-9b36fb9e3434\"}},{\"name\":\"as_float\",\"type\":\"float\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":5,\"delta.columnMapping.physicalName\":\"col-44632733-0978-43d3-bcb5-872df4fb3ace\"}},{\"name\":\"as_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":6,\"delta.columnMapping.physicalName\":\"col-7303ee44-17dd-403f-bb0e-1ccd77c17fab\"}},{\"name\":\"as_string\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":7,\"delta.columnMapping.physicalName\":\"col-4d17e3b0-1801-48d1-89c8-76a280dd8224\"}},{\"name\":\"as_date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":8,\"delta.columnMapping.physicalName\":\"col-5c129760-5d64-4673-a0df-434863f0ea1a\"}},{\"name\":\"as_timestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":9,\"delta.columnMapping.physicalName\":\"col-7296be94-39f4-4703-a791-e1ce50396a17\"}},{\"name\":\"as_big_decimal\",\"type\":\"decimal(1,0)\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":10,\"delta.columnMapping.physicalName\":\"col-715bf6a4-ad83-483e-9703-01a3e9fc23fb\"}}]}","partitionColumns":[],"configuration":{"delta.columnMapping.mode":"name","delta.columnMapping.maxColumnId":"10"},"createdTime":1704847189822}} {"protocol":{"minReaderVersion":2,"minWriterVersion":5}} {"add":{"path":"part-00001-97ba0cfd-25fe-4911-a28f-29d37288fdd0-c000.snappy.parquet","partitionValues":{},"size":4949,"modificationTime":1704847190000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col-51c5f1e8-f3d9-4504-86fb-003ea4d1b703\":0,\"col-e0a075f2-1847-4b21-a0f7-3974f2442f08\":0,\"col-3101512b-834d-41e9-83b6-e7a2c0ef318a\":0,\"col-d89267e7-eaa8-492b-81c6-9b36fb9e3434\":0,\"col-44632733-0978-43d3-bcb5-872df4fb3ace\":0.0,\"col-7303ee44-17dd-403f-bb0e-1ccd77c17fab\":0.0,\"col-4d17e3b0-1801-48d1-89c8-76a280dd8224\":\"0\",\"col-5c129760-5d64-4673-a0df-434863f0ea1a\":\"2000-01-01\",\"col-7296be94-39f4-4703-a791-e1ce50396a17\":\"2000-01-01T00:00:00.000-08:00\",\"col-715bf6a4-ad83-483e-9703-01a3e9fc23fb\":0},\"maxValues\":{\"col-51c5f1e8-f3d9-4504-86fb-003ea4d1b703\":0,\"col-e0a075f2-1847-4b21-a0f7-3974f2442f08\":0,\"col-3101512b-834d-41e9-83b6-e7a2c0ef318a\":0,\"col-d89267e7-eaa8-492b-81c6-9b36fb9e3434\":0,\"col-44632733-0978-43d3-bcb5-872df4fb3ace\":0.0,\"col-7303ee44-17dd-403f-bb0e-1ccd77c17fab\":0.0,\"col-4d17e3b0-1801-48d1-89c8-76a280dd8224\":\"0\",\"col-5c129760-5d64-4673-a0df-434863f0ea1a\":\"2000-01-01\",\"col-7296be94-39f4-4703-a791-e1ce50396a17\":\"2000-01-01T00:00:00.000-08:00\",\"col-715bf6a4-ad83-483e-9703-01a3e9fc23fb\":0},\"nullCount\":{\"col-51c5f1e8-f3d9-4504-86fb-003ea4d1b703\":0,\"col-e0a075f2-1847-4b21-a0f7-3974f2442f08\":0,\"col-3101512b-834d-41e9-83b6-e7a2c0ef318a\":0,\"col-d89267e7-eaa8-492b-81c6-9b36fb9e3434\":0,\"col-44632733-0978-43d3-bcb5-872df4fb3ace\":0,\"col-7303ee44-17dd-403f-bb0e-1ccd77c17fab\":0,\"col-4d17e3b0-1801-48d1-89c8-76a280dd8224\":0,\"col-5c129760-5d64-4673-a0df-434863f0ea1a\":0,\"col-7296be94-39f4-4703-a791-e1ce50396a17\":0,\"col-715bf6a4-ad83-483e-9703-01a3e9fc23fb\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-skipping-change-stats-collected-across-versions/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1704847199416,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"1","numOutputBytes":"1065"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0-SNAPSHOT","txnId":"e12d6dfe-db6a-4039-9fcf-6c13606d416d"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"col1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"col2\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1704847199041}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part-00001-c09e5ddb-2337-4e49-b8be-83fd96008375-c000.snappy.parquet","partitionValues":{},"size":684,"modificationTime":1704847199000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col1\":0,\"col2\":0},\"maxValues\":{\"col1\":0,\"col2\":0},\"nullCount\":{\"col1\":0,\"col2\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-skipping-change-stats-collected-across-versions/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1704847201143,"operation":"SET TBLPROPERTIES","operationParameters":{"properties":"{\"delta.dataSkippingNumIndexedCols\":\"1\"}"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0-SNAPSHOT","txnId":"992a1ee1-a32a-432c-9721-1c22c01cb90a"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"col1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"col2\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.dataSkippingNumIndexedCols":"1"},"createdTime":1704847199041}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-skipping-change-stats-collected-across-versions/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1704847203311,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"1","numOutputBytes":"1065"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0-SNAPSHOT","txnId":"307a5044-5868-4c05-9356-694a9c181a57"}} {"add":{"path":"part-00001-cb335794-98b0-43c3-a3a1-a4c86e3da38d-c000.snappy.parquet","partitionValues":{},"size":684,"modificationTime":1704847203000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col1\":0},\"maxValues\":{\"col1\":0},\"nullCount\":{\"col1\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-skipping-change-stats-collected-across-versions/_delta_log/00000000000000000003.json ================================================ {"commitInfo":{"timestamp":1704847204921,"operation":"SET TBLPROPERTIES","operationParameters":{"properties":"{\"delta.dataSkippingNumIndexedCols\":\"0\"}"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0-SNAPSHOT","txnId":"aaeb218d-6c17-41bb-a36d-35d75507b2c4"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"col1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"col2\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.dataSkippingNumIndexedCols":"0"},"createdTime":1704847199041}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-skipping-change-stats-collected-across-versions/_delta_log/00000000000000000004.json ================================================ {"commitInfo":{"timestamp":1704847206522,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":3,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"1","numOutputBytes":"1065"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0-SNAPSHOT","txnId":"7f7a7132-aba4-49f2-b618-aab176e978a7"}} {"add":{"path":"part-00001-e5d736b6-2ecd-457a-8bb2-947b61f9c67e-c000.snappy.parquet","partitionValues":{},"size":684,"modificationTime":1704847206000,"dataChange":true,"stats":"{\"numRecords\":1}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-skipping-partition-and-data-column/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1704847208280,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"1","numOutputBytes":"1055"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0-SNAPSHOT","txnId":"0003f637-f343-4c70-80bd-be1c8e7b0c59"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"part\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1704847207926}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part-00001-6dedd756-e903-46d7-9e6c-01b3c4ebeab3-c000.snappy.parquet","partitionValues":{},"size":678,"modificationTime":1704847208000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"part\":1,\"id\":0},\"maxValues\":{\"part\":1,\"id\":0},\"nullCount\":{\"part\":0,\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-skipping-partition-and-data-column/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1704847209741,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"1","numOutputBytes":"1055"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0-SNAPSHOT","txnId":"4a790d64-c653-421f-9423-137c982d9810"}} {"add":{"path":"part-00001-2c0ee02a-8591-4026-a5ab-952bdb347fc5-c000.snappy.parquet","partitionValues":{},"size":678,"modificationTime":1704847209000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"part\":1,\"id\":1},\"maxValues\":{\"part\":1,\"id\":1},\"nullCount\":{\"part\":0,\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-skipping-partition-and-data-column/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1704847211152,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"1","numOutputBytes":"1055"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0-SNAPSHOT","txnId":"ade7341b-c742-4861-afdd-43cfa39aaa54"}} {"add":{"path":"part-00001-442a6473-8d9a-41d3-8172-e2248e8be169-c000.snappy.parquet","partitionValues":{},"size":678,"modificationTime":1704847211000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"part\":0,\"id\":1},\"maxValues\":{\"part\":0,\"id\":1},\"nullCount\":{\"part\":0,\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/data-skipping-partition-and-data-column/_delta_log/00000000000000000003.json ================================================ {"commitInfo":{"timestamp":1704847212753,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"1","numOutputBytes":"1055"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0-SNAPSHOT","txnId":"91386c19-6a82-4708-bbe1-3049e2dbfd57"}} {"add":{"path":"part-00001-2822cff2-34ab-4b93-9cbb-4e751084a422-c000.snappy.parquet","partitionValues":{},"size":678,"modificationTime":1704847212000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"part\":0,\"id\":0},\"maxValues\":{\"part\":0,\"id\":0},\"nullCount\":{\"part\":0,\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/decimal-various-scale-precision/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1717778521300,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"3","numOutputBytes":"9126"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.3.0-SNAPSHOT","txnId":"5e3bfa16-cf0f-4d40-ad7d-b6426a6b4b7a"}} {"metaData":{"id":"7f750aff-9bf2-4e52-bfce-39811932da26","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"decimal_4_0\",\"type\":\"decimal(4,0)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_7_0\",\"type\":\"decimal(7,0)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_7_6\",\"type\":\"decimal(7,6)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_12_0\",\"type\":\"decimal(12,0)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_12_6\",\"type\":\"decimal(12,6)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_15_0\",\"type\":\"decimal(15,0)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_15_6\",\"type\":\"decimal(15,6)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_15_12\",\"type\":\"decimal(15,12)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_18_0\",\"type\":\"decimal(18,0)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_18_6\",\"type\":\"decimal(18,6)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_18_12\",\"type\":\"decimal(18,12)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_25_0\",\"type\":\"decimal(25,0)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_25_6\",\"type\":\"decimal(25,6)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_25_12\",\"type\":\"decimal(25,12)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_25_18\",\"type\":\"decimal(25,18)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_25_24\",\"type\":\"decimal(25,24)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_35_0\",\"type\":\"decimal(35,0)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_35_6\",\"type\":\"decimal(35,6)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_35_12\",\"type\":\"decimal(35,12)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_35_18\",\"type\":\"decimal(35,18)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_35_24\",\"type\":\"decimal(35,24)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_35_30\",\"type\":\"decimal(35,30)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_38_0\",\"type\":\"decimal(38,0)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_38_6\",\"type\":\"decimal(38,6)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_38_12\",\"type\":\"decimal(38,12)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_38_18\",\"type\":\"decimal(38,18)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_38_24\",\"type\":\"decimal(38,24)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_38_30\",\"type\":\"decimal(38,30)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_38_36\",\"type\":\"decimal(38,36)\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1717778519308}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part-00000-bb4b3e59-ddb9-4d26-beaf-de9554e14517-c000.snappy.parquet","partitionValues":{},"size":9126,"modificationTime":1717778521237,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"decimal_4_0\":-13,\"decimal_7_0\":0,\"decimal_12_0\":0,\"decimal_12_6\":-0.000098,\"decimal_15_0\":-157,\"decimal_15_6\":-3.346000,\"decimal_15_12\":-0.002162000000,\"decimal_18_0\":0,\"decimal_18_6\":-22641.000000,\"decimal_18_12\":-5.190000000000,\"decimal_25_0\":0,\"decimal_25_6\":-0.000013,\"decimal_25_12\":-3.1661E-8,\"decimal_25_18\":-24199.000000000000000000,\"decimal_35_0\":0,\"decimal_35_6\":-0.000161,\"decimal_35_12\":-2.59176E-7,\"decimal_35_18\":-1.36744000E-10,\"decimal_35_24\":-22827907.000000000000000000000000,\"decimal_35_30\":-32805.309000000000000000000000000000,\"decimal_38_0\":-17,\"decimal_38_6\":-0.027994,\"decimal_38_12\":-0.000024695819,\"decimal_38_18\":-4.614771000E-9,\"decimal_38_24\":-9.718032000000E-12,\"decimal_38_30\":-2.6626087000000000E-14,\"decimal_38_36\":-2.9546424000000000000E-17},\"maxValues\":{\"decimal_4_0\":4,\"decimal_7_0\":0,\"decimal_12_0\":0,\"decimal_12_6\":0.000062,\"decimal_15_0\":481,\"decimal_15_6\":3.302000,\"decimal_15_12\":0.001469000000,\"decimal_18_0\":0,\"decimal_18_6\":7998.000000,\"decimal_18_12\":10.994000000000,\"decimal_25_0\":0,\"decimal_25_6\":0.000021,\"decimal_25_12\":5.925E-9,\"decimal_25_18\":234942.000000000000000000,\"decimal_35_0\":0,\"decimal_35_6\":0.000161,\"decimal_35_12\":1.65519E-7,\"decimal_35_18\":1.52896000E-10,\"decimal_35_24\":14797356.000000000000000000000000,\"decimal_35_30\":8083.687000000000000000000000000000,\"decimal_38_0\":26,\"decimal_38_6\":0.021882,\"decimal_38_12\":0.000032950993,\"decimal_38_18\":1.2783803000E-8,\"decimal_38_24\":2.395564000000E-12,\"decimal_38_30\":2.9414203000000000E-14,\"decimal_38_36\":3.241836000000000000E-18},\"nullCount\":{\"decimal_4_0\":1,\"decimal_7_0\":1,\"decimal_7_6\":3,\"decimal_12_0\":1,\"decimal_12_6\":1,\"decimal_15_0\":1,\"decimal_15_6\":1,\"decimal_15_12\":1,\"decimal_18_0\":1,\"decimal_18_6\":1,\"decimal_18_12\":1,\"decimal_25_0\":1,\"decimal_25_6\":1,\"decimal_25_12\":1,\"decimal_25_18\":1,\"decimal_25_24\":3,\"decimal_35_0\":1,\"decimal_35_6\":1,\"decimal_35_12\":1,\"decimal_35_18\":1,\"decimal_35_24\":1,\"decimal_35_30\":1,\"decimal_38_0\":1,\"decimal_38_6\":1,\"decimal_38_12\":1,\"decimal_38_18\":1,\"decimal_38_24\":1,\"decimal_38_30\":1,\"decimal_38_36\":1}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/delete-re-add-same-file-different-transactions/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1697064953062,"operation":"Manual Update","operationParameters":{},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"fc238989-fb88-4fd1-8be4-328e0b719260"}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"intCol\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{}}} {"add":{"path":"foo","partitionValues":{},"size":1,"modificationTime":1600000000000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/delete-re-add-same-file-different-transactions/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1697064967361,"operation":"Manual Update","operationParameters":{},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"1bd8568f-6a12-41ac-983c-cd5e1b4e1caa"}} {"remove":{"path":"foo","deletionTimestamp":1697064967349,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/delete-re-add-same-file-different-transactions/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1697064970033,"operation":"Manual Update","operationParameters":{},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"2c2bb7f0-5a0f-4670-80b0-f787f7b23a80"}} {"add":{"path":"foo","partitionValues":{},"size":1,"modificationTime":1700000000000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/delete-re-add-same-file-different-transactions/_delta_log/00000000000000000003.json ================================================ {"commitInfo":{"timestamp":1697064972273,"operation":"Manual Update","operationParameters":{},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"52827868-a12a-4ea7-9d38-24bfe5e94e05"}} {"add":{"path":"bar","partitionValues":{},"size":1,"modificationTime":1697064972263,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-commit-info/_delta_log/00000000000000000000.json ================================================ {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"partitionColumns":[],"configuration":{},"createdTime":1607452026918}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"commitInfo":{"timestamp":1540415658000,"userId":"user_0","userName":"username_0","operation":"WRITE","operationParameters":{"test":"test"},"job":{"jobId":"job_id_0","jobName":"job_name_0","runId":"run_id_0","jobOwnerId":"job_owner_0","triggerType":"trigger_type_0"},"notebook":{"notebookId":"notebook_id_0"},"clusterId":"cluster_id_0","readVersion":-1,"isolationLevel":"default","isBlindAppend":true,"operationMetrics":{"test":"test"},"userMetadata":"foo"}} {"add":{"path":"abc","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-getChanges/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1704392842074,"operation":"Manual Update","operationParameters":{},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0-SNAPSHOT","txnId":"95ec924a-6859-4433-8008-6d6b4a0e3ba5"}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"part\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{}}} {"add":{"path":"fake/path/1","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-getChanges/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1704392846030,"operation":"Manual Update","operationParameters":{},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0-SNAPSHOT","txnId":"01d40235-c8b4-4f8e-8f19-8c97872217fd"}} {"cdc":{"path":"fake/path/2","partitionValues":{"partition_foo":"partition_bar"},"size":1,"tags":{"tag_foo":"tag_bar"},"dataChange":false}} {"remove":{"path":"fake/path/1","deletionTimestamp":100,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-getChanges/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1704392846603,"operation":"Manual Update","operationParameters":{},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0-SNAPSHOT","txnId":"6cef7579-ca93-4427-988e-9269e8db50c7"}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"txn":{"appId":"fakeAppId","version":3,"lastUpdated":200}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-invalid-protocol-version/_delta_log/00000000000000000000.json ================================================ {"protocol":{"minReaderVersion":99,"minWriterVersion":7,"readerFeatures":[],"writerFeatures":[]}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{}}} {"add":{"path":"abc","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-metadata/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603724008326,"operation":"Manual Update","operationParameters":{},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"d847eb65-8196-4f17-b2f8-021454e7a6b9","format":{"provider":"parquet","options":{}},"partitionColumns":[],"configuration":{},"createdTime":1603724008326}} {"add":{"path":"0","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-metadata/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1603724009009,"operation":"Manual Update","operationParameters":{},"readVersion":0,"isBlindAppend":true}} {"add":{"path":"1","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-metadata/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1603724009827,"operation":"Manual Update","operationParameters":{},"readVersion":1,"isBlindAppend":true}} {"add":{"path":"2","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-metadata/_delta_log/00000000000000000003.json ================================================ {"commitInfo":{"timestamp":1603724010438,"operation":"Manual Update","operationParameters":{},"readVersion":2,"isBlindAppend":true}} {"add":{"path":"3","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-metadata/_delta_log/00000000000000000004.json ================================================ {"commitInfo":{"timestamp":1603724011089,"operation":"Manual Update","operationParameters":{},"readVersion":3,"isBlindAppend":true}} {"add":{"path":"4","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-metadata/_delta_log/00000000000000000005.json ================================================ {"commitInfo":{"timestamp":1603724011784,"operation":"Manual Update","operationParameters":{},"readVersion":4,"isBlindAppend":true}} {"add":{"path":"5","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-metadata/_delta_log/00000000000000000006.json ================================================ {"commitInfo":{"timestamp":1603724012518,"operation":"Manual Update","operationParameters":{},"readVersion":5,"isBlindAppend":true}} {"add":{"path":"6","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-metadata/_delta_log/00000000000000000007.json ================================================ {"commitInfo":{"timestamp":1603724013308,"operation":"Manual Update","operationParameters":{},"readVersion":6,"isBlindAppend":true}} {"add":{"path":"7","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-metadata/_delta_log/00000000000000000008.json ================================================ {"commitInfo":{"timestamp":1603724014139,"operation":"Manual Update","operationParameters":{},"readVersion":7,"isBlindAppend":true}} {"add":{"path":"8","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-metadata/_delta_log/00000000000000000009.json ================================================ {"commitInfo":{"timestamp":1603724015017,"operation":"Manual Update","operationParameters":{},"readVersion":8,"isBlindAppend":true}} {"add":{"path":"9","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-metadata/_delta_log/00000000000000000010.json ================================================ {"commitInfo":{"timestamp":1603724016018,"operation":"Manual Update","operationParameters":{},"readVersion":9,"isBlindAppend":true}} {"add":{"path":"10","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-metadata/_delta_log/_last_checkpoint ================================================ {"version":10,"size":13} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-protocol/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603723997752,"operation":"Manual Update","operationParameters":{},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"8e276544-6bc2-4935-ac73-873ff9347d05","format":{"provider":"parquet","options":{}},"partitionColumns":[],"configuration":{},"createdTime":1603723997752}} {"add":{"path":"0","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-protocol/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1603723998268,"operation":"Manual Update","operationParameters":{},"readVersion":0,"isBlindAppend":true}} {"add":{"path":"1","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-protocol/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1603723998848,"operation":"Manual Update","operationParameters":{},"readVersion":1,"isBlindAppend":true}} {"add":{"path":"2","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-protocol/_delta_log/00000000000000000003.json ================================================ {"commitInfo":{"timestamp":1603723999470,"operation":"Manual Update","operationParameters":{},"readVersion":2,"isBlindAppend":true}} {"add":{"path":"3","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-protocol/_delta_log/00000000000000000004.json ================================================ {"commitInfo":{"timestamp":1603724000137,"operation":"Manual Update","operationParameters":{},"readVersion":3,"isBlindAppend":true}} {"add":{"path":"4","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-protocol/_delta_log/00000000000000000005.json ================================================ {"commitInfo":{"timestamp":1603724000823,"operation":"Manual Update","operationParameters":{},"readVersion":4,"isBlindAppend":true}} {"add":{"path":"5","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-protocol/_delta_log/00000000000000000006.json ================================================ {"commitInfo":{"timestamp":1603724001567,"operation":"Manual Update","operationParameters":{},"readVersion":5,"isBlindAppend":true}} {"add":{"path":"6","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-protocol/_delta_log/00000000000000000007.json ================================================ {"commitInfo":{"timestamp":1603724002323,"operation":"Manual Update","operationParameters":{},"readVersion":6,"isBlindAppend":true}} {"add":{"path":"7","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-protocol/_delta_log/00000000000000000008.json ================================================ {"commitInfo":{"timestamp":1603724003208,"operation":"Manual Update","operationParameters":{},"readVersion":7,"isBlindAppend":true}} {"add":{"path":"8","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-protocol/_delta_log/00000000000000000009.json ================================================ {"commitInfo":{"timestamp":1603724004087,"operation":"Manual Update","operationParameters":{},"readVersion":8,"isBlindAppend":true}} {"add":{"path":"9","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-protocol/_delta_log/00000000000000000010.json ================================================ {"commitInfo":{"timestamp":1603724005049,"operation":"Manual Update","operationParameters":{},"readVersion":9,"isBlindAppend":true}} {"add":{"path":"10","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-from-checkpoint-missing-protocol/_delta_log/_last_checkpoint ================================================ {"version":10,"size":13} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-without-metadata/_delta_log/00000000000000000000.json ================================================ {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":[],"writerFeatures":[]}} {"add":{"path":"abc","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/deltalog-state-reconstruction-without-protocol/_delta_log/00000000000000000000.json ================================================ {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"intCol\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{}}} {"add":{"path":"abc","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-partitioned-with-checkpoint/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1688691721441,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"part\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"20","numOutputRows":"50","numOutputBytes":"14678"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"bc4bfee2-8faf-4995-bfa1-a3c4930cbf22"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"part\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"col1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"col2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["part"],"configuration":{"delta.enableDeletionVectors":"true"},"createdTime":1688691715853}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["deletionVectors"],"writerFeatures":["deletionVectors"]}} {"add":{"path":"part=0/part-00000-8387c699-30b1-4734-a791-9278d560ec19.c000.snappy.parquet","partitionValues":{"part":"0"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":0,\"col2\":\"foo0\"},\"maxValues\":{\"col1\":20,\"col2\":\"foo0\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} {"add":{"path":"part=1/part-00000-a1586fa1-50e8-4f06-858a-b43b2e83010b.c000.snappy.parquet","partitionValues":{"part":"1"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":1,\"col2\":\"foo1\"},\"maxValues\":{\"col1\":21,\"col2\":\"foo1\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} {"add":{"path":"part=2/part-00000-ad58cb56-93db-4374-91ba-e65e7fa68e76.c000.snappy.parquet","partitionValues":{"part":"2"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":2,\"col2\":\"foo2\"},\"maxValues\":{\"col1\":22,\"col2\":\"foo2\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} {"add":{"path":"part=3/part-00000-319bea86-657f-4431-9b26-949dba99cf2c.c000.snappy.parquet","partitionValues":{"part":"3"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":3,\"col2\":\"foo3\"},\"maxValues\":{\"col1\":23,\"col2\":\"foo3\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} {"add":{"path":"part=4/part-00000-69ec928d-3737-4eb3-a3d8-9555a6b55ff5.c000.snappy.parquet","partitionValues":{"part":"4"},"size":735,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":4,\"col2\":\"foo4\"},\"maxValues\":{\"col1\":24,\"col2\":\"foo4\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} {"add":{"path":"part=5/part-00000-5c963f16-d5b8-4f8b-8d8a-0e3403228be2.c000.snappy.parquet","partitionValues":{"part":"5"},"size":732,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col1\":5,\"col2\":\"foo0\"},\"maxValues\":{\"col1\":15,\"col2\":\"foo0\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} {"add":{"path":"part=6/part-00000-be524334-115d-4d01-8614-e1bc8c630926.c000.snappy.parquet","partitionValues":{"part":"6"},"size":732,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col1\":6,\"col2\":\"foo1\"},\"maxValues\":{\"col1\":16,\"col2\":\"foo1\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} {"add":{"path":"part=7/part-00000-33cc19fc-3607-4ea7-ab6d-af4e3ebf62c4.c000.snappy.parquet","partitionValues":{"part":"7"},"size":732,"modificationTime":1688691721000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col1\":7,\"col2\":\"foo2\"},\"maxValues\":{\"col1\":17,\"col2\":\"foo2\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} {"add":{"path":"part=8/part-00000-02c66988-3465-4483-9f85-7155e6aee1f4.c000.snappy.parquet","partitionValues":{"part":"8"},"size":732,"modificationTime":1688691721000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col1\":8,\"col2\":\"foo3\"},\"maxValues\":{\"col1\":18,\"col2\":\"foo3\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} {"add":{"path":"part=9/part-00000-e4012c8c-cc60-44c0-babb-8c5d264a3a31.c000.snappy.parquet","partitionValues":{"part":"9"},"size":732,"modificationTime":1688691721000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col1\":9,\"col2\":\"foo4\"},\"maxValues\":{\"col1\":19,\"col2\":\"foo4\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} {"add":{"path":"part=0/part-00001-24cdbe06-d3dc-449f-bd38-575228ca42a7.c000.snappy.parquet","partitionValues":{"part":"0"},"size":732,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col1\":30,\"col2\":\"foo0\"},\"maxValues\":{\"col1\":40,\"col2\":\"foo0\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} {"add":{"path":"part=1/part-00001-d7e5d32a-55fa-410a-afee-adcdf46bc859.c000.snappy.parquet","partitionValues":{"part":"1"},"size":732,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col1\":31,\"col2\":\"foo1\"},\"maxValues\":{\"col1\":41,\"col2\":\"foo1\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} {"add":{"path":"part=2/part-00001-ab1247be-1f77-41e6-a392-50a99b2db864.c000.snappy.parquet","partitionValues":{"part":"2"},"size":732,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col1\":32,\"col2\":\"foo2\"},\"maxValues\":{\"col1\":42,\"col2\":\"foo2\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} {"add":{"path":"part=3/part-00001-afeef1dd-2517-49b9-873e-e9e6e8a74b19.c000.snappy.parquet","partitionValues":{"part":"3"},"size":731,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col1\":33,\"col2\":\"foo3\"},\"maxValues\":{\"col1\":43,\"col2\":\"foo3\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} {"add":{"path":"part=4/part-00001-e63d3db6-9e97-4472-aacc-6af9fa44e73d.c000.snappy.parquet","partitionValues":{"part":"4"},"size":732,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col1\":34,\"col2\":\"foo4\"},\"maxValues\":{\"col1\":44,\"col2\":\"foo4\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} {"add":{"path":"part=5/part-00001-f344b457-fbd0-4bc4-9502-2c07025e5bb1.c000.snappy.parquet","partitionValues":{"part":"5"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":25,\"col2\":\"foo0\"},\"maxValues\":{\"col1\":45,\"col2\":\"foo0\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} {"add":{"path":"part=6/part-00001-6fc16401-ac51-4b89-bf08-bb86cecb5cc2.c000.snappy.parquet","partitionValues":{"part":"6"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":26,\"col2\":\"foo1\"},\"maxValues\":{\"col1\":46,\"col2\":\"foo1\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} {"add":{"path":"part=7/part-00001-986abb06-e672-4134-83d4-261752b236b8.c000.snappy.parquet","partitionValues":{"part":"7"},"size":736,"modificationTime":1688691721000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":27,\"col2\":\"foo2\"},\"maxValues\":{\"col1\":47,\"col2\":\"foo2\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} {"add":{"path":"part=8/part-00001-7c58de64-d72f-4373-8d86-dfdc00fb264e.c000.snappy.parquet","partitionValues":{"part":"8"},"size":736,"modificationTime":1688691721000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":28,\"col2\":\"foo3\"},\"maxValues\":{\"col1\":48,\"col2\":\"foo3\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} {"add":{"path":"part=9/part-00001-c0430af8-a8e0-4b23-8776-b2fc549b3e4e.c000.snappy.parquet","partitionValues":{"part":"9"},"size":736,"modificationTime":1688691721000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":29,\"col2\":\"foo4\"},\"maxValues\":{\"col1\":49,\"col2\":\"foo4\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":true}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-partitioned-with-checkpoint/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1688691744588,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#463 = 0)\"]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"13565","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"33e9e33f-49f5-421c-bf62-c3e71c55cd32"}} {"add":{"path":"part=0/part-00000-8387c699-30b1-4734-a791-9278d560ec19.c000.snappy.parquet","partitionValues":{"part":"0"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":0,\"col2\":\"foo0\"},\"maxValues\":{\"col1\":20,\"col2\":\"foo0\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"0X9F0q2<2yJ-f)Gm2!e0","offset":1,"sizeInBytes":34,"cardinality":1}}} {"remove":{"path":"part=0/part-00000-8387c699-30b1-4734-a791-9278d560ec19.c000.snappy.parquet","deletionTimestamp":1688691741121,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"part":"0"},"size":736}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-partitioned-with-checkpoint/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1688691752166,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#1939 = 2)\"]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"5256","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"a9487f5e-84c9-44d8-a6d6-f7444e8405a9"}} {"add":{"path":"part=2/part-00000-ad58cb56-93db-4374-91ba-e65e7fa68e76.c000.snappy.parquet","partitionValues":{"part":"2"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":2,\"col2\":\"foo2\"},\"maxValues\":{\"col1\":22,\"col2\":\"foo2\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"]m5]-UtmB0Rl$j","offset":1,"sizeInBytes":34,"cardinality":1}}} {"remove":{"path":"part=8/part-00000-02c66988-3465-4483-9f85-7155e6aee1f4.c000.snappy.parquet","deletionTimestamp":1688691765550,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"part":"8"},"size":732}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-partitioned-with-checkpoint/_delta_log/00000000000000000006.json ================================================ {"commitInfo":{"timestamp":1688691770236,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#7069 = 10)\"]"},"readVersion":5,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"3110","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"7b578eea-e45c-4308-8c86-3c4c4f2bb9f9"}} {"add":{"path":"part=0/part-00000-8387c699-30b1-4734-a791-9278d560ec19.c000.snappy.parquet","partitionValues":{"part":"0"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":0,\"col2\":\"foo0\"},\"maxValues\":{\"col1\":20,\"col2\":\"foo0\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"Zcq?bs*HKQWCB[Sjf[.o","offset":1,"sizeInBytes":36,"cardinality":2}}} {"remove":{"path":"part=0/part-00000-8387c699-30b1-4734-a791-9278d560ec19.c000.snappy.parquet","deletionTimestamp":1688691769658,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"part":"0"},"size":736,"deletionVector":{"storageType":"u","pathOrInlineDv":"0X9F0q2<2yJ-f)Gm2!e0","offset":1,"sizeInBytes":34,"cardinality":1}}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-partitioned-with-checkpoint/_delta_log/00000000000000000007.json ================================================ {"commitInfo":{"timestamp":1688691774168,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#8351 = 12)\"]"},"readVersion":6,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"2645","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"970ae4f8-d941-4029-b282-20e244e00cd1"}} {"add":{"path":"part=2/part-00000-ad58cb56-93db-4374-91ba-e65e7fa68e76.c000.snappy.parquet","partitionValues":{"part":"2"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":2,\"col2\":\"foo2\"},\"maxValues\":{\"col1\":22,\"col2\":\"foo2\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"NZQXpd#3xxK!oBujr/=:","offset":1,"sizeInBytes":36,"cardinality":2}}} {"remove":{"path":"part=2/part-00000-ad58cb56-93db-4374-91ba-e65e7fa68e76.c000.snappy.parquet","deletionTimestamp":1688691773564,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"part":"2"},"size":736,"deletionVector":{"storageType":"u","pathOrInlineDv":"]m5]-UtmB0Rl$j","offset":1,"sizeInBytes":34,"cardinality":1}}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-partitioned-with-checkpoint/_delta_log/00000000000000000011.json ================================================ {"commitInfo":{"timestamp":1688691788332,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#13579 = 20)\"]"},"readVersion":10,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"2084","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"ad17e138-a1c2-4b6a-a373-e334c138c9e0"}} {"remove":{"path":"part=0/part-00000-8387c699-30b1-4734-a791-9278d560ec19.c000.snappy.parquet","deletionTimestamp":1688691788291,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"part":"0"},"size":736,"deletionVector":{"storageType":"u","pathOrInlineDv":"Zcq?bs*HKQWCB[Sjf[.o","offset":1,"sizeInBytes":36,"cardinality":2}}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-partitioned-with-checkpoint/_delta_log/00000000000000000012.json ================================================ {"commitInfo":{"timestamp":1688691791603,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#15220 = 22)\"]"},"readVersion":11,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1870","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"68c7ac7c-6e73-4aa4-864f-ef374e110de6"}} {"remove":{"path":"part=2/part-00000-ad58cb56-93db-4374-91ba-e65e7fa68e76.c000.snappy.parquet","deletionTimestamp":1688691791568,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"part":"2"},"size":736,"deletionVector":{"storageType":"u","pathOrInlineDv":"NZQXpd#3xxK!oBujr/=:","offset":1,"sizeInBytes":36,"cardinality":2}}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-partitioned-with-checkpoint/_delta_log/00000000000000000013.json ================================================ {"commitInfo":{"timestamp":1688691796131,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#16548 = 24)\"]"},"readVersion":12,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"3461","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"4e6f5590-7630-44d1-bc1c-5a2bfd637c4f"}} {"remove":{"path":"part=4/part-00000-69ec928d-3737-4eb3-a3d8-9555a6b55ff5.c000.snappy.parquet","deletionTimestamp":1688691796068,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"part":"4"},"size":735,"deletionVector":{"storageType":"u","pathOrInlineDv":"+V9Oq([R6mTaoM.}Isa)","offset":1,"sizeInBytes":36,"cardinality":2}}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-partitioned-with-checkpoint/_delta_log/00000000000000000014.json ================================================ {"commitInfo":{"timestamp":1688691798638,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#17876 = 26)\"]"},"readVersion":13,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1451","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"34f3a5ca-2166-4b49-a92b-f6b70ce4cd89"}} {"add":{"path":"part=6/part-00001-6fc16401-ac51-4b89-bf08-bb86cecb5cc2.c000.snappy.parquet","partitionValues":{"part":"6"},"size":736,"modificationTime":1688691720000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":26,\"col2\":\"foo1\"},\"maxValues\":{\"col1\":46,\"col2\":\"foo1\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"]g#Gi8g29gLy&KMkGJr?","offset":1,"sizeInBytes":34,"cardinality":1}}} {"remove":{"path":"part=6/part-00001-6fc16401-ac51-4b89-bf08-bb86cecb5cc2.c000.snappy.parquet","deletionTimestamp":1688691798060,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"part":"6"},"size":736}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-partitioned-with-checkpoint/_delta_log/00000000000000000015.json ================================================ {"commitInfo":{"timestamp":1688691801431,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#19379 = 28)\"]"},"readVersion":14,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1804","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"ace020dc-2202-4814-bbd4-36fc7a2a7988"}} {"add":{"path":"part=8/part-00001-7c58de64-d72f-4373-8d86-dfdc00fb264e.c000.snappy.parquet","partitionValues":{"part":"8"},"size":736,"modificationTime":1688691721000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col1\":28,\"col2\":\"foo3\"},\"maxValues\":{\"col1\":48,\"col2\":\"foo3\"},\"nullCount\":{\"col1\":0,\"col2\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"7NnC4LX-RqU.S:B4VD9n","offset":1,"sizeInBytes":34,"cardinality":1}}} {"remove":{"path":"part=8/part-00001-7c58de64-d72f-4373-8d86-dfdc00fb264e.c000.snappy.parquet","deletionTimestamp":1688691800672,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"part":"8"},"size":736}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-partitioned-with-checkpoint/_delta_log/_last_checkpoint ================================================ {"version":10,"size":30,"sizeInBytes":20573,"numOfAddFiles":18,"checkpointSchema":{"type":"struct","fields":[{"name":"txn","type":{"type":"struct","fields":[{"name":"appId","type":"string","nullable":true,"metadata":{}},{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"lastUpdated","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"add","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}},{"name":"stats","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues_parsed","type":{"type":"struct","fields":[{"name":"part","type":"integer","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"remove","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"deletionTimestamp","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"extendedFileMetadata","type":"boolean","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"metaData","type":{"type":"struct","fields":[{"name":"id","type":"string","nullable":true,"metadata":{}},{"name":"name","type":"string","nullable":true,"metadata":{}},{"name":"description","type":"string","nullable":true,"metadata":{}},{"name":"format","type":{"type":"struct","fields":[{"name":"provider","type":"string","nullable":true,"metadata":{}},{"name":"options","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"schemaString","type":"string","nullable":true,"metadata":{}},{"name":"partitionColumns","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"configuration","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"createdTime","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"protocol","type":{"type":"struct","fields":[{"name":"minReaderVersion","type":"integer","nullable":true,"metadata":{}},{"name":"minWriterVersion","type":"integer","nullable":true,"metadata":{}},{"name":"readerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"writerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"domainMetadata","type":{"type":"struct","fields":[{"name":"domain","type":"string","nullable":true,"metadata":{}},{"name":"configuration","type":"string","nullable":true,"metadata":{}},{"name":"removed","type":"boolean","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"checksum":"822eda70bb966b38d646f351ea753463"} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-with-columnmapping/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1697177838187,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"part\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"20","numOutputRows":"50","numOutputBytes":"24078"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"ccd115ea-c565-49a1-8b44-d97619864edc"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"part\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":1,\"delta.columnMapping.physicalName\":\"col-60c949ca-b8bc-4330-b931-b73fb4c60037\"}},{\"name\":\"col1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":2,\"delta.columnMapping.physicalName\":\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\"}},{\"name\":\"col2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":3,\"delta.columnMapping.physicalName\":\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\"}}]}","partitionColumns":["part"],"configuration":{"delta.columnMapping.mode":"name","delta.enableDeletionVectors":"true","delta.columnMapping.maxColumnId":"3"},"createdTime":1697177835151}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["deletionVectors","columnMapping"],"writerFeatures":["deletionVectors","columnMapping"]}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=0/part-00000-d1888b8a-150e-4fe3-a397-1514739499b4.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"0"},"size":1206,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo0\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":20,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo0\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":true}"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=1/part-00000-19513938-badc-4bd4-9513-3d043d1491dc.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"1"},"size":1206,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":1,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo1\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":21,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo1\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":true}"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=2/part-00000-3f054c46-7f8f-45f3-a541-25525787b631.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"2"},"size":1206,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":2,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo2\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":22,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo2\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":true}"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=3/part-00000-1a0ac64e-0ce2-493e-b1b0-6cf15c1988f5.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"3"},"size":1206,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":3,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo3\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":23,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo3\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":true}"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=4/part-00000-dbfb557b-b778-454e-bef4-bab9481bcea7.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"4"},"size":1205,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":4,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo4\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":24,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo4\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":true}"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=5/part-00000-6d057276-2da0-45c3-86eb-aed7fd3429b8.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"5"},"size":1202,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":5,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo0\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":15,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo0\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":true}"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=6/part-00000-c0ca807e-59eb-4c84-a67d-c65a2e03c3c5.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"6"},"size":1202,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":6,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo1\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":16,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo1\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":true}"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=7/part-00000-12e816f9-daa3-4197-98f2-217a983bdafd.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"7"},"size":1202,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":7,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo2\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":17,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo2\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":true}"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=8/part-00000-ee6122b8-1474-4764-8bdf-8f8b95c734af.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"8"},"size":1202,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":8,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo3\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":18,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo3\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":true}"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=9/part-00000-f2e5dc2f-b7c6-4772-85d7-23b273a9e54d.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"9"},"size":1202,"modificationTime":1697177838000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":9,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo4\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":19,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo4\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":true}"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=0/part-00001-0e48fbde-daec-44ff-b579-d5c49b6c827f.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"0"},"size":1202,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":30,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo0\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":40,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo0\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":true}"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=1/part-00001-fb5e7c74-75ab-4bee-8234-400040ae127a.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"1"},"size":1202,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":31,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo1\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":41,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo1\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":true}"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=2/part-00001-4825f848-06bb-4b91-94e8-deb40f05feca.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"2"},"size":1202,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":32,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo2\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":42,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo2\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":true}"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=3/part-00001-d1fc7b93-6ec3-4c75-8363-ffd8f1f43420.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"3"},"size":1201,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":33,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo3\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":43,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo3\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":true}"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=4/part-00001-ee7c50c6-4119-41ff-9c0a-285f844e7c31.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"4"},"size":1202,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":34,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo4\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":44,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo4\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":true}"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=5/part-00001-ee535eb0-972e-470f-b705-61884acbbe39.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"5"},"size":1206,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":25,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo0\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":45,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo0\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":true}"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=6/part-00001-8c38a718-ea0d-4ac1-9515-3a6ec23cc86b.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"6"},"size":1206,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":26,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo1\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":46,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo1\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":true}"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=7/part-00001-e2c8fd65-f478-4738-89f2-b4f63bdc166f.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"7"},"size":1206,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":27,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo2\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":47,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo2\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":true}"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=8/part-00001-0878dadb-c875-4347-92a3-8739c303d7bd.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"8"},"size":1206,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":28,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo3\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":48,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo3\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":true}"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=9/part-00001-8bbcb266-0863-4b31-adc0-e1c4d1194cec.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"9"},"size":1206,"modificationTime":1697177838000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":29,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo4\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":49,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo4\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":true}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-with-columnmapping/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1697177845210,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#493 = 0)\"]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numDeletionVectorsAdded":"1","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"0","executionTimeMs":"2463","numDeletionVectorsUpdated":"0","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"d0e5f096-0e13-4e72-a6c1-97e4d4786b8d"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=0/part-00000-d1888b8a-150e-4fe3-a397-1514739499b4.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"0"},"size":1206,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo0\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":20,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo0\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"5DxE90SCJoMhWOS&LEbJ","offset":1,"sizeInBytes":34,"cardinality":1}}} {"remove":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=0/part-00000-d1888b8a-150e-4fe3-a397-1514739499b4.c000.snappy.parquet","deletionTimestamp":1697177844817,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"0"},"size":1206,"stats":"{\"numRecords\":3}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-with-columnmapping/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1697177847623,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#1991 = 2)\"]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numDeletionVectorsAdded":"1","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"0","executionTimeMs":"1314","numDeletionVectorsUpdated":"0","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"d4007f70-5f27-40a0-9c4e-47e579230a67"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=2/part-00000-3f054c46-7f8f-45f3-a541-25525787b631.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"2"},"size":1206,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":2,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo2\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":22,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo2\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"&9sA9{u>E#S[qd3XLIaZ","offset":1,"sizeInBytes":34,"cardinality":1}}} {"remove":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=2/part-00000-3f054c46-7f8f-45f3-a541-25525787b631.c000.snappy.parquet","deletionTimestamp":1697177847357,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"2"},"size":1206,"stats":"{\"numRecords\":3}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-with-columnmapping/_delta_log/00000000000000000003.json ================================================ {"commitInfo":{"timestamp":1697177849801,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#3297 = 4)\"]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numDeletionVectorsAdded":"1","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"0","executionTimeMs":"1201","numDeletionVectorsUpdated":"0","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"ec2e636d-e0a8-4936-b754-ef2175810a1a"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=4/part-00000-dbfb557b-b778-454e-bef4-bab9481bcea7.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"4"},"size":1205,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":4,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo4\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":24,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo4\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"bmfc5TmwY>TJV$^4Zs$+","offset":1,"sizeInBytes":34,"cardinality":1}}} {"remove":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=4/part-00000-dbfb557b-b778-454e-bef4-bab9481bcea7.c000.snappy.parquet","deletionTimestamp":1697177849598,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"4"},"size":1205,"stats":"{\"numRecords\":3}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-with-columnmapping/_delta_log/00000000000000000004.json ================================================ {"commitInfo":{"timestamp":1697177851920,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#4601 = 6)\"]"},"readVersion":3,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numDeletionVectorsAdded":"1","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"0","executionTimeMs":"1180","numDeletionVectorsUpdated":"0","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"542a35b5-23c2-4193-96df-9e23284a1ad1"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=6/part-00000-c0ca807e-59eb-4c84-a67d-c65a2e03c3c5.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"6"},"size":1202,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":6,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo1\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":16,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo1\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"GOBG7L]@&bKr%xe4PF6?","offset":1,"sizeInBytes":34,"cardinality":1}}} {"remove":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=6/part-00000-c0ca807e-59eb-4c84-a67d-c65a2e03c3c5.c000.snappy.parquet","deletionTimestamp":1697177851705,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"6"},"size":1202,"stats":"{\"numRecords\":2}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-with-columnmapping/_delta_log/00000000000000000005.json ================================================ {"commitInfo":{"timestamp":1697177853860,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#5905 = 8)\"]"},"readVersion":4,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numDeletionVectorsAdded":"1","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"0","executionTimeMs":"1119","numDeletionVectorsUpdated":"0","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"b9b62b66-67fe-4e93-be55-4a6d077ec1b9"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=8/part-00000-ee6122b8-1474-4764-8bdf-8f8b95c734af.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"8"},"size":1202,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":8,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo3\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":18,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo3\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"m?5^MJ=4G^SC%md8dbcb","offset":1,"sizeInBytes":34,"cardinality":1}}} {"remove":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=8/part-00000-ee6122b8-1474-4764-8bdf-8f8b95c734af.c000.snappy.parquet","deletionTimestamp":1697177853670,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"8"},"size":1202,"stats":"{\"numRecords\":2}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-with-columnmapping/_delta_log/00000000000000000006.json ================================================ {"commitInfo":{"timestamp":1697177855845,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#7209 = 10)\"]"},"readVersion":5,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numDeletionVectorsAdded":"1","numDeletionVectorsRemoved":"1","numAddedChangeFiles":"0","executionTimeMs":"1181","numDeletionVectorsUpdated":"1","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"f211797b-f38e-4b65-bcec-e24c59d34b97"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=0/part-00000-d1888b8a-150e-4fe3-a397-1514739499b4.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"0"},"size":1206,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo0\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":20,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo0\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"0Mb{.bIUQYOtd%By%1f@","offset":1,"sizeInBytes":36,"cardinality":2}}} {"remove":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=0/part-00000-d1888b8a-150e-4fe3-a397-1514739499b4.c000.snappy.parquet","deletionTimestamp":1697177855619,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"0"},"size":1206,"deletionVector":{"storageType":"u","pathOrInlineDv":"5DxE90SCJoMhWOS&LEbJ","offset":1,"sizeInBytes":34,"cardinality":1},"stats":"{\"numRecords\":3}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-with-columnmapping/_delta_log/00000000000000000007.json ================================================ {"commitInfo":{"timestamp":1697177857441,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#8513 = 12)\"]"},"readVersion":6,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numDeletionVectorsAdded":"1","numDeletionVectorsRemoved":"1","numAddedChangeFiles":"0","executionTimeMs":"973","numDeletionVectorsUpdated":"1","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"8615ed9c-6975-4b52-8877-1e7ee0f6bdf8"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=2/part-00000-3f054c46-7f8f-45f3-a541-25525787b631.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"2"},"size":1206,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":2,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo2\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":22,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo2\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"oAq1>cIQ1AV0k$:hf0RA","offset":1,"sizeInBytes":36,"cardinality":2}}} {"remove":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=2/part-00000-3f054c46-7f8f-45f3-a541-25525787b631.c000.snappy.parquet","deletionTimestamp":1697177857274,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"2"},"size":1206,"deletionVector":{"storageType":"u","pathOrInlineDv":"&9sA9{u>E#S[qd3XLIaZ","offset":1,"sizeInBytes":34,"cardinality":1},"stats":"{\"numRecords\":3}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-with-columnmapping/_delta_log/00000000000000000008.json ================================================ {"commitInfo":{"timestamp":1697177859306,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#9817 = 14)\"]"},"readVersion":7,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numDeletionVectorsAdded":"1","numDeletionVectorsRemoved":"1","numAddedChangeFiles":"0","executionTimeMs":"1015","numDeletionVectorsUpdated":"1","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"dacd6651-46ce-44da-9097-42ff7c3ba430"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=4/part-00000-dbfb557b-b778-454e-bef4-bab9481bcea7.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"4"},"size":1205,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":4,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo4\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":24,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo4\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"kqleN6eh%pR:{MHoS{Q]","offset":1,"sizeInBytes":36,"cardinality":2}}} {"remove":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=4/part-00000-dbfb557b-b778-454e-bef4-bab9481bcea7.c000.snappy.parquet","deletionTimestamp":1697177859118,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"4"},"size":1205,"deletionVector":{"storageType":"u","pathOrInlineDv":"bmfc5TmwY>TJV$^4Zs$+","offset":1,"sizeInBytes":34,"cardinality":1},"stats":"{\"numRecords\":3}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-with-columnmapping/_delta_log/00000000000000000009.json ================================================ {"commitInfo":{"timestamp":1697177860988,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#11121 = 16)\"]"},"readVersion":8,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"0","numCopiedRows":"0","numDeletionVectorsAdded":"0","numDeletionVectorsRemoved":"1","numAddedChangeFiles":"0","executionTimeMs":"906","numDeletionVectorsUpdated":"0","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"b5fcd099-63f8-4673-bf68-2e4954a26c80"}} {"remove":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=6/part-00000-c0ca807e-59eb-4c84-a67d-c65a2e03c3c5.c000.snappy.parquet","deletionTimestamp":1697177860958,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"6"},"size":1202,"deletionVector":{"storageType":"u","pathOrInlineDv":"GOBG7L]@&bKr%xe4PF6?","offset":1,"sizeInBytes":34,"cardinality":1},"stats":"{\"numRecords\":2}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-with-columnmapping/_delta_log/00000000000000000010.json ================================================ {"commitInfo":{"timestamp":1697177862500,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#12425 = 18)\"]"},"readVersion":9,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"0","numCopiedRows":"0","numDeletionVectorsAdded":"0","numDeletionVectorsRemoved":"1","numAddedChangeFiles":"0","executionTimeMs":"808","numDeletionVectorsUpdated":"0","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"14d135d4-e072-40f1-9fb2-9137d54ac5c2"}} {"remove":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=8/part-00000-ee6122b8-1474-4764-8bdf-8f8b95c734af.c000.snappy.parquet","deletionTimestamp":1697177862473,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"8"},"size":1202,"deletionVector":{"storageType":"u","pathOrInlineDv":"m?5^MJ=4G^SC%md8dbcb","offset":1,"sizeInBytes":34,"cardinality":1},"stats":"{\"numRecords\":2}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-with-columnmapping/_delta_log/00000000000000000011.json ================================================ {"commitInfo":{"timestamp":1697177865395,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#13877 = 20)\"]"},"readVersion":10,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"0","numCopiedRows":"0","numDeletionVectorsAdded":"0","numDeletionVectorsRemoved":"1","numAddedChangeFiles":"0","executionTimeMs":"1201","numDeletionVectorsUpdated":"0","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"c35e56bf-cadb-4c9f-8b76-d14ed0527dee"}} {"remove":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=0/part-00000-d1888b8a-150e-4fe3-a397-1514739499b4.c000.snappy.parquet","deletionTimestamp":1697177865367,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"0"},"size":1206,"deletionVector":{"storageType":"u","pathOrInlineDv":"0Mb{.bIUQYOtd%By%1f@","offset":1,"sizeInBytes":36,"cardinality":2},"stats":"{\"numRecords\":3}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-with-columnmapping/_delta_log/00000000000000000012.json ================================================ {"commitInfo":{"timestamp":1697177867270,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#15575 = 22)\"]"},"readVersion":11,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"0","numCopiedRows":"0","numDeletionVectorsAdded":"0","numDeletionVectorsRemoved":"1","numAddedChangeFiles":"0","executionTimeMs":"721","numDeletionVectorsUpdated":"0","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"aedbce9b-48f2-4384-b65b-a07752818854"}} {"remove":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=2/part-00000-3f054c46-7f8f-45f3-a541-25525787b631.c000.snappy.parquet","deletionTimestamp":1697177867243,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"2"},"size":1206,"deletionVector":{"storageType":"u","pathOrInlineDv":"oAq1>cIQ1AV0k$:hf0RA","offset":1,"sizeInBytes":36,"cardinality":2},"stats":"{\"numRecords\":3}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-with-columnmapping/_delta_log/00000000000000000013.json ================================================ {"commitInfo":{"timestamp":1697177868948,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#16936 = 24)\"]"},"readVersion":12,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"1","numRemovedBytes":"0","numCopiedRows":"0","numDeletionVectorsAdded":"0","numDeletionVectorsRemoved":"1","numAddedChangeFiles":"0","executionTimeMs":"814","numDeletionVectorsUpdated":"0","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"65206981-895e-4279-a10d-ce15017dfa43"}} {"remove":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=4/part-00000-dbfb557b-b778-454e-bef4-bab9481bcea7.c000.snappy.parquet","deletionTimestamp":1697177868918,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"4"},"size":1205,"deletionVector":{"storageType":"u","pathOrInlineDv":"kqleN6eh%pR:{MHoS{Q]","offset":1,"sizeInBytes":36,"cardinality":2},"stats":"{\"numRecords\":3}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-with-columnmapping/_delta_log/00000000000000000014.json ================================================ {"commitInfo":{"timestamp":1697177870510,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#18297 = 26)\"]"},"readVersion":13,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numDeletionVectorsAdded":"1","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"0","executionTimeMs":"872","numDeletionVectorsUpdated":"0","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"dd7b833f-ff28-440e-8dd0-edb6cdfe10b0"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=6/part-00001-8c38a718-ea0d-4ac1-9515-3a6ec23cc86b.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"6"},"size":1206,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":26,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo1\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":46,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo1\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"^wV}@i>E%:Tv:EZ=7q>&","offset":1,"sizeInBytes":34,"cardinality":1}}} {"remove":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=6/part-00001-8c38a718-ea0d-4ac1-9515-3a6ec23cc86b.c000.snappy.parquet","deletionTimestamp":1697177870283,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"6"},"size":1206,"stats":"{\"numRecords\":3}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-with-columnmapping/_delta_log/00000000000000000015.json ================================================ {"commitInfo":{"timestamp":1697177872306,"operation":"DELETE","operationParameters":{"predicate":"[\"(col1#19833 = 28)\"]"},"readVersion":14,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numDeletionVectorsAdded":"1","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"0","executionTimeMs":"948","numDeletionVectorsUpdated":"0","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"a4b3afa7-3ad9-487f-bbe0-81fb606647a8"}} {"add":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=8/part-00001-0878dadb-c875-4347-92a3-8739c303d7bd.c000.snappy.parquet","partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"8"},"size":1206,"modificationTime":1697177837000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":28,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo3\"},\"maxValues\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":48,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":\"foo3\"},\"nullCount\":{\"col-9f0743fd-a52e-44a7-92e0-72a3d34231b1\":0,\"col-ae46e8ed-607b-4b7c-8a13-44991fe0a2e4\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"*Cr(.vsy[nS2z}u{${-v","offset":1,"sizeInBytes":34,"cardinality":1}}} {"remove":{"path":"col-60c949ca-b8bc-4330-b931-b73fb4c60037=8/part-00001-0878dadb-c875-4347-92a3-8739c303d7bd.c000.snappy.parquet","deletionTimestamp":1697177872136,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"col-60c949ca-b8bc-4330-b931-b73fb4c60037":"8"},"size":1206,"stats":"{\"numRecords\":3}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/dv-with-columnmapping/_delta_log/_last_checkpoint ================================================ {"version":10,"size":30,"sizeInBytes":23367,"numOfAddFiles":18,"checkpointSchema":{"type":"struct","fields":[{"name":"txn","type":{"type":"struct","fields":[{"name":"appId","type":"string","nullable":true,"metadata":{}},{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"lastUpdated","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"add","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}},{"name":"stats","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues_parsed","type":{"type":"struct","fields":[{"name":"col-60c949ca-b8bc-4330-b931-b73fb4c60037","type":"integer","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"remove","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"deletionTimestamp","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"extendedFileMetadata","type":"boolean","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"metaData","type":{"type":"struct","fields":[{"name":"id","type":"string","nullable":true,"metadata":{}},{"name":"name","type":"string","nullable":true,"metadata":{}},{"name":"description","type":"string","nullable":true,"metadata":{}},{"name":"format","type":{"type":"struct","fields":[{"name":"provider","type":"string","nullable":true,"metadata":{}},{"name":"options","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"schemaString","type":"string","nullable":true,"metadata":{}},{"name":"partitionColumns","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"configuration","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"createdTime","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"protocol","type":{"type":"struct","fields":[{"name":"minReaderVersion","type":"integer","nullable":true,"metadata":{}},{"name":"minWriterVersion","type":"integer","nullable":true,"metadata":{}},{"name":"readerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"writerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"domainMetadata","type":{"type":"struct","fields":[{"name":"domain","type":"string","nullable":true,"metadata":{}},{"name":"configuration","type":"string","nullable":true,"metadata":{}},{"name":"removed","type":"boolean","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"checksum":"0e712f2b035400d1c5b97f96fcef739f"} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/hive/deltatbl-column-names-case-insensitive/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1629874535433,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"BarFoo\"]"},"isBlindAppend":true,"operationMetrics":{"numFiles":"4","numOutputBytes":"1782","numOutputRows":"10"}}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"FooBar\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"BarFoo\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["BarFoo"],"configuration":{},"createdTime":1629874533636}} {"add":{"path":"BarFoo=foo0/part-00000-36c1f69c-21dc-4374-a89e-1c4468eff784.c000.snappy.parquet","partitionValues":{"BarFoo":"foo0"},"size":448,"modificationTime":1629874535000,"dataChange":true}} {"add":{"path":"BarFoo=foo1/part-00000-5c80a439-70eb-435a-92eb-04549d3f220e.c000.snappy.parquet","partitionValues":{"BarFoo":"foo1"},"size":443,"modificationTime":1629874535000,"dataChange":true}} {"add":{"path":"BarFoo=foo0/part-00001-27f5c1f6-2393-4021-9a0f-44d143761f88.c000.snappy.parquet","partitionValues":{"BarFoo":"foo0"},"size":443,"modificationTime":1629874535000,"dataChange":true}} {"add":{"path":"BarFoo=foo1/part-00001-b6134dd2-aa40-4868-a708-bec69fc562a2.c000.snappy.parquet","partitionValues":{"BarFoo":"foo1"},"size":448,"modificationTime":1629874535000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/hive/deltatbl-deleted-path/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1629874421524,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputBytes":"1318","numOutputRows":"10"}}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"c1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1629874419356}} {"add":{"path":"part-00000-377b2930-7ed7-41e6-bab2-d565a7ca5bfb-c000.snappy.parquet","partitionValues":{},"size":659,"modificationTime":1629874421000,"dataChange":true}} {"add":{"path":"part-00001-6537e97d-662a-430d-9ad9-f6d087ae7cb8-c000.snappy.parquet","partitionValues":{},"size":659,"modificationTime":1629874421000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/hive/deltatbl-incorrect-format-config/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1629874375835,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputBytes":"1306","numOutputRows":"10"}}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"a\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"b\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1629874374114}} {"add":{"path":"part-00000-7b3124df-d8a4-4a4a-9d99-e98cfde281cf-c000.snappy.parquet","partitionValues":{},"size":653,"modificationTime":1629874375000,"dataChange":true}} {"add":{"path":"part-00001-e8582398-602e-4697-a508-fc046c1c57cf-c000.snappy.parquet","partitionValues":{},"size":653,"modificationTime":1629874375000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/hive/deltatbl-map-types-correctly/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1629873175558,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputBytes":"4156","numOutputRows":"1"}}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"c1\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c2\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c3\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c4\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c5\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c6\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c7\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c8\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c9\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c10\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c11\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c12\",\"type\":\"decimal(38,18)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c13\",\"type\":{\"type\":\"array\",\"elementType\":\"string\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"c14\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"long\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"c15\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"f1\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"f2\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1629873173115}} {"add":{"path":"part-00000-c9259a22-ce39-45df-8d76-768bd813c3ff-c000.snappy.parquet","partitionValues":{},"size":4156,"modificationTime":1629873175000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/hive/deltatbl-non-partitioned/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1629872975334,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputBytes":"1318","numOutputRows":"10"}}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"c1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1629872972259}} {"add":{"path":"part-00000-e24c5388-1621-46bd-94eb-fea5209018d0-c000.snappy.parquet","partitionValues":{},"size":659,"modificationTime":1629872975000,"dataChange":true}} {"add":{"path":"part-00001-f2126b8d-1594-451b-9c89-c4c2481bfd93-c000.snappy.parquet","partitionValues":{},"size":659,"modificationTime":1629872975000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/hive/deltatbl-not-allow-write/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1629872770300,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputBytes":"1306","numOutputRows":"10"}}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"a\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"b\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1629872768383}} {"add":{"path":"part-00000-fab61bc4-5175-46ea-ac35-249c0f5750ff-c000.snappy.parquet","partitionValues":{},"size":653,"modificationTime":1629872770000,"dataChange":true}} {"add":{"path":"part-00001-6eb569ba-9300-49e7-9b5a-d064e8c5be2d-c000.snappy.parquet","partitionValues":{},"size":653,"modificationTime":1629872770000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/hive/deltatbl-partition-prune/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1629873077420,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"date\",\"city\"]"},"isBlindAppend":true,"operationMetrics":{"numFiles":"5","numOutputBytes":"3195","numOutputRows":"5"}}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"city\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"cnt\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["date","city"],"configuration":{},"createdTime":1629873075437}} {"add":{"path":"date=20180520/city=hz/part-00000-de1d5bcd-ad7e-4b88-ba9b-31fb8aeb8093.c000.snappy.parquet","partitionValues":{"date":"20180520","city":"hz"},"size":628,"modificationTime":1629873077000,"dataChange":true}} {"add":{"path":"date=20180718/city=hz/part-00000-f888e95b-c831-43fe-bba8-3dbf43b4eb86.c000.snappy.parquet","partitionValues":{"date":"20180718","city":"hz"},"size":639,"modificationTime":1629873077000,"dataChange":true}} {"add":{"path":"date=20180512/city=sh/part-00001-c87aeb63-6d9c-4511-b8b3-71d02178554f.c000.snappy.parquet","partitionValues":{"date":"20180512","city":"sh"},"size":628,"modificationTime":1629873077000,"dataChange":true}} {"add":{"path":"date=20180520/city=bj/part-00001-4c732f0f-a473-400a-8ba3-1499f599b8f1.c000.snappy.parquet","partitionValues":{"date":"20180520","city":"bj"},"size":650,"modificationTime":1629873077000,"dataChange":true}} {"add":{"path":"date=20181212/city=sz/part-00001-529ff89b-55c6-4405-a6cc-04759d5f692b.c000.snappy.parquet","partitionValues":{"date":"20181212","city":"sz"},"size":650,"modificationTime":1629873077000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/hive/deltatbl-partitioned/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1629873032991,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"c2\"]"},"isBlindAppend":true,"operationMetrics":{"numFiles":"4","numOutputBytes":"1734","numOutputRows":"10"}}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"c1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["c2"],"configuration":{},"createdTime":1629873029858}} {"add":{"path":"c2=foo0/part-00000-2bcc9ff6-0551-4401-bd22-d361a60627e3.c000.snappy.parquet","partitionValues":{"c2":"foo0"},"size":436,"modificationTime":1629873032000,"dataChange":true}} {"add":{"path":"c2=foo1/part-00000-786c7455-9587-454f-9a4c-de0b22b62bbd.c000.snappy.parquet","partitionValues":{"c2":"foo1"},"size":431,"modificationTime":1629873032000,"dataChange":true}} {"add":{"path":"c2=foo0/part-00001-ca647ee7-f1ad-4d70-bf02-5d1872324d6f.c000.snappy.parquet","partitionValues":{"c2":"foo0"},"size":431,"modificationTime":1629873032000,"dataChange":true}} {"add":{"path":"c2=foo1/part-00001-1c702e73-89b5-465a-9c6a-25f7559cd150.c000.snappy.parquet","partitionValues":{"c2":"foo1"},"size":436,"modificationTime":1629873032000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/hive/deltatbl-schema-match/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1629872936115,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"b\"]"},"isBlindAppend":true,"operationMetrics":{"numFiles":"4","numOutputBytes":"2494","numOutputRows":"10"}}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"a\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"b\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["b"],"configuration":{},"createdTime":1629872933338}} {"add":{"path":"b=foo0/part-00000-531fe778-e359-44c9-8c35-7ed2416c5ff5.c000.snappy.parquet","partitionValues":{"b":"foo0"},"size":629,"modificationTime":1629872935000,"dataChange":true}} {"add":{"path":"b=foo1/part-00000-7dad1d59-f42c-46c1-992e-35c2fb4d9c09.c000.snappy.parquet","partitionValues":{"b":"foo1"},"size":618,"modificationTime":1629872936000,"dataChange":true}} {"add":{"path":"b=foo0/part-00001-923b258c-b34c-4cb9-8da9-622005e49f2c.c000.snappy.parquet","partitionValues":{"b":"foo0"},"size":618,"modificationTime":1629872935000,"dataChange":true}} {"add":{"path":"b=foo1/part-00001-e44bca08-b26b-4f4d-8a22-5bb45a598dcf.c000.snappy.parquet","partitionValues":{"b":"foo1"},"size":629,"modificationTime":1629872936000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/hive/deltatbl-special-chars-in-partition-column/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1629873142667,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"c2\"]"},"isBlindAppend":true,"operationMetrics":{"numFiles":"4","numOutputBytes":"1734","numOutputRows":"10"}}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"c1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["c2"],"configuration":{},"createdTime":1629873139851}} {"add":{"path":"c2=+%20%253D%25250/part-00000-88ad45a3-9b80-4e66-b474-1748ba085060.c000.snappy.parquet","partitionValues":{"c2":"+ =%0"},"size":436,"modificationTime":1629873142000,"dataChange":true}} {"add":{"path":"c2=+%20%253D%25251/part-00000-180d1a36-4ba9-4321-8145-1e0d73406b02.c000.snappy.parquet","partitionValues":{"c2":"+ =%1"},"size":431,"modificationTime":1629873142000,"dataChange":true}} {"add":{"path":"c2=+%20%253D%25250/part-00001-aff2b410-c566-4e51-a968-acfa96d6f1e9.c000.snappy.parquet","partitionValues":{"c2":"+ =%0"},"size":431,"modificationTime":1629873142000,"dataChange":true}} {"add":{"path":"c2=+%20%253D%25251/part-00001-3379bbbf-1ab8-4781-8b7e-29038d983f83.c000.snappy.parquet","partitionValues":{"c2":"+ =%1"},"size":436,"modificationTime":1629873142000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/hive/deltatbl-touch-files-needed-for-partitioned/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1629873109640,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"c2\"]"},"isBlindAppend":true,"operationMetrics":{"numFiles":"4","numOutputBytes":"1734","numOutputRows":"10"}}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"c1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["c2"],"configuration":{},"createdTime":1629873107868}} {"add":{"path":"c2=foo0/part-00000-f1acd078-4e44-4d47-91b2-6568396e2ec3.c000.snappy.parquet","partitionValues":{"c2":"foo0"},"size":436,"modificationTime":1629873109000,"dataChange":true}} {"add":{"path":"c2=foo1/part-00000-1bb7c99b-be0e-4c49-ae73-9baf5a8a08d0.c000.snappy.parquet","partitionValues":{"c2":"foo1"},"size":431,"modificationTime":1629873109000,"dataChange":true}} {"add":{"path":"c2=foo0/part-00001-e7f40ed6-fefa-41f5-b8a6-c6e9b78a1448.c000.snappy.parquet","partitionValues":{"c2":"foo0"},"size":431,"modificationTime":1629873109000,"dataChange":true}} {"add":{"path":"c2=foo1/part-00001-c357f264-a317-4e93-a530-a8b1360ca9f6.c000.snappy.parquet","partitionValues":{"c2":"foo1"},"size":436,"modificationTime":1629873109000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/kernel-timestamp-INT96/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1692156979734,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"part\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"5","numOutputRows":"5","numOutputBytes":"3535"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"80d90401-5d96-4c45-8f57-2c4488eb4e78"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"part\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"time\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["part"],"configuration":{},"createdTime":1692156974547}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part=2020-01-01%2008%253A09%253A10.001/part-00000-bd889aef-417c-4493-b5f7-a9884ba4b247.c000.snappy.parquet","partitionValues":{"part":"2020-01-01 08:09:10.001"},"size":728,"modificationTime":1692156979000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":0,\"time\":\"2020-02-01T08:09:10.000Z\"},\"maxValues\":{\"id\":0,\"time\":\"2020-02-01T08:09:10.000Z\"},\"nullCount\":{\"id\":0,\"time\":0}}"}} {"add":{"path":"part=2021-10-01%2008%253A09%253A20/part-00000-57e97070-8fc8-485a-95c6-af55daf5e09b.c000.snappy.parquet","partitionValues":{"part":"2021-10-01 08:09:20"},"size":728,"modificationTime":1692156979000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1,\"time\":\"1999-01-01T09:00:00.000Z\"},\"maxValues\":{\"id\":1,\"time\":\"1999-01-01T09:00:00.000Z\"},\"nullCount\":{\"id\":0,\"time\":0}}"}} {"add":{"path":"part=__HIVE_DEFAULT_PARTITION__/part-00001-7cb5f53e-936c-4d24-bca1-9fa0fc7a66e4.c000.snappy.parquet","partitionValues":{"part":null},"size":623,"modificationTime":1692156979000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":4},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0,\"time\":1}}"}} {"add":{"path":"part=1969-01-01%2000%253A00%253A00/part-00001-75ac07ae-d2e8-4030-be59-c490d47c4496.c000.snappy.parquet","partitionValues":{"part":"1969-01-01 00:00:00"},"size":728,"modificationTime":1692156979000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":3,\"time\":\"1969-01-01T00:00:00.000Z\"},\"maxValues\":{\"id\":3,\"time\":\"1969-01-01T00:00:00.000Z\"},\"nullCount\":{\"id\":0,\"time\":0}}"}} {"add":{"path":"part=2021-10-01%2008%253A09%253A20/part-00001-bd0c6fb8-aafd-48dc-9bba-331c1c6f137b.c000.snappy.parquet","partitionValues":{"part":"2021-10-01 08:09:20"},"size":728,"modificationTime":1692156979000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":2,\"time\":\"2000-01-01T09:00:00.000Z\"},\"maxValues\":{\"id\":2,\"time\":\"2000-01-01T09:00:00.000Z\"},\"nullCount\":{\"id\":0,\"time\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/kernel-timestamp-PST/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1692156996195,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"part\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"5","numOutputRows":"5","numOutputBytes":"3535"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"40ad66e4-27a5-4882-96c7-c30c9f8d6fe9"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"part\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"time\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["part"],"configuration":{},"createdTime":1692156995593}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part=2020-01-01%2008%253A09%253A10.001/part-00000-a8be3fd2-1fd5-4dd7-84d2-6899a62d99e8.c000.snappy.parquet","partitionValues":{"part":"2020-01-01 08:09:10.001"},"size":728,"modificationTime":1692156995000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":0,\"time\":\"2020-02-01T08:09:10.000-08:00\"},\"maxValues\":{\"id\":0,\"time\":\"2020-02-01T08:09:10.000-08:00\"},\"nullCount\":{\"id\":0,\"time\":0}}"}} {"add":{"path":"part=2021-10-01%2008%253A09%253A20/part-00000-321ea6ca-841e-4654-9844-2d4041b6d0d6.c000.snappy.parquet","partitionValues":{"part":"2021-10-01 08:09:20"},"size":728,"modificationTime":1692156996000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1,\"time\":\"1999-01-01T09:00:00.000-08:00\"},\"maxValues\":{\"id\":1,\"time\":\"1999-01-01T09:00:00.000-08:00\"},\"nullCount\":{\"id\":0,\"time\":0}}"}} {"add":{"path":"part=__HIVE_DEFAULT_PARTITION__/part-00001-18484b3d-01e6-48bc-9e8b-2a75d36d9f7a.c000.snappy.parquet","partitionValues":{"part":null},"size":623,"modificationTime":1692156995000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":4},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0,\"time\":1}}"}} {"add":{"path":"part=1969-01-01%2000%253A00%253A00/part-00001-48d8c27a-3661-4e1e-95cb-02ef244c1cf4.c000.snappy.parquet","partitionValues":{"part":"1969-01-01 00:00:00"},"size":728,"modificationTime":1692156996000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":3,\"time\":\"1969-01-01T00:00:00.000-08:00\"},\"maxValues\":{\"id\":3,\"time\":\"1969-01-01T00:00:00.000-08:00\"},\"nullCount\":{\"id\":0,\"time\":0}}"}} {"add":{"path":"part=2021-10-01%2008%253A09%253A20/part-00001-b223f8fd-9d33-465b-b139-36c41abb10e8.c000.snappy.parquet","partitionValues":{"part":"2021-10-01 08:09:20"},"size":728,"modificationTime":1692156996000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":2,\"time\":\"2000-01-01T09:00:00.000-08:00\"},\"maxValues\":{\"id\":2,\"time\":\"2000-01-01T09:00:00.000-08:00\"},\"nullCount\":{\"id\":0,\"time\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/kernel-timestamp-TIMESTAMP_MICROS/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1692156990276,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"part\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"5","numOutputRows":"5","numOutputBytes":"3530"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"7c789624-c173-4a06-8abf-fc07d3cf45dc"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"part\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"time\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["part"],"configuration":{},"createdTime":1692156989451}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part=2020-01-01%2008%253A09%253A10.001/part-00000-3cac2575-d0b4-4647-a7a3-b4a9d910cb32.c000.snappy.parquet","partitionValues":{"part":"2020-01-01 08:09:10.001"},"size":719,"modificationTime":1692156989000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":0,\"time\":\"2020-02-01T08:09:10.000Z\"},\"maxValues\":{\"id\":0,\"time\":\"2020-02-01T08:09:10.000Z\"},\"nullCount\":{\"id\":0,\"time\":0}}"}} {"add":{"path":"part=2021-10-01%2008%253A09%253A20/part-00000-038fb25c-ca6b-43b6-b0dc-d987f38d0ab9.c000.snappy.parquet","partitionValues":{"part":"2021-10-01 08:09:20"},"size":719,"modificationTime":1692156990000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1,\"time\":\"1999-01-01T09:00:00.000Z\"},\"maxValues\":{\"id\":1,\"time\":\"1999-01-01T09:00:00.000Z\"},\"nullCount\":{\"id\":0,\"time\":0}}"}} {"add":{"path":"part=__HIVE_DEFAULT_PARTITION__/part-00001-107828e6-a4b9-42b1-9f1f-244c0efc1b08.c000.snappy.parquet","partitionValues":{"part":null},"size":654,"modificationTime":1692156989000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":4},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0,\"time\":1}}"}} {"add":{"path":"part=1969-01-01%2000%253A00%253A00/part-00001-2b5694f1-b839-4037-b264-353b31af6e7b.c000.snappy.parquet","partitionValues":{"part":"1969-01-01 00:00:00"},"size":719,"modificationTime":1692156990000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":3,\"time\":\"1969-01-01T00:00:00.000Z\"},\"maxValues\":{\"id\":3,\"time\":\"1969-01-01T00:00:00.000Z\"},\"nullCount\":{\"id\":0,\"time\":0}}"}} {"add":{"path":"part=2021-10-01%2008%253A09%253A20/part-00001-226faf2a-427a-40ee-bfb5-5d53c8642c8a.c000.snappy.parquet","partitionValues":{"part":"2021-10-01 08:09:20"},"size":719,"modificationTime":1692156990000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":2,\"time\":\"2000-01-01T09:00:00.000Z\"},\"maxValues\":{\"id\":2,\"time\":\"2000-01-01T09:00:00.000Z\"},\"nullCount\":{\"id\":0,\"time\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/kernel-timestamp-TIMESTAMP_MILLIS/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1692156993531,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"part\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"5","numOutputRows":"5","numOutputBytes":"3529"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"d84260a3-a773-4f10-8079-2bb1c1bf1303"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"part\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"time\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["part"],"configuration":{},"createdTime":1692156992918}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part=2020-01-01%2008%253A09%253A10.001/part-00000-4b5188f5-4784-47ce-b4ad-1d3eae80710e.c000.snappy.parquet","partitionValues":{"part":"2020-01-01 08:09:10.001"},"size":719,"modificationTime":1692156993000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":0,\"time\":\"2020-02-01T08:09:10.000Z\"},\"maxValues\":{\"id\":0,\"time\":\"2020-02-01T08:09:10.000Z\"},\"nullCount\":{\"id\":0,\"time\":0}}"}} {"add":{"path":"part=2021-10-01%2008%253A09%253A20/part-00000-086f164a-4d32-4631-b9f9-8aeab485f19c.c000.snappy.parquet","partitionValues":{"part":"2021-10-01 08:09:20"},"size":719,"modificationTime":1692156993000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1,\"time\":\"1999-01-01T09:00:00.000Z\"},\"maxValues\":{\"id\":1,\"time\":\"1999-01-01T09:00:00.000Z\"},\"nullCount\":{\"id\":0,\"time\":0}}"}} {"add":{"path":"part=__HIVE_DEFAULT_PARTITION__/part-00001-f81daebf-3993-4686-bf72-470e1fe078d9.c000.snappy.parquet","partitionValues":{"part":null},"size":654,"modificationTime":1692156993000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":4},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0,\"time\":1}}"}} {"add":{"path":"part=1969-01-01%2000%253A00%253A00/part-00001-4c527a95-ca90-4aeb-a61c-8d89b6330772.c000.snappy.parquet","partitionValues":{"part":"1969-01-01 00:00:00"},"size":718,"modificationTime":1692156993000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":3,\"time\":\"1969-01-01T00:00:00.000Z\"},\"maxValues\":{\"id\":3,\"time\":\"1969-01-01T00:00:00.000Z\"},\"nullCount\":{\"id\":0,\"time\":0}}"}} {"add":{"path":"part=2021-10-01%2008%253A09%253A20/part-00001-94d3f0af-754c-4cde-bc6e-08338a03a32e.c000.snappy.parquet","partitionValues":{"part":"2021-10-01 08:09:20"},"size":719,"modificationTime":1692156993000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":2,\"time\":\"2000-01-01T09:00:00.000Z\"},\"maxValues\":{\"id\":2,\"time\":\"2000-01-01T09:00:00.000Z\"},\"nullCount\":{\"id\":0,\"time\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/kernel-timestamp-partition-col-ISO8601/_delta_log/00000000000000000000.crc ================================================ {"txnId":"a069b528-fe6e-4f37-b4e7-37ff2105a1c2","tableSizeBytes":1140,"numFiles":2,"numMetadata":1,"numProtocol":1,"setTransactions":[],"domainMetadata":[],"metadata":{"id":"ede3d9ad-553d-4fcb-9d4e-22ce34c83e26","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"str\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ts\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["ts"],"configuration":{},"createdTime":1748542296251},"protocol":{"minReaderVersion":1,"minWriterVersion":2},"allFiles":[{"path":"ts=2024-01-01%2010%253A00%253A00/part-00000-9630b3f5-7ab4-4688-9822-3ef93a9d0559.c000.snappy.parquet","partitionValues":{"ts":"2024-01-01T10:00:00.000000Z"},"size":570,"modificationTime":1748542297686,"dataChange":false,"stats":"{\"numRecords\":1,\"minValues\":{\"str\":\"2024-01-01 10:00:00\"},\"maxValues\":{\"str\":\"2024-01-01 10:00:00\"},\"nullCount\":{\"str\":0}}"},{"path":"ts=2024-01-02%2012%253A30%253A00/part-00000-17b5fc05-b487-4b8b-82ff-9ef4352767a5.c000.snappy.parquet","partitionValues":{"ts":"2024-01-02T12:30:00.000000Z"},"size":570,"modificationTime":1748542297742,"dataChange":false,"stats":"{\"numRecords\":1,\"minValues\":{\"str\":\"2024-01-02 12:30:00\"},\"maxValues\":{\"str\":\"2024-01-02 12:30:00\"},\"nullCount\":{\"str\":0}}"}]} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/kernel-timestamp-partition-col-ISO8601/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1748542298780,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[\"ts\"]"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"2","numRemovedFiles":"0","numRemovedBytes":"0","numOutputRows":"2","numOutputBytes":"1140"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"a069b528-fe6e-4f37-b4e7-37ff2105a1c2"}} {"metaData":{"id":"ede3d9ad-553d-4fcb-9d4e-22ce34c83e26","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"str\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ts\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["ts"],"configuration":{},"createdTime":1748542296251}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"ts=2024-01-01%2010%253A00%253A00/part-00000-9630b3f5-7ab4-4688-9822-3ef93a9d0559.c000.snappy.parquet","partitionValues":{"ts":"2024-01-01T10:00:00.000000Z"},"size":570,"modificationTime":1748542297686,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"str\":\"2024-01-01 10:00:00\"},\"maxValues\":{\"str\":\"2024-01-01 10:00:00\"},\"nullCount\":{\"str\":0}}"}} {"add":{"path":"ts=2024-01-02%2012%253A30%253A00/part-00000-17b5fc05-b487-4b8b-82ff-9ef4352767a5.c000.snappy.parquet","partitionValues":{"ts":"2024-01-02T12:30:00.000000Z"},"size":570,"modificationTime":1748542297742,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"str\":\"2024-01-02 12:30:00\"},\"maxValues\":{\"str\":\"2024-01-02 12:30:00\"},\"nullCount\":{\"str\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/log-replay-dv-key-cases/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1697571663834,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"50","numOutputBytes":"765"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"1d19dc4c-3b35-4417-845a-0f44acef5056"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableDeletionVectors":"true"},"createdTime":1697571659174}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["deletionVectors"],"writerFeatures":["deletionVectors"]}} {"add":{"path":"part-00000-90177277-75c2-48db-92a2-20dcba39fd06-c000.snappy.parquet","partitionValues":{},"size":765,"modificationTime":1697571663000,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":49},\"nullCount\":{\"id\":0},\"tightBounds\":true}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/log-replay-dv-key-cases/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1697571681016,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#443L = 0)\"]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numDeletionVectorsAdded":"1","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"0","executionTimeMs":"6668","numDeletionVectorsUpdated":"0","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"7acb06d6-7473-406d-989d-275f025ef11f"}} {"add":{"path":"part-00000-90177277-75c2-48db-92a2-20dcba39fd06-c000.snappy.parquet","partitionValues":{},"size":765,"modificationTime":1697571663000,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":49},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"h{&8fAg]=QYJvl-}c!yH","offset":1,"sizeInBytes":34,"cardinality":1}}} {"remove":{"path":"part-00000-90177277-75c2-48db-92a2-20dcba39fd06-c000.snappy.parquet","deletionTimestamp":1697571680016,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":765,"stats":"{\"numRecords\":50}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/log-replay-dv-key-cases/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1697571686556,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#1913L = 7)\"]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numDeletionVectorsAdded":"1","numDeletionVectorsRemoved":"1","numAddedChangeFiles":"0","executionTimeMs":"3376","numDeletionVectorsUpdated":"1","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"4a09810b-39c6-4b29-875e-3562816750dd"}} {"add":{"path":"part-00000-90177277-75c2-48db-92a2-20dcba39fd06-c000.snappy.parquet","partitionValues":{},"size":765,"modificationTime":1697571663000,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":49},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"j=hZPftg7qJYIw^L+Oz9","offset":1,"sizeInBytes":36,"cardinality":2}}} {"remove":{"path":"part-00000-90177277-75c2-48db-92a2-20dcba39fd06-c000.snappy.parquet","deletionTimestamp":1697571685983,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":765,"deletionVector":{"storageType":"u","pathOrInlineDv":"h{&8fAg]=QYJvl-}c!yH","offset":1,"sizeInBytes":34,"cardinality":1},"stats":"{\"numRecords\":50}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/log-replay-dv-key-cases/_delta_log/00000000000000000003.json ================================================ {"commitInfo":{"timestamp":1697571690963,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#3191L = 14)\"]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numDeletionVectorsAdded":"1","numDeletionVectorsRemoved":"1","numAddedChangeFiles":"0","executionTimeMs":"2740","numDeletionVectorsUpdated":"1","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.0.0-SNAPSHOT","txnId":"c2f503a1-043c-4bbc-af37-b6c2de878a01"}} {"add":{"path":"part-00000-90177277-75c2-48db-92a2-20dcba39fd06-c000.snappy.parquet","partitionValues":{},"size":765,"modificationTime":1697571663000,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":49},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"^jP?. spark .sql(query) .where(s"id = $v") .write .format("delta") .mode("append") .insertInto(tableName) } ================================================ FILE: connectors/golden-tables/src/main/resources/golden/table-with-columnmapping-mode-id/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1723094980674,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"6","numOutputBytes":"35024"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.3.0-SNAPSHOT","txnId":"ad640169-85ab-4030-ada7-b70661040863"}} {"metaData":{"id":"6fb2dbf2-52a9-4632-8952-71c976b4bf77","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"ByteType\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":1,\"delta.columnMapping.physicalName\":\"col-91e8b36f-2559-4aa7-b451-d30f24d5bf28\"}},{\"name\":\"ShortType\",\"type\":\"short\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":2,\"delta.columnMapping.physicalName\":\"col-d1a51f0a-855a-4752-8a35-76f52cda06b9\"}},{\"name\":\"IntegerType\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":3,\"delta.columnMapping.physicalName\":\"col-0aa7e907-848d-47b7-9805-e014c0a09d83\"}},{\"name\":\"LongType\",\"type\":\"long\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":4,\"delta.columnMapping.physicalName\":\"col-b55a2771-5f84-432a-9fe4-39e1ec1edfd7\"}},{\"name\":\"FloatType\",\"type\":\"float\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":5,\"delta.columnMapping.physicalName\":\"col-7722f52d-c253-4df9-96f4-a87a1dbef4dd\"}},{\"name\":\"DoubleType\",\"type\":\"double\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":6,\"delta.columnMapping.physicalName\":\"col-1dda278c-a501-4499-a580-f3dff1b79834\"}},{\"name\":\"decimal\",\"type\":\"decimal(10,2)\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":7,\"delta.columnMapping.physicalName\":\"col-30298583-3b87-4b4d-ab7c-f1102050f720\"}},{\"name\":\"BooleanType\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":8,\"delta.columnMapping.physicalName\":\"col-836e8d56-f73a-4203-9c0f-a50476468c2c\"}},{\"name\":\"StringType\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":9,\"delta.columnMapping.physicalName\":\"col-ca55bcbb-453c-433c-995f-80e0ccc13cde\"}},{\"name\":\"BinaryType\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":10,\"delta.columnMapping.physicalName\":\"col-da9f099c-f6bc-459c-a740-b02f658221e2\"}},{\"name\":\"DateType\",\"type\":\"date\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":11,\"delta.columnMapping.physicalName\":\"col-7139e00c-abc4-4434-893d-1cf11d2b68bf\"}},{\"name\":\"TimestampType\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":12,\"delta.columnMapping.physicalName\":\"col-292ce280-d844-4477-b078-87a64a6972d2\"}},{\"name\":\"nested_struct\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aa\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":14,\"delta.columnMapping.physicalName\":\"col-b47b5204-0100-46bc-92df-aa446f634191\"}},{\"name\":\"ac\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aca\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":16,\"delta.columnMapping.physicalName\":\"col-75cd857b-0152-438b-a0b3-43db0d4daa55\"}}]},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":15,\"delta.columnMapping.physicalName\":\"col-94e598c8-0480-4710-a288-fc332ed449de\"}}]},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":13,\"delta.columnMapping.physicalName\":\"col-ee8dc621-b36b-4784-bdb5-26c5c364e254\"}},{\"name\":\"array_of_prims\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":17,\"delta.columnMapping.physicalName\":\"col-720537b9-44e0-4829-a189-f3742da4f095\"}},{\"name\":\"array_of_arrays\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"containsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":18,\"delta.columnMapping.physicalName\":\"col-9f8d855c-cc39-426c-b579-f1b85b1b6991\"}},{\"name\":\"array_of_map_of_arrays\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"valueContainsNull\":true},\"containsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":19,\"delta.columnMapping.physicalName\":\"col-37f1b990-e228-4046-b98e-23934eb972b0\"}},{\"name\":\"array_of_structs\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"ab\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":21,\"delta.columnMapping.physicalName\":\"col-87e5a113-d6ee-40d5-b647-fadedfb84adb\"}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":20,\"delta.columnMapping.physicalName\":\"col-11bac662-f43a-44e3-b419-17b2ee3ee612\"}},{\"name\":\"struct_of_arrays_maps_of_structs\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aa\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":23,\"delta.columnMapping.physicalName\":\"col-c4278c59-01d8-4f81-be2e-9d3ead4ddc9e\"}},{\"name\":\"ab\",\"type\":{\"type\":\"map\",\"keyType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"valueType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aca\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":25,\"delta.columnMapping.physicalName\":\"col-56a3728f-7e4b-4c2f-ac2d-98f9c3d8031e\"}}]},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":24,\"delta.columnMapping.physicalName\":\"col-10aa4b27-4657-4ee4-a43a-ecf2f8fbb4d3\"}}]},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":22,\"delta.columnMapping.physicalName\":\"col-dbc4c186-5c6b-4309-82a2-fce1be10512f\"}},{\"name\":\"map_of_prims\",\"type\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":\"long\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":26,\"delta.columnMapping.physicalName\":\"col-ee9a73a7-e466-49ee-9a1f-69d01b60b88e\"}},{\"name\":\"map_of_rows\",\"type\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"ab\",\"type\":\"long\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":28,\"delta.columnMapping.physicalName\":\"col-7643d991-48d1-4d3d-a737-46be59e9b200\"}}]},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":27,\"delta.columnMapping.physicalName\":\"col-9697fd7e-5f66-43c0-9fee-3cf2bc9c2d4b\"}},{\"name\":\"map_of_arrays\",\"type\":{\"type\":\"map\",\"keyType\":\"long\",\"valueType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":29,\"delta.columnMapping.physicalName\":\"col-e146f9dd-9ce2-4dbb-8a91-54a393bab453\"}},{\"name\":\"map_of_maps\",\"type\":{\"type\":\"map\",\"keyType\":\"long\",\"valueType\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":\"integer\",\"valueContainsNull\":true},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":30,\"delta.columnMapping.physicalName\":\"col-a6fcf96f-53a6-4a16-9df1-59eb1f6e06bf\"}}]}","partitionColumns":[],"configuration":{"delta.columnMapping.mode":"id","delta.columnMapping.maxColumnId":"30"},"createdTime":1723094978338}} {"protocol":{"minReaderVersion":2,"minWriterVersion":5}} {"add":{"path":"part-00000-37fc7686-b5a9-432d-8cdc-8caa8cf999e5-c000.snappy.parquet","partitionValues":{},"size":17281,"modificationTime":1723094980594,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col-91e8b36f-2559-4aa7-b451-d30f24d5bf28\":0,\"col-d1a51f0a-855a-4752-8a35-76f52cda06b9\":0,\"col-0aa7e907-848d-47b7-9805-e014c0a09d83\":0,\"col-b55a2771-5f84-432a-9fe4-39e1ec1edfd7\":0,\"col-7722f52d-c253-4df9-96f4-a87a1dbef4dd\":0.0,\"col-1dda278c-a501-4499-a580-f3dff1b79834\":0.0,\"col-30298583-3b87-4b4d-ab7c-f1102050f720\":0.00,\"col-ca55bcbb-453c-433c-995f-80e0ccc13cde\":\"0\",\"col-7139e00c-abc4-4434-893d-1cf11d2b68bf\":\"2021-11-18\",\"col-292ce280-d844-4477-b078-87a64a6972d2\":\"1970-01-01T00:00:00.000Z\",\"col-ee8dc621-b36b-4784-bdb5-26c5c364e254\":{\"col-b47b5204-0100-46bc-92df-aa446f634191\":\"0\",\"col-94e598c8-0480-4710-a288-fc332ed449de\":{\"col-75cd857b-0152-438b-a0b3-43db0d4daa55\":0}}},\"maxValues\":{\"col-91e8b36f-2559-4aa7-b451-d30f24d5bf28\":4,\"col-d1a51f0a-855a-4752-8a35-76f52cda06b9\":4,\"col-0aa7e907-848d-47b7-9805-e014c0a09d83\":4,\"col-b55a2771-5f84-432a-9fe4-39e1ec1edfd7\":4,\"col-7722f52d-c253-4df9-96f4-a87a1dbef4dd\":4.0,\"col-1dda278c-a501-4499-a580-f3dff1b79834\":4.0,\"col-30298583-3b87-4b4d-ab7c-f1102050f720\":4.00,\"col-ca55bcbb-453c-433c-995f-80e0ccc13cde\":\"4\",\"col-7139e00c-abc4-4434-893d-1cf11d2b68bf\":\"2021-11-18\",\"col-292ce280-d844-4477-b078-87a64a6972d2\":\"1970-01-01T00:00:00.004Z\",\"col-ee8dc621-b36b-4784-bdb5-26c5c364e254\":{\"col-b47b5204-0100-46bc-92df-aa446f634191\":\"4\",\"col-94e598c8-0480-4710-a288-fc332ed449de\":{\"col-75cd857b-0152-438b-a0b3-43db0d4daa55\":4}}},\"nullCount\":{\"col-91e8b36f-2559-4aa7-b451-d30f24d5bf28\":0,\"col-d1a51f0a-855a-4752-8a35-76f52cda06b9\":0,\"col-0aa7e907-848d-47b7-9805-e014c0a09d83\":0,\"col-b55a2771-5f84-432a-9fe4-39e1ec1edfd7\":0,\"col-7722f52d-c253-4df9-96f4-a87a1dbef4dd\":0,\"col-1dda278c-a501-4499-a580-f3dff1b79834\":0,\"col-30298583-3b87-4b4d-ab7c-f1102050f720\":0,\"col-836e8d56-f73a-4203-9c0f-a50476468c2c\":0,\"col-ca55bcbb-453c-433c-995f-80e0ccc13cde\":0,\"col-da9f099c-f6bc-459c-a740-b02f658221e2\":0,\"col-7139e00c-abc4-4434-893d-1cf11d2b68bf\":0,\"col-292ce280-d844-4477-b078-87a64a6972d2\":0,\"col-ee8dc621-b36b-4784-bdb5-26c5c364e254\":{\"col-b47b5204-0100-46bc-92df-aa446f634191\":0,\"col-94e598c8-0480-4710-a288-fc332ed449de\":{\"col-75cd857b-0152-438b-a0b3-43db0d4daa55\":0}},\"col-720537b9-44e0-4829-a189-f3742da4f095\":0,\"col-9f8d855c-cc39-426c-b579-f1b85b1b6991\":0,\"col-37f1b990-e228-4046-b98e-23934eb972b0\":0,\"col-11bac662-f43a-44e3-b419-17b2ee3ee612\":0,\"col-dbc4c186-5c6b-4309-82a2-fce1be10512f\":{\"col-c4278c59-01d8-4f81-be2e-9d3ead4ddc9e\":0,\"col-10aa4b27-4657-4ee4-a43a-ecf2f8fbb4d3\":0},\"col-ee9a73a7-e466-49ee-9a1f-69d01b60b88e\":0,\"col-9697fd7e-5f66-43c0-9fee-3cf2bc9c2d4b\":0,\"col-e146f9dd-9ce2-4dbb-8a91-54a393bab453\":0,\"col-a6fcf96f-53a6-4a16-9df1-59eb1f6e06bf\":0}}"}} {"add":{"path":"part-00001-0321adc4-f601-4c9d-bb7c-a0ddf759c7b2-c000.snappy.parquet","partitionValues":{},"size":17743,"modificationTime":1723094980594,"dataChange":true,"stats":"{\"numRecords\":4,\"minValues\":{\"col-91e8b36f-2559-4aa7-b451-d30f24d5bf28\":1,\"col-d1a51f0a-855a-4752-8a35-76f52cda06b9\":1,\"col-0aa7e907-848d-47b7-9805-e014c0a09d83\":1,\"col-b55a2771-5f84-432a-9fe4-39e1ec1edfd7\":1,\"col-7722f52d-c253-4df9-96f4-a87a1dbef4dd\":1.0,\"col-1dda278c-a501-4499-a580-f3dff1b79834\":1.0,\"col-30298583-3b87-4b4d-ab7c-f1102050f720\":1.00,\"col-ca55bcbb-453c-433c-995f-80e0ccc13cde\":\"1\",\"col-7139e00c-abc4-4434-893d-1cf11d2b68bf\":\"2021-11-18\",\"col-292ce280-d844-4477-b078-87a64a6972d2\":\"1970-01-01T00:00:00.001Z\",\"col-ee8dc621-b36b-4784-bdb5-26c5c364e254\":{\"col-b47b5204-0100-46bc-92df-aa446f634191\":\"1\",\"col-94e598c8-0480-4710-a288-fc332ed449de\":{\"col-75cd857b-0152-438b-a0b3-43db0d4daa55\":1}}},\"maxValues\":{\"col-91e8b36f-2559-4aa7-b451-d30f24d5bf28\":3,\"col-d1a51f0a-855a-4752-8a35-76f52cda06b9\":3,\"col-0aa7e907-848d-47b7-9805-e014c0a09d83\":3,\"col-b55a2771-5f84-432a-9fe4-39e1ec1edfd7\":3,\"col-7722f52d-c253-4df9-96f4-a87a1dbef4dd\":3.0,\"col-1dda278c-a501-4499-a580-f3dff1b79834\":3.0,\"col-30298583-3b87-4b4d-ab7c-f1102050f720\":3.00,\"col-ca55bcbb-453c-433c-995f-80e0ccc13cde\":\"3\",\"col-7139e00c-abc4-4434-893d-1cf11d2b68bf\":\"2021-11-18\",\"col-292ce280-d844-4477-b078-87a64a6972d2\":\"1970-01-01T00:00:00.003Z\",\"col-ee8dc621-b36b-4784-bdb5-26c5c364e254\":{\"col-b47b5204-0100-46bc-92df-aa446f634191\":\"3\",\"col-94e598c8-0480-4710-a288-fc332ed449de\":{\"col-75cd857b-0152-438b-a0b3-43db0d4daa55\":3}}},\"nullCount\":{\"col-91e8b36f-2559-4aa7-b451-d30f24d5bf28\":1,\"col-d1a51f0a-855a-4752-8a35-76f52cda06b9\":1,\"col-0aa7e907-848d-47b7-9805-e014c0a09d83\":1,\"col-b55a2771-5f84-432a-9fe4-39e1ec1edfd7\":1,\"col-7722f52d-c253-4df9-96f4-a87a1dbef4dd\":1,\"col-1dda278c-a501-4499-a580-f3dff1b79834\":1,\"col-30298583-3b87-4b4d-ab7c-f1102050f720\":1,\"col-836e8d56-f73a-4203-9c0f-a50476468c2c\":1,\"col-ca55bcbb-453c-433c-995f-80e0ccc13cde\":1,\"col-da9f099c-f6bc-459c-a740-b02f658221e2\":1,\"col-7139e00c-abc4-4434-893d-1cf11d2b68bf\":1,\"col-292ce280-d844-4477-b078-87a64a6972d2\":1,\"col-ee8dc621-b36b-4784-bdb5-26c5c364e254\":{\"col-b47b5204-0100-46bc-92df-aa446f634191\":1,\"col-94e598c8-0480-4710-a288-fc332ed449de\":{\"col-75cd857b-0152-438b-a0b3-43db0d4daa55\":1}},\"col-720537b9-44e0-4829-a189-f3742da4f095\":1,\"col-9f8d855c-cc39-426c-b579-f1b85b1b6991\":1,\"col-37f1b990-e228-4046-b98e-23934eb972b0\":1,\"col-11bac662-f43a-44e3-b419-17b2ee3ee612\":1,\"col-dbc4c186-5c6b-4309-82a2-fce1be10512f\":{\"col-c4278c59-01d8-4f81-be2e-9d3ead4ddc9e\":1,\"col-10aa4b27-4657-4ee4-a43a-ecf2f8fbb4d3\":1},\"col-ee9a73a7-e466-49ee-9a1f-69d01b60b88e\":1,\"col-9697fd7e-5f66-43c0-9fee-3cf2bc9c2d4b\":1,\"col-e146f9dd-9ce2-4dbb-8a91-54a393bab453\":1,\"col-a6fcf96f-53a6-4a16-9df1-59eb1f6e06bf\":1}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/table-with-columnmapping-mode-name/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1723094941755,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"6","numOutputBytes":"35024"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.3.0-SNAPSHOT","txnId":"244f65e5-5ecb-4563-a425-bf680f1c0546"}} {"metaData":{"id":"5c372fe7-d0fd-48b9-ae93-5fc346eea359","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"ByteType\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":1,\"delta.columnMapping.physicalName\":\"col-33314a5e-7dc1-438a-8f4d-df8c417071d6\"}},{\"name\":\"ShortType\",\"type\":\"short\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":2,\"delta.columnMapping.physicalName\":\"col-7237d244-656d-402c-a889-5540aa23d418\"}},{\"name\":\"IntegerType\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":3,\"delta.columnMapping.physicalName\":\"col-267caf03-cf2f-450d-a6ee-5dbe81c86497\"}},{\"name\":\"LongType\",\"type\":\"long\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":4,\"delta.columnMapping.physicalName\":\"col-f92689f0-399a-46e5-84b6-604670849d66\"}},{\"name\":\"FloatType\",\"type\":\"float\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":5,\"delta.columnMapping.physicalName\":\"col-9bd8f339-6e15-4ffb-8b53-ed3efe8e3482\"}},{\"name\":\"DoubleType\",\"type\":\"double\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":6,\"delta.columnMapping.physicalName\":\"col-d3069408-bcd3-4bf4-a125-166cfd56841f\"}},{\"name\":\"decimal\",\"type\":\"decimal(10,2)\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":7,\"delta.columnMapping.physicalName\":\"col-d2821ced-9890-4ac8-b196-4649c82547f4\"}},{\"name\":\"BooleanType\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":8,\"delta.columnMapping.physicalName\":\"col-a9fd703e-4984-4977-a0ac-72f2f880095d\"}},{\"name\":\"StringType\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":9,\"delta.columnMapping.physicalName\":\"col-7f007ffd-c4e2-4d7e-84c2-af985d4e3feb\"}},{\"name\":\"BinaryType\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":10,\"delta.columnMapping.physicalName\":\"col-05929041-8bb8-4db8-a4b3-0744cbef8116\"}},{\"name\":\"DateType\",\"type\":\"date\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":11,\"delta.columnMapping.physicalName\":\"col-af3ec5e1-41cc-4ccd-ad38-b094fb567ec5\"}},{\"name\":\"TimestampType\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":12,\"delta.columnMapping.physicalName\":\"col-50ab9e52-7811-4057-96b0-cf61d5be358e\"}},{\"name\":\"nested_struct\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aa\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":14,\"delta.columnMapping.physicalName\":\"col-7295e744-4c2c-48a1-baf5-3bb131ec68ea\"}},{\"name\":\"ac\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aca\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":16,\"delta.columnMapping.physicalName\":\"col-562033a2-86d8-4eb3-83e6-87eb2f27314f\"}}]},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":15,\"delta.columnMapping.physicalName\":\"col-15ecbe5f-906d-4d64-a627-eb16eb4b4410\"}}]},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":13,\"delta.columnMapping.physicalName\":\"col-e08b8668-6c8f-4081-aa77-357325212630\"}},{\"name\":\"array_of_prims\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":17,\"delta.columnMapping.physicalName\":\"col-98146e9e-2472-4c0f-b1d5-a01e9a41bc46\"}},{\"name\":\"array_of_arrays\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"containsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":18,\"delta.columnMapping.physicalName\":\"col-1b6de8ee-deb0-43b4-8c94-bc9f7a556da8\"}},{\"name\":\"array_of_map_of_arrays\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"valueContainsNull\":true},\"containsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":19,\"delta.columnMapping.physicalName\":\"col-ad0356f5-43c1-401c-b291-6a5d211bc63c\"}},{\"name\":\"array_of_structs\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"ab\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":21,\"delta.columnMapping.physicalName\":\"col-93cac05c-2edd-409e-83f1-cebfd09bfc75\"}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":20,\"delta.columnMapping.physicalName\":\"col-5d038bad-e2b1-4fe7-a444-22fc3ac6e52a\"}},{\"name\":\"struct_of_arrays_maps_of_structs\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aa\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":23,\"delta.columnMapping.physicalName\":\"col-fae23ffb-8aaa-43b7-982b-0930667ddae0\"}},{\"name\":\"ab\",\"type\":{\"type\":\"map\",\"keyType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"valueType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aca\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":25,\"delta.columnMapping.physicalName\":\"col-6cb288b5-d45e-46aa-98be-e2aa06665232\"}}]},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":24,\"delta.columnMapping.physicalName\":\"col-b816c867-4d4f-402f-85cf-786b58d14d05\"}}]},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":22,\"delta.columnMapping.physicalName\":\"col-d4888d0e-973f-486e-b345-d69a53dd193c\"}},{\"name\":\"map_of_prims\",\"type\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":\"long\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":26,\"delta.columnMapping.physicalName\":\"col-ccb0e770-7cce-4293-88bb-d98d74883f21\"}},{\"name\":\"map_of_rows\",\"type\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"ab\",\"type\":\"long\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":28,\"delta.columnMapping.physicalName\":\"col-d8438cc9-31ef-4b23-ac1c-1dff3e0a5530\"}}]},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":27,\"delta.columnMapping.physicalName\":\"col-5e848aab-e51a-46f0-8650-eee8f46f1149\"}},{\"name\":\"map_of_arrays\",\"type\":{\"type\":\"map\",\"keyType\":\"long\",\"valueType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":29,\"delta.columnMapping.physicalName\":\"col-990f4ddc-316c-4548-83e2-3e1d7868706c\"}},{\"name\":\"map_of_maps\",\"type\":{\"type\":\"map\",\"keyType\":\"long\",\"valueType\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":\"integer\",\"valueContainsNull\":true},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":30,\"delta.columnMapping.physicalName\":\"col-cca40ef1-caeb-4e56-ab0c-0d68b63f6785\"}}]}","partitionColumns":[],"configuration":{"delta.columnMapping.mode":"name","delta.columnMapping.maxColumnId":"30"},"createdTime":1723094939386}} {"protocol":{"minReaderVersion":2,"minWriterVersion":5}} {"add":{"path":"part-00000-2887cf52-61be-4009-afba-00b218602665-c000.snappy.parquet","partitionValues":{},"size":17281,"modificationTime":1723094941664,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col-33314a5e-7dc1-438a-8f4d-df8c417071d6\":0,\"col-7237d244-656d-402c-a889-5540aa23d418\":0,\"col-267caf03-cf2f-450d-a6ee-5dbe81c86497\":0,\"col-f92689f0-399a-46e5-84b6-604670849d66\":0,\"col-9bd8f339-6e15-4ffb-8b53-ed3efe8e3482\":0.0,\"col-d3069408-bcd3-4bf4-a125-166cfd56841f\":0.0,\"col-d2821ced-9890-4ac8-b196-4649c82547f4\":0.00,\"col-7f007ffd-c4e2-4d7e-84c2-af985d4e3feb\":\"0\",\"col-af3ec5e1-41cc-4ccd-ad38-b094fb567ec5\":\"2021-11-18\",\"col-50ab9e52-7811-4057-96b0-cf61d5be358e\":\"1970-01-01T00:00:00.000Z\",\"col-e08b8668-6c8f-4081-aa77-357325212630\":{\"col-7295e744-4c2c-48a1-baf5-3bb131ec68ea\":\"0\",\"col-15ecbe5f-906d-4d64-a627-eb16eb4b4410\":{\"col-562033a2-86d8-4eb3-83e6-87eb2f27314f\":0}}},\"maxValues\":{\"col-33314a5e-7dc1-438a-8f4d-df8c417071d6\":4,\"col-7237d244-656d-402c-a889-5540aa23d418\":4,\"col-267caf03-cf2f-450d-a6ee-5dbe81c86497\":4,\"col-f92689f0-399a-46e5-84b6-604670849d66\":4,\"col-9bd8f339-6e15-4ffb-8b53-ed3efe8e3482\":4.0,\"col-d3069408-bcd3-4bf4-a125-166cfd56841f\":4.0,\"col-d2821ced-9890-4ac8-b196-4649c82547f4\":4.00,\"col-7f007ffd-c4e2-4d7e-84c2-af985d4e3feb\":\"4\",\"col-af3ec5e1-41cc-4ccd-ad38-b094fb567ec5\":\"2021-11-18\",\"col-50ab9e52-7811-4057-96b0-cf61d5be358e\":\"1970-01-01T00:00:00.004Z\",\"col-e08b8668-6c8f-4081-aa77-357325212630\":{\"col-7295e744-4c2c-48a1-baf5-3bb131ec68ea\":\"4\",\"col-15ecbe5f-906d-4d64-a627-eb16eb4b4410\":{\"col-562033a2-86d8-4eb3-83e6-87eb2f27314f\":4}}},\"nullCount\":{\"col-33314a5e-7dc1-438a-8f4d-df8c417071d6\":0,\"col-7237d244-656d-402c-a889-5540aa23d418\":0,\"col-267caf03-cf2f-450d-a6ee-5dbe81c86497\":0,\"col-f92689f0-399a-46e5-84b6-604670849d66\":0,\"col-9bd8f339-6e15-4ffb-8b53-ed3efe8e3482\":0,\"col-d3069408-bcd3-4bf4-a125-166cfd56841f\":0,\"col-d2821ced-9890-4ac8-b196-4649c82547f4\":0,\"col-a9fd703e-4984-4977-a0ac-72f2f880095d\":0,\"col-7f007ffd-c4e2-4d7e-84c2-af985d4e3feb\":0,\"col-05929041-8bb8-4db8-a4b3-0744cbef8116\":0,\"col-af3ec5e1-41cc-4ccd-ad38-b094fb567ec5\":0,\"col-50ab9e52-7811-4057-96b0-cf61d5be358e\":0,\"col-e08b8668-6c8f-4081-aa77-357325212630\":{\"col-7295e744-4c2c-48a1-baf5-3bb131ec68ea\":0,\"col-15ecbe5f-906d-4d64-a627-eb16eb4b4410\":{\"col-562033a2-86d8-4eb3-83e6-87eb2f27314f\":0}},\"col-98146e9e-2472-4c0f-b1d5-a01e9a41bc46\":0,\"col-1b6de8ee-deb0-43b4-8c94-bc9f7a556da8\":0,\"col-ad0356f5-43c1-401c-b291-6a5d211bc63c\":0,\"col-5d038bad-e2b1-4fe7-a444-22fc3ac6e52a\":0,\"col-d4888d0e-973f-486e-b345-d69a53dd193c\":{\"col-fae23ffb-8aaa-43b7-982b-0930667ddae0\":0,\"col-b816c867-4d4f-402f-85cf-786b58d14d05\":0},\"col-ccb0e770-7cce-4293-88bb-d98d74883f21\":0,\"col-5e848aab-e51a-46f0-8650-eee8f46f1149\":0,\"col-990f4ddc-316c-4548-83e2-3e1d7868706c\":0,\"col-cca40ef1-caeb-4e56-ab0c-0d68b63f6785\":0}}"}} {"add":{"path":"part-00001-b664b3db-62d8-4e02-9dc5-26dbce3abfc1-c000.snappy.parquet","partitionValues":{},"size":17743,"modificationTime":1723094941664,"dataChange":true,"stats":"{\"numRecords\":4,\"minValues\":{\"col-33314a5e-7dc1-438a-8f4d-df8c417071d6\":1,\"col-7237d244-656d-402c-a889-5540aa23d418\":1,\"col-267caf03-cf2f-450d-a6ee-5dbe81c86497\":1,\"col-f92689f0-399a-46e5-84b6-604670849d66\":1,\"col-9bd8f339-6e15-4ffb-8b53-ed3efe8e3482\":1.0,\"col-d3069408-bcd3-4bf4-a125-166cfd56841f\":1.0,\"col-d2821ced-9890-4ac8-b196-4649c82547f4\":1.00,\"col-7f007ffd-c4e2-4d7e-84c2-af985d4e3feb\":\"1\",\"col-af3ec5e1-41cc-4ccd-ad38-b094fb567ec5\":\"2021-11-18\",\"col-50ab9e52-7811-4057-96b0-cf61d5be358e\":\"1970-01-01T00:00:00.001Z\",\"col-e08b8668-6c8f-4081-aa77-357325212630\":{\"col-7295e744-4c2c-48a1-baf5-3bb131ec68ea\":\"1\",\"col-15ecbe5f-906d-4d64-a627-eb16eb4b4410\":{\"col-562033a2-86d8-4eb3-83e6-87eb2f27314f\":1}}},\"maxValues\":{\"col-33314a5e-7dc1-438a-8f4d-df8c417071d6\":3,\"col-7237d244-656d-402c-a889-5540aa23d418\":3,\"col-267caf03-cf2f-450d-a6ee-5dbe81c86497\":3,\"col-f92689f0-399a-46e5-84b6-604670849d66\":3,\"col-9bd8f339-6e15-4ffb-8b53-ed3efe8e3482\":3.0,\"col-d3069408-bcd3-4bf4-a125-166cfd56841f\":3.0,\"col-d2821ced-9890-4ac8-b196-4649c82547f4\":3.00,\"col-7f007ffd-c4e2-4d7e-84c2-af985d4e3feb\":\"3\",\"col-af3ec5e1-41cc-4ccd-ad38-b094fb567ec5\":\"2021-11-18\",\"col-50ab9e52-7811-4057-96b0-cf61d5be358e\":\"1970-01-01T00:00:00.003Z\",\"col-e08b8668-6c8f-4081-aa77-357325212630\":{\"col-7295e744-4c2c-48a1-baf5-3bb131ec68ea\":\"3\",\"col-15ecbe5f-906d-4d64-a627-eb16eb4b4410\":{\"col-562033a2-86d8-4eb3-83e6-87eb2f27314f\":3}}},\"nullCount\":{\"col-33314a5e-7dc1-438a-8f4d-df8c417071d6\":1,\"col-7237d244-656d-402c-a889-5540aa23d418\":1,\"col-267caf03-cf2f-450d-a6ee-5dbe81c86497\":1,\"col-f92689f0-399a-46e5-84b6-604670849d66\":1,\"col-9bd8f339-6e15-4ffb-8b53-ed3efe8e3482\":1,\"col-d3069408-bcd3-4bf4-a125-166cfd56841f\":1,\"col-d2821ced-9890-4ac8-b196-4649c82547f4\":1,\"col-a9fd703e-4984-4977-a0ac-72f2f880095d\":1,\"col-7f007ffd-c4e2-4d7e-84c2-af985d4e3feb\":1,\"col-05929041-8bb8-4db8-a4b3-0744cbef8116\":1,\"col-af3ec5e1-41cc-4ccd-ad38-b094fb567ec5\":1,\"col-50ab9e52-7811-4057-96b0-cf61d5be358e\":1,\"col-e08b8668-6c8f-4081-aa77-357325212630\":{\"col-7295e744-4c2c-48a1-baf5-3bb131ec68ea\":1,\"col-15ecbe5f-906d-4d64-a627-eb16eb4b4410\":{\"col-562033a2-86d8-4eb3-83e6-87eb2f27314f\":1}},\"col-98146e9e-2472-4c0f-b1d5-a01e9a41bc46\":1,\"col-1b6de8ee-deb0-43b4-8c94-bc9f7a556da8\":1,\"col-ad0356f5-43c1-401c-b291-6a5d211bc63c\":1,\"col-5d038bad-e2b1-4fe7-a444-22fc3ac6e52a\":1,\"col-d4888d0e-973f-486e-b345-d69a53dd193c\":{\"col-fae23ffb-8aaa-43b7-982b-0930667ddae0\":1,\"col-b816c867-4d4f-402f-85cf-786b58d14d05\":1},\"col-ccb0e770-7cce-4293-88bb-d98d74883f21\":1,\"col-5e848aab-e51a-46f0-8650-eee8f46f1149\":1,\"col-990f4ddc-316c-4548-83e2-3e1d7868706c\":1,\"col-cca40ef1-caeb-4e56-ab0c-0d68b63f6785\":1}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/table-with-icebegCompatV2Enabled/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1723094912799,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"6","numOutputBytes":"44076"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.3.0-SNAPSHOT","txnId":"9e32df60-b1a0-4229-9820-bd9b56ccb304"}} {"metaData":{"id":"5d389c1e-778b-45c2-b1d9-01b7e60f63ec","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"ByteType\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":1,\"delta.columnMapping.nested.ids\":{},\"delta.columnMapping.physicalName\":\"col-150baa08-2441-4182-b1d2-6c33e9b2f6e6\"}},{\"name\":\"ShortType\",\"type\":\"short\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":2,\"delta.columnMapping.nested.ids\":{},\"delta.columnMapping.physicalName\":\"col-aaf5094f-3435-407f-bc7c-8d5cbd2bce22\"}},{\"name\":\"IntegerType\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":3,\"delta.columnMapping.nested.ids\":{},\"delta.columnMapping.physicalName\":\"col-a9123e30-97e4-428e-a179-36af4908b4f3\"}},{\"name\":\"LongType\",\"type\":\"long\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":4,\"delta.columnMapping.nested.ids\":{},\"delta.columnMapping.physicalName\":\"col-6a302d0e-d156-40e2-93b0-a26ee0a54eb4\"}},{\"name\":\"FloatType\",\"type\":\"float\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":5,\"delta.columnMapping.nested.ids\":{},\"delta.columnMapping.physicalName\":\"col-34f01baf-8cff-4abc-bf78-0494ba88070d\"}},{\"name\":\"DoubleType\",\"type\":\"double\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":6,\"delta.columnMapping.nested.ids\":{},\"delta.columnMapping.physicalName\":\"col-0ef81992-71a5-47da-9c4c-45c76084bd54\"}},{\"name\":\"decimal\",\"type\":\"decimal(10,2)\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":7,\"delta.columnMapping.nested.ids\":{},\"delta.columnMapping.physicalName\":\"col-2c4cb239-3129-476c-8c0b-8e1f9ef575b8\"}},{\"name\":\"BooleanType\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":8,\"delta.columnMapping.nested.ids\":{},\"delta.columnMapping.physicalName\":\"col-8dfa7127-0477-4ddf-9ce0-87baae1ca166\"}},{\"name\":\"StringType\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":9,\"delta.columnMapping.nested.ids\":{},\"delta.columnMapping.physicalName\":\"col-d588bf1d-e372-4f26-8572-5dd32730bc20\"}},{\"name\":\"BinaryType\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":10,\"delta.columnMapping.nested.ids\":{},\"delta.columnMapping.physicalName\":\"col-14d8b433-13d3-40b4-85bf-698b93a15edd\"}},{\"name\":\"DateType\",\"type\":\"date\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":11,\"delta.columnMapping.nested.ids\":{},\"delta.columnMapping.physicalName\":\"col-43df1513-711d-42c0-ae37-1988c0c7478f\"}},{\"name\":\"TimestampType\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":12,\"delta.columnMapping.nested.ids\":{},\"delta.columnMapping.physicalName\":\"col-1a000afa-7d0f-4822-85fc-6285cf51e460\"}},{\"name\":\"nested_struct\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aa\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":14,\"delta.columnMapping.nested.ids\":{},\"delta.columnMapping.physicalName\":\"col-c0d2b42d-365a-40ce-ac1d-e33605b4b595\"}},{\"name\":\"ac\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aca\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":16,\"delta.columnMapping.nested.ids\":{},\"delta.columnMapping.physicalName\":\"col-99fc36e3-1691-4d36-bc4b-91f63779d1c2\"}}]},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":15,\"delta.columnMapping.nested.ids\":{},\"delta.columnMapping.physicalName\":\"col-d2e13fb9-c6ae-4483-a580-7a27b4226700\"}}]},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":13,\"delta.columnMapping.nested.ids\":{},\"delta.columnMapping.physicalName\":\"col-8219db36-f0b0-4529-ac76-2e0da218668c\"}},{\"name\":\"array_of_prims\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":17,\"delta.columnMapping.nested.ids\":{\"col-9c4dbf49-152b-4f5d-8dfb-3a8884f78cb7.element\":31},\"delta.columnMapping.physicalName\":\"col-9c4dbf49-152b-4f5d-8dfb-3a8884f78cb7\"}},{\"name\":\"array_of_arrays\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"containsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":18,\"delta.columnMapping.nested.ids\":{\"col-77d17ccf-6d7d-4129-a93a-b6d4fd9c3483.element.element\":33,\"col-77d17ccf-6d7d-4129-a93a-b6d4fd9c3483.element\":32},\"delta.columnMapping.physicalName\":\"col-77d17ccf-6d7d-4129-a93a-b6d4fd9c3483\"}},{\"name\":\"array_of_map_of_arrays\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"valueContainsNull\":true},\"containsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":19,\"delta.columnMapping.nested.ids\":{\"col-1755e9a3-fa11-4062-8092-fe8f1664be80.element.value.element\":37,\"col-1755e9a3-fa11-4062-8092-fe8f1664be80.element.value\":36,\"col-1755e9a3-fa11-4062-8092-fe8f1664be80.element.key\":35,\"col-1755e9a3-fa11-4062-8092-fe8f1664be80.element\":34},\"delta.columnMapping.physicalName\":\"col-1755e9a3-fa11-4062-8092-fe8f1664be80\"}},{\"name\":\"array_of_structs\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"ab\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":21,\"delta.columnMapping.nested.ids\":{},\"delta.columnMapping.physicalName\":\"col-f52ebe0f-00e0-4602-a142-192ec849afa2\"}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":20,\"delta.columnMapping.nested.ids\":{\"col-ef476911-2c9b-49c5-8c76-7fc03c8953f8.element\":38},\"delta.columnMapping.physicalName\":\"col-ef476911-2c9b-49c5-8c76-7fc03c8953f8\"}},{\"name\":\"struct_of_arrays_maps_of_structs\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aa\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":23,\"delta.columnMapping.nested.ids\":{\"col-53e9356f-7271-4a56-8586-77976034c213.element\":39},\"delta.columnMapping.physicalName\":\"col-53e9356f-7271-4a56-8586-77976034c213\"}},{\"name\":\"ab\",\"type\":{\"type\":\"map\",\"keyType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"valueType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aca\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":25,\"delta.columnMapping.nested.ids\":{},\"delta.columnMapping.physicalName\":\"col-4cb3734b-e335-4aab-9d8c-decacaf891fc\"}}]},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":24,\"delta.columnMapping.nested.ids\":{\"col-70da66c1-651b-4562-908b-330e91ee6db2.key\":40,\"col-70da66c1-651b-4562-908b-330e91ee6db2.value\":42,\"col-70da66c1-651b-4562-908b-330e91ee6db2.key.element\":41},\"delta.columnMapping.physicalName\":\"col-70da66c1-651b-4562-908b-330e91ee6db2\"}}]},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":22,\"delta.columnMapping.nested.ids\":{},\"delta.columnMapping.physicalName\":\"col-7431b180-3300-4771-a556-693c9be39683\"}},{\"name\":\"map_of_prims\",\"type\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":\"long\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":26,\"delta.columnMapping.nested.ids\":{\"col-c5aff22c-e8de-4618-8318-31386aa7721f.key\":43,\"col-c5aff22c-e8de-4618-8318-31386aa7721f.value\":44},\"delta.columnMapping.physicalName\":\"col-c5aff22c-e8de-4618-8318-31386aa7721f\"}},{\"name\":\"map_of_rows\",\"type\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"ab\",\"type\":\"long\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":28,\"delta.columnMapping.nested.ids\":{},\"delta.columnMapping.physicalName\":\"col-be842636-62e7-4d96-b26c-64081616556b\"}}]},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":27,\"delta.columnMapping.nested.ids\":{\"col-8701e73b-45d6-4b23-87ac-c871996ad7a9.key\":45,\"col-8701e73b-45d6-4b23-87ac-c871996ad7a9.value\":46},\"delta.columnMapping.physicalName\":\"col-8701e73b-45d6-4b23-87ac-c871996ad7a9\"}},{\"name\":\"map_of_arrays\",\"type\":{\"type\":\"map\",\"keyType\":\"long\",\"valueType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":29,\"delta.columnMapping.nested.ids\":{\"col-8e1bb9a3-0c4d-4f5c-b95f-568c0703c77f.value\":48,\"col-8e1bb9a3-0c4d-4f5c-b95f-568c0703c77f.value.element\":49,\"col-8e1bb9a3-0c4d-4f5c-b95f-568c0703c77f.key\":47},\"delta.columnMapping.physicalName\":\"col-8e1bb9a3-0c4d-4f5c-b95f-568c0703c77f\"}},{\"name\":\"map_of_maps\",\"type\":{\"type\":\"map\",\"keyType\":\"long\",\"valueType\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":\"integer\",\"valueContainsNull\":true},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":30,\"delta.columnMapping.nested.ids\":{\"col-9da911c6-054c-4167-881e-4c03ce7a3fd1.value.key\":52,\"col-9da911c6-054c-4167-881e-4c03ce7a3fd1.key\":50,\"col-9da911c6-054c-4167-881e-4c03ce7a3fd1.value.value\":53,\"col-9da911c6-054c-4167-881e-4c03ce7a3fd1.value\":51},\"delta.columnMapping.physicalName\":\"col-9da911c6-054c-4167-881e-4c03ce7a3fd1\"}}]}","partitionColumns":[],"configuration":{"delta.enableIcebergCompatV2":"true","delta.columnMapping.mode":"id","delta.columnMapping.maxColumnId":"53"},"createdTime":1723094910279}} {"protocol":{"minReaderVersion":2,"minWriterVersion":7,"writerFeatures":["columnMapping","icebergCompatV2","appendOnly","invariants"]}} {"add":{"path":"part-00000-cbb3f19e-57e0-4922-a6c3-f211a65d918f-c000.snappy.parquet","partitionValues":{},"size":21807,"modificationTime":1723094912648,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"col-150baa08-2441-4182-b1d2-6c33e9b2f6e6\":0,\"col-aaf5094f-3435-407f-bc7c-8d5cbd2bce22\":0,\"col-a9123e30-97e4-428e-a179-36af4908b4f3\":0,\"col-6a302d0e-d156-40e2-93b0-a26ee0a54eb4\":0,\"col-34f01baf-8cff-4abc-bf78-0494ba88070d\":0.0,\"col-0ef81992-71a5-47da-9c4c-45c76084bd54\":0.0,\"col-2c4cb239-3129-476c-8c0b-8e1f9ef575b8\":0.00,\"col-d588bf1d-e372-4f26-8572-5dd32730bc20\":\"0\",\"col-43df1513-711d-42c0-ae37-1988c0c7478f\":\"2021-11-18\",\"col-1a000afa-7d0f-4822-85fc-6285cf51e460\":\"1970-01-01T00:00:00.000Z\",\"col-8219db36-f0b0-4529-ac76-2e0da218668c\":{\"col-c0d2b42d-365a-40ce-ac1d-e33605b4b595\":\"0\",\"col-d2e13fb9-c6ae-4483-a580-7a27b4226700\":{\"col-99fc36e3-1691-4d36-bc4b-91f63779d1c2\":0}}},\"maxValues\":{\"col-150baa08-2441-4182-b1d2-6c33e9b2f6e6\":4,\"col-aaf5094f-3435-407f-bc7c-8d5cbd2bce22\":4,\"col-a9123e30-97e4-428e-a179-36af4908b4f3\":4,\"col-6a302d0e-d156-40e2-93b0-a26ee0a54eb4\":4,\"col-34f01baf-8cff-4abc-bf78-0494ba88070d\":4.0,\"col-0ef81992-71a5-47da-9c4c-45c76084bd54\":4.0,\"col-2c4cb239-3129-476c-8c0b-8e1f9ef575b8\":4.00,\"col-d588bf1d-e372-4f26-8572-5dd32730bc20\":\"4\",\"col-43df1513-711d-42c0-ae37-1988c0c7478f\":\"2021-11-18\",\"col-1a000afa-7d0f-4822-85fc-6285cf51e460\":\"1970-01-01T00:00:00.004Z\",\"col-8219db36-f0b0-4529-ac76-2e0da218668c\":{\"col-c0d2b42d-365a-40ce-ac1d-e33605b4b595\":\"4\",\"col-d2e13fb9-c6ae-4483-a580-7a27b4226700\":{\"col-99fc36e3-1691-4d36-bc4b-91f63779d1c2\":4}}},\"nullCount\":{\"col-150baa08-2441-4182-b1d2-6c33e9b2f6e6\":0,\"col-aaf5094f-3435-407f-bc7c-8d5cbd2bce22\":0,\"col-a9123e30-97e4-428e-a179-36af4908b4f3\":0,\"col-6a302d0e-d156-40e2-93b0-a26ee0a54eb4\":0,\"col-34f01baf-8cff-4abc-bf78-0494ba88070d\":0,\"col-0ef81992-71a5-47da-9c4c-45c76084bd54\":0,\"col-2c4cb239-3129-476c-8c0b-8e1f9ef575b8\":0,\"col-8dfa7127-0477-4ddf-9ce0-87baae1ca166\":0,\"col-d588bf1d-e372-4f26-8572-5dd32730bc20\":0,\"col-14d8b433-13d3-40b4-85bf-698b93a15edd\":0,\"col-43df1513-711d-42c0-ae37-1988c0c7478f\":0,\"col-1a000afa-7d0f-4822-85fc-6285cf51e460\":0,\"col-8219db36-f0b0-4529-ac76-2e0da218668c\":{\"col-c0d2b42d-365a-40ce-ac1d-e33605b4b595\":0,\"col-d2e13fb9-c6ae-4483-a580-7a27b4226700\":{\"col-99fc36e3-1691-4d36-bc4b-91f63779d1c2\":0}},\"col-9c4dbf49-152b-4f5d-8dfb-3a8884f78cb7\":0,\"col-77d17ccf-6d7d-4129-a93a-b6d4fd9c3483\":0,\"col-1755e9a3-fa11-4062-8092-fe8f1664be80\":0,\"col-ef476911-2c9b-49c5-8c76-7fc03c8953f8\":0,\"col-7431b180-3300-4771-a556-693c9be39683\":{\"col-53e9356f-7271-4a56-8586-77976034c213\":0,\"col-70da66c1-651b-4562-908b-330e91ee6db2\":0},\"col-c5aff22c-e8de-4618-8318-31386aa7721f\":0,\"col-8701e73b-45d6-4b23-87ac-c871996ad7a9\":0,\"col-8e1bb9a3-0c4d-4f5c-b95f-568c0703c77f\":0,\"col-9da911c6-054c-4167-881e-4c03ce7a3fd1\":0}}","tags":{"ICEBERG_COMPAT_VERSION":"2"}}} {"add":{"path":"part-00001-5bf41539-fbc6-4b96-9f42-946d36a7f4c9-c000.snappy.parquet","partitionValues":{},"size":22269,"modificationTime":1723094912648,"dataChange":true,"stats":"{\"numRecords\":4,\"minValues\":{\"col-150baa08-2441-4182-b1d2-6c33e9b2f6e6\":1,\"col-aaf5094f-3435-407f-bc7c-8d5cbd2bce22\":1,\"col-a9123e30-97e4-428e-a179-36af4908b4f3\":1,\"col-6a302d0e-d156-40e2-93b0-a26ee0a54eb4\":1,\"col-34f01baf-8cff-4abc-bf78-0494ba88070d\":1.0,\"col-0ef81992-71a5-47da-9c4c-45c76084bd54\":1.0,\"col-2c4cb239-3129-476c-8c0b-8e1f9ef575b8\":1.00,\"col-d588bf1d-e372-4f26-8572-5dd32730bc20\":\"1\",\"col-43df1513-711d-42c0-ae37-1988c0c7478f\":\"2021-11-18\",\"col-1a000afa-7d0f-4822-85fc-6285cf51e460\":\"1970-01-01T00:00:00.001Z\",\"col-8219db36-f0b0-4529-ac76-2e0da218668c\":{\"col-c0d2b42d-365a-40ce-ac1d-e33605b4b595\":\"1\",\"col-d2e13fb9-c6ae-4483-a580-7a27b4226700\":{\"col-99fc36e3-1691-4d36-bc4b-91f63779d1c2\":1}}},\"maxValues\":{\"col-150baa08-2441-4182-b1d2-6c33e9b2f6e6\":3,\"col-aaf5094f-3435-407f-bc7c-8d5cbd2bce22\":3,\"col-a9123e30-97e4-428e-a179-36af4908b4f3\":3,\"col-6a302d0e-d156-40e2-93b0-a26ee0a54eb4\":3,\"col-34f01baf-8cff-4abc-bf78-0494ba88070d\":3.0,\"col-0ef81992-71a5-47da-9c4c-45c76084bd54\":3.0,\"col-2c4cb239-3129-476c-8c0b-8e1f9ef575b8\":3.00,\"col-d588bf1d-e372-4f26-8572-5dd32730bc20\":\"3\",\"col-43df1513-711d-42c0-ae37-1988c0c7478f\":\"2021-11-18\",\"col-1a000afa-7d0f-4822-85fc-6285cf51e460\":\"1970-01-01T00:00:00.003Z\",\"col-8219db36-f0b0-4529-ac76-2e0da218668c\":{\"col-c0d2b42d-365a-40ce-ac1d-e33605b4b595\":\"3\",\"col-d2e13fb9-c6ae-4483-a580-7a27b4226700\":{\"col-99fc36e3-1691-4d36-bc4b-91f63779d1c2\":3}}},\"nullCount\":{\"col-150baa08-2441-4182-b1d2-6c33e9b2f6e6\":1,\"col-aaf5094f-3435-407f-bc7c-8d5cbd2bce22\":1,\"col-a9123e30-97e4-428e-a179-36af4908b4f3\":1,\"col-6a302d0e-d156-40e2-93b0-a26ee0a54eb4\":1,\"col-34f01baf-8cff-4abc-bf78-0494ba88070d\":1,\"col-0ef81992-71a5-47da-9c4c-45c76084bd54\":1,\"col-2c4cb239-3129-476c-8c0b-8e1f9ef575b8\":1,\"col-8dfa7127-0477-4ddf-9ce0-87baae1ca166\":1,\"col-d588bf1d-e372-4f26-8572-5dd32730bc20\":1,\"col-14d8b433-13d3-40b4-85bf-698b93a15edd\":1,\"col-43df1513-711d-42c0-ae37-1988c0c7478f\":1,\"col-1a000afa-7d0f-4822-85fc-6285cf51e460\":1,\"col-8219db36-f0b0-4529-ac76-2e0da218668c\":{\"col-c0d2b42d-365a-40ce-ac1d-e33605b4b595\":1,\"col-d2e13fb9-c6ae-4483-a580-7a27b4226700\":{\"col-99fc36e3-1691-4d36-bc4b-91f63779d1c2\":1}},\"col-9c4dbf49-152b-4f5d-8dfb-3a8884f78cb7\":1,\"col-77d17ccf-6d7d-4129-a93a-b6d4fd9c3483\":1,\"col-1755e9a3-fa11-4062-8092-fe8f1664be80\":1,\"col-ef476911-2c9b-49c5-8c76-7fc03c8953f8\":1,\"col-7431b180-3300-4771-a556-693c9be39683\":{\"col-53e9356f-7271-4a56-8586-77976034c213\":1,\"col-70da66c1-651b-4562-908b-330e91ee6db2\":1},\"col-c5aff22c-e8de-4618-8318-31386aa7721f\":1,\"col-8701e73b-45d6-4b23-87ac-c871996ad7a9\":1,\"col-8e1bb9a3-0c4d-4f5c-b95f-568c0703c77f\":1,\"col-9da911c6-054c-4167-881e-4c03ce7a3fd1\":1}}","tags":{"ICEBERG_COMPAT_VERSION":"2"}}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/time-travel-partition-changes-a/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603724026157,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part5\"]"},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"9ce7bb6f-507b-4925-a820-f33601e5d700","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"part5\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["part5"],"configuration":{},"createdTime":1603724025794}} {"add":{"path":"part5=0/part-00000-67b6882e-f49f-4df5-9850-b5e8a72f4917.c000.snappy.parquet","partitionValues":{"part5":"0"},"size":429,"modificationTime":1603724025000,"dataChange":true}} {"add":{"path":"part5=1/part-00000-8a40c3d2-f658-4131-a17f-388265ab04b7.c000.snappy.parquet","partitionValues":{"part5":"1"},"size":429,"modificationTime":1603724026000,"dataChange":true}} {"add":{"path":"part5=2/part-00000-ec6e3a2e-ecbf-4d39-9076-37e523cd62f1.c000.snappy.parquet","partitionValues":{"part5":"2"},"size":429,"modificationTime":1603724026000,"dataChange":true}} {"add":{"path":"part5=3/part-00000-eaf1edf4-b9da-4df8-b957-08583e2a1d1b.c000.snappy.parquet","partitionValues":{"part5":"3"},"size":429,"modificationTime":1603724026000,"dataChange":true}} {"add":{"path":"part5=4/part-00000-ce66c2ca-8fdf-48d3-a6e7-5980a370461a.c000.snappy.parquet","partitionValues":{"part5":"4"},"size":429,"modificationTime":1603724026000,"dataChange":true}} {"add":{"path":"part5=0/part-00001-4f02a740-31dc-46c6-bc0e-c19d164ac82d.c000.snappy.parquet","partitionValues":{"part5":"0"},"size":429,"modificationTime":1603724025000,"dataChange":true}} {"add":{"path":"part5=1/part-00001-3dcad520-b001-4829-a6e5-3d578b0964f4.c000.snappy.parquet","partitionValues":{"part5":"1"},"size":429,"modificationTime":1603724026000,"dataChange":true}} {"add":{"path":"part5=2/part-00001-e20bae81-3f27-4c5c-aeca-5cfa6b38615c.c000.snappy.parquet","partitionValues":{"part5":"2"},"size":429,"modificationTime":1603724026000,"dataChange":true}} {"add":{"path":"part5=3/part-00001-b9c6b926-a274-4d8e-b882-31c4aac05038.c000.snappy.parquet","partitionValues":{"part5":"3"},"size":429,"modificationTime":1603724026000,"dataChange":true}} {"add":{"path":"part5=4/part-00001-5705917d-d837-4d7f-b8c4-f0ada8cf9663.c000.snappy.parquet","partitionValues":{"part5":"4"},"size":429,"modificationTime":1603724026000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/time-travel-partition-changes-b/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603724026157,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part5\"]"},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"9ce7bb6f-507b-4925-a820-f33601e5d700","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"part5\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["part5"],"configuration":{},"createdTime":1603724025794}} {"add":{"path":"part5=0/part-00000-67b6882e-f49f-4df5-9850-b5e8a72f4917.c000.snappy.parquet","partitionValues":{"part5":"0"},"size":429,"modificationTime":1603724025000,"dataChange":true}} {"add":{"path":"part5=1/part-00000-8a40c3d2-f658-4131-a17f-388265ab04b7.c000.snappy.parquet","partitionValues":{"part5":"1"},"size":429,"modificationTime":1603724026000,"dataChange":true}} {"add":{"path":"part5=2/part-00000-ec6e3a2e-ecbf-4d39-9076-37e523cd62f1.c000.snappy.parquet","partitionValues":{"part5":"2"},"size":429,"modificationTime":1603724026000,"dataChange":true}} {"add":{"path":"part5=3/part-00000-eaf1edf4-b9da-4df8-b957-08583e2a1d1b.c000.snappy.parquet","partitionValues":{"part5":"3"},"size":429,"modificationTime":1603724026000,"dataChange":true}} {"add":{"path":"part5=4/part-00000-ce66c2ca-8fdf-48d3-a6e7-5980a370461a.c000.snappy.parquet","partitionValues":{"part5":"4"},"size":429,"modificationTime":1603724026000,"dataChange":true}} {"add":{"path":"part5=0/part-00001-4f02a740-31dc-46c6-bc0e-c19d164ac82d.c000.snappy.parquet","partitionValues":{"part5":"0"},"size":429,"modificationTime":1603724025000,"dataChange":true}} {"add":{"path":"part5=1/part-00001-3dcad520-b001-4829-a6e5-3d578b0964f4.c000.snappy.parquet","partitionValues":{"part5":"1"},"size":429,"modificationTime":1603724026000,"dataChange":true}} {"add":{"path":"part5=2/part-00001-e20bae81-3f27-4c5c-aeca-5cfa6b38615c.c000.snappy.parquet","partitionValues":{"part5":"2"},"size":429,"modificationTime":1603724026000,"dataChange":true}} {"add":{"path":"part5=3/part-00001-b9c6b926-a274-4d8e-b882-31c4aac05038.c000.snappy.parquet","partitionValues":{"part5":"3"},"size":429,"modificationTime":1603724026000,"dataChange":true}} {"add":{"path":"part5=4/part-00001-5705917d-d837-4d7f-b8c4-f0ada8cf9663.c000.snappy.parquet","partitionValues":{"part5":"4"},"size":429,"modificationTime":1603724026000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/time-travel-partition-changes-b/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1603724028432,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[\"part2\"]"},"readVersion":0,"isBlindAppend":false}} {"metaData":{"id":"9ce7bb6f-507b-4925-a820-f33601e5d700","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"part2\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["part2"],"configuration":{},"createdTime":1603724025794}} {"add":{"path":"part2=0/part-00000-7bce012e-f358-4a97-91da-55c4d3266fbe.c000.snappy.parquet","partitionValues":{"part2":"0"},"size":442,"modificationTime":1603724028000,"dataChange":true}} {"add":{"path":"part2=1/part-00000-82368d1d-588b-487a-be01-16dc85260296.c000.snappy.parquet","partitionValues":{"part2":"1"},"size":437,"modificationTime":1603724028000,"dataChange":true}} {"add":{"path":"part2=0/part-00001-2a830e69-78f3-4d09-9b2c-3bfd9debc2f0.c000.snappy.parquet","partitionValues":{"part2":"0"},"size":437,"modificationTime":1603724028000,"dataChange":true}} {"add":{"path":"part2=1/part-00001-0a72544a-fb83-4eaa-8d62-9e6ab59afa8b.c000.snappy.parquet","partitionValues":{"part2":"1"},"size":442,"modificationTime":1603724028000,"dataChange":true}} {"remove":{"path":"part5=0/part-00000-67b6882e-f49f-4df5-9850-b5e8a72f4917.c000.snappy.parquet","deletionTimestamp":1603724028432,"dataChange":true}} {"remove":{"path":"part5=0/part-00001-4f02a740-31dc-46c6-bc0e-c19d164ac82d.c000.snappy.parquet","deletionTimestamp":1603724028432,"dataChange":true}} {"remove":{"path":"part5=1/part-00001-3dcad520-b001-4829-a6e5-3d578b0964f4.c000.snappy.parquet","deletionTimestamp":1603724028432,"dataChange":true}} {"remove":{"path":"part5=2/part-00000-ec6e3a2e-ecbf-4d39-9076-37e523cd62f1.c000.snappy.parquet","deletionTimestamp":1603724028432,"dataChange":true}} {"remove":{"path":"part5=2/part-00001-e20bae81-3f27-4c5c-aeca-5cfa6b38615c.c000.snappy.parquet","deletionTimestamp":1603724028432,"dataChange":true}} {"remove":{"path":"part5=4/part-00000-ce66c2ca-8fdf-48d3-a6e7-5980a370461a.c000.snappy.parquet","deletionTimestamp":1603724028432,"dataChange":true}} {"remove":{"path":"part5=1/part-00000-8a40c3d2-f658-4131-a17f-388265ab04b7.c000.snappy.parquet","deletionTimestamp":1603724028432,"dataChange":true}} {"remove":{"path":"part5=3/part-00000-eaf1edf4-b9da-4df8-b957-08583e2a1d1b.c000.snappy.parquet","deletionTimestamp":1603724028432,"dataChange":true}} {"remove":{"path":"part5=3/part-00001-b9c6b926-a274-4d8e-b882-31c4aac05038.c000.snappy.parquet","deletionTimestamp":1603724028432,"dataChange":true}} {"remove":{"path":"part5=4/part-00001-5705917d-d837-4d7f-b8c4-f0ada8cf9663.c000.snappy.parquet","deletionTimestamp":1603724028432,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/time-travel-schema-changes-a/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603724023478,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"37664cd7-239f-4dbc-a56b-d47437be8ddb","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724023419}} {"add":{"path":"part-00000-83680aa8-547c-40bc-8ca9-5c10997e307b-c000.snappy.parquet","partitionValues":{},"size":449,"modificationTime":1603724023000,"dataChange":true}} {"add":{"path":"part-00001-3c1f89ce-a996-4d44-a79c-21a6f3d53138-c000.snappy.parquet","partitionValues":{},"size":451,"modificationTime":1603724023000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/time-travel-schema-changes-b/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603724023478,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"37664cd7-239f-4dbc-a56b-d47437be8ddb","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724023419}} {"add":{"path":"part-00000-83680aa8-547c-40bc-8ca9-5c10997e307b-c000.snappy.parquet","partitionValues":{},"size":449,"modificationTime":1603724023000,"dataChange":true}} {"add":{"path":"part-00001-3c1f89ce-a996-4d44-a79c-21a6f3d53138-c000.snappy.parquet","partitionValues":{},"size":451,"modificationTime":1603724023000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/time-travel-schema-changes-b/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1603724024783,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isBlindAppend":true}} {"metaData":{"id":"37664cd7-239f-4dbc-a56b-d47437be8ddb","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"part\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724023419}} {"add":{"path":"part-00000-a830a49c-6cc8-4caf-80a5-7ff8a959bd53-c000.snappy.parquet","partitionValues":{},"size":711,"modificationTime":1603724024000,"dataChange":true}} {"add":{"path":"part-00001-5fdfd303-d5e8-4e77-9b5d-4e831fa723e1-c000.snappy.parquet","partitionValues":{},"size":711,"modificationTime":1603724024000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/time-travel-start/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603724019870,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"d49dc19d-c206-4b38-be18-d8b7bdb07a07","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724019791}} {"add":{"path":"part-00000-c6271e23-2077-455c-94f9-52866f930213-c000.snappy.parquet","partitionValues":{},"size":449,"modificationTime":1603724019000,"dataChange":true}} {"add":{"path":"part-00001-e6177404-aaf5-4e07-8dc0-543a90f4657f-c000.snappy.parquet","partitionValues":{},"size":451,"modificationTime":1603724019000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/time-travel-start-start20/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603724019870,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"d49dc19d-c206-4b38-be18-d8b7bdb07a07","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724019791}} {"add":{"path":"part-00000-c6271e23-2077-455c-94f9-52866f930213-c000.snappy.parquet","partitionValues":{},"size":449,"modificationTime":1603724019000,"dataChange":true}} {"add":{"path":"part-00001-e6177404-aaf5-4e07-8dc0-543a90f4657f-c000.snappy.parquet","partitionValues":{},"size":451,"modificationTime":1603724019000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/time-travel-start-start20/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1603724021190,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isBlindAppend":true}} {"add":{"path":"part-00000-632e29c6-fedf-4822-9223-233d6d8d9086-c000.snappy.parquet","partitionValues":{},"size":451,"modificationTime":1603724021000,"dataChange":true}} {"add":{"path":"part-00001-90fee26a-1483-44e3-b239-805343fec254-c000.snappy.parquet","partitionValues":{},"size":451,"modificationTime":1603724021000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/time-travel-start-start20-start40/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603724019870,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"d49dc19d-c206-4b38-be18-d8b7bdb07a07","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1603724019791}} {"add":{"path":"part-00000-c6271e23-2077-455c-94f9-52866f930213-c000.snappy.parquet","partitionValues":{},"size":449,"modificationTime":1603724019000,"dataChange":true}} {"add":{"path":"part-00001-e6177404-aaf5-4e07-8dc0-543a90f4657f-c000.snappy.parquet","partitionValues":{},"size":451,"modificationTime":1603724019000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/time-travel-start-start20-start40/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1603724021190,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isBlindAppend":true}} {"add":{"path":"part-00000-632e29c6-fedf-4822-9223-233d6d8d9086-c000.snappy.parquet","partitionValues":{},"size":451,"modificationTime":1603724021000,"dataChange":true}} {"add":{"path":"part-00001-90fee26a-1483-44e3-b239-805343fec254-c000.snappy.parquet","partitionValues":{},"size":451,"modificationTime":1603724021000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/time-travel-start-start20-start40/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1603724022561,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isBlindAppend":true}} {"add":{"path":"part-00000-aef3cbc1-92ef-43b1-8258-284d13163fbb-c000.snappy.parquet","partitionValues":{},"size":451,"modificationTime":1603724022000,"dataChange":true}} {"add":{"path":"part-00001-2b364e64-4212-4a35-a95f-ab64504f7c5c-c000.snappy.parquet","partitionValues":{},"size":451,"modificationTime":1603724022000,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/type-widening/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1727266110116,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"3694"},"engineInfo":"Apache-Spark/3.5.2 Delta-Lake/3.3.0-SNAPSHOT","txnId":"80c33fca-d936-40cf-81fb-7ef52b67e25b"}} {"metaData":{"id":"db0018ee-037b-41f7-8266-85058ceafb06","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"byte_long\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int_long\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"float_double\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}},{\"name\":\"byte_double\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}},{\"name\":\"short_double\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int_double\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_decimal_same_scale\",\"type\":\"decimal(10,2)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_decimal_greater_scale\",\"type\":\"decimal(10,2)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"byte_decimal\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}},{\"name\":\"short_decimal\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int_decimal\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"long_decimal\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date_timestamp_ntz\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1727266102938}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part-00000-1045efe0-45bb-4b99-9f83-5ffa04a63ab2-c000.snappy.parquet","partitionValues":{},"size":3694,"modificationTime":1727266109760,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"byte_long\":1,\"int_long\":2,\"float_double\":3.4,\"byte_double\":5,\"short_double\":6,\"int_double\":7,\"decimal_decimal_same_scale\":123.45,\"decimal_decimal_greater_scale\":67.89,\"byte_decimal\":1,\"short_decimal\":2,\"int_decimal\":3,\"long_decimal\":4,\"date_timestamp_ntz\":\"2024-09-09\"},\"maxValues\":{\"byte_long\":1,\"int_long\":2,\"float_double\":3.4,\"byte_double\":5,\"short_double\":6,\"int_double\":7,\"decimal_decimal_same_scale\":123.45,\"decimal_decimal_greater_scale\":67.89,\"byte_decimal\":1,\"short_decimal\":2,\"int_decimal\":3,\"long_decimal\":4,\"date_timestamp_ntz\":\"2024-09-09\"},\"nullCount\":{\"byte_long\":0,\"int_long\":0,\"float_double\":0,\"byte_double\":0,\"short_double\":0,\"int_double\":0,\"decimal_decimal_same_scale\":0,\"decimal_decimal_greater_scale\":0,\"byte_decimal\":0,\"short_decimal\":0,\"int_decimal\":0,\"long_decimal\":0,\"date_timestamp_ntz\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/type-widening/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1727266114275,"operation":"SET TBLPROPERTIES","operationParameters":{"properties":"{\"delta.enableTypeWidening\":\"true\",\"delta.feature.timestampntz\":\"supported\"}"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.2 Delta-Lake/3.3.0-SNAPSHOT","txnId":"7b869171-851a-4a8d-96e6-baee5496b98f"}} {"metaData":{"id":"db0018ee-037b-41f7-8266-85058ceafb06","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"byte_long\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int_long\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"float_double\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}},{\"name\":\"byte_double\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}},{\"name\":\"short_double\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int_double\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_decimal_same_scale\",\"type\":\"decimal(10,2)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal_decimal_greater_scale\",\"type\":\"decimal(10,2)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"byte_decimal\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}},{\"name\":\"short_decimal\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int_decimal\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"long_decimal\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date_timestamp_ntz\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableTypeWidening":"true"},"createdTime":1727266102938}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["timestampNtz","typeWidening-preview"],"writerFeatures":["timestampNtz","typeWidening-preview","appendOnly","invariants"]}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/type-widening/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1727266116833,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"4059"},"engineInfo":"Apache-Spark/3.5.2 Delta-Lake/3.3.0-SNAPSHOT","txnId":"2e63edee-6d96-4d12-90af-85b90f4fa9e5"}} {"metaData":{"id":"db0018ee-037b-41f7-8266-85058ceafb06","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"byte_long\",\"type\":\"long\",\"nullable\":true,\"metadata\":{\"delta.typeChanges\":[{\"toType\":\"long\",\"fromType\":\"byte\",\"tableVersion\":2}]}},{\"name\":\"int_long\",\"type\":\"long\",\"nullable\":true,\"metadata\":{\"delta.typeChanges\":[{\"toType\":\"long\",\"fromType\":\"integer\",\"tableVersion\":2}]}},{\"name\":\"float_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{\"delta.typeChanges\":[{\"toType\":\"double\",\"fromType\":\"float\",\"tableVersion\":2}]}},{\"name\":\"byte_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{\"delta.typeChanges\":[{\"toType\":\"double\",\"fromType\":\"byte\",\"tableVersion\":2}]}},{\"name\":\"short_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{\"delta.typeChanges\":[{\"toType\":\"double\",\"fromType\":\"short\",\"tableVersion\":2}]}},{\"name\":\"int_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{\"delta.typeChanges\":[{\"toType\":\"double\",\"fromType\":\"integer\",\"tableVersion\":2}]}},{\"name\":\"decimal_decimal_same_scale\",\"type\":\"decimal(20,2)\",\"nullable\":true,\"metadata\":{\"delta.typeChanges\":[{\"toType\":\"decimal(20,2)\",\"fromType\":\"decimal(10,2)\",\"tableVersion\":2}]}},{\"name\":\"decimal_decimal_greater_scale\",\"type\":\"decimal(20,5)\",\"nullable\":true,\"metadata\":{\"delta.typeChanges\":[{\"toType\":\"decimal(20,5)\",\"fromType\":\"decimal(10,2)\",\"tableVersion\":2}]}},{\"name\":\"byte_decimal\",\"type\":\"decimal(11,1)\",\"nullable\":true,\"metadata\":{\"delta.typeChanges\":[{\"toType\":\"decimal(11,1)\",\"fromType\":\"byte\",\"tableVersion\":2}]}},{\"name\":\"short_decimal\",\"type\":\"decimal(11,1)\",\"nullable\":true,\"metadata\":{\"delta.typeChanges\":[{\"toType\":\"decimal(11,1)\",\"fromType\":\"short\",\"tableVersion\":2}]}},{\"name\":\"int_decimal\",\"type\":\"decimal(11,1)\",\"nullable\":true,\"metadata\":{\"delta.typeChanges\":[{\"toType\":\"decimal(11,1)\",\"fromType\":\"integer\",\"tableVersion\":2}]}},{\"name\":\"long_decimal\",\"type\":\"decimal(21,1)\",\"nullable\":true,\"metadata\":{\"delta.typeChanges\":[{\"toType\":\"decimal(21,1)\",\"fromType\":\"long\",\"tableVersion\":2}]}},{\"name\":\"date_timestamp_ntz\",\"type\":\"timestamp_ntz\",\"nullable\":true,\"metadata\":{\"delta.typeChanges\":[{\"toType\":\"timestamp_ntz\",\"fromType\":\"date\",\"tableVersion\":2}]}}]}","partitionColumns":[],"configuration":{"delta.enableTypeWidening":"true"},"createdTime":1727266102938}} {"add":{"path":"part-00000-cd317895-4ae0-4292-b918-62d4ca832bd7-c000.snappy.parquet","partitionValues":{},"size":4059,"modificationTime":1727266116789,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"byte_long\":9223372036854775807,\"int_long\":9223372036854775807,\"float_double\":1.234567890123,\"byte_double\":1.234567890123,\"short_double\":1.234567890123,\"int_double\":1.234567890123,\"decimal_decimal_same_scale\":12345678901234.56,\"decimal_decimal_greater_scale\":12345678901.23456,\"byte_decimal\":123.4,\"short_decimal\":12345.6,\"int_decimal\":1234567890.1,\"long_decimal\":123456789012345678.9,\"date_timestamp_ntz\":\"2024-09-09T12:34:56.123\"},\"maxValues\":{\"byte_long\":9223372036854775807,\"int_long\":9223372036854775807,\"float_double\":1.234567890123,\"byte_double\":1.234567890123,\"short_double\":1.234567890123,\"int_double\":1.234567890123,\"decimal_decimal_same_scale\":12345678901234.56,\"decimal_decimal_greater_scale\":12345678901.23456,\"byte_decimal\":123.4,\"short_decimal\":12345.6,\"int_decimal\":1234567890.1,\"long_decimal\":123456789012345678.9,\"date_timestamp_ntz\":\"2024-09-09T12:34:56.123\"},\"nullCount\":{\"byte_long\":0,\"int_long\":0,\"float_double\":0,\"byte_double\":0,\"short_double\":0,\"int_double\":0,\"decimal_decimal_same_scale\":0,\"decimal_decimal_greater_scale\":0,\"byte_decimal\":0,\"short_decimal\":0,\"int_decimal\":0,\"long_decimal\":0,\"date_timestamp_ntz\":0}}","defaultRowCommitVersion":2}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/type-widening-nested/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1727266119620,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"1416"},"engineInfo":"Apache-Spark/3.5.2 Delta-Lake/3.3.0-SNAPSHOT","txnId":"034f1fec-b6d9-4957-93c1-09a19c323fc2"}} {"metaData":{"id":"43c8feba-0140-4d91-8c16-52f627a79cfe","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"struct\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"a\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"map\",\"type\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":\"integer\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"array\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1727266118466}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part-00000-138244f1-b939-40db-a4bd-d57cf3d214d2-c000.snappy.parquet","partitionValues":{},"size":1416,"modificationTime":1727266119587,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"struct\":{\"a\":1}},\"maxValues\":{\"struct\":{\"a\":1}},\"nullCount\":{\"struct\":{\"a\":0},\"map\":0,\"array\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/type-widening-nested/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1727266120320,"operation":"SET TBLPROPERTIES","operationParameters":{"properties":"{\"delta.enableTypeWidening\":\"true\"}"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.2 Delta-Lake/3.3.0-SNAPSHOT","txnId":"f4db4fbf-a05b-41b4-9049-e1f28a08ec5b"}} {"metaData":{"id":"43c8feba-0140-4d91-8c16-52f627a79cfe","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"struct\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"a\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"map\",\"type\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":\"integer\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"array\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableTypeWidening":"true"},"createdTime":1727266118466}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["typeWidening-preview"],"writerFeatures":["typeWidening-preview","appendOnly","invariants"]}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/type-widening-nested/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1727266121897,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"1519"},"engineInfo":"Apache-Spark/3.5.2 Delta-Lake/3.3.0-SNAPSHOT","txnId":"55211a5a-9d2b-4367-929a-ac5850f91b78"}} {"metaData":{"id":"43c8feba-0140-4d91-8c16-52f627a79cfe","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"struct\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"a\",\"type\":\"long\",\"nullable\":true,\"metadata\":{\"delta.typeChanges\":[{\"toType\":\"long\",\"fromType\":\"integer\",\"tableVersion\":2}]}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"map\",\"type\":{\"type\":\"map\",\"keyType\":\"long\",\"valueType\":\"long\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{\"delta.typeChanges\":[{\"toType\":\"long\",\"fromType\":\"integer\",\"tableVersion\":2,\"fieldPath\":\"key\"},{\"toType\":\"long\",\"fromType\":\"integer\",\"tableVersion\":2,\"fieldPath\":\"value\"}]}},{\"name\":\"array\",\"type\":{\"type\":\"array\",\"elementType\":\"long\",\"containsNull\":true},\"nullable\":true,\"metadata\":{\"delta.typeChanges\":[{\"toType\":\"long\",\"fromType\":\"integer\",\"tableVersion\":2,\"fieldPath\":\"element\"}]}}]}","partitionColumns":[],"configuration":{"delta.enableTypeWidening":"true"},"createdTime":1727266118466}} {"add":{"path":"part-00000-1f777f86-350c-4181-b7ef-73df70847eac-c000.snappy.parquet","partitionValues":{},"size":1519,"modificationTime":1727266121853,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"struct\":{\"a\":9223372036854775807}},\"maxValues\":{\"struct\":{\"a\":9223372036854775807}},\"nullCount\":{\"struct\":{\"a\":0},\"map\":0,\"array\":0}}","defaultRowCommitVersion":2}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/update-deleted-directory/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603723978664,"operation":"Manual Update","operationParameters":{},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"94c8d2b0-fbad-439b-a31f-17e17d93c2c7","format":{"provider":"parquet","options":{}},"partitionColumns":[],"configuration":{},"createdTime":1603723978664}} {"add":{"path":"1","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} {"add":{"path":"2","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} {"add":{"path":"3","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} {"add":{"path":"4","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} {"add":{"path":"5","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} {"add":{"path":"6","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} {"add":{"path":"7","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} {"add":{"path":"8","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} {"add":{"path":"9","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} {"add":{"path":"10","partitionValues":{},"size":1,"modificationTime":1,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/update-deleted-directory/_delta_log/_last_checkpoint ================================================ {"version":0,"size":12} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/v2-checkpoint-json/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1714496114594,"operation":"CREATE TABLE","operationParameters":{"partitionBy":"[]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{\"delta.checkpointInterval\":\"2\"}"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"f6282e54-afc6-4669-939b-0f8ba73062a0"}} {"metaData":{"id":"8a390218-e4ee-4341-b6de-4920e27d3f78","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2"},"createdTime":1714496114564}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/v2-checkpoint-json/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1714496114748,"operation":"SET TBLPROPERTIES","operationParameters":{"properties":"{\"delta.checkpointPolicy\":\"v2\"}"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"fddb3112-ca9b-48af-bf19-be23f1c36c22"}} {"metaData":{"id":"8a390218-e4ee-4341-b6de-4920e27d3f78","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2","delta.checkpointPolicy":"v2"},"createdTime":1714496114564}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["v2Checkpoint"],"writerFeatures":["v2Checkpoint","appendOnly","invariants"]}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/v2-checkpoint-json/_delta_log/00000000000000000002.checkpoint.6374b053-df23-479b-b2cf-c9c550132b49.json ================================================ {"checkpointMetadata":{"version":2}} {"sidecar":{"path":"00000000000000000002.checkpoint.0000000001.0000000002.bd1885fd-6ec0-4370-b0f5-43b5162fd4de.parquet","sizeInBytes":9367,"modificationTime":1714496115780}} {"sidecar":{"path":"00000000000000000002.checkpoint.0000000002.0000000002.0a8d73ee-aa83-49d0-9583-c99db75b89b2.parquet","sizeInBytes":9296,"modificationTime":1714496115788}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["v2Checkpoint"],"writerFeatures":["v2Checkpoint","appendOnly","invariants"]}} {"metaData":{"id":"8a390218-e4ee-4341-b6de-4920e27d3f78","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2","delta.checkpointPolicy":"v2"},"createdTime":1714496114564}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/v2-checkpoint-json/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1714496115090,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"4","numOutputRows":"10","numOutputBytes":"1952"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"a76e8fca-8bab-42cc-9618-77f8c536968c"}} {"add":{"path":"part-00000-240b5dd6-323b-4f74-b6bc-ab9fdcacc630-c000.snappy.parquet","partitionValues":{},"size":485,"modificationTime":1714496115046,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"id\":4},\"maxValues\":{\"id\":8},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-534ea355-2edd-4046-8d49-d932469170c7-c000.snappy.parquet","partitionValues":{},"size":496,"modificationTime":1714496115048,"dataChange":true,"stats":"{\"numRecords\":4,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00002-4438bc9d-9c60-4dd2-9343-574743ea4ca8-c000.snappy.parquet","partitionValues":{},"size":486,"modificationTime":1714496115087,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":5},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00003-ae431d66-23d5-4dc7-b961-136ce33e63da-c000.snappy.parquet","partitionValues":{},"size":485,"modificationTime":1714496115087,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"id\":2},\"maxValues\":{\"id\":6},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/v2-checkpoint-json/_delta_log/_last_checkpoint ================================================ {"version":2,"size":9,"sizeInBytes":19554,"numOfAddFiles":4,"v2Checkpoint":{"path":"00000000000000000002.checkpoint.6374b053-df23-479b-b2cf-c9c550132b49.json","sizeInBytes":891,"modificationTime":1714496115810,"nonFileActions":[{"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["v2Checkpoint"],"writerFeatures":["v2Checkpoint","appendOnly","invariants"]}},{"metaData":{"id":"8a390218-e4ee-4341-b6de-4920e27d3f78","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2","delta.checkpointPolicy":"v2"},"createdTime":1714496114564}},{"checkpointMetadata":{"version":2}}],"sidecarFiles":[{"path":"00000000000000000002.checkpoint.0000000001.0000000002.bd1885fd-6ec0-4370-b0f5-43b5162fd4de.parquet","sizeInBytes":9367,"modificationTime":1714496115780},{"path":"00000000000000000002.checkpoint.0000000002.0000000002.0a8d73ee-aa83-49d0-9583-c99db75b89b2.parquet","sizeInBytes":9296,"modificationTime":1714496115788}]},"checksum":"d09f95a326aab562c60d415a32ddd216"} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/v2-checkpoint-parquet/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1714496109365,"operation":"CREATE TABLE","operationParameters":{"partitionBy":"[]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{\"delta.checkpointInterval\":\"2\"}"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"7517176e-cff7-46ac-b133-3cf096e2620d"}} {"metaData":{"id":"7e2a1106-198b-4653-a612-2aa44685cb27","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2"},"createdTime":1714496109258}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/v2-checkpoint-parquet/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1714496110834,"operation":"SET TBLPROPERTIES","operationParameters":{"properties":"{\"delta.checkpointPolicy\":\"v2\"}"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"12ea26b9-c620-4104-95f6-654bcaabdda6"}} {"metaData":{"id":"7e2a1106-198b-4653-a612-2aa44685cb27","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2","delta.checkpointPolicy":"v2"},"createdTime":1714496109258}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["v2Checkpoint"],"writerFeatures":["v2Checkpoint","appendOnly","invariants"]}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/v2-checkpoint-parquet/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1714496112086,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"4","numOutputRows":"10","numOutputBytes":"1952"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"c9f86c17-1b30-44e7-873d-1e2102f54b0f"}} {"add":{"path":"part-00000-485b0fff-1c7b-4f14-92e9-a72300fcdf88-c000.snappy.parquet","partitionValues":{},"size":485,"modificationTime":1714496111974,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"id\":4},\"maxValues\":{\"id\":8},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-f7a80035-0622-431e-832e-a756c65cb2a5-c000.snappy.parquet","partitionValues":{},"size":496,"modificationTime":1714496111974,"dataChange":true,"stats":"{\"numRecords\":4,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00002-5754df9c-5a25-43a6-947b-f27840fddb1a-c000.snappy.parquet","partitionValues":{},"size":486,"modificationTime":1714496112068,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":5},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00003-6ab7bbbb-e14d-4fa3-8767-06b509e0a666-c000.snappy.parquet","partitionValues":{},"size":485,"modificationTime":1714496112071,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"id\":2},\"maxValues\":{\"id\":6},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/v2-checkpoint-parquet/_delta_log/_last_checkpoint ================================================ {"version":2,"size":9,"sizeInBytes":37269,"numOfAddFiles":4,"checkpointSchema":{"type":"struct","fields":[{"name":"txn","type":{"type":"struct","fields":[{"name":"appId","type":"string","nullable":true,"metadata":{}},{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"lastUpdated","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"add","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"stats","type":"string","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}},{"name":"clusteringProvider","type":"string","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"remove","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"deletionTimestamp","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"extendedFileMetadata","type":"boolean","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}},{"name":"stats","type":"string","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"metaData","type":{"type":"struct","fields":[{"name":"id","type":"string","nullable":true,"metadata":{}},{"name":"name","type":"string","nullable":true,"metadata":{}},{"name":"description","type":"string","nullable":true,"metadata":{}},{"name":"format","type":{"type":"struct","fields":[{"name":"provider","type":"string","nullable":true,"metadata":{}},{"name":"options","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"schemaString","type":"string","nullable":true,"metadata":{}},{"name":"partitionColumns","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"configuration","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"createdTime","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"protocol","type":{"type":"struct","fields":[{"name":"minReaderVersion","type":"integer","nullable":true,"metadata":{}},{"name":"minWriterVersion","type":"integer","nullable":true,"metadata":{}},{"name":"readerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"writerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"domainMetadata","type":{"type":"struct","fields":[{"name":"domain","type":"string","nullable":true,"metadata":{}},{"name":"configuration","type":"string","nullable":true,"metadata":{}},{"name":"removed","type":"boolean","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"checkpointMetadata","type":{"type":"struct","fields":[{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"sidecar","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"v2Checkpoint":{"path":"00000000000000000002.checkpoint.e8fa2696-9728-4e9c-b285-634743fdd4fb.parquet","sizeInBytes":18634,"modificationTime":1714496114276,"nonFileActions":[{"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["v2Checkpoint"],"writerFeatures":["v2Checkpoint","appendOnly","invariants"]}},{"metaData":{"id":"7e2a1106-198b-4653-a612-2aa44685cb27","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2","delta.checkpointPolicy":"v2"},"createdTime":1714496109258}},{"checkpointMetadata":{"version":2}}],"sidecarFiles":[{"path":"00000000000000000002.checkpoint.0000000001.0000000002.055454d8-329c-4e0e-864d-7f867075af33.parquet","sizeInBytes":9268,"modificationTime":1714496113961},{"path":"00000000000000000002.checkpoint.0000000002.0000000002.33321cc1-9c55-4d1f-8511-fafe6d2e1133.parquet","sizeInBytes":9367,"modificationTime":1714496113961}]},"checksum":"f81aaf268542b71bb3fc9b63f754f9df"} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/versions-not-contiguous/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1603723995084,"operation":"Manual Update","operationParameters":{},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"a564e335-d717-4a71-a1eb-25541a0f8d15","format":{"provider":"parquet","options":{}},"partitionColumns":[],"configuration":{},"createdTime":1603723995084}} {"add":{"path":"foo","partitionValues":{},"size":1,"modificationTime":1603723995077,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/resources/golden/versions-not-contiguous/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1603723996094,"operation":"Manual Update","operationParameters":{},"readVersion":1,"isBlindAppend":true}} {"add":{"path":"foo","partitionValues":{},"size":1,"modificationTime":1603723996088,"dataChange":true}} ================================================ FILE: connectors/golden-tables/src/main/scala/io/delta/golden/GoldenTableUtils.scala ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.golden import java.io.File object GoldenTableUtils { lazy val classLoader = GoldenTableUtils.getClass.getClassLoader() lazy val goldenResourceURL = classLoader.getResource("golden") def goldenTablePath(name: String): String = { classLoader.getResource(s"golden/$name").getPath } def goldenTableFile(name: String): File = { new File(classLoader.getResource(s"golden/$name").getFile) } def allTableNames(): Seq[String] = { val root = new File(goldenResourceURL.getFile) def loop(dir: File): Seq[File] = { val children = Option(dir.listFiles()).getOrElse(Array.empty[File]) val subdirs = children.filter(_.isDirectory) val here = if (new File(dir, "_delta_log").isDirectory) Seq(dir) else Seq.empty[File] here ++ subdirs.flatMap(loop) } val rootPath = root.toPath // Find all directories containing `_delta_log` under the `golden` resource and return their // relative paths (to the golden root), sorted. loop(root).map(f => rootPath.relativize(f.toPath).toString).sorted } } ================================================ FILE: connectors/golden-tables/src/test/scala/io/delta/golden/GoldenTables.scala ================================================ /* * Copyright (2020-present) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.golden import java.io.File import java.math.{BigInteger, BigDecimal => JBigDecimal} import java.sql.Timestamp import java.time.ZoneOffset.UTC import java.time.LocalDateTime import java.util.{Locale, Random, TimeZone} import scala.collection.mutable.ArrayBuffer import scala.concurrent.duration._ import scala.language.implicitConversions import io.delta.tables.DeltaTable import org.apache.commons.io.FileUtils import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.network.util.JavaUtils import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.delta.{DeltaLog, OptimisticTransaction} import org.apache.spark.sql.delta.DeltaOperations.ManualUpdate import org.apache.spark.sql.delta.actions.{Metadata, _} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ /** * This is a special class to generate golden tables for other projects. Run the following commands * to re-generate all golden tables: * ``` * GENERATE_GOLDEN_TABLES=1 build/sbt 'goldenTables/test' * ``` * * To generate a single table (that is specified below) run: * ``` * GENERATE_GOLDEN_TABLES=1 build/sbt 'goldenTables/testOnly *GoldenTables -- -z "tbl_name"' * ``` * * After generating golden tables, be sure to package or test project standalone, otherwise the * test resources won't be available when running tests with IntelliJ. */ class GoldenTables extends QueryTest with SharedSparkSession { import testImplicits._ override def sparkConf: SparkConf = super.sparkConf .set("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .set("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") // disable _SUCCESS files .set("spark.hadoop.mapreduce.fileoutputcommitter.marksuccessfuljobs", "false") // Timezone is fixed to America/Los_Angeles for timezone-sensitive tests TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles")) // Add Locale setting Locale.setDefault(Locale.US) private val shouldGenerateGoldenTables = sys.env.contains("GENERATE_GOLDEN_TABLES") private lazy val goldenTablePath = { val dir = new File("src/main/resources/golden").getCanonicalFile require(dir.exists(), s"Cannot find $dir. Please run `GENERATE_GOLDEN_TABLES=1 build/sbt 'goldenTables/test'`.") dir } private def copyDir(src: String, dest: String): Unit = { FileUtils.copyDirectory(createGoldenTableFile(src), createGoldenTableFile(dest)) } private def createGoldenTableFile(name: String): File = new File(goldenTablePath, name) private def createHiveGoldenTableFile(name: String): File = new File(createGoldenTableFile("hive"), name) private def generateGoldenTable(name: String, createTableFile: String => File = createGoldenTableFile) (generator: String => Unit): Unit = { if (shouldGenerateGoldenTables) { test(name) { val tablePath = createTableFile(name) JavaUtils.deleteRecursively(tablePath) generator(tablePath.getCanonicalPath) } } } /** * Helper class for to ensure initial commits contain a Metadata action. */ private implicit class OptimisticTxnTestHelper(txn: OptimisticTransaction) { def commitManually(actions: Action*): Long = { if (txn.readVersion == -1 && !actions.exists(_.isInstanceOf[Metadata])) { val schema = new StructType() .add("intCol", IntegerType) .json txn.commit(Metadata(schemaString = schema) +: actions, ManualUpdate) } else { txn.commit(actions, ManualUpdate) } } } /////////////////////////////////////////////////////////////////////////// // io.delta.standalone.internal.DeltaLogSuite /////////////////////////////////////////////////////////////////////////// /** TEST: DeltaLogSuite > checkpoint */ generateGoldenTable("checkpoint") { tablePath => val log = DeltaLog.forTable(spark, new Path(tablePath)) (1 to 15).foreach { i => val txn = log.startTransaction() val file = AddFile(i.toString, Map.empty, 1, 1, true) :: Nil val delete: Seq[Action] = if (i > 1) { RemoveFile((i - 1).toString, Some(System.currentTimeMillis()), true) :: Nil } else { Nil } txn.commitManually(delete ++ file: _*) } } /** TEST: DeltaLogSuite > snapshot */ private def writeData(data: Seq[(Int, String)], mode: String, tablePath: String): Unit = { data.toDS .toDF("col1", "col2") .write .mode(mode) .format("delta") .save(tablePath) } generateGoldenTable("snapshot-data0") { tablePath => writeData((0 until 10).map(x => (x, s"data-0-$x")), "append", tablePath) } generateGoldenTable("snapshot-data1") { tablePath => copyDir("snapshot-data0", "snapshot-data1") writeData((0 until 10).map(x => (x, s"data-1-$x")), "append", tablePath) } generateGoldenTable("snapshot-data2") { tablePath => copyDir("snapshot-data1", "snapshot-data2") writeData((0 until 10).map(x => (x, s"data-2-$x")), "overwrite", tablePath) } generateGoldenTable("snapshot-data3") { tablePath => copyDir("snapshot-data2", "snapshot-data3") writeData((0 until 20).map(x => (x, s"data-3-$x")), "append", tablePath) } generateGoldenTable("snapshot-data2-deleted") { tablePath => copyDir("snapshot-data3", "snapshot-data2-deleted") DeltaTable.forPath(spark, tablePath).delete("col2 like 'data-2-%'") } generateGoldenTable("snapshot-repartitioned") { tablePath => copyDir("snapshot-data2-deleted", "snapshot-repartitioned") spark.read .format("delta") .load(tablePath) .repartition(2) .write .option("dataChange", "false") .format("delta") .mode("overwrite") .save(tablePath) } generateGoldenTable("snapshot-vacuumed") { tablePath => copyDir("snapshot-repartitioned", "snapshot-vacuumed") withSQLConf(DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED.key -> "false") { DeltaTable.forPath(spark, tablePath).vacuum(0.0) } } /** TEST: DeltaLogSuite > SC-8078: update deleted directory */ generateGoldenTable("update-deleted-directory") { tablePath => val log = DeltaLog.forTable(spark, new Path(tablePath)) val txn = log.startTransaction() val files = (1 to 10).map(f => AddFile(f.toString, Map.empty, 1, 1, true)) txn.commitManually(files: _*) log.checkpoint() } /** TEST: DeltaLogSuite > handle corrupted '_last_checkpoint' file */ generateGoldenTable("corrupted-last-checkpoint") { tablePath => val log = DeltaLog.forTable(spark, new Path(tablePath)) val checkpointInterval = log.checkpointInterval(log.unsafeVolatileSnapshot.metadata) for (f <- 0 to checkpointInterval) { val txn = log.startTransaction() txn.commitManually(AddFile(f.toString, Map.empty, 1, 1, true)) } } generateGoldenTable("corrupted-last-checkpoint-kernel") { tablePath => val log = DeltaLog.forTable(spark, new Path(tablePath)) val checkpointInterval = log.checkpointInterval(log.unsafeVolatileSnapshot.metadata) for (f <- 0 to checkpointInterval) { spark.range(10).write.format("delta").mode("append").save(tablePath) } spark.range(100).write.format("delta").mode("overwrite").save(tablePath) // Create an empty "_last_checkpoint" (corrupted) val fs = log.LAST_CHECKPOINT.getFileSystem(log.newDeltaHadoopConf()) fs.create(log.LAST_CHECKPOINT, true /* overwrite */).close() } /** TEST: DeltaLogSuite > paths should be canonicalized */ { def helper(scheme: String, path: String, tableSuffix: String): Unit = { generateGoldenTable(s"canonicalized-paths-$tableSuffix") { tablePath => val log = DeltaLog.forTable(spark, new Path(tablePath)) new File(log.logPath.toUri).mkdirs() val add = AddFile(path, Map.empty, 100L, 10L, dataChange = true) val rm = RemoveFile(s"$scheme$path", Some(200L)) log.startTransaction().commitManually(add) log.startTransaction().commitManually(rm) } } // normal characters helper("file:", "/some/unqualified/absolute/path", "normal-a") helper("file://", "/some/unqualified/absolute/path", "normal-b") // special characters helper("file:", new Path("/some/unqualified/with space/p@#h").toUri.toString, "special-a") helper("file://", new Path("/some/unqualified/with space/p@#h").toUri.toString, "special-b") } /** TEST: DeltaLogSuite > delete and re-add the same file in different transactions */ generateGoldenTable(s"delete-re-add-same-file-different-transactions") { tablePath => val log = DeltaLog.forTable(spark, new Path(tablePath)) assert(new File(log.logPath.toUri).mkdirs()) val add1 = AddFile("foo", Map.empty, 1L, 1600000000000L, dataChange = true) log.startTransaction().commitManually(add1) val rm = add1.remove log.startTransaction().commit(rm :: Nil, ManualUpdate) val add2 = AddFile("foo", Map.empty, 1L, 1700000000000L, dataChange = true) log.startTransaction().commit(add2 :: Nil, ManualUpdate) // Add a new transaction to replay logs using the previous snapshot. If it contained // AddFile("foo") and RemoveFile("foo"), "foo" would get removed and fail this test. val otherAdd = AddFile("bar", Map.empty, 1L, System.currentTimeMillis(), dataChange = true) log.startTransaction().commit(otherAdd :: Nil, ManualUpdate) } /** TEST: DeltaLogSuite > error - versions not contiguous */ generateGoldenTable("versions-not-contiguous") { tablePath => val log = DeltaLog.forTable(spark, new Path(tablePath)) assert(new File(log.logPath.toUri).mkdirs()) val add1 = AddFile("foo", Map.empty, 1L, System.currentTimeMillis(), dataChange = true) log.startTransaction().commitManually(add1) val add2 = AddFile("foo", Map.empty, 1L, System.currentTimeMillis(), dataChange = true) log.startTransaction().commit(add2 :: Nil, ManualUpdate) val add3 = AddFile("foo", Map.empty, 1L, System.currentTimeMillis(), dataChange = true) log.startTransaction().commit(add3 :: Nil, ManualUpdate) new File(new Path(log.logPath, "00000000000000000001.json").toUri).delete() } /** TEST: DeltaLogSuite > state reconstruction without Protocol/Metadata should fail */ Seq("protocol", "metadata").foreach { action => generateGoldenTable(s"deltalog-state-reconstruction-without-$action") { tablePath => val log = DeltaLog.forTable(spark, new Path(tablePath)) assert(new File(log.logPath.toUri).mkdirs()) val selectedAction = if (action == "metadata") { Protocol() } else { val schema = new StructType() .add("intCol", IntegerType) .json Metadata(schemaString = schema) } val file = AddFile("abc", Map.empty, 1, 1, true) log.store.write( FileNames.unsafeDeltaFile(log.logPath, 0L), Iterator(selectedAction, file).map(a => JsonUtils.toJson(a.wrap))) } } /** * TEST: DeltaLogSuite > state reconstruction from checkpoint with missing Protocol/Metadata * should fail */ Seq("protocol", "metadata").foreach { action => generateGoldenTable(s"deltalog-state-reconstruction-from-checkpoint-missing-$action") { tablePath => val log = DeltaLog.forTable(spark, tablePath) val checkpointInterval = log.checkpointInterval(log.unsafeVolatileSnapshot.metadata) // Create a checkpoint regularly for (f <- 0 to checkpointInterval) { val txn = log.startTransaction() if (f == 0) { txn.commitManually(AddFile(f.toString, Map.empty, 1, 1, true)) } else { txn.commit(Seq(AddFile(f.toString, Map.empty, 1, 1, true)), ManualUpdate) } } // Create an incomplete checkpoint without the action and overwrite the // original checkpoint val checkpointPath = FileNames.checkpointFileSingular(log.logPath, log.snapshot.version) withTempDir { tmpCheckpoint => val takeAction = if (action == "metadata") { "protocol" } else { "metadata" } val corruptedCheckpointData = spark.read.parquet(checkpointPath.toString) .where(s"add is not null or $takeAction is not null") .as[SingleAction].collect() // Keep the add files and also filter by the additional condition corruptedCheckpointData.toSeq.toDS().coalesce(1).write .mode("overwrite").parquet(tmpCheckpoint.toString) val writtenCheckpoint = tmpCheckpoint.listFiles().toSeq.filter(_.getName.startsWith("part")).head val checkpointFile = new File(checkpointPath.toUri) new File(log.logPath.toUri).listFiles().toSeq.foreach { file => if (file.getName.startsWith(".0")) { // we need to delete checksum files, otherwise trying to replace our incomplete // checkpoint file fails due to the LocalFileSystem's checksum checks. require(file.delete(), "Failed to delete checksum file") } } require(checkpointFile.delete(), "Failed to delete old checkpoint") require(writtenCheckpoint.renameTo(checkpointFile), "Failed to rename corrupt checkpoint") } } } /** TEST: DeltaLogSuite > table protocol version greater than client reader protocol version */ generateGoldenTable("deltalog-invalid-protocol-version") { tablePath => val log = DeltaLog.forTable(spark, new Path(tablePath)) assert(new File(log.logPath.toUri).mkdirs()) val file = AddFile("abc", Map.empty, 1, 1, true) val metadata = Metadata( schemaString = new StructType().add("id", IntegerType).json ) log.store.write(FileNames.unsafeDeltaFile(log.logPath, 0L), // Protocol reader version explicitly set too high // Also include a Metadata Iterator(Protocol(99), metadata, file).map(a => JsonUtils.toJson(a.wrap))) } /** TEST: DeltaLogSuite > get commit info */ generateGoldenTable("deltalog-commit-info") { tablePath => val log = DeltaLog.forTable(spark, new Path(tablePath)) assert(new File(log.logPath.toUri).mkdirs()) val commitInfoFile = CommitInfo( version = Some(0L), inCommitTimestamp = None, timestamp = new Timestamp(1540415658000L), userId = Some("user_0"), userName = Some("username_0"), operation = "WRITE", operationParameters = Map("test" -> "\"test\""), job = Some(JobInfo( "job_id_0", "job_name_0", "job_run_id_0", "run_id_0", "job_owner_0", "trigger_type_0")), notebook = Some(NotebookInfo("notebook_id_0")), clusterId = Some("cluster_id_0"), readVersion = Some(-1L), isolationLevel = Some("default"), isBlindAppend = Some(true), operationMetrics = Some(Map("test" -> "test")), userMetadata = Some("foo"), tags = Some(Map("test" -> "test")), engineInfo = Some("OSS"), txnId = Some("txn_id_0") ) val addFile = AddFile("abc", Map.empty, 1, 1, true) log.store.write( FileNames.unsafeDeltaFile(log.logPath, 0L), Iterator(Metadata(), Protocol(), commitInfoFile, addFile).map(a => JsonUtils.toJson(a.wrap))) } /** TEST: DeltaLogSuite > getChanges - no data loss */ generateGoldenTable("deltalog-getChanges") { tablePath => val log = DeltaLog.forTable(spark, new Path(tablePath)) val schema = new StructType() .add("part", IntegerType) .add("id", IntegerType) val metadata = Metadata(schemaString = schema.json) val add1 = AddFile("fake/path/1", Map.empty, 1, 1, dataChange = true) val txn1 = log.startTransaction() txn1.commitManually(metadata :: add1 :: Nil: _*) val addCDC2 = AddCDCFile("fake/path/2", Map("partition_foo" -> "partition_bar"), 1, Map("tag_foo" -> "tag_bar")) val remove2 = RemoveFile("fake/path/1", Some(100), dataChange = true) val txn2 = log.startTransaction() txn2.commitManually(addCDC2 :: remove2 :: Nil: _*) val setTransaction3 = SetTransaction("fakeAppId", 3L, Some(200)) val txn3 = log.startTransaction() txn3.commitManually(Protocol(1, 2) :: setTransaction3 :: Nil: _*) } /////////////////////////////////////////////////////////////////////////// // io.delta.standalone.internal.ReadOnlyLogStoreSuite /////////////////////////////////////////////////////////////////////////// /** TEST: ReadOnlyLogStoreSuite > read */ generateGoldenTable("log-store-read") { tablePath => val log = DeltaLog.forTable(spark, new Path(tablePath)) assert(new File(log.logPath.toUri).mkdirs()) val deltas = Seq(0, 1).map(i => new File(tablePath, i.toString)).map(_.getCanonicalPath) log.store.write(deltas.head, Iterator("zero", "none")) log.store.write(deltas(1), Iterator("one")) } /** TEST: ReadOnlyLogStoreSuite > listFrom */ generateGoldenTable("log-store-listFrom") { tablePath => val log = DeltaLog.forTable(spark, new Path(tablePath)) assert(new File(log.logPath.toUri).mkdirs()) val deltas = Seq(0, 1, 2, 3, 4) .map(i => new File(tablePath, i.toString)) .map(_.getCanonicalPath) log.store.write(deltas(1), Iterator("zero")) log.store.write(deltas(2), Iterator("one")) log.store.write(deltas(3), Iterator("two")) } /////////////////////////////////////////////////////////////////////////// // io.delta.standalone.internal.DeltaTimeTravelSuite /////////////////////////////////////////////////////////////////////////// private implicit def durationToLong(duration: FiniteDuration): Long = { duration.toMillis } /** Generate commits with the given timestamp in millis. */ private def generateCommits(location: String, commits: Long*): Unit = { val deltaLog = DeltaLog.forTable(spark, location) var startVersion = deltaLog.snapshot.version + 1 commits.foreach { ts => val rangeStart = startVersion * 10 val rangeEnd = rangeStart + 10 spark.range(rangeStart, rangeEnd).write.format("delta").mode("append").save(location) val file = new File(FileNames.unsafeDeltaFile(deltaLog.logPath, startVersion).toUri) file.setLastModified(ts) startVersion += 1 } } val start = 1540415658000L generateGoldenTable("time-travel-start") { tablePath => generateCommits(tablePath, start) } generateGoldenTable("time-travel-start-start20") { tablePath => copyDir("time-travel-start", "time-travel-start-start20") generateCommits(tablePath, start + 20.minutes) } generateGoldenTable("time-travel-start-start20-start40") { tablePath => copyDir("time-travel-start-start20", "time-travel-start-start20-start40") generateCommits(tablePath, start + 40.minutes) } /** * TEST: DeltaTimeTravelSuite > time travel with schema changes - should instantiate old schema */ generateGoldenTable("time-travel-schema-changes-a") { tablePath => spark.range(10).write.format("delta").mode("append").save(tablePath) } generateGoldenTable("time-travel-schema-changes-b") { tablePath => copyDir("time-travel-schema-changes-a", "time-travel-schema-changes-b") spark.range(10, 20).withColumn("part", 'id) .write.format("delta").mode("append").option("mergeSchema", true).save(tablePath) } /** * TEST: DeltaTimeTravelSuite > time travel with partition changes - should instantiate old schema */ generateGoldenTable("time-travel-partition-changes-a") { tablePath => spark.range(10).withColumn("part5", 'id % 5).write.format("delta") .partitionBy("part5").mode("append").save(tablePath) } generateGoldenTable("time-travel-partition-changes-b") { tablePath => copyDir("time-travel-partition-changes-a", "time-travel-partition-changes-b") spark.range(10, 20).withColumn("part2", 'id % 2) .write .format("delta") .partitionBy("part2") .mode("overwrite") .option("overwriteSchema", true) .save(tablePath) } /////////////////////////////////////////////////////////////////////////// // io.delta.standalone.internal.DeltaDataReaderSuite /////////////////////////////////////////////////////////////////////////// private def writeDataWithSchema(tblLoc: String, data: Seq[Row], schema: StructType): Unit = { val df = spark.createDataFrame(spark.sparkContext.parallelize(data), schema) df.write.format("delta").mode("append").save(tblLoc) } /** TEST: DeltaDataReaderSuite > read - primitives */ generateGoldenTable("data-reader-primitives") { tablePath => def createRow(i: Int): Row = { Row(i, i.longValue, i.toByte, i.shortValue, i % 2 == 0, i.floatValue, i.doubleValue, i.toString, Array[Byte](i.toByte, i.toByte), new JBigDecimal(i)) } def createRowWithNullValues(): Row = { Row(null, null, null, null, null, null, null, null, null, null) } val schema = new StructType() .add("as_int", IntegerType) .add("as_long", LongType) .add("as_byte", ByteType) .add("as_short", ShortType) .add("as_boolean", BooleanType) .add("as_float", FloatType) .add("as_double", DoubleType) .add("as_string", StringType) .add("as_binary", BinaryType) .add("as_big_decimal", DecimalType(1, 0)) val data = createRowWithNullValues() +: (0 until 10).map(createRow) writeDataWithSchema(tablePath, data, schema) } /** TEST: DeltaDataReaderSuite > data reader can read partition values */ generateGoldenTable("data-reader-partition-values") { tablePath => def createRow(i: Int): Row = { Row(i, i.longValue, i.toByte, i.shortValue, i % 2 == 0, i.floatValue, i.doubleValue, i.toString, "null", java.sql.Date.valueOf("2021-09-08"), java.sql.Timestamp.valueOf("2021-09-08 11:11:11"), new JBigDecimal(i), Array(Row(i), Row(i), Row(i)), Row(i.toString, i.toString, Row(i, i.toLong)), i.toString) } def createRowWithNullPartitionValues(): Row = { Row( // partition values null, null, null, null, null, null, null, null, null, null, null, null, // data values Array(Row(2), Row(2), Row(2)), Row("2", "2", Row(2, 2L)), "2") } val schema = new StructType() // partition fields .add("as_int", IntegerType) .add("as_long", LongType) .add("as_byte", ByteType) .add("as_short", ShortType) .add("as_boolean", BooleanType) .add("as_float", FloatType) .add("as_double", DoubleType) .add("as_string", StringType) .add("as_string_lit_null", StringType) .add("as_date", DateType) .add("as_timestamp", TimestampType) .add("as_big_decimal", DecimalType(1, 0)) // data fields .add("as_list_of_records", ArrayType(new StructType().add("val", IntegerType))) .add("as_nested_struct", new StructType() .add("aa", StringType) .add("ab", StringType) .add("ac", new StructType() .add("aca", IntegerType) .add("acb", LongType) ) ) .add("value", StringType) val data = (0 until 2).map(createRow) :+ createRowWithNullPartitionValues() val df = spark.createDataFrame(spark.sparkContext.parallelize(data), schema) df.write .format("delta") .partitionBy("as_int", "as_long", "as_byte", "as_short", "as_boolean", "as_float", "as_double", "as_string", "as_string_lit_null", "as_date", "as_timestamp", "as_big_decimal") .save(tablePath) } Seq("name", "id").foreach { columnMappingMode => generateGoldenTable(s"table-with-columnmapping-mode-$columnMappingMode") { tablePath => withSQLConf( ("spark.databricks.delta.properties.defaults.columnMapping.mode", columnMappingMode)) { generateCMIcebegCompatTableHelper(tablePath) } } } generateGoldenTable("table-with-icebegCompatV2Enabled") { tablePath => withSQLConf( ("spark.databricks.delta.properties.defaults.columnMapping.mode", "id"), ("spark.databricks.delta.properties.defaults.enableIcebergCompatV2", "true")) { generateCMIcebegCompatTableHelper(tablePath) } } def generateCMIcebegCompatTableHelper(tablePath: String): Unit = { val timeZone = java.util.TimeZone.getTimeZone("UTC") java.util.TimeZone.setDefault(timeZone) import java.sql._ val decimalType = DecimalType(10, 2) val allDataTypes = Seq( ByteType, ShortType, IntegerType, LongType, FloatType, DoubleType, decimalType, BooleanType, StringType, BinaryType, DateType, TimestampType ) var fields = allDataTypes.map(dt => { val name = if (dt.isInstanceOf[DecimalType]) { "decimal" } else { dt.toString } StructField(name, dt) }) fields = fields :+ StructField("nested_struct", new StructType() .add("aa", StringType) .add("ac", new StructType() .add("aca", IntegerType) ) ) fields = fields :+ StructField("array_of_prims", ArrayType(IntegerType)) fields = fields :+ StructField("array_of_arrays", ArrayType(ArrayType(IntegerType))) fields = fields :+ StructField("array_of_map_of_arrays", ArrayType(MapType(IntegerType, ArrayType(IntegerType)))) fields = fields :+ StructField( "array_of_structs", ArrayType(new StructType().add("ab", IntegerType))) fields = fields :+ StructField( "struct_of_arrays_maps_of_structs", new StructType() .add("aa", ArrayType(IntegerType)) .add("ab", MapType(ArrayType(IntegerType), new StructType().add("aca", IntegerType))) ) fields = fields :+ StructField( "map_of_prims", MapType(IntegerType, LongType) ) fields = fields :+ StructField( "map_of_rows", MapType(IntegerType, new StructType().add("ab", LongType)) ) fields = fields :+ StructField( "map_of_arrays", MapType(LongType, ArrayType(IntegerType)) ) fields = fields :+ StructField( "map_of_maps", MapType(LongType, MapType(IntegerType, IntegerType)) ) val schema = StructType(fields) def createRow(i: Int): Row = { Row( i.toByte, // byte i.toShort, // short i, // integer i.toLong, // long i.toFloat, // float i.toDouble, // double new java.math.BigDecimal(i), // decimal i % 2 == 0, // boolean i.toString, // string i.toString.getBytes, // binary Date.valueOf("2021-11-18"), // date new Timestamp(i.toLong), // timestamp Row(i.toString, Row(i)), // nested_struct scala.Array(i, i + 1), // array_of_prims scala.Array(scala.Array(i, i + 1), scala.Array(i + 2, i + 3)), // array_of_arrays scala.Array( Map(i -> scala.Array(2, 3), i + 1 -> scala.Array(4, 5))), // array_of_map_of_arrays scala.Array(Row(i), Row(i)), // array_of_structs Row( // struct_of_arrays_maps_of_structs scala.Array(i, i + 1), Map(scala.Array(i, i + 1) -> Row(i + 2)) ), Map(i -> (i + 1).toLong, (i + 2) -> (i + 3).toLong), // map_of_prims Map(i + 1 -> Row((i * 20).toLong)), // map_of_rows { val val1 = scala.Array(i, null, i + 1) val val2 = scala.Array[Integer]() Map( i.longValue() -> val1, (i + 1).longValue() -> val2 ) // map_of_arrays }, Map( // map_of_maps i.toLong -> Map(i -> i), (i + 1).toLong -> Map(i + 2 -> i) ) ) } def createNullRow(): Row = { Row(Seq.fill(schema.length)(null): _*) } val rows = Seq.range(0, 5).map(i => createRow(i)) ++ Seq(createNullRow()) val df = spark.createDataFrame(spark.sparkContext.parallelize(rows), schema) df.repartition(2) .write .format("delta") .save(tablePath) } /** TEST: DeltaDataReaderSuite > read - date types */ Seq("UTC", "Iceland", "PST", "America/Los_Angeles", "Etc/GMT+9", "Asia/Beirut", "JST").foreach { timeZoneId => generateGoldenTable(s"data-reader-date-types-$timeZoneId") { tablePath => val timeZone = TimeZone.getTimeZone(timeZoneId) TimeZone.setDefault(timeZone) val timestamp = Timestamp.valueOf("2020-01-01 08:09:10") val date = java.sql.Date.valueOf("2020-01-01") val data = Row(timestamp, date) :: Nil val schema = new StructType() .add("timestamp", TimestampType) .add("date", DateType) writeDataWithSchema(tablePath, data, schema) } } /** TEST: DeltaDataReaderSuite > read - array of primitives */ generateGoldenTable("data-reader-array-primitives") { tablePath => def createRow(i: Int): Row = { Row(Array(i), Array(i.longValue), Array(i.toByte), Array(i.shortValue), Array(i % 2 == 0), Array(i.floatValue), Array(i.doubleValue), Array(i.toString), Array(Array(i.toByte, i.toByte)), Array(new JBigDecimal(i)) ) } val schema = new StructType() .add("as_array_int", ArrayType(IntegerType)) .add("as_array_long", ArrayType(LongType)) .add("as_array_byte", ArrayType(ByteType)) .add("as_array_short", ArrayType(ShortType)) .add("as_array_boolean", ArrayType(BooleanType)) .add("as_array_float", ArrayType(FloatType)) .add("as_array_double", ArrayType(DoubleType)) .add("as_array_string", ArrayType(StringType)) .add("as_array_binary", ArrayType(BinaryType)) .add("as_array_big_decimal", ArrayType(DecimalType(1, 0))) val data = (0 until 10).map(createRow) writeDataWithSchema(tablePath, data, schema) } /** TEST: DeltaDataReaderSuite > read - array of complex objects */ generateGoldenTable("data-reader-array-complex-objects") { tablePath => def createRow(i: Int): Row = { Row( i, Array(Array(Array(i, i, i), Array(i, i, i)), Array(Array(i, i, i), Array(i, i, i))), Array( Array(Array(Array(i, i, i), Array(i, i, i)), Array(Array(i, i, i), Array(i, i, i))), Array(Array(Array(i, i, i), Array(i, i, i)), Array(Array(i, i, i), Array(i, i, i))) ), Array( Map[String, Long](i.toString -> i.toLong), Map[String, Long](i.toString -> i.toLong) ), Array(Row(i), Row(i), Row(i)) ) } val schema = new StructType() .add("i", IntegerType) .add("3d_int_list", ArrayType(ArrayType(ArrayType(IntegerType)))) .add("4d_int_list", ArrayType(ArrayType(ArrayType(ArrayType(IntegerType))))) .add("list_of_maps", ArrayType(MapType(StringType, LongType))) .add("list_of_records", ArrayType(new StructType().add("val", IntegerType))) val data = (0 until 10).map(createRow) writeDataWithSchema(tablePath, data, schema) } /** TEST: DeltaDataReaderSuite > read - map */ generateGoldenTable("data-reader-map") { tablePath => def createRow(i: Int): Row = { Row( i, Map(i -> i), Map(i.toLong -> i.toByte), Map(i.toShort -> (i % 2 == 0)), Map(i.toFloat -> i.toDouble), Map(i.toString -> new JBigDecimal(i)), Map(i -> Array(Row(i), Row(i), Row(i))) ) } val schema = new StructType() .add("i", IntegerType) .add("a", MapType(IntegerType, IntegerType)) .add("b", MapType(LongType, ByteType)) .add("c", MapType(ShortType, BooleanType)) .add("d", MapType(FloatType, DoubleType)) .add("e", MapType(StringType, DecimalType(1, 0))) .add("f", MapType(IntegerType, ArrayType(new StructType().add("val", IntegerType)))) val data = (0 until 10).map(createRow) writeDataWithSchema(tablePath, data, schema) } /** TEST: DeltaDataReaderSuite > read - nested struct */ generateGoldenTable("data-reader-nested-struct") { tablePath => def createRow(i: Int): Row = Row(Row(i.toString, i.toString, Row(i, i.toLong)), i) val schema = new StructType() .add("a", new StructType() .add("aa", StringType) .add("ab", StringType) .add("ac", new StructType() .add("aca", IntegerType) .add("acb", LongType) ) ) .add("b", IntegerType) val data = (0 until 10).map(createRow) writeDataWithSchema(tablePath, data, schema) } /** TEST: DeltaDataReaderSuite > read - nullable field, invalid schema column key */ generateGoldenTable("data-reader-nullable-field-invalid-schema-key") { tablePath => val data = Row(Seq(null, null, null)) :: Nil val schema = new StructType() .add("array_can_contain_null", ArrayType(StringType, containsNull = true)) writeDataWithSchema(tablePath, data, schema) } /** TEST: DeltaDataReaderSuite > test escaped char sequences in path */ generateGoldenTable("data-reader-escaped-chars") { tablePath => val data = Seq("foo1" -> "bar+%21", "foo2" -> "bar+%22", "foo3" -> "bar+%23") data.foreach { row => Seq(row).toDF().write.format("delta").mode("append").partitionBy("_2").save(tablePath) } } /** TEST: DeltaDataReaderSuite > #124: decimal decode bug */ generateGoldenTable("124-decimal-decode-bug") { tablePath => val data = Seq(Row(new JBigDecimal(1000000))) val schema = new StructType().add("large_decimal", DecimalType(10, 0)) writeDataWithSchema(tablePath, data, schema) } /** TEST: DeltaDataReaderSuite > #125: iterator bug */ generateGoldenTable("125-iterator-bug") { tablePath => val datas = Seq( Seq(), Seq(1), Seq(2), Seq(), Seq(3), Seq(), Seq(), Seq(4), Seq(), Seq(), Seq(), Seq(5) ) datas.foreach { data => data.toDF("col1").write.format("delta").mode("append").save(tablePath) } } generateGoldenTable("deltatbl-not-allow-write", createHiveGoldenTableFile) { tablePath => val data = (0 until 10).map(x => (x, s"foo${x % 2}")) data.toDF("a", "b").write.format("delta").save(tablePath) } generateGoldenTable("deltatbl-schema-match", createHiveGoldenTableFile) { tablePath => val data = (0 until 10).map(x => (x, s"foo${x % 2}", s"test${x % 3}")) data.toDF("a", "b", "c").write.format("delta").partitionBy("b").save(tablePath) } generateGoldenTable("deltatbl-non-partitioned", createHiveGoldenTableFile) { tablePath => val data = (0 until 10).map(x => (x, s"foo${x % 2}")) data.toDF("c1", "c2").write.format("delta").save(tablePath) } generateGoldenTable("deltatbl-partitioned", createHiveGoldenTableFile) { tablePath => val data = (0 until 10).map(x => (x, s"foo${x % 2}")) data.toDF("c1", "c2").write.format("delta").partitionBy("c2").save(tablePath) } generateGoldenTable("deltatbl-partition-prune", createHiveGoldenTableFile) { tablePath => val data = Seq( ("hz", "20180520", "Jim", 3), ("hz", "20180718", "Jone", 7), ("bj", "20180520", "Trump", 1), ("sh", "20180512", "Jay", 4), ("sz", "20181212", "Linda", 8) ) data.toDF("city", "date", "name", "cnt") .write.format("delta").partitionBy("date", "city").save(tablePath) } generateGoldenTable("deltatbl-touch-files-needed-for-partitioned", createHiveGoldenTableFile) { tablePath => val data = (0 until 10).map(x => (x, s"foo${x % 2}")) data.toDF("c1", "c2").write.format("delta").partitionBy("c2").save(tablePath) } generateGoldenTable("deltatbl-special-chars-in-partition-column", createHiveGoldenTableFile) { tablePath => val data = (0 until 10).map(x => (x, s"+ =%${x % 2}")) data.toDF("c1", "c2").write.format("delta").partitionBy("c2").save(tablePath) } generateGoldenTable("deltatbl-map-types-correctly", createHiveGoldenTableFile) { tablePath => val data = Seq( TestClass( 97.toByte, Array(98.toByte, 99.toByte), true, 4, 5L, "foo", 6.0f, 7.0, 8.toShort, new java.sql.Date(60000000L), new java.sql.Timestamp(60000000L), new java.math.BigDecimal(12345.6789), Array("foo", "bar"), Map("foo" -> 123L), TestStruct("foo", 456L) ) ) data.toDF.write.format("delta").save(tablePath) } generateGoldenTable("deltatbl-column-names-case-insensitive", createHiveGoldenTableFile) { tablePath => val data = (0 until 10).map(x => (x, s"foo${x % 2}")) data.toDF("FooBar", "BarFoo").write.format("delta").partitionBy("BarFoo").save(tablePath) } generateGoldenTable("deltatbl-deleted-path", createHiveGoldenTableFile) { tablePath => val data = (0 until 10).map(x => (x, s"foo${x % 2}")) data.toDF("c1", "c2").write.format("delta").save(tablePath) } generateGoldenTable("deltatbl-incorrect-format-config", createHiveGoldenTableFile) { tablePath => val data = (0 until 10).map(x => (x, s"foo${x % 2}")) data.toDF("a", "b").write.format("delta").save(tablePath) } generateGoldenTable("dv-partitioned-with-checkpoint") { tablePath => withSQLConf(("spark.databricks.delta.properties.defaults.enableDeletionVectors", "true")) { val data = (0 until 50).map(x => (x%10, x, s"foo${x % 5}")) data.toDF("part", "col1", "col2").write .format("delta") .partitionBy("part") .save(tablePath) (0 until 15).foreach { n => spark.sql(s"DELETE FROM delta.`$tablePath` WHERE col1 = ${n*2}") } } } generateGoldenTable("dv-with-columnmapping") { tablePath => withSQLConf( ("spark.databricks.delta.properties.defaults.columnMapping.mode", "name"), ("spark.databricks.delta.properties.defaults.enableDeletionVectors", "true")) { val data = (0 until 50).map(x => (x%10, x, s"foo${x % 5}")) data.toDF("part", "col1", "col2").write .format("delta") .partitionBy("part") .save(tablePath) (0 until 15).foreach { n => spark.sql(s"DELETE FROM delta.`$tablePath` WHERE col1 = ${n*2}") } } } def writeBasicTimestampTable(path: String, timeZone: TimeZone): Unit = { TimeZone.setDefault(timeZone) // Create a partition value of both {year}-{month}-{day} {hour}:{minute}:{second} format and // {year}-{month}-{day} {hour}:{minute}:{second}.{microsecond} val data = Row(0, Timestamp.valueOf("2020-01-01 08:09:10.001"), Timestamp.valueOf("2020-02-01 08:09:10")) :: Row(1, Timestamp.valueOf("2021-10-01 08:09:20"), Timestamp.valueOf("1999-01-01 09:00:00")) :: Row(2, Timestamp.valueOf("2021-10-01 08:09:20"), Timestamp.valueOf("2000-01-01 09:00:00")) :: Row(3, Timestamp.valueOf("1969-01-01 00:00:00"), Timestamp.valueOf("1969-01-01 00:00:00")) :: Row(4, null, null) :: Nil val schema = new StructType() .add("id", IntegerType) .add("part", TimestampType) .add("time", TimestampType) spark.createDataFrame(spark.sparkContext.parallelize(data), schema) .write .format("delta") .partitionBy("part") .save(path) } for (parquetTimestampType <- SQLConf.ParquetOutputTimestampType.values) { generateGoldenTable(s"kernel-timestamp-${parquetTimestampType.toString}") { tablePath => withSQLConf(("spark.sql.parquet.outputTimestampType", parquetTimestampType.toString)) { writeBasicTimestampTable(tablePath, TimeZone.getTimeZone("UTC")) } } } generateGoldenTable("kernel-timestamp-PST") { tablePath => writeBasicTimestampTable(tablePath, TimeZone.getTimeZone("PST")) } generateGoldenTable("parquet-all-types-legacy-format") { tablePath => withSQLConf(("spark.sql.parquet.writeLegacyFormat", "true")) { generateAllTypesTable(tablePath) } } generateGoldenTable("parquet-all-types") { tablePath => // generating using the standard parquet format generateAllTypesTable(tablePath) } def generateAllTypesTable(tablePath: String): Unit = { val timeZone = java.util.TimeZone.getTimeZone("UTC") java.util.TimeZone.setDefault(timeZone) import java.sql._ val decimalType = DecimalType(10, 2) val allDataTypes = Seq( ByteType, ShortType, IntegerType, LongType, FloatType, DoubleType, decimalType, BooleanType, StringType, BinaryType, DateType, TimestampType, TimestampNTZType ) var fields = allDataTypes.map(dt => { val name = if (dt.isInstanceOf[DecimalType]) { "decimal" } else { dt.toString } StructField(name, dt) }) fields = fields :+ StructField("nested_struct", new StructType() .add("aa", StringType) .add("ac", new StructType() .add("aca", IntegerType) ) ) fields = fields :+ StructField("array_of_prims", ArrayType(IntegerType)) fields = fields :+ StructField("array_of_arrays", ArrayType(ArrayType(IntegerType))) fields = fields :+ StructField( "array_of_structs", ArrayType(new StructType().add("ab", LongType))) fields = fields :+ StructField( "map_of_prims", MapType(IntegerType, LongType) ) fields = fields :+ StructField( "map_of_rows", MapType(IntegerType, new StructType().add("ab", LongType)) ) fields = fields :+ StructField( "map_of_arrays", MapType(LongType, ArrayType(IntegerType)) ) val schema = StructType(fields) def createRow(i: Int): Row = { Row( if (i % 72 != 0) i.byteValue() else null, if (i % 56 != 0) i.shortValue() else null, if (i % 23 != 0) i else null, if (i % 25 != 0) (i + 1).longValue() else null, if (i % 28 != 0) (i * 0.234).floatValue() else null, if (i % 54 != 0) (i * 234234.23).doubleValue() else null, if (i % 67 != 0) new java.math.BigDecimal(i * 123.52) else null, if (i % 87 != 0) i % 2 == 0 else null, if (i % 57 != 0) (i).toString else null, if (i % 59 != 0) (i).toString.getBytes else null, if (i % 61 != 0) new java.sql.Date(i * 20000000L) else null, if (i % 62 != 0) new Timestamp(i * 23423523L) else null, if (i % 69 != 0) LocalDateTime.ofEpochSecond(i * 234234L, 200012, UTC) else null, // nested_struct if (i % 63 != 0) { if (i % 19 == 0) { // write a struct with all fields null Row(null, null) } else { Row(i.toString, if (i % 23 != 0) Row(i) else null) } } else null, // array_of_prims if (i % 25 != 0) { if (i % 29 == 0) { scala.Array() } else { scala.Array(i, null, i + 1) } } else null, // array_of_arrays if (i % 8 != 0) { val singleElemArray = scala.Array(i) val doubleElemArray = scala.Array(i + 10, i + 20) val arrayWithNulls = scala.Array(null, i + 200) val singleElemNullArray = scala.Array(null) val emptyArray = scala.Array() (i % 7) match { case 0 => scala.Array(singleElemArray, singleElemArray, arrayWithNulls) case 1 => scala.Array(singleElemArray, doubleElemArray, emptyArray) case 2 => scala.Array(arrayWithNulls) case 3 => scala.Array(singleElemNullArray) case 4 => scala.Array(null) case 5 => scala.Array(emptyArray) case 6 => scala.Array() } } else null, // array_of_structs if (i % 10 != 0) { scala.Array(Row(i.longValue()), null) } else null, // map_of_prims if (i % 28 != 0) { if (i % 30 == 0) { Map() } else { Map( i -> (if (i % 29 != 0) (i + 2).longValue() else null), (if (i % 27 != 0) i + 2 else i + 3) -> (i + 9).longValue() ) } } else null, // map_of_rows if (i % 25 != 0) { Map(i + 1 -> (if (i % 10 == 0) Row((i * 20).longValue()) else null)) } else null, // map_of_arrays if (i % 30 != 0) { if (i % 24 == 0) { Map() } else { val val1 = if (i % 4 == 0) scala.Array(i, null, i + 1) else scala.Array() val val2 = if (i % 7 == 0) scala.Array[Integer]() else scala.Array[Integer](null) Map( i.longValue() -> val1, (i + 1).longValue() -> val2 ) } } else null ) } val rows = Seq.range(0, 200).map(i => createRow(i)) val df = spark.createDataFrame(spark.sparkContext.parallelize(rows), schema) df.repartition(1) .write .format("delta") .mode("append") .save(tablePath) } def writeBasicDecimalTable(tablePath: String): Unit = { val data = Seq( Seq("234", "1", "2", "3"), Seq("2342222.23454", "111.11", "22222.22222", "3333333333.3333333333"), Seq("0.00004", "0.001", "0.000002", "0.00000000003"), Seq("-2342342.23423", "-999.99", "-99999.99999", "-9999999999.9999999999") ).map(_.map(new JBigDecimal(_))).map(Row(_: _*)) val schema = new StructType() .add("part", new DecimalType(12, 5)) // serialized to a string .add("col1", new DecimalType(5, 2)) // INT32: 1 <= precision <= 9 .add("col2", new DecimalType(10, 5)) // INT64: 10 <= precision <= 18 .add("col3", new DecimalType(20, 10)) // FIXED_LEN_BYTE_ARRAY spark.createDataFrame(spark.sparkContext.parallelize(data), schema) .repartition(1) .write .format("delta") .partitionBy("part") .save(tablePath) } generateGoldenTable("basic-decimal-table") { tablePath => writeBasicDecimalTable(tablePath) } generateGoldenTable("basic-decimal-table-legacy") { tablePath => withSQLConf(("spark.sql.parquet.writeLegacyFormat", "true")) { writeBasicDecimalTable(tablePath) } } generateGoldenTable("decimal-various-scale-precision") { tablePath => val fields = ArrayBuffer[StructField]() Seq(0, 4, 7, 12, 15, 18, 25, 35, 38).foreach { precision => Seq.range(start = 0, end = precision, step = 6).foreach { scale => fields.append( StructField(s"decimal_${precision}_${scale}", DecimalType(precision, scale))) } } val schema = StructType(fields) val random = new Random(27 /* seed */) def generateRandomBigDecimal(precision: Int, scale: Int): JBigDecimal = { // Generate a random BigInteger with the specified precision val unscaledValue = new BigInteger(precision, random) // Create a BigDecimal with the unscaled value and the specified scale new JBigDecimal(unscaledValue, scale) } val rows = ArrayBuffer[Row]() Seq.range(start = 0, end = 3).foreach { i => val rowValues = ArrayBuffer[BigDecimal]() Seq(0, 4, 7, 12, 15, 18, 25, 35, 38).foreach { precision => Seq.range(start = 0, end = precision, step = 3).foreach { scale => i match { case 0 => rowValues.append(null) case 1 => // Generate a positive random BigDecimal with the specified precision and scale rowValues.append(generateRandomBigDecimal(precision, scale)) case 2 => // Generate a negative random BigDecimal with the specified precision and scale rowValues.append(generateRandomBigDecimal(precision, scale).negate()) } } } rows.append(Row(rowValues: _*)) } spark.createDataFrame(spark.sparkContext.parallelize(rows), schema) .repartition(1) .write .format("delta") .save(tablePath) } for (parquetFormat <- Seq("v1", "v2")) { // PARQUET_1_0 doesn't support dictionary encoding for FIXED_LEN_BYTE_ARRAY (only PARQUET_2_0) generateGoldenTable(s"parquet-decimal-dictionaries-$parquetFormat") { tablePath => def withHadoopConf(key: String, value: String)(f: => Unit): Unit = { try { spark.sparkContext.hadoopConfiguration.set(key, value) f } finally { spark.sparkContext.hadoopConfiguration.unset(key) } } withHadoopConf("parquet.writer.version", parquetFormat) { val data = (0 until 1000000).map { i => Row(i, JBigDecimal.valueOf(i % 5), JBigDecimal.valueOf(i % 6), JBigDecimal.valueOf(i % 2)) } val schema = new StructType() .add("id", IntegerType) .add("col1", new DecimalType(9, 0)) // INT32: 1 <= precision <= 9 .add("col2", new DecimalType(12, 0)) // INT64: 10 <= precision <= 18 .add("col3", new DecimalType(25, 0)) // FIXED_LEN_BYTE_ARRAY spark.createDataFrame(spark.sparkContext.parallelize(data), schema) .repartition(1) .write .format("delta") .save(tablePath) } } } generateGoldenTable("parquet-decimal-type") { tablePath => def expand(n: JBigDecimal): JBigDecimal = { n.scaleByPowerOfTen(5).add(n) } val data = (0 until 99998).map { i => if (i % 85 == 0) { val n = JBigDecimal.valueOf(i) Row(i, n.movePointLeft(1), n, n) } else { val negation = if (i % 33 == 0) { -1 } else { 1 } val n = JBigDecimal.valueOf(i*negation) Row( i, n.movePointLeft(1), expand(n).movePointLeft(5), expand(expand(expand(n))).movePointLeft(5) ) } } val schema = new StructType() .add("id", IntegerType) .add("col1", new DecimalType(5, 1)) // INT32: 1 <= precision <= 9 .add("col2", new DecimalType(10, 5)) // INT64: 10 <= precision <= 18 .add("col3", new DecimalType(20, 5)) // FIXED_LEN_BYTE_ARRAY spark.createDataFrame(spark.sparkContext.parallelize(data), schema) .repartition(1) .write .format("delta") .save(tablePath) } /* START: TIMESTAMP_NTZ golden tables */ def generateTimestampNtzTable(tablePath: String): Unit = { spark.sql( s""" | CREATE TABLE delta.`$tablePath`(id INTEGER, tsNtz TIMESTAMP_NTZ, tsNtzPartition TIMESTAMP_NTZ) | USING DELTA | PARTITIONED BY(tsNtzPartition) """.stripMargin) spark.sql( s""" | INSERT INTO delta.`$tablePath` VALUES | (0, '2021-11-18 02:30:00.123456','2021-11-18 02:30:00.123456'), | (1, '2013-07-05 17:01:00.123456','2021-11-18 02:30:00.123456'), | (2, NULL,'2021-11-18 02:30:00.123456'), | (3, '2021-11-18 02:30:00.123456','2013-07-05 17:01:00.123456'), | (4, '2013-07-05 17:01:00.123456','2013-07-05 17:01:00.123456'), | (5, NULL,'2013-07-05 17:01:00.123456'), | (6, '2021-11-18 02:30:00.123456', NULL), | (7, '2013-07-05 17:01:00.123456', NULL), | (8, NULL, NULL) |""".stripMargin) } generateGoldenTable("data-reader-timestamp_ntz") { tablePath => generateTimestampNtzTable(tablePath) } Seq("id", "name").foreach { columnMappingMode => { generateGoldenTable(s"data-reader-timestamp_ntz-$columnMappingMode-mode") { tablePath => withSQLConf( ("spark.databricks.delta.properties.defaults.columnMapping.mode", columnMappingMode)) { generateTimestampNtzTable(tablePath) } } } } /* END: TIMESTAMP_NTZ golden tables */ generateGoldenTable("basic-with-inserts-deletes-checkpoint") { tablePath => // scalastyle:off line.size.limit spark.range(0, 10).repartition(1).write.format("delta").mode("append").save(tablePath) spark.range(10, 20).repartition(1).write.format("delta").mode("append").save(tablePath) spark.range(20, 30).repartition(1).write.format("delta").mode("append").save(tablePath) spark.range(30, 40).repartition(1).write.format("delta").mode("append").save(tablePath) spark.range(40, 50).repartition(1).write.format("delta").mode("append").save(tablePath) sql(s"DELETE FROM delta.`$tablePath` WHERE id >= 5 AND id <= 9") sql(s"DELETE FROM delta.`$tablePath` WHERE id >= 15 AND id <= 19") sql(s"DELETE FROM delta.`$tablePath` WHERE id >= 25 AND id <= 29") sql(s"DELETE FROM delta.`$tablePath` WHERE id >= 35 AND id <= 39") sql(s"DELETE FROM delta.`$tablePath` WHERE id >= 45 AND id <= 49") spark.range(50, 60).repartition(1).write.format("delta").mode("append").save(tablePath) spark.range(60, 70).repartition(1).write.format("delta").mode("append").save(tablePath) spark.range(70, 80).repartition(1).write.format("delta").mode("append").save(tablePath) sql(s"DELETE FROM delta.`$tablePath` WHERE id >= 66") // scalastyle:on line.size.limit } generateGoldenTable("multi-part-checkpoint") { tablePath => withSQLConf( ("spark.databricks.delta.checkpoint.partSize", "5"), ("spark.databricks.delta.properties.defaults.checkpointInterval", "1") ) { spark.range(1).repartition(1).write.format("delta").save(tablePath) spark.range(30).repartition(9).write.format("delta").mode("append").save(tablePath) } } Seq("parquet", "json").foreach { ckptFormat => val tbl = "tbl" generateGoldenTable(s"v2-checkpoint-$ckptFormat") { tablePath => withTable(tbl) { withSQLConf( (DeltaSQLConf.DELTA_CHECKPOINT_PART_SIZE.key, "2"), ("spark.databricks.delta.properties.defaults.checkpointInterval", "2"), (DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key, ckptFormat)) { spark.conf.set(DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key, ckptFormat) sql(s"CREATE TABLE $tbl (id LONG) USING delta LOCATION '$tablePath'") sql(s"ALTER TABLE $tbl SET TBLPROPERTIES('delta.checkpointPolicy' = 'v2')") spark.range(10).repartition(4) .write.format("delta").mode("append").saveAsTable(tbl) } } } } generateGoldenTable("no-delta-log-folder") { tablePath => spark.range(20).write.format("parquet").save(tablePath) } generateGoldenTable("log-replay-latest-metadata-protocol") { tablePath => spark.range(20).toDF("col1") .write.format("delta").save(tablePath) // update the table schema spark.range(20).toDF("col1").withColumn("col2", 'col1 % 2) .write.format("delta").mode("append").option("mergeSchema", "true").save(tablePath) // update the protocol version DeltaTable.forPath(spark, tablePath).upgradeTableProtocol(3, 7) } generateGoldenTable("only-checkpoint-files") { tablePath => withSQLConf(("spark.databricks.delta.properties.defaults.checkpointInterval", "1")) { spark.range(10).repartition(10).write.format("delta").save(tablePath) spark.sql(s"DELETE FROM delta.`$tablePath` WHERE id < 5") spark.range(20).write.format("delta").mode("append").save(tablePath) } } generateGoldenTable("log-replay-special-characters-a") { tablePath => val log = DeltaLog.forTable(spark, new Path(tablePath)) new File(log.logPath.toUri).mkdirs() val add = AddFile(new Path("special p@#h").toUri.toString, Map.empty, 100L, 10L, dataChange = true) val remove = add.remove log.startTransaction().commitManually(add) log.startTransaction().commitManually(remove) } generateGoldenTable("log-replay-special-characters-b") { tablePath => val log = DeltaLog.forTable(spark, new Path(tablePath)) new File(log.logPath.toUri).mkdirs() val add = AddFile(new Path("special p@#h").toUri.toString, Map.empty, 100L, 10L, dataChange = true) log.startTransaction().commitManually(add) } generateGoldenTable("log-replay-dv-key-cases") { tablePath => withSQLConf(("spark.databricks.delta.properties.defaults.enableDeletionVectors", "true")) { spark.range(50).repartition(1).write.format("delta").save(tablePath) (0 until 3).foreach { n => spark.sql(s"DELETE FROM delta.`$tablePath` WHERE id = ${n*7}") } } } generateGoldenTable("basic-with-vacuum-protocol-check-feature") { tablePath => val data = (0 until 100).map(x => (x, s"val=$x")) data.toDF("id", "str").write.format("delta").save(tablePath) sql(s""" |ALTER TABLE delta.`$tablePath` |SET TBLPROPERTIES('delta.feature.vacuumProtocolCheck' = 'supported') |""".stripMargin) } generateGoldenTable("basic-with-inserts-updates") { tablePath => val data = (0 until 100).map(x => (x, s"val=$x")) data.toDF("id", "str").write.format("delta").save(tablePath) sql(s"UPDATE delta.`$tablePath` SET str = 'N/A' WHERE id < 50") } generateGoldenTable("basic-with-inserts-merge") { tablePath => val data = (0 until 100).map(x => (x, s"val=$x")) data.toDF("id", "str").write.format("delta").save(tablePath) spark.range(50, 150).createTempView("source") sql( s""" |MERGE INTO delta.`$tablePath` t |USING source |ON source.id = t.id |WHEN MATCHED | THEN UPDATE SET str = 'N/A' |WHEN NOT MATCHED | THEN INSERT (id, str) VALUES (source.id, 'EXT') |WHEN NOT MATCHED BY SOURCE AND t.id < 10 | THEN DELETE |""".stripMargin) } generateGoldenTable("basic-with-inserts-overwrite-restore") { tablePath => spark.range(100).write.format("delta").save(tablePath) spark.range(100, 200).write.format("delta").mode("append").save(tablePath) spark.range(500, 1000).write.format("delta").mode("overwrite").save(tablePath) sql(s"RESTORE TABLE delta.`$tablePath` TO VERSION AS OF 1") } /* ----- Data skipping tables for Kernel ------ */ def writeBasicStatsAllTypesTable(tablePath: String): Unit = { val schema = new StructType() .add("as_int", IntegerType) .add("as_long", LongType) .add("as_byte", ByteType) .add("as_short", ShortType) .add("as_float", FloatType) .add("as_double", DoubleType) .add("as_string", StringType) .add("as_date", DateType) .add("as_timestamp", TimestampType) .add("as_big_decimal", DecimalType(1, 0)) writeDataWithSchema( tablePath, Row(0, 0.longValue, 0.byteValue, 0.shortValue, 0.floatValue, 0.doubleValue, "0", java.sql.Date.valueOf("2000-01-01"), Timestamp.valueOf("2000-01-01 00:00:00"), new JBigDecimal(0)) :: Nil, schema ) } generateGoldenTable("data-skipping-basic-stats-all-types") { tablePath => writeBasicStatsAllTypesTable(tablePath) } Seq("name", "id").foreach { columnMappingMode => generateGoldenTable(s"data-skipping-basic-stats-all-types-columnmapping-$columnMappingMode") { tablePath => withSQLConf( ("spark.databricks.delta.properties.defaults.columnMapping.mode", columnMappingMode)) { writeBasicStatsAllTypesTable(tablePath) } } } generateGoldenTable("data-skipping-basic-stats-all-types-checkpoint") { tablePath => withSQLConf( ("spark.databricks.delta.properties.defaults.checkpointInterval", "1") ) { writeBasicStatsAllTypesTable(tablePath) } } generateGoldenTable("data-skipping-change-stats-collected-across-versions") { tablePath => val schema = new StructType() .add("col1", IntegerType) .add("col2", IntegerType) // write stats for all columns writeDataWithSchema( tablePath, Row(0, 0) :: Nil, schema ) // write stats for just 1 column sql( s""" |ALTER TABLE delta.`$tablePath` |SET TBLPROPERTIES('delta.dataSkippingNumIndexedCols' = 1) |""".stripMargin) writeDataWithSchema( tablePath, Row(0, 0) :: Nil, schema) // write stats for no columns sql( s""" |ALTER TABLE delta.`$tablePath` |SET TBLPROPERTIES('delta.dataSkippingNumIndexedCols' = 0) |""".stripMargin) writeDataWithSchema( tablePath, Row(0, 0) :: Nil, schema) } generateGoldenTable("data-skipping-partition-and-data-column") { tablePath => val schema = new StructType() .add("part", IntegerType) .add("id", IntegerType) writeDataWithSchema( tablePath, Row(1, 0) :: Nil, schema ) writeDataWithSchema( tablePath, Row(1, 1) :: Nil, schema) writeDataWithSchema( tablePath, Row(0, 1) :: Nil, schema) writeDataWithSchema( tablePath, Row(0, 0) :: Nil, schema) } generateGoldenTable("commit-info-containing-arbitrary-operationParams-types") { tablePath => spark.sql( f""" |CREATE TABLE delta.`$tablePath` |USING DELTA |PARTITIONED BY (month) |TBLPROPERTIES (delta.enableChangeDataFeed = true) |AS |SELECT 1 AS id, 1 AS month""".stripMargin) // Add some data spark.sql("INSERT INTO delta.`%s` VALUES (2, 2)".format(tablePath)) // Run optimize that generates a commitInfo with arbitrary value types // operationParameters spark.sql("OPTIMIZE delta.`%s` ZORDER BY id".format(tablePath)) } } case class TestStruct(f1: String, f2: Long) /** A special test class that covers all Spark types we support in the Hive connector. */ case class TestClass( c1: Byte, c2: Array[Byte], c3: Boolean, c4: Int, c5: Long, c6: String, c7: Float, c8: Double, c9: Short, c10: java.sql.Date, c11: java.sql.Timestamp, c12: BigDecimal, c13: Array[String], c14: Map[String, Long], c15: TestStruct ) case class OneItem[T](t: T) ================================================ FILE: connectors/licenses/LICENSE-apache-spark.txt ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: connectors/licenses/LICENSE-parquet4s.txt ================================================ MIT License Copyright (c) 2018 Marcin Jakubowski Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: contribs/src/main/scala/io/delta/storage/IBMCOSLogStore.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage import com.google.common.base.Throwables import java.io.IOException import java.nio.charset.StandardCharsets.UTF_8 import java.nio.file.FileAlreadyExistsException import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.annotation.Unstable /** * :: Unstable :: * * LogStore implementation for IBM Cloud Object Storage. * * We assume the following from COS's [[FileSystem]] implementations: * - Write on COS is all-or-nothing, whether overwrite or not. * - Write is atomic. * Note: Write is atomic when using the Stocator v1.1.1+ - Storage Connector for Apache Spark * (https://github.com/CODAIT/stocator) by setting the configuration `fs.cos.atomic.write` to true * (for more info see the documentation for Stocator) * - List-after-write is consistent. * * @note This class is not meant for direct access but for configuration based on storage system. * See https://docs.delta.io/latest/delta-storage.html for details. */ @Unstable class IBMCOSLogStore(sparkConf: SparkConf, initHadoopConf: Configuration) extends org.apache.spark.sql.delta.storage.HadoopFileSystemLogStore(sparkConf, initHadoopConf) { val preconditionFailedExceptionMessage = "At least one of the preconditions you specified did not hold" assert(initHadoopConf.getBoolean("fs.cos.atomic.write", false) == true, "'fs.cos.atomic.write' must be set to true to use IBMCOSLogStore " + "in order to enable atomic write") override def write(path: Path, actions: Iterator[String], overwrite: Boolean = false): Unit = { write(path, actions, overwrite, getHadoopConfiguration) } override def write( path: Path, actions: Iterator[String], overwrite: Boolean, hadoopConf: Configuration): Unit = { val fs = path.getFileSystem(hadoopConf) val exists = fs.exists(path) if (exists && overwrite == false) { throw new FileAlreadyExistsException(path.toString) } else { // write is atomic when overwrite == false val stream = fs.create(path, overwrite) try { actions.map(_ + "\n").map(_.getBytes(UTF_8)).foreach(stream.write) stream.close() } catch { case e: IOException if isPreconditionFailure(e) => if (fs.exists(path)) { throw new FileAlreadyExistsException(path.toString) } else { throw new IllegalStateException(s"Failed due to concurrent write", e) } } } } private def isPreconditionFailure(x: Throwable): Boolean = { Throwables.getCausalChain(x) .stream() .filter(p => p != null) .filter(p => p.getMessage != null) .filter(p => p.getMessage.contains(preconditionFailedExceptionMessage)) .findFirst .isPresent; } override def invalidateCache(): Unit = {} override def isPartialWriteVisible(path: Path): Boolean = false override def isPartialWriteVisible(path: Path, hadoopConf: Configuration): Boolean = false } ================================================ FILE: contribs/src/main/scala/io/delta/storage/OracleCloudLogStore.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.annotation.Unstable /** * :: Unstable :: * * LogStore implementation for OCI (Oracle Cloud Infrastructure). * * We assume the following from OCI (Oracle Cloud Infrastructure)'s BmcFilesystem implementations: * - Rename without overwrite is atomic. * - List-after-write is consistent. * * Regarding file creation, this implementation: * - Uses atomic rename when overwrite is false; if the destination file exists or the rename * fails, throws an exception. * - Uses create-with-overwrite when overwrite is true. This does not make the file atomically * visible and therefore the caller must handle partial files. * * @note This class is not meant for direct access but for configuration based on storage system. * See https://docs.delta.io/latest/delta-storage.html for details. */ @Unstable class OracleCloudLogStore(sparkConf: SparkConf, initHadoopConf: Configuration) extends org.apache.spark.sql.delta.storage.HadoopFileSystemLogStore(sparkConf, initHadoopConf) { override def write(path: Path, actions: Iterator[String], overwrite: Boolean = false): Unit = { write(path, actions, overwrite, getHadoopConfiguration) } override def write( path: Path, actions: Iterator[String], overwrite: Boolean, hadoopConf: Configuration): Unit = { writeWithRename(path, actions, overwrite, hadoopConf) } override def invalidateCache(): Unit = {} override def isPartialWriteVisible(path: Path): Boolean = true override def isPartialWriteVisible(path: Path, hadoopConf: Configuration): Boolean = true } ================================================ FILE: contribs/src/test/scala/io/delta/storage/IBMCOSLogStoreSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage import org.apache.spark.sql.delta.{FakeFileSystem, LogStoreSuiteBase} class IBMCOSLogStoreSuite extends LogStoreSuiteBase { protected override def sparkConf = { super.sparkConf.set(logStoreClassConfKey, logStoreClassName) .set("spark.hadoop.fs.cos.atomic.write", "true") } override val logStoreClassName: String = classOf[IBMCOSLogStore].getName testHadoopConf( expectedErrMsg = ".*No FileSystem for scheme.*fake.*", "fs.fake.impl" -> classOf[FakeFileSystem].getName, "fs.fake.impl.disable.cache" -> "true") protected def shouldUseRenameToWriteCheckpoint: Boolean = false } ================================================ FILE: contribs/src/test/scala/io/delta/storage/OracleCloudLogStoreSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage import org.apache.spark.sql.delta.{FakeFileSystem, LogStoreSuiteBase} class OracleCloudLogStoreSuite extends LogStoreSuiteBase { override val logStoreClassName: String = classOf[OracleCloudLogStore].getName testHadoopConf( expectedErrMsg = "No FileSystem for scheme \"fake\"", "fs.fake.impl" -> classOf[FakeFileSystem].getName, "fs.fake.impl.disable.cache" -> "true") protected def shouldUseRenameToWriteCheckpoint: Boolean = true } ================================================ FILE: dev/check-delta-connect-codegen-python.py ================================================ #!/usr/bin/env python3 # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Utility for checking whether generated the Delta Connect Python protobuf codes are in sync. # usage: ./dev/check-delta-connect-codegen-python.py import os import sys import filecmp import tempfile import subprocess # Location of your Delta git development area DELTA_HOME = os.environ.get("DELTA_HOME", os.path.abspath(os.path.join(__file__, os.pardir, os.pardir))) def fail(msg): print(msg) sys.exit(-1) def run_cmd(cmd): print(f"RUN: {cmd}") if isinstance(cmd, list): return subprocess.check_output(cmd).decode("utf-8") else: return subprocess.check_output(cmd.split(" ")).decode("utf-8") def check_connect_protos(): generated_python_proto_codes_path = os.path.join(DELTA_HOME, "python", "delta", "connect", "proto") print(f"Start checking the generated codes in {generated_python_proto_codes_path}") generate_python_proto_codes_file = os.path.join(DELTA_HOME, "dev", "delta-connect-gen-protos.sh") with tempfile.TemporaryDirectory() as tmp: run_cmd([generate_python_proto_codes_file, tmp]) result = filecmp.dircmp( generated_python_proto_codes_path, tmp, ignore=["__init__.py", "__pycache__"], ) success = True if len(result.left_only) > 0: print(f"Unexpected files: {result.left_only}") success = False if len(result.right_only) > 0: print(f"Missing files: {result.right_only}") success = False if len(result.funny_files) > 0: print(f"Incomparable files: {result.funny_files}") success = False if len(result.diff_files) > 0: print(f"Different files: {result.diff_files}") success = False if success: print(f"Finish checking the generated codes in {generated_python_proto_codes_path}: SUCCESS") else: fail( f"Generated files for {generated_python_proto_codes_path} are out of sync! " + f"Please run {generate_python_proto_codes_file}." ) check_connect_protos() ================================================ FILE: dev/checkstyle-suppressions.xml ================================================ ================================================ FILE: dev/connectors-checkstyle.xml ================================================ ================================================ FILE: dev/copyrightHeader ================================================ \/\* \* Copyright \(\d\d\d\d\) The Delta Lake Project Authors\. \* \* Licensed under the Apache License, Version 2\.0 \(the "License"\); \* you may not use this file except in compliance with the License\. \* You may obtain a copy of the License at \* \* http:\/\/www\.apache\.org\/licenses\/LICENSE-2\.0 \* \* Unless required by applicable law or agreed to in writing, software \* distributed under the License is distributed on an "AS IS" BASIS, \* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied\. \* See the License for the specific language governing permissions and \* limitations under the License\. \*\/ ================================================ FILE: dev/delta-connect-gen-protos.sh ================================================ #!/usr/bin/env bash # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # export PATH="$PATH:~/buf/bin" set -ex if [[ $# -gt 1 ]]; then echo "Illegal number of parameters." echo "Usage: $0 [path]" exit -1 fi DELTA_HOME="$(cd "`dirname $0`"/..; pwd)" cd "$DELTA_HOME" OUTPUT_PATH=${DELTA_HOME}/python/delta/connect/proto/ if [[ $# -eq 1 ]]; then rm -Rf $1 mkdir -p $1 OUTPUT_PATH=$1 fi pushd ${DELTA_HOME}/spark-connect/common/src/main LICENSE=$(cat <<'EOF' # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # EOF) echo "$LICENSE" > /tmp/tmp_licence # Delete the old generated protobuf files. rm -Rf gen # Now, regenerate the new files buf generate --debug -vvv # We need to edit the generated python files to account for the actual package location and not # the location generated by proto. for f in `find gen/proto/python/delta/connect -name "*.py*"`; do # First fix the imports. if [[ $f == *_pb2.py || $f == *_pb2_grpc.py ]]; then sed \ -e 's/import spark.connect./import pyspark.sql.connect.proto./g' \ -e "s/DESCRIPTOR, 'spark.connect/DESCRIPTOR, 'pyspark.sql.connect.proto/g" \ -e 's/from spark.connect import/from pyspark.sql.connect.proto import/g' \ -e "s/DESCRIPTOR, 'delta.connect/DESCRIPTOR, 'delta.connect.proto/g" \ -e 's/from delta.connect import/from delta.connect.proto import/g' \ $f > $f.tmp mv $f.tmp $f elif [[ $f == *.pyi ]]; then sed \ -e 's/import spark.connect./import pyspark.sql.connect.proto./g' \ -e 's/spark.connect./pyspark.sql.connect.proto./g' \ -e 's/import delta.connect./import delta.connect.proto./g' \ -e 's/delta.connect./delta.connect.proto./g' \ $f > $f.tmp mv $f.tmp $f fi # Prepend the Apache licence header to the files. cp $f $f.bak cat /tmp/tmp_licence $f.bak > $f LC=$(wc -l < $f) echo $LC if [[ $f == *_grpc.py && $LC -eq 20 ]]; then rm $f fi rm $f.bak done black --config $DELTA_HOME/dev/pyproject.toml gen/proto/python/delta/connect # Last step copy the result files to the destination module. for f in `find gen/proto/python/delta/connect -name "*.py*"`; do cp $f $OUTPUT_PATH done # Clean up everything. rm -Rf gen ================================================ FILE: dev/kernel-checkstyle.xml ================================================ ================================================ FILE: dev/lint-python ================================================ #!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # define test binaries + versions PYDOCSTYLE_BUILD="pydocstyle" MINIMUM_PYDOCSTYLE="3.0.0" FLAKE8_BUILD="flake8" MINIMUM_FLAKE8="3.5.0" PYCODESTYLE_BUILD="pycodestyle" MINIMUM_PYCODESTYLE="2.4.0" function compile_python_test { local COMPILE_STATUS= local COMPILE_REPORT= if [[ ! "$1" ]]; then echo "No python files found! Something is very wrong -- exiting." exit 1; fi # compileall: https://docs.python.org/2/library/compileall.html echo "starting python compilation test..." COMPILE_REPORT=$( (python3 -B -mcompileall -q -l $1) 2>&1) COMPILE_STATUS=$? if [ $COMPILE_STATUS -ne 0 ]; then echo "Python compilation failed with the following errors:" echo "$COMPILE_REPORT" echo "$COMPILE_STATUS" exit "$COMPILE_STATUS" else echo "python compilation succeeded." echo fi } function pycodestyle_test { local PYCODESTYLE_STATUS= local PYCODESTYLE_REPORT= local RUN_LOCAL_PYCODESTYLE= local VERSION= local EXPECTED_PYCODESTYLE= local PYCODESTYLE_SCRIPT_PATH="$DELTA_ROOT_DIR/dev/pycodestyle-$MINIMUM_PYCODESTYLE.py" local PYCODESTYLE_SCRIPT_REMOTE_PATH="https://raw.githubusercontent.com/PyCQA/pycodestyle/$MINIMUM_PYCODESTYLE/pycodestyle.py" if [[ ! "$1" ]]; then echo "No python files found! Something is very wrong -- exiting." exit 1; fi # check for locally installed pycodestyle & version RUN_LOCAL_PYCODESTYLE="False" if hash "$PYCODESTYLE_BUILD" 2> /dev/null; then VERSION=$( $PYCODESTYLE_BUILD --version 2> /dev/null) EXPECTED_PYCODESTYLE=$( (python3 -c 'from distutils.version import LooseVersion; print(LooseVersion("""'${VERSION[0]}'""") >= LooseVersion("""'$MINIMUM_PYCODESTYLE'"""))')\ 2> /dev/null) if [ "$EXPECTED_PYCODESTYLE" == "True" ]; then RUN_LOCAL_PYCODESTYLE="True" fi fi # download the right version or run locally if [ $RUN_LOCAL_PYCODESTYLE == "False" ]; then # Get pycodestyle at runtime so that we don't rely on it being installed on the build server. # See: https://github.com/apache/spark/pull/1744#issuecomment-50982162 # Updated to the latest official version of pep8. pep8 is formally renamed to pycodestyle. echo "downloading pycodestyle from $PYCODESTYLE_SCRIPT_REMOTE_PATH..." if [ ! -e "$PYCODESTYLE_SCRIPT_PATH" ]; then curl --silent -o "$PYCODESTYLE_SCRIPT_PATH" "$PYCODESTYLE_SCRIPT_REMOTE_PATH" local curl_status="$?" if [ "$curl_status" -ne 0 ]; then echo "Failed to download pycodestyle.py from $PYCODESTYLE_SCRIPT_REMOTE_PATH" exit "$curl_status" fi fi echo "starting pycodestyle test..." PYCODESTYLE_REPORT=$( (python3 "$PYCODESTYLE_SCRIPT_PATH" --config=dev/tox.ini $1) 2>&1) PYCODESTYLE_STATUS=$? else # we have the right version installed, so run locally echo "starting pycodestyle test..." PYCODESTYLE_REPORT=$( ($PYCODESTYLE_BUILD --config=dev/tox.ini $1) 2>&1) PYCODESTYLE_STATUS=$? fi if [ $PYCODESTYLE_STATUS -ne 0 ]; then echo "pycodestyle checks failed:" echo "$PYCODESTYLE_REPORT" exit "$PYCODESTYLE_STATUS" else echo "pycodestyle checks passed." echo fi } function flake8_test { local FLAKE8_VERSION= local VERSION= local EXPECTED_FLAKE8= local FLAKE8_REPORT= local FLAKE8_STATUS= if ! hash "$FLAKE8_BUILD" 2> /dev/null; then echo "The flake8 command was not found." echo "flake8 checks failed." exit 1 fi FLAKE8_VERSION="$($FLAKE8_BUILD --version 2> /dev/null)" VERSION=($FLAKE8_VERSION) EXPECTED_FLAKE8=$( (python3 -c 'from distutils.version import LooseVersion; print(LooseVersion("""'${VERSION[0]}'""") >= LooseVersion("""'$MINIMUM_FLAKE8'"""))') \ 2> /dev/null) if [[ "$EXPECTED_FLAKE8" == "False" ]]; then echo "\ The minimum flake8 version needs to be $MINIMUM_FLAKE8. Your current version is $FLAKE8_VERSION flake8 checks failed." exit 1 fi echo "starting $FLAKE8_BUILD test..." FLAKE8_REPORT=$( ($FLAKE8_BUILD $1 --count --select=E901,E999,F821,F822,F823 \ --max-line-length=100 --show-source --statistics) 2>&1) FLAKE8_STATUS=$? if [ "$FLAKE8_STATUS" -ne 0 ]; then echo "flake8 checks failed:" echo "$FLAKE8_REPORT" echo "$FLAKE8_STATUS" exit "$FLAKE8_STATUS" else echo "flake8 checks passed." echo fi } function pydocstyle_test { local PYDOCSTYLE_REPORT= local PYDOCSTYLE_STATUS= local PYDOCSTYLE_VERSION= local EXPECTED_PYDOCSTYLE= # Exclude auto-generated configuration file. local DOC_PATHS_TO_CHECK="$( cd "${DELTA_ROOT_DIR}" && find . -name "*.py" | grep -vF 'functions.py' )" # Check python document style, skip check if pydocstyle is not installed. if ! hash "$PYDOCSTYLE_BUILD" 2> /dev/null; then echo "The pydocstyle command was not found. Skipping pydocstyle checks for now." echo return fi PYDOCSTYLE_VERSION="$($PYDOCSTYLEBUILD --version 2> /dev/null)" EXPECTED_PYDOCSTYLE=$(python3 -c 'from distutils.version import LooseVersion; \ print(LooseVersion("""'$PYDOCSTYLE_VERSION'""") >= LooseVersion("""'$MINIMUM_PYDOCSTYLE'"""))' \ 2> /dev/null) if [[ "$EXPECTED_PYDOCSTYLE" == "False" ]]; then echo "\ The minimum version of pydocstyle needs to be $MINIMUM_PYDOCSTYLE. Your current version is $PYDOCSTYLE_VERSION. Skipping pydocstyle checks for now." echo return fi echo "starting $PYDOCSTYLE_BUILD test..." PYDOCSTYLE_REPORT=$( ($PYDOCSTYLE_BUILD --config=dev/tox.ini $DOC_PATHS_TO_CHECK) 2>&1) PYDOCSTYLE_STATUS=$? if [ "$PYDOCSTYLE_STATUS" -ne 0 ]; then echo "pydocstyle checks failed:" echo "$PYDOCSTYLE_REPORT" exit "$PYDOCSTYLE_STATUS" else echo "pydocstyle checks passed." echo fi } SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" DELTA_ROOT_DIR="$(dirname "${SCRIPT_DIR}")" pushd "$DELTA_ROOT_DIR" &> /dev/null PYTHON_SOURCE="$(find "${DELTA_ROOT_DIR}/python" -name "*.py")" compile_python_test "$PYTHON_SOURCE" pycodestyle_test "$PYTHON_SOURCE" #flake8_test "$PYTHON_SOURCE" pydocstyle_test echo echo "all lint-python tests passed!" popd &> /dev/null ================================================ FILE: dev/pyproject.toml ================================================ # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # [tool.black] # When changing the version, we have to update # GitHub workflow version required-version = "23.12.1" line-length = 100 target-version = ['py38'] include = '\.pyi?$' ================================================ FILE: dev/requirements.txt ================================================ # Linter mypy==1.8.0 flake8==3.9.0 # Code Formatter black==23.12.1 # Spark Connect (required) grpcio>=1.67.0 grpcio-status>=1.67.0 googleapis-common-protos>=1.65.0 # Spark and Delta Connect python proto generation plugin (optional) mypy-protobuf==3.3.0 ================================================ FILE: dev/spark_structured_logging_style.py ================================================ #!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os import sys import re import glob def main(): log_pattern = r"log(?:Info|Warning|Error)\(.*?\)\n" inner_log_pattern = r'".*?"\.format\(.*\)|s?".*?(?:\$|\"\+(?!s?")).*|[^"]+\+\s*".*?"' compiled_inner_log_pattern = re.compile(inner_log_pattern) # Regex patterns for file paths to exclude from the Structured Logging style check excluded_file_patterns = [ "[Tt]est", "sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen" "/CodeGenerator.scala", "streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala", "sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver" "/SparkSQLCLIService.scala", "core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala", ] nonmigrated_files = {} target_directories = ["../spark", "../iceberg", "../hudi"] scala_files = [] for directory in target_directories: scala_files.extend(glob.glob(os.path.join(directory, "**", "*.scala"), recursive=True)) for file in scala_files: skip_file = False for exclude_pattern in excluded_file_patterns: if re.search(exclude_pattern, file): skip_file = True break if not skip_file and not os.path.isdir(file): with open(file, "r") as f: content = f.read() log_statements = re.finditer(log_pattern, content, re.DOTALL) if log_statements: nonmigrated_files[file] = [] for log_statement in log_statements: log_statement_str = log_statement.group(0).strip() # trim first ( and last ) first_paren_index = log_statement_str.find("(") inner_log_statement = re.sub( r"\s+", "", log_statement_str[first_paren_index + 1 : -1] ) if compiled_inner_log_pattern.fullmatch(inner_log_statement): start_pos = log_statement.start() preceding_content = content[:start_pos] line_number = preceding_content.count("\n") + 1 start_char = start_pos - preceding_content.rfind("\n") - 1 nonmigrated_files[file].append((line_number, start_char)) if all(len(issues) == 0 for issues in nonmigrated_files.values()): print("Structured logging style check passed.", file=sys.stderr) sys.exit(0) else: for file_path, issues in nonmigrated_files.items(): for line_number, start_char in issues: print(f"[error] {file_path}:{line_number}:{start_char}", file=sys.stderr) print( "[error]\tPlease use the Structured Logging Framework for logging messages " 'with variables. For example: log"...${{MDC(TASK_ID, taskId)}}..."' "\n\tRefer to the guidelines in the file `shims/LoggingShims.scala`.", file=sys.stderr, ) sys.exit(-1) if __name__ == "__main__": main() ================================================ FILE: dev/tox.ini ================================================ # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. [pycodestyle] ignore=E226,E231,E241,E305,E402,E722,E731,E741,W503,W504,W604 max-line-length=100 exclude=cloudpickle.py,heapq3.py,shared.py,python/docs/conf.py,work/*/*.py,python/.eggs/*,dist/*,*python/delta/connect* [pydocstyle] ignore=D100,D101,D102,D103,D104,D105,D106,D107,D200,D201,D202,D203,D204,D205,D206,D207,D208,D209,D210,D211,D212,D213,D214,D215,D300,D301,D302,D400,D401,D402,D403,D404,D405,D406,D407,D408,D409,D410,D411,D412,D413,D414,D415,D417 ================================================ FILE: docs/.gitignore ================================================ # build output dist/ public/api # generated types .astro/ # dependencies node_modules/ # logs npm-debug.log* yarn-debug.log* yarn-error.log* pnpm-debug.log* # environment variables .env .env.production # macOS-specific files .DS_Store # Local Netlify folder .netlify ================================================ FILE: docs/.nvmrc ================================================ v22.18.0 ================================================ FILE: docs/.prettierignore ================================================ coverage public dist .astro pnpm-lock.yaml .DS_Store src/content/**/*.mdx ================================================ FILE: docs/.prettierrc.json ================================================ { "plugins": ["prettier-plugin-astro"], "overrides": [ { "files": "*.astro", "options": { "parser": "astro" } }, { "files": ["*.md", "*.mdx"], "options": { "parser": "mdx", "proseWrap": "never", "embeddedLanguageFormatting": "auto", "htmlWhitespaceSensitivity": "css", "bracketSameLine": false, "singleAttributePerLine": true, "tabWidth": 2 } } ] } ================================================ FILE: docs/README.md ================================================ # Docs Generation Scripts This directory contains scripts to generate docs for https://docs.delta.io, including the API Docs for Scala, Java, and Python APIs. ## Setup Environment ### Install Node environment Install node v22.14.0 using [nvm](https://github.com/nvm-sh/nvm): ``` nvm install ``` Then, install [pnpm](https://pnpm.io/): ``` npm install --global corepack@latest corepack enable pnpm ``` Finally, install dependencies: ``` pnpm i ``` ### Install Conda environment Follow [Conda Download](https://www.anaconda.com/download/) to install Anaconda. Then, follow [Create Environment From Environment file](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#create-env-from-file) to create a Conda environment from `/delta/docs/environment.yml` and activate the newly created `delta_docs` environment. ``` # Note the `--file` argument should be a fully qualified path. Using `~` in file # path doesn't work. Example valid path: `/Users/macuser/delta/docs/environment.yml` conda env create --name delta_docs --file=/docs/environment.yml` ``` ### JDK Setup API doc generation needs JDK 1.8. Make sure to setup `JAVA_HOME` that points to JDK 1.8. ### Set the Delta Lake version Set the version of Delta Lake release these docs are being generated for. ``` export _DELTA_LAKE_RELEASE_VERSION_=3.3.0 ``` ## Usage Run the command from the `delta` repo root directory: ``` python3 docs/generate_docs.py --livehtml --api-docs ``` Above command will print a URL to preview the docs. ### Skip generating API docs Above command generates API docs which take time. If you are just interested in the docs that go on https://docs.delta.io, use the following command. ``` python3 docs/generate_docs.py --livehtml ``` ### Building for production To build the docs for production, run the following command: ```python python3 docs/generate_docs.py --api-docs ``` The resulting files will be found in `docs/dist`. ### Additional docs site commands The docs site is built on [Astro](https://astro.build/). Using pnpm, you can run a variety of commands: | Command | Description | | --------------------- | ----------------------------------------------- | | `pnpm run lint` | Run ESLint on the docs site code | | `pnpm run format` | Format docs site code using Prettier | | `pnpm run dev` | Start Astro in development mode | | `pnpm run build` | Build the Astro site for production | | `pnpm run preview` | Preview the built Astro site | | `pnpm run astro` | Run Astro CLI | ================================================ FILE: docs/apis/api-docs.css ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* Dynamically injected style for the API docs */ .unstable { background-color: #EE4B2B; } .developer { background-color: #0047AB; } .evolving { background-color: #44751E; } .badge { font-family: Arial, san-serif; float: right; } ================================================ FILE: docs/apis/api-docs.js ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* Dynamically injected post-processing code for the API docs */ $(document).ready(function() { var annotations = $("dt:contains('Annotations')").next("dd").children("span.name"); addBadges(annotations, "Unstable", ":: Unstable ::", 'Unstable API'); addBadges(annotations, "Developer", ":: DeveloperApi ::", 'Developer API'); addBadges(annotations, "Evolving", ":: Evolving ::", 'Evolving API'); }); function addBadges(allAnnotations, name, tag, html) { var annotations = allAnnotations.filter(":contains('" + name + "')") var tags = $(".cmt:contains(" + tag + ")") // Remove identifier tags from comments tags.each(function(index) { var oldHTML = $(this).html(); var newHTML = oldHTML.replace(tag, ""); $(this).html(newHTML); }); // Add badges to all containers tags // Scala 2.11 docs require these .prevAll("h4.signature") .add(annotations.closest("div.fullcommenttop")) .add(annotations.closest("div.fullcomment").prevAll("h4.signature")) // Scala 2.12 docs require this .add(tags.prevAll("span.symbol")) .add(annotations.closest("div.fullcomment").prevAll("span.symbol")) .prepend(html); } ================================================ FILE: docs/apis/api-javadocs.css ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* Dynamically injected style for the API docs */ .badge { font-family: Arial, san-serif; float: right; margin: 4px; /* The following declarations are taken from the ScalaDoc template.css */ display: inline-block; padding: 2px 4px; font-size: 11.844px; font-weight: bold; line-height: 14px; color: #ffffff; text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); white-space: nowrap; vertical-align: baseline; background-color: #999999; padding-right: 9px; padding-left: 9px; -webkit-border-radius: 9px; -moz-border-radius: 9px; border-radius: 9px; } .unstable { background-color: #EE4B2B; } .developer { background-color: #0047AB; } .evolving { background-color: #44751E; } ================================================ FILE: docs/apis/api-javadocs.js ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* Dynamically injected post-processing code for the API docs */ $(document).ready(function() { addBadges(":: Unstable ::", 'Unstable API'); addBadges(":: DeveloperApi ::", 'Developer API'); addBadges(":: Evolving ::", 'Evolving API'); }); function addBadges(tag, html) { var tags = $(".block:contains(" + tag + ")") // Remove identifier tags tags.each(function(index) { var oldHTML = $(this).html(); var newHTML = oldHTML.replace(tag, ""); $(this).html(newHTML); }); // Add html badge tags tags.each(function(index) { if ($(this).parent().is('td.colLast')) { $(this).parent().prepend(html); } else if ($(this).parent('li.blockList') .parent('ul.blockList') .parent('div.description') .parent().is('div.contentContainer')) { var contentContainer = $(this).parent('li.blockList') .parent('ul.blockList') .parent('div.description') .parent('div.contentContainer') var header = contentContainer.prev('div.header'); if (header.length > 0) { header.prepend(html); } else { contentContainer.prepend(html); } } else if ($(this).parent().is('li.blockList')) { $(this).parent().prepend(html); } else { $(this).prepend(html); } }); } ================================================ FILE: docs/apis/generate_api_docs.py ================================================ # !/usr/bin/env python3 # # Copyright (2021) The Delta Lake Project Authors. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os import sys import subprocess import argparse def main(): # Parse arguments parser = argparse.ArgumentParser() parser.add_argument("-v", "--verbose", default=False, action='store_true') args = parser.parse_args() global verbose verbose = args.verbose # Set up the directories docs_root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) repo_root_dir = os.path.dirname(docs_root_dir) # --- dirs where docs are generated spark_scaladoc_gen_dir = repo_root_dir + "/spark/target/scala-2.13/unidoc" spark_javadoc_gen_dir = repo_root_dir + "/spark/target/javaunidoc" spark_pythondoc_dir = repo_root_dir + "/docs/apis/python" spark_pythondoc_gen_dir = spark_pythondoc_dir + "/_build/html" kernel_javadoc_gen_dir = repo_root_dir + "/kernelGroup/target/javaunidoc" # --- final dirs where the docs will be copied to all_docs_final_dir = docs_root_dir + "/apis/_site/api" all_javadocs_final_dir = all_docs_final_dir + "/java" all_scaladocs_final_dir = all_docs_final_dir + "/scala" all_pythondocs_final_dir = all_docs_final_dir + "/python" spark_javadoc_final_dir = all_javadocs_final_dir + "/spark" spark_scaladoc_final_dir = all_scaladocs_final_dir + "/spark" spark_pythondoc_final_dir = all_pythondocs_final_dir + "/spark" kernel_javadoc_final_dir = all_javadocs_final_dir + "/kernel" # Generate Java and Scala docs print("## Generating Scala and Java docs ...") with WorkingDirectory(repo_root_dir): run_cmd(["build/sbt", ";clean;unidoc"], stream_output=verbose) # Update Scala docs print("## Patching Scala docs ...") patch_scala_docs(spark_scaladoc_gen_dir, docs_root_dir) # Update Java docs print("## Patching Java docs ...") jquery_path = spark_scaladoc_gen_dir + "/lib/jquery.min.js" # grab the JQuery library from Scaladocs all_javadoc_gen_dirs = [ spark_javadoc_gen_dir, kernel_javadoc_gen_dir, ] for javadoc_gen_dir in all_javadoc_gen_dirs: patch_java_docs(javadoc_gen_dir, docs_root_dir, jquery_path) # Generate Python docs print('## Generating Python docs ...') with WorkingDirectory(spark_pythondoc_dir): run_cmd(["make", "html"], stream_output=verbose) # Copy to final location log("## Copying to API doc directory %s" % all_docs_final_dir) src_dst_dirs = [ (spark_javadoc_gen_dir, spark_javadoc_final_dir), (spark_scaladoc_gen_dir, spark_scaladoc_final_dir), (spark_pythondoc_gen_dir, spark_pythondoc_final_dir), (kernel_javadoc_gen_dir, kernel_javadoc_final_dir), ] run_cmd(["rm", "-rf", all_docs_final_dir]) run_cmd(["mkdir", "-p", all_docs_final_dir]) for (src_dir, dst_dir) in src_dst_dirs: run_cmd(["mkdir", "-p", dst_dir]) run_cmd(["cp", "-r", src_dir.rstrip("/") + "/", dst_dir]) print("## API docs generated in " + all_docs_final_dir) def patch_scala_docs(scaladoc_dir, docs_root_dir): with WorkingDirectory(scaladoc_dir): # Patch the js and css files append(docs_root_dir + "/apis/api-docs.js", "./lib/template.js") # append new js functions append(docs_root_dir + "/apis/api-docs.css", "./lib/template.css") # append new styles def patch_java_docs(javadoc_dir, docs_root_dir, jquery_path): print("### Patching JavaDoc in %s ..." % javadoc_dir) with WorkingDirectory(javadoc_dir): # Find html files to patch (_, stdout, _) = run_cmd(["find", ".", "-name", "*.html", "-mindepth", "2"]) log("HTML files found:\n" + stdout) javadoc_files = [line for line in stdout.split('\n') if line.strip() != ''] js_script_start = '' # Patch the html files for javadoc_file in javadoc_files: # Generate relative path to js files based on how deep the html file is slash_count = javadoc_file.count("/") i = 1 path_to_js_file = "" while i < slash_count: path_to_js_file = path_to_js_file + "../" i += 1 # Create script elements to load new js files javadoc_jquery_script = \ js_script_start + path_to_js_file + "lib/jquery.min.js" + js_script_end javadoc_api_docs_script = \ js_script_start + path_to_js_file + "lib/api-javadocs.js" + js_script_end javadoc_script_elements = javadoc_jquery_script + javadoc_api_docs_script # Add script elements to body of the html file replace(javadoc_file, "", javadoc_script_elements + "") # Patch the js and css files run_cmd(["mkdir", "-p", "./lib"]) run_cmd(["cp", jquery_path, "./lib/"]) # copy from ScalaDocs run_cmd(["cp", docs_root_dir + "/apis/api-javadocs.js", "./lib/"]) # copy new js file append(docs_root_dir + "/apis/api-javadocs.css", "./stylesheet.css") # append new styles def run_cmd(cmd, throw_on_error=True, env=None, stream_output=False, **kwargs): """Runs a command as a child process. A convenience wrapper for running a command from a Python script. Keyword arguments: cmd -- the command to run, as a list of strings throw_on_error -- if true, raises an Exception if the exit code of the program is nonzero env -- additional environment variables to be defined when running the child process stream_output -- if true, does not capture standard output and error; if false, captures these streams and returns them Note on the return value: If stream_output is true, then only the exit code is returned. If stream_output is false, then a tuple of the exit code, standard output and standard error is returned. """ log("Running command %s" % str(cmd)) cmd_env = os.environ.copy() if env: cmd_env.update(env) if stream_output: child = subprocess.Popen(cmd, env=cmd_env, **kwargs) exit_code = child.wait() if throw_on_error and exit_code != 0: raise Exception("Non-zero exitcode: %s" % exit_code) return exit_code else: child = subprocess.Popen( cmd, env=cmd_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) (stdout, stderr) = child.communicate() if sys.version_info >= (3, 0): stdout = stdout.decode("UTF-8") stderr = stderr.decode("UTF-8") exit_code = child.wait() if throw_on_error and exit_code != 0: raise Exception( "Non-zero exitcode: %s\n\nSTDOUT:\n%s\n\nSTDERR:%s" % (exit_code, stdout, stderr)) return (exit_code, stdout, stderr) def append(src, dst): log("Appending %s to %s" % (src, dst)) fin = open(src, "r") str = fin.read() fin.close() fout = open(dst, "a") fout.write(str) fout.close() def replace(file, pattern, replacement): log("Replacing %s with %s in file %s" % (pattern, replacement, file)) fin = open(file, "r") str = fin.read() fin.close() str = str.replace(pattern, replacement) fout = open(file, "w") fout.write(str) fout.close() # pylint: disable=too-few-public-methods class WorkingDirectory(object): def __init__(self, working_directory): self.working_directory = working_directory self.old_workdir = os.getcwd() def __enter__(self): os.chdir(self.working_directory) def __exit__(self, tpe, value, traceback): os.chdir(self.old_workdir) def log(str): if verbose: print(str) verbose = False if __name__ == "__main__": # pylint: disable=e1120 main() ================================================ FILE: docs/apis/python/Makefile ================================================ # Minimal makefile for Sphinx documentation # # You can set these variables from the command line, and also # from the environment for the first two. SPHINXOPTS ?= -W SPHINXBUILD ?= sphinx-build SOURCEDIR = . BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) ================================================ FILE: docs/apis/python/conf.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html # -- Path setup -------------------------------------------------------------- # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # import os import sys import pathlib python_docs_root_dir = os.path.dirname(os.path.realpath(__file__)) docs_dir = os.path.dirname(python_docs_root_dir) root_dir = os.path.dirname(docs_dir) delta_root_dir = pathlib.Path(root_dir).parent version_file_path = os.path.join(delta_root_dir, 'version.sbt') sys.path.insert(0, os.path.abspath(os.path.join(root_dir, 'python'))) # -- Project information ----------------------------------------------------- project = 'delta-spark' # The full version, including alpha/beta/rc tags release = '0.0.0' for line in open(version_file_path): if "ThisBuild" in line: release = line.split("\"")[1] # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = ['sphinx.ext.autodoc'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = 'nature' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = [] # default value was ['_static'] # Fix for doc generation to work on older version of sphinx as well. master_doc = 'index' # Display the classes in the generated in the same order as the classes appear in the source files.` autodoc_member_order = 'bysource' ================================================ FILE: docs/apis/python/index.rst ================================================ .. delta documentation master file, created by sphinx-quickstart on Fri Sep 20 16:32:12 2019. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. Welcome to Delta Lake's Python documentation page ================================================= .. toctree:: :maxdepth: 2 :caption: Contents: DeltaTable ========== .. automodule:: delta.tables :members: :undoc-members: Exceptions ========== .. automodule:: delta.exceptions :members: :undoc-members: Others ====== .. automodule:: delta.pip_utils :members: :undoc-members: Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` ================================================ FILE: docs/astro.config.mjs ================================================ // @ts-check import { defineConfig } from "astro/config"; import starlight from "@astrojs/starlight"; import netlify from "@astrojs/netlify"; // https://astro.build/config export default defineConfig({ site: "https://docs.delta.io/", image: { service: { entrypoint: "astro/assets/services/sharp", }, }, adapter: netlify(), redirects: { "/latest/api/*": "/api/latest/:splat", "/:version/api/*": "/api/:version/:splat", "/latest/*": "/:splat", "/delta-intro.html": "/", "/delta-spark.html": "/", "/quick-start.html": "/quick-start", "/delta-batch.html": "/delta-batch", "/delta-streaming.html": "/delta-streaming", "/delta-update.html": "/delta-update", "/delta-change-data-feed.html": "/delta-change-data-feed", "/delta-utility.html": "/delta-utility", "/delta-constraints.html": "/delta-constraints", "/versioning.html": "/versioning", "/delta-default-columns.html": "/delta-default-columns", "/delta-column-mapping.html": "/delta-column-mapping", "/delta-clustering.html": "/delta-clustering", "/delta-deletion-vectors.html": "/delta-deletion-vectors", "/delta-drop-feature.html": "/delta-drop-feature", "/delta-row-tracking.html": "/delta-row-tracking", "/delta-spark-connect.html": "/delta-spark-connect", "/delta-storage.html": "/delta-storage", "/delta-type-widening.html": "/delta-type-widening", "/delta-uniform.html": "/delta-uniform", "/delta-sharing.html": "/delta-sharing", "/concurrency-control.html": "/concurrency-control", "/porting.html": "/porting", "/best-practices.html": "/best-practices", "/delta-faq.html": "/delta-faq", "/optimizations-oss.html": "/optimizations-oss", "/delta-trino-integration.html": "/delta-trino-integration", "/delta-presto-integration.html": "/delta-presto-integration", "/presto-integration.html": "/presto-integration", "/redshift-spectrum-integration.html": "/redshift-spectrum-integration", "/snowflake-integration.html": "/snowflake-integration", "/bigquery-integration.html": "/bigquery-integration", "/flink-integration.html": "/flink-integration", "/delta-more-connectors.html": "/delta-more-connectors", "/delta-kernel.html": "/delta-kernel", "/delta-standalone.html": "/delta-standalone", "/delta-apidoc.html": "/delta-apidoc", "/releases.html": "/releases", "/delta-resources.html": "/delta-resources", "/table-properties.html": "/table-properties", }, integrations: [ starlight({ customCss: ["./src/styles/custom.css"], title: "Delta Lake", social: [ { icon: "github", label: "GitHub", href: "https://github.com/delta-io/delta", }, ], editLink: { baseUrl: "https://github.com/jakebellacera/db-site-staging/tree/main/sites/delta-docs", }, lastUpdated: true, favicon: "/favicon.svg", logo: { light: "./src/assets/delta-lake-logo-light.svg", dark: "./src/assets/delta-lake-logo-dark.svg", replacesTitle: true, }, sidebar: [ { label: "Introduction", link: "/" }, { label: "Apache Spark connector", collapsed: true, items: [ { slug: "quick-start", }, { slug: "delta-batch", }, { slug: "delta-streaming", }, { slug: "delta-update", }, { slug: "delta-change-data-feed", }, { slug: "delta-utility", }, { slug: "delta-constraints", }, { slug: "versioning", }, { slug: "delta-default-columns", }, { slug: "delta-column-mapping", }, { slug: "delta-clustering", }, { slug: "delta-deletion-vectors", }, { slug: "delta-catalog-managed-tables", }, { slug: "delta-drop-feature", }, { slug: "delta-row-tracking", }, { slug: "delta-spark-connect", }, { slug: "delta-storage", }, { slug: "delta-type-widening", }, { slug: "delta-uniform", }, { slug: "delta-sharing", }, { slug: "concurrency-control", }, { slug: "porting", }, { slug: "best-practices", }, { slug: "delta-faq", }, { slug: "optimizations-oss", }, ], }, { slug: "delta-trino-integration", }, { slug: "delta-starburst-integration", }, { slug: "delta-presto-integration", }, { slug: "redshift-spectrum-integration", }, { slug: "snowflake-integration", }, { slug: "bigquery-integration", }, { slug: "flink-integration", }, { slug: "delta-more-connectors", }, { slug: "delta-kernel", }, { slug: "delta-standalone", }, { slug: "delta-apidoc", }, { slug: "releases", }, { slug: "delta-resources", }, { slug: "table-properties", }, { label: "Contribute", link: "https://github.com/delta-io/delta/blob/master/CONTRIBUTING.md", attrs: { target: "_blank" }, }, ], head: [ { tag: "script", attrs: { async: true, src: "https://www.googletagmanager.com/gtag/js?id=UA-138952006-1", }, }, { tag: "script", content: ` window.dataLayer = window.dataLayer || []; function gtag(){dataLayer.push(arguments);} gtag('js', new Date()); gtag('config', 'UA-138952006-1'); `, }, ], }), ], }); ================================================ FILE: docs/environment.yml ================================================ name: delta_docs channels: - defaults dependencies: - ca-certificates=2023.08.22 - libcxx=14.0.6 - libffi=3.4.4 - ncurses=6.4 - openssl=3.0.11 - pip=23.2.1 - python=3.8.18 - readline=8.2 - setuptools=68.0.0 - sqlite=3.41.2 - tk=8.6.12 - wheel=0.41.2 - xz=5.4.2 - zlib=1.2.13 - pip: - alabaster==0.7.13 - babel==2.13.0 - certifi==2023.7.22 - charset-normalizer==3.3.0 - colorama==0.4.6 - delta-spark==3.0.0 - docutils==0.15.2 - idna==3.4 - imagesize==1.4.1 - importlib-metadata==7.0.0 - jinja2==2.11.3 - livereload==2.6.3 - markupsafe==2.0.0 - packaging==23.2 - py4j==0.10.9.7 - pygments==2.16.1 - pyspark==3.5.3 - pytz==2023.3.post1 - requests==2.31.0 - six==1.16.0 - snowballstemmer==2.2.0 - sphinx==2.0.1 - sphinx-autobuild==2021.3.14 - sphinxcontrib-applehelp==1.0.4 - sphinxcontrib-devhelp==1.0.2 - sphinxcontrib-htmlhelp==2.0.1 - sphinxcontrib-jsmath==1.0.1 - sphinxcontrib-qthelp==1.0.3 - sphinxcontrib-serializinghtml==1.1.5 - tornado==6.3.3 - urllib3==2.0.6 - zipp==3.17.0 prefix: /anaconda3/envs/delta_docs ================================================ FILE: docs/eslint.config.mjs ================================================ import globals from "globals"; import pluginJs from "@eslint/js"; import eslintPluginPrettierRecommended from "eslint-plugin-prettier/recommended"; import tseslint from "typescript-eslint"; import eslintPluginAstro from "eslint-plugin-astro"; export default [ { ignores: [ "apis", "coverage", "**/public", "**/dist", "**/.astro", "pnpm-lock.yaml", "pnpm-workspace.yaml", ], }, { files: ["**/*.{js,mjs,cjs,ts}"] }, { languageOptions: { globals: globals.browser } }, pluginJs.configs.recommended, eslintPluginPrettierRecommended, ...tseslint.configs.recommended, ...eslintPluginAstro.configs.recommended, ]; ================================================ FILE: docs/generate_docs.py ================================================ #!/usr/bin/env python3 import argparse import os import subprocess import random import shutil import string import tempfile def main(): """Script to manage the deployment of Delta Lake docs to the hosting bucket. To build the docs: $ generate_docs --livehtml """ parser = argparse.ArgumentParser() parser.add_argument( "--livehtml", action="store_true", help="Build and serve a local build of docs") parser.add_argument( "--api-docs", action="store_true", help="Generate the API docs") args = parser.parse_args() docs_root_dir = os.path.dirname(os.path.realpath(__file__)) api_docs_root_dir = os.path.join(docs_root_dir, "apis") with WorkingDirectory(docs_root_dir): api_html_output = os.path.join(docs_root_dir, 'public', 'api', 'latest') print("Building content") build_docs_cmd = "pnpm run build" if args.livehtml: build_docs_cmd = "pnpm dev" if args.api_docs: # Assert that env var _DELTA_LAKE_RELEASE_VERSION_ (used by conf.py) is set try: os.environ["_DELTA_LAKE_RELEASE_VERSION_"] except KeyError: raise KeyError(f"Environment variable _DELTA_LAKE_RELEASE_VERSION_ not set.") generate_and_copy_api_docs(api_docs_root_dir, api_html_output) run_cmd(build_docs_cmd, shell=True, stream_output=True) def generate_and_copy_api_docs(api_docs_root_dir, target_loc): print("Building API docs") with WorkingDirectory(target_loc): script_path = os.path.join(api_docs_root_dir, "generate_api_docs.py") api_docs_dir = os.path.join(api_docs_root_dir, "_site", "api") run_cmd(["python3", script_path], stream_output=True) assert os.path.exists(api_docs_dir), \ "Doc generation didn't create the expected api directory" api_docs_dest_dir = target_loc shutil.copytree(api_docs_dir, api_docs_dest_dir) class WorkingDirectory(object): def __init__(self, working_directory): self.working_directory = working_directory self.old_workdir = os.getcwd() def __enter__(self): os.chdir(self.working_directory) def __exit__(self, type, value, traceback): os.chdir(self.old_workdir) def run_cmd(cmd, throw_on_error=True, env=None, stream_output=False, **kwargs): """Runs a command as a child process. A convenience wrapper for running a command from a Python script. Keyword arguments: cmd -- the command to run, as a list of strings throw_on_error -- if true, raises an Exception if the exit code of the program is nonzero env -- additional environment variables to be defined when running the child process stream_output -- if true, does not capture standard output and error; if false, captures these streams and returns them Note on the return value: If stream_output is true, then only the exit code is returned. If stream_output is false, then a tuple of the exit code, standard output and standard error is returned. """ cmd_env = os.environ.copy() if env: cmd_env.update(env) if stream_output: child = subprocess.Popen(cmd, env=cmd_env, **kwargs) exit_code = child.wait() if throw_on_error and exit_code != 0: raise Exception("Non-zero exitcode: %s" % exit_code) return exit_code else: child = subprocess.Popen( cmd, env=cmd_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) (stdout, stderr) = child.communicate() exit_code = child.wait() if throw_on_error and exit_code != 0: raise Exception( "Non-zero exitcode: %s\n\nSTDOUT:\n%s\n\nSTDERR:%s" % (exit_code, stdout, stderr)) return exit_code, stdout.decode("utf-8"), stderr.decode("utf-8") if __name__ == "__main__": main() ================================================ FILE: docs/package.json ================================================ { "name": "delta-docs", "private": true, "description": "The official documentation for delta.io", "main": "index.js", "engines": { "node": ">=22.18.0" }, "scripts": { "lint": "eslint .", "format": "prettier --write .", "dev": "astro dev", "build": "astro build", "preview": "astro preview", "astro": "astro" }, "keywords": [], "author": "delta-io", "license": "Apache-2.0", "packageManager": "pnpm@10.15.0+sha512.486ebc259d3e999a4e8691ce03b5cac4a71cbeca39372a9b762cb500cfdf0873e2cb16abe3d951b1ee2cf012503f027b98b6584e4df22524e0c7450d9ec7aa7b", "dependencies": { "@astrojs/check": "^0.9.5", "@astrojs/netlify": "^6.6.1", "@astrojs/starlight": "^0.36.2", "astro": "^5.15.9", "sharp": "^0.34.5", "typescript": "^5.9.3" }, "devDependencies": { "@eslint/js": "^9.39.1", "@typescript-eslint/parser": "^8.47.0", "eslint": "^9.39.1", "eslint-config-prettier": "^10.1.8", "eslint-plugin-astro": "^1.5.0", "eslint-plugin-prettier": "^5.5.4", "globals": "^16.5.0", "prettier": "^3.6.2", "prettier-plugin-astro": "^0.14.1", "typescript-eslint": "^8.47.0" } } ================================================ FILE: docs/scripts/download-api-docs ================================================ #!/bin/bash # GitHub repository details GITHUB_OWNER="delta-incubator" GITHUB_REPO="delta-docs" # Check if GitHub token is available if [ -z "$GITHUB_TOKEN" ]; then echo "Error: GITHUB_TOKEN environment variable is required for downloading artifacts" echo "Please set GITHUB_TOKEN with a personal access token that has scope: public_repo, actions:read" exit 1 fi # Check if an argument was provided if [ -z "$1" ]; then echo "Error: No version argument provided" echo "Usage: $0 " exit 1 fi # Get the first argument (e.g., "latest") arg="$1" # Concatenate with the environment variable prefix env_var_name="npm_package_config_apidocs_${arg}" # Get the value of the dynamically constructed environment variable apidocs_version="${!env_var_name}" # Check if the environment variable exists and has a value if [ -z "$apidocs_version" ]; then echo "Error: No configuration found for version '$arg'" echo "Environment variable '$env_var_name' is not set or empty" echo "Supported versions should be configured in package.json" echo "Available versions: latest, v3, v2" exit 1 fi echo "Resolved version '$arg' to '$apidocs_version'" # GitHub API base URL api_base_url="https://api.github.com/repos/${GITHUB_OWNER}/${GITHUB_REPO}" # Create target directory path target_dir="public/api/${apidocs_version}" zip_file="${apidocs_version}.zip" # Create public/api directory if it doesn't exist mkdir -p public/api # Remove existing target directory if it exists (for idempotence) if [ -d "$target_dir" ]; then echo "Removing existing directory: $target_dir" rm -rf "$target_dir" fi # First, get the list of artifacts to find the one matching our version echo "Searching for artifact with name containing '$apidocs_version'..." artifacts_response=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \ -H "Accept: application/vnd.github.v3+json" \ "$api_base_url/actions/artifacts") if [ $? -ne 0 ]; then echo "Error: Failed to fetch artifacts list from GitHub API" exit 1 fi # Extract artifact ID for the matching version (assumes artifact name contains the version) artifact_id=$(echo "$artifacts_response" | grep -A 10 -B 10 "\"name\".*$apidocs_version" | grep '"id"' | head -1 | sed 's/.*"id": *\([0-9]*\).*/\1/') if [ -z "$artifact_id" ]; then echo "Error: No artifact found with name containing '$apidocs_version'" echo "Available artifacts:" echo "$artifacts_response" | grep '"name"' | sed 's/.*"name": *"\([^"]*\)".*/ - \1/' exit 1 fi echo "Found artifact ID: $artifact_id" # Download the artifact echo "Downloading artifact $artifact_id..." download_url="$api_base_url/actions/artifacts/$artifact_id/zip" echo "Download URL: $download_url" if ! curl -L -H "Authorization: token $GITHUB_TOKEN" \ -H "Accept: application/vnd.github.v3+json" \ -o "$zip_file" "$download_url"; then echo "Error: Failed to download artifact from $download_url" echo "Please check if the artifact exists and your token has proper permissions" exit 1 fi # Verify the downloaded file exists and is not empty if [ ! -s "$zip_file" ]; then echo "Error: Downloaded file is empty or does not exist" rm -f "$zip_file" exit 1 fi # Create target directory mkdir -p "$target_dir" # Extract the archive echo "Extracting to $target_dir..." if ! unzip -o "$zip_file" -d "$target_dir"; then echo "Error: Failed to extract $zip_file" rm -f "$zip_file" rm -rf "$target_dir" exit 1 fi # Clean up the zip file rm -f "$zip_file" echo "Successfully extracted API docs for version '$apidocs_version' to '$target_dir'" echo "Done!" ================================================ FILE: docs/scripts/upgrade-dependencies ================================================ #!/bin/bash # Function to prompt for user confirmation confirm() { while true; do read -p "Confirm that you want to upgrade packages? (y/n): " yn case $yn in [Yy]* ) return 0;; [Nn]* ) echo "Upgrade cancelled."; exit 1;; * ) echo "Please answer yes or no.";; esac done } # Check for outdated dependencies echo "Checking for outdated dependencies..." if pnpm outdated --recursive; then echo "✅ All packages are up to date! No upgrades needed." exit 0 fi # Ask for confirmation before proceeding echo "⚠️ This script will upgrade Astro and all other dependencies to their latest versions." confirm echo "Starting upgrade process..." # Upgrade astro first pnpm dlx @astrojs/upgrade # Upgrade remaining packages pnpm upgrade --latest echo "✅ Upgrade process completed! Please verify that things work as expected." ================================================ FILE: docs/src/content/docs/best-practices.mdx ================================================ --- title: Best practices description: Learn best practices when using Delta Lake. --- import { Tabs, TabItem, Aside } from "@astrojs/starlight/components"; ## Choose the right partition column You can partition a Delta table by a column. The most commonly used partition column is `date`. Follow these two rules of thumb for deciding on what column to partition by: - If the cardinality of a column will be very high, do not use that column for partitioning. For example, if you partition by a column `userId` and if there can be 1M distinct user IDs, then that is a bad partitioning strategy. - Amount of data in each partition: You can partition by a column if you expect data in that partition to be at least 1 GB. ## Compact files If you continuously write data to a Delta table, it will over time accumulate a large number of files, especially if you add data in small batches. This can have an adverse effect on the efficiency of table reads, and it can also affect the performance of your file system. Ideally, a large number of small files should be rewritten into a smaller number of larger files on a regular basis. This is known as compaction. You can compact a table by repartitioning it to smaller number of files. In addition, you can specify the option `dataChange` to be `false` indicates that the operation does not change the data, only rearranges the data layout. This would ensure that other concurrent operations are minimally affected due to this compaction operation. For example, you can compact a table into 16 files: ```scala val path = "..." val numFiles = 16 spark.read .format("delta") .load(path) .repartition(numFiles) .write .option("dataChange", "false") .format("delta") .mode("overwrite") .save(path) ``` ```python path = "..." numFiles = 16 (spark.read .format("delta") .load(path) .repartition(numFiles) .write .option("dataChange", "false") .format("delta") .mode("overwrite") .save(path)) ``` If your table is partitioned and you want to repartition just one partition based on a predicate, you can read only the partition using `where` and write back to that using `replaceWhere`: ```scala val path = "..." val partition = "year = '2019'" val numFilesPerPartition = 16 spark.read .format("delta") .load(path) .where(partition) .repartition(numFilesPerPartition) .write .option("dataChange", "false") .format("delta") .mode("overwrite") .option("replaceWhere", partition) .save(path) ``` ```python path = "..." partition = "year = '2019'" numFilesPerPartition = 16 (spark.read .format("delta") .load(path) .where(partition) .repartition(numFilesPerPartition) .write .option("dataChange", "false") .format("delta") .mode("overwrite") .option("replaceWhere", partition) .save(path)) ``` ## Replace the content or schema of a table Sometimes you may want to replace a Delta table. For example: - You discover the data in the table is incorrect and want to replace the content. - You want to rewrite the whole table to do incompatible schema changes (such as changing column types). While you can delete the entire directory of a Delta table and create a new table on the same path, it's _not recommended_ because: - Deleting a directory is not efficient. A directory containing very large files can take hours or even days to delete. - You lose all of content in the deleted files; it's hard to recover if you delete the wrong table. - The directory deletion is not atomic. While you are deleting the table a concurrent query reading the table can fail or see a partial table. If you don't need to change the table schema, you can [delete](/delta-update/#delete-from-a-table) data from a Delta table and insert your new data, or [update](/delta-update/#update-a-table) the table to fix the incorrect values. If you want to change the table schema, you can replace the whole table atomically. For example: ```python dataframe.write \ .format("delta") \ .mode("overwrite") \ .option("overwriteSchema", "true") \ .partitionBy() \ .saveAsTable("") # Managed table dataframe.write \ .format("delta") \ .mode("overwrite") \ .option("overwriteSchema", "true") \ .option("path", "") \ .partitionBy() \ .saveAsTable("") # External table ``` ```sql REPLACE TABLE USING DELTA PARTITIONED BY () AS SELECT ... -- Managed table REPLACE TABLE USING DELTA PARTITIONED BY () LOCATION "" AS SELECT ... -- External table ``` ```scala dataframe.write .format("delta") .mode("overwrite") .option("overwriteSchema", "true") .partitionBy() .saveAsTable("") // Managed table dataframe.write .format("delta") .mode("overwrite") .option("overwriteSchema", "true") .option("path", "") .partitionBy() .saveAsTable("") // External table ``` There are multiple benefits with this approach: - Overwriting a table is much faster because it doesn't need to list the directory recursively or delete any files. - The old version of the table still exists. If you delete the wrong table you can easily retrieve the old data using [Time Travel](/delta-batch/#query-an-older-snapshot-of-a-table-time-travel). - It's an atomic operation. Concurrent queries can still read the table while you are deleting the table. - Because of Delta Lake ACID transaction guarantees, if overwriting the table fails, the table will be in its previous state. In addition, if you want to delete old files to save storage cost after overwriting the table, you can use [VACUUM](/delta-utility/#remove-files-no-longer-referenced-by-a-delta-table) to delete them. It's optimized for file deletion and usually faster than deleting the entire directory. ## Spark caching You should not use Spark caching for the following reasons: - You lose any data skipping that can come from additional filters added on top of the cached `DataFrame`. - The data that gets cached may not be updated if the table is accessed using a different identifier (for example, you do `spark.table(x).cache()` but then write to the table using `spark.write.save(/some/path)`. ================================================ FILE: docs/src/content/docs/bigquery-integration.mdx ================================================ --- title: Google BigQuery connector description: Learn how to read Delta Lake tables from Google BigQuery. --- Google BigQuery supports reading Delta Lake (reader version 3 with [Deletion Vectors](/delta-deletion-vectors) and [Column Mapping](/delta-column-mapping/)). Please refer to [Delta Lake BigLake tables documentation](https://cloud.google.com/bigquery/docs/create-delta-lake-table) for more details. ================================================ FILE: docs/src/content/docs/concurrency-control.mdx ================================================ --- title: Concurrency control description: Learn about the ACID transaction guarantees between reads and writes provided by Delta Lake. --- import { Tabs, TabItem, Aside } from "@astrojs/starlight/components"; Delta Lake provides ACID transaction guarantees between reads and writes. This means that: - For supported [storage systems](/delta-storage), multiple writers across multiple clusters can simultaneously modify a table partition and see a consistent snapshot view of the table and there will be a serial order for these writes. - Readers continue to see a consistent snapshot view of the table that the Apache Spark job started with, even when a table is modified during a job. ## Optimistic concurrency control Delta Lake uses [optimistic concurrency control](https://en.wikipedia.org/wiki/Optimistic_concurrency_control) to provide transactional guarantees between writes. Under this mechanism, writes operate in three stages: 1. **Read**: Reads (if needed) the latest available version of the table to identify which files need to be modified (that is, rewritten). 2. **Write**: Stages all the changes by writing new data files. 3. **Validate and commit**: Before committing the changes, checks whether the proposed changes conflict with any other changes that may have been concurrently committed since the snapshot that was read. If there are no conflicts, all the staged changes are committed as a new versioned snapshot, and the write operation succeeds. However, if there are conflicts, the write operation fails with a concurrent modification exception rather than corrupting the table as would happen with the write operation on a Parquet table. ## Write conflicts The following table describes which pairs of write operations can conflict. Compaction refers to [file compaction operation](/best-practices/#compact-files) written with the option `dataChange` set to `false`. | | INSERT | UPDATE, DELETE, MERGE INTO | COMPACTION | | --- | --- | --- | --- | | **INSERT** | Cannot conflict | | | | **UPDATE, DELETE, MERGE INTO** | Can conflict | Can conflict | | | **COMPACTION** | Cannot conflict | Can conflict | Can conflict | ## Avoid conflicts using partitioning and disjoint command conditions In all cases marked "can conflict", whether the two operations will conflict depends on whether they operate on the same set of files. You can make the two sets of files disjoint by partitioning the table by the same columns as those used in the conditions of the operations. For example, the two commands `UPDATE table WHERE date > '2010-01-01' ...` and `DELETE table WHERE date < '2010-01-01'` will conflict if the table is not partitioned by date, as both can attempt to modify the same set of files. Partitioning the table by `date` will avoid the conflict. Hence, partitioning a table according to the conditions commonly used on the command can reduce conflicts significantly. However, partitioning a table by a column that has high cardinality can lead to other performance issues due to large number of subdirectories. ## Conflict exceptions When a transaction conflict occurs, you will observe one of the following exceptions: ### ConcurrentAppendException This exception occurs when a concurrent operation adds files in the same partition (or anywhere in an unpartitioned table) that your operation reads. The file additions can be caused by `INSERT`, `DELETE`, `UPDATE`, or `MERGE` operations. This exception is often thrown during concurrent `DELETE`, `UPDATE`, or `MERGE` operations. While the concurrent operations may be physically updating different partition directories, one of them may read the same partition that the other one concurrently updates, thus causing a conflict. You can avoid this by making the separation explicit in the operation condition. Consider the following example. ```scala // Target 'deltaTable' is partitioned by date and country deltaTable.as("t") .merge( source.as("s"), "s.user_id = t.user_id AND s.date = t.date AND s.country = t.country" ) .whenMatched() .updateAll() .whenNotMatched() .insertAll() .execute() ``` Suppose you run the above code concurrently for different dates or countries. Since each job is working on an independent partition on the target Delta table, you don't expect any conflicts. However, the condition is not explicit enough and can scan the entire table and can conflict with concurrent operations updating any other partitions. Instead, you can rewrite your statement to add specific date and country to the merge condition, as shown in the following example. ```scala // Target 'deltaTable' is partitioned by date and country deltaTable.as("t") .merge( source.as("s"), "s.user_id = t.user_id AND s.date = t.date AND s.country = t.country AND t.date = '" + + "' AND t.country = '" + + "'" ) .whenMatched() .updateAll() .whenNotMatched() .insertAll() .execute() ``` This operation is now safe to run concurrently on different dates and countries. ### ConcurrentDeleteReadException This exception occurs when a concurrent operation deleted a file that your operation read. Common causes are a `DELETE`, `UPDATE`, or `MERGE` operation that rewrites files. ### ConcurrentDeleteDeleteException This exception occurs when a concurrent operation deleted a file that your operation also deletes. This could be caused by two concurrent compaction operations rewriting the same files. ### MetadataChangedException This exception occurs when a concurrent transaction updates the metadata of a Delta table. Common causes are `ALTER TABLE` operations or writes to your Delta table that update the schema of the table. ### ConcurrentTransactionException If a streaming query using the same checkpoint location is started multiple times concurrently and tries to write to the Delta table at the same time. You should never have two streaming queries use the same checkpoint location and run at the same time. ### ProtocolChangedException This exception can occur in the following cases: - When your Delta table is upgraded to a new version. For future operations to succeed you may need to upgrade your Delta Lake version. - When multiple writers are creating or replacing a table at the same time. - When multiple writers are writing to an empty path at the same time. ================================================ FILE: docs/src/content/docs/delta-apidoc.mdx ================================================ --- title: Delta Lake APIs description: Learn about the APIs provided by Delta Lake. --- import { Aside } from "@astrojs/starlight/components"; ## Delta Spark Delta Spark is a library for reading and writing Delta tables using Apache Spark™. For most read and write operations on Delta tables, you can use Apache Spark reader and writer APIs. For examples, see [Table batch reads and writes](/delta-batch/) and [Table streaming reads and writes](/delta-streaming/). However, there are some operations that are specific to Delta Lake and you must use Delta Lake APIs. For examples, see [Table utility commands](/delta-utility/). - [Scala API docs](/api/latest/scala/spark/io/delta/tables/index.html) - [Java API docs](/api/latest/java/spark/index.html) - [Python API docs](/api/latest/python/spark/index.html) ## Delta Kernel Delta Kernel is a library for operating on Delta tables. Specifically, it provides simple and narrow APIs for reading and writing to Delta tables without the need to understand the [Delta protocol](https://github.com/delta-io/delta/blob/master/PROTOCOL.md) details. You can use this library to do the following: - Read Delta tables from your applications. - Build a connector for a distributed engine like Apache Spark™, Apache Flink, or Trino for reading massive Delta tables. More details refer [here](https://github.com/delta-io/delta/blob/branch-3.0/kernel/USER_GUIDE.md). - [Java API docs](/api/latest/java/kernel/index.html) ## Delta Rust This [library](https://docs.rs/deltalake/latest/deltalake/) allows Rust (with Python bindings) low level access to Delta tables and is intended to be used with data processing frameworks like `datafusion`, `ballista`, `rust-dataframe`, `vega`, etc. ## Delta Standalone Delta Standalone, formerly known as the Delta Standalone Reader (DSR), is a JVM library to read and write Delta tables. Unlike Delta-Spark, this library doesn't use Spark to read or write tables and it has only a few transitive dependencies. It can be used by any application that cannot use a Spark cluster. More details refer [here](https://github.com/delta-io/delta/blob/master/connectors/README.md). - [Java API docs](/api/3.3.2/java/standalone/index.html) ## Delta Flink Flink/Delta Connector is a JVM library to read and write data from Apache Flink applications to Delta tables utilizing the Delta Standalone JVM library. More details refer [here](https://github.com/delta-io/delta/blob/master/connectors/flink/README.md). - [Java API docs](/api/3.3.2/java/flink/index.html) ================================================ FILE: docs/src/content/docs/delta-athena-integration.mdx ================================================ --- title: AWS Athena Delta Connector description: Learn how to set up an integration to enable you to read Delta tables from AWS Athena. --- # AWS Athena Delta Connector Since Athena [version 3](https://docs.aws.amazon.com/athena/latest/ug/engine-versions-reference-0003.html), Athena natively supports reading Delta Lake tables. For details on using the native Delta Lake connector, see [Querying Delta Lake tables](https://docs.aws.amazon.com/athena/latest/ug/delta-lake-tables.html). For Athena versions lower than [version 3](https://docs.aws.amazon.com/athena/latest/ug/engine-versions-reference-0003.html), you can use the manifest-based approach detailed in [Presto, Trino, and Athena to Delta Lake integration using manifests](/presto-integration). ================================================ FILE: docs/src/content/docs/delta-batch.mdx ================================================ --- title: Table batch reads and writes description: Learn how to perform batch reads and writes on Delta tables. --- import { Aside, Tabs, TabItem } from "@astrojs/starlight/components"; Delta Lake supports most of the options provided by Apache Spark DataFrame read and write APIs for performing batch reads and writes on tables. For many Delta Lake operations on tables, you enable integration with Apache Spark DataSourceV2 and Catalog APIs (since 3.0) by setting configurations when you create a new `SparkSession`. See [Configure SparkSession](#configure-sparksession). ## Create a table Delta Lake supports creating two types of tables—tables defined in the metastore and tables defined by path. To work with metastore-defined tables, you must enable integration with Apache Spark DataSourceV2 and Catalog APIs by setting configurations when you create a new `SparkSession`. See [Configure SparkSession](#configure-sparksession). You can create tables in the following ways: - **SQL DDL commands**: You can use standard SQL DDL commands supported in Apache Spark (for example, `CREATE TABLE` and `REPLACE TABLE`) to create Delta tables. ```sql CREATE TABLE IF NOT EXISTS default.people10m ( id INT, firstName STRING, middleName STRING, lastName STRING, gender STRING, birthDate TIMESTAMP, ssn STRING, salary INT ) USING DELTA CREATE OR REPLACE TABLE default.people10m ( id INT, firstName STRING, middleName STRING, lastName STRING, gender STRING, birthDate TIMESTAMP, ssn STRING, salary INT ) USING DELTA ``` SQL also supports creating a table at a path, without creating an entry in the Hive metastore. ```sql -- Create or replace table with path CREATE OR REPLACE TABLE delta.`/tmp/delta/people10m` ( id INT, firstName STRING, middleName STRING, lastName STRING, gender STRING, birthDate TIMESTAMP, ssn STRING, salary INT ) USING DELTA ``` - **`DataFrameWriter` API**: If you want to simultaneously create a table and insert data into it from Spark DataFrames or Datasets, you can use the Spark `DataFrameWriter` ([Scala or Java](https://spark.apache.org/docs/latest/api/latest/scala/org/apache/spark/sql/DataFrameWriter.html) and [Python](https://spark.apache.org/docs/latest/api/latest/python/reference/pyspark.sql/io.html)). ```python # Create table in the metastore using DataFrame's schema and write data to it df.write.format("delta").saveAsTable("default.people10m") # Create or replace partitioned table with path using DataFrame's schema and write/overwrite data to it df.write.format("delta").mode("overwrite").save("/tmp/delta/people10m") ``` ```scala // Create table in the metastore using DataFrame's schema and write data to it df.write.format("delta").saveAsTable("default.people10m") // Create table with path using DataFrame's schema and write data to it df.write.format("delta").mode("overwrite").save("/tmp/delta/people10m") ``` You can also create Delta tables using the Spark `DataFrameWriterV2` API. - **`DeltaTableBuilder` API**: You can also use the `DeltaTableBuilder` API in Delta Lake to create tables. Compared to the DataFrameWriter APIs, this API makes it easier to specify additional information like column comments, table properties, and [generated columns](#use-generated-columns). See the [API documentation](/delta-apidoc/) for details. ### Partition data You can partition data to speed up queries or DML that have predicates involving the partition columns. To partition data when you create a Delta table, specify a partition by columns. The following example partitions by gender. ```sql -- Create table in the metastore CREATE TABLE default.people10m ( id INT, firstName STRING, middleName STRING, lastName STRING, gender STRING, birthDate TIMESTAMP, ssn STRING, salary INT ) USING DELTA PARTITIONED BY (gender) ``` ```python df.write.format("delta").partitionBy("gender").saveAsTable("default.people10m") DeltaTable.create(spark) \ .tableName("default.people10m") \ .addColumn("id", "INT") \ .addColumn("firstName", "STRING") \ .addColumn("middleName", "STRING") \ .addColumn("lastName", "STRING", comment = "surname") \ .addColumn("gender", "STRING") \ .addColumn("birthDate", "TIMESTAMP") \ .addColumn("ssn", "STRING") \ .addColumn("salary", "INT") \ .partitionedBy("gender") \ .execute() ``` ```scala df.write.format("delta").partitionBy("gender").saveAsTable("default.people10m") DeltaTable.createOrReplace(spark) .tableName("default.people10m") .addColumn("id", "INT") .addColumn("firstName", "STRING") .addColumn("middleName", "STRING") .addColumn( DeltaTable.columnBuilder("lastName") .dataType("STRING") .comment("surname") .build()) .addColumn("lastName", "STRING", comment = "surname") .addColumn("gender", "STRING") .addColumn("birthDate", "TIMESTAMP") .addColumn("ssn", "STRING") .addColumn("salary", "INT") .partitionedBy("gender") .execute() ``` To determine whether a table contains a specific partition, use the statement `SELECT COUNT(*) > 0 FROM WHERE = `. If the partition exists, `true` is returned. For example: ```sql SELECT COUNT(*) > 0 AS `Partition exists` FROM default.people10m WHERE gender = "M" ``` ```python display(spark.sql("SELECT COUNT(*) > 0 AS `Partition exists` FROM default.people10m WHERE gender = 'M'")) ``` ```scala display(spark.sql("SELECT COUNT(*) > 0 AS `Partition exists` FROM default.people10m WHERE gender = 'M'")) ``` ### Control data location For tables defined in the metastore, you can optionally specify the `LOCATION` as a path. Tables created with a specified `LOCATION` are considered unmanaged by the metastore. Unlike a managed table, where no path is specified, an unmanaged table's files are not deleted when you `DROP` the table. When you run `CREATE TABLE` with a `LOCATION` that _already_ contains data stored using Delta Lake, Delta Lake does the following: - If you specify _only the table name and location_, for example: ```sql CREATE TABLE default.people10m USING DELTA LOCATION '/tmp/delta/people10m' ``` the table in the metastore automatically inherits the schema, partitioning, and table properties of the existing data. This functionality can be used to "import" data into the metastore. - If you specify _any configuration_ (schema, partitioning, or table properties), Delta Lake verifies that the specification exactly matches the configuration of the existing data. ### Use generated columns Delta Lake supports generated columns which are a special type of columns whose values are automatically generated based on a user-specified function over other columns in the Delta table. When you write to a table with generated columns and you do not explicitly provide values for them, Delta Lake automatically computes the values. For example, you can automatically generate a date column (for partitioning the table by date) from the timestamp column; any writes into the table need only specify the data for the timestamp column. However, if you explicitly provide values for them, the values must satisfy the [constraint](/delta-constraints/) `( <=> ) IS TRUE` or the write will fail with an error. The following example shows how to create a table with generated columns: ```python DeltaTable.create(spark) \ .tableName("default.people10m") \ .addColumn("id", "INT") \ .addColumn("firstName", "STRING") \ .addColumn("middleName", "STRING") \ .addColumn("lastName", "STRING", comment = "surname") \ .addColumn("gender", "STRING") \ .addColumn("birthDate", "TIMESTAMP") \ .addColumn("dateOfBirth", DateType(), generatedAlwaysAs="CAST(birthDate AS DATE)") \ .addColumn("ssn", "STRING") \ .addColumn("salary", "INT") \ .partitionedBy("gender") \ .execute() ``` ```scala DeltaTable.create(spark) .tableName("default.people10m") .addColumn("id", "INT") .addColumn("firstName", "STRING") .addColumn("middleName", "STRING") .addColumn( DeltaTable.columnBuilder("lastName") .dataType("STRING") .comment("surname") .build()) .addColumn("lastName", "STRING", comment = "surname") .addColumn("gender", "STRING") .addColumn("birthDate", "TIMESTAMP") .addColumn( DeltaTable.columnBuilder("dateOfBirth") .dataType(DateType) .generatedAlwaysAs("CAST(dateOfBirth AS DATE)") .build()) .addColumn("ssn", "STRING") .addColumn("salary", "INT") .partitionedBy("gender") .execute() ``` Generated columns are stored as if they were normal columns. That is, they occupy storage. The following restrictions apply to generated columns: - A generation expression can use any SQL functions in Spark that always return the same result when given the same argument values, except the following types of functions: - User-defined functions. - Aggregate functions. - Window functions. - Functions returning multiple rows. - For Delta Lake 1.1.0 and above, `MERGE` operations support generated columns when you set `spark.databricks.delta.schema.autoMerge.enabled` to true. Delta Lake may be able to generate partition filters for a query whenever a partition column is defined by one of the following expressions: - `CAST(col AS DATE)` and the type of `col` is `TIMESTAMP`. - `YEAR(col)` and the type of `col` is `TIMESTAMP`. - Two partition columns defined by `YEAR(col), MONTH(col)` and the type of `col` is `TIMESTAMP`. - Three partition columns defined by `YEAR(col), MONTH(col), DAY(col)` and the type of `col` is `TIMESTAMP`. - Four partition columns defined by `YEAR(col), MONTH(col), DAY(col), HOUR(col)` and the type of `col` is `TIMESTAMP`. - `SUBSTRING(col, pos, len)` and the type of `col` is `STRING` - `DATE_FORMAT(col, format)` and the type of `col` is `TIMESTAMP`. - `DATE_TRUNC(format, col)` and the type of the `col` is `TIMESTAMP` or `DATE`. - `TRUNC(col, format)` and type of the `col` is either `TIMESTAMP` or `DATE`. If a partition column is defined by one of the preceding expressions, and a query filters data using the underlying base column of a generation expression, Delta Lake looks at the relationship between the base column and the generated column, and populates partition filters based on the generated partition column if possible. For example, given the following table: ```python DeltaTable.create(spark) \ .tableName("default.events") \ .addColumn("eventId", "BIGINT") \ .addColumn("data", "STRING") \ .addColumn("eventType", "STRING") \ .addColumn("eventTime", "TIMESTAMP") \ .addColumn("eventDate", "DATE", generatedAlwaysAs="CAST(eventTime AS DATE)") \ .partitionedBy("eventType", "eventDate") \ .execute() ``` If you then run the following query: ```python spark.sql('SELECT * FROM default.events WHERE eventTime >= "2020-10-01 00:00:00" <= "2020-10-01 12:00:00"') ``` Delta Lake automatically generates a partition filter so that the preceding query only reads the data in partition `date=2020-10-01` even if a partition filter is not specified. As another example, given the following table: ```python DeltaTable.create(spark) \ .tableName("default.events") \ .addColumn("eventId", "BIGINT") \ .addColumn("data", "STRING") \ .addColumn("eventType", "STRING") \ .addColumn("eventTime", "TIMESTAMP") \ .addColumn("year", "INT", generatedAlwaysAs="YEAR(eventTime)") \ .addColumn("month", "INT", generatedAlwaysAs="MONTH(eventTime)") \ .addColumn("day", "INT", generatedAlwaysAs="DAY(eventTime)") \ .partitionedBy("eventType", "year", "month", "day") \ .execute() ``` If you then run the following query: ```python spark.sql('SELECT * FROM default.events WHERE eventTime >= "2020-10-01 00:00:00" <= "2020-10-01 12:00:00"') ``` Delta Lake automatically generates a partition filter so that the preceding query only reads the data in partition `year=2020/month=10/day=01` even if a partition filter is not specified. You can use an [EXPLAIN](https://spark.apache.org/docs/latest/sql-ref-syntax-qry-explain.html) clause and check the provided plan to see whether Delta Lake automatically generates any partition filters. ### Use identity columns Delta Lake identity columns are supported in Delta Lake 3.3 and above. They are a type of generated column that assigns unique values for each record inserted into a table. The following example shows how to declare an identity column during a create table command: ```python from delta.tables import DeltaTable, IdentityGenerator from pyspark.sql.types import LongType DeltaTable.create() .tableName("table_name") .addColumn("id_col1", dataType=LongType(), generatedAlwaysAs=IdentityGenerator()) .addColumn("id_col2", dataType=LongType(), generatedAlwaysAs=IdentityGenerator(start=-1, step=1)) .addColumn("id_col3", dataType=LongType(), generatedByDefaultAs=IdentityGenerator()) .addColumn("id_col4", dataType=LongType(), generatedByDefaultAs=IdentityGenerator(start=-1, step=1)) .execute() ``` ```scala import io.delta.tables.DeltaTable import org.apache.spark.sql.types.LongType DeltaTable.create(spark) .tableName("table_name") .addColumn( DeltaTable.columnBuilder(spark, "id_col1") .dataType(LongType) .generatedAlwaysAsIdentity().build()) .addColumn( DeltaTable.columnBuilder(spark, "id_col2") .dataType(LongType) .generatedAlwaysAsIdentity(start = -1L, step = 1L).build()) .addColumn( DeltaTable.columnBuilder(spark, "id_col3") .dataType(LongType) .generatedByDefaultAsIdentity().build()) .addColumn( DeltaTable.columnBuilder(spark, "id_col4") .dataType(LongType) .generatedByDefaultAsIdentity(start = -1L, step = 1L).build()) .execute() ``` You can optionally specify the following: - A starting value. - A step size, which can be positive or negative. Both the starting value and step size default to `1`. You cannot specify a step size of `0`. Values assigned by identity columns are unique and increment in the direction of the specified step, and in multiples of the specified step size, but are not guaranteed to be contiguous. For example, with a starting value of `0` and a step size of `2`, all values are positive even numbers but some even numbers might be skipped. When the identity column is specified to be `generated by default as identity`, insert operations can specify values for the identity column. Specify it to be `generated always as identity` to override the ability to manually set values. Identity columns only support `LongType`, and operations fail if the assigned value exceeds the range supported by `LongType`. You can use `ALTER TABLE table_name ALTER COLUMN column_name SYNC IDENTITY` to synchronize the metadata of an identity column with the actual data. When you write your own values to an identity column, it might not comply with the metadata. This option evaluates the state and updates the metadata to be consistent with the actual data. After this command, the next automatically assigned identity value will start from `start + (n + 1) * step`, where `n` is the smallest value that satisfies `start + n * step >= max()` (for a positive step). #### CTAS and identity columns You cannot define schema, identity column constraints, or any other table specifications when using a `CREATE TABLE table_name AS SELECT` (CTAS) statement. To create a new table with an identity column and populate it with existing data, do the following: 1. Create a table with the correct schema, including the identity column definition and other table properties. 2. Run an insertion operation. The following example define the identity column to be `generated by default as identity`. If data inserted into the table includes valid values for the identity column, these values are used. ```python from delta.tables import DeltaTable, IdentityGenerator from pyspark.sql.types import LongType, DateType DeltaTable.create(spark) .tableName("new_table") .addColumn("id", dataType=LongType(), generatedByDefaultAs=IdentityGenerator(start=5, step=1)) .addColumn("event_date", dataType=DateType()) .addColumn("some_value", dataType=LongType()) .execute() # Insert records including existing IDs old_table_df = spark.table("old_table").select("id", "event_date", "some_value") old_table_df.write .format("delta") .mode("append") .saveAsTable("new_table") # Insert records and generate new IDs new_records_df = spark.table("new_records").select("event_date", "some_value") new_records_df.write .format("delta") .mode("append") .saveAsTable("new_table") ``` ```scala import org.apache.spark.sql.types._ import io.delta.tables.DeltaTable DeltaTable.createOrReplace(spark) .tableName("new_table") .addColumn( DeltaTable.columnBuilder(spark, "id") .dataType(LongType) .generatedByDefaultAsIdentity(start = 5L, step = 1L) .build()) .addColumn( DeltaTable.columnBuilder(spark, "event_date") .dataType(DateType) .nullable(true) .build()) .addColumn( DeltaTable.columnBuilder(spark, "some_value") .dataType(LongType) .nullable(true) .build()) .execute() // Insert records including existing IDs val oldTableDF = spark.table("old_table").select("id", "event_date", "some_value") oldTableDF.write .format("delta") .mode("append") .saveAsTable("new_table") // Insert records and generate new IDs val newRecordsDF = spark.table("new_records").select("event_date", "some_value") newRecordsDF.write .format("delta") .mode("append") .saveAsTable("new_table") ``` #### Identity column limitations The following limitations exist when working with identity columns: - Concurrent transactions are not supported on tables with identity columns enabled. - You cannot partition a table by an identity column. - You cannot `ADD`, `REPLACE`, or `CHANGE` an identity column. - You cannot update the value of an identity column for an existing record. ### Specify default values for columns Delta enables the specification of [default expressions](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#default-columns) for columns in Delta tables. When users write to these tables without explicitly providing values for certain columns, or when they explicitly use the `DEFAULT` SQL keyword for a column, Delta automatically generates default values for those columns. For more information, please refer to the dedicated documentation page. ### Use special characters in column names By default, special characters such as spaces and any of the characters `,;{}()\n\t=` are not supported in table column names. To include these special characters in a table's column name, enable column mapping. ### Default table properties Delta Lake configurations set in the SparkSession override the default [table properties](/table-properties/) for new Delta Lake tables created in the session. The prefix used in the SparkSession is different from the configurations used in the table properties. | Delta Lake conf | SparkSession conf | | --------------- | --------------------------------------------------- | | `delta.` | `spark.databricks.delta.properties.defaults.` | For example, to set the `delta.appendOnly = true` property for all new Delta Lake tables created in a session, set the following: ```sql SET spark.databricks.delta.properties.defaults.appendOnly = true ``` ## Read a table You can load a Delta table as a DataFrame by specifying a table name or a path: ```sql SELECT * FROM default.people10m -- query table in the metastore SELECT * FROM delta.`/tmp/delta/people10m` -- query table by path ``` ```python spark.table("default.people10m") # query table in the metastore spark.read.format("delta").load("/tmp/delta/people10m") # query table by path ``` ```scala spark.table("default.people10m") // query table in the metastore spark.read.format("delta").load("/tmp/delta/people10m") // create table by path import io.delta.implicits._ spark.read.delta("/tmp/delta/people10m") ``` The DataFrame returned automatically reads the most recent snapshot of the table for any query; you never need to run `REFRESH TABLE`. Delta Lake automatically uses partitioning and statistics to read the minimum amount of data when there are applicable predicates in the query. ## Query an older snapshot of a table (time travel) Delta Lake time travel allows you to query an older snapshot of a Delta table. Time travel has many use cases, including: - Re-creating analyses, reports, or outputs (for example, the output of a machine learning model). This could be useful for debugging or auditing, especially in regulated industries. - Writing complex temporal queries. - Fixing mistakes in your data. - Providing snapshot isolation for a set of queries for fast changing tables. This section describes the supported methods for querying older versions of tables, data retention concerns, and provides examples. ### Syntax This section shows how to query an older version of a Delta table. #### SQL `AS OF` syntax ```sql SELECT * FROM table_name TIMESTAMP AS OF timestamp_expression SELECT * FROM table_name VERSION AS OF version ``` - `timestamp_expression` can be any one of: - `'2018-10-18T22:15:12.013Z'`, that is, a string that can be cast to a timestamp - `cast('2018-10-18 13:36:32 CEST' as timestamp)` - `'2018-10-18'`, that is, a date string - `current_timestamp() - interval 12 hours` - `date_sub(current_date(), 1)` - Any other expression that is or can be cast to a timestamp - `version` is a long value that can be obtained from the output of `DESCRIBE HISTORY table_spec`. Neither `timestamp_expression` nor `version` can be subqueries. ##### Example ```sql SELECT * FROM default.people10m TIMESTAMP AS OF '2018-10-18T22:15:12.013Z' SELECT * FROM delta.`/tmp/delta/people10m` VERSION AS OF 123 ``` #### DataFrameReader options DataFrameReader options allow you to create a DataFrame from a Delta table that is fixed to a specific version of the table. ```python df1 = spark.read.format("delta").option("timestampAsOf", timestamp_string).load("/tmp/delta/people10m") df2 = spark.read.format("delta").option("versionAsOf", version).load("/tmp/delta/people10m") ``` For `timestamp_string`, only date or timestamp strings are accepted. For example, `"2019-01-01"` and `"2019-01-01T00:00:00.000Z"`. A common pattern is to use the latest state of the Delta table throughout the execution of a job to update downstream applications. Because Delta tables auto update, a DataFrame loaded from a Delta table may return different results across invocations if the underlying data is updated. By using time travel, you can fix the data returned by the DataFrame across invocations: ```python history = spark.sql("DESCRIBE HISTORY delta.`/tmp/delta/people10m`") latest_version = history.selectExpr("max(version)").collect() df = spark.read.format("delta").option("versionAsOf", latest_version[0][0]).load("/tmp/delta/people10m") ``` ##### Examples - Fix accidental deletes to a table for the user 111: ```python yesterday = spark.sql("SELECT CAST(date_sub(current_date(), 1) AS STRING)").collect()[0][0] df = spark.read.format("delta").option("timestampAsOf", yesterday).load("/tmp/delta/events") df.where("userId = 111").write.format("delta").mode("append").save("/tmp/delta/events") ``` - Fix accidental incorrect updates to a table: ```python yesterday = spark.sql("SELECT CAST(date_sub(current_date(), 1) AS STRING)").collect()[0][0] df = spark.read.format("delta").option("timestampAsOf", yesterday).load("/tmp/delta/events") df.createOrReplaceTempView("my_table_yesterday") spark.sql(''' MERGE INTO delta.`/tmp/delta/events` target USING my_table_yesterday source ON source.userId = target.userId WHEN MATCHED THEN UPDATE SET * ''') ``` - Query the number of new customers added over the last week: ```python last_week = spark.sql("SELECT CAST(date_sub(current_date(), 7) AS STRING)").collect()[0][0] df = spark.read.format("delta").option("timestampAsOf", last_week).load("/tmp/delta/events") last_week_count = df.select("userId").distinct().count() count = spark.read.format("delta").load("/tmp/delta/events").select("userId").distinct().count() new_customers_count = count - last_week_count ``` ### Data retention To time travel to a previous version, you must retain _both_ the log and the data files for that version. The data files backing a Delta table are _never_ deleted automatically; data files are deleted only when you run [VACUUM](/delta-utility#remove-files-no-longer-referenced-by-a-delta-table). `VACUUM` _does not_ delete Delta log files; log files are automatically cleaned up after checkpoints are written. By default you can time travel to a Delta table up to 30 days old unless you have: - Run `VACUUM` on your Delta table. - Changed the data or log file retention periods using the following [table properties](/table-properties/): - `delta.logRetentionDuration = "interval "`: controls how long the history for a table is kept. The default is `interval 30 days`. Each time a checkpoint is written, Delta automatically cleans up log entries older than the retention interval. If you set this config to a large enough value, many log entries are retained. This should not impact performance as operations against the log are constant time. Operations on history are parallel but will become more expensive as the log size increases. - `delta.deletedFileRetentionDuration = "interval "`: controls how long ago a file must have been deleted _before being a candidate for_ `VACUUM`. The default is `interval 7 days`. To access 30 days of historical data even if you run `VACUUM` on the Delta table, set `delta.deletedFileRetentionDuration = "interval 30 days"`. This setting may cause your storage costs to go up. ## Write to table ### Append To atomically add new data to an existing Delta table, use `append` mode: ```sql INSERT INTO default.people10m SELECT * FROM morePeople ``` ```python df.write.format("delta").mode("append").save("/tmp/delta/people10m") df.write.format("delta").mode("append").saveAsTable("default.people10m") ``` ```scala df.write.format("delta").mode("append").save("/tmp/delta/people10m") df.write.format("delta").mode("append").saveAsTable("default.people10m") import io.delta.implicits._ df.write.mode("append").delta("/tmp/delta/people10m") ``` ### Overwrite To atomically replace all the data in a table, use `overwrite` mode: ```sql INSERT OVERWRITE TABLE default.people10m SELECT * FROM morePeople ``` ```python df.write.format("delta").mode("overwrite").save("/tmp/delta/people10m") df.write.format("delta").mode("overwrite").saveAsTable("default.people10m") ``` ```scala df.write.format("delta").mode("overwrite").save("/tmp/delta/people10m") df.write.format("delta").mode("overwrite").saveAsTable("default.people10m") import io.delta.implicits._ df.write.mode("overwrite").delta("/tmp/delta/people10m") ``` You can selectively overwrite only the data that matches an arbitrary expression. This feature is available with DataFrames in Delta Lake 1.1.0 and above and supported in SQL in Delta Lake 2.4.0 and above. The following command atomically replaces events in January in the target table, which is partitioned by `start_date`, with the data in `replace_data`: ```sql INSERT INTO TABLE events REPLACE WHERE start_data >= '2017-01-01' AND end_date <= '2017-01-31' SELECT * FROM replace_data ``` ```python replace_data.write \ .format("delta") \ .mode("overwrite") \ .option("replaceWhere", "start_date >= '2017-01-01' AND end_date <= '2017-01-31'") \ .save("/tmp/delta/events") ``` ```scala replace_data.write .format("delta") .mode("overwrite") .option("replaceWhere", "start_date >= '2017-01-01' AND end_date <= '2017-01-31'") .save("/tmp/delta/events") ``` This sample code writes out the data in `replace_data`, validates that it all matches the predicate, and performs an atomic replacement. If you want to write out data that doesn't all match the predicate, to replace the matching rows in the target table, you can disable the constraint check by setting `spark.databricks.delta.replaceWhere.constraintCheck.enabled` to false: ```sql SET spark.databricks.delta.replaceWhere.constraintCheck.enabled=false ``` ```python spark.conf.set("spark.databricks.delta.replaceWhere.constraintCheck.enabled", False) ``` ```scala spark.conf.set("spark.databricks.delta.replaceWhere.constraintCheck.enabled", false) ``` In Delta Lake 1.0.0 and below, `replaceWhere` overwrites data matching a predicate over partition columns only. The following command atomically replaces the month in January in the target table, which is partitioned by `date`, with the data in `df`: ```python df.write \ .format("delta") \ .mode("overwrite") \ .option("replaceWhere", "birthDate >= '2017-01-01' AND birthDate <= '2017-01-31'") \ .save("/tmp/delta/people10m") ``` ```scala df.write .format("delta") .mode("overwrite") .option("replaceWhere", "birthDate >= '2017-01-01' AND birthDate <= '2017-01-31'") .save("/tmp/delta/people10m") ``` In Delta Lake 1.1.0 and above, if you want to fall back to the old behavior, you can disable the `spark.databricks.delta.replaceWhere.dataColumns.enabled` flag: ```sql SET spark.databricks.delta.replaceWhere.dataColumns.enabled=false ``` ```python spark.conf.set("spark.databricks.delta.replaceWhere.dataColumns.enabled", False) ``` ```scala spark.conf.set("spark.databricks.delta.replaceWhere.dataColumns.enabled", false) ``` #### Dynamic Partition Overwrites Delta Lake 2.0 and above supports _dynamic_ partition overwrite mode for partitioned tables. When in dynamic partition overwrite mode, we overwrite all existing data in each logical partition for which the write will commit new data. Any existing logical partitions for which the write does not contain data will remain unchanged. This mode is only applicable when data is being written in overwrite mode: either `INSERT OVERWRITE` in SQL, or a DataFrame write with `df.write.mode("overwrite")`. Configure dynamic partition overwrite mode by setting the Spark session configuration `spark.sql.sources.partitionOverwriteMode` to `dynamic`. You can also enable this by setting the `DataFrameWriter` option `partitionOverwriteMode` to `dynamic`. If present, the query-specific option overrides the mode defined in the session configuration. The default for `partitionOverwriteMode` is `static`. ```sql SET spark.sql.sources.partitionOverwriteMode=dynamic; INSERT OVERWRITE TABLE default.people10m SELECT * FROM morePeople; ``` ```python df.write \ .format("delta") \ .mode("overwrite") \ .option("partitionOverwriteMode", "dynamic") \ .saveAsTable("default.people10m") ``` ```scala df.write .format("delta") .mode("overwrite") .option("partitionOverwriteMode", "dynamic") .saveAsTable("default.people10m") ``` For Delta Lake support for updating tables, see [Table deletes, updates, and merges](/delta-update/). ### Limit rows written in a file You can use the SQL session configuration `spark.sql.files.maxRecordsPerFile` to specify the maximum number of records to write to a single file for a Delta Lake table. Specifying a value of zero or a negative value represents no limit. You can also use the DataFrameWriter option `maxRecordsPerFile` when using the DataFrame APIs to write to a Delta Lake table. When `maxRecordsPerFile` is specified, the value of the SQL session configuration `spark.sql.files.maxRecordsPerFile` is ignored. ```python df.write.format("delta") \ .mode("append") \ .option("maxRecordsPerFile", "10000") \ .save("/tmp/delta/people10m") ``` ```scala df.write.format("delta") .mode("append") .option("maxRecordsPerFile", "10000") .save("/tmp/delta/people10m") ``` ### Idempotent writes Sometimes a job that writes data to a Delta table is restarted due to various reasons (for example, job encounters a failure). The failed job may or may not have written the data to Delta table before terminating. In the case where the data is written to the Delta table, the restarted job writes the same data to the Delta table which results in duplicate data. To address this, Delta tables support the following `DataFrameWriter` options to make the writes idempotent: - `txnAppId`: A unique string that you can pass on each `DataFrame` write. For example, this can be the name of the job. - `txnVersion`: A monotonically increasing number that acts as transaction version. This number needs to be unique for data that is being written to the Delta table(s). For example, this can be the epoch seconds of the instant when the query is attempted for the first time. Any subsequent restarts of the same job needs to have the same value for `txnVersion`. The above combination of options needs to be unique for each new data that is being ingested into the Delta table and the `txnVersion` needs to be higher than the last data that was ingested into the Delta table. For example: - Last successfully written data contains option values as `dailyETL:23423` (`txnAppId:txnVersion`). - Next write of data should have `txnAppId = dailyETL` and `txnVersion` as at least `23424` (one more than the last written data `txnVersion`). - Any attempt to write data with `txnAppId = dailyETL` and `txnVersion` as `23422` or less is ignored because the `txnVersion` is less than the last recorded `txnVersion` in the table. - Attempt to write data with `txnAppId:txnVersion` as `anotherETL:23424` is successful writing data to the table as it contains a different `txnAppId` compared to the same option value in last ingested data. You can also configure idempotent writes by setting the Spark session configuration `spark.databricks.delta.write.txnAppId` and `spark.databricks.delta.write.txnVersion`. In addition, you can set `spark.databricks.delta.write.txnVersion.autoReset.enabled` to true to automatically reset `spark.databricks.delta.write.txnVersion` after every write. When both the writer options and session configuration are set, we will use the writer option values. #### Example ```sql SET spark.databricks.delta.write.txnAppId = ...; SET spark.databricks.delta.write.txnVersion = ...; SET spark.databricks.delta.write.txnVersion.autoReset.enabled = true; -- if set to true, this will reset txnVersion after every write ``` ```python app_id = ... # A unique string that is used as an application ID. version = ... # A monotonically increasing number that acts as transaction version. dataFrame.write.format(...).option("txnVersion", version).option("txnAppId", app_id).save(...) ``` ```scala val appId = ... // A unique string that is used as an application ID. version = ... // A monotonically increasing number that acts as transaction version. dataFrame.write.format(...).option("txnVersion", version).option("txnAppId", appId).save(...) ``` ### Set user-defined commit metadata You can specify user-defined strings as metadata in commits made by these operations, either using the DataFrameWriter option `userMetadata` or the SparkSession configuration `spark.databricks.delta.commitInfo.userMetadata`. If both of them have been specified, then the option takes preference. This user-defined metadata is readable in the [history](/delta-utility/#retrieve-delta-table-history) operation. ```sql SET spark.databricks.delta.commitInfo.userMetadata=overwritten-for-fixing-incorrect-data INSERT OVERWRITE default.people10m SELECT * FROM morePeople ``` ```python df.write.format("delta") \ .mode("overwrite") \ .option("userMetadata", "overwritten-for-fixing-incorrect-data") \ .save("/tmp/delta/people10m") ``` ```scala df.write.format("delta") .mode("overwrite") .option("userMetadata", "overwritten-for-fixing-incorrect-data") .save("/tmp/delta/people10m") ``` ## Schema validation Delta Lake automatically validates that the schema of the DataFrame being written is compatible with the schema of the table. Delta Lake uses the following rules to determine whether a write from a DataFrame to a table is compatible: - All DataFrame columns must exist in the target table. If there are columns in the DataFrame not present in the table, an exception is raised. Columns present in the table but not in the DataFrame are set to null. - DataFrame column data types must match the column data types in the target table. If they don't match, an exception is raised. - DataFrame column names cannot differ only by case. This means that you cannot have columns such as "Foo" and "foo" defined in the same table. While you can use Spark in case sensitive or insensitive (default) mode, Parquet is case sensitive when storing and returning column information. Delta Lake is case-preserving but insensitive when storing the schema and has this restriction to avoid potential mistakes, data corruption, or loss issues. Delta Lake support DDL to add new columns explicitly and the ability to update schema automatically. If you specify other options, such as `partitionBy`, in combination with append mode, Delta Lake validates that they match and throws an error for any mismatch. When `partitionBy` is not present, appends automatically follow the partitioning of the existing data. ## Update table schema Delta Lake lets you update the schema of a table. The following types of changes are supported: - Adding new columns (at arbitrary positions) - Reordering existing columns You can make these changes explicitly using DDL or implicitly using DML. ### Explicitly update schema You can use the following DDL to explicitly change the schema of a table. #### Add columns ```sql ALTER TABLE table_name ADD COLUMNS (col_name data_type [COMMENT col_comment] [FIRST|AFTER colA_name], ...) ``` By default, nullability is `true`. To add a column to a nested field, use: ```sql ALTER TABLE table_name ADD COLUMNS (col_name.nested_col_name data_type [COMMENT col_comment] [FIRST|AFTER colA_name], ...) ``` ##### Example If the schema before running `ALTER TABLE boxes ADD COLUMNS (colB.nested STRING AFTER field1)` is: ``` - root | - colA | - colB | +-field1 | +-field2 ``` the schema after is: ``` - root | - colA | - colB | +-field1 | +-nested | +-field2 ``` #### Change column comment or ordering ```sql ALTER TABLE table_name ALTER [COLUMN] col_name col_name data_type [COMMENT col_comment] [FIRST|AFTER colA_name] ``` To change a column in a nested field, use: ```sql ALTER TABLE table_name ALTER [COLUMN] col_name.nested_col_name nested_col_name data_type [COMMENT col_comment] [FIRST|AFTER colA_name] ``` ##### Example If the schema before running `ALTER TABLE boxes CHANGE COLUMN colB.field2 field2 STRING FIRST` is: ``` - root | - colA | - colB | +-field1 | +-field2 ``` the schema after is: ``` - root | - colA | - colB | +-field2 | +-field1 ``` #### Replace columns ```sql ALTER TABLE table_name REPLACE COLUMNS (col_name1 col_type1 [COMMENT col_comment1], ...) ``` ##### Example When running the following DDL: ```sql ALTER TABLE boxes REPLACE COLUMNS (colC STRING, colB STRUCT, colA STRING) ``` if the schema before is: ``` - root | - colA | - colB | +-field1 | +-field2 ``` the schema after is: ``` - root | - colC | - colB | +-field2 | +-nested | +-field1 | - colA ``` #### Rename columns To rename columns without rewriting any of the columns' existing data, you must enable column mapping for the table. See [enable column mapping](/delta-column-mapping/). To rename a column: ```sql ALTER TABLE table_name RENAME COLUMN old_col_name TO new_col_name ``` To rename a nested field: ```sql ALTER TABLE table_name RENAME COLUMN col_name.old_nested_field TO new_nested_field ``` ##### Example When you run the following command: ```sql ALTER TABLE boxes RENAME COLUMN colB.field1 TO field001 ``` If the schema before is: ``` - root | - colA | - colB | +-field1 | +-field2 ``` Then the schema after is: ``` - root | - colA | - colB | +-field001 | +-field2 ``` #### Drop columns To drop columns as a metadata-only operation without rewriting any data files, you must enable column mapping for the table. See [enable column mapping](/delta-column-mapping/). To drop a column: ```sql ALTER TABLE table_name DROP COLUMN col_name ``` To drop multiple columns: ```sql ALTER TABLE table_name DROP COLUMNS (col_name_1, col_name_2) ``` #### Change column type or name You can change a column's type or name or drop a column by rewriting the table. To do this, use the `overwriteSchema` option: ##### Change a column type ```python spark.read.table(...) \ .withColumn("birthDate", col("birthDate").cast("date")) \ .write \ .format("delta") \ .mode("overwrite") .option("overwriteSchema", "true") \ .saveAsTable(...) ``` ##### Change a column name ```python spark.read.table(...) \ .withColumnRenamed("dateOfBirth", "birthDate") \ .write \ .format("delta") \ .mode("overwrite") \ .option("overwriteSchema", "true") \ .saveAsTable(...) ``` ### Automatic schema update Delta Lake can automatically update the schema of a table as part of a DML transaction (either appending or overwriting), and make the schema compatible with the data being written. #### Add columns Columns that are present in the DataFrame but missing from the table are automatically added as part of a write transaction when: - `write` or `writeStream` have `.option("mergeSchema", "true")` - `spark.databricks.delta.schema.autoMerge.enabled` is `true` When both options are specified, the option from the `DataFrameWriter` takes precedence. The added columns are appended to the end of the struct they are present in. Case is preserved when appending a new column. #### `NullType` columns Because Parquet doesn't support `NullType`, `NullType` columns are dropped from the DataFrame when writing into Delta tables, but are still stored in the schema. When a different data type is received for that column, Delta Lake merges the schema to the new data type. If Delta Lake receives a `NullType` for an existing column, the old schema is retained and the new column is dropped during the write. `NullType` in streaming is not supported. Since you must set schemas when using streaming this should be very rare. `NullType` is also not accepted for complex types such as `ArrayType` and `MapType`. ## Replace table schema By default, overwriting the data in a table does not overwrite the schema. When overwriting a table using mode `overwrite` without `replaceWhere`, you may still want to overwrite the schema of the data being written. You replace the schema and partitioning of the table by setting the `overwriteSchema` option to `true`: ```python df.write.option("overwriteSchema", "true") ``` ## Views on tables Delta Lake supports the creation of views on top of Delta tables just like you might with a data source table. The core challenge when you operate with views is resolving the schemas. If you alter a Delta table schema, you must recreate derivative views to account for any additions to the schema. For instance, if you add a new column to a Delta table, you must make sure that this column is available in the appropriate views built on top of that base table. ## Table properties You can store your own metadata as a table property using `TBLPROPERTIES` in `CREATE` and `ALTER`. You can then `SHOW` that metadata. For example: ```sql ALTER TABLE default.people10m SET TBLPROPERTIES ('department' = 'accounting', 'delta.appendOnly' = 'true'); -- Show the table's properties. SHOW TBLPROPERTIES default.people10m; -- Show just the 'department' table property. SHOW TBLPROPERTIES default.people10m ('department'); ``` `TBLPROPERTIES` are stored as part of Delta table metadata. You cannot define new `TBLPROPERTIES` in a `CREATE` statement if a Delta table already exists in a given location. In addition, to tailor behavior and performance, Delta Lake supports certain Delta table properties: - Block deletes and updates in a Delta table: `delta.appendOnly=true`. - Configure the [time travel](#query-an-older-snapshot-of-a-table-time-travel) retention properties: `delta.logRetentionDuration=` and `delta.deletedFileRetentionDuration=`. For details, see [Data retention](#data-retention). - Configure the number of columns for which statistics are collected: `delta.dataSkippingNumIndexedCols=n`. This property indicates to the writer that statistics are to be collected only for the first `n` columns in the table. Also the data skipping code ignores statistics for any column beyond this column index. This property takes affect only for new data that is written out. You can also set `delta.`-prefixed properties during the first commit to a Delta table using Spark configurations. For example, to initialize a Delta table with the property `delta.appendOnly=true`, set the Spark configuration `spark.databricks.delta.properties.defaults.appendOnly` to `true`. For example: ```sql spark.sql("SET spark.databricks.delta.properties.defaults.appendOnly = true") ``` ```python spark.conf.set("spark.databricks.delta.properties.defaults.appendOnly", "true") ``` ```scala spark.conf.set("spark.databricks.delta.properties.defaults.appendOnly", "true") ``` See also the [Delta table properties reference](/table-properties/). ## Syncing table schema and properties to the Hive metastore You can enable asynchronous syncing of table schema and properties to the metastore by setting `spark.databricks.delta.catalog.update.enabled` to `true`. Whenever the Delta client detects that either of these two were changed due to an update, it will sync the changes to the metastore. The schema is stored in the table properties in HMS. If the schema is small, it will be stored directly under the key `spark.sql.sources.schema`: ```json { "spark.sql.sources.schema": "{'name':'col1','type':'string','nullable':true, 'metadata':{}},{'name':'col2','type':'string','nullable':true,'metadata':{}}" } ``` If Schema is large, the schema will be broken down into multiple parts. Appending them together should give the correct schema. For example: ```json { "spark.sql.sources.schema.numParts": "4", "spark.sql.sources.schema.part.1": "{'name':'col1','type':'string','nullable':tr", "spark.sql.sources.schema.part.2": "ue, 'metadata':{}},{'name':'co", "spark.sql.sources.schema.part.3": "l2','type':'string','nullable':true,'meta", "spark.sql.sources.schema.part.4": "data':{}}" } ``` ## Table metadata Delta Lake has rich features for exploring table metadata. It supports `SHOW COLUMNS` and `DESCRIBE TABLE`. It also provides the following unique commands: ### `DESCRIBE DETAIL` Provides information about schema, partitioning, table size, and so on. For details, see [Retrieve Delta table details](/delta-utility/#retrieve-delta-table-history). ### `DESCRIBE HISTORY` Provides provenance information, including the operation, user, and so on, and operation metrics for each write to a table. Table history is retained for 30 days. For details, see [Retrieve Delta table history](/delta-utility/#retrieve-delta-table-history). ## Configure SparkSession For many Delta Lake operations, you enable integration with Apache Spark DataSourceV2 and Catalog APIs (since 3.0) by setting the following configurations when you create a new `SparkSession`. ```python from pyspark.sql import SparkSession spark = SparkSession \ .builder \ .appName("...") \ .master("...") \ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \ .getOrCreate() ``` ```scala import org.apache.spark.sql.SparkSession val spark = SparkSession .builder() .appName("...") .master("...") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") .getOrCreate() ``` ```java import org.apache.spark.sql.SparkSession; SparkSession spark = SparkSession .builder() .appName("...") .master("...") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") .getOrCreate(); ``` Alternatively, you can add configurations when submitting your Spark application using `spark-submit` or when starting `spark-shell` or `pyspark` by specifying them as command-line parameters. ```bash spark-submit --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" ... ``` ```bash pyspark --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" ``` ## Configure storage credentials Delta Lake uses Hadoop FileSystem APIs to access storage systems. The credentials for storage systems usually can be set through Hadoop configurations. Delta Lake provides multiple ways to set Hadoop configurations similar to Apache Spark. ### Spark configurations When you start a Spark application on a cluster, you can set the Spark configurations in the form of `spark.hadoop.*` to pass your custom Hadoop configurations. For example, setting a value for `spark.hadoop.a.b.c` will pass the value as a Hadoop configuration `a.b.c`, and Delta Lake will use it to access Hadoop FileSystem APIs. See [Spark documentation](http://spark.apache.org/docs/latest/configuration.html#custom-hadoophive-configuration) for more details. ### SQL session configurations Spark SQL will pass all of the current [SQL session configurations](http://spark.apache.org/docs/latest/configuration.html#runtime-sql-configuration) to Delta Lake, and Delta Lake will use them to access Hadoop FileSystem APIs. For example, `SET a.b.c=x.y.z` will tell Delta Lake to pass the value `x.y.z` as a Hadoop configuration `a.b.c`, and Delta Lake will use it to access Hadoop FileSystem APIs. ### DataFrame options Besides setting Hadoop file system configurations through the Spark (cluster) configurations or SQL session configurations, Delta supports reading Hadoop file system configurations from `DataFrameReader` and `DataFrameWriter` options (that is, option keys that start with the `fs.` prefix) when the table is read or written, by using `DataFrameReader.load(path)` or `DataFrameWriter.save(path)`. For example, you can pass your storage credentials through DataFrame options: ```python df1 = spark.read.format("delta") \ .option("fs.azure.account.key..dfs.core.windows.net", "") \ .read("...") df2 = spark.read.format("delta") \ .option("fs.azure.account.key..dfs.core.windows.net", "") \ .read("...") df1.union(df2).write.format("delta") \ .mode("overwrite") \ .option("fs.azure.account.key..dfs.core.windows.net", "") \ .save("...") ``` ```scala val df1 = spark.read.format("delta") .option("fs.azure.account.key..dfs.core.windows.net", "") .read("...") val df2 = spark.read.format("delta") .option("fs.azure.account.key..dfs.core.windows.net", "") .read("...") df1.union(df2).write.format("delta") .mode("overwrite") .option("fs.azure.account.key..dfs.core.windows.net", "") .save("...") ``` You can find the details of the Hadoop file system configurations for your storage in [Storage configuration](/delta-storage/). ================================================ FILE: docs/src/content/docs/delta-catalog-managed-tables.mdx ================================================ --- title: Use catalog-managed tables description: Learn how to enable and use catalog-managed commits in Delta Lake. --- import { Tabs, TabItem, Aside } from "@astrojs/starlight/components"; [Catalog-Managed Tables](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#catalog-managed-tables) introduce a `catalogManaged` reader-writer table feature that changes how Delta Lake discovers and accesses tables. With this feature enabled, the catalog coordinates commit atomicity, allowing for features like multi-table transactions that are difficult to achieve with filesystem-only primitives. ## Overview By default, Delta Lake relies entirely on the filesystem for read-time discovery and write-time commit atomicity. Each table manages its own transaction logs and conflict detection independently. Catalog-managed tables shift this responsibility to the managing catalog, which allows the catalog to orchestrate commits across multiple tables within a single transaction boundary while maintaining Delta Lake's ACID guarantees. ## Requirements - Catalog-managed tables requires the following Delta protocols: - Reader version 3 or above. - Writer version 7 or above. - The [In-Commit Timestamps](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#in-commit-timestamps) table feature must be enabled, as commit publishing can occur asynchronously and file modification timestamps may not reflect actual commit times. - The [VACUUM Protocol Check](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#vacuum-protocol-check) table feature must be enabled to provide proper vacuum operations on catalog-managed tables. ## Enable catalog-managed commits You can enable catalog-managed commits for new tables when using a catalog that supports this feature, such as [Unity Catalog](https://www.unitycatalog.io/). ### Enable catalog-managed commits for new tables Enable the `catalogManaged` table feature by setting the following table property when creating a table: ```sql CREATE TABLE sales_data ( sale_id BIGINT, amount DECIMAL(10,2), sale_date DATE ) TBLPROPERTIES ('delta.feature.catalogManaged' = 'supported'); ``` ## Check if catalog-managed commits are enabled To verify whether a table has catalog-managed commits enabled: ```sql DESCRIBE DETAIL sales_data; ``` If enabled, `catalogManaged` appears in the `tableFeatures` column. ## Limitations - Catalog-managed tables cannot be enabled on existing tables. Once enabled, the feature cannot be disabled. - `CREATE OR REPLACE TABLE` is not supported for tables with catalog-managed commits enabled. ================================================ FILE: docs/src/content/docs/delta-change-data-feed.mdx ================================================ --- title: Change data feed description: Learn how to get row-level change information from Delta tables using the Delta change data feed. --- import { Aside, Tabs, TabItem } from "@astrojs/starlight/components"; Change Data Feed (CDF) feature allows Delta tables to track row-level changes between versions of a Delta table. When enabled on a Delta table, the runtime records "change events" for all the data written into the table. This includes the row data along with metadata indicating whether the specified row was inserted, deleted, or updated. You can read the change events in batch queries using DataFrame APIs (that is, `df.read`) and in streaming queries using DataFrame APIs (that is, `df.readStream`). ## Use cases Change Data Feed is not enabled by default. The following use cases should drive when you enable the change data feed. - **Silver and Gold tables**: Improve Delta performance by processing only row-level changes following initial `MERGE`, `UPDATE`, or `DELETE` operations to accelerate and simplify ETL and ELT operations. - **Transmit changes**: Send a change data feed to downstream systems such as Kafka or RDBMS that can use it to incrementally process in later stages of data pipelines. - **Audit trail table**: Capture the change data feed as a Delta table provides perpetual storage and efficient query capability to see all changes over time, including when deletes occur and what updates were made. ## Enable change data feed You must explicitly enable the change data feed option using one of the following methods: - **New table**: Set the table property `delta.enableChangeDataFeed = true` in the `CREATE TABLE` command. ```sql CREATE TABLE student (id INT, name STRING, age INT) TBLPROPERTIES (delta.enableChangeDataFeed = true) ``` - **Existing table**: Set the table property `delta.enableChangeDataFeed = true` in the `ALTER TABLE` command. ```sql ALTER TABLE myDeltaTable SET TBLPROPERTIES (delta.enableChangeDataFeed = true) ``` - **All new tables**: ```sql set spark.databricks.delta.properties.defaults.enableChangeDataFeed = true; ``` ### Change data storage Delta Lake records change data for `UPDATE`, `DELETE`, and `MERGE` operations in the `_change_data` folder under the Delta table directory. These records may be skipped when Delta Lake detects it can efficiently compute the change data feed directly from the transaction log. In particular, insert-only operations and full partition deletes will not generate data in the `_change_data` directory. The files in the `_change_data` folder follow the retention policy of the table. Therefore, if you run the [VACUUM](/delta-utility/#remove-files-no-longer-referenced-by-a-delta-table) command, change data feed data is also deleted. ## Read changes in batch queries You can provide either version or timestamp for the start and end. The start and end versions and timestamps are inclusive in the queries. To read the changes from a particular start version to the _latest_ version of the table, specify only the starting version or timestamp. You specify a version as an integer and a timestamps as a string in the format `yyyy-MM-dd[ HH:mm:ss[.SSS]]`. If you provide a version lower or timestamp older than one that has recorded change events, that is, when the change data feed was enabled, an error is thrown indicating that the change data feed was not enabled. ```sql -- version as ints or longs e.g. changes from version 0 to 10 SELECT * FROM table_changes('tableName', 0, 10) -- timestamp as string formatted timestamps SELECT * FROM table_changes('tableName', '2021-04-21 05:45:46', '2021-05-21 12:00:00') -- providing only the startingVersion/timestamp SELECT * FROM table_changes('tableName', 0) -- database/schema names inside the string for table name, with backticks for escaping dots and special characters SELECT * FROM table_changes('dbName.`dotted.tableName`', '2021-04-21 06:45:46' , '2021-05-21 12:00:00') -- path based tables SELECT * FROM table_changes_by_path('\path', '2021-04-21 05:45:46') ``` ```python # version as ints or longs spark.read.format("delta") \ .option("readChangeFeed", "true") \ .option("startingVersion", 0) \ .option("endingVersion", 10) \ .table("myDeltaTable") # timestamps as formatted timestamp spark.read.format("delta") \ .option("readChangeFeed", "true") \ .option("startingTimestamp", '2021-04-21 05:45:46') \ .option("endingTimestamp", '2021-05-21 12:00:00') \ .table("myDeltaTable") # providing only the startingVersion/timestamp spark.read.format("delta") \ .option("readChangeFeed", "true") \ .option("startingVersion", 0) \ .table("myDeltaTable") # path based tables spark.read.format("delta") \ .option("readChangeFeed", "true") \ .option("startingTimestamp", '2021-04-21 05:45:46') \ .load("pathToMyDeltaTable") ``` ```scala // version as ints or longs spark.read.format("delta") .option("readChangeFeed", "true") .option("startingVersion", 0) .option("endingVersion", 10) .table("myDeltaTable") // timestamps as formatted timestamp spark.read.format("delta") .option("readChangeFeed", "true") .option("startingTimestamp", "2021-04-21 05:45:46") .option("endingTimestamp", "2021-05-21 12:00:00") .table("myDeltaTable") // providing only the startingVersion/timestamp spark.read.format("delta") .option("readChangeFeed", "true") .option("startingVersion", 0) .table("myDeltaTable") // path based tables spark.read.format("delta") .option("readChangeFeed", "true") .option("startingTimestamp", "2021-04-21 05:45:46") .load("pathToMyDeltaTable") ``` ## Read changes in streaming queries ```python # providing a starting version spark.readStream.format("delta") \ .option("readChangeFeed", "true") \ .option("startingVersion", 0) \ .table("myDeltaTable") # providing a starting timestamp spark.readStream.format("delta") \ .option("readChangeFeed", "true") \ .option("startingTimestamp", "2021-04-21 05:35:43") \ .load("/pathToMyDeltaTable") # not providing a starting version/timestamp will result in the latest snapshot being fetched first spark.readStream.format("delta") \ .option("readChangeFeed", "true") \ .table("myDeltaTable") ``` ```scala // providing a starting version spark.readStream.format("delta") .option("readChangeFeed", "true") .option("startingVersion", 0) .table("myDeltaTable") // providing a starting timestamp spark.readStream.format("delta") .option("readChangeFeed", "true") .option("startingVersion", "2021-04-21 05:35:43") .load("/pathToMyDeltaTable") // not providing a starting version/timestamp will result in the latest snapshot being fetched first spark.readStream.format("delta") .option("readChangeFeed", "true") .table("myDeltaTable") ``` To get the change data while reading the table, set the option `readChangeFeed` to `true`. The `startingVersion` or `startingTimestamp` are optional and if not provided the stream returns the latest snapshot of the table at the time of streaming as an `INSERT` and future changes as change data. Options like rate limits (`maxFilesPerTrigger`, `maxBytesPerTrigger`) and `excludeRegex` are also supported when reading change data. ## What is the schema for the change data feed? When you read from the change data feed for a table, the schema for the latest table version is used. In addition to the data columns from the schema of the Delta table, change data feed contains metadata columns that identify the type of change event: | Column name | Type | Values | | :-- | :-- | :-- | | `_change_type` | String | `insert`, `update_preimage` , `update_postimage`, `delete` [(1)](#-1) | | `_commit_version` | Long | The Delta log or table version containing the change. | | `_commit_timestamp` | Timestamp | The timestamp associated when the commit was created. | **(1)** `preimage` is the value before the update, `postimage` is the value after the update. ## Change data feed limitations for tables with column mapping enabled With column mapping enabled on a Delta table, you can drop or rename columns in the table without rewriting data files for existing data. With column mapping enabled, change data feed has limitations after performing non-additive schema changes such as renaming or dropping a column, changing data type, or nullability changes. ## Frequently asked questions (FAQ) ### What is the overhead of enabling the change data feed? There is no significant impact. The change data records are generated in line during the query execution process, and are generally much smaller than the total size of rewritten files. ### What is the retention policy for change records? Change records follow the same retention policy as out-of-date table versions, and will be cleaned up through VACUUM if they are outside the specified retention period. ### When do new records become available in the change data feed? Change data is committed along with the Delta Lake transaction, and will become available at the same time as the new data is available in the table. ================================================ FILE: docs/src/content/docs/delta-clustering.mdx ================================================ --- title: Use liquid clustering for Delta tables description: Learn about liquid clustering in Delta Lake. --- import { Tabs, TabItem, Aside } from "@astrojs/starlight/components"; Liquid clustering improves the existing partitioning and `ZORDER` techniques by simplifying data layout decisions in order to optimize query performance. Liquid clustering provides flexibility to redefine clustering columns without rewriting existing data, allowing data layout to evolve alongside analytic needs over time. ## What is liquid clustering used for? The following are examples of scenarios that benefit from clustering: - Tables often filtered by high cardinality columns. - Tables with significant skew in data distribution. - Tables that grow quickly and require maintenance and tuning effort. - Tables with access patterns that change over time. - Tables where a typical partition column could leave the table with too many or too few partitions. ## Enable liquid clustering You can enable liquid clustering on an existing table or during table creation. Clustering is not compatible with partitioning or `ZORDER`. Once enabled, run `OPTIMIZE` jobs as usual to incrementally cluster data. See [How to trigger clustering](#how-to-trigger-clustering). To enable liquid clustering, add the `CLUSTER BY` phrase to a table creation statement, as in the examples below: ```sql -- Create an empty table CREATE TABLE table1(col0 int, col1 string) USING DELTA CLUSTER BY (col0); -- Using a CTAS statement (Delta 3.3+) CREATE EXTERNAL TABLE table2 CLUSTER BY (col0) -- specify clustering after table name, not in subquery LOCATION 'table_location' AS SELECT * FROM table1; ``` ```python # Create an empty table DeltaTable.create() .tableName("table1") .addColumn("col0", dataType = "INT") .addColumn("col1", dataType = "STRING") .clusterBy("col0") .execute() ``` ```scala // Create an empty table DeltaTable.create() .tableName("table1") .addColumn("col0", dataType = "INT") .addColumn("col1", dataType = "STRING") .clusterBy("col0") .execute() ``` In Delta Lake 3.3 and above you can enable liquid clustering on an existing unpartitioned Delta table using the following syntax: ```sql ALTER TABLE CLUSTER BY () ``` ## Choose clustering columns Clustering columns can be defined in any order. If two columns are correlated, you only need to add one of them as a clustering column. If you're converting an existing table, consider the following recommendations: | Current data optimization technique | Recommendation for clustering columns | | --- | --- | | Hive-style partitioning | Use partition columns as clustering columns. | | Z-order indexing | Use the `ZORDER BY` columns as clustering columns. | | Hive-style partitioning and Z-order | Use both partition columns and `ZORDER BY` columns as clustering columns. | | Generated columns to reduce cardinality (for example, date for a timestamp) | Use the original column as a clustering column, and don't create a generated column. | ## Write data to a clustered table You must use a Delta writer client that supports `Clustering` and `DomainMetadata` table features. ## How to trigger clustering Use the `OPTIMIZE` command on your table, as in the following example: ```sql OPTIMIZE table_name; ``` Liquid clustering is incremental, meaning that data is only rewritten as necessary to accommodate data that needs to be clustered. Already clustered data files with different clustering columns are not rewritten. ### Recluster entire table In Delta Lake 3.3 and above, you can force reclustering of all records in a table with the following syntax: ```sql OPTIMIZE table_name FULL; ``` Run `OPTIMIZE FULL` when you change clustering columns. If you have previously run `OPTIMIZE FULL` and there has been no change to clustering columns, `OPTIMIZE FULL` runs the same as `OPTIMIZE`. Always use `OPTIMIZE FULL` to ensure that data layout reflects the current clustering columns. ## Read data from a clustered table You can read data in a clustered table using any Delta Lake client. For best query results, include clustering columns in your query filters, as in the following example: ```sql SELECT * FROM table_name WHERE clustering_column_name = "some_value"; ``` ## Change clustering columns You can change clustering columns for a table at any time by running an `ALTER TABLE` command, as in the following example: ```sql ALTER TABLE table_name CLUSTER BY (new_column1, new_column2); ``` When you change clustering columns, subsequent `OPTIMIZE` and write operations use the new clustering approach, but existing data is not rewritten. You can also turn off clustering by setting the columns to `NONE`, as in the following example: ```sql ALTER TABLE table_name CLUSTER BY NONE; ``` Setting cluster columns to `NONE` does not rewrite data that has already been clustered, but prevents future `OPTIMIZE` operations from using clustering columns. ## See how table is clustered You can use `DESCRIBE DETAIL` commands to see the clustering columns for a table, as in the following examples: ```sql DESCRIBE DETAIL table_name; ``` ## Limitations The following limitations exist: - You can only specify columns with statistics collected for clustering columns. By default, the first 32 columns in a Delta table have statistics collected. - You can specify up to 4 clustering columns. ================================================ FILE: docs/src/content/docs/delta-column-mapping.mdx ================================================ --- title: Delta column mapping description: Learn about column mapping in Delta. --- import { Tabs, TabItem, Aside } from "@astrojs/starlight/components"; [Column mapping feature](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#column-mapping) allows Delta table columns and the underlying Parquet file columns to use different names. This enables Delta schema evolution operations such as `RENAME COLUMN` and `DROP COLUMNS` on a Delta table without the need to rewrite the underlying Parquet files. It also allows users to name Delta table columns by using [characters that are not allowed](#supported-characters-in-column-names) by Parquet, such as spaces, so that users can directly ingest CSV or JSON data into Delta without the need to rename columns due to previous character constraints. ## How to enable Delta Lake column mapping Column mapping requires the following Delta protocols: - Reader version 2 or above. - Writer version 5 or above. For a Delta table with the required protocol versions, you can enable column mapping by setting `delta.columnMapping.mode` to `name`. You can use the following command to upgrade the table version and enable column mapping: ```sql ALTER TABLE SET TBLPROPERTIES ( 'delta.minReaderVersion' = '2', 'delta.minWriterVersion' = '5', 'delta.columnMapping.mode' = 'name' ) ``` ## Rename a column When column mapping is enabled for a Delta table, you can rename a column: ```sql ALTER TABLE RENAME COLUMN old_col_name TO new_col_name ``` For more examples, see [Rename columns](/delta-batch/#rename-columns). ## Drop columns When column mapping is enabled for a Delta table, you can drop one or more columns: ```sql ALTER TABLE table_name DROP COLUMN col_name; ALTER TABLE table_name DROP COLUMNS (col_name_1, col_name_2, ...); ``` For more details, see [Drop columns](/delta-batch/#drop-columns). ## Supported characters in column names When column mapping is enabled for a Delta table, you can include spaces as well as any of these characters in the table's column names: `,;{}()\n\t=`. ## Known limitations - Enabling column mapping on tables might break downstream operations that rely on Delta change data feed. See [Change data feed limitations for tables with column mapping enabled](/delta-change-data-feed/#change-data-feed-limitations-for-tables-with-column-mapping-enabled). - In Delta Lake 2.1 and below, [Spark Structured Streaming](https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html) reads are explicitly blocked on a column mapping enabled table. - In Delta Lake 2.2 and above, [Spark Structured Streaming](https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html) reads are explicitly blocked on a column mapping enabled table that underwent column renaming or column dropping. - In Delta Lake 3.0 and above, [Spark Structured Streaming](https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html) reads require schema tracking to be enabled on a column mapping enabled table that underwent column renaming or column dropping. See [Tracking non-additive schema changes](/delta-streaming/#schema-tracking) - The Delta table protocol specifies two modes of column mapping, by `name` and by `id`. Delta Lake 2.1 and below do not support `id` mode. ``` ================================================ FILE: docs/src/content/docs/delta-constraints.mdx ================================================ --- title: Constraints description: Learn how Delta tables apply constraints. --- import { Aside, Tabs, TabItem } from "@astrojs/starlight/components"; Delta tables support standard SQL constraint management clauses that ensure that the quality and integrity of data added to a table is automatically verified. When a constraint is violated, Delta Lake throws an `InvariantViolationException` to signal that the new data can't be added. Two types of constraints are supported: - `NOT NULL`: indicates that values in specific columns cannot be null. - `CHECK`: indicates that a specified Boolean expression must be true for each input row. ### `NOT NULL` constraint You specify `NOT NULL` constraints in the schema when you create a table and drop `NOT NULL` constraints using the `ALTER TABLE CHANGE COLUMN` command. ```sql CREATE TABLE default.people10m ( id INT NOT NULL, firstName STRING, middleName STRING NOT NULL, lastName STRING, gender STRING, birthDate TIMESTAMP, ssn STRING, salary INT ) USING DELTA; ALTER TABLE default.people10m CHANGE COLUMN middleName DROP NOT NULL; ``` If you specify a `NOT NULL` constraint on a column nested within a struct, the parent struct is also constrained to not be null. However, columns nested within array or map types do not accept `NOT NULL` constraints. ### `CHECK` constraint You manage `CHECK` constraints using the `ALTER TABLE ADD CONSTRAINT` and `ALTER TABLE DROP CONSTRAINT` commands. `ALTER TABLE ADD CONSTRAINT` verifies that all existing rows satisfy the constraint before adding it to the table. ```sql CREATE TABLE default.people10m ( id INT, firstName STRING, middleName STRING, lastName STRING, gender STRING, birthDate TIMESTAMP, ssn STRING, salary INT ) USING DELTA; ALTER TABLE default.people10m ADD CONSTRAINT dateWithinRange CHECK (birthDate > '1900-01-01'); ALTER TABLE default.people10m DROP CONSTRAINT dateWithinRange; ``` `CHECK` constraints are table properties in the output of the `DESCRIBE DETAIL` and `SHOW TBLPROPERTIES` commands. ```sql ALTER TABLE default.people10m ADD CONSTRAINT validIds CHECK (id > 1 and id < 99999999); DESCRIBE DETAIL default.people10m; SHOW TBLPROPERTIES default.people10m; ``` ================================================ FILE: docs/src/content/docs/delta-default-columns.mdx ================================================ --- title: Delta default column values description: Learn about default column values in Delta. --- import { Tabs, TabItem, Aside } from "@astrojs/starlight/components"; Delta enables the specification of [default expressions](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#default-columns) for columns in Delta tables. When users write to these tables without explicitly providing values for certain columns, or when they explicitly use the DEFAULT SQL keyword for a column, Delta automatically generates default values for those columns. This information is stored in the [StructField](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#struct-field) corresponding to the column of interest. ## How to enable Delta Lake default column values You can enable default column values for a table by setting `delta.feature.allowColumnDefaults` to `enabled`: ```sql ALTER TABLE SET TBLPROPERTIES ( 'delta.feature.allowColumnDefaults' = 'enabled' ) ``` ## How to use default columns in SQL commands - For SQL commands that perform table writes, such as `INSERT`, `UPDATE`, and `MERGE` commands, the `DEFAULT` keyword resolves to the most recently assigned default value for the corresponding column (or NULL if no default value exists). For instance, the following SQL command will use the default value for the second column in the table: `INSERT INTO t VALUES (16, DEFAULT);` - It is also possible for INSERT commands to specify lists of fewer columns than the target table, in which case the engine will assign default values for the remaining columns (or NULL for any columns where no defaults yet exist). - The `ALTER TABLE ... ADD COLUMN` command that introduces a new column to an existing table may not specify a default value for the new column. For instance, the following SQL command is not supported in Delta Lake: `ALTER TABLE t ADD COLUMN c INT DEFAULT 16;` - It is permissible, however, to assign or update default values for columns that were created in previous commands. For example, the following SQL command is valid: `ALTER TABLE t ALTER COLUMN c SET DEFAULT 16;` ================================================ FILE: docs/src/content/docs/delta-deletion-vectors.mdx ================================================ --- title: What are deletion vectors? description: Learn about deletion vectors in Delta Lake. --- import { Tabs, TabItem, Aside } from "@astrojs/starlight/components"; Deletion vectors are a storage optimization feature that can be enabled on Delta Lake tables. By default, when a single row in a data file is deleted, the entire Parquet file containing the record must be rewritten. With deletion vectors enabled for the table, some Delta operations use deletion vectors to mark existing rows as removed without rewriting the Parquet file. Subsequent reads on the table resolve current table state by applying the deletions noted by deletion vectors to the most recent table version. Support for deletion vectors was incrementally added with each Delta Lake version. The table below depicts the supported operations for each Delta Lake version. | Operation | First available Delta Lake version | Enabled by default since Delta Lake version | | --- | --- | --- | | `SCAN` | 2.3.0 | 2.3.0 | | `DELETE` | 2.4.0 | 2.4.0 | | `UPDATE` | 3.0.0 | 3.1.0 | | `MERGE` | 3.1.0 | 3.1.0 | ## Enable deletion vectors You enable support for deletion vectors on a Delta Lake table by setting a Delta Lake table property: ```sql ALTER TABLE SET TBLPROPERTIES('delta.enableDeletionVectors' = true); ``` ## Apply changes to Parquet data files Deletion vectors indicate changes to rows as soft-deletes that logically modify existing Parquet data files in the Delta Lake tables. These changes are applied physically when data files are rewritten, as triggered by one of the following events: - A DML command with deletion vectors disabled (by a command flag or a table property) is run on the table. - An `OPTIMIZE` command is run on the table. - `REORG TABLE ... APPLY (PURGE)` is run against the table. `UPDATE`, `MERGE`, and `OPTIMIZE` do not have strict guarantees for resolving changes recorded in deletion vectors, and some changes recorded in deletion vectors might not be applied if target data files contain no updated records, or would not otherwise be candidates for file compaction. `REORG TABLE ... APPLY (PURGE)` rewrites all data files containing records with modifications recorded using deletion vectors. See [Apply changes with REORG TABLE](#apply-changes-with-reorg-table) ### Apply changes with REORG TABLE Reorganize a Delta Lake table by rewriting files to purge soft-deleted data, such as rows marked as deleted by deletion vectors with `REORG TABLE`: ```sql REORG TABLE events APPLY (PURGE); -- If you have a large amount of data and only want to purge a subset of it, you can specify an optional partition predicate using `WHERE`: REORG TABLE events WHERE date >= '2022-01-01' APPLY (PURGE); REORG TABLE events WHERE date >= current_timestamp() - INTERVAL '1' DAY APPLY (PURGE); ``` ================================================ FILE: docs/src/content/docs/delta-drop-feature.mdx ================================================ --- title: Drop Delta table features description: Learn how to drop table features in Delta Lake to downgrade reader and writer protocol requirements and resolve compatibility issues. --- import { Tabs, TabItem, Aside } from "@astrojs/starlight/components"; This article describes how to drop Delta Lake table features and downgrade protocol versions. You should only use this functionality to support compatibility with earlier Delta Lake versions, Delta Sharing, or other Delta Lake reader or writer clients. ## How can I drop a Delta table feature? To remove a Delta table feature, you run an `ALTER TABLE DROP FEATURE ` command. ## What Delta table features can be dropped? You can drop the following Delta table features: - `deletionVectors`. See [What are deletion vectors?](/delta-deletion-vectors/). Drop support for deletion vectors is available in Delta Lake 4.0.0 and above. - `typeWidening-preview`. See [Delta type widening](/delta-type-widening/). Type widening is available in preview in Delta Lake 3.2.0 and above. - `typeWidening`. See [Delta type widening](/delta-type-widening/). Type widening is available in preview in Delta Lake 4.0.0 and above. - `v2Checkpoint`. See [V2 Checkpoint Spec](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#v2-spec). Drop support for V2 Checkpoints is available in Delta Lake 3.1.0 and above. - `columnMapping`. See [Delta column mapping](/delta-column-mapping/). Drop support for column mapping is available in Delta Lake 3.3.0 and above. - `vacuumProtocolCheck`. See [Vacuum Protocol Check Spec](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#vacuum-protocol-check). Drop support for vacuum protocol check is available in Delta Lake 3.3.0 and above. - `checkConstraints`. See [Constraints](/delta-constraints/). Drop support for check constraints is available in Delta Lake 3.3.0 and above. - `inCommitTimestamp`. See [In-Commit Timestamps](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#in-commit-timestamps). Drop support for In-Commit Timestamp is available in Delta Lake 3.3.0 and above. - `checkpointProtection`. See [Checkpoint Protection Spec](https://github.com/delta-io/delta/blob/master/protocol_rfcs/checkpoint-protection.md). Drop support for checkpoint protection is available in Delta Lake 4.0.0 and above. You cannot drop other [Delta table features](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#valid-feature-names-in-table-features). ## What happens when a table feature is dropped? When you drop a table feature, Delta Lake performs a series of atomic operations: - Disable table properties that use the table feature. - Rewrite data files as necessary to remove all traces of the table feature from the data files backing the table in the current version. - Create a set of protected checkpoints that allow reader clients to interpret table history correctly. - Add the writer table feature checkpointProtection to the table protocol. - Downgrade the table protocol to the lowest reader and writer versions that support all remaining table features. ## What is the checkpointProtection table feature? When you drop a feature, Delta Lake rewrites data and metadata in the table's history as protected checkpoints to respect the protocol downgrade. After the downgrade, the table should always be readable by more clients. This is because the protocol for the table now reflects that support for the dropped feature is no longer required to read the table. The protected checkpoints and the checkpointProtection feature accomplish the following: - Reader clients that understand the dropped table feature can access all available table history. - Reader clients that do not support the dropped table feature only need to read the table history starting from the protocol downgrade version. - Writer clients do not rewrite checkpoints prior to the protocol downgrade. - Table maintenance operations respect requirements set by `checkpointProtection`, which mark protocol downgrade checkpoints as protected. - While you can only drop one table feature with each DROP FEATURE command, a table can have multiple protected checkpoints and dropped features in its table history. The table feature `checkpointProtection` should not block read-only access from Delta Lake clients. To fully downgrade the table and remove the `checkpointProtection` table feature, you must use TRUNCATE HISTORY. The recommendation is to only use this pattern if you need to write to tables with external Delta clients that do not support checkpointProtection. ## Fully downgrade table protocols for legacy clients If integrations with external Delta Lake clients require writes that don't support the checkpointProtection table feature, you must use TRUNCATE HISTORY to fully remove all traces of the disabled table features and fully downgrade the table protocol. It is recommended to always test the default behavior for DROP FEATURE before proceeding with TRUNCATE HISTORY. Running TRUNCATE HISTORY removes all table history greater than 24 hours. Full table downgrade occurs in two steps that must occur at least 24 hours apart. ### Step 1: Prepare to drop a table feature During the first stage, the user prepares to drop the table feature. The following describes what happens during this stage: 1. You run the `ALTER TABLE DROP FEATURE TRUNCATE HISTORY` command. 2. Table properties that specifically enable a table feature have values set to disable the feature. 3. Table properties that control behaviors associated with the dropped feature have options set to default values before the feature was introduced. 4. As necessary, data and metadata files are rewritten respecting the updated table properties. 5. The command finishes running and returns an error message informing the user they must wait 24 hours to proceed with feature removal. After first disabling a feature, you can continue writing to the target table before completing the protocol downgrade, but you cannot use the table feature you are removing. ### Step 2: Downgrade the protocol and drop a table feature To fully remove all transaction history associated with the feature and downgrade the protocol: 1. After at least 24 hours have passed, you run the `ALTER TABLE DROP FEATURE TRUNCATE HISTORY` command. 2. The client confirms that no transactions in the specified retention threshold use the table feature, then truncates the table history to that threshold. 3. The protocol is downgraded, dropping the table feature. 4. If the table features that are present in the table can be represented by a legacy protocol version, the `minReaderVersion` and `minWriterVersion` for the table are downgraded to the lowest version that supports exactly all remaining features in use by the Delta table. See [How does Delta Lake manage feature compatibility?](/versioning/). ================================================ FILE: docs/src/content/docs/delta-faq.mdx ================================================ --- title: Frequently asked questions (FAQ) description: Find answers to commonly asked questions about Delta Lake. --- import { Tabs, TabItem, Aside } from "@astrojs/starlight/components"; ## What is Delta Lake? [Delta Lake](https://delta.io/) is an [open source storage layer](https://github.com/delta-io/delta) that brings reliability to [data lakes](https://databricks.com/discover/data-lakes/introduction). Delta Lake provides ACID transactions, scalable metadata handling, and unifies streaming and batch data processing. Delta Lake runs on top of your existing data lake and is fully compatible with Apache Spark APIs. ## How is Delta Lake related to Apache Spark? Delta Lake sits on top of Apache Spark. The format and the compute layer helps to simplify building big data pipelines and increase the overall efficiency of your pipelines. ## What format does Delta Lake use to store data? Delta Lake uses versioned Parquet files to store your data in your cloud storage. Apart from the versions, Delta Lake also stores a transaction log to keep track of all the commits made to the table or blob store directory to provide ACID transactions. ## How can I read and write data with Delta Lake? You can use your favorite Apache Spark APIs to read and write data with Delta Lake. See [Read a table](/delta-batch/#read-a-table) and [Write to a table](/delta-batch/#write-to-table). ## Where does Delta Lake store the data? When writing data, you can specify the location in your cloud storage. Delta Lake stores the data in that location in Parquet format. ## Can I copy my Delta Lake table to another location? Yes you can copy your Delta Lake table to another location. Remember to copy files without changing the timestamps to ensure that the time travel with timestamps will be consistent. ## Can I stream data directly into and from Delta tables? Yes, you can use Structured Streaming to directly write data into Delta tables and read from Delta tables. See [Stream data into Delta tables](/delta-streaming/#delta-table-as-a-sink) and [Stream data from Delta tables](/delta-streaming/#delta-table-as-a-source). ## Does Delta Lake support writes or reads using the Spark Streaming DStream API? Delta does not support the DStream API. We recommend [Table streaming reads and writes](/delta-streaming/). ## When I use Delta Lake, will I be able to port my code to other Spark platforms easily? Yes. When you use Delta Lake, you are using open Apache Spark APIs so you can easily port your code to other Spark platforms. To port your code, replace `delta` format with `parquet` format. ## Does Delta Lake support multi-table transactions? Delta Lake does not support multi-table transactions and foreign keys. Delta Lake supports transactions at the _table_ level. ## How can I change the type of a column? Changing a column's type or dropping a column requires rewriting the table. For an example, see [Change column type](/delta-batch/#change-column-type-or-name). ================================================ FILE: docs/src/content/docs/delta-kernel-java.mdx ================================================ --- title: Delta Kernel Java User Guide description: Learn how to build connectors to read and write Delta tables using Delta Kernel Java. --- import { Tabs, TabItem, Aside, Steps } from "@astrojs/starlight/components"; ## What is Delta Kernel? Delta Kernel is a library for operating on Delta tables. Specifically, it provides simple and narrow APIs for reading and writing to Delta tables without the need to understand the [Delta protocol](https://github.com/delta-io/delta/blob/master/PROTOCOL.md) details. You can use this library to do the following: * Read and write Delta tables from your applications. * Build a connector for a distributed engine like [Apache Spark™](https://github.com/apache/spark), [Apache Flink](https://github.com/apache/flink), or [Trino](https://github.com/trinodb/trino) for reading or writing massive Delta tables. ## Set up Delta Kernel for your project You need to `io.delta:delta-kernel-api` and `io.delta:delta-kernel-defaults` dependencies. Following is an example Maven `pom` file dependency list. The `delta-kernel-api` module contains the core of the Kernel that abstracts out the Delta protocol to enable reading and writing into Delta tables. It makes use of the `Engine` interface that is being passed to the Kernel API by the connector for heavy-lift operations such as reading/writing Parquet or JSON files, evaluating expressions or file system operations such as listing contents of the Delta Log directory, etc. Kernel supplies a default implementation of `Engine` in module `delta-kernel-defaults`. The connectors can implement their own version of `Engine` to make use of their native implementation of functionalities the `Engine` provides. For example: the connector can make use of their Parquet reader instead of using the reader from the `DefaultEngine`. More details on this [later](#step-2-build-your-own-engine). ```xml io.delta delta-kernel-api ${delta-kernel.version} io.delta delta-kernel-defaults ${delta-kernel.version} ``` If your connector is not using the [`DefaultEngine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) provided by the Kernel, the dependency `delta-kernel-defaults` from the above list can be skipped. ## Read a Delta table in a single process In this section, we will walk through how to build a very simple single-process Delta connector that can read a Delta table using the default [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) implementation provided by Delta Kernel. You can either write this code yourself in your project, or you can use the [examples](https://github.com/delta-io/delta/tree/master/kernel/examples) present in the Delta code repository. ### Step 1: Full scan on a Delta table The main entry point is [`io.delta.kernel.Table`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Table.html) which is a programmatic representation of a Delta table. Say you have a Delta table at the directory `myTablePath`. You can create a `Table` object as follows: ```java import io.delta.kernel.*; import io.delta.kernel.defaults.*; import org.apache.hadoop.conf.Configuration; String myTablePath = ; // fully qualified table path. Ex: file:/user/tables/myTable Configuration hadoopConf = new Configuration(); Engine myEngine = DefaultEngine.create(hadoopConf); Table myTable = Table.forPath(myEngine, myTablePath); ``` Note the default [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) we are creating to bootstrap the `myTable` object. This object allows you to plug in your own libraries for computationally intensive operations like Parquet file reading, JSON parsing, etc. You can ignore it for now. We will discuss more about this later when we discuss how to build more complex connectors for distributed processing engines. From this `myTable` object you can create a [`Snapshot`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Snapshot.html) object which represents the consistent state (a.k.a. a snapshot consistency) in a specific version of the table. ```java Snapshot mySnapshot = myTable.getLatestSnapshot(myEngine); ``` Now that we have a consistent snapshot view of the table, we can query more details about the table. For example, you can get the version and schema of this snapshot. ```java long version = mySnapshot.getVersion(); StructType tableSchema = mySnapshot.getSchema(); ``` Next, to read the table data, we have to *build* a [`Scan`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Scan.html) object. In order to build a `Scan` object, create a [`ScanBuilder`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/ScanBuilder.html) object which optionally allows selecting a subset of columns to read or setting a query filter. For now, ignore these optional settings. ```java Scan myScan = mySnapshot.getScanBuilder().build() // Common information about scanning for all data files to read. Row scanState = myScan.getScanState(myEngine) // Information about the list of scan files to read CloseableIterator scanFiles = myScan.getScanFiles(myEngine) ``` This [`Scan`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Scan.html) object has all the necessary metadata to start reading the table. There are two crucial pieces of information needed for reading data from a file in the table. * `myScan.getScanFiles(Engine)`: Returns scan files as columnar batches (represented as an iterator of `FilteredColumnarBatch`es, more on that later) where each selected row in the batch has information about a single file containing the table data. * `myScan.getScanState(Engine)`: Returns the snapshot-level information needed for reading any file. Note that this is a single row and common to all scan files. For each scan file the physical data must be read from the file. The columns to read are specified in the scan file state. Once the physical data is read, you have to call [`ScanFile.transformPhysicalData(...)`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Scan.html#transformPhysicalData-io.delta.kernel.engine.Engine-io.delta.kernel.data.Row-io.delta.kernel.data.Row-io.delta.kernel.utils.CloseableIterator-) with the scan state and the physical data read from scan file. This API takes care of transforming (e.g. adding partition columns) the physical data into logical data of the table. Here is an example of reading all the table data in a single thread. ```java CloserableIterator fileIter = scanObject.getScanFiles(myEngine); Row scanStateRow = scanObject.getScanState(myEngine); while(fileIter.hasNext()) { FilteredColumnarBatch scanFileColumnarBatch = fileIter.next(); // Get the physical read schema of columns to read from the Parquet data files StructType physicalReadSchema = ScanStateRow.getPhysicalDataReadSchema(engine, scanStateRow); try (CloseableIterator scanFileRows = scanFileColumnarBatch.getRows()) { while (scanFileRows.hasNext()) { Row scanFileRow = scanFileRows.next(); // From the scan file row, extract the file path, size and modification time metadata // needed to read the file. FileStatus fileStatus = InternalScanFileUtils.getAddFileStatus(scanFileRow); // Open the scan file which is a Parquet file using connector's own // Parquet reader or default Parquet reader provided by the Kernel (which // is used in this example). CloseableIterator physicalDataIter = engine.getParquetHandler().readParquetFiles( singletonCloseableIterator(fileStatus), physicalReadSchema, Optional.empty() /* optional predicate the connector can apply to filter data from the reader */ ); // Now the physical data read from the Parquet data file is converted to a table // logical data. Logical data may include the addition of partition columns and/or // subset of rows deleted try ( CloseableIterator transformedData = Scan.transformPhysicalData( engine, scanStateRow, scanFileRow, physicalDataIter)) { while (transformedData.hasNext()) { FilteredColumnarBatch logicalData = transformedData.next(); ColumnarBatch dataBatch = logicalData.getData(); // Not all rows in `dataBatch` are in the selected output. // An optional selection vector determines whether a row with a // specific row index is in the final output or not. Optional selectionVector = dataReadResult.getSelectionVector(); // access the data for the column at ordinal 0 ColumnVector column0 = dataBatch.getColumnVector(0); for (int rowIndex = 0; rowIndex < column0.getSize(); rowIndex++) { // check if the row is selected or not if (!selectionVector.isPresent() || // there is no selection vector, all records are selected (!selectionVector.get().isNullAt(rowId) && selectionVector.get().getBoolean(rowId))) { // Assuming the column type is String. // If it is a different type, call the relevant function on the `ColumnVector` System.out.println(column0.getString(rowIndex)); } } // access the data for column at ordinal 1 ColumnVector column1 = dataBatch.getColumnVector(1); for (int rowIndex = 0; rowIndex < column1.getSize(); rowIndex++) { // check if the row is selected or not if (!selectionVector.isPresent() || // there is no selection vector, all records are selected (!selectionVector.get().isNullAt(rowId) && selectionVector.get().getBoolean(rowId))) { // Assuming the column type is Long. // If it is a different type, call the relevant function on the `ColumnVector` System.out.println(column1.getLong(rowIndex)); } } // .. more .. } } } } } ``` A few working examples to read Delta tables within a single process are available [here](https://github.com/delta-io/delta/tree/master/kernel/examples). ### Step 2: Improve scan performance with file skipping We have explored how to do a full table scan. However, the real advantage of using the Delta format is that you can skip files using your query filters. To make this possible, Delta Kernel provides an [expression framework](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/expressions/package-summary.html) to encode your filters and provide them to Delta Kernel to skip files during the scan file generation. For example, say your table is partitioned by `columnX`, you want to query only the partition `columnX=1`. You can generate the expression and use it to build the scan as follows: ```java import io.delta.kernel.expressions.*; import io.delta.kernel.defaults.engine.*; Engine myEngine = DefaultEngine.create(new Configuration()); Predicate filter = new Predicate( "=", Arrays.asList(new Column("columnX"), Literal.ofInt(1))); Scan myFilteredScan = mySnapshot.getScanBuilder().withFilter(filter).build() // Subset of the given filter that is not guaranteed to be satisfied by // Delta Kernel when it returns data. This filter is used by Delta Kernel // to do data skipping as much as possible. The connector should use this filter // on top of the data returned by Delta Kernel in order for further filtering. Optional remainingFilter = myFilteredScan.getRemainingFilter(); ``` The scan files returned by `myFilteredScan.getScanFiles(myEngine)` will have rows representing files only of the required partition. Similarly, you can provide filters for non-partition columns, and if the data in the table is well clustered by those columns, then Delta Kernel will be able to skip files as much as possible. ## Create a Delta table In this section, we will walk through how to build a Delta connector that can create a Delta table using the default [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) implementation provided by Delta Kernel. You can either write this code yourself in your project, or you can use the [examples](https://github.com/delta-io/delta/tree/master/kernel/examples) present in the Delta code repository. The main entry point is [`io.delta.kernel.Table`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Table.html) which is a programmatic representation of a Delta table. Say you want to create Delta table at the directory `myTablePath`. You can create a `Table` object as follows: ```java package io.delta.kernel.examples; import io.delta.kernel.*; import io.delta.kernel.types.*; import io.delta.kernel.utils.CloseableIterable; String myTablePath = ; Configuration hadoopConf = new Configuration(); Engine myEngine = DefaultEngine.create(hadoopConf); Table myTable = Table.forPath(myEngine, myTablePath); ``` Note the default [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) we are creating to bootstrap the `myTable` object. This object allows you to plug in your own libraries for computationally intensive operations like Parquet file reading, JSON parsing, etc. You can ignore it for now. We will discuss more about this later when we discuss how to build more complex connectors for distributed processing engines. From this `myTable` object you can create a [`TransactionBuilder`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/TransactionBuilder.html) object which allows you to construct a [`Transaction`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Transaction.html) object ```java TransactionBuilder txnBuilder = myTable.createTransactionBuilder( myEngine, "Examples", /* engineInfo - connector can add its own identifier which is noted in the Delta Log */ Operation.CREATE_TABLE /* What is the operation we are trying to perform. This is noted in the Delta Log */ ); ``` Now that you have the `TransactionBuilder` object, you can set the table schema and partition columns of the table. ```java StructType mySchema = new StructType() .add("id", IntegerType.INTEGER) .add("name", StringType.STRING) .add("city", StringType.STRING) .add("salary", DoubleType.DOUBLE); // Partition columns are optional. Use it only if you are creating a partitioned table. List myPartitionColumns = Collections.singletonList("city"); // Set the schema of the new table on the transaction builder txnBuilder = txnBuilder .withSchema(engine, mySchema); // Set the partition columns of the new table only if you are creating // a partitioned table; otherwise, this step can be skipped. txnBuilder = txnBuilder .withPartitionColumns(engine, examplePartitionColumns); ``` `TransactionBuilder` allows setting additional properties of the table such as enabling a certain Delta feature or setting identifiers for idempotent writes. We will be visiting these in the next sections. The next step is to build `Transaction` out of the `TransactionBuilder` object. ```java // Build the transaction Transaction txn = txnBuilder.build(engine); ``` `Transaction` object allows the connector to optionally add any data and finally commit the transaction. A successful commit ensures that the table is created with the given schema. In this example, we are just creating a table and not adding any data as part of the table. ```java // Commit the transaction. // As we are just creating the table and not adding any data, the `dataActions` is empty. TransactionCommitResult commitResult = txn.commit( engine, CloseableIterable.emptyIterable() /* dataActions */ ); ``` The [`TransactionCommitResult`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/TransactionCommitResult.html) contains the what version the transaction is committed as and whether the table is ready for a checkpoint. As we are creating a table the version will be `0`. We will be discussing later on what a checkpoint is and what it means for the table to be ready for the checkpoint. A few working examples to create partitioned and un-partitioned Delta tables are available [here](https://github.com/delta-io/delta/tree/master/kernel/examples). ## Create a table and insert data into it In this section, we will walk through how to build a Delta connector that can create a Delta table and insert data into the table (similar to `CREATE TABLE AS ` construct in SQL) using the default [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) implementation provided by Delta Kernel. You can either write this code yourself in your project, or you can use the [examples](https://github.com/delta-io/delta/tree/master/kernel/examples) present in the Delta code repository. The first step is to construct a `Transaction`. Below is the code for that. For more details on what each step of the code means, please read the [create table](#create-a-delta-table) section. ``` package io.delta.kernel.examples; import io.delta.kernel.*; import io.delta.kernel.types.*; import io.delta.kernel.utils.CloseableIterable; String myTablePath = ; Configuration hadoopConf = new Configuration(); Engine myEngine = DefaultEngine.create(hadoopConf); Table myTable = Table.forPath(myEngine, myTablePath); StructType mySchema = new StructType() .add("id", IntegerType.INTEGER) .add("name", StringType.STRING) .add("city", StringType.STRING) .add("salary", DoubleType.DOUBLE); // Partition columns are optional. Use it only if you are creating a partitioned table. List myPartitionColumns = Collections.singletonList("city"); TransactionBuilder txnBuilder = myTable.createTransactionBuilder( myEngine, "Examples", /* engineInfo - connector can add its own identifier which is noted in the Delta Log */ Operation.WRITE /* What is the operation we are trying to perform? This is noted in the Delta Log */ ); // Set the schema of the new table on the transaction builder txnBuilder = txnBuilder .withSchema(engine, mySchema); // Set the partition columns of the new table only if you are creating // a partitioned table; otherwise, this step can be skipped. txnBuilder = txnBuilder .withPartitionColumns(engine, examplePartitionColumns); // Build the transaction Transaction txn = txnBuilder.build(engine); ``` Now that we have the [`Transaction`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Transaction.html) object, the next step is generating the data that confirms the table schema and partitioned according to the table partitions. ```java StructType dataSchema = txn.getSchema(engine) // Optional for un-partitioned tables List partitionColumnNames = txn.getPartitionColumns(engine) ``` Using the data schema and partition column names the connector can plan the query and generate data. At tasks that actually have the data to write to the table, the connector can ask the Kernel to transform the data given in the table schema into physical data that can actually be written to the Parquet data files. For partitioned tables, the data needs to be first partitioned by the partition columns, and then the connector should ask the Kernel to transform the data for each partition separately. The partitioning step is needed because any given data file in the Delta table contains data belonging to exactly one partition. Get the state of the transaction. The transaction state contains the information about how to convert the data in the table schema into physical data that needs to be written. The transformations depend on the protocol and features the table has. ```java Row txnState = txn.getTransactionState(engine); ``` Prepare the data. ```java // The data generated by the connector to write into a table CloseableIterator data = ... // Create partition value map Map partitionValues = Collections.singletonMap( "city", // partition column name // partition value. Depending upon the partition column type, the // partition value should be created. In this example, the partition // column is of type StringType, so we are creating a string literal. Literal.ofString(city) ); ``` The connector data is passed as an iterator of [`FilteredColumnarBatch`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/FilteredColumnarBatch.html). Each of the `FilteredColumnarBatch` contains a [`ColumnarBatch`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnarBatch.html) which actually contains the data in columnar access format and an optional section vector that allows the connector to specify which rows from the `ColumnarBatch` to write to the table. Partition values are passed as a map of the partition column name to the partition value. For an un-partitioned table, the map should be empty as it has no partition columns. ``` // Transform the logical data to physical data that needs to be written to the Parquet // files CloseableIterator physicalData = Transaction.transformLogicalData(engine, txnState, data, partitionValues); ``` The above code converts the given data for partitions into an iterator of `FilteredColumnarBatch` that needs to be written to the Parquet data files. In order to write the data files, the connector needs to get the [`WriteContext`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/DataWriteContext.html) from Kernel, which tells the connector where to write the data files and what columns to collect statistics from each data file. ```java // Get the write context DataWriteContext writeContext = Transaction.getWriteContext(engine, txnState, partitionValues); ``` Now, the connector has the physical data that needs to be written to Parquet data files, and where those files should be written, it can start writing the data files. ```java CloseableIterator dataFiles = engine.getParquetHandler() .writeParquetFiles( writeContext.getTargetDirectory(), physicalData, writeContext.getStatisticsColumns() ); ``` In the above code, the connector is making use of the `Engine` provided `ParquetHandler` to write the data, but the connector can choose its own Parquet file writer to write the data. Also note that the return of the above call is an iterator of [`DataFileStatus`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/utils/DataFileStatus.html) for each data file written. It basically contains the file path, file metadata, and optional file-level statistics for columns specified by the [`WriteContext.getStatisticsColumns()`]([https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/DataWriteContext.html](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/DataWriteContext.html#getStatisticsColumns--)) Convert each `DataFileStatus` into a Delta log action that can be written to the Delta table log. ```java CloseableIterator dataActions = Transaction.generateAppendActions(engine, txnState, dataFiles, writeContext); ``` The next step is constructing [`CloseableIterable`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/utils/CloseableIterable.html) out of the all the Delta log actions generated above. The reason for constructing an `Iterable` is that the transaction committing involves accessing the list of Delta log actions more than one time (in order to resolve conflicts when there are multiple writes to the table). Kernel provides a [utility method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/utils/CloseableIterable.html#inMemoryIterable-io.delta.kernel.utils.CloseableIterator-) to create an in-memory version of `CloseableIterable`. This interface also gives the connector an option to implement a custom implementation that spills the data actions to disk when the contents are too big to fit in memory. ```java // Create a iterable out of the data actions. If the contents are too big to fit in memory, // the connector may choose to write the data actions to a temporary file and return an // iterator that reads from the file. CloseableIterable dataActionsIterable = CloseableIterable.inMemoryIterable(dataActions); ``` The final step is committing the transaction! ```java TransactionCommitStatus commitStatus = txn.commit(engine, dataActionsIterable) ``` The [`TransactionCommitResult`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/TransactionCommitResult.html) contains the what version the transaction is committed as and whether the table is ready for a checkpoint. As we are creating a table the version will be `0`. We will be discussing later on what a checkpoint is and what it means for the table to be ready for the checkpoint. A few working examples to create and insert data into partitioned and un-partitioned Delta tables are available [here](https://github.com/delta-io/delta/tree/master/kernel/examples). ## Blind append into an existing Delta table In this section, we will walk through how to build a Delta connector that inserts data into an existing Delta table (similar to `INSERT INTO
` construct in SQL) using the default [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) implementation provided by Delta Kernel. You can either write this code yourself in your project, or you can use the [examples](https://github.com/delta-io/delta/tree/master/kernel/examples) present in the Delta code repository. The steps are exactly similar to [Create table and insert data into it](#create-a-table-and-insert-data-into-it) except that we won't be providing any schema or partition columns when building the `TransactionBuilder` ```java // Create a `Table` object with the given destination table path Table table = Table.forPath(engine, tablePath); // Create a transaction builder to build the transaction TransactionBuilder txnBuilder = table.createTransactionBuilder( engine, "Examples", /* engineInfo */ Operation.WRITE ); / Build the transaction - no need to provide the schema as the table already exists. Transaction txn = txnBuilder.build(engine); // Get the transaction state Row txnState = txn.getTransactionState(engine); List dataActions = new ArrayList<>(); // Generate the sample data for three partitions. Process each partition separately. // This is just an example. In a real-world scenario, the data may come from different // partitions. Connectors already have the capability to partition by partition values // before writing to the table // In the test data `city` is a partition column for (String city : Arrays.asList("San Francisco", "Campbell", "San Jose")) { FilteredColumnarBatch batch1 = generatedPartitionedDataBatch( 5 /* offset */, city /* partition value */); FilteredColumnarBatch batch2 = generatedPartitionedDataBatch( 5 /* offset */, city /* partition value */); FilteredColumnarBatch batch3 = generatedPartitionedDataBatch( 10 /* offset */, city /* partition value */); CloseableIterator data = toCloseableIterator(Arrays.asList(batch1, batch2, batch3).iterator()); // Create partition value map Map partitionValues = Collections.singletonMap( "city", // partition column name // partition value. Depending upon the parition column type, the // partition value should be created. In this example, the partition // column is of type StringType, so we are creating a string literal. Literal.ofString(city)); // First transform the logical data to physical data that needs to be written // to the Parquet // files CloseableIterator physicalData = Transaction.transformLogicalData(engine, txnState, data, partitionValues); // Get the write context DataWriteContext writeContext = Transaction.getWriteContext(engine, txnState, partitionValues); // Now write the physical data to Parquet files CloseableIterator dataFiles = engine.getParquetHandler() .writeParquetFiles( writeContext.getTargetDirectory(), physicalData, writeContext.getStatisticsColumns()); // Now convert the data file status to data actions that needs to be written to the Delta // table log CloseableIterator partitionDataActions = Transaction.generateAppendActions( engine, txnState, dataFiles, writeContext); // Now add all the partition data actions to the main data actions list. In a // distributed query engine, the partition data is written to files at tasks on executor // nodes. The data actions are collected at the driver node and then written to the // Delta table log using the `Transaction.commit` while (partitionDataActions.hasNext()) { dataActions.add(partitionDataActions.next()); } } // Create a iterable out of the data actions. If the contents are too big to fit in memory, // the connector may choose to write the data actions to a temporary file and return an // iterator that reads from the file. CloseableIterable dataActionsIterable = CloseableIterable.inMemoryIterable( toCloseableIterator(dataActions.iterator())); // Commit the transaction. TransactionCommitResult commitResult = txn.commit(engine, dataActionsIterable); ``` ## Idempotent Blind Appends to a Delta Table Idempotent writes allow the connector to make sure the data belonging to a particular transaction version and application id is inserted into the table at most once. In incremental processing systems (e.g. streaming systems), track progress using their own application-specific versions need to record what progress has been made in order to avoid duplicating data in the face of failures and retries during writes. By setting the transaction identifier, the Delta table can ensure that the data with the same identifier is not written multiple times. For more information refer to the Delta protocol section [Transaction Identifiers](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#transaction-identifiers) To make the data append idempotent, set the transaction identifier on the [`TransactionBuilder`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/TransactionBuilder.html#withTransactionId-io.delta.kernel.engine.Engine-java.lang.String-long-) ```java // Set the transaction identifiers for idempotent writes // Delta/Kernel makes sure that there exists only one transaction in the Delta log // with the given application id and txn version txnBuilder = txnBuilder.withTransactionId( engine, "my app id", /* application id */ 100 /* monotonically increasing txn version with each new data insert */ ); ``` That's all the connector need to do for idempotent blind appends. ## Checkpointing a Delta table [Checkpoints](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#checkpoints) are an optimization in Delta Log in order to construct the state of the Delta table faster. It basically contains the state of the table at the version the checkpoint is created. Delta Kernel allows the connector to optionally make the checkpoints. It is created for every few commits (configurable table property) on the table. The result of `Transaction.commit` returns a `TransactionCommitResult` that contains the version the transaction is committed as and whether the table is [read for checkpoint](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/TransactionCommitResult.html#isReadyForCheckpoint--). Creating a checkpoint takes time as it needs to construct the entire state of the table. If the connector doesn't want to checkpoint by itself but uses other connectors that are faster in creating a checkpoint, it can skip the checkpointing step. If it wants to checkpoint, the `Table` object has an [API](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Table.html#checkpoint-io.delta.kernel.engine.Engine-long-) to checkpoint the table. ```java TransactionCommitResult commitResult = txn.commit(engine, dataActionsIterable); if (commitResult.isReadyForCheckpoint()) { // Checkpoint the table Table.forPath(engine, tablePath).checkpoint(engine, commitResult.getVersion()); } ``` ## Build a Delta connector for a distributed processing engine Unlike simple applications that just read the table in a single process, building a connector for complex processing engines like Apache Spark™ and Trino can require quite a bit of additional effort. For example, to build a connector for an SQL engine you have to do the following * Understand the APIs provided by the engine to build connectors and how Delta Kernel can be used to provide the information necessary for the connector + engine to operate on a Delta table. * Decide what libraries to use to do computationally expensive operations like reading Parquet files, parsing JSON, computing expressions, etc. Delta Kernel provides all the extension points to allow you to plug in any library without having to understand all the low-level details of the Delta protocol. * Deal with details specific to distributed engines. For example, * Serialization of Delta table metadata provided by Delta Kernel. * Efficiently transforming data read from Parquet into the engine in-memory processing format. In this section, we are going to outline the steps needed to build a connector. ### Step 0: Validate the prerequisites In the previous section showing how to read a simple table, we were briefly introduced to the [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html). This is the main extension point where you can plug in your implementations of computationally-expensive operations like reading Parquet files, parsing JSON, etc. For the simple case, we were using a default implementation of the helper that works in most cases. However, for building a high-performance connector for a complex processing engine, you will very likely need to provide your own implementation using the libraries that work with your engine. So before you start building your connector, it is important to understand these requirements and plan for building your own engine. Here are the libraries/capabilities you need to build a connector that can read the Delta table * Perform file listing and file reads from your storage/file system. * Read Parquet files in columnar data, preferably in an in-memory columnar format. * Parse JSON data * Read JSON files * Evaluate expressions on in-memory columnar batches For each of these capabilities, you can choose to build your own implementation or reuse the default implementation. ### Step 1: Set up Delta Kernel in your connector project In the Delta Kernel project, there are multiple dependencies you can choose to depend on. 1. Delta Kernel core APIs - This is a must-have dependency, which contains all the main APIs like Table, Snapshot, and Scan that you will use to access the metadata and data of the Delta table. This has very few dependencies reducing the chance of conflicts with any dependencies in your connector and engine. This also provides the [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) interface which allows you to plug in your implementations of computationally expensive operations, but it does not provide any implementation of this interface. 2. Delta Kernel default- This has a default implementation called [`DefaultEngine`](https://delta-io.github.io/delta/snapshot/kernel-defaults/java/io/delta/kernel/defaults/engine/DefaultEngine.html) and additional dependencies such as `Hadoop`. If you wish to reuse all or parts of this implementation, then you can optionally depend on this. #### Set up Java projects As discussed above, you can import one or both of the artifacts as follows: ```xml io.delta delta-kernel-api ${delta-kernel.version} io.delta delta-kernel-defaults ${delta-kernel.version} ``` ### Step 2: Build your own Engine In this section, we are going to explore the [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) interface and walk through how to implement your own implementation so that you can plug in your connector/engine-specific implementations of computationally-intensive operations, threading model, resource management, etc. #### Step 2.1: Implement the `Engine` interface The [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) interface combines a bunch of sub-interfaces each of which is designed for a specific purpose. Here is a brief overview of the subinterfaces. See the API docs (Java) for a more detailed view. ```java interface Engine { /** * Get the connector provided {@link ExpressionHandler}. * @return An implementation of {@link ExpressionHandler}. */ ExpressionHandler getExpressionHandler(); /** * Get the connector provided {@link JsonHandler}. * @return An implementation of {@link JsonHandler}. */ JsonHandler getJsonHandler(); /** * Get the connector provided {@link FileSystemClient}. * @return An implementation of {@link FileSystemClient}. */ FileSystemClient getFileSystemClient(); /** * Get the connector provided {@link ParquetHandler}. * @return An implementation of {@link ParquetHandler}. */ ParquetHandler getParquetHandler(); } ``` To build your own `Engine` implementation, you can choose to either use the default implementations of each sub-interface or completely build every one from scratch. ```java class MyEngine extends DefaultEngine { FileSystemClient getFileSystemClient() { // Build a new implementation from scratch return new MyFileSystemClient(); } // For all other sub-clients, use the default implementations provided by the `DefaultEngine`. } ``` Next, we will walk through how to implement each interface. #### Step 2.2: Implement `FileSystemClient` interface The [`FileSystemClient`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/FileSystemClient.html) interface contains basic file system operations like listing directories, resolving paths into a fully qualified path and reading bytes from files. Implementation of this interface must take care of the following when interacting with storage systems such as S3, Hadoop, or ADLS: * Credentials and permissions: The connector must populate its `FileSystemClient` with the necessary configurations and credentials for the client to retrieve the necessary data from the storage system. For example, an implementation based on Hadoop's FileSystem abstractions can be passed S3 credentials via the Hadoop configurations. * Decryption: If file system objects are encrypted, then the implementation must decrypt the data before returning the data. #### Step 2.3: Implement `ParquetHandler` As the name suggests, this interface contains everything related to reading and writing Parquet files. It has been designed such that a connector can plug in a wide variety of implementations, from a simple single-threaded reader to a very advanced multi-threaded reader with pre-fetching and advanced connector-specific expression pushdown. Let's explore the methods to implement, and the guarantees associated with them. ##### Method `readParquetFiles(CloseableIterator fileIter, StructType physicalSchema, java.util.Optional predicate)` This [method]((https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/ParquetHandler.html#readParquetFiles-io.delta.kernel.utils.CloseableIterator-io.delta.kernel.types.StructType-)) takes as input `FileStatus`s which contains metadata such as file path, size etc. of the Parquet file to read. The columns to be read from the Parquet file are defined by the physical schema. To implement this method, you may have to first implement your own [`ColumnarBatch`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnarBatch.html) and [`ColumnVector`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnVector.html) which is used to represent the in-memory data generated from the Parquet files. When identifying the columns to read, note that there are multiple types of columns in the physical schema (represented as a [`StructType`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/types/StructType.html)). * Data columns: Columns that are expected to be read from the Parquet file. Based on the `StructField` object defining the column, read the column in the Parquet file that matches the same name or field id. If the column has a field id (stored as `parquet.field.id` in the `StructField` metadata) then the field id should be used to match the column in the Parquet file. Otherwise, the column name should be used for matching. * Metadata columns: These are special columns that must be populated using metadata about the Parquet file ([`StructField#isMetadataColumn`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/types/StructField.html#isMetadataColumn--) tells whether a column in `StructType` is a metadata column). To understand how to populate such a column, first match the column name against the set of standard metadata column name constants. For example, * `StructFileld#isMetadataColumn()` returns true and the column name is `StructField.METADATA_ROW_INDEX_COLUMN_NAME`, then you have to a generate column vector populated with the actual index of each row in the Parquet file (that is, not indexed by the possible subset of rows returned after Parquet data skipping). ##### Requirements and guarantees Any implementation must adhere to the following guarantees. * The schema of the returned `ColumnarBatch`es must match the physical schema. * If a data column is not found and the `StructField.isNullable = true`, then return a `ColumnVector` of nulls. Throw an error if it is not nullable. * The output iterator must maintain ordering as the input iterator. That is, if `file1` is before `file2` in the input iterator, then columnar batches of `file1` must be before those of `file2` in the output iterator. ##### Method `writeParquetFiles(String directoryPath, CloseableIterator dataIter, java.util.List statsColumns)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/ParquetHandler.html#writeParquetFiles-java.lang.String-io.delta.kernel.utils.CloseableIterator-java.util.List-) takes given data writes it into one or more Parquet files into the given directory. The data is given as an iterator of [FilteredColumnarBatches](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/FilteredColumnarBatch.html) which contains a [ColumnarBatch](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnarBatch.html) and an optional selection vector containing one entry for each row in `ColumnarBatch` indicating whether a row is selected or not selected. The `ColumnarBatch` also contains the schema of the data. This schema should be converted to Parquet schema, including any field IDs present [`FieldMetadata`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/types/FieldMetadata.html) for each column `StructField`. There is also the parameter `statsColumns`, which is a hint to the Parquet writer on what set of columns to collect stats for each file. The statistics include `min`, `max` and `null_count` for each column in the `statsColumns` list. Statistics collection is optional, but when present it is used by Kernel to persist the stats as part of the Delta table commit. This will help read queries prune un-needed data files based on the query predicate. For each written data file, the caller is expecting a [`DataFileStatus`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/utils/DataFileStatus.html) object. It contains the data file path, size, modification time, and optional column statistics. #### Method `writeParquetFileAtomically(String filePath, CloseableIterator data)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/ParquetHandler.html#writeParquetFiles-java.lang.String-io.delta.kernel.utils.CloseableIterator-java.util.List-) writes the given `data` into Parquet file at location `filePath`. The write is an atomic write i.e., either a Parquet file is created with all given content or no Parquet file is created at all. This should not create a file with partial content in it. The default implementation makes use of [`LogStore`](https://github.com/delta-io/delta/blob/master/storage/src/main/java/io/delta/storage/LogStore.java) implementations from the [`delta-storage`](https://github.com/delta-io/delta/tree/master/storage) module to accomplish the atomicity. A connector that wants to implement their own version of `ParquetHandler` can take a look at the default implementation for details. ##### Performance suggestions * The representation of data as `ColumnVector`s and `ColumnarBatch`es can have a significant impact on the query performance and it's best to read the Parquet file data directly into vectors and batches of the engine-native format to avoid potentially costly in-memory data format conversion. Create a Kernel `ColumnVector` and `ColumnarBatch` wrappers around the engine-native format equivalent classes. #### Step 2.4: Implement `ExpressionHandler` interface The [`ExpressionHandler`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/ExpressionHandler.html) interface has all the methods needed for handling expressions that may be applied on columnar data. ##### Method `getEvaluator(StructType batchSchema, Expression expresion, DataType outputType)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/ExpressionHandler.html#getEvaluator-io.delta.kernel.types.StructType-io.delta.kernel.expressions.Expression-io.delta.kernel.types.DataType-) generates an object of type [`ExpressionEvaluator`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/expressions/ExpressionEvaluator.html) that can evaluate the `expression` on a batch of row data to produce a result of a single column vector. To generate this function, the `getEvaluator()` method takes as input the expression and the schema of the `ColumnarBatch`es of data on which the expressions will be applied. The same object can be used to evaluate multiple columnar batches of input with the same schema and expression the evaluator is created for. ##### Method `getPredicateEvaluator(StructType inputSchema, Predicate predicate)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/ExpressionHandler.html#createSelectionVector-boolean:A-int-int-) is for creating an expression evaluator for `Predicate` type expressions. The `Predicate` type expressions return a boolean value as output. The returned object is of type [`PredicateEvaluator`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/expressions/PredicateEvaluator.html). This is a special interface for evaluating Predicate on input batch returns a selection vector containing one value for each row in input batch indicating whether the row has passed the predicate or not. Optionally it takes an existing selection vector along with the input batch for evaluation. The result selection vector is combined with the given existing selection vector and a new selection vector is returned. This mechanism allows running an input batch through several predicate evaluations without rewriting the input batch to remove rows that do not pass the predicate after each predicate evaluation. The new selection should be the same or more selective as the existing selection vector. For example, if a row is marked as unselected in the existing selection vector, then it should remain unselected in the returned selection vector even when the given predicate returns true for the row. ##### Method `createSelectionVector(boolean[] values, int from, int to)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/ExpressionHandler.html#createSelectionVector-boolean:A-int-int-) allows creating `ColumnVector` for boolean type values given as input. This allows the connector to maintain all `ColumnVector`s created in the desired memory format. ##### Requirements and guarantees Any implementation must adhere to the following guarantees. * Implementation must handle all possible variations of expressions. If the implementation encounters an expression type that it does not know how to handle, then it must throw a specific language-dependent exception. * Java: [NotSupportedException](https://docs.oracle.com/javaee/7/api/latest/javax/resource/NotSupportedException.html) * The `ColumnarBatch`es on which the generated `ExpressionEvaluator` is going to be used are guaranteed to have the schema provided during generation. Hence, it is safe to bind the expression evaluation logic to column ordinals instead of column names, thus making the actual evaluation faster. #### Step 2.5: Implement `JsonHandler` [This](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/JsonHandler.html) engine interface allows the connector to use plug-in their own JSON handling code and expose it to the Delta Kernel. ##### Method `readJsonFiles(CloseableIterator fileIter, StructType physicalSchema, java.util.Optional predicate)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/JsonHandler.html#readJsonFiles-io.delta.kernel.utils.CloseableIterator-io.delta.kernel.types.StructType-) takes as input `FileStatus`s of the JSON files and returns the data in a series of columnar batches. The columns to be read from the JSON file are defined by the physical schema, and the return batches must match that schema. To implement this method, you may have to first implement your own [`ColumnarBatch`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnarBatch.html)and [`ColumnVector`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnVector.html) which is used to represent the in-memory data generated from the JSON files. When identifying the columns to read, note that there are multiple types of columns in the physical schema (represented as a [`StructType`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/types/StructType.html)). ##### Method `parseJson(ColumnVector jsonStringVector, StructType outputSchema, java.util.Optional selectionVector)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/JsonHandler.html#parseJson-io.delta.kernel.data.ColumnVector-io.delta.kernel.types.StructType-) allows parsing a `ColumnVector` of string values which are in JSON format into the output format specified by the `outputSchema`. If a given column in `outputSchema` is not found, then a null value is returned. It optionally takes a selection vector which indicates what entries in the input `ColumnVector` of strings to parse. If an entry is not selected then a `null` value is returned as parsed output for that particular entry in the output. ##### Method `deserializeStructType(String structTypeJson)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/JsonHandler.html#deserializeStructType-java.lang.String-) allows parsing JSON encoded (according to [Delta schema serialization rules](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#schema-serialization-format)) `StructType` schema into a `StructType`. Most implementations of `JsonHandler` do not need to implement this method and instead use the one in the [default `JsonHandler`](https://delta-io.github.io/delta/snapshot/kernel-defaults/java/io/delta/kernel/defaults/engine/DefaultJsonHandler.html) implementation. #### Method `writeJsonFileAtomically(String filePath, CloseableIterator data, boolean overwrite)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/JsonHandler.html#writeJsonFileAtomically-java.lang.String-io.delta.kernel.utils.CloseableIterator-boolean-) writes the given `data` into a JSON file at location `filePath`. The write is an atomic write i.e., either a JSON file is created with all given content or no Parquet file is created at all. This should not create a file with partial content in it. The default implementation makes use of [`LogStore`](https://github.com/delta-io/delta/blob/master/storage/src/main/java/io/delta/storage/LogStore.java) implementations from the [`delta-storage`](https://github.com/delta-io/delta/tree/master/storage) module to accomplish the atomicity. A connector that wants to implement their own version of `JsonHandler` can take a look at the default implementation for details. The implementation is expected to handle the serialization rules (converting the `Row` object to JSON string) as described in the [API Javadoc](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/JsonHandler.html#writeJsonFileAtomically-java.lang.String-io.delta.kernel.utils.CloseableIterator-boolean-). #### Step 2.6: Implement `ColumnarBatch` and `ColumnVector` [`ColumnarBatch`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnarBatch.html) and [`ColumnVector`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnVector.html) are two interfaces to represent the data read into memory from files. This representation can have a significant impact on query performance. Each engine likely has a native representation of in-memory data with which it applies data transformation operations. For example, in Apache Spark™, the row data is internally represented as `UnsafeRow` for efficient processing. So it's best to read the Parquet file data directly into vectors and batches of the native format to avoid potentially costly in-memory data format conversions. So the recommended approach is to build wrapper classes that extend the two interfaces but internally use engine-native classes to store the data. When the connector has to forward the columnar batches received from the kernel to the engine, it has to be smart enough to skip converting vectors and batches that are already in the engine-native format. ### Step 3: Build read support in your connector In this section, we are going to walk through the likely sequence of Kernel API calls your connector will have to make to read a table. The exact timing of making these calls in your connector in the context of connector-engine interactions depends entirely on the engine-connector APIs and is therefore beyond the scope of this guide. However, we will try to provide broad guidelines that are likely (but not guaranteed) to apply to your connector-engine setup. For this purpose, we are going to assume that the engine goes through the following phases when processing a read/scan query - logical plan analysis, physical plan generation, and physical plan execution. Based on these broad characterizations, a typical control and data flow for reading a Delta table is going to be as follows: .. list-table:: :header-rows: 1 :widths: 30 70 Step Typical query phase when this step occurs Resolve the table snapshot to query Logical plan analysis phase when the plan's schema and other details need to be resolved and validated Resolve files to scan based on query parameters Physical plan generation, when the final parameters of the scan are available. For example: Schema of data to read after pruning away unused columns Query filters to apply after filter rearrangement Distribute the file information to workers Physical plan execution, only if it is a distributed engine. Read the columnar data using the file information Physical plan execution, when the data is being processed by the engine Let's understand the details of each step. #### Step 3.1: Resolve the table snapshot to query The first step is to resolve the consistent snapshot and the schema associated with it. This is often required by the connector/ engine to resolve and validate the logical plan of the scan query (if the concept of logical plan exists in your engine). To achieve this, the connector has to do the following. * Resolve the table path from the query: If the path is directly available, then this is easy. Otherwise, if it is a query based on a catalog table (for example, a Delta table defined in Hive Metastore), then the connector has to resolve the table path from the catalog. * Initialize the `Engine` object: Create a new instance of the `Engine` that you have chosen in [Step 2](#build-your-own Engine). * Initialize the Kernel objects and get the schema: Assuming the query is on the latest available version/snapshot of the table, you can get the table schema as follows: ```java import io.delta.kernel.*; import io.delta.kernel.defaults.engine.*; Engine myEngine = new MyEngine(); Table myTable = Table.forPath(myTablePath); Snapshot mySnapshot = myTable.getLatestSnapshot(myEngine); StructType mySchema = mySnapshot.getSchema(myEngine); ``` If you want to query a specific version of the table (that is, not the schema), then you can get the required snapshot as `myTable.getSnapshot(version)`. #### Step 3.2: Resolve files to scan Next, we need to build a Scan object using more information from the query. Here we are going to assume that the connector/engine has been able to extract the following details from the query (say, after optimizing the logical plan): * Read schema: The columns in the table that the query needs to read. This may be the full set of columns or a subset of columns. * Query filters: The filters on partitions or data columns that can be used skip reading table data. To provide this information to Kernel, you have to do the following: * Convert the engine-specific schema and filter expressions to Kernel schema and expressions: For schema, you have to create a `StructType` object. For the filters, you have to create an `Expression` object using all the available subclasses of `Expression`. * Build the scan with the converted information: Build the scan as follows: ```java import io.delta.kernel.expressions.*; import io.delta.kernel.types.*; StructType readSchema = ... ; // convert engine schema Predicate filterExpr = ... ; // convert engine filter expression Scan myScan = mySnapshot.getScanBuilder().withFilter(filterExpr).withReadSchema(readSchema).build(); ``` * Resolve the information required to file reads: The generated Scan object has two sets of information. * Scan files: `myScan.getScanFiles()` returns an iterator of `ColumnarBatch`es. Each batch in the iterator contains rows and each row has information about a single file that has been selected based on the query filter. * Scan state: `myScan.getScanState()` returns a `Row` that contains all the information that is common across all the files that need to be read. ```java Row myScanStateRow = myScan.getScanState(); CloseableIterator myScanFilesAsBatches = myScan.getScanFiles(); while (myScanFilesAsBatches.hasNext()) { FilteredColumnarBatch scanFileBatch = myScanFilesAsBatches.next(); CloseableIterator myScanFilesAsRows = scanFileBatch.getRows(); } ``` As we will soon see, reading the columnar data from a selected file will need to use both, the scan state row, and a scan file row with the file information. ##### Requirements and guarantees Here are the details you need to ensure when defining this scan. * The provided `readSchema` must be the exact schema of the data that the engine will expect when executing the query. Any mismatch in the schema defined during this query planning and the query execution will result in runtime failures. Hence you must build the scan with the readSchema only after the engine has finalized the logical plan after any optimizations like column pruning. * When applicable (for example, with Java Kernel APIs), you have to make sure to call the close() method as you consume the `ColumnarBatch`es of scan files (that is, either serialize the rows or use them to read the table data). #### Step 3.3: Distribute the file information to the workers If you are building a connector for a distributed engine like Spark/Presto/Trino/Flink, then your connector has to send all the scan metadata from the query planning machine (henceforth called the driver) to task execution machines (henceforth called the workers). You will have to serialize and deserialize the scan state and scan file rows. It is the connector job to implement serialization and deserialization utilities for a [`Row`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/Row.html). If the connector wants to split reading one scan file into multiple tasks, it can add additional connector specific split context to the task. At the task, the connector can use its own Parquet reader to read the specific part of the file indicated by the split info. ##### Custom `Row` Serializer/Deserializer Here are steps on how to build your own serializer/deserializer such that it will work with any [`Row`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/Row.html) of any schema. * Serializing * First serialize the row schema, that is, `StructType` object. * Then, use the schema to identify types of each column/ordinal in the `Row` and use that to serialize all the values one by one. * Deserializing * Define your own class that extends the Row interface. It must be able to handle complex types like arrays, nested structs and maps. * First deserialize the schema. * Then, use the schema to deserialize the values and put them in an instance of your custom Row class. ```java import io.delta.kernel.utils.*; // In the driver where query planning is being done Byte[] scanStateRowBytes = RowUtils.serialize(scanStateRow); Byte[] scanFileRowBytes = RowUtils.serialize(scanFileRow); // Optionally the connector adds a split info to the task (scan file, scan state) to // split reading of a Parquet file into multiple tasks. The task gets split info // along with the scan file row and scan state row. Split split = ...; // connector specific class, not related to Kernel // Send these over to the worker // In the worker when data will be read, after rowBytes have been sent over Row scanStateRow = RowUtils.deserialize(scanStateRowBytes); Row scanFileRow = RowUtils.deserialize(scanFileRowBytes); Split split = ... deserialize split info ...; ``` #### Step 3.4: Read the columnar data Finally, we are ready to read the columnar data. You will have to do the following: * Read the physical data from Parquet file as indicated by the scan file row, scan state, and optionally the split info * Convert the physical data into logical data of the table using the Kernel's APIs. ```java Row scanStateRow = ... ; Row scanFileRow = ... ; Split split = ...; // Additional option predicate such as dynamic filters the connector wants to // pass to the reader when reading files. Predicate optPredicate = ...; // Get the physical read schema of columns to read from the Parquet data files StructType physicalReadSchema = ScanStateRow.getPhysicalDataReadSchema(engine, scanStateRow); // From the scan file row, extract the file path, size and modification metadata // needed to read the file. FileStatus fileStatus = InternalScanFileUtils.getAddFileStatus(scanFileRow); // Open the scan file which is a Parquet file using connector's own // Parquet reader which supports reading specific parts (split) of the file. // If the connector doesn't have its own Parquet reader, it can use the // default Parquet reader provider which at the moment doesn't support reading // a specific part of the file, but reads the entire file from the beginning. CloseableIterator physicalDataIter = connectParquetReader.readParquetFile( fileStatus physicalReadSchema, split, // what part of the Parquet file to read data from optPredicate /* additional predicate the connector can apply to filter data from the reader */ ); // Now the physical data read from the Parquet data file is converted to logical data // the table represents. // Logical data may include the addition of partition columns and/or // subset of rows deleted CloseableIterator transformedData = Scan.transformPhysicalData( engine, scanState, scanFileRow, physicalDataIter)); ``` * Resolve the data in the batches: Each [`FilteredColumnarBatch`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/FilteredColumnarBatch.html) has two components: * Columnar batch (returned by `FilteredColumnarBatch.getData()`): This is the data read from the files having the schema matching the readSchema provided when the Scan object was built in the earlier step. * Optional selection vector (returned by `FilteredColumnarBatch.getSelectionVector()`): Optionally, a boolean vector that will define which rows in the batch are valid and should be consumed by the engine. If the selection vector is present, then you will have to apply it to the batch to resolve the final consumable data. * Convert to engine-specific data format: Each connector/engine has its own native row / columnar batch formats and interfaces. To return the read data batches to the engine, you have to convert them to fit those engine-specific formats and/or interfaces. Here are a few tips that you can follow to make this efficient. * Matching the engine-specific format: Some engines may expect the data in an in-memory format that may be different from the data produced by `getData()`. So you will have to do the data conversion for each column vector in the batch as needed. * Matching the engine-specific interfaces: You may have to implement wrapper classes that extend the engine-specific interfaces and appropriately encapsulate the row data. For best performance, you can implement your own Parquet reader and other `Engine` implementations to make sure that every `ColumnVector` generated is already in the engine-native format thus eliminating any need to convert. Now you should be able to read the Delta table correctly. ### Step 4: Build append support in your connector In this section, we are going to walk through the likely sequence of Kernel API calls your connector will have to make to append data to a table. The exact timing of making these calls in your connector in the context of connector-engine interactions depends entirely on the engine-connector APIs and is, therefore, beyond the scope of this guide. However, we will try to provide broad guidelines that are likely (but not guaranteed) to apply to your connector-engine setup. For this purpose, we are going to assume that the engine goes through the following phases when processing a write query - logical plan analysis, physical plan generation, and physical plan execution. Based on these broad characterizations, a typical control and data flow for reading a Delta table is going to be as follows:
Step Typical query phase when this step occurs
Determine the schema of the data that needs to be written to the table. Schema is derived from the existing table or from the parent operation of the write operator in the query plan when the table doesn't exist yet. Logical plan analysis phase when the plan's schema (write operator schema matches the table schema, etc.) and other details need to be resolved and validated.
Determine the physical partitioning of the data based on the table schema and partition columns either from the existing table or from the query plan (for new tables) Physical plan generation, where the number of writer tasks, data schema and partitioning is determined
Distribute the writer tasks definitions (which include the transaction state) to workers. Physical plan execution, only if it is a distributed engine.
Tasks write the data to data files and send the data file info to the driver. Physical plan execution, when the data is actually written to the table location
Finalize the query. Here, all the info of the data files written by the tasks is aggregated and committed to the transaction created at the beginning of the physical execution. Finalize the query. This happens on the driver where the query has started.
Let's understand the details of each step. #### Step 4.1: Determine the schema of the data that needs to be written to the table The first step is to resolve the output data schema. This is often required by the connector/ engine to resolve and validate the logical plan of the query (if the concept of logical plan exists in your engine). To achieve this, the connector has to do the following. At a high level query plan is a tree of operators where the leaf-level operators generate or read data from storage/tables and feed it upwards towards the parent operator nodes. This data transfer happens until it reaches the root operator node where the query is finalized (either the results are sent to the client or data is written to another table). * Create the `Table` object * From the `Table` object try to get the schema. * If the table is not found * the query includes creating the table (e.g., `CREATE TABLE AS` SQL query); * the schema is derived from the operator above the `write` that feeds the data to the `write` operator. * the query doesn't include creating new table, an exception is thrown saying the table is not found * If the table already exists * get the schema from the table and check if it matches the schema of the `write` operator. If not throw an exception. * Create a `TransactionBuilder` - this basically begins the steps of transaction construction. ```java import io.delta.kernel.*; import io.delta.kernel.defaults.engine.*; Engine myEngine = new MyEngine(); Table myTable = Table.forPath(myTablePath); StructType writeOperatorSchema = // ... derived from the query operator tree ... StructType dataSchema; boolean isNewTable = false; try { Snapshot mySnapshot = myTable.getLatestSnapshot(myEngine); dataSchema = mySnapshot.getSchema(myEngine); // .. check dataSchema and writeOperatorSchema match ... } catch(TableNotFoundException e) { isNewTable = true; dataSchema = writeOperatorSchema; } TransactionBuilder txnBuilder = myTable.createTransactionBuilder( myEngine, "Examples", /* engineInfo - connector can add its own identifier which is noted in the Delta Log */ Operation /* What is the operation we are trying to perform? This is noted in the Delta Log */ ); if (isNewTable) { // For a new table set the table schema in the transaction builder txnBuilder = txnBuilder.withSchema(engine, dataSchema) } ``` #### Step 4.2: Determine the physical partitioning of the data based on the table schema and partition columns Partition columns are found either from the query (for new tables, the query defines the partition columns) or from the existing table. ```java TransactionBuilder txnBuilder = ... from the last step ... Transaction txn; List partitionColumns = ... if (newTable) { partitionColumns = ... derive from the query parameters (ex. PARTITION BY clause in SQL) ... txnBuilder = txnBuilder.withPartitionColumns(engine, partitionColumns); txn = txnBuilder.build(engine); } else { txn = txnBuilder.build(engine); partitionColumns = txn.getPartitionColumns(engine); } ``` At the end of this step, we have the `Transaction` and schema of the data to generate and its partitioning. #### Step 4.3: Distribute the writer tasks definitions (which include the transaction state) to workers If you are building a connector for a distributed engine like Spark/Presto/Trino/Flink, then your connector has to send all the writer metadata from the query planning machine (henceforth called the driver) to task execution machines (henceforth called the workers). You will have to serialize and deserialize the transaction state. It is the connector job to implement serialization and deserialization utilities for a [`Row`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/Row.html). More details on a custom `Row` SerDe are found [here](#custom-row-serializerdeserializer). ```java Row txnState = txn.getState(engine); String jsonTxnState = serializeToJson(txnState); ``` #### Step 4.4: Tasks write the data to data files and send the data file info to the driver. In this step (which is executed on the worker nodes inside each task): * Deserialize the transaction state * Writer operator within the task gets the data from its parent operator. * The data is converted into a `FilteredColumnarBatch`. Each [`FilteredColumnarBatch`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/FilteredColumnarBatch.html) has two components: * Columnar batch (returned by `FilteredColumnarBatch.getData()`): This is the data read from the files having the schema matching the readSchema provided when the Scan object was built in the earlier step. * Optional selection vector (returned by `FilteredColumnarBatch.getSelectionVector()`): Optionally, a boolean vector that will define which rows in the batch are valid and should be consumed by the engine. * The connector can create `FilteredColumnBatch` wrapper around data in its own in-memory format. * Check if the data is partitioned or not. If not partitioned, partition the data by partition values. * For each partition generate the map of the partition column to the partition value * Use Kernel to convert the partitioned data into physical data that should go into the data files * Write the physical data into one or more data files. * Convert data file statues into a Delta log actions * Serialize the Delta log action `Row` objects and send them to the driver node ``` Row txnState = ... deserialize from JSON string sent by the driver ... CloseableIterator data = ... generate data ... // If the table is un-partitioned then this is an empty map Map partitionValues = ... prepare the partition values ... // First transform the logical data to physical data that needs to be written // to the Parquet files CloseableIterator physicalData = Transaction.transformLogicalData(engine, txnState, data, partitionValues); // Get the write context DataWriteContext writeContext = Transaction.getWriteContext(engine, txnState, partitionValues); // Now write the physical data to Parquet files CloseableIterator dataFiles = engine.getParquetHandler() .writeParquetFiles( writeContext.getTargetDirectory(), physicalData, writeContext.getStatisticsColumns()); // Now convert the data file status to data actions that needs to be written to the Delta table log CloseableIterator partitionDataActions = Transaction.generateAppendActions( engine, txnState, dataFiles, writeContext); .... serialize `partitionDataActions` and send them to driver node ``` #### Step 4.5: Finalize the query. At the driver node, the delta log actions from all the tasks are received and committed to the transaction. The tasks send the Delta log actions as a serialized JSON and deserialize them back to `Row` objects. ``` // Create a iterable out of the data actions. If the contents are too big to fit in memory, // the connector may choose to write the data actions to a temporary file and return an // iterator that reads from the file. CloseableIterable dataActionsIterable = CloseableIterable.inMemoryIterable( toCloseableIterator(dataActions.iterator())); // Commit the transaction. TransactionCommitResult commitResult = txn.commit(engine, dataActionsIterable); // Optional step if (commitResult.isReadyForCheckpoint()) { // Checkpoint the table Table.forPath(engine, tablePath).checkpoint(engine, commitResult.getVersion()); } ``` Thats it. Now you should be able to append data to Delta tables using the Kernel APIs. ## Migration guide Kernel APIs are still evolving and new features are being added. Kernel authors try to make the API changes backward compatible as much as they can with each new release, but sometimes it is hard to maintain the backward compatibility for a project that is evolving rapidly. This section provides guidance on how to migrate your connector to the latest version of Delta Kernel. With each new release the [examples](https://github.com/delta-io/delta/tree/master/kernel/examples) are kept up-to-date with the latest API changes. You can refer to the examples to understand how to use the new APIs. ### Migration from Delta Lake version 3.1.0 to 3.2.0 Following are API changes in Delta Kernel 3.2.0 that may require changes in your connector. #### Rename `TableClient` to [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) The `TableClient` interface has been renamed to `Engine`. This is the most significant API change in this release. The `TableClient` interface name is not exactly representing the functionality it provides. At a high level it provides capabilities such as reading Parquet files, JSON files, evaluating expressions on data and file system functionality. These are basically the heavy lift operations that Kernel depends on as a separate interface to allow the connectors to substitute their own custom implementation of the same functionality (e.g. custom Parquet reader). Essentially, these functionalities are the core of the `engine` functionalities. By renaming to `Engine`, we are representing the interface functionality with a proper name that is easy to understand. The `DefaultTableClient` has been renamed to [`DefaultEngine`](https://delta-io.github.io/delta/snapshot/kernel-defaults/java/io/delta/kernel/defaults/engine/DefaultEngine.html). #### [`Table.forPath(Engine engine, String tablePath)`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Table.html#forPath-io.delta.kernel.engine.Engine-java.lang.String-) behavior change Earlier when a non-existent table path is passed, the API used to throw `TableNotFoundException`. Now it doesn't throw the exception. Instead, it returns a `Table` object. When trying to get a `Snapshot` from the table object it throws the `TableNotFoundException`. #### [`FileSystemClient.resolvePath`](https://delta-io.github.io/delta/snapshot/kernel-defaults/java/io/delta/kernel/defaults/engine/DefaultFileSystemClient.html#resolvePath-java.lang.String-) behavior change Earlier when a non-existent path is passed, the API used to throw `FileNotFoundException`. Now it doesn't throw the exception. It still resolves the given path into a fully qualified path. ================================================ FILE: docs/src/content/docs/delta-kernel-rust.mdx ================================================ --- title: Delta Kernel Rust description: Learn how to build connectors to read and write Delta tables using Delta Kernel Rust. --- Work In Progress ================================================ FILE: docs/src/content/docs/delta-kernel.mdx ================================================ --- title: Delta Kernel description: Learn how to build connectors to read and write Delta tables. --- import { Tabs, TabItem, Aside, Steps } from "@astrojs/starlight/components"; The Delta Kernel project is a set of libraries ([Java](#kernel-java) and [Rust](#kernel-rust)) for building Delta connectors that can read from and write into Delta tables without the need to understand the [Delta protocol details](https://github.com/delta-io/delta/blob/master/PROTOCOL.md). You can use this library to do the following: - Read data from small Delta tables in a single thread in a single process. - Read data from large Delta tables using multiple threads in a single process. - Build a complex connector for a distributed processing engine and read very large Delta tables. - Insert data into a Delta table either from a single process or a complex distributed engine. Here is an example of a simple table scan with a filter: ```java Engine myEngine = DefaultEngine.create() ; // define a engine (more details below) Table myTable = Table.forPath("/delta/table/path"); // define what table to scan Snapshot mySnapshot = myTable.getLatestSnapshot(myEngine); // define which version of table to scan Scan myScan = mySnapshot.getScanBuilder(myEngine) // specify the scan details .withFilters(myEngine, scanFilter) .build(); CloseableIterator physicalData = // read the Parquet data files .. read from Parquet data files ... Scan.transformPhysicalData(...) // returns the table data ``` A complete version of the above example program and more examples of reading from and writing into a Delta table are available [here](https://github.com/delta-io/delta/tree/master/kernel/examples). Notice that there are two sets of public APIs to build connectors. - **Table APIs** - Interfaces like [`Table`](https://delta-io.github.io/delta/snapshot/kernel-api/java/index.html?io/delta/kernel/Table.html) and [`Snapshot`](https://delta-io.github.io/delta/snapshot/kernel-api/java/index.html?io/delta/kernel/Snapshot.html) that allow you to read (and soon write to) Delta tables - **Engine APIs** - The [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java//index.html?io/delta/kernel/engine/Engine.html) interface allows you to plug in connector-specific optimizations to compute-intensive components in the Kernel. For example, Delta Kernel provides a _default_ Parquet file reader via the `DefaultEngine`, but you may choose to replace that default with a custom `Engine` implementation that has a faster Parquet reader for your connector/processing engine. ## Kernel Java ## What is Delta Kernel? Delta Kernel is a library for operating on Delta tables. Specifically, it provides simple and narrow APIs for reading and writing to Delta tables without the need to understand the [Delta protocol](https://github.com/delta-io/delta/blob/master/PROTOCOL.md) details. You can use this library to do the following: - Read and write Delta tables from your applications. - Build a connector for a distributed engine like [Apache Spark™](https://github.com/apache/spark), [Apache Flink](https://github.com/apache/flink), or [Trino](https://github.com/trinodb/trino) for reading or writing massive Delta tables. ## Set up Delta Kernel for your project You need to `io.delta:delta-kernel-api` and `io.delta:delta-kernel-defaults` dependencies. Following is an example Maven `pom` file dependency list. The `delta-kernel-api` module contains the core of the Kernel that abstracts out the Delta protocol to enable reading and writing into Delta tables. It makes use of the `Engine` interface that is being passed to the Kernel API by the connector for heavy-lift operations such as reading/writing Parquet or JSON files, evaluating expressions or file system operations such as listing contents of the Delta Log directory, etc. Kernel supplies a default implementation of `Engine` in module `delta-kernel-defaults`. The connectors can implement their own version of `Engine` to make use of their native implementation of functionalities the `Engine` provides. For example: the connector can make use of their Parquet reader instead of using the reader from the `DefaultEngine`. More details on this [later](#step-2-build-your-own-engine). ```xml io.delta delta-kernel-api ${delta-kernel.version} io.delta delta-kernel-defaults ${delta-kernel.version} ``` If your connector is not using the [`DefaultEngine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) provided by the Kernel, the dependency `delta-kernel-defaults` from the above list can be skipped. ## Read a Delta table in a single process In this section, we will walk through how to build a very simple single-process Delta connector that can read a Delta table using the default [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) implementation provided by Delta Kernel. You can either write this code yourself in your project, or you can use the [examples](https://github.com/delta-io/delta/tree/master/kernel/examples) present in the Delta code repository. ### Step 1: Full scan on a Delta table The main entry point is [`io.delta.kernel.Table`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Table.html) which is a programmatic representation of a Delta table. Say you have a Delta table at the directory `myTablePath`. You can create a `Table` object as follows: ```java import io.delta.kernel.*; import io.delta.kernel.defaults.*; import org.apache.hadoop.conf.Configuration; String myTablePath = ; // fully qualified table path. Ex: file:/user/tables/myTable Configuration hadoopConf = new Configuration(); Engine myEngine = DefaultEngine.create(hadoopConf); Table myTable = Table.forPath(myEngine, myTablePath); ``` Note the default [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) we are creating to bootstrap the `myTable` object. This object allows you to plug in your own libraries for computationally intensive operations like Parquet file reading, JSON parsing, etc. You can ignore it for now. We will discuss more about this later when we discuss how to build more complex connectors for distributed processing engines. From this `myTable` object you can create a [`Snapshot`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Snapshot.html) object which represents the consistent state (a.k.a. a snapshot consistency) in a specific version of the table. ```java Snapshot mySnapshot = myTable.getLatestSnapshot(myEngine); ``` Now that we have a consistent snapshot view of the table, we can query more details about the table. For example, you can get the version and schema of this snapshot. ```java long version = mySnapshot.getVersion(myEngine); StructType tableSchema = mySnapshot.getSchema(myEngine); ``` Next, to read the table data, we have to _build_ a [`Scan`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Scan.html) object. In order to build a `Scan` object, create a [`ScanBuilder`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/ScanBuilder.html) object which optionally allows selecting a subset of columns to read or setting a query filter. For now, ignore these optional settings. ```java Scan myScan = mySnapshot.getScanBuilder(myEngine).build() // Common information about scanning for all data files to read. Row scanState = myScan.getScanState(myEngine) // Information about the list of scan files to read CloseableIterator scanFiles = myScan.getScanFiles(myEngine) ``` This [`Scan`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Scan.html) object has all the necessary metadata to start reading the table. There are two crucial pieces of information needed for reading data from a file in the table. - `myScan.getScanFiles(Engine)`: Returns scan files as columnar batches (represented as an iterator of `FilteredColumnarBatch`es, more on that later) where each selected row in the batch has information about a single file containing the table data. - `myScan.getScanState(Engine)`: Returns the snapshot-level information needed for reading any file. Note that this is a single row and common to all scan files. For each scan file the physical data must be read from the file. The columns to read are specified in the scan file state. Once the physical data is read, you have to call [`ScanFile.transformPhysicalData(…)`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Scan.html#transformPhysicalData-io.delta.kernel.engine.Engine-io.delta.kernel.data.Row-io.delta.kernel.data.Row-io.delta.kernel.utils.CloseableIterator-) with the scan state and the physical data read from scan file. This API takes care of transforming (e.g. adding partition columns) the physical data into logical data of the table. Here is an example of reading all the table data in a single thread. ```java CloserableIterator fileIter = scanObject.getScanFiles(myEngine); Row scanStateRow = scanObject.getScanState(myEngine); while(fileIter.hasNext()) { FilteredColumnarBatch scanFileColumnarBatch = fileIter.next(); // Get the physical read schema of columns to read from the Parquet data files StructType physicalReadSchema = ScanStateRow.getPhysicalDataReadSchema(engine, scanStateRow); try (CloseableIterator scanFileRows = scanFileColumnarBatch.getRows()) { while (scanFileRows.hasNext()) { Row scanFileRow = scanFileRows.next(); // From the scan file row, extract the file path, size and modification time metadata // needed to read the file. FileStatus fileStatus = InternalScanFileUtils.getAddFileStatus(scanFileRow); // Open the scan file which is a Parquet file using connector's own // Parquet reader or default Parquet reader provided by the Kernel (which // is used in this example). CloseableIterator physicalDataIter = engine.getParquetHandler().readParquetFiles( singletonCloseableIterator(fileStatus), physicalReadSchema, Optional.empty() /* optional predicate the connector can apply to filter data from the reader */ ); // Now the physical data read from the Parquet data file is converted to a table // logical data. Logical data may include the addition of partition columns and/or // subset of rows deleted try ( CloseableIterator transformedData = Scan.transformPhysicalData( engine, scanStateRow, scanFileRow, physicalDataIter)) { while (transformedData.hasNext()) { FilteredColumnarBatch logicalData = transformedData.next(); ColumnarBatch dataBatch = logicalData.getData(); // Not all rows in `dataBatch` are in the selected output. // An optional selection vector determines whether a row with a // specific row index is in the final output or not. Optional selectionVector = dataReadResult.getSelectionVector(); // access the data for the column at ordinal 0 ColumnVector column0 = dataBatch.getColumnVector(0); for (int rowIndex = 0; rowIndex < column0.getSize(); rowIndex++) { // check if the row is selected or not if (!selectionVector.isPresent() || // there is no selection vector, all records are selected (!selectionVector.get().isNullAt(rowId) && selectionVector.get().getBoolean(rowId))) { // Assuming the column type is String. // If it is a different type, call the relevant function on the `ColumnVector` System.out.println(column0.getString(rowIndex)); } } // access the data for column at ordinal 1 ColumnVector column1 = dataBatch.getColumnVector(1); for (int rowIndex = 0; rowIndex < column1.getSize(); rowIndex++) { // check if the row is selected or not if (!selectionVector.isPresent() || // there is no selection vector, all records are selected (!selectionVector.get().isNullAt(rowId) && selectionVector.get().getBoolean(rowId))) { // Assuming the column type is Long. // If it is a different type, call the relevant function on the `ColumnVector` System.out.println(column1.getLong(rowIndex)); } } // .. more .. } } } } } ``` A few working examples to read Delta tables within a single process are available [here](https://github.com/delta-io/delta/tree/master/kernel/examples). ### Step 2: Improve scan performance with file skipping We have explored how to do a full table scan. However, the real advantage of using the Delta format is that you can skip files using your query filters. To make this possible, Delta Kernel provides an [expression framework](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/expressions/package-summary.html) to encode your filters and provide them to Delta Kernel to skip files during the scan file generation. For example, say your table is partitioned by `columnX`, you want to query only the partition `columnX=1`. You can generate the expression and use it to build the scan as follows: ```java import io.delta.kernel.expressions.*; import io.delta.kernel.defaults.engine.*; Engine myEngine = DefaultEngine.create(new Configuration()); Predicate filter = new Predicate( "=", Arrays.asList(new Column("columnX"), Literal.ofInt(1))); Scan myFilteredScan = mySnapshot.buildScan(engine) .withFilter(myEngine, filter) .build() // Subset of the given filter that is not guaranteed to be satisfied by // Delta Kernel when it returns data. This filter is used by Delta Kernel // to do data skipping as much as possible. The connector should use this filter // on top of the data returned by Delta Kernel in order for further filtering. Optional remainingFilter = myFilteredScan.getRemainingFilter(); ``` The scan files returned by `myFilteredScan.getScanFiles(myEngine)` will have rows representing files only of the required partition. Similarly, you can provide filters for non-partition columns, and if the data in the table is well clustered by those columns, then Delta Kernel will be able to skip files as much as possible. ## Create a Delta table In this section, we will walk through how to build a Delta connector that can create a Delta table using the default [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) implementation provided by Delta Kernel. You can either write this code yourself in your project, or you can use the [examples](https://github.com/delta-io/delta/tree/master/kernel/examples) present in the Delta code repository. The main entry point is [`io.delta.kernel.Table`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Table.html) which is a programmatic representation of a Delta table. Say you want to create Delta table at the directory `myTablePath`. You can create a `Table` object as follows: ```java package io.delta.kernel.examples; import io.delta.kernel.*; import io.delta.kernel.types.*; import io.delta.kernel.utils.CloseableIterable; String myTablePath = ; Configuration hadoopConf = new Configuration(); Engine myEngine = DefaultEngine.create(hadoopConf); Table myTable = Table.forPath(myEngine, myTablePath); ``` Note the default [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) we are creating to bootstrap the `myTable` object. This object allows you to plug in your own libraries for computationally intensive operations like Parquet file reading, JSON parsing, etc. You can ignore it for now. We will discuss more about this later when we discuss how to build more complex connectors for distributed processing engines. From this `myTable` object you can create a [`TransactionBuilder`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/TransactionBuilder.html) object which allows you to construct a [`Transaction`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Transaction.html) object ```java TransactionBuilder txnBuilder = myTable.createTransactionBuilder( myEngine, "Examples", /* engineInfo - connector can add its own identifier which is noted in the Delta Log */ Operation.CREATE_TABLE /* What is the operation we are trying to perform. This is noted in the Delta Log */ ); ``` Now that you have the `TransactionBuilder` object, you can set the table schema and partition columns of the table. ```java StructType mySchema = new StructType() .add("id", IntegerType.INTEGER) .add("name", StringType.STRING) .add("city", StringType.STRING) .add("salary", DoubleType.DOUBLE); // Partition columns are optional. Use it only if you are creating a partitioned table. List myPartitionColumns = Collections.singletonList("city"); // Set the schema of the new table on the transaction builder txnBuilder = txnBuilder .withSchema(engine, mySchema); // Set the partition columns of the new table only if you are creating // a partitioned table; otherwise, this step can be skipped. txnBuilder = txnBuilder .withPartitionColumns(engine, examplePartitionColumns); ``` `TransactionBuilder` allows setting additional properties of the table such as enabling a certain Delta feature or setting identifiers for idempotent writes. We will be visiting these in the next sections. The next step is to build `Transaction` out of the `TransactionBuilder` object. ```java // Build the transaction Transaction txn = txnBuilder.build(engine); ``` `Transaction` object allows the connector to optionally add any data and finally commit the transaction. A successful commit ensures that the table is created with the given schema. In this example, we are just creating a table and not adding any data as part of the table. ```java // Commit the transaction. // As we are just creating the table and not adding any data, the `dataActions` is empty. TransactionCommitResult commitResult = txn.commit( engine, CloseableIterable.emptyIterable() /* dataActions */ ); ``` The [`TransactionCommitResult`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/TransactionCommitResult.html) contains the what version the transaction is committed as and whether the table is ready for a checkpoint. As we are creating a table the version will be `0`. We will be discussing later on what a checkpoint is and what it means for the table to be ready for the checkpoint. A few working examples to create partitioned and un-partitioned Delta tables are available [here](https://github.com/delta-io/delta/tree/master/kernel/examples). ## Create a table and insert data into it In this section, we will walk through how to build a Delta connector that can create a Delta table and insert data into the table (similar to `CREATE TABLE AS ` construct in SQL) using the default [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) implementation provided by Delta Kernel. You can either write this code yourself in your project, or you can use the [examples](https://github.com/delta-io/delta/tree/master/kernel/examples) present in the Delta code repository. The first step is to construct a `Transaction`. Below is the code for that. For more details on what each step of the code means, please read the [create table](#create-a-delta-table) section. ``` package io.delta.kernel.examples; import io.delta.kernel.*; import io.delta.kernel.types.*; import io.delta.kernel.utils.CloseableIterable; String myTablePath = ; Configuration hadoopConf = new Configuration(); Engine myEngine = DefaultEngine.create(hadoopConf); Table myTable = Table.forPath(myEngine, myTablePath); StructType mySchema = new StructType() .add("id", IntegerType.INTEGER) .add("name", StringType.STRING) .add("city", StringType.STRING) .add("salary", DoubleType.DOUBLE); // Partition columns are optional. Use it only if you are creating a partitioned table. List myPartitionColumns = Collections.singletonList("city"); TransactionBuilder txnBuilder = myTable.createTransactionBuilder( myEngine, "Examples", /* engineInfo - connector can add its own identifier which is noted in the Delta Log */ Operation.WRITE /* What is the operation we are trying to perform? This is noted in the Delta Log */ ); // Set the schema of the new table on the transaction builder txnBuilder = txnBuilder .withSchema(engine, mySchema); // Set the partition columns of the new table only if you are creating // a partitioned table; otherwise, this step can be skipped. txnBuilder = txnBuilder .withPartitionColumns(engine, examplePartitionColumns); // Build the transaction Transaction txn = txnBuilder.build(engine); ``` Now that we have the [`Transaction`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Transaction.html) object, the next step is generating the data that confirms the table schema and partitioned according to the table partitions. ```java StructType dataSchema = txn.getSchema(engine) // Optional for un-partitioned tables List partitionColumnNames = txn.getPartitionColumns(engine) ``` Using the data schema and partition column names the connector can plan the query and generate data. At tasks that actually have the data to write to the table, the connector can ask the Kernel to transform the data given in the table schema into physical data that can actually be written to the Parquet data files. For partitioned tables, the data needs to be first partitioned by the partition columns, and then the connector should ask the Kernel to transform the data for each partition separately. The partitioning step is needed because any given data file in the Delta table contains data belonging to exactly one partition. Get the state of the transaction. The transaction state contains the information about how to convert the data in the table schema into physical data that needs to be written. The transformations depend on the protocol and features the table has. ```java Row txnState = txn.getTransactionState(engine); ``` Prepare the data. ```java // The data generated by the connector to write into a table CloseableIterator data = ... // Create partition value map Map partitionValues = Collections.singletonMap( "city", // partition column name // partition value. Depending upon the partition column type, the // partition value should be created. In this example, the partition // column is of type StringType, so we are creating a string literal. Literal.ofString(city) ); ``` The connector data is passed as an iterator of [`FilteredColumnarBatch`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/FilteredColumnarBatch.html). Each of the `FilteredColumnarBatch` contains a [`ColumnarBatch`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnarBatch.html) which actually contains the data in columnar access format and an optional section vector that allows the connector to specify which rows from the `ColumnarBatch` to write to the table. Partition values are passed as a map of the partition column name to the partition value. For an un-partitioned table, the map should be empty as it has no partition columns. ``` // Transform the logical data to physical data that needs to be written to the Parquet // files CloseableIterator physicalData = Transaction.transformLogicalData(engine, txnState, data, partitionValues); ``` The above code converts the given data for partitions into an iterator of `FilteredColumnarBatch` that needs to be written to the Parquet data files. In order to write the data files, the connector needs to get the [`WriteContext`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/DataWriteContext.html) from Kernel, which tells the connector where to write the data files and what columns to collect statistics from each data file. ```java // Get the write context DataWriteContext writeContext = Transaction.getWriteContext(engine, txnState, partitionValues); ``` Now, the connector has the physical data that needs to be written to Parquet data files, and where those files should be written, it can start writing the data files. ```java CloseableIterator dataFiles = engine.getParquetHandler() .writeParquetFiles( writeContext.getTargetDirectory(), physicalData, writeContext.getStatisticsColumns() ); ``` In the above code, the connector is making use of the `Engine` provided `ParquetHandler` to write the data, but the connector can choose its own Parquet file writer to write the data. Also note that the return of the above call is an iterator of [`DataFileStatus`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/utils/DataFileStatus.html) for each data file written. It basically contains the file path, file metadata, and optional file-level statistics for columns specified by the [`WriteContext.getStatisticsColumns()`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/DataWriteContext.html#getStatisticsColumns--)) Convert each `DataFileStatus` into a Delta log action that can be written to the Delta table log. ```java CloseableIterator dataActions = Transaction.generateAppendActions(engine, txnState, dataFiles, writeContext); ``` The next step is constructing [`CloseableIterable`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/utils/CloseableIterable.html) out of the all the Delta log actions generated above. The reason for constructing an `Iterable` is that the transaction committing involves accessing the list of Delta log actions more than one time (in order to resolve conflicts when there are multiple writes to the table). Kernel provides a [utility method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/utils/CloseableIterable.html#inMemoryIterable-io.delta.kernel.utils.CloseableIterator-) to create an in-memory version of `CloseableIterable`. This interface also gives the connector an option to implement a custom implementation that spills the data actions to disk when the contents are too big to fit in memory. ```java // Create a iterable out of the data actions. If the contents are too big to fit in memory, // the connector may choose to write the data actions to a temporary file and return an // iterator that reads from the file. CloseableIterable dataActionsIterable = CloseableIterable.inMemoryIterable(dataActions); ``` The final step is committing the transaction! ```java TransactionCommitStatus commitStatus = txn.commit(engine, dataActionsIterable) ``` The [`TransactionCommitResult`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/TransactionCommitResult.html) contains the what version the transaction is committed as and whether the table is ready for a checkpoint. As we are creating a table the version will be `0`. We will be discussing later on what a checkpoint is and what it means for the table to be ready for the checkpoint. A few working examples to create and insert data into partitioned and un-partitioned Delta tables are available [here](https://github.com/delta-io/delta/tree/master/kernel/examples). ## Blind append into an existing Delta table In this section, we will walk through how to build a Delta connector that inserts data into an existing Delta table (similar to `INSERT INTO
` construct in SQL) using the default [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) implementation provided by Delta Kernel. You can either write this code yourself in your project, or you can use the [examples](https://github.com/delta-io/delta/tree/master/kernel/examples) present in the Delta code repository. The steps are exactly similar to [Create table and insert data into it](#create-a-table-and-insert-data-into-it) except that we won't be providing any schema or partition columns when building the `TransactionBuilder` ```java // Create a `Table` object with the given destination table path Table table = Table.forPath(engine, tablePath); // Create a transaction builder to build the transaction TransactionBuilder txnBuilder = table.createTransactionBuilder( engine, "Examples", /* engineInfo */ Operation.WRITE ); / Build the transaction - no need to provide the schema as the table already exists. Transaction txn = txnBuilder.build(engine); // Get the transaction state Row txnState = txn.getTransactionState(engine); List dataActions = new ArrayList<>(); // Generate the sample data for three partitions. Process each partition separately. // This is just an example. In a real-world scenario, the data may come from different // partitions. Connectors already have the capability to partition by partition values // before writing to the table // In the test data `city` is a partition column for (String city : Arrays.asList("San Francisco", "Campbell", "San Jose")) { FilteredColumnarBatch batch1 = generatedPartitionedDataBatch( 5 /* offset */, city /* partition value */); FilteredColumnarBatch batch2 = generatedPartitionedDataBatch( 5 /* offset */, city /* partition value */); FilteredColumnarBatch batch3 = generatedPartitionedDataBatch( 10 /* offset */, city /* partition value */); CloseableIterator data = toCloseableIterator(Arrays.asList(batch1, batch2, batch3).iterator()); // Create partition value map Map partitionValues = Collections.singletonMap( "city", // partition column name // partition value. Depending upon the parition column type, the // partition value should be created. In this example, the partition // column is of type StringType, so we are creating a string literal. Literal.ofString(city)); // First transform the logical data to physical data that needs to be written // to the Parquet // files CloseableIterator physicalData = Transaction.transformLogicalData(engine, txnState, data, partitionValues); // Get the write context DataWriteContext writeContext = Transaction.getWriteContext(engine, txnState, partitionValues); // Now write the physical data to Parquet files CloseableIterator dataFiles = engine.getParquetHandler() .writeParquetFiles( writeContext.getTargetDirectory(), physicalData, writeContext.getStatisticsColumns()); // Now convert the data file status to data actions that needs to be written to the Delta // table log CloseableIterator partitionDataActions = Transaction.generateAppendActions( engine, txnState, dataFiles, writeContext); // Now add all the partition data actions to the main data actions list. In a // distributed query engine, the partition data is written to files at tasks on executor // nodes. The data actions are collected at the driver node and then written to the // Delta table log using the `Transaction.commit` while (partitionDataActions.hasNext()) { dataActions.add(partitionDataActions.next()); } } // Create a iterable out of the data actions. If the contents are too big to fit in memory, // the connector may choose to write the data actions to a temporary file and return an // iterator that reads from the file. CloseableIterable dataActionsIterable = CloseableIterable.inMemoryIterable( toCloseableIterator(dataActions.iterator())); // Commit the transaction. TransactionCommitResult commitResult = txn.commit(engine, dataActionsIterable); ``` ## Idempotent Blind Appends to a Delta Table Idempotent writes allow the connector to make sure the data belonging to a particular transaction version and application id is inserted into the table at most once. In incremental processing systems (e.g. streaming systems), track progress using their own application-specific versions need to record what progress has been made in order to avoid duplicating data in the face of failures and retries during writes. By setting the transaction identifier, the Delta table can ensure that the data with the same identifier is not written multiple times. For more information refer to the Delta protocol section [Transaction Identifiers](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#transaction-identifiers) To make the data append idempotent, set the transaction identifier on the [`TransactionBuilder`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/TransactionBuilder.html#withTransactionId-io.delta.kernel.engine.Engine-java.lang.String-long-) ```java // Set the transaction identifiers for idempotent writes // Delta/Kernel makes sure that there exists only one transaction in the Delta log // with the given application id and txn version txnBuilder = txnBuilder.withTransactionId( engine, "my app id", /* application id */ 100 /* monotonically increasing txn version with each new data insert */ ); ``` That's all the connector need to do for idempotent blind appends. ## Checkpointing a Delta table [Checkpoints](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#checkpoints) are an optimization in Delta Log in order to construct the state of the Delta table faster. It basically contains the state of the table at the version the checkpoint is created. Delta Kernel allows the connector to optionally make the checkpoints. It is created for every few commits (configurable table property) on the table. The result of `Transaction.commit` returns a `TransactionCommitResult` that contains the version the transaction is committed as and whether the table is [read for checkpoint](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/TransactionCommitResult.html#isReadyForCheckpoint--). Creating a checkpoint takes time as it needs to construct the entire state of the table. If the connector doesn't want to checkpoint by itself but uses other connectors that are faster in creating a checkpoint, it can skip the checkpointing step. If it wants to checkpoint, the `Table` object has an [API](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Table.html#checkpoint-io.delta.kernel.engine.Engine-long-) to checkpoint the table. ```java TransactionCommitResult commitResult = txn.commit(engine, dataActionsIterable); if (commitResult.isReadyForCheckpoint()) { // Checkpoint the table Table.forPath(engine, tablePath).checkpoint(engine, commitResult.getVersion()); } ``` ## Build a Delta connector for a distributed processing engine Unlike simple applications that just read the table in a single process, building a connector for complex processing engines like Apache Spark™ and Trino can require quite a bit of additional effort. For example, to build a connector for an SQL engine you have to do the following - Understand the APIs provided by the engine to build connectors and how Delta Kernel can be used to provide the information necessary for the connector + engine to operate on a Delta table. - Decide what libraries to use to do computationally expensive operations like reading Parquet files, parsing JSON, computing expressions, etc. Delta Kernel provides all the extension points to allow you to plug in any library without having to understand all the low-level details of the Delta protocol. - Deal with details specific to distributed engines. For example, - Serialization of Delta table metadata provided by Delta Kernel. - Efficiently transforming data read from Parquet into the engine in-memory processing format. In this section, we are going to outline the steps needed to build a connector. ### Step 0: Validate the prerequisites In the previous section showing how to read a simple table, we were briefly introduced to the [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html). This is the main extension point where you can plug in your implementations of computationally-expensive operations like reading Parquet files, parsing JSON, etc. For the simple case, we were using a default implementation of the helper that works in most cases. However, for building a high-performance connector for a complex processing engine, you will very likely need to provide your own implementation using the libraries that work with your engine. So before you start building your connector, it is important to understand these requirements and plan for building your own engine. Here are the libraries/capabilities you need to build a connector that can read the Delta table - Perform file listing and file reads from your storage/file system. - Read Parquet files in columnar data, preferably in an in-memory columnar format. - Parse JSON data - Read JSON files - Evaluate expressions on in-memory columnar batches For each of these capabilities, you can choose to build your own implementation or reuse the default implementation. ### Step 1: Set up Delta Kernel in your connector project In the Delta Kernel project, there are multiple dependencies you can choose to depend on. 1. Delta Kernel core APIs - This is a must-have dependency, which contains all the main APIs like Table, Snapshot, and Scan that you will use to access the metadata and data of the Delta table. This has very few dependencies reducing the chance of conflicts with any dependencies in your connector and engine. This also provides the [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) interface which allows you to plug in your implementations of computationally expensive operations, but it does not provide any implementation of this interface. 2. Delta Kernel default- This has a default implementation called [`DefaultEngine`](https://delta-io.github.io/delta/snapshot/kernel-defaults/java/io/delta/kernel/defaults/engine/DefaultEngine.html) and additional dependencies such as `Hadoop`. If you wish to reuse all or parts of this implementation, then you can optionally depend on this. #### Set up Java projects As discussed above, you can import one or both of the artifacts as follows: ```xml io.delta delta-kernel-api ${delta-kernel.version} io.delta delta-kernel-defaults ${delta-kernel.version} ``` ### Step 2: Build your own Engine In this section, we are going to explore the [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) interface and walk through how to implement your own implementation so that you can plug in your connector/engine-specific implementations of computationally-intensive operations, threading model, resource management, etc. > [!IMPORTANT] During the validation process, if you believe that all the dependencies of the default `Engine` implementation can work with your connector and engine, then you can skip this step and jump to Step 3 of implementing your connector using the default engine. If later you have the need to customize the helper for your connector, you can revisit this step. #### Step 2.1: Implement the `Engine` interface The [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) interface combines a bunch of sub-interfaces each of which is designed for a specific purpose. Here is a brief overview of the subinterfaces. See the API docs (Java) for a more detailed view. ```java interface Engine { /** * Get the connector provided {@link ExpressionHandler}. * @return An implementation of {@link ExpressionHandler}. */ ExpressionHandler getExpressionHandler(); /** * Get the connector provided {@link JsonHandler}. * @return An implementation of {@link JsonHandler}. */ JsonHandler getJsonHandler(); /** * Get the connector provided {@link FileSystemClient}. * @return An implementation of {@link FileSystemClient}. */ FileSystemClient getFileSystemClient(); /** * Get the connector provided {@link ParquetHandler}. * @return An implementation of {@link ParquetHandler}. */ ParquetHandler getParquetHandler(); } ``` To build your own `Engine` implementation, you can choose to either use the default implementations of each sub-interface or completely build every one from scratch. ```java class MyEngine extends DefaultEngine { FileSystemClient getFileSystemClient() { // Build a new implementation from scratch return new MyFileSystemClient(); } // For all other sub-clients, use the default implementations provided by the `DefaultEngine`. } ``` Next, we will walk through how to implement each interface. #### Step 2.2: Implement `FileSystemClient` interface The [`FileSystemClient`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/FileSystemClient.html) interface contains basic file system operations like listing directories, resolving paths into a fully qualified path and reading bytes from files. Implementation of this interface must take care of the following when interacting with storage systems such as S3, Hadoop, or ADLS: - Credentials and permissions: The connector must populate its `FileSystemClient` with the necessary configurations and credentials for the client to retrieve the necessary data from the storage system. For example, an implementation based on Hadoop's FileSystem abstractions can be passed S3 credentials via the Hadoop configurations. - Decryption: If file system objects are encrypted, then the implementation must decrypt the data before returning the data. #### Step 2.3: Implement `ParquetHandler` As the name suggests, this interface contains everything related to reading and writing Parquet files. It has been designed such that a connector can plug in a wide variety of implementations, from a simple single-threaded reader to a very advanced multi-threaded reader with pre-fetching and advanced connector-specific expression pushdown. Let's explore the methods to implement, and the guarantees associated with them. ##### Method `readParquetFiles(CloseableIterator fileIter, StructType physicalSchema, java.util.Optional predicate)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/ParquetHandler.html#readParquetFiles-io.delta.kernel.utils.CloseableIterator-io.delta.kernel.types.StructType-)) takes as input `FileStatus`s which contains metadata such as file path, size etc. of the Parquet file to read. The columns to be read from the Parquet file are defined by the physical schema. To implement this method, you may have to first implement your own [`ColumnarBatch`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnarBatch.html) and [`ColumnVector`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnVector.html) which is used to represent the in-memory data generated from the Parquet files. When identifying the columns to read, note that there are multiple types of columns in the physical schema (represented as a [`StructType`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/types/StructType.html)). - Data columns: Columns that are expected to be read from the Parquet file. Based on the `StructField` object defining the column, read the column in the Parquet file that matches the same name or field id. If the column has a field id (stored as `parquet.field.id` in the `StructField` metadata) then the field id should be used to match the column in the Parquet file. Otherwise, the column name should be used for matching. - Metadata columns: These are special columns that must be populated using metadata about the Parquet file ([`StructField#isMetadataColumn`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/types/StructField.html#isMetadataColumn--) tells whether a column in `StructType` is a metadata column). To understand how to populate such a column, first match the column name against the set of standard metadata column name constants. For example, - `StructFileld#isMetadataColumn()` returns true and the column name is `StructField.METADATA_ROW_INDEX_COLUMN_NAME`, then you have to a generate column vector populated with the actual index of each row in the Parquet file (that is, not indexed by the possible subset of rows returned after Parquet data skipping). ##### Requirements and guarantees Any implementation must adhere to the following guarantees. - The schema of the returned `ColumnarBatch`es must match the physical schema. - If a data column is not found and the `StructField.isNullable = true`, then return a `ColumnVector` of nulls. Throw an error if it is not nullable. - The output iterator must maintain ordering as the input iterator. That is, if `file1` is before `file2` in the input iterator, then columnar batches of `file1` must be before those of `file2` in the output iterator. ##### Method `writeParquetFiles(String directoryPath, CloseableIterator dataIter, java.util.List statsColumns)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/ParquetHandler.html#writeParquetFiles-java.lang.String-io.delta.kernel.utils.CloseableIterator-java.util.List-) takes given data writes it into one or more Parquet files into the given directory. The data is given as an iterator of [FilteredColumnarBatches](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/FilteredColumnarBatch.html) which contains a [ColumnarBatch](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnarBatch.html) and an optional selection vector containing one entry for each row in `ColumnarBatch` indicating whether a row is selected or not selected. The `ColumnarBatch` also contains the schema of the data. This schema should be converted to Parquet schema, including any field IDs present [`FieldMetadata`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/types/FieldMetadata.html) for each column `StructField`. There is also the parameter `statsColumns`, which is a hint to the Parquet writer on what set of columns to collect stats for each file. The statistics include `min`, `max` and `null_count` for each column in the `statsColumns` list. Statistics collection is optional, but when present it is used by Kernel to persist the stats as part of the Delta table commit. This will help read queries prune un-needed data files based on the query predicate. For each written data file, the caller is expecting a [`DataFileStatus`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/utils/DataFileStatus.html) object. It contains the data file path, size, modification time, and optional column statistics. #### Method `writeParquetFileAtomically(String filePath, CloseableIterator data)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/ParquetHandler.html#writeParquetFiles-java.lang.String-io.delta.kernel.utils.CloseableIterator-java.util.List-) writes the given `data` into Parquet file at location `filePath`. The write is an atomic write i.e., either a Parquet file is created with all given content or no Parquet file is created at all. This should not create a file with partial content in it. The default implementation makes use of [`LogStore`](https://github.com/delta-io/delta/blob/master/storage/src/main/java/io/delta/storage/LogStore.java) implementations from the [`delta-storage`](https://github.com/delta-io/delta/tree/master/storage) module to accomplish the atomicity. A connector that wants to implement their own version of `ParquetHandler` can take a look at the default implementation for details. ##### Performance suggestions - The representation of data as `ColumnVector`s and `ColumnarBatch`es can have a significant impact on the query performance and it's best to read the Parquet file data directly into vectors and batches of the engine-native format to avoid potentially costly in-memory data format conversion. Create a Kernel `ColumnVector` and `ColumnarBatch` wrappers around the engine-native format equivalent classes. #### Step 2.4: Implement `ExpressionHandler` interface The [`ExpressionHandler`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/ExpressionHandler.html) interface has all the methods needed for handling expressions that may be applied on columnar data. ##### Method `getEvaluator(StructType batchSchema, Expression expresion, DataType outputType)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/ExpressionHandler.html#getEvaluator-io.delta.kernel.types.StructType-io.delta.kernel.expressions.Expression-io.delta.kernel.types.DataType-) generates an object of type [`ExpressionEvaluator`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/expressions/ExpressionEvaluator.html) that can evaluate the `expression` on a batch of row data to produce a result of a single column vector. To generate this function, the `getEvaluator()` method takes as input the expression and the schema of the `ColumnarBatch`es of data on which the expressions will be applied. The same object can be used to evaluate multiple columnar batches of input with the same schema and expression the evaluator is created for. ##### Method `getPredicateEvaluator(StructType inputSchema, Predicate predicate)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/ExpressionHandler.html#createSelectionVector-boolean:A-int-int-) is for creating an expression evaluator for `Predicate` type expressions. The `Predicate` type expressions return a boolean value as output. The returned object is of type [`PredicateEvaluator`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/expressions/PredicateEvaluator.html). This is a special interface for evaluating Predicate on input batch returns a selection vector containing one value for each row in input batch indicating whether the row has passed the predicate or not. Optionally it takes an existing selection vector along with the input batch for evaluation. The result selection vector is combined with the given existing selection vector and a new selection vector is returned. This mechanism allows running an input batch through several predicate evaluations without rewriting the input batch to remove rows that do not pass the predicate after each predicate evaluation. The new selection should be the same or more selective as the existing selection vector. For example, if a row is marked as unselected in the existing selection vector, then it should remain unselected in the returned selection vector even when the given predicate returns true for the row. ##### Method `createSelectionVector(boolean[] values, int from, int to)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/ExpressionHandler.html#createSelectionVector-boolean:A-int-int-) allows creating `ColumnVector` for boolean type values given as input. This allows the connector to maintain all `ColumnVector`s created in the desired memory format. ##### Requirements and guarantees Any implementation must adhere to the following guarantees. - Implementation must handle all possible variations of expressions. If the implementation encounters an expression type that it does not know how to handle, then it must throw a specific language-dependent exception. - Java: [NotSupportedException](https://docs.oracle.com/javaee/7/api/latest/javax/resource/NotSupportedException.html) - The `ColumnarBatch`es on which the generated `ExpressionEvaluator` is going to be used are guaranteed to have the schema provided during generation. Hence, it is safe to bind the expression evaluation logic to column ordinals instead of column names, thus making the actual evaluation faster. #### Step 2.5: Implement `JsonHandler` [This](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/JsonHandler.html) engine interface allows the connector to use plug-in their own JSON handling code and expose it to the Delta Kernel. ##### Method `readJsonFiles(CloseableIterator fileIter, StructType physicalSchema, java.util.Optional predicate)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/JsonHandler.html#readJsonFiles-io.delta.kernel.utils.CloseableIterator-io.delta.kernel.types.StructType-) takes as input `FileStatus`s of the JSON files and returns the data in a series of columnar batches. The columns to be read from the JSON file are defined by the physical schema, and the return batches must match that schema. To implement this method, you may have to first implement your own [`ColumnarBatch`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnarBatch.html)and [`ColumnVector`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnVector.html) which is used to represent the in-memory data generated from the JSON files. When identifying the columns to read, note that there are multiple types of columns in the physical schema (represented as a [`StructType`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/types/StructType.html)). ##### Method `parseJson(ColumnVector jsonStringVector, StructType outputSchema, java.util.Optional selectionVector)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/JsonHandler.html#parseJson-io.delta.kernel.data.ColumnVector-io.delta.kernel.types.StructType-) allows parsing a `ColumnVector` of string values which are in JSON format into the output format specified by the `outputSchema`. If a given column in `outputSchema` is not found, then a null value is returned. It optionally takes a selection vector which indicates what entries in the input `ColumnVector` of strings to parse. If an entry is not selected then a `null` value is returned as parsed output for that particular entry in the output. ##### Method `deserializeStructType(String structTypeJson)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/JsonHandler.html#deserializeStructType-java.lang.String-) allows parsing JSON encoded (according to [Delta schema serialization rules](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#schema-serialization-format)) `StructType` schema into a `StructType`. Most implementations of `JsonHandler` do not need to implement this method and instead use the one in the [default `JsonHandler`](https://delta-io.github.io/delta/snapshot/kernel-defaults/java/io/delta/kernel/defaults/engine/DefaultJsonHandler.html) implementation. #### Method `writeJsonFileAtomically(String filePath, CloseableIterator data, boolean overwrite)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/JsonHandler.html#writeJsonFileAtomically-java.lang.String-io.delta.kernel.utils.CloseableIterator-boolean-) writes the given `data` into a JSON file at location `filePath`. The write is an atomic write i.e., either a JSON file is created with all given content or no Parquet file is created at all. This should not create a file with partial content in it. The default implementation makes use of [`LogStore`](https://github.com/delta-io/delta/blob/master/storage/src/main/java/io/delta/storage/LogStore.java) implementations from the [`delta-storage`](https://github.com/delta-io/delta/tree/master/storage) module to accomplish the atomicity. A connector that wants to implement their own version of `JsonHandler` can take a look at the default implementation for details. The implementation is expected to handle the serialization rules (converting the `Row` object to JSON string) as described in the [API Javadoc](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/JsonHandler.html#writeJsonFileAtomically-java.lang.String-io.delta.kernel.utils.CloseableIterator-boolean-). #### Step 2.6: Implement `ColumnarBatch` and `ColumnVector` [`ColumnarBatch`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnarBatch.html) and [`ColumnVector`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnVector.html) are two interfaces to represent the data read into memory from files. This representation can have a significant impact on query performance. Each engine likely has a native representation of in-memory data with which it applies data transformation operations. For example, in Apache Spark™, the row data is internally represented as `UnsafeRow` for efficient processing. So it's best to read the Parquet file data directly into vectors and batches of the native format to avoid potentially costly in-memory data format conversions. So the recommended approach is to build wrapper classes that extend the two interfaces but internally use engine-native classes to store the data. When the connector has to forward the columnar batches received from the kernel to the engine, it has to be smart enough to skip converting vectors and batches that are already in the engine-native format. ### Step 3: Build read support in your connector In this section, we are going to walk through the likely sequence of Kernel API calls your connector will have to make to read a table. The exact timing of making these calls in your connector in the context of connector-engine interactions depends entirely on the engine-connector APIs and is therefore beyond the scope of this guide. However, we will try to provide broad guidelines that are likely (but not guaranteed) to apply to your connector-engine setup. For this purpose, we are going to assume that the engine goes through the following phases when processing a read/scan query - logical plan analysis, physical plan generation, and physical plan execution. Based on these broad characterizations, a typical control and data flow for reading a Delta table is going to be as follows: | Step | Typical query phase when this step occurs | | --- | --- | | Resolve the table snapshot to query | Logical plan analysis phase when the plan's schema and other details need to be resolved and validated | | Resolve files to scan based on query parameters | Physical plan generation, when the final parameters of the scan are available. For example: Schema of data to read after pruning away unused columns. Query filters to apply after filter rearrangement | | Distribute the file information to workers | Physical plan execution, only if it is a distributed engine. | | Read the columnar data using the file information | Physical plan execution, when the data is being processed by the engine | Let's understand the details of each step. #### Step 3.1: Resolve the table snapshot to query The first step is to resolve the consistent snapshot and the schema associated with it. This is often required by the connector/ engine to resolve and validate the logical plan of the scan query (if the concept of logical plan exists in your engine). To achieve this, the connector has to do the following. - Resolve the table path from the query: If the path is directly available, then this is easy. Otherwise, if it is a query based on a catalog table (for example, a Delta table defined in Hive Metastore), then the connector has to resolve the table path from the catalog. - Initialize the `Engine` object: Create a new instance of the `Engine` that you have chosen in Step 2. - Initialize the Kernel objects and get the schema: Assuming the query is on the latest available version/snapshot of the table, you can get the table schema as follows: ```java import io.delta.kernel.*; import io.delta.kernel.defaults.engine.*; Engine myEngine = new MyEngine(); Table myTable = Table.forPath(myTablePath); Snapshot mySnapshot = myTable.getLatestSnapshot(myEngine); StructType mySchema = mySnapshot.getSchema(myEngine); ``` If you want to query a specific version of the table (that is, not the schema), then you can get the required snapshot as `myTable.getSnapshot(version)`. #### Step 3.2: Resolve files to scan Next, we need to build a Scan object using more information from the query. Here we are going to assume that the connector/engine has been able to extract the following details from the query (say, after optimizing the logical plan): - Read schema: The columns in the table that the query needs to read. This may be the full set of columns or a subset of columns. - Query filters: The filters on partitions or data columns that can be used skip reading table data. To provide this information to Kernel, you have to do the following: - Convert the engine-specific schema and filter expressions to Kernel schema and expressions: For schema, you have to create a `StructType` object. For the filters, you have to create an `Expression` object using all the available subclasses of `Expression`. - Build the scan with the converted information: Build the scan as follows: ```java import io.delta.kernel.expressions.*; import io.delta.kernel.types.*; StructType readSchema = ... ; // convert engine schema Predicate filterExpr = ... ; // convert engine filter expression Scan myScan = mySnapshot.buildScan(engine) .withFilter(myEngine, filterExpr) .withReadSchema(myEngine, readSchema) .build() ``` - Resolve the information required to file reads: The generated Scan object has two sets of information. - Scan files: `myScan.getScanFiles()` returns an iterator of `ColumnarBatch`es. Each batch in the iterator contains rows and each row has information about a single file that has been selected based on the query filter. - Scan state: `myScan.getScanState()` returns a `Row` that contains all the information that is common across all the files that need to be read. ````java Row myScanStateRow = myScan.getScanState(); CloseableIterator myScanFilesAsBatches = myScan.getScanFiles(); ```java Row myScanStateRow = myScan.getScanState(); CloseableIterator myScanFilesAsBatches = myScan.getScanFiles(); while (myScanFilesAsBatches.hasNext()) { FilteredColumnarBatch scanFileBatch = myScanFilesAsBatches.next(); CloseableIterator myScanFilesAsRows = scanFileBatch.getRows(); } ```` As we will soon see, reading the columnar data from a selected file will need to use both, the scan state row, and a scan file row with the file information. ##### Requirements and guarantees Here are the details you need to ensure when defining this scan. - The provided `readSchema` must be the exact schema of the data that the engine will expect when executing the query. Any mismatch in the schema defined during this query planning and the query execution will result in runtime failures. Hence you must build the scan with the readSchema only after the engine has finalized the logical plan after any optimizations like column pruning. - When applicable (for example, with Java Kernel APIs), you have to make sure to call the close() method as you consume the `ColumnarBatch`es of scan files (that is, either serialize the rows or use them to read the table data). #### Step 3.3: Distribute the file information to the workers If you are building a connector for a distributed engine like Spark/Presto/Trino/Flink, then your connector has to send all the scan metadata from the query planning machine (henceforth called the driver) to task execution machines (henceforth called the workers). You will have to serialize and deserialize the scan state and scan file rows. It is the connector job to implement serialization and deserialization utilities for a [`Row`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/Row.html). If the connector wants to split reading one scan file into multiple tasks, it can add additional connector specific split context to the task. At the task, the connector can use its own Parquet reader to read the specific part of the file indicated by the split info. ##### Custom `Row` Serializer/Deserializer Here are steps on how to build your own serializer/deserializer such that it will work with any [`Row`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/Row.html) of any schema. - Serializing - First serialize the row schema, that is, `StructType` object. - Then, use the schema to identify types of each column/ordinal in the `Row` and use that to serialize all the values one by one. - Deserializing - Define your own class that extends the Row interface. It must be able to handle complex types like arrays, nested structs and maps. - First deserialize the schema. - Then, use the schema to deserialize the values and put them in an instance of your custom Row class. ```java import io.delta.kernel.utils.*; // In the driver where query planning is being done Byte[] scanStateRowBytes = RowUtils.serialize(scanStateRow); Byte[] scanFileRowBytes = RowUtils.serialize(scanFileRow); // Optionally the connector adds a split info to the task (scan file, scan state) to // split reading of a Parquet file into multiple tasks. The task gets split info // along with the scan file row and scan state row. Split split = ...; // connector specific class, not related to Kernel // Send these over to the worker // In the worker when data will be read, after rowBytes have been sent over Row scanStateRow = RowUtils.deserialize(scanStateRowBytes); Row scanFileRow = RowUtils.deserialize(scanFileRowBytes); Split split = ... deserialize split info ...; ``` #### Step 3.4: Read the columnar data Finally, we are ready to read the columnar data. You will have to do the following: - Read the physical data from Parquet file as indicated by the scan file row, scan state, and optionally the split info - Convert the physical data into logical data of the table using the Kernel's APIs. ```java Row scanStateRow = ... ; Row scanFileRow = ... ; Split split = ...; // Additional option predicate such as dynamic filters the connector wants to // pass to the reader when reading files. Predicate optPredicate = ...; // Get the physical read schema of columns to read from the Parquet data files StructType physicalReadSchema = ScanStateRow.getPhysicalDataReadSchema(engine, scanStateRow); // From the scan file row, extract the file path, size and modification metadata // needed to read the file. FileStatus fileStatus = InternalScanFileUtils.getAddFileStatus(scanFileRow); // Open the scan file which is a Parquet file using connector's own // Parquet reader which supports reading specific parts (split) of the file. // If the connector doesn't have its own Parquet reader, it can use the // default Parquet reader provider which at the moment doesn't support reading // a specific part of the file, but reads the entire file from the beginning. CloseableIterator physicalDataIter = connectParquetReader.readParquetFile( fileStatus physicalReadSchema, split, // what part of the Parquet file to read data from optPredicate /* additional predicate the connector can apply to filter data from the reader */ ); // Now the physical data read from the Parquet data file is converted to logical data // the table represents. // Logical data may include the addition of partition columns and/or // subset of rows deleted CloseableIterator transformedData = Scan.transformPhysicalData( engine, scanState, scanFileRow, physicalDataIter)); ``` - Resolve the data in the batches: Each [`FilteredColumnarBatch`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/FilteredColumnarBatch.html) has two components: - Columnar batch (returned by `FilteredColumnarBatch.getData()`): This is the data read from the files having the schema matching the readSchema provided when the Scan object was built in the earlier step. - Optional selection vector (returned by `FilteredColumnarBatch.getSelectionVector()`): Optionally, a boolean vector that will define which rows in the batch are valid and should be consumed by the engine. If the selection vector is present, then you will have to apply it to the batch to resolve the final consumable data. - Convert to engine-specific data format: Each connector/engine has its own native row / columnar batch formats and interfaces. To return the read data batches to the engine, you have to convert them to fit those engine-specific formats and/or interfaces. Here are a few tips that you can follow to make this efficient. - Matching the engine-specific format: Some engines may expect the data in an in-memory format that may be different from the data produced by `getData()`. So you will have to do the data conversion for each column vector in the batch as needed. - Matching the engine-specific interfaces: You may have to implement wrapper classes that extend the engine-specific interfaces and appropriately encapsulate the row data. For best performance, you can implement your own Parquet reader and other `Engine` implementations to make sure that every `ColumnVector` generated is already in the engine-native format thus eliminating any need to convert. Now you should be able to read the Delta table correctly. ### Step 4: Build append support in your connector In this section, we are going to walk through the likely sequence of Kernel API calls your connector will have to make to append data to a table. The exact timing of making these calls in your connector in the context of connector-engine interactions depends entirely on the engine-connector APIs and is, therefore, beyond the scope of this guide. However, we will try to provide broad guidelines that are likely (but not guaranteed) to apply to your connector-engine setup. For this purpose, we are going to assume that the engine goes through the following phases when processing a write query - logical plan analysis, physical plan generation, and physical plan execution. Based on these broad characterizations, a typical control and data flow for reading a Delta table is going to be as follows: | Step | Typical query phase when this step occurs | | --- | --- | | Determine the schema of the data that needs to be written to the table. Schema is derived from the existing table or from the parent operation of the `write` operator in the query plan when the table doesn't exist yet. | Logical plan analysis phase when the plan's schema (`write` operator schema matches the table schema, etc.) and other details need to be resolved and validated. | | Determine the physical partitioning of the data based on the table schema and partition columns either from the existing table or from the query plan (for new tables) | Physical plan generation, where the number of writer tasks, data schema and partitioning is determined | | Distribute the writer tasks definitions (which include the transaction state) to workers. | Physical plan execution, only if it is a distributed engine. | | Tasks write the data to data files and send the data file info to the driver. | Physical plan execution, when the data is actually written to the table location | | Finalize the query. Here, all the info of the data files written by the tasks is aggregated and committed to the transaction created at the beginning of the physical execution. | Finalize the query. This happens on the driver where the query has started. | Let's understand the details of each step. #### Step 4.1: Determine the schema of the data that needs to be written to the table The first step is to resolve the output data schema. This is often required by the connector/ engine to resolve and validate the logical plan of the query (if the concept of logical plan exists in your engine). To achieve this, the connector has to do the following. At a high level query plan is a tree of operators where the leaf-level operators generate or read data from storage/tables and feed it upwards towards the parent operator nodes. This data transfer happens until it reaches the root operator node where the query is finalized (either the results are sent to the client or data is written to another table). - Create the `Table` object - From the `Table` object try to get the schema. - If the table is not found - the query includes creating the table (e.g., `CREATE TABLE AS` SQL query); - the schema is derived from the operator above the `write` that feeds the data to the `write` operator. - the query doesn't include creating new table, an exception is thrown saying the table is not found - If the table already exists - get the schema from the table and check if it matches the schema of the `write` operator. If not throw an exception. - Create a `TransactionBuilder` - this basically begins the steps of transaction construction. ```java import io.delta.kernel.*; import io.delta.kernel.defaults.engine.*; Engine myEngine = new MyEngine(); Table myTable = Table.forPath(myTablePath); StructType writeOperatorSchema = // ... derived from the query operator tree ... StructType dataSchema; boolean isNewTable = false; try { Snapshot mySnapshot = myTable.getLatestSnapshot(myEngine); dataSchema = mySnapshot.getSchema(myEngine); // .. check dataSchema and writeOperatorSchema match ... } catch(TableNotFoundException e) { isNewTable = true; dataSchema = writeOperatorSchema; } TransactionBuilder txnBuilder = myTable.createTransactionBuilder( myEngine, "Examples", /* engineInfo - connector can add its own identifier which is noted in the Delta Log */ Operation /* What is the operation we are trying to perform? This is noted in the Delta Log */ ); if (isNewTable) { // For a new table set the table schema in the transaction builder txnBuilder = txnBuilder.withSchema(engine, dataSchema) } ``` #### Step 4.2: Determine the physical partitioning of the data based on the table schema and partition columns Partition columns are found either from the query (for new tables, the query defines the partition columns) or from the existing table. ```java TransactionBuilder txnBuilder = ... from the last step ... Transaction txn; List partitionColumns = ... if (newTable) { partitionColumns = ... derive from the query parameters (ex. PARTITION BY clause in SQL) ... txnBuilder = txnBuilder.withPartitionColumns(engine, partitionColumns); txn = txnBuilder.build(engine); } else { txn = txnBuilder.build(engine); partitionColumns = txn.getPartitionColumns(engine); } ``` At the end of this step, we have the `Transaction` and schema of the data to generate and its partitioning. #### Step 4.3: Distribute the writer tasks definitions (which include the transaction state) to workers If you are building a connector for a distributed engine like Spark/Presto/Trino/Flink, then your connector has to send all the writer metadata from the query planning machine (henceforth called the driver) to task execution machines (henceforth called the workers). You will have to serialize and deserialize the transaction state. It is the connector job to implement serialization and deserialization utilities for a [`Row`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/Row.html). More details on a custom `Row` SerDe are found [here](#custom-row-serializerdeserializer). ```java Row txnState = txn.getState(engine); String jsonTxnState = serializeToJson(txnState); ``` #### Step 4.4: Tasks write the data to data files and send the data file info to the driver In this step (which is executed on the worker nodes inside each task): - Deserialize the transaction state - Writer operator within the task gets the data from its parent operator. - The data is converted into a `FilteredColumnarBatch`. Each [`FilteredColumnarBatch`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/FilteredColumnarBatch.html) has two components: - Columnar batch (returned by `FilteredColumnarBatch.getData()`): This is the data read from the files having the schema matching the readSchema provided when the Scan object was built in the earlier step. - Optional selection vector (returned by `FilteredColumnarBatch.getSelectionVector()`): Optionally, a boolean vector that will define which rows in the batch are valid and should be consumed by the engine. - The connector can create `FilteredColumnBatch` wrapper around data in its own in-memory format. - Check if the data is partitioned or not. If not partitioned, partition the data by partition values. - For each partition generate the map of the partition column to the partition value - Use Kernel to convert the partitioned data into physical data that should go into the data files - Write the physical data into one or more data files. - Convert data file statues into a Delta log actions - Serialize the Delta log action `Row` objects and send them to the driver node ``` Row txnState = ... deserialize from JSON string sent by the driver ... CloseableIterator data = ... generate data ... // If the table is un-partitioned then this is an empty map Map partitionValues = ... prepare the partition values ... // First transform the logical data to physical data that needs to be written // to the Parquet files CloseableIterator physicalData = Transaction.transformLogicalData(engine, txnState, data, partitionValues); // Get the write context DataWriteContext writeContext = Transaction.getWriteContext(engine, txnState, partitionValues); // Now write the physical data to Parquet files CloseableIterator dataFiles = engine.getParquetHandler() .writeParquetFiles( writeContext.getTargetDirectory(), physicalData, writeContext.getStatisticsColumns()); // Now convert the data file status to data actions that needs to be written to the Delta table log CloseableIterator partitionDataActions = Transaction.generateAppendActions( engine, txnState, dataFiles, writeContext); .... serialize `partitionDataActions` and send them to driver node ``` #### Step 4.5: Finalize the query At the driver node, the delta log actions from all the tasks are received and committed to the transaction. The tasks send the Delta log actions as a serialized JSON and deserialize them back to `Row` objects. ``` // Create a iterable out of the data actions. If the contents are too big to fit in memory, // the connector may choose to write the data actions to a temporary file and return an // iterator that reads from the file. CloseableIterable dataActionsIterable = CloseableIterable.inMemoryIterable( toCloseableIterator(dataActions.iterator())); // Commit the transaction. TransactionCommitResult commitResult = txn.commit(engine, dataActionsIterable); // Optional step if (commitResult.isReadyForCheckpoint()) { // Checkpoint the table Table.forPath(engine, tablePath).checkpoint(engine, commitResult.getVersion()); } ``` Thats it. Now you should be able to append data to Delta tables using the Kernel APIs. ## Migration guide Kernel APIs are still evolving and new features are being added. Kernel authors try to make the API changes backward compatible as much as they can with each new release, but sometimes it is hard to maintain the backward compatibility for a project that is evolving rapidly. This section provides guidance on how to migrate your connector to the latest version of Delta Kernel. With each new release the [examples](https://github.com/delta-io/delta/tree/master/kernel/examples) are kept up-to-date with the latest API changes. You can refer to the examples to understand how to use the new APIs. ### Migration from Delta Lake version 3.1.0 to 3.2.0 Following are API changes in Delta Kernel 3.2.0 that may require changes in your connector. #### Rename `TableClient` to [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) The `TableClient` interface has been renamed to `Engine`. This is the most significant API change in this release. The `TableClient` interface name is not exactly representing the functionality it provides. At a high level it provides capabilities such as reading Parquet files, JSON files, evaluating expressions on data and file system functionality. These are basically the heavy lift operations that Kernel depends on as a separate interface to allow the connectors to substitute their own custom implementation of the same functionality (e.g. custom Parquet reader). Essentially, these functionalities are the core of the `engine` functionalities. By renaming to `Engine`, we are representing the interface functionality with a proper name that is easy to understand. The `DefaultTableClient` has been renamed to [`DefaultEngine`](https://delta-io.github.io/delta/snapshot/kernel-defaults/java/io/delta/kernel/defaults/engine/DefaultEngine.html). #### [`Table.forPath(Engine engine, String tablePath)`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Table.html#forPath-io.delta.kernel.engine.Engine-java.lang.String-) behavior change Earlier when a non-existent table path is passed, the API used to throw `TableNotFoundException`. Now it doesn't throw the exception. Instead, it returns a `Table` object. When trying to get a `Snapshot` from the table object it throws the `TableNotFoundException`. #### [`FileSystemClient.resolvePath`](https://delta-io.github.io/delta/snapshot/kernel-defaults/java/io/delta/kernel/defaults/engine/DefaultFileSystemClient.html#resolvePath-java.lang.String-) behavior change Earlier when a non-existent path is passed, the API used to throw `FileNotFoundException`. Now it doesn't throw the exception. It still resolves the given path into a fully qualified path. ## Kernel Rust The Rust Kernel is a set of libraries for building Delta connectors in native languages. Work in progress. ## More Information - [Talk](https://www.youtube.com/watch?v=KVUMFv7470I) explaining the rationale behind Kernel and the API design (slides are available [here](https://docs.google.com/presentation/d/1PGSSuJ8ndghucSF9GpYgCi9oeRpWolFyehjQbPh92-U/edit) which are kept up-to-date with the changes). - [User guide](https://github.com/delta-io/delta/blob/master/kernel/USER_GUIDE.md) on the step-by-step process of using Kernel in a standalone Java program or in a distributed processing connector for reading and writing to Delta tables. - Example [Java programs](https://github.com/delta-io/delta/tree/master/kernel/examples) that illustrate how to read and write Delta tables using the Kernel APIs. - Table and default Engine API Java [documentation](/api/latest/java/kernel/index.html) - [Migration guide](https://github.com/delta-io/delta/blob/master/kernel/USER_GUIDE.md#migration-guide) ================================================ FILE: docs/src/content/docs/delta-more-connectors.mdx ================================================ --- title: Other connectors --- import { Tabs, TabItem, Aside, Steps } from "@astrojs/starlight/components"; ## Apache Druid This [connector](https://druid.apache.org/docs/latest/development/extensions-contrib/delta-lake/) allows [Apache Druid](https://druid.apache.org/) to read from Delta Lake. ## Apache Pulsar This [connector](https://github.com/streamnative/pulsar-io-lakehouse/blob/master/docs/delta-lake-demo.md) allows [Apache Pulsar](https://pulsar.apache.org/) to read from and write to Delta Lake. ## ClickHouse [ClickHouse](https://clickhouse.com/) is a column-oriented database that allows users to run SQL queries on Delta Lake tables. This [connector](https://clickhouse.com/docs/en/engines/table-engines/integrations/deltalake) provides a read-only integration with existing Delta Lake tables in Amazon S3. ## Dagster Use the [Delta Lake IO Manager](https://delta-io.github.io/delta-rs/integrations/delta-lake-dagster/) to read from and write to Delta Lake tables in your [Dagster](https://dagster.io/) orchestration pipelines. ## FINOS Legend An [extension](https://github.com/finos/legend-community-delta/blob/main/README.md) to the [FINOS](https://landscape.finos.org/) Legend framework for Apache Spark™ / Delta Lake based environment, combining best of open data standards with open source technologies. This connector allows Trino to read from and write to Delta Lake. ## Hopsworks This [connectors](https://docs.hopsworks.ai/latest/user_guides/fs/feature_group/create/#batch-write-api) allows [Hopsworks Feature Store](https://www.hopsworks.ai/dictionary/feature-store) store, manage, and serve feature data in Delta Lake. ## Apache Hive This integration enables reading Delta tables from Apache Hive. For details on installing the integration, see the [Delta Lake repository](https://github.com/delta-io/delta/tree/master/connectors/hive). ## Kafka Delta Ingest This [project](https://github.com/delta-io/kafka-delta-ingest) builds a highly efficient daemon for streaming data through Apache Kafka into Delta Lake. ## SQL Delta Import This [utility](https://github.com/delta-io/delta/blob/master/connectors/sql-delta-import/readme.md) is for importing data from a JDBC source into a Delta Lake table. ## StarRocks [StarRocks](https://www.starrocks.io/), a Linux Foundation project, is a next-generation sub-second MPP OLAP database for full analytics scenarios, including multi-dimensional analytics, real-time analytics, and ad-hoc queries. StarRocks has the [ability to read](https://docs.starrocks.io/docs/introduction/StarRocks_intro/) from Delta Lake. ================================================ FILE: docs/src/content/docs/delta-presto-integration.mdx ================================================ --- title: Presto connector description: Learn how to set up an integration to enable you to read Delta tables from Presto. --- Since Presto [version 0.269](https://prestodb.io/docs/0.269/release/release-0.269.html#delta-lake-connector-changes), Presto natively supports reading Delta Lake tables. For details on using the native Delta Lake connector, see [Delta Lake Connector - Presto](https://prestodb.io/docs/current/connector/deltalake.html). For Presto versions lower than [0.269](https://prestodb.io/docs/0.269/release/release-0.269.html#delta-lake-connector-changes), you can use the manifest-based approach detailed in [Presto, Trino, and Athena to Delta Lake integration using manifests](/presto-integration/). ================================================ FILE: docs/src/content/docs/delta-resources.mdx ================================================ --- title: Delta Lake resources description: Learn about resources for understanding Delta Lake. --- ## Blog posts and talks [Delta Lake blog posts](https://delta.io/blog) [Delta Lake tutorials](https://delta.io/learn/tutorials/) [Delta Lake videos](https://delta.io/learn/videos/) ## VLDB 2020 paper [Delta Lake: High-Performance ACID Table Storage over Cloud Object Stores](https://databricks.com/wp-content/uploads/2020/08/p975-armbrust.pdf) ## Examples The Delta Lake GitHub repository has [Scala and Python examples](https://github.com/delta-io/delta/tree/master/examples/). ## Delta Lake transaction log specification The Delta Lake transaction log has a well-defined open protocol that can be used by any system to read the log. See [Delta Transaction Log Protocol](https://github.com/delta-io/delta/blob/master/PROTOCOL.md). ================================================ FILE: docs/src/content/docs/delta-row-tracking.mdx ================================================ --- title: Use row tracking for Delta tables description: Learn how Delta Lake row tracking allows tracking how rows change across table versions. --- import { Tabs, TabItem, Aside } from "@astrojs/starlight/components"; Row tracking allows Delta Lake to track row-level lineage in a Delta Lake table. When enabled on a Delta Lake table, row tracking adds two new metadata fields to the table: - **Row IDs** provide rows with an identifier that is unique within the table. A row keeps the same ID whenever it is modified using a `MERGE` or `UPDATE` statement. - **Row commit versions** record the last version of the table in which the row was modified. A row is assigned a new version whenever it is modified using a `MERGE` or `UPDATE` statement. ## Enable row tracking You must explicitly enable row tracking using one of the following methods: - **New table**: Set the table property `delta.enableRowTracking = true` in the `CREATE TABLE` command. ```sql -- Create an empty table CREATE TABLE student (id INT, name STRING, age INT) TBLPROPERTIES ('delta.enableRowTracking' = 'true'); -- Using a CTAS statement CREATE TABLE course_new TBLPROPERTIES ('delta.enableRowTracking' = 'true') AS SELECT * FROM course_old; -- Using a LIKE statement to copy configuration CREATE TABLE graduate LIKE student; -- Using a CLONE statement to copy configuration CREATE TABLE graduate CLONE student; ``` - **Existing table**: Available from Delta 3.3 and above, set the table property `'delta.enableRowTracking' = 'true'` in the `ALTER TABLE` command. ```sql ALTER TABLE grade SET TBLPROPERTIES ('delta.enableRowTracking' = 'true'); ``` - **All new tables**: Set the configuration `spark.databricks.delta.properties.defaults.enableRowTracking = true` for the current session in the `SET` command. ```sql SET spark.databricks.delta.properties.defaults.enableRowTracking = true; ``` ```python spark.conf.set("spark.databricks.delta.properties.defaults.enableRowTracking", True) ``` ```scala spark.conf.set("spark.databricks.delta.properties.defaults.enableRowTracking", true) ``` ### Row tracking storage Enabling row tracking may increase the size of the table. Delta Lake stores row tracking metadata fields in hidden metadata columns in the data files. Some operations, such as insert-only operations do not use these hidden columns and instead track the row ids and row commit versions using metadata in the Delta Lake log. Data reorganization operations such as `OPTIMIZE` and `REORG` cause the row ids and row commit versions to be tracked using the hidden metadata column, even when they were stored using metadata. ## Read row tracking metadata fields Row tracking adds the following metadata fields that can be accessed when reading a table: | Column name | Type | Values | | --- | --- | --- | | `_metadata.row_id` | Long | The unique identifier of the row. | | `_metadata.row_commit_version` | Long | The table version at which the row was last inserted or updated. | The row ids and row commit versions metadata fields are not automatically included when reading the table. Instead, these metadata fields must be manually selected from the hidden `_metadata` column which is available for all tables in Apache Spark. ```sql SELECT _metadata.row_id, _metadata.row_commit_version, * FROM table_name; ``` ```python spark.read.table("table_name") \ .select("_metadata.row_id", "_metadata.row_commit_version", "*") ``` ```scala spark.read.table("table_name") .select("_metadata.row_id", "_metadata.row_commit_version", "*") ``` ## Disable row tracking Row tracking can be disabled to reduce the storage overhead of the metadata fields. After disabling row tracking the metadata fields remain available, but all rows always get assigned a new id and commit version whenever they are touched by an operation. ```sql ALTER TABLE table_name SET TBLPROPERTIES (delta.enableRowTracking = false); ``` ```python spark.sql("ALTER TABLE table_name SET TBLPROPERTIES (delta.enableRowTracking = false)") ``` ```scala spark.sql("ALTER TABLE table_name SET TBLPROPERTIES (delta.enableRowTracking = false)") ``` ## Limitations The following limitations exist: - The row ids and row commit versions metadata fields cannot be accessed while reading the [Change data feed](/delta-change-data-feed/). - Once the Row Tracking feature is added to the table it cannot be removed without recreating the table. ================================================ FILE: docs/src/content/docs/delta-sharing.mdx ================================================ --- title: Read Delta Sharing Tables description: Learn how to perform reads on Delta Sharing tables. --- import { Tabs, TabItem, Aside } from "@astrojs/starlight/components"; [Delta Sharing](https://delta.io/sharing/) is an open protocol for secure real-time exchange of large datasets, which enables organizations to share data in real time regardless of which computing platforms they use. It is a simple REST protocol that securely grants access to part of a cloud dataset and leverages modern cloud storage systems, such as S3, ADLS, GCS or R2, to reliably transfer data. In Delta Sharing, data provider is the one who owns the original dataset or table, and shares it with a broad range of recipients. Each table can be configured to be shared with different options (history, filtering, etc.) We will focus on consuming the shared table in this doc. Delta Sharing data source supports most of the options provided by Apache Spark DataFrame for performing reads through [batch](/delta-batch), [streaming](/delta-streaming), or [table changes (CDF)](/delta-change-data-feed) APIs on shared tables. Delta Sharing doesn't support writing to a shared table. Please refer to the [Delta Sharing Repo](https://github.com/delta-io/delta-sharing/blob/main/README.md) for more details. Please follow the [quick start](https://github.com/delta-io/delta-sharing?tab=readme-ov-file#quick-start) to leverage the Delta Sharing python connector to discover the shared tables. For Delta Sharing reads on shared tables with advanced Delta Lake features such as [Deletion Vectors](/delta-deletion-vectors) and [Column Mapping](/delta-column-mapping), you need to enable integration with Apache Spark DataSourceV2 and Catalog APIs (since delta-sharing-spark 3.1) by setting the same configurations as Delta Lake when you create a new `SparkSession`. See [Configure SparkSession](/delta-batch/#configure-sparksession). ## Read a snapshot After you save the [Profile File](https://github.com/delta-io/delta-sharing/blob/main/PROTOCOL.md#profile-file-format) locally and launch Spark with the connector library, you can access shared tables. A profile file is provided by the data provider to the data recipient. ```sql -- A table path is the profile file path followed by `#` and the fully qualified name -- of a table (`..`). CREATE TABLE mytable USING deltaSharing LOCATION '#..'; SELECT * FROM mytable; ``` ```python # A table path is the profile file path followed by `#` and the fully qualified name # of a table (`..`). table_path = "#.." df = spark.read.format("deltaSharing").load(table_path) ``` ```scala // A table path is the profile file path followed by `#` and the fully qualified name // of a table (`..`). val tablePath = "#.." val df = spark.read.format("deltaSharing").load(tablePath) ``` ```java // A table path is the profile file path followed by `#` and the fully qualified name // of a table (`..`). String tablePath = "#.."; Dataset df = spark.read.format("deltaSharing").load(tablePath); ``` The DataFrame returned automatically reads the most recent snapshot of the table for any query. Delta Sharing supports [predicate pushdown](https://github.com/delta-io/delta-sharing/blob/main/PROTOCOL.md#json-predicates-for-filtering) to efficiently fetch data from the Delta Sharing server when there are applicable predicates in the query. ## Query an older snapshot of a shared table (time travel) Once the data provider enables history sharing of the shared table, Delta Sharing time travel allows you to query an older snapshot of a shared table. ```sql SELECT * FROM mytable TIMESTAMP AS OF timestamp_expression SELECT * FROM mytable VERSION AS OF version ``` ```python spark.read.format("deltaSharing").option("timestampAsOf", timestamp_string).load(tablePath) spark.read.format("deltaSharing").option("versionAsOf", version).load(tablePath) ``` ```scala spark.read.format("deltaSharing").option("timestampAsOf", timestamp_string).load(tablePath) spark.read.format("deltaSharing").option("versionAsOf", version).load(tablePath) ``` The `timestamp_expression` and `version` share the same syntax as [Delta](/delta-batch#timestamp-and-version-syntax). ## Read Table Changes (CDF) Once the data provider turns on CDF on the original Delta Lake table and shares it with history through Delta Sharing, the recipient can query CDF of a Delta Sharing table similar to [CDF of a Delta table](/delta-change-data-feed). ```sql CREATE TABLE mytable USING deltaSharing LOCATION '#..'; -- version as ints or longs e.g. changes from version 0 to 10 SELECT * FROM table_changes('mytable', 0, 10) -- timestamp as string formatted timestamps SELECT * FROM table_changes('mytable', '2021-04-21 05:45:46', '2021-05-21 12:00:00') -- providing only the startingVersion/timestamp SELECT * FROM table_changes('mytable', 0) ``` ```python table_path = "#.." # version as ints or longs spark.read.format("deltaSharing") \ .option("readChangeFeed", "true") \ .option("startingVersion", 0) \ .option("endingVersion", 10) \ .load(tablePath) # timestamps as formatted timestamp spark.read.format("deltaSharing") \ .option("readChangeFeed", "true") \ .option("startingTimestamp", '2021-04-21 05:45:46') \ .option("endingTimestamp", '2021-05-21 12:00:00') \ .load(tablePath) # providing only the startingVersion/timestamp spark.read.format("deltaSharing") \ .option("readChangeFeed", "true") \ .option("startingVersion", 0) \ .load(tablePath) ``` ```scala val tablePath = "#.." // version as ints or longs spark.read.format("deltaSharing") .option("readChangeFeed", "true") .option("startingVersion", 0) .option("endingVersion", 10) .load(tablePath) // timestamps as formatted timestamp spark.read.format("deltaSharing") .option("readChangeFeed", "true") .option("startingTimestamp", "2024-01-18 05:45:46") .option("endingTimestamp", "2024-01-18 12:00:00") .load(tablePath) // providing only the startingVersion/timestamp spark.read.format("deltaSharing") .option("readChangeFeed", "true") .option("startingVersion", 0) .load(tablePath) ``` ## Streaming Delta Sharing Streaming is deeply integrated with [Spark Structured Streaming](https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html) through `readStream`, and able to connect with any sink that is able to perform `writeStream`. Once the data provider shares a table with history, the recipient can perform a streaming query on the table. When you load a Delta Sharing table as a stream source and use it in a streaming query, the query processes all of the data present in the shared table as well as any new data that arrives after the stream has started. ```scala val tablePath = "#.." spark.readStream.format("deltaSharing").load(tablePath) ``` Delta Sharing Streaming supports the following functionalities in the same way as Delta Streaming: [Limit input rate](/delta-streaming/#limit-input-rate), [Ignore updates and deletes](/delta-streaming/#ignore-updates-and-deletes), [Specify initial position](/delta-streaming/#specify-initial-position) In addition, `maxVersionsPerRpc` is provided to decide how many versions of files are requested from the server in every Delta Sharing rpc. This is to help reduce the per rpc workload and make the Delta sharing streaming job more stable, especially when many new versions have accumulated when the streaming resumes from a checkpoint. The default is 100. ## Read Advanced Delta Lake Features in Delta Sharing In order to support advanced Delta Lake features in Delta Sharing, "Delta Format Sharing" was introduced since delta-sharing-client 1.0 and delta-sharing-spark 3.1, in which the actions of a shared table are returned in Delta Lake format, allowing a Delta Lake library to read it. Please remember to set the spark configurations mentioned in [Configure SparkSession](/delta-batch/#configure-sparksession) in order to read shared tables with Deletion Vectors and Column Mapping. | Read Table Feature | Available since version | | --- | --- | | [Deletion Vectors](/delta-deletion-vectors) | 3.1.0 | | [Column Mapping](/delta-column-mapping) | 3.1.0 | | [Timestamp without Timezone](https://spark.apache.org/docs/latest/sql-ref-datatypes.html) | 3.3.0 | | [Type widening (Preview)](/delta-type-widening) | 3.3.0 | | [Variant Type (Preview)](https://github.com/delta-io/delta/blob/master/protocol_rfcs/variant-type.md) | 3.3.0 | Batch queries can be performed as is, because it can automatically resolve the `responseFormat` based on the table features of the shared table. An additional option `responseFormat=delta` needs to be set for cdf and streaming queries when reading shared tables with Deletion Vectors or Column Mapping enabled. ```scala import org.apache.spark.sql.SparkSession val spark = SparkSession .builder() .appName("...") .master("...") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") .getOrCreate() val tablePath = "#.." // Batch query spark.read.format("deltaSharing").load(tablePath) // CDF query spark.read.format("deltaSharing") .option("readChangeFeed", "true") .option("responseFormat", "delta") .option("startingVersion", 1) .load(tablePath) // Streaming query spark.readStream.format("deltaSharing").option("responseFormat", "delta").load(tablePath) ``` ================================================ FILE: docs/src/content/docs/delta-spark-connect.mdx ================================================ --- title: Delta Connect (aka Spark Connect Support in Delta) description: Learn about Delta Connect - Spark Connect Support in Delta. --- import { Aside, Tabs, TabItem } from "@astrojs/starlight/components"; Delta Connect adds [Spark Connect](https://spark.apache.org/docs/latest/spark-connect-overview.html) support to Delta Lake for Apache Spark. Spark Connect is a new initiative that adds a decoupled client-server infrastructure which allows remote connectivity from Spark from everywhere. Delta Connect allows all Delta Lake operations to work in your application running as a client connected to the Spark server. ## Motivation Delta Connect is expected to br0ng the same benefits as Spark Connect: 1. Upgrading to more recent versions of Spark and Delta Lake is now easier because the client interface is being completely decoupled from the server. 2. Simpler integration of Spark and Delta Lake with developer tooling. IDEs no longer have to integrate with the full Spark and Delta Lake implementation, and instead can integrate with a thin-client. 3. Support for languages other than Java/Scala and Python. Clients "merely" have to generate Protocol Buffers and therefore become simpler to implement. 4. Spark and Delta Lake will become more stable, as user code is no longer running in the same JVM as Spark's driver. 5. Remote connectivity. Code can run anywhere now, as there is a gRPC layer between the user interface and the driver. ## How to start the Spark Server with Delta 1. Download `spark-4.0.0-bin-hadoop3.tgz` from [Spark 4.0.0](https://archive.apache.org/dist/spark/spark-4.0.0). 2. Start the Spark Connect server with the Delta Lake Connect plugins: ```bash sbin/start-connect-server.sh \ --packages io.delta:delta-connect-server_2.13:4.0.0,com.google.protobuf:protobuf-java:3.25.1 \ --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" \ --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" \ --conf "spark.connect.extensions.relation.classes=org.apache.spark.sql.connect.delta.DeltaRelationPlugin" \ --conf "spark.connect.extensions.command.classes=org.apache.spark.sql.connect.delta.DeltaCommandPlugin" ``` ## How to use the Python Spark Connect Client with Delta The Delta Lake Connect Python client is included in the same PyPi package as Delta Lake Spark. 1. `pip install pyspark==4.0.0`. 2. `pip install delta-spark==4.0.0`. 3. The usage is the same as Spark Connect (e.g. `./bin/pyspark --remote "sc://localhost"`). We just need to pass in a remote `SparkSession` (instead of a local one) to the `DeltaTable` API. An example: ```python from delta.tables import DeltaTable from pyspark.sql import SparkSession from pyspark.sql.functions import * deltaTable = DeltaTable.forName(spark, "my_table") deltaTable.toDF().show() deltaTable.update( condition = "id % 2 == 0", set = {"id": "id + 100"} ) ``` ## How to use the Scala Spark Connect Client with Delta Make sure you are using Java 17! ```bash ./bin/spark-shell --remote "sc://localhost" --packages io.delta:delta-connect-client_2.13:4.0.0,com.google.protobuf:protobuf-java:3.25.1 ``` An example: ```scala import io.delta.tables.DeltaTable val deltaTable = DeltaTable.forName(spark, "my_table") deltaTable.toDF.show() deltaTable.updateExpr( condition = "id % 2 == 0", set = Map("id" -> "id + 100") ) ``` ================================================ FILE: docs/src/content/docs/delta-standalone.mdx ================================================ --- title: Delta Standalone (deprecated) description: Learn how to read and write Delta tables from JVM applications without Apache Spark. --- import { Aside } from "@astrojs/starlight/components"; The Delta Standalone library is a single-node Java library that can be used to read from and write to Delta tables. Specifically, this library provides APIs to interact with a table's metadata in the transaction log, implementing the [Delta Transaction Log Protocol](https://github.com/delta-io/delta/blob/master/PROTOCOL.md) to achieve the transactional guarantees of the Delta Lake format. Notably, this project doesn't depend on Apache Spark and has only a few transitive dependencies. Therefore, it can be used by any processing engine or application to access Delta tables. ## Use cases Delta Standalone is optimized for cases when you want to read and write Delta tables by using a non-Spark engine of your choice. It is a "low-level" library, and we encourage developers to contribute open-source, higher-level connectors for their desired engines that use Delta Standalone for all Delta Lake metadata interaction. You can find a Hive source connector and Flink sink/source connector in the [Delta Lake](https://github.com/delta-io/delta) repository. Additional connectors are in development. ### Caveats Delta Standalone minimizes memory usage in the JVM by loading the Delta Lake transaction log incrementally, using an iterator. However, Delta Standalone runs in a single JVM, and is limited to the processing and memory capabilities of that JVM. Users must configure the JVM to avoid out of memory (OOM) issues. Delta Standalone does provide basic APIs for reading Parquet data, but does not include APIs for writing Parquet data. Users must write out new Parquet data files themselves and then use Delta Standalone to commit those changes to the Delta table and make the new data visible to readers. ## APIs Delta Standalone provides classes and entities to read data, query metadata, and commit to the transaction log. A few of them are highlighted here and with their key interfaces. See the [Java API docs](/api/latest/java/standalone/index.html) for the full set of classes and entities. ### DeltaLog [DeltaLog](/api/latest/java/io/delta/standalone/DeltaLog.html) is the main interface for programmatically interacting with the metadata in the transaction log of a Delta table. - Instantiate a `DeltaLog` with `DeltaLog.forTable(hadoopConf, path)` and pass in the `path` of the root location of the Delta table. - Access the current snapshot with `DeltaLog::snapshot`. - Get the latest snapshot, including any new data files that were added to the log, with `DeltaLog::update`. - Get the snapshot at some historical state of the log with `DeltaLog::getSnapshotForTimestampAsOf` or `DeltaLog::getSnapshotForVersionAsOf`. - Start a new transaction to commit to the transaction log by using `DeltaLog::startTransaction`. - Get all metadata actions without computing a full Snapshot using `DeltaLog::getChanges`. ### Snapshot A [Snapshot](/api/latest/java/io/delta/standalone/Snapshot.html) represents the state of the table at a specific version. - Get a list of the metadata files by using `Snapshot::getAllFiles`. - For a memory-optimized iterator over the metadata files, use `Snapshot::scan` to get a `DeltaScan` (as described later), optionally by passing in a `predicate` for partition filtering. - Read actual data with `Snapshot::open`, which returns an iterator over the rows of the Delta table. ### OptimisticTransaction The main class for committing a set of updates to the transaction log is [OptimisticTransaction](/api/latest/java/io/delta/standalone/OptimisticTransaction.html). During a transaction, all reads must go through the `OptimisticTransaction` instance rather than the `DeltaLog` in order to detect logical conflicts and concurrent updates. - Read metadata files during a transaction with `OptimisticTransaction::markFilesAsRead`, which returns a `DeltaScan` of files that match the `readPredicate`. - Commit to the transaction log with `OptimisticTransaction::commit`. - Get the latest version committed for a given application ID (for example, for idempotency) with `OptimisticTransaction::txnVersion`. (Note that this API requires users to commit `SetTransaction` actions.) - Update the medadata of the table upon committing with `OptimisticTransaction::updateMetadata`. ### DeltaScan [DeltaScan](/api/latest/java/io/delta/standalone/DeltaScan.html) is a wrapper class for the files inside a `Snapshot` that match a given `readPredicate`. - Access the files that match the partition filter portion of the `readPredicate` with `DeltaScan::getFiles`. This returns a memory-optimized iterator over the metadata files in the table. - To further filter the returned files on non-partition columns, get the portion of input predicate not applied with `DeltaScan::getResidualPredicate`. ## API compatibility The only public APIs currently provided by Delta Standalone are in the `io.delta.standalone` package. Classes and methods in the `io.delta.standalone.internal` package are considered internal and are subject to change across minor and patch releases. ## Project setup You can add the Delta Standalone library as a dependency by using your preferred build tool. Delta Standalone depends upon the `hadoop-client` and `parquet-hadoop` packages. Example build files are listed in the following sections. ### Environment requirements - JDK 8 or above. - Scala 2.11 or 2.12. ### Build files #### Maven Replace the version of `hadoop-client` with the one you are using. Scala 2.12: ```xml io.delta delta-standalone_2.12 0.5.0 org.apache.hadoop hadoop-client 3.1.0 ``` Scala 2.11: ```xml io.delta delta-standalone_2.11 0.5.0 org.apache.hadoop hadoop-client 3.1.0 ``` #### SBT Replace the version of `hadoop-client` with the one you are using. ``` libraryDependencies ++= Seq( "io.delta" %% "delta-standalone" % "0.5.0", "org.apache.hadoop" % "hadoop-client" % "3.1.0) ``` #### `ParquetSchemaConverter` caveat Delta Standalone shades its own Parquet dependencies so that it works out-of-the-box and reduces dependency conflicts in your environment. However, if you would like to use utility class `io.delta.standalone.util.ParquetSchemaConverter`, then you must provide your own version of `org.apache.parquet:parquet-hadoop`. ### Storage configuration Delta Lake ACID guarantees are based on the atomicity and durability guarantees of the storage system. Not all storage systems provide all the necessary guarantees. Because storage systems do not necessarily provide all of these guarantees out-of-the-box, Delta Lake transactional operations typically go through the [LogStore API](https://github.com/delta-io/delta/blob/master/storage/src/main/java/io/delta/storage/LogStore.java) instead of accessing the storage system directly. To provide the ACID guarantees for different storage systems, you may have to use different `LogStore` implementations. This section covers how to configure Delta Standalone for various storage systems. There are two categories of storage systems: - **Storage systems with built-in support**: For some storage systems, you do not need additional configurations. Delta Standalone uses the scheme of the path (that is, `s3a` in `s3a://path`) to dynamically identify the storage system and use the corresponding `LogStore` implementation that provides the transactional guarantees. However, for S3, there are additional caveats on concurrent writes. See the [section on S3](#amazon-s3-configuration) for details. - **Other storage systems**: The `LogStore`, similar to Apache Spark, uses the Hadoop `FileSystem` API to perform reads and writes. Delta Standalone supports concurrent reads on any storage system that provides an implementation of the `FileSystem` API. For concurrent writes with transactional guarantees, there are two cases based on the guarantees provided by the `FileSystem` implementation. If the implementation provides consistent listing and atomic renames-without-overwrite (that is, `rename(... , overwrite = false)` will either generate the target file atomically or fail if it already exists with `java.nio.file.FileAlreadyExistsException`), then the default `LogStore` implementation using renames will allow concurrent writes with guarantees. Otherwise, you must configure a custom implementation of `LogStore` by setting the following Hadoop configuration when you instantiate a `DeltaLog` with `DeltaLog.forTable(hadoopConf, path)`: ```java delta.logStore..impl= ``` Here, `` is the scheme of the paths of your storage system. This configures Delta Standalone to dynamically use the given `LogStore` implementation only for those paths. You can have multiple such configurations for different schemes in your application, thus allowing it to simultaneously read and write from different storage systems. #### Amazon S3 configuration Delta Standalone supports reads and writes to S3 in two different modes: Single-cluster and Multi-cluster. | | Single-cluster | Multi-cluster | | --- | --- | --- | | Configuration | Comes out-of-the-box | Is experimental and requires extra configuration | | Reads | Supports concurrent reads from multiple clusters | Supports concurrent reads from multiple clusters | | Writes | Supports concurrent writes from a single cluster | Supports multi-cluster writes | | Permissions | S3 credentials | S3 and DynamoDB operating permissions | ##### Single-cluster setup (default) By default, Delta Standalone supports concurrent reads from multiple clusters. However, concurrent writes to S3 must originate from a single cluster to provide transactional guarantees. This is because S3 currently does not provide mutual exclusion, that is, there is no way to ensure that only one writer is able to create a file. To use Delta Standalone with S3, you must meet the following requirements. If you are using access keys for authentication and authorization, you must configure a Hadoop Configuration specified as follows when you instantiate a `DeltaLog` with `DeltaLog.forTable(hadoopConf, path)`. ###### Requirements (S3 single-cluster) - S3 credentials: [IAM roles](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html) (recommended) or access keys. - Hadoop's [AWS connector (hadoop-aws)](https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws) for the version of Hadoop that Delta Standalone is compiled with. ###### Configuration (S3 single-cluster) 1. Include `hadoop-aws` JAR in the classpath. 2. Set up S3 credentials. We recommend that you use [IAM roles](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html) for authentication and authorization. But if you want to use keys, configure your `org.apache.hadoop.conf.Configuration` with: ```java conf.set("fs.s3a.access.key", ""); conf.set("fs.s3a.secret.key", ""); ``` ##### Multi-cluster setup ###### Requirements (S3 multi-cluster) - All of the requirements listed in the [Requirements (S3 single-cluster)](#requirements-s3-single-cluster) section - In additon to S3 credentials, you also need DynamoDB operating permissions ###### Configuration (S3 multi-cluster) 1. Create the DynamoDB table. See [Create the DynamoDB table](/delta-storage#setup-configuration-s3-multi-cluster) for more details on creating a table yourself (recommended) or having it created for you automatically. 2. Follow the configuration steps listed in [Configuration (S3 single-cluster)](#configuration-s3-single-cluster) section. 3. Include the `delta-storage-s3-dynamodb` JAR in the classpath. 4. Configure the `LogStore` implementation. First, configure this `LogStore` implementation for the scheme `s3`. You can replicate this command for schemes `s3a` and `s3n` as well. ```java conf.set("delta.logStore.s3.impl", "io.delta.storage.S3DynamoDBLogStore"); ``` | Configuration Key | Description | Default | | --- | --- | --- | | io.delta.storage.S3DynamoDBLogStore.ddb.tableName | The name of the DynamoDB table to use | delta_log | | io.delta.storage.S3DynamoDBLogStore.ddb.region | The region to be used by the client | us-east-1 | | io.delta.storage.S3DynamoDBLogStore.credentials.provider | The AWSCredentialsProvider\* used by the client | DefaultAWSCredentialsProviderChain | | io.delta.storage.S3DynamoDBLogStore.provisionedThroughput.rcu | (Table-creation-only\*\*) Read Capacity Units | 5 | | io.delta.storage.S3DynamoDBLogStore.provisionedThroughput.wcu | (Table-creation-only\*\*) Write Capacity Units | 5 |
\*For more details on AWS credential providers, see the [AWS documentation](https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html). \*\*These configurations are only used when the given DynamoDB table doesn't already exist and needs to be automatically created. ###### Production Configuration (S3 multi-cluster) By this point, this multi-cluster setup is fully operational. However, there is extra configuration you may do to improve performance and optimize storage when running in production. See the [Delta Lake documentation](/delta-storage#production-configuration-s3-multi-cluster) for more details. #### Microsoft Azure configuration Delta Standalone supports concurrent reads and writes from multiple clusters with full transactional guarantees for various Azure storage systems. To use an Azure storage system, you must satisfy the following requirements, and configure a Hadoop Configuration as specified when you instantiate a `DeltaLog` with `DeltaLog.forTable(hadoopConf, path)`. ##### Azure Blob Storage ###### Requirements (Azure Blob storage) - A [shared key](https://docs.microsoft.com/rest/api/latest/storageservices/authorize-with-shared-key) or [shared access signature (SAS)](https://docs.microsoft.com/azure/storage/common/storage-sas-overview). - Hadoop’s [Azure Blob Storage libraries](https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure) for a version compatible with the Hadoop version Delta Standalone was compiled with. - 2.9.1+ for Hadoop 2 - 3.0.1+ for Hadoop 3 ###### Configuration (Azure Blob storage) 1. Include `hadoop-azure` JAR in the classpath. 2. Set up credentials. - For an SAS token, configure `org.apache.hadoop.conf.Configuration`: ```java conf.set( "fs.azure.sas...blob.core.windows.net", ""); ``` - To specify an account access key: ```java conf.set( "fs.azure.account.key..blob.core.windows.net", ""); ``` ##### Azure Data Lake Storage Gen1 ###### Requirements (ADLS Gen 1) - A [service principal](https://docs.microsoft.com/azure/active-directory/develop/app-objects-and-service-principals) for OAuth 2.0 access. - Hadoop's [Azure Data Lake Storage Gen1 libraries](https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure-datalake) for a version that is compatible with the Hadoop version that was used to compile Delta Standalone. - 2.9.1+ for Hadoop 2 - 3.0.1+ for Hadoop 3 ###### Configuration (ADLS Gen 1) 1. Include `hadoop-azure-datalake` JAR in the classpath. 2. Set up Azure Data Lake Storage Gen1 credentials. Configure `org.apache.hadoop.conf.Configuration`: ```java conf.set("dfs.adls.oauth2.access.token.provider.type", "ClientCredential"); conf.set("dfs.adls.oauth2.client.id", ""); conf.set("dfs.adls.oauth2.credential", ""); conf.set("dfs.adls.oauth2.refresh.url", "https://login.microsoftonline.com//oauth2/token"); ``` ##### Azure Data Lake Storage Gen2 ###### Requirements (ADLS Gen 2) - Account created in [Azure Data Lake Storage Gen2](https://docs.microsoft.com/azure/storage/blobs/create-data-lake-storage-account). - Service principal [created](https://docs.microsoft.com/azure/active-directory/develop/howto-create-service-principal-portal) and [assigned the Storage Blob Data Contributor role](https://docs.microsoft.com/azure/storage/blobs/assign-azure-role-data-access) for the storage account. - Make a note of the storage-account-name, directory-id (also known as tenant-id), application-id, and password of the principal. These will be used for configuration. - Hadoop's [Azure Data Lake Storage Gen2 libraries](https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure-datalake) version 3.2+ and Delta Standalone compiled with Hadoop 3.2+. ###### Configuration (ADLS Gen 2) 1. Include `hadoop-azure-datalake` JAR in the classpath. In addition, you may also have to include JARs for Maven artifacts `hadoop-azure` and `wildfly-openssl`. 2. Set up Azure Data Lake Storage Gen2 credentials. Configure your `org.apache.hadoop.conf.Configuration` with: ```java conf.set("fs.azure.account.auth.type..dfs.core.windows.net", "OAuth"); conf.set("fs.azure.account.oauth.provider.type..dfs.core.windows.net", "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider"); conf.set("fs.azure.account.oauth2.client.id..dfs.core.windows.net", ""); conf.set("fs.azure.account.oauth2.client.secret..dfs.core.windows.net",""); conf.set("fs.azure.account.oauth2.client.endpoint..dfs.core.windows.net", "https://login.microsoftonline.com//oauth2/token"); ``` where ``, ``, `` and `` are details of the service principal we set as requirements earlier. #### HDFS Delta Standalone has built-in support for HDFS with full transactional guarantees on concurrent reads and writes from multiple clusters. See [Hadoop documentation](https://hadoop.apache.org/docs/stable/) for configuring credentials. #### Google Cloud Storage ##### Requirements (GCS) - JAR of the [GCS Connector (gcs-connector)](https://search.maven.org/search?q=a:gcs-connector) Maven artifact. - Google Cloud Storage account and credentials ##### Configuration (GCS) 1. Include the JAR for `gcs-connector` in the classpath. See the [documentation](https://cloud.google.com/dataproc/docs/tutorials/gcs-connector-spark-tutorial) for details on how to configure your project with the GCS connector. ## Usage This example shows how to use Delta Standalone to: - Find parquet files. - Write parquet data. - Commit to the transaction log. - Read from the transaction log. - Read back the Parquet data. Please note that this example uses a fictitious, non-Spark engine `Zappy` to write the actual parquet data, as Delta Standalone does not provide any data-writing APIs. Instead, Delta Standalone Writer lets you commit metadata to the Delta log after you've written your data. This is why Delta Standalone works well with so many connectors (e.g. Flink, Presto, Trino, etc.) since they provide the parquet-writing functionality instead. ### 1. SBT configuration The following SBT project configuration is used: ```scala // /build.sbt scalaVersion := "2.12.8" libraryDependencies ++= Seq( "io.delta" %% "delta-standalone" % "0.5.0", "org.apache.hadoop" % "hadoop-client" % "3.1.0") ``` ### 2. Mock situation We have a Delta table `Sales` storing sales data, but have realized all the data written on November 2021 for customer `XYZ` had incorrect `total_cost` values. Thus, we need to update all those records with the correct values. We will use a fictious distributed engine `Zappy` and Delta Standalone to update our Delta table. The sales table schema is given below. ``` Sales |-- year: int // partition column |-- month: int // partition column |-- day: int // partition column |-- customer: string |-- sale_id: string |-- total_cost: float ``` ### 3. Starting a transaction and finding relevant files Since we must read existing data in order to perform the desired update operation, we must use `OptimisticTransaction::markFilesAsRead` in order to automatically detect any concurrent modifications made to our read partitions. Since Delta Standalone only supports partition pruning, we must apply the residual predicate to further filter the returned files. ```java import io.delta.standalone.DeltaLog; import io.delta.standalone.DeltaScan; import io.delta.standalone.OptimisticTransaction; import io.delta.standalone.actions.AddFile; import io.delta.standalone.data.CloseableIterator; import io.delta.standalone.expressions.And; import io.delta.standalone.expressions.EqualTo; import io.delta.standalone.expressions.Literal; DeltaLog log = DeltaLog.forTable(new Configuration(), "/data/sales"); OptimisticTransaction txn = log.startTransaction(); DeltaScan scan = txn.markFilesAsRead( new And( new And( new EqualTo(schema.column("year"), Literal.of(2021)), // partition filter new EqualTo(schema.column("month"), Literal.of(11))), // partition filter new EqualTo(schema.column("customer"), Literal.of("XYZ")) // non-partition filter ) ); CloseableIterator iter = scan.getFiles(); Map addFileMap = new HashMap(); // partition filtered files: year=2021, month=11 while (iter.hasNext()) { AddFile addFile = iter.next(); addFileMap.put(addFile.getPath(), addFile); } iter.close(); List filteredFiles = ZappyReader.filterFiles( // fully filtered files: year=2021, month=11, customer=XYZ addFileMap.keySet(), toZappyExpression(scan.getResidualPredicate()) ); ``` ### 4. Writing updated Parquet data Since Delta Standalone does not provide any Parquet data write APIs, we use `Zappy` to write the data. ```java ZappyDataFrame correctedSaleIdToTotalCost = ...; ZappyDataFrame invalidSales = ZappyReader.readParquet(filteredFiles); ZappyDataFrame correctedSales = invalidSales.join(correctedSaleIdToTotalCost, "id"); ZappyWriteResult dataWriteResult = ZappyWritter.writeParquet("/data/sales", correctedSales); ``` The written data files from the preceding code will have a hierarchy similar to the following: ```shell $ tree /data/sales . ├── _delta_log │ └── ... │ └── 00000000000000001082.json │ └── 00000000000000001083.json ├── year=2019 │ └── month=1 ... ├── year=2020 │ └── month=1 │ └── day=1 │ └── part-00000-195768ae-bad8-4c53-b0c2-e900e0f3eaee-c000.snappy.parquet // previous │ └── part-00001-53c3c553-f74b-4384-b9b5-7aa45bc2291b-c000.snappy.parquet // new | ... │ └── day=2 │ └── part-00000-b9afbcf5-b90d-4f92-97fd-a2522aa2d4f6-c000.snappy.parquet // previous │ └── part-00001-c0569730-5008-42fa-b6cb-5a152c133fde-c000.snappy.parquet // new | ... ``` ### 5. Committing to our Delta table Now that we've written the correct data, we need to commit to the transaction log to add the new files, and remove the old incorrect files. ```java import io.delta.standalone.Operation; import io.delta.standalone.actions.RemoveFile; import io.delta.standalone.exceptions.DeltaConcurrentModificationException; import io.delta.standalone.types.StructType; List removeOldFiles = filteredFiles.stream() .map(path -> addFileMap.get(path).remove()) .collect(Collectors.toList()); List addNewFiles = dataWriteResult.getNewFiles() .map(file -> new AddFile( file.getPath(), file.getPartitionValues(), file.getSize(), System.currentTimeMillis(), true, // isDataChange null, // stats null // tags ); ).collect(Collectors.toList()); List totalCommitFiles = new ArrayList<>(); totalCommitFiles.addAll(removeOldFiles); totalCommitFiles.addAll(addNewFiles); try { txn.commit(totalCommitFiles, new Operation(Operation.Name.UPDATE), "Zippy/1.0.0"); } catch (DeltaConcurrentModificationException e) { // handle exception here } ``` ### 6. Reading from the Delta table Delta Standalone provides APIs that read both metadata and data, as follows. #### 6.1. Reading Parquet data (distributed) For most use cases, and especially when you deal with large volumes of data, we recommend that you use the Delta Standalone library as your metadata-only reader, and then perform the Parquet data reading yourself, most likely in a distributed manner. Delta Standalone provides two APIs for reading the files in a given table snapshot. `Snapshot::getAllFiles` returns an in-memory list. As of 0.3.0, we also provide `Snapshot::scan(filter)::getFiles`, which supports partition pruning and an optimized internal iterator implementation. We will use the latter here. ```java import io.delta.standalone.Snapshot; DeltaLog log = DeltaLog.forTable(new Configuration(), "/data/sales"); Snapshot latestSnapshot = log.update(); StructType schema = latestSnapshot.getMetadata().getSchema(); DeltaScan scan = latestSnapshot.scan( new And( new And( new EqualTo(schema.column("year"), Literal.of(2021)), new EqualTo(schema.column("month"), Literal.of(11))), new EqualTo(schema.column("customer"), Literal.of("XYZ")) ) ); CloseableIterator iter = scan.getFiles(); try { while (iter.hasNext()) { AddFile addFile = iter.next(); // Zappy engine to handle reading data in `addFile.getPath()` and apply any `scan.getResidualPredicate()` } } finally { iter.close(); } ``` #### 6.2. Reading Parquet data (single-JVM) Delta Standalone allows reading the Parquet data directly, using `Snapshot::open`. ```java import io.delta.standalone.data.RowRecord; CloseableIterator dataIter = log.update().open(); try { while (dataIter.hasNext()) { RowRecord row = dataIter.next(); int year = row.getInt("year"); String customer = row.getString("customer"); float totalCost = row.getFloat("total_cost"); } } finally { dataIter.close(); } ``` ## Reporting issues We use [GitHub Issues](https://github.com/delta-io/connectors/issues) to track community reported issues. You can also [contact](#community) the community for getting answers. ## Contributing We welcome contributions to Delta Lake repository. We use [GitHub Pull Requests](https://github.com/delta-io/delta/pulls) for accepting changes. ## Community There are two ways to communicate with the Delta Lake community: - Public Slack Channel - [Register to join the Slack channel](https://join.slack.com/t/delta-users/shared_invite/enQtNTY1NDg0ODcxOTI1LWJkZGU3ZmQ3MjkzNmY2ZDM0NjNlYjE4MWIzYjg2OWM1OTBmMWIxZTllMjg3ZmJkNjIwZmE1ZTZkMmQ0OTk5ZjA) - [Sign in to the Slack channel](https://delta-users.slack.com/) - Public [mailing list](https://groups.google.com/forum/#!forum/delta-users) ## Local development Before local debugging of `standalone` tests in IntelliJ, run all tests with `build/sbt standalone/test`. This helps IntelliJ recognize the golden tables as class resources. ================================================ FILE: docs/src/content/docs/delta-starburst-integration.mdx ================================================ --- title: Starburst connector description: Learn how to set up an integration to enable you to read Delta tables from Starburst. --- Starburst natively supports reading and writing Delta Lake tables. For details on using the native Delta Lake connector, see [Delta Lake Connector - Starburst](https://docs.starburst.io/latest/connector/delta-lake.html). ================================================ FILE: docs/src/content/docs/delta-storage.mdx ================================================ --- title: Storage configuration description: Learn how to configure Delta Lake on different storage systems. --- import { Tabs, TabItem, Aside } from "@astrojs/starlight/components"; Delta Lake ACID guarantees are predicated on the atomicity and durability guarantees of the storage system. Specifically, Delta Lake relies on the following when interacting with storage systems: - **Atomic visibility**: There must a way for a file to visible in its entirety or not visible at all. - **Mutual exclusion**: Only one writer must be able to create (or rename) a file at the final destination. - **Consistent listing**: Once a file has been written in a directory, all future listings for that directory must return that file. Because storage systems do not necessarily provide all of these guarantees out-of-the-box, Delta Lake transactional operations typically go through the [LogStore API](https://github.com/delta-io/delta/blob/master/storage/src/main/java/io/delta/storage/LogStore.java) instead of accessing the storage system directly. To provide the ACID guarantees for different storage systems, you may have to use different `LogStore` implementations. This article covers how to configure Delta Lake for various storage systems. There are two categories of storage systems: - **Storage systems with built-in support**: For some storage systems, you do not need additional configurations. Delta Lake uses the scheme of the path (that is, `s3a` in `s3a://path`) to dynamically identify the storage system and use the corresponding `LogStore` implementation that provides the transactional guarantees. However, for S3, there are additional caveats on concurrent writes. See the [section on S3](#amazon-s3) for details. - **Other storage systems**: The `LogStore`, similar to Apache Spark, uses Hadoop `FileSystem` API to perform reads and writes. So Delta Lake supports concurrent reads on any storage system that provides an implementation of `FileSystem` API. For concurrent writes with transactional guarantees, there are two cases based on the guarantees provided by `FileSystem` implementation. If the implementation provides consistent listing and atomic renames-without-overwrite (that is, `rename(..., overwrite = false)` will either generate the target file atomically or fail if it already exists with `java.nio.file.FileAlreadyExistsException`), then the default `LogStore` implementation using renames will allow concurrent writes with guarantees. Otherwise, you must configure a custom implementation of `LogStore` by setting the following Spark configuration: ```ini spark.delta.logStore..impl= ``` where `` is the scheme of the paths of your storage system. This configures Delta Lake to dynamically use the given `LogStore` implementation only for those paths. You can have multiple such configurations for different schemes in your application, thus allowing it to simultaneously read and write from different storage systems. ## Troubleshooting Delta Storage dependency error If you see an error like `java.lang.NoClassDefFoundError: io/delta/storage/LogStore` it usually means the **Delta Storage** dependency is missing from the Spark classpath. ##### Error Message with stack trace ```text com.google.common.util.concurrent.ExecutionError: java.lang.NoClassDefFoundError: io/delta/storage/LogStore Please ensure that the delta-storage dependency is included. If using Python, please ensure you call `configure_spark_with_delta_pip` or use `--packages io.delta:delta-spark_:`. See https://docs.delta.io/latest/quick-start.html#python. More information about this dependency and how to include it can be found here: https://docs.delta.io/latest/porting.html#delta-lake-1-1-or-below-to-delta-lake-1-2-or-above. at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2084) at com.google.common.cache.LocalCache.get(LocalCache.java:4017) at com.google.common.cache.LocalCache$LocalManualCache.get(LocalCache.java:4898) at org.apache.spark.sql.delta.DeltaLog$.getDeltaLogFromCache$1(DeltaLog.scala:995) at org.apache.spark.sql.delta.DeltaLog$.initializeDeltaLog$1(DeltaLog.scala:1006) at org.apache.spark.sql.delta.DeltaLog$.apply(DeltaLog.scala:1017) at org.apache.spark.sql.delta.DeltaLog$.forTable(DeltaLog.scala:801) at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:197) at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55) at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79) at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77) at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88) at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:155) at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:162) at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:268) at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:124) at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94) at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112) at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:106) at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111) at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:124) at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:291) at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:123) at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804) at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:77) at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:233) at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:155) at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:654) at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:154) at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:169) at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:164) at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:470) at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86) at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:470) at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37) at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360) at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356) at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37) at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37) at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:446) at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:164) at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:126) at scala.util.Try$.apply(Try.scala:217) at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1378) at org.apache.spark.util.Utils$.getTryWithCallerStacktrace(Utils.scala:1439) at org.apache.spark.util.LazyTry.get(LazyTry.scala:58) at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:131) at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:192) at org.apache.spark.sql.classic.DataFrameWriter.runCommand(DataFrameWriter.scala:622) at org.apache.spark.sql.classic.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:273) at org.apache.spark.sql.classic.DataFrameWriter.saveInternal(DataFrameWriter.scala:235) at org.apache.spark.sql.classic.DataFrameWriter.save(DataFrameWriter.scala:118) ... 42 elided ``` ### Why this happens When you provide the Delta Spark JAR using the `--jars` option (for example, when testing a locally-built JAR), Spark **does not automatically fetch transitive dependencies**. In this case, `delta-spark` may be present, but `delta-storage` is not. As a result, operations that need to initialize the Delta log (for example, writing a Delta table) can fail when Delta attempts to load the LogStore API: ```scala df.write.format("delta").save("/tmp/delta") ``` ### How to fix When using the `--jars` option, you can do either of the following: - Include both `delta-spark` and `delta-storage` JARs: ```bash spark-shell \ --jars delta-spark_-.jar,delta-storage-.jar \ --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" \ --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" ``` - Use the assembly JAR (build it using `build/sbt "spark/assembly"`): ```bash spark-shell \ --jars delta-spark-assembly-.jar \ --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" \ --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" ``` ## Amazon S3 Delta Lake supports reads and writes to S3 in two different modes: Single-cluster and Multi-cluster. | | Single-cluster | Multi-cluster | | --- | --- | --- | | Configuration | Comes with Delta Lake out-of-the-box | Is experimental and requires extra configuration | | Reads | Supports concurrent reads from multiple clusters | Supports concurrent reads from multiple clusters | | Writes | Supports concurrent writes from a _single_ Spark driver | Supports multi-cluster writes | | Permissions | S3 credentials | S3 and DynamoDB operating permissions | ### Single-cluster setup (default) In this default mode, Delta Lake supports concurrent reads from multiple clusters, but concurrent writes to S3 must originate from a _single_ Spark driver in order for Delta Lake to provide transactional guarantees. This is because S3 currently does not provide mutual exclusion, that is, there is no way to ensure that only one writer is able to create a file. #### Requirements (S3 single-cluster) - S3 credentials: [IAM roles](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html) (recommended) or access keys - Apache Spark associated with the corresponding Delta Lake version. - Hadoop's [AWS connector (hadoop-aws)](https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/) for the version of Hadoop that Apache Spark is compiled for. #### Quickstart (S3 single-cluster) This section explains how to quickly start reading and writing Delta tables on S3 using single-cluster mode. For a detailed explanation of the configuration, see [Setup Configuration (S3 multi-cluster)](#setup-configuration-s3-multi-cluster). 1. Use the following command to launch a Spark shell with Delta Lake and S3 support (assuming you use Spark 4.0.0 which is pre-built for Hadoop 3.4.0): ```bash bin/spark-shell \ --packages io.delta:delta-spark_2.13:4.0.0,org.apache.hadoop:hadoop-aws:3.4.0 \ --conf spark.hadoop.fs.s3a.access.key= \ --conf spark.hadoop.fs.s3a.secret.key= ``` 2. Try out some basic Delta table operations on S3 (in Scala): ```scala // Create a Delta table on S3: spark.range(5).write.format("delta").save("s3a:///") // Read a Delta table on S3: spark.read.format("delta").load("s3a:///").show() ``` For other languages and more examples of Delta table operations, see the [Quickstart](/quick-start/) page. For efficient listing of Delta Lake metadata files on S3, set the configuration `delta.enableFastS3AListFrom=true`. This performance optimization is in experimental support mode. It will only work on `S3A` filesystems and will not work on [Amazon's EMR](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-plan-file-systems.html) default filesystem `S3`. ```scala bin/spark-shell \ --packages io.delta:delta-spark_2.13:4.0.0,org.apache.hadoop:hadoop-aws:3.4.0 \ --conf spark.hadoop.fs.s3a.access.key= \ --conf spark.hadoop.fs.s3a.secret.key= \ --conf "spark.hadoop.delta.enableFastS3AListFrom=true ``` ### Multi-cluster setup This mode supports concurrent writes to S3 from multiple clusters and has to be explicitly enabled by configuring Delta Lake to use the right `LogStore` implementation. This implementation uses [DynamoDB](https://aws.amazon.com/dynamodb/) to provide the mutual exclusion that S3 is lacking. #### Requirements (S3 multi-cluster) - All of the requirements listed in [Requirements (S3 single-cluster)](#requirements-s3-single-cluster) section - In additon to S3 credentials, you also need DynamoDB operating permissions #### Quickstart (S3 multi-cluster) This section explains how to quickly start reading and writing Delta tables on S3 using multi-cluster mode. 1. Use the following command to launch a Spark shell with Delta Lake and S3 support (assuming you use Spark 4.0.0 which is pre-built for Hadoop 3.4.0): ```bash bin/spark-shell \ --packages io.delta:delta-spark_2.13:3,org.apache.hadoop:hadoop-aws:3.4.0,io.delta:delta-storage-s3-dynamodb:4.0.0 \ --conf spark.hadoop.fs.s3a.access.key= \ --conf spark.hadoop.fs.s3a.secret.key= \ --conf spark.delta.logStore.s3a.impl=io.delta.storage.S3DynamoDBLogStore \ --conf spark.io.delta.storage.S3DynamoDBLogStore.ddb.region=us-west-2 ``` 2. Try out some basic Delta table operations on S3 (in Scala): ```scala // Create a Delta table on S3: spark.range(5).write.format("delta").save("s3a:///") // Read a Delta table on S3: spark.read.format("delta").load("s3a:///").show() ``` For other languages and more examples of Delta table operations, see the [Quickstart](/quick-start/) page. #### Setup Configuration (S3 multi-cluster) 1. Create the DynamoDB table. You have the choice of creating the DynamoDB table yourself (recommended) or having it created for you automatically. - Creating the DynamoDB table yourself This DynamoDB table will maintain commit metadata for multiple Delta tables, and it is important that it is configured with the [Read/Write Capacity Mode](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.ReadWriteCapacityMode.html) (for example, on-demand or provisioned) that is right for your use cases. As such, we strongly recommend that you create your DynamoDB table yourself. The following example uses the AWS CLI. To learn more, see the [create-table](https://docs.aws.amazon.com/cli/latest/reference/dynamodb/create-table.html) command reference. ```bash aws dynamodb create-table \ --region us-east-1 \ --table-name delta_log \ --attribute-definitions AttributeName=tablePath,AttributeType=S \ AttributeName=fileName,AttributeType=S \ --key-schema AttributeName=tablePath,KeyType=HASH \ AttributeName=fileName,KeyType=RANGE \ --billing-mode PAY_PER_REQUEST ``` 2. Follow the configuration steps listed in [Configuration (S3 single-cluster)](#configuration-s3-single-cluster) section. 3. Include the `delta-storage-s3-dynamodb` JAR in the classpath. 4. Configure the `LogStore` implementation in your Spark session. First, configure this `LogStore` implementation for the scheme `s3`. You can replicate this command for schemes `s3a` and `s3n` as well. ```ini spark.delta.logStore.s3.impl=io.delta.storage.S3DynamoDBLogStore ``` Next, specify additional information necessary to instantiate the DynamoDB client. You must instantiate the DynamoDB client with the same `tableName` and `region` each Spark session for this multi-cluster mode to work correctly. A list of per-session configurations and their defaults is given below: ```ini spark.io.delta.storage.S3DynamoDBLogStore.ddb.tableName=delta_log spark.io.delta.storage.S3DynamoDBLogStore.ddb.region=us-east-1 spark.io.delta.storage.S3DynamoDBLogStore.credentials.provider= spark.io.delta.storage.S3DynamoDBLogStore.provisionedThroughput.rcu=5 spark.io.delta.storage.S3DynamoDBLogStore.provisionedThroughput.wcu=5 ``` #### Production Configuration (S3 multi-cluster) By this point, this multi-cluster setup is fully operational. However, there is extra configuration you may do to improve performance and optimize storage when running in production. 1. Adjust your Read and Write Capacity Mode. If you are using the default DynamoDB table created for you by this `LogStore` implementation, its default RCU and WCU might not be enough for your workloads. You can [adjust the provisioned throughput](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ProvisionedThroughput.html#ProvisionedThroughput.CapacityUnits.Modifying) or [update to On-Demand Mode](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/WorkingWithTables.Basics.html#WorkingWithTables.Basics.UpdateTable). 2. Cleanup old DynamoDB entries using Time to Live (TTL). Once a DynamoDB metadata entry is marked as complete, and after sufficient time such that we can now rely on S3 alone to prevent accidental overwrites on its corresponding Delta file, it is safe to delete that entry from DynamoDB. The cheapest way to do this is using [DynamoDB's TTL](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/TTL.html) feature which is a free, automated means to delete items from your DynamoDB table. Run the following command on your given DynamoDB table to enable TTL: ```bash aws dynamodb update-time-to-live \ --region us-east-1 \ --table-name delta_log \ --time-to-live-specification "Enabled=true, AttributeName=expireTime" ``` The default `expireTime` will be one day after the DynamoDB entry was marked as completed. 3. Cleanup old AWS S3 temp files using S3 Lifecycle Expiration. In this `LogStore` implementation, a temp file is created containing a copy of the metadata to be committed into the Delta log. Once that commit to the Delta log is complete, and after the corresponding DynamoDB entry has been removed, it is safe to delete this temp file. In practice, only the latest temp file will ever be used during recovery of a failed commit. Here are two simple options for deleting these temp files: 1. Delete manually using S3 CLI. This is the safest option. The following command will delete all but the latest temp file in your given `` and `
`: ```bash aws s3 ls s3:////_delta_log/.tmp/ --recursive | awk 'NF>1{print $4}' | grep . | sort | head -n -1 | while read -r line ; do echo "Removing ${line}" aws s3 rm s3:////_delta_log/.tmp/${line} done ``` 2. Delete using an S3 Lifecycle Expiration Rule A more automated option is to use an [S3 Lifecycle Expiration rule](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lifecycle-mgmt.html), with filter prefix pointing to the `/_delta_log/.tmp/` folder located in your table path, and an expiration value of 30 days. There are a variety of ways to configuring a bucket lifecycle configuration, described in AWS docs [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/how-to-set-lifecycle-configuration-intro.html). One way to do this is using S3's `put-bucket-lifecycle-configuration` command. See [S3 Lifecycle Configuration](https://docs.aws.amazon.com/cli/latest/reference/s3api/put-bucket-lifecycle-configuration.html) for details. An example rule and command invocation is given below: In a file referenced as `file://lifecycle.json`: ```json { "Rules":[ { "ID":"expire_tmp_files", "Filter":{ "Prefix":"path/to/table/_delta_log/.tmp/" }, "Status":"Enabled", "Expiration":{ "Days":30 } } ] } ``` ```bash aws s3api put-bucket-lifecycle-configuration \ --bucket my-bucket \ --lifecycle-configuration file://lifecycle.json ``` ## Microsoft Azure storage Delta Lake has built-in support for the various Azure storage systems with full transactional guarantees for concurrent reads and writes from multiple clusters. Delta Lake relies on Hadoop `FileSystem` APIs to access Azure storage services. Specifically, Delta Lake requires the implementation of `FileSystem.rename()` to be atomic, which is only supported in newer Hadoop versions ([Hadoop-15156](https://issues.apache.org/jira/browse/HADOOP-15156) and [Hadoop-15086](https://issues.apache.org/jira/browse/HADOOP-15086)). For this reason, you may need to build Spark with newer Hadoop versions and use them for deploying your application. See [Specifying the Hadoop Version and Enabling YARN](https://spark.apache.org/docs/latest/building-spark.html#specifying-the-hadoop-version-and-enabling-yarn) for building Spark with a specific Hadoop version and [Quickstart](/quick-start/) for setting up Spark with Delta Lake. Here is a list of requirements specific to each type of Azure storage system: ### Azure Blob storage #### Requirements (Azure Blob storage) - A [shared key](https://docs.microsoft.com/rest/api/latest/storageservices/authorize-with-shared-key) or [shared access signature (SAS)](https://docs.microsoft.com/azure/storage/common/storage-dotnet-shared-access-signature-part-1) - Delta Lake 0.2.0 or above - Hadoop's Azure Blob Storage libraries for deployment with the following versions: - 2.9.1+ for Hadoop 2 - 3.0.1+ for Hadoop 3 - Apache Spark associated with the corresponding Delta Lake version and [compiled with Hadoop version](https://spark.apache.org/docs/latest/building-spark.html#specifying-the-hadoop-version-and-enabling-yarn) that is compatible with the chosen Hadoop libraries. For example, a possible combination that will work is Delta 0.7.0 or above, along with Apache Spark 3.0 compiled and deployed with Hadoop 3.2. #### Configuration (Azure Blob storage) Here are the steps to configure Delta Lake on Azure Blob storage. 1. Include `hadoop-azure` JAR in the classpath. See the requirements above for version details. 2. Set up credentials. You can set up your credentials in the [Spark configuration property](https://spark.apache.org/docs/latest/configuration.html). We recommend that you use a SAS token. In Scala, you can use the following: ```scala spark.conf.set( "fs.azure.sas...blob.core.windows.net", "") ``` Or you can specify an account access key: ```scala spark.conf.set( "fs.azure.account.key..blob.core.windows.net", "") ``` #### Usage (Azure Blob storage) ```scala spark.range(5).write.format("delta").save("wasbs://@.blob.core.windows.net/") spark.read.format("delta").load("wasbs://@.blob.core.windows.net/").show() ``` ### Azure Data Lake Storage Gen1 #### Requirements (ADLS Gen1) - A [service principal](https://docs.microsoft.com/azure/active-directory/develop/app-objects-and-service-principals) for OAuth 2.0 access - Delta Lake 0.2.0 or above - Hadoop's Azure Data Lake Storage Gen1 libraries for deployment with the following versions: - 2.9.1+ for Hadoop 2 - 3.0.1+ for Hadoop 3 - Apache Spark associated with the corresponding Delta Lake version and [compiled with Hadoop version](https://spark.apache.org/docs/latest/building-spark.html#specifying-the-hadoop-version-and-enabling-yarn) that is compatible with the chosen Hadoop libraries. #### Configuration (ADLS Gen1) Here are the steps to configure Delta Lake on Azure Data Lake Storage Gen1. 1. Include `hadoop-azure-datalake` JAR in the classpath. See the requirements above for version details. 2. Set up Azure Data Lake Storage Gen1 credentials. You can set the following [Hadoop configurations](https://spark.apache.org/docs/latest/configuration.html#custom-hadoophive-configuration) with your credentials (in Scala): ```scala spark.conf.set("dfs.adls.oauth2.access.token.provider.type", "ClientCredential") spark.conf.set("dfs.adls.oauth2.client.id", "") spark.conf.set("dfs.adls.oauth2.credential", "") spark.conf.set("dfs.adls.oauth2.refresh.url", "https://login.microsoftonline.com//oauth2/token") ``` #### Usage (ADLS Gen1) ```scala spark.range(5).write.format("delta").save("adl://.azuredatalakestore.net/") spark.read.format("delta").load("adl://.azuredatalakestore.net/").show() ``` ### Azure Data Lake Storage Gen2 #### Requirements (ADLS Gen2) - A [service principal](https://docs.microsoft.com/azure/active-directory/develop/app-objects-and-service-principals) for OAuth 2.0 access or a [shared key](https://docs.microsoft.com/rest/api/latest/storageservices/authorize-with-shared-key) - Delta Lake 0.2.0 or above - Hadoop's Azure Data Lake Storage Gen2 libraries for deployment with the following versions: - 3.2.0+ for Hadoop 3 - Apache Spark associated with the corresponding Delta Lake version and [compiled with Hadoop version](https://spark.apache.org/docs/latest/building-spark.html#specifying-the-hadoop-version-and-enabling-yarn) that is compatible with the chosen Hadoop libraries. #### Configuration (ADLS Gen2) Here are the steps to configure Delta Lake on Azure Data Lake Storage Gen2. 1. Include `hadoop-azure` and `azure-storage` JARs in the classpath. See the requirements above for version details. 2. Set up credentials. You can use either OAuth 2.0 with service principal or shared key authentication: For OAuth 2.0 with service principal (recommended): ```scala spark.conf.set("fs.azure.account.auth.type..dfs.core.windows.net", "OAuth") spark.conf.set("fs.azure.account.oauth.provider.type..dfs.core.windows.net", "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider") spark.conf.set("fs.azure.account.oauth2.client.id..dfs.core.windows.net", "") spark.conf.set("fs.azure.account.oauth2.client.secret..dfs.core.windows.net", "") spark.conf.set("fs.azure.account.oauth2.client.endpoint..dfs.core.windows.net", "https://login.microsoftonline.com//oauth2/token") ``` For shared key authentication: ```scala spark.conf.set("fs.azure.account.key..dfs.core.windows.net", "") ``` #### Usage (ADLS Gen2) ```scala spark.range(5).write.format("delta").save("abfss://@.dfs.core.windows.net/") spark.read.format("delta").load("abfss://@.dfs.core.windows.net/").show() ``` ## HDFS Delta Lake has built-in support for HDFS with full transactional guarantees for concurrent reads and writes from multiple clusters. No additional configuration is required. #### Usage (HDFS) ```scala spark.range(5).write.format("delta").save("hdfs://:/") spark.read.format("delta").load("hdfs://:/").show() ``` ## Google Cloud Storage Delta Lake has built-in support for Google Cloud Storage (GCS) with full transactional guarantees for concurrent reads and writes from multiple clusters. ### Requirements (GCS) - Google Cloud Storage credentials - Delta Lake 0.2.0 or above - Hadoop's [GCS connector](https://cloud.google.com/dataproc/docs/concepts/connectors/cloud-storage) for the version of Hadoop that Apache Spark is compiled for ### Configuration (GCS) 1. Include the GCS connector JAR in the classpath. 2. Set up credentials using one of the following methods: - Use [Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials) - Configure service account credentials in Spark configuration ```scala spark.conf.set("google.cloud.auth.service.account.json.keyfile", "") ``` ### Usage (GCS) ```scala spark.range(5).write.format("delta").save("gs:///") spark.read.format("delta").load("gs:///").show() ``` ## Oracle Cloud Infrastructure Delta Lake supports Oracle Cloud Infrastructure (OCI) Object Storage with full transactional guarantees for concurrent reads and writes from multiple clusters. ### Requirements (OCI) - OCI credentials - Delta Lake 0.2.0 or above - Hadoop's [OCI connector](https://docs.oracle.com/en-us/iaas/Content/api/latest/SDKDocs/hdfsconnector.htm) for the version of Hadoop that Apache Spark is compiled for ### Configuration (OCI) 1. Include the OCI connector JAR in the classpath. 2. Set up credentials in Spark configuration: ```scala spark.conf.set("fs.oci.client.auth.tenantId", "") spark.conf.set("fs.oci.client.auth.userId", "") spark.conf.set("fs.oci.client.auth.fingerprint", "") spark.conf.set("fs.oci.client.auth.pemfilepath", "") ``` ### Usage (OCI) ```scala spark.range(5).write.format("delta").save("oci://@/") spark.read.format("delta").load("oci://@/").show() ``` ## IBM Cloud Object Storage Delta Lake supports IBM Cloud Object Storage with full transactional guarantees for concurrent reads and writes from multiple clusters. ### Requirements (IBM COS) - IBM Cloud Object Storage credentials - Delta Lake 0.2.0 or above - Hadoop's [Stocator connector](https://github.com/CODAIT/stocator) for the version of Hadoop that Apache Spark is compiled for ### Configuration (IBM COS) 1. Include the Stocator connector JAR in the classpath. 2. Set up credentials in Spark configuration: ```scala spark.conf.set("fs.cos.service.endpoint", "") spark.conf.set("fs.cos.service.access.key", "") spark.conf.set("fs.cos.service.secret.key", "") ``` ### Usage (IBM COS) ```scala spark.range(5).write.format("delta").save("cos://./") spark.read.format("delta").load("cos://./").show() ``` ================================================ FILE: docs/src/content/docs/delta-streaming/index.mdx ================================================ --- title: Table streaming reads and writes description: Learn how to use Delta tables as streaming sources and sinks. --- import { Aside, Tabs, TabItem } from "@astrojs/starlight/components"; Delta Lake is deeply integrated with [Spark Structured Streaming](https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html) through `readStream` and `writeStream`. Delta Lake overcomes many of the limitations typically associated with streaming systems and files, including: - Maintaining "exactly-once" processing with more than one stream (or concurrent batch jobs) - Efficiently discovering which files are new when using files as the source for a stream For many Delta Lake operations on tables, you enable integration with Apache Spark DataSourceV2 and Catalog APIs (since 3.0) by setting configurations when you create a new `SparkSession`. See [Configure SparkSession](/delta-batch/#configure-sparksession). ## Delta table as a source When you load a Delta table as a stream source and use it in a streaming query, the query processes all of the data present in the table as well as any new data that arrives after the stream is started. ```scala spark.readStream.format("delta") .load("/tmp/delta/events") import io.delta.implicits._ spark.readStream.delta("/tmp/delta/events") ``` ### Limit input rate The following options are available to control micro-batches: - `maxFilesPerTrigger`: How many new files to be considered in every micro-batch. The default is 1000. - `maxBytesPerTrigger`: How much data gets processed in each micro-batch. This option sets a "soft max", meaning that a batch processes approximately this amount of data and may process more than the limit in order to make the streaming query move forward in cases when the smallest input unit is larger than this limit. If you use `Trigger.Once` for your streaming, this option is ignored. This is not set by default. If you use `maxBytesPerTrigger` in conjunction with `maxFilesPerTrigger`, the micro-batch processes data until either the `maxFilesPerTrigger` or `maxBytesPerTrigger` limit is reached. ### Ignore updates and deletes Structured Streaming does not handle input that is not an append and throws an exception if any modifications occur on the table being used as a source. There are two main strategies for dealing with changes that cannot be automatically propagated downstream: - You can delete the output and checkpoint and restart the stream from the beginning. - You can set either of these two options: - `ignoreDeletes`: ignore transactions that delete data at partition boundaries. - `ignoreChanges`: re-process updates if files had to be rewritten in the source table due to a data changing operation such as `UPDATE`, `MERGE INTO`, `DELETE` (within partitions), or `OVERWRITE`. Unchanged rows may still be emitted, therefore your downstream consumers should be able to handle duplicates. Deletes are not propagated downstream. `ignoreChanges` subsumes `ignoreDeletes`. Therefore if you use `ignoreChanges`, your stream will not be disrupted by either deletions or updates to the source table. #### Example For example, suppose you have a table `user_events` with `date`, `user_email`, and `action` columns that is partitioned by `date`. You stream out of the `user_events` table and you need to delete data from it due to GDPR. When you delete at partition boundaries (that is, the `WHERE` is on a partition column), the files are already segmented by value so the delete just drops those files from the metadata. Thus, if you just want to delete data from some partitions, you can use: ```scala spark.readStream.format("delta") .option("ignoreDeletes", "true") .load("/tmp/delta/user_events") ``` However, if you have to delete data based on `user_email`, then you will need to use: ```scala spark.readStream.format("delta") .option("ignoreChanges", "true") .load("/tmp/delta/user_events") ``` If you update a `user_email` with the `UPDATE` statement, the file containing the `user_email` in question is rewritten. When you use `ignoreChanges`, the new record is propagated downstream with all other unchanged records that were in the same file. Your logic should be able to handle these incoming duplicate records. ### Specify initial position You can use the following options to specify the starting point of the Delta Lake streaming source without processing the entire table. - `startingVersion`: The Delta Lake version to start from. All table changes starting from this version (inclusive) will be read by the streaming source. You can obtain the commit versions from the `version` column of the [DESCRIBE HISTORY](/delta-utility/#retrieve-delta-table-history) command output. - To return only the latest changes, specify `latest`. - `startingTimestamp`: The timestamp to start from. All table changes committed at or after the timestamp (inclusive) will be read by the streaming source. One of: - A timestamp string. For example, `"2019-01-01T00:00:00.000Z"`. - A date string. For example, `"2019-01-01"`. You cannot set both options at the same time; you can use only one of them. They take effect only when starting a new streaming query. If a streaming query has started and the progress has been recorded in its checkpoint, these options are ignored. #### Example For example, suppose you have a table `user_events`. If you want to read changes since version 5, use: ```scala spark.readStream.format("delta") .option("startingVersion", "5") .load("/tmp/delta/user_events") ``` If you want to read changes since 2018-10-18, use: ```scala spark.readStream.format("delta") .option("startingTimestamp", "2018-10-18") .load("/tmp/delta/user_events") ``` ### Process initial snapshot without data being dropped When using a Delta table as a stream source, the query first processes all of the data present in the table. The Delta table at this version is called the initial snapshot. By default, the Delta table's data files are processed based on which file was last modified. However, the last modification time does not necessarily represent the record event time order. In a stateful streaming query with a defined watermark, processing files by modification time can result in records being processed in the wrong order. This could lead to records dropping as late events by the watermark. You can avoid the data drop issue by enabling the following option: - withEventTimeOrder: Whether the initial snapshot should be processed with event time order. With event time order enabled, the event time range of initial snapshot data is divided into time buckets. Each micro batch processes a bucket by filtering data within the time range. The maxFilesPerTrigger and maxBytesPerTrigger configuration options are still applicable to control the microbatch size but only in an approximate way due to the nature of the processing. The graphic below shows this process: ![Initial Snapshot](./delta-initial-snapshot-data-drop.png) Notable information about this feature: - The data drop issue only happens when the initial Delta snapshot of a stateful streaming query is processed in the default order. - You cannot change `withEventTimeOrder` once the stream query is started while the initial snapshot is still being processed. To restart with `withEventTimeOrder` changed, you need to delete the checkpoint. - If you are running a stream query with withEventTimeOrder enabled, you cannot downgrade it to a Delta version which doesn't support this feature until the initial snapshot processing is completed. If you need to downgrade, you can wait for the initial snapshot to finish, or delete the checkpoint and restart the query. - This feature is not supported in the following uncommon scenarios: - The event time column is a generated column and there are non-projection transformations between the Delta source and watermark. - There is a watermark that has more than one Delta source in the stream query. - With event time order enabled, the performance of the Delta initial snapshot processing might be slower. - Each micro batch scans the initial snapshot to filter data within the corresponding event time range. For faster filter action, it is advised to use a Delta source column as the event time so that data skipping can be applied (check \_ for when it's applicable). Additionally, table partitioning along the event time column can further speed the processing. You can check Spark UI to see how many delta files are scanned for a specific micro batch. #### Example Suppose you have a table `user_events` with an `event_time` column. Your streaming query is an aggregation query. If you want to ensure no data drop during the initial snapshot processing, you can use: ```scala spark.readStream.format("delta") .option("withEventTimeOrder", "true") .load("/tmp/delta/user_events") .withWatermark("event_time", "10 seconds") ``` ### Tracking non-additive schema changes You can provide a schema tracking location to enable streaming from Delta tables with column mapping enabled. This overcomes an issue in which non-additive schema changes could result in broken streams by allowing streams to read past table data in their exact schema as if the table is time-travelled. Each streaming read against a data source must have its own `schemaTrackingLocation` specified. The specified `schemaTrackingLocation` must be contained within the directory specified for the `checkpointLocation` of the target table for streaming write. #### Example The option `schemaTrackingLocation` is used to specify the path for schema tracking, as shown in the following code example: ```python checkpoint_path = "/path/to/checkpointLocation" (spark.readStream .option("schemaTrackingLocation", checkpoint_path) .table("delta_source_table") .writeStream .option("checkpointLocation", checkpoint_path) .toTable("output_table") ) ``` ## Delta table as a sink You can also write data into a Delta table using Structured Streaming. The transaction log enables Delta Lake to guarantee exactly-once processing, even when there are other streams or batch queries running concurrently against the table. ### Append mode By default, streams run in append mode, which adds new records to the table. You can use the path method: ```python events.writeStream .format("delta") .outputMode("append") .option("checkpointLocation", "/tmp/delta/_checkpoints/") .start("/delta/events") ``` ```scala events.writeStream .format("delta") .outputMode("append") .option("checkpointLocation", "/tmp/delta/events/_checkpoints/") .start("/tmp/delta/events") import io.delta.implicits._ events.writeStream .outputMode("append") .option("checkpointLocation", "/tmp/delta/events/_checkpoints/") .delta("/tmp/delta/events") ``` or the `toTable` method (in Spark 3.1 and higher) as follows: ```python events.writeStream .format("delta") .outputMode("append") .option("checkpointLocation", "/tmp/delta/events/_checkpoints/") .toTable("events") ``` ```scala events.writeStream .outputMode("append") .option("checkpointLocation", "/tmp/delta/events/_checkpoints/") .toTable("events") ``` ### Complete mode You can also use Structured Streaming to replace the entire table with every batch. One example use case is to compute a summary using aggregation: ```python (spark.readStream .format("delta") .load("/tmp/delta/events") .groupBy("customerId") .count() .writeStream .format("delta") .outputMode("complete") .option("checkpointLocation", "/tmp/delta/eventsByCustomer/_checkpoints/") .start("/tmp/delta/eventsByCustomer") ) ``` ```scala spark.readStream .format("delta") .load("/tmp/delta/events") .groupBy("customerId") .count() .writeStream .format("delta") .outputMode("complete") .option("checkpointLocation", "/tmp/delta/eventsByCustomer/_checkpoints/") .start("/tmp/delta/eventsByCustomer") ``` The preceding example continuously updates a table that contains the aggregate number of events by customer. For applications with more lenient latency requirements, you can save computing resources with one-time triggers. Use these to update summary aggregation tables on a given schedule, processing only new data that has arrived since the last update. ## Idempotent table writes in `foreachBatch` The command foreachBatch allows you to specify a function that is executed on the output of every micro-batch after arbitrary transformations in the streaming query. This allows implementating a `foreachBatch` function that can write the micro-batch output to one or more target Delta table destinations. However, `foreachBatch` does not make those writes idempotent as those write attempts lack the information of whether the batch is being re-executed or not. For example, rerunning a failed batch could result in duplicate data writes. To address this, Delta tables support the following `DataFrameWriter` options to make the writes idempotent: - `txnAppId`: A unique string that you can pass on each `DataFrame` write. For example, you can use the StreamingQuery ID as `txnAppId`. - `txnVersion`: A monotonically increasing number that acts as transaction version. Delta table uses the combination of `txnAppId` and `txnVersion` to identify duplicate writes and ignore them. If a batch write is interrupted with a failure, rerunning the batch uses the same application and batch ID, which would help the runtime correctly identify duplicate writes and ignore them. Application ID (`txnAppId`) can be any user-generated unique string and does not have to be related to the stream ID. The same `DataFrameWriter` options can be used to achieve the idempotent writes in non-Streaming job. For details [Idempotent writes](/delta-batch/#idempotent-writes). ### Example ```python app_id = ... # A unique string that is used as an application ID. def writeToDeltaLakeTableIdempotent(batch_df, batch_id): batch_df.write.format(...).option("txnVersion", batch_id).option("txnAppId", app_id).save(...) # location 1 batch_df.write.format(...).option("txnVersion", batch_id).option("txnAppId", app_id).save(...) # location 2 ``` ```scala val appId = ... // A unique string that is used as an application ID. streamingDF.writeStream.foreachBatch { (batchDF: DataFrame, batchId: Long) => batchDF.write.format(...).option("txnVersion", batchId).option("txnAppId", appId).save(...) // location 1 batchDF.write.format(...).option("txnVersion", batchId).option("txnAppId", appId).save(...) // location 2 } ``` ================================================ FILE: docs/src/content/docs/delta-trino-integration.mdx ================================================ --- title: Trino connector description: Learn how to set up an integration to enable you to read Delta tables from Trino. --- Since Trino [version 373](https://trino.io/docs/current/release/release-373.html), Trino natively supports reading and writing the Delta Lake tables. For details on using the native Delta Lake connector, see [Delta Lake Connector - Trino](https://trino.io/docs/current/connector/delta-lake.html). For Trino versions lower than [version 373](https://trino.io/docs/current/release/release-373.html), you can use the manifest-based approach detailed in [Presto, Trino, and Athena to Delta Lake integration using manifests](/presto-integration/). ================================================ FILE: docs/src/content/docs/delta-type-widening.mdx ================================================ --- title: Delta type widening description: Learn about type widening in Delta. --- import { Tabs, TabItem, Aside } from "@astrojs/starlight/components"; The type widening feature allows changing the type of columns in a Delta table to a wider type. This enables manual type changes using the `ALTER TABLE ALTER COLUMN` command and automatic type migration with schema evolution during write operations. ## Supported type changes The feature introduces a limited set of supported type changes in Delta Lake 3.2 and expands it in Delta Lake 4.0 and above. | Source type | Supported wider types - Delta 3.2 | Supported wider types - Delta 4.0 | |-------------|-----------------------------------|---------------------------------------------| | `byte` | `short`, `int` | `short`, `int`, `long`, `decimal`, `double` | | `short` | `int` | `int`, `long`, `decimal`, `double` | | `int` | | `long`, `decimal`, `double` | | `long` | | `decimal` | | `float` | | `double` | | `decimal` | | `decimal` with greater precision and scale | | `date` | | `timestampNTZ` | To avoid accidentally promoting integer values to decimals, you must **manually commit** type changes from `byte`, `short`, `int`, or `long` to `decimal` or `double`. When promoting an integer type to `decimal` or `double`, if any downstream ingestion writes this value back to an integer column, Spark will truncate the fractional part of the values by default. Type changes are supported for top-level columns as well as fields nested inside structs, maps and arrays. ## How to enable Delta Lake type widening You can enable type widening on an existing table by setting the `delta.enableTypeWidening` table property to `true`: ```sql ALTER TABLE SET TBLPROPERTIES ('delta.enableTypeWidening' = 'true') ``` Alternatively, you can enable type widening during table creation: ```sql CREATE TABLE USING DELTA TBLPROPERTIES('delta.enableTypeWidening' = 'true') ``` To disable type widening: ```sql ALTER TABLE SET TBLPROPERTIES ('delta.enableTypeWidening' = 'false') ``` Disabling type widening prevents future type changes from being applied to the table. It doesn't affect type changes previously applied and in particular, it doesn't remove the type widening table feature and doesn't allow clients that don't support the type widening table feature to read and write to the table. To remove the type widening table feature from the table and allow other clients that don't support this feature to read and write to the table, see [Removing the type widening table feature](#removing-the-type-widening-table-feature). ## Manually applying a type change When type widening is enabled on a Delta table, you can change the type of a column using the `ALTER COLUMN` command: ```sql ALTER TABLE ALTER COLUMN TYPE ``` The table schema is updated without rewriting the underlying Parquet files. ## Type changes with automatic schema evolution Schema evolution works with type widening to update data types in target tables to match the type of incoming data. To use schema evolution to widen the data type of a column during ingestion, you must meet the following conditions: - The write command runs with automatic schema evolution enabled. - The target table has type widening enabled. - The source column type is wider than the target column type. - Type widening supports the type change. - The type change is not one of `byte`, `short`, `int`, or `long` to `decimal` or `double`. These type changes can only be applied manually using ALTER TABLE to avoid accidental promotion of integers to decimals. Type mismatches that don't meet all of these conditions follow normal schema enforcement rules. ## Removing the type widening table feature The type widening feature can be removed from a Delta table using the `DROP FEATURE` command: ```sql ALTER TABLE DROP FEATURE 'typeWidening' [TRUNCATE HISTORY] ``` See [Drop Delta table features](/delta-drop-feature/) for more information on dropping Delta table features. When dropping the type widening feature, the underlying Parquet files are rewritten when necessary to ensure that the column types in the files match the column types in the Delta table schema. After the type widening feature is removed from the table, Delta clients that don't support the feature can read and write to the table. ## Limitations ### Iceberg Compatibility Iceberg doesn't support all type changes covered by type widening, see [Iceberg Schema Evolution](https://iceberg.apache.org/spec/#schema-evolution). In particular, Iceberg V2 does not support the following type changes: - `byte`, `short`, `int`, `long` to `decimal` or `double` - decimal scale increase - `date` to `timestampNTZ` When [UniForm with Iceberg compatibility](/delta-uniform) is enabled on a Delta table, applying one of these type changes results in an error. If you apply one of these unsupported type changes to a Delta table, enabling [Uniform with Iceberg compatibility](/delta-uniform) on the table results in an error. To resolve the error, you must [drop the type widening table feature](#removing-the-type-widening-table-feature). ================================================ FILE: docs/src/content/docs/delta-uniform.mdx ================================================ --- title: Universal Format (UniForm) description: Configure Delta tables to be read as Iceberg/Hudi tables using UniForm. --- import { Tabs, TabItem, Aside } from "@astrojs/starlight/components"; Delta Universal Format (UniForm) allows you to read Delta tables with Iceberg and Hudi clients. UniForm takes advantage of the fact that Delta Lake, Iceberg, and Hudi all consist of Parquet data files and a metadata layer. UniForm automatically generates Iceberg metadata asynchronously, allowing Iceberg clients to read Delta tables as if they were Iceberg or Hudi tables. You can expect negligible Delta write overhead when UniForm is enabled, as the metadata conversion and transaction occurs asynchronously after the Delta commit. A single copy of the data files provides access to clients of all formats. ## Requirements To enable UniForm, you must fulfill the following requirements: ### Uniform Iceberg - The table must have column mapping enabled. See [Delta column mapping](/delta-column-mapping). - The Delta table must have a `minReaderVersion` >= 2 and `minWriterVersion` >= 7. - Writes to the table must use Delta Lake 3.1 or above. - Hive Metastore (HMS) must be configured as the catalog. See [the HMS documentation](https://spark.apache.org/docs/latest/sql-data-sources-hive-tables.html) for how to configure Apache Spark to use Hive Metastore. ### Uniform Hudi (preview) - Writes to the table must use Delta Lake 3.2 or above. ## Enable Delta Lake UniForm The following table properties enable UniForm support for Iceberg. ``` 'delta.enableIcebergCompatV2' = 'true' 'delta.universalFormat.enabledFormats' = 'iceberg' ``` The following table properties enable UniForm support for Hudi. ``` 'delta.universalFormat.enabledFormats' = 'hudi' ``` The following table properties enable UniForm support for both. ``` 'delta.enableIcebergCompatV2' = 'true' 'delta.universalFormat.enabledFormats' = 'iceberg,hudi' ``` You must also enable column mapping to use UniForm. It is set automatically during table creation, as in the following example: ```sql CREATE TABLE T(c1 INT) USING DELTA TBLPROPERTIES( 'delta.enableIcebergCompatV2' = 'true', 'delta.universalFormat.enabledFormats' = 'iceberg'); ``` In Delta 3.3 and above, you can enable or upgrade UniForm Iceberg on an existing table using the following syntax: ```sql ALTER TABLE table_name SET TBLPROPERTIES( 'delta.enableIcebergCompatV2' = 'true', 'delta.universalFormat.enabledFormats' = 'iceberg'); ``` You can also use REORG to enable UniForm Iceberg and rewrite underlying data files, as in the following example: ```sql REORG TABLE table_name APPLY (UPGRADE UNIFORM(ICEBERG_COMPAT_VERSION=2)); ``` Use REORG if any of following are true: - Your table has deletion vectors enabled. - You previously enabled the IcebergCompatV1 version of UniForm Iceberg. - You need to read from Iceberg engines that don't support Hive-style Parquet files, such as Athena or Redshift. You can enable UniForm Hudi on an existing table using the following syntax: ```sql ALTER TABLE table_name SET TBLPROPERTIES ('delta.universalFormat.enabledFormats' = 'hudi'); ``` See [Limitations](#limitations). ## When does UniForm generate metadata? Delta Lake triggers Iceberg/Hudi metadata generation asynchronously after a Delta Lake write transaction completes using the same compute that completed the Delta transaction. Iceberg/Hudi can have significantly higher write latencies than Delta Lake. Delta tables with frequent commits might bundle multiple Delta commits into a single Iceberg/Hudi commit. Delta Lake ensures that only one metadata generation process per format is in progress at any time in a single cluster. Commits that would trigger a second concurrent metadata generation process successfully commit to Delta, but do not trigger asynchronous metadata generation. This prevents cascading latency for metadata generation for workloads with frequent commits (seconds to minutes between commits). ## Check Iceberg/Hudi metadata generation status UniForm adds the following properties to Iceberg/Hudi table metadata to track metadata generation status: | Table property | Description | | --- | --- | | `converted_delta_version` | The latest version of the Delta table for which metadata was successfully generated. | | `converted_delta_timestamp` | The timestamp of the latest Delta commit for which metadata was successfully generated. | See documentation for your Iceberg/Hudi reader client for how to review table properties outside Delta Lake. For Apache Spark, you can see these properties using the following syntax: ```sql SHOW TBLPROPERTIES ; ``` ## Read UniForm tables as Iceberg tables in Apache Spark You are able to read UniForm tables as Iceberg tables in Apache Spark with the following steps: - Start Apache Spark with Iceberg, and connect to the Hive Metastore used by UniForm. Please refer to the [Iceberg documentation](https://iceberg.apache.org/docs/latest/spark-configuration/#catalogs) for how to run Iceberg with Apache Spark and connect to a Hive Metastore. - Use the `SHOW TABLES` command to see a list of available Iceberg tables in the catalog. - Read an Iceberg table using standard SQL such as `SELECT`. ## Read UniForm tables as Iceberg tables using a metadata JSON path Some Iceberg clients allow you to register external Iceberg tables by providing a path to versioned metadata files. Each time UniForm converts a new version of the Delta table to Iceberg, it creates a new metadata JSON file. Clients that use metadata JSON paths for configuring Iceberg include BigQuery. Refer to documentation for the Iceberg reader client for configuration details. Delta Lake stores Iceberg metadata under the table directory, using the following pattern: ```ini /metadata/v-uuid.metadata.json ``` ## Read UniForm tables as Hudi tables in Apache Spark You are able to read UniForm tables as Hudi tables in Apache Spark with the following steps: - See [Hudi documentation](https://hudi.apache.org/docs/quick-start-guide#spark-shellsql) for how to run Hudi on Apache Spark ```scala spark.read.format("hudi") .option("hoodie.metadata.enable", "true") .load("PATH_TO_UNIFORM_TABLE_DIRECTORY") ``` ## Delta and Iceberg/Hudi table versions All Delta Lake, Iceberg and Hudi allow time travel queries using table versions or timestamps stored in table metadata. Delta and Iceberg table versions do not align by either the commit timestamp or the version ID. However, Delta and Hudi commit timestamp align, but version ID does not. If you wish to verify which version of a Delta table a given version of an Iceberg/Hudi table corresponds to, you can use the corresponding table properties set on the Iceberg/Hudi table. See [Check Iceberg/Hudi metadata generation status](#check-iceberghudi-metadata-generation-status). ## Limitations The following limitations exist: - UniForm does not work on tables with deletion vectors enabled. See [What are deletion vectors?](/delta-deletion-vectors). - Delta tables with UniForm enabled do not support `VOID` type. - Iceberg/Hudi clients can only read from UniForm. Writes are not supported. - Iceberg/Hudi reader clients might have individual limitations, regardless of UniForm. See documentation for your target client. The following Delta Lake features work for Delta clients when UniForm is enabled, but do not have support in Iceberg: - Change Data Feed - Delta Sharing ================================================ FILE: docs/src/content/docs/delta-update.mdx ================================================ --- title: Table deletes, updates, and merges description: Learn how to delete data from and update data in Delta tables. --- import { Aside, Tabs, TabItem } from "@astrojs/starlight/components"; Delta Lake supports several statements to facilitate deleting data from and updating data in Delta tables. ## Delete from a table You can remove data that matches a predicate from a Delta table. For instance, in a table named `people10m` or a path at `/tmp/delta/people-10m`, to delete all rows corresponding to people with a value in the `birthDate` column from before `1955`, you can run the following: ```sql DELETE FROM people10m WHERE birthDate < '1955-01-01' DELETE FROM delta.`/tmp/delta/people-10m` WHERE birthDate < '1955-01-01' ``` See [Configure SparkSession](/delta-batch#configure-sparksession) for the steps to enable support for SQL commands. ```python from delta.tables import * from pyspark.sql.functions import * deltaTable = DeltaTable.forPath(spark, '/tmp/delta/people-10m') # Declare the predicate by using a SQL-formatted string. deltaTable.delete("birthDate < '1955-01-01'") # Declare the predicate by using Spark SQL functions. deltaTable.delete(col('birthDate') < '1960-01-01') ``` ```scala import io.delta.tables._ val deltaTable = DeltaTable.forPath(spark, "/tmp/delta/people-10m") // Declare the predicate by using a SQL-formatted string. deltaTable.delete("birthDate < '1955-01-01'") import org.apache.spark.sql.functions._ import spark.implicits._ // Declare the predicate by using Spark SQL functions and implicits. deltaTable.delete(col("birthDate") < "1955-01-01") ``` ```java import io.delta.tables.*; import org.apache.spark.sql.functions; DeltaTable deltaTable = DeltaTable.forPath(spark, "/tmp/delta/people-10m"); // Declare the predicate by using a SQL-formatted string. deltaTable.delete("birthDate < '1955-01-01'"); // Declare the predicate by using Spark SQL functions. deltaTable.delete(functions.col("birthDate").lt(functions.lit("1955-01-01"))); ``` See the [Delta Lake APIs](/delta-apidoc/) for details. ## Update a table You can update data that matches a predicate in a Delta table. For example, in a table named `people10m` or a path at `/tmp/delta/people-10m`, to change an abbreviation in the `gender` column from `M` or `F` to `Male` or `Female`, you can run the following: ```sql UPDATE people10m SET gender = 'Female' WHERE gender = 'F'; UPDATE people10m SET gender = 'Male' WHERE gender = 'M'; UPDATE delta.`/tmp/delta/people-10m` SET gender = 'Female' WHERE gender = 'F'; UPDATE delta.`/tmp/delta/people-10m` SET gender = 'Male' WHERE gender = 'M'; ``` See [Configure SparkSession](/delta-batch#configure-sparksession) for the steps to enable support for SQL commands. ```python from delta.tables import * from pyspark.sql.functions import * deltaTable = DeltaTable.forPath(spark, '/tmp/delta/people-10m') # Declare the predicate by using a SQL-formatted string. deltaTable.update( condition = "gender = 'F'", set = { "gender": "'Female'" } ) # Declare the predicate by using Spark SQL functions. deltaTable.update( condition = col('gender') == 'M', set = { 'gender': lit('Male') } ) ``` ```scala import io.delta.tables._ val deltaTable = DeltaTable.forPath(spark, "/tmp/delta/people-10m") // Declare the predicate by using a SQL-formatted string. deltaTable.updateExpr( "gender = 'F'", Map("gender" -> "'Female'") import org.apache.spark.sql.functions._ import spark.implicits._ // Declare the predicate by using Spark SQL functions and implicits. deltaTable.update( col("gender") === "M", Map("gender" -> lit("Male"))); ``` ```java import io.delta.tables.*; import org.apache.spark.sql.functions; import java.util.HashMap; DeltaTable deltaTable = DeltaTable.forPath(spark, "/data/events/"); // Declare the predicate by using a SQL-formatted string. deltaTable.updateExpr( "gender = 'F'", new HashMap() {{ put("gender", "'Female'"); }} ); // Declare the predicate by using Spark SQL functions. deltaTable.update( functions.col(gender).eq("M"), new HashMap() {{ put("gender", functions.lit("Male")); }} ); ``` See the [Delta Lake APIs](/delta-apidoc/) for details. ## Upsert into a table using merge You can upsert data from a source table, view, or DataFrame into a target Delta table by using the `MERGE` SQL operation. Delta Lake supports inserts, updates and deletes in `MERGE`, and it supports extended syntax beyond the SQL standards to facilitate advanced use cases. Suppose you have a source table named `people10mupdates` or a source path at `/tmp/delta/people-10m-updates` that contains new data for a target table named `people10m` or a target path at `/tmp/delta/people-10m`. Some of these new records may already be present in the target data. To merge the new data, you want to update rows where the person's `id` is already present and insert the new rows where no matching `id` is present. You can run the following: ```sql MERGE INTO people10m USING people10mupdates ON people10m.id = people10mupdates.id WHEN MATCHED THEN UPDATE SET id = people10mupdates.id, firstName = people10mupdates.firstName, middleName = people10mupdates.middleName, lastName = people10mupdates.lastName, gender = people10mupdates.gender, birthDate = people10mupdates.birthDate, ssn = people10mupdates.ssn, salary = people10mupdates.salary WHEN NOT MATCHED THEN INSERT ( id, firstName, middleName, lastName, gender, birthDate, ssn, salary ) VALUES ( people10mupdates.id, people10mupdates.firstName, people10mupdates.middleName, people10mupdates.lastName, people10mupdates.gender, people10mupdates.birthDate, people10mupdates.ssn, people10mupdates.salary ) ``` See [Configure SparkSession](/delta-batch/#configure-sparksession) for the steps to enable support for SQL commands. ```python from delta.tables import * deltaTablePeople = DeltaTable.forPath(spark, '/tmp/delta/people-10m') deltaTablePeopleUpdates = DeltaTable.forPath(spark, '/tmp/delta/people-10m-updates') dfUpdates = deltaTablePeopleUpdates.toDF() deltaTablePeople.alias('people') \ .merge( dfUpdates.alias('updates'), 'people.id = updates.id' ) \ .whenMatchedUpdate(set = { "id": "updates.id", "firstName": "updates.firstName", "middleName": "updates.middleName", "lastName": "updates.lastName", "gender": "updates.gender", "birthDate": "updates.birthDate", "ssn": "updates.ssn", "salary": "updates.salary" } ) \ .whenNotMatchedInsert(values = { "id": "updates.id", "firstName": "updates.firstName", "middleName": "updates.middleName", "lastName": "updates.lastName", "gender": "updates.gender", "birthDate": "updates.birthDate", "ssn": "updates.ssn", "salary": "updates.salary" } ) \ .execute() ``` ```scala import io.delta.tables._ import org.apache.spark.sql.functions._ val deltaTablePeople = DeltaTable.forPath(spark, "/tmp/delta/people-10m") val deltaTablePeopleUpdates = DeltaTable.forPath(spark, "tmp/delta/people-10m-updates") val dfUpdates = deltaTablePeopleUpdates.toDF() deltaTablePeople .as("people") .merge( dfUpdates.as("updates"), "people.id = updates.id") .whenMatched .updateExpr( Map( "id" -> "updates.id", "firstName" -> "updates.firstName", "middleName" -> "updates.middleName", "lastName" -> "updates.lastName", "gender" -> "updates.gender", "birthDate" -> "updates.birthDate", "ssn" -> "updates.ssn", "salary" -> "updates.salary" )) .whenNotMatched .insertExpr( Map( "id" -> "updates.id", "firstName" -> "updates.firstName", "middleName" -> "updates.middleName", "lastName" -> "updates.lastName", "gender" -> "updates.gender", "birthDate" -> "updates.birthDate", "ssn" -> "updates.ssn", "salary" -> "updates.salary" )) .execute() ``` ```java import io.delta.tables.*; import org.apache.spark.sql.functions; import java.util.HashMap; DeltaTable deltaTable = DeltaTable.forPath(spark, "/tmp/delta/people-10m") Dataset dfUpdates = spark.read("delta").load("/tmp/delta/people-10m-updates") deltaTable .as("people") .merge( dfUpdates.as("updates"), "people.id = updates.id") .whenMatched() .updateExpr( new HashMap() {{ put("id", "updates.id"); put("firstName", "updates.firstName"); put("middleName", "updates.middleName"); put("lastName", "updates.lastName"); put("gender", "updates.gender"); put("birthDate", "updates.birthDate"); put("ssn", "updates.ssn"); put("salary", "updates.salary"); }}) .whenNotMatched() .insertExpr( new HashMap() {{ put("id", "updates.id"); put("firstName", "updates.firstName"); put("middleName", "updates.middleName"); put("lastName", "updates.lastName"); put("gender", "updates.gender"); put("birthDate", "updates.birthDate"); put("ssn", "updates.ssn"); put("salary", "updates.salary"); }}) .execute(); ``` See the [Delta Lake APIs](/delta-apidoc/) for Scala, Java, and Python syntax details. ### Modify all unmatched rows using merge You can use the `WHEN NOT MATCHED BY SOURCE` clause to `UPDATE` or `DELETE` records in the target table that do not have corresponding records in the source table. We recommend adding an optional conditional clause to avoid fully rewriting the target table. The following code example shows the basic syntax of using this for deletes, overwriting the target table with the contents of the source table and deleting unmatched records in the target table. ```sql MERGE INTO target USING source ON source.key = target.key WHEN MATCHED UPDATE SET * WHEN NOT MATCHED INSERT * WHEN NOT MATCHED BY SOURCE DELETE ``` ```python (targetDF .merge(sourceDF, "source.key = target.key") .whenMatchedUpdateAll() .whenNotMatchedInsertAll() .whenNotMatchedBySourceDelete() .execute() ) ``` ```scala targetDF .merge(sourceDF, "source.key = target.key") .whenMatched() .updateAll() .whenNotMatched() .insertAll() .whenNotMatchedBySource() .delete() .execute() ``` The following example adds conditions to the `WHEN NOT MATCHED BY SOURCE` clause and specifies values to update in unmatched target rows. ```sql MERGE INTO target USING source ON source.key = target.key WHEN MATCHED THEN UPDATE SET target.lastSeen = source.timestamp WHEN NOT MATCHED THEN INSERT (key, lastSeen, status) VALUES (source.key, source.timestamp, 'active') WHEN NOT MATCHED BY SOURCE AND target.lastSeen >= (current_date() - INTERVAL '5' DAY) THEN UPDATE SET target.status = 'inactive' ``` ```python (targetDF .merge(sourceDF, "source.key = target.key") .whenMatchedUpdate( set = {"target.lastSeen": "source.timestamp"} ) .whenNotMatchedInsert( values = { "target.key": "source.key", "target.lastSeen": "source.timestamp", "target.status": "'active'" } ) .whenNotMatchedBySourceUpdate( condition="target.lastSeen >= (current_date() - INTERVAL '5' DAY)", set = {"target.status": "'inactive'"} ) .execute() ) ``` ```scala targetDF .merge(sourceDF, "source.key = target.key") .whenMatched() .updateExpr(Map("target.lastSeen" -> "source.timestamp")) .whenNotMatched() .insertExpr(Map( "target.key" -> "source.key", "target.lastSeen" -> "source.timestamp", "target.status" -> "'active'", ) ) .whenNotMatchedBySource("target.lastSeen >= (current_date() - INTERVAL '5' DAY)") .updateExpr(Map("target.status" -> "'inactive'")) .execute() ``` ### Operation semantics Here is a detailed description of the `merge` programmatic operation. - There can be any number of `whenMatched` and `whenNotMatched` clauses. - `whenMatched` clauses are executed when a source row matches a target table row based on the match condition. These clauses have the following semantics. - `whenMatched` clauses can have at most one `update` and one `delete` action. The `update` action in `merge` only updates the specified columns (similar to the `update` [operation](/delta-update/#update-a-table)) of the matched target row. The `delete` action deletes the matched row. - Each `whenMatched` clause can have an optional condition. If this clause condition exists, the `update` or `delete` action is executed for any matching source-target row pair only when the clause condition is true. - If there are multiple `whenMatched` clauses, then they are evaluated in the order they are specified. All `whenMatched` clauses, except the last one, must have conditions. - If none of the `whenMatched` conditions evaluate to true for a source and target row pair that matches the merge condition, then the target row is left unchanged. - To update all the columns of the target Delta table with the corresponding columns of the source dataset, use `whenMatched(...).updateAll()`. This is equivalent to: ```scala whenMatched(...).updateExpr(Map("col1" -> "source.col1", "col2" -> "source.col2", ...)) ``` for all the columns of the target Delta table. Therefore, this action assumes that the source table has the same columns as those in the target table, otherwise the query throws an analysis error. - `whenNotMatched` clauses are executed when a source row does not match any target row based on the match condition. These clauses have the following semantics. - `whenNotMatched` clauses can have only the `insert` action. The new row is generated based on the specified column and corresponding expressions. You do not need to specify all the columns in the target table. For unspecified target columns, `NULL` is inserted. - Each `whenNotMatched` clause can have an optional condition. If the clause condition is present, a source row is inserted only if that condition is true for that row. Otherwise, the source column is ignored. - If there are multiple `whenNotMatched` clauses, then they are evaluated in the order they are specified. All `whenNotMatched` clauses, except the last one, must have conditions. - To insert all the columns of the target Delta table with the corresponding columns of the source dataset, use `whenNotMatched(...).insertAll()`. This is equivalent to: ```scala whenNotMatched(...).insertExpr(Map("col1" -> "source.col1", "col2" -> "source.col2", ...)) ``` for all the columns of the target Delta table. Therefore, this action assumes that the source table has the same columns as those in the target table, otherwise the query throws an analysis error. - `whenNotMatchedBySource` clauses are executed when a target row does not match any source row based on the merge condition. These clauses have the following semantics. - `whenNotMatchedBySource` clauses can specify `delete` and `update` actions. - Each `whenNotMatchedBySource` clause can have an optional condition. If the clause condition is present, a target row is modified only if that condition is true for that row. Otherwise, the target row is left unchanged. - If there are multiple `whenNotMatchedBySource` clauses, then they are evaluated in the order they are specified. All `whenNotMatchedBySource` clauses, except the last one, must have conditions. - By definition, `whenNotMatchedBySource` clauses do not have a source row to pull column values from, and so source columns can't be referenced. For each column to be modified, you can either specify a literal or perform an action on the target column, such as `SET target.deleted_count = target.deleted_count + 1`. ### Schema validation `merge` automatically validates that the schema of the data generated by insert and update expressions are compatible with the schema of the table. It uses the following rules to determine whether the `merge` operation is compatible: - For `update` and `insert` actions, the specified target columns must exist in the target Delta table. - For `updateAll` and `insertAll` actions, the source dataset must have all the columns of the target Delta table. The source dataset can have extra columns and they are ignored. If you do not want the extra columns to be ignored and instead want to update the target table schema to include new columns, see [Automatic schema evolution](/delta-update/#automatic-schema-evolution). - For all actions, if the data type generated by the expressions producing the target columns are different from the corresponding columns in the target Delta table, `merge` tries to cast them to the types in the table. ### Automatic schema evolution Schema evolution allows users to resolve schema mismatches between the target and source table in merge. It handles the following two cases: 1. A column in the source table is not present in the target table. The new column is added to the target schema, and its values are inserted or updated using the source values. 2. A column in the target table is not present in the source table. The target schema is left unchanged; the values in the additional target column are either left unchanged (for `UPDATE`) or set to `NULL` (for `INSERT`). Here are a few examples of the effects of `merge` operation with and without schema evolution. | Columns | Query (in SQL) | Behavior without schema evolution (default) | Behavior with schema evolution | | :-- | :-- | :-- | :-- | | Target: `key, value` Source: `key, value, new_value` | `sql MERGE INTO target_table t USING source_table s ON t.key = s.key WHEN MATCHED THEN UPDATE SET * WHEN NOT MATCHED THEN INSERT *` | The table schema remains unchanged; only columns `key`, `value` are updated/inserted. | The table schema is changed to `(key, value, new_value)`. Existing records with matches are updated with the `value` and `new_value` in the source. New rows are inserted with the schema `(key, value, new_value)`. | | Target: `key, old_value` Source: `key, new_value` | `sql MERGE INTO target_table t USING source_table s ON t.key = s.key WHEN MATCHED THEN UPDATE SET * WHEN NOT MATCHED THEN INSERT *` | `UPDATE` and `INSERT` actions throw an error because the target column `old_value` is not in the source. | The table schema is changed to `(key, old_value, new_value)`. Existing records with matches are updated with the `new_value` in the source leaving `old_value` unchanged. New records are inserted with the specified `key`, `new_value`, and `NULL` for the `old_value`. | | Target: `key, old_value` Source: `key, new_value` | `sql MERGE INTO target_table t USING source_table s ON t.key = s.key WHEN MATCHED THEN UPDATE SET new_value = s.new_value` | `UPDATE` throws an error because column `new_value` does not exist in the target table. | The table schema is changed to `(key, old_value, new_value)`. Existing records with matches are updated with the `new_value` in the source leaving `old_value` unchanged, and unmatched records have `NULL` entered for `new_value`. | | Target: `key, old_value` Source: `key, new_value` | `sql MERGE INTO target_table t USING source_table s ON t.key = s.key WHEN NOT MATCHED THEN INSERT (key, new_value) VALUES (s.key, s.new_value)` | `INSERT`throws an error because column`new_value`does not exist in the target table. | The table schema is changed to`(key, old_value, new_value)`. New records are inserted with the specified `key`, `new_value`, and `NULL`for the`old_value`. Existing records have `NULL`entered for`new_value`leaving`old_value` unchanged. See note (1). | ## Special considerations for schemas that contain arrays of structs Delta `MERGE INTO` supports resolving struct fields by name and evolving schemas for arrays of structs. With schema evolution enabled, target table schemas will evolve for arrays of structs, which also works with any nested structs inside of arrays. Here are a few examples of the effects of merge operations with and without schema evolution for arrays of structs. | Source schema | Target schema | Behavior without schema evolution (default) | Behavior with schema evolution | | :-- | :-- | :-- | :-- | | array<struct<b: string, a: string>> | array<struct<a: int, b: int>> | The table schema remains unchanged. Columns will be resolved by name and updated or inserted. | The table schema remains unchanged. Columns will be resolved by name and updated or inserted. | | array<struct<a: int, c: string, d: string>> | array<struct<a: string, b: string>> | `update` and `insert` throw errors because `c` and `d` do not exist in the target table. | The table schema is changed to array<struct<a: string, b: string, c: string, d: string>>. `c` and `d` are inserted as `NULL` for existing entries in the target table. `update` and `insert` fill entries in the source table with `a` casted to string and `b` as `NULL`. | | array<struct<a: string, b: struct<c: string, d: string>>> | array<struct<a: string, b: struct<c: string>>> | `update` and `insert` throw errors because `d` does not exist in the target table. | The target table schema is changed to array<struct<a: string, b: struct<c: string, d: string>>>. `d` is inserted as `NULL` for existing entries in the target table. | ### Performance tuning You can reduce the time taken by merge using the following approaches: - **Reduce the search space for matches**: By default, the `merge` operation searches the entire Delta table to find matches in the source table. One way to speed up `merge` is to reduce the search space by adding known constraints in the match condition. For example, suppose you have a table that is partitioned by `country` and `date` and you want to use `merge` to update information for the last day and a specific country. Adding the condition ```sql events.date = current_date() AND events.country = 'USA' ``` will make the query faster as it looks for matches only in the relevant partitions. Furthermore, it will also reduce the chances of conflicts with other concurrent operations. See [Concurrency control](/concurrency-control/) for more details. - **Compact files**: If the data is stored in many small files, reading the data to search for matches can become slow. You can compact small files into larger files to improve read throughput. See [Compact files](/best-practices/#compact-files) for details. - **Control the shuffle partitions for writes**: The `merge` operation shuffles data multiple times to compute and write the updated data. The number of tasks used to shuffle is controlled by the Spark session configuration `spark.sql.shuffle.partitions`. Setting this parameter not only controls the parallelism but also determines the number of output files. Increasing the value increases parallelism but also generates a larger number of smaller data files. - **Repartition output data before write**: For partitioned tables, `merge` can produce a much larger number of small files than the number of shuffle partitions. This is because every shuffle task can write multiple files in multiple partitions, and can become a performance bottleneck. In many cases, it helps to repartition the output data by the table's partition columns before writing it. You enable this by setting the Spark session configuration `spark.databricks.delta.merge.repartitionBeforeWrite.enabled` to `true`. ## Merge examples Here are a few examples on how to use `merge` in different scenarios. ### Data deduplication when writing into Delta tables A common ETL use case is to collect logs into Delta table by appending them to a table. However, often the sources can generate duplicate log records and downstream deduplication steps are needed to take care of them. With `merge`, you can avoid inserting the duplicate records. ```sql MERGE INTO logs USING newDedupedLogs ON logs.uniqueId = newDedupedLogs.uniqueId AND logs.date > current_date() - INTERVAL 7 DAYS WHEN NOT MATCHED AND newDedupedLogs.date > current_date() - INTERVAL 7 DAYS THEN INSERT * ``` ```python deltaTable.alias("logs").merge( newDedupedLogs.alias("newDedupedLogs"), "logs.uniqueId = newDedupedLogs.uniqueId") \ .whenNotMatchedInsertAll() \ .execute() ``` ```scala deltaTable .as("logs") .merge( newDedupedLogs.as("newDedupedLogs"), "logs.uniqueId = newDedupedLogs.uniqueId") .whenNotMatched() .insertAll() .execute() ``` ```java deltaTable .as("logs") .merge( newDedupedLogs.as("newDedupedLogs"), "logs.uniqueId = newDedupedLogs.uniqueId") .whenNotMatched() .insertAll() .execute(); ``` If you know that you may get duplicate records only for a few days, you can optimized your query further by partitioning the table by date, and then specifying the date range of the target table to match on. ```sql MERGE INTO logs USING newDedupedLogs ON logs.uniqueId = newDedupedLogs.uniqueId AND logs.date > current_date() - INTERVAL 7 DAYS WHEN NOT MATCHED AND newDedupedLogs.date > current_date() - INTERVAL 7 DAYS THEN INSERT * ``` ```python deltaTable.alias("logs").merge( newDedupedLogs.alias("newDedupedLogs"), "logs.uniqueId = newDedupedLogs.uniqueId AND logs.date > current_date() - INTERVAL 7 DAYS") \ .whenNotMatchedInsertAll("newDedupedLogs.date > current_date() - INTERVAL 7 DAYS") \ .execute() ``` ```scala deltaTable.as("logs").merge( newDedupedLogs.as("newDedupedLogs"), "logs.uniqueId = newDedupedLogs.uniqueId AND logs.date > current_date() - INTERVAL 7 DAYS") .whenNotMatched("newDedupedLogs.date > current_date() - INTERVAL 7 DAYS") .insertAll() .execute() ``` ```java deltaTable.as("logs").merge( newDedupedLogs.as("newDedupedLogs"), "logs.uniqueId = newDedupedLogs.uniqueId AND logs.date > current_date() - INTERVAL 7 DAYS") .whenNotMatched("newDedupedLogs.date > current_date() - INTERVAL 7 DAYS") .insertAll() .execute(); ``` This is more efficient than the previous command as it looks for duplicates only in the last 7 days of logs, not the entire table. Furthermore, you can use this insert-only merge with Structured Streaming to perform continuous deduplication of the logs. - In a streaming query, you can use merge operation in `foreachBatch` to continuously write any streaming data to a Delta table with deduplication. See the following [streaming example](#upsert-from-streaming-queries-using-foreachbatch) for more information on `foreachBatch`. - In another streaming query, you can continuously read deduplicated data from this Delta table. This is possible because an insert-only merge only appends new data to the Delta table. ### Slowly changing data (SCD) Type 2 operation into Delta tables Another common operation is SCD Type 2, which maintains history of all changes made to each key in a dimensional table. Such operations require updating existing rows to mark previous values of keys as old, and the inserting the new rows as the latest values. Given a source table with updates and the target table with the dimensional data, SCD Type 2 can be expressed with `merge`. Here is a concrete example of maintaining the history of addresses for a customer along with the active date range of each address. When a customer's address needs to be updated, you have to mark the previous address as not the current one, update its active date range, and add the new address as the current one. ```python customersTable = ... # DeltaTable with schema (customerId, address, current, effectiveDate, endDate) updatesDF = ... # DataFrame with schema (customerId, address, effectiveDate) # Rows to INSERT new addresses of existing customers newAddressesToInsert = updatesDF \ .alias("updates") \ .join(customersTable.toDF().alias("customers"), "customerid") \ .where("customers.current = true AND updates.address <> customers.address") # Stage the update by unioning two sets of rows # 1. Rows that will be inserted in the whenNotMatched clause # 2. Rows that will either update the current addresses of existing customers or insert the new addresses of new customers stagedUpdates = ( newAddressesToInsert .selectExpr("NULL as mergeKey", "updates.*") # Rows for 1 .union(updatesDF.selectExpr("updates.customerId as mergeKey", "*")) # Rows for 2. ) # Apply SCD Type 2 operation using merge customersTable.alias("customers").merge( stagedUpdates.alias("staged_updates"), "customers.customerId = mergeKey") \ .whenMatchedUpdate( condition = "customers.current = true AND customers.address <> staged_updates.address", set = { # Set current to false and endDate to source's effective date. "current": "false", "endDate": "staged_updates.effectiveDate" } ).whenNotMatchedInsert( values = { "customerid": "staged_updates.customerId", "address": "staged_updates.address", "current": "true", "effectiveDate": "staged_updates.effectiveDate", # Set current to true along with the new address and its effective date. "endDate": "null" } ).execute() ``` ```scala val customersTable: DeltaTable = ... // table with schema (customerId, address, current, effectiveDate, endDate) val updatesDF: DataFrame = ... // DataFrame with schema (customerId, address, effectiveDate) // Rows to INSERT new addresses of existing customers val newAddressesToInsert = updatesDF .as("updates") .join(customersTable.toDF.as("customers"), "customerid") .where("customers.current = true AND updates.address <> customers.address") // Stage the update by unioning two sets of rows // 1. Rows that will be inserted in the whenNotMatched clause // 2. Rows that will either update the current addresses of existing customers or insert the new addresses of new customers val stagedUpdates = newAddressesToInsert .selectExpr("NULL as mergeKey", "updates.*") // Rows for 1. .union( updatesDF.selectExpr("updates.customerId as mergeKey", "*") // Rows for 2. ) // Apply SCD Type 2 operation using merge customersTable .as("customers") .merge( stagedUpdates.as("staged_updates"), "customers.customerId = mergeKey") .whenMatched("customers.current = true AND customers.address <> staged_updates.address") .updateExpr(Map( // Set current to false and endDate to source's effective date. "current" -> "false", "endDate" -> "staged_updates.effectiveDate")) .whenNotMatched() .insertExpr(Map( "customerid" -> "staged_updates.customerId", "address" -> "staged_updates.address", "current" -> "true", "effectiveDate" -> "staged_updates.effectiveDate", // Set current to true along with the new address and its effective date. "endDate" -> "null")) .execute() ``` ### Write change data into a Delta table Similar to SCD, another common use case, often called change data capture (CDC), is to apply all data changes generated from an external database into a Delta table. In other words, a set of updates, deletes, and inserts applied to an external table needs to be applied to a Delta table. You can do this using `merge` as follows. ```python deltaTable = ... # DeltaTable with schema (key, value) # DataFrame with changes having following columns # - key: key of the change # - time: time of change for ordering between changes (can replaced by other ordering id) # - newValue: updated or inserted value if key was not deleted # - deleted: true if the key was deleted, false if the key was inserted or updated changesDF = spark.table("changes") # Find the latest change for each key based on the timestamp # Note: For nested structs, max on struct is computed as # max on first struct field, if equal fall back to second fields, and so on. latestChangeForEachKey = changesDF \ .selectExpr("key", "struct(time, newValue, deleted) as otherCols") \ .groupBy("key") \ .agg(max("otherCols").alias("latest")) \ .select("key", "latest.*") \ deltaTable.alias("t").merge( latestChangeForEachKey.alias("s"), "s.key = t.key") \ .whenMatchedDelete(condition = "s.deleted = true") \ .whenMatchedUpdate(set = { "key": "s.key", "value": "s.newValue" }) \ .whenNotMatchedInsert( condition = "s.deleted = false", values = { "key": "s.key", "value": "s.newValue" } ).execute() ``` ```scala val deltaTable: DeltaTable = ... // DeltaTable with schema (key, value) // DataFrame with changes having following columns // - key: key of the change // - time: time of change for ordering between changes (can replaced by other ordering id) // - newValue: updated or inserted value if key was not deleted // - deleted: true if the key was deleted, false if the key was inserted or updated val changesDF: DataFrame = ... // Find the latest change for each key based on the timestamp // Note: For nested structs, max on struct is computed as // max on first struct field, if equal fall back to second fields, and so on. val latestChangeForEachKey = changesDF .selectExpr("key", "struct(time, newValue, deleted) as otherCols" ) .groupBy("key") .agg(max("otherCols").as("latest")) .selectExpr("key", "latest.*") deltaTable.as("t") .merge( latestChangeForEachKey.as("s"), "s.key = t.key") .whenMatched("s.deleted = true") .delete() .whenMatched() .updateExpr(Map("key" -> "s.key", "value" -> "s.newValue")) .whenNotMatched("s.deleted = false") .insertExpr(Map("key" -> "s.key", "value" -> "s.newValue")) .execute() ``` ### Upsert from streaming queries using `foreachBatch` You can use a combination of `merge` and `foreachBatch` (see [foreachbatch](https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html#foreachbatch) for more information) to write complex upserts from a streaming query into a Delta table. For example: - **Write streaming aggregates in Update Mode**: This is much more efficient than Complete Mode. ```python from delta.tables import * deltaTable = DeltaTable.forPath(spark, "/data/aggregates") # Function to upsert microBatchOutputDF into Delta table using merge def upsertToDelta(microBatchOutputDF, batchId): deltaTable.alias("t").merge( microBatchOutputDF.alias("s"), "s.key = t.key") \ .whenMatchedUpdateAll() \ .whenNotMatchedInsertAll() \ .execute() } # Write the output of a streaming aggregation query into Delta table streamingAggregatesDF.writeStream \ .format("delta") \ .foreachBatch(upsertToDelta) \ .outputMode("update") \ .start() ``` ```scala import io.delta.tables.* val deltaTable = DeltaTable.forPath(spark, "/data/aggregates") // Function to upsert microBatchOutputDF into Delta table using merge def upsertToDelta(microBatchOutputDF: DataFrame, batchId: Long) { deltaTable.as("t") .merge( microBatchOutputDF.as("s"), "s.key = t.key") .whenMatched().updateAll() .whenNotMatched().insertAll() .execute() } // Write the output of a streaming aggregation query into Delta table streamingAggregatesDF.writeStream .format("delta") .foreachBatch(upsertToDelta _) .outputMode("update") .start() ``` - **Write a stream of database changes into a Delta table**: The [merge query for writing change data](#write-change-data-into-a-delta-table) can be used in `foreachBatch` to continuously apply a stream of changes to a Delta table. - **Write a stream data into Delta table with deduplication**: The [insert-only merge query for deduplication](#data-deduplication-when-writing-into-delta-tables) can be used in `foreachBatch` to continuously write data (with duplicates) to a Delta table with automatic deduplication. ================================================ FILE: docs/src/content/docs/delta-utility/index.mdx ================================================ --- title: Table utility commands description: Learn about Delta Lake utility commands. --- import { Aside, Tabs, TabItem } from "@astrojs/starlight/components"; Delta tables support a number of utility commands. For many Delta Lake operations, you enable integration with Apache Spark DataSourceV2 and Catalog APIs (since 3.0) by setting configurations when you create a new `SparkSession`. See [Configure SparkSession](/delta-batch/#configure-sparksession). ## Remove files no longer referenced by a Delta table You can remove files no longer referenced by a Delta table and are older than the retention threshold by running the `vacuum` command on the table. `vacuum` is not triggered automatically. The default retention threshold for the files is 7 days. To change this behavior, see [Data retention](/delta-batch/#data-retention). ```sql VACUUM eventsTable -- This runs VACUUM in ‘FULL’ mode and deletes data files outside of the retention duration and all files in the table directory not referenced by the table. VACUUM eventsTable LITE -- This VACUUM in ‘LITE’ mode runs faster. -- Instead of finding all files in the table directory, `VACUUM LITE` uses the Delta transaction log to identify and remove files no longer referenced by any table versions within the retention duration. -- If `VACUUM LITE` cannot be completed because the Delta log has been pruned a `DELTA_CANNOT_VACUUM_LITE` exception is raised. -- This mode is available only in Delta 3.3 and above. VACUUM '/data/events' -- vacuum files in path-based table VACUUM delta.`/data/events/` VACUUM delta.`/data/events/` RETAIN 100 HOURS -- vacuum files not required by versions more than 100 hours old VACUUM eventsTable DRY RUN -- do dry run to get the list of files to be deleted VACUUM eventsTable USING INVENTORY inventoryTable —- vacuum files based on a provided reservoir of files as a delta table VACUUM eventsTable USING INVENTORY (select * from inventoryTable) —- vacuum files based on a provided reservoir of files as spark SQL query ``` ```python from delta.tables import * deltaTable = DeltaTable.forPath(spark, pathToTable) # path-based tables, or deltaTable = DeltaTable.forName(spark, tableName) # Hive metastore-based tables deltaTable.vacuum() # vacuum files not required by versions older than the default retention period deltaTable.vacuum(100) # vacuum files not required by versions more than 100 hours old ``` ```scala import io.delta.tables._ val deltaTable = DeltaTable.forPath(spark, pathToTable) deltaTable.vacuum() // vacuum files not required by versions older than the default retention period deltaTable.vacuum(100) // vacuum files not required by versions more than 100 hours old ``` ```java import io.delta.tables.*; import org.apache.spark.sql.functions; DeltaTable deltaTable = DeltaTable.forPath(spark, pathToTable); deltaTable.vacuum(); // vacuum files not required by versions older than the default retention period deltaTable.vacuum(100); // vacuum files not required by versions more than 100 hours old ``` See the [Delta Lake APIs](/delta-apidoc/) for Scala, Java, and Python syntax details. ### Inventory Table An inventory table contains a list of file paths together with their size, type (directory or not), and the last modification time. When an INVENTORY option is provided, VACUUM will consider the files listed there instead of doing the full listing of the table directory, which can be time consuming for very large tables. The inventory table can be specified as a delta table or a spark SQL query that gives the expected table schema. The schema should be as follows: | Column Name | Type | Description | | :--------------- | :------ | :-------------------------------------- | | path | string | fully qualified uri | | length | integer | size in bytes | | isDir | boolean | boolean indicating if it is a directory | | modificationTime | integer | file update time in milliseconds | ## Retrieve Delta table history You can retrieve information on the operations, user, timestamp, and so on for each write to a Delta table by running the `history` command. The operations are returned in reverse chronological order. By default table history is retained for 30 days. See [Configure SparkSession](/delta-batch/#configure-sparksession) for the steps to enable support for SQL commands in Apache Spark. ```sql DESCRIBE HISTORY '/data/events/' -- get the full history of the table DESCRIBE HISTORY delta.`/data/events/` DESCRIBE HISTORY '/data/events/' LIMIT 1 -- get the last operation only DESCRIBE HISTORY eventsTable ``` ```python from delta.tables import * deltaTable = DeltaTable.forPath(spark, pathToTable) fullHistoryDF = deltaTable.history() # get the full history of the table lastOperationDF = deltaTable.history(1) # get the last operation ``` ```scala import io.delta.tables._ val deltaTable = DeltaTable.forPath(spark, pathToTable) val fullHistoryDF = deltaTable.history() // get the full history of the table val lastOperationDF = deltaTable.history(1) // get the last operation ``` ```java import io.delta.tables.*; DeltaTable deltaTable = DeltaTable.forPath(spark, pathToTable); DataFrame fullHistoryDF = deltaTable.history(); // get the full history of the table DataFrame lastOperationDF = deltaTable.history(1); // fetch the last operation on the DeltaTable ``` See the [Delta Lake APIs](/delta-apidoc/) for Scala/Java/Python syntax details. The output of the `history` operation has the following columns. | Column | Type | Description | | :-- | :-- | :-- | | version | long | Table version generated by the operation. | | timestamp | timestamp | When this version was committed. | | userId | string | ID of the user that ran the operation. | | userName | string | Name of the user that ran the operation. | | operation | string | Name of the operation. | | operationParameters | map | Parameters of the operation (for example, predicates.) | | job | struct | Details of the job that ran the operation. | | notebook | struct | Details of notebook from which the operation was run. | | clusterId | string | ID of the cluster on which the operation ran. | | readVersion | long | Version of the table that was read to perform the write operation. | | isolationLevel | string | Isolation level used for this operation. | | isBlindAppend | boolean | Whether this operation appended data. | | operationMetrics | map | Metrics of the operation (for example, number of rows and files modified.) | | userMetadata | string | User-defined commit metadata if it was specified | ``` +-------+-------------------+------+--------+---------+--------------------+----+--------+---------+-----------+--------------+-------------+--------------------+ |version| timestamp|userId|userName|operation| operationParameters| job|notebook|clusterId|readVersion|isolationLevel|isBlindAppend| operationMetrics| +-------+-------------------+------+--------+---------+--------------------+----+--------+---------+-----------+--------------+-------------+--------------------+ | 5|2019-07-29 14:07:47| null| null| DELETE|[predicate -> ["(...|null| null| null| 4| Serializable| false|[numTotalRows -> ...| | 4|2019-07-29 14:07:41| null| null| UPDATE|[predicate -> (id...|null| null| null| 3| Serializable| false|[numTotalRows -> ...| | 3|2019-07-29 14:07:29| null| null| DELETE|[predicate -> ["(...|null| null| null| 2| Serializable| false|[numTotalRows -> ...| | 2|2019-07-29 14:06:56| null| null| UPDATE|[predicate -> (id...|null| null| null| 1| Serializable| false|[numTotalRows -> ...| | 1|2019-07-29 14:04:31| null| null| DELETE|[predicate -> ["(...|null| null| null| 0| Serializable| false|[numTotalRows -> ...| | 0|2019-07-29 14:01:40| null| null| WRITE|[mode -> ErrorIfE...|null| null| null| null| Serializable| true|[numFiles -> 2, n...| +-------+-------------------+------+--------+---------+--------------------+----+--------+---------+-----------+--------------+-------------+--------------------+ ``` The `history` operation returns a collection of operations metrics in the `operationMetrics` column map. The following table lists the map key definitions by operation. | Operation | Metric name | Description | | :-- | :-- | :-- | | WRITE, CREATE TABLE AS SELECT, REPLACE TABLE AS SELECT, COPY INTO | | | | | numFiles | Number of files written. | | | numOutputBytes | Size in bytes of the written contents. | | | numOutputRows | Number of rows written. | | STREAMING UPDATE | | | | | numAddedFiles | Number of files added. | | | numRemovedFiles | Number of files removed. | | | numOutputRows | Number of rows written. | | | numOutputBytes | Size of write in bytes. | | DELETE | | | | | numAddedFiles | Number of files added. Not provided when partitions of the table are deleted. | | | numRemovedFiles | Number of files removed. | | | numDeletedRows | Number of rows removed. Not provided when partitions of the table are deleted. | | | numCopiedRows | Number of rows copied in the process of deleting files. | | | executionTimeMs | Time taken to execute the entire operation. | | | scanTimeMs | Time taken to scan the files for matches. | | | rewriteTimeMs | Time taken to rewrite the matched files. | | TRUNCATE | | | | | numRemovedFiles | Number of files removed. | | | executionTimeMs | Time taken to execute the entire operation. | | MERGE | | | | | numSourceRows | Number of rows in the source DataFrame. | | | numTargetRowsInserted | Number of rows inserted into the target table. | | | numTargetRowsUpdated | Number of rows updated in the target table. | | | numTargetRowsDeleted | Number of rows deleted in the target table. | | | numTargetRowsCopied | Number of target rows copied. | | | numOutputRows | Total number of rows written out. | | | numTargetFilesAdded | Number of files added to the sink(target). | | | numTargetFilesRemoved | Number of files removed from the sink(target). | | | executionTimeMs | Time taken to execute the entire operation. | | | scanTimeMs | Time taken to scan the files for matches. | | | rewriteTimeMs | Time taken to rewrite the matched files. | | UPDATE | | | | | numAddedFiles | Number of files added. | | | numRemovedFiles | Number of files removed. | | | numUpdatedRows | Number of rows updated. | | | numCopiedRows | Number of rows just copied over in the process of updating files. | | | executionTimeMs | Time taken to execute the entire operation. | | | scanTimeMs | Time taken to scan the files for matches. | | | rewriteTimeMs | Time taken to rewrite the matched files. | | FSCK | numRemovedFiles | Number of files removed. | | CONVERT | numConvertedFiles | Number of Parquet files that have been converted. | | OPTIMIZE | | | | | numAddedFiles | Number of files added. | | | numRemovedFiles | Number of files optimized. | | | numAddedBytes | Number of bytes added after the table was optimized. | | | numRemovedBytes | Number of bytes removed. | | | minFileSize | Size of the smallest file after the table was optimized. | | | p25FileSize | Size of the 25th percentile file after the table was optimized. | | | p50FileSize | Median file size after the table was optimized. | | | p75FileSize | Size of the 75th percentile file after the table was optimized. | | | maxFileSize | Size of the largest file after the table was optimized. | | VACUUM | | | | | numDeletedFiles | Number of deleted files. | | | numVacuumedDirectories | Number of vacuumed directories. | | | numFilesToDelete | Number of files to delete. | | RESTORE | | | | | tableSizeAfterRestore | Table size in bytes after restore. | | | numOfFilesAfterRestore | Number of files in the table after restore. | | | numRemovedFiles | Number of files removed by the restore operation. | | | numRestoredFiles | Number of files that were added as a result of the restore. | | | removedFilesSize | Size in bytes of files removed by the restore. | | | restoredFilesSize | Size in bytes of files added by the restore. | ## Retrieve Delta table details You can retrieve detailed information about a Delta table (for example, number of files, data size) using `DESCRIBE DETAIL`. See [Configure SparkSession](/delta-batch/#configure-sparksession) for the steps to enable support for SQL commands in Apache Spark. ```sql DESCRIBE DETAIL '/data/events/' DESCRIBE DETAIL eventsTable ``` ```python from delta.tables import * deltaTable = DeltaTable.forPath(spark, pathToTable) detailDF = deltaTable.detail() ``` ```scala import io.delta.tables._ val deltaTable = DeltaTable.forPath(spark, pathToTable) val detailDF = deltaTable.detail() ``` ```java import io.delta.tables.*; DeltaTable deltaTable = DeltaTable.forPath(spark, pathToTable); DataFrame detailDF = deltaTable.detail(); ``` See the [Delta Lake APIs](/delta-apidoc/) for Scala/Java/Python syntax details. The output of this operation has only one row with the following schema. | Column | Type | Description | | :-- | :-- | :-- | | format | string | Format of the table, that is, `delta`. | | id | string | Unique ID of the table. | | name | string | Name of the table as defined in the metastore. | | description | string | Description of the table. | | location | string | Location of the table. | | createdAt | timestamp | When the table was created. | | lastModified | timestamp | When the table was last modified. | | partitionColumns | array of strings | Names of the partition columns if the table is partitioned. | | numFiles | long | Number of the files in the latest version of the table. | | sizeInBytes | int | The size of the latest snapshot of the table in bytes. | | properties | string-string map | All the properties set for this table. | | minReaderVersion | int | Minimum version of readers (according to the log protocol) that can read the table. | | minWriterVersion | int | Minimum version of writers (according to the log protocol) that can write to the table. | ``` +------+--------------------+------------------+-----------+--------------------+--------------------+-------------------+----------------+--------+-----------+----------+----------------+----------------+ |format| id| name|description| location| createdAt| lastModified|partitionColumns|numFiles|sizeInBytes|properties|minReaderVersion|minWriterVersion| +------+--------------------+------------------+-----------+--------------------+--------------------+-------------------+----------------+--------+-----------+----------+----------------+----------------+ | delta|d31f82d2-a69f-42e...|default.deltatable| null|file:/Users/tuor/...|2020-06-05 12:20:...|2020-06-05 12:20:20| []| 10| 12345| []| 1| 2| +------+--------------------+------------------+-----------+--------------------+--------------------+-------------------+----------------+--------+-----------+----------+----------------+----------------+ ``` ## Generate a manifest file You can a generate manifest file for a Delta table that can be used by other processing engines (that is, other than Apache Spark) to read the Delta table. For example, to generate a manifest file that can be used by Presto and Athena to read a Delta table, you run the following: ```sql GENERATE symlink_format_manifest FOR TABLE delta.`` ``` ```python deltaTable = DeltaTable.forPath() deltaTable.generate("symlink_format_manifest") ``` ```scala val deltaTable = DeltaTable.forPath() deltaTable.generate("symlink_format_manifest") ``` ```java DeltaTable deltaTable = DeltaTable.forPath(); deltaTable.generate("symlink_format_manifest"); ``` See [Configure SparkSession](/delta-batch/#configure-sparksession) for the steps to enable support for SQL commands in Apache Spark. ## Convert a Parquet table to a Delta table Convert a Parquet table to a Delta table in-place. This command lists all the files in the directory, creates a Delta Lake transaction log that tracks these files, and automatically infers the data schema by reading the footers of all Parquet files. If your data is partitioned, you must specify the schema of the partition columns as a DDL-formatted string (that is, ` , , ...`). By default, this command will collect per-file statistics (e.g. minimum and maximum values for each column). These statistics will be used at query time to provide faster queries. You can disable this statistics collection in the SQL API using `NO STATISTICS`. ```sql -- Convert unpartitioned Parquet table at path '' CONVERT TO DELTA parquet.`` -- Convert unpartitioned Parquet table and disable statistics collection CONVERT TO DELTA parquet.`` NO STATISTICS -- Convert partitioned Parquet table at path '' and partitioned by integer columns named 'part' and 'part2' CONVERT TO DELTA parquet.`` PARTITIONED BY (part int, part2 int) -- Convert partitioned Parquet table and disable statistics collection CONVERT TO DELTA parquet.`` NO STATISTICS PARTITIONED BY (part int, part2 int) ``` ```python from delta.tables import * # Convert unpartitioned Parquet table at path '' deltaTable = DeltaTable.convertToDelta(spark, "parquet.``") # Convert partitioned parquet table at path '' and partitioned by integer column named 'part' partitionedDeltaTable = DeltaTable.convertToDelta(spark, "parquet.``", "part int") ``` ```scala import io.delta.tables._ // Convert unpartitioned Parquet table at path '' val deltaTable = DeltaTable.convertToDelta(spark, "parquet.``") // Convert partitioned Parquet table at path '' and partitioned by integer columns named 'part' and 'part2' val partitionedDeltaTable = DeltaTable.convertToDelta(spark, "parquet.``", "part int, part2 int") ``` ```java import io.delta.tables.*; // Convert unpartitioned Parquet table at path '' DeltaTable deltaTable = DeltaTable.convertToDelta(spark, "parquet.``"); // Convert partitioned Parquet table at path '' and partitioned by integer columns named 'part' and 'part2' DeltaTable deltaTable = DeltaTable.convertToDelta(spark, "parquet.``", "part int, part2 int"); ``` ## Convert an Iceberg table to a Delta table You can convert an Iceberg table to a Delta table in place if the underlying file format of the Iceberg table is Parquet. Similar to a conversion from a Parquet table, the conversion is in-place and there won't be any data copy or data rewrite. The original Iceberg table and the converted Delta table have separate history, so modifying the Delta table should not affect the Iceberg table as long as the source data Parquet files are not touched or deleted. The following command creates a Delta Lake transaction log based on the Iceberg table's native file manifest, schema and partitioning information. The converter also collects column stats during the conversion, unless `NO STATISTICS` is specified. ```sql -- Convert the Iceberg table in the path . CONVERT TO DELTA iceberg.\`\` -- Convert the Iceberg table in the path without collecting statistics. CONVERT TO DELTA iceberg.\`\` NO STATISTICS ``` ## Convert a Delta table to a Parquet table You can easily convert a Delta table back to a Parquet table using the following steps: 1. If you have performed Delta Lake operations that can change the data files (for example, `delete` or `merge`), run [vacuum](#remove-files-no-longer-referenced-by-a-delta-table) ) with retention of 0 hours to delete all data files that do not belong to the latest version of the table. 2. Delete the `_delta_log` directory in the table directory. ## Restore a Delta table to an earlier state You can restore a Delta table to its earlier state by using the `RESTORE` command. A Delta table internally maintains historic versions of the table that enable it to be restored to an earlier state. A version corresponding to the earlier state or a timestamp of when the earlier state was created are supported as options by the `RESTORE` command. ```sql RESTORE TABLE db.target_table TO VERSION AS OF RESTORE TABLE delta.`/data/target/` TO TIMESTAMP AS OF ``` ```python from delta.tables import * deltaTable = DeltaTable.forPath(spark, ) # path-based tables, or deltaTable = DeltaTable.forName(spark, ) # Hive metastore-based tables deltaTable.restoreToVersion(0) # restore table to oldest version deltaTable.restoreToTimestamp('2019-02-14') # restore to a specific timestamp ``` ```scala import io.delta.tables._ val deltaTable = DeltaTable.forPath(spark, ) val deltaTable = DeltaTable.forName(spark, ) deltaTable.restoreToVersion(0) // restore table to oldest version deltaTable.restoreToTimestamp("2019-02-14") // restore to a specific timestamp ``` ```java import io.delta.tables.*; DeltaTable deltaTable = DeltaTable.forPath(spark, ); DeltaTable deltaTable = DeltaTable.forName(spark, ); deltaTable.restoreToVersion(0) // restore table to oldest version deltaTable.restoreToTimestamp("2019-02-14") // restore to a specific timestamp ``` For example: | Table version | Operation | Delta log updates | Records in data change log updates | | :-- | :-- | :-- | :-- | | 0 | INSERT | AddFile(/path/to/file-1, dataChange = true) | (name = Viktor, age = 29), (name = George, age = 55) | | 1 | INSERT | AddFile(/path/to/file-2, dataChange = true) | (name = George, age = 39) | | 2 | OPTIMIZE | AddFile(/path/to/file-3, dataChange = false), RemoveFile(/path/to/file-1), RemoveFile(/path/to/file-2) | (No records as Optimize compaction does not change the data in the table) | | 3 | RESTORE(version=1) | RemoveFile(/path/to/file-3), AddFile(/path/to/file-1, dataChange = true), AddFile(/path/to/file-2, dataChange = true) | (name = Viktor, age = 29), (name = George, age = 55), (name = George, age = 39) | In the preceding example, the `RESTORE` command results in updates that were already seen when reading the Delta table version 0 and 1. If a streaming query was reading this table, then these files will be considered as newly added data and will be processed again. `RESTORE` reports the following metrics as a single row DataFrame once the operation is complete: - `table_size_after_restore`: The size of the table after restoring. - `num_of_files_after_restore`: The number of files in the table after restoring. - `num_removed_files`: Number of files removed (logically deleted) from the table. - `num_restored_files`: Number of files restored due to rolling back. - `removed_files_size`: Total size in bytes of the files that are removed from the table. - `restored_files_size`: Total size in bytes of the files that are restored. ![Restore metrics example](./restore-metrics.png) ## Shallow clone a Delta table You can create a shallow copy of an existing Delta table at a specific version using the `shallow clone` command. Any changes made to shallow clones affect only the clones themselves and not the source table, as long as they don't touch the source data Parquet files. The metadata that is cloned includes: schema, partitioning information, invariants, nullability. For shallow clones, stream metadata is not cloned. Metadata not cloned are the table description and [user-defined commit metadata](/delta-batch/#set-user-defined-commit-metadata). ```sql CREATE TABLE delta.`/data/target/` SHALLOW CLONE delta.`/data/source/` -- Create a shallow clone of /data/source at /data/target CREATE OR REPLACE TABLE db.target_table SHALLOW CLONE db.source_table -- Replace the target. target needs to be emptied CREATE TABLE IF NOT EXISTS delta.`/data/target/` SHALLOW CLONE db.source_table -- No-op if the target table exists CREATE TABLE db.target_table SHALLOW CLONE delta.`/data/source` CREATE TABLE db.target_table SHALLOW CLONE delta.`/data/source` VERSION AS OF version CREATE TABLE db.target_table SHALLOW CLONE delta.`/data/source` TIMESTAMP AS OF timestamp_expression -- timestamp can be like “2019-01-01” or like date_sub(current_date(), 1) ``` `CLONE` reports the following metrics as a single row DataFrame once the operation is complete: - `source_table_size`: Size of the source table that's being cloned in bytes. - `source_num_of_files`: The number of files in the source table. ### Cloud provider permissions If you have created a shallow clone, any user that reads the shallow clone needs permission to read the files in the original table, since the data files remain in the source table's directory where we cloned from. To make changes to the clone, users will need write access to the clone's directory. #### Clone use cases ### Machine learning flow reproduction When doing machine learning, you may want to archive a certain version of a table on which you trained an ML model. Future models can be tested using this archived data set. ```sql -- Trained model on version 15 of Delta table CREATE TABLE delta.`/model/dataset` SHALLOW CLONE entire_dataset VERSION AS OF 15 ``` ### Short-term experiments on a production table To test a workflow on a production table without corrupting the table, you can easily create a shallow clone. This allows you to run arbitrary workflows on the cloned table that contains all the production data but does not affect any production workloads. ```sql -- Perform shallow clone CREATE OR REPLACE TABLE my_test SHALLOW CLONE my_prod_table; UPDATE my_test WHERE user_id is null SET invalid=true; -- Run a bunch of validations. Once happy: -- This should leverage the update information in the clone to prune to only -- changed files in the clone if possible MERGE INTO my_prod_table USING my_test ON my_test.user_id <=> my_prod_table.user_id WHEN MATCHED AND my_test.user_id is null THEN UPDATE *; DROP TABLE my_test; ``` ### Table property overrides Table property overrides are particularly useful for: - Annotating tables with owner or user information when sharing data with different business units. - Archiving Delta tables and time travel is required. You can specify the log retention period independently for the archive table. For example: ```sql CREATE OR REPLACE TABLE archive.my_table SHALLOW CLONE prod.my_table TBLPROPERTIES ( delta.logRetentionDuration = '3650 days', delta.deletedFileRetentionDuration = '3650 days' ) LOCATION 'xx://archive/my_table' ``` ## Clone Parquet or Iceberg table to Delta Shallow clone for Parquet and Iceberg combines functionality used to clone Delta tables and convert tables to Delta Lake, you can use clone functionality to convert data from Parquet or Iceberg data sources to managed or external Delta tables with the same basic syntax. `replace` has the same limitation as Delta shallow clone, the target table must be emptied before applying replace. ```sql CREATE OR REPLACE TABLE SHALLOW CLONE parquet.`/path/to/data`; CREATE OR REPLACE TABLE SHALLOW CLONE iceberg.`/path/to/data`; ``` ================================================ FILE: docs/src/content/docs/flink-integration.mdx ================================================ --- title: Apache Flink connector description: Learn how to set up an integration to enable you to write Delta tables from Apache Flink. --- This integration enables reading from and writing to Delta tables from Apache Flink. For details on using the Flink/Delta Connector, see the [Delta Lake repository](https://github.com/delta-io/delta/tree/master/connectors/flink). ================================================ FILE: docs/src/content/docs/hive-integration.mdx ================================================ --- title: Apache Hive description: Learn how to set up an integration to enable you to read Delta tables from . --- Page moved to [Apache Hive](/delta-more-connectors#apache-hive) ================================================ FILE: docs/src/content/docs/index.md ================================================ --- title: Welcome to the Delta Lake documentation description: Learn how to use Delta Lake sidebar: label: Welcome --- [Delta Lake](https://delta.io) is an [open source project](https://github.com/delta-io/delta) that enables building a [Lakehouse architecture](https://www.databricks.com/blog/2020/01/30/what-is-a-data-lakehouse.html) on top of [data lakes](https://www.databricks.com/discover/data-lakes). Delta Lake provides [ACID transactions](/concurrency-control), scalable metadata handling, and unifies [streaming](/delta-streaming) and [batch](/delta-batch) data processing on top of existing data lakes, such as S3, ADLS, GCS, and HDFS. Specifically, Delta Lake offers: - [ACID transactions](/concurrency-control) on Spark: Serializable isolation levels ensure that readers never see inconsistent data. - Scalable metadata handling: Leverages Spark distributed processing power to handle all the metadata for petabyte-scale tables with billions of files at ease. - [Streaming](/delta-streaming) and [batch](/delta-batch) unification: A table in Delta Lake is a batch table as well as a streaming source and sink. Streaming data ingest, batch historic backfill, interactive queries all just work out of the box. - Schema enforcement: Automatically handles schema variations to prevent insertion of bad records during ingestion. - [Time travel](/delta-batch#query-an-older-snapshot-of-a-table-time-travel): Data versioning enables rollbacks, full historical audit trails, and reproducible machine learning experiments. - [Upserts](/delta-update#upsert-into-a-table-using-merge) and [deletes](/delta-update#delete-from-a-table): Supports merge, update and delete operations to enable complex use cases like change-data-capture, slowly-changing-dimension (SCD) operations, streaming upserts, and so on. - Vibrant connector ecosystem: Delta Lake has connectors read and write Delta tables from various data processing engines like Apache Spark, Apache Flink, Apache Hive, Apache Trino, AWS Athena, and more. To get started follow the [quickstart guide](/quick-start) to learn how to use Delta Lake with Apache Spark. ================================================ FILE: docs/src/content/docs/integrations.mdx ================================================ --- title: Integrations description: Learn how to access Delta tables from external data processing engines. --- This page is moved to [Welcome to the Delta Lake documentation](/). ================================================ FILE: docs/src/content/docs/optimizations-oss/index.mdx ================================================ --- title: Optimizations description: Learn about the optimizations available with Delta Lake. --- import { Aside, Tabs, TabItem } from "@astrojs/starlight/components"; import { Image } from "astro:assets"; ## Optimize performance with file management To improve query speed, Delta Lake supports the ability to optimize the layout of data in storage. There are various ways to optimize the layout. ### Compaction (bin-packing) Delta Lake can improve the speed of read queries from a table by coalescing small files into larger ones. ```sql OPTIMIZE '/path/to/delta/table' -- Optimizes the path-based Delta Lake table OPTIMIZE delta_table_name; OPTIMIZE delta.`/path/to/delta/table`; -- If you have a large amount of data and only want to optimize a subset of it, you can specify an optional partition predicate using `WHERE`: OPTIMIZE delta_table_name WHERE date >= '2017-01-01' ``` ```python from delta.tables import * deltaTable = DeltaTable.forPath(spark, pathToTable) # For path-based tables # For Hive metastore-based tables: deltaTable = DeltaTable.forName(spark, tableName) deltaTable.optimize().executeCompaction() # If you have a large amount of data and only want to optimize a subset of it, you can specify an optional partition predicate using `where` deltaTable.optimize().where("date='2021-11-18'").executeCompaction() ``` ```scala import io.delta.tables._ val deltaTable = DeltaTable.forPath(spark, pathToTable) // For path-based tables // For Hive metastore-based tables: val deltaTable = DeltaTable.forName(spark, tableName) deltaTable.optimize().executeCompaction() // If you have a large amount of data and only want to optimize a subset of it, you can specify an optional partition predicate using `where` deltaTable.optimize().where("date='2021-11-18'").executeCompaction() ``` For Scala, Java, and Python API syntax details, see the [Delta Lake APIs](/delta-apidoc/). Readers of Delta tables use snapshot isolation, which means that they are not interrupted when `OPTIMIZE` removes unnecessary files from the transaction log. `OPTIMIZE` makes no data related changes to the table, so a read before and after an `OPTIMIZE` has the same results. Performing `OPTIMIZE` on a table that is a streaming source does not affect any current or future streams that treat this table as a source. `OPTIMIZE` returns the file statistics (min, max, total, and so on) for the files removed and the files added by the operation. Optimize stats also contains the number of batches, and partitions optimized. ## Auto compaction Auto compaction combines small files within Delta table partitions to automatically reduce small file problems. Auto compaction occurs after a write to a table has succeeded and runs synchronously on the cluster that has performed the write. Auto compaction only compacts files that haven't been compacted previously. You can control the output file size by setting the configuration `spark.databricks.delta.autoCompact.maxFileSize`. Auto compaction is only triggered for partitions or tables that have at least a certain number of small files. You can optionally change the minimum number of files required to trigger auto compaction by setting `spark.databricks.delta.autoCompact.minNumFiles`. Auto compaction can be enabled at the table or session level using the following settings: - Table property: `delta.autoOptimize.autoCompact` - SparkSession setting: `spark.databricks.delta.autoCompact.enabled` These settings accept the following options: | Options | Behavior | | --- | --- | | `true` | Enable auto compaction. By default will use 128 MB as the target file size. | | `false` | Turns off auto compaction. Can be set at the session level to override auto compaction for all Delta tables modified in the workload. | ## Data skipping Data skipping information is collected automatically when you write data into a Delta Lake table. Delta Lake takes advantage of this information (minimum and maximum values for each column) at query time to provide faster queries. You do not need to configure data skipping; the feature is activated whenever applicable. However, its effectiveness depends on the layout of your data. For best results, apply [Z-Ordering](#z-ordering-multi-dimensional-clustering). Collecting statistics on a column containing long values such as `string` or `binary` is an expensive operation. To avoid collecting statistics on such columns you can configure the [table property](/delta-batch/#table-properties) `delta.dataSkippingNumIndexedCols`. This property indicates the position index of a column in the table's schema. All columns with a position index less than the `delta.dataSkippingNumIndexedCols` property will have statistics collected. For the purposes of collecting statistics, each field within a nested column is considered as an individual column. To avoid collecting statistics on columns containing long values, either set the `delta.dataSkippingNumIndexedCols` property so that the long value columns are after this index in the table's schema, or move columns containing long strings to an index position greater than the `delta.dataSkippingNumIndexedCols` property by using [ALTER TABLE ALTER COLUMN](https://spark.apache.org/docs/latest/sql-ref-syntax-ddl-alter-table.html#alter-or-change-column). ## Z-Ordering (multi-dimensional clustering) Z-Ordering is a [technique](https://en.wikipedia.org/wiki/Z-order_curve) to colocate related information in the same set of files. This co-locality is automatically used by Delta Lake in data-skipping algorithms. This behavior dramatically reduces the amount of data that Delta Lake on Apache Spark needs to read. To Z-Order data, you specify the columns to order on in the `ZORDER BY` clause: ```sql OPTIMIZE events ZORDER BY (eventType) -- If you have a large amount of data and only want to optimize a subset of it, you can specify an optional partition predicate by using "where". OPTIMIZE events WHERE date = '2021-11-18' ZORDER BY (eventType) ``` ```python from delta.tables import * deltaTable = DeltaTable.forPath(spark, pathToTable) # path-based table # For Hive metastore-based tables: deltaTable = DeltaTable.forName(spark, tableName) deltaTable.optimize().executeZOrderBy(eventType) # If you have a large amount of data and only want to optimize a subset of it, you can specify an optional partition predicate using `where` deltaTable.optimize().where("date='2021-11-18'").executeZOrderBy(eventType) ``` ```scala import io.delta.tables._ val deltaTable = DeltaTable.forPath(spark, pathToTable) // path-based table // For Hive metastore-based tables: val deltaTable = DeltaTable.forName(spark, tableName) deltaTable.optimize().executeZOrderBy(eventType) // If you have a large amount of data and only want to optimize a subset of it, you can specify an optional partition predicate by using "where". deltaTable.optimize().where("date='2021-11-18'").executeZOrderBy(eventType) ``` For Scala, Java, and Python API syntax details, see the [Delta Lake APIs](/delta-apidoc/) If you expect a column to be commonly used in query predicates and if that column has high cardinality (that is, a large number of distinct values), then use `ZORDER BY`. You can specify multiple columns for `ZORDER BY` as a comma-separated list. However, the effectiveness of the locality drops with each extra column. Z-Ordering on columns that do not have statistics collected on them would be ineffective and a waste of resources. This is because data skipping requires column-local stats such as min, max, and count. You can configure statistics collection on certain columns by reordering columns in the schema, or you can increase the number of columns to collect statistics on. See [Data skipping](#data-skipping). ## Multi-part checkpointing Delta Lake table periodically and automatically compacts all the incremental updates to the Delta log into a Parquet file. This "checkpointing" allows read queries to quickly reconstruct the current state of the table (that is, which files to process, what is the current schema) without reading too many files having incremental updates. Delta Lake protocol allows [splitting the checkpoint](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#checkpoints) into multiple Parquet files. This parallelizes and speeds up writing the checkpoint. In Delta Lake, by default each checkpoint is written as a single Parquet file. To to use this feature, set the SQL configuration `spark.databricks.delta.checkpoint.partSize=`, where `n` is the limit of number of actions (such as `AddFile`) at which Delta Lake on Apache Spark will start parallelizing the checkpoint and attempt to write a maximum of this many actions per checkpoint file. ## Log compactions Delta Lake protocol allows new log compaction files with the format `..compact.json`. These files contain the aggregated actions for commit range `[x, y]`. Log compactions reduce the need for frequent checkpoints and minimize the latency spikes caused by them. The read support for the log compaction files is available in Delta Lake 3.0.0 and above. It is enabled by default and can be disabled using the SQL conf `spark.databricks.delta.deltaLog.minorCompaction.useForReads=` where `value` can be `true/false`. The write support for the log compaction will be added in a future version of Delta. ## Optimized Write Optimized writes improve file size as data is written and benefit subsequent reads on the table. Optimized writes are most effective for partitioned tables, as they reduce the number of small files written to each partition. Writing fewer large files is more efficient than writing many small files, but you might still see an increase in write latency because data is shuffled before being written. The following image demonstrates how optimized writes works: ![Optimized writes](./optimized-writes.png) The optimized write feature is **disabled** by default. It can be enabled at the table, SQL session, and/or DataFrameWriter level using the following settings (in order of precedence from low to high): - The `delta.autoOptimize.optimizeWrite` table property (default=None); - The `spark.databricks.delta.optimizeWrite.enabled` SQL configuration (default=None); - The DataFrameWriter option `optimizeWrite` (default=None). Besides the above, the following advanced SQL configurations can be used to further fine-tune the number and size of files written: - `spark.databricks.delta.optimizeWrite.binSize` (default=512MiB), which controls the target in-memory size of each output file; - `spark.databricks.delta.optimizeWrite.numShuffleBlocks` (default=50,000,000), which controls "maximum number of shuffle blocks to target"; - `spark.databricks.delta.optimizeWrite.maxShufflePartitions` (default=2,000), which controls "max number of output buckets (reducers) that can be used by optimized writes". ================================================ FILE: docs/src/content/docs/porting.mdx ================================================ --- title: Migration guide description: Learn how to migrate existing workloads to Delta Lake. --- import { Tabs, TabItem, Aside } from "@astrojs/starlight/components"; ## Migrate workloads to Delta Lake When you migrate workloads to Delta Lake, you should be aware of the following simplifications and differences compared with the data sources provided by Apache Spark and Apache Hive. Delta Lake handles the following operations automatically, which you should never perform manually: - **Add and remove partitions**: Delta Lake automatically tracks the set of partitions present in a table and updates the list as data is added or removed. As a result, there is no need to run `ALTER TABLE [ADD|DROP] PARTITION` or `MSCK`. - **Load a single partition**: As an optimization, you may sometimes directly load the partition of data you are interested in. For example, `spark.read.format("parquet").load("/data/date=2017-01-01")`. This is unnecessary with Delta Lake, since it can quickly read the list of files from the transaction log to find the relevant ones. If you are interested in a single partition, specify it using a `WHERE` clause. For example, `spark.read.delta("/data").where("date = '2017-01-01'")`. For large tables with many files in the partition, this can be much faster than loading a single partition (with direct partition path, or with `WHERE`) from a Parquet table because listing the files in the directory is often slower than reading the list of files from the transaction log. When you port an existing application to Delta Lake, you should avoid the following operations, which bypass the transaction log: - **Manually modify data**: Delta Lake uses the transaction log to atomically commit changes to the table. Because the log is the source of truth, files that are written out but not added to the transaction log are not read by Spark. Similarly, even if you manually delete a file, a pointer to the file is still present in the transaction log. Instead of manually modifying files stored in a Delta table, always use the commands that are described in this guide. - **External readers**: Directly reading the data stored in Delta Lake. For information on how to read Delta tables, see [read a table](/delta-batch/#read-a-table). ### Example Suppose you have Parquet data stored in a directory named `/data-pipeline`, and you want to create a Delta table named `events`. The [first example](#save-as-delta-table) shows how to: - Read the Parquet data from its original location, `/data-pipeline`, into a DataFrame. - Save the DataFrame's contents in Delta format in a separate location, `/tmp/delta/data-pipeline/`. - Create the `events` table based on that separate location, `/tmp/delta/data-pipeline/`. The [second example](#convert-to-delta-table) shows how to use `CONVERT TO TABLE` to convert data from Parquet to Delta format without changing its original location, `/data-pipeline/`. #### Save as Delta table 1. Read the Parquet data into a DataFrame and then save the DataFrame's contents to a new directory in `delta` format: ```python data = spark.read.format("parquet").load("/data-pipeline") data.write.format("delta").save("/tmp/delta/data-pipeline/") ``` 2. Create a Delta table named `events` that refers to the files in the new directory: ```python spark.sql("CREATE TABLE events USING DELTA LOCATION '/tmp/delta/data-pipeline/'") ``` #### Convert to Delta table You have two options for converting a Parquet table to a Delta table: - Convert files to Delta Lake format and then create a Delta table: ```sql CONVERT TO DELTA parquet.`/data-pipeline/` CREATE TABLE events USING DELTA LOCATION '/data-pipeline/' ``` - Create a Parquet table and then convert it to a Delta table: ```sql CREATE TABLE events USING PARQUET OPTIONS (path '/data-pipeline/') CONVERT TO DELTA events ``` For details, see [Convert a Parquet table to a Delta table](/delta-utility/#convert-a-parquet-table-to-a-delta-table). ## Migrate Delta Lake workloads to newer versions This section discusses any changes that may be required in the user code when migrating from older to newer versions of Delta Lake. ### Below Delta Lake 3.0 to Delta Lake 3.0 or above Please note that the Delta Lake on Spark Maven artifact has been renamed from `delta-core` (before 3.0) to `delta-spark` (3.0 and above). ### Delta Lake 2.1.1 or below to Delta Lake 2.2 or above Delta Lake 2.2 collects statistics by default when converting a parquet table to a Delta Lake table (e.g. using the `CONVERT TO DELTA` command). To opt out of statistics collection and revert to the 2.1.1 or below default behavior, use the `NO STATISTICS` SQL API (e.g. `CONVERT TO DELTA parquet.`/path-to-table` NO STATISTICS`) ### Delta Lake 1.2.1, 2.0.0, or 2.1.0 to Delta Lake 2.0.1, 2.1.1 or above Delta Lake 1.2.1, 2.0.0 and 2.1.0 have a bug in their DynamoDB-based S3 multi-cluster configuration implementations where an incorrect timestamp value was written to DynamoDB. This caused [DynamoDB's TTL](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/TTL.html) feature to cleanup completed items before it was safe to do so. This has been fixed in Delta Lake versions 2.0.1 and 2.1.1, and the TTL attribute has been renamed from `commitTime` to `expireTime`. If you _already_ have TTL enabled on your DynamoDB table using the old attribute, you need to disable TTL for that attribute and then enable it for the new one. You may need to wait an hour between these two operations, as TTL settings changes may take some time to propagate. See the DynamoDB docs [here](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/time-to-live-ttl-before-you-start.html). If you don't do this, DyanmoDB's TTL feature will not remove any new and expired entries. There is no risk of data loss. ```bash # Disable TTL on old attribute aws dynamodb update-time-to-live \ --region \ --table-name \ --time-to-live-specification "Enabled=false, AttributeName=commitTime" # Enable TTL on new attribute aws dynamodb update-time-to-live \ --region \ --table-name \ --time-to-live-specification "Enabled=true, AttributeName=expireTime" ``` ### Delta Lake 2.0 or below to Delta Lake 2.1 or above When calling `CONVERT TO DELTA` on a catalog table Delta Lake 2.1 infers the data schema from the catalog. In version 2.0 and below, Delta Lake infers the data schema from the data. This means in Delta 2.1 data columns that are not defined in the original catalog table will not be present in the converted Delta table. This behavior can be disabled by setting the Spark session configuration `spark.databricks.delta.convert.useCatalogSchema=false`. ### Delta Lake 1.2 or below to Delta Lake 2.0 or above Delta Lake 2.0.0 introduced a behavior change for [DROP CONSTRAINT](/delta-constraints/#check-constraint). In version 1.2 and below, no error was thrown when trying to drop a non-existent constraint. In version 2.0.0 and above, the behavior is changed to throw a constraint not exists error. To avoid the error, use `IF EXISTS` construct (for example, `ALTER TABLE events DROP CONSTRAINT IF EXISTS constraint_name`). There is no change in behavior in dropping an existing constraint. Delta Lake 2.0.0 introduced support for [Dynamic Partition Overwrites](/delta-batch/#overwrite). In version 1.2 and below, enabling dynamic partition overwrite mode in either the Spark session configuration or a `DataFrameWriter` option was a no-op, and writes in `overwrite` mode replaced all existing data in every partition of the table. In version 2.0.0 and above, when dynamic partition overwrite mode is enabled, Delta Lake replaces all existing data in each logical partition for which the write will commit new data. ### Delta Lake 1.1 or below to Delta Lake 1.2 or above The [LogStore](/api/latest/java/index.html) related code is extracted out from the `delta-core` Maven module into a new module `delta-storage` as part of the issue [#951](https://github.com/delta-io/delta/issues/951) for better code manageability. This results in an additional JAR `delta-storage-.jar` dependency for `delta-core`. By default, the additional JAR is downloaded as part of the `delta-core-_.jar` dependency. In clusters where there is _no internet connectivity_, `delta-storage-.jar` cannot be downloaded. It is advised to download the `delta-storage-.jar` manually and place it in the Java classpath. ### Delta Lake 1.0 or below to Delta Lake 1.1 or above If the name of a partition column in a Delta table contains invalid characters (` ,;{}()\n\t=`), you cannot read it in Delta Lake 1.1 and above, due to [SPARK-36271](https://issues.apache.org/jira/browse/SPARK-36271). However, this should be rare as you cannot create such tables by using Delta Lake 0.6 and above. If you still have such legacy tables, you can overwrite your tables with new valid column names by using Delta Lake 1.0 and below before upgrading Delta Lake to 1.1 and above, such as the following: ```python spark.read \ .format("delta") \ .load("/the/delta/table/path") \ .withColumnRenamed("column name", "column-name") \ .write \ .format("delta")\ .mode("overwrite") \ .option("overwriteSchema", "true") \ .save("/the/delta/table/path") ``` ```scala spark.read .format("delta") .load("/the/delta/table/path") .withColumnRenamed("column name", "column-name") .write .format("delta") .mode("overwrite") .option("overwriteSchema", "true") .save("/the/delta/table/path") ``` ### Delta Lake 0.6 or below to Delta Lake 0.7 or above If you are using `DeltaTable` APIs in Scala, Java, or Python to [update](/delta-update/) or [run utility operations](/delta-utility/) on them, then you may have to add the following configurations when creating the `SparkSession` used to perform those operations. ```python from pyspark.sql import SparkSession spark = SparkSession \ .builder \ .appName("...") \ .master("...") \ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \ .getOrCreate() ``` ```scala import org.apache.spark.sql.SparkSession val spark = SparkSession .builder() .appName("...") .master("...") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") .getOrCreate() ``` ```java import org.apache.spark.sql.SparkSession; SparkSession spark = SparkSession .builder() .appName("...") .master("...") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") .getOrCreate(); ``` Alternatively, you can add additional configurations when submitting you Spark application using `spark-submit` or when starting `spark-shell`/`pyspark` by specifying them as command line parameters. ```bash spark-submit \ --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" \ --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" \ ... ``` ```bash pyspark \ --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" \ --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" \ ... ``` ================================================ FILE: docs/src/content/docs/presto-integration.mdx ================================================ --- title: Presto, Trino, and Athena to Delta Lake integration using manifests description: Learn how to set up an integration to enable you to read Delta tables from Presto, Trino, and Athena. --- import { Tabs, TabItem, Aside, Steps } from "@astrojs/starlight/components"; Presto, Trino, and Athena support reading from external tables using a _manifest file_, which is a text file containing the list of data files to read for querying a table. When an external table is defined in the Hive metastore using manifest files, Presto, Trino, and Athena can use the list of files in the manifest rather than finding the files by directory listing. This article describes how to set up a Presto, Trino, and Athena to Delta Lake integration using manifest files and query Delta tables. ## Set up the Presto, Trino, or Athena to Delta Lake integration and query Delta tables You set up a Presto, Trino, or Athena to Delta Lake integration using the following steps. ### Step 1: Generate manifests of a Delta table using Apache Spark Using Spark [configured](/quick-start#set-up-apache-spark-with-delta-lake) with Delta Lake, run any of the following commands on a Delta table at location ``: ```sql GENERATE symlink_format_manifest FOR TABLE delta.`` ``` ```scala val deltaTable = DeltaTable.forPath() deltaTable.generate("symlink_format_manifest") ``` ```java DeltaTable deltaTable = DeltaTable.forPath(); deltaTable.generate("symlink_format_manifest"); ``` ```python deltaTable = DeltaTable.forPath() deltaTable.generate("symlink_format_manifest") ``` See [Generate a manifest file](delta-utility.html#generate-a-manifest-file) for details. The `generate` command generates manifest files at `/_symlink_format_manifest/`. In other words, the files in this directory will contain the names of the data files (that is, Parquet files) that should be read for reading a snapshot of the Delta table. ### Step 2: Configure Presto, Trino, or Athena to read the generated manifests 1. Define a new table in the Hive metastore connected to Presto, Trino, or Athena using the format `SymlinkTextInputFormat` and the manifest location `/_symlink_format_manifest/`. ```sql CREATE EXTERNAL TABLE mytable ([(col_name1 col_datatype1, ...)]) [PARTITIONED BY (col_name2 col_datatype2, ...)] ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.SymlinkTextInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' LOCATION '/_symlink_format_manifest/' -- location of the generated manifest ``` `SymlinkTextInputFormat` configures Presto, Trino, or Athena to compute file splits for `mytable` by reading the manifest file instead of using a directory listing to find data files. Replace `mytable` with the name of the external table and `` with the absolute path to the Delta table. - You cannot use this table definition in Apache Spark; it can be used only by Presto, Trino, and Athena. The tool you use to run the command depends on whether Apache Spark and Presto, Trino, or Athena use the same Hive metastore. - **Same metastore**: If both Apache Spark and Presto, Trino, or Athena use the same Hive metastore, you can define the table using Apache Spark. - **Different metastores**: If Apache Spark and Presto, Trino, or Athena use different metastores, you must define the table using other tools. - Athena: You can define the external table in Athena. - Presto: Presto does not support the syntax `CREATE EXTERNAL TABLE ... STORED AS ...`, so you must use another tool (for example, Spark or Hive) connected to the same metastore as Presto to create the table. 2. If the Delta table is partitioned, run `MSCK REPAIR TABLE mytable` after generating the manifests to force the metastore (connected to Presto, Trino, or Athena) to discover the partitions. This is needed because the manifest of a partitioned table is itself partitioned in the same directory structure as the table. Run this command using _the same tool_ used to create the table. Furthermore, you should run this command: - **After every manifest generation**: New partitions are likely to be visible immediately after the manifest files have been updated. However, doing this too frequently can cause high load for the Hive metastore. - **As frequently as new partitions are expected**: For example, if a table is partitioned by date, then you can run repair once after every midnight, after the new partition has been created in the table and its corresponding manifest files have been generated. ### Step 3: Update manifests When the data in a Delta table is updated you must regenerate the manifests using either of the following approaches: - **Update explicitly**: After all the data updates, you can run the `generate` operation to update the manifests. - **Update automatically**: You can configure a Delta table so that all write operations on the table automatically update the manifests. To enable this automatic mode, set the corresponding table property using the following SQL command. ```sql ALTER TABLE delta.`` SET TBLPROPERTIES(delta.compatibility.symlinkFormatManifest.enabled=true) ``` To disable this automatic mode, set this property to `false`. In addition, for partitioned tables, you have to run `MSCK REPAIR` to ensure the metastore connected to Presto, Trino, or Athena to update partitions. Whether to update automatically or explicitly depends on the concurrent nature of write operations on the Delta table and the desired data consistency. For example, if automatic mode is enabled, concurrent write operations lead to concurrent overwrites to the manifest files. With such unordered writes, the manifest files are not guaranteed to point to the latest version of the table after the write operations complete. Hence, if concurrent writes are expected and you want to avoid stale manifests, you should consider explicitly updating the manifest after the expected write operations have completed. ## Limitations The Presto, Trino, and Athena integration has known limitations in its behavior. ### Data consistency Whenever Delta Lake generates updated manifests, it atomically overwrites existing manifest files. Therefore, Presto, Trino, and Athena will always see a consistent view of the data files; it will see all of the old version files or all of the new version files. However, the granularity of the consistency guarantees depends on whether or not the table is partitioned. - **Unpartitioned tables**: All the files names are written in one manifest file which is updated atomically. In this case Presto, Trino, and Athena will see full table snapshot consistency. - **Partitioned tables**: A manifest file is partitioned in the same Hive-partitioning-style directory structure as the original Delta table. This means that each partition is updated atomically, and Presto, Trino, or Athena will see a consistent view of each partition but not a consistent view across partitions. Furthermore, since all manifests of all partitions cannot be updated together, concurrent attempts to generate manifests can lead to different partitions having manifests of different versions. While this consistency guarantee under data change is weaker than that of reading Delta tables with Spark, it is still stronger than formats like Parquet as they do not provide partition-level consistency. Depending on what storage system you are using for Delta tables, it is possible to get incorrect results when Presto, Trino, or Athena concurrently queries the manifest while the manifest files are being rewritten. In file system implementations that lack atomic file overwrites, a manifest file may be momentarily unavailable. Hence, use manifests with caution if their updates are likely to coincide with queries from Presto, Trino, or Athena. ### Performance Very large numbers of files can hurt the performance of Presto, Trino, and Athena. Hence it is recommended that you [compact the files](/best-practices#compact-files) of the table before generating the manifests. The number of files should not exceed 1000 (for the entire unpartitioned table or for each partition in a partitioned table). ### Schema evolution Delta Lake supports schema evolution and queries on a Delta table automatically use the latest schema regardless of the schema defined in the table in the Hive metastore. However, Presto, Trino, or Athena uses the schema defined in the Hive metastore and will not query with the updated schema until the table used by Presto, Trino, or Athena is redefined to have the updated schema. ### Encrypted tables Athena does not support reading manifests from [CSE-KMS](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-emrfs-encryption-cse.html) encrypted tables. See the AWS documentation for the latest information. ================================================ FILE: docs/src/content/docs/quick-start.mdx ================================================ --- title: Quick Start description: Learn how to get started quickly with Delta Lake. --- import { Aside, Tabs, TabItem } from "@astrojs/starlight/components"; This guide helps you quickly explore the main features of Delta Lake. It provides code snippets that show how to read from and write to Delta tables from interactive, batch, and streaming queries. ## Set up Apache Spark with Delta Lake Follow these instructions to set up Delta Lake with Spark. You can run the steps in this guide on your local machine in the following two ways: 1. **Run interactively**: Start the Spark shell (Scala or Python) with Delta Lake and run the code snippets interactively in the shell. 2. **Run as a project**: Set up a Maven or SBT project (Scala or Java) with Delta Lake, copy the code snippets into a source file, and run the project. Alternatively, you can use the [examples provided in the Github repository](https://github.com/delta-io/delta/tree/master/examples). ### Prerequisite: set up Java As mentioned in the official Apache Spark installation instructions [here](https://spark.apache.org/docs/latest/index.html#downloading), make sure you have a valid Java version installed (8, 11, or 17) and that Java is configured correctly on your system using either the system `PATH` or `JAVA_HOME` environmental variable. Windows users should follow the instructions in this [blog](https://phoenixnap.com/kb/install-spark-on-windows-10), making sure to use the correct version of Apache Spark that is compatible with Delta Lake `4.0.0`. ### Set up interactive shell To use Delta Lake interactively within the Spark SQL, Scala, or Python shell, you need a local installation of Apache Spark. Depending on whether you want to use SQL, Python, or Scala, you can set up either the SQL, PySpark, or Spark shell, respectively. #### Spark SQL Shell Download the [compatible version](/releases) of Apache Spark by following instructions from [Downloading Spark](https://spark.apache.org/downloads.html), either using `pip` or by downloading and extracting the archive and running `spark-sql` in the extracted directory. ```bash bin/spark-sql --packages io.delta:delta-spark_2.13:4.0.0 --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" ``` #### PySpark Shell 1. Install the PySpark version that is [compatible](/releases) with the Delta Lake version by running the following: ```bash pip install pyspark== ``` 2. Run PySpark with the Delta Lake package and additional configurations: ```bash pyspark --packages io.delta:delta-spark_2.13:4.0.0 --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" ``` #### Spark Scala Shell Download the [compatible version](/releases/) of Apache Spark by following instructions from [Downloading Spark](https://spark.apache.org/downloads.html), either using `pip` or by downloading and extracting the archive and running `spark-shell` in the extracted directory. ```bash bin/spark-shell --packages io.delta:delta-spark_2.13:4.0.0 --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" ``` ### Set up project If you want to build a project using Delta Lake binaries from Maven Central Repository, you can use the following Maven coordinates. #### Maven You include Delta Lake in your Maven project by adding it as a dependency in your POM file. Delta Lake compiled with Scala 2.13. ```xml io.delta delta-spark_2.13 4.0.0 ``` #### SBT You include Delta Lake in your SBT project by adding the following line to your `build.sbt` file: ```scala libraryDependencies += "io.delta" %% "delta-spark" % "4.0.0" ``` #### Python To set up a Python project (for example, for unit testing), you can install Delta Lake using `pip install delta-spark==4.0.0` and then configure the SparkSession with the `configure_spark_with_delta_pip()` utility function in Delta Lake. ```python import pyspark from delta import * builder = pyspark.sql.SparkSession.builder.appName("MyApp") \ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") spark = configure_spark_with_delta_pip(builder).getOrCreate() ``` ## Create a table To create a Delta table, write a DataFrame out in the `delta` format. You can use existing Spark SQL code and change the format from `parquet`, `csv`, `json`, and so on, to `delta`. ```sql CREATE TABLE delta.`/tmp/delta-table` USING DELTA AS SELECT col1 as id FROM VALUES 0,1,2,3,4; ``` ```python data = spark.range(0, 5) data.write.format("delta").save("/tmp/delta-table") ``` ```scala val data = spark.range(0, 5) data.write.format("delta").save("/tmp/delta-table") ``` ```java import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; SparkSession spark = ... // create SparkSession Dataset data = spark.range(0, 5); data.write().format("delta").save("/tmp/delta-table"); ``` These operations create a new Delta table using the schema that was _inferred_ from your DataFrame. For the full set of options available when you create a new Delta table, see [Create a table](/delta-batch/#create-a-table) and [Write to a table](/delta-batch/#write-to-table). ## Read data You read data in your Delta table by specifying the path to the files: `"/tmp/delta-table"`: ```sql SELECT * FROM delta.`/tmp/delta-table`; ``` ```python df = spark.read.format("delta").load("/tmp/delta-table") df.show() ``` ```scala val df = spark.read.format("delta").load("/tmp/delta-table") df.show() ``` ```java Dataset df = spark.read().format("delta").load("/tmp/delta-table"); df.show(); ``` ## Update table data Delta Lake supports several operations to modify tables using standard DataFrame APIs. This example runs a batch job to overwrite the data in the table: ### Overwrite ```sql INSERT OVERWRITE delta.`/tmp/delta-table` SELECT col1 as id FROM VALUES 5,6,7,8,9; ``` ```python data = spark.range(5, 10) data.write.format("delta").mode("overwrite").save("/tmp/delta-table") ``` ```scala val data = spark.range(5, 10) data.write.format("delta").mode("overwrite").save("/tmp/delta-table") df.show() ``` ```java Dataset data = spark.range(5, 10); data.write().format("delta").mode("overwrite").save("/tmp/delta-table"); ``` If you read this table again, you should see only the values `5-9` you have added because you overwrote the previous data. ### Conditional update without overwrite Delta Lake provides programmatic APIs to conditional update, delete, and merge (upsert) data into tables. Here are a few examples. ```sql -- Update every even value by adding 100 to it UPDATE delta.`/tmp/delta-table` SET id = id + 100 WHERE id % 2 == 0; -- Delete every even value DELETE FROM delta.`/tmp/delta-table` WHERE id % 2 == 0; -- Upsert (merge) new data CREATE TEMP VIEW newData AS SELECT col1 AS id FROM VALUES 1,3,5,7,9,11,13,15,17,19; MERGE INTO delta.`/tmp/delta-table` AS oldData USING newData ON oldData.id = newData.id WHEN MATCHED THEN UPDATE SET id = newData.id WHEN NOT MATCHED THEN INSERT (id) VALUES (newData.id); SELECT * FROM delta.`/tmp/delta-table`; ``` ```python from delta.tables import * from pyspark.sql.functions import * deltaTable = DeltaTable.forPath(spark, "/tmp/delta-table") # Update every even value by adding 100 to it deltaTable.update( condition = expr("id % 2 == 0"), set = { "id": expr("id + 100") }) # Delete every even value deltaTable.delete(condition = expr("id % 2 == 0")) # Upsert (merge) new data newData = spark.range(0, 20) deltaTable.alias("oldData") \ .merge( newData.alias("newData"), "oldData.id = newData.id") \ .whenMatchedUpdate(set = { "id": col("newData.id") }) \ .whenNotMatchedInsert(values = { "id": col("newData.id") }) \ .execute() deltaTable.toDF().show() ``` ```scala import io.delta.tables._ import org.apache.spark.sql.functions._ val deltaTable = DeltaTable.forPath("/tmp/delta-table") // Update every even value by adding 100 to it deltaTable.update( condition = expr("id % 2 == 0"), set = Map("id" -> expr("id + 100"))) // Delete every even value deltaTable.delete(condition = expr("id % 2 == 0")) // Upsert (merge) new data val newData = spark.range(0, 20).toDF deltaTable.as("oldData") .merge( newData.as("newData"), "oldData.id = newData.id") .whenMatched .update(Map("id" -> col("newData.id"))) .whenNotMatched .insert(Map("id" -> col("newData.id"))) .execute() deltaTable.toDF.show() ``` ```java import io.delta.tables.*; import org.apache.spark.sql.functions; import java.util.HashMap; DeltaTable deltaTable = DeltaTable.forPath("/tmp/delta-table"); // Update every even value by adding 100 to it deltaTable.update( functions.expr("id % 2 == 0"), new HashMap() {{ put("id", functions.expr("id + 100")); }} ); // Delete every even value deltaTable.delete(condition = functions.expr("id % 2 == 0")); // Upsert (merge) new data Dataset newData = spark.range(0, 20).toDF(); deltaTable.as("oldData") .merge( newData.as("newData"), "oldData.id = newData.id") .whenMatched() .update( new HashMap() {{ put("id", functions.col("newData.id")); }}) .whenNotMatched() .insertExpr( new HashMap() {{ put("id", functions.col("newData.id")); }}) .execute(); deltaTable.toDF().show(); ``` You should see that some of the existing rows have been updated and new rows have been inserted. For more information on these operations, see [Table deletes, updates, and merges](/delta-update). ## Read older versions of data using time travel You can query previous snapshots of your Delta table by using time travel. If you want to access the data that you overwrote, you can query a snapshot of the table before you overwrote the first set of data using the `versionAsOf` option. ```sql SELECT * FROM delta.`/tmp/delta-table` VERSION AS OF 0; ``` ```python df = spark.read.format("delta").option("versionAsOf", 0).load("/tmp/delta-table") df.show() ``` ```scala val df = spark.read.format("delta").option("versionAsOf", 0).load("/tmp/delta-table") df.show() ``` ```java Dataset df = spark.read().format("delta").option("versionAsOf", 0).load("/tmp/delta-table"); df.show(); ``` You should see the first set of data, from before you overwrote it. Time travel takes advantage of the power of the Delta Lake transaction log to access data that is no longer in the table. Removing the version `0` option (or specifying version `1`) would let you see the newer data again. For more information, see [Query an older snapshot of a table (time travel)](/delta-batch#query-an-older-snapshot-of-a-table-time-travel). ## Write a stream of data to a table You can also write to a Delta table using Structured Streaming. The Delta Lake transaction log guarantees exactly-once processing, even when there are other streams or batch queries running concurrently against the table. By default, streams run in append mode, which adds new records to the table: ```python streamingDf = spark.readStream.format("rate").load() stream = streamingDf.selectExpr("value as id").writeStream.format("delta").option("checkpointLocation", "/tmp/checkpoint").start("/tmp/delta-table") ``` ```scala val streamingDf = spark.readStream.format("rate").load() val stream = streamingDf.select($"value" as "id").writeStream.format("delta").option("checkpointLocation", "/tmp/checkpoint").start("/tmp/delta-table") ``` ```java import org.apache.spark.sql.streaming.StreamingQuery; Dataset streamingDf = spark.readStream().format("rate").load(); StreamingQuery stream = streamingDf.selectExpr("value as id").writeStream().format("delta").option("checkpointLocation", "/tmp/checkpoint").start("/tmp/delta-table"); ``` While the stream is running, you can read the table using the earlier commands. You can stop the stream by running `stream.stop()` in the same terminal that started the stream. For more information about Delta Lake integration with Structured Streaming, see [Table streaming reads and writes](/delta-streaming). See also the [Structured Streaming Programming Guide](https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html) on the Apache Spark website. ## Read a stream of changes from a table While the stream is writing to the Delta table, you can also read from that table as streaming source. For example, you can start another streaming query that prints all the changes made to the Delta table. You can specify which version Structured Streaming should start from by providing the `startingVersion` or `startingTimestamp` option to get changes from that point onwards. See [Structured Streaming](/delta-streaming/#specify-initial-position) for details. ```python stream2 = spark.readStream.format("delta").load("/tmp/delta-table").writeStream.format("console").start() ``` ```scala val stream2 = spark.readStream.format("delta").load("/tmp/delta-table").writeStream.format("console").start() ``` ```java StreamingQuery stream2 = spark.readStream().format("delta").load("/tmp/delta-table").writeStream().format("console").start(); ``` ================================================ FILE: docs/src/content/docs/redshift-spectrum-integration.mdx ================================================ --- title: AWS Redshift Spectrum connector description: Learn how to set up an integration to enable you to read Delta tables from AWS Redshift. --- import { Aside, Tabs, TabItem } from "@astrojs/starlight/components"; A Delta table can be read by AWS Redshift Spectrum using a _manifest file_, which is a text file containing the list of data files to read for querying a Delta table. This article describes how to set up a AWS Redshift Spectrum to Delta Lake integration using manifest files and query Delta tables. ## Set up a AWS Redshift Spectrum to Delta Lake integration and query Delta tables You set up a AWS Redshift Spectrum to Delta Lake integration using the following steps. ### Step 1: Generate manifests of a Delta table using Apache Spark Run the `generate` operation on a Delta table at location ``: ```sql GENERATE symlink_format_manifest FOR TABLE delta.`` ``` ```scala val deltaTable = DeltaTable.forPath() deltaTable.generate("symlink_format_manifest") ``` ```java DeltaTable deltaTable = DeltaTable.forPath(); deltaTable.generate("symlink_format_manifest"); ``` ```python deltaTable = DeltaTable.forPath() deltaTable.generate("symlink_format_manifest") ``` See [Generate a manifest file](/delta-utility/#generate-a-manifest-file) for details. The `generate` operation generates manifest files at `/_symlink_format_manifest/`. In other words, the files in this directory contain the names of the data files (that is, Parquet files) that should be read for reading a snapshot of the Delta table. ### Step 2: Configure AWS Redshift Spectrum to read the generated manifests Run the following commands in your AWS Redshift Spectrum environment. 1. Define a new external table in AWS Redshift Spectrum using the format `SymlinkTextInputFormat` and the manifest location `/_symlink_format_manifest/`. ```sql CREATE EXTERNAL TABLE mytable ([(col_name1 col_datatype1, ...)]) [PARTITIONED BY (col_name2 col_datatype2, ...)] ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.SymlinkTextInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' LOCATION '/_symlink_format_manifest/' -- location of the generated manifest ``` `SymlinkTextInputFormat` configures AWS Redshift Spectrum to compute file splits for `mytable` by reading the manifest file instead of using a directory listing to find data files. Replace `mytable` with the name of the external table and `` with the absolute path to the Delta table. - You cannot use this table definition in Apache Spark; it can be used only by AWS Redshift Spectrum. 2. If the Delta table is partitioned, you must add the partitions explicitly to the AWS Redshift Spectrum table. This is needed because the manifest of a partitioned table is itself partitioned in the same directory structure as the table. - For every partition in the table, run the following in AWS Redshift Spectrum, either directly in AWS Redshift Spectrum, or using the AWS CLI or [Data API](https://docs.aws.amazon.com/redshift/latest/mgmt/data-api.html): ```sql ALTER TABLE mytable.redshiftdeltatable ADD IF NOT EXISTS PARTITION (col_name=col_value) LOCATION '/_symlink_format_manifest/col_name=col_value' ``` This steps will provide you with a [consistent](#data-consistency) view of the Delta table. ### Step 3: Update manifests When data in a Delta table is updated, you must regenerate the manifests using either of the following approaches: - **Update explicitly**: After all the data updates, you can run the `generate` operation to update the manifests. - **Update automatically**: You can configure a Delta table so that all write operations on the table automatically update the manifests. To enable this automatic mode, set the corresponding table property using the following SQL command. ```sql ALTER TABLE delta.`` SET TBLPROPERTIES(delta.compatibility.symlinkFormatManifest.enabled=true) ``` To disable this automatic mode, set this property to `false`. Whether to update automatically or explicitly depends on the concurrent nature of write operations on the Delta table and the desired data consistency. For example, if automatic mode is enabled, concurrent write operations lead to concurrent overwrites to the manifest files. With such unordered writes, the manifest files are not guaranteed to point to the latest version of the table after the write operations complete. Hence, if concurrent writes are expected and you want to avoid stale manifests, you should consider explicitly updating the manifest after the expected write operations have completed. In addition, if your table is partitioned, then you must add any new partitions or remove deleted partitions by following the same process as described in the preceding step. ## Limitations The AWS Redshift Spectrum integration has known limitations in its behavior. ### Data consistency Whenever Delta Lake generates updated manifests, it atomically overwrites existing manifest files. Therefore, AWS Redshift Spectrum will always see a consistent view of the data files; it will see all of the old version files or all of the new version files. However, the granularity of the consistency guarantees depends on whether or not the table is partitioned. - **Unpartitioned tables**: All the files names are written in one manifest file which is updated atomically. In this case AWS Redshift Spectrum will see full table snapshot consistency. - **Partitioned tables**: A manifest file is partitioned in the same Hive-partitioning-style directory structure as the original Delta table. This means that each partition is updated atomically, and AWS Redshift Spectrum will see a consistent view of each partition but not a consistent view across partitions. Furthermore, since all manifests of all partitions cannot be updated together, concurrent attempts to generate manifests can lead to different partitions having manifests of different versions. While this consistency guarantee under data change is weaker than that of reading Delta tables with Spark, it is still stronger than formats like Parquet as they do not provide partition-level consistency. Depending on what storage system you are using for Delta tables, it is possible to get incorrect results when AWS Redshift Spectrum concurrently queries the manifest while the manifest files are being rewritten. In file system implementations that lack atomic file overwrites, a manifest file may be momentarily unavailable. Hence, use manifests with caution if their updates are likely to coincide with queries from AWS Redshift Spectrum. ### Performance This is an experimental integration and its performance and scalability characteristics have not yet been tested. ### Schema evolution Delta Lake supports schema evolution and queries on a Delta table automatically use the latest schema regardless of the schema defined in the table in the Hive metastore. However, AWS Redshift Spectrum uses the schema defined in its table definition, and will not query with the updated schema until the table definition is updated to the new schema. ================================================ FILE: docs/src/content/docs/releases.mdx ================================================ --- title: Releases description: Learn about Delta Lake releases. --- ## Release notes The [GitHub releases page](https://github.com/delta-io/delta/releases/) describes features of each release. ## Compatibility with Apache Spark The following table lists Delta Lake versions and their compatible Apache Spark versions. | Delta Lake version | Apache Spark version | | ------------------ | ----------------------- | | 4.0.x | 4.0.x | | 3.3.x | 3.5.x | | 3.2.x | 3.5.x | | 3.1.x | 3.5.x | | 3.0.x | 3.5.x | | 2.4.x | 3.4.x | | 2.3.x | 3.3.x | | 2.2.x | 3.3.x | | 2.1.x | 3.3.x | | 2.0.x | 3.2.x | | 1.2.x | 3.2.x | | 1.1.x | 3.2.x | | 1.0.x | 3.1.x | | 0.7.x and 0.8.x | 3.0.x | | Below 0.7.0 | 2.4.2 - 2.4._\_ | ================================================ FILE: docs/src/content/docs/snowflake-integration.mdx ================================================ --- title: Snowflake connector description: Learn how to set up an integration to enable you to read Delta tables from Snowflake. --- import { Aside, Tabs, TabItem, Steps } from "@astrojs/starlight/components"; Visit the [Snowflake Delta Lake support](https://docs.snowflake.com/en/user-guide/tables-external-intro.html#delta-lake-support) documentation to use the connector. A Delta table can be read by Snowflake using a _manifest file_, which is a text file containing the list of data files to read for querying a Delta table. This article describes how to set up a Delta Lake to Snowflake integration using manifest files and query Delta tables. ## Set up a Delta Lake to Snowflake integration and query Delta tables You set up a Delta Lake to Snowflake integration using the following steps. ### Step 1: Generate manifests of a Delta table using Apache Spark Run the `generate` operation on a Delta table at location ``: ```sql GENERATE symlink_format_manifest FOR TABLE delta.`` ``` ```scala val deltaTable = DeltaTable.forPath() deltaTable.generate("symlink_format_manifest") ``` ```java DeltaTable deltaTable = DeltaTable.forPath(); deltaTable.generate("symlink_format_manifest"); ``` ```python deltaTable = DeltaTable.forPath() deltaTable.generate("symlink_format_manifest") ``` See [Generate a manifest file](/delta-utility/#generate-a-manifest-file) for details. The `generate` operation generates manifest files at `/_symlink_format_manifest/`. In other words, the files in this directory contain the names of the data files (that is, Parquet files) that should be read for reading a snapshot of the Delta table. ### Step 2: Configure Snowflake to read the generated manifests Run the following commands in your Snowflake environment. #### Define an external table on the manifest files To define an external table in Snowflake, you must first [define a external stage](https://docs.snowflake.net/manuals/user-guide/data-load-s3-create-stage.html) `my_staged_table` that points to the Delta table. In Snowflake, run the following. ```sql create or replace stage my_staged_table url='' ``` Replace `` with the full path to the Delta table. Using this stage, you can [define a table](https://docs.snowflake.net/manuals/sql-reference/sql/create-external-table.html) `delta_manifest_table` that reads the file names specified in the manifest files as follows: ```sql CREATE OR REPLACE EXTERNAL TABLE delta_manifest_table( filename VARCHAR AS split_part(VALUE:c1, '/', -1) ) WITH LOCATION = @my_staged_table/_symlink_format_manifest/ FILE_FORMAT = (TYPE = CSV) PATTERN = '.*[/]manifest' AUTO_REFRESH = true; ``` #### Define an external table on Parquet files You can define a table `my_parquet_data_table` that reads all the Parquet files in the Delta table. ```sql CREATE OR REPLACE EXTERNAL TABLE my_parquet_data_table( id INT AS (VALUE:id::INT), part INT AS (VALUE:part::INT), ..., parquet_filename VARCHAR AS split_part(metadata$filename, '/', -1) ) WITH LOCATION = @my_staged_table/ FILE_FORMAT = (TYPE = PARQUET) PATTERN = '.*[/]part-[^/]*[.]parquet' AUTO_REFRESH = true; ``` If your Delta table is partitioned, then you will have to explicitly extract the partition values in the table definition. For example, if the table was partitioned by a single integer column named `part`, you can extract the values as follows: ```sql CREATE OR REPLACE EXTERNAL TABLE my_parquet_data_partitioned_table( id INT AS (VALUE:id::INT), part INT AS ( nullif( regexp_replace(metadata$filename, '.*part\\=(.*)\\/.*', '\\1'), '__HIVE_DEFAULT_PARTITION__' )::INT ), ..., parquet_filename VARCHAR AS split_part(metadata$filename, '/', -1) ) WITH LOCATION = @my_staged_partitioned_table/ FILE_FORMAT = (TYPE = PARQUET) PATTERN = '.*[/]part-[^/]*[.]parquet' AUTO_REFRESH = true; ``` The regular expression is used to extract the partition value for the column `part`. Querying the Delta table as this Parquet table will produce incorrect results because this query will read all the Parquet files in this table rather than only those that define a consistent snapshot of the table. You can use the manifest table to get a consistent snapshot data. #### Define view to get correct contents of the Delta table using the manifest table To read only the rows belonging to the consistent snapshot defined in the generated manifests, you can apply a filter to keep only the rows in the Parquet table that came from the files defined in the manifest table. ```sql CREATE OR REPLACE VIEW my_delta_table AS SELECT id, part, ... FROM my_parquet_data_table WHERE parquet_filename IN ( SELECT filename FROM delta-manifest-table ); ``` Querying this view will provide you with a [consistent](#data-consistency) view of the Delta table. ### Step 3: Update manifests When data in a Delta table is updated, you must regenerate the manifests using either of the following approaches: - **Update explicitly**: After all the data updates, you can run the `generate` operation to update the manifests. - **Update automatically**: You can configure a Delta table so that all write operations on the table automatically update the manifests. To enable this automatic mode, set the corresponding table property using the following SQL command. ```sql ALTER TABLE delta.`` SET TBLPROPERTIES(delta.compatibility.symlinkFormatManifest.enabled=true) ``` ## Limitations The Snowflake integration has known limitations in its behavior. ### Data consistency Whenever Delta Lake generates updated manifests, it atomically overwrites existing manifest files. Therefore, Snowflake will always see a consistent view of the data files; it will see all of the old version files or all of the new version files. However, the granularity of the consistency guarantees depends on whether the table is partitioned or not. - **Unpartitioned tables**: All the files names are written in one manifest file which is updated atomically. In this case Snowflake will see full table snapshot consistency. - **Partitioned tables**: A manifest file is partitioned in the same Hive-partitioning-style directory structure as the original Delta table. This means that each partition is updated atomically, and Snowflake will see a consistent view of each partition but not a consistent view across partitions. Furthermore, since all manifests of all partitions cannot be updated together, concurrent attempts to generate manifests can lead to different partitions having manifests of different versions. Depending on what storage system you are using for Delta tables, it is possible to get incorrect results when Snowflake concurrently queries the manifest while the manifest files are being rewritten. In file system implementations that lack atomic file overwrites, a manifest file may be momentarily unavailable. Hence, use manifests with caution if their updates are likely to coincide with queries from Snowflake. ### Performance This is an experimental integration and its performance and scalability characteristics have not yet been tested. ### Schema evolution Delta Lake supports schema evolution and queries on a Delta table automatically use the latest schema regardless of the schema defined in the table in the Hive metastore. However, Snowflake uses the schema defined in its table definition, and will not query with the updated schema until the table definition is updated to the new schema. ================================================ FILE: docs/src/content/docs/table-properties.mdx ================================================ --- title: Delta Table Properties Reference description: Access the list of available Delta table properties. --- | Property | Description | Data type | Default | |----------|-------------|-----------|---------| | `delta.appendOnly` | `true` for this Delta table to be append-only. If append-only, existing records cannot be deleted, and existing values cannot be updated. See [Table properties](/delta-batch/#table-properties). | `Boolean` | `false` | | `delta.checkpoint.writeStatsAsJson` | `true` for Delta Lake to write file statistics in checkpoints in JSON format for the `stats` column. | `Boolean` | `true` | | `delta.checkpoint.writeStatsAsStruct` | `true` for Delta Lake to write file statistics to checkpoints in struct format for the `stats_parsed` column and to write partition values as a struct for `partitionValues_parsed`. | `Boolean` | (none) | | `delta.compatibility.symlinkFormatManifest.enabled` | `true` for Delta Lake to configure the Delta table so that all write operations on the table automatically update the manifests. See [Update manifests](/presto-integration/#step-3-update-manifests). | `Boolean` | `false` | | `delta.dataSkippingNumIndexedCols` | The number of columns for Delta Lake to collect statistics about for data skipping. A value of `-1` means to collect statistics for all columns. Updating this property does not automatically collect statistics again; instead, it redefines the statistics schema of the Delta table. For example, it changes the behavior of future statistics collection (such as during appends and optimizations) as well as data skipping (such as ignoring column statistics beyond this number, even when such statistics exist). See [Data skipping](/optimizations-oss/#data-skipping). | `Int` | `32` | | `delta.deletedFileRetentionDuration` | The shortest duration for Delta Lake to keep logically deleted data files before deleting them physically. This is to prevent failures in stale readers after compactions or partition overwrites. This value should be large enough to ensure that: - It is larger than the longest possible duration of a job if you run `VACUUM` when there are concurrent readers or writers accessing the Delta table. - If you run a streaming query that reads from the table, that the query does not stop for longer than this value. Otherwise, the query may not be able to restart, as it must still read old files. See [Data retention](/delta-batch/#data-retention). | `CalendarInterval` | `interval 1 week` | | `delta.enableChangeDataFeed` | `true` to enable change data feed. See [Enable change data feed](/delta-change-data-feed/#enable-change-data-feed). | `Boolean` | `false` | | `delta.logRetentionDuration` | How long the history for a Delta table is kept. Each time a checkpoint is written, Delta Lake automatically cleans up log entries older than the retention interval. If you set this property to a large enough value, many log entries are retained. This should not impact performance as operations against the log are constant time. Operations on history are parallel but will become more expensive as the log size increases. See [Data retention](/delta-batch/#data-retention). | `CalendarInterval` | `interval 30 days` | | `delta.minReaderVersion` | The minimum required protocol reader version for a reader that allows to read from this Delta table. See [Versioning](/versioning). | `Int` | `1` | | `delta.minWriterVersion` | The minimum required protocol writer version for a writer that allows to write to this Delta table. See [Versioning](/versioning). | `Int` | `2` | | `delta.setTransactionRetentionDuration` | The shortest duration within which new snapshots will retain transaction identifiers (for example, `SetTransaction`s). When a new snapshot sees a transaction identifier older than or equal to the duration specified by this property, the snapshot considers it expired and ignores it. The `SetTransaction` identifier is used when making the writes idempotent. See [Idempotent table writes in foreachBatch](/delta-streaming/#idempotent-table-writes-in-foreachbatch) for details. | `CalendarInterval` | (none) | | `delta.checkpointPolicy` | `classic` for classic Delta Lake checkpoints. `v2` for v2 checkpoints. See [V2 Checkpoint Spec](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#v2-spec) for details. See [Versioning](/versioning) for details around compatibility. | `String` | `classic` | ================================================ FILE: docs/src/content/docs/versioning.mdx ================================================ --- title: How does Delta Lake manage feature compatibility? description: Learn how Delta table protocols are versioned. --- import { Aside, Tabs, TabItem } from "@astrojs/starlight/components"; Many Delta Lake optimizations require enabling Delta Lake features on a table. Delta Lake features are always backwards compatible, so tables written by a lower Delta Lake version can always be read and written by a higher Delta Lake version. Enabling some features breaks forward compatibility with workloads running in a lower Delta Lake version. For features that break forward compatibility, you must update all workloads that reference the upgraded tables to use a compliant Delta Lake version. ## What Delta Lake features require client upgrades? The following Delta Lake features break forward compatibility. Features are enabled on a table-by-table basis. | Feature | Requires Delta Lake version or later | Documentation | | --- | --- | --- | | `CHECK` constraints | [Delta Lake 0.8.0](https://github.com/delta-io/delta/releases/tag/v0.8.0) | [CHECK constraint](/delta-constraints/#check-constraint) | | Generated columns | [Delta Lake 1.0.0](https://github.com/delta-io/delta/releases/tag/v1.0.0) | [Use generated columns](/delta-batch/#use-generated-columns) | | Column mapping | [Delta Lake 1.2.0](https://github.com/delta-io/delta/releases/tag/v1.2.0) | [Delta column mapping](/delta-column-mapping/) | | Change data feed | [Delta Lake 2.0.0](https://github.com/delta-io/delta/releases/tag/v2.0.0) | [Change data feed](/delta-change-data-feed/) | | Deletion vectors | [Delta Lake 2.3.0](https://github.com/delta-io/delta/releases/tag/v2.3.0) | [What are deletion vectors?](/delta-deletion-vectors/) | | Table features | [Delta Lake 2.3.0](https://github.com/delta-io/delta/releases/tag/v2.3.0) | [What are table features?](#what-are-table-features) | | Timestamp without Timezone | [Delta Lake 2.4.0](https://github.com/delta-io/delta/releases/tag/v2.4.0) | [TimestampNTZType](https://spark.apache.org/docs/latest/sql-ref-datatypes.html) | | Iceberg Compatibility V1 | [Delta Lake 3.0.0](https://github.com/delta-io/delta/releases/tag/v3.0.0) | [IcebergCompatV1](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#iceberg-compatibility-v1) | | Iceberg Compatibility V2 | [Delta Lake 3.1.0](https://github.com/delta-io/delta/releases/tag/v3.1.0) | [IcebergCompatV2](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#iceberg-compatibility-v2) | | V2 Checkpoints | [Delta Lake 3.0.0](https://github.com/delta-io/delta/releases/tag/v3.0.0) | [V2 Checkpoint Spec](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#v2-spec) | | Domain metadata | [Delta Lake 3.0.0](https://github.com/delta-io/delta/releases/tag/v3.0.0) | [Domain Metadata Spec](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#domain-metadata) | | Clustering | [Delta Lake 3.1.0](https://github.com/delta-io/delta/releases/tag/v3.1.0) | [Use liquid clustering for Delta tables](/delta-clustering/) | | Row Tracking | [Delta Lake 3.2.0](https://github.com/delta-io/delta/releases/tag/v3.2.0) | [Use row tracking for Delta tables](/row-tracking/) | | Type widening (Preview) | [Delta Lake 3.2.0](https://github.com/delta-io/delta/releases/tag/v3.2.0) | [Delta type widening](/delta-type-widening/) | | Type widening | [Delta Lake 4.0.0](https://github.com/delta-io/delta/releases/tag/v4.0.0) | [Delta type widening](/delta-type-widening/) | | Identity columns | [Delta Lake 3.3.0](https://github.com/delta-io/delta/releases/tag/v3.3.0) | [Use identity columns](/delta-batch/#use-identity-columns) | | Variant Type | [Delta Lake 4.0.0](https://github.com/delta-io/delta/releases/tag/v4.0.0) | [Delta type widening](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#variant-data-type) | | Variant Shredding (Preview) | [Delta Lake 4.0.0](https://github.com/delta-io/delta/releases/tag/v4.0.0) | [Variant Shredding](https://github.com/delta-io/delta/blob/master/protocol_rfcs/variant-shredding.md) | | Checkpoint Protection | [Delta Lake 4.0.0](https://github.com/delta-io/delta/releases/tag/v4.0.0) | [Checkpoint Protection](https://github.com/delta-io/delta/blob/master/protocol_rfcs/checkpoint-protection.md) ## What is a table protocol specification? Every Delta table has a protocol specification which indicates the set of features that the table supports. The protocol specification is used by applications that read or write the table to determine if they can handle all the features that the table supports. If an application does not know how to handle a feature that is listed as supported in the protocol of a table, then that application is not be able to read or write that table. The protocol specification is separated into two components: the _read protocol_ and the _write protocol_. ### Read protocol The read protocol lists all features that a table supports and that an application must understand in order to read the table correctly. Upgrading the read protocol of a table requires that all reader applications support the added features. ### Write protocol The write protocol lists all features that a table supports and that an application must understand in order to write to the table correctly. Upgrading the write protocol of a table requires that all writer applications support the added features. It does not affect read-only applications, unless the read protocol is also upgraded. ## Which protocols must be upgraded? Some features require upgrading both the read protocol and the write protocol. Other features only require upgrading the write protocol. As an example, support for `CHECK` constraints is a write protocol feature: only writing applications need to know about `CHECK` constraints and enforce them. In contrast, column mapping requires upgrading both the read and write protocols. Because the data is stored differently in the table, reader applications must understand column mapping so they can read the data correctly. For more on upgrading, see [Upgrading protocol versions](#upgrading-protocol-versions). ## What are table features? In Delta Lake 2.3.0 and above, Delta Lake table features introduce granular flags specifying which features are supported by a given table. Table features are the successor to protocol versions and are designed with the goal of improved flexibility for clients that read and write Delta Lake. See [What is a protocol version?](#what-is-a-protocol-version). A Delta table feature is a marker that indicates that the table supports a particular feature. Every feature is either a write protocol feature (meaning it only upgrades the write protocol) or a read/write protocol feature (meaning both read and write protocols are upgraded to enable the feature). To learn more about supported table features in Delta Lake, see the [Delta Lake protocol](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#valid-feature-names-in-table-features). ## Do table features change how Delta Lake features are enabled? If you only interact with Delta tables through Delta Lake, you can continue to track support for Delta Lake features using minimum Delta Lake requirements. If you read and write from Delta tables using other systems, you might need to consider how table features impact compatibility, because there is a risk that the system could not understand the upgraded protocol versions. ## What is a protocol version? A protocol version is a protocol number that indicates a particular grouping of table features. In Delta Lake 2.3.0 and below, you cannot enable table features individually. Protocol versions bundle a group of features. Delta tables specify a separate protocol version for read protocol and write protocol. The transaction log for a Delta table contains protocol versioning information that supports Delta Lake evolution. The protocol versions bundle all features from previous protocols. See [Features by protocol version](#features-by-protocol-version). ## Features by protocol version The following table shows minimum protocol versions required for Delta Lake features. | Feature | `minWriterVersion` | `minReaderVersion` | Documentation | | --- | --- | --- | --- | | Basic functionality | 2 | 1 | [Welcome to the Delta Lake documentation](/) | | `CHECK` constraints | 3 | 1 | [CHECK constraint](/delta-constraints/#check-constraint) | | Change data feed | 4 | 1 | [Change data feed](/delta-change-data-feed/) | | Generated columns | 4 | 1 | [Use generated columns](/delta-batch/#use-generated-columns) | | Column mapping | 5 | 2 | [Delta column mapping](/delta-column-mapping/) | | Identity columns | 6 | 1 | [Use identity columns](/delta-batch/#use-identity-columns) | | Table features read | 7 | 1 | [What are table features?](#what-are-table-features) | | Table features write | 7 | 3 | [What are table features?](#what-are-table-features) | | Deletion vectors | 7 | 3 | [What are deletion vectors?](/delta-deletion-vectors/) | | Timestamp without Timezone | 7 | 3 | [TimestampNTZType](https://spark.apache.org/docs/latest/sql-ref-datatypes.html) | | Iceberg Compatibility V1 | 7 | 2 | [IcebergCompatV1](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#iceberg-compatibility-v1) | | V2 Checkpoints | 7 | 3 | [V2 Checkpoint Spec](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#v2-spec) | | Vacuum Protocol Check | 7 | 3 | [Vacuum Protocol Check Spec](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#vacuum-protocol-check) | | Row Tracking | 7 | 3 | [Use row tracking for Delta tables](/delta-row-tracking/) | | Type widening (Preview) | 7 | 3 | [Delta type widening](/delta-type-widening/) | | Type widening | 7 | 3 | [Delta type widening](/delta-type-widening/) | | Variant Type | 7 | 3 | [Variant Type](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#variant-data-type) | | Variant Shredding (Preview) | 7 | 3 | [Variant Shredding](https://github.com/delta-io/delta/blob/master/protocol_rfcs/variant-shredding.md) | ## Upgrading protocol versions You can choose to manually update a table to a newer protocol version. We recommend using the lowest protocol versions that support the Delta Lake features required for your table. Upgrading the writer protocol might cause less disruption than upgrading the reader protocol since systems and workloads using older Delta Lake versions can still read from tables, even if they do not support the updated writer protocol. To upgrade a table to a newer protocol version, use the `DeltaTable.upgradeTableProtocol` method: ```sql -- Upgrades the reader protocol version to 1 and the writer protocol version to 3. ALTER TABLE SET TBLPROPERTIES('delta.minReaderVersion' = '1', 'delta.minWriterVersion' = '3') ``` ```python from delta.tables import DeltaTable delta = DeltaTable.forPath(spark, "path_to_table") # or DeltaTable.forName delta.upgradeTableProtocol(1, 3) # upgrades to readerVersion=1, writerVersion=3 ``` ```scala import io.delta.tables.DeltaTable val delta = DeltaTable.forPath(spark, "path_to_table") // or DeltaTable.forName delta.upgradeTableProtocol(1, 3) // Upgrades to readerVersion=1, writerVersion=3. ``` ================================================ FILE: docs/src/content.config.ts ================================================ import { defineCollection } from "astro:content"; import { docsLoader } from "@astrojs/starlight/loaders"; import { docsSchema } from "@astrojs/starlight/schema"; export const collections = { docs: defineCollection({ loader: docsLoader(), schema: docsSchema() }), }; ================================================ FILE: docs/src/env.d.ts ================================================ /* eslint-disable @typescript-eslint/triple-slash-reference */ /// /// ================================================ FILE: docs/src/pages/robots.txt.ts ================================================ import type { APIRoute } from "astro"; const getRobotsTxt = (sitemapURL: URL) => ` User-agent: * Allow: / Sitemap: ${sitemapURL.href} `; export const GET: APIRoute = ({ site }) => { const sitemapURL = new URL("sitemap-index.xml", site); return new Response(getRobotsTxt(sitemapURL)); }; ================================================ FILE: docs/src/styles/custom.css ================================================ /* Dark mode colors. */ :root { --sl-color-accent-low: #00273c; --sl-color-accent: #0072a4; --sl-color-accent-high: #90d2fd; --sl-color-white: #ffffff; --sl-color-gray-1: #eceef2; --sl-color-gray-2: #c0c2c7; --sl-color-gray-3: #888b96; --sl-color-gray-4: #545861; --sl-color-gray-5: #353841; --sl-color-gray-6: #24272f; --sl-color-black: #17181c; } /* Light mode colors. */ :root[data-theme="light"] { --sl-color-accent-low: #aedeff; --sl-color-accent: #0074a7; --sl-color-accent-high: #003652; --sl-color-white: #17181c; --sl-color-gray-1: #24272f; --sl-color-gray-2: #353841; --sl-color-gray-3: #545861; --sl-color-gray-4: #888b96; --sl-color-gray-5: #c0c2c7; --sl-color-gray-6: #eceef2; --sl-color-gray-7: #f5f6f8; --sl-color-black: #ffffff; } ================================================ FILE: docs/tsconfig.json ================================================ { "extends": "astro/tsconfigs/strictest" } ================================================ FILE: examples/README.md ================================================ ## Delta Lake examples In this folder there are examples taken from the delta.io quickstart guide and docs. They are available in both Scala and Python and can be run if the prerequisites are satisfied. ### Prerequisites * See [Set up Apache Spark with Delta Lake](https://docs.delta.io/latest/quick-start.html#set-up-apache-spark-with-delta-lake). ### Instructions * To run an example in Python run `spark-submit --packages io.delta:delta-spark_2.12:{Delta Lake version} PATH/TO/EXAMPLE` * To run the Scala examples, `cd examples/scala` and run `./build/sbt "runMain example.{Example class name}"` e.g. `./build/sbt "runMain example.Quickstart"` ================================================ FILE: examples/python/change_data_feed.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from pyspark.sql import SparkSession from pyspark.sql.functions import col, expr from delta.tables import DeltaTable import shutil path = "/tmp/delta-change-data-feed/student" otherPath = "/tmp/delta-change-data-feed/student_source" # Enable SQL commands and Update/Delete/Merge for the current spark session. # we need to set the following configs spark = SparkSession.builder \ .appName("Change Data Feed") \ .master("local[*]") \ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \ .getOrCreate() def cleanup(): shutil.rmtree(path, ignore_errors=True) shutil.rmtree(otherPath, ignore_errors=True) spark.sql("DROP TABLE IF EXISTS student") spark.sql("DROP TABLE IF EXISTS student_source") def read_cdc_by_table_name(starting_version): return spark.read.format("delta") \ .option("readChangeFeed", "true") \ .option("startingVersion", str(starting_version)) \ .table("student") \ .orderBy("_change_type", "id") def stream_cdc_by_table_name(starting_version): return spark.readStream.format("delta") \ .option("readChangeFeed", "true") \ .option("startingVersion", str(starting_version)) \ .table("student") \ .writeStream \ .format("console") \ .option("numRows", 1000) \ .start() cleanup() try: # =============== Create student table =============== spark.sql('''CREATE TABLE student (id INT, name STRING, age INT) USING DELTA PARTITIONED BY (age) TBLPROPERTIES (delta.enableChangeDataFeed = true) LOCATION '{0}' '''.format(path)) spark.range(0, 10) \ .selectExpr( "CAST(id as INT) as id", "CAST(id as STRING) as name", "CAST(id % 4 + 18 as INT) as age") \ .write.format("delta").mode("append").save(path) # v1 # =============== Show table data + changes =============== print("(v1) Initial Table") spark.read.format("delta").load(path).orderBy("id").show() print("(v1) CDC changes") read_cdc_by_table_name(1).show() table = DeltaTable.forPath(spark, path) # =============== Perform UPDATE =============== print("(v2) Updated id -> id + 1") table.update(set={"id": expr("id + 1")}) # v2 read_cdc_by_table_name(2).show() # =============== Perform DELETE =============== print("(v3) Deleted where id >= 7") table.delete(condition=expr("id >= 7")) # v3 read_cdc_by_table_name(3).show() # =============== Perform partition DELETE =============== print("(v4) Deleted where age = 18") table.delete(condition=expr("age = 18")) # v4, partition delete read_cdc_by_table_name(4).show() # =============== Create source table for MERGE =============== spark.sql('''CREATE TABLE student_source (id INT, name STRING, age INT) USING DELTA LOCATION '{0}' '''.format(otherPath)) spark.range(0, 3) \ .selectExpr( "CAST(id as INT) as id", "CAST(id as STRING) as name", "CAST(id % 4 + 18 as INT) as age") \ .write.format("delta").mode("append").saveAsTable("student_source") source = spark.sql("SELECT * FROM student_source") # =============== Perform MERGE =============== table.alias("target") \ .merge( source.alias("source"), "target.id = source.id")\ .whenMatchedUpdate(set={"id": "source.id", "age": "source.age + 10"}) \ .whenNotMatchedInsertAll() \ .execute() # v5 print("(v5) Merged with a source table") read_cdc_by_table_name(5).show() # =============== Stream changes =============== print("Streaming by table name") cdfStream = stream_cdc_by_table_name(0) cdfStream.awaitTermination(10) cdfStream.stop() finally: cleanup() spark.stop() ================================================ FILE: examples/python/delta_connect.py ================================================ # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """ To run this example you must follow these steps: Requirements: - Using Java 17 - Spark 4.0.0-preview1+ - delta-spark (python package) 4.0.0rc1+ and pyspark 4.0.0.dev1+ (1) Start a local Spark connect server using this command: sbin/start-connect-server.sh \ --packages org.apache.spark:spark-connect_2.13:4.0.0-preview1,io.delta:delta-connect-server_2.13:{DELTA_VERSION},io.delta:delta-spark_2.13:{DELTA_VERSION},com.google.protobuf:protobuf-java:3.25.1 \ --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" \ --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" \ --conf "spark.connect.extensions.relation.classes"="org.apache.spark.sql.connect.delta.DeltaRelationPlugin" \ --conf "spark.connect.extensions.command.classes"="org.apache.spark.sql.connect.delta.DeltaCommandPlugin" * Be sure to replace DELTA_VERSION with the version you are using (2) Set the SPARK_REMOTE environment variable to point to your local Spark server export SPARK_REMOTE="sc://localhost:15002" (3) Run this file i.e. python3 examples/python/delta_connect.py """ import os from pyspark.sql import SparkSession from delta.tables import DeltaTable import shutil filePath = "/tmp/delta_connect" tableName = "delta_connect_table" def assert_dataframe_equals(df1, df2): assert(df1.collect().sort() == df2.collect().sort()) def cleanup(spark): shutil.rmtree(filePath, ignore_errors=True) spark.sql(f"DROP TABLE IF EXISTS {tableName}") # --------------------- Set up Spark Connect spark session ------------------------ assert os.getenv("SPARK_REMOTE"), "Must point to Spark Connect server using SPARK_REMOTE" spark = SparkSession.builder \ .appName("delta_connect") \ .remote(os.getenv("SPARK_REMOTE")) \ .getOrCreate() # Clean up any previous runs cleanup(spark) # -------------- Try reading non-existent table (should fail with an exception) ---------------- # Using forPath try: DeltaTable.forPath(spark, filePath).toDF().show() except Exception as e: assert "DELTA_MISSING_DELTA_TABLE" in str(e) else: assert False, "Expected exception to be thrown for missing table" # Using forName try: DeltaTable.forName(spark, tableName).toDF().show() except Exception as e: assert "DELTA_MISSING_DELTA_TABLE" in str(e) else: assert False, "Expected exception to be thrown for missing table" # ------------------------ Write basic table and check that results match ---------------------- # By table name spark.range(5).write.format("delta").saveAsTable(tableName) assert_dataframe_equals(DeltaTable.forName(spark, tableName).toDF(), spark.range(5)) assert_dataframe_equals(spark.read.format("delta").table(tableName), spark.range(5)) assert_dataframe_equals(spark.sql(f"SELECT * FROM {tableName}"), spark.range(5)) # By table path spark.range(10).write.format("delta").save(filePath) assert_dataframe_equals(DeltaTable.forPath(spark, filePath).toDF(), spark.range(10)) assert_dataframe_equals(spark.read.format("delta").load(filePath), spark.range(10)) assert_dataframe_equals(spark.sql(f"SELECT * FROM delta.`{filePath}`"), spark.range(10)) # ---------------------------------- Clean up ---------------------------------------- cleanup(spark) ================================================ FILE: examples/python/image_storage.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # This example shows # 1 - How to load the TensorFlow Flowers Images into a dataframe # 2 - Manipulate the dataframe # 3 - Write the dataframe to a Delta Lake table # 4 - Read the new Delta Lake table import pyspark.sql.functions as fn from pyspark.sql import SparkSession from delta import configure_spark_with_delta_pip import shutil from urllib import request import os # To run this example directly, set up the spark session using the following 2 commands # You will need to run using Python3 # You will also need to install the python packages pyspark and delta-spark, we advise using pip builder = ( SparkSession.builder .appName('image_storage') .config('spark.sql.extensions', 'io.delta.sql.DeltaSparkSessionExtension') .config('spark.sql.catalog.spark_catalog', 'org.apache.spark.sql.delta.catalog.DeltaCatalog') ) # This is only for testing staged release artifacts. Ignore this completely. if os.getenv('EXTRA_MAVEN_REPO'): builder = builder.config("spark.jars.repositories", os.getenv('EXTRA_MAVEN_REPO')) spark = configure_spark_with_delta_pip(builder).getOrCreate() # Flowers dataset from the TensorFlow team - https://www.tensorflow.org/datasets/catalog/tf_flowers imageGzipUrl = "https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz" imageGzipPath = "/tmp/flower_photos.tgz" imagePath = "/tmp/image-folder" deltaPath = "/tmp/delta-table" # Clear previous run's zipper file, image folder and delta tables if os.path.exists(imageGzipPath): os.remove(imageGzipPath) shutil.rmtree(imagePath, ignore_errors=True) shutil.rmtree(deltaPath, ignore_errors=True) request.urlretrieve(imageGzipUrl, imageGzipPath) shutil.unpack_archive(imageGzipPath, imagePath) # read the images from the flowers dataset images = spark.read.format("binaryFile").\ option("recursiveFileLookup", "true").\ option("pathGlobFilter", "*.jpg").\ load(imagePath) # Knowing the file path, extract the flower type and filename using substring_index # Remember, Spark dataframes are immutable, here we are just reusing the images dataframe images = images.withColumn("flowerType_filename", fn.substring_index(images.path, "/", -2)) images = images.withColumn("flowerType", fn.substring_index(images.flowerType_filename, "/", 1)) images = images.withColumn("filename", fn.substring_index(images.flowerType_filename, "/", -1)) images = images.drop("flowerType_filename") images.show() # Select the columns we want to write out to df = images.select("path", "content", "flowerType", "filename").repartition(4) df.show() # Write out the delta table to the given path, this will overwrite any table that is currently there df.write.format("delta").mode("overwrite").save(deltaPath) # Reads the delta table that was just written dfDelta = spark.read.format("delta").load(deltaPath) dfDelta.show() # Cleanup if os.path.exists(imageGzipPath): os.remove(imageGzipPath) shutil.rmtree(imagePath) shutil.rmtree(deltaPath) ================================================ FILE: examples/python/missing_delta_storage_jar.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from pyspark.sql import SparkSession import shutil path = "/tmp/delta-table/missing_logstore_jar" try: # Clear any previous runs shutil.rmtree(path, ignore_errors=True) spark = SparkSession.builder \ .appName("missing logstore jar") \ .master("local[*]") \ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \ .getOrCreate() spark.range(0, 5).write.format("delta").save(path) except Exception as e: assert "Please ensure that the delta-storage dependency is included." in str(e) print("SUCCESS - error was thrown, as expected") else: assert False, "The write to the delta table should have thrown without the delta-storage JAR." finally: # cleanup shutil.rmtree(path, ignore_errors=True) ================================================ FILE: examples/python/quickstart.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from pyspark.sql import SparkSession from pyspark.sql.functions import col, expr from delta.tables import DeltaTable import shutil # Clear any previous runs shutil.rmtree("/tmp/delta-table", ignore_errors=True) # Enable SQL commands and Update/Delete/Merge for the current spark session. # we need to set the following configs spark = SparkSession.builder \ .appName("quickstart") \ .master("local[*]") \ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \ .getOrCreate() # Create a table print("############# Creating a table ###############") data = spark.range(0, 5) data.write.format("delta").save("/tmp/delta-table") # Read the table print("############ Reading the table ###############") df = spark.read.format("delta").load("/tmp/delta-table") df.show() # Upsert (merge) new data print("########### Upsert new data #############") newData = spark.range(0, 20) deltaTable = DeltaTable.forPath(spark, "/tmp/delta-table") deltaTable.alias("oldData")\ .merge( newData.alias("newData"), "oldData.id = newData.id")\ .whenMatchedUpdate(set={"id": col("newData.id")})\ .whenNotMatchedInsert(values={"id": col("newData.id")})\ .execute() deltaTable.toDF().show() # Update table data print("########## Overwrite the table ###########") data = spark.range(5, 10) data.write.format("delta").mode("overwrite").save("/tmp/delta-table") deltaTable.toDF().show() deltaTable = DeltaTable.forPath(spark, "/tmp/delta-table") # Update every even value by adding 100 to it print("########### Update to the table(add 100 to every even value) ##############") deltaTable.update( condition=expr("id % 2 == 0"), set={"id": expr("id + 100")}) deltaTable.toDF().show() # Delete every even value print("######### Delete every even value ##############") deltaTable.delete(condition=expr("id % 2 == 0")) deltaTable.toDF().show() # Read old version of data using time travel print("######## Read old data using time travel ############") df = spark.read.format("delta").option("versionAsOf", 0).load("/tmp/delta-table") df.show() # cleanup shutil.rmtree("/tmp/delta-table") ================================================ FILE: examples/python/quickstart_sql.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from pyspark.sql import SparkSession tableName = "tbltestpython" # Enable SQL/DML commands and Metastore tables for the current spark session. # We need to set the following configs spark = SparkSession.builder \ .appName("quickstart_sql") \ .master("local[*]") \ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \ .getOrCreate() # Clear any previous runs spark.sql("DROP TABLE IF EXISTS " + tableName) spark.sql("DROP TABLE IF EXISTS newData") try: # Create a table print("############# Creating a table ###############") spark.sql("CREATE TABLE %s(id LONG) USING delta" % tableName) spark.sql("INSERT INTO %s VALUES 0, 1, 2, 3, 4" % tableName) # Read the table print("############ Reading the table ###############") spark.sql("SELECT * FROM %s" % tableName).show() # Upsert (merge) new data print("########### Upsert new data #############") spark.sql("CREATE TABLE newData(id LONG) USING parquet") spark.sql("INSERT INTO newData VALUES 3, 4, 5, 6") spark.sql('''MERGE INTO {0} USING newData ON {0}.id = newData.id WHEN MATCHED THEN UPDATE SET {0}.id = newData.id WHEN NOT MATCHED THEN INSERT * '''.format(tableName)) spark.sql("SELECT * FROM %s" % tableName).show() # Update table data print("########## Overwrite the table ###########") spark.sql("INSERT OVERWRITE %s select * FROM (VALUES 5, 6, 7, 8, 9) x (id)" % tableName) spark.sql("SELECT * FROM %s" % tableName).show() # Update every even value by adding 100 to it print("########### Update to the table(add 100 to every even value) ##############") spark.sql("UPDATE {0} SET id = (id + 100) WHERE (id % 2 == 0)".format(tableName)) spark.sql("SELECT * FROM %s" % tableName).show() # Delete every even value print("######### Delete every even value ##############") spark.sql("DELETE FROM {0} WHERE (id % 2 == 0)".format(tableName)) spark.sql("SELECT * FROM %s" % tableName).show() # Read old version of data using time travel print("######## Read old data using time travel ############") df = spark.read.format("delta").option("versionAsOf", 0).table(tableName) df.show() finally: # cleanup spark.sql("DROP TABLE " + tableName) spark.sql("DROP TABLE IF EXISTS newData") spark.stop() ================================================ FILE: examples/python/quickstart_sql_on_paths.py ================================================ from pyspark.sql import SparkSession import shutil table_dir = "/tmp/delta-table" # Clear any previous runs shutil.rmtree(table_dir, ignore_errors=True) # Enable SQL/DML commands and Metastore tables for the current spark session. # We need to set the following configs spark = SparkSession.builder \ .appName("quickstart_sql_on_paths") \ .master("local[*]") \ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \ .getOrCreate() # Clear any previous runs spark.sql("DROP TABLE IF EXISTS newData") try: # Create a table print("############# Creating a table ###############") spark.sql("CREATE TABLE delta.`%s`(id LONG) USING delta" % table_dir) spark.sql("INSERT INTO delta.`%s` VALUES 0, 1, 2, 3, 4" % table_dir) # Read the table print("############ Reading the table ###############") spark.sql("SELECT * FROM delta.`%s`" % table_dir).show() # Upsert (merge) new data print("########### Upsert new data #############") spark.sql("CREATE TABLE newData(id LONG) USING parquet") spark.sql("INSERT INTO newData VALUES 3, 4, 5, 6") spark.sql('''MERGE INTO delta.`{0}` AS data USING newData ON data.id = newData.id WHEN MATCHED THEN UPDATE SET data.id = newData.id WHEN NOT MATCHED THEN INSERT * '''.format(table_dir)) spark.sql("SELECT * FROM delta.`%s`" % table_dir).show() # Update table data print("########## Overwrite the table ###########") spark.sql("INSERT OVERWRITE delta.`%s` select * FROM (VALUES 5, 6, 7, 8, 9) x (id)" % table_dir) spark.sql("SELECT * FROM delta.`%s`" % table_dir).show() # Update every even value by adding 100 to it print("########### Update to the table(add 100 to every even value) ##############") spark.sql("UPDATE delta.`{0}` SET id = (id + 100) WHERE (id % 2 == 0)".format(table_dir)) spark.sql("SELECT * FROM delta.`%s`" % table_dir).show() # Delete every even value print("######### Delete every even value ##############") spark.sql("DELETE FROM delta.`{0}` WHERE (id % 2 == 0)".format(table_dir)) spark.sql("SELECT * FROM delta.`%s`" % table_dir).show() finally: # cleanup spark.sql("DROP TABLE IF EXISTS newData") spark.stop() ================================================ FILE: examples/python/streaming.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from pyspark.sql import SparkSession from pyspark.sql.functions import col from delta.tables import DeltaTable import shutil import random # Enable SQL commands and Update/Delete/Merge for the current spark session. # we need to set the following configs spark = SparkSession.builder \ .appName("streaming") \ .master("local[*]") \ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \ .getOrCreate() shutil.rmtree("/tmp/delta-streaming/", ignore_errors=True) # Create a table(key, value) of some data data = spark.range(8) data = data.withColumn("value", data.id + random.randint(0, 5000)) data.write.format("delta").save("/tmp/delta-streaming/delta-table") # Stream writes to the table print("####### Streaming write ######") streamingDf = spark.readStream.format("rate").load() stream = streamingDf.selectExpr("value as id").writeStream\ .format("delta")\ .option("checkpointLocation", "/tmp/delta-streaming/checkpoint")\ .start("/tmp/delta-streaming/delta-table2") stream.awaitTermination(10) stream.stop() # Stream reads from a table print("##### Reading from stream ######") stream2 = spark.readStream.format("delta").load("/tmp/delta-streaming/delta-table2")\ .writeStream\ .format("console")\ .start() stream2.awaitTermination(10) stream2.stop() # Streaming aggregates in Update mode print("####### Streaming upgrades in update mode ########") # Function to upsert microBatchOutputDF into Delta Lake table using merge def upsertToDelta(microBatchOutputDF, batchId): t = deltaTable.alias("t").merge(microBatchOutputDF.alias("s"), "s.id = t.id")\ .whenMatchedUpdateAll()\ .whenNotMatchedInsertAll()\ .execute() streamingAggregatesDF = spark.readStream.format("rate").load()\ .withColumn("id", col("value") % 10)\ .drop("timestamp") # Write the output of a streaming aggregation query into Delta Lake table deltaTable = DeltaTable.forPath(spark, "/tmp/delta-streaming/delta-table") print("############# Original Delta Table ###############") deltaTable.toDF().show() stream3 = streamingAggregatesDF.writeStream\ .format("delta") \ .foreachBatch(upsertToDelta) \ .outputMode("update") \ .start() stream3.awaitTermination(10) stream3.stop() print("########### DeltaTable after streaming upsert #########") deltaTable.toDF().show() # Streaming append and concurrent repartition using data change = false # tbl1 is the sink and tbl2 is the source print("############ Streaming appends with concurrent table repartition ##########") tbl1 = "/tmp/delta-streaming/delta-table4" tbl2 = "/tmp/delta-streaming/delta-table5" numRows = 10 spark.range(numRows).write.mode("overwrite").format("delta").save(tbl1) spark.read.format("delta").load(tbl1).show() spark.range(numRows, numRows * 10).write.mode("overwrite").format("delta").save(tbl2) # Start reading tbl2 as a stream and do a streaming write to tbl1 # Prior to Delta 0.5.0 this would throw StreamingQueryException: Detected a data update in the # source table. This is currently not supported. stream4 = spark.readStream.format("delta").load(tbl2).writeStream.format("delta")\ .option("checkpointLocation", "/tmp/delta-streaming/checkpoint/tbl1") \ .outputMode("append") \ .start(tbl1) # repartition table while streaming job is running spark.read.format("delta").load(tbl2).repartition(10).write\ .format("delta")\ .mode("overwrite")\ .option("dataChange", "false")\ .save(tbl2) stream4.awaitTermination(10) stream4.stop() print("######### After streaming write #########") spark.read.format("delta").load(tbl1).show() # cleanup shutil.rmtree("/tmp/delta-streaming/", ignore_errors=True) ================================================ FILE: examples/python/table_exists.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from pyspark.sql import SparkSession from pyspark.sql.utils import AnalysisException import shutil def exists(spark, filepath): """Checks if a delta table exists at `filepath`""" try: spark.read.load(path=filepath, format="delta") except AnalysisException as exception: if "is not a Delta table" in str(exception) or "Path does not exist" in str(exception): return False raise exception return True spark = SparkSession.builder \ .appName("table_exists") \ .master("local[*]") \ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \ .getOrCreate() filepath = "/tmp/table_exists" # Clear any previous runs shutil.rmtree(filepath, ignore_errors=True) # Verify table doesn't exist yet print(f"Verifying table does not exist at {filepath}") assert not exists(spark, filepath) # Create a delta table at filepath print(f"Creating delta table at {filepath}") data = spark.range(0, 5) data.write.format("delta").save(filepath) # Verify table now exists print(f"Verifying table exists at {filepath}") assert exists(spark, filepath) # Clean up shutil.rmtree(filepath) ================================================ FILE: examples/python/using_with_pip.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import shutil # flake8: noqa import os from pyspark.sql import SparkSession from delta import * builder = SparkSession.builder \ .appName("with-pip") \ .master("local[*]") \ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \ .config("spark.jars.repositories", \ ("https://maven-central.storage-download.googleapis.com/maven2/," "https://repo1.maven.org/maven2/")\ ) # This is only for testing staged release artifacts. Ignore this completely. if os.getenv('EXTRA_MAVEN_REPO'): builder = builder.config("spark.jars.repositories", os.getenv('EXTRA_MAVEN_REPO')) # This configuration tells Spark to download the Delta Lake JAR that is needed to operate # in Spark. Use this only when the Pypi package Delta Lake is locally installed with pip. # This configuration is not needed if the this python program is executed with # spark-submit or pyspark shell with the --package arguments. spark = configure_spark_with_delta_pip(builder).getOrCreate() # Clear previous run's delta-tables shutil.rmtree("/tmp/delta-table", ignore_errors=True) print("########### Create a Parquet table ##############") data = spark.range(0, 5) data.write.format("parquet").save("/tmp/delta-table") print("########### Convert to Delta ###########") DeltaTable.convertToDelta(spark, "parquet.`/tmp/delta-table`") print("########### Read table with DataFrames ###########") df = spark.read.format("delta").load("/tmp/delta-table") df.show() print("########### Read table with DeltaTable ###########") deltaTable = DeltaTable.forPath(spark, "/tmp/delta-table") deltaTable.toDF().show() print("########### All import submodules work ###########") from delta.exceptions import MetadataChangedException spark.stop() # cleanup shutil.rmtree("/tmp/delta-table") ================================================ FILE: examples/python/utilities.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from pyspark.sql import SparkSession from delta.tables import DeltaTable import shutil spark = SparkSession.builder \ .appName("utilities") \ .master("local[*]") \ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \ .config("spark.sql.sources.parallelPartitionDiscovery.parallelism", "4") \ .getOrCreate() # Clear previous run's delta-tables shutil.rmtree("/tmp/delta-table", ignore_errors=True) # Create a table print("########### Create a Parquet table ##############") data = spark.range(0, 5) data.write.format("parquet").save("/tmp/delta-table") # Convert to delta print("########### Convert to Delta ###########") DeltaTable.convertToDelta(spark, "parquet.`/tmp/delta-table`") # Read the table df = spark.read.format("delta").load("/tmp/delta-table") df.show() deltaTable = DeltaTable.forPath(spark, "/tmp/delta-table") print("######## Vacuum the table ########") deltaTable.vacuum() print("######## Describe history for the table ######") deltaTable.history().show() print("######## Describe details for the table ######") deltaTable.detail().show() # Generate manifest print("######## Generating manifest ######") deltaTable.generate("SYMLINK_FORMAT_MANIFEST") # SQL Vacuum print("####### SQL Vacuum #######") spark.sql("VACUUM '%s' RETAIN 169 HOURS" % "/tmp/delta-table").collect() # SQL describe history print("####### SQL Describe History ########") print(spark.sql("DESCRIBE HISTORY delta.`%s`" % ("/tmp/delta-table")).collect()) # cleanup shutil.rmtree("/tmp/delta-table") ================================================ FILE: examples/scala/.scalafmt.conf ================================================ version = "3.4.0" runner.dialect = scala213 ================================================ FILE: examples/scala/README.md ================================================ # delta scala examples This directory contains a set of spark & delta examples. Execute `./build/sbt run` and choose which main class to run. ``` Multiple main classes detected. Select one to run: [1] example.Quickstart [2] example.QuickstartSQL [3] example.QuickstartSQLOnPaths [4] example.Streaming [5] example.Utilities ``` You can specify delta lake version and scala version with environment variables `DELTA_VERSION`, `SCALA_VERSION` or editing `build.sbt`. If you are faced with `java.lang.IllegalAccessError: class org.apache.spark.storage.StorageUtils$ (in unnamed module @0x******) cannot access class sun.nio.ch.DirectBuffer (in module java.base) because module java.base does not export sun.nio.ch to unnamed module` when you use Java 9 or later, add jvm option in `build.sbt`. ```diff lazy val root = (project in file(".")) .settings( run / fork := true, + run / javaOptions ++= Seq( + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED" + ), ``` ================================================ FILE: examples/scala/build/sbt ================================================ #!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # # This file contains code from the Apache Spark project (original license above). # It contains modifications, which are licensed as follows: # # # Copyright (2021) The Delta Lake Project Authors. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # When creating new tests for Spark SQL Hive, the HADOOP_CLASSPATH must contain the hive jars so # that we can run Hive to generate the golden answer. This is not required for normal development # or testing. if [ -n "$HIVE_HOME" ]; then for i in "$HIVE_HOME"/lib/* do HADOOP_CLASSPATH="$HADOOP_CLASSPATH:$i" done export HADOOP_CLASSPATH fi realpath () { ( TARGET_FILE="$1" cd "$(dirname "$TARGET_FILE")" TARGET_FILE="$(basename "$TARGET_FILE")" COUNT=0 while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] do TARGET_FILE="$(readlink "$TARGET_FILE")" cd $(dirname "$TARGET_FILE") TARGET_FILE="$(basename $TARGET_FILE)" COUNT=$(($COUNT + 1)) done echo "$(pwd -P)/"$TARGET_FILE"" ) } if [[ "$JENKINS_URL" != "" ]]; then # Make Jenkins use Google Mirror first as Maven Central may ban us SBT_REPOSITORIES_CONFIG="$(dirname "$(realpath "$0")")/sbt-config/repositories" export SBT_OPTS="-Dsbt.override.build.repos=true -Dsbt.repository.config=$SBT_REPOSITORIES_CONFIG" fi . "$(dirname "$(realpath "$0")")"/sbt-launch-lib.bash declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy" declare -r sbt_opts_file=".sbtopts" declare -r etc_sbt_opts_file="/etc/sbt/sbtopts" usage() { cat < path to global settings/plugins directory (default: ~/.sbt) -sbt-boot path to shared boot directory (default: ~/.sbt/boot in 0.11 series) -ivy path to local Ivy repository (default: ~/.ivy2) -mem set memory options (default: $sbt_mem, which is $(get_mem_opts $sbt_mem)) -no-share use all local caches; no sharing -no-global uses global caches, but does not use global ~/.sbt directory. -jvm-debug Turn on JVM debugging, open at the given port. -batch Disable interactive mode # sbt version (default: from project/build.properties if present, else latest release) -sbt-version use the specified version of sbt -sbt-jar use the specified jar as the sbt launcher -sbt-rc use an RC version of sbt -sbt-snapshot use a snapshot version of sbt # java version (default: java from PATH, currently $(java -version 2>&1 | grep version)) -java-home alternate JAVA_HOME # jvm options and output control JAVA_OPTS environment variable, if unset uses "$java_opts" SBT_OPTS environment variable, if unset uses "$default_sbt_opts" .sbtopts if this file exists in the current directory, it is prepended to the runner args /etc/sbt/sbtopts if this file exists, it is prepended to the runner args -Dkey=val pass -Dkey=val directly to the java runtime -J-X pass option -X directly to the java runtime (-J is stripped) -S-X add -X to sbt's scalacOptions (-S is stripped) -PmavenProfiles Enable a maven profile for the build. In the case of duplicated or conflicting options, the order above shows precedence: JAVA_OPTS lowest, command line options highest. EOM } process_my_args () { while [[ $# -gt 0 ]]; do case "$1" in -no-colors) addJava "-Dsbt.log.noformat=true" && shift ;; -no-share) addJava "$noshare_opts" && shift ;; -no-global) addJava "-Dsbt.global.base=$(pwd)/project/.sbtboot" && shift ;; -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;; -sbt-dir) require_arg path "$1" "$2" && addJava "-Dsbt.global.base=$2" && shift 2 ;; -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;; -batch) exec /dev/null) if [[ ! $? ]]; then saved_stty="" fi } saveSttySettings trap onExit INT run "$@" exit_status=$? onExit ================================================ FILE: examples/scala/build/sbt-config/repositories ================================================ [repositories] local local-preloaded-ivy: file:///${sbt.preloaded-${sbt.global.base-${user.home}/.sbt}/preloaded/}, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext] local-preloaded: file:///${sbt.preloaded-${sbt.global.base-${user.home}/.sbt}/preloaded/} gcs-maven-central-mirror: https://maven-central.storage-download.googleapis.com/repos/central/data/ maven-central typesafe-ivy-releases: https://repo.typesafe.com/typesafe/ivy-releases/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly sbt-ivy-snapshots: https://repo.scala-sbt.org/scalasbt/ivy-snapshots/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly sbt-plugin-releases: https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/, [organization]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] repos-spark-packages: https://repos.spark-packages.org typesafe-releases: https://repo.typesafe.com/typesafe/releases/ ================================================ FILE: examples/scala/build/sbt-launch-lib.bash ================================================ #!/usr/bin/env bash # # A library to simplify using the SBT launcher from other packages. # Note: This should be used by tools like giter8/conscript etc. # TODO - Should we merge the main SBT script with this library? if test -z "$HOME"; then declare -r script_dir="$(dirname "$script_path")" else declare -r script_dir="$HOME/.sbt" fi declare -a residual_args declare -a java_args declare -a scalac_args declare -a sbt_commands declare -a maven_profiles if test -x "$JAVA_HOME/bin/java"; then echo -e "Using $JAVA_HOME as default JAVA_HOME." echo "Note, this will be overridden by -java-home if it is set." declare java_cmd="$JAVA_HOME/bin/java" else declare java_cmd=java fi echoerr () { echo 1>&2 "$@" } vlog () { [[ $verbose || $debug ]] && echoerr "$@" } dlog () { [[ $debug ]] && echoerr "$@" } acquire_sbt_jar () { SBT_VERSION=`awk -F "=" '/sbt\.version/ {print $2}' ./project/build.properties` URL1=${DEFAULT_ARTIFACT_REPOSITORY:-https://repo1.maven.org/maven2/}org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar JAR=build/sbt-launch-${SBT_VERSION}.jar sbt_jar=$JAR if [[ ! -f "$sbt_jar" ]]; then # Download sbt launch jar if it hasn't been downloaded yet if [ ! -f "${JAR}" ]; then # Download printf "Attempting to fetch sbt\n" JAR_DL="${JAR}.part" if [ $(command -v curl) ]; then curl --fail --location --silent ${URL1} > "${JAR_DL}" &&\ mv "${JAR_DL}" "${JAR}" elif [ $(command -v wget) ]; then wget --quiet ${URL1} -O "${JAR_DL}" &&\ mv "${JAR_DL}" "${JAR}" else printf "You do not have curl or wget installed, please install sbt manually from http://www.scala-sbt.org/\n" exit -1 fi fi if [ ! -f "${JAR}" ]; then # We failed to download printf "Our attempt to download sbt locally to ${JAR} failed. Please install sbt manually from http://www.scala-sbt.org/\n" exit -1 fi printf "Launching sbt from ${JAR}\n" fi } execRunner () { # print the arguments one to a line, quoting any containing spaces [[ $verbose || $debug ]] && echo "# Executing command line:" && { for arg; do if printf "%s\n" "$arg" | grep -q ' '; then printf "\"%s\"\n" "$arg" else printf "%s\n" "$arg" fi done echo "" } "$@" } addJava () { dlog "[addJava] arg = '$1'" java_args=( "${java_args[@]}" "$1" ) } enableProfile () { dlog "[enableProfile] arg = '$1'" maven_profiles=( "${maven_profiles[@]}" "$1" ) export SBT_MAVEN_PROFILES="${maven_profiles[@]}" } addSbt () { dlog "[addSbt] arg = '$1'" sbt_commands=( "${sbt_commands[@]}" "$1" ) } addResidual () { dlog "[residual] arg = '$1'" residual_args=( "${residual_args[@]}" "$1" ) } addDebugger () { addJava "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$1" } # a ham-fisted attempt to move some memory settings in concert # so they need not be dicked around with individually. get_mem_opts () { local mem=${1:-1000} local perm=$(( $mem / 4 )) (( $perm > 256 )) || perm=256 (( $perm < 4096 )) || perm=4096 local codecache=$(( $perm / 2 )) echo "-Xms${mem}m -Xmx${mem}m -XX:ReservedCodeCacheSize=${codecache}m" } require_arg () { local type="$1" local opt="$2" local arg="$3" if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then echo "$opt requires <$type> argument" 1>&2 exit 1 fi } is_function_defined() { declare -f "$1" > /dev/null } process_args () { while [[ $# -gt 0 ]]; do case "$1" in -h|-help) usage; exit 1 ;; -v|-verbose) verbose=1 && shift ;; -d|-debug) debug=1 && shift ;; -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;; -mem) require_arg integer "$1" "$2" && sbt_mem="$2" && shift 2 ;; -jvm-debug) require_arg port "$1" "$2" && addDebugger $2 && shift 2 ;; -batch) exec m.group(1) case None => throw new Exception("Could not parse version from version.sbt") } } def getMajorMinor(version: String): (Int, Int) = { val majorMinor = Try { val splitVersion = version.split('.') (splitVersion(0).toInt, splitVersion(1).toInt) } majorMinor match { case Success(_) => (majorMinor.get._1, majorMinor.get._2) case _ => throw new RuntimeException(s"Unsupported delta version: $version. " + s"Please check https://docs.delta.io/latest/releases.html") } } // Maps Delta version (major, minor) to the compatible Spark version. // Used as a fallback for local dev when SPARK_VERSION env var is not set. val lookupSparkVersion: PartialFunction[(Int, Int), String] = { // TODO: how to run integration tests for multiple Spark versions case (major, minor) if major >= 4 && minor >= 1 => "4.1.0" // version 4.0.0 case (major, minor) if major >= 4 => "4.0.0" // versions 3.3.x+ case (major, minor) if major >= 3 && minor >=3 => "3.5.3" // versions 3.0.0 to 3.2.x case (major, minor) if major >= 3 && minor <=2 => "3.5.0" // versions 2.4.x case (major, minor) if major == 2 && minor == 4 => "3.4.0" // versions 2.3.x case (major, minor) if major == 2 && minor == 3 => "3.3.2" // versions 2.2.x case (major, minor) if major == 2 && minor == 2 => "3.3.1" // versions 2.1.x case (major, minor) if major == 2 && minor == 1 => "3.3.0" // versions 1.0.0 to 2.0.x case (major, minor) if major == 1 || (major == 2 && minor == 0) => "3.2.1" // versions 0.7.x to 0.8.x case (major, minor) if major == 0 && (minor == 7 || minor == 8) => "3.0.2" // versions below 0.7 case (major, minor) if major == 0 && minor < 7 => "2.4.4" } val getScalaVersion = settingKey[String]( s"get scala version from environment variable SCALA_VERSION. If it doesn't exist, use $scala213" ) val getDeltaVersion = settingKey[String]( s"get delta version from environment variable DELTA_VERSION. If it doesn't exist, use $defaultDeltaVersion" ) val getDeltaArtifactName = settingKey[String]( s"get delta artifact name based on the delta version. either `delta-core` or `delta-spark`." ) val getIcebergSparkRuntimeArtifactName = settingKey[String]( s"get iceberg-spark-runtime name based on the delta version." ) getScalaVersion := { sys.env.get("SCALA_VERSION") match { case Some("2.13") | Some(`scala213`) => scala213 case Some(v) => println( s"[warn] Invalid SCALA_VERSION. Expected one of {2.13, $scala213} but " + s"got $v. Fallback to $scala213." ) scala213 case None => scala213 } } scalaVersion := getScalaVersion.value version := "0.1.0" getDeltaVersion := { sys.env.get("DELTA_VERSION") match { case Some(v) => println(s"Using DELTA_VERSION Delta version $v") v case None => println(s"Using default Delta version $defaultDeltaVersion") defaultDeltaVersion } } getDeltaArtifactName := { val deltaVersion = getDeltaVersion.value if (deltaVersion.charAt(0).asDigit >= 3) "delta-spark" else "delta-core" } val getSparkPackageSuffix = settingKey[String]( s"get package suffix for cross-build artifact name from environment variable SPARK_PACKAGE_SUFFIX. " + s"This is derived from CrossSparkVersions.scala (single source of truth)." ) getSparkPackageSuffix := { sys.env.getOrElse("SPARK_PACKAGE_SUFFIX", "") } val getSupportIceberg = settingKey[String]( s"get supportIceberg for cross-build artifact name from environment variable SUPPORT_ICEBERG. " + s"This is derived from CrossSparkVersions.scala (single source of truth)." ) getSupportIceberg := { sys.env.getOrElse("SUPPORT_ICEBERG", "false") } getIcebergSparkRuntimeArtifactName := { val (expMaj, expMin) = getMajorMinor(lookupSparkVersion.apply( getMajorMinor(getDeltaVersion.value))) s"iceberg-spark-runtime-$expMaj.$expMin" } lazy val extraMavenRepo = sys.env.get("EXTRA_MAVEN_REPO").toSeq.map { repo => resolvers += "Delta" at repo } lazy val java17Settings = Seq( fork := true, javaOptions ++= Seq( "--add-exports=java.base/sun.nio.ch=ALL-UNNAMED" ) ) // Use SPARK_VERSION env var if set, otherwise fall back to lookupSparkVersion (for local dev) def resolveSparkVersion(deltaVersion: String): String = { val envVersion = sys.env.getOrElse("SPARK_VERSION", "") if (envVersion.nonEmpty) envVersion else lookupSparkVersion.apply(getMajorMinor(deltaVersion)) } def getLibraryDependencies( deltaVersion: String, deltaArtifactName: String, icebergSparkRuntimeArtifactName: String, sparkPackageSuffix: String, scalaBinVersion: String, supportIceberg: String): Seq[ModuleID] = { // Package suffix comes from CrossSparkVersions.scala (single source of truth) // e.g., "" for default Spark, "_4.1" for Spark 4.1 val deltaCoreDep = "io.delta" % s"${deltaArtifactName}${sparkPackageSuffix}_${scalaBinVersion}" % deltaVersion val deltaIcebergDep = "io.delta" % s"delta-iceberg_${scalaBinVersion}" % deltaVersion val resolvedSparkVersion = resolveSparkVersion(deltaVersion) val baseDeps = Seq( deltaCoreDep, "org.apache.spark" %% "spark-sql" % resolvedSparkVersion, "org.apache.spark" %% "spark-hive" % resolvedSparkVersion, "org.apache.iceberg" % "iceberg-hive-metastore" % icebergVersion ) // Include Iceberg dependencies only if supportIceberg is enabled val icebergDeps = if (supportIceberg == "true") { getMajorMinor(deltaVersion) match { case (major, _) if major >= 4 => // Don't include the iceberg dependencies for 4.0.0rc1 and later Seq.empty case _ => Seq( deltaIcebergDep, "org.apache.iceberg" %% icebergSparkRuntimeArtifactName % icebergVersion, ) } } else { Seq.empty } baseDeps ++ icebergDeps } lazy val root = (project in file(".")) .settings( run / fork := true, name := "hello-world", crossScalaVersions := Seq(scala213), libraryDependencies ++= getLibraryDependencies( getDeltaVersion.value, getDeltaArtifactName.value, getIcebergSparkRuntimeArtifactName.value, getSparkPackageSuffix.value, scalaBinaryVersion.value, getSupportIceberg.value), libraryDependencies ++= Seq( "io.unitycatalog" %% "unitycatalog-spark" % unityCatalogVersion excludeAll( ExclusionRule(organization = "com.fasterxml.jackson.core"), ExclusionRule(organization = "com.fasterxml.jackson.module"), ExclusionRule(organization = "com.fasterxml.jackson.datatype"), ExclusionRule(organization = "com.fasterxml.jackson.dataformat") ), "io.unitycatalog" % "unitycatalog-server" % unityCatalogVersion excludeAll( ExclusionRule(organization = "com.fasterxml.jackson.core"), ExclusionRule(organization = "com.fasterxml.jackson.module"), ExclusionRule(organization = "com.fasterxml.jackson.datatype"), ExclusionRule(organization = "com.fasterxml.jackson.dataformat") ) ), dependencyOverrides ++= Seq( "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion, "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion, "com.fasterxml.jackson.dataformat" % "jackson-dataformat-yaml" % jacksonVersion, "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % jacksonVersion, "com.fasterxml.jackson.datatype" % "jackson-datatype-jdk8" % jacksonVersion ), extraMavenRepo, resolvers += Resolver.mavenLocal, scalacOptions ++= Seq( "-deprecation", "-feature" ), // Conditionally exclude IcebergCompatV2.scala when supportIceberg is "false" Compile / unmanagedSources / excludeFilter := { if (getSupportIceberg.value == "false") { HiddenFileFilter || "IcebergCompatV2.scala" } else { HiddenFileFilter } }, java17Settings ) ================================================ FILE: examples/scala/project/build.properties ================================================ # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # # This file contains code from the Apache Spark project (original license above). # It contains modifications, which are licensed as follows: # # # Copyright (2021) The Delta Lake Project Authors. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # sbt.version=1.9.9 ================================================ FILE: examples/scala/src/main/resources/log4j2.properties ================================================ # # Copyright (2021) The Delta Lake Project Authors. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # rootLogger.level = error rootLogger.appenderRef.stdout.ref = console appender.console.type = Console appender.console.name = console appender.console.layout.type = PatternLayout appender.console.layout.pattern = [%t] %-5p %c %x - %m%n ================================================ FILE: examples/scala/src/main/scala/example/ChangeDataFeed.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package example import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.sql.{DataFrame, SQLContext, SparkSession} import org.apache.spark.sql.streaming.{StreamingQuery} import io.delta.tables._ import org.apache.spark.sql.functions._ import org.apache.commons.io.FileUtils import java.io.File object ChangeDataFeed { def main(args: Array[String]): Unit = { val spark = SparkSession .builder() .appName("ChangeDataFeed") .master("local[*]") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config( "spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog" ) .getOrCreate() val path = "/tmp/delta-change-data-feed/student" val otherPath = "/tmp/delta-change-data-feed/student_source" def cleanup(): Unit = { Seq(path, otherPath).foreach { p => val file = new File(p) if (file.exists()) FileUtils.deleteDirectory(file) } spark.sql(s"DROP TABLE IF EXISTS student") spark.sql(s"DROP TABLE IF EXISTS student_source") } // Note: one could also read by path using `.load(path)` def readCDCByTableName(startingVersion: Int): DataFrame = { spark.read.format("delta") .option("readChangeFeed", "true") .option("startingVersion", startingVersion.toString) .table("student") .orderBy("_change_type", "id") } // Note: one could also stream by path using `.load(path)` def streamCDCByTableName(startingVersion: Int): StreamingQuery = { spark.readStream.format("delta") .option("readChangeFeed", "true") .option("startingVersion", startingVersion.toString) .table("student") .writeStream .format("console") .option("numRows", 1000) .start() } cleanup() try { // =============== Create student table =============== spark.sql( s""" |CREATE TABLE student (id INT, name STRING, age INT) |USING DELTA |PARTITIONED BY (age) |TBLPROPERTIES (delta.enableChangeDataFeed = true) |LOCATION '$path'""".stripMargin) // v0 spark.range(0, 10) .selectExpr( "CAST(id as INT) as id", "CAST(id as STRING) as name", "CAST(id % 4 + 18 as INT) as age") .write.format("delta").mode("append").save(path) // v1 // =============== Show table data + changes =============== println("(v1) Initial Table") spark.read.format("delta").load(path).orderBy("id").show() println("(v1) CDC changes") readCDCByTableName(1).show() val table = io.delta.tables.DeltaTable.forPath(path) // =============== Perform UPDATE =============== println("(v2) Updated id -> id + 1") table.update(Map("id" -> expr("id + 1"))) // v2 readCDCByTableName(2).show() // =============== Perform DELETE =============== println("(v3) Deleted where id >= 7") table.delete(expr("id >= 7")) // v3 readCDCByTableName(3).show() // =============== Perform partition DELETE =============== println("(v4) Deleted where age = 18") table.delete(expr("age = 18")) // v4, partition delete readCDCByTableName(4).show() // =============== Create source table for MERGE =============== spark.sql( s""" |CREATE TABLE student_source (id INT, name STRING, age INT) |USING DELTA |LOCATION '$otherPath'""".stripMargin) spark.range(0, 3).selectExpr( "CAST(id as INT) as id", "CAST(id as STRING) as name", "CAST(id % 4 + 18 as INT) as age") .write.format("delta").mode("append").saveAsTable("student_source") val source = spark.sql("SELECT * FROM student_source") // =============== Perform MERGE =============== table .as("target") .merge(source.as("source"), "target.id = source.id") .whenMatched() .updateExpr( Map("id" -> "source.id", "age" -> "source.age + 10")) .whenNotMatched() .insertAll() .execute() // v5 println("(v5) Merged with a source table") readCDCByTableName(5).show() // =============== Stream changes =============== println("Streaming by table name") val cdfStream = streamCDCByTableName(0) cdfStream.awaitTermination(5000) cdfStream.stop() } finally { cleanup() spark.stop() } } } ================================================ FILE: examples/scala/src/main/scala/example/Clustering.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package example import io.delta.tables.DeltaTable import org.apache.spark.sql.SparkSession object Clustering { def main(args: Array[String]): Unit = { val tableName = "deltatable" val deltaSpark = SparkSession .builder() .appName("Clustering-Delta") .master("local[*]") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") .getOrCreate() // Clear up old session deltaSpark.sql(s"DROP TABLE IF EXISTS $tableName") // Enable preview config for clustering deltaSpark.conf.set( "spark.databricks.delta.clusteredTable.enableClusteringTablePreview", "true") try { // Create a table println("Creating a table") deltaSpark.sql( s"""CREATE TABLE $tableName (col1 INT, col2 STRING) using DELTA |CLUSTER BY (col1, col2)""".stripMargin) // Insert new data println("Insert new data") deltaSpark.sql(s"INSERT INTO $tableName VALUES (123, '123')") // Optimize the table println("Optimize the table") deltaSpark.sql(s"OPTIMIZE $tableName") // Change the clustering columns println("Change the clustering columns") deltaSpark.sql( s"""ALTER TABLE $tableName CLUSTER BY (col2, col1)""".stripMargin) // Check the clustering columns println("Check the clustering columns") deltaSpark.sql(s"DESCRIBE DETAIL $tableName").show(false) } finally { // Cleanup deltaSpark.sql(s"DROP TABLE IF EXISTS $tableName") } // DeltaTable clusterBy Scala API try { val table = io.delta.tables.DeltaTable.create() .tableName(tableName) .addColumn("col1", "INT") .addColumn("col2", "STRING") .clusterBy("col1", "col2") .execute() } finally { // Cleanup deltaSpark.sql(s"DROP TABLE IF EXISTS $tableName") deltaSpark.stop() } } } ================================================ FILE: examples/scala/src/main/scala/example/EvolutionWithMap.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package example import org.apache.spark.sql.types._ import org.apache.spark.sql.functions._ import org.apache.spark.sql.Row import org.apache.spark.sql.SparkSession object EvolutionWithMap { def main(args: Array[String]): Unit = { val spark = SparkSession.builder() .appName("EvolutionWithMap") .master("local[*]") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") .getOrCreate() import spark.implicits._ val tableName = "insert_map_schema_evolution" try { // Define initial schema val initialSchema = StructType(Seq( StructField("key", IntegerType, nullable = false), StructField("metrics", MapType(StringType, StructType(Seq( StructField("id", IntegerType, nullable = false), StructField("value", IntegerType, nullable = false) )))) )) val data = Seq( Row(1, Map("event" -> Row(1, 1))) ) val rdd = spark.sparkContext.parallelize(data) val initialDf = spark.createDataFrame(rdd, initialSchema) initialDf.write .option("overwriteSchema", "true") .mode("overwrite") .format("delta") .saveAsTable(s"$tableName") // Define the schema with simulteneous change in a StructField name // And additional field in a map column val evolvedSchema = StructType(Seq( StructField("renamed_key", IntegerType, nullable = false), StructField("metrics", MapType(StringType, StructType(Seq( StructField("id", IntegerType, nullable = false), StructField("value", IntegerType, nullable = false), StructField("comment", StringType, nullable = true) )))) )) val evolvedData = Seq( Row(1, Map("event" -> Row(1, 1, "deprecated"))) ) val evolvedRDD = spark.sparkContext.parallelize(evolvedData) val modifiedDf = spark.createDataFrame(evolvedRDD, evolvedSchema) // The below would fail without schema evolution for map types modifiedDf.write .mode("append") .option("mergeSchema", "true") .format("delta") .insertInto(s"$tableName") spark.sql(s"SELECT * FROM $tableName").show(false) } finally { // Cleanup spark.sql(s"DROP TABLE IF EXISTS $tableName") spark.stop() } } } ================================================ FILE: examples/scala/src/main/scala/example/IcebergCompatV2.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package example import java.io.{File, IOException} import java.net.ServerSocket import org.apache.commons.io.FileUtils import org.apache.spark.sql.SparkSession /** * This example relies on an external Hive metastore (HMS) instance to run. * * A standalone HMS can be created using the following docker command. * ************************************************************ * docker run -d -p 9083:9083 --env SERVICE_NAME=metastore \ * --name metastore-standalone apache/hive:4.0.0-beta-1 * ************************************************************ * The URL of this standalone HMS is thrift://localhost:9083 * * By default this hms will use `/opt/hive/data/warehouse` as warehouse path. * Please make sure this path exists or change it prior to running the example. */ object IcebergCompatV2 { def main(args: Array[String]): Unit = { // Update this according to the metastore config val port = 9083 val warehousePath = "/opt/hive/data/warehouse/" if (!UniForm.hmsReady(port)) { print("HMS not available. Exit.") return } val testTableName = "uniform_table3" FileUtils.deleteDirectory(new File(s"${warehousePath}${testTableName}")) val deltaSpark = SparkSession .builder() .appName("UniForm-Delta") .master("local[*]") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") .config("hive.metastore.uris", s"thrift://localhost:$port") .config("spark.sql.catalogImplementation", "hive") .getOrCreate() deltaSpark.sql(s"DROP TABLE IF EXISTS ${testTableName}") deltaSpark.sql( s"""CREATE TABLE `${testTableName}` | (id INT, ts TIMESTAMP, array_data array, map_data map) | using DELTA""".stripMargin) deltaSpark.sql( s""" |INSERT INTO `$testTableName` (id, ts, array_data, map_data) | VALUES (123, '2024-01-01 00:00:00', array(2, 3, 4, 5), map(3, 6, 8, 7))""".stripMargin) deltaSpark.sql( s"""REORG TABLE `$testTableName` APPLY (UPGRADE UNIFORM | (ICEBERG_COMPAT_VERSION = 2))""".stripMargin) val icebergSpark = SparkSession.builder() .master("local[*]") .appName("UniForm-Iceberg") .config("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions") .config("spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("hive.metastore.uris", s"thrift://localhost:$port") .config("spark.sql.catalogImplementation", "hive") .getOrCreate() icebergSpark.sql(s"SELECT * FROM ${testTableName}").show() } } ================================================ FILE: examples/scala/src/main/scala/example/Quickstart.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package example import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.sql.{SparkSession, SQLContext} import io.delta.tables._ import org.apache.spark.sql.functions._ import org.apache.commons.io.FileUtils import java.io.File object Quickstart { def main(args: Array[String]): Unit = { val spark = SparkSession .builder() .appName("Quickstart") .master("local[*]") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config( "spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog" ) .getOrCreate() val file = new File("/tmp/delta-table") if (file.exists()) FileUtils.deleteDirectory(file) // Create a table println("Creating a table") val path = file.getCanonicalPath var data = spark.range(0, 5) data.write.format("delta").save(path) // Read table println("Reading the table") val df = spark.read.format("delta").load(path) df.show() // Upsert (merge) new data println("Upsert new data") val newData = spark.range(0, 20).toDF() val deltaTable = DeltaTable.forPath(path) deltaTable .as("oldData") .merge(newData.as("newData"), "oldData.id = newData.id") .whenMatched() .update(Map("id" -> col("newData.id"))) .whenNotMatched() .insert(Map("id" -> col("newData.id"))) .execute() deltaTable.toDF.show() // Update table data println("Overwrite the table") data = spark.range(5, 10) data.write.format("delta").mode("overwrite").save(path) deltaTable.toDF.show() // Update every even value by adding 100 to it println("Update to the table (add 100 to every even value)") deltaTable.update( condition = expr("id % 2 == 0"), set = Map("id" -> expr("id + 100")) ) deltaTable.toDF.show() // Delete every even value deltaTable.delete(condition = expr("id % 2 == 0")) deltaTable.toDF.show() // Read old version of the data using time travel print("Read old data using time travel") val df2 = spark.read.format("delta").option("versionAsOf", 0).load(path) df2.show() // Cleanup FileUtils.deleteDirectory(file) spark.stop() } } ================================================ FILE: examples/scala/src/main/scala/example/QuickstartSQL.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package example import org.apache.spark.sql.SparkSession import io.delta.tables._ import org.apache.spark.sql.functions._ import org.apache.commons.io.FileUtils import java.io.File object QuickstartSQL { def main(args: Array[String]): Unit = { // Create Spark Conf val spark = SparkSession .builder() .appName("QuickstartSQL") .master("local[*]") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") .getOrCreate() val tableName = "tblname" // Clear up old session spark.sql(s"DROP TABLE IF EXISTS $tableName") spark.sql(s"DROP TABLE IF EXISTS newData") try { // Create a table println("Creating a table") spark.sql(s"CREATE TABLE $tableName(id LONG) USING delta") spark.sql(s"INSERT INTO $tableName VALUES 0, 1, 2, 3, 4") // Read table println("Reading the table") spark.sql(s"SELECT * FROM $tableName").show() // Upsert (merge) new data println("Upsert new data") spark.sql("CREATE TABLE newData(id LONG) USING parquet") spark.sql("INSERT INTO newData VALUES 3, 4, 5, 6") spark.sql(s"""MERGE INTO $tableName USING newData ON ${tableName}.id = newData.id WHEN MATCHED THEN UPDATE SET ${tableName}.id = newData.id WHEN NOT MATCHED THEN INSERT * """) spark.sql(s"SELECT * FROM $tableName").show() // Update table data println("Overwrite the table") spark.sql(s"INSERT OVERWRITE $tableName VALUES 5, 6, 7, 8, 9") spark.sql(s"SELECT * FROM $tableName").show() // Update every even value by adding 100 to it println("Update to the table (add 100 to every even value)") spark.sql(s"UPDATE $tableName SET id = (id + 100) WHERE (id % 2 == 0)") spark.sql(s"SELECT * FROM $tableName").show() // Delete every even value spark.sql(s"DELETE FROM $tableName WHERE (id % 2 == 0)") spark.sql(s"SELECT * FROM $tableName").show() // Read old version of the data using time travel print("Read old data using time travel") spark.sql(s"SELECT * FROM $tableName VERSION AS OF 0").show() } finally { // Cleanup spark.sql(s"DROP TABLE IF EXISTS $tableName") spark.sql(s"DROP TABLE IF EXISTS newData") spark.stop() } } } ================================================ FILE: examples/scala/src/main/scala/example/QuickstartSQLOnPaths.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package example import org.apache.spark.sql.SparkSession import io.delta.tables._ import org.apache.spark.sql.functions._ import org.apache.commons.io.FileUtils import java.io.File object QuickstartSQLOnPaths { def main(args: Array[String]): Unit = { // Create Spark Conf val spark = SparkSession .builder() .appName("QuickstartSQLOnPaths") .master("local[*]") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") .getOrCreate() val tablePath = new File("/tmp/delta-table") if (tablePath.exists()) FileUtils.deleteDirectory(tablePath) // Clear up old session spark.sql(s"DROP TABLE IF EXISTS newData") try { // Create a table println("Creating a table") spark.sql(s"CREATE TABLE delta.`$tablePath`(id LONG) USING delta") spark.sql(s"INSERT INTO delta.`$tablePath` VALUES 0, 1, 2, 3, 4") // Read table println("Reading the table") spark.sql(s"SELECT * FROM delta.`$tablePath`").show() // Upsert (merge) new data println("Upsert new data") spark.sql("CREATE TABLE newData(id LONG) USING parquet") spark.sql("INSERT INTO newData VALUES 3, 4, 5, 6") spark.sql(s"""MERGE INTO delta.`$tablePath` data USING newData ON data.id = newData.id WHEN MATCHED THEN UPDATE SET data.id = newData.id WHEN NOT MATCHED THEN INSERT * """) spark.sql(s"SELECT * FROM delta.`$tablePath`").show() // Update table data println("Overwrite the table") spark.sql(s"INSERT OVERWRITE delta.`$tablePath` VALUES 5, 6, 7, 8, 9") spark.sql(s"SELECT * FROM delta.`$tablePath`").show() // Update every even value by adding 100 to it println("Update to the table (add 100 to every even value)") spark.sql(s"UPDATE delta.`$tablePath` SET id = (id + 100) WHERE (id % 2 == 0)") spark.sql(s"SELECT * FROM delta.`$tablePath`").show() // Delete every even value spark.sql(s"DELETE FROM delta.`$tablePath` WHERE (id % 2 == 0)") spark.sql(s"SELECT * FROM delta.`$tablePath`").show() } finally { // Cleanup spark.sql(s"DROP TABLE IF EXISTS newData") spark.stop() } } } ================================================ FILE: examples/scala/src/main/scala/example/Streaming.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package example import java.io.File import io.delta.tables.DeltaTable import org.apache.commons.io.FileUtils import org.apache.spark.sql.{DataFrame, SparkSession} import org.apache.spark.sql.functions.col object Streaming { def main(args: Array[String]): Unit = { // Create a Spark Session val spark = SparkSession .builder() .appName("Streaming") .master("local[*]") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config( "spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog" ) .getOrCreate() import spark.implicits._ val exampleDir = new File("/tmp/delta-streaming/") if (exampleDir.exists()) FileUtils.deleteDirectory(exampleDir) println( "=== Section 1: write and read delta table using batch queries, and initialize table for later sections" ) // Create a table val data = spark.range(0, 5) val path = new File("/tmp/delta-streaming/delta-table").getAbsolutePath data.write.format("delta").save(path) // Read table val df = spark.read.format("delta").load(path) df.show() println("=== Section 2: write and read delta using structured streaming") val streamingDf = spark.readStream.format("rate").load() val tablePath2 = new File( "/tmp/delta-streaming/delta-table2" ).getCanonicalPath val checkpointPath = new File( "/tmp/delta-streaming/checkpoint" ).getCanonicalPath val stream = streamingDf .select($"value" as "id") .writeStream .format("delta") .option("checkpointLocation", checkpointPath) .start(tablePath2) stream.awaitTermination(10000) stream.stop() val stream2 = spark.readStream .format("delta") .load(tablePath2) .writeStream .format("console") .start() stream2.awaitTermination(10000) stream2.stop() println("=== Section 3: Streaming upserts using MERGE") // Function to upsert microBatchOutputDF into Delta Lake table using merge def upsertToDelta(microBatchOutputDF: DataFrame, batchId: Long): Unit = { val deltaTable = DeltaTable.forPath(path) deltaTable .as("t") .merge( microBatchOutputDF.select($"value" as "id").as("s"), "s.id = t.id" ) .whenMatched() .updateAll() .whenNotMatched() .insertAll() .execute() } val streamingAggregatesDf = spark.readStream .format("rate") .load() .withColumn("key", col("value") % 10) .drop("timestamp") // Write the output of a streaming aggregation query into Delta Lake table println("Original Delta Table") val deltaTable = DeltaTable.forPath(path) deltaTable.toDF.show() val stream3 = streamingAggregatesDf.writeStream .format("delta") .foreachBatch(upsertToDelta _) .outputMode("update") .start() stream3.awaitTermination(20000) stream3.stop() println("Delta Table after streaming upsert") deltaTable.toDF.show() // Streaming append and concurrent repartition using data change = false // tbl1 is the sink and tbl2 is the source println( "############ Streaming appends with concurrent table repartition ##########" ) val tbl1 = "/tmp/delta-streaming/delta-table4" val tbl2 = "/tmp/delta-streaming/delta-table5" val numRows = 10 spark.range(numRows).write.mode("overwrite").format("delta").save(tbl1) spark.read.format("delta").load(tbl1).show() spark .range(numRows, numRows * 10) .write .mode("overwrite") .format("delta") .save(tbl2) // Start reading tbl2 as a stream and do a streaming write to tbl1 // Prior to Delta 0.5.0 this would throw StreamingQueryException: Detected a data update in the source table. This is currently not supported. val stream4 = spark.readStream .format("delta") .load(tbl2) .writeStream .format("delta") .option( "checkpointLocation", new File("/tmp/delta-streaming/checkpoint/tbl1").getCanonicalPath ) .outputMode("append") .start(tbl1) Thread.sleep(10 * 1000) // repartition table while streaming job is running spark.read .format("delta") .load(tbl2) .repartition(10) .write .format("delta") .mode("overwrite") .option("dataChange", "false") .save(tbl2) stream4.awaitTermination(5 * 1000) stream4.stop() println("######### After streaming write #########") spark.read.format("delta").load(tbl1).show() println("=== In the end, clean all paths") // Cleanup if (exampleDir.exists()) FileUtils.deleteDirectory(exampleDir) spark.stop() } } ================================================ FILE: examples/scala/src/main/scala/example/UniForm.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package example import java.io.{File, IOException} import java.net.ServerSocket import org.apache.commons.io.FileUtils import org.apache.spark.sql.SparkSession /** * This example relies on an external Hive metastore (HMS) instance to run. * * A standalone HMS can be created using the following docker command. * ************************************************************ * docker run -d -p 9083:9083 --env SERVICE_NAME=metastore \ * --name metastore-standalone apache/hive:4.0.0-beta-1 * ************************************************************ * The URL of this standalone HMS is thrift://localhost:9083 * * By default this hms will use `/opt/hive/data/warehouse` as warehouse path. * Please make sure this path exists or change it prior to running the example. */ object UniForm { def main(args: Array[String]): Unit = { // Update this according to the metastore config val port = 9083 val warehousePath = "/opt/hive/data/warehouse/" if (!hmsReady(port)) { print("HMS not available. Exit.") return } val testTableName = "deltatable" FileUtils.deleteDirectory(new File(s"${warehousePath}${testTableName}")) val deltaSpark = SparkSession .builder() .appName("UniForm-Delta") .master("local[*]") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") .config("hive.metastore.uris", s"thrift://localhost:$port") .config("spark.sql.catalogImplementation", "hive") .getOrCreate() val schema = """ |col0 INT, |col1 STRUCT< | col2: MAP, | col3: ARRAY, | col4: STRUCT |>, |col6 INT, |col7 INT |""".stripMargin def getRowToInsertStr(id: Int): String = { s""" |$id, |struct(map($id, $id), array($id), struct($id)), |$id, |$id |""".stripMargin } deltaSpark.sql(s"DROP TABLE IF EXISTS ${testTableName}") deltaSpark.sql( s"""CREATE TABLE `${testTableName}` ($schema) using DELTA |PARTITIONED BY (col0, col6, col7) |TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name', | 'delta.enableIcebergCompatV2' = 'true', | 'delta.universalFormat.enabledFormats' = 'iceberg' |)""".stripMargin) deltaSpark.sql(s"INSERT INTO $testTableName VALUES (${getRowToInsertStr(1)})") // Wait for the conversion to be done Thread.sleep(10000) val icebergSpark = SparkSession.builder() .master("local[*]") .appName("UniForm-Iceberg") .config("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions") .config("spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("hive.metastore.uris", s"thrift://localhost:$port") .config("spark.sql.catalogImplementation", "hive") .getOrCreate() icebergSpark.sql(s"SELECT * FROM ${testTableName}").show() } def hmsReady(port: Int): Boolean = { var ss: ServerSocket = null try { ss = new ServerSocket(port) ss.setReuseAddress(true) return false } catch { case e: IOException => } finally { if (ss != null) { try ss.close() catch { case e: IOException => } } } true } } ================================================ FILE: examples/scala/src/main/scala/example/UnityCatalogQuickstart.scala ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package example import io.unitycatalog.client.ApiClientBuilder import io.unitycatalog.client.api.{CatalogsApi, SchemasApi} import io.unitycatalog.client.auth.TokenProvider import io.unitycatalog.client.model.{CreateCatalog, CreateSchema} import io.unitycatalog.server.UnityCatalogServer import io.unitycatalog.server.utils.ServerProperties import org.apache.commons.io.FileUtils import org.apache.spark.sql.SparkSession import java.io.File import java.net.ServerSocket import java.nio.file.Files import java.util.Properties import scala.collection.JavaConverters._ /** * Example of testing streaming read from UC managed table with OSS UC */ object UnityCatalogQuickstart { private val StaticToken = "static-token" def main(args: Array[String]): Unit = { val serverDir = Files.createTempDirectory("uc-integration-test-").toFile val tableDir = Files.createTempDirectory("uc-table-location-").toFile val port = { val socket = new ServerSocket(0) val p = socket.getLocalPort socket.close() p } val serverProps = new Properties() serverProps.setProperty("server.env", "test") serverProps.setProperty("server.managed-table.enabled", "true") serverProps.setProperty("storage-root.tables", new File(serverDir, "ucroot").getAbsolutePath) val server = UnityCatalogServer.builder() .port(port) .serverProperties(new ServerProperties(serverProps)) .build() server.start() val serverUri = s"http://localhost:$port/" try { waitForServer(serverUri) createCatalogAndSchema(serverUri) runDeltaWorkload(serverUri, tableDir) println("SUCCESS: Unity Catalog + Delta integration test passed") } finally { server.stop() FileUtils.deleteQuietly(serverDir) FileUtils.deleteQuietly(tableDir) } } private def waitForServer(serverUri: String): Unit = { var ready = false var retries = 0 while (!ready && retries < 30) { try { new CatalogsApi(createApiClient(serverUri)).listCatalogs(null, null) ready = true } catch { case _: Exception => Thread.sleep(500) retries += 1 } } if (!ready) { throw new RuntimeException("Unity Catalog server did not become ready within 15 seconds") } } private def createCatalogAndSchema(serverUri: String): Unit = { val client = createApiClient(serverUri) new CatalogsApi(client).createCatalog( new CreateCatalog().name("unity").comment("Integration test catalog")) new SchemasApi(client).createSchema( new CreateSchema().name("default").catalogName("unity")) } private def runDeltaWorkload(serverUri: String, tableDir: File): Unit = { val spark = SparkSession.builder() .appName("UC Delta Integration Test") .master("local[2]") .config("spark.ui.enabled", "false") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") .config("spark.sql.catalog.unity", "io.unitycatalog.spark.UCSingleCatalog") .config("spark.sql.catalog.unity.uri", serverUri) .config("spark.sql.catalog.unity.token", StaticToken) .getOrCreate() try { val checkpointPath = new File(tableDir, "checkpoint").getAbsolutePath spark.sql( s"""CREATE TABLE unity.default.test_table (id BIGINT, data STRING) |USING DELTA |TBLPROPERTIES('delta.feature.catalogManaged' = 'supported')""".stripMargin) spark.sql("INSERT INTO unity.default.test_table VALUES (1, 'hello'), (2, 'world')") val stream = spark.readStream .table("unity.default.test_table") .writeStream .format("console") .option("checkpointLocation", checkpointPath) .start() stream.awaitTermination(10000) stream.stop() spark.sql("DROP TABLE IF EXISTS unity.default.test_table") } finally { spark.stop() } } private def createApiClient(serverUri: String) = { ApiClientBuilder.create() .uri(serverUri) .tokenProvider( TokenProvider.create(Map("type" -> "static", "token" -> StaticToken).asJava)) .build() } } ================================================ FILE: examples/scala/src/main/scala/example/Utilities.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package example import java.io.File import io.delta.tables.DeltaTable import org.apache.commons.io.FileUtils import org.apache.spark.sql.SparkSession object Utilities { def main(args: Array[String]): Unit = { // Create a Spark Session with SQL enabled val spark = SparkSession .builder() .appName("Utilities") .master("local[*]") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") // control the parallelism for vacuum .config("spark.sql.sources.parallelPartitionDiscovery.parallelism", "4") .getOrCreate() // Create a table println("Create a parquet table") val data = spark.range(0, 5) val file = new File("/tmp/parquet-table") val path = file.getAbsolutePath data.write.format("parquet").save(path) // Convert to delta println("Convert to Delta") DeltaTable.convertToDelta(spark, s"parquet.`$path`") // Read table as delta var df = spark.read.format("delta").load(path) // Read old version of data using time travel df = spark.read.format("delta").option("versionAsOf", 0).load(path) df.show() val deltaTable = DeltaTable.forPath(path) // Utility commands println("Vacuum the table") deltaTable.vacuum() println("Describe History for the table") deltaTable.history().show() println("Describe Details for the table") deltaTable.detail().show() // Generate manifest println("Generate Manifest files") deltaTable.generate("SYMLINK_FORMAT_MANIFEST") // SQL utility commands println("SQL Vacuum") spark.sql(s"VACUUM '$path' RETAIN 169 HOURS") println("SQL Describe History") println(spark.sql(s"DESCRIBE HISTORY '$path'").collect()) // Cleanup FileUtils.deleteDirectory(new File(path)) spark.stop() } } ================================================ FILE: examples/scala/src/main/scala/example/Variant.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package example import io.delta.tables.DeltaTable import org.apache.spark.sql.SparkSession object Variant { def main(args: Array[String]): Unit = { val tableName = "tbl" val spark = SparkSession .builder() .appName("Variant-Delta") .master("local[*]") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") .getOrCreate() // Only run this example for Spark versions >= 4.0.0 if (spark.version.split("\\.").head.toInt < 4) { println(s"Skipping Variant.scala since Spark version ${spark.version} is too low") return } // Create and insert variant values. try { println("Creating and inserting variant values") spark.sql(s"DROP TABLE IF EXISTS $tableName") spark.sql(s"CREATE TABLE $tableName(v VARIANT) USING DELTA") spark.sql(s"INSERT INTO $tableName VALUES (parse_json('1'))") spark.sql(s"""INSERT INTO $tableName SELECT parse_json(format_string('{\"k\": %s}', id)) FROM range(0, 10)""") val ids = spark.sql("SELECT variant_get(v, '$.k', 'INT') out " + s"""FROM $tableName WHERE contains(schema_of_variant(v), 'k') ORDER BY out""") .collect().map { r => r.getInt(0) }.toSeq val expected = (0 until 10).toSeq assert(expected == ids) spark.sql(s"DELETE FROM $tableName WHERE variant_get(v, '$$.k', 'INT') = 0") val idsWithDelete = spark.sql("SELECT variant_get(v, '$.k', 'INT') out " + s"""FROM $tableName WHERE contains(schema_of_variant(v), 'k') ORDER BY out""") .collect().map { r => r.getInt(0) }.toSeq val expectedWithDelete = (1 until 10).toSeq assert(idsWithDelete == expectedWithDelete) } finally { spark.sql(s"DROP TABLE IF EXISTS $tableName") } // Convert Parquet table with variant values to Delta. try { println("Converting a parquet table with variant values to Delta") spark.sql(s"DROP TABLE IF EXISTS $tableName") spark.sql(s"""CREATE TABLE $tableName USING PARQUET AS ( SELECT parse_json(format_string('%s', id)) v FROM range(0, 10))""") spark.sql(s"CONVERT TO DELTA $tableName") val convertToDeltaIds = spark.sql(s"SELECT v::int v FROM $tableName ORDER BY v") .collect() .map { r => r.getInt(0) } .toSeq val convertToDeltaExpected = (0 until 10).toSeq assert(convertToDeltaIds == convertToDeltaExpected) } finally { spark.sql(s"DROP TABLE IF EXISTS $tableName") } // DeltaTable create with variant Scala API. try { println("Creating a delta table with variant type using the DeltaTable API") spark.sql(s"DROP TABLE IF EXISTS $tableName") val table = io.delta.tables.DeltaTable.create() .tableName(tableName) .addColumn("v", "VARIANT") .execute() table .as("tgt") .merge( spark.sql("select parse_json(format_string('%s', id)) v from range(0, 10)").as("source"), "source.v::int == tgt.v::int" ) .whenMatched() .updateAll() .whenNotMatched() .insertAll() .execute() val insertedVals = spark.sql(s"SELECT v::int v FROM $tableName ORDER BY v") .collect() .map { r => r.getInt(0) } .toSeq val expected = (0 until 10).toSeq assert(insertedVals == expected) } finally { spark.sql(s"DROP TABLE IF EXISTS $tableName") spark.stop() } } } ================================================ FILE: flink/src/main/java/.placeholder ================================================ ================================================ FILE: flink/src/main/java/io/delta/flink/Conf.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink; import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Global configuration for Delta Flink sinks. * *

This class loads process-wide configuration from the {@code delta-flink.properties} file and * exposes shared settings that apply to all {@code DeltaSink} instances within the JVM. * These configurations are intended for operational tuning and common defaults, such as retry * behavior, thread pool sizing, caching, and credential refresh. * *

The configuration is loaded once and managed as a singleton. All sinks created in the same * process observe the same global configuration values. * *

Per-sink or per-table behavior should be configured explicitly when constructing the {@code * DeltaSink} instance. Sink-level configuration takes precedence over global defaults defined here. * *

This class is not intended to be instantiated or mutated directly by users. */ public final class Conf { private static final Logger LOG = LoggerFactory.getLogger(Conf.class); public static String SINK_RETRY_MAX_ATTEMPT = "sink.retry.max_attempt"; // The i-th retry will have a delay of `delay-ms * (2 ^ i)` public static String SINK_RETRY_DELAY_MS = "sink.retry.delay_ms"; // Retry will stop if the delay exceeds max-delay public static String SINK_RETRY_MAX_DELAY_MS = "sink.retry.max_delay_ms"; public static String SINK_WRITER_NUM_CONCURRENT_FILE = "sink.writer.num_concurrent_file"; public static String TABLE_THREAD_POOL_SIZE = "table.thread_pool_size"; public static String TABLE_CACHE_ENABLE = "table.cache.enable"; public static String TABLE_CACHE_SIZE = "table.cache.size"; public static String TABLE_CACHE_EXPIRE_MS = "table.cache.expire_ms"; public static String CREDENTIALS_REFRESH_THREAD_POOL_SIZE = "credentials.refresh.thread_pool_size"; public static String CREDENTIALS_REFRESH_AHEAD_MS = "credentials.refresh.ahead_ms"; private static final String CONFIG_FILE = "delta-flink.properties"; private static final Conf INSTANCE = new Conf(); private final Map props; // For debug purpose private URL sourcePath; private Conf() { this.props = load(); } public static Conf getInstance() { return INSTANCE; } /*================ * Confs *================*/ public int getSinkRetryMaxAttempt() { return Integer.parseInt(getOrDefault(SINK_RETRY_MAX_ATTEMPT, "4")); } public long getSinkRetryDelayMs() { return Long.parseLong(getOrDefault(SINK_RETRY_DELAY_MS, "200")); } public long getSinkRetryMaxDelayMs() { return Long.parseLong(getOrDefault(SINK_RETRY_MAX_DELAY_MS, "20000")); } public int getSinkWriterNumConcurrentFiles() { return Integer.parseInt(getOrDefault(SINK_WRITER_NUM_CONCURRENT_FILE, "1000")); } public int getTableThreadPoolSize() { return Integer.parseInt(getOrDefault(TABLE_THREAD_POOL_SIZE, "5")); } public boolean getTableCacheEnable() { return Boolean.parseBoolean(getOrDefault(TABLE_CACHE_ENABLE, "true")); } public int getTableCacheSize() { return Integer.parseInt(getOrDefault(TABLE_CACHE_SIZE, "100")); } public long getTableCacheExpireInMs() { return Long.parseLong(getOrDefault(TABLE_CACHE_EXPIRE_MS, "300000")); } public int getCredentialsRefreshThreadPoolSize() { return Integer.parseInt(getOrDefault(CREDENTIALS_REFRESH_THREAD_POOL_SIZE, "10")); } public long getCredentialsRefreshAheadInMs() { return Long.parseLong(getOrDefault(CREDENTIALS_REFRESH_AHEAD_MS, "60000")); } /** Returns an immutable view of all configuration entries. */ public Map asMap() { return props; } /** Returns a configuration value or null if missing. */ public String get(String key) { return props.get(key); } /** Returns a configuration value or default if missing. */ public String getOrDefault(String key, String defaultValue) { return props.getOrDefault(key, defaultValue); } // ----------------- internals ----------------- private Map load() { Properties p = new Properties(); try (InputStream in = Conf.class.getClassLoader().getResourceAsStream(CONFIG_FILE)) { if (in == null) { return Map.of(); } sourcePath = Conf.class.getClassLoader().getResource(CONFIG_FILE); LOG.info("Loaded configuration from {}", sourcePath); p.load(in); } catch (IOException e) { throw new RuntimeException("Failed to load " + CONFIG_FILE, e); } Map map = new HashMap<>(); for (String name : p.stringPropertyNames()) { map.put(name, p.getProperty(name)); } return Collections.unmodifiableMap(map); } } ================================================ FILE: flink/src/main/java/io/delta/flink/kernel/CheckpointActionRow.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink.kernel; import io.delta.kernel.data.ArrayValue; import io.delta.kernel.data.MapValue; import io.delta.kernel.data.Row; import io.delta.kernel.internal.actions.*; import io.delta.kernel.internal.checkpoints.*; import io.delta.kernel.types.StructType; import java.math.BigDecimal; import java.util.List; import java.util.function.Function; /** Represent a row in a v2 checkpoint */ public class CheckpointActionRow implements Row { /** Schema used to read/write v2 checkpoint files produced by this writer. */ public static final StructType CHECKPOINT_SCHEMA = new StructType() .add("checkpointMetadata", CheckpointMetadataAction.FULL_SCHEMA) .add("metaData", Metadata.FULL_SCHEMA) .add("protocol", Protocol.FULL_SCHEMA) .add("txn", SetTransaction.FULL_SCHEMA) .add("sidecar", SidecarFile.READ_SCHEMA) .add("domainMetadata", DomainMetadata.FULL_SCHEMA) .add("add", AddFile.FULL_SCHEMA); static final List> ROW_MAPPERS = List.of( obj -> ((CheckpointMetadataAction) obj).toRow(), obj -> ((Metadata) obj).toRow(), obj -> ((Protocol) obj).toRow(), obj -> ((SetTransaction) obj).toRow(), obj -> ((SidecarFile) obj).toRow(), obj -> ((DomainMetadata) obj).toRow()); private final Object action; public CheckpointActionRow(Object action) { this.action = action; } @Override public StructType getSchema() { return CHECKPOINT_SCHEMA; } @Override public boolean isNullAt(int ordinal) { if (ordinal >= ROW_MAPPERS.size()) { return true; } try { ROW_MAPPERS.get(ordinal).apply(action); return false; } catch (ClassCastException e) { return true; } } @Override public Row getStruct(int ordinal) { try { return ROW_MAPPERS.get(ordinal).apply(action); } catch (ClassCastException e) { return null; } } @Override public boolean getBoolean(int ordinal) { return false; } @Override public byte getByte(int ordinal) { return 0; } @Override public short getShort(int ordinal) { return 0; } @Override public int getInt(int ordinal) { return 0; } @Override public long getLong(int ordinal) { return 0; } @Override public float getFloat(int ordinal) { return 0; } @Override public double getDouble(int ordinal) { return 0; } @Override public String getString(int ordinal) { return ""; } @Override public BigDecimal getDecimal(int ordinal) { return null; } @Override public byte[] getBinary(int ordinal) { return new byte[0]; } @Override public ArrayValue getArray(int ordinal) { return null; } @Override public MapValue getMap(int ordinal) { return null; } } ================================================ FILE: flink/src/main/java/io/delta/flink/kernel/CheckpointWriter.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink.kernel; import static io.delta.flink.kernel.CheckpointActionRow.CHECKPOINT_SCHEMA; import static io.delta.kernel.internal.checkpoints.Checkpointer.LAST_CHECKPOINT_FILE_NAME; import static io.delta.kernel.internal.util.FileNames.*; import static io.delta.kernel.internal.util.Utils.singletonCloseableIterator; import static io.delta.kernel.internal.util.Utils.toCloseableIterator; import io.delta.flink.table.ExceptionUtils; import io.delta.kernel.CommitActions; import io.delta.kernel.Snapshot; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.defaults.internal.data.DefaultRowBasedColumnarBatch; import io.delta.kernel.engine.Engine; import io.delta.kernel.engine.FileReadResult; import io.delta.kernel.internal.DeltaLogActionUtils; import io.delta.kernel.internal.DeltaLogActionUtils.DeltaAction; import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.actions.AddFile; import io.delta.kernel.internal.actions.DomainMetadata; import io.delta.kernel.internal.actions.SetTransaction; import io.delta.kernel.internal.checkpoints.CheckpointMetaData; import io.delta.kernel.internal.checkpoints.CheckpointMetadataAction; import io.delta.kernel.internal.checkpoints.SidecarFile; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.internal.util.InternalUtils; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import java.io.IOException; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.LongStream; import java.util.stream.Stream; import org.apache.flink.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Flink-specialized Delta v2 checkpoint writer. * *

This writer assumes the table is configured to use v2 checkpoints with sidecar files and is * most efficient when used for incremental checkpoint creation on Flink sink tables. * *

Because the Flink sink performs blind appends, the writer generates a new sidecar file * containing all {@code AddFile} and {@code SetTransaction} actions in the range {@code * (previousCheckpointVersion + 1, currentSnapshotVersion]}. It then writes a new singular v2 * checkpoint that includes: * *

    *
  • protocol, metadata, and checkpointMetadata actions *
  • the newly generated sidecar file *
  • all sidecars referenced by the previous checkpoint, if that checkpoint was written by this * class *
  • aggregated {@code SetTransaction} actions computed from commits in the version range *
* * If the writer detects that the number of existing sidecar files exceeds a threshold, it will * merge them into a new single sidecar files to reduce the total number of files. * *

## Fallback If any of the condition is true, the writer will ignore all existing sidecars, and * create a new checkpoint with a single sidecar containing all actions up to the version. * *

    *
  • No previous checkpoint exists *
  • {@code _last_checkpoint} is not tagged with {@link #TAG_DELTASINK_CHECKPOINT}. *
  • Remove files appear since the last checkpoint. *
  • this writer is invoked on a snapshot version earlier than the version recorded in {@code * * _last_checkpoint} *
* * We choose to not support reading checkpoints written by other writers assuming that the case is * rare. The support can be added in the future if being requested. * *

## Limitations This writer does not support tables with domain metadata feature. The support * will be added in the future. */ public class CheckpointWriter { /** * Tag written into {@code _last_checkpoint} to indicate the checkpoint was produced by DeltaSink. */ public static final String TAG_DELTASINK_CHECKPOINT = "io.delta.flink.sink.checkpoint"; public static final String TAG_SIDECAR_COUNT = "io.delta.flink.num_sidecar"; private static final Logger LOG = LoggerFactory.getLogger(CheckpointWriter.class); private static final StructType SIDECAR_SCHEMA = new StructType().add("add", AddFile.FULL_SCHEMA); private static final CheckpointMetaData EMPTY_META = new CheckpointMetaData(-1L, 0, Optional.empty(), Map.of()); private static CloseableIterator EMPTY_ITERATOR() { return toCloseableIterator(Collections.emptyIterator()); } private final Engine engine; private final SnapshotImpl snapshot; private final Path lastCheckpointFilePath; private final int sidecarMergeThreshold; private final CheckpointMetaData lastCheckpointMeta; private final boolean lastCheckpointByMe; private int lastSidecarCount; /** Guard to prevent reusing a single writer instance. */ private boolean used = false; /** Max transaction version per appId observed while scanning commits. */ private final Map transactionIds = new HashMap<>(); private final Map domainMetadatas = new HashMap<>(); /** * Creates a checkpoint writer bound to a snapshot. * * @param engine kernel engine * @param snapshot snapshot to checkpoint * @param sidecarMergeThreshold threshold to merge sidecars. Set negative to disable merging. */ public CheckpointWriter(Engine engine, Snapshot snapshot, int sidecarMergeThreshold) { this.engine = engine; this.snapshot = (SnapshotImpl) snapshot; Preconditions.checkArgument( this.snapshot.getProtocol().supportsFeature(TableFeatures.CHECKPOINT_V2_RW_FEATURE)); if (this.snapshot.getProtocol().supportsFeature(TableFeatures.CATALOG_MANAGED_RW_FEATURE)) { // Make sure we are creating checkpoint for a published version Preconditions.checkArgument( this.snapshot.getLogSegment().getMaxPublishedDeltaVersion().orElse(-1L) >= this.snapshot.getVersion()); } Preconditions.checkArgument(sidecarMergeThreshold < 0 || sidecarMergeThreshold >= 2); this.sidecarMergeThreshold = sidecarMergeThreshold; // Read information from _last_checkpoint this.lastCheckpointFilePath = new Path(this.snapshot.getLogPath(), LAST_CHECKPOINT_FILE_NAME); lastCheckpointMeta = readLastCheckpointInfo(); lastCheckpointByMe = Boolean.parseBoolean( lastCheckpointMeta.tags.getOrDefault(TAG_DELTASINK_CHECKPOINT, "false")); lastSidecarCount = 0; try { lastSidecarCount = Integer.parseInt(lastCheckpointMeta.tags.getOrDefault(TAG_SIDECAR_COUNT, "0")); } catch (Exception ignore) { } } public CheckpointWriter(Engine engine, Snapshot snapshot) { this(engine, snapshot, -1); } /** * Writes an incremental v2 checkpoint for the bound snapshot. * *

This operation includes the following steps: * *

    *
  • 1. Determine the baseCheckpoint. *
  • 2. Fetch the actions between baseCheckpoint and current version. *
  • 3. If the actions do not contain remove files, generate a new sidecar from them, and read * existing sidecars from baseCheckpoint. Otherwise, generate a new sidecar for all actions * in current snapshot, and DO NOT read existing sidecars. *
  • 4. Write existing sidecars and new sidecar as a new V2 checkpoint. *
  • 5. Update _last_checkpoint *
* *

Note: This method is single-use. Calling it more than once on the same instance * throws {@link IllegalStateException}. * * @throws IOException if reading commit files or writing checkpoint/sidecar files fails */ public void write() throws IOException { if (used) { throw new IllegalStateException("Checkpoint writer must not be reused."); } used = true; transactionIds.clear(); Path logPath = snapshot.getLogPath(); long version = snapshot.getVersion(); // =========== // Step 1: // =========== // Use _last_checkpoint as baseCheckpoint when // 1. It is written by this writer // 2. _last_checkpoint version is smaller than this snapshot version // Otherwise assume there's no baseCheckpoint // ==================================================================== Optional baseCheckpointMeta = (lastCheckpointByMe && lastCheckpointMeta.version < version) ? Optional.of(lastCheckpointMeta) : Optional.empty(); long baseVersion = baseCheckpointMeta.map(m -> m.version).orElse(-1L); Path baseCheckpointPath = checkpointFileSingular(logPath, baseVersion); Path newCheckpointPath = checkpointFileSingular(logPath, version); // ============ // Step 2 // ============ // Read actions between (baseCheckpoint.version, version]. We can assume this since we require // that the snapshot be fully published on catalog-managed tables List deltaFiles = LongStream.range(baseVersion + 1, version + 1) .mapToObj(v -> FileStatus.of(FileNames.deltaFile(snapshot.getLogPath(), v))) .collect(Collectors.toList()); AtomicInteger addFileCounter = new AtomicInteger(); AtomicInteger removeFileCounter = new AtomicInteger(); // =========== // Step 3 // =========== // Read AddFile and txn actions from incremental commit files, and generate a new sidecar // including AddFiles. It also checks if remove file exists. SidecarFile newSidecar; CloseableIterator existingSidecars = EMPTY_ITERATOR(); try (CloseableIterator actions = DeltaLogActionUtils.getActionsFromCommitFilesWithProtocolValidation( engine, snapshot.getPath(), deltaFiles, Set.of( DeltaAction.ADD, DeltaAction.REMOVE, DeltaAction.TXN, DeltaAction.DOMAINMETADATA)) .flatMap(CommitActions::getActions) .map(filterActions(Map.of("add", addFileCounter, "remove", removeFileCounter)))) { newSidecar = sidecarFromAddFiles(actions); // If remove file exists, fallback to generating a new sidecar including everything. if (removeFileCounter.get() > 0) { try (CloseableIterator allActions = snapshot.getCreateCheckpointIterator(engine).map(filterAddFiles())) { newSidecar = sidecarFromAddFiles(allActions); } } else if (baseCheckpointMeta.isPresent()) { // When there's no remove files, read existing sidecars from the base checkpoint if it // exists existingSidecars = sidecarsFromCheckpoint(baseCheckpointPath); } } // ========== // Step 4 // ========== // Build new checkpoint including: // - protocol // - metadata // - checkpointMetadata // - txn // - domainMetadata // - existing sidecars // - new sidecar. CheckpointMetadataAction checkpointMetadata = new CheckpointMetadataAction(version, Map.of()); try (CloseableIterator merged = rowsToBatch( Stream.of( snapshot.getProtocol(), snapshot.getMetadata(), checkpointMetadata, newSidecar)) .combine(existingSidecars) .combine(rowsToBatch(getTransactions())) .combine(rowsToBatch(getDomainMetadatas()))) { engine .getParquetHandler() .writeParquetFileAtomically(String.valueOf(newCheckpointPath), merged); } // ========== // Step 5 // ========== // Write _last_checkpoint file with our tag, so we can recognize our own checkpoints later. if (version > lastCheckpointMeta.version) { engine .getJsonHandler() .writeJsonFileAtomically( lastCheckpointFilePath.toString(), singletonCloseableIterator( new CheckpointMetaData( version, lastCheckpointMeta.size + addFileCounter.get() - removeFileCounter.get(), Optional.empty(), Map.of( TAG_DELTASINK_CHECKPOINT, "true", TAG_SIDECAR_COUNT, String.valueOf(lastSidecarCount + 1))) .toRow()), true /* overwrite */); } } /** * Read existing sidecars from the given checkpoint Path. If the total number of sidecars exceeds * the threshold, merge existing sidecars into one to reduce the number of files to read. * * @param checkpointPath path for the checkpoint * @return an iterator of sidecars fetched from the checkpoint. * @throws IOException when exception happens during read or write. */ private CloseableIterator sidecarsFromCheckpoint(Path checkpointPath) throws IOException { AtomicInteger sidecarCounter = new AtomicInteger(); CloseableIterator existingSidecars; existingSidecars = engine .getParquetHandler() .readParquetFiles( singletonCloseableIterator(getFileStatus(checkpointPath)), CHECKPOINT_SCHEMA, Optional.empty()) .map(FileReadResult::getData) .map(filterActions(Map.of("sidecar", sidecarCounter))); if (sidecarMergeThreshold > 0 && lastSidecarCount >= sidecarMergeThreshold - 1) { // Too many existing sidecars. Merge them into one. try (CloseableIterator sidecarFiles = existingSidecars .flatMap(FilteredColumnarBatch::getRows) .map( row -> { Row sidecar = row.getStruct(CHECKPOINT_SCHEMA.indexOf("sidecar")); String path = sidecar.getString(SidecarFile.READ_SCHEMA.indexOf("path")); Path fullPath = new Path(sidecarFile(snapshot.getLogPath(), path)); return getFileStatus(fullPath); }); CloseableIterator addFileRows = engine .getParquetHandler() .readParquetFiles(sidecarFiles, SIDECAR_SCHEMA, Optional.empty()) .map(FileReadResult::getData) .map(ColumnVectorUtils::wrap)) { existingSidecars = rowsToBatch(Stream.of(sidecarFromAddFiles(addFileRows))); lastSidecarCount = 1; } } return existingSidecars; } /** * Return a mapping function that further filter add files from a filtered column batch * * @return a mapping function that applies to a filtered column batch */ private Function filterAddFiles() { return (input) -> { int addOrdinal = input.getData().getSchema().indexOf("add"); return new FilteredColumnarBatch( input.getData(), ColumnVectorUtils.filter( input.getData().getSize(), (rowId) -> input.getSelectionVector().map(cv -> cv.getBoolean(rowId)).orElse(true) && !input.getData().getColumnVector(addOrdinal).isNullAt(rowId))); }; } /** * Returns a mapping function that: * *

    *
  • Aggregates {@code txn} actions into {@link #transactionIds} (max version per appId) *
  • Filters rows where {@code notNullName} is non-null *
  • Optionally increments {@code counter} for each retained row *
* *

This is used both for selecting {@code AddFile} rows ("add") when generating the sidecar and * for selecting {@code sidecar} rows when merging sidecar references from a prior checkpoint. * * @param nameToCounters a map of key: name of the column that must be non-null for a row to be * retained. value: counter incremented for each retained row; * @return mapping function that applies the filter (and aggregation side-effects) to a batch */ private Function filterActions( Map nameToCounters) { return (columnarBatch) -> { int txnOrdinal = columnarBatch.getSchema().indexOf("txn"); int dmOrdinal = columnarBatch.getSchema().indexOf("domainMetadata"); var entries = new ArrayList<>(nameToCounters.entrySet()); Integer[] ordinals = new Integer[nameToCounters.size()]; AtomicInteger[] counters = new AtomicInteger[nameToCounters.size()]; for (int i = 0; i < entries.size(); i++) { ordinals[i] = columnarBatch.getSchema().indexOf(entries.get(i).getKey()); counters[i] = entries.get(i).getValue(); } return new FilteredColumnarBatch( columnarBatch, ColumnVectorUtils.filter( columnarBatch.getSize(), (rowId) -> { ColumnVector txnVector = columnarBatch.getColumnVector(txnOrdinal); if (!txnVector.isNullAt(rowId)) { String appId = txnVector.getChild(0).getString(rowId); long txnVersion = txnVector.getChild(1).getLong(rowId); transactionIds.merge(appId, txnVersion, Math::max); } ColumnVector dmVector = columnarBatch.getColumnVector(dmOrdinal); if (!dmVector.isNullAt(rowId)) { String domain = dmVector.getChild(0).getString(rowId); String configuration = dmVector.getChild(1).getString(rowId); boolean removed = dmVector.getChild(2).getBoolean(rowId); if (removed) { domainMetadatas.remove(domain); } else { domainMetadatas.put(domain, configuration); } } for (int i = 0; i < ordinals.length; i++) { if (!columnarBatch.getColumnVector(ordinals[i]).isNullAt(rowId)) { counters[i].incrementAndGet(); return true; } } return false; })); }; } /** Reads {@code _last_checkpoint} and returns the checkpoint metadata. */ private CheckpointMetaData readLastCheckpointInfo() { try (CloseableIterator jsonIter = engine .getJsonHandler() .readJsonFiles( singletonCloseableIterator(FileStatus.of(lastCheckpointFilePath.toString())), CheckpointMetaData.READ_SCHEMA, Optional.empty())) { return InternalUtils.getSingularRow(jsonIter) .map(CheckpointMetaData::fromRow) .orElse(EMPTY_META); } catch (Exception ignore) { // Best-effort: absence or parse errors mean "no usable previous checkpoint." return EMPTY_META; } } /* Wrap a stream of actions as column batches */ private CloseableIterator rowsToBatch(Stream input) { FilteredColumnarBatch checkpointContent = new FilteredColumnarBatch( new DefaultRowBasedColumnarBatch( CHECKPOINT_SCHEMA, input.map(CheckpointActionRow::new).collect(Collectors.toList())), Optional.empty()); return singletonCloseableIterator(checkpointContent); } /* write a new sidecar from the given addfiles */ private SidecarFile sidecarFromAddFiles(CloseableIterator actions) throws IOException { String sidecarName = UUID.randomUUID().toString(); Path sidecarPath = v2CheckpointSidecarFile(snapshot.getLogPath(), sidecarName); engine.getParquetHandler().writeParquetFileAtomically(String.valueOf(sidecarPath), actions); FileStatus fileStatus = getFileStatus(sidecarPath); return new SidecarFile( String.format("%s.parquet", sidecarName), // Kernel does not support absolute paths. fileStatus.getSize(), fileStatus.getModificationTime()); } /** * Resolves the {@link FileStatus} for a path using the engine filesystem client. * * @param path file path * @return file status */ private FileStatus getFileStatus(Path path) { try { return engine.getFileSystemClient().getFileStatus(path.toString()); } catch (IOException e) { throw ExceptionUtils.wrap(e); } } private Stream getTransactions() { return transactionIds.entrySet().stream() .map(e -> new SetTransaction(e.getKey(), e.getValue(), Optional.empty())); } private Stream getDomainMetadatas() { return domainMetadatas.entrySet().stream() .map(e -> new DomainMetadata(e.getKey(), e.getValue(), false)); } } ================================================ FILE: flink/src/main/java/io/delta/flink/kernel/ColumnVectorUtils.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink.kernel; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.types.BooleanType; import io.delta.kernel.types.DataType; import io.delta.kernel.types.StructType; import java.util.Optional; import java.util.function.Function; import java.util.function.Predicate; public class ColumnVectorUtils { public static FilteredColumnarBatch wrap(ColumnarBatch data) { return new FilteredColumnarBatch(data, Optional.empty()); } public static Function notNullAt(int ordinal) { return (batch) -> new FilteredColumnarBatch(batch, notNull(batch.getColumnVector(ordinal))); } public static Function child(String childName) { return (batch) -> { StructType childSchema = (StructType) batch.getData().getSchema().get(childName).getDataType(); int childIndex = batch.getData().getSchema().indexOf(childName); ColumnarBatch newData = new ColumnarBatch() { @Override public StructType getSchema() { return childSchema; } @Override public ColumnVector getColumnVector(int ordinal) { return batch.getData().getColumnVector(childIndex).getChild(ordinal); } @Override public int getSize() { return batch.getData().getSize(); } }; return new FilteredColumnarBatch(newData, batch.getSelectionVector()); }; } /** * Create a column vector that filter out data based on the pred(data, rowId) * * @param size the filter size * @param pred a predicate taking (data, rowId) as input * @return a column vector masking out the rows with pred returning false */ public static Optional filter(int size, Predicate pred) { return Optional.of( new ColumnVector() { @Override public DataType getDataType() { return BooleanType.BOOLEAN; } @Override public int getSize() { return size; } @Override public void close() {} @Override public boolean isNullAt(int rowId) { return false; } @Override public boolean getBoolean(int rowId) { return pred.test(rowId); } }); } public static Optional notNull(ColumnVector input) { return filter(input.getSize(), (rowId) -> !input.isNullAt(rowId)); } } ================================================ FILE: flink/src/main/java/io/delta/flink/table/AbstractKernelTable.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink.table; import dev.failsafe.Failsafe; import dev.failsafe.Fallback; import dev.failsafe.RetryPolicy; import dev.failsafe.function.CheckedRunnable; import dev.failsafe.function.CheckedSupplier; import io.delta.flink.Conf; import io.delta.flink.table.postcommit.ChecksumListener; import io.delta.flink.table.postcommit.MaintenanceListener; import io.delta.kernel.*; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.defaults.internal.json.JsonUtils; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.TableAlreadyExistsException; import io.delta.kernel.expressions.Column; import io.delta.kernel.expressions.Literal; import io.delta.kernel.internal.DeltaLogActionUtils; import io.delta.kernel.internal.data.TransactionStateRow; import io.delta.kernel.transaction.CreateTableTransactionBuilder; import io.delta.kernel.transaction.DataLayoutSpec; import io.delta.kernel.transaction.UpdateTableTransactionBuilder; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterable; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.DataFileStatus; import java.io.File; import java.net.URI; import java.net.URISyntaxException; import java.time.Duration; import java.util.*; import java.util.concurrent.*; import java.util.function.Function; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * An abstract base class for {@link DeltaTable} implementations backed by the Delta Kernel. * *

{@code AbstractKernelTable} provides common functionality for interacting with Delta tables, * including access to table metadata, schema, partitioning information, and commit operations. * Concrete subclasses are responsible for supplying catalog-specific or filesystem-specific logic * such as table discovery, path resolution, and storage I/O. * *

This class centralizes shared behavior so that different table backends (e.g., Hadoop-based * tables, catalog-managed tables, custom catalogs) can implement only the backend-specific portions * while inheriting consistent Delta table semantics. * *

Subclasses must provide their own mechanisms for interpreting table identifiers and resolving * them into physical locations or catalog entries. See also @link{io.delta.flink.table.Catalog} */ public abstract class AbstractKernelTable implements DeltaTable { protected static String ENGINE_INFO = "DeltaSink"; protected static Logger LOG = LoggerFactory.getLogger(AbstractKernelTable.class); /** * Normalizes the given URI string to a canonical form. The normalization includes: * *

    *
  • Ensuring file URIs use the standard triple-slash form (e.g., {@code file:/abc/def} → * {@code file:///abc/def}). *
  • Appending a trailing slash to paths that do not already end with {@code /}. *
* *

This method is useful for making URI comparisons consistent and avoiding issues caused by * variations in file URI formatting or missing trailing path delimiters. * * @param input the URI to normalize; * @return the normalized URI */ public static URI normalize(URI input) { if (input == null) { return null; } URI target = input; if (target.getScheme() == null) { target = new File(input.toString()).toPath().toUri(); } else if (target.getScheme().equals("file")) { // Normalize "file:/xxx/" to "file:///xxx/" target = new File(input).toPath().toUri(); } try { // Normalize "abc://def/xxx" to "abc://def/xxx/" if (!target.getPath().endsWith("/")) { target = new URI( target.getScheme(), Optional.ofNullable(target.getHost()).orElse(""), target.getPath() + "/", target.getFragment()); } } catch (URISyntaxException e) { throw new RuntimeException(e); } return target; } /** * Normalize the provided partition column names. * * @param rawPartitions input list of column names. * @return a partition info that does not contain null or empty string. */ protected static List normalize(List rawPartitions) { if (rawPartitions == null) { return List.of(); } return rawPartitions.stream().filter(StringUtils::isNotEmpty).collect(Collectors.toList()); } protected final DeltaCatalog catalog; protected String tableId; protected String tableUUID; protected URI tablePath; protected final TableConf conf; /* * This is the TransactionStateRow in json. Needed mainly by {@link #writeParquet} */ protected String serializedTableState; protected List partitionColumns; private SnapshotCacheManager cacheManager; protected final List metricListeners; protected final List eventListeners; // Engine is not serializable, it will be lazily re-created protected transient volatile Engine engine; // These fields are not serializable. They will be reinitialized in {@link #open} protected transient StructType schema; protected transient Row tableState; protected transient CredentialManager credentialManager; // Single-thread thread pool for executing interruptible operation. protected transient ExecutorService refreshThreadPool = null; // Thread pool for all kinds of async works protected transient ExecutorService generalThreadPool = null; public AbstractKernelTable( DeltaCatalog catalog, String tableId, Map conf, StructType schema, List partitionColumns) { this.catalog = catalog; this.tableId = tableId; // Allow subclasses to provide extra confs Map mergedConfs = new HashMap<>(conf); mergedConfs.putAll(extraConf()); this.conf = new TableConf(mergedConfs); this.schema = schema; this.partitionColumns = normalize(partitionColumns); this.cacheManager = SnapshotCacheManager.getInstance(); this.metricListeners = new ArrayList<>(); this.eventListeners = new ArrayList<>(); addEventListener(new MaintenanceListener()); addEventListener(new ChecksumListener()); } public AbstractKernelTable(DeltaCatalog catalog, String tableId, Map conf) { this(catalog, tableId, conf, null, null); } // ===================== // Override methods // ===================== @Override public String getId() { return tableId; } @Override public StructType getSchema() { return schema; } @Override public List getPartitionColumns() { return partitionColumns; } @Override public void open() { catalog.open(); // init all transient variables if (refreshThreadPool == null) { refreshThreadPool = Executors.newSingleThreadExecutor(); } if (generalThreadPool == null) { generalThreadPool = Executors.newFixedThreadPool(Conf.getInstance().getTableThreadPoolSize()); } if (credentialManager == null) { credentialManager = createCredentialManager(); } if (serializedTableState == null) { withRetry( () -> { loadDeltaTable(); return null; }); } if (tableState == null) { tableState = JsonUtils.rowFromJson(serializedTableState, TransactionStateRow.SCHEMA); } if (schema == null) { schema = TransactionStateRow.getLogicalSchema(tableState); } } @Override public synchronized void close() throws InterruptedException { LOG.info("Closing table : {}", getId()); if (refreshThreadPool != null) { withTiming( "close", () -> { refreshThreadPool.shutdownNow(); // This should return quickly if all tasks are interruptible refreshThreadPool.awaitTermination(10, TimeUnit.MINUTES); refreshThreadPool = null; }); } } @Override public void refresh() { refresh(null); } @Override public Optional commit( CloseableIterable actions, String appId, long txnId, Map properties) { return withTiming( "commit", () -> withRetry( () -> { Engine localEngine = getEngine(); Optional snapshotOpt = snapshot(); if (snapshotOpt.isEmpty()) { throw new IllegalStateException("Snapshot should exist"); } Snapshot snapshot = snapshotOpt.get(); UpdateTableTransactionBuilder txnBuilder = snapshot.buildUpdateTableTransaction(ENGINE_INFO, Operation.WRITE); txnBuilder.withTransactionId(appId, txnId); txnBuilder.withTablePropertiesAdded(properties); Transaction txn = txnBuilder.build(engine); TransactionCommitResult result = withTiming("commit.txn", () -> txn.commit(localEngine, actions)); return result .getPostCommitSnapshot() .map( pcSnapshot -> { this.refresh(pcSnapshot); onPostCommit(pcSnapshot); return pcSnapshot; }); })); } @Override public CloseableIterator writeParquet( String pathSuffix, CloseableIterator data, Map partitionValues) { return withRetry( () -> { Engine localEngine = getEngine(); Row writeState = getWriteState(); final CloseableIterator physicalData = Transaction.transformLogicalData(localEngine, writeState, data, partitionValues); final DataWriteContext writeContext = Transaction.getWriteContext(localEngine, writeState, partitionValues); LOG.debug("Writing file to path {} with suffix {}", getTablePath(), pathSuffix); final CloseableIterator dataFiles = localEngine .getParquetHandler() .writeParquetFiles( getTablePath().resolve(pathSuffix).toString(), physicalData, writeContext.getStatisticsColumns()); return Transaction.generateAppendActions( localEngine, writeState, dataFiles, writeContext); }); } /** * Load snapshot using a separated thread. This will allow external request to interrupt the * thread during time-consuming operations in loading snapshot, such as log replay. * * @return loaded snapshot, null if the table does not exist */ protected Optional snapshot() { Function> body = (key) -> { try { return withTiming( "loadLatestSnapshot", () -> Optional.of(refreshThreadPool.submit(this::loadLatestSnapshot).get())); } catch (Exception e) { if (ExceptionUtils.isTableNotFound.test(e)) { return Optional.empty(); } throw ExceptionUtils.wrap(e); } }; String path = tablePath.toString(); LOG.debug("Loading snapshot for path {}", path); return cacheManager.get(path, this::versionExists, body); } /** * Subclass must implement this method to fetch a Kernel snapshot * * @return latest snapshot of the table */ protected abstract Snapshot loadLatestSnapshot(); /** * Subclass may implement this to achieve fast cache validation. This method is expected to be * faster than {@link #loadLatestSnapshot()}. The default implementation checks if a file with the * given version exists. NOTE: catalog-managed tables need to override this method to check * against catalog. * * @return the latest version of the table, null if unknown / not supported */ protected boolean versionExists(Long version) { try { return !DeltaLogActionUtils.getCommitFilesForVersionRange( getEngine(), new io.delta.kernel.internal.fs.Path(tablePath), version, Optional.empty()) .isEmpty(); } catch (Exception e) { return false; } } /** Refresh with the provided snapshot */ protected void refresh(Snapshot snapshot) { withTiming( "refresh", () -> withRetry( () -> { Snapshot currentSnapshot = snapshot; if (currentSnapshot == null) { currentSnapshot = snapshot().orElse(null); } if (currentSnapshot == null) { return null; } this.schema = currentSnapshot.getSchema(); this.partitionColumns = currentSnapshot.getPartitionColumnNames(); // Refresh table state this.tableState = currentSnapshot .buildUpdateTableTransaction("dummy", Operation.WRITE) .build(getEngine()) .getTransactionState(getEngine()); this.serializedTableState = JsonUtils.rowToJson(this.tableState); return null; })); } protected CreateTableTransactionBuilder buildCreateTableTransaction() { return TableManager.buildCreateTableTransaction(tablePath.toString(), schema, ENGINE_INFO); } /** Create a new Delta snapshot representing the empty table at the given location. */ protected void createDeltaTable() { Engine engine = getEngine(); CreateTableTransactionBuilder txnBuilder = buildCreateTableTransaction().withTableProperties(conf.catalogConf()); if (!partitionColumns.isEmpty()) { txnBuilder.withDataLayoutSpec( DataLayoutSpec.partitioned( Optional.of(partitionColumns) .map(nonEmpty -> nonEmpty.stream().map(Column::new).collect(Collectors.toList())) .orElseGet(Collections::emptyList))); } try { TransactionCommitResult result = txnBuilder.build(engine).commit(engine, CloseableIterable.emptyIterable()); result.getPostCommitSnapshot().ifPresent(this::onPostCommit); } catch (TableAlreadyExistsException ignore) { // Concurrent open may cause this. Ignore it safely. } } /** * Load table information from the delta table. This method loads the table if it exists, or * creates a new table entry in catalog if the table does not exist */ protected void loadDeltaTable() { DeltaCatalog.TableDescriptor info; try { info = catalog.getTable(tableId); tableUUID = info.uuid; tablePath = normalize(info.tablePath); } catch (ExceptionUtils.ResourceNotFoundException notFound) { catalog.createTable( tableId, schema, partitionColumns, conf.catalogConf(), tableDesc -> { this.tablePath = normalize(tableDesc.tablePath); this.tableUUID = tableDesc.uuid; createDeltaTable(); }); } final Optional latestSnapshotOpt = snapshot(); if (latestSnapshotOpt.isEmpty()) { throw new IllegalStateException("Snapshot not initialized"); } Snapshot latestSnapshot = latestSnapshotOpt.get(); // We use a temporary transaction to generate a TransactionStateRow. // It serves as a holder for schema and partition columns. // The transaction will not be committed, and is discarded afterward. Row existingTableState = latestSnapshot .buildUpdateTableTransaction(ENGINE_INFO, Operation.WRITE) .build(getEngine()) .getTransactionState(getEngine()); this.serializedTableState = JsonUtils.rowToJson(existingTableState); this.schema = latestSnapshot.getSchema(); this.partitionColumns = latestSnapshot.getPartitionColumnNames(); } // Engine will be invalidated when credentials expire public Engine getEngine() { if (engine == null) { synchronized (this) { if (engine == null) { engine = createEngine(); } } } return engine; } /** * Subclass may implement this method to generate an engine. * * @return engine to access the tables */ protected Engine createEngine() { Configuration conf = new Configuration(); // Built-in configurations for common file system access conf.set("fs.file.impl", "org.apache.hadoop.fs.LocalFileSystem"); conf.set("fs.AbstractFileSystem.file.impl", "org.apache.hadoop.fs.local.LocalFs"); conf.set("fs.s3.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem"); conf.set("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem"); conf.set("fs.s3a.path.style.access", "false"); conf.set("fs.s3.impl.disable.cache", "true"); conf.set("fs.s3a.impl.disable.cache", "true"); conf.set("fs.abfs.impl", "org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem"); conf.set("fs.abfss.impl", "org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem"); conf.set("fs.AbstractFileSystem.abfs.impl", "org.apache.hadoop.fs.azurebfs.Abfs"); conf.set("fs.AbstractFileSystem.abfss.impl", "org.apache.hadoop.fs.azurebfs.Abfss"); conf.set("fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem"); conf.set("fs.AbstractFileSystem.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS"); this.conf.engineConf().forEach(conf::set); this.credentialManager.getCredentials().forEach(conf::set); // Explicitly load external conf files // TODO this is because Flink does not auto load this file in Docker conf.addResource(new Path("/opt/flink/conf/core-site.xml")); return DefaultEngine.create(conf); } public SnapshotCacheManager getCacheManager() { return cacheManager; } public void setCacheManager(SnapshotCacheManager cacheManager) { this.cacheManager = cacheManager; } public DeltaCatalog getCatalog() { return catalog; } public String getTableUUID() { return tableUUID; } public TableConf getConf() { return conf; } protected Row getWriteState() { return tableState; } /** The table storage location where all data and metadata files should be stored. */ public URI getTablePath() { return tablePath; } protected Map extraConf() { return Map.of(); } private CredentialManager createCredentialManager() { return new CredentialManager( () -> catalog.getCredentials(this.getTableUUID()), this::refreshCredential); } /** * Retry on retryable exceptions. It must be used on all methods that need storage credentials. * {@see ExceptionUtils.isRetryableException} * * @param body the execution body. * @return the return value from body */ protected RET withRetry(CheckedSupplier body) { RetryPolicy retryPolicy = RetryPolicy.builder() .handleIf(ExceptionUtils::isRetryableException) .withBackoff( Duration.ofMillis(Conf.getInstance().getSinkRetryDelayMs()), Duration.ofMillis(Conf.getInstance().getSinkRetryMaxDelayMs()), 2.0) .withMaxAttempts(Conf.getInstance().getSinkRetryMaxAttempt()) .onRetry( e -> { LOG.warn( "Retrying attempt {} on exception {}", e.getAttemptCount(), e.getLastFailure()); if (CredentialManager.isCredentialsExpired.test(e.getLastFailure())) { refreshCredential(); } else { reloadSnapshot(); } }) .build(); Fallback fallback = Fallback.builder((Object) Optional.empty()).handleIf(ExceptionUtils.isSwallowable).build(); return Failsafe.with(retryPolicy, fallback).get(body); } public RET withTiming(String name, Callable body) { long start = System.nanoTime(); try { return body.call(); } catch (Throwable t) { throw ExceptionUtils.wrap(t); } finally { long elapse = System.nanoTime() - start; onMetric(name, elapse); } } public void withTiming(String name, CheckedRunnable body) { long start = System.nanoTime(); try { body.run(); } catch (Throwable t) { throw ExceptionUtils.wrap(t); } finally { long elapse = System.nanoTime() - start; onMetric(name, elapse); } } public Future executeWithTiming(String name, Callable body) { return generalThreadPool.submit(() -> withTiming(name, body)); } public Future executeWithTiming(String name, CheckedRunnable body) { return generalThreadPool.submit(() -> withTiming(name, body)); } // =================== // Table Listeners // =================== public void addMetricListener(MetricListener listener) { this.metricListeners.add(listener); } public void removeMetricListener(MetricListener listener) { this.metricListeners.remove(listener); } protected void onMetric(String event, long time) { this.metricListeners.forEach(listener -> listener.onEvent(event, time)); } public void addEventListener(TableEventListener listener) { this.eventListeners.add(listener); } public void removeEventListener(TableEventListener listener) { this.eventListeners.remove(listener); } public void onPostCommit(Snapshot snapshot) { eventListeners.forEach( listener -> { try { listener.onPostCommit(this, snapshot); } catch (Exception e) { LOG.error("Suppressed exception from listener", e); } }); } /** Callback invoked when retry need to refresh credentials (credential exception) */ protected void refreshCredential() { // Force the recreation of engine (and reload credentials) next time on use. synchronized (this) { this.engine = null; } } /** Callback invoked when retry need to reload snapshot (concurrent exception). */ protected void reloadSnapshot() { // Client need to clean up snapshot cache if any cacheManager.invalidate(getTablePath().toString()); } } ================================================ FILE: flink/src/main/java/io/delta/flink/table/CredentialManager.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink.table; import io.delta.flink.Conf; import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; import java.util.function.Supplier; /** * Manages credentials with proactive refresh semantics. * *

{@code CredentialManager} is responsible for fetching credentials from an external source and * refreshing them before expiration. The expiration time is expected to be encoded in the * credential map under the key {@link #CREDENTIAL_EXPIRATION_KEY}. * *

The manager caches the most recently fetched credentials and re-fetches them when the * expiration time is approaching. * *

This class is intentionally agnostic of the underlying credential source and refresh strategy. * Callers provide: * *

    *
  • A {@link Supplier} that fetches the latest credentials *
  • A {@link Runnable} callback that is invoked when a refresh occurs (for example, to notify * dependent components) *
* * Typical usage: * CredentialManager credManager = new CredentialManager(loadCredFromCatalog, callback); * credManager.getCredentials(); // Guaranteed to be an refreshed credential * // Wait for a long time * credManager.getCredentials(); // Internally refreshed, still return valid credentials. * * *

This class is thread-safe. */ public class CredentialManager { /** * Key in the credential map that represents the credential expiration time. * *

The value is expected to be a string representation of a timestamp (for example, epoch * milliseconds), interpretable by the implementation. */ protected static String CREDENTIAL_EXPIRATION_KEY = "credential.expiration"; protected static ScheduledExecutorService refreshExecutors = Executors.newScheduledThreadPool(Conf.getInstance().getCredentialsRefreshThreadPoolSize()); /** * Determines whether the given exception indicates a credential-related failure. * *

This predicate can be used by callers to detect failures that should trigger a credential * refresh or retry logic. * * @return a Predicate that returns {@code true} if the exception is related to invalid or expired * credentials; {@code false} otherwise */ public static Predicate isCredentialsExpired = ExceptionUtils.recursiveCheck(ex -> ex instanceof java.nio.file.AccessDeniedException); /** Supplier used to fetch the latest credentials from the underlying source. */ private final Supplier> credSupplier; /** * Callback invoked after credentials are refreshed. * *

This can be used to trigger downstream updates or reconfiguration when credentials change. */ private final Runnable refreshCallback; private AtomicReference> cachedCredentials = new AtomicReference<>(); public CredentialManager(Supplier> supplier, Runnable refreshCallback) { this.credSupplier = supplier; this.refreshCallback = refreshCallback; } /** * Returns the current credentials, refreshing them if expiration is approaching. * *

If no credentials have been fetched yet, this method will fetch and cache them. On * subsequent calls, the cached credentials are returned unless they are near expiration, in which * case a refresh is triggered. * *

The refresh strategy (for example, how close to expiration a refresh occurs) is * implementation-defined. * * @return the current valid credentials */ Map getCredentials() { Map cached = cachedCredentials.get(); if (cached != null) return cached; Map newCredentials = this.credSupplier.get(); if (cachedCredentials.compareAndSet(null, newCredentials)) { scheduleNextRefresh(newCredentials); return newCredentials; } return cachedCredentials.get(); } protected void scheduleNextRefresh(Map newCredentials) { long expiration = -1; try { expiration = Long.parseLong(newCredentials.getOrDefault(CREDENTIAL_EXPIRATION_KEY, "-1")); } catch (NumberFormatException ignore) { } if (expiration >= 0) { long refreshDelay = Math.max( 100, // A minimal wait of 100ms if the refresh delay is too small expiration - Conf.getInstance().getCredentialsRefreshAheadInMs() - System.currentTimeMillis()); refreshExecutors.schedule( () -> { Map existingCredential = cachedCredentials.get(); Map refreshedCredential = this.credSupplier.get(); if (cachedCredentials.compareAndSet(existingCredential, refreshedCredential)) { this.refreshCallback.run(); scheduleNextRefresh(refreshedCredential); } }, refreshDelay, TimeUnit.MILLISECONDS); } } } ================================================ FILE: flink/src/main/java/io/delta/flink/table/DeltaCatalog.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink.table; import io.delta.kernel.types.StructType; import java.io.Serializable; import java.net.URI; import java.util.List; import java.util.Map; import java.util.function.Consumer; /** * A {@code Catalog} abstracts interaction with an external table catalog or metadata service. * *

The catalog is responsible for resolving logical table identifiers into concrete table * metadata and for providing the credentials required to access the underlying storage system. This * abstraction allows different catalog implementations (e.g., filesystem-based catalogs, * metastore-backed catalogs, or REST-based catalogs) to be used interchangeably by higher-level * components. * *

Typical responsibilities of a {@code Catalog} include: * *

    *
  • mapping table identifiers to physical table locations, *
  • providing stable table UUIDs for identification and caching, and *
  • supplying credential or configuration information required for table access. *
*/ public interface DeltaCatalog extends Serializable { /** * Init the catalog instance and make it ready for use. Should be called at least once before the * catalog can be safely used. Calling open on an already opened table has no effect. */ default void open() {} /** * Loads metadata for a table identified by the given table identifier. * *

The identifier format and naming conventions are defined by the specific catalog * implementation. Implementations may interpret the identifier as a logical name, a * fully-qualified path, or another catalog-specific reference. * * @param tableId the logical identifier of the table to load; must not be {@code null} * @return a {@link TableDescriptor} object describing the resolved table * @throws IllegalArgumentException if the identifier is invalid * @throws ExceptionUtils.ResourceNotFoundException if the table cannot be resolved or loaded */ TableDescriptor getTable(String tableId); /** * Creates a new table in the catalog with the given schema, partitioning, and properties. * *

The table is identified by {@code tableId} and is initialized with the provided {@link * StructType} schema. Optional partition columns define how the table data is physically * organized, and table properties supply additional configuration such as format-specific options * or metadata. * * @param tableId The unique identifier of the table to create within the catalog. * @param schema The logical schema of the table, describing column names, data types, and * nullability. * @param partitions A list of column names used for partitioning the table; an empty list * indicates an unpartitioned table. * @param properties A map of table properties for configuration and metadata; may be empty but * must not be {@code null}. * @param callback for the caller to init the table when the storage URI is allocated. * @throws ExceptionUtils.ResourceAlreadyExistException If a table with the same identifier * already exists in the catalog. */ void createTable( String tableId, StructType schema, List partitions, Map properties, Consumer callback); /** * Returns the credentials or configuration properties required to access the table identified by * the given UUID. * *

The returned map may contain authentication information, endpoint configuration, or other * filesystem- or catalog-specific properties. The exact contents and semantics are defined by the * catalog implementation. * * @param uuid the unique identifier of the table * @return a map of credential or configuration properties; may be empty but never {@code null} */ Map getCredentials(String uuid); /** * A container for table metadata resolved by a {@link DeltaCatalog}. * *

{@code TableInfo} describes the essential properties needed to locate and access a table, * independent of the underlying catalog implementation. */ class TableDescriptor { /** The logical identifier used to resolve the table. */ String tableId; /** A stable UUID that uniquely identifies the table. */ String uuid; /** The normalized physical location of the table. */ URI tablePath; public TableDescriptor() {} public TableDescriptor(String tableId, String uuid, URI tablePath) { this.tableId = tableId; this.uuid = uuid; this.tablePath = tablePath; } } } ================================================ FILE: flink/src/main/java/io/delta/flink/table/DeltaTable.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink.table; import io.delta.kernel.Snapshot; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.expressions.Literal; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterable; import io.delta.kernel.utils.CloseableIterator; import java.io.IOException; import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.Optional; /** * A {@code DeltaTable} represents a logical view of a Delta table and provides access to both table * metadata (such as schema and partitioning) and operations for reading and writing table data. * *

A {@code DeltaTable} instance abstracts the underlying Delta transaction log and storage * layout. Implementations are responsible for: * *

    *
  • exposing immutable table metadata (schema, partition information) *
  • managing transaction boundaries and versioned commits *
  • coordinating reads and writes against the physical table storage *
  • serializing table changes into Delta {@code actions} and committing them atomically *
* *

All implementations must be {@link Serializable} to allow use in distributed execution * environments. */ public interface DeltaTable extends Serializable, AutoCloseable { /** * Returns a stable identifier that uniquely represents this table within its catalog or storage * system. * *

These are some examples that may be used as identifiers, depending on the subclass * implementation and the catalog in use. * *

    *
  • a logical table name (e.g., {@code "catalog.database.table"}) *
  • a filesystem or object-store URI *
* * @return a unique logical identifier for the table */ String getId(); /** * Returns the table schema as a {@link StructType}. * *

The schema defines the logical column structure of the table. Implementations should * guarantee that the schema corresponds to the latest committed version unless otherwise * documented. * * @return the table schema */ StructType getSchema(); /** * Returns the list of partition columns for this table. * *

The returned list defines the physical partitioning strategy used by the table. The ordering * of columns follows the table’s partition specification and should be stable across versions. * * @return an ordered list of partition column names */ List getPartitionColumns(); /** * Init the table instance and make it ready for use. Should be called at least once before the * table can be safely used. Calling open on an already opened table has no effect. */ void open(); /** * Commits a new version to the table by applying the provided Delta actions. * *

Actions may include (but are not limited to): * *

    *
  • {@code AddFile} records representing new data files *
  • {@code RemoveFile} records removing obsolete files *
  • metadata updates or protocol changes *
* *

Implementations must ensure atomicity: either all provided actions are committed as part of * a new table version, or none are. Commit conflicts should be detected and surfaced as * exceptions. * * @param actions an iterable collection of Delta actions to commit; the caller is responsible for * closing the iterable * @param appId application id used for this commit. See transaction identifier in Delta protocol. * @param txnId the transaction identifier to be used for this commit. * @param properties table properties to be updated with this commit. */ Optional commit( CloseableIterable actions, String appId, long txnId, Map properties); /** * Refreshes the table state by reloading the latest snapshot metadata. * *

This method updates the in-memory view of the table to reflect the most recently committed * version, including: * *

    *
  • the latest table schema, *
  • partition column definitions, and *
  • any other metadata derived from the current Delta log snapshot. *
* *

{@code refresh()} should be invoked when external changes to the table may have occurred * (for example, commits from other writers) and the caller requires an up-to-date view before * performing read or write operations. * *

Implementations may perform I/O and metadata parsing as part of this operation. */ void refresh(); /** * Writes one or more Parquet files as part of the table and emits the corresponding {@code * AddFile} action describing the newly written data. * *

This operation is responsible for: * *

    *
  • writes to the underlying storage layer using the specified {@code data} *
  • constructing physical file paths by appending {@code pathSuffix} to the table root *
  • materializing partition values into the file metadata *
  • returning a Row describing the resulting {@code AddFile} action *
* *

The returned iterator typically contains exactly one row (the AddFile action), but * implementations may return multiple actions depending on file-splitting behavior. * * @param pathSuffix a suffix appended to the table path when generating file locations. The * result path will be `//` * @param data an iterator over row batches to be written as Parquet files; this method will close * it on consumption. * @param partitionValues a mapping of partition column names to their literal values * @return an iterator over {@code Row} objects representing the AddFile actions generated during * the write * @throws IOException if data writing or file creation fails */ CloseableIterator writeParquet( String pathSuffix, CloseableIterator data, Map partitionValues) throws IOException; } ================================================ FILE: flink/src/main/java/io/delta/flink/table/ExceptionUtils.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink.table; import dev.failsafe.function.CheckedConsumer; import io.delta.kernel.exceptions.ConcurrentTransactionException; import io.delta.kernel.exceptions.ConcurrentWriteException; import io.delta.kernel.exceptions.TableAlreadyExistsException; import io.delta.kernel.exceptions.TableNotFoundException; import java.util.ConcurrentModificationException; import java.util.function.Consumer; import java.util.function.Predicate; /** Utility methods and common exception types for exception inspection and handling. */ public class ExceptionUtils { public static Predicate isTableNotFound = ExceptionUtils.recursiveCheck(ex -> ex instanceof TableNotFoundException); public static Predicate isSnapshotUpdated = ExceptionUtils.recursiveCheck( ex -> ex instanceof ConcurrentModificationException || ex instanceof ConcurrentWriteException || ex instanceof TableAlreadyExistsException); public static Predicate isSwallowable = ExceptionUtils.recursiveCheck(ex -> ex instanceof ConcurrentTransactionException); /** * Check if an exception is retryable. * * @param e exception * @return true if the exception is Authentication or Concurrency related. */ public static boolean isRetryableException(Throwable e) { return CredentialManager.isCredentialsExpired.test(e) || isSnapshotUpdated.test(e); } /** * Creates a predicate that applies the given predicate recursively to an exception and its causal * chain. * *

The returned predicate returns {@code true} if the supplied predicate matches the exception * itself or any of its causes. * * @param pred the predicate to apply to each exception in the causal chain * @return a predicate that recursively checks the exception and its causes */ public static Predicate recursiveCheck(Predicate pred) { return new Predicate() { @Override public boolean test(Throwable e) { if (e == null) { return false; } if (pred.test(e)) { return true; } return test(e.getCause()); } }; } /** Exception indicating that a requested resource does not exist. */ public static class ResourceNotFoundException extends RuntimeException { public ResourceNotFoundException(String message) { super(message); } } /** Exception indicating that a resource already exists and cannot be created again. */ public static class ResourceAlreadyExistException extends RuntimeException { public ResourceAlreadyExistException(String message) { super(message); } } public static RuntimeException wrap(Throwable t) { if (t instanceof RuntimeException) { return (RuntimeException) t; } return new RuntimeException(t); } public static Consumer wrap(CheckedConsumer body) { return t -> { try { body.accept(t); } catch (Throwable e) { throw new RuntimeException(e); } }; } } ================================================ FILE: flink/src/main/java/io/delta/flink/table/MetricListener.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink.table; import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; /** * A listener interface for receiving performance-related metric events emitted by Delta components. * *

Each event represents a completed operation identified by {@code eventName}, along with its * execution time in nanoseconds. * *

Implementations of this interface are expected to be lightweight and non-blocking, as * callbacks may be invoked on performance-critical execution paths. */ public interface MetricListener extends Serializable { /** * Called when a performance-related event occurs. * * @param eventName a logical name identifying the metric or operation (e.g., {@code * "snapshot.load"}, {@code "commit.retry"}) * @param elapseNano the elapsed time of the operation in nanoseconds */ void onEvent(String eventName, long elapseNano); /** * A {@link MetricListener} implementation that aggregates basic statistics (minimum, maximum, and * average elapsed time) for each metric. * *

Statistics are maintained per {@code eventName} and updated incrementally as events are * received. * *

This implementation is intended for lightweight in-memory aggregation and diagnostics. * Thread-safety guarantees depend on the concrete implementation and should be documented * accordingly. */ class StatsListener implements MetricListener { // (metricName, [count, max, min, sum]) Map summary = new HashMap<>(); @Override public void onEvent(String eventName, long elapseNano) { summary.merge( eventName, new long[] {1, elapseNano, elapseNano, elapseNano}, (existing, newvalue) -> { existing[0] += newvalue[0]; existing[1] = Math.max(existing[1], newvalue[1]); existing[2] = Math.min(existing[2], newvalue[2]); existing[3] += newvalue[3]; return existing; }); } /** * Get the statistical results of metrics * * @return a map of (metricName, [count, max, min, average]) */ public Map report() { return summary.entrySet().stream() .collect( Collectors.toMap( Map.Entry::getKey, entry -> new long[] { entry.getValue()[0], entry.getValue()[1], entry.getValue()[2], entry.getValue()[3] / entry.getValue()[0] })); } } /** Record the point-wise data points. */ class PointListener implements MetricListener { private final Map> data = new HashMap<>(); @Override public void onEvent(String eventName, long elapseNano) { data.computeIfAbsent(eventName, (key) -> new ArrayList<>()).add(elapseNano); } public Map> report() { return data; } } } ================================================ FILE: flink/src/main/java/io/delta/flink/table/SnapshotCacheManager.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink.table; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import io.delta.flink.Conf; import io.delta.kernel.Snapshot; import java.io.Serializable; import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.function.Predicate; /** * A {@code SnapshotCacheManager} provides a pluggable abstraction for caching {@link Snapshot} * instances keyed by a string identifier (for example, a table path). * *

The cache is primarily used to avoid repeatedly loading and reconstructing Delta table * snapshots from remote storage, which can be expensive for tables with long histories or high * access frequency. * *

Implementations may choose different caching strategies, including no-op caching or in-memory * eviction-based caching. * *

This interface is {@link Serializable} so that it can be safely used in distributed Flink jobs * and reconstructed across task restarts. */ public interface SnapshotCacheManager extends Serializable { /** * Inserts or updates a cached snapshot for the given key. * * @param key the cache key (for example, a table path) * @param snapshot the snapshot to cache */ default void put(String key, Snapshot snapshot) {} /** * Invalidates the cached snapshot associated with the given key. * * @param key the cache key to invalidate */ default void invalidate(String key) {} /** * Retrieves a cached snapshot for the given key, loading it on demand if it is not already * present in the cache. * *

If the snapshot is not cached, the provided {@code body} is invoked to compute the value, * which may then be stored in the cache depending on the implementation. * *

If a version exists, versionProbe is used to quickly verify if the version is up-to-date. * * @param key the cache key * @param versionProbe checks if a version exists. Expected to be faster than body. Used to * quickly verify if the cached snapshot is updated. It takes a version number and return true * if the version already exists. * @param body a callable used to compute the snapshot on cache miss * @return the cached or newly loaded snapshot, empty means no snapshot exists ( empty table ) */ default Optional get( String key, Predicate versionProbe, Function> body) { try { return body.apply(key); } catch (Exception e) { throw new RuntimeException(e); } } static SnapshotCacheManager getInstance() { return Conf.getInstance().getTableCacheEnable() ? new LocalCacheManager() : new NoCacheManager(); } /** * A no-op {@link SnapshotCacheManager} implementation that performs no caching. * *

This implementation always bypasses caching logic and is useful in scenarios where caching * is undesirable or needs to be disabled explicitly. */ class NoCacheManager implements SnapshotCacheManager { // Intentionally empty } /** * The default {@link SnapshotCacheManager} implementation backed by an in-memory, path-based * cache. * *

This implementation uses a bounded cache with time-based eviction to store {@link Snapshot} * instances, providing faster snapshot access while preventing unbounded memory growth. * *

Cache size and expiration are controlled via {@link Conf}. */ class LocalCacheManager implements SnapshotCacheManager { /** * A path-based snapshot cache used to speed up snapshot loading. * *

The cache is bounded in size and evicts entries based on access time to balance * performance and memory usage. */ static final Cache> SNAPSHOT_CACHE = Caffeine.newBuilder() .maximumSize(Conf.getInstance().getTableCacheSize()) .expireAfterAccess(Conf.getInstance().getTableCacheExpireInMs(), TimeUnit.MILLISECONDS) .build(); @Override public void put(String key, Snapshot snapshot) { SNAPSHOT_CACHE.put(key, Optional.ofNullable(snapshot)); } @Override public void invalidate(String key) { SNAPSHOT_CACHE.invalidate(key); } @Override public Optional get( String key, Predicate versionProbe, Function> body) { Optional cached = SNAPSHOT_CACHE.get(key, body); // Probe if `version + 1` already exists. long versionToProbe = cached.map(Snapshot::getVersion).orElse(-1L) + 1; // `version + 1` exists. It means current cache at `version` is outdated. if (versionProbe.test(versionToProbe)) { // Cache is outdated and needs reload SNAPSHOT_CACHE.invalidate(key); return SNAPSHOT_CACHE.get(key, body); } return cached; } } } ================================================ FILE: flink/src/main/java/io/delta/flink/table/TableConf.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink.table; import java.io.Serializable; import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Random; import java.util.stream.Collectors; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; import org.apache.flink.configuration.Configuration; /** * Per-table configuration for DeltaSink table maintenance behavior. * *

This class parses a raw string map (typically table options) into typed Flink {@link * ConfigOption} values and exposes: * *

    *
  • Catalog config: options that should be persisted with the table definition. *
  • Engine config: options that should be forwarded to the Delta Kernel engine at * runtime. *
*/ public class TableConf implements Serializable { private static final long serialVersionUID = 1L; /** Probability in [0.0, 1.0] to create a checkpoint on a commit. */ public static final ConfigOption CHECKPOINT_FREQUENCY = ConfigOptions.key("checkpoint.frequency") .doubleType() .defaultValue(0.0) .withDescription( "Probability in [0.0, 1.0] to create a checkpoint on a commit. " + "0.0 disables checkpoint creation; 1.0 creates a checkpoint on every commit."); /** Whether checksum file creation is enabled for this table. */ public static final ConfigOption CHECKSUM_ENABLED = ConfigOptions.key("checksum.enable") .booleanType() .defaultValue(true) .withDescription("Whether to generate checksum files for commits on this table."); private static final Map DEFAULT_CONFS = Map.of("delta.feature.v2Checkpoint", "supported"); private final Map raw; private final Configuration cfg; private final Random randgen = new Random(System.currentTimeMillis()); /** * Creates a {@link TableConf} from a raw key/value map (e.g., table options). * *

Unknown keys are preserved in {@link #raw} but ignored by typed accessors unless explicitly * surfaced via {@link #catalogConf()} or {@link #engineConf()}. * * @param conf raw configuration map; must not be null */ public TableConf(Map conf) { raw = Map.copyOf(Objects.requireNonNull(conf, "conf")); cfg = Configuration.fromMap(raw); validate(); } /** * Configuration to be persisted in the catalog. * *

This returns a subset of options that are intended to be stored with the table definition. * Now it includes configuration starts with "delta." * * @return a map of catalog-persisted configuration entries */ public Map catalogConf() { Map merged = new HashMap<>(DEFAULT_CONFS); merged.putAll( raw.entrySet().stream() .filter(entry -> entry.getKey().startsWith("delta.")) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); return merged; } /** * Configuration to be forwarded to the Kernel engine. * *

This returns the subset of configuration entries that are relevant to engine-side behavior. * If your engine uses different option names, translate them here. * * @return a map of engine configuration entries */ public Map engineConf() { return Map.of(); } /** @return whether checksum file creation is enabled for this table */ public boolean isChecksumEnabled() { return cfg.get(CHECKSUM_ENABLED); } /** * Returns the checkpoint creation frequency as a probability. * * @return probability in [0.0, 1.0] */ public double getCheckpointFrequency() { return cfg.get(CHECKPOINT_FREQUENCY); } /** * Returns whether a checkpoint should be created for the current commit attempt. * *

The decision is made by sampling a uniform random number in [0.0, 1.0) and comparing it to * {@link #getCheckpointFrequency()}. * * @return {@code true} if a random number is smaller than the configured frequency */ public boolean shouldCreateCheckpoint() { double p = getCheckpointFrequency(); if (p <= 0.0) return false; if (p >= 1.0) return true; return randgen.nextDouble() < p; } private void validate() { double p = cfg.get(CHECKPOINT_FREQUENCY); if (Double.isNaN(p) || p < 0.0 || p > 1.0) { throw new IllegalArgumentException( "Invalid checkpoint-frequency: " + p + " (expected a probability in [0.0, 1.0])"); } } } ================================================ FILE: flink/src/main/java/io/delta/flink/table/TableEventListener.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink.table; import io.delta.kernel.Snapshot; import java.io.Serializable; public interface TableEventListener extends Serializable { default void onPostCommit(AbstractKernelTable source, Snapshot snapshot) {} } ================================================ FILE: flink/src/main/java/io/delta/flink/table/postcommit/ChecksumListener.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink.table.postcommit; import io.delta.flink.table.AbstractKernelTable; import io.delta.flink.table.ExceptionUtils; import io.delta.flink.table.TableEventListener; import io.delta.kernel.Snapshot; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Post-commit listener that generate checksum for the snapshot */ public class ChecksumListener implements TableEventListener { public static final Logger LOG = LoggerFactory.getLogger(ChecksumListener.class); @Override public void onPostCommit(AbstractKernelTable source, Snapshot snapshot) { if (source.getConf().isChecksumEnabled()) { // Write checksum asynchronously source.executeWithTiming( "postcommit.checksum", () -> { snapshot .getStatistics() .getChecksumWriteMode() .ifPresent( ExceptionUtils.wrap(mode -> snapshot.writeChecksum(source.getEngine(), mode))); }); } } } ================================================ FILE: flink/src/main/java/io/delta/flink/table/postcommit/MaintenanceListener.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink.table.postcommit; import dev.failsafe.function.CheckedRunnable; import io.delta.flink.kernel.CheckpointWriter; import io.delta.flink.table.AbstractKernelTable; import io.delta.flink.table.TableEventListener; import io.delta.kernel.Snapshot; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.tablefeatures.TableFeatures; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** This post-commit listener will publish the snapshot, and optionally create checkpoints. */ public class MaintenanceListener implements TableEventListener { @Override public void onPostCommit(AbstractKernelTable source, Snapshot snapshot) { source.executeWithTiming("postcommit.maintenance", new MaintenanceTask(source, snapshot)); } static class MaintenanceTask implements CheckedRunnable { static final Logger LOG = LoggerFactory.getLogger(MaintenanceTask.class); final AbstractKernelTable table; final Snapshot snapshot; public MaintenanceTask(AbstractKernelTable table, Snapshot snapshot) { this.table = table; this.snapshot = snapshot; } public void run() { Engine engine = table.getEngine(); try { // Publish commits Snapshot published = table.withTiming("postcommit.maintenance.publish", () -> snapshot.publish(engine)); // Update cache table.getCacheManager().put(table.getTablePath().toString(), published); // Checkpoint can be done only on published snapshots if (table.getConf().shouldCreateCheckpoint()) { if (snapshot instanceof SnapshotImpl && ((SnapshotImpl) snapshot) .getProtocol() .getWriterFeatures() .contains(TableFeatures.CHECKPOINT_V2_RW_FEATURE.featureName())) { // Use v2 incremental checkpoint when possible table.withTiming( "postcommit.maintenance.checkpoint", () -> new CheckpointWriter(engine, snapshot).write()); } else { table.withTiming( "postcommit.maintenance.checkpoint", () -> snapshot.writeCheckpoint(engine)); } } } catch (Exception e) { LOG.error("Exception while maintenance", e); } } } } ================================================ FILE: flink/src/main/resources/delta-flink.properties ================================================ # # Copyright (2026) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # #sink.retry.max_attempt=4 #sink.retry.delay_ms=200 #sink.retry.max_delay_ms=20000 #table.cache.enable=true #table.cache.size=100 #table.cache.expire_ms=300000 #credentials.refresh.thread_pool_size=10 #credentials.refresh.ahead_ms=60000 ================================================ FILE: flink/src/test/java/io/delta/flink/DummyHttp.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink; import static com.github.tomakehurst.wiremock.client.WireMock.*; import com.github.tomakehurst.wiremock.WireMockServer; import com.github.tomakehurst.wiremock.core.WireMockConfiguration; import java.util.HashMap; import java.util.Map; public class DummyHttp { private final WireMockServer wireMockServer; public DummyHttp(Map returns) { this.wireMockServer = new WireMockServer(WireMockConfiguration.options().dynamicPort()); this.wireMockServer.start(); configureFor("localhost", this.wireMockServer.port()); returns.forEach( (url, content) -> { this.wireMockServer.stubFor( any(urlPathMatching(url)) .willReturn( aResponse() .withStatus(200) .withHeader("Content-Type", "application/json") .withBody(content))); }); } public int port() { return this.wireMockServer.port(); } public static DummyHttp forUC(String tablePath) { Map stubs = new HashMap<>(); stubs.put("/api/2.1/unity-catalog/tables", "{}"); // For write stubs.put( "/api/2.1/unity-catalog/tables/.*", // For read String.format("{\"storage_location\": \"%s\", \"table_id\": \"dummy_id\"}", tablePath)); stubs.put("/api/2.1/unity-catalog/temporary-table-credentials", "{}"); stubs.put( "/api/2.1/unity-catalog/delta/preview/commits", "{\"commits\": [], \"latest_table_version\": 1230}"); return new DummyHttp(stubs); } } ================================================ FILE: flink/src/test/java/io/delta/flink/TestHelper.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink; import dev.failsafe.function.CheckedConsumer; import io.delta.kernel.Operation; import io.delta.kernel.Snapshot; import io.delta.kernel.TableManager; import io.delta.kernel.data.Row; import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.engine.Engine; import io.delta.kernel.expressions.Column; import io.delta.kernel.expressions.Literal; import io.delta.kernel.internal.ScanImpl; import io.delta.kernel.internal.actions.AddFile; import io.delta.kernel.internal.actions.SingleAction; import io.delta.kernel.internal.data.GenericRow; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.statistics.DataFileStatistics; import io.delta.kernel.transaction.DataLayoutSpec; import io.delta.kernel.types.DataType; import io.delta.kernel.types.IntegerType; import io.delta.kernel.types.LongType; import io.delta.kernel.types.StringType; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterable; import io.delta.kernel.utils.DataFileStatus; import io.delta.kernel.utils.FileStatus; import java.io.*; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.StreamSupport; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.shaded.org.apache.commons.io.FileUtils; /** * Abstract base class for JUnit 6 (Jupiter) tests providing common test utilities for Delta Lake * operations. */ public abstract class TestHelper { protected final Random random = new Random(System.currentTimeMillis()); protected static Consumer wrap(CheckedConsumer body) { return t -> { try { body.accept(t); } catch (Throwable e) { throw new RuntimeException(e); } }; } /** * Executes the given function with a temporary directory. The directory is automatically deleted * after the function completes. * * @param f Consumer function that receives the temporary directory */ protected void withTempDir(CheckedConsumer f) { File tempDir = null; try { tempDir = Files.createTempDirectory(UUID.randomUUID().toString()).toFile(); f.accept(tempDir); } catch (Throwable e) { throw new RuntimeException(e); } finally { if (tempDir != null) { FileUtils.deleteQuietly(tempDir); } } } /** * Creates a dummy row with a random ID. * * @return A Row containing a random integer ID */ protected Row dummyRow() { int id = random.nextInt(1048576); Map map = new HashMap<>(); map.put(0, id); StructType schema = new StructType().add("id", IntegerType.INTEGER); return new GenericRow(schema, map); } /** * Creates dummy file statistics with the specified number of records. * * @param numRecords Number of records in the file * @return DataFileStatistics with the given record count */ protected DataFileStatistics dummyStatistics(long numRecords) { return new DataFileStatistics( numRecords, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Optional.empty()); } /** * Creates a dummy AddFile row for testing. * * @param schema Schema of the table * @param numRows Number of rows in the file * @param partitionValues Partition column values * @return Row representing an AddFile action */ protected Row dummyAddFileRow( StructType schema, long numRows, Map partitionValues) { AddFile addFile = AddFile.convertDataFileStatus( schema, URI.create("s3://abc/def"), new DataFileStatus( "s3://abc/def/" + UUID.randomUUID().toString(), 1000L, 2000L, Optional.of(dummyStatistics(numRows))), partitionValues, /* dataChange= */ true, /* tags= */ Collections.emptyMap(), /* baseRowId= */ Optional.empty(), /* defaultRowCommitVersion= */ Optional.empty(), /* deletionVectorDescriptor= */ Optional.empty()); return SingleAction.createAddFileSingleAction(addFile.toRow()); } /** Create Multiple add files rows */ protected List dummyAddFileRows( StructType schema, int count, Function> partgen) { return IntStream.range(0, count) .mapToObj(i -> dummyAddFileRow(schema, 10 + i, partgen.apply(i))) .collect(Collectors.toList()); } /** * Creates a dummy writer context for testing. * * @param engine Delta Engine instance * @param tablePath Path to the table * @param schema Table schema * @param partitionCols Partition column names * @return Transaction state row */ protected Row dummyWriterContext( Engine engine, String tablePath, StructType schema, List partitionCols) { var txnBuilder = TableManager.buildCreateTableTransaction(tablePath, schema, "dummy"); if (!partitionCols.isEmpty()) { List partitionColumns = partitionCols.stream().map(Column::new).collect(Collectors.toList()); txnBuilder.withDataLayoutSpec(DataLayoutSpec.partitioned(partitionColumns)); } var txn = txnBuilder.build(engine); return txn.getTransactionState(engine); } /** Overload with empty partition columns. */ protected Row dummyWriterContext(Engine engine, String tablePath, StructType schema) { return dummyWriterContext(engine, tablePath, schema, Collections.emptyList()); } /** * Creates a non-empty table with dummy data. * * @param engine Delta Engine instance * @param tablePath Path to the table * @param schema Table schema * @param partitionCols Partition column names * @param numRows Number of rows to add * @param properties Table properties * @return Optional snapshot after creation */ protected Optional createNonEmptyTable( Engine engine, String tablePath, StructType schema, List partitionCols, long numRows, Map properties) { var txnBuilder = TableManager.buildCreateTableTransaction(tablePath, schema, "dummy") .withTableProperties(properties); if (!partitionCols.isEmpty()) { List partitionColumns = partitionCols.stream().map(Column::new).collect(Collectors.toList()); txnBuilder.withDataLayoutSpec(DataLayoutSpec.partitioned(partitionColumns)); } var txn = txnBuilder.build(engine); Map partitionMap = new HashMap<>(); for (String colName : partitionCols) { partitionMap.put(colName, dummyRandomLiteral(schema.get(colName).getDataType())); } // Prepare some dummy AddFile AddFile dummyAddFile = AddFile.convertDataFileStatus( schema, URI.create(tablePath), new DataFileStatus( UUID.randomUUID().toString(), 1000L, 2000L, Optional.of(dummyStatistics(numRows))), partitionMap, true, Collections.emptyMap(), Optional.empty(), Optional.empty(), Optional.empty()); return txn.commit( engine, CloseableIterable.inMemoryIterable( Utils.singletonCloseableIterator( SingleAction.createAddFileSingleAction(dummyAddFile.toRow())))) .getPostCommitSnapshot(); } /** Overloads with default values. */ protected Optional createNonEmptyTable( Engine engine, String tablePath, StructType schema) { return createNonEmptyTable( engine, tablePath, schema, Collections.emptyList(), 0L, Collections.emptyMap()); } protected Optional createNonEmptyTable( Engine engine, String tablePath, StructType schema, List partitionCols) { return createNonEmptyTable( engine, tablePath, schema, partitionCols, 0L, Collections.emptyMap()); } protected Optional createNonEmptyTable( Engine engine, String tablePath, StructType schema, List partitionCols, long numRows) { return createNonEmptyTable( engine, tablePath, schema, partitionCols, numRows, Collections.emptyMap()); } protected Optional createNonEmptyTable( Engine engine, String tablePath, StructType schema, List partitionCols, Map properties) { return createNonEmptyTable(engine, tablePath, schema, partitionCols, 0L, properties); } /** * Makes a random write to an existing table. * * @param engine Delta Engine instance * @param tablePath Path to the table * @param schema Table schema * @param partitionCols Partition column names * @return Optional snapshot after write */ protected Optional writeTable( Engine engine, String tablePath, StructType schema, List partitionCols) { Map partitionMap = new HashMap<>(); for (String colName : partitionCols) { partitionMap.put(colName, dummyRandomLiteral(schema.get(colName).getDataType())); } // Prepare some dummy AddFile AddFile dummyAddFile = AddFile.convertDataFileStatus( schema, URI.create(tablePath), new DataFileStatus(UUID.randomUUID().toString(), 1000L, 2000L, Optional.empty()), partitionMap, true, Collections.emptyMap(), Optional.empty(), Optional.empty(), Optional.empty()); var txn = TableManager.loadSnapshot(tablePath) .build(engine) .buildUpdateTableTransaction("dummy", Operation.WRITE) .build(engine); return txn.commit( engine, CloseableIterable.inMemoryIterable( Utils.singletonCloseableIterator( SingleAction.createAddFileSingleAction(dummyAddFile.toRow())))) .getPostCommitSnapshot(); } /** Overload with empty partition columns. */ protected Optional writeTable(Engine engine, String tablePath, StructType schema) { return writeTable(engine, tablePath, schema, Collections.emptyList()); } /** * Verifies table content using a custom checker function. * * @param tablePath Path to the table * @param checker Consumer that receives version, AddFiles, and properties */ protected void verifyTableContent(String tablePath, TableContentChecker checker) { Engine engine = DefaultEngine.create(new Configuration()); Snapshot snapshot = TableManager.loadSnapshot(tablePath).build(engine); var filesList = ((ScanImpl) snapshot.getScanBuilder().build()).getScanFiles(engine, true).toInMemoryList(); List actions = StreamSupport.stream(filesList.spliterator(), false) .flatMap( scanFile -> StreamSupport.stream(scanFile.getRows().toInMemoryList().spliterator(), false)) .map(row -> new AddFile(row.getStruct(0))) .collect(Collectors.toList()); Map properties = snapshot.getTableProperties(); checker.check(snapshot.getVersion(), actions, properties); } /** Functional interface for table content verification. */ @FunctionalInterface protected interface TableContentChecker { void check(long version, Iterable addFiles, Map properties); } /** * Reads a Parquet file and returns its rows. * * @param filePath Path to the Parquet file * @param schema Expected schema * @return List of rows from the file */ protected List readParquet(Path filePath, StructType schema) { try { FileStatus fileStatus = FileStatus.of( filePath.toString(), Files.size(filePath), Files.getLastModifiedTime(filePath).toMillis()); var results = DefaultEngine.create(new Configuration()) .getParquetHandler() .readParquetFiles( Utils.singletonCloseableIterator(fileStatus), schema, Optional.empty()); return StreamSupport.stream(results.toInMemoryList().spliterator(), false) .flatMap( result -> StreamSupport.stream( result.getData().getRows().toInMemoryList().spliterator(), false)) .collect(Collectors.toList()); } catch (IOException e) { throw new UncheckedIOException("Failed to read Parquet file: " + filePath, e); } } /** * Creates a random literal value for the given data type. * * @param dataType Data type for the literal * @return Random literal of the specified type */ protected Literal dummyRandomLiteral(DataType dataType) { if (dataType.equals(IntegerType.INTEGER)) { return Literal.ofInt(random.nextInt()); } else if (dataType.equals(StringType.STRING)) { return Literal.ofString("p" + random.nextInt()); } else if (dataType.equals(LongType.LONG)) { return Literal.ofLong(random.nextLong()); } else { throw new UnsupportedOperationException( "Unsupported data type for random literal: " + dataType); } } /** * Checks if an object is serializable by attempting to serialize and deserialize it. * * @param input Object to check for serializability * @throws AssertionError if the object cannot be serialized or deserialized correctly */ protected void checkSerializability(Object input) { try { ByteArrayOutputStream baos = new ByteArrayOutputStream(); ObjectOutputStream oos = new ObjectOutputStream(baos); oos.writeObject(input); oos.close(); byte[] bytes = baos.toByteArray(); ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes)); Object restored = ois.readObject(); ois.close(); if (!restored.getClass().equals(input.getClass())) { throw new AssertionError( "Restored object class " + restored.getClass() + " does not match input class " + input.getClass()); } } catch (IOException | ClassNotFoundException e) { throw new AssertionError("Object is not serializable", e); } } } ================================================ FILE: flink/src/test/java/io/delta/flink/kernel/CheckpointWriterTest.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink.kernel; import static io.delta.flink.kernel.CheckpointWriter.TAG_SIDECAR_COUNT; import static io.delta.kernel.internal.util.Utils.singletonCloseableIterator; import static org.junit.jupiter.api.Assertions.*; import io.delta.flink.TestHelper; import io.delta.kernel.Operation; import io.delta.kernel.Snapshot; import io.delta.kernel.TableManager; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.engine.Engine; import io.delta.kernel.expressions.Literal; import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.actions.AddFile; import io.delta.kernel.internal.actions.DomainMetadata; import io.delta.kernel.internal.actions.SingleAction; import io.delta.kernel.internal.checkpoints.CheckpointMetaData; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.types.IntegerType; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterable; import io.delta.kernel.utils.DataFileStatus; import io.delta.kernel.utils.FileStatus; import java.net.URI; import java.util.*; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.junit.jupiter.api.Test; /** JUnit test suite for checkpoint creation and validation. */ class CheckpointWriterTest extends TestHelper { /** * Helper method to create a table and write commits with a callback on each commit. * * @param engine Delta engine * @param tablePath path to the table * @param schema table schema * @param partitionCols partition columns * @param numCommits number of commits to write * @param callback callback invoked after each commit with the commit index (0-based) and snapshot */ private void createTableAndWriteCommits( Engine engine, String tablePath, StructType schema, List partitionCols, int numCommits, CommitCallback callback) { Map properties = Map.of("delta.feature.v2Checkpoint", "supported"); createNonEmptyTable(engine, tablePath, schema, partitionCols, properties); for (int i = 0; i < numCommits; i++) { Optional snapshot = writeTable(engine, tablePath, schema, partitionCols); if (snapshot.isPresent() && callback != null) { callback.onCommit(i, snapshot.get()); } } } @FunctionalInterface interface CommitCallback { void onCommit(int commitIndex, Snapshot snapshot); } private Optional writeRemoveFile( Engine engine, String tablePath, StructType schema, List partitionCols) { Map partitionMap = new HashMap<>(); for (String colName : partitionCols) { partitionMap.put(colName, dummyRandomLiteral(schema.get(colName).getDataType())); } // Prepare some dummy AddFile AddFile dummyAddFile = AddFile.convertDataFileStatus( schema, URI.create(tablePath), new DataFileStatus(UUID.randomUUID().toString(), 1000L, 2000L, Optional.empty()), partitionMap, true, Collections.emptyMap(), Optional.empty(), Optional.empty(), Optional.empty()); var txn = TableManager.loadSnapshot(tablePath) .build(engine) .buildUpdateTableTransaction("dummy", Operation.WRITE) .build(engine); txn.commit( engine, CloseableIterable.inMemoryIterable( Utils.singletonCloseableIterator( SingleAction.createAddFileSingleAction(dummyAddFile.toRow())))) .getPostCommitSnapshot(); txn = TableManager.loadSnapshot(tablePath) .build(engine) .buildUpdateTableTransaction("dummy", Operation.WRITE) .build(engine); return txn.commit( engine, CloseableIterable.inMemoryIterable( Utils.singletonCloseableIterator( SingleAction.createRemoveFileSingleAction( dummyAddFile.toRemoveFileRow(true, Optional.empty()))))) .getPostCommitSnapshot(); } private Optional writeDomainMetadata( Engine engine, String tablePath, String domain, String conf) { // Prepare some dummy AddFile DomainMetadata dm = new DomainMetadata(domain, conf, false); var txn = TableManager.loadSnapshot(tablePath) .build(engine) .buildUpdateTableTransaction("dummy", Operation.WRITE) .build(engine); return txn.commit( engine, CloseableIterable.inMemoryIterable( Utils.singletonCloseableIterator( SingleAction.createDomainMetadataSingleAction(dm.toRow())))) .getPostCommitSnapshot(); } private void assertSnapshotRead( Engine engine, String tablePath, long version, int numSidecars, int numActions) { SnapshotImpl latest = (SnapshotImpl) TableManager.loadSnapshot(tablePath).build(engine); assertSnapshotRead(engine, latest, version, numSidecars, numActions, Map.of()); } private void assertSnapshotRead( Engine engine, SnapshotImpl snapshot, long version, int numSidecars, int numActions) { assertSnapshotRead(engine, snapshot, version, numSidecars, numActions, Map.of()); } private void assertSnapshotRead( Engine engine, SnapshotImpl snapshot, long version, int numSidecars, int numActions, Map dms) { if (version >= 0) { assertEquals(version, snapshot.getSnapshotReport().getCheckpointVersion().orElse(-1L)); } var files = snapshot.getScanBuilder().build().getScanFiles(engine).toInMemoryList(); if (numSidecars >= 0) { long sidecarCount = files.stream() .map(FilteredColumnarBatch::getFilePath) .filter(path -> path.orElse("").contains("_sidecar")) .count(); assertEquals(numSidecars, sidecarCount); } if (numActions >= 0) { List actions = files.stream() .flatMap(scanFile -> scanFile.getRows().toInMemoryList().stream()) .map(row -> new AddFile(row.getStruct(0))) .collect(Collectors.toList()); assertEquals(numActions, actions.size()); } for (Map.Entry e : dms.entrySet()) { assertTrue(snapshot.getDomainMetadata(e.getKey()).isPresent()); assertEquals(e.getValue(), snapshot.getDomainMetadata(e.getKey()).get()); } } /** * Reads and verifies the _last_checkpoint file. * * @param engine Delta engine * @param snapshot snapshot to get table path from * @param expectedVersion expected checkpoint version (-1 to skip verification) * @param expectedNumSidecars expected number of sidecars (-1 to skip verification) */ private void assertLastCheckpointFile( Engine engine, Snapshot snapshot, int expectedVersion, int expectedNumSidecars) { String tablePath = snapshot.getPath(); String lastCheckpointPath = tablePath + "/_delta_log/_last_checkpoint"; try { var content = engine .getJsonHandler() .readJsonFiles( singletonCloseableIterator(FileStatus.of(lastCheckpointPath)), CheckpointMetaData.READ_SCHEMA, Optional.empty()) .flatMap(ColumnarBatch::getRows) .toInMemoryList() .get(0); CheckpointMetaData metadata = CheckpointMetaData.fromRow(content); // Verify version if (expectedVersion >= 0) { assertEquals(expectedVersion, metadata.version, "Checkpoint version mismatch"); } // Verify tags contain TAG_FLINK_DELTASINK_CHECKPOINT assertTrue( Boolean.parseBoolean( metadata.tags.getOrDefault(CheckpointWriter.TAG_DELTASINK_CHECKPOINT, "false")), "Expected TAG_DELTASINK_CHECKPOINT to be true in _last_checkpoint"); // Verify number of sidecars if parts are present if (expectedNumSidecars >= 0 && metadata.parts.isPresent()) { assertEquals( expectedNumSidecars, Integer.parseInt(metadata.tags.getOrDefault(TAG_SIDECAR_COUNT, "0")), "Number of sidecars mismatch in _last_checkpoint"); } } catch (Exception e) { throw new AssertionError("Failed to read or verify _last_checkpoint", e); } } @Test void testCreateIncrementalCheckpoint() { withTempDir( dir -> { String tablePath = dir.getAbsolutePath(); Engine engine = DefaultEngine.create(new Configuration()); StructType schema = new StructType().add("id", IntegerType.INTEGER); createTableAndWriteCommits( engine, tablePath, schema, Collections.emptyList(), 25, (i, snapshot) -> { if (i % 7 == 6) { TestHelper.wrap(s -> new CheckpointWriter(engine, s).write()) .accept(snapshot); assertLastCheckpointFile( engine, snapshot, /* version */ i + 1, /* numSidecar */ (i + 1) / 7); } }); assertSnapshotRead(engine, tablePath, 21, 3, 26); }); } @Test void testIgnoreNotMyCheckpoints() { withTempDir( dir -> { String tablePath = dir.getAbsolutePath(); Engine engine = DefaultEngine.create(new Configuration()); StructType schema = new StructType().add("id", IntegerType.INTEGER); createTableAndWriteCommits( engine, tablePath, schema, Collections.emptyList(), 25, (i, snapshot) -> { if (i % 7 == 6) { TestHelper.wrap(s -> s.writeCheckpoint(engine)).accept(snapshot); } if (i == 24) { TestHelper.wrap(s -> new CheckpointWriter(engine, s).write()) .accept(snapshot); } }); // 1 checkpoint, 1 sidecars assertSnapshotRead(engine, tablePath, 25, 1, 26); }); } @Test void testCreateOnOlderSnapshots() { withTempDir( dir -> { String tablePath = dir.getAbsolutePath(); Engine engine = DefaultEngine.create(new Configuration()); StructType schema = new StructType().add("id", IntegerType.INTEGER); createTableAndWriteCommits(engine, tablePath, schema, Collections.emptyList(), 25, null); SnapshotImpl snapshot = (SnapshotImpl) TableManager.loadSnapshot(tablePath).build(engine); snapshot.writeCheckpoint(engine); SnapshotImpl oldSnapshot = (SnapshotImpl) TableManager.loadSnapshot(tablePath).atVersion(20).build(engine); new CheckpointWriter(engine, oldSnapshot).write(); SnapshotImpl oldSnapshotAgain = (SnapshotImpl) TableManager.loadSnapshot(tablePath).atVersion(20).build(engine); assertEquals(20, oldSnapshotAgain.getSnapshotReport().getCheckpointVersion().orElse(-1L)); assertSnapshotRead(engine, oldSnapshotAgain, -1, 1, 21); // Make sure _last_checkpoint is not changed String fileName = tablePath + "/_delta_log/_last_checkpoint"; var content = engine .getJsonHandler() .readJsonFiles( singletonCloseableIterator(FileStatus.of(fileName)), CheckpointMetaData.READ_SCHEMA, Optional.empty()) .toInMemoryList(); List checkpoints = content.stream() .flatMap(columnarBatch -> columnarBatch.getRows().toInMemoryList().stream()) .map(CheckpointMetaData::fromRow) .collect(Collectors.toList()); assertEquals(25, checkpoints.get(0).version); }); } @Test void testFallbackOnRemoveFiles() { withTempDir( dir -> { String tablePath = dir.getAbsolutePath(); Engine engine = DefaultEngine.create(new Configuration()); StructType schema = new StructType().add("id", IntegerType.INTEGER); createTableAndWriteCommits( engine, tablePath, schema, Collections.emptyList(), 25, (i, snapshot) -> { if (i % 7 == 6) { TestHelper.wrap(s -> new CheckpointWriter(engine, s).write()) .accept(snapshot); } }); var s = writeRemoveFile(engine, tablePath, schema, Collections.emptyList()); s.ifPresent(wrap(sn -> new CheckpointWriter(engine, sn).write())); assertSnapshotRead(engine, tablePath, 27, 1, 26); }); } @Test void testMergeManySidecars() { withTempDir( dir -> { String tablePath = dir.getAbsolutePath(); Engine engine = DefaultEngine.create(new Configuration()); StructType schema = new StructType().add("id", IntegerType.INTEGER); // Create multiple checkpoints to accumulate 5 sidecars createTableAndWriteCommits( engine, tablePath, schema, Collections.emptyList(), 40, (i, snapshot) -> { if (i % 7 == 6) { TestHelper.wrap(s -> new CheckpointWriter(engine, s).write()) .accept(snapshot); } }); // Verify we have accumulated the expected sidecars assertSnapshotRead(engine, tablePath, -1L, 5, -1); // Write one more checkpoint - this should merge the 5 old sidecars SnapshotImpl finalSnapshot = (SnapshotImpl) TableManager.loadSnapshot(tablePath).build(engine); new CheckpointWriter(engine, finalSnapshot, 5).write(); // Load the snapshot again to get the latest checkpoint SnapshotImpl snapshotAfter = (SnapshotImpl) TableManager.loadSnapshot(tablePath).build(engine); assertSnapshotRead(engine, snapshotAfter, -1, 2, 41); }); } @Test public void testMergeDomainMetadata() { withTempDir( dir -> { String tablePath = dir.getAbsolutePath(); Engine engine = DefaultEngine.create(new Configuration()); StructType schema = new StructType().add("id", IntegerType.INTEGER); Map properties = Map.of( "delta.feature.v2Checkpoint", "supported", "delta.feature.domainMetadata", "supported"); Optional snapshot = createNonEmptyTable(engine, tablePath, schema, List.of(), properties); writeDomainMetadata(engine, tablePath, "domain1", "conf1"); writeDomainMetadata(engine, tablePath, "domain2", "conf2"); snapshot = writeDomainMetadata(engine, tablePath, "domain1", "conf2"); new CheckpointWriter(engine, snapshot.get()).write(); // Write a new commit then read the table writeTable(engine, tablePath, schema).get(); SnapshotImpl snapshotAfter = (SnapshotImpl) TableManager.loadSnapshot(tablePath).build(engine); assertSnapshotRead( engine, snapshotAfter, 3, 1, 2, Map.of("domain1", "conf2", "domain2", "conf2")); }); } } ================================================ FILE: flink/src/test/java/io/delta/flink/table/AbstractKernelTableTest.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink.table; import static io.delta.kernel.internal.util.Utils.singletonCloseableIterator; import static org.junit.jupiter.api.Assertions.*; import dev.failsafe.function.CheckedConsumer; import io.delta.flink.TestHelper; import io.delta.kernel.Snapshot; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.defaults.internal.data.DefaultColumnarBatch; import io.delta.kernel.engine.Engine; import io.delta.kernel.expressions.Literal; import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.actions.AddFile; import io.delta.kernel.internal.actions.SingleAction; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.types.IntegerType; import io.delta.kernel.types.StringType; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterable; import io.delta.kernel.utils.CloseableIterator; import java.io.File; import java.net.URI; import java.nio.file.AccessDeniedException; import java.util.*; import java.util.stream.Collectors; import org.apache.flink.util.InstantiationUtil; import org.apache.hadoop.conf.Configuration; import org.junit.jupiter.api.Test; /** JUnit 6 test suite for AbstractKernelTable. */ class AbstractKernelTableTest extends TestHelper { /** * Helper method to create a test table with default configuration. * * @param schema table schema * @param partitionCols partition columns * @param callback callback invoked with the created table */ private void withTestTable( StructType schema, List partitionCols, CheckedConsumer callback) { withTestTable(schema, partitionCols, Collections.emptyMap(), callback); } /** * Helper method to create a test table with custom configuration. * * @param schema table schema * @param partitionCols partition columns * @param tableConfig table configuration * @param callback callback invoked with the created table */ private void withTestTable( StructType schema, List partitionCols, Map tableConfig, CheckedConsumer callback) { withTempDir( dir -> { LocalFileSystemTable table = new LocalFileSystemTable(dir.toURI(), tableConfig, schema, partitionCols); table.open(); callback.accept(table); }); } @Test void testNormalizeURI() { assertEquals( "file:///var/char/good/", AbstractKernelTable.normalize(URI.create("file:/var/char/good")).toString()); assertEquals( "file:///var/char/good/", AbstractKernelTable.normalize(URI.create("/var/char/good")).toString()); assertEquals( "file:///var/char/good/", AbstractKernelTable.normalize(URI.create("file:///var/char/good")).toString()); assertEquals( "s3://host/var/", AbstractKernelTable.normalize(URI.create("s3://host/var")).toString()); } @Test void testTableIsSerializable() throws Exception { StructType schema = new StructType().add("id", IntegerType.INTEGER); withTestTable( schema, Collections.emptyList(), table -> { byte[] serialized = InstantiationUtil.serializeObject(table); AbstractKernelTable copy = InstantiationUtil.deserializeObject(serialized, getClass().getClassLoader()); assertNotNull(copy); assertNull(copy.getSchema()); assertNull(copy.tableState); assertEquals(copy.getId(), table.getId()); assertEquals(copy.getTablePath(), table.getTablePath()); assertEquals(copy.getTableUUID(), table.getTableUUID()); assertEquals(copy.getPartitionColumns(), table.getPartitionColumns()); }); } @Test void testTableStoredConfIntoDeltaLogs() { StructType schema = new StructType().add("id", IntegerType.INTEGER); Map tableConfig = new HashMap<>(); tableConfig.put("delta.enableDeletionVectors", "true"); tableConfig.put("showme", "themoney"); tableConfig.put("something", "fornothing"); withTestTable( schema, Collections.emptyList(), tableConfig, table -> { table.commit( CloseableIterable.inMemoryIterable( singletonCloseableIterator(dummyAddFileRow(schema, 1, Collections.emptyMap()))), "app", 100L, Collections.emptyMap()); Snapshot snapshot = table.snapshot().get(); assertEquals("true", snapshot.getTableProperties().get("delta.enableDeletionVectors")); assertFalse(snapshot.getTableProperties().containsKey("showme")); assertFalse(snapshot.getTableProperties().containsKey("something")); assertTrue( ((SnapshotImpl) snapshot).getProtocol().getWriterFeatures().contains("v2Checkpoint")); }); } @Test void testCreateTableAndCommitWithoutPartition() { StructType schema = new StructType().add("id", IntegerType.INTEGER).add("part", StringType.STRING); withTestTable( schema, Collections.emptyList(), table -> { List actions = dummyAddFileRows(schema, 5, (i) -> Map.of("part", Literal.ofString("p" + i))); CloseableIterable dataActions = new CloseableIterable() { @Override public CloseableIterator iterator() { return Utils.toCloseableIterator(actions.iterator()); } @Override public void close() { // Nothing to close } }; table.commit(dataActions, "a", 100, Collections.emptyMap()); // The target table should have one version verifyTableContent( table.getTablePath().toString(), (version, addFiles, properties) -> { assertEquals(1L, version); // There should be 5 files to scan List actionsList = new ArrayList<>(); addFiles.forEach(actionsList::add); assertEquals(5, actionsList.size()); long sum = actionsList.stream().mapToLong(af -> af.getNumRecords().get()).sum(); assertEquals(60, sum); }); }); } @Test void testCreateNewTableAndCommitWithPartition() { withTempDir( dir -> { String tablePath = dir.getAbsolutePath(); StructType schema = new StructType().add("id", IntegerType.INTEGER).add("part", StringType.STRING); LocalFileSystemTable table = new LocalFileSystemTable( URI.create(tablePath), Collections.emptyMap(), schema, List.of("part")); table.open(); List actions = dummyAddFileRows(schema, 5, (i) -> Map.of("part", Literal.ofString("p" + i))); CloseableIterable dataActions = new CloseableIterable() { @Override public CloseableIterator iterator() { return Utils.toCloseableIterator(actions.iterator()); } @Override public void close() { // Nothing to close } }; table.commit(dataActions, "a", 100, Collections.emptyMap()); // The target table should have one version verifyTableContent( dir.toString(), (version, addFiles, properties) -> { assertEquals(1L, version); // There should be 5 files to scan List actionsList = new ArrayList<>(); addFiles.forEach(actionsList::add); assertEquals(5, actionsList.size()); Set partitionValues = actionsList.stream() .map(af -> af.getPartitionValues().getValues().getString(0)) .collect(Collectors.toSet()); assertEquals(Set.of("p0", "p1", "p2", "p3", "p4"), partitionValues); long sum = actionsList.stream().mapToLong(af -> af.getNumRecords().get()).sum(); assertEquals(60, sum); }); }); } @Test void testCommitToExistingTableWithoutPartition() { withTempDir( dir -> { String tablePath = dir.getAbsolutePath(); StructType schema = new StructType().add("id", IntegerType.INTEGER).add("part", StringType.STRING); createNonEmptyTable( DefaultEngine.create(new Configuration()), tablePath, schema, Collections.emptyList(), 30); LocalFileSystemTable table = new LocalFileSystemTable( URI.create(tablePath), Collections.emptyMap(), schema, Collections.emptyList()); table.open(); List actions = dummyAddFileRows(schema, 5, (i) -> Map.of()); CloseableIterable dataActions = CloseableIterable.inMemoryIterable(Utils.toCloseableIterator(actions.iterator())); table.commit(dataActions, "a", 100, Collections.emptyMap()); // The target table should have one version verifyTableContent( dir.toString(), (version, addFiles, properties) -> { assertEquals(1L, version); // There should be 6 files to scan List actionsList = new ArrayList<>(); addFiles.forEach(actionsList::add); assertEquals(6, actionsList.size()); long sum = actionsList.stream().mapToLong(af -> af.getNumRecords().get()).sum(); assertEquals(90, sum); }); }); } @Test void testCommitToExistingTable() { withTempDir( dir -> { String tablePath = dir.getAbsolutePath(); StructType schema = new StructType().add("id", IntegerType.INTEGER).add("part", StringType.STRING); createNonEmptyTable( DefaultEngine.create(new Configuration()), tablePath, schema, List.of("part"), 30); LocalFileSystemTable table = new LocalFileSystemTable( URI.create(tablePath), Collections.emptyMap(), schema, List.of("part")); table.open(); List actions = dummyAddFileRows(schema, 5, (i) -> Map.of("part", Literal.ofString("p" + i))); CloseableIterable dataActions = CloseableIterable.inMemoryIterable(Utils.toCloseableIterator(actions.iterator())); table.commit(dataActions, "a", 100, Collections.emptyMap()); // The target table should have one version verifyTableContent( dir.toString(), (version, addFiles, properties) -> { assertEquals(1L, version); // There should be 6 files to scan List actionsList = new ArrayList<>(); addFiles.forEach(actionsList::add); assertEquals(6, actionsList.size()); Set partitionValues = actionsList.stream() .map(af -> af.getPartitionValues().getValues().getString(0)) .collect(Collectors.toSet()); assertTrue(partitionValues.containsAll(Set.of("p0", "p1", "p2", "p3", "p4"))); long sum = actionsList.stream().mapToLong(af -> af.getNumRecords().get()).sum(); assertEquals(90, sum); }); }); } @Test void testRefreshOnEmptyTable() { withTempDir( dir -> { String tablePath = dir.getAbsolutePath(); StructType schema = new StructType().add("id", IntegerType.INTEGER).add("part", StringType.STRING); LocalFileSystemTable table = new LocalFileSystemTable( URI.create(tablePath), Collections.emptyMap(), schema, List.of("part")); table.open(); table.refresh(); assertTrue(table.snapshot().isPresent()); Snapshot snapshot = table.snapshot().get(); assertEquals(0, snapshot.getVersion()); }); } @Test void testRefreshOnExistingTable() { withTempDir( dir -> { String tablePath = dir.getAbsolutePath(); StructType schema = new StructType().add("id", IntegerType.INTEGER).add("part", StringType.STRING); createNonEmptyTable( DefaultEngine.create(new Configuration()), tablePath, schema, List.of("part"), 30); LocalFileSystemTable table = new LocalFileSystemTable( URI.create(tablePath), Collections.emptyMap(), schema, List.of("part")); table.open(); table.refresh(); assertEquals(0, table.snapshot().get().getVersion()); }); } @Test void testCloseCancelOngoingOperations() { withTempDir( dir -> { Engine engine = DefaultEngine.create(new Configuration()); StructType schema = new StructType().add("id", IntegerType.INTEGER).add("name", StringType.STRING); createNonEmptyTable(engine, dir.getAbsolutePath(), schema); int[] callCounter = {0}; LocalFileSystemTable table = new LocalFileSystemTable( dir.toURI(), Collections.emptyMap(), schema, Collections.emptyList()) { @Override protected Snapshot loadLatestSnapshot() { Snapshot snapshot = super.loadLatestSnapshot(); callCounter[0]++; if (callCounter[0] >= 2) { for (int i = 0; i < 50; i++) { try { Thread.sleep(100); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; } } } return snapshot; } }; table.open(); // With cache, load will not be called again table.setCacheManager(new SnapshotCacheManager.NoCacheManager()); // this thread will refresh the table Thread thread1 = new Thread( () -> { table.refresh(); }); thread1.start(); // If we do not call close, the refresh will take ~5s to stop long wcstart = System.currentTimeMillis(); while (thread1.isAlive()) { try { Thread.sleep(100); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; } } long elapse = System.currentTimeMillis() - wcstart; assertTrue(elapse >= 4500); // this thread will refresh the table Thread thread2 = new Thread( () -> { try { table.refresh(); } catch (Exception e) { // Ignore the InterruptException } }); thread2.start(); // If we call close, the refresh was interrupted quickly try { Thread.sleep(100); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } wcstart = System.currentTimeMillis(); table.close(); while (thread2.isAlive()) { try { Thread.sleep(100); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; } } elapse = System.currentTimeMillis() - wcstart; assertTrue(elapse < 200); }); } @Test void testRetryConcurrencyException() { withTempDir( dir -> { Engine engine = DefaultEngine.create(new Configuration()); StructType schema = new StructType().add("id", IntegerType.INTEGER).add("name", StringType.STRING); createNonEmptyTable(engine, dir.getAbsolutePath(), schema); int[] retryCounter = {0}; int[] loadCounter = {0}; LocalFileSystemTable testHadoopTable = new LocalFileSystemTable( dir.toURI(), Collections.emptyMap(), schema, Collections.emptyList()) { @Override protected Snapshot loadLatestSnapshot() { loadCounter[0]++; Snapshot result = super.loadLatestSnapshot(); if (loadCounter[0] == 2) { throw new ConcurrentModificationException(); } return result; } @Override public void reloadSnapshot() { // This should be called once retryCounter[0]++; } }; testHadoopTable.open(); // Disable cache for retry to work testHadoopTable.setCacheManager(new SnapshotCacheManager.NoCacheManager()); testHadoopTable.commit( CloseableIterable.inMemoryIterable( Utils.singletonCloseableIterator(dummyAddFileRow(schema, 4, Map.of()))), "a", 1000L, Collections.emptyMap()); assertEquals(1, retryCounter[0]); assertEquals(3, loadCounter[0]); }); } @Test void testRetryCredentialExceptionToSucceed() { withTempDir( dir -> { Engine engine = DefaultEngine.create(new Configuration()); StructType schema = new StructType().add("id", IntegerType.INTEGER).add("name", StringType.STRING); createNonEmptyTable(engine, dir.getAbsolutePath(), schema); int[] retryCounter = {0}; int[] loadCounter = {0}; LocalFileSystemTable testHadoopTable = new LocalFileSystemTable( dir.toURI(), Collections.emptyMap(), schema, Collections.emptyList()) { @Override protected Snapshot loadLatestSnapshot() { loadCounter[0]++; Snapshot result = super.loadLatestSnapshot(); if (loadCounter[0] == 2) { throw new RuntimeException(new AccessDeniedException("")); } return result; } @Override public void refreshCredential() { // This should be called once retryCounter[0]++; } }; testHadoopTable.open(); testHadoopTable.setCacheManager(new SnapshotCacheManager.NoCacheManager()); testHadoopTable.refresh(); assertEquals(1, retryCounter[0]); }); } @Test void testRetryCredentialExceptionToExceedMaxAttempts() { withTempDir( dir -> { Engine engine = DefaultEngine.create(new Configuration()); StructType schema = new StructType().add("id", IntegerType.INTEGER).add("name", StringType.STRING); createNonEmptyTable(engine, dir.getAbsolutePath(), schema); int[] retryCounter = {0}; int[] loadCounter = {0}; LocalFileSystemTable testHadoopTable = new LocalFileSystemTable( dir.toURI(), Collections.emptyMap(), schema, Collections.emptyList()) { @Override protected Snapshot loadLatestSnapshot() { loadCounter[0]++; Snapshot result = super.loadLatestSnapshot(); if (loadCounter[0] >= 2) { throw new RuntimeException(new AccessDeniedException("")); } return result; } @Override public void refreshCredential() { // This should be called three times retryCounter[0]++; } }; testHadoopTable.open(); // Disable cache for retry to work testHadoopTable.setCacheManager(new SnapshotCacheManager.NoCacheManager()); Exception e = assertThrows(Exception.class, () -> testHadoopTable.refresh()); assertTrue( ExceptionUtils.recursiveCheck(ex -> ex instanceof AccessDeniedException).test(e)); assertEquals(3, retryCounter[0]); }); } @Test void testWriteResultHasProperStats() { StructType schema = new StructType().add("id", IntegerType.INTEGER).add("name", StringType.STRING); withTestTable( schema, Collections.emptyList(), table -> { int numColumns = 2; ColumnVector[] columnVectors = new ColumnVector[numColumns]; List> dataBuffer = List.of(List.of(1, "Jack"), List.of(2, "Amy")); for (int colIdx = 0; colIdx < numColumns; colIdx++) { var colDataType = schema.at(colIdx).getDataType(); columnVectors[colIdx] = new DataColumnVectorView(dataBuffer, colIdx, colDataType); } CloseableIterator data = Utils.singletonCloseableIterator( new FilteredColumnarBatch( new DefaultColumnarBatch(dataBuffer.size(), schema, columnVectors), Optional.empty())); CloseableIterator result = table.writeParquet("", data, Collections.emptyMap()); result.toInMemoryList().stream() .map(r -> new AddFile(r.getStruct(SingleAction.ADD_FILE_ORDINAL))) .forEach( file -> { assertFalse(file.getStatsJson().isEmpty()); assertEquals( Optional.of( "{\"numRecords\":2,\"minValues\":{\"id\":1,\"name\":\"Amy\"}," + "\"maxValues\":{\"id\":2,\"name\":\"Jack\"},\"nullCount\":{\"id\":0,\"name\":0}}"), file.getStatsJson()); }); }); } @Test public void testGenerateChecksum() { withTempDir( dir -> { StructType schema = new StructType().add("id", IntegerType.INTEGER).add("name", StringType.STRING); LocalFileSystemTable table = new LocalFileSystemTable( dir.toURI(), Collections.emptyMap(), schema, Collections.emptyList()); table.open(); for (int i = 0; i < 10; i++) { table.commit( CloseableIterable.inMemoryIterable( Utils.singletonCloseableIterator(dummyAddFileRow(schema, 10, Map.of()))), "a", 1000L + i, Collections.emptyMap()); String checksumPath = String.format("%s/_delta_log/%020d.crc", dir.getAbsolutePath(), i); File checksumFile = new File(checksumPath); // Async creation, wait for file to appear for (int j = 0; j < 100; j++) { if (checksumFile.exists()) { break; } Thread.sleep(100); } assertTrue(checksumFile.exists(), checksumPath); } }); } @Test public void testCheckpoint() { withTempDir( dir -> { StructType schema = new StructType().add("id", IntegerType.INTEGER).add("name", StringType.STRING); LocalFileSystemTable table = new LocalFileSystemTable( dir.toURI(), Map.of(TableConf.CHECKPOINT_FREQUENCY.key(), "1.0"), schema, Collections.emptyList()); table.open(); for (int i = 0; i < 10; i++) { table.commit( CloseableIterable.inMemoryIterable( Utils.singletonCloseableIterator(dummyAddFileRow(schema, 10, Map.of()))), "a", 1000L + i, Collections.emptyMap()); String checkpointPath = String.format("%s/_delta_log/%020d.checkpoint.parquet", dir.getAbsolutePath(), i); File checkpointFile = new File(checkpointPath); // Async creation, wait for file to appear for (int j = 0; j < 100; j++) { if (checkpointFile.exists()) { break; } Thread.sleep(100); } assertTrue(checkpointFile.exists(), checkpointPath); // Ensure cache is updated var cachedSnapshot = table.snapshot().get(); assertEquals(i + 1, cachedSnapshot.getVersion()); } }); } } ================================================ FILE: flink/src/test/java/io/delta/flink/table/CredentialManagerTest.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink.table; import static org.junit.jupiter.api.Assertions.assertEquals; import io.delta.flink.Conf; import java.util.HashMap; import java.util.Map; import java.util.function.Supplier; import org.junit.jupiter.api.Test; /** JUnit test suite for CredentialManager. */ class CredentialManagerTest { @Test void testGetAndAutoRefreshCredentials() throws InterruptedException { Supplier> supplier = new Supplier>() { private int callCount = 0; @Override public Map get() { long currentTime = System.currentTimeMillis(); long refreshInterval = Conf.getInstance().getCredentialsRefreshAheadInMs(); Map result = new HashMap<>(); result.put("authKey", "authValue" + callCount); // Refresh after around 100 ms result.put( CredentialManager.CREDENTIAL_EXPIRATION_KEY, String.valueOf(currentTime + refreshInterval + 100)); callCount++; return result; } }; CredentialManager manager = new CredentialManager(supplier, () -> {}); // Initial values Map initialResult = manager.getCredentials(); Thread.sleep(150); // Refreshed values Map refreshedResult = manager.getCredentials(); assertEquals("authValue0", initialResult.get("authKey")); assertEquals("authValue1", refreshedResult.get("authKey")); } } ================================================ FILE: flink/src/test/java/io/delta/flink/table/DataColumnVectorView.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink.table; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.types.*; import java.math.BigDecimal; import java.util.List; /** * A wrapper to provide a ColumnVector view backed by a nested list. Usage example: * val dataBuffer = java.util.List.of( * java.util.List.of(1, "Jack"), * java.util.List.of(2, "Amy")) * val cv1 = new DataColumnVectorView(dataBuffer, 0, IntegerType.INTEGER) * val cv2 = new DataColumnVectorView(dataBuffer, 1, StringType.STRING) * */ public class DataColumnVectorView implements ColumnVector { private final List> rows; private final int colIdx; private final DataType dataType; public DataColumnVectorView(List> rows, int colIdx, DataType dataType) { this.rows = rows; this.colIdx = colIdx; this.dataType = dataType; } @Override public DataType getDataType() { return this.dataType; } @Override public int getSize() { return this.rows.size(); } @Override public void close() {} @Override public boolean isNullAt(int rowId) { checkValidRowId(rowId); return rows.get(rowId).get(colIdx) == null; } protected void checkValidRowId(int rowId) { if (rowId < 0 || rowId >= getSize()) { throw new IllegalArgumentException("RowId out of range: " + rowId + " <-> " + getSize()); } } protected void checkValidDataType(io.delta.kernel.types.DataType dataType) { if (!this.getDataType().equivalent(dataType)) { throw new UnsupportedOperationException("Invalid value request for data type"); } } @Override public int getInt(int rowId) { checkValidRowId(rowId); checkValidDataType(IntegerType.INTEGER); return (Integer) rows.get(rowId).get(colIdx); } @Override public long getLong(int rowId) { checkValidRowId(rowId); checkValidDataType(LongType.LONG); return (Long) rows.get(rowId).get(colIdx); } @Override public String getString(int rowId) { checkValidRowId(rowId); checkValidDataType(StringType.STRING); return rows.get(rowId).get(colIdx).toString(); } @Override public float getFloat(int rowId) { checkValidRowId(rowId); checkValidDataType(FloatType.FLOAT); return (Float) rows.get(rowId).get(colIdx); } @Override public double getDouble(int rowId) { checkValidRowId(rowId); checkValidDataType(DoubleType.DOUBLE); return (Double) rows.get(rowId).get(colIdx); } @Override public BigDecimal getDecimal(int rowId) { checkValidRowId(rowId); // Do not check precision and scale here because RowData support conversion if (!(this.getDataType() instanceof DecimalType)) { throw new UnsupportedOperationException("Invalid value request for data type"); } DecimalType actualType = (DecimalType) dataType; return (BigDecimal) rows.get(rowId).get(colIdx); } } ================================================ FILE: flink/src/test/java/io/delta/flink/table/LocalFileSystemCatalog.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink.table; import io.delta.kernel.types.StructType; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Map; import java.util.function.Consumer; public class LocalFileSystemCatalog implements DeltaCatalog { private final Map configurations; public LocalFileSystemCatalog(Map conf) { this.configurations = conf; } @Override public TableDescriptor getTable(String tableId) { URI tablePath = AbstractKernelTable.normalize(URI.create(tableId)); if (!Files.exists(Path.of(tablePath.resolve("_delta_log")))) { throw new ExceptionUtils.ResourceNotFoundException(""); } TableDescriptor info = new TableDescriptor(); info.tableId = tableId; info.tablePath = tablePath; info.uuid = tableId; return info; } @Override public void createTable( String tableId, StructType schema, List partitions, Map properties, Consumer callback) { TableDescriptor desc = new TableDescriptor(tableId, tableId, URI.create(tableId)); callback.accept(desc); } @Override public Map getCredentials(String uuid) { return configurations; } } ================================================ FILE: flink/src/test/java/io/delta/flink/table/LocalFileSystemTable.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.flink.table; import io.delta.kernel.Snapshot; import io.delta.kernel.TableManager; import io.delta.kernel.types.StructType; import java.net.URI; import java.util.List; import java.util.Map; public class LocalFileSystemTable extends AbstractKernelTable { public LocalFileSystemTable( URI tablePath, Map conf, StructType schema, List partitionColumns) { super(new LocalFileSystemCatalog(conf), tablePath.toString(), conf, schema, partitionColumns); } @Override protected Snapshot loadLatestSnapshot() { return TableManager.loadSnapshot(getTablePath().toString()).build(getEngine()); } } ================================================ FILE: flink/src/test/resources/log4j2-test.properties ================================================ # # Copyright (2026) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Console appender appender.console.type = Console appender.console.name = console appender.console.target = SYSTEM_OUT appender.console.layout.type = PatternLayout appender.console.layout.pattern = %d{HH:mm:ss} %-5level %c{1} - %msg%n # Root logger rootLogger.level = warn rootLogger.appenderRefs = console rootLogger.appenderRef.console.ref = console # Package-specific log levels logger.delta.name = io.delta.kernel logger.delta.level = warn logger.flink.name = org.apache.flink logger.flink.level = warn logger.kafka.name = org.apache.kafka logger.kafka.level = warn logger.hadoop.name = org.apache.hadoop logger.hadoop.level = warn ================================================ FILE: hudi/README.md ================================================ # Converting to Hudi with UniForm ## Create a table with Hudi UniForm enabled Using spark-sql you can create a table and insert a few records into it. You will need to include the delta-hudi-assembly jar on the path. ``` spark-sql --packages io.delta:delta-spark_2.12:3.2.0-SNAPSHOT --jars delta-hudi-assembly_2.12-3.2.0-SNAPSHOT.jar --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" ``` Then you can create a table with Hudi UniForm enabled. ``` CREATE TABLE `delta_table_with_hudi` (col1 INT) USING DELTA TBLPROPERTIES('delta.universalFormat.enabledFormats' = 'hudi') LOCATION '/tmp/delta-table-with-hudi'; ``` And insert a record into it. ``` INSERT INTO delta_table_with_hudi VALUES (1); ``` ## Read the table with Hudi Hudi does not currently support spark 3.5.X so you will need to launch a spark shell with spark 3.4.X or earlier. Instructions for launching the spark-shell with Hudi can be found [here](https://hudi.apache.org/docs/quick-start-guide#spark-shellsql). After launching the shell, you can read the table by enabling the hudi metadata table in the reader and loading from the path used in the create table step. ```scala val df = spark.read.format("hudi").option("hoodie.metadata.enable", "true").load("/tmp/delta-table-with-hudi") ``` ================================================ FILE: hudi/integration_tests/write_uniform_hudi.py ================================================ from pyspark.sql import SparkSession from pyspark.sql.functions import current_date, current_timestamp from pyspark.testing import assertDataFrameEqual from delta.tables import DeltaTable import shutil import random import os import time ###################### Setup ###################### test_root = "/tmp/delta-uniform-hudi/" warehouse_path = test_root + "uniform_tables" shutil.rmtree(test_root, ignore_errors=True) hudi_table_base_name = "delta_table_with_hudi" # we need to set the following configs spark_delta = SparkSession.builder \ .appName("delta-uniform-hudi-writer") \ .master("local[*]") \ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \ .config("spark.sql.warehouse.dir", warehouse_path) \ .getOrCreate() ###################### Helper functions ###################### def get_delta_df(spark, table_name): hudi_table_path = os.path.join(warehouse_path, table_name) print('hudi_table_path:', hudi_table_path) df_delta = spark.read.format("delta").load(hudi_table_path) return df_delta def get_hudi_df(spark, table_name): hudi_table_path = os.path.join(warehouse_path, table_name) df_hudi = (spark.read.format("hudi") .option("hoodie.metadata.enable", "true") .option("hoodie.datasource.write.hive_style_partitioning", "true") .load(hudi_table_path)) return df_hudi ###################### Create tables in Delta ###################### print('Delta tables:') # validate various data types spark_delta.sql(f"""CREATE TABLE `{hudi_table_base_name}_0` (col1 BIGINT, col2 BOOLEAN, col3 DATE, col4 DOUBLE, col5 FLOAT, col6 INT, col7 STRING, col8 TIMESTAMP, col9 BINARY, col10 DECIMAL(5, 2), col11 STRUCT>, col12 ARRAY>, col13 MAP>) USING DELTA TBLPROPERTIES('delta.universalFormat.enabledFormats' = 'hudi') """) spark_delta.sql(f"""INSERT INTO `{hudi_table_base_name}_0` VALUES (123, true, date(current_timestamp()), 32.1, 1.23, 456, 'hello world', current_timestamp(), X'1ABF', -999.99, STRUCT(1, 'hello', STRUCT(2, 3, 'world')), ARRAY( STRUCT(1, 'first'), STRUCT(2, 'second') ), MAP( 'key1', STRUCT(1, 'delta'), 'key2', STRUCT(1, 'lake') )); """) df_delta_0 = get_delta_df(spark_delta, f"{hudi_table_base_name}_0") df_delta_0.show() # conversion happens correctly when enabling property after table creation spark_delta.sql(f"CREATE TABLE {hudi_table_base_name}_1 (col1 INT, col2 STRING) USING DELTA") spark_delta.sql(f"INSERT INTO {hudi_table_base_name}_1 VALUES (1, 'a'), (2, 'b')") spark_delta.sql(f"ALTER TABLE {hudi_table_base_name}_1 SET TBLPROPERTIES('delta.universalFormat.enabledFormats' = 'hudi')") df_delta_1 = get_delta_df(spark_delta, f"{hudi_table_base_name}_1") df_delta_1.show() # validate deletes spark_delta.sql(f"""CREATE TABLE {hudi_table_base_name}_2 (col1 INT, col2 STRING) USING DELTA TBLPROPERTIES('delta.universalFormat.enabledFormats' = 'hudi')""") spark_delta.sql(f"INSERT INTO {hudi_table_base_name}_2 VALUES (1, 'a'), (2, 'b')") spark_delta.sql(f"DELETE FROM {hudi_table_base_name}_2 WHERE col1 = 1") df_delta_2 = get_delta_df(spark_delta, f"{hudi_table_base_name}_2") df_delta_2.show() # basic schema evolution spark_delta.sql(f"""CREATE TABLE {hudi_table_base_name}_3 (col1 INT, col2 STRING) USING DELTA TBLPROPERTIES('delta.universalFormat.enabledFormats' = 'hudi')""") spark_delta.sql(f"INSERT INTO {hudi_table_base_name}_3 VALUES (1, 'a'), (2, 'b')") spark_delta.sql(f"ALTER TABLE {hudi_table_base_name}_3 ADD COLUMN col3 INT FIRST") spark_delta.sql(f"INSERT INTO {hudi_table_base_name}_3 VALUES (3, 4, 'c')") df_delta_3 = get_delta_df(spark_delta, f"{hudi_table_base_name}_3") df_delta_3.show() # schema evolution for nested fields spark_delta.sql(f"""CREATE TABLE {hudi_table_base_name}_4 (col1 STRUCT) USING DELTA TBLPROPERTIES('delta.universalFormat.enabledFormats' = 'hudi')""") spark_delta.sql(f"""INSERT INTO {hudi_table_base_name}_4 VALUES (named_struct('field1', 1, 'field2', 'hello')) """) spark_delta.sql(f"ALTER TABLE {hudi_table_base_name}_4 ADD COLUMN col1.field3 INT AFTER field1") spark_delta.sql(f"INSERT INTO {hudi_table_base_name}_4 VALUES (named_struct('field1', 3, 'field3', 4, 'field2', 'delta'))") df_delta_4 = get_delta_df(spark_delta, f"{hudi_table_base_name}_4") df_delta_4.show() # time travel spark_delta.sql(f"""CREATE TABLE {hudi_table_base_name}_5 (col1 INT, col2 STRING) USING DELTA TBLPROPERTIES('delta.universalFormat.enabledFormats' = 'hudi', 'delta.columnMapping.mode' = 'name')""") spark_delta.sql(f"INSERT INTO {hudi_table_base_name}_5 VALUES (1, 'a')") spark_delta.sql(f"INSERT INTO {hudi_table_base_name}_5 VALUES (2, 'b')") df_history_5 = spark_delta.sql(f"DESCRIBE HISTORY {hudi_table_base_name}_5") timestamp = df_history_5.collect()[0]['timestamp'] # get the timestamp of the first commit df_delta_5 = spark_delta.sql(f""" SELECT * FROM {hudi_table_base_name}_5 TIMESTAMP AS OF '{timestamp}'""") df_delta_5.show() time.sleep(5) ###################### Read tables from Hudi engine ###################### print('Hudi tables:') spark_hudi = SparkSession.builder \ .appName("delta-uniform-hudi-reader") \ .master("local[*]") \ .config("spark.sql.extensions", "org.apache.spark.sql.hudi.HoodieSparkSessionExtension") \ .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.hudi.catalog.HoodieCatalog") \ .config("spark.sql.warehouse.dir", warehouse_path) \ .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") \ .config("spark.kryo.registrator", "org.apache.spark.HoodieSparkKryoRegistrar") \ .getOrCreate() df_hudi_0 = get_hudi_df(spark_hudi, f"{hudi_table_base_name}_0") df_hudi_0.show() assertDataFrameEqual(df_delta_0, df_hudi_0) df_hudi_1 = get_hudi_df(spark_hudi, f"{hudi_table_base_name}_1") df_hudi_1.show() assertDataFrameEqual(df_delta_1, df_hudi_1) df_hudi_2 = get_hudi_df(spark_hudi, f"{hudi_table_base_name}_2") df_hudi_2.show() assertDataFrameEqual(df_delta_2, df_hudi_2) df_hudi_3 = get_hudi_df(spark_hudi, f"{hudi_table_base_name}_3") df_hudi_3.show() assertDataFrameEqual(df_delta_3, df_hudi_3) df_hudi_4 = get_hudi_df(spark_hudi, f"{hudi_table_base_name}_4") df_hudi_4.show() assertDataFrameEqual(df_delta_4, df_hudi_4) df_hudi_5 = spark_hudi.sql(f""" SELECT * FROM {hudi_table_base_name}_5 TIMESTAMP AS OF '{timestamp}'""") df_hudi_5.show() assertDataFrameEqual(df_delta_5, df_hudi_5) print('UniForm Hudi integration test passed!') ================================================ FILE: hudi/src/main/scala/org/apache/spark/sql/delta/hudi/HudiConversionTransaction.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.hudi import java.io.{IOException, UncheckedIOException} import java.time.{Instant, LocalDateTime, ZoneId} import java.time.format.{DateTimeFormatterBuilder, DateTimeParseException} import java.time.temporal.{ChronoField, ChronoUnit} import java.util import java.util.{Collections, Properties} import java.util.stream.Collectors import scala.collection.JavaConverters._ import scala.collection.mutable._ import scala.util.control.NonFatal import org.apache.spark.sql.delta.Snapshot import org.apache.spark.sql.delta.actions.Action import org.apache.spark.sql.delta.hudi.HudiSchemaUtils._ import org.apache.spark.sql.delta.hudi.HudiTransactionUtils._ import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.avro.Schema import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.hadoop.conf.Configuration import org.apache.hudi.avro.model.HoodieActionInstant import org.apache.hudi.avro.model.HoodieCleanerPlan import org.apache.hudi.avro.model.HoodieCleanFileInfo import org.apache.hudi.client.HoodieJavaWriteClient import org.apache.hudi.client.HoodieTimelineArchiver import org.apache.hudi.client.WriteStatus import org.apache.hudi.client.common.HoodieJavaEngineContext import org.apache.hudi.common.HoodieCleanStat import org.apache.hudi.common.config.HoodieMetadataConfig import org.apache.hudi.common.engine.HoodieEngineContext import org.apache.hudi.common.model.{HoodieAvroPayload, HoodieBaseFile, HoodieCleaningPolicy} import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieInstantTimeGenerator, HoodieTimeline, TimelineMetadataUtils} import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator.{MILLIS_INSTANT_TIMESTAMP_FORMAT_LENGTH, SECS_INSTANT_ID_LENGTH, SECS_INSTANT_TIMESTAMP_FORMAT} import org.apache.hudi.common.util.{Option => HudiOption} import org.apache.hudi.common.util.CleanerUtils import org.apache.hudi.common.util.ExternalFilePathUtil import org.apache.hudi.common.util.collection.Pair import org.apache.hudi.config.HoodieArchivalConfig import org.apache.hudi.config.HoodieCleanConfig import org.apache.hudi.config.HoodieIndexConfig import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.exception.{HoodieException, HoodieRollbackException} import org.apache.hudi.index.HoodieIndex.IndexType.INMEMORY import org.apache.hudi.table.HoodieJavaTable import org.apache.hudi.table.action.clean.CleanPlanner import org.apache.spark.internal.MDC /** * Used to prepare (convert) and then commit a set of Delta actions into the Hudi table located * at the same path as [[postCommitSnapshot]] * * * @param conf Configuration for Hudi Hadoop interactions. * @param postCommitSnapshot Latest Delta snapshot associated with this Hudi commit. */ class HudiConversionTransaction( protected val conf: Configuration, protected val postCommitSnapshot: Snapshot, protected val providedMetaClient: HoodieTableMetaClient, protected val lastConvertedDeltaVersion: Option[Long] = None) extends DeltaLogging { ////////////////////// // Member variables // ////////////////////// private val tablePath = postCommitSnapshot.deltaLog.dataPath private val hudiSchema: Schema = convertDeltaSchemaToHudiSchema(postCommitSnapshot.metadata.schema) private var metaClient = providedMetaClient private val instantTime = convertInstantToCommit( Instant.ofEpochMilli(postCommitSnapshot.timestamp)) private var writeStatuses: util.List[WriteStatus] = new util.ArrayList[WriteStatus]() private var partitionToReplacedFileIds: util.Map[String, util.List[String]] = new util.HashMap[String, util.List[String]]() private val version = postCommitSnapshot.version /** Tracks if this transaction has already committed. You can only commit once. */ private var committed = false ///////////////// // Public APIs // ///////////////// def setCommitFileUpdates(actions: scala.collection.Seq[Action]): Unit = { // for all removed files, group by partition path and then map to // the file group ID (name in this case) val newPartitionToReplacedFileIds = actions .map(_.wrap) .filter(action => action.remove != null) .map(_.remove) .map(remove => { val path = remove.toPath val partitionPath = getPartitionPath(tablePath, path) (partitionPath, path.getName)}) .groupBy(_._1).map(v => (v._1, v._2.map(_._2).asJava)) .asJava partitionToReplacedFileIds.putAll(newPartitionToReplacedFileIds) // Convert the AddFiles to write statuses for the commit val newWriteStatuses = actions .map(_.wrap) .filter(action => action.add != null) .map(_.add) .map(add => { convertAddFile(add, tablePath, instantTime) }) .asJava writeStatuses.addAll(newWriteStatuses) } def commit(): Unit = { assert(!committed, "Cannot commit. Transaction already committed.") val writeConfig = getWriteConfig(hudiSchema, getNumInstantsToRetain, 10, 7*24) val engineContext: HoodieEngineContext = new HoodieJavaEngineContext(metaClient.getStorageConf) val writeClient = new HoodieJavaWriteClient[AnyRef](engineContext, writeConfig) try { writeClient.startCommitWithTime(instantTime, HoodieTimeline.REPLACE_COMMIT_ACTION) metaClient.getActiveTimeline.transitionReplaceRequestedToInflight( new HoodieInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.REPLACE_COMMIT_ACTION, instantTime), HudiOption.empty[Array[Byte]]) val syncMetadata: Map[String, String] = Map( HudiConverter.DELTA_VERSION_PROPERTY -> version.toString, HudiConverter.DELTA_TIMESTAMP_PROPERTY -> postCommitSnapshot.timestamp.toString) writeClient.commit(instantTime, writeStatuses, HudiOption.of(syncMetadata.asJava), HoodieTimeline.REPLACE_COMMIT_ACTION, partitionToReplacedFileIds) // if the metaclient was created before the table's first commit, we need to reload it to // pick up the metadata table context if (!metaClient.getTableConfig.isMetadataTableAvailable) { metaClient = HoodieTableMetaClient.reload(metaClient) } val table = HoodieJavaTable.create(writeClient.getConfig, engineContext, metaClient) // clean up old commits and archive them markInstantsAsCleaned(table, writeClient.getConfig, engineContext) runArchiver(table, writeClient.getConfig, engineContext) } catch { case e: HoodieException if e.getMessage == "Failed to update metadata" || e.getMessage == "Error getting all file groups in pending clustering" || e.getMessage == "Error fetching partition paths from metadata table" => logInfo(log"[Thread=${MDC(DeltaLogKeys.THREAD_NAME, Thread.currentThread().getName)}] " + log"Failed to fully update Hudi metadata table for Delta snapshot version " + log"${MDC(DeltaLogKeys.VERSION, version)}. This is likely due to a concurrent " + log"commit and should not lead to data corruption.") case e: HoodieRollbackException => logInfo(log"[Thread=${MDC(DeltaLogKeys.THREAD_NAME, Thread.currentThread().getName)}] " + log"Failed to rollback Hudi metadata table for Delta snapshot version " + log"${MDC(DeltaLogKeys.VERSION, version)}. This is likely due to a concurrent " + log"commit and should not lead to data corruption.") case NonFatal(e) => recordHudiCommit(Some(e)) throw e } finally { if (writeClient != null) writeClient.close() recordHudiCommit() } committed = true } //////////////////// // Helper Methods // //////////////////// private def getNumInstantsToRetain = { val commitCutoff = convertInstantToCommit( parseFromInstantTime(instantTime).minus(7*24, ChronoUnit.HOURS)) // count number of completed commits after the cutoff metaClient .getActiveTimeline .filterCompletedInstants .findInstantsAfter(commitCutoff) .countInstants } private def markInstantsAsCleaned(table: HoodieJavaTable[_], writeConfig: HoodieWriteConfig, engineContext: HoodieEngineContext): Unit = { val planner = new CleanPlanner(engineContext, table, writeConfig) val earliestInstant = planner.getEarliestCommitToRetain // since we're retaining based on time, we should exit early if earliestInstant is empty if (!earliestInstant.isPresent) return var partitionsToClean: util.List[String] = null try partitionsToClean = planner.getPartitionPathsToClean(earliestInstant) catch { case ex: IOException => throw new UncheckedIOException("Unable to get partitions to clean", ex) } if (partitionsToClean.isEmpty) return val activeTimeline = metaClient.getActiveTimeline val fsView = table.getHoodieView val cleanInfoPerPartition = partitionsToClean.asScala.map(partition => Pair.of(partition, planner.getDeletePaths(partition, earliestInstant))) .filter(deletePaths => !deletePaths.getValue.getValue.isEmpty) .map(deletePathsForPartition => deletePathsForPartition.getKey -> { val partition = deletePathsForPartition.getKey // we need to manipulate the path to properly clean from the metadata table, // so we map the file path to the base file val baseFiles = fsView.getAllReplacedFileGroups(partition) .flatMap(fileGroup => fileGroup.getAllBaseFiles) .collect(Collectors.toList[HoodieBaseFile]) val baseFilesByPath = baseFiles.asScala .map(baseFile => baseFile.getPath -> baseFile).toMap deletePathsForPartition.getValue.getValue.asScala.map(cleanFileInfo => { val baseFile = baseFilesByPath.getOrElse(cleanFileInfo.getFilePath, null) new HoodieCleanFileInfo(ExternalFilePathUtil.appendCommitTimeAndExternalFileMarker( baseFile.getFileName, baseFile.getCommitTime), false) }).asJava }).toMap.asJava // there is nothing to clean, so exit early if (cleanInfoPerPartition.isEmpty) return // create a clean instant write after this latest commit val cleanTime = convertInstantToCommit(parseFromInstantTime(instantTime) .plus(1, ChronoUnit.SECONDS)) // create a metadata table writer in order to mark files as deleted in the table // the deleted entries are cleaned up in the metadata table during compaction to control the // growth of the table val hoodieTableMetadataWriter = table.getMetadataWriter(cleanTime).get try { val earliestInstantToRetain = earliestInstant .map[HoodieActionInstant]((earliestInstantToRetain: HoodieInstant) => new HoodieActionInstant( earliestInstantToRetain.getTimestamp, earliestInstantToRetain.getAction, earliestInstantToRetain.getState.name)) .orElse(null) val cleanerPlan = new HoodieCleanerPlan(earliestInstantToRetain, instantTime, writeConfig.getCleanerPolicy.name, Collections.emptyMap[String, util.List[String]], CleanPlanner.LATEST_CLEAN_PLAN_VERSION, cleanInfoPerPartition, Collections.emptyList[String], Collections.emptyMap[String, String]) // create a clean instant and mark it as requested with the clean plan val requestedCleanInstant = new HoodieInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.CLEAN_ACTION, cleanTime) activeTimeline.saveToCleanRequested( requestedCleanInstant, TimelineMetadataUtils.serializeCleanerPlan(cleanerPlan)) val inflightClean = activeTimeline .transitionCleanRequestedToInflight(requestedCleanInstant, HudiOption.empty[Array[Byte]]) val cleanStats = cleanInfoPerPartition.entrySet.asScala.map(entry => { val partitionPath = entry.getKey val deletePaths = entry.getValue.asScala.map(_.getFilePath).asJava new HoodieCleanStat(HoodieCleaningPolicy.KEEP_LATEST_COMMITS, partitionPath, deletePaths, deletePaths, Collections.emptyList[String], earliestInstant.get.getTimestamp, instantTime) }).toSeq.asJava val cleanMetadata = CleanerUtils.convertCleanMetadata(cleanTime, HudiOption.empty[java.lang.Long], cleanStats, java.util.Collections.emptyMap[String, String]) // update the metadata table with the clean metadata so the files' metadata are marked for // deletion hoodieTableMetadataWriter.performTableServices(HudiOption.empty[String]) hoodieTableMetadataWriter.update(cleanMetadata, cleanTime) // mark the commit as complete on the table timeline activeTimeline.transitionCleanInflightToComplete(inflightClean, TimelineMetadataUtils.serializeCleanMetadata(cleanMetadata)) } catch { case ex: IOException => throw new UncheckedIOException("Unable to clean Hudi timeline", ex) } finally if (hoodieTableMetadataWriter != null) hoodieTableMetadataWriter.close() } private def runArchiver(table: HoodieJavaTable[_ <: HoodieAvroPayload], config: HoodieWriteConfig, engineContext: HoodieEngineContext): Unit = { // trigger archiver manually val archiver = new HoodieTimelineArchiver(config, table) archiver.archiveIfRequired(engineContext, true) } private def getWriteConfig(schema: Schema, numCommitsToKeep: Int, maxNumDeltaCommitsBeforeCompaction: Int, timelineRetentionInHours: Int) = { val properties = new Properties properties.setProperty(HoodieMetadataConfig.AUTO_INITIALIZE.key, "false") HoodieWriteConfig.newBuilder .withIndexConfig(HoodieIndexConfig.newBuilder.withIndexType(INMEMORY).build) .withPath(metaClient.getBasePathV2.toString) .withPopulateMetaFields(metaClient.getTableConfig.populateMetaFields) .withEmbeddedTimelineServerEnabled(false) .withSchema(if (schema == null) "" else schema.toString) .withArchivalConfig(HoodieArchivalConfig.newBuilder .archiveCommitsWith(Math.max(0, numCommitsToKeep - 1), Math.max(1, numCommitsToKeep)) .withAutoArchive(false) .build) .withCleanConfig( HoodieCleanConfig.newBuilder .withCleanerPolicy(HoodieCleaningPolicy.KEEP_LATEST_BY_HOURS) .cleanerNumHoursRetained(timelineRetentionInHours) .withAutoClean(false) .build) .withMetadataConfig(HoodieMetadataConfig.newBuilder .enable(true) .withProperties(properties) .withMetadataIndexColumnStats(true) .withMaxNumDeltaCommitsBeforeCompaction(maxNumDeltaCommitsBeforeCompaction) .build) .build } /** * Copied mostly from {@link * org.apache.hudi.common.table.timeline.HoodieActiveTimeline#parseDateFromInstantTime(String)} * but forces the timestamp to use UTC unlike the Hudi code. * * @param timestamp input commit timestamp * @return timestamp parsed as Instant */ private def parseFromInstantTime(timestamp: String): Instant = { try { var timestampInMillis: String = timestamp if (isSecondGranularity(timestamp)) { timestampInMillis = timestamp + "999" } else { if (timestamp.length > MILLIS_INSTANT_TIMESTAMP_FORMAT_LENGTH) { timestampInMillis = timestamp.substring(0, MILLIS_INSTANT_TIMESTAMP_FORMAT_LENGTH) } } val dt: LocalDateTime = LocalDateTime.parse(timestampInMillis, MILLIS_INSTANT_TIME_FORMATTER) dt.atZone(ZoneId.of("UTC")).toInstant } catch { case ex: DateTimeParseException => throw new RuntimeException("Unable to parse date from commit timestamp: " + timestamp, ex) } } private def isSecondGranularity(instant: String) = instant.length == SECS_INSTANT_ID_LENGTH private def convertInstantToCommit(instant: Instant): String = { val instantTime = instant.atZone(ZoneId.of("UTC")).toLocalDateTime HoodieInstantTimeGenerator.getInstantFromTemporalAccessor(instantTime) } private def recordHudiCommit(errorOpt: Option[Throwable] = None): Unit = { val errorData = errorOpt.map { e => Map( "exception" -> ExceptionUtils.getMessage(e), "stackTrace" -> ExceptionUtils.getStackTrace(e) ) }.getOrElse(Map.empty) recordDeltaEvent( postCommitSnapshot.deltaLog, s"delta.hudi.conversion.commit.${if (errorOpt.isEmpty) "success" else "error"}", data = Map( "version" -> postCommitSnapshot.version, "timestamp" -> postCommitSnapshot.timestamp, "prevConvertedDeltaVersion" -> lastConvertedDeltaVersion ) ++ errorData ) } private val MILLIS_INSTANT_TIME_FORMATTER = new DateTimeFormatterBuilder() .appendPattern(SECS_INSTANT_TIMESTAMP_FORMAT) .appendValue(ChronoField.MILLI_OF_SECOND, 3) .toFormatter .withZone(ZoneId.of("UTC")) } ================================================ FILE: hudi/src/main/scala/org/apache/spark/sql/delta/hudi/HudiConverter.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.hudi import java.io.{IOException, UncheckedIOException} import java.util.concurrent.atomic.AtomicReference import javax.annotation.concurrent.GuardedBy import scala.collection.JavaConverters._ import scala.util.control.NonFatal import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.Snapshot import org.apache.spark.sql.delta.UniversalFormatConverter import org.apache.spark.sql.delta.actions.Action import org.apache.spark.sql.delta.hooks.HudiConverterHook import org.apache.spark.sql.delta.hudi.HudiTransactionUtils._ import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.hudi.common.model.{HoodieCommitMetadata, HoodieReplaceCommitMetadata} import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline} import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration import org.apache.spark.internal.MDC import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.catalog.CatalogTable object HudiConverter { /** * Property to be set in translated Hudi commit metadata. * Indicates the delta commit version # that it corresponds to. */ val DELTA_VERSION_PROPERTY = "delta-version" /** * Property to be set in translated Hudi commit metadata. * Indicates the timestamp (milliseconds) of the delta commit that it corresponds to. */ val DELTA_TIMESTAMP_PROPERTY = "delta-timestamp" } /** * This class manages the transformation of delta snapshots into their Hudi equivalent. */ class HudiConverter extends UniversalFormatConverter with DeltaLogging { // Save an atomic reference of the snapshot being converted, and the txn that triggered // resulted in the specified snapshot protected val currentConversion = new AtomicReference[(Snapshot, CommittedTransaction)]() protected val standbyConversion = new AtomicReference[(Snapshot, CommittedTransaction)]() // Whether our async converter thread is active. We may already have an alive thread that is // about to shutdown, but in such cases this value should return false. @GuardedBy("asyncThreadLock") private var asyncConverterThreadActive: Boolean = false private val asyncThreadLock = new Object /** * Enqueue the specified snapshot to be converted to Hudi. This will start an async * job to run the conversion, unless there already is an async conversion running for * this table. In that case, it will queue up the provided snapshot to be run after * the existing job completes. * Note that if there is another snapshot already queued, the previous snapshot will get * removed from the wait queue. Only one snapshot is queued at any point of time. * */ override def enqueueSnapshotForConversion( snapshotToConvert: Snapshot, txn: CommittedTransaction): Unit = { if (!UniversalFormat.hudiEnabled(snapshotToConvert.metadata)) { return } val log = snapshotToConvert.deltaLog // Replace any previously queued snapshot val previouslyQueued = standbyConversion.getAndSet((snapshotToConvert, txn)) asyncThreadLock.synchronized { if (!asyncConverterThreadActive) { val threadName = HudiConverterHook.ASYNC_HUDI_CONVERTER_THREAD_NAME + s" [id=${snapshotToConvert.metadata.id}]" val asyncConverterThread: Thread = new Thread(threadName) { setDaemon(true) override def run(): Unit = try { var snapshotAndTxn = getNextSnapshot while (snapshotAndTxn != null) { val snapshotVal = snapshotAndTxn._1 val prevTxn = snapshotAndTxn._2 try { logInfo(log"Converting Delta table [path=" + log"${MDC(DeltaLogKeys.PATH, log.logPath)}, " + log"tableId=${MDC(DeltaLogKeys.TABLE_ID, log.unsafeVolatileTableId)}, " + log"version=${MDC(DeltaLogKeys.VERSION, snapshotVal.version)}] into Hudi") convertSnapshot(snapshotVal, prevTxn) } catch { case NonFatal(e) => logWarning(log"Error when writing Hudi metadata asynchronously", e) recordDeltaEvent( log, "delta.hudi.conversion.async.error", data = Map( "exception" -> ExceptionUtils.getMessage(e), "stackTrace" -> ExceptionUtils.getStackTrace(e) ) ) } currentConversion.set(null) // Pick next snapshot to convert if there's a new one snapshotAndTxn = getNextSnapshot } } finally { // shuttingdown thread asyncThreadLock.synchronized { asyncConverterThreadActive = false } } // Get a snapshot to convert from the hudiQueue. Sets the queue to null after. private def getNextSnapshot: (Snapshot, CommittedTransaction) = asyncThreadLock.synchronized { val potentialSnapshotAndTxn = standbyConversion.get() currentConversion.set(potentialSnapshotAndTxn) standbyConversion.compareAndSet(potentialSnapshotAndTxn, null) if (potentialSnapshotAndTxn == null) { asyncConverterThreadActive = false } potentialSnapshotAndTxn } } asyncConverterThread.start() asyncConverterThreadActive = true } } // If there already was a snapshot waiting to be converted, log that snapshot info. if (previouslyQueued != null) { recordDeltaEvent( snapshotToConvert.deltaLog, "delta.hudi.conversion.async.backlog", data = Map( "newVersion" -> snapshotToConvert.version, "replacedVersion" -> previouslyQueued._1.version) ) } } /** * Convert the specified snapshot into Hudi for the given catalogTable * @param snapshotToConvert the snapshot that needs to be converted to Hudi * @param catalogTable the catalogTable this conversion targets. * @return Converted Delta version and commit timestamp */ override def convertSnapshot( snapshotToConvert: Snapshot, catalogTable: CatalogTable): Option[(Long, Long)] = { if (!UniversalFormat.hudiEnabled(snapshotToConvert.metadata)) { return None } convertSnapshot(snapshotToConvert, None, Some(catalogTable)) } /** * Convert the specified snapshot into Hudi when performing an OptimisticTransaction * on a delta table. * @param snapshotToConvert the snapshot that needs to be converted to Hudi * @param txn the transaction that triggers the conversion. It must * contain the catalogTable this conversion targets. * @return Converted Delta version and commit timestamp */ override def convertSnapshot( snapshotToConvert: Snapshot, txn: CommittedTransaction): Option[(Long, Long)] = { if (!UniversalFormat.hudiEnabled(snapshotToConvert.metadata)) { return None } convertSnapshot(snapshotToConvert, Some(txn), txn.catalogTable) } /** * Convert the specified snapshot into Hudi. NOTE: This operation is blocking. Call * enqueueSnapshotForConversion to run the operation asynchronously. * @param snapshotToConvert the snapshot that needs to be converted to Hudi * @param txnOpt the OptimisticTransaction that created snapshotToConvert. * Used as a hint to avoid recomputing old metadata. * @param catalogTable the catalogTable this conversion targets * @return Converted Delta version and commit timestamp */ private def convertSnapshot( snapshotToConvert: Snapshot, txnOpt: Option[CommittedTransaction], catalogTable: Option[CatalogTable]): Option[(Long, Long)] = recordFrameProfile("Delta", "HudiConverter.convertSnapshot") { val log = snapshotToConvert.deltaLog val metaClient = loadTableMetaClient( snapshotToConvert.deltaLog.dataPath.toString, catalogTable.flatMap(ct => Option(ct.identifier.table)), snapshotToConvert.metadata.partitionColumns, new HadoopStorageConfiguration(log.newDeltaHadoopConf())) val lastDeltaVersionConverted: Option[Long] = loadLastDeltaVersionConverted(metaClient) val maxCommitsToConvert = spark.sessionState.conf.getConf(DeltaSQLConf.HUDI_MAX_COMMITS_TO_CONVERT) // Nth to convert if (lastDeltaVersionConverted.exists(_ == snapshotToConvert.version)) { return None } // Get the most recently converted delta snapshot, if applicable val prevConvertedSnapshotOpt = (lastDeltaVersionConverted, txnOpt) match { case (Some(version), Some(txn)) if version == txn.readSnapshot.version => Some(txn.readSnapshot) // Check how long it has been since we last converted to Hudi. If outside the threshold, // fall back to state reconstruction to get the actions, to protect driver from OOMing. case (Some(version), _) if snapshotToConvert.version - version <= maxCommitsToConvert => try { // TODO: We can optimize this by providing a checkpointHint to getSnapshotAt. Check if // txn.snapshot.version < version. If true, use txn.snapshot's checkpoint as a hint. Some(log.getSnapshotAt(version, catalogTableOpt = catalogTable)) } catch { // If we can't load the file since the last time Hudi was converted, it's likely that // the commit file expired. Treat this like a new Hudi table conversion. case _: DeltaFileNotFoundException => None } case (_, _) => None } val hudiTxn = new HudiConversionTransaction(log.newDeltaHadoopConf(), snapshotToConvert, metaClient, lastDeltaVersionConverted) // Write out the actions taken since the last conversion (or since table creation). // This is done in batches, with each batch corresponding either to one delta file, // or to the specified batch size. val actionBatchSize = spark.sessionState.conf.getConf(DeltaSQLConf.HUDI_MAX_COMMITS_TO_CONVERT) prevConvertedSnapshotOpt match { case Some(prevSnapshot) => // Read the actions directly from the delta json files. // TODO: Run this as a spark job on executors val deltaFiles = DeltaFileProviderUtils.getDeltaFilesInVersionRange( spark = spark, deltaLog = log, startVersion = prevSnapshot.version + 1, endVersion = snapshotToConvert.version, catalogTableOpt = catalogTable) recordDeltaEvent( snapshotToConvert.deltaLog, "delta.hudi.conversion.deltaCommitRange", data = Map( "fromVersion" -> (prevSnapshot.version + 1), "toVersion" -> snapshotToConvert.version, "numDeltaFiles" -> deltaFiles.length ) ) val actionsToConvert = DeltaFileProviderUtils.parallelReadAndParseDeltaFilesAsIterator( log, spark, deltaFiles) actionsToConvert.foreach { actionsIter => try { actionsIter.grouped(actionBatchSize).foreach { actionStrs => runHudiConversionForActions( hudiTxn, actionStrs.map(Action.fromJson)) } } finally { actionsIter.close() } } // If we don't have a snapshot of the last converted version, get all the table addFiles // (via state reconstruction). case None => val actionsToConvert = snapshotToConvert.allFiles.toLocalIterator().asScala recordDeltaEvent( snapshotToConvert.deltaLog, "delta.hudi.conversion.batch", data = Map( "version" -> snapshotToConvert.version, "numDeltaFiles" -> snapshotToConvert.numOfFiles ) ) actionsToConvert.grouped(actionBatchSize) .foreach { actions => runHudiConversionForActions(hudiTxn, actions) } } hudiTxn.commit() Some(snapshotToConvert.version, snapshotToConvert.timestamp) } def loadLastDeltaVersionConverted(snapshot: Snapshot, table: CatalogTable): Option[Long] = { val metaClient = loadTableMetaClient(snapshot.deltaLog.dataPath.toString, Option.apply(table.identifier.table), snapshot.metadata.partitionColumns, new HadoopStorageConfiguration(snapshot.deltaLog.newDeltaHadoopConf())) loadLastDeltaVersionConverted(metaClient) } private def loadLastDeltaVersionConverted(metaClient: HoodieTableMetaClient): Option[Long] = { val lastCompletedCommit = metaClient.getCommitsTimeline.filterCompletedInstants.lastInstant if (!lastCompletedCommit.isPresent) { return None } val extraMetadata = parseCommitExtraMetadata(lastCompletedCommit.get(), metaClient) extraMetadata.get(HudiConverter.DELTA_VERSION_PROPERTY).map(_.toLong) } private def parseCommitExtraMetadata(instant: HoodieInstant, metaClient: HoodieTableMetaClient): Map[String, String] = { try { if (instant.getAction == HoodieTimeline.REPLACE_COMMIT_ACTION) { HoodieReplaceCommitMetadata.fromBytes( metaClient.getActiveTimeline.getInstantDetails(instant).get, classOf[HoodieReplaceCommitMetadata]).getExtraMetadata.asScala.toMap } else { HoodieCommitMetadata.fromBytes( metaClient.getActiveTimeline.getInstantDetails(instant).get, classOf[HoodieCommitMetadata]).getExtraMetadata.asScala.toMap } } catch { case ex: IOException => throw new UncheckedIOException("Unable to read Hudi commit metadata", ex) } } private[delta] def runHudiConversionForActions( hudiTxn: HudiConversionTransaction, actionsToCommit: Seq[Action]): Unit = { hudiTxn.setCommitFileUpdates(actionsToCommit) } } ================================================ FILE: hudi/src/main/scala/org/apache/spark/sql/delta/hudi/HudiSchemaUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.hudi import java.util import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.avro.{LogicalTypes, Schema} import org.apache.spark.sql.types._ object HudiSchemaUtils extends DeltaLogging { ///////////////// // Public APIs // ///////////////// def convertDeltaSchemaToHudiSchema(deltaSchema: StructType): Schema = { /** * Recursively (i.e. for all nested elements) transforms the delta DataType `elem` into its * corresponding Avro type. */ def transform[E <: DataType](elem: E, isNullable: Boolean, currentPath: String): Schema = elem match { case StructType(fields) => val avroFields: util.List[Schema.Field] = fields.map(f => new Schema.Field( f.name, transform(f.dataType, f.nullable, s"$currentPath.${f.name}"), f.getComment().orNull)).toList.asJava finalizeSchema( Schema.createRecord(currentPath, null, null, false, avroFields), isNullable) case ArrayType(elementType, containsNull) => finalizeSchema( Schema.createArray(transform(elementType, containsNull, currentPath)), isNullable) case MapType(keyType, valueType, valueContainsNull) => finalizeSchema( Schema.createMap(transform(valueType, valueContainsNull, currentPath)), isNullable) case atomicType: AtomicType => convertAtomic(atomicType, isNullable) case other => throw new UnsupportedOperationException(s"Cannot convert Delta type $other to Hudi") } transform(deltaSchema, false, "root") } private def finalizeSchema(targetSchema: Schema, isNullable: Boolean): Schema = { if (isNullable) return Schema.createUnion(Schema.create(Schema.Type.NULL), targetSchema) targetSchema } private def convertAtomic[E <: DataType](elem: E, isNullable: Boolean) = elem match { case StringType => finalizeSchema(Schema.create(Schema.Type.STRING), isNullable) case LongType => finalizeSchema(Schema.create(Schema.Type.LONG), isNullable) case IntegerType => finalizeSchema( Schema.create(Schema.Type.INT), isNullable) case FloatType => finalizeSchema(Schema.create(Schema.Type.FLOAT), isNullable) case DoubleType => finalizeSchema(Schema.create(Schema.Type.DOUBLE), isNullable) case d: DecimalType => finalizeSchema(LogicalTypes.decimal(d.precision, d.scale) .addToSchema(Schema.create(Schema.Type.BYTES)), isNullable) case BooleanType => finalizeSchema(Schema.create(Schema.Type.BOOLEAN), isNullable) case BinaryType => finalizeSchema(Schema.create(Schema.Type.BYTES), isNullable) case DateType => finalizeSchema( LogicalTypes.date.addToSchema(Schema.create(Schema.Type.INT)), isNullable) case TimestampType => finalizeSchema( LogicalTypes.timestampMicros.addToSchema(Schema.create(Schema.Type.LONG)), isNullable) case _ => throw new UnsupportedOperationException(s"Could not convert atomic type $elem") } } ================================================ FILE: hudi/src/main/scala/org/apache/spark/sql/delta/hudi/HudiTransactionUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.hudi import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.hadoop.fs.Path import org.apache.hudi.client.WriteStatus import org.apache.hudi.common.model.{HoodieAvroPayload, HoodieDeltaWriteStat, HoodieTableType, HoodieTimelineTimeZone} import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.common.util.ExternalFilePathUtil import org.apache.hudi.exception.TableNotFoundException import org.apache.hudi.storage.StorageConfiguration object HudiTransactionUtils extends DeltaLogging { ///////////////// // Public APIs // ///////////////// def convertAddFile(addFile: AddFile, tablePath: Path, commitTime: String): WriteStatus = { val writeStatus = new WriteStatus val path = addFile.toPath val partitionPath = getPartitionPath(tablePath, path) val fileName = path.getName val fileId = fileName val filePath = if (partitionPath.isEmpty) fileName else partitionPath + "/" + fileName writeStatus.setFileId(fileId) writeStatus.setPartitionPath(partitionPath) val writeStat = new HoodieDeltaWriteStat writeStat.setFileId(fileId) writeStat.setPath( ExternalFilePathUtil.appendCommitTimeAndExternalFileMarker(filePath, commitTime)) writeStat.setPartitionPath(partitionPath) writeStat.setNumWrites(addFile.numLogicalRecords.getOrElse(0L)) writeStat.setTotalWriteBytes(addFile.getFileSize) writeStat.setFileSizeInBytes(addFile.getFileSize) writeStatus.setStat(writeStat) writeStatus } def getPartitionPath(tableBasePath: Path, filePath: Path): String = { val fileName = filePath.getName val pathStr = filePath.toUri.getPath val tableBasePathStr = tableBasePath.toUri.getPath if (pathStr.contains(tableBasePathStr)) { // input file path is absolute val startIndex = tableBasePath.toUri.getPath.length + 1 val endIndex = pathStr.length - fileName.length - 1 if (endIndex <= startIndex) "" else pathStr.substring(startIndex, endIndex) } else { val lastSlash = pathStr.lastIndexOf("/") if (lastSlash <= 0) "" else pathStr.substring(0, pathStr.lastIndexOf("/")) } } /** * Loads the meta client for the table at the base path if it exists. * If it does not exist, initializes the Hudi table and returns the meta client. * * @param tableDataPath the path for the table * @param tableName the name of the table * @param partitionFields the fields used for partitioning * @param conf the hadoop configuration * @return {@link HoodieTableMetaClient} for the existing table or that was created */ def loadTableMetaClient(tableDataPath: String, tableName: Option[String], partitionFields: Seq[String], conf: StorageConfiguration[_]): HoodieTableMetaClient = { try HoodieTableMetaClient.builder .setBasePath(tableDataPath).setConf(conf) .setLoadActiveTimelineOnLoad(false) .build catch { case ex: TableNotFoundException => log.debug("Hudi table does not exist, creating now.") if (tableName.isEmpty) { log.warn("No name is specified for the table. " + "Creating a new Hudi table with a default name: 'table'.") } initializeHudiTable(tableDataPath, tableName.getOrElse("table"), partitionFields, conf) } } /** * Initializes a Hudi table with the provided properties * * @param tableDataPath the base path for the data files in the table * @param tableName the name of the table * @param partitionFields the fields used for partitioning * @param conf the hadoop configuration * @return {@link HoodieTableMetaClient} for the table that was created */ private def initializeHudiTable(tableDataPath: String, tableName: String, partitionFields: Seq[String], conf: StorageConfiguration[_]): HoodieTableMetaClient = { val keyGeneratorClass = getKeyGeneratorClass(partitionFields) HoodieTableMetaClient .withPropertyBuilder .setCommitTimezone(HoodieTimelineTimeZone.UTC) .setHiveStylePartitioningEnable(true) .setTableType(HoodieTableType.COPY_ON_WRITE) .setTableName(tableName) .setPayloadClass(classOf[HoodieAvroPayload]) .setKeyGeneratorClassProp(keyGeneratorClass) .setPopulateMetaFields(false) .setPartitionFields(partitionFields.mkString(",")) .initTable(conf, tableDataPath) } private def getKeyGeneratorClass(partitionFields: Seq[String]): String = { if (partitionFields.isEmpty) { "org.apache.hudi.keygen.NonpartitionedKeyGenerator" } else if (partitionFields.size > 1) { "org.apache.hudi.keygen.CustomKeyGenerator" } else { "org.apache.hudi.keygen.SimpleKeyGenerator" } } } ================================================ FILE: hudi/src/test/scala/org/apache/spark/sql/delta/hudi/ConvertToHudiSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.hudi import java.time.Instant import java.util.UUID import java.util.stream.Collectors import scala.collection.JavaConverters import org.apache.spark.sql.delta.{DeltaConfigs, DeltaLog, DeltaUnsupportedOperationException, OptimisticTransaction} import org.apache.spark.sql.delta.DeltaOperations.Truncate import org.apache.spark.sql.delta.actions.{Action, AddFile, Metadata} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.hudi.common.config.HoodieMetadataConfig import org.apache.hudi.common.engine.HoodieLocalEngineContext import org.apache.hudi.common.fs.FSUtils import org.apache.hudi.common.model.HoodieBaseFile import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver} import org.apache.hudi.metadata.HoodieMetadataFileSystemView import org.apache.hudi.storage.StorageConfiguration import org.apache.hudi.storage.hadoop.{HadoopStorageConfiguration, HoodieHadoopStorage} import org.scalatest.concurrent.Eventually import org.scalatest.time.SpanSugar._ import org.apache.spark.sql.QueryTest import org.apache.spark.sql.SparkSession import org.apache.spark.sql.avro.SchemaConverters import org.apache.spark.sql.types.StructType import org.apache.spark.util.ManualClock trait HudiTestBase extends QueryTest with Eventually { /** * Executes `f` with params (tableId, tempPath). * * We want to use a temp directory in addition to a unique temp table so that when the async * Hudi conversion runs and completes, the parent folder is still removed. */ def withTempTableAndDir(f: (String, String) => Unit): Unit protected def spark: SparkSession def buildHudiMetaClient(testTablePath: String): HoodieTableMetaClient = { // scalastyle:off deltahadoopconfiguration val hadoopConf: Configuration = spark.sessionState.newHadoopConf() // scalastyle:on deltahadoopconfiguration val storageConf : StorageConfiguration[_] = new HadoopStorageConfiguration(hadoopConf) HoodieTableMetaClient.builder .setConf(storageConf).setBasePath(testTablePath) .setLoadActiveTimelineOnLoad(true) .build } def verifyFilesAndSchemaMatch(testTableName: String, testTablePath: String): Unit = { eventually(timeout(30.seconds)) { // To avoid requiring Hudi spark dependencies, we first lookup the active base files and then // assert by reading those active base files (parquet) directly // scalastyle:off deltahadoopconfiguration val hadoopConf: Configuration = spark.sessionState.newHadoopConf() // scalastyle:on deltahadoopconfiguration val storageConf : StorageConfiguration[_] = new HadoopStorageConfiguration(hadoopConf) val metaClient: HoodieTableMetaClient = buildHudiMetaClient(testTablePath) val engContext: HoodieLocalEngineContext = new HoodieLocalEngineContext(storageConf) val fsView: HoodieMetadataFileSystemView = new HoodieMetadataFileSystemView(engContext, metaClient, metaClient.getActiveTimeline.getCommitsTimeline.filterCompletedInstants, HoodieMetadataConfig.newBuilder.enable(true).build) val hoodieStorage = new HoodieHadoopStorage(testTablePath, storageConf) val paths = JavaConverters.asScalaBuffer( FSUtils.getAllPartitionPaths(engContext, hoodieStorage, testTablePath, true, false)) .flatMap(partition => JavaConverters.asScalaBuffer(fsView.getLatestBaseFiles(partition) .collect(Collectors.toList[HoodieBaseFile]))) .map(baseFile => baseFile.getPath).sorted val avroSchema = new TableSchemaResolver(metaClient).getTableAvroSchema val hudiSchemaAsStruct = SchemaConverters.toSqlType(avroSchema).dataType .asInstanceOf[StructType] val deltaDF = spark.sql(s"SELECT * FROM $testTableName") // Assert file paths are equivalent val expectedFiles = deltaDF.inputFiles.map(path => path.substring(5)).toSeq.sorted assert(paths.equals(expectedFiles), s"Files do not match.\nExpected: $expectedFiles\nActual: $paths") // Assert schemas are equal val expectedSchema = deltaDF.schema assert(hudiSchemaAsStruct.equals(expectedSchema), s"Schemas do not match.\nExpected: $expectedSchema\nActual: $hudiSchemaAsStruct") } } def withDefaultTablePropsInSQLConf(enableInCommitTimestamp: Boolean, f: => Unit): Unit = { withSQLConf( DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey -> "name", DeltaConfigs.UNIVERSAL_FORMAT_ENABLED_FORMATS.defaultTablePropertyKey -> "hudi", DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey -> enableInCommitTimestamp.toString ) { f } } protected def startTxnWithManualLogCleanup(log: DeltaLog): OptimisticTransaction = { val txn = log.startTransaction() // This will pick up `spark.databricks.delta.properties.defaults.enableExpiredLogCleanup` to // disable log cleanup. txn.updateMetadata(Metadata()) txn } def verifyNumHudiCommits(count: Integer, testTablePath: String): Unit = { eventually(timeout(30.seconds)) { val metaClient: HoodieTableMetaClient = buildHudiMetaClient(testTablePath) val activeCommits = metaClient.getActiveTimeline.getCommitsTimeline .filterCompletedInstants.countInstants val archivedCommits = metaClient.getArchivedTimeline.getCommitsTimeline .filterCompletedInstants.countInstants assert(activeCommits + archivedCommits == count) } } } trait ConvertToHudiTestBase extends HudiTestBase { test("basic test - managed table created with SQL") { withTempTableAndDir { case (testTableName, testTablePath) => spark.sql( s""" |CREATE TABLE $testTableName (ID INT) USING DELTA |LOCATION '$testTablePath' |TBLPROPERTIES ( | 'delta.universalFormat.enabledFormats' = 'hudi' |)""".stripMargin) spark.sql(s"INSERT INTO $testTableName VALUES (123)") verifyFilesAndSchemaMatch(testTableName, testTablePath) } } test("basic test - catalog table created with DataFrame") { withTempTableAndDir { case (testTableName, testTablePath) => withDefaultTablePropsInSQLConf(false, { spark.range(10).write.format("delta") .option("path", testTablePath) .saveAsTable(testTableName) }) verifyFilesAndSchemaMatch(testTableName, testTablePath) withDefaultTablePropsInSQLConf(false, { spark.range(10, 20, 1) .write.format("delta").mode("append") .save(testTablePath) }) verifyFilesAndSchemaMatch(testTableName, testTablePath) } } for (isPartitioned <- Seq(true, false)) { test(s"validate multiple commits (partitioned = $isPartitioned)") { withTempTableAndDir { case (testTableName, testTablePath) => spark.sql( s"""CREATE TABLE $testTableName (col1 INT, col2 STRING, col3 STRING) USING DELTA |${if (isPartitioned) "PARTITIONED BY (col3)" else ""} |LOCATION '$testTablePath' |TBLPROPERTIES ( | 'delta.universalFormat.enabledFormats' = 'hudi' |)""".stripMargin) // perform some inserts spark.sql(s"INSERT INTO $testTableName VALUES (1, 'instant1', 'a'), (2, 'instant1', 'a')") verifyFilesAndSchemaMatch(testTableName, testTablePath) spark.sql(s"INSERT INTO `$testTableName` VALUES (3, 'instant2', 'b'), (4, 'instant2', 'b')") verifyFilesAndSchemaMatch(testTableName, testTablePath) spark.sql(s"INSERT INTO `$testTableName` VALUES (5, 'instant3', 'b'), (6, 'instant3', 'a')") verifyFilesAndSchemaMatch(testTableName, testTablePath) // update the data from the first instant spark.sql(s"UPDATE `$testTableName` SET col2 = 'instant4' WHERE col2 = 'instant1'") verifyFilesAndSchemaMatch(testTableName, testTablePath) // delete a single row spark.sql(s"DELETE FROM `$testTableName` WHERE col1 = 5") verifyFilesAndSchemaMatch(testTableName, testTablePath) } } } test("Enabling Delete Vector Throws Exception") { withTempTableAndDir { case (testTableName, testTablePath) => intercept[DeltaUnsupportedOperationException] { spark.sql( s"""CREATE TABLE `$testTableName` (col1 INT, col2 STRING) USING DELTA |LOCATION '$testTablePath' |TBLPROPERTIES ( | 'delta.universalFormat.enabledFormats' = 'hudi', | 'delta.enableDeletionVectors' = true |)""".stripMargin) } } } test("Enabling Delete Vector After Hudi Enabled Already Throws Exception") { withTempTableAndDir { case (testTableName, testTablePath) => spark.sql( s"""CREATE TABLE `$testTableName` (col1 INT, col2 STRING) USING DELTA |LOCATION '$testTablePath' |TBLPROPERTIES ( | 'delta.universalFormat.enabledFormats' = 'hudi' |)""".stripMargin) intercept[DeltaUnsupportedOperationException] { spark.sql( s"""ALTER TABLE `$testTableName` SET TBLPROPERTIES ( | 'delta.enableDeletionVectors' = true |)""".stripMargin) } } } test(s"Conversion behavior for lists") { withTempTableAndDir { case (testTableName, testTablePath) => spark.sql( s"""CREATE TABLE `$testTableName` (col1 ARRAY) USING DELTA |LOCATION '$testTablePath' |TBLPROPERTIES ( | 'delta.universalFormat.enabledFormats' = 'hudi' |)""".stripMargin) spark.sql(s"INSERT INTO `$testTableName` VALUES (array(1, 2, 3))") verifyFilesAndSchemaMatch(testTableName, testTablePath) } } test(s"Conversion behavior for lists of structs") { withTempTableAndDir { case (testTableName, testTablePath) => spark.sql( s"""CREATE TABLE `$testTableName` |(col1 ARRAY>) USING DELTA |LOCATION '$testTablePath' |TBLPROPERTIES ( | 'delta.universalFormat.enabledFormats' = 'hudi' |)""".stripMargin) spark.sql(s"INSERT INTO `$testTableName` " + s"VALUES (array(named_struct('field1', 1, 'field2', 'hello'), " + s"named_struct('field1', 2, 'field2', 'world')))") verifyFilesAndSchemaMatch(testTableName, testTablePath) } } test(s"Conversion behavior for lists of lists") { withTempTableAndDir { case (testTableName, testTablePath) => spark.sql( s"""CREATE TABLE `$testTableName` |(col1 ARRAY>) USING DELTA |LOCATION '$testTablePath' |TBLPROPERTIES ( | 'delta.universalFormat.enabledFormats' = 'hudi' |)""".stripMargin) spark.sql(s"INSERT INTO `$testTableName` " + s"VALUES (array(array(1, 2, 3), array(4, 5, 6)))") verifyFilesAndSchemaMatch(testTableName, testTablePath) } } test(s"Conversion behavior for maps") { withTempTableAndDir { case (testTableName, testTablePath) => spark.sql( s"""CREATE TABLE `$testTableName` (col1 MAP) USING DELTA |LOCATION '$testTablePath' |TBLPROPERTIES ( | 'delta.universalFormat.enabledFormats' = 'hudi' |)""".stripMargin) spark.sql( s"INSERT INTO `$testTableName` VALUES (map('a', 1, 'b', 2, 'c', 3))" ) verifyFilesAndSchemaMatch(testTableName, testTablePath) } } test(s"Conversion behavior for nested structs") { withTempTableAndDir { case (testTableName, testTablePath) => spark.sql( s"""CREATE TABLE `$testTableName` (col1 STRUCT>) |USING DELTA |LOCATION '$testTablePath' |TBLPROPERTIES ( | 'delta.universalFormat.enabledFormats' = 'hudi' |)""".stripMargin) spark.sql( s"INSERT INTO `$testTableName` VALUES (named_struct('field1', 1, 'field2', 'hello', " + "'field3', named_struct('field4', 2, 'field5', 3, 'field6', 'world')))" ) verifyFilesAndSchemaMatch(testTableName, testTablePath) } } test("validate Hudi timeline archival and cleaning") { withTempTableAndDir { case (_, testTablePath) => val testOp = Truncate() withDefaultTablePropsInSQLConf(true, { val startTime = System.currentTimeMillis() - 12 * 24 * 60 * 60 * 1000 val clock = new ManualClock(startTime) val actualTestStartTime = System.currentTimeMillis() val log = DeltaLog.forTable(spark, new Path(testTablePath), clock) (1 to 20).foreach { i => val txn = if (i == 1) startTxnWithManualLogCleanup(log) else log.startTransaction() val file = AddFile(i.toString + ".parquet", Map.empty, 1, 1, true) :: Nil val delete: Seq[Action] = if (i > 1) { val timestamp = startTime + (System.currentTimeMillis() - actualTestStartTime) val prevFile = AddFile((i - 1).toString + ".parquet", Map.empty, 1, 1, true) prevFile.removeWithTimestamp(timestamp) :: Nil } else { Nil } txn.commit(delete ++ file, testOp) clock.advance(12.hours.toMillis) // wait for each Hudi sync to complete verifyNumHudiCommits(i, testTablePath) } val metaClient: HoodieTableMetaClient = HoodieTableMetaClient.builder .setConf(new HadoopStorageConfiguration(log.newDeltaHadoopConf())) .setBasePath(log.dataPath.toString) .setLoadActiveTimelineOnLoad(true) .build // Timeline requires a clean commit for proper removal of entries from the Hudi // Metadata Table assert(metaClient.getActiveTimeline.getCleanerTimeline.countInstants() == 1, "Cleaner timeline should have 1 instant") // Older commits should move from active to archive timeline // TODO Fix the flaky tests /* assert(metaClient.getArchivedTimeline.getCommitsTimeline.filterInflights.countInstants == 2, "Archived timeline should have 2 instants") */ }) } } test("validate various data types") { withTempTableAndDir { case (testTableName, testTablePath) => spark.sql( s"""CREATE TABLE `$testTableName` (col1 BIGINT, col2 BOOLEAN, col3 DATE, | col4 DOUBLE, col5 FLOAT, col6 INT, col7 STRING, col8 TIMESTAMP, | col9 BINARY, col10 DECIMAL(5, 2), | col11 STRUCT>) | USING DELTA |LOCATION '$testTablePath' |TBLPROPERTIES ( | 'delta.universalFormat.enabledFormats' = 'hudi' |)""".stripMargin) val nowSeconds = Instant.now().getEpochSecond spark.sql(s"INSERT INTO `$testTableName` VALUES (123, true, " + s"date(from_unixtime($nowSeconds)), 32.1, 1.23, 456, 'hello world', " + s"timestamp(from_unixtime($nowSeconds)), X'1ABF', -999.99," + s"STRUCT(1, 'hello', STRUCT(2, 3, 'world')))") verifyFilesAndSchemaMatch(testTableName, testTablePath) } } for (invalidType <- Seq("SMALLINT", "TINYINT", "TIMESTAMP_NTZ", "VOID")) { test(s"Unsupported Type $invalidType Throws Exception") { withTempTableAndDir { case (testTableName, testTablePath) => intercept[DeltaUnsupportedOperationException] { spark.sql( s"""CREATE TABLE `$testTableName` (col1 $invalidType) USING DELTA |LOCATION '$testTablePath' |TBLPROPERTIES ( | 'delta.universalFormat.enabledFormats' = 'hudi' |)""".stripMargin) } } } } test("all batches of actions are converted") { withTempTableAndDir { case (testTableName, testTablePath) => withSQLConf( DeltaSQLConf.HUDI_MAX_COMMITS_TO_CONVERT.key -> "3" ) { spark.sql( s"""CREATE TABLE `$testTableName` (col1 INT) | USING DELTA |LOCATION '$testTablePath'""".stripMargin) for (i <- 1 to 10) { spark.sql(s"INSERT INTO `$testTableName` VALUES ($i)") } spark.sql( s"""ALTER TABLE `$testTableName` SET TBLPROPERTIES ( | 'delta.universalFormat.enabledFormats' = 'hudi' |)""".stripMargin) verifyFilesAndSchemaMatch(testTableName, testTablePath) } } } } class ConvertToHudiSuite extends ConvertToHudiTestBase { private var _sparkSession: SparkSession = null override def spark: SparkSession = _sparkSession override def beforeAll(): Unit = { super.beforeAll() _sparkSession = createSparkSession() _sparkSession.conf.set( DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey, "true") } def createSparkSession(): SparkSession = { SparkSession.clearActiveSession() SparkSession.clearDefaultSession() SparkSession.builder() .master("local[*]") .appName("UniformSession") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") .getOrCreate() } override def withTempTableAndDir(f: (String, String) => Unit): Unit = { val tableId = s"testTable${UUID.randomUUID()}".replace("-", "_") withTempDir { externalLocation => val tablePath = new Path(externalLocation.toString, "table") f(tableId, s"$tablePath") } } } ================================================ FILE: iceberg/integration_tests/iceberg_converter.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from pyspark.sql import SparkSession from pyspark.sql.functions import col from delta.tables import DeltaTable import shutil import random testRoot = "/tmp/delta-iceberg-converter/" warehousePath = testRoot + "iceberg_tables" shutil.rmtree(testRoot, ignore_errors=True) # we need to set the following configs spark = SparkSession.builder \ .appName("delta-iceberg-converter") \ .master("local[*]") \ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \ .config("spark.sql.catalog.local", "org.apache.iceberg.spark.SparkCatalog") \ .config("spark.sql.catalog.local.type", "hadoop") \ .config("spark.sql.catalog.local.warehouse", warehousePath) \ .getOrCreate() table = "local.db.table" tablePath = "file://" + warehousePath + "/db/table" try: print("Creating Iceberg table with partitions...") spark.sql( "CREATE TABLE {} (id BIGINT, data STRING) USING ICEBERG PARTITIONED BY (data)".format(table)) spark.sql("INSERT INTO {} VALUES (1, 'a'), (2, 'b')".format(table)) spark.sql("INSERT INTO {} VALUES (3, 'c')".format(table)) print("Converting Iceberg table to Delta table...") spark.sql("CONVERT TO DELTA iceberg.`{}`".format(tablePath)) print("Reading from converted Delta table...") spark.read.format("delta").load(tablePath).show() print("Modifying the converted table...") spark.sql("INSERT INTO delta.`{}` VALUES (4, 'd')".format(tablePath)) print("Reading the final Delta table...") spark.read.format("delta").load(tablePath).show() print("Create an external catalog table using Delta...") spark.sql("CREATE TABLE converted_delta_table USING delta LOCATION '{}'".format(tablePath)) print("Read from the catalog table...") spark.read.table("converted_delta_table").show() finally: # cleanup shutil.rmtree(testRoot, ignore_errors=True) ================================================ FILE: iceberg/src/main/java/org/apache/spark/sql/delta/serverSidePlanning/FixedGcsAccessTokenProvider.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.serverSidePlanning; import com.google.cloud.hadoop.util.AccessTokenProvider; import org.apache.hadoop.conf.Configuration; /** * A custom AccessTokenProvider used for server-side planning with temporary GCS credentials from * credential vending services. * * Configuration keys: * - fs.gs.auth.access.token: The OAuth2 access token * - fs.gs.auth.access.token.expiration.ms: Optional expiration timestamp in epoch milliseconds * * If no expiration is provided, defaults to 1 hour from current time. This default does not * guarantee that the token will be valid for the entire duration of the query. If the actual token expires earlier, queries will fail. */ public class FixedGcsAccessTokenProvider implements AccessTokenProvider { private static final String CONFIG_TOKEN = "fs.gs.auth.access.token"; private static final String CONFIG_EXPIRATION_MS = "fs.gs.auth.access.token.expiration.ms"; private static final long FALLBACK_EXPIRATION_MS = 3600_000L; private Configuration conf; @Override public AccessTokenProvider.AccessToken getAccessToken() { String token = conf.get(CONFIG_TOKEN); if (token == null || token.isEmpty()) { throw new RuntimeException("Missing GCS access token in configuration: " + CONFIG_TOKEN); } // Read expiration timestamp from config, or use fallback long expirationMs; String expirationStr = conf.get(CONFIG_EXPIRATION_MS); if (expirationStr != null && !expirationStr.isEmpty()) { try { expirationMs = Long.parseLong(expirationStr); } catch (NumberFormatException e) { // If parsing fails, use fallback expirationMs = System.currentTimeMillis() + FALLBACK_EXPIRATION_MS; } } else { // No expiration provided, use fallback. expirationMs = System.currentTimeMillis() + FALLBACK_EXPIRATION_MS; } return new AccessTokenProvider.AccessToken(token, expirationMs); } @Override public void refresh() { // Refresh is not supported, token is static and expires. } @Override public void setConf(Configuration conf) { this.conf = conf; } @Override public Configuration getConf() { return conf; } } ================================================ FILE: iceberg/src/main/scala/org/apache/iceberg/transforms/IcebergPartitionUtil.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package shadedForDelta.org.apache.iceberg.transforms import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.DeltaColumnMapping import org.apache.spark.sql.delta.commands.convert.TypeToSparkTypeWithCustomCast import org.apache.spark.sql.delta.sources.DeltaSourceUtils.GENERATION_EXPRESSION_METADATA_KEY import org.apache.spark.sql.delta.util.{DateFormatter, TimestampFormatter} import shadedForDelta.org.apache.iceberg.{PartitionField, PartitionSpec, Schema, StructLike} import shadedForDelta.org.apache.iceberg.types.Type.TypeID import shadedForDelta.org.apache.iceberg.types.Types import shadedForDelta.org.apache.iceberg.types.TypeUtil import org.apache.spark.sql.types.{DateType, IntegerType, MetadataBuilder, StringType, StructField} /** * Utils to translate Iceberg's partition expressions to Delta generated column expressions. */ object IcebergPartitionUtil { // scalastyle:off line.size.limit /** * Convert the partition values stored in Iceberg metadata to string values, which we will * directly use in the partitionValues field of AddFiles. Here is how we generate the string * value from the Iceberg stored partition value for each of the transforms: * * Identity * - Iceberg source code: https://github.com/apache/iceberg/blob/4c98a0f6408d4ccd0d47b076b2f7743d836d28ec/api/src/main/java/org/apache/iceberg/transforms/Identity.java * - Source column type: any * - Stored partition value type: same as source type * - String value generation: for timestamp and date, use our Spark formatter; other types use toString * * Timestamps (year, month, day, hour) * - Iceberg source code: https://github.com/apache/iceberg/blob/4c98a0f6408d4ccd0d47b076b2f7743d836d28ec/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java * - Source column type: timestamp * - Stored partition value type: integer * - String value generation: use Iceberg's Timestamps.toHumanString (which uses yyyy-MM-dd-HH format) * * Dates (year, month, day) * - Iceberg source code: https://github.com/apache/iceberg/blob/4c98a0f6408d4ccd0d47b076b2f7743d836d28ec/api/src/main/java/org/apache/iceberg/transforms/Dates.java * - Source column type: date * - Stored partition value type: integer * - String value generation: use Iceberg's Dates.toHumanString (which uses yyyy-MM-dd format) * * Truncate * - Iceberg source code: https://github.com/apache/iceberg/blob/4c98a0f6408d4ccd0d47b076b2f7743d836d28ec/api/src/main/java/org/apache/iceberg/transforms/Truncate.java * - Source column type: string, long and int * - Stored partition value type: string, long and int * - String value generation: directly use toString */ // scalastyle:on line.size.limit def partitionValueToString( partField: PartitionField, partValue: Object, schema: Schema, dateFormatter: DateFormatter, timestampFormatter: TimestampFormatter): String = { if (partValue == null) return null partField.transform() match { case _: Identity[_] => // Identity transform // We use our own date and timestamp formatter for date and timestamp types, while simply // use toString for other input types. val sourceField = schema.findField(partField.sourceId()) val sourceType = sourceField.`type`() if (sourceType.typeId() == TypeID.DATE) { // convert epoch days to Spark date formatted string dateFormatter.format(partValue.asInstanceOf[Int]) } else if (sourceType.typeId == TypeID.TIMESTAMP) { // convert timestamps to Spark timestamp formatted string timestampFormatter.format(partValue.asInstanceOf[Long]) } else { // all other types can directly toString partValue.toString } case ts: Timestamps => // Matches all transforms on Timestamp input type: YEAR, MONTH, DAY, HOUR // We directly use Iceberg's toHumanString(), which takes a timestamp type source column and // generates the partition value in the string format as follows: // - YEAR: yyyy // - MONTH: yyyy-MM // - DAY: yyyy-MM-dd // - HOUR: yyyy-MM-dd-HH ts.toHumanString(Types.TimestampType.withoutZone(), partValue.asInstanceOf[Int]) case dt: Dates => // Matches all transform on Date input type: YEAR, MONTH, DAY // We directly use Iceberg's toHumanString(), which takes a date type source column and // generates the partition value in the string format as follows: // - YEAR: yyyy // - MONTH: yyyy-MM // - DAY: yyyy-MM-dd dt.toHumanString(Types.DateType.get(), partValue.asInstanceOf[Int]) case _: Truncate[_] => // Truncate transform // While Iceberg Truncate transform supports multiple input types, our converter // only supports string and block all other input types. So simply toString suffices. partValue.toString case other => throw new UnsupportedOperationException( s"unsupported partition transform expression when converting to Delta: $other") } } def getPartitionFields( partSpec: PartitionSpec, schema: Schema, castTimeType: Boolean): Seq[StructField] = { // Skip removed partition fields due to partition evolution. partSpec.fields.asScala.toSeq.collect { case partField if !partField.transform().isInstanceOf[VoidTransform[_]] && !partField.transform().isInstanceOf[Bucket[_]] => val sourceColumnName = schema.findColumnName(partField.sourceId()) val sourceField = schema.findField(partField.sourceId()) val sourceType = sourceField.`type`() val metadataBuilder = new MetadataBuilder() // TODO: Support truncate[Decimal] in partition val (transformExpr, targetType) = partField.transform() match { // binary partition values are problematic in Delta, so we block converting if the iceberg // table has a binary type partition column case _: Identity[_] if sourceType.typeId() != TypeID.BINARY => // copy id only for identity transform because source id will be the converted column id // ids for other columns will be assigned later automatically during schema evolution metadataBuilder .putLong(DeltaColumnMapping.COLUMN_MAPPING_METADATA_ID_KEY, sourceField.fieldId()) ("", TypeUtil.visit(sourceType, new TypeToSparkTypeWithCustomCast(castTimeType))) case Timestamps.MICROS_TO_YEAR | Dates.YEAR => (s"year($sourceColumnName)", IntegerType) case Timestamps.MICROS_TO_DAY | Dates.DAY => (s"cast($sourceColumnName as date)", DateType) case t: Truncate[_] if sourceType.typeId() == TypeID.STRING => (s"substring($sourceColumnName, 0, ${t.width()})", StringType) case t: Truncate[_] if sourceType.typeId() == TypeID.LONG || sourceType.typeId() == TypeID.INTEGER => (icebergNumericTruncateExpression(sourceColumnName, t.width().toLong), TypeUtil.visit(sourceType, new TypeToSparkTypeWithCustomCast(castTimeType))) case Timestamps.MICROS_TO_MONTH | Dates.MONTH => (s"date_format($sourceColumnName, 'yyyy-MM')", StringType) case Timestamps.MICROS_TO_HOUR => (s"date_format($sourceColumnName, 'yyyy-MM-dd-HH')", StringType) case other => throw new UnsupportedOperationException( s"Unsupported partition transform expression when converting to Delta: " + s"transform: $other, source data type: ${sourceType.typeId()}") } if (transformExpr != "") { metadataBuilder.putString(GENERATION_EXPRESSION_METADATA_KEY, transformExpr) } Option(sourceField.doc()).foreach { comment => metadataBuilder.putString("comment", comment) } val metadata = metadataBuilder.build() StructField(partField.name(), targetType, nullable = sourceField.isOptional(), metadata = metadata) } } /** * Returns the iceberg transform function of truncate[Integer] and truncate[Long] as an * expression string, please check the iceberg documents for more details: * * https://iceberg.apache.org/spec/#truncate-transform-details * * TODO: make this partition expression optimizable. */ private def icebergNumericTruncateExpression(colName: String, width: Long): String = s"$colName - (($colName % $width) + $width) % $width" def hasBucketPartition(partSpec: PartitionSpec): Boolean = { partSpec.fields.asScala.toSeq.exists(spec => spec.transform().isInstanceOf[Bucket[_]]) } // return true if the partition spec has a partition that is not a bucket partition def hasNonBucketPartition(partSpec: PartitionSpec): Boolean = { partSpec.isPartitioned && partSpec.fields().asScala.exists { field => !field.transform().isInstanceOf[Bucket[_]] } } } ================================================ FILE: iceberg/src/main/scala/org/apache/spark/sql/catalyst/analysis/NoSuchProcedureException.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.analysis import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.connector.catalog.Identifier class NoSuchProcedureException(ident: Identifier) extends AnalysisException("Procedure " + ident + " not found") ================================================ FILE: iceberg/src/main/scala/org/apache/spark/sql/delta/IcebergFileManifest.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.convert import scala.collection.JavaConverters._ import scala.collection.mutable import org.apache.spark.sql.delta.{DeltaColumnMapping, DeltaLog, SerializableFileStatus, Snapshot => DeltaSnapshot} import org.apache.spark.sql.delta.DeltaErrors.cloneFromIcebergSourceWithPartitionEvolution import org.apache.spark.sql.delta.commands.convert.IcebergTable.ERR_MULTIPLE_PARTITION_SPECS import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.hadoop.fs.Path import shadedForDelta.org.apache.iceberg.{BaseTable, DataFile, DataFiles, DeleteFile, FileContent, FileFormat, ManifestContent, ManifestFile, ManifestFiles, PartitionData, PartitionSpec, RowLevelOperationMode, Schema, StructLike, Table, TableProperties} import shadedForDelta.org.apache.iceberg.transforms.IcebergPartitionUtil import shadedForDelta.org.apache.iceberg.types.Type.TypeID import org.apache.spark.SparkThrowable import org.apache.spark.sql.{Dataset, SparkSession} import org.apache.spark.sql.types.StructType class IcebergFileManifest( spark: SparkSession, table: IcebergTableLike, partitionSchema: StructType, convertStats: Boolean = true) extends ConvertTargetFileManifest { // scalastyle:off sparkimplicits import spark.implicits._ // scalastyle:on sparkimplicits private var fileSparkResults: Option[Dataset[ConvertTargetFile]] = None private var _numFiles: Option[Long] = None private var _sizeInBytes: Option[Long] = None private val specIdsToIfSpecHasNonBucketPartition = table.specs().asScala.map { case (specId, spec) => specId.toInt -> IcebergPartitionUtil.hasNonBucketPartition(spec) } private val partitionEvolutionEnabled = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_CONVERT_ICEBERG_PARTITION_EVOLUTION_ENABLED) private val statsAllowTypes: Set[TypeID] = IcebergStatsUtils.typesAllowStatsConversion(spark) private val allowPartialStatsConverted: Boolean = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_CLONE_ICEBERG_ALLOW_PARTIAL_STATS ) val basePath = table.location() override def numFiles: Long = { if (_numFiles.isEmpty) _numFiles = Some(allFiles.count()) _numFiles.get } override def sizeInBytes: Long = { if (_sizeInBytes.isEmpty) { _sizeInBytes = Some(if (allFiles.isEmpty) 0L else allFiles.map(_.fileStatus.length).reduce(_ + _)) } _sizeInBytes.get } def allFiles: Dataset[ConvertTargetFile] = { if (fileSparkResults.isEmpty) fileSparkResults = Some(getFileSparkResults()) fileSparkResults.get } private def getFileSparkResults(): Dataset[ConvertTargetFile] = { val format = table .properties() .getOrDefault( TableProperties.DEFAULT_FILE_FORMAT, TableProperties.DEFAULT_FILE_FORMAT_DEFAULT) if (format.toLowerCase() != "parquet") { throw new UnsupportedOperationException( s"Cannot convert Iceberg tables with file format $format. Only parquet is supported.") } if (table.currentSnapshot() == null) { return spark.emptyDataset[ConvertTargetFile] } val hasMergeOnReadDeletionFiles = table.currentSnapshot().deleteManifests(table.io()).size() > 0 if (hasMergeOnReadDeletionFiles) { throw new UnsupportedOperationException( s"Cannot support convert Iceberg table with row-level deletes." + s"Please trigger an Iceberg compaction and retry the command.") } // Localize variables so we don't need to serialize the File Manifest class // Some contexts: Spark needs all variables in closure to be serializable // while class members carry the entire class, so they require serialization of the class // As IcebergFileManifest is not serializable, // we localize member variables to avoid serialization of the class val localTable = table // We use the latest snapshot timestamp for all generated Delta AddFiles due to the fact that // retrieving timestamp for each DataFile is non-trivial time-consuming. This can be improved // in the future. val snapshotTs = table.currentSnapshot().timestampMillis val shouldConvertPartition = spark.sessionState.conf .getConf(DeltaSQLConf.DELTA_CONVERT_ICEBERG_USE_NATIVE_PARTITION_VALUES) val convertPartition = if (shouldConvertPartition) { new IcebergPartitionConverter(localTable, partitionSchema, partitionEvolutionEnabled) } else { null } val shouldConvertStats = convertStats val partialStatsConvertedEnabled = allowPartialStatsConverted val statsAllowTypesSet = statsAllowTypes val shouldCheckPartitionEvolution = !partitionEvolutionEnabled val specIdsToIfSpecHasNonBucketPartitionMap = specIdsToIfSpecHasNonBucketPartition val tableSpecsSize = table.specs().size() val dataFiles = loadIcebergFiles() dataFiles.map { dataFile: DataFileWrapper => if (shouldCheckPartitionEvolution) { IcebergFileManifest.validateLimitedPartitionEvolution( dataFile.specId, tableSpecsSize, specIdsToIfSpecHasNonBucketPartitionMap ) } ConvertTargetFile( SerializableFileStatus( path = dataFile.path, length = dataFile.fileSizeInBytes, isDir = false, modificationTime = snapshotTs ), partitionValues = if (shouldConvertPartition) { Some(convertPartition.toDelta(dataFile.partition())) } else None, stats = if (shouldConvertStats) { IcebergStatsUtils.icebergStatsToDelta( localTable.schema, dataFile, statsAllowTypesSet, shouldSkipForFile = (df: DataFile) => { !partialStatsConvertedEnabled && IcebergStatsUtils.hasPartialStats(df) } ) } else None ) } .cache() } private def loadIcebergFiles(): ( Dataset[DataFileWrapper] ) = { val localTable = table val manifestFiles = localTable .currentSnapshot() .dataManifests(localTable.io()) .asScala .map(new ManifestFileWrapper(_)) .toSeq val dataFiles = spark .createDataset(manifestFiles) .flatMap( ManifestFiles.read(_, localTable.io(), localTable.specs()) .asScala.map(new DataFileWrapper(_) ) ) dataFiles } override def close(): Unit = { fileSparkResults.map(_.unpersist()) fileSparkResults = None } } object IcebergFileManifest { // scalastyle:off /** * Validates on partition evolution for proposed partitionSpecId * We don't support the conversion of tables with partition evolution * * However, we allow one special case where * all data files have either no-partition or bucket-partition * regardless of multiple partition spec present in the table */ // scalastyle:on private def validateLimitedPartitionEvolution( partitionSpecId: Int, tableSpecsSize: Int, specIdsToIfSpecHasNonBucketPartition: mutable.Map[Int, Boolean]): Unit = { if (hasPartitionEvolved( partitionSpecId, tableSpecsSize, specIdsToIfSpecHasNonBucketPartition) ) { throw cloneFromIcebergSourceWithPartitionEvolution() } } private def hasPartitionEvolved( partitionSpecID: Int, tableSpecsSize: Int, specIdsToIfSpecHasNonBucketPartition: mutable.Map[Int, Boolean]): Boolean = { val isSpecPartitioned = specIdsToIfSpecHasNonBucketPartition(partitionSpecID) isSpecPartitioned && tableSpecsSize > 1 } } ================================================ FILE: iceberg/src/main/scala/org/apache/spark/sql/delta/IcebergPartitionConverter.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.convert import java.lang.{Integer => JInt, Long => JLong} import java.nio.ByteBuffer import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.DeltaColumnMapping import org.apache.spark.sql.delta.util.{DateFormatter, TimestampFormatter} import shadedForDelta.org.apache.iceberg.{PartitionData, PartitionField, PartitionSpec, Schema, StructLike, Table} import shadedForDelta.org.apache.iceberg.transforms.IcebergPartitionUtil import shadedForDelta.org.apache.iceberg.types.{Conversions, Type => IcebergType} import shadedForDelta.org.apache.iceberg.types.Type.{PrimitiveType => IcebergPrimitiveType, TypeID} import shadedForDelta.org.apache.iceberg.types.Types.{ ListType => IcebergListType, MapType => IcebergMapType, NestedField, StringType => IcebergStringType, StructType => IcebergStructType } import org.apache.spark.sql.types.StructType object IcebergPartitionConverter { // we must use field id to look up the partition value; consider scenario with iceberg // behavior chance since 1.4.0: // 1) create table with partition schema (a[col_name]: 1[field_id]), add file1; // The partition data for file1 is (a:1:some_part_value) // 2) add new partition col b and the partition schema becomes (a: 1, b: 2), add file2; // the partition data for file2 is (a:1:some_part_value, b:2:some_part_value) // 3) remove partition col a, then add file3; // for iceberg < 1.4.0: the partFields is (a:1(void), b:2); the partition data for // file3 is (a:1(void):null, b:2:some_part_value); // for iceberg 1.4.0: the partFields is (b:2); When it reads file1 (a:1:some_part_value), // it must use the field_id instead of index to look up the partition // value, as the partField and partitionData from file1 have different // ordering and thus same index indicates different column. def physicalNameToPartitionField( table: IcebergTableLike, partitionSchema: StructType): Map[String, PartitionField] = table.spec().fields().asScala.collect { case field if field.transform().toString != "void" && !field.transform().toString.contains("bucket") => DeltaColumnMapping.getPhysicalName(partitionSchema(field.name)) -> field }.toMap } case class IcebergPartitionConverter( icebergSchema: Schema, physicalNameToPartitionField: Map[String, PartitionField]) { val dateFormatter: DateFormatter = DateFormatter() val timestampFormatter: TimestampFormatter = TimestampFormatter(ConvertUtils.timestampPartitionPattern, java.util.TimeZone.getDefault) def this( table: IcebergTableLike, partitionSchema: StructType, partitionEvolutionEnabled: Boolean) = this(table.schema(), // We only allow empty partition when partition evolution happened // This is an extra safety mechanism as we should have already passed // a non-bucket partitionSchema when table has >1 specs if (table.specs().size() > 1 && !partitionEvolutionEnabled) { Map.empty[String, PartitionField] } else { IcebergPartitionConverter.physicalNameToPartitionField(table, partitionSchema) } ) /** * Convert an Iceberg [[PartitionData]] into a Map of (columnID -> partitionValue) used by Delta */ def toDelta(partition: StructLike): Map[String, String] = { val icebergPartitionData = partition.asInstanceOf[PartitionData] val fieldIdToIdx = icebergPartitionData.getPartitionType .fields() .asScala .zipWithIndex .map(kv => kv._1.fieldId() -> kv._2) .toMap val physicalNameToPartValueMap = physicalNameToPartitionField .map { case (physicalName, field) => val fieldIndex = fieldIdToIdx.get(field.fieldId()) val partValueAsString = fieldIndex .map { idx => val partValue = icebergPartitionData.get(idx) IcebergPartitionUtil.partitionValueToString( field, partValue, icebergSchema, dateFormatter, timestampFormatter ) } .getOrElse(null) physicalName -> partValueAsString } physicalNameToPartValueMap } } ================================================ FILE: iceberg/src/main/scala/org/apache/spark/sql/delta/IcebergSchemaUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.convert import org.apache.spark.sql.delta.DeltaColumnMapping import org.apache.spark.sql.delta.schema.SchemaMergingUtils import shadedForDelta.org.apache.iceberg.Schema import shadedForDelta.org.apache.iceberg.types.TypeUtil import org.apache.spark.sql.types.{MetadataBuilder, StructType} object IcebergSchemaUtils { /** * Given an iceberg schema, convert it to a Spark schema. This conversion will keep the Iceberg * column IDs (used to read Parquet files) in the field metadata * * @param icebergSchema Iceberg schema * @param castTimeType cast Iceberg TIME type to Spark Long * @return Spark schema converted from Iceberg schema */ def convertIcebergSchemaToSpark(icebergSchema: Schema, castTimeType: Boolean = false): StructType = { // Convert from Iceberg schema to Spark schema but without the column IDs val baseConvertedSchema = TypeUtil.visit( icebergSchema, new TypeToSparkTypeWithCustomCast(castTimeType) ).asInstanceOf[StructType] // For each field, find the column ID (fieldId) and add to the StructField metadata SchemaMergingUtils.transformColumns(baseConvertedSchema) { (path, field, _) => // This should be safe to access fields // scalastyle:off // https://github.com/apache/iceberg/blob/d98224a82b104888281d4e901ccf948f9642590b/api/src/main/java/org/apache/iceberg/types/IndexByName.java#L171 // scalastyle:on val fieldPath = (path :+ field.name).mkString(".") val id = icebergSchema.findField(fieldPath).fieldId() field.copy( metadata = new MetadataBuilder() .withMetadata(field.metadata) .putLong(DeltaColumnMapping.COLUMN_MAPPING_METADATA_ID_KEY, id) .build()) } } } ================================================ FILE: iceberg/src/main/scala/org/apache/spark/sql/delta/IcebergSparkWrappers.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.convert import java.io.{ByteArrayInputStream, ByteArrayOutputStream, ObjectInputStream, ObjectOutputStream} import java.lang.{Integer => JInt, Long => JLong} import java.nio.ByteBuffer import java.util.{List => JList, Map => JMap} import java.util.stream.Collectors import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor import shadedForDelta.org.apache.iceberg.{DataFile, DeleteFile, FileContent, FileFormat, ManifestContent, ManifestFile, PartitionData, StructLike} import shadedForDelta.org.apache.iceberg.ManifestFile.PartitionFieldSummary /** * The classes in this file are wrappers of Iceberg classes * in the format of case classes so they can be serialized by * Spark automatically. */ case class ManifestFileWrapper( path: String, length: Long, partitionSpecId: Int, content: ManifestContent, sequenceNumber: Long, minSequenceNumber: Long, snapshotId: JLong, addedFilesCount: JInt, addedRowsCount: JLong, existingFilesCount: JInt, existingRowsCount: JLong, deletedFilesCount: JInt, deletedRowsCount: JLong, _partitions: Option[Seq[PartitionFieldSummaryWrapper]]) extends ManifestFile { def this(manifest: ManifestFile) = this( manifest.path, manifest.length, manifest.partitionSpecId, manifest.content(), manifest.sequenceNumber, manifest.minSequenceNumber, manifest.snapshotId, manifest.addedFilesCount, manifest.addedRowsCount, manifest.existingFilesCount, manifest.existingRowsCount, manifest.deletedFilesCount, manifest.deletedRowsCount, Option(manifest.partitions).map(_.asScala.map(new PartitionFieldSummaryWrapper(_)).toSeq) ) override def partitions: JList[PartitionFieldSummary] = _partitions.map(_.asJava.asInstanceOf[JList[PartitionFieldSummary]]).orNull override def copy: ManifestFile = this.copy } case class PartitionFieldSummaryWrapper( containsNull: Boolean, _lowerBound: Option[Array[Byte]], _upperBound: Option[Array[Byte]]) extends PartitionFieldSummary { def this(src: PartitionFieldSummary) = this( src.containsNull, IcebergSparkWrappers.serializeByteBuffer(src.lowerBound), IcebergSparkWrappers.serializeByteBuffer(src.upperBound) ) override def lowerBound: ByteBuffer = IcebergSparkWrappers.deserializeByteBuffer(_lowerBound) override def upperBound: ByteBuffer = IcebergSparkWrappers.deserializeByteBuffer(_upperBound) override def copy: PartitionFieldSummary = this.copy } case class DataFileWrapper( pos: JLong, specId: Int, path: String, recordCount: Long, fileSizeInBytes: Long, _partition: Array[Byte], _columnSizes: Option[Map[JInt, JLong]], _valueCounts: Option[Map[JInt, JLong]], _nullValueCounts: Option[Map[JInt, JLong]], _nanValueCounts: Option[Map[JInt, JLong]], _lowerBounds: Option[Map[JInt, Option[Array[Byte]]]], _upperBounds: Option[Map[JInt, Option[Array[Byte]]]], _keyMetadata: Option[Array[Byte]], _splitOffsets: Option[Seq[JLong]]) extends DataFile { def this(df: DataFile) = { this( df.pos, df.specId, df.path.toString, df.recordCount, df.fileSizeInBytes, IcebergSparkWrappers.serialize(df.partition.asInstanceOf[java.io.Serializable]), Option(df.columnSizes).map(_.asScala.toMap), Option(df.valueCounts).map(_.asScala.toMap), Option(df.nullValueCounts).map(_.asScala.toMap), Option(df.nanValueCounts).map(_.asScala.toMap), IcebergSparkWrappers.serializeMap(df.lowerBounds), IcebergSparkWrappers.serializeMap(df.upperBounds), IcebergSparkWrappers.serializeByteBuffer(df.keyMetadata), Option(df.splitOffsets).map(_.asScala.toSeq)) require(df.content == FileContent.DATA) require(df.format == FileFormat.PARQUET) } override def content: FileContent = FileContent.DATA override def format: FileFormat = FileFormat.PARQUET override def partition: StructLike = IcebergSparkWrappers.deserialize[PartitionData](this._partition) override def columnSizes: JMap[JInt, JLong] = _columnSizes.map(_.asJava).orNull override def valueCounts: JMap[JInt, JLong] = _valueCounts.map(_.asJava).orNull override def nullValueCounts: JMap[JInt, JLong] = _nullValueCounts.map(_.asJava).orNull override def nanValueCounts: JMap[JInt, JLong] = _nanValueCounts.map(_.asJava).orNull override def lowerBounds: JMap[JInt, ByteBuffer] = IcebergSparkWrappers.deserializeMap(_lowerBounds) override def upperBounds: JMap[JInt, ByteBuffer] = IcebergSparkWrappers.deserializeMap(_upperBounds) override def keyMetadata: ByteBuffer = IcebergSparkWrappers.deserializeByteBuffer(_keyMetadata) override def splitOffsets: JList[JLong] = _splitOffsets.map(_.asJava).orNull override def copy: DataFile = this.copy override def copyWithoutStats: DataFile = this.copy } object IcebergSparkWrappers { def serialize(obj: java.io.Serializable): Array[Byte] = { val baos = new ByteArrayOutputStream() val oos = new ObjectOutputStream(baos) try { oos.writeObject(obj) oos.flush() baos.toByteArray } finally { oos.close() baos.close() } } def deserialize[T](bytes: Array[Byte]): T = { val bais = new ByteArrayInputStream(bytes) val ois = new ObjectInputStream(bais) try { ois.readObject().asInstanceOf[T] // Cast to the expected type } finally { ois.close() bais.close() } } def serializeByteBuffer(byteBuffer: ByteBuffer): Option[Array[Byte]] = { Option(byteBuffer).map(_.array()) } def deserializeByteBuffer(byteBufferWrapper: Option[Array[Byte]]): ByteBuffer = { byteBufferWrapper.map(ByteBuffer.wrap).orNull } def serializeMap[T](map: JMap[T, ByteBuffer]): Option[Map[T, Option[Array[Byte]]]] = Option(map).map(_.asScala.mapValues(serializeByteBuffer).toMap) def deserializeMap[T](map: Option[Map[T, Option[Array[Byte]]]]): JMap[T, ByteBuffer] = map.map(_.mapValues(deserializeByteBuffer).toMap.asJava).orNull } ================================================ FILE: iceberg/src/main/scala/org/apache/spark/sql/delta/IcebergStatsUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.convert import java.lang.{Integer => JInt, Long => JLong} import java.nio.ByteBuffer import java.util.{Map => JMap} import scala.collection.JavaConverters._ import scala.util.control.NonFatal import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.DeltaStatistics._ import org.apache.spark.sql.delta.util.JsonUtils import shadedForDelta.org.apache.iceberg.{DataFile, PartitionData, PartitionField, Schema, StructLike, Table} import shadedForDelta.org.apache.iceberg.types.{Conversions, Type => IcebergType} import shadedForDelta.org.apache.iceberg.types.Type.{PrimitiveType => IcebergPrimitiveType, TypeID} import shadedForDelta.org.apache.iceberg.types.Types.{ DateType => IcebergDateType, ListType => IcebergListType, MapType => IcebergMapType, NestedField, StringType => IcebergStringType, StructType => IcebergStructType, TimestampType => IcebergTimestampType } import shadedForDelta.org.apache.iceberg.util.DateTimeUtil import org.apache.spark.sql.SparkSession object IcebergStatsUtils extends DeltaLogging { // Types that are currently supported for converting stats to delta // The stats for these types will be converted to Delta stats // except for following types: // DECIMAL (decided by DeltaSQLConf.DELTA_CONVERT_ICEBERG_DECIMAL_STATS) // DATE (decided by DeltaSQLConf.DELTA_CONVERT_ICEBERG_DATE_STATS) // TIMESTAMP (decided by DeltaSQLConf.DELTA_CONVERT_ICEBERG_TIMESTAMP_STATS) // which are decided by spark configs dynamically private val STATS_ALLOW_TYPES = Set[TypeID]( TypeID.BOOLEAN, TypeID.INTEGER, TypeID.LONG, TypeID.FLOAT, TypeID.DOUBLE, TypeID.DATE, // TypeID.TIME, TypeID.TIMESTAMP, // TypeID.TIMESTAMP_NANO, TypeID.STRING, // TypeID.UUID, // TypeID.FIXED, TypeID.BINARY, TypeID.DECIMAL ) private val STATS_NULLCOUNT_ALLOW_TYPES = Set[TypeID]( TypeID.LIST, TypeID.MAP, TypeID.DATE, TypeID.TIMESTAMP, TypeID.DECIMAL ) private val CONFIGS_TO_STATS_ALLOW_TYPES = Map( DeltaSQLConf.DELTA_CONVERT_ICEBERG_DATE_STATS -> TypeID.DATE, DeltaSQLConf.DELTA_CONVERT_ICEBERG_TIMESTAMP_STATS -> TypeID.TIMESTAMP, DeltaSQLConf.DELTA_CONVERT_ICEBERG_DECIMAL_STATS -> TypeID.DECIMAL ) def typesAllowStatsConversion(spark: SparkSession): Set[TypeID] = { val statsDisallowTypes = CONFIGS_TO_STATS_ALLOW_TYPES.filter { case (conf, _) => !spark.sessionState.conf.getConf(conf) }.values.toSet typesAllowStatsConversion(statsDisallowTypes) } def typesAllowStatsConversion(statsDisallowTypes: Set[TypeID]): Set[TypeID] = { STATS_ALLOW_TYPES -- statsDisallowTypes } /** * Convert Iceberg DataFile stats into a Json string containing Delta stats. * We will abandon conversion if Iceberg DataFile has a null or empty stats for * any criteria used in the conversion. * * @param icebergSchema Iceberg table schema * @param dataFile Iceberg DataFile that contains stats info * @param statsAllowTypes Iceberg types that are allowed to convert stats * @param shouldSkipForFile Function => true if a data file should be skipped * @return None if stats is missing on the DataFile or error occurs during conversion */ def icebergStatsToDelta( icebergSchema: Schema, dataFile: DataFile, statsAllowTypes: Set[TypeID], shouldSkipForFile: DataFile => Boolean ): Option[String] = { if (shouldSkipForFile(dataFile)) { return None } try { Some(icebergStatsToDelta( icebergSchema, dataFile.recordCount, Option(dataFile.upperBounds).map(_.asScala.toMap).filter(_.nonEmpty), Option(dataFile.lowerBounds).map(_.asScala.toMap).filter(_.nonEmpty), Option(dataFile.nullValueCounts).map(_.asScala.toMap).filter(_.nonEmpty), statsAllowTypes )) } catch { case NonFatal(e) => logInfo("[Iceberg-Stats-Conversion] " + "Exception while converting Iceberg stats to Delta format", e) None } } def hasPartialStats(dataFile: DataFile): Boolean = { def nonEmptyMap[K, V](m: JMap[K, V]): Boolean = { m != null && !m.isEmpty } // nullValueCounts is less common, so we ignore it val hasPartialStats = !nonEmptyMap(dataFile.upperBounds()) || !nonEmptyMap(dataFile.lowerBounds()) if (hasPartialStats) { logInfo(s"[Iceberg-Stats-Conversion] $dataFile only has partial stats:" + s"upperBounds=${dataFile.upperBounds}, lowerBounds = ${dataFile.lowerBounds()}") } hasPartialStats } /** * Convert Iceberg DataFile stats into Delta stats. * * Iceberg stats consist of multiple maps from field_id to value. The maps include * max_value, min_value and null_counts. * Delta stats is a Json string. * ********************************************************** * Example: ********************************************************** * Assume we have an Iceberg table of schema * ( col1: int, field_id = 1, col2: string, field_id = 2 ) * * The following Iceberg stats: * numRecords 100 * max_value { 1 -> 200, 2 -> "max value" } * min_value { 1 -> 10, 2 -> "min value" } * null_counts { 1 -> 0, 2 -> 20 } * will be converted into the following Delta style stats as a Json str * * { * numRecords: 100, * maxValues: { * "col1": 200, * "col2" "max value" * }, * minValues: { * "col1": 10, * "col2": "min value" * }, * nullCount: { * "col1": 0, * "col2": 20 * } * } ********************************************************** * * See also [[org.apache.spark.sql.delta.stats.StatsCollectionUtils]] for more * about Delta stats. * * @param icebergSchema Iceberg table schema * @param numRecords Iceberg stats of numRecords * @param maxMap Iceberg stats of max value ( field_id -> value ) * @param minMap Iceberg stats of min value ( field_id -> value ) * @param nullCountMap Iceberg stats of null count ( field_id -> value ) * @param statsAllowTypes dataTypes that will convert stats to Delta * @return json string representing Delta stats */ private[convert] def icebergStatsToDelta( icebergSchema: Schema, numRecords: Long, maxMap: Option[Map[JInt, ByteBuffer]], minMap: Option[Map[JInt, ByteBuffer]], nullCountMap: Option[Map[JInt, JLong]], statsAllowTypes: Set[TypeID] ): String = { def deserialize(ftype: IcebergType, value: Any): Any = { (ftype, value) match { case (_, null) => null case (_: IcebergStringType, bb: ByteBuffer) => Conversions.fromByteBuffer(ftype, bb).toString case (_: IcebergDateType, bb: ByteBuffer) => val daysFromEpoch = Conversions.fromByteBuffer(ftype, bb).asInstanceOf[Int] DateTimeUtil.dateFromDays(daysFromEpoch).toString case (tsType: IcebergTimestampType, bb: ByteBuffer) => val microts = Conversions.fromByteBuffer(tsType, bb).asInstanceOf[JLong] microTimestampToString(microts, tsType) case (_, bb: ByteBuffer) => Conversions.fromByteBuffer(ftype, bb) case _ => throw new IllegalArgumentException("unable to deserialize unknown values") } } // Recursively collect stats from the given fields list and values and // use the given deserializer to format the value. // The result is a map of ( delta column physical name -> value ) def collectStats( fields: java.util.List[NestedField], valueMap: Map[JInt, Any], deserializer: (IcebergType, Any) => Any, statsAllowTypes: Set[TypeID]): Map[String, Any] = { fields.asScala.flatMap { field => field.`type`() match { case st: IcebergStructType => Some(field.name -> collectStats(st.fields, valueMap, deserializer, statsAllowTypes)) case pt: IcebergPrimitiveType if valueMap.contains(field.fieldId) && statsAllowTypes.contains(pt.typeId) => Option(deserializer(pt, valueMap(field.fieldId))).map(field.name -> _) case pt: IcebergListType if valueMap.contains(field.fieldId) && statsAllowTypes.contains(pt.typeId) => Option(deserializer(pt, valueMap(field.fieldId))).map(field.name -> _) case pt: IcebergMapType if valueMap.contains(field.fieldId) && statsAllowTypes.contains(pt.typeId) => Option(deserializer(pt, valueMap(field.fieldId))).map(field.name -> _) case _ => None } }.toMap } JsonUtils.toJson( Map( NUM_RECORDS -> numRecords ) ++ maxMap.map( MAX -> collectStats(icebergSchema.columns, _, deserialize, statsAllowTypes) ) ++ minMap.map( MIN -> collectStats(icebergSchema.columns, _, deserialize, statsAllowTypes) ) ++ nullCountMap.map( NULL_COUNT -> collectStats( icebergSchema.columns, _, (_: IcebergType, v: Any) => v, statsAllowTypes ++ STATS_NULLCOUNT_ALLOW_TYPES ) ) ) } private def microTimestampToString( microTS: JLong, tsType: IcebergTimestampType): String = { // iceberg timestamptz will have shouldAdjustToUTC() as true if (tsType.shouldAdjustToUTC()) { DateTimeUtil.microsToIsoTimestamptz(microTS) } else { // iceberg timestamp doesn't need to adjust to UTC DateTimeUtil.microsToIsoTimestamp(microTS) } } } ================================================ FILE: iceberg/src/main/scala/org/apache/spark/sql/delta/IcebergTable.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.convert import java.util.Locale import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.{DeltaColumnMapping, DeltaColumnMappingMode, DeltaConfigs, IdMapping, SerializableFileStatus, Snapshot} import org.apache.spark.sql.delta.DeltaErrors.{cloneFromIcebergSourceWithoutSpecs, cloneFromIcebergSourceWithPartitionEvolution} import org.apache.spark.sql.delta.schema.SchemaMergingUtils import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import shadedForDelta.org.apache.iceberg.{PartitionSpec, Schema, Snapshot => IcebergSnapshot, Table, TableProperties} import shadedForDelta.org.apache.iceberg.hadoop.HadoopTables import shadedForDelta.org.apache.iceberg.io.FileIO import shadedForDelta.org.apache.iceberg.transforms.{Bucket, IcebergPartitionUtil} import shadedForDelta.org.apache.iceberg.util.PropertyUtil import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.spark.sql.execution.datasources.PartitioningUtils import org.apache.spark.sql.types.StructType /** Subset of [[Table]] functionality required for conversion to Delta. */ trait IcebergTableLike { def location(): String def schema(): Schema def properties(): java.util.Map[String, String] def specs(): java.util.Map[Integer, PartitionSpec] def spec(): PartitionSpec def currentSnapshot(): IcebergSnapshot def snapshot(id: Long): IcebergSnapshot def io(): FileIO } /** * Implementation of [[IcebergTableLike]] that can safely rely on the functionality of an * underlying [[Table]]. */ case class DelegatingIcebergTable(table: Table) extends IcebergTableLike { override def location(): String = table.location() override def schema(): Schema = table.schema() override def properties(): java.util.Map[String, String] = table.properties() override def specs(): java.util.Map[Integer, PartitionSpec] = table.specs() override def spec(): PartitionSpec = table.spec() override def currentSnapshot(): IcebergSnapshot = table.currentSnapshot() override def snapshot(id: Long): IcebergSnapshot = table.snapshot(id) override def io(): FileIO = table.io() } /** * A target Iceberg table for conversion to a Delta table. * * @param icebergTable the Iceberg table underneath. * @param deltaSnapshot the delta snapshot used for incremental update, none for initial conversion. * @param convertStats flag for disabling convert iceberg stats directly into Delta stats. * If you wonder why we need this flag, you are not alone. * This flag is only used by the old, obsolete, legacy command * `CONVERT TO DELTA NO STATISTICS`. * We believe that back then the CONVERT command suffered performance * problem due to stats collection and design `NO STATISTICS` as a workaround. * Now we are able to generate stats much faster, but when this flag is true, * we still have to honor it and give up generating stats. What a pity! */ class IcebergTable( spark: SparkSession, icebergTable: IcebergTableLike, deltaSnapshot: Option[Snapshot], convertStats: Boolean) extends ConvertTargetTable { def this(spark: SparkSession, basePath: String, deltaTable: Option[Snapshot], convertStats: Boolean = true) = // scalastyle:off deltahadoopconfiguration this( spark, DelegatingIcebergTable(new HadoopTables(spark.sessionState.newHadoopConf).load(basePath)), deltaTable, convertStats) // scalastyle:on deltahadoopconfiguration protected val existingSchema: Option[StructType] = deltaSnapshot.map(_.schema) private val partitionEvolutionEnabled = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_CONVERT_ICEBERG_PARTITION_EVOLUTION_ENABLED) private val bucketPartitionEnabled = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_CONVERT_ICEBERG_BUCKET_PARTITION_ENABLED) || deltaSnapshot.exists(s => DeltaConfigs.IGNORE_ICEBERG_BUCKET_PARTITION.fromMetaData(s.metadata) ) // When a table is CLONED/federated with the session conf ON, it will have the table property // set and will continue to support CAST TIME TYPE even when later the session conf is OFF. private val castTimeType = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_CONVERT_ICEBERG_CAST_TIME_TYPE) || deltaSnapshot.exists(s => DeltaConfigs.CAST_ICEBERG_TIME_TYPE.fromMetaData(s.metadata)) protected val fieldPathToPhysicalName: Map[Seq[String], String] = existingSchema.map { SchemaMergingUtils.explode(_).collect { case (path, field) if DeltaColumnMapping.hasPhysicalName(field) => path.map(_.toLowerCase(Locale.ROOT)) -> DeltaColumnMapping.getPhysicalName(field) }.toMap }.getOrElse(Map.empty[Seq[String], String]) private val convertedSchema = { // Reuse physical names of existing columns. val mergedSchema = DeltaColumnMapping.setPhysicalNames( IcebergSchemaUtils.convertIcebergSchemaToSpark(icebergTable.schema(), castTimeType), fieldPathToPhysicalName) // Assign physical names to new columns. DeltaColumnMapping.assignPhysicalNames(mergedSchema, reuseLogicalName = true) } override val requiredColumnMappingMode: DeltaColumnMappingMode = IdMapping override val properties: Map[String, String] = { val maxSnapshotAgeMs = PropertyUtil.propertyAsLong(icebergTable.properties, TableProperties.MAX_SNAPSHOT_AGE_MS, TableProperties.MAX_SNAPSHOT_AGE_MS_DEFAULT) val castTimeTypeConf = if (castTimeType) { Some((DeltaConfigs.CAST_ICEBERG_TIME_TYPE.key -> "true")) } else { None } val bucketPartitionToNonPartition = if (bucketPartitionEnabled) { Some((DeltaConfigs.IGNORE_ICEBERG_BUCKET_PARTITION.key -> "true")) } else { None } val timestampNtz = if (SchemaUtils.checkForTimestampNTZColumnsRecursively(convertedSchema)) { Some("delta.feature.timestampNtz" -> "supported") } else None icebergTable.properties().asScala.toMap + (DeltaConfigs.COLUMN_MAPPING_MODE.key -> "id") + (DeltaConfigs.LOG_RETENTION.key -> s"$maxSnapshotAgeMs millisecond") ++ castTimeTypeConf ++ timestampNtz ++ bucketPartitionToNonPartition } val tablePartitionSpec: PartitionSpec = { // Validate && Get Partition Spec from Iceberg table // We don't support conversion from iceberg tables with partition evolution // So normally we only allow table having one partition spec // // However, we allow one special case where // all data files have either no-partition or bucket-partition // in this case we will convert them into non-partition, so // we will use an arbitrary non-bucket-partition spec as table's spec if (icebergTable.specs().size() == 1 || partitionEvolutionEnabled || !bucketPartitionEnabled) { icebergTable.spec() } else if (icebergTable.specs().isEmpty) { throw cloneFromIcebergSourceWithoutSpecs() } else { icebergTable.specs().asScala.values.find( !IcebergPartitionUtil.hasNonBucketPartition(_) ).getOrElse { throw cloneFromIcebergSourceWithPartitionEvolution() } } } override val partitionSchema: StructType = { // Reuse physical names of existing columns. val mergedPartitionSchema = DeltaColumnMapping.setPhysicalNames( StructType( IcebergPartitionUtil.getPartitionFields( tablePartitionSpec, icebergTable.schema(), castTimeType ) ), fieldPathToPhysicalName) // Assign physical names to new partition columns. DeltaColumnMapping.assignPhysicalNames(mergedPartitionSchema, reuseLogicalName = true) } val tableSchema: StructType = PartitioningUtils.mergeDataAndPartitionSchema( convertedSchema, partitionSchema, spark.sessionState.conf.caseSensitiveAnalysis)._1 checkConvertible() val fileManifest = new IcebergFileManifest(spark, icebergTable, partitionSchema, convertStats) lazy val numFiles: Long = Option(icebergTable.currentSnapshot()) .flatMap { snapshot => Option(snapshot.summary()).flatMap(_.asScala.get("total-data-files").map(_.toLong)) } .getOrElse(fileManifest.numFiles) lazy val sizeInBytes: Long = Option(icebergTable.currentSnapshot()) .flatMap { snapshot => Option(snapshot.summary()).flatMap(_.asScala.get("total-files-size").map(_.toLong)) } .getOrElse(fileManifest.sizeInBytes) override val format: String = "iceberg" def checkConvertible(): Unit = { /** * If the sql conf bucketPartitionEnabled is true, then convert iceberg table with * bucket partition to unpartitioned delta table; if bucketPartitionEnabled is false, * block conversion. */ if (!bucketPartitionEnabled && IcebergPartitionUtil.hasBucketPartition(icebergTable.spec())) { throw new UnsupportedOperationException(IcebergTable.ERR_BUCKET_PARTITION) } /** * Existing Iceberg Table that has data imported from table without field ids will need * to add a custom property to enable the mapping for Iceberg. * Therefore, we can simply check for the existence of this property to see if there was * a custom mapping within Iceberg. * * Ref: https://www.mail-archive.com/dev@iceberg.apache.org/msg01638.html */ if (icebergTable.properties().containsKey(TableProperties.DEFAULT_NAME_MAPPING)) { throw new UnsupportedOperationException(IcebergTable.ERR_CUSTOM_NAME_MAPPING) } /** * Delta does not support case sensitive columns while Iceberg does. We should check for * this here to throw a better message tailored to converting to Delta than the default * AnalysisException */ try { SchemaMergingUtils.checkColumnNameDuplication(tableSchema, "during convert to Delta") } catch { case e: AnalysisException if e.getMessage.contains("during convert to Delta") => throw new UnsupportedOperationException( IcebergTable.caseSensitiveConversionExceptionMsg(e.getMessage)) } } } object IcebergTable { /** Error message constants */ val ERR_MULTIPLE_PARTITION_SPECS = s"Source iceberg table has undergone partition evolution" val ERR_CUSTOM_NAME_MAPPING = "Cannot convert Iceberg tables with column name mapping" val ERR_BUCKET_PARTITION = "Cannot convert Iceberg tables with bucket partition" def caseSensitiveConversionExceptionMsg(conflictingColumns: String): String = s"""Cannot convert table to Delta as the table contains column names that only differ by case. |$conflictingColumns. Delta does not support case sensitive column names. |Please rename these columns before converting to Delta. """.stripMargin } ================================================ FILE: iceberg/src/main/scala/org/apache/spark/sql/delta/TypeToSparkTypeWithCustomCast.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.convert import java.util import scala.collection.JavaConverters._ import shadedForDelta.org.apache.iceberg.MetadataColumns import shadedForDelta.org.apache.iceberg.Schema import shadedForDelta.org.apache.iceberg.relocated.com.google.common.collect.Lists import shadedForDelta.org.apache.iceberg.types.Type import shadedForDelta.org.apache.iceberg.types.Type.TypeID._ import shadedForDelta.org.apache.iceberg.types.Types import shadedForDelta.org.apache.iceberg.types.TypeUtil import org.apache.spark.sql.types.ArrayType import org.apache.spark.sql.types.BinaryType import org.apache.spark.sql.types.BooleanType import org.apache.spark.sql.types.DataType import org.apache.spark.sql.types.DateType import org.apache.spark.sql.types.DecimalType import org.apache.spark.sql.types.DoubleType import org.apache.spark.sql.types.FloatType import org.apache.spark.sql.types.IntegerType import org.apache.spark.sql.types.LongType import org.apache.spark.sql.types.MapType import org.apache.spark.sql.types.Metadata import org.apache.spark.sql.types.MetadataBuilder import org.apache.spark.sql.types.StringType import org.apache.spark.sql.types.StructField import org.apache.spark.sql.types.StructType import org.apache.spark.sql.types.TimestampNTZType import org.apache.spark.sql.types.TimestampType /** * This class is copied from [[org.apache.iceberg.spark.TypeToSparkType]] to * add custom type casting. Currently, it supports the following casting * * Iceberg TIME -> Spark Long */ class TypeToSparkTypeWithCustomCast(castTimeType: Boolean) extends TypeUtil.SchemaVisitor[DataType] { val METADATA_COL_ATTR_KEY = "__metadata_col"; override def schema(schema: Schema, structType: DataType): DataType = structType override def struct(struct: Types.StructType, fieldResults: util.List[DataType]): DataType = { val fields = struct.fields(); val sparkFields: util.List[StructField] = Lists.newArrayListWithExpectedSize(fieldResults.size()) for (i <- 0 until fields.size()) { val field = fields.get(i) val `type` = fieldResults.get(i) val metadata = fieldMetadata(field.fieldId()) var sparkField = StructField.apply(field.name(), `type`, field.isOptional(), metadata) if (field.doc() != null) { sparkField = sparkField.withComment(field.doc()) } sparkFields.add(sparkField) } StructType.apply(sparkFields) } override def field(field: Types.NestedField, fieldResult: DataType): DataType = fieldResult override def list(list: Types.ListType, elementResult: DataType): DataType = ArrayType.apply(elementResult, list.isElementOptional()) override def map(map: Types.MapType, keyResult: DataType, valueResult: DataType): DataType = MapType.apply(keyResult, valueResult, map.isValueOptional()) override def primitive(primitive: Type.PrimitiveType): DataType = { primitive.typeId() match { case BOOLEAN => BooleanType case INTEGER => IntegerType case LONG => LongType case FLOAT => FloatType case DOUBLE => DoubleType case DATE => DateType // Changed to allow casting TIME to Spark Long. // The result is microseconds since midnight. case TIME => if (castTimeType) { LongType } else { throw new UnsupportedOperationException("Spark does not support time fields") } case TIMESTAMP => val ts = primitive.asInstanceOf[Types.TimestampType] if (ts.shouldAdjustToUTC()) { TimestampType } else { TimestampNTZType } case STRING => StringType case UUID => // use String StringType case FIXED => BinaryType case BINARY => BinaryType case DECIMAL => val decimal = primitive.asInstanceOf[Types.DecimalType] DecimalType.apply(decimal.precision(), decimal.scale()); case _ => throw new UnsupportedOperationException( "Cannot convert unknown type to Spark: " + primitive); } } private def fieldMetadata(fieldId: Int): Metadata = { if (MetadataColumns.metadataFieldIds().contains(fieldId)) { return new MetadataBuilder().putBoolean(METADATA_COL_ATTR_KEY, value = true).build() } Metadata.empty } } ================================================ FILE: iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/DeltaToIcebergConvert.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.icebergShaded import java.nio.ByteBuffer import java.sql.Timestamp import java.time.{LocalDateTime, OffsetDateTime, ZoneOffset} import java.time.format._ import java.util.{Base64, List => JList} import scala.util.control.NonFatal import org.apache.spark.sql.delta.{DeltaConfig, DeltaConfigs, IcebergCompat, NoMapping, Snapshot, SnapshotDescriptor} import org.apache.spark.sql.delta.DeltaConfigs.{LOG_RETENTION, TOMBSTONE_RETENTION} import org.apache.spark.sql.delta.actions.{AddFile, FileAction} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.util.JsonUtils import shadedForDelta.org.apache.iceberg.{FileMetadata, PartitionData, PartitionSpec, Schema => IcebergSchema, StructLike, TableProperties => IcebergTableProperties} import shadedForDelta.org.apache.iceberg.expressions.Literal import shadedForDelta.org.apache.iceberg.types.{Conversions, Type => IcebergType, Types => IcebergTypes} import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns.CURRENT_DEFAULT_COLUMN_METADATA_KEY import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.CalendarInterval /** * Generate Iceberg table metadata (schema, partition, etc.) from a Delta [[Snapshot]] */ class DeltaToIcebergConverter(val snapshot: SnapshotDescriptor, val catalogTable: CatalogTable) { private val schemaUtils: IcebergSchemaUtils = IcebergSchemaUtils(snapshot.metadata.columnMappingMode == NoMapping) def maxFieldId: Int = schemaUtils.maxFieldId(snapshot) val schema: IcebergSchema = IcebergCompat .getEnabledVersion(snapshot.metadata) .orElse(Some(0)) .map { compatVersion => val icebergStruct = schemaUtils.convertStruct(snapshot.schema)(compatVersion) new IcebergSchema(icebergStruct.fields()) }.getOrElse(throw new IllegalArgumentException("No IcebergCompat available")) val partition: PartitionSpec = IcebergTransactionUtils .createPartitionSpec(schema, snapshot.metadata.partitionColumns) val properties: Map[String, String] = DeltaToIcebergConvert.TableProperties(snapshot.metadata.configuration) } /** * Utils for converting a Delta Table to Iceberg Table */ object DeltaToIcebergConvert extends DeltaLogging { object Action extends DeltaLogging { def buildPartitionValues( builder: FileMetadata.Builder, fileAction: FileAction, partitionSpec: PartitionSpec, snapshot: Snapshot, logicalToPhysicalPartitionNames: Map[String, String]): Unit = { if (partitionSpec.isPartitioned) { builder.withPartition( DeltaToIcebergConvert.Partition.convertPartitionValues( snapshot, partitionSpec, fileAction.partitionValues, logicalToPhysicalPartitionNames)) } } } /** * Utils used when converting Delta schema to Iceberg */ object Schema { /** * Extract Delta Column Default values in Iceberg Literal format * @param field column * @return Right(Some(Literal)) if the column contains a literal default * Right(None) if the column does not have a default * Left(errorMessage) if the column contains a non-literal default */ def extractLiteralDefault(field: StructField): Either[String, Option[Literal[_]]] = { if (field.metadata.contains(CURRENT_DEFAULT_COLUMN_METADATA_KEY)) { val defaultValueStr = field.metadata.getString(CURRENT_DEFAULT_COLUMN_METADATA_KEY) try { Right(Some(stringToLiteral(defaultValueStr, field.dataType))) } catch { case NonFatal(e) => Left("Unsupported default value:" + s"${field.dataType.typeName}:$defaultValueStr:${e.getMessage}") case unknown: Throwable => throw unknown } } else { Right(None) } } /** * Follow Spark's string escape rule to unescape a default value string * @param input string to be unescaped * @return unescaped string */ def unescapeString(input: String): String = { val table = Map[Char, String]( 'b' -> "\u0008", 't' -> "\t", 'n' -> "\n", 'r' -> "\r", 'Z' -> "\u001A", '\\' -> "\\", '%' -> "\\%", '_' -> "\\_", '\'' -> "'" ) def isHex(c: Char): Boolean = Character.digit(c, 16) >= 0 def isOct(c: Char): Boolean = c >= '0' && c <= '7' def hexAt(pos: Int, n: Int): String = { if (pos + n <= input.length && (0 until n).forall(k => isHex(input.charAt(pos + k)))) { val cp = Integer.parseInt(input.substring(pos, pos + n), 16) new String(Character.toChars(cp)) } else null } def octAt(pos: Int): String = { if (pos + 3 <= input.length && (0 until 3).forall(k => isOct(input.charAt(pos + k)))) { val cp = Integer.parseInt(input.substring(pos, pos + 3), 8) if (cp <= 255) new String(Character.toChars(cp)) else null } else null } val out = new StringBuilder var i = 0 while (i < input.length) { val c = input.charAt(i) if (c != '\\') { out.append(c); i += 1 } else { if (i + 1 >= input.length) throw new IllegalStateException("dangling escape") val d = input.charAt(i + 1) if (d >= '0' && d <= '7') { val oct = octAt(i + 1) if (oct != null) { out.append(oct); i += 4 } else if (d == '0') { out.append("\u0000"); i += 2 } else { out.append(d); i += 2 } } else if (d == 'u' || d == 'U') { val h8 = hexAt(i + 2, 8) val h4 = if (h8 == null) hexAt(i + 2, 4) else null val h = if (h8 != null) h8 else h4 if (h != null) { out.append(h); i += (if (h8 != null) 10 else 6) } else { out.append(d); i += 2 } // \x => x rule } else { out.append(table.getOrElse(d, d.toString)) i += 2 } } } out.toString } /** * Convert Delta default value string to an Iceberg Literal based on data type. * @param str default value in Delta column metadata * @param dataType Delta column data type * @return converted Literal */ def stringToLiteral(str: String, dataType: DataType): Literal[_] = { def parseString(input: String) = { if (input.length > 1 && ((input.head == '\'' && input.last == '\'') || (input.head == '"' && input.last == '"'))) { Literal.of(unescapeString(input.substring(1, input.length - 1))) } else { throw new UnsupportedOperationException(s"String missing quotation marks: $input") } } // Parse either hex encoded literal x'....' or string literal(utf8) into binary def parseBinary(input: String) = { if (input.startsWith("x") || input.startsWith("X")) { // Hex encoded literal var hexString = parseString(input.substring(1)).value().toString if (hexString.length % 2 == 1) { hexString = hexString.substring(1) + "00" } Literal.of(hexString.sliding(2, 2).map(Integer.parseInt(_, 16).toByte).toArray) } else { Literal.of(parseString(input).value().toString .getBytes(java.nio.charset.StandardCharsets.UTF_8)) } } // Parse timestamp string without time zone info def parseLocalTimestamp(input: String) = { val formats = Seq(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"), DateTimeFormatter.ISO_LOCAL_DATE_TIME) val stripped = parseString(input).value() val parsed = formats.flatMap { format => try { val ldt = LocalDateTime.parse(stripped, format) Some( Literal.of( ldt.toInstant(ZoneOffset.UTC).getEpochSecond * 1000000 + ldt.getNano / 1000 ) ) } catch { case NonFatal(_) => None } } if (parsed.nonEmpty) { parsed.head } else { throw new IllegalArgumentException(input) } } // Parse string with time zone info. If the input has no time zone, assume its UTC. def parseTimestamp(input: String) = { val stripped = parseString(input).value() try { val instant = OffsetDateTime.parse(stripped, DateTimeFormatter.ISO_DATE_TIME).toInstant Literal.of(instant.getEpochSecond * 1000000 + instant.getNano / 1000) } catch { case NonFatal(_) => parseLocalTimestamp(input) } } dataType match { case StringType => parseString(str) case LongType => Literal.of(java.lang.Long.valueOf(str.replaceAll("[lL]$", ""))) case IntegerType | ShortType | ByteType => Literal.of(Integer.valueOf(str)) case FloatType => Literal.of(java.lang.Float.valueOf(str)) case DoubleType => Literal.of(java.lang.Double.valueOf(str)) // The number should be correctly formatted without need to rounding case d: DecimalType => Literal.of( new java.math.BigDecimal(str, new java.math.MathContext(d.precision)).setScale(d.scale) ) case BooleanType => Literal.of(java.lang.Boolean.valueOf(str)) case BinaryType => parseBinary(str) case DateType => parseString(str).to(IcebergTypes.DateType.get()) case TimestampType => parseTimestamp(str) case TimestampNTZType => parseLocalTimestamp(str) case _ => throw new UnsupportedOperationException( s"Could not convert default value: $dataType: $str") } } } object TableProperties { /** * We generate Iceberg Table properties from Delta table properties * using two methods. * 1. If a Delta property key starts with "delta.universalformat.config.iceberg" * we strip the prefix from the key and include the property pair. * Note the key is already normalized to lower case. * 2. We compute Iceberg properties from Delta using custom logic * This now includes * a) Iceberg format version * b) Iceberg snapshot retention */ def apply(deltaProperties: Map[String, String]): Map[String, String] = { val prefix = DeltaConfigs.DELTA_UNIVERSAL_FORMAT_ICEBERG_CONFIG_PREFIX val copiedFromDelta = deltaProperties .filterKeys(_.startsWith(prefix)) .map { case (key, value) => key.stripPrefix(prefix) -> value } .toSeq .toMap val computers = Seq(FormatVersionComputer, RetentionPeriodComputer) val computed: Map[String, String] = computers .map(_.apply(deltaProperties ++ copiedFromDelta)) .reduce((a, b) => a ++ b) copiedFromDelta ++ computed } private trait IcebergPropertiesComputer { /** * Compute Iceberg properties from Delta properties. */ def apply(deltaProperties: Map[String, String]): Map[String, String] } /** * Compute Iceberg FORMAT_VERSION from IcebergCompat */ private object FormatVersionComputer extends IcebergPropertiesComputer { override def apply(deltaProperties: Map[String, String]): Map[String, String] = IcebergCompat .anyEnabled(deltaProperties) .map(IcebergTableProperties.FORMAT_VERSION -> _.icebergFormatVersion.toString) .toMap } /** * Compute Iceberg MAX_SNAPSHOT_AGE_MS as the minimal of * Delta's LOG_RETENTION and TOMBSTONE_RETENTION. * If users explicitly provide a MAX_SNAPSHOT_AGE_MS, also ensure the provided * value is no larger than Delta's retention. */ private object RetentionPeriodComputer extends IcebergPropertiesComputer { override def apply(deltaProperties: Map[String, String]): Map[String, String] = { def getAsMilliSeconds(conf: DeltaConfig[CalendarInterval], properties: Map[String, String], useDefault: Boolean = false): Option[Long] = properties.get(conf.key) .orElse(if (useDefault) Some(conf.defaultValue) else None) .map(conf.fromString) .map(DeltaConfigs.getMilliSeconds) // Set Iceberg max snapshot age as minimal of Delta log retention and tombstone retention val deltaRetention = ( getAsMilliSeconds(LOG_RETENTION, deltaProperties), getAsMilliSeconds(TOMBSTONE_RETENTION, deltaProperties) ) match { case (Some(a), Some(b)) => Some(a min b) case (a, b) => a orElse b } // If user provided max snapshot age, check that it is smaller than Delta's retention lazy val maxAllowedRetention = getAsMilliSeconds(LOG_RETENTION, deltaProperties, useDefault = true).get min getAsMilliSeconds(TOMBSTONE_RETENTION, deltaProperties, useDefault = true).get deltaProperties.get(IcebergTableProperties.MAX_SNAPSHOT_AGE_MS) .foreach { providedRetention => if (providedRetention.toLong > maxAllowedRetention) { throw new IllegalArgumentException( s"""Uniform iceberg's ${IcebergTableProperties.MAX_SNAPSHOT_AGE_MS} should be | no less than the min of delta's ${LOG_RETENTION.key} and | ${TOMBSTONE_RETENTION.key}. | Current delta retention min in MS: $maxAllowedRetention. | Proposed iceberg retention in Ms: $providedRetention""".stripMargin) } } deltaRetention .filter(_ < IcebergTableProperties.MAX_SNAPSHOT_AGE_MS_DEFAULT) .map { IcebergTableProperties.MAX_SNAPSHOT_AGE_MS -> _.toString } .toMap } } } object Partition { private[delta] def convertPartitionValues( snapshot: Snapshot, partitionSpec: PartitionSpec, partitionValues: Map[String, String], logicalToPhysicalPartitionNames: Map[String, String]): StructLike = { val schema = snapshot.schema val ICEBERG_NULL_PARTITION_VALUE = "__HIVE_DEFAULT_PARTITION__" val partitionPath = partitionSpec.fields() val partitionVals = new Array[Any](partitionSpec.fields().size()) val nameToDataTypes: Map[String, DataType] = schema.fields.map(f => f.name -> f.dataType).toMap for (i <- partitionVals.indices) { val logicalPartCol = partitionPath.get(i).name() val physicalPartKey = logicalToPhysicalPartitionNames(logicalPartCol) // ICEBERG_NULL_PARTITION_VALUE is referred in Iceberg lib to mark NULL partition value val partValue = Option(partitionValues.getOrElse(physicalPartKey, null)) .getOrElse(ICEBERG_NULL_PARTITION_VALUE) val partitionColumnDataType = nameToDataTypes(logicalPartCol) val icebergPartitionValue = IcebergTransactionUtils.stringToIcebergPartitionValue( partitionColumnDataType, partValue, snapshot.version) partitionVals(i) = icebergPartitionValue } new IcebergTransactionUtils.Row(partitionVals) } } } ================================================ FILE: iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConversionTransaction.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.icebergShaded import java.util.ConcurrentModificationException import java.util.function.Consumer import scala.collection.JavaConverters._ import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import scala.jdk.OptionConverters._ import scala.util.control.NonFatal import org.apache.spark.sql.delta.{DeltaFileProviderUtils, DummySnapshot, IcebergConstants, NoMapping, Snapshot} import org.apache.spark.sql.delta.actions.{AddFile, Metadata, RemoveFile} import org.apache.spark.sql.delta.icebergShaded.IcebergTransactionUtils._ import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.hadoop.conf.Configuration import shadedForDelta.org.apache.iceberg.{AppendFiles, BaseTransaction, DataFile, DeleteFiles, ExpireSnapshots, OverwriteFiles, PartitionSpec, PendingUpdate, RewriteFiles, Schema => IcebergSchema, TableMetadata, Transaction => IcebergTransaction} import shadedForDelta.org.apache.iceberg.MetadataUpdate import shadedForDelta.org.apache.iceberg.MetadataUpdate.{AddPartitionSpec, AddSchema} import shadedForDelta.org.apache.iceberg.mapping.MappingUtil import shadedForDelta.org.apache.iceberg.mapping.NameMappingParser import shadedForDelta.org.apache.iceberg.unityCatalog.{UnityCatalog, UnityCatalogTableOperations} import shadedForDelta.org.apache.iceberg.util.LocationUtil import org.apache.spark.internal.MDC import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.catalog.CatalogTable sealed trait IcebergTableOp case object CREATE_TABLE extends IcebergTableOp case object WRITE_TABLE extends IcebergTableOp case object REPLACE_TABLE extends IcebergTableOp sealed trait IcebergConversionMode { } // Used by Post-commit Delta UniForm (Iceberg conversion in Delta post commit hook) case object UNIFORM_POST_COMMIT_MODE extends IcebergConversionMode { } // Used by atomic Delta UniForm case object UNIFORM_CC_MODE extends IcebergConversionMode { } /** * Used to prepare (convert) and then commit a set of Delta actions into the Iceberg table located * at the same path as [[postCommitSnapshot]] * * * @param conf Configuration for Iceberg Hadoop interactions. * @param postCommitSnapshot Latest Delta snapshot associated with this Iceberg commit. * @param tableOp How to instantiate the underlying Iceberg table. Defaults to WRITE_TABLE. * @param lastConvertedIcebergSnapshotId the iceberg snapshot this Iceberg txn should write to. * @param lastConvertedDeltaVersion the delta version this Iceberg txn starts from. */ class IcebergConversionTransaction( protected val spark: SparkSession, protected val catalogTable: CatalogTable, protected val conf: Configuration, protected val postCommitSnapshot: Snapshot, protected val tableOp: IcebergTableOp = WRITE_TABLE, protected val lastConvertedIcebergSnapshotId: Option[Long] = None, protected val lastConvertedDeltaVersion: Option[Long] = None, protected val lastConvertedIcebergMetadataPath: Option[String] = None, protected val metadataUpdates: java.util.ArrayList[MetadataUpdate] = new java.util.ArrayList[MetadataUpdate]() ) extends DeltaLogging { /////////////////////////// // Nested Helper Classes // /////////////////////////// implicit class AddFileConversion(addFile: AddFile) { def toDataFile: DataFile = convertDeltaAddFileToIcebergDataFile( addFile, tablePath, currentPartitionSpec, logicalToPhysicalPartitionNames, statsParser, postCommitSnapshot) } implicit class RemoveFileConversion(removeFile: RemoveFile) { def toDataFile: DataFile = convertDeltaRemoveFileToIcebergDataFile( removeFile, tablePath, currentPartitionSpec, logicalToPhysicalPartitionNames, postCommitSnapshot) } protected abstract class TransactionHelper(protected val impl: PendingUpdate[_]) { protected var committed = false var writeSize = 0L def opType: String def add(add: AddFile): Unit = throw new UnsupportedOperationException def add(remove: RemoveFile): Unit = throw new UnsupportedOperationException def commit(expectedSequenceNumber: Long): Unit = { assert(!committed, "Already committed.") impl.commit() committed = true } private[icebergShaded]def hasCommitted: Boolean = committed protected def currentSnapshotId: Option[Long] = Option(txn.table().currentSnapshot()).map(_.snapshotId()) } class NullHelper extends TransactionHelper(null) { override def opType: String = "null" override def add(add: AddFile): Unit = {} override def add(remove: RemoveFile): Unit = {} override def commit(deltaCommitVersion: Long): Unit = {} } /** * API for appending new files in a table. * * e.g. INSERT */ class AppendOnlyHelper(appender: AppendFiles) extends TransactionHelper(appender) { override def opType: String = "append" override def add(add: AddFile): Unit = { writeSize += add.size appender.appendFile(add.toDataFile) } } /** * API for deleting files from a table. * * e.g. DELETE */ class RemoveOnlyHelper(deleter: DeleteFiles) extends TransactionHelper(deleter) { override def opType: String = "delete" override def add(remove: RemoveFile): Unit = { // We can just use the canonical RemoveFile.path instead of converting RemoveFile to DataFile. // Note that in other helper APIs, converting a FileAction to a DataFile will also take care // of canonicalizing the path. deleter.deleteFile(canonicalizeFilePath(remove, tablePath)) } } /** * API for overwriting files in a table. Replaces all the deleted files with the set of additions. * * e.g. UPDATE, MERGE */ class OverwriteHelper(overwriter: OverwriteFiles) extends TransactionHelper(overwriter) { override def opType: String = "overwrite" override def add(add: AddFile): Unit = { writeSize += add.size overwriter.addFile(add.toDataFile) } override def add(remove: RemoveFile): Unit = { overwriter.deleteFile(remove.toDataFile) } } /** * API for rewriting existing files in the table (i.e. replaces one set of data files with another * set that contains the same data). * * e.g. OPTIMIZE */ class RewriteHelper(rewriter: RewriteFiles) extends TransactionHelper(rewriter) { override def opType: String = "rewrite" private val addBuffer: mutable.HashSet[DataFile] = new mutable.HashSet[DataFile] private val removeBuffer: mutable.HashSet[DataFile] = new mutable.HashSet[DataFile] override def add(add: AddFile): Unit = { writeSize += add.size assert(!add.dataChange, "Rewrite operation should not add data") addBuffer += add.toDataFile } override def add(remove: RemoveFile): Unit = { assert(!remove.dataChange, "Rewrite operation should not add data") removeBuffer += remove.toDataFile } override def commit(deltaCommitVersion: Long): Unit = { if (removeBuffer.nonEmpty) { rewriter.rewriteFiles(removeBuffer.asJava, addBuffer.asJava, 0) } currentSnapshotId.foreach(rewriter.validateFromSnapshot) super.commit(deltaCommitVersion) } } class ExpireSnapshotHelper(expireSnapshot: ExpireSnapshots) extends TransactionHelper(expireSnapshot) { def cleanExpiredFiles(clean: Boolean): ExpireSnapshotHelper = { expireSnapshot.cleanExpiredFiles(clean) this } def deleteWith(newDeleteFunc: Consumer[String]): ExpireSnapshotHelper = { expireSnapshot.deleteWith(newDeleteFunc) this } override def opType: String = "expireSnapshot" } ////////////////////// // Member variables // ////////////////////// protected val tablePath = postCommitSnapshot.deltaLog.dataPath protected val convert = new DeltaToIcebergConverter(postCommitSnapshot, catalogTable) protected def icebergSchema: IcebergSchema = convert.schema // Initial partition spec converted from Delta protected def partitionSpec: PartitionSpec = convert.partition // Current partition spec from iceberg table def currentPartitionSpec: PartitionSpec = { Some(txn.table()).map(_.spec()).getOrElse(partitionSpec) } protected val logicalToPhysicalPartitionNames = getPartitionPhysicalNameMapping(postCommitSnapshot.metadata.partitionSchema) /** Parses the stats JSON string to convert Delta stats to Iceberg stats. */ private val statsParser = DeltaFileProviderUtils.createJsonStatsParser(postCommitSnapshot.statsSchema) /** Visible for testing. */ private[icebergShaded]val (txn, startFromSnapshotId) = withStartSnapshotId(createIcebergTxn()) /** Tracks if this transaction has already committed. You can only commit once. */ private var committed = false /** Tracks the file updates (add, remove, overwrite, rewrite) made to this table. */ protected val fileUpdates = new ArrayBuffer[TransactionHelper]() /** Tracks if this transaction updates only the differences between a prev and new metadata. */ private var isMetadataUpdate = false ///////////////// // Public APIs // ///////////////// def getNullHelper: NullHelper = new NullHelper() def getAppendOnlyHelper: AppendOnlyHelper = { val ret = new AppendOnlyHelper(txn.newAppend()) fileUpdates += ret ret } def getRemoveOnlyHelper: RemoveOnlyHelper = { val ret = new RemoveOnlyHelper(txn.newDelete()) fileUpdates += ret ret } def getOverwriteHelper: OverwriteHelper = { val ret = new OverwriteHelper(txn.newOverwrite()) fileUpdates += ret ret } def getRewriteHelper: RewriteHelper = { val ret = new RewriteHelper(txn.newRewrite()) fileUpdates += ret ret } def getExpireSnapshotHelper(): ExpireSnapshotHelper = { val ret = new ExpireSnapshotHelper(txn.expireSnapshots()) fileUpdates += ret ret } /** * Handles the following update scenarios * - partition update -> throws * - schema update -> sets the full new schema * - properties update -> applies only the new properties */ def updateTableMetadata(prevMetadata: Metadata): Unit = { assert(!isMetadataUpdate, "updateTableMetadata already called") isMetadataUpdate = true val newMetadata = postCommitSnapshot.metadata // Throws if partition evolution detected if (newMetadata.partitionColumns != prevMetadata.partitionColumns) { throw new IllegalStateException("Delta does not support partition evolution") } // As we do not have a second set schema txn for REPLACE_TABLE, we need to set // the schema as part of this transaction if (newMetadata.schema != prevMetadata.schema || tableOp == REPLACE_TABLE) { val differenceStr = SchemaUtils.reportDifferences(prevMetadata.schema, newMetadata.schema) logInfo( log"Detected schema update for table with name=" + log"${MDC(DeltaLogKeys.TABLE_NAME, newMetadata.name)}, " + log"id=${MDC(DeltaLogKeys.METADATA_ID, newMetadata.id)}:\n" + log"${MDC(DeltaLogKeys.SCHEMA_DIFF, differenceStr)}, " + s"tableOp=$tableOp, " + log"Setting new Iceberg schema:\n " + log"${MDC(DeltaLogKeys.SCHEMA, icebergSchema)}" ) metadataUpdates.add(new AddSchema(icebergSchema, convert.maxFieldId)) recordDeltaEvent( postCommitSnapshot.deltaLog, "delta.iceberg.conversion.schemaChange", data = Map( "version" -> postCommitSnapshot.version, "deltaSchemaDiff" -> differenceStr, "icebergSchema" -> icebergSchema.toString.replace('\n', ';') ) ) } // Compute and apply properties changes val (propertyDeletes, propertyAdditions) = { val newIcebergProperties = convert.properties val prevIcebergProperties = new DeltaToIcebergConverter( new DummySnapshot( logPath = postCommitSnapshot.path, deltaLog = postCommitSnapshot.deltaLog, metadata = prevMetadata), catalogTable ).properties if (prevIcebergProperties == newIcebergProperties) { (Set.empty, Map.empty) } else { ( prevIcebergProperties.keySet.diff(newIcebergProperties.keySet), newIcebergProperties ) } } if (propertyDeletes.nonEmpty || propertyAdditions.nonEmpty) { val updater = txn.updateProperties() propertyDeletes.foreach(updater.remove) propertyAdditions.foreach(kv => updater.set(kv._1, kv._2)) updater.commit() recordDeltaEvent( postCommitSnapshot.deltaLog, "delta.iceberg.conversion.propertyChange", data = Map("version" -> postCommitSnapshot.version) ++ (if (propertyDeletes.nonEmpty) Map("deletes" -> propertyDeletes.toSeq) else Map.empty) ++ (if (propertyAdditions.nonEmpty) Map("adds" -> propertyAdditions) else Map.empty) ) } } def commit(): Unit = { assert(!committed, "Cannot commit. Transaction already committed.") // At least one file or metadata updates is required when writing to an existing table. If // creating or replacing a table, we can create an empty table with just the table metadata // (schema, properties, etc.) if (tableOp == WRITE_TABLE) { assert(fileUpdates.nonEmpty || isMetadataUpdate, "Cannot commit WRITE. Transaction is empty.") } assert(fileUpdates.forall(_.hasCommitted), "Cannot commit. You have uncommitted changes.") val nameMapping = NameMappingParser.toJson(MappingUtil.create(icebergSchema)) var updateTxn = txn.updateProperties() updateTxn = updateTxn.set(IcebergConverter.DELTA_VERSION_PROPERTY, postCommitSnapshot.version.toString) .set(IcebergConverter.DELTA_TIMESTAMP_PROPERTY, postCommitSnapshot.timestamp.toString) .set(IcebergConstants.ICEBERG_NAME_MAPPING_PROPERTY, nameMapping) val includeBaseVersion = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_UNIFORM_ICEBERG_INCLUDE_BASE_CONVERTED_VERSION) updateTxn = lastConvertedDeltaVersion match { case Some(v) if includeBaseVersion => updateTxn.set(IcebergConverter.BASE_DELTA_VERSION_PROPERTY, v.toString) case _ => updateTxn.remove(IcebergConverter.BASE_DELTA_VERSION_PROPERTY) } updateTxn.commit() // We ensure the iceberg txns are serializable by only allowing them to commit against // lastConvertedIcebergSnapshotId. // // If the startFromSnapshotId is non-empty and not the same as lastConvertedIcebergSnapshotId, // there is a new iceberg transaction committed after we read lastConvertedIcebergSnapshotId, // and before this check. We explicitly abort by throwing exceptions. // // If startFromSnapshotId is empty, the txn must be one of the following: // 1. CREATE_TABLE // 2. Writing to an empty table // 3. REPLACE_TABLE // In either case this txn is safe to commit. // // Iceberg will further guarantee that txns passed this check are serializable. if (startFromSnapshotId.isDefined && lastConvertedIcebergSnapshotId != startFromSnapshotId) { throw new ConcurrentModificationException("Cannot commit because the converted " + s"metadata is based on a stale iceberg snapshot $lastConvertedIcebergSnapshotId" ) } try { // Iceberg CREATE_TABLE reassigns the field id in schema, which // is overwritten by setting Delta schema with Delta generated field id to ensure // consistency between field id in Iceberg schema after conversion and field id in // parquet files written by Delta. if (tableOp == CREATE_TABLE) { metadataUpdates.add( new AddSchema(icebergSchema, postCommitSnapshot.metadata.columnMappingMaxId.toInt) ) if (postCommitSnapshot.metadata.partitionColumns.nonEmpty) { metadataUpdates.add( new AddPartitionSpec(partitionSpec) ) } } txn.commitTransaction() recordIcebergCommit() } catch { case NonFatal(e) => recordIcebergCommit(Some(e)) throw e } committed = true } /** * Retrieves the converted Iceberg metadata location and its current snapshot. * This method should only be called after a successful table conversion operation * * @return A tuple containing: * - String: The path where the Iceberg metadata file was written * - IcebergMetadata: The converted Iceberg metadata * @throws IllegalStateException if the Iceberg metadata has not been converted * @throws UnsupportedOperationException if called on non-UnityCatalogTableOperations */ def getConvertedIcebergMetadata: (String, TableMetadata) = txn.asInstanceOf[BaseTransaction].underlyingOps() match { case ops: UnityCatalogTableOperations => ops.getLastWrittenTableMetadataWithLocation.toScala match { case Some((metadataPath, tableMetadata)) => (metadataPath, tableMetadata) case _ => throw new IllegalStateException( "Could not get converted Iceberg metadata: new written metadata not found") } case _ => throw new IllegalStateException( "Could not get converted Iceberg metadata:" + " underlying UnityCatalogTableOperations not found" ) } /////////////////////// // Protected Methods // /////////////////////// protected def createIcebergTxn(tableOpOpt: Option[IcebergTableOp] = None): IcebergTransaction = { val baseMetadataPath = (tableOpOpt.getOrElse(tableOp), lastConvertedIcebergMetadataPath) match { case (CREATE_TABLE, None) => None case (CREATE_TABLE, Some(_)) => throw new IllegalStateException( "Unexpected base metadata path for CREATE_TABLE operation") case (op, None) => throw new IllegalStateException(s"Missing base metadata path for $op operation") case (_, Some(path)) => Some(path) } val ucTable = new UnityCatalog( metadataUpdates, baseMetadataPath.toJava ) ucTable.initialize(null, new java.util.HashMap[String, String]()) ucTable.setConf(conf) val icebergIdentifier = IcebergTransactionUtils.convertSparkTableIdentifierToIceberg(catalogTable.identifier) val tableExists = ucTable.tableExists(icebergIdentifier) def tableBuilder = { val tableLocation = postCommitSnapshot.deltaLog.dataPath.toString ucTable .buildTable(icebergIdentifier, icebergSchema) .withPartitionSpec(partitionSpec) .withProperties(convert.properties.asJava) .withLocation(tableLocation) } val txn = tableOpOpt.getOrElse(tableOp) match { case WRITE_TABLE => if (tableExists) { recordFrameProfile("IcebergConversionTransaction", "loadTable") { ucTable.loadTable(icebergIdentifier).newTransaction() } } else { throw new IllegalStateException(s"Cannot write to table $tablePath. Table doesn't exist.") } case CREATE_TABLE => if (tableExists) { throw new IllegalStateException(s"Cannot create table $tablePath. Table already exists.") } else { recordFrameProfile("IcebergConversionTransaction", "createTable") { tableBuilder.createTransaction() } } case REPLACE_TABLE => if (tableExists) { recordFrameProfile("IcebergConversionTransaction", "replaceTable") { tableBuilder.replaceTransaction() } } else { throw new IllegalStateException(s"Cannot replace table $tablePath. Table doesn't exist.") } } txn } //////////////////// // Helper Methods // //////////////////// /** * We fetch the txn table's current snapshot id before any writing is made on the transaction. * This id should equal [[lastConvertedIcebergSnapshotId]] for the transaction to commit. * * @param txn the iceberg transaction * @return txn and the snapshot id just before this txn */ private def withStartSnapshotId(txn: IcebergTransaction): (IcebergTransaction, Option[Long]) = (txn, Option(txn.table().currentSnapshot()).map(_.snapshotId())) private def recordIcebergCommit(errorOpt: Option[Throwable] = None): Unit = { val icebergTxnTypes = if (fileUpdates.nonEmpty) Map("icebergTxnTypes" -> fileUpdates.map(_.opType)) else Map.empty val errorData = errorOpt.map { e => Map( "exception" -> ExceptionUtils.getMessage(e), "stackTrace" -> ExceptionUtils.getStackTrace(e) ) }.getOrElse(Map.empty) recordDeltaEvent( postCommitSnapshot.deltaLog, s"delta.iceberg.conversion.commit.${if (errorOpt.isEmpty) "success" else "error"}", data = Map( "version" -> postCommitSnapshot.version, "timestamp" -> postCommitSnapshot.timestamp, "tableOp" -> tableOp.getClass.getSimpleName.stripSuffix("$"), "prevConvertedDeltaVersion" -> lastConvertedDeltaVersion, "tableSize" -> postCommitSnapshot.sizeInBytes, "commitWriteSize" -> fileUpdates.map(_.writeSize).sum ) ++ icebergTxnTypes ++ errorData ) } } ================================================ FILE: iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConverter.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.icebergShaded import java.util.concurrent.atomic.AtomicReference import javax.annotation.concurrent.GuardedBy import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer import scala.util.control.Breaks._ import scala.util.control.NonFatal import org.apache.spark.sql.delta.{CommittedTransaction, CurrentTransactionInfo, DeltaErrors, DeltaFileNotFoundException, DeltaFileProviderUtils, DeltaLog, DeltaOperations, DummySnapshot, IcebergCompat, IcebergConstants, Snapshot, SnapshotDescriptor, UniversalFormat, UniversalFormatConverter} import org.apache.spark.sql.delta.DeltaOperations.OPTIMIZE_OPERATION_NAME import org.apache.spark.sql.delta.RowId.RowTrackingMetadataDomain import org.apache.spark.sql.delta.actions.{Action, AddFile, CommitInfo, DomainMetadata, FileAction, InMemoryLogReplay, Metadata, Protocol, RemoveFile} import org.apache.spark.sql.delta.hooks.IcebergConverterHook import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.TransactionHelper import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.hadoop.fs.Path import shadedForDelta.org.apache.iceberg.{Table => IcebergTable, TableProperties} import shadedForDelta.org.apache.iceberg.exceptions.CommitFailedException import shadedForDelta.org.apache.iceberg.hadoop.HadoopTables import shadedForDelta.org.apache.iceberg.hive.{HiveCatalog, HiveTableOperations} import shadedForDelta.org.apache.iceberg.util.LocationUtil import org.apache.spark.internal.MDC import org.apache.spark.sql.{Dataset, SparkSession} import org.apache.spark.sql.catalyst.catalog.CatalogTable object IcebergConverter { /** * Property to be set in translated Iceberg metadata files. * Indicates the delta commit version # that it corresponds to. */ val DELTA_VERSION_PROPERTY = "delta-version" /** * Property to be set in translated Iceberg metadata files. * Indicates the timestamp (milliseconds) of the delta commit that it corresponds to. */ val DELTA_TIMESTAMP_PROPERTY = "delta-timestamp" /** * Property to be set in translated Iceberg metadata files. * Indicates the base delta commit version # that the conversion started from */ val BASE_DELTA_VERSION_PROPERTY = "base-delta-version" def getLastConvertedDeltaVersion(table: Option[IcebergTable]): Option[Long] = table.flatMap(_.properties().asScala.get(DELTA_VERSION_PROPERTY)).map(_.toLong) def getLastConvertedDeltaTimestamp(table: Option[IcebergTable]): Option[Long] = table.flatMap(_.properties().asScala.get(DELTA_TIMESTAMP_PROPERTY)).map(_.toLong) } /** * This class manages the transformation of delta snapshots into their Iceberg equivalent. */ class IcebergConverter extends UniversalFormatConverter with DeltaLogging { // Save an atomic reference of the snapshot being converted, and the txn that triggered // resulted in the specified snapshot protected val currentConversion = new AtomicReference[(Snapshot, CommittedTransaction)]() protected val standbyConversion = new AtomicReference[(Snapshot, CommittedTransaction)]() // Whether our async converter thread is active. We may already have an alive thread that is // about to shutdown, but in such cases this value should return false. @GuardedBy("asyncThreadLock") private var asyncConverterThreadActive: Boolean = false private val asyncThreadLock = new Object private[icebergShaded]var targetSnapshot: SnapshotDescriptor = _ /** * Enqueue the specified snapshot to be converted to Iceberg. This will start an async * job to run the conversion, unless there already is an async conversion running for * this table. In that case, it will queue up the provided snapshot to be run after * the existing job completes. * Note that if there is another snapshot already queued, the previous snapshot will get * removed from the wait queue. Only one snapshot is queued at any point of time. * */ override def enqueueSnapshotForConversion( snapshotToConvert: Snapshot, txn: CommittedTransaction): Unit = { throw new IllegalStateException("enqueueSnapshotForConversion is no longer supported") } /** * Convert the specified snapshot into Iceberg for the given catalogTable * @param snapshotToConvert the snapshot that needs to be converted to Iceberg * @param catalogTable the catalogTable this conversion targets. * @return Converted Delta version and commit timestamp */ override def convertSnapshot( snapshotToConvert: Snapshot, catalogTable: CatalogTable): Option[(Long, Long)] = { throw new IllegalStateException("convertSnapshot is no longer supported") } /** * Convert the specified snapshot into Iceberg when performing an OptimisticTransaction * on a delta table. * @param snapshotToConvert the snapshot that needs to be converted to Iceberg * @param txn the transaction that triggers the conversion. It must * contain the catalogTable this conversion targets. * @return Converted Delta version and commit timestamp */ override def convertSnapshot( snapshotToConvert: Snapshot, txn: CommittedTransaction): Option[(Long, Long)] = { throw new IllegalStateException("convertSnapshot is no longer supported") } // Used for tracking last converted Iceberg metadata information // It would be used for incremental conversion for all Iceberg conversion modes protected case class LastConvertedIcebergInfo( icebergTable: Option[IcebergTable], icebergSnapshotId: Option[Long], deltaVersionConverted: Option[Long], baseMetadataLocationOpt: Option[String] ) /** * Used for tracking Iceberg Conversion Context by Conversion Mode * UNIFORM_POST_COMMIT_MODE => no context required * @param conversionMode * @param additionalDeltaActionsToCommit */ protected class ConversionContext( val conversionMode: IcebergConversionMode, val additionalDeltaActionsToCommit: Option[Seq[Action]], val opType: String ) { validate() // Validation on parameters def validate(): Unit = { conversionMode match { case _ => assert(additionalDeltaActionsToCommit.isEmpty) } } def hasAdditionalDeltaActionsToCommit: Boolean = { additionalDeltaActionsToCommit.nonEmpty } def getAdditionalDeltaActionsToCommit: Seq[Action] = { additionalDeltaActionsToCommit.get } } /** * The core implementation of convertSnapshot * 'delta.iceberg.conversion.convertSnapshot' -> * Convert Iceberg Metadata for a complete snapshot. Used for conversion * after delta commits and in create table */ protected def convertSnapshotInternal( snapshotToConvert: Snapshot, readSnapshotOpt: Option[Snapshot], lastConvertedInfo: LastConvertedIcebergInfo, conversionContext: ConversionContext, catalogTable: CatalogTable ): IcebergConversionTransaction = recordFrameProfile("Delta", "IcebergConverter.convertSnapshotImpl") { val conversionMode = conversionContext.conversionMode val log = snapshotToConvert.deltaLog targetSnapshot = snapshotToConvert val lastConvertedIcebergTable = lastConvertedInfo.icebergTable val lastConvertedIcebergSnapshotId = lastConvertedInfo.icebergSnapshotId val lastDeltaVersionConverted = lastConvertedInfo.deltaVersionConverted val baseMetadataLocation = lastConvertedInfo.baseMetadataLocationOpt val maxCommitsToConvert = spark.sessionState.conf.getConf(DeltaSQLConf.ICEBERG_MAX_COMMITS_TO_CONVERT) val conversionStartTime = System.currentTimeMillis() val prevConvertedSnapshotOpt = (lastDeltaVersionConverted, readSnapshotOpt) match { // The provided Snapshot is the last converted Snapshot case (Some(version), Some(readSnapshot)) if version == readSnapshot.version => Some(readSnapshot) // Some snapshots are pending conversion since last conversion case (Some(version), _) if snapshotToConvert.version - version <= maxCommitsToConvert => try { Some(log.getSnapshotAt(version, catalogTableOpt = Some(catalogTable))) } catch { // If we can't load the file since the last time Iceberg was converted, it's likely that // the commit file expired. Treat this like a new Iceberg table conversion. case _: DeltaFileNotFoundException => None } // Never converted before case _ => None } val tableOp = (lastDeltaVersionConverted, prevConvertedSnapshotOpt) match { case (Some(_), Some(_)) => WRITE_TABLE case (Some(_), None) => REPLACE_TABLE case (None, None) => CREATE_TABLE } val icebergTxn = new IcebergConversionTransaction( spark, catalogTable, log.newDeltaHadoopConf(), snapshotToConvert, tableOp, lastConvertedIcebergSnapshotId, lastDeltaVersionConverted ) val convertedCommits: Seq[Option[CommitInfo]] = prevConvertedSnapshotOpt match { case Some(prevSnapshot) => // Read the actions directly from the delta json files. // TODO: Run this as a spark job on executors val endVersion = conversionMode match { case _ => snapshotToConvert.version } val deltaFiles = DeltaFileProviderUtils.getDeltaFilesInVersionRange( spark = spark, deltaLog = log, startVersion = prevSnapshot.version + 1, endVersion = endVersion, catalogTableOpt = Some(catalogTable)) recordDeltaEvent( snapshotToConvert.deltaLog, "delta.iceberg.conversion.deltaCommitRange", data = Map( "fromVersion" -> (prevSnapshot.version + 1), "toVersion" -> snapshotToConvert.version, "numDeltaFiles" -> deltaFiles.length ) ) val actionsToConvert = DeltaFileProviderUtils.parallelReadAndParseDeltaFilesAsIterator( log, spark, deltaFiles) var deltaVersion = prevSnapshot.version val commitInfos = actionsToConvert.map { actionsIter => try { deltaVersion += 1 runIcebergConversionForActions( icebergTxn, actionsIter.map(Action.fromJson).toSeq, prevConvertedSnapshotOpt, deltaVersion) } finally { actionsIter.close() } } val additionalCommitInfo = if (conversionContext.hasAdditionalDeltaActionsToCommit) { runIcebergConversionForActions( icebergTxn, actionsToCommit = conversionContext.getAdditionalDeltaActionsToCommit, prevConvertedSnapshotOpt, deltaVersion + 1 ) } else { None } // If the metadata hasn't changed, this will no-op. icebergTxn.updateTableMetadata(prevSnapshot.metadata) commitInfos :+ additionalCommitInfo case None => // If we don't have a snapshot of the last converted version, get all the AddFiles // (via state reconstruction). // Batch is always disabled but we still want to reuse the event for conversion recordDeltaEvent( snapshotToConvert.deltaLog, "delta.iceberg.conversion.batch", data = Map( "version" -> snapshotToConvert.version, "numOfFiles" -> snapshotToConvert.numOfFiles, "actionBatchSize" -> -1, // This param is ignored as batch is deprecated "numOfPartitions" -> 1 ) ) runIcebergConversionForActions( icebergTxn, snapshotToConvert.allFiles.toLocalIterator().asScala.toSeq, None, snapshotToConvert.version) // Always attempt to update table metadata (schema/properties) for REPLACE_TABLE if (tableOp == REPLACE_TABLE) { icebergTxn.updateTableMetadata(snapshotToConvert.metadata) } Nil } // OPTIMIZE will trigger snapshot expiration for iceberg table val OPR_TRIGGER_EXPIRE = Set(DeltaOperations.OPTIMIZE_OPERATION_NAME) val needsExpireSnapshot = OPR_TRIGGER_EXPIRE.intersect(convertedCommits.flatten.map(_.operation).toSet).nonEmpty if (needsExpireSnapshot) { logInfo(log"Committing iceberg snapshot expiration for uniform table " + log"[path = ${MDC(DeltaLogKeys.PATH, log.logPath)}] tableId=" + log"${MDC(DeltaLogKeys.TABLE_ID, log.unsafeVolatileTableId)}]") expireIcebergSnapshot(snapshotToConvert, icebergTxn) } icebergTxn.commit() logInfo(s"icebergTxn committed for table ${Option(catalogTable).map(_.identifier)} " + s"with converted delta version ${snapshotToConvert.version}") recordDeltaEvent( snapshotToConvert.deltaLog, conversionContext.opType, data = Map( "deltaVersion" -> snapshotToConvert.version, "compatVersion" -> IcebergCompat.getEnabledVersion(snapshotToConvert.metadata) .getOrElse(0), "elapsedTimeMs" -> (System.currentTimeMillis() - conversionStartTime) ) ) icebergTxn } /** * Helper function to execute and commit Iceberg snapshot expiry * @param snapshotToConvert the Delta snapshot that needs to be converted to Iceberg * @param icebergTxn the IcebergConversionTransaction created in convertSnapshot, used * to create a table object and expiration helper */ private def expireIcebergSnapshot( snapshotToConvert: Snapshot, icebergTxn: IcebergConversionTransaction): Unit = { val expireSnapshotHelper = icebergTxn.getExpireSnapshotHelper() val table = icebergTxn.txn.table() val tableLocation = LocationUtil.stripTrailingSlash(table.location) val defaultWriteMetadataLocation = s"$tableLocation/metadata" val writeMetadataLocation = LocationUtil.stripTrailingSlash( table.properties().getOrDefault( TableProperties.WRITE_METADATA_LOCATION, defaultWriteMetadataLocation)) val shouldKeepPhysicalFiles = // Don't attempt any file cleanup in the edge-case configuration // that the data location (in Uniform the table root location) // is the same as the Iceberg metadata location (snapshotToConvert.path.toString == writeMetadataLocation) if (shouldKeepPhysicalFiles) { expireSnapshotHelper.cleanExpiredFiles(false) } else { expireSnapshotHelper.deleteWith(path => { if (path.startsWith(writeMetadataLocation)) { table.io().deleteFile(path) } }) } expireSnapshotHelper.commit(snapshotToConvert.version) } // This is for newly enabling uniform table to // start a new history line for iceberg metadata // so that if a uniform table is corrupted, // user can unset and re-enable to unblock private def cleanCatalogTableIfEnablingUniform( table: CatalogTable, snapshotToConvert: Snapshot, txnOpt: Option[CommittedTransaction]): CatalogTable = { val disabledIceberg = txnOpt.map(txn => !UniversalFormat.icebergEnabled(txn.readSnapshot.metadata) ).getOrElse(!UniversalFormat.icebergEnabled(table.properties)) val enablingUniform = disabledIceberg && UniversalFormat.icebergEnabled(snapshotToConvert.metadata) if (enablingUniform) { clearDeltaUniformMetadata(table) } else { table } } protected def clearDeltaUniformMetadata(table: CatalogTable): CatalogTable = { val metadata_key = IcebergConstants.ICEBERG_TBLPROP_METADATA_LOCATION if (table.properties.contains(metadata_key)) { val cleanedCatalogTable = table.copy(properties = table.properties - metadata_key - IcebergConverter.DELTA_VERSION_PROPERTY - IcebergConverter.DELTA_TIMESTAMP_PROPERTY ) spark.sessionState.catalog.alterTable(cleanedCatalogTable) cleanedCatalogTable } else { table } } override def loadLastDeltaVersionConverted( snapshot: Snapshot, catalogTable: CatalogTable): Option[Long] = recordFrameProfile("Delta", "IcebergConverter.loadLastDeltaVersionConverted") { IcebergConverter.getLastConvertedDeltaVersion(loadIcebergTable(snapshot, catalogTable)) } protected def loadIcebergTable( snapshot: Snapshot, catalogTable: CatalogTable): Option[IcebergTable] = { recordFrameProfile("Delta", "IcebergConverter.loadLastConvertedIcebergTable") { val hiveCatalog = IcebergTransactionUtils .createHiveCatalog(snapshot.deltaLog.newDeltaHadoopConf()) val icebergTableId = IcebergTransactionUtils .convertSparkTableIdentifierToIcebergHive(catalogTable.identifier) if (hiveCatalog.tableExists(icebergTableId)) { Some(hiveCatalog.loadTable(icebergTableId)) } else { None } } } /** * Commit the set of changes into an Iceberg snapshot. Each call to this function will * build exactly one Iceberg Snapshot. * * We determine what type of [[IcebergConversionTransaction.TransactionHelper]] to use * (and what type of Iceberg snapshot to create) based on the types of actions and * whether they contain data change. An [[UnsupportedOperationException]] will be * thrown for cases not listed in the table below. It means the combination of actions are * not recognized/supported. IcebergConverter will do a re-try with REPLACE TABLE, which * collects all valid data files from the target Delta snapshot and commit to Iceberg. * * Some Delta operations are known to contain only AddFiles(dataChange=false), intended to * replace/overwrite existing AddFiles. They rely on Delta's dedup in state reconstruction * and cannot be action-to-action translated to Iceberg, which lacks dedup abilities. * We create corresponding RemoveFile entries for the AddFiles so these operations can be * properly translated into [[RewriteFiles]] in Iceberg. These operations are marked as * [[needAutoRewrite]] in the code and the table below. * * The following table demonstrates how to choose the appropriate TransactionHelper. * The conditions can overlap and should be checked in order. * +-------------------+---------------+---------------------+--------------------+ * | Type of actions | Data Change | TransactionHelper | Example / Note | * +-------------------+---------------+---------------------+--------------------+ * | Create table | Any | AppendHelper | Note 1 | * +-------------------+---------------+---------------------+--------------------+ * | | All | AppendHelper | INSERT | * | Add only +---------------+---------------------+--------------------+ * | | None | needAutoRewrite | Note 2 | * | | | else | | * | | | NullHelper | Add Tag | * | +---------------+---------------------+--------------------+ * | | Some | Unsupported | (unknown) | * +-------------------+---------------+---------------------+--------------------+ * | Remove only | Any | RemoveHelper | DELETE | * +-------------------+---------------+---------------------+--------------------+ * | | All | OverwriteHelper | UPDATE | * | Add + Remove +---------------+---------------------+--------------------+ * | | None | RewriteHelper | OPTIMIZE | * | +---------------+---------------------+--------------------+ * | | Some | Unsupported | (unknown) | * +-------------------+---------------+---------------------+--------------------+ * Note: * 1. We assume a Create/Replace table operation will only contain AddFiles. * 2. DV is allowed but ignored as known operations (ComputeStats) do not touch DV. */ private[delta] def runIcebergConversionForActions( icebergTxn: IcebergConversionTransaction, actionsToCommit: Seq[Action], prevSnapshotOpt: Option[SnapshotDescriptor], deltaVersion: Long): Option[CommitInfo] = { var commitInfo: Option[CommitInfo] = None var addFiles: Seq[AddFile] = Nil var removeFiles: Seq[RemoveFile] = Nil // Determining what txnHelper to use for this group of Actions requires a full-scan // of [[actionsToCommit]], which is not too expensive as the actions are already in-memory. val txnHelper = prevSnapshotOpt match { // Having no previous Snapshot implies that the table is either being created or replaced. // This guarantees that the actions are fetched via [[Snapshot.allFiles]] and are unique. case None => addFiles = actionsToCommit.asInstanceOf[Seq[AddFile]] if (addFiles.isEmpty) { icebergTxn.getNullHelper } else if (addFiles.exists(_.deletionVector != null)) { throw new UnsupportedOperationException("Deletion Vector is not supported") } else { icebergTxn.getAppendOnlyHelper } case Some(_) => val addBuffer = new ArrayBuffer[AddFile]() val removeBuffer = new ArrayBuffer[RemoveFile]() // Scan the actions to collect info needed to determine which txnHelper to use object DataChange extends Enumeration { val Empty = Value(0, "Empty") val None = Value(1, "None") val All = Value(2, "All") val Some = Value(3, "Some") } var dataChangeBits = 0 var hasDv: Boolean = false val autoRewriteOprs = Set("COMPUTE STATS") var needAutoRewrite = false actionsToCommit.foreach { case file: FileAction => addBuffer ++= Option(file.wrap.add) removeBuffer ++= Option(file.wrap.remove) if (file.wrap.add != null || file.wrap.remove != null) { // We only care about data changes in add and remove actions dataChangeBits |= (1 << (if (file.dataChange) 1 else 0)) } hasDv |= file.deletionVector != null case c: CommitInfo => commitInfo = Some(c) needAutoRewrite = autoRewriteOprs.contains(c.operation) case _ => // Ignore other actions } addFiles = addBuffer.toSeq removeFiles = removeBuffer.toSeq val dataChange = DataChange(dataChangeBits) (addFiles.nonEmpty, removeFiles.nonEmpty, dataChange) match { case (true, false, DataChange.All) if !hasDv => icebergTxn.getAppendOnlyHelper case (true, false, DataChange.None) => if (!needAutoRewrite) { icebergTxn.getNullHelper // Ignore } else { // Create RemoveFiles to refresh these AddFiles without data change removeFiles = addBuffer.map(_.removeWithTimestamp(dataChange = false)).toSeq icebergTxn.getRewriteHelper } case (false, true, _) => icebergTxn.getRemoveOnlyHelper case (true, true, DataChange.All) if !hasDv => icebergTxn.getOverwriteHelper case (true, true, DataChange.None) if !hasDv => icebergTxn.getRewriteHelper case (false, false, _) => icebergTxn.getNullHelper case _ => recordDeltaEvent( targetSnapshot.deltaLog, "delta.iceberg.conversion.unsupportedActions", data = Map( "version" -> targetSnapshot.version, "commitInfo" -> commitInfo.map(_.operation).getOrElse(""), "hasAdd" -> addFiles.nonEmpty.toString, "hasRemove" -> removeFiles.nonEmpty.toString, "dataChange" -> dataChange.toString, "hasDv" -> hasDv.toString ) ) logError( s"""Unsupported combination of actions for incremental conversion. Context: |version -> ${targetSnapshot.version}, |commitInfo -> ${commitInfo.map(_.operation).getOrElse("")}, |hasAdd -> ${addFiles.nonEmpty.toString}, |hasRemove -> ${removeFiles.nonEmpty.toString}, |dataChange -> ${dataChange.toString}, |hasDv -> ${hasDv.toString}""".stripMargin) throw new UnsupportedOperationException( "Unsupported combination of actions for incremental conversion.") } } recordDeltaEvent( targetSnapshot.deltaLog, "delta.iceberg.conversion.convertActions", data = Map( "version" -> targetSnapshot.version, "commitInfo" -> commitInfo.map(_.operation).getOrElse(""), "txnHelper" -> txnHelper.getClass.getSimpleName ) ) removeFiles.foreach(txnHelper.add) addFiles.foreach(txnHelper.add) // Make sure the next snapshot sequence number is deltaVersion txnHelper.commit(deltaVersion) commitInfo } /** * Validate the Iceberg conversion by comparing the number of files and size in bytes * between the converted Iceberg table and the Delta table. * TODO: throw exception and proactively abort conversion transaction */ private def validateIcebergCommit(snapshotToConvert: Snapshot, catalogTable: CatalogTable) = { val table = loadIcebergTable(snapshotToConvert, catalogTable) val lastConvertedDeltaVersion = IcebergConverter.getLastConvertedDeltaVersion(table) table.map {t => if (lastConvertedDeltaVersion.contains(snapshotToConvert.version) && t.currentSnapshot() != null) { val icebergNumOfFiles = t.currentSnapshot().summary().asScala .getOrElse("total-data-files", "-1").toLong val icebergTotalBytes = t.currentSnapshot().summary().asScala .getOrElse("total-files-size", "-1").toLong if (icebergNumOfFiles != snapshotToConvert.numOfFiles || icebergTotalBytes != snapshotToConvert.sizeInBytes) { recordDeltaEvent( snapshotToConvert.deltaLog, "delta.iceberg.conversion.mismatch", data = Map( "lastConvertedDeltaVersion" -> snapshotToConvert.version, "numOfFiles" -> snapshotToConvert.numOfFiles, "icebergNumOfFiles" -> icebergNumOfFiles, "sizeInBytes" -> snapshotToConvert.sizeInBytes, "icebergTotalBytes" -> icebergTotalBytes ) ) } } } } } ================================================ FILE: iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergSchemaUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.icebergShaded import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.{DeltaColumnMapping, SnapshotDescriptor} import org.apache.spark.sql.delta.actions.Protocol import org.apache.spark.sql.delta.metering.DeltaLogging import shadedForDelta.org.apache.iceberg.{Schema => IcebergSchema} import shadedForDelta.org.apache.iceberg.types.{Type => IcebergType, Types => IcebergTypes} import org.apache.spark.sql.types._ trait IcebergSchemaUtils extends DeltaLogging { import IcebergSchemaUtils._ ///////////////// // Public APIs // ///////////////// // scalastyle:off line.size.limit /** * Delta types are defined here: https://github.com/delta-io/delta/blob/master/PROTOCOL.md#schema-serialization-format * * Iceberg types are defined here: https://iceberg.apache.org/spec/#schemas-and-data-types */ // scalastyle:on line.size.limit def convertDeltaSchemaToIcebergSchema(deltaSchema: StructType): IcebergSchema = { val icebergStruct = convertStruct(deltaSchema) new IcebergSchema(icebergStruct.fields()) } def maxFieldId(snapshot: SnapshotDescriptor): Int //////////////////// // Helper Methods // //////////////////// protected def getFieldId(field: Option[StructField]): Int private[delta] def getNestedFieldId(field: Option[StructField], path: Seq[String]): Int /** Visible for testing */ private[delta] def convertStruct(deltaSchema: StructType)( implicit compatVersion: Int = 0): IcebergTypes.StructType = { /** * Recursively (i.e. for all nested elements) transforms the delta DataType `elem` into its * corresponding Iceberg type. * * - StructType -> IcebergTypes.StructType * - ArrayType -> IcebergTypes.ListType * - MapType -> IcebergTypes.MapType * - primitive -> IcebergType.PrimitiveType */ def transform[E <: DataType](elem: E, field: Option[StructField], name: Seq[String]) : IcebergType = elem match { case StructType(fields) => IcebergTypes.StructType.of(fields.map { f => val icebergField = IcebergTypes.NestedField.of( getFieldId(Some(f)), f.nullable, f.name, transform(f.dataType, Some(f), Seq(DeltaColumnMapping.getPhysicalName(f))), f.getComment().orNull ) // Translate column default value if (compatVersion >= 3) { DeltaToIcebergConvert.Schema.extractLiteralDefault(f) match { case Left(errorMsg) => throw new UnsupportedOperationException(errorMsg) case _ => icebergField } } else { icebergField } }.toList.asJava) case ArrayType(elementType, containsNull) => val currName = name :+ DeltaColumnMapping.PARQUET_LIST_ELEMENT_FIELD_NAME val id = getNestedFieldId(field, currName) if (containsNull) { IcebergTypes.ListType.ofOptional(id, transform(elementType, field, currName)) } else { IcebergTypes.ListType.ofRequired(id, transform(elementType, field, currName)) } case MapType(keyType, valueType, valueContainsNull) => val currKeyName = name :+ DeltaColumnMapping.PARQUET_MAP_KEY_FIELD_NAME val currValName = name :+ DeltaColumnMapping.PARQUET_MAP_VALUE_FIELD_NAME val keyId = getNestedFieldId(field, currKeyName) val valId = getNestedFieldId(field, currValName) if (valueContainsNull) { IcebergTypes.MapType.ofOptional( keyId, valId, transform(keyType, field, currKeyName), transform(valueType, field, currValName) ) } else { IcebergTypes.MapType.ofRequired( keyId, valId, transform(keyType, field, currKeyName), transform(valueType, field, currValName) ) } case atomicType: AtomicType => convertAtomic(atomicType) case other => throw new UnsupportedOperationException(s"Cannot convert Delta type $other to Iceberg") } transform(deltaSchema, None, Seq.empty).asStructType() } } object IcebergSchemaUtils { /** * Creates a schema utility for Delta to Iceberg schema conversion. * @param icebergDefaultNameMapping: whether to generate schemas for Iceberg default name mapping, * where the column name is the ground of truth. * @return an Iceberg schema utility. */ def apply(icebergDefaultNameMapping: Boolean = false): IcebergSchemaUtils = { if (icebergDefaultNameMapping) new IcebergSchemaUtilsNameMapping() else new IcebergSchemaUtilsIdMapping() } private class IcebergSchemaUtilsNameMapping() extends IcebergSchemaUtils { // Dummy field ID to support Delta table with NoMapping mode, where logical column name is the // ground of truth and no column Id is available. private var dummyId: Int = 1 def maxFieldId(snapshot: SnapshotDescriptor): Int = dummyId def getFieldId(field: Option[StructField]): Int = { val fieldId = dummyId dummyId += 1 fieldId } def getNestedFieldId(field: Option[StructField], path: Seq[String]): Int = getFieldId(field) } private class IcebergSchemaUtilsIdMapping() extends IcebergSchemaUtils { def maxFieldId(snapshot: SnapshotDescriptor): Int = snapshot.metadata.columnMappingMaxId.toInt def getFieldId(field: Option[StructField]): Int = { if (!field.exists(f => DeltaColumnMapping.hasColumnId(f))) { throw new UnsupportedOperationException("UniForm requires Column Mapping") } DeltaColumnMapping.getColumnId(field.get) } def getNestedFieldId(field: Option[StructField], path: Seq[String]): Int = { field.get.metadata .getMetadata(DeltaColumnMapping.COLUMN_MAPPING_METADATA_NESTED_IDS_KEY) .getLong(path.mkString(".")) .toInt } } /** * Converts delta atomic into an iceberg primitive. * * Visible for testing. * * https://github.com/delta-io/delta/blob/master/PROTOCOL.md#primitive-types */ private[delta] def convertAtomic[E <: DataType](elem: E): IcebergType.PrimitiveType = elem match { case StringType => IcebergTypes.StringType.get() case LongType => IcebergTypes.LongType.get() case IntegerType | ShortType | ByteType => IcebergTypes.IntegerType.get() case FloatType => IcebergTypes.FloatType.get() case DoubleType => IcebergTypes.DoubleType.get() case d: DecimalType => IcebergTypes.DecimalType.of(d.precision, d.scale) case BooleanType => IcebergTypes.BooleanType.get() case BinaryType => IcebergTypes.BinaryType.get() case DateType => IcebergTypes.DateType.get() case TimestampType => IcebergTypes.TimestampType.withZone() case TimestampNTZType => IcebergTypes.TimestampType.withoutZone() case _ => throw new UnsupportedOperationException(s"Could not convert atomic type $elem") } } ================================================ FILE: iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergStatsConverter.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.icebergShaded import java.lang.{Long => JLong} import java.nio.ByteBuffer import org.apache.spark.sql.delta.DeltaColumnMapping import org.apache.spark.sql.delta.stats.{DeltaStatistics, SkippingEligibleDataType} import shadedForDelta.org.apache.iceberg.types.Conversions import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String /** * Converts Delta stats to Iceberg stats given an Internal Row representing Delta stats and the * row's schema. * * Iceberg stores stats as a map from column ID to the statistic. For example, lower/upper bound * statistics are represented as a map from column ID to byte buffer where the byte buffer stores * any type. * * For example, given the following Delta stats schema with column IDs: * | -- id(0): INT * | -- person(1): STRUCT * | name(2): STRUCT * | -- first(3): STRING * | -- last(4): STRING * | height(5): LONG * * Iceberg's upper bound statistic map will be: * {0 -> MAX_ID, 3 -> MAX_FIRST, 4 -> MAX_LAST, 5 -> MAX_HEIGHT} * * Iceberg requires the "record count" stat while the "upper bounds", "lower bounds", and * "null value counts" are optional. See iceberg/DataFile.java. * Iceberg's "record count" metric is set in `convertFileAction` before the stats conversion. * If additional metrics are attached to the Iceberg data file, the "record count" metric must be * left non-null. */ case class IcebergStatsConverter(statsRow: InternalRow, statsSchema: StructType) { val numRecordsStat: JLong = statsSchema.getFieldIndex(DeltaStatistics.NUM_RECORDS) match { case Some(fieldIndex) => new JLong(statsRow.getLong(fieldIndex)) case None => throw new IllegalArgumentException("Delta is missing the 'num records' stat. " + "Iceberg requires this stat when attaching statistics to the output data file.") } val lowerBoundsStat: Option[Map[Integer, ByteBuffer]] = getByteBufferBackedColStats(DeltaStatistics.MIN) val upperBoundsStat: Option[Map[Integer, ByteBuffer]] = getByteBufferBackedColStats(DeltaStatistics.MAX) val nullValueCountsStat: Option[Map[Integer, JLong]] = statsSchema.getFieldIndex(DeltaStatistics.NULL_COUNT) match { case Some(nullCountFieldIdx) => val nullCountStatSchema = statsSchema.fields(nullCountFieldIdx).dataType.asInstanceOf[StructType] Some( generateIcebergLongMetricMap( statsRow.getStruct(nullCountFieldIdx, nullCountStatSchema.fields.length), nullCountStatSchema ) ) case None => None } /** * Generates Iceberg's metric representation by recursively flattening the Delta stat struct * (represented as an internal row) and converts the column's physical name to its ID. * * Ignores null Delta stats. * * @param stats An internal row holding the `ByteBuffer`-based Delta column stats * (i.e. lower bound). * @param statsSchema The schema of the `stats` internal row. * @return Iceberg's ByteBuffer-backed metric representation. */ private def generateIcebergByteBufferMetricMap( stats: InternalRow, statsSchema: StructType): Map[Integer, ByteBuffer] = { // If the entire Delta stats struct is missing (for example, min or max values are missing for // all columns), then the stats row may be null. if (stats == null) return Map.empty statsSchema.fields.zipWithIndex.flatMap { case (field, idx) => field.dataType match { // Iceberg statistics cannot be null. case _ if stats.isNullAt(idx) => Map[Integer, ByteBuffer]().empty // If the stats schema contains a struct type, there is a corresponding struct in the data // schema. The struct's per-field stats are also stored in the Delta stats struct. See the // `StatisticsCollection` trait comment for more. case st: StructType => generateIcebergByteBufferMetricMap(stats.getStruct(idx, st.fields.length), st) // Ignore the Delta statistic if the conversion doesn't support the given data type or the // column ID for this field is missing. case dt if !DeltaColumnMapping.hasColumnId(field) || !IcebergStatsConverter.isMinMaxStatTypeSupported(dt) => Map[Integer, ByteBuffer]().empty case b: ByteType => // Iceberg stores bytes using integers. val statVal = stats.getByte(idx).toInt Map[Integer, ByteBuffer](Integer.valueOf(DeltaColumnMapping.getColumnId(field)) -> Conversions.toByteBuffer(IcebergSchemaUtils.convertAtomic(b), statVal)) case s: ShortType => // Iceberg stores shorts using integers. val statVal = stats.getShort(idx).toInt Map[Integer, ByteBuffer](Integer.valueOf(DeltaColumnMapping.getColumnId(field)) -> Conversions.toByteBuffer(IcebergSchemaUtils.convertAtomic(s), statVal)) case dt if IcebergStatsConverter.isMinMaxStatTypeSupported(dt) => val statVal = stats.get(idx, dt) // Iceberg's `Conversions.toByteBuffer` method expects the Java object representation // for string and decimal types. // Other types supported by Delta's min/max stat such as int, long, boolean, etc., do not // require a different representation. val compatibleStatsVal = statVal match { case u: UTF8String => u.toString case d: Decimal => d.toJavaBigDecimal case _ => statVal } Map[Integer, ByteBuffer](Integer.valueOf(DeltaColumnMapping.getColumnId(field)) -> Conversions.toByteBuffer(IcebergSchemaUtils.convertAtomic(dt), compatibleStatsVal)) } }.toMap } /** * Generates Iceberg's metric representation by recursively flattening the Delta stat struct * (represented as an internal row) and converts the column's physical name to its ID. * * @param stats An internal row holding the long-backed Delta column stats (i.e. null counts). * @param statsSchema The schema of the `stats` internal row. * @return a map in Iceberg's metric representation. */ private def generateIcebergLongMetricMap( stats: InternalRow, statsSchema: StructType): Map[Integer, JLong] = { // If the entire Delta stats struct is missing, then the Iceberg stats would be empty map. if (stats == null) return Map.empty statsSchema.fields.zipWithIndex.flatMap { case (field, idx) => field.dataType match { // If the stats schema contains a struct type, there is a corresponding struct in the data // schema. The struct's per-field stats are also stored in the Delta stats struct. See the // `StatisticsCollection` trait comment for more. case st: StructType => generateIcebergLongMetricMap(stats.getStruct(idx, st.fields.length), st) case lt: LongType => // Skip null values - InternalRow.getLong returns 0 for nulls, which would incorrectly // add 0 to Iceberg stats instead of omitting them if (!stats.isNullAt(idx) && DeltaColumnMapping.hasColumnId(field)) { Map[Integer, JLong](Integer.valueOf(DeltaColumnMapping.getColumnId(field)) -> new JLong(stats.getLong(idx))) } else { Map[Integer, JLong]().empty } case _ => throw new UnsupportedOperationException("Expected metric to be a long type.") } }.toMap } /** * @param statName The name of the Delta stat that is being converted. Must be one of the field * names in the `DeltaStatistics` object. * @return An option holding Iceberg's statistic representation. Returns `None` if the output * would otherwise be empty. */ private def getByteBufferBackedColStats(statName: String): Option[Map[Integer, ByteBuffer]] = { statsSchema.getFieldIndex(statName) match { case Some(statFieldIdx) => val colStatSchema = statsSchema.fields(statFieldIdx).dataType.asInstanceOf[StructType] val icebergMetricsMap = generateIcebergByteBufferMetricMap( statsRow.getStruct(statFieldIdx, colStatSchema.fields.length), colStatSchema ) if (icebergMetricsMap.nonEmpty) { Some(icebergMetricsMap) } else { // The iceberg metrics map may be empty when all Delta stats are null. None } case None => None } } } object IcebergStatsConverter { /** * Returns true if a min/max statistic of the given Delta data type can be converted into an * Iceberg metric of equivalent data type. * * Currently, nested types and null types are unsupported. */ def isMinMaxStatTypeSupported(dt: DataType): Boolean = { if (!SkippingEligibleDataType(dt)) return false dt match { case _: StringType | _: IntegerType | _: FloatType | _: DoubleType | _: DoubleType | _: DecimalType | _: BooleanType | _: DateType | _: TimestampType | // _: LongType TODO: enable after https://github.com/apache/spark/pull/42083 is released _: TimestampNTZType | _: ByteType | _: ShortType => true case _ => false } } } ================================================ FILE: iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergTransactionUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.icebergShaded import java.nio.ByteBuffer import java.time.Instant import java.time.format.DateTimeParseException import scala.collection.JavaConverters._ import scala.util.control.NonFatal import org.apache.spark.sql.delta.{DeltaColumnMapping, DeltaErrors, Snapshot} import org.apache.spark.sql.delta.actions.{AddFile, FileAction, RemoveFile} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.util.PartitionUtils.{timestampPartitionPattern, utcFormatter} import org.apache.spark.sql.delta.util.TimestampFormatter import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import shadedForDelta.org.apache.iceberg.{DataFile, DataFiles, FileFormat, MetadataUpdate, PartitionSpec, Schema => IcebergSchema} import shadedForDelta.org.apache.iceberg.Metrics import shadedForDelta.org.apache.iceberg.StructLike import shadedForDelta.org.apache.iceberg.catalog.{Namespace, TableIdentifier => IcebergTableIdentifier} import shadedForDelta.org.apache.iceberg.hive.HiveCatalog import shadedForDelta.org.apache.iceberg.util.DateTimeUtil import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier => SparkTableIdentifier} import org.apache.spark.sql.types.{BinaryType, BooleanType, ByteType, DataType, DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType, ShortType, StringType, StructType, TimestampNTZType, TimestampType} object IcebergTransactionUtils extends DeltaLogging { ///////////////// // Public APIs // ///////////////// def createPartitionSpec( icebergSchema: IcebergSchema, partitionColumns: Seq[String]): PartitionSpec = { if (partitionColumns.isEmpty) { PartitionSpec.unpartitioned } else { val builder = PartitionSpec.builderFor(icebergSchema) for (partitionName <- partitionColumns) { builder.identity(partitionName) } builder.build() } } /** * We expose this as a public API since APIs like * [[shadedForDelta.org.apache.iceberg.DeleteFiles#deleteFile]] actually only need to take in * a file path String, thus we don't need to actually convert a [[RemoveFile]] into a [[DataFile]] * in this case. */ def canonicalizeFilePath(f: FileAction, tablePath: Path): String = { // Recall that FileActions can have either relative paths or absolute paths (i.e. from shallow- // cloned files). // Iceberg spec requires path be fully qualified path, suitable for constructing a Hadoop Path if (f.pathAsUri.isAbsolute) new Path(f.pathAsUri).toString else new Path(tablePath, f.toPath.toString).toString } /** Returns the mapping of logicalPartitionColName -> physicalPartitionColName */ def getPartitionPhysicalNameMapping(partitionSchema: StructType): Map[String, String] = { partitionSchema.fields.map(f => f.name -> DeltaColumnMapping.getPhysicalName(f)).toMap } class Row (val values: Array[Any]) extends StructLike { override def size: Int = values.length override def get[T <: Any](pos: Int, javaClass: Class[T]): T = javaClass.cast(values(pos)) override def set[T <: Any](pos: Int, value: T): Unit = { values(pos) = value } } //////////////////// // Helper Methods // //////////////////// /** Visible for testing. */ private[delta] def convertDeltaAddFileToIcebergDataFile( add: AddFile, tablePath: Path, partitionSpec: PartitionSpec, logicalToPhysicalPartitionNames: Map[String, String], statsParser: String => InternalRow, snapshot: Snapshot): DataFile = { var dataFileBuilder = convertFileAction( add, tablePath, partitionSpec, logicalToPhysicalPartitionNames, snapshot) // Attempt to attach the number of records metric regardless of whether the Delta stats // string is null/empty or not because this metric is required by Iceberg. If the number // of records is both unavailable here and unavailable in the Delta stats, Iceberg will // throw an exception when building the data file. .withRecordCount(add.numLogicalRecords.getOrElse(-1L)) try { if (add.stats != null && add.stats.nonEmpty) { dataFileBuilder = dataFileBuilder.withMetrics( getMetricsForIcebergDataFile(statsParser, add.stats, snapshot.statsSchema)) } } catch { case NonFatal(e) => logWarning(log"Failed to convert Delta stats to Iceberg stats. Iceberg conversion will " + "attempt to proceed without stats.", e) } dataFileBuilder.build() } private[delta] def convertDeltaRemoveFileToIcebergDataFile( remove: RemoveFile, tablePath: Path, partitionSpec: PartitionSpec, logicalToPhysicalPartitionNames: Map[String, String], snapshot: Snapshot): DataFile = { convertFileAction( remove, tablePath, partitionSpec, logicalToPhysicalPartitionNames, snapshot) .withRecordCount(remove.numLogicalRecords.getOrElse(0L)) .build() } private[delta] def convertFileAction( f: FileAction, tablePath: Path, partitionSpec: PartitionSpec, logicalToPhysicalPartitionNames: Map[String, String], snapshot: Snapshot): DataFiles.Builder = { val absPath = canonicalizeFilePath(f, tablePath) var builder = DataFiles .builder(partitionSpec) .withPath(absPath) .withFileSizeInBytes(f.getFileSize) .withFormat(FileFormat.PARQUET) if (partitionSpec.isPartitioned) { builder = builder.withPartition( DeltaToIcebergConvert.Partition.convertPartitionValues( snapshot, partitionSpec, f.partitionValues, logicalToPhysicalPartitionNames)) } builder } private lazy val timestampFormatter = TimestampFormatter(timestampPartitionPattern, java.util.TimeZone.getDefault) /** * Follows deserialization as specified here * https://github.com/delta-io/delta/blob/master/PROTOCOL.md#Partition-Value-Serialization */ private[delta] def stringToIcebergPartitionValue( elemType: DataType, partitionVal: String, version: Long): Any = { if (partitionVal == null || partitionVal == "__HIVE_DEFAULT_PARTITION__") { return null } elemType match { case _: StringType => partitionVal case _: DateType => java.sql.Date.valueOf(partitionVal).toLocalDate.toEpochDay.asInstanceOf[Int] case _: IntegerType => partitionVal.toInt.asInstanceOf[Integer] case _: ShortType => partitionVal.toInt.asInstanceOf[Integer] case _: ByteType => partitionVal.toInt.asInstanceOf[Integer] case _: LongType => partitionVal.toLong case _: BooleanType => partitionVal.toBoolean case _: FloatType => partitionVal.toFloat case _: DoubleType => partitionVal.toDouble case _: DecimalType => new java.math.BigDecimal(partitionVal) case _: BinaryType => ByteBuffer.wrap(partitionVal.getBytes("UTF-8")) case _: TimestampNTZType => DateTimeUtil.isoTimestampToMicros( partitionVal.replace(" ", "T")) case _: TimestampType => try { getMicrosSinceEpoch(partitionVal) } catch { case _: DateTimeParseException => // In case of non-ISO timestamps, parse and interpret the timestamp as system time // and then convert to UTC val utcInstant = utcFormatter.format(timestampFormatter.parse(partitionVal)) getMicrosSinceEpoch(utcInstant) } case _ => throw DeltaErrors.universalFormatConversionFailedException( version, "iceberg", "Unexpected partition data type " + elemType) } } private def getMicrosSinceEpoch(instant: String): Long = { DateTimeUtil.microsFromInstant( Instant.parse(instant)) } private def getMetricsForIcebergDataFile( statsParser: String => InternalRow, stats: String, statsSchema: StructType): Metrics = { val statsRow = statsParser(stats) val metricsConverter = IcebergStatsConverter(statsRow, statsSchema) new Metrics( metricsConverter.numRecordsStat, // rowCount null, // columnSizes null, // valueCounts metricsConverter.nullValueCountsStat.getOrElse(null).asJava, // nullValueCounts null, // nanValueCounts metricsConverter.lowerBoundsStat.getOrElse(null).asJava, // lowerBounds metricsConverter.upperBoundsStat.getOrElse(null).asJava // upperBounds ) } /** * Create an Iceberg HiveCatalog * @param conf: Hadoop Configuration * @return */ def createHiveCatalog( conf: Configuration, metadataUpdates: java.util.ArrayList[MetadataUpdate] = new java.util.ArrayList[MetadataUpdate]()) : HiveCatalog = { val catalog = new HiveCatalog() catalog.setConf(conf) catalog.initialize("spark_catalog", Map.empty[String, String].asJava, metadataUpdates) catalog } /** * Encode Spark table identifier to Iceberg table identifier by putting "database" and "catalog" * to the "namespace" in Iceberg table identifier */ def convertSparkTableIdentifierToIceberg( identifier: SparkTableIdentifier): IcebergTableIdentifier = { val namespace = (identifier.database, identifier.catalog) match { case (Some(database), Some(catalog)) => Namespace.of(database, catalog) case (Some(database), None) => Namespace.of(database) case (None, Some(catalog)) => throw new IllegalArgumentException( "Spark does not allow the constructors to skip the `database` when `catalog` is used" ) case (None, None) => Namespace.empty() } IcebergTableIdentifier.of(namespace, identifier.table) } /** * Encode Spark table identifier to Iceberg table identifier by putting * only "database" to the "namespace" in Iceberg table identifier. * See [[HiveCatalog.isValidateNamespace]] */ def convertSparkTableIdentifierToIcebergHive( identifier: SparkTableIdentifier): IcebergTableIdentifier = { val namespace = (identifier.database) match { case Some(database) => Namespace.of(database) case _ => Namespace.empty() } IcebergTableIdentifier.of(namespace, identifier.table) } } ================================================ FILE: iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.serverSidePlanning import java.io.IOException import java.lang.reflect.Method import java.util.Locale import scala.jdk.CollectionConverters._ import scala.util.Try import org.apache.hadoop.conf.Configuration import org.apache.http.client.methods.{HttpGet, HttpPost} import org.apache.http.entity.{ContentType, StringEntity} import org.apache.http.util.EntityUtils import org.apache.http.{HttpHeaders, HttpResponse, HttpStatus} import org.apache.http.client.ServiceUnavailableRetryStrategy import org.apache.http.impl.client.{DefaultHttpRequestRetryHandler, HttpClientBuilder} import org.apache.http.protocol.HttpContext import org.apache.http.message.BasicHeader import org.apache.spark.internal.Logging import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.sql.sources.Filter import org.apache.spark.sql.types.StructType import org.apache.spark.util.Utils import org.json4s._ import org.json4s.jackson.JsonMethods._ import shadedForDelta.org.apache.iceberg.PartitionSpec import shadedForDelta.org.apache.iceberg.expressions.Expressions import shadedForDelta.org.apache.iceberg.rest.requests.{PlanTableScanRequest, PlanTableScanRequestParser} import shadedForDelta.org.apache.iceberg.rest.responses.PlanTableScanResponse /** * Case class for parsing Iceberg REST catalog /v1/config response. * Per the Iceberg REST spec, the config endpoint returns defaults and overrides. * The optional "prefix" in overrides is used for multi-tenant catalog paths. */ private case class CatalogConfigResponse( defaults: Map[String, String], overrides: Map[String, String]) /** * Iceberg REST implementation of ServerSidePlanningClient that calls Iceberg REST catalog server. * * This implementation calls the Iceberg REST catalog's `/plan` endpoint to perform server-side * scan planning. The server returns the list of data files to read, which eliminates the need * for client-side listing operations. * * Thread safety: This class creates a shared HTTP client that is thread-safe for concurrent * requests. The HTTP client should be explicitly closed by calling close() when done. * * @param baseUriRaw Base URI of the Iceberg REST catalog up to /v1, e.g., * "http:///iceberg/v1". Trailing slashes are handled automatically. * @param catalogName Name of the catalog for config endpoint query parameter. * @param token Authentication token for the catalog server. */ class IcebergRESTCatalogPlanningClient( baseUriRaw: String, catalogName: String, token: String) extends ServerSidePlanningClient with Logging { // Normalize baseUri to handle trailing slashes private val baseUri = baseUriRaw.stripSuffix("/") // Sentinel value indicating "use current snapshot" in Iceberg REST API private val CURRENT_SNAPSHOT_ID = 0L // Partition spec ID for unpartitioned tables private val UNPARTITIONED_SPEC_ID = 0 // IRC config key mappings for each credential type private val S3_KEYS = Seq("s3.access-key-id", "s3.secret-access-key", "s3.session-token") private val AZURE_SAS_TOKEN_KEY_PREFIX = "adls.sas-token." private val GCS_TOKEN_KEY = "gcs.oauth2.token" private val GCS_EXPIRY_KEY = "gcs.oauth2.token-expires-at" private case class S3Credentials( accessKeyId: String, secretAccessKey: String, sessionToken: String) extends ScanPlanStorageCredentials { override def configure(conf: Configuration): Unit = { conf.set("fs.s3a.path.style.access", "true") conf.set("fs.s3.impl.disable.cache", "true") conf.set("fs.s3a.impl.disable.cache", "true") conf.set("fs.s3a.access.key", accessKeyId) conf.set("fs.s3a.secret.key", secretAccessKey) conf.set("fs.s3a.session.token", sessionToken) } } private case class AzureCredentials( accountName: String, sasToken: String) extends ScanPlanStorageCredentials { override def configure(conf: Configuration): Unit = { val accountSuffix = s"$accountName.dfs.core.windows.net" conf.set("fs.abfs.impl.disable.cache", "true") conf.set("fs.abfss.impl.disable.cache", "true") conf.set(s"fs.azure.account.auth.type.$accountSuffix", "SAS") conf.set(s"fs.azure.sas.fixed.token.$accountSuffix", sasToken) } } private case class GcsCredentials( oauth2Token: String, expirationEpochMs: Option[Long] = None) extends ScanPlanStorageCredentials { override def configure(conf: Configuration): Unit = { conf.set("fs.gs.impl.disable.cache", "true") conf.set("fs.gs.auth.type", "ACCESS_TOKEN_PROVIDER") conf.set("fs.gs.auth.access.token.provider.impl", classOf[FixedGcsAccessTokenProvider].getName) conf.set("fs.gs.auth.access.token", oauth2Token) expirationEpochMs.foreach { ms => conf.set("fs.gs.auth.access.token.expiration.ms", ms.toString) } } } private def hasAzureKeys(config: Map[String, String]): Boolean = config.keys.exists(_.startsWith(AZURE_SAS_TOKEN_KEY_PREFIX)) private def buildAzureCredentials(config: Map[String, String]): AzureCredentials = { val sasTokenKey = config.keys .find(_.startsWith(AZURE_SAS_TOKEN_KEY_PREFIX)) .getOrElse(throw new IllegalStateException( s"Missing Azure SAS token key starting with: $AZURE_SAS_TOKEN_KEY_PREFIX")) val accountName = sasTokenKey .stripPrefix(AZURE_SAS_TOKEN_KEY_PREFIX) .stripSuffix(".dfs.core.windows.net") val sasToken = config(sasTokenKey) AzureCredentials(accountName = accountName, sasToken = sasToken) } private def fromConfig(config: Map[String, String]): ScanPlanStorageCredentials = { def get(key: String): String = config.getOrElse(key, throw new IllegalStateException(s"Missing required credential: $key")) def hasAny(keys: Seq[String]): Boolean = keys.exists(config.contains) if (hasAny(S3_KEYS)) { S3Credentials( get("s3.access-key-id"), get("s3.secret-access-key"), get("s3.session-token")) } else if (hasAzureKeys(config)) { buildAzureCredentials(config) } else if (config.contains(GCS_TOKEN_KEY)) { val token = get(GCS_TOKEN_KEY) val expirationEpochMs = config.get(GCS_EXPIRY_KEY) .flatMap(s => scala.util.Try(s.toLong).toOption) GcsCredentials(token, expirationEpochMs) } else { throw new IllegalStateException( "Unrecognized credential keys. " + "Expected S3 (s3.*), Azure (adls.*), or GCS (gcs.*) properties.") } } /** * Lazily fetch the catalog configuration and construct the endpoint URI root. * Calls /v1/config?warehouse= per Iceberg REST catalog spec to get the prefix. * If no prefix is returned, uses baseUri directly without any prefix per Iceberg spec. */ private lazy val icebergRestCatalogUriRoot: String = { fetchCatalogPrefix() match { case Some(prefix) => s"$baseUri/$prefix" case None => baseUri } } /** * Fetch catalog prefix from /v1/config endpoint per Iceberg REST catalog spec. * Returns None on any error or if no prefix is defined in the config. */ private def fetchCatalogPrefix(): Option[String] = { val configUri = s"$baseUri/config?warehouse=$catalogName" try { val httpGet = new HttpGet(configUri) val response = httpClient.execute(httpGet) try { if (response.getStatusLine.getStatusCode == HttpStatus.SC_OK) { val body = EntityUtils.toString(response.getEntity) val config = JsonUtils.fromJson[CatalogConfigResponse](body) // Apply overrides on top of defaults per Iceberg REST spec config.overrides.get("prefix").orElse(config.defaults.get("prefix")) } else { None } } finally { response.close() } } catch { case e: Exception => logWarning(s"Failed to fetch catalog prefix from $configUri. " + s"Falling back to base URI. Error: ${e.getMessage}") None } } private val httpHeaders = { val baseHeaders = Map( HttpHeaders.ACCEPT -> ContentType.APPLICATION_JSON.getMimeType, HttpHeaders.CONTENT_TYPE -> ContentType.APPLICATION_JSON.getMimeType, HttpHeaders.USER_AGENT -> buildUserAgent() ) // Add Bearer token authentication if token is provided val headersWithAuth = if (token.nonEmpty) { baseHeaders + (HttpHeaders.AUTHORIZATION -> s"Bearer $token") } else { baseHeaders } headersWithAuth.map { case (k, v) => new BasicHeader(k, v) }.toSeq.asJava } /** * Build User-Agent header with Delta, Spark, Java and Scala version information. * Format: "Delta/ Spark/ Java/ Scala/" * Example: "Delta/4.0.0 Spark/3.5.0 Java/17.0.10 Scala/2.12.18" */ private def buildUserAgent(): String = { val deltaVersion = getDeltaVersion().getOrElse("unknown") val sparkVersion = getSparkVersion().getOrElse("unknown") val javaVersion = getJavaVersion() val scalaVersion = getScalaVersion() s"Delta/$deltaVersion Spark/$sparkVersion Java/$javaVersion Scala/$scalaVersion" } /** * Get the User-Agent header value used by this client. * Format: "Delta/ Spark/ Java/ Scala/" * * @return The User-Agent string used in HTTP requests */ def getUserAgent(): String = { buildUserAgent() } /** * Get Spark version. Returns None if Spark version cannot be determined. */ private def getSparkVersion(): Option[String] = { try { val packageClass = Utils.classForName("org.apache.spark.package$") val moduleField = packageClass.getField("MODULE$") val moduleObj = moduleField.get(null) val versionObj = packageClass.getMethod("SPARK_VERSION").invoke(moduleObj) if (versionObj != null) { Some(versionObj.toString) } else { None } } catch { case _: Exception => None } } /** * Get Delta version. Returns None if Delta is not available or version cannot be determined. */ private def getDeltaVersion(): Option[String] = { // Try io.delta.Version.getVersion() first (preferred method) try { val versionClass = Utils.classForName("io.delta.Version") val versionObj = versionClass.getMethod("getVersion").invoke(null) if (versionObj != null) { return Some(versionObj.toString) } } catch { case _: Exception => // Fall through to fallback } // Fall back to io.delta.VERSION constant try { val packageClass = Utils.classForName("io.delta.package$") val moduleField = packageClass.getField("MODULE$") val moduleObj = moduleField.get(null) val versionObj = packageClass.getMethod("VERSION").invoke(moduleObj) if (versionObj != null) { return Some(versionObj.toString) } } catch { case _: Exception => // Delta not available or version not accessible } None } /** * Get Java version from system properties. */ private def getJavaVersion(): String = { System.getProperty("java.version", "unknown") } /** * Get Scala version from the scala.util.Properties.versionNumberString property. */ private def getScalaVersion(): String = { scala.util.Properties.versionNumberString } // Maximum number of retries for transient HTTP failures (IOException, 5xx server errors) private val HTTP_MAX_RETRIES = 3 private lazy val httpClient = HttpClientBuilder.create() .setDefaultHeaders(httpHeaders) .setConnectionTimeToLive(30, java.util.concurrent.TimeUnit.SECONDS) // requestSentRetryEnabled=true: safe to retry already-sent requests because // planScan is a read-only operation (idempotent POST to /plan endpoint) .setRetryHandler(new DefaultHttpRequestRetryHandler(HTTP_MAX_RETRIES, true)) .setServiceUnavailableRetryStrategy(new ServerErrorRetryStrategy(HTTP_MAX_RETRIES)) .build() override def canConvertFilters(filters: Array[Filter]): Boolean = { // Check if all filters can be converted to Iceberg expressions // Returns true only if ALL filters successfully convert filters.forall { filter => SparkToIcebergExpressionConverter.convert(filter).isDefined } } override def planScan( database: String, table: String, sparkFilterOption: Option[Filter] = None, sparkProjectionOption: Option[Seq[String]] = None, sparkLimitOption: Option[Int] = None): ScanPlan = { // Construct the /plan endpoint URI. For Unity Catalog tables, the // Call /v1/config to get the catalog prefix, then construct the full endpoint. // icebergRestCatalogUriRoot is lazily constructed as: {baseUri}/{prefix} // where prefix comes from /v1/config?warehouse= per Iceberg REST spec. // See: https://iceberg.apache.org/rest-catalog-spec/ val planTableScanUri = s"$icebergRestCatalogUriRoot/namespaces/$database/tables/$table/plan" // Request planning for current snapshot. snapshotId = 0 means "use current snapshot" // in the Iceberg REST API spec. Time-travel queries are not yet supported. val builder = new PlanTableScanRequest.Builder() .withSnapshotId(CURRENT_SNAPSHOT_ID) // Set caseSensitive=false (defaults to true in spec) to match Spark's case-insensitive // column handling. Server should validate and block requests with caseSensitive=true. .withCaseSensitive(false) // Convert Spark Filter to Iceberg Expression and add to request if filter is present. sparkFilterOption.foreach { sparkFilter => SparkToIcebergExpressionConverter.convert(sparkFilter).foreach { icebergExpr => builder.withFilter(icebergExpr) } } // Add projection to request if present. sparkProjectionOption.foreach { columnNames => builder.withSelect(columnNames.asJava) } val request = builder.build() // Iceberg 1.11 adds withMinRowsRequested() support. For now, manually inject the field. val requestJson = sparkLimitOption match { case Some(limit) => implicit val formats: Formats = DefaultFormats val jsonAst = parse(PlanTableScanRequestParser.toJson(request)) val modifiedJson = jsonAst merge JObject("min-rows-requested" -> JLong(limit.toLong)) compact(render(modifiedJson)) case None => PlanTableScanRequestParser.toJson(request) } val httpPost = new HttpPost(planTableScanUri) httpPost.setEntity(new StringEntity(requestJson, ContentType.APPLICATION_JSON)) val httpResponse = httpClient.execute(httpPost) // Only unpartitioned tables are supported. This map is used when parsing the response // to resolve partition specs. The validation that the table is actually unpartitioned // happens later in convertToScanPlan when we check file.partition().size(). val unpartitionedSpecMap = Map(UNPARTITIONED_SPEC_ID -> PartitionSpec.unpartitioned()) try { val statusCode = httpResponse.getStatusLine.getStatusCode val responseBody = EntityUtils.toString(httpResponse.getEntity) if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_CREATED) { // Parse response with caseSensitive=false to match request and Spark's case-insensitive // column handling val icebergResponse = parsePlanTableScanResponse( responseBody, unpartitionedSpecMap, caseSensitive = false) // Verify plan status is "completed". The Iceberg REST spec allows async planning // where the server returns "submitted" status and the client must poll for results. // We don't support async planning yet, so we require "completed" status. val planStatus = icebergResponse.planStatus() if (planStatus != null && planStatus.toString.toLowerCase(Locale.ROOT) != "completed") { throw new UnsupportedOperationException( s"Async planning not supported. Plan status was '$planStatus' but " + s"expected 'completed'. Table: $database.$table") } convertToScanPlan(icebergResponse, responseBody) } else { // TODO: Parse structured ErrorResponse JSON from Iceberg REST spec instead of raw body throw new IOException( s"Failed to plan table scan for $database.$table. " + s"HTTP status: $statusCode, Response: $responseBody") } } finally { httpResponse.close() } } /** * Convert Iceberg PlanTableScanResponse to simple ScanPlan data class. * * Validates response structure and ensures the table is unpartitioned. */ private def convertToScanPlan( response: PlanTableScanResponse, responseBody: String): ScanPlan = { require(response != null, "PlanTableScanResponse cannot be null") require(response.fileScanTasks() != null, "File scan tasks cannot be null") val files = response.fileScanTasks().asScala.map { task => require(task != null, "FileScanTask cannot be null") require(task.file() != null, "DataFile cannot be null") // Validate that the server does not expect the application of a residual. The application of // a residual filter is currently not supported, and its ignorance leads to wrong results. val residual = task.residual() if (residual != null && !residual.isEquivalentTo(Expressions.alwaysTrue)) { throw new UnsupportedOperationException( s"Found FileScanTask with residual: ${residual}. " + s"Only FileScanTasks with no or alwaysTrue residual are currently supported.") } val file = task.file() // Validate that table is unpartitioned. Partitioned tables are not supported yet. if (file.partition().size() > 0) { throw new UnsupportedOperationException( s"Table has partition data: ${file.partition()}. " + s"Only unpartitioned tables (spec ID $UNPARTITIONED_SPEC_ID) are currently supported.") } ScanFile( filePath = file.path().toString, fileSizeInBytes = file.fileSizeInBytes(), fileFormat = file.format().toString.toLowerCase(Locale.ROOT) ) }.toSeq val credentials = extractCredentials(responseBody) ScanPlan(files = files, credentials = credentials) } /** * Extract storage credentials from IRC server response. * Uses sealed trait pattern - tries each credential type in priority order. * * JSON structure: * { * "storage-credentials": [{ * "config": { * "s3.access-key-id": "...", * "azure.account-name": "...", * "gcs.oauth2.token": "...", * ... * } * }] * } */ /** * Extract storage credentials from response using sealed trait factory. * Returns None if no credentials section exists. * Throws IllegalStateException if credentials are incomplete or malformed. */ private def extractCredentials(responseBody: String): Option[ScanPlanStorageCredentials] = { implicit val formats: Formats = DefaultFormats val json = parse(responseBody) // Extract config map from storage-credentials[0].config val config: Option[Map[String, String]] = try { (json \ "storage-credentials")(0) \ "config" match { case JNothing | JNull => None case c => Some(c.extract[Map[String, String]]) } } catch { case _: Exception => None // No credentials section in response } // If config exists and is non-empty, use factory (throws on incomplete credentials) config.filter(_.nonEmpty).map(fromConfig) } /** * Close the HTTP client and release resources. * * This should be called when the client is no longer needed to prevent resource leaks. * After calling close(), this client instance should not be used for further requests. */ override def close(): Unit = { if (httpClient != null) { httpClient.close() } } /** * Retry strategy for server errors (5xx status codes) with exponential backoff. * Retries up to maxRetries times with doubling intervals (1s, 2s, 4s, ...). * Does NOT retry on client errors (4xx) since those indicate request-level issues. * * The ServiceUnavailableRetryStrategy interface calls retryRequest() first, then * getRetryInterval(), so we capture the execution count in retryRequest() and * use it to compute the backoff in getRetryInterval(). */ private class ServerErrorRetryStrategy(maxRetries: Int) extends ServiceUnavailableRetryStrategy { // ThreadLocal so concurrent planScan calls each track their own retry attempt. // The HTTP client is shared and thread-safe (see class doc), so multiple threads // can be retrying independently through the same strategy instance. private val lastExecutionCount = new ThreadLocal[Int] { override def initialValue(): Int = 1 } override def retryRequest( response: HttpResponse, executionCount: Int, context: HttpContext): Boolean = { lastExecutionCount.set(executionCount) val statusCode = response.getStatusLine.getStatusCode statusCode >= 500 && executionCount <= maxRetries } // Exponential backoff: 1s, 2s, 4s, ... override def getRetryInterval: Long = java.util.concurrent.TimeUnit.SECONDS.toMillis(1L << (lastExecutionCount.get() - 1)) } private def parsePlanTableScanResponse( json: String, specsById: Map[Int, PartitionSpec], caseSensitive: Boolean): PlanTableScanResponse = { // Use reflection to access the private fromJson method in the Iceberg parser class. // The method is not part of the public API, so we need reflection and setAccessible. val parserClass = Utils.classForName( "shadedForDelta.org.apache.iceberg.rest.responses.PlanTableScanResponseParser") val fromJsonMethod: Method = parserClass.getDeclaredMethod( "fromJson", classOf[String], classOf[java.util.Map[_, _]], classOf[Boolean]) fromJsonMethod.setAccessible(true) fromJsonMethod.invoke( null, // static method json, specsById.map { case (k, v) => Int.box(k) -> v }.asJava, Boolean.box(caseSensitive) ).asInstanceOf[PlanTableScanResponse] } } ================================================ FILE: iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientFactory.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.serverSidePlanning import org.apache.spark.sql.SparkSession /** * Factory that creates IcebergRESTCatalogPlanningClient instances. * Lives in the iceberg module alongside the implementation. */ class IcebergRESTCatalogPlanningClientFactory extends ServerSidePlanningClientFactory { override def buildClient( spark: SparkSession, metadata: ServerSidePlanningMetadata): ServerSidePlanningClient = { val baseUri = metadata.planningEndpointUri val token = metadata.authToken.getOrElse("") val catalogName = metadata.catalogName new IcebergRESTCatalogPlanningClient(baseUri, catalogName, token) } } ================================================ FILE: iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/SparkToIcebergExpressionConverter.scala ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.serverSidePlanning import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.sources._ import shadedForDelta.org.apache.iceberg.expressions.{Expression, Expressions} /** * Converts Spark Filter expressions to Iceberg Expression objects for server-side planning. * * Filter Mapping Table: * {{{ * +--------------------------+--------------------------------+ * | Spark Filter | Iceberg Expression | * +--------------------------+--------------------------------+ * | EqualTo | Expressions.equal() | * | EqualTo(col, null) | Expressions.isNull() | * | EqualTo(col, NaN) | Expressions.isNaN() | * | NotEqualTo | Expressions.notEqual() | * | NotEqualTo(col, NaN) | Expressions.notNaN() | * | LessThan | Expressions.lessThan() | * | GreaterThan | Expressions.greaterThan() | * | LessThanOrEqual | Expressions.lessThanOrEqual() | * | GreaterThanOrEqual | Expressions.greaterThanOrEqual()| * | In | Expressions.in() | * | Not(In) | Expressions.notIn() | * | IsNull | Expressions.isNull() | * | IsNotNull | Expressions.notNull() | * | Not(IsNull) | Expressions.notNull() | * | And | Expressions.and() | * | Or | Expressions.or() | * | StringStartsWith | Expressions.startsWith() | * | Not(StringStartsWith) | Expressions.notStartsWith() | * | AlwaysTrue | Expressions.alwaysTrue() | * | AlwaysFalse | Expressions.alwaysFalse() | * +--------------------------+--------------------------------+ * }}} * * * Example usage: * {{{ * val sparkFilter = EqualTo("id", 5) * SparkToIcebergExpressionConverter.convert(sparkFilter) match { * case Some(icebergExpr) => // Use expression * case None => // Filter not supported * } * }}} */ private[serverSidePlanning] object SparkToIcebergExpressionConverter extends Logging { /** * Convert a Spark Filter to an Iceberg Expression. * * @param sparkFilter The Spark filter to convert * @return Some(Expression) if the filter is supported, None otherwise */ private[serverSidePlanning] def convert(sparkFilter: Filter): Option[Expression] = { logInfo(s"Converting Spark filter to Iceberg expression: $sparkFilter") val result = try { sparkFilter match { // Equality and Comparison Operators case EqualTo(attribute, sparkValue) => Some(convertEqualTo(attribute, sparkValue)) case LessThan(attribute, sparkValue) => Some(convertLessThan(attribute, sparkValue)) case GreaterThan(attribute, sparkValue) => Some(convertGreaterThan(attribute, sparkValue)) case LessThanOrEqual(attribute, sparkValue) => Some(convertLessThanOrEqual(attribute, sparkValue)) case GreaterThanOrEqual(attribute, sparkValue) => Some(convertGreaterThanOrEqual(attribute, sparkValue)) case In(attribute, values) => Some(convertIn(attribute, values)) // Null Checks case IsNull(attribute) => Some(Expressions.isNull(attribute)) case IsNotNull(attribute) => Some(Expressions.notNull(attribute)) // Logical Combinators case And(left, right) => for { leftIcebergExpr <- convert(left) rightIcebergExpr <- convert(right) } yield Expressions.and(leftIcebergExpr, rightIcebergExpr) case Or(left, right) => for { leftIcebergExpr <- convert(left) rightIcebergExpr <- convert(right) } yield Expressions.or(leftIcebergExpr, rightIcebergExpr) // NOT Operator (special case) case Not(innerFilter) => convertNot(innerFilter) // String Operations case StringStartsWith(attribute, value) => Some(Expressions.startsWith(attribute, value)) // Always True/False case AlwaysTrue() => Some(Expressions.alwaysTrue()) case AlwaysFalse() => Some(Expressions.alwaysFalse()) /* * Unsupported Filters: * - StringEndsWith, StringContains: Iceberg API doesn't provide these predicates */ case _ => logInfo(s"Unsupported Spark filter (no Iceberg equivalent): " + s"${sparkFilter.getClass.getSimpleName} - $sparkFilter") None } } catch { case e: IllegalArgumentException => /* * The filter is supported but conversion failed as the type or value is unsupported. * - NaN in comparison operators (LessThan, GreaterThan, etc.) * - Unsupported types (e.g., Array, Map, binary types) */ logWarning(s"Failed to convert Spark filter due to unsupported type or value: " + s"$sparkFilter", e) None } logDebug(s"Conversion result for $sparkFilter: " + s"${result.map(_.toString).getOrElse("None (unsupported)")}") result } // Private helper methods for type-specific conversions private def isNaN(value: Any): Boolean = value match { case v: Float => v.isNaN case v: Double => v.isNaN case _ => false } /** * Convert a Spark value to Iceberg-compatible type with proper coercion. * @param supportBoolean if true, also handles Boolean type. * Note: Comparison operators (LessThan, GreaterThan, etc.) don't support Boolean. * Only equality operators (EqualTo, NotEqualTo) should set this to true. * @throws IllegalArgumentException if the value is an unsupported type (complex types, * unknown types, or Boolean when supportBoolean=false) */ private[serverSidePlanning] def toIcebergValue( value: Any, supportBoolean: Boolean = false): Any = { value match { // Date/Timestamp conversion (semantic change) because // Iceberg Literals.from() doesn't accept java.sql.Date/Timestamp, expects Int/Long case v: java.sql.Date => // Iceberg expects days since epoch (1970-01-01) as Int DateTimeUtils.fromJavaDate(v): Integer case v: java.sql.Timestamp => // Iceberg expects microseconds since epoch as Long DateTimeUtils.fromJavaTimestamp(v): java.lang.Long case v: java.time.Instant => // Iceberg expects microseconds since epoch as Long (for TIMESTAMP WITH TIMEZONE) DateTimeUtils.instantToMicros(v): java.lang.Long case v: java.time.LocalDateTime => // Iceberg expects microseconds since epoch as Long (for TIMESTAMP_NTZ) DateTimeUtils.localDateTimeToMicros(v): java.lang.Long case v: java.time.LocalDate => // Iceberg expects days since epoch (1970-01-01) as Int (for DATE) v.toEpochDay.toInt: Integer // Type coercion (Scala to Java boxed types) case v: Int => v: Integer case v: Long => v: java.lang.Long case v: Float => v: java.lang.Float case v: Double => v: java.lang.Double case v: java.math.BigDecimal => v case v: String => v case v: Boolean if supportBoolean => v: java.lang.Boolean case _ => throw new IllegalArgumentException( s"Unsupported type for Iceberg filter pushdown: ${value.getClass.getName}") } } /* * Convert EqualTo with special handling for null and NaN. * Note: We cannot use Expressions.equal(col, null/NaN) because Iceberg models these * with specialized predicates (isNull/isNaN) that have different evaluation semantics: * - SQL: col = NULL returns NULL (unknown), but col IS NULL returns TRUE/FALSE * Reference: OSS Iceberg SparkV2Filters.handleEqual() */ private def convertEqualTo(attribute: String, sparkValue: Any): Expression = { sparkValue match { case null => Expressions.isNull(attribute) case _ if isNaN(sparkValue) => Expressions.isNaN(attribute) case _ => Expressions.equal(attribute, toIcebergValue(sparkValue, supportBoolean = true)) } } /* * Convert NotEqualTo with special handling for null and NaN. * Note: Not(EqualTo(col, null)) from Spark (representing IS NOT NULL) is converted here. */ private def convertNotEqualTo(attribute: String, sparkValue: Any): Expression = { sparkValue match { case null => Expressions.notNull(attribute) case _ if isNaN(sparkValue) => Expressions.notNaN(attribute) case _ => Expressions.notEqual(attribute, toIcebergValue(sparkValue, supportBoolean = true)) } } /** * Convert a Spark NOT filter to an Iceberg Expression. * * Supported conversions: * - Not(EqualTo(col, value)) -> Expressions.notEqual * - Not(EqualTo(col, null)) -> Expressions.notNull * - Not(EqualTo(col, NaN)) -> Expressions.notNaN * - Not(In(col, values)) -> Expressions.notIn * - Not(IsNull(col)) -> Expressions.notNull * - Not(StringStartsWith(col, value)) -> Expressions.notStartsWith * * All other NOT expressions (Not(LessThan), Not(And), etc.) are unsupported because Iceberg * doesn't provide equivalent predicates. This is consistent with OSS Iceberg's SparkV2Filters. */ private def convertNot(sparkInnerFilter: Filter): Option[Expression] = { sparkInnerFilter match { case EqualTo(attribute, sparkValue) => Some(convertNotEqualTo(attribute, sparkValue)) case In(attribute, values) => Some(convertNotIn(attribute, values)) case IsNull(attribute) => Some(Expressions.notNull(attribute)) case StringStartsWith(attribute, value) => Some(Expressions.notStartsWith(attribute, value)) case _ => None // All other NOT expressions are unsupported } } private def convertIn(attribute: String, values: Array[Any]): Expression = { // Iceberg expects IN to filter out null values and convert Date/Timestamp to Int/Long val nonNullValues = values.filter(_ != null).map(v => toIcebergValue(v, supportBoolean = true) ) Expressions.in(attribute, nonNullValues: _*) } /** * Convert NOT IN filter to Iceberg notIn expression. * Example: NOT IN ("id", [1, 2, 3]) -> Expressions.notIn("id", 1, 2, 3) */ private def convertNotIn(attribute: String, values: Array[Any]): Expression = { // Iceberg expects NOT IN to filter out null values and convert Date/Timestamp to Int/Long val nonNullValues = values.filter(_ != null).map(v => toIcebergValue(v, supportBoolean = true) ) Expressions.notIn(attribute, nonNullValues: _*) } private def convertLessThan(attribute: String, sparkValue: Any): Expression = Expressions.lessThan(attribute, toIcebergValue(sparkValue, supportBoolean = false)) private def convertGreaterThan(attribute: String, sparkValue: Any): Expression = Expressions.greaterThan(attribute, toIcebergValue(sparkValue, supportBoolean = false)) private def convertLessThanOrEqual(attribute: String, sparkValue: Any): Expression = Expressions.lessThanOrEqual(attribute, toIcebergValue(sparkValue, supportBoolean = false)) private def convertGreaterThanOrEqual(attribute: String, sparkValue: Any): Expression = Expressions.greaterThanOrEqual(attribute, toIcebergValue(sparkValue, supportBoolean = false)) } ================================================ FILE: iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTCatalogAdapterWithPlanSupport.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package shadedForDelta.org.apache.iceberg.rest; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import shadedForDelta.org.apache.iceberg.BaseFileScanTask; import shadedForDelta.org.apache.iceberg.DeleteFile; import shadedForDelta.org.apache.iceberg.FileScanTask; import shadedForDelta.org.apache.iceberg.PartitionSpecParser; import shadedForDelta.org.apache.iceberg.SchemaParser; import shadedForDelta.org.apache.iceberg.Table; import shadedForDelta.org.apache.iceberg.TableScan; import shadedForDelta.org.apache.iceberg.catalog.Catalog; import shadedForDelta.org.apache.iceberg.catalog.TableIdentifier; import shadedForDelta.org.apache.iceberg.io.CloseableIterable; import shadedForDelta.org.apache.iceberg.rest.HTTPRequest; import shadedForDelta.org.apache.iceberg.rest.RESTCatalogAdapter; import shadedForDelta.org.apache.iceberg.rest.requests.PlanTableScanRequest; import shadedForDelta.org.apache.iceberg.rest.requests.PlanTableScanRequestParser; import shadedForDelta.org.apache.iceberg.rest.responses.ErrorResponse; import shadedForDelta.org.apache.iceberg.rest.PlanStatus; import shadedForDelta.org.apache.iceberg.rest.responses.PlanTableScanResponse; import shadedForDelta.org.apache.iceberg.expressions.Expression; import shadedForDelta.org.apache.iceberg.expressions.ResidualEvaluator; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Extends RESTCatalogAdapter to add support for server-side scan planning via the /plan endpoint. * This adapter intercepts /plan requests and handles them by executing Iceberg table scans locally, * returning file scan tasks to the client. Other catalog operations are delegated to the parent * RESTCatalogAdapter implementation. */ class IcebergRESTCatalogAdapterWithPlanSupport extends RESTCatalogAdapter { private static final Logger LOG = LoggerFactory.getLogger(IcebergRESTCatalogAdapterWithPlanSupport.class); private final Catalog catalog; // Catalog prefix returned in /v1/config that gets inserted into REST paths. // Example: prefix="iceberg" transforms /v1/namespaces/db/tables/t1/plan // to /v1/iceberg/namespaces/db/tables/t1/plan private String catalogPrefix = null; // null = no prefix (fallback case) // Static fields for test verification - captures filter and projection from requests // Volatile is used to guarantee correct cross-thread access (test thread and Jetty server thread). private static volatile Expression capturedFilter = null; private static volatile List capturedProjection = null; private static volatile Long capturedMinRowsRequested = null; private static volatile Boolean capturedCaseSensitive = null; // Static field for test credential injection - credentials to inject into /plan responses // Volatile is used to guarantee correct cross-thread access (test thread and Jetty server thread). private static volatile Map testCredentials = null; // Static field for test residual injection - residual expression to override in /plan responses. // When set, all FileScanTasks in the response will have this residual instead of the default. // Volatile is used to guarantee correct cross-thread access (test thread and Jetty server thread). private static volatile Expression testResidual = null; // Static field to capture the request path of /plan requests for test verification // Volatile is used to guarantee correct cross-thread access (test thread and Jetty server thread). private static volatile String capturedPlanRequestPath = null; // Failure injection fields for testing HTTP retry logic. // planRequestFailCount: number of remaining /plan requests to fail before allowing success. // planRequestFailStatusCode: HTTP status code to return for injected failures. // planRequestCount: total number of /plan requests received (for verifying retry behavior). private static final AtomicInteger planRequestFailCount = new AtomicInteger(0); private static volatile int planRequestFailStatusCode = 503; private static final AtomicInteger planRequestCount = new AtomicInteger(0); IcebergRESTCatalogAdapterWithPlanSupport(Catalog catalog) { super(catalog); this.catalog = catalog; } /** * Set the catalog prefix to be returned by /v1/config endpoint. * The prefix is inserted into REST paths: /v1/{prefix}/namespaces/{namespace}/tables/{table}/plan * Used for testing prefix-based endpoint construction. * Package-private as this is an implementation detail - tests should use * IcebergRESTServer.setCatalogPrefix() instead. * * @param prefix The prefix to return in config.overrides, or null for no prefix */ void setCatalogPrefix(String prefix) { this.catalogPrefix = prefix; } /** * Get the catalog prefix for testing. * Package-private for servlet access. */ String getCatalogPrefix() { return this.catalogPrefix; } /** * Get the filter captured from the most recent /plan request. * Package-private for test access. */ static Expression getCapturedFilter() { return capturedFilter; } /** * Get the projection (list of column names) captured from the most recent /plan request. * Package-private for test access. */ static List getCapturedProjection() { return capturedProjection; } /** * Get the min-rows-requested captured from the most recent /plan request. * Package-private for test access. */ static Long getCapturedMinRowsRequested() { return capturedMinRowsRequested; } /** * Get the caseSensitive flag captured from the most recent /plan request. * Package-private for test access. */ static Boolean getCapturedCaseSensitive() { return capturedCaseSensitive; } /** * Get the request path captured from the most recent /plan request. * Package-private for test access. */ static String getCapturedPlanRequestPath() { return capturedPlanRequestPath; } /** * Set test credentials to inject into /plan responses. * Package-private for test access. * * @param credentials Map of credential config (e.g., "s3.access-key-id" -> "...") */ static void setTestCredentials(Map credentials) { testCredentials = credentials; } /** * Get the test credentials configured for injection into /plan responses. * Package-private for servlet access. */ static Map getTestCredentials() { return testCredentials; } /** * Set test residual expression to inject into /plan responses. * When set, all FileScanTasks in the response will have this residual expression * instead of the default (alwaysTrue). Used for testing client-side residual validation. * Package-private for test access via IcebergRESTServer. * * @param residual The residual expression to inject, or null to use the default */ static void setTestResidual(Expression residual) { testResidual = residual; } /** * Clear captured filter, projection, and limit. Call between tests to avoid pollution. * Package-private for test access. */ static void clearCaptured() { capturedFilter = null; capturedProjection = null; capturedMinRowsRequested = null; capturedCaseSensitive = null; testCredentials = null; testResidual = null; capturedPlanRequestPath = null; planRequestFailCount.set(0); planRequestCount.set(0); } /** * Configure the server to fail the next N /plan requests with the specified HTTP status code. * After N failures, subsequent requests proceed normally. * Used for testing HTTP retry logic in the client. * * @param count Number of /plan requests to fail * @param statusCode HTTP status code to return for injected failures (e.g., 503, 404) */ static void setFailNextPlanRequests(int count, int statusCode) { planRequestFailCount.set(count); planRequestFailStatusCode = statusCode; } /** * Atomically get and decrement the remaining failure count. * Returns the value before decrement. If > 0, the request should be failed. * Package-private for servlet access. */ static int getAndDecrementFailCount() { return planRequestFailCount.getAndDecrement(); } /** * Get the HTTP status code to use for injected failures. * Package-private for servlet access. */ static int getPlanRequestFailStatusCode() { return planRequestFailStatusCode; } /** * Increment and return the total /plan request count. * Package-private for servlet access. */ static void incrementPlanRequestCount() { planRequestCount.incrementAndGet(); } /** * Get the total number of /plan requests received. * Package-private for test access. */ static int getPlanRequestCount() { return planRequestCount.get(); } @Override protected T execute( HTTPRequest request, Class responseType, Consumer errorHandler, Consumer> responseHeaders, ParserContext parserContext) { LOG.debug("Executing request: {} {}", request.method(), request.path()); // Intercept /plan requests before they reach the base adapter if (isPlanTableScanRequest(request)) { capturedPlanRequestPath = request.path(); // Capture the path for test verification try { PlanTableScanResponse response = handlePlanTableScan(request, parserContext); return (T) response; } catch (Exception e) { LOG.error("Error handling plan table scan: {}", e.getMessage(), e); ErrorResponse error = ErrorResponse.builder() .responseCode(500) .withType("InternalServerError") .withMessage("Failed to plan table scan: " + e.getMessage()) .build(); errorHandler.accept(error); return null; } } return super.execute( request, responseType, errorHandler, responseHeaders, parserContext); } private boolean isPlanTableScanRequest(HTTPRequest request) { return HTTPRequest.HTTPMethod.POST.equals(request.method()) && request.path().endsWith("/plan"); } private TableIdentifier extractTableIdentifier(String path) { // Path format: /v1/namespaces/{namespace}/tables/{table}/plan // or: /v1/{prefix}/namespaces/{namespace}/tables/{table}/plan String[] parts = path.split("/"); int namespacesIdx = -1; for (int i = 0; i < parts.length; i++) { if ("namespaces".equals(parts[i])) { namespacesIdx = i; break; } } if (namespacesIdx == -1 || namespacesIdx + 3 >= parts.length) { throw new IllegalArgumentException("Invalid path format: " + path); } String namespace = parts[namespacesIdx + 1]; String tableName = parts[namespacesIdx + 3]; // skip "tables" return TableIdentifier.of(namespace, tableName); } /** * Extract min-rows-requested from JSON string using Jackson. * Iceberg 1.11 added this field, but we're on 1.10.0, so we parse it from JSON. */ private Long extractMinRowsRequested(String jsonBody) { if (jsonBody == null || jsonBody.trim().isEmpty()) { return null; } try { ObjectMapper mapper = new ObjectMapper(); JsonNode root = mapper.readTree(jsonBody); JsonNode minRowsNode = root.get("min-rows-requested"); return minRowsNode != null ? minRowsNode.asLong() : null; } catch (Exception e) { LOG.warn("Failed to extract min-rows-requested from JSON: {}", e.getMessage()); return null; } } private PlanTableScanRequest parsePlanRequest(HTTPRequest request) { // The request body should be a JSON string Object body = request.body(); if (body == null) { throw new IllegalArgumentException("Request body is null"); } String jsonBody = body.toString(); return PlanTableScanRequestParser.fromJson(jsonBody); } private PlanTableScanResponse handlePlanTableScan( HTTPRequest request, ParserContext parserContext) throws Exception { LOG.debug("Handling plan table scan request"); // Extract table identifier TableIdentifier tableIdent = extractTableIdentifier(request.path()); LOG.debug("Table identifier: {}", tableIdent); // Extract JSON body for parsing both the request and min-rows-requested Object body = request.body(); if (body == null) { throw new IllegalArgumentException("Request body is null"); } String jsonBody = body.toString(); // Extract min-rows-requested (not supported in Iceberg 1.10, so parse from JSON) Long minRowsRequested = extractMinRowsRequested(jsonBody); LOG.debug("Extracted min-rows-requested: {}", minRowsRequested); // Parse request PlanTableScanRequest planRequest = PlanTableScanRequestParser.fromJson(jsonBody); LOG.debug("Plan request parsed: snapshotId={}", planRequest.snapshotId()); // Load table from catalog Table table = catalog.loadTable(tableIdent); LOG.debug("Table loaded: {}", table); // Create table scan TableScan tableScan = table.newScan(); // Apply snapshot if specified and valid if (planRequest.snapshotId() != null && planRequest.snapshotId() != 0) { tableScan = tableScan.useSnapshot(planRequest.snapshotId()); LOG.debug("Using snapshot: {}", planRequest.snapshotId()); } else { LOG.debug("Using current snapshot (snapshotId was null or 0)"); } // Capture filter, projection, and limit for test verification capturedFilter = planRequest.filter(); capturedProjection = planRequest.select(); capturedMinRowsRequested = minRowsRequested; capturedCaseSensitive = planRequest.caseSensitive(); LOG.debug("Captured filter: {}", capturedFilter); LOG.debug("Captured projection: {}", capturedProjection); LOG.debug("Captured min-rows-requested: {}", capturedMinRowsRequested); LOG.debug("Captured caseSensitive: {}", capturedCaseSensitive); // Validate caseSensitive=false requirement if (planRequest.caseSensitive()) { throw new IllegalArgumentException("caseSensitive=true is not supported"); } // Validate that unsupported features are not requested if (planRequest.startSnapshotId() != null) { throw new UnsupportedOperationException( "Incremental scans are not supported in this test implementation"); } if (planRequest.endSnapshotId() != null) { throw new UnsupportedOperationException( "Incremental scans are not supported in this test implementation"); } if (planRequest.statsFields() != null && !planRequest.statsFields().isEmpty()) { throw new UnsupportedOperationException( "Column stats are not supported in this test implementation"); } // Execute scan planning List fileScanTasks = new ArrayList<>(); try (CloseableIterable tasks = tableScan.planFiles()) { tasks.forEach(task -> fileScanTasks.add(task)); } LOG.debug("Planned {} file scan tasks", fileScanTasks.size()); // If a test residual is configured, rebuild tasks with the injected residual Expression residualOverride = testResidual; List tasksToReturn; if (residualOverride != null) { tasksToReturn = new ArrayList<>(); for (FileScanTask task : fileScanTasks) { tasksToReturn.add(new BaseFileScanTask( task.file(), task.deletes().toArray(new DeleteFile[0]), SchemaParser.toJson(task.spec().schema()), PartitionSpecParser.toJson(task.spec()), ResidualEvaluator.of(task.spec(), residualOverride, capturedCaseSensitive))); } LOG.debug("Injected test residual into {} file scan tasks", tasksToReturn.size()); } else { tasksToReturn = fileScanTasks; } // Get partition specs for serialization Map specsById = table.specs(); LOG.debug("Table has {} partition specs", specsById.size()); // Build response (Pattern 1: COMPLETED with direct tasks) return PlanTableScanResponse.builder() .withPlanStatus(PlanStatus.COMPLETED) .withFileScanTasks(tasksToReturn) .withSpecsById(specsById) .build(); } } ================================================ FILE: iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServer.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package shadedForDelta.org.apache.iceberg.rest; import java.io.File; import java.io.IOException; import java.util.Map; import org.apache.hadoop.conf.Configuration; import shadedForDelta.org.apache.iceberg.CatalogProperties; import shadedForDelta.org.apache.iceberg.CatalogUtil; import shadedForDelta.org.apache.iceberg.catalog.Catalog; import shadedForDelta.org.apache.iceberg.jdbc.JdbcCatalog; import shadedForDelta.org.apache.iceberg.relocated.com.google.common.collect.Maps; import shadedForDelta.org.apache.iceberg.util.PropertyUtil; import shadedForDelta.org.apache.iceberg.expressions.Expression; import java.util.List; import org.eclipse.jetty.server.Connector; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.handler.gzip.GzipHandler; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * HTTP server for testing Iceberg REST catalog operations with server-side scan planning support. * Uses Jetty to serve REST catalog endpoints and extends the standard REST catalog with a /plan * endpoint for server-side table scan planning. This implementation is suitable for integration * tests and does not require external services. */ public class IcebergRESTServer { private static final Logger LOG = LoggerFactory.getLogger(IcebergRESTServer.class); public static final String REST_PORT = "rest.port"; static final int REST_PORT_DEFAULT = 8181; public static final String CATALOG_NAME = "catalog.name"; static final String CATALOG_NAME_DEFAULT = "rest_backend"; private Server httpServer; private final Map config; private Catalog catalog; private Map catalogConfiguration; private IcebergRESTCatalogAdapterWithPlanSupport adapter; public IcebergRESTServer() { this.config = Maps.newHashMap(); } public IcebergRESTServer(Map config) { this.config = config; } private void initializeBackendCatalog() throws IOException { // Translate environment variables to catalog properties Map catalogProperties = Maps.newHashMap(); catalogProperties.putAll(config); // Fallback to a JDBCCatalog impl if one is not set catalogProperties.putIfAbsent(CatalogProperties.CATALOG_IMPL, JdbcCatalog.class.getName()); catalogProperties.putIfAbsent(CatalogProperties.URI, "jdbc:sqlite::memory:"); catalogProperties.putIfAbsent("jdbc.schema-version", "V1"); // Configure a default location if one is not specified String warehouseLocation = catalogProperties.get(CatalogProperties.WAREHOUSE_LOCATION); if (warehouseLocation == null) { File tmp = java.nio.file.Files.createTempDirectory("iceberg_warehouse").toFile(); tmp.deleteOnExit(); warehouseLocation = new File(tmp, "iceberg_data").getAbsolutePath(); catalogProperties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouseLocation); LOG.info("No warehouse location set. Defaulting to temp location: {}", warehouseLocation); } String catalogName = PropertyUtil.propertyAsString(catalogProperties, CATALOG_NAME, CATALOG_NAME_DEFAULT); LOG.info("Creating {} catalog with properties: {}", catalogName, catalogProperties); this.catalog = CatalogUtil.buildIcebergCatalog(catalogName, catalogProperties, new Configuration()); this.catalogConfiguration = catalogProperties; } public void start(boolean join) throws Exception { initializeBackendCatalog(); this.adapter = new IcebergRESTCatalogAdapterWithPlanSupport(catalog); // Use custom servlet that supports the /plan endpoint RESTCatalogServlet servlet = new IcebergRESTServletWithPlanSupport(adapter); ServletContextHandler servletContext = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); ServletHolder servletHolder = new ServletHolder(servlet); // Serve on root path for IcebergRESTCatalogPlanningClient tests servletContext.addServlet(servletHolder, "/*"); servletContext.insertHandler(new GzipHandler()); this.httpServer = new Server( PropertyUtil.propertyAsInt(catalogConfiguration, REST_PORT, REST_PORT_DEFAULT)); httpServer.setHandler(servletContext); for (Connector connector : httpServer.getConnectors()) { ((ServerConnector) connector).setReusePort(true); } httpServer.start(); if (join) { httpServer.join(); } } public Catalog getCatalog() { return catalog; } public Map getConfiguration() { return catalogConfiguration; } public int getPort() { Connector[] connectors = httpServer.getConnectors(); if (connectors.length > 0) { return ((ServerConnector) connectors[0]).getLocalPort(); } else { throw new IllegalStateException("HTTP server has no connectors"); } } /** * Set the catalog prefix to be returned by /v1/config endpoint. * Used for testing prefix-based endpoint construction in Unity Catalog metadata. * Delegates to the adapter which handles the actual /v1/config request interception. * * @param prefix The prefix to return in config.overrides, or null for no prefix */ public void setCatalogPrefix(String prefix) { if (adapter != null) { adapter.setCatalogPrefix(prefix); } } /** * Get the filter captured from the most recent /plan request. * Delegates to adapter. For test verification. */ public Expression getCapturedFilter() { return IcebergRESTCatalogAdapterWithPlanSupport.getCapturedFilter(); } /** * Get the projection (list of column names) captured from the most recent /plan request. * Delegates to adapter. For test verification. */ public List getCapturedProjection() { return IcebergRESTCatalogAdapterWithPlanSupport.getCapturedProjection(); } /** * Get the limit (min-rows-requested) captured from the most recent /plan request. * Delegates to adapter. For test verification. */ public Long getCapturedLimit() { return IcebergRESTCatalogAdapterWithPlanSupport.getCapturedMinRowsRequested(); } /** * Get the caseSensitive flag captured from the most recent /plan request. * For test verification. */ public Boolean getCapturedCaseSensitive() { return IcebergRESTCatalogAdapterWithPlanSupport.getCapturedCaseSensitive(); } /** * Get the request path captured from the most recent /plan request. * Delegates to adapter. For test verification of endpoint construction. */ public String getCapturedPlanRequestPath() { return IcebergRESTCatalogAdapterWithPlanSupport.getCapturedPlanRequestPath(); } /** * Set test credentials to inject into /plan responses. * Used for testing credential extraction in clients. * * @param credentials Map of credential config (e.g., "s3.access-key-id" -> "...") */ public void setTestCredentials(Map credentials) { IcebergRESTCatalogAdapterWithPlanSupport.setTestCredentials(credentials); } /** * Set test residual expression to inject into /plan responses. * When set, all FileScanTasks in the response will have this residual expression * instead of the default (alwaysTrue). Used for testing client-side residual validation. * * @param residual The residual expression to inject, or null to use the default */ public void setTestResidual(shadedForDelta.org.apache.iceberg.expressions.Expression residual) { IcebergRESTCatalogAdapterWithPlanSupport.setTestResidual(residual); } /** * Configure the server to fail the next N /plan requests with the specified HTTP status code. * After N failures, subsequent requests proceed normally. * Used for testing HTTP retry logic in the client. * * @param count Number of /plan requests to fail * @param statusCode HTTP status code to return for injected failures (e.g., 503, 404) */ public void setFailNextPlanRequests(int count, int statusCode) { IcebergRESTCatalogAdapterWithPlanSupport.setFailNextPlanRequests(count, statusCode); } /** * Get the total number of /plan requests received since last clearCaptured(). * Used for verifying retry behavior in tests. */ public int getPlanRequestCount() { return IcebergRESTCatalogAdapterWithPlanSupport.getPlanRequestCount(); } /** * Clear captured filter and projection. Call between tests. */ public void clearCaptured() { IcebergRESTCatalogAdapterWithPlanSupport.clearCaptured(); } public void stop() throws Exception { if (httpServer != null) { httpServer.stop(); } } public static void main(String[] args) throws Exception { new IcebergRESTServer().start(true); } } ================================================ FILE: iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServletWithPlanSupport.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package shadedForDelta.org.apache.iceberg.rest; import java.io.BufferedReader; import java.io.IOException; import java.io.PrintWriter; import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; import com.fasterxml.jackson.databind.ObjectMapper; import shadedForDelta.org.apache.iceberg.rest.responses.ConfigResponse; import shadedForDelta.org.apache.iceberg.rest.responses.ErrorResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Extension of RESTCatalogServlet that adds support for the /plan endpoint. */ public class IcebergRESTServletWithPlanSupport extends RESTCatalogServlet { private static final Logger LOG = LoggerFactory.getLogger(IcebergRESTServletWithPlanSupport.class); private final RESTCatalogAdapter adapter; private final ObjectMapper mapper; public IcebergRESTServletWithPlanSupport(RESTCatalogAdapter adapter) { super(adapter); this.adapter = adapter; this.mapper = RESTObjectMapper.mapper(); } /** * Override GET to handle /v1/config requests with catalog prefix. * Note: We handle this at servlet level because the shaded RESTCatalogAdapter * doesn't expose the server-side execute() method needed for interception. */ @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { String path = req.getPathInfo(); // Check if this is a /v1/config request if (path != null && (path.equals("/v1/config") || path.endsWith("/v1/config"))) { LOG.debug("Custom servlet handling /v1/config request"); handleConfigRequest(req, resp); } else { // For all other GET requests, use standard handling super.doGet(req, resp); } } @Override protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException { String path = req.getPathInfo(); // Check if this is a /plan endpoint request if (path != null && path.endsWith("/plan")) { LOG.debug("Custom servlet handling /plan request for path: {}", path); handlePlanRequest(req, resp); } else { // For all other requests, use standard handling super.doPost(req, resp); } } /** * Test helper for Iceberg REST /v1/config endpoint that returns optional catalog * prefix, following the Iceberg REST catalog spec pattern. */ private void handleConfigRequest(HttpServletRequest req, HttpServletResponse resp) throws IOException { try { // Build ConfigResponse with prefix if adapter has one set ConfigResponse.Builder builder = ConfigResponse.builder(); // If adapter is our custom type, get the prefix and add it to overrides if (adapter instanceof IcebergRESTCatalogAdapterWithPlanSupport) { IcebergRESTCatalogAdapterWithPlanSupport customAdapter = (IcebergRESTCatalogAdapterWithPlanSupport) adapter; String prefix = customAdapter.getCatalogPrefix(); if (prefix != null && !prefix.isEmpty()) { LOG.info("Adding prefix to /v1/config response: {}", prefix); builder.withOverride("prefix", prefix); } } ConfigResponse config = builder.build(); // Write JSON response resp.setStatus(200); resp.setContentType("application/json"); mapper.writeValue(resp.getWriter(), config); } catch (Exception e) { LOG.error("Error handling /v1/config request: {}", e.getMessage(), e); resp.setStatus(500); } } private void handlePlanRequest(HttpServletRequest req, HttpServletResponse resp) throws IOException { // Track plan request count for testing retry behavior IcebergRESTCatalogAdapterWithPlanSupport.incrementPlanRequestCount(); // Check if we should inject a failure for testing HTTP retry logic int remainingFailures = IcebergRESTCatalogAdapterWithPlanSupport.getAndDecrementFailCount(); if (remainingFailures > 0) { int failStatusCode = IcebergRESTCatalogAdapterWithPlanSupport.getPlanRequestFailStatusCode(); LOG.info("Injecting test failure: returning HTTP {} ({} failures remaining)", failStatusCode, remainingFailures - 1); resp.setStatus(failStatusCode); resp.setContentType("application/json"); resp.getWriter().write( "{\"error\": {\"message\": \"Injected test failure\", \"type\": \"TestError\", \"code\": " + failStatusCode + "}}"); return; } try { // Extract request components String path = req.getPathInfo(); // HTTPRequest paths should not start with /, so strip it if (path != null && path.startsWith("/")) { path = path.substring(1); } Map headers = extractHeaders(req); Map queryParams = extractQueryParams(req); String body = extractBody(req); LOG.debug("Plan request - path: {}", path); LOG.debug("Plan request - body: {}", body); // Build HTTPRequest - body should be kept as string, not parsed HTTPRequest httpRequest = adapter.buildRequest( HTTPRequest.HTTPMethod.POST, path, queryParams, headers, body // Pass body as string, not parsed ); // Set up response handling resp.setStatus(200); resp.setContentType("application/json"); // Execute the request through the adapter RESTResponse response = adapter.execute( httpRequest, RESTResponse.class, error -> handleError(resp, error), responseHeaders -> responseHeaders.forEach((k, v) -> resp.setHeader(k, v)) ); // Write response if (response != null) { PrintWriter writer = resp.getWriter(); // Check if we need to inject test credentials Map testCredentials = IcebergRESTCatalogAdapterWithPlanSupport.getTestCredentials(); if (testCredentials != null && !testCredentials.isEmpty()) { // Inject storage-credentials into the response JSON String responseJson = mapper.writeValueAsString(response); String modifiedJson = injectStorageCredentials(responseJson, testCredentials); writer.write(modifiedJson); } else { // No credentials to inject, write response as-is mapper.writeValue(writer, response); } writer.flush(); } } catch (Exception e) { LOG.error("Error handling /plan request: {}", e.getMessage(), e); resp.setStatus(500); ErrorResponse error = ErrorResponse.builder() .responseCode(500) .withType("InternalServerError") .withMessage("Failed to process plan request: " + e.getMessage()) .build(); mapper.writeValue(resp.getWriter(), error); } } private void handleError(HttpServletResponse resp, ErrorResponse error) { try { resp.setStatus(error.code()); mapper.writeValue(resp.getWriter(), error); } catch (IOException e) { LOG.error("Failed to write error response: {}", e.getMessage(), e); } } private Map extractHeaders(HttpServletRequest req) { Map headers = new HashMap<>(); Enumeration headerNames = req.getHeaderNames(); while (headerNames.hasMoreElements()) { String name = headerNames.nextElement(); headers.put(name, req.getHeader(name)); } return headers; } private Map extractQueryParams(HttpServletRequest req) { Map params = new HashMap<>(); if (req.getQueryString() != null) { for (String param : req.getQueryString().split("&")) { String[] pair = param.split("=", 2); if (pair.length == 2) { params.put(pair[0], pair[1]); } } } return params; } private String extractBody(HttpServletRequest req) throws IOException { BufferedReader reader = req.getReader(); return reader.lines().collect(Collectors.joining()); } /** * Inject storage-credentials section into the plan response JSON. * Follows Iceberg REST catalog spec structure for credentials: * { * "storage-credentials": [{ * "config": { * "s3.access-key-id": "...", * ... * } * }] * } */ private String injectStorageCredentials( String originalJson, Map credentials) throws IOException { // Parse original JSON Map responseMap = mapper.readValue(originalJson, Map.class); // Build storage-credentials structure Map credConfig = new HashMap<>(credentials); Map credWrapper = new HashMap<>(); credWrapper.put("config", credConfig); // Add as array (spec requires array even with single element) responseMap.put("storage-credentials", Collections.singletonList(credWrapper)); // Serialize back to JSON return mapper.writeValueAsString(responseMap); } } ================================================ FILE: iceberg/src/test/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister ================================================ org.apache.iceberg.spark.source.IcebergSource ================================================ FILE: iceberg/src/test/scala/org/apache/spark/sql/delta/CloneIcebergSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.sql.Date import java.sql.Timestamp import java.time.LocalDateTime import java.time.LocalTime import java.time.format.DateTimeFormatter import java.util.TimeZone import scala.collection.JavaConverters._ import scala.util.Try import org.apache.spark.sql.delta.commands.convert.ConvertUtils import org.apache.spark.sql.delta.schema.SchemaMergingUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.{DataSkippingDeltaTestsUtils, StatisticsCollection} import org.apache.spark.sql.delta.util.JsonUtils import org.apache.iceberg.Schema import org.apache.iceberg.hadoop.HadoopTables import org.apache.iceberg.spark.{SparkSchemaUtil => IcebergSparkSchemaUtil} import org.apache.iceberg.types.Types import org.apache.iceberg.types.Types.NestedField import org.apache.spark.SparkConf import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, microsToLocalDateTime, stringToDate, stringToTimestamp, stringToTimestampWithoutTimeZone, toJavaDate, toJavaTimestamp} import org.apache.spark.sql.functions.{col, expr, from_json, lit, struct, substring} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{Decimal, DecimalType, LongType, StringType, StructField, StructType, TimestampType} import org.apache.spark.unsafe.types.UTF8String // scalastyle:on import.ordering.noEmptyLine case class DeltaStatsClass( numRecords: Int, maxValues: Map[String, String], minValues: Map[String, String], nullCount: Map[String, Int]) trait CloneIcebergSuiteBase extends QueryTest with DataSkippingDeltaTestsUtils with ConvertIcebergToDeltaUtils { override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaSQLConf.DELTA_CONVERT_ICEBERG_PARTITION_EVOLUTION_ENABLED.key, "true") } protected val cloneTable = "clone" // The identifier of clone source, can be either path-based or name-based. protected def sourceIdentifier: String protected def supportedModes: Seq[String] = Seq("SHALLOW") protected def toDate(date: String): Date = { toJavaDate(stringToDate(UTF8String.fromString(date)).get) } protected def physicalNamesAreEqual( sourceSchema: StructType, targetSchema: StructType): Boolean = { val sourcePathToPhysicalName = SchemaMergingUtils.explode(sourceSchema).map { case (path, field) => path -> DeltaColumnMapping.getPhysicalName(field) }.toMap val targetPathToPhysicalName = SchemaMergingUtils.explode(targetSchema).map { case (path, field) => path -> DeltaColumnMapping.getPhysicalName(field) }.toMap targetPathToPhysicalName.foreach { case (path, physicalName) => if (!sourcePathToPhysicalName.contains(path) || physicalName != sourcePathToPhysicalName(path)) { return false } } sourcePathToPhysicalName.size == targetPathToPhysicalName.size } protected def testClone(testName: String)(f: String => Unit): Unit = supportedModes.foreach { mode => test(s"$testName - $mode") { f(mode) } } testClone("table with deleted files") { mode => withTable(table, cloneTable) { spark.sql( s"""CREATE TABLE $table (id bigint, data string) |USING iceberg PARTITIONED BY (data) |TBLPROPERTIES ('write.format.default' = 'PARQUET')""".stripMargin) spark.sql(s"INSERT INTO $table VALUES (1, 'a'), (2, 'b'), (3, 'c')") spark.sql(s"DELETE FROM $table WHERE data > 'a'") checkAnswer(spark.sql(s"SELECT * from $table"), Row(1, "a") :: Nil) spark.sql(s"CREATE TABLE $cloneTable $mode CLONE $sourceIdentifier") assert(SchemaMergingUtils.equalsIgnoreCaseAndCompatibleNullability( DeltaLog.forTable(spark, TableIdentifier(cloneTable)).snapshot.schema, new StructType().add("id", LongType).add("data", StringType))) checkAnswer(spark.table(cloneTable), Row(1, "a") :: Nil) } } protected def runCreateOrReplace(mode: String, source: String): DataFrame = { Try(spark.sql(s"DELETE FROM $cloneTable")) spark.sql(s"CREATE OR REPLACE TABLE $cloneTable $mode CLONE $source") } testClone("table with renamed columns") { mode => withTable(table, cloneTable) { spark.sql( s"""CREATE TABLE $table (id bigint, data string) |USING iceberg PARTITIONED BY (data)""".stripMargin) spark.sql(s"INSERT INTO $table VALUES (1, 'a'), (2, 'b')") spark.sql("ALTER TABLE local.db.table RENAME COLUMN id TO id2") spark.sql(s"INSERT INTO $table VALUES (3, 'c')") // Parquet files still have the old schema assert( SchemaMergingUtils.equalsIgnoreCaseAndCompatibleNullability( spark.read.format("parquet").load(tablePath + "/data").schema, new StructType().add("id", LongType).add("data", StringType))) runCreateOrReplace(mode, sourceIdentifier) // The converted delta table will get the updated schema assert( SchemaMergingUtils.equalsIgnoreCaseAndCompatibleNullability( DeltaLog.forTable(spark, TableIdentifier(cloneTable)).snapshot.schema, new StructType().add("id2", LongType).add("data", StringType))) checkAnswer(spark.table(cloneTable), Row(1, "a") :: Row(2, "b") :: Row(3, "c") :: Nil) } } testClone("create or replace table - same schema") { mode => withTable(table, cloneTable) { spark.sql( s"""CREATE TABLE $table (id bigint, data string) |USING iceberg PARTITIONED BY (data)""".stripMargin) // Add some rows to check the initial CLONE. spark.sql(s"INSERT INTO $table VALUES (1, 'a'), (2, 'b')") runCreateOrReplace(mode, sourceIdentifier) checkAnswer(spark.table(cloneTable), Row(1, "a") :: Row(2, "b") :: Nil) // Add more rows to check incremental update with REPLACE. spark.sql(s"INSERT INTO $table VALUES (3, 'c')") runCreateOrReplace(mode, sourceIdentifier) checkAnswer(spark.table(cloneTable), Row(1, "a") :: Row(2, "b") :: Row(3, "c") :: Nil) } } testClone("create or replace table - renamed column") { mode => withTable(table, cloneTable) { spark.sql( s"""CREATE TABLE $table (id bigint, data string) |USING iceberg PARTITIONED BY (data)""".stripMargin) spark.sql(s"INSERT INTO $table VALUES (1, 'a'), (2, 'b')") runCreateOrReplace(mode, sourceIdentifier) assert( SchemaMergingUtils.equalsIgnoreCaseAndCompatibleNullability( DeltaLog.forTable(spark, TableIdentifier(cloneTable)).snapshot.schema, new StructType().add("id", LongType).add("data", StringType))) checkAnswer(spark.table(cloneTable), Row(1, "a") :: Row(2, "b") :: Nil) // Rename column 'id' into column 'id2'. spark.sql("ALTER TABLE local.db.table RENAME COLUMN id TO id2") spark.sql(s"INSERT INTO $table VALUES (3, 'c')") // Update the cloned delta table with REPLACE. runCreateOrReplace(mode, sourceIdentifier) assert( SchemaMergingUtils.equalsIgnoreCaseAndCompatibleNullability( DeltaLog.forTable(spark, TableIdentifier(cloneTable)).snapshot.schema, new StructType().add("id2", LongType).add("data", StringType))) checkAnswer(spark.table(cloneTable), Row(1, "a") :: Row(2, "b") :: Row(3, "c") :: Nil) } } testClone("create or replace table - deleted rows") { mode => withTable(table, cloneTable) { spark.sql( s"""CREATE TABLE $table (id bigint, data string) |USING iceberg PARTITIONED BY (data)""".stripMargin) spark.sql(s"INSERT INTO $table VALUES (1, 'a'), (2, 'b'), (3, 'c')") runCreateOrReplace(mode, sourceIdentifier) checkAnswer(spark.table(cloneTable), Row(1, "a") :: Row(2, "b") :: Row(3, "c") :: Nil) // Delete some rows from the iceberg table. spark.sql(s"DELETE FROM $table WHERE data > 'a'") checkAnswer( spark.sql(s"SELECT * from $table"), Row(1, "a") :: Nil) runCreateOrReplace(mode, sourceIdentifier) checkAnswer(spark.table(cloneTable), Row(1, "a") :: Nil) } } testClone("create or replace table - schema with nested column") { mode => withTable(table, cloneTable) { spark.sql( s"""CREATE TABLE $table (id bigint, person struct) |USING iceberg PARTITIONED BY (truncate(person.name, 2))""".stripMargin) spark.sql(s"INSERT INTO $table VALUES (1, ('AaAaAa', 10)), (2, ('BbBbBb', 20))") runCreateOrReplace(mode, sourceIdentifier) checkAnswer( spark.table(cloneTable), Row(1, Row("AaAaAa", 10), "Aa") :: Row(2, Row("BbBbBb", 20), "Bb") :: Nil) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(cloneTable)) val schemaBefore = deltaLog.update().schema spark.sql(s"INSERT INTO $table VALUES (3, ('AaZzZz', 30)), (4, ('CcCcCc', 40))") runCreateOrReplace(mode, sourceIdentifier) checkAnswer( spark.table(cloneTable), Row(1, Row("AaAaAa", 10), "Aa") :: Row(2, Row("BbBbBb", 20), "Bb") :: Row(3, Row("AaZzZz", 30), "Aa") :: Row(4, Row("CcCcCc", 40), "Cc") :: Nil) assert(physicalNamesAreEqual(schemaBefore, deltaLog.update().schema)) } } testClone("create or replace table - add partition field") { mode => withTable(table, cloneTable) { spark.sql( s"""CREATE TABLE $table (date date, id bigint, category string, price double) | USING iceberg PARTITIONED BY (date)""".stripMargin) // scalastyle:off deltahadoopconfiguration val hadoopTables = new HadoopTables(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration val icebergTable = hadoopTables.load(tablePath) val icebergTableSchema = IcebergSparkSchemaUtil.convert(icebergTable.schema()) val df1 = spark.createDataFrame( Seq( Row(toDate("2022-01-01"), 1L, "toy", 2.5D), Row(toDate("2022-01-01"), 2L, "food", 0.6D), Row(toDate("2022-02-05"), 3L, "food", 1.4D), Row(toDate("2022-02-05"), 4L, "toy", 10.2D)).asJava, icebergTableSchema) df1.writeTo(table).append() runCreateOrReplace(mode, sourceIdentifier) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(cloneTable)) assert(deltaLog.snapshot.metadata.partitionColumns == Seq("date")) checkAnswer(spark.table(cloneTable), df1) // Add a new partition field from the existing column "category" icebergTable.refresh() icebergTable.updateSpec().addField("category").commit() // Invalidate cache and load the updated partition spec spark.sql(s"REFRESH TABLE $table") val df2 = spark.createDataFrame( Seq( Row(toDate("2022-02-05"), 5L, "toy", 5.8D), Row(toDate("2022-06-04"), 6L, "toy", 20.1D)).asJava, icebergTableSchema) df2.writeTo(table).append() runCreateOrReplace(mode, sourceIdentifier) assert(deltaLog.update().metadata.partitionColumns == Seq("date", "category")) // Old data of cloned Delta table has null on the new partition field. checkAnswer(spark.table(cloneTable), df1.withColumn("category", lit(null)).union(df2)) // Iceberg table projects existing value of old data to the new partition field though. checkAnswer(spark.sql(s"SELECT * FROM $table"), df1.union(df2)) } } testClone("create or replace table - remove partition field") { mode => withTable(table, cloneTable) { spark.sql( s"""CREATE TABLE $table (date date, id bigint, category string, price double) | USING iceberg PARTITIONED BY (date)""".stripMargin) // scalastyle:off deltahadoopconfiguration val hadoopTables = new HadoopTables(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration val icebergTable = hadoopTables.load(tablePath) val icebergTableSchema = IcebergSparkSchemaUtil.convert(icebergTable.schema()) val df1 = spark.createDataFrame( Seq( Row(toDate("2022-01-01"), 1L, "toy", 2.5D), Row(toDate("2022-01-01"), 2L, "food", 0.6D), Row(toDate("2022-02-05"), 3L, "food", 1.4D), Row(toDate("2022-02-05"), 4L, "toy", 10.2D)).asJava, icebergTableSchema) df1.writeTo(table).append() runCreateOrReplace(mode, sourceIdentifier) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(cloneTable)) assert(deltaLog.snapshot.metadata.partitionColumns == Seq("date")) checkAnswer(spark.table(cloneTable), df1) // Remove the partition field "date" icebergTable.refresh() icebergTable.updateSpec().removeField("date").commit() // Invalidate cache and load the updated partition spec spark.sql(s"REFRESH TABLE $table") val df2 = spark.createDataFrame( Seq( Row(toDate("2022-02-05"), 5L, "toy", 5.8D), Row(toDate("2022-06-04"), 6L, "toy", 20.1D)).asJava, icebergTableSchema) df2.writeTo(table).append() runCreateOrReplace(mode, sourceIdentifier) assert(deltaLog.update().metadata.partitionColumns.isEmpty) // Both cloned Delta table and Iceberg table has data for the removed partition field. checkAnswer(spark.table(cloneTable), df1.union(df2)) checkAnswer(spark.table(cloneTable), spark.sql(s"SELECT * FROM $table")) } } testClone("create or replace table - replace partition field") { mode => withTable(table, cloneTable) { spark.sql( s"""CREATE TABLE $table (date date, id bigint, category string, price double) | USING iceberg PARTITIONED BY (date)""".stripMargin) // scalastyle:off deltahadoopconfiguration val hadoopTables = new HadoopTables(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration val icebergTable = hadoopTables.load(tablePath) val icebergTableSchema = IcebergSparkSchemaUtil.convert(icebergTable.schema()) val df1 = spark.createDataFrame( Seq( Row(toDate("2022-01-01"), 1L, "toy", 2.5D), Row(toDate("2022-01-01"), 2L, "food", 0.6D), Row(toDate("2022-02-05"), 3L, "food", 1.4D), Row(toDate("2022-02-05"), 4L, "toy", 10.2D)).asJava, icebergTableSchema) df1.writeTo(table).append() runCreateOrReplace(mode, sourceIdentifier) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(cloneTable)) assert(deltaLog.snapshot.metadata.partitionColumns == Seq("date")) checkAnswer(spark.table(cloneTable), df1) // Replace the partition field "date" with a transformed field "month(date)" icebergTable.refresh() icebergTable.updateSpec().removeField("date") .addField(org.apache.iceberg.expressions.Expressions.month("date")) .commit() // Invalidate cache and load the updated partition spec spark.sql(s"REFRESH TABLE $table") val df2 = spark.createDataFrame( Seq( Row(toDate("2022-02-05"), 5L, "toy", 5.8D), Row(toDate("2022-06-04"), 6L, "toy", 20.1D)).asJava, icebergTableSchema) df2.writeTo(table).append() runCreateOrReplace(mode, sourceIdentifier) assert(deltaLog.update().metadata.partitionColumns == Seq("date_month")) // Old data of cloned Delta table has null on the new partition field. checkAnswer(spark.table(cloneTable), df1.withColumn("date_month", lit(null)) .union(df2.withColumn("date_month", substring(col("date") cast "String", 1, 7)))) // The new partition field is a hidden metadata column in Iceberg. checkAnswer( spark.table(cloneTable).drop("date_month"), spark.sql(s"SELECT * FROM $table")) } } testClone("Enables column mapping table feature") { mode => withTable(table, cloneTable) { spark.sql( s"""CREATE TABLE $table (id bigint, data string) |USING iceberg PARTITIONED BY (data)""".stripMargin) spark.sql(s"CREATE TABLE $cloneTable $mode CLONE $sourceIdentifier") val log = DeltaLog.forTable(spark, TableIdentifier(cloneTable)) val protocol = log.update().protocol assert(protocol.isFeatureSupported(ColumnMappingTableFeature)) } } testClone("Iceberg bucket partition should be converted to unpartitioned delta table") { mode => withTable(table, cloneTable) { spark.sql( s"""CREATE TABLE $table (date date, id bigint, category string, price double) | USING iceberg PARTITIONED BY (bucket(2, id))""".stripMargin) // scalastyle:off deltahadoopconfiguration val hadoopTables = new HadoopTables(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration val icebergTable = hadoopTables.load(tablePath) val icebergTableSchema = IcebergSparkSchemaUtil.convert(icebergTable.schema()) val df1 = spark.createDataFrame( Seq( Row(toDate("2022-01-01"), 1L, "toy", 2.5D), Row(toDate("2022-01-01"), 2L, "food", 0.6D), Row(toDate("2022-02-05"), 3L, "food", 1.4D), Row(toDate("2022-02-05"), 4L, "toy", 10.2D)).asJava, icebergTableSchema) df1.writeTo(table).append() runCreateOrReplace(mode, sourceIdentifier) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(cloneTable)) assert(deltaLog.snapshot.metadata.partitionColumns.isEmpty) checkAnswer(spark.table(cloneTable), df1) checkAnswer(spark.sql(s"select * from $cloneTable where id = 1"), df1.where("id = 1")) // clone should fail with flag off withSQLConf(DeltaSQLConf.DELTA_CONVERT_ICEBERG_BUCKET_PARTITION_ENABLED.key -> "false") { df1.writeTo(table).append() val ae = intercept[UnsupportedOperationException] { runCreateOrReplace(mode, sourceIdentifier) } assert(ae.getMessage.contains("bucket partition")) } } } private def assertStats(deltaLog: DeltaLog, expectedStats: Seq[String]): Unit = { val addFiles = deltaLog.update().allFiles.collectAsList().iterator().asScala val addFilesSortedByIndices = addFiles.toList.sortBy { f => f.partitionValues.head._2 } addFilesSortedByIndices.zip(expectedStats).foreach { case (f, expectedStat) => val parsedStats = JsonUtils.fromJson[DeltaStatsClass]( f.stats ) assert(parsedStats.numRecords == 1) assert(parsedStats.minValues("col2") == expectedStat) assert(parsedStats.maxValues("col2") == expectedStat) } } private def assertStateReconstruction( deltaLog: DeltaLog, extractFunc: Row => String, expectedStats: Seq[String]): Unit = { val snapshot = deltaLog.update() val analyzedDf = snapshot.withStatsDeduplicated.queryExecution.analyzed.toString val statsCol = if (analyzedDf.contains("stats_parsed")) "stats_parsed" else "stats" val stats = snapshot.withStats.select(statsCol) val minStats = stats.select(s"$statsCol.minValues.col2").collect() assert(minStats.map(extractFunc(_)).toSet == expectedStats.toSet) val maxStats = stats.select(s"$statsCol.maxValues.col2").collect() assert(maxStats.map(extractFunc(_)).toSet == expectedStats.toSet) } private case class DataSkippingTestParam( predicate: String, expectedFilesReadNum: Int, expectedFilesReadIndices: Set[Int]) /** * E2E test stats conversions and dataSkipping for an iceberg dataType * It will write data into the iceberg table, * verify stats of the addFiles and results of dataSkipping on cloned delta table * * @param icebergDataType Iceberg data type to test * For example, "date" for date dataType * @param tableData Data to write into the table corresponding to data type * For example, Seq( * toDate("2015-01-25"), // index 1 * toDate("1917-02-10") // index 2 * ) * It will be written into col2 * @param extractFunc Function to extract the value from the row containing only stat * For example, for date type, it would be row => row.getDate(0).toString * @param expectedStats: Expected stat values in json string after extraction * For example, for Date("2025-01-25"), it would be "2025-01-25" * @param dataSkippingTestParams DataSkipping performed and what to verify * For example, * DataSkippingTestParam( * predicate = "col2 < '1918-01-25'", * expectedFilesReadNum = 1, * expectedFilesReadIndices = Set(2) // indices of files expected to select out * ) * @param mode Clone mode, for example, by path */ private def testStatsConversionAndDataSkipping( icebergDataType: String, tableData: Seq[Any], extractFunc: Row => String, expectedStats: Seq[String], dataSkippingTestParams: Seq[DataSkippingTestParam], mode: String): Unit = { withSQLConf(DeltaSQLConf.DELTA_CONVERT_ICEBERG_STATS.key-> "true") { withTable(table, cloneTable) { // Create Iceberg table with date type spark.sql( s"""CREATE TABLE $table (col1 int, col2 $icebergDataType) | USING iceberg PARTITIONED BY (col1)""".stripMargin) // Write into Iceberg table // scalastyle:off deltahadoopconfiguration val hadoopTables = new HadoopTables(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration val icebergTable = hadoopTables.load(tablePath) val icebergTableSchema = IcebergSparkSchemaUtil.convert(icebergTable.schema()) val df = spark.createDataFrame( tableData.zipWithIndex.map { case (elem, index) => Row(index + 1, elem) }.asJava, icebergTableSchema ) df.writeTo(table).append() runCreateOrReplace(mode, sourceIdentifier) val deltaLog = DeltaLog.forTable( spark, spark.sessionState.catalog.getTableMetadata(TableIdentifier(cloneTable)) ) // Verify converted stats against expected stats assertStats(deltaLog, expectedStats) assertStateReconstruction(deltaLog, extractFunc, expectedStats) // Check table read results checkAnswer(spark.table(cloneTable), df) // Check data skipping results dataSkippingTestParams.foreach { dataSkippingParam => val (predicate, expectedFilesReadNum, expectedFilesReadIndices) = (dataSkippingParam.predicate, dataSkippingParam.expectedFilesReadNum, dataSkippingParam.expectedFilesReadIndices) val filesRead = getFilesRead(spark, deltaLog, predicate, checkEmptyUnusedFilters = false) try { checkAnswer( spark.sql(s"select * from $cloneTable where $predicate"), df.where(predicate) ) assert(filesRead.size == expectedFilesReadNum) assert(filesRead.map(_.partitionValues.head._2).toSet == expectedFilesReadIndices.map(_.toString)) } catch { case e: Throwable => throw new RuntimeException( s"DataSkipping Failed for predicate: $predicate: " + s"expectedFilesReadNum: $expectedFilesReadNum, " + s"expectedFilesReadIndices: $expectedFilesReadIndices " + s"actualFilesRead: $filesRead" + s"actualFilesIndices: ${filesRead.map(_.partitionValues.head._2)}", e) } } } } } testClone("Convert Iceberg date type") { mode => testStatsConversionAndDataSkipping( icebergDataType = "date", tableData = Seq( toDate("2015-01-25"), // index 1 toDate("1917-02-10"), // index 2 toDate("2050-06-23") // index 3 ), extractFunc = row => row.getDate(0).toString, expectedStats = Seq("2015-01-25", "1917-02-10", "2050-06-23"), dataSkippingTestParams = Seq( DataSkippingTestParam( predicate = "col2 > '2030-01-25'", expectedFilesReadNum = 1, expectedFilesReadIndices = Set(3) ), DataSkippingTestParam( predicate = "col2 < '1917-02-11'", expectedFilesReadNum = 1, expectedFilesReadIndices = Set(2) ) ), mode = mode ) } // int32 for 1 <= precision <= 9 testClone("Convert Iceberg decimal type - int32 in parquet") { mode => testStatsConversionAndDataSkipping( icebergDataType = "decimal(6, 5)", tableData = Seq(Decimal(0.123)), extractFunc = row => row.getDecimal(0).toString, expectedStats = Seq("0.12300"), dataSkippingTestParams = Seq( DataSkippingTestParam( predicate = "col2 > 0.123", expectedFilesReadNum = 0, expectedFilesReadIndices = Set() ), DataSkippingTestParam( predicate = "col2 >= 0.123", expectedFilesReadNum = 1, expectedFilesReadIndices = Set(1) ) ), mode = mode ) } // int64 for 10 <= precision <= 18 testClone("Convert Iceberg decimal type - int64 in parquet") { mode => testStatsConversionAndDataSkipping( icebergDataType = "decimal(16, 4)", tableData = Seq(BigDecimal("123456789123.4567")), extractFunc = row => row.getDecimal(0).toString, expectedStats = Seq("123456789123.4567"), dataSkippingTestParams = Seq( DataSkippingTestParam( predicate = "col2 < 123456789123.4567", expectedFilesReadNum = 0, expectedFilesReadIndices = Set() ), DataSkippingTestParam( predicate = "col2 <= 123456789123.4567", expectedFilesReadNum = 1, expectedFilesReadIndices = Set(1) ) ), mode = mode ) } // array for precision > 18 testClone("Convert Iceberg decimal type - array in parquet") { mode => testStatsConversionAndDataSkipping( icebergDataType = "decimal(20, 8)", tableData = Seq( BigDecimal("111111.111"), // index 1 BigDecimal("111111.112"), // index 2 Decimal(123.5) // index 3 ), extractFunc = row => row.getDecimal(0).toString, expectedStats = Seq("111111.11100000", "111111.11200000", "123.50000000"), dataSkippingTestParams = Seq( DataSkippingTestParam( predicate = "col2 > 111111.111", expectedFilesReadNum = 1, expectedFilesReadIndices = Set(2) ), DataSkippingTestParam( predicate = "col2 <= 111111.111", expectedFilesReadNum = 2, expectedFilesReadIndices = Set(1, 3) ), DataSkippingTestParam( predicate = "col2 < 123.5001", expectedFilesReadNum = 1, expectedFilesReadIndices = Set(3) ), DataSkippingTestParam( predicate = "col2 < 123.5", expectedFilesReadNum = 0, expectedFilesReadIndices = Set() ) ), mode = mode ) } // common decimal type used in iceberg testClone("Convert Iceberg decimal type - mixed") { mode => testStatsConversionAndDataSkipping( icebergDataType = "decimal(38, 0)", tableData = Seq( BigDecimal("123456789"), // index 1 BigDecimal("123456789123456789"), // index 2 BigDecimal("123456789123456789123456789"), // index 3 BigDecimal("123456789123456789123456789123456789"), // index 4 BigDecimal("12345678912345678912345678912345678912") // index 5 ), extractFunc = row => row.getDecimal(0).toString, expectedStats = Seq( "123456789", "123456789123456789", "123456789123456789123456789", "123456789123456789123456789123456789", "12345678912345678912345678912345678912" ), dataSkippingTestParams = Seq( DataSkippingTestParam( predicate = "col2 <= 123456789", expectedFilesReadNum = 1, expectedFilesReadIndices = Set(1) ), DataSkippingTestParam( predicate = "col2 == 123456789123456789", expectedFilesReadNum = 1, expectedFilesReadIndices = Set(2) ), DataSkippingTestParam( predicate = "col2 < 12345678912345678912345678912345678912", expectedFilesReadNum = 4, expectedFilesReadIndices = Set(1, 2, 3, 4) ), DataSkippingTestParam( predicate = "col2 >= 123456789123456789123456789123456789", expectedFilesReadNum = 2, expectedFilesReadIndices = Set(4, 5) ) ), mode = mode ) } // Exactly on minutes testClone("Convert Iceberg timestamptz type - 1") { mode => testStatsConversionAndDataSkipping( icebergDataType = "timestamp", // spark timestamp => iceberg timestamptz tableData = Seq( toTimestamp("1908-03-15 10:1:17") ), extractFunc = row => { timestamptzExtracter(row, pattern = "yyyy-MM-dd'T'HH:mm:ssXXX") }, expectedStats = Seq("1908-03-15T10:01:17+00:00"), dataSkippingTestParams = Seq( DataSkippingTestParam( predicate = "col2 > TIMESTAMP'1908-03-15T10:01:18+00:00'", expectedFilesReadNum = 0, expectedFilesReadIndices = Set() ), DataSkippingTestParam( predicate = "col2 <= TIMESTAMP'1908-03-15T10:01:17+00:00'", expectedFilesReadNum = 1, expectedFilesReadIndices = Set(1) ) ), mode = mode ) } // Fractional time testClone("Convert Iceberg timestamptz type - 2") { mode => testStatsConversionAndDataSkipping( icebergDataType = "timestamp", // spark timestamp => iceberg timestamptz tableData = Seq( toTimestamp("1997-12-11 5:40:19.23349") ), extractFunc = row => { timestamptzExtracter(row, pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSSSXXX") }, expectedStats = Seq("1997-12-11T05:40:19.23349+00:00"), dataSkippingTestParams = Seq( DataSkippingTestParam( predicate = "col2 > TIMESTAMP'1997-12-11T05:40:19.233+00:00'", expectedFilesReadNum = 1, expectedFilesReadIndices = Set(1) ), DataSkippingTestParam( predicate = "col2 <= TIMESTAMP'1997-12-11T05:40:19.10+00:00'", expectedFilesReadNum = 0, expectedFilesReadIndices = Set() ) ), mode = mode ) } // Customized timezone testClone("Convert Iceberg timestamptz type - 3") { mode => testStatsConversionAndDataSkipping( icebergDataType = "timestamp", // spark timestamp => iceberg timestamptz tableData = Seq( toTimestamp("2077-11-11 3:23:11.23456+02:15") ), extractFunc = row => { timestamptzExtracter(row, pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSSSXXX") }, expectedStats = Seq("2077-11-11T01:08:11.23456+00:00"), dataSkippingTestParams = Seq( DataSkippingTestParam( predicate = "col2 > TIMESTAMP'2077-11-11T03:23:11.23456+02:16'", expectedFilesReadNum = 1, expectedFilesReadIndices = Set(1) ), DataSkippingTestParam( predicate = "col2 < TIMESTAMP'2077-11-11T03:23:11.23456+02:16'", expectedFilesReadNum = 0, expectedFilesReadIndices = Set() ) ), mode = mode ) } // Exactly on minutes testClone("Convert Iceberg timestamp type - 1") { mode => testStatsConversionAndDataSkipping( icebergDataType = "timestamp_ntz", // spark timestamp_ntz => iceberg timestamp tableData = Seq( toTimestampNTZ("2024-01-02T02:04:05.123456") ), extractFunc = row => { row.get(0).asInstanceOf[LocalDateTime].toString }, expectedStats = Seq("2024-01-02T02:04:05.123456"), dataSkippingTestParams = Seq( DataSkippingTestParam( predicate = "col2 > TIMESTAMP'2024-01-02T02:04:04.123456'", expectedFilesReadNum = 1, expectedFilesReadIndices = Set(1) ) ), mode = mode ) } // Fractional time testClone("Convert Iceberg timestamp type - 2") { mode => testStatsConversionAndDataSkipping( icebergDataType = "timestamp_ntz", // spark timestamp_ntz => iceberg timestamp tableData = Seq( toTimestampNTZ("1712-4-29T06:23:49.12") ), extractFunc = row => { row.get(0).asInstanceOf[LocalDateTime].toString .replaceAll("0+$", "") // remove trailing zeros }, expectedStats = Seq("1712-04-29T06:23:49.12"), dataSkippingTestParams = Seq( DataSkippingTestParam( predicate = "col2 > TIMESTAMP'1712-04-29T06:23:49.11'", expectedFilesReadNum = 1, expectedFilesReadIndices = Set(1) ) ), mode = mode ) } private def toTimestamp(timestamp: String): Timestamp = { toJavaTimestamp(stringToTimestamp(UTF8String.fromString(timestamp), getZoneId(SQLConf.get.sessionLocalTimeZone)).get) } private def toTimestampNTZ(timestampNTZ: String): LocalDateTime = { microsToLocalDateTime( stringToTimestampWithoutTimeZone( UTF8String.fromString(timestampNTZ) ).get ) } private def timestamptzExtracter(row: Row, pattern: String): String = { val ts = row.getTimestamp(0).toLocalDateTime.atZone( getZoneId(TimeZone.getDefault.getID) ) ts.withZoneSameInstant(getZoneId(SQLConf.get.sessionLocalTimeZone)) .format(DateTimeFormatter.ofPattern(pattern)) .replace("UTC", "+00:00") .replace("Z", "+00:00") } } class CloneIcebergByPathSuite extends CloneIcebergSuiteBase { override def sourceIdentifier: String = s"iceberg.`$tablePath`" test("negative case: select from iceberg table using path") { withTable(table) { val ae = intercept[AnalysisException] { sql(s"SELECT * FROM $sourceIdentifier") } assert(ae.getMessage.contains("does not support batch scan")) } } } /** * This suite test features in Iceberg that is not directly supported by Spark. * See also [[NonSparkIcebergTestUtils]]. * We do not put these tests in or extend from [[CloneIcebergSuiteBase]] because they * use non-Spark way to create test data. */ class CloneNonSparkIcebergByPathSuite extends QueryTest with ConvertIcebergToDeltaUtils { protected val cloneTable = "clone" private def sourceIdentifier: String = s"iceberg.`$tablePath`" private def runCreateOrReplace(mode: String, source: String): DataFrame = { Try(spark.sql(s"DELETE FROM $cloneTable")) spark.sql(s"CREATE OR REPLACE TABLE $cloneTable $mode CLONE $source") } private val mode = "SHALLOW" test("cast Iceberg TIME to Spark long") { withTable(table, cloneTable) { val schema = new Schema( Seq[NestedField]( NestedField.required(1, "id", Types.IntegerType.get), NestedField.required(2, "event_time", Types.TimeType.get) ).asJava ) val rows = Seq( Map( "id" -> 1, "event_time" -> LocalTime.of(14, 30, 11) ) ) NonSparkIcebergTestUtils.createIcebergTable(spark, tablePath, schema, rows) intercept[UnsupportedOperationException] { runCreateOrReplace(mode, sourceIdentifier) } withSQLConf(DeltaSQLConf.DELTA_CONVERT_ICEBERG_CAST_TIME_TYPE.key -> "true") { runCreateOrReplace(mode, sourceIdentifier) val expectedMicrosec = (14 * 3600 + 30 * 60 + 11) * 1000000L checkAnswer(spark.table(cloneTable), Row(1, expectedMicrosec) :: Nil) val clonedDeltaTable = DeltaLog.forTable( spark, spark.sessionState.catalog.getTableMetadata(TableIdentifier(cloneTable)) ) assert(DeltaConfigs.CAST_ICEBERG_TIME_TYPE.fromMetaData(clonedDeltaTable.update().metadata)) } } } test("block data path not under table path") { withTable(table, cloneTable) { val schema = new Schema( Seq[NestedField]( NestedField.required(1, "id", Types.IntegerType.get), NestedField.required(2, "name", Types.StringType.get) ).asJava ) val rows = Seq( Map( "id" -> 1, "name" -> "alice" ) ) val table = NonSparkIcebergTestUtils.createIcebergTable(spark, tablePath, schema, rows) // Create a new data file not under the table path withTempDir { dir => val dataPath = dir.toPath.resolve("out_of_table.parquet").toAbsolutePath.toString NonSparkIcebergTestUtils.writeIntoIcebergTable( table, Seq(Map("id" -> 2, "name" -> "bob")), 2, Some(dataPath) ) val e = intercept[org.apache.spark.SparkException] { runCreateOrReplace(mode, sourceIdentifier) } assert(e.getMessage.contains("assertion failed: Fail to relativize path")) } } } } class CloneIcebergByNameSuite extends CloneIcebergSuiteBase { override def sourceIdentifier: String = table } trait DisablingConvertIcebergStats extends CloneIcebergSuiteBase { override def sparkConf: SparkConf = super.sparkConf.set(DeltaSQLConf.DELTA_CONVERT_ICEBERG_STATS.key, "false") } class CloneIcebergByPathNoConvertStatsSuite extends CloneIcebergByPathSuite with DisablingConvertIcebergStats class CloneIcebergByNameNoConvertStatsSuite extends CloneIcebergByNameSuite with DisablingConvertIcebergStats ================================================ FILE: iceberg/src/test/scala/org/apache/spark/sql/delta/ConvertIcebergToDeltaPartitionSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.io.File import java.sql.Timestamp import java.util.concurrent.TimeUnit import scala.collection.JavaConverters._ import scala.collection.mutable import org.apache.spark.sql.delta.commands.ConvertToDeltaCommand import org.apache.spark.sql.delta.schema.SchemaMergingUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.commons.io.FileUtils import org.apache.hadoop.fs.Path import org.apache.iceberg.Table import org.apache.spark.sql.{AnalysisException, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.types._ // scalastyle:on import.ordering.noEmptyLine abstract class ConvertIcebergToDeltaPartitioningUtils extends QueryTest with ConvertIcebergToDeltaUtils { override protected val schemaDDL = "id bigint, data string, size int, ts timestamp, dt date" protected lazy val schemaColumnNames: Seq[String] = schema.map(_.name) /** Original iceberg data used to check the correctness of conversion. */ protected def initRows: Seq[String] = Seq( "1L, 'abc', 100, cast('2021-06-01 18:00:00' as timestamp), cast('2021-06-01' as date)", "2L, 'ace', 200, cast('2022-07-01 20:00:00' as timestamp), cast('2022-07-01' as date)" ) /** Data added into both iceberg and converted delta to check post-conversion consistency. */ protected def incrRows: Seq[String] = Seq( "3L, 'acf', 300, cast('2023-07-01 03:00:00' as timestamp), cast('2023-07-01' as date)" ) protected override def test(testName: String, testTags: org.scalatest.Tag*) (testFun: => Any) (implicit pos: org.scalactic.source.Position): Unit = { Seq("true", "false").foreach { flag => val msg = if (flag == "true") "- with native partition values" else "- with inferred partition values" super.test(testName + msg, testTags : _*) { withSQLConf(DeltaSQLConf.DELTA_CONVERT_ICEBERG_USE_NATIVE_PARTITION_VALUES.key -> flag) { testFun } }(pos) } } /** * Creates an iceberg table with the default schema and the provided partition columns, writes * some original rows into the iceberg table for conversion. */ protected def createIcebergTable( tableName: String, partitionColumns: Seq[String], withRows: Seq[String] = initRows): Unit = { val partitionClause = if (partitionColumns.nonEmpty) s"PARTITIONED BY (${partitionColumns.mkString(",")})" else "" spark.sql(s"CREATE TABLE $tableName ($schemaDDL) USING iceberg $partitionClause") withRows.foreach{ row => spark.sql(s"INSERT INTO $tableName VALUES ($row)") } } /** * Tests ConvertToDelta on the provided iceberg table, and checks both schema and data of the * converted delta table. * * @param tableName: the iceberg table name. * @param tablePath: the iceberg table path. * @param partitionSchemaDDL: the expected partition schema DDL. * @param deltaPath: the location for the converted delta table. */ protected def testConvertToDelta( tableName: String, tablePath: String, partitionSchemaDDL: String, deltaPath: String): Unit = { // Convert at an external location to ease testing. ConvertToDeltaCommand( tableIdentifier = TableIdentifier(tablePath, Some("iceberg")), partitionSchema = None, collectStats = true, Some(deltaPath)).run(spark) // Check the converted table schema. validateConvertedSchema( readIcebergHadoopTable(tablePath), DeltaLog.forTable(spark, new Path(deltaPath)), StructType.fromDDL(partitionSchemaDDL)) // Check converted data. checkAnswer( // The converted delta table will have partition columns. spark.sql(s"select ${schemaColumnNames.mkString(",")} from delta.`$deltaPath`"), spark.sql(s"select * from $tableName")) } /** * Checks partition-based file skipping on the iceberg table (as parquet) and the converted delta * table to verify post-conversion partition consistency. * * @param icebergTableName: the iceberg table name. * @param icebergTablePath: the iceberg table path. * @param deltaTablePath: the converted delta table path. * @param filterAndFiles: a map from filter expression to the expected number of scanned files. */ protected def checkSkipping( icebergTableName: String, icebergTablePath: String, deltaTablePath: String, filterAndFiles: Map[String, Int] = Map.empty[String, Int]): Unit = { // Add the same data into both iceberg table and converted delta table. writeRows(icebergTableName, deltaTablePath, incrRows) // Disable file stats to check file skipping solely based on partition, please note this only // works for optimizable partition expressions, check 'optimizablePartitionExpressions.scala' // for the whole list of supported partition expressions. sql( s""" |ALTER TABLE delta.`$deltaTablePath` |SET TBLPROPERTIES ( | '${DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.key}' = '0')""".stripMargin) // Always check full scan. (filterAndFiles ++ Map("" -> 3)).foreach { case (filter, numFilesScanned) => val filterExpr = if (filter == "") "" else s"where $filter" checkAnswer( // The converted delta table will have partition columns. spark.sql( s"""SELECT ${schemaColumnNames.mkString(",")} FROM delta.`$deltaTablePath` | WHERE $filter""".stripMargin), spark.sql(s"SELECT * FROM $icebergTableName $filterExpr")) // Check the raw parquet partition directories written out by Iceberg checkAnswer( spark.sql(s"select * from parquet.`$icebergTablePath/data` $filterExpr"), spark.sql(s"select * from delta.`$deltaTablePath` $filterExpr")) assert( spark.sql(s"select * from delta.`$deltaTablePath` $filterExpr").inputFiles.length == numFilesScanned) } } /** * Validates the table schema and partition schema of the iceberg table and the converted delta * table. */ private def validateConvertedSchema( icebergTable: Table, convertedDeltaLog: DeltaLog, expectedPartitionSchema: StructType): Unit = { def mergeSchema(dataSchema: StructType, partitionSchema: StructType): StructType = { StructType(dataSchema.fields ++ partitionSchema.fields.filter { partField => !dataSchema.fields.exists(f => spark.sessionState.conf.resolver(partField.name, f.name))}) } val columnIds = mutable.Set[Long]() val schemaWithoutMetadata = SchemaMergingUtils.transformColumns(convertedDeltaLog.update().schema) { (_, field, _) => // all columns should have the columnID metadata assert(DeltaColumnMapping.hasColumnId(field)) // all columns should have physical name metadata assert(DeltaColumnMapping.hasPhysicalName(field)) // nest column ids should be distinct val id = DeltaColumnMapping.getColumnId(field) assert(!columnIds.contains(id)) columnIds.add(id) // the id can either be a data schema id or a identity transform partition field // or it is generated because it's a non-identity transform partition field assert( Option(icebergTable.schema().findField(id)).map(_.name()).contains(field.name) || icebergTable.spec().fields().asScala.map(_.name()).contains(field.name) ) field.copy(metadata = Metadata.empty) } assert(schemaWithoutMetadata == mergeSchema(schema, expectedPartitionSchema)) // check partition columns assert( expectedPartitionSchema.map(_.name) == convertedDeltaLog.update().metadata.partitionColumns) } /** * Writes the same rows into both the iceberg table and the converted delta table using the * default schema. */ protected def writeRows( icebergTableName: String, deltaTablePath: String, rows: Seq[String]): Unit = { // Write Iceberg rows.foreach { row => spark.sql(s"INSERT INTO $icebergTableName VALUES ($row)") } // Write Delta rows.foreach { row => val values = row.split(",") assert(values.length == schemaColumnNames.length) val valueAsColumns = values.zip(schemaColumnNames).map { case (value, column) => s"$value AS $column" } val df = spark.sql(valueAsColumns.mkString("SELECT ", ",", "")) df.write.format("delta").mode("append").save(deltaTablePath) } } } class ConvertIcebergToDeltaPartitioningSuite extends ConvertIcebergToDeltaPartitioningUtils { import testImplicits._ test("partition by timestamp year") { withTable(table) { createIcebergTable(table, Seq("years(ts)")) withTempDir { dir => testConvertToDelta(table, tablePath, "ts_year int", dir.getCanonicalPath) checkSkipping( table, tablePath, dir.getCanonicalPath, Map( "ts < cast('2021-06-01 00:00:00' as timestamp)" -> 1, "ts <= cast('2021-06-01 00:00:00' as timestamp)" -> 1, "ts > cast('2021-06-01 00:00:00' as timestamp)" -> 3, "ts > cast('2022-01-01 00:00:00' as timestamp)" -> 2) ) } } } test("partition by date year") { withTable(table) { createIcebergTable(table, Seq("years(dt)")) withTempDir { dir => testConvertToDelta(table, tablePath, "dt_year int", dir.getCanonicalPath) checkSkipping( table, tablePath, dir.getCanonicalPath, Map( "dt < cast('2021-06-01' as date)" -> 1, "dt <= cast('2021-06-01' as date)" -> 1, "dt > cast('2021-06-01' as date)" -> 3, "dt = cast('2022-08-01' as date)" -> 1) ) } } } test("partition by timestamp day") { withTable(table) { createIcebergTable(table, Seq("days(ts)")) withTempDir { dir => testConvertToDelta(table, tablePath, "ts_day date", dir.getCanonicalPath) checkSkipping( table, tablePath, dir.getCanonicalPath, Map("ts < cast('2021-07-01 00:00:00' as timestamp)" -> 1)) } } } test("partition by date day") { withTable(table) { createIcebergTable(table, Seq("days(dt)")) withTempDir { dir => testConvertToDelta(table, tablePath, "dt_day date", dir.getCanonicalPath) checkSkipping( table, tablePath, dir.getCanonicalPath, Map( "dt < cast('2021-06-01' as date)" -> 1, "dt <= cast('2021-06-01' as date)" -> 1, "dt > cast('2021-06-01' as date)" -> 3, "dt = cast('2022-07-01' as date)" -> 1) ) } } } test("partition by truncate string") { withTable(table) { createIcebergTable(table, Seq("truncate(data, 2)")) withTempDir { dir => testConvertToDelta(table, tablePath, "data_trunc string", dir.getCanonicalPath) checkSkipping( table, tablePath, dir.getCanonicalPath, Map( "data >= 'ac'" -> 2, "data >= 'ad'" -> 0 ) ) } } } test("partition by truncate long and int") { withTable(table) { // Include both positive and negative long values in the rows: positive will be rounded up // while negative will be rounded down. val sampleRows = Seq( "111L, 'abc', 100, cast('2021-06-01 18:00:00' as timestamp), cast('2021-06-01' as date)", "-11L, 'ace', -10, cast('2022-07-01 20:00:00' as timestamp), cast('2022-07-01' as date)") createIcebergTable(table, Seq("truncate(id, 10)", "truncate(size, 8)"), sampleRows) withTempDir { dir => val deltaPath = dir.getCanonicalPath testConvertToDelta(table, tablePath, "id_trunc long, size_trunc int", deltaPath) // TODO: make iceberg truncate partition expression optimizable and check file skipping. // Write the same rows again into the converted delta table and make sure the partition // value computed by delta are the same with iceberg. writeRows(table, deltaPath, sampleRows) checkAnswer( spark.sql(s"SELECT id_trunc, size_trunc FROM delta.`$deltaPath`"), Row(110L, 96) :: Row(-20L, -16) :: Row(110L, 96) :: Row(-20L, -16) :: Nil) } } } test("partition by identity") { withTable(table) { createIcebergTable(table, Seq("data")) withTempDir { dir => val deltaPath = new File(dir, "delta-table").getCanonicalPath testConvertToDelta(table, tablePath, "data string", deltaPath) checkSkipping(table, tablePath, deltaPath) spark.read.format("delta").load(deltaPath).inputFiles.foreach { fileName => val sourceFile = new File(fileName.stripPrefix("file:")) val targetFile = new File(dir, sourceFile.getName) FileUtils.copyFile(sourceFile, targetFile) val parquetFileSchema = spark.read.format("parquet").load(targetFile.getCanonicalPath).schema if (fileName.contains("acf")) { // new file written by delta SchemaMergingUtils.equalsIgnoreCaseAndCompatibleNullability( parquetFileSchema, StructType(schema.fields.filter(_.name != "data"))) } else { SchemaMergingUtils.equalsIgnoreCaseAndCompatibleNullability(parquetFileSchema, schema) } } } } } test("df writes and Insert Into with composite partitioning") { withTable(table) { createIcebergTable(table, Seq("years(dt), truncate(data, 3), id")) withTempDir { dir => val deltaPath = new File(dir, "/delta").getCanonicalPath testConvertToDelta( table, tablePath, "dt_year int, data_trunc string, id bigint", deltaPath) checkSkipping( table, tablePath, deltaPath, Map( "data >= 'ac'" -> 2, "data >= 'acg'" -> 0, "dt = cast('2022-07-01' as date) and data >= 'ac'" -> 1 ) ) // for Dataframe, we don't need to explicitly mention partition columns Seq((4L, "bcddddd", 400, new Timestamp(TimeUnit.DAYS.toMillis(10)), new java.sql.Date(TimeUnit.DAYS.toMillis(10)))) .toDF(schemaColumnNames: _*) .write.format("delta").mode("append").save(deltaPath) checkAnswer( spark.read.format("delta").load(deltaPath).where("id = 4") .select("id", "data", "dt_year", "data_trunc"), Row( 4, "bcddddd", // generated partition columns 1970, "bcd") :: Nil) val tempTablePath = dir.getCanonicalPath + "/temp" Seq((5, "c", 500, new Timestamp(TimeUnit.DAYS.toMillis(20)), new java.sql.Date(TimeUnit.DAYS.toMillis(20))) ).toDF(schemaColumnNames: _*) .write.format("delta").save(tempTablePath) val e = intercept[AnalysisException] { spark.sql( s""" | INSERT INTO delta.`$deltaPath` | SELECT * from delta.`$tempTablePath` |""".stripMargin) } assert(e.getMessage.contains("not enough data columns")) } } } test("partition by timestamp month") { withTable(table) { createIcebergTable(table, Seq("months(ts)")) withTempDir { dir => testConvertToDelta(table, tablePath, "ts_month string", dir.getCanonicalPath) // Do NOT infer partition column type for ts_month and dt_month since: 2020-01 will be // inferred as a date and cast it to 2020-01-01. withSQLConf("spark.sql.sources.partitionColumnTypeInference.enabled" -> "false") { checkSkipping( table, tablePath, dir.getCanonicalPath, Map( "ts < cast('2021-06-01 00:00:00' as timestamp)" -> 1, "ts <= cast('2021-06-01 00:00:00' as timestamp)" -> 1, "ts > cast('2021-06-01 00:00:00' as timestamp)" -> 3, "ts >= cast('2021-06-01 00:00:00' as timestamp)" -> 3, "ts < cast('2021-05-01 00:00:00' as timestamp)" -> 0, "ts > cast('2021-07-01 00:00:00' as timestamp)" -> 2, "ts = cast('2023-07-30 00:00:00' as timestamp)" -> 1, "ts > cast('2023-08-01 00:00:00' as timestamp)" -> 0)) } } } } test("partition by date month") { withTable(table) { createIcebergTable(table, Seq("months(dt)")) withTempDir { dir => testConvertToDelta(table, tablePath, "dt_month string", dir.getCanonicalPath) // Do NOT infer partition column type for ts_month and dt_month since: 2020-01 will be // inferred as a date and cast it to 2020-01-01. withSQLConf("spark.sql.sources.partitionColumnTypeInference.enabled" -> "false") { checkSkipping( table, tablePath, dir.getCanonicalPath, Map( "dt < cast('2021-06-01' as date)" -> 1, "dt <= cast('2021-06-01' as date)" -> 1, "dt > cast('2021-06-01' as date)" -> 3, "dt >= cast('2021-06-01' as date)" -> 3, "dt < cast('2021-05-01' as date)" -> 0, "dt > cast('2021-07-01' as date)" -> 2, "dt = cast('2023-07-30' as date)" -> 1, "dt > cast('2023-08-01' as date)" -> 0)) } } } } test("partition by timestamp hour") { withTable(table) { createIcebergTable(table, Seq("hours(ts)")) withTempDir { dir => testConvertToDelta(table, tablePath, "ts_hour string", dir.getCanonicalPath) checkSkipping(table, tablePath, dir.getCanonicalPath, Map( "ts < cast('2021-06-01 18:00:00' as timestamp)" -> 1, "ts <= cast('2021-06-01 18:00:00' as timestamp)" -> 1, "ts > cast('2021-06-01 18:00:00' as timestamp)" -> 3, "ts >= cast('2021-06-01 18:30:00' as timestamp)" -> 3, "ts < cast('2021-06-01 17:59:59' as timestamp)" -> 0, "ts = cast('2021-06-01 18:30:10' as timestamp)" -> 1, "ts > cast('2022-07-01 20:00:00' as timestamp)" -> 2, "ts > cast('2023-07-01 02:00:00' as timestamp)" -> 1, "ts > cast('2023-07-01 04:00:00' as timestamp)" -> 0)) } } } } ///////////////////////////////// // 5-DIGIT-YEAR TIMESTAMP TEST // ///////////////////////////////// class ConvertIcebergToDeltaPartitioningFiveDigitYearSuite extends ConvertIcebergToDeltaPartitioningUtils { override protected def initRows: Seq[String] = Seq( "1, 'abc', 100, cast('13168-11-15 18:00:00' as timestamp), cast('13168-11-15' as date)", "2, 'abc', 200, cast('2021-08-24 18:00:00' as timestamp), cast('2021-08-24' as date)" ) override protected def incrRows: Seq[String] = Seq( "3, 'acf', 300, cast('11267-07-15 18:00:00' as timestamp), cast('11267-07-15' as date)", "4, 'acf', 400, cast('2008-07-15 18:00:00' as timestamp), cast('2008-07-15' as date)" ) /** * Checks filtering on 5-digit year based on different policies. * * @param icebergTableName: the iceberg table name. * @param deltaTablePath: the converted delta table path. * @param partitionSchemaDDL: the partition schema DDL. * @param policy: time parser policy to determine 5-digit year handling. * @param filters: a list of filter expressions to check. */ private def checkFiltering( icebergTableName: String, deltaTablePath: String, partitionSchemaDDL: String, policy: String, filters: Seq[String]): Unit = { filters.foreach { filter => val filterExpr = if (filter == "") "" else s"where $filter" if (policy == "EXCEPTION" && filterExpr != "" && partitionSchemaDDL != "ts_year int" && partitionSchemaDDL != "ts_day date") { var thrownError = false val msg = try { spark.sql(s"select * from delta.`$deltaTablePath` $filterExpr").collect() } catch { case e: Throwable if e.isInstanceOf[org.apache.spark.SparkThrowable] && e.getMessage.contains("spark.sql.legacy.timeParserPolicy") => thrownError = true case other: Throwable => throw other } assert(thrownError, s"Error message $msg is incorrect.") } else { // check results of iceberg == delta checkAnswer( // the converted delta table will have partition columns spark.sql( s"select ${schema.fields.map(_.name).mkString(",")} from delta.`$deltaTablePath`"), spark.sql(s"select * from $icebergTableName")) } } } Seq("EXCEPTION", "CORRECTED", "LEGACY").foreach { policy => test(s"future timestamp: partition by month when timeParserPolicy is: $policy") { withSQLConf("spark.sql.legacy.timeParserPolicy" -> policy) { withTable(table) { createIcebergTable(table, Seq("months(ts)")) withTempDir { dir => val partitionSchemaDDL = "ts_month string" testConvertToDelta(table, tablePath, partitionSchemaDDL, dir.getCanonicalPath) checkFiltering( table, dir.getCanonicalPath, partitionSchemaDDL, policy, Seq("", "ts > cast('2021-06-01 00:00:00' as timestamp)", "ts < cast('12000-06-01 00:00:00' as timestamp)", "ts >= cast('13000-06-01 00:00:00' as timestamp)", "ts <= cast('2009-06-01 00:00:00' as timestamp)", "ts = cast('11267-07-15 00:00:00' as timestamp)" ) ) } } } } test(s"future timestamp: partition by hour when timeParserPolicy is: $policy") { withSQLConf("spark.sql.legacy.timeParserPolicy" -> policy) { withTable(table) { createIcebergTable(table, Seq("hours(ts)")) withTempDir { dir => val partitionSchemaDDL = "ts_hour string" testConvertToDelta(table, tablePath, partitionSchemaDDL, dir.getCanonicalPath) checkFiltering( table, dir.getCanonicalPath, partitionSchemaDDL, policy, Seq("", "ts > cast('2021-06-01 18:00:00' as timestamp)", "ts < cast('12000-06-01 18:00:00' as timestamp)", "ts >= cast('13000-06-01 19:00:00' as timestamp)", "ts <= cast('2009-06-01 16:00:00' as timestamp)", "ts = cast('11267-07-15 18:30:00' as timestamp)" ) ) } } } } test(s"future timestamp: partition by year when timeParserPolicy is: $policy") { withSQLConf("spark.sql.legacy.timeParserPolicy" -> policy) { withTable(table) { createIcebergTable(table, Seq("years(ts)")) withTempDir { dir => val partitionSchemaDDL = "ts_year int" testConvertToDelta(table, tablePath, partitionSchemaDDL, dir.getCanonicalPath) checkFiltering( table, dir.getCanonicalPath, partitionSchemaDDL, policy, Seq("", "ts > cast('2021-06-01 18:00:00' as timestamp)", "ts < cast('12000-06-01 18:00:00' as timestamp)", "ts >= cast('13000-06-01 19:00:00' as timestamp)", "ts <= cast('2009-06-01 16:00:00' as timestamp)", "ts = cast('11267-07-15 18:30:00' as timestamp)" ) ) } } } } test(s"future timestamp: partition by day when timeParserPolicy is: $policy") { withSQLConf("spark.sql.legacy.timeParserPolicy" -> policy) { withTable(table) { createIcebergTable(table, Seq("days(ts)")) withTempDir { dir => val partitionSchemaDDL = "ts_day date" testConvertToDelta(table, tablePath, partitionSchemaDDL, dir.getCanonicalPath) checkFiltering( table, dir.getCanonicalPath, partitionSchemaDDL, policy, Seq("", "ts > cast('2021-06-01 18:00:00' as timestamp)", "ts < cast('12000-06-01 18:00:00' as timestamp)", "ts >= cast('13000-06-01 19:00:00' as timestamp)", "ts <= cast('2009-06-01 16:00:00' as timestamp)", "ts = cast('11267-07-15 18:30:00' as timestamp)" ) ) } } } } } } ================================================ FILE: iceberg/src/test/scala/org/apache/spark/sql/delta/ConvertIcebergToDeltaSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.io.{ByteArrayOutputStream, File} import java.text.SimpleDateFormat import java.util.TimeZone import scala.collection.JavaConverters._ import scala.collection.mutable import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.delta.commands.ConvertToDeltaCommand import org.apache.spark.sql.delta.commands.convert.{ConvertUtils, IcebergTable} import org.apache.spark.sql.delta.schema.SchemaMergingUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.StatsUtils import io.delta.sql.DeltaSparkSessionExtension import org.apache.hadoop.fs.Path import org.apache.avro.file.{DataFileReader, DataFileWriter, SeekableByteArrayInput} import org.apache.avro.generic.{GenericDatumReader, GenericDatumWriter, GenericRecord} import org.apache.iceberg.{Table, TableProperties} import org.apache.iceberg.hadoop.HadoopTables import org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions import org.apache.spark.SparkConf import org.apache.spark.sql.{AnalysisException, QueryTest, Row, SparkSession, SparkSessionExtensions} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.{col, expr, from_json} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.{SharedSparkSession, TestSparkSession} import org.apache.spark.sql.types._ import org.apache.spark.util.Utils // scalastyle:on import.ordering.noEmptyLine class IcebergCompatibleDeltaTestSparkSession(sparkConf: SparkConf) extends TestSparkSession(sparkConf) { override val extensions: SparkSessionExtensions = { val extensions = new SparkSessionExtensions new DeltaSparkSessionExtension().apply(extensions) new IcebergSparkSessionExtensions().apply(extensions) extensions } } trait ConvertIcebergToDeltaUtils extends SharedSparkSession { protected var warehousePath: File = null protected lazy val table: String = "local.db.table" protected lazy val tablePath: String = "file://" + warehousePath.getCanonicalPath + "/db/table" protected lazy val nestedTable: String = "local.db.nested_table" protected lazy val nestedTablePath: String = "file://" + warehousePath.getCanonicalPath + "/db/nested_table" protected def collectStatisticsStringOption(collectStats: Boolean): String = Option(collectStats) .filterNot(identity).map(_ => "NO STATISTICS").getOrElse("") override def beforeAll(): Unit = { warehousePath = Utils.createTempDir() super.beforeAll() } override def afterAll(): Unit = { super.afterAll() if (warehousePath != null) Utils.deleteRecursively(warehousePath) } override def afterEach(): Unit = { sql(s"DROP TABLE IF EXISTS $table") super.afterEach() } /** * Setting the java default timezone, as we use java.util.TimeZone.getDefault for partition * values... * * In production clusters, the default timezone is always set as UTC. */ def withDefaultTimeZone(timeZoneId: String)(func: => Unit): Unit = { val previousTimeZone = TimeZone.getDefault() try { TimeZone.setDefault(TimeZone.getTimeZone(timeZoneId)) func } finally { TimeZone.setDefault(previousTimeZone) } } override protected def createSparkSession: TestSparkSession = { // Clean up any existing session (Spark 4.0 API) SparkSession.getActiveSession.foreach(_.stop()) SparkSession.clearActiveSession() SparkSession.clearDefaultSession() val session = new IcebergCompatibleDeltaTestSparkSession(sparkConf) session.conf.set(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[DeltaCatalog].getName) session } protected override def sparkConf = super.sparkConf .set( "spark.sql.catalog.local", "org.apache.iceberg.spark.SparkCatalog") .set( "spark.sql.catalog.local.type", "hadoop") .set( "spark.sql.catalog.local.warehouse", warehousePath.getCanonicalPath) .set("spark.sql.session.timeZone", "UTC") protected val schemaDDL = "id bigint, data string, ts timestamp, dt date" protected lazy val schema = StructType.fromDDL(schemaDDL) protected def readIcebergHadoopTable(tablePath: String): Table = { // scalastyle:off deltahadoopconfiguration new HadoopTables(spark.sessionState.newHadoopConf).load(tablePath) // scalastyle:on deltahadoopconfiguration } } trait ConvertIcebergToDeltaSuiteBase extends QueryTest with ConvertIcebergToDeltaUtils with StatsUtils { import testImplicits._ protected def convert(tableIdentifier: String, partitioning: Option[String] = None, collectStats: Boolean = true): Unit test("convert with statistics") { withTable(table) { spark.sql( s"""CREATE TABLE $table (id bigint, data string) |USING iceberg PARTITIONED BY (data)""".stripMargin) spark.sql(s"INSERT INTO $table VALUES (1, 'a'), (2, 'b')") spark.sql(s"INSERT INTO $table VALUES (3, 'c')") convert(s"iceberg.`$tablePath`", collectStats = true) // Check statistics val deltaLog = DeltaLog.forTable(spark, new Path(tablePath)) val statsDf = deltaLog.unsafeVolatileSnapshot.allFiles .select( from_json(col("stats"), deltaLog.unsafeVolatileSnapshot.statsSchema).as("stats")) .select("stats.*") assert(statsDf.filter(col("numRecords").isNull).count == 0) val history = io.delta.tables.DeltaTable.forPath(tablePath).history() assert(history.count == 1) } } test("table with deleted files") { withTable(table) { spark.sql( s"""CREATE TABLE $table (id bigint, data string) |USING iceberg PARTITIONED BY (data)""".stripMargin) spark.sql(s"INSERT INTO $table VALUES (1, 'a'), (2, 'b'), (3, 'c')") spark.sql(s"DELETE FROM $table WHERE data > 'a'") checkAnswer( spark.sql(s"SELECT * from $table"), Row(1, "a") :: Nil) convert(s"iceberg.`$tablePath`") assert(SchemaMergingUtils.equalsIgnoreCaseAndCompatibleNullability( spark.read.format("delta").load(tablePath).schema, new StructType().add("id", LongType).add("data", StringType))) checkAnswer( spark.read.format("delta").load(tablePath), Row(1, "a") :: Nil) } } test("missing iceberg library should throw a sensical error") { val validIcebergSparkTableClassPath = ConvertUtils.icebergSparkTableClassPath Seq( () => { ConvertUtils.icebergSparkTableClassPath = validIcebergSparkTableClassPath + "2" }).foreach { makeInvalid => try { makeInvalid() withTable(table) { spark.sql( s"""CREATE TABLE $table (`1 id` bigint, 2data string) |USING iceberg PARTITIONED BY (2data)""".stripMargin) spark.sql(s"INSERT INTO $table VALUES (1, 'a'), (2, 'b'), (3, 'c')") val e = intercept[DeltaIllegalStateException] { convert(s"iceberg.`$tablePath`") } assert(e.getErrorClass == "DELTA_MISSING_ICEBERG_CLASS") } } finally { ConvertUtils.icebergSparkTableClassPath = validIcebergSparkTableClassPath } } } test("non-parquet table") { withTable(table) { spark.sql( s"""CREATE TABLE $table (id bigint, data string) |USING iceberg PARTITIONED BY (data) |TBLPROPERTIES ('write.format.default'='orc') |""".stripMargin) spark.sql(s"INSERT INTO $table VALUES (1, 'a'), (2, 'b'), (3, 'c')") val e = intercept[UnsupportedOperationException] { convert(s"iceberg.`$tablePath`") } assert(e.getMessage.contains("Cannot convert") && e.getMessage.contains("orc")) } } test("external location") { withTempDir { dir => withTable(table) { spark.sql( s"""CREATE TABLE $table (id bigint, data string) |USING iceberg PARTITIONED BY (data)""".stripMargin) spark.sql(s"INSERT INTO $table VALUES (1, 'a'), (2, 'b')") spark.sql(s"INSERT INTO $table VALUES (3, 'c')") ConvertToDeltaCommand( TableIdentifier(tablePath, Some("iceberg")), None, collectStats = true, Some(dir.getCanonicalPath)).run(spark) checkAnswer( spark.read.format("delta").load(dir.getCanonicalPath), Row(1, "a") :: Row(2, "b") :: Row(3, "c") :: Nil) } } } test("table with renamed columns") { withTable(table) { spark.sql( s"""CREATE TABLE $table (id bigint, data string) |USING iceberg PARTITIONED BY (data)""".stripMargin) spark.sql(s"INSERT INTO $table VALUES (1, 'a'), (2, 'b')") spark.sql("ALTER TABLE local.db.table RENAME COLUMN id TO id2") spark.sql(s"INSERT INTO $table VALUES (3, 'c')") convert(s"iceberg.`$tablePath`") // The converted delta table will get the updated schema assert( SchemaMergingUtils.equalsIgnoreCaseAndCompatibleNullability( spark.read.format("delta").load(tablePath).schema, new StructType().add("id2", LongType).add("data", StringType))) // Parquet files still have the old schema assert( SchemaMergingUtils.equalsIgnoreCaseAndCompatibleNullability( spark.read.format("parquet").load(tablePath + "/data").schema, new StructType().add("id", LongType).add("data", StringType))) val properties = readIcebergHadoopTable(tablePath).properties() // This confirms that name mapping is not used for this case assert(properties.get(TableProperties.DEFAULT_NAME_MAPPING) == null) // As of right now, the data added before rename will be nulls. checkAnswer( spark.read.format("delta").load(tablePath), Row(1, "a") :: Row(2, "b") :: Row(3, "c") :: Nil) } } test("columns starting with numbers") { val table2 = "local.db.table2" val tablePath2 = tablePath + "2" withTable(table2) { spark.sql( s"""CREATE TABLE $table2 (1id bigint, 2data string) |USING iceberg PARTITIONED BY (2data)""".stripMargin) spark.sql(s"INSERT INTO $table2 VALUES (1, 'a'), (2, 'b')") spark.sql(s"INSERT INTO $table2 VALUES (3, 'c')") assert(spark.sql(s"select * from $table2").schema == new StructType().add("1id", LongType).add("2data", StringType)) checkAnswer( spark.sql(s"select * from $table2"), Row(1, "a") :: Row(2, "b") :: Row(3, "c") :: Nil) val properties = readIcebergHadoopTable(tablePath2).properties() // This confirms that name mapping is not used for this case assert(properties.get(TableProperties.DEFAULT_NAME_MAPPING) == null) convert(s"iceberg.`$tablePath2`") // The converted delta table gets the updated schema assert( SchemaMergingUtils.equalsIgnoreCaseAndCompatibleNullability( spark.read.format("delta").load(tablePath2).schema, new StructType().add("1id", LongType).add("2data", StringType))) // parquet file schema has been modified assert( spark.read.format("parquet").load(tablePath2 + "/data").schema == new StructType() .add("_1id", LongType) .add("_2data", StringType) // this is the partition column, which stays as-is .add("2data", StringType)) checkAnswer( spark.read.format("delta").load(tablePath2), Row(1, "a") :: Row(2, "b") :: Row(3, "c") :: Nil) } } test("nested schema") { withTable(table) { def createDDL(tname: String): String = s"""CREATE TABLE $tname (id bigint, person struct) |USING iceberg PARTITIONED BY (truncate(person.name, 2))""".stripMargin def insertDDL(tname: String): String = s"INSERT INTO $tname VALUES (1, ('aaaaa', 10)), (2, ('bbbbb', 20))" testNestedColumnIDs(createDDL(nestedTable), insertDDL(nestedTable)) spark.sql(createDDL(table)) spark.sql(s"INSERT INTO $table VALUES (1, ('aaaaa', 10)), (2, ('bbbbb', 20))") checkAnswer( spark.sql(s"SELECT * from $table"), Row(1, Row("aaaaa", 10)) :: Row(2, Row("bbbbb", 20)) :: Nil) convert(s"iceberg.`$tablePath`") val tblSchema = spark.read.format("delta").load(tablePath).schema val expectedSchema = new StructType() .add("id", LongType) .add("person", new StructType().add("name", StringType).add("phone", IntegerType)) .add("person.name_trunc", StringType) assert(SchemaMergingUtils.equalsIgnoreCaseAndCompatibleNullability(expectedSchema, tblSchema)) checkAnswer( spark.read.format("delta").load(tablePath), Row(1, Row("aaaaa", 10), "aa") :: Row(2, Row("bbbbb", 20), "bb") :: Nil) assert( spark.sql(s"select * from delta.`$tablePath` where person.name > 'b'") .inputFiles.length == 1) spark.sql( s""" |insert into $table (id, person) |values (3, struct("ccccc", 30)) |""".stripMargin) val insertDataSchema = StructType.fromDDL("id bigint, person struct") val df = spark.createDataFrame(Seq(Row(3L, Row("ccccc", 30))).asJava, insertDataSchema) df.write.format("delta").mode("append").save(tablePath) checkAnswer( // check the raw parquet partition directories written out by Iceberg spark.sql(s"select * from parquet.`$tablePath/data`"), spark.sql(s"select * from delta.`$tablePath`") ) assert( spark.sql(s"select * from delta.`$tablePath` where person.name > 'b'") .inputFiles.length == 2) } } private def schemaTestNoDataSkipping( createTableSql: String, initialInsertValuesSql: String, expectedInitialRows: Seq[Row], expectedSchema: StructType, finalInsertValuesSql: String) : Unit = { withTable(table) { spark.sql(s"DROP TABLE IF EXISTS $table") spark.sql(s"CREATE TABLE $table $createTableSql USING iceberg") spark.sql(s"INSERT INTO $table VALUES $initialInsertValuesSql") checkAnswer(spark.sql(s"SELECT * FROM $table"), expectedInitialRows) convert(s"iceberg.`$tablePath`") val tblSchema = spark.read.format("delta").load(tablePath).schema assert(SchemaMergingUtils.equalsIgnoreCaseAndCompatibleNullability(expectedSchema, tblSchema)) checkAnswer(spark.read.format("delta").load(tablePath), expectedInitialRows) spark.sql( s""" |INSERT INTO $table |VALUES $finalInsertValuesSql |""".stripMargin) spark.sql( s""" |INSERT INTO delta.`$tablePath` |VALUES $finalInsertValuesSql |""".stripMargin) checkAnswer( // check the raw parquet partition directories written out by Iceberg spark.sql(s"SELECT * FROM parquet.`$tablePath/data`"), spark.sql(s"SELECT * FROM delta.`$tablePath`") ) } } test("array of struct schema") { val createTableSql = "(id bigint, grades array>)" val initialInsertValuesSql = "(1, array(('mat', 10), ('cs', 90))), (2, array(('eng', 80)))" val expectedInitialRows = Row(1, Seq(Row("mat", 10), Row("cs", 90))) :: Row(2, Seq(Row("eng", 80))) :: Nil val arrayType = ArrayType(new StructType().add("class", StringType).add("score", IntegerType)) val expectedSchema = new StructType() .add("id", LongType) .add("grades", arrayType) val finalInsertValuesSql = "(3, array(struct(\"mat\", 100), struct(\"cs\", 100)))" schemaTestNoDataSkipping(createTableSql, initialInsertValuesSql, expectedInitialRows, expectedSchema, finalInsertValuesSql) } test("map schema") { val createTableSql = "(id bigint, grades map)" val initialInsertValuesSql = "(1, map('mat', 10, 'cs', 90)), (2, map('eng', 80))" val expectedInitialRows = Row(1, Map[String, Int]("mat" -> 10, "cs" -> 90)) :: Row(2, Map[String, Int]("eng" -> 80)) :: Nil val expectedSchema = new StructType() .add("id", LongType) .add("grades", MapType(StringType, IntegerType)) val finalInsertValuesSql = "(3, map(\"mat\", 100, \"cs\", 100))" schemaTestNoDataSkipping(createTableSql, initialInsertValuesSql, expectedInitialRows, expectedSchema, finalInsertValuesSql) } test("partition schema is not allowed") { withTable(table) { spark.sql( s"""CREATE TABLE $table (id bigint, data string) |USING iceberg PARTITIONED BY (data) |""".stripMargin) spark.sql(s"INSERT INTO $table VALUES (1, 'a'), (2, 'b'), (3, 'c')") val e = intercept[IllegalArgumentException] { convert(s"iceberg.`$tablePath`", Some("data string")) } assert(e.getMessage.contains("Partition schema cannot be specified")) } } test("copy over Iceberg table properties") { withTable(table) { spark.sql( s"""CREATE TABLE $table (id bigint, data string) |USING iceberg PARTITIONED BY (data)""".stripMargin) spark.sql(s"INSERT INTO $table VALUES (1, 'a'), (2, 'b'), (3, 'c')") spark.sql( s"""ALTER TABLE $table SET TBLPROPERTIES( | 'read.split.target-size'='268435456' |)""".stripMargin) convert(s"iceberg.`$tablePath`") checkAnswer( spark.sql(s"SHOW TBLPROPERTIES delta.`$tablePath`") .filter(col("key").startsWith("read.")), Row("read.split.target-size", "268435456") :: Nil ) } } test("converted table columns have metadata containing iceberg column ids") { val nested1 = s"""CREATE TABLE $nestedTable (name string, age int, |pokemon array>) |USING iceberg""".stripMargin val insert1 = s"""INSERT INTO $nestedTable VALUES ('Ash', 10, |array(struct('Charizard', 'Fire/Flying'), struct('Pikachu', 'Electric'))) """.stripMargin testNestedColumnIDs(nested1, insert1) val nested2 = s"""CREATE TABLE $nestedTable (name string, |info struct, id:int>) |USING iceberg""".stripMargin val insert2 = s"""INSERT INTO $nestedTable VALUES ('Zigzagoon', |struct(struct('Hoenn', 'Common'), 263)) """.stripMargin testNestedColumnIDs(nested2, insert2) val nested3 = s"""CREATE TABLE $nestedTable (name string, |moves map>) |USING iceberg""".stripMargin val insert3 = s"""INSERT INTO $nestedTable VALUES ('Heatran', |map('Fire Fang', struct(17, 7))) """.stripMargin testNestedColumnIDs(nested3, insert3) } test("comments are retained from Iceberg") { withTable(table) { spark.sql( s"""CREATE TABLE $table (id bigint comment "myexample", data string comment "myexample") |USING iceberg PARTITIONED BY (data)""".stripMargin) spark.sql(s"INSERT INTO $table VALUES (1, 'a'), (2, 'b'), (3, 'c')") convert(s"iceberg.`$tablePath`") val readSchema = spark.read.format("delta").load(tablePath).schema readSchema.foreach { field => assert(field.getComment().contains("myexample")) } } } private def testNestedColumnIDs(createString: String, insertString: String): Unit = { // Nested schema withTable(nestedTable) { // Create table and insert into it spark.sql(createString) spark.sql(insertString) // Convert to Delta convert(s"iceberg.`$nestedTablePath`") // Check Delta schema val schema = DeltaLog.forTable(spark, new Path(nestedTablePath)).update().schema // Get initial Iceberg schema val icebergTable = readIcebergHadoopTable(nestedTablePath) val icebergSchema = icebergTable.schema() // Check all nested fields to see if they all have a column ID then check the iceberg schema // for whether that column ID corresponds to the same column name val columnIds = mutable.Set[Long]() SchemaMergingUtils.transformColumns(schema) { (_, field, _) => assert(DeltaColumnMapping.hasColumnId(field)) // nest column ids should be distinct val id = DeltaColumnMapping.getColumnId(field) assert(!columnIds.contains(id)) columnIds.add(id) // the id can either be a data schema id or a identity transform partition field // or it is generated bc it's a non-identity transform partition field assert( Option(icebergSchema.findField(id)).map(_.name()).contains(field.name) || icebergTable.spec().fields().asScala.map(_.name()).contains(field.name) ) field } } } test("conversion should fail if had partition evolution / multiple partition specs") { /** * Per https://iceberg.apache.org/evolution/#partition-evolution, if partition evolution happens * in Iceberg, multiple partition specs are persisted, thus convert to Delta cannot be * supported w/o repartitioning because Delta only supports one consistent spec */ withTable(table) { spark.sql( s"""CREATE TABLE $table (id bigint, data string, data2 string) |USING iceberg PARTITIONED BY (data)""".stripMargin) spark.sql(s"INSERT INTO $table VALUES (1, 'a', 'x'), (2, 'b', 'y'), (3, 'c', 'z')") // add new partition spec readIcebergHadoopTable(tablePath).updateSpec().addField("data2").commit() spark.sql(s"INSERT INTO $table VALUES (1, 'a', 'x'), (2, 'b', 'y'), (3, 'c', 'z')") // partition evolution happens, convert will fail val e1 = intercept[DeltaAnalysisException] { convert(s"iceberg.`$tablePath`") } assert(e1.getMessage.contains(IcebergTable.ERR_MULTIPLE_PARTITION_SPECS)) // drop old partition spec readIcebergHadoopTable(tablePath).updateSpec().removeField("data2").commit() spark.sql(s"INSERT INTO $table VALUES (1, 'a', 'x'), (2, 'b', 'y'), (3, 'c', 'z')") // partition spec is reverted, but partition evolution happens already // use assert explicitly bc we do not want checks in IcebergPartitionUtils to run first assert(readIcebergHadoopTable(tablePath).specs().size() > 1) } } /** * Strips the "schema" metadata key from all manifest Avro files of the table's current snapshot. * This simulates a V2 writer that omits the optional schema field. Without passing specsById * to ManifestFiles.read, reading such manifests causes an NPE in SchemaParser.fromJson. */ private def stripSchemaFromManifests(table: Table): Unit = { val manifests = table.currentSnapshot().dataManifests(table.io()).asScala // scalastyle:off deltahadoopconfiguration val conf = spark.sessionState.newHadoopConf() // scalastyle:on deltahadoopconfiguration manifests.foreach { manifest => val path = new Path(manifest.path()) val fs = path.getFileSystem(conf) // Read the entire manifest file into a byte array val inputStream = fs.open(path) val bytes = try { org.apache.commons.io.IOUtils.toByteArray(inputStream) } finally { inputStream.close() } val datumReader = new GenericDatumReader[GenericRecord]() val reader = new DataFileReader[GenericRecord]( new SeekableByteArrayInput(bytes), datumReader) // Collect records and metadata val records = new java.util.ArrayList[GenericRecord]() while (reader.hasNext) { records.add(reader.next()) } val avroSchema = reader.getSchema val metaKeys = reader.getMetaKeys.asScala.toSeq // Write back without the "schema" metadata key val out = new ByteArrayOutputStream() val datumWriter = new GenericDatumWriter[GenericRecord](avroSchema) val writer = new DataFileWriter[GenericRecord](datumWriter) val reservedKeys = Set("schema", "avro.schema", "avro.codec") metaKeys.filterNot(reservedKeys.contains).foreach { key => writer.setMeta(key, reader.getMeta(key)) } writer.create(avroSchema, out) records.asScala.foreach(writer.append) writer.close() reader.close() // Overwrite the original file val outputStream = fs.create(path, true) try { outputStream.write(out.toByteArray) } finally { outputStream.close() } } } test("convert Iceberg table with manifest missing schema metadata") { withTable(table) { spark.sql( s"""CREATE TABLE $table (id bigint, data string) |USING iceberg PARTITIONED BY (data) |TBLPROPERTIES ("format-version" = "2")""".stripMargin) Seq((1L, "a"), (2L, "b")).toDF("id", "data") .write.format("iceberg").mode("append").saveAsTable(table) // Strip the "schema" metadata from manifest Avro files to simulate a V2 writer // that omits the optional schema field. Without passing specsById to // ManifestFiles.read, this causes an NPE in SchemaParser.fromJson. val iceTable = readIcebergHadoopTable(tablePath) stripSchemaFromManifests(iceTable) convert(s"iceberg.`$tablePath`") checkAnswer( spark.read.format("delta").load(tablePath), Row(1, "a") :: Row(2, "b") :: Nil) } } test("convert Iceberg table with not null columns") { withTable(table) { spark.sql( s"""CREATE TABLE $table (id bigint NOT NULL, data string, name string NOT NULL) |USING iceberg PARTITIONED BY (id)""".stripMargin) spark.sql(s"INSERT INTO $table VALUES (1, 'a', 'b'), (2, 'b', 'c'), (3, 'c', 'd')") convert(s"iceberg.`$tablePath`") val data = spark.read.format("delta").load(tablePath) // verify data is converted properly checkAnswer(data, Seq(Row(1, "a", "b"), Row(2, "b", "c"), Row(3, "c", "d"))) // Verify schema contains not null constraint where appropriate val dataSchema = data.schema dataSchema.foreach { field => // both partition columns and data columns should have the correct nullability if (field.name == "id" || field.name == "name") { assert(!field.nullable) } else { assert(field.nullable) } } // Should not be able to write nulls to not null data column var ex = intercept[Exception] { spark.sql(s"INSERT INTO $table VALUES (4, 'd', null)") } // Spark 4.0+ uses uppercase NULL, 3.4+ uses capitalized Null, <3.4 has column name assert(ex.getMessage.contains("NULL value appeared in non-nullable field") || ex.getMessage.contains("Null value appeared in non-nullable field") || ex.getMessage.contains("""Cannot write nullable values to non-null column 'name'""")) // Should not be able to write nulls to not null partition column ex = intercept[Exception] { spark.sql(s"INSERT INTO $table VALUES (null, 'e', 'e')") } // Spark 4.0+ uses uppercase NULL, 3.4+ uses capitalized Null, <3.4 has column name assert(ex.getMessage.contains("NULL value appeared in non-nullable field") || ex.getMessage.contains("Null value appeared in non-nullable field") || ex.getMessage.contains("""Cannot write nullable values to non-null column 'id'""")) // Should be able to write nulls to nullable column spark.sql(s"INSERT INTO $table VALUES (5, null, 'e')") } } test("convert Iceberg table with case sensitive columns") { withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { withTable(table) { spark.sql( s"""CREATE TABLE $table (i bigint NOT NULL, I string) |USING iceberg PARTITIONED BY (I)""".stripMargin) spark.sql(s"INSERT INTO $table VALUES (1, 'a'), (2, 'b'), (3, 'c')") val ex = intercept[UnsupportedOperationException] { convert(s"iceberg.`$tablePath`") } assert(ex.getMessage.contains("contains column names that only differ by case")) } } } test("should block converting Iceberg table with name mapping") { withTable(table) { spark.sql( s"""CREATE TABLE $table (id bigint, data string) |USING iceberg PARTITIONED BY (data) |""".stripMargin ) spark.sql( s"""ALTER TABLE $table SET TBLPROPERTIES( | 'schema.name-mapping.default' = | '[{"field-id": 1, "names": ["my_id"]},{"field-id": 2, "names": ["my_data"]}]' |)""".stripMargin) val e = intercept[UnsupportedOperationException] { convert(s"iceberg.`$tablePath`") } assert(e.getMessage.contains(IcebergTable.ERR_CUSTOM_NAME_MAPPING)) } } private def testNullPartitionValues(): Unit = { withTable(table) { spark.sql( s"""CREATE TABLE $table (id bigint, data string, dt date) |USING iceberg PARTITIONED BY (dt)""".stripMargin) spark.sql(s"INSERT INTO $table" + s" VALUES (1, 'a', null), (2, 'b', null), (3, 'c', cast('2021-01-03' as date))") convert(s"iceberg.`$tablePath`") val data = spark.read.format("delta").load(tablePath) val fmt = new SimpleDateFormat("yyyy-MM-dd") checkAnswer(data, Seq( Row(1, "a", null), Row(2, "b", null), Row(3, "c", new java.sql.Date(fmt.parse("2021-01-03").getTime)))) } } test("partition columns are null") { withSQLConf(DeltaSQLConf.DELTA_CONVERT_ICEBERG_USE_NATIVE_PARTITION_VALUES.key -> "false") { val e = intercept[RuntimeException] { testNullPartitionValues() } assert(e.getMessage.contains("Failed to cast partition value")) } withSQLConf( DeltaSQLConf.DELTA_CONVERT_PARTITION_VALUES_IGNORE_CAST_FAILURE.key -> "true", DeltaSQLConf.DELTA_CONVERT_ICEBERG_USE_NATIVE_PARTITION_VALUES.key -> "false") { testNullPartitionValues() } // default setting should work testNullPartitionValues() } test("arbitrary name") { def col(name: String): String = name + "with_special_chars_;{}()\n\t=" // turns out Iceberg would fail when partition col names have special chars def partCol(name: String): String = "0123" + name withTable(table) { spark.sql( s"""CREATE TABLE $table ( | `${col("data")}` int, | `${partCol("part1")}` bigint, | `${partCol("part2")}` string) |USING iceberg |PARTITIONED BY ( | `${partCol("part1")}`, | truncate(`${partCol("part2")}`, 4)) |""".stripMargin) spark.sql( s""" |INSERT INTO $table |VALUES (123, 1234567890123, 'str11') |""".stripMargin) convert(s"iceberg.`$tablePath`") spark.sql( s""" |INSERT INTO delta.`$tablePath` |VALUES (456, 4567890123456, 'str22', 'str2') |""".stripMargin) checkAnswer(spark.sql(s"select * from delta.`$tablePath`"), Seq( Row(123, 1234567890123L, "str11", "str1"), Row(456, 4567890123456L, "str22", "str2"))) // projection and filter checkAnswer( spark.table(s"delta.`$tablePath`") .select(s"`${col("data")}`", s"`${partCol("part1")}`") .where(s"`${partCol("part2")}` = 'str22'"), Seq(Row(456, 4567890123456L))) } } test("partition by identity, using native partition values") { withDefaultTimeZone("UTC") { withTable(table) { spark.sql( s"""CREATE TABLE $table ( | data_binary binary, | part_ts timestamp, | part_date date, | part_bool boolean, | part_int integer, | part_long long, | part_float float, | part_double double, | part_decimal decimal(3, 2), | part_string string | ) |USING iceberg PARTITIONED BY (part_ts, part_date, part_bool, part_int, part_long, | part_float, part_double, part_decimal, part_string)""".stripMargin) def insertData(targetTable: String): Unit = { spark.sql( s""" |INSERT INTO $targetTable |VALUES (cast('this is binary' as binary), | cast(1635728400000 as timestamp), | cast('2021-11-15' as date), | true, | 123, | 12345678901234, | 123.4, | 123.4, | 1.23, | 'this is a string')""".stripMargin) } insertData(table) withTempDir { dir => val deltaPath = dir.getCanonicalPath ConvertToDeltaCommand( tableIdentifier = TableIdentifier(tablePath, Some("iceberg")), partitionSchema = None, collectStats = true, Some(deltaPath)).run(spark) // check that all the partition value types can be converted correctly checkAnswer(spark.table(s"delta.`$deltaPath`"), spark.table(table)) insertData(s"delta.`$deltaPath`") insertData(table) // check that new writes to both Delta and Iceberg can be read back the same checkAnswer(spark.table(s"delta.`$deltaPath`"), spark.table(table)) } } } } test("mor table without deletion files") { withTable(table) { spark.sql( s"""CREATE TABLE $table (id bigint, data string) |USING iceberg |TBLPROPERTIES ( | "format-version" = "2", | "write.delete.mode" = "merge-on-read" |) |""".stripMargin) spark.sql(s"INSERT INTO $table VALUES (1, 'a')") spark.sql(s"INSERT INTO $table VALUES (2, 'b')") spark.sql(s"DELETE FROM $table WHERE id = 1") // The two rows above should've been in separate files, and DELETE will remove all rows from // one file completely, in this case, we could still convert the table as Spark scan will // ignore the completely deleted file. convert(s"iceberg.`$tablePath`") checkAnswer( spark.read.format("delta").load(tablePath), Row(2, "b") :: Nil ) } } test("block convert: mor table with deletion files") { def setupBulkMorTable(): Unit = { spark.sql( s"""CREATE TABLE $table (id bigint, data string) |USING iceberg |TBLPROPERTIES ( | "format-version" = "2", | "write.delete.mode" = "merge-on-read", | "write.update.mode" = "merge-on-read", | "write.merge.mode" = "merge-on-read" |) |""".stripMargin) // Now we need to write a considerable amount of data in a dataframe fashion so Iceberg can // combine multiple records in one Parquet file. (0 until 100).map(i => (i.toLong, s"name_$i")).toDF("id", "data") .write.format("iceberg").mode("append").saveAsTable(table) } def assertConversionFailed(): Unit = { // By default, conversion should fail because it is unsafe. val e = intercept[UnsupportedOperationException] { convert(s"iceberg.`$tablePath`") } assert(e.getMessage.contains("convert Iceberg table with row-level deletes")) } // --- DELETE withTable(table) { setupBulkMorTable() // This should touch part of one Parquet file spark.sql(s"DELETE FROM $table WHERE id = 1") // By default, conversion should fail because it is unsafe. assertConversionFailed() } // --- UPDATE withTable(table) { setupBulkMorTable() // This should touch part of one Parquet file spark.sql(s"UPDATE $table SET id = id * 2 WHERE id = 1") // By default, conversion should fail because it is unsafe. assertConversionFailed() } // --- MERGE withTable(table) { setupBulkMorTable() (0 until 100).filter(_ % 2 == 0) .toDF("id") .createOrReplaceTempView("tempdata") // This should touch part of one Parquet file spark.sql( s""" |MERGE INTO $table t |USING tempdata s |ON t.id = s.id |WHEN MATCHED THEN UPDATE SET t.data = "some_other" |""".stripMargin) // By default, conversion should fail because it is unsafe. assertConversionFailed() } } test("block convert: binary type partition columns") { withTable(table) { spark.sql( s"""CREATE TABLE $table ( | data int, | part binary) |USING iceberg |PARTITIONED BY (part) |""".stripMargin) spark.sql(s"insert into $table values (123, cast('str1' as binary))") val e = intercept[UnsupportedOperationException] { convert(s"iceberg.`$tablePath`") } assert(e.getMessage.contains("Unsupported partition transform expression")) } } test("block convert: partition transform truncate decimal type") { withTable(table) { spark.sql( s"""CREATE TABLE $table ( | data int, | part decimal) |USING iceberg |PARTITIONED BY (truncate(part, 3)) |""".stripMargin) spark.sql(s"insert into $table values (123, 123456)") val e = intercept[UnsupportedOperationException] { convert(s"iceberg.`$tablePath`") } assert(e.getMessage.contains("Unsupported partition transform expression")) } } } class ConvertIcebergToDeltaScalaSuite extends ConvertIcebergToDeltaSuiteBase { override protected def convert( tableIdentifier: String, partitioning: Option[String] = None, collectStats: Boolean = true): Unit = { if (partitioning.isDefined) { io.delta.tables.DeltaTable.convertToDelta(spark, tableIdentifier, partitioning.get) } else { io.delta.tables.DeltaTable.convertToDelta(spark, tableIdentifier) } } } class ConvertIcebergToDeltaSQLSuite extends ConvertIcebergToDeltaSuiteBase { override protected def convert( tableIdentifier: String, partitioning: Option[String] = None, collectStats: Boolean = true): Unit = { val statement = partitioning.map(p => s" PARTITIONED BY ($p)").getOrElse("") spark.sql(s"CONVERT TO DELTA ${tableIdentifier}${statement} " + s"${collectStatisticsStringOption(collectStats)}") } // TODO: Move to base once DeltaAPI support collectStats parameter test("convert without statistics") { withTempDir { dir => withTable(table) { spark.sql( s"""CREATE TABLE $table (id bigint, data string) |USING iceberg PARTITIONED BY (data)""".stripMargin) spark.sql(s"INSERT INTO $table VALUES (1, 'a'), (2, 'b')") spark.sql(s"INSERT INTO $table VALUES (3, 'c')") ConvertToDeltaCommand( TableIdentifier(tablePath, Some("iceberg")), None, collectStats = false, Some(dir.getCanonicalPath)).run(spark) // Check statistics val deltaLog = DeltaLog.forTable(spark, new Path(dir.getPath)) val statsDf = deltaLog.unsafeVolatileSnapshot.allFiles .select(from_json(col("stats"), deltaLog.unsafeVolatileSnapshot.statsSchema).as("stats")) .select("stats.*") assert(statsDf.filter(col("numRecords").isNotNull).count == 0) val history = io.delta.tables.DeltaTable.forPath(dir.getPath).history() assert(history.count == 1) } } } } ================================================ FILE: iceberg/src/test/scala/org/apache/spark/sql/delta/ConvertToIcebergSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.{File, IOException} import java.net.ServerSocket import org.scalatest.concurrent.Eventually import org.scalatest.time.SpanSugar._ import org.apache.spark.SparkContext import org.apache.spark.sql.{QueryTest, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType, CatalogStorageFormat} import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.types.{IntegerType, StringType, StructType, StructField} import org.apache.spark.util.Utils /** * This test suite relies on an external Hive metastore (HMS) instance to run. * * A standalone HMS can be created using the following docker command. * ************************************************************ * docker run -d -p 9083:9083 --env SERVICE_NAME=metastore \ * --name metastore-standalone apache/hive:4.0.0-beta-1 * ************************************************************ * The URL of this standalone HMS is thrift://localhost:9083 * * By default this hms will use `/opt/hive/data/warehouse` as warehouse path. * Please make sure this path exists prior to running the suite. */ class ConvertToIcebergSuite extends QueryTest with Eventually { private var _sparkSession: SparkSession = null private var _sparkSessionWithDelta: SparkSession = null private var _sparkSessionWithIceberg: SparkSession = null private val PORT = 9083 private val WAREHOUSE_PATH = "/opt/hive/data/warehouse/" private val testTableName: String = "deltatable" private var testTablePath: String = s"$WAREHOUSE_PATH$testTableName" override def spark: SparkSession = _sparkSession override def beforeAll(): Unit = { super.beforeAll() if (hmsReady(PORT)) { _sparkSessionWithDelta = createSparkSessionWithDelta() _sparkSessionWithIceberg = createSparkSessionWithIceberg() require(!_sparkSessionWithDelta.eq(_sparkSessionWithIceberg), "separate sessions expected") } } override def afterEach(): Unit = { super.afterEach() if (hmsReady(PORT)) { _sparkSessionWithDelta.sql(s"DROP TABLE IF EXISTS $testTableName") } Utils.deleteRecursively(new File(testTablePath)) } override def afterAll(): Unit = { super.afterAll() SparkContext.getActive.foreach(_.stop()) } test("enforceSupportInCatalog") { var testTable = new CatalogTable( TableIdentifier("table"), CatalogTableType.EXTERNAL, CatalogStorageFormat(None, None, None, None, compressed = false, Map.empty), new StructType(Array(StructField("col1", IntegerType), StructField("col2", StringType)))) var testMetadata = Metadata() assert(UniversalFormat.enforceSupportInCatalog(testTable, testMetadata).isEmpty) testTable = testTable.copy(properties = Map("table_type" -> "iceberg")) var resultTable = UniversalFormat.enforceSupportInCatalog(testTable, testMetadata) assert(resultTable.nonEmpty) assert(!resultTable.get.properties.contains("table_type")) testMetadata = testMetadata.copy( configuration = Map("delta.universalFormat.enabledFormats" -> "iceberg")) assert(UniversalFormat.enforceSupportInCatalog(testTable, testMetadata).isEmpty) testTable = testTable.copy(properties = Map.empty) resultTable = UniversalFormat.enforceSupportInCatalog(testTable, testMetadata) assert(resultTable.isEmpty) } test("basic test - managed table created with SQL") { if (hmsReady(PORT)) { runDeltaSql( s"""CREATE TABLE `${testTableName}` (col1 INT) USING DELTA |TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name', | 'delta.enableIcebergCompatV2' = 'true', | 'delta.universalFormat.enabledFormats' = 'iceberg' |)""".stripMargin) runDeltaSql(s"INSERT INTO `$testTableName` VALUES (123)") verifyReadWithIceberg(testTableName, Seq(Row(123))) } } test("basic test - catalog table created with DataFrame") { if (hmsReady(PORT)) { withDeltaSparkSession { deltaSpark => withDefaultTablePropsInSQLConf { deltaSpark.range(10).write.format("delta") .option("path", testTablePath) .saveAsTable(testTableName) } } withDeltaSparkSession { deltaSpark => deltaSpark.range(10, 20, 1) .write.format("delta").mode("append") .option("path", testTablePath) .saveAsTable(testTableName) } verifyReadWithIceberg(testTableName, 0 to 19 map (Row(_))) } } def runDeltaSql(sqlStr: String): Unit = { withDeltaSparkSession { deltaSpark => deltaSpark.sql(sqlStr) } } def verifyReadWithIceberg(tableName: String, expectedAnswer: Seq[Row]): Unit = { withIcebergSparkSession { icebergSparkSession => eventually(timeout(10.seconds)) { icebergSparkSession.sql(s"REFRESH TABLE ${tableName}") val icebergDf = icebergSparkSession.read.format("iceberg").load(tableName) checkAnswer(icebergDf, expectedAnswer) } } } def withDefaultTablePropsInSQLConf(f: => Unit): Unit = { withSQLConf( DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey -> "name", DeltaConfigs.ICEBERG_COMPAT_V1_ENABLED.defaultTablePropertyKey -> "true", DeltaConfigs.UNIVERSAL_FORMAT_ENABLED_FORMATS.defaultTablePropertyKey -> "iceberg" ) { f } } def withDeltaSparkSession[T](f: SparkSession => T): T = { withSparkSession(_sparkSessionWithDelta, f) } def withIcebergSparkSession[T](f: SparkSession => T): T = { withSparkSession(_sparkSessionWithIceberg, f) } def withSparkSession[T](sessionToUse: SparkSession, f: SparkSession => T): T = { try { SparkSession.setDefaultSession(sessionToUse) SparkSession.setActiveSession(sessionToUse) _sparkSession = sessionToUse f(sessionToUse) } finally { SparkSession.clearActiveSession() SparkSession.clearDefaultSession() _sparkSession = null } } protected def createSparkSessionWithDelta(): SparkSession = { SparkSession.clearActiveSession() SparkSession.clearDefaultSession() val sparkSession = SparkSession.builder() .master("local[*]") .appName("DeltaSession") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") .config("hive.metastore.uris", s"thrift://localhost:$PORT") .config("spark.sql.catalogImplementation", "hive") .getOrCreate() SparkSession.clearActiveSession() SparkSession.clearDefaultSession() sparkSession } protected def createSparkSessionWithIceberg(): SparkSession = { SparkSession.clearActiveSession() SparkSession.clearDefaultSession() val sparkSession = SparkSession.builder() .master("local[*]") .appName("IcebergSession") .config("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions") .config("spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("hive.metastore.uris", s"thrift://localhost:$PORT") .config("spark.sql.catalogImplementation", "hive") .getOrCreate() SparkSession.clearActiveSession() SparkSession.clearDefaultSession() sparkSession } def hmsReady(port: Int): Boolean = { var ss: ServerSocket = null try { ss = new ServerSocket(port) ss.setReuseAddress(true) logWarning("No HMS detected, test suite will not run") return false } catch { case e: IOException => } finally { if (ss != null) { try ss.close() catch { case e: IOException => } } } true } } ================================================ FILE: iceberg/src/test/scala/org/apache/spark/sql/delta/NonSparkIcebergTestUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.collection.JavaConverters._ import org.apache.iceberg.{DataFile, DataFiles, Files, PartitionSpec, Schema, Table} import org.apache.iceberg.data.GenericRecord import org.apache.iceberg.data.parquet.GenericParquetWriter import org.apache.iceberg.hadoop.HadoopTables import org.apache.iceberg.io.FileAppender import org.apache.iceberg.parquet.Parquet import org.apache.iceberg.types.Types import org.apache.iceberg.types.Types.NestedField import org.apache.spark.sql.SparkSession object NonSparkIcebergTestUtils { /** * Create an Iceberg table with formats/data types not supported by Spark. * This is primarily used for compatibility tests. It includes the following features * * TIME data type that is not supported by Spark. * @param location Iceberg table root path * @param schema Iceberg table schema * @param rows Data rows we write into the table * @param dataFileIdx index of the parquet file going to be written in the data folder */ def createIcebergTable( spark: SparkSession, location: String, schema: Schema, rows: Seq[Map[String, Any]], dataFileIdx: Int = 1): Table = { // scalastyle:off deltahadoopconfiguration val tables = new HadoopTables(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration val table = tables.create( schema, PartitionSpec.unpartitioned(), location ) writeIntoIcebergTable(table, rows, dataFileIdx) table } /** * Writes into an Iceberg table with formats/data types not supported by Spark. * This is primarily used for compatibility tests. It includes the following features * * TIME data type that is not supported by Spark. * @param table Iceberg table * @param rows Data rows we write into the table * @param dataFileIdx index of the parquet file going to be written in the data folder * @param dataPath Optional path to write the data file */ def writeIntoIcebergTable( table: Table, rows: Seq[Map[String, Any]], dataFileIdx: Int, dataPath: Option[String] = None): Unit = { val schema = table.schema() val records = rows.map { row => val record = GenericRecord.create(schema) row.foreach { case (key, value) => record.setField(key, value) } record } val parquetLocation = dataPath.getOrElse(table.location() + s"/data/$dataFileIdx.parquet") val fileAppender: FileAppender[GenericRecord] = Parquet .write(table.io().newOutputFile(parquetLocation)) .schema(schema) .createWriterFunc(GenericParquetWriter.create _) // Iceberg 1.10.0 API .overwrite() .build(); try { fileAppender.addAll(records.asJava) } finally { fileAppender.close } val dataFile = DataFiles.builder(PartitionSpec.unpartitioned()) .withInputFile(table.io().newInputFile(parquetLocation)) .withMetrics(fileAppender.metrics()) .build(); table .newAppend .appendFile(dataFile) .commit } } ================================================ FILE: iceberg/src/test/scala/org/apache/spark/sql/delta/commands/convert/IcebergPartitionConverterSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.convert import java.lang.{Integer => JInt, Long => JLong} import java.math.BigDecimal import java.util.{List => JList} import scala.collection.JavaConverters._ import shadedForDelta.org.apache.iceberg.{PartitionData, PartitionSpec, Schema} import shadedForDelta.org.apache.iceberg.transforms._ import shadedForDelta.org.apache.iceberg.types.Conversions import shadedForDelta.org.apache.iceberg.types.Types._ import org.apache.spark.SparkFunSuite class IcebergPartitionConverterSuite extends SparkFunSuite { test("convert partition simple case, including empty and null") { val icebergSchema = new Schema(10, Seq[NestedField]( NestedField.required(1, "col_int", IntegerType.get), NestedField.required(2, "col_long", LongType.get), NestedField.required(3, "col_st", StringType.get) ).asJava) val icebergPartSpec = PartitionSpec .builderFor(icebergSchema) .identity("col_int") .truncate("col_st", 3) .identity("col_long") .build val physicalNameToField = Map( "pname1" -> icebergPartSpec.fields().get(0), "pname2" -> icebergPartSpec.fields().get(1), "pname3" -> icebergPartSpec.fields().get(2) ) val partitionConverter = IcebergPartitionConverter(icebergSchema, physicalNameToField) val partData = new PartitionData( StructType.of( NestedField.required(1000, "col_int", IntegerType.get), NestedField.required(1001, "col_st", StringType.get) ) ) partData.put(0, 100) partData.put(1, "alo") assertResult("Map(pname1 -> 100, pname2 -> alo, pname3 -> null)")( partitionConverter.toDelta(partData).toString) val partData2 = new PartitionData( StructType.of( NestedField.required(1000, "col_int", IntegerType.get), NestedField.required(1001, "col_long", LongType.get), NestedField.required(1002, "col_st", StringType.get) ) ) partData2.put(2, 100000000000000L) partData2.put(1, null) assertResult("Map(pname1 -> null, pname2 -> null, pname3 -> 100000000000000)")( partitionConverter.toDelta(partData2).toString) } test("convert partition with complex types") { val icebergSchema = new Schema(10, Seq[NestedField]( NestedField.required(4, "col_date", DateType.get), NestedField.required(5, "col_ts", TimestampType.withZone), NestedField.required(6, "col_tsnz", TimestampType.withoutZone) ).asJava) val icebergPartSpec = PartitionSpec .builderFor(icebergSchema) .identity("col_date") .identity("col_ts") .identity("col_tsnz") .build val physicalNameToField = Map( "pname1" -> icebergPartSpec.fields().get(0), "pname2" -> icebergPartSpec.fields().get(1), "pname3" -> icebergPartSpec.fields().get(2) ) val partitionConverter = IcebergPartitionConverter(icebergSchema, physicalNameToField) val partData = new PartitionData( StructType.of( NestedField.required(1000, "col_date", DateType.get), NestedField.required(1001, "col_ts", TimestampType.withZone), NestedField.required(1002, "col_tsnz", TimestampType.withoutZone) ) ) partData.put(0, 12800) partData.put(1, 1790040414914000L) partData.put(2, 1790040414914000L) assertResult("Map(pname1 -> 2005-01-17, " + "pname2 -> 2026-09-21 18:26:54.9, pname3 -> 2026-09-21 18:26:54.9)")( partitionConverter.toDelta(partData).toString) } } ================================================ FILE: iceberg/src/test/scala/org/apache/spark/sql/delta/commands/convert/IcebergStatsUtilsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.convert import java.lang.{Boolean => JBoolean, Double => JDouble, Float => JFloat, Integer => JInt, Long => JLong} import java.math.BigDecimal import java.nio.ByteBuffer import java.util.{HashMap => JHashMap, List => JList, Map => JMap} import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.JsonUtils import shadedForDelta.org.apache.iceberg.{DataFile, FileContent, FileFormat, PartitionData, PartitionSpec, Schema, StructLike} import shadedForDelta.org.apache.iceberg.transforms._ import shadedForDelta.org.apache.iceberg.types.Conversions import shadedForDelta.org.apache.iceberg.types.Type import shadedForDelta.org.apache.iceberg.types.Type.TypeID import shadedForDelta.org.apache.iceberg.types.Types._ import org.apache.spark.SparkFunSuite import org.apache.spark.internal.config.ConfigEntry import org.apache.spark.sql.test.SharedSparkSession class IcebergStatsUtilsSuite extends SparkFunSuite with SharedSparkSession { private val StatsAllowTypes = IcebergStatsUtils.typesAllowStatsConversion(statsDisallowTypes = Set.empty) test("stats conversion from basic columns") { val icebergSchema = new Schema(10, Seq[NestedField]( NestedField.required(1, "col_int", IntegerType.get), NestedField.required(2, "col_long", LongType.get), NestedField.required(3, "col_st", StringType.get), NestedField.required(4, "col_boolean", BooleanType.get), NestedField.required(5, "col_float", FloatType.get), NestedField.required(6, "col_double", DoubleType.get), NestedField.required(7, "col_date", DateType.get), NestedField.required(8, "col_binary", BinaryType.get), NestedField.required(9, "col_strt", StructType.of( NestedField.required(10, "sc_int", IntegerType.get), NestedField.required(11, "sc_int2", IntegerType.get) )), NestedField.required(12, "col_array", ListType.ofRequired(13, IntegerType.get)), NestedField.required(14, "col_map", MapType.ofRequired(15, 16, IntegerType.get, StringType.get))).asJava ) val minMap = Map( Integer.valueOf(1) -> Conversions.toByteBuffer(IntegerType.get, JInt.valueOf(-5)), Integer.valueOf(2) -> Conversions.toByteBuffer(LongType.get, JLong.valueOf(-4)), Integer.valueOf(3) -> Conversions.toByteBuffer(StringType.get, "minval"), Integer.valueOf(4) -> Conversions.toByteBuffer(BooleanType.get, JBoolean.FALSE), Integer.valueOf(5) -> Conversions.toByteBuffer(FloatType.get, JFloat.valueOf("0.001")), Integer.valueOf(6) -> Conversions.toByteBuffer(DoubleType.get, JDouble.valueOf("0.0001")), Integer.valueOf(7) -> Conversions.toByteBuffer(DateType.get, JInt.valueOf(12800)), Integer.valueOf(8) -> Conversions.toByteBuffer(BinaryType.get, ByteBuffer.wrap(Array(1, 2, 3, 4))), Integer.valueOf(10) -> Conversions.toByteBuffer(IntegerType.get, JInt.valueOf(-1)), Integer.valueOf(11) -> Conversions.toByteBuffer(IntegerType.get, JInt.valueOf(-1)) ) val maxMap = Map( Integer.valueOf(1) -> Conversions.toByteBuffer(IntegerType.get, JInt.valueOf(5)), Integer.valueOf(2) -> Conversions.toByteBuffer(LongType.get, JLong.valueOf(4)), Integer.valueOf(3) -> Conversions.toByteBuffer(StringType.get, "maxval"), Integer.valueOf(4) -> Conversions.toByteBuffer(BooleanType.get, JBoolean.TRUE), Integer.valueOf(5) -> Conversions.toByteBuffer(FloatType.get, JFloat.valueOf("10.001")), Integer.valueOf(6) -> Conversions.toByteBuffer(DoubleType.get, JDouble.valueOf("10.0001")), Integer.valueOf(7) -> Conversions.toByteBuffer(DateType.get, JInt.valueOf(13800)), Integer.valueOf(8) -> Conversions.toByteBuffer(BinaryType.get, ByteBuffer.wrap(Array(2, 2, 3, 4))), Integer.valueOf(10) -> Conversions.toByteBuffer(IntegerType.get, JInt.valueOf(128)), Integer.valueOf(11) -> Conversions.toByteBuffer(IntegerType.get, JInt.valueOf(512)) ) val nullCountMap = Map( Integer.valueOf(1) -> JLong.valueOf(0), Integer.valueOf(2) -> JLong.valueOf(1), Integer.valueOf(3) -> JLong.valueOf(2), Integer.valueOf(5) -> JLong.valueOf(3), Integer.valueOf(6) -> JLong.valueOf(4), Integer.valueOf(7) -> JLong.valueOf(5), Integer.valueOf(8) -> JLong.valueOf(6), Integer.valueOf(10) -> JLong.valueOf(7), Integer.valueOf(11) -> JLong.valueOf(8), Integer.valueOf(12) -> JLong.valueOf(9), Integer.valueOf(14) -> JLong.valueOf(10) ) val deltaStats = IcebergStatsUtils.icebergStatsToDelta( icebergSchema, 1251, Some(minMap), Some(maxMap), Some(nullCountMap), statsAllowTypes = StatsAllowTypes ) val actualStatsObj = JsonUtils.fromJson[StatsObject](deltaStats) val expectedStatsObj = JsonUtils.fromJson[StatsObject]( """{"numRecords":1251, |"maxValues":{"col_date":"2005-01-17","col_int":-5,"col_double":1.0E-4, |"col_float":0.001,"col_long":-4,"col_strt":{"sc_int":-1,"sc_int2":-1}, |"col_boolean":false,"col_st":"minval","col_binary":"AQIDBA=="}, |"minValues":{"col_date":"2007-10-14","col_int":5,"col_double":10.0001, |"col_float":10.001,"col_long":4,"col_strt":{"sc_int":128,"sc_int2":512}, |"col_boolean":true,"col_st":"maxval","col_binary":"AgIDBA=="}, |"nullCount":{"col_int":0,"col_double":4,"col_date":5,"col_float":3,"col_long":1, |"col_strt":{"sc_int":7,"sc_int2":8},"col_st":2,"col_binary":6,"col_array":9,"col_map":10}} |""".stripMargin.replaceAll("\n", "")) assertResult(expectedStatsObj)(actualStatsObj) } test("stats conversion for decimal and timestamp") { val icebergSchema = new Schema(10, Seq[NestedField]( NestedField.required(1, "col_ts", TimestampType.withZone), NestedField.required(2, "col_tsnz", TimestampType.withoutZone), NestedField.required(3, "col_decimal", DecimalType.of(10, 5)) ).asJava) val deltaStats = IcebergStatsUtils.icebergStatsToDelta( icebergSchema, 1251, minMap = Some(Map( Integer.valueOf(1) -> Conversions.toByteBuffer(TimestampType.withZone, JLong.valueOf(1734391979000000L)), Integer.valueOf(2) -> Conversions.toByteBuffer(TimestampType.withoutZone, JLong.valueOf(1734391979000000L)), Integer.valueOf(3) -> Conversions.toByteBuffer(DecimalType.of(10, 5), new BigDecimal("3.44141")) )), maxMap = Some(Map( Integer.valueOf(1) -> Conversions.toByteBuffer(TimestampType.withZone, JLong.valueOf(1734394979000000L)), Integer.valueOf(2) -> Conversions.toByteBuffer(TimestampType.withoutZone, JLong.valueOf(1734394979000000L)), Integer.valueOf(3) -> Conversions.toByteBuffer(DecimalType.of(10, 5), new BigDecimal("9.99999")) )), nullCountMap = Some(Map( Integer.valueOf(1) -> JLong.valueOf(20), Integer.valueOf(2) -> JLong.valueOf(10), Integer.valueOf(3) -> JLong.valueOf(31) )), statsAllowTypes = StatsAllowTypes ) assertResult( JsonUtils.fromJson[StatsObject]( """{"numRecords":1251, |"maxValues":{ | "col_ts":"2024-12-17T00:22:59+00:00", | "col_tsnz":"2024-12-17T00:22:59", | "col_decimal":9.99999 | }, |"minValues":{ | "col_ts":"2024-12-16T23:32:59+00:00", | "col_tsnz":"2024-12-16T23:32:59", | "col_decimal":3.44141 | }, |"nullCount":{"col_ts":20,"col_tsnz":10,"col_decimal":31}}""".stripMargin))( JsonUtils.fromJson[StatsObject](deltaStats)) } test("stats conversion when value is missing or is null") { val icebergSchema = new Schema(10, Seq[NestedField]( NestedField.required(1, "col_int", IntegerType.get), NestedField.required(2, "col_long", LongType.get), NestedField.required(3, "col_st", StringType.get) ).asJava) val deltaStats = IcebergStatsUtils.icebergStatsToDelta( icebergSchema, 1251, minMap = Some(Map( Integer.valueOf(1) -> Conversions.toByteBuffer(IntegerType.get, JInt.valueOf(-5)), Integer.valueOf(2) -> Conversions.toByteBuffer(LongType.get, null), Integer.valueOf(3) -> null )), maxMap = Some(Map( Integer.valueOf(1) -> Conversions.toByteBuffer(IntegerType.get, JInt.valueOf(5)), // stats for value 2 is missing Integer.valueOf(3) -> Conversions.toByteBuffer(StringType.get, "maxval"), Integer.valueOf(5) -> Conversions.toByteBuffer(StringType.get, "maxval") )), nullCountMap = Some(Map( Integer.valueOf(1) -> JLong.valueOf(0), Integer.valueOf(2) -> null, Integer.valueOf(3) -> JLong.valueOf(2), Integer.valueOf(5) -> JLong.valueOf(3) )), statsAllowTypes = StatsAllowTypes ) assertResult( JsonUtils.fromJson[StatsObject]( """{"numRecords":1251, |"maxValues":{"col_int":5,"col_st":"maxval"}, |"minValues":{"col_int":-5}, |"nullCount":{"col_int":0,"col_st":2}} |""".stripMargin))( JsonUtils.fromJson[StatsObject](deltaStats)) } private def testStatsConversion( expectedStatsJson: String, dataFile: DataFile, icebergSchema: Schema): Unit = { val expectedStats = JsonUtils.fromJson[StatsObject](expectedStatsJson) val actualStats = IcebergStatsUtils.icebergStatsToDelta( icebergSchema, dataFile, StatsAllowTypes, shouldSkipForFile = _ => false ) .map(JsonUtils.fromJson[StatsObject](_)) .get assertResult(expectedStats)(actualStats) } test("stats conversion while DataFile misses the stats fields") { val icebergSchema = new Schema(10, Seq[NestedField]( NestedField.required(1, "col_int", IntegerType.get), NestedField.required(2, "col_long", LongType.get), NestedField.required(3, "col_st", StringType.get) ).asJava) val expectedStatsJson = """{"numRecords":0,"maxValues":{"col_int":100992003}, |"minValues":{"col_int":100992003},"nullCount":{"col_int":2}}""" .stripMargin testStatsConversion(expectedStatsJson, DummyDataFile(), icebergSchema) val expectedStatsWithoutUpperBound = """{"numRecords":0,"minValues":{"col_int":100992003}, |"nullCount":{"col_int":2}}""" .stripMargin testStatsConversion( expectedStatsWithoutUpperBound, DummyDataFile(upperBounds = null), icebergSchema ) testStatsConversion( expectedStatsWithoutUpperBound, DummyDataFile(upperBounds = new JHashMap[Integer, ByteBuffer]()), icebergSchema ) val expectedStatsWithoutLowerBound = """{"numRecords":0,"maxValues":{"col_int":100992003}, |"nullCount":{"col_int":2}}""" .stripMargin testStatsConversion( expectedStatsWithoutLowerBound, DummyDataFile(lowerBounds = null), icebergSchema ) testStatsConversion( expectedStatsWithoutLowerBound, DummyDataFile(lowerBounds = new JHashMap[Integer, ByteBuffer]()), icebergSchema ) val expectedStatsWithoutNullCounts = """{"numRecords":0,"maxValues":{"col_int":100992003}, |"minValues":{"col_int":100992003}}""" .stripMargin testStatsConversion( expectedStatsWithoutNullCounts, DummyDataFile(nullValueCounts = null), icebergSchema ) testStatsConversion( expectedStatsWithoutNullCounts, DummyDataFile(nullValueCounts = new JHashMap[Integer, JLong]()), icebergSchema ) } } private case class StatsObject( numRecords: Long, maxValues: Map[String, Any], minValues: Map[String, Any], nullCount: Map[String, Long]) private case class DummyDataFile( upperBounds: JMap[JInt, ByteBuffer] = Map(JInt.valueOf(1) -> ByteBuffer.wrap(Array(3, 4, 5, 6))).asJava, lowerBounds: JMap[JInt, ByteBuffer] = Map(JInt.valueOf(1) -> ByteBuffer.wrap(Array(3, 4, 5, 6))).asJava, nullValueCounts: JMap[JInt, JLong] = Map(JInt.valueOf(1) -> JLong.valueOf(2)).asJava) extends DataFile { override def pos: JLong = 0L override def specId: Int = 0 override def path: String = "dummy" override def recordCount: Long = 0 override def fileSizeInBytes: Long = 0 override def content: FileContent = FileContent.DATA override def format: FileFormat = FileFormat.PARQUET override def partition: StructLike = null override def columnSizes: JMap[JInt, JLong] = null override def valueCounts: JMap[JInt, JLong] = null override def nanValueCounts: JMap[JInt, JLong] = null override def keyMetadata: ByteBuffer = null override def splitOffsets: JList[JLong] = null override def copy: DataFile = this.copy override def copyWithoutStats: DataFile = this.copy } ================================================ FILE: iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.serverSidePlanning import scala.jdk.CollectionConverters._ import org.apache.hadoop.fs.Path import org.apache.spark.sql.QueryTest import org.apache.spark.sql.sources._ import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructField, StructType} import shadedForDelta.org.apache.iceberg.{PartitionSpec, Schema, Table} import shadedForDelta.org.apache.iceberg.catalog._ import shadedForDelta.org.apache.iceberg.expressions.{Binder, Expressions} import shadedForDelta.org.apache.iceberg.rest.IcebergRESTServer import shadedForDelta.org.apache.iceberg.types.Types class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSession { import testImplicits._ private val defaultNamespace = Namespace.of("testDatabase") private val defaultSchema = TestSchemas.testSchema private val defaultSpec = PartitionSpec.unpartitioned() private lazy val server = IcebergRESTServerTestUtils.startServer() private lazy val catalog = server.getCatalog() private lazy val serverUri = s"http://localhost:${server.getPort}" override def beforeAll(): Unit = { super.beforeAll() // Configure Spark to use the Iceberg REST catalog spark.conf.set(s"spark.sql.catalog.rest_catalog", "org.apache.iceberg.spark.SparkCatalog") spark.conf.set(s"spark.sql.catalog.rest_catalog.type", "rest") spark.conf.set(s"spark.sql.catalog.rest_catalog.uri", serverUri) if (catalog.isInstanceOf[SupportsNamespaces]) { catalog.asInstanceOf[SupportsNamespaces].createNamespace(defaultNamespace) } else { throw new IllegalStateException("Catalog does not support namespaces") } } override def afterAll(): Unit = { try { if (server != null) { server.clearCaptured() server.stop() } } finally { super.afterAll() } } test("IcebergRESTCatalogPlanningClientFactory is auto-registered by default") { // Verify that calling getFactory() returns the Iceberg factory via auto-registration val factory = ServerSidePlanningClientFactory.getFactory() assert(factory != null, "Factory should not be null after auto-registration") assert(factory.getClass.getName.contains("IcebergRESTCatalogPlanningClientFactory"), s"Expected IcebergRESTCatalogPlanningClientFactory, got: ${factory.getClass.getName}") } // Tests that the REST /plan endpoint returns 0 files for an empty table. test("basic plan table scan via IcebergRESTCatalogPlanningClient") { withTempTable("testTable") { table => val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") try { val scanPlan = client.planScan(defaultNamespace.toString, "testTable") assert(scanPlan != null, "Scan plan should not be null") assert(scanPlan.files != null, "Scan plan files should not be null") assert(scanPlan.files.isEmpty, s"Empty table should have 0 files, got ${scanPlan.files.length}") } finally { client.close() } } } // Tests that the REST /plan endpoint returns the correct number of files for a non-empty table. // Creates a table, writes actual parquet files with data, then verifies the response includes // them. test("plan scan on non-empty table with data files") { withTempTable("tableWithData") { table => val tableName = s"rest_catalog.${defaultNamespace}.tableWithData" populateTestData(tableName) // Get the actual data files from the table metadata to verify against scan plan val expectedFiles = spark.sql( s"SELECT file_path, file_size_in_bytes FROM ${tableName}.files") .collect() .map(row => (new Path(row.getString(0)).getName, row.getLong(1))) .toMap val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") try { val scanPlan = client.planScan(defaultNamespace.toString, "tableWithData") assert(scanPlan != null, "Scan plan should not be null") assert(scanPlan.files != null, "Scan plan files should not be null") assert(scanPlan.files.length == 2, s"Expected 2 files but got ${scanPlan.files.length}") // Get scanned files as map of filename -> size val scannedFiles = scanPlan.files.map { file => (new Path(file.filePath).getName, file.fileSizeInBytes) }.toMap // Verify scan plan files match expected files assert(scannedFiles == expectedFiles, s"Scan plan files don't match expected files.\n" + s"Expected: $expectedFiles\n" + s"Got: $scannedFiles") } finally { client.close() } } } // TODO: Add test for partitioned table rejection // Once the test server (IcebergRESTCatalogAdapterWithPlanSupport) properly retains and serves // partition data through the commit/serialize/deserialize cycle, add a test that verifies: // 1. Creates a partitioned table with data files containing partition info // 2. Calls client.planScan() and expects UnsupportedOperationException // 3. Verifies exception message contains "partition data" // This will test the client's partition validation logic at // IcebergRESTCatalogPlanningClient:160-164 test("IcebergRESTCatalogPlanningClient uses prefix from /v1/config endpoint") { server.clearCaptured() // Clear any previous state server.setCatalogPrefix("catalogs/test-catalog-prefix") withTempTable("testTable") { table => // Client expects baseUri to include the /v1 path (per Iceberg REST spec) val client = new IcebergRESTCatalogPlanningClient(s"$serverUri/v1", "test_catalog", "") try { // Make a call that will trigger the lazy initialization of icebergRestCatalogUriRoot // which internally calls fetchCatalogPrefix() val scanPlan = client.planScan(defaultNamespace.toString, "testTable") assert(scanPlan != null, "Scan plan should not be null") // Verify the server received a /plan request with the correct prefix // This confirms that the config endpoint returned the correct prefix and that the client // correctly constructed the full plan request path. val capturedPath = server.getCapturedPlanRequestPath() assert(capturedPath != null, "Server should have captured the request path") assert(capturedPath.startsWith("v1/catalogs/test-catalog-prefix/"), s"Expected path to start with 'v1/catalogs/test-catalog-prefix/' but got: $capturedPath") } finally { client.close() } } } test("IcebergRESTCatalogPlanningClient uses baseUri directly when /v1/config returns no prefix") { server.clearCaptured() // Clear any previous state // Configure server to return no prefix server.setCatalogPrefix(null) withTempTable("testTable") { table => // Client expects baseUri to include the /v1 path (per Iceberg REST spec) val client = new IcebergRESTCatalogPlanningClient(s"$serverUri/v1", "test_catalog", "") try { // Make a call that will trigger the lazy initialization val scanPlan = client.planScan(defaultNamespace.toString, "testTable") assert(scanPlan != null, "Scan plan should not be null") // Verify the server received a /plan request using baseUri directly (no prefix) val capturedPath = server.getCapturedPlanRequestPath() assert(capturedPath != null, "Server should have captured the request path") // When no prefix is returned, use baseUri directly without adding prefix assert( !capturedPath.contains("catalogs/"), s"Expected path to NOT contain 'catalogs/' when no prefix, but got: $capturedPath") assert( capturedPath.startsWith("v1/namespaces/"), s"Expected path to start with 'v1/namespaces/' (using baseUri directly), but got: " + s"$capturedPath") } finally { client.close() } } } test("filter sent to IRC server over HTTP") { withTempTable("filterTest") { table => populateTestData(s"rest_catalog.${defaultNamespace}.filterTest") val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") try { val testCases = Seq( (EqualTo("longCol", 2L), "EqualTo numeric (long)"), (EqualTo("intCol", 30), "EqualTo numeric (int)"), (EqualTo("stringCol", "bob"), "EqualTo string"), (EqualTo("boolCol", true), "EqualTo boolean"), (Not(EqualTo("longCol", 2L)), "NotEqualTo numeric (long)"), (Not(EqualTo("stringCol", "bob")), "NotEqualTo string"), (LessThan("longCol", 10L), "LessThan (long)"), (LessThan("floatCol", 4.5f), "LessThan (float)"), (GreaterThan("longCol", 5L), "GreaterThan (long)"), (GreaterThan("doubleCol", 100.0), "GreaterThan (double)"), (LessThanOrEqual("intCol", 30), "LessThanOrEqual (int)"), (GreaterThanOrEqual("doubleCol", 100.0), "GreaterThanOrEqual (double)"), (In("longCol", Array(1L, 2L, 3L)), "In numeric (long)"), (In("stringCol", Array("alice", "bob", "charlie")), "In string"), (IsNull("stringCol"), "IsNull"), (IsNotNull("stringCol"), "IsNotNull"), (StringStartsWith("stringCol", "ali"), "StringStartsWith"), (AlwaysTrue(), "AlwaysTrue"), (AlwaysFalse(), "AlwaysFalse"), (And(EqualTo("longCol", 2L), EqualTo("stringCol", "bob")), "And"), (Or(EqualTo("longCol", 1L), EqualTo("longCol", 3L)), "Or"), (EqualTo("address.intCol", 200), "EqualTo on nested numeric field"), (EqualTo("metadata.stringCol", "meta_bob"), "EqualTo on nested string field"), (GreaterThan("address.intCol", 500), "GreaterThan on nested numeric field")) testCases.foreach { case (filter, description) => // Clear previous captured filter server.clearCaptured() // Convert Spark filter to expected Iceberg expression val expectedExpr = SparkToIcebergExpressionConverter.convert(filter) assert( expectedExpr.isDefined, s"[$description] Filter conversion should succeed for: $filter") // Call client with filter client.planScan( defaultNamespace.toString, "filterTest", sparkFilterOption = Some(filter)) // Verify server captured the filter val capturedFilter = server.getCapturedFilter assert(capturedFilter != null, s"[$description] Server should have captured filter") // isEquivalentTo() only works on bound expressions, so bind both to schema for comparison // Binding resolves field references from names to schema-specific field IDs and types val boundExpected = Binder.bind(defaultSchema.asStruct(), expectedExpr.get, true) val boundCaptured = Binder.bind(defaultSchema.asStruct(), capturedFilter, true) assert( boundCaptured.isEquivalentTo(boundExpected), s"[$description] Expected expression: $boundExpected, got: $boundCaptured") } } finally { client.close() } } } // Test case classes for structured test data private case class ProjectionTestCase( description: String, projection: Seq[String], expected: Set[String]) private case class PushdownTestCase( description: String, filter: Filter, projection: Seq[String], limit: Option[Int]) test("projection sent to IRC server over HTTP") { withTempTable("projectionTest") { table => // Populate test data using the shared helper method val tableName = s"rest_catalog.${defaultNamespace}.projectionTest" populateTestData(tableName) // Test cases covering different projection scenarios // Note: At this HTTP layer, we're only testing that column name strings are correctly // sent and received. Type serialization and data reading are tested end-to-end. val testCases = Seq( // Basic projections ProjectionTestCase( "single column", Seq("intCol"), Set("intCol")), ProjectionTestCase( "multiple columns", Seq("intCol", "stringCol"), Set("intCol", "stringCol")), // Nested field projections - test dot-notation string handling ProjectionTestCase( "individual nested field", Seq("address.intCol"), Set("address.intCol")), ProjectionTestCase( "dotted field name inside struct with escaping", Seq("parent.`child.name`"), Set("parent.`child.name`")), ProjectionTestCase( "dotted column name with escaping", Seq("`address.city`"), Set("`address.city`")) ) val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") try { testCases.foreach { testCase => // Clear previous captured projection server.clearCaptured() client.planScan( defaultNamespace.toString, "projectionTest", sparkProjectionOption = Some(testCase.projection)) // Verify server captured the projection val capturedProjection = server.getCapturedProjection assert(capturedProjection != null, s"[${testCase.description}] Server should have captured projection") // Verify field names match expected val fieldNames = capturedProjection.asScala.toSet assert(fieldNames == testCase.expected, s"[${testCase.description}] Expected ${testCase.expected}, got: $fieldNames") } } finally { client.close() } } } test("limit sent to IRC server over HTTP") { withTempTable("limitTest") { table => // Populate test data using the shared helper method val tableName = s"rest_catalog.${defaultNamespace}.limitTest" populateTestData(tableName) val client = new IcebergRESTCatalogPlanningClient(serverUri, null, "") try { // Test different limit values val testCases = Seq( (Some(10), Some(10L), "limit = 10"), (Some(100), Some(100L), "limit = 100"), (Some(1), Some(1L), "limit = 1"), (None, None, "no limit")) testCases.foreach { case (limitOption, expectedCaptured, description) => // Clear previous captured state server.clearCaptured() client.planScan( defaultNamespace.toString, "limitTest", sparkLimitOption = limitOption) // Verify server captured the limit val capturedLimit = Option(server.getCapturedLimit) assert(capturedLimit == expectedCaptured, s"[$description] Expected $expectedCaptured, got: $capturedLimit") } } finally { client.close() } } } test("filter, projection, and limit sent together to IRC server over HTTP") { withTempTable("filterProjectionLimitTest") { table => // Populate test data using the shared helper method val tableName = s"rest_catalog.${defaultNamespace}.filterProjectionLimitTest" populateTestData(tableName) val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") try { // Note: Filter types are already tested in "filter sent to IRC server" test. // Here we verify filter, projection, AND limit are sent together correctly. val testCases = Seq( PushdownTestCase( "filter + projection + limit", EqualTo("longCol", 2L), Seq("intCol", "stringCol"), Some(10)), PushdownTestCase( "nested field in both filter and projection + limit", EqualTo("address.intCol", 200), Seq("intCol", "address.intCol"), Some(5)) ) testCases.foreach { testCase => // Clear previous captured state server.clearCaptured() // Convert Spark filter to expected Iceberg expression val expectedExpr = SparkToIcebergExpressionConverter.convert(testCase.filter) assert( expectedExpr.isDefined, s"[${testCase.description}] Filter conversion should succeed for: ${testCase.filter}") // Call client with filter, projection, and limit client.planScan( defaultNamespace.toString, "filterProjectionLimitTest", sparkFilterOption = Some(testCase.filter), sparkProjectionOption = Some(testCase.projection), sparkLimitOption = testCase.limit) // Verify server captured filter, projection, and limit val capturedFilter = server.getCapturedFilter val capturedProjection = server.getCapturedProjection val capturedLimit = server.getCapturedLimit assert(capturedFilter != null, s"[${testCase.description}] Server should have captured filter") assert(capturedProjection != null, s"[${testCase.description}] Server should have captured projection") assert(capturedLimit != null, s"[${testCase.description}] Server should have captured limit") // Verify filter is correct val boundExpected = Binder.bind(defaultSchema.asStruct(), expectedExpr.get, true) val boundCaptured = Binder.bind(defaultSchema.asStruct(), capturedFilter, true) assert( boundCaptured.isEquivalentTo(boundExpected), s"[${testCase.description}] Filter mismatch. Expected: $boundExpected, " + s"got: $boundCaptured") // Verify projection is correct val projectionFields = capturedProjection.asScala.toSet val expectedFields = testCase.projection.toSet assert(projectionFields == expectedFields, s"[${testCase.description}] Projection mismatch. Expected: $expectedFields, " + s"got: $projectionFields") // Verify limit is correct val expectedLimit = testCase.limit.map(_.toLong) assert(Option(capturedLimit) == expectedLimit, s"[${testCase.description}] Limit mismatch. Expected: $expectedLimit, " + s"got: ${Option(capturedLimit)}") } } finally { client.close() } } } test("caseSensitive=false sent to IRC server") { withTempTable("caseSensitiveTest") { table => populateTestData(s"rest_catalog.${defaultNamespace}.caseSensitiveTest") val client = new IcebergRESTCatalogPlanningClient(serverUri, null, "") try { server.clearCaptured() // Call planScan - the client sets caseSensitive=false in the request val scanPlan = client.planScan(defaultNamespace.toString, "caseSensitiveTest") // Verify the scan succeeds and returns files assert(scanPlan.files.nonEmpty, "Expected planScan to return files for the test table") // Verify server captured caseSensitive=false val capturedCaseSensitive = server.getCapturedCaseSensitive() assert(capturedCaseSensitive == false, s"Expected server to capture caseSensitive=false, got $capturedCaseSensitive") } finally { client.close() } } } test("rejects FileScanTask with non-trivial residual") { withTempTable("residualTest") { table => populateTestData(s"rest_catalog.${defaultNamespace}.residualTest") val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") try { // Verify that a trivial (alwaysTrue) residual is accepted server.setTestResidual(Expressions.alwaysTrue()) val scanPlan = client.planScan(defaultNamespace.toString, "residualTest") assert(scanPlan.files.nonEmpty, "Scan with alwaysTrue residual should succeed and return files") // Configure server to inject a non-trivial residual expression into the response. // This simulates a server that expects the client to apply a residual filter, // which is currently unsupported. server.setTestResidual(Expressions.greaterThan("longCol", 42L)) val exception = intercept[UnsupportedOperationException] { client.planScan(defaultNamespace.toString, "residualTest") } assert(exception.getMessage.contains("residual"), s"Error message should mention 'residual'. Got: ${exception.getMessage}") } finally { client.close() } } } /** * Convenience wrapper for withTempTable that uses the test suite's default values. */ private def withTempTable[T](tableName: String)(func: Table => T): T = { IcebergRESTServerTestUtils.withTempTable( catalog, defaultNamespace, tableName, defaultSchema, defaultSpec, Some(server) )(func) } /** * Convenience wrapper for populateTestData that uses the test suite's SparkSession. */ private def populateTestData(tableName: String): Unit = { IcebergRESTServerTestUtils.populateTestData(spark, tableName) } test("retry on transient 503 server error") { withTempTable("retryTest503") { table => populateTestData(s"rest_catalog.${defaultNamespace}.retryTest503") val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") try { server.clearCaptured() // Configure server to fail the first plan request with 503 server.setFailNextPlanRequests(1, 503) // Client should retry and succeed on the second attempt val scanPlan = client.planScan(defaultNamespace.toString, "retryTest503") assert(scanPlan != null, "Scan plan should not be null after retry") assert(scanPlan.files.nonEmpty, "Scan plan should have files after successful retry") // Verify 2 requests were made: 1 failed (503) + 1 success assert(server.getPlanRequestCount() == 2, s"Expected 2 plan requests (1 retry), got ${server.getPlanRequestCount()}") } finally { server.clearCaptured() client.close() } } } test("retries exhausted on persistent 503 server error") { // No populateTestData needed: failure injection intercepts at the servlet level before // table data is accessed, so we only need the table to exist for a valid URI. withTempTable("retryTestExhausted") { table => val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") try { server.clearCaptured() // Configure server to fail more requests than the client will retry (max 3 retries = 4 // total attempts). Setting 10 failures ensures all retries see 503. server.setFailNextPlanRequests(10, 503) val exception = intercept[java.io.IOException] { client.planScan(defaultNamespace.toString, "retryTestExhausted") } assert(exception.getMessage.contains("503"), s"Error should mention 503 status code. Got: ${exception.getMessage}") // Verify 4 requests were made: 1 original + 3 retries (max retries = 3) assert(server.getPlanRequestCount() == 4, s"Expected 4 plan requests (1 + 3 retries), got ${server.getPlanRequestCount()}") } finally { server.clearCaptured() client.close() } } } test("no retry on 404 client error") { // No populateTestData needed: failure injection intercepts at the servlet level before // table data is accessed, so we only need the table to exist for a valid URI. withTempTable("retryTest404") { table => val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") try { server.clearCaptured() // Configure server to fail all plan requests with 404 // Using a high count ensures the test fails if the client retries server.setFailNextPlanRequests(10, 404) // Client should NOT retry 404 and should throw immediately val exception = intercept[java.io.IOException] { client.planScan(defaultNamespace.toString, "retryTest404") } assert(exception.getMessage.contains("404"), s"Error should mention 404 status code. Got: ${exception.getMessage}") // Verify only 1 request was made (no retry for 404) assert(server.getPlanRequestCount() == 1, s"Expected 1 plan request (no retry for 404), got ${server.getPlanRequestCount()}") } finally { server.clearCaptured() client.close() } } } test("fetchCatalogPrefix falls back to baseUri on connection failure") { // Use a port that's expected to have no listener. fetchCatalogPrefix() makes an HTTP GET // to /config which will fail with a connection error. It should catch the exception, log a // warning, and return None — causing icebergRestCatalogUriRoot to fall back to baseUri. // The subsequent planScan HTTP POST will also fail (same unreachable host). val unreachableUri = "http://localhost:1" val client = new IcebergRESTCatalogPlanningClient(unreachableUri, "test_catalog", "") try { val ex = intercept[Exception] { client.planScan("test_db", "test_table") } // Verify the exception is a connection error. This confirms fetchCatalogPrefix() // did not throw a different exception type (e.g., NPE, parse error) and that the // client progressed past the config fetch to attempt the plan HTTP POST. assert(ex.getMessage != null, "Expected a connection error with a message from the HTTP client") } finally { client.close() } } test("User-Agent header format") { val client = new IcebergRESTCatalogPlanningClient("http://localhost:8080", "test_catalog", "") try { val userAgent = client.getUserAgent() // Verify the format follows RFC 7231: product/version [product/version ...] val parts = userAgent.split(" ") assert(parts.length == 4, s"User-Agent should have 4 space-separated components, got ${parts.length}: $userAgent") // First part should be Delta/version assert(parts(0).matches("Delta/.*"), s"First component should match 'Delta/', got: ${parts(0)}") // Second part should be Spark/version assert(parts(1).matches("Spark/.*"), s"Second component should match 'Spark/', got: ${parts(1)}") // Third part should be Java/version assert(parts(2).matches("Java/.*"), s"Third component should match 'Java/', got: ${parts(2)}") // Fourth part should be Scala/version assert(parts(3).matches("Scala/.*"), s"Fourth component should match 'Scala/', got: ${parts(3)}") // Verify versions are not "unknown" in test environment where all dependencies are available assert(!userAgent.contains("Spark/unknown"), s"Spark version should not be 'unknown' in test environment, got: $userAgent") assert(!userAgent.contains("Delta/unknown"), s"Delta version should not be 'unknown' in test environment, got: $userAgent") assert(!userAgent.contains("Java/unknown"), s"Java version should not be 'unknown' in test environment, got: $userAgent") assert(!userAgent.contains("Scala/unknown"), s"Scala version should not be 'unknown' in test environment, got: $userAgent") } finally { client.close() } } } ================================================ FILE: iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTServerTestUtils.scala ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.serverSidePlanning import scala.jdk.CollectionConverters._ import org.apache.http.HttpHeaders import org.apache.http.client.methods.HttpGet import org.apache.http.entity.ContentType import org.apache.http.impl.client.HttpClientBuilder import org.apache.http.message.BasicHeader import org.apache.spark.sql.{Row, SparkSession} import shadedForDelta.org.apache.iceberg.{PartitionSpec, Table} import shadedForDelta.org.apache.iceberg.catalog._ import shadedForDelta.org.apache.iceberg.rest.IcebergRESTServer /** * Shared test utilities for IcebergRESTServer-based tests. * * Provides helper methods for: * - Starting and verifying IcebergRESTServer instances * - Managing table lifecycle with guaranteed cleanup * - Populating test data with consistent schemas */ object IcebergRESTServerTestUtils { /** * Starts an IcebergRESTServer on a dynamic port and verifies it's reachable. * * @return Started and verified IcebergRESTServer instance * @throws IllegalStateException if server fails to start or become reachable */ def startServer(): IcebergRESTServer = { val config = Map(IcebergRESTServer.REST_PORT -> "0").asJava val newServer = new IcebergRESTServer(config) newServer.start(/* join = */ false) if (!isServerReachable(newServer)) { throw new IllegalStateException("Failed to start IcebergRESTServer") } newServer } /** * Checks if an IcebergRESTServer is reachable via HTTP. * * Makes a GET request to /v1/config endpoint to verify server is responding. * * @param server The IcebergRESTServer to check * @return true if server returns 200 OK, false otherwise */ def isServerReachable(server: IcebergRESTServer): Boolean = { val httpHeaders = Map( HttpHeaders.ACCEPT -> ContentType.APPLICATION_JSON.getMimeType, HttpHeaders.CONTENT_TYPE -> ContentType.APPLICATION_JSON.getMimeType ).map { case (k, v) => new BasicHeader(k, v) }.toSeq.asJava val httpClient = HttpClientBuilder.create() .setDefaultHeaders(httpHeaders) .build() try { val httpGet = new HttpGet(s"http://localhost:${server.getPort}/v1/config") val httpResponse = httpClient.execute(httpGet) try { val statusCode = httpResponse.getStatusLine.getStatusCode statusCode == 200 } finally { httpResponse.close() } } finally { httpClient.close() } } /** * Executes a function with a temporary table, guaranteeing cleanup. * * @param catalog The Iceberg catalog to use * @param namespace The namespace for the table * @param tableName The table name * @param schema The table schema * @param spec The partition spec * @param server Optional server to clear captures after cleanup * @param func Function to execute with the created table * @return The result of executing func */ def withTempTable[T]( catalog: Catalog, namespace: Namespace, tableName: String, schema: shadedForDelta.org.apache.iceberg.Schema, spec: PartitionSpec, server: Option[IcebergRESTServer] = None )(func: Table => T): T = { val tableId = TableIdentifier.of(namespace, tableName) val table = catalog.createTable(tableId, schema, spec) try { func(table) } finally { catalog.dropTable(tableId, false) server.foreach(_.clearCaptured()) } } /** * Populates an Iceberg table with test data. * * Creates 250 rows of test data using TestSchemas.sparkSchema, distributed * across 2 partitions to create 2 data files. * * @param spark The SparkSession to use * @param tableName The fully-qualified table name (e.g., "catalog.db.table") */ def populateTestData(spark: SparkSession, tableName: String): Unit = { // scalastyle:off sparkimplicits import spark.implicits._ // scalastyle:on sparkimplicits val data = spark.sparkContext.parallelize(0 until 250, numSlices = 2) .map(i => Row( i, // intCol i.toLong, // longCol i * 10.0, // doubleCol i.toFloat, // floatCol s"test_$i", // stringCol i % 2 == 0, // boolCol BigDecimal(i).bigDecimal, // decimalCol java.sql.Date.valueOf("2024-01-01"), // dateCol java.sql.Timestamp.valueOf("2024-01-01 00:00:00"), // timestampCol java.sql.Date.valueOf("2024-01-01"), // localDateCol java.sql.Timestamp.valueOf("2024-01-01 00:00:00"), // localDateTimeCol java.sql.Timestamp.valueOf("2024-01-01 00:00:00"), // instantCol Row(i * 100), // address.intCol (nested struct) Row(s"meta_$i"), // metadata.stringCol (nested struct) Row(s"child_$i"), // parent.`child.name` (nested struct with dotted field name) s"city_$i", // address.city (literal top-level dotted column) s"abc_$i" // a.b.c (literal top-level dotted column) )) spark.createDataFrame(data, TestSchemas.sparkSchema) .write .format("iceberg") .mode("append") .save(tableName) } } ================================================ FILE: iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningCredentialsSuite.scala ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.serverSidePlanning import scala.jdk.CollectionConverters._ import org.apache.hadoop.conf.Configuration import org.apache.spark.sql.QueryTest import org.apache.spark.sql.test.SharedSparkSession import shadedForDelta.org.apache.iceberg.{PartitionSpec, Table} import shadedForDelta.org.apache.iceberg.catalog._ /** * Test suite for server-side planning credential handling. * Tests credential parsing and Hadoop configuration injection for S3, Azure, and GCS. */ class ServerSidePlanningCredentialsSuite extends QueryTest with SharedSparkSession { import CredentialTestHelpers._ private val defaultNamespace = Namespace.of("testDatabase") private val defaultSchema = TestSchemas.testSchema private val defaultSpec = PartitionSpec.unpartitioned() private lazy val server = IcebergRESTServerTestUtils.startServer() private lazy val catalog = server.getCatalog() private lazy val serverUri = s"http://localhost:${server.getPort}" override def beforeAll(): Unit = { super.beforeAll() // Configure Spark to use the Iceberg REST catalog spark.conf.set(s"spark.sql.catalog.rest_catalog", "org.apache.iceberg.spark.SparkCatalog") spark.conf.set(s"spark.sql.catalog.rest_catalog.type", "rest") spark.conf.set(s"spark.sql.catalog.rest_catalog.uri", serverUri) if (catalog.isInstanceOf[SupportsNamespaces]) { catalog.asInstanceOf[SupportsNamespaces].createNamespace(defaultNamespace) } else { throw new IllegalStateException("Catalog does not support namespaces") } } override def afterAll(): Unit = { try { if (server != null) { server.clearCaptured() server.stop() } } finally { super.afterAll() } } test("Credentials: server response parsing and Hadoop configuration") { withTempTable("credentialsTest") { table => populateTestData(s"rest_catalog.${defaultNamespace}.credentialsTest") val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") try { // Covers the successful credential extraction and Hadoop configuration injection cases. val testCases: Seq[CredentialTestCase] = Seq( // S3 S3CredentialTestCase( description = "S3 with session token", accessKeyId = "test-access-key", secretAccessKey = "test-secret-key", sessionToken = "test-session-token" ), // Azure without expiration AzureCredentialTestCase( description = "Azure without expiration", accountName = "unitycatalogmetastore", sasToken = "sv=2023-01-03&ss=b&srt=sco&sp=rwdlac&se=2025-12-31T23:59:59Z&sig=test", expirationMs = None ), // Azure with expiration AzureCredentialTestCase( description = "Azure with expiration", accountName = "unitycatalogmetastore", sasToken = "sv=2023-01-03&ss=b&srt=sco&sp=rwdlac&se=2025-12-31T23:59:59Z&sig=test", expirationMs = Some(1771456336352L) ), // GCS without expiration GcsCredentialTestCase( description = "GCS without expiration", token = "ya29.c.c0AY_VpZg_test_token", expirationMs = None ), // GCS with expiration GcsCredentialTestCase( description = "GCS with expiration", token = "ya29.c.c0AY_VpZg_test_token", expirationMs = Some(1771456336352L) ) ) testCases.foreach { testCase => // Set server to return credentials. server.setTestCredentials(testCase.serverResponse.asJava) val scanPlan = client.planScan(defaultNamespace.toString, "credentialsTest") assert(scanPlan.credentials.isDefined, s"[${testCase.description}] Credentials should be present in ScanPlan") val testConf = new Configuration() scanPlan.credentials.foreach(_.configure(testConf)) // Validate Hadoop config matches expectation. testCase.expectedHadoopConfig.foreach { case (key, expectedValue) => val actualValue = testConf.get(key) assert(actualValue == expectedValue, s"[${testCase.description}] Hadoop config mismatch for key '$key'.\n" + s"Expected: $expectedValue\n" + s"Got: $actualValue") } // Clear for next test case server.clearCaptured() } } finally { client.close() } } } test("incomplete/missing credentials throw errors") { withTempTable("incompleteCredsTest") { table => populateTestData(s"rest_catalog.${defaultNamespace}.incompleteCredsTest") val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") try { // Test cases for incomplete credentials that should throw errors val errorTestCases = Seq( ("Incomplete S3 (missing secret and token)", Map("s3.access-key-id" -> "test-key"), "Missing required credential"), ("GCS incomplete: only expiration", Map("gcs.oauth2.token-expires-at" -> "1771456336352"), "Unrecognized credential keys"), // Expiration-only Azure entry is unrecognized: without the token key // (adls.sas-token.), hasAzureKeys() returns false and we can't // construct valid credentials. ("Azure incomplete: expiration key only, no token key", Map("adls.sas-token-expires-at-ms.myaccount.dfs.core.windows.net" -> "1771456336352"), "Unrecognized credential keys") ) errorTestCases.foreach { case (description, incompleteConfig, expectedMessageFragment) => // Configure server with incomplete credentials server.setTestCredentials(incompleteConfig.asJava) // Verify that planScan throws IllegalStateException val exception = intercept[IllegalStateException] { client.planScan(defaultNamespace.toString, "incompleteCredsTest") } // Verify error message contains relevant fragment assert(exception.getMessage.contains(expectedMessageFragment), s"[$description] Error message should contain '$expectedMessageFragment'. " + s"Got: ${exception.getMessage}") // Clear for next test case server.clearCaptured() } } finally { client.close() } } } /** * Convenience wrapper for withTempTable that uses the test suite's default values. */ private def withTempTable[T](tableName: String)(func: Table => T): T = { IcebergRESTServerTestUtils.withTempTable( catalog, defaultNamespace, tableName, defaultSchema, defaultSpec, Some(server) )(func) } /** * Convenience wrapper for populateTestData that uses the test suite's SparkSession. */ private def populateTestData(tableName: String): Unit = { IcebergRESTServerTestUtils.populateTestData(spark, tableName) } /** * Credential test helper traits and case classes. * Private to this test suite - these are test-only utilities. */ private object CredentialTestHelpers { /** * Test case for end-to-end credential validation. * * Flow: serverResponse → (client parses) → creds.configure(conf) → expectedHadoopConfig */ sealed trait CredentialTestCase { /** Test case name */ def description: String /** Cloud provider type (S3, Azure, GCS) */ def cloudProvider: String /** INPUT: Credential config map that server returns */ def serverResponse: Map[String, String] /** EXPECTED OUTPUT: Hadoop configuration keys and values */ def expectedHadoopConfig: Map[String, String] } /** * S3 credential test case. */ case class S3CredentialTestCase( description: String, accessKeyId: String, secretAccessKey: String, sessionToken: String ) extends CredentialTestCase { override def cloudProvider: String = "S3" override def serverResponse: Map[String, String] = Map( "s3.access-key-id" -> accessKeyId, "s3.secret-access-key" -> secretAccessKey, "s3.session-token" -> sessionToken ) override def expectedHadoopConfig: Map[String, String] = Map( "fs.s3a.path.style.access" -> "true", "fs.s3.impl.disable.cache" -> "true", "fs.s3a.impl.disable.cache" -> "true", "fs.s3a.access.key" -> accessKeyId, "fs.s3a.secret.key" -> secretAccessKey, "fs.s3a.session.token" -> sessionToken ) } /** * Azure credential test case. */ case class AzureCredentialTestCase( description: String, accountName: String, sasToken: String, expirationMs: Option[Long] = None ) extends CredentialTestCase { override def cloudProvider: String = "Azure" override def serverResponse: Map[String, String] = { val base = Map( s"adls.sas-token.$accountName.dfs.core.windows.net" -> sasToken ) expirationMs match { case Some(ms) => val expiryKey = s"adls.sas-token-expires-at-ms.$accountName.dfs.core.windows.net" base + (expiryKey -> ms.toString) case None => base } } override def expectedHadoopConfig: Map[String, String] = { val accountSuffix = s"$accountName.dfs.core.windows.net" Map( "fs.abfs.impl.disable.cache" -> "true", "fs.abfss.impl.disable.cache" -> "true", s"fs.azure.account.auth.type.$accountSuffix" -> "SAS", s"fs.azure.sas.fixed.token.$accountSuffix" -> sasToken ) } } /** * GCS credential test case. */ case class GcsCredentialTestCase( description: String, token: String, expirationMs: Option[Long] = None ) extends CredentialTestCase { override def cloudProvider: String = "GCS" override def serverResponse: Map[String, String] = { val base = Map("gcs.oauth2.token" -> token) expirationMs match { case Some(ms) => base + ("gcs.oauth2.token-expires-at" -> ms.toString) case None => base } } override def expectedHadoopConfig: Map[String, String] = { val base = Map( "fs.gs.impl.disable.cache" -> "true", "fs.gs.auth.type" -> "ACCESS_TOKEN_PROVIDER", "fs.gs.auth.access.token.provider.impl" -> "org.apache.spark.sql.delta.serverSidePlanning.FixedGcsAccessTokenProvider", "fs.gs.auth.access.token" -> token ) expirationMs match { case Some(ms) => base + ("fs.gs.auth.access.token.expiration.ms" -> ms.toString) case None => base } } } } } ================================================ FILE: iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/SparkToIcebergExpressionConverterSuite.scala ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.serverSidePlanning import org.apache.spark.sql.Row import org.apache.spark.sql.sources._ import org.scalatest.funsuite.AnyFunSuite import shadedForDelta.org.apache.iceberg.expressions.{Expression, ExpressionUtil, Expressions} class SparkToIcebergExpressionConverterSuite extends AnyFunSuite { private case class ExprConvTestCase( label: String, spark: Filter, iceberg: Option[Expression] ) // Types that support equality and ordering operations // (EqualTo, NotEqualTo, LessThan, GreaterThan, LessThanOrEqual, GreaterThanOrEqual) // // Note on Date/Timestamp: Spark Filter API sends java.sql.Date/Timestamp, but our converter // transforms them to Int (days since epoch) and Long (microseconds since epoch) for Iceberg. // // Note on escaped column names: Per Spark's Filter documentation, column names with special // characters (dots, spaces, etc.) arrive already escaped with backticks when they represent // literal column names. We verify the converter passes these through unchanged. Examples: // - "address.intCol" = nested field access (struct address, field intCol) // - "`address.city`" = literal column named "address.city" (backtick-quoted by Spark) // - "parent.`child.name`" = nested access where child field name is literally "child.name" private val orderableTypeTestCases = Seq( ("intCol", 42, "Int"), // (column name, test value, label to identify test case) ("longCol", 100L, "Long"), ("doubleCol", 99.99, "Double"), ("floatCol", 10.5f, "Float"), ("decimalCol", BigDecimal("123.45").bigDecimal, "Decimal"), ("stringCol", "test", "String"), ("dateCol", java.sql.Date.valueOf("2024-01-01"), "Date"), ("timestampCol", java.sql.Timestamp.valueOf("2024-01-01 12:00:00"), "Timestamp"), ("localDateCol", java.time.LocalDate.of(2024, 1, 1), "LocalDate"), ("localDateTimeCol", java.time.LocalDateTime.of(2024, 1, 1, 12, 0, 0), "LocalDateTime"), ("instantCol", java.time.Instant.parse("2024-01-01T12:00:00Z"), "Instant"), ("address.intCol", 42, "Nested Int"), ("metadata.stringCol", "test", "Nested String"), ("`address.city`", "Seattle", "Escaped Literal String"), ("`a.b.c`", "value", "Escaped Multi-Dot String"), ("parent.`child.name`", "childValue", "Nested with Escaped Field") ) // Types that only support equality operators (EqualTo, NotEqualTo, IsNull, IsNotNull) private val equalityOnlyTypesTestCases = Seq( ("boolCol", true, "Boolean") ) private val allTypesTestCases = orderableTypeTestCases ++ equalityOnlyTypesTestCases private val testSchema = TestSchemas.testSchema.asStruct() private def assertConvert(testCases: Seq[ExprConvTestCase]): Unit = { testCases.foreach { tc => val result = SparkToIcebergExpressionConverter.convert(tc.spark) tc.iceberg match { case Some(expected) => assert(result.isDefined, s"[${tc.label}] Should convert: ${tc.spark}") assert( ExpressionUtil.equivalent(expected, result.get, testSchema, true), s"[${tc.label}] Expected: $expected, got: ${result.get}" ) case None => assert(result.isEmpty, s"[${tc.label}] Should return None for: ${tc.spark}") } } } // ======================================================================== // EQUALITY OPERATORS (=, !=) // ======================================================================== test("equality operators (=, !=) on all types including null and NaN handling") { val equalityOpMappings = Seq( ("EqualTo", // Test case label (col: String, v: Any) => EqualTo(col, v), // Spark filter builder (col: String, v: Any) => Expressions.equal(col, v)), // Iceberg expression builder ("NotEqualTo", (col: String, v: Any) => Not(EqualTo(col, v)), (col: String, v: Any) => Expressions.notEqual(col, v)) ) // Generate all combinations: all types x equality operators val standardTests = for { (col, value, typeDesc) <- allTypesTestCases (opName, sparkOp, icebergOp) <- equalityOpMappings } yield ExprConvTestCase( s"$opName $typeDesc", sparkOp(col, value), // supportBoolean=true because equality operators work on all types including Boolean Some(icebergOp(col, SparkToIcebergExpressionConverter.toIcebergValue( value, supportBoolean = true))) ) // Null handling: EqualTo(col, null) -> isNull, Not(EqualTo(col, null)) -> notNull val nullHandlingTests = Seq( ExprConvTestCase( "EqualTo(col, null) converts to isNull", // Test case label EqualTo("stringCol", null), // Spark filter builder Some(Expressions.isNull("stringCol")) // Iceberg expression builder ), ExprConvTestCase( "Not(EqualTo(col, null)) converts to notNull (IS NOT NULL)", Not(EqualTo("stringCol", null)), Some(Expressions.notNull("stringCol")) ) ) // NaN handling: EqualTo/NotEqualTo with NaN convert to isNaN/notNaN predicates val nanHandlingTests = Seq( ExprConvTestCase( "EqualTo with Double.NaN converts to isNaN", // Test case label EqualTo("doubleCol", Double.NaN), // Spark filter builder Some(Expressions.isNaN("doubleCol")) // Iceberg expression builder ), ExprConvTestCase( "EqualTo with Float.NaN converts to isNaN", EqualTo("floatCol", Float.NaN), Some(Expressions.isNaN("floatCol")) ), ExprConvTestCase( "Not(EqualTo) with Double.NaN converts to notNaN", Not(EqualTo("doubleCol", Double.NaN)), Some(Expressions.notNaN("doubleCol")) ), ExprConvTestCase( "Not(EqualTo) with Float.NaN converts to notNaN", Not(EqualTo("floatCol", Float.NaN)), Some(Expressions.notNaN("floatCol")) ) ) assertConvert(standardTests ++ nullHandlingTests ++ nanHandlingTests) } // ======================================================================== // ORDERING COMPARISON OPERATORS (<, >, <=, >=) // ======================================================================== test("ordering comparison operators (<, >, <=, >=) on orderable types") { // Note: This only tests ordering comparisons (<, >, <=, >=), not equality or other operations val comparisonOpMappings = Seq( ("LessThan", // Test case label (col: String, v: Any) => LessThan(col, v), // Spark filter builder (col: String, v: Any) => Expressions.lessThan(col, v)), // Iceberg expression builder ("GreaterThan", (col: String, v: Any) => GreaterThan(col, v), (col: String, v: Any) => Expressions.greaterThan(col, v)), ("LessThanOrEqual", (col: String, v: Any) => LessThanOrEqual(col, v), (col: String, v: Any) => Expressions.lessThanOrEqual(col, v)), ("GreaterThanOrEqual", (col: String, v: Any) => GreaterThanOrEqual(col, v), (col: String, v: Any) => Expressions.greaterThanOrEqual(col, v)) ) // Generate all combinations: orderable types x comparison operators val supportedTests = for { (col, value, typeDesc) <- orderableTypeTestCases (opName, sparkOp, icebergOp) <- comparisonOpMappings } yield ExprConvTestCase( s"$opName $typeDesc", sparkOp(col, value), // supportBoolean=false because ordering operators don't work on Boolean type Some(icebergOp(col, SparkToIcebergExpressionConverter.toIcebergValue( value, supportBoolean = false))) ) // NaN with comparison operators returns None val nanRejectionTests = Seq( ExprConvTestCase( "LessThan with NaN returns None (undefined)", // Test case label LessThan("doubleCol", Double.NaN), // Spark filter builder None // Iceberg expression builder ), ExprConvTestCase( "GreaterThan with NaN returns None (undefined)", GreaterThan("floatCol", Float.NaN), None ), ExprConvTestCase( "LessThanOrEqual with NaN returns None (undefined)", LessThanOrEqual("doubleCol", Double.NaN), None ), ExprConvTestCase( "GreaterThanOrEqual with NaN returns None (undefined)", GreaterThanOrEqual("floatCol", Float.NaN), None ) ) assertConvert(supportedTests ++ nanRejectionTests) } // ======================================================================== // NULL CHECK OPERATORS (IsNull, IsNotNull, Not(IsNull)) // ======================================================================== test("null check operators (IsNull, IsNotNull, Not(IsNull)) on all types") { val nullCheckOpMappings = Seq( ("IsNull", // Test case label (col: String, _: Any) => IsNull(col), // Spark filter builder (col: String, _: Any) => Expressions.isNull(col)), // Iceberg expression builder ("IsNotNull", (col: String, _: Any) => IsNotNull(col), (col: String, _: Any) => Expressions.notNull(col)), ("Not(IsNull)", (col: String, _: Any) => Not(IsNull(col)), (col: String, _: Any) => Expressions.notNull(col)) ) // Generate all combinations: all types x null check operators val testCases = for { (col, value, typeDesc) <- allTypesTestCases (opName, sparkOp, icebergOp) <- nullCheckOpMappings } yield ExprConvTestCase( s"$opName $typeDesc", sparkOp(col, value), Some(icebergOp(col, SparkToIcebergExpressionConverter.toIcebergValue( value, supportBoolean = true))) ) assertConvert(testCases) } // ======================================================================== // IN AND NOT IN OPERATORS // ======================================================================== // IN and NOT IN operators require special handling because: // - They accept arrays of values, requiring per-element type coercion // - Null values must be filtered out (SQL semantics: col IN (1, NULL) = col IN (1)) // - Empty arrays after null filtering result in always-false/true predicates // - Type conversion needed for each array element (Scala -> Java types) test("IN and NOT IN operators with type coercion and null handling") { // Helper to generate multiple test values for IN/NOT IN operators def generateInValues(value: Any): Array[Any] = value match { case v: Int => Array(v, v + 1, v + 2) case v: Long => Array(v, v + 1L, v + 2L) case v: Float => Array(v, v + 1.0f, v + 2.0f) case v: Double => Array(v, v + 1.0, v + 2.0) case v: String => Array(v, s"${v}_2", s"${v}_3") case v: java.math.BigDecimal => Array(v, v.add(java.math.BigDecimal.ONE), v.add(java.math.BigDecimal.TEN)) case v: Boolean => Array(v, !v) case v: java.sql.Date => Array(v, new java.sql.Date(v.getTime + 86400000L)) // +1 day in millis case v: java.sql.Timestamp => Array(v, new java.sql.Timestamp(v.getTime + 3600000L)) // +1 hour in millis case v: java.time.LocalDate => Array(v, v.plusDays(1), v.plusDays(2)) case v: java.time.LocalDateTime => Array(v, v.plusHours(1), v.plusHours(2)) case v: java.time.Instant => Array(v, v.plusSeconds(3600), v.plusSeconds(7200)) // +1 hour, +2 hours case _ => Array(value) } val inOpMappings = Seq( ("In", (col: String, values: Array[Any]) => In(col, values), (col: String, values: Array[Any]) => Expressions.in(col, values: _*)), ("Not(In)", (col: String, values: Array[Any]) => Not(In(col, values)), (col: String, values: Array[Any]) => Expressions.notIn(col, values: _*)) ) // Test IN and NOT IN operators for all types val typeTests = for { (col, value, typeDesc) <- allTypesTestCases (opName, sparkOp, icebergOp) <- inOpMappings } yield { val values = generateInValues(value) val icebergValues = values.map(v => SparkToIcebergExpressionConverter.toIcebergValue(v, supportBoolean = true)) ExprConvTestCase( s"$opName with $typeDesc", sparkOp(col, values), Some(icebergOp(col, icebergValues)) ) } // Null handling tests for both In and Not(In) val nullHandlingTests = for { (opName, sparkOp, icebergOp) <- inOpMappings } yield Seq( ExprConvTestCase( s"$opName with null values (nulls filtered)", sparkOp("stringCol", Array(null, "value1", "value2")), Some(icebergOp("stringCol", Array("value1", "value2"))) ), ExprConvTestCase( s"$opName with null and integers", sparkOp("intCol", Array(null, 1, 2)), Some(icebergOp("intCol", Array(1: Integer, 2: Integer))) ), ExprConvTestCase( s"$opName with only null", sparkOp("stringCol", Array(null)), Some(icebergOp("stringCol", Array())) ), ExprConvTestCase( s"$opName with empty array", sparkOp("intCol", Array()), Some(icebergOp("intCol", Array())) ) ) // Specific examples for both In and Not(In) val specificExamples = for { (opName, sparkOp, icebergOp) <- inOpMappings } yield Seq( ExprConvTestCase( s"$opName with string values", sparkOp("stringCol", Array("value1", "value2")), Some(icebergOp("stringCol", Array("value1", "value2"))) ), ExprConvTestCase( s"$opName with single value", sparkOp("intCol", Array(42)), Some(icebergOp("intCol", Array(42: Integer))) ), ExprConvTestCase( s"$opName with nested column", sparkOp("address.intCol", Array(1, 2, 3)), Some(icebergOp("address.intCol", Array(1: Integer, 2: Integer, 3: Integer))) ) ) assertConvert(typeTests ++ nullHandlingTests.flatten ++ specificExamples.flatten) } // ======================================================================== // STRING OPERATIONS // ======================================================================== test("string operations (startsWith/notStartsWith supported, endsWith/contains unsupported)") { val stringOpMappings = Seq( ("StringStartsWith", (col: String, prefix: String) => StringStartsWith(col, prefix), (col: String, prefix: String) => Expressions.startsWith(col, prefix)), ("Not(StringStartsWith)", (col: String, prefix: String) => Not(StringStartsWith(col, prefix)), (col: String, prefix: String) => Expressions.notStartsWith(col, prefix)) ) val stringColumns = Seq( ("stringCol", "string column"), ("metadata.stringCol", "nested string column") ) val prefixTestCases = Seq( ("prefix", "basic prefix"), ("", "empty prefix") ) // Generate all combinations: string columns x prefixes x [startsWith, notStartsWith] val supportedTests = for { (col, colDesc) <- stringColumns (prefix, prefixDesc) <- prefixTestCases (opName, sparkOp, icebergOp) <- stringOpMappings } yield ExprConvTestCase( s"$opName with $prefixDesc on $colDesc", sparkOp(col, prefix), Some(icebergOp(col, prefix)) ) // Unsupported: StringEndsWith, StringContains val unsupportedTests = Seq( ExprConvTestCase( "StringEndsWith (unsupported)", StringEndsWith("stringCol", "suffix"), None ), ExprConvTestCase( "StringContains (unsupported)", StringContains("stringCol", "substr"), None ) ) assertConvert(supportedTests ++ unsupportedTests) } // ======================================================================== // LOGICAL OPERATORS (AND, OR) // ======================================================================== test("logical operators (AND, OR) with valid and invalid combinations") { // Valid combinations: both sides convert successfully val validCombinations = Seq( ExprConvTestCase( "AND with two different types", // Test case label And( // Spark filter builder EqualTo("intCol", 42), GreaterThan("longCol", 100L) ), Some( // Iceberg expression builder Expressions.and( Expressions.equal("intCol", 42), Expressions.greaterThan("longCol", 100L)) ) ), ExprConvTestCase( "OR with two different types", Or( LessThan("doubleCol", 99.99), IsNull("stringCol") ), Some( Expressions.or( Expressions.lessThan("doubleCol", 99.99), Expressions.isNull("stringCol") ) ) ), ExprConvTestCase( "Nested logical operators", And( Or( EqualTo("intCol", 1), EqualTo("intCol", 2) ), And( GreaterThan("longCol", 0L), LessThan("longCol", 100L) ) ), Some( Expressions.and( Expressions.or( Expressions.equal("intCol", 1), Expressions.equal("intCol", 2) ), Expressions.and( Expressions.greaterThan("longCol", 0L), Expressions.lessThan("longCol", 100L) ) ) ) ), ExprConvTestCase( "Range filter: 0 < intCol < 100", And(GreaterThan("intCol", 0), LessThan("intCol", 100)), Some(Expressions.and( Expressions.greaterThan("intCol", 0), Expressions.lessThan("intCol", 100) )) ) ) // Invalid combinations: when one side fails conversion, the whole expression returns None val validFilter = EqualTo("intCol", 42) val unsupportedFilter = StringEndsWith("stringCol", "suffix") val invalidCombinations = Seq( ExprConvTestCase( "AND with unsupported right side", // Test case label And(validFilter, unsupportedFilter), // Spark filter builder None // Iceberg expression builder ), ExprConvTestCase( "AND with unsupported left side", And(unsupportedFilter, validFilter), None ), ExprConvTestCase( "OR with unsupported right side", Or(validFilter, unsupportedFilter), None ), ExprConvTestCase( "OR with unsupported left side", Or(unsupportedFilter, validFilter), None ), ExprConvTestCase( "Nested AND with unsupported in OR", And( validFilter, Or( validFilter, unsupportedFilter ) ), None ) ) assertConvert(validCombinations ++ invalidCombinations) } // ======================================================================== // NOT OPERATOR (unsupported cases) // ======================================================================== test("NOT operator with unsupported inner filters") { // Note: Supported NOT patterns are tested in their respective operator pair tests: // - Not(EqualTo) tested in equality operators // - Not(In) tested in IN and NOT IN operators // - Not(IsNull) tested in null check operators // - Not(StringStartsWith) tested in string operations // // This test only covers unsupported NOT patterns val testCases = Seq( ExprConvTestCase( "Not(LessThan) is unsupported", Not(LessThan("intCol", 5)), None ), ExprConvTestCase( "Not(GreaterThan) is unsupported", Not(GreaterThan("longCol", 100L)), None ), ExprConvTestCase( "Not(And) is unsupported", Not(And(EqualTo("intCol", 1), EqualTo("longCol", 2L))), None ) ) assertConvert(testCases) } // ======================================================================== // BOOLEAN LITERALS // ======================================================================== test("boolean literals (AlwaysTrue, AlwaysFalse)") { val testCases = Seq( ExprConvTestCase( "AlwaysTrue", // Test case label AlwaysTrue(), // Spark filter builder Some(Expressions.alwaysTrue()) // Iceberg expression builder ), ExprConvTestCase( "AlwaysFalse", AlwaysFalse(), Some(Expressions.alwaysFalse()) ) ) assertConvert(testCases) } // ======================================================================== // TYPE CONVERSIONS AND BOUNDARY VALUES // ======================================================================== test("type conversions (Date/Timestamp) and boundary values") { val testDate = java.sql.Date.valueOf("2024-01-01") val expectedDateDays = (testDate.getTime / (1000L * 60 * 60 * 24)).toInt val testTimestamp = java.sql.Timestamp.valueOf("2024-01-01 00:00:00") val expectedTimestampMicros = testTimestamp.getTime * 1000 + (testTimestamp.getNanos % 1000000) / 1000 // java.time types val testLocalDate = java.time.LocalDate.of(2024, 1, 1) val expectedLocalDateDays = testLocalDate.toEpochDay.toInt val testLocalDateTime = java.time.LocalDateTime.of(2024, 1, 1, 12, 30, 45) val expectedLocalDateTimeMicros = testLocalDateTime.toEpochSecond( java.time.ZoneOffset.UTC) * 1000000 + testLocalDateTime.getNano / 1000 val testInstant = java.time.Instant.parse("2024-01-01T12:30:45.123456Z") val expectedInstantMicros = testInstant.getEpochSecond * 1000000 + testInstant.getNano / 1000 val testCases = Seq( // Date/Timestamp: Spark sends java.sql types, but we convert to Int/Long for Iceberg ExprConvTestCase( "Date converted to days since epoch", // Test case label EqualTo("dateCol", testDate), // Spark filter builder Some(Expressions.equal("dateCol", expectedDateDays: Integer)) // Iceberg expression builder ), ExprConvTestCase( "Timestamp converted to microseconds since epoch", EqualTo("timestampCol", testTimestamp), Some(Expressions.equal("timestampCol", expectedTimestampMicros: java.lang.Long)) ), // java.time types: converted to Int (days) or Long (microseconds) ExprConvTestCase( "LocalDate converted to days since epoch", EqualTo("localDateCol", testLocalDate), Some(Expressions.equal("localDateCol", expectedLocalDateDays: Integer)) ), ExprConvTestCase( "LocalDateTime converted to microseconds since epoch", EqualTo("localDateTimeCol", testLocalDateTime), Some(Expressions.equal("localDateTimeCol", expectedLocalDateTimeMicros: java.lang.Long)) ), ExprConvTestCase( "Instant converted to microseconds since epoch", EqualTo("instantCol", testInstant), Some(Expressions.equal("instantCol", expectedInstantMicros: java.lang.Long)) ), // Boundary values ExprConvTestCase( "Int.MinValue boundary", // Test case label EqualTo("intCol", Int.MinValue), // Spark filter builder Some(Expressions.equal("intCol", Int.MinValue)) // Iceberg expression builder ), ExprConvTestCase( "Int.MaxValue boundary", EqualTo("intCol", Int.MaxValue), Some(Expressions.equal("intCol", Int.MaxValue)) ), ExprConvTestCase( "Long.MinValue boundary", EqualTo("longCol", Long.MinValue), Some(Expressions.equal("longCol", Long.MinValue)) ), ExprConvTestCase( "Long.MaxValue boundary", EqualTo("longCol", Long.MaxValue), Some(Expressions.equal("longCol", Long.MaxValue)) ) ) assertConvert(testCases) } // ======================================================================== // UNSUPPORTED FILTERS // ======================================================================== test("unsupported filters return None") { // This test ensures that all known unsupported Spark Filter types return None // If Spark adds new filter types, our converter will skip them via case _ => None val testCases = Seq( // EqualNullSafe - Iceberg doesn't have null-safe equality ExprConvTestCase( "EqualNullSafe", // Test case label EqualNullSafe("intCol", 5), // Spark filter builder None // Iceberg expression builder ), // StringEndsWith - Iceberg API doesn't provide this predicate ExprConvTestCase( "StringEndsWith", StringEndsWith("stringCol", "suffix"), None ), // StringContains - Iceberg API doesn't provide this predicate ExprConvTestCase( "StringContains", StringContains("stringCol", "substring"), None ), // Not with non-EqualTo inner filter - Iceberg doesn't support arbitrary NOT // Only Not(EqualTo) is converted as a special case ExprConvTestCase( "Not(LessThan) - arbitrary NOT unsupported", Not(LessThan("intCol", 10)), None ), ExprConvTestCase( "Not(GreaterThan) - arbitrary NOT unsupported", Not(GreaterThan("intCol", 10)), None ), ExprConvTestCase( "Not(And) - arbitrary NOT unsupported", Not(And(EqualTo("intCol", 1), EqualTo("longCol", 2L))), None ) ) assertConvert(testCases) } // ======================================================================== // UNSUPPORTED VALUE TYPES // ======================================================================== test("filters with unsupported value types return None") { // Define unsupported types for which conversion must fail val unsupportedTypes = Seq( (Array(1, 2, 3), "Array"), (Map("key" -> 1), "Map"), (Row(1, "test"), "Row/Struct"), (Array[Byte](1, 2, 3), "byte array"), (5.toByte, "Byte"), (5.toShort, "Short") ) // Define operators that must reject unsupported types val operators = Seq( ("EqualTo", (col: String, v: Any) => EqualTo(col, v)), ("LessThan", (col: String, v: Any) => LessThan(col, v)), ("GreaterThan", (col: String, v: Any) => GreaterThan(col, v)), ("LessThanOrEqual", (col: String, v: Any) => LessThanOrEqual(col, v)), ("GreaterThanOrEqual", (col: String, v: Any) => GreaterThanOrEqual(col, v)) ) // Generate all combinations: unsupported types x operators val operatorTests = for { (value, typeDesc) <- unsupportedTypes (opName, sparkOp) <- operators } yield ExprConvTestCase( s"$opName with $typeDesc should be unsupported", // Test case label sparkOp("intCol", value), // Spark filter builder None // Iceberg expression builder ) // Boolean with comparison operators (supportBoolean=false for these) val booleanComparisonTests = Seq( ("LessThan", (col: String, v: Any) => LessThan(col, v)), ("GreaterThan", (col: String, v: Any) => GreaterThan(col, v)), ("LessThanOrEqual", (col: String, v: Any) => LessThanOrEqual(col, v)), ("GreaterThanOrEqual", (col: String, v: Any) => GreaterThanOrEqual(col, v)) ).map { case (opName, sparkOp) => ExprConvTestCase( s"$opName with Boolean (unsupported for comparison)", sparkOp("boolCol", true), None ) } // Special case: In with nested Array values val inWithNestedArrays = ExprConvTestCase( "In with nested Array values", In("intCol", Array(Array(1), Array(2))), None ) assertConvert(operatorTests ++ booleanComparisonTests ++ Seq(inWithNestedArrays)) } } ================================================ FILE: iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/TestSchemas.scala ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.serverSidePlanning import shadedForDelta.org.apache.iceberg.Schema import shadedForDelta.org.apache.iceberg.types.Types import org.apache.spark.sql.types._ private[serverSidePlanning] object TestSchemas { /** * Shared test schema used across all server-side planning test suites. * Structure: * - Flat fields (12 types): intCol, longCol, doubleCol, floatCol, stringCol, boolCol, * decimalCol, dateCol, timestampCol, localDateCol, * localDateTimeCol, instantCol * - Nested struct (ID 13): address with intCol - tests nested field access * - Nested struct (ID 14): metadata with stringCol - tests nested string field * - Nested struct with dotted field (ID 15): parent with "child.name" - tests escaping at * nested level * - Literal top-level dotted columns (IDs 16-17): address.city, a.b.c - tests top-level * escaping */ val testSchema = new Schema( // Flat fields (IDs 1-12) Types.NestedField.required(1, "intCol", Types.IntegerType.get), Types.NestedField.required(2, "longCol", Types.LongType.get), Types.NestedField.required(3, "doubleCol", Types.DoubleType.get), Types.NestedField.required(4, "floatCol", Types.FloatType.get), Types.NestedField.required(5, "stringCol", Types.StringType.get), Types.NestedField.required(6, "boolCol", Types.BooleanType.get), Types.NestedField.required(7, "decimalCol", Types.DecimalType.of(10, 2)), Types.NestedField.required(8, "dateCol", Types.DateType.get), Types.NestedField.required(9, "timestampCol", Types.TimestampType.withoutZone), Types.NestedField.required(10, "localDateCol", Types.DateType.get), Types.NestedField.required(11, "localDateTimeCol", Types.TimestampType.withoutZone), Types.NestedField.required(12, "instantCol", Types.TimestampType.withZone), // Nested struct for testing nested field access (ID 13) Types.NestedField.required(13, "address", Types.StructType.of( Types.NestedField.required(101, "intCol", Types.IntegerType.get) )), // Nested struct for testing nested string field (ID 14) Types.NestedField.required(14, "metadata", Types.StructType.of( Types.NestedField.required(111, "stringCol", Types.StringType.get) )), // Nested struct with field that has dots in its name (ID 15) // Tests escaping at nested level: parent.`child.name` Types.NestedField.required(15, "parent", Types.StructType.of( Types.NestedField.required(121, "`child.name`", Types.StringType.get) )), // Literal top-level column names with dots (IDs 16-17) - Test escaping Types.NestedField.required(16, "`address.city`", Types.StringType.get), Types.NestedField.required(17, "`a.b.c`", Types.StringType.get) ) /** * Spark StructType corresponding to the testSchema above. * Used for filter conversion in tests. */ val sparkSchema: StructType = StructType(Seq( StructField("intCol", IntegerType, nullable = false), StructField("longCol", LongType, nullable = false), StructField("doubleCol", DoubleType, nullable = false), StructField("floatCol", FloatType, nullable = false), StructField("stringCol", StringType, nullable = false), StructField("boolCol", BooleanType, nullable = false), StructField("decimalCol", DecimalType(10, 2), nullable = false), StructField("dateCol", DateType, nullable = false), StructField("timestampCol", TimestampType, nullable = false), StructField("localDateCol", DateType, nullable = false), StructField("localDateTimeCol", TimestampType, nullable = false), StructField("instantCol", TimestampType, nullable = false), // Nested struct for testing nested field access StructField("address", StructType(Seq( StructField("intCol", IntegerType, nullable = false) )), nullable = false), // Nested struct for testing nested string field StructField("metadata", StructType(Seq( StructField("stringCol", StringType, nullable = false) )), nullable = false), // Nested struct with field that has dots in its name // Tests escaping at nested level: parent.`child.name` StructField("parent", StructType(Seq( StructField("`child.name`", StringType, nullable = false) )), nullable = false), // Literal top-level column names with dots - Test escaping StructField("`address.city`", StringType, nullable = false), StructField("`a.b.c`", StringType, nullable = false) )) } ================================================ FILE: iceberg/src/test/scala/org/apache/spark/sql/delta/uniform/IcebergCompatV2EnableUniformByAlterTableSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.uniform import java.util.UUID import org.apache.spark.sql.delta.{ColumnMappingTableFeature, DeltaLog, IcebergCompatV2TableFeature, UniversalFormat} import org.apache.spark.sql.delta.uniform.IcebergCompatV2EnableUniformByAlterTableSuiteBase import org.apache.hadoop.fs.Path import org.apache.spark.sql.DataFrame import org.apache.spark.sql.catalyst.TableIdentifier class IcebergCompatV2EnableUniformByAlterTableSuite extends IcebergCompatV2EnableUniformByAlterTableSuiteBase with WriteDeltaHMSReadIceberg { override def withTempTableAndDir(f: (String, String) => Unit): Unit = { val tableId = s"testTable${UUID.randomUUID()}".replace("-", "_") withTempDir { dir => val tablePath = new Path(dir.toString, "table") withTable(tableId) { f(tableId, s"'$tablePath'") } } } override def executeSql(sqlStr: String): DataFrame = write(sqlStr) override def assertUniFormIcebergProtocolAndProperties(id: String): Unit = { val snapshot = DeltaLog.forTable(spark, new TableIdentifier(id)).update() val protocol = snapshot.protocol val tblProperties = snapshot.getProperties val tableFeature = IcebergCompatV2TableFeature val expectedMinReaderVersion = Math.max( ColumnMappingTableFeature.minReaderVersion, tableFeature.minReaderVersion ) val expectedMinWriterVersion = Math.max( ColumnMappingTableFeature.minWriterVersion, tableFeature.minWriterVersion ) assert(protocol.minReaderVersion >= expectedMinReaderVersion) assert(protocol.minWriterVersion >= expectedMinWriterVersion) assert(protocol.writerFeatures.get.contains(tableFeature.name)) assert(tblProperties(s"delta.enableIcebergCompatV2") === "true") assert(Seq("name", "id").contains(tblProperties("delta.columnMapping.mode"))) assert(UniversalFormat.icebergEnabled(snapshot.metadata)) } } ================================================ FILE: iceberg/src/test/scala/org/apache/spark/sql/delta/uniform/TypeWideningUniformSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.uniform import org.apache.spark.sql.delta.typewidening.TypeWideningUniformTests /** * Suite running Uniform Iceberg + type widening tests against HMS. */ class TypeWideningUniformSuite extends TypeWideningUniformTests with WriteDeltaHMSReadIceberg ================================================ FILE: iceberg/src/test/scala/org/apache/spark/sql/delta/uniform/UniFormConverterSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.uniform import shadedForDelta.org.apache.iceberg.hadoop.HadoopTables import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.delta.{DeltaLog, Snapshot} import org.apache.spark.sql.delta.icebergShaded.{IcebergConverter, UNIFORM_CC_MODE} import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.test.SharedSparkSession class IcebergConverterForTest extends IcebergConverter { def convertSnapshotAndReturnMetadataPath( snapshotToConvert: Snapshot, catalogTable: CatalogTable): String = { val icebergTxn = convertSnapshotInternal( snapshotToConvert, readSnapshotOpt = None, lastConvertedInfo = LastConvertedIcebergInfo( icebergTable = None, icebergSnapshotId = None, deltaVersionConverted = None, baseMetadataLocationOpt = None ), conversionContext = new ConversionContext( conversionMode = UNIFORM_CC_MODE, additionalDeltaActionsToCommit = None, opType = "delta.iceberg.conversion.convertSnapshot" ), catalogTable ) icebergTxn.getConvertedIcebergMetadata._1 } } class UniFormConverterSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { test("convertSnapshot writes Iceberg metadata and file count matches Delta snapshot") { val tableName = "test_iceberg_converter" withTable(tableName) { spark.sql( s"""CREATE TABLE $tableName (id INT, name STRING) USING DELTA |TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name', | 'delta.enableIcebergCompatV2' = 'true', | 'delta.universalFormat.enabledFormats' = 'iceberg' |)""".stripMargin) // Do write spark.sql(s"INSERT INTO $tableName VALUES (1, 'alice'), (2, 'bob'), (3, 'carol')") val tableId = TableIdentifier(tableName) val deltaLog = DeltaLog.forTable(spark, tableId) val snapshot = deltaLog.update() val catalogTable = spark.sessionState.catalog.getTableMetadata(tableId) // Trigger conversion val converter = new IcebergConverterForTest() val metadataPath = converter.convertSnapshotAndReturnMetadataPath(snapshot, catalogTable) // Check match val icebergTable = new HadoopTables(deltaLog.newDeltaHadoopConf()).load(metadataPath) val numFilesInIceberg = icebergTable.currentSnapshot().summary().get("total-data-files").toInt assert( numFilesInIceberg == snapshot.numOfFiles, s"Iceberg total-data-files ($numFilesInIceberg) must equal " + s"Delta numOfFiles (${snapshot.numOfFiles})") } } test("convertSnapshot file count matches Delta snapshot after multiple inserts") { val tableName = "test_iceberg_converter_multi" withTable(tableName) { spark.sql( s"""CREATE TABLE $tableName (id INT) USING DELTA |TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name', | 'delta.enableIcebergCompatV2' = 'true', | 'delta.universalFormat.enabledFormats' = 'iceberg' |)""".stripMargin) // Do some writes spark.sql(s"INSERT INTO $tableName VALUES (1)") spark.sql(s"INSERT INTO $tableName VALUES (2)") spark.sql(s"INSERT INTO $tableName VALUES (3)") val tableId = TableIdentifier(tableName) val deltaLog = DeltaLog.forTable(spark, tableId) val snapshot = deltaLog.update() val catalogTable = spark.sessionState.catalog.getTableMetadata(tableId) assert(snapshot.numOfFiles == 3) // Trigger conversion val converter = new IcebergConverterForTest() val metadataPath = converter.convertSnapshotAndReturnMetadataPath(snapshot, catalogTable) // Check match val icebergTable = new HadoopTables(deltaLog.newDeltaHadoopConf()).load(metadataPath) val numFilesInIceberg = icebergTable.currentSnapshot().summary().get("total-data-files").toInt assert( numFilesInIceberg == snapshot.numOfFiles, s"Iceberg total-data-files ($numFilesInIceberg) must equal " + s"Delta numOfFiles (${snapshot.numOfFiles})") } } } ================================================ FILE: iceberg/src/test/scala/org/apache/spark/sql/delta/uniform/UniFormE2EIcebergSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.uniform import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.{SparkConf, SparkSessionSwitch} import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.uniform.hms.HMSTest /** * This trait allows the tests to write with Delta * using a in-memory HiveMetaStore as catalog, * and read from the same HiveMetaStore with Iceberg. */ trait WriteDeltaHMSReadIceberg extends UniFormE2ETest with DeltaSQLCommandTest with HMSTest with SparkSessionSwitch { override protected def sparkConf: SparkConf = setupSparkConfWithHMS(super.sparkConf) .set(DeltaSQLConf.DELTA_UNIFORM_ICEBERG_SYNC_CONVERT_ENABLED.key, "true") .set(DeltaSQLConf.DELTA_OPTIMIZE_WRITE_ENABLED.key, "true") .set(DeltaSQLConf.DELTA_AUTO_COMPACT_ENABLED.key, "true") private var _readerSparkSession: Option[SparkSession] = None /** * Verify the result by reading from the reader session and compare the result to the expected. * * @param table write table name * @param fields fields to verify, separated by comma. E.g., "col1, col2" * @param orderBy fields to order the results, separated by comma. * @param expect expected result */ protected override def readAndVerify( table: String, fields: String, orderBy: String, expect: Seq[Row]): Unit = { val translated = tableNameForRead(table) withSession(readerSparkSession) { session => checkAnswer(session.sql(s"SELECT $fields FROM $translated ORDER BY $orderBy"), expect) } } protected def readerSparkSession: SparkSession = { if (_readerSparkSession.isEmpty) { // call to newSession makes sure // [[SparkSession.getOrCreate]] gives a new session // and [[SparkContext.getOrCreate]] uses a new context _readerSparkSession = Some(newSession(createIcebergSparkSession)) } _readerSparkSession.get } } /** * No test should go here. Please add tests in [[UniFormE2EIcebergSuiteBase]] */ ================================================ FILE: iceberg/src/test/scala/org/apache/spark/sql/delta/uniform/UniversalFormatSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.uniform import java.util.UUID import org.apache.spark.sql.delta.{ DeltaLog, IcebergCompatUtilsBase, UniFormWithIcebergCompatV1SuiteBase, UniFormWithIcebergCompatV2SuiteBase, UniversalFormatMiscSuiteBase, UniversalFormatSuiteBase} import org.apache.spark.sql.delta.commands.DeltaReorgTableCommand import org.apache.spark.sql.delta.icebergShaded.IcebergTransactionUtils import org.apache.hadoop.fs.Path import org.apache.spark.sql.{DataFrame, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.parser.ParseException /** Contains shared utils for both IcebergCompatV1, IcebergCompatV2 and MISC suites. */ trait UniversalFormatSuiteUtilsBase extends IcebergCompatUtilsBase with WriteDeltaHMSReadIceberg { override def withTempTableAndDir(f: (String, String) => Unit): Unit = { val tableId = s"testTable${UUID.randomUUID()}".replace("-", "_") withTempDir { dir => val tablePath = new Path(dir.toString, "table") withTable(tableId) { f(tableId, s"'$tablePath'") } } } override def executeSql(sqlStr: String): DataFrame = write(sqlStr) override protected val allReaderWriterVersions: Seq[(Int, Int)] = (1 to 3) .flatMap { r => (1 to 7).filter(_ != 6).map(w => (r, w)) } // can only be at minReaderVersion >= 3 if minWriterVersion is >= 7 .filterNot { case (r, w) => w < 7 && r >= 3 } } class UniversalFormatSuite extends UniversalFormatMiscSuiteBase with UniversalFormatSuiteUtilsBase class UniFormWithIcebergCompatV1Suite extends UniversalFormatSuiteUtilsBase with UniFormWithIcebergCompatV1SuiteBase class UniFormWithIcebergCompatV2Suite extends UniversalFormatSuiteUtilsBase with UniFormWithIcebergCompatV2SuiteBase ================================================ FILE: icebergShaded/src/main/java/org/apache/iceberg/MetadataUpdate.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.iceberg; import java.io.Serializable; import java.util.Locale; import java.util.Map; import java.util.Set; import org.apache.iceberg.encryption.EncryptedKey; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.view.ViewMetadata; import org.apache.iceberg.view.ViewVersion; /** Represents a change to table or view metadata. */ /** * This class is directly copied from iceberg repo 1.10.0 with following changes * Changes: L91 Added back the deprecated API in 1.9.0 * public AddSchema(Schema schema, int lastColumnId) */ public interface MetadataUpdate extends Serializable { default void applyTo(TableMetadata.Builder metadataBuilder) { throw new UnsupportedOperationException( String.format("Cannot apply update %s to a table", this.getClass().getSimpleName())); } default void applyTo(ViewMetadata.Builder viewMetadataBuilder) { throw new UnsupportedOperationException( String.format("Cannot apply update %s to a view", this.getClass().getSimpleName())); } class AssignUUID implements MetadataUpdate { private final String uuid; public AssignUUID(String uuid) { this.uuid = uuid; } public String uuid() { return uuid; } @Override public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.assignUUID(uuid); } @Override public void applyTo(ViewMetadata.Builder metadataBuilder) { metadataBuilder.assignUUID(uuid); } } class UpgradeFormatVersion implements MetadataUpdate { private final int formatVersion; public UpgradeFormatVersion(int formatVersion) { this.formatVersion = formatVersion; } public int formatVersion() { return formatVersion; } @Override public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.upgradeFormatVersion(formatVersion); } @Override public void applyTo(ViewMetadata.Builder viewMetadataBuilder) { viewMetadataBuilder.upgradeFormatVersion(formatVersion); } } class AddSchema implements MetadataUpdate { private final Schema schema; private final int lastColumnId; public AddSchema(Schema schema) { this(schema, schema.highestFieldId()); } /** * HACK-HACK This is added * Set the schema * @deprecated in 1.9.0 */ @Deprecated public AddSchema(Schema schema, int lastColumnId) { this.schema = schema; this.lastColumnId = lastColumnId; } public Schema schema() { return schema; } // HACK-HACK This is modified public int lastColumnId() { return lastColumnId; } // HACK-HACK This is modified @Override public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.addSchema(schema, lastColumnId); } @Override public void applyTo(ViewMetadata.Builder viewMetadataBuilder) { viewMetadataBuilder.addSchema(schema); } } class SetCurrentSchema implements MetadataUpdate { private final int schemaId; public SetCurrentSchema(int schemaId) { this.schemaId = schemaId; } public int schemaId() { return schemaId; } @Override public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.setCurrentSchema(schemaId); } } class AddPartitionSpec implements MetadataUpdate { private final UnboundPartitionSpec spec; public AddPartitionSpec(PartitionSpec spec) { this(spec.toUnbound()); } public AddPartitionSpec(UnboundPartitionSpec spec) { this.spec = spec; } public UnboundPartitionSpec spec() { return spec; } @Override public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.addPartitionSpec(spec); } } class SetDefaultPartitionSpec implements MetadataUpdate { private final int specId; public SetDefaultPartitionSpec(int specId) { this.specId = specId; } public int specId() { return specId; } @Override public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.setDefaultPartitionSpec(specId); } } class RemovePartitionSpecs implements MetadataUpdate { private final Set specIds; public RemovePartitionSpecs(Set specIds) { this.specIds = specIds; } public Set specIds() { return specIds; } @Override public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.removeSpecs(specIds); } } class RemoveSchemas implements MetadataUpdate { private final Set schemaIds; public RemoveSchemas(Set schemaIds) { this.schemaIds = schemaIds; } public Set schemaIds() { return schemaIds; } @Override public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.removeSchemas(schemaIds); } } class AddSortOrder implements MetadataUpdate { private final UnboundSortOrder sortOrder; public AddSortOrder(SortOrder sortOrder) { this(sortOrder.toUnbound()); } public AddSortOrder(UnboundSortOrder sortOrder) { this.sortOrder = sortOrder; } public UnboundSortOrder sortOrder() { return sortOrder; } @Override public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.addSortOrder(sortOrder); } } class SetDefaultSortOrder implements MetadataUpdate { private final int sortOrderId; public SetDefaultSortOrder(int sortOrderId) { this.sortOrderId = sortOrderId; } public int sortOrderId() { return sortOrderId; } @Override public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.setDefaultSortOrder(sortOrderId); } } class SetStatistics implements MetadataUpdate { private final StatisticsFile statisticsFile; public SetStatistics(StatisticsFile statisticsFile) { this.statisticsFile = statisticsFile; } public long snapshotId() { return statisticsFile.snapshotId(); } public StatisticsFile statisticsFile() { return statisticsFile; } @Override public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.setStatistics(statisticsFile); } } class RemoveStatistics implements MetadataUpdate { private final long snapshotId; public RemoveStatistics(long snapshotId) { this.snapshotId = snapshotId; } public long snapshotId() { return snapshotId; } @Override public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.removeStatistics(snapshotId); } } class SetPartitionStatistics implements MetadataUpdate { private final PartitionStatisticsFile partitionStatisticsFile; public SetPartitionStatistics(PartitionStatisticsFile partitionStatisticsFile) { this.partitionStatisticsFile = partitionStatisticsFile; } public long snapshotId() { return partitionStatisticsFile.snapshotId(); } public PartitionStatisticsFile partitionStatisticsFile() { return partitionStatisticsFile; } @Override public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.setPartitionStatistics(partitionStatisticsFile); } } class RemovePartitionStatistics implements MetadataUpdate { private final long snapshotId; public RemovePartitionStatistics(long snapshotId) { this.snapshotId = snapshotId; } public long snapshotId() { return snapshotId; } @Override public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.removePartitionStatistics(snapshotId); } } class AddSnapshot implements MetadataUpdate { private final Snapshot snapshot; public AddSnapshot(Snapshot snapshot) { this.snapshot = snapshot; } public Snapshot snapshot() { return snapshot; } @Override public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.addSnapshot(snapshot); } } class RemoveSnapshots implements MetadataUpdate { private final Set snapshotIds; public RemoveSnapshots(long snapshotId) { this.snapshotIds = ImmutableSet.of(snapshotId); } public RemoveSnapshots(Set snapshotIds) { this.snapshotIds = snapshotIds; } public Set snapshotIds() { return snapshotIds; } @Override public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.removeSnapshots(snapshotIds); } } class RemoveSnapshotRef implements MetadataUpdate { private final String refName; public RemoveSnapshotRef(String refName) { this.refName = refName; } public String name() { return refName; } @Override public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.removeRef(refName); } } class SetSnapshotRef implements MetadataUpdate { private final String refName; private final Long snapshotId; private final SnapshotRefType type; private final Integer minSnapshotsToKeep; private final Long maxSnapshotAgeMs; private final Long maxRefAgeMs; public SetSnapshotRef( String refName, Long snapshotId, SnapshotRefType type, Integer minSnapshotsToKeep, Long maxSnapshotAgeMs, Long maxRefAgeMs) { this.refName = refName; this.snapshotId = snapshotId; this.type = type; this.minSnapshotsToKeep = minSnapshotsToKeep; this.maxSnapshotAgeMs = maxSnapshotAgeMs; this.maxRefAgeMs = maxRefAgeMs; } public String name() { return refName; } public String type() { return type.name().toLowerCase(Locale.ROOT); } public long snapshotId() { return snapshotId; } public Integer minSnapshotsToKeep() { return minSnapshotsToKeep; } public Long maxSnapshotAgeMs() { return maxSnapshotAgeMs; } public Long maxRefAgeMs() { return maxRefAgeMs; } @Override public void applyTo(TableMetadata.Builder metadataBuilder) { SnapshotRef ref = SnapshotRef.builderFor(snapshotId, type) .minSnapshotsToKeep(minSnapshotsToKeep) .maxSnapshotAgeMs(maxSnapshotAgeMs) .maxRefAgeMs(maxRefAgeMs) .build(); metadataBuilder.setRef(refName, ref); } } class SetProperties implements MetadataUpdate { private final Map updated; public SetProperties(Map updated) { this.updated = updated; } public Map updated() { return updated; } @Override public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.setProperties(updated); } @Override public void applyTo(ViewMetadata.Builder viewMetadataBuilder) { viewMetadataBuilder.setProperties(updated); } } class RemoveProperties implements MetadataUpdate { private final Set removed; public RemoveProperties(Set removed) { this.removed = removed; } public Set removed() { return removed; } @Override public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.removeProperties(removed); } @Override public void applyTo(ViewMetadata.Builder viewMetadataBuilder) { viewMetadataBuilder.removeProperties(removed); } } class SetLocation implements MetadataUpdate { private final String location; public SetLocation(String location) { this.location = location; } public String location() { return location; } @Override public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.setLocation(location); } @Override public void applyTo(ViewMetadata.Builder viewMetadataBuilder) { viewMetadataBuilder.setLocation(location); } } class AddViewVersion implements MetadataUpdate { private final ViewVersion viewVersion; public AddViewVersion(ViewVersion viewVersion) { this.viewVersion = viewVersion; } public ViewVersion viewVersion() { return viewVersion; } @Override public void applyTo(ViewMetadata.Builder viewMetadataBuilder) { viewMetadataBuilder.addVersion(viewVersion); } } class SetCurrentViewVersion implements MetadataUpdate { private final int versionId; public SetCurrentViewVersion(int versionId) { this.versionId = versionId; } public int versionId() { return versionId; } @Override public void applyTo(ViewMetadata.Builder viewMetadataBuilder) { viewMetadataBuilder.setCurrentVersionId(versionId); } } class AddEncryptionKey implements MetadataUpdate { private final EncryptedKey key; public AddEncryptionKey(EncryptedKey key) { this.key = key; } public EncryptedKey key() { return key; } @Override public void applyTo(TableMetadata.Builder builder) { builder.addEncryptionKey(key); } } class RemoveEncryptionKey implements MetadataUpdate { private final String keyId; public RemoveEncryptionKey(String keyId) { this.keyId = keyId; } public String keyId() { return keyId; } @Override public void applyTo(TableMetadata.Builder builder) { builder.removeEncryptionKey(keyId); } } } ================================================ FILE: icebergShaded/src/main/java/org/apache/iceberg/PartitionSpec.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.iceberg; import java.io.Serializable; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.AbstractMap; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ListMultimap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Multimaps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.transforms.UnknownTransform; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.StructType; /** * Represents how to produce partition data for a table. * *

Partition data is produced by transforming columns in a table. Each column transform is * represented by a named {@link PartitionField}. * * This class is directly copied from iceberg repo 1.10.0; The only change is this sets checkConflicts * to false by default for partition spec converted from Delta to honor the field id assigned by Delta */ public class PartitionSpec implements Serializable { // IDs for partition fields start at 1000 private static final int PARTITION_DATA_ID_START = 1000; private final Schema schema; // this is ordered so that DataFile has a consistent schema private final int specId; private final PartitionField[] fields; private transient volatile ListMultimap fieldsBySourceId = null; private transient volatile Class[] lazyJavaClasses = null; private transient volatile StructType lazyPartitionType = null; private transient volatile StructType lazyRawPartitionType = null; private transient volatile List fieldList = null; private final int lastAssignedFieldId; private PartitionSpec( Schema schema, int specId, List fields, int lastAssignedFieldId) { this.schema = schema; this.specId = specId; this.fields = fields.toArray(new PartitionField[0]); this.lastAssignedFieldId = lastAssignedFieldId; } /** Returns the {@link Schema} for this spec. */ public Schema schema() { return schema; } /** Returns the ID of this spec. */ public int specId() { return specId; } /** Returns the list of {@link PartitionField partition fields} for this spec. */ public List fields() { return lazyFieldList(); } public boolean isPartitioned() { return fields.length > 0 && fields().stream().anyMatch(f -> !f.transform().isVoid()); } public boolean isUnpartitioned() { return !isPartitioned(); } int lastAssignedFieldId() { return lastAssignedFieldId; } public UnboundPartitionSpec toUnbound() { UnboundPartitionSpec.Builder builder = UnboundPartitionSpec.builder().withSpecId(specId); for (PartitionField field : fields) { builder.addField( field.transform().toString(), field.sourceId(), field.fieldId(), field.name()); } return builder.build(); } /** * Returns the {@link PartitionField field} that partitions the given source field * * @param fieldId a field id from the source schema * @return the {@link PartitionField field} that partitions the given source field */ public List getFieldsBySourceId(int fieldId) { return lazyFieldsBySourceId().get(fieldId); } /** Returns a {@link StructType} for partition data defined by this spec. */ public StructType partitionType() { if (lazyPartitionType == null) { synchronized (this) { if (lazyPartitionType == null) { List structFields = Lists.newArrayListWithExpectedSize(fields.length); for (PartitionField field : fields) { Type sourceType = schema.findType(field.sourceId()); Type resultType = field.transform().getResultType(sourceType); // When the source field has been dropped we cannot determine the type if (sourceType == null) { resultType = Types.UnknownType.get(); } structFields.add(Types.NestedField.optional(field.fieldId(), field.name(), resultType)); } this.lazyPartitionType = Types.StructType.of(structFields); } } } return lazyPartitionType; } /** * Returns a struct matching partition information as written into manifest files. See {@link * #partitionType()} for a struct with field ID's potentially re-assigned to avoid conflict. */ public StructType rawPartitionType() { if (schema.idsToOriginal().isEmpty()) { // not re-assigned. return partitionType(); } if (lazyRawPartitionType == null) { synchronized (this) { if (lazyRawPartitionType == null) { this.lazyRawPartitionType = StructType.of( partitionType().fields().stream() .map(f -> f.withFieldId(schema.idsToOriginal().get(f.fieldId()))) .collect(Collectors.toList())); } } } return lazyRawPartitionType; } public Class[] javaClasses() { if (lazyJavaClasses == null) { synchronized (this) { if (lazyJavaClasses == null) { Class[] classes = new Class[fields.length]; for (int i = 0; i < fields.length; i += 1) { PartitionField field = fields[i]; if (field.transform() instanceof UnknownTransform) { classes[i] = Object.class; } else { Type sourceType = schema.findType(field.sourceId()); Type result = field.transform().getResultType(sourceType); classes[i] = result.typeId().javaClass(); } } this.lazyJavaClasses = classes; } } } return lazyJavaClasses; } @SuppressWarnings("unchecked") private T get(StructLike data, int pos, Class javaClass) { return data.get(pos, (Class) javaClass); } private String escape(String string) { try { return URLEncoder.encode(string, "UTF-8"); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } } public String partitionToPath(StructLike data) { StringBuilder sb = new StringBuilder(); Class[] javaClasses = javaClasses(); List outputFields = partitionType().fields(); for (int i = 0; i < javaClasses.length; i += 1) { PartitionField field = fields[i]; Type type = outputFields.get(i).type(); String valueString = field.transform().toHumanString(type, get(data, i, javaClasses[i])); if (i > 0) { sb.append("/"); } sb.append(escape(field.name())).append("=").append(escape(valueString)); } return sb.toString(); } /** * Returns true if this spec is equivalent to the other, with partition field ids ignored. That * is, if both specs have the same number of fields, field order, field name, source columns, and * transforms. * * @param other another PartitionSpec * @return true if the specs have the same fields, source columns, and transforms. */ public boolean compatibleWith(PartitionSpec other) { if (equals(other)) { return true; } if (fields.length != other.fields.length) { return false; } for (int i = 0; i < fields.length; i += 1) { PartitionField thisField = fields[i]; PartitionField thatField = other.fields[i]; if (thisField.sourceId() != thatField.sourceId() || !thisField.transform().toString().equals(thatField.transform().toString()) || !thisField.name().equals(thatField.name())) { return false; } } return true; } @Override public boolean equals(Object other) { if (this == other) { return true; } else if (!(other instanceof PartitionSpec)) { return false; } PartitionSpec that = (PartitionSpec) other; if (this.specId != that.specId) { return false; } return Arrays.equals(fields, that.fields); } @Override public int hashCode() { return 31 * Integer.hashCode(specId) + Arrays.hashCode(fields); } private List lazyFieldList() { if (fieldList == null) { synchronized (this) { if (fieldList == null) { this.fieldList = ImmutableList.copyOf(fields); } } } return fieldList; } private ListMultimap lazyFieldsBySourceId() { if (fieldsBySourceId == null) { synchronized (this) { if (fieldsBySourceId == null) { ListMultimap multiMap = Multimaps.newListMultimap( Maps.newHashMap(), () -> Lists.newArrayListWithCapacity(fields.length)); for (PartitionField field : fields) { multiMap.put(field.sourceId(), field); } this.fieldsBySourceId = multiMap; } } } return fieldsBySourceId; } /** * Returns the source field ids for identity partitions. * * @return a set of source ids for the identity partitions. */ public Set identitySourceIds() { Set sourceIds = Sets.newHashSet(); for (PartitionField field : fields()) { if ("identity".equals(field.transform().toString())) { sourceIds.add(field.sourceId()); } } return sourceIds; } @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("["); for (PartitionField field : fields) { sb.append("\n"); sb.append(" ").append(field); } if (fields.length > 0) { sb.append("\n"); } sb.append("]"); return sb.toString(); } private static final PartitionSpec UNPARTITIONED_SPEC = new PartitionSpec(new Schema(), 0, ImmutableList.of(), unpartitionedLastAssignedId()); /** * Returns a spec for unpartitioned tables. * * @return a partition spec with no partitions */ public static PartitionSpec unpartitioned() { return UNPARTITIONED_SPEC; } private static int unpartitionedLastAssignedId() { return PARTITION_DATA_ID_START - 1; } /** * Creates a new {@link Builder partition spec builder} for the given {@link Schema}. * * @param schema a schema * @return a partition spec builder for the given schema */ public static Builder builderFor(Schema schema) { return new Builder(schema); } /** * Used to create valid {@link PartitionSpec partition specs}. * *

Call {@link #builderFor(Schema)} to create a new builder. */ public static class Builder { private final Schema schema; private final List fields = Lists.newArrayList(); private final Set partitionNames = Sets.newHashSet(); private final Map, PartitionField> dedupFields = Maps.newHashMap(); private int specId = 0; private final AtomicInteger lastAssignedFieldId = new AtomicInteger(unpartitionedLastAssignedId()); // check if there are conflicts between partition and schema field name // HACK-HACK: disable checkConflicts for partition spec converted from Delta // to honor the field id assigned by Delta private boolean checkConflicts = false; private boolean caseSensitive = true; private Builder(Schema schema) { this.schema = schema; } private int nextFieldId() { return lastAssignedFieldId.incrementAndGet(); } private void checkAndAddPartitionName(String name) { checkAndAddPartitionName(name, null); } Builder checkConflicts(boolean check) { checkConflicts = check; return this; } private void checkAndAddPartitionName(String name, Integer sourceColumnId) { Types.NestedField schemaField = this.caseSensitive ? schema.findField(name) : schema.caseInsensitiveFindField(name); if (checkConflicts) { if (sourceColumnId != null) { // for identity transform case we allow conflicts between partition and schema field name // as // long as they are sourced from the same schema field Preconditions.checkArgument( schemaField == null || schemaField.fieldId() == sourceColumnId, "Cannot create identity partition sourced from different field in schema: %s", name); } else { // for all other transforms we don't allow conflicts between partition name and schema // field name Preconditions.checkArgument( schemaField == null, "Cannot create partition from name that exists in schema: %s", name); } } Preconditions.checkArgument(!name.isEmpty(), "Cannot use empty partition name: %s", name); Preconditions.checkArgument( !partitionNames.contains(name), "Cannot use partition name more than once: %s", name); partitionNames.add(name); } private void checkForRedundantPartitions(PartitionField field) { Map.Entry dedupKey = new AbstractMap.SimpleEntry<>(field.sourceId(), field.transform().dedupName()); PartitionField partitionField = dedupFields.get(dedupKey); Preconditions.checkArgument( partitionField == null, "Cannot add redundant partition: %s conflicts with %s", partitionField, field); dedupFields.put(dedupKey, field); } public Builder caseSensitive(boolean sensitive) { this.caseSensitive = sensitive; return this; } public Builder withSpecId(int newSpecId) { this.specId = newSpecId; return this; } private Types.NestedField findSourceColumn(String sourceName) { Types.NestedField sourceColumn = this.caseSensitive ? schema.findField(sourceName) : schema.caseInsensitiveFindField(sourceName); Preconditions.checkArgument( sourceColumn != null, "Cannot find source column: %s", sourceName); return sourceColumn; } public Builder identity(String sourceName, String targetName) { return identity(findSourceColumn(sourceName), targetName); } private Builder identity(Types.NestedField sourceColumn, String targetName) { checkAndAddPartitionName(targetName, sourceColumn.fieldId()); PartitionField field = new PartitionField( sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.identity()); checkForRedundantPartitions(field); fields.add(field); return this; } public Builder identity(String sourceName) { Types.NestedField sourceColumn = findSourceColumn(sourceName); return identity(sourceColumn, schema.findColumnName(sourceColumn.fieldId())); } public Builder year(String sourceName, String targetName) { return year(findSourceColumn(sourceName), targetName); } private Builder year(Types.NestedField sourceColumn, String targetName) { checkAndAddPartitionName(targetName); PartitionField field = new PartitionField(sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.year()); checkForRedundantPartitions(field); fields.add(field); return this; } public Builder year(String sourceName) { Types.NestedField sourceColumn = findSourceColumn(sourceName); String columnName = schema.findColumnName(sourceColumn.fieldId()); return year(sourceColumn, columnName + "_year"); } public Builder month(String sourceName, String targetName) { return month(findSourceColumn(sourceName), targetName); } private Builder month(Types.NestedField sourceColumn, String targetName) { checkAndAddPartitionName(targetName); PartitionField field = new PartitionField(sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.month()); checkForRedundantPartitions(field); fields.add(field); return this; } public Builder month(String sourceName) { Types.NestedField sourceColumn = findSourceColumn(sourceName); String columnName = schema.findColumnName(sourceColumn.fieldId()); return month(sourceColumn, columnName + "_month"); } public Builder day(String sourceName, String targetName) { return day(findSourceColumn(sourceName), targetName); } private Builder day(Types.NestedField sourceColumn, String targetName) { checkAndAddPartitionName(targetName); PartitionField field = new PartitionField(sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.day()); checkForRedundantPartitions(field); fields.add(field); return this; } public Builder day(String sourceName) { Types.NestedField sourceColumn = findSourceColumn(sourceName); String columnName = schema.findColumnName(sourceColumn.fieldId()); return day(sourceColumn, columnName + "_day"); } public Builder hour(String sourceName, String targetName) { return hour(findSourceColumn(sourceName), targetName); } private Builder hour(Types.NestedField sourceColumn, String targetName) { checkAndAddPartitionName(targetName); PartitionField field = new PartitionField(sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.hour()); checkForRedundantPartitions(field); fields.add(field); return this; } public Builder hour(String sourceName) { Types.NestedField sourceColumn = findSourceColumn(sourceName); String columnName = schema.findColumnName(sourceColumn.fieldId()); return hour(sourceColumn, columnName + "_hour"); } public Builder bucket(String sourceName, int numBuckets, String targetName) { return bucket(findSourceColumn(sourceName), numBuckets, targetName); } private Builder bucket(Types.NestedField sourceColumn, int numBuckets, String targetName) { checkAndAddPartitionName(targetName); fields.add( new PartitionField( sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.bucket(numBuckets))); return this; } public Builder bucket(String sourceName, int numBuckets) { Types.NestedField sourceColumn = findSourceColumn(sourceName); String columnName = schema.findColumnName(sourceColumn.fieldId()); return bucket(sourceColumn, numBuckets, columnName + "_bucket"); } public Builder truncate(String sourceName, int width, String targetName) { return truncate(findSourceColumn(sourceName), width, targetName); } private Builder truncate(Types.NestedField sourceColumn, int width, String targetName) { checkAndAddPartitionName(targetName); fields.add( new PartitionField( sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.truncate(width))); return this; } public Builder truncate(String sourceName, int width) { Types.NestedField sourceColumn = findSourceColumn(sourceName); String columnName = schema.findColumnName(sourceColumn.fieldId()); return truncate(sourceColumn, width, columnName + "_trunc"); } public Builder alwaysNull(String sourceName, String targetName) { return alwaysNull(findSourceColumn(sourceName), targetName); } private Builder alwaysNull(Types.NestedField sourceColumn, String targetName) { checkAndAddPartitionName( targetName, sourceColumn.fieldId()); // can duplicate a source column name fields.add( new PartitionField( sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.alwaysNull())); return this; } public Builder alwaysNull(String sourceName) { Types.NestedField sourceColumn = findSourceColumn(sourceName); String columnName = schema.findColumnName(sourceColumn.fieldId()); return alwaysNull(sourceColumn, columnName + "_null"); } // add a partition field with an auto-increment partition field id starting from // PARTITION_DATA_ID_START Builder add(int sourceId, String name, Transform transform) { return add(sourceId, nextFieldId(), name, transform); } Builder add(int sourceId, int fieldId, String name, Transform transform) { checkAndAddPartitionName(name, sourceId); fields.add(new PartitionField(sourceId, fieldId, name, transform)); lastAssignedFieldId.getAndAccumulate(fieldId, Math::max); return this; } public PartitionSpec build() { return build(false); } public PartitionSpec build(boolean allowMissingFields) { PartitionSpec spec = buildUnchecked(); checkCompatibility(spec, schema, allowMissingFields); return spec; } PartitionSpec buildUnchecked() { return new PartitionSpec(schema, specId, fields, lastAssignedFieldId.get()); } } static void checkCompatibility(PartitionSpec spec, Schema schema) { checkCompatibility(spec, schema, false); } static void checkCompatibility(PartitionSpec spec, Schema schema, boolean allowMissingFields) { final Map parents = TypeUtil.indexParents(schema.asStruct()); for (PartitionField field : spec.fields) { Type sourceType = schema.findType(field.sourceId()); Transform transform = field.transform(); // In the case the underlying field is dropped, we cannot check if they are compatible if (allowMissingFields && sourceType == null) { continue; } // In the case of a Version 1 partition-spec field gets deleted, // it is replaced with a void transform, see: // https://iceberg.apache.org/spec/#partition-transforms // We don't care about the source type since a VoidTransform is always compatible and skip the // checks if (!transform.equals(Transforms.alwaysNull())) { ValidationException.check( sourceType != null, "Cannot find source column for partition field: %s", field); ValidationException.check( sourceType.isPrimitiveType(), "Cannot partition by non-primitive source field: %s", sourceType); ValidationException.check( transform.canTransform(sourceType), "Invalid source type %s for transform: %s", sourceType, transform); // The only valid parent types for a PartitionField are StructTypes. This must be checked // recursively. Integer parentId = parents.get(field.sourceId()); while (parentId != null) { Type parentType = schema.findType(parentId); ValidationException.check( parentType.isStructType(), "Invalid partition field parent: %s", parentType); parentId = parents.get(parentId); } } } } static boolean hasSequentialIds(PartitionSpec spec) { for (int i = 0; i < spec.fields.length; i += 1) { if (spec.fields[i].fieldId() != PARTITION_DATA_ID_START + i) { return false; } } return true; } } ================================================ FILE: icebergShaded/src/main/java/org/apache/iceberg/TableMetadata.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.iceberg; import java.io.Serializable; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.iceberg.encryption.EncryptedKey; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; import org.apache.iceberg.relocated.com.google.common.base.Objects; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.util.LocationUtil; import org.apache.iceberg.util.Pair; import org.apache.iceberg.util.PartitionUtil; import org.apache.iceberg.util.PropertyUtil; import org.apache.iceberg.util.SerializableSupplier; /** * This class is directly copied from iceberg repo 1.10.0 with following changes * Change: L602 add back the deprecated API * public TableMetadata updateSchema(Schema newSchema, int newLastColumnId) * L848 add the sql conf check to bypass snap sequenceNumber check * L1048 add back the deprecated API * public Builder withNextRowId(Long newRowId) * L1110 add the sql conf check to bypass downgrade check * L1174 add back the deprecated API * public Builder addSchema(Schema schema, int newLastColumnId) */ /** Metadata for a table. */ public class TableMetadata implements Serializable { static final long INITIAL_SEQUENCE_NUMBER = 0; static final long INVALID_SEQUENCE_NUMBER = -1; static final int DEFAULT_TABLE_FORMAT_VERSION = 2; static final int SUPPORTED_TABLE_FORMAT_VERSION = 4; static final int MIN_FORMAT_VERSION_ROW_LINEAGE = 3; static final int INITIAL_SPEC_ID = 0; static final int INITIAL_SORT_ORDER_ID = 1; static final int INITIAL_SCHEMA_ID = 0; static final int INITIAL_ROW_ID = 0; private static final long ONE_MINUTE = TimeUnit.MINUTES.toMillis(1); public static TableMetadata newTableMetadata( Schema schema, PartitionSpec spec, SortOrder sortOrder, String location, Map properties) { int formatVersion = PropertyUtil.propertyAsInt( properties, TableProperties.FORMAT_VERSION, DEFAULT_TABLE_FORMAT_VERSION); return newTableMetadata( schema, spec, sortOrder, location, persistedProperties(properties), formatVersion); } public static TableMetadata newTableMetadata( Schema schema, PartitionSpec spec, String location, Map properties) { return newTableMetadata(schema, spec, SortOrder.unsorted(), location, properties); } private static Map unreservedProperties(Map rawProperties) { return rawProperties.entrySet().stream() .filter(e -> !TableProperties.RESERVED_PROPERTIES.contains(e.getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } private static Map persistedProperties(Map rawProperties) { Map persistedProperties = Maps.newHashMap(); // explicitly set defaults that apply only to new tables persistedProperties.put( TableProperties.PARQUET_COMPRESSION, TableProperties.PARQUET_COMPRESSION_DEFAULT_SINCE_1_4_0); rawProperties.entrySet().stream() .filter(entry -> !TableProperties.RESERVED_PROPERTIES.contains(entry.getKey())) .forEach(entry -> persistedProperties.put(entry.getKey(), entry.getValue())); return persistedProperties; } static TableMetadata newTableMetadata( Schema schema, PartitionSpec spec, SortOrder sortOrder, String location, Map properties, int formatVersion) { Preconditions.checkArgument( properties.keySet().stream().noneMatch(TableProperties.RESERVED_PROPERTIES::contains), "Table properties should not contain reserved properties, but got %s", properties); // reassign all column ids to ensure consistency AtomicInteger lastColumnId = new AtomicInteger(0); Schema freshSchema = TypeUtil.assignFreshIds(INITIAL_SCHEMA_ID, schema, lastColumnId::incrementAndGet); // rebuild the partition spec using the new column ids PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(freshSchema).withSpecId(INITIAL_SPEC_ID); for (PartitionField field : spec.fields()) { // look up the name of the source field in the old schema to get the new schema's id String sourceName = schema.findColumnName(field.sourceId()); // reassign all partition fields with fresh partition field Ids to ensure consistency specBuilder.add(freshSchema.findField(sourceName).fieldId(), field.name(), field.transform()); } PartitionSpec freshSpec = specBuilder.build(); // rebuild the sort order using the new column ids int freshSortOrderId = sortOrder.isUnsorted() ? sortOrder.orderId() : INITIAL_SORT_ORDER_ID; SortOrder freshSortOrder = freshSortOrder(freshSortOrderId, freshSchema, sortOrder); // Validate the metrics configuration. Note: we only do this on new tables to we don't // break existing tables. MetricsConfig.fromProperties(properties).validateReferencedColumns(schema); PropertyUtil.validateCommitProperties(properties); return new Builder() .setInitialFormatVersion(formatVersion) .setCurrentSchema(freshSchema, lastColumnId.get()) .setDefaultPartitionSpec(freshSpec) .setDefaultSortOrder(freshSortOrder) .setLocation(location) .setProperties(properties) .build(); } public static class SnapshotLogEntry implements HistoryEntry { private final long timestampMillis; private final long snapshotId; SnapshotLogEntry(long timestampMillis, long snapshotId) { this.timestampMillis = timestampMillis; this.snapshotId = snapshotId; } @Override public long timestampMillis() { return timestampMillis; } @Override public long snapshotId() { return snapshotId; } @Override public boolean equals(Object other) { if (this == other) { return true; } else if (!(other instanceof SnapshotLogEntry)) { return false; } SnapshotLogEntry that = (SnapshotLogEntry) other; return timestampMillis == that.timestampMillis && snapshotId == that.snapshotId; } @Override public int hashCode() { return Objects.hashCode(timestampMillis, snapshotId); } @Override public String toString() { return MoreObjects.toStringHelper(this) .add("timestampMillis", timestampMillis) .add("snapshotId", snapshotId) .toString(); } } public static class MetadataLogEntry { private final long timestampMillis; private final String file; MetadataLogEntry(long timestampMillis, String file) { this.timestampMillis = timestampMillis; this.file = file; } public long timestampMillis() { return timestampMillis; } public String file() { return file; } @Override public boolean equals(Object other) { if (this == other) { return true; } else if (!(other instanceof MetadataLogEntry)) { return false; } MetadataLogEntry that = (MetadataLogEntry) other; return timestampMillis == that.timestampMillis && java.util.Objects.equals(file, that.file); } @Override public int hashCode() { return Objects.hashCode(timestampMillis, file); } @Override public String toString() { return MoreObjects.toStringHelper(this) .add("timestampMillis", timestampMillis) .add("file", file) .toString(); } } // stored metadata private final String metadataFileLocation; private final int formatVersion; private final String uuid; private final String location; private final long lastSequenceNumber; private final long lastUpdatedMillis; private final int lastColumnId; private final int currentSchemaId; private final List schemas; private final int defaultSpecId; private final List specs; private final int lastAssignedPartitionId; private final int defaultSortOrderId; private final List sortOrders; private final Map properties; private final long currentSnapshotId; private final Map schemasById; private final Map specsById; private final Map sortOrdersById; private final List snapshotLog; private final List previousFiles; private final List statisticsFiles; private final List partitionStatisticsFiles; private final List changes; private final long nextRowId; private final List encryptionKeys; private SerializableSupplier> snapshotsSupplier; private volatile List snapshots; private volatile Map snapshotsById; private volatile Map refs; private volatile boolean snapshotsLoaded; @SuppressWarnings("checkstyle:CyclomaticComplexity") TableMetadata( String metadataFileLocation, int formatVersion, String uuid, String location, long lastSequenceNumber, long lastUpdatedMillis, int lastColumnId, int currentSchemaId, List schemas, int defaultSpecId, List specs, int lastAssignedPartitionId, int defaultSortOrderId, List sortOrders, Map properties, long currentSnapshotId, List snapshots, SerializableSupplier> snapshotsSupplier, List snapshotLog, List previousFiles, Map refs, List statisticsFiles, List partitionStatisticsFiles, long nextRowId, List encryptionKeys, List changes) { Preconditions.checkArgument( specs != null && !specs.isEmpty(), "Partition specs cannot be null or empty"); Preconditions.checkArgument( sortOrders != null && !sortOrders.isEmpty(), "Sort orders cannot be null or empty"); Preconditions.checkArgument( formatVersion <= SUPPORTED_TABLE_FORMAT_VERSION, "Unsupported format version: v%s (supported: v%s)", formatVersion, SUPPORTED_TABLE_FORMAT_VERSION); Preconditions.checkArgument( formatVersion == 1 || uuid != null, "UUID is required in format v%s", formatVersion); Preconditions.checkArgument( formatVersion > 1 || lastSequenceNumber == 0, "Sequence number must be 0 in v1: %s", lastSequenceNumber); Preconditions.checkArgument( metadataFileLocation == null || changes.isEmpty(), "Cannot create TableMetadata with a metadata location and changes"); Preconditions.checkArgument(encryptionKeys != null, "Encryption keys cannot be null"); this.metadataFileLocation = metadataFileLocation; this.formatVersion = formatVersion; this.uuid = uuid; this.location = location != null ? LocationUtil.stripTrailingSlash(location) : null; this.lastSequenceNumber = lastSequenceNumber; this.lastUpdatedMillis = lastUpdatedMillis; this.lastColumnId = lastColumnId; this.currentSchemaId = currentSchemaId; this.schemas = schemas; this.specs = specs; this.defaultSpecId = defaultSpecId; this.lastAssignedPartitionId = lastAssignedPartitionId; this.defaultSortOrderId = defaultSortOrderId; this.sortOrders = sortOrders; this.properties = properties; this.currentSnapshotId = currentSnapshotId; this.snapshots = snapshots; this.snapshotsSupplier = snapshotsSupplier; this.snapshotsLoaded = snapshotsSupplier == null; this.snapshotLog = snapshotLog; this.previousFiles = previousFiles; this.encryptionKeys = encryptionKeys; // changes are carried through until metadata is read from a file this.changes = changes; this.snapshotsById = indexAndValidateSnapshots(snapshots, lastSequenceNumber); this.schemasById = indexSchemas(); this.specsById = PartitionUtil.indexSpecs(specs); this.sortOrdersById = indexSortOrders(sortOrders); this.refs = validateRefs(currentSnapshotId, refs, snapshotsById); this.statisticsFiles = ImmutableList.copyOf(statisticsFiles); this.partitionStatisticsFiles = ImmutableList.copyOf(partitionStatisticsFiles); // row lineage this.nextRowId = nextRowId; HistoryEntry last = null; for (HistoryEntry logEntry : snapshotLog) { if (last != null) { Preconditions.checkArgument( (logEntry.timestampMillis() - last.timestampMillis()) >= -ONE_MINUTE, "[BUG] Expected sorted snapshot log entries."); } last = logEntry; } if (last != null) { Preconditions.checkArgument( // commits can happen concurrently from different machines. // A tolerance helps us avoid failure for small clock skew lastUpdatedMillis - last.timestampMillis() >= -ONE_MINUTE, "Invalid update timestamp %s: before last snapshot log entry at %s", lastUpdatedMillis, last.timestampMillis()); } MetadataLogEntry previous = null; for (MetadataLogEntry metadataEntry : previousFiles) { if (previous != null) { Preconditions.checkArgument( // commits can happen concurrently from different machines. // A tolerance helps us avoid failure for small clock skew (metadataEntry.timestampMillis() - previous.timestampMillis()) >= -ONE_MINUTE, "[BUG] Expected sorted previous metadata log entries."); } previous = metadataEntry; } // Make sure that this update's lastUpdatedMillis is > max(previousFile's timestamp) if (previous != null) { Preconditions.checkArgument( // commits can happen concurrently from different machines. // A tolerance helps us avoid failure for small clock skew lastUpdatedMillis - previous.timestampMillis >= -ONE_MINUTE, "Invalid update timestamp %s: before the latest metadata log entry timestamp %s", lastUpdatedMillis, previous.timestampMillis); } validateCurrentSnapshot(); } public int formatVersion() { return formatVersion; } public String metadataFileLocation() { return metadataFileLocation; } public String uuid() { return uuid; } public long lastSequenceNumber() { return lastSequenceNumber; } public long nextSequenceNumber() { return formatVersion > 1 ? lastSequenceNumber + 1 : INITIAL_SEQUENCE_NUMBER; } public long lastUpdatedMillis() { return lastUpdatedMillis; } public int lastColumnId() { return lastColumnId; } public Schema schema() { return schemasById.get(currentSchemaId); } public List schemas() { return schemas; } public Map schemasById() { return schemasById; } public int currentSchemaId() { return currentSchemaId; } public PartitionSpec spec() { return specsById.get(defaultSpecId); } public PartitionSpec spec(int id) { return specsById.get(id); } public List specs() { return specs; } public Map specsById() { return specsById; } public int lastAssignedPartitionId() { return lastAssignedPartitionId; } public int defaultSpecId() { return defaultSpecId; } public int defaultSortOrderId() { return defaultSortOrderId; } public SortOrder sortOrder() { return sortOrdersById.get(defaultSortOrderId); } public List sortOrders() { return sortOrders; } public Map sortOrdersById() { return sortOrdersById; } public String location() { return location; } public Map properties() { return properties; } public String property(String property, String defaultValue) { return properties.getOrDefault(property, defaultValue); } public boolean propertyAsBoolean(String property, boolean defaultValue) { return PropertyUtil.propertyAsBoolean(properties, property, defaultValue); } public int propertyAsInt(String property, int defaultValue) { return PropertyUtil.propertyAsInt(properties, property, defaultValue); } public int propertyTryAsInt(String property, int defaultValue) { return PropertyUtil.propertyTryAsInt(properties, property, defaultValue); } public long propertyAsLong(String property, long defaultValue) { return PropertyUtil.propertyAsLong(properties, property, defaultValue); } public Snapshot snapshot(long snapshotId) { if (!snapshotsById.containsKey(snapshotId)) { ensureSnapshotsLoaded(); } return snapshotsById.get(snapshotId); } public Snapshot currentSnapshot() { return snapshotsById.get(currentSnapshotId); } public List snapshots() { ensureSnapshotsLoaded(); return snapshots; } private synchronized void ensureSnapshotsLoaded() { if (!snapshotsLoaded) { List loadedSnapshots = Lists.newArrayList(snapshotsSupplier.get()); loadedSnapshots.removeIf(s -> s.sequenceNumber() > lastSequenceNumber); this.snapshots = ImmutableList.copyOf(loadedSnapshots); this.snapshotsById = indexAndValidateSnapshots(snapshots, lastSequenceNumber); validateCurrentSnapshot(); this.refs = validateRefs(currentSnapshotId, refs, snapshotsById); this.snapshotsLoaded = true; this.snapshotsSupplier = null; } } public SnapshotRef ref(String name) { return refs.get(name); } public Map refs() { return refs; } public List statisticsFiles() { return statisticsFiles; } public List partitionStatisticsFiles() { return partitionStatisticsFiles; } public List snapshotLog() { return snapshotLog; } public List previousFiles() { return previousFiles; } public List changes() { return changes; } public TableMetadata withUUID() { return new Builder(this).assignUUID().build(); } public long nextRowId() { return nextRowId; } public List encryptionKeys() { return encryptionKeys; } /** * HACK-HACK This is added * Updates the schema * @deprecated in 1.9.0 */ @Deprecated public TableMetadata updateSchema(Schema newSchema, int newLastColumnId) { return new Builder(this).setCurrentSchema(newSchema, newLastColumnId).build(); } /** Updates the schema */ public TableMetadata updateSchema(Schema newSchema) { return new Builder(this) .setCurrentSchema(newSchema, Math.max(this.lastColumnId, newSchema.highestFieldId())) .build(); } // The caller is responsible to pass a newPartitionSpec with correct partition field IDs public TableMetadata updatePartitionSpec(PartitionSpec newPartitionSpec) { return new Builder(this).setDefaultPartitionSpec(newPartitionSpec).build(); } public TableMetadata addPartitionSpec(PartitionSpec newPartitionSpec) { return new Builder(this).addPartitionSpec(newPartitionSpec).build(); } public TableMetadata replaceSortOrder(SortOrder newOrder) { return new Builder(this).setDefaultSortOrder(newOrder).build(); } public TableMetadata removeSnapshotsIf(Predicate removeIf) { List toRemove = snapshots().stream().filter(removeIf).collect(Collectors.toList()); return new Builder(this).removeSnapshots(toRemove).build(); } public TableMetadata replaceProperties(Map rawProperties) { ValidationException.check(rawProperties != null, "Cannot set properties to null"); Map newProperties = unreservedProperties(rawProperties); Set removed = Sets.newHashSet(properties.keySet()); Map updated = Maps.newHashMap(); for (Map.Entry entry : newProperties.entrySet()) { removed.remove(entry.getKey()); String current = properties.get(entry.getKey()); if (current == null || !current.equals(entry.getValue())) { updated.put(entry.getKey(), entry.getValue()); } } int newFormatVersion = PropertyUtil.propertyAsInt(rawProperties, TableProperties.FORMAT_VERSION, formatVersion); return new Builder(this) .setProperties(updated) .removeProperties(removed) .upgradeFormatVersion(newFormatVersion) .build(); } private void validateCurrentSnapshot() { Preconditions.checkArgument( currentSnapshotId < 0 || snapshotsById.containsKey(currentSnapshotId), "Invalid table metadata: Cannot find current version"); } private PartitionSpec reassignPartitionIds(PartitionSpec partitionSpec, TypeUtil.NextID nextID) { PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(partitionSpec.schema()).withSpecId(partitionSpec.specId()); if (formatVersion > 1) { // for v2 and later, reuse any existing field IDs, but reproduce the same spec Map, Integer> transformToFieldId = specs.stream() .flatMap(spec -> spec.fields().stream()) .collect( Collectors.toMap( field -> Pair.of(field.sourceId(), field.transform().toString()), PartitionField::fieldId, Math::max)); for (PartitionField field : partitionSpec.fields()) { // reassign the partition field ids int partitionFieldId = transformToFieldId.computeIfAbsent( Pair.of(field.sourceId(), field.transform().toString()), k -> nextID.get()); specBuilder.add(field.sourceId(), partitionFieldId, field.name(), field.transform()); } } else { // for v1, preserve the existing spec and carry forward all fields, replacing missing fields // with void Map, PartitionField> newFields = Maps.newLinkedHashMap(); for (PartitionField newField : partitionSpec.fields()) { newFields.put(Pair.of(newField.sourceId(), newField.transform().toString()), newField); } List newFieldNames = newFields.values().stream().map(PartitionField::name).collect(Collectors.toList()); for (PartitionField field : spec().fields()) { // ensure each field is either carried forward or replaced with void PartitionField newField = newFields.remove(Pair.of(field.sourceId(), field.transform().toString())); if (newField != null) { // copy the new field with the existing field ID specBuilder.add( newField.sourceId(), field.fieldId(), newField.name(), newField.transform()); } else { // Rename old void transforms that would otherwise conflict String voidName = newFieldNames.contains(field.name()) ? field.name() + "_" + field.fieldId() : field.name(); specBuilder.add(field.sourceId(), field.fieldId(), voidName, Transforms.alwaysNull()); } } // add any remaining new fields at the end and assign new partition field IDs for (PartitionField newField : newFields.values()) { specBuilder.add(newField.sourceId(), nextID.get(), newField.name(), newField.transform()); } } return specBuilder.build(); } // The caller is responsible to pass a updatedPartitionSpec with correct partition field IDs public TableMetadata buildReplacement( Schema updatedSchema, PartitionSpec updatedPartitionSpec, SortOrder updatedSortOrder, String newLocation, Map updatedProperties) { ValidationException.check( formatVersion > 1 || PartitionSpec.hasSequentialIds(updatedPartitionSpec), "Spec does not use sequential IDs that are required in v1: %s", updatedPartitionSpec); AtomicInteger newLastColumnId = new AtomicInteger(lastColumnId); Schema freshSchema = TypeUtil.assignFreshIds(updatedSchema, schema(), newLastColumnId::incrementAndGet); // rebuild the partition spec using the new column ids and reassign partition field ids to align // with existing // partition specs in the table PartitionSpec freshSpec = reassignPartitionIds( freshSpec(INITIAL_SPEC_ID, freshSchema, updatedPartitionSpec), new AtomicInteger(lastAssignedPartitionId)::incrementAndGet); // rebuild the sort order using new column ids SortOrder freshSortOrder = freshSortOrder(INITIAL_SORT_ORDER_ID, freshSchema, updatedSortOrder); // check if there is format version override int newFormatVersion = PropertyUtil.propertyAsInt( updatedProperties, TableProperties.FORMAT_VERSION, formatVersion); return new Builder(this) .upgradeFormatVersion(newFormatVersion) .removeRef(SnapshotRef.MAIN_BRANCH) .setCurrentSchema(freshSchema, newLastColumnId.get()) .setDefaultPartitionSpec(freshSpec) .setDefaultSortOrder(freshSortOrder) .setLocation(newLocation) .setProperties(persistedProperties(updatedProperties)) .build(); } public TableMetadata updateLocation(String newLocation) { return new Builder(this).setLocation(newLocation).build(); } public TableMetadata upgradeToFormatVersion(int newFormatVersion) { return new Builder(this).upgradeFormatVersion(newFormatVersion).build(); } private static PartitionSpec updateSpecSchema(Schema schema, PartitionSpec partitionSpec) { PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(schema).withSpecId(partitionSpec.specId()); // add all the fields to the builder. IDs should not change. for (PartitionField field : partitionSpec.fields()) { specBuilder.add(field.sourceId(), field.fieldId(), field.name(), field.transform()); } // build without validation because the schema may have changed in a way that makes this spec // invalid. the spec // should still be preserved so that older metadata can be interpreted. return specBuilder.buildUnchecked(); } private static SortOrder updateSortOrderSchema(Schema schema, SortOrder sortOrder) { SortOrder.Builder builder = SortOrder.builderFor(schema).withOrderId(sortOrder.orderId()); // add all the fields to the builder. IDs should not change. for (SortField field : sortOrder.fields()) { builder.addSortField( field.transform(), field.sourceId(), field.direction(), field.nullOrder()); } // build without validation because the schema may have changed in a way that makes this order // invalid. the order // should still be preserved so that older metadata can be interpreted. return builder.buildUnchecked(); } private static PartitionSpec freshSpec(int specId, Schema schema, PartitionSpec partitionSpec) { UnboundPartitionSpec.Builder specBuilder = UnboundPartitionSpec.builder().withSpecId(specId); for (PartitionField field : partitionSpec.fields()) { // look up the name of the source field in the old schema to get the new schema's id String sourceName = partitionSpec.schema().findColumnName(field.sourceId()); final int fieldId; if (sourceName != null) { fieldId = schema.findField(sourceName).fieldId(); } else { // In the case of a null sourceName, the column has been deleted. // This only happens in V1 tables where the reference is still around as a void transform fieldId = field.sourceId(); } specBuilder.addField(field.transform().toString(), fieldId, field.fieldId(), field.name()); } return specBuilder.build().bind(schema); } private static SortOrder freshSortOrder(int orderId, Schema schema, SortOrder sortOrder) { UnboundSortOrder.Builder builder = UnboundSortOrder.builder(); if (sortOrder.isSorted()) { builder.withOrderId(orderId); } for (SortField field : sortOrder.fields()) { // look up the name of the source field in the old schema to get the new schema's id String sourceName = sortOrder.schema().findColumnName(field.sourceId()); // reassign all sort fields with fresh sort field IDs int newSourceId = schema.findField(sourceName).fieldId(); builder.addSortField( field.transform().toString(), newSourceId, field.direction(), field.nullOrder()); } return builder.build().bind(schema); } private static Map indexAndValidateSnapshots( List snapshots, long lastSequenceNumber) { ImmutableMap.Builder builder = ImmutableMap.builder(); for (Snapshot snap : snapshots) { ValidationException.check( snap.sequenceNumber() <= lastSequenceNumber, "Invalid snapshot with sequence number %s greater than last sequence number %s", snap.sequenceNumber(), lastSequenceNumber); builder.put(snap.snapshotId(), snap); } return builder.build(); } private Map indexSchemas() { ImmutableMap.Builder builder = ImmutableMap.builder(); for (Schema schema : schemas) { builder.put(schema.schemaId(), schema); } return builder.build(); } private static Map indexSortOrders(List sortOrders) { ImmutableMap.Builder builder = ImmutableMap.builder(); for (SortOrder sortOrder : sortOrders) { builder.put(sortOrder.orderId(), sortOrder); } return builder.build(); } private static Map validateRefs( Long currentSnapshotId, Map inputRefs, Map snapshotsById) { for (SnapshotRef ref : inputRefs.values()) { Preconditions.checkArgument( snapshotsById.containsKey(ref.snapshotId()), "Snapshot for reference %s does not exist in the existing snapshots list", ref); } SnapshotRef main = inputRefs.get(SnapshotRef.MAIN_BRANCH); if (currentSnapshotId != -1) { Preconditions.checkArgument( main == null || currentSnapshotId == main.snapshotId(), "Current snapshot ID does not match main branch (%s != %s)", currentSnapshotId, main != null ? main.snapshotId() : null); } else { Preconditions.checkArgument( main == null, "Current snapshot is not set, but main branch exists: %s", main); } return inputRefs; } public static Builder buildFrom(TableMetadata base) { return new Builder(base); } public static Builder buildFromEmpty() { return new Builder(DEFAULT_TABLE_FORMAT_VERSION); } public static Builder buildFromEmpty(int formatVersion) { return new Builder(formatVersion); } public static class Builder { private static final int LAST_ADDED = -1; private final TableMetadata base; private String metadataLocation; private int formatVersion; private String uuid; private Long lastUpdatedMillis; private String location; private long lastSequenceNumber; private int lastColumnId; private int currentSchemaId; private List schemas; private int defaultSpecId; private List specs; private int lastAssignedPartitionId; private int defaultSortOrderId; private List sortOrders; private final Map properties; private long currentSnapshotId; private List snapshots; private SerializableSupplier> snapshotsSupplier; private final Map refs; private final Map> statisticsFiles; private final Map> partitionStatisticsFiles; private boolean suppressHistoricalSnapshots = false; private long nextRowId; private final List encryptionKeys; // change tracking private final List changes; private final int startingChangeCount; private boolean discardChanges = false; private Integer lastAddedSchemaId = null; private Integer lastAddedSpecId = null; private Integer lastAddedOrderId = null; // handled in build private final List snapshotLog; private String previousFileLocation; private final List previousFiles; // indexes for convenience private final Map snapshotsById; private final Map schemasById; private final Map specsById; private final Map sortOrdersById; private final Map keysById; private Builder() { this(DEFAULT_TABLE_FORMAT_VERSION); } private Builder(int formatVersion) { this.base = null; this.formatVersion = formatVersion; this.lastSequenceNumber = INITIAL_SEQUENCE_NUMBER; this.uuid = UUID.randomUUID().toString(); this.schemas = Lists.newArrayList(); this.specs = Lists.newArrayList(); this.sortOrders = Lists.newArrayList(); this.properties = Maps.newHashMap(); this.snapshots = Lists.newArrayList(); this.currentSnapshotId = -1; this.changes = Lists.newArrayList(); this.startingChangeCount = 0; this.snapshotLog = Lists.newArrayList(); this.previousFiles = Lists.newArrayList(); this.encryptionKeys = Lists.newArrayList(); this.refs = Maps.newHashMap(); this.statisticsFiles = Maps.newHashMap(); this.partitionStatisticsFiles = Maps.newHashMap(); this.snapshotsById = Maps.newHashMap(); this.schemasById = Maps.newHashMap(); this.specsById = Maps.newHashMap(); this.sortOrdersById = Maps.newHashMap(); this.keysById = Maps.newHashMap(); this.nextRowId = INITIAL_ROW_ID; } private Builder(TableMetadata base) { this.base = base; this.formatVersion = base.formatVersion; this.uuid = base.uuid; this.lastUpdatedMillis = null; this.location = base.location; this.lastSequenceNumber = base.lastSequenceNumber; this.lastColumnId = base.lastColumnId; this.currentSchemaId = base.currentSchemaId; this.schemas = Lists.newArrayList(base.schemas); this.defaultSpecId = base.defaultSpecId; this.specs = Lists.newArrayList(base.specs); this.lastAssignedPartitionId = base.lastAssignedPartitionId; this.defaultSortOrderId = base.defaultSortOrderId; this.sortOrders = Lists.newArrayList(base.sortOrders); this.properties = Maps.newHashMap(base.properties); this.currentSnapshotId = base.currentSnapshotId; this.snapshots = Lists.newArrayList(base.snapshots()); this.encryptionKeys = Lists.newArrayList(base.encryptionKeys); this.changes = Lists.newArrayList(base.changes); this.startingChangeCount = changes.size(); this.snapshotLog = Lists.newArrayList(base.snapshotLog); this.previousFileLocation = base.metadataFileLocation; this.previousFiles = base.previousFiles; this.refs = Maps.newHashMap(base.refs); this.statisticsFiles = indexStatistics(base.statisticsFiles); this.partitionStatisticsFiles = indexPartitionStatistics(base.partitionStatisticsFiles); this.snapshotsById = Maps.newHashMap(base.snapshotsById); this.schemasById = Maps.newHashMap(base.schemasById); this.specsById = Maps.newHashMap(base.specsById); this.sortOrdersById = Maps.newHashMap(base.sortOrdersById); this.keysById = encryptionKeys.stream() .collect(Collectors.toMap(EncryptedKey::keyId, Function.identity())); this.nextRowId = base.nextRowId; } // Hack-Hack This is added public Builder withNextRowId(Long newRowId) { this.nextRowId = newRowId; return this; } public Builder withMetadataLocation(String newMetadataLocation) { this.metadataLocation = newMetadataLocation; if (null != base) { // carry over lastUpdatedMillis from base and set previousFileLocation to null to avoid // writing a new metadata log entry // this is safe since setting metadata location doesn't cause any changes and no other // changes can be added when metadata location is configured this.lastUpdatedMillis = base.lastUpdatedMillis(); this.previousFileLocation = null; } return this; } public Builder assignUUID() { if (uuid == null) { this.uuid = UUID.randomUUID().toString(); changes.add(new MetadataUpdate.AssignUUID(uuid)); } return this; } public Builder assignUUID(String newUuid) { Preconditions.checkArgument(newUuid != null, "Cannot set uuid to null"); if (!newUuid.equals(uuid)) { this.uuid = newUuid; changes.add(new MetadataUpdate.AssignUUID(uuid)); } return this; } // it is only safe to set the format version directly while creating tables // in all other cases, use upgradeFormatVersion private Builder setInitialFormatVersion(int newFormatVersion) { Preconditions.checkArgument( newFormatVersion <= SUPPORTED_TABLE_FORMAT_VERSION, "Unsupported format version: v%s (supported: v%s)", newFormatVersion, SUPPORTED_TABLE_FORMAT_VERSION); this.formatVersion = newFormatVersion; return this; } public Builder upgradeFormatVersion(int newFormatVersion) { Preconditions.checkArgument( newFormatVersion <= SUPPORTED_TABLE_FORMAT_VERSION, "Cannot upgrade table to unsupported format version: v%s (supported: v%s)", newFormatVersion, SUPPORTED_TABLE_FORMAT_VERSION); Preconditions.checkArgument( newFormatVersion >= formatVersion, "Cannot downgrade v%s table to v%s", formatVersion, newFormatVersion); if (newFormatVersion == formatVersion) { return this; } this.formatVersion = newFormatVersion; changes.add(new MetadataUpdate.UpgradeFormatVersion(newFormatVersion)); return this; } public Builder setCurrentSchema(Schema newSchema, int newLastColumnId) { setCurrentSchema(addSchemaInternal(newSchema, newLastColumnId)); return this; } public Builder setCurrentSchema(int schemaId) { if (schemaId == -1) { ValidationException.check( lastAddedSchemaId != null, "Cannot set last added schema: no schema has been added"); return setCurrentSchema(lastAddedSchemaId); } if (currentSchemaId == schemaId) { return this; } Schema schema = schemasById.get(schemaId); Preconditions.checkArgument( schema != null, "Cannot set current schema to unknown schema: %s", schemaId); // rebuild all the partition specs and sort orders for the new current schema this.specs = Lists.newArrayList(Iterables.transform(specs, spec -> updateSpecSchema(schema, spec))); specsById.clear(); specsById.putAll(PartitionUtil.indexSpecs(specs)); this.sortOrders = Lists.newArrayList( Iterables.transform(sortOrders, order -> updateSortOrderSchema(schema, order))); sortOrdersById.clear(); sortOrdersById.putAll(indexSortOrders(sortOrders)); this.currentSchemaId = schemaId; if (lastAddedSchemaId != null && lastAddedSchemaId == schemaId) { changes.add(new MetadataUpdate.SetCurrentSchema(LAST_ADDED)); } else { changes.add(new MetadataUpdate.SetCurrentSchema(schemaId)); } return this; } public Builder addSchema(Schema schema) { addSchemaInternal(schema, Math.max(lastColumnId, schema.highestFieldId())); return this; } /** * Hack-Hack This is added * Add a new schema. * @deprecated since 1.8.0, will be removed in 1.9.0 or 2.0.0, use AddSchema(schema). */ @Deprecated public Builder addSchema(Schema schema, int newLastColumnId) { addSchemaInternal(schema, newLastColumnId); return this; } public Builder setDefaultPartitionSpec(PartitionSpec spec) { setDefaultPartitionSpec(addPartitionSpecInternal(spec)); return this; } public Builder setDefaultPartitionSpec(int specId) { if (specId == -1) { ValidationException.check( lastAddedSpecId != null, "Cannot set last added spec: no spec has been added"); return setDefaultPartitionSpec(lastAddedSpecId); } if (defaultSpecId == specId) { // the new spec is already current and no change is needed return this; } this.defaultSpecId = specId; if (lastAddedSpecId != null && lastAddedSpecId == specId) { changes.add(new MetadataUpdate.SetDefaultPartitionSpec(LAST_ADDED)); } else { changes.add(new MetadataUpdate.SetDefaultPartitionSpec(specId)); } return this; } Builder removeSpecs(Iterable specIds) { Set specIdsToRemove = Sets.newHashSet(specIds); Preconditions.checkArgument( !specIdsToRemove.contains(defaultSpecId), "Cannot remove the default partition spec"); if (!specIdsToRemove.isEmpty()) { this.specs = specs.stream() .filter(s -> !specIdsToRemove.contains(s.specId())) .collect(Collectors.toList()); changes.add(new MetadataUpdate.RemovePartitionSpecs(specIdsToRemove)); } return this; } Builder removeSchemas(Iterable schemaIds) { Set schemaIdsToRemove = Sets.newHashSet(schemaIds); Preconditions.checkArgument( !schemaIdsToRemove.contains(currentSchemaId), "Cannot remove the current schema"); if (!schemaIdsToRemove.isEmpty()) { this.schemas = schemas.stream() .filter(s -> !schemaIdsToRemove.contains(s.schemaId())) .collect(Collectors.toList()); changes.add(new MetadataUpdate.RemoveSchemas(schemaIdsToRemove)); } return this; } public Builder addPartitionSpec(UnboundPartitionSpec spec) { addPartitionSpecInternal(spec.bind(schemasById.get(currentSchemaId))); return this; } public Builder addPartitionSpec(PartitionSpec spec) { addPartitionSpecInternal(spec); return this; } public Builder setDefaultSortOrder(SortOrder order) { setDefaultSortOrder(addSortOrderInternal(order)); return this; } public Builder setDefaultSortOrder(int sortOrderId) { if (sortOrderId == -1) { ValidationException.check( lastAddedOrderId != null, "Cannot set last added sort order: no sort order has been added"); return setDefaultSortOrder(lastAddedOrderId); } if (sortOrderId == defaultSortOrderId) { return this; } this.defaultSortOrderId = sortOrderId; if (lastAddedOrderId != null && lastAddedOrderId == sortOrderId) { changes.add(new MetadataUpdate.SetDefaultSortOrder(LAST_ADDED)); } else { changes.add(new MetadataUpdate.SetDefaultSortOrder(sortOrderId)); } return this; } public Builder addSortOrder(UnboundSortOrder order) { addSortOrderInternal(order.bind(schemasById.get(currentSchemaId))); return this; } public Builder addSortOrder(SortOrder order) { addSortOrderInternal(order); return this; } public Builder addSnapshot(Snapshot snapshot) { if (snapshot == null) { // change is a noop return this; } ValidationException.check( !schemas.isEmpty(), "Attempting to add a snapshot before a schema is added"); ValidationException.check( !specs.isEmpty(), "Attempting to add a snapshot before a partition spec is added"); ValidationException.check( !sortOrders.isEmpty(), "Attempting to add a snapshot before a sort order is added"); ValidationException.check( !snapshotsById.containsKey(snapshot.snapshotId()), "Snapshot already exists for id: %s", snapshot.snapshotId()); ValidationException.check( formatVersion == 1 || snapshot.sequenceNumber() > lastSequenceNumber || snapshot.parentId() == null, "Cannot add snapshot with sequence number %s older than last sequence number %s", snapshot.sequenceNumber(), lastSequenceNumber); this.lastUpdatedMillis = snapshot.timestampMillis(); this.lastSequenceNumber = snapshot.sequenceNumber(); snapshots.add(snapshot); snapshotsById.put(snapshot.snapshotId(), snapshot); changes.add(new MetadataUpdate.AddSnapshot(snapshot)); if (formatVersion >= MIN_FORMAT_VERSION_ROW_LINEAGE) { ValidationException.check( snapshot.firstRowId() != null, "Cannot add a snapshot: first-row-id is null"); ValidationException.check( snapshot.firstRowId() != null && snapshot.firstRowId() >= nextRowId, "Cannot add a snapshot, first-row-id is behind table next-row-id: %s < %s", snapshot.firstRowId(), nextRowId); this.nextRowId += snapshot.addedRows(); } return this; } public Builder setSnapshotsSupplier(SerializableSupplier> snapshotsSupplier) { this.snapshotsSupplier = snapshotsSupplier; return this; } public Builder setBranchSnapshot(Snapshot snapshot, String branch) { addSnapshot(snapshot); setBranchSnapshotInternal(snapshot, branch); return this; } public Builder setBranchSnapshot(long snapshotId, String branch) { SnapshotRef ref = refs.get(branch); if (ref != null && ref.snapshotId() == snapshotId) { // change is a noop return this; } Snapshot snapshot = snapshotsById.get(snapshotId); ValidationException.check( snapshot != null, "Cannot set %s to unknown snapshot: %s", branch, snapshotId); setBranchSnapshotInternal(snapshot, branch); return this; } public Builder setRef(String name, SnapshotRef ref) { SnapshotRef existingRef = refs.get(name); if (existingRef != null && existingRef.equals(ref)) { return this; } long snapshotId = ref.snapshotId(); Snapshot snapshot = snapshotsById.get(snapshotId); ValidationException.check( snapshot != null, "Cannot set %s to unknown snapshot: %s", name, snapshotId); if (isAddedSnapshot(snapshotId)) { this.lastUpdatedMillis = snapshot.timestampMillis(); } if (SnapshotRef.MAIN_BRANCH.equals(name)) { this.currentSnapshotId = ref.snapshotId(); if (lastUpdatedMillis == null) { this.lastUpdatedMillis = System.currentTimeMillis(); } snapshotLog.add(new SnapshotLogEntry(lastUpdatedMillis, ref.snapshotId())); } refs.put(name, ref); MetadataUpdate.SetSnapshotRef refUpdate = new MetadataUpdate.SetSnapshotRef( name, ref.snapshotId(), ref.type(), ref.minSnapshotsToKeep(), ref.maxSnapshotAgeMs(), ref.maxRefAgeMs()); changes.add(refUpdate); return this; } public Builder removeRef(String name) { if (SnapshotRef.MAIN_BRANCH.equals(name)) { this.currentSnapshotId = -1; } SnapshotRef ref = refs.remove(name); if (ref != null) { changes.add(new MetadataUpdate.RemoveSnapshotRef(name)); } return this; } public Builder setStatistics(StatisticsFile statisticsFile) { Preconditions.checkNotNull(statisticsFile, "statisticsFile is null"); statisticsFiles.put(statisticsFile.snapshotId(), ImmutableList.of(statisticsFile)); changes.add(new MetadataUpdate.SetStatistics(statisticsFile)); return this; } public Builder removeStatistics(long snapshotId) { if (statisticsFiles.remove(snapshotId) == null) { return this; } changes.add(new MetadataUpdate.RemoveStatistics(snapshotId)); return this; } /** * Suppresses snapshots that are historical, removing the metadata for lazy snapshot loading. * *

Note that the snapshots are not considered removed from metadata and no RemoveSnapshot * changes are created. * *

A snapshot is historical if no ref directly references its ID. * * @return this for method chaining */ public Builder suppressHistoricalSnapshots() { this.suppressHistoricalSnapshots = true; Set refSnapshotIds = refs.values().stream().map(SnapshotRef::snapshotId).collect(Collectors.toSet()); Set suppressedSnapshotIds = Sets.difference(snapshotsById.keySet(), refSnapshotIds); rewriteSnapshotsInternal(suppressedSnapshotIds, true); return this; } public Builder setPartitionStatistics(PartitionStatisticsFile file) { Preconditions.checkNotNull(file, "partition statistics file is null"); partitionStatisticsFiles.put(file.snapshotId(), ImmutableList.of(file)); changes.add(new MetadataUpdate.SetPartitionStatistics(file)); return this; } public Builder removePartitionStatistics(long snapshotId) { if (partitionStatisticsFiles.remove(snapshotId) == null) { return this; } changes.add(new MetadataUpdate.RemovePartitionStatistics(snapshotId)); return this; } public Builder removeSnapshots(List snapshotsToRemove) { Set idsToRemove = snapshotsToRemove.stream().map(Snapshot::snapshotId).collect(Collectors.toSet()); return removeSnapshots(idsToRemove); } public Builder removeSnapshots(Collection idsToRemove) { return rewriteSnapshotsInternal(idsToRemove, false); } /** * Rewrite this builder's snapshots by removing the snapshots for a list of IDs. * *

If suppress is true, changes are not created. * * @param idsToRemove collection of snapshot IDs to remove from this builder * @param suppress whether the operation is suppressing snapshots (retains history) or removing * @return this for method chaining */ private Builder rewriteSnapshotsInternal(Collection idsToRemove, boolean suppress) { List retainedSnapshots = Lists.newArrayListWithExpectedSize(snapshots.size() - idsToRemove.size()); for (Snapshot snapshot : snapshots) { long snapshotId = snapshot.snapshotId(); if (idsToRemove.contains(snapshotId)) { snapshotsById.remove(snapshotId); if (!suppress) { changes.add(new MetadataUpdate.RemoveSnapshots(snapshotId)); } removeStatistics(snapshotId); removePartitionStatistics(snapshotId); } else { retainedSnapshots.add(snapshot); } } this.snapshots = retainedSnapshots; // remove any refs that are no longer valid Set danglingRefs = Sets.newHashSet(); for (Map.Entry refEntry : refs.entrySet()) { if (!snapshotsById.containsKey(refEntry.getValue().snapshotId())) { danglingRefs.add(refEntry.getKey()); } } danglingRefs.forEach(this::removeRef); return this; } public Builder setProperties(Map updated) { if (updated.isEmpty()) { return this; } properties.putAll(updated); changes.add(new MetadataUpdate.SetProperties(updated)); return this; } public Builder removeProperties(Set removed) { if (removed.isEmpty()) { return this; } removed.forEach(properties::remove); changes.add(new MetadataUpdate.RemoveProperties(removed)); return this; } public Builder setLocation(String newLocation) { if (location != null && location.equals(newLocation)) { return this; } this.location = newLocation; changes.add(new MetadataUpdate.SetLocation(newLocation)); return this; } public Builder addEncryptionKey(EncryptedKey key) { if (keysById.containsKey(key.keyId())) { // already exists return this; } encryptionKeys.add(key); keysById.put(key.keyId(), key); changes.add(new MetadataUpdate.AddEncryptionKey(key)); return this; } public Builder removeEncryptionKey(String keyId) { boolean removed = encryptionKeys.removeIf(key -> key.keyId().equals(keyId)); keysById.remove(keyId); if (removed) { changes.add(new MetadataUpdate.RemoveEncryptionKey(keyId)); } return this; } public Builder discardChanges() { this.discardChanges = true; return this; } public Builder setPreviousFileLocation(String previousFileLocation) { this.previousFileLocation = previousFileLocation; return this; } private boolean hasChanges() { return changes.size() != startingChangeCount || (discardChanges && !changes.isEmpty()) || metadataLocation != null || suppressHistoricalSnapshots || null != snapshotsSupplier; } public TableMetadata build() { if (!hasChanges()) { return base; } if (lastUpdatedMillis == null) { this.lastUpdatedMillis = System.currentTimeMillis(); } // when associated with a metadata file, table metadata must have no changes so that the // metadata matches exactly // what is in the metadata file, which does not store changes. metadata location with changes // is inconsistent. Preconditions.checkArgument( changes.isEmpty() || discardChanges || metadataLocation == null, "Cannot set metadata location with changes to table metadata: %s changes", changes.size()); Schema schema = schemasById.get(currentSchemaId); PartitionSpec.checkCompatibility(specsById.get(defaultSpecId), schema); SortOrder.checkCompatibility(sortOrdersById.get(defaultSortOrderId), schema); List metadataHistory; if (base == null) { metadataHistory = Lists.newArrayList(); } else { metadataHistory = addPreviousFile( previousFiles, previousFileLocation, base.lastUpdatedMillis(), properties); } List newSnapshotLog = updateSnapshotLog(snapshotLog, snapshotsById, currentSnapshotId, changes); return new TableMetadata( metadataLocation, formatVersion, uuid, location, lastSequenceNumber, lastUpdatedMillis, lastColumnId, currentSchemaId, ImmutableList.copyOf(schemas), defaultSpecId, ImmutableList.copyOf(specs), lastAssignedPartitionId, defaultSortOrderId, ImmutableList.copyOf(sortOrders), ImmutableMap.copyOf(properties), currentSnapshotId, ImmutableList.copyOf(snapshots), snapshotsSupplier, ImmutableList.copyOf(newSnapshotLog), ImmutableList.copyOf(metadataHistory), ImmutableMap.copyOf(refs), statisticsFiles.values().stream().flatMap(List::stream).collect(Collectors.toList()), partitionStatisticsFiles.values().stream() .flatMap(List::stream) .collect(Collectors.toList()), nextRowId, encryptionKeys, discardChanges ? ImmutableList.of() : ImmutableList.copyOf(changes)); } private int addSchemaInternal(Schema schema, int newLastColumnId) { Preconditions.checkArgument( newLastColumnId >= lastColumnId, "Invalid last column ID: %s < %s (previous last column ID)", newLastColumnId, lastColumnId); Schema.checkCompatibility(schema, formatVersion); int newSchemaId = reuseOrCreateNewSchemaId(schema); boolean schemaFound = schemasById.containsKey(newSchemaId); if (schemaFound && newLastColumnId == lastColumnId) { // the new spec and last column id is already current and no change is needed // update lastAddedSchemaId if the schema was added in this set of changes (since it is now // the last) boolean isNewSchema = lastAddedSchemaId != null && changes(MetadataUpdate.AddSchema.class) .anyMatch(added -> added.schema().schemaId() == newSchemaId); this.lastAddedSchemaId = isNewSchema ? newSchemaId : null; return newSchemaId; } this.lastColumnId = newLastColumnId; Schema newSchema; if (newSchemaId != schema.schemaId()) { newSchema = new Schema(newSchemaId, schema.columns(), schema.identifierFieldIds()); } else { newSchema = schema; } if (!schemaFound) { schemas.add(newSchema); schemasById.put(newSchema.schemaId(), newSchema); } changes.add(new MetadataUpdate.AddSchema(newSchema)); this.lastAddedSchemaId = newSchemaId; return newSchemaId; } private int reuseOrCreateNewSchemaId(Schema newSchema) { // if the schema already exists, use its id; otherwise use the highest id + 1 int newSchemaId = currentSchemaId; for (Schema schema : schemas) { if (schema.sameSchema(newSchema)) { return schema.schemaId(); } else if (schema.schemaId() >= newSchemaId) { newSchemaId = schema.schemaId() + 1; } } return newSchemaId; } private int addPartitionSpecInternal(PartitionSpec spec) { int newSpecId = reuseOrCreateNewSpecId(spec); if (specsById.containsKey(newSpecId)) { // update lastAddedSpecId if the spec was added in this set of changes (since it is now the // last) boolean isNewSpec = lastAddedSpecId != null && changes(MetadataUpdate.AddPartitionSpec.class) .anyMatch(added -> added.spec().specId() == lastAddedSpecId); this.lastAddedSpecId = isNewSpec ? newSpecId : null; return newSpecId; } Schema schema = schemasById.get(currentSchemaId); PartitionSpec.checkCompatibility(spec, schema); ValidationException.check( formatVersion > 1 || PartitionSpec.hasSequentialIds(spec), "Spec does not use sequential IDs that are required in v1: %s", spec); PartitionSpec newSpec = freshSpec(newSpecId, schema, spec); this.lastAssignedPartitionId = Math.max(lastAssignedPartitionId, newSpec.lastAssignedFieldId()); specs.add(newSpec); specsById.put(newSpecId, newSpec); changes.add(new MetadataUpdate.AddPartitionSpec(newSpec)); this.lastAddedSpecId = newSpecId; return newSpecId; } private int reuseOrCreateNewSpecId(PartitionSpec newSpec) { // if the spec already exists, use the same ID. otherwise, use 1 more than the highest ID. int newSpecId = INITIAL_SPEC_ID; for (PartitionSpec spec : specs) { if (newSpec.compatibleWith(spec)) { return spec.specId(); } else if (newSpecId <= spec.specId()) { newSpecId = spec.specId() + 1; } } return newSpecId; } private int addSortOrderInternal(SortOrder order) { int newOrderId = reuseOrCreateNewSortOrderId(order); if (sortOrdersById.containsKey(newOrderId)) { // update lastAddedOrderId if the order was added in this set of changes (since it is now // the last) boolean isNewOrder = lastAddedOrderId != null && changes(MetadataUpdate.AddSortOrder.class) .anyMatch(added -> added.sortOrder().orderId() == lastAddedOrderId); this.lastAddedOrderId = isNewOrder ? newOrderId : null; return newOrderId; } Schema schema = schemasById.get(currentSchemaId); SortOrder.checkCompatibility(order, schema); SortOrder newOrder; if (order.isUnsorted()) { newOrder = SortOrder.unsorted(); } else { // rebuild the sort order using new column ids newOrder = freshSortOrder(newOrderId, schema, order); } sortOrders.add(newOrder); sortOrdersById.put(newOrderId, newOrder); changes.add(new MetadataUpdate.AddSortOrder(newOrder)); this.lastAddedOrderId = newOrderId; return newOrderId; } private int reuseOrCreateNewSortOrderId(SortOrder newOrder) { if (newOrder.isUnsorted()) { return SortOrder.unsorted().orderId(); } // determine the next order id int newOrderId = INITIAL_SORT_ORDER_ID; for (SortOrder order : sortOrders) { if (order.sameOrder(newOrder)) { return order.orderId(); } else if (newOrderId <= order.orderId()) { newOrderId = order.orderId() + 1; } } return newOrderId; } private void setBranchSnapshotInternal(Snapshot snapshot, String branch) { long replacementSnapshotId = snapshot.snapshotId(); SnapshotRef ref = refs.get(branch); if (ref != null) { ValidationException.check(ref.isBranch(), "Cannot update branch: %s is a tag", branch); if (ref.snapshotId() == replacementSnapshotId) { return; } } ValidationException.check( formatVersion == 1 || snapshot.sequenceNumber() <= lastSequenceNumber, "Last sequence number %s is less than existing snapshot sequence number %s", lastSequenceNumber, snapshot.sequenceNumber()); SnapshotRef newRef; if (ref != null) { newRef = SnapshotRef.builderFrom(ref, replacementSnapshotId).build(); } else { newRef = SnapshotRef.branchBuilder(replacementSnapshotId).build(); } setRef(branch, newRef); } private static List addPreviousFile( List previousFiles, String previousFileLocation, long timestampMillis, Map properties) { if (previousFileLocation == null) { return previousFiles; } int maxSize = Math.max( 1, PropertyUtil.propertyAsInt( properties, TableProperties.METADATA_PREVIOUS_VERSIONS_MAX, TableProperties.METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT)); List newMetadataLog; if (previousFiles.size() >= maxSize) { int removeIndex = previousFiles.size() - maxSize + 1; newMetadataLog = Lists.newArrayList(previousFiles.subList(removeIndex, previousFiles.size())); } else { newMetadataLog = Lists.newArrayList(previousFiles); } newMetadataLog.add(new MetadataLogEntry(timestampMillis, previousFileLocation)); return newMetadataLog; } /** * Finds intermediate snapshots that have not been committed as the current snapshot. * *

Transactions can create snapshots that are never the current snapshot because several * changes are combined by the transaction into one table metadata update. when each * intermediate snapshot is added to table metadata, it is added to the snapshot log, assuming * that it will be the current snapshot. when there are multiple snapshot updates, the log must * be corrected by suppressing the intermediate snapshot entries. * *

A snapshot is an intermediate snapshot if it was added but is not the current snapshot. * * @return a set of snapshot ids for all added snapshots that were later replaced as the current * snapshot in changes */ private static Set intermediateSnapshotIdSet( List changes, long currentSnapshotId) { Set addedSnapshotIds = Sets.newHashSet(); Set intermediateSnapshotIds = Sets.newHashSet(); for (MetadataUpdate update : changes) { if (update instanceof MetadataUpdate.AddSnapshot) { // adds must always come before set current snapshot MetadataUpdate.AddSnapshot addSnapshot = (MetadataUpdate.AddSnapshot) update; addedSnapshotIds.add(addSnapshot.snapshot().snapshotId()); } else if (update instanceof MetadataUpdate.SetSnapshotRef) { MetadataUpdate.SetSnapshotRef setRef = (MetadataUpdate.SetSnapshotRef) update; long snapshotId = setRef.snapshotId(); if (addedSnapshotIds.contains(snapshotId) && SnapshotRef.MAIN_BRANCH.equals(setRef.name()) && snapshotId != currentSnapshotId) { intermediateSnapshotIds.add(snapshotId); } } } return intermediateSnapshotIds; } private static List updateSnapshotLog( List snapshotLog, Map snapshotsById, long currentSnapshotId, List changes) { Set intermediateSnapshotIds = intermediateSnapshotIdSet(changes, currentSnapshotId); boolean hasIntermediateSnapshots = !intermediateSnapshotIds.isEmpty(); boolean hasRemovedSnapshots = changes.stream().anyMatch(change -> change instanceof MetadataUpdate.RemoveSnapshots); if (!hasIntermediateSnapshots && !hasRemovedSnapshots) { return snapshotLog; } // update the snapshot log List newSnapshotLog = Lists.newArrayList(); for (HistoryEntry logEntry : snapshotLog) { long snapshotId = logEntry.snapshotId(); if (snapshotsById.containsKey(snapshotId)) { if (!intermediateSnapshotIds.contains(snapshotId)) { // copy the log entries that are still valid newSnapshotLog.add(logEntry); } } else if (hasRemovedSnapshots) { // any invalid entry causes the history before it to be removed. otherwise, there could be // history gaps that cause time-travel queries to produce incorrect results. for example, // if history is [(t1, s1), (t2, s2), (t3, s3)] and s2 is removed, the history cannot be // [(t1, s1), (t3, s3)] because it appears that s3 was current during the time between t2 // and t3 when in fact s2 was the current snapshot. newSnapshotLog.clear(); } } if (snapshotsById.get(currentSnapshotId) != null) { ValidationException.check( Iterables.getLast(newSnapshotLog).snapshotId() == currentSnapshotId, "Cannot set invalid snapshot log: latest entry is not the current snapshot"); } return newSnapshotLog; } private static Map> indexStatistics(List files) { return files.stream().collect(Collectors.groupingBy(StatisticsFile::snapshotId)); } private static Map> indexPartitionStatistics( List files) { return files.stream().collect(Collectors.groupingBy(PartitionStatisticsFile::snapshotId)); } private boolean isAddedSnapshot(long snapshotId) { return changes(MetadataUpdate.AddSnapshot.class) .anyMatch(add -> add.snapshot().snapshotId() == snapshotId); } private Stream changes(Class updateClass) { return changes.stream().filter(updateClass::isInstance).map(updateClass::cast); } } } ================================================ FILE: icebergShaded/src/main/java/org/apache/iceberg/hive/HiveCatalog.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.iceberg.hive; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.UnknownDBException; import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.ClientPool; import org.apache.iceberg.MetadataUpdate; import org.apache.iceberg.Schema; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; import org.apache.iceberg.Transaction; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.*; import org.apache.iceberg.hadoop.HadoopFileIO; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.util.LocationUtil; import org.apache.iceberg.view.BaseMetastoreViewCatalog; import org.apache.iceberg.view.View; import org.apache.iceberg.view.ViewBuilder; import org.apache.iceberg.view.ViewMetadata; import org.apache.iceberg.view.ViewOperations; import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * This class is directly copied from iceberg 1.10.0; The only change made is * 1. * accept metadataUpdates in constructor and pass to HiveTableOperations * to support using schema/partitionSpec with field ids assigned by Delta lake * 2. * Validate metadataLocation for validating table as Iceberg in tableExists */ public class HiveCatalog extends BaseMetastoreViewCatalog implements SupportsNamespaces, Configurable { public static final String LIST_ALL_TABLES = "list-all-tables"; public static final String LIST_ALL_TABLES_DEFAULT = "false"; public static final String HMS_TABLE_OWNER = "hive.metastore.table.owner"; public static final String HMS_DB_OWNER = "hive.metastore.database.owner"; public static final String HMS_DB_OWNER_TYPE = "hive.metastore.database.owner-type"; // MetastoreConf is not available with current Hive version static final String HIVE_CONF_CATALOG = "metastore.catalog.default"; private static final Logger LOG = LoggerFactory.getLogger(HiveCatalog.class); private String name; private Configuration conf; private FileIO fileIO; private ClientPool clients; private boolean listAllTables = false; private Map catalogProperties; // HACK-HACK This is newly added private List metadataUpdates = new ArrayList(); public HiveCatalog() {} // HACK-HACK This is newly added public void initialize(String inputName, Map properties, List metadataUpdates) { initialize(inputName, properties); this.metadataUpdates = metadataUpdates; } @Override public void initialize(String inputName, Map properties) { this.catalogProperties = ImmutableMap.copyOf(properties); this.name = inputName; if (conf == null) { LOG.warn("No Hadoop Configuration was set, using the default environment Configuration"); this.conf = new Configuration(); } if (properties.containsKey(CatalogProperties.URI)) { this.conf.set(HiveConf.ConfVars.METASTOREURIS.varname, properties.get(CatalogProperties.URI)); } if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) { this.conf.set( HiveConf.ConfVars.METASTOREWAREHOUSE.varname, LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION))); } this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, LIST_ALL_TABLES_DEFAULT)); String fileIOImpl = properties.get(CatalogProperties.FILE_IO_IMPL); this.fileIO = fileIOImpl == null ? new HadoopFileIO(conf) : CatalogUtil.loadFileIO(fileIOImpl, properties, conf); this.clients = new CachedClientPool(conf, properties); } @Override public TableBuilder buildTable(TableIdentifier identifier, Schema schema) { return new ViewAwareTableBuilder(identifier, schema); } @Override public ViewBuilder buildView(TableIdentifier identifier) { return new TableAwareViewBuilder(identifier); } @Override public List listTables(Namespace namespace) { Preconditions.checkArgument( isValidateNamespace(namespace), "Missing database in namespace: %s", namespace); String database = namespace.level(0); try { List tableNames = clients.run(client -> client.getAllTables(database)); List tableIdentifiers; if (listAllTables) { tableIdentifiers = tableNames.stream() .map(t -> TableIdentifier.of(namespace, t)) .collect(Collectors.toList()); } else { tableIdentifiers = listIcebergTables( tableNames, namespace, BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE); } LOG.debug( "Listing of namespace: {} resulted in the following tables: {}", namespace, tableIdentifiers); return tableIdentifiers; } catch (UnknownDBException e) { throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace); } catch (TException e) { throw new RuntimeException("Failed to list all tables under namespace " + namespace, e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException("Interrupted in call to listTables", e); } } @Override public List listViews(Namespace namespace) { Preconditions.checkArgument( isValidateNamespace(namespace), "Missing database in namespace: %s", namespace); try { String database = namespace.level(0); List viewNames = clients.run(client -> client.getTables(database, "*", TableType.VIRTUAL_VIEW)); // Retrieving the Table objects from HMS in batches to avoid OOM List filteredTableIdentifiers = Lists.newArrayList(); Iterable> viewNameSets = Iterables.partition(viewNames, 100); for (List viewNameSet : viewNameSets) { filteredTableIdentifiers.addAll( listIcebergTables(viewNameSet, namespace, HiveOperationsBase.ICEBERG_VIEW_TYPE_VALUE)); } return filteredTableIdentifiers; } catch (UnknownDBException e) { throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace); } catch (TException e) { throw new RuntimeException("Failed to list all views under namespace " + namespace, e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException("Interrupted in call to listViews", e); } } @Override public String name() { return name; } @Override public boolean dropTable(TableIdentifier identifier, boolean purge) { if (!isValidIdentifier(identifier)) { return false; } String database = identifier.namespace().level(0); TableOperations ops = newTableOps(identifier); TableMetadata lastMetadata = null; if (purge) { try { lastMetadata = ops.current(); } catch (NotFoundException e) { LOG.warn( "Failed to load table metadata for table: {}, continuing drop without purge", identifier, e); } } try { clients.run( client -> { client.dropTable( database, identifier.name(), false /* do not delete data */, false /* throw NoSuchObjectException if the table doesn't exist */); return null; }); if (purge && lastMetadata != null) { CatalogUtil.dropTableData(ops.io(), lastMetadata); } LOG.info("Dropped table: {}", identifier); return true; } catch (NoSuchTableException | NoSuchObjectException e) { LOG.info("Skipping drop, table does not exist: {}", identifier, e); return false; } catch (TException e) { throw new RuntimeException("Failed to drop " + identifier, e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException("Interrupted in call to dropTable", e); } } @Override public boolean dropView(TableIdentifier identifier) { if (!isValidIdentifier(identifier)) { return false; } try { String database = identifier.namespace().level(0); String viewName = identifier.name(); HiveViewOperations ops = (HiveViewOperations) newViewOps(identifier); ViewMetadata lastViewMetadata = null; try { lastViewMetadata = ops.current(); } catch (NotFoundException e) { LOG.warn("Failed to load view metadata for view: {}", identifier, e); } clients.run( client -> { client.dropTable(database, viewName, false, false); return null; }); if (lastViewMetadata != null) { CatalogUtil.dropViewMetadata(ops.io(), lastViewMetadata); } LOG.info("Dropped view: {}", identifier); return true; } catch (NoSuchObjectException e) { LOG.info("Skipping drop, view does not exist: {}", identifier, e); return false; } catch (TException e) { throw new RuntimeException("Failed to drop view " + identifier, e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException("Interrupted in call to dropView", e); } } @Override public void renameTable(TableIdentifier from, TableIdentifier originalTo) { renameTableOrView(from, originalTo, HiveOperationsBase.ContentType.TABLE); } @Override public void renameView(TableIdentifier from, TableIdentifier to) { renameTableOrView(from, to, HiveOperationsBase.ContentType.VIEW); } private List listIcebergTables( List tableNames, Namespace namespace, String tableTypeProp) throws TException, InterruptedException { List

tableObjects = clients.run(client -> client.getTableObjectsByName(namespace.level(0), tableNames)); return tableObjects.stream() .filter( table -> table.getParameters() != null && tableTypeProp.equalsIgnoreCase( table.getParameters().get(BaseMetastoreTableOperations.TABLE_TYPE_PROP))) .map(table -> TableIdentifier.of(namespace, table.getTableName())) .collect(Collectors.toList()); } @SuppressWarnings("checkstyle:CyclomaticComplexity") private void renameTableOrView( TableIdentifier from, TableIdentifier originalTo, HiveOperationsBase.ContentType contentType) { Preconditions.checkArgument(isValidIdentifier(from), "Invalid identifier: %s", from); TableIdentifier to = removeCatalogName(originalTo); Preconditions.checkArgument(isValidIdentifier(to), "Invalid identifier: %s", to); if (!namespaceExists(to.namespace())) { throw new NoSuchNamespaceException( "Cannot rename %s to %s. Namespace does not exist: %s", from, to, to.namespace()); } if (tableExists(to)) { throw new org.apache.iceberg.exceptions.AlreadyExistsException( "Cannot rename %s to %s. Table already exists", from, to); } if (viewExists(to)) { throw new org.apache.iceberg.exceptions.AlreadyExistsException( "Cannot rename %s to %s. View already exists", from, to); } String toDatabase = to.namespace().level(0); String fromDatabase = from.namespace().level(0); String fromName = from.name(); try { Table table = clients.run(client -> client.getTable(fromDatabase, fromName)); validateTableIsIcebergTableOrView(contentType, table, CatalogUtil.fullTableName(name, from)); table.setDbName(toDatabase); table.setTableName(to.name()); clients.run( client -> { MetastoreUtil.alterTable(client, fromDatabase, fromName, table); return null; }); LOG.info("Renamed {} from {}, to {}", contentType.value(), from, to); } catch (NoSuchObjectException e) { switch (contentType) { case TABLE: throw new NoSuchTableException("Cannot rename %s to %s. Table does not exist", from, to); case VIEW: throw new NoSuchViewException("Cannot rename %s to %s. View does not exist", from, to); } } catch (InvalidOperationException e) { if (e.getMessage() != null && e.getMessage().contains(String.format("new table %s already exists", to))) { throw new org.apache.iceberg.exceptions.AlreadyExistsException( "Table already exists: %s", to); } else { throw new RuntimeException("Failed to rename " + from + " to " + to, e); } } catch (TException e) { throw new RuntimeException("Failed to rename " + from + " to " + to, e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException("Interrupted in call to rename", e); } } private void validateTableIsIcebergTableOrView( HiveOperationsBase.ContentType contentType, Table table, String fullName) { switch (contentType) { case TABLE: HiveOperationsBase.validateTableIsIceberg(table, fullName); break; case VIEW: HiveOperationsBase.validateTableIsIcebergView(table, fullName); } } /** * Check whether table or metadata table exists. * *

Note: If a hive table with the same identifier exists in catalog, this method will return * {@code false}. * * @param identifier a table identifier * @return true if the table exists, false otherwise */ @Override public boolean tableExists(TableIdentifier identifier) { TableIdentifier baseTableIdentifier = identifier; if (!isValidIdentifier(identifier)) { if (!isValidMetadataIdentifier(identifier)) { return false; } else { baseTableIdentifier = TableIdentifier.of(identifier.namespace().levels()); } } String database = baseTableIdentifier.namespace().level(0); String tableName = baseTableIdentifier.name(); try { Table table = clients.run(client -> client.getTable(database, tableName)); // HACK-HACK This is modified validateTableIsIceberg(table, fullTableName(name, baseTableIdentifier)); return true; } catch (NoSuchTableException | NoSuchObjectException e) { return false; } catch (TException e) { throw new RuntimeException("Failed to check table existence of " + baseTableIdentifier, e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException( "Interrupted in call to check table existence of " + baseTableIdentifier, e); } } // HACK-HACK This is added private void validateTableIsIceberg(Table table, String fullName) { HiveOperationsBase.validateTableIsIceberg(table, fullName); String metadataLocation = table.getParameters().get(BaseMetastoreTableOperations.METADATA_LOCATION_PROP); NoSuchIcebergTableException.check( metadataLocation != null, "Not an iceberg table: %s, metadataLocation is null", fullName); } @Override public boolean viewExists(TableIdentifier viewIdentifier) { if (!isValidIdentifier(viewIdentifier)) { return false; } String database = viewIdentifier.namespace().level(0); String viewName = viewIdentifier.name(); try { Table table = clients.run(client -> client.getTable(database, viewName)); HiveOperationsBase.validateTableIsIcebergView(table, fullTableName(name, viewIdentifier)); return true; } catch (NoSuchIcebergViewException | NoSuchObjectException e) { return false; } catch (TException e) { throw new RuntimeException("Failed to check view existence of " + viewIdentifier, e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException( "Interrupted in call to check view existence of " + viewIdentifier, e); } } @Override public void createNamespace(Namespace namespace, Map meta) { Preconditions.checkArgument( !namespace.isEmpty(), "Cannot create namespace with invalid name: %s", namespace); Preconditions.checkArgument( isValidateNamespace(namespace), "Cannot support multi part namespace in Hive Metastore: %s", namespace); Preconditions.checkArgument( meta.get(HMS_DB_OWNER_TYPE) == null || meta.get(HMS_DB_OWNER) != null, "Create namespace setting %s without setting %s is not allowed", HMS_DB_OWNER_TYPE, HMS_DB_OWNER); try { clients.run( client -> { client.createDatabase(convertToDatabase(namespace, meta)); return null; }); LOG.info("Created namespace: {}", namespace); } catch (AlreadyExistsException e) { throw new org.apache.iceberg.exceptions.AlreadyExistsException( e, "Namespace already exists: %s", namespace); } catch (TException e) { throw new RuntimeException( "Failed to create namespace " + namespace + " in Hive Metastore", e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException( "Interrupted in call to createDatabase(name) " + namespace + " in Hive Metastore", e); } } @Override public List listNamespaces(Namespace namespace) { if (!namespace.isEmpty() && (!isValidateNamespace(namespace) || !namespaceExists(namespace))) { throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace); } if (!namespace.isEmpty()) { return ImmutableList.of(); } try { List namespaces = clients.run(IMetaStoreClient::getAllDatabases).stream() .map(Namespace::of) .collect(Collectors.toList()); LOG.debug("Listing namespace {} returned tables: {}", namespace, namespaces); return namespaces; } catch (TException e) { throw new RuntimeException( "Failed to list all namespace: " + namespace + " in Hive Metastore", e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException( "Interrupted in call to getAllDatabases() " + namespace + " in Hive Metastore", e); } } @Override public boolean dropNamespace(Namespace namespace) { if (!isValidateNamespace(namespace)) { return false; } try { clients.run( client -> { client.dropDatabase( namespace.level(0), false /* deleteData */, false /* ignoreUnknownDb */, false /* cascade */); return null; }); LOG.info("Dropped namespace: {}", namespace); return true; } catch (InvalidOperationException e) { throw new NamespaceNotEmptyException( e, "Namespace %s is not empty. One or more tables exist.", namespace); } catch (NoSuchObjectException e) { return false; } catch (TException e) { throw new RuntimeException("Failed to drop namespace " + namespace + " in Hive Metastore", e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException( "Interrupted in call to drop dropDatabase(name) " + namespace + " in Hive Metastore", e); } } @Override public boolean setProperties(Namespace namespace, Map properties) { Preconditions.checkArgument( (properties.get(HMS_DB_OWNER_TYPE) == null) == (properties.get(HMS_DB_OWNER) == null), "Setting %s and %s has to be performed together or not at all", HMS_DB_OWNER_TYPE, HMS_DB_OWNER); Map parameter = Maps.newHashMap(); parameter.putAll(loadNamespaceMetadata(namespace)); parameter.putAll(properties); Database database = convertToDatabase(namespace, parameter); alterHiveDataBase(namespace, database); LOG.debug("Successfully set properties {} for {}", properties.keySet(), namespace); // Always successful, otherwise exception is thrown return true; } @Override public boolean removeProperties(Namespace namespace, Set properties) { Preconditions.checkArgument( properties.contains(HMS_DB_OWNER_TYPE) == properties.contains(HMS_DB_OWNER), "Removing %s and %s has to be performed together or not at all", HMS_DB_OWNER_TYPE, HMS_DB_OWNER); Map parameter = Maps.newHashMap(); parameter.putAll(loadNamespaceMetadata(namespace)); properties.forEach(key -> parameter.put(key, null)); Database database = convertToDatabase(namespace, parameter); alterHiveDataBase(namespace, database); LOG.debug("Successfully removed properties {} from {}", properties, namespace); // Always successful, otherwise exception is thrown return true; } private void alterHiveDataBase(Namespace namespace, Database database) { try { clients.run( client -> { client.alterDatabase(namespace.level(0), database); return null; }); } catch (NoSuchObjectException | UnknownDBException e) { throw new NoSuchNamespaceException(e, "Namespace does not exist: %s", namespace); } catch (TException e) { throw new RuntimeException( "Failed to list namespace under namespace: " + namespace + " in Hive Metastore", e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException( "Interrupted in call to getDatabase(name) " + namespace + " in Hive Metastore", e); } } @Override public Map loadNamespaceMetadata(Namespace namespace) { if (!isValidateNamespace(namespace)) { throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace); } try { Database database = clients.run(client -> client.getDatabase(namespace.level(0))); Map metadata = convertToMetadata(database); LOG.debug("Loaded metadata for namespace {} found {}", namespace, metadata.keySet()); return metadata; } catch (NoSuchObjectException | UnknownDBException e) { throw new NoSuchNamespaceException(e, "Namespace does not exist: %s", namespace); } catch (TException e) { throw new RuntimeException( "Failed to list namespace under namespace: " + namespace + " in Hive Metastore", e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException( "Interrupted in call to getDatabase(name) " + namespace + " in Hive Metastore", e); } } @Override protected boolean isValidIdentifier(TableIdentifier tableIdentifier) { return tableIdentifier.namespace().levels().length == 1; } private TableIdentifier removeCatalogName(TableIdentifier to) { if (isValidIdentifier(to)) { return to; } // check if the identifier includes the catalog name and remove it if (to.namespace().levels().length == 2 && name().equalsIgnoreCase(to.namespace().level(0))) { return TableIdentifier.of(Namespace.of(to.namespace().level(1)), to.name()); } // return the original unmodified return to; } private boolean isValidateNamespace(Namespace namespace) { return namespace.levels().length == 1; } @Override public TableOperations newTableOps(TableIdentifier tableIdentifier) { String dbName = tableIdentifier.namespace().level(0); String tableName = tableIdentifier.name(); // HACK-HACK This is modified return new HiveTableOperations(conf, clients, fileIO, name, dbName, tableName, metadataUpdates); } @Override protected ViewOperations newViewOps(TableIdentifier identifier) { return new HiveViewOperations(conf, clients, fileIO, name, identifier); } @Override protected String defaultWarehouseLocation(TableIdentifier tableIdentifier) { // This is a little edgy since we basically duplicate the HMS location generation logic. // Sadly I do not see a good way around this if we want to keep the order of events, like: // - Create meta files // - Create the metadata in HMS, and this way committing the changes // Create a new location based on the namespace / database if it is set on database level try { Database databaseData = clients.run(client -> client.getDatabase(tableIdentifier.namespace().levels()[0])); if (databaseData.getLocationUri() != null) { // If the database location is set use it as a base. return String.format("%s/%s", databaseData.getLocationUri(), tableIdentifier.name()); } } catch (NoSuchObjectException e) { throw new NoSuchNamespaceException( e, "Namespace does not exist: %s", tableIdentifier.namespace().levels()[0]); } catch (TException e) { throw new RuntimeException( String.format("Metastore operation failed for %s", tableIdentifier), e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException("Interrupted during commit", e); } // Otherwise, stick to the {WAREHOUSE_DIR}/{DB_NAME}.db/{TABLE_NAME} path String databaseLocation = databaseLocation(tableIdentifier.namespace().levels()[0]); return String.format("%s/%s", databaseLocation, tableIdentifier.name()); } private String databaseLocation(String databaseName) { String warehouseLocation = conf.get(HiveConf.ConfVars.METASTOREWAREHOUSE.varname); Preconditions.checkNotNull( warehouseLocation, "Warehouse location is not set: hive.metastore.warehouse.dir=null"); warehouseLocation = LocationUtil.stripTrailingSlash(warehouseLocation); return String.format("%s/%s.db", warehouseLocation, databaseName); } private Map convertToMetadata(Database database) { Map meta = Maps.newHashMap(); meta.putAll(database.getParameters()); meta.put("location", database.getLocationUri()); if (database.getDescription() != null) { meta.put("comment", database.getDescription()); } if (database.getOwnerName() != null) { meta.put(HMS_DB_OWNER, database.getOwnerName()); if (database.getOwnerType() != null) { meta.put(HMS_DB_OWNER_TYPE, database.getOwnerType().name()); } } return meta; } Database convertToDatabase(Namespace namespace, Map meta) { if (!isValidateNamespace(namespace)) { throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace); } Database database = new Database(); Map parameter = Maps.newHashMap(); database.setName(namespace.level(0)); database.setLocationUri(databaseLocation(namespace.level(0))); meta.forEach( (key, value) -> { if (key.equals("comment")) { database.setDescription(value); } else if (key.equals("location")) { database.setLocationUri(value); } else if (key.equals(HMS_DB_OWNER)) { database.setOwnerName(value); } else if (key.equals(HMS_DB_OWNER_TYPE) && value != null) { database.setOwnerType(PrincipalType.valueOf(value)); } else { if (value != null) { parameter.put(key, value); } } }); if (database.getOwnerName() == null) { database.setOwnerName(HiveHadoopUtil.currentUser()); database.setOwnerType(PrincipalType.USER); } database.setParameters(parameter); return database; } @Override public String toString() { return MoreObjects.toStringHelper(this) .add("name", name) .add("uri", this.conf == null ? "" : this.conf.get(HiveConf.ConfVars.METASTOREURIS.varname)) .toString(); } @Override public void setConf(Configuration conf) { this.conf = new Configuration(conf); } @Override public Configuration getConf() { return conf; } @Override protected Map properties() { return catalogProperties == null ? ImmutableMap.of() : catalogProperties; } @VisibleForTesting void setListAllTables(boolean listAllTables) { this.listAllTables = listAllTables; } @VisibleForTesting ClientPool clientPool() { return clients; } /** * The purpose of this class is to add view detection only for Hive-Specific tables. Hive catalog * follows checks at different levels: 1. During refresh, it validates if the table is an iceberg * table or not. 2. During commit, it validates if there is any concurrent commit with table or * table-name already exists. This class helps to do the validation on an early basis. */ private class ViewAwareTableBuilder extends BaseMetastoreViewCatalogTableBuilder { private final TableIdentifier identifier; private ViewAwareTableBuilder(TableIdentifier identifier, Schema schema) { super(identifier, schema); this.identifier = identifier; } @Override public Transaction createOrReplaceTransaction() { if (viewExists(identifier)) { throw new org.apache.iceberg.exceptions.AlreadyExistsException( "View with same name already exists: %s", identifier); } return super.createOrReplaceTransaction(); } @Override public org.apache.iceberg.Table create() { if (viewExists(identifier)) { throw new org.apache.iceberg.exceptions.AlreadyExistsException( "View with same name already exists: %s", identifier); } return super.create(); } } /** * The purpose of this class is to add table detection only for Hive-Specific view. Hive catalog * follows checks at different levels: 1. During refresh, it validates if the view is an iceberg * view or not. 2. During commit, it validates if there is any concurrent commit with view or * view-name already exists. This class helps to do the validation on an early basis. */ private class TableAwareViewBuilder extends BaseViewBuilder { private final TableIdentifier identifier; private TableAwareViewBuilder(TableIdentifier identifier) { super(identifier); this.identifier = identifier; } @Override public View createOrReplace() { if (tableExists(identifier)) { throw new org.apache.iceberg.exceptions.AlreadyExistsException( "Table with same name already exists: %s", identifier); } return super.createOrReplace(); } @Override public View create() { if (tableExists(identifier)) { throw new org.apache.iceberg.exceptions.AlreadyExistsException( "Table with same name already exists: %s", identifier); } return super.create(); } } } ================================================ FILE: icebergShaded/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.iceberg.hive; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.iceberg.BaseMetastoreOperations; import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.ClientPool; import org.apache.iceberg.MetadataUpdate; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableProperties; import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.exceptions.CommitStateUnknownException; import org.apache.iceberg.exceptions.NoSuchIcebergTableException; import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.hadoop.ConfigProperties; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * TODO we should be able to extract some more commonalities to BaseMetastoreTableOperations to * avoid code duplication between this class and Metacat Tables. * This class is directly copied from iceberg 1.10.0; The only change made are * 1) accept metadataUpdates in constructor apply those before writing metadata * to support using schema/partitionSpec with field ids assigned by Delta lake; * 2) handle NoSuchIcebergTableException in doRefresh to regard a table entry * that exists in HMS but does not have "table_type" = "ICEBERG" as table does * not exist, so Delta lake can correctly start create table transaction */ public class HiveTableOperations extends BaseMetastoreTableOperations implements HiveOperationsBase { private static final Logger LOG = LoggerFactory.getLogger(HiveTableOperations.class); private static final String HIVE_ICEBERG_METADATA_REFRESH_MAX_RETRIES = "iceberg.hive.metadata-refresh-max-retries"; private static final int HIVE_ICEBERG_METADATA_REFRESH_MAX_RETRIES_DEFAULT = 2; private final String fullName; private final String catalogName; private final String database; private final String tableName; private final Configuration conf; private final long maxHiveTablePropertySize; private final int metadataRefreshMaxRetries; private final FileIO fileIO; private final ClientPool metaClients; // HACK-HACK This is newly added private List metadataUpdates = new ArrayList(); // HACK-HACK This is newly added protected HiveTableOperations( Configuration conf, ClientPool metaClients, FileIO fileIO, String catalogName, String database, String table, List metadataUpdates) { this(conf, metaClients, fileIO, catalogName, database, table); this.metadataUpdates = metadataUpdates; } protected HiveTableOperations( Configuration conf, ClientPool metaClients, FileIO fileIO, String catalogName, String database, String table) { this.conf = conf; this.metaClients = metaClients; this.fileIO = fileIO; this.fullName = catalogName + "." + database + "." + table; this.catalogName = catalogName; this.database = database; this.tableName = table; this.metadataRefreshMaxRetries = conf.getInt( HIVE_ICEBERG_METADATA_REFRESH_MAX_RETRIES, HIVE_ICEBERG_METADATA_REFRESH_MAX_RETRIES_DEFAULT); this.maxHiveTablePropertySize = conf.getLong(HIVE_TABLE_PROPERTY_MAX_SIZE, HIVE_TABLE_PROPERTY_MAX_SIZE_DEFAULT); } @Override protected String tableName() { return fullName; } @Override public FileIO io() { return fileIO; } @Override protected void doRefresh() { String metadataLocation = null; try { Table table = metaClients.run(client -> client.getTable(database, tableName)); // Check if we are trying to load an Iceberg View as a Table HiveOperationsBase.validateIcebergViewNotLoadedAsIcebergTable(table, fullName); // Check if it is a valid Iceberg Table HiveOperationsBase.validateTableIsIceberg(table, fullName); metadataLocation = table.getParameters().get(METADATA_LOCATION_PROP); } catch (NoSuchObjectException e) { if (currentMetadataLocation() != null) { throw new NoSuchTableException("No such table: %s.%s", database, tableName); } } catch (NoSuchIcebergTableException e) { // HACK-HACK This is newly added // NoSuchIcebergTableException is throw when table exists in catalog but not with // table_type=iceberg; in that case we want to swallow so createTable // txn can proceed with creating the iceberg table/metadata and set table_type=iceberg if (currentMetadataLocation() != null) { throw new NoSuchTableException("No such table: %s.%s", database, tableName); } } catch (TException e) { String errMsg = String.format("Failed to get table info from metastore %s.%s", database, tableName); throw new RuntimeException(errMsg, e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException("Interrupted during refresh", e); } refreshFromMetadataLocation(metadataLocation, metadataRefreshMaxRetries); } @SuppressWarnings({"checkstyle:CyclomaticComplexity", "MethodLength"}) @Override protected void doCommit(TableMetadata base, TableMetadata metadata) { boolean newTable = base == null; // HACK-HACK This is newly added // Apply metadata updates so adjustedMetadata has field id and partition spec created // from Delta lake TableMetadata.Builder builder = TableMetadata.buildFrom(metadata); Schema lastAddedSchema = metadata.schema(); for (MetadataUpdate update : metadataUpdates) { if (update instanceof MetadataUpdate.AddSchema) { MetadataUpdate.AddSchema addSchema = (MetadataUpdate.AddSchema) update; builder.setCurrentSchema(addSchema.schema(), addSchema.lastColumnId()); lastAddedSchema = addSchema.schema(); } else if (update instanceof MetadataUpdate.AddPartitionSpec) { // regard AddPartitionSpec as replace all existing specs as Delta Uniform only // support one partition spec PartitionSpec specToAdd = ((MetadataUpdate.AddPartitionSpec) update).spec().bind(lastAddedSchema); if (!specToAdd.compatibleWith(metadata.spec())) { HashSet idsToRemove = new HashSet(); for (PartitionSpec spec : metadata.specs()) { idsToRemove.add(spec.specId()); } builder.setDefaultPartitionSpec(specToAdd); MetadataUpdate.RemovePartitionSpecs removeSpecs = new MetadataUpdate.RemovePartitionSpecs(idsToRemove); removeSpecs.applyTo(builder); } } else { update.applyTo(builder); } } TableMetadata adjustedMetadata = builder.build(); // HACK-HACK This is modified String newMetadataLocation = writeNewMetadataIfRequired(newTable, adjustedMetadata); boolean hiveEngineEnabled = hiveEngineEnabled(metadata, conf); boolean keepHiveStats = conf.getBoolean(ConfigProperties.KEEP_HIVE_STATS, false); BaseMetastoreOperations.CommitStatus commitStatus = BaseMetastoreOperations.CommitStatus.FAILURE; boolean updateHiveTable = false; HiveLock lock = lockObject(base); try { lock.lock(); Table tbl = loadHmsTable(); if (tbl != null) { // If we try to create the table but the metadata location is already set, then we had a // concurrent commit if (newTable && tbl.getParameters().get(BaseMetastoreTableOperations.METADATA_LOCATION_PROP) != null) { if (TableType.VIRTUAL_VIEW.name().equalsIgnoreCase(tbl.getTableType())) { throw new AlreadyExistsException( "View with same name already exists: %s.%s", database, tableName); } throw new AlreadyExistsException("Table already exists: %s.%s", database, tableName); } updateHiveTable = true; LOG.debug("Committing existing table: {}", fullName); } else { tbl = newHmsTable( metadata.property(HiveCatalog.HMS_TABLE_OWNER, HiveHadoopUtil.currentUser())); LOG.debug("Committing new table: {}", fullName); } // HACK-HACK This is newely added StorageDescriptor newsd = HiveOperationsBase.storageDescriptor( adjustedMetadata.schema(), adjustedMetadata.location(), hiveEngineEnabled); // HACK-HACK This is newely added: use storage descriptor from Delta newsd.getSerdeInfo().setParameters(tbl.getSd().getSerdeInfo().getParameters()); tbl.setSd(newsd); // HACK-HACK This is newely added: set schema to be empty to match Delta behavior tbl.getSd().setCols(Collections.singletonList(new FieldSchema("col", "array", ""))); String metadataLocation = tbl.getParameters().get(METADATA_LOCATION_PROP); String baseMetadataLocation = base != null ? base.metadataFileLocation() : null; if (!Objects.equals(baseMetadataLocation, metadataLocation)) { throw new CommitFailedException( "Cannot commit: Base metadata location '%s' is not same as the current table metadata location '%s' for %s.%s", baseMetadataLocation, metadataLocation, database, tableName); } // get Iceberg props that have been removed Set removedProps = Collections.emptySet(); if (base != null) { removedProps = base.properties().keySet().stream() .filter(key -> !metadata.properties().containsKey(key)) .collect(Collectors.toSet()); } HMSTablePropertyHelper.updateHmsTableForIcebergTable( newMetadataLocation, tbl, metadata, removedProps, hiveEngineEnabled, maxHiveTablePropertySize, currentMetadataLocation()); if (!keepHiveStats) { tbl.getParameters().remove(StatsSetupConst.COLUMN_STATS_ACCURATE); tbl.getParameters().put(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE); } lock.ensureActive(); try { persistTable( tbl, updateHiveTable, hiveLockEnabled(base, conf) ? null : baseMetadataLocation); lock.ensureActive(); commitStatus = BaseMetastoreOperations.CommitStatus.SUCCESS; } catch (LockException le) { commitStatus = BaseMetastoreOperations.CommitStatus.UNKNOWN; throw new CommitStateUnknownException( "Failed to heartbeat for hive lock while " + "committing changes. This can lead to a concurrent commit attempt be able to overwrite this commit. " + "Please check the commit history. If you are running into this issue, try reducing " + "iceberg.hive.lock-heartbeat-interval-ms.", le); } catch (org.apache.hadoop.hive.metastore.api.AlreadyExistsException e) { throw new AlreadyExistsException(e, "Table already exists: %s.%s", database, tableName); } catch (InvalidObjectException e) { throw new ValidationException(e, "Invalid Hive object for %s.%s", database, tableName); } catch (CommitFailedException | CommitStateUnknownException e) { throw e; } catch (Throwable e) { if (e.getMessage() != null && e.getMessage().contains("Table/View 'HIVE_LOCKS' does not exist")) { throw new RuntimeException( "Failed to acquire locks from metastore because the underlying metastore " + "table 'HIVE_LOCKS' does not exist. This can occur when using an embedded metastore which does not " + "support transactions. To fix this use an alternative metastore.", e); } commitStatus = BaseMetastoreOperations.CommitStatus.UNKNOWN; if (e.getMessage() != null && e.getMessage() .contains( "The table has been modified. The parameter value for key '" + HiveTableOperations.METADATA_LOCATION_PROP + "' is")) { // It's possible the HMS client incorrectly retries a successful operation, due to network // issue for example, and triggers this exception. So we need double-check to make sure // this is really a concurrent modification. Hitting this exception means no pending // requests, if any, can succeed later, so it's safe to check status in strict mode commitStatus = checkCommitStatusStrict(newMetadataLocation, metadata); if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) { throw new CommitFailedException( e, "The table %s.%s has been modified concurrently", database, tableName); } } else { LOG.error( "Cannot tell if commit to {}.{} succeeded, attempting to reconnect and check.", database, tableName, e); commitStatus = checkCommitStatus(newMetadataLocation, metadata); } switch (commitStatus) { case SUCCESS: break; case FAILURE: throw e; case UNKNOWN: throw new CommitStateUnknownException(e); } } } catch (TException e) { throw new RuntimeException( String.format("Metastore operation failed for %s.%s", database, tableName), e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException("Interrupted during commit", e); } catch (LockException e) { throw new CommitFailedException(e); } finally { HiveOperationsBase.cleanupMetadataAndUnlock(io(), commitStatus, newMetadataLocation, lock); } LOG.info( "Committed to table {} with the new metadata location {}", fullName, newMetadataLocation); } @Override public long maxHiveTablePropertySize() { return maxHiveTablePropertySize; } @Override public String database() { return database; } @Override public String table() { return tableName; } @Override public TableType tableType() { return TableType.EXTERNAL_TABLE; } @Override public ClientPool metaClients() { return metaClients; } /** * Returns if the hive engine related values should be enabled on the table, or not. * *

The decision is made like this: * *

    *
  1. Table property value {@link TableProperties#ENGINE_HIVE_ENABLED} *
  2. If the table property is not set then check the hive-site.xml property value {@link * ConfigProperties#ENGINE_HIVE_ENABLED} *
  3. If none of the above is enabled then use the default value {@link * TableProperties#ENGINE_HIVE_ENABLED_DEFAULT} *
* * @param metadata Table metadata to use * @param conf The hive configuration to use * @return if the hive engine related values should be enabled or not */ private static boolean hiveEngineEnabled(TableMetadata metadata, Configuration conf) { if (metadata.properties().get(TableProperties.ENGINE_HIVE_ENABLED) != null) { // We know that the property is set, so default value will not be used, return metadata.propertyAsBoolean(TableProperties.ENGINE_HIVE_ENABLED, false); } return conf.getBoolean( ConfigProperties.ENGINE_HIVE_ENABLED, TableProperties.ENGINE_HIVE_ENABLED_DEFAULT); } /** * Returns if the hive locking should be enabled on the table, or not. * *

The decision is made like this: * *

    *
  1. Table property value {@link TableProperties#HIVE_LOCK_ENABLED} *
  2. If the table property is not set then check the hive-site.xml property value {@link * ConfigProperties#LOCK_HIVE_ENABLED} *
  3. If none of the above is enabled then use the default value {@link * TableProperties#HIVE_LOCK_ENABLED_DEFAULT} *
* * @param metadata Table metadata to use * @param conf The hive configuration to use * @return if the hive engine related values should be enabled or not */ private static boolean hiveLockEnabled(TableMetadata metadata, Configuration conf) { if (metadata != null && metadata.properties().get(TableProperties.HIVE_LOCK_ENABLED) != null) { // We know that the property is set, so default value will not be used, return metadata.propertyAsBoolean(TableProperties.HIVE_LOCK_ENABLED, false); } return conf.getBoolean( ConfigProperties.LOCK_HIVE_ENABLED, TableProperties.HIVE_LOCK_ENABLED_DEFAULT); } @VisibleForTesting HiveLock lockObject(TableMetadata metadata) { if (hiveLockEnabled(metadata, conf)) { return new MetastoreLock(conf, metaClients, catalogName, database, tableName); } else { return new NoLock(); } } } ================================================ FILE: icebergShaded/src/main/java/org/apache/iceberg/rest/RESTFileScanTaskParser.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ /* * DELTA LAKE PATCH: * This file is patched from Apache Iceberg 1.10.0 to fix a bug in the fromJson method. * * Bug: The original code crashes with NoSuchElementException when parsing JSON responses * containing empty delete-file-references arrays (delete-file-references: []). * * This occurs because Collections.max(indices) throws NoSuchElementException on empty * collections, but empty delete-file-references arrays are valid per the Iceberg REST spec. * * Fix: Added check for indices.isEmpty() before calling Collections.max() at line 93. * * Without this patch, the Delta Lake server-side scan planning client cannot parse responses * for tables with data files, as the Iceberg REST server always includes the * delete-file-references field even when empty. */ package org.apache.iceberg.rest; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonNode; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; import org.apache.iceberg.BaseFileScanTask; import org.apache.iceberg.ContentFileParser; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.SchemaParser; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.ExpressionParser; import org.apache.iceberg.expressions.ResidualEvaluator; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.util.JsonUtil; class RESTFileScanTaskParser { private static final String DATA_FILE = "data-file"; private static final String DELETE_FILE_REFERENCES = "delete-file-references"; private static final String RESIDUAL_FILTER = "residual-filter"; private RESTFileScanTaskParser() {} public static void toJson( FileScanTask fileScanTask, Set deleteFileReferences, PartitionSpec partitionSpec, JsonGenerator generator) throws IOException { Preconditions.checkArgument(fileScanTask != null, "Invalid file scan task: null"); Preconditions.checkArgument(generator != null, "Invalid JSON generator: null"); generator.writeStartObject(); generator.writeFieldName(DATA_FILE); ContentFileParser.toJson(fileScanTask.file(), partitionSpec, generator); if (deleteFileReferences != null) { JsonUtil.writeIntegerArray(DELETE_FILE_REFERENCES, deleteFileReferences, generator); } if (fileScanTask.residual() != null) { generator.writeFieldName(RESIDUAL_FILTER); ExpressionParser.toJson(fileScanTask.residual(), generator); } generator.writeEndObject(); } public static FileScanTask fromJson( JsonNode jsonNode, List allDeleteFiles, Map specsById, boolean isCaseSensitive) { Preconditions.checkArgument(jsonNode != null, "Invalid JSON node for file scan task: null"); Preconditions.checkArgument( jsonNode.isObject(), "Invalid JSON node for file scan task: non-object (%s)", jsonNode); DataFile dataFile = (DataFile) ContentFileParser.fromJson(JsonUtil.get(DATA_FILE, jsonNode), specsById); int specId = dataFile.specId(); DeleteFile[] deleteFiles = null; if (jsonNode.has(DELETE_FILE_REFERENCES)) { List indices = JsonUtil.getIntegerList(DELETE_FILE_REFERENCES, jsonNode); // DELTA LAKE PATCH: Added indices.isEmpty() check to fix NoSuchElementException // when delete-file-references is an empty array Preconditions.checkArgument( indices.isEmpty() || Collections.max(indices) < allDeleteFiles.size(), "Invalid delete file references: %s, expected indices < %s", indices, allDeleteFiles.size()); deleteFiles = indices.stream().map(allDeleteFiles::get).toArray(DeleteFile[]::new); } Expression filter = null; if (jsonNode.has(RESIDUAL_FILTER)) { filter = ExpressionParser.fromJson(jsonNode.get(RESIDUAL_FILTER)); } String schemaString = SchemaParser.toJson(specsById.get(specId).schema()); String specString = PartitionSpecParser.toJson(specsById.get(specId)); ResidualEvaluator boundResidual = ResidualEvaluator.of(specsById.get(specId), filter, isCaseSensitive); return new BaseFileScanTask(dataFile, deleteFiles, schemaString, specString, boundResidual); } } ================================================ FILE: icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/UnityCatalog.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.iceberg.unityCatalog; import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; import org.apache.iceberg.BaseMetastoreCatalog; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.MetadataUpdate; import org.apache.iceberg.TableOperations; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.hadoop.Configurable; import org.apache.iceberg.io.CloseableGroup; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.exceptions.NotFoundException; /** * UnityCatalog manages delta table with iceberg metadata conversion in unity catalog * Only newTableOps needs implementation as it is used to commit to Iceberg. All other methods are * not required (eg, listTables, dropTable, renameTable) because the tables are managed by * unity catalog outside of iceberg context. */ public class UnityCatalog extends BaseMetastoreCatalog implements Closeable, Configurable { private static final String DEFAULT_FILE_IO_IMPL = "org.apache.iceberg.hadoop.HadoopFileIO"; // Injectable factory for testing purposes. static class FileIOFactory { public FileIO newFileIO(String impl, Map properties, Object hadoopConf) { return CatalogUtil.loadFileIO(impl, properties, hadoopConf); } } private Object conf; private Map catalogProperties; private FileIOFactory fileIOFactory; private CloseableGroup closeableGroup; private List metadataUpdates; // If set, all table option in this catalog will be built based on the snapshot baseMetadataLocation points to. private final Optional baseMetadataLocation; public UnityCatalog( List metadataUpdates , Optional baseMetadataLocation ) { this.metadataUpdates = metadataUpdates; this.baseMetadataLocation = baseMetadataLocation; } @Override protected TableOperations newTableOps(TableIdentifier tableIdentifier) { FileIO fileIO = fileIOFactory.newFileIO(DEFAULT_FILE_IO_IMPL, catalogProperties, conf); closeableGroup.addCloseable(fileIO); return new UnityCatalogTableOperations( fileIO , tableIdentifier , metadataUpdates , baseMetadataLocation ); } @Override public void initialize(String name, Map properties) { this.catalogProperties = properties; this.fileIOFactory = new FileIOFactory(); this.closeableGroup = new CloseableGroup(); } @Override public void close() throws IOException { if (closeableGroup != null) { closeableGroup.close(); } } @Override protected String defaultWarehouseLocation(TableIdentifier tableIdentifier) { throw new UnsupportedOperationException( "UnityCatalog does not currently support defaultWarehouseLocation"); } @Override public void setConf(Object conf) { this.conf = conf; } @Override public List listTables(Namespace namespace) { throw new UnsupportedOperationException("UnityCatalog does not currently support listTables"); } @Override public boolean dropTable(TableIdentifier identifier, boolean purge) { throw new UnsupportedOperationException("UnityCatalog does not currently support dropTable"); } @Override public void renameTable(TableIdentifier from, TableIdentifier to) { throw new UnsupportedOperationException("UnityCatalog does not currently support renameTable"); } // If the given metadataLocation is invalid, we also see it as the table doesn't exist @Override public boolean tableExists(TableIdentifier identifier) { try { loadTable(identifier); return true; } catch (NoSuchTableException | NotFoundException e) { return false; } } } ================================================ FILE: icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/UnityCatalogTableOperations.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.iceberg.unityCatalog; import java.net.URI; import java.net.URISyntaxException; import java.time.Instant; import java.util.*; import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.MetadataUpdate; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableProperties; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import scala.Option; import scala.Tuple2; /** * UnityCatalogTableOperations supports load and commit to Iceberg metadata through unity catalog */ public class UnityCatalogTableOperations extends BaseMetastoreTableOperations { private static final Logger LOG = LoggerFactory.getLogger(UnityCatalogTableOperations.class); /** * Property to be set in translated Iceberg metadata files. * Indicates the delta commit version # that it corresponds to. */ public static final String DELTA_VERSION_PROPERTY = "delta-version"; public static final String DELTA_TIMESTAMP_PROPERTY = "delta-timestamp"; public static final String BASE_DELTA_VERSION_PROPERTY = "base-delta-version"; public static final String DELTA_HIGH_WATER_MARK_PROPERTY = "delta-high-water-mark"; public static final String DELTA_SQL_CONF_BYPASS_SEQUENCE_NUMBER_CHECK = "spark.databricks.delta.uniform.bypassSnapshotSequenceNumberCheck"; public static final String DELTA_SQL_CONF_BYPASS_FORMAT_VERSION_DOWNGRAGDE_CHECK = "spark.databricks.delta.uniform.bypassFormatVersionDowngradeCheck"; private final FileIO fileIO; private final TableIdentifier tableIdentifier; private final Optional baseMetadataLocation; private List metadataUpdates; private Optional> lastWrittenTableMetadataWithLocation; public UnityCatalogTableOperations( FileIO fileIO , TableIdentifier tableIdentifier , List metadataUpdates , Optional baseMetadataLocation ) { this.fileIO = fileIO; this.tableIdentifier = tableIdentifier; this.metadataUpdates = metadataUpdates; this.baseMetadataLocation = baseMetadataLocation; this.lastWrittenTableMetadataWithLocation = Optional.empty(); } @Override public FileIO io() { return fileIO; } @Override protected String tableName() { return tableIdentifier.toString(); } @Override public void doCommit(TableMetadata base, TableMetadata metadata) { TableMetadata.Builder builder = TableMetadata.buildFrom(metadata); // hasAddPartitionSpec indicates if the current commit attempt have added a new // partition spec through metadata update. boolean hasAddPartitionSpec = false; boolean initialPartitionSpecExistsAndIsPartitioned = false; for (PartitionSpec spec : metadata.specs()) { if (spec.specId() == 0 && spec.isPartitioned()) { initialPartitionSpecExistsAndIsPartitioned = true; } } Long deltaVersion = Long.parseLong( metadata.properties().get(DELTA_VERSION_PROPERTY)); String baseDeltaVersionStr = metadata.properties().get(BASE_DELTA_VERSION_PROPERTY); String deltaHighWaterMarkStr = metadata.properties().get(DELTA_HIGH_WATER_MARK_PROPERTY); Schema lastAddedSchema = metadata.schema(); for (MetadataUpdate update : metadataUpdates) { // iceberg-core reassigns field id in its schema when firstly creates table metadata; // we should always use the schema (with field ids assigned by delta) from MetadataUpdate // because the parquet data files are already written with field ids assigned by Delta. if (update instanceof MetadataUpdate.AddSchema) { MetadataUpdate.AddSchema addSchema = (MetadataUpdate.AddSchema) update; // lastColumnId must be monotonically increasing. builder.setCurrentSchema( addSchema.schema(), Math.max(metadata.lastColumnId(), addSchema.lastColumnId())); lastAddedSchema = addSchema.schema(); } else if (update instanceof MetadataUpdate.AddPartitionSpec) { // Use the partition spec from MetadataUpdate because the partition spec contains // the correct field ids assigned by Delta PartitionSpec specToAdd = ((MetadataUpdate.AddPartitionSpec) update).spec().bind(lastAddedSchema); if (!specToAdd.compatibleWith(metadata.spec())) { builder.setDefaultPartitionSpec(specToAdd); hasAddPartitionSpec = true; } } else { update.applyTo(builder); } } // Remove the initial partitioned partition spec (id=0) if new partition spec is added // because the initial partition spec has field ids assigned by iceberg-core // which may mismatch with field id assigned by Delta if (hasAddPartitionSpec && initialPartitionSpecExistsAndIsPartitioned) { MetadataUpdate.RemovePartitionSpecs removeSpecs = new MetadataUpdate.RemovePartitionSpecs(Set.of(0)); removeSpecs.applyTo(builder); } metadata = builder.build(); if (base != current()) { throw new CommitFailedException("Cannot commit changes based on stale table metadata"); } if (base == metadata) { LOG.info("Nothing to commit."); return; } final int newVersion = currentVersion() + 1; String newMetadataLocation = writeNewMetadata(metadata, newVersion); lastWrittenTableMetadataWithLocation = Optional.of(Tuple2.apply(newMetadataLocation, metadata)); } @Override public TableMetadata refresh() { // during Delta to iceberg conversion, the table should always exist in catalog and the only // case with NoSuchTableException is the very first metadata conversion where iceberg // metadata and metadata location table prop does not exist yet. try { return super.refresh(); } catch (NoSuchTableException e) { return null; } } @Override public void doRefresh() { LOG.debug("Getting metadata location for table {}", tableIdentifier); String location = loadTableMetadataLocation(); Preconditions.checkState( location != null && !location.isEmpty(), "Got null or empty location %s for table %s", location, tableIdentifier); refreshFromMetadataLocation(location); } /** * Returns both the location of the Iceberg metadata file and its corresponding table metadata. * * @return An Optional containing a tuple of: * - String: The path where the metadata file was written * - TableMetadata: The corresponding Iceberg table metadata */ public Optional> getLastWrittenTableMetadataWithLocation() { return lastWrittenTableMetadataWithLocation; } private String loadTableMetadataLocation() { if (baseMetadataLocation.isPresent()) { return baseMetadataLocation.get(); } throw new NoSuchTableException("Cannot find iceberg table %s. Either the table does " + "not exist or the corresponding Delta table has not been converted yet.", tableIdentifier.toString()); } } ================================================ FILE: kernel/EXCEPTION_PRINCIPLES.md ================================================ # Exception principles in Delta Kernel ## Introduction Exceptions thrown in Delta Kernel are either user-facing or developer-facing. - **User-facing exceptions** are expected to be thrown. Delta Kernel is unable to complete the requested operation for a fundamental reason inherent to the nature of the request, the table of interest, and the capabilities of Delta Kernel and the Delta protocol. These errors are intentional and are used to communicate with the end-user why an operation cannot be completed. - **Developer-facing exceptions** are unexpected and generally indicate that something has gone wrong or is incorrect. They can target either Kernel developers or connector developers that are using Kernel APIs. These exceptions should be used for debugging; a perfectly working connector + Kernel should never encounter these. See [User-facing vs developer-facing exceptions](#User-facing-vs-developer-facing-exceptions) for examples of these types of exceptions. ## Principles These are the general exception principles to follow and enforce when contributing code or reviewing pull requests. - All **user-facing exceptions** should be of type `KernelException`. - Create a new subclass for exceptions that may require special handling (such as `TableNotFoundException`) otherwise just use `KernelException`. Subclasses should expose useful exception parameters on a case-by-case basis. - All `KernelException`s should be instantiated with a method in the [DeltaErrors](https://github.com/delta-io/delta/blob/master/kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaErrors.java) file. - Error messages should be clear and actionable. - Clearly state (1) the problem, (2) why it occurred and (3) how it can be solved. - **User-facing exceptions** should be consistent across releases. Any changes to user-facing exception classes or messages should be carefully reviewed. - Any unchecked exceptions originating from the `Engine` implementation should be wrapped with `KernelEngineException` and should include additional context about the failing operation. - This means all method calls to the `Engine` implementation should be wrapped. See [Wrapping exceptions thrown from the Engine implementation](#Wrapping-exceptions-thrown-from-the-Engine-implementation) for more details. - **Developer-facing exceptions** should be informative and provide useful information for debugging. ## Further details ### User-facing vs developer-facing exceptions User-facing exceptions: - `TableNotFoundException` when there is no Delta table at the provided path. - Reading the Change Data Feed from a table without CDF enabled. - The input data violates table constraints when writing to the table. - Kernel doesn’t support reading a table with XXX table feature. Developer-facing exceptions: - `getInt` is called on a boolean `ColumnVector`. - A column mapping mode besides “none”, “id”, and “name” is encountered. - An empty iterator is returned from the `Engine` implementation when reading files. ### Wrapping exceptions thrown from the Engine implementation We want to wrap any unchecked exceptions thrown from the `Engine` implementation with `KernelEngineException` and include additional context about the failing operation. This makes it clear where the exception is originating from, and the additional context can help future debugging. This requires wrapping all method calls into the `Engine` implementation. We do this using helper methods in `DeltaErrors` like [wrapEngineException](https://github.com/delta-io/delta/blob/4fefba182f81d39f1d11e2f2b85bfa140079ea11/kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaErrors.java#L228-L240). For usage see [example 1](https://github.com/delta-io/delta/blob/2b2ef732533c707b7ca1af30e2a059da86c3c3ff/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionImpl.java#L246-L256) and [example 2](https://github.com/delta-io/delta/blob/2b2ef732533c707b7ca1af30e2a059da86c3c3ff/kernel/kernel-api/src/main/java/io/delta/kernel/internal/ScanImpl.java#L236-L244). Note: this does not catch all exceptions originating from the engine implementation, as exceptions that are not thrown until access will not be wrapped (i.e. exceptions thrown within iterators, in `ColumnVector` implementations, etc) - When checked exceptions cannot be thrown we instead wrap the checked exception in a `KernelEngineException`. See [here](https://github.com/delta-io/delta/blob/2b2ef732533c707b7ca1af30e2a059da86c3c3ff/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetFileReader.java#L148-L150) for an example. ================================================ FILE: kernel/README.md ================================================ # Delta Kernel The Delta Kernel project is a set of Java libraries for building Delta connectors that can read from and write into Delta tables without the need to understand the [Delta protocol details](https://github.com/delta-io/delta/blob/master/PROTOCOL.md). You can use this library to do the following: - Read data from small Delta tables in a single thread in a single process. - Read data from large Delta tables using multiple threads in a single process. - Build a complex connector for a distributed processing engine and read very large Delta tables. - Insert data into a Delta table either from a single process or a complex distributed engine. Here is an example of a simple table scan with a filter: ```java Engine myEngine = DefaultEngine.create() ; // define a engine (more details below) Table myTable = Table.forPath("/delta/table/path"); // define what table to scan Snapshot mySnapshot = myTable.getLatestSnapshot(myEngine); // define which version of table to scan Scan myScan = mySnapshot.getScanBuilder(myEngine) // specify the scan details .withFilters(myEngine, scanFilter) .build(); CloseableIterator physicalData = // read the Parquet data files .. read from Parquet data files ... Scan.transformPhysicalData(...) // returns the table data ``` A complete version of the above example program and more examples of reading from and writing into a Delta table are available [here](https://github.com/delta-io/delta/tree/master/kernel/examples). Notice that there are two sets of public APIs to build connectors. - **Table APIs** - Interfaces like [`Table`](https://delta-io.github.io/delta/snapshot/kernel-api/java/index.html?io/delta/kernel/Table.html), [`Snapshot`](https://delta-io.github.io/delta/snapshot/kernel-api/java/index.html?io/delta/kernel/Snapshot.html), and [`Transaction`](https://delta-io.github.io/delta/snapshot/kernel-api/java/index.html?io/delta/kernel/Transaction.html) that allow you to read from and write to Delta tables - **Engine APIs** - The [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java//index.html?io/delta/kernel/engine/Engine.html) interface allows you to plug in connector-specific optimizations to compute-intensive components in the Kernel. For example, Delta Kernel provides a *default* Parquet file reader via the `DefaultEngine`, but you may choose to replace that default with a custom `Engine` implementation that has a faster Parquet reader for your connector/processing engine. # Project setup with Delta Kernel The Delta Kernel project provides the following two Maven artifacts: - `delta-kernel-api`: This is a must-have dependency and contains all the public `Table` and `Engine` APIs discussed earlier. - `delta-kernel-defaults`: This is an optional dependency that contains *default* implementations of the `Engine` interfaces using Hadoop libraries. Developers can optionally use these default implementations to speed up the development of their Delta connector. ```xml io.delta delta-kernel-api VERSION io.delta delta-kernel-defaults VERSION ``` # API Guarantees **Note: This project is currently in `preview` and all APIs are currently in an evolving state. We welcome trying out the APIs to build Delta Lake connectors and providing feedback (see below) to the project authors.** The Java API docs are available [here](https://docs.delta.io/latest/api/java/kernel/index.html). Only the classes and interfaces documented here are considered public APIs with backward compatibility guarantees (when marked as **Stable** APIs). All other classes and interfaces available in the JAR are considered private APIs with no backward compatibility guarantees. Kernel APIs are still evolving and new features are being added with every release. Kernel authors try to make the API changes backward compatible as much as they can with each new release even when an API is marked as **Evolving**, but sometimes it is hard to maintain the backward compatibility for a project that is evolving rapidly. With each new release, the [examples](https://github.com/delta-io/delta/tree/master/kernel/examples) are kept up-to-date with the latest API changes, and a detailed [migration guide](https://github.com/delta-io/delta/blob/master/kernel/USER_GUIDE.md#migration-guide) is provided for the connectors to upgrade to use the updated APIs. # More Information - [Talk](https://www.youtube.com/watch?v=KVUMFv7470I) explaining the rationale behind Kernel and the API design (slides are available [here](https://docs.google.com/presentation/d/1PGSSuJ8ndghucSF9GpYgCi9oeRpWolFyehjQbPh92-U/edit) which are kept up-to-date with the changes). - [User guide](https://github.com/delta-io/delta/blob/master/kernel/USER_GUIDE.md) on the step-by-step process of using Kernel in a standalone Java program or in a distributed processing connector for reading and writing to Delta tables. - Example [Java programs](https://github.com/delta-io/delta/tree/master/kernel/examples) that illustrate how to read and write Delta tables using the Kernel APIs. - Table and default Engine API Java [documentation](https://docs.delta.io/latest/api/java/kernel/index.html) - [Migration guide](https://github.com/delta-io/delta/blob/master/kernel/USER_GUIDE.md#migration-guide) # Providing feedback We use [GitHub Issues](https://github.com/delta-io/delta/issues) to track community-reported issues. You can also contact the community to get answers. # Contributing We welcome contributions to Delta Lake and we accept contributions via Pull Requests. See our [CONTRIBUTING.md](https://github.com/delta-io/delta/blob/master/CONTRIBUTING.md) for more details. We also adhere to the [Delta Lake Code of Conduct](https://github.com/delta-io/delta/blob/master/CODE_OF_CONDUCT.md). # Setting up IDE Java code adheres to the [Google style](https://google.github.io/styleguide/javaguide.html), which is verified via `build/sbt javafmtCheckAll` during builds. In order to automatically fix Java code style issues, please use `build/sbt javafmtAll`. ## Configuring Code Formatter for Eclipse/IntelliJ Follow the instructions for [Eclipse](https://github.com/google/google-java-format#eclipse) or [IntelliJ](https://github.com/google/google-java-format#intellij-android-studio-and-other-jetbrains-ides) to install the **google-java-format** plugin (note the required manual actions for IntelliJ). ================================================ FILE: kernel/USER_GUIDE.md ================================================ ## Delta Kernel User Guide ## What is Delta Kernel? Delta Kernel is a library for operating on Delta tables. Specifically, it provides simple and narrow APIs for reading and writing to Delta tables without the need to understand the [Delta protocol](https://github.com/delta-io/delta/blob/master/PROTOCOL.md) details. You can use this library to do the following: * Read and write Delta tables from your applications. * Build a connector for a distributed engine like [Apache Spark™](https://github.com/apache/spark), [Apache Flink](https://github.com/apache/flink), or [Trino](https://github.com/trinodb/trino) for reading or writing massive Delta tables. * [Delta Kernel User Guide](#delta-kernel-user-guide) * [What is Delta Kernel?](#what-is-delta-kernel) * [Set up Delta Kernel for your project](#set-up-delta-kernel-for-your-project) * [Read a Delta table in a single process](#read-a-delta-table-in-a-single-process) * [Step 1: Full scan on a Delta table](#step-1-full-scan-on-a-delta-table) * [Step 2: Improve scan performance with file skipping](#step-2-improve-scan-performance-with-file-skipping) * [Create a Delta table](#create-a-delta-table) * [Create a table and insert data into it](#create-a-table-and-insert-data-into-it) * [Blind append into an existing Delta table](#blind-append-into-an-existing-delta-table) * [Idempotent Blind Appends to a Delta Table](#idempotent-blind-appends-to-a-delta-table) * [Checkpointing a Delta table](#checkpointing-a-delta-table) * [Build a Delta connector for a distributed processing engine](#build-a-delta-connector-for-a-distributed-processing-engine) * [Step 0: Validate the prerequisites](#step-0-validate-the-prerequisites) * [Step 1: Set up Delta Kernel in your connector project](#step-1-set-up-delta-kernel-in-your-connector-project) * [Set up Java projects](#set-up-java-projects) * [Step 2: Build your own Engine](#step-2-build-your-own-engine) * [Step 2.1: Implement the Engine interface](#step-21-implement-the-engine-interface) * [Step 2.2: Implement FileSystemClient interface](#step-22-implement-filesystemclient-interface) * [Step 2.3: Implement ParquetHandler](#step-23-implement-parquethandler) * [Step 2.5: Implement JsonHandler](#step-25-implement-jsonhandler) * [Step 2.6: Implement ColumnarBatch and ColumnVector](#step-26-implement-columnarbatch-and-columnvector) * [Step 3: Build read support in your connector](#step-3-build-read-support-in-your-connector) * [Step 3.1: Resolve the table snapshot to query](#step-31-resolve-the-table-snapshot-to-query) * [Step 3.2: Resolve files to scan](#step-32-resolve-files-to-scan) * [Step 3.3: Distribute the file information to the workers](#step-33-distribute-the-file-information-to-the-workers) * [Step 3.4: Read the columnar data](#step-34-read-the-columnar-data) * [Step 4: Build write support in your connector](#step-4-build-write-support-in-your-connector) * [Step 4.1: Determine the schema of the data that needs to be written to the table](#step-41-determine-the-schema-of-the-data-that-needs-to-be-written-to-the-table) * [Step 4.2: Determine the physical partitioning of the data based on the table schema and partition columns](#step-42-determine-the-physical-partitioning-of-the-data-based-on-the-table-schema-and-partition-columns) * [Step 4.3: Distribute the writer tasks definitions (which include the transaction state) to workers](#step-43-distribute-the-writer-tasks-definitions-which-include-the-transaction-state-to-workers) * [Step 4.4: Tasks write the data to data files and send the data file info to the driver.](#step-44-tasks-write-the-data-to-data-files-and-send-the-data-file-info-to-the-driver) * [Step 4.5: Finalize the query.](#step-45-finalize-the-query) * [Migration guide](#migration-guide) * [Migration from Delta Lake version 3.1.0 to 3.2.0](#migration-from-delta-lake-version-310-to-320) ## Set up Delta Kernel for your project You need to `io.delta:delta-kernel-api` and `io.delta:delta-kernel-defaults` dependencies. Following is an example Maven `pom` file dependency list. The `delta-kernel-api` module contains the core of the Kernel that abstracts out the Delta protocol to enable reading and writing into Delta tables. It makes use of the `Engine` interface that is being passed to the Kernel API by the connector for heavy-lift operations such as reading/writing Parquet or JSON files, evaluating expressions or file system operations such as listing contents of the Delta Log directory, etc. Kernel supplies a default implementation of `Engine` in module `delta-kernel-defaults`. The connectors can implement their own version of `Engine` to make use of their native implementation of functionalities the `Engine` provides. For example: the connector can make use of their Parquet reader instead of using the reader from the `DefaultEngine`. More details on this [later](#step-2-build-your-own-engine). ```xml io.delta delta-kernel-api ${delta-kernel.version} io.delta delta-kernel-defaults ${delta-kernel.version} ``` If your connector is not using the [`DefaultEngine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) provided by the Kernel, the dependency `delta-kernel-defaults` from the above list can be skipped. ## Read a Delta table in a single process In this section, we will walk through how to build a very simple single-process Delta connector that can read a Delta table using the default [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) implementation provided by Delta Kernel. You can either write this code yourself in your project, or you can use the [examples](https://github.com/delta-io/delta/tree/master/kernel/examples) present in the Delta code repository. ### Step 1: Full scan on a Delta table The main entry point is [`io.delta.kernel.Table`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Table.html) which is a programmatic representation of a Delta table. Say you have a Delta table at the directory `myTablePath`. You can create a `Table` object as follows: ```java import io.delta.kernel.*; import io.delta.kernel.defaults.*; import org.apache.hadoop.conf.Configuration; String myTablePath = ; // fully qualified table path. Ex: file:/user/tables/myTable Configuration hadoopConf = new Configuration(); Engine myEngine = DefaultEngine.create(hadoopConf); Table myTable = Table.forPath(myEngine, myTablePath); ``` Note the default [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) we are creating to bootstrap the `myTable` object. This object allows you to plug in your own libraries for computationally intensive operations like Parquet file reading, JSON parsing, etc. You can ignore it for now. We will discuss more about this later when we discuss how to build more complex connectors for distributed processing engines. From this `myTable` object you can create a [`Snapshot`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Snapshot.html) object which represents the consistent state (a.k.a. a snapshot consistency) in a specific version of the table. ```java Snapshot mySnapshot = myTable.getLatestSnapshot(myEngine); ``` Now that we have a consistent snapshot view of the table, we can query more details about the table. For example, you can get the version and schema of this snapshot. ```java long version = mySnapshot.getVersion(); StructType tableSchema = mySnapshot.getSchema(); ``` Next, to read the table data, we have to *build* a [`Scan`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Scan.html) object. In order to build a `Scan` object, create a [`ScanBuilder`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/ScanBuilder.html) object which optionally allows selecting a subset of columns to read or setting a query filter. For now, ignore these optional settings. ```java Scan myScan = mySnapshot.getScanBuilder(myEngine).build() // Common information about scanning for all data files to read. Row scanState = myScan.getScanState(myEngine) // Information about the list of scan files to read CloseableIterator scanFiles = myScan.getScanFiles(myEngine) ``` This [`Scan`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Scan.html) object has all the necessary metadata to start reading the table. There are two crucial pieces of information needed for reading data from a file in the table. * `myScan.getScanFiles(Engine)`: Returns scan files as columnar batches (represented as an iterator of `FilteredColumnarBatch`es, more on that later) where each selected row in the batch has information about a single file containing the table data. * `myScan.getScanState(Engine)`: Returns the snapshot-level information needed for reading any file. Note that this is a single row and common to all scan files. For each scan file the physical data must be read from the file. The columns to read are specified in the scan file state. Once the physical data is read, you have to call [`ScanFile.transformPhysicalData(...)`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Scan.html#transformPhysicalData-io.delta.kernel.engine.Engine-io.delta.kernel.data.Row-io.delta.kernel.data.Row-io.delta.kernel.utils.CloseableIterator-) with the scan state and the physical data read from scan file. This API takes care of transforming (e.g. adding partition columns) the physical data into logical data of the table. Here is an example of reading all the table data in a single thread. ```java CloserableIterator fileIter = scanObject.getScanFiles(myEngine); Row scanStateRow = scanObject.getScanState(myEngine); while(fileIter.hasNext()) { FilteredColumnarBatch scanFileColumnarBatch = fileIter.next(); // Get the physical read schema of columns to read from the Parquet data files StructType physicalReadSchema = ScanStateRow.getPhysicalDataReadSchema(engine, scanStateRow); try (CloseableIterator scanFileRows = scanFileColumnarBatch.getRows()) { while (scanFileRows.hasNext()) { Row scanFileRow = scanFileRows.next(); // From the scan file row, extract the file path, size and modification time metadata // needed to read the file. FileStatus fileStatus = InternalScanFileUtils.getAddFileStatus(scanFileRow); // Open the scan file which is a Parquet file using connector's own // Parquet reader or default Parquet reader provided by the Kernel (which // is used in this example). CloseableIterator physicalDataIter = engine.getParquetHandler().readParquetFiles( singletonCloseableIterator(fileStatus), physicalReadSchema, Optional.empty() /* optional predicate the connector can apply to filter data from the reader */ ); // Now the physical data read from the Parquet data file is converted to a table // logical data. Logical data may include the addition of partition columns and/or // subset of rows deleted try ( CloseableIterator transformedData = Scan.transformPhysicalData( engine, scanStateRow, scanFileRow, physicalDataIter)) { while (transformedData.hasNext()) { FilteredColumnarBatch logicalData = transformedData.next(); ColumnarBatch dataBatch = logicalData.getData(); // Not all rows in `dataBatch` are in the selected output. // An optional selection vector determines whether a row with a // specific row index is in the final output or not. Optional selectionVector = dataReadResult.getSelectionVector(); // access the data for the column at ordinal 0 ColumnVector column0 = dataBatch.getColumnVector(0); for (int rowIndex = 0; rowIndex < column0.getSize(); rowIndex++) { // check if the row is selected or not if (!selectionVector.isPresent() || // there is no selection vector, all records are selected (!selectionVector.get().isNullAt(rowId) && selectionVector.get().getBoolean(rowId))) { // Assuming the column type is String. // If it is a different type, call the relevant function on the `ColumnVector` System.out.println(column0.getString(rowIndex)); } } // access the data for column at ordinal 1 ColumnVector column1 = dataBatch.getColumnVector(1); for (int rowIndex = 0; rowIndex < column1.getSize(); rowIndex++) { // check if the row is selected or not if (!selectionVector.isPresent() || // there is no selection vector, all records are selected (!selectionVector.get().isNullAt(rowId) && selectionVector.get().getBoolean(rowId))) { // Assuming the column type is Long. // If it is a different type, call the relevant function on the `ColumnVector` System.out.println(column1.getLong(rowIndex)); } } // .. more .. } } } } } ``` A few working examples to read Delta tables within a single process are available [here](https://github.com/delta-io/delta/tree/master/kernel/examples). > [!IMPORTANT] > All the Delta protocol-level details are encoded in the rows returned by `Scan.getScanFiles` API, but you do not have to understand them in order to read the table data correctly. All you need is to get the Parquet file status from each scan file row and read the data from the Parquet file into the `ColumnarBatch` format. The physical data is converted into the logical data of the table using [`Scan.transformPhysicalData`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Scan.html#transformPhysicalData-io.delta.kernel.engine.Engine-io.delta.kernel.data.Row-io.delta.kernel.data.Row-io.delta.kernel.utils.CloseableIterator-). Transformation to logical data is dictated by the protocol and the metadata of the table and the scan file. As the Delta protocol evolves this transformation step will evolve with it and your code will not have to change to accommodate protocol changes. This is the major advantage of the abstractions provided by Delta Kernel. > [!NOTE] > Observe that the same `Engine` instance `myEngine` is passed multiple times whenever a call to Delta Kernel API is made. The reason for passing this instance for every call is because it is the connector context, it should maintained outside of the Delta Kernel APIs to give the connector control over the `Engine`. ### Step 2: Improve scan performance with file skipping We have explored how to do a full table scan. However, the real advantage of using the Delta format is that you can skip files using your query filters. To make this possible, Delta Kernel provides an [expression framework](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/expressions/package-summary.html) to encode your filters and provide them to Delta Kernel to skip files during the scan file generation. For example, say your table is partitioned by `columnX`, you want to query only the partition `columnX=1`. You can generate the expression and use it to build the scan as follows: ```java import io.delta.kernel.expressions.*; import io.delta.kernel.defaults.engine.*; Engine myEngine = DefaultEngine.create(new Configuration()); Predicate filter = new Predicate( "=", Arrays.asList(new Column("columnX"), Literal.ofInt(1))); Scan myFilteredScan = mySnapshot.getScanBuilder().withFilter(filter).build() // Subset of the given filter that is not guaranteed to be satisfied by // Delta Kernel when it returns data. This filter is used by Delta Kernel // to do data skipping as much as possible. The connector should use this filter // on top of the data returned by Delta Kernel in order for further filtering. Optional remainingFilter = myFilteredScan.getRemainingFilter(); ``` The scan files returned by `myFilteredScan.getScanFiles(myEngine)` will have rows representing files only of the required partition. Similarly, you can provide filters for non-partition columns, and if the data in the table is well clustered by those columns, then Delta Kernel will be able to skip files as much as possible. ## Create a Delta table In this section, we will walk through how to build a Delta connector that can create a Delta table using the default [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) implementation provided by Delta Kernel. You can either write this code yourself in your project, or you can use the [examples](https://github.com/delta-io/delta/tree/master/kernel/examples) present in the Delta code repository. The main entry point is [`io.delta.kernel.Table`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Table.html) which is a programmatic representation of a Delta table. Say you want to create Delta table at the directory `myTablePath`. You can create a `Table` object as follows: ```java package io.delta.kernel.examples; import io.delta.kernel.*; import io.delta.kernel.types.*; import io.delta.kernel.utils.CloseableIterable; String myTablePath = ; Configuration hadoopConf = new Configuration(); Engine myEngine = DefaultEngine.create(hadoopConf); Table myTable = Table.forPath(myEngine, myTablePath); ``` Note the default [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) we are creating to bootstrap the `myTable` object. This object allows you to plug in your own libraries for computationally intensive operations like Parquet file reading, JSON parsing, etc. You can ignore it for now. We will discuss more about this later when we discuss how to build more complex connectors for distributed processing engines. From this `myTable` object you can create a [`TransactionBuilder`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/TransactionBuilder.html) object which allows you to construct a [`Transaction`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Transaction.html) object ```java TransactionBuilder txnBuilder = myTable.createTransactionBuilder( myEngine, "Examples", /* engineInfo - connector can add its own identifier which is noted in the Delta Log */ Operation.CREATE_TABLE /* What is the operation we are trying to perform. This is noted in the Delta Log */ ); ``` Now that you have the `TransactionBuilder` object, you can set the table schema and partition columns of the table. ```java StructType mySchema = new StructType() .add("id", IntegerType.INTEGER) .add("name", StringType.STRING) .add("city", StringType.STRING) .add("salary", DoubleType.DOUBLE); // Partition columns are optional. Use it only if you are creating a partitioned table. List myPartitionColumns = Collections.singletonList("city"); // Set the schema of the new table on the transaction builder txnBuilder = txnBuilder .withSchema(engine, mySchema); // Set the partition columns of the new table only if you are creating // a partitioned table; otherwise, this step can be skipped. txnBuilder = txnBuilder .withPartitionColumns(engine, examplePartitionColumns); ``` `TransactionBuilder` allows setting additional properties of the table such as enabling a certain Delta feature or setting identifiers for idempotent writes. We will be visiting these in the next sections. The next step is to build `Transaction` out of the `TransactionBuilder` object. ```java // Build the transaction Transaction txn = txnBuilder.build(engine); ``` `Transaction` object allows the connector to optionally add any data and finally commit the transaction. A successful commit ensures that the table is created with the given schema. In this example, we are just creating a table and not adding any data as part of the table. ```java // Commit the transaction. // As we are just creating the table and not adding any data, the `dataActions` is empty. TransactionCommitResult commitResult = txn.commit( engine, CloseableIterable.emptyIterable() /* dataActions */ ); ``` The [`TransactionCommitResult`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/TransactionCommitResult.html) contains the what version the transaction is committed as and whether the table is ready for a checkpoint. As we are creating a table the version will be `0`. We will be discussing later on what a checkpoint is and what it means for the table to be ready for the checkpoint. A few working examples to create partitioned and un-partitioned Delta tables are available [here](https://github.com/delta-io/delta/tree/master/kernel/examples). ## Create a table and insert data into it In this section, we will walk through how to build a Delta connector that can create a Delta table and insert data into the table (similar to `CREATE TABLE
AS ` construct in SQL) using the default [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) implementation provided by Delta Kernel. You can either write this code yourself in your project, or you can use the [examples](https://github.com/delta-io/delta/tree/master/kernel/examples) present in the Delta code repository. The first step is to construct a `Transaction`. Below is the code for that. For more details on what each step of the code means, please read the [create table](#create-a-delta-table) section. ``` package io.delta.kernel.examples; import io.delta.kernel.*; import io.delta.kernel.types.*; import io.delta.kernel.utils.CloseableIterable; String myTablePath = ; Configuration hadoopConf = new Configuration(); Engine myEngine = DefaultEngine.create(hadoopConf); Table myTable = Table.forPath(myEngine, myTablePath); StructType mySchema = new StructType() .add("id", IntegerType.INTEGER) .add("name", StringType.STRING) .add("city", StringType.STRING) .add("salary", DoubleType.DOUBLE); // Partition columns are optional. Use it only if you are creating a partitioned table. List myPartitionColumns = Collections.singletonList("city"); TransactionBuilder txnBuilder = myTable.createTransactionBuilder( myEngine, "Examples", /* engineInfo - connector can add its own identifier which is noted in the Delta Log */ Operation.WRITE /* What is the operation we are trying to perform? This is noted in the Delta Log */ ); // Set the schema of the new table on the transaction builder txnBuilder = txnBuilder .withSchema(engine, mySchema); // Set the partition columns of the new table only if you are creating // a partitioned table; otherwise, this step can be skipped. txnBuilder = txnBuilder .withPartitionColumns(engine, examplePartitionColumns); // Build the transaction Transaction txn = txnBuilder.build(engine); ``` Now that we have the [`Transaction`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Transaction.html) object, the next step is generating the data that confirms the table schema and partitioned according to the table partitions. ```java StructType dataSchema = txn.getSchema(engine) // Optional for un-partitioned tables List partitionColumnNames = txn.getPartitionColumns(engine) ``` Using the data schema and partition column names the connector can plan the query and generate data. At tasks that actually have the data to write to the table, the connector can ask the Kernel to transform the data given in the table schema into physical data that can actually be written to the Parquet data files. For partitioned tables, the data needs to be first partitioned by the partition columns, and then the connector should ask the Kernel to transform the data for each partition separately. The partitioning step is needed because any given data file in the Delta table contains data belonging to exactly one partition. Get the state of the transaction. The transaction state contains the information about how to convert the data in the table schema into physical data that needs to be written. The transformations depend on the protocol and features the table has. ```java Row txnState = txn.getTransactionState(engine); ``` Prepare the data. ```java // The data generated by the connector to write into a table CloseableIterator data = ... // Create partition value map Map partitionValues = Collections.singletonMap( "city", // partition column name // partition value. Depending upon the partition column type, the // partition value should be created. In this example, the partition // column is of type StringType, so we are creating a string literal. Literal.ofString(city) ); ``` The connector data is passed as an iterator of [`FilteredColumnarBatch`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/FilteredColumnarBatch.html). Each of the `FilteredColumnarBatch` contains a [`ColumnarBatch`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnarBatch.html) which actually contains the data in columnar access format and an optional section vector that allows the connector to specify which rows from the `ColumnarBatch` to write to the table. Partition values are passed as a map of the partition column name to the partition value. For an un-partitioned table, the map should be empty as it has no partition columns. ``` // Transform the logical data to physical data that needs to be written to the Parquet // files CloseableIterator physicalData = Transaction.transformLogicalData(engine, txnState, data, partitionValues); ``` The above code converts the given data for partitions into an iterator of `FilteredColumnarBatch` that needs to be written to the Parquet data files. In order to write the data files, the connector needs to get the [`WriteContext`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/DataWriteContext.html) from Kernel, which tells the connector where to write the data files and what columns to collect statistics from each data file. ```java // Get the write context DataWriteContext writeContext = Transaction.getWriteContext(engine, txnState, partitionValues); ``` Now, the connector has the physical data that needs to be written to Parquet data files, and where those files should be written, it can start writing the data files. ```java CloseableIterator dataFiles = engine.getParquetHandler() .writeParquetFiles( writeContext.getTargetDirectory(), physicalData, writeContext.getStatisticsColumns() ); ``` In the above code, the connector is making use of the `Engine` provided `ParquetHandler` to write the data, but the connector can choose its own Parquet file writer to write the data. Also note that the return of the above call is an iterator of [`DataFileStatus`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/utils/DataFileStatus.html) for each data file written. It basically contains the file path, file metadata, and optional file-level statistics for columns specified by the [`WriteContext.getStatisticsColumns()`]([https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/DataWriteContext.html](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/DataWriteContext.html#getStatisticsColumns--)) Convert each `DataFileStatus` into a Delta log action that can be written to the Delta table log. ```java CloseableIterator dataActions = Transaction.generateAppendActions(engine, txnState, dataFiles, writeContext); ``` The next step is constructing [`CloseableIterable`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/utils/CloseableIterable.html) out of the all the Delta log actions generated above. The reason for constructing an `Iterable` is that the transaction committing involves accessing the list of Delta log actions more than one time (in order to resolve conflicts when there are multiple writes to the table). Kernel provides a [utility method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/utils/CloseableIterable.html#inMemoryIterable-io.delta.kernel.utils.CloseableIterator-) to create an in-memory version of `CloseableIterable`. This interface also gives the connector an option to implement a custom implementation that spills the data actions to disk when the contents are too big to fit in memory. ```java // Create a iterable out of the data actions. If the contents are too big to fit in memory, // the connector may choose to write the data actions to a temporary file and return an // iterator that reads from the file. CloseableIterable dataActionsIterable = CloseableIterable.inMemoryIterable(dataActions); ``` The final step is committing the transaction! ```java TransactionCommitStatus commitStatus = txn.commit(engine, dataActionsIterable) ``` The [`TransactionCommitResult`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/TransactionCommitResult.html) contains the what version the transaction is committed as and whether the table is ready for a checkpoint. As we are creating a table the version will be `0`. We will be discussing later on what a checkpoint is and what it means for the table to be ready for the checkpoint. A few working examples to create and insert data into partitioned and un-partitioned Delta tables are available [here](https://github.com/delta-io/delta/tree/master/kernel/examples). ## Blind append into an existing Delta table In this section, we will walk through how to build a Delta connector that inserts data into an existing Delta table (similar to `INSERT INTO
` construct in SQL) using the default [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) implementation provided by Delta Kernel. You can either write this code yourself in your project, or you can use the [examples](https://github.com/delta-io/delta/tree/master/kernel/examples) present in the Delta code repository. The steps are exactly similar to [Create table and insert data into it](#create-a-table-and-insert-data-into-it) except that we won't be providing any schema or partition columns when building the `TransactionBuilder` ```java // Create a `Table` object with the given destination table path Table table = Table.forPath(engine, tablePath); // Create a transaction builder to build the transaction TransactionBuilder txnBuilder = table.createTransactionBuilder( engine, "Examples", /* engineInfo */ Operation.WRITE ); / Build the transaction - no need to provide the schema as the table already exists. Transaction txn = txnBuilder.build(engine); // Get the transaction state Row txnState = txn.getTransactionState(engine); List dataActions = new ArrayList<>(); // Generate the sample data for three partitions. Process each partition separately. // This is just an example. In a real-world scenario, the data may come from different // partitions. Connectors already have the capability to partition by partition values // before writing to the table // In the test data `city` is a partition column for (String city : Arrays.asList("San Francisco", "Campbell", "San Jose")) { FilteredColumnarBatch batch1 = generatedPartitionedDataBatch( 5 /* offset */, city /* partition value */); FilteredColumnarBatch batch2 = generatedPartitionedDataBatch( 5 /* offset */, city /* partition value */); FilteredColumnarBatch batch3 = generatedPartitionedDataBatch( 10 /* offset */, city /* partition value */); CloseableIterator data = toCloseableIterator(Arrays.asList(batch1, batch2, batch3).iterator()); // Create partition value map Map partitionValues = Collections.singletonMap( "city", // partition column name // partition value. Depending upon the parition column type, the // partition value should be created. In this example, the partition // column is of type StringType, so we are creating a string literal. Literal.ofString(city)); // First transform the logical data to physical data that needs to be written // to the Parquet // files CloseableIterator physicalData = Transaction.transformLogicalData(engine, txnState, data, partitionValues); // Get the write context DataWriteContext writeContext = Transaction.getWriteContext(engine, txnState, partitionValues); // Now write the physical data to Parquet files CloseableIterator dataFiles = engine.getParquetHandler() .writeParquetFiles( writeContext.getTargetDirectory(), physicalData, writeContext.getStatisticsColumns()); // Now convert the data file status to data actions that needs to be written to the Delta // table log CloseableIterator partitionDataActions = Transaction.generateAppendActions( engine, txnState, dataFiles, writeContext); // Now add all the partition data actions to the main data actions list. In a // distributed query engine, the partition data is written to files at tasks on executor // nodes. The data actions are collected at the driver node and then written to the // Delta table log using the `Transaction.commit` while (partitionDataActions.hasNext()) { dataActions.add(partitionDataActions.next()); } } // Create a iterable out of the data actions. If the contents are too big to fit in memory, // the connector may choose to write the data actions to a temporary file and return an // iterator that reads from the file. CloseableIterable dataActionsIterable = CloseableIterable.inMemoryIterable( toCloseableIterator(dataActions.iterator())); // Commit the transaction. TransactionCommitResult commitResult = txn.commit(engine, dataActionsIterable); ``` ## Idempotent Blind Appends to a Delta Table Idempotent writes allow the connector to make sure the data belonging to a particular transaction version and application id is inserted into the table at most once. In incremental processing systems (e.g. streaming systems), track progress using their own application-specific versions need to record what progress has been made in order to avoid duplicating data in the face of failures and retries during writes. By setting the transaction identifier, the Delta table can ensure that the data with the same identifier is not written multiple times. For more information refer to the Delta protocol section [Transaction Identifiers](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#transaction-identifiers) To make the data append idempotent, set the transaction identifier on the [`TransactionBuilder`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/TransactionBuilder.html#withTransactionId-io.delta.kernel.engine.Engine-java.lang.String-long-) ```java // Set the transaction identifiers for idempotent writes // Delta/Kernel makes sure that there exists only one transaction in the Delta log // with the given application id and txn version txnBuilder = txnBuilder.withTransactionId( engine, "my app id", /* application id */ 100 /* monotonically increasing txn version with each new data insert */ ); ``` That's all the connector need to do for idempotent blind appends. ## Checkpointing a Delta table [Checkpoints](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#checkpoints) are an optimization in Delta Log in order to construct the state of the Delta table faster. It basically contains the state of the table at the version the checkpoint is created. Delta Kernel allows the connector to optionally make the checkpoints. It is created for every few commits (configurable table property) on the table. The result of `Transaction.commit` returns a `TransactionCommitResult` that contains the version the transaction is committed as and whether the table is [read for checkpoint](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/TransactionCommitResult.html#isReadyForCheckpoint--). Creating a checkpoint takes time as it needs to construct the entire state of the table. If the connector doesn't want to checkpoint by itself but uses other connectors that are faster in creating a checkpoint, it can skip the checkpointing step. If it wants to checkpoint, the `Table` object has an [API](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Table.html#checkpoint-io.delta.kernel.engine.Engine-long-) to checkpoint the table. ```java TransactionCommitResult commitResult = txn.commit(engine, dataActionsIterable); if (commitResult.isReadyForCheckpoint()) { // Checkpoint the table Table.forPath(engine, tablePath).checkpoint(engine, commitResult.getVersion()); } ``` ## Build a Delta connector for a distributed processing engine Unlike simple applications that just read the table in a single process, building a connector for complex processing engines like Apache Spark™ and Trino can require quite a bit of additional effort. For example, to build a connector for an SQL engine you have to do the following * Understand the APIs provided by the engine to build connectors and how Delta Kernel can be used to provide the information necessary for the connector + engine to operate on a Delta table. * Decide what libraries to use to do computationally expensive operations like reading Parquet files, parsing JSON, computing expressions, etc. Delta Kernel provides all the extension points to allow you to plug in any library without having to understand all the low-level details of the Delta protocol. * Deal with details specific to distributed engines. For example, * Serialization of Delta table metadata provided by Delta Kernel. * Efficiently transforming data read from Parquet into the engine in-memory processing format. In this section, we are going to outline the steps needed to build a connector. ### Step 0: Validate the prerequisites In the previous section showing how to read a simple table, we were briefly introduced to the [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html). This is the main extension point where you can plug in your implementations of computationally-expensive operations like reading Parquet files, parsing JSON, etc. For the simple case, we were using a default implementation of the helper that works in most cases. However, for building a high-performance connector for a complex processing engine, you will very likely need to provide your own implementation using the libraries that work with your engine. So before you start building your connector, it is important to understand these requirements and plan for building your own engine. Here are the libraries/capabilities you need to build a connector that can read the Delta table * Perform file listing and file reads from your storage/file system. * Read Parquet files in columnar data, preferably in an in-memory columnar format. * Parse JSON data * Read JSON files * Evaluate expressions on in-memory columnar batches For each of these capabilities, you can choose to build your own implementation or reuse the default implementation. ### Step 1: Set up Delta Kernel in your connector project In the Delta Kernel project, there are multiple dependencies you can choose to depend on. 1. Delta Kernel core APIs - This is a must-have dependency, which contains all the main APIs like Table, Snapshot, and Scan that you will use to access the metadata and data of the Delta table. This has very few dependencies reducing the chance of conflicts with any dependencies in your connector and engine. This also provides the [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) interface which allows you to plug in your implementations of computationally expensive operations, but it does not provide any implementation of this interface. 2. Delta Kernel default- This has a default implementation called [`DefaultEngine`](https://delta-io.github.io/delta/snapshot/kernel-defaults/java/io/delta/kernel/defaults/engine/DefaultEngine.html) and additional dependencies such as `Hadoop`. If you wish to reuse all or parts of this implementation, then you can optionally depend on this. #### Set up Java projects As discussed above, you can import one or both of the artifacts as follows: ```xml io.delta delta-kernel-api ${delta-kernel.version} io.delta delta-kernel-defaults ${delta-kernel.version} ``` ### Step 2: Build your own Engine In this section, we are going to explore the [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) interface and walk through how to implement your own implementation so that you can plug in your connector/engine-specific implementations of computationally-intensive operations, threading model, resource management, etc. > [!IMPORTANT] > During the validation process, if you believe that all the dependencies of the default `Engine` implementation can work with your connector and engine, then you can skip this step and jump to Step 3 of implementing your connector using the default engine. If later you have the need to customize the helper for your connector, you can revisit this step. #### Step 2.1: Implement the `Engine` interface The [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) interface combines a bunch of sub-interfaces each of which is designed for a specific purpose. Here is a brief overview of the subinterfaces. See the API docs (Java) for a more detailed view. ```java interface Engine { /** * Get the connector provided {@link ExpressionHandler}. * @return An implementation of {@link ExpressionHandler}. */ ExpressionHandler getExpressionHandler(); /** * Get the connector provided {@link JsonHandler}. * @return An implementation of {@link JsonHandler}. */ JsonHandler getJsonHandler(); /** * Get the connector provided {@link FileSystemClient}. * @return An implementation of {@link FileSystemClient}. */ FileSystemClient getFileSystemClient(); /** * Get the connector provided {@link ParquetHandler}. * @return An implementation of {@link ParquetHandler}. */ ParquetHandler getParquetHandler(); } ``` To build your own `Engine` implementation, you can choose to either use the default implementations of each sub-interface or completely build every one from scratch. ```java class MyEngine extends DefaultEngine { FileSystemClient getFileSystemClient() { // Build a new implementation from scratch return new MyFileSystemClient(); } // For all other sub-clients, use the default implementations provided by the `DefaultEngine`. } ``` Next, we will walk through how to implement each interface. #### Step 2.2: Implement `FileSystemClient` interface The [`FileSystemClient`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/FileSystemClient.html) interface contains basic file system operations like listing directories, resolving paths into a fully qualified path and reading bytes from files. Implementation of this interface must take care of the following when interacting with storage systems such as S3, Hadoop, or ADLS: * Credentials and permissions: The connector must populate its `FileSystemClient` with the necessary configurations and credentials for the client to retrieve the necessary data from the storage system. For example, an implementation based on Hadoop's FileSystem abstractions can be passed S3 credentials via the Hadoop configurations. * Decryption: If file system objects are encrypted, then the implementation must decrypt the data before returning the data. #### Step 2.3: Implement `ParquetHandler` As the name suggests, this interface contains everything related to reading and writing Parquet files. It has been designed such that a connector can plug in a wide variety of implementations, from a simple single-threaded reader to a very advanced multi-threaded reader with pre-fetching and advanced connector-specific expression pushdown. Let's explore the methods to implement, and the guarantees associated with them. ##### Method `readParquetFiles(CloseableIterator fileIter, StructType physicalSchema, java.util.Optional predicate)` This [method]((https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/ParquetHandler.html#readParquetFiles-io.delta.kernel.utils.CloseableIterator-io.delta.kernel.types.StructType-)) takes as input `FileStatus`s which contains metadata such as file path, size etc. of the Parquet file to read. The columns to be read from the Parquet file are defined by the physical schema. To implement this method, you may have to first implement your own [`ColumnarBatch`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnarBatch.html) and [`ColumnVector`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnVector.html) which is used to represent the in-memory data generated from the Parquet files. When identifying the columns to read, note that there are multiple types of columns in the physical schema (represented as a [`StructType`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/types/StructType.html)). * Data columns: Columns that are expected to be read from the Parquet file. Based on the `StructField` object defining the column, read the column in the Parquet file that matches the same name or field id. If the column has a field id (stored as `parquet.field.id` in the `StructField` metadata) then the field id should be used to match the column in the Parquet file. Otherwise, the column name should be used for matching. * Metadata columns: These are special columns that must be populated using metadata about the Parquet file ([`StructField#isMetadataColumn`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/types/StructField.html#isMetadataColumn--) tells whether a column in `StructType` is a metadata column). To understand how to populate such a column, first match the column name against the set of standard metadata column name constants. For example, * `StructFileld#isMetadataColumn()` returns true and the column name is `StructField.METADATA_ROW_INDEX_COLUMN_NAME`, then you have to a generate column vector populated with the actual index of each row in the Parquet file (that is, not indexed by the possible subset of rows returned after Parquet data skipping). ##### Requirements and guarantees Any implementation must adhere to the following guarantees. * The schema of the returned `ColumnarBatch`es must match the physical schema. * If a data column is not found and the `StructField.isNullable = true`, then return a `ColumnVector` of nulls. Throw an error if it is not nullable. * The output iterator must maintain ordering as the input iterator. That is, if `file1` is before `file2` in the input iterator, then columnar batches of `file1` must be before those of `file2` in the output iterator. ##### Method `writeParquetFiles(String directoryPath, CloseableIterator dataIter, java.util.List statsColumns)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/ParquetHandler.html#writeParquetFiles-java.lang.String-io.delta.kernel.utils.CloseableIterator-java.util.List-) takes given data writes it into one or more Parquet files into the given directory. The data is given as an iterator of [FilteredColumnarBatches](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/FilteredColumnarBatch.html) which contains a [ColumnarBatch](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnarBatch.html) and an optional selection vector containing one entry for each row in `ColumnarBatch` indicating whether a row is selected or not selected. The `ColumnarBatch` also contains the schema of the data. This schema should be converted to Parquet schema, including any field IDs present [`FieldMetadata`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/types/FieldMetadata.html) for each column `StructField`. There is also the parameter `statsColumns`, which is a hint to the Parquet writer on what set of columns to collect stats for each file. The statistics include `min`, `max` and `null_count` for each column in the `statsColumns` list. Statistics collection is optional, but when present it is used by Kernel to persist the stats as part of the Delta table commit. This will help read queries prune un-needed data files based on the query predicate. For each written data file, the caller is expecting a [`DataFileStatus`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/utils/DataFileStatus.html) object. It contains the data file path, size, modification time, and optional column statistics. #### Method `writeParquetFileAtomically(String filePath, CloseableIterator data)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/ParquetHandler.html#writeParquetFiles-java.lang.String-io.delta.kernel.utils.CloseableIterator-java.util.List-) writes the given `data` into Parquet file at location `filePath`. The write is an atomic write i.e., either a Parquet file is created with all given content or no Parquet file is created at all. This should not create a file with partial content in it. The default implementation makes use of [`LogStore`](https://github.com/delta-io/delta/blob/master/storage/src/main/java/io/delta/storage/LogStore.java) implementations from the [`delta-storage`](https://github.com/delta-io/delta/tree/master/storage) module to accomplish the atomicity. A connector that wants to implement their own version of `ParquetHandler` can take a look at the default implementation for details. ##### Performance suggestions * The representation of data as `ColumnVector`s and `ColumnarBatch`es can have a significant impact on the query performance and it's best to read the Parquet file data directly into vectors and batches of the engine-native format to avoid potentially costly in-memory data format conversion. Create a Kernel `ColumnVector` and `ColumnarBatch` wrappers around the engine-native format equivalent classes. #### Step 2.4: Implement `ExpressionHandler` interface The [`ExpressionHandler`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/ExpressionHandler.html) interface has all the methods needed for handling expressions that may be applied on columnar data. ##### Method `getEvaluator(StructType batchSchema, Expression expresion, DataType outputType)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/ExpressionHandler.html#getEvaluator-io.delta.kernel.types.StructType-io.delta.kernel.expressions.Expression-io.delta.kernel.types.DataType-) generates an object of type [`ExpressionEvaluator`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/expressions/ExpressionEvaluator.html) that can evaluate the `expression` on a batch of row data to produce a result of a single column vector. To generate this function, the `getEvaluator()` method takes as input the expression and the schema of the `ColumnarBatch`es of data on which the expressions will be applied. The same object can be used to evaluate multiple columnar batches of input with the same schema and expression the evaluator is created for. ##### Method `getPredicateEvaluator(StructType inputSchema, Predicate predicate)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/ExpressionHandler.html#createSelectionVector-boolean:A-int-int-) is for creating an expression evaluator for `Predicate` type expressions. The `Predicate` type expressions return a boolean value as output. The returned object is of type [`PredicateEvaluator`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/expressions/PredicateEvaluator.html). This is a special interface for evaluating Predicate on input batch returns a selection vector containing one value for each row in input batch indicating whether the row has passed the predicate or not. Optionally it takes an existing selection vector along with the input batch for evaluation. The result selection vector is combined with the given existing selection vector and a new selection vector is returned. This mechanism allows running an input batch through several predicate evaluations without rewriting the input batch to remove rows that do not pass the predicate after each predicate evaluation. The new selection should be the same or more selective as the existing selection vector. For example, if a row is marked as unselected in the existing selection vector, then it should remain unselected in the returned selection vector even when the given predicate returns true for the row. ##### Method `createSelectionVector(boolean[] values, int from, int to)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/ExpressionHandler.html#createSelectionVector-boolean:A-int-int-) allows creating `ColumnVector` for boolean type values given as input. This allows the connector to maintain all `ColumnVector`s created in the desired memory format. ##### Requirements and guarantees Any implementation must adhere to the following guarantees. * Implementation must handle all possible variations of expressions. If the implementation encounters an expression type that it does not know how to handle, then it must throw a specific language-dependent exception. * Java: [NotSupportedException](https://docs.oracle.com/javaee/7/api/javax/resource/NotSupportedException.html) * The `ColumnarBatch`es on which the generated `ExpressionEvaluator` is going to be used are guaranteed to have the schema provided during generation. Hence, it is safe to bind the expression evaluation logic to column ordinals instead of column names, thus making the actual evaluation faster. #### Step 2.5: Implement `JsonHandler` [This](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/JsonHandler.html) engine interface allows the connector to use plug-in their own JSON handling code and expose it to the Delta Kernel. ##### Method `readJsonFiles(CloseableIterator fileIter, StructType physicalSchema, java.util.Optional predicate)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/JsonHandler.html#readJsonFiles-io.delta.kernel.utils.CloseableIterator-io.delta.kernel.types.StructType-) takes as input `FileStatus`s of the JSON files and returns the data in a series of columnar batches. The columns to be read from the JSON file are defined by the physical schema, and the return batches must match that schema. To implement this method, you may have to first implement your own [`ColumnarBatch`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnarBatch.html)and [`ColumnVector`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnVector.html) which is used to represent the in-memory data generated from the JSON files. When identifying the columns to read, note that there are multiple types of columns in the physical schema (represented as a [`StructType`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/types/StructType.html)). ##### Method `parseJson(ColumnVector jsonStringVector, StructType outputSchema, java.util.Optional selectionVector)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/JsonHandler.html#parseJson-io.delta.kernel.data.ColumnVector-io.delta.kernel.types.StructType-) allows parsing a `ColumnVector` of string values which are in JSON format into the output format specified by the `outputSchema`. If a given column in `outputSchema` is not found, then a null value is returned. It optionally takes a selection vector which indicates what entries in the input `ColumnVector` of strings to parse. If an entry is not selected then a `null` value is returned as parsed output for that particular entry in the output. ##### Method `deserializeStructType(String structTypeJson)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/JsonHandler.html#deserializeStructType-java.lang.String-) allows parsing JSON encoded (according to [Delta schema serialization rules](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#schema-serialization-format)) `StructType` schema into a `StructType`. Most implementations of `JsonHandler` do not need to implement this method and instead use the one in the [default `JsonHandler`](https://delta-io.github.io/delta/snapshot/kernel-defaults/java/io/delta/kernel/defaults/engine/DefaultJsonHandler.html) implementation. #### Method `writeJsonFileAtomically(String filePath, CloseableIterator data, boolean overwrite)` This [method](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/JsonHandler.html#writeJsonFileAtomically-java.lang.String-io.delta.kernel.utils.CloseableIterator-boolean-) writes the given `data` into a JSON file at location `filePath`. The write is an atomic write i.e., either a JSON file is created with all given content or no Parquet file is created at all. This should not create a file with partial content in it. The default implementation makes use of [`LogStore`](https://github.com/delta-io/delta/blob/master/storage/src/main/java/io/delta/storage/LogStore.java) implementations from the [`delta-storage`](https://github.com/delta-io/delta/tree/master/storage) module to accomplish the atomicity. A connector that wants to implement their own version of `JsonHandler` can take a look at the default implementation for details. The implementation is expected to handle the serialization rules (converting the `Row` object to JSON string) as described in the [API Javadoc](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/JsonHandler.html#writeJsonFileAtomically-java.lang.String-io.delta.kernel.utils.CloseableIterator-boolean-). #### Step 2.6: Implement `ColumnarBatch` and `ColumnVector` [`ColumnarBatch`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnarBatch.html) and [`ColumnVector`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/ColumnVector.html) are two interfaces to represent the data read into memory from files. This representation can have a significant impact on query performance. Each engine likely has a native representation of in-memory data with which it applies data transformation operations. For example, in Apache Spark™, the row data is internally represented as `UnsafeRow` for efficient processing. So it's best to read the Parquet file data directly into vectors and batches of the native format to avoid potentially costly in-memory data format conversions. So the recommended approach is to build wrapper classes that extend the two interfaces but internally use engine-native classes to store the data. When the connector has to forward the columnar batches received from the kernel to the engine, it has to be smart enough to skip converting vectors and batches that are already in the engine-native format. ### Step 3: Build read support in your connector In this section, we are going to walk through the likely sequence of Kernel API calls your connector will have to make to read a table. The exact timing of making these calls in your connector in the context of connector-engine interactions depends entirely on the engine-connector APIs and is therefore beyond the scope of this guide. However, we will try to provide broad guidelines that are likely (but not guaranteed) to apply to your connector-engine setup. For this purpose, we are going to assume that the engine goes through the following phases when processing a read/scan query - logical plan analysis, physical plan generation, and physical plan execution. Based on these broad characterizations, a typical control and data flow for reading a Delta table is going to be as follows:
Step Typical query phase when this step occurs
Resolve the table snapshot to query Logical plan analysis phase when the plan's schema and other details need to be resolved and validated
Resolve files to scan based on query parameters Physical plan generation, when the final parameters of the scan are available. For example:
  • Schema of data to read after pruning away unused columns
  • Query filters to apply after filter rearrangement
Distribute the file information to workers Physical plan execution, only if it is a distributed engine.
Read the columnar data using the file information Physical plan execution, when the data is being processed by the engine
Let's understand the details of each step. #### Step 3.1: Resolve the table snapshot to query The first step is to resolve the consistent snapshot and the schema associated with it. This is often required by the connector/ engine to resolve and validate the logical plan of the scan query (if the concept of logical plan exists in your engine). To achieve this, the connector has to do the following. * Resolve the table path from the query: If the path is directly available, then this is easy. Otherwise, if it is a query based on a catalog table (for example, a Delta table defined in Hive Metastore), then the connector has to resolve the table path from the catalog. * Initialize the `Engine` object: Create a new instance of the `Engine` that you have chosen in [Step 2](#build-your-own Engine). * Initialize the Kernel objects and get the schema: Assuming the query is on the latest available version/snapshot of the table, you can get the table schema as follows: ```java import io.delta.kernel.*; import io.delta.kernel.defaults.engine.*; Engine myEngine = new MyEngine(); Table myTable = Table.forPath(myTablePath); Snapshot mySnapshot = myTable.getLatestSnapshot(myEngine); StructType mySchema = mySnapshot.getSchema(myEngine); ``` If you want to query a specific version of the table (that is, not the schema), then you can get the required snapshot as `myTable.getSnapshot(version)`. #### Step 3.2: Resolve files to scan Next, we need to build a Scan object using more information from the query. Here we are going to assume that the connector/engine has been able to extract the following details from the query (say, after optimizing the logical plan): * Read schema: The columns in the table that the query needs to read. This may be the full set of columns or a subset of columns. * Query filters: The filters on partitions or data columns that can be used skip reading table data. To provide this information to Kernel, you have to do the following: * Convert the engine-specific schema and filter expressions to Kernel schema and expressions: For schema, you have to create a `StructType` object. For the filters, you have to create an `Expression` object using all the available subclasses of `Expression`. * Build the scan with the converted information: Build the scan as follows: ```java import io.delta.kernel.expressions.*; import io.delta.kernel.types.*; StructType readSchema = ... ; // convert engine schema Predicate filterExpr = ... ; // convert engine filter expression Scan myScan = mySnapshot.getScanBuilder().withFilter(filterExpr).withReadSchema(readSchema).build(); ``` * Resolve the information required to file reads: The generated Scan object has two sets of information. * Scan files: `myScan.getScanFiles()` returns an iterator of `ColumnarBatch`es. Each batch in the iterator contains rows and each row has information about a single file that has been selected based on the query filter. * Scan state: `myScan.getScanState()` returns a `Row` that contains all the information that is common across all the files that need to be read. ```java Row myScanStateRow = myScan.getScanState(); CloseableIterator myScanFilesAsBatches = myScan.getScanFiles(); while (myScanFilesAsBatches.hasNext()) { FilteredColumnarBatch scanFileBatch = myScanFilesAsBatches.next(); CloseableIterator myScanFilesAsRows = scanFileBatch.getRows(); } ``` As we will soon see, reading the columnar data from a selected file will need to use both, the scan state row, and a scan file row with the file information. ##### Requirements and guarantees Here are the details you need to ensure when defining this scan. * The provided `readSchema` must be the exact schema of the data that the engine will expect when executing the query. Any mismatch in the schema defined during this query planning and the query execution will result in runtime failures. Hence you must build the scan with the readSchema only after the engine has finalized the logical plan after any optimizations like column pruning. * When applicable (for example, with Java Kernel APIs), you have to make sure to call the close() method as you consume the `ColumnarBatch`es of scan files (that is, either serialize the rows or use them to read the table data). #### Step 3.3: Distribute the file information to the workers If you are building a connector for a distributed engine like Spark/Presto/Trino/Flink, then your connector has to send all the scan metadata from the query planning machine (henceforth called the driver) to task execution machines (henceforth called the workers). You will have to serialize and deserialize the scan state and scan file rows. It is the connector job to implement serialization and deserialization utilities for a [`Row`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/Row.html). If the connector wants to split reading one scan file into multiple tasks, it can add additional connector specific split context to the task. At the task, the connector can use its own Parquet reader to read the specific part of the file indicated by the split info. ##### Custom `Row` Serializer/Deserializer Here are steps on how to build your own serializer/deserializer such that it will work with any [`Row`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/Row.html) of any schema. * Serializing * First serialize the row schema, that is, `StructType` object. * Then, use the schema to identify types of each column/ordinal in the `Row` and use that to serialize all the values one by one. * Deserializing * Define your own class that extends the Row interface. It must be able to handle complex types like arrays, nested structs and maps. * First deserialize the schema. * Then, use the schema to deserialize the values and put them in an instance of your custom Row class. ```java import io.delta.kernel.utils.*; // In the driver where query planning is being done Byte[] scanStateRowBytes = RowUtils.serialize(scanStateRow); Byte[] scanFileRowBytes = RowUtils.serialize(scanFileRow); // Optionally the connector adds a split info to the task (scan file, scan state) to // split reading of a Parquet file into multiple tasks. The task gets split info // along with the scan file row and scan state row. Split split = ...; // connector specific class, not related to Kernel // Send these over to the worker // In the worker when data will be read, after rowBytes have been sent over Row scanStateRow = RowUtils.deserialize(scanStateRowBytes); Row scanFileRow = RowUtils.deserialize(scanFileRowBytes); Split split = ... deserialize split info ...; ``` #### Step 3.4: Read the columnar data Finally, we are ready to read the columnar data. You will have to do the following: * Read the physical data from Parquet file as indicated by the scan file row, scan state, and optionally the split info * Convert the physical data into logical data of the table using the Kernel's APIs. ```java Row scanStateRow = ... ; Row scanFileRow = ... ; Split split = ...; // Additional option predicate such as dynamic filters the connector wants to // pass to the reader when reading files. Predicate optPredicate = ...; // Get the physical read schema of columns to read from the Parquet data files StructType physicalReadSchema = ScanStateRow.getPhysicalDataReadSchema(engine, scanStateRow); // From the scan file row, extract the file path, size and modification metadata // needed to read the file. FileStatus fileStatus = InternalScanFileUtils.getAddFileStatus(scanFileRow); // Open the scan file which is a Parquet file using connector's own // Parquet reader which supports reading specific parts (split) of the file. // If the connector doesn't have its own Parquet reader, it can use the // default Parquet reader provider which at the moment doesn't support reading // a specific part of the file, but reads the entire file from the beginning. CloseableIterator physicalDataIter = connectParquetReader.readParquetFile( fileStatus physicalReadSchema, split, // what part of the Parquet file to read data from optPredicate /* additional predicate the connector can apply to filter data from the reader */ ); // Now the physical data read from the Parquet data file is converted to logical data // the table represents. // Logical data may include the addition of partition columns and/or // subset of rows deleted CloseableIterator transformedData = Scan.transformPhysicalData( engine, scanState, scanFileRow, physicalDataIter)); ``` * Resolve the data in the batches: Each [`FilteredColumnarBatch`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/FilteredColumnarBatch.html) has two components: * Columnar batch (returned by `FilteredColumnarBatch.getData()`): This is the data read from the files having the schema matching the readSchema provided when the Scan object was built in the earlier step. * Optional selection vector (returned by `FilteredColumnarBatch.getSelectionVector()`): Optionally, a boolean vector that will define which rows in the batch are valid and should be consumed by the engine. If the selection vector is present, then you will have to apply it to the batch to resolve the final consumable data. * Convert to engine-specific data format: Each connector/engine has its own native row / columnar batch formats and interfaces. To return the read data batches to the engine, you have to convert them to fit those engine-specific formats and/or interfaces. Here are a few tips that you can follow to make this efficient. * Matching the engine-specific format: Some engines may expect the data in an in-memory format that may be different from the data produced by `getData()`. So you will have to do the data conversion for each column vector in the batch as needed. * Matching the engine-specific interfaces: You may have to implement wrapper classes that extend the engine-specific interfaces and appropriately encapsulate the row data. For best performance, you can implement your own Parquet reader and other `Engine` implementations to make sure that every `ColumnVector` generated is already in the engine-native format thus eliminating any need to convert. Now you should be able to read the Delta table correctly. ### Step 4: Build append support in your connector In this section, we are going to walk through the likely sequence of Kernel API calls your connector will have to make to append data to a table. The exact timing of making these calls in your connector in the context of connector-engine interactions depends entirely on the engine-connector APIs and is, therefore, beyond the scope of this guide. However, we will try to provide broad guidelines that are likely (but not guaranteed) to apply to your connector-engine setup. For this purpose, we are going to assume that the engine goes through the following phases when processing a write query - logical plan analysis, physical plan generation, and physical plan execution. Based on these broad characterizations, a typical control and data flow for reading a Delta table is going to be as follows:
Step Typical query phase when this step occurs
Determine the schema of the data that needs to be written to the table. Schema is derived from the existing table or from the parent operation of the `write` operator in the query plan when the table doesn't exist yet. Logical plan analysis phase when the plan's schema (`write` operator schema matches the table schema, etc.) and other details need to be resolved and validated.
Determine the physical partitioning of the data based on the table schema and partition columns either from the existing table or from the query plan (for new tables) Physical plan generation, where the number of writer tasks, data schema and partitioning is determined
Distribute the writer tasks definitions (which include the transaction state) to workers. Physical plan execution, only if it is a distributed engine.
Tasks write the data to data files and send the data file info to the driver. Physical plan execution, when the data is actually written to the table location
Finalize the query. Here, all the info of the data files written by the tasks is aggregated and committed to the transaction created at the beginning of the physical execution. Finalize the query. This happens on the driver where the query has started.
Let's understand the details of each step. #### Step 4.1: Determine the schema of the data that needs to be written to the table The first step is to resolve the output data schema. This is often required by the connector/ engine to resolve and validate the logical plan of the query (if the concept of logical plan exists in your engine). To achieve this, the connector has to do the following. At a high level query plan is a tree of operators where the leaf-level operators generate or read data from storage/tables and feed it upwards towards the parent operator nodes. This data transfer happens until it reaches the root operator node where the query is finalized (either the results are sent to the client or data is written to another table). * Create the `Table` object * From the `Table` object try to get the schema. * If the table is not found * the query includes creating the table (e.g., `CREATE TABLE AS` SQL query); * the schema is derived from the operator above the `write` that feeds the data to the `write` operator. * the query doesn't include creating new table, an exception is thrown saying the table is not found * If the table already exists * get the schema from the table and check if it matches the schema of the `write` operator. If not throw an exception. * Create a `TransactionBuilder` - this basically begins the steps of transaction construction. ```java import io.delta.kernel.*; import io.delta.kernel.defaults.engine.*; Engine myEngine = new MyEngine(); Table myTable = Table.forPath(myTablePath); StructType writeOperatorSchema = // ... derived from the query operator tree ... StructType dataSchema; boolean isNewTable = false; try { Snapshot mySnapshot = myTable.getLatestSnapshot(myEngine); dataSchema = mySnapshot.getSchema(myEngine); // .. check dataSchema and writeOperatorSchema match ... } catch(TableNotFoundException e) { isNewTable = true; dataSchema = writeOperatorSchema; } TransactionBuilder txnBuilder = myTable.createTransactionBuilder( myEngine, "Examples", /* engineInfo - connector can add its own identifier which is noted in the Delta Log */ Operation /* What is the operation we are trying to perform? This is noted in the Delta Log */ ); if (isNewTable) { // For a new table set the table schema in the transaction builder txnBuilder = txnBuilder.withSchema(engine, dataSchema) } ``` #### Step 4.2: Determine the physical partitioning of the data based on the table schema and partition columns Partition columns are found either from the query (for new tables, the query defines the partition columns) or from the existing table. ```java TransactionBuilder txnBuilder = ... from the last step ... Transaction txn; List partitionColumns = ... if (newTable) { partitionColumns = ... derive from the query parameters (ex. PARTITION BY clause in SQL) ... txnBuilder = txnBuilder.withPartitionColumns(engine, partitionColumns); txn = txnBuilder.build(engine); } else { txn = txnBuilder.build(engine); partitionColumns = txn.getPartitionColumns(engine); } ``` At the end of this step, we have the `Transaction` and schema of the data to generate and its partitioning. #### Step 4.3: Distribute the writer tasks definitions (which include the transaction state) to workers If you are building a connector for a distributed engine like Spark/Presto/Trino/Flink, then your connector has to send all the writer metadata from the query planning machine (henceforth called the driver) to task execution machines (henceforth called the workers). You will have to serialize and deserialize the transaction state. It is the connector job to implement serialization and deserialization utilities for a [`Row`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/Row.html). More details on a custom `Row` SerDe are found [here](#custom-row-serializerdeserializer). ```java Row txnState = txn.getState(engine); String jsonTxnState = serializeToJson(txnState); ``` #### Step 4.4: Tasks write the data to data files and send the data file info to the driver. In this step (which is executed on the worker nodes inside each task): * Deserialize the transaction state * Writer operator within the task gets the data from its parent operator. * The data is converted into a `FilteredColumnarBatch`. Each [`FilteredColumnarBatch`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/data/FilteredColumnarBatch.html) has two components: * Columnar batch (returned by `FilteredColumnarBatch.getData()`): This is the data read from the files having the schema matching the readSchema provided when the Scan object was built in the earlier step. * Optional selection vector (returned by `FilteredColumnarBatch.getSelectionVector()`): Optionally, a boolean vector that will define which rows in the batch are valid and should be consumed by the engine. * The connector can create `FilteredColumnBatch` wrapper around data in its own in-memory format. * Check if the data is partitioned or not. If not partitioned, partition the data by partition values. * For each partition generate the map of the partition column to the partition value * Use Kernel to convert the partitioned data into physical data that should go into the data files * Write the physical data into one or more data files. * Convert data file statues into a Delta log actions * Serialize the Delta log action `Row` objects and send them to the driver node ``` Row txnState = ... deserialize from JSON string sent by the driver ... CloseableIterator data = ... generate data ... // If the table is un-partitioned then this is an empty map Map partitionValues = ... prepare the partition values ... // First transform the logical data to physical data that needs to be written // to the Parquet files CloseableIterator physicalData = Transaction.transformLogicalData(engine, txnState, data, partitionValues); // Get the write context DataWriteContext writeContext = Transaction.getWriteContext(engine, txnState, partitionValues); // Now write the physical data to Parquet files CloseableIterator dataFiles = engine.getParquetHandler() .writeParquetFiles( writeContext.getTargetDirectory(), physicalData, writeContext.getStatisticsColumns()); // Now convert the data file status to data actions that needs to be written to the Delta table log CloseableIterator partitionDataActions = Transaction.generateAppendActions( engine, txnState, dataFiles, writeContext); .... serialize `partitionDataActions` and send them to driver node ``` #### Step 4.5: Finalize the query. At the driver node, the delta log actions from all the tasks are received and committed to the transaction. The tasks send the Delta log actions as a serialized JSON and deserialize them back to `Row` objects. ``` // Create a iterable out of the data actions. If the contents are too big to fit in memory, // the connector may choose to write the data actions to a temporary file and return an // iterator that reads from the file. CloseableIterable dataActionsIterable = CloseableIterable.inMemoryIterable( toCloseableIterator(dataActions.iterator())); // Commit the transaction. TransactionCommitResult commitResult = txn.commit(engine, dataActionsIterable); // Optional step if (commitResult.isReadyForCheckpoint()) { // Checkpoint the table Table.forPath(engine, tablePath).checkpoint(engine, commitResult.getVersion()); } ``` Thats it. Now you should be able to append data to Delta tables using the Kernel APIs. ## Migration guide Kernel APIs are still evolving and new features are being added. Kernel authors try to make the API changes backward compatible as much as they can with each new release, but sometimes it is hard to maintain the backward compatibility for a project that is evolving rapidly. This section provides guidance on how to migrate your connector to the latest version of Delta Kernel. With each new release the [examples](https://github.com/delta-io/delta/tree/master/kernel/examples) are kept up-to-date with the latest API changes. You can refer to the examples to understand how to use the new APIs. ### Migration from Delta Lake version 3.1.0 to 3.2.0 Following are API changes in Delta Kernel 3.2.0 that may require changes in your connector. #### Rename `TableClient` to [`Engine`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/engine/Engine.html) The `TableClient` interface has been renamed to `Engine`. This is the most significant API change in this release. The `TableClient` interface name is not exactly representing the functionality it provides. At a high level it provides capabilities such as reading Parquet files, JSON files, evaluating expressions on data and file system functionality. These are basically the heavy lift operations that Kernel depends on as a separate interface to allow the connectors to substitute their own custom implementation of the same functionality (e.g. custom Parquet reader). Essentially, these functionalities are the core of the `engine` functionalities. By renaming to `Engine`, we are representing the interface functionality with a proper name that is easy to understand. The `DefaultTableClient` has been renamed to [`DefaultEngine`](https://delta-io.github.io/delta/snapshot/kernel-defaults/java/io/delta/kernel/defaults/engine/DefaultEngine.html). #### [`Table.forPath(Engine engine, String tablePath)`](https://delta-io.github.io/delta/snapshot/kernel-api/java/io/delta/kernel/Table.html#forPath-io.delta.kernel.engine.Engine-java.lang.String-) behavior change Earlier when a non-existent table path is passed, the API used to throw `TableNotFoundException`. Now it doesn't throw the exception. Instead, it returns a `Table` object. When trying to get a `Snapshot` from the table object it throws the `TableNotFoundException`. #### [`FileSystemClient.resolvePath`](https://delta-io.github.io/delta/snapshot/kernel-defaults/java/io/delta/kernel/defaults/engine/DefaultFileSystemClient.html#resolvePath-java.lang.String-) behavior change Earlier when a non-existent path is passed, the API used to throw `FileNotFoundException`. Now it doesn't throw the exception. It still resolves the given path into a fully qualified path. ================================================ FILE: kernel/build/sbt ================================================ #!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # # This file contains code from the Apache Spark project (original license above). # It contains modifications, which are licensed as follows: # # # Copyright (2021) The Delta Lake Project Authors. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # When creating new tests for Spark SQL Hive, the HADOOP_CLASSPATH must contain the hive jars so # that we can run Hive to generate the golden answer. This is not required for normal development # or testing. if [ -n "$HIVE_HOME" ]; then for i in "$HIVE_HOME"/lib/* do HADOOP_CLASSPATH="$HADOOP_CLASSPATH:$i" done export HADOOP_CLASSPATH fi realpath () { ( TARGET_FILE="$1" cd "$(dirname "$TARGET_FILE")" TARGET_FILE="$(basename "$TARGET_FILE")" COUNT=0 while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] do TARGET_FILE="$(readlink "$TARGET_FILE")" cd $(dirname "$TARGET_FILE") TARGET_FILE="$(basename $TARGET_FILE)" COUNT=$(($COUNT + 1)) done echo "$(pwd -P)/"$TARGET_FILE"" ) } if [[ "$JENKINS_URL" != "" ]]; then # Make Jenkins use Google Mirror first as Maven Central may ban us SBT_REPOSITORIES_CONFIG="$(dirname "$(realpath "$0")")/sbt-config/repositories" export SBT_OPTS="-Dsbt.override.build.repos=true -Dsbt.repository.config=$SBT_REPOSITORIES_CONFIG" fi . "$(dirname "$(realpath "$0")")"/sbt-launch-lib.bash declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy" declare -r sbt_opts_file=".sbtopts" declare -r etc_sbt_opts_file="/etc/sbt/sbtopts" usage() { cat < path to global settings/plugins directory (default: ~/.sbt) -sbt-boot path to shared boot directory (default: ~/.sbt/boot in 0.11 series) -ivy path to local Ivy repository (default: ~/.ivy2) -mem set memory options (default: $sbt_mem, which is $(get_mem_opts $sbt_mem)) -no-share use all local caches; no sharing -no-global uses global caches, but does not use global ~/.sbt directory. -jvm-debug Turn on JVM debugging, open at the given port. -batch Disable interactive mode # sbt version (default: from project/build.properties if present, else latest release) -sbt-version use the specified version of sbt -sbt-jar use the specified jar as the sbt launcher -sbt-rc use an RC version of sbt -sbt-snapshot use a snapshot version of sbt # java version (default: java from PATH, currently $(java -version 2>&1 | grep version)) -java-home alternate JAVA_HOME # jvm options and output control JAVA_OPTS environment variable, if unset uses "$java_opts" SBT_OPTS environment variable, if unset uses "$default_sbt_opts" .sbtopts if this file exists in the current directory, it is prepended to the runner args /etc/sbt/sbtopts if this file exists, it is prepended to the runner args -Dkey=val pass -Dkey=val directly to the java runtime -J-X pass option -X directly to the java runtime (-J is stripped) -S-X add -X to sbt's scalacOptions (-S is stripped) -PmavenProfiles Enable a maven profile for the build. In the case of duplicated or conflicting options, the order above shows precedence: JAVA_OPTS lowest, command line options highest. EOM } process_my_args () { while [[ $# -gt 0 ]]; do case "$1" in -no-colors) addJava "-Dsbt.log.noformat=true" && shift ;; -no-share) addJava "$noshare_opts" && shift ;; -no-global) addJava "-Dsbt.global.base=$(pwd)/project/.sbtboot" && shift ;; -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;; -sbt-dir) require_arg path "$1" "$2" && addJava "-Dsbt.global.base=$2" && shift 2 ;; -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;; -batch) exec /dev/null) if [[ ! $? ]]; then saved_stty="" fi } saveSttySettings trap onExit INT run "$@" exit_status=$? onExit ================================================ FILE: kernel/build/sbt-config/repositories ================================================ [repositories] local local-preloaded-ivy: file:///${sbt.preloaded-${sbt.global.base-${user.home}/.sbt}/preloaded/}, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext] local-preloaded: file:///${sbt.preloaded-${sbt.global.base-${user.home}/.sbt}/preloaded/} gcs-maven-central-mirror: https://maven-central.storage-download.googleapis.com/repos/central/data/ maven-central typesafe-ivy-releases: https://repo.typesafe.com/typesafe/ivy-releases/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly sbt-ivy-snapshots: https://repo.scala-sbt.org/scalasbt/ivy-snapshots/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly sbt-plugin-releases: https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/, [organization]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] bintray-typesafe-sbt-plugin-releases: https://dl.bintray.com/typesafe/sbt-plugins/, [organization]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] bintray-spark-packages: https://dl.bintray.com/spark-packages/maven/ typesafe-releases: https://repo.typesafe.com/typesafe/releases/ ================================================ FILE: kernel/build/sbt-launch-lib.bash ================================================ #!/usr/bin/env bash # # A library to simplify using the SBT launcher from other packages. # Note: This should be used by tools like giter8/conscript etc. # TODO - Should we merge the main SBT script with this library? if test -z "$HOME"; then declare -r script_dir="$(dirname "$script_path")" else declare -r script_dir="$HOME/.sbt" fi declare -a residual_args declare -a java_args declare -a scalac_args declare -a sbt_commands declare -a maven_profiles if test -x "$JAVA_HOME/bin/java"; then echo -e "Using $JAVA_HOME as default JAVA_HOME." echo "Note, this will be overridden by -java-home if it is set." declare java_cmd="$JAVA_HOME/bin/java" else declare java_cmd=java fi echoerr () { echo 1>&2 "$@" } vlog () { [[ $verbose || $debug ]] && echoerr "$@" } dlog () { [[ $debug ]] && echoerr "$@" } acquire_sbt_jar () { SBT_VERSION=`awk -F "=" '/sbt\.version/ {print $2}' ./project/build.properties` # Download sbt from mirror URL if the environment variable is provided if [[ "${SBT_VERSION}" == "0.13.18" ]] && [[ -n "${SBT_MIRROR_JAR_URL}" ]]; then URL1="${SBT_MIRROR_JAR_URL}" elif [[ "${SBT_VERSION}" == "1.5.5" ]] && [[ -n "${SBT_1_5_5_MIRROR_JAR_URL}" ]]; then URL1="${SBT_1_5_5_MIRROR_JAR_URL}" else URL1=${DEFAULT_ARTIFACT_REPOSITORY:-https://repo1.maven.org/maven2/}org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar fi JAR=build/sbt-launch-${SBT_VERSION}.jar sbt_jar=$JAR if [[ ! -f "$sbt_jar" ]]; then # Download sbt launch jar if it hasn't been downloaded yet if [ ! -f "${JAR}" ]; then # Download printf 'Attempting to fetch sbt from %s\n' "${URL1}" JAR_DL="${JAR}.part" if [ $(command -v curl) ]; then curl --fail --location --silent ${URL1} > "${JAR_DL}" &&\ mv "${JAR_DL}" "${JAR}" elif [ $(command -v wget) ]; then wget --quiet ${URL1} -O "${JAR_DL}" &&\ mv "${JAR_DL}" "${JAR}" else printf "You do not have curl or wget installed, please install sbt manually from https://www.scala-sbt.org/\n" exit -1 fi fi if [ ! -f "${JAR}" ]; then # We failed to download printf "Our attempt to download sbt locally to ${JAR} failed. Please install sbt manually from https://www.scala-sbt.org/\n" exit -1 fi printf "Launching sbt from ${JAR}\n" fi } execRunner () { # print the arguments one to a line, quoting any containing spaces [[ $verbose || $debug ]] && echo "# Executing command line:" && { for arg; do if printf "%s\n" "$arg" | grep -q ' '; then printf "\"%s\"\n" "$arg" else printf "%s\n" "$arg" fi done echo "" } "$@" } addJava () { dlog "[addJava] arg = '$1'" java_args=( "${java_args[@]}" "$1" ) } enableProfile () { dlog "[enableProfile] arg = '$1'" maven_profiles=( "${maven_profiles[@]}" "$1" ) export SBT_MAVEN_PROFILES="${maven_profiles[@]}" } addSbt () { dlog "[addSbt] arg = '$1'" sbt_commands=( "${sbt_commands[@]}" "$1" ) } addResidual () { dlog "[residual] arg = '$1'" residual_args=( "${residual_args[@]}" "$1" ) } addDebugger () { addJava "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$1" } # a ham-fisted attempt to move some memory settings in concert # so they need not be dicked around with individually. get_mem_opts () { local mem=${1:-1000} local perm=$(( $mem / 4 )) (( $perm > 256 )) || perm=256 (( $perm < 4096 )) || perm=4096 local codecache=$(( $perm / 2 )) echo "-Xms${mem}m -Xmx${mem}m -XX:ReservedCodeCacheSize=${codecache}m" } require_arg () { local type="$1" local opt="$2" local arg="$3" if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then echo "$opt requires <$type> argument" 1>&2 exit 1 fi } is_function_defined() { declare -f "$1" > /dev/null } process_args () { while [[ $# -gt 0 ]]; do case "$1" in -h|-help) usage; exit 1 ;; -v|-verbose) verbose=1 && shift ;; -d|-debug) debug=1 && shift ;; -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;; -mem) require_arg integer "$1" "$2" && sbt_mem="$2" && shift 2 ;; -jvm-debug) require_arg port "$1" "$2" && addDebugger $2 && shift 2 ;; -batch) exec 4.0.0 org.example kernel-examples 0.1-SNAPSHOT 1.8 1.8 "" 3.2.0-SNAPSHOT 3.3.1 staging-repo ${staging.repo.url} io.delta delta-kernel-api ${delta-kernel.version} io.delta delta-kernel-defaults ${delta-kernel.version} io.delta delta-storage ${delta-kernel.version} org.apache.hadoop hadoop-client-runtime ${hadoop.version} org.apache.hadoop hadoop-client-api ${hadoop.version} commons-cli commons-cli 1.5.0 com.fasterxml.jackson.core jackson-databind 2.13.5 ================================================ FILE: kernel/examples/kernel-examples/src/main/java/io/delta/kernel/examples/BaseTableReader.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.examples; import java.io.IOException; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.IntStream; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.hadoop.conf.Configuration; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.TableNotFoundException; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.types.*; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.defaults.engine.DefaultEngine; /** * Base class for reading Delta Lake tables using the Delta Kernel APIs. */ public abstract class BaseTableReader { public static final int DEFAULT_LIMIT = 20; protected final String tablePath; protected final Engine engine; public BaseTableReader(String tablePath) { this.tablePath = requireNonNull(tablePath); this.engine = DefaultEngine.create(new Configuration()); } /** * Show the given {@code limit} rows containing the given columns with the predicate from the * table. * * @param limit Max number of rows to show. * @param columnsOpt If null, show all columns in the table. * @param predicateOpt Optional predicate * @return Number of rows returned by the query. * @throws TableNotFoundException * @throws IOException */ public abstract int show( int limit, Optional> columnsOpt, Optional predicateOpt) throws TableNotFoundException, IOException; /** * Utility method to return a pruned schema that contains the given {@code columns} from * {@code baseSchema} */ protected static StructType pruneSchema(StructType baseSchema, Optional> columns) { if (!columns.isPresent()) { return baseSchema; } List selectedFields = columns.get().stream().map(column -> { if (baseSchema.indexOf(column) == -1) { throw new IllegalArgumentException( format("Column %s is not found in table", column)); } return baseSchema.get(column); }).collect(Collectors.toList()); return new StructType(selectedFields); } protected static int printData(FilteredColumnarBatch data, int maxRowsToPrint) { int printedRowCount = 0; try (CloseableIterator rows = data.getRows()) { while (rows.hasNext()) { printRow(rows.next()); printedRowCount++; if (printedRowCount == maxRowsToPrint) { break; } } } catch (Exception e) { throw new RuntimeException(e); } return printedRowCount; } protected static void printSchema(StructType schema) { System.out.printf(formatter(schema.length()), schema.fieldNames().toArray(new String[0])); } protected static void printRow(Row row){ int numCols = row.getSchema().length(); Object[] rowValues = IntStream.range(0, numCols) .mapToObj(colOrdinal -> getValue(row, colOrdinal)) .toArray(); // TODO: Need to handle the Row, Map, Array, Timestamp, Date types specially to // print them in the format they need. Copy this code from Spark CLI. System.out.printf(formatter(numCols), rowValues); } /** * Minimum command line options for any implementation of this reader. */ protected static Options baseOptions() { return new Options() .addRequiredOption("t", "table", true, "Fully qualified table path") .addOption("c", "columns", true, "Comma separated list of columns to read from the table. " + "Ex. --columns=id,name,address") .addOption( Option.builder() .option("l") .longOpt("limit") .hasArg(true) .desc("Maximum number of rows to read from the table (default 20).") .type(Number.class) .build() ); } protected static Optional> parseColumnList(CommandLine cli, String optionName) { return Optional.ofNullable(cli.getOptionValue(optionName)) .map(colString -> Arrays.asList(colString.split(",[ ]*"))); } protected static int parseInt(CommandLine cli, String optionName, int defaultValue) throws ParseException { return Optional.ofNullable(cli.getParsedOptionValue(optionName)) .map(Number.class::cast) .map(Number::intValue) .orElse(defaultValue); } private static String formatter(int length) { return IntStream.range(0, length) .mapToObj(i -> "%20s") .collect(Collectors.joining("|")) + "\n"; } private static String getValue(Row row, int columnOrdinal) { DataType dataType = row.getSchema().at(columnOrdinal).getDataType(); if (row.isNullAt(columnOrdinal)) { return null; } else if (dataType instanceof BooleanType) { return Boolean.toString(row.getBoolean(columnOrdinal)); } else if (dataType instanceof ByteType) { return Byte.toString(row.getByte(columnOrdinal)); } else if (dataType instanceof ShortType) { return Short.toString(row.getShort(columnOrdinal)); } else if (dataType instanceof IntegerType) { return Integer.toString(row.getInt(columnOrdinal)); } else if (dataType instanceof DateType) { // DateType data is stored internally as the number of days since 1970-01-01 int daysSinceEpochUTC = row.getInt(columnOrdinal); return LocalDate.ofEpochDay(daysSinceEpochUTC).toString(); } else if (dataType instanceof LongType) { return Long.toString(row.getLong(columnOrdinal)); } else if (dataType instanceof TimestampType || dataType instanceof TimestampNTZType) { // Timestamps are stored internally as the number of microseconds since epoch. // TODO: TimestampType should use the session timezone to display values. long microSecsSinceEpochUTC = row.getLong(columnOrdinal); LocalDateTime dateTime = LocalDateTime.ofEpochSecond( microSecsSinceEpochUTC / 1_000_000 /* epochSecond */, (int) (1000 * microSecsSinceEpochUTC % 1_000_000) /* nanoOfSecond */, ZoneOffset.UTC); return dateTime.toString(); } else if (dataType instanceof FloatType) { return Float.toString(row.getFloat(columnOrdinal)); } else if (dataType instanceof DoubleType) { return Double.toString(row.getDouble(columnOrdinal)); } else if (dataType instanceof StringType) { return row.getString(columnOrdinal); } else if (dataType instanceof BinaryType) { return new String(row.getBinary(columnOrdinal)); } else if (dataType instanceof DecimalType) { return row.getDecimal(columnOrdinal).toString(); } else if (dataType instanceof StructType) { return "TODO: struct value"; } else if (dataType instanceof ArrayType) { return "TODO: list value"; } else if (dataType instanceof MapType) { return "TODO: map value"; } else { throw new UnsupportedOperationException("unsupported data type: " + dataType); } } } ================================================ FILE: kernel/examples/kernel-examples/src/main/java/io/delta/kernel/examples/BaseTableWriter.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.examples; import java.util.*; import org.apache.hadoop.conf.Configuration; import io.delta.kernel.TransactionCommitResult; import io.delta.kernel.data.*; import io.delta.kernel.engine.Engine; import io.delta.kernel.types.*; import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.defaults.internal.data.DefaultColumnarBatch; public class BaseTableWriter { protected final Engine engine = DefaultEngine.create(new Configuration()); /** * Schema used in examples for table create and/or writes */ protected final StructType exampleTableSchema = new StructType() .add("id", IntegerType.INTEGER) .add("name", StringType.STRING) .add("address", StringType.STRING) .add("salary", DoubleType.DOUBLE); /** * Schema and partition columns used in examples for partitioned table create and/or writes. */ protected final StructType examplePartitionedTableSchema = new StructType() .add("id", IntegerType.INTEGER) .add("name", StringType.STRING) .add("city", StringType.STRING) .add("salary", DoubleType.DOUBLE); protected final List examplePartitionColumns = Collections.singletonList("city"); void verifyCommitSuccess(String tablePath, TransactionCommitResult result) { // Verify the commit was successful if (result.getVersion() >= 0) { System.out.println("Table created successfully at: " + tablePath); } else { // This should never happen. If there is a reason for table be not created // `Transaction.commit` always throws an exception. throw new RuntimeException("Table creation failed"); } } /** * Create data batch for a un-partitioned table with schema {@link #exampleTableSchema}. * * @param offset Offset that affects the generated data. * @return */ FilteredColumnarBatch generateUnpartitionedDataBatch(int offset) { ColumnVector[] vectors = new ColumnVector[exampleTableSchema.length()]; // Create a batch with 5 rows // id vectors[0] = intVector( Arrays.asList(offset, 1 + offset, 2 + offset, 3 + offset, 4 + offset)); // name vectors[1] = stringVector( Arrays.asList("Alice", "Bob", "Charlie", "David", "Eve")); // address vectors[2] = stringVector( Arrays.asList( "123 Main St", "456 Elm St", "789 Cedar St", "101 Oak St", "121 Pine St")); // salary vectors[3] = doubleVector( Arrays.asList( 100.0d + offset, 200.0d + offset, 300.0d + offset, 400.0d + offset, 500.0d + offset)); ColumnarBatch batch = new DefaultColumnarBatch(5, exampleTableSchema, vectors); return new FilteredColumnarBatch( batch, // data // Optional selection vector. If want to write only a subset of rows from the batch. Optional.empty()); } /** * Create data batch for a partitioned table with schema {@link #examplePartitionedTableSchema}. * * @param offset Offset that affects the generated data. * @param city City value for the partition column. * @return */ FilteredColumnarBatch generatedPartitionedDataBatch(int offset, String city) { ColumnVector[] vectors = new ColumnVector[examplePartitionedTableSchema.length()]; // Create a batch with 5 rows // id vectors[0] = intVector( Arrays.asList(offset, 1 + offset, 2 + offset, 3 + offset, 4 + offset)); // name vectors[1] = stringVector( Arrays.asList("Alice", "Bob", "Charlie", "David", "Eve")); // city - given city is a partition column we expect the batch to contain the same // value for all rows. vectors[2] = stringSingleValueVector(city, 5); // salary vectors[3] = doubleVector( Arrays.asList( 100.0d + offset, 200.0d + offset, 300.0d + offset, 400.0d + offset, 500.0d + offset)); ColumnarBatch batch = new DefaultColumnarBatch(5, examplePartitionedTableSchema, vectors); return new FilteredColumnarBatch( batch, // data // Optional selection vector. If want to write only a subset of rows from the batch. Optional.empty()); } //////////////////////// Helper methods to create ColumnVectors //////////////////////// // These are sample vectors which can be created as wrappers as engine specific // // vector types. // //////////////////////////////////////////////////////////////////////////////////////// static ColumnVector intVector(List data) { return new ColumnVector() { @Override public DataType getDataType() { return IntegerType.INTEGER; } @Override public int getSize() { return data.size(); } @Override public void close() { } @Override public boolean isNullAt(int rowId) { return data.get(rowId) == null; } @Override public int getInt(int rowId) { return data.get(rowId); } }; } static ColumnVector doubleVector(List data) { return new ColumnVector() { @Override public DataType getDataType() { return DoubleType.DOUBLE; } @Override public int getSize() { return data.size(); } @Override public void close() { } @Override public boolean isNullAt(int rowId) { return data.get(rowId) == null; } @Override public double getDouble(int rowId) { return data.get(rowId); } }; } static ColumnVector stringVector(List data) { return new ColumnVector() { @Override public DataType getDataType() { return StringType.STRING; } @Override public int getSize() { return data.size(); } @Override public void close() { } @Override public boolean isNullAt(int rowId) { return data.get(rowId) == null; } @Override public String getString(int rowId) { return data.get(rowId); } }; } static ColumnVector stringSingleValueVector(String value, int size) { return new ColumnVector() { @Override public DataType getDataType() { return StringType.STRING; } @Override public int getSize() { return size; } @Override public void close() { } @Override public boolean isNullAt(int rowId) { return value == null; } @Override public String getString(int rowId) { return value; } }; } } ================================================ FILE: kernel/examples/kernel-examples/src/main/java/io/delta/kernel/examples/CreateTable.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.examples; import org.apache.commons.cli.Options; import io.delta.kernel.*; import io.delta.kernel.utils.CloseableIterable; import static io.delta.kernel.examples.utils.Utils.parseArgs; /** * Example program to create a Delta table (no data is written) using the Kernel APIs. *

* Creates two tables with the following schema and partition columns in the input given directory * location. * 

 *     Table 1: un-partitioned table
 *     CREATE TABLE example (id INT, name STRING, address STRING, salary DECIMAL(1, 3))
 *
 *     Table 2: partitioned table
 *     CREATE TABLE example_partitioned (id INT, name STRING, salary DECIMAL(1, 3), city STRING))
 *     PARTITIONED BY (city)
 *
*

*

* It prints the table locations at the end of the successful execution. */ public class CreateTable extends BaseTableWriter { public static void main(String[] args) throws Exception { Options options = new Options() .addOption("l", "location", true, "Locations where the sample tables are created"); new CreateTable().runExamples(parseArgs(options, args).getOptionValue("location")); } public void runExamples(String location) { createUnpartitionedTable(location + "/example"); createPartitionedTable(location + "/example_partitioned"); } public void createUnpartitionedTable(String tablePath) { // Create a `Table` object with the given destination table path Table table = Table.forPath(engine, tablePath); // Create a transaction builder to build the transaction TransactionBuilder txnBuilder = table.createTransactionBuilder( engine, "Examples", /* engineInfo */ Operation.CREATE_TABLE); // Set the schema of the new table on the transaction builder txnBuilder = txnBuilder.withSchema(engine, exampleTableSchema); // Build the transaction Transaction txn = txnBuilder.build(engine); // Commit the transaction. // As we are just creating the table and not adding any data, the `dataActions` is empty. TransactionCommitResult commitResult = txn.commit( engine, CloseableIterable.emptyIterable() /* dataActions */); // Check the transaction commit result verifyCommitSuccess(tablePath, commitResult); } public void createPartitionedTable(String tablePath) { // Create a `Table` object with the given destination table path Table table = Table.forPath(engine, tablePath); // Create a transaction builder to build the transaction TransactionBuilder txnBuilder = table.createTransactionBuilder( engine, "Examples", /* engineInfo */ Operation.CREATE_TABLE); txnBuilder = txnBuilder // Set the schema of the new table .withSchema(engine, examplePartitionedTableSchema) // set the partition columns of the new table .withPartitionColumns(engine, examplePartitionColumns); // Build the transaction Transaction txn = txnBuilder.build(engine); // Commit the transaction. // As we are just creating the table and not adding any data, the `dataActions` is empty. TransactionCommitResult commitResult = txn.commit( engine, CloseableIterable.emptyIterable() /* dataActions */); // Check the transaction commit result verifyCommitSuccess(tablePath, commitResult); } } ================================================ FILE: kernel/examples/kernel-examples/src/main/java/io/delta/kernel/examples/CreateTableAndInsertData.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.examples; import java.io.IOException; import java.util.*; import java.util.concurrent.CompletableFuture; import org.apache.commons.cli.Options; import io.delta.kernel.*; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.expressions.Literal; import io.delta.kernel.hook.PostCommitHook; import io.delta.kernel.hook.PostCommitHook.PostCommitHookType; import io.delta.kernel.utils.*; import static io.delta.kernel.examples.utils.Utils.parseArgs; import static io.delta.kernel.internal.util.Utils.toCloseableIterator; /** * Example program that demonstrates how to: * *

    *
  • * create a partiitoned and unpartitioned table and insert data into it * (Basically the CREATE TABLE AS command). *
    • *
  • * Insert into an existing table *
    • *
  • * Idempotent data write to a table. *
    • *
*/ public class CreateTableAndInsertData extends BaseTableWriter { public static void main(String[] args) throws IOException { Options options = new Options() .addOption("l", "location", true, "Locations where the sample tables are created"); new CreateTableAndInsertData().runExamples( parseArgs(options, args).getOptionValue("location")); } public void runExamples(String location) throws IOException { String unpartitionedTblPath = location + "/example"; String partitionTblPath = location + "/example_partitioned"; // CTAS example for unpartitioned tables createTableWithSampleData(unpartitionedTblPath); // CTAS example for partitioned tables createPartitionedTableWithSampleData(partitionTblPath); // Insert into an existing table. insertDataIntoUnpartitionedTable(unpartitionedTblPath); // Example of idempotent inserts idempotentInserts(unpartitionedTblPath); // Example of checkpointg insertWithOptionalCheckpoint(unpartitionedTblPath); } public TransactionCommitResult createTableWithSampleData(String tablePath) throws IOException { // Create a `Table` object with the given destination table path Table table = Table.forPath(engine, tablePath); // Create a transaction builder to build the transaction TransactionBuilder txnBuilder = table.createTransactionBuilder( engine, "Examples", /* engineInfo */ Operation.CREATE_TABLE); // Set the schema of the new table on the transaction builder txnBuilder = txnBuilder.withSchema(engine, exampleTableSchema); // Build the transaction Transaction txn = txnBuilder.build(engine); // Get the transaction state Row txnState = txn.getTransactionState(engine); // Generate the sample data for the table that confirms to the table schema FilteredColumnarBatch batch1 = generateUnpartitionedDataBatch(5 /* offset */); FilteredColumnarBatch batch2 = generateUnpartitionedDataBatch(10 /* offset */); FilteredColumnarBatch batch3 = generateUnpartitionedDataBatch(25 /* offset */); CloseableIterator data = toCloseableIterator(Arrays.asList(batch1, batch2, batch3).iterator()); // First transform the logical data to physical data that needs to be written to the Parquet // files CloseableIterator physicalData = Transaction.transformLogicalData( engine, txnState, data, // partition values - as this table is unpartitioned, it should be empty Collections.emptyMap()); // Get the write context DataWriteContext writeContext = Transaction.getWriteContext( engine, txnState, // partition values - as this table is unpartitioned, it should be empty Collections.emptyMap()); // Now write the physical data to Parquet files CloseableIterator dataFiles = engine.getParquetHandler() .writeParquetFiles( writeContext.getTargetDirectory(), physicalData, writeContext.getStatisticsColumns()); // Now convert the data file status to data actions that needs to be written to the Delta // table log CloseableIterator dataActions = Transaction.generateAppendActions(engine, txnState, dataFiles, writeContext); // Create a iterable out of the data actions. If the contents are too big to fit in memory, // the connector may choose to write the data actions to a temporary file and return an // iterator that reads from the file. CloseableIterable dataActionsIterable = CloseableIterable.inMemoryIterable(dataActions); // Commit the transaction. TransactionCommitResult commitResult = txn.commit(engine, dataActionsIterable); // Check the transaction commit result verifyCommitSuccess(tablePath, commitResult); return commitResult; } public TransactionCommitResult createPartitionedTableWithSampleData(String tablePath) throws IOException { // Create a `Table` object with the given destination table path Table table = Table.forPath(engine, tablePath); // Create a transaction builder to build the transaction TransactionBuilder txnBuilder = table.createTransactionBuilder( engine, "Examples", /* engineInfo */ Operation.CREATE_TABLE); txnBuilder = txnBuilder // Set the schema of the new table .withSchema(engine, examplePartitionedTableSchema) // set the partition columns of the new table .withPartitionColumns(engine, examplePartitionColumns); // Build the transaction Transaction txn = txnBuilder.build(engine); // Get the transaction state Row txnState = txn.getTransactionState(engine); List dataActions = new ArrayList<>(); // Generate the sample data for three partitions. Process each partition separately. // This is just an example. In a real-world scenario, the data may come from different // partitions. Connectors already have the capability to partition by partition values // before writing to the table // In the test data `city` is a partition column for (String city : Arrays.asList("San Francisco", "Campbell", "San Jose")) { FilteredColumnarBatch batch1 = generatedPartitionedDataBatch( 5 /* offset */, city /* partition value */); FilteredColumnarBatch batch2 = generatedPartitionedDataBatch( 5 /* offset */, city /* partition value */); FilteredColumnarBatch batch3 = generatedPartitionedDataBatch( 10 /* offset */, city /* partition value */); CloseableIterator data = toCloseableIterator(Arrays.asList(batch1, batch2, batch3).iterator()); // Create partition value map Map partitionValues = Collections.singletonMap( "city", // partition column name // partition value. Depending upon the parition column type, the // partition value should be created. In this example, the partition // column is of type StringType, so we are creating a string literal. Literal.ofString(city)); // First transform the logical data to physical data that needs to be written // to the Parquet // files CloseableIterator physicalData = Transaction.transformLogicalData(engine, txnState, data, partitionValues); // Get the write context DataWriteContext writeContext = Transaction.getWriteContext(engine, txnState, partitionValues); // Now write the physical data to Parquet files CloseableIterator dataFiles = engine.getParquetHandler() .writeParquetFiles( writeContext.getTargetDirectory(), physicalData, writeContext.getStatisticsColumns()); // Now convert the data file status to data actions that needs to be written to the Delta // table log CloseableIterator partitionDataActions = Transaction.generateAppendActions( engine, txnState, dataFiles, writeContext); // Now add all the partition data actions to the main data actions list. In a // distributed query engine, the partition data is written to files at tasks on executor // nodes. The data actions are collected at the driver node and then written to the // Delta table log using the `Transaction.commit` while (partitionDataActions.hasNext()) { dataActions.add(partitionDataActions.next()); } } // Create a iterable out of the data actions. If the contents are too big to fit in memory, // the connector may choose to write the data actions to a temporary file and return an // iterator that reads from the file. CloseableIterable dataActionsIterable = CloseableIterable.inMemoryIterable( toCloseableIterator(dataActions.iterator())); // Commit the transaction. TransactionCommitResult commitResult = txn.commit(engine, dataActionsIterable); // Check the transaction commit result verifyCommitSuccess(tablePath, commitResult); return commitResult; } public TransactionCommitResult insertDataIntoUnpartitionedTable(String tablePath) throws IOException { // Create a `Table` object with the given destination table path Table table = Table.forPath(engine, tablePath); // Create a transaction builder to build the transaction TransactionBuilder txnBuilder = table.createTransactionBuilder( engine, "Examples", /* engineInfo */ Operation.WRITE); // Build the transaction - no need to provide the schema as the table already exists. Transaction txn = txnBuilder.build(engine); // Get the transaction state Row txnState = txn.getTransactionState(engine); // Generate the sample data for the table that confirms to the table schema FilteredColumnarBatch batch1 = generateUnpartitionedDataBatch(5 /* offset */); FilteredColumnarBatch batch2 = generateUnpartitionedDataBatch(10 /* offset */); FilteredColumnarBatch batch3 = generateUnpartitionedDataBatch(25 /* offset */); CloseableIterator data = toCloseableIterator(Arrays.asList(batch1, batch2, batch3).iterator()); // First transform the logical data to physical data that needs to be written to the Parquet // files CloseableIterator physicalData = Transaction.transformLogicalData( engine, txnState, data, // partition values - as this table is unpartitioned, it should be empty Collections.emptyMap()); // Get the write context DataWriteContext writeContext = Transaction.getWriteContext( engine, txnState, // partition values - as this table is unpartitioned, it should be empty Collections.emptyMap()); // Now write the physical data to Parquet files CloseableIterator dataFiles = engine.getParquetHandler() .writeParquetFiles( writeContext.getTargetDirectory(), physicalData, writeContext.getStatisticsColumns()); // Now convert the data file status to data actions that needs to be written to the Delta // table log CloseableIterator dataActions = Transaction.generateAppendActions(engine, txnState, dataFiles, writeContext); // Create a iterable out of the data actions. If the contents are too big to fit in memory, // the connector may choose to write the data actions to a temporary file and return an // iterator that reads from the file. CloseableIterable dataActionsIterable = CloseableIterable.inMemoryIterable(dataActions); // Commit the transaction. TransactionCommitResult commitResult = txn.commit(engine, dataActionsIterable); // Check the transaction commit result verifyCommitSuccess(tablePath, commitResult); return commitResult; } public TransactionCommitResult idempotentInserts(String tablePath) throws IOException { // Create a `Table` object with the given destination table path Table table = Table.forPath(engine, tablePath); // Create a transaction builder to build the transaction TransactionBuilder txnBuilder = table.createTransactionBuilder( engine, "Examples", /* engineInfo */ Operation.WRITE); // Set the transaction identifiers for idempotent writes // Delta/Kernel makes sure that there exists only one transaction in the Delta log // with the given application id and txn version txnBuilder = txnBuilder.withTransactionId( engine, "my app id", /* application id */ 100 /* txn version */); // Build the transaction - no need to provide the schema as the table already exists. Transaction txn = txnBuilder.build(engine); // Get the transaction state Row txnState = txn.getTransactionState(engine); // Generate the sample data for the table that confirms to the table schema FilteredColumnarBatch batch1 = generateUnpartitionedDataBatch(5 /* offset */); FilteredColumnarBatch batch2 = generateUnpartitionedDataBatch(10 /* offset */); FilteredColumnarBatch batch3 = generateUnpartitionedDataBatch(25 /* offset */); CloseableIterator data = toCloseableIterator(Arrays.asList(batch1, batch2, batch3).iterator()); // First transform the logical data to physical data that needs to be written to the Parquet // files CloseableIterator physicalData = Transaction.transformLogicalData( engine, txnState, data, // partition values - as this table is unpartitioned, it should be empty Collections.emptyMap()); // Get the write context DataWriteContext writeContext = Transaction.getWriteContext( engine, txnState, // partition values - as this table is unpartitioned, it should be empty Collections.emptyMap()); // Now write the physical data to Parquet files CloseableIterator dataFiles = engine.getParquetHandler() .writeParquetFiles( writeContext.getTargetDirectory(), physicalData, writeContext.getStatisticsColumns()); // Now convert the data file status to data actions that needs to be written to the Delta // table log CloseableIterator dataActions = Transaction.generateAppendActions(engine, txnState, dataFiles, writeContext); // Create a iterable out of the data actions. If the contents are too big to fit in memory, // the connector may choose to write the data actions to a temporary file and return an // iterator that reads from the file. CloseableIterable dataActionsIterable = CloseableIterable.inMemoryIterable(dataActions); // Commit the transaction. TransactionCommitResult commitResult = txn.commit(engine, dataActionsIterable); // Check the transaction commit result verifyCommitSuccess(tablePath, commitResult); return commitResult; } public void insertWithOptionalCheckpoint(String tablePath) throws IOException { boolean didCheckpoint = false; // insert data multiple times to trigger a checkpoint. By default checkpoint is needed // for every 10 versions. for (int i = 0; i < 12; i++) { TransactionCommitResult commitResult = insertDataIntoUnpartitionedTable(tablePath); for(PostCommitHook hook: commitResult.getPostCommitHooks()) // Checkpoint the table didCheckpoint = didCheckpoint || CompletableFuture.supplyAsync(() -> { // run the code async try{ hook.threadSafeInvoke(engine); } catch (IOException e) { return false; } return hook.getType().equals(PostCommitHookType.CHECKPOINT); }).join(); // wait async finish. } if (!didCheckpoint) { throw new RuntimeException("Table should have checkpointed by now"); } } } ================================================ FILE: kernel/examples/kernel-examples/src/main/java/io/delta/kernel/examples/MultiThreadedTableReader.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.examples; import java.util.List; import java.util.Optional; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import io.delta.kernel.*; import io.delta.kernel.engine.Engine; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.examples.utils.RowSerDe; import io.delta.kernel.exceptions.TableNotFoundException; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import static io.delta.kernel.examples.utils.Utils.parseArgs; import io.delta.kernel.internal.InternalScanFileUtils; import io.delta.kernel.internal.data.ScanStateRow; import io.delta.kernel.internal.util.Utils; import static io.delta.kernel.internal.util.Utils.singletonCloseableIterator; /** * Multi-threaded Delta Lake table reader using the Delta Kernel APIs. It illustrates * how to use the scan files rows received from the Delta Kernel in distributed engine. *

* For this example serialization and deserialization is not needed as the work generator and * work executors share the same memory, but it illustrates an example of how Delta Kernel can * work in a distributed query engine. High level steps are: * - The query engine asks the Delta Kernel APIs for scan file and scan state rows at the driver * (or equivalent) node * - The query engine serializes the scan file and scan state at the driver node * - The driver sends the serialized bytes to remote worker node(s) * - Worker nodes deserialize the scan file and scan state rows from the serialized bytes * - Worker nodes read the data from given scan file(s) and scan state using the Delta Kernel APIs. * *

* Usage: * java io.delta.kernel.examples.SingleThreadedTableReader [-c ][-l ] [-p ] -t * -c,--columns Comma separated list of columns to read from the * table. Ex. --columns=id,name,address * -l,--limit Maximum number of rows to read from the table (default 20). * -p,--parallelism Number of parallel readers to use (default 3). * -t,--table Fully qualified table path *

*/ public class MultiThreadedTableReader extends BaseTableReader { private static final int DEFAULT_NUM_THREADS = 3; private final int numThreads; public MultiThreadedTableReader(int numThreads, String tablePath) { super(tablePath); this.numThreads = numThreads; } public int show(int limit, Optional> columnsOpt, Optional predicate) throws TableNotFoundException { Table table = Table.forPath(engine, tablePath); Snapshot snapshot = table.getLatestSnapshot(engine); StructType readSchema = pruneSchema(snapshot.getSchema(), columnsOpt); ScanBuilder scanBuilder = snapshot.getScanBuilder().withReadSchema(readSchema); if (predicate.isPresent()) { scanBuilder = scanBuilder.withFilter(predicate.get()); } return new Reader(limit) .readData(readSchema, scanBuilder.build()); } public static void main(String[] args) throws Exception { Options cliOptions = baseOptions().addOption( Option.builder() .option("p") .longOpt("parallelism") .hasArg() .desc("Number of parallel readers to use (default 3).") .type(Number.class) .build()); CommandLine commandLine = parseArgs(cliOptions, args); String tablePath = commandLine.getOptionValue("table"); int limit = parseInt(commandLine, "limit", DEFAULT_LIMIT); int numThreads = parseInt(commandLine, "parallelism", DEFAULT_NUM_THREADS); Optional> columns = parseColumnList(commandLine, "columns"); new MultiThreadedTableReader(numThreads, tablePath) .show(limit, columns, Optional.empty()); } /** * Work unit representing the scan state and scan file in serialized format. */ private static class ScanFile { /** * Special instance of the {@link ScanFile} to indicate to the worker that there are no * more scan files to scan and stop the worker thread. */ private static final ScanFile POISON_PILL = new ScanFile("", ""); final String stateJson; final String fileJson; ScanFile(Row scanStateRow, Row scanFileRow) { this.stateJson = RowSerDe.serializeRowToJson(scanStateRow); this.fileJson = RowSerDe.serializeRowToJson(scanFileRow); } ScanFile(String stateJson, String fileJson) { this.stateJson = stateJson; this.fileJson = fileJson; } /** * Get the deserialized scan state as {@link Row} object */ Row getScanRow(Engine engine) { return RowSerDe.deserializeRowFromJson(stateJson); } /** * Get the deserialized scan file as {@link Row} object */ Row getScanFileRow(Engine engine) { return RowSerDe.deserializeRowFromJson(fileJson); } } private class Reader { private final int limit; private final AtomicBoolean stopSignal = new AtomicBoolean(false); private final CountDownLatch countDownLatch = new CountDownLatch(numThreads); private final ExecutorService executorService = Executors.newFixedThreadPool(numThreads + 1); private final BlockingQueue workQueue = new ArrayBlockingQueue<>(20); private int readRecordCount; // Number of rows read so far, synchronized with `this` object private AtomicReference error = new AtomicReference<>(); Reader(int limit) { this.limit = limit; } /** * Read the data from the given {@code snapshot}. * * @param readSchema Subset of columns to read from the snapshot. * @param scan Scan object to read data from. * @return Number of rows read */ int readData(StructType readSchema, Scan scan) { printSchema(readSchema); try { executorService.submit(workGenerator(scan)); for (int i = 0; i < numThreads; i++) { executorService.submit(workConsumer(i)); } countDownLatch.await(); } catch (InterruptedException ie) { System.out.println("Interrupted exiting now.."); throw new RuntimeException(ie); } finally { stopSignal.set(true); executorService.shutdownNow(); if (error.get() != null) { throw new RuntimeException(error.get()); } } return readRecordCount; } private Runnable workGenerator(Scan scan) { return (() -> { Row scanStateRow = scan.getScanState(engine); try(CloseableIterator scanFileIter = scan.getScanFiles(engine)) { while (scanFileIter.hasNext() && !stopSignal.get()) { try (CloseableIterator scanFileRows = scanFileIter.next().getRows()) { while (scanFileRows.hasNext() && !stopSignal.get()) { workQueue.put(new ScanFile(scanStateRow, scanFileRows.next())); } } } for (int i = 0; i < numThreads; i++) { // poison pill for each worker threads to stop the work. workQueue.put(ScanFile.POISON_PILL); } } catch (InterruptedException ie) { System.out.print("Work generator is interrupted"); } catch (Exception e) { error.compareAndSet(null /* expected */, e); throw new RuntimeException(e); } }); } private Runnable workConsumer(int workerId) { return (() -> { try { ScanFile work = workQueue.take(); if (work == ScanFile.POISON_PILL) { return; // exit as there are no more work units } Row scanState = work.getScanRow(engine); Row scanFile = work.getScanFileRow(engine); FileStatus fileStatus = InternalScanFileUtils.getAddFileStatus(scanFile); StructType physicalReadSchema = ScanStateRow.getPhysicalDataReadSchema(scanState); CloseableIterator physicalDataIter = engine.getParquetHandler().readParquetFiles( singletonCloseableIterator(fileStatus), physicalReadSchema, Optional.empty()).map(res -> res.getData()); try ( CloseableIterator dataIter = Scan.transformPhysicalData( engine, scanState, scanFile, physicalDataIter)) { while (dataIter.hasNext()) { if (printDataBatch(dataIter.next())) { // Have enough records, exit now. break; } } } } catch (InterruptedException ie) { System.out.printf("Worker %d is interrupted." + workerId); } catch (Exception e) { error.compareAndSet(null /* expected */, e); throw new RuntimeException(e); } finally { countDownLatch.countDown(); } }); } /** * Returns true when sufficient amount of rows are received */ private boolean printDataBatch(FilteredColumnarBatch data) { synchronized (this) { if (readRecordCount >= limit) { return true; } readRecordCount += printData(data, limit - readRecordCount); return readRecordCount >= limit; } } } } ================================================ FILE: kernel/examples/kernel-examples/src/main/java/io/delta/kernel/examples/SingleThreadedTableReader.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.examples; import java.io.IOException; import java.util.List; import java.util.Optional; import org.apache.commons.cli.CommandLine; import io.delta.kernel.*; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.exceptions.TableNotFoundException; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import static io.delta.kernel.examples.utils.Utils.parseArgs; import io.delta.kernel.internal.InternalScanFileUtils; import io.delta.kernel.internal.data.ScanStateRow; import static io.delta.kernel.internal.util.Utils.singletonCloseableIterator; /** * Single threaded Delta Lake table reader using the Delta Kernel APIs. * *

* Usage: java io.delta.kernel.examples.SingleThreadedTableReader [-c ] [-l ] -t *

* -c,--columns Comma separated list of columns to read from the * table. Ex. --columns=id,name,address * -l,--limit Maximum number of rows to read from the table * (default 20). * -t,--table Fully qualified table path *

*/ public class SingleThreadedTableReader extends BaseTableReader { public SingleThreadedTableReader(String tablePath) { super(tablePath); } @Override public int show(int limit, Optional> columnsOpt, Optional predicate) throws TableNotFoundException, IOException { Table table = Table.forPath(engine, tablePath); Snapshot snapshot = table.getLatestSnapshot(engine); StructType readSchema = pruneSchema(snapshot.getSchema(), columnsOpt); ScanBuilder scanBuilder = snapshot.getScanBuilder().withReadSchema(readSchema); if (predicate.isPresent()) { scanBuilder = scanBuilder.withFilter(predicate.get()); } return readData(readSchema, scanBuilder.build(), limit); } public static void main(String[] args) throws Exception { CommandLine commandLine = parseArgs(baseOptions(), args); String tablePath = commandLine.getOptionValue("table"); int limit = parseInt(commandLine, "limit", DEFAULT_LIMIT); Optional> columns = parseColumnList(commandLine, "columns"); new SingleThreadedTableReader(tablePath) .show(limit, columns, Optional.empty()); } /** * Utility method to read and print the data from the given {@code snapshot}. * * @param readSchema Subset of columns to read from the snapshot. * @param scan Table scan object * @param maxRowCount Not a hard limit but use this limit to stop reading more columnar batches * once the already read columnar batches have at least these many rows. * @return Number of rows read. * @throws Exception */ private int readData(StructType readSchema, Scan scan, int maxRowCount) throws IOException { printSchema(readSchema); Row scanState = scan.getScanState(engine); CloseableIterator scanFileIter = scan.getScanFiles(engine); int readRecordCount = 0; try { StructType physicalReadSchema = ScanStateRow.getPhysicalDataReadSchema(scanState); while (scanFileIter.hasNext()) { FilteredColumnarBatch scanFilesBatch = scanFileIter.next(); try (CloseableIterator scanFileRows = scanFilesBatch.getRows()) { while (scanFileRows.hasNext()) { Row scanFileRow = scanFileRows.next(); FileStatus fileStatus = InternalScanFileUtils.getAddFileStatus(scanFileRow); CloseableIterator physicalDataIter = engine.getParquetHandler().readParquetFiles( singletonCloseableIterator(fileStatus), physicalReadSchema, Optional.empty()).map(res -> res.getData()); try ( CloseableIterator transformedData = Scan.transformPhysicalData( engine, scanState, scanFileRow, physicalDataIter)) { while (transformedData.hasNext()) { FilteredColumnarBatch filteredData = transformedData.next(); readRecordCount += printData(filteredData, maxRowCount - readRecordCount); if (readRecordCount >= maxRowCount) { return readRecordCount; } } } } } } } finally { scanFileIter.close(); } return readRecordCount; } } ================================================ FILE: kernel/examples/kernel-examples/src/main/java/io/delta/kernel/examples/utils/RowSerDe.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.examples.utils; import java.io.UncheckedIOException; import java.util.HashMap; import java.util.Map; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import io.delta.kernel.engine.Engine; import io.delta.kernel.data.Row; import io.delta.kernel.types.*; import io.delta.kernel.internal.types.DataTypeJsonSerDe; import io.delta.kernel.internal.util.VectorUtils; import io.delta.kernel.defaults.internal.data.DefaultJsonRow; /** * Utility class to serialize and deserialize {@link Row} object. */ public class RowSerDe { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private RowSerDe() { } /** * Utility method to serialize a {@link Row} as a JSON string */ public static String serializeRowToJson(Row row) { Map rowObject = convertRowToJsonObject(row); try { Map rowWithSchema = new HashMap<>(); rowWithSchema.put("schema", row.getSchema().toJson()); rowWithSchema.put("row", rowObject); return OBJECT_MAPPER.writeValueAsString(rowWithSchema); } catch (JsonProcessingException e) { throw new UncheckedIOException(e); } } /** * Utility method to deserialize a {@link Row} object from the JSON form. */ public static Row deserializeRowFromJson(String jsonRowWithSchema) { try { JsonNode jsonNode = OBJECT_MAPPER.readTree(jsonRowWithSchema); JsonNode schemaNode = jsonNode.get("schema"); StructType schema = DataTypeJsonSerDe.deserializeStructType(schemaNode.asText()); return parseRowFromJsonWithSchema((ObjectNode) jsonNode.get("row"), schema); } catch (JsonProcessingException ex) { throw new UncheckedIOException(ex); } } private static Map convertRowToJsonObject(Row row) { StructType rowType = row.getSchema(); Map rowObject = new HashMap<>(); for (int fieldId = 0; fieldId < rowType.length(); fieldId++) { StructField field = rowType.at(fieldId); DataType fieldType = field.getDataType(); String name = field.getName(); if (row.isNullAt(fieldId)) { rowObject.put(name, null); continue; } Object value; if (fieldType instanceof BooleanType) { value = row.getBoolean(fieldId); } else if (fieldType instanceof ByteType) { value = row.getByte(fieldId); } else if (fieldType instanceof ShortType) { value = row.getShort(fieldId); } else if (fieldType instanceof IntegerType) { value = row.getInt(fieldId); } else if (fieldType instanceof LongType) { value = row.getLong(fieldId); } else if (fieldType instanceof FloatType) { value = row.getFloat(fieldId); } else if (fieldType instanceof DoubleType) { value = row.getDouble(fieldId); } else if (fieldType instanceof DateType) { value = row.getInt(fieldId); } else if (fieldType instanceof TimestampType) { value = row.getLong(fieldId); } else if (fieldType instanceof StringType) { value = row.getString(fieldId); } else if (fieldType instanceof ArrayType) { value = VectorUtils.toJavaList(row.getArray(fieldId)); } else if (fieldType instanceof MapType) { value = VectorUtils.toJavaMap(row.getMap(fieldId)); } else if (fieldType instanceof StructType) { Row subRow = row.getStruct(fieldId); value = convertRowToJsonObject(subRow); } else { throw new UnsupportedOperationException("NYI"); } rowObject.put(name, value); } return rowObject; } private static Row parseRowFromJsonWithSchema(ObjectNode rowJsonNode, StructType rowType) { return new DefaultJsonRow(rowJsonNode, rowType); } } ================================================ FILE: kernel/examples/kernel-examples/src/main/java/io/delta/kernel/examples/utils/Utils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.examples.utils; import org.apache.commons.cli.*; import io.delta.kernel.examples.SingleThreadedTableReader; public class Utils { /** * Helper method to parse the command line arguments. */ public static CommandLine parseArgs(Options options, String[] args) { CommandLineParser cliParser = new DefaultParser(); try { return cliParser.parse(options, args); } catch (ParseException parseException) { new HelpFormatter().printHelp( "java " + SingleThreadedTableReader.class.getCanonicalName(), options, true ); } System.exit(-1); return null; } } ================================================ FILE: kernel/examples/kernel-examples/src/main/java/io/delta/kernel/integration/ReadIntegrationTestSuite.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.integration; import java.math.BigDecimal; import java.util.List; import java.util.Optional; import static java.util.Arrays.asList; import io.delta.kernel.examples.SingleThreadedTableReader; import io.delta.kernel.expressions.And; import io.delta.kernel.expressions.Column; import io.delta.kernel.expressions.Literal; import io.delta.kernel.expressions.Predicate; /** * Test suite that runs various integration tests for sanity testing the staged/released artifacts. * It only verifies the number of rows in the results and not the specific values of rows. * For full scale results verification we rely on unit tests which are run as part of the CI jobs. */ public class ReadIntegrationTestSuite { private final String goldenTableDir; public static void main(String[] args) throws Exception { new ReadIntegrationTestSuite(args[0]) .runTests(); } public ReadIntegrationTestSuite(String goldenTableDir) { this.goldenTableDir = goldenTableDir; } public void runTests() throws Exception { // Definitions of golden tables is present in // /connectors/golden-tables/src/test/scala/io/delta/golden/GoldenTables.scala // Basic reads: Simple table runAndVerifyRowCount( "basic_read_simple_table", "data-reader-primitives", Optional.empty(), /* read schema - read all columns */ Optional.empty(), /* predicate */ 11 /* expected row count */); // Basic reads: Partitioned table runAndVerifyRowCount( "basic_read_partitioned_table", "data-reader-array-primitives", Optional.empty(), /* read schema - read all columns */ Optional.empty(), /* predicate */ 10 /* expected row count */); // Basic reads: Table with DVs runAndVerifyRowCount( "basic_read_table_with_deletionvectors", "dv-partitioned-with-checkpoint", Optional.empty(), /* read schema - read all columns */ Optional.empty(), /* predicate */ 35 /* expected row count */); // Basic reads: select subset of columns runAndVerifyRowCount( "basic_read_subset_of_columns", "dv-partitioned-with-checkpoint", Optional.of(asList("part", "col2")), /* read schema */ Optional.empty(), /* predicate */ 35 /* expected row count */); // Basic reads: Table with DVs and column mapping name runAndVerifyRowCount( "basic_read_table_with_columnmapping_deletionvectors", "dv-with-columnmapping", Optional.of(asList("col1", "col2")), /* read schema */ Optional.empty(), /* predicate */ 35 /* expected row count */); // Basic read: table with column mapping mode id runAndVerifyRowCount( "basic_read_table_columnmapping_id", "table-with-columnmapping-mode-id", Optional.of( asList("ByteType", "decimal", "nested_struct", "array_of_prims", "map_of_prims")), Optional.empty(), /* predicate */ 6 /* expected row count */); // Basic read: table with JSON V2 checkpoint runAndVerifyRowCount( "basic_read_table_v2_checkpoint_json", "v2-checkpoint-json", Optional.empty(), /* read schema - read all columns */ Optional.empty(), /* predicate */ 10 /* expected row count */); // Basic read: table with Parquet V2 checkpoint runAndVerifyRowCount( "basic_read_table_v2_checkpoint_parquet", "v2-checkpoint-parquet", Optional.empty(), /* read schema - read all columns */ Optional.empty(), /* predicate */ 10 /* expected row count */); // Partition pruning: simple expression runAndVerifyRowCount( "partition_pruning_simple_filter", "basic-decimal-table", Optional.empty(), /* read schema - read all columns */ Optional.of(new Predicate( "=", asList( new Column("part"), Literal.ofDecimal(new BigDecimal("2342222.23454"), 12, 5)))), 1 /* expected row count */); // Partition pruning: simple expression where nothing is pruned runAndVerifyRowCount( "partition_pruning_simple_filter_no_pruning", "basic-decimal-table", Optional.empty(), /* read schema - read all columns */ Optional.of( new Predicate( "NOT", asList( new Predicate( "=", asList(new Column("part"), Literal.ofDecimal(new BigDecimal(0), 12, 5))) ))), 4 /* expected row count */); // Partition pruning + data skipping: filter on data and metadata columns where // data filter doesn't prune anything runAndVerifyRowCount( "partition_pruning_filter_on_data_and_metadata_columns_1", "dv-partitioned-with-checkpoint", Optional.of(asList("part", "col2")), /* read schema */ Optional.of( new And( new Predicate(">=", asList(new Column("part"), Literal.ofInt(7))), new Predicate(">=", asList(new Column("col1"), Literal.ofInt(0))))), 12 /* expected row count */); // Partition pruning + data skipping: filter on data and metadata columns where // data filter also prunes few files based on the stats based skipping runAndVerifyRowCount( "partition_pruning_filter_on_data_and_metadata_columns_2", "dv-partitioned-with-checkpoint", Optional.of(asList("part", "col2")), /* read schema */ Optional.of( new And( new Predicate(">=", asList(new Column("part"), Literal.ofInt(7))), new Predicate("=", asList(new Column("col1"), Literal.ofInt(28))))), 5 /* expected row count */); // Data skipping: filter on a table with checkpoint runAndVerifyRowCount( "data_skipping_table_with_checkpoint", "data-skipping-basic-stats-all-types-checkpoint", Optional.empty(), /* read schema - read all columns */ Optional.of( new Predicate( ">", asList(new Column("as_int"), Literal.ofInt(0)) )), 0 /* expected row count */); // Partition pruning: table with column mapping mode name runAndVerifyRowCount( "partition_pruning_columnmapping_name", "dv-with-columnmapping", Optional.empty(), Optional.of( new Predicate( "=", asList(new Column("part"), Literal.ofInt(0)) )), 2 /* expected row count */); // Data skipping: table with column mapping mode id runAndVerifyRowCount( "data_skipping_columnmapping_id", "data-skipping-basic-stats-all-types-columnmapping-id", Optional.empty(), Optional.of( new Predicate( "=", asList(new Column("as_int"), Literal.ofInt(1)) )), 0 /* expected row count */); // Type widening: table with various type changes. runAndVerifyRowCount( "type_widening", "type-widening", Optional.empty(), /* read schema - read all columns */ Optional.empty(), /* predicate */ 2 /* expected row count */); // Type widening: table with type changes inside nested struct/array/map. runAndVerifyRowCount( "type_widening_nested", "type-widening-nested", Optional.empty(), /* read schema - read all columns */ Optional.empty(), /* predicate */ 2 /* expected row count */); } private void runAndVerifyRowCount( String testName, String goldenTable, Optional> readColumns, Optional predicate, int expectedRowCount) throws Exception { System.out.println("\n========== TEST START: " + testName + " =============="); try { String path = goldenTableDir + "/" + goldenTable; SingleThreadedTableReader reader = new SingleThreadedTableReader(path); // Select a large number of rows (1M), so that everything in the table is read. int actRowCount = reader.show(1_000_000, readColumns, predicate); if (actRowCount != expectedRowCount) { throw new RuntimeException(String.format( "Test (%s) failed: expected row count = %s, actual row count = %s", testName, expectedRowCount, actRowCount)); } } finally { System.out.println("========== TEST END: " + testName + " ==============\n"); } } } ================================================ FILE: kernel/examples/kernel-examples/src/main/java/io/delta/kernel/integration/WriteIntegrationTestSuite.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.integration; import java.io.IOException; import java.nio.file.Files; import java.util.Optional; import java.util.UUID; import io.delta.kernel.examples.*; /** * Test suite that runs various integration tests for sanity testing the staged/released artifacts. * It only verifies the number of rows in the results and not the specific values of rows. For full * scale results verification we rely on unit tests which are run as part of the CI jobs. */ public class WriteIntegrationTestSuite { public static void main(String[] args) throws Exception { new WriteIntegrationTestSuite().runTests(); } public void runTests() throws Exception { verifyRowCount( "Create un-partitioned table", 0 /* expected row count */, tblLocation -> new CreateTable().createUnpartitionedTable(tblLocation) ); verifyRowCount( "Create partitioned table", 0 /* expected row count */, tblLocation -> new CreateTable().createPartitionedTable(tblLocation) ); verifyRowCount( "Create un-partitioned table and insert data", 15 /* expected row count */, tblLocation -> new CreateTableAndInsertData().createTableWithSampleData(tblLocation) ); verifyRowCount( "Create partitioned table and insert data", 45 /* expected row count */, tblLocation -> new CreateTableAndInsertData() .createPartitionedTableWithSampleData(tblLocation) ); verifyRowCount( "insert data into an existing table", 30 /* expected row count */, tblLocation -> { CreateTableAndInsertData createTableAndInsertData = new CreateTableAndInsertData(); createTableAndInsertData.createTableWithSampleData(tblLocation); createTableAndInsertData.insertDataIntoUnpartitionedTable(tblLocation); }); verifyRowCount( "idempotent inserts into a table", 30 /* expected row count */, tblLocation -> { CreateTableAndInsertData createTableAndInsertData = new CreateTableAndInsertData(); createTableAndInsertData.createTableWithSampleData(tblLocation); createTableAndInsertData.idempotentInserts(tblLocation); }); verifyRowCount( "inserts with an optional checkpoint", 195 /* expected row count */, tblLocation -> { CreateTableAndInsertData createTableAndInsertData = new CreateTableAndInsertData(); createTableAndInsertData.createTableWithSampleData(tblLocation); createTableAndInsertData.insertWithOptionalCheckpoint(tblLocation); }); } private void verifyRowCount(String testName, int expectedRowCount, CheckedFunction test) throws Exception { System.out.println("\n========== TEST START: " + testName + " =============="); try { String tblLocation = tmpLocation(); test.apply(tblLocation); SingleThreadedTableReader reader = new SingleThreadedTableReader(tblLocation); // Select a large number of rows (1M), so that everything in the table is read. int actRowCount = reader.show(1_000_000, Optional.empty(), Optional.empty()); if (actRowCount != expectedRowCount) { throw new RuntimeException(String.format( "Test (%s) failed: expected row count = %s, actual row count = %s", testName, expectedRowCount, actRowCount)); } } finally { System.out.println("========== TEST END: " + testName + " ==============\n"); } } private String tmpLocation() throws IOException { return Files.createTempDirectory("delta" + UUID.randomUUID()).toString(); } interface CheckedFunction { void apply(T tblLocation) throws Exception; } } ================================================ FILE: kernel/examples/run-kernel-examples.py ================================================ #!/usr/bin/env python3 # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ''' To run examples by building the artifacts from code and using them: ``` /kernel/examples/run-kernel-examples.py --use-local ``` To run examples using artifacts from a Maven repository: ``` /kernel/examples/run-kernel-examples.py --version --maven-repo ``` ''' import os import subprocess from os import path import shutil import argparse def run_single_threaded_examples(version, maven_repo, examples_root_dir, golden_tables_dir): main_class = "io.delta.kernel.examples.SingleThreadedTableReader" test_cases = [ f"--table={golden_tables_dir}/data-reader-primitives --columns=as_int,as_long --limit=5", f"--table={golden_tables_dir}/data-reader-primitives --columns=as_int,as_long,as_double,as_string --limit=20", f"--table={golden_tables_dir}/data-reader-partition-values --columns=as_string,as_byte,as_list_of_records,as_nested_struct --limit=20" ] project_dir = path.join(examples_root_dir, "kernel-examples") run_example(version, maven_repo, project_dir, main_class, test_cases) def run_multi_threaded_examples(version, maven_repo, examples_root_dir, golden_tables_dir): main_class = "io.delta.kernel.examples.MultiThreadedTableReader" test_cases = [ f"--table={golden_tables_dir}/data-reader-primitives --columns=as_int,as_long --limit=5 --parallelism=5", f"--table={golden_tables_dir}/data-reader-primitives --columns=as_int,as_long,as_double,as_string --limit=20 --parallelism=20", f"--table={golden_tables_dir}/data-reader-partition-values --columns=as_string,as_byte,as_list_of_records,as_nested_struct --limit=20 --parallelism=2" ] project_dir = path.join(examples_root_dir, "kernel-examples") run_example(version, maven_repo, project_dir, main_class, test_cases) def run_integration_tests(version, maven_repo, examples_root_dir, golden_tables_dir): main_classes = ["io.delta.kernel.integration.ReadIntegrationTestSuite", "io.delta.kernel.integration.WriteIntegrationTestSuite"] for main_class in main_classes: project_dir = path.join(examples_root_dir, "kernel-examples") with WorkingDirectory(project_dir): cmd = ["mvn", "package", "exec:java", f"-Dexec.mainClass={main_class}", f"-Dstaging.repo.url={maven_repo}", f"-Ddelta-kernel.version={version}", f"-Dexec.args={golden_tables_dir}"] run_cmd(cmd, stream_output=True) def run_example(version, maven_repo, project_dir, main_class, test_cases): with WorkingDirectory(project_dir): for test in test_cases: cmd = ["mvn", "package", "exec:java", f"-Dexec.mainClass={main_class}", f"-Dstaging.repo.url={maven_repo}", f"-Ddelta-kernel.version={version}", f"-Dexec.args={test}"] run_cmd(cmd, stream_output=True) def clear_artifact_cache(): print("Clearing Delta Kernel artifacts from ivy2 and mvn cache") ivy_caches_to_clear = [filepath for filepath in os.listdir(os.path.expanduser("~")) if filepath.startswith(".ivy")] print(f"Clearing Ivy caches in: {ivy_caches_to_clear}") for filepath in ivy_caches_to_clear: for subpath in ["io.delta", "io.delta.kernel"]: delete_if_exists(os.path.expanduser(f"~/{filepath}/cache/{subpath}")) delete_if_exists(os.path.expanduser(f"~/{filepath}/local/{subpath}")) delete_if_exists(os.path.expanduser("~/.m2/repository/io/delta/")) def delete_if_exists(path): # if path exists, delete it. if os.path.exists(path): shutil.rmtree(path) print("Deleted %s " % path) # pylint: disable=too-few-public-methods class WorkingDirectory(object): def __init__(self, working_directory): self.working_directory = working_directory self.old_workdir = os.getcwd() def __enter__(self): os.chdir(self.working_directory) def __exit__(self, tpe, value, traceback): os.chdir(self.old_workdir) def run_cmd(cmd, throw_on_error=True, env=None, stream_output=False, **kwargs): cmd_env = os.environ.copy() if env: cmd_env.update(env) if stream_output: child = subprocess.Popen(cmd, env=cmd_env, **kwargs) exit_code = child.wait() if throw_on_error and exit_code != 0: raise Exception("Non-zero exitcode: %s" % (exit_code)) return exit_code else: child = subprocess.Popen( cmd, env=cmd_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) (stdout, stderr) = child.communicate() exit_code = child.wait() if throw_on_error and exit_code != 0: raise Exception( "Non-zero exitcode: %s\n\nSTDOUT:\n%s\n\nSTDERR:%s" % (exit_code, stdout, stderr)) return (exit_code, stdout, stderr) if __name__ == "__main__": """ Script to run Delta Kernel examples which are located in the kernel/examples directory. call this by running `python run-kernel-examples.py` additionally the version can be provided as a command line argument. """ # get the version of the package examples_root_dir = path.abspath(path.dirname(__file__)) project_root_dir = path.join(examples_root_dir, "../../") with open(path.join(project_root_dir, "version.sbt")) as fd: default_version = fd.readline().split('"')[1] parser = argparse.ArgumentParser() parser.add_argument( "--version", required=False, default=default_version, help="Delta Kernel version to use to run the examples") parser.add_argument( "--maven-repo", required=False, default=None, help="Additional Maven repo to resolve staged new release artifacts") parser.add_argument( "--use-local", required=False, default=False, action="store_true", help="Generate JARs from local source code and use to run tests") args = parser.parse_args() if args.use_local and (args.version != default_version): raise Exception("Cannot specify --use-local with a --version different than in version.sbt") clear_artifact_cache() if args.use_local: with WorkingDirectory(project_root_dir): run_cmd(["build/sbt", "kernelGroup/publishM2", "storage/publishM2"], stream_output=True) golden_file_dir = path.join( examples_root_dir, "../../connectors/golden-tables/src/main/resources/golden/") run_single_threaded_examples(args.version, args.maven_repo, examples_root_dir, golden_file_dir) run_multi_threaded_examples(args.version, args.maven_repo, examples_root_dir, golden_file_dir) run_integration_tests(args.version, args.maven_repo, examples_root_dir, golden_file_dir) ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/CommitActions.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.utils.CloseableIterator; /** * Represents all actions from a single commit version in a table. * *

Resource Management: * *

    *
  • Each iterator returned by {@link #getActions()} must be closed after use to release * underlying resources. *
  • The {@code CommitActions} object should be closed correctly (preferably using * try-with-resources) to ensure any resources allocated during construction are properly * released. *
* *

Example usage: * * 

{@code
 * try (CommitActions commitActions = ...) {
 *   long version = commitActions.getVersion();
 *   long timestamp = commitActions.getTimestamp();
 *
 *   try (CloseableIterator actions = commitActions.getActions()) {
 *     while (actions.hasNext()) {
 *       ColumnarBatch batch = actions.next();
 *       // process batch
 *     }
 *   }
 * }
 * }
* * @since 4.1.0 */ @Evolving public interface CommitActions extends AutoCloseable { /** * Returns the commit version number. * * @return the version number of this commit */ long getVersion(); /** * Returns the commit timestamp in milliseconds since Unix epoch. * * @return the timestamp of this commit */ long getTimestamp(); /** * Returns an iterator over the action batches for this commit. * *

Each {@link ColumnarBatch} contains actions from this commit only. * *

Note: All rows within all batches have the same version (returned by {@link #getVersion()}). * *

This method can be called multiple times, and each call returns a new iterator over the same * set of batches. This supports use cases like two-pass processing (e.g., validation pass * followed by processing pass). * *

Callers are responsible for closing each iterator returned by this method. Each * iterator must be closed after use to release underlying resources. * * @return a {@link CloseableIterator} over columnar batches containing this commit's actions */ CloseableIterator getActions(); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/CommitRange.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.internal.DeltaLogActionUtils; import io.delta.kernel.utils.CloseableIterator; import java.util.Optional; import java.util.Set; /** * Represents a range of contiguous commits in a Delta Lake table with a defined start and end * version. Supports operation on the range of commits, such as reading the delta actions committed * in each commit in the version range. * *

Commit ranges are created using a {@link CommitRangeBuilder}, which supports specifying the * start and end boundaries of the range. The boundaries can be defined using either versions or * timestamps. * * @since 3.4.0 */ @Evolving public interface CommitRange { /** * Returns the starting version number (inclusive) of this commit range. * * @return the starting version number of the commit range */ long getStartVersion(); /** * Returns the ending version number (inclusive) of this commit range. * * @return the ending version number of the commit range */ long getEndVersion(); /** * Returns the original query boundary used to define the start boundary of this commit range. * *

The boundary indicates whether the range was defined using a specific version number or a * timestamp. * * @return the start boundary for this commit range */ CommitRangeBuilder.CommitBoundary getQueryStartBoundary(); /** * Returns the original query boundary used to define the end boundary of this commit range, if * available. * *

The boundary indicates whether the range was defined using a specific version number or a * timestamp. * * @return an {@link Optional} containing the end boundary, or empty if the range was created with * default end parameters (latest version) */ Optional getQueryEndBoundary(); /** * Returns an iterator of the requested actions for the commits in this commit range. * *

For the returned columnar batches: * *

    *
  • Each row within the same batch is guaranteed to have the same commit version *
  • The batch commit versions are monotonically increasing *
  • The top-level columns include "version", "timestamp", and the actions requested in * actionSet. "version" and "timestamp" are the first and second columns in the schema, * respectively. The remaining columns are based on the actions requested and each have the * schema found in {@code DeltaAction.schema}. *
* *

The iterator must be closed after use to release any underlying resources. * * @param engine the {@link Engine} to use for reading the Delta log files * @param startSnapshot the snapshot for startVersion, required to ensure the table is readable by * Kernel at startVersion * @param actionSet the set of action types to include in the results. Only actions of these types * will be returned in the iterator * @return a {@link CloseableIterator} over columnar batches containing the requested actions * within this commit range * @throws IllegalArgumentException if startSnapshot.getVersion() != startVersion * @throws KernelException if the version range contains a version with reader protocol that is * unsupported by Kernel */ CloseableIterator getActions( Engine engine, Snapshot startSnapshot, Set actionSet); /** * Returns an iterator of commits in this commit range, where each commit is represented as a * {@link CommitActions} object. * * @param engine the {@link Engine} to use for reading the Delta log files * @param startSnapshot the snapshot for startVersion, required to ensure the table is readable by * Kernel at startVersion * @param actionSet the set of action types to include in the results. Only actions of these types * will be returned in each commit's actions iterator * @return a {@link CloseableIterator} over {@link CommitActions}, one per commit version in this * range * @throws IllegalArgumentException if startSnapshot.getVersion() != startVersion * @throws KernelException if the version range contains a version with reader protocol that is * unsupported by Kernel */ CloseableIterator getCommitActions( Engine engine, Snapshot startSnapshot, Set actionSet); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/CommitRangeBuilder.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.annotation.Experimental; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.files.ParsedLogData; import java.util.List; import java.util.Optional; /** * A builder for creating {@link CommitRange} instances that define a contiguous range of commits in * a Delta Lake table. * *

The start boundary is required and provided via {@link TableManager#loadCommitRange(String, * CommitBoundary)}. If no end specification is provided, the range defaults to the latest available * version. * * @since 3.4.0 */ @Experimental public interface CommitRangeBuilder { /** * Configures the builder to end the commit range at a specific version or timestamp. * *

If not specified, the commit range will default to ending at the latest available version. * * @param endBoundary the boundary specification for the end of the commit range, must not be null * @return this builder instance configured with the specified end boundary */ CommitRangeBuilder withEndBoundary(CommitBoundary endBoundary); /** * Provides parsed log data to optimize the commit range construction. * *

Note: If no end boundary is provided via {@link * #withEndBoundary(CommitBoundary)}, or a timestamp-based end boundary is provided, the provided * log data must include all available ratified commits. If a version-based end boundary is * provided, the log data must include commits up to at least the end version (i.e., the tail of * the log data must have a version greater than or equal to the end version). * * @param logData the list of pre-parsed log data, must not be null * @return this builder instance configured with the specified log data */ // TODO: should we change this to take in a ParsedDeltaData instead? CommitRangeBuilder withLogData(List logData); /** * Specifies the maximum table version known by the catalog. * *

This method is used by catalog implementations for catalog-managed Delta tables to indicate * the latest ratified version of the table. This ensures that any commit range operations respect * the catalog's view of the table state. * *

Important: This method is required for catalog-managed tables and must not be used for * file-system managed tables. * *

When specified, the following additional constraints are enforced: * *

    *
  • When the provided startBoundary is version-based, the start version must be less than or * equal to the max catalog version. *
  • If {@link #withEndBoundary(CommitBoundary)} is used with a version, the requested version * must be less than or equal to the max catalog version. *
  • If the provided startBoundary is timestamp-based, or {@link * #withEndBoundary(CommitBoundary)} is used with a timestamp, the provided latest snapshot * must have a version equal to the max catalog version. *
  • If {@link #withLogData(List)} is provided and no end boundary is specified (resolving to * latest), the log data must end with the max catalog version. *
* * @param version the maximum table version known by the catalog (must be {@code >= 0}) * @return a new builder instance with the specified max catalog version * @throws IllegalArgumentException if version is negative */ CommitRangeBuilder withMaxCatalogVersion(long version); /** * Builds and returns a {@link CommitRange} instance with the configured specifications. * *

This method validates the builder configuration and constructs the commit range by resolving * version numbers from timestamps if necessary and determining the actual commit files that fall * within the specified range. * * @param engine the {@link Engine} to use for file system operations and log parsing * @return a new {@link CommitRange} instance configured according to this builder's * specifications * @throws IllegalArgumentException if the builder configuration is invalid (e.g., start version * {@code >} end version) */ CommitRange build(Engine engine); /** * Defines a boundary (start or end) of a commit range in a Delta Lake table. * *

A {@code CommitBoundary} can be based on either a specific version number or a timestamp. * When using timestamps, the boundary requires the latest snapshot to help resolve the timestamp * to the appropriate version. * *

Use the static factory methods {@link #atVersion(long)} and {@link #atTimestamp(long, * Snapshot)} to create instances. */ final class CommitBoundary { /** * Creates a commit boundary based on a specific version number. * * @param version the commit version number, must be non-negative * @return a new {@code CommitBoundary} representing the specified version * @throws IllegalArgumentException if {@code version} is negative */ public static CommitBoundary atVersion(long version) { checkArgument(version >= 0, "Version must be >= 0, but got: %d", version); return new CommitBoundary(true, version, Optional.empty()); } /** * Creates a commit boundary based on a timestamp. * *

The timestamp represents a point in time, and the boundary will resolve to the appropriate * commit version. * * @param timestamp the timestamp in milliseconds since epoch * @param latestSnapshot the latest snapshot of the table, used for timestamp resolution * @return a new {@code CommitBoundary} representing the specified timestamp */ public static CommitBoundary atTimestamp(long timestamp, Snapshot latestSnapshot) { checkArgument( latestSnapshot instanceof SnapshotImpl, "latestSnapshot must be instance of SnapshotImpl"); return new CommitBoundary(false, timestamp, Optional.of(latestSnapshot)); } private final boolean isVersion; private final long value; private final Optional latestSnapshot; private CommitBoundary(boolean isVersion, long value, Optional latestSnapshot) { checkArgument(isVersion || latestSnapshot.isPresent()); this.isVersion = isVersion; this.value = value; this.latestSnapshot = latestSnapshot; } /** @return {@code true} if this is a version-based boundary, {@code false} otherwise */ public boolean isVersion() { return isVersion; } /** @return {@code true} if this is a timestamp-based boundary, {@code false} otherwise */ public boolean isTimestamp() { return !isVersion; } /** * Returns the version number for version-based boundaries. Callers should check {@link * CommitBoundary#isVersion()} before access. * * @return the version number * @throws IllegalStateException if this boundary is timestamp-based */ public long getVersion() { if (!isVersion) { throw new IllegalStateException("This boundary is not version-based"); } return value; } /** * Returns the timestamp for timestamp-based boundaries. Callers should check {@link * CommitBoundary#isTimestamp()} before access. * * @return the timestamp in milliseconds since epoch * @throws IllegalStateException if this boundary is version-based */ public long getTimestamp() { if (isVersion) { throw new IllegalStateException("This boundary is not timestamp-based"); } return value; } /** * Returns the latest snapshot used for timestamp resolution in timestamp-based boundaries. * Callers should check {@link CommitBoundary#isTimestamp()} before access. * * @return the latest snapshot * @throws IllegalStateException if this boundary is version-based */ public Snapshot getLatestSnapshot() { if (isVersion) { throw new IllegalStateException("This boundary is not timestamp-based"); } return latestSnapshot.get(); } @Override public String toString() { if (isVersion) { return String.format("CommitBoundary{version=%d}", value); } else { return String.format( "CommitBoundary{timestamp=%d, latestSnapshot=%s}", value, latestSnapshot.get()); } } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/DataWriteContext.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; import io.delta.kernel.expressions.Column; import java.util.List; import java.util.Map; /** * Contains the context for writing data to Delta table. The context is created for each partition * for partitioned table or once per table for un-partitioned table. It is created using {@link * Transaction#getWriteContext(Engine, Row, Map)} (String, Map, List)}. * * @since 3.2.0 */ @Evolving public interface DataWriteContext { /** * Returns the target directory where the data should be written. * * @return fully qualified path of the target directory */ String getTargetDirectory(); /** * Returns the list of {@link Column} that the connector can optionally collect statistics. Each * {@link Column} is a reference to a top-level or nested column in the table. * *

Statistics collections can be skipped or collected for a partial list of the returned {@link * Column}s. When stats are present in the written Delta log, they can be used to optimize query * performance. * * @return schema of the statistics */ List getStatisticsColumns(); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/Operation.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel; /** * An operation that can be performed on a Delta table. * *

An operation is tracked as the first line in commit info action inside the Delta Log It also * shows up when {@code DESCRIBE HISTORY} on the table is executed. */ public enum Operation { /** Recorded when the table is created. */ CREATE_TABLE("CREATE TABLE"), /** Recorded during batch inserts. */ WRITE("WRITE"), /** Recorded during streaming inserts. */ STREAMING_UPDATE("STREAMING UPDATE"), /** Recorded during REPLACE operation (may also be considered an overwrite) */ REPLACE_TABLE("REPLACE TABLE"), /** For any operation that doesn't fit the above categories. */ MANUAL_UPDATE("Manual Update"); /** Actual value that will be recorded in the transaction log */ private final String description; Operation(String description) { this.description = description; } /** Returns the string that will be recorded in the transaction log. */ public String getDescription() { return description; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/PaginatedScan.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.engine.Engine; /** * Extension of {@link Scan} that supports pagination. * *

This interface allows consumers to retrieve scan results in discrete, ordered pages rather * than all at once. This is particularly useful for large datasets where materializing the full * result set would be expensive in terms of memory or compute resources. * *

Pagination is achieved via a combination of {@code pageSize} and {@code pageToken}. The {@code * pageSize} controls how many Scan files are returned in each page, while the {@code pageToken} * encodes the location of next batch to read and is used to resume the scan from exactly where the * last page ended. For the first page, the {@code pageToken} should be {@code Optional.empty()}. * *

Consumers typically use {@link PaginatedScan} in a loop: they call {@code getScanFiles()} to * retrieve an iterator over the current page's scan files. After consuming the iterator, users * should call {@link PaginatedScanFilesIterator#getCurrentPageToken} to retrieve a token to pass * into the next page request. This allows users to scan the dataset incrementally, resuming from * where they left off. */ public interface PaginatedScan extends Scan { /** * Get a paginated iterator of Scan files for the current page. * * @param engine {@link Engine} instance to use in Delta Kernel. * @return iterator of {@link FilteredColumnarBatch}s for the current page. */ @Override PaginatedScanFilesIterator getScanFiles(Engine engine); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/PaginatedScanFilesIterator.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.utils.CloseableIterator; import java.util.Optional; /** * An iterator over {@link FilteredColumnarBatch}, each representing a batch of Scan Files in a * paginated scan. This iterator also exposes the page token that can be used to resume the scan * from the exact position current page ends in a subsequent request. * *

This interface extends {@link CloseableIterator} and should be closed when the iteration is * complete. */ public interface PaginatedScanFilesIterator extends CloseableIterator { /** * Returns an optional page token representing the starting position of next page. This token is * used to resume the scan from the exact position current page ends in a subsequent request. Page * token also contains metadata for validation purpose, such as detecting changes in query * parameters or the underlying log files. * *

The page token represents the position of current iterator at the time it's called. If the * iterator is only partially consumed, the returned token will always point to the beginning of * the next unconsumed {@link FilteredColumnarBatch}. This method will return Option.empty() if * all data in the Scan is consumed (no more non-empty pages remain). */ Optional getCurrentPageToken(); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/Scan.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.data.*; import io.delta.kernel.engine.Engine; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.internal.InternalScanFileUtils; import io.delta.kernel.internal.ScanImpl; import io.delta.kernel.internal.TableConfig; import io.delta.kernel.internal.actions.DeletionVectorDescriptor; import io.delta.kernel.internal.data.ScanStateRow; import io.delta.kernel.internal.data.SelectionColumnVector; import io.delta.kernel.internal.deletionvectors.DeletionVectorUtils; import io.delta.kernel.internal.deletionvectors.RoaringBitmapArray; import io.delta.kernel.internal.rowtracking.MaterializedRowTrackingColumn; import io.delta.kernel.internal.util.ColumnMapping.ColumnMappingMode; import io.delta.kernel.internal.util.PartitionUtils; import io.delta.kernel.internal.util.Tuple2; import io.delta.kernel.types.MetadataColumnSpec; import io.delta.kernel.types.StructField; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterator; import java.io.IOException; import java.util.Map; import java.util.Optional; /** * Represents a scan of a Delta table. * * @since 3.0.0 */ @Evolving public interface Scan { /** * Get an iterator of data files to scan. * * @param engine {@link Engine} instance to use in Delta Kernel. * @return iterator of {@link FilteredColumnarBatch}s where each selected row in the batch * corresponds to one scan file. Schema of each row is defined as follows: *

*

    *
  1. *
      *
    • name: {@code add}, type: {@code struct} *
    • Description: Represents `AddFile` DeltaLog action *
    • *
        *
      • name: {@code path}, type: {@code string}, description: location of the * file. The path is a URI as specified by RFC 2396 URI Generic Syntax, * which needs to be decoded to get the data file path. *
      • name: {@code partitionValues}, type: {@code map(string, string)}, * description: A map from partition column to value for this logical file. *
      • name: {@code size}, type: {@code long}, description: size of the file. *
      • name: {@code modificationTime}, type: {@code log}, description: the time * this logical file was created, as milliseconds since the epoch. *
      • name: {@code dataChange}, type: {@code boolean}, description: When false * the logical file must already be present in the table or the records in * the added file must be contained in one or more remove actions in the * same version *
      • name: {@code deletionVector}, type: {@code string}, description: Either * null (or absent in JSON) when no DV is associated with this data file, or * a struct (described below) that contains necessary information about the * DV that is part of this logical file. For description of each member * variable in `deletionVector` @see * Protocol *
          *
        • name: {@code storageType}, type: {@code string} *
        • name: {@code pathOrInlineDv}, type: {@code string}, description: * The path is a URI as specified by RFC 2396 URI Generic Syntax, * which needs to be decoded to get the data file path. *
        • name: {@code offset}, type: {@code log} *
        • name: {@code sizeInBytes}, type: {@code log} *
        • name: {@code cardinality}, type: {@code log} *
        *
      • name: {@code tags}, type: {@code map(string, string)}, description: Map * containing metadata about the scan file. *
      *
    *
  2. *
      *
    • name: {@code tableRoot}, type: {@code string} *
    • Description: Absolute path of the table location. The path is a URI as * specified by RFC 2396 URI Generic Syntax, which needs to be decoded to get the * data file path. NOTE: this is temporary. Will be removed in the future. *
    *
* * @see */ CloseableIterator getScanFiles(Engine engine); /** * Get the remaining filter that is not guaranteed to be satisfied for the data Delta Kernel * returns. This filter is used by Delta Kernel to do data skipping when possible. * * @return the remaining filter as a {@link Predicate}. */ Optional getRemainingFilter(); /** * Get the scan state associated with the current scan. This state is common across all files in * the scan to be read. * * @param engine {@link Engine} instance to use in Delta Kernel. * @return Scan state in {@link Row} format. */ Row getScanState(Engine engine); /** * Transform the physical data read from the table data file to the logical data that are expected * out of the Delta table. * *

This iterator effectively reverses the logical-to-physical schema transformation performed * in {@link ScanImpl#getScanState(Engine)} by transforming physical data batches into the logical * data requested by the connector. * * @param engine Connector provided {@link Engine} implementation. * @param scanState Scan state returned by {@link Scan#getScanState(Engine)} * @param scanFile Scan file from where the physical data {@code physicalDataIter} is read from. * @param physicalDataIter Iterator of {@link ColumnarBatch}s containing the physical data read * from the {@code scanFile}. * @return Data read from the input scan files as an iterator of {@link FilteredColumnarBatch}s. * Each {@link FilteredColumnarBatch} instance contains the data read and an optional * selection vector that indicates data rows as valid or invalid. It is the responsibility of * the caller to close this iterator. * @throws IOException when error occurs while reading the data. */ static CloseableIterator transformPhysicalData( Engine engine, Row scanState, Row scanFile, CloseableIterator physicalDataIter) throws IOException { return new CloseableIterator() { boolean inited = false; // initialized as part of init() StructType logicalSchema = null; Map configuration = null; ColumnMappingMode columnMappingMode = null; String tablePath = null; RoaringBitmapArray currBitmap = null; DeletionVectorDescriptor currDV = null; private void initIfRequired() { if (inited) { return; } logicalSchema = ScanStateRow.getLogicalSchema(scanState); configuration = ScanStateRow.getConfiguration(scanState); columnMappingMode = ScanStateRow.getColumnMappingMode(scanState); tablePath = ScanStateRow.getTableRoot(scanState).toString(); inited = true; } @Override public void close() throws IOException { physicalDataIter.close(); } @Override public boolean hasNext() { initIfRequired(); return physicalDataIter.hasNext(); } @Override public FilteredColumnarBatch next() { initIfRequired(); ColumnarBatch nextDataBatch = physicalDataIter.next(); // Step 1: If row tracking is enabled, check for physical row tracking columns in the data // batch and transform them to logical row tracking columns as needed if (TableConfig.ROW_TRACKING_ENABLED.fromMetadata(configuration)) { nextDataBatch = MaterializedRowTrackingColumn.transformPhysicalData( nextDataBatch, scanFile, logicalSchema, configuration, engine); } // Step 2: Get the selectionVector if DV is present DeletionVectorDescriptor dv = InternalScanFileUtils.getDeletionVectorDescriptorFromRow(scanFile); Optional selectionVector; if (dv == null) { selectionVector = Optional.empty(); } else { int rowIndexOrdinal = nextDataBatch.getSchema().indexOf(MetadataColumnSpec.ROW_INDEX); if (rowIndexOrdinal == -1) { throw new IllegalArgumentException( "Row index column is not present in the data read from the Parquet file."); } if (!dv.equals(currDV)) { Tuple2 dvInfo = DeletionVectorUtils.loadNewDvAndBitmap(engine, tablePath, dv); this.currDV = dvInfo._1; this.currBitmap = dvInfo._2; } ColumnVector rowIndexVector = nextDataBatch.getColumnVector(rowIndexOrdinal); selectionVector = Optional.of(new SelectionColumnVector(currBitmap, rowIndexVector)); } // Step 3: If a column was only requested to compute other columns, we remove it for (StructField field : nextDataBatch.getSchema().fields()) { if (field.isInternalColumn()) { int columnOrdinal = nextDataBatch.getSchema().indexOf(field.getName()); if (columnOrdinal == -1) { // This should never happen since we only interact with a single schema throw new IllegalArgumentException( String.format( "Column %s was requested internally but is not present in the data batch.", field.getName())); } nextDataBatch = nextDataBatch.withDeletedColumnAt(columnOrdinal); } } // Step 4: Add partition columns back to the data batch nextDataBatch = PartitionUtils.withPartitionColumns( nextDataBatch, logicalSchema, InternalScanFileUtils.getPartitionValues(scanFile), engine.getExpressionHandler()); // Step 5: Transform column names back to logical names if column mapping is enabled switch (columnMappingMode) { case NAME: // fall through case ID: nextDataBatch = nextDataBatch.withNewSchema(logicalSchema); break; case NONE: break; default: throw new UnsupportedOperationException( "Column mapping mode is not yet supported: " + columnMappingMode); } return new FilteredColumnarBatch(nextDataBatch, selectionVector); } }; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/ScanBuilder.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.types.StructType; import java.util.Optional; /** * Builder to construct {@link Scan} object. * * @since 3.0.0 */ @Evolving public interface ScanBuilder { /** * Apply the given filter expression to prune any files that do not possibly contain the data that * satisfies the given filter. * *

Kernel makes use of the scan file partition values (for partitioned tables) and file-level * column statistics (min, max, null count etc.) in the Delta metadata for filtering. Sometimes * these metadata is not enough to deterministically say a scan file doesn't contain data that * satisfies the filter. * *

E.g. given filter is {@code a = 2}. In file A, column {@code a} has min value as -40 and max * value as 200. In file B, column {@code a} has min value as 78 and max value as 323. File B can * be ruled out as it cannot possibly have rows where `a = 2`, but file A cannot be ruled out as * it may contain rows where {@code a = 2}. * *

As filtering is a best effort, the {@link Scan} object may return scan files (through {@link * Scan#getScanFiles(Engine)}) that does not satisfy the filter. It is the responsibility of the * caller to apply the remaining filter returned by {@link Scan#getRemainingFilter()} to the data * read from the scan files (returned by {@link Scan#getScanFiles(Engine)}) to completely filter * out the data that doesn't satisfy the filter.``` * * @param predicate a {@link Predicate} to prune the metadata or data. * @return A {@link ScanBuilder} with filter applied. */ ScanBuilder withFilter(Predicate predicate); /** * Apply the given readSchema. If the builder already has a projection applied, calling * this again replaces the existing projection. * * @param readSchema Subset of columns to read from the Delta table. * @return A {@link ScanBuilder} with projection pruning. */ ScanBuilder withReadSchema(StructType readSchema); /** @return Build the {@link Scan instance} */ Scan build(); /** * Build a Paginated Scan with a required page size and an optional page token. * * @param pageSize Maximum number of Scan Files to return in this page. * @param pageToken Optional page token representing the current scan position; empty to start * from the beginning. * @return A {@link PaginatedScan} configured for pagination. */ PaginatedScan buildPaginated(long pageSize, Optional pageToken); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/Snapshot.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.commit.PublishFailedException; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.CheckpointAlreadyExistsException; import io.delta.kernel.statistics.SnapshotStatistics; import io.delta.kernel.transaction.UpdateTableTransactionBuilder; import io.delta.kernel.types.StructType; import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Optional; /** * Represents a snapshot of a Delta table at a specific version. * *

A {@code Snapshot} is a consistent view of a Delta table at a specific point in time, * identified by a version number. It provides access to the table's metadata, schema, and * capabilities for both reading and writing data. This interface serves as the entry point for * table operations after resolving a table through a {@link SnapshotBuilder}. * *

The snapshot represents a consistent view of the table at the resolved version. All operations * on this snapshot will see the same data and metadata, ensuring consistency across reads and * writes within the same snapshot. * *

There are two ways to create a {@code Snapshot}: * *

    *
  • New API (recommended): Use {@link TableManager#loadSnapshot(String)} to get a {@link * SnapshotBuilder}, which can then be configured and built into a snapshot *
  • Legacy API: Use {@code Table.forPath(path)} followed by methods like {@code * getLatestSnapshot()}, {@code getSnapshotAtTimestamp()}, etc. *
* * @since 3.0.0 */ @Evolving public interface Snapshot { /** * Indicates how a checksum file should be written for this Snapshot. Use with {@link * #writeChecksum(Engine, ChecksumWriteMode)}. */ enum ChecksumWriteMode { /** * Checksum info is already loaded in this Snapshot and can be written cheaply. This mode uses * pre-computed CRC information already in memory. */ SIMPLE, /** * Checksum info is not loaded in this Snapshot and requires replaying the delta log since the * latest checksum (if present) to compute. This mode performs full computation and writing. */ FULL } /** @return the file system path to this table */ String getPath(); /** @return the version of this snapshot in the Delta table */ long getVersion(); /** * Get the names of the partition columns in the Delta table at this snapshot. * *

The partition column names are returned in the order they are defined in the Delta table * schema. If the table does not define any partition columns, this method returns an empty list. * * @return a list of partition column names, or an empty list if the table is not partitioned. */ List getPartitionColumnNames(); /** * Get the timestamp (in milliseconds since the Unix epoch) of the latest commit in this snapshot. * * @param engine the engine to use for IO operations * @return the timestamp of the latest commit */ long getTimestamp(Engine engine); /** @return the schema of the Delta table at this snapshot */ StructType getSchema(); /** * Returns the configuration for the provided domain if it exists in the snapshot. Returns empty * if the domain is not present in the snapshot. * * @param domain the domain to look up * @return the domain configuration or empty */ Optional getDomainMetadata(String domain); /** * Get all table properties for the Delta table at this snapshot. * * @return a {@link Map} of table properties. */ Map getTableProperties(); /** @return statistics about this snapshot */ SnapshotStatistics getStatistics(); /** @return a scan builder to construct a {@link Scan} to read data from this snapshot */ ScanBuilder getScanBuilder(); /** * @return a {@link UpdateTableTransactionBuilder} to build an update table transaction * @since 3.4.0 */ UpdateTableTransactionBuilder buildUpdateTableTransaction(String engineInfo, Operation operation); /** * Publishes all catalog commits at this table version. Applicable only to catalog-managed tables. * This method is a no-op for filesystem-managed tables, if the committer doesn't support * publishing, or if there's no catalog commits to publish. * *

Publishing copies ratified catalog commits to the Delta log as published Delta files, * reducing catalog storage requirements and enabling some table maintenance operations, like * checkpointing. * * @param engine the engine to use for publishing commits * @see io.delta.kernel.commit.CatalogCommitter#publish * @throws PublishFailedException if the publish operation fails * @return a new Snapshot reflecting the published state */ Snapshot publish(Engine engine) throws PublishFailedException; /** * Writes a checksum file for this snapshot using the specified mode: * *

    *
  • SIMPLE: Uses pre-computed CRC information already loaded in memory. This is the fastest * approach but requires CRC info to be available. Throws {@link IllegalStateException} if * CRC information is not available. *
  • FULL: Computes the necessary CRC information by replaying the delta log since the latest * checksum (if present). This may be expensive for large tables when CRC information is not * available. *
* *

Use {@link SnapshotStatistics#getChecksumWriteMode()} to check if writing is needed and to * determine the appropriate mode. * *

This method should only be called if a checksum file does not already exist at this version. * If it already does, this method is a no-op. * *

If a concurrent writer creates the checksum file for this version between when this snapshot * was loaded and when this method is called, the method will detect the existing checksum and * return successfully without error. This ensures safe concurrent checksum writing. * * @param engine the engine to use for writing the checksum file and potentially reading the log * @param mode the mode specifying how to write the checksum (SIMPLE or FULL) * @throws IOException if an I/O error occurs during checksum computation or writing * @throws IllegalStateException if mode is SIMPLE but CRC information is not available * @see SnapshotStatistics#getChecksumWriteMode() */ void writeChecksum(Engine engine, ChecksumWriteMode mode) throws IOException; /** * Writes a checkpoint for the current snapshot. * * @param engine The execution engine used to write the checkpoint and, if necessary, read log * entries required to compute it. * @throws IOException If an I/O error occurs while computing or writing the checkpoint. * @throws IllegalStateException If attempting to create a checkpoint on an unpublished catalog * managed commit. * @throws CheckpointAlreadyExistsException If a checkpoint already exists for the target snapshot * version. */ void writeCheckpoint(Engine engine) throws IOException, CheckpointAlreadyExistsException; } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/SnapshotBuilder.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel; import io.delta.kernel.annotation.Experimental; import io.delta.kernel.commit.Committer; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.files.ParsedLogData; import java.util.List; /** * Builder for constructing a {@link Snapshot} instance. * *

This builder allows table managers (filesystems, catalogs) to provide any information they may * know about a Delta table and get back a {@link Snapshot}. When {@link #build(Engine)} is invoked, * Kernel will automatically fill any missing information needed to construct the {@link Snapshot} * by reading from the filesystem as needed. * *

If no version is specified, the builder will resolve to the latest version. Depending on the * {@link ParsedLogData} provided, Kernel can avoid expensive filesystem operations to improve * performance. */ @Experimental public interface SnapshotBuilder { /** * Configures the builder to resolve the table at a specific version. * *

This method is mutually exclusive with {@link #atTimestamp(long, Snapshot)}. If both are * called, an {@link IllegalArgumentException} will be thrown. * * @param version the version number to resolve to * @return a new builder instance configured for the specified version */ SnapshotBuilder atVersion(long version); /** * Configures the builder to resolve the table at a specific timestamp. * *

This returns a Snapshot for the latest version of the table that was committed before or at * the given timestamp. Specifically: * *

    *
  • If a commit version exactly matches the provided timestamp, the snapshot at that version * is resolved. *
  • Otherwise, the latest commit version with a timestamp less than the provided one is * resolved. *
  • If the provided timestamp is less than the timestamp of any committed version, snapshot * resolution will fail. *
  • If the provided timestamp is after (strictly greater than) the timestamp of the latest * version of the table, snapshot resolution will fail. *
* *

This method is mutually exclusive with {@link #atVersion(long)}. If both are called, an * {@link IllegalArgumentException} will be thrown. * * @param millisSinceEpochUTC timestamp to resolve the snapshot for in milliseconds since the unix * epoch * @return a new builder instance configured for the specified timestamp */ SnapshotBuilder atTimestamp(long millisSinceEpochUTC, Snapshot latestSnapshot); /** * Provides a custom committer to use at transaction commit time. * *

Catalog implementations that wish to support the catalogManaged Delta table feature should * provide to engines their own catalog-specific Committer implementation which may, for example, * send a commit RPC to the catalog service to finalize the commit. * *

If no committer is provided, a default committer will be created that only supports writing * into filesystem-managed Delta tables. * * @param committer the committer to use * @return a new builder instance with the provided committer * @see Committer */ SnapshotBuilder withCommitter(Committer committer); /** * Provides parsed log data to optimize table resolution. * *

When log data is provided, Kernel can avoid reading from the filesystem for information that * is already available in the parsed data, improving performance. Currently, only ratified staged * commits are supported. * * @param logData the parsed log data to use for optimization * @return a new builder instance with the provided log data */ SnapshotBuilder withLogData(List logData); /** * Provides protocol and metadata information to optimize table resolution. * *

When protocol and metadata are provided, Kernel can avoid reading this information from the * filesystem, improving performance. * * @param protocol the protocol information * @param metadata the metadata information * @return a new builder instance with the provided protocol and metadata */ // TODO: [delta-io/delta#4820] Public Protocol API // TODO: [delta-io/delta#4821] Public Metadata API SnapshotBuilder withProtocolAndMetadata(Protocol protocol, Metadata metadata); /** * Specifies the maximum table version known by the catalog. * *

This method is used by catalog implementations for catalog-managed Delta tables to indicate * the latest ratified version of the table. This ensures that any snapshot resolution operations * respect the catalog's view of the table state. * *

Important: This method is required for catalog-managed tables and must not be used for * file-system managed tables. An {@link IllegalArgumentException} will be thrown at build time if * this constraint is violated. * *

When specified, the following additional constraints are enforced: * *

    *
  • If {@link #atVersion(long)} is used for time travel, the requested version must be less * than or equal to the max catalog version. *
  • If {@link #atTimestamp(long, Snapshot)} is used for time travel, the provided {@code * latestSnapshot} must have a version equal to the max catalog version. *
  • If {@link #withLogData(List)} is provided and {@link #atVersion(long)} is used, the log * data must include the requested version (i.e., the tail of the log data must have a * version greater than or equal to the requested version). *
  • If {@link #withLogData(List)} is provided and no version is specified (resolving to * latest), the log data must end with the max catalog version. *
* * @param version the maximum table version known by the catalog (must be {@code >= 0}) * @return a new builder instance with the specified max catalog version * @throws IllegalArgumentException if version is negative */ SnapshotBuilder withMaxCatalogVersion(long version); /** * Constructs the {@link Snapshot} using the provided engine. * *

This method will read any missing information from the filesystem using the provided engine * to complete the snapshot resolution process. * * @param engine the engine to use for filesystem operations * @return the resolved snapshot instance */ Snapshot build(Engine engine); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/Table.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.CheckpointAlreadyExistsException; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.exceptions.TableNotFoundException; import io.delta.kernel.internal.TableImpl; import java.io.IOException; /** * Represents the Delta Lake table for a given path. * * @since 3.0.0 */ @Evolving public interface Table { /** * Instantiate a table object for the Delta Lake table at the given path. * *

    *
  • Behavior when the table location doesn't exist: *
      *
    • Reads will fail with a {@link TableNotFoundException} *
    • Writes will create the location *
    *
  • Behavior when the table location exists (with contents or not) but not a Delta table: *
      *
    • Reads will fail with a {@link TableNotFoundException} *
    • Writes will create a Delta table at the given location. If there are any existing * files in the location that are not already part of the Delta table, they will * remain excluded from the Delta table. *
    *
* * @param engine {@link Engine} instance to use in Delta Kernel. * @param path location of the table. Path is resolved to fully qualified path using the given * {@code engine}. * @return an instance of {@link Table} representing the Delta table at the given path */ static Table forPath(Engine engine, String path) { return TableImpl.forPath(engine, path); } /** * The fully qualified path of this {@link Table} instance. * * @param engine {@link Engine} instance. * @return the table path. * @since 3.2.0 */ String getPath(Engine engine); /** * Get the latest snapshot of the table. * * @param engine {@link Engine} instance to use in Delta Kernel. * @return an instance of {@link Snapshot} * @throws TableNotFoundException if the table is not found */ Snapshot getLatestSnapshot(Engine engine) throws TableNotFoundException; /** * Get the snapshot at the given {@code versionId}. * * @param engine {@link Engine} instance to use in Delta Kernel. * @param versionId snapshot version to retrieve * @return an instance of {@link Snapshot} * @throws TableNotFoundException if the table is not found * @throws KernelException if the provided version is less than the first available version or * greater than the last available version * @since 3.2.0 */ Snapshot getSnapshotAsOfVersion(Engine engine, long versionId) throws TableNotFoundException; /** * Get the snapshot of the table at the given {@code timestamp}. This is the latest version of the * table that was committed before or at {@code timestamp}. * *

Specifically: * *

    *
  • If a commit version exactly matches the provided timestamp, we return the table snapshot * at that version. *
  • Else, we return the latest commit version with a timestamp less than the provided one. *
  • If the provided timestamp is less than the timestamp of any committed version, we throw * an error. *
  • If the provided timestamp is after (strictly greater than) the timestamp of the latest * version of the table, we throw an error *
* * . * * @param engine {@link Engine} instance to use in Delta Kernel. * @param millisSinceEpochUTC timestamp to fetch the snapshot for in milliseconds since the unix * epoch * @return an instance of {@link Snapshot} * @throws TableNotFoundException if the table is not found * @throws KernelException if the provided timestamp is before the earliest available version or * after the latest available version * @since 3.2.0 */ Snapshot getSnapshotAsOfTimestamp(Engine engine, long millisSinceEpochUTC) throws TableNotFoundException; /** * Create a {@link TransactionBuilder} which can create a {@link Transaction} object to mutate the * table. * * @param engine {@link Engine} instance to use. * @param engineInfo information about the engine that is making the updates. * @param operation metadata of operation that is being performed. E.g. "insert", "delete". * @return {@link TransactionBuilder} instance to build the transaction. * @since 3.2.0 */ TransactionBuilder createTransactionBuilder( Engine engine, String engineInfo, Operation operation); /** * Checkpoint the table at given version. It writes a single checkpoint file. * * @param engine {@link Engine} instance to use. * @param version Version to checkpoint. * @throws TableNotFoundException if the table is not found * @throws CheckpointAlreadyExistsException if a checkpoint already exists at the given version * @throws IOException for any I/O error. * @since 3.2.0 */ void checkpoint(Engine engine, long version) throws TableNotFoundException, CheckpointAlreadyExistsException, IOException; /** * Computes and writes a checksum file for the table at given version. If a checksum file already * exists, this method does nothing. * *

Note: For very large tables, this operation may be expensive as it requires scanning the log * to compute table statistics. * * @param engine {@link Engine} instance to use. * @param version Version to generate checksum file for. * @throws TableNotFoundException if the table is not found * @throws IOException for any I/O error. * @since 4.0.0 */ void checksum(Engine engine, long version) throws TableNotFoundException, IOException; } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/TableManager.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel; import io.delta.kernel.annotation.Experimental; import io.delta.kernel.internal.CreateTableTransactionBuilderImpl; import io.delta.kernel.internal.commitrange.CommitRangeBuilderImpl; import io.delta.kernel.internal.table.SnapshotBuilderImpl; import io.delta.kernel.transaction.CreateTableTransactionBuilder; import io.delta.kernel.types.StructType; /** * The entry point for loading and creating Delta tables. * *

TableManager provides static factory methods for creating builders that can resolve Delta * tables to specific snapshots. This is the primary interface for table discovery and resolution in * the Delta Kernel. */ @Experimental public interface TableManager { /** * Creates a builder for loading a snapshot at the given path. * *

The returned builder can be configured to load the snapshot at a specific version or with * additional metadata to optimize the loading process. If no version is specified, the builder * will resolve to the latest version of the table. * * @param path the file system path to the Delta table * @return a {@link SnapshotBuilder} that can be used to load a {@link Snapshot} at the given path */ static SnapshotBuilder loadSnapshot(String path) { return new SnapshotBuilderImpl(path); } /** * Creates a {@link CreateTableTransactionBuilder} to build a create table transaction. * * @param path the file system path for the delta table being created * @param engineInfo information about the engine that is making the update. * @param schema the schema for the delta table being created * @return create table builder instance to build the transaction * @since 3.4.0 */ static CreateTableTransactionBuilder buildCreateTableTransaction( String path, StructType schema, String engineInfo) { return new CreateTableTransactionBuilderImpl(path, schema, engineInfo); } /** * Creates a builder for loading a CommitRange at a given path. * *

The returned builder can be configured with an end version or timestamp, and with additional * metadata to optimize the loading process. * * @param path the file system path to the Delta table * @param startBoundary the boundary specification for the start of the commit range, must not be * null * @return a {@link CommitRangeBuilder} that can be used to load a {@link CommitRange} at the * given path */ static CommitRangeBuilder loadCommitRange( String path, CommitRangeBuilder.CommitBoundary startBoundary) { return new CommitRangeBuilderImpl(path, startBoundary); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/Transaction.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel; import static io.delta.kernel.internal.DeltaErrors.dataSchemaMismatch; import static io.delta.kernel.internal.DeltaErrors.partitionColumnMissingInData; import static io.delta.kernel.internal.TransactionImpl.getStatisticsColumns; import static io.delta.kernel.internal.data.TransactionStateRow.*; import static io.delta.kernel.internal.util.ColumnMapping.blockIfColumnMappingEnabled; import static io.delta.kernel.internal.util.PartitionUtils.getTargetDirectory; import static io.delta.kernel.internal.util.PartitionUtils.validateAndSanitizePartitionValues; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static io.delta.kernel.internal.util.SchemaUtils.findColIndex; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.annotation.Experimental; import io.delta.kernel.commit.Committer; import io.delta.kernel.data.*; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.ConcurrentWriteException; import io.delta.kernel.exceptions.DomainDoesNotExistException; import io.delta.kernel.expressions.Literal; import io.delta.kernel.internal.DataWriteContextImpl; import io.delta.kernel.internal.actions.AddFile; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.actions.SingleAction; import io.delta.kernel.internal.columndefaults.ColumnDefaults; import io.delta.kernel.internal.data.TransactionStateRow; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.icebergcompat.IcebergCompatV2MetadataValidatorAndUpdater; import io.delta.kernel.internal.icebergcompat.IcebergCompatV3MetadataValidatorAndUpdater; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.internal.util.SchemaIterable; import io.delta.kernel.statistics.DataFileStatistics; import io.delta.kernel.types.StructField; import io.delta.kernel.types.StructType; import io.delta.kernel.types.VariantType; import io.delta.kernel.utils.*; import java.net.URI; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.Supplier; /** * Represents a transaction to mutate a Delta table. * * @since 3.2.0 */ @Evolving public interface Transaction { /** * Get the schema of the table. If the connector is adding any data to the table through this * transaction, it should have the same schema as the table schema. */ StructType getSchema(Engine engine); /** * Get the list of logical names of the partition columns. This helps the connector to do physical * partitioning of the data before asking the Kernel to stage the data per partition. */ List getPartitionColumns(Engine engine); /** * Gets the latest version of the table used as the base of this transaction. This returns -1 when * the table is being created in this transaction. * * @return The version of the table as of the beginning of this Transaction */ long getReadTableVersion(); /** * Get the state of the transaction. The state helps Kernel do the transformations to logical data * according to the Delta protocol and table features enabled on the table. The engine should use * this at the data writer task to transform the logical data that the engine wants to write to * the table in to physical data that goes in data files using {@link * Transaction#transformLogicalData(Engine, Row, CloseableIterator, Map)} */ Row getTransactionState(Engine engine); /** @return a committer that owns and controls commits to this table */ @Experimental Committer getCommitter(); /** * Commit the transaction including the data action rows generated by {@link * Transaction#generateAppendActions}. * * @param engine {@link Engine} instance. * @param dataActions Iterable of data actions to commit. These data actions are generated by the * {@link Transaction#generateAppendActions(Engine, Row, CloseableIterator, * DataWriteContext)}. The {@link CloseableIterable} allows the Kernel to access the list of * actions multiple times (in case of retries to resolve the conflicts due to other writers to * the table). Kernel provides a in-memory based implementation of {@link CloseableIterable} * with utility API {@link CloseableIterable#inMemoryIterable(CloseableIterator)} * @return {@link TransactionCommitResult} status of the successful transaction. * @throws ConcurrentWriteException when the transaction has encountered a non-retryable conflicts * or exceeded the maximum number of retries reached. The connector needs to rerun the query * on top of the latest table state and retry the transaction. */ TransactionCommitResult commit(Engine engine, CloseableIterable dataActions) throws ConcurrentWriteException; /** * Adds custom properties that will be passed through to the committer. These properties allow * connectors to inject catalog-specific metadata without Kernel inspection. Repeated calls to * this method will overwrite any previously set properties. */ void withCommitterProperties(Supplier> committerProperties); /** * Commit the provided domain metadata as part of this transaction. If this is called more than * once with the same {@code domain} the latest provided {@code config} will be committed in the * transaction. Only user-controlled domains are allowed (aka. domains with a `delta.` prefix are * not allowed). Adding and removing a domain with the same identifier in the same txn is not * allowed. Adding domain metadata to a table that does not support the table feature is not * allowed. To enable the table feature, make sure to call {@link * TransactionBuilder#withDomainMetadataSupported} * * @param domain the domain identifier * @param config configuration string for this domain */ void addDomainMetadata(String domain, String config); /** * Mark the domain metadata with identifier {@code domain} as removed in this transaction. If this * domain does not exist in the latest version of the table, calling {@link * Transaction#commit(Engine, CloseableIterable)} will throw a {@link * DomainDoesNotExistException}. Adding and removing a domain with the same identifier in one txn * is not allowed. * * @param domain the domain identifier for the domain to remove */ void removeDomainMetadata(String domain); /** * Given the logical data that needs to be written to the table, convert it into the required * physical data depending upon the table Delta protocol and features enabled on the table. Kernel * takes care of adding any additional column or removing existing columns that doesn't need to be * in physical data files. All these transformations are driven by the Delta protocol and table * features enabled on the table. * *

The given data should belong to exactly one partition. It is the job of the connector to do * partitioning of the data before calling the API. Partition values are provided as map of column * name to partition value (as {@link Literal}). If the table is an un-partitioned table, then map * should be empty. * * @param engine {@link Engine} instance to use. * @param transactionState The transaction state * @param dataIter Iterator of logical data (with schema same as the table schema) to transform to * physical data. All the data n this iterator should belong to one physical partition and it * should also include the partition data. * @param partitionValues The partition values for the data. If the table is un-partitioned, the * map should be empty * @return Iterator of physical data to write to the data files. */ static CloseableIterator transformLogicalData( Engine engine, Row transactionState, CloseableIterator dataIter, Map partitionValues) { // Note: `partitionValues` are not used as of now in this API, but taking the partition // values as input forces the connector to not pass data from multiple partitions this // API in a single call. StructType tableSchema = getLogicalSchema(transactionState); List partitionColNames = getPartitionColumnsList(transactionState); validateAndSanitizePartitionValues(tableSchema, partitionColNames, partitionValues); // TODO: add support for: // - enforcing the constraints // - generating the default value columns // - generating the generated columns boolean isIcebergCompatEnabled = isIcebergCompatV2Enabled(transactionState) || isIcebergCompatV3Enabled(transactionState); Protocol protocol = getProtocol(transactionState); boolean materializePartitionColumnsEnabled = protocol.supportsFeature(TableFeatures.MATERIALIZE_PARTITION_COLUMNS_W_FEATURE); blockIfColumnMappingEnabled(transactionState); blockIfVariantDataTypeIsDefined(tableSchema); // We recognize the AllowColumnDefaults feature for Iceberg v3 // but do not support writing with it yet ColumnDefaults.blockWriteIfEnabled(transactionState); // TODO: set the correct schema once writing into column mapping enabled table is supported. String tablePath = getTablePath(transactionState); return dataIter.map( filteredBatch -> { ColumnarBatch data = filteredBatch.getData(); if (!data.getSchema().isWriteCompatible(tableSchema)) { throw dataSchemaMismatch(tablePath, tableSchema, data.getSchema()); } if (isIcebergCompatEnabled || materializePartitionColumnsEnabled) { // Move partition columns to the end of the schema for iceberg compat enabled tables // or when materialize partition columns feature is enabled. for (String partitionColName : partitionColNames) { int partitionColIndex = findColIndex(data.getSchema(), partitionColName); if (partitionColIndex < 0) { throw partitionColumnMissingInData(tablePath, partitionColName); } StructField partitionColField = data.getSchema().at(partitionColIndex); ColumnVector partitionColVector = data.getColumnVector(partitionColIndex); data = data.withDeletedColumnAt(partitionColIndex); // Add the partition column at the end data = data.withNewColumn( data.getSchema().length(), partitionColField, partitionColVector); } } else { // Remove partition columns entirely for non-materialized partitions, and non-iceberg // compat tables. for (String partitionColName : partitionColNames) { int partitionColIndex = findColIndex(data.getSchema(), partitionColName); if (partitionColIndex < 0) { throw partitionColumnMissingInData(tablePath, partitionColName); } data = data.withDeletedColumnAt(partitionColIndex); } } return new FilteredColumnarBatch(data, filteredBatch.getSelectionVector()); }); } /** * Currently Kernel supports only metadata updates for variants (including shredded values). Block * any physical data writes if variant exists in the schema */ static void blockIfVariantDataTypeIsDefined(StructType tableSchema) { boolean variantFieldExists = new SchemaIterable(tableSchema) .stream().anyMatch(field -> field.getField().getDataType() instanceof VariantType); if (variantFieldExists) { throw new UnsupportedOperationException( "Transforming logical data with variant data is currently unsupported"); } } /** * Get the context for writing data into a table. The context tells the connector where the data * should be written. For partitioned table context is generated per partition. So, the connector * should call this API for each partition. For un-partitioned table, the context is same for all * the data. * * @param engine {@link Engine} instance to use. * @param transactionState The transaction state * @param partitionValues The partition values for the data. If the table is un-partitioned, the * map should be empty * @return {@link DataWriteContext} containing metadata about where and how the data for partition * should be written. */ static DataWriteContext getWriteContext( Engine engine, Row transactionState, Map partitionValues) { blockIfColumnMappingEnabled(transactionState); StructType tableSchema = getLogicalSchema(transactionState); List partitionColNames = getPartitionColumnsList(transactionState); partitionValues = validateAndSanitizePartitionValues(tableSchema, partitionColNames, partitionValues); String targetDirectory = getTargetDirectory(getTablePath(transactionState), partitionColNames, partitionValues); return new DataWriteContextImpl( targetDirectory, partitionValues, getStatisticsColumns(transactionState)); } /** * For given data files, generate Delta actions that can be committed in a transaction. These data * files are the result of writing the data returned by {@link Transaction#transformLogicalData} * with the context returned by {@link Transaction#getWriteContext}. * * @param engine {@link Engine} instance. * @param transactionState State of the transaction. * @param fileStatusIter Iterator of row objects representing each data file written. When {@code * delta.icebergCompatV2} is enabled, each data file status should contain {@link * DataFileStatistics} with at least the {@link DataFileStatistics#getNumRecords()} field set. * @param dataWriteContext The context used when writing the data files given in {@code * fileStatusIter} * @return {@link CloseableIterator} of {@link Row} representing the actions to commit using * {@link Transaction#commit}. */ static CloseableIterator generateAppendActions( Engine engine, Row transactionState, CloseableIterator fileStatusIter, DataWriteContext dataWriteContext) { checkArgument( dataWriteContext instanceof DataWriteContextImpl, "DataWriteContext is not created by the `Transaction.getWriteContext()`"); boolean isIcebergCompatV2Enabled = isIcebergCompatV2Enabled(transactionState); boolean isIcebergCompatV3Enabled = isIcebergCompatV3Enabled(transactionState); URI tableRoot = new Path(getTablePath(transactionState)).toUri(); StructType physicalSchema = TransactionStateRow.getPhysicalSchema(transactionState); return fileStatusIter.map( dataFileStatus -> { if (isIcebergCompatV2Enabled) { IcebergCompatV2MetadataValidatorAndUpdater.validateDataFileStatus(dataFileStatus); } else if (isIcebergCompatV3Enabled) { IcebergCompatV3MetadataValidatorAndUpdater.validateDataFileStatus(dataFileStatus); } AddFile addFileRow = AddFile.convertDataFileStatus( physicalSchema, tableRoot, dataFileStatus, ((DataWriteContextImpl) dataWriteContext).getPartitionValues(), true /* dataChange */, // TODO: populate tags in generateAppendActions Collections.emptyMap() /* tags */, Optional.empty() /* baseRowId */, Optional.empty() /* defaultRowCommitVersion */, Optional.empty() /* deletionVectorDescriptor */); return SingleAction.createAddFileSingleAction(addFileRow.toRow()); }); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/TransactionBuilder.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.ConcurrentTransactionException; import io.delta.kernel.exceptions.DomainDoesNotExistException; import io.delta.kernel.exceptions.InvalidConfigurationValueException; import io.delta.kernel.exceptions.TableAlreadyExistsException; import io.delta.kernel.exceptions.UnknownConfigurationException; import io.delta.kernel.expressions.Column; import io.delta.kernel.internal.TableConfig; import io.delta.kernel.types.StructType; import java.util.List; import java.util.Map; import java.util.Set; /** * Builder for creating a {@link Transaction} to mutate a Delta table. * * @since 3.2.0 */ @Evolving public interface TransactionBuilder { /** * Set the schema of the table. If setting the schema on an existing table for a schema evolution, * then column mapping must be enabled. This API will preserve field metadata for fields such as * field IDs and physical names. If field metadata is not specified for a field, it is considered * as a new column and new IDs/physical names will be specified. The possible schema evolutions * supported include column additions, removals, renames, and moves. If a schema evolution is * performed, implementations must perform the following validations: * *

    *
  • No duplicate columns are allowed *
  • Column names contain only valid characters *
  • Data types are supported *
  • No new non-nullable fields are added *
  • Physical column name consistency is preserved in the new schema *
  • No type changes *
  • ToDo: Nested IDs for array/map types are preserved in the new schema *
  • ToDo: Validate invalid field reorderings *
* * @param engine {@link Engine} instance to use. * @param schema The new schema of the table. * @return updated {@link TransactionBuilder} instance. * @throws io.delta.kernel.exceptions.KernelException in case column mapping is not enabled * @throws IllegalArgumentException in case of any validation failure */ TransactionBuilder withSchema(Engine engine, StructType schema); /** * Set the list of partitions columns when create a new partitioned table. * * @param engine {@link Engine} instance to use. * @param partitionColumns The partition columns of the table. These should be a subset of the * columns in the schema. Only top-level columns are allowed to be partitioned. Note: * Clustering columns and partition columns cannot coexist in a table. * @return updated {@link TransactionBuilder} instance. */ TransactionBuilder withPartitionColumns(Engine engine, List partitionColumns); /** * Set the list of clustering columns when create a new clustered table. * * @param engine {@link Engine} instance to use. * @param clusteringColumns The clustering columns of the table. These should be a subset of the * columns in the schema. Both top-level and nested columns are allowed to be clustered. Note: * Clustering columns and partition columns cannot coexist in a table. * @return updated {@link TransactionBuilder} instance. */ TransactionBuilder withClusteringColumns(Engine engine, List clusteringColumns); /** * Set the transaction identifier for idempotent writes. Incremental processing systems (e.g., * streaming systems) that track progress using their own application-specific versions need to * record what progress has been made, in order to avoid duplicating data in the face of failures * and retries during writes. By setting the transaction identifier, the Delta table can ensure * that the data with same identifier is not written multiple times. For more information refer to * the Delta protocol section * Transaction Identifiers. * * @param engine {@link Engine} instance to use. * @param applicationId The application ID that is writing to the table. * @param transactionVersion The version of the transaction. This should be monotonically * increasing with each write for the same application ID. * @return updated {@link TransactionBuilder} instance. */ TransactionBuilder withTransactionId( Engine engine, String applicationId, long transactionVersion); /** * Set the table properties for the table. When the table already contains the property with same * key, it gets replaced if it doesn't have the same value. Note, user-properties (those without a * '.delta' prefix) are case-sensitive. Delta-properties are case-insensitive and are normalized * to their expected case before writing to the log. * * @param engine {@link Engine} instance to use. * @param properties The table properties to set. These are key-value pairs that can be used to * configure the table. And these properties are stored in the table metadata. * @return updated {@link TransactionBuilder} instance. * @since 3.3.0 */ TransactionBuilder withTableProperties(Engine engine, Map properties); /** * Unset the provided table properties on the table. If a property does not exist this is a no-op. * For now this is only supported for user-properties (in other words, does not support 'delta.' * prefixed properties). An exception will be thrown upon calling {@link * TransactionBuilder#build(Engine)} if the same key is both set and unset in the same * transaction. Note, user-properties (those without a '.delta' prefix) are case-sensitive. * * @param propertyKeys the table property keys to unset (remove from the table properties) * @return updated {@link TransactionBuilder} instance. * @throws IllegalArgumentException if 'delta.' prefixed keys are provided */ TransactionBuilder withTablePropertiesRemoved(Set propertyKeys); /** * Set the maximum number of times to retry a transaction if a concurrent write is detected. This * defaults to 200 * * @param maxRetries The number of times to retry * @return updated {@link TransactionBuilder} instance */ TransactionBuilder withMaxRetries(int maxRetries); /** * Set the number of commits between log compactions. Defaults to 0 (disabled). For more * information see the Delta protocol section Log * Compaction Files. * * @param logCompactionInterval The commits between log compactions * @return updated {@link TransactionBuilder} instance */ TransactionBuilder withLogCompactionInverval(int logCompactionInterval); /** * Enables support for Domain Metadata on this table if it is not supported already. The table * feature _must_ be supported on the table to add or remove domain metadata using {@link * Transaction#addDomainMetadata} or {@link Transaction#removeDomainMetadata}. See * How does Delta Lake manage feature compatibility? for more details on table feature * support. * *

See the Delta protocol for more information on how to use Domain * Metadata. This may break existing writers that do not support the Domain Metadata feature; * readers will be unaffected. */ TransactionBuilder withDomainMetadataSupported(); /** * Build the transaction. Also validates the given info to ensure that a valid transaction can be * created. * * @param engine {@link Engine} instance to use. * @throws ConcurrentTransactionException if the table already has a committed transaction with * the same given transaction identifier. * @throws InvalidConfigurationValueException if the value of the property is invalid. * @throws UnknownConfigurationException if any of the properties are unknown to {@link * TableConfig}. * @throws DomainDoesNotExistException if removing a domain that does not exist in the latest * version of the table * @throws TableAlreadyExistsException if the operation provided when calling {@link * Table#createTransactionBuilder(Engine, String, Operation)} is CREATE_TABLE and the table * already exists */ Transaction build(Engine engine); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/TransactionCommitResult.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel; import static java.util.Objects.requireNonNull; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.engine.Engine; import io.delta.kernel.hook.PostCommitHook; import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.metrics.TransactionReport; import io.delta.kernel.utils.CloseableIterable; import java.util.List; import java.util.Optional; /** * Contains the result of a successful transaction commit. Returned by {@link * Transaction#commit(Engine, CloseableIterable)}. * * @since 3.2.0 */ @Evolving public class TransactionCommitResult { private final long version; private final List postCommitHooks; private final TransactionReport transactionReport; private final Optional postCommitSnapshotOpt; public TransactionCommitResult( long version, List postCommitHooks, TransactionReport transactionReport, Optional postCommitSnapshotOpt) { this.version = version; this.postCommitHooks = requireNonNull(postCommitHooks); this.transactionReport = requireNonNull(transactionReport); this.postCommitSnapshotOpt = requireNonNull(postCommitSnapshotOpt); } /** * Contains the version of the transaction committed as. * * @return version the transaction is committed as. */ public long getVersion() { return version; } /** * Operations for connector to trigger post-commit. * *

Usage: * *

    *
  • Async: Call {@link PostCommitHook#threadSafeInvoke(Engine)} in separate thread. *
  • Sync: Direct call {@link PostCommitHook#threadSafeInvoke(Engine)} and block until * operation ends. *
* * @return list of post-commit operations */ public List getPostCommitHooks() { return postCommitHooks; } /** @return the report and metrics for this transaction */ public TransactionReport getTransactionReport() { return transactionReport; } /** * Return the snapshot at the committed version. * *

Currently, Kernel does not support getting the post-commit snapshot for transactions that * experienced conflicts. */ public Optional getPostCommitSnapshot() { return postCommitSnapshotOpt.map(s -> s); // Map needed to upcast to Optional } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/annotation/Evolving.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.annotation; import java.lang.annotation.*; /** * APIs that are meant to evolve towards becoming stable APIs, but are not stable APIs yet. Evolving * interfaces can change from one feature release to another release (i.e. 3.0 to 3.1). */ @Documented @Retention(RetentionPolicy.RUNTIME) @Target({ ElementType.TYPE, ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER, ElementType.CONSTRUCTOR, ElementType.LOCAL_VARIABLE, ElementType.PACKAGE }) public @interface Evolving {} ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/annotation/Experimental.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.annotation; import java.lang.annotation.*; /** * APIs that are still under active development and are expected to change. Experimental interfaces * can change, break, or be deleted from one feature release to another release (i.e. 3.0 to 3.1). */ @Documented @Retention(RetentionPolicy.RUNTIME) @Target({ ElementType.TYPE, ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER, ElementType.CONSTRUCTOR, ElementType.LOCAL_VARIABLE, ElementType.PACKAGE }) public @interface Experimental {} ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/commit/CatalogCommitter.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.commit; import io.delta.kernel.annotation.Experimental; import io.delta.kernel.engine.Engine; import java.util.Collections; import java.util.Map; /** * {@link Committer} sub-interface for catalog-managed tables. Provides catalog-specific operations * not applicable to filesystem-managed tables. */ @Experimental public interface CatalogCommitter extends Committer { /** * Returns required catalog table properties that must be set in the Delta metadata. * *

These properties are automatically injected during CREATE and REPLACE operations and cannot * be changed or removed by users. Any attempt to set these properties to different values or * remove them will result in a validation error. * * @return a map of required catalog properties */ default Map getRequiredTableProperties() { return Collections.emptyMap(); } /** * Publishes catalog commits to the Delta log. Applicable only to catalog-managed tables. * *

Publishing is the act of copying ratified catalog commits to the Delta log as published * Delta files (e.g., {@code _delta_log/00000000000000000001.json}). * *

The benefits of publishing include: * *

    *
  • Reduces the number of commits the catalog needs to store internally and serve to readers *
  • Enables table maintenance operations that must operate on published versions only, such * as checkpointing and log compaction *
* *

Requirements: * *

    *
  • This method must ensure that all catalog commits are published to the Delta log up to and * including the snapshot version specified in {@code publishMetadata} *
  • Commits must be published in order: version V-1 must be published before version V *
* *

Catalog-specific semantics: Each catalog implementation may specify its own rules and * semantics for publishing, including whether it expects to be notified immediately upon * publishing success, whether published deltas must appear with PUT-if-absent semantics in the * Delta log, and whether publishing happens in the client-side or server-side catalog-component. * * @param engine the {@link Engine} instance used for publishing commits * @param publishMetadata the {@link PublishMetadata} containing the snapshot version up to which * all catalog commits must be published, the log path, and list of catalog commits * @throws PublishFailedException if the publish operation fails */ void publish(Engine engine, PublishMetadata publishMetadata) throws PublishFailedException; } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/commit/CatalogCommitterUtils.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.commit; import io.delta.kernel.internal.TableConfig; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.tablefeatures.TableFeatures; import java.util.HashMap; import java.util.Map; public class CatalogCommitterUtils { private CatalogCommitterUtils() {} /** Property key that specifies which version last updated the catalog entry. */ public static final String METASTORE_LAST_UPDATE_VERSION = "delta.lastUpdateVersion"; /** * Property key that specifies the timestamp (in milliseconds since the Unix epoch) of the last * commit that updated the catalog entry. */ public static final String METASTORE_LAST_COMMIT_TIMESTAMP = "delta.lastCommitTimestamp"; /** * Extract protocol-related properties from the given protocol. * *

For a Protocol(3, 7) with reader features ["columnMapping", "deletionVectors"] and writer * features ["appendOnly", "columnMapping"], this would return properties like: * *

    *
  • delta.minReaderVersion: 3 *
  • delta.minWriterVersion: 7 *
  • delta.feature.columnMapping: supported *
  • delta.feature.deletionVectors: supported *
  • delta.feature.appendOnly: supported *
*/ public static Map extractProtocolProperties(Protocol protocol) { final Map properties = new HashMap<>(); properties.put( TableConfig.MIN_PROTOCOL_READER_VERSION_KEY, String.valueOf(protocol.getMinReaderVersion())); properties.put( TableConfig.MIN_PROTOCOL_WRITER_VERSION_KEY, String.valueOf(protocol.getMinWriterVersion())); if (protocol.supportsReaderFeatures() || protocol.supportsWriterFeatures()) { for (String featureName : protocol.getReaderAndWriterFeatures()) { properties.put( TableFeatures.getTableFeature(featureName).getTableFeatureSupportKey(), TableFeatures.SET_TABLE_FEATURE_SUPPORTED_VALUE); } } return properties; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/commit/CommitFailedException.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.commit; import io.delta.kernel.annotation.Experimental; /** * Exception raised by {@link Committer#commit}. * * 
 *  | retryable | conflict  | meaning                                                         |
 *  |   no      |   no      | something bad happened (e.g. auth failure)                      |
 *  |   no      |   yes     | permanent transaction conflict (e.g. multi-table commit failed) |
 *  |   yes     |   no      | transient error (e.g. network hiccup)                           |
 *  |   yes     |   yes     | physical conflict (allowed to rebase and retry)                 |
 *
*/ @Experimental public class CommitFailedException extends Exception { private final boolean retryable; private final boolean conflict; // TODO: [delta-io/delta#4908] Include the winning, conflicting catalog ratified commits here public CommitFailedException(boolean retryable, boolean conflict, String message) { super(message); this.retryable = retryable; this.conflict = conflict; } public CommitFailedException( boolean retryable, boolean conflict, String message, Throwable cause) { super(message, cause); this.retryable = retryable; this.conflict = conflict; } /** Returns whether the commit can be retried. */ public boolean isRetryable() { return retryable; } /** Returns whether the commit failed due to a conflict. */ public boolean isConflict() { return conflict; } @Override public String toString() { return String.format( "%s: retryable=%s, conflict=%s, msg=%s", getClass().getName(), retryable, conflict, getMessage()); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/commit/CommitMetadata.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.commit; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.annotation.Experimental; import io.delta.kernel.internal.actions.CommitInfo; import io.delta.kernel.internal.actions.DomainMetadata; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.internal.util.Tuple2; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.Supplier; /** * Contains all information (excluding the iterator of finalized actions) required to commit changes * to a Delta table. */ @Experimental public class CommitMetadata { /** * Represents the different types of commits based on filesystem-managed and catalog-managed state * transitions. */ public enum CommitType { /** Creating a new filesystem-managed table */ FILESYSTEM_CREATE, /** Creating a new catalog-managed table */ CATALOG_CREATE, /** Writing to an existing filesystem-managed table */ FILESYSTEM_WRITE, /** Writing to an existing catalog-managed table */ CATALOG_WRITE, /** Upgrading a filesystem-managed table to a catalog-managed table */ FILESYSTEM_UPGRADE_TO_CATALOG, /** Downgrading a catalog-managed table to a filesystem-managed table */ CATALOG_DOWNGRADE_TO_FILESYSTEM } private final long version; private final String logPath; private final CommitInfo commitInfo; private final List commitDomainMetadatas; private final Supplier> committerProperties; private final Optional> readPandMOpt; private final Optional newProtocolOpt; private final Optional newMetadataOpt; private final Optional maxKnownPublishedDeltaVersion; public CommitMetadata( long version, String logPath, CommitInfo commitInfo, List commitDomainMetadatas, Supplier> committerProperties, Optional> readPandMOpt, Optional newProtocolOpt, Optional newMetadataOpt, Optional maxKnownPublishedDeltaVersion) { checkArgument(version >= 0, "version must be non-negative: %d", version); this.version = version; this.logPath = requireNonNull(logPath, "logPath is null"); this.commitInfo = requireNonNull(commitInfo, "commitInfo is null"); this.commitDomainMetadatas = Collections.unmodifiableList( requireNonNull(commitDomainMetadatas, "txnDomainMetadatas is null")); this.committerProperties = requireNonNull(committerProperties, "committerProperties is null"); this.readPandMOpt = requireNonNull(readPandMOpt, "readPandMOpt is null"); this.newProtocolOpt = requireNonNull(newProtocolOpt, "newProtocolOpt is null"); this.newMetadataOpt = requireNonNull(newMetadataOpt, "newMetadataOpt is null"); this.maxKnownPublishedDeltaVersion = requireNonNull(maxKnownPublishedDeltaVersion, "maxKnownPublishedDeltaVersion is null"); checkArgument( readPandMOpt.isPresent() || newProtocolOpt.isPresent(), "At least one of readPandMOpt.protocol or newProtocolOpt must be present"); checkArgument( readPandMOpt.isPresent() || newMetadataOpt.isPresent(), "At least one of readPandMOpt.metadata or newMetadataOpt must be present"); checkReadStateAbsentIfAndOnlyIfVersion0(); checkInCommitTimestampPresentIfCatalogManaged(); } /** The version of the Delta table this commit is targeting. */ public long getVersion() { return version; } /** The path to the Delta log directory, located at {@code /_delta_log}. */ public String getDeltaLogDirPath() { return logPath; } /** The {@link CommitInfo} that is being written as part of this commit. */ public CommitInfo getCommitInfo() { return commitInfo; } /** * The {@link DomainMetadata}s that are being written as part of this commit. Includes those that * are being explicitly added and those that are being explicitly removed (tombstoned). * *

Does not include the domain metadatas that already exist in the transaction's read snapshot, * if any. */ public List getCommitDomainMetadatas() { return commitDomainMetadatas; } /** * Returns custom properties provided by the connector to be passed through to the committer. * These properties are not inspected by Kernel and are used for catalog-specific functionality. */ public Supplier> getCommitterProperties() { return committerProperties; } /** * The {@link Protocol} that was read at the beginning of the commit. Empty if a new table is * being created. */ public Optional getReadProtocolOpt() { return readPandMOpt.map(x -> x._1); } /** * The {@link Metadata} that was read at the beginning of the commit. Empty if a new table is * being created. */ public Optional getReadMetadataOpt() { return readPandMOpt.map(x -> x._2); } /** * The {@link Protocol} that is being written as part of this commit. Empty if the protocol is not * being changed. */ public Optional getNewProtocolOpt() { return newProtocolOpt; } /** * The {@link Metadata} that is being written as part of this commit. Empty if the metadata is not * being changed. */ public Optional getNewMetadataOpt() { return newMetadataOpt; } /** * Returns the maximum known published delta version at commit time. * *

This is a best-effort API that returns what was actually seen during Snapshot and * Transaction construction, not the authoritative maximum published delta version in the log. * *

{@code Optional.empty()} means "we don't know" - not necessarily that no deltas have been * published. * *

{@code Optional.of(-1)} means it is known that there are no published deltas (e.g., during * CREATE) * * @return the maximum known published delta version, or empty if unknown */ public Optional getMaxKnownPublishedDeltaVersion() { return maxKnownPublishedDeltaVersion; } /** * Returns the effective {@link Protocol} that will be in place after this commit. If a new * protocol is being written as part of this commit, returns the new protocol. Otherwise, returns * the protocol that was read at the beginning of the commit. */ public Protocol getEffectiveProtocol() { return newProtocolOpt.orElseGet(() -> getReadProtocolOpt().get()); } /** * Returns the effective {@link Metadata} that will be in place after this commit. If new metadata * is being written as part of this commit, returns the new metadata. Otherwise, returns the * metadata that was read at the beginning of the commit. */ public Metadata getEffectiveMetadata() { return newMetadataOpt.orElseGet(() -> getReadMetadataOpt().get()); } /** * Determines the type of commit based on whether this is a table creation and the catalog-managed * status of the table before and after the commit. */ public CommitType getCommitType() { final boolean isCreate = version == 0; final boolean readVersionCatalogManaged = readPandMOpt.map(x -> x._1).map(TableFeatures::isCatalogManagedSupported).orElse(false); final boolean writeVersionCatalogManaged = TableFeatures.isCatalogManagedSupported(getEffectiveProtocol()); if (isCreate && writeVersionCatalogManaged) { return CommitType.CATALOG_CREATE; } else if (isCreate && !writeVersionCatalogManaged) { return CommitType.FILESYSTEM_CREATE; } else if (readVersionCatalogManaged && writeVersionCatalogManaged) { return CommitType.CATALOG_WRITE; } else if (readVersionCatalogManaged && !writeVersionCatalogManaged) { return CommitType.CATALOG_DOWNGRADE_TO_FILESYSTEM; } else if (!readVersionCatalogManaged && writeVersionCatalogManaged) { return CommitType.FILESYSTEM_UPGRADE_TO_CATALOG; } else { return CommitType.FILESYSTEM_WRITE; } } /** * Returns the corresponding published Delta log file path for this commit, which is in the form * of {@code /_delta_log/0000000000000000000.json}. * *

Usages: * *

    *
  • Filesystem-managed committers must write to this file path. *
  • Catalog-managed committers must publish to this file path, if/when they so choose. *
*/ public String getPublishedDeltaFilePath() { return FileNames.deltaFile(logPath, version); } /** * Returns a new staged commit file path with a unique UUID for this commit. Each invocation * returns a new, unique value, in the form of {@code * /_delta_log/_staged_commits/0000000000000000000..json} * *

Catalog-managed committers may use this path to write new staged commits. */ public String generateNewStagedCommitFilePath() { return FileNames.stagedCommitFile(logPath, version); } private void checkReadStateAbsentIfAndOnlyIfVersion0() { checkArgument( (version == 0) == (!readPandMOpt.isPresent()), "Table creation (version 0) requires absent readPandMOpt, while existing table writes " + "(version > 0) require present readPandMOpt"); } private void checkInCommitTimestampPresentIfCatalogManaged() { if (TableFeatures.isCatalogManagedSupported(getEffectiveProtocol())) { checkArgument( commitInfo.getInCommitTimestamp().isPresent(), "InCommitTimestamp must be present for commits to catalogManaged tables"); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/commit/CommitResponse.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.commit; import io.delta.kernel.annotation.Experimental; import io.delta.kernel.internal.files.ParsedDeltaData; /** Response container for the result of a commit operation. */ @Experimental public class CommitResponse { // TODO: Create a DeltaLogData extends ParsedLogData that includes commit timestamp information. private final ParsedDeltaData commitLogData; public CommitResponse(ParsedDeltaData commitLogData) { this.commitLogData = commitLogData; } /** * The parsed log data resulting from the commit operation. Note that for catalog-managed tables, * this may be the ratified staged commit, the ratified inline commit, or even a published Delta * file that the {@link Committer} implementation decided to publish after committing to the * managing catalog. */ public ParsedDeltaData getCommitLogData() { return commitLogData; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/commit/Committer.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.commit; import io.delta.kernel.annotation.Experimental; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; import io.delta.kernel.utils.CloseableIterator; /** * Interface for committing changes to Delta tables, supporting both filesystem-managed and * catalog-managed tables. */ @Experimental public interface Committer { /** * Commits the given {@code finalizedActions} and {@code commitMetadata} to the table. * *

Filesystem-managed tables: Implementations must write the {@code finalizedActions} into a * new Delta JSON file at version {@link CommitMetadata#getVersion()} using atomic file operations * (PUT-if-absent semantics). * *

Catalog-managed tables: Implementations must follow the commit rules and requirements as * dictated by the managing catalog to ensure commit atomicity and consistency. This may involve: * *

    *
  1. Writing the finalized actions into a staged commit file *
  2. Calling catalog commit APIs with the staged commit location (or inline content) and * additional metadata (such as the commit Protocol and Metadata) *
  3. Publishing ratified catalog commits into the Delta log *
* * @param engine the {@link Engine} instance used for committing changes * @param finalizedActions the iterator of finalized actions to be committed * @param commitMetadata the {@link CommitMetadata} associated with this commit, which contains * additional metadata required to commit the finalized actions to the table, such as the * commit version, Delta log path, and more. * @return CommitResponse containing the resultant commit * @throws CommitFailedException if the commit operation fails */ CommitResponse commit( Engine engine, CloseableIterator finalizedActions, CommitMetadata commitMetadata) throws CommitFailedException; } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/commit/PublishFailedException.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.commit; /** Exception thrown when publishing catalog commits to the Delta log fails. */ public class PublishFailedException extends RuntimeException { /** * Constructs a new PublishFailedException with the specified detail message. * * @param message the detail message */ public PublishFailedException(String message) { super(message); } /** * Constructs a new PublishFailedException with the specified detail message and cause. * * @param message the detail message * @param cause the cause of the exception */ public PublishFailedException(String message, Throwable cause) { super(message, cause); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/commit/PublishMetadata.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.commit; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.annotation.Experimental; import io.delta.kernel.internal.files.LogDataUtils; import io.delta.kernel.internal.files.ParsedCatalogCommitData; import io.delta.kernel.internal.lang.ListUtils; import java.util.List; /** Metadata required for publishing catalog commits to the Delta log. */ @Experimental public class PublishMetadata { private final long snapshotVersion; private final String logPath; /** * List of contiguous catalog commits to be published, in ascending order of version number. * *

Must be non-empty and must end with a catalog commit whose version matches {@code * snapshotVersion}. */ private final List ascendingCatalogCommits; public PublishMetadata( long snapshotVersion, String logPath, List ascendingCatalogCommits) { this.snapshotVersion = snapshotVersion; this.logPath = requireNonNull(logPath, "logPath is null"); this.ascendingCatalogCommits = requireNonNull(ascendingCatalogCommits, "ascendingCatalogCommits is null"); validateCommitsNonEmpty(); validateCommitsContiguous(); validateLastCommitMatchesSnapshotVersion(); } /** @return the snapshot version up to which all catalog commits must be published */ public long getSnapshotVersion() { return snapshotVersion; } /** @return the path to the Delta log directory, located at {@code /_delta_log} */ public String getLogPath() { return logPath; } /** * @return the list of contiguous catalog commits to be published, in ascending order of version * number */ public List getAscendingCatalogCommits() { return ascendingCatalogCommits; } private void validateCommitsNonEmpty() { checkArgument(!ascendingCatalogCommits.isEmpty(), "ascendingCatalogCommits must be non-empty"); } private void validateCommitsContiguous() { LogDataUtils.validateLogDataIsSortedContiguous(ascendingCatalogCommits); } private void validateLastCommitMatchesSnapshotVersion() { final long lastCommitVersion = ListUtils.getLast(ascendingCatalogCommits).getVersion(); checkArgument( lastCommitVersion == snapshotVersion, "Last catalog commit version %d must equal snapshot version %d", lastCommitVersion, snapshotVersion); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/data/ArrayValue.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.data; /** Abstraction to represent a single array value in a {@link ColumnVector}. */ public interface ArrayValue { /** The number of elements in the array */ int getSize(); /** * A {@link ColumnVector} containing the array elements with exactly {@link ArrayValue#getSize()} * elements. */ ColumnVector getElements(); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/data/ColumnVector.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.data; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.types.DataType; import java.math.BigDecimal; /** * Represents zero or more values of a single column. * * @since 3.0.0 */ @Evolving public interface ColumnVector extends AutoCloseable { /** @return the data type of this column vector. */ DataType getDataType(); /** @return number of elements in the vector */ int getSize(); /** Cleans up memory for this column vector. The column vector is not usable after this. */ @Override void close(); /** * @param rowId * @return whether the value at {@code rowId} is NULL. */ boolean isNullAt(int rowId); /** * Returns the boolean type value for {@code rowId}. The return value is undefined and can be * anything, if the slot for {@code rowId} is null. * * @param rowId * @return Boolean value at the given row id */ default boolean getBoolean(int rowId) { throw new UnsupportedOperationException("Invalid value request for data type"); } /** * Returns the byte type value for {@code rowId}. The return value is undefined and can be * anything, if the slot for {@code rowId} is null. * * @param rowId * @return Byte value at the given row id */ default byte getByte(int rowId) { throw new UnsupportedOperationException("Invalid value request for data type"); } /** * Returns the short type value for {@code rowId}. The return value is undefined and can be * anything, if the slot for {@code rowId} is null. * * @param rowId * @return Short value at the given row id */ default short getShort(int rowId) { throw new UnsupportedOperationException("Invalid value request for data type"); } /** * Returns the int type value for {@code rowId}. The return value is undefined and can be * anything, if the slot for {@code rowId} is null. * * @param rowId * @return Integer value at the given row id */ default int getInt(int rowId) { throw new UnsupportedOperationException("Invalid value request for data type"); } /** * Returns the long type value for {@code rowId}. The return value is undefined and can be * anything, if the slot for {@code rowId} is null. * * @param rowId * @return Long value at the given row id */ default long getLong(int rowId) { throw new UnsupportedOperationException("Invalid value request for data type"); } /** * Returns the float type value for {@code rowId}. The return value is undefined and can be * anything, if the slot for {@code rowId} is null. * * @param rowId * @return Float value at the given row id */ default float getFloat(int rowId) { throw new UnsupportedOperationException("Invalid value request for data type"); } /** * Returns the double type value for {@code rowId}. The return value is undefined and can be * anything, if the slot for {@code rowId} is null. * * @param rowId * @return Double value at the given row id */ default double getDouble(int rowId) { throw new UnsupportedOperationException("Invalid value request for data type"); } /** * Returns the binary type value for {@code rowId}. The return value is undefined and can be * anything, if the slot for {@code rowId} is null. * * @param rowId * @return Binary value at the given row id */ default byte[] getBinary(int rowId) { throw new UnsupportedOperationException("Invalid value request for data type"); } /** * Returns the string type value for {@code rowId}. The return value is undefined and can be * anything, if the slot for {@code rowId} is null. * * @param rowId * @return String value at the given row id */ default String getString(int rowId) { throw new UnsupportedOperationException("Invalid value request for data type"); } /** * Returns the decimal type value for {@code rowId}. The return value is undefined and can be * anything, if the slot for {@code rowId} is null. * * @param rowId * @return Decimal value at the given row id */ default BigDecimal getDecimal(int rowId) { throw new UnsupportedOperationException("Invalid value request for data type"); } /** * Return the map value located at {@code rowId}. Returns null if the slot for {@code rowId} is * null */ default MapValue getMap(int rowId) { throw new UnsupportedOperationException("Invalid value request for data type"); } /** * Return the array value located at {@code rowId}. Returns null if the slot for {@code rowId} is * null */ default ArrayValue getArray(int rowId) { throw new UnsupportedOperationException("Invalid value request for data type"); } /** * Get the child vector associated with the given ordinal. This method is applicable only to the * {@code struct} type columns. * * @param ordinal Ordinal of the child vector to return. */ default ColumnVector getChild(int ordinal) { throw new UnsupportedOperationException( "Child vectors are not available for vector of type " + getDataType()); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/data/ColumnarBatch.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.data; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.internal.data.ColumnarBatchRow; import io.delta.kernel.types.StructField; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterator; import java.util.NoSuchElementException; /** * Represents zero or more rows of records with same schema type. * * @since 3.0.0 */ @Evolving public interface ColumnarBatch { /** @return the schema of the data in this batch. */ StructType getSchema(); /** * Return the {@link ColumnVector} for the given ordinal in the columnar batch. If the ordinal is * not valid throws error. * * @param ordinal the ordinal of the column to retrieve * @return the {@link ColumnVector} for the given ordinal in the columnar batch */ ColumnVector getColumnVector(int ordinal); /** @return the number of rows/records in the columnar batch */ int getSize(); /** * Return a copy of the {@link ColumnarBatch} with given new column vector inserted at the given * {@code columnVector} at given {@code ordinal}. Shift the existing {@link ColumnVector}s located * at from {@code ordinal} to the end by one position. The schema of the new {@link ColumnarBatch} * will also be changed to reflect the newly inserted vector. * * @param ordinal * @param columnSchema Column name and schema details of the new column vector. * @param columnVector * @return {@link ColumnarBatch} with new vector inserted. * @throws IllegalArgumentException If the ordinal is not valid (ie less than zero or greater than * the current number of vectors). */ default ColumnarBatch withNewColumn( int ordinal, StructField columnSchema, ColumnVector columnVector) { throw new UnsupportedOperationException("Not yet implemented"); } /** * Return a copy of this {@link ColumnarBatch} with the column at given {@code ordinal} removed. * All columns after the {@code ordinal} will be shifted to left by one position. * * @param ordinal Column ordinal to delete. * @return {@link ColumnarBatch} with a column vector deleted. */ default ColumnarBatch withDeletedColumnAt(int ordinal) { throw new UnsupportedOperationException("Not yet implemented"); } /** * Generate a copy of this {@link ColumnarBatch} with the given {@code newSchema}. The data types * of elements in the given new schema and existing schema should be the same. Rest of the details * such as name of the column or column metadata could be different. * * @param newSchema * @return {@link ColumnarBatch} with given new schema. */ default ColumnarBatch withNewSchema(StructType newSchema) { throw new UnsupportedOperationException("Not yet implemented"); } /** @return iterator of {@link Row}s in this batch */ default CloseableIterator getRows() { final ColumnarBatch batch = this; return new CloseableIterator() { int rowId = 0; int maxRowId = getSize(); @Override public boolean hasNext() { return rowId < maxRowId; } @Override public Row next() { if (!hasNext()) { throw new NoSuchElementException(); } Row row = new ColumnarBatchRow(batch, rowId); rowId += 1; return row; } @Override public void close() {} }; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/data/FilteredColumnarBatch.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.data; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.internal.data.ColumnarBatchRow; import io.delta.kernel.utils.CloseableIterator; import java.util.NoSuchElementException; import java.util.Optional; /** * Represents a filtered version of {@link ColumnarBatch}. Contains original {@link ColumnarBatch} * with an optional selection vector to select only a subset of rows for the original columnar * batch. * *

The selection vector is of type boolean and has the same size as the data in the corresponding * {@link ColumnarBatch}. For each row index, a value of true in the selection vector indicates the * row at the same index in the data {@link ColumnarBatch} is valid; a value of false indicates the * row should be ignored. If there is no selection vector then all the rows are valid. * * @since 3.0.0 */ @Evolving public class FilteredColumnarBatch { private final ColumnarBatch data; private final Optional selectionVector; private final Optional filePath; private final Optional preComputedNumSelectedRows; // TODO: use static factory for constructors public FilteredColumnarBatch(ColumnarBatch data, Optional selectionVector) { this.data = data; this.selectionVector = selectionVector; this.filePath = Optional.empty(); this.preComputedNumSelectedRows = !selectionVector.isPresent() ? Optional.of(data.getSize()) : Optional.empty(); } public FilteredColumnarBatch( ColumnarBatch data, Optional selectionVector, String filePath, int preComputedNumSelectedRows) { this.data = data; this.selectionVector = selectionVector; checkArgument( selectionVector.isPresent() || preComputedNumSelectedRows == data.getSize(), "Invalid precomputedNumSelectedRows: must be equal to batch size " + "when selectionVector is empty."); checkArgument( preComputedNumSelectedRows >= 0 && preComputedNumSelectedRows <= data.getSize(), "Invalid precomputedNumSelectedRows: " + "must be no less than 0 and no larger than batch size."); this.filePath = Optional.of(filePath); this.preComputedNumSelectedRows = Optional.of(preComputedNumSelectedRows); } /** * Return the data as {@link ColumnarBatch}. Not all rows in the data are valid for this result. * An optional selectionVector determines which rows are selected. If there is no selection * vector that means all rows in this columnar batch are valid for this result. * * @return all the data read from the file */ public ColumnarBatch getData() { return data; } /** * Returns the file path from which the data originates, if available. * *

Note: The file path may not be present. It is only set if explicitly provided in the * constructor. * * @return an {@link Optional} containing the file path if available, otherwise an empty Optional */ public Optional getFilePath() { return filePath; } /** * Optional selection vector containing one entry for each row in data indicating whether a * row is selected or not selected. If there is no selection vector then all the rows are valid. * * @return an optional {@link ColumnVector} indicating which rows are valid */ public Optional getSelectionVector() { return selectionVector; } /** * Iterator of rows that survived the filter. * * @return Closeable iterator of rows that survived the filter. It is responsibility of the caller * to the close the iterator. */ public CloseableIterator getRows() { if (!selectionVector.isPresent()) { return data.getRows(); } return new CloseableIterator() { private int rowId = 0; private int maxRowId = data.getSize(); private int nextRowId = -1; @Override public boolean hasNext() { for (; rowId < maxRowId && nextRowId == -1; rowId++) { boolean isSelected = !selectionVector.get().isNullAt(rowId) && selectionVector.get().getBoolean(rowId); if (isSelected) { nextRowId = rowId; rowId++; break; } } return nextRowId != -1; } @Override public Row next() { if (!hasNext()) { throw new NoSuchElementException(); } Row row = new ColumnarBatchRow(data, nextRowId); nextRowId = -1; return row; } @Override public void close() {} }; } /** * @return an {@link Optional} containing the pre-computed number of selected rows, if available. *

If present, this value was computed ahead of time and can be used without incurring any * additional cost. This occurs in two cases: *

    *
  • When the selection vector is absent, which implies that all rows are selected — in * this case, the number of selected rows is equal to the batch size. *
  • When the number of selected rows was explicitly pre-computed and passed in. *
*

If empty, the caller must compute the number of selected rows manually from the * selection vector. */ public Optional getPreComputedNumSelectedRows() { return preComputedNumSelectedRows; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/data/MapValue.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.data; /** Abstraction to represent a single map value in a {@link ColumnVector}. */ public interface MapValue { /** The number of elements in the map */ int getSize(); /** * A {@link ColumnVector} containing the keys. There are exactly {@link MapValue#getSize()} keys * in the vector, and each key maps one-to-one to the value at the same index in {@link * MapValue#getValues()}. */ ColumnVector getKeys(); /** * A {@link ColumnVector} containing the values. There are exactly {@link MapValue#getSize()} * values in the vector, and maps one-to-one to the keys in {@link MapValue#getKeys()} */ ColumnVector getValues(); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/data/Row.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.data; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.types.StructType; import java.math.BigDecimal; /** * Represent a single record * * @since 3.0.0 */ @Evolving public interface Row { /** @return Schema of the record. */ StructType getSchema(); /** * @param ordinal the ordinal of the column to check * @return whether the column at {@code ordinal} is null */ boolean isNullAt(int ordinal); /** * Return boolean value of the column located at the given ordinal. Throws error if the column at * given ordinal is not of boolean type, */ boolean getBoolean(int ordinal); /** * Return byte value of the column located at the given ordinal. Throws error if the column at * given ordinal is not of boolean type, */ byte getByte(int ordinal); /** * Return short value of the column located at the given ordinal. Throws error if the column at * given ordinal is not of boolean type, */ short getShort(int ordinal); /** * Return integer value of the column located at the given ordinal. Throws error if the column at * given ordinal is not of integer type, */ int getInt(int ordinal); /** * Return long value of the column located at the given ordinal. Throws error if the column at * given ordinal is not of long type, */ long getLong(int ordinal); /** * Return float value of the column located at the given ordinal. Throws error if the column at * given ordinal is not of long type, */ float getFloat(int ordinal); /** * Return double value of the column located at the given ordinal. Throws error if the column at * given ordinal is not of long type, */ double getDouble(int ordinal); /** * Return string value of the column located at the given ordinal. Throws error if the column at * given ordinal is not of varchar type, */ String getString(int ordinal); /** * Return decimal value of the column located at the given ordinal. Throws error if the column at * given ordinal is not of decimal type, */ BigDecimal getDecimal(int ordinal); /** * Return binary value of the column located at the given ordinal. Throws error if the column at * given ordinal is not of varchar type, */ byte[] getBinary(int ordinal); /** * Return struct value of the column located at the given ordinal. Throws error if the column at * given ordinal is not of struct type, */ Row getStruct(int ordinal); /** * Return array value of the column located at the given ordinal. Throws error if the column at * given ordinal is not of array type, */ ArrayValue getArray(int ordinal); /** * Return map value of the column located at the given ordinal. Throws error if the column at * given ordinal is not of map type, */ MapValue getMap(int ordinal); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/data/package-info.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /** Delta Kernel interfaces for representing data in columnar and row format. */ package io.delta.kernel.data; ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/engine/Engine.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.engine; import io.delta.kernel.annotation.Evolving; import java.util.Collections; import java.util.List; /** * Interface encapsulating all clients needed by the Delta Kernel in order to read the Delta table. * Connectors are expected to pass an implementation of this interface when reading a Delta table. * * @since 3.0.0 */ @Evolving public interface Engine { /** * Get the connector provided {@link ExpressionHandler}. * * @return An implementation of {@link ExpressionHandler}. */ ExpressionHandler getExpressionHandler(); /** * Get the connector provided {@link JsonHandler}. * * @return An implementation of {@link JsonHandler}. */ JsonHandler getJsonHandler(); /** * Get the connector provided {@link FileSystemClient}. * * @return An implementation of {@link FileSystemClient}. */ FileSystemClient getFileSystemClient(); /** * Get the connector provided {@link ParquetHandler}. * * @return An implementation of {@link ParquetHandler}. */ ParquetHandler getParquetHandler(); /** Get the engine's {@link MetricsReporter} instances to push reports to. */ default List getMetricsReporters() { return Collections.emptyList(); }; } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/engine/ExpressionHandler.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.engine; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.expressions.Expression; import io.delta.kernel.expressions.ExpressionEvaluator; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.expressions.PredicateEvaluator; import io.delta.kernel.types.DataType; import io.delta.kernel.types.StructType; /** * Provides expression evaluation capability to Delta Kernel. Delta Kernel can use this client to * evaluate predicate on partition filters, fill up partition column values and any computation on * data using {@link Expression}s. * * @since 3.0.0 */ @Evolving public interface ExpressionHandler { /** * Create an {@link ExpressionEvaluator} that can evaluate the given expression on {@link * ColumnarBatch}s with the given batchSchema. The expression is expected to be a * scalar expression where for each one input row there is a one output value. * * @param inputSchema Input data schema * @param expression Expression to evaluate. * @param outputType Expected result data type. */ ExpressionEvaluator getEvaluator( StructType inputSchema, Expression expression, DataType outputType); /** * Create a {@link PredicateEvaluator} that can evaluate the given predicate expression and * return a selection vector ({@link ColumnVector} of {@code boolean} type). * * @param inputSchema Schema of the data referred by the given predicate expression. * @param predicate Predicate expression to evaluate. * @return */ PredicateEvaluator getPredicateEvaluator(StructType inputSchema, Predicate predicate); /** * Create a selection vector, a boolean type {@link ColumnVector}, on top of the range of values * given in values array. * * @param values Array of initial boolean values for the selection vector. The ownership of this * array is with the caller and this method shouldn't depend on it after the call is complete. * @param from start index of the range, inclusive. * @param to end index of the range, exclusive. * @return A {@link ColumnVector} of {@code boolean} type values. */ ColumnVector createSelectionVector(boolean[] values, int from, int to); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/engine/FileReadRequest.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.engine; import io.delta.kernel.annotation.Evolving; /** Represents a request to read a range of bytes from a given file. */ @Evolving public interface FileReadRequest { /** Get the fully qualified path of the file from which to read the data. */ String getPath(); /** Get the start offset in the file from where to start reading the data. */ int getStartOffset(); /** Get the length of the data to read from the file starting at the startOffset. */ int getReadLength(); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/engine/FileReadResult.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.engine; import io.delta.kernel.data.ColumnarBatch; import java.util.Objects; /** * The result of reading a batch of data in a file. * *

Encapsulates both the data read (as a {@link ColumnarBatch}) and the full path of the file * from which the data was read. */ public class FileReadResult { private final ColumnarBatch data; private final String filePath; /** * Constructs a {@code FileReadResult} object with the given data and file path. * * @param data the columnar batch of data read from the file * @param filePath the path of the file from which the data was read */ public FileReadResult(ColumnarBatch data, String filePath) { this.data = Objects.requireNonNull(data, "data must not be null"); this.filePath = Objects.requireNonNull(filePath, "filePath must not be null"); } /** @return {@link ColumnarBatch} of data that was read from the file. */ public ColumnarBatch getData() { return data; } /** @return the path of the file that this data was read from. */ public String getFilePath() { return filePath; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/engine/FileSystemClient.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.engine; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; import java.io.IOException; /** * Provides file system related functionalities to Delta Kernel. Delta Kernel uses this client * whenever it needs to access the underlying file system where the Delta table is present. * Connector implementation of this interface can hide filesystem specific details from Delta * Kernel. * * @since 3.0.0 */ @Evolving public interface FileSystemClient { /** * List the paths in the same directory that are lexicographically greater or equal to (UTF-8 * sorting) the given `path`. The result should also be sorted by the file name. * * @param filePath Fully qualified path to a file * @return Closeable iterator of files. It is the responsibility of the caller to close the * iterator. * @throws FileNotFoundException if the file at the given path is not found * @throws IOException for any other IO error. */ CloseableIterator listFrom(String filePath) throws IOException; /** * Resolve the given path to a fully qualified path. * * @param path Input path * @return Fully qualified path. * @throws FileNotFoundException If the given path doesn't exist. * @throws IOException for any other IO error. */ String resolvePath(String path) throws IOException; /** * Return an iterator of byte streams one for each read request in {@code readRequests}. The * returned streams are in the same order as the given {@link FileReadRequest}s. It is the * responsibility of the caller to close each returned stream. * * @param readRequests Iterator of read requests * @return Data for each request as one {@link ByteArrayInputStream}. * @throws IOException */ CloseableIterator readFiles(CloseableIterator readRequests) throws IOException; /** * Create a directory at the given path including parent directories. This mimicks the behavior of * `mkdir -p` in Unix. * * @param path Full qualified path to create a directory at. * @return true if the directory was created successfully, false otherwise. * @throws IOException for any IO error. */ boolean mkdirs(String path) throws IOException; /** * Delete the file at given path. * * @param path the path to delete. If path is a directory throws an exception. * @return true if delete is successful else false. * @throws IOException for any IO error. */ boolean delete(String path) throws IOException; /** * Get the metadata of the file at the given path. * * @param path Fully qualified path to the file. * @return Metadata of the file. * @throws IOException for any IO error. */ FileStatus getFileStatus(String path) throws IOException; /** * Atomically copy a file from source path to destination path. The copy operation should be * atomic to ensure that the destination file is either fully copied or not present at all. * * @param srcPath Fully qualified path to the source file to copy * @param destPath Fully qualified path to the destination where the file will be copied * @param overwrite If true, overwrite the destination file if it already exists. If false, throw * an exception if the destination exists. * @throws java.nio.file.FileAlreadyExistsException if the destination file already exists and * {@code overwrite} is false. * @throws FileNotFoundException if the source file does not exist * @throws IOException for any other IO error */ void copyFileAtomically(String srcPath, String destPath, boolean overwrite) throws IOException; } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/engine/JsonHandler.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.engine; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.data.*; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import java.io.IOException; import java.nio.file.FileAlreadyExistsException; import java.util.Optional; /** * Provides JSON handling functionality to Delta Kernel. Delta Kernel can use this client to parse * JSON strings into {@link ColumnarBatch} or read content from JSON files. Connectors can leverage * this interface to provide their best implementation of the JSON parsing capability to Delta * Kernel. * * @since 3.0.0 */ @Evolving public interface JsonHandler { /** * Parse the given json strings and return the fields requested by {@code outputSchema} as * columns in a {@link ColumnarBatch}. * *

There are a couple special cases that should be handled for specific data types: * *

    *
  • FloatType and DoubleType: handle non-numeric numbers encoded as strings *
      *
    • NaN: "NaN" *
    • Positive infinity: "+INF", "Infinity", "+Infinity" *
    • Negative infinity: "-INF", "-Infinity"" *
    *
  • DateType: handle dates encoded as strings in the format "yyyy-MM-dd" *
  • TimestampType: handle timestamps encoded as strings in the format * "yyyy-MM-dd'T'HH:mm:ss.SSSXXX" *
* * @param jsonStringVector String {@link ColumnVector} of valid JSON strings. * @param outputSchema Schema of the data to return from the parsed JSON. If any requested fields * are missing in the JSON string, a null is returned for that particular field in the * returned {@link Row}. The type for each given field is expected to match the type in the * JSON. * @param selectionVector Optional selection vector indicating which rows to parse the JSON. If * present, only the selected rows should be parsed. Unselected rows should be all null in the * returned batch. * @return a {@link ColumnarBatch} of schema {@code outputSchema} with one row for each entry in * {@code jsonStringVector} */ ColumnarBatch parseJson( ColumnVector jsonStringVector, StructType outputSchema, Optional selectionVector); /** * Read and parse the JSON format file at given locations and return the data as a {@link * ColumnarBatch} with the columns requested by {@code physicalSchema}. * * @param fileIter Iterator of files to read data from. * @param physicalSchema Select list of columns to read from the JSON file. * @param predicate Optional predicate which the JSON reader can optionally use to prune rows that * don't satisfy the predicate. Because pruning is optional and may be incomplete, caller is * still responsible apply the predicate on the data returned by this method. * @return an iterator of {@link ColumnarBatch}s containing the data in columnar format. It is the * responsibility of the caller to close the iterator. The data returned is in the same as the * order of files given in {@code scanFileIter} * @throws IOException if an I/O error occurs during the read. */ CloseableIterator readJsonFiles( CloseableIterator fileIter, StructType physicalSchema, Optional predicate) throws IOException; /** * Serialize each {@code Row} in the iterator as JSON and write as a separate line in destination * file. This call either succeeds in creating the file with given contents or no file is created * at all. It won't leave behind a partially written file. * *

Following are the supported data types and their serialization rules. At a high-level, the * JSON serialization is similar to that of {@code jackson} JSON serializer. * *

    *
  • Primitive types: @code boolean, byte, short, int, long, float, double, string} *
  • {@code struct}: any element whose value is null is not written to file *
  • {@code map}: only a {@code map} with {@code string} key type is supported. If an entry * value is {@code null}, it should be written to the file. *
  • {@code array}: {@code null} value elements are written to file *
* * @param filePath Fully qualified destination file path * @param data Iterator of {@link Row} objects where each row should be serialized as JSON and * written as separate line in the destination file. It is the responsibility of the * implementation to close this iterator. * @param overwrite If {@code true}, the file is overwritten if it already exists. If {@code * false} and a file exists {@link FileAlreadyExistsException} is thrown. * @throws FileAlreadyExistsException if the file already exists and {@code overwrite} is false. * @throws IOException if any other I/O error occurs. */ void writeJsonFileAtomically(String filePath, CloseableIterator data, boolean overwrite) throws IOException; } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/engine/MetricsReporter.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.engine; import io.delta.kernel.metrics.MetricsReport; /** Interface for reporting metrics for operations to a Delta table */ public interface MetricsReporter { /** Indicates that an operation is done by reporting a {@link MetricsReport} */ void report(MetricsReport report); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/engine/ParquetHandler.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.engine; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.data.*; import io.delta.kernel.expressions.Column; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.types.MetadataColumnSpec; import io.delta.kernel.types.StructField; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.*; import java.io.IOException; import java.nio.file.FileAlreadyExistsException; import java.util.List; import java.util.Optional; /** * Provides Parquet file related functionalities to Delta Kernel. Connectors can leverage this * interface to provide their own custom implementation of Parquet data file functionalities to * Delta Kernel. * * @since 3.0.0 */ @Evolving public interface ParquetHandler { /** * Read the Parquet format files at the given locations and return the data as a {@link * ColumnarBatch} with the columns requested by {@code physicalSchema}. * *

If {@code physicalSchema} has a {@link StructField} that is a metadata column {@link * StructField#isMetadataColumn()} of type {@link MetadataColumnSpec#ROW_INDEX}, the column must * be populated with the file row index. * *

How does a column in {@code physicalSchema} match to the column in the Parquet file? If the * {@link StructField} has a field id in the {@code metadata} with key `parquet.field.id` the * column is attempted to match by id. If the column is not found by id, the column is matched by * name. When trying to find the column in Parquet by name, first case-sensitive match is used. If * not found then a case-insensitive match is attempted. * * @param fileIter Iterator of files to read data from. * @param physicalSchema Select list of columns to read from the Parquet file. * @param predicate Optional predicate which the Parquet reader can optionally use to prune rows * that don't satisfy the predicate. Because pruning is optional and may be incomplete, caller * is still responsible apply the predicate on the data returned by this method. * @return an iterator of {@link FileReadResult}s containing the data in columnar format along * with metadata. It is the responsibility of the caller to close the iterator. The data * returned is in the same as the order of files given in {@code scanFileIter}. * @throws IOException if an I/O error occurs during the read. */ CloseableIterator readParquetFiles( CloseableIterator fileIter, StructType physicalSchema, Optional predicate) throws IOException; /** * Write the given data batches to a Parquet files. Try to keep the Parquet file size to given * size. If the current file exceeds this size close the current file and start writing to a new * file. * *

* * @param directoryPath Location where the data files should be written. * @param dataIter Iterator of data batches to write. It is the responsibility of the calle to * close the iterator. * @param statsColumns List of columns to collect statistics for. The statistics collection is * optional. If the implementation does not support statistics collection, it is ok to return * no statistics. * @return an iterator of {@link DataFileStatus} containing the status of the written files. Each * status contains the file path and the optionally collected statistics for the file It is * the responsibility of the caller to close the iterator. * @throws IOException if an I/O error occurs during the file writing. This may leave some files * already written in the directory. It is the responsibility of the caller to clean up. * @since 3.2.0 */ CloseableIterator writeParquetFiles( String directoryPath, CloseableIterator dataIter, List statsColumns) throws IOException; /** * Write the given data as a Parquet file. This call either succeeds in creating the file with * given contents or no file is created at all. It won't leave behind a partially written file. * *

* * @param filePath Fully qualified destination file path * @param data Iterator of {@link FilteredColumnarBatch} * @throws FileAlreadyExistsException if the file already exists and {@code overwrite} is false. * @throws IOException if any other I/O error occurs. */ void writeParquetFileAtomically(String filePath, CloseableIterator data) throws IOException; } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/engine/package-info.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /** * Interfaces to allow the connector to bring their own implementation of functions such as reading * parquet files, listing files in a file system, parsing a JSON string etc. to Delta Kernel. */ package io.delta.kernel.engine; ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/exceptions/CheckpointAlreadyExistsException.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.exceptions; import static java.lang.String.format; import io.delta.kernel.annotation.Evolving; /** * Thrown when trying to create a checkpoint at version {@code v}, but there already exists a * checkpoint at version {@code v}. * * @since 3.2.0 */ @Evolving public class CheckpointAlreadyExistsException extends KernelException { public CheckpointAlreadyExistsException(long version) { super(format("Checkpoint for given version %d already exists in the table", version)); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/exceptions/CommitRangeNotFoundException.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.exceptions; import io.delta.kernel.annotation.Evolving; import java.util.Optional; /** * Exception thrown when Kernel cannot find any commit files in the requested version range. This * can happen when the requested versions don't exist in the table. * * @since 4.1.0 */ @Evolving public class CommitRangeNotFoundException extends KernelException { private final String tablePath; private final long startVersion; private final Optional endVersion; public CommitRangeNotFoundException( String tablePath, long startVersion, Optional endVersion) { super( String.format( "%s: Requested table changes between [%s, %s] but no log files found in the requested" + " version range.", tablePath, startVersion, endVersion)); this.tablePath = tablePath; this.startVersion = startVersion; this.endVersion = endVersion; } /** @return the table path where the commit range was not found */ public String getTablePath() { return tablePath; } /** @return the start version of the requested commit range */ public long getStartVersion() { return startVersion; } /** * @return the end version of the requested commit range, or empty if no end version was specified */ public Optional getEndVersion() { return endVersion; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/exceptions/CommitStateUnknownException.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.exceptions; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.commit.CommitFailedException; /** * Exception thrown when the Delta transaction commit system cannot determine whether a previous * commit attempt succeeded or failed, making it unsafe to continue with automatic retries. * *

This exception occurs in a specific sequence during transaction commit retries: * *

    *
  1. First commit attempt fails with a retryable, non-conflict exception (e.g., IOException) *
  2. Second commit attempt fails with a conflict exception (e.g., FileAlreadyExistsException) *
* *

In this scenario, the system cannot determine whether the first attempt actually wrote the * commit file successfully but failed to report success, or whether the commit file was never * written. * *

Since the system cannot distinguish between these cases, it cannot safely determine whether to * retry at the current version or advance to version N+1. * *

Resolution: When this exception occurs, manual intervention is required to: * *

    *
  • Examine the commit history to determine if the first attempt actually succeeded *
  • If the commit succeeded, avoid retrying to prevent duplicate records *
  • If the commit failed, retry the operation from the beginning *
*/ @Evolving public class CommitStateUnknownException extends RuntimeException { public CommitStateUnknownException( long commitVersion, int commitAttempt, CommitFailedException cfe) { super( String.format( "Commit attempt %d for version %d failed due to a concurrent write conflict after a " + "previous retry. Since Kernel cannot determine if that previous attempt actually " + "succeeded, retrying could create duplicate records. Please manually validate " + "the commit history to resolve this conflict and retry the operation.", commitAttempt, commitVersion), cfe); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/exceptions/ConcurrentTransactionException.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.exceptions; import io.delta.kernel.TransactionBuilder; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.engine.Engine; /** * Thrown when concurrent transaction both attempt to update the table with same transaction * identifier set through {@link TransactionBuilder#withTransactionId(Engine, String, long)} * (String)}. * *

Incremental processing systems (e.g., streaming systems) that track progress using their own * application-specific versions need to record what progress has been made, in order to avoid * duplicating data in the face of failures and retries during writes. For more information refer to * the Delta protocol section * Transaction Identifiers * * @since 3.2.0 */ @Evolving public class ConcurrentTransactionException extends ConcurrentWriteException { private static final String message = "This error occurs when multiple updates are " + "using the same transaction identifier to write into this table.\n" + "Application ID: %s, Attempted version: %s, Latest version in table: %s"; public ConcurrentTransactionException(String appId, long txnVersion, long lastUpdated) { super(String.format(message, appId, txnVersion, lastUpdated)); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/exceptions/ConcurrentWriteException.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.exceptions; import io.delta.kernel.annotation.Evolving; /** * Thrown when a concurrent transaction has written data after the current transaction has started. * * @since 3.2.0 */ @Evolving public class ConcurrentWriteException extends KernelException { public ConcurrentWriteException() { super( "Transaction has encountered a conflict and can not be committed. " + "Query needs to be re-executed using the latest version of the table."); } public ConcurrentWriteException(String message) { super(message); } public ConcurrentWriteException(String message, Throwable cause) { super(message, cause); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/exceptions/DomainDoesNotExistException.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.exceptions; import io.delta.kernel.annotation.Evolving; /** Thrown when attempting to remove a domain metadata that does not exist in the read snapshot. */ @Evolving public class DomainDoesNotExistException extends KernelException { public DomainDoesNotExistException(String tablePath, String domain, long snapshotVersion) { super( String.format( "%s: Cannot remove domain metadata with identifier %s because it does not exist in the " + "read snapshot at version %s", tablePath, domain, snapshotVersion)); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/exceptions/InvalidConfigurationValueException.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.exceptions; import io.delta.kernel.annotation.Evolving; /** * Thrown when an illegal value is specified for a table property. * * @since 3.3.0 */ @Evolving public class InvalidConfigurationValueException extends KernelException { public InvalidConfigurationValueException(String key, String value, String helpMessage) { super( String.format("Invalid value for table property '%s': '%s'. %s", key, value, helpMessage)); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/exceptions/InvalidTableException.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.exceptions; /** * Thrown when an invalid table is encountered; the table's log and/or checkpoint files are in an * invalid state. */ public class InvalidTableException extends KernelException { private static final String message = "Invalid table found at %s: %s"; public InvalidTableException(String tablePath, String reason) { super(String.format(message, tablePath, reason)); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/exceptions/KernelEngineException.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.exceptions; import io.delta.kernel.engine.Engine; /** Throws when the {@link Engine} encountered an error while executing an operation. */ public class KernelEngineException extends RuntimeException { private static final String msgTemplate = "Encountered an error from the underlying engine " + "implementation while trying to %s: %s"; public KernelEngineException(String attemptedOperation, Throwable cause) { super(String.format(msgTemplate, attemptedOperation, cause.getMessage()), cause); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/exceptions/KernelException.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.exceptions; /** * Thrown when Kernel cannot execute the requested operation due to the operation being invalid or * unsupported. */ public class KernelException extends RuntimeException { public KernelException() { super(); } public KernelException(String message) { super(message); } public KernelException(Throwable cause) { super(cause); } public KernelException(String message, Throwable cause) { super(message, cause); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/exceptions/MaxCommitRetryLimitReachedException.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.exceptions; import io.delta.kernel.annotation.Evolving; @Evolving public class MaxCommitRetryLimitReachedException extends KernelException { public MaxCommitRetryLimitReachedException(long commitVersion, int maxRetries, Exception cause) { super( String.format( "Commit attempt for version %d failed with a retryable exception but will not be " + "retried because the maximum number of retries (%d) has been reached.", commitVersion, maxRetries), cause); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/exceptions/MetadataChangedException.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.exceptions; import io.delta.kernel.annotation.Evolving; /** * Thrown when the metadata of the Delta table has changed between the time of transaction start and * the time of commit. * * @since 3.2.0 */ @Evolving public class MetadataChangedException extends ConcurrentWriteException { public MetadataChangedException() { super( "The metadata of the Delta table has been changed by a concurrent update. " + "Please try the operation again."); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/exceptions/ProtocolChangedException.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.exceptions; import io.delta.kernel.annotation.Evolving; /** * Thrown when the protocol of the Delta table has changed between the time of transaction start and * the time of commit. * * @since 3.2.0 */ @Evolving public class ProtocolChangedException extends ConcurrentWriteException { private static final String helpfulMsgForNewTables = " This happens when multiple writers " + "are writing to an empty directory. Creating the table ahead of time will avoid this " + "conflict."; public ProtocolChangedException(long attemptVersion) { super( String.format( "Transaction has encountered a conflict and can not be committed. " + "Query needs to be re-executed using the latest version of the table.%s", attemptVersion == 0 ? helpfulMsgForNewTables : "")); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/exceptions/TableAlreadyExistsException.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.exceptions; import io.delta.kernel.annotation.Evolving; import java.util.Optional; /** * Thrown when trying to create a Delta table at a location where a Delta table already exists. * * @since 3.2.0 */ @Evolving public class TableAlreadyExistsException extends KernelException { private final String tablePath; private final Optional context; public TableAlreadyExistsException(String tablePath, String context) { this.tablePath = tablePath; this.context = Optional.ofNullable(context); } public TableAlreadyExistsException(String tablePath) { this(tablePath, null); } @Override public String getMessage() { return String.format( "Delta table already exists at `%s`.%s", tablePath, context.map(c -> " Context: " + c).orElse("")); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/exceptions/TableNotFoundException.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.exceptions; import io.delta.kernel.annotation.Evolving; /** * Thrown when there is no Delta table at the given location. * * @since 3.0.0 */ @Evolving public class TableNotFoundException extends KernelException { private final String tablePath; public TableNotFoundException(String tablePath) { this(tablePath, null); } public TableNotFoundException(String tablePath, String context) { super( String.format( "Delta table at path `%s` is not found.%s", tablePath, context == null ? "" : " Context: " + context)); this.tablePath = tablePath; } /** @return the provided path where no Delta table was found */ public String getTablePath() { return tablePath; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/exceptions/UnknownConfigurationException.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.exceptions; import io.delta.kernel.annotation.Evolving; /** * Thrown when an unknown configuration key is specified. * * @since 3.3.0 */ @Evolving public class UnknownConfigurationException extends KernelException { public UnknownConfigurationException(String confKey) { super(String.format("Unknown configuration was specified: %s", confKey)); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/exceptions/UnsupportedProtocolVersionException.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.exceptions; import io.delta.kernel.annotation.Evolving; /** * Exception thrown when Kernel encounters unsupported protocol versions. * * @since 4.1.0 */ @Evolving public class UnsupportedProtocolVersionException extends KernelException { /** Enum representing the type of Delta protocol version. */ public enum ProtocolVersionType { /** Reader protocol version */ READER, /** Writer protocol version */ WRITER } private final String tablePath; private final int version; private final ProtocolVersionType versionType; public UnsupportedProtocolVersionException( String tablePath, int version, ProtocolVersionType versionType) { super( String.format( "Unsupported Delta protocol %s version: table `%s` requires %s version %s " + "which is unsupported by this version of Delta Kernel.", versionType.name().toLowerCase(), tablePath, versionType.name().toLowerCase(), version)); this.tablePath = tablePath; this.version = version; this.versionType = versionType; } /** @return the table path where the unsupported protocol was encountered */ public String getTablePath() { return tablePath; } /** @return the unsupported protocol version */ public int getVersion() { return version; } /** @return the type of protocol version (READER or WRITER) */ public ProtocolVersionType getVersionType() { return versionType; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/exceptions/UnsupportedTableFeatureException.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.exceptions; import io.delta.kernel.annotation.Evolving; import java.util.Collections; import java.util.Set; /** * Base exception thrown when Kernel encounters unsupported table features. * * @since 4.1.0 */ @Evolving public class UnsupportedTableFeatureException extends KernelException { private final String tablePath; private final Set unsupportedFeatures; public UnsupportedTableFeatureException( String tablePath, Set unsupportedFeatures, String message) { super(message); this.tablePath = tablePath; this.unsupportedFeatures = unsupportedFeatures != null ? Collections.unmodifiableSet(unsupportedFeatures) : Collections.emptySet(); } public UnsupportedTableFeatureException( String tablePath, String unsupportedFeature, String message) { this(tablePath, Collections.singleton(unsupportedFeature), message); } /** * @return the table path where the unsupported features were encountered, or null if not * applicable */ public String getTablePath() { return tablePath; } /** @return an unmodifiable set of unsupported feature names */ public Set getUnsupportedFeatures() { return unsupportedFeatures; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/expressions/AlwaysFalse.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.expressions; import io.delta.kernel.annotation.Evolving; import java.util.Collections; /** * Predicate which always evaluates to {@code false}. * * @since 3.0.0 */ @Evolving public final class AlwaysFalse extends Predicate { public static final AlwaysFalse ALWAYS_FALSE = new AlwaysFalse(); private AlwaysFalse() { super("ALWAYS_FALSE", Collections.emptyList()); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/expressions/AlwaysTrue.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.expressions; import io.delta.kernel.annotation.Evolving; import java.util.Collections; /** * Predicate which always evaluates to {@code true}. * * @since 3.0.0 */ @Evolving public final class AlwaysTrue extends Predicate { public static final AlwaysTrue ALWAYS_TRUE = new AlwaysTrue(); private AlwaysTrue() { super("ALWAYS_TRUE", Collections.emptyList()); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/expressions/And.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.expressions; import io.delta.kernel.annotation.Evolving; import java.util.Arrays; /** * {@code AND} expression * *

Definition: * *

* *

    *
  • Logical {@code expr1} AND {@code expr2} on two inputs. *
  • Requires both left and right input expressions of type {@link Predicate}. *
  • Result is null when both inputs are null, or when one input is null and the other is {@code * true}. *
* * @since 3.0.0 */ @Evolving public final class And extends Predicate { public And(Predicate left, Predicate right) { super("AND", Arrays.asList(left, right)); } /** @return Left side operand. */ public Predicate getLeft() { return (Predicate) getChildren().get(0); } /** @return Right side operand. */ public Predicate getRight() { return (Predicate) getChildren().get(1); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/expressions/Column.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.expressions; import static java.lang.String.format; import io.delta.kernel.annotation.Evolving; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; /** * An expression type that refers to a column (case-sensitive) in the input. The column name is * either a single name or array of names (when referring to a nested column). * * @since 3.0.0 */ @Evolving public final class Column implements Expression { private final String[] names; /** Create a column expression for referring to a column. */ public Column(String name) { this.names = new String[] {name}; } /** Create a column expression to refer to a nested column. */ public Column(String[] names) { this.names = names; } /** * @return the column names. Each part in the name correspond to one level of nested reference. */ public String[] getNames() { return names; } @Override public List getChildren() { return Collections.emptyList(); } @Override public String toString() { return "column(" + quoteColumnPath(names) + ")"; } /** * Returns a new column that appends the input column name to the current column. Corresponds to * an additional level of nested reference. * * @param name the column name to append * @return the new column */ public Column appendNestedField(String name) { String[] newNames = new String[names.length + 1]; System.arraycopy(names, 0, newNames, 0, names.length); newNames[names.length] = name; return new Column(newNames); } private static String quoteColumnPath(String[] names) { return Arrays.stream(names) .map(s -> format("`%s`", s.replace("`", "``"))) .collect(Collectors.joining(".")); } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } Column other = (Column) o; return Arrays.equals(names, other.getNames()); } @Override public int hashCode() { return Arrays.hashCode(names); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/expressions/Expression.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.expressions; import io.delta.kernel.annotation.Evolving; import java.util.List; /** * Base interface for all Kernel expressions. * * @since 3.0.0 */ @Evolving public interface Expression { /** @return a list of expressions that are input to this expression. */ List getChildren(); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/expressions/ExpressionEvaluator.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.expressions; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; /** * Interface for implementing an {@link Expression} evaluator. It contains one {@link Expression} * which can be evaluated on multiple {@link ColumnarBatch}es Connectors can implement this * interface to optimize the evaluation using the connector specific capabilities. * * @since 3.0.0 */ @Evolving public interface ExpressionEvaluator extends AutoCloseable { /** * Evaluate the expression on given {@link ColumnarBatch} data. * * @param input input data in columnar format. * @return Result of the expression as a {@link ColumnVector}. Contains one value for each row of * the input. The data type of the output is same as the type output of the expression this * evaluator is using. */ ColumnVector eval(ColumnarBatch input); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/expressions/In.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.expressions; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.types.CollationIdentifier; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; /** * {@code IN} expression * *

Definition: * *

    *
  • SQL semantic: {@code expr IN (expr1, expr2, ...) [COLLATE collationIdentifier]} *
  • Requires the value expression to be evaluated against a list of literal expressions. *
  • Result is true if the value matches any element in the list, false if no matches and no * nulls, null if the value is null or any comparison results in null. *
  • Supports collation for string comparisons. *
  • Only supports primitive types . Nested types are not supported. *
* * @since 4.0.0 */ @Evolving public final class In extends Predicate { /** * Creates an IN predicate expression. * * @param valueExpression The expression to evaluate (left side of IN) * @param inListElements The list of literal expressions to check against */ public In(Expression valueExpression, List inListElements) { super("IN", buildChildren(valueExpression, inListElements)); } /** * Creates an IN predicate expression with collation support. * * @param valueExpression The expression to evaluate (left side of IN) * @param inListElements The list of literal expressions to check against * @param collationIdentifier The collation identifier for string comparisons */ public In( Expression valueExpression, List inListElements, CollationIdentifier collationIdentifier) { super("IN", buildChildren(valueExpression, inListElements), collationIdentifier); } /** @return The value expression to be evaluated (left side of IN). */ public Expression getValueExpression() { return getChildren().get(0); } /** @return The list of expressions to check against (right side of IN). */ public List getInListElements() { return Collections.unmodifiableList( new ArrayList<>(getChildren().subList(1, getChildren().size()))); } @Override public String toString() { String collationSuffix = getCollationIdentifier().map(c -> " COLLATE " + c).orElse(""); String inValues = getInListElements().stream().map(Object::toString).collect(Collectors.joining(", ")); return String.format("(%s IN (%s)%s)", getValueExpression(), inValues, collationSuffix); } private static List buildChildren( Expression valueExpression, List inListElements) { List children = new ArrayList<>(); children.add(valueExpression); children.addAll(inListElements); return children; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/expressions/Literal.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.expressions; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.types.*; import java.math.BigDecimal; import java.util.Collections; import java.util.List; import java.util.Objects; /** * A literal value. * *

Definition: * *

    *
  • Represents literal of primitive types as defined in the protocol Delta * Transaction Log Protocol: Primitive Types *
  • Use {@link #getValue()} to fetch the literal value. Returned value type depends on the type * of the literal data type. See the {@link #getValue()} for further details. *
* * @since 3.0.0 */ @Evolving public final class Literal implements Expression { /** * Create a {@code boolean} type literal expression. * * @param value literal value * @return a {@link Literal} of type {@link BooleanType} */ public static Literal ofBoolean(boolean value) { return new Literal(value, BooleanType.BOOLEAN); } /** * Create a {@code byte} type literal expression. * * @param value literal value * @return a {@link Literal} of type {@link ByteType} */ public static Literal ofByte(byte value) { return new Literal(value, ByteType.BYTE); } /** * Create a {@code short} type literal expression. * * @param value literal value * @return a {@link Literal} of type {@link ShortType} */ public static Literal ofShort(short value) { return new Literal(value, ShortType.SHORT); } /** * Create a {@code integer} type literal expression. * * @param value literal value * @return a {@link Literal} of type {@link IntegerType} */ public static Literal ofInt(int value) { return new Literal(value, IntegerType.INTEGER); } /** * Create a {@code long} type literal expression. * * @param value literal value * @return a {@link Literal} of type {@link LongType} */ public static Literal ofLong(long value) { return new Literal(value, LongType.LONG); } /** * Create a {@code float} type literal expression. * * @param value literal value * @return a {@link Literal} of type {@link FloatType} */ public static Literal ofFloat(float value) { return new Literal(value, FloatType.FLOAT); } /** * Create a {@code double} type literal expression. * * @param value literal value * @return a {@link Literal} of type {@link DoubleType} */ public static Literal ofDouble(double value) { return new Literal(value, DoubleType.DOUBLE); } /** * Create a {@code string} type literal expression. * * @param value literal value * @return a {@link Literal} of type {@link StringType} */ public static Literal ofString(String value) { return new Literal(value, StringType.STRING); } /** * Create a {@code string} type literal expression with collated {@link StringType}. * * @param value literal value * @param collationIdentifier collation identifier for the string literal * @return a {@link Literal} of type {@link StringType} with the given collation */ public static Literal ofString(String value, CollationIdentifier collationIdentifier) { return new Literal(value, new StringType(collationIdentifier)); } /** * Create a {@code binary} type literal expression. * * @param value binary literal value as an array of bytes * @return a {@link Literal} of type {@link BinaryType} */ public static Literal ofBinary(byte[] value) { return new Literal(value, BinaryType.BINARY); } /** * Create a {@code date} type literal expression. * * @param daysSinceEpochUTC number of days since the epoch in UTC timezone. * @return a {@link Literal} of type {@link DateType} */ public static Literal ofDate(int daysSinceEpochUTC) { return new Literal(daysSinceEpochUTC, DateType.DATE); } /** * Create a {@code timestamp} type literal expression. * * @param microsSinceEpochUTC microseconds since epoch time in UTC timezone. * @return a {@link Literal} with data type {@link TimestampType} */ public static Literal ofTimestamp(long microsSinceEpochUTC) { return new Literal(microsSinceEpochUTC, TimestampType.TIMESTAMP); } /** * Create a {@code timestamp_ntz} type literal expression. * * @param microSecondsEpoch Microseconds since epoch with no timezone. * @return a {@link Literal} with data type {@link TimestampNTZType} */ public static Literal ofTimestampNtz(long microSecondsEpoch) { return new Literal(microSecondsEpoch, TimestampNTZType.TIMESTAMP_NTZ); } /** * Create a {@code decimal} type literal expression. * * @param value decimal literal value * @param precision precision of the decimal literal * @param scale scale of the decimal literal * @return a {@link Literal} with data type {@link DecimalType} with given {@code precision} and * {@code scale}. */ public static Literal ofDecimal(BigDecimal value, int precision, int scale) { // throws an error if rounding is required to set the specified scale BigDecimal valueToStore = value.setScale(scale); checkArgument( valueToStore.precision() <= precision, "Decimal precision=%s for decimal %s exceeds max precision %s", valueToStore.precision(), valueToStore, precision); return new Literal(valueToStore, new DecimalType(precision, scale)); } /** * Create {@code null} value literal. * * @param dataType {@link DataType} of the null literal. * @return a null {@link Literal} with the given data type */ public static Literal ofNull(DataType dataType) { return new Literal(null, dataType); } private final Object value; private final DataType dataType; private Literal(Object value, DataType dataType) { if (dataType instanceof ArrayType || dataType instanceof MapType || dataType instanceof StructType) { throw new IllegalArgumentException(dataType + " is an invalid data type for Literal."); } this.value = value; this.dataType = dataType; } /** * Get the literal value. If the value is null a {@code null} is returned. For non-null literal * the returned value is one of the following types based on the literal data type. * *
    *
  • BOOLEAN: {@link Boolean} *
  • BYTE: {@link Byte} *
  • SHORT: {@link Short} *
  • INTEGER: {@link Integer} *
  • LONG: {@link Long} *
  • FLOAT: {@link Float} *
  • DOUBLE: {@link Double} *
  • DATE: {@link Integer} represents the number of days since epoch in UTC *
  • TIMESTAMP: {@link Long} represents the microseconds since epoch in UTC *
  • TIMESTAMP_NTZ: {@link Long} represents the microseconds since epoch with no timezone *
  • DECIMAL: {@link BigDecimal}.Use {@link #getDataType()} to find the precision and scale *
* * @return Literal value. */ public Object getValue() { return value; } /** * Get the datatype of the literal object. Datatype lets the caller interpret the value of the * literal object returned by {@link #getValue()} * * @return Datatype of the literal object. */ public DataType getDataType() { return dataType; } @Override public String toString() { return String.valueOf(value); } @Override public List getChildren() { return Collections.emptyList(); } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } Literal other = (Literal) o; return Objects.equals(dataType, other.dataType) && Objects.equals(value, other.value); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/expressions/Or.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.expressions; import io.delta.kernel.annotation.Evolving; import java.util.Arrays; /** * {@code OR} expression * *

Definition: * *

* *

    *
  • Logical {@code expr1} OR {@code expr2} on two inputs. *
  • Requires both left and right input expressions of type {@link Predicate}. *
  • Result is null when both inputs are null, or when one input is null and the other is {@code * false}. *
* * @since 3.0.0 */ @Evolving public final class Or extends Predicate { public Or(Predicate left, Predicate right) { super("OR", Arrays.asList(left, right)); } /** @return Left side operand. */ public Predicate getLeft() { return (Predicate) getChildren().get(0); } /** @return Right side operand. */ public Predicate getRight() { return (Predicate) getChildren().get(1); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/expressions/PartitionValueExpression.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.expressions; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.types.DataType; import java.util.Collections; import java.util.List; /** * Expression to decode the serialized partition value into partition type value according the * Delta Protocol spec. Currently all valid partition types are supported except the `timestamp` * and `timestamp without timezone` types. * *

* *

    *
  • Name: partition_value *
  • Semantic: partition_value(string, datatype). Decode the partition value of * type datatype from the serialized string format. *
* * @since 3.0.0 */ @Evolving public class PartitionValueExpression implements Expression { private final DataType partitionValueType; private final Expression serializedPartitionValue; /** * Create {@code partition_value} expression. * * @param serializedPartitionValue Input expression providing the partition values in serialized * format. * @param partitionDataType Partition data type to which string partition value is deserialized as * according to the Delta Protocol. */ public PartitionValueExpression(Expression serializedPartitionValue, DataType partitionDataType) { this.serializedPartitionValue = requireNonNull(serializedPartitionValue); this.partitionValueType = requireNonNull(partitionDataType); } /** Get the expression reference to the serialized partition value. */ public Expression getInput() { return serializedPartitionValue; } /** Get the data type of the partition value. */ public DataType getDataType() { return partitionValueType; } @Override public List getChildren() { return Collections.singletonList(serializedPartitionValue); } @Override public String toString() { return format("partition_value(%s, %s)", serializedPartitionValue, partitionValueType); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/expressions/Predicate.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.expressions; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.engine.ExpressionHandler; import io.delta.kernel.types.CollationIdentifier; import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; /** * Defines predicate scalar expression which is an extension of {@link ScalarExpression} that * evaluates to true, false, or null for each input row. * *

Currently, implementations of {@link ExpressionHandler} requires support for at least the * following scalar expressions. * *

    *
  1. Name: = *
      *
    • SQL semantic: expr1 = expr2 [COLLATE collationIdentifier] *
    • Since version: 3.0.0 *
    *
  2. Name: < *
      *
    • SQL semantic: expr1 < expr2 [COLLATE collationIdentifier] *
    • Since version: 3.0.0 *
    *
  3. Name: <= *
      *
    • SQL semantic: expr1 <= expr2 [COLLATE collationIdentifier] *
    • Since version: 3.0.0 *
    *
  4. Name: > *
      *
    • SQL semantic: expr1 > expr2 [COLLATE collationIdentifier] *
    • Since version: 3.0.0 *
    *
  5. Name: >= *
      *
    • SQL semantic: expr1 >= expr2 [COLLATE collationIdentifier] *
    • Since version: 3.0.0 *
    *
  6. Name: ALWAYS_TRUE *
      *
    • SQL semantic: Constant expression whose value is `true` *
    • Since version: 3.0.0 *
    *
  7. Name: ALWAYS_FALSE *
      *
    • SQL semantic: Constant expression whose value is `false` *
    • Since version: 3.0.0 *
    *
  8. Name: AND *
      *
    • SQL semantic: expr1 AND expr2 *
    • Since version: 3.0.0 *
    *
  9. Name: OR *
      *
    • SQL semantic: expr1 OR expr2 *
    • Since version: 3.0.0 *
    *
  10. Name: NOT *
      *
    • SQL semantic: NOT expr *
    • Since version: 3.1.0 *
    *
  11. Name: IS_NOT_NULL *
      *
    • SQL semantic: expr IS NOT NULL *
    • Since version: 3.1.0 *
    *
  12. Name: IS_NULL *
      *
    • SQL semantic: expr IS NULL *
    • Since version: 3.2.0 *
    *
  13. Name: LIKE *
      *
    • SQL semantic: expr LIKE expr *
    • Since version: 3.3.0 *
    *
  14. Name: IS NOT DISTINCT FROM *
      *
    • SQL semantic: expr1 IS NOT DISTINCT FROM expr2 [COLLATE collationIdentifier] * *
    • Since version: 3.3.0 *
    *
  15. Name: STARTS_WITH *
      *
    • SQL semantic: expr STARTS_WITH expr [COLLATE collationIdentifier] *
    • Since version: 3.4.0 *
    *
  16. Name: IN *
      *
    • SQL semantic: expr IN (expr1, expr2, ...) [COLLATE collationIdentifier] *
    • Since version: 4.0.0 *
    *
* * @since 3.0.0 */ @Evolving public class Predicate extends ScalarExpression { /** Optional collation to be used for string comparison in this predicate. */ private Optional collationIdentifier; public Predicate(String name, List children) { super(name, children); collationIdentifier = Optional.empty(); } /** Constructor for a unary Predicate expression */ public Predicate(String name, Expression child) { this(name, Arrays.asList(child)); } /** Constructor for a binary Predicate expression */ public Predicate(String name, Expression left, Expression right) { this(name, Arrays.asList(left, right)); } /** Constructor for a Predicate expression with collation support. */ public Predicate( String name, Expression left, Expression right, CollationIdentifier collationIdentifier) { this(name, Arrays.asList(left, right), collationIdentifier); } /** Constructor for a Predicate expression with collation support. */ public Predicate( String name, List children, CollationIdentifier collationIdentifier) { this(name, children); checkArgument( COLLATION_SUPPORTED_OPERATORS.contains(this.name), "Collation is not supported for operator %s. Supported operators are %s", this.name, COLLATION_SUPPORTED_OPERATORS); // For operators with multiple children, we need at least 2 children // For other collated expressions, we require exactly 2 children if (OPERATORS_WITH_MULTIPLE_CHILDREN.contains(this.name)) { checkArgument( this.children.size() >= 2, "Invalid Predicate: collated predicate '%s' with multiple children requires at least 2 " + "children, but found %d.", this.name, this.children.size()); } else { checkArgument( this.children.size() == 2, "Invalid Predicate: collated predicate '%s' requires exactly 2 children, but found %d.", this.name, this.children.size()); } this.collationIdentifier = Optional.of(collationIdentifier); } /** Returns the collation identifier used for this predicate, if specified. */ public Optional getCollationIdentifier() { return collationIdentifier; } /** * Returns string representation of the predicate. * *

Format for binary operators: {@code (left OP right)} or {@code (left OP right COLLATE * collation)} * *

Examples: * *

    *
  • {@code (col = 5)} *
  • {@code (name = 'John' COLLATE SPARK.UTF8_BINARY)} *
* *

Note: Specialized operators like IN override this method to provide their own string * representation. */ @Override public String toString() { String collationSuffix = collationIdentifier.map(c -> " COLLATE " + c).orElse(""); if (BINARY_OPERATORS.contains(name) || collationIdentifier.isPresent()) { return String.format("(%s %s %s%s)", children.get(0), name, children.get(1), collationSuffix); } return super.toString(); } @Override public int hashCode() { return toString().hashCode(); } @Override public boolean equals(Object o) { if (this == o) return true; if (!(o instanceof Predicate)) return false; return this.hashCode() == o.hashCode(); } private static final Set BINARY_OPERATORS = Stream.of("<", "<=", ">", ">=", "=", "AND", "OR", "IS NOT DISTINCT FROM", "STARTS_WITH") .collect(Collectors.toSet()); /** Operators that can have multiple children (more than 2). */ private static final Set OPERATORS_WITH_MULTIPLE_CHILDREN = Collections.singleton("IN"); /** Operators that support collation-based string comparison. */ private static final Set COLLATION_SUPPORTED_OPERATORS = Stream.of("<", "<=", ">", ">=", "=", "IS NOT DISTINCT FROM", "STARTS_WITH", "IN") .collect(Collectors.toSet()); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/expressions/PredicateEvaluator.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.expressions; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import java.util.Optional; /** * Special interface for evaluating {@link Predicate} on input batch and return a selection vector * containing one value for each row in input batch indicating whether the row has passed the * predicate or not. * *

Optionally it takes an existing selection vector along with the input batch for evaluation. * Result selection vector is combined with the given existing selection vector and a new selection * vector is returned. This mechanism allows running an input batch through several predicate * evaluations without rewriting the input batch to remove rows that do not pass the predicate after * each predicate evaluation. The new selection should be same or more selective as the existing * selection vector. For example if a row is marked as unselected in existing selection vector, then * it should remain unselected in the returned selection vector even when the given predicate * returns true for the row. * * @since 3.0.0 */ @Evolving public interface PredicateEvaluator { /** * Evaluate the predicate on given inputData. Combine the existing selection vector with the * output of the predicate result and return a new selection vector. * * @param inputData {@link ColumnarBatch} of data to which the predicate expression refers for * input. * @param existingSelectionVector Optional existing selection vector. If not empty, it is combined * with the predicate result. The caller is also releasing the ownership of * `existingSelectionVector` to this callee, and the callee is responsible for closing it. * @return A {@link ColumnVector} of boolean type that captures the predicate result for each row * together with the existing selection vector. */ ColumnVector eval(ColumnarBatch inputData, Optional existingSelectionVector); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/expressions/ScalarExpression.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.expressions; import static java.util.Objects.requireNonNull; import io.delta.kernel.annotation.Evolving; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.stream.Collectors; /** * Scalar SQL expressions which take zero or more inputs and for each input row generate one output * value. A subclass of these expressions are of type {@link Predicate} whose result type is * `boolean`. See {@link Predicate} for predicate type scalar expressions. Supported non-predicate * type scalar expressions are listed below. * *

    *
  1. Name: ELEMENT_AT *
      *
    • Semantic: ELEMENT_AT(map, key). Return the value of given key * from the map type input. Returns null if the given key is not in * the map Ex: `ELEMENT_AT(map(1, 'a', 2, 'b'), 2)` returns 'b'. *
    • Since version: 3.0.0 *
    *
  2. Name: COALESCE *
      *
    • Semantic: COALESCE(expr1, ..., exprN). Return the first non-null * argument. If all arguments are null, returns null. *
    • Since version: 3.1.0 *
    *
  3. Name: ADD *
      *
    • Semantic: ADD(expr1, expr2). Return the sum of two numeric expressions. * If either of the expressions is null, returns null. *
    • Since Version: 4.1.0 *
    *
  4. Name: TIMEADD *
      *
    • Semantic: TIMEADD(colExpr, milliseconds). Add the specified number of * milliseconds to the timestamp represented by colExpr. The adjustment does not * alter the original value but returns a new timestamp increased by the given * milliseconds. Ex: `TIMEADD(timestampColumn, 1000)` returns a timestamp 1 second * later. *
    • Since version: 3.3.0 *
    *
  5. Name: SUBSTRING *
      *
    • Semantic: SUBSTRING(colExpr, pos, len). Returns the slice of byte array * or string, that starts at pos and has the length len. *
        *
      • pos is 1 based. If pos is negative the start is determined by counting * characters (or bytes for BINARY) from the end. *
      • If len is less than 1 the result is empty. *
      • If len is omitted the function returns on characters or bytes starting with * pos. *
      *
    • Since version: 3.4.0 *
    *
* * @since 3.0.0 */ @Evolving public class ScalarExpression implements Expression { protected final String name; protected final List children; public ScalarExpression(String name, List children) { this.name = requireNonNull(name, "name is null").toUpperCase(Locale.ENGLISH); this.children = Collections.unmodifiableList(new ArrayList<>(children)); } @Override public String toString() { return String.format( "%s(%s)", name, children.stream().map(Object::toString).collect(Collectors.joining(", "))); } public String getName() { return name; } @Override public List getChildren() { return children; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/expressions/package-info.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /** * Expressions framework that defines the most common expressions which the connectors can use to * pass predicates to Delta Kernel. */ package io.delta.kernel.expressions; ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/hook/PostCommitHook.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.hook; import io.delta.kernel.engine.Engine; import java.io.IOException; /** * A hook for executing operation after a transaction commit. Hooks are added in the Transaction and * engine need to invoke the hook explicitly for executing the operation. Supported operations are * listed in {@link PostCommitHookType}. */ public interface PostCommitHook { enum PostCommitHookType { /** * Writes a new checkpoint at the version committed by the transaction. This hook is present * when the table is ready for checkpoint according to its configured checkpoint interval. To * perform this operation, reading previous checkpoint + logs is required to construct a new * checkpoint, with latency scaling based on log size (typically seconds to minutes). */ CHECKPOINT, /** * Writes a checksum file at the version committed by the transaction. This hook is present when * all required table statistics (e.g. table size) for checksum file are known when a * transaction commits. This operation has a minimal latency with no requirement of reading * previous checkpoint or logs. */ CHECKSUM_SIMPLE, /** * Writes a checksum file at the version committed by the transaction. This hook is present when * CHECKSUM_SIMPLE is missing. It requires constructing table stats via full log replay, which * can be expensive for large tables, with latency scaling based on log size. Unlike * CHECKSUM_SIMPLE, this always performs a full table state construction rather than * incrementally computing from a previous CRC. */ CHECKSUM_FULL, /** * Writes a log compaction file that merges a range of commit JSON files into a single file. * This hook is triggered on a configurable interval (e.g., every 10 commits) and reduces the * number of small log files that need to be read when reconstructing the table state, thereby * improving read performance. */ LOG_COMPACTION } /** Invokes the post commit operation whose implementation must be thread safe. */ void threadSafeInvoke(Engine engine) throws IOException; PostCommitHookType getType(); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/CommitActionsImpl.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal; import static java.util.Objects.requireNonNull; import io.delta.kernel.CommitActions; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.replay.ActionWrapper; import io.delta.kernel.internal.replay.ActionsIterator; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.internal.util.Preconditions; import io.delta.kernel.internal.util.Tuple2; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.types.StructField; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import java.io.IOException; import java.util.Collections; import java.util.HashSet; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; /** * Implementation of {@link CommitActions}. * *

This implementation owns the commit file and supports multiple calls to {@link #getActions()}. * The first call reuses initially-read data to avoid double I/O, while subsequent calls re-read the * commit file for memory efficiency. * *

Resource Management: * *

    *
  • Calling {@link #getActions()} transfers resource ownership to the returned iterator. *
  • Callers MUST close the returned iterator (preferably using try-with-resources) to release * file handles and other resources. *
  • If {@link #getActions()} is never called, callers should explicitly call {@link #close()} * to release internal resources. Otherwise, resources will be released when the object is * garbage collected. *
*/ public class CommitActionsImpl implements CommitActions, AutoCloseable { private final Engine engine; private final FileStatus commitFile; private final StructType readSchema; private final String tablePath; private final boolean shouldDropProtocolColumn; private final boolean shouldDropCommitInfoColumn; private final long version; private final long timestamp; /** * Iterator over ActionWrappers. The first call to {@link #getActions()} uses this iterator which * was created during construction (to extract metadata). Subsequent calls lazily create new * iterators, by constructing an ActionsIterator which does not open the file. */ private CloseableIterator iterator; /** * Creates a CommitActions from a commit file. * * @param engine the engine for file I/O * @param commitFile the commit file to read * @param tablePath the table path for error messages * @param actionSet the set of actions to read from the commit file */ public CommitActionsImpl( Engine engine, FileStatus commitFile, String tablePath, Set actionSet) { requireNonNull(engine, "engine cannot be null"); this.commitFile = requireNonNull(commitFile, "commitFile cannot be null"); this.tablePath = requireNonNull(tablePath, "tablePath cannot be null"); // Create a new action set which is a super set of the requested actions. // The extra actions are needed either for checks or to extract // extra information. We will strip out the extra actions before // returning the result. Set copySet = new HashSet<>(actionSet); copySet.add(DeltaLogActionUtils.DeltaAction.PROTOCOL); // commitInfo is needed to extract the inCommitTimestamp of delta files, this is used in // ActionsIterator to resolve the timestamp when available copySet.add(DeltaLogActionUtils.DeltaAction.COMMITINFO); // Determine whether the additional actions were in the original set. this.shouldDropProtocolColumn = !actionSet.contains(DeltaLogActionUtils.DeltaAction.PROTOCOL); this.shouldDropCommitInfoColumn = !actionSet.contains(DeltaLogActionUtils.DeltaAction.COMMITINFO); this.readSchema = new StructType( copySet.stream() .map(action -> new StructField(action.colName, action.schema, true)) .collect(Collectors.toList())); this.engine = engine; // Create initial iterator and peek at the first element to extract metadata CloseableIterator actionsIter = new ActionsIterator( engine, Collections.singletonList(commitFile), readSchema, Optional.empty()); Tuple2, CloseableIterator> headAndIter = peekHeadAndGetFullIterator(actionsIter); this.iterator = headAndIter._2; // Extract version and timestamp from first action (or use reading file if not exists) if (headAndIter._1.isPresent()) { ActionWrapper firstWrapper = headAndIter._1.get(); this.version = firstWrapper.getVersion(); this.timestamp = firstWrapper .getTimestamp() .orElseThrow( () -> new RuntimeException("timestamp should always exist for Delta File")); } else { // Empty commit file - extract from file metadata this.version = FileNames.deltaVersion(new Path(commitFile.getPath())); this.timestamp = commitFile.getModificationTime(); } } /** * Helper to peek at the first element and return both the head and a full iterator (head + rest). * * @return Tuple2 where _1 is the head element (Optional) and _2 is the full iterator */ private static Tuple2, CloseableIterator> peekHeadAndGetFullIterator(CloseableIterator iter) { Optional head = iter.hasNext() ? Optional.of(iter.next()) : Optional.empty(); CloseableIterator fullIterator = head.isPresent() ? Utils.singletonCloseableIterator(head.get()).combine(iter) : iter; return new Tuple2<>(head, fullIterator); } @Override public long getVersion() { return version; } @Override public long getTimestamp() { return timestamp; } @Override public synchronized CloseableIterator getActions() { CloseableIterator result = iterator.map( wrapper -> validateProtocolAndDropInternalColumns( wrapper.getColumnarBatch(), tablePath, shouldDropProtocolColumn, shouldDropCommitInfoColumn)); // Constructing an ActionsIterator does not open the file. iterator = new ActionsIterator( engine, Collections.singletonList(commitFile), readSchema, Optional.empty()); return result; } /** Validates protocol and drops protocol/commitInfo columns if not requested. */ private static ColumnarBatch validateProtocolAndDropInternalColumns( ColumnarBatch batch, String tablePath, boolean shouldDropProtocolColumn, boolean shouldDropCommitInfoColumn) { // Validate protocol if present in the batch. int protocolIdx = batch.getSchema().indexOf("protocol"); Preconditions.checkState(protocolIdx >= 0, "protocol column must be present in readSchema"); ColumnVector protocolVector = batch.getColumnVector(protocolIdx); for (int rowId = 0; rowId < protocolVector.getSize(); rowId++) { if (!protocolVector.isNullAt(rowId)) { Protocol protocol = Protocol.fromColumnVector(protocolVector, rowId); TableFeatures.validateKernelCanReadTheTable(protocol, tablePath); } } // Drop columns if not requested ColumnarBatch result = batch; if (shouldDropProtocolColumn && protocolIdx >= 0) { result = result.withDeletedColumnAt(protocolIdx); } int commitInfoIdx = result.getSchema().indexOf("commitInfo"); if (shouldDropCommitInfoColumn && commitInfoIdx >= 0) { result = result.withDeletedColumnAt(commitInfoIdx); } return result; } /** * Closes this CommitActionsImpl and releases any underlying resources. * * @throws IOException if an I/O error occurs while closing resources */ @Override public synchronized void close() throws IOException { Utils.closeCloseables(iterator); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/CreateTableTransactionBuilderImpl.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static io.delta.kernel.internal.util.Utils.resolvePath; import static java.util.Collections.emptyMap; import static java.util.Objects.requireNonNull; import io.delta.kernel.*; import io.delta.kernel.commit.Committer; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.TableAlreadyExistsException; import io.delta.kernel.exceptions.TableNotFoundException; import io.delta.kernel.expressions.Column; import io.delta.kernel.internal.annotation.VisibleForTesting; import io.delta.kernel.internal.commit.DefaultFileSystemManagedTableOnlyCommitter; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.internal.util.Clock; import io.delta.kernel.transaction.CreateTableTransactionBuilder; import io.delta.kernel.transaction.DataLayoutSpec; import io.delta.kernel.types.StructType; import java.util.*; public class CreateTableTransactionBuilderImpl implements CreateTableTransactionBuilder { private Clock clock = System::currentTimeMillis; private final String unresolvedPath; private final StructType schema; private final String engineInfo; private Optional> tableProperties = Optional.empty(); private Optional dataLayoutSpec = Optional.empty(); private Optional userProvidedMaxRetries = Optional.empty(); private Optional userProvidedCommitter = Optional.empty(); public CreateTableTransactionBuilderImpl(String tablePath, StructType schema, String engineInfo) { this.unresolvedPath = requireNonNull(tablePath, "tablePath is null"); this.schema = requireNonNull(schema, "schema is null"); this.engineInfo = requireNonNull(engineInfo, "engineInfo is null"); } @Override public CreateTableTransactionBuilder withTableProperties(Map properties) { requireNonNull(properties, "properties cannot be null"); final Map normalizedNewProperties = TableConfig.validateAndNormalizeDeltaProperties(properties); // Case 1: First time properties are being set if (!this.tableProperties.isPresent()) { this.tableProperties = Optional.of(Collections.unmodifiableMap(normalizedNewProperties)); return this; } // Case 2: Properties have already been set; ensure no duplicates with different values final Map existingProperties = this.tableProperties.get(); for (String key : normalizedNewProperties.keySet()) { final String existingValue = existingProperties.get(key); if (existingValue != null) { final String newValue = normalizedNewProperties.get(key); if (!Objects.equals(existingValue, newValue)) { throw new IllegalArgumentException( String.format( "Table property '%s' has already been set. Existing value: '%s', New value: '%s'", key, existingValue, newValue)); } } } final Map mergedProperties = new HashMap<>(existingProperties); mergedProperties.putAll(normalizedNewProperties); this.tableProperties = Optional.of(Collections.unmodifiableMap(mergedProperties)); return this; } @Override public CreateTableTransactionBuilder withDataLayoutSpec(DataLayoutSpec spec) { requireNonNull(spec, "spec cannot be null"); this.dataLayoutSpec = Optional.of(spec); return this; } @Override public CreateTableTransactionBuilder withMaxRetries(int maxRetries) { checkArgument(maxRetries >= 0, "maxRetries must be >= 0"); this.userProvidedMaxRetries = Optional.of(maxRetries); return this; } @Override public CreateTableTransactionBuilder withCommitter(Committer committer) { userProvidedCommitter = Optional.of(requireNonNull(committer, "committer cannot be null")); return this; } @VisibleForTesting public CreateTableTransactionBuilder withClock(Clock clock) { this.clock = requireNonNull(clock, "clock cannot be null"); return this; } @Override public Transaction build(Engine engine) { requireNonNull(engine, "engine cannot be null"); String resolvedPath = resolvePath(engine, unresolvedPath); throwIfTableAlreadyExists(engine, resolvedPath); // Extract partition and clustering columns from the data layout spec Optional> partitionColumns = dataLayoutSpec .filter(DataLayoutSpec::hasPartitioning) .map(DataLayoutSpec::getPartitionColumnsAsStrings); Optional> clusteringColumns = dataLayoutSpec .filter(DataLayoutSpec::hasClustering) .map(DataLayoutSpec::getClusteringColumns); TransactionMetadataFactory.Output txnMetadata = TransactionMetadataFactory.buildCreateTableMetadata( resolvedPath, schema, tableProperties.orElse(emptyMap()), partitionColumns, clusteringColumns, userProvidedCommitter); Path dataPath = new Path(resolvedPath); return new TransactionImpl( true, // isCreateOrReplace dataPath, Optional.empty(), // no existing snapshot for create table engineInfo, Operation.CREATE_TABLE, txnMetadata.newProtocol, txnMetadata.newMetadata, userProvidedCommitter.orElse(DefaultFileSystemManagedTableOnlyCommitter.INSTANCE), Optional.empty(), // no setTransaction for create table txnMetadata.physicalNewClusteringColumns, userProvidedMaxRetries, 0, // logCompactionInterval - no compaction for new table clock); } @VisibleForTesting public Optional> getTablePropertiesOpt() { return tableProperties; } @VisibleForTesting public Optional getCommitterOpt() { return userProvidedCommitter; } private void throwIfTableAlreadyExists(Engine engine, String tablePath) { final boolean isCatalogManaged = tableProperties .map( props -> TableFeatures.isPropertiesManuallySupportingTableFeature( props, TableFeatures.CATALOG_MANAGED_RW_FEATURE)) .orElse(false); if (isCatalogManaged) { // For catalog managed tables we assume the catalog has ensured the table loc is not already // a Delta table; return early return; } // Otherwise, try loading the latest snapshot to ensure the table does not exist try { Snapshot snapshot = TableManager.loadSnapshot(tablePath).build(engine); throw new TableAlreadyExistsException( tablePath, "Found table with latest version " + snapshot.getVersion()); } catch (TableNotFoundException tblf) { // This is the desired scenario as the table should not exist yet } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/DataWriteContextImpl.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal; import static java.util.Collections.unmodifiableList; import static java.util.Collections.unmodifiableMap; import io.delta.kernel.DataWriteContext; import io.delta.kernel.expressions.Column; import io.delta.kernel.expressions.Literal; import io.delta.kernel.internal.fs.Path; import java.util.List; import java.util.Map; /** * Implements the {@link DataWriteContext} interface. In addition to the data needed for the * interface, it also contains the partition values of the targeted partition. In case of * un-partitioned tables, the partition values will be empty. */ public class DataWriteContextImpl implements DataWriteContext { private final String targetDirectory; private final Map partitionValues; private final List statsColumns; /** * Creates a new instance of WriteContext. * * @param targetDirectory fully qualified path of the target directory * @param partitionValues partition values for the data to be written. If the table is * un-partitioned, this should be an empty map. * @param statsColumns Set of columns that need statistics for the data to be written. The column * can be a top-level column or a nested column. E.g. "a.b.c" is a nested column. "d" is a * top-level column. */ public DataWriteContextImpl( String targetDirectory, Map partitionValues, List statsColumns) { this.targetDirectory = targetDirectory; this.partitionValues = unmodifiableMap(partitionValues); this.statsColumns = unmodifiableList(statsColumns); } /** * Returns the target directory where the data should be written. * * @return fully qualified path of the target directory */ public String getTargetDirectory() { // TODO: this is temporary until paths are uniform (i.e. they are actually file system paths // or URIs everywhere, but not a combination of the two). return new Path(targetDirectory).toUri().toString(); } /** * Returns the partition values for the data to be written. If the table is un-partitioned, this * should be an empty map. * * @return partition values */ public Map getPartitionValues() { return partitionValues; } /** * Returns the list of {@link Column} that the connector can optionally collect statistics. Each * {@link Column} is a reference to a top-level or nested column in the table. * *

Statistics collections can be skipped or collected for a partial list of the returned {@link * Column}s. When stats are present in the written Delta log, they can be used to optimize query * performance. * * @return schema of the statistics */ public List getStatisticsColumns() { return statsColumns; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaErrors.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal; import static java.lang.String.format; import io.delta.kernel.commit.CommitFailedException; import io.delta.kernel.exceptions.*; import io.delta.kernel.expressions.Column; import io.delta.kernel.internal.actions.DomainMetadata; import io.delta.kernel.internal.tablefeatures.TableFeature; import io.delta.kernel.internal.util.SchemaIterable; import io.delta.kernel.types.DataType; import io.delta.kernel.types.StructField; import io.delta.kernel.types.StructType; import io.delta.kernel.types.TypeChange; import io.delta.kernel.utils.DataFileStatus; import java.io.IOException; import java.sql.Timestamp; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Supplier; import java.util.stream.Collectors; /** Contains methods to create user-facing Delta exceptions. */ public final class DeltaErrors { private DeltaErrors() {} public static KernelException missingCheckpoint(String tablePath, long checkpointVersion) { return new InvalidTableException( tablePath, String.format("Missing checkpoint at version %s", checkpointVersion)); } public static KernelException versionBeforeFirstAvailableCommit( String tablePath, long versionToLoad, long earliestVersion) { String message = String.format( "%s: Cannot load table version %s as the transaction log has been truncated due to " + "manual deletion or the log/checkpoint retention policy. The earliest available " + "version is %s.", tablePath, versionToLoad, earliestVersion); return new KernelException(message); } public static KernelException versionToLoadAfterLatestCommit( String tablePath, long versionToLoad, long latestVersion) { String message = String.format( "%s: Cannot load table version %s as it does not exist. " + "The latest available version is %s.", tablePath, versionToLoad, latestVersion); return new KernelException(message); } public static KernelException timestampBeforeFirstAvailableCommit( String tablePath, long providedTimestamp, long earliestCommitTimestamp, long earliestCommitVersion) { String message = String.format( "%s: The provided timestamp %s ms (%s) is before the earliest available version %s. " + "Please use a timestamp greater than or equal to %s ms (%s).", tablePath, providedTimestamp, formatTimestamp(providedTimestamp), earliestCommitVersion, earliestCommitTimestamp, formatTimestamp(earliestCommitTimestamp)); return new KernelException(message); } public static KernelException timestampAfterLatestCommit( String tablePath, long providedTimestamp, long latestCommitTimestamp, long latestCommitVersion) { String message = String.format( "%s: The provided timestamp %s ms (%s) is after the latest available version %s. " + "Please use a timestamp less than or equal to %s ms (%s).", tablePath, providedTimestamp, formatTimestamp(providedTimestamp), latestCommitVersion, latestCommitTimestamp, formatTimestamp(latestCommitTimestamp)); return new KernelException(message); } public static CommitRangeNotFoundException noCommitFilesFoundForVersionRange( String tablePath, long startVersion, Optional endVersionOpt) { return new CommitRangeNotFoundException(tablePath, startVersion, endVersionOpt); } public static KernelException startVersionNotFound( String tablePath, long startVersionRequested, Optional earliestAvailableVersion) { String message = String.format( "%s: Requested table changes beginning with startVersion=%s but no log file found for " + "version %s.", tablePath, startVersionRequested, startVersionRequested); if (earliestAvailableVersion.isPresent()) { message = message + String.format(" Earliest available version is %s", earliestAvailableVersion.get()); } return new KernelException(message); } public static KernelException endVersionNotFound( String tablePath, long endVersionRequested, long latestAvailableVersion) { String message = String.format( "%s: Requested table changes ending with endVersion=%d but no log file found for " + "version %d. Latest available version is %d", tablePath, endVersionRequested, endVersionRequested, latestAvailableVersion); return new KernelException(message); } public static KernelException invalidVersionRange(long startVersion, long endVersion) { String message = String.format( "Invalid version range: requested table changes for version range [%s, %s]. " + "Requires startVersion >= 0 and endVersion >= startVersion.", startVersion, endVersion); return new KernelException(message); } public static KernelException invalidResolvedVersionRange( String tablePath, long startVersion, long endVersion) { String message = String.format( "%s: Invalid resolved version range: after timestamp resolution, " + "startVersion=%d > endVersion=%d. " + "Please adjust the provided timestamp boundaries.", tablePath, startVersion, endVersion); return new KernelException(message); } public static KernelException resolvedEndVersionAfterMaxCatalogVersion( String tablePath, long resolvedEndVersion, long maxCatalogVersion) { String message = String.format( "%s: Resolved end version to %s which is after max catalog version %s", tablePath, resolvedEndVersion, maxCatalogVersion); return new KernelException(message); } /* ------------------------ PROTOCOL EXCEPTIONS ----------------------------- */ public static UnsupportedProtocolVersionException unsupportedReaderProtocol( String tablePath, int tableReaderVersion) { return new UnsupportedProtocolVersionException( tablePath, tableReaderVersion, UnsupportedProtocolVersionException.ProtocolVersionType.READER); } public static UnsupportedProtocolVersionException unsupportedWriterProtocol( String tablePath, int tableWriterVersion) { return new UnsupportedProtocolVersionException( tablePath, tableWriterVersion, UnsupportedProtocolVersionException.ProtocolVersionType.WRITER); } public static UnsupportedTableFeatureException unsupportedTableFeature(String feature) { String message = String.format( "Unsupported Delta table feature: table requires feature \"%s\" " + "which is unsupported by this version of Delta Kernel.", feature); return new UnsupportedTableFeatureException(null, feature, message); } public static UnsupportedTableFeatureException unsupportedReaderFeatures( String tablePath, Set readerFeatures) { String message = String.format( "Unsupported Delta reader features: table `%s` requires reader table features [%s] " + "which is unsupported by this version of Delta Kernel.", tablePath, String.join(", ", readerFeatures)); return new UnsupportedTableFeatureException(tablePath, readerFeatures, message); } public static UnsupportedTableFeatureException unsupportedWriterFeatures( String tablePath, Set writerFeatures) { String message = String.format( "Unsupported Delta writer features: table `%s` requires writer table features [%s] " + "which is unsupported by this version of Delta Kernel.", tablePath, String.join(", ", writerFeatures)); return new UnsupportedTableFeatureException(tablePath, writerFeatures, message); } public static KernelException columnInvariantsNotSupported() { String message = "This version of Delta Kernel does not support writing to tables with " + "column invariants present."; return new KernelException(message); } public static KernelException checkpointOnUnpublishedCommits( String tablePath, long version, long maxPublishedVersion) { String message = String.format( "Unable to create checkpoint: Snapshot at at path" + " `%s` with version %d has unpublished commits. " + "Max known published version is %d", tablePath, version, maxPublishedVersion); return new KernelException(message); } public static KernelException unsupportedDataType(DataType dataType) { return new KernelException("Kernel doesn't support writing data of type: " + dataType); } public static KernelException unsupportedStatsDataType(DataType dataType) { return new KernelException("Kernel doesn't support writing stats data of type: " + dataType); } public static KernelException unsupportedPartitionDataType(String colName, DataType dataType) { String msgT = "Kernel doesn't support writing data with partition column (%s) of type: %s"; return new KernelException(format(msgT, colName, dataType)); } public static KernelException duplicateColumnsInSchema( StructType schema, List duplicateColumns) { String msg = format( "Schema contains duplicate columns: %s.\nSchema: %s", String.join(", ", duplicateColumns), schema); return new KernelException(msg); } public static KernelException conflictWithReservedInternalColumnName(String columnName) { return new KernelException( format("Cannot use column name '%s' because it is reserved for internal use", columnName)); } public static KernelException invalidColumnName(String columnName, String unsupportedChars) { return new KernelException( format( "Column name '%s' contains one of the unsupported (%s) characters.", columnName, unsupportedChars)); } public static KernelException requiresSchemaForNewTable(String tablePath) { return new TableNotFoundException( tablePath, "Must provide a new schema to write to a new table."); } public static KernelException requireSchemaForReplaceTable() { return new KernelException("Must provide a new schema for REPLACE TABLE"); } public static KernelException tableAlreadyExists(String tablePath, String message) { return new TableAlreadyExistsException(tablePath, message); } public static KernelException dataSchemaMismatch( String tablePath, StructType tableSchema, StructType dataSchema) { String msgT = "The schema of the data to be written to the table doesn't match " + "the table schema. \nTable: %s\nTable schema: %s, \nData schema: %s"; return new KernelException(format(msgT, tablePath, tableSchema, dataSchema)); } public static KernelException statsTypeMismatch( String fieldName, DataType expected, DataType actual) { String msgFormat = "Type mismatch for field '%s' when writing statistics: expected %s, but found %s"; return new KernelException(format(msgFormat, fieldName, expected, actual)); } public static KernelException columnNotFoundInSchema(Column column, StructType tableSchema) { return new KernelException( format("Column '%s' was not found in the table schema: %s", column, tableSchema)); } public static KernelException overlappingTablePropertiesSetAndUnset(Set violatingKeys) { return new KernelException( format( "Cannot set and unset the same table property in the same transaction. " + "Properties set and unset: %s", violatingKeys)); } /// Start: icebergCompat exceptions public static KernelException icebergCompatMissingNumRecordsStats( String compatVersion, DataFileStatus dataFileStatus) { throw new KernelException( format( "%s compatibility requires 'numRecords' statistic.\n DataFileStatus: %s", compatVersion, dataFileStatus)); } public static KernelException icebergCompatIncompatibleVersionEnabled( String compatVersion, String incompatibleIcebergCompatVersion) { throw new KernelException( format( "%s: Only one IcebergCompat version can be enabled. Incompatible version enabled: %s", compatVersion, incompatibleIcebergCompatVersion)); } public static KernelException icebergCompatUnsupportedTypeColumns( String compatVersion, List dataTypes) { throw new KernelException( format("%s does not support the data types: %s.", compatVersion, dataTypes)); } public static KernelException icebergCompatUnsupportedTypeWidening( String compatVersion, TypeChange typeChange) { throw new KernelException( format( "%s does not support type widening present in table: %s.", compatVersion, typeChange)); } public static KernelException icebergCompatUnsupportedTypePartitionColumn( String compatVersion, DataType dataType) { throw new KernelException( format( "%s does not support the data type '%s' for a partition column.", compatVersion, dataType)); } public static KernelException icebergCompatRequiresLiteralDefaultValue( String compatVersion, DataType dataType, String value) { throw new KernelException( format( "%s requires the default value to be literal with correct data types for " + "a column. '%s: %s' is invalid.", compatVersion, dataType, value)); } public static KernelException icebergCompatIncompatibleTableFeatures( String compatVersion, Set incompatibleFeatures) { throw new KernelException( format( "Table features %s are incompatible with %s.", incompatibleFeatures.stream() .map(TableFeature::featureName) .collect(Collectors.toList()), compatVersion)); } public static KernelException icebergCompatRequiredFeatureMissing( String compatVersion, String feature) { throw new KernelException( format("%s: requires the feature '%s' to be enabled.", compatVersion, feature)); } public static KernelException enablingIcebergCompatFeatureOnExistingTable(String key) { return new KernelException( String.format( "Cannot enable %s on an existing table. " + "Enablement is only supported upon table creation.", key)); } public static KernelException icebergWriterCompatInvalidPhysicalName(List invalidFields) { return new KernelException( String.format( "IcebergWriterCompatV1 requires column mapping field physical names be equal to " + "'col-[fieldId]', but this is not true for the following fields %s", invalidFields)); } public static KernelException disablingIcebergCompatFeatureOnExistingTable(String key) { return new KernelException( String.format("Disabling %s on an existing table is not allowed.", key)); } // End: icebergCompat exceptions // Start: Column Defaults Exceptions // TODO migrate this to InvalidTableException when table info is available at the call site public static KernelException defaultValueRequiresTableFeature() { return new KernelException( "Found column defaults in the schema but the table does not support the " + "columnDefaults table feature."); } public static KernelException defaultValueRequireIcebergV3() { return new KernelException( "In Delta Kernel, default values table feature requires " + "IcebergCompatV3 to be enabled."); } public static KernelException unsupportedDataTypeForDefaultValue( String fieldName, String fieldType) { return new KernelException( String.format( "Kernel does not support default value for " + "data type %s: %s", fieldType, fieldName)); } public static KernelException nonLiteralDefaultValue(String value) { return new KernelException( String.format( "currently only literal values are supported for default values in Kernel." + " %s is an invalid default value", value)); } // End: Column Defaults Exceptions public static KernelException partitionColumnMissingInData( String tablePath, String partitionColumn) { String msgT = "Missing partition column '%s' in the data to be written to the table '%s'."; return new KernelException(format(msgT, partitionColumn, tablePath)); } public static KernelException enablingClusteringOnPartitionedTableNotAllowed( String tablePath, Set partitionColNames, List clusteringCols) { return new KernelException( String.format( "Cannot enable clustering on a partitioned table '%s'. " + "Existing partition columns: '%s', Clustering columns: '%s'.", tablePath, partitionColNames, clusteringCols)); } public static RuntimeException nonRetryableCommitException( int attempt, long commitAsVersion, CommitFailedException cause) { throw new RuntimeException( String.format( "Commit attempt %d for version %d failed with a non-retryable exception.", attempt, commitAsVersion), cause); } public static KernelException concurrentTransaction( String appId, long txnVersion, long lastUpdated) { return new ConcurrentTransactionException(appId, txnVersion, lastUpdated); } public static KernelException metadataChangedException() { return new MetadataChangedException(); } public static KernelException protocolChangedException(long attemptVersion) { return new ProtocolChangedException(attemptVersion); } public static KernelException voidTypeEncountered() { return new KernelException( "Failed to parse the schema. Encountered unsupported Delta data type: VOID"); } public static KernelException cannotModifyTableProperty(String key) { String msg = format("The Delta table property '%s' is an internal property and cannot be updated.", key); return new KernelException(msg); } public static KernelException unknownConfigurationException(String confKey) { return new UnknownConfigurationException(confKey); } public static KernelException invalidConfigurationValueException( String key, String value, String helpMessage) { return new InvalidConfigurationValueException(key, value, helpMessage); } public static KernelException domainMetadataUnsupported() { String message = "Cannot commit DomainMetadata action(s) because the feature 'domainMetadata' " + "is not supported on this table."; return new KernelException(message); } public static ConcurrentWriteException concurrentDomainMetadataAction( DomainMetadata domainMetadataAttempt, DomainMetadata winningDomainMetadata) { String message = String.format( "A concurrent writer added a domainMetadata action for the same domain: %s. " + "No domain-specific conflict resolution is available for this domain. " + "Attempted domainMetadata: %s. Winning domainMetadata: %s", domainMetadataAttempt.getDomain(), domainMetadataAttempt, winningDomainMetadata); return new ConcurrentWriteException(message); } public static KernelException missingNumRecordsStatsForRowTracking() { return new KernelException( "Cannot write to a rowTracking-supported table without 'numRecords' statistics. " + "Connectors are expected to populate the number of records statistics when " + "writing to a Delta table with 'rowTracking' table feature supported."); } public static KernelException rowTrackingSupportedWithDomainMetadataUnsupported() { return new KernelException( "Feature 'rowTracking' is supported and depends on feature 'domainMetadata'," + " but 'domainMetadata' is unsupported"); } public static KernelException rowTrackingRequiredForRowIdHighWatermark( String tablePath, String rowIdHighWatermark) { return new KernelException( String.format( "Cannot assign a row id high water mark (`%s`) to a table `%s` that does not support " + "`rowTracking` table feature. Please enable the `rowTracking` table feature.", rowIdHighWatermark, tablePath)); } public static KernelException cannotToggleRowTrackingOnExistingTable() { return new KernelException("Row tracking support cannot be changed once the table is created."); } public static KernelException missingRowTrackingColumnRequested(String columnName) { return new KernelException( String.format( "Row tracking is not enabled, but row tracking column '%s' was requested.", columnName)); } public static KernelException cannotModifyAppendOnlyTable(String tablePath) { return new KernelException( String.format( "Cannot modify append-only table. Table `%s` has configuration %s=true.", tablePath, TableConfig.APPEND_ONLY_ENABLED.getKey())); } public static KernelException rowTrackingMetadataMissingInFile(String entry, String filePath) { return new KernelException( String.format("Required metadata key %s is not present in scan file %s.", entry, filePath)); } public static InvalidTableException tableWithIctMissingCommitInfo(String dataPath, long version) { return new InvalidTableException( dataPath, String.format( "This table has the feature inCommitTimestamp enabled which requires the presence of " + "the CommitInfo action in every commit. However, the CommitInfo action is " + "missing from commit version %d.", version)); } public static InvalidTableException tableWithIctMissingIct(String dataPath, long version) { return new InvalidTableException( dataPath, String.format( "This table has the feature inCommitTimestamp enabled which requires the presence of " + "inCommitTimestamp in the CommitInfo action. However, this field has not been " + "set in commit version %d.", version)); } public static KernelException metadataMissingRequiredCatalogTableProperty( String committerClassName, Map missingOrViolatingProperties, Map requiredCatalogTableProperties) { final String details = missingOrViolatingProperties.entrySet().stream() .map( entry -> String.format( "%s (current: '%s', required: '%s')", entry.getKey(), entry.getValue(), requiredCatalogTableProperties.get(entry.getKey()))) .collect(Collectors.joining(", ")); return new KernelException( String.format( "[%s] Metadata is missing or has incorrect values for required catalog properties: %s.", committerClassName, details)); } public static KernelException invalidFieldMove( int columnId, Optional currentParent, Optional newParent) { return new KernelException( String.format( "Cannot move fields between different levels of nesting: " + "field with fieldId=%s is nested under %s in the current schema and under %s in " + "the new schema", columnId, formatParentField(currentParent), formatParentField(newParent))); } /* ------------------------ HELPER METHODS ----------------------------- */ private static String formatParentField(Optional parent) { if (!parent.isPresent()) { return "ROOT"; } StructField parentField = parent.get().getParentField(); String pathToParentField = parent.get().getPathFromParent(); if (pathToParentField.isEmpty()) { // Example: "StructField(name=c1, ...)" return parentField.toString(); } else { // Example: "StructField(name=c1, ...) at path=key.element" return parentField.toString() + " at path=" + pathToParentField; } } private static String formatTimestamp(long millisSinceEpochUTC) { return new Timestamp(millisSinceEpochUTC).toInstant().toString(); } // We use the `Supplier` interface to avoid silently wrapping any checked exceptions public static T wrapEngineException(Supplier f, String msgString, Object... args) { try { return f.get(); } catch (KernelException e) { // Let any KernelExceptions fall through (even though these generally shouldn't // originate from the engine implementation there are some edge cases such as // deserializeStructType) throw e; } catch (RuntimeException e) { throw new KernelEngineException(String.format(msgString, args), e); } } // Functional interface for a fx that throws an `IOException` (but no other checked exceptions) public interface SupplierWithIOException { T get() throws IOException; } public static T wrapEngineExceptionThrowsIO( SupplierWithIOException f, String msgString, Object... args) throws IOException { try { return f.get(); } catch (KernelException e) { // Let any KernelExceptions fall through (even though these generally shouldn't // originate from the engine implementation there are some edge cases such as // deserializeStructType) throw e; } catch (RuntimeException e) { throw new KernelEngineException(String.format(msgString, args), e); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaErrorsInternal.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal; import io.delta.kernel.exceptions.InvalidTableException; import java.util.List; /** * Contains methods to create developer-facing exceptions. See Exception * Principles for more information on what these are and how to use them. */ public class DeltaErrorsInternal { private DeltaErrorsInternal() {} public static IllegalStateException missingRemoveFileSizeDuringCommit() { return new IllegalStateException( "Kernel APIs for creating remove file rows require that " + "file size be provided but found null file size"); } public static IllegalStateException invalidTimestampFormatForPartitionValue( String partitionValue) { return new IllegalStateException( String.format( "Invalid timestamp format for value: %s. Expected formats: " + "'yyyy-MM-dd HH:mm:ss[.SSSSSS]' or ISO-8601 (e.g. 2020-01-01T00:00:00Z)'", partitionValue)); } public static UnsupportedOperationException defaultCommitterDoesNotSupportCatalogManagedTables() { return new UnsupportedOperationException( "No io.delta.kernel.commit.Committer has been provided to Kernel, so Kernel is using a " + "default Committer that only supports committing to filesystem-managed Delta tables, " + "not catalog-managed Delta tables. Since this table is catalog-managed, this " + "commit operation is unsupported."); } public static IllegalStateException logicalPhysicalSchemaMismatch( int num_partition_cols, int physical_size, int logical_size) { return new IllegalStateException( String.format( "The number of partition columns (%s) plus the physical schema size (%s) does not " + "equal the logical schema size (%s).", num_partition_cols, physical_size, logical_size)); } public static InvalidTableException catalogCommitsPrecedePublishedDeltas( String tablePath, long earliestCatalogCommitVersion, List foundPublishedVersions) { return new InvalidTableException( tablePath, String.format( "Missing delta file: found staged ratified commit for version %s but no published " + "delta file. Found published deltas for later versions: %s", earliestCatalogCommitVersion, foundPublishedVersions)); } public static InvalidTableException publishedDeltasAndCatalogCommitsNotContiguous( String tablePath, List publishedDeltaVersions, List catalogCommitVersions) { return new InvalidTableException( tablePath, String.format( "Missing delta files: found published delta files for versions %s and staged " + "ratified commits for versions %s", publishedDeltaVersions, catalogCommitVersions)); } public static InvalidTableException publishedDeltasNotContiguous( String tablePath, List foundVersions) { return new InvalidTableException( tablePath, String.format("Missing delta files: versions are not contiguous: (%s)", foundVersions)); } public static IllegalArgumentException invalidLatestSnapshotForMaxCatalogVersion( long latestSnapshotVersion, long maxCatalogVersion) { return new IllegalArgumentException( String.format( "When using timestamp boundaries with maxCatalogVersion, the provided " + "snapshot version (%d) must equal maxCatalogVersion (%d)", latestSnapshotVersion, maxCatalogVersion)); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaHistoryManager.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal; import static io.delta.kernel.internal.DeltaErrors.wrapEngineExceptionThrowsIO; import static io.delta.kernel.internal.TableConfig.*; import static io.delta.kernel.internal.fs.Path.getName; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.exceptions.TableNotFoundException; import io.delta.kernel.internal.actions.CommitInfo; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.checkpoints.CheckpointInstance; import io.delta.kernel.internal.files.LogDataUtils; import io.delta.kernel.internal.files.ParsedCatalogCommitData; import io.delta.kernel.internal.files.ParsedLogData; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.internal.util.InCommitTimestampUtils; import io.delta.kernel.internal.util.Tuple2; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import java.io.FileNotFoundException; import java.io.IOException; import java.io.UncheckedIOException; import java.util.*; import java.util.function.Function; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public final class DeltaHistoryManager { private DeltaHistoryManager() {} private static final Logger logger = LoggerFactory.getLogger(DeltaHistoryManager.class); /** * Returns the latest version that was committed at or after {@code millisSinceEpochUTC}. If no * version exists, throws a {@link KernelException} * *

Specifically: * *

    *
  • if a commit version exactly matches the provided timestamp, we return it *
  • else, we return the earliest commit version with a timestamp greater than the provided * one *
  • If the provided timestamp is larger than the timestamp of any committed version, we throw * an error. *
* * @param millisSinceEpochUTC the number of milliseconds since midnight, January 1, 1970 UTC * @param catalogCommits parsed log Deltas to use (must be sorted and contiguous) * @return latest commit that happened at or before {@code timestamp}. * @throws KernelException if the timestamp is more than the timestamp of any committed version */ public static long getVersionAtOrAfterTimestamp( Engine engine, Path logPath, long millisSinceEpochUTC, SnapshotImpl latestSnapshot, List catalogCommits) { DeltaHistoryManager.Commit commit = DeltaHistoryManager.getActiveCommitAtTimestamp( engine, latestSnapshot, logPath, millisSinceEpochUTC, false, /* mustBeRecreatable */ // e.g. if we give time T+2 and last commit has time T, then we do NOT want that last // commit false, /* canReturnLastCommit */ // e.g. we give time T-1 and first commit has time T, then we DO want that earliest // commit true /* canReturnEarliestCommit */, catalogCommits); if (commit.getTimestamp() >= millisSinceEpochUTC) { return commit.getVersion(); } else { // this commit.timestamp is before the input timestamp. if this is the last commit, then // the input timestamp is after the last commit and `getActiveCommitAtTimestamp` would have // thrown an KernelException. So, clearly, this can't be the last commit, so we can safely // return commit.version + 1 as the version that is at or after the input timestamp. return commit.getVersion() + 1; } } /** * Returns the latest version that was committed before or at {@code millisSinceEpochUTC}. If no * version exists, throws a {@link KernelException} * *

Specifically: * *

    *
  • if a commit version exactly matches the provided timestamp, we return it *
  • else, we return the latest commit version with a timestamp less than the provided one *
  • If the provided timestamp is less than the timestamp of any committed version, we throw * an error. *
* * @param millisSinceEpochUTC the number of milliseconds since midnight, January 1, 1970 UTC * @param catalogCommits parsed log Deltas to use (must be sorted and contiguous) * @return latest commit that happened before or at {@code timestamp}. * @throws KernelException if the timestamp is less than the timestamp of any committed version */ public static long getVersionBeforeOrAtTimestamp( Engine engine, Path logPath, long millisSinceEpochUTC, SnapshotImpl latestSnapshot, List catalogCommits) { return DeltaHistoryManager.getActiveCommitAtTimestamp( engine, latestSnapshot, logPath, millisSinceEpochUTC, false, /* mustBeRecreatable */ // e.g. if we give time T+2 and last commit has time T, then we DO want that last commit true, /* canReturnLastCommit */ // e.g. we give time T-1 and first commit has time T, then do NOT want that earliest // commit false /* canReturnEarliestCommit */, catalogCommits) .getVersion(); } /** * Returns the latest commit that happened at or before {@code timestamp}. * *

If the timestamp is outside the range of [earliestCommit, latestCommit] then use parameters * {@code canReturnLastCommit} and {@code canReturnEarliestCommit} to control whether an exception * is thrown or the corresponding earliest/latest commit is returned. * * @param engine instance of {@link Engine} to use * @param logPath the _delta_log path of the table * @param timestamp the timestamp find the version for in milliseconds since the unix epoch * @param mustBeRecreatable whether the state at the returned commit should be recreatable * @param canReturnLastCommit whether we can return the latest version of the table if the * provided timestamp is after the latest commit * @param canReturnEarliestCommit whether we can return the earliest version of the table if the * provided timestamp is before the earliest commit * @param catalogCommits parsed log Deltas to use (must be sorted and contiguous) * @throws KernelException if the provided timestamp is before the earliest commit and * canReturnEarliestCommit is false * @throws KernelException if the provided timestamp is after the latest commit and * canReturnLastCommit is false * @throws TableNotFoundException when there is no Delta table at the given path */ public static Commit getActiveCommitAtTimestamp( Engine engine, SnapshotImpl latestSnapshot, Path logPath, long timestamp, boolean mustBeRecreatable, boolean canReturnLastCommit, boolean canReturnEarliestCommit, List catalogCommits) throws TableNotFoundException { // For now, we only accept *staged* ratified commits (not inline) LogDataUtils.validateLogDataContainsOnlyRatifiedStagedCommits(catalogCommits); // Create a mapper for delta version -> file status that takes into account ratified commits Function versionToFileStatusFunction = getVersionToFileStatusFunction(catalogCommits, logPath); Optional earliestRatifiedCommitVersion = catalogCommits.stream().map(ParsedLogData::getVersion).min(Long::compare); long earliestVersion = (mustBeRecreatable) ? getEarliestRecreatableCommit(engine, logPath, earliestRatifiedCommitVersion) : getEarliestDeltaFile(engine, logPath, earliestRatifiedCommitVersion); Commit placeholderEarliestCommit = new Commit(earliestVersion, -1L /* timestamp */); Commit ictEnablementCommit = getICTEnablementCommit(latestSnapshot, placeholderEarliestCommit); // Validate our assumptions: ICT must be enabled for all catalog-provided commits earliestRatifiedCommitVersion.ifPresent( v -> checkArgument( ictEnablementCommit.version <= v, "catalogManaged tables must have ICT enabled but given catalog-provided commit " + "with version < ictEnablementVersion")); Commit searchResult; if (ictEnablementCommit.getTimestamp() <= timestamp) { // The target commit is in the ICT range. long latestSnapshotTimestamp = latestSnapshot.getTimestamp(engine); if (latestSnapshotTimestamp <= timestamp) { // We just proved we should use the latest snapshot // Note that if `latestSnapshotTimestamp` is less than `timestamp`, we only // return this search result if `canReturnLastCommit` is true. // If `canReturnLastCommit` is false, we still need this commit to // throw the timestampAfterLatestCommit error. searchResult = new Commit(latestSnapshot.getVersion(), latestSnapshotTimestamp); } else { // start ICT search over [earliest available ICT version, latestVersion) boolean ictEnabledForEntireWindow = (ictEnablementCommit.version <= earliestVersion); long searchWindowLowerBound = ictEnabledForEntireWindow ? placeholderEarliestCommit.getVersion() : ictEnablementCommit.getVersion(); try { searchResult = getActiveCommitAtTimeFromICTRange( timestamp, searchWindowLowerBound, latestSnapshot.getVersion(), engine, latestSnapshot.getLogPath(), versionToFileStatusFunction); } catch (IOException e) { throw new RuntimeException( "There was an error while reading a historical commit while performing a timestamp-" + "based lookup. This usually happens when there is a parallel operation like " + "metadata cleanup that is deleting commits. Please retry the query.", e); } } } else { // ICT was NOT enabled as-of the requested time if (ictEnablementCommit.version <= earliestVersion) { // We're searching for a non-ICT time but the non-ICT commits are all missing. // If `canReturnEarliestCommit` is `false`, we need the details of the // earliest commit to populate the timestampBeforeFirstAvailableCommit // error correctly. // Else, when `canReturnEarliestCommit` is `true`, the earliest commit // is the desired result. long ict = CommitInfo.getRequiredIctFromDeltaFile( engine, logPath.getParent(), versionToFileStatusFunction.apply(placeholderEarliestCommit.getVersion()), placeholderEarliestCommit.getVersion()); searchResult = new Commit(placeholderEarliestCommit.getVersion(), ict); } else { // We know the table was not catalogManaged here since ICT was not enabled ==> we don't // need to worry about catalogCommits // start non-ICT linear search over [earliestVersion, ) List commits = getCommits(engine, logPath, earliestVersion); searchResult = lastCommitBeforeOrAtTimestamp(commits, timestamp) .orElse( commits.get(0)); // This is only returned if canReturnEarliestCommit (see below) } } // If timestamp is before the earliest commit if (searchResult.timestamp > timestamp && !canReturnEarliestCommit) { throw DeltaErrors.timestampBeforeFirstAvailableCommit( logPath.getParent().toString(), /* use dataPath */ timestamp, searchResult.timestamp, searchResult.version); } // If timestamp is after the last commit of the table if (searchResult.version == latestSnapshot.getVersion() && searchResult.timestamp < timestamp && !canReturnLastCommit) { throw DeltaErrors.timestampAfterLatestCommit( logPath.getParent().toString(), /* use dataPath */ timestamp, searchResult.timestamp, searchResult.version); } return searchResult; } /** * Finds the commit with the latest in-commit timestamp that is less than or equal to the * searchTimestamp. All commits from `startCommitVersionInclusive` till * `endCommitVersionInclusive` must have ICT enabled. Also, this method assumes that we have * already proven that `searchTimestamp` is in the given range. */ private static Commit getActiveCommitAtTimeFromICTRange( long searchTimestamp, long startCommitVersionInclusive, long endCommitVersionInclusive, Engine engine, Path logPath, Function versionToFileStatusFunction) throws IOException { // Now we have a range of commits to search through. We can use binary search to find the // commit that is closest to the search timestamp. Optional> greatestLowerBoundOpt = InCommitTimestampUtils.greatestLowerBound( searchTimestamp, startCommitVersionInclusive, endCommitVersionInclusive, version -> CommitInfo.getRequiredIctFromDeltaFile( engine, logPath.getParent(), versionToFileStatusFunction.apply(version), version)); // This indicates that the search timestamp is less than the earliest commit. if (!greatestLowerBoundOpt.isPresent()) { long startIct = CommitInfo.getRequiredIctFromDeltaFile( engine, logPath.getParent(), versionToFileStatusFunction.apply(startCommitVersionInclusive), startCommitVersionInclusive); return new Commit(startCommitVersionInclusive, startIct); } Tuple2 greatestLowerBound = greatestLowerBoundOpt.get(); return new Commit(greatestLowerBound._1, greatestLowerBound._2); } /** * Gets the commit that enabled in-commit timestamps. * * @param snapshot The latest snapshot of the table. This is used to determine when in-commit * timestamps were enabled. * @param earliestCommit The earliest commit under consideration. If in-commit timestamps were * enabled for the entire history, this function will return this commit. * @return The commit that enabled in-commit timestamps. If the table does not have in-commit * timestamps enabled, this will be the commit after the latest version. If in-commit * timestamps were enabled for the entire history, this will be `earliestCommit`. */ private static Commit getICTEnablementCommit(SnapshotImpl snapshot, Commit earliestCommit) { Metadata metadata = snapshot.getMetadata(); if (!IN_COMMIT_TIMESTAMPS_ENABLED.fromMetadata(metadata)) { // Pretend ICT will be enabled after the latest version and requested timestamp. // This will force us to use the non-ICT search path. return new Commit(snapshot.getVersion() + 1, Long.MAX_VALUE); } Optional enablementTimestampOpt = IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.fromMetadata(metadata); Optional enablementVersionOpt = IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.fromMetadata(metadata); if (enablementTimestampOpt.isPresent() && enablementVersionOpt.isPresent()) { return new Commit(enablementVersionOpt.get(), enablementTimestampOpt.get()); } else if (!enablementTimestampOpt.isPresent() && !enablementVersionOpt.isPresent()) { // This means that ICT has been enabled for the entire history. return earliestCommit; } else { throw new IllegalStateException( String.format( "Both %s and %s should be present or absent together" + "when inCommitTimestamp is enabled.", IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.getKey(), IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.getKey())); } } /** * Gets the earliest commit that we can recreate. Note that this version isn't guaranteed to exist * when performing an action as a concurrent operation can delete the file during cleanup. This * value must be used as a lower bound. * *

We search for the earliest checkpoint we have, or whether we have the 0th delta file. This * method assumes that the commits are contiguous. */ public static long getEarliestRecreatableCommit( Engine engine, Path logPath, Optional earliestRatifiedCommitVersion) throws TableNotFoundException { // For a catalogManaged table, the only time no published commits exist is when v0 has not yet // been published. Otherwise, since checkpoints must have a published delta file, and log clean // up must always preserve a checkpoint, there must be published commits present on the // file-system if (earliestRatifiedCommitVersion.isPresent() && earliestRatifiedCommitVersion.get() == 0) { return 0; } // Thus, if there is no v0 ratified commit, then there must be published commit try (CloseableIterator files = listFrom(engine, logPath, 0) .filter( fs -> FileNames.isCommitFile(getName(fs.getPath())) || FileNames.isCheckpointFile(getName(fs.getPath())))) { if (!files.hasNext()) { // listFrom already throws an error if the directory is truly empty, thus this must // be because no files are checkpoint or delta files throw new RuntimeException( String.format("No delta files found in the directory: %s", logPath)); } // A map of checkpoint version and number of parts to number of parts observed Map, Integer> checkpointMap = new HashMap<>(); long smallestDeltaVersion = Long.MAX_VALUE; Optional lastCompleteCheckpoint = Optional.empty(); // Iterate through the log files - this will be in order starting from the lowest // version. Checkpoint files come before deltas, so when we see a checkpoint, we // remember it and return it once we detect that we've seen a smaller or equal delta // version. while (files.hasNext()) { String nextFilePath = files.next().getPath(); if (FileNames.isCommitFile(getName(nextFilePath))) { long version = FileNames.deltaVersion(nextFilePath); if (version == 0L) { return version; } smallestDeltaVersion = Math.min(version, smallestDeltaVersion); // Note that we also check this condition at the end of the function - we check // it here too to try and avoid more file listing when it's unnecessary. if (lastCompleteCheckpoint.isPresent() && lastCompleteCheckpoint.get() >= smallestDeltaVersion) { return lastCompleteCheckpoint.get(); } } else if (FileNames.isCheckpointFile(nextFilePath)) { long checkpointVersion = FileNames.checkpointVersion(nextFilePath); CheckpointInstance checkpointInstance = new CheckpointInstance(nextFilePath); if (!checkpointInstance.numParts.isPresent()) { lastCompleteCheckpoint = Optional.of(checkpointVersion); } else { // if we have a multi-part checkpoint, we need to check that all parts exist int numParts = checkpointInstance.numParts.orElse(1); int preCount = checkpointMap.getOrDefault(new Tuple2<>(checkpointVersion, numParts), 0); if (numParts == preCount + 1) { lastCompleteCheckpoint = Optional.of(checkpointVersion); } checkpointMap.put(new Tuple2<>(checkpointVersion, numParts), preCount + 1); } } } if (lastCompleteCheckpoint.isPresent() && lastCompleteCheckpoint.get() >= smallestDeltaVersion) { return lastCompleteCheckpoint.get(); } else if (smallestDeltaVersion < Long.MAX_VALUE) { // This is a corrupt table where 000.json does not exist and there are no complete // checkpoints OR the earliest complete checkpoint does not have a corresponding // commit file (but there are other later commit files present) throw new RuntimeException(String.format("No recreatable commits found at %s", logPath)); } else { throw new RuntimeException(String.format("No commits found at %s", logPath)); } } catch (IOException e) { throw new RuntimeException("Could not close iterator", e); } } /** * Get the earliest commit available for this table. Note that this version isn't guaranteed to * exist when performing an action as a concurrent operation can delete the file during cleanup. * This value must be used as a lower bound. */ public static long getEarliestDeltaFile( Engine engine, Path logPath, Optional earliestRatifiedCommitVersion) throws TableNotFoundException { // For a catalogManaged table, the only time no published commits exist is when v0 has not yet // been published. Otherwise, since checkpoints must have a published delta file, and log clean // up must always preserve a checkpoint, there must be published commits present on the // file-system if (earliestRatifiedCommitVersion.isPresent() && earliestRatifiedCommitVersion.get() == 0) { return 0; } // Thus, if there is no v0 ratified commit, then there must be published commit. // Due to *ordered backfill* we know the following: // minFSPublishedCommitVersion <= minCatalogProvidedCommitVersion try (CloseableIterator files = listFrom(engine, logPath, 0).filter(fs -> FileNames.isCommitFile(getName(fs.getPath())))) { if (files.hasNext()) { return FileNames.deltaVersion(files.next().getPath()); } else { // listFrom already throws an error if the directory is truly empty, thus this must // be because no files are delta files throw new RuntimeException( String.format("No delta files found in the directory: %s", logPath)); } } catch (IOException e) { throw new RuntimeException("Could not close iterator", e); } } /** * Returns an iterator containing a list of files found in the _delta_log directory starting with * {@code startVersion}. Throws a {@link TableNotFoundException} if the directory doesn't exist or * is empty. */ private static CloseableIterator listFrom( Engine engine, Path logPath, long startVersion) throws TableNotFoundException { Path tablePath = logPath.getParent(); try { CloseableIterator files = wrapEngineExceptionThrowsIO( () -> engine .getFileSystemClient() .listFrom(FileNames.listingPrefix(logPath, startVersion)), "Listing files in the delta log starting from %s", FileNames.listingPrefix(logPath, startVersion)); if (!files.hasNext()) { // We treat an empty directory as table not found throw new TableNotFoundException(tablePath.toString()); } return files; } catch (FileNotFoundException e) { throw new TableNotFoundException(tablePath.toString()); } catch (IOException io) { throw new UncheckedIOException("Failed to list the files in delta log", io); } } /** * Returns the commit version and timestamps of all commits starting from version {@code start}. * Guarantees that the commits returned have both monotonically increasing versions and * timestamps. */ private static List getCommits(Engine engine, Path logPath, long start) throws TableNotFoundException { CloseableIterator commits = listFrom(engine, logPath, start) .filter(fs -> FileNames.isCommitFile(getName(fs.getPath()))) .map(fs -> new Commit(FileNames.deltaVersion(fs.getPath()), fs.getModificationTime())); return monotonizeCommitTimestamps(commits); } /** * Makes sure that the commit timestamps are monotonically increasing with respect to commit * versions. Requires the input commits to be sorted by the commit version. */ private static List monotonizeCommitTimestamps(CloseableIterator commits) { List monotonizedCommits = new ArrayList<>(); long prevTimestamp = Long.MIN_VALUE; long prevVersion = Long.MIN_VALUE; while (commits.hasNext()) { Commit newElem = commits.next(); assert (prevVersion < newElem.version); // Verify commits are ordered if (prevTimestamp >= newElem.timestamp) { logger.warn( "Found Delta commit {} with a timestamp {} which is greater than the next " + "commit timestamp {}.", prevVersion, prevTimestamp, newElem.timestamp); newElem = new Commit(newElem.version, prevTimestamp + 1); } monotonizedCommits.add(newElem); prevTimestamp = newElem.timestamp; prevVersion = newElem.version; } return monotonizedCommits; } /** Returns the latest commit that happened at or before {@code timestamp} */ private static Optional lastCommitBeforeOrAtTimestamp( List commits, long timestamp) { int i = -1; while (i + 1 < commits.size() && commits.get(i + 1).timestamp <= timestamp) { i++; } return Optional.ofNullable((i < 0) ? null : commits.get(i)); } public static class Commit { private final long version; private final long timestamp; Commit(long version, long timestamp) { this.version = version; this.timestamp = timestamp; } public long getVersion() { return version; } public long getTimestamp() { return timestamp; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } Commit other = (Commit) o; return Objects.equals(version, other.version) && Objects.equals(timestamp, other.timestamp); } @Override public int hashCode() { return Objects.hash(version, timestamp); } } /** * Returns a function for resolving version-to-file-status given a list of ratified staged * commits. We prefer to read a ratified commit over the published file whenever it is present. * *

Note, this assumes that for any version provided to the function, it has already been * validated that the version _should_ exist by listing the _delta_log and finding the earliest * available commit. * *

DeltaHistoryManager doesn't, and has never, done a full LIST of the _delta_log. Instead, it * only lists to find the earliest commit, exists early, and then relies on the assumption that * any commits after the earliest commit exist and are contiguous. With CCV2, we continue this * assumption, such that for any version after the earliest available commit, the commit must * exist either in the list of ratified commits provided by the catalog or on the file-system. */ private static Function getVersionToFileStatusFunction( List catalogCommits, Path logPath) { Map versionToFileStatusMap = new HashMap<>(); for (ParsedCatalogCommitData catalogCommit : catalogCommits) { versionToFileStatusMap.put(catalogCommit.getVersion(), catalogCommit.getFileStatus()); } return version -> { if (versionToFileStatusMap.containsKey(version)) { return versionToFileStatusMap.get(version); } else { return FileStatus.of( FileNames.deltaFile(logPath, version), /* path */ 0, /* size */ 0 /* modification time */); } }; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaLogActionUtils.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal; import static io.delta.kernel.internal.DeltaErrors.*; import static io.delta.kernel.internal.fs.Path.getName; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static io.delta.kernel.internal.util.Utils.toCloseableIterator; import io.delta.kernel.CommitActions; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.InvalidTableException; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.exceptions.TableNotFoundException; import io.delta.kernel.internal.actions.*; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.lang.ListUtils; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.internal.util.FileNames.DeltaLogFileType; import io.delta.kernel.internal.util.Tuple2; import io.delta.kernel.types.*; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.CloseableIterator.BreakableFilterResult; import io.delta.kernel.utils.FileStatus; import java.io.FileNotFoundException; import java.io.IOException; import java.io.UncheckedIOException; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Exposes APIs to read the raw actions within the *commit files* of the _delta_log. This is used * for CDF, streaming, and more. */ public class DeltaLogActionUtils { private DeltaLogActionUtils() {} private static final Logger logger = LoggerFactory.getLogger(DeltaLogActionUtils.class); ///////////////// // Public APIs // ///////////////// /** * Represents a Delta action. This is used to request which actions to read from the commit files * in {@link TableImpl#getChanges(Engine, long, long, Set)}. * *

See the Delta protocol for more details * https://github.com/delta-io/delta/blob/master/PROTOCOL.md#actions */ public enum DeltaAction { REMOVE("remove", RemoveFile.FULL_SCHEMA), ADD("add", AddFile.FULL_SCHEMA), METADATA("metaData", Metadata.FULL_SCHEMA), PROTOCOL("protocol", Protocol.FULL_SCHEMA), COMMITINFO("commitInfo", CommitInfo.FULL_SCHEMA), CDC("cdc", AddCDCFile.FULL_SCHEMA), TXN("txn", SetTransaction.FULL_SCHEMA), DOMAINMETADATA("domainMetadata", DomainMetadata.FULL_SCHEMA); public final String colName; public final StructType schema; DeltaAction(String colName, StructType schema) { this.colName = colName; this.schema = schema; } } /** * For a table get the list of commit log files for the provided version range. * * @param tablePath path for the given table * @param startVersion start version of the range (inclusive) * @param endVersionOpt end version of the range (inclusive) * @return the list of commit files in increasing order between startVersion and endVersion * @throws TableNotFoundException if the table does not exist or if it is not a delta table * @throws KernelException if a commit file does not exist for any of the versions in the provided * range * @throws KernelException if provided an invalid version range */ public static List getCommitFilesForVersionRange( Engine engine, Path tablePath, long startVersion, Optional endVersionOpt) { logger.info( "{}: Getting the commit files for versions [{}, {}]", tablePath, startVersion, endVersionOpt); // Validate arguments endVersionOpt.ifPresent( endVersion -> { if (startVersion < 0 || endVersion < startVersion) { throw invalidVersionRange(startVersion, endVersion); } }); // Get any available commit files within the version range final List commitFiles = listDeltaLogFilesAsIter( engine, Collections.singleton(DeltaLogFileType.COMMIT), tablePath, startVersion, endVersionOpt, false /* mustBeRecreatable */) .toInMemoryList(); // There are no available commit files within the version range. // This can be due to (1) an empty directory, (2) no valid delta files in the directory, // (3) only delta files less than startVersion prefix (4) only delta files after endVersion if (commitFiles.isEmpty()) { throw noCommitFilesFoundForVersionRange(tablePath.toString(), startVersion, endVersionOpt); } // Verify commit files found // (check that they are continuous and start with startVersion and end with endVersion) verifyDeltaVersions(commitFiles, startVersion, endVersionOpt, tablePath); return commitFiles; } /** * Returns a {@link CloseableIterator} of files of type $fileTypes in the _delta_log directory of * the given $tablePath, in increasing order from $startVersion to the optional $endVersion. * * @throws TableNotFoundException if the table or its _delta_log does not exist * @throws KernelException if mustBeRecreatable is true, endVersionOpt is present, and the * _delta_log history has been truncated so that we cannot load the desired end version */ public static CloseableIterator listDeltaLogFilesAsIter( Engine engine, Set fileTypes, Path tablePath, long startVersion, Optional endVersionOpt, boolean mustBeRecreatable) { checkArgument(!fileTypes.isEmpty(), "At least one file type must be provided"); endVersionOpt.ifPresent( endVersion -> { checkArgument( endVersion >= startVersion, "endVersion=%s provided is less than startVersion=%s", endVersion, startVersion); }); final Path logPath = new Path(tablePath, "_delta_log"); logger.info( "Listing log files types={} in path={} starting from {} and ending with {}", fileTypes, logPath, startVersion, endVersionOpt); // This variable is used to help determine if we should throw an error if the table history is // not reconstructable. Only commit and checkpoint files are applicable. // Must be final to be used in lambda final AtomicBoolean hasReturnedCommitOrCheckpoint = new AtomicBoolean(false); return listLogDir(engine, tablePath, startVersion) .breakableFilter( fs -> { String fileName = getName(fs.getPath()); if (fileTypes.contains(DeltaLogFileType.COMMIT) && FileNames.isCommitFile(fileName)) { // Here, we do nothing (we will consume this file). } else if (fileTypes.contains(DeltaLogFileType.LOG_COMPACTION) && FileNames.isLogCompactionFile(fileName)) { // Here, we do nothing (we will consume this file). } else if (fileTypes.contains(DeltaLogFileType.CHECKPOINT) && FileNames.isCheckpointFile(fileName) && fs.getSize() > 0) { // Checkpoint files of 0 size are invalid but may be ignored silently when read, // hence we ignore them so that we never pick up such checkpoints. // Here, we do nothing (we will consume this file). } else if (fileTypes.contains(DeltaLogFileType.CHECKSUM) && FileNames.isChecksumFile(fileName)) { // Here, we do nothing (we will consume this file). } else { logger.debug("Ignoring file {} as it is not of the desired type", fs.getPath()); return BreakableFilterResult.EXCLUDE; // Here, we exclude and filter out this file. } final long fileVersion; if (FileNames.isLogCompactionFile(fileName)) { Tuple2 compactionVersions = FileNames.logCompactionVersions(new Path(fs.getPath())); // We use start version here. Below this is used to determine if we should stop // listing because we've listed past the required version. But with a log compaction // file, if the end version is passed the requested version, we don't want to stop, // we just won't use the compaction file. fileVersion = compactionVersions._1; // Now check if the compaction end version is too far in the future, and don't // include this file if it is if (endVersionOpt.isPresent()) { final long endVersion = endVersionOpt.get(); if (compactionVersions._2 > endVersion) { logger.debug( "Excluding compaction file as it covers past the end version {}", fileName); return BreakableFilterResult.EXCLUDE; } } } else { fileVersion = FileNames.getFileVersion(new Path(fs.getPath())); } if (fileVersion < startVersion) { throw new RuntimeException( String.format( "Listing files in %s with startVersion %s yet found file %s.", logPath, startVersion, fs.getPath())); } if (endVersionOpt.isPresent()) { final long endVersion = endVersionOpt.get(); if (fileVersion > endVersion) { if (mustBeRecreatable && !hasReturnedCommitOrCheckpoint.get()) { final long earliestVersion = DeltaHistoryManager.getEarliestRecreatableCommit( engine, logPath, Optional.empty()); throw DeltaErrors.versionBeforeFirstAvailableCommit( tablePath.toString(), endVersion, earliestVersion); } else { logger.debug( "Stopping listing; found file {} with version greater than endVersion {}", fs.getPath(), endVersion); return BreakableFilterResult.BREAK; } } } if (FileNames.isCommitFile(fileName) || FileNames.isCheckpointFile(fileName) || FileNames.isLogCompactionFile(fileName)) { hasReturnedCommitOrCheckpoint.set(true); } return BreakableFilterResult.INCLUDE; }); } /** * Returns CommitActions for each commit file. CommitActions are ordered by increasing version. * *

This function automatically: * *

    *
  • Performs protocol validation by reading and validating the protocol action *
  • Extracts commit timestamp using inCommitTimestamp if available, otherwise file * modification time *
  • Filters out protocol and commitInfo actions if not requested in actionSet *
*/ public static CloseableIterator getActionsFromCommitFilesWithProtocolValidation( Engine engine, String tablePath, List commitFiles, Set actionSet) { // For each commit file, create a CommitActions return toCloseableIterator(commitFiles.iterator()) .map(commitFile -> new CommitActionsImpl(engine, commitFile, tablePath, actionSet)); } ////////////////////// // Private helpers // ///////////////////// /** Column name storing the commit version for a given file action */ private static final String COMMIT_VERSION_COL_NAME = "version"; private static final DataType COMMIT_VERSION_DATA_TYPE = LongType.LONG; private static final StructField COMMIT_VERSION_STRUCT_FIELD = new StructField(COMMIT_VERSION_COL_NAME, COMMIT_VERSION_DATA_TYPE, false /* nullable */); /** Column name storing the commit timestamp for a given file action */ private static final String COMMIT_TIMESTAMP_COL_NAME = "timestamp"; private static final DataType COMMIT_TIMESTAMP_DATA_TYPE = LongType.LONG; private static final StructField COMMIT_TIMESTAMP_STRUCT_FIELD = new StructField(COMMIT_TIMESTAMP_COL_NAME, COMMIT_TIMESTAMP_DATA_TYPE, false /* nullable */); /** * Given a list of delta versions, verifies that they are (1) contiguous (2) versions starts with * expectedStartVersion and (3) end with expectedEndVersion. Throws an exception if any of these * are not true. * *

Public to expose for testing only. * * @param commitFiles in sorted increasing order according to the commit version */ static void verifyDeltaVersions( List commitFiles, long expectedStartVersion, Optional expectedEndVersionOpt, Path tablePath) { List commitVersions = commitFiles.stream() .map(fs -> FileNames.deltaVersion(new Path(fs.getPath()))) .collect(Collectors.toList()); for (int i = 1; i < commitVersions.size(); i++) { if (commitVersions.get(i) != commitVersions.get(i - 1) + 1) { throw new InvalidTableException( tablePath.toString(), String.format( "Missing delta files: versions are not contiguous: (%s)", commitVersions)); } } if (commitVersions.isEmpty() || !Objects.equals(commitVersions.get(0), expectedStartVersion)) { throw startVersionNotFound( tablePath.toString(), expectedStartVersion, commitVersions.isEmpty() ? Optional.empty() : Optional.of(commitVersions.get(0))); } expectedEndVersionOpt.ifPresent( expectedEndVersion -> { if (!Objects.equals(ListUtils.getLast(commitVersions), expectedEndVersion)) { throw endVersionNotFound( tablePath.toString(), expectedEndVersion, ListUtils.getLast(commitVersions)); } }); } /** * Gets an iterator of files in the _delta_log directory starting with the startVersion. * * @throws TableNotFoundException if the directory does not exist */ private static CloseableIterator listLogDir( Engine engine, Path tablePath, long startVersion) { final Path logPath = new Path(tablePath, "_delta_log"); try { return wrapEngineExceptionThrowsIO( () -> engine.getFileSystemClient().listFrom(FileNames.listingPrefix(logPath, startVersion)), "Listing from %s", FileNames.listingPrefix(logPath, startVersion)); } catch (FileNotFoundException e) { // Did not find the _delta_log directory. throw new TableNotFoundException(tablePath.toString()); } catch (IOException io) { throw new UncheckedIOException("Failed to list the files in delta log", io); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/InternalScanFileUtils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal; import io.delta.kernel.Scan; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; import io.delta.kernel.expressions.Column; import io.delta.kernel.internal.actions.AddFile; import io.delta.kernel.internal.actions.DeletionVectorDescriptor; import io.delta.kernel.internal.data.GenericRow; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.util.VectorUtils; import io.delta.kernel.types.DataType; import io.delta.kernel.types.StringType; import io.delta.kernel.types.StructField; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.FileStatus; import java.net.URI; import java.util.HashMap; import java.util.Map; import java.util.Optional; /** * Utilities to extract information out of the scan file rows returned by {@link * Scan#getScanFiles(Engine)}. */ public class InternalScanFileUtils { private InternalScanFileUtils() {} private static final String TABLE_ROOT_COL_NAME = "tableRoot"; private static final DataType TABLE_ROOT_DATA_TYPE = StringType.STRING; /** {@link Column} expression referring to the `partitionValues` in scan `add` file. */ public static final Column ADD_FILE_PARTITION_COL_REF = new Column(new String[] {"add", "partitionValues"}); public static StructField TABLE_ROOT_STRUCT_FIELD = new StructField(TABLE_ROOT_COL_NAME, TABLE_ROOT_DATA_TYPE, false /* nullable */); // TODO update this when stats columns are dropped from the returned scan files /** * Schema of the returned scan files. May have an additional column "add.stats" at the end of the * "add" columns that is not represented in the schema here. This column is conditionally read * when a valid data skipping filter can be generated. */ public static final StructType SCAN_FILE_SCHEMA = new StructType() .add("add", AddFile.SCHEMA_WITHOUT_STATS) // NOTE: table root is temporary, until the path in `add.path` is converted to // an absolute path. https://github.com/delta-io/delta/issues/2089 .add(TABLE_ROOT_STRUCT_FIELD); /** * Schema of the returned scan files when {@link ScanImpl#getScanFiles(Engine, boolean)} is called * with {@code includeStats=true}. */ public static final StructType SCAN_FILE_SCHEMA_WITH_STATS = new StructType().add("add", AddFile.SCHEMA_WITH_STATS).add(TABLE_ROOT_STRUCT_FIELD); public static final int ADD_FILE_ORDINAL = SCAN_FILE_SCHEMA.indexOf("add"); private static final StructType ADD_FILE_SCHEMA = (StructType) SCAN_FILE_SCHEMA.get("add").getDataType(); private static final int ADD_FILE_PATH_ORDINAL = ADD_FILE_SCHEMA.indexOf("path"); private static final int ADD_FILE_PARTITION_VALUES_ORDINAL = ADD_FILE_SCHEMA.indexOf("partitionValues"); private static final int ADD_FILE_SIZE_ORDINAL = ADD_FILE_SCHEMA.indexOf("size"); private static final int ADD_FILE_MOD_TIME_ORDINAL = ADD_FILE_SCHEMA.indexOf("modificationTime"); private static final int ADD_FILE_DATA_CHANGE_ORDINAL = ADD_FILE_SCHEMA.indexOf("dataChange"); private static final int ADD_FILE_DV_ORDINAL = ADD_FILE_SCHEMA.indexOf("deletionVector"); private static final int TABLE_ROOT_ORDINAL = SCAN_FILE_SCHEMA.indexOf(TABLE_ROOT_COL_NAME); public static final int ADD_FILE_STATS_ORDINAL = AddFile.SCHEMA_WITH_STATS.indexOf("stats"); /** * Get the {@link FileStatus} of {@code AddFile} from given scan file {@link Row}. The {@link * FileStatus} contains file metadata about the file. * * @param scanFileInfo {@link Row} representing one scan file. * @return a {@link FileStatus} object created from the given scan file row. */ public static FileStatus getAddFileStatus(Row scanFileInfo) { Row addFile = getAddFileEntry(scanFileInfo); String path = addFile.getString(ADD_FILE_PATH_ORDINAL); long size = addFile.getLong(ADD_FILE_SIZE_ORDINAL); long modificationTime = addFile.getLong(ADD_FILE_MOD_TIME_ORDINAL); // TODO: this is hack until the path in `add.path` is converted to an absolute path String tableRoot = scanFileInfo.getString(TABLE_ROOT_ORDINAL); String absolutePath = new Path(new Path(URI.create(tableRoot)), new Path(URI.create(path))).toString(); return FileStatus.of(absolutePath, size, modificationTime); } /** * Get the partition columns and values belonging to the {@code AddFile} from given scan file row. * * @param scanFileInfo {@link Row} representing one scan file. * @return Map of partition column name to partition column value. */ public static Map getPartitionValues(Row scanFileInfo) { Row addFile = getAddFileEntry(scanFileInfo); return VectorUtils.toJavaMap(addFile.getMap(ADD_FILE_PARTITION_VALUES_ORDINAL)); } /** * Helper method to get the {@code AddFile} struct from the scan file row. * * @param scanFileInfo * @return {@link Row} representing the {@code AddFile} * @throws IllegalArgumentException If the scan file row doesn't contain {@code add} file entry. */ protected static Row getAddFileEntry(Row scanFileInfo) { if (scanFileInfo.isNullAt(ADD_FILE_ORDINAL)) { throw new IllegalArgumentException("There is no `add` entry in the scan file row"); } return scanFileInfo.getStruct(ADD_FILE_ORDINAL); } /** * Create a scan file row conforming to the schema {@link #SCAN_FILE_SCHEMA} for given file * status. This is used when creating the ScanFile row for reading commit or checkpoint files. * * @param fileStatus * @return */ public static Row generateScanFileRow(FileStatus fileStatus) { Row addFile = new GenericRow( ADD_FILE_SCHEMA, new HashMap() { { put(ADD_FILE_PATH_ORDINAL, fileStatus.getPath()); put(ADD_FILE_PARTITION_VALUES_ORDINAL, null); // partitionValues put(ADD_FILE_SIZE_ORDINAL, fileStatus.getSize()); put(ADD_FILE_MOD_TIME_ORDINAL, fileStatus.getModificationTime()); put(ADD_FILE_DATA_CHANGE_ORDINAL, null); // dataChange put(ADD_FILE_DV_ORDINAL, null); // deletionVector } }); return new GenericRow( SCAN_FILE_SCHEMA, new HashMap() { { put(ADD_FILE_ORDINAL, addFile); put(TABLE_ROOT_ORDINAL, "/"); } }); } /** * Create a {@link DeletionVectorDescriptor} from {@code add} entry in the given scan file row. * * @param scanFile {@link Row} representing one scan file. * @return */ public static DeletionVectorDescriptor getDeletionVectorDescriptorFromRow(Row scanFile) { Row addFile = getAddFileEntry(scanFile); return DeletionVectorDescriptor.fromRow(addFile.getStruct(ADD_FILE_DV_ORDINAL)); } /** * Get a references column for given partition column name in partitionValues_parsed column in * scan file row. * * @param partitionColName Partition column name * @return {@link Column} reference */ public static Column getPartitionValuesParsedRefInAddFile(String partitionColName) { return new Column(new String[] {"add", "partitionValues_parsed", partitionColName}); } public static Optional getBaseRowId(Row scanFile) { Row addFileRow = getAddFileEntry(scanFile); return new AddFile(addFileRow).getBaseRowId(); } public static Optional getDefaultRowCommitVersion(Row scanFile) { Row addFileRow = getAddFileEntry(scanFile); return new AddFile(addFileRow).getDefaultRowCommitVersion(); } public static String getFilePath(Row scanFile) { Row addFileRow = getAddFileEntry(scanFile); return new AddFile(addFileRow).getPath(); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/PaginatedScanImpl.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal; import io.delta.kernel.PaginatedScan; import io.delta.kernel.PaginatedScanFilesIterator; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.internal.replay.PageToken; import io.delta.kernel.internal.replay.PaginatedScanFilesIteratorImpl; import io.delta.kernel.internal.replay.PaginationContext; import io.delta.kernel.internal.snapshot.LogSegment; import io.delta.kernel.utils.CloseableIterator; import java.util.Optional; /** Implementation of {@link PaginatedScan} */ public class PaginatedScanImpl implements PaginatedScan { private final PaginationContext paginationContext; private final ScanImpl baseScan; private final long pageSize; private final Optional pageTokenOpt; public PaginatedScanImpl( ScanImpl baseScan, String tablePath, long tableVersion, long pageSize, LogSegment logSegment, Optional predicate, Optional pageTokenRowOpt) { this.baseScan = baseScan; this.pageSize = pageSize; this.pageTokenOpt = pageTokenRowOpt.map(PageToken::fromRow); // TODO: get hash value of predicate & log segment and check values in pagination context this.paginationContext = pageTokenOpt .map( token -> PaginationContext.forPageWithPageToken( tablePath, tableVersion, logSegment.hashCode(), predicate.hashCode(), pageSize, token)) .orElseGet( () -> PaginationContext.forFirstPage( tablePath, tableVersion, logSegment.hashCode(), predicate.hashCode(), pageSize)); } @Override public Optional getRemainingFilter() { return baseScan.getRemainingFilter(); } @Override public Row getScanState(Engine engine) { return baseScan.getScanState(engine); } @Override public PaginatedScanFilesIterator getScanFiles(Engine engine) { return this.getScanFiles(engine, false /* include stats */); } public PaginatedScanFilesIterator getScanFiles(Engine engine, boolean includeStates) { CloseableIterator filteredScanFilesIter = baseScan.getScanFiles(engine, includeStates, Optional.of(paginationContext)); return new PaginatedScanFilesIteratorImpl(filteredScanFilesIter, paginationContext); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/ReplaceTableTransactionBuilderImpl.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal; import static io.delta.kernel.internal.DeltaErrors.requireSchemaForReplaceTable; import io.delta.kernel.Operation; import io.delta.kernel.Transaction; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.TableNotFoundException; import java.util.Optional; public class ReplaceTableTransactionBuilderImpl extends TransactionBuilderImpl { public ReplaceTableTransactionBuilderImpl(TableImpl table, String engineInfo) { super(table, engineInfo, Operation.REPLACE_TABLE); } @Override public Transaction build(Engine engine) { try { withMaxRetries(0); // We don't support conflict resolution yet so disable retries for now schema.orElseThrow(() -> requireSchemaForReplaceTable()); SnapshotImpl snapshot = table.getLatestSnapshot(engine); return buildTransactionInternal(engine, true, Optional.of(snapshot)); } catch (TableNotFoundException tblf) { throw new TableNotFoundException( tblf.getTablePath(), "Trying to replace a table that does not exist."); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/ReplaceTableTransactionBuilderV2Impl.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Collections.emptyMap; import static java.util.Objects.requireNonNull; import io.delta.kernel.Operation; import io.delta.kernel.Transaction; import io.delta.kernel.engine.Engine; import io.delta.kernel.expressions.Column; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.transaction.DataLayoutSpec; import io.delta.kernel.transaction.ReplaceTableTransactionBuilder; import io.delta.kernel.types.StructType; import java.util.*; public class ReplaceTableTransactionBuilderV2Impl implements ReplaceTableTransactionBuilder { /** * Delta-specific properties that should be preserved during REPLACE operations, unless their * value is specifically set (overridden) during the REPLACE. All other properties should be * reset. * *

For example, suppose at the time of REPLACE the table has property 'delta.foo' = 'bar' and * that such property is included in this set. * *

    *
  • If the REPLACE statement does not specify 'delta.foo', then the new table will still have * 'delta.foo' = 'bar'. *
  • If the REPLACE statement specifies 'delta.foo' = 'baz', then the new table will of course * have 'delta.foo' = 'baz'. *
*/ static final Set TABLE_PROPERTY_KEYS_TO_PRESERVE = new HashSet() { { add(TableConfig.COLUMN_MAPPING_MAX_COLUMN_ID.getKey()); // Must retain all ICT properties, else a client would not know when ICT was enabled, // which could result in a failed query or incorrect results. // // If ICT is explicitly disabled during REPLACE (or during any operation), we should then // explicitly remove the ICT enablement version and timestamp properties. add(TableConfig.IN_COMMIT_TIMESTAMPS_ENABLED.getKey()); add(TableConfig.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.getKey()); add(TableConfig.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.getKey()); } }; private final SnapshotImpl snapshot; private final StructType schema; private final String engineInfo; private Optional> tableProperties = Optional.empty(); private Optional dataLayoutSpec = Optional.empty(); private Optional userProvidedMaxRetries = Optional.empty(); public ReplaceTableTransactionBuilderV2Impl( SnapshotImpl snapshot, StructType schema, String engineInfo) { this.snapshot = requireNonNull(snapshot, "snapshot is null"); this.schema = requireNonNull(schema, "schema is null"); this.engineInfo = requireNonNull(engineInfo, "engineInfo is null"); TableFeatures.validateKernelCanWriteToTable( snapshot.getProtocol(), snapshot.getMetadata(), snapshot.getPath()); } @Override public ReplaceTableTransactionBuilder withTableProperties(Map properties) { requireNonNull(properties, "properties cannot be null"); this.tableProperties = Optional.of( java.util.Collections.unmodifiableMap( TableConfig.validateAndNormalizeDeltaProperties(properties))); return this; } @Override public ReplaceTableTransactionBuilder withDataLayoutSpec(DataLayoutSpec spec) { requireNonNull(spec, "spec cannot be null"); this.dataLayoutSpec = Optional.of(spec); return this; } @Override public ReplaceTableTransactionBuilder withMaxRetries(int maxRetries) { checkArgument(maxRetries >= 0, "maxRetries must be >= 0"); this.userProvidedMaxRetries = Optional.of(maxRetries); return this; } @Override public Transaction build(Engine engine) { requireNonNull(engine, "engine cannot be null"); Optional> partitionColumns = dataLayoutSpec .filter(DataLayoutSpec::hasPartitioning) .map(DataLayoutSpec::getPartitionColumnsAsStrings); Optional> clusteringColumns = dataLayoutSpec .filter(DataLayoutSpec::hasClustering) .map(DataLayoutSpec::getClusteringColumns); TransactionMetadataFactory.Output txnMetadata = TransactionMetadataFactory.buildReplaceTableMetadata( snapshot.getPath(), snapshot, schema, tableProperties.orElse(emptyMap()), partitionColumns, clusteringColumns); return new TransactionImpl( true, // isCreateOrReplace snapshot.getDataPath(), Optional.of(snapshot), engineInfo, Operation.REPLACE_TABLE, txnMetadata.newProtocol, txnMetadata.newMetadata, snapshot.getCommitter(), Optional.empty(), // no setTransaction for replace table txnMetadata.physicalNewClusteringColumns, // We don't support conflict resolution yet for replace so disable retries for now Optional.of(0), 0, // logCompactionInterval System::currentTimeMillis); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/ScanBuilderImpl.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal; import io.delta.kernel.PaginatedScan; import io.delta.kernel.ScanBuilder; import io.delta.kernel.data.Row; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.replay.LogReplay; import io.delta.kernel.metrics.SnapshotReport; import io.delta.kernel.types.StructType; import java.util.Optional; /** Implementation of {@link ScanBuilder}. */ public class ScanBuilderImpl implements ScanBuilder { private final Path dataPath; private final long tableVersion; private final Protocol protocol; private final Metadata metadata; private final StructType snapshotSchema; private final LogReplay logReplay; private final SnapshotReport snapshotReport; private StructType readSchema; private Optional predicate; public ScanBuilderImpl( Path dataPath, long tableVersion, Protocol protocol, Metadata metadata, StructType snapshotSchema, LogReplay logReplay, SnapshotReport snapshotReport) { this.dataPath = dataPath; this.tableVersion = tableVersion; this.protocol = protocol; this.metadata = metadata; this.snapshotSchema = snapshotSchema; this.logReplay = logReplay; this.readSchema = snapshotSchema; this.predicate = Optional.empty(); this.snapshotReport = snapshotReport; } @Override public ScanBuilder withFilter(Predicate predicate) { if (this.predicate.isPresent()) { throw new IllegalArgumentException("There already exists a filter in current builder"); } this.predicate = Optional.of(predicate); return this; } @Override public ScanBuilder withReadSchema(StructType readSchema) { // TODO: Validate that readSchema is a subset of the table schema or that extra fields are // metadata columns. this.readSchema = readSchema; return this; } @Override public ScanImpl build() { return new ScanImpl( snapshotSchema, readSchema, protocol, metadata, logReplay, predicate, dataPath, snapshotReport); } @Override public PaginatedScan buildPaginated(long pageSize, Optional pageTokenRowOpt) { ScanImpl baseScan = this.build(); return new PaginatedScanImpl( baseScan, dataPath.toString(), tableVersion, pageSize, logReplay.getLogSegment(), predicate, pageTokenRowOpt); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/ScanImpl.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal; import static io.delta.kernel.internal.DeltaErrors.wrapEngineException; import static io.delta.kernel.internal.skipping.StatsSchemaHelper.getStatsSchema; import static io.delta.kernel.internal.util.PartitionUtils.rewritePartitionPredicateOnCheckpointFileSchema; import static io.delta.kernel.internal.util.PartitionUtils.rewritePartitionPredicateOnScanFileSchema; import static java.util.function.Function.identity; import static java.util.stream.Collectors.toMap; import io.delta.kernel.Scan; import io.delta.kernel.data.*; import io.delta.kernel.engine.Engine; import io.delta.kernel.expressions.*; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.data.ScanStateRow; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.metrics.ScanMetrics; import io.delta.kernel.internal.metrics.ScanReportImpl; import io.delta.kernel.internal.metrics.Timer; import io.delta.kernel.internal.replay.LogReplay; import io.delta.kernel.internal.replay.PaginationContext; import io.delta.kernel.internal.rowtracking.MaterializedRowTrackingColumn; import io.delta.kernel.internal.rowtracking.RowTracking; import io.delta.kernel.internal.skipping.DataSkippingPredicate; import io.delta.kernel.internal.skipping.DataSkippingUtils; import io.delta.kernel.internal.util.*; import io.delta.kernel.metrics.ScanReport; import io.delta.kernel.metrics.SnapshotReport; import io.delta.kernel.types.MetadataColumnSpec; import io.delta.kernel.types.StructField; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterator; import java.io.IOException; import java.util.*; import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Implementation of {@link Scan} */ public class ScanImpl implements Scan { private static final Logger logger = LoggerFactory.getLogger(ScanImpl.class); /** * Schema of the snapshot from the Delta log being scanned in this scan. It is a logical schema * with metadata properties to derive the physical schema. */ private final StructType snapshotSchema; /** Schema that we actually want to read. */ private final StructType readSchema; private final Protocol protocol; private final Metadata metadata; private final LogReplay logReplay; private final Path dataPath; private final Optional filter; private final Optional> partitionAndDataFilters; private final Supplier> partitionColToStructFieldMap; private boolean accessedScanFiles; private final SnapshotReport snapshotReport; private final ScanMetrics scanMetrics = new ScanMetrics(); public ScanImpl( StructType snapshotSchema, StructType readSchema, Protocol protocol, Metadata metadata, LogReplay logReplay, Optional filter, Path dataPath, SnapshotReport snapshotReport) { this.snapshotSchema = snapshotSchema; this.readSchema = readSchema; this.protocol = protocol; this.metadata = metadata; this.logReplay = logReplay; this.filter = filter; this.partitionAndDataFilters = splitFilters(filter); this.dataPath = dataPath; this.partitionColToStructFieldMap = () -> { Set partitionColNames = metadata.getPartitionColNames(); return metadata.getSchema().fields().stream() .filter(field -> partitionColNames.contains(field.getName().toLowerCase(Locale.ROOT))) .collect(toMap(field -> field.getName().toLowerCase(Locale.ROOT), identity())); }; this.snapshotReport = snapshotReport; } /** * Get an iterator of data files in this version of scan that survived the predicate pruning. * * @return data in {@link ColumnarBatch} batch format. Each row correspond to one survived file. */ @Override public CloseableIterator getScanFiles(Engine engine) { return getScanFiles(engine, false /* includeStats */); } /** * Get an iterator of data files in this version of scan that survived the predicate pruning. * * @return data in {@link ColumnarBatch} batch format. Each row correspond to one survived file. */ public CloseableIterator getScanFiles( Engine engine, boolean includeStats) { return getScanFiles(engine, includeStats, Optional.empty() /* paginationContextOpt */); } /** * Get an iterator of data files in this version of scan that survived the predicate pruning. * *

When {@code includeStats=true} the JSON file statistics are always read from the log and * included in the returned columnar batches which have schema {@link * InternalScanFileUtils#SCAN_FILE_SCHEMA_WITH_STATS}. When {@code includeStats=false} the JSON * file statistics may or may not be present in the returned columnar batches. * * @param engine the {@link Engine} instance to use * @param includeStats whether to read and include the JSON statistics * @param paginationContextOpt pagination context if present * @return the surviving scan files as {@link FilteredColumnarBatch}s */ protected CloseableIterator getScanFiles( Engine engine, boolean includeStats, Optional paginationContextOpt) { if (accessedScanFiles) { throw new IllegalStateException("Scan files are already fetched from this instance"); } accessedScanFiles = true; // Generate data skipping filter and decide if we should read the stats column logger.info( "Trying to generate data skipping filter for data filter = {} and data schema = {}", getDataFilters(), metadata.getDataSchema()); Optional dataSkippingFilter = getDataSkippingFilter(); boolean hasDataSkippingFilter = dataSkippingFilter.isPresent(); boolean shouldReadStats = hasDataSkippingFilter || includeStats; logger.info("Generated data skipping filter = {}", dataSkippingFilter); Timer.Timed planningDuration = scanMetrics.totalPlanningTimer.start(); // ScanReportReporter stores the current context and can be invoked (in the future) with // `reportError` or `reportSuccess` to stop the planning duration timer and push a report to // the engine ScanReportReporter reportReporter = (exceptionOpt, isFullyConsumed) -> { planningDuration.stop(); ScanReport scanReport = new ScanReportImpl( dataPath.toString() /* tablePath */, logReplay.getVersion() /* table version */, snapshotSchema, snapshotReport.getReportUUID(), filter, readSchema, getPartitionsFilters() /* partitionPredicate */, dataSkippingFilter.map(p -> p), isFullyConsumed, scanMetrics, exceptionOpt); engine.getMetricsReporters().forEach(reporter -> reporter.report(scanReport)); }; try { // Get active AddFiles via log replay // If there is a partition predicate, construct a predicate to prune checkpoint files // while constructing the table state. CloseableIterator scanFileIter = logReplay.getAddFilesAsColumnarBatches( engine, shouldReadStats, getPartitionsFilters() .map( predicate -> rewritePartitionPredicateOnCheckpointFileSchema( predicate, partitionColToStructFieldMap.get())), scanMetrics, paginationContextOpt); // Apply partition pruning scanFileIter = applyPartitionPruning(engine, scanFileIter); // Apply data skipping if (hasDataSkippingFilter) { // there was a usable data skipping filter --> apply data skipping scanFileIter = applyDataSkipping(engine, scanFileIter, dataSkippingFilter.get()); } // TODO when !includeStats drop the stats column if present before returning return wrapWithMetricsReporting(scanFileIter, reportReporter); } catch (Exception e) { reportReporter.reportError(e); throw e; } } @Override public Row getScanState(Engine engine) { StructType physicalSchema = createPhysicalSchema(); return ScanStateRow.of( metadata, protocol, readSchema.toJson() /* logical schema */, physicalSchema.toJson(), dataPath.toUri().toString()); } @Override public Optional getRemainingFilter() { return getDataFilters(); } /** * Transform the logical schema requested by the connector into a physical schema that is passed * to the engine's parquet reader. * *

The logical-to-physical conversion is reversed in {@link Scan#transformPhysicalData(Engine, * Row, Row, CloseableIterator)} when physical data batches returned by the parquet reader are * converted into logical data batches requested by the connector. * *

The logical-to-physical conversion follows these high-level steps: * *

    *
  • Partition columns are excluded from the physical schema. *
  • Regular columns are converted based on the column mapping mode. *
  • Metadata columns are converted to their physical counterparts if applicable. *
  • Additional columns (such as the row index) are requested if necessary. *
* * @return The physical schema to read data from the data files. */ private StructType createPhysicalSchema() { ArrayList physicalFields = new ArrayList<>(); ColumnMapping.ColumnMappingMode mode = ColumnMapping.getColumnMappingMode(metadata.getConfiguration()); for (StructField logicalField : readSchema.fields()) { if (!metadata .getPartitionColNames() .contains(logicalField.getName().toLowerCase(Locale.ROOT))) { physicalFields.addAll(convertField(logicalField, mode)); } } if (protocol.getReaderFeatures().contains("deletionVectors") && physicalFields.stream() .map(StructField::getMetadataColumnSpec) .noneMatch(MetadataColumnSpec.ROW_INDEX::equals)) { // If the row index column is not already present, add it to the physical read schema physicalFields.add(SchemaUtils.asInternalColumn(StructField.DEFAULT_ROW_INDEX_COLUMN)); } return new StructType(physicalFields); } private List convertField( StructField logicalField, ColumnMapping.ColumnMappingMode mode) { if (logicalField.isDataColumn()) { return Collections.singletonList( ColumnMapping.convertToPhysicalColumn(logicalField, snapshotSchema, mode)); } if (RowTracking.isRowTrackingColumn(logicalField)) { return MaterializedRowTrackingColumn.convertToPhysicalColumn( logicalField, readSchema, metadata); } // As of now, metadata columns other than row tracking columns do not require any special // handling, so we can just add them to the physical schema as is. return Collections.singletonList(logicalField); } private Optional> splitFilters(Optional filter) { return filter.map( predicate -> PartitionUtils.splitMetadataAndDataPredicates( predicate, metadata.getPartitionColNames())); } private Optional getDataFilters() { return removeAlwaysTrue(partitionAndDataFilters.map(filters -> filters._2)); } private Optional getPartitionsFilters() { return removeAlwaysTrue(partitionAndDataFilters.map(filters -> filters._1)); } /** Consider `ALWAYS_TRUE` as no predicate. */ private Optional removeAlwaysTrue(Optional predicate) { return predicate.filter(filter -> !filter.getName().equalsIgnoreCase("ALWAYS_TRUE")); } private CloseableIterator applyPartitionPruning( Engine engine, CloseableIterator scanFileIter) { Optional partitionPredicate = getPartitionsFilters(); if (!partitionPredicate.isPresent()) { // There is no partition filter, return the scan file iterator as is. return scanFileIter; } Predicate predicateOnScanFileBatch = rewritePartitionPredicateOnScanFileSchema( partitionPredicate.get(), partitionColToStructFieldMap.get()); return new CloseableIterator() { PredicateEvaluator predicateEvaluator = null; @Override public boolean hasNext() { return scanFileIter.hasNext(); } @Override public FilteredColumnarBatch next() { FilteredColumnarBatch next = scanFileIter.next(); if (predicateEvaluator == null) { predicateEvaluator = wrapEngineException( () -> engine .getExpressionHandler() .getPredicateEvaluator( next.getData().getSchema(), predicateOnScanFileBatch), "Get the predicate evaluator for partition pruning with schema=%s and" + " filter=%s", next.getData().getSchema(), predicateOnScanFileBatch); } ColumnVector newSelectionVector = wrapEngineException( () -> predicateEvaluator.eval(next.getData(), next.getSelectionVector()), "Evaluating the partition expression %s", predicateOnScanFileBatch); return new FilteredColumnarBatch(next.getData(), Optional.of(newSelectionVector)); } @Override public void close() throws IOException { scanFileIter.close(); } }; } private Optional getDataSkippingFilter() { return getDataFilters() .flatMap( dataFilters -> DataSkippingUtils.constructDataSkippingFilter( dataFilters, metadata.getDataSchema())); } private CloseableIterator applyDataSkipping( Engine engine, CloseableIterator scanFileIter, DataSkippingPredicate dataSkippingFilter) { // Get the stats schema // It's possible to instead provide the referenced columns when building the schema but // pruning it after is much simpler StructType prunedStatsSchema = DataSkippingUtils.pruneStatsSchema( getStatsSchema(metadata.getDataSchema(), dataSkippingFilter.getReferencedCollations()), dataSkippingFilter.getReferencedCols()); logger.info("For stats JSON parsing: prunedStatsSchema={}", prunedStatsSchema); // Skipping happens in two steps: // 1. The predicate produces false for any file whose stats prove we can safely skip it. A // value of true means the stats say we must keep the file, and null means we could not // determine whether the file is safe to skip, because its stats were missing/null. // 2. The coalesce(skip, true) converts null (= keep) to true Predicate filterToEval = new Predicate( "=", new ScalarExpression( "COALESCE", Arrays.asList(dataSkippingFilter, Literal.ofBoolean(true))), AlwaysTrue.ALWAYS_TRUE); PredicateEvaluator predicateEvaluator = wrapEngineException( () -> engine .getExpressionHandler() .getPredicateEvaluator(prunedStatsSchema, filterToEval), "Get the predicate evaluator for data skipping with schema=%s and filter=%s", prunedStatsSchema, filterToEval); return scanFileIter.map( filteredScanFileBatch -> { ColumnVector newSelectionVector = wrapEngineException( () -> predicateEvaluator.eval( DataSkippingUtils.parseJsonStats( engine, filteredScanFileBatch, prunedStatsSchema), filteredScanFileBatch.getSelectionVector()), "Evaluating the data skipping filter %s", filterToEval); return new FilteredColumnarBatch( filteredScanFileBatch.getData(), Optional.of(newSelectionVector)); }); } /** * Wraps a scan file iterator such that we emit {@link ScanReport} to the engine upon success and * failure. Since most of our scan building code is lazily executed (since it occurs as * maps/filters over an iterator) potential errors don't occur just within `getScanFile`s * execution, but rather may occur as the returned iterator is consumed. Similarly, we cannot * report a successful scan until the iterator has been fully consumed and the log read/filtered * etc. This means we cannot report the successful scan within `getScanFiles` but rather must * report after the iterator has been consumed. * *

This method wraps an inner scan file iterator with an outer iterator wrapper that reports * {@link ScanReport}s as needed. It reports a failed {@link ScanReport} in the case of any * exceptions originating from the inner iterator `next` and `hasNext` impl. It reports a complete * or incomplete {@link ScanReport} when the iterator is closed. */ private CloseableIterator wrapWithMetricsReporting( CloseableIterator scanIter, ScanReportReporter reporter) { return new CloseableIterator() { /* Whether this iterator has reported an error report */ private boolean errorReported = false; @Override public void close() throws IOException { try { // If a ScanReport has already been pushed in the case of an exception don't double report if (!errorReported) { if (!scanIter.hasNext()) { // The entire scan file iterator has been successfully consumed report a complete Scan reporter.reportCompleteScan(); } else { // The scan file iterator has NOT been fully consumed before being closed // We have no way of knowing the reason why, this could be due to an exception in the // connector code, or intentional early termination such as for a LIMIT query reporter.reportIncompleteScan(); } } } finally { scanIter.close(); } } @Override public boolean hasNext() { return wrapWithErrorReporting(() -> scanIter.hasNext()); } @Override public FilteredColumnarBatch next() { return wrapWithErrorReporting(() -> scanIter.next()); } private T wrapWithErrorReporting(Supplier s) { try { return s.get(); } catch (Exception e) { reporter.reportError(e); errorReported = true; throw e; } } }; } /** * Defines methods to report {@link ScanReport} to the engine. This allows us to avoid ambiguous * lambdas/anonymous classes as well as reuse the defined default methods. */ private interface ScanReportReporter { default void reportError(Exception e) { report(Optional.of(e), false /* isFullyConsumed */); } default void reportCompleteScan() { report(Optional.empty(), true /* isFullyConsumed */); } default void reportIncompleteScan() { report(Optional.empty(), false /* isFullyConsumed */); } /** Given an optional exception, reports a {@link ScanReport} to the engine */ void report(Optional exceptionOpt, boolean isFullyConsumed); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/SnapshotImpl.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal; import static io.delta.kernel.internal.TableConfig.*; import static io.delta.kernel.internal.TableConfig.TOMBSTONE_RETENTION; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.Operation; import io.delta.kernel.ScanBuilder; import io.delta.kernel.Snapshot; import io.delta.kernel.commit.CatalogCommitter; import io.delta.kernel.commit.Committer; import io.delta.kernel.commit.PublishFailedException; import io.delta.kernel.commit.PublishMetadata; import io.delta.kernel.engine.Engine; import io.delta.kernel.expressions.Column; import io.delta.kernel.internal.actions.CommitInfo; import io.delta.kernel.internal.actions.DomainMetadata; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.annotation.VisibleForTesting; import io.delta.kernel.internal.checkpoints.Checkpointer; import io.delta.kernel.internal.checksum.CRCInfo; import io.delta.kernel.internal.checksum.ChecksumUtils; import io.delta.kernel.internal.checksum.ChecksumWriter; import io.delta.kernel.internal.clustering.ClusteringMetadataDomain; import io.delta.kernel.internal.files.ParsedCatalogCommitData; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.lang.Lazy; import io.delta.kernel.internal.metrics.SnapshotQueryContext; import io.delta.kernel.internal.metrics.SnapshotReportImpl; import io.delta.kernel.internal.replay.CreateCheckpointIterator; import io.delta.kernel.internal.replay.LogReplay; import io.delta.kernel.internal.snapshot.LogSegment; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.internal.util.VectorUtils; import io.delta.kernel.metrics.SnapshotReport; import io.delta.kernel.statistics.SnapshotStatistics; import io.delta.kernel.transaction.ReplaceTableTransactionBuilder; import io.delta.kernel.transaction.UpdateTableTransactionBuilder; import io.delta.kernel.types.StructType; import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Implementation of {@link Snapshot}. */ public class SnapshotImpl implements Snapshot { private static final Logger logger = LoggerFactory.getLogger(SnapshotImpl.class); private final Path logPath; private final Path dataPath; private final long version; private final Lazy lazyLogSegment; private final LogReplay logReplay; private final Protocol protocol; private final Metadata metadata; private final Committer committer; /** * If this snapshot does not have the InCommitTimestamp (ICT) table feature enabled, then this is * always Optional.empty(). If it does, then this is: * *

    *
  • Optional.empty(): if the ICT value is not yet known (i.e. has not yet been read from the * CRC or CommitInfo) *
  • Optional.of(timestamp): if the ICT value has been read from the CRC or CommitInfo, or was * injected into this Snapshot at construction time (e.g. for a post-commit snapshot) *
*/ private Optional inCommitTimestampOpt; private Lazy lazySnapshotReport; private Lazy>> lazyClusteringColumns; /** * Indicates whether this snapshot was built as a "latest" snapshot query (i.e., no time-travel * parameters were provided). This is intent-based - it indicates what the user requested, not * whether the snapshot is actually the latest version. */ private final boolean wasBuiltAsLatest; // TODO: Do not take in LogReplay as a constructor argument. // TODO: Also take in clustering columns for post-commit snapshot public SnapshotImpl( Path dataPath, long version, Lazy lazyLogSegment, LogReplay logReplay, Protocol protocol, Metadata metadata, Committer committer, SnapshotQueryContext snapshotContext, Optional inCommitTimestampOpt) { checkArgument(version >= 0, "A snapshot cannot have version < 0"); this.logPath = new Path(dataPath, "_delta_log"); this.dataPath = dataPath; this.version = version; this.lazyLogSegment = lazyLogSegment; this.logReplay = logReplay; this.protocol = requireNonNull(protocol); this.metadata = requireNonNull(metadata); this.committer = committer; this.inCommitTimestampOpt = inCommitTimestampOpt; // TODO: Post-commit snapshots build a version-based SnapshotQueryContext // (see TransactionImpl.buildPostCommitSnapshotOpt), so isLatestQuery() may be false even // when this snapshot is intended to be the latest version. this.wasBuiltAsLatest = snapshotContext.isLatestQuery(); // We create the actual Snapshot report lazily (on first access) instead of eagerly in this // constructor because some Snapshot metrics, like {@link // io.delta.kernel.metrics.SnapshotMetricsResult#getLoadSnapshotTotalDurationNs}, are only // completed *after* the Snapshot has been constructed. this.lazySnapshotReport = new Lazy<>(() -> SnapshotReportImpl.forSuccess(snapshotContext)); this.lazyClusteringColumns = new Lazy<>( () -> ClusteringMetadataDomain.fromSnapshot(this) .map(ClusteringMetadataDomain::getClusteringColumns)); } ///////////////// // Public APIs // ///////////////// @Override public String getPath() { return dataPath.toString(); } @Override public long getVersion() { return version; } @Override public List getPartitionColumnNames() { return VectorUtils.toJavaList(getMetadata().getPartitionColumns()); } /** * Get the timestamp (in milliseconds since the Unix epoch) of the latest commit in this Snapshot. * *

When InCommitTimestampTableFeature is enabled, the timestamp is retrieved from the * CommitInfo of the latest commit in this Snapshot, which can result in an IO operation. * *

For non-ICT tables, this is the same as the file modification time of the latest commit in * this Snapshot. */ // TODO: Support reading from CRC file if available @Override public long getTimestamp(Engine engine) { if (IN_COMMIT_TIMESTAMPS_ENABLED.fromMetadata(metadata)) { if (!inCommitTimestampOpt.isPresent()) { final Optional commitInfoOpt = CommitInfo.tryReadCommitInfoFromDeltaFile( engine, getLogSegment().getDeltaFileAtEndVersion()); inCommitTimestampOpt = Optional.of( CommitInfo.extractRequiredIctFromCommitInfoOpt(commitInfoOpt, version, dataPath)); } return inCommitTimestampOpt.get(); } else { return getLogSegment().getDeltaFileAtEndVersion().getModificationTime(); } } @Override public StructType getSchema() { return getMetadata().getSchema(); } @Override public Optional getDomainMetadata(String domain) { return Optional.ofNullable(getActiveDomainMetadataMap().get(domain)) .map(DomainMetadata::getConfiguration); } @Override public Map getTableProperties() { return metadata.getConfiguration(); } @Override public SnapshotStatistics getStatistics() { return new SnapshotStatisticsImpl(); } @Override public ScanBuilder getScanBuilder() { return new ScanBuilderImpl( dataPath, version, protocol, metadata, getSchema(), logReplay, getSnapshotReport()); } @Override public UpdateTableTransactionBuilder buildUpdateTableTransaction( String engineInfo, Operation operation) { return new UpdateTableTransactionBuilderImpl(this, engineInfo, operation); } @Override public Snapshot publish(Engine engine) throws PublishFailedException { final List allCatalogCommits = getLogSegment().getAllCatalogCommits(); final boolean isFileSystemBasedTable = !TableFeatures.isCatalogManagedSupported(protocol); final boolean isCatalogCommitter = committer instanceof CatalogCommitter; if (!allCatalogCommits.isEmpty()) { if (isFileSystemBasedTable) { throw new IllegalStateException( // This case should be impossible "Cannot have catalog commits on a filesystem-managed table"); } if (!isCatalogCommitter) { throw new UnsupportedOperationException( // This case should also be impossible String.format( "[%s] Cannot publish: committer does not support publishing", committer.getClass().getName())); } } else { if (isFileSystemBasedTable) { logger.info("Publishing not applicable: this is a filesystem-managed table"); return this; } if (!isCatalogCommitter) { logger.info( "[{}] Publishing not applicable: committer does not support publishing", committer.getClass().getName()); return this; } } // TODO: When we return a post-publish Snapshot, ensure to replace *all* catalog commits with // their published versions, not just the catalog commits that were published. For // example: if we have catalog commits v11, v12, and v13 but the maxPublishedVersion is // 12, we will only publish v13. Nonetheless, our post-publish Snapshot must include the // published versions of v11 and v12, too. final long maxPublishedDeltaVersion = getMaxPublishedDeltaVersionOrThrow(); final List catalogCommitsToPublish = allCatalogCommits.stream() .filter(commit -> commit.getVersion() > maxPublishedDeltaVersion) .collect(Collectors.toList()); if (catalogCommitsToPublish.isEmpty()) { logger.info("No catalog commits need to be published"); return this; } final PublishMetadata publishMetadata = new PublishMetadata(version, logPath.toString(), catalogCommitsToPublish); ((CatalogCommitter) committer).publish(engine, publishMetadata); LogSegment updatedLogSegment = getLogSegment().newAsPublished(); return new SnapshotImpl( dataPath, version, new Lazy<>(() -> updatedLogSegment), logReplay, protocol, metadata, committer, SnapshotQueryContext.forVersionSnapshot(dataPath.toString(), version), this.inCommitTimestampOpt); } @Override public void writeChecksum(Engine engine, Snapshot.ChecksumWriteMode mode) throws IOException { final Optional actualOpt = getStatistics().getChecksumWriteMode(); if (actualOpt.isEmpty()) { logger.warn("Not writing checksum: checksum file already exists at version {}", version); return; } final Snapshot.ChecksumWriteMode actual = actualOpt.get(); switch (mode) { case SIMPLE: if (actual == ChecksumWriteMode.FULL) { throw new IllegalStateException( "Cannot write checksum in SIMPLE mode: FULL mode required"); } final CRCInfo crcInfo = logReplay.getCrcInfoAtSnapshotVersion().get(); logger.info("Executing checksum write in SIMPLE mode"); new ChecksumWriter(logPath).writeCheckSum(engine, crcInfo); return; case FULL: if (actual == ChecksumWriteMode.SIMPLE) { logger.warn("Requested checksum write in FULL mode, but SIMPLE mode is available"); } logger.info("Executing checksum write in FULL mode"); ChecksumUtils.computeStateAndWriteChecksum(engine, getLogSegment()); return; default: throw new IllegalStateException("Unknown checksum write mode: " + mode); } } public void writeCheckpoint(Engine engine) throws IOException { // Refuse to create a checkpoint if the table is CatalogManaged but the current snapshot is not // published if (TableFeatures.isCatalogManagedSupported(protocol) && getLogSegment().getMaxPublishedDeltaVersion().orElse(-1L) < version) { throw DeltaErrors.checkpointOnUnpublishedCommits( getPath(), version, getLogSegment().getMaxPublishedDeltaVersion().orElse(-1L)); } Checkpointer.checkpoint(engine, System::currentTimeMillis, this); } /////////////////// // Internal APIs // /////////////////// // TODO: make this API public after closing open threads for Replace Table operation public ReplaceTableTransactionBuilder buildReplaceTableTransaction( StructType schema, String engineInfo) { return new ReplaceTableTransactionBuilderV2Impl(this, schema, engineInfo); } public Committer getCommitter() { return committer; } public Path getLogPath() { return logPath; } public Path getDataPath() { return dataPath; } /** * Returns true if this snapshot was built as a "latest" snapshot query (i.e., no time-travel * parameters were provided). This is intent-based - it indicates what the user requested, not * whether the snapshot is actually the latest version. */ public boolean wasBuiltAsLatest() { return wasBuiltAsLatest; } public Protocol getProtocol() { return protocol; } public SnapshotReport getSnapshotReport() { return lazySnapshotReport.get(); } /** * Returns the clustering columns for this snapshot. * *

    *
  • Optional.empty() - unclustered table (clustering is not enabled) *
  • Optional.of([]) - clustered table with no clustering columns (clustering is enabled) *
  • Optional.of([col1, col2]) - clustered table with the given physical clustering columns *
* * @return the physical clustering columns in this snapshot */ public Optional> getPhysicalClusteringColumns() { return lazyClusteringColumns.get(); } /** * Get the domain metadata map from the log replay, which lazily loads and replays a history of * domain metadata actions, resolving them to produce the current state of the domain metadata. * Only active domain metadata are included in this map. * * @return A map where the keys are domain names and the values are {@link DomainMetadata} * objects. */ public Map getActiveDomainMetadataMap() { return logReplay.getActiveDomainMetadataMap(); } /** Returns the crc info for the current snapshot if the checksum file is read */ public Optional getCurrentCrcInfo() { return logReplay.getCrcInfoAtSnapshotVersion(); } public Metadata getMetadata() { return metadata; } public LogSegment getLogSegment() { return lazyLogSegment.get(); } @VisibleForTesting public Lazy getLazyLogSegment() { return lazyLogSegment; } public CreateCheckpointIterator getCreateCheckpointIterator(Engine engine) { long minFileRetentionTimestampMillis = System.currentTimeMillis() - TOMBSTONE_RETENTION.fromMetadata(metadata); return new CreateCheckpointIterator(engine, getLogSegment(), minFileRetentionTimestampMillis); } /** * Get the latest transaction version for given applicationId. This information comes from * the transactions identifiers stored in Delta transaction log. This API is not a public API. For * now keep this internal to enable Flink upgrade to use Kernel. * * @param applicationId Identifier of the application that put transaction identifiers in Delta * transaction log * @return Last transaction version or {@link Optional#empty()} if no transaction identifier * exists for this application. */ public Optional getLatestTransactionVersion(Engine engine, String applicationId) { return logReplay.getLatestTransactionIdentifier(engine, applicationId); } //////////////////// // Helper Methods // //////////////////// private long getMaxPublishedDeltaVersionOrThrow() { // The maxPublishedDeltaVersion is required for publishing to ensure published deltas are // contiguous. The cases where it is unknown should be very rare (e.g. Kernel loaded a // LogSegment consisting only of a checkpoint with no corresponding published delta). // TODO: Kernel should LIST to authoritatively determine the maxPublishedDeltaVersion, or give // such utilities to CatalogCommitters for them to do this. return getLogSegment() .getMaxPublishedDeltaVersion() .orElseThrow( () -> new IllegalStateException( "maxPublishedDeltaVersion is unknown. This is required for publishing.")); } /////////////////// // Inner Classes // /////////////////// private class SnapshotStatisticsImpl implements SnapshotStatistics { @Override public Optional getChecksumWriteMode() { final boolean checksumFileExists = getLogSegment() .getLastSeenChecksum() .map(checksumFile -> FileNames.checksumVersion(checksumFile.getPath()) == version) .orElse(false); if (checksumFileExists) { return Optional.empty(); } if (logReplay.getCrcInfoAtSnapshotVersion().isPresent()) { return Optional.of(Snapshot.ChecksumWriteMode.SIMPLE); } return Optional.of(Snapshot.ChecksumWriteMode.FULL); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/TableChangesUtils.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal; import static io.delta.kernel.internal.DeltaErrors.wrapEngineException; import io.delta.kernel.CommitActions; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.engine.Engine; import io.delta.kernel.expressions.ExpressionEvaluator; import io.delta.kernel.expressions.Literal; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.types.LongType; import io.delta.kernel.types.StructField; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterator; /** Utility class for table changes operations. */ public class TableChangesUtils { /** Column name for the version metadata column added to getActions results. */ public static final String VERSION_COLUMN_NAME = "version"; /** Column name for the timestamp metadata column added to getActions results. */ public static final String TIMESTAMP_COLUMN_NAME = "timestamp"; /** StructField for the version metadata column. */ private static final StructField VERSION_STRUCT_FIELD = new StructField(VERSION_COLUMN_NAME, LongType.LONG, false); /** StructField for the timestamp metadata column. */ private static final StructField TIMESTAMP_STRUCT_FIELD = new StructField(TIMESTAMP_COLUMN_NAME, LongType.LONG, false); private TableChangesUtils() {} /** * Adds version and timestamp columns to a columnar batch. * *

The version and timestamp columns are added as the first two columns in the batch. * * @param engine the engine for expression evaluation * @param batch the original batch * @param version the version value to add * @param timestamp the timestamp value to add * @return a new batch with version and timestamp columns prepended */ public static ColumnarBatch addVersionAndTimestampColumns( Engine engine, ColumnarBatch batch, long version, long timestamp) { StructType schemaForEval = batch.getSchema(); ExpressionEvaluator commitVersionGenerator = wrapEngineException( () -> engine .getExpressionHandler() .getEvaluator(schemaForEval, Literal.ofLong(version), LongType.LONG), "Get the expression evaluator for the commit version"); ExpressionEvaluator commitTimestampGenerator = wrapEngineException( () -> engine .getExpressionHandler() .getEvaluator(schemaForEval, Literal.ofLong(timestamp), LongType.LONG), "Get the expression evaluator for the commit timestamp"); ColumnVector commitVersionVector = wrapEngineException( () -> commitVersionGenerator.eval(batch), "Evaluating the commit version expression"); ColumnVector commitTimestampVector = wrapEngineException( () -> commitTimestampGenerator.eval(batch), "Evaluating the commit timestamp expression"); return batch .withNewColumn(0, VERSION_STRUCT_FIELD, commitVersionVector) .withNewColumn(1, TIMESTAMP_STRUCT_FIELD, commitTimestampVector); } /** * Flattens an iterator of CommitActions into an iterator of ColumnarBatch, adding version and * timestamp columns to each batch. * * @param engine the engine for expression evaluation * @param commits the iterator of CommitActions to flatten * @return an iterator of ColumnarBatch with version and timestamp columns added */ public static CloseableIterator flattenCommitsAndAddMetadata( Engine engine, CloseableIterator commits) { CloseableIterator> nestedIterator = commits.map( commit -> { long version = commit.getVersion(); long timestamp = commit.getTimestamp(); CloseableIterator actions = commit.getActions(); // Map each batch to add version and timestamp columns return actions.map( batch -> addVersionAndTimestampColumns(engine, batch, version, timestamp)); }); return Utils.flatten(nestedIterator); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/TableConfig.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal; import io.delta.kernel.exceptions.InvalidConfigurationValueException; import io.delta.kernel.exceptions.UnknownConfigurationException; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.internal.util.ColumnMapping.ColumnMappingMode; import io.delta.kernel.internal.util.IntervalParserUtils; import java.util.*; import java.util.function.Function; import java.util.function.Predicate; /** * Represents the table properties. Also provides methods to access the property values from the * table metadata. */ public class TableConfig { public static final String MIN_PROTOCOL_READER_VERSION_KEY = "delta.minReaderVersion"; public static final String MIN_PROTOCOL_WRITER_VERSION_KEY = "delta.minWriterVersion"; ////////////////// // TableConfigs // ////////////////// /** * Whether this Delta table is append-only. Files can't be deleted, or values can't be updated. */ public static final TableConfig APPEND_ONLY_ENABLED = new TableConfig<>( "delta.appendOnly", "false", Boolean::valueOf, value -> true, "needs to be a boolean.", true); /** * Enable change data feed output. When enabled, DELETE, UPDATE, and MERGE INTO operations will * need to do additional work to output their change data in an efficiently readable format. */ public static final TableConfig CHANGE_DATA_FEED_ENABLED = new TableConfig<>( "delta.enableChangeDataFeed", "false", Boolean::valueOf, value -> true, "needs to be a boolean.", true); public static final TableConfig CHECKPOINT_POLICY = new TableConfig<>( "delta.checkpointPolicy", "classic", v -> v, value -> value.equals("classic") || value.equals("v2"), "needs to be a string and one of 'classic' or 'v2'.", true); /** Whether commands modifying this Delta table are allowed to create new deletion vectors. */ public static final TableConfig DELETION_VECTORS_CREATION_ENABLED = new TableConfig<>( "delta.enableDeletionVectors", "false", Boolean::valueOf, value -> true, "needs to be a boolean.", true); /** * Whether widening the type of an existing column or field is allowed, either manually using * ALTER TABLE CHANGE COLUMN or automatically if automatic schema evolution is enabled. */ public static final TableConfig TYPE_WIDENING_ENABLED = new TableConfig<>( "delta.enableTypeWidening", "false", Boolean::valueOf, value -> true, "needs to be a boolean.", true); /** * Indicates whether Row Tracking is enabled on the table. When this flag is turned on, all rows * are guaranteed to have Row IDs and Row Commit Versions assigned to them, and writers are * expected to preserve them by materializing them to hidden columns in the data files. */ public static final TableConfig ROW_TRACKING_ENABLED = new TableConfig<>( "delta.enableRowTracking", "false", Boolean::valueOf, value -> true, "needs to be a boolean.", true); /** * The shortest duration we have to keep logically deleted data files around before deleting them * physically. * *

Note: this value should be large enough: * *

    *
  • It should be larger than the longest possible duration of a job if you decide to run * "VACUUM" when there are concurrent readers or writers accessing the table. *
  • If you are running a streaming query reading from the table, you should make sure the * query doesn't stop longer than this value. Otherwise, the query may not be able to * restart as it still needs to read old files. *
*/ public static final TableConfig TOMBSTONE_RETENTION = new TableConfig<>( "delta.deletedFileRetentionDuration", "interval 1 week", IntervalParserUtils::safeParseIntervalAsMillis, value -> value >= 0, "needs to be provided as a calendar interval such as '2 weeks'. Months" + " and years are not accepted. You may specify '365 days' for a year instead.", true); /** * How often to checkpoint the delta log? For every N (this config) commits to the log, we will * suggest write out a checkpoint file that can speed up the Delta table state reconstruction. */ public static final TableConfig CHECKPOINT_INTERVAL = new TableConfig<>( "delta.checkpointInterval", "10", Integer::valueOf, value -> value > 0, "needs to be a positive integer.", true); /** * The shortest duration we have to keep delta/checkpoint files around before deleting them. We * can only delete delta files that are before a checkpoint. */ public static final TableConfig LOG_RETENTION = new TableConfig<>( "delta.logRetentionDuration", "interval 30 days", IntervalParserUtils::safeParseIntervalAsMillis, value -> true, "needs to be provided as a calendar interval such as '2 weeks'. Months " + "and years are not accepted. You may specify '365 days' for a year instead.", true /* editable */); /** Whether to clean up expired checkpoints and delta logs. */ public static final TableConfig EXPIRED_LOG_CLEANUP_ENABLED = new TableConfig<>( "delta.enableExpiredLogCleanup", "true", Boolean::valueOf, value -> true, "needs to be a boolean.", true /* editable */); /** * This table property is used to track the enablement of the {@code inCommitTimestamps}. * *

When enabled, commit metadata includes a monotonically increasing timestamp that allows for * reliable TIMESTAMP AS OF time travel even if filesystem operations change a commit file's * modification timestamp. */ public static final TableConfig IN_COMMIT_TIMESTAMPS_ENABLED = new TableConfig<>( "delta.enableInCommitTimestamps", "false", /* default values */ v -> Boolean.valueOf(v), value -> true, "needs to be a boolean.", true); /** * This table property is used to track the version of the table at which {@code * inCommitTimestamps} were enabled. */ public static final TableConfig> IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION = new TableConfig<>( "delta.inCommitTimestampEnablementVersion", null, /* default values */ v -> Optional.ofNullable(v).map(Long::valueOf), value -> true, "needs to be a long.", true); /** * This table property is used to track the timestamp at which {@code inCommitTimestamps} were * enabled. More specifically, it is the {@code inCommitTimestamps} of the commit with the version * specified in {@link #IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION}. */ public static final TableConfig> IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP = new TableConfig<>( "delta.inCommitTimestampEnablementTimestamp", null, /* default values */ v -> Optional.ofNullable(v).map(Long::valueOf), value -> true, "needs to be a long.", true); /** This table property is used to control the column mapping mode. */ public static final TableConfig COLUMN_MAPPING_MODE = new TableConfig<>( "delta.columnMapping.mode", "none", /* default values */ ColumnMappingMode::fromTableConfig, value -> true, "Needs to be one of none, id, name.", true); /** This table property is used to control the maximum column mapping ID. */ public static final TableConfig COLUMN_MAPPING_MAX_COLUMN_ID = new TableConfig<>( "delta.columnMapping.maxColumnId", "0", Long::valueOf, value -> value >= 0, "", false); /** * Table property that enables modifying the table in accordance with the Delta-Iceberg * Compatibility V2 protocol. * * @see * Delta-Iceberg Compatibility V2 Protocol */ public static final TableConfig ICEBERG_COMPAT_V2_ENABLED = new TableConfig<>( "delta.enableIcebergCompatV2", "false", Boolean::valueOf, value -> true, "needs to be a boolean.", true); /** * Table property that enables modifying the table in accordance with the Delta-Iceberg * Compatibility V3 protocol. TODO: add the delta protocol link once updated * [https://github.com/delta-io/delta/issues/4574] */ public static final TableConfig ICEBERG_COMPAT_V3_ENABLED = new TableConfig<>( "delta.enableIcebergCompatV3", "false", Boolean::valueOf, value -> true, "needs to be a boolean.", true); /** * The number of columns to collect stats on for data skipping. A value of -1 means collecting * stats for all columns. * *

For Struct types, all leaf fields count individually toward this limit in depth-first order. * For example, if a table has columns a, b.c, b.d, and e, then the first three indexed columns * would be a, b.c, and b.d. Map and array types are not supported for statistics collection. */ public static final TableConfig DATA_SKIPPING_NUM_INDEXED_COLS = new TableConfig<>( "delta.dataSkippingNumIndexedCols", "32", Integer::valueOf, value -> value >= -1, "needs to be larger than or equal to -1.", true); /** * IMPORTANT: This table property is recognized but is not yet validated, enforced, or implemented * by Kernel. * *

The names of specific columns to collect stats on for data skipping. If present, it takes * precedence over {@link #DATA_SKIPPING_NUM_INDEXED_COLS}, and the system will only collect stats * for columns that exactly match those specified. If a nested column is specified, the system * will collect stats for all leaf fields of that column. If a non-existent column is specified, * it will be ignored. Updating this config does not trigger stats re-collection, but redefines * the stats schema of the table, i.e., it will change the behavior of future stats collection * (e.g., in append and OPTIMIZE) as well as data skipping (e.g., the column stats not mentioned * by this config will be ignored even if they exist). * *

The value is a comma-separated list of case-insensitive column identifiers. Each column * identifier can consist of letters, digits, and underscores. If a column identifier includes * special characters, the column name should be enclosed in backticks (`) to escape the special * characters. * *

A column identifier can refer to one of the following: the name of a non-struct column, the * leaf field's name of a struct column, or the name of a struct column. When a struct column's * name is specified, statistics for all its leaf fields will be collected. */ public static final TableConfig> DATA_SKIPPING_STATS_COLUMNS = new TableConfig<>( "delta.dataSkippingStatsColumns", null, v -> Optional.ofNullable(v), value -> true, "needs to be a comma-separated list of column identifiers.", true); /** * Table property that enables modifying the table in accordance with the Delta-Iceberg Writer * Compatibility V1 ({@code icebergCompatWriterV1}) protocol. */ public static final TableConfig ICEBERG_WRITER_COMPAT_V1_ENABLED = new TableConfig<>( "delta.enableIcebergWriterCompatV1", "false", Boolean::valueOf, value -> true, "needs to be a boolean.", true); /** * Table property that enables modifying the table in accordance with the Delta-Iceberg Writer * Compatibility V3 ({@code icebergCompatWriterV3}) protocol. V2 is skipped to align with the * iceberg v3 spec. */ public static final TableConfig ICEBERG_WRITER_COMPAT_V3_ENABLED = new TableConfig<>( "delta.enableIcebergWriterCompatV3", "false", Boolean::valueOf, value -> true, "needs to be a boolean.", true); public static class UniversalFormats { /** * The value that enables uniform exports to Iceberg for {@linkplain * TableConfig#UNIVERSAL_FORMAT_ENABLED_FORMATS}. * *

{@link #ICEBERG_COMPAT_V2_ENABLED but also be set to true} to fully enable this feature. */ public static final String FORMAT_ICEBERG = "iceberg"; /** * The value to use to enable uniform exports to Hudi for {@linkplain * TableConfig#UNIVERSAL_FORMAT_ENABLED_FORMATS}. */ public static final String FORMAT_HUDI = "hudi"; } private static final Collection ALLOWED_UNIFORM_FORMATS = Collections.unmodifiableList( Arrays.asList(UniversalFormats.FORMAT_HUDI, UniversalFormats.FORMAT_ICEBERG)); /** Table config that allows for translation of Delta metadata to other table formats metadata. */ public static final TableConfig> UNIVERSAL_FORMAT_ENABLED_FORMATS = new TableConfig<>( "delta.universalFormat.enabledFormats", null, TableConfig::parseStringSet, value -> ALLOWED_UNIFORM_FORMATS.containsAll(value), String.format("each value must in the the set: %s", ALLOWED_UNIFORM_FORMATS), true); /** * Table property that enables modifying the table in accordance with the Delta-Variant Shredding * protocol. * * @see * Delta-Variant Shredding Protocol */ public static final TableConfig VARIANT_SHREDDING_ENABLED = new TableConfig<>( "delta.enableVariantShredding", "false", Boolean::valueOf, value -> true, "needs to be a boolean.", true); public static final TableConfig MATERIALIZED_ROW_ID_COLUMN_NAME = new TableConfig<>( "delta.rowTracking.materializedRowIdColumnName", null, v -> v, value -> true, "need to be a string.", false); public static final TableConfig MATERIALIZED_ROW_COMMIT_VERSION_COLUMN_NAME = new TableConfig<>( "delta.rowTracking.materializedRowCommitVersionColumnName", null, v -> v, value -> true, "need to be a string.", false); /** All the valid properties that can be set on the table. */ private static final Map> VALID_PROPERTIES = Collections.unmodifiableMap( new HashMap>() { { addConfig(this, APPEND_ONLY_ENABLED); addConfig(this, CHANGE_DATA_FEED_ENABLED); addConfig(this, CHECKPOINT_POLICY); addConfig(this, DELETION_VECTORS_CREATION_ENABLED); addConfig(this, TYPE_WIDENING_ENABLED); addConfig(this, ROW_TRACKING_ENABLED); addConfig(this, LOG_RETENTION); addConfig(this, EXPIRED_LOG_CLEANUP_ENABLED); addConfig(this, TOMBSTONE_RETENTION); addConfig(this, CHECKPOINT_INTERVAL); addConfig(this, IN_COMMIT_TIMESTAMPS_ENABLED); addConfig(this, IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION); addConfig(this, IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP); addConfig(this, COLUMN_MAPPING_MODE); addConfig(this, ICEBERG_COMPAT_V2_ENABLED); addConfig(this, ICEBERG_COMPAT_V3_ENABLED); addConfig(this, ICEBERG_WRITER_COMPAT_V1_ENABLED); addConfig(this, ICEBERG_WRITER_COMPAT_V3_ENABLED); addConfig(this, COLUMN_MAPPING_MAX_COLUMN_ID); addConfig(this, DATA_SKIPPING_NUM_INDEXED_COLS); addConfig(this, UNIVERSAL_FORMAT_ENABLED_FORMATS); addConfig(this, MATERIALIZED_ROW_ID_COLUMN_NAME); addConfig(this, MATERIALIZED_ROW_COMMIT_VERSION_COLUMN_NAME); addConfig(this, VARIANT_SHREDDING_ENABLED); // The below configs do not yet have their behavior correctly implemented in Kernel. addConfig(this, DATA_SKIPPING_STATS_COLUMNS); } }); /////////////////////////// // Static Helper Methods // /////////////////////////// /** * Validates that the given new properties that the txn is trying to update in table. Properties * that have `delta.` prefix in the key name should be in valid list and are editable. The caller * is expected to store the returned properties in the table metadata after further validation * from a protocol point of view. The returned properties will have the key's case normalized as * defined in its {@link TableConfig}. * * @param newProperties the properties to validate * @throws InvalidConfigurationValueException if any of the properties are invalid * @throws UnknownConfigurationException if any of the properties are unknown */ public static Map validateAndNormalizeDeltaProperties( Map newProperties) { Map validatedProperties = new HashMap<>(); for (Map.Entry kv : newProperties.entrySet()) { String key = kv.getKey().toLowerCase(Locale.ROOT); String value = kv.getValue(); boolean isTableFeatureOverrideKey = key.startsWith(TableFeatures.SET_TABLE_FEATURE_SUPPORTED_PREFIX); boolean isTableConfigKey = key.startsWith("delta."); // TableFeature override properties validation is handled separately in TransactionBuilder. boolean shouldValidateProperties = isTableConfigKey && !isTableFeatureOverrideKey; if (shouldValidateProperties) { // If it is a delta table property, make sure it is a supported property and editable if (!VALID_PROPERTIES.containsKey(key)) { throw DeltaErrors.unknownConfigurationException(kv.getKey()); } TableConfig tableConfig = VALID_PROPERTIES.get(key); if (!tableConfig.editable) { throw DeltaErrors.cannotModifyTableProperty(kv.getKey()); } tableConfig.validate(value); validatedProperties.put(tableConfig.getKey(), value); } else { // allow unknown properties to be set (and preserve their original case!) validatedProperties.put(kv.getKey(), value); } } return validatedProperties; } private static void addConfig(HashMap> configs, TableConfig config) { configs.put(config.getKey().toLowerCase(Locale.ROOT), config); } ///////////////////////////// // Member Fields / Methods // ///////////////////////////// private final String key; private final String defaultValue; private final Function fromString; private final Predicate validator; private final boolean editable; private final String helpMessage; private TableConfig( String key, String defaultValue, Function fromString, Predicate validator, String helpMessage, boolean editable) { this.key = key; this.defaultValue = defaultValue; this.fromString = fromString; this.validator = validator; this.helpMessage = helpMessage; this.editable = editable; } /** * Returns the value of the table property from the given metadata. * * @param metadata the table metadata * @return the value of the table property */ public T fromMetadata(Metadata metadata) { return fromMetadata(metadata.getConfiguration()); } /** * Returns the value of the table property from the given configuration. * * @param configuration the table configuration * @return the value of the table property */ public T fromMetadata(Map configuration) { String value = configuration.getOrDefault(key, defaultValue); validate(value); return fromString.apply(value); } /** * Returns the key of the table property. * * @return the key of the table property */ public String getKey() { return key; } private void validate(String value) { T parsedValue = fromString.apply(value); if (!validator.test(parsedValue)) { throw DeltaErrors.invalidConfigurationValueException(key, value, helpMessage); } } private static Set parseStringSet(String value) { if (value == null || value.isEmpty()) { return Collections.emptySet(); } String[] formats = value.split(","); Set config = new HashSet<>(); for (String format : formats) { config.add(format.trim()); } return config; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/TableImpl.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal; import static io.delta.kernel.internal.DeltaErrors.wrapEngineExceptionThrowsIO; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Collections.emptyList; import io.delta.kernel.*; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.CheckpointAlreadyExistsException; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.exceptions.TableNotFoundException; import io.delta.kernel.internal.checkpoints.Checkpointer; import io.delta.kernel.internal.checksum.ChecksumUtils; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.metrics.SnapshotQueryContext; import io.delta.kernel.internal.snapshot.LogSegment; import io.delta.kernel.internal.snapshot.SnapshotManager; import io.delta.kernel.internal.util.Clock; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import java.io.IOException; import java.io.UncheckedIOException; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class TableImpl implements Table { private static final Logger logger = LoggerFactory.getLogger(TableImpl.class); public static Table forPath(Engine engine, String path) { return forPath(engine, path, System::currentTimeMillis); } /** * Instantiate a table object for the Delta Lake table at the given path. It takes an additional * parameter called {@link Clock} which helps in testing. * * @param engine {@link Engine} instance to use in Delta Kernel. * @param path location of the table. * @param clock {@link Clock} instance to use for time-related operations. * @return an instance of {@link Table} representing the Delta table at the given path */ public static Table forPath(Engine engine, String path, Clock clock) { String resolvedPath; try { resolvedPath = wrapEngineExceptionThrowsIO( () -> engine.getFileSystemClient().resolvePath(path), "Resolving path %s", path); } catch (IOException io) { throw new UncheckedIOException(io); } return new TableImpl(resolvedPath, clock); } private final String tablePath; private final Checkpointer checkpointer; private final SnapshotManager snapshotManager; private final Clock clock; public TableImpl(String tablePath, Clock clock) { this.tablePath = tablePath; final Path dataPath = new Path(tablePath); final Path logPath = new Path(dataPath, "_delta_log"); this.checkpointer = new Checkpointer(logPath); this.snapshotManager = new SnapshotManager(dataPath); this.clock = clock; } @Override public String getPath(Engine engine) { return tablePath; } @Override public SnapshotImpl getLatestSnapshot(Engine engine) throws TableNotFoundException { SnapshotQueryContext snapshotContext = SnapshotQueryContext.forLatestSnapshot(tablePath); return loadSnapshotWithMetrics( engine, () -> snapshotManager.buildLatestSnapshot(engine, snapshotContext), snapshotContext); } @Override public SnapshotImpl getSnapshotAsOfVersion(Engine engine, long versionId) throws TableNotFoundException { SnapshotQueryContext snapshotContext = SnapshotQueryContext.forVersionSnapshot(tablePath, versionId); return loadSnapshotWithMetrics( engine, () -> snapshotManager.getSnapshotAt(engine, versionId, snapshotContext), snapshotContext); } @Override public SnapshotImpl getSnapshotAsOfTimestamp(Engine engine, long millisSinceEpochUTC) throws TableNotFoundException { SnapshotQueryContext snapshotContext = SnapshotQueryContext.forTimestampSnapshot(tablePath, millisSinceEpochUTC); SnapshotImpl latestSnapshot = getLatestSnapshot(engine); return loadSnapshotWithMetrics( engine, () -> snapshotManager.getSnapshotForTimestamp( engine, latestSnapshot, millisSinceEpochUTC, snapshotContext), snapshotContext); } @Override public void checkpoint(Engine engine, long version) throws TableNotFoundException, CheckpointAlreadyExistsException, IOException { final SnapshotImpl snapshotToCheckpoint = getSnapshotAsOfVersion(engine, version); checkpointer.checkpoint(engine, clock, snapshotToCheckpoint); } @Override public void checksum(Engine engine, long version) throws TableNotFoundException, IOException { final LogSegment logSegmentAtVersion = snapshotManager.getLogSegmentForVersion(engine, Optional.of(version)); ChecksumUtils.computeStateAndWriteChecksum(engine, logSegmentAtVersion); } @Override public TransactionBuilder createTransactionBuilder( Engine engine, String engineInfo, Operation operation) { return new TransactionBuilderImpl(this, engineInfo, operation); } public TransactionBuilder createReplaceTableTransactionBuilder(Engine engine, String engineInfo) { return new ReplaceTableTransactionBuilderImpl(this, engineInfo); } public Clock getClock() { return clock; } /** * Returns delta actions for each version between startVersion and endVersion. Only returns the * actions requested in actionSet. * *

For the returned columnar batches: * *

    *
  • Each row within the same batch is guaranteed to have the same commit version *
  • The batch commit versions are monotonically increasing *
  • The top-level columns include "version", "timestamp", and the actions requested in * actionSet. "version" and "timestamp" are the first and second columns in the schema, * respectively. The remaining columns are based on the actions requested and each have the * schema found in {@code DeltaAction.schema}. *
* * @param engine {@link Engine} instance to use in Delta Kernel. * @param startVersion start version (inclusive) * @param endVersion end version (inclusive) * @param actionSet the actions to read and return from the JSON log files * @return an iterator of batches where each row in the batch has exactly one non-null action and * its commit version and timestamp * @throws TableNotFoundException if the table does not exist or if it is not a delta table * @throws KernelException if a commit file does not exist for any of the versions in the provided * range * @throws KernelException if provided an invalid version range * @throws KernelException if the version range contains a version with reader protocol that is * unsupported by Kernel */ public CloseableIterator getChanges( Engine engine, long startVersion, long endVersion, Set actionSet) { checkArgument(startVersion >= 0, "startVersion must be >= 0"); checkArgument(startVersion <= endVersion, "startVersion must be <= endVersion"); List commitFiles = DeltaLogActionUtils.getCommitFilesForVersionRange( engine, new Path(tablePath), startVersion, Optional.of(endVersion)); // Get CommitActions for each file CloseableIterator commits = DeltaLogActionUtils.getActionsFromCommitFilesWithProtocolValidation( engine, tablePath, commitFiles, actionSet); // Flatten and add version/timestamp columns return TableChangesUtils.flattenCommitsAndAddMetadata(engine, commits); } protected Path getDataPath() { return new Path(tablePath); } protected Path getLogPath() { return new Path(tablePath, "_delta_log"); } /** * Returns the latest version that was committed before or at {@code millisSinceEpochUTC}. If no * version exists, throws a {@link KernelException} * *

Specifically: * *

    *
  • if a commit version exactly matches the provided timestamp, we return it *
  • else, we return the latest commit version with a timestamp less than the provided one *
  • If the provided timestamp is less than the timestamp of any committed version, we throw * an error. *
* * . * * @param millisSinceEpochUTC the number of milliseconds since midnight, January 1, 1970 UTC * @return latest commit that happened before or at {@code timestamp}. * @throws KernelException if the timestamp is less than the timestamp of any committed version * @throws TableNotFoundException if no delta table is found */ public long getVersionBeforeOrAtTimestamp(Engine engine, long millisSinceEpochUTC) { SnapshotImpl latestSnapshot = (SnapshotImpl) getLatestSnapshot(engine); return DeltaHistoryManager.getVersionBeforeOrAtTimestamp( engine, getLogPath(), millisSinceEpochUTC, latestSnapshot, emptyList() /* catalogCommits */); } /** * Returns the latest version that was committed at or after {@code millisSinceEpochUTC}. If no * version exists, throws a {@link KernelException} * *

Specifically: * *

    *
  • if a commit version exactly matches the provided timestamp, we return it *
  • else, we return the earliest commit version with a timestamp greater than the provided * one *
  • If the provided timestamp is larger than the timestamp of any committed version, we throw * an error. *
* * . * * @param millisSinceEpochUTC the number of milliseconds since midnight, January 1, 1970 UTC * @return latest commit that happened at or before {@code timestamp}. * @throws KernelException if the timestamp is more than the timestamp of any committed version * @throws TableNotFoundException if no delta table is found */ public long getVersionAtOrAfterTimestamp(Engine engine, long millisSinceEpochUTC) { SnapshotImpl latestSnapshot = (SnapshotImpl) getLatestSnapshot(engine); return DeltaHistoryManager.getVersionAtOrAfterTimestamp( engine, getLogPath(), millisSinceEpochUTC, latestSnapshot, emptyList() /* catalogCommits */); } /** Helper method that loads a snapshot with proper metrics recording, logging, and reporting. */ private SnapshotImpl loadSnapshotWithMetrics( Engine engine, Supplier loadSnapshot, SnapshotQueryContext snapshotContext) throws TableNotFoundException { try { final SnapshotImpl snapshot = snapshotContext.getSnapshotMetrics().loadSnapshotTotalTimer.time(loadSnapshot); logger.info( "[{}] Took {}ms to load snapshot (version = {}) for snapshot query {}", tablePath, snapshotContext.getSnapshotMetrics().loadSnapshotTotalTimer.totalDurationMs(), snapshot.getVersion(), snapshotContext.getQueryDisplayStr()); engine .getMetricsReporters() .forEach(reporter -> reporter.report(snapshot.getSnapshotReport())); return snapshot; } catch (Exception e) { snapshotContext.recordSnapshotErrorReport(engine, e); throw e; } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionBuilderImpl.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal; import static io.delta.kernel.internal.DeltaErrors.*; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Collections.emptyMap; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toSet; import io.delta.kernel.*; import io.delta.kernel.commit.Committer; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.TableAlreadyExistsException; import io.delta.kernel.exceptions.TableNotFoundException; import io.delta.kernel.expressions.Column; import io.delta.kernel.internal.actions.*; import io.delta.kernel.internal.commit.DefaultFileSystemManagedTableOnlyCommitter; import io.delta.kernel.internal.rowtracking.MaterializedRowTrackingColumn; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.types.StructType; import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class TransactionBuilderImpl implements TransactionBuilder { private static final Logger logger = LoggerFactory.getLogger(TransactionBuilderImpl.class); private final long currentTimeMillis = System.currentTimeMillis(); private final String engineInfo; private final Operation operation; private Optional> partitionColumns = Optional.empty(); private Optional setTxnOpt = Optional.empty(); private Optional> tableProperties = Optional.empty(); private Optional> unsetTablePropertiesKeys = Optional.empty(); private boolean needDomainMetadataSupport = false; // The original clustering columns provided by the user when building the transaction. // This represents logical column references before schema resolution is applied. // (e.g., case sensitivity, column mapping) private Optional> inputLogicalClusteringColumns = Optional.empty(); // The resolved clustering columns that will be written into domain metadata in the txn. This // reflects case-preserved column names or physical column names if column mapping is enabled. // This is set during transaction building after the schema has been updated/resolved with any // column mapping info. These are the physical columns of `inputLogicalClusteringColumns`. private Optional> newResolvedClusteringColumns = Optional.empty(); protected final TableImpl table; protected Optional schema = Optional.empty(); private Optional userProvidedMaxRetries = Optional.empty(); /** Number of commits between producing a log compaction file. */ private int logCompactionInterval = 0; public TransactionBuilderImpl(TableImpl table, String engineInfo, Operation operation) { this.table = table; this.engineInfo = engineInfo; this.operation = operation; } @Override public TransactionBuilder withSchema(Engine engine, StructType newSchema) { this.schema = Optional.of(newSchema); // will be verified as part of the build() call return this; } @Override public TransactionBuilder withPartitionColumns(Engine engine, List partitionColumns) { if (!partitionColumns.isEmpty()) { this.partitionColumns = Optional.of(partitionColumns); } return this; } /** * There are three possible cases when handling clustering columns via `withClusteringColumns`: * *
    *
  • Clustering columns are not set (i.e., `withClusteringColumns` is not called): *
      *
    • No changes are made related to clustering. *
    • For table creation, the table is initialized as a non-clustered table. *
    • For table updates, the existing clustered or non-clustered state remains unchanged * (i.e., no protocol or domain metadata updates). *
    *
  • Clustering columns are an empty list: *
      *
    • This is equivalent to executing `ALTER TABLE ... CLUSTER BY NONE` in Delta. *
    • The table remains a clustered table, but its clustering domain metadata is updated * to reflect an empty list of clustering columns. *
    *
  • Clustering columns are a non-empty list: *
      *
    • The table is treated as a clustered table. *
    • We update the protocol (if needed) to include clustering writer support and set the * clustering domain metadata accordingly. *
    *
*/ @Override public TransactionBuilder withClusteringColumns(Engine engine, List clusteringColumns) { this.inputLogicalClusteringColumns = Optional.of(clusteringColumns); return this; } @Override public TransactionBuilder withTransactionId( Engine engine, String applicationId, long transactionVersion) { SetTransaction txnId = new SetTransaction( requireNonNull(applicationId, "applicationId is null"), transactionVersion, Optional.of(currentTimeMillis)); this.setTxnOpt = Optional.of(txnId); return this; } @Override public TransactionBuilder withTableProperties(Engine engine, Map properties) { this.tableProperties = Optional.of( Collections.unmodifiableMap( TableConfig.validateAndNormalizeDeltaProperties(properties))); return this; } @Override public TransactionBuilder withTablePropertiesRemoved(Set propertyKeys) { checkArgument( propertyKeys.stream().noneMatch(key -> key.toLowerCase(Locale.ROOT).startsWith("delta.")), "Unsetting 'delta.' table properties is currently unsupported"); this.unsetTablePropertiesKeys = Optional.of(Collections.unmodifiableSet(propertyKeys)); return this; } @Override public TransactionBuilder withMaxRetries(int maxRetries) { checkArgument(maxRetries >= 0, "maxRetries must be >= 0"); this.userProvidedMaxRetries = Optional.of(maxRetries); return this; } @Override public TransactionBuilder withLogCompactionInverval(int logCompactionInterval) { checkArgument(logCompactionInterval >= 0, "logCompactionInterval must be >= 0"); this.logCompactionInterval = logCompactionInterval; return this; } @Override public TransactionBuilder withDomainMetadataSupported() { needDomainMetadataSupport = true; return this; } @Override public Transaction build(Engine engine) { if (operation == Operation.REPLACE_TABLE) { throw new UnsupportedOperationException("REPLACE TABLE is not yet supported"); } SnapshotImpl snapshot; try { snapshot = table.getLatestSnapshot(engine); if (operation == Operation.CREATE_TABLE) { throw new TableAlreadyExistsException(table.getPath(engine), "Operation = CREATE_TABLE"); } return buildTransactionInternal(engine, false /* isCreateOrReplace */, Optional.of(snapshot)); } catch (TableNotFoundException tblf) { String tablePath = table.getPath(engine); logger.info("Table {} doesn't exist yet. Trying to create a new table.", tablePath); schema.orElseThrow(() -> requiresSchemaForNewTable(tablePath)); return buildTransactionInternal(engine, true /* isNewTableDef */, Optional.empty()); } } /** * Returns a built {@link Transaction} for this transaction builder (with the input provided by * the user) given the provided parameters. This includes validation and updates as defined in the * builder. * * @param isCreateOrReplace whether we are defining a new table definition or not. This determines * what metadata to commit in the returned transaction, and what operations to allow or block. * @param latestSnapshot the latest snapshot of the table if it exists. For a new table this * should be empty. For replace table, this should be the latest snapshot of the table. This * is used to validate that we can write to the table, and to get the protocol/metadata when * isCreateOrReplace=false. */ protected TransactionImpl buildTransactionInternal( Engine engine, boolean isCreateOrReplace, Optional latestSnapshot) { checkArgument( isCreateOrReplace || latestSnapshot.isPresent(), "Existing snapshot must be provided if not defining a new table definition"); latestSnapshot.ifPresent( snapshot -> validateWriteToExistingTable(engine, snapshot, isCreateOrReplace)); validateTransactionInputs(engine, isCreateOrReplace); final Committer committer = latestSnapshot .map(SnapshotImpl::getCommitter) .orElse(DefaultFileSystemManagedTableOnlyCommitter.INSTANCE); boolean enablesDomainMetadataSupport = needDomainMetadataSupport && latestSnapshot.isPresent() && !latestSnapshot .get() .getProtocol() .supportsFeature(TableFeatures.DOMAIN_METADATA_W_FEATURE); boolean needsMetadataOrProtocolUpdate = isCreateOrReplace || schema.isPresent() // schema evolution || tableProperties.isPresent() // table properties updated || unsetTablePropertiesKeys.isPresent() // table properties unset || inputLogicalClusteringColumns.isPresent() // clustering columns changed || enablesDomainMetadataSupport; // domain metadata support added if (!needsMetadataOrProtocolUpdate) { // Return early if there is no metadata or protocol updates and isCreateOrReplace=false return new TransactionImpl( false, // isCreateOrReplace table.getDataPath(), latestSnapshot, engineInfo, operation, Optional.empty(), // newProtocol Optional.empty(), // newMetadata committer, setTxnOpt, Optional.empty(), /* clustering cols=empty */ userProvidedMaxRetries, logCompactionInterval, table.getClock()); } // Instead of special casing enabling domain metadata, we should just add them // to the table properties which we already handle. boolean domainMetadataEnabled = !isCreateOrReplace && latestSnapshot .get() .getProtocol() .supportsFeature(TableFeatures.DOMAIN_METADATA_W_FEATURE); if (needDomainMetadataSupport && !domainMetadataEnabled) { Map tablePropertiesWithDomainMetadataEnabled = new HashMap<>(tableProperties.orElse(emptyMap())); tablePropertiesWithDomainMetadataEnabled.put( TableFeatures.DOMAIN_METADATA_W_FEATURE.getTableFeatureSupportKey(), "supported"); tableProperties = Optional.of(tablePropertiesWithDomainMetadataEnabled); } TransactionMetadataFactory.Output outputMetadata; if (!isCreateOrReplace) { outputMetadata = TransactionMetadataFactory.buildUpdateTableMetadata( table.getPath(engine), latestSnapshot.get(), tableProperties, unsetTablePropertiesKeys, schema, inputLogicalClusteringColumns); } else if (latestSnapshot.isPresent()) { // is REPLACE outputMetadata = TransactionMetadataFactory.buildReplaceTableMetadata( table.getPath(engine), latestSnapshot.get(), // when isCreateOrReplace we know schema is present schema.get(), tableProperties.orElse(emptyMap()), partitionColumns, inputLogicalClusteringColumns); } else { outputMetadata = TransactionMetadataFactory.buildCreateTableMetadata( table.getPath(engine), // when isCreateOrReplace we know schema is present schema.get(), tableProperties.orElse(emptyMap()), partitionColumns, inputLogicalClusteringColumns, Optional.empty() /* committerOpt */); } return new TransactionImpl( isCreateOrReplace, table.getDataPath(), latestSnapshot, engineInfo, operation, outputMetadata.newProtocol, outputMetadata.newMetadata, committer, setTxnOpt, outputMetadata.physicalNewClusteringColumns, userProvidedMaxRetries, logCompactionInterval, table.getClock()); } /** * Validates that Kernel can write to the existing table with the latest snapshot as provided. * This means (1) Kernel supports the reader and writer protocol of the table (2) if a transaction * identifier has been provided in this txn builder, a concurrent write has not already committed * this transaction (3) Updating a partitioned table with clustering columns is not allowed (4) * Row tracking configs are present when row tracking is enabled. */ protected void validateWriteToExistingTable( Engine engine, SnapshotImpl snapshot, boolean isCreateOrReplace) { // Validate the table has no features that Kernel doesn't yet support writing into it. TableFeatures.validateKernelCanWriteToTable( snapshot.getProtocol(), snapshot.getMetadata(), table.getPath(engine)); setTxnOpt.ifPresent( txnId -> { Optional lastTxnVersion = snapshot.getLatestTransactionVersion(engine, txnId.getAppId()); if (lastTxnVersion.isPresent() && lastTxnVersion.get() >= txnId.getVersion()) { throw DeltaErrors.concurrentTransaction( txnId.getAppId(), txnId.getVersion(), lastTxnVersion.get()); } }); if (!isCreateOrReplace && inputLogicalClusteringColumns.isPresent() && snapshot.getMetadata().getPartitionColumns().getSize() != 0) { throw DeltaErrors.enablingClusteringOnPartitionedTableNotAllowed( table.getPath(engine), snapshot.getMetadata().getPartitionColNames(), inputLogicalClusteringColumns.get()); } // Validate row tracking configs are present when row tracking is enabled. This must run // on every write, including the early-return path that skips TransactionMetadataFactory. if (!isCreateOrReplace) { MaterializedRowTrackingColumn.validateRowTrackingConfigsNotMissing( snapshot.getMetadata(), table.getPath(engine)); } } /** * Validates the inputs to this transaction builder. This includes * *
    *
  • Partition columns are only set for a new table definition. *
  • Partition columns and clustering columns are not set at the same time. *
  • The provided schema is valid. *
  • The provided partition columns are valid. *
  • The provided table properties to set and unset do not overlap with each other. *
*/ protected void validateTransactionInputs(Engine engine, boolean isCreateOrReplace) { String tablePath = table.getPath(engine); if (!isCreateOrReplace) { if (partitionColumns.isPresent()) { throw tableAlreadyExists( tablePath, "Table already exists, but provided new partition columns. " + "Partition columns can only be set on a new table."); } } else { checkArgument( !(partitionColumns.isPresent() && inputLogicalClusteringColumns.isPresent()), "Partition Columns and Clustering Columns cannot be set at the same time"); } if (unsetTablePropertiesKeys.isPresent() && tableProperties.isPresent()) { Set invalidPropertyKeys = unsetTablePropertiesKeys.get().stream() .filter(key -> tableProperties.get().containsKey(key)) .collect(toSet()); if (!invalidPropertyKeys.isEmpty()) { throw DeltaErrors.overlappingTablePropertiesSetAndUnset(invalidPropertyKeys); } } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionImpl.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal; import static io.delta.kernel.internal.TableConfig.*; import static io.delta.kernel.internal.actions.SingleAction.*; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static io.delta.kernel.internal.util.Preconditions.checkState; import static io.delta.kernel.internal.util.Utils.toCloseableIterator; import static java.util.Collections.emptyMap; import static java.util.Objects.requireNonNull; import io.delta.kernel.*; import io.delta.kernel.commit.CommitFailedException; import io.delta.kernel.commit.CommitMetadata; import io.delta.kernel.commit.Committer; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.*; import io.delta.kernel.expressions.Column; import io.delta.kernel.hook.PostCommitHook; import io.delta.kernel.internal.actions.*; import io.delta.kernel.internal.annotation.VisibleForTesting; import io.delta.kernel.internal.checksum.CRCInfo; import io.delta.kernel.internal.clustering.ClusteringUtils; import io.delta.kernel.internal.compaction.LogCompactionWriter; import io.delta.kernel.internal.data.TransactionStateRow; import io.delta.kernel.internal.files.ParsedDeltaData; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.hook.CheckpointHook; import io.delta.kernel.internal.hook.ChecksumFullHook; import io.delta.kernel.internal.hook.ChecksumSimpleHook; import io.delta.kernel.internal.hook.LogCompactionHook; import io.delta.kernel.internal.lang.Lazy; import io.delta.kernel.internal.metrics.SnapshotQueryContext; import io.delta.kernel.internal.metrics.TransactionMetrics; import io.delta.kernel.internal.metrics.TransactionReportImpl; import io.delta.kernel.internal.replay.ConflictChecker; import io.delta.kernel.internal.replay.ConflictChecker.TransactionRebaseState; import io.delta.kernel.internal.replay.LogReplay; import io.delta.kernel.internal.rowtracking.RowTracking; import io.delta.kernel.internal.rowtracking.RowTrackingMetadataDomain; import io.delta.kernel.internal.snapshot.LogSegment; import io.delta.kernel.internal.stats.FileSizeHistogram; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.internal.util.*; import io.delta.kernel.internal.util.Clock; import io.delta.kernel.internal.util.InCommitTimestampUtils; import io.delta.kernel.internal.util.VectorUtils; import io.delta.kernel.metrics.TransactionMetricsResult; import io.delta.kernel.metrics.TransactionReport; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterable; import io.delta.kernel.utils.CloseableIterator; import java.io.IOException; import java.io.UncheckedIOException; import java.util.*; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class TransactionImpl implements Transaction { /////////////////////////////// // Static methods and fields // /////////////////////////////// /** Get the part of the schema of the table that needs the statistics to be collected per file. */ public static List getStatisticsColumns(Row transactionState) { int numIndexedCols = TableConfig.DATA_SKIPPING_NUM_INDEXED_COLS.fromMetadata( TransactionStateRow.getConfiguration(transactionState)); // Get the list of partition columns to exclude Set partitionColumns = new HashSet<>(TransactionStateRow.getPartitionColumnsList(transactionState)); // Collect the leaf-level columns for statistics calculation. // This call selects only the first 'numIndexedCols' leaf columns from the logical schema, // excluding any column whose top-level name appears in 'partitionColumns'. // NOTE: Nested columns (i.e. each leaf within a StructType) count individually toward the // numIndexedCols limit (not Map/ArrayTypes - they're not stats compatible types). // // For example, given the following schema: // root // ├─ col1 (int) // ├─ col2 (string) // └─ col3 (struct) // ├─ a (int) // └─ b (double) // // And if 'numIndexedCols' is set to 2 with no partition columns to exclude, then the returned // stats columns // would be: [col1, col2]. If 'col1' were a partition column, the returned list would be: // [col2, col3.a] (assuming col3.a is encountered before col3.b). return SchemaUtils.collectLeafColumns( TransactionStateRow.getPhysicalSchema(transactionState), partitionColumns, numIndexedCols); } private static final Logger logger = LoggerFactory.getLogger(TransactionImpl.class); public static final int DEFAULT_READ_VERSION = 1; public static final int DEFAULT_WRITE_VERSION = 2; /** * Default retries for concurrent write exceptions to resolve conflicts and retry commit. In * Delta-Spark, for historical reasons the number of retries is really high (10m). We are starting * with a lower number by default for now. If this is not sufficient we can update it. */ private static final int DEFAULT_MAX_RETRIES = 200; ///////////////////// // Instance fields // ///////////////////// private final UUID txnId = UUID.randomUUID(); /** If the transaction is defining a new table from scratch (i.e. create table, replace table) */ private final boolean isCreateOrReplace; private final Path dataPath; private final Path logPath; private final Optional readSnapshotOpt; private final String engineInfo; private final Operation operation; private final Protocol protocol; private final boolean shouldUpdateProtocol; private Metadata metadata; private boolean shouldUpdateMetadata; private final Committer committer; private final Optional setTxnOpt; /** * The new clustering columns to write in the domain metadata in this transaction if provided. * *
    *
  • Optional.empty() - do not update the clustering domain metadata in this txn *
  • Optional.of([]) - update the clustering domain metadata to store an empty list in this * txn *
  • Optional.of([col1, col2]) - update the clustering domain metadata to store these columns * in this txn *
*/ private final Optional> newClusteringColumnsOpt; private int maxRetries; private final int logCompactionInterval; private final Clock clock; private final DomainMetadataState domainMetadataState = new DomainMetadataState(); private Optional currentCrcInfo; private Optional providedRowIdHighWatermark = Optional.empty(); private Supplier> committerProperties = Collections::emptyMap; private boolean closed; // To avoid trying to commit the same transaction again. public TransactionImpl( boolean isCreateOrReplace, Path dataPath, Optional readSnapshotOpt, String engineInfo, Operation operation, Optional newProtocol, Optional newMetadata, Committer committer, Optional setTxnOpt, Optional> newClusteringColumnsOpt, Optional maxRetriesOpt, int logCompactionInterval, Clock clock) { checkArgument(isCreateOrReplace || readSnapshotOpt.isPresent()); // For a new table, a protocol and metadata must be provided checkArgument( (newProtocol.isPresent() && newMetadata.isPresent()) || readSnapshotOpt.isPresent()); // TODO: look into migrating entire class into just (newMetadata, newProtocol, readSnapshotOpt) this.isCreateOrReplace = isCreateOrReplace; this.dataPath = dataPath; this.logPath = new Path(dataPath, "_delta_log"); this.readSnapshotOpt = readSnapshotOpt; this.engineInfo = engineInfo; this.operation = operation; this.protocol = newProtocol.orElseGet(() -> readSnapshotOpt.get().getProtocol()); this.shouldUpdateProtocol = newProtocol.isPresent(); this.metadata = newMetadata.orElseGet(() -> readSnapshotOpt.get().getMetadata()); this.shouldUpdateMetadata = newMetadata.isPresent(); this.committer = committer; this.setTxnOpt = setTxnOpt; this.newClusteringColumnsOpt = newClusteringColumnsOpt; this.maxRetries = maxRetriesOpt.orElse(DEFAULT_MAX_RETRIES); this.logCompactionInterval = logCompactionInterval; this.clock = clock; this.currentCrcInfo = readSnapshotOpt.flatMap(SnapshotImpl::getCurrentCrcInfo); } //////////////// // Public API // //////////////// @Override public Row getTransactionState(Engine engine) { return TransactionStateRow.of(metadata, protocol, dataPath.toString(), maxRetries); } @Override public Committer getCommitter() { return committer; } @Override public List getPartitionColumns(Engine engine) { return VectorUtils.toJavaList(metadata.getPartitionColumns()); } @Override public StructType getSchema(Engine engine) { return metadata.getSchema(); } @Override public long getReadTableVersion() { return readSnapshotOpt.map(SnapshotImpl::getVersion).orElse(-1L); } @Override public void withCommitterProperties(Supplier> committerProperties) { this.committerProperties = requireNonNull(committerProperties, "committerProperties is null"); } @Override public void addDomainMetadata(String domain, String config) { checkState( TableFeatures.isDomainMetadataSupported(protocol), "Unable to add domain metadata when the domain metadata table feature is disabled"); checkArgument( DomainMetadata.isUserControlledDomain(domain) || DomainMetadata.isSystemDomainSupportedSetFromTxn(domain), "Setting a non-supported system-controlled domain is not allowed: " + domain); // Specific handling for system domain metadata if (DomainMetadata.isSystemDomainSupportedSetFromTxn(domain)) { handleSystemDomainMetadata(domain, config); } else { domainMetadataState.addDomain(domain, config); } } @Override public void removeDomainMetadata(String domain) { checkState( TableFeatures.isDomainMetadataSupported(protocol), "Unable to add domain metadata when the domain metadata table feature is disabled"); checkArgument( DomainMetadata.isUserControlledDomain(domain), "Removing a system-controlled domain is not allowed: " + domain); domainMetadataState.removeDomain(domain); } @Override public TransactionCommitResult commit(Engine engine, CloseableIterable dataActions) throws ConcurrentWriteException { checkState(!closed, "Transaction is already attempted to commit. Create a new transaction."); // For a new table or when fileSizeHistogram is available in the CRC of the readSnapshotOpt // we update it in the commit. When it is not available we do nothing. TransactionMetrics txnMetrics = readSnapshotOpt .map( snapshot -> TransactionMetrics.withExistingTableFileSizeHistogram( snapshot.getCurrentCrcInfo().flatMap(CRCInfo::getFileSizeHistogram))) .orElse(TransactionMetrics.forNewTable()); try { final Tuple2> committedDeltaAndIct = txnMetrics.totalCommitTimer.time(() -> commitWithRetry(engine, dataActions, txnMetrics)); return buildTransactionCommitResult( engine, committedDeltaAndIct._1, txnMetrics, committedDeltaAndIct._2); } catch (Exception e) { recordTransactionReport( engine, Optional.empty() /* committedVersion */, getEffectiveClusteringColumns(), txnMetrics, Optional.of(e) /* exception */); throw e; } } ////////////////// // Internal API // ////////////////// @VisibleForTesting public void addDomainMetadataInternal(String domain, String config) { domainMetadataState.addDomain(domain, config); } @VisibleForTesting public void removeDomainMetadataInternal(String domain) { domainMetadataState.removeDomain(domain); } public Path getDataPath() { return dataPath; } public Path getLogPath() { return logPath; } public Protocol getProtocol() { return protocol; } public Optional getSetTxnOpt() { return setTxnOpt; } public Optional> getEffectiveClusteringColumns() { if (isCreateOrReplace) { // if isCreateOrReplace return the columns set in this txn return newClusteringColumnsOpt; } else { // since !isCreateOrReplace must be an update to an existing table if (newClusteringColumnsOpt.isPresent()) { // if the clustering columns are being updated in this txn return those return newClusteringColumnsOpt; } else { // else, return the current existing clustering columns (readSnapshotOpt must be present) return readSnapshotOpt.flatMap(SnapshotImpl::getPhysicalClusteringColumns); } } } /////////////////////////////// // Other getters and setters // /////////////////////////////// private boolean isReplaceTable() { return isCreateOrReplace && readSnapshotOpt.isPresent(); } /** * Returns the maximum number of commit attempts, including the first attempt. * *

This is explicitly a method instead of a constant as the maxRetries variable is itself * mutable, and can for example be set to 0 when the rowIdHighWatermark is explicitly provided. */ private int getMaxCommitAttempts() { return maxRetries + 1; // +1 because the first attempt is a try, not a retry. } private Optional isBlindAppend() { // TODO: for now we hard code this to false to avoid erroneously setting this to true for a // non-blind-append operation. We should revisit how to safely set this to true for actual // blind appends. return Optional.of(false); } private void updateMetadata(Metadata metadata) { logger.info( "Updated metadata from {} to {}", shouldUpdateMetadata ? this.metadata : "-", metadata); this.metadata = metadata; this.shouldUpdateMetadata = true; } private void handleSystemDomainMetadata(String domain, String config) { if (domain.equals(RowTrackingMetadataDomain.DOMAIN_NAME)) { if (!TableFeatures.isRowTrackingSupported(protocol)) { throw DeltaErrors.rowTrackingRequiredForRowIdHighWatermark(dataPath.toString(), config); } long providedHighWaterMark = RowTrackingMetadataDomain.fromJsonConfiguration(config).getRowIdHighWaterMark(); checkArgument(providedHighWaterMark >= 0, "rowIdHighWatermark must be >= 0"); this.providedRowIdHighWatermark = Optional.of(providedHighWaterMark); // Conflict resolution is disabled when providedRowIdHighWatermark is set, // because it must be updated according to the latest table state. maxRetries = 0; } } ////////////////////////////////// // Commit Execution (Main Flow) // ////////////////////////////////// /** Returns (commitDeltaData, inCommitTimestamp). */ private Tuple2> commitWithRetry( Engine engine, CloseableIterable dataActions, TransactionMetrics transactionMetrics) { try { long commitAsVersion = getReadTableVersion() + 1; // Generate the commit action with the inCommitTimestamp if ICT is enabled. CommitInfo attemptCommitInfo = generateCommitAction(engine); updateMetadataWithICTIfRequired( engine, attemptCommitInfo.getInCommitTimestamp(), getReadTableVersion()); List resolvedDomainMetadatas = domainMetadataState.getComputedDomainMetadatasToCommit(); // If row tracking is supported, assign base row IDs and default row commit versions to any // AddFile actions that do not yet have them. If the row ID high watermark changes, emit a // DomainMetadata action to update it. if (TableFeatures.isRowTrackingSupported(protocol)) { List updatedDomainMetadata = RowTracking.updateRowIdHighWatermarkIfNeeded( readSnapshotOpt, protocol, Optional.empty() /* winningTxnRowIdHighWatermark */, dataActions, resolvedDomainMetadatas, providedRowIdHighWatermark); domainMetadataState.setComputedDomainMetadatas(updatedDomainMetadata); dataActions = RowTracking.assignBaseRowIdAndDefaultRowCommitVersion( readSnapshotOpt, protocol, Optional.empty() /* winningTxnRowIdHighWatermark */, Optional.empty() /* prevCommitVersion */, commitAsVersion, dataActions); } int attempt = 1; boolean seenRetryableNonConflictException = false; while (true) { // This loop exits upon either (a) commit success (return statement) or (b) commit failure. logger.info( "Attempting to commit transaction at table version {}. Attempt {}/{}", commitAsVersion, attempt, getMaxCommitAttempts()); try { transactionMetrics.commitAttemptsCounter.increment(); return doCommit( engine, commitAsVersion, attemptCommitInfo, dataActions, transactionMetrics); } catch (CommitFailedException cfe) { if (!cfe.isRetryable()) { // Case 1: Non-retryable exception. We must throw this. We don't expect connectors to // be able to recover from this. throw DeltaErrors.nonRetryableCommitException(attempt, commitAsVersion, cfe); } if (attempt >= getMaxCommitAttempts()) { // Case 2: Despite the error being retryable, we have exhausted the maximum number of // retries. We must throw here, too. throw new MaxCommitRetryLimitReachedException(commitAsVersion, maxRetries, cfe); } // We know the commit is retryable. if (!cfe.isConflict()) { // Case 3: No conflict => No conflict resolution needed. Just retry with same version. printLogForRetryableNonConflictException(attempt, commitAsVersion, cfe); seenRetryableNonConflictException = true; } else if (seenRetryableNonConflictException) { checkState(cfe.isRetryable() && cfe.isConflict(), "expect retryable and conflict"); // Case 4: There is a conflict, and we have previously seen a retryable exception // without conflict and then retried. This means that something like the // following has happened: // - Commit Attempt #1: IOException => CFE(retryable=true, conflict=false). We set // seenRetryableNonConflictException to true. Here, there's two possible cases: // (A) N.json was written successfully (and we just never learned about it), // or (B) N.json was not written. // - Commit Attempt #2: FileAlreadyExistsException => CFE(retryable=true, conflict=true) // Should we retry this commit? If it's case (A), then we should not, as we are // just conflicting with our previous commit attempt. If it's case (B), then we // should retry, since we are conflicting with some *other* writer's commit. In // the future we can add detection capabilities between these two cases (e.g. // check if the CommitInfo action is present and has a txnId, else compare the // other contents of the delta files). throw new CommitStateUnknownException(commitAsVersion, attempt, cfe); } else { checkState(cfe.isRetryable() && cfe.isConflict(), "expect retryable and conflict"); // Case 5: There is a conflict, and we have not previously seen a retryable and // non-conflict exception. We will resolve the conflict and retry. printLogForRetryableWithConflictException(attempt, commitAsVersion, cfe); TransactionRebaseState rebaseState = resolveConflicts(engine, commitAsVersion, attemptCommitInfo, attempt, dataActions); commitAsVersion = rebaseState.getLatestVersion() + 1; dataActions = rebaseState.getUpdatedDataActions(); domainMetadataState.setComputedDomainMetadatas(rebaseState.getUpdatedDomainMetadatas()); currentCrcInfo = rebaseState.getUpdatedCrcInfo(); } } // We will be retrying the commit (either from case 3 or 5 above). // // Action counters may be partially incremented from previous tries, reset the counters // to 0 and drop fileSizeHistogram // TODO: [delta-io/delta#5047] reconcile fileSizeHistogram transactionMetrics.resetActionMetricsForRetry(); attempt++; } } finally { closed = true; } } /** Returns (commitDeltaData, inCommitTimestamp). */ private Tuple2> doCommit( Engine engine, long commitAsVersion, CommitInfo attemptCommitInfo, CloseableIterable dataActions, TransactionMetrics transactionMetrics) throws CommitFailedException { List metadataActions = new ArrayList<>(); metadataActions.add(createCommitInfoSingleAction(attemptCommitInfo.toRow())); if (shouldUpdateMetadata) { metadataActions.add(createMetadataSingleAction(metadata.toRow())); } if (shouldUpdateProtocol) { // In the future, we need to add metadata and action when there are any changes to them. metadataActions.add(createProtocolSingleAction(protocol.toRow())); } setTxnOpt.ifPresent(setTxn -> metadataActions.add(createTxnSingleAction(setTxn.toRow()))); List resolvedDomainMetadatas = domainMetadataState.getComputedDomainMetadatasToCommit(); // Check for duplicate domain metadata and if the protocol supports DomainMetadataUtils.validateDomainMetadatas(resolvedDomainMetadatas, protocol); resolvedDomainMetadatas.forEach( dm -> metadataActions.add(createDomainMetadataSingleAction(dm.toRow()))); try (CloseableIterator userStageDataIter = dataActions.iterator()) { final CloseableIterator completeFileActionIter; if (isReplaceTable()) { // If this is a replace table operation we need to internally generate the remove file // actions to reset the table state completeFileActionIter = getRemoveActionsForReplace(engine).combine(userStageDataIter); } else { completeFileActionIter = userStageDataIter; } boolean isAppendOnlyTable = APPEND_ONLY_ENABLED.fromMetadata(metadata); // Create a new CloseableIterator that will return the metadata actions followed by the // data actions. CloseableIterator dataAndMetadataActions = toCloseableIterator(metadataActions.iterator()) .combine(completeFileActionIter) .map( action -> { incrementMetricsForFileActionRow(transactionMetrics, action); if (!action.isNullAt(REMOVE_FILE_ORDINAL)) { RemoveFile removeFile = new RemoveFile(action.getStruct(REMOVE_FILE_ORDINAL)); if (isAppendOnlyTable && removeFile.getDataChange()) { throw DeltaErrors.cannotModifyAppendOnlyTable(dataPath.toString()); } } return action; }); final CommitMetadata commitMetadata = new CommitMetadata( commitAsVersion, logPath.toString(), attemptCommitInfo, resolvedDomainMetadatas, committerProperties, readSnapshotOpt.map(x -> new Tuple2<>(x.getProtocol(), x.getMetadata())), shouldUpdateProtocol ? Optional.of(protocol) : Optional.empty(), shouldUpdateMetadata ? Optional.of(metadata) : Optional.empty(), readSnapshotOpt .map(x -> x.getLogSegment().getMaxPublishedDeltaVersion()) .orElse(Optional.of(-1L))); DirectoryCreationUtils.createAllDeltaDirectoriesAsNeeded( engine, logPath, commitAsVersion, commitMetadata.getReadProtocolOpt(), protocol); return new Tuple2<>( committer.commit(engine, dataAndMetadataActions, commitMetadata).getCommitLogData(), attemptCommitInfo.getInCommitTimestamp()); } catch (IOException ioe) { // Error closing the CloseableIterator of actions or error creating the delta log directory throw new UncheckedIOException(ioe); } } //////////////////////////////// // Commit Execution (Helpers) // //////////////////////////////// private CommitInfo generateCommitAction(Engine engine) { long commitAttemptStartTime = clock.getTimeMillis(); return new CommitInfo( generateInCommitTimestampForFirstCommitAttempt(engine, commitAttemptStartTime), commitAttemptStartTime, /* timestamp */ Optional.of("Kernel-" + Meta.KERNEL_VERSION + "/" + engineInfo), /* engineInfo */ Optional.of(operation.getDescription()), /* description */ getOperationParameters(), /* operationParameters */ isBlindAppend(), /* isBlindAppend */ Optional.of(txnId.toString()), /* txnId */ emptyMap() /* operationMetrics */); } /** * Generates a timestamp which is greater than the commit timestamp of the readSnapshotOpt. This * can result in an additional file read and that this will only happen if ICT is enabled. */ private Optional generateInCommitTimestampForFirstCommitAttempt( Engine engine, long currentTimestamp) { if (IN_COMMIT_TIMESTAMPS_ENABLED.fromMetadata(metadata)) { if (readSnapshotOpt.isPresent()) { long lastCommitTimestamp = readSnapshotOpt.get().getTimestamp(engine); return Optional.of(Math.max(currentTimestamp, lastCommitTimestamp + 1)); } else { // For a new table this is just the current timestamp return Optional.of(currentTimestamp); } } else { return Optional.empty(); } } private Map getOperationParameters() { if (isCreateOrReplace) { List partitionCols = VectorUtils.toJavaList(metadata.getPartitionColumns()); String partitionBy = partitionCols.stream() .map(col -> "\"" + col + "\"") .collect(Collectors.joining(",", "[", "]")); return Collections.singletonMap("partitionBy", partitionBy); } return emptyMap(); } private void updateMetadataWithICTIfRequired( Engine engine, Optional inCommitTimestampOpt, long lastCommitVersion) { // If ICT is enabled for the current transaction, update the metadata with the ICT // enablement info. inCommitTimestampOpt.ifPresent( inCommitTimestamp -> { Optional metadataWithICTInfo = InCommitTimestampUtils.getUpdatedMetadataWithICTEnablementInfo( engine, inCommitTimestamp, readSnapshotOpt, metadata, lastCommitVersion + 1L); metadataWithICTInfo.ifPresent(this::updateMetadata); }); } private void incrementMetricsForFileActionRow(TransactionMetrics txnMetrics, Row fileActionRow) { txnMetrics.totalActionsCounter.increment(); if (!fileActionRow.isNullAt(ADD_FILE_ORDINAL)) { txnMetrics.updateForAddFile(new AddFile(fileActionRow.getStruct(ADD_FILE_ORDINAL)).getSize()); } else if (!fileActionRow.isNullAt(REMOVE_FILE_ORDINAL)) { RemoveFile removeFile = new RemoveFile(fileActionRow.getStruct(REMOVE_FILE_ORDINAL)); long removeFileSize = removeFile.getSize().orElseThrow(DeltaErrorsInternal::missingRemoveFileSizeDuringCommit); txnMetrics.updateForRemoveFile(removeFileSize); } } /** * Returns the remove file rows needed to remove every active add file in the table. These rows * are already formatted as {@link SingleAction} rows and are ready to be committed. */ private CloseableIterator getRemoveActionsForReplace(Engine engine) { checkArgument( readSnapshotOpt.isPresent(), "Cannot generate removes for a snapshot with version < 0"); Scan scan = readSnapshotOpt.get().getScanBuilder().build(); return Utils.intoRows(scan.getScanFiles(engine)) .map( scanRow -> { AddFile add = new AddFile(scanRow.getStruct(InternalScanFileUtils.ADD_FILE_ORDINAL)); return SingleAction.createRemoveFileSingleAction( add.toRemoveFileRow(true /* dataChange */, Optional.empty())); }); } ///////////////////////// // Conflict Resolution // ///////////////////////// private TransactionRebaseState resolveConflicts( Engine engine, long commitAsVersion, CommitInfo attemptCommitInfo, int attempt, CloseableIterable dataActions) { logger.info( "[{}] Trying to resolve conflicts and retry commit. Attempt {}/{}.", dataPath, attempt, getMaxCommitAttempts()); TransactionRebaseState rebaseState = ConflictChecker.resolveConflicts( engine, readSnapshotOpt, commitAsVersion, this, domainMetadataState.getComputedDomainMetadatasToCommit(), dataActions); long newCommitAsVersion = rebaseState.getLatestVersion() + 1; checkArgument( commitAsVersion < newCommitAsVersion, "New commit version %d should be greater than the previous commit attempt version %d.", newCommitAsVersion, commitAsVersion); Optional updatedInCommitTimestamp = getUpdatedInCommitTimestampAfterConflict( rebaseState.getLatestCommitTimestamp(), attemptCommitInfo.getInCommitTimestamp()); updateMetadataWithICTIfRequired( engine, updatedInCommitTimestamp, rebaseState.getLatestVersion()); attemptCommitInfo.setInCommitTimestamp(updatedInCommitTimestamp); return rebaseState; } private Optional getUpdatedInCommitTimestampAfterConflict( long winningCommitTimestamp, Optional attemptInCommitTimestamp) { if (attemptInCommitTimestamp.isPresent()) { long updatedInCommitTimestamp = Math.max(attemptInCommitTimestamp.get(), winningCommitTimestamp + 1); return Optional.of(updatedInCommitTimestamp); } return attemptInCommitTimestamp; } //////////////////////////// // Post-Commit Processing // //////////////////////////// private TransactionCommitResult buildTransactionCommitResult( Engine engine, ParsedDeltaData committedDelta, TransactionMetrics txnMetrics, Optional committedIctOpt) { final long committedVersion = committedDelta.getVersion(); final TransactionReport transactionReport = recordTransactionReport( engine, Optional.of(committedVersion), getEffectiveClusteringColumns(), txnMetrics, Optional.empty() /* exception */); final TransactionMetricsResult txnMetricsCaptured = txnMetrics.captureTransactionMetricsResult(); final Optional postCommitCrcOpt = buildPostCommitCrcInfoIfCurrentCrcAvailable(committedVersion, txnMetricsCaptured); final Optional postCommitSnapshotOpt = buildPostCommitSnapshotOpt(engine, committedDelta, committedIctOpt, postCommitCrcOpt); return new TransactionCommitResult( committedVersion, generatePostCommitHooks(committedVersion, postCommitCrcOpt), transactionReport, postCommitSnapshotOpt); } private Optional buildPostCommitSnapshotOpt( Engine engine, ParsedDeltaData committedDelta, Optional committedIctOpt, Optional postCommitCrcOpt) { // TODO: Support building post-commit Snapshots after conflicts. If there was a conflict, then // we'd need to keep track of each of the conflicting commit files in order to build the // new LogSegment for our post-commit Snapshot. This is currently not done, today. Note // that for catalogManaged tables, we would need the Committer to provide the conflicting // commits as part of the CommitFailedException. if (committedDelta.getVersion() != getReadTableVersion() + 1) { return Optional.empty(); } final LogSegment postCommitLogSegment = buildPostCommitLogSegment(committedDelta); final Lazy lazyLogSegment = new Lazy<>(() -> postCommitLogSegment); final Lazy> lazyCrcInfo = new Lazy<>(() -> postCommitCrcOpt); final LogReplay logReplay = new LogReplay(engine, dataPath, lazyLogSegment, lazyCrcInfo); // TODO: SnapshotQueryContext.forPostCommitSnapshot final SnapshotQueryContext snapshotContext = SnapshotQueryContext.forVersionSnapshot(dataPath.toString(), committedDelta.getVersion()); final SnapshotImpl postCommitSnapshot = new SnapshotImpl( dataPath, committedDelta.getVersion(), lazyLogSegment, logReplay, protocol, metadata, committer, snapshotContext, committedIctOpt); return Optional.of(postCommitSnapshot); } private LogSegment buildPostCommitLogSegment(ParsedDeltaData committedDelta) { if (readSnapshotOpt.isPresent()) { return readSnapshotOpt .get() .getLogSegment() .newWithAddedDeltas(Collections.singletonList(committedDelta)); } return LogSegment.createForNewTable(logPath, committedDelta); } private List generatePostCommitHooks( long committedVersion, Optional postCommitCrcOpt) { final List postCommitHooks = new ArrayList<>(); if (isReadyForCheckpoint(committedVersion)) { postCommitHooks.add(new CheckpointHook(dataPath, committedVersion)); } if (postCommitCrcOpt.isPresent()) { postCommitHooks.add(new ChecksumSimpleHook(postCommitCrcOpt.get(), logPath)); } else { postCommitHooks.add(new ChecksumFullHook(dataPath, committedVersion)); } if (logCompactionInterval > 0 && LogCompactionWriter.shouldCompact(committedVersion, logCompactionInterval)) { // add one here because commits start a 0 long startVersion = committedVersion + 1 - logCompactionInterval; long minFileRetentionTimestampMillis = clock.getTimeMillis() - TOMBSTONE_RETENTION.fromMetadata(metadata); postCommitHooks.add( new LogCompactionHook( dataPath, logPath, startVersion, committedVersion, minFileRetentionTimestampMillis)); } return postCommitHooks; } private boolean isReadyForCheckpoint(long newVersion) { int checkpointInterval = CHECKPOINT_INTERVAL.fromMetadata(metadata); return newVersion > 0 && newVersion % checkpointInterval == 0; } private Optional buildPostCommitCrcInfoIfCurrentCrcAvailable( long commitAtVersion, TransactionMetricsResult metricsResult) { if (isCreateOrReplace) { // We don't need to worry about conflicting transaction here since new tables always commit // metadata (and thus fail any conflicts) return Optional.of( new CRCInfo( commitAtVersion, metadata, protocol, metricsResult.getTotalAddFilesSizeInBytes(), metricsResult.getNumAddFiles(), Optional.of(txnId.toString()), domainMetadataState.getPostCommitDomainMetadatas(), metricsResult .getTableFileSizeHistogram() .map(FileSizeHistogram::fromFileSizeHistogramResult))); } return currentCrcInfo // Ensure current currentCrcInfo is exactly commitAtVersion - 1 .filter(crcInfo -> commitAtVersion == crcInfo.getVersion() + 1) .map( lastCrcInfo -> new CRCInfo( commitAtVersion, metadata, protocol, lastCrcInfo.getTableSizeBytes() + metricsResult.getTotalAddFilesSizeInBytes() - metricsResult.getTotalRemoveFilesSizeInBytes(), lastCrcInfo.getNumFiles() + metricsResult.getNumAddFiles() - metricsResult.getNumRemoveFiles(), Optional.of(txnId.toString()), domainMetadataState.getPostCommitDomainMetadatas(), metricsResult .getTableFileSizeHistogram() .map(FileSizeHistogram::fromFileSizeHistogramResult))); } private TransactionReport recordTransactionReport( Engine engine, Optional committedVersion, Optional> clusteringColumnsOpt, TransactionMetrics transactionMetrics, Optional exception) { TransactionReport transactionReport = new TransactionReportImpl( dataPath.toString() /* tablePath */, operation.toString(), engineInfo, committedVersion, clusteringColumnsOpt, transactionMetrics, readSnapshotOpt.map(SnapshotImpl::getSnapshotReport), exception); engine.getMetricsReporters().forEach(reporter -> reporter.report(transactionReport)); return transactionReport; } ///////////////////// // Logging Helpers // ///////////////////// private void printLogForRetryableNonConflictException( int attempt, long commitAsVersion, CommitFailedException cfe) { logger.warn( "Commit attempt {} for table version {} failed with a retryable exception and without " + "conflict. Skipping conflict resolution and trying again. Exception: {}", attempt, commitAsVersion, cfe); } private void printLogForRetryableWithConflictException( int attempt, long commitAsVersion, CommitFailedException cfe) { logger.warn( "Commit attempt {} for version {} failed with a retryable exception due to a physical " + "conflict. Performing conflict resolution and trying again. Exception: {}", attempt, commitAsVersion, cfe); } //////////////////// // Helper Classes // //////////////////// /** Encapsulates the state of domain metadata within a transaction. */ private class DomainMetadataState { private final Map domainsToAdd = new HashMap<>(); private final Set domainsToRemove = new HashSet<>(); private Optional> computedMetadatas = Optional.empty(); /** Adds a domain metadata. Invalidates any cached computed state. */ public void addDomain(String domain, String config) { checkArgument( !domainsToRemove.contains(domain), "Cannot add a domain that is removed in this transaction"); checkState(!closed, "Cannot add a domain metadata after the transaction has completed"); // Add the domain and invalidate cache domainsToAdd.put(domain, new DomainMetadata(domain, config, false /* removed */)); computedMetadatas = Optional.empty(); } /** Marks a domain for removal. Invalidates any cached computed state. */ public void removeDomain(String domain) { checkArgument( !domainsToAdd.containsKey(domain), "Cannot remove a domain that is added in this transaction"); checkState(!closed, "Cannot remove a domain after the transaction has completed"); // Mark for removal and invalidate cache domainsToRemove.add(domain); computedMetadatas = Optional.empty(); } /** * Returns a list of the domain metadatas to commit. This consists of the domain metadatas added * in the transaction using {@link Transaction#addDomainMetadata(String, String)} and the * tombstones for the domain metadatas removed in the transaction using {@link * Transaction#removeDomainMetadata(String)}. * * @return A list of {@link DomainMetadata} containing domain metadata to be committed in this * transaction. */ public List getComputedDomainMetadatasToCommit() { if (computedMetadatas.isPresent()) { return computedMetadatas.get(); } generateClusteringDomainMetadataIfNeeded(); if (isReplaceTable()) { // In the case of replace table we need to completely reset the table state by removing // any existing domain metadata readSnapshotOpt .get() // if replaceTable we know snapshot is present .getActiveDomainMetadataMap() .forEach( (domainName, domainMetadata) -> { if (!domainsToAdd.containsKey(domainName)) { // We only need to remove the domain if it is not added (& thus overwritten) // in this current transaction. We cannot add and remove the same domain in // one transaction. removeDomain(domainName); } }); } // Add all domains added in the transaction List result = new ArrayList<>(domainsToAdd.values()); if (domainsToRemove.isEmpty()) { // If no domain metadatas are removed we don't need to load the existing domain metadatas // from the snapshot (which is an expensive operation) computedMetadatas = Optional.of(result); return result; } // Generate the tombstones for removed domains Map snapshotDomainMetadataMap = readSnapshotOpt.map(SnapshotImpl::getActiveDomainMetadataMap).orElse(emptyMap()); for (String domainName : domainsToRemove) { if (snapshotDomainMetadataMap.containsKey(domainName)) { // Note: we know domainName is not already in finalDomainMetadatas because we do not allow // removing and adding a domain with the same identifier in a single txn! DomainMetadata domainToRemove = snapshotDomainMetadataMap.get(domainName); checkState( !domainToRemove.isRemoved(), "snapshotDomainMetadataMap should only contain active domain metadata"); result.add(domainToRemove.removed()); } else { // We must throw an error if the domain does not exist. Otherwise, there could be // unexpected // behavior within conflict resolution. For example, consider the following // 1. Table has no domains set in V0 // 2. txnA is started and wants to remove domain "foo" // 3. txnB is started and adds domain "foo" and commits V1 before txnA // 4. txnA needs to perform conflict resolution against the V1 commit from txnB // Conflict resolution should fail but since the domain does not exist we cannot create // a tombstone to mark it as removed and correctly perform conflict resolution. throw new DomainDoesNotExistException( dataPath.toString(), domainName, getReadTableVersion()); } } computedMetadatas = Optional.of(result); return result; } /** Sets the computed domain metadata list directly. Used during conflict resolution. */ public void setComputedDomainMetadatas(List updatedDomainMetadatas) { computedMetadatas = Optional.of(updatedDomainMetadatas); } /** * Returns the set of active domain metadata of the table, removed domain metadata are excluded. */ public Optional> getPostCommitDomainMetadatas() { if (!readSnapshotOpt.isPresent()) { return Optional.of( getComputedDomainMetadatasToCommit().stream() .filter(dm -> !dm.isRemoved()) .collect(Collectors.toSet())); } return currentCrcInfo .flatMap(CRCInfo::getDomainMetadata) .map( oldDomainMetadata -> { Map domainMetadataMap = oldDomainMetadata.stream() .collect(Collectors.toMap(DomainMetadata::getDomain, Function.identity())); getComputedDomainMetadatasToCommit() .forEach( domainMetadata -> { if (domainMetadata.isRemoved()) { domainMetadataMap.remove(domainMetadata.getDomain()); } else { domainMetadataMap.put(domainMetadata.getDomain(), domainMetadata); } }); return new HashSet<>(domainMetadataMap.values()); }); } /** * Generate the domain metadata for the clustering columns if they are present in the * transaction. */ private void generateClusteringDomainMetadataIfNeeded() { if (TableFeatures.isClusteringTableFeatureSupported(protocol) && newClusteringColumnsOpt.isPresent()) { DomainMetadata clusteringDomainMetadata = ClusteringUtils.getClusteringDomainMetadata(newClusteringColumnsOpt.get()); addDomain( clusteringDomainMetadata.getDomain(), clusteringDomainMetadata.getConfiguration()); } else if (TableFeatures.isClusteringTableFeatureSupported(protocol) && isReplaceTable() && !newClusteringColumnsOpt.isPresent()) { // When clustering is in the writer features we require there to be a clustering domain // metadata present; when the table is no longer a clustered table this means we must have // a domain metadata with clusteringColumns=[] DomainMetadata emptyClusteringDomainMetadata = ClusteringUtils.getClusteringDomainMetadata(Collections.emptyList()); addDomain( emptyClusteringDomainMetadata.getDomain(), emptyClusteringDomainMetadata.getConfiguration()); } } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionMetadataFactory.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal; import static io.delta.kernel.internal.ReplaceTableTransactionBuilderV2Impl.TABLE_PROPERTY_KEYS_TO_PRESERVE; import static io.delta.kernel.internal.TransactionImpl.DEFAULT_READ_VERSION; import static io.delta.kernel.internal.TransactionImpl.DEFAULT_WRITE_VERSION; import static io.delta.kernel.internal.tablefeatures.TableFeatures.ALLOW_COLUMN_DEFAULTS_W_FEATURE; import static io.delta.kernel.internal.util.ColumnMapping.isColumnMappingModeEnabled; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static io.delta.kernel.internal.util.Preconditions.checkState; import static io.delta.kernel.internal.util.SchemaUtils.casePreservingPartitionColNames; import static io.delta.kernel.internal.util.VectorUtils.buildArrayValue; import static io.delta.kernel.internal.util.VectorUtils.stringStringMapValue; import static java.util.Collections.*; import static java.util.stream.Collectors.toSet; import io.delta.kernel.commit.CatalogCommitter; import io.delta.kernel.commit.Committer; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.expressions.Column; import io.delta.kernel.internal.actions.*; import io.delta.kernel.internal.icebergcompat.*; import io.delta.kernel.internal.rowtracking.MaterializedRowTrackingColumn; import io.delta.kernel.internal.rowtracking.RowTracking; import io.delta.kernel.internal.tablefeatures.TableFeature; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.internal.util.ColumnMapping; import io.delta.kernel.internal.util.ColumnMapping.ColumnMappingMode; import io.delta.kernel.internal.util.SchemaUtils; import io.delta.kernel.internal.util.Tuple2; import io.delta.kernel.types.StringType; import io.delta.kernel.types.StructType; import java.util.*; import java.util.function.Consumer; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Class for building the Protocol, Metadata, and ResolvedClusteringColumns to commit in a * transaction. This means it validates and updates the protocol and metadata according to the * inputs. */ public class TransactionMetadataFactory { private static final Logger logger = LoggerFactory.getLogger(TransactionMetadataFactory.class); /** * Expectations with respect to the given operation: * *

    *
  • Create table: both protocol and metadata will be present *
  • Replace table: both protocol and metadata will be present *
  • Update table: metadata and protocol may or may not be present depending on whether there * should be a metadata or protocol update *
*/ public static class Output { /* New metadata, present if the transaction should commit a new metadata action */ public final Optional newProtocol; /* New protocol, present if the transaction should commit a new protocol action */ public final Optional newMetadata; /* Resolved _new_ clustering columns if the transaction should update the clustering columns */ public final Optional> physicalNewClusteringColumns; public Output( Optional newProtocol, Optional newMetadata, Optional> physicalNewClusteringColumns) { this.newProtocol = newProtocol; this.newMetadata = newMetadata; this.physicalNewClusteringColumns = physicalNewClusteringColumns; } } //////////////////////////// // Static factory methods // //////////////////////////// static Output buildCreateTableMetadata( String tablePath, StructType schema, Map userInputTableProperties, Optional> partitionColumns, Optional> clusteringColumns, Optional committerOpt) { checkArgument( !partitionColumns.isPresent() || !clusteringColumns.isPresent(), "Cannot provide both partition columns and clustering columns"); validateSchemaAndPartColsCreateOrReplace( userInputTableProperties, schema, partitionColumns.orElse(emptyList())); final Map requiredCatalogTableProperties = committerOpt .map(TransactionMetadataFactory::getRequiredCatalogTablePropertiesIfApplicable) .orElse(Collections.emptyMap()); // We put the required catalog table properties *first* so that we persist the intent, if any, // of the user explicitly setting a required catalog table property. If it's set to an invalid // value, we will detect this and fail later inside TransactionMetadataFactory. final Map allCreateTableProperties = new HashMap<>(); allCreateTableProperties.putAll(requiredCatalogTableProperties); allCreateTableProperties.putAll(userInputTableProperties); Output output = new TransactionMetadataFactory( tablePath, Optional.empty() /* readSnapshot */, Optional.of( getInitialMetadata( schema, allCreateTableProperties, partitionColumns.orElse(emptyList()))), Optional.of(getInitialProtocol()), userInputTableProperties /* originalUserInputProperties */, true /* isCreateOrReplace */, clusteringColumns, false /* isSchemaEvolution */, committerOpt) .finalOutput; checkState( output.newMetadata.isPresent() && output.newProtocol.isPresent(), "Expected non-null metadata and protocol for create table"); return output; } static Output buildReplaceTableMetadata( String tablePath, SnapshotImpl readSnapshot, StructType schema, Map userInputTableProperties, Optional> partitionColumns, Optional> clusteringColumns) { checkArgument( !partitionColumns.isPresent() || !clusteringColumns.isPresent(), "Cannot provide both partition columns and clustering columns"); validateSchemaAndPartColsCreateOrReplace( userInputTableProperties, schema, partitionColumns.orElse(emptyList())); validateNotEnablingCatalogManagedOnReplace(userInputTableProperties); final Map requiredCatalogTableProperties = getRequiredCatalogTablePropertiesIfApplicable(readSnapshot.getCommitter()); final Map allReplaceTableProperties = new HashMap<>(); // Step 1: We put the required catalog table properties *first* so that we persist the intent, // if any, of the user explicitly setting a required catalog table property. If it's set to an // invalid value, we will detect this and fail later inside TransactionMetadataFactory. allReplaceTableProperties.putAll(requiredCatalogTableProperties); // Step 2: Preserve a few important delta properties allReplaceTableProperties.putAll( readSnapshot.getMetadata().getConfiguration().entrySet().stream() .filter(e -> TABLE_PROPERTY_KEYS_TO_PRESERVE.contains(e.getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); // Step 3: Insert the new user-provided table properties allReplaceTableProperties.putAll(userInputTableProperties); Output output = new TransactionMetadataFactory( tablePath, Optional.of(readSnapshot), Optional.of( getInitialMetadata( schema, allReplaceTableProperties, partitionColumns.orElse(emptyList()))), Optional.of(readSnapshot.getProtocol()), userInputTableProperties, true /* isCreateOrReplace */, clusteringColumns, false /* isSchemaEvolution */, Optional.of(readSnapshot.getCommitter())) .finalOutput; // TODO: reconsider whether we should always commit a new Protocol action regardless of whether // there is a protocol upgrade checkState( output.newMetadata.isPresent() && output.newProtocol.isPresent(), "Expected non-null metadata and protocol for replace table"); if (output .newProtocol .orElse(readSnapshot.getProtocol()) .supportsFeature(TableFeatures.ICEBERG_COMPAT_V3_W_FEATURE)) { // Block this for now to be safe, we will return to this in the future // once replace for rowTracking is enabled throw new UnsupportedOperationException( "REPLACE TABLE is not yet supported on IcebergCompatV3 tables"); } return output; } static Output buildUpdateTableMetadata( String tablePath, SnapshotImpl readSnapshot, Optional> propertiesAdded, Optional> propertyKeysRemoved, Optional newSchema, Optional> clusteringColumns) { if (propertiesAdded.isPresent() && propertyKeysRemoved.isPresent()) { Set overlappingPropertyKeys = propertyKeysRemoved.get().stream() .filter(key -> propertiesAdded.get().containsKey(key)) .collect(toSet()); if (!overlappingPropertyKeys.isEmpty()) { throw DeltaErrors.overlappingTablePropertiesSetAndUnset(overlappingPropertyKeys); } } Optional newMetadata = Optional.empty(); Map newProperties = readSnapshot .getMetadata() .filterOutUnchangedProperties(propertiesAdded.orElse(Collections.emptyMap())); if (!newProperties.isEmpty()) { newMetadata = Optional.of(readSnapshot.getMetadata().withMergedConfiguration(newProperties)); } if (propertyKeysRemoved.isPresent()) { newMetadata = Optional.of( newMetadata .orElse(readSnapshot.getMetadata()) .withConfigurationKeysUnset(propertyKeysRemoved.get())); } if (newSchema.isPresent()) { newMetadata = Optional.of( newMetadata.orElse(readSnapshot.getMetadata()).withNewSchema(newSchema.get())); } return new TransactionMetadataFactory( tablePath, Optional.of(readSnapshot), newMetadata, Optional.empty(), propertiesAdded.orElse(Collections.emptyMap()) /* originalUserInputProperties */, false /* isCreateOrReplace */, clusteringColumns, newSchema.isPresent() /* isSchemaEvolution */, Optional.of(readSnapshot.getCommitter())) .finalOutput; } /////////////////////////////// // Instance Fields / Methods // /////////////////////////////// // ===== Fields that set by input ===== private final String tablePath; private final Optional latestSnapshotOpt; /** * The table properties provided in this transaction. i.e. excludes any properties in the read * snapshot. * *

This helps validation code understand what the user is trying to do in *this* transaction, * as opposed to what is the current state already in the table. */ private final Map originalUserInputProperties; private final boolean isCreateOrReplace; private final boolean isSchemaEvolution; // ===== Fields that are updated by helper methods when updating and validating the metadata ===== private Optional newMetadata; private Optional newProtocol; private Optional> physicalNewClusteringColumns; // ===== Fields that are fixed after validation and updates are finished ===== private final Output finalOutput; /** * @param initialNewMetadata the initial metadata that we should validate and transform. It is a * function of the readSnapshot's metadata (if applicable) joined with any _user provided_ * table property updates. Specifically: *

    *
  • CREATE: default empty metadata merged with schema, partCols, and user-specified table * properties *
  • UPDATE: readSnapshot's metadata merged wth user-specified added/removed table * properties *
  • REPLACE: readSnapshot's metadata, with all table properties removed except for those * that are included in TABLE_PROPERTY_KEYS_TO_PRESERVE, merged with schema, partCols, * and user-specified table properties *
*

This class may apply additional updates to transform the {@code initialNewMetadata} the * final output (e.g. auto-enabling column mapping for iceberg compat, adding column mapping * metadata to the schema, etc.) */ private TransactionMetadataFactory( String tablePath, Optional latestSnapshotOpt, Optional initialNewMetadata, Optional initialNewProtocol, Map originalUserInputProperties, boolean isCreateOrReplace, Optional> userProvidedLogicalClusteringColumns, boolean isSchemaEvolution, Optional committerOpt) { checkArgument( (initialNewMetadata.isPresent() && initialNewProtocol.isPresent()) || latestSnapshotOpt.isPresent(), "initial protocol and metadata must be present for create table"); checkArgument( !isSchemaEvolution || !isCreateOrReplace, "isSchemaEvolution can only be true for update table"); checkArgument( isCreateOrReplace || latestSnapshotOpt.isPresent(), "update table must provide a latest snapshot"); this.tablePath = tablePath; this.latestSnapshotOpt = latestSnapshotOpt; this.originalUserInputProperties = originalUserInputProperties; this.isCreateOrReplace = isCreateOrReplace; this.isSchemaEvolution = isSchemaEvolution; this.newMetadata = initialNewMetadata; this.newProtocol = initialNewProtocol; performProtocolUpgrades(userProvidedLogicalClusteringColumns.isPresent()); handleCatalogManagedEnablement(); handleInCommitTimestampDisablement(); performIcebergCompatUpgradesAndValidation(); updateColumnMappingMetadataAndResolveClusteringColumns(userProvidedLogicalClusteringColumns); updateRowTrackingMetadata(); validateMetadataChangeAndApplyTypeWidening(); validateRequiredCatalogTablePropertiesSet(committerOpt); this.finalOutput = new Output(newProtocol, newMetadata, physicalNewClusteringColumns); } private Metadata getEffectiveMetadata() { // Fact: either newMetadata is defined upon initiation or latestSnapshotOpt is present return newMetadata.orElseGet(() -> latestSnapshotOpt.get().getMetadata()); } private Protocol getEffectiveProtocol() { // Fact: either newProtocol is defined upon initiation or latestSnapshotOpt is present return newProtocol.orElseGet(() -> latestSnapshotOpt.get().getProtocol()); } private void validateForUpdateTableUsingOldMetadata(Consumer validateFn) { if (!isCreateOrReplace) { // For update table we know latestSnapshotOpt is present Metadata oldMetadata = latestSnapshotOpt.get().getMetadata(); validateFn.accept(oldMetadata); } } /** STEP 1: Update the PROTOCOL based on the table properties or schema */ // TODO: if you only update the feature properties we currently write a new Metadata even though // this should just be a protocol upgrade (this could be an issue if for example you use // .withDomainMetadata supported in every txn --- we always write a new Metadata action) private void performProtocolUpgrades(boolean clusteringRequired) { // This is the only place we update the protocol action; takes care of any dependent features // Ex: We enable feature `icebergCompatV2` plus dependent features `columnMapping` // This will remove feature properties (i.e. metadata properties in the form of // "delta.feature.*") from metadata. There should be one TableFeature in the returned set for // each property removed. Tuple2, Optional> newFeaturesAndMetadata = TableFeatures.extractFeaturePropertyOverrides(getEffectiveMetadata()); if (newFeaturesAndMetadata._2.isPresent()) { newMetadata = newFeaturesAndMetadata._2; } // Enable clustering if not already enabled and clustering columns are set. This isn't handled // in autoUpgradeProtocolBasedOnMetadata since clustering is stored in the domain metadata if (clusteringRequired && !getEffectiveProtocol().supportsFeature(TableFeatures.CLUSTERING_W_FEATURE)) { newFeaturesAndMetadata._1.add(TableFeatures.CLUSTERING_W_FEATURE); } Optional>> newProtocolAndFeatures = TableFeatures.autoUpgradeProtocolBasedOnMetadata( getEffectiveMetadata(), newFeaturesAndMetadata._1, getEffectiveProtocol()); if (newProtocolAndFeatures.isPresent()) { logger.info( "Automatically enabling table features: {}", newProtocolAndFeatures.get()._2.stream().map(TableFeature::featureName).collect(toSet())); newProtocol = Optional.of(newProtocolAndFeatures.get()._1); TableFeatures.validateKernelCanWriteToTable( getEffectiveProtocol(), getEffectiveMetadata(), tablePath); } } /** STEP 1.1: Handle catalogManaged enablement. Updates the METADATA if applicable. */ private void handleCatalogManagedEnablement() { final boolean readVersionSupportsCatalogManaged = latestSnapshotOpt .map(x -> TableFeatures.isCatalogManagedSupported(x.getProtocol())) .orElse(false); final boolean writeVersionSupportsCatalogManaged = TableFeatures.isCatalogManagedSupported(getEffectiveProtocol()); final boolean txnEnablesCatalogManaged = !readVersionSupportsCatalogManaged && writeVersionSupportsCatalogManaged; // Case 1: Txn is not enabling catalogManaged. Exit. if (!txnEnablesCatalogManaged) { return; } // catalogManaged is being enabled in this transaction. catalogManaged dependsOn // inCommitTimestamp. The inCommitTimestamp feature should have been auto-supported in the // protocol by now. checkState( getEffectiveProtocol().supportsFeature(TableFeatures.IN_COMMIT_TIMESTAMP_W_FEATURE), "inCommitTimestamp feature expected to have already been supported"); // Case 2: Txn is explicitly disabling ICT. Throw. final String txnIctEnabledValue = originalUserInputProperties.get(TableConfig.IN_COMMIT_TIMESTAMPS_ENABLED.getKey()); final boolean txnExplicitlyDisablesICT = txnIctEnabledValue != null && txnIctEnabledValue.equalsIgnoreCase("false"); if (txnExplicitlyDisablesICT) { throw new KernelException("Cannot disable inCommitTimestamp when enabling catalogManaged"); } // Case 3: ICT already enabled. Exit. final boolean isIctAlreadyEnabled = TableConfig.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetadata(getEffectiveMetadata()); if (isIctAlreadyEnabled) { return; } // Case 4: ICT is not enabled. Enable it. newMetadata = Optional.of( getEffectiveMetadata() .withMergedConfiguration( Collections.singletonMap( TableConfig.IN_COMMIT_TIMESTAMPS_ENABLED.getKey(), "true"))); } /** * Step 1.2: Handle inCommitTimestamp disablement. Updates the METADATA if applicable. * *

If the user explicitly disables inCommitTimestamp in this transaction, we then also * explicitly remove the ICT enablement version and timestamp properties from the metadata. */ private void handleInCommitTimestampDisablement() { final String txnIctEnabledValue = originalUserInputProperties.get(TableConfig.IN_COMMIT_TIMESTAMPS_ENABLED.getKey()); final boolean txnExplicitlyDisablesICT = txnIctEnabledValue != null && txnIctEnabledValue.equalsIgnoreCase("false"); // Case 1: Txn is not explicitly disabling ICT. Exit. if (!txnExplicitlyDisablesICT) { return; } // Case 2: Txn is explicitly disabling ICT on a catalogManaged table. Throw. if (getEffectiveProtocol().supportsFeature(TableFeatures.CATALOG_MANAGED_RW_FEATURE)) { throw new KernelException("Cannot disable inCommitTimestamp on a catalogManaged table"); } // Case 3 (normal case): Txn is explicitly disabling ICT. Remove the ICT enablement properties. final Set ictKeysToRemove = new HashSet<>( Arrays.asList( TableConfig.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.getKey(), TableConfig.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.getKey())); newMetadata = Optional.of(getEffectiveMetadata().withConfigurationKeysUnset(ictKeysToRemove)); } /** * STEP 2: Validate the METADATA and PROTOCOL and possibly update the METADATA for IcebergCompat */ private void performIcebergCompatUpgradesAndValidation() { // IcebergCompat validates that the current metadata and protocol is compatible (e.g. all the // required TF are present, no incompatible types, etc). It also updates the metadata for new // tables if needed (e.g. enables column mapping) // Ex: We enable column mapping mode in the configuration such that our properties now include // Map(delta.enableIcebergCompatV2 -> true, delta.columnMapping.mode -> name) // Validate this is a valid config change earlier for a clearer error message validateForUpdateTableUsingOldMetadata( oldMetadata -> { newMetadata.ifPresent( metadata -> IcebergWriterCompatV1MetadataValidatorAndUpdater .validateIcebergWriterCompatV1Change( oldMetadata.getConfiguration(), metadata.getConfiguration())); newMetadata.ifPresent( metadata -> IcebergCompatV3MetadataValidatorAndUpdater.validateIcebergCompatV3Change( oldMetadata.getConfiguration(), metadata.getConfiguration())); }); // Pass the previous protocol to IcebergCompat checks as defense-in-depth: if the read // snapshot already had deletion vectors enabled, the check should reject it even when the // new protocol in this transaction does not include DVs. The conflict checker may also // catch this at commit time, but checking early gives a clearer error message. Optional prevProtocol = latestSnapshotOpt.map(s -> s.getProtocol()); // We must do our icebergWriterCompatV1 checks/updates FIRST since it has stricter column // mapping requirements (id mode) than icebergCompatV2. It also may enable icebergCompatV2. Optional icebergWriterCompatV1 = IcebergWriterCompatV1MetadataValidatorAndUpdater .validateAndUpdateIcebergWriterCompatV1Metadata( isCreateOrReplace, getEffectiveMetadata(), getEffectiveProtocol(), prevProtocol); if (icebergWriterCompatV1.isPresent()) { newMetadata = icebergWriterCompatV1; } Optional icebergWriterCompatV3 = IcebergWriterCompatV3MetadataValidatorAndUpdater .validateAndUpdateIcebergWriterCompatV3Metadata( isCreateOrReplace, getEffectiveMetadata(), getEffectiveProtocol(), prevProtocol); if (icebergWriterCompatV3.isPresent()) { newMetadata = icebergWriterCompatV3; } // TODO: refactor this method to use a single validator and updater. Optional icebergCompatV2Metadata = IcebergCompatV2MetadataValidatorAndUpdater.validateAndUpdateIcebergCompatV2Metadata( isCreateOrReplace, getEffectiveMetadata(), getEffectiveProtocol(), prevProtocol); if (icebergCompatV2Metadata.isPresent()) { newMetadata = icebergCompatV2Metadata; } Optional icebergCompatV3Metadata = IcebergCompatV3MetadataValidatorAndUpdater.validateAndUpdateIcebergCompatV3Metadata( isCreateOrReplace, getEffectiveMetadata(), getEffectiveProtocol(), prevProtocol); if (icebergCompatV3Metadata.isPresent()) { newMetadata = icebergCompatV3Metadata; } } /** * STEP 3: Update the METADATA with column mapping info if applicable, and resolve the provided * clustering columns using the updated metadata. */ private void updateColumnMappingMetadataAndResolveClusteringColumns( Optional> userProvidedClusteringColumns) { // We update the column mapping info here after all configuration changes are finished Optional columnMappingMetadata = ColumnMapping.updateColumnMappingMetadataIfNeeded( getEffectiveMetadata(), isCreateOrReplace); if (columnMappingMetadata.isPresent()) { newMetadata = columnMappingMetadata; } // We also resolve the user provided clustering columns here using the updated schema StructType updatedSchema = getEffectiveMetadata().getSchema(); this.physicalNewClusteringColumns = userProvidedClusteringColumns.map( cols -> SchemaUtils.casePreservingEligibleClusterColumns(updatedSchema, cols)); } /** STEP 4: Update the METADATA with materialized row tracking column name if applicable */ private void updateRowTrackingMetadata() { if (isCreateOrReplace) { // For new tables, assign materialized column names if row tracking is enabled Optional rowTrackingMetadata = MaterializedRowTrackingColumn.assignMaterializedColumnNamesIfNeeded( getEffectiveMetadata()); if (rowTrackingMetadata.isPresent()) { newMetadata = rowTrackingMetadata; } } validateForUpdateTableUsingOldMetadata( oldMetadata -> { // For existing tables, we block enabling/disabling row tracking because: // 1. Enabling requires backfilling row IDs/commit versions, which is not supported in // Kernel // 2. Disabling is irreversible in Kernel (re-enabling not supported) newMetadata.ifPresent( metadata -> RowTracking.throwIfRowTrackingToggled(oldMetadata, metadata)); // For existing tables, validate that row tracking configs are present when row tracking // is enabled MaterializedRowTrackingColumn.validateRowTrackingConfigsNotMissing( getEffectiveMetadata(), tablePath); }); } /** * STEP 5: Validate the metadata change and update the type widening metadata if needed. Note: we * update the type widening info at the same time to avoid traversing the schema more than * required. * *

Validate that the change from oldMetadata to newMetadata is a valid change. For example, * this checks the following * *

    *
  • Column mapping mode can only go from none->name for existing table *
  • icebergWriterCompatV1 cannot be enabled on existing tables (only supported upon table * creation) *
  • Validates the universal format configs are valid. *
  • If there is schema evolution validates *
      *
    • column mapping is enabled *
    • column mapping mode is not changed in the same txn as schema change *
    • the new schema is a valid schema *
    • the schema change is a valid schema change *
    • the schema change is a valid schema change given the tables partition and * clustering columns *
    *
  • Materialized row tracking column names do not conflict with schema *
*/ private void validateMetadataChangeAndApplyTypeWidening() { validateForUpdateTableUsingOldMetadata( oldMetadata -> { ColumnMapping.verifyColumnMappingChange( oldMetadata.getConfiguration(), getEffectiveMetadata().getConfiguration()); IcebergWriterCompatV1MetadataValidatorAndUpdater.validateIcebergWriterCompatV1Change( oldMetadata.getConfiguration(), getEffectiveMetadata().getConfiguration()); IcebergCompatV3MetadataValidatorAndUpdater.validateIcebergCompatV3Change( oldMetadata.getConfiguration(), getEffectiveMetadata().getConfiguration()); }); IcebergUniversalFormatMetadataValidatorAndUpdater.validate(getEffectiveMetadata()); validateForUpdateTableUsingOldMetadata( oldMetadata -> { if (isSchemaEvolution) { ColumnMappingMode updatedMappingMode = ColumnMapping.getColumnMappingMode(getEffectiveMetadata().getConfiguration()); ColumnMappingMode currentMappingMode = ColumnMapping.getColumnMappingMode(oldMetadata.getConfiguration()); if (currentMappingMode != updatedMappingMode) { throw new KernelException("Cannot update mapping mode and perform schema evolution"); } // If the column mapping restriction is removed, clustering columns // will need special handling during schema evolution since they won't have physical // names // ToDo: Support adding clustering columns if (!isColumnMappingModeEnabled(updatedMappingMode)) { throw new KernelException( "Cannot update schema for table when column mapping is disabled"); } // Clustering columns will be guaranteed to have physical names at this point // Only the leaf part of the overall column needs to be taken since // validation is performed on the leaf struct fields // E.g. getClusteringColumns returns ., // Only physical_name_inner is required for validation Optional> effectiveClusteringCols = physicalNewClusteringColumns.isPresent() ? physicalNewClusteringColumns : latestSnapshotOpt.get().getPhysicalClusteringColumns(); Set clusteringColumnPhysicalNames = effectiveClusteringCols.orElse(Collections.emptyList()).stream() .map(col -> col.getNames()[col.getNames().length - 1]) .collect(toSet()); Optional schemaWithTypeWidening = SchemaUtils.validateUpdatedSchemaAndGetUpdatedSchema( oldMetadata, getEffectiveMetadata(), getEffectiveProtocol(), clusteringColumnPhysicalNames, false /* allowNewRequiredFields */); schemaWithTypeWidening.ifPresent( structType -> newMetadata = Optional.of(getEffectiveMetadata().withNewSchema(structType))); } }); // For replace table we need to do special validation in the case of fieldId re-use if (isCreateOrReplace && latestSnapshotOpt.isPresent()) { // For now, we don't support changing column mapping mode during replace, in a future PR we // will loosen this restriction ColumnMappingMode oldMode = ColumnMapping.getColumnMappingMode( latestSnapshotOpt.get().getMetadata().getConfiguration()); ColumnMappingMode newMode = ColumnMapping.getColumnMappingMode(getEffectiveMetadata().getConfiguration()); if (oldMode != newMode) { throw new UnsupportedOperationException( String.format( "Changing column mapping mode from %s to %s is not currently supported in Kernel " + "during REPLACE TABLE operations", oldMode, newMode)); } // We only need to check fieldId re-use when cmMode != none if (newMode != ColumnMappingMode.NONE) { Optional schemaWithTypeWidening = SchemaUtils.validateUpdatedSchemaAndGetUpdatedSchema( latestSnapshotOpt.get().getMetadata(), getEffectiveMetadata(), getEffectiveProtocol(), // We already validate clustering columns elsewhere for isCreateOrReplace no // need to // duplicate this check here emptySet() /* clusteringCols */, // We allow new non-null fields in REPLACE since we know all existing data is // removed true /* allowNewRequiredFields */); schemaWithTypeWidening.ifPresent( structType -> newMetadata = Optional.of(getEffectiveMetadata().withNewSchema(structType))); } } MaterializedRowTrackingColumn.throwIfColumnNamesConflictWithSchema(getEffectiveMetadata()); } /** * STEP 6: Validate that required catalog table properties are set. Below is a complete summary of * our required-catalog-property setting and validation: * *

First, during CREATE and REPLACE, we inject and set the required catalog table properties. * Note that we do this *before* setting any user properties such that if a user overrides a * required catalog property, we will detect that here. * *

Next, here, we validate that all required catalog table properties are, in fact, set to * their required values. Thus, if a property was explicitly removed during UPDATE, changed to an * invalid value during UPDATE, or set to an invalid value during CREATE or REPLACE, we will * detect and fail. */ private void validateRequiredCatalogTablePropertiesSet(Optional committerOpt) { if (!committerOpt.isPresent()) { return; } final Committer committer = committerOpt.get(); final Map requiredCatalogTableProperties = getRequiredCatalogTablePropertiesIfApplicable(committer); if (requiredCatalogTableProperties.isEmpty()) { return; } final Map effectiveTableProperties = getEffectiveMetadata().getConfiguration(); final Map missingOrViolatingProperties = new HashMap<>(); for (Map.Entry requiredEntry : requiredCatalogTableProperties.entrySet()) { final String requiredKey = requiredEntry.getKey(); final String requiredValue = requiredEntry.getValue(); final String currentValue = effectiveTableProperties.get(requiredKey); if (!Objects.equals(requiredValue, currentValue)) { missingOrViolatingProperties.put( requiredKey, Optional.ofNullable(currentValue).orElse("")); } } if (!missingOrViolatingProperties.isEmpty()) { throw DeltaErrors.metadataMissingRequiredCatalogTableProperty( committer.getClass().getName(), missingOrViolatingProperties, requiredCatalogTableProperties); } } private static Metadata getInitialMetadata( StructType schema, Map tableProperties, List partitionColumns) { return new Metadata( java.util.UUID.randomUUID().toString(), /* id */ Optional.empty(), /* name */ Optional.empty(), /* description */ new Format(), /* format */ schema.toJson(), /* schemaString */ schema, /* schema */ buildArrayValue( casePreservingPartitionColNames(schema, partitionColumns), StringType.STRING), /* partitionColumns */ Optional.of(System.currentTimeMillis()), /* createdTime */ stringStringMapValue(tableProperties) /* configuration */); } private static Protocol getInitialProtocol() { return new Protocol(DEFAULT_READ_VERSION, DEFAULT_WRITE_VERSION); } private static void validateSchemaAndPartColsCreateOrReplace( Map tableProperties, StructType schema, List partitionColumns) { // New table verify the given schema and partition columns ColumnMappingMode mappingMode = ColumnMapping.getColumnMappingMode(tableProperties); SchemaUtils.validateSchema( schema, isColumnMappingModeEnabled(mappingMode), TableFeatures.isPropertiesManuallySupportingTableFeature( tableProperties, ALLOW_COLUMN_DEFAULTS_W_FEATURE), TableConfig.ICEBERG_COMPAT_V3_ENABLED.fromMetadata(tableProperties)); SchemaUtils.validatePartitionColumns(schema, partitionColumns); } private static void validateNotEnablingCatalogManagedOnReplace( Map userInputTableProperties) { if (TableFeatures.isPropertiesManuallySupportingTableFeature( userInputTableProperties, TableFeatures.CATALOG_MANAGED_RW_FEATURE)) { throw new UnsupportedOperationException( "Cannot enable the catalogManaged feature during a REPLACE command."); } } private static Map getRequiredCatalogTablePropertiesIfApplicable( Committer committer) { if (committer instanceof CatalogCommitter) { return ((CatalogCommitter) committer).getRequiredTableProperties(); } return Collections.emptyMap(); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/UpdateTableTransactionBuilderImpl.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toSet; import io.delta.kernel.Operation; import io.delta.kernel.Transaction; import io.delta.kernel.engine.Engine; import io.delta.kernel.expressions.Column; import io.delta.kernel.internal.actions.SetTransaction; import io.delta.kernel.internal.annotation.VisibleForTesting; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.internal.util.Clock; import io.delta.kernel.transaction.UpdateTableTransactionBuilder; import io.delta.kernel.types.StructType; import java.util.*; public class UpdateTableTransactionBuilderImpl implements UpdateTableTransactionBuilder { private Clock clock = System::currentTimeMillis; /** Timestamp when this builder was created, used for populating any {@link SetTransaction} */ private final long txnBuilderStartTime = System.currentTimeMillis(); /* Class fields provided in the constructor */ private final SnapshotImpl snapshot; private final String engineInfo; private final Operation operation; /* Optional metadata configured in this builder by the connector */ private Optional setTxnOpt = Optional.empty(); private Optional> tablePropertiesAddedOpt = Optional.empty(); private Optional> tablePropertiesRemovedOpt = Optional.empty(); private Optional updatedSchemaOpt = Optional.empty(); private Optional> inputLogicalClusteringColumnsOpt = Optional.empty(); private Optional userProvidedMaxRetries = Optional.empty(); /** Number of commits between producing a log compaction file. */ private int logCompactionInterval = 0; public UpdateTableTransactionBuilderImpl( SnapshotImpl snapshot, String engineInfo, Operation operation) { validateIsUpdateOperation(operation); this.snapshot = snapshot; this.engineInfo = engineInfo; this.operation = operation; TableFeatures.validateKernelCanWriteToTable( snapshot.getProtocol(), snapshot.getMetadata(), snapshot.getPath()); } @Override public UpdateTableTransactionBuilder withUpdatedSchema(StructType schema) { this.updatedSchemaOpt = Optional.of(schema); return this; } @Override public UpdateTableTransactionBuilder withTablePropertiesAdded(Map properties) { this.tablePropertiesAddedOpt = Optional.of( Collections.unmodifiableMap( TableConfig.validateAndNormalizeDeltaProperties(properties))); validateTablePropertiesAddedRemovedNoOverlap(); return this; } @Override public UpdateTableTransactionBuilder withTablePropertiesRemoved(Set propertyKeys) { checkArgument( propertyKeys.stream().noneMatch(key -> key.toLowerCase(Locale.ROOT).startsWith("delta.")), "Unsetting 'delta.' table properties is currently unsupported"); this.tablePropertiesRemovedOpt = Optional.of(propertyKeys); validateTablePropertiesAddedRemovedNoOverlap(); return this; } @Override public UpdateTableTransactionBuilder withClusteringColumns(List clusteringColumns) { if (snapshot.getPartitionColumnNames().size() > 0) { throw DeltaErrors.enablingClusteringOnPartitionedTableNotAllowed( snapshot.getPath(), snapshot.getMetadata().getPartitionColNames(), clusteringColumns); } this.inputLogicalClusteringColumnsOpt = Optional.of(clusteringColumns); return this; } @Override public UpdateTableTransactionBuilder withTransactionId( String applicationId, long transactionVersion) { SetTransaction txnId = new SetTransaction( requireNonNull(applicationId, "applicationId is null"), transactionVersion, Optional.of(txnBuilderStartTime)); this.setTxnOpt = Optional.of(txnId); return this; } @Override public UpdateTableTransactionBuilder withMaxRetries(int maxRetries) { checkArgument(maxRetries >= 0, "maxRetries must be >= 0"); this.userProvidedMaxRetries = Optional.of(maxRetries); return this; } @Override public UpdateTableTransactionBuilder withLogCompactionInterval(int logCompactionInterval) { checkArgument(logCompactionInterval >= 0, "logCompactionInterval must be >= 0"); this.logCompactionInterval = logCompactionInterval; return this; } @VisibleForTesting public UpdateTableTransactionBuilder withClock(Clock clock) { this.clock = requireNonNull(clock, "clock cannot be null"); return this; } @Override public Transaction build(Engine engine) { setTxnOpt.ifPresent( txnId -> { Optional lastTxnVersion = snapshot.getLatestTransactionVersion(engine, txnId.getAppId()); if (lastTxnVersion.isPresent() && lastTxnVersion.get() >= txnId.getVersion()) { throw DeltaErrors.concurrentTransaction( txnId.getAppId(), txnId.getVersion(), lastTxnVersion.get()); } }); TransactionMetadataFactory.Output txnMetadata = TransactionMetadataFactory.buildUpdateTableMetadata( snapshot.getPath(), snapshot, tablePropertiesAddedOpt, tablePropertiesRemovedOpt, updatedSchemaOpt, inputLogicalClusteringColumnsOpt); return new TransactionImpl( false /* isCreateOrReplace */, snapshot.getDataPath(), Optional.of(snapshot), engineInfo, operation, txnMetadata.newProtocol, txnMetadata.newMetadata, snapshot.getCommitter(), setTxnOpt, txnMetadata.physicalNewClusteringColumns, userProvidedMaxRetries, logCompactionInterval, clock); } private void validateTablePropertiesAddedRemovedNoOverlap() { if (tablePropertiesAddedOpt.isPresent() && tablePropertiesRemovedOpt.isPresent()) { Set invalidPropertyKeys = tablePropertiesRemovedOpt.get().stream() .filter(tablePropertiesAddedOpt.get()::containsKey) .collect(toSet()); if (!invalidPropertyKeys.isEmpty()) { throw DeltaErrors.overlappingTablePropertiesSetAndUnset(invalidPropertyKeys); } } } private void validateIsUpdateOperation(Operation operation) { if (operation == Operation.CREATE_TABLE || operation == Operation.REPLACE_TABLE) { throw new IllegalArgumentException( String.format( "Operation %s is not compatible with Snapshot::buildUpdateTableTransaction", operation)); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/AddCDCFile.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.actions; import io.delta.kernel.types.*; /** Metadata about {@code cdc} action in the Delta Log. */ public class AddCDCFile { /** Full schema of the {@code cdc} action in the Delta Log. */ public static final StructType FULL_SCHEMA = new StructType() .add("path", StringType.STRING, false /* nullable */) .add( "partitionValues", new MapType(StringType.STRING, StringType.STRING, true), false /* nullable*/) .add("size", LongType.LONG, false /* nullable*/) .add( "tags", new MapType(StringType.STRING, StringType.STRING, true), true /* nullable */); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/AddFile.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.actions; import static io.delta.kernel.internal.util.InternalUtils.relativizePath; import static io.delta.kernel.internal.util.PartitionUtils.serializePartitionMap; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static io.delta.kernel.internal.util.VectorUtils.toJavaMap; import io.delta.kernel.data.MapValue; import io.delta.kernel.data.Row; import io.delta.kernel.expressions.Literal; import io.delta.kernel.internal.data.GenericRow; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.util.VectorUtils; import io.delta.kernel.statistics.DataFileStatistics; import io.delta.kernel.types.*; import io.delta.kernel.utils.DataFileStatus; import java.net.URI; import java.util.*; /** Delta log action representing an `AddFile` */ public class AddFile extends RowBackedAction { /* We conditionally read this field based on the query filter */ private static final StructField JSON_STATS_FIELD = new StructField("stats", StringType.STRING, true /* nullable */); /** * Schema of the {@code add} action in the Delta Log without stats. Used for constructing table * snapshot to read data from the table. */ public static final StructType SCHEMA_WITHOUT_STATS = new StructType() .add("path", StringType.STRING, false /* nullable */) .add( "partitionValues", new MapType(StringType.STRING, StringType.STRING, true), false /* nullable*/) .add("size", LongType.LONG, false /* nullable*/) .add("modificationTime", LongType.LONG, false /* nullable*/) .add("dataChange", BooleanType.BOOLEAN, false /* nullable*/) .add("deletionVector", DeletionVectorDescriptor.READ_SCHEMA, true /* nullable */) .add( "tags", new MapType(StringType.STRING, StringType.STRING, true /* valueContainsNull */), true /* nullable */) .add("baseRowId", LongType.LONG, true /* nullable */) .add("defaultRowCommitVersion", LongType.LONG, true /* nullable */); public static final StructType SCHEMA_WITH_STATS = SCHEMA_WITHOUT_STATS.add(JSON_STATS_FIELD); /** Full schema of the {@code add} action in the Delta Log. */ public static final StructType FULL_SCHEMA = SCHEMA_WITH_STATS; // There are more fields which are added when row-id tracking and clustering is enabled. // When Kernel starts supporting row-ids and clustering, we should add those fields here. /** * Utility to generate {@link AddFile} action instance from the given {@link DataFileStatus} and * partition values. */ public static AddFile convertDataFileStatus( StructType physicalSchema, URI tableRoot, DataFileStatus dataFileStatus, Map partitionValues, boolean dataChange, Map tags, Optional baseRowId, Optional defaultRowCommitVersion, Optional deletionVectorDescriptor) { Optional tagMapValue = !tags.isEmpty() ? Optional.of(VectorUtils.stringStringMapValue(tags)) : Optional.empty(); Row row = createAddFileRow( physicalSchema, relativizePath(new Path(dataFileStatus.getPath()), tableRoot).toUri().toString(), serializePartitionMap(partitionValues), dataFileStatus.getSize(), dataFileStatus.getModificationTime(), dataChange, deletionVectorDescriptor, tagMapValue, // tags baseRowId, defaultRowCommitVersion, dataFileStatus.getStatistics()); return new AddFile(row); } /** Utility to generate an 'AddFile' row from the given fields. */ public static Row createAddFileRow( StructType physicalSchema, String path, MapValue partitionValues, long size, long modificationTime, boolean dataChange, Optional deletionVector, Optional tags, Optional baseRowId, Optional defaultRowCommitVersion, Optional stats) { checkArgument(path != null, "path is not nullable"); checkArgument(partitionValues != null, "partitionValues is not nullable"); Map fieldMap = new HashMap<>(); fieldMap.put(FULL_SCHEMA.indexOf("path"), path); fieldMap.put(FULL_SCHEMA.indexOf("partitionValues"), partitionValues); fieldMap.put(FULL_SCHEMA.indexOf("size"), size); fieldMap.put(FULL_SCHEMA.indexOf("modificationTime"), modificationTime); fieldMap.put(FULL_SCHEMA.indexOf("dataChange"), dataChange); tags.ifPresent(tag -> fieldMap.put(FULL_SCHEMA.indexOf("tags"), tag)); baseRowId.ifPresent(id -> fieldMap.put(FULL_SCHEMA.indexOf("baseRowId"), id)); defaultRowCommitVersion.ifPresent( version -> fieldMap.put(FULL_SCHEMA.indexOf("defaultRowCommitVersion"), version)); stats.ifPresent( stat -> fieldMap.put(FULL_SCHEMA.indexOf("stats"), stat.serializeAsJson(physicalSchema))); deletionVector.ifPresent( dv -> { Row dvRow = dv.toRow(); fieldMap.put(FULL_SCHEMA.indexOf("deletionVector"), dvRow); }); return new GenericRow(FULL_SCHEMA, fieldMap); } //////////////////////////////////// // Constructor and Member Methods // //////////////////////////////////// /** Constructs an {@link AddFile} action from the given 'AddFile' {@link Row}. */ public AddFile(Row row) { super(row); } public String getPath() { return row.getString(getFieldIndex("path")); } public MapValue getPartitionValues() { return row.getMap(getFieldIndex("partitionValues")); } public long getSize() { return row.getLong(getFieldIndex("size")); } public long getModificationTime() { return row.getLong(getFieldIndex("modificationTime")); } public boolean getDataChange() { return row.getBoolean(getFieldIndex("dataChange")); } public Optional getDeletionVector() { int index = getFieldIndex("deletionVector"); return Optional.ofNullable( row.isNullAt(index) ? null : DeletionVectorDescriptor.fromRow(row.getStruct(index))); } public Optional getTags() { int index = getFieldIndex("tags"); return Optional.ofNullable(row.isNullAt(index) ? null : row.getMap(index)); } public Optional getStatsJson() { Optional statsIndexOpt = getFieldIndexOpt("stats"); return statsIndexOpt.map( index -> { if (row.isNullAt(index)) { return null; } return row.getString(index); }); } public Optional getBaseRowId() { int index = getFieldIndex("baseRowId"); return Optional.ofNullable(row.isNullAt(index) ? null : row.getLong(index)); } public Optional getDefaultRowCommitVersion() { int index = getFieldIndex("defaultRowCommitVersion"); return Optional.ofNullable(row.isNullAt(index) ? null : row.getLong(index)); } public Optional getNumRecords() { return getFieldIndexOpt("stats") .flatMap( index -> row.isNullAt(index) ? Optional.empty() : DataFileStatistics.getNumRecords(row.getString(index))); } /** * Returns the file statistics parsed from the stats JSON string using the provided schema. This * method deserializes the statistics JSON with full type information, ensuring that min/max * values and null counts are correctly typed according to the physical schema. * * @param physicalSchema the physical schema of the table, used to correctly parse and type the * statistics values (min/max values and null counts) * @return an {@link Optional} containing the deserialized {@link DataFileStatistics} if the stats * field is present and non-null, or {@link Optional#empty()} otherwise * @throws io.delta.kernel.exceptions.KernelException if the stats JSON is malformed or if values * don't match the expected types from the schema * @see DataFileStatistics#deserializeFromJson(String, StructType) for details on the * deserialization process */ public Optional getStats(StructType physicalSchema) { return getFieldIndexOpt("stats") .flatMap( index -> row.isNullAt(index) ? Optional.empty() : DataFileStatistics.deserializeFromJson(row.getString(index), physicalSchema)); } /** Returns a new {@link AddFile} with the provided baseRowId. */ public AddFile withNewBaseRowId(long baseRowId) { return new AddFile(toRowWithOverriddenValue("baseRowId", baseRowId)); } /** Returns a new {@link AddFile} with the provided defaultRowCommitVersion. */ public AddFile withNewDefaultRowCommitVersion(long defaultRowCommitVersion) { return new AddFile( toRowWithOverriddenValue("defaultRowCommitVersion", defaultRowCommitVersion)); } /** * Utility to generate a 'RemoveFile' action from this AddFile action. * * @param dataChange this will override the dataChange field in current AddFile * @param deletionTimestamp the deletion timestamp of the operation, this will override the * modificationTime field in current AddFile */ public Row toRemoveFileRow(boolean dataChange, Optional deletionTimestamp) { Map fieldMap = new HashMap<>(); fieldMap.put(RemoveFile.FULL_SCHEMA.indexOf("path"), getPath()); fieldMap.put( RemoveFile.FULL_SCHEMA.indexOf("deletionTimestamp"), deletionTimestamp.orElse(System.currentTimeMillis())); fieldMap.put(RemoveFile.FULL_SCHEMA.indexOf("dataChange"), dataChange); fieldMap.put(RemoveFile.FULL_SCHEMA.indexOf("extendedFileMetadata"), true); fieldMap.put(RemoveFile.FULL_SCHEMA.indexOf("partitionValues"), getPartitionValues()); fieldMap.put(RemoveFile.FULL_SCHEMA.indexOf("size"), getSize()); getStatsJson() .ifPresent(statsJson -> fieldMap.put(RemoveFile.FULL_SCHEMA.indexOf("stats"), statsJson)); getTags().ifPresent(tags -> fieldMap.put(RemoveFile.FULL_SCHEMA.indexOf("tags"), tags)); if (!row.isNullAt(getFieldIndex("deletionVector"))) { fieldMap.put( RemoveFile.FULL_SCHEMA.indexOf("deletionVector"), row.getStruct(getFieldIndex("deletionVector"))); } getBaseRowId() .ifPresent( baseRowId -> fieldMap.put(RemoveFile.FULL_SCHEMA.indexOf("baseRowId"), baseRowId)); getDefaultRowCommitVersion() .ifPresent( defaultRowCommitVersion -> fieldMap.put( RemoveFile.FULL_SCHEMA.indexOf("defaultRowCommitVersion"), defaultRowCommitVersion)); return new GenericRow(RemoveFile.FULL_SCHEMA, fieldMap); } @Override public String toString() { // No specific ordering is guaranteed for partitionValues and tags in the returned string StringBuilder sb = new StringBuilder(); sb.append("AddFile{"); sb.append("path='").append(getPath()).append('\''); sb.append(", partitionValues=").append(toJavaMap(getPartitionValues())); sb.append(", size=").append(getSize()); sb.append(", modificationTime=").append(getModificationTime()); sb.append(", dataChange=").append(getDataChange()); sb.append(", deletionVector=").append(getDeletionVector()); sb.append(", tags=").append(getTags().map(VectorUtils::toJavaMap)); sb.append(", baseRowId=").append(getBaseRowId()); sb.append(", defaultRowCommitVersion=").append(getDefaultRowCommitVersion()); sb.append(", stats=").append(getStats(null).map(d -> d.serializeAsJson(null)).orElse("")); sb.append('}'); return sb.toString(); } @Override public boolean equals(Object obj) { if (this == obj) return true; if (!(obj instanceof AddFile)) return false; AddFile other = (AddFile) obj; // MapValue and DataFileStatistics don't implement equals(), so we need to convert // partitionValues and tags to Java Maps, and stats to strings to compare them return getSize() == other.getSize() && getModificationTime() == other.getModificationTime() && getDataChange() == other.getDataChange() && Objects.equals(getPath(), other.getPath()) && Objects.equals(toJavaMap(getPartitionValues()), toJavaMap(other.getPartitionValues())) && Objects.equals(getDeletionVector(), other.getDeletionVector()) && Objects.equals( getTags().map(VectorUtils::toJavaMap), other.getTags().map(VectorUtils::toJavaMap)) && Objects.equals(getBaseRowId(), other.getBaseRowId()) && Objects.equals(getDefaultRowCommitVersion(), other.getDefaultRowCommitVersion()) && Objects.equals(getStats(null), other.getStats(null)); } @Override public int hashCode() { // MapValue and DataFileStatistics don't implement hashCode(), so we need to convert // partitionValues and tags to Java Maps, and stats to strings to compute the hash code return Objects.hash( getPath(), toJavaMap(getPartitionValues()), getSize(), getModificationTime(), getDataChange(), getDeletionVector(), getTags().map(VectorUtils::toJavaMap), getBaseRowId(), getDefaultRowCommitVersion(), getStats(null)); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/CommitInfo.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.actions; import static io.delta.kernel.internal.DeltaErrors.wrapEngineExceptionThrowsIO; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static io.delta.kernel.internal.util.Utils.singletonCloseableIterator; import static io.delta.kernel.internal.util.VectorUtils.stringStringMapValue; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toMap; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.DeltaErrors; import io.delta.kernel.internal.data.GenericRow; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.internal.util.VectorUtils; import io.delta.kernel.types.*; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import java.io.IOException; import java.io.UncheckedIOException; import java.util.*; import java.util.stream.IntStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Delta log action representing a commit information action. According to the Delta protocol there * isn't any specific schema for this action, but we use the following schema: * *

    *
  • inCommitTimestamp: Long - A monotonically increasing timestamp that represents the time * since epoch in milliseconds when the commit write was started *
  • timestamp: Long - Milliseconds since epoch UTC of when this commit happened *
  • engineInfo: String - Engine that made this commit *
  • operation: String - Operation (e.g. insert, delete, merge etc.) *
  • operationParameters: Map(String, String) - each operation depending upon the type may add * zero or more parameters about the operation. E.g. when creating a table `partitionBy` key * with list of partition columns is added. *
  • isBlindAppend: Boolean - Is this commit a blind append? *
  • txnId: String - a unique transaction id of this commit *
* * The Delta-Spark connector adds lot more fields to this action. We can add them as needed. */ public class CommitInfo { ////////////////////////////////// // Static variables and methods // ////////////////////////////////// public static final StructType FULL_SCHEMA = new StructType() .add("inCommitTimestamp", LongType.LONG, true /* nullable */) .add("timestamp", LongType.LONG) .add("engineInfo", StringType.STRING) .add("operation", StringType.STRING) .add( "operationParameters", new MapType(StringType.STRING, StringType.STRING, true /* nullable */)) .add("isBlindAppend", BooleanType.BOOLEAN, true /* nullable */) .add("txnId", StringType.STRING) .add( "operationMetrics", new MapType(StringType.STRING, StringType.STRING, true /* nullable */)); private static final StructType READ_SCHEMA = new StructType().add("commitInfo", CommitInfo.FULL_SCHEMA); private static final Map COL_NAME_TO_ORDINAL = IntStream.range(0, FULL_SCHEMA.length()) .boxed() .collect(toMap(i -> FULL_SCHEMA.at(i).getName(), i -> i)); private static final Logger logger = LoggerFactory.getLogger(CommitInfo.class); public static CommitInfo fromColumnVector(ColumnVector vector, int rowId) { if (vector.isNullAt(rowId)) { return null; } ColumnVector[] children = new ColumnVector[8]; for (int i = 0; i < children.length; i++) { children[i] = vector.getChild(i); } checkArgument(!children[1].isNullAt(rowId), "CommitInfo is missing required timestamp field"); return new CommitInfo( Optional.ofNullable(children[0].isNullAt(rowId) ? null : children[0].getLong(rowId)), children[1].getLong(rowId), Optional.ofNullable(children[2].isNullAt(rowId) ? null : children[2].getString(rowId)), Optional.ofNullable(children[3].isNullAt(rowId) ? null : children[3].getString(rowId)), children[4].isNullAt(rowId) ? Collections.emptyMap() : VectorUtils.toJavaMap(children[4].getMap(rowId)), Optional.ofNullable(children[5].isNullAt(rowId) ? null : children[5].getBoolean(rowId)), Optional.ofNullable(children[6].isNullAt(rowId) ? null : children[6].getString(rowId)), children[7].isNullAt(rowId) ? Collections.emptyMap() : VectorUtils.toJavaMap(children[7].getMap(rowId))); } /** * Returns the `inCommitTimestamp` of delta file at the requested version. Throws an exception if * the delta file does not exist or does not have a commitInfo action or if the commitInfo action * contains an empty `inCommitTimestamp`. * *

WARNING: UNSAFE METHOD because this assumes that 00N.json is published. */ // TODO: [delta-io/delta#5147] Can't just use the logPath & version on catalogManaged tables. public static long unsafeGetRequiredIctFromPublishedDeltaFile( Engine engine, Path logPath, long version) { return extractRequiredIctFromCommitInfoOpt( unsafeTryReadCommitInfoFromPublishedDeltaFile(engine, logPath, version), version, logPath); } /** * Returns the `inCommitTimestamp` of the provided delta file. Throws an exception if the delta * file does not exist or does not have a commitInfo action or if the commitInfo action contains * an empty `inCommitTimestamp`. The delta file can be either a published or staged commit file. */ public static long getRequiredIctFromDeltaFile( Engine engine, Path tablePath, FileStatus deltaFileStatus, long version) { checkArgument( FileNames.isCommitFile(deltaFileStatus.getPath()), "Must provide a valid commit file"); return extractRequiredIctFromCommitInfoOpt( tryReadCommitInfoFromDeltaFile(engine, deltaFileStatus), version, tablePath); } /** * Returns the `inCommitTimestamp` of the given `commitInfoOpt` if it is defined. Throws an * exception if `commitInfoOpt` is empty or contains an empty `inCommitTimestamp`. */ // TODO: [delta-io/delta#5147] Can't just use the logPath & version on catalogManaged tables. public static long extractRequiredIctFromCommitInfoOpt( Optional commitInfoOpt, long version, Path dataPath) { CommitInfo commitInfo = commitInfoOpt.orElseThrow( () -> DeltaErrors.tableWithIctMissingCommitInfo(dataPath.toString(), version)); return commitInfo.inCommitTimestamp.orElseThrow( () -> DeltaErrors.tableWithIctMissingIct(dataPath.toString(), version)); } /** * Get the CommitInfo action (if available) from the delta file at the given logPath and version. * *

WARNING: UNSAFE METHOD because this assumes that 00N.json is published. */ // TODO: [delta-io/delta#5147] Can't just use the logPath & version on catalogManaged tables. public static Optional unsafeTryReadCommitInfoFromPublishedDeltaFile( Engine engine, Path logPath, long version) { final FileStatus file = FileStatus.of( FileNames.deltaFile(logPath, version), /* path */ 0, /* size */ 0 /* modification time */); return tryReadCommitInfoFromDeltaFile(engine, file); } /** Read the CommitInfo action (if available) from the given delta file. */ public static Optional tryReadCommitInfoFromDeltaFile( Engine engine, FileStatus deltaFileStatus) { try (CloseableIterator columnarBatchIter = wrapEngineExceptionThrowsIO( () -> engine .getJsonHandler() .readJsonFiles( singletonCloseableIterator(deltaFileStatus), READ_SCHEMA, Optional.empty()), "Reading the CommitInfo with schema=%s from delta file %s", READ_SCHEMA, deltaFileStatus.getPath())) { while (columnarBatchIter.hasNext()) { final ColumnarBatch columnarBatch = columnarBatchIter.next(); assert (columnarBatch.getSchema().equals(READ_SCHEMA)); final ColumnVector commitInfoVector = columnarBatch.getColumnVector(0); for (int i = 0; i < commitInfoVector.getSize(); i++) { if (!commitInfoVector.isNullAt(i)) { CommitInfo commitInfo = CommitInfo.fromColumnVector(commitInfoVector, i); if (commitInfo != null) { return Optional.of(commitInfo); } } } } } catch (IOException ex) { throw new UncheckedIOException("Could not close iterator", ex); } logger.info("No CommitInfo found in delta file {}", deltaFileStatus.getPath()); return Optional.empty(); } ////////////////////////////////// // Member variables and methods // ////////////////////////////////// private final long timestamp; private final Optional engineInfo; private final Optional operation; private final Map operationParameters; private final Optional isBlindAppend; private final Optional txnId; private Optional inCommitTimestamp; private final Map operationMetrics; public CommitInfo( Optional inCommitTimestamp, long timestamp, Optional engineInfo, Optional operation, Map operationParameters, Optional isBlindAppend, Optional txnId, Map operationMetrics) { this.inCommitTimestamp = requireNonNull(inCommitTimestamp); this.timestamp = timestamp; this.engineInfo = requireNonNull(engineInfo); this.operation = requireNonNull(operation); this.operationParameters = Collections.unmodifiableMap(requireNonNull(operationParameters)); this.isBlindAppend = requireNonNull(isBlindAppend); this.txnId = requireNonNull(txnId); this.operationMetrics = Collections.unmodifiableMap(requireNonNull(operationMetrics)); } public long getTimestamp() { return timestamp; } public Optional getEngineInfo() { return engineInfo; } public Optional getOperation() { return operation; } public Map getOperationParameters() { return operationParameters; } public Optional getIsBlindAppend() { return isBlindAppend; } public Optional getTxnId() { return txnId; } public Optional getInCommitTimestamp() { return inCommitTimestamp; } public Map getOperationMetrics() { return operationMetrics; } public void setInCommitTimestamp(Optional inCommitTimestamp) { this.inCommitTimestamp = inCommitTimestamp; } /** * Encode as a {@link Row} object with the schema {@link CommitInfo#FULL_SCHEMA}. * * @return {@link Row} object with the schema {@link CommitInfo#FULL_SCHEMA} */ public Row toRow() { Map commitInfo = new HashMap<>(); commitInfo.put(COL_NAME_TO_ORDINAL.get("inCommitTimestamp"), inCommitTimestamp.orElse(null)); commitInfo.put(COL_NAME_TO_ORDINAL.get("timestamp"), timestamp); commitInfo.put(COL_NAME_TO_ORDINAL.get("engineInfo"), engineInfo.orElse(null)); commitInfo.put(COL_NAME_TO_ORDINAL.get("operation"), operation.orElse(null)); commitInfo.put( COL_NAME_TO_ORDINAL.get("operationParameters"), stringStringMapValue(operationParameters)); commitInfo.put(COL_NAME_TO_ORDINAL.get("isBlindAppend"), isBlindAppend.orElse(null)); commitInfo.put(COL_NAME_TO_ORDINAL.get("txnId"), txnId.orElse(null)); commitInfo.put( COL_NAME_TO_ORDINAL.get("operationMetrics"), stringStringMapValue(operationMetrics)); return new GenericRow(CommitInfo.FULL_SCHEMA, commitInfo); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/DeletionVectorDescriptor.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.actions; import static io.delta.kernel.internal.util.InternalUtils.requireNonNull; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.stream.Collectors.toMap; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.Row; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.internal.data.GenericRow; import io.delta.kernel.internal.deletionvectors.Base85Codec; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.types.IntegerType; import io.delta.kernel.types.LongType; import io.delta.kernel.types.StringType; import io.delta.kernel.types.StructType; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.*; import java.util.stream.IntStream; /** Information about a deletion vector attached to a file action. */ public class DeletionVectorDescriptor { //////////////////////////////////////////////////////////////////////////////// // Static Fields / Methods //////////////////////////////////////////////////////////////////////////////// public static DeletionVectorDescriptor fromRow(Row row) { if (row == null) { return null; } final String storageType = requireNonNull(row, 0, "storageType").getString(0); final String pathOrInlineDv = requireNonNull(row, 1, "pathOrInlineDv").getString(1); final Optional offset = Optional.ofNullable(row.isNullAt(2) ? null : row.getInt(2)); final int sizeInBytes = requireNonNull(row, 3, "sizeInBytes").getInt(3); final long cardinality = requireNonNull(row, 4, "cardinality").getLong(4); return new DeletionVectorDescriptor( storageType, pathOrInlineDv, offset, sizeInBytes, cardinality); } public static DeletionVectorDescriptor fromColumnVector(ColumnVector vector, int rowId) { if (vector.isNullAt(rowId)) { return null; } final String storageType = requireNonNull(vector.getChild(0), rowId, "storageType").getString(rowId); final String pathOrInlineDv = requireNonNull(vector.getChild(1), rowId, "pathOrInlineDv").getString(rowId); final Optional offset = Optional.ofNullable( vector.getChild(2).isNullAt(rowId) ? null : vector.getChild(2).getInt(rowId)); final int sizeInBytes = requireNonNull(vector.getChild(3), rowId, "sizeInBytes").getInt(rowId); final long cardinality = requireNonNull(vector.getChild(4), rowId, "cardinality").getLong(rowId); return new DeletionVectorDescriptor( storageType, pathOrInlineDv, offset, sizeInBytes, cardinality); } // Markers to separate different kinds of DV storage. public static final String PATH_DV_MARKER = "p"; public static final String INLINE_DV_MARKER = "i"; public static final String UUID_DV_MARKER = "u"; public static final StructType READ_SCHEMA = new StructType() .add("storageType", StringType.STRING, false /* nullable*/) .add("pathOrInlineDv", StringType.STRING, false /* nullable*/) .add("offset", IntegerType.INTEGER, true /* nullable*/) .add("sizeInBytes", IntegerType.INTEGER, false /* nullable*/) .add("cardinality", LongType.LONG, false /* nullable*/); private static final Map COL_NAME_TO_ORDINAL = IntStream.range(0, READ_SCHEMA.length()) .boxed() .collect(toMap(i -> READ_SCHEMA.at(i).getName(), i -> i)); /** String that is used in all file names generated by deletion vector store */ private static final String DELETION_VECTOR_FILE_NAME_CORE = "deletion_vector"; //////////////////////////////////////////////////////////////////////////////// // Instance Fields / Methods //////////////////////////////////////////////////////////////////////////////// /** Indicates how the DV is stored. Should be a single letter (see [[pathOrInlineDv]] below.) */ private final String storageType; /** * Contains the actual data that allows accessing the DV. * *

Three options are currently supported: - `storageType="u"` format: `` The deletion vector is stored in a file with a path relative to * the data directory of this Delta Table, and the file name can be reconstructed from the UUID. * The encoded UUID is always exactly 20 characters, so the random prefix length can be determined * any characters exceeding 20. - `storageType="i"` format: `` The deletion * vector is stored inline in the log. - `storageType="p"` format: `` The DV is * stored in a file with an absolute path given by this url. */ private final String pathOrInlineDv; /** * Start of the data for this DV in number of bytes from the beginning of the file it is stored * in. * *

Always None when storageType = "i". */ private final Optional offset; /** Size of the serialized DV in bytes (raw data size, i.e. before base85 encoding). */ private final int sizeInBytes; /** Number of rows the DV logically removes from the file. */ private final long cardinality; public DeletionVectorDescriptor( String storageType, String pathOrInlineDv, Optional offset, int sizeInBytes, long cardinality) { this.storageType = storageType; this.pathOrInlineDv = pathOrInlineDv; this.offset = offset; this.sizeInBytes = sizeInBytes; this.cardinality = cardinality; } public String getStorageType() { return storageType; } public String getPathOrInlineDv() { return pathOrInlineDv; } public Optional getOffset() { return offset; } public int getSizeInBytes() { return sizeInBytes; } public long getCardinality() { return cardinality; } public String getUniqueId() { String uniqueFileId = storageType + pathOrInlineDv; if (offset.isPresent()) { return uniqueFileId + "@" + offset; } else { return uniqueFileId; } } /** * Serialize this DV descriptor to a base64 encoded string. * *

Format is compatible with Spark's DeletionVectorDescriptor.serializeToBase64(). */ public String serializeToBase64() { try (ByteArrayOutputStream bs = new ByteArrayOutputStream(); DataOutputStream ds = new DataOutputStream(bs)) { ds.writeLong(cardinality); ds.writeInt(sizeInBytes); byte[] storageTypeBytes = storageType.getBytes(); checkArgument(storageTypeBytes.length == 1, "Storage type must be 1 byte: " + storageType); ds.writeByte(storageTypeBytes[0]); // Inline DVs (storageType="i") have no offset if (!storageType.equals(INLINE_DV_MARKER)) { checkArgument(offset.isPresent(), "Non-inline DV must have offset"); ds.writeInt(offset.get()); } ds.writeUTF(pathOrInlineDv); return Base64.getEncoder().encodeToString(bs.toByteArray()); } catch (IOException e) { throw new KernelException("Failed to serialize DeletionVectorDescriptor", e); } } public boolean isInline() { return INLINE_DV_MARKER.equals(storageType); } public boolean isOnDisk() { return !isInline(); } public byte[] inlineData() { checkArgument(isInline(), "Can't get data for an on-disk DV from the log."); // The sizeInBytes is used to remove any padding that might have been added during encoding. return Base85Codec.decodeBytes(pathOrInlineDv, sizeInBytes); } public String getAbsolutePath(String tableLocation) { checkArgument(isOnDisk(), "Can't get a path for an inline deletion vector"); if (storageType.equals(UUID_DV_MARKER)) { // If the file was written with a random prefix, we have to extract that, // before decoding the UUID. int randomPrefixLength = pathOrInlineDv.length() - Base85Codec.ENCODED_UUID_LENGTH; String randomPrefix = pathOrInlineDv.substring(0, randomPrefixLength); String encodedUuid = pathOrInlineDv.substring(randomPrefixLength); UUID uuid = Base85Codec.decodeUUID(encodedUuid); return assembleDeletionVectorPath(tableLocation, uuid, randomPrefix).toString(); } else if (storageType.equals(PATH_DV_MARKER)) { // Since there is no need for legacy support for relative paths for DVs, // relative DVs should *always* use the UUID variant. try { URI parsedUri = new URI(pathOrInlineDv); checkArgument(parsedUri.isAbsolute(), "Relative URIs are not supported for DVs"); return new Path(parsedUri).toString(); } catch (URISyntaxException e) { throw new RuntimeException("Couldn't parse uri:\n" + e); } } else { throw new RuntimeException( "A uri " + pathOrInlineDv + " which cannot be turned into a relative path as found in the transaction log"); } } /** * Return the unique path under `parentPath` that is based on `id`. * *

Optionally, prepend a `prefix` to the name. */ private Path assembleDeletionVectorPath(String targetParentPath, UUID id, String prefix) { String fileName = String.format("%s_%s.bin", DELETION_VECTOR_FILE_NAME_CORE, id.toString()); if (prefix.length() > 0) { return new Path(new Path(targetParentPath, prefix), fileName); } else { return new Path(targetParentPath, fileName); } } @Override public String toString() { return String.format( "DeletionVectorDescriptor(storageType=%s, pathOrInlineDv=%s, offset=%s, " + "sizeInBytes=%s, cardinality=%s)", storageType, pathOrInlineDv, offset, sizeInBytes, cardinality); } /** @return Row representation of this deletion vector descriptor */ public Row toRow() { Map fieldMap = new HashMap<>(); fieldMap.put(COL_NAME_TO_ORDINAL.get("storageType"), storageType); fieldMap.put(COL_NAME_TO_ORDINAL.get("pathOrInlineDv"), pathOrInlineDv); // Only add offset if it's present if (offset.isPresent()) { fieldMap.put(COL_NAME_TO_ORDINAL.get("offset"), offset.get()); } // If offset is not present, the field remains null in the map fieldMap.put(COL_NAME_TO_ORDINAL.get("sizeInBytes"), sizeInBytes); fieldMap.put(COL_NAME_TO_ORDINAL.get("cardinality"), cardinality); return new GenericRow(READ_SCHEMA, fieldMap); } @Override public boolean equals(Object o) { if (o == this) { return true; } if (!(o instanceof DeletionVectorDescriptor)) { return false; } DeletionVectorDescriptor dv = (DeletionVectorDescriptor) o; return Objects.equals(storageType, dv.storageType) && Objects.equals(pathOrInlineDv, dv.pathOrInlineDv) && Objects.equals(offset, dv.offset) && this.sizeInBytes == dv.sizeInBytes && this.cardinality == dv.cardinality; } @Override public int hashCode() { return Objects.hash(storageType, pathOrInlineDv, offset, sizeInBytes, cardinality); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/DomainMetadata.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.actions; import static io.delta.kernel.internal.util.InternalUtils.requireNonNull; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.Row; import io.delta.kernel.internal.data.GenericRow; import io.delta.kernel.internal.rowtracking.RowTrackingMetadataDomain; import io.delta.kernel.types.BooleanType; import io.delta.kernel.types.StringType; import io.delta.kernel.types.StructType; import java.util.*; /** Delta log action representing an `DomainMetadata` action */ public class DomainMetadata { private static final Set SUPPORTED_SYSTEM_DOMAINS = Collections.unmodifiableSet( new HashSet<>(Collections.singletonList(RowTrackingMetadataDomain.DOMAIN_NAME))); /** Whether the provided {@code domain} is a user-controlled domain */ public static boolean isUserControlledDomain(String domain) { // Domain identifiers are case-sensitive, but we don't want to allow users to set domains // with prefixes like `DELTA.` either, so perform case-insensitive check for this purpose return !domain.toLowerCase(Locale.ROOT).startsWith("delta."); } /** * Checks whether the provided {@code domain} is a system domain that is supported for set in a * Delta transaction via addDomainMetadata. * *

By default, system domains are not allowed to be set through transaction-level domain * metadata due to their reserved nature. However, there are specific system domains—such as * {@link RowTrackingMetadataDomain#DOMAIN_NAME}—that are explicitly allowed to be set in this * context. This method defines the allowlist of such supported domains and checks against it. */ public static boolean isSystemDomainSupportedSetFromTxn(String domain) { return SUPPORTED_SYSTEM_DOMAINS.contains(domain); } /** Full schema of the {@link DomainMetadata} action in the Delta Log. */ public static final StructType FULL_SCHEMA = new StructType() .add("domain", StringType.STRING, false /* nullable */) .add("configuration", StringType.STRING, false /* nullable */) .add("removed", BooleanType.BOOLEAN, false /* nullable */); public static DomainMetadata fromColumnVector(ColumnVector vector, int rowId) { if (vector.isNullAt(rowId)) { return null; } return new DomainMetadata( requireNonNull(vector.getChild(0), rowId, "domain").getString(rowId), requireNonNull(vector.getChild(1), rowId, "configuration").getString(rowId), requireNonNull(vector.getChild(2), rowId, "removed").getBoolean(rowId)); } /** * Creates a {@link DomainMetadata} instance from a Row with the schema being {@link * DomainMetadata#FULL_SCHEMA}. * * @param row the Row object containing the DomainMetadata action * @return a DomainMetadata instance or null if the row is null * @throws IllegalArgumentException if the schema of the row does not match {@link * DomainMetadata#FULL_SCHEMA} */ public static DomainMetadata fromRow(Row row) { if (row == null) { return null; } checkArgument( row.getSchema().equals(FULL_SCHEMA), "Expected schema: %s, found: %s", FULL_SCHEMA, row.getSchema()); return new DomainMetadata( requireNonNull(row, 0, "domain").getString(0), requireNonNull(row, 1, "configuration").getString(1), requireNonNull(row, 2, "removed").getBoolean(2)); } private final String domain; private final String configuration; private final boolean removed; /** * The domain metadata action contains a configuration string for a named metadata domain. Two * overlapping transactions conflict if they both contain a domain metadata action for the same * metadata domain. Per-domain conflict resolution logic can be implemented. * * @param domain A string used to identify a specific domain. * @param configuration A string containing configuration for the metadata domain. * @param removed If it is true it serves as a tombstone to logically delete a {@link * DomainMetadata} action. */ public DomainMetadata(String domain, String configuration, boolean removed) { this.domain = requireNonNull(domain, "domain is null"); this.configuration = requireNonNull(configuration, "configuration is null"); this.removed = removed; } public String getDomain() { return domain; } public String getConfiguration() { return configuration; } public boolean isRemoved() { return removed; } /** * Encode as a {@link Row} object with the schema {@link DomainMetadata#FULL_SCHEMA}. * * @return {@link Row} object with the schema {@link DomainMetadata#FULL_SCHEMA} */ public Row toRow() { Map domainMetadataMap = new HashMap<>(); domainMetadataMap.put(0, domain); domainMetadataMap.put(1, configuration); domainMetadataMap.put(2, removed); return new GenericRow(DomainMetadata.FULL_SCHEMA, domainMetadataMap); } public DomainMetadata removed() { checkArgument(!removed, "Cannot remove a domain metadata tombstone (already removed)"); return new DomainMetadata(domain, configuration, true /* removed */); } @Override public String toString() { return String.format( "DomainMetadata{domain='%s', configuration='%s', removed='%b'}", domain, configuration, removed); } @Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null || getClass() != obj.getClass()) return false; DomainMetadata that = (DomainMetadata) obj; return removed == that.removed && domain.equals(that.domain) && configuration.equals(that.configuration); } @Override public int hashCode() { return java.util.Objects.hash(domain, configuration, removed); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/Format.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.actions; import static io.delta.kernel.internal.util.InternalUtils.requireNonNull; import static io.delta.kernel.internal.util.VectorUtils.*; import static java.util.Collections.emptyMap; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.Row; import io.delta.kernel.internal.data.GenericRow; import io.delta.kernel.types.*; import java.io.Serializable; import java.util.*; public class Format implements Serializable { private static final long serialVersionUID = 1L; public static Format fromColumnVector(ColumnVector vector, int rowId) { if (vector.isNullAt(rowId)) { return null; } final String provider = requireNonNull(vector.getChild(0), rowId, "provider").getString(rowId); final Map options = vector.getChild(1).isNullAt(rowId) ? Collections.emptyMap() : toJavaMap(vector.getChild(1).getMap(rowId)); return new Format(provider, options); } public static final StructType FULL_SCHEMA = new StructType() .add("provider", StringType.STRING, false /* nullable */) .add( "options", new MapType(StringType.STRING, StringType.STRING, false), true /* nullable */); private final String provider; private final Map options; public Format(String provider, Map options) { this.provider = provider; this.options = options; } public Format() { this.provider = "parquet"; this.options = emptyMap(); } public String getProvider() { return provider; } public Map getOptions() { return Collections.unmodifiableMap(options); } /** * Encode as a {@link Row} object with the schema {@link Format#FULL_SCHEMA}. * * @return {@link Row} object with the schema {@link Format#FULL_SCHEMA} */ public Row toRow() { Map formatMap = new HashMap<>(); formatMap.put(0, provider); formatMap.put(1, stringStringMapValue(options)); return new GenericRow(Format.FULL_SCHEMA, formatMap); } @Override public String toString() { return "Format{" + "provider='" + provider + '\'' + ", options=" + options + '}'; } @Override public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { return false; } Format format = (Format) o; return provider.equals(format.provider) && options.equals(format.options); } @Override public int hashCode() { return Objects.hash(provider, options); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/GenerateIcebergCompatActionUtils.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.actions; import static io.delta.kernel.internal.util.InternalUtils.relativizePath; import static io.delta.kernel.internal.util.PartitionUtils.serializePartitionMap; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static io.delta.kernel.internal.util.Preconditions.checkState; import static java.util.Objects.requireNonNull; import io.delta.kernel.Transaction; import io.delta.kernel.data.MapValue; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.expressions.Literal; import io.delta.kernel.internal.DeltaErrors; import io.delta.kernel.internal.TableConfig; import io.delta.kernel.internal.annotation.VisibleForTesting; import io.delta.kernel.internal.data.GenericRow; import io.delta.kernel.internal.data.TransactionStateRow; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.icebergcompat.IcebergCompatV2MetadataValidatorAndUpdater; import io.delta.kernel.internal.icebergcompat.IcebergCompatV3MetadataValidatorAndUpdater; import io.delta.kernel.statistics.DataFileStatistics; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterable; import io.delta.kernel.utils.DataFileStatus; import java.net.URI; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; /** Utilities to convert Iceberg add/removes to Delta Kernel add/removes */ public final class GenerateIcebergCompatActionUtils { /** * Create an add action {@link Row} that can be passed to {@link Transaction#commit(Engine, * CloseableIterable)} from an Iceberg add. * * @param transactionState the transaction state from the built transaction * @param fileStatus the file status to create the add with (contains path, time, size, and stats) * @param partitionValues the partition values for the add * @param dataChange whether or not the add constitutes a dataChange (i.e. append vs. compaction) * @param tags key-value metadata to be attached to the add action * @param physicalSchemaOpt An optional pre-parsed physical schema. Improves performance for batch * operations by avoiding repeated JSON parsing. Recommended when generating many actions with * the same schema. * @return add action row that can be included in the transaction * @throws UnsupportedOperationException if icebergWriterCompatV1 is not enabled * @throws UnsupportedOperationException if maxRetries != 0 in the transaction * @throws KernelException if stats are not present (required for icebergCompatV2) * @throws UnsupportedOperationException if the table is partitioned (currently unsupported) */ public static Row generateIcebergCompatWriterV1AddAction( Row transactionState, DataFileStatus fileStatus, Map partitionValues, boolean dataChange, Map tags, Optional physicalSchemaOpt) { Map configuration = TransactionStateRow.getConfiguration(transactionState); /* ----- Validate that this is a valid usage of this API ----- */ validateIcebergWriterCompatV1Enabled(configuration); validateMaxRetriesSetToZero(transactionState); /* ----- Validate this is valid write given the table's protocol & configurations ----- */ checkState( TableConfig.ICEBERG_COMPAT_V2_ENABLED.fromMetadata(configuration), "icebergCompatV2 not enabled despite icebergWriterCompatV1 enabled"); // We require field `numRecords` when icebergCompatV2 is enabled IcebergCompatV2MetadataValidatorAndUpdater.validateDataFileStatus(fileStatus); /* --- Validate and update partitionValues ---- */ // Currently we don't support partitioned tables; fail here blockPartitionedTables(transactionState, partitionValues); URI tableRoot = new Path(TransactionStateRow.getTablePath(transactionState)).toUri(); // This takes care of relativizing the file path and serializing the file statistics AddFile addFile = AddFile.convertDataFileStatus( physicalSchemaOpt.orElseGet( () -> TransactionStateRow.getPhysicalSchema(transactionState)), tableRoot, fileStatus, partitionValues, dataChange, tags, Optional.empty() /* baseRowId */, Optional.empty() /* defaultRowCommitVersion */, Optional.empty() /* deletionVectorDescriptor */); return SingleAction.createAddFileSingleAction(addFile.toRow()); } public static Row generateIcebergCompatWriterV1AddAction( Row transactionState, DataFileStatus fileStatus, Map partitionValues, boolean dataChange, Optional physicalSchemaOpt) { return generateIcebergCompatWriterV1AddAction( transactionState, fileStatus, partitionValues, dataChange, Collections.emptyMap(), physicalSchemaOpt); } /** * Create an add action {@link Row} that can be passed to {@link Transaction#commit(Engine, * CloseableIterable)} from an Iceberg add. * * @param transactionState the transaction state from the built transaction * @param fileStatus the file status to create the add with (contains path, time, size, and stats) * @param partitionValues the partition values for the add * @param dataChange whether or not the add constitutes a dataChange (i.e. append vs. compaction) * @param tags key-value metadata to be attached to the add action * @param deletionVectorDescriptor optional deletion vector descriptor for the add action * @param physicalSchemaOpt An optional pre-parsed physical schema. Improves performance for batch * operations by avoiding repeated JSON parsing. Recommended when generating many actions with * the same schema. * @return add action row that can be included in the transaction * @throws UnsupportedOperationException if icebergWriterCompatV3 is not enabled * @throws UnsupportedOperationException if maxRetries != 0 in the transaction * @throws KernelException if stats are not present (required for icebergCompatV3) * @throws UnsupportedOperationException if the table is partitioned (currently unsupported) */ public static Row generateIcebergCompatWriterV3AddAction( Row transactionState, DataFileStatus fileStatus, Map partitionValues, boolean dataChange, Map tags, Optional baseRowId, Optional defaultRowCommitVersion, Optional deletionVectorDescriptor, Optional physicalSchemaOpt) { Map configuration = TransactionStateRow.getConfiguration(transactionState); /* ----- Validate that this is a valid usage of this API ----- */ validateIcebergWriterCompatV3Enabled(configuration); validateMaxRetriesSetToZero(transactionState); /* ----- Validate that deletion vector is passed in only when the table supports it ----- */ deletionVectorDescriptor.ifPresent(dv -> validateIcebergDeletionVectorsEnabled(configuration)); /* ----- Validate this is valid write given the table's protocol & configurations ----- */ checkState( TableConfig.ICEBERG_COMPAT_V3_ENABLED.fromMetadata(configuration), "icebergCompatV3 not enabled despite icebergWriterCompatV3 enabled"); // We require field `numRecords` when icebergCompatV3 is enabled IcebergCompatV3MetadataValidatorAndUpdater.validateDataFileStatus(fileStatus); /* --- Validate and update partitionValues ---- */ // Currently we don't support partitioned tables; fail here blockPartitionedTables(transactionState, partitionValues); URI tableRoot = new Path(TransactionStateRow.getTablePath(transactionState)).toUri(); // This takes care of relativizing the file path and serializing the file statistics AddFile addFile = AddFile.convertDataFileStatus( physicalSchemaOpt.orElseGet( () -> TransactionStateRow.getPhysicalSchema(transactionState)), tableRoot, fileStatus, partitionValues, dataChange, tags, baseRowId, defaultRowCommitVersion, deletionVectorDescriptor); return SingleAction.createAddFileSingleAction(addFile.toRow()); } /** * Create a remove action {@link Row} that can be passed to {@link Transaction#commit(Engine, * CloseableIterable)} from an Iceberg remove. * * @param transactionState the transaction state from the built transaction * @param fileStatus the file status to create the remove with (contains path, time, size, and * stats) * @param partitionValues the partition values for the remove * @param dataChange whether or not the remove constitutes a dataChange (i.e. delete vs. * compaction) * @param physicalSchemaOpt An optional pre-parsed physical schema. Improves performance for batch * operations by avoiding repeated JSON parsing. Recommended when generating many actions with * the same schema. * @return remove action row that can be committed to the transaction * @throws UnsupportedOperationException if icebergWriterCompatV1 is not enabled * @throws UnsupportedOperationException if maxRetries != 0 in the transaction * @throws KernelException if the table is an append-only table and dataChange=true * @throws UnsupportedOperationException if the table is partitioned (currently unsupported) */ public static Row generateIcebergCompatWriterV1RemoveAction( Row transactionState, DataFileStatus fileStatus, Map partitionValues, boolean dataChange, Optional physicalSchemaOpt) { Map config = TransactionStateRow.getConfiguration(transactionState); /* ----- Validate that this is a valid usage of this API ----- */ validateIcebergWriterCompatV1Enabled(config); validateMaxRetriesSetToZero(transactionState); /* ----- Validate this is valid write given the table's protocol & configurations ----- */ // We only allow removes with dataChange=false when appendOnly=true blockUpdatingAppendOnlyTables(dataChange, transactionState, config); /* --- Validate and update partitionValues ---- */ // Currently we don't support partitioned tables; fail here blockPartitionedTables(transactionState, partitionValues); URI tableRoot = new Path(TransactionStateRow.getTablePath(transactionState)).toUri(); // This takes care of relativizing the file path and serializing the file statistics Row removeFileRow = convertRemoveDataFileStatus( physicalSchemaOpt.orElseGet( () -> TransactionStateRow.getPhysicalSchema(transactionState)), tableRoot, fileStatus, partitionValues, dataChange, Optional.empty() /* baseRowId */, Optional.empty() /* defaultRowCommitVersion */, Optional.empty() /* deletionVectorDescriptor */); return SingleAction.createRemoveFileSingleAction(removeFileRow); } /** * Create a remove action {@link Row} that can be passed to {@link Transaction#commit(Engine, * CloseableIterable)} from an Iceberg remove. * * @param transactionState the transaction state from the built transaction * @param fileStatus the file status to create the remove with (contains path, time, size, and * stats) * @param partitionValues the partition values for the remove * @param dataChange whether or not the remove constitutes a dataChange (i.e. delete vs. * compaction) * @param deletionVectorDescriptor optional deletion vector descriptor for the add action * @param physicalSchemaOpt An optional pre-parsed physical schema. Improves performance for batch * operations by avoiding repeated JSON parsing. Recommended when generating many actions with * the same schema. * @return remove action row that can be committed to the transaction * @throws UnsupportedOperationException if icebergWriterCompatV3 is not enabled * @throws UnsupportedOperationException if maxRetries != 0 in the transaction * @throws KernelException if the table is an append-only table and dataChange=true * @throws UnsupportedOperationException if the table is partitioned (currently unsupported) */ public static Row generateIcebergCompatWriterV3RemoveAction( Row transactionState, DataFileStatus fileStatus, Map partitionValues, boolean dataChange, Optional baseRowId, Optional defaultRowCommitVersion, Optional deletionVectorDescriptor, Optional physicalSchemaOpt) { Map config = TransactionStateRow.getConfiguration(transactionState); /* ----- Validate that this is a valid usage of this API ----- */ validateIcebergWriterCompatV3Enabled(config); validateMaxRetriesSetToZero(transactionState); /* ----- Validate that deletion vector is passed in only when the table supports it ----- */ deletionVectorDescriptor.ifPresent(dv -> validateIcebergDeletionVectorsEnabled(config)); /* ----- Validate this is valid write given the table's protocol & configurations ----- */ // We only allow removes with dataChange=false when appendOnly=true if (dataChange && TableConfig.APPEND_ONLY_ENABLED.fromMetadata(config)) { throw DeltaErrors.cannotModifyAppendOnlyTable( TransactionStateRow.getTablePath(transactionState)); } /* ----- Validate this is valid write given the table's protocol & configurations ----- */ // We only allow removes with dataChange=false when appendOnly=true blockUpdatingAppendOnlyTables(dataChange, transactionState, config); /* --- Validate and update partitionValues ---- */ // Currently we don't support partitioned tables; fail here blockPartitionedTables(transactionState, partitionValues); URI tableRoot = new Path(TransactionStateRow.getTablePath(transactionState)).toUri(); // This takes care of relativizing the file path and serializing the file statistics Row removeFileRow = convertRemoveDataFileStatus( physicalSchemaOpt.orElseGet( () -> TransactionStateRow.getPhysicalSchema(transactionState)), tableRoot, fileStatus, partitionValues, dataChange, baseRowId, defaultRowCommitVersion, deletionVectorDescriptor); return SingleAction.createRemoveFileSingleAction(removeFileRow); } ///////////////////// // Private helpers // ///////////////////// /** * Validates that table feature `icebergWriterCompatV1` is enabled. We restrict usage of these * APIs to require that this table feature is enabled to prevent any unsafe usage due to the table * features that are blocked via `icebergWriterCompatV1` (for example, rowTracking or * deletionVectors). */ private static void validateIcebergWriterCompatV1Enabled(Map config) { if (!TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.fromMetadata(config)) { throw new UnsupportedOperationException( String.format( "APIs within GenerateIcebergCompatActionUtils are only supported on tables with" + " '%s' set to true", TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.getKey())); } } /** * Validates that table feature `icebergWriterCompatV3` is enabled. We restrict usage of these * APIs to require that this table feature is enabled to prevent any unsafe usage due to the table * features that are blocked via `icebergWriterCompatV3`. */ private static void validateIcebergWriterCompatV3Enabled(Map config) { if (!TableConfig.ICEBERG_WRITER_COMPAT_V3_ENABLED.fromMetadata(config)) { throw new UnsupportedOperationException( String.format( "APIs within GenerateIcebergCompatActionUtils are only supported on tables with" + " '%s' set to true", TableConfig.ICEBERG_WRITER_COMPAT_V3_ENABLED.getKey())); } } /** * Validates that table feature `deletion vectors` is enabled. Checked when a deletion vector * descriptor is passed to generateIcebergCompatWriterV3AddAction. */ private static void validateIcebergDeletionVectorsEnabled(Map config) { if (!TableConfig.DELETION_VECTORS_CREATION_ENABLED.fromMetadata(config)) { throw new UnsupportedOperationException( String.format( "APIs within GenerateIcebergCompatActionUtils are only supported on tables with" + " '%s' set to true", TableConfig.DELETION_VECTORS_CREATION_ENABLED.getKey())); } } /** * Throws an exception if `maxRetries` was not set to 0 in the transaction. We restrict these APIs * to require `maxRetries = 0` since conflict resolution is not supported for operations other * than blind appends. */ private static void validateMaxRetriesSetToZero(Row transactionState) { if (TransactionStateRow.getMaxRetries(transactionState) > 0) { throw new UnsupportedOperationException( String.format( "Usage of GenerateIcebergCompatActionUtils requires maxRetries=0, " + "found maxRetries=%s", TransactionStateRow.getMaxRetries(transactionState))); } } private static void blockUpdatingAppendOnlyTables( boolean dataChange, Row transactionState, Map config) { // We only allow removes with dataChange=false when appendOnly=true if (dataChange && TableConfig.APPEND_ONLY_ENABLED.fromMetadata(config)) { throw DeltaErrors.cannotModifyAppendOnlyTable( TransactionStateRow.getTablePath(transactionState)); } } private static void blockPartitionedTables( Row transactionState, Map partitionValues) { if (!TransactionStateRow.getPartitionColumnsList(transactionState).isEmpty()) { throw new UnsupportedOperationException( "Currently GenerateIcebergCompatActionUtils " + "is not supported for partitioned tables"); } checkArgument( partitionValues.isEmpty(), "Non-empty partitionValues provided for an unpartitioned table"); } ////////////////////////////////////////////////// // Private methods for creating RemoveFile rows // ////////////////////////////////////////////////// // I've added these APIs here since they rely on the assumptions validated within // GenerateIcebergCompatActionUtils such as icebergWriterCompatV1 is enabled --> rowTracking is // disabled. Since these APIs are not valid without these assumptions, holding off on putting them // within RemoveFile.java until we add full support for deletes (which will likely involve // generating RemoveFiles directly from AddFiles anyway) @VisibleForTesting public static Row convertRemoveDataFileStatus( StructType physicalSchema, URI tableRoot, DataFileStatus dataFileStatus, Map partitionValues, boolean dataChange, Optional baseRowId, Optional defaultRowCommitVersion, Optional deletionVectorDescriptor) { return createRemoveFileRowWithExtendedFileMetadata( relativizePath(new Path(dataFileStatus.getPath()), tableRoot).toUri().toString(), dataFileStatus.getModificationTime(), dataChange, serializePartitionMap(partitionValues), dataFileStatus.getSize(), dataFileStatus.getStatistics(), physicalSchema, baseRowId, defaultRowCommitVersion, deletionVectorDescriptor); } @VisibleForTesting public static Row createRemoveFileRowWithExtendedFileMetadata( String path, long deletionTimestamp, boolean dataChange, MapValue partitionValues, long size, Optional stats, StructType physicalSchema, Optional baseRowId, Optional defaultRowCommitVersion, Optional deletionVector) { Map fieldMap = new HashMap<>(); fieldMap.put(RemoveFile.FULL_SCHEMA.indexOf("path"), requireNonNull(path)); fieldMap.put(RemoveFile.FULL_SCHEMA.indexOf("deletionTimestamp"), deletionTimestamp); fieldMap.put(RemoveFile.FULL_SCHEMA.indexOf("dataChange"), dataChange); fieldMap.put(RemoveFile.FULL_SCHEMA.indexOf("extendedFileMetadata"), true); fieldMap.put( RemoveFile.FULL_SCHEMA.indexOf("partitionValues"), requireNonNull(partitionValues)); fieldMap.put(RemoveFile.FULL_SCHEMA.indexOf("size"), size); stats.ifPresent( stat -> fieldMap.put( RemoveFile.FULL_SCHEMA.indexOf("stats"), stat.serializeAsJson(physicalSchema))); baseRowId.ifPresent(id -> fieldMap.put(RemoveFile.FULL_SCHEMA.indexOf("baseRowId"), id)); defaultRowCommitVersion.ifPresent( version -> fieldMap.put(RemoveFile.FULL_SCHEMA.indexOf("defaultRowCommitVersion"), version)); deletionVector.ifPresent( dv -> { Row dvRow = dv.toRow(); fieldMap.put(RemoveFile.FULL_SCHEMA.indexOf("deletionVector"), dvRow); }); return new GenericRow(RemoveFile.FULL_SCHEMA, fieldMap); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/Metadata.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.actions; import static io.delta.kernel.internal.util.InternalUtils.requireNonNull; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.data.*; import io.delta.kernel.internal.data.GenericRow; import io.delta.kernel.internal.lang.Lazy; import io.delta.kernel.internal.types.DataTypeJsonSerDe; import io.delta.kernel.internal.util.ColumnMapping; import io.delta.kernel.internal.util.VectorUtils; import io.delta.kernel.types.*; import java.io.InvalidObjectException; import java.io.ObjectInputStream; import java.io.Serializable; import java.util.*; import java.util.stream.Collectors; import javax.annotation.Nullable; public class Metadata implements Serializable { private static final long serialVersionUID = 1L; public static Metadata fromRow(Row row) { requireNonNull(row); checkArgument(FULL_SCHEMA.equals(row.getSchema())); return fromColumnVector( VectorUtils.buildColumnVector(Collections.singletonList(row), FULL_SCHEMA), /* rowId */ 0); } public static Metadata fromColumnVector(ColumnVector vector, int rowId) { if (vector.isNullAt(rowId)) { return null; } final String schemaJson = requireNonNull(vector.getChild(4), rowId, "schemaString").getString(rowId); Lazy lazySchema = new Lazy<>(() -> DataTypeJsonSerDe.deserializeStructType(schemaJson)); return new Metadata( requireNonNull(vector.getChild(0), rowId, "id").getString(rowId), Optional.ofNullable( vector.getChild(1).isNullAt(rowId) ? null : vector.getChild(1).getString(rowId)), Optional.ofNullable( vector.getChild(2).isNullAt(rowId) ? null : vector.getChild(2).getString(rowId)), Format.fromColumnVector(requireNonNull(vector.getChild(3), rowId, "format"), rowId), schemaJson, lazySchema, vector.getChild(5).getArray(rowId), Optional.ofNullable( vector.getChild(6).isNullAt(rowId) ? null : vector.getChild(6).getLong(rowId)), vector.getChild(7).getMap(rowId)); } public static final StructType FULL_SCHEMA = new StructType() .add("id", StringType.STRING, false /* nullable */) .add("name", StringType.STRING, true /* nullable */) .add("description", StringType.STRING, true /* nullable */) .add("format", Format.FULL_SCHEMA, false /* nullable */) .add("schemaString", StringType.STRING, false /* nullable */) .add( "partitionColumns", new ArrayType(StringType.STRING, false /* contains null */), false /* nullable */) .add("createdTime", LongType.LONG, true /* contains null */) .add( "configuration", new MapType(StringType.STRING, StringType.STRING, false), false /* nullable */); private final String id; private final Optional name; private final Optional description; private final Format format; private final String schemaString; private final Lazy schema; private final ArrayValue partitionColumns; private final Optional createdTime; private final MapValue configurationMapValue; private final Lazy> configuration; // Partition column names in lower case. private final Lazy> partitionColNames; // Logical data schema excluding partition columns private final Lazy dataSchema; public Metadata( String id, Optional name, Optional description, Format format, String schemaString, StructType schema, ArrayValue partitionColumns, Optional createdTime, MapValue configurationMapValue) { this( id, name, description, format, schemaString, new Lazy<>(() -> schema), partitionColumns, createdTime, configurationMapValue); } private Metadata( String id, Optional name, Optional description, Format format, String schemaString, Lazy lazySchema, ArrayValue partitionColumns, Optional createdTime, MapValue configurationMapValue) { this.id = requireNonNull(id, "id is null"); this.name = name; this.description = requireNonNull(description, "description is null"); this.format = requireNonNull(format, "format is null"); this.schemaString = requireNonNull(schemaString, "schemaString is null"); this.schema = new Lazy<>( () -> { StructType s = lazySchema.get(); ensureNoMetadataColumns(s); return s; }); this.partitionColumns = requireNonNull(partitionColumns, "partitionColumns is null"); this.createdTime = createdTime; this.configurationMapValue = requireNonNull(configurationMapValue, "configuration is null"); this.configuration = new Lazy<>(() -> VectorUtils.toJavaMap(configurationMapValue)); this.partitionColNames = new Lazy<>(this::loadPartitionColNames); this.dataSchema = new Lazy<>( () -> new StructType( this.schema.get().fields().stream() .filter( field -> !partitionColNames .get() .contains(field.getName().toLowerCase(Locale.ROOT))) .collect(Collectors.toList()))); } /** * Returns a new metadata object that has a new configuration which is the combination of its * current configuration and {@code configuration}. * *

For overlapping keys the values from {@code configuration} take precedence. */ public Metadata withMergedConfiguration(Map configuration) { Map newConfiguration = new HashMap<>(getConfiguration()); newConfiguration.putAll(configuration); return withReplacedConfiguration(newConfiguration); } /** * Returns a new metadata object that has a new configuration which does not contain any of the * keys provided in {@code keysToUnset}. */ public Metadata withConfigurationKeysUnset(Set keysToUnset) { Map newConfiguration = new HashMap<>(getConfiguration()); keysToUnset.forEach(newConfiguration::remove); return withReplacedConfiguration(newConfiguration); } /** * Returns a new Metadata object with the configuration provided with newConfiguration (any prior * configuration is replaced). */ public Metadata withReplacedConfiguration(Map newConfiguration) { return new Metadata( this.id, this.name, this.description, this.format, this.schemaString, this.schema, // pass Lazy directly to avoid forcing evaluation this.partitionColumns, this.createdTime, VectorUtils.stringStringMapValue(newConfiguration)); } public Metadata withNewSchema(StructType schema) { return new Metadata( this.id, this.name, this.description, this.format, schema.toJson(), schema, this.partitionColumns, this.createdTime, this.configurationMapValue); } @Override public String toString() { List partitionColumnsStr = VectorUtils.toJavaList(partitionColumns); StringBuilder sb = new StringBuilder(); sb.append("List("); for (String partitionColumn : partitionColumnsStr) { sb.append(partitionColumn).append(", "); } if (sb.substring(sb.length() - 2).equals(", ")) { sb.setLength(sb.length() - 2); // Remove the last comma and space } sb.append(")"); return "Metadata{" + "id='" + id + '\'' + ", name=" + name + ", description=" + description + ", format=" + format + ", schemaString='" + schemaString + '\'' + ", partitionColumns=" + sb + ", createdTime=" + createdTime + ", configuration=" + configuration.get() + '}'; } public String getSchemaString() { return schemaString; } public StructType getSchema() { return schema.get(); } public ArrayValue getPartitionColumns() { return partitionColumns; } /** Set of lowercase partition column names */ public Set getPartitionColNames() { return partitionColNames.get(); } /** The logical data schema which excludes partition columns */ public StructType getDataSchema() { return dataSchema.get(); } public String getId() { return id; } public Optional getName() { return name; } public Optional getDescription() { return description; } public Format getFormat() { return format; } public Optional getCreatedTime() { return createdTime; } public MapValue getConfigurationMapValue() { return configurationMapValue; } public Map getConfiguration() { return Collections.unmodifiableMap(configuration.get()); } /** * The full schema (including partition columns) with the field names converted to their physical * names (column names used in the data files) based on the table's column mapping mode. When * column mapping mode is ID, fieldId metadata is preserved in the field metadata; all column * metadata is otherwise removed. */ public StructType getPhysicalSchema() { ColumnMapping.ColumnMappingMode mappingMode = ColumnMapping.getColumnMappingMode(getConfiguration()); return ColumnMapping.convertToPhysicalSchema(getSchema(), getSchema(), mappingMode); } /** * Filter out the key-value pair matches exactly with the old properties. * * @param newProperties the new properties to be filtered * @return the filtered properties */ public Map filterOutUnchangedProperties(Map newProperties) { Map oldProperties = getConfiguration(); return newProperties.entrySet().stream() .filter( entry -> !oldProperties.containsKey(entry.getKey()) || !oldProperties.get(entry.getKey()).equals(entry.getValue())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } /** * Encode as a {@link Row} object with the schema {@link Metadata#FULL_SCHEMA}. * * @return {@link Row} object with the schema {@link Metadata#FULL_SCHEMA} */ public Row toRow() { Map metadataMap = new HashMap<>(); metadataMap.put(0, id); metadataMap.put(1, name.orElse(null)); metadataMap.put(2, description.orElse(null)); metadataMap.put(3, format.toRow()); metadataMap.put(4, schemaString); metadataMap.put(5, partitionColumns); metadataMap.put(6, createdTime.orElse(null)); metadataMap.put(7, configurationMapValue); return new GenericRow(Metadata.FULL_SCHEMA, metadataMap); } @Override public int hashCode() { return Objects.hash( id, name, description, format, schema.get(), partitionColNames.get(), createdTime, configuration.get()); } @Override public boolean equals(Object o) { if (!(o instanceof Metadata)) { return false; } Metadata other = (Metadata) o; return id.equals(other.id) && name.equals(other.name) && description.equals(other.description) && format.equals(other.format) && schema.get().equals(other.schema.get()) && partitionColNames.get().equals(other.partitionColNames.get()) && createdTime.equals(other.createdTime) && configuration.get().equals(other.configuration.get()); } /** Helper method to load the partition column names. */ private Set loadPartitionColNames() { ColumnVector partitionColNameVector = partitionColumns.getElements(); Set partitionColumnNames = new HashSet<>(); for (int i = 0; i < partitionColumns.getSize(); i++) { checkArgument( !partitionColNameVector.isNullAt(i), "Expected a non-null partition column name"); String partitionColName = partitionColNameVector.getString(i); checkArgument( partitionColName != null && !partitionColName.isEmpty(), "Expected non-null and non-empty partition column name"); partitionColumnNames.add(partitionColName.toLowerCase(Locale.ROOT)); } return Collections.unmodifiableSet(partitionColumnNames); } /** Helper method to ensure that a table schema never contains metadata columns. */ private void ensureNoMetadataColumns(StructType schema) { for (StructField field : schema.fields()) { if (field.isMetadataColumn()) { throw new IllegalArgumentException( "Table schema cannot contain metadata columns: " + field.getName()); } } } /** * Serializable representation of Metadata. Converts complex Kernel types (ArrayValue, MapValue) * to simple Java types (List, Map) that are serializable. */ private static class SerializableMetadata implements Serializable { private static final long serialVersionUID = 1L; private final String id; @Nullable private final String name; @Nullable private final String description; private final String formatProvider; private final Map formatOptions; private final String schemaString; private final List partitionColumnsList; @Nullable private final Long createdTime; private final Map configuration; SerializableMetadata(Metadata metadata) { this.id = metadata.id; this.name = metadata.name.orElse(null); this.description = metadata.description.orElse(null); this.formatProvider = metadata.format.getProvider(); this.formatOptions = metadata.format.getOptions(); this.schemaString = metadata.schemaString; this.partitionColumnsList = VectorUtils.toJavaList(metadata.partitionColumns); this.createdTime = metadata.createdTime.orElse(null); this.configuration = VectorUtils.toJavaMap(metadata.configurationMapValue); } // Reconstruct Metadata from serialized data private Object readResolve() { Format format = new Format(formatProvider, formatOptions); Lazy lazySchema = new Lazy<>(() -> DataTypeJsonSerDe.deserializeStructType(schemaString)); ArrayValue partitionColumns = VectorUtils.buildArrayValue(partitionColumnsList, StringType.STRING); MapValue configurationMapValue = VectorUtils.stringStringMapValue(configuration); return new Metadata( id, Optional.ofNullable(name), Optional.ofNullable(description), format, schemaString, lazySchema, partitionColumns, Optional.ofNullable(createdTime), configurationMapValue); } } /** * Replace this Metadata with SerializableMetadata during serialization. This is the standard Java * serialization proxy pattern for immutable objects with complex fields. */ private Object writeReplace() { return new SerializableMetadata(this); } /** Prevent direct deserialization of Metadata (must use SerializableMetadata). */ private void readObject(ObjectInputStream stream) throws InvalidObjectException { throw new InvalidObjectException("Use SerializableMetadata"); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/Protocol.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.actions; import static io.delta.kernel.internal.tablefeatures.TableFeatures.TABLE_FEATURES; import static io.delta.kernel.internal.tablefeatures.TableFeatures.TABLE_FEATURES_MIN_WRITER_VERSION; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static io.delta.kernel.internal.util.VectorUtils.buildArrayValue; import static java.lang.String.format; import static java.util.Collections.emptySet; import static java.util.Collections.unmodifiableSet; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toSet; import io.delta.kernel.data.*; import io.delta.kernel.exceptions.UnsupportedTableFeatureException; import io.delta.kernel.internal.data.GenericRow; import io.delta.kernel.internal.tablefeatures.TableFeature; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.internal.util.Tuple2; import io.delta.kernel.internal.util.VectorUtils; import io.delta.kernel.types.ArrayType; import io.delta.kernel.types.IntegerType; import io.delta.kernel.types.StringType; import io.delta.kernel.types.StructType; import java.io.Serializable; import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; public class Protocol implements Serializable { private static final long serialVersionUID = 1L; ///////////////////////////////////////////////////////////////////////////////////////////////// /// Public static variables and methods /// ///////////////////////////////////////////////////////////////////////////////////////////////// /** * Helper method to get the Protocol from the row representation. * * @param row Row representation of the Protocol. * @return the Protocol object */ public static Protocol fromRow(Row row) { requireNonNull(row); Set readerFeatures = row.isNullAt(2) ? Collections.emptySet() : Collections.unmodifiableSet(new HashSet<>(VectorUtils.toJavaList(row.getArray(2)))); Set writerFeatures = row.isNullAt(3) ? Collections.emptySet() : Collections.unmodifiableSet(new HashSet<>(VectorUtils.toJavaList(row.getArray(3)))); return new Protocol(row.getInt(0), row.getInt(1), readerFeatures, writerFeatures); } public static Protocol fromColumnVector(ColumnVector vector, int rowId) { if (vector.isNullAt(rowId)) { return null; } return new Protocol( vector.getChild(0).getInt(rowId), vector.getChild(1).getInt(rowId), vector.getChild(2).isNullAt(rowId) ? emptySet() : new HashSet<>(VectorUtils.toJavaList(vector.getChild(2).getArray(rowId))), vector.getChild(3).isNullAt(rowId) ? emptySet() : new HashSet<>(VectorUtils.toJavaList(vector.getChild(3).getArray(rowId)))); } public static final StructType FULL_SCHEMA = new StructType() .add("minReaderVersion", IntegerType.INTEGER, false /* nullable */) .add("minWriterVersion", IntegerType.INTEGER, false /* nullable */) .add("readerFeatures", new ArrayType(StringType.STRING, false /* contains null */)) .add("writerFeatures", new ArrayType(StringType.STRING, false /* contains null */)); private final int minReaderVersion; private final int minWriterVersion; private final Set readerFeatures; private final Set writerFeatures; // These are derived fields from minReaderVersion and minWriterVersion private final boolean supportsReaderFeatures; private final boolean supportsWriterFeatures; public Protocol(int minReaderVersion, int minWriterVersion) { this(minReaderVersion, minWriterVersion, emptySet(), emptySet()); } public Protocol( int minReaderVersion, int minWriterVersion, Set readerFeatures, Set writerFeatures) { this.minReaderVersion = minReaderVersion; this.minWriterVersion = minWriterVersion; this.readerFeatures = unmodifiableSet(requireNonNull(readerFeatures, "readerFeatures cannot be null")); this.writerFeatures = unmodifiableSet(requireNonNull(writerFeatures, "writerFeatures cannot be null")); this.supportsReaderFeatures = TableFeatures.supportsReaderFeatures(minReaderVersion); this.supportsWriterFeatures = TableFeatures.supportsWriterFeatures(minWriterVersion); } /** @return The minimum reader version required for this protocol */ public int getMinReaderVersion() { return minReaderVersion; } /** @return The minimum writer version required for this protocol */ public int getMinWriterVersion() { return minWriterVersion; } /** * @return The set of explicitly specified reader features for this protocol. Will be empty if * this protocol does not support reader features. */ public Set getReaderFeatures() { return readerFeatures; } /** * @return The set of explicitly specified writer features for this protocol. Will be empty if * this protocol does not support writer features. */ public Set getWriterFeatures() { return writerFeatures; } /** * @return The combined set of all reader and writer features for this protocol. Will be empty if * this protocol does not support reader or writer features. */ public Set getReaderAndWriterFeatures() { final Set allFeatureNames = new HashSet<>(); allFeatureNames.addAll(readerFeatures); allFeatureNames.addAll(writerFeatures); return allFeatureNames; } /** * @return Whether this protocol supports explicitly specifying reader features, which occurs when * the minReaderVersion is greater than or equal to 3. */ public boolean supportsReaderFeatures() { return supportsReaderFeatures; } /** * @return Whether this protocol supports explicitly specifying writer features, which occurs when * the minWriterVersion is greater than or equal to 7. */ public boolean supportsWriterFeatures() { return supportsWriterFeatures; } @Override public String toString() { final StringBuilder sb = new StringBuilder("Protocol{"); sb.append("minReaderVersion=").append(minReaderVersion); sb.append(", minWriterVersion=").append(minWriterVersion); sb.append(", readerFeatures=").append(readerFeatures); sb.append(", writerFeatures=").append(writerFeatures); sb.append('}'); return sb.toString(); } @Override public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { return false; } Protocol protocol = (Protocol) o; return minReaderVersion == protocol.minReaderVersion && minWriterVersion == protocol.minWriterVersion && Objects.equals(readerFeatures, protocol.readerFeatures) && Objects.equals(writerFeatures, protocol.writerFeatures); } @Override public int hashCode() { return Objects.hash(minReaderVersion, minWriterVersion, readerFeatures, writerFeatures); } /** * Encode as a {@link Row} object with the schema {@link Protocol#FULL_SCHEMA}. Write any empty * `readerFeatures` and `writerFeatures` as null. * * @return {@link Row} object with the schema {@link Protocol#FULL_SCHEMA} */ public Row toRow() { Map protocolMap = new HashMap<>(); protocolMap.put(0, minReaderVersion); protocolMap.put(1, minWriterVersion); if (supportsReaderFeatures) { protocolMap.put(2, buildArrayValue(new ArrayList<>(readerFeatures), StringType.STRING)); } if (supportsWriterFeatures) { protocolMap.put(3, buildArrayValue(new ArrayList<>(writerFeatures), StringType.STRING)); } return new GenericRow(Protocol.FULL_SCHEMA, protocolMap); } ///////////////////////////////////////////////////////////////////////////////////////////////// /// Public methods related to table features interaction with the protocol /// ///////////////////////////////////////////////////////////////////////////////////////////////// /** * Get the set of features that are implicitly supported by the protocol. Features are implicitly * supported if the reader and/or writer version is less than the versions that supports the * explicit features specified in `readerFeatures` and `writerFeatures` sets. Examples: * *

* *

    *
  • (minRV = 1, minWV = 7, readerFeatures=[], writerFeatures=[domainMetadata]) results in [] *
  • (minRV = 1, minWV = 3) results in [appendOnly, invariants, checkConstraints] *
  • (minRV = 3, minWV = 7, readerFeatures=[v2Checkpoint], writerFeatures=[v2Checkpoint]) * results in [] *
  • (minRV = 2, minWV = 6) results in [appendOnly, invariants, checkConstraints, * changeDataFeed, generatedColumns, columnMapping, identityColumns] *
*/ public Set getImplicitlySupportedFeatures() { if (supportsReaderFeatures && supportsWriterFeatures) { return emptySet(); } else { return TABLE_FEATURES.stream() .filter(f -> !supportsReaderFeatures && f.minReaderVersion() <= minReaderVersion) .filter(f -> !supportsWriterFeatures && f.minWriterVersion() <= minWriterVersion) .collect(Collectors.toSet()); } } /** * Get the set of features that are explicitly supported by the protocol. Features are explicitly * supported if they are present in the `readerFeatures` and/or `writerFeatures` sets. Examples: * *

* *

    *
  • (minRV = 1, minWV = 7, writerFeatures=[appendOnly, invariants, checkConstraints]) results * in [appendOnly, invariants, checkConstraints] *
  • (minRV = 3, minWV = 7, readerFeatures = [columnMapping], writerFeatures=[columnMapping, * invariants]) results in [columnMapping, invariants] *
  • (minRV = 1, minWV = 2, readerFeatures = [], writerFeatures=[]) results in [] *
* * @throws UnsupportedTableFeatureException if any table features in the protocol's list of * readerFeatures or writerFeatures are unsupported by Kernel */ public Set getExplicitlySupportedFeatures() { return Stream.of(readerFeatures, writerFeatures) .flatMap(Set::stream) .map(TableFeatures::getTableFeature) // if a feature is not known, will throw an exception .collect(Collectors.toSet()); } /** * Get the set of features that are both implicitly and explicitly supported by the protocol. * Usually, the protocol has either implicit or explicit features, but not both. This API provides * a way to get all enabled features. * * @throws UnsupportedTableFeatureException if any table features in the protocol's list of * readerFeatures or writerFeatures are unsupported by Kernel */ public Set getImplicitlyAndExplicitlySupportedFeatures() { Set supportedFeatures = new HashSet<>(); supportedFeatures.addAll(getImplicitlySupportedFeatures()); supportedFeatures.addAll(getExplicitlySupportedFeatures()); return supportedFeatures; } /** * Get the set of reader writer features that are both implicitly and explicitly supported by the * protocol. Usually, the protocol has either implicit or explicit features, but not both. This * API provides a way to get all enabled reader writer features. It doesn't return any writer only * features. */ public Set getImplicitlyAndExplicitlySupportedReaderWriterFeatures() { return Stream.concat( // implicit supported features TABLE_FEATURES.stream() .filter( f -> !supportsReaderFeatures && f.minReaderVersion() <= this.getMinReaderVersion()), // explicitly supported features readerFeatures.stream().map(TableFeatures::getTableFeature)) .collect(toSet()); } /** Create a new {@link Protocol} object with the given {@link TableFeature} supported. */ public Protocol withFeatures(Iterable newFeatures) { Protocol result = this; for (TableFeature feature : newFeatures) { result = result.withFeature(feature); } return result; } /** * Get a new Protocol object that has `feature` supported. Writer-only features will be added to * `writerFeatures` field, and reader-writer features will be added to `readerFeatures` and * `writerFeatures` fields. * *

If `feature` is already implicitly supported in the current protocol's legacy reader or * writer protocol version, the new protocol will not modify the original protocol version, i.e., * the feature will not be explicitly added to the protocol's `readerFeatures` or * `writerFeatures`. This is to avoid unnecessary protocol upgrade for feature that it already * supports. * *

Examples: * *

    *
  • current protocol (2, 5) and new feature to add 'invariants` result in (2, 5) as this * protocol already supports 'invariants' implicitly. *
  • current protocol is (1, 7, writerFeature='rowTracking,domainMetadata' and the new feature * to add is 'appendOnly' results in (1, 7, * writerFeature='rowTracking,domainMetadata,appendOnly') *
  • current protocol is (1, 7, writerFeature='rowTracking,domainMetadata' and the new feature * to add is 'columnMapping' results in throwing UnsupportedOperationException as * 'columnMapping' requires higher reader version (2) than the current protocol's reader * version (1). *
*/ public Protocol withFeature(TableFeature feature) { // Add required dependencies of the feature Protocol protocolWithDependencies = withFeatures(feature.requiredFeatures()); if (feature.minReaderVersion() > protocolWithDependencies.minReaderVersion) { throw new UnsupportedOperationException( "TableFeature requires higher reader protocol version"); } if (feature.minWriterVersion() > protocolWithDependencies.minWriterVersion) { throw new UnsupportedOperationException( "TableFeature requires higher writer protocol version"); } boolean shouldAddToReaderFeatures = feature.isReaderWriterFeature() && // protocol already has support for `readerFeatures` set and the new feature // can be explicitly added to the protocol's `readerFeatures` supportsReaderFeatures; Set newReaderFeatures = protocolWithDependencies.readerFeatures; Set newWriterFeatures = protocolWithDependencies.writerFeatures; if (shouldAddToReaderFeatures) { newReaderFeatures = new HashSet<>(protocolWithDependencies.readerFeatures); newReaderFeatures.add(feature.featureName()); } if (supportsWriterFeatures) { newWriterFeatures = new HashSet<>(protocolWithDependencies.writerFeatures); newWriterFeatures.add(feature.featureName()); } return new Protocol( protocolWithDependencies.minReaderVersion, protocolWithDependencies.minWriterVersion, newReaderFeatures, newWriterFeatures); } /** * Determine whether this protocol can be safely upgraded to a new protocol `to`. This means all * features supported by this protocol are supported by `to`. * *

Examples regarding feature status: * *

    *
  • `[appendOnly]` to `[appendOnly]` results in allowed. *
  • `[appendOnly, changeDataFeed]` to `[appendOnly]` results in not allowed. *
*/ public boolean canUpgradeTo(Protocol to) { return to.getImplicitlyAndExplicitlySupportedFeatures() .containsAll(this.getImplicitlyAndExplicitlySupportedFeatures()); } /** * Protocol normalization is the process of converting a table features protocol to the weakest * possible form. This primarily refers to converting a table features protocol to a legacy * protocol. A Table Features protocol can be represented with the legacy representation only when * the features set of the former exactly matches a legacy protocol. * *

Normalization can also decrease the reader version of a table features protocol when it is * higher than necessary. * *

For example: * *

    *
  • (1, 7, AppendOnly, Invariants, CheckConstraints) results in (1, 3) *
  • (3, 7, RowTracking) results in (1, 7, RowTracking) *
*/ public Protocol normalized() { // Normalization can only be applied to table feature protocols. if (!isFeatureProtocol()) { return this; } Tuple2 versions = TableFeatures.minimumRequiredVersions(getExplicitlySupportedFeatures()); int minReaderVersion = versions._1; int minWriterVersion = versions._2; Protocol newProtocol = new Protocol(minReaderVersion, minWriterVersion); if (this.getImplicitlyAndExplicitlySupportedFeatures() .equals(newProtocol.getImplicitlyAndExplicitlySupportedFeatures())) { return newProtocol; } else { // means we have some feature that is added after table feature support. // Whatever the feature (reader or readerWriter), it is always going to // have minWriterVersion as 7. Required minReaderVersion // should be based on the supported features. return new Protocol(minReaderVersion, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(getExplicitlySupportedFeatures()); } } /** * Protocol denormalization is the process of converting a legacy protocol to the equivalent table * features protocol. This is the inverse of protocol normalization. It can be used to allow * operations on legacy protocols that yield results which cannot be represented anymore by a * legacy protocol. For example * *
    *
  • (1, 3) results in (1, 7, readerFeatures=[], writerFeatures=[appendOnly, invariants, * checkConstraints]) *
  • (2, 5) results in (2, 7, readerFeatures=[], writerFeatures=[appendOnly, invariants, * checkConstraints, changeDataFeed, generatedColumns, columnMapping]) *
*/ public Protocol denormalized() { // Denormalization can only be applied to legacy protocols. if (!isLegacyProtocol()) { return this; } Tuple2 versions = TableFeatures.minimumRequiredVersions(getImplicitlySupportedFeatures()); int minReaderVersion = versions._1; return new Protocol(minReaderVersion, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(getImplicitlySupportedFeatures()); } /** * Helper method that applies both denormalization and normalization. This can be used to * normalize invalid legacy protocols such as (2, 3), (1, 5). A legacy protocol is invalid when * the version numbers are higher than required to support the implied feature set. */ public Protocol denormalizedNormalized() { return this.denormalized().normalized(); } /** * Merge this protocol with multiple `protocols` to have the highest reader and writer versions * plus all explicitly and implicitly supported features. */ public Protocol merge(Protocol... others) { List protocols = new ArrayList<>(); protocols.add(this); protocols.addAll(Arrays.asList(others)); int mergedReaderVersion = protocols.stream().mapToInt(Protocol::getMinReaderVersion).max().orElse(0); int mergedWriterVersion = protocols.stream().mapToInt(Protocol::getMinWriterVersion).max().orElse(0); Set mergedReaderFeatures = protocols.stream().flatMap(p -> p.readerFeatures.stream()).collect(Collectors.toSet()); Set mergedWriterFeatures = protocols.stream().flatMap(p -> p.writerFeatures.stream()).collect(Collectors.toSet()); Set mergedImplicitFeatures = protocols.stream() .flatMap(p -> p.getImplicitlySupportedFeatures().stream()) .collect(Collectors.toSet()); Protocol mergedProtocol = new Protocol( mergedReaderVersion, mergedWriterVersion, mergedReaderFeatures, mergedWriterFeatures) .withFeatures(mergedImplicitFeatures); // The merged protocol is always normalized in order to represent the protocol // with the weakest possible form. This enables backward compatibility. // This is preceded by a denormalization step. This allows to fix invalid legacy Protocols. // For example, (2, 3) is normalized to (1, 3). This is because there is no legacy feature // in the set with reader version 2 unless the writer version is at least 5. return mergedProtocol.denormalizedNormalized(); } /** Check if the protocol supports the given table feature */ public boolean supportsFeature(TableFeature feature) { if (feature.isReaderWriterFeature()) { if (supportsReaderFeatures) { return readerFeatures.contains(feature.featureName()); } else { return feature.minReaderVersion() <= minReaderVersion; } } else { if (supportsWriterFeatures) { return writerFeatures.contains(feature.featureName()); } else { return feature.minWriterVersion() <= minWriterVersion; } } } /** Validate the protocol contents represents a valid state */ protected void validate() { checkArgument(minReaderVersion >= 1, "minReaderVersion should be at least 1"); checkArgument(minWriterVersion >= 1, "minWriterVersion should be at least 1"); // expect the reader and writer features to be empty if the protocol version does not support checkArgument( readerFeatures.isEmpty() || supportsReaderFeatures, "Reader features are not supported for the reader version: " + minReaderVersion); checkArgument( writerFeatures.isEmpty() || supportsWriterFeatures, "Writer features are not supported for the writer version: " + minWriterVersion); // If reader versions are supported, expect the writer versions to be supported as well // We don't have any reader only features. if (supportsReaderFeatures) { checkArgument( supportsWriterFeatures, "writer version doesn't support writer features: " + minWriterVersion); } if (supportsWriterFeatures) { // ensure that the reader version supports all the readerWriter features Set supportedFeatures = getExplicitlySupportedFeatures(); supportedFeatures.stream() .filter(TableFeature::isReaderWriterFeature) .forEach( feature -> { checkArgument( feature.minReaderVersion() <= minReaderVersion, format( "Reader version %d does not support readerWriter feature %s", minReaderVersion, feature.featureName())); if (supportsReaderFeatures) { // if the protocol supports reader features, then it should be part of the // readerFeatures checkArgument( readerFeatures.contains(feature.featureName()), format( "ReaderWriter feature %s is not present in readerFeatures", feature.featureName())); } }); } else { // ensure we don't get (minReaderVersion, minWriterVersion) that satisfy the readerWriter // feature version requirements. E.g. (1, 5) is invalid as writer version indicates // columnMapping supported but reader version does not support it (requires 2). TABLE_FEATURES.stream() .filter(TableFeature::isReaderWriterFeature) .forEach( f -> { if (f.minWriterVersion() <= minWriterVersion) { checkArgument( f.minReaderVersion() <= minReaderVersion, format( "Reader version %d does not support readerWriter feature %s", minReaderVersion, f.featureName())); } }); } } /** is the protocol a legacy protocol, i.e before (3, 7) */ private boolean isLegacyProtocol() { return !supportsReaderFeatures && !supportsWriterFeatures; } /** is the protocol a table feature protocol, i.e after (3, 7) */ private boolean isFeatureProtocol() { // checking for writer feature support is enough as we have // writerOnly or readerWriter features, but no readerOnly features. return supportsWriterFeatures; } // Note: Protocol uses default Java serialization because all fields are Serializable: // - int, boolean: primitive types (automatically serializable) // - Set: Set and String both implement Serializable // No need for custom writeObject/readObject! } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/RemoveFile.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.actions; import io.delta.kernel.data.MapValue; import io.delta.kernel.data.Row; import io.delta.kernel.internal.util.VectorUtils; import io.delta.kernel.statistics.DataFileStatistics; import io.delta.kernel.types.*; import java.util.Optional; /** Metadata about {@code remove} action in the Delta Log. */ public class RemoveFile extends RowBackedAction { /** Full schema of the {@code remove} action in the Delta Log. */ public static final StructType FULL_SCHEMA = new StructType() .add("path", StringType.STRING, false /* nullable */) .add("deletionTimestamp", LongType.LONG, true /* nullable */) .add("dataChange", BooleanType.BOOLEAN, false /* nullable*/) .add("extendedFileMetadata", BooleanType.BOOLEAN, true /* nullable */) .add( "partitionValues", new MapType(StringType.STRING, StringType.STRING, true), true /* nullable*/) .add("size", LongType.LONG, true /* nullable*/) .add("stats", StringType.STRING, true /* nullable */) .add("tags", new MapType(StringType.STRING, StringType.STRING, true), true /* nullable */) .add("deletionVector", DeletionVectorDescriptor.READ_SCHEMA, true /* nullable */) .add("baseRowId", LongType.LONG, true /* nullable */) .add("defaultRowCommitVersion", LongType.LONG, true /* nullable */); // TODO: Currently Kernel doesn't support RemoveFile actions when rowTracking is enabled (or have // any public API for generating RemoveFile actions). Once we // do this we need to ensure that the baseRowId and defaultRowCommitVersion fields are correctly // populated to match the corresponding AddFile actions /** Constructs an {@link RemoveFile} action from the given 'RemoveFile' {@link Row}. */ public RemoveFile(Row row) { super(row); } public String getPath() { return row.getString(getFieldIndex("path")); } public Optional getDeletionTimestamp() { return row.isNullAt(getFieldIndex("deletionTimestamp")) ? Optional.empty() : Optional.of(row.getLong(getFieldIndex("deletionTimestamp"))); } public boolean getDataChange() { return row.getBoolean(getFieldIndex("dataChange")); } public Optional getExtendedFileMetadata() { return row.isNullAt(getFieldIndex("extendedFileMetadata")) ? Optional.empty() : Optional.of(row.getBoolean(getFieldIndex("extendedFileMetadata"))); } public Optional getPartitionValues() { return row.isNullAt(getFieldIndex("partitionValues")) ? Optional.empty() : Optional.of(row.getMap(getFieldIndex("partitionValues"))); } public Optional getSize() { return row.isNullAt(getFieldIndex("size")) ? Optional.empty() : Optional.of(row.getLong(getFieldIndex("size"))); } public Optional getStatsJson() { return getFieldIndexOpt("stats") .flatMap( index -> row.isNullAt(index) ? Optional.empty() : Optional.of(row.getString(index))); } public Optional getNumRecords() { return getFieldIndexOpt("stats") .flatMap( index -> row.isNullAt(index) ? Optional.empty() : DataFileStatistics.getNumRecords(row.getString(index))); } /** * Returns the file statistics parsed from the stats JSON string using the provided schema. This * method deserializes the statistics JSON with full type information, ensuring that min/max * values and null counts are correctly typed according to the physical schema. * * @param physicalSchema the physical schema of the table, used to correctly parse and type the * statistics values (min/max values and null counts) * @return an {@link Optional} containing the deserialized {@link DataFileStatistics} if the stats * field is present and non-null, or {@link Optional#empty()} otherwise * @throws io.delta.kernel.exceptions.KernelException if the stats JSON is malformed or if values * don't match the expected types from the schema * @see DataFileStatistics#deserializeFromJson(String, StructType) for details on the * deserialization process */ public Optional getStats(StructType physicalSchema) { return getFieldIndexOpt("stats") .flatMap( index -> row.isNullAt(index) ? Optional.empty() : DataFileStatistics.deserializeFromJson(row.getString(index), physicalSchema)); } public Optional getTags() { int index = getFieldIndex("tags"); return Optional.ofNullable(row.isNullAt(index) ? null : row.getMap(index)); } public Optional getDeletionVector() { int index = getFieldIndex("deletionVector"); return Optional.ofNullable( row.isNullAt(index) ? null : DeletionVectorDescriptor.fromRow(row.getStruct(index))); } public Optional getBaseRowId() { int index = getFieldIndex("baseRowId"); return Optional.ofNullable(row.isNullAt(index) ? null : row.getLong(index)); } public Optional getDefaultRowCommitVersion() { int index = getFieldIndex("defaultRowCommitVersion"); return Optional.ofNullable(row.isNullAt(index) ? null : row.getLong(index)); } @Override public String toString() { // No specific ordering is guaranteed for partitionValues and tags in the returned string StringBuilder sb = new StringBuilder(); sb.append("RemoveFile{"); sb.append("path='").append(getPath()).append('\''); sb.append(", deletionTimestamp=").append(getDeletionTimestamp()); sb.append(", dataChange=").append(getDataChange()); sb.append(", extendedFileMetadata=").append(getExtendedFileMetadata()); sb.append(", partitionValues=").append(getPartitionValues().map(VectorUtils::toJavaMap)); sb.append(", size=").append(getSize()); sb.append(", stats=").append(getStats(null).map(d -> d.serializeAsJson(null)).orElse("")); sb.append(", tags=").append(getTags().map(VectorUtils::toJavaMap)); sb.append(", deletionVector=").append(getDeletionVector()); sb.append(", baseRowId=").append(getBaseRowId()); sb.append(", defaultRowCommitVersion=").append(getDefaultRowCommitVersion()); sb.append('}'); return sb.toString(); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/RowBackedAction.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.actions; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.Row; import io.delta.kernel.internal.data.DelegateRow; import io.delta.kernel.types.*; import java.util.Collections; import java.util.Map; import java.util.Optional; /** * An abstract base class for Delta Log actions that are backed by a {@link Row}. This design is to * avoid materialization of all fields when creating action instances from action rows within * Kernel. Actions like {@link AddFile} can extend this class to maintain just a reference to the * underlying action row. */ public abstract class RowBackedAction { /** The underlying {@link Row} that represents an action and contains all its field values. */ protected final Row row; protected RowBackedAction(Row row) { this.row = row; } /** * Returns the index of the field with the given name in the schema of the row. Throws an {@link * IllegalArgumentException} if the field is not found. */ protected int getFieldIndex(String fieldName) { int index = row.getSchema().indexOf(fieldName); checkArgument(index >= 0, "Field '%s' not found in schema: %s", fieldName, row.getSchema()); return index; } /** * Returns the index of the field with the given name in the schema of the row, or {@link * Optional#empty()} if the field is not found. This should be used when the underlying row may or * may not contain that field. */ protected Optional getFieldIndexOpt(String fieldName) { int index = row.getSchema().indexOf(fieldName); return index >= 0 ? Optional.of(index) : Optional.empty(); } /** * Returns a new {@link Row} with the same schema and values as the row backing this action, but * with the value of the field with the given name overridden by the given value. */ protected Row toRowWithOverriddenValue(String fieldName, Object value) { Map overrides = Collections.singletonMap(getFieldIndex(fieldName), value); return new DelegateRow(row, overrides); } /** Returns the underlying {@link Row} that represents this action. */ public final Row toRow() { return row; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/SetTransaction.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.actions; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.Row; import io.delta.kernel.internal.data.GenericRow; import io.delta.kernel.types.LongType; import io.delta.kernel.types.StringType; import io.delta.kernel.types.StructType; import java.util.*; /** Delta log action representing a transaction identifier action. */ public class SetTransaction { public static final StructType FULL_SCHEMA = new StructType() .add("appId", StringType.STRING, false /* nullable */) .add("version", LongType.LONG, false /* nullable*/) .add("lastUpdated", LongType.LONG, true /* nullable*/); public static SetTransaction fromColumnVector(ColumnVector vector, int rowId) { if (vector.isNullAt(rowId)) { return null; } return new SetTransaction( vector.getChild(0).getString(rowId), vector.getChild(1).getLong(rowId), vector.getChild(2).isNullAt(rowId) ? Optional.empty() : Optional.of(vector.getChild(2).getLong(rowId))); } private final String appId; private final long version; private final Optional lastUpdated; public SetTransaction(String appId, Long version, Optional lastUpdated) { this.appId = appId; this.version = version; this.lastUpdated = lastUpdated; } public String getAppId() { return appId; } public long getVersion() { return version; } public Optional getLastUpdated() { return lastUpdated; } /** * Encode as a {@link Row} object with the schema {@link SetTransaction#FULL_SCHEMA}. * * @return {@link Row} object with the schema {@link SetTransaction#FULL_SCHEMA} */ public Row toRow() { Map setTransactionMap = new HashMap<>(); setTransactionMap.put(0, appId); setTransactionMap.put(1, version); setTransactionMap.put(2, lastUpdated.orElse(null)); return new GenericRow(SetTransaction.FULL_SCHEMA, setTransactionMap); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/SingleAction.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.actions; import io.delta.kernel.data.Row; import io.delta.kernel.internal.data.GenericRow; import io.delta.kernel.types.StructType; import java.util.Collections; import java.util.HashMap; import java.util.Map; public class SingleAction { /** * Get the schema of reading entries from Delta Log delta and checkpoint files for construction of * new checkpoint. */ public static StructType CHECKPOINT_SCHEMA = new StructType() .add("txn", SetTransaction.FULL_SCHEMA) .add("add", AddFile.FULL_SCHEMA) .add("remove", RemoveFile.FULL_SCHEMA) .add("metaData", Metadata.FULL_SCHEMA) .add("protocol", Protocol.FULL_SCHEMA) .add("domainMetadata", DomainMetadata.FULL_SCHEMA); // Once we start supporting updating CDC or domain metadata enabled tables, we should add the // schema for those fields here. /** * Schema to use when reading the winning commit files for conflict resolution. This schema is * just for resolving conflicts when doing a blind append. It doesn't cover case when the txn is * reading data from the table and updating the table. */ public static StructType CONFLICT_RESOLUTION_SCHEMA = new StructType() .add("txn", SetTransaction.FULL_SCHEMA) // .add("add", AddFile.FULL_SCHEMA) // not needed for blind appends // .add("remove", RemoveFile.FULL_SCHEMA) // not needed for blind appends .add("metaData", Metadata.FULL_SCHEMA) .add("protocol", Protocol.FULL_SCHEMA) .add("commitInfo", CommitInfo.FULL_SCHEMA) .add("domainMetadata", DomainMetadata.FULL_SCHEMA); // Once we start supporting domain metadata/row tracking enabled tables, we should add the // schema for domain metadata fields here. // Schema to use when writing out the single action to the Delta Log. public static StructType FULL_SCHEMA = new StructType() .add("txn", SetTransaction.FULL_SCHEMA) .add("add", AddFile.FULL_SCHEMA) .add("remove", RemoveFile.FULL_SCHEMA) .add("metaData", Metadata.FULL_SCHEMA) .add("protocol", Protocol.FULL_SCHEMA) .add("cdc", new StructType()) .add("commitInfo", CommitInfo.FULL_SCHEMA) .add("domainMetadata", DomainMetadata.FULL_SCHEMA); // Once we start supporting updating CDC or domain metadata enabled tables, we should add the // schema for those fields here. public static final int TXN_ORDINAL = FULL_SCHEMA.indexOf("txn"); public static final int ADD_FILE_ORDINAL = FULL_SCHEMA.indexOf("add"); public static final int REMOVE_FILE_ORDINAL = FULL_SCHEMA.indexOf("remove"); public static final int METADATA_ORDINAL = FULL_SCHEMA.indexOf("metaData"); public static final int PROTOCOL_ORDINAL = FULL_SCHEMA.indexOf("protocol"); public static final int COMMIT_INFO_ORDINAL = FULL_SCHEMA.indexOf("commitInfo"); private static final int DOMAIN_METADATA_ORDINAL = FULL_SCHEMA.indexOf("domainMetadata"); public static Row createAddFileSingleAction(Row addFile) { Map singleActionValueMap = new HashMap<>(); singleActionValueMap.put(ADD_FILE_ORDINAL, addFile); return new GenericRow(FULL_SCHEMA, singleActionValueMap); } public static Row createProtocolSingleAction(Row protocol) { Map singleActionValueMap = new HashMap<>(); singleActionValueMap.put(PROTOCOL_ORDINAL, protocol); return new GenericRow(FULL_SCHEMA, singleActionValueMap); } public static Row createMetadataSingleAction(Row metadata) { Map singleActionValueMap = new HashMap<>(); singleActionValueMap.put(METADATA_ORDINAL, metadata); return new GenericRow(FULL_SCHEMA, singleActionValueMap); } public static Row createRemoveFileSingleAction(Row remove) { Map singleActionValueMap = new HashMap<>(); singleActionValueMap.put(REMOVE_FILE_ORDINAL, remove); return new GenericRow(FULL_SCHEMA, singleActionValueMap); } public static Row createCommitInfoSingleAction(Row commitInfo) { Map singleActionValueMap = new HashMap<>(); singleActionValueMap.put(COMMIT_INFO_ORDINAL, commitInfo); return new GenericRow(FULL_SCHEMA, singleActionValueMap); } public static Row createDomainMetadataSingleAction(Row domainMetadata) { return new GenericRow( FULL_SCHEMA, Collections.singletonMap(DOMAIN_METADATA_ORDINAL, domainMetadata)); } public static Row createTxnSingleAction(Row txn) { Map singleActionValueMap = new HashMap<>(); singleActionValueMap.put(TXN_ORDINAL, txn); return new GenericRow(FULL_SCHEMA, singleActionValueMap); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/annotation/VisibleForTesting.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.annotation; import java.lang.annotation.*; /** * Indicates that the visibility of a program element (such as a field, method, or class) is * intentionally wider than necessary for testing purposes. This annotation serves as documentation * for developers and tooling, clarifying that the element is not intended for production use but * must be visible for test code. */ @Documented @Retention(RetentionPolicy.CLASS) @Target({ElementType.TYPE, ElementType.FIELD, ElementType.METHOD, ElementType.CONSTRUCTOR}) public @interface VisibleForTesting {} ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/checkpoints/CheckpointInstance.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.checkpoints; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.internal.util.Preconditions; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Optional; // TODO: Delete this in favor of ParsedCheckpointData. /** Metadata about Delta checkpoint. */ public class CheckpointInstance implements Comparable { public enum CheckpointFormat { // Note that the order of these enum values is important for comparison of checkpoint // instances (we prefer V2 > MULTI_PART > CLASSIC). CLASSIC, MULTI_PART, V2; // Indicates that the checkpoint (may) contain SidecarFile actions. For compatibility, // V2 checkpoints can be named with classic-style names, so any checkpoint other than a // multipart checkpoint may contain SidecarFile actions. public boolean usesSidecars() { return this == CLASSIC || this == V2; } } /** Placeholder to identify the version that is always the latest on timeline */ public static final CheckpointInstance MAX_VALUE = new CheckpointInstance(Long.MAX_VALUE); public final long version; public final Optional numParts; public final CheckpointFormat format; public final Optional filePath; // Guaranteed to be present for V2 checkpoints. public CheckpointInstance(String path) { Preconditions.checkArgument( FileNames.isCheckpointFile(path), "not a valid checkpoint file name"); String[] pathParts = getPathName(path).split("\\."); if (pathParts.length == 3 && pathParts[2].equals("parquet")) { // Classic checkpoint 00000000000000000010.checkpoint.parquet this.version = Long.parseLong(pathParts[0]); this.numParts = Optional.empty(); this.format = CheckpointFormat.CLASSIC; this.filePath = Optional.empty(); } else if (pathParts.length == 5 && pathParts[4].equals("parquet")) { // Multi-part checkpoint 00000000000000000010.checkpoint.0000000001.0000000003.parquet this.version = Long.parseLong(pathParts[0]); this.numParts = Optional.of(Integer.parseInt(pathParts[3])); this.format = CheckpointFormat.MULTI_PART; this.filePath = Optional.empty(); } else if (pathParts.length == 4 && (pathParts[3].equals("parquet") || pathParts[3].equals("json"))) { // V2 checkpoint 00000000000000000010.checkpoint.UUID.(parquet|json) this.version = Long.parseLong(pathParts[0]); this.numParts = Optional.empty(); this.format = CheckpointFormat.V2; this.filePath = Optional.of(new Path(path)); } else { throw new RuntimeException("Unrecognized checkpoint path format: " + getPathName(path)); } } public CheckpointInstance(long version) { this(version, Optional.empty()); } public CheckpointInstance(long version, Optional numParts) { this.version = version; this.numParts = numParts; this.filePath = Optional.empty(); if (numParts.orElse(0) == 0) { this.format = CheckpointFormat.CLASSIC; } else { this.format = CheckpointFormat.MULTI_PART; } } boolean isNotLaterThan(CheckpointInstance other) { if (other == CheckpointInstance.MAX_VALUE) { return true; } return version <= other.version; } boolean isEarlierThan(CheckpointInstance other) { if (other == CheckpointInstance.MAX_VALUE) { return true; } return version < other.version; } public List getCorrespondingFiles(Path path) { if (this == CheckpointInstance.MAX_VALUE) { throw new IllegalStateException("Can't get files for CheckpointVersion.MaxValue."); } // This is safe because the only way to construct a V2 CheckpointInstance is with the path. if (format == CheckpointFormat.V2) { return Collections.singletonList(filePath.get()); } return numParts .map(parts -> FileNames.checkpointFileWithParts(path, version, parts)) .orElseGet( () -> Collections.singletonList(FileNames.checkpointFileSingular(path, version))); } /** * Comparison rules: 1. A CheckpointInstance with higher version is greater than the one with * lower version. 2. A CheckpointInstance for a V2 checkpoint is greater than a classic checkpoint * (to filter avoid selecting the compatibility file) or a multipart checkpoint. 3. For * CheckpointInstances with same version, a Multi-part checkpoint is greater than a Single part * checkpoint. 4. For Multi-part CheckpointInstance corresponding to same version, the one with * more parts is greater than the one with fewer parts. 5. For V2 checkpoints, use the file path * to break ties. */ @Override public int compareTo(CheckpointInstance that) { // Compare versions. if (version != that.version) { return Long.compare(version, that.version); } // Compare formats. if (format != that.format) { return Integer.compare(format.ordinal(), that.format.ordinal()); } // Use format-specific tiebreakers if versions and formats are the same. switch (format) { case CLASSIC: return 0; // No way to break ties if both are classic checkpoints. case MULTI_PART: return Long.compare(numParts.orElse(1), that.numParts.orElse(1)); case V2: return filePath.get().getName().compareTo(that.filePath.get().getName()); default: throw new IllegalStateException("Unexpected format: " + format); } } @Override public String toString() { return "CheckpointInstance{version=" + version + ", numParts=" + numParts + ", format=" + format + ", filePath=" + filePath + "}"; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } CheckpointInstance checkpointInstance = (CheckpointInstance) o; return this.compareTo(checkpointInstance) == 0; } @Override public int hashCode() { // For V2 checkpoints, the filepath is included in the hash of the instance (as we consider // different UUID checkpoints to be different checkpoint instances. Otherwise, ignore // the filepath (which is empty) when hashing. return Objects.hash(version, numParts, format, filePath); } private String getPathName(String path) { int slash = path.lastIndexOf("/"); return path.substring(slash + 1); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/checkpoints/CheckpointMetaData.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.checkpoints; import static io.delta.kernel.internal.util.VectorUtils.stringStringMapValue; import static io.delta.kernel.internal.util.VectorUtils.toJavaMap; import io.delta.kernel.data.Row; import io.delta.kernel.internal.data.GenericRow; import io.delta.kernel.types.LongType; import io.delta.kernel.types.MapType; import io.delta.kernel.types.StringType; import io.delta.kernel.types.StructType; import java.util.*; public class CheckpointMetaData { public static CheckpointMetaData fromRow(Row row) { return new CheckpointMetaData( row.getLong(0), row.getLong(1), row.isNullAt(2) ? Optional.empty() : Optional.of(row.getLong(2)), row.isNullAt(3) ? Map.of() : toJavaMap(row.getMap(3))); } public static StructType READ_SCHEMA = new StructType() .add("version", LongType.LONG, false /* nullable */) .add("size", LongType.LONG, false /* nullable */) .add("parts", LongType.LONG) .add("tags", new MapType(StringType.STRING, StringType.STRING, false)); public final long version; public final long size; public final Optional parts; public final Map tags; public CheckpointMetaData(long version, long size, Optional parts) { this(version, size, parts, Map.of()); } public CheckpointMetaData( long version, long size, Optional parts, Map tags) { this.version = version; this.size = size; this.parts = parts; this.tags = tags; } public Row toRow() { Map dataMap = new HashMap<>(); dataMap.put(0, version); dataMap.put(1, size); parts.ifPresent(aLong -> dataMap.put(2, aLong)); if (!tags.isEmpty()) { dataMap.put(3, stringStringMapValue(tags)); } return new GenericRow(READ_SCHEMA, dataMap); } @Override public String toString() { return "CheckpointMetaData{" + "version=" + version + ", size=" + size + ", parts=" + parts + ", tags=" + tags + '}'; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/checkpoints/CheckpointMetadataAction.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.checkpoints; import io.delta.kernel.data.Row; import io.delta.kernel.internal.data.GenericRow; import io.delta.kernel.internal.util.VectorUtils; import io.delta.kernel.types.LongType; import io.delta.kernel.types.MapType; import io.delta.kernel.types.StringType; import io.delta.kernel.types.StructType; import java.util.HashMap; import java.util.Map; /** Action representing a checkpointMetadata action in a top-level V2 checkpoint file */ public class CheckpointMetadataAction { public static final StructType FULL_SCHEMA = new StructType() .add("version", LongType.LONG, false /* nullable */) .add("tags", new MapType(StringType.STRING, StringType.STRING, false /* nullable */)); private final long version; private final Map tags; public CheckpointMetadataAction(long version, Map tags) { this.version = version; this.tags = tags; } public long getVersion() { return version; } public Map getTags() { return tags; } public Row toRow() { Map contentMap = new HashMap<>(); contentMap.put(0, version); contentMap.put(1, VectorUtils.stringStringMapValue(tags)); return new GenericRow(FULL_SCHEMA, contentMap); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/checkpoints/Checkpointer.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.checkpoints; import static io.delta.kernel.internal.DeltaErrors.wrapEngineExceptionThrowsIO; import static io.delta.kernel.internal.TableConfig.EXPIRED_LOG_CLEANUP_ENABLED; import static io.delta.kernel.internal.TableConfig.LOG_RETENTION; import static io.delta.kernel.internal.snapshot.MetadataCleanup.cleanupExpiredLogs; import static io.delta.kernel.internal.tablefeatures.TableFeatures.CHECKPOINT_PROTECTION_W_FEATURE; import static io.delta.kernel.internal.util.Utils.singletonCloseableIterator; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.CheckpointAlreadyExistsException; import io.delta.kernel.exceptions.KernelEngineException; import io.delta.kernel.exceptions.TableNotFoundException; import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.replay.CreateCheckpointIterator; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.internal.util.*; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import java.io.*; import java.nio.file.FileAlreadyExistsException; import java.util.*; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Class to load and write the {@link CheckpointMetaData} from `_last_checkpoint` file. */ public class Checkpointer { //////////////////////////////// // Static variables / methods // //////////////////////////////// private static final Logger logger = LoggerFactory.getLogger(Checkpointer.class); private static final int READ_LAST_CHECKPOINT_FILE_MAX_RETRIES = 3; /** The name of the last checkpoint file */ public static final String LAST_CHECKPOINT_FILE_NAME = "_last_checkpoint"; public static void checkpoint(Engine engine, Clock clock, SnapshotImpl snapshot) throws TableNotFoundException, IOException { final Path tablePath = snapshot.getDataPath(); final Path logPath = snapshot.getLogPath(); final long version = snapshot.getVersion(); logger.info("{}: Starting checkpoint for version: {}", tablePath, version); // Check if writing to the given table protocol version/features is supported in Kernel TableFeatures.validateKernelCanWriteToTable( snapshot.getProtocol(), snapshot.getMetadata(), snapshot.getDataPath().toString()); final Path checkpointPath = FileNames.checkpointFileSingular(logPath, version); long numberOfAddFiles = 0; try (CreateCheckpointIterator checkpointDataIter = snapshot.getCreateCheckpointIterator(engine)) { // Write the iterator actions to the checkpoint using the Parquet handler wrapEngineExceptionThrowsIO( () -> { engine .getParquetHandler() .writeParquetFileAtomically(checkpointPath.toString(), checkpointDataIter); logger.info("{}: Finished writing checkpoint file for version: {}", tablePath, version); return null; }, "Writing checkpoint file %s", checkpointPath.toString()); // Get the metadata of the checkpoint file numberOfAddFiles = checkpointDataIter.getNumberOfAddActions(); } catch (FileAlreadyExistsException faee) { throw new CheckpointAlreadyExistsException(version); } catch (IOException io) { if (io.getCause() instanceof FileAlreadyExistsException) { throw new CheckpointAlreadyExistsException(version); } throw io; } final CheckpointMetaData checkpointMetaData = new CheckpointMetaData(version, numberOfAddFiles, Optional.empty()); new Checkpointer(logPath).writeLastCheckpointFile(engine, checkpointMetaData); logger.info( "{}: Finished writing last checkpoint metadata file for version: {}", tablePath, version); final Metadata metadata = snapshot.getMetadata(); if (shouldPerformLogCleanup(snapshot)) { cleanupExpiredLogs(engine, clock, tablePath, LOG_RETENTION.fromMetadata(metadata)); } else { logger.info( "{}: Log cleanup is disabled. Skipping the deletion of expired log files", tablePath); } } /** * Only clean up expired log files when: * *
    *
  • Snapshot was built as "latest" by intent (not time-traveled) *
  • checkpointProtection feature is not enabled *
  • delta.enableExpiredLogCleanup table property is set to true *
*/ private static boolean shouldPerformLogCleanup(SnapshotImpl snapshot) { final boolean hasCheckpointProtection = snapshot .getProtocol() .getWriterFeatures() .contains(CHECKPOINT_PROTECTION_W_FEATURE.featureName()); return snapshot.wasBuiltAsLatest() && EXPIRED_LOG_CLEANUP_ENABLED.fromMetadata(snapshot.getMetadata()) && !hasCheckpointProtection; } /** * Given a list of checkpoint files, pick the latest complete checkpoint instance which is not * later than `notLaterThan`. */ public static Optional getLatestCompleteCheckpointFromList( List instances, CheckpointInstance notLaterThan) { final List completeCheckpoints = instances.stream() .filter(c -> c.isNotLaterThan(notLaterThan)) .collect(Collectors.groupingBy(c -> c)) .entrySet() .stream() .filter( entry -> { final CheckpointInstance key = entry.getKey(); final List inst = entry.getValue(); if (key.numParts.isPresent()) { return inst.size() == entry.getKey().numParts.get(); } else { return inst.size() == 1; } }) .map(Map.Entry::getKey) .collect(Collectors.toList()); if (completeCheckpoints.isEmpty()) { return Optional.empty(); } else { return Optional.of(Collections.max(completeCheckpoints)); } } /** Find the last complete checkpoint before (strictly less than) a given version. */ public static Optional findLastCompleteCheckpointBefore( Engine engine, Path tableLogPath, long version) { return findLastCompleteCheckpointBeforeHelper(engine, tableLogPath, version)._1; } /** * Helper method for `findLastCompleteCheckpointBefore` which also return the number of files * searched. This helps in testing */ public static Tuple2, Long> findLastCompleteCheckpointBeforeHelper( Engine engine, Path tableLogPath, long version) { CheckpointInstance upperBoundCheckpoint = new CheckpointInstance(version); logger.info("Try to find the last complete checkpoint before version {}", version); // This is a just a tracker for testing purposes long numberOfFilesSearched = 0; long currentVersion = version; // Some cloud storage APIs make a calls to fetch 1000 at a time. // To make use of that observation and to avoid making more listing calls than // necessary, list 1000 at a time (backwards from the given version). Search // within that list if a checkpoint is found. If found stop, otherwise list the previous // 1000 entries. Repeat until a checkpoint is found or there are no more delta commits. while (currentVersion >= 0) { try { long searchLowerBound = Math.max(0, currentVersion - 1000); CloseableIterator deltaLogFileIter = wrapEngineExceptionThrowsIO( () -> engine .getFileSystemClient() .listFrom(FileNames.listingPrefix(tableLogPath, searchLowerBound)), "Listing from %s", FileNames.listingPrefix(tableLogPath, searchLowerBound)); List checkpoints = new ArrayList<>(); while (deltaLogFileIter.hasNext()) { FileStatus fileStatus = deltaLogFileIter.next(); String fileName = new Path(fileStatus.getPath()).getName(); long currentFileVersion; if (FileNames.isCommitFile(fileName)) { currentFileVersion = FileNames.deltaVersion(fileName); } else if (FileNames.isCheckpointFile(fileName)) { currentFileVersion = FileNames.checkpointVersion(fileName); } else { // allow all other types of files. currentFileVersion = currentVersion; } boolean shouldContinue = // only consider files with version in the range and // before the target version (currentVersion == 0 || currentFileVersion <= currentVersion) && currentFileVersion < version; if (!shouldContinue) { break; } if (validCheckpointFile(fileStatus)) { checkpoints.add(new CheckpointInstance(fileStatus.getPath())); } numberOfFilesSearched++; } Optional latestCheckpoint = getLatestCompleteCheckpointFromList(checkpoints, upperBoundCheckpoint); if (latestCheckpoint.isPresent()) { logger.info( "Found the last complete checkpoint before version {} at {}", version, latestCheckpoint.get()); return new Tuple2<>(latestCheckpoint, numberOfFilesSearched); } currentVersion -= 1000; // search for checkpoint in previous 1000 entries } catch (IOException e) { String msg = String.format( "Failed to list checkpoint files for version %s in %s.", version, tableLogPath); logger.warn(msg, e); return new Tuple2<>(Optional.empty(), numberOfFilesSearched); } } logger.info("No complete checkpoint found before version {} in {}", version, tableLogPath); return new Tuple2<>(Optional.empty(), numberOfFilesSearched); } private static boolean validCheckpointFile(FileStatus fileStatus) { return FileNames.isCheckpointFile(new Path(fileStatus.getPath()).getName()) && fileStatus.getSize() > 0; } //////////////////////////////// // Member variables / methods // //////////////////////////////// /** The path to the file that holds metadata about the most recent checkpoint. */ private final Path lastCheckpointFilePath; public Checkpointer(Path logPath) { this.lastCheckpointFilePath = new Path(logPath, LAST_CHECKPOINT_FILE_NAME); } /** Returns information about the most recent checkpoint. */ public Optional readLastCheckpointFile(Engine engine) { return loadMetadataFromFile(engine, 0 /* tries */); } /** * Write the given data to last checkpoint metadata file. * * @param engine {@link Engine} instance to use for writing * @param checkpointMetaData Checkpoint metadata to write * @throws IOException For any I/O issues. */ public void writeLastCheckpointFile(Engine engine, CheckpointMetaData checkpointMetaData) throws IOException { wrapEngineExceptionThrowsIO( () -> { engine .getJsonHandler() .writeJsonFileAtomically( lastCheckpointFilePath.toString(), singletonCloseableIterator(checkpointMetaData.toRow()), true /* overwrite */); return null; }, "Writing last checkpoint file at `%s`", lastCheckpointFilePath); } /** * Loads the checkpoint metadata from the _last_checkpoint file. * * @param engine {@link Engine instance to use} * @param tries Number of times already tried to load the metadata before this call. */ private Optional loadMetadataFromFile(Engine engine, int tries) { if (tries >= READ_LAST_CHECKPOINT_FILE_MAX_RETRIES) { // We have tried 3 times and failed. Assume the checkpoint metadata file is corrupt. logger.warn( "Failed to load checkpoint metadata from file {} after {} attempts.", lastCheckpointFilePath, READ_LAST_CHECKPOINT_FILE_MAX_RETRIES); return Optional.empty(); } logger.info( "Loading last checkpoint from the _last_checkpoint file. Attempt: {} / {}", tries + 1, READ_LAST_CHECKPOINT_FILE_MAX_RETRIES); try { // Use arbitrary values for size and mod time as they are not available. // We could list and find the values, but it is an unnecessary FS call. FileStatus lastCheckpointFile = FileStatus.of(lastCheckpointFilePath.toString(), 0 /* size */, 0 /* modTime */); try (CloseableIterator jsonIter = wrapEngineExceptionThrowsIO( () -> engine .getJsonHandler() .readJsonFiles( singletonCloseableIterator(lastCheckpointFile), CheckpointMetaData.READ_SCHEMA, Optional.empty()), "Reading the last checkpoint file as JSON")) { Optional checkpointRow = InternalUtils.getSingularRow(jsonIter); if (checkpointRow.isPresent()) { return Optional.of(CheckpointMetaData.fromRow(checkpointRow.get())); } // Checkpoint has no data. This is a valid case on some file systems where the // contents are not visible until the file stream is closed. // Sleep for one second and retry. logger.warn( "Last checkpoint file {} has no data. " + "Retrying after 1sec. (current attempt = {})", lastCheckpointFilePath, tries); try { Thread.sleep(1000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return Optional.empty(); } return loadMetadataFromFile(engine, tries + 1); } } catch (Exception e) { if (e instanceof FileNotFoundException || (e instanceof KernelEngineException && e.getCause() instanceof FileNotFoundException)) { return Optional.empty(); // there's no point in retrying } String msg = String.format( "Failed to load checkpoint metadata from file %s. " + "It must be in the process of being written. " + "Retrying after 1sec. (current attempt of %s (max 3)", lastCheckpointFilePath, tries); logger.warn(msg, e); // we can retry until max tries are exhausted. It saves latency as the alternative // is to list files and find the last checkpoint file. And the `_last_checkpoint` // file is possibly being written to. return loadMetadataFromFile(engine, tries + 1); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/checkpoints/SidecarFile.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.checkpoints; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.Row; import io.delta.kernel.internal.data.GenericRow; import io.delta.kernel.types.LongType; import io.delta.kernel.types.StringType; import io.delta.kernel.types.StructType; import java.util.HashMap; import java.util.Map; /** Action representing a SidecarFile in a top-level V2 checkpoint file. */ public class SidecarFile { public static StructType READ_SCHEMA = new StructType() .add("path", StringType.STRING, false /* nullable */) .add("sizeInBytes", LongType.LONG, false /* nullable */) .add("modificationTime", LongType.LONG, false /* nullable */); public static SidecarFile fromColumnVector(ColumnVector vector, int rowIndex) { if (vector.isNullAt(rowIndex)) { return null; } return new SidecarFile( vector.getChild(0).getString(rowIndex), vector.getChild(1).getLong(rowIndex), vector.getChild(2).getLong(rowIndex)); } private final String path; private final long sizeInBytes; private final long modificationTime; public SidecarFile(String path, long sizeInBytes, long modificationTime) { this.path = path; this.sizeInBytes = sizeInBytes; this.modificationTime = modificationTime; } public String getPath() { return path; } public long getSizeInBytes() { return sizeInBytes; } public long getModificationTime() { return modificationTime; } public Row toRow() { Map dataMap = new HashMap<>(); dataMap.put(0, path); dataMap.put(1, sizeInBytes); dataMap.put(2, modificationTime); return new GenericRow(READ_SCHEMA, dataMap); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/CRCInfo.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.checksum; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.internal.actions.DomainMetadata; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.data.GenericRow; import io.delta.kernel.internal.data.StructRow; import io.delta.kernel.internal.stats.FileSizeHistogram; import io.delta.kernel.internal.util.InternalUtils; import io.delta.kernel.internal.util.VectorUtils; import io.delta.kernel.types.ArrayType; import io.delta.kernel.types.LongType; import io.delta.kernel.types.StringType; import io.delta.kernel.types.StructType; import java.util.*; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class CRCInfo { private static final Logger logger = LoggerFactory.getLogger(CRCInfo.class); // Constants for schema field names private static final String TABLE_SIZE_BYTES = "tableSizeBytes"; private static final String NUM_FILES = "numFiles"; private static final String NUM_METADATA = "numMetadata"; private static final String NUM_PROTOCOL = "numProtocol"; private static final String METADATA = "metadata"; private static final String PROTOCOL = "protocol"; private static final String TXN_ID = "txnId"; private static final String DOMAIN_METADATA = "domainMetadata"; private static final String FILE_SIZE_HISTOGRAM = "fileSizeHistogram"; private static final String HISTOGRAM_OPT = "histogramOpt"; public static final StructType CRC_FILE_SCHEMA = new StructType() .add(TABLE_SIZE_BYTES, LongType.LONG) .add(NUM_FILES, LongType.LONG) .add(NUM_METADATA, LongType.LONG) .add(NUM_PROTOCOL, LongType.LONG) .add(METADATA, Metadata.FULL_SCHEMA) .add(PROTOCOL, Protocol.FULL_SCHEMA) .add(TXN_ID, StringType.STRING, /*nullable*/ true) .add(DOMAIN_METADATA, new ArrayType(DomainMetadata.FULL_SCHEMA, false), /*nullable*/ true) .add(FILE_SIZE_HISTOGRAM, FileSizeHistogram.FULL_SCHEMA, /*nullable*/ true); // Used by ChecksumReader to support reading CRC files with the legacy "histogramOpt" field. public static final StructType CRC_FILE_READ_SCHEMA = CRC_FILE_SCHEMA.add(HISTOGRAM_OPT, FileSizeHistogram.FULL_SCHEMA, /*nullable*/ true); public static Optional fromColumnarBatch( long version, ColumnarBatch batch, int rowId, String crcFilePath) { // Read required fields. Protocol protocol = Protocol.fromColumnVector(batch.getColumnVector(getSchemaIndex(PROTOCOL)), rowId); Metadata metadata = Metadata.fromColumnVector(batch.getColumnVector(getSchemaIndex(METADATA)), rowId); long tableSizeBytes = InternalUtils.requireNonNull( batch.getColumnVector(getSchemaIndex(TABLE_SIZE_BYTES)), rowId, TABLE_SIZE_BYTES) .getLong(rowId); long numFiles = InternalUtils.requireNonNull( batch.getColumnVector(getSchemaIndex(NUM_FILES)), rowId, NUM_FILES) .getLong(rowId); // Read optional fields ColumnVector txnIdColumnVector = batch.getColumnVector(getSchemaIndex(TXN_ID)); Optional txnId = txnIdColumnVector.isNullAt(rowId) ? Optional.empty() : Optional.of(txnIdColumnVector.getString(rowId)); Optional fileSizeHistogram = FileSizeHistogram.fromColumnVector( batch.getColumnVector(getSchemaIndex(FILE_SIZE_HISTOGRAM)), rowId); if (!fileSizeHistogram.isPresent()) { int histogramOptIdx = batch.getSchema().indexOf(HISTOGRAM_OPT); if (histogramOptIdx >= 0) { fileSizeHistogram = FileSizeHistogram.fromColumnVector(batch.getColumnVector(histogramOptIdx), rowId); } } ColumnVector domainMetadataVector = batch.getColumnVector(getSchemaIndex(DOMAIN_METADATA)); Optional> domainMetadata = domainMetadataVector.isNullAt(rowId) ? Optional.empty() : Optional.of( VectorUtils.toJavaList(domainMetadataVector.getArray(rowId)).stream() .map(row -> DomainMetadata.fromRow((StructRow) row)) .collect(Collectors.toSet())); // protocol and metadata are nullable per fromColumnVector's implementation. if (protocol == null || metadata == null) { logger.warn("Invalid checksum file missing protocol and/or metadata: {}", crcFilePath); return Optional.empty(); } return Optional.of( new CRCInfo( version, metadata, protocol, tableSizeBytes, numFiles, txnId, domainMetadata, fileSizeHistogram)); } private final long version; private final Metadata metadata; private final Protocol protocol; private final long tableSizeBytes; private final long numFiles; private final Optional txnId; private final Optional> domainMetadata; private final Optional fileSizeHistogram; public CRCInfo( long version, Metadata metadata, Protocol protocol, long tableSizeBytes, long numFiles, Optional txnId, Optional> domainMetadata, Optional fileSizeHistogram) { checkArgument(tableSizeBytes >= 0); checkArgument(numFiles >= 0); // Live Domain Metadata actions at this version, excluding tombstones. this.domainMetadata = requireNonNull(domainMetadata); domainMetadata.ifPresent( dms -> dms.forEach( dm -> checkArgument( !dm.isRemoved(), String.format( "Domain metadata in CRC should exclude tombstones, " + "found removed domain metadata: %s.", dm.getDomain())))); this.version = version; this.metadata = requireNonNull(metadata); this.protocol = requireNonNull(protocol); this.tableSizeBytes = tableSizeBytes; this.numFiles = numFiles; this.txnId = requireNonNull(txnId); this.fileSizeHistogram = requireNonNull(fileSizeHistogram); } /** The version of the Delta table that this CRCInfo represents. */ public long getVersion() { return version; } /** The {@link Metadata} stored in this CRCInfo. */ public Metadata getMetadata() { return metadata; } /** The {@link Protocol} stored in this CRCInfo. */ public Protocol getProtocol() { return protocol; } public long getNumFiles() { return numFiles; } public long getTableSizeBytes() { return tableSizeBytes; } public Optional getTxnId() { return txnId; } public Optional> getDomainMetadata() { return domainMetadata; } /** The {@link FileSizeHistogram} stored in this CRCInfo. */ public Optional getFileSizeHistogram() { return fileSizeHistogram; } /** * Encode as a {@link Row} object with the schema {@link CRCInfo#CRC_FILE_SCHEMA}. * * @return {@link Row} object with the schema {@link CRCInfo#CRC_FILE_SCHEMA} */ public Row toRow() { Map values = new HashMap<>(); // Add required fields values.put(getSchemaIndex(TABLE_SIZE_BYTES), tableSizeBytes); values.put(getSchemaIndex(NUM_FILES), numFiles); values.put(getSchemaIndex(NUM_METADATA), 1L); values.put(getSchemaIndex(NUM_PROTOCOL), 1L); values.put(getSchemaIndex(METADATA), metadata.toRow()); values.put(getSchemaIndex(PROTOCOL), protocol.toRow()); // Add optional fields txnId.ifPresent(txn -> values.put(getSchemaIndex(TXN_ID), txn)); domainMetadata.ifPresent( domainMetadataSet -> values.put( getSchemaIndex(DOMAIN_METADATA), VectorUtils.buildArrayValue( domainMetadataSet.stream() .map(DomainMetadata::toRow) .collect(Collectors.toList()), DomainMetadata.FULL_SCHEMA))); fileSizeHistogram.ifPresent( fileSizeHistogram -> values.put(getSchemaIndex(FILE_SIZE_HISTOGRAM), fileSizeHistogram.toRow())); return new GenericRow(CRC_FILE_SCHEMA, values); } @Override public int hashCode() { return Objects.hash( version, metadata, protocol, tableSizeBytes, numFiles, txnId, domainMetadata, fileSizeHistogram); } @Override public boolean equals(Object o) { if (!(o instanceof CRCInfo)) { return false; } CRCInfo other = (CRCInfo) o; return version == other.version && tableSizeBytes == other.tableSizeBytes && numFiles == other.numFiles && metadata.equals(other.metadata) && protocol.equals(other.protocol) && txnId.equals(other.txnId) && domainMetadata.equals(other.domainMetadata) && fileSizeHistogram.equals(other.fileSizeHistogram); } private static int getSchemaIndex(String fieldName) { return CRC_FILE_SCHEMA.indexOf(fieldName); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/ChecksumReader.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.checksum; import static io.delta.kernel.internal.util.Utils.singletonCloseableIterator; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Utility method to load protocol and metadata from the Delta log checksum files. */ public class ChecksumReader { private static final Logger logger = LoggerFactory.getLogger(ChecksumReader.class); /** * Load the CRCInfo from the provided checksum file. * * @param engine the engine to use for reading the checksum file * @param checkSumFile the file status of the checksum file to read * @return Optional {@link CRCInfo} containing the information included in the checksum file, such * as protocol, metadata. */ public static Optional tryReadChecksumFile(Engine engine, FileStatus checkSumFile) { try (CloseableIterator iter = engine .getJsonHandler() .readJsonFiles( singletonCloseableIterator(checkSumFile), CRCInfo.CRC_FILE_READ_SCHEMA, Optional.empty())) { // We do this instead of iterating through the rows or using `getSingularRow` so we // can use the existing fromColumnVector methods in Protocol, Metadata, Format etc if (!iter.hasNext()) { logger.warn("Checksum file is empty: {}", checkSumFile.getPath()); return Optional.empty(); } ColumnarBatch batch = iter.next(); if (batch.getSize() != 1) { String msg = "Expected exactly one row in the checksum file {}, found {} rows"; logger.warn(msg, checkSumFile.getPath(), batch.getSize()); return Optional.empty(); } long crcVersion = FileNames.checksumVersion(new Path(checkSumFile.getPath())); return CRCInfo.fromColumnarBatch(crcVersion, batch, 0 /* rowId */, checkSumFile.getPath()); } catch (Exception e) { // This can happen when the version does not have a checksum file logger.warn("Failed to read checksum file {}", checkSumFile.getPath(), e); return Optional.empty(); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/ChecksumUtils.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.checksum; import static io.delta.kernel.internal.actions.SingleAction.CHECKPOINT_SCHEMA; import static io.delta.kernel.internal.util.Preconditions.checkState; import static java.util.Objects.requireNonNull; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.actions.*; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.replay.ActionWrapper; import io.delta.kernel.internal.replay.ActionsIterator; import io.delta.kernel.internal.replay.CreateCheckpointIterator; import io.delta.kernel.internal.snapshot.LogSegment; import io.delta.kernel.internal.stats.FileSizeHistogram; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import java.io.IOException; import java.time.Instant; import java.util.*; import java.util.concurrent.atomic.LongAdder; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Utility methods for computing and writing checksums for Delta tables. */ public class ChecksumUtils { private ChecksumUtils() {} private static final Logger logger = LoggerFactory.getLogger(ChecksumUtils.class); private static final int PROTOCOL_INDEX = CHECKPOINT_SCHEMA.indexOf("protocol"); private static final int METADATA_INDEX = CHECKPOINT_SCHEMA.indexOf("metaData"); // TODO: simplify the schema to only read size from addFile. private static final int ADD_INDEX = CHECKPOINT_SCHEMA.indexOf("add"); private static final int REMOVE_INDEX = CHECKPOINT_SCHEMA.indexOf("remove"); private static final int DOMAIN_METADATA_INDEX = CHECKPOINT_SCHEMA.indexOf("domainMetadata"); private static final int ADD_SIZE_INDEX = AddFile.FULL_SCHEMA.indexOf("size"); private static final int REMOVE_SIZE_INDEX = RemoveFile.FULL_SCHEMA.indexOf("size"); // commitInfo is appended after CHECKPOINT_SCHEMA in incremental read private static final int COMMIT_INFO_INDEX = CHECKPOINT_SCHEMA.length(); private static final Set INCREMENTAL_SUPPORTED_OPS = Collections.unmodifiableSet( new HashSet<>( Arrays.asList( "WRITE", "MERGE", "UPDATE", "DELETE", "OPTIMIZE", "CREATE TABLE", "REPLACE TABLE", "CREATE TABLE AS SELECT", "REPLACE TABLE AS SELECT", "CREATE OR REPLACE TABLE AS SELECT"))); /** * Computes the state of a Delta table and writes a checksum file for the provided snapshot's * version. If a checksum file already exists for this version, this method returns without any * changes. * *

The checksum file contains table statistics including: * *

    *
  • Total table size in bytes *
  • Total number of files *
  • File size histogram *
  • Domain metadata information *
* *

Note: For very large tables, this operation may be expensive as it requires scanning the * table state to compute statistics. * * @param engine The Engine instance used to access the underlying storage * @param logSegmentAtVersion The LogSegment instance of the table at a specific version * @throws IOException If an I/O error occurs during checksum computation or writing */ public static void computeStateAndWriteChecksum(Engine engine, LogSegment logSegmentAtVersion) throws IOException { requireNonNull(engine); requireNonNull(logSegmentAtVersion); // Check for existing checksum for this version Optional lastSeenCrcVersion = logSegmentAtVersion .getLastSeenChecksum() .map(file -> FileNames.getFileVersion(new Path(file.getPath()))); if (lastSeenCrcVersion.isPresent() && lastSeenCrcVersion.get().equals(logSegmentAtVersion.getVersion())) { logger.info("Checksum file already exists for version {}", logSegmentAtVersion.getVersion()); return; } Optional lastSeenCrcInfo = logSegmentAtVersion .getLastSeenChecksum() .flatMap(file -> ChecksumReader.tryReadChecksumFile(engine, file)); // Try to build CRC incrementally if possible Optional incrementallyBuiltCrc = lastSeenCrcInfo.isPresent() ? buildCrcInfoIncrementally(lastSeenCrcInfo.get(), engine, logSegmentAtVersion) : Optional.empty(); // Use incrementally built CRC if available, otherwise do full log replay CRCInfo crcInfo = incrementallyBuiltCrc.isPresent() ? incrementallyBuiltCrc.get() : buildCrcInfoWithFullLogReplay(engine, logSegmentAtVersion); ChecksumWriter checksumWriter = new ChecksumWriter(logSegmentAtVersion.getLogPath()); checksumWriter.writeCheckSum(engine, crcInfo); } /** * Builds CRC info by replaying the full log. * * @param engine The engine instance * @param logSegmentAtVersion The log segment at the target version * @return The complete CRC info */ private static CRCInfo buildCrcInfoWithFullLogReplay( Engine engine, LogSegment logSegmentAtVersion) throws IOException { StateTracker state = new StateTracker(); // Process logs and update state try (CreateCheckpointIterator checkpointIterator = new CreateCheckpointIterator( // Set minFileRetentionTimestampMillis to infinite future to skip all removed files engine, logSegmentAtVersion, Instant.ofEpochMilli(Long.MAX_VALUE).toEpochMilli())) { // Process all checkpoint batches while (checkpointIterator.hasNext()) { FilteredColumnarBatch filteredBatch = checkpointIterator.next(); ColumnarBatch batch = filteredBatch.getData(); Optional selectionVector = filteredBatch.getSelectionVector(); final int rowCount = batch.getSize(); ColumnVector metadataVector = batch.getColumnVector(METADATA_INDEX); ColumnVector protocolVector = batch.getColumnVector(PROTOCOL_INDEX); ColumnVector removeVector = batch.getColumnVector(REMOVE_INDEX); ColumnVector addVector = batch.getColumnVector(ADD_INDEX); ColumnVector domainMetadataVector = batch.getColumnVector(DOMAIN_METADATA_INDEX); // Process all selected rows in a single pass for optimal performance for (int i = 0; i < rowCount; i++) { // Fields referenced in the lambda should be effectively final. int rowId = i; boolean isSelected = selectionVector .map(vec -> !vec.isNullAt(rowId) && vec.getBoolean(rowId)) .orElse(true); if (!isSelected) continue; // Step 1: Ensure there are no remove records // We set minFileRetentionTimestampMillis to infinite future to skip all removed files, // so there should be no remove actions. checkState( removeVector.isNullAt(i), "unexpected remove row found when " + "setting minFileRetentionTimestampMillis to infinite future"); // Step 2: Process add files, domain metadata, metadata, and protocol processAddRecord(addVector, state, i); processDomainMetadataRecord(domainMetadataVector, state, i); processMetadataRecord(metadataVector, state, i); processProtocolRecord(protocolVector, state, i); } } } // Get final metadata and protocol Metadata finalMetadata = state.metadataFromLog.orElseThrow(() -> new IllegalStateException("No metadata found")); Protocol finalProtocol = state.protocolFromLog.orElseThrow(() -> new IllegalStateException("No protocol found")); // Filter to only non-removed domain metadata Set finalDomainMetadata = getNonRemovedDomainMetadata(state); return new CRCInfo( logSegmentAtVersion.getVersion(), finalMetadata, finalProtocol, state.tableSizeByte.longValue(), state.fileCount.longValue(), Optional.empty(), Optional.of(finalDomainMetadata), Optional.of(state.addedFileSizeHistogram)); } /** * Attempts to build CRC info incrementally from the last seen checksum. Falls back if incremental * computation is not possible. * * @param lastSeenCrcInfo The last available CRC info to build upon * @param engine The engine to use for file operations * @param logSegment The log segment to process * @return Optional containing the new CRC info, or empty if fallback is needed */ private static Optional buildCrcInfoIncrementally( CRCInfo lastSeenCrcInfo, Engine engine, LogSegment logSegment) throws IOException { long startTime = System.currentTimeMillis(); // Can only build incrementally if we have domain metadata and file size histogram if (!lastSeenCrcInfo.getDomainMetadata().isPresent()) { logger.info( "Falling back to full replay after {}ms: detected current crc missing domain metadata.", System.currentTimeMillis() - startTime); return Optional.empty(); } if (!lastSeenCrcInfo.getFileSizeHistogram().isPresent()) { logger.info( "Falling back to full replay after {}ms: " + "detected current crc missing file size histogram.", System.currentTimeMillis() - startTime); return Optional.empty(); } // Initialize state tracking StateTracker state = new StateTracker(); // TODO: use compacted logs. List deltaFiles = logSegment.getDeltas().stream() .filter( file -> FileNames.getFileVersion(new Path(file.getPath())) > lastSeenCrcInfo.getVersion()) .sorted( Comparator.comparingLong( (FileStatus file) -> FileNames.getFileVersion(new Path(file.getPath())))) .collect(Collectors.toList()); validateDeltaContinuity(deltaFiles, lastSeenCrcInfo.getVersion()); Collections.reverse(deltaFiles); // Create iterator for delta files newer than last CRC StructType readSchema = CHECKPOINT_SCHEMA.add("commitInfo", CommitInfo.FULL_SCHEMA); try (CloseableIterator iterator = new ActionsIterator(engine, deltaFiles, readSchema, java.util.Optional.empty())) { Optional lastSeenVersion = Optional.empty(); while (iterator.hasNext()) { ActionWrapper currentAction = iterator.next(); ColumnarBatch batch = currentAction.getColumnarBatch(); final int rowCount = batch.getSize(); if (rowCount == 0) { continue; } ColumnVector addVector = batch.getColumnVector(ADD_INDEX); ColumnVector removeVector = batch.getColumnVector(REMOVE_INDEX); ColumnVector metadataVector = batch.getColumnVector(METADATA_INDEX); ColumnVector protocolVector = batch.getColumnVector(PROTOCOL_INDEX); ColumnVector domainMetadataVector = batch.getColumnVector(DOMAIN_METADATA_INDEX); ColumnVector commitInfoVector = batch.getColumnVector(COMMIT_INFO_INDEX); for (int i = 0; i < rowCount; i++) { long newVersion = currentAction.getVersion(); // Detect version change if (!lastSeenVersion.isPresent() || newVersion != lastSeenVersion.get()) { // New version detected - current row must be commit info if (commitInfoVector.isNullAt(i)) { logger.info( "Falling back to full replay: first row of version {} is not commit info", newVersion); return Optional.empty(); } CommitInfo commitInfo = CommitInfo.fromColumnVector(commitInfoVector, i); if (commitInfo == null || !commitInfo .getOperation() .filter(INCREMENTAL_SUPPORTED_OPS::contains) .isPresent()) { logger.info( "Falling back to full replay after {}ms: " + "unsupported operation '{}' for version {}", System.currentTimeMillis() - startTime, commitInfo != null ? commitInfo.getOperation().orElse("null") : "null", newVersion); return Optional.empty(); } lastSeenVersion = Optional.of(newVersion); continue; } // Process the row if (!addVector.isNullAt(i)) { processAddRecord(addVector, state, i); } // Process remove file records if (!removeVector.isNullAt(i)) { ColumnVector sizeVector = removeVector.getChild(REMOVE_SIZE_INDEX); if (sizeVector.isNullAt(i)) { logger.info( "Falling back to full replay after {}ms: " + "detected remove without file size in version {}", System.currentTimeMillis() - startTime, newVersion); return Optional.empty(); } long fileSize = sizeVector.getLong(i); state.tableSizeByte.add(-fileSize); state.removedFileSizeHistogram.insert(fileSize); state.fileCount.decrement(); } // Process domain metadata, protocol, and metadata processDomainMetadataRecord(domainMetadataVector, state, i); processMetadataRecord(metadataVector, state, i); processProtocolRecord(protocolVector, state, i); } } } // Merge with existing domain metadata lastSeenCrcInfo .getDomainMetadata() .get() .forEach( dm -> { if (!state.domainMetadataMap.containsKey(dm.getDomain())) { state.domainMetadataMap.put(dm.getDomain(), dm); } }); // Filter to only non-removed domain metadata Set finalDomainMetadata = getNonRemovedDomainMetadata(state); logger.info( "Successfully completed incremental CRC computation in {} ms", System.currentTimeMillis() - startTime); // Build and return the new CRC info return Optional.of( new CRCInfo( logSegment.getVersion(), state.metadataFromLog.orElseGet(lastSeenCrcInfo::getMetadata), state.protocolFromLog.orElseGet(lastSeenCrcInfo::getProtocol), state.tableSizeByte.longValue() + lastSeenCrcInfo.getTableSizeBytes(), state.fileCount.longValue() + lastSeenCrcInfo.getNumFiles(), Optional.empty(), Optional.of(finalDomainMetadata), Optional.of( state .addedFileSizeHistogram .plus(lastSeenCrcInfo.getFileSizeHistogram().get()) .minus(state.removedFileSizeHistogram)))); } /** Processes an add file record and updates the state tracker. */ private static void processAddRecord(ColumnVector addVector, StateTracker state, int rowId) { if (!addVector.isNullAt(rowId)) { ColumnVector sizeVector = addVector.getChild(ADD_SIZE_INDEX); checkState(!sizeVector.isNullAt(rowId), "Add record has null file size"); long fileSize = sizeVector.getLong(rowId); checkState(fileSize >= 0, "Add record has negative file size: " + fileSize); state.tableSizeByte.add(fileSize); state.addedFileSizeHistogram.insert(fileSize); state.fileCount.increment(); } } /** Processes a domain metadata record and updates the state tracker. */ private static void processDomainMetadataRecord( ColumnVector domainMetadataVector, StateTracker state, int rowId) { if (!domainMetadataVector.isNullAt(rowId)) { DomainMetadata domainMetadata = DomainMetadata.fromColumnVector(domainMetadataVector, rowId); if (!state.domainMetadataMap.containsKey(domainMetadata.getDomain())) { state.domainMetadataMap.put(domainMetadata.getDomain(), domainMetadata); } } } /** Processes a metadata record and updates the state tracker. */ private static void processMetadataRecord( ColumnVector metadataVector, StateTracker state, int rowId) { if (!metadataVector.isNullAt(rowId) && !state.metadataFromLog.isPresent()) { Metadata metadata = Metadata.fromColumnVector(metadataVector, rowId); checkState(metadata != null, "Metadata is null"); state.metadataFromLog = Optional.of(metadata); } } /** Processes a protocol record and updates the state tracker. */ private static void processProtocolRecord( ColumnVector protocolVector, StateTracker state, int rowId) { if (!protocolVector.isNullAt(rowId) && !state.protocolFromLog.isPresent()) { Protocol protocol = Protocol.fromColumnVector(protocolVector, rowId); checkState(protocol != null, "Protocol is null"); state.protocolFromLog = Optional.of(protocol); } } /** Get non-removed domain metadata. */ private static Set getNonRemovedDomainMetadata(StateTracker state) { return state.domainMetadataMap.values().stream() .filter(dm -> !dm.isRemoved()) .collect(Collectors.toSet()); } /** Class for tracking state during log processing. */ private static class StateTracker { Optional metadataFromLog = Optional.empty(); Optional protocolFromLog = Optional.empty(); LongAdder tableSizeByte = new LongAdder(); LongAdder fileCount = new LongAdder(); FileSizeHistogram addedFileSizeHistogram = FileSizeHistogram.createDefaultHistogram(); FileSizeHistogram removedFileSizeHistogram = FileSizeHistogram.createDefaultHistogram(); Map domainMetadataMap = new HashMap<>(); } private static void validateDeltaContinuity(List deltas, long checksumVersion) { if (deltas.isEmpty()) { return; } long expectedVersion = checksumVersion + 1; for (FileStatus delta : deltas) { long version = FileNames.getFileVersion(new Path(delta.getPath())); if (version != expectedVersion) { throw new IllegalStateException( String.format( "Gap detected in delta files: expected version %d, found %d", expectedVersion, version)); } expectedVersion++; } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/ChecksumWriter.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.checksum; import static io.delta.kernel.internal.DeltaErrors.wrapEngineExceptionThrowsIO; import static io.delta.kernel.internal.util.Utils.singletonCloseableIterator; import static java.util.Objects.requireNonNull; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.util.FileNames; import java.io.IOException; import java.nio.file.FileAlreadyExistsException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Writers for writing checksum files from a snapshot */ public class ChecksumWriter { private static final Logger logger = LoggerFactory.getLogger(ChecksumWriter.class); private final Path logPath; public ChecksumWriter(Path logPath) { this.logPath = requireNonNull(logPath); } /** Writes a checksum file */ public void writeCheckSum(Engine engine, CRCInfo crcInfo) throws IOException { Path newChecksumPath = FileNames.checksumFile(logPath, crcInfo.getVersion()); logger.info("Writing checksum file to path: {}", newChecksumPath); try { wrapEngineExceptionThrowsIO( () -> { engine .getJsonHandler() .writeJsonFileAtomically( newChecksumPath.toString(), singletonCloseableIterator(crcInfo.toRow()), false /* overwrite */); logger.info("Write checksum file `{}` succeeds", newChecksumPath); return null; }, "Write checksum file `%s`", newChecksumPath); } catch (FileAlreadyExistsException e) { logger.info("Checksum file already exists for version {}", crcInfo.getVersion()); // Checksum file has been created while we were computing it. // This is fine - the checksum now exists, which was our goal. } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/clustering/ClusteringMetadataDomain.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.clustering; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import io.delta.kernel.expressions.Column; import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.metadatadomain.JsonMetadataDomain; import java.util.*; import java.util.stream.Collectors; /** Represents the metadata domain for clustering. */ public final class ClusteringMetadataDomain extends JsonMetadataDomain { public static final String DOMAIN_NAME = "delta.clustering"; /** * Constructs a ClusteringMetadataDomain with the clustering columns. * * @param clusteringColumns the columns used for clustering. If column mapping is enabled, use the * physical name assigned; otherwise, use the logical column name. */ public static ClusteringMetadataDomain fromClusteringColumns(List clusteringColumns) { return new ClusteringMetadataDomain( clusteringColumns.stream() .map(column -> Arrays.asList(column.getNames())) .collect(Collectors.toList())); } /** * Creates an instance of {@link ClusteringMetadataDomain} from a JSON configuration string. * * @param json the JSON configuration string */ public static ClusteringMetadataDomain fromJsonConfiguration(String json) { return JsonMetadataDomain.fromJsonConfiguration(json, ClusteringMetadataDomain.class); } /** * Creates an optional instance of {@link ClusteringMetadataDomain} from a {@link SnapshotImpl} if * present. * * @param snapshot the snapshot instance */ // TODO: Add the test coverage for this function in the integration test. public static Optional fromSnapshot(SnapshotImpl snapshot) { return JsonMetadataDomain.fromSnapshot(snapshot, ClusteringMetadataDomain.class, DOMAIN_NAME); } /** * The column names used for clustering. If column mapping is enabled, we use physical column * names, otherwise we would store its logical column names. Stored as a List of Lists to avoid * customized serialization and deserialization logic. */ @JsonProperty("clusteringColumns") private final List> clusteringColumns; @JsonCreator private ClusteringMetadataDomain( @JsonProperty("clusteringColumns") List> physicalClusteringColumns) { this.clusteringColumns = physicalClusteringColumns; } @Override public String getDomainName() { return DOMAIN_NAME; } @JsonIgnore public List getClusteringColumns() { return clusteringColumns.stream() .map(list -> new Column(list.toArray(new String[0]))) .collect(Collectors.toList()); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/clustering/ClusteringUtils.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.clustering; import io.delta.kernel.expressions.Column; import io.delta.kernel.internal.actions.DomainMetadata; import java.util.List; public class ClusteringUtils { private ClusteringUtils() { // Empty private constructor to prevent instantiation } /** * Get the domain metadata for the clustering columns. If column mapping is enabled, pass the list * of physical names assigned; otherwise, use the logical column names. */ public static DomainMetadata getClusteringDomainMetadata(List clusteringColumns) { ClusteringMetadataDomain clusteringMetadataDomain = ClusteringMetadataDomain.fromClusteringColumns(clusteringColumns); return clusteringMetadataDomain.toDomainMetadata(); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/columndefaults/ColumnDefaults.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.columndefaults; import io.delta.kernel.data.Row; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.internal.DeltaErrors; import io.delta.kernel.internal.data.TransactionStateRow; import io.delta.kernel.internal.util.SchemaIterable; import io.delta.kernel.types.*; import java.math.BigDecimal; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.stream.Stream; /** * Utilities class for TableFeature "allowColumnDefaults". NOTE: As of Aug 2025, kernel only * supports reading Delta tables with the table feature, or modifying table metadata. Writing actual * data to the table is not allowed. */ public class ColumnDefaults { private static final String DEFAULT_VALUE_METADATA_KEY = "CURRENT_DEFAULT"; /** Don't allow data writes to tables with default values */ public static void blockWriteIfEnabled(Row transactionState) { if (extractFieldsWithDefaultValues(TransactionStateRow.getLogicalSchema(transactionState)) .findAny() .isPresent()) { throw new UnsupportedOperationException( "Writing with Column Default values is not supported yet."); } } /** * Validate Column Default values in the provided metadata. Kernel only supports literal default * values. See {validateLiteral}. * * @param schema target table schema * @param isEnabled When the feature is disabled, no column default is allowed. * @param isIcebergCompatV3Enabled Kernel currently requires IcebergCompatV3 to be enabled when * using Column Defaults * @throws KernelException when the table contains invalid default value */ public static void validateSchema( StructType schema, boolean isEnabled, boolean isIcebergCompatV3Enabled) { if (isEnabled && !isIcebergCompatV3Enabled) { throw DeltaErrors.defaultValueRequireIcebergV3(); } Stream defaultValues = extractFieldsWithDefaultValues(schema); if (!isEnabled) { if (defaultValues.findAny().isPresent()) { throw DeltaErrors.defaultValueRequiresTableFeature(); } } else { // This check will be relaxed once kernel supports default values with expression defaultValues.forEach( field -> { String defaultValue = getRawDefaultValue(field); try { validateLiteral(field.getDataType(), defaultValue); } catch (IllegalArgumentException e) { throw DeltaErrors.nonLiteralDefaultValue(defaultValue); } catch (UnsupportedOperationException e) { throw DeltaErrors.unsupportedDataTypeForDefaultValue( field.getName(), field.getDataType().toString()); } }); } } /** * Validate that the schema only contains literal default values as a requirement of * IcebergCompat. * * @param schema table schema */ public static void validateSchemaForIcebergCompat(StructType schema, String compatVersion) { extractFieldsWithDefaultValues(schema) .forEach( field -> { String defaultValue = getRawDefaultValue(field); try { validateLiteral(field.getDataType(), defaultValue); } catch (IllegalArgumentException e) { throw DeltaErrors.icebergCompatRequiresLiteralDefaultValue( compatVersion, field.getDataType(), defaultValue); } }); } private static Stream extractFieldsWithDefaultValues(StructType schema) { return new SchemaIterable(schema) .stream() .map(SchemaIterable.SchemaElement::getField) .filter(f -> getRawDefaultValue(f) != null); } public static String getRawDefaultValue(StructField field) { return field.getMetadata().getString(DEFAULT_VALUE_METADATA_KEY); } /** * Validate that the provided default value is a literal (not an expression) and can be cast to * the given data type. We only support a limited set of literals. Example: * *

    *
  • 'CURRENT_VALUE()' is a valid String literal. CURRENT_VALUE (no quotes) is not. *
  • 4.95 and '4.95' are both valid Double/Float literals. *
  • '2022-01-01' is a valid Date literal, '09/01/2022' is not. *
* * @throws IllegalArgumentException if the value is not a literal value matching the data type * @throws UnsupportedOperationException when kernel does not support column defaults for the data * type */ private static void validateLiteral(DataType type, String value) { String stripped = stripQuotes(value, false); if ((type instanceof StringType || type instanceof BinaryType)) { // String literals are required to be enclosed in quotes stripQuotes(value, true); } else if (type instanceof LongType) { Long.parseLong(stripped); } else if (type instanceof IntegerType) { Integer.parseInt(stripped); } else if (type instanceof ShortType) { Short.parseShort(stripped); } else if (type instanceof FloatType) { Float.parseFloat(stripped); } else if (type instanceof DoubleType) { Double.parseDouble(stripped); } else if (type instanceof DecimalType) { DecimalType dtype = (DecimalType) type; BigDecimal input = new BigDecimal(stripped); if (input.scale() > dtype.getScale() || input.precision() > dtype.getPrecision()) { throw new IllegalArgumentException("invalid default value " + value + " for " + type); } } else if (type instanceof BooleanType) { Boolean.parseBoolean(stripped); } else if (type instanceof DateType) { try { LocalDate.parse(stripped, DateTimeFormatter.ISO_LOCAL_DATE); } catch (DateTimeParseException e) { throw new IllegalArgumentException(e); } } else if (type instanceof TimestampType) { try { OffsetDateTime.parse(stripped, DateTimeFormatter.ISO_DATE_TIME); } catch (DateTimeParseException e) { throw new IllegalArgumentException(e); } } else if (type instanceof TimestampNTZType) { try { LocalDateTime.parse(stripped, DateTimeFormatter.ISO_LOCAL_DATE_TIME); } catch (DateTimeParseException e) { throw new IllegalArgumentException(e); } } else { throw new UnsupportedOperationException( "Kernel does not support column defaults for " + type.toString()); } } /** * Remove the quotes from input string. * * @param input input to remove quotes * @param require require the input to have quotes * @return string with enclosing quotes removed */ private static String stripQuotes(String input, boolean require) { if (input.length() > 1 && ((input.charAt(0) == '\'' && input.charAt(input.length() - 1) == '\'') || (input.charAt(0) == '"' && input.charAt(input.length() - 1) == '"'))) { return input.substring(1, input.length() - 1); } if (require) { throw new IllegalArgumentException("String literal not enclosed in quotes: " + input); } return input; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/commit/DefaultFileSystemManagedTableOnlyCommitter.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.commit; import static io.delta.kernel.internal.DeltaErrors.wrapEngineExceptionThrowsIO; import io.delta.kernel.commit.CommitFailedException; import io.delta.kernel.commit.CommitMetadata; import io.delta.kernel.commit.CommitResponse; import io.delta.kernel.commit.Committer; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.DeltaErrorsInternal; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.files.ParsedPublishedDeltaData; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import java.io.IOException; import java.nio.file.FileAlreadyExistsException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class DefaultFileSystemManagedTableOnlyCommitter implements Committer { private static final Logger logger = LoggerFactory.getLogger(DefaultFileSystemManagedTableOnlyCommitter.class); public static final DefaultFileSystemManagedTableOnlyCommitter INSTANCE = new DefaultFileSystemManagedTableOnlyCommitter(); private DefaultFileSystemManagedTableOnlyCommitter() {} @Override public CommitResponse commit( Engine engine, CloseableIterator finalizedActions, CommitMetadata commitMetadata) throws CommitFailedException { commitMetadata.getReadProtocolOpt().ifPresent(this::validateProtocol); commitMetadata.getNewProtocolOpt().ifPresent(this::validateProtocol); final String jsonCommitFile = FileNames.deltaFile(commitMetadata.getDeltaLogDirPath(), commitMetadata.getVersion()); logger.info("Attempting to commit {}", jsonCommitFile); try { return wrapEngineExceptionThrowsIO( () -> { engine .getJsonHandler() .writeJsonFileAtomically(jsonCommitFile, finalizedActions, false /* overwrite */); final FileStatus writtenDeltaFileStatus = engine.getFileSystemClient().getFileStatus(jsonCommitFile); return new CommitResponse( ParsedPublishedDeltaData.forFileStatus(writtenDeltaFileStatus)); }, String.format("Write file actions to JSON log file `%s`", jsonCommitFile)); } catch (FileAlreadyExistsException e) { throw new CommitFailedException( true /* retryable */, true /* conflict */, "Concurrent write detected for version " + commitMetadata.getVersion(), e); } catch (IOException e) { throw new CommitFailedException( true /* retryable */, false /* conflict */, "Failed to write commit file due to I/O error: " + e.getMessage(), e); } } private void validateProtocol(Protocol protocol) { if (TableFeatures.isCatalogManagedSupported(protocol)) { throw DeltaErrorsInternal.defaultCommitterDoesNotSupportCatalogManagedTables(); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/commitrange/CommitRangeBuilderImpl.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.commitrange; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.CommitRange; import io.delta.kernel.CommitRangeBuilder; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.DeltaErrorsInternal; import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.files.LogDataUtils; import io.delta.kernel.internal.files.ParsedLogData; import java.util.Collections; import java.util.List; import java.util.Optional; /** * An implementation of {@link CommitRangeBuilder}. * *

Note: The primary responsibility of this class is to take input, validate that input, and then * create a {@link CommitRange} instance with the specified configuration. */ public class CommitRangeBuilderImpl implements CommitRangeBuilder { public static class Context { public final String unresolvedPath; public final CommitBoundary startBoundary; public Optional endBoundaryOpt = Optional.empty(); public List logDatas = Collections.emptyList(); public Optional maxCatalogVersion = Optional.empty(); public Context(String unresolvedPath, CommitBoundary startBoundary) { this.unresolvedPath = requireNonNull(unresolvedPath, "unresolvedPath is null"); this.startBoundary = requireNonNull(startBoundary, "startBoundary is null"); } } private final Context ctx; public CommitRangeBuilderImpl(String unresolvedPath, CommitBoundary startBoundary) { ctx = new Context(unresolvedPath, startBoundary); } /////////////////////////////////////// // Public CommitRangeBuilder Methods // /////////////////////////////////////// @Override public CommitRangeBuilderImpl withEndBoundary(CommitBoundary endBoundary) { ctx.endBoundaryOpt = Optional.of(requireNonNull(endBoundary, "endBoundary is null")); return this; } @Override public CommitRangeBuilderImpl withLogData(List logData) { ctx.logDatas = requireNonNull(logData, "logData is null"); return this; } @Override public CommitRangeBuilderImpl withMaxCatalogVersion(long version) { checkArgument(version >= 0, "maxCatalogVersion must be >= 0, but got: %d", version); ctx.maxCatalogVersion = Optional.of(version); return this; } @Override public CommitRange build(Engine engine) { validateInputOnBuild(); return new CommitRangeFactory(engine, ctx).create(engine); } //////////////////////////// // Private Helper Methods // //////////////////////////// private void validateInputOnBuild() { // Validate that start boundary is less than or equal to end boundary if end boundary is // provided if (ctx.endBoundaryOpt.isPresent()) { CommitBoundary startBoundary = ctx.startBoundary; CommitBoundary endBoundary = ctx.endBoundaryOpt.get(); // If both are version-based, compare versions if (startBoundary.isVersion() && endBoundary.isVersion()) { checkArgument( startBoundary.getVersion() <= endBoundary.getVersion(), "startVersion must be <= endVersion"); } // If both are timestamp-based, compare timestamps else if (startBoundary.isTimestamp() && endBoundary.isTimestamp()) { checkArgument( startBoundary.getTimestamp() <= endBoundary.getTimestamp(), "startTimestamp must be <= endTimestamp"); } // Mixed types are allowed but will need runtime resolution } // Validate max catalog version constraints if provided if (ctx.maxCatalogVersion.isPresent()) { long maxVersion = ctx.maxCatalogVersion.get(); // Validate start boundary against max catalog version if (ctx.startBoundary.isVersion()) { checkArgument( ctx.startBoundary.getVersion() <= maxVersion, String.format( "startVersion (%d) must be <= maxCatalogVersion (%d)", ctx.startBoundary.getVersion(), maxVersion)); } else if (ctx.startBoundary.isTimestamp()) { long latestSnapshotVersion = ((SnapshotImpl) ctx.startBoundary.getLatestSnapshot()).getVersion(); if (latestSnapshotVersion != maxVersion) { throw DeltaErrorsInternal.invalidLatestSnapshotForMaxCatalogVersion( latestSnapshotVersion, maxVersion); } } // Validate end boundary against max catalog version if (ctx.endBoundaryOpt.isPresent()) { CommitBoundary endBoundary = ctx.endBoundaryOpt.get(); if (endBoundary.isVersion()) { checkArgument( endBoundary.getVersion() <= maxVersion, String.format( "endVersion (%d) must be <= maxCatalogVersion (%d)", endBoundary.getVersion(), maxVersion)); } else if (endBoundary.isTimestamp()) { long latestSnapshotVersion = ((SnapshotImpl) endBoundary.getLatestSnapshot()).getVersion(); if (latestSnapshotVersion != maxVersion) { throw DeltaErrorsInternal.invalidLatestSnapshotForMaxCatalogVersion( latestSnapshotVersion, maxVersion); } } } // Validate logData ends with maxCatalogVersion when no end boundary is provided if (!ctx.endBoundaryOpt.isPresent() && !ctx.logDatas.isEmpty()) { long lastLogDataVersion = ctx.logDatas.get(ctx.logDatas.size() - 1).getVersion(); checkArgument( lastLogDataVersion == maxVersion, String.format( "When maxCatalogVersion is specified without an end boundary, the last " + "logData version (%d) must equal maxCatalogVersion (%d)", lastLogDataVersion, maxVersion)); } } // Validate logData input LogDataUtils.validateLogDataContainsOnlyRatifiedStagedCommits(ctx.logDatas); LogDataUtils.validateLogDataIsSortedContiguous(ctx.logDatas); // Validate that when endVersion and logData are both provided, the logData includes endVersion // This is applicable for catalog-managed tables since the catalog must provide sufficient // ratified commits to cover the requested endVersion if (ctx.endBoundaryOpt.isPresent() && !ctx.logDatas.isEmpty()) { CommitBoundary endBoundary = ctx.endBoundaryOpt.get(); if (endBoundary.isVersion()) { long endVersion = endBoundary.getVersion(); long lastLogDataVersion = ctx.logDatas.get(ctx.logDatas.size() - 1).getVersion(); checkArgument( lastLogDataVersion >= endVersion, String.format( "When endVersion is specified with logData, the last logData version (%d) " + "must be >= endVersion (%d) to cover the requested range", lastLogDataVersion, endVersion)); } // Note: For timestamp boundaries, we can't validate at build time since the timestamp // needs to be resolved to a version first in CommitRangeFactory } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/commitrange/CommitRangeFactory.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.commitrange; import static io.delta.kernel.internal.DeltaErrors.*; import static io.delta.kernel.internal.DeltaErrorsInternal.*; import static io.delta.kernel.internal.DeltaLogActionUtils.listDeltaLogFilesAsIter; import static io.delta.kernel.internal.util.Utils.resolvePath; import io.delta.kernel.CommitRangeBuilder; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.DeltaErrors; import io.delta.kernel.internal.DeltaHistoryManager; import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.files.LogDataUtils; import io.delta.kernel.internal.files.ParsedCatalogCommitData; import io.delta.kernel.internal.files.ParsedDeltaData; import io.delta.kernel.internal.files.ParsedLogData; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.lang.ListUtils; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.utils.FileStatus; import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; class CommitRangeFactory { private static final Logger logger = LoggerFactory.getLogger(CommitRangeFactory.class); private final CommitRangeBuilderImpl.Context ctx; private final Path tablePath; private final Path logPath; CommitRangeFactory(Engine engine, CommitRangeBuilderImpl.Context ctx) { this.ctx = ctx; this.tablePath = new Path(resolvePath(engine, ctx.unresolvedPath)); this.logPath = new Path(tablePath, "_delta_log"); } CommitRangeImpl create(Engine engine) { List ratifiedCommits = getFileBasedRatifiedCommits(); long startVersion = resolveStartVersion(engine, ratifiedCommits); Optional endVersionOpt = resolveEndVersionIfSpecified(engine, ratifiedCommits); // Apply maxCatalogVersion constraint if (ctx.maxCatalogVersion.isPresent()) { if (!endVersionOpt.isPresent()) { // When maxCatalogVersion is specified and no end boundary is provided, // the end version should be maxCatalogVersion endVersionOpt = ctx.maxCatalogVersion; logger.info( "{}: Using maxCatalogVersion {} as end version", tablePath, endVersionOpt.get()); } else { // Check that endVersion is <= maxCatalogVersion if (endVersionOpt.get() > ctx.maxCatalogVersion.get()) { throw DeltaErrors.resolvedEndVersionAfterMaxCatalogVersion( tablePath.toString(), endVersionOpt.get(), ctx.maxCatalogVersion.get()); } } } validateVersionRange(startVersion, endVersionOpt); logResolvedVersions(startVersion, endVersionOpt); List deltas = getDeltasForVersionRangeWithCatalogPriority( engine, startVersion, endVersionOpt, ratifiedCommits); // Once we have a list of deltas, we can resolve endVersion=latestVersion for the default case long endVersion = endVersionOpt.orElseGet(() -> extractLatestVersion(deltas)); if (!endVersionOpt.isPresent()) { logger.info("{}: Resolved end-boundary to the latest version {}", tablePath, endVersion); } return new CommitRangeImpl( tablePath, ctx.startBoundary, ctx.endBoundaryOpt, startVersion, endVersion, deltas); } private long resolveStartVersion(Engine engine, List catalogCommits) { if (ctx.startBoundary.isVersion()) { return ctx.startBoundary.getVersion(); } else { logger.info( "{}: Trying to resolve start-boundary timestamp {} to version", tablePath, ctx.startBoundary.getTimestamp()); return DeltaHistoryManager.getVersionAtOrAfterTimestamp( engine, logPath, ctx.startBoundary.getTimestamp(), (SnapshotImpl) ctx.startBoundary.getLatestSnapshot(), catalogCommits); } } /** * This method resolves the endBoundary to a version if it is specified. For a version-based * boundary, this just returns the version. For a timestamp-based boundary, this resolves the * timestamp to version. When the boundary is not specified, this returns empty, as the version * cannot be resolved until later after we have performed any listing. */ private Optional resolveEndVersionIfSpecified( Engine engine, List catalogCommits) { if (!ctx.endBoundaryOpt.isPresent()) { // When endBoundary is not provided, we default to the latest version. We cannot resolve the // latest version until later after we have performed any listing. return Optional.empty(); } CommitRangeBuilder.CommitBoundary endBoundary = ctx.endBoundaryOpt.get(); if (endBoundary.isVersion()) { return Optional.of(endBoundary.getVersion()); } else { logger.info( "{}: Trying to resolve end-boundary timestamp {} to version", tablePath, endBoundary.getTimestamp()); long resolvedVersion = DeltaHistoryManager.getVersionBeforeOrAtTimestamp( engine, logPath, endBoundary.getTimestamp(), (SnapshotImpl) endBoundary.getLatestSnapshot(), catalogCommits); return Optional.of(resolvedVersion); } } private void validateVersionRange(long startVersion, Optional endVersionOpt) { endVersionOpt.ifPresent( endVersion -> { if (startVersion > endVersion) { throw invalidResolvedVersionRange(tablePath.toString(), startVersion, endVersion); } }); } private void logResolvedVersions(long startVersion, Optional endVersionOpt) { logger.info( "{}: Resolved startVersion={} and endVersion={} from startBoundary={} endBoundary={}", tablePath, startVersion, endVersionOpt, ctx.startBoundary, ctx.endBoundaryOpt); } private long extractLatestVersion(List deltaDatas) { return ListUtils.getLast(deltaDatas).getVersion(); } private List getFileBasedRatifiedCommits() { // Note: currently this is all we allow in CommitRangeBuilder anyway, but in the future that // could change return ctx.logDatas.stream() .filter(logData -> logData instanceof ParsedCatalogCommitData) .map(logData -> (ParsedCatalogCommitData) logData) .filter(deltaData -> deltaData.isFile()) .collect(Collectors.toList()); } /** * Lists the _delta_log and combines the found published deltas with the catalog commits to * compile a single contiguous list of deltas. Catalog commits take priority over published deltas * when both are present for the same commit version. Returned deltas are guaranteed to start with * startVersion, end with endVersion if endVersionOpt is non-empty, and are contiguous. Throws an * exception if no deltas are found in the version range, or if startVersion or endVersion cannot * be found. */ private List getDeltasForVersionRangeWithCatalogPriority( Engine engine, long startVersion, Optional endVersionOpt, List ratifiedCommits) { // Get published deltas between startVersion and endVersionOpt List publishedDeltas = getPublishedDeltasInVersionRange(engine, startVersion, endVersionOpt); // Get ratified deltas between startVersion and endVersionOpt List ratifiedDeltas = ratifiedCommits.stream() .filter( x -> x.getVersion() >= startVersion && x.getVersion() <= endVersionOpt.orElse(Long.MAX_VALUE)) .collect(Collectors.toList()); // Validate they are contiguous and valid (i.e. backfill is ordered) validatePublishedPlusRatifiedDeltas(publishedDeltas, ratifiedDeltas); List combinedDeltas = LogDataUtils.combinePublishedAndRatifiedDeltasWithCatalogPriority( publishedDeltas, ratifiedDeltas); validateDeltasMatchVersionRange(combinedDeltas, startVersion, endVersionOpt); return combinedDeltas; } /** * Returns any published deltas found on the file-system within the version range provided and * validates that they are contiguous. Returns a sorted and contiguous list of deltas for the * version range, but does no validation that the list fills the range. */ private List getPublishedDeltasInVersionRange( Engine engine, long startVersion, Optional endVersionOpt) { final List commitFiles = listDeltaLogFilesAsIter( engine, Collections.singleton(FileNames.DeltaLogFileType.COMMIT), tablePath, startVersion, endVersionOpt, false /* mustBeRecreatable */) .toInMemoryList(); List publishedDeltas = commitFiles.stream().map(ParsedDeltaData::forFileStatus).collect(Collectors.toList()); // Validate listed delta files are contiguous if (publishedDeltas.size() > 1) { for (int i = 1; i < publishedDeltas.size(); i++) { final ParsedLogData prev = publishedDeltas.get(i - 1); final ParsedLogData curr = publishedDeltas.get(i); if (prev.getVersion() + 1 != curr.getVersion()) { throw publishedDeltasNotContiguous( tablePath.toString(), publishedDeltas.stream() .map(ParsedDeltaData::getVersion) .collect(Collectors.toList())); } } } return publishedDeltas; } private void validatePublishedPlusRatifiedDeltas( List publishedDeltas, List ratifiedDeltas) { // Valid example: P0, P1, P2 + R1 (ratified within published) // Valid example: P0, P1 + R2, R3 (no overlap) // Valid example: P0, P1 + R1, R2 (overlap) if (!publishedDeltas.isEmpty() && !ratifiedDeltas.isEmpty()) { long earliestPublishedVersion = publishedDeltas.get(0).getVersion(); long earliestRatifiedVersion = ratifiedDeltas.get(0).getVersion(); // We cannot have ratifiedDeltas.head.version < publishedDeltas.head.version // Invalid example: P2, P3 + R1, R2, R3 if (earliestRatifiedVersion < earliestPublishedVersion) { throw catalogCommitsPrecedePublishedDeltas( tablePath.toString(), earliestRatifiedVersion, publishedDeltas.stream().map(ParsedDeltaData::getVersion).collect(Collectors.toList())); } long lastPublishedVersion = ListUtils.getLast(publishedDeltas).getVersion(); // We must have publishedDeltas + ratifiedDeltas be contiguous // Invalid example: P0, P1 + R3, R4 if (lastPublishedVersion + 1 < earliestRatifiedVersion) { throw publishedDeltasAndCatalogCommitsNotContiguous( tablePath.toString(), publishedDeltas.stream().map(ParsedDeltaData::getVersion).collect(Collectors.toList()), ratifiedDeltas.stream().map(ParsedDeltaData::getVersion).collect(Collectors.toList())); } } } private void validateDeltasMatchVersionRange( List deltas, long startVersion, Optional endVersionOpt) { // This can only happen if publishedDeltas.isEmpty && ratifiedDeltas.isEmpty if (deltas.isEmpty()) { throw noCommitFilesFoundForVersionRange(tablePath.toString(), startVersion, endVersionOpt); } long earliestVersion = ListUtils.getFirst(deltas).getVersion(); long latestVersion = ListUtils.getLast(deltas).getVersion(); if (earliestVersion != startVersion) { throw startVersionNotFound(tablePath.toString(), startVersion, Optional.of(earliestVersion)); } endVersionOpt.ifPresent( endVersion -> { if (latestVersion != endVersion) { throw endVersionNotFound(tablePath.toString(), endVersion, latestVersion); } }); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/commitrange/CommitRangeImpl.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.commitrange; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.CommitActions; import io.delta.kernel.CommitRange; import io.delta.kernel.CommitRangeBuilder; import io.delta.kernel.Snapshot; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.DeltaLogActionUtils; import io.delta.kernel.internal.TableChangesUtils; import io.delta.kernel.internal.annotation.VisibleForTesting; import io.delta.kernel.internal.files.ParsedDeltaData; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; /** Implementation of {@link CommitRange}. */ public class CommitRangeImpl implements CommitRange { private final Path dataPath; private final CommitRangeBuilder.CommitBoundary startBoundary; private final Optional endBoundaryOpt; private final long startVersion; private final long endVersion; private final List deltas; public CommitRangeImpl( Path dataPath, CommitRangeBuilder.CommitBoundary startBoundary, Optional endBoundaryOpt, long startVersion, long endVersion, List deltas) { checkArgument(startVersion <= endVersion, "must have startVersion <= endVersion"); checkArgument( deltas.size() == endVersion - startVersion + 1, "deltaFiles size must match size of range"); this.dataPath = requireNonNull(dataPath, "dataPath cannot be null"); this.startBoundary = requireNonNull(startBoundary, "startBoundary cannot be null"); this.endBoundaryOpt = requireNonNull(endBoundaryOpt, "endSpecOpt cannot be null"); this.startVersion = startVersion; this.endVersion = endVersion; this.deltas = requireNonNull(deltas, "deltas cannot be null"); } //////////////////////////////////////// // Public CommitRange Implementation // //////////////////////////////////////// @Override public long getStartVersion() { return startVersion; } @Override public long getEndVersion() { return endVersion; } @Override public CommitRangeBuilder.CommitBoundary getQueryStartBoundary() { return startBoundary; } @Override public Optional getQueryEndBoundary() { return endBoundaryOpt; } @VisibleForTesting public List getDeltaFiles() { return deltas.stream().map(ParsedDeltaData::getFileStatus).collect(Collectors.toList()); } @Override public CloseableIterator getActions( Engine engine, Snapshot startSnapshot, Set actionSet) { validateParameters(engine, startSnapshot, actionSet); // Build on top of getCommitActions() by flattening and adding version/timestamp columns CloseableIterator commits = getCommitActions(engine, startSnapshot, actionSet); return TableChangesUtils.flattenCommitsAndAddMetadata(engine, commits); } @Override public CloseableIterator getCommitActions( Engine engine, Snapshot startSnapshot, Set actionSet) { validateParameters(engine, startSnapshot, actionSet); return DeltaLogActionUtils.getActionsFromCommitFilesWithProtocolValidation( engine, dataPath.toString(), getDeltaFiles(), actionSet); } ////////////////////// // Private helpers // ////////////////////// private void validateParameters( Engine engine, Snapshot startSnapshot, Set actionSet) { requireNonNull(engine, "engine cannot be null"); requireNonNull(startSnapshot, "startSnapshot cannot be null"); requireNonNull(actionSet, "actionSet cannot be null"); checkArgument( startSnapshot.getVersion() == startVersion, "startSnapshot must have version = startVersion"); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/compaction/LogCompactionWriter.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.compaction; import static io.delta.kernel.internal.DeltaErrors.wrapEngineExceptionThrowsIO; import static io.delta.kernel.internal.lang.ListUtils.getLast; import static java.util.Objects.requireNonNull; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.DeltaLogActionUtils; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.replay.CreateCheckpointIterator; import io.delta.kernel.internal.snapshot.LogSegment; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.internal.util.FileNames.DeltaLogFileType; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Utility for writing out log compactions. */ public class LogCompactionWriter { private static final Logger logger = LoggerFactory.getLogger(LogCompactionWriter.class); private final Path dataPath; private final Path logPath; private final long startVersion; private final long endVersion; // We need to know after what time we can cleanup remove tombstones. This is pulled from the table // metadata, which we have at hook creation time in TransactionImpl, so we just store it here so // we can use it when we run this hook private final long minFileRetentionTimestampMillis; public LogCompactionWriter( Path dataPath, Path logPath, long startVersion, long endVersion, long minFileRetentionTimestampMillis) { this.dataPath = requireNonNull(dataPath); this.logPath = requireNonNull(logPath); this.startVersion = startVersion; this.endVersion = endVersion; this.minFileRetentionTimestampMillis = minFileRetentionTimestampMillis; } public void writeLogCompactionFile(Engine engine) throws IOException { Path compactedPath = FileNames.logCompactionPath(logPath, startVersion, endVersion); logger.info( "Writing log compaction file for versions {} to {} to path: {}", startVersion, endVersion, compactedPath); final long startTimeMillis = System.currentTimeMillis(); final List deltas = DeltaLogActionUtils.listDeltaLogFilesAsIter( engine, Collections.singleton(DeltaLogFileType.COMMIT), dataPath, startVersion, Optional.of(endVersion), false /* mustBeRecreatable */) .toInMemoryList(); logger.info( "{}: Took {}ms to list commit files for log compaction", dataPath, System.currentTimeMillis() - startTimeMillis); if (deltas.size() != (endVersion - startVersion + 1)) { throw new IllegalArgumentException( String.format( "Asked to compact between versions %d and %d, but found %d delta files", startVersion, endVersion, deltas.size())); } LogSegment segment = new LogSegment( dataPath, endVersion, deltas, Collections.emptyList(), Collections.emptyList(), getLast(deltas), Optional.empty() /* lastSeemChecksum */, Optional.empty() /* maxPublishedDeltaVersion */); CreateCheckpointIterator checkpointIterator = new CreateCheckpointIterator(engine, segment, minFileRetentionTimestampMillis); wrapEngineExceptionThrowsIO( () -> { try (CloseableIterator rows = Utils.intoRows(checkpointIterator)) { engine.getJsonHandler().writeJsonFileAtomically(compactedPath.toString(), rows, false); } logger.info("Successfully wrote log compaction file `{}`", compactedPath); return null; }, "Writing log compaction file `%s`", compactedPath); } /** Utility to determine if log compaction should run for the given commit version. */ public static boolean shouldCompact(long commitVersion, long compactionInterval) { // commits start at 0, so we add one to the commit version to check if we've hit the interval return commitVersion > 0 && ((commitVersion + 1) % compactionInterval == 0); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/data/ChildVectorBasedRow.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.data; import io.delta.kernel.data.ArrayValue; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.MapValue; import io.delta.kernel.data.Row; import io.delta.kernel.types.StructType; import java.math.BigDecimal; /** A {@link Row} implementation that wraps a set of child vectors for a specific {@code rowId}. */ public abstract class ChildVectorBasedRow implements Row { private final int rowId; private final StructType schema; public ChildVectorBasedRow(int rowId, StructType schema) { this.rowId = rowId; this.schema = schema; } @Override public StructType getSchema() { return schema; } @Override public boolean isNullAt(int ordinal) { return getChild(ordinal).isNullAt(rowId); } @Override public boolean getBoolean(int ordinal) { return getChild(ordinal).getBoolean(rowId); } @Override public byte getByte(int ordinal) { return getChild(ordinal).getByte(rowId); } @Override public short getShort(int ordinal) { return getChild(ordinal).getShort(rowId); } @Override public int getInt(int ordinal) { return getChild(ordinal).getInt(rowId); } @Override public long getLong(int ordinal) { return getChild(ordinal).getLong(rowId); } @Override public float getFloat(int ordinal) { return getChild(ordinal).getFloat(rowId); } @Override public double getDouble(int ordinal) { return getChild(ordinal).getDouble(rowId); } @Override public String getString(int ordinal) { return getChild(ordinal).getString(rowId); } @Override public BigDecimal getDecimal(int ordinal) { return getChild(ordinal).getDecimal(rowId); } @Override public byte[] getBinary(int ordinal) { return getChild(ordinal).getBinary(rowId); } @Override public Row getStruct(int ordinal) { return StructRow.fromStructVector(getChild(ordinal), rowId); } @Override public ArrayValue getArray(int ordinal) { return getChild(ordinal).getArray(rowId); } @Override public MapValue getMap(int ordinal) { return getChild(ordinal).getMap(rowId); } protected abstract ColumnVector getChild(int ordinal); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/data/ColumnarBatchRow.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.data; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import java.util.Objects; /** Row abstraction around a columnar batch and a particular row within the columnar batch. */ public class ColumnarBatchRow extends ChildVectorBasedRow { private final ColumnarBatch columnarBatch; public ColumnarBatchRow(ColumnarBatch columnarBatch, int rowId) { super(rowId, Objects.requireNonNull(columnarBatch, "columnarBatch is null").getSchema()); this.columnarBatch = columnarBatch; } @Override protected ColumnVector getChild(int ordinal) { return columnarBatch.getColumnVector(ordinal); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/data/DelegateRow.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.data; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.ArrayValue; import io.delta.kernel.data.MapValue; import io.delta.kernel.data.Row; import io.delta.kernel.types.*; import java.math.BigDecimal; import java.util.HashMap; import java.util.Map; import java.util.Objects; /** * This wraps an existing {@link Row} and allows overriding values for some particular ordinals. * This enables creating a modified view of a row without mutating the original row. */ public class DelegateRow implements Row { /** The underlying row being delegated to. */ private final Row row; /** * A map of ordinal-to-value overrides that takes precedence over the underlying row's data. When * accessing data, this map is checked first before falling back to the underlying row. */ private final Map overrides; public DelegateRow(Row row, Map overrides) { Objects.requireNonNull(row, "row is null"); Objects.requireNonNull(overrides, "map of overrides is null"); if (row instanceof DelegateRow) { // If the row is already a delegation of another row, we merge the overrides and keep only // one layer of delegation. DelegateRow delegateRow = (DelegateRow) row; this.row = delegateRow.row; this.overrides = new HashMap<>(delegateRow.overrides); this.overrides.putAll(overrides); } else { this.row = row; this.overrides = new HashMap<>(overrides); } } @Override public StructType getSchema() { return row.getSchema(); } @Override public boolean isNullAt(int ordinal) { if (overrides.containsKey(ordinal)) { return overrides.get(ordinal) == null; } return row.isNullAt(ordinal); } @Override public boolean getBoolean(int ordinal) { if (overrides.containsKey(ordinal)) { throwIfUnsafeAccess(ordinal, BooleanType.class, "boolean"); return (boolean) overrides.get(ordinal); } return row.getBoolean(ordinal); } @Override public byte getByte(int ordinal) { if (overrides.containsKey(ordinal)) { throwIfUnsafeAccess(ordinal, ByteType.class, "byte"); return (byte) overrides.get(ordinal); } return row.getByte(ordinal); } @Override public short getShort(int ordinal) { throwIfUnsafeAccess(ordinal, ShortType.class, "short"); if (overrides.containsKey(ordinal)) { return (short) overrides.get(ordinal); } return row.getShort(ordinal); } @Override public int getInt(int ordinal) { if (overrides.containsKey(ordinal)) { throwIfUnsafeAccess(ordinal, IntegerType.class, "integer"); return (int) overrides.get(ordinal); } return row.getInt(ordinal); } @Override public long getLong(int ordinal) { if (overrides.containsKey(ordinal)) { throwIfUnsafeAccess(ordinal, LongType.class, "long"); return (long) overrides.get(ordinal); } return row.getLong(ordinal); } @Override public float getFloat(int ordinal) { if (overrides.containsKey(ordinal)) { throwIfUnsafeAccess(ordinal, FloatType.class, "float"); return (float) overrides.get(ordinal); } return row.getFloat(ordinal); } @Override public double getDouble(int ordinal) { if (overrides.containsKey(ordinal)) { throwIfUnsafeAccess(ordinal, DoubleType.class, "double"); return (double) overrides.get(ordinal); } return row.getDouble(ordinal); } @Override public String getString(int ordinal) { if (overrides.containsKey(ordinal)) { throwIfUnsafeAccess(ordinal, StringType.class, "string"); return (String) overrides.get(ordinal); } return row.getString(ordinal); } @Override public BigDecimal getDecimal(int ordinal) { if (overrides.containsKey(ordinal)) { throwIfUnsafeAccess(ordinal, DecimalType.class, "decimal"); return (BigDecimal) overrides.get(ordinal); } return row.getDecimal(ordinal); } @Override public byte[] getBinary(int ordinal) { if (overrides.containsKey(ordinal)) { throwIfUnsafeAccess(ordinal, BinaryType.class, "binary"); return (byte[]) overrides.get(ordinal); } return row.getBinary(ordinal); } @Override public Row getStruct(int ordinal) { if (overrides.containsKey(ordinal)) { throwIfUnsafeAccess(ordinal, StructType.class, "struct"); return (Row) overrides.get(ordinal); } return row.getStruct(ordinal); } @Override public ArrayValue getArray(int ordinal) { if (overrides.containsKey(ordinal)) { // TODO: Not sufficient check, also need to check the element type. This should be revisited // together with the GenericRow. throwIfUnsafeAccess(ordinal, ArrayType.class, "array"); return (ArrayValue) overrides.get(ordinal); } return row.getArray(ordinal); } @Override public MapValue getMap(int ordinal) { if (overrides.containsKey(ordinal)) { // TODO: Not sufficient check, also need to check the element type. This should be revisited // together with the GenericRow. throwIfUnsafeAccess(ordinal, MapType.class, "map"); return (MapValue) overrides.get(ordinal); } return row.getMap(ordinal); } private void throwIfUnsafeAccess( int ordinal, Class expDataType, String accessType) { final StructType schema = row.getSchema(); checkArgument( ordinal >= 0 && ordinal < schema.length(), "Invalid ordinal %d for schema with length %d", ordinal, schema.length()); DataType actualDataType = schema.at(ordinal).getDataType(); if (!expDataType.isAssignableFrom(actualDataType.getClass())) { String msg = String.format( "Fail to access a '%s' value from a field of type '%s'", accessType, actualDataType); throw new UnsupportedOperationException(msg); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/data/GenericColumnVector.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.data; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.*; import io.delta.kernel.types.*; import java.math.BigDecimal; import java.util.List; import java.util.stream.Collectors; /** A generic implementation of {@link ColumnVector} that wraps a list of values. */ public class GenericColumnVector implements ColumnVector { private final List values; private final DataType dataType; public GenericColumnVector(List values, DataType dataType) { this.values = values; this.dataType = dataType; } @Override public DataType getDataType() { return dataType; } @Override public int getSize() { return values.size(); } @Override public void close() { // no-op } @Override public boolean isNullAt(int rowId) { validateRowId(rowId); return values.get(rowId) == null; } @Override public boolean getBoolean(int rowId) { checkArgument(BooleanType.BOOLEAN.equals(dataType)); return (Boolean) getValidatedValue(rowId, Boolean.class); } @Override public byte getByte(int rowId) { checkArgument(ByteType.BYTE.equals(dataType)); return (Byte) getValidatedValue(rowId, Byte.class); } @Override public short getShort(int rowId) { checkArgument(ShortType.SHORT.equals(dataType)); return (Short) getValidatedValue(rowId, Short.class); } @Override public int getInt(int rowId) { checkArgument(IntegerType.INTEGER.equals(dataType) || DateType.DATE.equals(dataType)); return (Integer) getValidatedValue(rowId, Integer.class); } @Override public long getLong(int rowId) { checkArgument( LongType.LONG.equals(dataType) || TimestampType.TIMESTAMP.equals(dataType) || TimestampNTZType.TIMESTAMP_NTZ.equals(dataType)); return (Long) getValidatedValue(rowId, Long.class); } @Override public float getFloat(int rowId) { checkArgument(FloatType.FLOAT.equals(dataType)); return (Float) getValidatedValue(rowId, Float.class); } @Override public double getDouble(int rowId) { checkArgument(DoubleType.DOUBLE.equals(dataType)); return (Double) getValidatedValue(rowId, Double.class); } @Override public BigDecimal getDecimal(int rowId) { checkArgument(dataType instanceof DecimalType); return (BigDecimal) getValidatedValue(rowId, BigDecimal.class); } @Override public String getString(int rowId) { checkArgument(StringType.STRING.equals(dataType)); return (String) getValidatedValue(rowId, String.class); } @Override public byte[] getBinary(int rowId) { checkArgument(BinaryType.BINARY.equals(dataType)); return (byte[]) getValidatedValue(rowId, byte[].class); } @Override public ArrayValue getArray(int rowId) { checkArgument(dataType instanceof ArrayType); return (ArrayValue) getValidatedValue(rowId, ArrayValue.class); } @Override public MapValue getMap(int rowId) { checkArgument(dataType instanceof MapType); return (MapValue) getValidatedValue(rowId, MapValue.class); } @Override public ColumnVector getChild(int ordinal) { checkArgument(dataType instanceof StructType); checkArgument(ordinal < ((StructType) dataType).length()); DataType childDatatype = ((StructType) dataType).at(ordinal).getDataType(); List childValues = extractChildValues(ordinal, childDatatype); return new GenericColumnVector(childValues, childDatatype); } private void validateRowId(int rowId) { checkArgument(rowId >= 0 && rowId < values.size(), "Invalid rowId: %s", rowId); } private Object getValidatedValue(int rowId, Class expectedType) { validateRowId(rowId); Object value = values.get(rowId); checkArgument( expectedType.isInstance(value), "Value must be of type %s", expectedType.getSimpleName()); return value; } private List extractChildValues(int ordinal, DataType childDatatype) { return values.stream() .map(e -> extractChildValue(e, ordinal, childDatatype)) .collect(Collectors.toList()); } private Object extractChildValue(Object element, int ordinal, DataType childDatatype) { checkArgument(element instanceof Row); Row row = (Row) element; if (row.isNullAt(ordinal)) { return null; } return extractTypedValue(row, ordinal, childDatatype); } private Object extractTypedValue(Row row, int ordinal, DataType childDatatype) { // Primitive Types if (childDatatype instanceof BooleanType) { return row.getBoolean(ordinal); } if (childDatatype instanceof ByteType) { return row.getByte(ordinal); } if (childDatatype instanceof ShortType) { return row.getShort(ordinal); } if (childDatatype instanceof IntegerType || childDatatype instanceof DateType) { return row.getInt(ordinal); } if (childDatatype instanceof LongType || childDatatype instanceof TimestampType || childDatatype instanceof TimestampNTZType) { return row.getLong(ordinal); } if (childDatatype instanceof FloatType) { return row.getFloat(ordinal); } if (childDatatype instanceof DoubleType) { return row.getDouble(ordinal); } // Complex Types if (childDatatype instanceof StringType) { return row.getString(ordinal); } if (childDatatype instanceof BinaryType) { return row.getBinary(ordinal); } if (childDatatype instanceof DecimalType) { return row.getDecimal(ordinal); } // Nested Types if (childDatatype instanceof StructType) { return row.getStruct(ordinal); } if (childDatatype instanceof ArrayType) { return row.getArray(ordinal); } if (childDatatype instanceof MapType) { return row.getMap(ordinal); } throw new UnsupportedOperationException( String.format("Unsupported data type: %s", childDatatype.getClass().getSimpleName())); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/data/GenericRow.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.data; import static java.util.Objects.requireNonNull; import io.delta.kernel.data.ArrayValue; import io.delta.kernel.data.MapValue; import io.delta.kernel.data.Row; import io.delta.kernel.types.*; import java.math.BigDecimal; import java.util.Map; /** Exposes a given map of values as a {@link Row} */ public class GenericRow implements Row { private final StructType schema; private final Map ordinalToValue; /** * @param schema the schema of the row * @param ordinalToValue a mapping of column ordinal to objects; for each column the object must * be of the return type corresponding to the data type's getter method in the Row interface */ public GenericRow(StructType schema, Map ordinalToValue) { this.schema = requireNonNull(schema, "schema is null"); this.ordinalToValue = requireNonNull(ordinalToValue, "ordinalToValue is null"); } @Override public StructType getSchema() { return schema; } @Override public boolean isNullAt(int ordinal) { return getValue(ordinal) == null; } @Override public boolean getBoolean(int ordinal) { throwIfUnsafeAccess(ordinal, BooleanType.class, "boolean"); return (boolean) getValue(ordinal); } @Override public byte getByte(int ordinal) { throwIfUnsafeAccess(ordinal, ByteType.class, "byte"); return (byte) getValue(ordinal); } @Override public short getShort(int ordinal) { throwIfUnsafeAccess(ordinal, ShortType.class, "short"); return (short) getValue(ordinal); } @Override public int getInt(int ordinal) { throwIfUnsafeAccess(ordinal, IntegerType.class, "integer"); return (int) getValue(ordinal); } @Override public long getLong(int ordinal) { throwIfUnsafeAccess(ordinal, LongType.class, "long"); return (long) getValue(ordinal); } @Override public float getFloat(int ordinal) { throwIfUnsafeAccess(ordinal, FloatType.class, "float"); return (float) getValue(ordinal); } @Override public double getDouble(int ordinal) { throwIfUnsafeAccess(ordinal, DoubleType.class, "double"); return (double) getValue(ordinal); } @Override public String getString(int ordinal) { throwIfUnsafeAccess(ordinal, StringType.class, "string"); return (String) getValue(ordinal); } @Override public BigDecimal getDecimal(int ordinal) { throwIfUnsafeAccess(ordinal, DecimalType.class, "decimal"); return (BigDecimal) getValue(ordinal); } @Override public byte[] getBinary(int ordinal) { throwIfUnsafeAccess(ordinal, BinaryType.class, "binary"); return (byte[]) getValue(ordinal); } @Override public Row getStruct(int ordinal) { throwIfUnsafeAccess(ordinal, StructType.class, "struct"); return (Row) getValue(ordinal); } @Override public ArrayValue getArray(int ordinal) { // TODO: not sufficient check, also need to check the element type throwIfUnsafeAccess(ordinal, ArrayType.class, "array"); return (ArrayValue) getValue(ordinal); } @Override public MapValue getMap(int ordinal) { // TODO: not sufficient check, also need to check the element types throwIfUnsafeAccess(ordinal, MapType.class, "map"); return (MapValue) getValue(ordinal); } private Object getValue(int ordinal) { return ordinalToValue.get(ordinal); } private void throwIfUnsafeAccess( int ordinal, Class expDataType, String accessType) { DataType actualDataType = dataType(ordinal); if (!expDataType.isAssignableFrom(actualDataType.getClass())) { String msg = String.format( "Trying to access a `%s` value from vector of type `%s`", accessType, actualDataType); throw new UnsupportedOperationException(msg); } } private DataType dataType(int ordinal) { if (schema.length() <= ordinal) { throw new IllegalArgumentException("invalid ordinal: " + ordinal); } return schema.at(ordinal).getDataType(); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/data/ScanStateRow.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.data; import static java.util.stream.Collectors.toMap; import io.delta.kernel.Scan; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.types.DataTypeJsonSerDe; import io.delta.kernel.internal.util.ColumnMapping; import io.delta.kernel.internal.util.ColumnMapping.ColumnMappingMode; import io.delta.kernel.internal.util.VectorUtils; import io.delta.kernel.types.*; import java.net.URI; import java.util.*; import java.util.stream.IntStream; /** Encapsulate the scan state (common info for all scan files) as a {@link Row} */ public class ScanStateRow extends GenericRow { private static final StructType SCHEMA = new StructType() .add("configuration", new MapType(StringType.STRING, StringType.STRING, false)) .add("logicalSchemaJson", StringType.STRING) .add("physicalSchemaJson", StringType.STRING) .add("partitionColumns", new ArrayType(StringType.STRING, false)) .add("minReaderVersion", IntegerType.INTEGER) .add("minWriterVersion", IntegerType.INTEGER) .add("tablePath", StringType.STRING); private static final Map COL_NAME_TO_ORDINAL = IntStream.range(0, SCHEMA.length()) .boxed() .collect(toMap(i -> SCHEMA.at(i).getName(), i -> i)); public static ScanStateRow of( Metadata metadata, Protocol protocol, String logicalSchemaJson, String physicalSchemaJson, String tablePath) { HashMap valueMap = new HashMap<>(); valueMap.put(COL_NAME_TO_ORDINAL.get("configuration"), metadata.getConfigurationMapValue()); valueMap.put(COL_NAME_TO_ORDINAL.get("logicalSchemaJson"), logicalSchemaJson); valueMap.put(COL_NAME_TO_ORDINAL.get("physicalSchemaJson"), physicalSchemaJson); valueMap.put(COL_NAME_TO_ORDINAL.get("partitionColumns"), metadata.getPartitionColumns()); valueMap.put(COL_NAME_TO_ORDINAL.get("minReaderVersion"), protocol.getMinReaderVersion()); valueMap.put(COL_NAME_TO_ORDINAL.get("minWriterVersion"), protocol.getMinWriterVersion()); valueMap.put(COL_NAME_TO_ORDINAL.get("tablePath"), tablePath); return new ScanStateRow(valueMap); } public ScanStateRow(HashMap valueMap) { super(SCHEMA, valueMap); } /** * Utility method to get the configuration map from the scan state {@link Row} returned by {@link * Scan#getScanState(Engine)}. * * @param scanState Scan state {@link Row} * @return Map of configuration key-value pairs. */ public static Map getConfiguration(Row scanState) { return VectorUtils.toJavaMap(scanState.getMap(COL_NAME_TO_ORDINAL.get("configuration"))); } /** * Utility method to get the logical schema from the scan state {@link Row} returned by {@link * Scan#getScanState(Engine)}. * * @param scanState Scan state {@link Row} * @return Logical schema to read from the data files. */ public static StructType getLogicalSchema(Row scanState) { String serializedSchema = scanState.getString(COL_NAME_TO_ORDINAL.get("logicalSchemaJson")); return DataTypeJsonSerDe.deserializeStructType(serializedSchema); } /** * Utility method to get the physical schema from the scan state {@link Row} returned by {@link * Scan#getScanState(Engine)}. This schema is used to request data from the scan files for the * query. * * @param scanState Scan state {@link Row} * @return Physical schema to read from the data files. */ public static StructType getPhysicalDataReadSchema(Row scanState) { String serializedSchema = scanState.getString(COL_NAME_TO_ORDINAL.get("physicalSchemaJson")); return DataTypeJsonSerDe.deserializeStructType(serializedSchema); } /** * Get the list of partition column names from the scan state {@link Row} returned by {@link * Scan#getScanState(Engine)}. * * @param scanState Scan state {@link Row} * @return List of partition column names according to the scan state. */ public static List getPartitionColumns(Row scanState) { return VectorUtils.toJavaList(scanState.getArray(COL_NAME_TO_ORDINAL.get("partitionColumns"))); } /** * Get the column mapping mode from the scan state {@link Row} returned by {@link * Scan#getScanState(Engine)}. */ public static ColumnMappingMode getColumnMappingMode(Row scanState) { return ColumnMapping.getColumnMappingMode(getConfiguration(scanState)); } /** * Get the table root from scan state {@link Row} returned by {@link Scan#getScanState(Engine)} * * @param scanState Scan state {@link Row} * @return Fully qualified path to the location of the table. */ public static Path getTableRoot(Row scanState) { return new Path(URI.create(scanState.getString(COL_NAME_TO_ORDINAL.get("tablePath")))); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/data/SelectionColumnVector.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.data; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.internal.deletionvectors.RoaringBitmapArray; import io.delta.kernel.types.BooleanType; import io.delta.kernel.types.DataType; /** The selection vector for a columnar batch as a boolean {@link ColumnVector}. */ public class SelectionColumnVector implements ColumnVector { private final RoaringBitmapArray bitmap; private final ColumnVector rowIndices; public SelectionColumnVector(RoaringBitmapArray bitmap, ColumnVector rowIndices) { this.bitmap = bitmap; this.rowIndices = rowIndices; } @Override public DataType getDataType() { return BooleanType.BOOLEAN; } @Override public int getSize() { return rowIndices.getSize(); } @Override public void close() { rowIndices.close(); } @Override public boolean isNullAt(int rowId) { return false; } @Override public boolean getBoolean(int rowId) { return !bitmap.contains(rowIndices.getLong(rowId)); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/data/StructRow.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.data; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.Row; import io.delta.kernel.types.StructType; /** A {@link Row} abstraction for a struct type column vector and a specific {@code rowId}. */ public class StructRow extends ChildVectorBasedRow { public static StructRow fromStructVector(ColumnVector columnVector, int rowId) { checkArgument(columnVector.getDataType() instanceof StructType); if (columnVector.isNullAt(rowId)) { return null; } else { return new StructRow(columnVector, rowId, (StructType) columnVector.getDataType()); } } private final ColumnVector structVector; private StructRow(ColumnVector structVector, int rowId, StructType schema) { super(rowId, schema); this.structVector = structVector; } @Override protected ColumnVector getChild(int ordinal) { return structVector.getChild(ordinal); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/data/TransactionStateRow.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.data; import static java.util.stream.Collectors.toMap; import io.delta.kernel.Transaction; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.TableConfig; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.types.DataTypeJsonSerDe; import io.delta.kernel.internal.util.ColumnMapping; import io.delta.kernel.internal.util.VectorUtils; import io.delta.kernel.types.*; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.IntStream; public class TransactionStateRow extends GenericRow { public static final StructType SCHEMA = new StructType() .add("logicalSchemaString", StringType.STRING) .add("physicalSchemaString", StringType.STRING) .add("partitionColumns", new ArrayType(StringType.STRING, false /* containsNull */)) .add( "configuration", new MapType(StringType.STRING, StringType.STRING, false /* valueContainsNull */)) .add("tablePath", StringType.STRING) .add("maxRetries", IntegerType.INTEGER) .add("protocol", Protocol.FULL_SCHEMA); private static final Map COL_NAME_TO_ORDINAL = IntStream.range(0, SCHEMA.length()) .boxed() .collect(toMap(i -> SCHEMA.at(i).getName(), i -> i)); public static TransactionStateRow of( Metadata metadata, Protocol protocol, String tablePath, int maxRetries) { HashMap valueMap = new HashMap<>(); valueMap.put(COL_NAME_TO_ORDINAL.get("logicalSchemaString"), metadata.getSchemaString()); valueMap.put( COL_NAME_TO_ORDINAL.get("physicalSchemaString"), metadata.getPhysicalSchema().toJson()); valueMap.put(COL_NAME_TO_ORDINAL.get("partitionColumns"), metadata.getPartitionColumns()); valueMap.put(COL_NAME_TO_ORDINAL.get("configuration"), metadata.getConfigurationMapValue()); valueMap.put(COL_NAME_TO_ORDINAL.get("tablePath"), tablePath); valueMap.put(COL_NAME_TO_ORDINAL.get("maxRetries"), maxRetries); valueMap.put(COL_NAME_TO_ORDINAL.get("protocol"), protocol.toRow()); return new TransactionStateRow(valueMap); } private TransactionStateRow(HashMap valueMap) { super(SCHEMA, valueMap); } /** * Get the logical schema of the table from the transaction state {@link Row} returned by {@link * Transaction#getTransactionState(Engine)}} * * @param transactionState Transaction state state {@link Row} * @return Logical schema of the table as {@link StructType} */ public static StructType getLogicalSchema(Row transactionState) { String serializedSchema = transactionState.getString(COL_NAME_TO_ORDINAL.get("logicalSchemaString")); return DataTypeJsonSerDe.deserializeStructType(serializedSchema); } /** * Get the physical schema of the table from the transaction state {@link Row} returned by {@link * Transaction#getTransactionState(Engine)}} * * @param transactionState Transaction state state {@link Row} * @return Logical schema of the table as {@link StructType} */ public static StructType getPhysicalSchema(Row transactionState) { String serializedSchema = transactionState.getString(COL_NAME_TO_ORDINAL.get("physicalSchemaString")); return DataTypeJsonSerDe.deserializeStructType(serializedSchema); } /** * Get the configuration from the transaction state {@link Row} returned by {@link * Transaction#getTransactionState(Engine)} * * @param transactionState * @return Configuration as a map of key-value pairs. */ public static Map getConfiguration(Row transactionState) { return VectorUtils.toJavaMap(transactionState.getMap(COL_NAME_TO_ORDINAL.get("configuration"))); } /** * Get the iceberg compatibility enabled or not from the transaction state {@link Row} returned by * {@link Transaction#getTransactionState(Engine)} * * @param transactionState Transaction state state {@link Row} * @return True if iceberg compatibility is enabled, false otherwise. */ public static boolean isIcebergCompatV2Enabled(Row transactionState) { return Boolean.parseBoolean( getConfiguration(transactionState) .getOrDefault(TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey(), "false")); } /** * Get the iceberg compatibility enabled or not from the transaction state {@link Row} returned by * {@link Transaction#getTransactionState(Engine)} * * @param transactionState Transaction state state {@link Row} * @return True if iceberg compatibility is enabled, false otherwise. */ public static boolean isIcebergCompatV3Enabled(Row transactionState) { return Boolean.parseBoolean( getConfiguration(transactionState) .getOrDefault(TableConfig.ICEBERG_COMPAT_V3_ENABLED.getKey(), "false")); } /** * Get the column mapping mode from the transaction state {@link Row} returned by {@link * Transaction#getTransactionState(Engine)} * * @param transactionState * @return ColumnMapping mode as {@link ColumnMapping.ColumnMappingMode} */ public static ColumnMapping.ColumnMappingMode getColumnMappingMode(Row transactionState) { String columnMappingModeStr = getConfiguration(transactionState) .getOrDefault(TableConfig.COLUMN_MAPPING_MODE.getKey(), "none"); return ColumnMapping.ColumnMappingMode.fromTableConfig(columnMappingModeStr); } /** * Get the list of partition column names from the transaction state {@link Row} returned by * {@link Transaction#getTransactionState(Engine)} * * @param transactionState Transaction state state {@link Row} * @return List of partition column names according to the scan state. */ public static List getPartitionColumnsList(Row transactionState) { return VectorUtils.toJavaList( transactionState.getArray(COL_NAME_TO_ORDINAL.get("partitionColumns"))); } /** * Get the table path from transaction state {@link Row} returned by {@link * Transaction#getTransactionState(Engine)} * * @param transactionState Transaction state state {@link Row} * @return Fully qualified path to the location of the table. */ public static String getTablePath(Row transactionState) { return transactionState.getString(COL_NAME_TO_ORDINAL.get("tablePath")); } /** * Get the maxRetries from transaction state {@link Row} returned by {@link * Transaction#getTransactionState(Engine)} */ public static int getMaxRetries(Row transactionState) { return transactionState.getInt(COL_NAME_TO_ORDINAL.get("maxRetries")); } /** * Get the Protocol from transaction state {@link Row} returned by {@link * Transaction#getTransactionState(Engine)} * * @param transactionState Transaction state state {@link Row} * @return Protocol object */ public static Protocol getProtocol(Row transactionState) { Row protocolRow = transactionState.getStruct(COL_NAME_TO_ORDINAL.get("protocol")); return Protocol.fromRow(protocolRow); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/deletionvectors/Base85Codec.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.deletionvectors; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.nio.charset.StandardCharsets.US_ASCII; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.UUID; /** * This implements Base85 using the 4 byte block aligned encoding and character set from Z85. * * @see Z85 encoding *

Taken from * https://github.com/delta-io/delta/blob/master/spark/src/main/scala/org/apache/spark * /sql/delta/util/Codec.scala */ public final class Base85Codec { static final long BASE = 85L; static final long BASE_2ND_POWER = 7225L; // 85^2 static final long BASE_3RD_POWER = 614125L; // 85^3 static final long BASE_4TH_POWER = 52200625L; // 85^4 static final int ASCII_BITMASK = 0x7F; // UUIDs always encode into 20 characters. public static final int ENCODED_UUID_LENGTH = 20; private static byte[] getEncodeMap() { byte[] map = new byte[85]; int i = 0; for (char c = '0'; c <= '9'; c++) { map[i] = (byte) c; i++; } for (char c = 'a'; c <= 'z'; c++) { map[i] = (byte) c; i++; } for (char c = 'A'; c <= 'Z'; c++) { map[i] = (byte) c; i++; } for (char c : ".-:+=^!/*?&<>()[]{}@%$#".toCharArray()) { map[i] = (byte) c; i++; } return map; } private static byte[] getDecodeMap() { checkArgument(ENCODE_MAP.length - 1 <= Byte.MAX_VALUE); // The bitmask is the same as largest possible value, so the length of the array must // be one greater. byte[] map = new byte[ASCII_BITMASK + 1]; Arrays.fill(map, (byte) -1); for (int i = 0; i < ENCODE_MAP.length; i++) { byte b = ENCODE_MAP[i]; map[b] = (byte) i; } return map; } public static final byte[] ENCODE_MAP = getEncodeMap(); public static final byte[] DECODE_MAP = getDecodeMap(); /** Decode a 16 byte UUID. */ public static UUID decodeUUID(String encoded) { ByteBuffer buffer = decodeBlocks(encoded); return uuidFromByteBuffer(buffer); } /** * Decode an arbitrary byte array. * *

Only `outputLength` bytes will be returned. Any extra bytes, such as padding added because * the input was unaligned, will be dropped. */ public static byte[] decodeBytes(String encoded, int outputLength) { ByteBuffer result = decodeBlocks(encoded); if (result.remaining() > outputLength) { // Only read the expected number of bytes byte[] output = new byte[outputLength]; result.get(output); return output; } else { return result.array(); } } /** * Decode an arbitrary byte array. * *

Output may contain padding bytes, if the input was not 4 byte aligned. Use [[decodeBytes]] * in that case and specify the expected number of output bytes without padding. */ public static byte[] decodeAlignedBytes(String encoded) { return decodeBlocks(encoded).array(); } /** * Decode an arbitrary byte array. * *

Output may contain padding bytes, if the input was not 4 byte aligned. */ private static ByteBuffer decodeBlocks(String encoded) { char[] input = encoded.toCharArray(); checkArgument(input.length % 5 == 0, "input should be 5 character aligned"); ByteBuffer buffer = ByteBuffer.allocate(input.length / 5 * 4); // A mechanism to detect invalid characters in the input while decoding, that only has a // single conditional at the very end, instead of branching for every character. class InputCharDecoder { int canary = 0; long decodeInputChar(int i) { char c = input[i]; canary |= c; // non-ascii char has bits outside of ASCII_BITMASK byte b = DECODE_MAP[c & ASCII_BITMASK]; canary |= b; // invalid char maps to -1, which has bits outside ASCII_BITMASK return (long) b; } } int inputIndex = 0; InputCharDecoder inputCharDecoder = new InputCharDecoder(); while (buffer.hasRemaining()) { long sum = 0; sum += inputCharDecoder.decodeInputChar(inputIndex) * BASE_4TH_POWER; sum += inputCharDecoder.decodeInputChar(inputIndex + 1) * BASE_3RD_POWER; sum += inputCharDecoder.decodeInputChar(inputIndex + 2) * BASE_2ND_POWER; sum += inputCharDecoder.decodeInputChar(inputIndex + 3) * BASE; sum += inputCharDecoder.decodeInputChar(inputIndex + 4); buffer.putInt((int) sum); inputIndex += 5; } checkArgument( (inputCharDecoder.canary & ~ASCII_BITMASK) == 0, "Input is not valid Z85: " + encoded); buffer.rewind(); return buffer; } private static UUID uuidFromByteBuffer(ByteBuffer buffer) { checkArgument(buffer.remaining() >= 16); long highBits = buffer.getLong(); long lowBits = buffer.getLong(); return new UUID(highBits, lowBits); } //////////////////////////////////////////////////////////////////////////////// // Methods implemented for testing only //////////////////////////////////////////////////////////////////////////////// /** Encode a 16 byte UUID. */ public static String encodeUUID(UUID id) { ByteBuffer buffer = uuidToByteBuffer(id); return encodeBlocks(buffer); } private static ByteBuffer uuidToByteBuffer(UUID id) { ByteBuffer buffer = ByteBuffer.allocate(16); buffer.putLong(id.getMostSignificantBits()); buffer.putLong(id.getLeastSignificantBits()); buffer.rewind(); return buffer; } /** * Encode an arbitrary byte array using 4 byte blocks. * *

Expects the input to be 4 byte aligned. */ private static String encodeBlocks(ByteBuffer buffer) { checkArgument(buffer.remaining() % 4 == 0); int numBlocks = buffer.remaining() / 4; // Every 4 byte block gets encoded into 5 bytes/chars int outputLength = numBlocks * 5; byte[] output = new byte[outputLength]; int outputIndex = 0; while (buffer.hasRemaining()) { long sum = buffer.getInt() & 0x00000000ffffffffL; output[outputIndex] = ENCODE_MAP[(int) (sum / BASE_4TH_POWER)]; sum %= BASE_4TH_POWER; output[outputIndex + 1] = ENCODE_MAP[(int) (sum / BASE_3RD_POWER)]; sum %= BASE_3RD_POWER; output[outputIndex + 2] = ENCODE_MAP[(int) (sum / BASE_2ND_POWER)]; sum %= BASE_2ND_POWER; output[outputIndex + 3] = ENCODE_MAP[(int) (sum / BASE)]; output[outputIndex + 4] = ENCODE_MAP[(int) (sum % BASE)]; outputIndex += 5; } return new String(output, US_ASCII); } /** * Encode an arbitrary byte array. * *

Unaligned input will be padded to a multiple of 4 bytes. */ public static String encodeBytes(byte[] input) { if (input.length % 4 == 0) { return encodeBlocks(ByteBuffer.wrap(input)); } else { int alignedLength = ((input.length + 4) / 4) * 4; ByteBuffer buffer = ByteBuffer.allocate(alignedLength); buffer.put(input); while (buffer.hasRemaining()) { buffer.put((byte) 0); } buffer.rewind(); return encodeBlocks(buffer); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/deletionvectors/DeletionVectorStoredBitmap.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.deletionvectors; import static io.delta.kernel.internal.DeltaErrors.wrapEngineExceptionThrowsIO; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.engine.FileReadRequest; import io.delta.kernel.engine.FileSystemClient; import io.delta.kernel.internal.actions.DeletionVectorDescriptor; import io.delta.kernel.internal.util.InternalUtils; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.utils.CloseableIterator; import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.IOException; import java.util.Optional; import java.util.zip.CRC32; /** * Bitmap for a Deletion Vector, implemented as a thin wrapper around a Deletion Vector Descriptor. * The bitmap can be empty, inline or on-disk. In case of on-disk deletion vectors, `tableDataPath` * must be set to the data path of the Delta table, which is where deletion vectors are stored. */ public class DeletionVectorStoredBitmap { private final DeletionVectorDescriptor dvDescriptor; private final Optional tableDataPath; public DeletionVectorStoredBitmap( DeletionVectorDescriptor dvDescriptor, Optional tableDataPath) { checkArgument( tableDataPath.isPresent() || !dvDescriptor.isOnDisk(), "Table path is required for on-disk deletion vectors"); this.dvDescriptor = dvDescriptor; this.tableDataPath = tableDataPath; } // TODO: for now we request 1 stream at a time public RoaringBitmapArray load(FileSystemClient fileSystemClient) throws IOException { if (dvDescriptor.getCardinality() == 0) { // isEmpty return new RoaringBitmapArray(); } else if (dvDescriptor.isInline()) { return RoaringBitmapArray.readFrom(dvDescriptor.inlineData()); } else { // isOnDisk String onDiskPath = dvDescriptor.getAbsolutePath(tableDataPath.get()); FileReadRequest dvToRead = new FileReadRequest() { @Override public String getPath() { return onDiskPath; } @Override public int getStartOffset() { return dvDescriptor.getOffset().orElse(0); } @Override public int getReadLength() { // We pad 4 bytes in the front for the size and 4 bytes at the end for // CRC-32 checksum return dvDescriptor.getSizeInBytes() + 8; } }; CloseableIterator streamIter = wrapEngineExceptionThrowsIO( () -> fileSystemClient.readFiles(Utils.singletonCloseableIterator(dvToRead)), "Reading file %s", dvToRead); ByteArrayInputStream stream = InternalUtils.getSingularElement(streamIter) .orElseThrow(() -> new IllegalStateException("Iterator should not be empty")); return loadFromStream(stream); } } /** Read a serialized deletion vector from a data stream. */ private RoaringBitmapArray loadFromStream(ByteArrayInputStream stream) throws IOException { DataInputStream dataStream = new DataInputStream(stream); try { int sizeAccordingToFile = dataStream.readInt(); if (dvDescriptor.getSizeInBytes() != sizeAccordingToFile) { throw new RuntimeException("DV size mismatch"); } byte[] buffer = new byte[sizeAccordingToFile]; dataStream.readFully(buffer); int expectedChecksum = dataStream.readInt(); int actualChecksum = calculateChecksum(buffer); if (expectedChecksum != actualChecksum) { throw new RuntimeException("DV checksum mismatch"); } return RoaringBitmapArray.readFrom(buffer); } finally { stream.close(); dataStream.close(); } } /** * Calculate checksum of a serialized deletion vector. We are using CRC32 which has 4bytes size, * but CRC32 implementation conforms to Java Checksum interface which requires a long. However, * the high-order bytes are zero, so here is safe to cast to Int. This will result in negative * checksums, but this is not a problem because we only care about equality. */ private int calculateChecksum(byte[] data) { CRC32 crc = new CRC32(); crc.update(data); return (int) crc.getValue(); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/deletionvectors/DeletionVectorUtils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.deletionvectors; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.actions.DeletionVectorDescriptor; import io.delta.kernel.internal.util.Tuple2; import java.io.IOException; import java.util.Optional; /** Utility methods regarding deletion vectors. */ public class DeletionVectorUtils { public static Tuple2 loadNewDvAndBitmap( Engine engine, String tablePath, DeletionVectorDescriptor dv) { DeletionVectorStoredBitmap storedBitmap = new DeletionVectorStoredBitmap(dv, Optional.of(tablePath)); try { RoaringBitmapArray bitmap = storedBitmap.load(engine.getFileSystemClient()); return new Tuple2<>(dv, bitmap); } catch (IOException e) { throw new RuntimeException("Couldn't load dv", e); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/deletionvectors/RoaringBitmapArray.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.deletionvectors; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.internal.util.Tuple2; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; import org.roaringbitmap.IntIterator; import org.roaringbitmap.RoaringBitmap; /** * A 64-bit extension of [[RoaringBitmap]] that is optimized for cases that usually fit within a * 32-bit bitmap, but may run over by a few bits on occasion. * *

This focus makes it different from [[org.roaringbitmap.longlong.Roaring64NavigableMap]] and * [[org.roaringbitmap.longlong.Roaring64Bitmap]] which focus on sparse bitmaps over the whole * 64-bit range. * *

Structurally, this implementation simply uses the most-significant 4 bytes to index into an * array of 32-bit [[RoaringBitmap]] instances. The array is grown as necessary to accommodate the * largest value in the bitmap. * *

*Note:* As opposed to the other two 64-bit bitmap implementations mentioned above, this * implementation cannot accommodate `Long` values where the most significant bit is non-zero (i.e., * negative `Long` values). It cannot even accommodate values where the 4 high-order bytes are * `Int.MaxValue`, because then the length of the `bitmaps` array would be a negative number * (`Int.MaxValue + 1`). * *

Taken from https://github.com/delta-io/delta/blob/master/spark/src/main/scala/org/apache/spark * /sql/delta/deletionvectors/RoaringBitmapArray.scala */ public final class RoaringBitmapArray { //////////////////////////////////////////////////////////////////////////////// // Static Fields / Methods //////////////////////////////////////////////////////////////////////////////// /** The largest value a [[RoaringBitmapArray]] can possibly represent. */ static final long MAX_REPRESENTABLE_VALUE = composeFromHighLowBytes(Integer.MAX_VALUE - 1, Integer.MIN_VALUE); /** * @param value Any `Long`; positive or negative. * @return An `Int` holding the 4 high-order bytes of information of the input `value`. */ static int highBytes(long value) { return (int) (value >> 32); } /** * @param value Any `Long`; positive or negative. * @return An `Int` holding the 4 low-order bytes of information of the input `value`. */ static int lowBytes(long value) { return (int) value; } /** * Combine high and low 4 bytes of a pair of `Int`s into a `Long`. * *

This is essentially the inverse of [[decomposeHighLowBytes()]]. * * @param high An `Int` representing the 4 high-order bytes of the output `Long` * @param low An `Int` representing the 4 low-order bytes of the output `Long` * @return A `Long` composing the `high` and `low` bytes. */ static long composeFromHighLowBytes(int high, int low) { // Must bitmask to avoid sign extension. return (((long) high) << 32) | (((long) low) & 0xFFFFFFFFL); } /** Deserialize the right instance from the given bytes */ static RoaringBitmapArray readFrom(byte[] bytes) throws IOException { ByteBuffer buffer = ByteBuffer.wrap(bytes); buffer.order(ByteOrder.LITTLE_ENDIAN); RoaringBitmapArray bitmap = new RoaringBitmapArray(); bitmap.deserialize(buffer); return bitmap; } //////////////////////////////////////////////////////////////////////////////// // Instance Fields / Methods //////////////////////////////////////////////////////////////////////////////// private RoaringBitmap[] bitmaps = new RoaringBitmap[0]; /** * Deserialize the contents of `buffer` into this [[RoaringBitmapArray]]. * *

All existing content will be discarded! * *

== Serialization Format == - A Magic Number indicating the format used (4 bytes) - The * actual data as specified by the format. */ void deserialize(ByteBuffer buffer) throws IOException { checkArgument( ByteOrder.LITTLE_ENDIAN == buffer.order(), "RoaringBitmapArray has to be deserialized using a little endian buffer"); int magicNumber = buffer.getInt(); if (magicNumber == NativeRoaringBitmapArraySerializationFormat.MAGIC_NUMBER) { bitmaps = NativeRoaringBitmapArraySerializationFormat.deserialize(buffer); } else if (magicNumber == PortableRoaringBitmapArraySerializationFormat.MAGIC_NUMBER) { bitmaps = PortableRoaringBitmapArraySerializationFormat.deserialize(buffer); } else { throw new IOException("Unexpected RoaringBitmapArray magic number " + magicNumber); } } /** * Checks whether the value is included, which is equivalent to checking if the corresponding bit * is set. */ public boolean contains(long value) { checkArgument(value >= 0 && value <= MAX_REPRESENTABLE_VALUE); int high = highBytes(value); if (high >= bitmaps.length) { return false; } else { RoaringBitmap highBitmap = bitmaps[high]; int low = lowBytes(value); return highBitmap.contains(low); } } //////////////////////////////////////////////////////////////////////////////// // Serialization Formats //////////////////////////////////////////////////////////////////////////////// /** * == Serialization Format == - Number of bitmaps (4 bytes) - For each individual bitmap: - Length * of the serialized bitmap - Serialized bitmap data using the standard format (see * https://github.com/RoaringBitmap/RoaringFormatSpec) */ static class NativeRoaringBitmapArraySerializationFormat { /** Magic number prefix for serialization with this format. */ static final int MAGIC_NUMBER = 1681511376; /** Deserialize all bitmaps from the `buffer` into a fresh array. */ static RoaringBitmap[] deserialize(ByteBuffer buffer) throws IOException { int numberOfBitmaps = buffer.getInt(); if (numberOfBitmaps < 0) { throw new IOException( String.format("Invalid RoaringBitmapArray length (%s < 0)", numberOfBitmaps)); } RoaringBitmap[] bitmaps = new RoaringBitmap[numberOfBitmaps]; for (int i = 0; i < numberOfBitmaps; i++) { bitmaps[i] = new RoaringBitmap(); int bitmapSize = buffer.getInt(); bitmaps[i].deserialize(buffer); // RoaringBitmap.deserialize doesn't move the buffer's pointer buffer.position(buffer.position() + bitmapSize); } return bitmaps; } } /** * This is the "official" portable format defined in the spec. * *

See [[https://github.com/RoaringBitmap/RoaringFormatSpec#extention-for-64-bit * -implementations]] * *

== Serialization Format == - Number of bitmaps (8 bytes, upper 4 are basically padding) - * For each individual bitmap, in increasing key order (unsigned, technically, but * RoaringBitmapArray doesn't support negative keys anyway.): - key of the bitmap (upper 32 bit) - * Serialized bitmap data using the standard format (see * https://github.com/RoaringBitmap/RoaringFormatSpec) */ static class PortableRoaringBitmapArraySerializationFormat { /** Magic number prefix for serialization with this format. */ static final int MAGIC_NUMBER = 1681511377; /** Deserialize all bitmaps from the `buffer` into a fresh array. */ static RoaringBitmap[] deserialize(ByteBuffer buffer) throws IOException { long numberOfBitmaps = buffer.getLong(); if (numberOfBitmaps < 0) { throw new IOException( String.format("Invalid RoaringBitmapArray length (%s < 0)", numberOfBitmaps)); } if (numberOfBitmaps > Integer.MAX_VALUE) { throw new IOException( String.format( "Invalid RoaringBitmapArray length (%s > %s)", numberOfBitmaps, Integer.MAX_VALUE)); } // This format is designed for sparse bitmaps, so numberOfBitmaps is only a lower bound // for the actual size of the array. int minimumArraySize = (int) numberOfBitmaps; ArrayList bitmaps = new ArrayList<>(minimumArraySize); int lastIndex = 0; for (long i = 0; i < numberOfBitmaps; i++) { int key = buffer.getInt(); if (key < 0L) { throw new IOException( String.format("Invalid unsigned entry in RoaringBitmapArray (%s)", key)); } assert key >= lastIndex : "Keys are required to be sorted in ascending order."; // Fill gaps in sparse data. while (lastIndex < key) { bitmaps.add(new RoaringBitmap()); lastIndex += 1; } RoaringBitmap bitmap = new RoaringBitmap(); bitmap.deserialize(buffer); bitmaps.add(bitmap); lastIndex += 1; // RoaringBitmap.deserialize doesn't move the buffer's pointer buffer.position(buffer.position() + bitmap.serializedSizeInBytes()); } return bitmaps.toArray(new RoaringBitmap[0]); } } //////////////////////////////////////////////////////////////////////////////// // Methods implemented for testing only //////////////////////////////////////////////////////////////////////////////// static Tuple2 decomposeHighLowBytes(long value) { return new Tuple2<>(highBytes(value), lowBytes(value)); } public void add(long value) { checkArgument(value >= 0 && value <= MAX_REPRESENTABLE_VALUE); Tuple2 tup = decomposeHighLowBytes(value); // (high, low) if (tup._1 >= bitmaps.length) { extendBitmaps(tup._1 + 1); } RoaringBitmap highBitmap = bitmaps[tup._1]; highBitmap.add(tup._2); } private void extendBitmaps(int newLength) { if (bitmaps.length == 0 && newLength == 1) { bitmaps = new RoaringBitmap[] {new RoaringBitmap()}; return; } RoaringBitmap[] newBitmaps = new RoaringBitmap[newLength]; System.arraycopy( bitmaps, // source 0, // source start pos newBitmaps, // dest 0, // dest start pos bitmaps.length); // number of entries to copy for (int i = bitmaps.length; i < newBitmaps.length; i++) { newBitmaps[i] = new RoaringBitmap(); } bitmaps = newBitmaps; } public static RoaringBitmapArray create(long... values) { RoaringBitmapArray bitmap = new RoaringBitmapArray(); for (long value : values) { bitmap.add(value); } return bitmap; } public long[] toArray() { long cardinality = 0; for (final RoaringBitmap bitmap : bitmaps) { cardinality += bitmap.getCardinality(); } checkArgument(cardinality <= Integer.MAX_VALUE, "Cardinality higher than max int value"); final long[] values = new long[(int) cardinality]; int valuesIndex = 0; for (int bitmapIndex = 0; bitmapIndex < bitmaps.length; bitmapIndex++) { final IntIterator valueIterator = bitmaps[bitmapIndex].getIntIterator(); while (valueIterator.hasNext()) { final int value = valueIterator.next(); values[valuesIndex++] = composeFromHighLowBytes(bitmapIndex, value); } } return values; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/files/LogDataUtils.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.files; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.internal.lang.ListUtils; import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; public final class LogDataUtils { private LogDataUtils() {} public static void validateLogDataContainsOnlyRatifiedStagedCommits( List logDatas) { for (ParsedLogData logData : logDatas) { checkArgument( logData instanceof ParsedCatalogCommitData && logData.isFile(), "Only staged ratified commits are supported, but found: " + logData); } } public static void validateLogDataIsSortedContiguous(List logDatas) { if (logDatas.size() > 1) { for (int i = 1; i < logDatas.size(); i++) { final ParsedLogData prev = logDatas.get(i - 1); final ParsedLogData curr = logDatas.get(i); checkArgument( prev.getVersion() + 1 == curr.getVersion(), String.format( "Log data must be sorted and contiguous, but found: %s and %s", prev, curr)); } } } /** * Combines a list of published Deltas and ratified Deltas into a single list of Deltas such that * there is exactly one {@link ParsedDeltaData} per version. When there is both a published Delta * and a ratified staged Delta for the same version, prioritizes the ratified Delta. * *

The method requires but does not validate the following: * *

    *
  • {@code publishedDeltas} are sorted and contiguous *
  • {@code ratifiedDeltas} are sorted and contiguous *
  • the commit versions present in {@code publishedDeltas} and {@code ratifiedDeltas}, when * combined, reflect a contiguous version range. In other words, if the two do not overlap, * publishedDeltas.last = ratifiedDeltas.first + 1). *
*/ public static List combinePublishedAndRatifiedDeltasWithCatalogPriority( List publishedDeltas, List ratifiedDeltas) { if (ratifiedDeltas.isEmpty()) { return publishedDeltas; } if (publishedDeltas.isEmpty()) { return ratifiedDeltas; } final long firstRatified = ratifiedDeltas.get(0).getVersion(); final long lastRatified = ListUtils.getLast(ratifiedDeltas).getVersion(); return Stream.of( publishedDeltas.stream().filter(x -> x.getVersion() < firstRatified), ratifiedDeltas.stream(), publishedDeltas.stream().filter(x -> x.getVersion() > lastRatified)) .flatMap(Function.identity()) .collect(Collectors.toList()); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/files/ParsedCatalogCommitData.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.files; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.utils.FileStatus; import java.util.Optional; /** * A catalog commit represents an atomic change to a table. * *

Can be staged and written to a staged commit file, like so: {@code * _delta_log/_staged_commits/00000000000000000001.uuid-1234.json}. * *

Can also be inline. */ public final class ParsedCatalogCommitData extends ParsedDeltaData { public static ParsedCatalogCommitData forFileStatus(FileStatus fileStatus) { checkArgument( FileNames.isStagedDeltaFile(fileStatus.getPath()), "Expected a staged commit file but got %s", fileStatus.getPath()); final String path = fileStatus.getPath(); final long version = FileNames.deltaVersion(path); return new ParsedCatalogCommitData(version, Optional.of(fileStatus), Optional.empty()); } public static ParsedCatalogCommitData forInlineData(long version, ColumnarBatch inlineData) { return new ParsedCatalogCommitData(version, Optional.empty(), Optional.of(inlineData)); } private ParsedCatalogCommitData( long version, Optional fileStatusOpt, Optional inlineDataOpt) { super(version, fileStatusOpt, inlineDataOpt); } @Override public Class getGroupByCategoryClass() { return ParsedCatalogCommitData.class; } // TODO: String getPublishedDeltaFilePath(); Requires forInlineData to take in logPath } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/files/ParsedCheckpointData.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.files; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.utils.FileStatus; import java.util.Optional; /** * Abstract checkpoint file that contains a complete snapshot of table state at a specific version. */ public abstract class ParsedCheckpointData extends ParsedLogData implements Comparable { public static ParsedCheckpointData forFileStatus(FileStatus fileStatus) { final String path = fileStatus.getPath(); if (FileNames.isClassicCheckpointFile(path)) { return ParsedClassicCheckpointData.forFileStatus(fileStatus); } else if (FileNames.isV2CheckpointFile(path)) { return ParsedV2CheckpointData.forFileStatus(fileStatus); } else if (FileNames.isMultiPartCheckpointFile(path)) { return ParsedMultiPartCheckpointData.forFileStatus(fileStatus); } else { throw new IllegalArgumentException("Unknown checkpoint file type: " + path); } } /** * Enum representing checkpoint type priorities for comparison when multiple checkpoint types * exist at the same version. Higher ordinal values indicate higher priority. */ protected enum CheckpointTypePriority { CLASSIC, // priority 0 - least preferred MULTIPART, // priority 1 - better than classic V2 // priority 2 - most preferred } protected ParsedCheckpointData( long version, Optional fileStatusOpt, Optional inlineDataOpt) { super(version, fileStatusOpt, inlineDataOpt); } /** * Returns the checkpoint type priority used as a tiebreaker when multiple checkpoint types exist * at the same version. */ protected abstract CheckpointTypePriority getCheckpointTypePriority(); /** * Compares two checkpoints of the same version and same type. Subclasses should implement * type-specific comparison logic for final tiebreaking. */ protected abstract int compareToSameType(ParsedCheckpointData that); @Override public Class getGroupByCategoryClass() { return ParsedCheckpointData.class; } /** * Compares checkpoints for ordering preference. Returns positive if *this* checkpoint is * preferred over *that* checkpoint, negative if *that* is preferred, or zero if equal. * *

Comparison hierarchy: * *

    *
  1. Version (most important): Higher version numbers are always preferred * over lower ones, as newer checkpoints contain more recent data *
  2. Checkpoint type: When versions are equal, prefer by type priority based * on safety and performance characteristics (V2 > MultiPart > Classic) *
  3. Type-specific logic: When version and type are equal, use type-specific * comparison (e.g., MultiPart prefers more parts for better parallelization) *
*/ @Override public int compareTo(ParsedCheckpointData that) { // 1. Compare versions - newer checkpoints are always preferred if (version != that.version) { return Long.compare(version, that.version); } // 2. Compare types by priority (V2 > MultiPart > Classic) CheckpointTypePriority thisTypePriority = this.getCheckpointTypePriority(); CheckpointTypePriority thatTypePriority = that.getCheckpointTypePriority(); if (thisTypePriority != thatTypePriority) { return thisTypePriority.compareTo(thatTypePriority); } // 3. Use type-specific comparison when version and type are the same return compareToSameType(that); } /** * Compares checkpoints by data source preference and deterministic tiebreaking. * *

Prefers inline data to file data because inline data is already loaded in memory, avoiding * the need for additional file I/O operations. * *

When both are files or both are inline, uses lexicographic path comparison as an arbitrary * but deterministic tiebreaker to ensure consistent ordering. */ protected final int compareByDataSource(ParsedCheckpointData that) { if (this.isInline() && that.isFile()) { return 1; // Prefer this (inline data) } else if (this.isFile() && that.isInline()) { return -1; // Prefer that (inline data) } else if (this.isFile() && that.isFile()) { // Both are files - use path as arbitrary but deterministic tiebreaker return this.getFileStatus().getPath().compareTo(that.getFileStatus().getPath()); } else { // Both are inline - no preference return 0; } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/files/ParsedChecksumData.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.files; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.utils.FileStatus; import java.util.Optional; /** * Version checksum file containing table state information for integrity validation. * *

These auxiliary files contain important metadata about the table state at a specific version * to enable detection of non-compliant modifications to Delta files. Contains information like * table size, file counts, and metadata. Example: {@code 00000000000000000001.crc} */ public final class ParsedChecksumData extends ParsedLogData { public static ParsedChecksumData forFileStatus(FileStatus fileStatus) { checkArgument( FileNames.isChecksumFile(fileStatus.getPath()), "Expected a checksum file but got %s", fileStatus.getPath()); final String path = fileStatus.getPath(); final long version = FileNames.checksumVersion(path); return new ParsedChecksumData(version, Optional.of(fileStatus), Optional.empty()); } private ParsedChecksumData( long version, Optional fileStatusOpt, Optional inlineDataOpt) { super(version, fileStatusOpt, inlineDataOpt); } @Override public Class getGroupByCategoryClass() { return ParsedChecksumData.class; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/files/ParsedClassicCheckpointData.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.files; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.utils.FileStatus; import java.util.Optional; /** * Classic checkpoint stored as a single Parquet file. * *

Example: {@code 00000000000000000001.checkpoint.parquet} */ public final class ParsedClassicCheckpointData extends ParsedCheckpointData { public static ParsedClassicCheckpointData forFileStatus(FileStatus fileStatus) { checkArgument( FileNames.isClassicCheckpointFile(fileStatus.getPath()), "Expected a classic checkpoint file but got %s", fileStatus.getPath()); final String path = fileStatus.getPath(); final long version = FileNames.checkpointVersion(path); return new ParsedClassicCheckpointData(version, Optional.of(fileStatus), Optional.empty()); } private ParsedClassicCheckpointData( long version, Optional fileStatusOpt, Optional inlineDataOpt) { super(version, fileStatusOpt, inlineDataOpt); } @Override protected CheckpointTypePriority getCheckpointTypePriority() { return CheckpointTypePriority.CLASSIC; } @Override protected int compareToSameType(ParsedCheckpointData that) { return compareByDataSource(that); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/files/ParsedDeltaData.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.files; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.utils.FileStatus; import java.util.Optional; /** Base class for Delta types that represent atomic changes to a table. */ public abstract class ParsedDeltaData extends ParsedLogData { public static ParsedDeltaData forFileStatus(FileStatus fileStatus) { final String path = fileStatus.getPath(); if (FileNames.isPublishedDeltaFile(path)) { return ParsedPublishedDeltaData.forFileStatus(fileStatus); } else if (FileNames.isStagedDeltaFile(path)) { return ParsedCatalogCommitData.forFileStatus(fileStatus); } else { throw new IllegalArgumentException("Unknown delta file type: " + path); } } protected ParsedDeltaData( long version, Optional fileStatusOpt, Optional inlineDataOpt) { super(version, fileStatusOpt, inlineDataOpt); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/files/ParsedLogCompactionData.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.files; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.internal.util.Tuple2; import io.delta.kernel.utils.FileStatus; import java.util.Objects; import java.util.Optional; /** * Log compaction file containing compacted delta entries across a version range. * *

These files compact multiple delta files into a single JSON file to reduce the number of files * readers need to process. Example: {@code * 00000000000000000001.00000000000000000009.compacted.json} represents compacted entries from * version 1 to 9. */ // TODO: Add the comparable logic from CheckpointInstance. public final class ParsedLogCompactionData extends ParsedLogData { public static ParsedLogCompactionData forFileStatus(FileStatus fileStatus) { checkArgument( FileNames.isLogCompactionFile(fileStatus.getPath()), "Expected a log compaction file but got %s", fileStatus.getPath()); final Tuple2 startEnd = FileNames.logCompactionVersions(fileStatus.getPath()); return new ParsedLogCompactionData( startEnd._1, startEnd._2, Optional.of(fileStatus), Optional.empty()); } public final long startVersion; public final long endVersion; private ParsedLogCompactionData( long startVersion, long endVersion, Optional fileStatusOpt, Optional inlineDataOpt) { super(endVersion, fileStatusOpt, inlineDataOpt); checkArgument( startVersion >= 0 && endVersion >= 0, "startVersion and endVersion must be non-negative"); checkArgument(startVersion < endVersion, "startVersion must be less than endVersion"); this.startVersion = startVersion; this.endVersion = endVersion; } @Override public Class getGroupByCategoryClass() { return ParsedLogCompactionData.class; } @Override protected void appendAdditionalToStringFields(StringBuilder sb) { sb.append(", startVersion=").append(startVersion); } @Override public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { return false; } if (!super.equals(o)) { return false; } ParsedLogCompactionData that = (ParsedLogCompactionData) o; return startVersion == that.startVersion && endVersion == that.endVersion; } @Override public int hashCode() { return Objects.hash(super.hashCode(), startVersion, endVersion); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/files/ParsedLogData.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.files; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.utils.FileStatus; import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; /** * Abstract representation of any valid log type in the Delta log. * *

Different child classes are used to represent the different log types. * *

Any given log type can be written as a file or represented inline and given to Kernel as a * {@link ColumnarBatch}. That is: Kernel just needs to know how to parse and interpret a given log * type (Kernel will of course treat Deltas differently than Checksums) as well as how to get that * log type's bytes. This is why a given log type can be represented as either a file or inline. * *

For now, our APIs only allow creating {@link ParsedCatalogCommitData} inline, but we may * change and expand this capability in the future. * *

The supported log types are: * *

    *
  • Published Deltas: {@code 00000000000000000001.json} *
  • Catalog Commits: {@code _staged_commits/00000000000000000001.uuid-1234.json} *
  • Log compaction files: {@code 00000000000000000001.00000000000000000009.compacted.json} *
  • Checksum files: {@code 00000000000000000001.crc} *
  • Classic Checkpoint files: {@code 00000000000000000001.checkpoint.parquet} *
  • V2 checkpoint files: {@code 00000000000000000001.checkpoint.uuid-1234.json} *
  • Multi-part checkpoint files: {@code * 00000000000000000001.checkpoint.0000000001.0000000010.parquet} *
*/ // TODO: Move this to be a public API public abstract class ParsedLogData { public static ParsedLogData forFileStatus(FileStatus fileStatus) { final String path = fileStatus.getPath(); if (FileNames.isCommitFile(path)) { return ParsedDeltaData.forFileStatus(fileStatus); } else if (FileNames.isCheckpointFile(path)) { return ParsedCheckpointData.forFileStatus(fileStatus); } else if (FileNames.isLogCompactionFile(path)) { return ParsedLogCompactionData.forFileStatus(fileStatus); } else if (FileNames.isChecksumFile(path)) { return ParsedChecksumData.forFileStatus(fileStatus); } else { throw new IllegalArgumentException("Unknown log file type: " + path); } } /////////////////////////////// // Member fields and methods // /////////////////////////////// protected final long version; protected final Optional fileStatusOpt; protected final Optional inlineDataOpt; protected ParsedLogData( long version, Optional fileStatusOpt, Optional inlineDataOpt) { checkArgument( fileStatusOpt.isPresent() ^ inlineDataOpt.isPresent(), "Exactly one of fileStatusOpt or inlineDataOpt must be present"); checkArgument(version >= 0, "version must be non-negative"); this.version = version; this.fileStatusOpt = fileStatusOpt; this.inlineDataOpt = inlineDataOpt; } /** * Returns true if this log data is stored as a file on disk. When false, the data is stored * inline. */ public boolean isFile() { return fileStatusOpt.isPresent(); } /** * Returns true if this log data is stored inline as a ColumnarBatch. When false, the data is * stored as a file on disk. */ public boolean isInline() { return inlineDataOpt.isPresent(); } /** Return the version of this log data. */ public long getVersion() { return version; } /** * Callers must check {@link #isFile()} before calling this method. * * @throws NoSuchElementException if {@link #isFile()} is false */ public FileStatus getFileStatus() { return fileStatusOpt.get(); } /** * Callers must check {@link #isInline()} before calling this method. * * @throws NoSuchElementException if {@link #isInline()} is false */ public ColumnarBatch getInlineData() { return inlineDataOpt.get(); } /** Returns the category class used for grouping collections of LISTed ParsedLogData instances. */ public abstract Class getGroupByCategoryClass(); /** Protected method for subclasses to override to add output to {@link #toString}. */ protected void appendAdditionalToStringFields(StringBuilder sb) { // Default implementation does nothing } @Override public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { return false; } ParsedLogData that = (ParsedLogData) o; return version == that.version && Objects.equals(fileStatusOpt, that.fileStatusOpt) && Objects.equals(inlineDataOpt, that.inlineDataOpt); } @Override public int hashCode() { return Objects.hash(version, fileStatusOpt, inlineDataOpt); } @Override public String toString() { final StringBuilder sb = new StringBuilder(getClass().getSimpleName()) .append("{version=") .append(version) .append(", source="); if (isFile()) { sb.append(fileStatusOpt.get()); } else { sb.append("inline"); } appendAdditionalToStringFields(sb); sb.append('}'); return sb.toString(); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/files/ParsedMultiPartCheckpointData.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.files; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.internal.util.Tuple2; import io.delta.kernel.utils.FileStatus; import java.util.Objects; import java.util.Optional; /** * Multi-part checkpoint split across multiple Parquet files for parallel reading. * *

Example: {@code 00000000000000000001.checkpoint.0000000001.0000000010.parquet} */ public final class ParsedMultiPartCheckpointData extends ParsedCheckpointData { public static ParsedMultiPartCheckpointData forFileStatus(FileStatus fileStatus) { checkArgument( FileNames.isMultiPartCheckpointFile(fileStatus.getPath()), "Expected a multi-part checkpoint file but got %s", fileStatus.getPath()); final long version = FileNames.checkpointVersion(fileStatus.getPath()); final Tuple2 partInfo = FileNames.multiPartCheckpointPartAndNumParts(fileStatus.getPath()); return new ParsedMultiPartCheckpointData( version, partInfo._1, partInfo._2, Optional.of(fileStatus), Optional.empty()); } public final int part; public final int numParts; private ParsedMultiPartCheckpointData( long version, int part, int numParts, Optional fileStatusOpt, Optional inlineDataOpt) { super(version, fileStatusOpt, inlineDataOpt); checkArgument(numParts > 0, "numParts must be greater than 0"); checkArgument(part > 0 && part <= numParts, "part must be between 1 and numParts"); this.part = part; this.numParts = numParts; } @Override protected CheckpointTypePriority getCheckpointTypePriority() { return CheckpointTypePriority.MULTIPART; } @Override protected int compareToSameType(ParsedCheckpointData that) { // For multi-part checkpoints, prefer more parts as they enable better parallelization if (that instanceof ParsedMultiPartCheckpointData) { ParsedMultiPartCheckpointData other = (ParsedMultiPartCheckpointData) that; int numPartsComparison = Integer.compare(this.numParts, other.numParts); if (numPartsComparison != 0) { return numPartsComparison; } } return compareByDataSource(that); } @Override protected void appendAdditionalToStringFields(StringBuilder sb) { sb.append(", part=").append(part).append(", numParts=").append(numParts); } @Override public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { return false; } if (!super.equals(o)) { return false; } ParsedMultiPartCheckpointData that = (ParsedMultiPartCheckpointData) o; return part == that.part && numParts == that.numParts; } @Override public int hashCode() { return Objects.hash(super.hashCode(), part, numParts); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/files/ParsedPublishedDeltaData.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.files; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.utils.FileStatus; import java.util.Optional; /** * A published Delta commit file represent an atomic change to a table. * *

Example: {@code _delta_log/00000000000000000001.json} */ public final class ParsedPublishedDeltaData extends ParsedDeltaData { public static ParsedPublishedDeltaData forFileStatus(FileStatus fileStatus) { checkArgument( FileNames.isPublishedDeltaFile(fileStatus.getPath()), "Expected a published Delta file but got %s", fileStatus.getPath()); final String path = fileStatus.getPath(); final long version = FileNames.deltaVersion(path); return new ParsedPublishedDeltaData(version, Optional.of(fileStatus), Optional.empty()); } private ParsedPublishedDeltaData( long version, Optional fileStatusOpt, Optional inlineDataOpt) { super(version, fileStatusOpt, inlineDataOpt); } @Override public Class getGroupByCategoryClass() { return ParsedPublishedDeltaData.class; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/files/ParsedV2CheckpointData.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.files; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.utils.FileStatus; import java.util.Optional; /** * V2 checkpoint with UUID-based naming. * *

Example: {@code 00000000000000000001.checkpoint.80a083e8-7026-4e79-81be-64bd76c43a11.json} */ public final class ParsedV2CheckpointData extends ParsedCheckpointData { public static ParsedV2CheckpointData forFileStatus(FileStatus fileStatus) { checkArgument( FileNames.isV2CheckpointFile(fileStatus.getPath()), "Expected a V2 checkpoint file but got %s", fileStatus.getPath()); final String path = fileStatus.getPath(); final long version = FileNames.checkpointVersion(path); return new ParsedV2CheckpointData(version, Optional.of(fileStatus), Optional.empty()); } private ParsedV2CheckpointData( long version, Optional fileStatusOpt, Optional inlineDataOpt) { super(version, fileStatusOpt, inlineDataOpt); } @Override protected CheckpointTypePriority getCheckpointTypePriority() { return CheckpointTypePriority.V2; } @Override protected int compareToSameType(ParsedCheckpointData that) { return compareByDataSource(that); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/fs/Path.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.fs; import java.io.InvalidObjectException; import java.io.ObjectInputValidation; import java.io.Serializable; import java.net.URI; import java.net.URISyntaxException; import java.util.regex.Pattern; /** * Names a file or directory in a FileSystem. Path strings use slash as the directory separator. * *

Taken from https://github.com/apache/hadoop/blob/branch-3.3 * .4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java */ public class Path implements Comparable, Serializable, ObjectInputValidation { /** The directory separator, a slash. */ public static final String SEPARATOR = "/"; /** The directory separator, a slash, as a character. */ public static final char SEPARATOR_CHAR = '/'; /** The current directory, ".". */ public static final String CUR_DIR = "."; /** Whether the current host is a Windows machine. */ public static final boolean WINDOWS = System.getProperty("os.name").startsWith("Windows"); /** Pre-compiled regular expressions to detect path formats. */ private static final Pattern HAS_DRIVE_LETTER_SPECIFIER = Pattern.compile("^/?[a-zA-Z]:"); /** Pre-compiled regular expressions to detect duplicated slashes. */ private static final Pattern SLASHES = Pattern.compile("/+"); private static final long serialVersionUID = 0xad00f; private URI uri; // a hierarchical uri /** * Create a new Path based on the child path resolved against the parent path. * * @param parent the parent path * @param child the child path */ public Path(String parent, String child) { this(new Path(parent), new Path(child)); } /** * Create a new Path based on the child path resolved against the parent path. * * @param parent the parent path * @param child the child path */ public Path(Path parent, String child) { this(parent, new Path(child)); } /** * Create a new Path based on the child path resolved against the parent path. * * @param parent the parent path * @param child the child path */ public Path(String parent, Path child) { this(new Path(parent), child); } /** * Create a new Path based on the child path resolved against the parent path. * * @param parent the parent path * @param child the child path */ public Path(Path parent, Path child) { // Add a slash to parent's path so resolution is compatible with URI's URI parentUri = parent.uri; String parentPath = parentUri.getPath(); if (!(parentPath.equals("/") || parentPath.isEmpty())) { try { parentUri = new URI( parentUri.getScheme(), parentUri.getAuthority(), parentUri.getPath() + "/", null, parentUri.getFragment()); } catch (URISyntaxException e) { throw new IllegalArgumentException(e); } } URI resolved = parentUri.resolve(child.uri); initialize( resolved.getScheme(), resolved.getAuthority(), resolved.getPath(), resolved.getFragment()); } private void checkPathArg(String path) throws IllegalArgumentException { // disallow construction of a Path from an empty string if (path == null) { throw new IllegalArgumentException("Can not create a Path from a null string"); } if (path.length() == 0) { throw new IllegalArgumentException("Can not create a Path from an empty string"); } } /** * Construct a path from a String. Path strings are URIs, but with unescaped elements and some * additional normalization. * * @param pathString the path string */ public Path(String pathString) throws IllegalArgumentException { checkPathArg(pathString); // We can't use 'new URI(String)' directly, since it assumes things are // escaped, which we don't require of Paths. // add a slash in front of paths with Windows drive letters if (hasWindowsDrive(pathString) && pathString.charAt(0) != '/') { pathString = "/" + pathString; } // parse uri components String scheme = null; String authority = null; int start = 0; // parse uri scheme, if any int colon = pathString.indexOf(':'); int slash = pathString.indexOf('/'); if ((colon != -1) && ((slash == -1) || (colon < slash))) { // has a scheme scheme = pathString.substring(0, colon); start = colon + 1; } // parse uri authority, if any if (pathString.startsWith("//", start) && (pathString.length() - start > 2)) { // has authority int nextSlash = pathString.indexOf('/', start + 2); int authEnd = nextSlash > 0 ? nextSlash : pathString.length(); authority = pathString.substring(start + 2, authEnd); start = authEnd; } // uri path is the rest of the string -- query & fragment not supported String path = pathString.substring(start, pathString.length()); initialize(scheme, authority, path, null); } /** * Construct a path from a URI * * @param aUri the source URI */ public Path(URI aUri) { uri = aUri.normalize(); } /** * Construct a Path from components. * * @param scheme the scheme * @param authority the authority * @param path the path */ public Path(String scheme, String authority, String path) { checkPathArg(path); // add a slash in front of paths with Windows drive letters if (hasWindowsDrive(path) && path.charAt(0) != '/') { path = "/" + path; } // add "./" in front of Linux relative paths so that a path containing // a colon e.q. "a:b" will not be interpreted as scheme "a". if (!WINDOWS && path.charAt(0) != '/') { path = "./" + path; } initialize(scheme, authority, path, null); } private void initialize(String scheme, String authority, String path, String fragment) { try { this.uri = new URI(scheme, authority, normalizePath(scheme, path), null, fragment).normalize(); } catch (URISyntaxException e) { throw new IllegalArgumentException(e); } } /** * Normalize a path string to use non-duplicated forward slashes as the path separator and remove * any trailing path separators. * * @param scheme the URI scheme. Used to deduce whether we should replace backslashes or not * @param path the scheme-specific part * @return the normalized path string */ private static String normalizePath(String scheme, String path) { // In most cases the path is expected to not have repeated slashes. // Validating this first before applying the regex saves ~40-50% of // Path construction time, with a potentially small overhead for // cases when paths need normalization. if (containsRepeatedSlash(path)) { // Remove duplicated slashes to ensure all equivalent Path's have // the same representation. path = SLASHES.matcher(path).replaceAll("/"); } // Remove backslashes if this looks like a Windows path. Avoid // the substitution if it looks like a non-local URI. if (WINDOWS && (hasWindowsDrive(path) || (scheme == null) || (scheme.isEmpty()) || (scheme.equals("file")))) { path = path.replace("\\", "/"); } // trim trailing slash from non-root path (ignoring windows drive) int minLength = startPositionWithoutWindowsDrive(path) + 1; if (path.length() > minLength && path.endsWith(SEPARATOR)) { path = path.substring(0, path.length() - 1); } return path; } private static boolean containsRepeatedSlash(String path) { // Not inlining this method back into normalizePath() appears // to be slightly faster (there is a high probability this is noise // but keep out-of-line for now until there is definitive evidence // one way or another). return path.contains("//"); } private static boolean hasWindowsDrive(String path) { return (WINDOWS && HAS_DRIVE_LETTER_SPECIFIER.matcher(path).find()); } private static int startPositionWithoutWindowsDrive(String path) { if (hasWindowsDrive(path)) { return path.charAt(0) == SEPARATOR_CHAR ? 3 : 2; } else { return 0; } } /** * Convert this Path to a URI. * * @return this Path as a URI */ public URI toUri() { return uri; } /** * Returns true if the path component (i.e. directory) of this URI is absolute. * * @return whether this URI's path is absolute */ public boolean isUriPathAbsolute() { int start = startPositionWithoutWindowsDrive(uri.getPath()); return uri.getPath().startsWith(SEPARATOR, start); } /** * Returns true if the path component (i.e. directory) of this URI is absolute. This method is a * wrapper for {@link #isUriPathAbsolute()}. * * @return whether this URI's path is absolute */ public boolean isAbsolute() { return isUriPathAbsolute(); } /** * Returns true if and only if this path represents the root of a file system. * * @return true if and only if this path represents the root of a file system */ public boolean isRoot() { return getParent() == null; } /** * Returns the final component of this path. * * @return the final component of this path */ public String getName() { String path = uri.getPath(); int slash = path.lastIndexOf(SEPARATOR); return path.substring(slash + 1); } /** * Returns the parent of a path or null if at root. * * @return the parent of a path or null if at root */ public Path getParent() { String path = uri.getPath(); int lastSlash = path.lastIndexOf('/'); int start = startPositionWithoutWindowsDrive(path); if ((path.length() == start) || // empty path (lastSlash == start && path.length() == start + 1)) { // at root return null; } String parent; if (lastSlash == -1) { parent = CUR_DIR; } else { parent = path.substring(0, lastSlash == start ? start + 1 : lastSlash); } return new Path(uri.getScheme(), uri.getAuthority(), parent); } @Override public String toString() { // we can't use uri.toString(), which escapes everything, because we want // illegal characters unescaped in the string, for glob processing, etc. StringBuilder buffer = new StringBuilder(); if (uri.getScheme() != null) { buffer.append(uri.getScheme()).append(":"); } if (uri.getAuthority() != null) { buffer.append("//").append(uri.getAuthority()); } if (uri.getPath() != null) { String path = uri.getPath(); if (path.indexOf('/') == 0 && hasWindowsDrive(path) && // has windows drive uri.getScheme() == null && // but no scheme uri.getAuthority() == null) { // or authority path = path.substring(1); // remove slash before drive } buffer.append(path); } if (uri.getFragment() != null) { buffer.append("#").append(uri.getFragment()); } return buffer.toString(); } @Override public boolean equals(Object o) { if (!(o instanceof Path)) { return false; } Path that = (Path) o; return this.uri.equals(that.uri); } @Override public int hashCode() { return uri.hashCode(); } @Override public int compareTo(Path o) { return this.uri.compareTo(o.uri); } /** * Validate the contents of a deserialized Path, so as to defend against malicious object streams. * * @throws InvalidObjectException if there's no URI */ @Override public void validateObject() throws InvalidObjectException { if (uri == null) { throw new InvalidObjectException("No URI in deserialized Path"); } } public static String getName(String pathString) { return new Path(pathString).getName(); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/hook/CheckpointHook.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.hook; import io.delta.kernel.Table; import io.delta.kernel.engine.Engine; import io.delta.kernel.hook.PostCommitHook; import io.delta.kernel.internal.fs.Path; import java.io.IOException; /** Write a new checkpoint at the version committed by the txn. */ public class CheckpointHook implements PostCommitHook { private final Path tablePath; private final long checkpointVersion; public CheckpointHook(Path tablePath, long checkpointVersion) { this.tablePath = tablePath; this.checkpointVersion = checkpointVersion; } @Override public void threadSafeInvoke(Engine engine) throws IOException { Table.forPath(engine, tablePath.toString()).checkpoint(engine, checkpointVersion); } @Override public PostCommitHookType getType() { return PostCommitHookType.CHECKPOINT; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/hook/ChecksumFullHook.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.hook; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.Table; import io.delta.kernel.engine.Engine; import io.delta.kernel.hook.PostCommitHook; import io.delta.kernel.internal.fs.Path; import java.io.IOException; /** * A post-commit hook that writes a new checksum file at the version committed by the transaction. * This hook performs a writing checksum operation with table state construction for log replay. */ public class ChecksumFullHook implements PostCommitHook { private final Path tablePath; private final long version; public ChecksumFullHook(Path tablePath, long version) { this.tablePath = requireNonNull(tablePath); this.version = version; } @Override public void threadSafeInvoke(Engine engine) throws IOException { checkArgument(engine != null); Table.forPath(engine, tablePath.toString()).checksum(engine, version); } @Override public PostCommitHookType getType() { return PostCommitHookType.CHECKSUM_FULL; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/hook/ChecksumSimpleHook.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.hook; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.engine.Engine; import io.delta.kernel.hook.PostCommitHook; import io.delta.kernel.internal.checksum.CRCInfo; import io.delta.kernel.internal.checksum.ChecksumWriter; import io.delta.kernel.internal.fs.Path; import java.io.IOException; /** * A post-commit hook that writes a new checksum file at the version committed by the transaction. * This hook performs a simple checksum operation without requiring previous checkpoint or log * reading. */ public class ChecksumSimpleHook implements PostCommitHook { private final CRCInfo crcInfo; private final Path logPath; public ChecksumSimpleHook(CRCInfo crcInfo, Path logPath) { this.crcInfo = requireNonNull(crcInfo); this.logPath = requireNonNull(logPath); } @Override public void threadSafeInvoke(Engine engine) throws IOException { checkArgument(engine != null); new ChecksumWriter(logPath).writeCheckSum(engine, crcInfo); } @Override public PostCommitHookType getType() { return PostCommitHookType.CHECKSUM_SIMPLE; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/hook/LogCompactionHook.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.hook; import static java.util.Objects.requireNonNull; import io.delta.kernel.engine.Engine; import io.delta.kernel.hook.PostCommitHook; import io.delta.kernel.internal.annotation.VisibleForTesting; import io.delta.kernel.internal.compaction.LogCompactionWriter; import io.delta.kernel.internal.fs.Path; import java.io.IOException; /** * A post-commit hook that performs inline log compaction. It merges commit JSON files over a * compaction interval into a single compacted JSON file. */ public class LogCompactionHook implements PostCommitHook { private final Path dataPath; private final Path logPath; private final long startVersion; private final long commitVersion; private final long minFileRetentionTimestampMillis; public LogCompactionHook( Path dataPath, Path logPath, long startVersion, long commitVersion, long minFileRetentionTimestampMillis) { this.dataPath = requireNonNull(dataPath, "dataPath cannot be null"); this.logPath = requireNonNull(logPath, "logPath cannot be null"); this.startVersion = startVersion; this.commitVersion = commitVersion; this.minFileRetentionTimestampMillis = minFileRetentionTimestampMillis; } @Override public void threadSafeInvoke(Engine engine) throws IOException { LogCompactionWriter compactionWriter = new LogCompactionWriter( dataPath, logPath, startVersion, commitVersion, minFileRetentionTimestampMillis); compactionWriter.writeLogCompactionFile(engine); } @Override public PostCommitHookType getType() { return PostCommitHookType.LOG_COMPACTION; } @VisibleForTesting public Path getDataPath() { return dataPath; } @VisibleForTesting public Path getLogPath() { return logPath; } @VisibleForTesting public long getStartVersion() { return startVersion; } @VisibleForTesting public long getCommitVersion() { return commitVersion; } @VisibleForTesting public long getMinFileRetentionTimestampMillis() { return minFileRetentionTimestampMillis; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatMetadataValidatorAndUpdater.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.icebergcompat; import static io.delta.kernel.internal.tablefeatures.TableFeatures.*; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Collections.singletonMap; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.internal.DeltaErrors; import io.delta.kernel.internal.TableConfig; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.tablefeatures.TableFeature; import io.delta.kernel.internal.types.TypeWideningChecker; import io.delta.kernel.internal.util.ColumnMapping; import io.delta.kernel.internal.util.SchemaIterable; import io.delta.kernel.types.*; import io.delta.kernel.utils.DataFileStatus; import java.util.*; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; /** * Contains interfaces and common utility classes for defining the iceberg conversion compatibility * checks and metadata updates. * *

Main class is {@link IcebergCompatMetadataValidatorAndUpdater} which takes: * *

    *
  • {@link TableConfig} to check if the table is enabled iceberg compat property enabled. When * enabled, the metadata will be validated and updated. *
  • List of {@link TableFeature}s expected to be supported by the protocol *
  • List of {@link IcebergCompatRequiredTablePropertyEnforcer} that enforce certain properties * must be set for IcebergV2 compatibility. If the property is not set, we will set it to a * default value. It will also update the metadata to make it compatible with Iceberg compat * version targeted. *
  • List of {@link IcebergCompatCheck} to validate the metadata and protocol. The checks can be * like what are the table features not supported and in what cases a certain table feature is * supported (e.g. type widening is enabled, but iceberg compat only if the widening is * supported in the Iceberg). *
*/ public abstract class IcebergCompatMetadataValidatorAndUpdater { /** * Returns whether Iceberg compatibility is enabled for the given table metadata. This checks if * either `icebergCompatV2` or `icebergCompatV3` table property is enabled. * * @param metadata The table metadata to check. * @return true if either Iceberg compatibility V2 or V3 is enabled; false otherwise. */ public static Boolean isIcebergCompatEnabled(Metadata metadata) { return TableConfig.ICEBERG_COMPAT_V2_ENABLED.fromMetadata(metadata) || TableConfig.ICEBERG_COMPAT_V3_ENABLED.fromMetadata(metadata); } ///////////////////////////////////////////////////////////////////////////////// /// Interfaces for defining checks for the compat validation and updating /// ///////////////////////////////////////////////////////////////////////////////// /** Defines the input context for the metadata validator and updater. */ public static class IcebergCompatInputContext { final String compatFeatureName; final boolean isCreatingNewTable; final Metadata newMetadata; final Protocol newProtocol; /** * The protocol from the previous snapshot, used to guard against concurrent writers that may * have enabled incompatible features (e.g. deletion vectors). Empty for new tables. */ final Optional prevProtocol; public IcebergCompatInputContext( String compatFeatureName, boolean isCreatingNewTable, Metadata newMetadata, Protocol newProtocol, Optional prevProtocol) { this.compatFeatureName = compatFeatureName; this.isCreatingNewTable = isCreatingNewTable; this.newMetadata = newMetadata; this.newProtocol = newProtocol; this.prevProtocol = prevProtocol; } public IcebergCompatInputContext withUpdatedMetadata(Metadata newMetadata) { return new IcebergCompatInputContext( compatFeatureName, isCreatingNewTable, newMetadata, newProtocol, prevProtocol); } } /** Defines a callback to post-process the metadata. */ interface PostMetadataProcessor { Optional postProcess(IcebergCompatInputContext inputContext); } /** * Defines a required table property that must be set for IcebergV2 compatibility. If the property * is not set, we will set it to a default value. It will also update the metadata to make it * compatible with Iceberg compat version targeted. */ protected static class IcebergCompatRequiredTablePropertyEnforcer { public final TableConfig property; public final Predicate validator; public final String autoSetValue; public final PostMetadataProcessor postMetadataProcessor; /** * Constructor for RequiredDeltaTableProperty * * @param property DeltaConfig we are checking * @param validator A generic method to validate the given value * @param autoSetValue The value to set if we can auto-set this value (e.g. during table * creation) * @param postMetadataProcessor A callback to post-process the metadata */ IcebergCompatRequiredTablePropertyEnforcer( TableConfig property, Predicate validator, String autoSetValue, PostMetadataProcessor postMetadataProcessor) { this.property = property; this.validator = validator; this.autoSetValue = autoSetValue; this.postMetadataProcessor = postMetadataProcessor; } /** * Constructor for RequiredDeltaTableProperty * * @param property DeltaConfig we are checking * @param validator A generic method to validate the given value * @param autoSetValue The value to set if we can auto-set this value (e.g. during table * creation) */ IcebergCompatRequiredTablePropertyEnforcer( TableConfig property, Predicate validator, String autoSetValue) { this(property, validator, autoSetValue, (c) -> Optional.empty()); } Optional validateAndUpdate( IcebergCompatInputContext inputContext, String compatVersion) { Metadata newMetadata = inputContext.newMetadata; T newestValue = property.fromMetadata(newMetadata); boolean newestValueOkay = validator.test(newestValue); boolean newestValueExplicitlySet = newMetadata.getConfiguration().containsKey(property.getKey()); if (!newestValueOkay) { if (!newestValueExplicitlySet && inputContext.isCreatingNewTable) { // Covers the case CREATE that did not explicitly specify the required table property. // In these cases, we set the property automatically. newMetadata = newMetadata.withMergedConfiguration(singletonMap(property.getKey(), autoSetValue)); return Optional.of(newMetadata); } else { // In all other cases, if the property value is not compatible throw new KernelException( String.format( "The value '%s' for the property '%s' is not compatible with " + "%s requirements", newestValue, property.getKey(), compatVersion)); } } return Optional.empty(); } } ///////////////////////////////////////////////////////////////////////////////// /// Implementation of {@link IcebergCompatRequiredTablePropertyEnforcer} /// ///////////////////////////////////////////////////////////////////////////////// protected static final IcebergCompatRequiredTablePropertyEnforcer COLUMN_MAPPING_REQUIREMENT = new IcebergCompatRequiredTablePropertyEnforcer<>( TableConfig.COLUMN_MAPPING_MODE, (value) -> ColumnMapping.ColumnMappingMode.NAME == value || ColumnMapping.ColumnMappingMode.ID == value, ColumnMapping.ColumnMappingMode.NAME.value); protected static final IcebergCompatRequiredTablePropertyEnforcer ROW_TRACKING_ENABLED = new IcebergCompatRequiredTablePropertyEnforcer<>( TableConfig.ROW_TRACKING_ENABLED, (value) -> value, "true"); /** * Defines checks for compatibility with the targeted iceberg features (icebergCompatV1 or * icebergCompatV2 etc.) */ protected interface IcebergCompatCheck { void check(IcebergCompatInputContext inputContext); } /////////////////////////////////////////////////////////// /// Implementation of {@link IcebergCompatCheck} /// /////////////////////////////////////////////////////////// protected static IcebergCompatCheck disallowOtherCompatVersions(List incompatibleProps) { return (inputContext) -> { for (String prop : incompatibleProps) { if (Boolean.parseBoolean( inputContext.newMetadata.getConfiguration().getOrDefault(prop, "false"))) { throw DeltaErrors.icebergCompatIncompatibleVersionEnabled( inputContext.compatFeatureName, prop); } } }; } protected static final IcebergCompatCheck CHECK_ONLY_ICEBERG_COMPAT_V2_ENABLED = disallowOtherCompatVersions( Arrays.asList("delta.enableIcebergCompatV1", "delta.enableIcebergCompatV3")); protected static final IcebergCompatCheck CHECK_ONLY_ICEBERG_COMPAT_V3_ENABLED = disallowOtherCompatVersions( Arrays.asList("delta.enableIcebergCompatV1", "delta.enableIcebergCompatV2")); protected static IcebergCompatCheck hasOnlySupportedTypes( Set> supportedTypes) { return (inputContext) -> { Set matches = new SchemaIterable(inputContext.newMetadata.getSchema()) .stream() .map(element -> element.getField().getDataType()) .filter( dataType -> { for (Class clazz : supportedTypes) { if (clazz.isInstance(dataType)) return false; } return true; }) .collect(Collectors.toSet()); if (!matches.isEmpty()) { List unsupportedTypes = new ArrayList<>(matches); unsupportedTypes.sort(Comparator.comparing(DataType::toString)); throw DeltaErrors.icebergCompatUnsupportedTypeColumns( inputContext.compatFeatureName, unsupportedTypes); } }; } private static final Set> V2_SUPPORTED_TYPES = new HashSet<>( Arrays.asList( ByteType.class, ShortType.class, IntegerType.class, LongType.class, FloatType.class, DoubleType.class, DecimalType.class, StringType.class, BinaryType.class, BooleanType.class, DateType.class, TimestampType.class, TimestampNTZType.class, ArrayType.class, MapType.class, StructType.class)); private static final Set> V3_SUPPORTED_TYPES = Stream.concat(V2_SUPPORTED_TYPES.stream(), Stream.of(VariantType.class)) .collect(Collectors.toSet()); protected static final IcebergCompatCheck V2_CHECK_HAS_SUPPORTED_TYPES = hasOnlySupportedTypes(V2_SUPPORTED_TYPES); protected static final IcebergCompatCheck V3_CHECK_HAS_SUPPORTED_TYPES = hasOnlySupportedTypes(V3_SUPPORTED_TYPES); // These are the common supported partition types for both Iceberg compat V2 and V3 protected static final IcebergCompatCheck CHECK_HAS_ALLOWED_PARTITION_TYPES = (inputContext) -> inputContext .newMetadata .getPartitionColNames() .forEach( partitionCol -> { int partitionFieldIndex = inputContext.newMetadata.getSchema().indexOf(partitionCol); checkArgument( partitionFieldIndex != -1, "Partition column %s not found in the schema", partitionCol); DataType dataType = inputContext.newMetadata.getSchema().at(partitionFieldIndex).getDataType(); boolean validType = dataType instanceof ByteType || dataType instanceof ShortType || dataType instanceof IntegerType || dataType instanceof LongType || dataType instanceof FloatType || dataType instanceof DoubleType || dataType instanceof DecimalType || dataType instanceof StringType || dataType instanceof BinaryType || dataType instanceof BooleanType || dataType instanceof DateType || dataType instanceof TimestampType || dataType instanceof TimestampNTZType; if (!validType) { throw DeltaErrors.icebergCompatUnsupportedTypePartitionColumn( inputContext.compatFeatureName, dataType); } }); protected static final IcebergCompatCheck CHECK_HAS_NO_PARTITION_EVOLUTION = (inputContext) -> { // TODO: Kernel doesn't support replace table yet. When it is supported, extend // this to allow checking the partition columns aren't changed }; protected static final IcebergCompatCheck CHECK_HAS_NO_DELETION_VECTORS = (inputContext) -> { // Check both newProtocol and prevProtocol as defense-in-depth against cases where // the previous snapshot already had deletion vectors enabled. This matches Spark's // CheckDeletionVectorDisabled which checks both prevSnapshot and newestProtocol. // Note: the conflict checker may also catch concurrent DV enablement at commit time. boolean dvInNewProtocol = inputContext.newProtocol.supportsFeature(DELETION_VECTORS_RW_FEATURE); boolean dvInPrevProtocol = inputContext .prevProtocol .map(prev -> prev.supportsFeature(DELETION_VECTORS_RW_FEATURE)) .orElse(false); if (dvInNewProtocol || dvInPrevProtocol) { throw DeltaErrors.icebergCompatIncompatibleTableFeatures( inputContext.compatFeatureName, Collections.singleton(DELETION_VECTORS_RW_FEATURE)); } }; protected static final IcebergCompatCheck CHECK_HAS_SUPPORTED_TYPE_WIDENING = (inputContext) -> { Protocol protocol = inputContext.newProtocol; if (!protocol.supportsFeature(TYPE_WIDENING_RW_FEATURE) && !protocol.supportsFeature(TYPE_WIDENING_RW_PREVIEW_FEATURE)) { return; } for (SchemaIterable.SchemaElement element : new SchemaIterable(inputContext.newMetadata.getSchema())) { for (TypeChange typeChange : element.getField().getTypeChanges()) { if (!TypeWideningChecker.isIcebergV2Compatible( typeChange.getFrom(), typeChange.getTo())) { throw DeltaErrors.icebergCompatUnsupportedTypeWidening( inputContext.compatFeatureName, typeChange); } } } }; ///////////////////////////////////////////////////////////////////////////////// /// Implementation of {@link IcebergCompatMetadataValidatorAndUpdater} /// ///////////////////////////////////////////////////////////////////////////////// /** * If the iceberg compat is enabled, validate and update the metadata for Iceberg compatibility. * * @param inputContext input containing the metadata, protocol, if the table is being created etc. * @return the updated metadata. If no updates are done, then returns empty * @throws {@link io.delta.kernel.exceptions.KernelException} for any validation errors */ Optional validateAndUpdateMetadata(IcebergCompatInputContext inputContext) { if (!requiredDeltaTableProperty().fromMetadata(inputContext.newMetadata)) { return Optional.empty(); } boolean metadataUpdated = false; // table property checks and metadata updates List requiredDeltaTableProperties = requiredDeltaTableProperties(); for (IcebergCompatRequiredTablePropertyEnforcer requiredDeltaTableProperty : requiredDeltaTableProperties) { Optional updated = requiredDeltaTableProperty.validateAndUpdate(inputContext, compatFeatureName()); if (updated.isPresent()) { inputContext = inputContext.withUpdatedMetadata(updated.get()); metadataUpdated = true; } } // post-process metadata after the table property checks are done and updated for (IcebergCompatRequiredTablePropertyEnforcer requiredDeltaTableProperty : requiredDeltaTableProperties) { Optional updated = requiredDeltaTableProperty.postMetadataProcessor.postProcess(inputContext); if (updated.isPresent()) { metadataUpdated = true; inputContext = inputContext.withUpdatedMetadata(updated.get()); } } // check for required dependency table features for (TableFeature requiredDependencyTableFeature : requiredDependencyTableFeatures()) { if (!inputContext.newProtocol.supportsFeature(requiredDependencyTableFeature)) { throw DeltaErrors.icebergCompatRequiredFeatureMissing( compatFeatureName(), requiredDependencyTableFeature.featureName()); } } // check for Iceberg compatibility checks for (IcebergCompatCheck icebergCompatCheck : icebergCompatChecks()) { icebergCompatCheck.check(inputContext); } return metadataUpdated ? Optional.of(inputContext.newMetadata) : Optional.empty(); } abstract String compatFeatureName(); abstract TableConfig requiredDeltaTableProperty(); abstract List requiredDeltaTableProperties(); abstract List requiredDependencyTableFeatures(); abstract List icebergCompatChecks(); ///////////////////////////// /// Helper function /// ///////////////////////////// /** * Validate the given {@link DataFileStatus} that is being added as a {@code add} action to Delta * Log. Currently, it checks that the statistics are not empty. * * @param dataFileStatus The {@link DataFileStatus} to validate. * @param compatFeatureName The name of the compatibility feature being validated (e.g. * "icebergCompatV2"). */ protected static void validateDataFileStatus( DataFileStatus dataFileStatus, String compatFeatureName) { if (!dataFileStatus.getStatistics().isPresent()) { // presence of stats means always has a non-null `numRecords` throw DeltaErrors.icebergCompatMissingNumRecordsStats(compatFeatureName, dataFileStatus); } } /** * Block the Iceberg Compat config related changes that we do not support and for which we throw * an {@link KernelException}, * *
    *
  • Disabling on an existing table (true to false) *
  • Enabling on an existing table (false to true) *
*/ protected static void blockConfigChangeOnExistingTable( TableConfig tableConfig, Map oldConfig, Map newConfig) { boolean wasEnabled = tableConfig.fromMetadata(oldConfig); boolean isEnabled = tableConfig.fromMetadata(newConfig); if (!wasEnabled && isEnabled) { throw DeltaErrors.enablingIcebergCompatFeatureOnExistingTable(tableConfig.getKey()); } if (wasEnabled && !isEnabled) { throw DeltaErrors.disablingIcebergCompatFeatureOnExistingTable(tableConfig.getKey()); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatV2MetadataValidatorAndUpdater.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.icebergcompat; import static io.delta.kernel.internal.tablefeatures.TableFeatures.*; import static java.util.Collections.singletonList; import static java.util.stream.Collectors.toList; import io.delta.kernel.internal.TableConfig; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.tablefeatures.TableFeature; import io.delta.kernel.types.*; import io.delta.kernel.utils.DataFileStatus; import java.util.List; import java.util.Optional; import java.util.stream.Stream; /** Utility methods for validation and compatibility checks for Iceberg V2. */ public class IcebergCompatV2MetadataValidatorAndUpdater extends IcebergCompatMetadataValidatorAndUpdater { /** * Validate and update the given Iceberg V2 metadata. * * @param newMetadata Metadata after the current updates * @param newProtocol Protocol after the current updates * @return The updated metadata if the metadata is valid and updated, otherwise empty. * @throws UnsupportedOperationException if the metadata is not compatible with Iceberg V2 * requirements */ public static Optional validateAndUpdateIcebergCompatV2Metadata( boolean isCreatingNewTable, Metadata newMetadata, Protocol newProtocol, Optional prevProtocol) { return INSTANCE.validateAndUpdateMetadata( new IcebergCompatInputContext( INSTANCE.compatFeatureName(), isCreatingNewTable, newMetadata, newProtocol, prevProtocol)); } /** * Validate the given {@link DataFileStatus} that is being added as a {@code add} action to Delta * Log. Currently, it checks that the statistics are not empty. * * @param dataFileStatus The {@link DataFileStatus} to validate. */ public static void validateDataFileStatus(DataFileStatus dataFileStatus) { validateDataFileStatus(dataFileStatus, INSTANCE.compatFeatureName()); } /// ////////////////////////////////////////////////////////////////////////////// /// Define the compatibility and update checks for icebergCompatV2 /// /// ////////////////////////////////////////////////////////////////////////////// private static final IcebergCompatV2MetadataValidatorAndUpdater INSTANCE = new IcebergCompatV2MetadataValidatorAndUpdater(); @Override String compatFeatureName() { return "icebergCompatV2"; } @Override TableConfig requiredDeltaTableProperty() { return TableConfig.ICEBERG_COMPAT_V2_ENABLED; } @Override List requiredDeltaTableProperties() { return singletonList(COLUMN_MAPPING_REQUIREMENT); } @Override List requiredDependencyTableFeatures() { return Stream.of(ICEBERG_COMPAT_V2_W_FEATURE, COLUMN_MAPPING_RW_FEATURE).collect(toList()); } @Override List icebergCompatChecks() { return Stream.of( CHECK_ONLY_ICEBERG_COMPAT_V2_ENABLED, V2_CHECK_HAS_SUPPORTED_TYPES, CHECK_HAS_ALLOWED_PARTITION_TYPES, CHECK_HAS_NO_PARTITION_EVOLUTION, CHECK_HAS_NO_DELETION_VECTORS, CHECK_HAS_SUPPORTED_TYPE_WIDENING) .collect(toList()); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatV3MetadataValidatorAndUpdater.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.icebergcompat; import static io.delta.kernel.internal.tablefeatures.TableFeatures.*; import static java.util.stream.Collectors.toList; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.internal.TableConfig; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.columndefaults.ColumnDefaults; import io.delta.kernel.internal.tablefeatures.TableFeature; import io.delta.kernel.utils.DataFileStatus; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Stream; /** Utility methods for validation and compatibility checks for Iceberg V3. */ public class IcebergCompatV3MetadataValidatorAndUpdater extends IcebergCompatMetadataValidatorAndUpdater { /** * Validates that any change to property {@link TableConfig#ICEBERG_COMPAT_V3_ENABLED} is valid. * Currently, the changes we support are * *
    *
  • No change in enablement (true to true or false to false) *
* * The changes that we do not support and for which we throw an {@link KernelException} are * *
    *
  • Disabling on an existing table (true to false) *
  • Enabling on an existing table (false to true) *
*/ public static void validateIcebergCompatV3Change( Map oldConfig, Map newConfig) { blockConfigChangeOnExistingTable(TableConfig.ICEBERG_COMPAT_V3_ENABLED, oldConfig, newConfig); } /** * Validate and update the given Iceberg V3 metadata. * * @param newMetadata Metadata after the current updates * @param newProtocol Protocol after the current updates * @return The updated metadata if the metadata is valid and updated, otherwise empty. * @throws UnsupportedOperationException if the metadata is not compatible with Iceberg V3 * requirements */ public static Optional validateAndUpdateIcebergCompatV3Metadata( boolean isCreatingNewTable, Metadata newMetadata, Protocol newProtocol, Optional prevProtocol) { return INSTANCE.validateAndUpdateMetadata( new IcebergCompatInputContext( INSTANCE.compatFeatureName(), isCreatingNewTable, newMetadata, newProtocol, prevProtocol)); } /** * Validate the given {@link DataFileStatus} that is being added as a {@code add} action to Delta * Log. Currently, it checks that the statistics are not empty. * * @param dataFileStatus The {@link DataFileStatus} to validate. */ public static void validateDataFileStatus(DataFileStatus dataFileStatus) { validateDataFileStatus(dataFileStatus, INSTANCE.compatFeatureName()); } /// ////////////////////////////////////////////////////////////////////////////// /// Define the compatibility and update checks for icebergCompatV3 /// /// ////////////////////////////////////////////////////////////////////////////// private static final IcebergCompatV3MetadataValidatorAndUpdater INSTANCE = new IcebergCompatV3MetadataValidatorAndUpdater(); @Override String compatFeatureName() { return "icebergCompatV3"; } @Override TableConfig requiredDeltaTableProperty() { return TableConfig.ICEBERG_COMPAT_V3_ENABLED; } @Override List requiredDeltaTableProperties() { return Stream.of(COLUMN_MAPPING_REQUIREMENT, ROW_TRACKING_ENABLED).collect(toList()); } @Override List requiredDependencyTableFeatures() { return Stream.of(ICEBERG_COMPAT_V3_W_FEATURE, COLUMN_MAPPING_RW_FEATURE, ROW_TRACKING_W_FEATURE) .collect(toList()); } @Override List icebergCompatChecks() { return Stream.of( V3_CHECK_HAS_SUPPORTED_TYPES, CHECK_ONLY_ICEBERG_COMPAT_V3_ENABLED, CHECK_HAS_ALLOWED_PARTITION_TYPES, CHECK_HAS_NO_PARTITION_EVOLUTION, CHECK_HAS_SUPPORTED_TYPE_WIDENING, CHECK_LITERAL_DEFAULT_VALUE) .collect(toList()); } protected static IcebergCompatCheck CHECK_LITERAL_DEFAULT_VALUE = (inputContext) -> ColumnDefaults.validateSchemaForIcebergCompat( inputContext.newMetadata.getSchema(), ICEBERG_COMPAT_V3_W_FEATURE.featureName()); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergUniversalFormatMetadataValidatorAndUpdater.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.icebergcompat; import io.delta.kernel.exceptions.InvalidConfigurationValueException; import io.delta.kernel.internal.TableConfig; import io.delta.kernel.internal.actions.Metadata; import java.util.Arrays; import java.util.List; import java.util.Set; /** * Utility class that enforces dependencies of UNIVERSAL_FORMAT_* options. * *

This class currently only does validation, in the future it might also update metadata to be * conformant and thus has "Updater suffix". */ public class IcebergUniversalFormatMetadataValidatorAndUpdater { private IcebergUniversalFormatMetadataValidatorAndUpdater() {} /** * Ensures the metadata is consistent with the enabled Universal output targets. * *

If required dependent {@linkplain TableConfig}s are not set in {@code metadata} then an * exception is raised. * *

"hudi" is trivially compatible with Metadata. * *

"iceberg" requires that {@linkplain TableConfig#ICEBERG_COMPAT_V2_ENABLED} is set to true. * * @throws InvalidConfigurationValueException metadata has ICEBERG universal format enabled and * {@linkplain TableConfig#ICEBERG_COMPAT_V2_ENABLED} is not enabled in metadata */ public static void validate(Metadata metadata) { if (!metadata .getConfiguration() .containsKey(TableConfig.UNIVERSAL_FORMAT_ENABLED_FORMATS.getKey())) { return; } Set targetFormats = TableConfig.UNIVERSAL_FORMAT_ENABLED_FORMATS.fromMetadata(metadata); boolean isIcebergEnabled = targetFormats.contains(TableConfig.UniversalFormats.FORMAT_ICEBERG); List> icebergCompatOptions = Arrays.asList(TableConfig.ICEBERG_COMPAT_V2_ENABLED, TableConfig.ICEBERG_COMPAT_V3_ENABLED); long enabledCompatCount = icebergCompatOptions.stream().filter(opt -> opt.fromMetadata(metadata)).count(); if (isIcebergEnabled && enabledCompatCount == 0) { String optionKeys = String.join( " or ", icebergCompatOptions.stream().map(TableConfig::getKey).toArray(String[]::new)); throw new InvalidConfigurationValueException( TableConfig.UNIVERSAL_FORMAT_ENABLED_FORMATS.getKey(), TableConfig.UniversalFormats.FORMAT_ICEBERG, String.format( "One of %s must be set to \"true\" to enable iceberg uniform format.", optionKeys)); } if (enabledCompatCount > 1) { String optionKeys = String.join( "' and '", icebergCompatOptions.stream().map(TableConfig::getKey).toArray(String[]::new)); throw new InvalidConfigurationValueException( TableConfig.UNIVERSAL_FORMAT_ENABLED_FORMATS.getKey(), TableConfig.UniversalFormats.FORMAT_ICEBERG, String.format("'%s' cannot be enabled at the same time.", optionKeys)); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatMetadataValidatorAndUpdater.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.icebergcompat; import static io.delta.kernel.internal.tablefeatures.TableFeatures.*; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toSet; import io.delta.kernel.internal.DeltaErrors; import io.delta.kernel.internal.TableConfig; import io.delta.kernel.internal.tablefeatures.TableFeature; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.internal.util.ColumnMapping; import io.delta.kernel.internal.util.SchemaIterable; import io.delta.kernel.types.*; import java.util.*; import java.util.stream.Stream; /** * Contains interfaces and common utility classes performing the validations and updates necessary * to support the table feature IcebergWriterCompats when it is enabled by the table properties such * as "delta.enableIcebergWriterCompatV3". */ abstract class IcebergWriterCompatMetadataValidatorAndUpdater extends IcebergCompatMetadataValidatorAndUpdater { ///////////////////////////////////////////////////////////////////////////////// /// Interfaces for defining validations and updates necessary to support IcebergWriterCompats // /// ///////////////////////////////////////////////////////////////////////////////// /** * Common property enforcer for Column Mapping ID mode requirement. This is identical across all * Writer Compat versions. */ protected static final IcebergCompatRequiredTablePropertyEnforcer CM_ID_MODE_ENABLED = new IcebergCompatRequiredTablePropertyEnforcer<>( TableConfig.COLUMN_MAPPING_MODE, (value) -> ColumnMapping.ColumnMappingMode.ID == value, ColumnMapping.ColumnMappingMode.ID.value, // We need to update the CM info in the schema here because we check that the physical // name is correctly set as part of icebergWriterCompat checks (inputContext) -> ColumnMapping.updateColumnMappingMetadataIfNeeded( inputContext.newMetadata, inputContext.isCreatingNewTable)); /** * Creates an IcebergCompatRequiredTablePropertyEnforcer for enabling a specific Iceberg * compatibility version. The enforcer ensures the property is set to "true" and delegates * validation to the appropriate metadata validator. * * @param tableConfigProperty the table configuration property to enforce * @param postProcessor the version-specific validation and metadata update processor * @return configured enforcer for the specified Iceberg compatibility version */ protected static IcebergCompatRequiredTablePropertyEnforcer createIcebergCompatEnforcer( TableConfig tableConfigProperty, PostMetadataProcessor postProcessor) { return new IcebergCompatRequiredTablePropertyEnforcer<>( tableConfigProperty, (value) -> value, "true", postProcessor); } /** * Common set of allowed table features shared across all Iceberg writer compatibility versions. * This includes the incompatible legacy features (invariants, changeDataFeed, checkConstraints, * identityColumns, generatedColumns) because they may be present in the table protocol even when * they are not in use. In later checks we validate that these incompatible features are inactive * in the table. See the protocol spec for more details. */ protected static final Set COMMON_ALLOWED_FEATURES = Stream.of( // Incompatible, but not active, legacy table features INVARIANTS_W_FEATURE, CHANGE_DATA_FEED_W_FEATURE, ROW_TRACKING_W_FEATURE, CONSTRAINTS_W_FEATURE, IDENTITY_COLUMNS_W_FEATURE, GENERATED_COLUMNS_W_FEATURE, // Compatible table features APPEND_ONLY_W_FEATURE, COLUMN_MAPPING_RW_FEATURE, DOMAIN_METADATA_W_FEATURE, VACUUM_PROTOCOL_CHECK_RW_FEATURE, CHECKPOINT_V2_RW_FEATURE, CHECKPOINT_PROTECTION_W_FEATURE, IN_COMMIT_TIMESTAMP_W_FEATURE, CLUSTERING_W_FEATURE, TIMESTAMP_NTZ_RW_FEATURE, TYPE_WIDENING_RW_FEATURE, TYPE_WIDENING_RW_PREVIEW_FEATURE, CATALOG_MANAGED_RW_FEATURE) .collect(toSet()); protected static IcebergCompatCheck createUnsupportedFeaturesCheck( IcebergWriterCompatMetadataValidatorAndUpdater instance) { return (inputContext) -> { Set allowedTableFeatures = instance.getAllowedTableFeatures(); if (!allowedTableFeatures.containsAll( inputContext.newProtocol.getImplicitlyAndExplicitlySupportedFeatures())) { Set incompatibleFeatures = inputContext.newProtocol.getImplicitlyAndExplicitlySupportedFeatures(); incompatibleFeatures.removeAll(allowedTableFeatures); throw DeltaErrors.icebergCompatIncompatibleTableFeatures( inputContext.compatFeatureName, incompatibleFeatures); } }; } /** * Checks that there are no unsupported types in the schema. Data types {@link ByteType} and * {@link ShortType} are unsupported for IcebergWriterCompatV1 and V3 tables. */ protected static final IcebergCompatCheck UNSUPPORTED_TYPES_CHECK = (inputContext) -> { Set matches = new SchemaIterable(inputContext.newMetadata.getSchema()) .stream() .map(element -> element.getField().getDataType()) .filter( dataType -> dataType instanceof ByteType || dataType instanceof ShortType) .collect(toSet()); if (!matches.isEmpty()) { List unsupportedTypes = new ArrayList<>(matches); unsupportedTypes.sort(Comparator.comparing(DataType::toString)); throw DeltaErrors.icebergCompatUnsupportedTypeColumns( inputContext.compatFeatureName, unsupportedTypes); } }; /** * Checks that in the schema column mapping is set up such that the physicalName is equal to * "col-[fieldId]". This check assumes column mapping is enabled (and so should be performed after * that check). */ protected static final IcebergCompatCheck PHYSICAL_NAMES_MATCH_FIELD_IDS_CHECK = (inputContext) -> { List invalidFields = new SchemaIterable(inputContext.newMetadata.getSchema()) .stream() // ID info is only on struct fields. .filter(SchemaIterable.SchemaElement::isStructField) .filter( element -> { StructField field = element.getField(); String physicalName = ColumnMapping.getPhysicalName(field); long columnId = ColumnMapping.getColumnId(field); return !physicalName.equals(String.format("col-%s", columnId)); }) .map( element -> { StructField field = element.getField(); return String.format( "%s(physicalName='%s', columnId=%s)", element.getNamePath(), ColumnMapping.getPhysicalName(field), ColumnMapping.getColumnId(field)); }) .collect(toList()); if (!invalidFields.isEmpty()) { throw DeltaErrors.icebergWriterCompatInvalidPhysicalName(invalidFields); } }; /** * Checks that the table feature `invariants` is not active in the table, meaning there are no * invariants stored in the table schema. */ protected static final IcebergCompatCheck INVARIANTS_INACTIVE_CHECK = (inputContext) -> { // Note - since Kernel currently does not support the table feature `invariants` we will not // hit this check for E2E writes since we will fail early due to unsupported write // If Kernel starts supporting the feature `invariants` this check will become applicable if (TableFeatures.hasInvariants(inputContext.newMetadata.getSchema())) { throw DeltaErrors.icebergCompatIncompatibleTableFeatures( inputContext.compatFeatureName, Collections.singleton(INVARIANTS_W_FEATURE)); } }; /** * Checks that the table feature `changeDataFeed` is not active in the table, meaning the table * property `delta.enableChangeDataFeed` is not enabled. */ protected static final IcebergCompatCheck CHANGE_DATA_FEED_INACTIVE_CHECK = (inputContext) -> { // Note - since Kernel currently does not support the table feature `changeDataFeed` we will // not hit this check for E2E writes since we will fail early due to unsupported write // If Kernel starts supporting the feature `changeDataFeed` this check will become // applicable if (TableConfig.CHANGE_DATA_FEED_ENABLED.fromMetadata(inputContext.newMetadata)) { throw DeltaErrors.icebergCompatIncompatibleTableFeatures( inputContext.compatFeatureName, Collections.singleton(CHANGE_DATA_FEED_W_FEATURE)); } }; /** * Checks that the table feature `rowTracking` is not active in the table, meaning the table * property `delta.enableRowTracking` is not enabled. */ protected static final IcebergCompatCheck ROW_TRACKING_INACTIVE_CHECK = (inputContext) -> { if (TableConfig.ROW_TRACKING_ENABLED.fromMetadata(inputContext.newMetadata)) { throw DeltaErrors.icebergCompatIncompatibleTableFeatures( inputContext.compatFeatureName, Collections.singleton(ROW_TRACKING_W_FEATURE)); } }; /** * Checks that the table feature `checkConstraints` is not active in the table, meaning the table * has no check constraints stored in its metadata configuration. */ protected static final IcebergCompatCheck CHECK_CONSTRAINTS_INACTIVE_CHECK = (inputContext) -> { // Note - since Kernel currently does not support the table feature `checkConstraints` we // will // not hit this check for E2E writes since we will fail early due to unsupported write // If Kernel starts supporting the feature `checkConstraints` this check will become // applicable if (TableFeatures.hasCheckConstraints(inputContext.newMetadata)) { throw DeltaErrors.icebergCompatIncompatibleTableFeatures( inputContext.compatFeatureName, Collections.singleton(CONSTRAINTS_W_FEATURE)); } }; /** * Checks that the table feature `identityColumns` is not active in the table, meaning no identity * columns exist in the table schema. */ protected static final IcebergCompatCheck IDENTITY_COLUMNS_INACTIVE_CHECK = (inputContext) -> { // Note - since Kernel currently does not support the table feature `identityColumns` we // will // not hit this check for E2E writes since we will fail early due to unsupported write // If Kernel starts supporting the feature `identityColumns` this check will become // applicable if (TableFeatures.hasIdentityColumns(inputContext.newMetadata)) { throw DeltaErrors.icebergCompatIncompatibleTableFeatures( inputContext.compatFeatureName, Collections.singleton(IDENTITY_COLUMNS_W_FEATURE)); } }; /** * Checks that the table feature `generatedColumns` is not active in the table, meaning no * generated columns exist in the table schema. */ protected static final IcebergCompatCheck GENERATED_COLUMNS_INACTIVE_CHECK = (inputContext) -> { // Note - since Kernel currently does not support the table feature `generatedColumns` we // will // not hit this check for E2E writes since we will fail early due to unsupported write // If Kernel starts supporting the feature `generatedColumns` this check will become // applicable if (TableFeatures.hasGeneratedColumns(inputContext.newMetadata)) { throw DeltaErrors.icebergCompatIncompatibleTableFeatures( inputContext.compatFeatureName, Collections.singleton(GENERATED_COLUMNS_W_FEATURE)); } }; protected static final List COMMON_CHECKS = Arrays.asList( UNSUPPORTED_TYPES_CHECK, PHYSICAL_NAMES_MATCH_FIELD_IDS_CHECK, INVARIANTS_INACTIVE_CHECK, CHANGE_DATA_FEED_INACTIVE_CHECK, CHECK_CONSTRAINTS_INACTIVE_CHECK, IDENTITY_COLUMNS_INACTIVE_CHECK, GENERATED_COLUMNS_INACTIVE_CHECK); @Override abstract String compatFeatureName(); @Override abstract TableConfig requiredDeltaTableProperty(); @Override abstract List requiredDeltaTableProperties(); @Override abstract List requiredDependencyTableFeatures(); @Override abstract List icebergCompatChecks(); abstract Set getAllowedTableFeatures(); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdater.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.icebergcompat; import static io.delta.kernel.internal.tablefeatures.TableFeatures.*; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toSet; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.internal.TableConfig; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.tablefeatures.TableFeature; import io.delta.kernel.types.*; import java.util.*; import java.util.stream.Stream; /** * Performs the validations and updates necessary to support the table feature IcebergWriterCompatV1 * when it is enabled by the table property "delta.enableIcebergWriterCompatV1". * *

Requires that the following table properties are set to the specified values. If they are set * to an invalid value, throws an exception. If they are not set, enable them if possible. * *

    *
  • Requires ID column mapping mode (cannot be enabled on existing table). *
  • Requires icebergCompatV2 to be enabled. *
* *

Checks that required table features are enabled: icebergCompatWriterV1, icebergCompatV2, * columnMapping * *

Checks the following: * *

    *
  • Checks that all table features supported in the table's protocol are in the allow-list of * table features. This simultaneously ensures that any unsupported features are not present * (e.g. CDF, variant type, etc). *
  • Checks that there are no fields with data type byte or short. *
  • Checks that the table feature `invariants` is not active in the table (i.e. there are no * invariants in the table schema). This is a special case where the incompatible feature * `invariants` is in the allow-list of features since it is included by default in the table * protocol for new tables. Since it is incompatible we must verify that it is inactive in the * table. *
* * TODO additional enforcements coming in (ie physicalName=fieldId) */ public class IcebergWriterCompatV1MetadataValidatorAndUpdater extends IcebergWriterCompatMetadataValidatorAndUpdater { /** * Validates that any change to property {@link TableConfig#ICEBERG_WRITER_COMPAT_V1_ENABLED} is * valid (for existing table). Currently, the changes we support are * *
    *
  • No change in enablement (true to true or false to false) *
* * The changes that we do not support and for which we throw an {@link KernelException} are * *
    *
  • Disabling on an existing table (true to false) *
  • Enabling on an existing table (false to true) *
*/ public static void validateIcebergWriterCompatV1Change( Map oldConfig, Map newConfig) { blockConfigChangeOnExistingTable( TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED, oldConfig, newConfig); } /** * Validate and update the given Iceberg Writer Compat V1 metadata. * * @param newMetadata Metadata after the current updates * @param newProtocol Protocol after the current updates * @return The updated metadata if the metadata is valid and updated, otherwise empty. * @throws UnsupportedOperationException if the metadata is not compatible with Iceberg Writer V1 * requirements */ public static Optional validateAndUpdateIcebergWriterCompatV1Metadata( boolean isCreatingNewTable, Metadata newMetadata, Protocol newProtocol, Optional prevProtocol) { return INSTANCE.validateAndUpdateMetadata( new IcebergCompatInputContext( INSTANCE.compatFeatureName(), isCreatingNewTable, newMetadata, newProtocol, prevProtocol)); } /// ////////////////////////////////////////////////////////////////////////////// /// Define the compatibility and update checks for icebergWriterCompatV1 /// /// ////////////////////////////////////////////////////////////////////////////// private static final IcebergWriterCompatV1MetadataValidatorAndUpdater INSTANCE = new IcebergWriterCompatV1MetadataValidatorAndUpdater(); /** * Enforcer for Iceberg compatibility V2 (required by V1). Ensures the ICEBERG_COMPAT_V2_ENABLED * property is set to "true" and delegates validation to the V2 metadata validator. */ private static final IcebergCompatRequiredTablePropertyEnforcer ICEBERG_COMPAT_V2_ENABLED = createIcebergCompatEnforcer( TableConfig.ICEBERG_COMPAT_V2_ENABLED, (inputContext) -> IcebergCompatV2MetadataValidatorAndUpdater .validateAndUpdateIcebergCompatV2Metadata( inputContext.isCreatingNewTable, inputContext.newMetadata, inputContext.newProtocol, inputContext.prevProtocol)); /** * Current set of allowed table features for Iceberg writer compat V1. This combines the common * features with V1-specific features (ICEBERG_COMPAT_V2_W_FEATURE, ICEBERG_WRITER_COMPAT_V1). */ private static Set ALLOWED_TABLE_FEATURES = Stream.concat( COMMON_ALLOWED_FEATURES.stream(), Stream.of(ICEBERG_COMPAT_V2_W_FEATURE, ICEBERG_WRITER_COMPAT_V1)) .collect(toSet()); @Override String compatFeatureName() { return "icebergWriterCompatV1"; } @Override TableConfig requiredDeltaTableProperty() { return TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED; } @Override List requiredDeltaTableProperties() { return Stream.of(CM_ID_MODE_ENABLED, ICEBERG_COMPAT_V2_ENABLED).collect(toList()); } @Override List requiredDependencyTableFeatures() { return Stream.of( ICEBERG_WRITER_COMPAT_V1, ICEBERG_COMPAT_V2_W_FEATURE, COLUMN_MAPPING_RW_FEATURE) .collect(toList()); } @Override protected Set getAllowedTableFeatures() { return ALLOWED_TABLE_FEATURES; } @Override List icebergCompatChecks() { return Stream.concat( Stream.of(createUnsupportedFeaturesCheck(this), ROW_TRACKING_INACTIVE_CHECK), COMMON_CHECKS.stream()) .collect(toList()); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdater.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.icebergcompat; import static io.delta.kernel.internal.tablefeatures.TableFeatures.*; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toSet; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.internal.TableConfig; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.tablefeatures.TableFeature; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Stream; public class IcebergWriterCompatV3MetadataValidatorAndUpdater extends IcebergWriterCompatMetadataValidatorAndUpdater { /** * Validates that any change to property {@link TableConfig#ICEBERG_WRITER_COMPAT_V3_ENABLED} is * valid. Currently, the changes we support are * *
    *
  • No change in enablement (true to true or false to false) *
* * The changes that we do not support and for which we throw an {@link KernelException} are * *
    *
  • Disabling on an existing table (true to false) *
  • Enabling on an existing table (false to true) *
*/ public static void validateIcebergWriterCompatV3Change( Map oldConfig, Map newConfig) { blockConfigChangeOnExistingTable( TableConfig.ICEBERG_WRITER_COMPAT_V3_ENABLED, oldConfig, newConfig); } /** * Validate and update the given Iceberg Writer Compat V3 metadata. * * @param newMetadata Metadata after the current updates * @param newProtocol Protocol after the current updates * @return The updated metadata if the metadata is valid and updated, otherwise empty. * @throws UnsupportedOperationException if the metadata is not compatible with Iceberg Writer V3 * requirements */ public static Optional validateAndUpdateIcebergWriterCompatV3Metadata( boolean isCreatingNewTable, Metadata newMetadata, Protocol newProtocol, Optional prevProtocol) { return INSTANCE.validateAndUpdateMetadata( new IcebergCompatInputContext( INSTANCE.compatFeatureName(), isCreatingNewTable, newMetadata, newProtocol, prevProtocol)); } /// ////////////////////////////////////////////////////////////////////////////// /// Define the compatibility and update checks for icebergWriterCompatV3 /// /// ////////////////////////////////////////////////////////////////////////////// private static final IcebergWriterCompatV3MetadataValidatorAndUpdater INSTANCE = new IcebergWriterCompatV3MetadataValidatorAndUpdater(); /** * Enforcer for Iceberg compatibility V3. Ensures the ICEBERG_COMPAT_V3_ENABLED property is set to * "true" and delegates validation to the V3 metadata validator. */ private static final IcebergCompatRequiredTablePropertyEnforcer ICEBERG_COMPAT_V3_ENABLED = createIcebergCompatEnforcer( TableConfig.ICEBERG_COMPAT_V3_ENABLED, (inputContext) -> IcebergCompatV3MetadataValidatorAndUpdater .validateAndUpdateIcebergCompatV3Metadata( inputContext.isCreatingNewTable, inputContext.newMetadata, inputContext.newProtocol, inputContext.prevProtocol)); /** * Current set of allowed table features for Iceberg writer compat V3. This combines the common * features, v1-specific features with V3-specific features including variant support, deletion * vectors, and row tracking. */ private static Set ALLOWED_TABLE_FEATURES = Stream.concat( COMMON_ALLOWED_FEATURES.stream(), Stream.of( ICEBERG_COMPAT_V3_W_FEATURE, ICEBERG_WRITER_COMPAT_V3, DELETION_VECTORS_RW_FEATURE, VARIANT_RW_FEATURE, VARIANT_SHREDDING_RW_FEATURE, VARIANT_SHREDDING_PREVIEW_RW_FEATURE, VARIANT_RW_PREVIEW_FEATURE, // Also allow writerV1 features for backward compatibility. // // Note: We already enforce that these features cannot be enabled // through the CHECK_ONLY_ICEBERG_COMPAT_V3_ENABLED validation in // IcebergCompatV3MetadataValidatorAndUpdater. This ensures that // writerV1-related configs remain disabled even though the features // are listed here for protocol compatibility. ICEBERG_COMPAT_V2_W_FEATURE, ICEBERG_WRITER_COMPAT_V1)) .collect(toSet());; @Override String compatFeatureName() { return "icebergWriterCompatV3"; } @Override TableConfig requiredDeltaTableProperty() { return TableConfig.ICEBERG_WRITER_COMPAT_V3_ENABLED; } @Override List requiredDeltaTableProperties() { return Stream.of(CM_ID_MODE_ENABLED, ICEBERG_COMPAT_V3_ENABLED).collect(toList()); } @Override List requiredDependencyTableFeatures() { return Stream.of( ICEBERG_WRITER_COMPAT_V3, ICEBERG_COMPAT_V3_W_FEATURE, COLUMN_MAPPING_RW_FEATURE) .collect(toList()); } @Override List icebergCompatChecks() { return Stream.concat(Stream.of(createUnsupportedFeaturesCheck(this)), COMMON_CHECKS.stream()) .collect(toList()); } @Override protected Set getAllowedTableFeatures() { return ALLOWED_TABLE_FEATURES; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/lang/Lazy.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.lang; import java.util.Optional; import java.util.function.Supplier; public class Lazy { private final Supplier supplier; private Optional instance = Optional.empty(); public Lazy(Supplier supplier) { this.supplier = supplier; } public T get() { if (!instance.isPresent()) { instance = Optional.of(supplier.get()); } return instance.get(); } public boolean isPresent() { return instance.isPresent(); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/lang/ListUtils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.lang; import io.delta.kernel.internal.util.Tuple2; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.function.Predicate; import java.util.stream.Collectors; public final class ListUtils { private ListUtils() {} public static Tuple2, List> partition( List list, Predicate predicate) { final Map> partitionMap = list.stream().collect(Collectors.partitioningBy(predicate)); return new Tuple2<>(partitionMap.get(true), partitionMap.get(false)); } /** Remove once supported JDK (build) version is 21 or above */ public static T getFirst(List list) { if (list.isEmpty()) { throw new NoSuchElementException(); } else { return list.get(0); } } /** Remove once supported JDK (build) version is 21 or above */ public static T getLast(List list) { if (list.isEmpty()) { throw new NoSuchElementException(); } else { return list.get(list.size() - 1); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/metadatadomain/JsonMetadataDomain.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.metadatadomain; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.*; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.actions.DomainMetadata; import io.delta.kernel.internal.rowtracking.RowTrackingMetadataDomain; import java.util.Optional; /** * Abstract class representing a metadata domain, whose configuration string is a JSON serialization * of a domain object. This class provides methods to serialize and deserialize a metadata domain to * and from JSON. Concrete implementations, such as {@link RowTrackingMetadataDomain}, should extend * this class to define a specific metadata domain. * *

A metadata domain differs from {@link DomainMetadata}: {@link DomainMetadata} represents an * action that modifies the table's state by updating the configuration of a named metadata domain. * A metadata domain is a named domain used to organize configurations related to a specific table * feature. * *

For example, the row tracking feature uses a {@link RowTrackingMetadataDomain} to store the * highest assigned fresh row id of the table. When updated, the row tracking feature creates and * commits a new {@link DomainMetadata} action to reflect the change. * *

Serialization and deserialization are handled using Jackson's annotations. By default, all * public fields and getters are included in the serialization. When creating subclasses, ensure * that all fields to be serialized are accessible either through public fields or getters. * *

To control this behavior: * *

    *
  • Annotate methods/fields with {@link JsonIgnore} if they should be excluded from * serialization/deserialization. *
  • Annotate constructor with {@link JsonCreator} to specify which constructor to use during * deserialization. *
  • Use {@link JsonProperty} on constructor parameters to define the JSON field names during * deserialization. *
*/ public abstract class JsonMetadataDomain { // Configure the ObjectMapper with the same settings used in Delta-Spark private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper() .registerModule(new Jdk8Module()) // To support Optional .setSerializationInclusion(JsonInclude.Include.NON_ABSENT) // Exclude empty Optionals .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false /* state */); /** * Deserializes a JSON string into an instance of the specified metadata domain. * * @param json the JSON string to deserialize * @param clazz the concrete class of the metadata domain object to deserialize into * @param the type of the object * @return the deserialized object * @throws KernelException if the JSON string cannot be parsed */ protected static T fromJsonConfiguration(String json, Class clazz) { try { return OBJECT_MAPPER.readValue(json, clazz); } catch (JsonProcessingException e) { throw new KernelException( String.format( "Failed to parse JSON string into a %s instance. JSON content: %s", clazz.getSimpleName(), json), e); } } /** * Retrieves the domain metadata from a snapshot for a given domain, and deserializes it into an * instance of the specified metadata domain class. * * @param snapshot the snapshot to read from * @param clazz the metadata domain class of the object to deserialize into * @param domainName the name of the domain * @param the type of the metadata domain object * @return an Optional containing the deserialized object if the domain metadata is found, * otherwise an empty Optional */ protected static Optional fromSnapshot( SnapshotImpl snapshot, Class clazz, String domainName) { return snapshot .getDomainMetadata(domainName) .map(config -> fromJsonConfiguration(config, clazz)); } /** * Returns the name of the domain. * * @return the domain name */ @JsonIgnore public abstract String getDomainName(); /** * Serializes this object into a JSON string. * * @return the JSON string representation of this object * @throws KernelException if the object cannot be serialized */ public String toJsonConfiguration() { try { return OBJECT_MAPPER.writeValueAsString(this); } catch (JsonProcessingException e) { throw new KernelException( String.format( "Could not serialize %s (domain: %s) to JSON", this.getClass().getSimpleName(), getDomainName()), e); } } /** * Generate a {@link DomainMetadata} action from this metadata domain. * * @return the DomainMetadata action instance */ public DomainMetadata toDomainMetadata() { return new DomainMetadata(getDomainName(), toJsonConfiguration(), false /* removed */); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/metrics/Counter.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.metrics; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.LongAdder; /** A long counter that uses {@link AtomicLong} to count events. */ public class Counter { private final LongAdder counter = new LongAdder(); /** Increment the counter by 1. */ public void increment() { increment(1L); } /** * Increment the counter by the provided amount. * * @param amount to be incremented. */ public void increment(long amount) { counter.add(amount); } /** * Reports the current count. * * @return The current count. */ public long value() { return counter.longValue(); } /** Resets the current count to 0. */ public void reset() { counter.reset(); } @Override public String toString() { return String.format("Counter(%s)", counter.longValue()); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/metrics/DeltaOperationReportImpl.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.metrics; import static java.util.Objects.requireNonNull; import io.delta.kernel.metrics.DeltaOperationReport; import java.util.Optional; import java.util.UUID; /** Basic POJO implementation of {@link DeltaOperationReport} */ public abstract class DeltaOperationReportImpl implements DeltaOperationReport { private final String tablePath; private final UUID reportUUID; private final Optional exception; protected DeltaOperationReportImpl(String tablePath, Optional exception) { this.tablePath = requireNonNull(tablePath); this.reportUUID = UUID.randomUUID(); this.exception = requireNonNull(exception); } @Override public String getTablePath() { return tablePath; } @Override public UUID getReportUUID() { return reportUUID; } @Override public Optional getException() { return exception; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/metrics/MetricsReportSerializer.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.metrics; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import io.delta.kernel.expressions.Column; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.metrics.MetricsReport; import io.delta.kernel.types.StructType; import java.io.IOException; /** Provides Jackson ObjectMapper configuration for serializing {@link MetricsReport} types */ public final class MetricsReportSerializer { /** * ObjectMapper configured for serializing metrics reports. * *

This ObjectMapper is pre-configured with custom serializers for: * *

    *
  • Java 8 Optional types (serialized as null when empty) *
  • Exceptions (serialized using their toString() representation) *
  • Complex types like StructType and Predicate (using string representation) *
  • Column objects (serialized as arrays of field names) *
*/ public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper() .registerModule(new Jdk8Module()) // To support Optional .registerModule( // Serialize Exception using toString() new SimpleModule().addSerializer(Exception.class, new ToStringSerializer())) .registerModule( // Serialize StructType using toString new SimpleModule().addSerializer(StructType.class, new ToStringSerializer())) .registerModule( // Serialize Predicate using toString new SimpleModule().addSerializer(Predicate.class, new ToStringSerializer())) .registerModule( // Serialize Column to exclude un-necessary fields new SimpleModule().addSerializer(Column.class, new ColumnSerializer())); private static class ColumnSerializer extends JsonSerializer { @Override public void serialize(Column value, JsonGenerator gen, SerializerProvider serializers) throws IOException { gen.writeStartArray(); for (String name : value.getNames()) { gen.writeString(name); } gen.writeEndArray(); } } private MetricsReportSerializer() {} } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/metrics/ScanMetrics.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.metrics; import io.delta.kernel.metrics.ScanMetricsResult; /** * Stores the metrics for an ongoing scan. These metrics are updated and recorded throughout the * scan using this class. * *

At report time, we create an immutable {@link ScanMetricsResult} from an instance of {@link * ScanMetrics} to capture the metrics collected during the scan. The {@link ScanMetricsResult} * interface exposes getters for any metrics collected in this class. */ public class ScanMetrics { public final Timer totalPlanningTimer = new Timer(); public final Counter addFilesCounter = new Counter(); public final Counter addFilesFromDeltaFilesCounter = new Counter(); public final Counter activeAddFilesCounter = new Counter(); public final Counter duplicateAddFilesCounter = new Counter(); public final Counter removeFilesFromDeltaFilesCounter = new Counter(); public ScanMetricsResult captureScanMetricsResult() { return new ScanMetricsResult() { final long totalPlanningDurationNs = totalPlanningTimer.totalDurationNs(); final long numAddFilesSeen = addFilesCounter.value(); final long numAddFilesSeenFromDeltaFiles = addFilesFromDeltaFilesCounter.value(); final long numActiveAddFiles = activeAddFilesCounter.value(); final long numDuplicateAddFiles = duplicateAddFilesCounter.value(); final long numRemoveFilesSeenFromDeltaFiles = removeFilesFromDeltaFilesCounter.value(); @Override public long getTotalPlanningDurationNs() { return totalPlanningDurationNs; } @Override public long getNumAddFilesSeen() { return numAddFilesSeen; } @Override public long getNumAddFilesSeenFromDeltaFiles() { return numAddFilesSeenFromDeltaFiles; } @Override public long getNumActiveAddFiles() { return numActiveAddFiles; } @Override public long getNumDuplicateAddFiles() { return numDuplicateAddFiles; } @Override public long getNumRemoveFilesSeenFromDeltaFiles() { return numRemoveFilesSeenFromDeltaFiles; } }; } @Override public String toString() { return String.format( "ScanMetrics(totalPlanningTimer=%s, addFilesCounter=%s, addFilesFromDeltaFilesCounter=%s," + " activeAddFilesCounter=%s, duplicateAddFilesCounter=%s, " + "removeFilesFromDeltaFilesCounter=%s", totalPlanningTimer, addFilesCounter, addFilesFromDeltaFilesCounter, activeAddFilesCounter, duplicateAddFilesCounter, removeFilesFromDeltaFilesCounter); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/metrics/ScanReportImpl.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.metrics; import static java.util.Objects.requireNonNull; import com.fasterxml.jackson.core.JsonProcessingException; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.metrics.ScanMetricsResult; import io.delta.kernel.metrics.ScanReport; import io.delta.kernel.types.StructType; import java.util.Optional; import java.util.UUID; /** A basic POJO implementation of {@link ScanReport} for creating them */ public class ScanReportImpl extends DeltaOperationReportImpl implements ScanReport { private final long tableVersion; private final StructType tableSchema; private final UUID snapshotReportUUID; private final Optional filter; private final StructType readSchema; private final Optional partitionPredicate; private final Optional dataSkippingFilter; private final boolean isFullyConsumed; private final ScanMetricsResult scanMetricsResult; public ScanReportImpl( String tablePath, long tableVersion, StructType tableSchema, UUID snapshotReportUUID, Optional filter, StructType readSchema, Optional partitionPredicate, Optional dataSkippingFilter, boolean isFullyConsumed, ScanMetrics scanMetrics, Optional exception) { super(tablePath, exception); this.tableVersion = tableVersion; this.tableSchema = requireNonNull(tableSchema); this.snapshotReportUUID = requireNonNull(snapshotReportUUID); this.filter = requireNonNull(filter); this.readSchema = requireNonNull(readSchema); this.partitionPredicate = requireNonNull(partitionPredicate); this.dataSkippingFilter = requireNonNull(dataSkippingFilter); this.isFullyConsumed = isFullyConsumed; this.scanMetricsResult = requireNonNull(scanMetrics).captureScanMetricsResult(); } @Override public long getTableVersion() { return tableVersion; } @Override public StructType getTableSchema() { return tableSchema; } @Override public UUID getSnapshotReportUUID() { return snapshotReportUUID; } @Override public Optional getFilter() { return filter; } @Override public StructType getReadSchema() { return readSchema; } @Override public Optional getPartitionPredicate() { return partitionPredicate; } @Override public Optional getDataSkippingFilter() { return dataSkippingFilter; } @Override public boolean getIsFullyConsumed() { return isFullyConsumed; } @Override public ScanMetricsResult getScanMetrics() { return scanMetricsResult; } @Override public String toJson() throws JsonProcessingException { return MetricsReportSerializer.OBJECT_MAPPER.writeValueAsString(this); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/metrics/SnapshotMetrics.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.metrics; import io.delta.kernel.metrics.SnapshotMetricsResult; import java.util.Optional; /** * Stores the metrics for an ongoing snapshot construction. These metrics are updated and recorded * throughout the snapshot query using this class. * *

At report time, we create an immutable {@link SnapshotMetricsResult} from an instance of * {@link SnapshotMetrics} to capture the metrics collected during the query. The {@link * SnapshotMetricsResult} interface exposes getters for any metrics collected in this class. */ public class SnapshotMetrics { public final Timer loadSnapshotTotalTimer = new Timer(); public final Timer computeTimestampToVersionTotalDurationTimer = new Timer(); public final Timer loadProtocolMetadataTotalDurationTimer = new Timer(); public final Timer loadLogSegmentTotalDurationTimer = new Timer(); public final Timer loadCrcTotalDurationTimer = new Timer(); public SnapshotMetricsResult captureSnapshotMetricsResult() { return new SnapshotMetricsResult() { final Optional computeTimestampToVersionTotalDurationResult = computeTimestampToVersionTotalDurationTimer.totalDurationIfRecorded(); final long loadSnapshotTotalDurationResult = loadSnapshotTotalTimer.totalDurationNs(); final long loadProtocolMetadataTotalDurationResult = loadProtocolMetadataTotalDurationTimer.totalDurationNs(); final long loadLogSegmentTotalDurationResult = loadLogSegmentTotalDurationTimer.totalDurationNs(); final long loadCrcTotalDurationResult = loadCrcTotalDurationTimer.totalDurationNs(); @Override public Optional getComputeTimestampToVersionTotalDurationNs() { return computeTimestampToVersionTotalDurationResult; } @Override public long getLoadSnapshotTotalDurationNs() { return loadSnapshotTotalDurationResult; } @Override public long getLoadProtocolMetadataTotalDurationNs() { return loadProtocolMetadataTotalDurationResult; } @Override public long getLoadLogSegmentTotalDurationNs() { return loadLogSegmentTotalDurationResult; } @Override public long getLoadCrcTotalDurationNs() { return loadCrcTotalDurationResult; } }; } @Override public String toString() { return String.format( "SnapshotMetrics(" + "computeTimestampToVersionTotalDurationTimer=%s, " + "loadSnapshotTotalTimer=%s," + "loadProtocolMetadataTotalDurationTimer=%s, " + "timeToBuildLogSegmentForVersionTimer=%s, " + "loadCrcTotalDurationNsTimer=%s)", computeTimestampToVersionTotalDurationTimer, loadSnapshotTotalTimer, loadProtocolMetadataTotalDurationTimer, loadLogSegmentTotalDurationTimer, loadCrcTotalDurationTimer); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/metrics/SnapshotQueryContext.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.metrics; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.engine.Engine; import io.delta.kernel.engine.MetricsReporter; import io.delta.kernel.metrics.SnapshotReport; import java.util.Optional; /** * Stores the context for a given Snapshot query. This includes information about the query * parameters (i.e. table path, time travel parameters), updated state as the snapshot query * progresses (i.e. resolved version), and metrics. * *

This is used to generate a {@link SnapshotReport}. It exists from snapshot query initiation * until either successful snapshot construction or failure. */ public class SnapshotQueryContext { /** Creates a {@link SnapshotQueryContext} for a Snapshot created by a latest snapshot query */ public static SnapshotQueryContext forLatestSnapshot(String tablePath) { return new SnapshotQueryContext( tablePath, Optional.empty(), Optional.empty(), Optional.empty()); } /** Creates a {@link SnapshotQueryContext} for a Snapshot created by a AS OF VERSION query */ public static SnapshotQueryContext forVersionSnapshot(String tablePath, long version) { return new SnapshotQueryContext( tablePath, Optional.of(version), Optional.empty(), Optional.empty()); } /** Creates a {@link SnapshotQueryContext} for a Snapshot created by a AS OF TIMESTAMP query */ public static SnapshotQueryContext forTimestampSnapshot(String tablePath, long timestamp) { return new SnapshotQueryContext( tablePath, Optional.empty(), Optional.empty(), Optional.of(timestamp)); } private final String tablePath; /** The version provided in a time-travel-by-version query, if any. */ private final Optional providedVersion; /** The timestamp provided in a time-travel-by-timestamp query, if any. */ private final Optional providedTimestamp; private final SnapshotMetrics snapshotMetrics = new SnapshotMetrics(); /** The table version that this snapshot is actually resolved to. */ private Optional resolvedVersion; private Optional checkpointVersion; /** * @param tablePath the table path for the table being queried * @param providedVersion the provided version for a time-travel-by-version query, empty if this * is not a time-travel-by-version query * @param checkpointVersion the version of the checkpoint used for this snapshot, empty if no * checkpoint was used or if this is a failed snapshot construction * @param providedTimestamp the provided timestamp for a time-travel-by-timestamp query, empty if * this is not a time-travel-by-timestamp query */ private SnapshotQueryContext( String tablePath, Optional providedVersion, Optional checkpointVersion, Optional providedTimestamp) { this.tablePath = tablePath; this.providedVersion = providedVersion; this.resolvedVersion = providedVersion; this.checkpointVersion = checkpointVersion; this.providedTimestamp = providedTimestamp; } public String getTablePath() { return tablePath; } public Optional getResolvedVersion() { return resolvedVersion; } public Optional getCheckpointVersion() { return checkpointVersion; } public Optional getProvidedTimestamp() { return providedTimestamp; } /** * Returns true if this snapshot was requested as the latest snapshot (i.e., no time-travel * parameters were provided). Note that this is intent-based - it indicates what the user * requested, not whether the snapshot is actually the latest version. */ public boolean isLatestQuery() { return !providedVersion.isPresent() && !providedTimestamp.isPresent(); } public SnapshotMetrics getSnapshotMetrics() { return snapshotMetrics; } public String getQueryDisplayStr() { final String resolvedVersionStr = resolvedVersion.map(v -> String.format(" (RESOLVED TO VERSION %d)", v)).orElse(""); if (providedVersion.isPresent()) { return "AS OF VERSION " + resolvedVersion.get(); } else if (providedTimestamp.isPresent()) { return "AS OF TIMESTAMP " + providedTimestamp.get() + resolvedVersionStr; } else { return "LATEST SNAPSHOT" + resolvedVersionStr; } } /** * Set the resolved version that was actually loaded for this snapshot query. * *

For AS OF TIMESTAMP queries, this should be set upon timestamp-to-version resolution. * *

For AS OF LATEST queries, this should be set after log segment construction, when we learn * what the latest version of the table really is. */ public void setResolvedVersion(long resolvedVersion) { this.resolvedVersion = Optional.of(resolvedVersion); } /** Updates the {@code checkpointVersion} stored in this snapshot context. */ public void setCheckpointVersion(Optional checkpointVersion) { requireNonNull(checkpointVersion, "checkpointVersion cannot be null"); checkArgument( !checkpointVersion.isPresent() || checkpointVersion.get() >= 0, "Invalid checkpoint version: %s", checkpointVersion); this.checkpointVersion = checkpointVersion; } /** Creates a {@link SnapshotReport} and pushes it to any {@link MetricsReporter}s. */ public void recordSnapshotErrorReport(Engine engine, Exception e) { SnapshotReport snapshotReport = SnapshotReportImpl.forError(this, e); engine.getMetricsReporters().forEach(reporter -> reporter.report(snapshotReport)); } @Override public String toString() { return String.format( "SnapshotQueryContext(tablePath=%s, version=%s, providedTimestamp=%s, " + "checkpointVersion=%s, snapshotMetric=%s)", tablePath, resolvedVersion, providedTimestamp, checkpointVersion, snapshotMetrics); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/metrics/SnapshotReportImpl.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.metrics; import static java.util.Objects.requireNonNull; import com.fasterxml.jackson.core.JsonProcessingException; import io.delta.kernel.metrics.SnapshotMetricsResult; import io.delta.kernel.metrics.SnapshotReport; import java.util.Optional; /** A basic POJO implementation of {@link SnapshotReport} for creating them */ public class SnapshotReportImpl extends DeltaOperationReportImpl implements SnapshotReport { /** * Creates a {@link SnapshotReport} for a failed snapshot query. * * @param snapshotContext context/metadata about the snapshot query * @param e the exception that was thrown */ public static SnapshotReport forError(SnapshotQueryContext snapshotContext, Exception e) { return new SnapshotReportImpl( snapshotContext.getTablePath(), snapshotContext.getSnapshotMetrics(), snapshotContext.getResolvedVersion(), snapshotContext.getCheckpointVersion(), snapshotContext.getProvidedTimestamp(), Optional.of(e)); } /** * Creates a {@link SnapshotReport} for a successful snapshot query. * * @param snapshotContext context/metadata about the snapshot query */ public static SnapshotReport forSuccess(SnapshotQueryContext snapshotContext) { return new SnapshotReportImpl( snapshotContext.getTablePath(), snapshotContext.getSnapshotMetrics(), snapshotContext.getResolvedVersion(), snapshotContext.getCheckpointVersion(), snapshotContext.getProvidedTimestamp(), Optional.empty() /* exception */); } private final SnapshotMetricsResult snapshotMetrics; private final Optional version; private final Optional checkpointVersion; private final Optional providedTimestamp; private SnapshotReportImpl( String tablePath, SnapshotMetrics snapshotMetrics, Optional version, Optional checkpointVersion, Optional providedTimestamp, Optional exception) { super(tablePath, exception); this.snapshotMetrics = requireNonNull(snapshotMetrics).captureSnapshotMetricsResult(); this.version = requireNonNull(version); this.checkpointVersion = requireNonNull(checkpointVersion); this.providedTimestamp = requireNonNull(providedTimestamp); } @Override public SnapshotMetricsResult getSnapshotMetrics() { return snapshotMetrics; } @Override public Optional getVersion() { return version; } @Override public Optional getCheckpointVersion() { return checkpointVersion; } @Override public Optional getProvidedTimestamp() { return providedTimestamp; } @Override public String toJson() throws JsonProcessingException { return MetricsReportSerializer.OBJECT_MAPPER.writeValueAsString(this); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/metrics/Timer.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.metrics; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import java.util.Optional; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; import java.util.function.Supplier; /** A timer class for measuring the duration of operations in nanoseconds */ public class Timer { private final LongAdder count = new LongAdder(); private final LongAdder totalTime = new LongAdder(); /** @return the number of times this timer was used to record a duration. */ public long count() { return count.longValue(); } /** @return the total duration that was recorded in nanoseconds */ public long totalDurationNs() { return totalTime.longValue(); } /** @return the total duration that was recorded in milliseconds */ public long totalDurationMs() { return TimeUnit.NANOSECONDS.toMillis(totalDurationNs()); } /** * @return An optional storing the total duration recorded in the timer if the timer has been used * to record a duration at least once. If the timer has not been used, returns empty. */ public Optional totalDurationIfRecorded() { return count() > 0 ? Optional.of(totalDurationNs()) : Optional.empty(); } /** * Starts the timer and returns a {@link Timed} instance. Call {@link Timed#stop()} to complete * the timing. * * @return A {@link Timed} instance with the start time recorded. */ public Timed start() { return new DefaultTimed(this); } /** * Records a custom amount. * * @param amount The amount to record in nanoseconds */ public void record(long amount) { checkArgument(amount >= 0, "Cannot record %s: must be >= 0", amount); this.totalTime.add(amount); this.count.increment(); } public T time(Supplier supplier) { try (Timed ignore = start()) { return supplier.get(); } } public T timeCallable(Callable callable) throws Exception { try (Timed ignore = start()) { return callable.call(); } } /** * Times an operation that can throw a specific checked exception type. * * @param The return type * @param The exception type that can be thrown */ @FunctionalInterface public interface ThrowingSupplier { T get() throws E; } /** Times an operation that can throw a specific checked exception type. */ @SuppressWarnings("unchecked") public T timeChecked(ThrowingSupplier operation) throws E { try (Timed ignore = start()) { return operation.get(); } catch (RuntimeException | Error e) { throw e; } catch (Exception e) { // Safe cast since operation can only throw E or unchecked exceptions (handled above) throw (E) e; } } public void time(Runnable runnable) { try (Timed ignore = start()) { runnable.run(); } } @Override public String toString() { return String.format("Timer(duration=%s ns, count=%s)", totalDurationNs(), count()); } /** * A timing sample that carries internal state about the Timer's start position. The timing can be * completed by calling {@link Timed#stop()}. */ public interface Timed extends AutoCloseable { /** Stops the timer and records the total duration up until {@link Timer#start()} was called. */ void stop(); @Override default void close() { stop(); } Timed NOOP = () -> {}; } private static class DefaultTimed implements Timed { private final Timer timer; private final long startTime; private boolean closed; private DefaultTimed(Timer timer) { this.timer = timer; this.startTime = System.nanoTime(); } @Override public void stop() { if (closed) { throw new IllegalStateException("called stop() multiple times"); } timer.record(System.nanoTime() - startTime); closed = true; } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/metrics/TransactionMetrics.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.metrics; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.internal.stats.FileSizeHistogram; import io.delta.kernel.metrics.FileSizeHistogramResult; import io.delta.kernel.metrics.TransactionMetricsResult; import java.util.Optional; /** * Stores the metrics for an ongoing transaction. These metrics are updated and recorded throughout * the transaction using this class. * *

At report time, we create an immutable {@link TransactionMetricsResult} from an instance of * {@link TransactionMetrics} to capture the metrics collected during the transaction. The {@link * TransactionMetricsResult} interface exposes getters for any metrics collected in this class. */ public class TransactionMetrics { /** * @return a fresh TransactionMetrics object with a default tableFileSizeHistogram (with 0 counts) */ public static TransactionMetrics forNewTable() { return new TransactionMetrics(Optional.of(FileSizeHistogram.createDefaultHistogram())); } /** * @return a fresh TransactionMetrics object with an initial tableFileSizeHistogram as provided */ public static TransactionMetrics withExistingTableFileSizeHistogram( Optional tableFileSizeHistogram) { return new TransactionMetrics(tableFileSizeHistogram); } public final Timer totalCommitTimer = new Timer(); public final Counter commitAttemptsCounter = new Counter(); private final Counter addFilesCounter = new Counter(); private final Counter removeFilesCounter = new Counter(); public final Counter totalActionsCounter = new Counter(); private final Counter addFilesSizeInBytesCounter = new Counter(); private final Counter removeFilesSizeInBytesCounter = new Counter(); private Optional tableFileSizeHistogram; private TransactionMetrics(Optional tableFileSizeHistogram) { this.tableFileSizeHistogram = tableFileSizeHistogram; } /** * Updates the metrics for a seen AddFile with size {@code addFileSize}. Specifically, updates * addFilesCounter, addFilesSizeInBytesCounter, and tableFileSizeHistogram. Note, it does NOT * increment totalActionsCounter, this needs to be done separately. * * @param addFileSize the size of the add file to update the metrics for */ public void updateForAddFile(long addFileSize) { checkArgument(addFileSize >= 0, "File size must be non-negative, got %s", addFileSize); addFilesCounter.increment(); addFilesSizeInBytesCounter.increment(addFileSize); tableFileSizeHistogram.ifPresent(histogram -> histogram.insert(addFileSize)); } /** * Updates the metrics for a seen RemoveFile with size {@code removeFileSize}. Specifically, * updates removeFilesCounter, removeFilesSizeInBytesCounter, and tableFileSizeHistogram. Note, it * does NOT increment totalActionsCounter, this needs to be done separately. * * @param removeFileSize the size of the remove file to update the metrics for */ public void updateForRemoveFile(long removeFileSize) { checkArgument(removeFileSize >= 0, "File size must be non-negative, got %s", removeFileSize); removeFilesCounter.increment(); removeFilesSizeInBytesCounter.increment(removeFileSize); tableFileSizeHistogram.ifPresent(histogram -> histogram.remove(removeFileSize)); } /** * Resets any action metrics for a failed commit to prepare them for retrying. Specifically, * *

    *
  • Resets addFilesCounter, removeFilesCounter, totalActionsCounter, * addFilesSizeInBytesCounter, and removeFilesSizeInBytesCounter to 0 *
  • Sets tableFileSizeHistogram to be empty since we don't know the updated distribution * after the conflicting txn committed *
* * Action counters / tableFileSizeHistogram may be partially incremented if an action iterator is * not read to completion (i.e. if an exception interrupts a file write). This allows us to reset * the counters so that we can increment them correctly from 0 on a retry. */ public void resetActionMetricsForRetry() { addFilesCounter.reset(); addFilesSizeInBytesCounter.reset(); removeFilesCounter.reset(); totalActionsCounter.reset(); removeFilesSizeInBytesCounter.reset(); // For now, on retry we set tableFileSizeHistogram = Optional.empty() because we don't know the // correct state of tableFileSizeHistogram after conflicting transaction has committed tableFileSizeHistogram = Optional.empty(); } public TransactionMetricsResult captureTransactionMetricsResult() { return new TransactionMetricsResult() { final long totalCommitDurationNs = totalCommitTimer.totalDurationNs(); final long numCommitAttempts = commitAttemptsCounter.value(); final long numAddFiles = addFilesCounter.value(); final long totalAddFilesSizeInBytes = addFilesSizeInBytesCounter.value(); final long numRemoveFiles = removeFilesCounter.value(); final long numTotalActions = totalActionsCounter.value(); final long totalRemoveFileSizeInBytes = removeFilesSizeInBytesCounter.value(); final Optional tableFileSizeHistogramResult = tableFileSizeHistogram.map(FileSizeHistogram::captureFileSizeHistogramResult); @Override public long getTotalCommitDurationNs() { return totalCommitDurationNs; } @Override public long getNumCommitAttempts() { return numCommitAttempts; } @Override public long getNumAddFiles() { return numAddFiles; } @Override public long getNumRemoveFiles() { return numRemoveFiles; } @Override public long getNumTotalActions() { return numTotalActions; } @Override public long getTotalAddFilesSizeInBytes() { return totalAddFilesSizeInBytes; } @Override public long getTotalRemoveFilesSizeInBytes() { return totalRemoveFileSizeInBytes; } @Override public Optional getTableFileSizeHistogram() { return tableFileSizeHistogramResult; } }; } @Override public String toString() { return String.format( "TransactionMetrics(totalCommitTimer=%s, commitAttemptsCounter=%s, addFilesCounter=%s, " + "removeFilesCounter=%s, totalActionsCounter=%s, totalAddFilesSizeInBytes=%s," + "totalRemoveFilesSizeInBytes=%s, tableFileSizeHistogram=%s)", totalCommitTimer, commitAttemptsCounter, addFilesCounter, removeFilesCounter, totalActionsCounter, addFilesSizeInBytesCounter, removeFilesSizeInBytesCounter, tableFileSizeHistogram); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/metrics/TransactionReportImpl.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.metrics; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import com.fasterxml.jackson.core.JsonProcessingException; import io.delta.kernel.expressions.Column; import io.delta.kernel.metrics.SnapshotReport; import io.delta.kernel.metrics.TransactionMetricsResult; import io.delta.kernel.metrics.TransactionReport; import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.UUID; /** A basic POJO implementation of {@link TransactionReport} for creating them */ public class TransactionReportImpl extends DeltaOperationReportImpl implements TransactionReport { private final String operation; private final String engineInfo; private final long snapshotVersion; private final Optional snapshotReportUUID; private final Optional committedVersion; private final List clusteringColumns; private final TransactionMetricsResult transactionMetrics; /** * @param tablePath the path of the table for the transaction * @param operation the operation provided by the connector when the transaction was created * @param engineInfo the engineInfo provided by the connector when the transaction was created * @param committedVersion the version committed to the table. Empty for a failed transaction. * @param clusteringColumns the clustering columns for the table, if any. Empty if not set. * @param transactionMetrics the metrics for the transaction * @param snapshotReport the SnapshotReport for the base snapshot of this transaction. Note, in * the case of a new table (when version = -1), this SnapshotReport is just a placeholder and * was never emitted to the engine's metrics reporters. * @param exception the exception thrown. Empty for a successful transaction. */ public TransactionReportImpl( String tablePath, String operation, String engineInfo, Optional committedVersion, Optional> clusteringColumns, TransactionMetrics transactionMetrics, Optional snapshotReport, Optional exception) { super(tablePath, exception); this.operation = requireNonNull(operation); this.engineInfo = requireNonNull(engineInfo); this.transactionMetrics = requireNonNull(transactionMetrics).captureTransactionMetricsResult(); this.committedVersion = committedVersion; this.clusteringColumns = requireNonNull(clusteringColumns).orElse(Collections.emptyList()); requireNonNull(snapshotReport); if (snapshotReport.isPresent()) { checkArgument( !snapshotReport.get().getException().isPresent(), "Expected a successful SnapshotReport provided report has exception"); checkArgument( snapshotReport.get().getVersion().isPresent(), "Expected a successful SnapshotReport but missing version"); this.snapshotVersion = snapshotReport.get().getVersion().get(); this.snapshotReportUUID = Optional.of(snapshotReport.get().getReportUUID()); } else { this.snapshotVersion = -1; this.snapshotReportUUID = Optional.empty(); } } @Override public String getOperation() { return operation; } @Override public String getEngineInfo() { return engineInfo; } @Override public long getBaseSnapshotVersion() { return snapshotVersion; } @Override public List getClusteringColumns() { return clusteringColumns; } @Override public Optional getSnapshotReportUUID() { return snapshotReportUUID; } @Override public Optional getCommittedVersion() { return committedVersion; } @Override public TransactionMetricsResult getTransactionMetrics() { return transactionMetrics; } @Override public String toJson() throws JsonProcessingException { return MetricsReportSerializer.OBJECT_MAPPER.writeValueAsString(this); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/ActionWrapper.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.replay; import io.delta.kernel.data.ColumnarBatch; import java.util.Optional; /** Internal wrapper class holding information needed to perform log replay. */ public class ActionWrapper { private final ColumnarBatch columnarBatch; private final boolean isFromCheckpoint; private final long version; private final String filePath; /* Timestamp of the commit file if isFromCheckpoint=false */ private final Optional timestamp; ActionWrapper( ColumnarBatch data, boolean isFromCheckpoint, long version, Optional timestamp, String filePath) { this.columnarBatch = data; this.isFromCheckpoint = isFromCheckpoint; this.version = version; this.timestamp = timestamp; this.filePath = filePath; } public ColumnarBatch getColumnarBatch() { return columnarBatch; } public boolean isFromCheckpoint() { return isFromCheckpoint; } public long getVersion() { return version; } public Optional getTimestamp() { return timestamp; } public String getFilePath() { return filePath; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/ActionsIterator.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.replay; import static io.delta.kernel.internal.DeltaErrors.wrapEngineExceptionThrowsIO; import static io.delta.kernel.internal.replay.DeltaLogFile.LogType.*; import static io.delta.kernel.internal.util.FileNames.*; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static io.delta.kernel.internal.util.Utils.singletonCloseableIterator; import static io.delta.kernel.internal.util.Utils.toCloseableIterator; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.engine.Engine; import io.delta.kernel.engine.FileReadResult; import io.delta.kernel.expressions.*; import io.delta.kernel.internal.annotation.VisibleForTesting; import io.delta.kernel.internal.checkpoints.SidecarFile; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.util.*; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import java.io.IOException; import java.io.UncheckedIOException; import java.util.*; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * This class takes as input a list of delta files (.json, .checkpoint.parquet) and produces an * iterator of (ColumnarBatch, isFromCheckpoint) tuples, where the schema of the ColumnarBatch * semantically represents actions (or, a subset of action fields) parsed from the Delta Log. * *

Users must pass in a `deltaReadSchema` to select which actions and sub-fields they want to * consume. * *

Users can also pass in an optional `checkpointReadSchema` if it is different from * `deltaReadSchema`. */ public class ActionsIterator implements CloseableIterator { private static final Logger logger = LoggerFactory.getLogger(ActionsIterator.class); private final Engine engine; private final Optional checkpointPredicate; /** * Linked list of iterator files (commit files and/or checkpoint files) {@link LinkedList} to * allow removing the head of the list and also to peek at the head of the list. The {@link * Iterator} doesn't provide a way to peek. * *

Each of these files return an iterator of {@link ColumnarBatch} containing the actions */ private final LinkedList filesList; /** Schema used for reading delta files. */ private final StructType deltaReadSchema; /** * Schema to be used for reading checkpoint files. Checkpoint files can be Parquet or JSON in the * case of v2 checkpoints. */ private final StructType checkpointReadSchema; private final boolean schemaContainsAddOrRemoveFiles; /** * The current (ColumnarBatch, isFromCheckpoint) tuple. Whenever this iterator is exhausted, we * will try and fetch the next one from the `filesList`. * *

If it is ever empty, that means there are no more batches to produce. */ private Optional> actionsIter; private final Optional paginationContextOpt; private long numCheckpointFilesSkipped = 0; private long numSidecarFilesSkipped = 0; private boolean closed; public ActionsIterator( Engine engine, List files, StructType deltaReadSchema, Optional checkpointPredicate) { this( engine, files, deltaReadSchema, deltaReadSchema, checkpointPredicate, Optional.empty() /* paginationContextOpt */); } public ActionsIterator( Engine engine, List files, StructType deltaReadSchema, StructType checkpointReadSchema, Optional checkpointPredicate, Optional paginationContextOpt) { this.engine = engine; this.checkpointPredicate = checkpointPredicate; this.filesList = new LinkedList<>(); this.paginationContextOpt = paginationContextOpt; this.filesList.addAll( files.stream() .map(DeltaLogFile::forFileStatus) .filter(this::paginatedFilter) .collect(Collectors.toList())); this.deltaReadSchema = deltaReadSchema; this.checkpointReadSchema = checkpointReadSchema; this.actionsIter = Optional.empty(); this.schemaContainsAddOrRemoveFiles = LogReplay.containsAddOrRemoveFileActions(deltaReadSchema); } /** * Filters a log segment file based on the pagination context. * *

If this method returns {@code true}, the current file will be kept; otherwise, it will be * skipped. * *

    *
  • If pagination is not enabled (i.e., {@code paginationContextOpt} is not present), return * {@code true}. *
  • If the pagination context is present but doesn't include a last read log file path, * return {@code true} (indicates reading the first page). *
  • If the file is a JSON log file, return {@code true} — we never skip JSON files as they're * needed to build hash sets. *
  • If the file is a V2 checkpoint manifest, return {@code true} — these should never be * skipped. *
  • If the file is a checkpoint file and comes after the last log file recorded in the page * token, return {@code false} (skip it). *
* *

Note: The {@code nextLogFile} parameter cannot be a sidecar file because sidecar * files are not included in the log segment list. Sidecar files are handled separately later, * after the V2 manifest file has been read, specifically in the {@code extractSidecarFiles()} * method. * * @param nextLogFile the log file to evaluate * @return {@code true} to include the file; {@code false} to skip it */ @VisibleForTesting // TODO: verify numCheckpointFilesSkipped is correct in E2E test // TODO: add unit test for this method public boolean paginatedFilter(DeltaLogFile nextLogFile) { Objects.requireNonNull(paginationContextOpt); // Pagination isn't enabled. if (!paginationContextOpt.isPresent()) return true; String nextFilePath = nextLogFile.getFile().getPath(); Optional lastReadLogFilePathOpt = paginationContextOpt.get().getLastReadLogFilePath(); // Reading the first page if (!lastReadLogFilePathOpt.isPresent()) { logger.info("Pagination: no page token present, reading the first page"); return true; } logger.info("Pagination: lastReadLogFilePath in token is {}", lastReadLogFilePathOpt.get()); logger.info("Pagination: nextFilePath is {}", nextFilePath); switch (nextLogFile.getLogType()) { case COMMIT: case LOG_COMPACTION: case CHECKPOINT_CLASSIC: case V2_CHECKPOINT_MANIFEST: return true; case MULTIPART_CHECKPOINT: if (isFullyConsumedFile(nextFilePath, lastReadLogFilePathOpt.get())) { logger.info("Pagination: skip reading multi-part checkpoint file {}", nextFilePath); numCheckpointFilesSkipped++; return false; } else { return true; } case SIDECAR: throw new IllegalArgumentException( "Sidecar file shouldn't exist in log segment! Path: " + nextFilePath); default: throw new IllegalArgumentException("Unknown log file type!"); } } /** * Returns true if the given file was already fully consumed in a previous page that ends at * lastReadLogFilePath. */ private boolean isFullyConsumedFile(String filePath, String lastReadLogFilePath) { // Files are sorted in reverse lexicographic order.so if `filePath` is *greater* than // `lastReadLogFilePath`, // it actually comes before lastReadLogFilePath in the log stream, meaning we have already // paginated past it. return filePath.compareTo(lastReadLogFilePath) > 0; } @Override public boolean hasNext() { if (closed) { throw new IllegalStateException("Can't call `hasNext` on a closed iterator."); } tryEnsureNextActionsIterIsReady(); // By definition of tryEnsureNextActionsIterIsReady, we know that if actionsIter // is non-empty then it has a next element return actionsIter.isPresent(); } /** * @return a tuple of (ColumnarBatch, isFromCheckpoint), where ColumnarBatch conforms to the * instance {@link #deltaReadSchema} or {@link #checkpointReadSchema} (the latter when when * isFromCheckpoint=true). */ @Override public ActionWrapper next() { if (closed) { throw new IllegalStateException("Can't call `next` on a closed iterator."); } if (Thread.currentThread().isInterrupted()) { throw new IllegalStateException("Thread was interrupted"); } if (!hasNext()) { throw new NoSuchElementException("No next element"); } return actionsIter.get().next(); } @Override public void close() throws IOException { if (!closed && actionsIter.isPresent()) { actionsIter.get().close(); actionsIter = Optional.empty(); closed = true; } } /** * If the current `actionsIter` has no more elements, this function finds the next non-empty file * in `filesList` and uses it to set `actionsIter`. */ private void tryEnsureNextActionsIterIsReady() { if (actionsIter.isPresent()) { // This iterator already has a next element, so we can exit early; if (actionsIter.get().hasNext()) { return; } // Clean up resources Utils.closeCloseables(actionsIter.get()); // Set this to empty since we don't know if there's a next file yet actionsIter = Optional.empty(); } // Search for the next non-empty file and use that iter while (!filesList.isEmpty()) { actionsIter = Optional.of(getNextActionsIter()); if (actionsIter.get().hasNext()) { // It is ready, we are done return; } // It was an empty file. Clean up resources Utils.closeCloseables(actionsIter.get()); // Set this to empty since we don't know if there's a next file yet actionsIter = Optional.empty(); } } /** * Get an iterator of actions from the v2 checkpoint file that may contain sidecar files. If the * current read schema includes Add/Remove files, then inject the sidecar column into this schema * to read the sidecar files from the top-level v2 checkpoint file. When the returned * ColumnarBatches are processed, these sidecars will be appended to the end of the file list and * read as part of a subsequent batch (avoiding reading the top-level v2 checkpoint files more * than once). */ private CloseableIterator getActionsIterFromSinglePartOrV2Checkpoint( FileStatus file, String fileName) throws IOException { // If the sidecars may contain the current action, read sidecars from the top-level v2 // checkpoint file(to be read later). StructType modifiedReadSchema = checkpointReadSchema; if (schemaContainsAddOrRemoveFiles) { modifiedReadSchema = LogReplay.withSidecarFileSchema(checkpointReadSchema); } long checkpointVersion = checkpointVersion(file.getPath()); // If the read schema contains Add/Remove files, we should always read the sidecar file // actions from the checkpoint manifest regardless of the checkpoint predicate. Optional checkpointPredicateIncludingSidecars; if (schemaContainsAddOrRemoveFiles) { Predicate containsSidecarPredicate = new Predicate("IS_NOT_NULL", new Column(LogReplay.SIDECAR_FIELD_NAME)); checkpointPredicateIncludingSidecars = checkpointPredicate.map(p -> new Or(p, containsSidecarPredicate)); } else { checkpointPredicateIncludingSidecars = checkpointPredicate; } final CloseableIterator topLevelIter; StructType finalReadSchema = modifiedReadSchema; if (fileName.endsWith(".parquet")) { topLevelIter = wrapEngineExceptionThrowsIO( () -> engine .getParquetHandler() .readParquetFiles( singletonCloseableIterator(file), finalReadSchema, checkpointPredicateIncludingSidecars) .map(FileReadResult::getData), "Reading parquet log file `%s` with readSchema=%s and predicate=%s", file, finalReadSchema, checkpointPredicateIncludingSidecars); } else if (fileName.endsWith(".json")) { topLevelIter = wrapEngineExceptionThrowsIO( () -> engine .getJsonHandler() .readJsonFiles( singletonCloseableIterator(file), finalReadSchema, checkpointPredicateIncludingSidecars), "Reading JSON log file `%s` with readSchema=%s and predicate=%s", file, finalReadSchema, checkpointPredicateIncludingSidecars); } else { throw new IOException("Unrecognized top level v2 checkpoint file format: " + fileName); } return new CloseableIterator() { @Override public void close() throws IOException { topLevelIter.close(); } @Override public boolean hasNext() { return topLevelIter.hasNext(); } @Override public ColumnarBatch next() { ColumnarBatch batch = topLevelIter.next(); if (schemaContainsAddOrRemoveFiles) { return extractSidecarsFromBatch(file, checkpointVersion, batch); } return batch; } }; } /** * Reads SidecarFile actions from ColumnarBatch, removing sidecar actions from the ColumnarBatch. * Returns a list of SidecarFile actions found. */ public ColumnarBatch extractSidecarsFromBatch( FileStatus checkpointFileStatus, long checkpointVersion, ColumnarBatch columnarBatch) { checkArgument(columnarBatch.getSchema().fieldNames().contains(LogReplay.SIDECAR_FIELD_NAME)); Path deltaLogPath = new Path(checkpointFileStatus.getPath()).getParent(); // Sidecars will exist in schema. Extract sidecar files, then remove sidecar files from // batch output. List outputFiles = new ArrayList<>(); int sidecarFieldIndexInSchema = columnarBatch.getSchema().fieldNames().indexOf(LogReplay.SIDECAR_FIELD_NAME); ColumnVector sidecarVector = columnarBatch.getColumnVector(sidecarFieldIndexInSchema); int sidecarIndexInV2Manifest = -1; // sidecar file index start from 0 in v2 manifest checkpoint for (int i = 0; i < columnarBatch.getSize(); i++) { SidecarFile sidecarFile = SidecarFile.fromColumnVector(sidecarVector, i); if (sidecarFile == null) { continue; } sidecarIndexInV2Manifest++; if (paginationContextOpt.isPresent()) { // Different from regular log files, name of sidecars are not in order, so we need to use // sidecar index in V2 manifest to compare if (paginationContextOpt.get().getLastReadSidecarFileIdx().isPresent() && sidecarIndexInV2Manifest < paginationContextOpt.get().getLastReadSidecarFileIdx().get()) { logger.info( "Pagination: skip reading sidecar file: index={}, path={}", sidecarIndexInV2Manifest, sidecarFile.getPath()); numSidecarFilesSkipped++; continue; } } FileStatus sideCarFileStatus = FileStatus.of( FileNames.sidecarFile(deltaLogPath, sidecarFile.getPath()), sidecarFile.getSizeInBytes(), sidecarFile.getModificationTime()); filesList.add(DeltaLogFile.ofSideCar(sideCarFileStatus, checkpointVersion)); } if (paginationContextOpt.isPresent()) { logger.info("Pagination: number of sidecar files skipped is {}", numSidecarFilesSkipped); } // Delete SidecarFile actions from the schema. return columnarBatch.withDeletedColumnAt(sidecarFieldIndexInSchema); } /** * Get the next file from `filesList` (.json or .checkpoint.parquet) read it + inject the * `isFromCheckpoint` information. * *

Requires that `filesList.isEmpty` is false. */ private CloseableIterator getNextActionsIter() { final DeltaLogFile nextLogFile = filesList.pop(); final FileStatus nextFile = nextLogFile.getFile(); final Path nextFilePath = new Path(nextFile.getPath()); final String fileName = nextFilePath.getName(); try { switch (nextLogFile.getLogType()) { case COMMIT: { final long fileVersion = FileNames.deltaVersion(nextFilePath); return readCommitOrCompactionFile(fileVersion, nextFile); } case LOG_COMPACTION: { // use end version as this is like a mini checkpoint, and that's what checkpoints do final long fileVersion = FileNames.logCompactionVersions(nextFilePath)._2; return readCommitOrCompactionFile(fileVersion, nextFile); } case CHECKPOINT_CLASSIC: case V2_CHECKPOINT_MANIFEST: { // If the checkpoint file is a UUID or classic checkpoint, read the top-level // checkpoint file and any potential sidecars. Otherwise, look for any other // parts of the current multipart checkpoint. CloseableIterator dataIter = getActionsIterFromSinglePartOrV2Checkpoint(nextFile, fileName) .map(batch -> new FileReadResult(batch, nextFile.getPath())); long version = checkpointVersion(nextFilePath); return combine(dataIter, true /* isFromCheckpoint */, version, Optional.empty()); } case MULTIPART_CHECKPOINT: case SIDECAR: { // Try to retrieve the remaining checkpoint files (if there are any) and issue // read request for all in one go, so that the {@link ParquetHandler} can do // optimizations like reading multiple files in parallel. CloseableIterator checkpointFiles = retrieveRemainingCheckpointFiles(nextLogFile); CloseableIterator dataIter = wrapEngineExceptionThrowsIO( () -> engine .getParquetHandler() .readParquetFiles( checkpointFiles, deltaReadSchema, checkpointPredicate), "Reading checkpoint sidecars [%s] with readSchema=%s and predicate=%s", checkpointFiles, deltaReadSchema, checkpointPredicate); long version = nextLogFile.getVersion(); return combine(dataIter, true /* isFromCheckpoint */, version, Optional.empty()); } default: throw new IOException("Unrecognized log type: " + nextLogFile.getLogType()); } } catch (IOException ex) { throw new UncheckedIOException(ex); } } private CloseableIterator readCommitOrCompactionFile( long fileVersion, FileStatus nextFile) throws IOException { // We can not read multiple JSON files in parallel (like the checkpoint files), // because each one has a different version, and we need to associate the // version with actions read from the JSON file for further optimizations later // on (faster metadata & protocol loading in subsequent runs by remembering // the version of the last version where the metadata and protocol are found). CloseableIterator dataIter = null; try { dataIter = wrapEngineExceptionThrowsIO( () -> engine .getJsonHandler() .readJsonFiles( singletonCloseableIterator(nextFile), deltaReadSchema, Optional.empty()) .map(batch -> new FileReadResult(batch, nextFile.getPath())), "Reading JSON log file `%s` with readSchema=%s", nextFile, deltaReadSchema); return combine( dataIter, false /* isFromCheckpoint */, fileVersion, Optional.of(nextFile.getModificationTime()) /* timestamp */); } catch (Exception e) { if (dataIter != null) { Utils.closeCloseablesSilently(dataIter); // close it avoid leaking resources } throw e; } } /** * Takes an input iterator of actions read from the file and metadata about the file read, and * combines it to return an Iterator. The timestamp in the ActionWrapper is only * set when the input file is not a Checkpoint. The timestamp will be set to be the * inCommitTimestamp of the delta file when available, otherwise it will be the modification time * of the file. */ private CloseableIterator combine( CloseableIterator fileReadDataIter, boolean isFromCheckpoint, long version, Optional timestamp) { // For delta files, we want to use the inCommitTimestamp from commitInfo // as the commit timestamp for the file. // Since CommitInfo should be the first action in the delta when inCommitTimestamp is // enabled, we will read the first batch and try to extract the timestamp from it. // We also ensure that rewoundFileReadDataIter is identical to the original // fileReadDataIter before any data was consumed. final CloseableIterator rewoundFileReadDataIter; Optional inCommitTimestampOpt = Optional.empty(); if (!isFromCheckpoint && fileReadDataIter.hasNext()) { FileReadResult fileReadResult = fileReadDataIter.next(); rewoundFileReadDataIter = singletonCloseableIterator(fileReadResult).combine(fileReadDataIter); inCommitTimestampOpt = InCommitTimestampUtils.tryExtractInCommitTimestamp(fileReadResult.getData()); } else { rewoundFileReadDataIter = fileReadDataIter; } final Optional finalResolvedCommitTimestamp = inCommitTimestampOpt.isPresent() ? inCommitTimestampOpt : timestamp; return new CloseableIterator() { @Override public boolean hasNext() { return rewoundFileReadDataIter.hasNext(); } @Override public ActionWrapper next() { FileReadResult fileReadResult = rewoundFileReadDataIter.next(); return new ActionWrapper( fileReadResult.getData(), isFromCheckpoint, version, finalResolvedCommitTimestamp, fileReadResult.getFilePath()); } @Override public void close() throws IOException { fileReadDataIter.close(); } }; } /** * Given a checkpoint file, retrieve all the files that are part of the same checkpoint or sidecar * files. * *

This is done by looking at the log file type and finding all the files that have the same * version number. */ private CloseableIterator retrieveRemainingCheckpointFiles( DeltaLogFile deltaLogFile) { // Find the contiguous parquet files that are part of the same checkpoint final List checkpointFiles = new ArrayList<>(); // Add the already retrieved checkpoint file to the list. checkpointFiles.add(deltaLogFile.getFile()); // Sidecar or multipart checkpoint types are the only files that can have multiple parts. if (deltaLogFile.getLogType() == SIDECAR || deltaLogFile.getLogType() == MULTIPART_CHECKPOINT) { DeltaLogFile peek = filesList.peek(); while (peek != null && deltaLogFile.getLogType() == peek.getLogType() && deltaLogFile.getVersion() == peek.getVersion()) { checkpointFiles.add(filesList.pop().getFile()); peek = filesList.peek(); } } return toCloseableIterator(checkpointFiles.iterator()); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/ActiveAddFilesIterator.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.replay; import static io.delta.kernel.internal.DeltaErrors.wrapEngineException; import static io.delta.kernel.internal.replay.LogReplay.ADD_FILE_DV_ORDINAL; import static io.delta.kernel.internal.replay.LogReplay.ADD_FILE_ORDINAL; import static io.delta.kernel.internal.replay.LogReplay.ADD_FILE_PATH_ORDINAL; import static io.delta.kernel.internal.replay.LogReplay.REMOVE_FILE_DV_ORDINAL; import static io.delta.kernel.internal.replay.LogReplay.REMOVE_FILE_ORDINAL; import static io.delta.kernel.internal.replay.LogReplay.REMOVE_FILE_PATH_ORDINAL; import static io.delta.kernel.internal.replay.LogReplayUtils.pathToUri; import static io.delta.kernel.internal.replay.LogReplayUtils.prepareSelectionVectorBuffer; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.engine.Engine; import io.delta.kernel.expressions.ExpressionEvaluator; import io.delta.kernel.expressions.Literal; import io.delta.kernel.internal.InternalScanFileUtils; import io.delta.kernel.internal.actions.DeletionVectorDescriptor; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.metrics.ScanMetrics; import io.delta.kernel.internal.replay.LogReplayUtils.UniqueFileActionTuple; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.types.StringType; import io.delta.kernel.utils.CloseableIterator; import java.io.IOException; import java.net.URI; import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * This class takes an iterator of ({@link ColumnarBatch}, isFromCheckpoint), where the columnar * data inside the columnar batch represents has top level columns "add" and "remove", and produces * an iterator of {@link FilteredColumnarBatch} with only the "add" column and with a selection * vector indicating which AddFiles are still active in the table (have not been tombstoned). */ public class ActiveAddFilesIterator implements CloseableIterator { private static final Logger logger = LoggerFactory.getLogger(ActiveAddFilesIterator.class); private final Engine engine; private final Path tableRoot; private final CloseableIterator iter; private final Set tombstonesFromJson; private final Set addFilesFromJson; private Optional next; /** * This buffer is reused across batches to keep the memory allocations minimal. It is resized as * required and the array entries are reset between batches. */ private boolean[] selectionVectorBuffer; private ExpressionEvaluator tableRootVectorGenerator; private boolean closed; /** * Metrics capturing log replay for scan building. These counters are updated as the iterator is * consumed and reported to the {@link Engine#getMetricsReporters()} when the scan is complete. */ private ScanMetrics metrics; ActiveAddFilesIterator( Engine engine, CloseableIterator iter, Path tableRoot, ScanMetrics metrics) { this.engine = engine; this.tableRoot = tableRoot; this.iter = iter; this.tombstonesFromJson = new HashSet<>(); this.addFilesFromJson = new HashSet<>(); this.next = Optional.empty(); this.metrics = metrics; } @Override public boolean hasNext() { if (closed) { throw new IllegalStateException("Can't call `hasNext` on a closed iterator."); } if (!next.isPresent()) { prepareNext(); } return next.isPresent(); } @Override public FilteredColumnarBatch next() { if (closed) { throw new IllegalStateException("Can't call `next` on a closed iterator."); } if (!hasNext()) { throw new NoSuchElementException(); } // By the definition of `hasNext`, we know that `next` is non-empty final FilteredColumnarBatch ret = next.get(); next = Optional.empty(); return ret; } @Override public void close() throws IOException { closed = true; Utils.closeCloseables(iter); // Log the metrics of the log replay of actions that are consumed so far. If the iterator // is closed before consuming all the actions, the metrics will be partial. logger.info("Active add file finding log replay metrics: {}", metrics); } /** * Grabs the next FileDataReadResult from `iter` and updates the value of `next`. * *

Internally, implements the following algorithm: 1. read all the RemoveFiles in the next * ColumnarBatch to update the `tombstonesFromJson` set 2. read all the AddFiles in that same * ColumnarBatch, unselecting ones that have already been removed or returned by updating a * selection vector 3. produces a DataReadResult by dropping that RemoveFile column from the * ColumnarBatch and using that selection vector * *

Note that, according to the Delta protocol, "a valid [Delta] version is restricted to * contain at most one file action of the same type (i.e. add/remove) for any one combination of * path and dvId". This means that step 2 could actually come before 1 - there's no temporal * dependency between them. * *

Ensures that - `next` is non-empty if there is a next result - `next` is empty if there is * no next result */ private void prepareNext() { if (next.isPresent()) { return; // already have a next result } if (!iter.hasNext()) { return; // no next result, and no batches to read } final ActionWrapper _next = iter.next(); final ColumnarBatch addRemoveColumnarBatch = _next.getColumnarBatch(); final boolean isFromCheckpoint = _next.isFromCheckpoint(); // Step 1: Update `tombstonesFromJson` with all the RemoveFiles in this columnar batch, if // and only if this batch is not from a checkpoint. // // There's no reason to put a RemoveFile from a checkpoint into `tombstonesFromJson` // since, when we generate a checkpoint, any corresponding AddFile would have // been excluded already if (!isFromCheckpoint) { final ColumnVector removesVector = addRemoveColumnarBatch.getColumnVector(REMOVE_FILE_ORDINAL); for (int rowId = 0; rowId < removesVector.getSize(); rowId++) { if (removesVector.isNullAt(rowId)) { continue; } // Note: this row doesn't represent the complete RemoveFile schema. It only contains // the fields we need for this replay. final String path = getRemoveFilePath(removesVector, rowId); final URI pathAsUri = pathToUri(path); final Optional dvId = Optional.ofNullable(getRemoveFileDV(removesVector, rowId)) .map(DeletionVectorDescriptor::getUniqueId); final UniqueFileActionTuple key = new UniqueFileActionTuple(pathAsUri, dvId); tombstonesFromJson.add(key); metrics.removeFilesFromDeltaFilesCounter.increment(); } } // Step 2: Iterate over all the AddFiles in this columnar batch in order to build up the // selection vector. We unselect an AddFile when it was removed by a RemoveFile final ColumnVector addsVector = addRemoveColumnarBatch.getColumnVector(ADD_FILE_ORDINAL); selectionVectorBuffer = prepareSelectionVectorBuffer(selectionVectorBuffer, addsVector.getSize()); boolean atLeastOneUnselected = false; int numSelectedRows = 0; for (int rowId = 0; rowId < addsVector.getSize(); rowId++) { if (addsVector.isNullAt(rowId)) { atLeastOneUnselected = true; continue; // selectionVector will be `false` at rowId by default } metrics.addFilesCounter.increment(); if (!isFromCheckpoint) { metrics.addFilesFromDeltaFilesCounter.increment(); } final String path = getAddFilePath(addsVector, rowId); final URI pathAsUri = pathToUri(path); final Optional dvId = Optional.ofNullable(getAddFileDV(addsVector, rowId)) .map(DeletionVectorDescriptor::getUniqueId); final UniqueFileActionTuple key = new UniqueFileActionTuple(pathAsUri, dvId); final boolean alreadyDeleted = tombstonesFromJson.contains(key); final boolean alreadyReturned = addFilesFromJson.contains(key); boolean doSelect = false; if (!alreadyReturned) { // Note: No AddFile will appear twice in a checkpoint, so we only need // non-checkpoint AddFiles in the set. When stats are recomputed the same // AddFile is added with stats without remove it first. if (!isFromCheckpoint) { addFilesFromJson.add(key); } if (!alreadyDeleted) { doSelect = true; selectionVectorBuffer[rowId] = true; numSelectedRows++; metrics.activeAddFilesCounter.increment(); } } else { metrics.duplicateAddFilesCounter.increment(); } if (!doSelect) { atLeastOneUnselected = true; } } ColumnarBatch scanAddFiles = addRemoveColumnarBatch; // Step 3: Drop the RemoveFile column and use the selection vector to build a new // FilteredColumnarBatch // For checkpoint files, we would only have read the adds, not the removes. if (!isFromCheckpoint) { scanAddFiles = scanAddFiles.withDeletedColumnAt(1); } // Step 4: TODO: remove this step. This is a temporary requirement until the path // in `add` is converted to absolute path. final ColumnarBatch finalScanAddFiles = scanAddFiles; if (tableRootVectorGenerator == null) { tableRootVectorGenerator = wrapEngineException( () -> engine .getExpressionHandler() .getEvaluator( finalScanAddFiles.getSchema(), Literal.ofString(tableRoot.toUri().toString()), StringType.STRING), "Get the expression evaluator for the table root"); } ColumnVector tableRootVector = wrapEngineException( () -> tableRootVectorGenerator.eval(finalScanAddFiles), "Evaluating the table root expression"); scanAddFiles = scanAddFiles.withNewColumn( 1, InternalScanFileUtils.TABLE_ROOT_STRUCT_FIELD, tableRootVector); Optional selectionColumnVector = Optional.empty(); if (atLeastOneUnselected) { selectionColumnVector = Optional.of( wrapEngineException( () -> engine .getExpressionHandler() .createSelectionVector(selectionVectorBuffer, 0, addsVector.getSize()), "Create selection vector for selected scan files")); } // TODO: skip batch if all AddFiles are unselected; issue #4941 next = Optional.of( new FilteredColumnarBatch( scanAddFiles, selectionColumnVector, _next.getFilePath(), numSelectedRows)); } public static String getAddFilePath(ColumnVector addFileVector, int rowId) { return addFileVector.getChild(ADD_FILE_PATH_ORDINAL).getString(rowId); } public static DeletionVectorDescriptor getAddFileDV(ColumnVector addFileVector, int rowId) { return DeletionVectorDescriptor.fromColumnVector( addFileVector.getChild(ADD_FILE_DV_ORDINAL), rowId); } public static String getRemoveFilePath(ColumnVector removeFileVector, int rowId) { return removeFileVector.getChild(REMOVE_FILE_PATH_ORDINAL).getString(rowId); } public static DeletionVectorDescriptor getRemoveFileDV(ColumnVector removeFileVector, int rowId) { return DeletionVectorDescriptor.fromColumnVector( removeFileVector.getChild(REMOVE_FILE_DV_ORDINAL), rowId); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/ConflictChecker.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.replay; import static io.delta.kernel.internal.DeltaErrors.concurrentDomainMetadataAction; import static io.delta.kernel.internal.DeltaErrors.wrapEngineExceptionThrowsIO; import static io.delta.kernel.internal.TableConfig.IN_COMMIT_TIMESTAMPS_ENABLED; import static io.delta.kernel.internal.actions.SingleAction.*; import static io.delta.kernel.internal.util.FileNames.checksumFile; import static io.delta.kernel.internal.util.FileNames.deltaFile; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static io.delta.kernel.internal.util.Preconditions.checkState; import static java.lang.String.format; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.ConcurrentWriteException; import io.delta.kernel.internal.*; import io.delta.kernel.internal.actions.CommitInfo; import io.delta.kernel.internal.actions.DomainMetadata; import io.delta.kernel.internal.actions.SetTransaction; import io.delta.kernel.internal.checksum.CRCInfo; import io.delta.kernel.internal.checksum.ChecksumReader; import io.delta.kernel.internal.rowtracking.RowTracking; import io.delta.kernel.internal.rowtracking.RowTrackingMetadataDomain; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.internal.util.DomainMetadataUtils; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.utils.CloseableIterable; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import java.io.*; import java.util.*; import java.util.concurrent.atomic.AtomicReference; /** * Class containing the conflict resolution logic when writing to a Delta table. * *

Currently, the support is to allow blind appends. Later on this can be extended to add support * for read-after-write scenarios. */ public class ConflictChecker { private static final int PROTOCOL_ORDINAL = CONFLICT_RESOLUTION_SCHEMA.indexOf("protocol"); private static final int METADATA_ORDINAL = CONFLICT_RESOLUTION_SCHEMA.indexOf("metaData"); private static final int TXN_ORDINAL = CONFLICT_RESOLUTION_SCHEMA.indexOf("txn"); private static final int COMMITINFO_ORDINAL = CONFLICT_RESOLUTION_SCHEMA.indexOf("commitInfo"); private static final int DOMAIN_METADATA_ORDINAL = CONFLICT_RESOLUTION_SCHEMA.indexOf("domainMetadata"); // Snapshot of the table read by the transaction that encountered the conflict // (a.k.a the losing transaction) private final Optional snapshotOpt; // Losing transaction private final TransactionImpl transaction; private final long attemptVersion; private final CloseableIterable attemptDataActions; private final List attemptDomainMetadatas; // Helper states during conflict resolution private Optional lastWinningRowIdHighWatermark = Optional.empty(); private ConflictChecker( Optional snapshotOpt, TransactionImpl transaction, long attemptVersion, List domainMetadatas, CloseableIterable dataActions) { this.snapshotOpt = snapshotOpt; this.transaction = transaction; this.attemptVersion = attemptVersion; this.attemptDomainMetadatas = domainMetadatas; this.attemptDataActions = dataActions; } /** * Resolve conflicts between the losing transaction and the winning transactions and return a * rebase state that the losing transaction needs to rebase against before attempting the commit. * * @param engine {@link Engine} instance to use * @param snapshot {@link SnapshotImpl} of the table when the losing transaction has started * @param transaction {@link TransactionImpl} that encountered the conflict (a.k.a the losing * transaction) * @param domainMetadatas List of {@link DomainMetadata} that the losing transaction is trying to * commit * @param dataActions {@link CloseableIterable} of data actions that the losing transaction is * trying to commit * @return {@link TransactionRebaseState} that the losing transaction needs to rebase against * @throws ConcurrentWriteException if there are logical conflicts between the losing transaction * and the winning transactions that cannot be resolved. */ public static TransactionRebaseState resolveConflicts( Engine engine, Optional snapshot, long attemptVersion, TransactionImpl transaction, List domainMetadatas, CloseableIterable dataActions) throws ConcurrentWriteException { // We currently set isBlindAppend=false in our CommitInfo to avoid unsafe resolution by other // connectors. Here, we still can assume that conflict resolution is safe to perform in Kernel. // checkArgument(transaction.isBlindAppend(), "Current support is for blind appends only."); return new ConflictChecker(snapshot, transaction, attemptVersion, domainMetadatas, dataActions) .resolveConflicts(engine); } public TransactionRebaseState resolveConflicts(Engine engine) throws ConcurrentWriteException { List winningCommits = getWinningCommitFiles(engine); AtomicReference> winningCommitInfoOpt = new AtomicReference<>(Optional.empty()); // no winning commits. why did we get the transaction conflict? checkState(!winningCommits.isEmpty(), "No winning commits found."); FileStatus lastWinningTxn = winningCommits.get(winningCommits.size() - 1); long lastWinningVersion = FileNames.deltaVersion(lastWinningTxn.getPath()); // Read the actions from the winning commits try (ActionsIterator actionsIterator = new ActionsIterator(engine, winningCommits, CONFLICT_RESOLUTION_SCHEMA, Optional.empty())) { actionsIterator.forEachRemaining( actionBatch -> { checkArgument(!actionBatch.isFromCheckpoint()); // no checkpoints should be read ColumnarBatch batch = actionBatch.getColumnarBatch(); if (actionBatch.getVersion() == lastWinningVersion) { Optional commitInfo = getCommitInfo(batch.getColumnVector(COMMITINFO_ORDINAL)); winningCommitInfoOpt.set(commitInfo); } handleProtocol(batch.getColumnVector(PROTOCOL_ORDINAL)); handleMetadata(batch.getColumnVector(METADATA_ORDINAL)); handleTxn(batch.getColumnVector(TXN_ORDINAL)); handleDomainMetadata(batch.getColumnVector(DOMAIN_METADATA_ORDINAL)); }); } catch (IOException ioe) { throw new UncheckedIOException("Error reading actions from winning commits.", ioe); } // Initialize updated actions for the next commit attempt with the current attempt's actions CloseableIterable updatedDataActions = attemptDataActions; List updatedDomainMetadatas = attemptDomainMetadatas; if (TableFeatures.isRowTrackingSupported(transaction.getProtocol())) { updatedDomainMetadatas = RowTracking.updateRowIdHighWatermarkIfNeeded( snapshotOpt, transaction.getProtocol(), lastWinningRowIdHighWatermark, attemptDataActions, attemptDomainMetadatas, Optional.empty() /* providedRowIdHighWatermark */); updatedDataActions = RowTracking.assignBaseRowIdAndDefaultRowCommitVersion( snapshotOpt, transaction.getProtocol(), lastWinningRowIdHighWatermark, Optional.of(attemptVersion), lastWinningVersion + 1, attemptDataActions); } Optional updatedCrcInfo = ChecksumReader.tryReadChecksumFile( engine, FileStatus.of(checksumFile(transaction.getLogPath(), lastWinningVersion).toString())); // if we get here, we have successfully rebased (i.e no logical conflicts) // against the winning transactions return new TransactionRebaseState( lastWinningVersion, getLastCommitTimestamp(lastWinningVersion, lastWinningTxn, winningCommitInfoOpt.get()), updatedDataActions, updatedDomainMetadatas, updatedCrcInfo); } /** * Class containing the rebase state from winning transactions that the current transaction needs * to rebase against before attempting the commit. * *

Currently, the rebase state is just the latest winning version of the table plus the updated * data actions and domainMetadata actions to commit. In future once we start supporting * read-after-write, row tracking, etc., we will have more state to add. For example * read-after-write will need to know the files deleted in the winning transactions to make sure * the same files are not deleted by the current (losing) transaction. */ public static class TransactionRebaseState { private final long latestVersion; private final long latestCommitTimestamp; private final CloseableIterable updatedDataActions; private final List updatedDomainMetadatas; private final Optional updatedCrcInfo; public TransactionRebaseState( long latestVersion, long latestCommitTimestamp, CloseableIterable updatedDataActions, List updatedDomainMetadatas, Optional updatedCrcInfo) { this.latestVersion = latestVersion; this.latestCommitTimestamp = latestCommitTimestamp; this.updatedDataActions = updatedDataActions; this.updatedDomainMetadatas = updatedDomainMetadatas; this.updatedCrcInfo = updatedCrcInfo; } /** * Return the latest winning version of the table. * * @return latest winning version of the table. */ public long getLatestVersion() { return latestVersion; } /** * Return the latest commit timestamp of the table. For ICT enabled tables, this is the ICT of * the latest winning transaction commit file. For non-ICT enabled tables, this is the * modification time of the latest winning transaction commit file. * * @return latest commit timestamp of the table. */ public long getLatestCommitTimestamp() { return latestCommitTimestamp; } public CloseableIterable getUpdatedDataActions() { return updatedDataActions; } public List getUpdatedDomainMetadatas() { return updatedDomainMetadatas; } public Optional getUpdatedCrcInfo() { return updatedCrcInfo; } } /** * Any protocol changes between the losing transaction and the winning transactions are not * allowed. In future once we start supporting more table features on the write side, this can be * changed to handle safe protocol changes. For now the write support in Kernel is supported at a * very first version of the protocol. * * @param protocolVector protocol rows from the winning transactions */ private void handleProtocol(ColumnVector protocolVector) { for (int rowId = 0; rowId < protocolVector.getSize(); rowId++) { if (!protocolVector.isNullAt(rowId)) { throw DeltaErrors.protocolChangedException(attemptVersion); } } } /** * Any metadata changes between the losing transaction and the winning transactions are not * allowed. * * @param metadataVector metadata rows from the winning transactions */ private void handleMetadata(ColumnVector metadataVector) { for (int rowId = 0; rowId < metadataVector.getSize(); rowId++) { if (!metadataVector.isNullAt(rowId)) { throw DeltaErrors.metadataChangedException(); } } } /** * Checks whether each of the current transaction's {@link DomainMetadata} conflicts with the * winning transaction at any domain. * *

    *
  1. Accept the current transaction if its set of metadata domains does not overlap with the * winning transaction's set of metadata domains. *
  2. Otherwise, fail the current transaction unless each conflicting domain is associated with * a domain-specific way of resolving the conflict. *
* * @param domainMetadataVector domainMetadata rows from the winning transactions * @return a map of domain name to {@link DomainMetadata} from the winning transaction */ private Map handleDomainMetadata(ColumnVector domainMetadataVector) { // Build a domain metadata map from the winning transaction. Map winningTxnDomainMetadataMap = new HashMap<>(); DomainMetadataUtils.populateDomainMetadataMap( domainMetadataVector, winningTxnDomainMetadataMap); for (DomainMetadata currentTxnDM : attemptDomainMetadatas) { // For each domain metadata action in the current transaction, check if it has a conflict with // the winning transaction. String domainName = currentTxnDM.getDomain(); DomainMetadata winningTxnDM = winningTxnDomainMetadataMap.get(domainName); if (winningTxnDM != null) { // Conflict - check if the conflict can be resolved. // Domain-specific ways of resolving the conflict can be added here. switch (domainName) { case RowTrackingMetadataDomain.DOMAIN_NAME: // We keep updating the new row ID high watermark we have seen from all winning txns. // The latest one will be used to reassign row IDs later final long winningRowIdHighWatermark = RowTrackingMetadataDomain.fromJsonConfiguration(winningTxnDM.getConfiguration()) .getRowIdHighWaterMark(); checkState( !lastWinningRowIdHighWatermark.isPresent() || lastWinningRowIdHighWatermark.get() <= winningRowIdHighWatermark, "row ID high watermark should be monotonically increasing"); this.lastWinningRowIdHighWatermark = Optional.of(winningRowIdHighWatermark); break; default: throw concurrentDomainMetadataAction(currentTxnDM, winningTxnDM); } } } return winningTxnDomainMetadataMap; } /** * Get the commit info from the winning transactions. * * @param commitInfoVector commit info rows from the winning transactions * @return the commit info */ private Optional getCommitInfo(ColumnVector commitInfoVector) { for (int rowId = 0; rowId < commitInfoVector.getSize(); rowId++) { if (!commitInfoVector.isNullAt(rowId)) { return Optional.of(CommitInfo.fromColumnVector(commitInfoVector, rowId)); } } return Optional.empty(); } private void handleTxn(ColumnVector txnVector) { // Check if the losing transaction has any txn identifier. If it does, go through the // winning transactions and make sure that the losing transaction is valid from a // idempotent perspective. Optional losingTxnIdOpt = transaction.getSetTxnOpt(); losingTxnIdOpt.ifPresent( losingTxnId -> { for (int rowId = 0; rowId < txnVector.getSize(); rowId++) { SetTransaction winningTxn = SetTransaction.fromColumnVector(txnVector, rowId); if (winningTxn != null && winningTxn.getAppId().equals(losingTxnId.getAppId()) && winningTxn.getVersion() >= losingTxnId.getVersion()) { throw DeltaErrors.concurrentTransaction( losingTxnId.getAppId(), losingTxnId.getVersion(), winningTxn.getVersion()); } } }); } private List getWinningCommitFiles(Engine engine) { // TODO delta-io/delta#5018 this should be based on attemptVersion not readSnapshot version String firstWinningCommitFile = deltaFile(transaction.getLogPath(), transaction.getReadTableVersion() + 1); try (CloseableIterator files = wrapEngineExceptionThrowsIO( () -> engine.getFileSystemClient().listFrom(firstWinningCommitFile), "Listing from %s", firstWinningCommitFile)) { // Select all winning transaction commit files. List winningCommitFiles = new ArrayList<>(); while (files.hasNext()) { FileStatus file = files.next(); if (FileNames.isCommitFile(file.getPath())) { winningCommitFiles.add(file); } } return ensureNoGapsInWinningCommits(winningCommitFiles); } catch (FileNotFoundException nfe) { // no winning commits. why did we get here? throw new IllegalStateException("No winning commits found.", nfe); } catch (IOException ioe) { throw new UncheckedIOException("Error listing files from " + firstWinningCommitFile, ioe); } } /** * Get the last commit timestamp of the table. For ICT enabled tables, this is the ICT of the * latest winning transaction commit file. For non-ICT enabled tables, this is the modification * time of the latest winning transaction commit file. * * @param lastWinningVersion last winning version of the table * @param lastWinningTxn the last winning transaction commit file * @param winningCommitInfoOpt winning commit info * @return last commit timestamp of the table */ private long getLastCommitTimestamp( long lastWinningVersion, FileStatus lastWinningTxn, Optional winningCommitInfoOpt) { if (!snapshotOpt.isPresent() || !IN_COMMIT_TIMESTAMPS_ENABLED.fromMetadata(snapshotOpt.get().getMetadata())) { return lastWinningTxn.getModificationTime(); } else { return CommitInfo.extractRequiredIctFromCommitInfoOpt( winningCommitInfoOpt, lastWinningVersion, transaction.getDataPath()); } } private static List ensureNoGapsInWinningCommits(List winningCommits) { long lastVersion = -1; for (FileStatus commit : winningCommits) { long version = FileNames.deltaVersion(commit.getPath()); checkState( lastVersion == -1 || version == lastVersion + 1, format( "Gaps in Delta log commit files. Expected version %d but got %d", (lastVersion + 1), version)); lastVersion = version; } return winningCommits; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/CreateCheckpointIterator.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.replay; import static io.delta.kernel.internal.DeltaErrors.wrapEngineException; import static io.delta.kernel.internal.actions.SingleAction.CHECKPOINT_SCHEMA; import static io.delta.kernel.internal.replay.LogReplayUtils.*; import static io.delta.kernel.internal.util.Preconditions.checkState; import io.delta.kernel.data.*; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.actions.SetTransaction; import io.delta.kernel.internal.snapshot.LogSegment; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.utils.CloseableIterator; import java.io.IOException; import java.util.*; /** * Replays a history of actions from the transaction log to reconstruct the checkpoint state of the * table. The rules for constructing the checkpoint state are defined in the Delta Protocol: Checkpoint * Reconciliation Rules. * *

Currently, the following rules are implemented: * *

    *
  • The latest protocol action seen wins *
  • The latest metaData action seen wins *
  • For txn actions, the latest version seen for a given appId wins *
  • Logical files in a table are identified by their (path, deletionVector.uniqueId) primary * key. File actions (add or remove) reference logical files, and a log can contain any number * of references to a single file. *
  • To replay the log, scan all file actions and keep only the newest reference for each * logical file. *
  • add actions in the result identify logical files currently present in the table (for * queries). remove actions in the result identify tombstones of logical files no longer * present in the table (for VACUUM). *
  • commit info actions are not included *
* *

Following rules are not implemented. They will be implemented as we add support for more table * features over time. * *

    *
  • For domainMetadata, the latest domainMetadata seen for a given domain wins. *
*/ public class CreateCheckpointIterator implements CloseableIterator { private static final int[] ADD_ORDINAL = getPathOrdinals(CHECKPOINT_SCHEMA, "add"); private static final int[] ADD_PATH_ORDINAL = getPathOrdinals(CHECKPOINT_SCHEMA, "add", "path"); private static final int[] ADD_DV_ORDINAL = getPathOrdinals(CHECKPOINT_SCHEMA, "add", "deletionVector"); private static final int[] REMOVE_ORDINAL = getPathOrdinals(CHECKPOINT_SCHEMA, "remove"); private static final int[] REMOVE_PATH_ORDINAL = getPathOrdinals(CHECKPOINT_SCHEMA, "remove", "path"); private static final int[] REMOVE_DV_ORDINAL = getPathOrdinals(CHECKPOINT_SCHEMA, "remove", "deletionVector"); private static final int[] REMOVE_DELETE_TIMESTAMP_ORDINAL = getPathOrdinals(CHECKPOINT_SCHEMA, "remove", "deletionTimestamp"); private static final int[] PROTOCOL_ORDINAL = getPathOrdinals(CHECKPOINT_SCHEMA, "protocol"); private static final int[] METADATA_ORDINAL = getPathOrdinals(CHECKPOINT_SCHEMA, "metaData"); private static final int[] TXN_ORDINAL = getPathOrdinals(CHECKPOINT_SCHEMA, "txn"); private static final int[] DOMAIN_METADATA_DOMAIN_NAME_ORDINAL = getPathOrdinals(CHECKPOINT_SCHEMA, "domainMetadata", "domain"); private final Engine engine; private final LogSegment logSegment; /** * Tombstones (i.e. RemoveFile) will be still kept in checkpoint until the tombstone timestamp is * earlier than this retention timestamp. */ private final long minFileRetentionTimestampMillis; // State of the iterator and current batch being worked on private CloseableIterator actionsIter; private boolean closed; private Optional toReturnNext = Optional.empty(); /** * This buffer is reused across batches to keep the memory allocations minimal. It is resized as * required and the array entries are reset between batches. */ private boolean[] selectionVectorBuffer; // Current state of the tombstones and add files from delta files private final Set tombstonesFromJson = new HashSet<>(); private final Set addFilesFromJson = new HashSet<>(); // Current state of the protocol and metadata. Captures whether protocol or metadata is seen. // We traverse the log in reverse, so the first encounter of protocol or metadata is considered // latest. private boolean isMetadataAlreadySeen; private boolean isProtocolAlreadySeen; // Current state of the transaction identifier (a.k.a. SetTransaction). We traverse the log in // reverse, so storing the first seen transaction version for each appId is enough for // checkpoint private final Map txnAppIdToVersion = new HashMap<>(); // Current state of all domains we have seen in {@link DomainMetadata} during the log replay. We // traverse the log in reverse, so remembering the domains we have seen is enough for creating a // checkpoint. private final Set domainSeen = new HashSet<>(); // Metadata about the checkpoint to store in `_last_checkpoint` file private long numberOfAddActions = 0; // final number of add actions survived in the checkpoint ///////////////// // Public APIs // ///////////////// public CreateCheckpointIterator( Engine engine, LogSegment logSegment, long minFileRetentionTimestampMillis) { this.engine = engine; this.logSegment = logSegment; this.minFileRetentionTimestampMillis = minFileRetentionTimestampMillis; } @Override public boolean hasNext() { initActionIterIfRequired(); checkState(!closed, "Can't call `hasNext` on a closed iterator."); return prepareNext(); } @Override public FilteredColumnarBatch next() { checkState(!closed, "Can't call `next` on a closed iterator."); if (!hasNext()) { throw new NoSuchElementException(); } FilteredColumnarBatch toReturn = toReturnNext.get(); toReturnNext = Optional.empty(); return toReturn; } @Override public void close() throws IOException { closed = true; Utils.closeCloseables(actionsIter); } /** * Number of add files in the final checkpoint. Should be called once the entire data of this * iterator is consumed. * * @return Number of add files in checkpoint. */ public long getNumberOfAddActions() { checkState(closed, "Iterator is not fully consumed yet."); return numberOfAddActions; } //////////////////////////// // Private Helper Methods // //////////////////////////// private void initActionIterIfRequired() { if (this.actionsIter == null) { this.actionsIter = new ActionsIterator( engine, logSegment.allLogFilesReversed(), CHECKPOINT_SCHEMA, Optional.empty() /* checkpoint predicate */); } } /** * Prepare the next batch to return and store it in {@link #toReturnNext} * * @return true if there is data to return, false otherwise. */ private boolean prepareNext() { if (toReturnNext.isPresent()) { return true; } if (!actionsIter.hasNext()) { return false; } ActionWrapper actionWrapper = actionsIter.next(); final ColumnarBatch actionsBatch = actionWrapper.getColumnarBatch(); final boolean isFromCheckpoint = actionWrapper.isFromCheckpoint(); // Prepare the selection vector to attach to the batch to indicate which records to // write to checkpoint and which one or not selectionVectorBuffer = prepareSelectionVectorBuffer(selectionVectorBuffer, actionsBatch.getSize()); // Step 1: Update `tombstonesFromJson` with all the RemoveFiles in this columnar batch, if // and only if this batch is not from a checkpoint. There's no reason to put a RemoveFile // from a checkpoint into `tombstonesFromJson` since, when we generate a checkpoint, // any corresponding AddFile would have been excluded already if (!isFromCheckpoint) { processRemoves( getVector(actionsBatch, REMOVE_ORDINAL), getVector(actionsBatch, REMOVE_PATH_ORDINAL), getVector(actionsBatch, REMOVE_DV_ORDINAL), getVector(actionsBatch, REMOVE_DELETE_TIMESTAMP_ORDINAL), selectionVectorBuffer); } // Step 2: Iterate over all the AddFiles in this columnar batch in order to build up the // selection vector. We unselect an AddFile when it was removed by a RemoveFile processAdds( getVector(actionsBatch, ADD_ORDINAL), getVector(actionsBatch, ADD_PATH_ORDINAL), getVector(actionsBatch, ADD_DV_ORDINAL), isFromCheckpoint, selectionVectorBuffer); // Step 3: Process the protocol final ColumnVector protocolVector = getVector(actionsBatch, PROTOCOL_ORDINAL); processProtocol(protocolVector, selectionVectorBuffer); // Step 3: Process the metadata final ColumnVector metadataVector = getVector(actionsBatch, METADATA_ORDINAL); processMetadata(metadataVector, selectionVectorBuffer); // Step 4: Process the transaction identifiers final ColumnVector txnVector = getVector(actionsBatch, TXN_ORDINAL); processTxn(txnVector, selectionVectorBuffer); // Step 5: Process the domain metadata final ColumnVector domainMetadataDomainNameVector = getVector(actionsBatch, DOMAIN_METADATA_DOMAIN_NAME_ORDINAL); processDomainMetadata(domainMetadataDomainNameVector, selectionVectorBuffer); Optional selectionVector = Optional.of(createSelectionVector(selectionVectorBuffer, actionsBatch.getSize())); toReturnNext = Optional.of(new FilteredColumnarBatch(actionsBatch, selectionVector)); return true; } private void processRemoves( ColumnVector removesVector, ColumnVector removePathVector, ColumnVector removeDvVector, ColumnVector removeDeleteTimestampVector, boolean[] selectionVectorBuffer) { for (int rowId = 0; rowId < removesVector.getSize(); rowId++) { if (removesVector.isNullAt(rowId)) { continue; // selectionVector will be `false` at rowId by default } final UniqueFileActionTuple key = getUniqueFileAction(removePathVector, removeDvVector, rowId); tombstonesFromJson.add(key); // Default is zero. Not sure if this the correct way, but it is same Delta Spark. // Ideally this should never be zero, but we are following the same behavior as Delta // Spark here. long deleteTimestamp = 0; if (!removeDeleteTimestampVector.isNullAt(rowId)) { deleteTimestamp = removeDeleteTimestampVector.getLong(rowId); } if (deleteTimestamp > minFileRetentionTimestampMillis) { // We still keep remove files in checkpoint as tombstones until the minimum // retention period has passed select(selectionVectorBuffer, rowId); } } } private void processAdds( ColumnVector addsVector, ColumnVector addPathVector, ColumnVector addDvVector, boolean isFromCheckpoint, boolean[] selectionVectorBuffer) { for (int rowId = 0; rowId < addsVector.getSize(); rowId++) { if (addsVector.isNullAt(rowId)) { continue; // selectionVector will be `false` at rowId by default } final UniqueFileActionTuple key = getUniqueFileAction(addPathVector, addDvVector, rowId); final boolean alreadyDeleted = tombstonesFromJson.contains(key); final boolean alreadyReturned = addFilesFromJson.contains(key); if (!alreadyReturned) { // Note: No AddFile will appear twice in a checkpoint, so we only need // non-checkpoint AddFiles in the set if (!isFromCheckpoint) { addFilesFromJson.add(key); } if (!alreadyDeleted) { numberOfAddActions++; select(selectionVectorBuffer, rowId); } } } } private void processProtocol(ColumnVector protocolVector, boolean[] selectionVectorBuffer) { for (int rowId = 0; rowId < protocolVector.getSize(); rowId++) { if (protocolVector.isNullAt(rowId)) { continue; // selectionVector will be `false` at rowId by default } if (isProtocolAlreadySeen) { // We do a reverse log replay. The latest always the one that should be written // to the checkpoint. Anything after the first one shouldn't be in checkpoint unselect(selectionVectorBuffer, rowId); } else { select(selectionVectorBuffer, rowId); isProtocolAlreadySeen = true; } } } private void processMetadata(ColumnVector metadataVector, boolean[] selectionVectorBuffer) { for (int rowId = 0; rowId < metadataVector.getSize(); rowId++) { if (metadataVector.isNullAt(rowId)) { continue; // selectionVector will be `false` at rowId by default } if (isMetadataAlreadySeen) { // We do a reverse log replay. The latest always the one that should be written // to the checkpoint. Anything after the first one shouldn't be in checkpoint unselect(selectionVectorBuffer, rowId); } else { select(selectionVectorBuffer, rowId); isMetadataAlreadySeen = true; } } } private void processTxn(ColumnVector txnVector, boolean[] selectionVectorBuffer) { for (int rowId = 0; rowId < txnVector.getSize(); rowId++) { SetTransaction txn = SetTransaction.fromColumnVector(txnVector, rowId); if (txn == null) { continue; // selectionVector will be `false` at rowId by default } if (txnAppIdToVersion.containsKey(txn.getAppId())) { // We do a reverse log replay. The latest txn version is the one that should be // written to the checkpoint. Anything after the first one shouldn't be in // checkpoint unselect(selectionVectorBuffer, rowId); } else { select(selectionVectorBuffer, rowId); txnAppIdToVersion.put(txn.getAppId(), txn.getVersion()); } } } /** * Processes domain metadata actions during checkpoint creation. During the reverse log replay, * for each domain, we only keep the first (latest) domain metadata action encountered by * selecting them in the selection vector, and ignore any older ones for the same domain by * unselecting them. * * @param domainMetadataVector Column vector containing domain names of domain metadata actions. * @param selectionVectorBuffer The selection vector to attach to the batch to indicate which * records to write to the checkpoint and which ones not to. */ private void processDomainMetadata( ColumnVector domainMetadataVector, boolean[] selectionVectorBuffer) { final int vectorSize = domainMetadataVector.getSize(); for (int rowId = 0; rowId < vectorSize; rowId++) { if (domainMetadataVector.isNullAt(rowId)) { continue; // selectionVector will be `false` at rowId by default } final String domain = domainMetadataVector.getString(rowId); if (domainSeen.contains(domain)) { // We do a reverse log replay. The latest domainMetadata seen for a given domain wins and // should be written to the checkpoint. Anything after the first one shouldn't be in // checkpoint. unselect(selectionVectorBuffer, rowId); } else { select(selectionVectorBuffer, rowId); domainSeen.add(domain); } } } private void unselect(boolean[] selectionVectorBuffer, int rowId) { // Just use the java assert (which are enabled in tests) for sanity checks. This should // never happen. Given this is going to be on the hot path, we want to avoid cost in // production. assert !selectionVectorBuffer[rowId] : "Row is already marked for selection, can't unselect now: " + rowId; selectionVectorBuffer[rowId] = false; } private void select(boolean[] selectionVectorBuffer, int rowId) { selectionVectorBuffer[rowId] = true; } private ColumnVector createSelectionVector(boolean[] selectionVectorBuffer, int size) { return wrapEngineException( () -> engine.getExpressionHandler().createSelectionVector(selectionVectorBuffer, 0, size), "Create selection vector for writing actions to checkpoints"); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/DeltaLogFile.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.replay; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.utils.FileStatus; /** * Internal wrapper class holding information needed to perform log replay. Represents either a * Delta commit file, classic checkpoint, a multipart checkpoint, a V2 checkpoint, or a sidecar * checkpoint. */ public class DeltaLogFile { public enum LogType { COMMIT, LOG_COMPACTION, CHECKPOINT_CLASSIC, MULTIPART_CHECKPOINT, V2_CHECKPOINT_MANIFEST, SIDECAR } public static DeltaLogFile forFileStatus(FileStatus file) { String fileName = new Path(file.getPath()).getName(); LogType logType = null; long version = -1; if (FileNames.isCommitFile(fileName)) { logType = LogType.COMMIT; version = FileNames.deltaVersion(fileName); } else if (FileNames.isLogCompactionFile(fileName)) { logType = LogType.LOG_COMPACTION; // use end version, similar to a checkpoint version = FileNames.logCompactionVersions(fileName)._2; } else if (FileNames.isClassicCheckpointFile(fileName)) { logType = LogType.CHECKPOINT_CLASSIC; version = FileNames.checkpointVersion(fileName); } else if (FileNames.isMultiPartCheckpointFile(fileName)) { logType = LogType.MULTIPART_CHECKPOINT; version = FileNames.checkpointVersion(fileName); } else if (FileNames.isV2CheckpointFile(fileName)) { logType = LogType.V2_CHECKPOINT_MANIFEST; version = FileNames.checkpointVersion(fileName); } else { throw new IllegalArgumentException( "File is not a recognized delta log type: " + file.getPath()); } return new DeltaLogFile(file, logType, version); } public static DeltaLogFile ofSideCar(FileStatus file, long version) { return new DeltaLogFile(file, LogType.SIDECAR, version); } private final FileStatus file; private final LogType logType; private final long version; private DeltaLogFile(FileStatus file, LogType logType, long version) { this.file = file; this.logType = logType; this.version = version; } public FileStatus getFile() { return file; } public LogType getLogType() { return logType; } /** * Get the version for this log file. Note that for LOG_COMPACTION files this returns the end * version, similar to a checkpoint */ public long getVersion() { return version; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/LogReplay.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.replay; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.engine.Engine; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.internal.actions.*; import io.delta.kernel.internal.checkpoints.SidecarFile; import io.delta.kernel.internal.checksum.CRCInfo; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.lang.Lazy; import io.delta.kernel.internal.metrics.ScanMetrics; import io.delta.kernel.internal.snapshot.LogSegment; import io.delta.kernel.internal.util.DomainMetadataUtils; import io.delta.kernel.types.StringType; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import java.io.IOException; import java.io.UncheckedIOException; import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Replays a history of actions, resolving them to produce the current state of the table. The * protocol for resolution is as follows: * *
    *
  • The most recent {@code AddFile} and accompanying metadata for any `(path, dv id)` tuple * wins. *
  • {@code RemoveFile} deletes a corresponding AddFile. A {@code RemoveFile} "corresponds" to * the AddFile that matches both the parquet file URI *and* the deletion vector's URI (if * any). *
  • The most recent {@code Metadata} wins. *
  • The most recent {@code Protocol} version wins. *
  • For each `(path, dv id)` tuple, this class should always output only one {@code * * FileAction} (either {@code AddFile} or {@code RemoveFile}) *
*/ public class LogReplay { private static final Logger logger = LoggerFactory.getLogger(LogReplay.class); ////////////////////////// // Static Schema Fields // ///////////////////////// /** We don't need to read the entire RemoveFile, only the path and dv info */ private static StructType REMOVE_FILE_SCHEMA = new StructType() .add("path", StringType.STRING, false /* nullable */) .add("deletionVector", DeletionVectorDescriptor.READ_SCHEMA, true /* nullable */); /** Read schema when searching for just the transaction identifiers */ public static final StructType SET_TRANSACTION_READ_SCHEMA = new StructType().add("txn", SetTransaction.FULL_SCHEMA); private static StructType getAddSchema(boolean shouldReadStats) { return shouldReadStats ? AddFile.SCHEMA_WITH_STATS : AddFile.SCHEMA_WITHOUT_STATS; } /** Read schema when searching for just the domain metadata */ public static final StructType DOMAIN_METADATA_READ_SCHEMA = new StructType().add("domainMetadata", DomainMetadata.FULL_SCHEMA); public static String SIDECAR_FIELD_NAME = "sidecar"; public static String ADDFILE_FIELD_NAME = "add"; public static String REMOVEFILE_FIELD_NAME = "remove"; public static StructType withSidecarFileSchema(StructType schema) { return schema.add(SIDECAR_FIELD_NAME, SidecarFile.READ_SCHEMA); } public static boolean containsAddOrRemoveFileActions(StructType schema) { return schema.fieldNames().contains(ADDFILE_FIELD_NAME) || schema.fieldNames().contains(REMOVEFILE_FIELD_NAME); } /** Read schema when searching for all the active AddFiles */ public static StructType getAddRemoveReadSchema(boolean shouldReadStats) { return new StructType() .add(ADDFILE_FIELD_NAME, getAddSchema(shouldReadStats)) .add(REMOVEFILE_FIELD_NAME, REMOVE_FILE_SCHEMA); } /** Read schema when searching only for AddFiles */ public static StructType getAddReadSchema(boolean shouldReadStats) { return new StructType().add(ADDFILE_FIELD_NAME, getAddSchema(shouldReadStats)); } public static int ADD_FILE_ORDINAL = 0; public static int ADD_FILE_PATH_ORDINAL = AddFile.SCHEMA_WITHOUT_STATS.indexOf("path"); public static int ADD_FILE_DV_ORDINAL = AddFile.SCHEMA_WITHOUT_STATS.indexOf("deletionVector"); public static int REMOVE_FILE_ORDINAL = 1; public static int REMOVE_FILE_PATH_ORDINAL = REMOVE_FILE_SCHEMA.indexOf("path"); public static int REMOVE_FILE_DV_ORDINAL = REMOVE_FILE_SCHEMA.indexOf("deletionVector"); /////////////////////////////////// // Member fields and constructor // /////////////////////////////////// private final Path dataPath; private final Lazy lazyLogSegment; private final Lazy> lazyLatestCrcInfo; private final Lazy> lazyActiveDomainMetadataMap; /** * Creates a new LogReplay instance. * * @param dataPath the path to the Delta table * @param engine the engine to use for reading log files * @param lazyLogSegment lazy loader for the log segment * @param lazyCrcInfo lazy loader for the CRC file (shared with ProtocolMetadataLogReplay) */ public LogReplay( Engine engine, Path dataPath, Lazy lazyLogSegment, Lazy> lazyCrcInfo) { this.dataPath = dataPath; this.lazyLogSegment = lazyLogSegment; this.lazyLatestCrcInfo = lazyCrcInfo; // TODO: Refactor DomainMetadata loading to static utility, just like P & M loading // Lazy loading of domain metadata only when needed this.lazyActiveDomainMetadataMap = new Lazy<>( () -> loadDomainMetadataMap(engine).entrySet().stream() .filter(entry -> !entry.getValue().isRemoved()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); } ///////////////// // Public APIs // ///////////////// public long getVersion() { return getLogSegment().getVersion(); } public Optional getLatestTransactionIdentifier(Engine engine, String applicationId) { return loadLatestTransactionVersion(engine, applicationId); } /** Returns map for all active domain metadata. */ public Map getActiveDomainMetadataMap() { return lazyActiveDomainMetadataMap.get(); } /** * Returns the CRC info for the current snapshot version if available. Lazily loads and caches the * CRC file on first access. Returns empty if no CRC file exists at the snapshot version. */ public Optional getCrcInfoAtSnapshotVersion() { // TODO: We should first just check if the checksum file in the LogSegment is at this snapshot // version. return lazyLatestCrcInfo.get().filter(crcInfo -> crcInfo.getVersion() == getVersion()); } /** * Returns an iterator of {@link FilteredColumnarBatch} representing all the active AddFiles in * the table. * *

Statistics are conditionally read for the AddFiles based on {@code shouldReadStats}. The * returned batches have schema: * *

    *
  1. name: {@code add} *

    type: {@link AddFile#SCHEMA_WITH_STATS} if {@code shouldReadStats=true}, otherwise * {@link AddFile#SCHEMA_WITHOUT_STATS} *

*/ public CloseableIterator getAddFilesAsColumnarBatches( Engine engine, boolean shouldReadStats, Optional checkpointPredicate, ScanMetrics scanMetrics, Optional paginationContextOpt) { // We do not need to look at any `remove` files from the checkpoints. Skip the column to save // I/O. Note that we are still going to process the row groups. Adds and removes are randomly // scattered through checkpoint part files, so row group push down is unlikely to be useful. final CloseableIterator addRemoveIter = new ActionsIterator( engine, getLogReplayFiles(getLogSegment()), getAddRemoveReadSchema(shouldReadStats), getAddReadSchema(shouldReadStats), checkpointPredicate, paginationContextOpt); return new ActiveAddFilesIterator(engine, addRemoveIter, dataPath, scanMetrics); } public LogSegment getLogSegment() { return lazyLogSegment.get(); } //////////////////// // Helper Methods // //////////////////// // For now we always read log compaction files. Plumb an option through to here if we ever want to // make it configurable private boolean readLogCompactionFiles = true; /** * Get the files to use for this log replay, can be configured for example to use or not use log * compaction files */ private List getLogReplayFiles(LogSegment logSegment) { if (readLogCompactionFiles) { return logSegment.allFilesWithCompactionsReversed(); } else { return logSegment.allLogFilesReversed(); } } private Optional loadLatestTransactionVersion(Engine engine, String applicationId) { try (CloseableIterator reverseIter = new ActionsIterator( engine, getLogReplayFiles(getLogSegment()), SET_TRANSACTION_READ_SCHEMA, Optional.empty())) { while (reverseIter.hasNext()) { final ColumnarBatch columnarBatch = reverseIter.next().getColumnarBatch(); assert (columnarBatch.getSchema().equals(SET_TRANSACTION_READ_SCHEMA)); final ColumnVector txnVector = columnarBatch.getColumnVector(0); for (int rowId = 0; rowId < txnVector.getSize(); rowId++) { if (!txnVector.isNullAt(rowId)) { SetTransaction txn = SetTransaction.fromColumnVector(txnVector, rowId); if (txn != null && applicationId.equals(txn.getAppId())) { return Optional.of(txn.getVersion()); } } } } } catch (IOException ex) { throw new RuntimeException("Failed to fetch the transaction identifier", ex); } return Optional.empty(); } /** * Loads the domain metadata map, either from CRC info (if available) or from the transaction log. * * @param engine The engine to use for loading from log when necessary * @return A map of domain names to their metadata */ private Map loadDomainMetadataMap(Engine engine) { long startTimeMillis = System.currentTimeMillis(); // Case 1: CRC does not exist or does not have domain metadata => read DM from log final Optional latestCrcInfoOpt = lazyLatestCrcInfo.get(); if (!latestCrcInfoOpt.isPresent() || !latestCrcInfoOpt.get().getDomainMetadata().isPresent()) { Map domainMetadataMap = loadDomainMetadataMapFromLog(engine, Optional.empty()); logger.info( "{}:No domain metadata available in CRC info," + " loading domain metadata for version {} from logs took {}ms", dataPath.toString(), getVersion(), System.currentTimeMillis() - startTimeMillis); return domainMetadataMap; } final CRCInfo latestCrcInfo = latestCrcInfoOpt.get(); // Case 2: CRC exists at the snapshot version and has domain metadata => read DM from CRC if (latestCrcInfo.getVersion() == getVersion()) { Map domainMetadataMap = latestCrcInfo.getDomainMetadata().get().stream() .collect(Collectors.toMap(DomainMetadata::getDomain, Function.identity())); logger.info( "{}:CRC for version {} found, loading domain metadata from CRC took {}ms", dataPath.toString(), getVersion(), System.currentTimeMillis() - startTimeMillis); return domainMetadataMap; } // Case 3: CRC exists at an *earlier* version and has domain metadata => read DM from CRC and // read DM from log for newer versions Map finalDomainMetadataMap = loadDomainMetadataMapFromLog(engine, Optional.of(latestCrcInfo.getVersion() + 1)); // Add domains from the CRC that don't exist in the incremental log data // - If a domain is updated to the newer versions or removed, it will exist in // finalDomainMetadataMap, use the one in the map. // - If a domain is only in the CRC file, use the one from CRC. latestCrcInfo .getDomainMetadata() .get() .forEach( domainMetadataInCrc -> { if (!finalDomainMetadataMap.containsKey(domainMetadataInCrc.getDomain())) { finalDomainMetadataMap.put(domainMetadataInCrc.getDomain(), domainMetadataInCrc); } }); logger.info( "{}: Loading domain metadata for version {} from logs with crc version {} took {}ms", dataPath.toString(), getVersion(), latestCrcInfo.getVersion(), System.currentTimeMillis() - startTimeMillis); return finalDomainMetadataMap; } /** * Retrieves a map of domainName to {@link DomainMetadata} from the log files. * *

Loading domain metadata requires an additional round of log replay so this is done lazily * and only when domain metadata is requested. * * @param engine The engine used to process the log files. * @param minLogVersion The minimum log version to read (inclusive). When provided, only reads log * files * starting from this version. When not provided, reads the entire log. * For * incremental loading from crc, this is typically set to (crc version + 1). * @return A map where the keys are domain names and the values are the corresponding {@link * DomainMetadata} objects. * @throws UncheckedIOException if an I/O error occurs while closing the iterator. */ private Map loadDomainMetadataMapFromLog( Engine engine, Optional minLogVersion) { long logReadCount = 0; try (CloseableIterator reverseIter = new ActionsIterator( engine, getLogReplayFiles(getLogSegment()), DOMAIN_METADATA_READ_SCHEMA, Optional.empty() /* checkpointPredicate */)) { Map domainMetadataMap = new HashMap<>(); while (reverseIter.hasNext()) { final ActionWrapper nextElem = reverseIter.next(); final long version = nextElem.getVersion(); // Stop before processing any batch from a version below minLogVersion. We use // less-than (not equality) to ensure all batches from the minLogVersion file are // fully processed, since a single large log file may produce multiple batches. if (minLogVersion.isPresent() && version < minLogVersion.get()) { break; } final ColumnarBatch columnarBatch = nextElem.getColumnarBatch(); logReadCount++; assert (columnarBatch.getSchema().equals(DOMAIN_METADATA_READ_SCHEMA)); final ColumnVector dmVector = columnarBatch.getColumnVector(0); // We are performing a reverse log replay. This function ensures that only the first // encountered domain metadata for each domain is added to the map. DomainMetadataUtils.populateDomainMetadataMap(dmVector, domainMetadataMap); } logger.info( "{}: Loading domain metadata from log for version {}, " + "read {} actions, using crc version {}", dataPath.toString(), getVersion(), logReadCount, minLogVersion.map(String::valueOf).orElse("N/A")); return domainMetadataMap; } catch (IOException ex) { throw new UncheckedIOException("Could not close iterator", ex); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/LogReplayUtils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.replay; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.internal.actions.DeletionVectorDescriptor; import io.delta.kernel.internal.util.Tuple2; import io.delta.kernel.types.DataType; import io.delta.kernel.types.StructType; import java.net.URI; import java.net.URISyntaxException; import java.util.*; public class LogReplayUtils { private LogReplayUtils() {} public static class UniqueFileActionTuple extends Tuple2> { UniqueFileActionTuple(URI fileURI, Optional deletionVectorId) { super(fileURI, deletionVectorId); } } public static UniqueFileActionTuple getUniqueFileAction( ColumnVector pathVector, ColumnVector dvVector, int rowId) { final String path = pathVector.getString(rowId); final URI pathAsUri = pathToUri(path); final Optional dvId = Optional.ofNullable(DeletionVectorDescriptor.fromColumnVector(dvVector, rowId)) .map(DeletionVectorDescriptor::getUniqueId); return new UniqueFileActionTuple(pathAsUri, dvId); } static boolean[] prepareSelectionVectorBuffer(boolean[] currentSelectionVector, int newSize) { if (currentSelectionVector == null || currentSelectionVector.length < newSize) { currentSelectionVector = new boolean[newSize]; } else { // reset the array - if we are reusing the same buffer. Arrays.fill(currentSelectionVector, false); } return currentSelectionVector; } static URI pathToUri(String path) { try { return new URI(path); } catch (URISyntaxException ex) { throw new RuntimeException(ex); } } /** * Get the ordinals of the column path at each level. Ordinal refers position of a column within a * struct type column. For example: `struct(a: struct(a1: int, b1: long))` and lookup path is * `a.b1` returns `0, 1`. */ static int[] getPathOrdinals(StructType schema, String... path) { checkArgument(path.length > 0, "Invalid path"); int[] pathOrdinals = new int[path.length]; DataType currentLevelDataType = schema; for (int level = 0; level < path.length; level++) { checkArgument(currentLevelDataType instanceof StructType, "Invalid search path"); StructType asStructType = (StructType) currentLevelDataType; pathOrdinals[level] = asStructType.indexOf(path[level]); currentLevelDataType = asStructType.at(pathOrdinals[level]).getDataType(); } return pathOrdinals; } /** Get the vector corresponding to the given ordinals at each level of the column path. */ static ColumnVector getVector(ColumnarBatch batch, int[] pathOrdinals) { checkArgument(pathOrdinals.length > 0, "Invalid path ordinals size"); ColumnVector vector = null; for (int level = 0; level < pathOrdinals.length; level++) { int levelOrdinal = pathOrdinals[level]; vector = (level == 0) ? batch.getColumnVector(levelOrdinal) : vector.getChild(levelOrdinal); } return vector; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/PageToken.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.replay; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.data.Row; import io.delta.kernel.internal.data.GenericRow; import io.delta.kernel.types.*; import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Optional; /** Page Token Class for Pagination Support */ public class PageToken { public static PageToken fromRow(Row row) { requireNonNull(row); // Check #1: Correct schema checkArgument( PAGE_TOKEN_SCHEMA.equals(row.getSchema()), String.format( "Invalid Page Token: input row schema does not match expected PageToken schema." + "\nExpected: %s\nGot: %s", PAGE_TOKEN_SCHEMA, row.getSchema())); // Check #2: All required fields are present for (int i = 0; i < PAGE_TOKEN_SCHEMA.length(); i++) { if (PAGE_TOKEN_SCHEMA.at(i).getName().equals("lastReadSidecarFileIdx")) continue; checkArgument( !row.isNullAt(i), String.format( "Invalid Page Token: required field '%s' is null at index %d", PAGE_TOKEN_SCHEMA.at(i).getName(), i)); } return new PageToken( row.getString(0), // lastReadLogFilePath row.getLong(1), // lastReturnedRowIndex Optional.ofNullable(row.isNullAt(2) ? null : row.getLong(2)), // lastReadSidecarFileIdx row.getString(3), // kernelVersion row.getString(4), // tablePath row.getLong(5), // tableVersion row.getInt(6), // predicateHash row.getInt(7)); // logSegmentHash } public static final StructType PAGE_TOKEN_SCHEMA = new StructType() .add("lastReadLogFilePath", StringType.STRING, false /* nullable */) .add("lastReturnedRowIndex", LongType.LONG, false /* nullable */) .add("lastReadSidecarFileIdx", LongType.LONG, true /* nullable */) .add("kernelVersion", StringType.STRING, false /* nullable */) .add("tablePath", StringType.STRING, false /* nullable */) .add("tableVersion", LongType.LONG, false /* nullable */) .add("predicateHash", IntegerType.INTEGER, false /* nullable */) .add("logSegmentHash", IntegerType.INTEGER, false /* nullable */); // ===== Variables to mark where the last page ended (and the current page starts) ===== /** The last log file read in the previous page. */ private final String lastReadLogFilePath; /** * The index of the last row that was returned from the last read log file during the previous * page. This row index is relative to the file. The current page should begin from the row * immediately after this row index. */ private final long lastReturnedRowIndex; /** * Optional index of the last sidecar checkpoint file read in the previous page. This index is * based on the ordering of sidecar files in the V2 manifest checkpoint file. If present, it must * represent the final sidecar file that was read and must correspond to the same file as * `lastReadLogFilePath`. */ private final Optional lastReadSidecarFileIdx; // ===== Variables for validating query params and detecting changes in log segment ===== private final String kernelVersion; private final String tablePath; private final long tableVersion; private final int predicateHash; private final int logSegmentHash; public PageToken( String lastReadLogFilePath, long lastReturnedRowIndex, Optional lastReadSidecarFileIdx, String kernelVersion, String tablePath, long tableVersion, int predicateHash, int logSegmentHash) { this.lastReadLogFilePath = requireNonNull(lastReadLogFilePath, "lastReadLogFilePath is null"); this.lastReturnedRowIndex = lastReturnedRowIndex; this.lastReadSidecarFileIdx = lastReadSidecarFileIdx; this.kernelVersion = requireNonNull(kernelVersion, "kernelVersion is null"); this.tablePath = requireNonNull(tablePath, "tablePath is null"); this.tableVersion = tableVersion; this.predicateHash = predicateHash; this.logSegmentHash = logSegmentHash; } public Row toRow() { Map pageTokenMap = new HashMap<>(); pageTokenMap.put(0, lastReadLogFilePath); pageTokenMap.put(1, lastReturnedRowIndex); pageTokenMap.put(2, lastReadSidecarFileIdx.orElse(null)); pageTokenMap.put(3, kernelVersion); pageTokenMap.put(4, tablePath); pageTokenMap.put(5, tableVersion); pageTokenMap.put(6, predicateHash); pageTokenMap.put(7, logSegmentHash); return new GenericRow(PAGE_TOKEN_SCHEMA, pageTokenMap); } public String getLastReadLogFilePath() { return lastReadLogFilePath; } public long getLastReturnedRowIndex() { return lastReturnedRowIndex; } public Optional getLastReadSidecarFileIdx() { return lastReadSidecarFileIdx; } public String getTablePath() { return tablePath; } public long getTableVersion() { return tableVersion; } public String getKernelVersion() { return kernelVersion; } public int getPredicateHash() { return predicateHash; } public int getLogSegmentHash() { return logSegmentHash; } @Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null || getClass() != obj.getClass()) { return false; } PageToken other = (PageToken) obj; return lastReturnedRowIndex == other.lastReturnedRowIndex && tableVersion == other.tableVersion && predicateHash == other.predicateHash && logSegmentHash == other.logSegmentHash && Objects.equals(lastReadSidecarFileIdx, other.lastReadSidecarFileIdx) && Objects.equals(lastReadLogFilePath, other.lastReadLogFilePath) && Objects.equals(kernelVersion, other.kernelVersion) && Objects.equals(tablePath, other.tablePath); } @Override public int hashCode() { return Objects.hash( lastReadLogFilePath, lastReturnedRowIndex, lastReadSidecarFileIdx, kernelVersion, tablePath, tableVersion, predicateHash, logSegmentHash); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/PaginatedScanFilesIteratorImpl.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.replay; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static io.delta.kernel.internal.util.Preconditions.checkState; import io.delta.kernel.Meta; import io.delta.kernel.PaginatedScanFilesIterator; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.utils.CloseableIterator; import java.io.IOException; import java.util.NoSuchElementException; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Implementation of {@link PaginatedScanFilesIterator} */ public class PaginatedScanFilesIteratorImpl implements PaginatedScanFilesIterator { private static final Logger logger = LoggerFactory.getLogger(PaginatedScanFilesIteratorImpl.class); /** * Filtered ScanFiles iterator from the base scan, excluding batches from fully consumed log * files. For example, if previous pages have fully consumed sidecar files A and B, and partially * consumed sidecar C, this iterator will exclude all batches from A and B, but include all * batches from C. * *

Note: When no cached hash sets are available, this iterator will include all batches from * JSON log files. */ private final CloseableIterator baseFilteredScanFilesIter; /** Pagination Context that carries page token and page size info. */ private final PaginationContext paginationContext; /** Maximum number of ScanFiles to include in current page */ private final long pageSize; /** Total number of ScanFiles returned in current page */ private long numScanFilesReturned; /** * The name of the last log file that was read during pagination. * *

This value is used to track which log file the current scan is processing. * *

Initialization: * *

    *
  • If the pagination token includes a log file path, this value is initialized from it. *
  • If the pagination token does not include a log file path (i.e., the previous page did not * read any log file), this value is initialized to {@code null}. *
*/ private String lastReadLogFilePath = null; /** * Tracks the index of the last read sidecar file during pagination. * *

The index starts from 0 for the first sidecar file read. It is incremented each time a new * sidecar file is encountered during scanning. * *

Initialization: * *

    *
  • If the pagination token includes a sidecar index, this value is initialized from it. *
  • If the pagination token does not include a sidecar index (i.e., no sidecar file was read * in the previous page), this value is initialized to {@code -1}. *
*/ private long lastSidecarIndex = -1; /** * The index of the last row returned from the last log file that was read. * *

For example, if the last page contains 3 batches from the same log file, and each batch has * 10 rows, this value will be 29 (since row indices start at 0). * *

This value corresponds to the one saved in the page token if present. */ private long lastReturnedRowIndex = -1; private Optional currentBatch = Optional.empty(); private boolean closed = false; private boolean isBaseScanExhausted = false; private boolean hasLeastOneBatchConsumed = false; /** * Constructs a paginated iterator over scan files on top of a given filtered scan files iterator * and pagination context. * * @param baseFilteredScanFilesIter The underlying scan files iterator with data skipping and * partition pruning applied. This iterator serves as the source of filtered scan results for * pagination. * @param paginationContext The pagination context that carries pagination-related information, * such as the maximum number of files to return in a page. */ public PaginatedScanFilesIteratorImpl( CloseableIterator baseFilteredScanFilesIter, PaginationContext paginationContext) { this.baseFilteredScanFilesIter = baseFilteredScanFilesIter; this.paginationContext = paginationContext; this.pageSize = paginationContext.getPageSize(); if (paginationContext.getLastReadLogFilePath().isPresent()) { lastReadLogFilePath = paginationContext.getLastReadLogFilePath().get(); } if (paginationContext.getLastReadSidecarFileIdx().isPresent()) { lastSidecarIndex = paginationContext.getLastReadSidecarFileIdx().get(); } } /** * Returns a page token representing the position of the last consumed batch, corresponding to the * most recent {@code next()} call. * *

Note: This method can be called after the paginated iterator has been closed. */ @Override public Optional getCurrentPageToken() { // User must call getCurrentPageToken() after they call next() at least once. checkState( hasLeastOneBatchConsumed, "Can't call getCurrentPageToken() without consuming any batches!"); // Return empty page token to signal pagination completes. if (isBaseScanExhausted) { return Optional.empty(); } // TODO: replace hash value of log segment Row pageTokenRow = new PageToken( lastReadLogFilePath, lastReturnedRowIndex, (lastSidecarIndex == -1) ? Optional.empty() : Optional.of(lastSidecarIndex), Meta.KERNEL_VERSION, paginationContext.getTablePath(), paginationContext.getTableVersion(), paginationContext.getPredicateHash(), paginationContext.getLogSegmentHash()) .toRow(); return Optional.of(pageTokenRow); } @Override public boolean hasNext() { checkState(!closed, "Can't call `hasNext` on a closed iterator."); if (!currentBatch.isPresent()) { prepareNext(); } return currentBatch.isPresent(); } /** * Prepares the next FilteredColumnarBatch to return in the current page. Skips batches that have * already been returned in the previous page, based on file path, row index and sidecar index * stored in the pagination context. */ private void prepareNext() { if (currentBatch.isPresent()) return; if (!baseFilteredScanFilesIter.hasNext()) { isBaseScanExhausted = true; return; } if (numScanFilesReturned >= pageSize) return; Optional tokenLastReadFilePathOpt = paginationContext.getLastReadLogFilePath(); Optional tokenLastReadSidecarFileIdxOpt = paginationContext.getLastReadSidecarFileIdx(); while (baseFilteredScanFilesIter.hasNext() && numScanFilesReturned < pageSize) { final FilteredColumnarBatch batch = baseFilteredScanFilesIter.next(); validateBatch(batch); final String batchFilePath = batch.getFilePath().get(); final long numRowsInBatch = batch.getData().getSize(); // Case 1: Skip batches from fully consumed files. // A file is considered fully consumed if it appears earlier (in reverse lexicographic order) // than the last read file recorded in the page token. // // Example: // - Suppose the previous page ends at file 13.json // - Then, all files after 13.json (e.g., 14.json, 15.json, etc.) // have already been processed and are considered fully consumed. // - Any batches from these files should be skipped here. if (isBatchFromFullyConsumedFile( batchFilePath, tokenLastReadFilePathOpt, tokenLastReadSidecarFileIdxOpt)) { // All fully consumed multi-part checkpoints should already be skipped in ActionsIterator. // Fully consumed delta commit, log compaction and V2 checkpoint files won't be skipped in // ActionsIterator. checkArgument(!FileNames.isMultiPartCheckpointFile(batchFilePath)); logger.info("Pagination: skipping batch from a fully consumed file : {}", batchFilePath); continue; } // Case 2: The batch belongs to the same last read log file recorded in the page token. // In this case, we may have partially consumed this file on the previous page, // so we need to decide whether to skip the current batch based on row index. // // Example: // - Page 1 ends after processing row index 9 in file 13.json (i.e., the first 10 rows). // - This includes two batches: batch 1 (rows 0–4), batch 2 (rows 5–9). // - When reading page 2, we may re-encounter these batches. // * Batch 1 ends at row 4 → skip (already returned). // * Batch 2 ends at row 9 → skip (already returned). // * Batch 3 starts at row 10 → keep (new data). else if (isBatchFromLastFileInToken( batchFilePath, tokenLastReadFilePathOpt, tokenLastReadSidecarFileIdxOpt)) { Optional tokenLastReturnedRowIndexOpt = paginationContext.getLastReturnedRowIndex(); // Skip this batch if its last row index is smaller than or equal to last returned row index // in token. if (tokenLastReturnedRowIndexOpt.isPresent() && lastReturnedRowIndex + numRowsInBatch <= tokenLastReturnedRowIndexOpt.get()) { lastReturnedRowIndex += numRowsInBatch; logger.info( "Pagination: skipping batch from a partially consumed file : {}, " + "last row index is {}", batchFilePath, lastReturnedRowIndex); continue; } } // Case 3: If this batch belongs to an "unseen file" — meaning a file whose content was // not read at all in any previous page. In other words, this file is fully unconsumed: // it was neither partially read nor fully read before. // Batches from such files won't be skipped. // currentBatch will be included in the current page. currentBatch = Optional.of(batch); // Found a valid batch, break out of the loop break; } } @Override public FilteredColumnarBatch next() { checkState(!closed, "Can't call `next` on a closed iterator."); if (!hasNext()) { throw new NoSuchElementException(); } hasLeastOneBatchConsumed = true; final FilteredColumnarBatch ret = currentBatch.get(); validateBatch(ret); final long numSelectedAddFilesInBatch = ret.getPreComputedNumSelectedRows().get(); final String batchFilePath = ret.getFilePath().get(); // This batch is the first one we've seen from an "unseen file" during the current page // read; // update tracking state to reflect that we're now reading an "unseen file". // Example: // - Page 1 ends midway through 18.json. // - Page 2 resumes, completes 18.json, then sees 17.json. // - 17.json was not seen in Page 1, so it's an "unseen file". // - In 17.json, all batches are emitted. // - Only the first batch triggers state reset via `isFirstBatchFromNewFile`. if (isFirstBatchFromUnseenFile(batchFilePath)) { lastReadLogFilePath = batchFilePath; // Start from -1 so adding the first batch size gives correct 0-based row index. lastReturnedRowIndex = -1; logger.info("Pagination: reading new file: {}", lastReadLogFilePath); if (isSidecar(batchFilePath)) { lastSidecarIndex++; } } // Calculate the row index of the last row in current batch within the file. lastReturnedRowIndex += ret.getData().getSize(); numScanFilesReturned += numSelectedAddFilesInBatch; logger.info( "Pagination: emit a new batch: lastReadLogFilePath: {}, lastReturnedRowIndex: {}", lastReadLogFilePath, lastReturnedRowIndex); logger.info("Pagination: total numScanFilesReturned: {}", numScanFilesReturned); currentBatch = Optional.empty(); return ret; } void validateBatch(FilteredColumnarBatch batch) { // FilePath and pre-computed number of selected rows are expected to be present; both are // computed and set in ActiveAddFilesIterator (when building FilteredColumnarBatch from // ActionWrapper) checkArgument(batch.getFilePath().isPresent(), "File path doesn't exist!"); checkArgument( batch.getPreComputedNumSelectedRows().isPresent(), "Pre-computed number of selected rows doesn't exist!"); } /** * Returns {@code true} if the current batch comes from a fully consumed file in previous pages, * as determined by the page token. */ private boolean isBatchFromFullyConsumedFile( String batchFilePath, Optional tokenLastReadLogFilePathOpt, Optional tokenLastReadSidecarFileIdxOpt) { if (tokenLastReadSidecarFileIdxOpt.isPresent()) { // If a sidecar file was read in the previous page, all non-sidecar files (i.e., delta commit, // log compaction and V2 checkpoint) are considered fully consumed. return !isSidecar(batchFilePath); } else if (tokenLastReadLogFilePathOpt.isPresent()) { // Delta log files are ordered in reverse lexicographic order (i.e., higher file names appear // earlier). // If the current batch’s log file name is greater than the last one recorded in the token, // it means this file appeared earlier in the segment and has already been processed. return !isSidecar(batchFilePath) && batchFilePath.compareTo(tokenLastReadLogFilePathOpt.get()) > 0; } return false; } /** * Returns true if the current batch is from the same file (either log or sidecar) that the * previous page ended at, based on the file path and sidecar index recorded in the page token. */ private boolean isBatchFromLastFileInToken( String batchFilePath, Optional tokenLastReadFilePathOpt, Optional tokenLastReadSidecarFileIdxOpt) { // Check if batch file path matches last read file path recorded in the page token (if // present). boolean isSameFilePath = tokenLastReadFilePathOpt.isPresent() && batchFilePath.equals(tokenLastReadFilePathOpt.get()); if (!isSameFilePath) return false; // For sidecar files, if file path matches, sidecar index must also present and match. if (isSidecar(batchFilePath)) { checkArgument( tokenLastReadSidecarFileIdxOpt.isPresent() && lastSidecarIndex == tokenLastReadSidecarFileIdxOpt.get(), "Sidecar index mismatch for file: %s", batchFilePath); } return true; } /** * Returns {@code true} if the current batch is the first one from a different file than the last * seen, indicating the start of a new unseen file during pagination. */ private boolean isFirstBatchFromUnseenFile(String batchFilePath) { // If the batch's file path differs from {@code lastReadLogFilePath}, it's considered an // unseen file. if (!batchFilePath.equals(lastReadLogFilePath)) { // For non-sidecar files, files must appear in reverse lexicographic order — // i.e., the current file must come *before* the last seen file. checkArgument( isSidecar(batchFilePath) || lastReadLogFilePath == null || batchFilePath.compareTo(lastReadLogFilePath) < 0, "Expected file '%s' to appear after last read file '%s' in reverse lexicographic order, " + "unless it's a sidecar file", batchFilePath, lastReadLogFilePath); return true; } return false; } // TODO: move isSidecar() to FileNames private boolean isSidecar(String filePath) { return filePath.contains("/_delta_log/_sidecars/") && filePath.endsWith(".parquet"); } @Override public void close() throws IOException { if (!closed) { closed = true; Utils.closeCloseables(baseFilteredScanFilesIter); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/PaginationContext.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.replay; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.Meta; import java.util.Objects; import java.util.Optional; /** {@code PaginationContext} carries pagination-related information. */ public class PaginationContext { public static PaginationContext forPageWithPageToken( String tablePath, long tableVersion, int logSegmentHash, int predicateHash, long pageSize, PageToken pageToken) { Objects.requireNonNull(pageToken, "page token is null"); Objects.requireNonNull(tablePath, "table path is null"); checkArgument( tablePath.equals(pageToken.getTablePath()), "Invalid page token: token table path does not match the requested table path. " + "Expected: %s, Found: %s", tablePath, pageToken.getTablePath()); checkArgument( tableVersion == pageToken.getTableVersion(), "Invalid page token: token table version does not match the requested table version. " + "Expected: %d, Found: %d", tableVersion, pageToken.getTableVersion()); checkArgument( Meta.KERNEL_VERSION.equals(pageToken.getKernelVersion()), "Invalid page token: token kernel version does not match the requested kernel version. " + "Expected: %s, Found: %s", Meta.KERNEL_VERSION, pageToken.getKernelVersion()); checkArgument( predicateHash == pageToken.getPredicateHash(), "Invalid page token: token predicate hash does not match the requested predicate hash. " + "Expected: %s, Found: %s", predicateHash, pageToken.getPredicateHash()); checkArgument( logSegmentHash == pageToken.getLogSegmentHash(), "Invalid page token: token log segment hash does not match the requested log segment hash. " + "Expected: %s, Found: %s", logSegmentHash, pageToken.getLogSegmentHash()); return new PaginationContext( tablePath, tableVersion, logSegmentHash, predicateHash, pageSize, Optional.of(pageToken)); } public static PaginationContext forFirstPage( String tablePath, long tableVersion, int logSegmentHash, int predicateHash, long pageSize) { Objects.requireNonNull(tablePath, "table path is null"); return new PaginationContext( tablePath, tableVersion, logSegmentHash, predicateHash, pageSize, Optional.empty() /* page token */); } private final String tablePath; private final long tableVersion; private final int predicateHash; private final int logSegmentHash; /** maximum number of ScanFiles to return in the current page */ private final long pageSize; /** Optional Page Token */ private final Optional pageToken; // TODO: add cached log replay hashsets related info private PaginationContext( String tablePath, long tableVersion, int logSegmentHash, int predicateHash, long pageSize, Optional pageToken) { checkArgument(pageSize > 0, "Page size must be greater than zero!"); this.tablePath = tablePath; this.tableVersion = tableVersion; this.logSegmentHash = logSegmentHash; this.predicateHash = predicateHash; this.pageSize = pageSize; this.pageToken = pageToken; } public String getTablePath() { return tablePath; } public long getTableVersion() { return tableVersion; } public int getPredicateHash() { return predicateHash; } public int getLogSegmentHash() { return logSegmentHash; } public long getPageSize() { return pageSize; } public Optional getLastReadLogFilePath() { return pageToken.map(PageToken::getLastReadLogFilePath); } public Optional getLastReturnedRowIndex() { if (!pageToken.isPresent()) return Optional.empty(); return Optional.of(pageToken.get().getLastReturnedRowIndex()); } public Optional getLastReadSidecarFileIdx() { if (!pageToken.isPresent()) return Optional.empty(); return pageToken.get().getLastReadSidecarFileIdx(); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/ProtocolMetadataLogReplay.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.replay; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.checksum.CRCInfo; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.lang.Lazy; import io.delta.kernel.internal.metrics.SnapshotMetrics; import io.delta.kernel.internal.snapshot.LogSegment; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import java.io.IOException; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Static utility class for loading Protocol and Metadata from Delta log files. */ public class ProtocolMetadataLogReplay { private static final Logger logger = LoggerFactory.getLogger(ProtocolMetadataLogReplay.class); /** Read schema when searching for the latest Protocol and Metadata. */ public static final StructType PROTOCOL_METADATA_READ_SCHEMA = new StructType().add("protocol", Protocol.FULL_SCHEMA).add("metaData", Metadata.FULL_SCHEMA); /** Result of loading Protocol and Metadata from a LogSegment. */ public static class Result { public final Protocol protocol; public final Metadata metadata; private final long numDeltaFilesRead; public Result(Protocol protocol, Metadata metadata, long numDeltaFilesRead) { this.protocol = protocol; this.metadata = metadata; this.numDeltaFilesRead = numDeltaFilesRead; } } /** * Loads the latest Protocol and Metadata from the log files in the given LogSegment. * *

Uses the provided lazy CRC loader to bound how many delta files it reads, and to ensure we * only read the CRC file if needed. * *

We read delta files in reverse order (newest first) searching for the latest Protocol and * Metadata. When we reach the version just before (greater than) the CRC file version (if * present), we lazily load the CRC file to fill in any missing Protocol or Metadata, avoiding * reading older delta files. * *

Also validates that the Kernel can read the table at the loaded protocol version. */ public static Result loadProtocolAndMetadata( Engine engine, Path dataPath, LogSegment logSegment, Lazy> lazyCrcInfo, SnapshotMetrics snapshotMetrics) { final Result result = snapshotMetrics.loadProtocolMetadataTotalDurationTimer.time( () -> loadProtocolAndMetadataInternal(engine, logSegment, lazyCrcInfo)); TableFeatures.validateKernelCanReadTheTable(result.protocol, dataPath.toString()); logger.info( "[{}] Took {}ms to load Protocol and Metadata at version {}, read {} log files", dataPath, snapshotMetrics.loadProtocolMetadataTotalDurationTimer.totalDurationMs(), logSegment.getVersion(), result.numDeltaFilesRead); return result; } private static Result loadProtocolAndMetadataInternal( Engine engine, LogSegment logSegment, Lazy> lazyCrcInfo) { final long snapshotVersion = logSegment.getVersion(); final Optional crcFileOpt = logSegment.getLastSeenChecksum(); final Optional crcVersionOpt = crcFileOpt.map(f -> FileNames.checksumVersion(f.getPath())); // If CRC is at this exact snapshot version, use it directly if (crcVersionOpt.isPresent() && crcVersionOpt.get() == snapshotVersion) { final Optional crcInfo = lazyCrcInfo.get(); if (crcInfo.isPresent()) { validateCrcInfoMatchesExpectedVersion(crcInfo.get(), crcVersionOpt.get()); final Protocol protocol = crcInfo.get().getProtocol(); final Metadata metadata = crcInfo.get().getMetadata(); return new Result(protocol, metadata, 0 /* logFilesRead */); } } // Otherwise, we need to read log files. The CRC (if present) might still be useful to avoid // reading older files. long numDeltaFilesRead = 0; Protocol protocol = null; Metadata metadata = null; try (CloseableIterator reverseIter = new ActionsIterator( engine, logSegment.allFilesWithCompactionsReversed(), PROTOCOL_METADATA_READ_SCHEMA, Optional.empty())) { while (reverseIter.hasNext()) { final ActionWrapper nextElem = reverseIter.next(); final long version = nextElem.getVersion(); numDeltaFilesRead++; // Load this lazily (as needed). We may be able to just use the CRC. ColumnarBatch columnarBatch = null; if (protocol == null) { columnarBatch = nextElem.getColumnarBatch(); assert (columnarBatch.getSchema().equals(PROTOCOL_METADATA_READ_SCHEMA)); final ColumnVector protocolVector = columnarBatch.getColumnVector(0); for (int i = 0; i < protocolVector.getSize(); i++) { if (!protocolVector.isNullAt(i)) { protocol = Protocol.fromColumnVector(protocolVector, i); if (metadata != null) { // Stop since we have found the latest Protocol and Metadata. return new Result(protocol, metadata, numDeltaFilesRead); } break; // We just found the protocol, exit this for-loop } } } if (metadata == null) { if (columnarBatch == null) { columnarBatch = nextElem.getColumnarBatch(); assert (columnarBatch.getSchema().equals(PROTOCOL_METADATA_READ_SCHEMA)); } final ColumnVector metadataVector = columnarBatch.getColumnVector(1); for (int i = 0; i < metadataVector.getSize(); i++) { if (!metadataVector.isNullAt(i)) { metadata = Metadata.fromColumnVector(metadataVector, i); if (protocol != null) { // Stop since we have found the latest Protocol and Metadata. return new Result(protocol, metadata, numDeltaFilesRead); } break; // We just found the metadata, exit this for-loop } } } // Since we haven't returned, then at least one of P or M is null. // Note: Suppose the CRC is at version N. We check the CRC eagerly at N + 1 so // that we don't read or open any files at version N. if (crcVersionOpt.isPresent() && version == crcVersionOpt.get() + 1) { final Optional crcInfo = lazyCrcInfo.get(); if (crcInfo.isPresent()) { validateCrcInfoMatchesExpectedVersion(crcInfo.get(), crcVersionOpt.get()); if (protocol == null) { protocol = crcInfo.get().getProtocol(); } if (metadata == null) { metadata = crcInfo.get().getMetadata(); } return new Result(protocol, metadata, numDeltaFilesRead); } } } } catch (IOException ex) { throw new RuntimeException("Could not close iterator", ex); } if (protocol == null) { throw new IllegalStateException( String.format("No protocol found at version %s", logSegment.getVersion())); } if (metadata == null) { throw new IllegalStateException( String.format("No metadata found at version %s", logSegment.getVersion())); } return new Result(protocol, metadata, numDeltaFilesRead); } private static void validateCrcInfoMatchesExpectedVersion(CRCInfo crcInfo, long expectedVersion) { if (crcInfo.getVersion() != expectedVersion) { throw new IllegalStateException( String.format( "Expected a CRC at version %d but got one at version %d", expectedVersion, crcInfo.getVersion())); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/rowtracking/MaterializedRowTrackingColumn.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.rowtracking; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; import io.delta.kernel.engine.ExpressionHandler; import io.delta.kernel.exceptions.InvalidTableException; import io.delta.kernel.expressions.*; import io.delta.kernel.internal.DeltaErrors; import io.delta.kernel.internal.InternalScanFileUtils; import io.delta.kernel.internal.TableConfig; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.util.ColumnMapping; import io.delta.kernel.internal.util.SchemaUtils; import io.delta.kernel.types.*; import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; /** A collection of helper methods for working with materialized row tracking columns. */ public final class MaterializedRowTrackingColumn { /** Static instance for the materialized row ID column. */ public static final MaterializedRowTrackingColumn MATERIALIZED_ROW_ID = new MaterializedRowTrackingColumn( TableConfig.MATERIALIZED_ROW_ID_COLUMN_NAME, "_row-id-col-"); /** Static instance for the materialized row commit version column. */ public static final MaterializedRowTrackingColumn MATERIALIZED_ROW_COMMIT_VERSION = new MaterializedRowTrackingColumn( TableConfig.MATERIALIZED_ROW_COMMIT_VERSION_COLUMN_NAME, "_row-commit-version-col-"); private final TableConfig tableConfig; private final String materializedColumnNamePrefix; /** Private constructor to enforce the use of static instances. */ private MaterializedRowTrackingColumn(TableConfig tableConfig, String prefix) { this.tableConfig = tableConfig; this.materializedColumnNamePrefix = prefix; } /** Returns the configuration property name associated with this materialized column. */ public String getMaterializedColumnNameProperty() { return tableConfig.getKey(); } /** Returns the prefix to use for generating the materialized column name. */ public String getMaterializedColumnNamePrefix() { return materializedColumnNamePrefix; } /** * Validates that the materialized column names for ROW_ID and ROW_COMMIT_VERSION do not conflict * with any existing logical or physical column names in the table's schema. * * @param metadata The current {@link Metadata} of the table. */ public static void throwIfColumnNamesConflictWithSchema(Metadata metadata) { StructType schema = metadata.getSchema(); Set logicalColNames = schema.fields().stream().map(StructField::getName).collect(Collectors.toSet()); Set physicalColNames = schema.fields().stream().map(ColumnMapping::getPhysicalName).collect(Collectors.toSet()); Stream.of(MATERIALIZED_ROW_ID, MATERIALIZED_ROW_COMMIT_VERSION) .map(column -> metadata.getConfiguration().get(column.getMaterializedColumnNameProperty())) .filter(Objects::nonNull) .forEach( columnName -> { if (logicalColNames.contains(columnName) || physicalColNames.contains(columnName)) { throw DeltaErrors.conflictWithReservedInternalColumnName(columnName); } }); } /** * Validates that materialized column names for ROW_ID and ROW_COMMIT_VERSION are not missing when * row tracking is enabled. This should be called for existing tables to ensure that row tracking * configs are present when they should be. * * @param metadata The current {@link Metadata} of the table. */ public static void validateRowTrackingConfigsNotMissing(Metadata metadata, String tablePath) { // No validation needed if row tracking is disabled if (!TableConfig.ROW_TRACKING_ENABLED.fromMetadata(metadata)) { return; } // Check that both materialized column name configs are present when row tracking is enabled Stream.of(MATERIALIZED_ROW_ID, MATERIALIZED_ROW_COMMIT_VERSION) .forEach( column -> { if (!metadata .getConfiguration() .containsKey(column.getMaterializedColumnNameProperty())) { throw new InvalidTableException( tablePath, String.format( "Row tracking is enabled but the materialized column name `%s` is missing.", column.getMaterializedColumnNameProperty())); } }); } /** * Assigns materialized column names for ROW_ID and ROW_COMMIT_VERSION if row tracking is enabled * and the column names have not been assigned yet. * * @param metadata The current Metadata of the table. * @return An Optional containing updated metadata if any assignments occurred; Optional.empty() * otherwise. */ public static Optional assignMaterializedColumnNamesIfNeeded(Metadata metadata) { // No assignment if row tracking is disabled if (!TableConfig.ROW_TRACKING_ENABLED.fromMetadata(metadata)) { return Optional.empty(); } Map configsToAdd = new HashMap<>(); Stream.of(MATERIALIZED_ROW_ID, MATERIALIZED_ROW_COMMIT_VERSION) .filter( column -> !metadata .getConfiguration() .containsKey(column.getMaterializedColumnNameProperty())) .forEach( column -> { configsToAdd.put( column.getMaterializedColumnNameProperty(), column.generateMaterializedColumnName()); }); return configsToAdd.isEmpty() ? Optional.empty() : Optional.of(metadata.withMergedConfiguration(configsToAdd)); } /** * Converts a logical row tracking field to its physical counterpart(s). * *

Since computing the row ID requires the row index, requesting a row tracking column can * require adding two columns to the physical schema. * *

Note that we must not mark the physical columns as metadata columns because as far as the * parquet reader is concerned, these columns are not metadata columns. * * @param logicalField The logical field to convert. * @param logicalSchema The logical schema containing the field. * @param metadata The current metadata of the table. * @return A list of {@link StructField}s representing the physical columns corresponding to the * logical field. */ public static List convertToPhysicalColumn( StructField logicalField, StructType logicalSchema, Metadata metadata) { if (!TableConfig.ROW_TRACKING_ENABLED.fromMetadata(metadata)) { throw DeltaErrors.missingRowTrackingColumnRequested(logicalField.getName()); } if (logicalField.getMetadataColumnSpec() == MetadataColumnSpec.ROW_ID) { List physicalFields = new ArrayList<>(2); physicalFields.add( new StructField( MATERIALIZED_ROW_ID.getPhysicalColumnName(metadata.getConfiguration()), LongType.LONG, true /* nullable */)); if (!logicalSchema.contains(MetadataColumnSpec.ROW_INDEX)) { physicalFields.add(SchemaUtils.asInternalColumn(StructField.DEFAULT_ROW_INDEX_COLUMN)); } return physicalFields; } else if (logicalField.getMetadataColumnSpec() == MetadataColumnSpec.ROW_COMMIT_VERSION) { return Collections.singletonList( new StructField( MATERIALIZED_ROW_COMMIT_VERSION.getPhysicalColumnName(metadata.getConfiguration()), LongType.LONG, true /* nullable */)); } throw new IllegalArgumentException( String.format( "Logical field `%s` is not a recognized materialized row tracking column.", logicalField.getName())); } /** * Computes row IDs and row commit versions based on their materialized values if present in the * data returned by the Parquet reader, using the base row ID and default row commit version from * the AddFile otherwise. * * @param dataBatch a batch of physical data read from the table. * @param scanFile the {@link Row} representing the scan file metadata. * @param logicalSchema the logical schema of the query. * @param configuration the configuration map containing table metadata. * @param engine the {@link Engine} to use for expression evaluation. * @return a new {@link ColumnarBatch} with logical row tracking columns */ public static ColumnarBatch transformPhysicalData( ColumnarBatch dataBatch, Row scanFile, StructType logicalSchema, Map configuration, Engine engine) { StructType physicalSchema = dataBatch.getSchema(); ExpressionHandler exprHandler = engine.getExpressionHandler(); // NOTE: We assume that each column is requested at most once in the read schema. This is // consistent with other parts of the codebase. String rowIdColumnName = MATERIALIZED_ROW_ID.getPhysicalColumnName(configuration); int rowIdOrdinal = physicalSchema.indexOf(rowIdColumnName); if (rowIdOrdinal != -1) { dataBatch = transformPhysicalRowId( dataBatch, scanFile, rowIdColumnName, exprHandler, logicalSchema.at(logicalSchema.indexOf(MetadataColumnSpec.ROW_ID)), physicalSchema, rowIdOrdinal); } String rowCommitVersionColumnName = MATERIALIZED_ROW_COMMIT_VERSION.getPhysicalColumnName(configuration); int commitVersionOrdinal = physicalSchema.indexOf(rowCommitVersionColumnName); if (commitVersionOrdinal != -1) { dataBatch = transformPhysicalCommitVersion( dataBatch, scanFile, rowCommitVersionColumnName, exprHandler, logicalSchema.at(logicalSchema.indexOf(MetadataColumnSpec.ROW_COMMIT_VERSION)), physicalSchema, commitVersionOrdinal); } return dataBatch; } /** Row IDs are computed as COALESCE(materializedRowId, baseRowId + rowIndex). */ private static ColumnarBatch transformPhysicalRowId( ColumnarBatch dataBatch, Row scanFile, String rowIdColumnName, ExpressionHandler exprHandler, StructField logicalRowIdColumn, StructType physicalSchema, int rowIdOrdinal) { long baseRowId = InternalScanFileUtils.getBaseRowId(scanFile) .orElseThrow( () -> DeltaErrors.rowTrackingMetadataMissingInFile( "baseRowId", InternalScanFileUtils.getFilePath(scanFile))); String rowIndexMetadataColName = dataBatch .getSchema() .at(dataBatch.getSchema().indexOf(MetadataColumnSpec.ROW_INDEX)) .getName(); Expression rowIdExpr = new ScalarExpression( "COALESCE", Arrays.asList( new Column(rowIdColumnName), new ScalarExpression( "ADD", Arrays.asList( new Column(rowIndexMetadataColName), Literal.ofLong(baseRowId))))); ColumnVector rowIdVector = exprHandler.getEvaluator(physicalSchema, rowIdExpr, LongType.LONG).eval(dataBatch); // Remove the materialized row ID column and replace it with the coalesced vector dataBatch = dataBatch .withDeletedColumnAt(rowIdOrdinal) .withNewColumn(rowIdOrdinal, logicalRowIdColumn, rowIdVector); return dataBatch; } /** * Row commit versions are computed as COALESCE(materializedRowCommitVersion, * defaultRowCommitVersion). */ private static ColumnarBatch transformPhysicalCommitVersion( ColumnarBatch dataBatch, Row scanFile, String commitVersionColumnName, ExpressionHandler exprHandler, StructField logicalCommitVersionColumn, StructType physicalSchema, int commitVersionOrdinal) { long defaultRowCommitVersion = InternalScanFileUtils.getDefaultRowCommitVersion(scanFile) .orElseThrow( () -> DeltaErrors.rowTrackingMetadataMissingInFile( "defaultRowCommitVersion", InternalScanFileUtils.getFilePath(scanFile))); Expression commitVersionExpr = new ScalarExpression( "COALESCE", Arrays.asList( new Column(commitVersionColumnName), Literal.ofLong(defaultRowCommitVersion))); ColumnVector commitVersionVector = exprHandler.getEvaluator(physicalSchema, commitVersionExpr, LongType.LONG).eval(dataBatch); // Remove the materialized row commit version column and replace it with the coalesced vector dataBatch = dataBatch .withDeletedColumnAt(commitVersionOrdinal) .withNewColumn(commitVersionOrdinal, logicalCommitVersionColumn, commitVersionVector); return dataBatch; } /** * Gets the physical column name from the table configuration. * * @param configuration the table configuration map * @return the physical column name * @throws IllegalArgumentException if the materialized column name is missing from the * configuration */ public String getPhysicalColumnName(Map configuration) { return Optional.ofNullable(configuration.get(getMaterializedColumnNameProperty())) .orElseThrow( () -> new IllegalArgumentException( String.format( "Materialized column name `%s` is missing in the metadata config: %s", getMaterializedColumnNameProperty(), configuration))); } /** Generates a random name by concatenating the prefix with a random UUID. */ private String generateMaterializedColumnName() { return materializedColumnNamePrefix + UUID.randomUUID().toString(); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/rowtracking/RowTracking.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.rowtracking; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.Row; import io.delta.kernel.internal.DeltaErrors; import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.TableConfig; import io.delta.kernel.internal.actions.*; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.types.MetadataColumnSpec; import io.delta.kernel.types.StructField; import io.delta.kernel.utils.CloseableIterable; import io.delta.kernel.utils.CloseableIterator; import java.io.IOException; import java.util.List; import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; /** A collection of helper methods for working with row tracking. */ public class RowTracking { private RowTracking() { // Empty constructor to prevent instantiation of this class } /** * Check if row tracking is enabled for reading. * * @param protocol the protocol to check * @param metadata the metadata to check * @return true if row tracking is enabled * @throws IllegalStateException if row tracking is enabled in metadata but not supported by * protocol */ public static boolean isEnabled(Protocol protocol, Metadata metadata) { boolean isEnabled = TableConfig.ROW_TRACKING_ENABLED.fromMetadata(metadata); if (isEnabled && !TableFeatures.isRowTrackingSupported(protocol)) { throw new IllegalStateException( "Table property 'delta.enableRowTracking' is set on the table but this table version " + "doesn't support table feature 'delta.feature.rowTracking'."); } return isEnabled; } /** * Checks if the provided field is a row tracking column, i.e., either the row ID or the row * commit version column. * * @param field the field to check * @return true if the field is a row tracking column, false otherwise */ public static boolean isRowTrackingColumn(StructField field) { return field.isMetadataColumn() && (field.getMetadataColumnSpec() == MetadataColumnSpec.ROW_ID || field.getMetadataColumnSpec() == MetadataColumnSpec.ROW_COMMIT_VERSION); } /** * Assigns or reassigns baseRowIds and defaultRowCommitVersions to {@link AddFile} actions in the * provided {@code dataActions}. This method should be invoked only when the 'rowTracking' feature * is supported and is used in two scenarios: * *

    *
  1. Initial Assignment: Assigns row tracking fields to AddFile actions during commit * preparation before they are committed. *
  2. Conflict Resolution: Updates row tracking fields when a transaction conflict occurs. * Since the losing transaction gets a new commit version and winning transactions may have * increased the row ID high watermark, this method reassigns the fields for the losing * transaction using the latest state from winning transactions before retrying the commit. *
* * @param txnReadSnapshotOpt the snapshot of the table that this transaction is reading from * @param txnProtocol the (updated, if any) protocol that will result from this txn * @param winningTxnRowIdHighWatermark the latest row ID high watermark from the winning * transactions. Should be empty for initial assignment and present for conflict resolution. * @param prevCommitVersion the commit version used by this transaction in the previous commit * attempt. Should be empty for initial assignment and present for conflict resolution. * @param currCommitVersion the transaction's (latest) commit version * @param txnDataActions a {@link CloseableIterable} of data actions this txn is trying to commit * @return a {@link CloseableIterable} of data actions with baseRowIds and * defaultRowCommitVersions assigned or reassigned */ public static CloseableIterable assignBaseRowIdAndDefaultRowCommitVersion( Optional txnReadSnapshotOpt, Protocol txnProtocol, Optional winningTxnRowIdHighWatermark, Optional prevCommitVersion, long currCommitVersion, CloseableIterable txnDataActions) { checkArgument( TableFeatures.isRowTrackingSupported(txnProtocol), "Base row ID and default row commit version are assigned " + "only when feature 'rowTracking' is supported."); return new CloseableIterable() { @Override public void close() throws IOException { txnDataActions.close(); } @Override public CloseableIterator iterator() { // The row ID high watermark from the snapshot of the table that this transaction is reading // at. Any baseRowIds higher than this watermark are assigned by this transaction. final long prevRowIdHighWatermark = readRowIdHighWaterMark(txnReadSnapshotOpt); // Used to track the current high watermark as we iterate through the data actions and // assign baseRowIds. Use an AtomicLong to allow for updating in the lambda. final AtomicLong currRowIdHighWatermark = new AtomicLong(winningTxnRowIdHighWatermark.orElse(prevRowIdHighWatermark)); // The row ID high watermark must increase monotonically, so the winning transaction's high // watermark must be greater than or equal to the high watermark from the current // transaction's read snapshot. checkArgument( currRowIdHighWatermark.get() >= prevRowIdHighWatermark, "The current row ID high watermark must be greater than or equal to " + "the high watermark from the transaction's read snapshot"); return txnDataActions .iterator() .map( row -> { // Non-AddFile actions are returned unchanged if (row.isNullAt(SingleAction.ADD_FILE_ORDINAL)) { return row; } AddFile addFile = new AddFile(row.getStruct(SingleAction.ADD_FILE_ORDINAL)); // Assign a baseRowId if not present, or update it if previously assigned // by this transaction if (!addFile.getBaseRowId().isPresent() || addFile.getBaseRowId().get() > prevRowIdHighWatermark) { addFile = addFile.withNewBaseRowId(currRowIdHighWatermark.get() + 1L); currRowIdHighWatermark.addAndGet(getNumRecordsOrThrow(addFile)); } // Assign a defaultRowCommitVersion if not present, or update it if previously // assigned by this transaction if (!addFile.getDefaultRowCommitVersion().isPresent() || addFile.getDefaultRowCommitVersion().get() == prevCommitVersion.orElse(-1L)) { addFile = addFile.withNewDefaultRowCommitVersion(currCommitVersion); } return SingleAction.createAddFileSingleAction(addFile.toRow()); }); } }; } /** * Inserts or updates the {@link DomainMetadata} action reflecting the new row ID high watermark * when this transaction adds rows and pushed it higher. * *

This method should only be called when the 'rowTracking' feature is supported. Similar to * {@link #assignBaseRowIdAndDefaultRowCommitVersion}, it should be called during the initial row * ID assignment or conflict resolution to reflect the change to the row ID high watermark. * * @param txnReadSnapshotOpt the snapshot of the table that this transaction is reading at * @param txnProtocol the (updated, if any) protocol that will result from this txn * @param winningTxnRowIdHighWatermark the latest row ID high watermark from the winning * transaction. Should be empty for initial assignment and present for conflict resolution. * @param txnDataActions a {@link CloseableIterable} of data actions this txn is trying to commit * @param txnDomainMetadatas a list of domain metadata actions this txn is trying to commit * @param providedRowIdHighWatermark Optional row ID high watermark explicitly provided by the * transaction builder. * @return Updated list of domain metadata actions for commit */ public static List updateRowIdHighWatermarkIfNeeded( Optional txnReadSnapshotOpt, Protocol txnProtocol, Optional winningTxnRowIdHighWatermark, CloseableIterable txnDataActions, List txnDomainMetadatas, Optional providedRowIdHighWatermark) { checkArgument( TableFeatures.isRowTrackingSupported(txnProtocol), "Row ID high watermark is updated only when feature 'rowTracking' is supported."); checkArgument( !(providedRowIdHighWatermark.isPresent() && winningTxnRowIdHighWatermark.isPresent()), "Conflict resolution is not allowed when an explicit row tracking high " + "watermark is provided. Please recommit."); // Filter out existing row tracking domainMetadata action, if any List nonRowTrackingDomainMetadatas = txnDomainMetadatas.stream() .filter(dm -> !dm.getDomain().equals(RowTrackingMetadataDomain.DOMAIN_NAME)) .collect(Collectors.toList()); // The row ID high watermark from the snapshot of the table that this transaction is reading at. // Any baseRowIds higher than this watermark are assigned by this transaction. final long prevRowIdHighWatermark = readRowIdHighWaterMark(txnReadSnapshotOpt); // Tracks the new row ID high watermark as we iterate through data actions and counting new rows // added in this transaction. final AtomicLong currCalculatedRowIdHighWatermark = new AtomicLong(winningTxnRowIdHighWatermark.orElse(prevRowIdHighWatermark)); // The row ID high watermark must increase monotonically, so the winning transaction's high // watermark (if present) must be greater than or equal to the high watermark from the // current transaction's read snapshot. checkArgument( currCalculatedRowIdHighWatermark.get() >= prevRowIdHighWatermark, "The current row ID high watermark must be greater than or equal to " + "the high watermark from the transaction's read snapshot"); txnDataActions.forEach( row -> { if (!row.isNullAt(SingleAction.ADD_FILE_ORDINAL)) { AddFile addFile = new AddFile(row.getStruct(SingleAction.ADD_FILE_ORDINAL)); if (!addFile.getBaseRowId().isPresent() || addFile.getBaseRowId().get() > prevRowIdHighWatermark) { currCalculatedRowIdHighWatermark.addAndGet(getNumRecordsOrThrow(addFile)); } } }); // If the txn builder has explicitly provided a row ID high watermark, we should use that value // instead of the one calculated from the current row ID high watermark and uncommitted data // actions. Validate that the provided value is greater than or equal to the calculated // watermark to // ensure consistency. providedRowIdHighWatermark.ifPresent( providedHighWatermark -> checkArgument( providedHighWatermark >= currCalculatedRowIdHighWatermark.get(), String.format( "The provided row ID high watermark (%d) must be greater than or equal to " + "the calculated row ID high watermark (%d) based on the transaction's " + "data actions.", providedHighWatermark, currCalculatedRowIdHighWatermark.get()))); final long finalRowIdHighWatermark = providedRowIdHighWatermark.orElse(currCalculatedRowIdHighWatermark.get()); if (finalRowIdHighWatermark != prevRowIdHighWatermark) { nonRowTrackingDomainMetadatas.add( new RowTrackingMetadataDomain(finalRowIdHighWatermark).toDomainMetadata()); } return nonRowTrackingDomainMetadatas; } /** * Throws an exception if row tracking enablement is toggled between the old and the new metadata. */ public static void throwIfRowTrackingToggled(Metadata oldMetadata, Metadata newMetadata) { boolean oldRowTrackingEnabledValue = TableConfig.ROW_TRACKING_ENABLED.fromMetadata(oldMetadata); boolean newRowTrackingEnabledValue = TableConfig.ROW_TRACKING_ENABLED.fromMetadata(newMetadata); if (oldRowTrackingEnabledValue != newRowTrackingEnabledValue) { throw DeltaErrors.cannotToggleRowTrackingOnExistingTable(); } } /** * Reads the current row ID high watermark from the snapshot, or returns a default value if * missing. */ private static long readRowIdHighWaterMark(Optional snapshotOpt) { return snapshotOpt .flatMap(RowTrackingMetadataDomain::fromSnapshot) .map(RowTrackingMetadataDomain::getRowIdHighWaterMark) .orElse(RowTrackingMetadataDomain.MISSING_ROW_ID_HIGH_WATERMARK); } /** * Get the number of records from the AddFile's statistics. It errors out if statistics are * missing. */ private static long getNumRecordsOrThrow(AddFile addFile) { return addFile.getNumRecords().orElseThrow(DeltaErrors::missingNumRecordsStatsForRowTracking); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/rowtracking/RowTrackingMetadataDomain.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.rowtracking; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.metadatadomain.JsonMetadataDomain; import java.util.Optional; /** Represents the metadata domain for row tracking. */ public final class RowTrackingMetadataDomain extends JsonMetadataDomain { public static final String DOMAIN_NAME = "delta.rowTracking"; public static final long MISSING_ROW_ID_HIGH_WATERMARK = -1L; /** * Creates an instance of {@link RowTrackingMetadataDomain} from a JSON configuration string. * * @param json the JSON configuration string * @return an instance of {@link RowTrackingMetadataDomain} */ public static RowTrackingMetadataDomain fromJsonConfiguration(String json) { return JsonMetadataDomain.fromJsonConfiguration(json, RowTrackingMetadataDomain.class); } /** * Creates an instance of {@link RowTrackingMetadataDomain} from a {@link SnapshotImpl}. * * @param snapshot the snapshot instance * @return an {@link Optional} containing the {@link RowTrackingMetadataDomain} if present */ public static Optional fromSnapshot(SnapshotImpl snapshot) { return JsonMetadataDomain.fromSnapshot(snapshot, RowTrackingMetadataDomain.class, DOMAIN_NAME); } /** The highest assigned fresh row id for the table */ private final long rowIdHighWaterMark; /** * Constructs a RowTrackingMetadataDomain with the specified row ID high water mark. * * @param rowIdHighWaterMark the row ID high water mark */ @JsonCreator public RowTrackingMetadataDomain(@JsonProperty("rowIdHighWaterMark") long rowIdHighWaterMark) { this.rowIdHighWaterMark = rowIdHighWaterMark; } @Override public String getDomainName() { return DOMAIN_NAME; } public long getRowIdHighWaterMark() { return rowIdHighWaterMark; } @Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null || getClass() != obj.getClass()) return false; RowTrackingMetadataDomain that = (RowTrackingMetadataDomain) obj; return rowIdHighWaterMark == that.rowIdHighWaterMark; } @Override public int hashCode() { return java.util.Objects.hash(DOMAIN_NAME, rowIdHighWaterMark); } @Override public String toString() { return "RowTrackingMetadataDomain{" + "rowIdHighWaterMark=" + rowIdHighWaterMark + '}'; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingPredicate.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.skipping; import io.delta.kernel.expressions.Column; import io.delta.kernel.expressions.Expression; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.types.CollationIdentifier; import java.util.*; /** A {@link Predicate} with a set of columns referenced by the expression. */ public class DataSkippingPredicate extends Predicate { /** Set of {@link Column}s referenced by the predicate or any of its child expressions */ private final Set referencedCols; /** * @param name the predicate name * @param children list of expressions that are input to this predicate. * @param referencedCols set of columns referenced by this predicate or any of its child * expressions */ DataSkippingPredicate(String name, List children, Set referencedCols) { super(name, children); this.referencedCols = Collections.unmodifiableSet(referencedCols); } /** * @param name the predicate name * @param children list of expressions that are input to this predicate. * @param collationIdentifier collation identifier used for this predicate * @param referencedCols set of columns referenced by this predicate or any of its child * expressions */ DataSkippingPredicate( String name, List children, CollationIdentifier collationIdentifier, Set referencedCols) { super(name, children, collationIdentifier); this.referencedCols = Collections.unmodifiableSet(referencedCols); } /** * Constructor for a binary {@link DataSkippingPredicate} where both children are instances of * {@link DataSkippingPredicate}. * * @param name the predicate name * @param left left input to this predicate * @param right right input to this predicate */ DataSkippingPredicate(String name, DataSkippingPredicate left, DataSkippingPredicate right) { super(name, Arrays.asList(left, right)); this.referencedCols = immutableUnion(left.referencedCols, right.referencedCols); } /** @return set of columns referenced by this predicate or any of its child expressions */ public Set getReferencedCols() { return referencedCols; } /** * @return set of collation identifiers referenced by this predicate or any of its child * expressions */ public Set getReferencedCollations() { Set referencedCollations = new HashSet<>(); if (this.getCollationIdentifier().isPresent()) { referencedCollations.add(this.getCollationIdentifier().get()); } for (Expression child : children) { if (child instanceof DataSkippingPredicate) { referencedCollations.addAll(((DataSkippingPredicate) child).getReferencedCollations()); } else if (child instanceof Predicate) { throw new IllegalStateException( String.format( "Expected child Predicate of DataSkippingPredicate to be an instance of" + " DataSkippingPredicate but found: %s", child, this)); } } return Collections.unmodifiableSet(referencedCollations); } /** @return an unmodifiable set containing all elements from both sets. */ private Set immutableUnion(Set set1, Set set2) { return Collections.unmodifiableSet( new HashSet() { { addAll(set1); addAll(set2); } }); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingUtils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.skipping; import static io.delta.kernel.internal.DeltaErrors.wrapEngineException; import static io.delta.kernel.internal.InternalScanFileUtils.ADD_FILE_ORDINAL; import static io.delta.kernel.internal.InternalScanFileUtils.ADD_FILE_STATS_ORDINAL; import static io.delta.kernel.internal.util.ExpressionUtils.*; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.engine.Engine; import io.delta.kernel.expressions.*; import io.delta.kernel.internal.util.Tuple2; import io.delta.kernel.types.CollationIdentifier; import io.delta.kernel.types.StructField; import io.delta.kernel.types.StructType; import java.util.*; import java.util.function.BiFunction; public class DataSkippingUtils { /** * Given a {@code FilteredColumnarBatch} of scan files and the statistics schema to parse, return * the parsed JSON stats from the scan files. */ public static ColumnarBatch parseJsonStats( Engine engine, FilteredColumnarBatch scanFileBatch, StructType statsSchema) { ColumnVector statsVector = scanFileBatch.getData().getColumnVector(ADD_FILE_ORDINAL).getChild(ADD_FILE_STATS_ORDINAL); return wrapEngineException( () -> engine .getJsonHandler() .parseJson(statsVector, statsSchema, scanFileBatch.getSelectionVector()), "Parsing the JSON statistics with statsSchema=%s", statsSchema); } /** * Prunes the given schema to only include the referenced leaf columns. If a leaf column is a * nested column it must be referenced using the full column path, e.g. "C_0.C_1.C_leaf" * * @param schema the schema to prune * @param referencedLeafCols set of leaf columns in {@code schema} */ public static StructType pruneStatsSchema(StructType schema, Set referencedLeafCols) { return pruneSchema(referencedLeafCols, schema, new String[0]); } /** * Constructs a data skipping filter to prune files using column statistics given a query data * filter if possible. The returned filter will evaluate to FALSE for any files that can be safely * skipped. If the filter evaluates to NULL or TRUE, the file should not be skipped. * * @param dataFilters query filters on the data columns * @param dataSchema the data schema of the table * @return data skipping filter to prune files if it exists as a {@link DataSkippingPredicate} */ public static Optional constructDataSkippingFilter( Predicate dataFilters, StructType dataSchema) { StatsSchemaHelper schemaHelper = new StatsSchemaHelper(dataSchema); return constructDataSkippingFilter(dataFilters, schemaHelper); } ////////////////////////////////////////////////////////////////////////////////// // Helper functions ////////////////////////////////////////////////////////////////////////////////// /** * Returns a file skipping predicate expression, derived from the user query, which uses column * statistics to prune away files that provably contain no rows the query cares about. * *

Specifically, the filter extraction code must obey the following rules: * *

1. Given a query predicate `e`, `constructDataSkippingFilter(e)` must return TRUE for a file * unless we can prove `e` will not return TRUE for any row the file might contain. For example, * given `a = 3` and min/max stat values [0, 100], this skipping predicate is safe: * *

AND(minValues.a <= 3, maxValues.a >= 3) * *

Because that condition must be true for any file that might possibly contain `a = 3`; the * skipping predicate could return FALSE only if the max is too low, or the min too high; it could * return NULL only if a is NULL in every row of the file. In both latter cases, it is safe to * skip the file because `a = 3` can never evaluate to TRUE. * *

2. It is unsafe to apply skipping to operators that can evaluate to NULL or produce an error * for non-NULL inputs. For example, consider this query predicate involving integer addition: * *

a + 1 = 3 * *

It might be tempting to apply the standard equality skipping predicate: * *

AND(minValues.a + 1 <= 3, 3 <= maxValues.a + 1) * *

However, the skipping predicate would be unsound, because the addition operator could * trigger integer overflow (e.g. minValues.a = 0 and maxValues.a = INT_MAX), even though the file * could very well contain rows satisfying a + 1 = 3. * *

3. Predicates involving NOT are ineligible for skipping, because * `Not(constructDataSkippingFilter(e))` is seldom equivalent to `constructDataSkippingFilter * (Not(e))`. For example, consider the query predicate: * *

NOT(a = 1) * *

A simple inversion of the data skipping predicate would be: * *

NOT(AND(minValues.a <= 1, maxValues.a >= 1)) ==> OR(NOT(minValues.a <= 1), NOT(maxValues.a * >= 1)) ==> OR(minValues.a > 1, maxValues.a < 1) * *

By contrast, if we first combine the NOT with = to obtain * *

a != 1 * *

We get a different skipping predicate: * *

NOT(AND(minValues.a = 1, maxValues.a = 1)) ==> OR(NOT(minValues.a = 1), NOT(maxValues.a = * 1)) ==> OR(minValues.a != 1, maxValues.a != 1) * *

A truth table confirms that the first (naively inverted) skipping predicate is incorrect: * *

minValues.a | maxValues.a | | OR(minValues.a > 1, maxValues.a < 1) | | | OR(minValues.a != * 1, maxValues.a != 1) 0 0 T T 0 1 F T !! first predicate wrongly skipped a = 0 1 1 F F * *

Fortunately, we may be able to eliminate NOT from some (branches of some) predicates: * *

a. It is safe to push the NOT into the children of AND and OR using de Morgan's Law, e.g. * *

NOT(AND(a, b)) ==> OR(NOT(a), NOT(B)). * *

b. It is safe to fold NOT into other operators, when a negated form of the operator exists: * *

NOT(NOT(x)) ==> x NOT(a == b) ==> a != b NOT(a > b) ==> a <= b * *

NOTE: The skipping predicate must handle the case where min and max stats for a column are * both NULL -- which indicates that all values in the file are NULL. Fortunately, most of the * operators we support data skipping for are NULL intolerant, and thus trivially satisfy this * requirement because they never return TRUE for NULL inputs. The only NULL tolerant operator we * support -- IS [NOT] NULL -- is specifically NULL aware. The predicate evaluates to NULL if any * required statistics are missing. */ private static Optional constructDataSkippingFilter( Predicate dataFilters, StatsSchemaHelper schemaHelper) { switch (dataFilters.getName().toUpperCase(Locale.ROOT)) { // Push skipping predicate generation through the AND: // // constructDataSkippingFilter(AND(a, b)) // ==> AND(constructDataSkippingFilter(a), constructDataSkippingFilter(b)) // // To see why this transformation is safe, consider that // `constructDataSkippingFilter(a)` must evaluate to TRUE *UNLESS* we can prove that // `a` would not evaluate to TRUE for any // row the file might contain. Thus, if the rewritten form of the skipping predicate // does not evaluate to TRUE, at least one of the skipping predicates must not have // evaluated to TRUE, which in turn means we were able to prove that `a` and/or `b` // will not evaluate to TRUE for any row of the file. If that is the case, then // `AND(a, b)` also cannot evaluate to TRUE for any row of the file, which proves we // have a valid data skipping predicate. // // NOTE: AND is special -- we can safely skip the file if one leg does not evaluate to // TRUE, even if we cannot construct a skipping filter for the other leg. case "AND": Optional e1AndFilter = constructDataSkippingFilter(asPredicate(getLeft(dataFilters)), schemaHelper); Optional e2AndFilter = constructDataSkippingFilter(asPredicate(getRight(dataFilters)), schemaHelper); if (e1AndFilter.isPresent() && e2AndFilter.isPresent()) { return Optional.of( new DataSkippingPredicate("AND", e1AndFilter.get(), e2AndFilter.get())); } else if (e1AndFilter.isPresent()) { return e1AndFilter; } else { return e2AndFilter; // possibly none } // Push skipping predicate generation through OR (similar to AND case). // // constructDataFilters(OR(a, b)) // ==> OR(constructDataFilters(a), constructDataFilters(b)) // // Similar to AND case, if the rewritten predicate does not evaluate to TRUE, then it // means that neither `constructDataFilters(a)` nor `constructDataFilters(b)` evaluated // to TRUE, which in turn means that neither `a` nor `b` could evaluate to TRUE for any // row the file might contain, which proves we have a valid data skipping predicate. // // Unlike AND, a single leg of an OR expression provides no filtering power -- we can // only reject a file if both legs evaluate to false. case "OR": Optional e1OrFilter = constructDataSkippingFilter(asPredicate(getLeft(dataFilters)), schemaHelper); Optional e2OrFilter = constructDataSkippingFilter(asPredicate(getRight(dataFilters)), schemaHelper); if (e1OrFilter.isPresent() && e2OrFilter.isPresent()) { return Optional.of(new DataSkippingPredicate("OR", e1OrFilter.get(), e2OrFilter.get())); } else { return Optional.empty(); } // Match any file whose null count is less than the row count. case "IS_NOT_NULL": Expression child = getUnaryChild(dataFilters); if (child instanceof Column) { Column childColumn = (Column) child; if (schemaHelper.isSkippingEligibleNullCountColumn((Column) child)) { Column nullCountCol = schemaHelper.getNullCountColumn(childColumn); Column numRecordsCol = schemaHelper.getNumRecordsColumn(); return Optional.of( new DataSkippingPredicate( "<", Arrays.asList(nullCountCol, numRecordsCol), new HashSet() { { add(nullCountCol); add(numRecordsCol); } })); } } break; // Match any file whose null count is larger than zero. // Note DVs might result in a redundant read of a file. // However, they cannot lead to a correctness issue. case "IS_NULL": Expression unaryChild = getUnaryChild(dataFilters); if (unaryChild instanceof Column) { Column childColumn = (Column) unaryChild; if (schemaHelper.isSkippingEligibleNullCountColumn((Column) unaryChild)) { Column nullCountCol = schemaHelper.getNullCountColumn(childColumn); Literal zero = Literal.ofLong(0); return Optional.of( new DataSkippingPredicate( ">", Arrays.asList(nullCountCol, zero), Collections.singleton(nullCountCol))); } } break; case "=": case "<": case "<=": case ">": case ">=": case "IS NOT DISTINCT FROM": Expression left = getLeft(dataFilters); Expression right = getRight(dataFilters); Optional collationIdentifier = dataFilters.getCollationIdentifier(); if (collationIdentifier .filter(ci -> !ci.isSparkUTF8BinaryCollation() && ci.getVersion().isEmpty()) .isPresent()) { // Each collated statistics is stored with a specific version, so collation must specify a // version to be used for data skipping. return Optional.empty(); } if (left instanceof Column && right instanceof Literal) { Column leftCol = (Column) left; Literal rightLit = (Literal) right; if (schemaHelper.isSkippingEligibleMinMaxColumn(leftCol) && schemaHelper.isSkippingEligibleLiteral(rightLit)) { return constructComparatorDataSkippingFilters( dataFilters.getName(), leftCol, rightLit, collationIdentifier, schemaHelper); } } else if (right instanceof Column && left instanceof Literal) { return constructDataSkippingFilter(reverseComparatorFilter(dataFilters), schemaHelper); } break; case "NOT": return constructNotDataSkippingFilters( asPredicate(getUnaryChild(dataFilters)), schemaHelper); // TODO more expressions } return Optional.empty(); } /** Construct the skipping predicate for a given comparator */ private static Optional constructComparatorDataSkippingFilters( String comparator, Column leftCol, Literal rightLit, Optional collationIdentifier, StatsSchemaHelper schemaHelper) { switch (comparator.toUpperCase(Locale.ROOT)) { // Match any file whose min/max range contains the requested point. case "=": // For example a = 1 --> minValue.a <= 1 AND maxValue.a >= 1 return Optional.of( new DataSkippingPredicate( "AND", constructBinaryDataSkippingPredicate( "<=", schemaHelper.getMinColumn(leftCol, collationIdentifier), rightLit, collationIdentifier), constructBinaryDataSkippingPredicate( ">=", schemaHelper.getMaxColumn(leftCol, collationIdentifier), rightLit, collationIdentifier))); // Match any file whose min is less than the requested upper bound. case "<": return Optional.of( constructBinaryDataSkippingPredicate( "<", schemaHelper.getMinColumn(leftCol, collationIdentifier), rightLit, collationIdentifier)); // Match any file whose min is less than or equal to the requested upper bound case "<=": return Optional.of( constructBinaryDataSkippingPredicate( "<=", schemaHelper.getMinColumn(leftCol, collationIdentifier), rightLit, collationIdentifier)); // Match any file whose max is larger than the requested lower bound. case ">": return Optional.of( constructBinaryDataSkippingPredicate( ">", schemaHelper.getMaxColumn(leftCol, collationIdentifier), rightLit, collationIdentifier)); // Match any file whose max is larger than or equal to the requested lower bound. case ">=": return Optional.of( constructBinaryDataSkippingPredicate( ">=", schemaHelper.getMaxColumn(leftCol, collationIdentifier), rightLit, collationIdentifier)); case "IS NOT DISTINCT FROM": return constructDataSkippingFilter( rewriteEqualNullSafe(leftCol, rightLit, collationIdentifier), schemaHelper); default: throw new IllegalArgumentException( String.format("Unsupported comparator expression %s", comparator)); } } /** * Constructs a {@link DataSkippingPredicate} for a binary predicate expression with a left * column, an optional column adjustment expression and a right expression of type {@link * Literal}. */ private static DataSkippingPredicate constructBinaryDataSkippingPredicate( String exprName, Tuple2> colExpr, Literal lit, Optional collationIdentifier) { Column column = colExpr._1; Expression adjColExpr = colExpr._2.isPresent() ? colExpr._2.get() : column; if (collationIdentifier.isPresent()) { return new DataSkippingPredicate( exprName, Arrays.asList(adjColExpr, lit), collationIdentifier.get(), Collections.singleton(column)); } else { return new DataSkippingPredicate( exprName, Arrays.asList(adjColExpr, lit), Collections.singleton(column)); } } private static final Map REVERSE_COMPARATORS = new HashMap() { { put("=", "="); put("<", ">"); put("<=", ">="); put(">", "<"); put(">=", "<="); put("IS NOT DISTINCT FROM", "IS NOT DISTINCT FROM"); } }; private static Predicate reverseComparatorFilter(Predicate predicate) { return createPredicate( REVERSE_COMPARATORS.get(predicate.getName().toUpperCase(Locale.ROOT)), getRight(predicate), getLeft(predicate), predicate.getCollationIdentifier()); } /** Construct the skipping predicate for a NOT expression child if possible */ private static Optional constructNotDataSkippingFilters( Predicate childPredicate, StatsSchemaHelper schemaHelper) { Optional collationIdentifier = childPredicate.getCollationIdentifier(); switch (childPredicate.getName().toUpperCase(Locale.ROOT)) { // Use deMorgan's law to push the NOT past the AND. This is safe even with SQL // tri-valued logic (see below), and is desirable because we cannot generally push // predicate filters // through NOT, but we *CAN* push predicate filters through AND and OR: // // constructDataFilters(NOT(AND(a, b))) // ==> constructDataFilters(OR(NOT(a), NOT(b))) // ==> OR(constructDataFilters(NOT(a)), constructDataFilters(NOT(b))) // // Assuming we can push the resulting NOT operations all the way down to some leaf // operation it can fold into, the rewrite allows us to create a data skipping filter // from the expression. // // a b AND(a, b) // | | | NOT(AND(a, b)) // | | | | OR(NOT(a), NOT(b)) // T T T F F // T F F T T // T N N N N // F F F T T // F N F T T // N N N N N case "AND": return constructDataSkippingFilter( new Or( new Predicate("NOT", asPredicate(getLeft(childPredicate))), new Predicate("NOT", asPredicate(getRight(childPredicate)))), schemaHelper); // Similar to AND, we can (and want to) push the NOT past the OR using deMorgan's law. case "OR": return constructDataSkippingFilter( new And( new Predicate("NOT", asPredicate(getLeft(childPredicate))), new Predicate("NOT", asPredicate(getRight(childPredicate)))), schemaHelper); case "IS_NOT_NULL": return constructDataSkippingFilter( new Predicate("IS_NULL", getUnaryChild(childPredicate)), schemaHelper); case "IS_NULL": return constructDataSkippingFilter( new Predicate("IS_NOT_NULL", getUnaryChild(childPredicate)), schemaHelper); case "=": return constructDataSkippingFiltersForNotEqual( childPredicate, schemaHelper, (leftColumn, rightLiteral) -> { // Match any file whose min/max range contains anything other than the // rejected point. // For example a != 1 --> minValue.a < 1 OR maxValue.a > 1 return Optional.of( new DataSkippingPredicate( "OR", constructBinaryDataSkippingPredicate( "<", schemaHelper.getMinColumn(leftColumn, collationIdentifier), rightLiteral, collationIdentifier), constructBinaryDataSkippingPredicate( ">", schemaHelper.getMaxColumn(leftColumn, collationIdentifier), rightLiteral, collationIdentifier))); }); case "<": return constructDataSkippingFilter( createPredicate(">=", childPredicate.getChildren(), collationIdentifier), schemaHelper); case "<=": return constructDataSkippingFilter( createPredicate(">", childPredicate.getChildren(), collationIdentifier), schemaHelper); case ">": return constructDataSkippingFilter( createPredicate("<=", childPredicate.getChildren(), collationIdentifier), schemaHelper); case ">=": return constructDataSkippingFilter( createPredicate("<", childPredicate.getChildren(), collationIdentifier), schemaHelper); case "IS NOT DISTINCT FROM": return constructDataSkippingFiltersForNotEqual( childPredicate, schemaHelper, (leftColumn, rightLiteral) -> constructDataSkippingFilter( new Predicate( "NOT", rewriteEqualNullSafe(leftColumn, rightLiteral, collationIdentifier)), schemaHelper)); case "NOT": // Remove redundant pairs of NOT return constructDataSkippingFilter( asPredicate(getUnaryChild(childPredicate)), schemaHelper); } return Optional.empty(); } /** * Prunes the given schema to include only the referenced leaf columns to keep. These leaf columns * (possible nested) are relative to the root schema, not to the current level of recursion. * {@code leafColumnsToKeep} is unchanged at any level of recursion. * *

For example consider the following schema: * * 

   * |--level1_struct: struct
   * |   |--level2_struct: struct
   * |       |--level3_struct: struct
   * |           |--level_4_col: int
   * |       |--level_3_col: int
   *
* * At the second level of recursion on field {@code level2_struct} we would have parameters * *
    *
  1. leafColumnsToKeep=Set(Column(level1_struct.level2_struct.level_3_col)) *
  2. schema: * 
       *          |--level3_struct: struct
   *          |   |--level_4_col: int
   *          |--level_3_col: int
   *
    *
  3. parentPath=["level1_struct", "level2_struct"] *
* * @param leafColumnsToKeep set of leaf columns relative to the schema root * @param schema schema to prune * @param parentPath parent path of the fields in {@code schema} relative to the schema root */ private static StructType pruneSchema( Set leafColumnsToKeep, StructType schema, String[] parentPath) { List prunedFields = new ArrayList<>(); for (StructField field : schema.fields()) { String[] colPath = appendArray(parentPath, field.getName()); if (field.getDataType() instanceof StructType) { StructType prunedNestedSchema = pruneSchema(leafColumnsToKeep, (StructType) field.getDataType(), colPath); if (prunedNestedSchema.length() > 0) { // Only add a struct field it has un-pruned nested columns prunedFields.add( new StructField( field.getName(), prunedNestedSchema, field.isNullable(), field.getMetadata())); } } else { if (leafColumnsToKeep.contains(new Column(colPath))) { prunedFields.add(field); } } } return new StructType(prunedFields); } /** * Given an array {@code arr} and a string element {@code appendElem} return a new array with * {@code appendElem} inserted at the end */ private static String[] appendArray(String[] arr, String appendElem) { String[] newNames = new String[arr.length + 1]; System.arraycopy(arr, 0, newNames, 0, arr.length); newNames[arr.length] = appendElem; return newNames; } /** * Rewrite `EqualNullSafe(a, NotNullLiteral)` as `And(IsNotNull(a), EqualTo(a, NotNullLiteral))` * and rewrite `EqualNullSafe(a, null)` as `IsNull(a)` */ private static Predicate rewriteEqualNullSafe( Column leftCol, Literal rightLit, Optional collationIdentifier) { if (rightLit.getValue() == null) { return new Predicate("IS_NULL", leftCol); } return new Predicate( "AND", new Predicate("IS_NOT_NULL", leftCol), createPredicate("=", leftCol, rightLit, collationIdentifier)); } /** Helper method for building DataSkippingPredicate for NOT =/IS NOT DISTINCT FROM */ private static Optional constructDataSkippingFiltersForNotEqual( Predicate equalPredicate, StatsSchemaHelper schemaHelper, BiFunction> buildDataSkippingPredicateFunc) { checkArgument( "=".equals(equalPredicate.getName()) || "IS NOT DISTINCT FROM".equals(equalPredicate.getName()), "Expects predicate to be = or IS NOT DISTINCT FROM"); Expression leftChild = getLeft(equalPredicate); Expression rightChild = getRight(equalPredicate); Optional collationIdentifier = equalPredicate.getCollationIdentifier(); if (rightChild instanceof Column && leftChild instanceof Literal) { return constructDataSkippingFilter( new Predicate( "NOT", createPredicate( equalPredicate.getName(), rightChild, leftChild, collationIdentifier)), schemaHelper); } if (leftChild instanceof Column && rightChild instanceof Literal) { Column leftCol = (Column) leftChild; Literal rightLit = (Literal) rightChild; if (schemaHelper.isSkippingEligibleMinMaxColumn(leftCol) && schemaHelper.isSkippingEligibleLiteral(rightLit)) { return buildDataSkippingPredicateFunc.apply(leftCol, rightLit); } } return Optional.empty(); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/StatsSchemaHelper.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.skipping; import static io.delta.kernel.internal.util.ColumnMapping.getPhysicalName; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.expressions.Column; import io.delta.kernel.expressions.Expression; import io.delta.kernel.expressions.Literal; import io.delta.kernel.expressions.ScalarExpression; import io.delta.kernel.internal.util.Tuple2; import io.delta.kernel.types.*; import java.util.*; import java.util.stream.Collectors; /** * Provides information and utilities for statistics columns given a table schema. Specifically, it * is used to: * *
    *
  1. Get the expected statistics schema given a table schema *
  2. Check if a {@link Literal} or {@link Column} is skipping eligible *
  3. Get the statistics column for a given stat type and logical column *
*/ public class StatsSchemaHelper { ////////////////////////////////////////////////////////////////////////////////// // Public static fields and methods ////////////////////////////////////////////////////////////////////////////////// /* Delta statistics field names for file statistics */ public static final String NUM_RECORDS = "numRecords"; public static final String MIN = "minValues"; public static final String MAX = "maxValues"; public static final String NULL_COUNT = "nullCount"; public static final String TIGHT_BOUNDS = "tightBounds"; public static final String STATS_WITH_COLLATION = "statsWithCollation"; /** * Returns true if the given literal is skipping-eligible. Delta tracks min/max stats for a * limited set of data types and only literals of those types are skipping eligible. */ public static boolean isSkippingEligibleLiteral(Literal literal) { return isSkippingEligibleDataType(literal.getDataType()); } /** Returns true if the given data type is eligible for MIN/MAX data skipping. */ public static boolean isSkippingEligibleDataType(DataType dataType) { return SKIPPING_ELIGIBLE_TYPE_NAMES.contains(dataType.toString()) || // DecimalType is eligible, but since its string includes scale + precision, it needs to // be matched separately. dataType instanceof DecimalType || // StringType is eligible, but since its string can include collation info, it needs to // be matched separately. dataType instanceof StringType; } /** * Returns the expected statistics schema given a table schema. * *

Here is an example of a data schema along with the schema of the statistics that would be * collected. * *

Data Schema: * * 

   * |-- a: struct (nullable = true)
   * |  |-- b: struct (nullable = true)
   * |  |  |-- c: long (nullable = true)
   * |  |  |-- d: string (nullable = true)
   *
* *

Collected Statistics: * * 

   * |-- stats: struct (nullable = true)
   * |  |-- numRecords: long (nullable = false)
   * |  |-- minValues: struct (nullable = false)
   * |  |  |-- a: struct (nullable = false)
   * |  |  |  |-- b: struct (nullable = false)
   * |  |  |  |  |-- c: long (nullable = true)
   * |  |  |  |  |-- d: string (nullable = true)
   * |  |-- maxValues: struct (nullable = false)
   * |  |  |-- a: struct (nullable = false)
   * |  |  |  |-- b: struct (nullable = false)
   * |  |  |  |  |-- c: long (nullable = true)
   * |  |  |  |  |-- d: string (nullable = true)
   * |  |-- nullCount: struct (nullable = false)
   * |  |  |-- a: struct (nullable = false)
   * |  |  |  |-- b: struct (nullable = false)
   * |  |  |  |  |-- c: long (nullable = true)
   * |  |  |  |  |-- d: string (nullable = true)
   * |  |-- tightBounds: boolean (nullable = true)
   * |  |-- statsWithCollation: struct (nullable = true)
   * |  |  |-- collationName: struct (nullable = true)
   * |  |  |  |-- min: struct (nullable = true)
   * |  |  |  |  |-- a: struct (nullable = true)
   * |  |  |  |  |  |-- b: struct (nullable = true)
   * |  |  |  |  |  |  |-- d: string (nullable = true)
   * |  |  |  |-- max: struct (nullable = true)
   * |  |  |  |  |-- a: struct (nullable = true)
   * |  |  |  |  |  |-- b: struct (nullable = true)
   * |  |  |  |  |  |  |-- d: string (nullable = true)
   *
*/ public static StructType getStatsSchema( StructType dataSchema, Set collationIdentifiers) { StructType statsSchema = new StructType().add(NUM_RECORDS, LongType.LONG, true); StructType minMaxStatsSchema = getMinMaxStatsSchema(dataSchema); if (minMaxStatsSchema.length() > 0) { statsSchema = statsSchema.add(MIN, minMaxStatsSchema, true).add(MAX, minMaxStatsSchema, true); } StructType nullCountSchema = getNullCountSchema(dataSchema); if (nullCountSchema.length() > 0) { statsSchema = statsSchema.add(NULL_COUNT, nullCountSchema, true); } statsSchema = statsSchema.add(TIGHT_BOUNDS, BooleanType.BOOLEAN, true); StructType collatedMinMaxStatsSchema = getCollatedStatsSchema(dataSchema, collationIdentifiers); if (collatedMinMaxStatsSchema.length() > 0) { statsSchema = statsSchema.add(STATS_WITH_COLLATION, collatedMinMaxStatsSchema, true); } return statsSchema; } ////////////////////////////////////////////////////////////////////////////////// // Instance fields and public methods ////////////////////////////////////////////////////////////////////////////////// private final StructType dataSchema; /* Map of all leaf columns from logical to physical names */ private final Map logicalToPhysicalColumn; /* Map of all leaf logical columns to their data type */ private final Map logicalToDataType; public StatsSchemaHelper(StructType dataSchema) { this.dataSchema = dataSchema; Map> logicalToPhysicalColumnAndDataType = getLogicalToPhysicalColumnAndDataType(dataSchema); this.logicalToPhysicalColumn = logicalToPhysicalColumnAndDataType.entrySet().stream() .collect( Collectors.toMap( Map.Entry::getKey, e -> e.getValue()._1 // map to just the column )); this.logicalToDataType = logicalToPhysicalColumnAndDataType.entrySet().stream() .collect( Collectors.toMap( Map.Entry::getKey, e -> e.getValue()._2 // map to just the data type )); } /** * Given a logical column in the data schema provided when creating {@code this}, return the * corresponding MIN column and an optional column adjustment expression from the statistic schema * that stores the MIN values for the provided logical column. * * @param column the logical column name. * @param collationIdentifier optional collation identifier if getting a collated stats column. * @return a tuple of the MIN column and an optional adjustment expression. */ public Tuple2> getMinColumn( Column column, Optional collationIdentifier) { checkArgument( isSkippingEligibleMinMaxColumn(column), "%s is not a valid min column%s for data schema %s", column, collationIdentifier.isPresent() ? (" for collation " + collationIdentifier) : "", dataSchema); return new Tuple2<>(getStatsColumn(column, MIN, collationIdentifier), Optional.empty()); } /** * Given a logical column in the data schema provided when creating {@code this}, return the * corresponding MAX column and an optional column adjustment expression from the statistic schema * that stores the MAX values for the provided logical column. * * @param column the logical column name. * @param collationIdentifier optional collation identifier if getting a collated stats column. * @return a tuple of the MAX column and an optional adjustment expression. */ public Tuple2> getMaxColumn( Column column, Optional collationIdentifier) { checkArgument( isSkippingEligibleMinMaxColumn(column), "%s is not a valid min column%s for data schema %s", column, collationIdentifier.isPresent() ? (" for collation " + collationIdentifier) : "", dataSchema); DataType dataType = logicalToDataType.get(column); Column maxColumn = getStatsColumn(column, MAX, collationIdentifier); // If this is a column of type Timestamp or TimestampNTZ // compensate for the truncation from microseconds to milliseconds // by adding 1 millisecond. For example, a file containing only // 01:02:03.456789 will be written with min == max == 01:02:03.456, so we must consider it // to contain the range from 01:02:03.456 to 01:02:03.457. if (dataType instanceof TimestampType || dataType instanceof TimestampNTZType) { return new Tuple2<>( maxColumn, Optional.of( new ScalarExpression("TIMEADD", Arrays.asList(maxColumn, Literal.ofLong(1))))); } return new Tuple2<>(maxColumn, Optional.empty()); } /** * Given a logical column in the data schema provided when creating {@code this}, return the * corresponding NULL_COUNT column in the statistic schema that stores the null count values for * the provided logical column. */ public Column getNullCountColumn(Column column) { checkArgument( isSkippingEligibleNullCountColumn(column), "%s is not a valid null_count column for data schema %s", column, dataSchema); return getStatsColumn(column, NULL_COUNT, Optional.empty()); } /** Returns the NUM_RECORDS column in the statistic schema */ public Column getNumRecordsColumn() { return new Column(NUM_RECORDS); } /** * Returns true if the given column is skipping-eligible using min/max statistics. This means the * column exists, is a leaf column, and is of a skipping-eligible data-type. */ public boolean isSkippingEligibleMinMaxColumn(Column column) { return logicalToDataType.containsKey(column) && isSkippingEligibleDataType(logicalToDataType.get(column)); } /** * Returns true if the given column is skipping-eligible using null count statistics. This means * the column exists and is a leaf column as we only collect stats for leaf columns. */ public boolean isSkippingEligibleNullCountColumn(Column column) { return logicalToPhysicalColumn.containsKey(column); } ////////////////////////////////////////////////////////////////////////////////// // Private static fields and methods ////////////////////////////////////////////////////////////////////////////////// private static final Set SKIPPING_ELIGIBLE_TYPE_NAMES = new HashSet() { { add("byte"); add("short"); add("integer"); add("long"); add("float"); add("double"); add("date"); add("timestamp"); add("timestamp_ntz"); } }; /** * Given a data schema returns the expected schema for a min or max statistics column. This means * 1) replace logical names with physical names 2) set nullable=true 3) only keep stats eligible * fields (i.e. don't include fields with isSkippingEligibleDataType=false) */ private static StructType getMinMaxStatsSchema(StructType dataSchema) { List fields = new ArrayList<>(); for (StructField field : dataSchema.fields()) { if (isSkippingEligibleDataType(field.getDataType())) { fields.add(new StructField(getPhysicalName(field), field.getDataType(), true)); } else if (field.getDataType() instanceof StructType) { fields.add( new StructField( getPhysicalName(field), getMinMaxStatsSchema((StructType) field.getDataType()), true)); } } return new StructType(fields); } /** * Given a data schema and a set of collation identifiers returns the expected schema for * collation-aware statistics columns. */ private static StructType getCollatedStatsSchema( StructType dataSchema, Set collationIdentifiers) { StructType statsWithCollation = new StructType(); StructType collationAwareFields = getCollationAwareFields(dataSchema); for (CollationIdentifier collationIdentifier : collationIdentifiers) { if (collationIdentifier.isSparkUTF8BinaryCollation()) { // For SPARK.UTF8_BINARY collation we use the binary stats continue; } if (collationIdentifier.getVersion().isEmpty()) { throw new IllegalArgumentException( String.format( "Collation identifier %s must specify a collation version for collation-aware " + "statistics.", collationIdentifier)); } if (collationAwareFields.length() > 0) { statsWithCollation = statsWithCollation.add( collationIdentifier.toString(), new StructType() .add(MIN, collationAwareFields, true) .add(MAX, collationAwareFields, true), true); } } return statsWithCollation; } /** Given a data schema returns its collation aware fields. */ private static StructType getCollationAwareFields(StructType dataSchema) { StructType collationAwareFields = new StructType(); for (StructField field : dataSchema.fields()) { DataType dataType = field.getDataType(); if (dataType instanceof StructType) { StructType nestedCollationAwareFields = getCollationAwareFields((StructType) dataType); if (nestedCollationAwareFields.length() > 0) { collationAwareFields = collationAwareFields.add(getPhysicalName(field), nestedCollationAwareFields, true); } } else if (dataType instanceof StringType) { collationAwareFields = collationAwareFields.add(getPhysicalName(field), dataType, true); } } return collationAwareFields; } /** * Given a data schema returns the expected schema for a null_count statistics column. This means * 1) replace logical names with physical names 2) set nullable=true 3) use LongType for all * fields */ private static StructType getNullCountSchema(StructType dataSchema) { List fields = new ArrayList<>(); for (StructField field : dataSchema.fields()) { if (field.getDataType() instanceof StructType) { fields.add( new StructField( getPhysicalName(field), getNullCountSchema((StructType) field.getDataType()), true)); } else { fields.add(new StructField(getPhysicalName(field), LongType.LONG, true)); } } return new StructType(fields); } ////////////////////////////////////////////////////////////////////////////////// // Private class helpers ////////////////////////////////////////////////////////////////////////////////// /** * Given a logical column and a stats type returns the corresponding column in the statistics * schema */ private Column getStatsColumn( Column column, String statType, Optional collationIdentifier) { checkArgument( logicalToPhysicalColumn.containsKey(column), "%s is not a valid leaf column for data schema: %s", column, dataSchema); Column physicalColumn = logicalToPhysicalColumn.get(column); // Use binary stats if collation is not specified or if it is the default Spark collation. if (collationIdentifier.isPresent() && collationIdentifier.get() != CollationIdentifier.SPARK_UTF8_BINARY) { // Collation-aware stats are stored under `statsWithCollation.collationName.statType`. return getChildColumn( physicalColumn, Arrays.asList(STATS_WITH_COLLATION, collationIdentifier.get().toString(), statType)); } else { // Binary stats are stored under `statType`. return getChildColumn(physicalColumn, statType); } } /** * Given a data schema returns a map of {logical column -> (physical column, data type)} for all * leaf columns in the schema. */ private Map> getLogicalToPhysicalColumnAndDataType( StructType dataSchema) { Map> result = new HashMap<>(); for (StructField field : dataSchema.fields()) { if (field.getDataType() instanceof StructType) { Map> nestedCols = getLogicalToPhysicalColumnAndDataType((StructType) field.getDataType()); for (Column childLogicalCol : nestedCols.keySet()) { Tuple2 childCol = nestedCols.get(childLogicalCol); Column childPhysicalCol = childCol._1; DataType childColDataType = childCol._2; result.put( getChildColumn(childLogicalCol, field.getName()), new Tuple2<>( getChildColumn(childPhysicalCol, getPhysicalName(field)), childColDataType)); } } else { result.put( new Column(field.getName()), new Tuple2<>(new Column(getPhysicalName(field)), field.getDataType())); } } return result; } /** Returns the provided column as a child column nested under {@code parentName} */ private static Column getChildColumn(Column column, String parentName) { return new Column(prependArray(column.getNames(), parentName)); } /** Returns the provided column as a child column nested under {@code nestedPath} */ private static Column getChildColumn(Column column, List nestedPath) { for (int i = nestedPath.size() - 1; i >= 0; i--) { String name = nestedPath.get(i); column = getChildColumn(column, name); } return column; } /** * Given an array {@code names} and a string element {@code preElem} return a new array with * {@code preElem} inserted at the beginning */ private static String[] prependArray(String[] arr, String preElem) { String[] newNames = new String[arr.length + 1]; newNames[0] = preElem; System.arraycopy(arr, 0, newNames, 1, arr.length); return newNames; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/snapshot/LogSegment.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.snapshot; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.internal.annotation.VisibleForTesting; import io.delta.kernel.internal.files.ParsedCatalogCommitData; import io.delta.kernel.internal.files.ParsedDeltaData; import io.delta.kernel.internal.files.ParsedPublishedDeltaData; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.lang.Lazy; import io.delta.kernel.internal.lang.ListUtils; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.internal.util.Tuple2; import io.delta.kernel.utils.FileStatus; import java.util.*; import java.util.stream.Collectors; import java.util.stream.LongStream; import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class LogSegment { ////////////////////////////////////////// // Static factory methods and constants // ////////////////////////////////////////// /** * Creates a LogSegment for a newly created table from a single {@link ParsedDeltaData}. Used to * construct a post-commit Snapshot after a CREATE transaction. * * @param logPath The path to the _delta_log directory * @param parsedDeltaVersion0 The ParsedDeltaData that must be for version 0 * @return A new LogSegment with just this delta * @throws IllegalArgumentException if the ParsedDeltaData is not file-based */ public static LogSegment createForNewTable(Path logPath, ParsedDeltaData parsedDeltaVersion0) { checkArgument(parsedDeltaVersion0.isFile(), "Currently, only file-based deltas are supported"); checkArgument( parsedDeltaVersion0.getVersion() == 0L, "Version must be 0 for a LogSegment with only a single delta"); final FileStatus deltaFile = parsedDeltaVersion0.getFileStatus(); final List deltas = Collections.singletonList(deltaFile); final List checkpoints = Collections.emptyList(); final List compactions = Collections.emptyList(); final Optional maxPublishedDeltaVersion = parsedDeltaVersion0 instanceof ParsedPublishedDeltaData ? Optional.of(0L) : Optional.empty(); return new LogSegment( logPath, 0 /* version */, deltas, compactions, checkpoints, deltaFile /* deltaAtEndVersion */, Optional.empty() /* lastSeenChecksum */, maxPublishedDeltaVersion); } private static final Logger logger = LoggerFactory.getLogger(LogSegment.class); ////////////////////////////////// // Member methods and variables // ////////////////////////////////// private final Path logPath; private final long version; private final List deltas; private final List compactions; private final List checkpoints; private final FileStatus deltaAtEndVersion; private final Optional checkpointVersionOpt; private final Optional lastSeenChecksum; private final Optional maxPublishedDeltaVersion; private final List deltasAndCheckpoints; private final Lazy> deltasAndCheckpointsReversed; private final Lazy> compactionsReversed; private final Lazy> deltasCheckpointsCompactionsReversed; /** * Provides information around which files in the transaction log need to be read to create the * given version of the log. * *

This constructor validates and guarantees that: * *

    *
  • All deltas are valid deltas files *
  • All checkpoints are valid checkpoint files *
  • All checkpoint files have the same version *
  • All deltas are contiguous and range from {@link #checkpointVersionOpt} + 1 to version *
  • If no deltas are present then {@link #checkpointVersionOpt} is equal to version *
* * @param logPath The path to the _delta_log directory * @param version The Snapshot version to generate * @param deltas The delta commit files (.json) to read * @param compactions Any found log compactions files that can be used in place of some or all of * the deltas * @param checkpoints The checkpoint file(s) to read * @param deltaAtEndVersion The delta file at the end version of this LogSegment. If this * LogSegment contains only checkpoints (e.g. 10.checkpoint only) then this is the delta at * that checkpoint version. * @param lastSeenChecksum The most recent checksum file encountered during log directory listing, * if available. * @param maxPublishedDeltaVersion The maximum version among all published delta files seen during * log segment construction, if available. Note that the Published Delta file for this version * may not be included as a Delta in this LogSegment, if there was a catalog commit that took * priority over it. */ public LogSegment( Path logPath, long version, List deltas, List compactions, List checkpoints, FileStatus deltaAtEndVersion, Optional lastSeenChecksum, Optional maxPublishedDeltaVersion) { /////////////////////// // Input validations // /////////////////////// requireNonNull(logPath, "logPath is null"); requireNonNull(deltas, "deltas is null"); requireNonNull(compactions, "compactions is null"); requireNonNull(checkpoints, "checkpoints is null"); requireNonNull(deltaAtEndVersion, "deltaAtEndVersion is null"); requireNonNull(lastSeenChecksum, "lastSeenChecksum null"); checkArgument(version >= 0, "version must be >= 0"); validateDeltasAreDeltas(deltas); validateCompactionsAreCompactions(compactions); validateCheckpointsAreCheckpoints(checkpoints); validateIndividualCompactionVersions(compactions); this.checkpointVersionOpt = checkpoints.isEmpty() ? Optional.empty() : Optional.of(FileNames.checkpointVersion(new Path(checkpoints.get(0).getPath()))); validateCheckpointVersionsAreSame(checkpoints, checkpointVersionOpt); validateLastSeenChecksumWithinLogSegmentStartEndVersionRange( lastSeenChecksum, version, checkpointVersionOpt); checkArgument(!deltas.isEmpty() || !checkpoints.isEmpty(), "No files to read"); if (!deltas.isEmpty()) { final List deltaVersions = deltas.stream() .map(fs -> FileNames.deltaVersion(new Path(fs.getPath()))) .collect(Collectors.toList()); validateFirstDeltaVersionIsCheckpointVersionPlusOne(deltaVersions, checkpointVersionOpt); validateLastDeltaVersionIsLogSegmentVersion(deltaVersions, version); validateDeltaVersionsAreContiguous(deltaVersions); validateCompactionVersionsAreInRange(compactions, version, checkpointVersionOpt); } else { validateCheckpointVersionEqualsLogSegmentVersion(checkpointVersionOpt, version); } validateDeltaAtEndVersion(version, deltaAtEndVersion); // Make sure input delta commits (JSON file), checkpoints and log compactions are valid. assertLogFilesBelongToTable( logPath, Stream.concat(checkpoints.stream(), Stream.concat(deltas.stream(), compactions.stream())) .collect(Collectors.toList())); //////////////////////////////// // Member variable assignment // //////////////////////////////// this.logPath = logPath; this.version = version; this.deltas = deltas; this.compactions = compactions; this.checkpoints = checkpoints; this.deltaAtEndVersion = deltaAtEndVersion; this.lastSeenChecksum = lastSeenChecksum; this.maxPublishedDeltaVersion = maxPublishedDeltaVersion; this.deltasAndCheckpoints = Stream.concat(checkpoints.stream(), deltas.stream()).collect(Collectors.toList()); this.deltasAndCheckpointsReversed = lazyLoadDeltasAndCheckpointsReversed(deltasAndCheckpoints); // We sort by the end version. since we work backward through the list, so this is the same as // lexicographic, except when a compaction has a bigger range, which makes it "better", so we // prefer it this.compactionsReversed = lazyLoadCompactionsReversed(compactions); this.deltasCheckpointsCompactionsReversed = lazyLoadDeltasCheckpointsCompactionsReversed( deltasAndCheckpointsReversed, compactionsReversed, compactions); logger.debug("Created LogSegment: {}", this); } ///////////////// // Public APIs // ///////////////// public Path getLogPath() { return logPath; } public long getVersion() { return version; } public List getDeltas() { return deltas; } public List getCompactions() { return compactions; } public List getCheckpoints() { return checkpoints; } public Optional getCheckpointVersionOpt() { return checkpointVersionOpt; } /** * Returns the most recent checksum file encountered during log directory listing, if available. * *

Note: This checksum file's version is guaranteed to: * *

    *
  • Be less than or equal to the LogSegment version (enforced by constructor) *
  • Be greater than or equal to the checkpoint version if a checkpoint exists (filtered * during initialization) *
* * @return Optional containing the most recent valid checksum file encountered, or empty if none * found */ public Optional getLastSeenChecksum() { return lastSeenChecksum; } /** * Returns the maximum published delta version observed during log segment construction. * *

This is a best-effort API that returns what was actually seen during construction, not the * authoritative maximum published delta version in the log. * *

{@code Optional.empty()} means "we don't know" - not necessarily that no deltas have been * published. This can occur when: * *

    *
  • Only checkpoint files were found during listing (e.g., due to log cleanup) *
  • Listing bounds did not include published delta files *
  • The table contains only catalog commits with no published deltas *
* * @return the maximum published delta version seen during construction, or empty if unknown */ public Optional getMaxPublishedDeltaVersion() { return maxPublishedDeltaVersion; } /** * Returns the Delta file at the end {@code version} of this LogSegment. * *

If this LogSegment has checkpoints and deltas, then this is the last delta. * *

If this LogSegment has only checkpoints (i.e. 10.checkpoint only) then this is the delta at * that checkpoint version. */ public FileStatus getDeltaFileAtEndVersion() { return deltaAtEndVersion; } /** * @return all deltas (.json) and checkpoint (.checkpoint.parquet) files in this LogSegment, * sorted in reverse (00012.json, 00011.json, 00010.checkpoint.parquet) order. */ public List allLogFilesReversed() { return deltasAndCheckpointsReversed.get(); } /** * @return all files sorted in reverse order in this log segment, but omitting the deltas (.json) * files that are covered by log compaction files. This will include deltas (xxx.json) that * are not covered by a log compaction, compaction files (xxx.xxx.json), and checkpoints * (.checkpoint.parquet). */ public List allFilesWithCompactionsReversed() { return deltasCheckpointsCompactionsReversed.get(); } public List getAllCatalogCommits() { return deltas.stream() .map(ParsedDeltaData::forFileStatus) .filter(x -> x instanceof ParsedCatalogCommitData) .map(ParsedCatalogCommitData.class::cast) .collect(Collectors.toList()); } /** * Creates a new LogSegment by extending this LogSegment with additional deltas. Used to construct * a post-commit Snapshot from a previous Snapshot. * *

The additional deltas must be contiguous and start at version + 1. * * @param addedDeltas List of ParsedDeltaData to add (must be contiguous and start at current * version + 1) * @return A new LogSegment with the additional deltas * @throws IllegalArgumentException if deltas are not contiguous or don't start at version + 1 */ public LogSegment newWithAddedDeltas(List addedDeltas) { if (addedDeltas.isEmpty()) { return this; } // Validate file-based (not inline), contiguous, and starts at version + 1. Then, convert to // file status. final List newDeltaFileStatuses = new ArrayList<>(addedDeltas.size()); long expectedVersion = version + 1; for (ParsedDeltaData delta : addedDeltas) { checkArgument(delta.isFile(), "Currently, only file-based deltas are supported"); checkArgument( delta.getVersion() == expectedVersion, "Delta versions must be contiguous. Expected %d but got %d", expectedVersion, delta.getVersion()); newDeltaFileStatuses.add(delta.getFileStatus()); expectedVersion++; } final List combinedDeltas = new ArrayList<>(deltas); combinedDeltas.addAll(newDeltaFileStatuses); final ParsedDeltaData lastAddedDelta = ListUtils.getLast(addedDeltas); return new LogSegment( logPath, lastAddedDelta.getVersion(), // Use the updated version combinedDeltas, compactions, // Keep existing compactions checkpoints, // Keep existing checkpoints lastAddedDelta.getFileStatus(), lastSeenChecksum, // Keep existing lastSeenChecksum maxPublishedDeltaVersion); // Keep existing maxPublishedDeltaVersion } /** * Creates a new LogSegment that reflects the published commits. Used to construct a post-publish * Snapshot from a previous Snapshot. * * @return A new LogSegment with published commits */ public LogSegment newAsPublished() { FileStatus lastDeltaFileStatus = FileStatus.of(FileNames.deltaFile(logPath, version)); long deltaStartVersion = this.checkpointVersionOpt.map(i -> i + 1).orElse(0L); return new LogSegment( logPath, version, LongStream.rangeClosed(deltaStartVersion, version) .mapToObj(v -> FileStatus.of(FileNames.deltaFile(logPath, v))) .collect(Collectors.toList()), getCompactions(), getCheckpoints(), lastDeltaFileStatus, getLastSeenChecksum(), Optional.of(version)); } @Override public String toString() { return String.format( "LogSegment {\n" + " logPath='%s',\n" + " version=%d,\n" + " deltas=[%s\n ],\n" + " checkpoints=[%s\n ],\n" + " deltaAtEndVersion=%s,\n" + " lastSeenChecksum=%s,\n" + " checkpointVersion=%s,\n" + " maxPublishedDeltaVersion=%s\n" + "}", logPath, version, formatList(deltas), formatList(checkpoints), deltaAtEndVersion, lastSeenChecksum.map(FileStatus::toString).orElse("None"), checkpointVersionOpt.map(String::valueOf).orElse("None"), maxPublishedDeltaVersion.map(String::valueOf).orElse("None")); } @Override public int hashCode() { // TODO: support staged commits #4927 return Objects.hash(deltas, checkpoints, compactions); } ////////////////////////////// // Input validation methods // ////////////////////////////// private void validateDeltasAreDeltas(List deltas) { checkArgument( deltas.stream().allMatch(fs -> FileNames.isCommitFile(fs.getPath())), () -> "deltas must all be actual delta (commit) files: " + deltas); } private void validateCompactionsAreCompactions(List compactions) { checkArgument( compactions.stream().allMatch(fs -> FileNames.isLogCompactionFile(fs.getPath())), () -> "compactions must all be actual log compaction files: " + compactions); } private void validateCheckpointsAreCheckpoints(List checkpoints) { checkArgument( checkpoints.stream().allMatch(fs -> FileNames.isCheckpointFile(fs.getPath())), () -> "checkpoints must all be actual checkpoint files: " + checkpoints); } private void validateIndividualCompactionVersions(List compactions) { checkArgument( compactions.stream() .allMatch( fs -> { Tuple2 versions = FileNames.logCompactionVersions(fs.getPath()); return versions._1 < versions._2; }), () -> "compactions must have start version less than end version: " + compactions); } private void validateCheckpointVersionsAreSame( List checkpoints, Optional checkpointVersionOpt) { if (!checkpoints.isEmpty()) { checkArgument( checkpoints.stream() .map(fs -> FileNames.checkpointVersion(new Path(fs.getPath()))) .allMatch(v -> checkpointVersionOpt.get().equals(v)), () -> "All checkpoint files must have the same version: " + checkpoints); } } private void validateLastSeenChecksumWithinLogSegmentStartEndVersionRange( Optional lastSeenChecksum, long version, Optional checkpointVersionOpt) { lastSeenChecksum.ifPresent( checksumFile -> { long checksumVersion = FileNames.checksumVersion(new Path(checksumFile.getPath())); checkArgument( checksumVersion <= version, "checksum version (%d) should be less than or equal to LogSegment version (%d)", checksumVersion, version); checkpointVersionOpt.ifPresent( checkpointVersion -> checkArgument( checksumVersion >= checkpointVersion, "checksum version (%d) should be greater than or equal to checkpoint " + "version (%d)", checksumVersion, checkpointVersion)); }); } private void validateFirstDeltaVersionIsCheckpointVersionPlusOne( List deltaVersions, Optional checkpointVersionOpt) { checkpointVersionOpt.ifPresent( checkpointVersion -> { checkArgument( deltaVersions.get(0) == checkpointVersion + 1, "First delta file version (%d) must equal checkpointVersion + 1 (%d)", deltaVersions.get(0), checkpointVersion + 1); }); } private void validateLastDeltaVersionIsLogSegmentVersion(List deltaVersions, long version) { checkArgument( ListUtils.getLast(deltaVersions) == version, "Last delta file version (%d) must equal LogSegment version (%d)", ListUtils.getLast(deltaVersions), version); } private void validateDeltaVersionsAreContiguous(List deltaVersions) { for (int i = 1; i < deltaVersions.size(); i++) { checkArgument( deltaVersions.get(i) == deltaVersions.get(i - 1) + 1, () -> "Delta versions must be contiguous: " + deltaVersions); } } private void validateCompactionVersionsAreInRange( List compactions, long version, Optional checkpointVersionOpt) { checkArgument( compactions.stream() .allMatch( fs -> { Tuple2 versions = FileNames.logCompactionVersions(fs.getPath()); boolean checkpointVersionOkay = checkpointVersionOpt .map(checkpointVersion -> versions._1 > checkpointVersion) .orElse(true); return checkpointVersionOkay && versions._2 <= version; }), () -> String.format( "compactions must have startVersion > checkpointVersion (%d) AND endVersion <= " + "version (%d): %s", checkpointVersionOpt.orElse(-1L), version, compactions)); } private void validateCheckpointVersionEqualsLogSegmentVersion( Optional checkpointVersionOpt, long version) { checkpointVersionOpt.ifPresent( checkpointVersion -> { checkArgument( checkpointVersion == version, "If no deltas, then checkpointVersion (%d) must equal LogSegment version (%d)", checkpointVersion, version); }); } private void validateDeltaAtEndVersion(long version, FileStatus deltaAtEndVersion) { checkArgument( FileNames.isCommitFile(deltaAtEndVersion.getPath()), "deltaAtEndVersion must be a delta file: " + deltaAtEndVersion); final long deltaVersion = FileNames.deltaVersion(deltaAtEndVersion.getPath()); checkArgument( deltaVersion == version, "deltaAtEndVersion (%d) must be equal to LogSegment version (%d)", deltaVersion, version); } ////////////////////////// // Other helper methods // ////////////////////////// private Lazy> lazyLoadDeltasAndCheckpointsReversed( List deltasAndCheckpoints) { return new Lazy<>( () -> deltasAndCheckpoints.stream() .sorted( Comparator.comparing((FileStatus a) -> new Path(a.getPath()).getName()) .reversed()) .collect(Collectors.toList())); } private Lazy> lazyLoadCompactionsReversed(List compactions) { return new Lazy<>( () -> compactions.stream() .sorted( Comparator.comparing( (FileStatus a) -> FileNames.logCompactionVersions(a.getPath())._2) .reversed()) .collect(Collectors.toList())); } private Lazy> lazyLoadDeltasCheckpointsCompactionsReversed( Lazy> deltasAndCheckpointsReversed, Lazy> compactionsReversed, List compactions) { return new Lazy<>( () -> { if (compactions.isEmpty()) { return deltasAndCheckpointsReversed.get(); } else { LogCompactionResolver resolver = new LogCompactionResolver( deltasAndCheckpointsReversed.get(), compactionsReversed.get()); return resolver.resolveFiles(); } }); } private String formatList(List list) { if (list.isEmpty()) { return ""; } return "\n " + list.stream().map(FileStatus::toString).collect(Collectors.joining(",\n ")); } /** * Verifies that a set of delta or checkpoint files to be read actually belongs to this table. * Visible only for testing. */ @VisibleForTesting static void assertLogFilesBelongToTable(Path logPath, List allFiles) { String logPathStr = logPath.toString(); // fully qualified path for (FileStatus fileStatus : allFiles) { String filePath = fileStatus.getPath(); if (!filePath.startsWith(logPathStr)) { throw new RuntimeException( String.format( "File (%s) doesn't belong in the transaction log at %s.", filePath, logPathStr)); } } } //////////////////// // Helper classes // //////////////////// // Class to resolve the final list of deltas + log compactions to return private class LogCompactionResolver { // note that currentCompactionPos _always_ points to a valid compaction we'll be including, _or_ // past the end of the list of compactions (meaning we've consumed them all). The compaction // pointed to will be added to the output when we hit a delta with a version equal to the low // version of the compaction. int currentCompactionPos = 0; long currentCompactionHi = -1; long currentCompactionLo = -1; final List compactionsReversed; final Iterator deltaIt; LogCompactionResolver(List allFilesReversed, List compactionsReversed) { this.deltaIt = allFilesReversed.iterator(); this.compactionsReversed = compactionsReversed; } // We have two lists, one of deltas and one of compactions. Each is sorted in DESCENDING // order. Given this, resolves as follows: // - set a "hi/lo" goalpost around the next compactions // - for each delta, if its version is: // - greater than the current compaction high point, include it, move to next delta // - less than (but not equal to) the current compaction low point, skip it, move to next // delta // - equal to the current compaction low point, we're about to transition out of the // compaction, so, include the compaction, find the next compaction that has a high // point lower than our current low point and set that to the current compaction to // consider. This deals with overlapping compactions in a greedy way, ensuring we // ignore any overlapping compactions. List resolveFiles() { ArrayList ret = new ArrayList(); setHiLo(); while (deltaIt.hasNext()) { FileStatus currentDelta = deltaIt.next(); long deltaVersion = FileNames.deltaVersion(currentDelta.getPath()); if (deltaVersion == currentCompactionLo) { // we're about to cross out of the compaction. insert the compaction and advance to the // next compaction. We don't want to include this delta here. ret.add(compactionsReversed.get(currentCompactionPos)); advanceCompactionPos(); setHiLo(); } else if (deltaVersion > currentCompactionHi) { // this delta is not covered by the next compaction, include it. ret.add(currentDelta); } // just skip the file if none of the above are true, it's covered by the current compaction } return ret; } // Advance the compaction pos until we're pointing a compaction that has a end lower than our // current low mark (recall we move backwards through versions). This takes compactions in a // greedy manner, and ensures we don't use any overlapping compactions. private void advanceCompactionPos() { currentCompactionPos += 1; while (currentCompactionPos < compactionsReversed.size()) { Tuple2 versions = FileNames.logCompactionVersions( compactionsReversed.get(currentCompactionPos).getPath()); if (versions._2 < currentCompactionLo) { break; } currentCompactionPos += 1; } } // Set the high/low position based on the current currentCompactionPos private void setHiLo() { if (currentCompactionPos < compactionsReversed.size()) { Tuple2 versions = FileNames.logCompactionVersions( compactionsReversed.get(currentCompactionPos).getPath()); currentCompactionLo = versions._1; currentCompactionHi = versions._2; } else { currentCompactionLo = currentCompactionHi = -1; } } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/snapshot/MetadataCleanup.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.snapshot; import static io.delta.kernel.internal.DeltaErrors.wrapEngineExceptionThrowsIO; import static io.delta.kernel.internal.checkpoints.Checkpointer.getLatestCompleteCheckpointFromList; import static io.delta.kernel.internal.lang.ListUtils.getFirst; import static io.delta.kernel.internal.lang.ListUtils.getLast; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.stream.Collectors.toList; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.checkpoints.CheckpointInstance; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.util.Clock; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class MetadataCleanup { private static final Logger logger = LoggerFactory.getLogger(MetadataCleanup.class); private MetadataCleanup() {} /** * Delete the Delta log files (delta and checkpoint files) that are expired according to the table * metadata retention settings. While deleting the log files, it makes sure the time travel * continues to work for all unexpired table versions. * *

Here is algorithm: * *

    *
  • Initial the potential delete file list: `potentialFilesToDelete` as an empty list *
  • Initialize the last seen checkpoint file list: `lastSeenCheckpointFiles`. There could be * one or more checkpoint files for a given version. *
  • List the delta log files starting with prefix "00000000000000000000." (%020d). For each * file: *
      *
    • Step 1: Check if the `lastSeenCheckpointFiles` contains a complete checkpoint, then *
        *
      • Step 1.1: delete all files in `potentialFilesToDelete`. Now we know there is * a checkpoint that contains the compacted Delta log up to the checkpoint * version and all commit/checkpoint files before this checkpoint version are * not needed. *
      • Step 1.2: add `lastCheckpointFiles` to `potentialFileStoDelete` list. This * checkpoint is potential candidate to delete later if we find another * checkpoint *
      *
    • Step 2: If the timestamp falls within the retention period, stop *
    • Step 3: If the file is a delta log file, add it to the `potentialFilesToDelete` * list *
    • Step 4: If the file is a checkpoint file, add it to the `lastSeenCheckpointFiles` *
    *
* * @param engine {@link Engine} instance to delete the expired log files * @param clock {@link Clock} instance to get the current time. Useful in testing to mock the * current time. * @param tablePath Table location * @param retentionMillis Log file retention period in milliseconds * @return number of log files deleted * @throws IOException if an error occurs while deleting the log files */ public static long cleanupExpiredLogs( Engine engine, Clock clock, Path tablePath, long retentionMillis) throws IOException { checkArgument(retentionMillis >= 0, "Retention period must be non-negative"); List potentialLogFilesToDelete = new ArrayList<>(); long lastSeenCheckpointVersion = -1; // -1 indicates no checkpoint seen yet List lastSeenCheckpointFiles = new ArrayList<>(); long fileCutOffTime = clock.getTimeMillis() - retentionMillis; String tableName = tablePath.getName(); logger.info( "[tableName={}] Starting the deletion of log files older than {}", tableName, fileCutOffTime); long numDeleted = 0; try (CloseableIterator files = listDeltaLogs(engine, tablePath)) { while (files.hasNext()) { // Step 1: Check if the `lastSeenCheckpointFiles` contains a complete checkpoint Optional lastCompleteCheckpoint = getLatestCompleteCheckpointFromList( lastSeenCheckpointFiles.stream().map(CheckpointInstance::new).collect(toList()), CheckpointInstance.MAX_VALUE); if (lastCompleteCheckpoint.isPresent()) { // Step 1.1: delete all files in `potentialFilesToDelete`. Now we know there is a // checkpoint that contains the compacted Delta log up to the checkpoint version and all // commit/checkpoint files before this checkpoint version are not needed. add // `lastCheckpointFiles` to `potentialFileStoDelete` list. This checkpoint is potential // candidate to delete later if we find another checkpoint if (!potentialLogFilesToDelete.isEmpty()) { logger.info( "[tableName={}] Deleting log files (start = {}, end = {}) because a checkpoint at " + "version {} indicates that these log files are no longer needed.", tableName, getFirst(potentialLogFilesToDelete), getLast(potentialLogFilesToDelete), lastSeenCheckpointVersion); numDeleted += deleteLogFiles(engine, potentialLogFilesToDelete); potentialLogFilesToDelete.clear(); } // Step 1.2: add `lastCheckpointFiles` to `potentialFileStoDelete` list. This checkpoint // is potential candidate to delete later if we find another checkpoint potentialLogFilesToDelete.addAll(lastSeenCheckpointFiles); lastSeenCheckpointFiles.clear(); lastSeenCheckpointVersion = -1; } FileStatus nextFile = files.next(); // Step 2: If the timestamp is earlier than the retention period, stop if (nextFile.getModificationTime() > fileCutOffTime) { if (!potentialLogFilesToDelete.isEmpty()) { logger.info( "[tableName={}] Skipping deletion of expired log files {}, because there is " + "no checkpoint file that indicates that the log files are no longer " + "needed. ", tableName, potentialLogFilesToDelete.size()); } break; } if (FileNames.isCommitFile(nextFile.getPath())) { // Step 3: If the file is a delta log file, add it to the `potentialFilesToDelete` list // We can't delete these files until we encounter a checkpoint later that indicates // that the log files are no longer needed. potentialLogFilesToDelete.add(nextFile.getPath()); } else if (FileNames.isCheckpointFile(nextFile.getPath())) { // Step 4: If the file is a checkpoint file, add it to the `lastSeenCheckpointFiles` long newLastSeenCheckpointVersion = FileNames.checkpointVersion(nextFile.getPath()); checkArgument( lastSeenCheckpointVersion == -1 || newLastSeenCheckpointVersion >= lastSeenCheckpointVersion); if (lastSeenCheckpointVersion != -1 && newLastSeenCheckpointVersion > lastSeenCheckpointVersion) { // We have found checkpoint file for a new version. This means the files gathered for // the last checkpoint version are not complete (most likely an incomplete multipart // checkpoint). We should delete the files gathered so far and start fresh // last seen checkpoint state logger.info( "[tableName={}] Incomplete checkpoint files found at version {}, ignoring " + "the checkpoint files and adding them to potential log file delete list", tableName, lastSeenCheckpointVersion); potentialLogFilesToDelete.addAll(lastSeenCheckpointFiles); lastSeenCheckpointFiles.clear(); } lastSeenCheckpointFiles.add(nextFile.getPath()); lastSeenCheckpointVersion = newLastSeenCheckpointVersion; } // Ignore non-delta and non-checkpoint files. } } logger.info( "[tableName={}] Deleted {} log files older than {}", tableName, numDeleted, fileCutOffTime); return numDeleted; } private static CloseableIterator listDeltaLogs(Engine engine, Path tablePath) throws IOException { Path logPath = new Path(tablePath, "_delta_log"); // TODO: Currently we don't update the timestamps of files to be monotonically increasing. // In future we can do something similar to Delta Spark to make the timestamps monotonically // increasing. See `BufferingLogDeletionIterator` in Delta Spark. return engine.getFileSystemClient().listFrom(FileNames.listingPrefix(logPath, 0)); } private static int deleteLogFiles(Engine engine, List logFiles) throws IOException { int numDeleted = 0; for (String logFile : logFiles) { if (wrapEngineExceptionThrowsIO( () -> engine.getFileSystemClient().delete(logFile), "Failed to delete the log file as part of the metadata cleanup %s", logFile)) { numDeleted++; } } return numDeleted; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/snapshot/SnapshotManager.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.snapshot; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.lang.String.format; import io.delta.kernel.*; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.InvalidTableException; import io.delta.kernel.exceptions.TableNotFoundException; import io.delta.kernel.internal.*; import io.delta.kernel.internal.annotation.VisibleForTesting; import io.delta.kernel.internal.checkpoints.*; import io.delta.kernel.internal.checksum.CRCInfo; import io.delta.kernel.internal.commit.DefaultFileSystemManagedTableOnlyCommitter; import io.delta.kernel.internal.files.*; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.lang.Lazy; import io.delta.kernel.internal.lang.ListUtils; import io.delta.kernel.internal.metrics.SnapshotQueryContext; import io.delta.kernel.internal.replay.LogReplay; import io.delta.kernel.internal.replay.ProtocolMetadataLogReplay; import io.delta.kernel.internal.table.SnapshotFactory; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.internal.util.FileNames.DeltaLogFileType; import io.delta.kernel.internal.util.Tuple2; import io.delta.kernel.utils.FileStatus; import java.util.*; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class SnapshotManager { private final Path tablePath; private final Path logPath; public SnapshotManager(Path tablePath) { this.tablePath = tablePath; this.logPath = new Path(tablePath, "_delta_log"); } private static final Logger logger = LoggerFactory.getLogger(SnapshotManager.class); ///////////////// // Public APIs // ///////////////// /** * Construct the latest snapshot for given table. * * @param engine Instance of {@link Engine} to use. * @return the latest {@link Snapshot} of the table * @throws TableNotFoundException if the table does not exist * @throws InvalidTableException if the table is in an invalid state */ public SnapshotImpl buildLatestSnapshot(Engine engine, SnapshotQueryContext snapshotContext) throws TableNotFoundException { final LogSegment logSegment = snapshotContext .getSnapshotMetrics() .loadLogSegmentTotalDurationTimer .time(() -> getLogSegmentForVersion(engine, Optional.empty() /* versionToLoad */)); snapshotContext.setResolvedVersion(logSegment.getVersion()); snapshotContext.setCheckpointVersion(logSegment.getCheckpointVersionOpt()); return createSnapshot(logSegment, engine, snapshotContext); } /** * Construct the snapshot for the given table at the version provided. * * @param engine Instance of {@link Engine} to use. * @param version The snapshot version to construct * @return a {@link Snapshot} of the table at version {@code version} * @throws TableNotFoundException if the table does not exist * @throws InvalidTableException if the table is in an invalid state */ public SnapshotImpl getSnapshotAt( Engine engine, long version, SnapshotQueryContext snapshotContext) throws TableNotFoundException { final LogSegment logSegment = snapshotContext .getSnapshotMetrics() .loadLogSegmentTotalDurationTimer .time( () -> getLogSegmentForVersion(engine, Optional.of(version) /* versionToLoadOpt */)); snapshotContext.setCheckpointVersion(logSegment.getCheckpointVersionOpt()); snapshotContext.setResolvedVersion(logSegment.getVersion()); return createSnapshot(logSegment, engine, snapshotContext); } /** * Construct the snapshot for the given table at the provided timestamp. * * @param engine Instance of {@link Engine} to use. * @param millisSinceEpochUTC timestamp to fetch the snapshot for in milliseconds since the unix * epoch * @return a {@link Snapshot} of the table at the provided timestamp * @throws TableNotFoundException if the table does not exist * @throws InvalidTableException if the table is in an invalid state */ public SnapshotImpl getSnapshotForTimestamp( Engine engine, SnapshotImpl latestSnapshot, long millisSinceEpochUTC, SnapshotQueryContext snapshotContext) throws TableNotFoundException { final long versionToLoad = SnapshotFactory.resolveTimestampToSnapshotVersion( engine, snapshotContext, latestSnapshot, millisSinceEpochUTC, Collections.emptyList() /* logDatas */); return getSnapshotAt(engine, versionToLoad, snapshotContext); } //////////////////// // Helper Methods // //////////////////// /** * Verify that a list of delta versions is contiguous. * * @throws InvalidTableException if the versions are not contiguous */ @VisibleForTesting public static void verifyDeltaVersionsContiguous(List versions, Path tablePath) { for (int i = 1; i < versions.size(); i++) { if (versions.get(i) != versions.get(i - 1) + 1) { throw new InvalidTableException( tablePath.toString(), String.format("Missing delta files: versions are not contiguous: (%s)", versions)); } } } private SnapshotImpl createSnapshot( LogSegment initSegment, Engine engine, SnapshotQueryContext snapshotContext) { final Lazy lazyLogSegment = new Lazy<>(() -> initSegment); final Lazy> lazyCrcInfo = SnapshotFactory.createLazyChecksumFileLoaderWithMetrics( engine, lazyLogSegment, snapshotContext.getSnapshotMetrics()); final ProtocolMetadataLogReplay.Result protocolMetadataResult = ProtocolMetadataLogReplay.loadProtocolAndMetadata( engine, tablePath, initSegment, lazyCrcInfo, snapshotContext.getSnapshotMetrics()); // TODO: When LogReplay becomes static utilities, we can create it inside of SnapshotImpl final LogReplay logReplay = new LogReplay(engine, tablePath, lazyLogSegment, lazyCrcInfo); final SnapshotImpl snapshot = new SnapshotImpl( tablePath, initSegment.getVersion(), lazyLogSegment, logReplay, protocolMetadataResult.protocol, protocolMetadataResult.metadata, DefaultFileSystemManagedTableOnlyCommitter.INSTANCE, snapshotContext, Optional.empty() /* inCommitTimestampOpt */); return snapshot; } /** * Generates a {@link LogSegment} for the given `versionToLoadOpt`. If no `versionToLoadOpt` is * provided, generates a {@code LogSegment} for the latest version of the table. * *

This primarily consists of three steps: * *

    *
  1. First, determine the starting checkpoint version that is at or before `versionToLoadOpt`. * If no `versionToLoadOpt` is provided, will use the checkpoint pointed to by the * _last_checkpoint file. *
  2. Second, LIST the _delta_log for all delta and checkpoint files newer than the starting * checkpoint version. *
  3. Third, process and validate this list of _delta_log files to yield a {@code LogSegment}. *
*/ public LogSegment getLogSegmentForVersion(Engine engine, Optional versionToLoadOpt) { return getLogSegmentForVersion( engine, versionToLoadOpt, Collections.emptyList() /* parsedLogDatas */, Optional.empty() /* maxCatalogVersionOpt */); } /** * [delta-io/delta#4765]: Right now, we only support sorted and contiguous ratified commit log * data. * * @param timeTravelVersionOpt the version to time-travel to for a time-travel query * @param parsedLogDatas the parsed log data from the catalog * @param maxCatalogVersionOpt the maximum version ratified by the catalog for catalog managed * tables. Empty for file-system managed tables. */ public LogSegment getLogSegmentForVersion( Engine engine, Optional timeTravelVersionOpt, List parsedLogDatas, Optional maxCatalogVersionOpt) { // This is the actual version we want to load. For "latest" (aka non-time-travel) queries for // catalogManaged tables we want to load the maxCatalogVersion final Optional versionToLoadOpt = timeTravelVersionOpt.isPresent() ? timeTravelVersionOpt : maxCatalogVersionOpt; final long versionToLoad = versionToLoadOpt.orElse(Long.MAX_VALUE); final String versionToLoadStr = versionToLoadOpt.map(String::valueOf).orElse("latest"); logger.info("Loading log segment for version {}", versionToLoadStr); final long logSegmentBuildingStartTimeMillis = System.currentTimeMillis(); /////////////////////////////////////////////////////////////////////////////////////////// // Step 1: Find the latest checkpoint version. If timeTravelVersionOpt is empty, use the // // version referenced by the _LAST_CHECKPOINT file. If timeTravelVersionOpt is // // present, search for the previous latest complete checkpoint at or before the // // version to load // /////////////////////////////////////////////////////////////////////////////////////////// final Optional startCheckpointVersionOpt = getStartCheckpointVersion(engine, timeTravelVersionOpt, maxCatalogVersionOpt); ///////////////////////////////////////////////////////////////// // Step 2: Determine the actual version to start listing from. // ///////////////////////////////////////////////////////////////// final long listFromStartVersion = startCheckpointVersionOpt .map( version -> { logger.info("Found a complete checkpoint at version {}.", version); return version; }) .orElseGet( () -> { logger.warn("Cannot find a complete checkpoint. Listing from version 0."); return 0L; }); ///////////////////////////////////////////////////////////////// // Step 3: List the files from $startVersion to $versionToLoad // ///////////////////////////////////////////////////////////////// Set fileTypes = new HashSet<>( Arrays.asList( DeltaLogFileType.COMMIT, DeltaLogFileType.CHECKPOINT, DeltaLogFileType.CHECKSUM, DeltaLogFileType.LOG_COMPACTION)); final long listingStartTimeMillis = System.currentTimeMillis(); final List listedFileStatuses = DeltaLogActionUtils.listDeltaLogFilesAsIter( engine, fileTypes, tablePath, listFromStartVersion, versionToLoadOpt, true /* mustBeRecreatable */) .toInMemoryList(); logger.info( "{}: Took {}ms to list the files after starting checkpoint", tablePath, System.currentTimeMillis() - listingStartTimeMillis); //////////////////////////////////////////////////////////////////////// // Step 4: Perform some basic validations on the listed file statuses // //////////////////////////////////////////////////////////////////////// if (listedFileStatuses.isEmpty()) { if (startCheckpointVersionOpt.isPresent()) { // We either (a) determined this checkpoint version from the _LAST_CHECKPOINT file, or (b) // found the last complete checkpoint before our versionToLoad. In either case, we didn't // see the checkpoint file in the listing. // TODO: throw a more specific error based on case (a) or (b) throw DeltaErrors.missingCheckpoint(tablePath.toString(), startCheckpointVersionOpt.get()); } else { // Either no files found OR no *delta* files found even when listing from 0. This means that // the delta table does not exist yet. throw new TableNotFoundException( tablePath.toString(), format("No delta files found in the directory: %s", logPath)); } } logDebugFileStatuses("listedFileStatuses", listedFileStatuses); ////////////////////////////////////////////////////////////////////////////////////////// // Step 5: Partition $listedFileStatuses into the checkpoints, deltas, and compactions. // ////////////////////////////////////////////////////////////////////////////////////////// final Map, List> partitionedFiles = listedFileStatuses.stream() .map(ParsedLogData::forFileStatus) .collect( Collectors.groupingBy( ParsedLogData::getGroupByCategoryClass, LinkedHashMap::new, // Ensure order is maintained Collectors.toList())); final List allPublishedDeltas = partitionedFiles.getOrDefault(ParsedPublishedDeltaData.class, Collections.emptyList()) .stream() .map(ParsedPublishedDeltaData.class::cast) .collect(Collectors.toList()); final List listedCheckpointFileStatuses = partitionedFiles.getOrDefault(ParsedCheckpointData.class, Collections.emptyList()).stream() .map(ParsedLogData::getFileStatus) .collect(Collectors.toList()); final List listedCompactionFileStatuses = partitionedFiles.getOrDefault(ParsedLogCompactionData.class, Collections.emptyList()) .stream() .map(ParsedLogData::getFileStatus) .collect(Collectors.toList()); final List listedChecksumFileStatuses = partitionedFiles.getOrDefault(ParsedChecksumData.class, Collections.emptyList()).stream() .map(ParsedLogData::getFileStatus) .collect(Collectors.toList()); logDebugParsedLogDatas("allPublishedDeltas", allPublishedDeltas); logDebugFileStatuses("listedCheckpointFileStatuses", listedCheckpointFileStatuses); logDebugFileStatuses("listedCompactionFileStatuses", listedCompactionFileStatuses); logDebugFileStatuses("listedCheckSumFileStatuses", listedChecksumFileStatuses); ///////////////////////////////////////////////////////////////////////////////////////////// // Step 6: Determine the latest complete checkpoint version. The intuition here is that we // // LISTed from the startingCheckpoint but may have found a newer complete // // checkpoint. // ///////////////////////////////////////////////////////////////////////////////////////////// final List listedCheckpointInstances = listedCheckpointFileStatuses.stream() .map(f -> new CheckpointInstance(f.getPath())) .collect(Collectors.toList()); final CheckpointInstance notLaterThanCheckpoint = versionToLoadOpt.map(CheckpointInstance::new).orElse(CheckpointInstance.MAX_VALUE); final Optional latestCompleteCheckpointOpt = Checkpointer.getLatestCompleteCheckpointFromList( listedCheckpointInstances, notLaterThanCheckpoint); if (!latestCompleteCheckpointOpt.isPresent() && startCheckpointVersionOpt.isPresent()) { // In Step 1 we found a $startCheckpointVersion but now our LIST of the file system doesn't // see it. This means that the checkpoint we thought should exist no longer does. throw DeltaErrors.missingCheckpoint(tablePath.toString(), startCheckpointVersionOpt.get()); } final long latestCompleteCheckpointVersion = latestCompleteCheckpointOpt.map(x -> x.version).orElse(-1L); logger.info("Latest complete checkpoint version: {}", latestCompleteCheckpointVersion); ///////////////////////////////////////////////////////////////////////////////////////////// // Step 7: Grab all deltas in range [$latestCompleteCheckpointVersion + 1, $versionToLoad] // ///////////////////////////////////////////////////////////////////////////////////////////// final List allDeltasAfterCheckpoint = getAllDeltasAfterCheckpointWithCatalogPriority( allPublishedDeltas, parsedLogDatas, latestCompleteCheckpointVersion, versionToLoad); logDebugParsedLogDatas("allDeltasAfterCheckpoint", allDeltasAfterCheckpoint); ////////////////////////////////////////////////////////////////////////////////// // Step 8: Grab all compactions in range [$latestCompleteCheckpointVersion + 1, // // $versionToLoad] // ////////////////////////////////////////////////////////////////////////////////// final List compactionsAfterCheckpoint = listedCompactionFileStatuses.stream() .filter( fs -> { final Tuple2 compactionVersions = FileNames.logCompactionVersions(new Path(fs.getPath())); return latestCompleteCheckpointVersion + 1 <= compactionVersions._1 && compactionVersions._2 <= versionToLoad; }) .collect(Collectors.toList()); logDebugFileStatuses("compactionsAfterCheckpoint", compactionsAfterCheckpoint); //////////////////////////////////////////////////////////////////// // Step 9: Determine the version of the snapshot we can now load. // //////////////////////////////////////////////////////////////////// final long newVersion = allDeltasAfterCheckpoint.isEmpty() ? latestCompleteCheckpointVersion : ListUtils.getLast(allDeltasAfterCheckpoint).getVersion(); logger.info("New version to load: {}", newVersion); ///////////////////////////////////////////// // Step 10: Perform some basic validations. // ///////////////////////////////////////////// // Check that we have found at least one checkpoint or delta file if (!latestCompleteCheckpointOpt.isPresent() && allDeltasAfterCheckpoint.isEmpty()) { throw new InvalidTableException( tablePath.toString(), "No complete checkpoint found and no delta files found"); } final Optional deltaAtCheckpointVersionOpt = allPublishedDeltas.stream() .filter(x -> x.getVersion() == latestCompleteCheckpointVersion) .findFirst(); // Check that, for a checkpoint at version N, there's a delta file at N, too. if (latestCompleteCheckpointOpt.isPresent() && !deltaAtCheckpointVersionOpt.isPresent()) { throw new InvalidTableException( tablePath.toString(), String.format("Missing delta file for version %s", latestCompleteCheckpointVersion)); } // Check that the $newVersion we actually loaded is the desired $versionToLoad if (versionToLoadOpt.isPresent()) { if (newVersion < versionToLoad) { throw DeltaErrors.versionToLoadAfterLatestCommit( tablePath.toString(), versionToLoad, newVersion); } else if (newVersion > versionToLoad) { throw new IllegalStateException( String.format( "%s: Expected to load version %s but actually loaded version %s", tablePath, versionToLoad, newVersion)); } } if (!allDeltasAfterCheckpoint.isEmpty()) { // Check that the delta versions are contiguous verifyDeltaVersionsContiguous( // TODO: refactor `verifyDeltaVersionsContiguous` to operate on ParsedLogData so we can // avoid making an entirely new list here allDeltasAfterCheckpoint.stream().map(x -> x.getVersion()).collect(Collectors.toList()), tablePath); // Check that the delta versions start with $latestCompleteCheckpointVersion + 1. If they // don't, then we have a gap in between the checkpoint and the first delta file. if (allDeltasAfterCheckpoint.get(0).getVersion() != latestCompleteCheckpointVersion + 1) { throw new InvalidTableException( tablePath.toString(), String.format( "Cannot compute snapshot. Missing delta file version %d.", latestCompleteCheckpointVersion + 1)); } // Note: We have already asserted above that $versionToLoad equals $newVersion. // Note: We already know that the last element of deltasAfterCheckpoint is $newVersion IF // $deltasAfterCheckpoint is not empty. logger.info( "Verified delta files are contiguous from version {} to {}", latestCompleteCheckpointVersion + 1, newVersion); } //////////////////////////////////////////////////////////////////////////////////////////// // Step 11: Grab the actual checkpoint file statuses for latestCompleteCheckpointVersion. // //////////////////////////////////////////////////////////////////////////////////////////// final List latestCompleteCheckpointFileStatuses = latestCompleteCheckpointOpt .map( latestCompleteCheckpoint -> { final Set newCheckpointPaths = new HashSet<>(latestCompleteCheckpoint.getCorrespondingFiles(logPath)); final List newCheckpointFileStatuses = listedCheckpointFileStatuses.stream() .filter(f -> newCheckpointPaths.contains(new Path(f.getPath()))) .collect(Collectors.toList()); logDebugFileStatuses("newCheckpointFileStatuses", newCheckpointFileStatuses); if (newCheckpointFileStatuses.size() != newCheckpointPaths.size()) { final String msg = format( "Seems like the checkpoint is corrupted. Failed in getting the file " + "information for:\n%s\namong\n%s", newCheckpointPaths.stream() .map(Path::toString) .collect(Collectors.joining("\n - ")), listedCheckpointFileStatuses.stream() .map(FileStatus::getPath) .collect(Collectors.joining("\n - "))); throw new IllegalStateException(msg); } return newCheckpointFileStatuses; }) .orElse(Collections.emptyList()); //////////////////////////////////////////////////////// // Step 12: Calculate the remaining LogSegment params // //////////////////////////////////////////////////////// // If our LogSegment has deltas (allDeltasAfterCheckpoint), we use the last delta. // Else, our LogSegment only has a checkpoint, and we have checked above that if there's a // checkpoint then the `deltaAtCheckpointVersionOpt` exists. final FileStatus deltaAtEndVersion = allDeltasAfterCheckpoint.isEmpty() ? deltaAtCheckpointVersionOpt.get().getFileStatus() : ListUtils.getLast(allDeltasAfterCheckpoint).getFileStatus(); final Optional maxPublishedDeltaVersion = allPublishedDeltas.stream().map(ParsedPublishedDeltaData::getVersion).max(Long::compareTo); Optional lastSeenChecksumFile = Optional.empty(); if (!listedChecksumFileStatuses.isEmpty()) { FileStatus latestChecksum = ListUtils.getLast(listedChecksumFileStatuses); long checksumVersion = FileNames.checksumVersion(new Path(latestChecksum.getPath())); if (checksumVersion >= latestCompleteCheckpointVersion) { lastSeenChecksumFile = Optional.of(latestChecksum); } } /////////////////////////////////////////////////// // Step 13: Construct the LogSegment and return. // /////////////////////////////////////////////////// logger.info( "Successfully constructed LogSegment at version {}, took {}ms", newVersion, System.currentTimeMillis() - logSegmentBuildingStartTimeMillis); return new LogSegment( logPath, newVersion, allDeltasAfterCheckpoint.stream() .map(ParsedLogData::getFileStatus) .collect(Collectors.toList()), compactionsAfterCheckpoint, latestCompleteCheckpointFileStatuses, deltaAtEndVersion, lastSeenChecksumFile, maxPublishedDeltaVersion); } ///////////////////////// // getLogSegment utils // ///////////////////////// /** * Filters and concats (a) a list of published Deltas (from cloud LIST call), and (b) a list of * {@link ParsedLogData} injected by the {@link TableManager}, to return a new list of all Deltas * since the latest complete checkpoint, up to and including the target version to load. * *
    *
  • Assumes that {@code allPublishedDeltas} is sorted and contiguous. *
  • Assumes that {@code parsedLogDatas} is sorted and contiguous. *
  • [delta-io/delta#4765] For now, only accepts parsedLogData of type {@link * ParsedCatalogCommitData} (written to file). *
  • If there is both a published Delta and a ratified staged commit for the same version, * prioritizes the ratified staged commit *
*/ private List getAllDeltasAfterCheckpointWithCatalogPriority( List allPublishedDeltas, List parsedLogDatas, long latestCompleteCheckpointVersion, long versionToLoad) { final List allPublishedDeltasAfterCheckpoint = allPublishedDeltas.stream() .filter(ParsedLogData::isFile) .filter( x -> latestCompleteCheckpointVersion < x.getVersion() && x.getVersion() <= versionToLoad) .collect(Collectors.toList()); if (parsedLogDatas.isEmpty()) { return allPublishedDeltasAfterCheckpoint; } final List allRatifiedCommitsAfterCheckpoint = parsedLogDatas.stream() .filter(x -> x instanceof ParsedCatalogCommitData && x.isFile()) .filter( x -> latestCompleteCheckpointVersion < x.getVersion() && x.getVersion() <= versionToLoad) .map(ParsedCatalogCommitData.class::cast) .collect(Collectors.toList()); return LogDataUtils.combinePublishedAndRatifiedDeltasWithCatalogPriority( allPublishedDeltasAfterCheckpoint, allRatifiedCommitsAfterCheckpoint); } /** * Determine the starting checkpoint version that is at or before the version to load. * *

Version to load: For time-travel queries, this is the time-travel version. For latest * queries on catalog maanged tables, this is the max ratified catalog version. For latest queries * on file-system managed tables, this is the latest available version on the file-system. * *

For non-time travel queries we will use the checkpoint pointed to by the _last_checkpoint * file (except for when it is after the maxRatifiedCatalogVersion, in which case we will search * backwards for a checkpoint). */ private Optional getStartCheckpointVersion( Engine engine, Optional timeTravelVersionOpt, Optional maxCatalogVersionOpt) { // This is a "latest" query, let's try to use the _last_checkpoint file if possible if (!timeTravelVersionOpt.isPresent()) { logger.info("Reading the _last_checkpoint file for 'latest' query"); Optional lastCheckpointFileVersionOpt = new Checkpointer(logPath).readLastCheckpointFile(engine).map(x -> x.version); if (!lastCheckpointFileVersionOpt.isPresent()) { logger.info("No _last_checkpoint file found, default to listing from 0"); return Optional.empty(); } long lastCheckpointFileVersion = lastCheckpointFileVersionOpt.get(); if (!maxCatalogVersionOpt.isPresent()) { // If there is no maxCatalogVersion we don't have to do anything special --> just return return Optional.of(lastCheckpointFileVersion); } else { // When there is a maxCatalogVersion we only want to return the version from the // _last_checkpoint file if it is less than or equal to the maxCatalogVersion. Otherwise, // we should revert to listing backwards from the version to load. // This situation is possible due to race conditions. Since fetching the maxCatalogVersion // from the catalog, it is possible that a concurrent writer has committed, published // and checkpointed before this listing code is executed. Thus, it is possible that the // _last_checkpoint file points to a checkpoint later than the maxCatalogVersion. if (lastCheckpointFileVersion <= maxCatalogVersionOpt.get()) { return Optional.of(lastCheckpointFileVersion); } logger.info( "Found checkpoint at version {} in _last_checkpoint file but cannot be used because " + "maxCatalogVersion = {}.", lastCheckpointFileVersion, maxCatalogVersionOpt.get()); } } long versionToLoad = timeTravelVersionOpt.orElseGet( () -> maxCatalogVersionOpt.orElseThrow( () -> new IllegalStateException( "Impossible state: If timeTravelToVersionOpt and maxCatalogVersion " + "is empty we should always have returned earlier"))); logger.info("Finding last complete checkpoint at or before version {}", versionToLoad); final long startTimeMillis = System.currentTimeMillis(); return Checkpointer.findLastCompleteCheckpointBefore(engine, logPath, versionToLoad + 1) .map(checkpointInstance -> checkpointInstance.version) .map( checkpointVersion -> { checkArgument( checkpointVersion <= versionToLoad, "Last complete checkpoint version %s was not <= targetVersion %s", checkpointVersion, versionToLoad); logger.info( "{}: Took {}ms to find last complete checkpoint <= targetVersion {}", tablePath, System.currentTimeMillis() - startTimeMillis, versionToLoad); return checkpointVersion; }); } private void logDebugFileStatuses(String varName, List fileStatuses) { if (logger.isDebugEnabled()) { logger.debug( "{}: {}", varName, Arrays.toString( fileStatuses.stream().map(x -> new Path(x.getPath()).getName()).toArray())); } } private void logDebugParsedLogDatas(String varName, List logDatas) { if (logger.isDebugEnabled()) { logger.debug( "{}:\n {}", varName, logDatas.stream().map(Object::toString).collect(Collectors.joining("\n "))); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/stats/FileSizeHistogram.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.stats; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.Row; import io.delta.kernel.internal.annotation.VisibleForTesting; import io.delta.kernel.internal.data.GenericRow; import io.delta.kernel.internal.util.InternalUtils; import io.delta.kernel.internal.util.VectorUtils; import io.delta.kernel.metrics.FileSizeHistogramResult; import io.delta.kernel.types.ArrayType; import io.delta.kernel.types.LongType; import io.delta.kernel.types.StructType; import java.util.*; import java.util.stream.Collectors; /** A histogram that tracks file size distributions and their counts. */ public class FileSizeHistogram { ////////////////////////////////// // Static variables and methods // ////////////////////////////////// private static final long KB = 1024; private static final long MB = KB * 1024; private static final long GB = MB * 1024; public static final StructType FULL_SCHEMA = new StructType() .add("sortedBinBoundaries", new ArrayType(LongType.LONG, false)) .add("fileCounts", new ArrayType(LongType.LONG, false)) .add("totalBytes", new ArrayType(LongType.LONG, false)); /** Creates a default FileSizeHistogram with predefined bin boundaries and zero counts. */ public static FileSizeHistogram createDefaultHistogram() { long[] defaultBoundaries = createDefaultBinBoundaries(); long[] zeroCounts = new long[defaultBoundaries.length]; long[] zeroBytes = new long[defaultBoundaries.length]; return new FileSizeHistogram(defaultBoundaries, zeroCounts, zeroBytes); } /** Creates a FileSizeHistogram from a column vector. */ public static Optional fromColumnVector(ColumnVector vector, int rowId) { if (vector.isNullAt(rowId)) { return Optional.empty(); } int boundariesIdx = FULL_SCHEMA.indexOf("sortedBinBoundaries"); int totalBytesIdx = FULL_SCHEMA.indexOf("totalBytes"); int fileCountsIdx = FULL_SCHEMA.indexOf("fileCounts"); List boundariesList = VectorUtils.toJavaList( InternalUtils.requireNonNull( vector.getChild(boundariesIdx), rowId, "sortedBinBoundaries") .getArray(rowId)); List totalBytesList = VectorUtils.toJavaList( InternalUtils.requireNonNull(vector.getChild(totalBytesIdx), rowId, "totalBytes") .getArray(rowId)); List fileCountsList = VectorUtils.toJavaList( InternalUtils.requireNonNull(vector.getChild(fileCountsIdx), rowId, "fileCounts") .getArray(rowId)); long[] boundaries = boundariesList.stream().mapToLong(Long::longValue).toArray(); long[] totalBytesArray = totalBytesList.stream().mapToLong(Long::longValue).toArray(); long[] fileCountsArray = fileCountsList.stream().mapToLong(Long::longValue).toArray(); return Optional.of(new FileSizeHistogram(boundaries, fileCountsArray, totalBytesArray)); } /** * Creates the default bin boundaries for file size categorization. * *

    *
  • Starts with 0 and powers of 2 from 8KB to 4MB *
  • 4MB jumps from 8MB to 40MB *
  • 8MB jumps from 48MB to 120MB *
  • 4MB jumps from 124MB to 144MB *
  • 16MB jumps from 160MB to 576MB *
  • 64MB jumps from 640MB to 1408MB *
  • 128MB jumps from 1536MB to 2GB *
  • 256MB jumps from 2304MB to 4GB *
  • Powers of 2 from 8GB to 256GB *
* * @return An array of bin boundaries sorted in ascending order */ private static long[] createDefaultBinBoundaries() { // Pre-calculate the size to avoid resizing int totalSize = 95; // Known size from all the boundaries long[] boundaries = new long[totalSize]; int idx = 0; // 0 and powers of 2 till 4 MB boundaries[idx++] = 0L; for (long size = 8 * KB; size <= 4 * MB; size *= 2) { boundaries[idx++] = size; } // 4 MB jumps till 40 MB for (long size = 8 * MB; size <= 40 * MB; size += 4 * MB) { boundaries[idx++] = size; } // 8 MB jumps till 120 MB for (long size = 48 * MB; size <= 120 * MB; size += 8 * MB) { boundaries[idx++] = size; } // 4 MB jumps till 144 MB for (long size = 124 * MB; size <= 144 * MB; size += 4 * MB) { boundaries[idx++] = size; } // 16 MB jumps till 576 MB for (long size = 160 * MB; size <= 576 * MB; size += 16 * MB) { boundaries[idx++] = size; } // 64 MB jumps till 1408 MB for (long size = 640 * MB; size <= 1408 * MB; size += 64 * MB) { boundaries[idx++] = size; } // 128 MB jumps till 2 GB for (long size = 1536 * MB; size <= 2048 * MB; size += 128 * MB) { boundaries[idx++] = size; } // 256 MB jumps till 4 GB for (long size = 2304 * MB; size <= 4096 * MB; size += 256 * MB) { boundaries[idx++] = size; } // Power of 2 till 256 GB for (long size = 8 * GB; size <= 256 * GB; size *= 2) { boundaries[idx++] = size; } checkArgument( idx == totalSize, "Incorrect pre-calculated size. Expected %s but got %s", totalSize, idx); return boundaries; } public static FileSizeHistogram fromFileSizeHistogramResult( FileSizeHistogramResult fileSizeHistogramResult) { requireNonNull(fileSizeHistogramResult); return new FileSizeHistogram( fileSizeHistogramResult.getSortedBinBoundaries(), fileSizeHistogramResult.getFileCounts(), fileSizeHistogramResult.getTotalBytes()); } //////////////////////////////////// // Member variables and methods // //////////////////////////////////// private final long[] sortedBinBoundaries; private final long[] fileCounts; private final long[] totalBytes; @VisibleForTesting public FileSizeHistogram(long[] sortedBinBoundaries, long[] fileCounts, long[] totalBytes) { requireNonNull(sortedBinBoundaries, "sortedBinBoundaries cannot be null"); requireNonNull(fileCounts, "fileCounts cannot be null"); requireNonNull(totalBytes, "totalBytes cannot be null"); checkArgument( sortedBinBoundaries.length >= 2, "sortedBinBoundaries must have at least 2 elements to define a range"); checkArgument( sortedBinBoundaries[0] == 0, "First boundary must be 0, got %s", sortedBinBoundaries[0]); checkArgument( sortedBinBoundaries.length == fileCounts.length && sortedBinBoundaries.length == totalBytes.length, "All arrays must have the same length"); this.sortedBinBoundaries = sortedBinBoundaries; this.fileCounts = fileCounts; this.totalBytes = totalBytes; } /** * Adds a file size to the histogram, incrementing the appropriate bin's count and total bytes. * The appropriate bin refers to a bin with boundary that is less than or equal to the file size. * Files larger than the maximum bin boundary (256 GB) are placed in the last bin. * * @param fileSize The size of the file in bytes * @throws IllegalArgumentException if fileSize is negative or if getBinIndex returns an invalid * index */ public void insert(long fileSize) { checkArgument(fileSize >= 0, "File size must be non-negative, got %s", fileSize); int index = getBinIndex(fileSize); checkArgument( index >= 0, "getBinIndex must return non-negative index for non-negative fileSize, got %s", index); fileCounts[index]++; totalBytes[index] += fileSize; } /** * Removes a file size from the histogram, decrementing the appropriate bin's count and total * bytes. * * @param fileSize The size of the file in bytes * @throws IllegalArgumentException if fileSize is negative */ public void remove(long fileSize) { checkArgument(fileSize >= 0, "File size must be non-negative, got %s", fileSize); int index = getBinIndex(fileSize); checkArgument( index >= 0, "getBinIndex must return non-negative index for non-negative fileSize, got %s", index); checkArgument( totalBytes[index] >= fileSize && fileCounts[index] > 0, "Cannot remove %s bytes from bin %d which only has %s bytes or does not have any files", fileSize, index, totalBytes[index]); fileCounts[index]--; totalBytes[index] -= fileSize; } private int getBinIndex(long fileSize) { int index = Arrays.binarySearch(sortedBinBoundaries, fileSize); // When fileSize is not found in the array, binarySearch returns -(insertion_point) - 1 // We need to get the bin that comes before the insertion point, which is (insertion_point - 1) return index >= 0 ? index : -(index + 1) - 1; } /** Encode as a {@link Row} object with the schema {@link FileSizeHistogram#FULL_SCHEMA}. */ public Row toRow() { Map value = new HashMap<>(); value.put( FULL_SCHEMA.indexOf("sortedBinBoundaries"), VectorUtils.buildArrayValue( Arrays.stream(sortedBinBoundaries).boxed().collect(Collectors.toList()), LongType.LONG)); value.put( FULL_SCHEMA.indexOf("fileCounts"), VectorUtils.buildArrayValue( Arrays.stream(fileCounts).boxed().collect(Collectors.toList()), LongType.LONG)); value.put( FULL_SCHEMA.indexOf("totalBytes"), VectorUtils.buildArrayValue( Arrays.stream(totalBytes).boxed().collect(Collectors.toList()), LongType.LONG)); return new GenericRow(FULL_SCHEMA, value); } public FileSizeHistogramResult captureFileSizeHistogramResult() { return new FileSizeHistogramResult() { final long[] copiedSortedBinBoundaries = Arrays.copyOf(sortedBinBoundaries, sortedBinBoundaries.length); final long[] copiedFileCounts = Arrays.copyOf(fileCounts, fileCounts.length); final long[] copiedTotalBytes = Arrays.copyOf(totalBytes, totalBytes.length); @Override public long[] getSortedBinBoundaries() { return copiedSortedBinBoundaries; } @Override public long[] getFileCounts() { return copiedFileCounts; } @Override public long[] getTotalBytes() { return copiedTotalBytes; } }; } /** * Adds two histograms together by combining their counts and total bytes. Both histograms must * have the same bin boundaries. * * @param other The histogram to add to this histogram * @return A new histogram with combined statistics * @throws IllegalArgumentException if the histograms have different bin boundaries */ public FileSizeHistogram plus(FileSizeHistogram other) { requireNonNull(other, "histogram to add cannot be null"); checkArgument( Arrays.equals(this.sortedBinBoundaries, other.sortedBinBoundaries), "Cannot add histograms with different bin boundaries"); int length = this.sortedBinBoundaries.length; long[] combinedCounts = new long[length]; long[] combinedBytes = new long[length]; for (int i = 0; i < length; i++) { combinedCounts[i] = this.fileCounts[i] + other.fileCounts[i]; combinedBytes[i] = this.totalBytes[i] + other.totalBytes[i]; } return new FileSizeHistogram( Arrays.copyOf(this.sortedBinBoundaries, length), combinedCounts, combinedBytes); } /** * Subtracts another histogram from this one. Both histograms must have the same bin boundaries. * The result will ensure no negative counts or bytes. * * @param other The histogram to subtract from this histogram * @return A new histogram with the difference in statistics * @throws IllegalArgumentException if the histograms have different bin boundaries * @throws IllegalArgumentException if subtraction would result in negative counts or bytes */ public FileSizeHistogram minus(FileSizeHistogram other) { requireNonNull(other, "histogram to minus cannot be null"); checkArgument( Arrays.equals(this.sortedBinBoundaries, other.sortedBinBoundaries), "Cannot subtract histograms with different bin boundaries"); int length = this.sortedBinBoundaries.length; long[] resultCounts = new long[length]; long[] resultBytes = new long[length]; for (int i = 0; i < length; i++) { resultCounts[i] = this.fileCounts[i] - other.fileCounts[i]; resultBytes[i] = this.totalBytes[i] - other.totalBytes[i]; checkArgument( resultCounts[i] >= 0 && resultBytes[i] >= 0, "Subtraction would result in negative counts or bytes at bin %d", i); } return new FileSizeHistogram( Arrays.copyOf(this.sortedBinBoundaries, length), resultCounts, resultBytes); } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; FileSizeHistogram that = (FileSizeHistogram) o; return Arrays.equals(sortedBinBoundaries, that.sortedBinBoundaries) && Arrays.equals(fileCounts, that.fileCounts) && Arrays.equals(totalBytes, that.totalBytes); } @Override public int hashCode() { return Objects.hash( Arrays.hashCode(sortedBinBoundaries), Arrays.hashCode(fileCounts), Arrays.hashCode(totalBytes)); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/table/SnapshotBuilderImpl.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.table; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.Snapshot; import io.delta.kernel.SnapshotBuilder; import io.delta.kernel.commit.Committer; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.DeltaErrors; import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.files.LogDataUtils; import io.delta.kernel.internal.files.ParsedLogData; import io.delta.kernel.internal.lang.ListUtils; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.internal.util.Tuple2; import java.util.Collections; import java.util.List; import java.util.Optional; /** * An implementation of {@link SnapshotBuilder}. * *

Note: The primary responsibility of this class is to take input, validate that input, and then * pass the input to the {@link SnapshotFactory}, which is then responsible for actually creating * the {@link Snapshot} instance. */ public class SnapshotBuilderImpl implements SnapshotBuilder { public static class Context { public final String unresolvedPath; public Optional versionOpt = Optional.empty(); public Optional> timestampQueryContextOpt = Optional.empty(); public Optional committerOpt = Optional.empty(); public List logDatas = Collections.emptyList(); public Optional> protocolAndMetadataOpt = Optional.empty(); public Optional maxCatalogVersion = Optional.empty(); public Context(String unresolvedPath) { this.unresolvedPath = requireNonNull(unresolvedPath, "unresolvedPath is null"); } } private final Context ctx; public SnapshotBuilderImpl(String unresolvedPath) { ctx = new Context(unresolvedPath); } //////////////////////////////////// // Public SnapshotBuilder Methods // //////////////////////////////////// @Override public SnapshotBuilderImpl atVersion(long version) { checkArgument(version >= 0, "version must be >= 0"); ctx.versionOpt = Optional.of(version); return this; } @Override public SnapshotBuilderImpl atTimestamp(long millisSinceEpochUTC, Snapshot latestSnapshot) { requireNonNull(latestSnapshot, "latestSnapshot is null"); checkArgument(latestSnapshot instanceof SnapshotImpl, "latestSnapshot must be a SnapshotImpl"); ctx.timestampQueryContextOpt = Optional.of(new Tuple2<>((SnapshotImpl) latestSnapshot, millisSinceEpochUTC)); return this; } @Override public SnapshotBuilderImpl withCommitter(Committer committer) { ctx.committerOpt = Optional.of(requireNonNull(committer, "committer is null")); return this; } @Override public SnapshotBuilderImpl withLogData(List logDatas) { ctx.logDatas = requireNonNull(logDatas, "logDatas is null"); return this; } @Override public SnapshotBuilderImpl withProtocolAndMetadata(Protocol protocol, Metadata metadata) { ctx.protocolAndMetadataOpt = Optional.of( new Tuple2<>( requireNonNull(protocol, "protocol is null"), requireNonNull(metadata, "metadata is null"))); return this; } @Override public SnapshotBuilderImpl withMaxCatalogVersion(long version) { checkArgument(version >= 0, "A valid version must be >= 0"); ctx.maxCatalogVersion = Optional.of(version); return this; } @Override public SnapshotImpl build(Engine engine) { validateInputOnBuild(engine); return new SnapshotFactory(engine, ctx).create(engine); } //////////////////////////// // Private Helper Methods // //////////////////////////// private void validateInputOnBuild(Engine engine) { validateTimestampNotGreaterThanLatestSnapshot(engine); validateVersionAndTimestampMutuallyExclusive(); validateProtocolAndMetadataOnlyIfVersionProvided(); validateProtocolRead(); // TODO: delta-io/delta#4765 support other types LogDataUtils.validateLogDataContainsOnlyRatifiedStagedCommits(ctx.logDatas); LogDataUtils.validateLogDataIsSortedContiguous(ctx.logDatas); validateMaxCatalogVersionCompatibleWithTimeTravelParams(); validateLogTailEndsWithMaxCatalogVersionOrVersionToLoad(); } /** * Recall the semantics of time-travel by timestamp: "If the provided timestamp is after (strictly * greater than) the timestamp of the latest version of the table, snapshot resolution will fail." */ private void validateTimestampNotGreaterThanLatestSnapshot(Engine engine) { ctx.timestampQueryContextOpt.ifPresent( x -> { final long latestSnapshotVersion = x._1.getVersion(); final long latestSnapshotTimestamp = x._1.getTimestamp(engine); final long requestedTimestamp = x._2; if (requestedTimestamp > latestSnapshotTimestamp) { throw DeltaErrors.timestampAfterLatestCommit( ctx.unresolvedPath, requestedTimestamp, latestSnapshotTimestamp, latestSnapshotVersion); } }); } private void validateVersionAndTimestampMutuallyExclusive() { checkArgument( !ctx.timestampQueryContextOpt.isPresent() || !ctx.versionOpt.isPresent(), "timestamp and version cannot be provided together"); } private void validateProtocolAndMetadataOnlyIfVersionProvided() { checkArgument( ctx.versionOpt.isPresent() || !ctx.protocolAndMetadataOpt.isPresent(), "protocol and metadata can only be provided if a version is provided"); } private void validateProtocolRead() { ctx.protocolAndMetadataOpt.ifPresent( x -> TableFeatures.validateKernelCanReadTheTable(x._1, ctx.unresolvedPath)); } /** * For catalog managed tables we cannot time-travel to a version after the max catalog version. We * also require that the latestSnapshot provided for timestamp-based queries has the max catalog * version. */ private void validateMaxCatalogVersionCompatibleWithTimeTravelParams() { ctx.maxCatalogVersion.ifPresent( maxVersion -> { ctx.versionOpt.ifPresent( version -> checkArgument( version <= maxVersion, String.format( "Cannot time-travel to version %s after the max catalog version %s", version, maxVersion))); ctx.timestampQueryContextOpt.ifPresent( queryContext -> checkArgument( queryContext._1.getVersion() == maxVersion, "The latestSnapshot provided for timestamp-based time-travel queries " + "must have version = maxCatalogVersion")); }); } /** * When a catalog implementation has provided catalog commits we require that they provide up to * and including the version that we will load (which for a latest query is the max catalog * version, and for a time-travel-by-version query is the version to load). This is to validate * that the catalog has queried and provided sufficient catalog commits to correctly read the * table. */ private void validateLogTailEndsWithMaxCatalogVersionOrVersionToLoad() { ctx.maxCatalogVersion.ifPresent( maxVersion -> { if (!ctx.logDatas.isEmpty()) { ParsedLogData tailLogData = ListUtils.getLast(ctx.logDatas); if (ctx.versionOpt.isPresent()) { checkArgument( tailLogData.getVersion() >= ctx.versionOpt.get(), "Provided catalog commits must include versionToLoad for time-travel queries"); } else { checkArgument( maxVersion == tailLogData.getVersion(), "Provided catalog commits must end with max catalog version"); } } }); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/table/SnapshotFactory.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.table; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static io.delta.kernel.internal.util.Utils.resolvePath; import io.delta.kernel.Snapshot; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.DeltaHistoryManager; import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.checksum.CRCInfo; import io.delta.kernel.internal.checksum.ChecksumReader; import io.delta.kernel.internal.commit.DefaultFileSystemManagedTableOnlyCommitter; import io.delta.kernel.internal.files.ParsedCatalogCommitData; import io.delta.kernel.internal.files.ParsedLogData; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.lang.Lazy; import io.delta.kernel.internal.metrics.SnapshotMetrics; import io.delta.kernel.internal.metrics.SnapshotQueryContext; import io.delta.kernel.internal.replay.LogReplay; import io.delta.kernel.internal.replay.ProtocolMetadataLogReplay; import io.delta.kernel.internal.snapshot.LogSegment; import io.delta.kernel.internal.snapshot.SnapshotManager; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.utils.FileStatus; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Factory class responsible for creating {@link Snapshot} instances. * *

This factory takes validated parameters from {@link SnapshotBuilderImpl} and orchestrates the * actual snapshot creation process. It handles path resolution, log segment loading, and * coordinates with various internal components to construct a fully initialized {@link Snapshot}. * *

Note: The {@link SnapshotBuilderImpl} is responsible for receiving and validating all builder * parameters, and then passing that information to this factory to actually create the {@link * Snapshot}. */ public class SnapshotFactory { ////////////////////////////////////////// // Static utility methods and variables // ////////////////////////////////////////// /** * Resolves the latest table version that exists at or before the given {@code * millisSinceEpochUTC}. * *

Updates the given {@code snapshotQueryCtx} with the resolved version and prints out useful * log statements. */ public static long resolveTimestampToSnapshotVersion( Engine engine, SnapshotQueryContext snapshotQueryCtx, SnapshotImpl latestSnapshot, long millisSinceEpochUTC, List logDatas) { List parsedCatalogCommits = logDatas.stream() .filter(logData -> logData instanceof ParsedCatalogCommitData && logData.isFile()) .map(catalogCommit -> (ParsedCatalogCommitData) catalogCommit) .collect(Collectors.toList()); final long resolvedVersionToLoad = snapshotQueryCtx .getSnapshotMetrics() .computeTimestampToVersionTotalDurationTimer .time( () -> DeltaHistoryManager.getActiveCommitAtTimestamp( engine, latestSnapshot, latestSnapshot.getLogPath(), millisSinceEpochUTC, true /* mustBeRecreatable */, false /* canReturnLastCommit */, false /* canReturnEarliestCommit */, parsedCatalogCommits) .getVersion()); snapshotQueryCtx.setResolvedVersion(resolvedVersionToLoad); logger.info( "{}: Took {} ms to resolve timestamp {} to snapshot version {}", latestSnapshot.getPath(), snapshotQueryCtx .getSnapshotMetrics() .computeTimestampToVersionTotalDurationTimer .totalDurationMs(), millisSinceEpochUTC, resolvedVersionToLoad); return resolvedVersionToLoad; } /** * Creates a lazy loader for CRC file information. The CRC file is loaded only once when needed. * *

If {@link Lazy#isPresent()} is false, then the CRC file was never attempted to be loaded. * *

If {@link Lazy#isPresent()} is true, then the result is: * *

    *
  • {@code Optional.empty()} if there is no CRC file in this LogSegment, we failed to read * it, or we failed to parse it (e.g. missing required fields) *
  • {@code Optional.of(crcInfo)} if the file exists and was successfully read and parsed *
*/ public static Lazy> createLazyChecksumFileLoaderWithMetrics( Engine engine, Lazy lazyLogSegment, SnapshotMetrics snapshotMetrics) { return new Lazy<>( () -> { final Optional crcFileOpt = lazyLogSegment.get().getLastSeenChecksum(); if (!crcFileOpt.isPresent()) { return Optional.empty(); } return snapshotMetrics.loadCrcTotalDurationTimer.time( () -> ChecksumReader.tryReadChecksumFile(engine, crcFileOpt.get())); }); } private static final Logger logger = LoggerFactory.getLogger(SnapshotFactory.class); ////////////////////////////////// // Member methods and variables // ////////////////////////////////// private final SnapshotBuilderImpl.Context ctx; private final Path tablePath; SnapshotFactory(Engine engine, SnapshotBuilderImpl.Context ctx) { this.ctx = ctx; this.tablePath = new Path(resolvePath(engine, ctx.unresolvedPath)); } SnapshotImpl create(Engine engine) { final SnapshotQueryContext snapshotCtx = getSnapshotQueryContext(); try { final SnapshotImpl snapshot = snapshotCtx .getSnapshotMetrics() .loadSnapshotTotalTimer .time(() -> createSnapshot(engine, snapshotCtx)); logger.info( "[{}] Took {}ms to load snapshot (version = {}) for snapshot query {}", tablePath.toString(), snapshotCtx.getSnapshotMetrics().loadSnapshotTotalTimer.totalDurationMs(), snapshot.getVersion(), snapshotCtx.getQueryDisplayStr()); engine .getMetricsReporters() .forEach(reporter -> reporter.report(snapshot.getSnapshotReport())); return snapshot; } catch (Exception e) { snapshotCtx.recordSnapshotErrorReport(engine, e); throw e; } } private SnapshotImpl createSnapshot(Engine engine, SnapshotQueryContext snapshotCtx) { final Optional timeTravelVersion = getTargetTimeTravelVersion(engine, snapshotCtx); final Lazy lazyLogSegment = getLazyLogSegment(engine, snapshotCtx, timeTravelVersion); final Lazy> lazyCrcInfo = createLazyChecksumFileLoaderWithMetrics( engine, lazyLogSegment, snapshotCtx.getSnapshotMetrics()); Protocol protocol; Metadata metadata; if (ctx.protocolAndMetadataOpt.isPresent()) { protocol = ctx.protocolAndMetadataOpt.get()._1; metadata = ctx.protocolAndMetadataOpt.get()._2; } else { ProtocolMetadataLogReplay.Result result = ProtocolMetadataLogReplay.loadProtocolAndMetadata( engine, tablePath, lazyLogSegment.get(), lazyCrcInfo, snapshotCtx.getSnapshotMetrics()); protocol = result.protocol; metadata = result.metadata; } // We require maxCatalogVersion to be provided for catalogManaged tables. We cannot validate // this earlier since we need to first load the protocol. validateMaxCatalogVersionPresence(protocol); // TODO: When LogReplay becomes static utilities, we can create it inside of SnapshotImpl final LogReplay logReplay = new LogReplay(engine, tablePath, lazyLogSegment, lazyCrcInfo); return new SnapshotImpl( tablePath, timeTravelVersion.orElseGet(() -> lazyLogSegment.get().getVersion()), lazyLogSegment, logReplay, protocol, metadata, ctx.committerOpt.orElse(DefaultFileSystemManagedTableOnlyCommitter.INSTANCE), snapshotCtx, Optional.empty() /* inCommitTimestampOpt */); } private SnapshotQueryContext getSnapshotQueryContext() { if (ctx.versionOpt.isPresent()) { return SnapshotQueryContext.forVersionSnapshot(tablePath.toString(), ctx.versionOpt.get()); } if (ctx.timestampQueryContextOpt.isPresent()) { return SnapshotQueryContext.forTimestampSnapshot( tablePath.toString(), ctx.timestampQueryContextOpt.get()._2); } return SnapshotQueryContext.forLatestSnapshot(tablePath.toString()); } private Lazy getLazyLogSegment( Engine engine, SnapshotQueryContext snapshotCtx, Optional timeTravelVersion) { return new Lazy<>( () -> { final LogSegment logSegment = snapshotCtx .getSnapshotMetrics() .loadLogSegmentTotalDurationTimer .time( () -> new SnapshotManager(tablePath) .getLogSegmentForVersion( engine, timeTravelVersion, ctx.logDatas, ctx.maxCatalogVersion)); snapshotCtx.setResolvedVersion(logSegment.getVersion()); snapshotCtx.setCheckpointVersion(logSegment.getCheckpointVersionOpt()); return logSegment; }); } private Optional getTargetTimeTravelVersion( Engine engine, SnapshotQueryContext snapshotCtx) { if (ctx.timestampQueryContextOpt.isPresent()) { return Optional.of( resolveTimestampToSnapshotVersion( engine, snapshotCtx, ctx.timestampQueryContextOpt.get()._1, ctx.timestampQueryContextOpt.get()._2, ctx.logDatas)); } else if (ctx.versionOpt.isPresent()) { return ctx.versionOpt; } return Optional.empty(); } private void validateMaxCatalogVersionPresence(Protocol protocol) { boolean isCatalogManaged = TableFeatures.isCatalogManagedSupported(protocol); if (isCatalogManaged) { checkArgument( ctx.maxCatalogVersion.isPresent(), "Must provide maxCatalogVersion for catalogManaged tables"); } else { checkArgument( !ctx.maxCatalogVersion.isPresent(), "Should not provide maxCatalogVersion for file-system managed tables"); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/FeatureAutoEnabledByMetadata.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.tablefeatures; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; /** * Defines behavior for {@link TableFeature} that can be automatically enabled via a change in a * table's metadata, e.g., through setting particular values of certain feature-specific table * properties. When the requirements are satisfied, the feature is automatically enabled. */ public interface FeatureAutoEnabledByMetadata { /** * Determine whether the feature must be supported and enabled because its metadata requirements * are satisfied. * * @param protocol the protocol of the table for features that are already enabled. * @param metadata the metadata of the table for properties that can enable the feature. */ boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeature.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.tablefeatures; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.internal.actions.Metadata; import java.util.Collections; import java.util.Set; /** * Base class for table features. * *

A feature can be explicitly supported by a table's protocol when the protocol contains * a feature's `name`. Writers (for writer-only features) or readers and writers (for reader-writer * features) must recognize supported features and must handle them appropriately. * *

A table feature that released before Delta Table Features (reader version 3 and writer version * 7) is considered as a legacy feature. Legacy features are implicitly * supported when (a) the protocol does not support table features, i.e., has reader * version less than 3 or writer version less than 7 and (b) the feature's minimum reader/writer * version is less than or equal to the current protocol's reader/writer version. * *

Separately, a feature can be automatically supported by a table's metadata when certain * feature-specific table properties are set. For example, `changeDataFeed` is automatically * supported when there's a table property `delta.enableChangeDataFeed=true`. See {@link * FeatureAutoEnabledByMetadata} for details on how to define such features. This is independent of * the table's enabled features. When a feature is supported (explicitly or implicitly) by the table * protocol but its metadata requirements are not satisfied, then clients still have to understand * the feature (at least to the extent that they can read and preserve the existing data in the * table that uses the feature). * *

Important note: uses the default implementation of `equals` and `hashCode` methods. We expect * that the feature instances are singletons, so we don't need to compare the fields. */ public abstract class TableFeature { ///////////////////////////////////////////////////////////////////////////////// /// Instance variables. /// ///////////////////////////////////////////////////////////////////////////////// private final String featureName; private final int minReaderVersion; private final int minWriterVersion; ///////////////////////////////////////////////////////////////////////////////// /// Public methods. /// ///////////////////////////////////////////////////////////////////////////////// /** * Constructor. Does validations to make sure: * *

    *
  • Feature name is not null or empty and has valid characters *
  • minReaderVersion is always 0 for writer features *
* * @param featureName a globally-unique string indicator to represent the feature. All characters * must be letters (a-z, A-Z), digits (0-9), '-', or '_'. Words must be in camelCase. * @param minReaderVersion the minimum reader version this feature requires. For a feature that * can only be explicitly supported, this is either `0` (i.e writerOnly feature) or `3` (the * reader protocol version that supports table features), depending on the feature is * writer-only or reader-writer. For a legacy feature that can be implicitly supported, this * is the first protocol version which the feature is introduced. * @param minWriterVersion the minimum writer version this feature requires. For a feature that * can only be explicitly supported, this is the writer protocol `7` that supports table * features. For a legacy feature that can be implicitly supported, this is the first protocol * version which the feature is introduced. */ public TableFeature(String featureName, int minReaderVersion, int minWriterVersion) { this.featureName = requireNonNull(featureName, "name is null"); checkArgument(!featureName.isEmpty(), "name is empty"); checkArgument( featureName.chars().allMatch(c -> Character.isLetterOrDigit(c) || c == '-' || c == '_'), "name contains invalid characters: " + featureName); checkArgument(minReaderVersion >= 0, "minReaderVersion is negative"); checkArgument(minWriterVersion >= 1, "minWriterVersion is less than 1"); this.minReaderVersion = minReaderVersion; this.minWriterVersion = minWriterVersion; validate(); } /** @return the name of the table feature. */ public String featureName() { return featureName; } /** * @return true if this feature is applicable to both reader and writer, false if it is * writer-only. */ public boolean isReaderWriterFeature() { return this instanceof ReaderWriterFeatureType; } /** @return the minimum reader version this feature requires */ public int minReaderVersion() { return minReaderVersion; } /** @return the minimum writer version that this feature requires. */ public int minWriterVersion() { return minWriterVersion; } /** @return if this feature is a legacy feature? */ public boolean isLegacyFeature() { return this instanceof LegacyFeatureType; } /** * Set of table features that this table feature depends on. I.e. the set of features that need to * be enabled if this table feature is enabled. * * @return the set of table features that this table feature depends on. */ public Set requiredFeatures() { return Collections.emptySet(); } /** * Does Kernel has support to read a table containing this feature? Default implementation returns * true. Features should override this method if they have special requirements or not supported * by the Kernel yet. * * @return true if Kernel has support to read a table containing this feature. */ public boolean hasKernelReadSupport() { checkArgument(isReaderWriterFeature(), "Should be called only for reader-writer features"); return true; } /** * Does Kernel has support to write a table containing this feature? Default implementation * returns true. Features should override this method if they have special requirements or not * supported by the Kernel yet. * * @param metadata the metadata of the table. Sometimes checking the metadata is necessary to know * the Kernel can write the table or not. * @return true if Kernel has support to write a table containing this feature. */ public boolean hasKernelWriteSupport(Metadata metadata) { return true; } /** * Gets the key that turns on support for the respective table feature. * * @return the feature support key for the respective feature. */ public String getTableFeatureSupportKey() { return TableFeatures.SET_TABLE_FEATURE_SUPPORTED_PREFIX + featureName(); } ///////////////////////////////////////////////////////////////////////////////// /// Define the {@link TableFeature}s traits that define behavior/attributes. /// ///////////////////////////////////////////////////////////////////////////////// /** * An interface to indicate a feature is legacy, i.e., released before Table Features. All legacy * features are auto enabled by metadata. */ public interface LegacyFeatureType extends FeatureAutoEnabledByMetadata {} /** An interface to indicate a feature applies to readers and writers. */ public interface ReaderWriterFeatureType {} ///////////////////////////////////////////////////////////////////////////////// /// Base classes for each of the feature category. /// ///////////////////////////////////////////////////////////////////////////////// /** A base class for all table legacy writer-only features. */ public abstract static class LegacyWriterFeature extends TableFeature implements LegacyFeatureType { public LegacyWriterFeature(String featureName, int minWriterVersion) { super(featureName, /* minReaderVersion= */ 0, minWriterVersion); } @Override public boolean hasKernelReadSupport() { return true; } } /** A base class for all table legacy reader-writer features. */ public abstract static class LegacyReaderWriterFeature extends TableFeature implements LegacyFeatureType, ReaderWriterFeatureType { public LegacyReaderWriterFeature( String featureName, int minReaderVersion, int minWriterVersion) { super(featureName, minReaderVersion, minWriterVersion); } } /** A base class for all non-legacy table writer features. */ public abstract static class WriterFeature extends TableFeature { public WriterFeature(String featureName, int minWriterVersion) { super(featureName, /* minReaderVersion= */ 0, minWriterVersion); } @Override public boolean hasKernelReadSupport() { return true; } } /** A base class for all non-legacy table reader-writer features. */ public abstract static class ReaderWriterFeature extends TableFeature implements ReaderWriterFeatureType { public ReaderWriterFeature(String featureName, int minReaderVersion, int minWriterVersion) { super(featureName, minReaderVersion, minWriterVersion); } } /** * Validate the table feature. This method should throw an exception if the table feature * properties are invalid. Should be called after the object deriving the {@link TableFeature} is * constructed. */ private void validate() { if (!isReaderWriterFeature()) { checkArgument(minReaderVersion() == 0, "Writer-only feature must have minReaderVersion=0"); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.tablefeatures; import static io.delta.kernel.internal.DeltaErrors.*; import static io.delta.kernel.internal.util.ColumnMapping.ColumnMappingMode.NONE; import static io.delta.kernel.types.TimestampNTZType.TIMESTAMP_NTZ; import static io.delta.kernel.types.VariantType.VARIANT; import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toSet; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.internal.DeltaErrors; import io.delta.kernel.internal.TableConfig; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.util.CaseInsensitiveMap; import io.delta.kernel.internal.util.SchemaIterable; import io.delta.kernel.internal.util.Tuple2; import io.delta.kernel.types.*; import java.util.*; import java.util.stream.Stream; /** Contains utility methods related to the Delta table feature support in protocol. */ public class TableFeatures { /** * The prefix for setting an override of a feature option in {@linkplain Metadata} configuration. * *

Keys with this prefix should never be persisted in the Metadata action. The keys can be * filtered out by using {@linkplain #extractFeaturePropertyOverrides}. * *

These overrides only support add the feature as supported in the Protocol action. * *

Disabling features via this method is unsupported. */ public static String SET_TABLE_FEATURE_SUPPORTED_PREFIX = "delta.feature."; /** * Configuration value to turn on support for a table feature when used with {@link * #SET_TABLE_FEATURE_SUPPORTED_PREFIX}. * *

Example: {@code "delta.feature.myFeature" -> "supported"} */ public static String SET_TABLE_FEATURE_SUPPORTED_VALUE = "supported"; ///////////////////////////////////////////////////////////////////////////////// /// START: Define the {@link TableFeature}s /// /// If feature instance variable ends with /// /// 1) `_W_FEATURE` it is a writer only feature. /// /// 2) `_RW_FEATURE` it is a reader-writer feature. /// ///////////////////////////////////////////////////////////////////////////////// public static final TableFeature APPEND_ONLY_W_FEATURE = new AppendOnlyFeature(); private static class AppendOnlyFeature extends TableFeature.LegacyWriterFeature { AppendOnlyFeature() { super(/* featureName = */ "appendOnly", /* minWriterVersion = */ 2); } @Override public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { return TableConfig.APPEND_ONLY_ENABLED.fromMetadata(metadata); } @Override public boolean hasKernelWriteSupport(Metadata metadata) { return true; } } public static final TableFeature CATALOG_MANAGED_RW_FEATURE = new CatalogManagedFeatureBase("catalogManaged"); private static class CatalogManagedFeatureBase extends TableFeature.ReaderWriterFeature { CatalogManagedFeatureBase(String featureName) { super(featureName, /* minReaderVersion = */ 3, /* minWriterVersion = */ 7); } @Override public Set requiredFeatures() { return Collections.singleton(IN_COMMIT_TIMESTAMP_W_FEATURE); } } public static final TableFeature INVARIANTS_W_FEATURE = new InvariantsFeature(); private static class InvariantsFeature extends TableFeature.LegacyWriterFeature { InvariantsFeature() { super(/* featureName = */ "invariants", /* minWriterVersion = */ 2); } @Override public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { return hasInvariants(metadata.getSchema()); } @Override public boolean hasKernelWriteSupport(Metadata metadata) { // If there is no invariant, then the table is supported return !hasInvariants(metadata.getSchema()); } } public static final TableFeature CONSTRAINTS_W_FEATURE = new ConstraintsFeature(); private static class ConstraintsFeature extends TableFeature.LegacyWriterFeature { ConstraintsFeature() { super("checkConstraints", /* minWriterVersion = */ 3); } @Override public boolean hasKernelWriteSupport(Metadata metadata) { // Kernel doesn't support table with constraints. return !hasCheckConstraints(metadata); } @Override public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { return hasCheckConstraints(metadata); } } public static final TableFeature CHANGE_DATA_FEED_W_FEATURE = new ChangeDataFeedFeature(); private static class ChangeDataFeedFeature extends TableFeature.LegacyWriterFeature { ChangeDataFeedFeature() { super("changeDataFeed", /* minWriterVersion = */ 4); } @Override public boolean hasKernelWriteSupport(Metadata metadata) { // writable if change data feed is disabled return !TableConfig.CHANGE_DATA_FEED_ENABLED.fromMetadata(metadata); } @Override public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { return TableConfig.CHANGE_DATA_FEED_ENABLED.fromMetadata(metadata); } } public static final TableFeature COLUMN_MAPPING_RW_FEATURE = new ColumnMappingFeature(); private static class ColumnMappingFeature extends TableFeature.LegacyReaderWriterFeature { ColumnMappingFeature() { super("columnMapping", /*minReaderVersion = */ 2, /* minWriterVersion = */ 5); } @Override public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { return TableConfig.COLUMN_MAPPING_MODE.fromMetadata(metadata) != NONE; } } public static final TableFeature GENERATED_COLUMNS_W_FEATURE = new GeneratedColumnsFeature(); private static class GeneratedColumnsFeature extends TableFeature.LegacyWriterFeature { GeneratedColumnsFeature() { super("generatedColumns", /* minWriterVersion = */ 4); } @Override public boolean hasKernelWriteSupport(Metadata metadata) { // Kernel can write as long as there are no generated columns defined return !hasGeneratedColumns(metadata); } @Override public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { return hasGeneratedColumns(metadata); } } public static final TableFeature IDENTITY_COLUMNS_W_FEATURE = new IdentityColumnsFeature(); private static class IdentityColumnsFeature extends TableFeature.LegacyWriterFeature { IdentityColumnsFeature() { super("identityColumns", /* minWriterVersion = */ 6); } @Override public boolean hasKernelWriteSupport(Metadata metadata) { return !hasIdentityColumns(metadata); } @Override public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { return hasIdentityColumns(metadata); } } /* ---- Start: variantType ---- */ // Base class for variantType and variantType-preview features. Both features are same in terms // of behavior and given the feature is graduated, we will enable the `variantType` by default // if the metadata requirements are satisfied and the table doesn't already contain the // `variantType-preview` feature. Also to note, with this version of Kernel, one can't // auto upgrade to `variantType-preview` with metadata requirements. It can only be set // manually using `delta.feature.variantType-preview=supported` property. private static class VariantTypeTableFeatureBase extends TableFeature.ReaderWriterFeature { VariantTypeTableFeatureBase(String featureName) { super(featureName, /* minReaderVersion = */ 3, /* minWriterVersion = */ 7); } } private static class VariantTypeTableFeature extends VariantTypeTableFeatureBase implements FeatureAutoEnabledByMetadata { VariantTypeTableFeature() { super("variantType"); } @Override public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { return hasTypeColumn(metadata.getSchema(), VARIANT) && // Don't automatically enable the stable feature if the preview feature is // already supported, to avoid possibly breaking old clients that only // support the preview feature. !protocol.supportsFeature(VARIANT_RW_PREVIEW_FEATURE); } } public static final TableFeature VARIANT_RW_FEATURE = new VariantTypeTableFeature(); public static final TableFeature VARIANT_RW_PREVIEW_FEATURE = new VariantTypeTableFeatureBase("variantType-preview"); /* ---- End: variantType ---- */ /* ---- Start: variantShredding ---- */ // Base class for variantShredding and variantShredding-preview features. Both features have // identical behavior. Now that variantShredding has graduated to GA: // // - When `delta.enableVariantShredding` is set to true, the GA feature (`variantShredding`) // is automatically enabled unless the table already has `variantShredding-preview` in its // protocol (to avoid breaking clients that only understand the preview feature). // - The preview feature (`variantShredding-preview`) is never auto-enabled. To use it on a // new table, a user must explicitly set `delta.feature.variantShredding-preview=supported` // in the table properties. private static class VariantShreddingTableFeatureBase extends TableFeature.ReaderWriterFeature { VariantShreddingTableFeatureBase(String featureName) { super(featureName, /* minReaderVersion = */ 3, /* minWriterVersion = */ 7); } @Override public Set requiredFeatures() { return new HashSet<>(Arrays.asList(TableFeatures.VARIANT_RW_FEATURE)); } } private static class VariantShreddingTableFeature extends VariantShreddingTableFeatureBase implements FeatureAutoEnabledByMetadata { VariantShreddingTableFeature() { super("variantShredding"); } @Override public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { return TableConfig.VARIANT_SHREDDING_ENABLED.fromMetadata(metadata) && // Don't automatically enable the stable feature if the preview feature is // already supported, to avoid possibly breaking old clients that only // support the preview feature. !protocol.supportsFeature(VARIANT_SHREDDING_PREVIEW_RW_FEATURE); } } public static final TableFeature VARIANT_SHREDDING_RW_FEATURE = new VariantShreddingTableFeature(); public static final TableFeature VARIANT_SHREDDING_PREVIEW_RW_FEATURE = new VariantShreddingTableFeatureBase("variantShredding-preview"); /* ---- End: variantShredding ---- */ public static final TableFeature DOMAIN_METADATA_W_FEATURE = new DomainMetadataFeature(); private static class DomainMetadataFeature extends TableFeature.WriterFeature { DomainMetadataFeature() { super("domainMetadata", /* minWriterVersion = */ 7); } } public static final TableFeature CLUSTERING_W_FEATURE = new ClusteringTableFeature(); private static class ClusteringTableFeature extends TableFeature.WriterFeature { ClusteringTableFeature() { super("clustering", /* minWriterVersion = */ 7); } @Override public Set requiredFeatures() { return Collections.singleton(DOMAIN_METADATA_W_FEATURE); } } public static final TableFeature ROW_TRACKING_W_FEATURE = new RowTrackingFeature(); private static class RowTrackingFeature extends TableFeature.WriterFeature implements FeatureAutoEnabledByMetadata { RowTrackingFeature() { super("rowTracking", /* minWriterVersion = */ 7); } @Override public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { return TableConfig.ROW_TRACKING_ENABLED.fromMetadata(metadata); } @Override public Set requiredFeatures() { return Collections.singleton(DOMAIN_METADATA_W_FEATURE); } } public static final TableFeature DELETION_VECTORS_RW_FEATURE = new DeletionVectorsTableFeature(); /** * Kernel currently only support blind appends. So we don't need to do anything special for * writing into a table with deletion vectors enabled (i.e a table feature with DV enabled is both * readable and writable). */ private static class DeletionVectorsTableFeature extends TableFeature.ReaderWriterFeature implements FeatureAutoEnabledByMetadata { DeletionVectorsTableFeature() { super("deletionVectors", /* minReaderVersion = */ 3, /* minWriterVersion = */ 7); } @Override public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { return TableConfig.DELETION_VECTORS_CREATION_ENABLED.fromMetadata(metadata); } } public static final TableFeature ICEBERG_COMPAT_V2_W_FEATURE = new IcebergCompatV2TableFeature(); private static class IcebergCompatV2TableFeature extends TableFeature.WriterFeature implements FeatureAutoEnabledByMetadata { IcebergCompatV2TableFeature() { super("icebergCompatV2", /* minWriterVersion = */ 7); } @Override public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { return TableConfig.ICEBERG_COMPAT_V2_ENABLED.fromMetadata(metadata); } public @Override Set requiredFeatures() { return Collections.singleton(COLUMN_MAPPING_RW_FEATURE); } } public static final TableFeature ICEBERG_COMPAT_V3_W_FEATURE = new IcebergCompatV3TableFeature(); private static class IcebergCompatV3TableFeature extends TableFeature.WriterFeature implements FeatureAutoEnabledByMetadata { IcebergCompatV3TableFeature() { super("icebergCompatV3", /* minWriterVersion = */ 7); } @Override public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { return TableConfig.ICEBERG_COMPAT_V3_ENABLED.fromMetadata(metadata); } public @Override Set requiredFeatures() { return Collections.unmodifiableSet( new HashSet<>(Arrays.asList(COLUMN_MAPPING_RW_FEATURE, ROW_TRACKING_W_FEATURE))); } } /* ---- Start: type widening ---- */ // Base class for typeWidening and typeWidening-preview features. Both features are same in terms // of behavior and given the feature is graduated, we will enable the `typeWidening` by default // if the metadata requirements are satisfied and the table doesn't already contain the // `typeWidening-preview` feature. Also to note, with this version of Kernel, one can't // auto upgrade to `typeWidening-preview` with metadata requirements. It can only be set // manually using `delta.feature.typeWidening-preview=supported` property. private static class TypeWideningTableFeatureBase extends TableFeature.ReaderWriterFeature { TypeWideningTableFeatureBase(String featureName) { super(featureName, /* minReaderVersion = */ 3, /* minWriterVersion = */ 7); } } private static class TypeWideningTableFeature extends TypeWideningTableFeatureBase implements FeatureAutoEnabledByMetadata { TypeWideningTableFeature() { super("typeWidening"); } @Override public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { return TableConfig.TYPE_WIDENING_ENABLED.fromMetadata(metadata) && // Don't automatically enable the stable feature if the preview feature is already // supported, to // avoid possibly breaking old clients that only support the preview feature. !protocol.supportsFeature(TYPE_WIDENING_RW_PREVIEW_FEATURE); } } public static final TableFeature TYPE_WIDENING_RW_FEATURE = new TypeWideningTableFeature(); public static final TableFeature TYPE_WIDENING_RW_PREVIEW_FEATURE = new TypeWideningTableFeatureBase("typeWidening-preview"); /* ---- End: type widening ---- */ public static final TableFeature IN_COMMIT_TIMESTAMP_W_FEATURE = new InCommitTimestampTableFeature(); private static class InCommitTimestampTableFeature extends TableFeature.WriterFeature implements FeatureAutoEnabledByMetadata { InCommitTimestampTableFeature() { super("inCommitTimestamp", /* minWriterVersion = */ 7); } @Override public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { return TableConfig.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetadata(metadata); } } public static final TableFeature TIMESTAMP_NTZ_RW_FEATURE = new TimestampNtzTableFeature(); private static class TimestampNtzTableFeature extends TableFeature.ReaderWriterFeature implements FeatureAutoEnabledByMetadata { TimestampNtzTableFeature() { super("timestampNtz", /* minReaderVersion = */ 3, /* minWriterVersion = */ 7); } @Override public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { return hasTypeColumn(metadata.getSchema(), TIMESTAMP_NTZ); } } public static final TableFeature CHECKPOINT_V2_RW_FEATURE = new CheckpointV2TableFeature(); /** * In order to commit, there is no extra work required when v2 checkpoint is enabled. This affects * the checkpoint format only. When v2 is enabled, writing classic checkpoints is still allowed. */ private static class CheckpointV2TableFeature extends TableFeature.ReaderWriterFeature implements FeatureAutoEnabledByMetadata { CheckpointV2TableFeature() { super("v2Checkpoint", /* minReaderVersion = */ 3, /* minWriterVersion = */ 7); } @Override public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { // TODO: define an enum for checkpoint policy when we start supporting writing v2 checkpoints return "v2".equals(TableConfig.CHECKPOINT_POLICY.fromMetadata(metadata)); } } public static final TableFeature VACUUM_PROTOCOL_CHECK_RW_FEATURE = new VacuumProtocolCheckTableFeature(); private static class VacuumProtocolCheckTableFeature extends TableFeature.ReaderWriterFeature { VacuumProtocolCheckTableFeature() { super("vacuumProtocolCheck", /* minReaderVersion = */ 3, /* minWriterVersion = */ 7); } } public static final TableFeature CHECKPOINT_PROTECTION_W_FEATURE = new CheckpointProtectionTableFeature(); private static class CheckpointProtectionTableFeature extends TableFeature.WriterFeature { CheckpointProtectionTableFeature() { super("checkpointProtection", /* minWriterVersion = */ 7); } } /** * Support reading / metadata writes on tables with the feature. Don't support writing new data * rows with default values. Don't allow updating the types of columns with default values. */ public static final TableFeature ALLOW_COLUMN_DEFAULTS_W_FEATURE = new AllowColumnDefaultsTableFeature(); private static class AllowColumnDefaultsTableFeature extends TableFeature.WriterFeature { AllowColumnDefaultsTableFeature() { super("allowColumnDefaults", /* minWriterVersion = */ 7); } } public static final TableFeature ICEBERG_WRITER_COMPAT_V1 = new IcebergWriterCompatV1(); private static class IcebergWriterCompatV1 extends TableFeature.WriterFeature implements FeatureAutoEnabledByMetadata { IcebergWriterCompatV1() { super("icebergWriterCompatV1", /* minWriterVersion = */ 7); } @Override public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { return TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.fromMetadata(metadata); } public @Override Set requiredFeatures() { return Collections.singleton(ICEBERG_COMPAT_V2_W_FEATURE); } } public static final TableFeature ICEBERG_WRITER_COMPAT_V3 = new IcebergWriterCompatV3(); private static class IcebergWriterCompatV3 extends TableFeature.WriterFeature implements FeatureAutoEnabledByMetadata { IcebergWriterCompatV3() { super("icebergWriterCompatV3", /* minWriterVersion = */ 7); } @Override public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { return TableConfig.ICEBERG_WRITER_COMPAT_V3_ENABLED.fromMetadata(metadata); } public @Override Set requiredFeatures() { return Collections.singleton(ICEBERG_COMPAT_V3_W_FEATURE); } } public static final TableFeature MATERIALIZE_PARTITION_COLUMNS_W_FEATURE = new MaterializePartitionColumnsFeature(); private static class MaterializePartitionColumnsFeature extends TableFeature.WriterFeature { MaterializePartitionColumnsFeature() { super("materializePartitionColumns", /* minWriterVersion = */ 7); } } ///////////////////////////////////////////////////////////////////////////////// /// END: Define the {@link TableFeature}s /// ///////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////// /// Public static variables and methods /// ///////////////////////////////////////////////////////////////////////////////// /** Min reader version that supports reader features. */ public static final int TABLE_FEATURES_MIN_READER_VERSION = 3; /** Min reader version that supports writer features. */ public static final int TABLE_FEATURES_MIN_WRITER_VERSION = 7; public static final List TABLE_FEATURES = Collections.unmodifiableList( Arrays.asList( ALLOW_COLUMN_DEFAULTS_W_FEATURE, APPEND_ONLY_W_FEATURE, CATALOG_MANAGED_RW_FEATURE, CHECKPOINT_PROTECTION_W_FEATURE, CHECKPOINT_V2_RW_FEATURE, CHANGE_DATA_FEED_W_FEATURE, CLUSTERING_W_FEATURE, COLUMN_MAPPING_RW_FEATURE, CONSTRAINTS_W_FEATURE, DELETION_VECTORS_RW_FEATURE, GENERATED_COLUMNS_W_FEATURE, DOMAIN_METADATA_W_FEATURE, ICEBERG_COMPAT_V2_W_FEATURE, ICEBERG_COMPAT_V3_W_FEATURE, IDENTITY_COLUMNS_W_FEATURE, IN_COMMIT_TIMESTAMP_W_FEATURE, INVARIANTS_W_FEATURE, MATERIALIZE_PARTITION_COLUMNS_W_FEATURE, ROW_TRACKING_W_FEATURE, TIMESTAMP_NTZ_RW_FEATURE, TYPE_WIDENING_RW_PREVIEW_FEATURE, TYPE_WIDENING_RW_FEATURE, VACUUM_PROTOCOL_CHECK_RW_FEATURE, VARIANT_RW_FEATURE, VARIANT_RW_PREVIEW_FEATURE, VARIANT_SHREDDING_RW_FEATURE, VARIANT_SHREDDING_PREVIEW_RW_FEATURE, ICEBERG_WRITER_COMPAT_V1, ICEBERG_WRITER_COMPAT_V3)); public static final Map TABLE_FEATURE_MAP = Collections.unmodifiableMap( new CaseInsensitiveMap() { { for (TableFeature feature : TABLE_FEATURES) { put(feature.featureName(), feature); } } }); /** Get the table feature by name. Case-insensitive lookup. If not found, throws error. */ public static TableFeature getTableFeature(String featureName) { TableFeature tableFeature = TABLE_FEATURE_MAP.get(featureName); if (tableFeature == null) { throw DeltaErrors.unsupportedTableFeature(featureName); } return tableFeature; } /** Does reader version supports explicitly specifying reader feature set in protocol? */ public static boolean supportsReaderFeatures(int minReaderVersion) { return minReaderVersion >= TABLE_FEATURES_MIN_READER_VERSION; } /** Does writer version supports explicitly specifying writer feature set in protocol? */ public static boolean supportsWriterFeatures(int minWriterVersion) { return minWriterVersion >= TABLE_FEATURES_MIN_WRITER_VERSION; } /** Returns the minimum reader/writer versions required to support all provided features. */ public static Tuple2 minimumRequiredVersions(Set features) { int minReaderVersion = features.stream().mapToInt(TableFeature::minReaderVersion).max().orElse(0); int minWriterVersion = features.stream().mapToInt(TableFeature::minWriterVersion).max().orElse(0); return new Tuple2<>(Math.max(minReaderVersion, 1), Math.max(minWriterVersion, 1)); } /** * Upgrade the current protocol to satisfy all auto-update capable features required by the given * metadata. If the current protocol already satisfies the metadata requirements, return empty. * * @param newMetadata the new metadata to be applied to the table. * @param manuallyEnabledFeatures features that were requested to be added to the protocol. * @param currentProtocol the current protocol of the table. * @return the upgraded protocol and the set of new features that were enabled in the upgrade. */ public static Optional>> autoUpgradeProtocolBasedOnMetadata( Metadata newMetadata, Collection manuallyEnabledFeatures, Protocol currentProtocol) { Set allNeededTableFeatures = extractAllNeededTableFeatures(newMetadata, currentProtocol); if (manuallyEnabledFeatures != null && !manuallyEnabledFeatures.isEmpty()) { // Note that any dependent features are handled below in the withFeatures call. allNeededTableFeatures = Stream.concat(allNeededTableFeatures.stream(), manuallyEnabledFeatures.stream()) .collect(toSet()); } Protocol required = new Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(allNeededTableFeatures) .normalized(); // See if all the required features are already supported in the current protocol. if (!required.canUpgradeTo(currentProtocol)) { // `required` has one or more features that are not supported in `currentProtocol`. Set newFeatures = new HashSet<>(required.getImplicitlyAndExplicitlySupportedFeatures()); newFeatures.removeAll(currentProtocol.getImplicitlyAndExplicitlySupportedFeatures()); return Optional.of(new Tuple2<>(required.merge(currentProtocol), newFeatures)); } else { return Optional.empty(); } } /** * Checks if a table feature is being manually supported through user property {@code * delta.feature.=supported}. */ public static boolean isPropertiesManuallySupportingTableFeature( Map userProperties, TableFeature tableFeature) { final String featurePropKey = SET_TABLE_FEATURE_SUPPORTED_PREFIX + tableFeature.featureName(); final String propertyValue = userProperties.get(featurePropKey); // will be null if not found return SET_TABLE_FEATURE_SUPPORTED_VALUE.equals(propertyValue); } /** * Extracts features overrides from Metadata properties and returns an updated metadata if any * overrides are present. * *

Overrides are specified using a key in th form {@linkplain * #SET_TABLE_FEATURE_SUPPORTED_PREFIX} + {featureName}. (e.g. {@code * delta.feature.icebergWriterCompatV1}). The value must be "supported" to add the feature. * Currently, removing values is not handled. * * @return A set of features that had overrides and Metadata object with the properties removed if * any overrides were present. * @throws KernelException if the feature name for the override is invalid or the value is not * equal to "supported". */ public static Tuple2, Optional> extractFeaturePropertyOverrides( Metadata currentMetadata) { Set features = new HashSet<>(); Map properties = currentMetadata.getConfiguration(); for (Map.Entry entry : properties.entrySet()) { if (entry.getKey().startsWith(SET_TABLE_FEATURE_SUPPORTED_PREFIX)) { String featureName = entry.getKey().substring(SET_TABLE_FEATURE_SUPPORTED_PREFIX.length()); TableFeature feature = getTableFeature(featureName); features.add(feature); if (!entry.getValue().equals(SET_TABLE_FEATURE_SUPPORTED_VALUE)) { throw DeltaErrors.invalidConfigurationValueException( entry.getKey(), entry.getValue(), "TableFeature override options may only have \"supported\" as there value"); } } } if (features.isEmpty()) { return new Tuple2<>(features, Optional.empty()); } Map cleanedProperties = properties.entrySet().stream() .filter(e -> !e.getKey().startsWith(SET_TABLE_FEATURE_SUPPORTED_PREFIX)) .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); return new Tuple2<>( features, Optional.of(currentMetadata.withReplacedConfiguration(cleanedProperties))); } /** Utility method to check if the table with given protocol is readable by the Kernel. */ public static void validateKernelCanReadTheTable(Protocol protocol, String tablePath) { if (protocol.getMinReaderVersion() > TABLE_FEATURES_MIN_READER_VERSION) { throw DeltaErrors.unsupportedReaderProtocol(tablePath, protocol.getMinReaderVersion()); } Set unsupportedFeatures = protocol.getImplicitlyAndExplicitlySupportedReaderWriterFeatures().stream() .filter(f -> !f.hasKernelReadSupport()) .collect(toSet()); if (!unsupportedFeatures.isEmpty()) { throw unsupportedReaderFeatures( tablePath, unsupportedFeatures.stream().map(TableFeature::featureName).collect(toSet())); } } /** * Utility method to check if the table with given protocol and metadata is writable by the * Kernel. */ public static void validateKernelCanWriteToTable( Protocol protocol, Metadata metadata, String tablePath) { validateKernelCanReadTheTable(protocol, tablePath); if (protocol.getMinWriterVersion() > TABLE_FEATURES_MIN_WRITER_VERSION) { throw unsupportedWriterProtocol(tablePath, protocol.getMinWriterVersion()); } Set unsupportedFeatures = protocol.getImplicitlyAndExplicitlySupportedFeatures().stream() .filter(f -> !f.hasKernelWriteSupport(metadata)) .collect(toSet()); if (!unsupportedFeatures.isEmpty()) { throw unsupportedWriterFeatures( tablePath, unsupportedFeatures.stream().map(TableFeature::featureName).collect(toSet())); } } ///////////////////////////// // Is feature X supported? // ///////////////////////////// public static boolean isCatalogManagedSupported(Protocol protocol) { return protocol.supportsFeature(CATALOG_MANAGED_RW_FEATURE); } public static boolean isRowTrackingSupported(Protocol protocol) { return protocol.supportsFeature(ROW_TRACKING_W_FEATURE); } public static boolean isDomainMetadataSupported(Protocol protocol) { return protocol.supportsFeature(DOMAIN_METADATA_W_FEATURE); } public static boolean isClusteringTableFeatureSupported(Protocol protocol) { return protocol.supportsFeature(CLUSTERING_W_FEATURE); } /////////////////////////// // Does protocol have X? // /////////////////////////// public static boolean hasInvariants(StructType tableSchema) { return SchemaIterable.newSchemaIterableWithIgnoredRecursion( tableSchema, // invariants are not allowed in maps or arrays new Class[] {MapType.class, ArrayType.class}) .stream() .anyMatch(element -> element.getField().getMetadata().contains("delta.invariants")); } public static boolean hasCheckConstraints(Metadata metadata) { return metadata.getConfiguration().keySet().stream() .anyMatch(s -> s.startsWith("delta.constraints.")); } public static boolean hasIdentityColumns(Metadata metadata) { return SchemaIterable.newSchemaIterableWithIgnoredRecursion( metadata.getSchema(), // invariants are not allowed in maps or arrays new Class[] {MapType.class, ArrayType.class}) .stream() .anyMatch( element -> { StructField field = element.getField(); FieldMetadata fieldMetadata = field.getMetadata(); // Check if the metadata contains the required keys boolean hasStart = fieldMetadata.contains("delta.identity.start"); boolean hasStep = fieldMetadata.contains("delta.identity.step"); boolean hasInsert = fieldMetadata.contains("delta.identity.allowExplicitInsert"); // Verify that all or none of the three fields are present if (!((hasStart == hasStep) && (hasStart == hasInsert))) { throw new KernelException( String.format( "Inconsistent IDENTITY metadata for column %s detected: %s, %s, %s", field.getName(), hasStart, hasStep, hasInsert)); } // Return true only if all three fields are present return hasStart && hasStep && hasInsert; }); } public static boolean hasGeneratedColumns(Metadata metadata) { return SchemaIterable.newSchemaIterableWithIgnoredRecursion( metadata.getSchema(), // don't expected generated columns in // nested columns new Class[] {MapType.class, ArrayType.class}) .stream() .anyMatch( element -> element.getField().getMetadata().contains("delta.generationExpression")); } ///////////////////////////////////////////////////////////////////////////////// /// Private methods /// ///////////////////////////////////////////////////////////////////////////////// /** * Extracts all table features (and their dependency features) that are enabled by the given * metadata and supported in existing protocol. */ private static Set extractAllNeededTableFeatures( Metadata newMetadata, Protocol currentProtocol) { Set protocolSupportedFeatures = currentProtocol.getImplicitlyAndExplicitlySupportedFeatures(); Set metadataEnabledFeatures = TableFeatures.TABLE_FEATURES.stream() .filter(f -> f instanceof FeatureAutoEnabledByMetadata) .filter( f -> ((FeatureAutoEnabledByMetadata) f) .metadataRequiresFeatureToBeEnabled(currentProtocol, newMetadata)) .collect(toSet()); // Each feature may have dependencies that are not yet enabled in the protocol. Set newFeatures = getDependencyFeatures(metadataEnabledFeatures); return Stream.concat(protocolSupportedFeatures.stream(), newFeatures.stream()).collect(toSet()); } /** * Returns the smallest set of table features that contains `features` and that also contains all * dependencies of all features in the returned set. */ private static Set getDependencyFeatures(Set features) { Set requiredFeatures = new HashSet<>(features); features.forEach(feature -> requiredFeatures.addAll(feature.requiredFeatures())); if (features.equals(requiredFeatures)) { return features; } else { return getDependencyFeatures(requiredFeatures); } } /** * Check if the table schema has a column of type. Caution: works only for the primitive types. */ private static boolean hasTypeColumn(StructType tableSchema, DataType type) { return new SchemaIterable(tableSchema) .stream().anyMatch(element -> element.getField().getDataType().equals(type)); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/types/DataTypeJsonSerDe.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.types; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.lang.String.format; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.*; import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.databind.ser.std.StdSerializer; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.internal.DeltaErrors; import io.delta.kernel.internal.util.SchemaIterable; import io.delta.kernel.types.*; import java.io.IOException; import java.io.StringWriter; import java.util.*; import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; /** * Serialize and deserialize Delta data types {@link DataType} to JSON and from JSON class based on * the serialization * rules outlined in the Delta Protocol. */ public class DataTypeJsonSerDe { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper() .registerModule( new SimpleModule().addSerializer(StructType.class, new StructTypeSerializer())); private DataTypeJsonSerDe() {} /** * Serializes a {@link StructType} to a JSON string * * @param structType * @return */ public static String serializeStructType(StructType structType) { try { return OBJECT_MAPPER.writeValueAsString(structType); } catch (JsonProcessingException ex) { throw new KernelException("Could not serialize StructType to JSON", ex); } } /** * Serializes a {@link DataType} to a JSON string according to the Delta Protocol. TODO: Only * reason why this API added was due to Flink-Kernel dependency. Currently Flink-Kernel uses the * Kernel DataType.toJson and Standalone DataType.fromJson to convert between types. * * @param dataType * @return JSON string representing the data type */ public static String serializeDataType(DataType dataType) { try { StringWriter stringWriter = new StringWriter(); JsonGenerator generator = OBJECT_MAPPER.createGenerator(stringWriter); writeDataType(generator, dataType); generator.flush(); return stringWriter.toString(); } catch (IOException ex) { throw new KernelException("Could not serialize DataType to JSON", ex); } } /** * Deserializes a JSON string representing a Delta data type to a {@link DataType}. * * @param structTypeJson JSON string representing a {@link StructType} data type */ public static StructType deserializeStructType(String structTypeJson) { try { DataType parsedType = parseDataType(OBJECT_MAPPER.reader().readTree(structTypeJson)); if (parsedType instanceof StructType) { return (StructType) parsedType; } else { throw new IllegalArgumentException( String.format( "Could not parse the following JSON as a valid StructType:\n%s", structTypeJson)); } } catch (JsonProcessingException ex) { throw new KernelException( format("Could not parse schema given as JSON string: %s", structTypeJson), ex); } } /** * Parses a Delta data type from JSON. Data types can either be serialized as strings (for * primitive types) or as objects (for complex types). * *

For example: * * 

   * // Map type field is serialized as:
   * {
   *   "name" : "f",
   *   "type" : {
   *     "type" : "map",
   *     "keyType" : "string",
   *     "valueType" : "string",
   *     "valueContainsNull" : true
   *   },
   *   "nullable" : true,
   *   "metadata" : { }
   * }
   *
   * // Integer type field serialized as:
   * {
   *   "name" : "a",
   *   "type" : "integer",
   *   "nullable" : false,
   *   "metadata" : { }
   * }
   *
   *
   *
* * Note: Metadata for individual schema element (e.g. collations are handled when parsing * StructField) */ static DataType parseDataType(JsonNode json) { switch (json.getNodeType()) { case STRING: // simple types are stored as just a string return nameToType(json.textValue()); case OBJECT: // complex types (array, map, or struct are stored as JSON objects) String type = getStringField(json, "type"); switch (type) { case "struct": return parseStructType(json); case "array": return parseArrayType(json); case "map": return parseMapType(json); // No default case here; fall through to the following error when no match } default: throw new IllegalArgumentException( String.format( "Could not parse the following JSON as a valid Delta data type:\n%s", json)); } } /** * Parses an array * type */ private static ArrayType parseArrayType(JsonNode json) { checkArgument( json.isObject() && json.size() == 3, "Expected JSON object with 3 fields for array data type but got:\n%s", json); boolean containsNull = getBooleanField(json, "containsNull"); DataType dataType = parseDataType(getNonNullField(json, "elementType")); return new ArrayType(dataType, containsNull); } /** * Parses an map type * */ private static MapType parseMapType(JsonNode json) { checkArgument( json.isObject() && json.size() == 4, "Expected JSON object with 4 fields for map data type but got:\n%s", json); boolean valueContainsNull = getBooleanField(json, "valueContainsNull"); DataType keyType = parseDataType(getNonNullField(json, "keyType")); DataType valueType = parseDataType(getNonNullField(json, "valueType")); return new MapType(keyType, valueType, valueContainsNull); } /** * Parses an * struct type */ private static StructType parseStructType(JsonNode json) { checkArgument( json.isObject() && json.size() == 2, "Expected JSON object with 2 fields for struct data type but got:\n%s", json); JsonNode fieldsNode = getNonNullField(json, "fields"); checkArgument(fieldsNode.isArray(), "Expected array for fieldName=%s in:\n%s", "fields", json); Iterator fields = fieldsNode.elements(); List parsedFields = new ArrayList<>(); while (fields.hasNext()) { parsedFields.add(parseStructField(fields.next())); } return new StructType(parsedFields); } /** * Parses an * struct field */ private static StructField parseStructField(JsonNode json) { checkArgument(json.isObject(), "Expected JSON object for struct field"); String name = getStringField(json, "name"); FieldMetadata metadata = parseFieldMetadata(json.get("metadata")); DataType type = parseDataType(getNonNullField(json, "type")); boolean nullable = getBooleanField(json, "nullable"); StructField structField = new StructField(name, type, nullable, metadata); return fixupFieldLevelMetadata(structField, metadata); } /** * * * 
   * // Collated string type field serialized as:
   * {
   *   "name" : "s",
   *   "type" : "string",
   *   "nullable", false,
   *   "metadata" : {
   *     "__COLLATIONS": { "s": "ICU.de_DE" }
   *   }
   * }
   *
   * // Array with collated strings field serialized as:
   * {
   *   "name" : "arr",
   *   "type" : {
   *     "type" : "array",
   *     "elementType" : "string",
   *     "containsNull" : false
   *   }
   *   "nullable" : false,
   *   "metadata" : {
   *     "__COLLATIONS": { "arr.element": "ICU.de_DE" }
   *   }
   * }
   *
*/ private static StructField fixupFieldLevelMetadata( StructField structField, FieldMetadata metadata) { if (structField.getMetadata().contains(StructField.COLLATIONS_METADATA_KEY) || structField.getMetadata().contains(StructField.DELTA_TYPE_CHANGES_KEY)) { FieldMetadata collationsMetadata = metadata.getMetadata(StructField.COLLATIONS_METADATA_KEY); if (collationsMetadata == null) { collationsMetadata = FieldMetadata.empty(); } Map> pathToTypeChanges = parseTypeChangesFromMetadata(metadata); // Don't recurse on Structs because they would have already been constructed/fixed up in this // code path when assembling the child structs. SchemaIterable iterable = SchemaIterable.newSchemaIterableWithIgnoredRecursion( new StructType().add(structField), new Class[] {StructType.class}); Iterator elements = iterable.newMutableIterator(); String fieldName = structField.getName(); while (elements.hasNext()) { SchemaIterable.MutableSchemaElement element = elements.next(); String pathFromAncestor = element.getPathFromNearestStructFieldAncestor(""); String pathWithFieldName = pathFromAncestor.isEmpty() ? fieldName : String.format("%s.%s", fieldName, pathFromAncestor); if (collationsMetadata.contains(pathWithFieldName)) { updateCollation(element, collationsMetadata, pathWithFieldName); } List changes = pathToTypeChanges.get(pathFromAncestor); if (changes != null) { if (element.getField().getDataType().isNested()) { throw new KernelException( format("Invalid data type for type change: \"%s\"", element.getField())); } element.updateField(element.getField().withTypeChanges(changes)); } } structField = iterable .getSchema() .at(0) .withNewMetadata( FieldMetadata.builder() .fromMetadata(metadata) .remove(StructField.COLLATIONS_METADATA_KEY) .remove(StructField.DELTA_TYPE_CHANGES_KEY) .build()); } return structField; } private static void updateCollation( SchemaIterable.MutableSchemaElement element, FieldMetadata collationsMetadata, String pathFromAncestor) { StructField field = element.getField(); DataType fieldType = field.getDataType(); if (!(fieldType instanceof StringType)) { throw new IllegalArgumentException( format("Invalid data type for collations: \"%s\"", fieldType)); } StringType collatedType = new StringType(collationsMetadata.getString(pathFromAncestor)); element.updateField(field.withDataType(collatedType)); } /** * Processes a FieldMetadata and a map of paths to type changes to create TypeChange objects. This * function parses the type changes from the metadata and organizes them by path. */ private static Map> parseTypeChangesFromMetadata( FieldMetadata metadata) { Map> pathToTypeChanges = new HashMap<>(); if (metadata == null || !metadata.contains(StructField.DELTA_TYPE_CHANGES_KEY)) { return pathToTypeChanges; } FieldMetadata[] typeChangesArray = metadata.getMetadataArray(StructField.DELTA_TYPE_CHANGES_KEY); if (typeChangesArray == null) { return pathToTypeChanges; } for (FieldMetadata changeMetadata : typeChangesArray) { String fromTypeStr = changeMetadata.getString(StructField.FROM_TYPE_KEY); String toTypeStr = changeMetadata.getString(StructField.TO_TYPE_KEY); DataType fromType = nameToType(fromTypeStr); DataType toType = nameToType(toTypeStr); TypeChange typeChange = new TypeChange(fromType, toType); // Empty string is default because it means type change is directly on struct field. String path = ""; if (changeMetadata.contains(StructField.FIELD_PATH_KEY)) { path = changeMetadata.getString(StructField.FIELD_PATH_KEY); } pathToTypeChanges.computeIfAbsent(path, k -> new ArrayList<>()).add(typeChange); } return pathToTypeChanges; } /** Parses a {@link FieldMetadata}. */ private static FieldMetadata parseFieldMetadata(JsonNode json) { if (json == null || json.isNull()) { return FieldMetadata.empty(); } checkArgument(json.isObject(), "Expected JSON object for struct field metadata"); final Iterator> iterator = json.fields(); final FieldMetadata.Builder builder = FieldMetadata.builder(); while (iterator.hasNext()) { Map.Entry entry = iterator.next(); JsonNode value = entry.getValue(); String key = entry.getKey(); if (key.equals(StructField.METADATA_SPEC_KEY)) { builder.putMetadataColumnSpec(key, MetadataColumnSpec.fromString(value.textValue())); } else if (value.isNull()) { builder.putNull(key); } else if (value.isIntegralNumber()) { // covers both int and long builder.putLong(key, value.longValue()); } else if (value.isDouble()) { builder.putDouble(key, value.doubleValue()); } else if (value.isBoolean()) { builder.putBoolean(key, value.booleanValue()); } else if (value.isTextual()) { builder.putString(key, value.textValue()); } else if (value.isObject()) { builder.putFieldMetadata(key, parseFieldMetadata(value)); } else if (value.isArray()) { final Iterator fields = value.elements(); if (!fields.hasNext()) { // If it is an empty array, we cannot infer its element type. // We put an empty Array[Long]. builder.putLongArray(key, new Long[0]); } else { final JsonNode head = fields.next(); if (head.isInt()) { builder.putLongArray( key, buildList(value, node -> (long) node.intValue()).toArray(new Long[0])); } else if (head.isDouble()) { builder.putDoubleArray( key, buildList(value, JsonNode::doubleValue).toArray(new Double[0])); } else if (head.isBoolean()) { builder.putBooleanArray( key, buildList(value, JsonNode::booleanValue).toArray(new Boolean[0])); } else if (head.isTextual()) { builder.putStringArray( key, buildList(value, JsonNode::textValue).toArray(new String[0])); } else if (head.isObject()) { builder.putFieldMetadataArray( key, buildList(value, DataTypeJsonSerDe::parseFieldMetadata) .toArray(new FieldMetadata[0])); } else { throw new IllegalArgumentException( String.format("Unsupported type for Array as field metadata value: %s", value)); } } } else { throw new IllegalArgumentException( String.format("Unsupported type for field metadata value: %s", value)); } } return builder.build(); } /** * For an array JSON node builds a {@link List} using the provided {@code accessor} for each * element. */ private static List buildList(JsonNode json, Function accessor) { List result = new ArrayList<>(); Iterator elements = json.elements(); while (elements.hasNext()) { result.add(accessor.apply(elements.next())); } return result; } private static String FIXED_DECIMAL_REGEX = "decimal\\(\\s*(\\d+)\\s*,\\s*(\\-?\\d+)\\s*\\)"; private static Pattern FIXED_DECIMAL_PATTERN = Pattern.compile(FIXED_DECIMAL_REGEX); // Geometry patterns private static final String GEOMETRY_REGEX = "geometry\\(\\s*([\\w]+:-?[\\w]+)\\s*\\)"; private static final Pattern GEOMETRY_PATTERN = Pattern.compile(GEOMETRY_REGEX); // Geography patterns private static final String GEOGRAPHY_CRS_ALG_REGEX = "geography\\(\\s*(\\w+:-?\\w+)\\s*,\\s*(\\w+)\\s*\\)"; private static final Pattern GEOGRAPHY_CRS_ALG_PATTERN = Pattern.compile(GEOGRAPHY_CRS_ALG_REGEX); private static final String GEOGRAPHY_CRS_REGEX = "geography\\(\\s*(\\w+:-?\\w+)\\s*\\)"; private static final Pattern GEOGRAPHY_CRS_PATTERN = Pattern.compile(GEOGRAPHY_CRS_REGEX); private static final String GEOGRAPHY_ALG_REGEX = "geography\\(\\s*(\\w+)\\s*\\)"; private static final Pattern GEOGRAPHY_ALG_PATTERN = Pattern.compile(GEOGRAPHY_ALG_REGEX); /** Parses primitive string type names to a {@link DataType} */ private static DataType nameToType(String name) { if (BasePrimitiveType.isPrimitiveType(name)) { return BasePrimitiveType.createPrimitive(name); } else if (name.equals("decimal")) { return DecimalType.USER_DEFAULT; } else if (name.equals("geometry")) { return GeometryType.ofDefault(); } else if (name.equals("geography")) { return GeographyType.ofDefault(); } else if ("void".equalsIgnoreCase(name)) { // Earlier versions of Delta had VOID type which is not specified in Delta Protocol. // It is not readable or writable. Throw a user-friendly error message. throw DeltaErrors.voidTypeEncountered(); } else { // decimal has a special pattern with a precision and scale Matcher decimalMatcher = FIXED_DECIMAL_PATTERN.matcher(name); if (decimalMatcher.matches()) { int precision = Integer.parseInt(decimalMatcher.group(1)); int scale = Integer.parseInt(decimalMatcher.group(2)); return new DecimalType(precision, scale); } // geometry has a special pattern with an SRID Matcher geometryMatcher = GEOMETRY_PATTERN.matcher(name); if (geometryMatcher.matches()) { String srid = geometryMatcher.group(1); return GeometryType.ofSRID(srid); } // geography has different patterns: // 1. geography(, ) - both specified // 2. geography() - SRID specified (contains colon) // 3. geography() - algorithm specified (no colon) // First check for both CRS and algorithm (contains comma) Matcher geographyCrsAlgMatcher = GEOGRAPHY_CRS_ALG_PATTERN.matcher(name); if (geographyCrsAlgMatcher.matches()) { String srid = geographyCrsAlgMatcher.group(1); String algorithm = geographyCrsAlgMatcher.group(2); return new GeographyType(srid, algorithm); } // Check for CRS pattern (contains colon) Matcher geographyCrsMatcher = GEOGRAPHY_CRS_PATTERN.matcher(name); if (geographyCrsMatcher.matches()) { String srid = geographyCrsMatcher.group(1); return GeographyType.ofSRID(srid); } // Check for algorithm pattern (no colon) Matcher geographyAlgMatcher = GEOGRAPHY_ALG_PATTERN.matcher(name); if (geographyAlgMatcher.matches()) { String algorithm = geographyAlgMatcher.group(1); return GeographyType.ofAlgorithm(algorithm); } // We have encountered a type that is beyond the specification of the protocol // checks. This must be an invalid type (according to protocol) and // not an unsupported data type by Kernel. throw new IllegalArgumentException( String.format("%s is not a supported delta data type", name)); } } private static JsonNode getNonNullField(JsonNode rootNode, String fieldName) { JsonNode node = rootNode.get(fieldName); if (node == null || node.isNull()) { throw new IllegalArgumentException( String.format("Expected non-null for fieldName=%s in:\n%s", fieldName, rootNode)); } return node; } private static String getStringField(JsonNode rootNode, String fieldName) { JsonNode node = getNonNullField(rootNode, fieldName); checkArgument( node.isTextual(), "Expected string for fieldName=%s in:\n%s", fieldName, rootNode); return node.textValue(); // double check this only works for string values! and isTextual()! } private static boolean getBooleanField(JsonNode rootNode, String fieldName) { JsonNode node = getNonNullField(rootNode, fieldName); checkArgument( node.isBoolean(), "Expected boolean for fieldName=%s in:\n%s", fieldName, rootNode); return node.booleanValue(); } protected static class StructTypeSerializer extends StdSerializer { public StructTypeSerializer() { super(StructType.class); } @Override public void serialize(StructType structType, JsonGenerator gen, SerializerProvider provider) throws IOException { writeDataType(gen, structType); } } private static void writeDataType(JsonGenerator gen, DataType dataType) throws IOException { if (dataType instanceof StructType) { writeStructType(gen, (StructType) dataType); } else if (dataType instanceof ArrayType) { writeArrayType(gen, (ArrayType) dataType); } else if (dataType instanceof MapType) { writeMapType(gen, (MapType) dataType); } else if (dataType instanceof DecimalType) { DecimalType decimalType = (DecimalType) dataType; gen.writeString(format("decimal(%d,%d)", decimalType.getPrecision(), decimalType.getScale())); } else if (dataType instanceof StringType) { // Always serialize `StringType` as "string" without including collation info, since collation // is stored in the field metadata. gen.writeString("string"); } else if (dataType instanceof GeometryType) { GeometryType geometryType = (GeometryType) dataType; gen.writeString(geometryType.simpleString()); } else if (dataType instanceof GeographyType) { GeographyType geographyType = (GeographyType) dataType; gen.writeString(geographyType.simpleString()); } else { gen.writeString(dataType.toString()); } } private static void writeArrayType(JsonGenerator gen, ArrayType arrayType) throws IOException { gen.writeStartObject(); gen.writeStringField("type", "array"); gen.writeFieldName("elementType"); writeDataType(gen, arrayType.getElementType()); gen.writeBooleanField("containsNull", arrayType.containsNull()); gen.writeEndObject(); } private static void writeMapType(JsonGenerator gen, MapType mapType) throws IOException { gen.writeStartObject(); gen.writeStringField("type", "map"); gen.writeFieldName("keyType"); writeDataType(gen, mapType.getKeyType()); gen.writeFieldName("valueType"); writeDataType(gen, mapType.getValueType()); gen.writeBooleanField("valueContainsNull", mapType.isValueContainsNull()); gen.writeEndObject(); } private static void writeStructType(JsonGenerator gen, StructType structType) throws IOException { gen.writeStartObject(); gen.writeStringField("type", "struct"); gen.writeArrayFieldStart("fields"); for (StructField field : structType.fields()) { writeStructField(gen, field); } gen.writeEndArray(); gen.writeEndObject(); } private static void writeStructField(JsonGenerator gen, StructField field) throws IOException { gen.writeStartObject(); gen.writeStringField("name", field.getName()); gen.writeFieldName("type"); writeDataType(gen, field.getDataType()); gen.writeBooleanField("nullable", field.isNullable()); gen.writeFieldName("metadata"); writeFieldMetadata(gen, field.getMetadata()); gen.writeEndObject(); } private static void writeFieldMetadata(JsonGenerator gen, FieldMetadata metadata) throws IOException { gen.writeStartObject(); for (Map.Entry entry : metadata.getEntries().entrySet()) { gen.writeFieldName(entry.getKey()); Object value = entry.getValue(); if (value instanceof Long) { gen.writeNumber((Long) value); } else if (value instanceof Double) { gen.writeNumber((Double) value); } else if (value instanceof Boolean) { gen.writeBoolean((Boolean) value); } else if (value instanceof String) { gen.writeString((String) value); } else if (value instanceof MetadataColumnSpec) { gen.writeString(value.toString()); } else if (value instanceof FieldMetadata) { writeFieldMetadata(gen, (FieldMetadata) value); } else if (value instanceof Long[]) { gen.writeStartArray(); for (Long v : (Long[]) value) { gen.writeNumber(v); } gen.writeEndArray(); } else if (value instanceof Double[]) { gen.writeStartArray(); for (Double v : (Double[]) value) { gen.writeNumber(v); } gen.writeEndArray(); } else if (value instanceof Boolean[]) { gen.writeStartArray(); for (Boolean v : (Boolean[]) value) { gen.writeBoolean(v); } gen.writeEndArray(); } else if (value instanceof String[]) { gen.writeStartArray(); for (String v : (String[]) value) { gen.writeString(v); } gen.writeEndArray(); } else if (value instanceof FieldMetadata[]) { gen.writeStartArray(); for (FieldMetadata v : (FieldMetadata[]) value) { writeFieldMetadata(gen, v); } gen.writeEndArray(); } else if (value == null) { gen.writeNull(); } else { throw new IllegalArgumentException( format("Unsupported type for field metadata value: %s", value)); } } gen.writeEndObject(); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/types/TypeWideningChecker.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.types; import io.delta.kernel.types.*; /** * Utility class for checking type widening compatibility according to the Delta protocol. * *

The Type Widening feature enables changing the type of a column or field in an existing Delta * table to a wider type. This class implements the checks required by the protocol to ensure that * only supported type changes are allowed. * *

Supported type changes as per protocol: * *

    *
  • Integer widening: {@code Byte -> Short -> Int -> Long} *
  • Floating-point widening: {@code Float -> Double} *
  • Floating-point widening: {@code Byte, Short or Int -> Double} *
  • Date widening: {@code Date -> Timestamp without timezone} *
  • Decimal widening: {@code Decimal(p, s) -> Decimal(p + k1, s + k2)} where {@code k1 >= k2 >= * 0} *
  • Decimal widening: {@code Byte, Short or Int -> Decimal(10 + k1, k2)} where {@code k1 >= k2 * >= 0} *
  • Decimal widening: {@code Long -> Decimal(20 + k1, k2)} where {@code k1 >= k2 >= 0} *
*/ public class TypeWideningChecker { private TypeWideningChecker() {} /** * Checks if a type change from sourceType to targetType is a supported widening operation. * * @param sourceType The original data type * @param targetType The target data type to widen to * @return true if the type change is a supported widening operation (or the types are equal), * false otherwise */ public static boolean isWideningSupported(DataType sourceType, DataType targetType) { // Iceberg V2 type widening is a strict subset of Delta type widening if (isIcebergV2Compatible(sourceType, targetType)) { return true; } // Floating-point widening: Byte, Short or Int -> Double if (isIntegerToDoubleWidening(sourceType, targetType)) { return true; } // Date widening: Date -> Timestamp without timezone if (isDateToTimestampNtzWidening(sourceType, targetType)) { return true; } // Decimal widening if (isDecimalWidening(sourceType, targetType)) { return true; } // No supported widening found return false; } /** * Checks if the type change is supported by Iceberg V2 schema evolution. This is required when * Iceberg compatibility is enabled. * * @param sourceType The original data type * @param targetType The target data type to widen to * @return true if the type change is supported by Iceberg V2 (sourceType == targetType returns * true), false otherwise */ public static boolean isIcebergV2Compatible(DataType sourceType, DataType targetType) { // If types are the same, it's not a widening operation if (sourceType.equals(targetType)) { return true; } // Integer widening: Byte -> Short -> Int -> Long if (isIntegerWidening(sourceType, targetType)) { return true; } // Floating-point widening: Float -> Double if (isFloatToDoubleWidening(sourceType, targetType)) { return true; } // Check if it's a decimal widening with scale increase if (sourceType instanceof DecimalType && targetType instanceof DecimalType) { DecimalType sourceDecimal = (DecimalType) sourceType; DecimalType targetDecimal = (DecimalType) targetType; // If scale changes, are not supported by Iceberg if (targetDecimal.getScale() != sourceDecimal.getScale()) { return false; } // Precision increase with same scale is supported. return targetDecimal.getPrecision() >= sourceDecimal.getPrecision(); } // No other supported widening for Iceberg return false; } /** Checks if the type change is an integer widening (Byte -> Short -> Int -> Long). */ private static boolean isIntegerWidening(DataType sourceType, DataType targetType) { if (sourceType instanceof ByteType) { return targetType instanceof ShortType || targetType instanceof IntegerType || targetType instanceof LongType; } else if (sourceType instanceof ShortType) { return targetType instanceof IntegerType || targetType instanceof LongType; } else if (sourceType instanceof IntegerType) { return targetType instanceof LongType; } return false; } /** Checks if the type change is a Float to Double widening. */ private static boolean isFloatToDoubleWidening(DataType sourceType, DataType targetType) { return sourceType instanceof FloatType && targetType instanceof DoubleType; } /** Checks if the type change is an integer to Double widening. */ private static boolean isIntegerToDoubleWidening(DataType sourceType, DataType targetType) { return (sourceType instanceof ByteType || sourceType instanceof ShortType || sourceType instanceof IntegerType) && targetType instanceof DoubleType; } /** Checks if the type change is a Date to TimestampNTZ widening. */ private static boolean isDateToTimestampNtzWidening(DataType sourceType, DataType targetType) { return sourceType instanceof DateType && targetType instanceof TimestampNTZType; } /** Checks if the type change is a supported decimal widening. */ private static boolean isDecimalWidening(DataType sourceType, DataType targetType) { // Decimal(p, s) -> Decimal(p + k1, s + k2) where k1 >= k2 >= 0 if (sourceType instanceof DecimalType && targetType instanceof DecimalType) { DecimalType sourceDecimal = (DecimalType) sourceType; DecimalType targetDecimal = (DecimalType) targetType; int precisionDiff = targetDecimal.getPrecision() - sourceDecimal.getPrecision(); int scaleDiff = targetDecimal.getScale() - sourceDecimal.getScale(); return precisionDiff >= scaleDiff && scaleDiff >= 0; } // Byte, Short or Int -> Decimal(10 + k1, k2) where k1 >= k2 >= 0 if ((sourceType instanceof ByteType || sourceType instanceof ShortType || sourceType instanceof IntegerType) && targetType instanceof DecimalType) { DecimalType targetDecimal = (DecimalType) targetType; int basePrecision = 10; return targetDecimal.getPrecision() >= basePrecision && (targetDecimal.getPrecision() - basePrecision) >= targetDecimal.getScale() && targetDecimal.getScale() >= 0; } // Long -> Decimal(20 + k1, k2) where k1 >= k2 >= 0 if (sourceType instanceof LongType && targetType instanceof DecimalType) { DecimalType targetDecimal = (DecimalType) targetType; int basePrecision = 20; return targetDecimal.getPrecision() >= basePrecision && (targetDecimal.getPrecision() - basePrecision) >= targetDecimal.getScale() && targetDecimal.getScale() >= 0; } return false; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/CaseInsensitiveMap.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import java.util.Collection; import java.util.HashMap; import java.util.Locale; import java.util.Map; import java.util.Set; /** * A map that is case-insensitive in its keys. This map is not thread-safe, and key is * case-insensitive and of string type. * * @param */ public class CaseInsensitiveMap implements Map { private final Map innerMap = new HashMap<>(); @Override public V get(Object key) { return innerMap.get(toLowerCase(key)); } @Override public V put(String key, V value) { return innerMap.put(toLowerCase(key), value); } @Override public void putAll(Map m) { // behavior of this method is not defined on how to handle duplicates // don't support this use case, as it is not needed in Kernel throw new UnsupportedOperationException("putAll"); } @Override public V remove(Object key) { return innerMap.remove(toLowerCase(key)); } @Override public boolean containsKey(Object key) { return innerMap.containsKey(toLowerCase(key)); } @Override public boolean containsValue(Object value) { return innerMap.containsValue(value); } @Override public Set keySet() { // no need to convert to lower case here as the inserted keys are already in lower case return innerMap.keySet(); } @Override public Set> entrySet() { // no need to convert to lower case here as the inserted keys are already in lower case return innerMap.entrySet(); } @Override public Collection values() { return innerMap.values(); } @Override public int size() { return innerMap.size(); } @Override public boolean isEmpty() { return innerMap.isEmpty(); } @Override public void clear() { innerMap.clear(); } private String toLowerCase(Object key) { if (key == null) { return null; } checkArgument(key instanceof String, "Key must be a string"); return ((String) key).toLowerCase(Locale.ROOT); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/Clock.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; /** An interface to represent clocks, so that they can be mocked out in unit tests. */ public interface Clock { /** @return Current system time, in ms. */ long getTimeMillis(); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/ColumnMapping.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; import static io.delta.kernel.internal.DeltaErrors.columnNotFoundInSchema; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Collections.singletonMap; import io.delta.kernel.data.Row; import io.delta.kernel.exceptions.InvalidConfigurationValueException; import io.delta.kernel.expressions.Column; import io.delta.kernel.internal.TableConfig; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.data.TransactionStateRow; import io.delta.kernel.internal.icebergcompat.IcebergCompatMetadataValidatorAndUpdater; import io.delta.kernel.types.*; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.stream.Collectors; /** Utilities related to the column mapping feature. */ public class ColumnMapping { private ColumnMapping() {} public enum ColumnMappingMode { NONE("none"), ID("id"), NAME("name"); public final String value; ColumnMappingMode(String value) { this.value = value; } public static ColumnMappingMode fromTableConfig(String modeString) { for (ColumnMappingMode mode : ColumnMappingMode.values()) { if (mode.value.equalsIgnoreCase(modeString)) { return mode; } } throw new InvalidConfigurationValueException( COLUMN_MAPPING_MODE_KEY, modeString, String.format("Needs to be one of: %s.", Arrays.toString(ColumnMappingMode.values()))); } @Override public String toString() { return this.value; } } private enum SchemaConversionDirection { LOGICAL_TO_PHYSICAL, PHYSICAL_TO_LOGICAL } public static final String COLUMN_MAPPING_MODE_KEY = "delta.columnMapping.mode"; public static final String COLUMN_MAPPING_PHYSICAL_NAME_KEY = "delta.columnMapping.physicalName"; public static final String COLUMN_MAPPING_ID_KEY = "delta.columnMapping.id"; public static final String COLUMN_MAPPING_NESTED_IDS_KEY = "delta.columnMapping.nested.ids"; public static final String PARQUET_FIELD_ID_KEY = "parquet.field.id"; public static final String PARQUET_FIELD_NESTED_IDS_METADATA_KEY = "parquet.field.nested.ids"; public static final String COLUMN_MAPPING_MAX_COLUMN_ID_KEY = "delta.columnMapping.maxColumnId"; ///////////////// // Public APIs // ///////////////// /** * Returns the column mapping mode from the given configuration. * * @param configuration Configuration * @return Column mapping mode. One of ("none", "name", "id") */ public static ColumnMappingMode getColumnMappingMode(Map configuration) { return Optional.ofNullable(configuration.get(COLUMN_MAPPING_MODE_KEY)) .map(ColumnMappingMode::fromTableConfig) .orElse(ColumnMappingMode.NONE); } /** * Helper method that converts the logical schema (requested by the connector) to physical schema * of the data stored in data files based on the table's column mapping mode. Field-id column * metadata is preserved when cmMode = ID, all column metadata is otherwise removed. * *

We require {@code fullSchema} in addition to the pruned schema we want to convert since we * need the complete field metadata as it is stored in the schema in the _delta_log. We cannot be * sure (and do not enforce) that this metadata is preserved by the connector. * * @param prunedSchema the logical read schema requested by the connector * @param fullSchema the full delta schema (with complete metadata) as read from the _delta_log * @param columnMappingMode Column mapping mode */ public static StructType convertToPhysicalSchema( StructType prunedSchema, StructType fullSchema, ColumnMappingMode columnMappingMode) { switch (columnMappingMode) { case NONE: return prunedSchema; case ID: // fall through case NAME: boolean includeFieldIds = columnMappingMode == ColumnMappingMode.ID; return convertToPhysicalSchema(prunedSchema, fullSchema, includeFieldIds); default: throw new UnsupportedOperationException( "Unsupported column mapping mode: " + columnMappingMode); } } /** * Converts a logical column to a physical column based on the table's column mapping mode. The * field-id metadata is preserved when cmMode = ID, all column metadata is otherwise removed. * *

We require {@code fullSchema} in addition to the logical field we want to convert since we * need the complete field metadata as it is stored in the schema in the _delta_log. We cannot be * sure (and do not enforce) that this metadata is preserved by the connector. * * @param logicalField the logical read column requested by the connector * @param fullSchema the full delta schema (with complete metadata) as read from the _delta_log * @param columnMappingMode Column mapping mode */ public static StructField convertToPhysicalColumn( StructField logicalField, StructType fullSchema, ColumnMappingMode columnMappingMode) { switch (columnMappingMode) { case NONE: return logicalField; case ID: // fall through case NAME: boolean includeFieldIds = columnMappingMode == ColumnMappingMode.ID; return convertToPhysicalColumn(logicalField, fullSchema, includeFieldIds); default: throw new UnsupportedOperationException( "Unsupported column mapping mode: " + columnMappingMode); } } /** Returns the physical name for a given {@link StructField} */ public static String getPhysicalName(StructField field) { if (hasPhysicalName(field)) { return field.getMetadata().getString(COLUMN_MAPPING_PHYSICAL_NAME_KEY); } else { return field.getName(); } } /** Returns the column id for a given {@link StructField} */ public static int getColumnId(StructField field) { checkArgument( field.getMetadata().contains(COLUMN_MAPPING_ID_KEY), "Field does not have column id set in it's metadata"); return field.getMetadata().getLong(COLUMN_MAPPING_ID_KEY).intValue(); } public static void verifyColumnMappingChange( Map oldConfig, Map newConfig) { ColumnMappingMode oldMappingMode = getColumnMappingMode(oldConfig); ColumnMappingMode newMappingMode = getColumnMappingMode(newConfig); checkArgument( validModeChange(oldMappingMode, newMappingMode), "Changing column mapping mode from '%s' to '%s' is not supported", oldMappingMode, newMappingMode); } public static boolean isColumnMappingModeEnabled(ColumnMappingMode columnMappingMode) { return columnMappingMode == ColumnMappingMode.ID || columnMappingMode == ColumnMappingMode.NAME; } /** * Updates the column mapping metadata if needed based on the column mapping mode and whether the * icebergCompatV2 is enabled. If column mapping/iceberg compat info is already present in the * metadata, this method does nothing and returns an empty Optional. Callers can avoid updating * the metadata if the metadata has not changed. * * @param metadata Current metadata. * @param isNewTable Whether this is part of a commit that sets the mapping mode on a new table. * @return Optional of the updated metadata if it has changed, Optional.empty() otherwise. */ public static Optional updateColumnMappingMetadataIfNeeded( Metadata metadata, boolean isNewTable) { ColumnMappingMode columnMappingMode = getColumnMappingMode(metadata.getConfiguration()); switch (columnMappingMode) { case NONE: return Optional.empty(); case ID: // fall through case NAME: return assignColumnIdAndPhysicalName(metadata, isNewTable); default: throw new UnsupportedOperationException( "Unsupported column mapping mode: " + columnMappingMode); } } /** Returns the physical column and data type for a given logical column based on the schema. */ public static Tuple2 getPhysicalColumnNameAndDataType( StructType schema, Column logicalColumn) { return convertColumnName(schema, logicalColumn, SchemaConversionDirection.LOGICAL_TO_PHYSICAL); } /** Returns the logical column and data type for a given physical column based on the schema. */ public static Tuple2 getLogicalColumnNameAndDataType( StructType schema, Column physicalColumn) { return convertColumnName(schema, physicalColumn, SchemaConversionDirection.PHYSICAL_TO_LOGICAL); } /** * Utility method to block writing into a table with column mapping enabled. Currently Kernel only * supports the metadata updates on tables with column mapping enabled. Data writes into such * tables using the data transformation APIs provided by the Kernel are not supported yet. */ public static void blockIfColumnMappingEnabled(Row transactionState) { ColumnMapping.ColumnMappingMode columnMappingMode = TransactionStateRow.getColumnMappingMode(transactionState); if (columnMappingMode != ColumnMapping.ColumnMappingMode.NONE) { throw new UnsupportedOperationException( "Writing into column mapping enabled table is not supported yet."); } } //////////////////////////// // Private Helper Methods // //////////////////////////// /** * Common helper method for column name conversion between logical and physical representations. * * @param schema The schema to traverse * @param inputColumn The column to convert * @param conversionDirection The direction of schema conversion, either from logical to physical * or physical to logical * @return Tuple of the converted column and its data type */ private static Tuple2 convertColumnName( StructType schema, Column inputColumn, SchemaConversionDirection conversionDirection) { Function sourceNameExtractor; Function targetNameExtractor; switch (conversionDirection) { case LOGICAL_TO_PHYSICAL: sourceNameExtractor = StructField::getName; targetNameExtractor = ColumnMapping::getPhysicalName; break; case PHYSICAL_TO_LOGICAL: sourceNameExtractor = ColumnMapping::getPhysicalName; targetNameExtractor = StructField::getName; break; default: throw new IllegalArgumentException("Unknown conversion direction: " + conversionDirection); } final List outputNameParts = new ArrayList<>(); DataType currentType = schema; // Traverse through each level to resolve the corresponding name mapping for (String inputNamePart : inputColumn.getNames()) { if (!(currentType instanceof StructType)) { throw columnNotFoundInSchema(inputColumn, schema); } final StructType structType = (StructType) currentType; // Find the field that matches the input name using the appropriate matching function final StructField field = structType.fields().stream() .filter(f -> sourceNameExtractor.apply(f).equalsIgnoreCase(inputNamePart)) .findFirst() .orElseThrow(() -> columnNotFoundInSchema(inputColumn, schema)); outputNameParts.add(targetNameExtractor.apply(field)); currentType = field.getDataType(); } return new Tuple2<>(new Column(outputNameParts.toArray(new String[0])), currentType); } /** Visible for testing */ static int findMaxColumnId(StructType schema) { return new SchemaIterable(schema) .stream() .mapToInt( e -> { int columnId = hasColumnId(e.getField()) ? getColumnId(e.getField()) : 0; int nestedMaxId = hasNestedColumnIds(e.getField()) ? getMaxNestedColumnId(e.getField()) : 0; return Math.max(columnId, nestedMaxId); }) .max() .orElse(0); } static boolean hasColumnId(StructField field) { return field.getMetadata().contains(COLUMN_MAPPING_ID_KEY); } static boolean hasPhysicalName(StructField field) { return field.getMetadata().contains(COLUMN_MAPPING_PHYSICAL_NAME_KEY); } /** * Utility method to convert the given logical schema to physical schema, recursively converting * sub-types in case of complex types. When {@code includeFieldId} is true, converted physical * schema will have field ids in the metadata. Column metadata is otherwise removed. */ private static StructType convertToPhysicalSchema( StructType prunedSchema, StructType fullSchema, boolean includeFieldId) { StructType newSchema = new StructType(); for (StructField prunedField : prunedSchema.fields()) { newSchema = newSchema.add(convertToPhysicalColumn(prunedField, fullSchema, includeFieldId)); } return newSchema; } /** * Utility method to convert the given logical field to a physical field, recursively converting * sub-types in case of complex types. When {@code includeFieldId} is true, converted physical * schema will have field ids in the metadata. Column metadata is otherwise removed. */ private static StructField convertToPhysicalColumn( StructField logicalField, StructType fullSchema, boolean includeFieldId) { StructField completeField = fullSchema.get(logicalField.getName()); DataType physicalType = convertToPhysicalType( logicalField.getDataType(), completeField.getDataType(), includeFieldId); String physicalName = completeField.getMetadata().getString(COLUMN_MAPPING_PHYSICAL_NAME_KEY); if (!includeFieldId) { return new StructField(physicalName, physicalType, logicalField.isNullable()); } Long fieldId = completeField.getMetadata().getLong(COLUMN_MAPPING_ID_KEY); FieldMetadata.Builder builder = FieldMetadata.builder().putLong(PARQUET_FIELD_ID_KEY, fieldId); // convertToPhysicalSchema(..) gets called when trying to find the read schema // for the Parquet reader. This currently assumes that if the nested field IDs for // the 'element' and 'key'/'value' fields of Arrays/Maps haven been written, // then IcebergCompatV2 is enabled because the schema we are looking at is from // the DeltaLog and has nested field IDs setup if (hasNestedColumnIds(completeField)) { builder.putFieldMetadata( PARQUET_FIELD_NESTED_IDS_METADATA_KEY, getNestedColumnIds(completeField)); } return new StructField(physicalName, physicalType, logicalField.isNullable(), builder.build()); } private static DataType convertToPhysicalType( DataType logicalType, DataType physicalType, boolean includeFieldId) { if (logicalType instanceof StructType) { return convertToPhysicalSchema( (StructType) logicalType, (StructType) physicalType, includeFieldId); } else if (logicalType instanceof ArrayType) { ArrayType logicalArrayType = (ArrayType) logicalType; return new ArrayType( convertToPhysicalType( logicalArrayType.getElementType(), ((ArrayType) physicalType).getElementType(), includeFieldId), logicalArrayType.containsNull()); } else if (logicalType instanceof MapType) { MapType logicalMapType = (MapType) logicalType; MapType physicalMapType = (MapType) physicalType; return new MapType( convertToPhysicalType( logicalMapType.getKeyType(), physicalMapType.getKeyType(), includeFieldId), convertToPhysicalType( logicalMapType.getValueType(), physicalMapType.getValueType(), includeFieldId), logicalMapType.isValueContainsNull()); } return logicalType; } private static boolean validModeChange(ColumnMappingMode oldMode, ColumnMappingMode newMode) { // only upgrade from none to name mapping is allowed return oldMode.equals(newMode) || (oldMode == ColumnMappingMode.NONE && newMode == ColumnMappingMode.NAME); } /** * For each column/field in a {@link Metadata}'s schema, assign an id using the current maximum id * as the basis and increment from there. Additionally, assign a physical name based on a random * UUID or re-use the old display name if the mapping mode is updated on an existing table. When * `icebergWriterCompatV1` is enabled, we assign physical names as 'col-[colId]'. * * @param metadata The new metadata to assign ids and physical names to * @param isNewTable whether this is part of a commit that sets the mapping mode on a new table * @return Optional {@link Metadata} with a new schema where ids and physical names have been * assigned if the schema has changed, returns Optional.empty() otherwise */ private static Optional assignColumnIdAndPhysicalName( Metadata metadata, boolean isNewTable) { StructType oldSchema = metadata.getSchema(); // When icebergWriterCompatV1 or icebergWriterCompatV3 is enabled we require // physicalName='col-[columnId]' boolean useColumnIdForPhysicalName = TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.fromMetadata(metadata) || TableConfig.ICEBERG_WRITER_COMPAT_V3_ENABLED.fromMetadata(metadata); // This is the maxColumnId to use when assigning any new field-ids; we update this as we // traverse the schema and after traversal this is the value that should be stored in the // metadata. Note - this could be greater than the current value stored in the metadata if // the connector has added new fields with field-ids AtomicInteger maxColumnId = new AtomicInteger( Math.max( Integer.parseInt( metadata .getConfiguration() .getOrDefault(COLUMN_MAPPING_MAX_COLUMN_ID_KEY, "0")), findMaxColumnId(oldSchema))); StructType newSchema = new StructType(); for (StructField field : oldSchema.fields()) { newSchema = newSchema.add( transformAndAssignColumnIdAndPhysicalName( assignColumnIdAndPhysicalNameToField( field, maxColumnId, isNewTable, useColumnIdForPhysicalName), maxColumnId, isNewTable, useColumnIdForPhysicalName)); } if (IcebergCompatMetadataValidatorAndUpdater.isIcebergCompatEnabled(metadata)) { newSchema = rewriteFieldIdsForIceberg(newSchema, maxColumnId); } // The maxColumnId in the metadata may be out-of-date either due to field-id assignment // performed in this function, or due to connector adding new fields boolean shouldUpdateMaxId = TableConfig.COLUMN_MAPPING_MAX_COLUMN_ID.fromMetadata(metadata) != maxColumnId.get(); // We are comparing the old schema with the new schema to determine if the schema has changed. // If this becomes hotspot, we can consider updating the methods to pass around AtomicBoolean // to track if the schema has changed. It is a bit convoluted to pass around and update the // AtomicBoolean in the recursive and multiple methods. if (oldSchema.equals(newSchema) && !shouldUpdateMaxId) { return Optional.empty(); } String maxFieldId = Integer.toString(maxColumnId.get()); return Optional.of( metadata .withNewSchema(newSchema) .withMergedConfiguration(singletonMap(COLUMN_MAPPING_MAX_COLUMN_ID_KEY, maxFieldId))); } /** * Recursively visits each nested struct / array / map type and assigns an id using the current * maximum id as the basis and increments from there. Additionally, assigns a physical name based * on a random UUID or re-uses the old display name if the mapping mode is updated on an existing * table. Note that key / value fields of a map and the element field of an array are not assigned * an id / physical name. Such functionality is actually being handled by {@link * ColumnMapping#rewriteFieldIdsForIceberg(StructType, AtomicInteger)}. * * @param field The current {@link StructField} * @param maxColumnId Holds the current maximum id. Value is incremented whenever the current max * id value is used to keep the current value always the max id * @param isNewTable Whether this is a new or an existing table. For existing tables the physical * name will be re-used from the old display name * @param useColumnIdForPhysicalName Whether we should assign physical names to 'col-[colId]'. * When false uses the default behavior described above. * @return A new {@link StructField} with updated metadata under the {@link * ColumnMapping#COLUMN_MAPPING_ID_KEY} and the {@link * ColumnMapping#COLUMN_MAPPING_PHYSICAL_NAME_KEY} keys */ private static StructField transformAndAssignColumnIdAndPhysicalName( StructField field, AtomicInteger maxColumnId, boolean isNewTable, boolean useColumnIdForPhysicalName) { DataType dataType = field.getDataType(); if (dataType instanceof StructType) { StructType type = (StructType) dataType; StructType schema = new StructType(); for (StructField f : type.fields()) { schema = schema.add( transformAndAssignColumnIdAndPhysicalName( assignColumnIdAndPhysicalNameToField( f, maxColumnId, isNewTable, useColumnIdForPhysicalName), maxColumnId, isNewTable, useColumnIdForPhysicalName)); } return new StructField(field.getName(), schema, field.isNullable(), field.getMetadata()); } else if (dataType instanceof ArrayType) { ArrayType type = (ArrayType) dataType; StructField elementField = transformAndAssignColumnIdAndPhysicalName( type.getElementField(), maxColumnId, isNewTable, useColumnIdForPhysicalName); return new StructField( field.getName(), new ArrayType(elementField), field.isNullable(), field.getMetadata()); } else if (dataType instanceof MapType) { MapType type = (MapType) dataType; StructField key = transformAndAssignColumnIdAndPhysicalName( type.getKeyField(), maxColumnId, isNewTable, useColumnIdForPhysicalName); StructField value = transformAndAssignColumnIdAndPhysicalName( type.getValueField(), maxColumnId, isNewTable, useColumnIdForPhysicalName); return new StructField( field.getName(), new MapType(key, value), field.isNullable(), field.getMetadata()); } return field; } /** * Assigns an id using the current maximum id as the basis and increments from there. * Additionally, assigns a physical name based on a random UUID or re-uses the old display name if * the mapping mode is updated on an existing table. * * @param field The current {@link StructField} to assign an id / physical name to * @param maxColumnId Holds the current maximum id. Value is incremented whenever the current max * id value is used to keep the current value always the max id * @param isNewTable Whether this is a new or an existing table. For existing tables the physical * name will be re-used from the old display name * @param useColumnIdForPhysicalName Whether we should assign physical names to 'col-[colId]'. * When false uses the default behavior described above. * @return A new {@link StructField} with updated metadata under the {@link * ColumnMapping#COLUMN_MAPPING_ID_KEY} and the {@link * ColumnMapping#COLUMN_MAPPING_PHYSICAL_NAME_KEY} keys */ private static StructField assignColumnIdAndPhysicalNameToField( StructField field, AtomicInteger maxColumnId, boolean isNewTable, boolean useColumnIdForPhysicalName) { if (hasColumnId(field) ^ hasPhysicalName(field)) { // If a connector is providing column mapping metadata in the given schema we require it to be // complete throw new IllegalArgumentException( String.format( "Both columnId and physicalName must be present if one is present. " + "Found this field with incomplete column mapping metadata: %s", field)); } if (!hasColumnId(field)) { field = field.withNewMetadata( FieldMetadata.builder() .fromMetadata(field.getMetadata()) .putLong(COLUMN_MAPPING_ID_KEY, maxColumnId.incrementAndGet()) .build()); } if (!hasPhysicalName(field)) { // re-use old display names as physical names when a table is updated String physicalName; if (useColumnIdForPhysicalName) { long columnId = getColumnId(field); physicalName = String.format("col-%s", columnId); } else { physicalName = isNewTable ? "col-" + UUID.randomUUID() : field.getName(); } field = field.withNewMetadata( FieldMetadata.builder() .fromMetadata(field.getMetadata()) .putString(COLUMN_MAPPING_PHYSICAL_NAME_KEY, physicalName) .build()); } return field; } private static boolean hasNestedColumnIds(StructField field) { return field.getMetadata().contains(COLUMN_MAPPING_NESTED_IDS_KEY); } private static FieldMetadata getNestedColumnIds(StructField field) { return field.getMetadata().getMetadata(COLUMN_MAPPING_NESTED_IDS_KEY); } private static int getMaxNestedColumnId(StructField field) { return getNestedColumnIds(field).getEntries().values().stream() .filter(Long.class::isInstance) .map(Long.class::cast) .max(Comparator.naturalOrder()) .orElse(0L) .intValue(); } /** * Adds the nested field IDs required by Iceberg. * *

In parquet, list-type columns have a nested, implicitly defined {@code element} field and * map-type columns have implicitly defined {@code key} and {@code value} fields. By default, * Spark does not write field IDs for these fields in the parquet files. However, Iceberg requires * these *nested* field IDs to be present. This method rewrites the specified schema to add those * nested field IDs. * *

Nested field IDs are stored in a map as part of the metadata of the *nearest* parent {@link * StructField}. For example, consider the following schema: * *

col1 ARRAY(INT) col2 MAP(INT, INT) col3 STRUCT(a INT, b ARRAY(STRUCT(c INT, d MAP(INT, * INT)))) * *

col1 is a list and so requires one nested field ID for the {@code element} field in parquet. * This nested field ID will be stored in a map that is part of col1's {@link * StructField#getMetadata()}. The same applies to the nested field IDs for col2's implicit {@code * key} and {@code value} fields. col3 itself is a Struct, consisting of an integer field and a * list field named 'b'. The nested field ID for the list of 'b' is stored in b's {@link * StructField#getMetadata()}. Finally, the list type itself is again a struct consisting of an * integer field and a map field named 'd'. The nested field IDs for the map of 'd' are stored in * d's {@link StructField#getMetadata()}. * * @param schema The schema to which nested field IDs should be added * @param startId The first field ID to use for the nested field IDs */ private static StructType rewriteFieldIdsForIceberg(StructType schema, AtomicInteger startId) { StructType newSchema = new StructType(); for (StructField field : schema.fields()) { FieldMetadata.Builder builder = FieldMetadata.builder().fromMetadata(field.getMetadata()); newSchema = newSchema.add( transformSchema( startId, field, "", /** current column path */ builder) .withNewMetadata(builder.build())); } return newSchema; } /** * Recursively visits each nested struct / array / map type and returns a new {@link StructField} * with updated {@link FieldMetadata}. For array / map types the field IDs of their nested * elements are under the {@link ColumnMapping#COLUMN_MAPPING_NESTED_IDS_KEY} key. A concrete * schema example can be seen at {@link ColumnMapping#rewriteFieldIdsForIceberg(StructType, * AtomicInteger)}. * * @param currentFieldId The current maximum field id to increment and use for assignment * @param structField The field where to start from * @param path The current field path relative to the parent field (aka most recent ancestor with * a StructField). An empty path indicates that there's no parent and we're at the root * @param closestStructFieldParentMetadata The metadata builder of the closest struct field parent * where nested IDs will be stored. For StructFields this is the current field. For * map/arrays, it is the closest parent that is a struct field. * @return A new {@link StructField} with updated {@link FieldMetadata} */ private static StructField transformSchema( AtomicInteger currentFieldId, StructField structField, String path, FieldMetadata.Builder closestStructFieldParentMetadata) { DataType dataType = structField.getDataType(); if (dataType instanceof StructType) { StructType type = (StructType) dataType; List fields = type.fields().stream() .map( field -> { FieldMetadata.Builder metadataBuilder = FieldMetadata.builder().fromMetadata(field.getMetadata()); return transformSchema( currentFieldId, field, getPhysicalName(field), metadataBuilder) .withNewMetadata(metadataBuilder.build()); }) .collect(Collectors.toList()); return new StructField( structField.getName(), new StructType(fields), structField.isNullable(), structField.getMetadata()); } else if (dataType instanceof ArrayType) { ArrayType type = (ArrayType) dataType; String basePath = "".equals(path) ? getPhysicalName(structField) : path; // update element type metadata and recurse into element type String elementPath = basePath + "." + type.getElementField().getName(); maybeUpdateFieldId(closestStructFieldParentMetadata, elementPath, currentFieldId); StructField elementType = transformSchema( currentFieldId, type.getElementField(), elementPath, closestStructFieldParentMetadata); return new StructField( structField.getName(), new ArrayType(elementType), structField.isNullable(), structField.getMetadata()); } else if (dataType instanceof MapType) { MapType type = (MapType) dataType; // update key type metadata and recurse into key type String basePath = "".equals(path) ? getPhysicalName(structField) : path; String keyPath = basePath + "." + type.getKeyField().getName(); maybeUpdateFieldId(closestStructFieldParentMetadata, keyPath, currentFieldId); StructField key = transformSchema( currentFieldId, type.getKeyField(), keyPath, closestStructFieldParentMetadata); // update value type metadata and recurse into value type String valuePath = basePath + "." + type.getValueField().getName(); maybeUpdateFieldId(closestStructFieldParentMetadata, valuePath, currentFieldId); StructField value = transformSchema( currentFieldId, type.getValueField(), valuePath, closestStructFieldParentMetadata); return new StructField( structField.getName(), new MapType(key, value), structField.isNullable(), structField.getMetadata()); } return structField; } /** * The {@code field} being passed here is either {@link ArrayType#getElementField()} or one of * {@link MapType#getKeyField()}, {@link MapType#getValueField()}. For a map the passed in {@code * key} will be one of columnName.key or columnName.value. For an array the passed {@code key} * will be columnName.element. The columnName in this case is either the physical or the display * name of the column. * *

Below is an example that shows the {@link FieldMetadata} of an array named b, where * the array itself is assigned id = 2 with a physical name that includes a UUID. That metadata * field then holds a nested {@link FieldMetadata} under the {@code COLUMN_MAPPING_NESTED_IDS_KEY} * key as can be seen below, which in turn contains the assigned id. * *

* * 
   * {
   *   "name": "b",
   *   "type": {
   *     "type": "array",
   *     "elementType": "integer",
   *     "containsNull": true
   *   },
   *   "nullable": true,
   *   "metadata": {
   *     "delta.columnMapping.id": 2,
   *     "delta.columnMapping.physicalName": "col-859d81a5-6e36-4e43-9c8e-46aa7d80dce6"
   *     "delta.columnMapping.nested.ids": {
   *       "col-859d81a5-6e36-4e43-9c8e-46aa7d80dce6.element": 4
   *     },
   *   }
   * }
   *
* *
* * @param fieldMetadataBuilder The FieldMetadata.Builder to update with nested IDs * @param key For a map this is .key or .value. For an array this is * .element * @param currentFieldId The current maximum field id to increment and use for assignment */ private static void maybeUpdateFieldId( FieldMetadata.Builder fieldMetadataBuilder, String key, AtomicInteger currentFieldId) { // init the nested metadata that holds the nested ids FieldMetadata nestedMetadata = fieldMetadataBuilder.getMetadata(COLUMN_MAPPING_NESTED_IDS_KEY); if (fieldMetadataBuilder.getMetadata(COLUMN_MAPPING_NESTED_IDS_KEY) == null) { fieldMetadataBuilder.putFieldMetadata(COLUMN_MAPPING_NESTED_IDS_KEY, FieldMetadata.empty()); nestedMetadata = fieldMetadataBuilder.getMetadata(COLUMN_MAPPING_NESTED_IDS_KEY); } // assign an id to the nested element and update the metadata if (!nestedMetadata.contains(key)) { FieldMetadata newNestedMeta = FieldMetadata.builder() .fromMetadata(nestedMetadata) .putLong(key, currentFieldId.incrementAndGet()) .build(); fieldMetadataBuilder.putFieldMetadata(COLUMN_MAPPING_NESTED_IDS_KEY, newNestedMeta); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/DateTimeConstants.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; public class DateTimeConstants { public static final int MONTHS_PER_YEAR = 12; public static final byte DAYS_PER_WEEK = 7; public static final long HOURS_PER_DAY = 24L; public static final long MINUTES_PER_HOUR = 60L; public static final long SECONDS_PER_MINUTE = 60L; public static final long SECONDS_PER_HOUR = MINUTES_PER_HOUR * SECONDS_PER_MINUTE; public static final long SECONDS_PER_DAY = HOURS_PER_DAY * SECONDS_PER_HOUR; public static final long MILLIS_PER_SECOND = 1000L; public static final long MILLIS_PER_MINUTE = SECONDS_PER_MINUTE * MILLIS_PER_SECOND; public static final long MILLIS_PER_HOUR = MINUTES_PER_HOUR * MILLIS_PER_MINUTE; public static final long MILLIS_PER_DAY = HOURS_PER_DAY * MILLIS_PER_HOUR; public static final long MICROS_PER_MILLIS = 1000L; public static final long MICROS_PER_SECOND = MILLIS_PER_SECOND * MICROS_PER_MILLIS; public static final long MICROS_PER_MINUTE = SECONDS_PER_MINUTE * MICROS_PER_SECOND; public static final long MICROS_PER_HOUR = MINUTES_PER_HOUR * MICROS_PER_MINUTE; public static final long MICROS_PER_DAY = HOURS_PER_DAY * MICROS_PER_HOUR; public static final long NANOS_PER_MICROS = 1000L; public static final long NANOS_PER_MILLIS = MICROS_PER_MILLIS * NANOS_PER_MICROS; public static final long NANOS_PER_SECOND = MILLIS_PER_SECOND * NANOS_PER_MILLIS; } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/DirectoryCreationUtils.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; import static io.delta.kernel.internal.tablefeatures.TableFeatures.CHECKPOINT_V2_RW_FEATURE; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.tablefeatures.TableFeatures; import java.io.IOException; import java.util.Optional; /** Utility class for creating Delta directories based on commit version and protocol features. */ public class DirectoryCreationUtils { private DirectoryCreationUtils() {} /** Creates all required Delta directories based on commit version and protocol features. */ public static void createAllDeltaDirectoriesAsNeeded( Engine engine, Path logPath, long commitAsVersion, Optional readProtocol, Protocol writeProtocol) throws IOException { createDeltaLogDirectoryIfNeeded(engine, logPath, commitAsVersion); createStagedCommitDirectoryIfNeeded(engine, logPath, readProtocol, writeProtocol); createSidecarDirectoryIfNeeded(engine, logPath, readProtocol, writeProtocol); } /** Creates delta log directory (_delta_log) if this is the initial commit (version 0). */ private static void createDeltaLogDirectoryIfNeeded( Engine engine, Path logPath, long commitAsVersion) throws IOException { if (commitAsVersion == 0) { createDirectoryOrThrow(engine, logPath.toString()); } } /** * Creates staged commit directory (_delta_log/_staged_commits) when enabling catalog managed * feature. */ private static void createStagedCommitDirectoryIfNeeded( Engine engine, Path logPath, Optional readProtocol, Protocol writeProtocol) throws IOException { final boolean readVersionSupportsCatalogManaged = readProtocol.map(TableFeatures::isCatalogManagedSupported).orElse(false); final boolean writeVersionSupportsCatalogManaged = TableFeatures.isCatalogManagedSupported(writeProtocol); if (!readVersionSupportsCatalogManaged && writeVersionSupportsCatalogManaged) { createDirectoryOrThrow(engine, FileNames.stagedCommitDirectory(logPath)); } } /** Creates sidecar directory (_delta_log/_sidecar) when enabling v2 checkpoint feature. */ private static void createSidecarDirectoryIfNeeded( Engine engine, Path logPath, Optional readProtocol, Protocol writeProtocol) throws IOException { final boolean readVersionSupportsV2Checkpoints = readProtocol.map(p -> p.supportsFeature(CHECKPOINT_V2_RW_FEATURE)).orElse(false); final boolean writeVersionSupportsV2Checkpoints = writeProtocol.supportsFeature(CHECKPOINT_V2_RW_FEATURE); if (!readVersionSupportsV2Checkpoints && writeVersionSupportsV2Checkpoints) { createDirectoryOrThrow(engine, FileNames.sidecarDirectory(logPath)); } } /** Creates directory using engine filesystem client, throws on failure. */ private static void createDirectoryOrThrow(Engine engine, String directoryPath) throws IOException { try { if (!engine.getFileSystemClient().mkdirs(directoryPath)) { throw new RuntimeException("Failed to create directory: " + directoryPath); } } catch (Exception e) { throw new IOException("Creating directories for path " + directoryPath, e); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/DomainMetadataUtils.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.internal.DeltaErrors; import io.delta.kernel.internal.actions.DomainMetadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.tablefeatures.TableFeatures; import java.util.HashMap; import java.util.List; import java.util.Map; public class DomainMetadataUtils { private DomainMetadataUtils() { // Empty private constructor to prevent instantiation } /** * Populate the map of domain metadata from actions. When encountering duplicate domain metadata * actions for the same domain, this method preserves the first seen entry and skips subsequent * entries. This behavior is especially useful for log replay as we want to ensure that earlier * domain metadata entries take precedence over later ones. * * @param domainMetadataActionVector A {@link ColumnVector} containing the domain metadata rows * @param domainMetadataMap The existing map to be populated with domain metadata entries, where * the key is the domain name and the value is the domain metadata */ public static void populateDomainMetadataMap( ColumnVector domainMetadataActionVector, Map domainMetadataMap) { final int vectorSize = domainMetadataActionVector.getSize(); for (int rowId = 0; rowId < vectorSize; rowId++) { DomainMetadata dm = DomainMetadata.fromColumnVector(domainMetadataActionVector, rowId); if (dm != null && !domainMetadataMap.containsKey(dm.getDomain())) { // We only add the domain metadata if its domain name not already present in the map domainMetadataMap.put(dm.getDomain(), dm); } } } /** * Validates the list of domain metadata actions before committing them. It ensures that * *
    *
  1. domain metadata actions are only present when supported by the table protocol *
  2. there are no duplicate domain metadata actions for the same domain in the provided * actions. *
* * @param domainMetadataActions The list of domain metadata actions to validate * @param protocol The protocol to check for domain metadata support */ public static void validateDomainMetadatas( List domainMetadataActions, Protocol protocol) { if (domainMetadataActions.isEmpty()) return; // The list of domain metadata is non-empty, so the protocol must support domain metadata if (!TableFeatures.isDomainMetadataSupported(protocol)) { throw DeltaErrors.domainMetadataUnsupported(); } Map domainMetadataMap = new HashMap<>(); for (DomainMetadata domainMetadata : domainMetadataActions) { String domain = domainMetadata.getDomain(); if (domainMetadataMap.containsKey(domain)) { String message = String.format( "Multiple actions detected for domain '%s' in single transaction: '%s' and '%s'. " + "Only one action per domain is allowed.", domain, domainMetadataMap.get(domain).toString(), domainMetadata.toString()); throw new IllegalArgumentException(message); } domainMetadataMap.put(domain, domainMetadata); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/ExpressionUtils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.expressions.Expression; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.types.CollationIdentifier; import java.util.Arrays; import java.util.List; import java.util.Optional; public class ExpressionUtils { /** Return an expression cast as a predicate, throw an error if it is not a predicate */ public static Predicate asPredicate(Expression expression) { checkArgument(expression instanceof Predicate, "Expected predicate but got %s", expression); return (Predicate) expression; } /** Utility method to return the left child of the binary input expression */ public static Expression getLeft(Expression expression) { List children = expression.getChildren(); checkArgument( children.size() == 2, "%s: expected two inputs, but got %s", expression, children.size()); return children.get(0); } /** Utility method to return the right child of the binary input expression */ public static Expression getRight(Expression expression) { List children = expression.getChildren(); checkArgument( children.size() == 2, "%s: expected two inputs, but got %s", expression, children.size()); return children.get(1); } /** Utility method to return the single child of the unary input expression */ public static Expression getUnaryChild(Expression expression) { List children = expression.getChildren(); checkArgument( children.size() == 1, "%s: expected one inputs, but got %s", expression, children.size()); return children.get(0); } /** Utility method to create a predicate with an optional collation identifier */ public static Predicate createPredicate( String name, List children, Optional collationIdentifier) { if (collationIdentifier.isPresent()) { return new Predicate(name, children, collationIdentifier.get()); } else { return new Predicate(name, children); } } /** Utility method to create a binary predicate with an optional collation identifier */ public static Predicate createPredicate( String name, Expression left, Expression right, Optional collationIdentifier) { return createPredicate(name, Arrays.asList(left, right), collationIdentifier); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/FileNames.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.utils.FileStatus; import java.util.ArrayList; import java.util.List; import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; public final class FileNames { private FileNames() {} //////////////////////////////////////////////// // File name patterns and other static values // //////////////////////////////////////////////// // TODO: Delete this in favor of ParsedLogCategory. public enum DeltaLogFileType { COMMIT, LOG_COMPACTION, CHECKPOINT, CHECKSUM } /** Example: 00000000000000000001.json */ private static final Pattern DELTA_FILE_PATTERN = Pattern.compile("\\d+\\.json"); /** Example: 00000000000000000001.00000000000000000009.compacted.json */ private static final Pattern COMPACTION_FILE_PATTERN = Pattern.compile("\\d+\\.\\d+\\.compacted\\.json"); /** Example: 00000000000000000001.dc0f9f58-a1a0-46fd-971a-bd8b2e9dbb81.json */ private static final Pattern UUID_DELTA_FILE_REGEX = Pattern.compile("(\\d+)\\.([^\\.]+)\\.json"); /** * Examples: * *
    *
  • Classic V1 - 00000000000000000001.checkpoint.parquet *
  • Multi-part V1 - 00000000000000000001.checkpoint.0000000001.0000000010.parquet *
  • V2 JSON - 00000000000000000001.checkpoint.uuid-1234abcd.json *
  • V2 Parquet - 00000000000000000001.checkpoint.uuid-1234abcd.parquet *
*/ private static final Pattern CHECKPOINT_FILE_PATTERN = Pattern.compile("(\\d+)\\.checkpoint((\\.\\d+\\.\\d+)?\\.parquet|\\.[^.]+\\.(json|parquet))"); /** Example: 00000000000000000001.checkpoint.parquet */ private static final Pattern CLASSIC_CHECKPOINT_FILE_PATTERN = Pattern.compile("\\d+\\.checkpoint\\.parquet"); /** Example: 00000000000000000001.crc */ private static final Pattern CHECK_SUM_FILE_PATTERN = Pattern.compile("(\\d+)\\.crc"); /** * Examples: * *
    *
  • 00000000000000000001.checkpoint.dc0f9f58-a1a0-46fd-971a-bd8b2e9dbb81.json *
  • 00000000000000000001.checkpoint.dc0f9f58-a1a0-46fd-971a-bd8b2e9dbb81.parquet *
*/ private static final Pattern V2_CHECKPOINT_FILE_PATTERN = Pattern.compile("(\\d+)\\.checkpoint\\.[^.]+\\.(json|parquet)"); /** Example: 00000000000000000001.checkpoint.0000000020.0000000060.parquet */ public static final Pattern MULTI_PART_CHECKPOINT_FILE_PATTERN = Pattern.compile("(\\d+)\\.checkpoint\\.(\\d+)\\.(\\d+)\\.parquet"); public static final String STAGED_COMMIT_DIRECTORY = "_staged_commits"; public static final String SIDECAR_DIRECTORY = "_sidecars"; public static DeltaLogFileType determineFileType(FileStatus file) { final String fileName = file.getPath().toString(); if (isCommitFile(fileName)) { return DeltaLogFileType.COMMIT; } else if (isCheckpointFile(fileName)) { return DeltaLogFileType.CHECKPOINT; } else if (isLogCompactionFile(fileName)) { return DeltaLogFileType.LOG_COMPACTION; } else if (isChecksumFile(fileName)) { return DeltaLogFileType.CHECKSUM; } else { throw new IllegalStateException("Unexpected file type: " + fileName); } } //////////////////////// // Version extractors // //////////////////////// /** * Get the version of the checkpoint, checksum or delta file. Throws an error if an unexpected * file type is seen. These unexpected files should be filtered out to ensure forward * compatibility in cases where new file types are added, but without an explicit protocol * upgrade. */ public static long getFileVersion(Path path) { if (isCheckpointFile(path.getName())) { return checkpointVersion(path); } else if (isCommitFile(path.getName())) { return deltaVersion(path); } else if (isChecksumFile(path.getName())) { return checksumVersion(path); } else { throw new IllegalArgumentException( String.format("Unexpected file type found in transaction log: %s", path)); } } /** Returns the version for the given delta path. */ public static long deltaVersion(Path path) { return Long.parseLong(path.getName().split("\\.")[0]); } public static long deltaVersion(String path) { final int slashIdx = path.lastIndexOf(Path.SEPARATOR); final String name = path.substring(slashIdx + 1); return Long.parseLong(name.split("\\.")[0]); } /** Returns the start and end versions for the given compaction path. */ public static Tuple2 logCompactionVersions(Path path) { final String[] split = path.getName().split("\\."); return new Tuple2<>(Long.parseLong(split[0]), Long.parseLong(split[1])); } public static Tuple2 logCompactionVersions(String path) { return logCompactionVersions(new Path(path)); } /** Returns the version for the given checkpoint path. */ public static long checkpointVersion(Path path) { return Long.parseLong(path.getName().split("\\.")[0]); } public static long checkpointVersion(String path) { final int slashIdx = path.lastIndexOf(Path.SEPARATOR); final String name = path.substring(slashIdx + 1); return Long.parseLong(name.split("\\.")[0]); } public static Tuple2 multiPartCheckpointPartAndNumParts(Path path) { final String fileName = path.getName(); final Matcher matcher = MULTI_PART_CHECKPOINT_FILE_PATTERN.matcher(fileName); checkArgument( matcher.matches(), String.format("Path is not a multi-part checkpoint file: %s", fileName)); final int partNum = Integer.parseInt(matcher.group(2)); final int numParts = Integer.parseInt(matcher.group(3)); return new Tuple2<>(partNum, numParts); } public static Tuple2 multiPartCheckpointPartAndNumParts(String path) { return multiPartCheckpointPartAndNumParts(new Path(path)); } ///////////////////// // Directory paths // ///////////////////// public static String stagedCommitDirectory(Path logPath) { return new Path(logPath, STAGED_COMMIT_DIRECTORY).toString(); } public static String sidecarDirectory(Path logPath) { return new Path(logPath, SIDECAR_DIRECTORY).toString(); } /////////////////////////////////// // File path and prefix builders // /////////////////////////////////// /** Returns the delta (json format) path for a given delta file. */ public static String deltaFile(Path path, long version) { return String.format("%s/%020d.json", path, version); } /** Returns the delta (json format) path for a given delta file. */ public static String deltaFile(String path, long version) { return deltaFile(new Path(path), version); } public static String stagedCommitFile(Path logPath, long version) { final Path stagedCommitPath = new Path(logPath, STAGED_COMMIT_DIRECTORY); return String.format("%s/%020d.%s.json", stagedCommitPath, version, UUID.randomUUID()); } public static String stagedCommitFile(String logPath, long version) { return stagedCommitFile(new Path(logPath), version); } /** Example: /a/_sidecars/3a0d65cd-4056-49b8-937b-95f9e3ee90e5.parquet */ public static String sidecarFile(Path path, String sidecar) { return String.format("%s/%s/%s", path.toString(), SIDECAR_DIRECTORY, sidecar); } /** Returns the path to the checksum file for the given version. */ public static Path checksumFile(Path path, long version) { return new Path(path, String.format("%020d.crc", version)); } public static long checksumVersion(Path path) { return Long.parseLong(path.getName().split("\\.")[0]); } public static long checksumVersion(String path) { return checksumVersion(new Path(path)); } /** * Returns the prefix of all delta log files for the given version. * *

Intended for use with listFrom to get all files from this version onwards. The returned Path * will not exist as a file. */ public static String listingPrefix(Path path, long version) { return String.format("%s/%020d.", path, version); } /** * Returns the path for a singular checkpoint up to the given version. * *

In a future protocol version this path will stop being written. */ public static Path checkpointFileSingular(Path path, long version) { return new Path(path, String.format("%020d.checkpoint.parquet", version)); } /** * Returns the path for a top-level V2 checkpoint file up to the given version with a given UUID * and filetype (JSON or Parquet). */ public static Path topLevelV2CheckpointFile( Path path, long version, String uuid, String fileType) { assert (fileType.equals("json") || fileType.equals("parquet")); return new Path(path, String.format("%020d.checkpoint.%s.%s", version, uuid, fileType)); } /** Returns the path for a V2 sidecar file with a given UUID. */ public static Path v2CheckpointSidecarFile(Path path, String uuid) { return new Path(String.format("%s/%s/%s.parquet", path.toString(), SIDECAR_DIRECTORY, uuid)); } public static Path multiPartCheckpointFile(Path path, long version, int part, int numParts) { return new Path( path, String.format("%020d.checkpoint.%010d.%010d.parquet", version, part, numParts)); } /** * Returns the paths for all parts of the checkpoint up to the given version. * *

In a future protocol version we will write this path instead of checkpointFileSingular. * *

Example of the format: 00000000000000004915.checkpoint.0000000020.0000000060.parquet is * checkpoint part 20 out of 60 for the snapshot at version 4915. Zero padding is for * lexicographic sorting. */ public static List checkpointFileWithParts(Path path, long version, int numParts) { final List output = new ArrayList<>(); for (int i = 1; i < numParts + 1; i++) { output.add(multiPartCheckpointFile(path, version, i, numParts)); } return output; } /** * Return the path that should be used for a log compaction file. * * @param logPath path to the delta log location * @param startVersion the start version for the log compaction * @param endVersion the end version for the log compaction */ public static Path logCompactionPath(Path logPath, long startVersion, long endVersion) { String fileName = String.format("%020d.%020d.compacted.json", startVersion, endVersion); return new Path(logPath, fileName); } ///////////////////////////// // Is file checkers // ///////////////////////////// public static boolean isCheckpointFile(String path) { return CHECKPOINT_FILE_PATTERN.matcher(new Path(path).getName()).matches(); } public static boolean isClassicCheckpointFile(String path) { return CLASSIC_CHECKPOINT_FILE_PATTERN.matcher(new Path(path).getName()).matches(); } public static boolean isMultiPartCheckpointFile(String path) { return MULTI_PART_CHECKPOINT_FILE_PATTERN.matcher(new Path(path).getName()).matches(); } public static boolean isV2CheckpointFile(String path) { return V2_CHECKPOINT_FILE_PATTERN.matcher(new Path(path).getName()).matches(); } public static boolean isCommitFile(String path) { final String fileName = new Path(path).getName(); return DELTA_FILE_PATTERN.matcher(fileName).matches() || UUID_DELTA_FILE_REGEX.matcher(fileName).matches(); } public static boolean isPublishedDeltaFile(String path) { final String fileName = new Path(path).getName(); return DELTA_FILE_PATTERN.matcher(fileName).matches(); } public static boolean isStagedDeltaFile(String path) { final Path p = new Path(path); if (!p.getParent().getName().equals(STAGED_COMMIT_DIRECTORY)) { return false; } return UUID_DELTA_FILE_REGEX.matcher(p.getName()).matches(); } public static boolean isLogCompactionFile(String path) { final String fileName = new Path(path).getName(); return COMPACTION_FILE_PATTERN.matcher(fileName).matches(); } public static boolean isChecksumFile(String checksumFilePath) { return CHECK_SUM_FILE_PATTERN.matcher(new Path(checksumFilePath).getName()).matches(); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/InCommitTimestampUtils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; import static io.delta.kernel.internal.TableConfig.IN_COMMIT_TIMESTAMPS_ENABLED; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.TableConfig; import io.delta.kernel.internal.actions.CommitInfo; import io.delta.kernel.internal.actions.Metadata; import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.function.Function; public class InCommitTimestampUtils { /** * Returns the updated {@link Metadata} with inCommitTimestamp enablement related info (version * and timestamp) correctly set. This is done only 1. If this transaction enables * inCommitTimestamp. 2. If the commit version is not 0. This is because we only need to persist * the enablement info if there are non-ICT commits in the Delta log. Note: This function must * only be called after transaction conflicts have been resolved. */ public static Optional getUpdatedMetadataWithICTEnablementInfo( Engine engine, long inCommitTimestamp, Optional readSnapshotOpt, Metadata metadata, long commitVersion) { if (readSnapshotOpt.isPresent() && didCurrentTransactionEnableICT(engine, metadata, readSnapshotOpt.get())) { Map enablementTrackingProperties = new HashMap<>(); enablementTrackingProperties.put( TableConfig.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.getKey(), Long.toString(commitVersion)); enablementTrackingProperties.put( TableConfig.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.getKey(), Long.toString(inCommitTimestamp)); Metadata newMetadata = metadata.withMergedConfiguration(enablementTrackingProperties); return Optional.of(newMetadata); } else { return Optional.empty(); } } /** * Tries to extract the inCommitTimestamp from the commitInfo action in the given ColumnarBatch. * When inCommitTimestamp is enabled, the commitInfo action is always the first action in the * delta file. This function assumes that this batch is the leading batch of a single delta file * and attempts to extract the commitInfo action from the first row. If the commitInfo action is * not present or does not contain an inCommitTimestamp, this function returns an empty Optional. */ public static Optional tryExtractInCommitTimestamp( ColumnarBatch firstActionsBatchFromSingleDelta) { final int commitInfoOrdinal = firstActionsBatchFromSingleDelta.getSchema().indexOf("commitInfo"); if (commitInfoOrdinal == -1) { return Optional.empty(); } ColumnVector commitInfoVector = firstActionsBatchFromSingleDelta.getColumnVector(commitInfoOrdinal); // CommitInfo is always the first action in the batch when inCommitTimestamp is enabled. int expectedRowIdOfCommitInfo = 0; CommitInfo commitInfo = CommitInfo.fromColumnVector(commitInfoVector, expectedRowIdOfCommitInfo); return commitInfo != null ? commitInfo.getInCommitTimestamp() : Optional.empty(); } /** Returns true if the current transaction implicitly/explicitly enables ICT. */ private static boolean didCurrentTransactionEnableICT( Engine engine, Metadata currentTransactionMetadata, SnapshotImpl readSnapshot) { // If ICT is currently enabled, and the read snapshot did not have ICT enabled, // then the current transaction must have enabled it. // In case of a conflict, any winning transaction that enabled it after // our read snapshot would have caused a metadata conflict abort // (see {@link ConflictChecker.handleMetadata}), so we know that // all winning transactions' ICT enablement status must match the read snapshot. // // WARNING: To ensure that this function returns true if ICT is enabled during the first // commit, we explicitly handle the case where the readSnapshot.version is -1. boolean isICTCurrentlyEnabled = IN_COMMIT_TIMESTAMPS_ENABLED.fromMetadata(currentTransactionMetadata); boolean wasICTEnabledInReadSnapshot = IN_COMMIT_TIMESTAMPS_ENABLED.fromMetadata(readSnapshot.getMetadata()); return isICTCurrentlyEnabled && !wasICTEnabledInReadSnapshot; } /** * Finds the greatest lower bound of the target value in the range [lowerBoundInclusive, * upperBoundInclusive] using binary search. The indexToValueMapper function is used to map the * index to the corresponding value. Note that this function assumes that the values are sorted in * ascending order. * * @param target The target value to find the greatest lower bound for. * @param lowerBoundInclusive The lower bound of the search range (inclusive). * @param upperBoundInclusive The upper bound of the search range (inclusive). * @param indexToValueMapper A function that maps an index to its corresponding value. * @return An optional which contains a tuple containing the index and the value of the greatest * lower bound when found, or an empty optional if not found. */ public static Optional> greatestLowerBound( long target, long lowerBoundInclusive, long upperBoundInclusive, Function indexToValueMapper) { if (lowerBoundInclusive > upperBoundInclusive) { return Optional.empty(); } long start = lowerBoundInclusive; long end = upperBoundInclusive; long resultIndex = -1; long resultValue = 0; while (start <= end) { long mid = start + (end - start) / 2; long midValue = indexToValueMapper.apply(mid); if (midValue == target) { return Optional.of(new Tuple2<>(mid, midValue)); } else if (midValue < target) { resultIndex = mid; resultValue = midValue; start = mid + 1; } else { end = mid - 1; } } if (resultIndex == -1) { return Optional.empty(); } else { return Optional.of(new Tuple2<>(resultIndex, resultValue)); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/InternalUtils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.types.DataType; import io.delta.kernel.types.StringType; import io.delta.kernel.utils.CloseableIterator; import java.io.IOException; import java.net.URI; import java.sql.Date; import java.sql.Timestamp; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.util.*; import java.util.stream.Collectors; public class InternalUtils { private static final LocalDate EPOCH_DAY = LocalDate.ofEpochDay(0); private static final LocalDateTime EPOCH_DATETIME = LocalDateTime.ofEpochSecond(0, 0, ZoneOffset.UTC); private InternalUtils() {} /** * Utility method to read at most one row from the given data {@link ColumnarBatch} iterator. If * there is more than one row, an exception will be thrown. * * @param dataIter * @return */ public static Optional getSingularRow(CloseableIterator dataIter) throws IOException { Row row = null; while (dataIter.hasNext()) { try (CloseableIterator rows = dataIter.next().getRows()) { while (rows.hasNext()) { if (row != null) { throw new IllegalArgumentException("Given data batch contains more than one row"); } row = rows.next(); } } } return Optional.ofNullable(row); } /** * Utility method to read at most one element from a {@link CloseableIterator}. If there is more * than element row, an exception will be thrown. */ public static Optional getSingularElement(CloseableIterator iter) throws IOException { try { T result = null; while (iter.hasNext()) { if (result != null) { throw new IllegalArgumentException("Iterator contains more than one element"); } result = iter.next(); } return Optional.ofNullable(result); } finally { iter.close(); } } /** Utility method to get the number of days since epoch this given date is. */ public static int daysSinceEpoch(Date date) { LocalDate localDate = date.toLocalDate(); return (int) localDate.toEpochDay(); } /** * Utility method to get the number of microseconds since the unix epoch for the given timestamp * interpreted in UTC. */ public static long microsSinceEpoch(Timestamp timestamp) { LocalDateTime localTimestamp = timestamp.toLocalDateTime(); return TimestampUtils.toEpochMicros(localTimestamp); } /** * Utility method to create a singleton string {@link ColumnVector} * * @param value the string element to create the vector with * @return A {@link ColumnVector} with a single element {@code value} */ public static ColumnVector singletonStringColumnVector(String value) { return new ColumnVector() { @Override public DataType getDataType() { return StringType.STRING; } @Override public int getSize() { return 1; } @Override public void close() {} @Override public boolean isNullAt(int rowId) { return value == null; } @Override public String getString(int rowId) { if (rowId != 0) { throw new IllegalArgumentException("Invalid row id: " + rowId); } return value; } }; } public static Row requireNonNull(Row row, int ordinal, String columnName) { if (row.isNullAt(ordinal)) { throw new IllegalArgumentException("Expected a non-null value for column: " + columnName); } return row; } public static ColumnVector requireNonNull(ColumnVector vector, int rowId, String columnName) { if (vector.isNullAt(rowId)) { throw new IllegalArgumentException("Expected a non-null value for column: " + columnName); } return vector; } /** * Relativize the given child path with respect to the given root URI. If the child path is * already a relative path, it is returned as is. * * @param child * @param root Root directory as URI. Relativization is done with respect to this root. The * relativize operation requires conversion to URI, so the caller is expected to convert the * root directory to URI once and use it for relativizing for multiple child paths. * @return */ public static Path relativizePath(Path child, URI root) { if (child.isAbsolute()) { return new Path(root.relativize(child.toUri())); } return child; } public static Set toLowerCaseSet(Collection set) { return set.stream().map(String::toLowerCase).collect(Collectors.toSet()); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/IntervalParserUtils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; import static io.delta.kernel.internal.util.DateTimeConstants.*; import static io.delta.kernel.internal.util.IntervalParserUtils.ParseState.*; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.lang.String.format; import static java.nio.charset.StandardCharsets.UTF_8; import java.util.Locale; /** * Copy of `org/apache/spark/sql/catalyst/util/SparkIntervalUtils.scala` from Apache Spark. Delta * table properties store the interval format. We need a parser in order to parse these values in * Kernel. */ public class IntervalParserUtils { private IntervalParserUtils() {} /** * Parse the given interval string into milliseconds. For configs accepting an interval, we * require the user specified string must obey: * *

    *
  • Doesn't use months or years, since an internal like this is not deterministic. *
  • Doesn't use microseconds or nanoseconds part as it too granular to use. *
* * @return parsed interval as milliseconds. */ public static long safeParseIntervalAsMillis(String input) { checkArgument(input != null, "interval string cannot be null"); String inputInLowerCase = input.trim().toLowerCase(Locale.ROOT); checkArgument(!inputInLowerCase.isEmpty(), "interval string cannot be empty"); if (!inputInLowerCase.startsWith("interval ")) { inputInLowerCase = "interval " + inputInLowerCase; } return parseIntervalAsMicros(inputInLowerCase) / 1000; // convert to milliseconds } public static long parseIntervalAsMicros(String input) { return new IntervalParser(input).toMicroSeconds(); } enum ParseState { PREFIX, TRIM_BEFORE_SIGN, SIGN, TRIM_BEFORE_VALUE, VALUE, VALUE_FRACTIONAL_PART, TRIM_BEFORE_UNIT, UNIT_BEGIN, UNIT_SUFFIX, UNIT_END; } private static final String INTERVAL_STR = "interval"; private static final String YEAR_STR = "year"; private static final String MONTH_STR = "month"; private static final String WEEK_STR = "week"; private static final String DAY_STR = "day"; private static final String HOUR_STR = "hour"; private static final String MINUTE_STR = "minute"; private static final String SECOND_STR = "second"; private static final String MILLIS_STR = "millisecond"; private static final String MICROS_STR = "microsecond"; private static class IntervalParser { private final String input; private String s; // trimmed input in lowercase private ParseState state = ParseState.PREFIX; private int i = 0; private long currentValue = 0; private boolean isNegative = false; private int days = 0; private long microseconds = 0; private int fractionScale = 0; private int fraction = 0; private boolean pointPrefixed = false; // Expected trimmed lower case input string. IntervalParser(String input) { this.input = input; if (input == null) { throwIAE("interval string cannot be null"); } String inputInLowerCase = input.trim().toLowerCase(Locale.ROOT); if (inputInLowerCase.isEmpty()) { throwIAE(format("Error parsing '%s' to interval", input)); } this.s = inputInLowerCase; } long toMicroSeconds() { // UTF-8 encoded bytes of the trimmed input byte[] bytes = s.getBytes(UTF_8); checkArgument(bytes.length > 0, "Interval string cannot be empty"); while (i < bytes.length) { byte b = bytes[i]; int initialFractionScale = (int) (NANOS_PER_SECOND / 10); switch (state) { case PREFIX: if (s.startsWith(INTERVAL_STR)) { if (s.length() == INTERVAL_STR.length()) { throwIAE("interval string cannot be empty"); } else if (!Character.isWhitespace(bytes[i + INTERVAL_STR.length()])) { throwIAE("invalid interval prefix " + currentWord()); } else { i += INTERVAL_STR.length(); } } state = ParseState.TRIM_BEFORE_SIGN; break; case TRIM_BEFORE_SIGN: trimToNextState(b, SIGN); break; case SIGN: currentValue = 0; fraction = 0; // We preset next state from SIGN to TRIM_BEFORE_VALUE. If we meet '.' // in the SIGN state, it means that the interval value we deal with here is // a numeric with only fractional part, such as '.11 second', which can be // parsed to 0.11 seconds. In this case, we need to reset next state to // `VALUE_FRACTIONAL_PART` to go parse the fraction part of the interval // value. state = TRIM_BEFORE_VALUE; fractionScale = -1; if (b == '-' || b == '+') { i++; isNegative = b == '-'; } else if ('0' <= b && b <= '9') { isNegative = false; } else if (b == '.') { isNegative = false; fractionScale = initialFractionScale; pointPrefixed = true; i++; state = VALUE_FRACTIONAL_PART; } else { throwIAE(format("unrecognized number '%s'", currentWord())); } break; case TRIM_BEFORE_VALUE: trimToNextState(b, VALUE); break; case VALUE: if ('0' <= b && b <= '9') { try { currentValue = Math.addExact(Math.multiplyExact(10, currentValue), (b - '0')); } catch (ArithmeticException e) { throwIAE(e.getMessage(), e); } } else if (Character.isWhitespace(b)) { state = TRIM_BEFORE_UNIT; } else if (b == '.') { fractionScale = initialFractionScale; state = VALUE_FRACTIONAL_PART; } else { throwIAE(format("invalid value '%s'", currentWord())); } i++; break; case VALUE_FRACTIONAL_PART: if ('0' <= b && b <= '9' && fractionScale > 0) { fraction += (b - '0') * fractionScale; fractionScale /= 10; } else if (Character.isWhitespace(b) && (!pointPrefixed || fractionScale < initialFractionScale)) { fraction /= ((int) NANOS_PER_MICROS); state = TRIM_BEFORE_UNIT; } else if ('0' <= b && b <= '9') { throwIAE( format( "interval can only support nanosecond " + "precision, '%s' is out of range", currentWord())); } else { throwIAE(format("invalid value '%s'", currentWord())); } i += 1; break; case TRIM_BEFORE_UNIT: trimToNextState(b, UNIT_BEGIN); break; case UNIT_BEGIN: // Checks that only seconds can have the fractional part if (b != 's' && fractionScale >= 0) { throwIAE(format("'%s' cannot have fractional part", currentWord())); } if (isNegative) { currentValue = -currentValue; fraction = -fraction; } try { if (b == 'y' && matchAt(i, YEAR_STR)) { throwIAE("year is not supported, use days instead"); } else if (b == 'w' && matchAt(i, WEEK_STR)) { long daysInWeeks = Math.multiplyExact(DAYS_PER_WEEK, currentValue); days = Math.toIntExact(Math.addExact(days, daysInWeeks)); i += WEEK_STR.length(); } else if (b == 'd' && matchAt(i, DAY_STR)) { days = Math.addExact(days, Math.toIntExact(currentValue)); i += DAY_STR.length(); } else if (b == 'h' && matchAt(i, HOUR_STR)) { long hoursUs = Math.multiplyExact(currentValue, MICROS_PER_HOUR); microseconds = Math.addExact(microseconds, hoursUs); i += HOUR_STR.length(); } else if (b == 's' && matchAt(i, SECOND_STR)) { long secondsUs = Math.multiplyExact(currentValue, MICROS_PER_SECOND); microseconds = Math.addExact(Math.addExact(microseconds, secondsUs), fraction); i += SECOND_STR.length(); } else if (b == 'm') { if (matchAt(i, MONTH_STR)) { throwIAE("month is not supported, use days instead"); } else if (matchAt(i, MINUTE_STR)) { long minutesUs = Math.multiplyExact(currentValue, MICROS_PER_MINUTE); microseconds = Math.addExact(microseconds, minutesUs); i += MINUTE_STR.length(); } else if (matchAt(i, MILLIS_STR)) { long millisUs = Math.multiplyExact(currentValue, MICROS_PER_MILLIS); microseconds = Math.addExact(microseconds, millisUs); i += MILLIS_STR.length(); } else if (matchAt(i, MICROS_STR)) { microseconds = Math.addExact(microseconds, currentValue); i += MICROS_STR.length(); } else { throwIAE(format("invalid unit '%s'", currentWord())); } } else { throwIAE(format("invalid unit '%s'", currentWord())); } } catch (ArithmeticException e) { throwIAE(e.getMessage(), e); } state = UNIT_SUFFIX; break; case UNIT_SUFFIX: if (b == 's') { state = UNIT_END; } else if (Character.isWhitespace(b)) { state = TRIM_BEFORE_SIGN; } else { throwIAE(format("invalid unit '%s'", currentWord())); } i++; break; case UNIT_END: if (Character.isWhitespace(b)) { i++; state = TRIM_BEFORE_SIGN; } else { throwIAE(format("invalid unit '%s'", currentWord())); } break; default: throwIAE("invalid input: " + s); } } switch (state) { case UNIT_SUFFIX: // fall through case UNIT_END: // fall through case TRIM_BEFORE_SIGN: return days * MICROS_PER_DAY + microseconds; case TRIM_BEFORE_VALUE: throwIAE(format("expect a number after '%s' but hit EOL", currentWord())); break; case VALUE: case VALUE_FRACTIONAL_PART: throwIAE(format("expect a unit name after '%s' but hit EOL", currentWord())); break; default: throwIAE(format("unknown error when parsing '%s'", currentWord())); } throwIAE("invalid interval"); return 0; // should never reach. } private void trimToNextState(byte b, ParseState next) { if (Character.isWhitespace(b)) { i++; } else { state = next; } } private String currentWord() { String sep = "\\s+"; String[] strings = s.split(sep); int lenRight = s.substring(i).split(sep).length; return strings[strings.length - lenRight]; } private boolean matchAt(int i, String str) { if (i + str.length() > s.length()) { return false; } return s.substring(i, i + str.length()).equals(str); } private void throwIAE(String msg, Exception e) { throw new IllegalArgumentException( format("Error parsing '%s' to interval, %s", input, msg), e); } private void throwIAE(String msg) { throw new IllegalArgumentException(format("Error parsing '%s' to interval, %s", input, msg)); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/JsonUtils.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; import static io.delta.kernel.internal.DeltaErrors.unsupportedStatsDataType; import static io.delta.kernel.statistics.DataFileStatistics.EPOCH; import static io.delta.kernel.statistics.DataFileStatistics.TIMESTAMP_FORMATTER; import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.expressions.Literal; import io.delta.kernel.types.*; import java.io.IOException; import java.io.StringWriter; import java.io.UncheckedIOException; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; import java.time.*; import java.time.format.DateTimeFormatter; import java.util.Collections; import java.util.Map; public class JsonUtils { private JsonUtils() {} private static final ObjectMapper MAPPER = new ObjectMapper(); private static final JsonFactory FACTORY = new JsonFactory(); public static JsonFactory factory() { return FACTORY; } public static ObjectMapper mapper() { return MAPPER; } @FunctionalInterface public interface ToJson { void generate(JsonGenerator generator) throws IOException; } @FunctionalInterface public interface JsonValueWriter { void write(JsonGenerator generator, T value) throws IOException; } /** * Utility class for writing JSON with a Jackson {@link JsonGenerator}. * * @param toJson function that produces JSON using a {@link JsonGenerator} * @return a JSON string produced from the generator */ public static String generate(ToJson toJson) { try (StringWriter writer = new StringWriter(); JsonGenerator generator = factory().createGenerator(writer)) { toJson.generate(generator); generator.flush(); return writer.toString(); } catch (IOException e) { throw new UncheckedIOException(e); } } /** * Parses the given JSON string into a map of key-value pairs. * *

The JSON string should be in the format: * * 

{@code {"key1": "value1", "key2": "value2", ...}}
* * where both keys and values are strings. * * @param jsonString The JSON string to parse * @return A map containing the key-value pairs extracted from the JSON string */ public static Map parseJSONKeyValueMap(String jsonString) { if (jsonString == null || jsonString.trim().isEmpty()) { return Collections.emptyMap(); } try { return MAPPER.readValue(jsonString, new TypeReference>() {}); } catch (Exception e) { throw new KernelException(String.format("Failed to parse JSON string: %s", jsonString), e); } } /** * Helper method to convert JSON node value to Literal based on the expected data type from * schema. Uses the schema type information to eliminate ambiguity when parsing JSON values. * * @param valueNode The JSON node containing the value * @param dataType The expected data type from the schema * @return The corresponding Literal, or null if the value is null * @throws KernelException if the JSON value cannot be parsed as the expected type */ public static Literal parseJsonValueToLiteral(JsonNode valueNode, DataType dataType) { if (valueNode == null || valueNode.isNull()) { return null; } try { if (dataType instanceof BooleanType) { if (!valueNode.isBoolean()) { throw new KernelException( String.format("Expected boolean value but got: %s", valueNode.toString())); } return Literal.ofBoolean(valueNode.asBoolean()); } else if (dataType instanceof ByteType) { if (!valueNode.isNumber()) { throw new KernelException( String.format("Expected byte value but got: %s", valueNode.toString())); } return Literal.ofByte((byte) valueNode.asInt()); } else if (dataType instanceof ShortType) { if (!valueNode.isNumber()) { throw new KernelException( String.format("Expected short value but got: %s", valueNode.toString())); } return Literal.ofShort(valueNode.shortValue()); } else if (dataType instanceof IntegerType) { if (!valueNode.isNumber()) { throw new KernelException( String.format("Expected integer value but got: %s", valueNode.toString())); } return Literal.ofInt(valueNode.asInt()); } else if (dataType instanceof LongType) { if (!valueNode.isNumber()) { throw new KernelException( String.format("Expected long value but got: %s", valueNode.toString())); } return Literal.ofLong(valueNode.asLong()); } else if (dataType instanceof FloatType) { if (valueNode.isTextual()) { // Special float values are stored as strings during serialization String textValue = valueNode.asText(); switch (textValue) { case "NaN": return Literal.ofFloat(Float.NaN); case "Infinity": return Literal.ofFloat(Float.POSITIVE_INFINITY); case "-Infinity": return Literal.ofFloat(Float.NEGATIVE_INFINITY); default: throw new KernelException( String.format("Expected float value but got unexpected string: %s", textValue)); } } if (!valueNode.isNumber()) { throw new KernelException( String.format("Expected float value but got: %s", valueNode.toString())); } return Literal.ofFloat(valueNode.floatValue()); } else if (dataType instanceof DoubleType) { if (valueNode.isTextual()) { // Special double values are stored as strings during serialization String textValue = valueNode.asText(); switch (textValue) { case "NaN": return Literal.ofDouble(Double.NaN); case "Infinity": return Literal.ofDouble(Double.POSITIVE_INFINITY); case "-Infinity": return Literal.ofDouble(Double.NEGATIVE_INFINITY); default: throw new KernelException( String.format("Expected double value but got unexpected string: %s", textValue)); } } if (!valueNode.isNumber()) { throw new KernelException( String.format("Expected double value but got: %s", valueNode.toString())); } return Literal.ofDouble(valueNode.asDouble()); } else if (dataType instanceof StringType) { if (!valueNode.isTextual()) { throw new KernelException( String.format("Expected string value but got: %s", valueNode.toString())); } return Literal.ofString(valueNode.asText()); } else if (dataType instanceof BinaryType) { if (!valueNode.isTextual()) { throw new KernelException( String.format("Expected binary (as string) value but got: %s", valueNode.toString())); } // Binary data was stored as UTF-8 string during serialization return Literal.ofBinary(valueNode.asText().getBytes(StandardCharsets.UTF_8)); } else if (dataType instanceof DecimalType) { if (!valueNode.isNumber()) { throw new KernelException( String.format("Expected decimal value but got: %s", valueNode.toString())); } DecimalType decimalType = (DecimalType) dataType; BigDecimal decimal = valueNode.decimalValue(); return Literal.ofDecimal(decimal, decimalType.getPrecision(), decimalType.getScale()); } else if (dataType instanceof DateType) { if (!valueNode.isTextual()) { throw new KernelException( String.format("Expected date (as string) value but got: %s", valueNode.toString())); } String textValue = valueNode.asText(); LocalDate date = LocalDate.parse(textValue, ISO_LOCAL_DATE); return Literal.ofDate((int) date.toEpochDay()); } else if (dataType instanceof TimestampType) { if (!valueNode.isTextual()) { throw new KernelException( String.format( "Expected timestamp (as string) value but got: %s", valueNode.toString())); } String textValue = valueNode.asText(); OffsetDateTime offsetDateTime = OffsetDateTime.parse(textValue, TIMESTAMP_FORMATTER); return Literal.ofTimestamp(TimestampUtils.toEpochMicros(offsetDateTime)); } else if (dataType instanceof TimestampNTZType) { if (!valueNode.isTextual()) { throw new KernelException( String.format( "Expected timestamp NTZ (as string) value but got: %s", valueNode.toString())); } String textValue = valueNode.asText(); LocalDateTime localDateTime = LocalDateTime.parse(textValue, DateTimeFormatter.ISO_LOCAL_DATE_TIME); return Literal.ofTimestampNtz(TimestampUtils.toEpochMicros(localDateTime)); } else if (dataType instanceof VariantType) { if (!valueNode.isTextual()) { throw new KernelException( String.format("Expected variant as string value but got: %s", valueNode)); } String textValue = valueNode.asText(); return Literal.ofString(textValue); } else { throw unsupportedStatsDataType(dataType); } } catch (Exception e) { if (e instanceof KernelException) { throw (KernelException) e; } throw new KernelException( String.format( "Failed to parse value '%s' as %s", valueNode.toString(), dataType.toString()), e); } } /** * Get the timestamp formatter used for parsing/formatting timestamps. Package-private for use by * DataFileStatistics. */ static DateTimeFormatter getTimestampFormatter() { return TIMESTAMP_FORMATTER; } /** Get the epoch offset date time constant. Package-private for use by DataFileStatistics. */ static OffsetDateTime getEpoch() { return EPOCH; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/ManualClock.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; /** A clock whose time can be manually set and modified. */ public class ManualClock implements Clock { private long timeMillis; public ManualClock(long timeMillis) { this.timeMillis = timeMillis; } /** @param timeToSet new time (in milliseconds) that the clock should represent */ public synchronized void setTime(long timeToSet) { this.timeMillis = timeToSet; this.notifyAll(); } @Override public long getTimeMillis() { return timeMillis; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/PartitionUtils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; import static io.delta.kernel.expressions.AlwaysFalse.ALWAYS_FALSE; import static io.delta.kernel.expressions.AlwaysTrue.ALWAYS_TRUE; import static io.delta.kernel.internal.DeltaErrors.wrapEngineException; import static io.delta.kernel.internal.util.ExpressionUtils.createPredicate; import static io.delta.kernel.internal.util.InternalUtils.toLowerCaseSet; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static io.delta.kernel.internal.util.SchemaUtils.casePreservingPartitionColNames; import static java.util.Arrays.asList; import io.delta.kernel.data.*; import io.delta.kernel.engine.Engine; import io.delta.kernel.engine.ExpressionHandler; import io.delta.kernel.expressions.*; import io.delta.kernel.internal.DeltaErrorsInternal; import io.delta.kernel.internal.InternalScanFileUtils; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.types.*; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; import java.sql.Date; import java.sql.Timestamp; import java.time.*; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.*; import java.util.stream.Collectors; public class PartitionUtils { private static final DateTimeFormatter PARTITION_TIMESTAMP_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSS"); private PartitionUtils() {} /** * Utility method to attach partition columns to the given data batch. * * @param dataBatch Data batch to which the partition columns will be added. * @param logicalReadSchema Logical schema of the table scan. Used to insert partition columns at * the right positions in the data batch. This logical schema must contain column mapping * metadata if column mapping is enabled. * @param partitionValues Map of partition column name to value. * @param expressionHandler Expression handler used to evaluate the partition values. * @return A new {@link ColumnarBatch} with the partition columns added. */ public static ColumnarBatch withPartitionColumns( ColumnarBatch dataBatch, StructType logicalReadSchema, Map partitionValues, ExpressionHandler expressionHandler) { if (partitionValues == null || partitionValues.isEmpty()) { // No partition column vectors to attach to the data batch return dataBatch; } // We verify that the number of partition columns in the logical schema plus the number of // columns in the data batch schema is equal to the length of the logical schema. // `partitionValues` contains all partition columns of the table (not just the requested ones), // so we first need to count the number of partition columns in the logical schema. int numPartitionColumnsInSchema = (int) logicalReadSchema.fields().stream() .map(ColumnMapping::getPhysicalName) .filter(partitionValues::containsKey) .count(); if (numPartitionColumnsInSchema + dataBatch.getSchema().length() != logicalReadSchema.length()) { throw DeltaErrorsInternal.logicalPhysicalSchemaMismatch( numPartitionColumnsInSchema, dataBatch.getSchema().length(), logicalReadSchema.length()); } for (int colIdx = 0; colIdx < logicalReadSchema.length(); colIdx++) { // We must iterate the logical schema in order since we insert partition columns into the data // batch according to their ordinal in the logical schema. StructField structField = logicalReadSchema.at(colIdx); String physicalName = ColumnMapping.getPhysicalName(structField); if (partitionValues.containsKey(physicalName)) { // Create a partition column vector final ColumnarBatch finalDataBatch = dataBatch; Literal partitionValue = literalForPartitionValue(structField.getDataType(), partitionValues.get(physicalName)); ExpressionEvaluator evaluator = wrapEngineException( () -> expressionHandler.getEvaluator( finalDataBatch.getSchema(), partitionValue, structField.getDataType()), "Get the expression evaluator for partition column %s with type=%s and value=%s", physicalName, structField.getDataType(), partitionValues.get(physicalName)); ColumnVector partitionVector = wrapEngineException( () -> evaluator.eval(finalDataBatch), "Evaluating the partition value expression %s", partitionValue); dataBatch = dataBatch.withNewColumn(colIdx, structField, partitionVector); } } return dataBatch; } /** * Convert the given partition values to a {@link MapValue} that can be serialized to a Delta * commit file. * * @param partitionValueMap Expected the partition column names to be same case as in the schema. * We want to preserve the case of the partition column names when serializing to the Delta * commit file. * @return {@link MapValue} representing the serialized partition values that can be written to a * Delta commit file. */ public static MapValue serializePartitionMap(Map partitionValueMap) { if (partitionValueMap == null || partitionValueMap.isEmpty()) { return VectorUtils.stringStringMapValue(Collections.emptyMap()); } Map serializedPartValues = new HashMap<>(); for (Map.Entry entry : partitionValueMap.entrySet()) { serializedPartValues.put( entry.getKey(), // partition column name serializePartitionValue(entry.getValue())); // serialized partition value as str } return VectorUtils.stringStringMapValue(serializedPartValues); } /** * Validate {@code partitionValues} contains values for every partition column in the table and * the type of the value is correct. Once validated the partition values are sanitized to match * the case of the partition column names in the table schema and returned * * @param tableSchema Schema of the table. * @param partitionColNames Partition column name. These should be from the table metadata that * retain the same case as in the table schema. * @param partitionValues Map of partition column to value map given by the connector * @return Sanitized partition values. */ public static Map validateAndSanitizePartitionValues( StructType tableSchema, List partitionColNames, Map partitionValues) { if (!toLowerCaseSet(partitionColNames).equals(toLowerCaseSet(partitionValues.keySet()))) { throw new IllegalArgumentException( String.format( "Partition values provided are not matching the partition columns. " + "Partition columns: %s, Partition values: %s", partitionColNames, partitionValues)); } // Convert the partition column names in given `partitionValues` to schema case. Schema // case is the exact case the column name was given by the connector when creating the // table. Comparing the column names is case-insensitive, but preserve the case as stored // in the table metadata when writing the partition column name to DeltaLog // (`partitionValues` in `AddFile`) or generating the target directory for writing the // data belonging to a partition. Map schemaCasePartitionValues = casePreservingPartitionColNames(partitionColNames, partitionValues); // validate types are the same schemaCasePartitionValues .entrySet() .forEach( entry -> { String partColName = entry.getKey(); Literal partValue = entry.getValue(); StructField partColField = tableSchema.get(partColName); // this shouldn't happen as we have already validated the partition column names checkArgument( partColField != null, "Partition column %s is not present in the table schema", partColName); DataType partColType = partColField.getDataType(); if (!partColType.isWriteCompatible(partValue.getDataType())) { throw new IllegalArgumentException( String.format( "Partition column %s is of type %s but the value provided is of type %s", partColName, partColType, partValue.getDataType())); } }); return schemaCasePartitionValues; } /** * Validate that the given predicate references only (and at least one) partition columns. * * @throws IllegalArgumentException if the predicate does not reference any partition columns or * if it references any data columns */ public static void validatePredicateOnlyOnPartitionColumns( Predicate predicate, Set partitionColNames) { final Tuple2 metadataAndDataPredicates = splitMetadataAndDataPredicates(predicate, partitionColNames); final Predicate metadataPredicate = metadataAndDataPredicates._1; final Predicate dataPredicate = metadataAndDataPredicates._2; if (metadataPredicate == AlwaysTrue.ALWAYS_TRUE) { throw new IllegalArgumentException( String.format( "Partition predicate must contain at least one partition column: %s", predicate)); } if (dataPredicate != AlwaysTrue.ALWAYS_TRUE) { throw new IllegalArgumentException( String.format("Partition predicate must contain only partition columns: %s", predicate)); } } /** * Split the given predicate into predicate on partition columns and predicate on data columns. * * @param predicate * @param partitionColNames * @return Tuple of partition column predicate and data column predicate. */ public static Tuple2 splitMetadataAndDataPredicates( Predicate predicate, Set partitionColNames) { String predicateName = predicate.getName(); List children = predicate.getChildren(); if ("AND".equalsIgnoreCase(predicateName)) { Predicate left = (Predicate) children.get(0); Predicate right = (Predicate) children.get(1); Tuple2 leftResult = splitMetadataAndDataPredicates(left, partitionColNames); Tuple2 rightResult = splitMetadataAndDataPredicates(right, partitionColNames); return new Tuple2<>( combineWithAndOp(leftResult._1, rightResult._1), combineWithAndOp(leftResult._2, rightResult._2)); } if (hasNonPartitionColumns(children, partitionColNames)) { return new Tuple2<>(ALWAYS_TRUE, predicate); } else { return new Tuple2<>(predicate, ALWAYS_TRUE); } } /** * Rewrite the given predicate on partition columns on `partitionValues_parsed` in checkpoint * schema. The rewritten predicate can be pushed to the Parquet reader when reading the checkpoint * files. * * @param predicate Predicate on partition columns. * @param partitionColNameToField Map of partition column name (in lower case) to its {@link * StructField}. * @return Rewritten {@link Predicate} on `partitionValues_parsed` in `add`. */ public static Predicate rewritePartitionPredicateOnCheckpointFileSchema( Predicate predicate, Map partitionColNameToField) { return createPredicate( predicate.getName(), predicate.getChildren().stream() .map(child -> rewriteColRefOnPartitionValuesParsed(child, partitionColNameToField)) .collect(Collectors.toList()), predicate.getCollationIdentifier()); } private static Expression rewriteColRefOnPartitionValuesParsed( Expression expression, Map partitionColMetadata) { if (expression instanceof Column) { Column column = (Column) expression; String partColName = column.getNames()[0]; StructField partColField = partitionColMetadata.get(partColName.toLowerCase(Locale.ROOT)); if (partColField == null) { throw new IllegalArgumentException(partColName + " is not present in metadata"); } String partColPhysicalName = ColumnMapping.getPhysicalName(partColField); return InternalScanFileUtils.getPartitionValuesParsedRefInAddFile(partColPhysicalName); } else if (expression instanceof Predicate) { return rewritePartitionPredicateOnCheckpointFileSchema( (Predicate) expression, partitionColMetadata); } return expression; } /** * Utility method to rewrite the partition predicate referring to the table schema as predicate * referring to the {@code partitionValues} in scan files read from Delta log. The scan file batch * is returned by the {@link io.delta.kernel.Scan#getScanFiles(Engine)}. * *

E.g. given predicate on partition columns: {@code p1 = 'new york' && p2 >= 26} where p1 is * of type string and p2 is of int Rewritten expression looks like: {@code * element_at(Column('add', 'partitionValues'), 'p1') = 'new york' && * partition_value(element_at(Column('add', 'partitionValues'), 'p2'), 'integer') >= 26} * *

The column `add.partitionValues` is a {@literal map(string -> string)} type. Each partition * values is in string serialization format according to the Delta protocol. Expression * `partition_value` deserializes the string value into the given partition column type value. * String type partition values don't need any deserialization. * * @param predicate Predicate containing filters only on partition columns. * @param partitionColMetadata Map of partition column name (in lower case) to its type. * @return */ public static Predicate rewritePartitionPredicateOnScanFileSchema( Predicate predicate, Map partitionColMetadata) { return createPredicate( predicate.getName(), predicate.getChildren().stream() .map(child -> rewritePartitionColumnRef(child, partitionColMetadata)) .collect(Collectors.toList()), predicate.getCollationIdentifier()); } private static Expression rewritePartitionColumnRef( Expression expression, Map partitionColMetadata) { Column scanFilePartitionValuesRef = InternalScanFileUtils.ADD_FILE_PARTITION_COL_REF; if (expression instanceof Column) { Column column = (Column) expression; String partColName = column.getNames()[0]; StructField partColField = partitionColMetadata.get(partColName.toLowerCase(Locale.ROOT)); if (partColField == null) { throw new IllegalArgumentException(partColName + " is not present in metadata"); } DataType partColType = partColField.getDataType(); String partColPhysicalName = ColumnMapping.getPhysicalName(partColField); Expression elementAt = new ScalarExpression( "element_at", asList(scanFilePartitionValuesRef, Literal.ofString(partColPhysicalName))); if (partColType instanceof StringType) { return elementAt; } // Add expression to decode the partition value based on the partition column type. return new PartitionValueExpression(elementAt, partColType); } else if (expression instanceof Predicate) { return rewritePartitionPredicateOnScanFileSchema( (Predicate) expression, partitionColMetadata); } return expression; } /** * Get the target directory for writing data for given partition values. Example: Given partition * values (part1=1, part2='abc'), the target directory will be for a table rooted at * 's3://bucket/table': 's3://bucket/table/part1=1/part2=abc'. * * @param dataRoot Root directory where the data is stored. * @param partitionColNames Partition column names. We need this to create the target directory * structure that is consistent levels of directories. * @param partitionValues Partition values to create the target directory. * @return Target directory path. */ public static String getTargetDirectory( String dataRoot, List partitionColNames, Map partitionValues) { Path targetDirectory = new Path(dataRoot); for (String partitionColName : partitionColNames) { Literal partitionValue = partitionValues.get(partitionColName); checkArgument( partitionValue != null, "Partition column value is missing for column: %s", partitionColName); String serializedValue = serializePartitionValue(partitionValue); if (serializedValue == null) { // Follow the delta-spark behavior to use "__HIVE_DEFAULT_PARTITION__" for null serializedValue = "__HIVE_DEFAULT_PARTITION__"; } else { serializedValue = escapePartitionValue(serializedValue); } String partitionDirectory = partitionColName + "=" + serializedValue; targetDirectory = new Path(targetDirectory, partitionDirectory); } return targetDirectory.toString(); } private static boolean hasNonPartitionColumns( List children, Set partitionColNames) { for (Expression child : children) { if (child instanceof Column) { String[] names = ((Column) child).getNames(); // Partition columns are never of nested types. if (names.length != 1 || !partitionColNames.contains(names[0].toLowerCase(Locale.ROOT))) { return true; } } else { if (hasNonPartitionColumns(child.getChildren(), partitionColNames)) { return true; } } } return false; } private static Predicate combineWithAndOp(Predicate left, Predicate right) { String leftName = left.getName().toUpperCase(); String rightName = right.getName().toUpperCase(); if (leftName.equals("ALWAYS_FALSE") || rightName.equals("ALWAYS_FALSE")) { return ALWAYS_FALSE; } if (leftName.equals("ALWAYS_TRUE")) { return right; } if (rightName.equals("ALWAYS_TRUE")) { return left; } return new And(left, right); } /** * Try parsing the standard formatted timestamp (e.g. 2024-03-11 11:00:00.123456). Return the * number of microseconds since epoch. */ private static Optional tryParseStandardTimestamp(String value) { try { Timestamp ts = Timestamp.valueOf(value); return Optional.of(InternalUtils.microsSinceEpoch(ts)); } catch (IllegalArgumentException e) { return Optional.empty(); } } /** * Try parsing the ISO8601 formatted timestamp (e.g. 1970-01-01T00:00:00.123456Z). Return the * number of microseconds since epoch. */ private static Optional tryParseIsoTimestamp(String value) { try { Instant instant = Instant.parse(value); long micros = instant.getEpochSecond() * 1_000_000L + instant.getNano() / 1000L; return Optional.of(micros); } catch (DateTimeParseException e) { return Optional.empty(); } } /** * Try parsing the timestamp, could be in the standard format or ISO8601 format. Return the * Literal Object. */ public static long tryParseTimestamp(String partitionValue) { // ISO8601 format contains 'T' separator, standard format uses space Optional micros = partitionValue.contains("T") ? tryParseIsoTimestamp(partitionValue) : tryParseStandardTimestamp(partitionValue); // If the first attempt failed, try the other format as fallback (this really shouldn't happen) if (!micros.isPresent()) { micros = partitionValue.contains("T") ? tryParseStandardTimestamp(partitionValue) : tryParseIsoTimestamp(partitionValue); } return micros.orElseThrow( () -> DeltaErrorsInternal.invalidTimestampFormatForPartitionValue(partitionValue)); } protected static Literal literalForPartitionValue(DataType dataType, String partitionValue) { if (partitionValue == null) { return Literal.ofNull(dataType); } if (dataType instanceof BooleanType) { return Literal.ofBoolean(Boolean.parseBoolean(partitionValue)); } if (dataType instanceof ByteType) { return Literal.ofByte(Byte.parseByte(partitionValue)); } if (dataType instanceof ShortType) { return Literal.ofShort(Short.parseShort(partitionValue)); } if (dataType instanceof IntegerType) { return Literal.ofInt(Integer.parseInt(partitionValue)); } if (dataType instanceof LongType) { return Literal.ofLong(Long.parseLong(partitionValue)); } if (dataType instanceof FloatType) { return Literal.ofFloat(Float.parseFloat(partitionValue)); } if (dataType instanceof DoubleType) { return Literal.ofDouble(Double.parseDouble(partitionValue)); } if (dataType instanceof StringType) { return Literal.ofString(partitionValue); } if (dataType instanceof BinaryType) { return Literal.ofBinary(partitionValue.getBytes()); } if (dataType instanceof DateType) { return Literal.ofDate(InternalUtils.daysSinceEpoch(Date.valueOf(partitionValue))); } if (dataType instanceof DecimalType) { DecimalType decimalType = (DecimalType) dataType; return Literal.ofDecimal( new BigDecimal(partitionValue), decimalType.getPrecision(), decimalType.getScale()); } if (dataType instanceof TimestampType) { return Literal.ofTimestamp(tryParseTimestamp(partitionValue)); } if (dataType instanceof TimestampNTZType) { // Both the timestamp and timestamp_ntz have no timezone info, so they are interpreted // in local time zone. return Literal.ofTimestampNtz( InternalUtils.microsSinceEpoch(Timestamp.valueOf(partitionValue))); } throw new UnsupportedOperationException("Unsupported partition column: " + dataType); } /** * Serialize the given partition value to a string according to the Delta protocol partition value serialization rules. * * @param literal Literal representing the partition value of specific datatype. * @return Serialized string representation of the partition value. */ protected static String serializePartitionValue(Literal literal) { Object value = literal.getValue(); if (value == null) { return null; } DataType dataType = literal.getDataType(); if (dataType instanceof ByteType || dataType instanceof ShortType || dataType instanceof IntegerType || dataType instanceof LongType || dataType instanceof FloatType || dataType instanceof DoubleType || dataType instanceof BooleanType) { return String.valueOf(value); } else if (dataType instanceof StringType) { return (String) value; } else if (dataType instanceof DateType) { int daysSinceEpochUTC = (int) value; return LocalDate.ofEpochDay(daysSinceEpochUTC).toString(); } else if (dataType instanceof TimestampType || dataType instanceof TimestampNTZType) { long microsSinceEpochUTC = (long) value; long seconds = microsSinceEpochUTC / 1_000_000; int microsOfSecond = (int) (microsSinceEpochUTC % 1_000_000); if (microsOfSecond < 0) { // also adjust for negative microsSinceEpochUTC microsOfSecond = 1_000_000 + microsOfSecond; } int nanosOfSecond = microsOfSecond * 1_000; LocalDateTime localDateTime = LocalDateTime.ofEpochSecond(seconds, nanosOfSecond, ZoneOffset.UTC); return localDateTime.format(PARTITION_TIMESTAMP_FORMATTER); } else if (dataType instanceof DecimalType) { return ((BigDecimal) value).toString(); } else if (dataType instanceof BinaryType) { return new String((byte[]) value, StandardCharsets.UTF_8); } throw new UnsupportedOperationException("Unsupported partition column type: " + dataType); } //////////////////////////////////////////////////////////////////////////////////////////////// // The following string escaping code is mainly copied from Spark // // (org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils) which is copied from // // Hive (o.a.h.h.common.FileUtils). // //////////////////////////////////////////////////////////////////////////////////////////////// private static final BitSet CHARS_TO_ESCAPE = new BitSet(128); static { // ASCII 01-1F are HTTP control characters that need to be escaped. char[] controlChars = new char[] { '\u0001', '\u0002', '\u0003', '\u0004', '\u0005', '\u0006', '\u0007', '\b', '\t', '\n', '\u000B', '\f', '\r', '\u000E', '\u000F', '\u0010', '\u0011', '\u0012', '\u0013', '\u0014', '\u0015', '\u0016', '\u0017', '\u0018', '\u0019', '\u001A', '\u001B', '\u001C', '\u001D', '\u001E', '\u001F', '"', '#', '%', '\'', '*', '/', ':', '=', '?', '\\', '\u007F', '{', '[', ']', '^' }; for (char c : controlChars) { CHARS_TO_ESCAPE.set(c); } } /** * Escapes the given string to be used as a partition value in the path. Basically this escapes * *

    *
  • characters that can't be in a file path. E.g. `a\nb` will be escaped to `a%0Ab`. *
  • character that are cause ambiguity in partition value parsing. E.g. For partition column * `a` having value `b=c`, the path should be `a=b%3Dc` *
* * @param value The partition value to escape. * @return The escaped partition value. */ private static String escapePartitionValue(String value) { StringBuilder escaped = new StringBuilder(value.length()); for (int i = 0; i < value.length(); i++) { char c = value.charAt(i); if (c >= 0 && c < CHARS_TO_ESCAPE.size() && CHARS_TO_ESCAPE.get(c)) { escaped.append('%'); escaped.append(String.format("%02X", (int) c)); } else { escaped.append(c); } } return escaped.toString(); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/Preconditions.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; import java.util.function.Supplier; /** * Static convenience methods that help a method or constructor check whether it was invoked * correctly (that is, whether its preconditions were met). */ public class Preconditions { private Preconditions() {} /** * Precondition-style validation that throws {@link IllegalArgumentException}. * * @param isValid {@code true} if valid, {@code false} if an exception should be thrown * @throws IllegalArgumentException if {@code isValid} is false */ public static void checkArgument(boolean isValid) throws IllegalArgumentException { if (!isValid) { throw new IllegalArgumentException(); } } /** * Precondition-style validation that throws {@link IllegalArgumentException}. * * @param isValid {@code true} if valid, {@code false} if an exception should be thrown * @param message A String message for the exception. * @throws IllegalArgumentException if {@code isValid} is false */ public static void checkArgument(boolean isValid, String message) throws IllegalArgumentException { if (!isValid) { throw new IllegalArgumentException(message); } } /** * Precondition-style validation that throws {@link IllegalArgumentException}. The message is only * evaluated if the validation fails. * * @param isValid {@code true} if valid, {@code false} if an exception should be thrown * @param messageSupplier A supplier that provides the exception message (evaluated lazily) * @throws IllegalArgumentException if {@code isValid} is false */ public static void checkArgument(boolean isValid, Supplier messageSupplier) throws IllegalArgumentException { if (!isValid) { throw new IllegalArgumentException(messageSupplier.get()); } } /** * Precondition-style validation that throws {@link IllegalArgumentException}. * * @param isValid {@code true} if valid, {@code false} if an exception should be thrown * @param message A String message for the exception. * @param args Objects used to fill in {@code %s} placeholders in the message * @throws IllegalArgumentException if {@code isValid} is false */ public static void checkArgument(boolean isValid, String message, Object... args) throws IllegalArgumentException { if (!isValid) { throw new IllegalArgumentException(String.format(String.valueOf(message), args)); } } /** * Ensures the truth of an expression involving the state of the calling instance. * * @param expression a boolean expression * @param errorMessage the exception message to use if the check fails * @throws IllegalStateException if {@code expression} is false */ public static void checkState(boolean expression, String errorMessage) { if (!expression) { throw new IllegalStateException(errorMessage); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/SchemaChanges.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; import io.delta.kernel.types.StructField; import io.delta.kernel.types.StructType; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; /** * SchemaChanges encapsulates a list of added, removed, renamed, or updated fields in a schema * change. Updates include renamed fields, reordered fields, type changes, nullability changes, and * metadata attribute changes. This set of updates can apply to nested fields within structs. In * case any update is applied to a nested field, an update will be produced for every level of * nesting. This includes re-ordered columns in a nested field. Note that SchemaChanges does not * capture re-ordered columns in top level schema. * *

For example, given a field struct_col: struct> if id is renamed to * `renamed_id` 1 update will be produced for the change to struct_col and 1 update will be produced * for the change to inner_struct * *

ToDo: Possibly track moves/renames independently, enable capturing re-ordered columns in top * level schema */ class SchemaChanges { public static class SchemaUpdate { private final StructField fieldBefore; private final StructField fieldAfter; // This is a "." concatenated path to the field. Names containing "." are wrapped in // back-ticks (`). // For example in the schema >> the path to "c" would be: // "`a.b`.element.c". In general, though the format should not be relid upon since this // is used for surfacing errors to users. // Note this is a by name. If we want to be able to track changes // at the where an element is moved to a different location in the // schema we need to add more paths here. private final String pathToAfterField; SchemaUpdate(StructField fieldBefore, StructField fieldAfter, String pathToAfterField) { this.fieldBefore = fieldBefore; this.fieldAfter = fieldAfter; this.pathToAfterField = pathToAfterField; } public StructField before() { return fieldBefore; } public StructField after() { return fieldAfter; } public String getPathToAfterField() { return pathToAfterField; } } private List addedFields; private List removedFields; private List updatedFields; private Optional updatedSchema; private SchemaChanges( List addedFields, List removedFields, List updatedFields, Optional updatedSchema) { this.addedFields = Collections.unmodifiableList(addedFields); this.removedFields = Collections.unmodifiableList(removedFields); this.updatedFields = Collections.unmodifiableList(updatedFields); this.updatedSchema = updatedSchema; } static class Builder { private List addedFields = new ArrayList<>(); private List removedFields = new ArrayList<>(); private List updatedFields = new ArrayList<>(); private Optional updatedSchema = Optional.empty(); public Builder withAddedField(StructField addedField) { addedFields.add(addedField); return this; } public Builder withRemovedField(StructField removedField) { removedFields.add(removedField); return this; } public Builder withUpdatedField( StructField existingField, StructField newField, String pathToAfterField) { updatedFields.add(new SchemaUpdate(existingField, newField, pathToAfterField)); return this; } public Builder withUpdatedSchema(StructType updatedSchema) { this.updatedSchema = Optional.of(updatedSchema); return this; } public SchemaChanges build() { return new SchemaChanges(addedFields, removedFields, updatedFields, updatedSchema); } } public static Builder builder() { return new Builder(); } /* Added Fields */ public List addedFields() { return addedFields; } /* Removed Fields */ public List removedFields() { return removedFields; } /* Updated Fields (e.g. rename, type change) represented */ public List updatedFields() { return updatedFields; } public Optional updatedSchema() { return updatedSchema; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/SchemaIterable.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.types.*; import java.util.*; import java.util.function.Consumer; import java.util.stream.Stream; import java.util.stream.StreamSupport; /** * Utility class for iterating over schema structures and modifying them. * *

Sample usage for iterating schemas: * * 

{@code
 * StructType schema = ...;
 * for (SchemaIterable.SchemaElement element : new SchemaIterable(schema)) {
 *    StructField field = element.getField();
 *    // Get info from field in some way.
 * }
 * }
* *

Sample usage for mutating schemas: * * 

{@code
 * StructType schema = ...;
 * SchemaIterable schemaIterable = new SchemaIterable(schema);
 * Iterator iterator = schemaIterable.newMutableIterator();
 * while (iterator.hasNext()) {
 *    SchemaIterable.MutableSchemaElement element = iterator.next();
 *    // Calculate a new field in some way then call updateField.
 *    element.updateField(...)
 * }
 * updatedSchema = schemaIterable.getSchema();
 * }
*/ public class SchemaIterable implements Iterable { private final Class[] typesToSkipRecursion; private StructType schema; /** Construct a new Iterable for the schema. */ public SchemaIterable(StructType schema) { this(schema, new Class[0]); } /** * Constructs a new iterable that skips recursion for some data types. * *

No recursion will be done on any type that is an instance of a class in * typesToSkipRecursion. * *

For example with schema: * * 

{@code
   * a: Map
   * b: Array
   * }
* *

If {@code typesToSkipRecursion = {MapType.class}} is provided then the iterator will only * visit "a", "b.element", and "b". * *

If {@code typesToSkipRecursion = {ArrayType.class}} is provided then the iterator wil only * visit "a.key", "a.value", "a" and "b". */ public static SchemaIterable newSchemaIterableWithIgnoredRecursion( StructType schema, Class[] typesToSkipRecursion) { return new SchemaIterable(schema, typesToSkipRecursion); } private SchemaIterable(StructType schema, Class[] typesToSkipRecursion) { this.schema = schema; this.typesToSkipRecursion = Arrays.copyOf(typesToSkipRecursion, typesToSkipRecursion.length); } /** * Gets the latest schema (either the initial schema or the one set after a mutable iterator is * fully consumed). */ public StructType getSchema() { return schema; } @Override public Iterator iterator() { Iterator iterator = newMutableIterator(); return new Iterator() { @Override public boolean hasNext() { return iterator.hasNext(); } @Override public SchemaElement next() { return iterator.next(); } }; } public Stream stream() { return StreamSupport.stream(spliterator(), /* parallel = */ false); } public Stream mutableStream() { return StreamSupport.stream( Spliterators.spliteratorUnknownSize(newMutableIterator(), /*characteristics = */ 0), /* parallel = */ false); } /** * Returns a new iterator that can be used to iterate and update elements in the schema. The * current schema on this iterable is updated once the returned iterator is fully consumed. * *

Consuming multiple iterators across different threads concurrently is not thread safe. * *

Example usage: * * 

{@code
   * Iterator iterator = schemaIterable.newMutableIterator();
   * while (iterator.hasNext()) {
   *    SchemaIterable.MutableSchemaElement element = iterator.next();
   *    element.updateField(...)
   * }
   * updatedSchema = schemaIterable.getSchema();
   * }
*/ public Iterator newMutableIterator() { return new SchemaIterator(schema, this::setSchema, typesToSkipRecursion); } private void setSchema(StructType newSchema) { this.schema = newSchema; } /** * Iterator that performs a depth-first traversal of a schema structure using a zipper pattern. * Each call to next() returns the next zipper in the traversal sequence. */ private static class SchemaIterator implements Iterator { private final Consumer finalizedSchemaConsumer; private SchemaZipper nextZipper; private boolean finishedVisitingCurrent = false; SchemaIterator( StructType schema, Consumer finalizedSchemaConsumer, Class[] typesToSkipRecursion) { this.nextZipper = SchemaZipper.createZipper(schema, typesToSkipRecursion); this.finalizedSchemaConsumer = finalizedSchemaConsumer; // Special case if struct is empty no force no iteration. this.finishedVisitingCurrent = schema.fields().isEmpty(); } @Override public boolean hasNext() { boolean nextAvailable = nextZipper != null && (!finishedVisitingCurrent // Implies there are children. // Without children there must be siblings or parents left to visit to have a next // value. || nextZipper.hasMoreSiblings() || nextZipper.hasParents()); if (!nextAvailable && nextZipper != null) { finalizedSchemaConsumer.accept((StructType) nextZipper.extractDataTypeFromFields()); nextZipper = null; } return nextAvailable; } @Override public MutableSchemaElement next() { if (!hasNext()) { throw new NoSuchElementException(); } advanceNext(); return nextZipper; } private void advanceNext() { while (nextZipper != null) { if (!finishedVisitingCurrent) { if (nextZipper.hasChildren()) { // Try to go deeper if we haven't visited this node yet while (nextZipper.hasChildren()) { nextZipper = nextZipper.childrenZipper(); } } // At a leaf node, so by definition it is finished visiting. finishedVisitingCurrent = true; return; } // Try moving to sibling if there is no need to go down further. if (nextZipper.hasMoreSiblings()) { nextZipper = nextZipper.moveToSibling(); if (nextZipper.hasChildren()) { // Force visiting children first. finishedVisitingCurrent = false; continue; } finishedVisitingCurrent = true; return; } // Last remaining direction with no children and no // siblings is to pop back up and note that visiting // has finished on the current value. nextZipper = nextZipper.moveToParent(); finishedVisitingCurrent = true; return; } } } /** * Container for parent StructField information returned by {@link * SchemaElement#getParentStructFieldAndPath()}. */ public static class ParentStructFieldInfo { private final StructField parentField; private final String pathFromParent; public ParentStructFieldInfo(StructField parentField, String pathFromParent) { this.parentField = parentField; this.pathFromParent = pathFromParent; } /** Returns the nearest parent StructField. */ public StructField getParentField() { return parentField; } /** * Returns the path from the nearest parent StructField. If the nearest parent StructField is a * direct ancestor, this is "". Otherwise, the path is of the format {@code * ((key|value|element).)*(key|value|element)} */ public String getPathFromParent() { return pathFromParent; } } /** * Interface for representing a schema element as part of a traversal. * *

This object should always be treated ephemeral and not be referenced once {@code next()} is * called on the iterator. */ public interface SchemaElement { /** Get the current field. */ StructField getField(); /** * Returns the nearest parent StructField (is a member of a StructType) if it exists. For an * element that is at the root-level of the schema, this returns Optional.empty(). * *

Also returns the path from the nearest parent StructField. If the nearest parent * StructField is a direct ancestor, this is "". Otherwise, the path is of the format {@code * ((key|value|element).)*(key|value|element)} */ Optional getParentStructFieldAndPath(); /** Returns the path to the node via user facing names. */ String getNamePath(); /** * Returns the nearest ancestor that is a member of a StructType (could be the current element). * *

Maps Keys and Values and Array elements are skipped over when finding the nearest * ancestor. */ StructField getNearestStructFieldAncestor(); /** * Returns the path to this node from the nearest ancestor that is a member of a StructType. * *

Prefix is prepend to any path with an added "." * *

If this element is a StructField returns prefix * *

Otherwise the grammar of the returned field is: {@code * [.]((key|value|element).)*(key|value|element)} */ String getPathFromNearestStructFieldAncestor(String prefix); /** * Returns true if this element is a StructField (as compared to an array element or a map * key/value). */ default boolean isStructField() { return false; } } /** * Interface for manipulating Schema elements. * *

This object should always be treated ephemeral and not be referenced once {@code next()} is * called on the iterator. */ public interface MutableSchemaElement extends SchemaElement { /** Replace the current targeted field with a new field. */ void updateField(StructField structField); /** Replace the metadata on the nearest struct ancestor with new metadata. */ void setMetadataOnNearestStructFieldAncestor(FieldMetadata metadata); } /** * SchemaZipper implements an adaptation of the functional zipper pattern for manipulating schema * structures. * *

As a high-level summary, keeps state of the path used to get a certain element as it moves * through Schema elements. As it moves back up, it reconstructs the schema data types as * necessary. * *

N.B. For clients using this class only one instance of a Zipper should be kept around since * the internal state is mutable. */ private abstract static class SchemaZipper implements MutableSchemaElement { // Path to the current zipper. Note parents is shared between all elements // on the path. private final List parents; private final Class[] typesToSkipRecursion; abstract DataType constructType(); protected List fields; // Current focus element in fields. private int index = 0; private boolean modified = false; private SchemaZipper(List parents, List fields) { this(parents, fields, new Class[0]); } private SchemaZipper( List parents, List fields, Class[] typesToSkipRecursion) { this.parents = parents; this.fields = fields; this.typesToSkipRecursion = typesToSkipRecursion; } /** Returns if the zipper has any children can be traversed. */ public boolean hasChildren() { DataType currentType = currentField().getDataType(); // Skip recursion for specified types for (Class typeToSkip : typesToSkipRecursion) { if (typeToSkip.isInstance(currentType)) { return false; } } boolean isStructType = currentType instanceof StructType; // TODO(#4571): this concept should be centralized. boolean isNested = isStructType || currentType instanceof ArrayType || currentType instanceof MapType; boolean isEmptyStruct = isStructType && ((StructType) currentType).fields().isEmpty(); return isNested && !isEmptyStruct; } static SchemaZipper createZipper(StructType schema, Class[] typesToSkipRecursion) { return createZipper(/*parents=*/ new ArrayList<>(), schema, typesToSkipRecursion); } private static SchemaZipper createZipper( List parents, DataType type, Class[] typesToSkipRecursion) { if (type instanceof ArrayType) { ArrayType arrayType = (ArrayType) type; return new ArraySchemaZipper(parents, arrayType, typesToSkipRecursion); } else if (type instanceof MapType) { MapType mapType = (MapType) type; return new MapSchemaZipper(parents, mapType, typesToSkipRecursion); } else if (type instanceof StructType) { StructType structType = (StructType) type; return new StructSchemaZipper(parents, structType, typesToSkipRecursion); } else { throw new KernelException("Unsupported data type: " + type); } } private static SchemaZipper createZipper( List parents, StructField field, Class[] typesToSkipRecursion) { return createZipper(parents, field.getDataType(), typesToSkipRecursion); } /** Returns a zipper pointing the left-most child field of this zipper. */ public SchemaZipper childrenZipper() { if (!hasChildren()) { return null; } parents.add(this); return createZipper(parents, fields.get(index), typesToSkipRecursion); } /** * Returns a zipper pointing to the next sibling of this zipper. (moving right across zippers). */ public SchemaZipper moveToSibling() { if (!hasMoreSiblings()) { return null; } index++; return this; } public boolean hasMoreSiblings() { return index < fields.size() - 1; } @Override public StructField getField() { return currentField(); } private SchemaZipper getParent() { if (parents.isEmpty()) { return null; } return parents.get(parents.size() - 1); } @Override public Optional getParentStructFieldAndPath() { if (parents.isEmpty()) { return Optional.empty(); } LinkedList names = new LinkedList<>(); for (int i = parents.size() - 1; i >= 0; i--) { SchemaZipper parent = parents.get(i); if (parent.isStructField()) { return Optional.of( new ParentStructFieldInfo(parent.getField(), SchemaUtils.concatWithDot(names))); } // We are traversing parents in reverse so need to insert at the start names.addFirst(parent.currentField().getName()); } throw new IllegalStateException( "At least one parent must be a struct field for a valid schema"); } /** * Returns a new zipper pointing to the parent of this zipper. * *

If the zipper has any modifications they are propagated up to the parent. */ public SchemaZipper moveToParent() { SchemaZipper parent = getParent(); if (!parents.isEmpty()) { parents.remove(parents.size() - 1); } else { return null; } if (modified) { // Propagate changes to parent. StructField currentParentField = parent.currentField(); StructField newParentField = currentParentField.withDataType(extractDataTypeFromFields()); parent.updateField(newParentField); } return parent; } public DataType extractDataTypeFromFields() { return constructType(); } public StructField currentField() { return fields.get(index); } @Override public void updateField(StructField structField) { try { fields.set(index, structField); } catch (UnsupportedOperationException e) { // Field might be immutable, copy and set if this is the case. fields = new ArrayList<>(fields); fields.set(index, structField); } modified = true; } @Override public String getNamePath() { List names = new ArrayList<>(); for (SchemaZipper parent : parents) { names.add(parent.currentField().getName()); } names.add(currentField().getName()); return SchemaUtils.concatWithDot(names); } @Override public StructField getNearestStructFieldAncestor() { if (parents.isEmpty() || this instanceof StructSchemaZipper) { return currentField(); } ListIterator iterator = findNearestStructFieldAncestor(); return iterator.next().currentField(); } /** Returns an iterator to a zipper that has a focus on a StructType child StructField. */ private ListIterator findNearestStructFieldAncestor() { ListIterator iterator = parents.listIterator(parents.size()); while (iterator.hasPrevious()) { SchemaZipper parent = iterator.previous(); if (parent instanceof StructSchemaZipper) { return iterator; } } throw new IllegalArgumentException("no top level parent struct field, this shouldn't happen"); } @Override public void setMetadataOnNearestStructFieldAncestor(FieldMetadata metadata) { if (parents.isEmpty() || this instanceof StructSchemaZipper) { updateField(currentField().withNewMetadata(metadata)); return; } ListIterator iterator = findNearestStructFieldAncestor(); SchemaZipper parent = iterator.next(); parent.updateField(parent.currentField().withNewMetadata(metadata)); } @Override public String getPathFromNearestStructFieldAncestor(String prefix) { if (parents.isEmpty() || this instanceof StructSchemaZipper) { return prefix; } ListIterator iterator = parents.listIterator(parents.size()); int pathSize = prefix.length() + currentField().getName().length(); while (iterator.hasPrevious()) { SchemaZipper parent = iterator.previous(); if (parent instanceof StructSchemaZipper) { break; } pathSize += parent.currentField().getName().length(); } StringBuilder sb = new StringBuilder( pathSize + (prefix.isEmpty() ? 0 : 1 + (parents.size()) - iterator.nextIndex())); if (!prefix.isEmpty()) { sb.append(prefix); sb.append("."); } iterator.next(); while (iterator.hasNext()) { sb.append(iterator.next().currentField().getName()); sb.append("."); } sb.append(currentField().getName()); return sb.toString(); } public boolean hasParents() { return !parents.isEmpty(); } } private static class ArraySchemaZipper extends SchemaZipper { ArraySchemaZipper(List parents, ArrayType arrayType) { super(parents, Collections.singletonList(arrayType.getElementField())); if (!fields.get(0).getName().equals(ArrayType.ARRAY_ELEMENT_NAME)) { throw new KernelException( "ArrayType must have a single field named 'element', found: " + fields.get(0).getName()); } } ArraySchemaZipper( List parents, ArrayType arrayType, Class[] typesToSkipRecursion) { super(parents, Collections.singletonList(arrayType.getElementField()), typesToSkipRecursion); } @Override DataType constructType() { return new ArrayType(fields.get(0)); } } private static class MapSchemaZipper extends SchemaZipper { MapSchemaZipper(List parents, MapType mapType) { super(parents, Arrays.asList(mapType.getKeyField(), mapType.getValueField())); if (!fields.get(0).getName().equals(MapType.MAP_KEY_NAME) || !fields.get(1).getName().equals(MapType.MAP_VALUE_NAME)) { throw new KernelException( "MapType must have two fields named 'key' and 'value', found: " + fields.get(0).getName() + ", " + fields.get(1).getName()); } } MapSchemaZipper(List parents, MapType mapType, Class[] typesToSkipRecursion) { super( parents, Arrays.asList(mapType.getKeyField(), mapType.getValueField()), typesToSkipRecursion); } @Override DataType constructType() { return new MapType(fields.get(0), fields.get(1)); } } private static class StructSchemaZipper extends SchemaZipper { StructSchemaZipper(List parents, StructType structType) { super(parents, structType.fields()); } StructSchemaZipper( List parents, StructType structType, Class[] typesToSkipRecursion) { super(parents, structType.fields(), typesToSkipRecursion); } @Override public boolean isStructField() { return true; } @Override DataType constructType() { return new StructType(fields); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/SchemaUtils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; import static io.delta.kernel.internal.DeltaErrors.*; import static io.delta.kernel.internal.tablefeatures.TableFeatures.*; import static io.delta.kernel.internal.util.ColumnMapping.*; import static io.delta.kernel.internal.util.ColumnMapping.getColumnId; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.lang.String.format; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.expressions.Column; import io.delta.kernel.expressions.Literal; import io.delta.kernel.internal.DeltaErrors; import io.delta.kernel.internal.TableConfig; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.columndefaults.ColumnDefaults; import io.delta.kernel.internal.skipping.StatsSchemaHelper; import io.delta.kernel.internal.types.TypeWideningChecker; import io.delta.kernel.types.*; import java.util.*; import java.util.stream.Collectors; /** * Utility methods for schema related operations such as validating the schema has no duplicate * columns and the names contain only valid characters. */ public class SchemaUtils { private SchemaUtils() {} /** * Validate the schema. This method checks if the schema has no duplicate columns, the names * contain only valid characters, the data types are supported, and the column metadata is valid. * * @param schema the schema to validate * @param isColumnMappingEnabled whether column mapping is enabled. When column mapping is * enabled, the column names in the schema can contain special characters that are allowed as * column names in the Parquet file * @param isColumnDefaultEnabled whether column defaults is enabled * @throws IllegalArgumentException if the schema is invalid */ public static void validateSchema( StructType schema, boolean isColumnMappingEnabled, boolean isColumnDefaultEnabled, boolean isIcebergCompatV3Enabled) { checkArgument(schema.length() > 0, "Schema should contain at least one column"); List flattenColNames = new SchemaIterable(schema) .stream() // Paths to struct fields are sufficient to find duplicate columns, as arrays/maps // always have the same names. .filter(SchemaIterable.SchemaElement::isStructField) .map(SchemaIterable.SchemaElement::getNamePath) .collect(Collectors.toList()); // check there are no duplicate column names in the schema Set uniqueColNames = flattenColNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); if (uniqueColNames.size() != flattenColNames.size()) { Set uniqueCols = new HashSet<>(); List duplicateColumns = flattenColNames.stream() .map(String::toLowerCase) .filter(n -> !uniqueCols.add(n)) .sorted(String::compareTo) .collect(Collectors.toList()); throw DeltaErrors.duplicateColumnsInSchema(schema, duplicateColumns); } // Check the column names are valid if (!isColumnMappingEnabled) { validParquetColumnNames(flattenColNames); } else { // when column mapping is enabled, just check the name contains no new line in it. flattenColNames.forEach( name -> { if (name.contains("\\n")) { throw invalidColumnName(name, "\\n"); } }); } validateSupportedType(schema); ColumnDefaults.validateSchema(schema, isColumnDefaultEnabled, isIcebergCompatV3Enabled); } /** * Performs the following validations on an updated table schema using the current schema as a * base for validation. ColumnMapping must be enabled to call this. * *

Returns an updated schema if metadata (i.e. TypeChanges needs to be copied over from * currentSchema and new type changes need to be recorded. Kernel is expected to handle this work * instead of clients). * *

The following checks are performed: * *

    *
  • No duplicate columns are allowed *
  • Column names contain only valid characters *
  • Data types are supported *
  • Physical column name consistency is preserved in the new schema *
  • If IcebergWriterCompatV1 is enabled, that map struct keys have not changed *
  • ToDo: Nested IDs for array/map types are preserved in the new schema for IcebergCompatV2 *
*/ public static Optional validateUpdatedSchemaAndGetUpdatedSchema( Metadata currentMetadata, Metadata newMetadata, Protocol newProtocol, Set clusteringColumnPhysicalNames, boolean allowNewRequiredFields) { checkArgument( isColumnMappingModeEnabled( ColumnMapping.getColumnMappingMode(newMetadata.getConfiguration())), "Cannot validate updated schema when column mapping is disabled"); validateSchema( newMetadata.getSchema(), true /*columnMappingEnabled*/, newProtocol.supportsFeature(ALLOW_COLUMN_DEFAULTS_W_FEATURE), TableConfig.ICEBERG_COMPAT_V3_ENABLED.fromMetadata(newMetadata)); validatePartitionColumns( newMetadata.getSchema(), new ArrayList<>(newMetadata.getPartitionColNames())); int currentMaxFieldId = Integer.parseInt( currentMetadata.getConfiguration().getOrDefault(COLUMN_MAPPING_MAX_COLUMN_ID_KEY, "0")); return validateSchemaEvolution( currentMetadata.getSchema(), newMetadata.getSchema(), ColumnMapping.getColumnMappingMode(newMetadata.getConfiguration()), clusteringColumnPhysicalNames, currentMaxFieldId, allowNewRequiredFields, TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.fromMetadata(newMetadata.getConfiguration()), TableConfig.TYPE_WIDENING_ENABLED.fromMetadata(newMetadata.getConfiguration())); } /** * Validates a given schema evolution by using field ID as the source of truth for identifying * fields * * @param currentSchema the schema that is present the table schema _before_ the schema evolution * @param newSchema the new schema that is present the table schema _after_ the schema evolution * @param clusteringColumnPhysicalNames The clustering columns present in the table before the * schema update * @param oldMaxFieldId the maximum field id in the table before the schema update * @param allowNewRequiredFields If `false`, adding new required columns throws an error. If * `true`, new required columns are allowed * @param icebergWriterCompatV1Enabled `true` if icebergCompatV1 is enabled on the table * @return an updated schema if metadata (e.g. TypeChanges needs to be copied over from the old * schema * @throws IllegalArgumentException if the schema evolution is invalid */ // TODO: Consider renaming or refactoring to avoid returning the // StructType here. public static Optional validateSchemaEvolutionById( StructType currentSchema, StructType newSchema, Set clusteringColumnPhysicalNames, int oldMaxFieldId, boolean allowNewRequiredFields, boolean icebergWriterCompatV1Enabled, boolean typeWideningEnabled) { SchemaChanges schemaChanges = computeSchemaChangesById(currentSchema, newSchema); validatePhysicalNameConsistency(schemaChanges.updatedFields()); // Validates that the updated schema does not contain breaking changes in terms of types and // nullability validateUpdatedSchemaCompatibility( schemaChanges, oldMaxFieldId, allowNewRequiredFields, icebergWriterCompatV1Enabled, typeWideningEnabled); validateClusteringColumnsNotDropped( schemaChanges.removedFields(), clusteringColumnPhysicalNames); return schemaChanges.updatedSchema(); // ToDo Potentially validate IcebergCompatV2 nested IDs } /** * Verify the partition columns exists in the table schema and a supported data type for a * partition column. * * @param schema * @param partitionCols */ public static void validatePartitionColumns(StructType schema, List partitionCols) { // partition columns are always the top-level columns Map columnNameToType = schema.fields().stream() .collect( Collectors.toMap( field -> field.getName().toLowerCase(Locale.ROOT), StructField::getDataType)); partitionCols.stream() .forEach( partitionCol -> { DataType dataType = columnNameToType.get(partitionCol.toLowerCase(Locale.ROOT)); checkArgument( dataType != null, "Partition column %s not found in the schema", partitionCol); if (!(dataType instanceof BooleanType || dataType instanceof ByteType || dataType instanceof ShortType || dataType instanceof IntegerType || dataType instanceof LongType || dataType instanceof FloatType || dataType instanceof DoubleType || dataType instanceof DecimalType || dataType instanceof StringType || dataType instanceof BinaryType || dataType instanceof DateType || dataType instanceof TimestampType || dataType instanceof TimestampNTZType)) { throw unsupportedPartitionDataType(partitionCol, dataType); } }); } /** * Delta expects partition column names to be same case preserving as the name in the schema. E.g: * Schema: (a INT, B STRING) and partition columns: (b). In this case we store the schema as (a * INT, B STRING) and partition columns as (B). * *

This method expects the inputs are already validated (i.e. schema contains all the partition * columns). */ public static List casePreservingPartitionColNames( StructType tableSchema, List partitionColumns) { Map columnNameMap = new HashMap<>(); tableSchema .fieldNames() .forEach(colName -> columnNameMap.put(colName.toLowerCase(Locale.ROOT), colName)); return partitionColumns.stream() .map(colName -> columnNameMap.get(colName.toLowerCase(Locale.ROOT))) .collect(Collectors.toList()); } /** * Convert the partition column names in {@code partitionValues} map into the same case as the * column in the table metadata. Delta expects the partition column names to preserve the case * same as the table schema. * * @param partitionColNames List of partition columns in the table metadata. The names preserve * the case as given by the connector when the table is created. * @param partitionValues Map of partition column name to partition value. Convert the partition * column name to be same case preserving name as its equivalent column in the {@code * partitionColName}. Column name comparison is case-insensitive. * @return Rewritten {@code partitionValues} map with names case preserved. */ public static Map casePreservingPartitionColNames( List partitionColNames, Map partitionValues) { Map partitionColNameMap = new HashMap<>(); partitionColNames.forEach( colName -> partitionColNameMap.put(colName.toLowerCase(Locale.ROOT), colName)); return partitionValues.entrySet().stream() .collect( Collectors.toMap( entry -> partitionColNameMap.get(entry.getKey().toLowerCase(Locale.ROOT)), Map.Entry::getValue)); } /** * Verify the clustering columns exists in the table schema. * * @param schema The schema of the table * @param clusteringCols List of clustering columns */ public static List casePreservingEligibleClusterColumns( StructType schema, List clusteringCols) { List> physicalColumnsWithTypes = clusteringCols.stream() .map(col -> ColumnMapping.getPhysicalColumnNameAndDataType(schema, col)) .collect(Collectors.toList()); List nonSkippingEligibleColumns = physicalColumnsWithTypes.stream() .filter(tuple -> !StatsSchemaHelper.isSkippingEligibleDataType(tuple._2)) .map(tuple -> tuple._1.toString() + " : " + tuple._2) .collect(Collectors.toList()); if (!nonSkippingEligibleColumns.isEmpty()) { throw new KernelException( format( "Clustering is not supported because the following column(s): %s " + "don't support data skipping", nonSkippingEligibleColumns)); } return physicalColumnsWithTypes.stream().map(tuple -> tuple._1).collect(Collectors.toList()); } /** * Search (case-insensitive) for the given {@code colName} in the {@code schema} and return its * position in the {@code schema}. * * @param schema {@link StructType} * @param colName Name of the column whose index is needed. * @return Valid index or -1 if not found. */ public static int findColIndex(StructType schema, String colName) { for (int i = 0; i < schema.length(); i++) { if (schema.at(i).getName().equalsIgnoreCase(colName)) { return i; } } return -1; } /** * Collects all leaf columns from the given schema (including flattened columns only for * StructTypes), up to maxColumns. NOTE: If maxColumns = -1, we collect ALL leaf columns in the * schema. */ public static List collectLeafColumns( StructType schema, Set excludedColumns, int maxColumns) { List result = new ArrayList<>(); collectLeafColumnsInternal(schema, null, excludedColumns, result, maxColumns); return result; } /** @return column name by concatenating the column path elements (think of nested) with dots */ public static String concatWithDot(List columnPath) { return columnPath.stream().map(SchemaUtils::escapeDots).collect(Collectors.joining(".")); } /** Helper method to create a copy of a column that is marked as an internal column. */ public static StructField asInternalColumn(StructField field) { FieldMetadata metadata = FieldMetadata.builder() .fromMetadata(field.getMetadata()) .putBoolean(StructField.IS_INTERNAL_COLUMN_KEY, true) .build(); return field.withNewMetadata(metadata); } ///////////////////////////////////////////////////////////////////////////////////////////////// /// Private methods /// ///////////////////////////////////////////////////////////////////////////////////////////////// /** * Compute the SchemaChanges using field IDs * * @throws KernelException if any existing fields have been illegally moved outside their parent * struct */ static SchemaChanges computeSchemaChangesById(StructType currentSchema, StructType newSchema) { SchemaChanges.Builder schemaDiff = SchemaChanges.builder(); findAndAddRemovedFields(currentSchema, newSchema, schemaDiff); // Given a schema like struct> // This map would contain: // {<"", 1> : StructField("a", MapType), // <"key", 1> : StructField("key", IntegerType), // <"value", 1> : StructField("value", StructType), // <"", 2>: StructField("b", IntegerType)} Map currentFieldIdToField = fieldsByElementId(currentSchema); // This map only contains struct field keys (does not include complex type elements). // Using the earlier example, this map would contain: // {1: Optional.empty(), // 2: Optional.of(StructField("a", MapType), "value"))} Map> currentFieldIdToParent = mapStructFieldsToParent(currentSchema); Set addedFieldIds = new HashSet<>(); SchemaIterable newSchemaIterable = new SchemaIterable(newSchema); Iterator newSchemaIterator = newSchemaIterable.newMutableIterator(); boolean newSchemaHasUpdates = false; while (newSchemaIterator.hasNext()) { SchemaIterable.MutableSchemaElement newElement = newSchemaIterator.next(); SchemaElementId id = getSchemaElementId(newElement); // If the element is a struct field we need to validate that it has not been moved out of // its parent struct. To do this, we check that in the old schema and the new schema // its parent struct field (and the path to it) is unchanged. if (newElement.isStructField()) { int columnId = getColumnId(newElement.getField()); if (currentFieldIdToParent.containsKey(columnId)) { // If it's an existing field // We need both the parent struct field and the path to it in case of nested arrays/maps Optional currentParent = currentFieldIdToParent.get(columnId); Optional newParent = newElement.getParentStructFieldAndPath(); Optional currentParentId = currentParent.map(SchemaUtils::getSchemaElementId); Optional newParentId = newParent.map(SchemaUtils::getSchemaElementId); if (!Objects.equals(currentParentId, newParentId)) { throw DeltaErrors.invalidFieldMove(columnId, currentParent, newParent); } } } if (addedFieldIds.contains(id.getId())) { // Skip early if this is a descendant of an added field. continue; } StructField existingField = currentFieldIdToField.get(id); if (existingField == null) { // If the field wasn't present in the schema before then it represents either // a schema change or a newly added field. To check if it is a new struct field, // we check the old schema for just the field ID (i.e. no nested path) to make // a determination between these two cases. // If new StructField where added to the schema example above (e.g // >> ) // This would eventually probe the currentFieldIdToField map for <"", 3> and // find it is an addition. SchemaElementId rootId = new SchemaElementId(/* nestedPath= */ "", id.getId()); if (!currentFieldIdToField.containsKey(rootId)) { addedFieldIds.add(id.getId()); schemaDiff.withAddedField(newElement.getNearestStructFieldAncestor()); } // Getting here implies a structural change in nested maps/arrays which will be caught at // when comparing the higher level element (or we added a field in the if statement above). // This can be somewhat subtle, if this is a type change. // Consider the case of the changing a field from Map to an Array. This would imply that // path would be "element" here and either "key" or "value" in the previous schema. Nothing // is done for the new "element" path if it isn't a field addition there must be at least // one ancestor node in common (at least the nearest struct field) which would get added as // an update below. This logic is inductive. If the previous schema was a array> // and was not a new addition and the new schema was array>> then this path // would be reached on element.element.element but the type the code would move past this // block for element.element which would have a type change detected from x to array. // concretely if the new schema was >> then // <"element, "1"> would be skipped here but the type change would be detected for <"", 1> // from map to array. continue; } StructField newField = newElement.getField(); List existingTypeChanges = existingField.getTypeChanges(); if (!existingTypeChanges.isEmpty() && newField.getTypeChanges().isEmpty()) { // Copy over type changes from existing field because new schemas // might be constructed elsewhere and not have the persisted type // change metadata. newField = newField.withTypeChanges(existingTypeChanges); newElement.updateField(newField); newSchemaHasUpdates = true; } // Ensure the schemas are equal now, updating type changes from clients is not supported // so they should be. if (!existingTypeChanges.equals(newField.getTypeChanges())) { throw new KernelException( String.format( "Detected a modified type changes field at '%s' %s != %s", newElement.getNamePath(), existingTypeChanges, newField.getTypeChanges())); } if (!existingField.equals(newField)) { if (!existingField.getDataType().isNested() && !newField.getDataType().isNested() && !existingField.getDataType().equivalent(newField.getDataType())) { // Type changes only apply to non-nested types. This loop evaluates both // nested and non-nested, so we narrow down updates here. Actual type differences // between nested types are validated against SchemaChanges returned by this // function. List changes = new ArrayList<>(newField.getTypeChanges()); changes.add(new TypeChange(existingField.getDataType(), newField.getDataType())); newElement.updateField(newField.withTypeChanges(changes)); newSchemaHasUpdates = true; } // Field changed name, nullability, metadata or type schemaDiff.withUpdatedField(existingField, newField, newElement.getNamePath()); } } if (newSchemaHasUpdates) { schemaDiff.withUpdatedSchema(newSchemaIterable.getSchema()); } return schemaDiff.build(); } private static Map fieldsByElementId(StructType schema) { Map fieldIdToField = new HashMap<>(); for (SchemaIterable.SchemaElement element : new SchemaIterable(schema)) { SchemaElementId id = getSchemaElementId(element); checkArgument( !fieldIdToField.containsKey(id), "Field %s with id %d already exists", element.getNamePath(), id); fieldIdToField.put(id, element.getField()); } return fieldIdToField; } private static Map> mapStructFieldsToParent(StructType schema) { Map> fieldIdToParent = new HashMap<>(); for (SchemaIterable.SchemaElement element : new SchemaIterable(schema)) { if (element.isStructField()) { StructField elementField = element.getField(); int columnId = getColumnId(elementField); Optional parentInfo = element.getParentStructFieldAndPath(); fieldIdToParent.put(columnId, parentInfo); } } return fieldIdToParent; } private static SchemaElementId getSchemaElementId(SchemaIterable.SchemaElement element) { int columnId = getColumnId(element.getNearestStructFieldAncestor()); return new SchemaElementId(element.getPathFromNearestStructFieldAncestor(""), columnId); } private static SchemaElementId getSchemaElementId( SchemaIterable.ParentStructFieldInfo parentInfo) { int columnId = getColumnId(parentInfo.getParentField()); return new SchemaElementId(parentInfo.getPathFromParent(), columnId); } private static void findAndAddRemovedFields( StructType currentSchema, StructType newSchema, SchemaChanges.Builder schemaDiff) { // With schema: >> // contains {1: StructField("a", MapType), 3: StructField("c", IntegerType)} Map fieldIdToField = fieldsById(newSchema); for (SchemaIterable.SchemaElement element : new SchemaIterable(currentSchema)) { // Removed fields are always calculated at the Struct level. // From the example above, we only "c" or "a" are removed, as all other StructFields // returned by the iterator (e.g. a.key, a.value cannot be removed without a type // change). if (!element.isStructField()) { continue; } StructField field = element.getField(); int columnId = getCheckedColumnId(field); if (!fieldIdToField.containsKey(columnId)) { schemaDiff.withRemovedField(element.getField()); } } } private static void validatePhysicalNameConsistency( List updatedFields) { for (SchemaChanges.SchemaUpdate updatedField : updatedFields) { StructField currentField = updatedField.before(); StructField newField = updatedField.after(); if (!getPhysicalName(currentField).equals(getPhysicalName(newField))) { throw new IllegalArgumentException( String.format( "Existing field with id %s in current schema has " + "physical name %s which is different from %s", getColumnId(currentField), getPhysicalName(currentField), getPhysicalName(newField))); } } } /* * Validate if a given schema evolution is safe for a given column mapping mode * *

Returns an updated schema if metadata (i.e. TypeChanges needs to be copied * over from currentSchema). */ private static Optional validateSchemaEvolution( StructType currentSchema, StructType newSchema, ColumnMappingMode columnMappingMode, Set clusteringColumnPhysicalNames, int currentMaxFieldId, boolean allowNewRequiredFields, boolean icebergWriterCompatV1Enabled, boolean typeWideningEnabled) { switch (columnMappingMode) { case ID: case NAME: return validateSchemaEvolutionById( currentSchema, newSchema, clusteringColumnPhysicalNames, currentMaxFieldId, allowNewRequiredFields, icebergWriterCompatV1Enabled, typeWideningEnabled); case NONE: throw new UnsupportedOperationException( "Schema evolution without column mapping is not supported"); default: throw new UnsupportedOperationException( "Unknown column mapping mode: " + columnMappingMode); } } private static void validateClusteringColumnsNotDropped( List droppedFields, Set clusteringColumnPhysicalNames) { for (StructField droppedField : droppedFields) { // ToDo: At some point plumb through mapping of ID to full name, so we get better error // messages if (clusteringColumnPhysicalNames.contains(getPhysicalName(droppedField))) { throw new KernelException( String.format("Cannot drop clustering column %s", droppedField.getName())); } } } /** * Verifies that no non-nullable fields are added, no existing field nullability is tightened and * no invalid type changes are performed * *

ToDo: Prevent moving fields outside of their containing struct */ private static void validateUpdatedSchemaCompatibility( SchemaChanges schemaChanges, int oldMaxFieldId, boolean allowNewRequiredFields, boolean icebergWriterCompatV1Enabled, boolean typeWideningEnabled) { for (StructField addedField : schemaChanges.addedFields()) { if (!allowNewRequiredFields && !addedField.isNullable()) { throw new KernelException( String.format("Cannot add non-nullable field %s", addedField.getName())); } int colId = getColumnId(addedField); if (colId <= oldMaxFieldId) { throw new IllegalArgumentException( String.format( "Cannot add a new column with a fieldId <= maxFieldId. Found field: %s with" + "fieldId=%s. Current maxFieldId in the table is: %s", addedField, colId, oldMaxFieldId)); } } for (SchemaChanges.SchemaUpdate updatedFields : schemaChanges.updatedFields()) { validateFieldCompatibility(updatedFields, icebergWriterCompatV1Enabled, typeWideningEnabled); } } /** * Validate that there was no change in type from existing field from new field, excluding * modified, dropped, or added fields to structs. Validates that a field's nullability is not * tightened */ private static void validateFieldCompatibility( SchemaChanges.SchemaUpdate update, boolean icebergWriterCompatV1Enabled, boolean typeWideningEnabled) { StructField existingField = update.before(); StructField newField = update.after(); if (existingField.isNullable() && !newField.isNullable()) { throw new KernelException( String.format( "Cannot tighten the nullability of existing field %s", update.getPathToAfterField())); } if (existingField.getDataType() instanceof MapType && newField.getDataType() instanceof MapType) { MapType existingMapType = (MapType) existingField.getDataType(); MapType newMapType = (MapType) newField.getDataType(); if (icebergWriterCompatV1Enabled && existingMapType.getKeyType() instanceof StructType && newMapType.getKeyType() instanceof StructType) { // Enforce that we don't change map struct keys. This is a requirement for // IcebergWriterCompatV1 StructType currentKeyType = (StructType) existingMapType.getKeyType(); StructType newKeyType = (StructType) newMapType.getKeyType(); if (!currentKeyType.equals(newKeyType)) { throw new KernelException( String.format( "Cannot change the type key of Map field %s from %s to %s", newField.getName(), currentKeyType, newKeyType)); } } } else { // Note because computeSchemaChangesById() adds all changed struct fields and any nested // elements that have the same path but different types then the following scenarios are // handled in this block. // 1. This is non-leaf node (e.g. StructType, ArrayType) type, in which case it is sufficient // to ensure that the types are of the same class. For a struct type there might be field // additions or removals, which shouldn't be considered invalid schema transitions // (and hence `equivalent(...)` is not used in this case). If the nested type changes // (e.g. ArrayType to Primitive) then the classes would be different and the types by // definition would not be equivalent. // // 2. This is a leaf node, in which case it sufficient to check that the types are equivalent // (or once implemented the transition of types is valid). // // The subtle point here is for any non-leaf node change, the computed changes will include at // least one ancestor change where the type change can be detected. // for (Class clazz : new Class[] {StructType.class, ArrayType.class}) { if (existingField.getDataType().getClass() == clazz && newField.getDataType().getClass() == clazz) { return; } } DataType existingType = existingField.getDataType(); DataType newType = newField.getDataType(); if (!existingType.equivalent(newType)) { if (typeWideningEnabled) { if ((icebergWriterCompatV1Enabled && TypeWideningChecker.isIcebergV2Compatible(existingType, newType)) || (!icebergWriterCompatV1Enabled && TypeWideningChecker.isWideningSupported(existingType, newType))) { return; } } throw new KernelException( String.format( "Cannot change the type of existing field %s from %s to %s", existingField.getName(), existingField.getDataType(), newField.getDataType())); } } } /** * A composite class that uniquely identifiers an element in a schema. * *

When column mapping is enabled, the every field in structs has a unique ID. For other nested * types (maps and arrays), elements are identified from there path from the struct field. */ private static class SchemaElementId { private final String nestedPath; private final int id; SchemaElementId(String nestedPath, int id) { this.nestedPath = nestedPath; this.id = id; } public int getId() { return id; } @Override public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; SchemaElementId that = (SchemaElementId) o; return id == that.id && Objects.equals(nestedPath, that.nestedPath); } @Override public int hashCode() { return Objects.hash(nestedPath, id); } @Override public String toString() { return "SchemaElementId{" + "getPath='" + nestedPath + '\'' + ", id=" + id + '}'; } } private static Map fieldsById(StructType schema) { Map columnIdToField = new HashMap<>(); for (SchemaIterable.SchemaElement element : new SchemaIterable(schema)) { if (!element.isStructField()) { continue; } StructField field = element.getField(); int columnId = getCheckedColumnId(field); checkArgument( !columnIdToField.containsKey(columnId), "Field %s with id %d already exists", field.getName(), columnId); columnIdToField.put(columnId, field); } return columnIdToField; } private static int getCheckedColumnId(StructField field) { checkArgument(hasColumnId(field), "Field %s is missing column id", field.getName()); checkArgument(hasPhysicalName(field), "Field %s is missing physical name", field.getName()); return ColumnMapping.getColumnId(field); } private static String escapeDots(String name) { return name.contains(".") ? "`" + name + "`" : name; } private static void collectLeafColumnsInternal( StructType schema, Column parentColumn, Set excludedColumns, List result, int maxColumns) { boolean hasLimit = maxColumns != -1; for (StructField field : schema.fields()) { if (hasLimit && result.size() >= maxColumns) { return; } Column currentColumn = null; if (parentColumn == null) { // Skip excluded top-level columns if (excludedColumns.contains(field.getName())) { continue; } currentColumn = new Column(field.getName()); } else { currentColumn = parentColumn.appendNestedField(field.getName()); } if (field.getDataType() instanceof StructType) { collectLeafColumnsInternal( (StructType) field.getDataType(), currentColumn, excludedColumns, result, maxColumns); } else { result.add(currentColumn); } } } protected static void validParquetColumnNames(List columnNames) { for (String name : columnNames) { // ,;{}()\n\t= and space are special characters in Parquet schema if (name.matches(".*[ ,;{}()\n\t=].*")) { throw invalidColumnName(name, "[ ,;{}()\\n\\t=]"); } } } /** * Validate the supported data types. Once we start supporting additional types, take input the * protocol features and validate the schema. * * @param dataType the data type to validate */ protected static void validateSupportedType(DataType dataType) { if (dataType instanceof BooleanType || dataType instanceof ByteType || dataType instanceof ShortType || dataType instanceof IntegerType || dataType instanceof LongType || dataType instanceof FloatType || dataType instanceof DoubleType || dataType instanceof DecimalType || dataType instanceof StringType || dataType instanceof BinaryType || dataType instanceof DateType || dataType instanceof TimestampType || dataType instanceof TimestampNTZType || dataType instanceof VariantType) { // supported types return; } else if (dataType instanceof StructType) { ((StructType) dataType).fields().forEach(field -> validateSupportedType(field.getDataType())); } else if (dataType instanceof ArrayType) { validateSupportedType(((ArrayType) dataType).getElementType()); } else if (dataType instanceof MapType) { validateSupportedType(((MapType) dataType).getKeyType()); validateSupportedType(((MapType) dataType).getValueType()); } else { throw unsupportedDataType(dataType); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/TimestampUtils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; import java.time.Instant; import java.time.LocalDateTime; import java.time.OffsetDateTime; /** * Utilities for timestamp conversions that avoid overflow issues. * *

Note: {@code ChronoUnit.MICROS.between()} internally computes {@code (seconds * 1_000_000_000) * / 1000}, where the intermediate nanoseconds value overflows for timestamps beyond ~292 years from * epoch. These methods compute {@code seconds * 1_000_000} directly to avoid overflow. */ public final class TimestampUtils { private TimestampUtils() {} /** Converts an Instant to microseconds since epoch. */ public static long toEpochMicros(Instant instant) { long microsFromSeconds = Math.multiplyExact(instant.getEpochSecond(), 1_000_000L); long microsFromNanos = instant.getNano() / 1000; return Math.addExact(microsFromSeconds, microsFromNanos); } /** Converts an OffsetDateTime to microseconds since epoch. */ public static long toEpochMicros(OffsetDateTime dateTime) { return toEpochMicros(dateTime.toInstant()); } /** Converts a LocalDateTime (interpreted as UTC) to microseconds since epoch. */ public static long toEpochMicros(LocalDateTime dateTime) { return toEpochMicros(dateTime.toInstant(java.time.ZoneOffset.UTC)); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/Tuple2.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; import io.delta.kernel.annotation.Evolving; import java.util.Objects; /** * Represents tuple of objects. * * @param Type of the first element in the tuple * @param Type of the second element in the tuple * @since 3.0.0 */ @Evolving public class Tuple2 { public final K _1; public final V _2; public Tuple2(K _1, V _2) { this._1 = _1; this._2 = _2; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } Tuple2 tuple2 = (Tuple2) o; return Objects.equals(_1, tuple2._1) && Objects.equals(_2, tuple2._2); } @Override public int hashCode() { return Objects.hash(_1, _2); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/Utils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; import static io.delta.kernel.internal.DeltaErrors.wrapEngineExceptionThrowsIO; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; import io.delta.kernel.utils.CloseableIterator; import java.io.IOException; import java.io.UncheckedIOException; import java.util.Iterator; import java.util.NoSuchElementException; import java.util.Optional; /** * Various utility methods to help the connectors work with data objects returned by Kernel * * @since 3.0.0 */ @Evolving public class Utils { /** * Utility method to create a singleton {@link CloseableIterator}. * * @param elem Element to create iterator with. * @param Element type. * @return A {@link CloseableIterator} with just one element. */ public static CloseableIterator singletonCloseableIterator(T elem) { return new CloseableIterator() { private boolean accessed; @Override public void close() throws IOException { // nothing to close } @Override public boolean hasNext() { return !accessed; } @Override public T next() { accessed = true; return elem; } }; } /** * Convert a {@link Iterator} to {@link CloseableIterator}. Useful when passing normal iterators * for arguments that require {@link CloseableIterator} type. * * @param iter {@link Iterator} instance * @param Element type * @return A {@link CloseableIterator} wrapping the given {@link Iterator} */ public static CloseableIterator toCloseableIterator(Iterator iter) { return new CloseableIterator() { @Override public void close() {} @Override public boolean hasNext() { return iter.hasNext(); } @Override public T next() { return iter.next(); } }; } /** * Close the given one or more {@link AutoCloseable}s. {@link AutoCloseable#close()} will be * called on all given non-null closeables. Will throw unchecked {@link RuntimeException} if an * error occurs while closing. If multiple closeables causes exceptions in closing, the exceptions * will be added as suppressed to the main exception that is thrown. * * @param closeables */ public static void closeCloseables(AutoCloseable... closeables) { RuntimeException exception = null; for (AutoCloseable closeable : closeables) { if (closeable == null) { continue; } try { closeable.close(); } catch (Exception ex) { if (exception == null) { exception = new RuntimeException(ex); } else { exception.addSuppressed(ex); } } } if (exception != null) { throw exception; } } /** * Close the given list of {@link AutoCloseable} objects. Any exception thrown is silently * ignored. * * @param closeables */ public static void closeCloseablesSilently(AutoCloseable... closeables) { try { closeCloseables(closeables); } catch (Throwable throwable) { // ignore } } // Utility class to support `intoRows` below private static class FilteredBatchToRowIter implements CloseableIterator { private final CloseableIterator sourceBatches; private CloseableIterator current; private boolean isClosed = false; FilteredBatchToRowIter(CloseableIterator sourceBatches) { this.sourceBatches = sourceBatches; } @Override public boolean hasNext() { if (isClosed) { return false; } while ((current == null || !current.hasNext()) && sourceBatches.hasNext()) { closeCloseables(current); FilteredColumnarBatch next = sourceBatches.next(); current = next.getRows(); } return current != null && current.hasNext(); } @Override public Row next() { if (!hasNext()) { throw new java.util.NoSuchElementException("No more rows available"); } return current.next(); } @Override public void close() throws IOException { isClosed = true; closeCloseables(current, sourceBatches); } } /** Convert a ClosableIterator of FilteredColumnarBatch into a CloseableIterator of Row */ public static CloseableIterator intoRows( CloseableIterator sourceBatches) { return new FilteredBatchToRowIter(sourceBatches); } /** * Flattens a nested {@link CloseableIterator} structure into a single flat iterator. This method * takes an iterator of iterators (nested structure) and flattens it into a single iterator that * yields all elements from all inner iterators in sequence. * *

Important: Callers must call {@link CloseableIterator#close()} on the returned * iterator even if it is not fully consumed, to ensure all inner iterators are properly closed * and resources are released. * * @param nestedIterator An iterator of iterators to flatten * @param The type of elements in the inner iterators * @return A new {@link CloseableIterator} that flattens all nested iterators */ public static CloseableIterator flatten( CloseableIterator> nestedIterator) { return new CloseableIterator<>() { private CloseableIterator currentInnerIterator = null; @Override public boolean hasNext() { while (true) { if (currentInnerIterator != null && currentInnerIterator.hasNext()) { return true; } if (currentInnerIterator != null) { closeCloseables(currentInnerIterator); currentInnerIterator = null; } if (!nestedIterator.hasNext()) { return false; } try { currentInnerIterator = nestedIterator.next(); } catch (Exception e) { // Ensure cleanup on exception closeCloseables(nestedIterator); throw e; } } } @Override public T next() { if (!hasNext()) { throw new NoSuchElementException(); } return currentInnerIterator.next(); } @Override public void close() { // Close both the current inner iterator and the outer iterator // closeCloseables works with null closeable. closeCloseables(currentInnerIterator, nestedIterator); } }; } /** * Returns the last element from the given {@link CloseableIterator}, if present. * *

This method iterates through all elements of the iterator to find the last one. Once * iteration is complete, the iterator is automatically closed to release any underlying * resources. * * @param iterator The iterator to get the last element from * @param The type of elements in the iterator * @return An {@link Optional} containing the last element, or {@link Optional#empty()} if the * iterator is empty. * @throws UncheckedIOException If an {@link IOException} occurs while closing the iterator. */ public static Optional iteratorLast(CloseableIterator iterator) { try (CloseableIterator iter = iterator) { T last = null; while (iter.hasNext()) { last = iter.next(); } return Optional.ofNullable(last); } catch (IOException e) { throw new UncheckedIOException("Failed to close the CloseableIterator", e); } } public static String resolvePath(Engine engine, String path) { try { return wrapEngineExceptionThrowsIO( () -> engine.getFileSystemClient().resolvePath(path), "Resolving path %s", path); } catch (IOException io) { throw new UncheckedIOException(io); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/VectorUtils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util; import io.delta.kernel.data.ArrayValue; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.MapValue; import io.delta.kernel.data.Row; import io.delta.kernel.internal.data.GenericColumnVector; import io.delta.kernel.internal.data.StructRow; import io.delta.kernel.types.*; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; public final class VectorUtils { private VectorUtils() {} /** * Converts an {@link ArrayValue} to a Java list. Any nested complex types are also converted to * their Java type. */ public static List toJavaList(ArrayValue arrayValue) { final ColumnVector elementVector = arrayValue.getElements(); final DataType dataType = elementVector.getDataType(); List elements = new ArrayList<>(); for (int i = 0; i < arrayValue.getSize(); i++) { elements.add((T) getValueAsObject(elementVector, dataType, i)); } return elements; } /** * Converts a {@link MapValue} to a Java map. Any nested complex types are also converted to their * Java type. * *

Please note not all key types override hashCode/equals. Be careful when using with keys of: * - Struct type at any nesting level (i.e. ArrayType(StructType) does not) - Binary type */ public static Map toJavaMap(MapValue mapValue) { final ColumnVector keyVector = mapValue.getKeys(); final DataType keyDataType = keyVector.getDataType(); final ColumnVector valueVector = mapValue.getValues(); final DataType valueDataType = valueVector.getDataType(); Map values = new HashMap<>(); for (int i = 0; i < mapValue.getSize(); i++) { Object key = getValueAsObject(keyVector, keyDataType, i); Object value = getValueAsObject(valueVector, valueDataType, i); values.put((K) key, (V) value); } return values; } /** * Creates a {@link MapValue} from map of string keys and string values. The type {@code * map(string -> string)} is a common occurrence in Delta Log schema. * * @param keyValues * @return */ public static MapValue stringStringMapValue(Map keyValues) { List keys = new ArrayList<>(); List values = new ArrayList<>(); for (Map.Entry entry : keyValues.entrySet()) { keys.add(entry.getKey()); values.add(entry.getValue()); } return new MapValue() { @Override public int getSize() { return values.size(); } @Override public ColumnVector getKeys() { return buildColumnVector(keys, StringType.STRING); } @Override public ColumnVector getValues() { return buildColumnVector(values, StringType.STRING); } }; } /** Creates an {@link ArrayValue} from list of objects. */ public static ArrayValue buildArrayValue(List values, DataType dataType) { if (values == null) { return null; } return new ArrayValue() { @Override public int getSize() { return values.size(); } @Override public ColumnVector getElements() { return buildColumnVector(values, dataType); } }; } /** * Utility method to create a {@link ColumnVector} for given list of object, the object should be * primitive type or an Row instance. * * @param values list of strings * @return a {@link ColumnVector} with the given values type. */ public static ColumnVector buildColumnVector(List values, DataType dataType) { return new GenericColumnVector(values, dataType); } /** * Gets the value at {@code rowId} from the column vector. The type of the Object returned depends * on the data type of the column vector. For complex types array and map, returns the value as * Java list or Java map. For struct type, returns an {@link Row}. */ public static Object getValueAsObject(ColumnVector columnVector, DataType dataType, int rowId) { if (columnVector.isNullAt(rowId)) { return null; } else if (dataType instanceof BooleanType) { return columnVector.getBoolean(rowId); } else if (dataType instanceof ByteType) { return columnVector.getByte(rowId); } else if (dataType instanceof ShortType) { return columnVector.getShort(rowId); } else if (dataType instanceof IntegerType || dataType instanceof DateType) { // DateType data is stored internally as the number of days since 1970-01-01 return columnVector.getInt(rowId); } else if (dataType instanceof LongType || dataType instanceof TimestampType) { // TimestampType data is stored internally as the number of microseconds since the unix // epoch return columnVector.getLong(rowId); } else if (dataType instanceof FloatType) { return columnVector.getFloat(rowId); } else if (dataType instanceof DoubleType) { return columnVector.getDouble(rowId); } else if (dataType instanceof StringType) { return columnVector.getString(rowId); } else if (dataType instanceof BinaryType) { return columnVector.getBinary(rowId); } else if (dataType instanceof StructType) { // TODO are we okay with this usage of StructRow? return StructRow.fromStructVector(columnVector, rowId); } else if (dataType instanceof DecimalType) { return columnVector.getDecimal(rowId); } else if (dataType instanceof ArrayType) { return toJavaList(columnVector.getArray(rowId)); } else if (dataType instanceof MapType) { return toJavaMap(columnVector.getMap(rowId)); } else { throw new UnsupportedOperationException("unsupported data type"); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/metrics/DeltaOperationReport.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.metrics; import java.util.Optional; import java.util.UUID; /** Defines the common fields that are shared by reports for Delta operations */ public interface DeltaOperationReport extends MetricsReport { /** @return the path of the table */ String getTablePath(); /** @return a string representation of the operation this report is for */ String getOperationType(); /** @return a unique ID for this report */ UUID getReportUUID(); /** @return the exception thrown if this report is for a failed operation, otherwise empty */ Optional getException(); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/metrics/FileSizeHistogramResult.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.metrics; /** Stores the file size histogram information to track file size distribution and their counts. */ public interface FileSizeHistogramResult { /** * Sorted list of bin boundaries where each element represents the start of the bin (inclusive) * and the next element represents the end of the bin (exclusive). */ long[] getSortedBinBoundaries(); /** * The total number of files in each bin of {@link * FileSizeHistogramResult#getSortedBinBoundaries()} */ long[] getFileCounts(); /** * The total number of bytes in each bin of {@link * FileSizeHistogramResult#getSortedBinBoundaries()} */ long[] getTotalBytes(); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/metrics/MetricsReport.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.metrics; import com.fasterxml.jackson.core.JsonProcessingException; /** * Interface containing the metrics for a given operation. * *

Implementations of this interface capture performance metrics and diagnostic information for * various Delta table operations. These metrics can be used for monitoring, debugging, and * performance analysis. */ public interface MetricsReport { /** * Converts this metrics report to a JSON string representation. * * @return a JSON string representation of this metrics report * @throws JsonProcessingException if the report cannot be serialized to JSON */ String toJson() throws JsonProcessingException; } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/metrics/ScanMetricsResult.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.metrics; import com.fasterxml.jackson.annotation.JsonPropertyOrder; /** Stores the metrics results for a {@link ScanReport} */ @JsonPropertyOrder({ "totalPlanningDurationNs", "numAddFilesSeen", "numAddFilesSeenFromDeltaFiles", "numActiveAddFiles", "numDuplicateAddFiles", "numRemoveFilesSeenFromDeltaFiles" }) public interface ScanMetricsResult { /** * Returns the total duration to find, filter, and consume the scan files. This begins at the * request for the scan files and terminates once all the scan files have been consumed and the * scan file iterator closed. It includes reading the _delta_log, log replay, filtering * optimizations, and any work from the connector before closing the scan file iterator. * * @return the total duration to find, filter, and consume the scan files */ long getTotalPlanningDurationNs(); /** * @return the number of AddFile actions seen during log replay (from both checkpoint and delta * files). For a failed or incomplete scan this metric may be incomplete. */ long getNumAddFilesSeen(); /** * @return the number of AddFile actions seen during log replay from delta files only. For a * failed or incomplete scan this metric may be incomplete. */ long getNumAddFilesSeenFromDeltaFiles(); /** * @return the number of active AddFile actions that survived log replay (i.e. belong to the table * state). For a failed or incomplete scan this metric may be incomplete. */ long getNumActiveAddFiles(); /** * Returns the number of duplicate AddFile actions seen during log replay. The same AddFile (same * path and DV) can be present in multiple commit files when stats collection is run on the table. * In this case, the same AddFile will be added with stats without removing the original. * * @return the number of AddFile actions seen during log replay that are duplicates. For a failed * or incomplete scan this metric may be incomplete. */ long getNumDuplicateAddFiles(); /** * @return the number of RemoveFiles seen in log replay (only from delta files). For a failed or * incomplete scan this metric may be incomplete. */ long getNumRemoveFilesSeenFromDeltaFiles(); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/metrics/ScanReport.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.metrics; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.types.StructType; import java.util.Optional; import java.util.UUID; /** Defines the metadata and metrics for a Scan {@link MetricsReport} */ @JsonSerialize(as = ScanReport.class) @JsonPropertyOrder({ "tablePath", "operationType", "reportUUID", "exception", "tableVersion", "tableSchema", "snapshotReportUUID", "filter", "readSchema", "partitionPredicate", "dataSkippingFilter", "isFullyConsumed", "scanMetrics" }) public interface ScanReport extends DeltaOperationReport { /** @return the version of the table in this scan */ long getTableVersion(); /** @return the schema of the table for this scan */ StructType getTableSchema(); /** * @return the {@link SnapshotReport#getReportUUID} for the snapshot this scan was created from */ UUID getSnapshotReportUUID(); /** @return the filter provided when building the scan */ Optional getFilter(); /** @return the read schema provided when building the scan */ StructType getReadSchema(); /** @return the part of {@link ScanReport#getFilter()} that was used for partition pruning */ Optional getPartitionPredicate(); /** @return the filter used for data skipping using the file statistics */ Optional getDataSkippingFilter(); /** * Whether the scan file iterator had been fully consumed when it was closed. The iterator may be * closed early (before being fully consumed) either due to an exception originating within * connector code or intentionally (such as for a LIMIT query). * * @return whether the scan file iterator had been fully consumed when it was closed */ boolean getIsFullyConsumed(); /** @return the metrics for this scan */ ScanMetricsResult getScanMetrics(); @Override default String getOperationType() { return "Scan"; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/metrics/SnapshotMetricsResult.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.metrics; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import java.util.Optional; /** Stores the metrics results for a {@link SnapshotReport} */ @JsonPropertyOrder({ "computeTimestampToVersionTotalDurationNs", "loadSnapshotTotalDurationNs", "loadProtocolMetadataTotalDurationNs", "loadLogSegmentTotalDurationNs", "loadCrcTotalDurationNs" }) public interface SnapshotMetricsResult { /** * @return the duration (ns) to resolve the provided timestamp to a table version for timestamp * time-travel queries. Empty for time-travel by version or non-time-travel queries. */ Optional getComputeTimestampToVersionTotalDurationNs(); /** * @return the total duration (ns) to load the snapshot, including all steps such as resolving * timestamp to version, LISTing the _delta_log, building the log segment, and determining the * latest protocol and metadata. */ long getLoadSnapshotTotalDurationNs(); /** * @return the duration (ns) to load the initial delta actions for the snapshot (such as the table * protocol and metadata). 0 if snapshot construction fails before log replay. */ long getLoadProtocolMetadataTotalDurationNs(); /** * @return the duration (ns) to build the log segment for the specified version during snapshot * construction. 0 if snapshot construction fails before this step. */ long getLoadLogSegmentTotalDurationNs(); /** * @return the duration (ns) to get CRC information during snapshot construction. 0 if snapshot * construction fails before this step or if CRC is not read in loading snapshot. */ long getLoadCrcTotalDurationNs(); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/metrics/SnapshotReport.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.metrics; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.util.Optional; /** Defines the metadata and metrics for a snapshot construction {@link MetricsReport} */ @JsonSerialize(as = SnapshotReport.class) @JsonPropertyOrder({ "tablePath", "operationType", "reportUUID", "exception", "version", "checkpointVersion", "providedTimestamp", "snapshotMetrics" }) public interface SnapshotReport extends DeltaOperationReport { /** * For a time-travel by version query, this is the version provided. For a time-travel by * timestamp query, this is the version resolved from the provided timestamp. For a latest * snapshot, this is the version read from the delta log. * *

This is empty when this report is for a failed snapshot construction, and the error occurs * before a version can be resolved. * * @return the version of the snapshot */ Optional getVersion(); /** * @return the timestamp provided for time-travel, empty if this is not a timestamp-based * time-travel query */ Optional getProvidedTimestamp(); /** * @return the version of the checkpoint used for this snapshot, empty if no checkpoint was used * or if this is a failed snapshot construction */ Optional getCheckpointVersion(); /** @return the metrics for this snapshot construction */ SnapshotMetricsResult getSnapshotMetrics(); @Override default String getOperationType() { return "Snapshot"; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/metrics/TransactionMetricsResult.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.metrics; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import java.util.Optional; /** Stores the metrics results for a {@link TransactionReport} */ @JsonPropertyOrder({ "totalCommitDurationNs", "numCommitAttempts", "numAddFiles", "numRemoveFiles", "numTotalActions", "totalAddFilesSizeInBytes", "totalRemoveFilesSizeInBytes" }) public interface TransactionMetricsResult { /** @return the total duration (ns) this transaction spent committing or trying to commit */ long getTotalCommitDurationNs(); /** @return the total number of commit attempts this transaction made */ long getNumCommitAttempts(); /** * @return the number of add files committed in this transaction. For a failed transaction this * metric may be incomplete. */ long getNumAddFiles(); /** * @return the number of remove files committed in this transaction. For a failed transaction this * metric may be incomplete. */ long getNumRemoveFiles(); /** * @return the total number of delta actions committed in this transaction. For a failed * transaction this metric may be incomplete. */ long getNumTotalActions(); /** * @return the sum of size of added files committed in this transaction. For a failed transaction * this metric may be incomplete. */ long getTotalAddFilesSizeInBytes(); /** * @return the sum of size of removed files committed in this transaction. For a failed * transaction this metric may be incomplete. */ long getTotalRemoveFilesSizeInBytes(); /** * @return the file size histogram information for the table version committed in this * transaction. For a failed transaction this metric may be incomplete. */ @JsonIgnore Optional getTableFileSizeHistogram(); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/metrics/TransactionReport.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.metrics; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.delta.kernel.expressions.Column; import java.util.List; import java.util.Optional; import java.util.UUID; /** Defines the metadata and metrics for a transaction {@link MetricsReport} */ @JsonSerialize(as = TransactionReport.class) @JsonPropertyOrder({ "tablePath", "operationType", "reportUUID", "exception", "operation", "engineInfo", "baseSnapshotVersion", "snapshotReportUUID", "committedVersion", "clusteringColumns", "transactionMetrics" }) public interface TransactionReport extends DeltaOperationReport { /** * @return The {@link io.delta.kernel.Operation} provided when the transaction was created using * {@link io.delta.kernel.Table#createTransactionBuilder}. */ String getOperation(); /** * @return The engineInfo provided when the transaction was created using {@link * io.delta.kernel.Table#createTransactionBuilder}. */ String getEngineInfo(); /** * The version of the table the transaction was created from. For example, if the latest table * version is 4 when the transaction is created, the transaction is based off of the snapshot of * the table at version 4. For a new table (e.g. a transaction that is creating a table) this is * -1. * * @return the table version of the snapshot the transaction was started from */ long getBaseSnapshotVersion(); /** * Get the list of clustering columns the table data is expected to be clustered by. This is * optional because clustering columns are not always defined for a table. Consumers of the * transaction report trigger clustering operations based on this list. * * @return list of clustering columns for the table. The columns are physical names of how the * data is written in the data files. Each column can contain one or more elements * representing the hierarchy of the column names in case of nested columns. */ List getClusteringColumns(); /** * @return the {@link SnapshotReport#getReportUUID} of the SnapshotReport for the transaction's * snapshot construction. Empty for a new table transaction. */ Optional getSnapshotReportUUID(); /** * @return the version committed to the table in this transaction. Empty for a failed transaction. */ Optional getCommittedVersion(); /** @return the metrics for this transaction */ TransactionMetricsResult getTransactionMetrics(); @Override default String getOperationType() { return "Transaction"; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/package-info.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /** * Delta Kernel interfaces for constructing table object representing a Delta Lake table, getting * snapshot from the table and building a scan object to scan a subset of the data in the table. */ package io.delta.kernel; ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/statistics/DataFileStatistics.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.statistics; import static io.delta.kernel.internal.DeltaErrors.unsupportedStatsDataType; import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonNode; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.expressions.Column; import io.delta.kernel.expressions.Literal; import io.delta.kernel.internal.DeltaErrors; import io.delta.kernel.internal.skipping.StatsSchemaHelper; import io.delta.kernel.internal.util.JsonUtils; import io.delta.kernel.types.*; import java.io.IOException; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; import java.time.*; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; import java.util.*; /** * Encapsulates statistics for a data file in a Delta Lake table and provides methods to serialize * those stats to JSON with basic physical-type validation. Note that connectors (e.g. Spark, Flink) * are responsible for ensuring the correctness of collected stats, including any necessary string * truncation, prior to constructing this class. */ public class DataFileStatistics { public static final DateTimeFormatter TIMESTAMP_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSX"); public static final OffsetDateTime EPOCH = Instant.ofEpochSecond(0).atOffset(ZoneOffset.UTC); private final long numRecords; private final Map minValues; private final Map maxValues; private final Map nullCount; private final Optional tightBounds; /** * Create a new instance of {@link DataFileStatistics}. The minValues, maxValues, and nullCount * are required fields. The tightBounds field is optional - pass Optional.empty() if not * specified, Optional.of(true) or Optional.of(false) for explicit values. * * @param numRecords Number of records in the data file. * @param minValues Map of column to minimum value of it in the data file. If the data file has * all nulls for the column, the value will be null or not present in the map. * @param maxValues Map of column to maximum value of it in the data file. If the data file has * all nulls for the column, the value will be null or not present in the map. * @param nullCount Map of column to number of nulls in the data file. * @param tightBounds Optional boolean indicating if bounds are tight (accurate). Pass * Optional.empty() if not specified. */ public DataFileStatistics( long numRecords, Map minValues, Map maxValues, Map nullCount, Optional tightBounds) { Objects.requireNonNull(minValues, "minValues must not be null to serialize stats."); Objects.requireNonNull(maxValues, "maxValues must not be null to serialize stats."); Objects.requireNonNull(nullCount, "nullCount must not be null to serialize stats."); this.numRecords = numRecords; this.minValues = Collections.unmodifiableMap(minValues); this.maxValues = Collections.unmodifiableMap(maxValues); this.nullCount = Collections.unmodifiableMap(nullCount); this.tightBounds = tightBounds; } /** * Utility method to extract only the numRecords field from a statistics JSON string. * * @param json Data statistics JSON string to deserialize. * @return An {@link Optional} containing the numRecords value if present. * @throws KernelException if JSON parsing fails */ public static Optional getNumRecords(String json) { // Delegate to the full deserialization method with null schema to only parse numRecords Optional stats = deserializeFromJson(json, null); return stats.map(DataFileStatistics::getNumRecords); } /** * Utility method to deserialize statistics from a JSON string with full type information. This * overloaded version uses the provided schema to correctly parse min/max values and null counts * with their appropriate data types. * * @param json Data statistics JSON string to deserialize. * @param physicalSchema The physical schema providing type information for columns. Must match * the schema used during serialization. * @return An {@link Optional} containing the deserialized {@link DataFileStatistics} if present. * @throws KernelException if JSON parsing fails or if values don't match expected types */ public static Optional deserializeFromJson( String json, StructType physicalSchema) { JsonNode root; try { root = JsonUtils.mapper().readTree(json); } catch (IOException e) { throw new KernelException(String.format("Failed to parse JSON string: %s", json), e); } // Parse numRecords JsonNode numRecordsNode = root.get(StatsSchemaHelper.NUM_RECORDS); if (numRecordsNode == null || !numRecordsNode.isNumber()) { return Optional.empty(); } long numRecords = numRecordsNode.asLong(); // Check if schema is null or empty if (physicalSchema == null || physicalSchema.fields().isEmpty()) { // Return statistics with only numRecords return Optional.of( new DataFileStatistics( numRecords, new HashMap<>(), new HashMap<>(), new HashMap<>(), Optional.empty())); } // Parse minValues Map minValues = new HashMap<>(); JsonNode minNode = root.get(StatsSchemaHelper.MIN); if (minNode != null && minNode.isObject()) { parseMinMaxValues(minNode, minValues, new Column(new String[0]), physicalSchema); } // Parse maxValues Map maxValues = new HashMap<>(); JsonNode maxNode = root.get(StatsSchemaHelper.MAX); if (maxNode != null && maxNode.isObject()) { parseMinMaxValues(maxNode, maxValues, new Column(new String[0]), physicalSchema); } // Parse nullCount Map nullCount = new HashMap<>(); JsonNode nullCountNode = root.get(StatsSchemaHelper.NULL_COUNT); if (nullCountNode != null && nullCountNode.isObject()) { parseNullCounts(nullCountNode, nullCount, new Column(new String[0]), physicalSchema); } // Parse tightBounds Optional tightBounds = Optional.empty(); JsonNode tightBoundsNode = root.get(StatsSchemaHelper.TIGHT_BOUNDS); if (tightBoundsNode != null && tightBoundsNode.isBoolean()) { tightBounds = Optional.of(tightBoundsNode.asBoolean()); } return Optional.of( new DataFileStatistics(numRecords, minValues, maxValues, nullCount, tightBounds)); } /** * Get the number of records in the data file. * * @return Number of records in the data file. */ public long getNumRecords() { return numRecords; } /** * Get the minimum values of the columns in the data file. The map may contain statistics for only * a subset of columns in the data file. * * @return Map of column to minimum value of it in the data file. */ public Map getMinValues() { return minValues; } /** * Get the maximum values of the columns in the data file. The map may contain statistics for only * a subset of columns in the data file. * * @return Map of column to minimum value of it in the data file. */ public Map getMaxValues() { return maxValues; } /** * Get the number of nulls of columns in the data file. The map may contain statistics for only a * subset of columns in the data file. * * @return Map of column to number of nulls in the data file. */ public Map getNullCount() { return nullCount; } /** * Get the tight bounds information for the data file. Tight bounds indicate whether the values * are guaranteed to be accurate bounds for the data. * * @return The tight bounds boolean value. */ public Optional getTightBounds() { return tightBounds; } /** * Returns a new DataFileStatistics instance with tightBounds set to false. This is useful when * the statistics bounds are no longer guaranteed to be tight, such as after applying deletion * vectors. * * @return A new DataFileStatistics with tightBounds set to false */ public DataFileStatistics withoutTightBounds() { return new DataFileStatistics( this.numRecords, this.minValues, this.maxValues, this.nullCount, Optional.of(false)); } /** * Serializes the statistics as a JSON string. * *

Example: For nested column structures: * * 

   * Input:
   *   minValues = {
   *     new Column(new String[]{"a", "b", "c"}) mapped to Literal.ofInt(10),
   *     new Column("d") mapped to Literal.ofString("value")
   *   }
   *
   * Output JSON:
   *   {
   *     "minValues": {
   *       "a": {
   *         "b": {
   *           "c": 10
   *         }
   *       },
   *       "d": "value"
   *     }
   *   }
   *
* * @param physicalSchema the optional physical schema. If provided, all min/max values and null * counts will be included and validated against their physical types. If null, only * numRecords will be serialized without validation. * @return a JSON representation of the statistics. * @throws KernelException if dataSchema is provided and there's a type mismatch between the * Literal values and the expected types in the schema, or if an unsupported data type is * found. */ public String serializeAsJson(StructType physicalSchema) { return JsonUtils.generate( gen -> { gen.writeStartObject(); gen.writeNumberField(StatsSchemaHelper.NUM_RECORDS, numRecords); if (physicalSchema != null) { gen.writeObjectFieldStart(StatsSchemaHelper.MIN); writeJsonValues( gen, physicalSchema, minValues, new Column(new String[0]), (g, v) -> writeJsonValue(g, v)); gen.writeEndObject(); gen.writeObjectFieldStart(StatsSchemaHelper.MAX); writeJsonValues( gen, physicalSchema, maxValues, new Column(new String[0]), (g, v) -> writeJsonValue(g, v)); gen.writeEndObject(); gen.writeObjectFieldStart(StatsSchemaHelper.NULL_COUNT); writeJsonValues( gen, physicalSchema, nullCount, new Column(new String[0]), (g, v) -> g.writeNumber(v)); gen.writeEndObject(); if (tightBounds.isPresent()) { gen.writeBooleanField(StatsSchemaHelper.TIGHT_BOUNDS, tightBounds.get()); } } gen.writeEndObject(); }); } @Override public boolean equals(Object o) { if (this == o) { return true; } if (!(o instanceof DataFileStatistics)) { return false; } DataFileStatistics that = (DataFileStatistics) o; return numRecords == that.numRecords && Objects.equals(minValues, that.minValues) && Objects.equals(maxValues, that.maxValues) && Objects.equals(nullCount, that.nullCount) && Objects.equals(tightBounds, that.tightBounds); } @Override public int hashCode() { int result = Long.hashCode(numRecords); result = 31 * result + Objects.hash(minValues.keySet()); result = 31 * result + Objects.hash(maxValues.keySet()); result = 31 * result + Objects.hash(nullCount.keySet()); result = 31 * result + Objects.hash(tightBounds); return result; } @Override public String toString() { return String.format( "DataFileStatistics(numRecords=%s, minValues=%s, maxValues=%s," + "nullCount=%s, tightBounds=%s)", numRecords, minValues, maxValues, nullCount, tightBounds.map(Object::toString).orElse("empty")); } ///////////////////////////////////////////////////////////////////////////////// /// Private methods /// ///////////////////////////////////////////////////////////////////////////////// private void writeJsonValues( JsonGenerator generator, StructType schema, Map values, Column parentCol, JsonUtils.JsonValueWriter writer) throws IOException { if (schema == null) { return; } for (StructField field : schema.fields()) { Column colPath = parentCol.appendNestedField(field.getName()); if (field.getDataType() instanceof StructType) { generator.writeObjectFieldStart(field.getName()); writeJsonValues(generator, (StructType) field.getDataType(), values, colPath, writer); generator.writeEndObject(); } else { T value = values.get(colPath); if (value != null) { if (value instanceof Literal) { validateLiteralType(field, (Literal) value); } generator.writeFieldName(field.getName()); writer.write(generator, value); } } } } /** * Validates that the literal's data type matches the expected field type. * * @param field The schema field with the expected data type * @param literal The literal to validate * @throws KernelException if the data types don't match */ private void validateLiteralType(StructField field, Literal literal) { // Variant stats in JSON are Z85 encoded strings, all other stats should match the field type DataType expectedLiteralType = field.getDataType() instanceof VariantType ? StringType.STRING : field.getDataType(); if (literal.getDataType() == null || !expectedLiteralType.isWriteCompatible(literal.getDataType())) { throw DeltaErrors.statsTypeMismatch( field.getName(), expectedLiteralType, literal.getDataType()); } } private void writeJsonValue(JsonGenerator generator, Literal literal) throws IOException { if (literal == null || literal.getValue() == null) { generator.writeNull(); return; } DataType type = literal.getDataType(); Object value = literal.getValue(); if (type instanceof BooleanType) { generator.writeBoolean((Boolean) value); } else if (type instanceof ByteType) { generator.writeNumber(((Number) value).byteValue()); } else if (type instanceof ShortType) { generator.writeNumber(((Number) value).shortValue()); } else if (type instanceof IntegerType) { generator.writeNumber(((Number) value).intValue()); } else if (type instanceof LongType) { generator.writeNumber(((Number) value).longValue()); } else if (type instanceof FloatType) { float f = ((Number) value).floatValue(); if (Float.isNaN(f) || Float.isInfinite(f)) { generator.writeString(String.valueOf(f)); } else { generator.writeNumber(f); } } else if (type instanceof DoubleType) { double d = ((Number) value).doubleValue(); if (Double.isNaN(d) || Double.isInfinite(d)) { generator.writeString(String.valueOf(d)); } else { generator.writeNumber(d); } } else if (type instanceof StringType) { generator.writeString((String) value); } else if (type instanceof BinaryType) { generator.writeString(new String((byte[]) value, StandardCharsets.UTF_8)); } else if (type instanceof DecimalType) { generator.writeNumber((BigDecimal) value); } else if (type instanceof DateType) { generator.writeString( LocalDate.ofEpochDay(((Number) value).longValue()).format(ISO_LOCAL_DATE)); } else if (type instanceof TimestampType) { long epochMicros = (long) value; LocalDateTime localDateTime = ChronoUnit.MICROS.addTo(EPOCH, epochMicros).toLocalDateTime(); LocalDateTime truncated = localDateTime.truncatedTo(ChronoUnit.MILLIS); generator.writeString(TIMESTAMP_FORMATTER.format(truncated.atOffset(ZoneOffset.UTC))); } else if (type instanceof TimestampNTZType) { long epochMicros = (long) value; LocalDateTime localDateTime = ChronoUnit.MICROS.addTo(EPOCH, epochMicros).toLocalDateTime(); LocalDateTime truncated = localDateTime.truncatedTo(ChronoUnit.MILLIS); generator.writeString(truncated.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)); } else { throw unsupportedStatsDataType(type); } } /** * Helper method to recursively parse nested JSON values back into Column->Literal maps for * min/max values using the schema for type information. This is the inverse of the * writeJsonValues method in DataFileStatistics.serializeAsJson. * *

Example JSON structure being parsed: * * 

   * {
   *   "simpleColumn": 42,
   *   "nestedColumn": {
   *     "field1": "value1",
   *     "field2": {
   *       "subfield": 10.5
   *     }
   *   }
   * }
   *
* *

This would create Column entries: * *

    *
  • Column(["simpleColumn"]) → Literal.ofInt(42) *
  • Column(["nestedColumn", "field1"]) → Literal.ofString("value1") *
  • Column(["nestedColumn", "field2", "subfield"]) → Literal.ofDouble(10.5) *
* * @param node The JSON node to parse * @param resultMap The map to populate with Column->Literal mappings * @param currentColumn The current column path being built * @param schema The schema for the current level */ private static void parseMinMaxValues( JsonNode node, Map resultMap, Column currentColumn, StructType schema) { if (node == null || !node.isObject() || schema == null) { return; } for (StructField field : schema.fields()) { String fieldName = field.getName(); JsonNode valueNode = node.get(fieldName); if (valueNode == null) { // Field not present in JSON, skip continue; } Column newColumn = currentColumn.appendNestedField(fieldName); DataType fieldType = field.getDataType(); if (fieldType instanceof StructType) { // This is a nested structure, recurse deeper if (valueNode.isObject()) { parseMinMaxValues(valueNode, resultMap, newColumn, (StructType) fieldType); } } else { // This is a leaf value Literal literal = JsonUtils.parseJsonValueToLiteral(valueNode, fieldType); if (literal != null) { resultMap.put(newColumn, literal); } } } } /** * Helper method to recursively parse nested JSON null count values back into Column->Long maps * using the schema for type information. * *

Example JSON structure being parsed: * * 

   * {
   *   "simpleColumn": 5,
   *   "nestedColumn": {
   *     "field1": 0,
   *     "field2": {
   *       "subfield": 10
   *     }
   *   }
   * }
   *
* *

This would create Column entries: * *

    *
  • Column(["simpleColumn"]) → 5L *
  • Column(["nestedColumn", "field1"]) → 0L *
  • Column(["nestedColumn", "field2", "subfield"]) → 10L *
* * @param node The JSON node to parse * @param resultMap The map to populate with Column->Long mappings * @param currentColumn The current column path being built * @param schema The schema for the current level */ private static void parseNullCounts( JsonNode node, Map resultMap, Column currentColumn, StructType schema) { if (node == null || !node.isObject() || schema == null) { return; } for (StructField field : schema.fields()) { String fieldName = field.getName(); JsonNode valueNode = node.get(fieldName); if (valueNode == null) { // Field not present in JSON, skip continue; } Column newColumn = currentColumn.appendNestedField(fieldName); DataType fieldType = field.getDataType(); if (fieldType instanceof StructType) { // This is a nested structure, recurse deeper if (valueNode.isObject()) { parseNullCounts(valueNode, resultMap, newColumn, (StructType) fieldType); } } else { // This is a leaf value - parse as long for null count if (valueNode.isNumber()) { resultMap.put(newColumn, valueNode.asLong()); } } } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/statistics/SnapshotStatistics.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.statistics; import io.delta.kernel.Snapshot; import io.delta.kernel.annotation.Evolving; import java.util.Optional; /** Provides statistics and metadata about a {@link Snapshot}. */ @Evolving public interface SnapshotStatistics { /** * Determines the appropriate mode for writing a checksum file for this Snapshot. * *

The returned value can be passed to {@link Snapshot#writeChecksum} to write the checksum * file using the most efficient approach available: * *

    *
  • {@link Optional#empty()} - Checksum already exists, no write needed *
  • {@link Optional} of {@link Snapshot.ChecksumWriteMode#SIMPLE} - Fast write using * in-memory CRC info *
  • {@link Optional} of {@link Snapshot.ChecksumWriteMode#FULL} - Requires log replay to * compute CRC info *
* * @return the recommended checksum write mode, or empty if checksum already exists */ Optional getChecksumWriteMode(); // TODO: getNumUnpublishedCatalogCommits // TODO: getNumDeltasSinceLastCheckpoint // TODO: getCheckpointInterval } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/transaction/CreateTableTransactionBuilder.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.transaction; import io.delta.kernel.Transaction; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.commit.Committer; import io.delta.kernel.engine.Engine; import io.delta.kernel.utils.CloseableIterable; import java.util.Map; /** * Builder for creating a {@link Transaction} to create a new Delta table. * * @since 3.4.0 */ @Evolving public interface CreateTableTransactionBuilder { /** * Set table properties for the new Delta table. * *

This method can be called multiple times to add additional properties. If a property key * already exists from a previous call with a different value, an {@link IllegalArgumentException} * will be thrown. * *

Note, user-properties (those without a '.delta' prefix) are case-sensitive. Delta-properties * are case-insensitive and are normalized to their expected case before writing to the log. * * @param properties A map of table property names to their values. * @throws IllegalArgumentException if any property key already exists from a previous call with a * different value. */ CreateTableTransactionBuilder withTableProperties(Map properties); /** * Set the data layout specification for the new Delta table. * *

The data layout specification determines how data files are organized within the table, such * as partitioning and clustering strategies. * *

The default, if not specified in the builder, is unpartitioned. * * @param spec The data layout specification. * @see DataLayoutSpec */ CreateTableTransactionBuilder withDataLayoutSpec(DataLayoutSpec spec); /** * Set the maximum number of retries to retry the commit in the case of a retryable error. * * @param maxRetries The maximum number of retries. Must be at least 0. Default is 200. */ CreateTableTransactionBuilder withMaxRetries(int maxRetries); /** * Provides a custom committer to use at transaction commit time. * *

Catalog implementations that wish to support the catalogManaged Delta table feature should * provide to engines their own catalog-specific Committer implementation which may, for example, * send a commit RPC to the catalog service to finalize the commit. * *

If no committer is provided, a default committer will be created that only supports writing * into filesystem-managed Delta tables. * * @param committer the committer to use * @return a new builder instance with the provided committer * @see Committer */ CreateTableTransactionBuilder withCommitter(Committer committer); /** * Build the transaction for creating the Delta table. * *

This validates all the configuration and creates a {@link Transaction} that can be used to * create the new Delta table. The transaction must be committed using {@link * Transaction#commit(Engine, CloseableIterable)} to actually create the table. * * @param engine The {@link Engine} instance to use. * @return A configured {@link Transaction} for creating the table. */ Transaction build(Engine engine); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/transaction/DataLayoutSpec.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.transaction; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.expressions.Column; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; /** * Specification for the data layout of a Delta table, including partitioning and clustering * configurations. * *

This class supports different layout strategies: * *

    *
  • No data layout spec: No special data layout *
  • Partitioned: Data is partitioned by specified columns. *
  • Clustered: Data is clustered by specified columns. *
* * @since 3.4.0 */ @Evolving public class DataLayoutSpec { /** * Creates data layout spec with no special layout. * * @return A new {@link DataLayoutSpec} with no special layout. */ public static DataLayoutSpec noDataLayout() { return new DataLayoutSpec(null, null); } /** * Creates a data layout specification for a partitioned table. * * @param partitionColumns The columns to partition by. Cannot be null or empty. Only top-level * columns are supported for partitioning. * @return A new {@link DataLayoutSpec} for a partitioned table. */ public static DataLayoutSpec partitioned(List partitionColumns) { checkArgument( partitionColumns != null && !partitionColumns.isEmpty(), "Partition columns cannot be null or empty"); checkArgument( partitionColumns.stream().allMatch(col -> col.getNames().length == 1), "Partition columns must be only top-level columns"); return new DataLayoutSpec(partitionColumns, null); } /** * Creates a data layout specification for a clustered table. * * @param clusteringColumns The columns to cluster by. Cannot be null, but can be empty to * indicate clustering is enabled without specific column definitions. * @return A new {@link DataLayoutSpec} for a clustered table. */ public static DataLayoutSpec clustered(List clusteringColumns) { checkArgument( clusteringColumns != null, "Clustering columns cannot be null (but can be empty)"); return new DataLayoutSpec(null, clusteringColumns); } private final List partitionColumns; private final List clusteringColumns; private DataLayoutSpec(List partitionColumns, List clusteringColumns) { if (partitionColumns != null && clusteringColumns != null) { throw new IllegalArgumentException("Cannot specify both partition and clustering columns"); } if (partitionColumns != null && partitionColumns.isEmpty()) { throw new IllegalArgumentException("Partition columns cannot be empty"); } this.partitionColumns = partitionColumns; this.clusteringColumns = clusteringColumns; } /** * Returns true if this layout specification includes partitioning. * *

Partitioning requires non-empty partition columns. An empty list of partition columns is not * considered valid partitioning. */ public boolean hasPartitioning() { return partitionColumns != null && !partitionColumns.isEmpty(); } /** * Returns true if this layout specification includes clustering. * *

Clustering can be enabled even with empty clustering columns, which indicates that * clustering is enabled on the table but no specific columns are defined yet. */ public boolean hasClustering() { return clusteringColumns != null; } /** * Returns true if this is a data layout spec with no special layout. * *

This means it has neither partitioning nor clustering enabled. */ public boolean hasNoDataLayoutSpec() { return !hasPartitioning() && !hasClustering(); } /** * Returns the partition columns for this layout. * * @throws IllegalStateException if partitioning is not enabled on this layout. Use {@link * #hasPartitioning()} to check first. */ public List getPartitionColumns() { if (!hasPartitioning()) { throw new IllegalStateException( "Cannot get partition columns: partitioning is not enabled on this layout"); } return Collections.unmodifiableList(partitionColumns); } /** * Returns the partition columns for this layout as strings. * * @throws IllegalStateException if partitioning is not enabled on this layout. Use {@link * #hasPartitioning()} to check first. */ public List getPartitionColumnsAsStrings() { return getPartitionColumns().stream() .map(col -> col.getNames()[0]) .collect(Collectors.toList()); } /** * Returns the clustering columns for this layout. * *

The returned list may be empty if clustering is enabled but no specific columns are defined. * * @throws IllegalStateException if clustering is not enabled on this layout. Use {@link * #hasClustering()} to check first. */ public List getClusteringColumns() { if (!hasClustering()) { throw new IllegalStateException( "Cannot get clustering columns: clustering is not enabled on this layout"); } return Collections.unmodifiableList(clusteringColumns); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/transaction/ReplaceTableTransactionBuilder.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.transaction; import io.delta.kernel.Transaction; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.engine.Engine; import java.util.Map; /** * Builds a {@link Transaction} to replace an existing Delta table. * *

Replace table creates a new table definition (schema, properties, layout) and atomically * replaces the existing table. * * @since 3.4.0 */ @Evolving public interface ReplaceTableTransactionBuilder { /** * Set table properties for the new table definition. * * @param properties A map of table property names to their values. */ ReplaceTableTransactionBuilder withTableProperties(Map properties); /** * Set the data layout specification for the new table definition. * * @param spec The data layout specification. * @see DataLayoutSpec */ ReplaceTableTransactionBuilder withDataLayoutSpec(DataLayoutSpec spec); /** * Set the maximum number of retries to retry the commit in the case of a retryable error. * * @param maxRetries The maximum number of retries. Must be at least 0. Default is 200. */ ReplaceTableTransactionBuilder withMaxRetries(int maxRetries); /** * Build the transaction for replacing the Delta table. * *

The transaction must be committed using {@link Transaction#commit(Engine, * io.delta.kernel.utils.CloseableIterable)} to apply the changes. * * @param engine The {@link Engine} instance to use for the transaction. Cannot be null. * @return A configured {@link Transaction} for replacing the table. */ Transaction build(Engine engine); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/transaction/UpdateTableTransactionBuilder.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.transaction; import io.delta.kernel.Transaction; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.*; import io.delta.kernel.expressions.Column; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterable; import java.util.List; import java.util.Map; import java.util.Set; /** * Builds a {@link Transaction} to update an existing Delta table. * * @since 3.4.0 */ @Evolving public interface UpdateTableTransactionBuilder { /** * Set a new schema for the table, enabling schema evolution. * *

Schema evolution allows you to modify the table's structure by adding, removing, renaming, * or reordering columns. Column mapping must be enabled on the table for schema evolution to be * supported. * *

The provided schema should preserve field metadata (such as field IDs and physical names) * for existing columns. Columns without metadata will be considered new columns and be assigned * new IDs and physical names automatically. * *

Supported schema evolution operations: * *

    *
  • Add columns: New columns can be added at any position *
  • Rename columns: Change the logical name while preserving the physical name *
  • Type widening: Compatible type changes (e.g., int to long) *
  • Reorder columns: Change the position of columns in the schema *
  • Drop columns: Remove columns (with restrictions) *
* * @param schema The new schema for the table. Cannot be null. Must be compatible with the current * schema and follow schema evolution rules. */ UpdateTableTransactionBuilder withUpdatedSchema(StructType schema); /** * Add or update table properties (configuration). * *

Properties specified here will be added to the table or override existing values. To remove * properties, use {@link #withTablePropertiesRemoved(Set)}. * * @param properties A map of property names to their values. The properties will be validated and * normalized. Cannot be null. */ UpdateTableTransactionBuilder withTablePropertiesAdded(Map properties); /** * Remove table properties from the table configuration. * *

The specified property keys will be removed from the table's configuration. Attempting to * remove a property that doesn't exist is not an error. * *

Currently only user-properties (in other words, ones that are not prefixed by 'delta.') can * be removed using this API. Adding and removing the same key in the same transaction is not * allowed. * * @param propertyKeys A set of property names to remove. Cannot be null. * @throws IllegalArgumentException if attempting to remove a 'delta.' property */ UpdateTableTransactionBuilder withTablePropertiesRemoved(Set propertyKeys); /** * Update the clustering columns for the table and enable clustering if it is not already enabled. * Note: clustering cannot be enabled for a partitioned table. * * @param clusteringColumns The columns to cluster by. Cannot be null. * @throws IllegalArgumentException if the table is partitioned */ UpdateTableTransactionBuilder withClusteringColumns(List clusteringColumns); /** * Set a transaction identifier for idempotent operations. * *

Transaction identifiers allow you to implement idempotent operations by ensuring that * multiple attempts to perform the same logical operation don't result in duplicate effects. This * is useful for: * *

    *
  • Retry logic in distributed systems *
  • Exactly-once processing guarantees *
  • Recovery from failures *
* *

If a transaction with the same application ID and version (or higher) has already been * committed the transaction will fail. * * @param applicationId A unique identifier for the application or process. Cannot be null. * @param transactionVersion A monotonically increasing version number for this application ID. */ UpdateTableTransactionBuilder withTransactionId(String applicationId, long transactionVersion); /** * Set the maximum number of retries for handling concurrent write conflicts. * *

When multiple writers attempt to modify the same Delta table simultaneously, conflicts can * occur. This setting controls how many times the operation will be retried with conflict * resolution before giving up. * * @param maxRetries The maximum number of retries. Must be at least 0. Default is 200. */ UpdateTableTransactionBuilder withMaxRetries(int maxRetries); /** * Set the log compaction interval for optimizing the transaction log. * *

Log compaction creates periodic checkpoint files that consolidate multiple transaction log * entries, improving read performance and reducing the number of files that need to be processed * when reading table metadata. * *

A value of 0 disables automatic log compaction for this transaction. Positive values specify * how many commits should occur between compactions. Defaults to 0. * * @param logCompactionInterval The number of commits between checkpoints. Must be at least 0. A * value of 0 disables log compaction. */ UpdateTableTransactionBuilder withLogCompactionInterval(int logCompactionInterval); /** * Build the transaction for updating the Delta table. * *

This validates all the configuration and creates a {@link Transaction} that can be used to * update the existing Delta table. The transaction must be committed using {@link * Transaction#commit(Engine, CloseableIterable)} to actually apply the changes. * * @param engine The {@link Engine} instance to use for the transaction. Cannot be null. * @return A configured {@link Transaction} for updating the table. * @throws ConcurrentTransactionException if the table already has a committed transaction with * the same given transaction identifier. */ Transaction build(Engine engine); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/ArrayType.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import io.delta.kernel.annotation.Evolving; import java.util.Objects; /** * Represent {@code array} data type * * @since 3.0.0 */ @Evolving public class ArrayType extends DataType { private final StructField elementField; public static final String ARRAY_ELEMENT_NAME = "element"; public ArrayType(DataType elementType, boolean containsNull) { this.elementField = new StructField(ARRAY_ELEMENT_NAME, elementType, containsNull); } public ArrayType(StructField elementField) { this.elementField = elementField; } public StructField getElementField() { return elementField; } public DataType getElementType() { return elementField.getDataType(); } public boolean containsNull() { return elementField.isNullable(); } @Override public boolean equivalent(DataType dataType) { return dataType instanceof ArrayType && ((ArrayType) dataType).getElementType().equivalent(getElementType()); } /** * Checks whether the given {@code dataType} is compatible with this type when writing data. * Collation differences are ignored. * *

This method is intended to be used during the write path to validate that an input type * matches the expected schema before data is written. * *

It should not be used in other cases, such as the read path. * * @param dataType the input data type being written * @return {@code true} if the input type is compatible with this type. */ @Override public boolean isWriteCompatible(DataType dataType) { if (this == dataType) { return true; } if (dataType == null || getClass() != dataType.getClass()) { return false; } ArrayType arrayType = (ArrayType) dataType; return (elementField == null && arrayType.elementField == null) || (elementField != null && elementField.isWriteCompatible(arrayType.elementField)); } @Override public boolean isNested() { return true; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } ArrayType arrayType = (ArrayType) o; return Objects.equals(elementField, arrayType.elementField); } @Override public int hashCode() { return Objects.hash(elementField); } @Override public String toString() { return "array[" + getElementType() + "]"; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/BasePrimitiveType.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import java.util.*; import java.util.function.Supplier; import java.util.stream.Collectors; /** Base class for all primitive types {@link DataType}. */ public abstract class BasePrimitiveType extends DataType { /** * Create a primitive type {@link DataType} * * @param primitiveTypeName Primitive type name. * @return {@link DataType} for given primitive type name */ public static DataType createPrimitive(String primitiveTypeName) { return Optional.ofNullable(nameToPrimitiveTypeMap.get().get(primitiveTypeName)) .orElseThrow( () -> new IllegalArgumentException("Unknown primitive type " + primitiveTypeName)); } /** Is the given type name a primitive type? */ public static boolean isPrimitiveType(String typeName) { return nameToPrimitiveTypeMap.get().containsKey(typeName); } /** For testing only */ public static List getAllPrimitiveTypes() { return nameToPrimitiveTypeMap.get().values().stream().collect(Collectors.toList()); } private static final Supplier> nameToPrimitiveTypeMap = () -> Collections.unmodifiableMap( new HashMap() { { put("boolean", BooleanType.BOOLEAN); put("byte", ByteType.BYTE); put("short", ShortType.SHORT); put("integer", IntegerType.INTEGER); put("long", LongType.LONG); put("float", FloatType.FLOAT); put("double", DoubleType.DOUBLE); put("date", DateType.DATE); put("timestamp", TimestampType.TIMESTAMP); put("timestamp_ntz", TimestampNTZType.TIMESTAMP_NTZ); put("binary", BinaryType.BINARY); put("string", StringType.STRING); put("variant", VariantType.VARIANT); } }); private final String primitiveTypeName; protected BasePrimitiveType(String primitiveTypeName) { this.primitiveTypeName = primitiveTypeName; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } BasePrimitiveType that = (BasePrimitiveType) o; return primitiveTypeName.equals(that.primitiveTypeName); } @Override public boolean isNested() { return false; } @Override public int hashCode() { return Objects.hash(primitiveTypeName); } @Override public String toString() { return primitiveTypeName; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/BinaryType.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import io.delta.kernel.annotation.Evolving; /** * The data type representing {@code byte[]} values. * * @since 3.0.0 */ @Evolving public class BinaryType extends BasePrimitiveType { public static final BinaryType BINARY = new BinaryType(); private BinaryType() { super("binary"); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/BooleanType.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import io.delta.kernel.annotation.Evolving; /** * Data type representing {@code boolean} type values. * * @since 3.0.0 */ @Evolving public class BooleanType extends BasePrimitiveType { public static final BooleanType BOOLEAN = new BooleanType(); private BooleanType() { super("boolean"); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/ByteType.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import io.delta.kernel.annotation.Evolving; /** * The data type representing {@code byte} type values. * * @since 3.0.0 */ @Evolving public class ByteType extends BasePrimitiveType { public static final ByteType BYTE = new ByteType(); private ByteType() { super("byte"); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/CollationIdentifier.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.annotation.Evolving; import java.util.Objects; import java.util.Optional; /** * Identifies collation for string type. * Collation identifiers * * @since 3.3.0 */ @Evolving public class CollationIdentifier { /** The default Spark UTF8_BINARY collation. */ public static final CollationIdentifier SPARK_UTF8_BINARY = new CollationIdentifier("SPARK", "UTF8_BINARY"); private final String provider; private final String name; private final Optional version; private CollationIdentifier(String provider, String collationName) { this(provider, collationName, Optional.empty()); } private CollationIdentifier(String provider, String collationName, Optional version) { Objects.requireNonNull(provider, "Collation provider cannot be null."); Objects.requireNonNull(collationName, "Collation name cannot be null."); Objects.requireNonNull(version, "Collation version cannot be null."); this.provider = provider.toUpperCase(); this.name = collationName.toUpperCase(); this.version = version.map(String::toUpperCase); } /** @return collation provider. */ public String getProvider() { return provider; } /** @return collation name. */ public String getName() { return name; } /** @return collation version. */ public Optional getVersion() { return version; } /** @return if this collation is the default Spark UTF8_BINARY collation. */ public boolean isSparkUTF8BinaryCollation() { return equals(SPARK_UTF8_BINARY); } /** * @param identifier collation identifier in string form of
* {@code PROVIDER.COLLATION_NAME[.COLLATION_VERSION]}. * @return appropriate collation identifier object */ public static CollationIdentifier fromString(String identifier) { long numDots = identifier.chars().filter(ch -> ch == '.').count(); checkArgument(numDots > 0, "Invalid collation identifier: %s", identifier); if (numDots == 1) { String[] parts = identifier.split("\\."); return new CollationIdentifier(parts[0], parts[1]); } else { String[] parts = identifier.split("\\.", 3); return new CollationIdentifier(parts[0], parts[1], Optional.of(parts[2])); } } /** Collation identifiers are identical when the provider, name, and version are the same. */ @Override public boolean equals(Object o) { if (!(o instanceof CollationIdentifier)) { return false; } CollationIdentifier other = (CollationIdentifier) o; return this.provider.equals(other.provider) && this.name.equals(other.name) && this.version.equals(other.version); } /** @return collation identifier in form of {@code PROVIDER.COLLATION_NAME}. */ public String toStringWithoutVersion() { return String.format("%s.%s", provider, name); } /** @return collation identifier in form of {@code PROVIDER.COLLATION_NAME[.COLLATION_VERSION]} */ @Override public String toString() { if (version.isPresent()) { return String.format("%s.%s.%s", provider, name, version.get()); } else { return String.format("%s.%s", provider, name); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/DataType.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import io.delta.kernel.annotation.Evolving; /** * Base class for all data types. * * @since 3.0.0 */ @Evolving public abstract class DataType { /** * Are the data types same? The metadata, collations or column names could be different. * *

Should be used for schema comparisons during schema evolution. * * @param dataType * @return */ public boolean equivalent(DataType dataType) { return equals(dataType); } /** * Checks whether the given {@code dataType} is compatible with this type when writing data. * Collation differences are ignored. * *

This method is intended to be used during the write path to validate that an input type * matches the expected schema before data is written. * *

It should not be used in other cases, such as the read path. * * @param dataType the input data type being written * @return {@code true} if the input type is compatible with this type. */ public boolean isWriteCompatible(DataType dataType) { return equals(dataType); } /** * Returns true iff this data is a nested data type (it logically parameterized by other types). * *

For example StructType, ArrayType, MapType are nested data types. */ public abstract boolean isNested(); @Override public abstract int hashCode(); @Override public abstract boolean equals(Object obj); @Override public abstract String toString(); } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/DateType.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import io.delta.kernel.annotation.Evolving; /** * A date type, supporting "0001-01-01" through "9999-12-31". Internally, this is represented as the * number of days from 1970-01-01. * * @since 3.0.0 */ @Evolving public class DateType extends BasePrimitiveType { public static final DateType DATE = new DateType(); private DateType() { super("date"); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/DecimalType.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import io.delta.kernel.annotation.Evolving; import java.util.Objects; /** * The data type representing {@code java.math.BigDecimal} values. A Decimal that must have fixed * precision (the maximum number of digits) and scale (the number of digits on right side of dot). * *

The precision can be up to 38, scale can also be up to 38 (less or equal to precision). * *

The default precision and scale is (10, 0). * * @since 3.0.0 */ @Evolving public final class DecimalType extends DataType { public static final DecimalType USER_DEFAULT = new DecimalType(10, 0); private final int precision; private final int scale; public DecimalType(int precision, int scale) { if (precision < 0 || precision > 38 || scale < 0 || scale > 38 || scale > precision) { throw new IllegalArgumentException( String.format( "Invalid precision and scale combo (%d, %d). They should be in the range [0, 38] " + "and scale can not be more than the precision.", precision, scale)); } this.precision = precision; this.scale = scale; } /** @return the maximum number of digits of the decimal */ public int getPrecision() { return precision; } /** @return the number of digits on the right side of the decimal point (dot) */ public int getScale() { return scale; } @Override public boolean isNested() { return false; } @Override public String toString() { return String.format("Decimal(%d, %d)", precision, scale); } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } DecimalType that = (DecimalType) o; return precision == that.precision && scale == that.scale; } @Override public int hashCode() { return Objects.hash(precision, scale); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/DoubleType.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import io.delta.kernel.annotation.Evolving; /** * The data type representing {@code double} type values. * * @since 3.0.0 */ @Evolving public class DoubleType extends BasePrimitiveType { public static final DoubleType DOUBLE = new DoubleType(); private DoubleType() { super("double"); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/FieldMetadata.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license below). * It contains modifications which are licensed as specified above. */ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import io.delta.kernel.internal.util.Preconditions; import java.util.*; import java.util.stream.Collectors; /** The metadata for a given {@link StructField}. The contents are immutable. */ public final class FieldMetadata { private final Map metadata; private FieldMetadata(Map metadata) { this.metadata = Collections.unmodifiableMap(metadata); } /** @return list of the key-value pairs in this {@link FieldMetadata} */ public Map getEntries() { return metadata; } /** * @param key the key to check for * @return True if {@code this} contains a mapping for the given key, False otherwise */ public boolean contains(String key) { return metadata.containsKey(key); } /** * @param key the key to check for * @return the value to which the specified key is mapped, or null if there is no mapping for the * given key */ public Object get(String key) { return metadata.get(key); } public Long getLong(String key) { return get(key, Long.class); } public Double getDouble(String key) { return get(key, Double.class); } public Boolean getBoolean(String key) { return get(key, Boolean.class); } public String getString(String key) { return get(key, String.class); } public MetadataColumnSpec getMetadataColumnSpec(String key) { return get(key, MetadataColumnSpec.class); } public FieldMetadata getMetadata(String key) { return get(key, FieldMetadata.class); } public Long[] getLongArray(String key) { return get(key, Long[].class); } public Double[] getDoubleArray(String key) { return get(key, Double[].class); } public Boolean[] getBooleanArray(String key) { return get(key, Boolean[].class); } public String[] getStringArray(String key) { return get(key, String[].class); } public FieldMetadata[] getMetadataArray(String key) { return get(key, FieldMetadata[].class); } @Override public String toString() { return metadata.entrySet().stream() .map( entry -> { String key = entry.getKey(); Object value = entry.getValue(); if (value == null) { return key + "=null"; } String valueStr = value.getClass().isArray() ? Arrays.toString((Object[]) value) : value.toString(); return key + "=" + valueStr; }) .collect(Collectors.joining(", ", "{", "}")); } /** Are the metadata same, ignoring the specified keys? */ public boolean equalsIgnoreKeys(FieldMetadata other, Set keys) { Preconditions.checkArgument(keys != null, "keys must not be null"); if (this == other) { return true; } if (other == null) { return false; } Map filteredMetadata = new HashMap<>(); for (Map.Entry entry : this.metadata.entrySet()) { if (!keys.contains(entry.getKey())) { filteredMetadata.put(entry.getKey(), entry.getValue()); } } Map otherFilteredMetadata = new HashMap<>(); for (Map.Entry entry : other.metadata.entrySet()) { if (!keys.contains(entry.getKey())) { otherFilteredMetadata.put(entry.getKey(), entry.getValue()); } } if (filteredMetadata.size() != otherFilteredMetadata.size()) { return false; } return filteredMetadata.entrySet().stream() .allMatch( e -> { Object value = e.getValue(); Object otherValue = otherFilteredMetadata.get(e.getKey()); return Objects.deepEquals(value, otherValue); }); } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; FieldMetadata other = (FieldMetadata) o; if (this.metadata.size() != other.metadata.size()) return false; return this.metadata.entrySet().stream() .allMatch( e -> { Object value = e.getValue(); Object otherValue = other.metadata.get(e.getKey()); return Objects.deepEquals(value, otherValue); }); } @Override public int hashCode() { return metadata.entrySet().stream() .mapToInt( entry -> (entry.getValue().getClass().isArray() ? (entry.getKey() == null ? 0 : entry.getKey().hashCode()) ^ Arrays.hashCode((Object[]) entry.getValue()) : entry.hashCode())) .sum(); } /** @return a new {@link FieldMetadata.Builder} */ public static Builder builder() { return new Builder(); } /** @return an empty {@link FieldMetadata} instance */ public static FieldMetadata empty() { return builder().build(); } /** * @param key the key to check for * @param type the type to cast the value to * @return the value (casted to the given type) to which the specified key is mapped, or null if * there is no mapping for the given key */ private T get(String key, Class type) { Object value = get(key); if (null == value) { return (T) value; } Preconditions.checkArgument( value.getClass().isAssignableFrom(type), "Expected '%s' to be of type '%s' but was '%s'", value, type, value.getClass()); return type.cast(value); } /** Builder class for {@link FieldMetadata}. */ public static class Builder { private Map metadata = new HashMap(); public Builder putNull(String key) { metadata.put(key, null); return this; } public Builder putLong(String key, long value) { metadata.put(key, value); return this; } public Builder putDouble(String key, double value) { metadata.put(key, value); return this; } public Builder putBoolean(String key, boolean value) { metadata.put(key, value); return this; } public Builder putString(String key, String value) { metadata.put(key, value); return this; } public Builder putMetadataColumnSpec(String key, MetadataColumnSpec value) { metadata.put(key, value); return this; } public Builder putFieldMetadata(String key, FieldMetadata value) { metadata.put(key, value); return this; } public Builder putLongArray(String key, Long[] value) { metadata.put(key, value); return this; } public Builder putDoubleArray(String key, Double[] value) { metadata.put(key, value); return this; } public Builder putBooleanArray(String key, Boolean[] value) { metadata.put(key, value); return this; } public Builder putStringArray(String key, String[] value) { metadata.put(key, value); return this; } public Builder putFieldMetadataArray(String key, FieldMetadata[] value) { metadata.put(key, value); return this; } /** * Adds all metadata from {@code meta.metadata} to the builder's {@code metadata}. Entries in * the builder's {@code metadata} are overwritten with the entries from {@code meta.metadata}. * * @param meta The {@link FieldMetadata} instance holding metadata * @return this */ public Builder fromMetadata(FieldMetadata meta) { metadata.putAll(meta.metadata); return this; } /** @return a new {@link FieldMetadata} with the mappings added to the builder */ public FieldMetadata build() { return new FieldMetadata(this.metadata); } public FieldMetadata getMetadata(String key) { Object value = metadata.get(key); if (null == value) { return null; } if (value instanceof FieldMetadata) { return (FieldMetadata) value; } throw new io.delta.kernel.exceptions.KernelException( String.format( "Expected '%s' to be of type 'FieldMetadata' but was '%s'", value, value.getClass().getName())); } public Builder remove(String s) { metadata.remove(s); return this; } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/FloatType.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import io.delta.kernel.annotation.Evolving; /** * The data type representing {@code float} type values. * * @since 3.0.0 */ @Evolving public class FloatType extends BasePrimitiveType { public static final FloatType FLOAT = new FloatType(); private FloatType() { super("float"); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/GeographyType.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import io.delta.kernel.annotation.Evolving; import java.util.Objects; import java.util.Set; /** * The data type representing geography values. A Geography must have a fixed Spatial Reference * System Identifier (SRID) that defines the coordinate system and an algorithm that determines how * geometric calculations are performed. * *

The SRID is specified as a string and the algorithm defines the calculation method. The engine * is responsible for validating and interpreting the SRID and algorithm values. * * @since 3.0.0 */ @Evolving public final class GeographyType extends DataType { public static final String DEFAULT_SRID = "OGC:CRS84"; public static final String DEFAULT_ALGORITHM = "spherical"; public static final Set VALID_ALGORITHMS = Set.of("spherical", "vincenty", "thomas", "andoyer", "karney"); private final String srid; private final String algorithm; /** Returns a GeographyType with the default SRID and algorithm. */ public static GeographyType ofDefault() { return new GeographyType(DEFAULT_SRID, DEFAULT_ALGORITHM); } /** * Returns a GeographyType with the specified SRID and default algorithm. * * @param srid the Spatial Reference System Identifier (any non-null, non-empty string) */ public static GeographyType ofSRID(String srid) { return new GeographyType(srid, DEFAULT_ALGORITHM); } /** * Returns a GeographyType with the default SRID and the specified algorithm. * * @param algorithm one of: spherical, vincenty, thomas, andoyer, karney */ public static GeographyType ofAlgorithm(String algorithm) { return new GeographyType(DEFAULT_SRID, algorithm); } /** * Create a GeographyType with the specified SRID and algorithm. * * @param srid the Spatial Reference System Identifier (any non-null, non-empty string) * @param algorithm the algorithm for geometric calculations (any non-null, non-empty string) * @throws IllegalArgumentException if the SRID or algorithm is null or empty or algorithm is * invalid */ public GeographyType(String srid, String algorithm) { if (srid == null || srid.isEmpty()) { throw new IllegalArgumentException("SRID cannot be null or empty"); } if (algorithm == null || algorithm.isEmpty()) { throw new IllegalArgumentException("Algorithm cannot be null or empty"); } if (!VALID_ALGORITHMS.contains(algorithm)) { throw new IllegalArgumentException( "Algorithm must be one of: spherical, vincenty, thomas, andoyer, karney, got: " + algorithm); } this.srid = srid; this.algorithm = algorithm; } /** * Get the Spatial Reference System Identifier. * * @return the SRID string */ public String getSRID() { return srid; } /** * Get the algorithm for geometric calculations. * * @return the algorithm string */ public String getAlgorithm() { return algorithm; } @Override public boolean isNested() { return false; } /** * Serialize this GeographyType to its string representation with minimal info. * * @return the serialized string representation */ public String simpleString() { return String.format("geography(%s, %s)", srid, algorithm); } @Override public String toString() { return String.format("Geography(srid=%s, algorithm=%s)", srid, algorithm); } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } GeographyType that = (GeographyType) o; return srid.equals(that.srid) && algorithm.equals(that.algorithm); } @Override public int hashCode() { return Objects.hash(srid, algorithm); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/GeometryType.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import io.delta.kernel.annotation.Evolving; import java.util.Objects; /** * The data type representing geometry values. A Geometry must have a fixed Spatial Reference System * Identifier (SRID) that defines the coordinate system. * *

The SRID is specified as a string The engine is responsible for validating and interpreting * the SRID value. * * @since 3.0.0 */ @Evolving public final class GeometryType extends DataType { public static final String DEFAULT_SRID = "OGC:CRS84"; private final String srid; /** Returns a GeometryType with the default SRID. */ public static GeometryType ofDefault() { return new GeometryType(DEFAULT_SRID); } /** * Returns a GeometryType with the specified SRID. * * @param srid the Spatial Reference System Identifier (any non-null, non-empty string) */ public static GeometryType ofSRID(String srid) { return new GeometryType(srid); } /** * Create a GeometryType with the specified SRID. * * @param srid the Spatial Reference System Identifier (any non-null, non-empty string) * @throws IllegalArgumentException if the SRID is null or empty */ public GeometryType(String srid) { if (srid == null || srid.isEmpty()) { throw new IllegalArgumentException("SRID cannot be null or empty"); } this.srid = srid; } /** * Get the Spatial Reference System Identifier. * * @return the SRID string */ public String getSRID() { return srid; } @Override public boolean isNested() { return false; } /** * Serialize this GeometryType to its string representation with minimal info. * * @return the serialized string representation */ public String simpleString() { return String.format("geometry(%s)", srid); } @Override public String toString() { return String.format("Geometry(srid=%s)", srid); } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } GeometryType that = (GeometryType) o; return srid.equals(that.srid); } @Override public int hashCode() { return Objects.hash(srid); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/IntegerType.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import io.delta.kernel.annotation.Evolving; /** * The data type representing {@code integer} type values. * * @since 3.0.0 */ @Evolving public class IntegerType extends BasePrimitiveType { public static final IntegerType INTEGER = new IntegerType(); private IntegerType() { super("integer"); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/LongType.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import io.delta.kernel.annotation.Evolving; /** * The data type representing {@code long} type values. * * @since 3.0.0 */ @Evolving public class LongType extends BasePrimitiveType { public static final LongType LONG = new LongType(); private LongType() { super("long"); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/MapType.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import io.delta.kernel.annotation.Evolving; import java.util.Objects; /** * Data type representing a {@code map} type. * * @since 3.0.0 */ @Evolving public class MapType extends DataType { private final StructField keyField; private final StructField valueField; public static final String MAP_KEY_NAME = "key"; public static final String MAP_VALUE_NAME = "value"; public MapType(DataType keyType, DataType valueType, boolean valueContainsNull) { validateKeyType(keyType); this.keyField = new StructField(MAP_KEY_NAME, keyType, false); this.valueField = new StructField(MAP_VALUE_NAME, valueType, valueContainsNull); } public MapType(StructField keyField, StructField valueField) { validateKeyType(keyField.getDataType()); this.keyField = keyField; this.valueField = valueField; } /** * The Delta protocol does not support collated string types as map keys. Only StringType with the * default UTF8_BINARY collation is allowed. * * @see * Collated String Type RFC */ private static void validateKeyType(DataType keyType) { if (keyType instanceof StringType && !((StringType) keyType).isUTF8BinaryCollated()) { throw new IllegalArgumentException( String.format( "MapType does not support collated string types as keys. " + "Found key type '%s', but only StringType with default UTF8_BINARY " + "collation is allowed.", keyType)); } } public StructField getKeyField() { return keyField; } public StructField getValueField() { return valueField; } public DataType getKeyType() { return getKeyField().getDataType(); } public DataType getValueType() { return getValueField().getDataType(); } public boolean isValueContainsNull() { return valueField.isNullable(); } @Override public boolean equivalent(DataType dataType) { return dataType instanceof MapType && ((MapType) dataType).getKeyType().equivalent(getKeyType()) && ((MapType) dataType).getValueType().equivalent(getValueType()) && ((MapType) dataType).isValueContainsNull() == isValueContainsNull(); } /** * Checks whether the given {@code dataType} is compatible with this type when writing data. * Collation differences are ignored. * *

This method is intended to be used during the write path to validate that an input type * matches the expected schema before data is written. * *

It should not be used in other cases, such as the read path. * * @param dataType the input data type being written * @return {@code true} if the input type is compatible with this type. */ @Override public boolean isWriteCompatible(DataType dataType) { if (this == dataType) { return true; } if (dataType == null || getClass() != dataType.getClass()) { return false; } MapType mapType = (MapType) dataType; return ((keyField == null && mapType.keyField == null) || (keyField != null && keyField.isWriteCompatible(mapType.keyField))) && ((valueField == null && mapType.valueField == null) || (valueField != null && valueField.isWriteCompatible(mapType.valueField))); } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } MapType mapType = (MapType) o; return Objects.equals(keyField, mapType.keyField) && Objects.equals(valueField, mapType.valueField); } @Override public boolean isNested() { return true; } @Override public int hashCode() { return Objects.hash(keyField, valueField); } @Override public String toString() { return String.format("map[%s, %s]", getKeyType(), getValueType()); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/MetadataColumnSpec.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; /** * Enumeration of metadata columns recognized by Delta Kernel. * *

Metadata columns provide additional information about rows in a Delta table. */ public enum MetadataColumnSpec { ROW_INDEX("row_index", LongType.LONG, false), ROW_ID("row_id", LongType.LONG, false), ROW_COMMIT_VERSION("row_commit_version", LongType.LONG, false); public final String textValue; public final DataType dataType; public final boolean nullable; MetadataColumnSpec(String textValue, DataType dataType, boolean nullable) { this.textValue = textValue; this.dataType = dataType; this.nullable = nullable; } public String toString() { return textValue; } public static MetadataColumnSpec fromString(String text) { for (MetadataColumnSpec type : MetadataColumnSpec.values()) { if (type.textValue.equalsIgnoreCase(text)) { return type; } } throw new IllegalArgumentException("Unknown MetadataColumnType: " + text); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/ShortType.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import io.delta.kernel.annotation.Evolving; /** * The data type representing {@code short} type values. * * @since 3.0.0 */ @Evolving public class ShortType extends BasePrimitiveType { public static final ShortType SHORT = new ShortType(); private ShortType() { super("short"); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/StringType.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import io.delta.kernel.annotation.Evolving; /** * The data type representing {@code string} type values. * * @since 3.0.0 */ @Evolving public class StringType extends BasePrimitiveType { public static final StringType STRING = new StringType(CollationIdentifier.SPARK_UTF8_BINARY); private final CollationIdentifier collationIdentifier; /** * @param collationIdentifier An identifier representing the collation to be used for string * comparison and sorting. This determines how strings will be ordered and compared in query * operations. */ public StringType(CollationIdentifier collationIdentifier) { super("string"); this.collationIdentifier = collationIdentifier; } /** * @param collationName name of collation in which this StringType will be observed. In form of * {@code PROVIDER.COLLATION_NAME[.VERSION]} */ public StringType(String collationName) { super("string"); this.collationIdentifier = CollationIdentifier.fromString(collationName); } /** @return StringType's collation identifier */ public CollationIdentifier getCollationIdentifier() { return collationIdentifier; } /** * Are the data types same? The metadata, collations or column names could be different. * * @param dataType * @return */ public boolean equivalent(DataType dataType) { return dataType instanceof StringType; } /** * Checks whether the given {@code dataType} is compatible with this type when writing data. * Collation differences are ignored. * *

This method is intended to be used during the write path to validate that an input type * matches the expected schema before data is written. * *

It should not be used in other cases, such as the read path. * * @param dataType the input data type being written * @return {@code true} if the input type is compatible with this type. */ @Override public boolean isWriteCompatible(DataType dataType) { return dataType instanceof StringType; } /** @return true if this StringType uses the default Spark UTF8_BINARY collation. */ public boolean isUTF8BinaryCollated() { return collationIdentifier.isSparkUTF8BinaryCollation(); } @Override public boolean equals(Object o) { if (!(o instanceof StringType)) { return false; } StringType that = (StringType) o; return collationIdentifier.equals(that.collationIdentifier); } /** * Override is needed because {@code toString()} may be used for schema serialization and similar * contexts, so collation information must be included. * * @return string representation of the StringType. */ @Override public String toString() { if (isUTF8BinaryCollated()) { return super.toString(); } else { return String.format("string collate %s", collationIdentifier.getName()); } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/StructField.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import static io.delta.kernel.types.MetadataColumnSpec.*; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.internal.types.DataTypeJsonSerDe; import io.delta.kernel.internal.util.SchemaIterable; import java.util.*; /** * Represents a subfield of {@link StructType} with additional properties and metadata. * * @since 3.0.0 */ @Evolving public class StructField { //////////////////////////////////////////////////////////////////////////////// // Static Fields / Methods //////////////////////////////////////////////////////////////////////////////// /** * The existence of this key indicates that a column is a metadata column and its values indicates * what kind of {@link MetadataColumnSpec} it is. */ public static final String METADATA_SPEC_KEY = "delta.metadataSpec"; /** * Indicates that a column was requested for internal computations and should not be returned to * the user. */ public static final String IS_INTERNAL_COLUMN_KEY = "delta.isInternalColumn"; /** The name of the default row index metadata column. */ private static final String DEFAULT_ROW_INDEX_COLUMN_NAME = "_metadata.row_index"; /** * The default row index metadata column used by Kernel. When present, this column must be * populated with row index of each row when reading from Parquet. */ public static StructField DEFAULT_ROW_INDEX_COLUMN = createMetadataColumn(DEFAULT_ROW_INDEX_COLUMN_NAME, MetadataColumnSpec.ROW_INDEX); public static final String COLLATIONS_METADATA_KEY = "__COLLATIONS"; public static final String FROM_TYPE_KEY = "fromType"; public static final String TO_TYPE_KEY = "toType"; public static final String FIELD_PATH_KEY = "fieldPath"; public static final String DELTA_TYPE_CHANGES_KEY = "delta.typeChanges"; /** * Creates a metadata column of the given {@code colSpec} with the given {@code name}. * * @param name Name of the metadata column * @param colSpec Type of the metadata column * @return A StructField representing the metadata column */ public static StructField createMetadataColumn(String name, MetadataColumnSpec colSpec) { switch (colSpec) { case ROW_INDEX: return new StructField( name, colSpec.dataType, colSpec.nullable, new FieldMetadata.Builder() .putMetadataColumnSpec(METADATA_SPEC_KEY, ROW_INDEX) .build()); case ROW_ID: return new StructField( name, colSpec.dataType, colSpec.nullable, new FieldMetadata.Builder().putMetadataColumnSpec(METADATA_SPEC_KEY, ROW_ID).build()); case ROW_COMMIT_VERSION: return new StructField( name, colSpec.dataType, colSpec.nullable, new FieldMetadata.Builder() .putMetadataColumnSpec(METADATA_SPEC_KEY, ROW_COMMIT_VERSION) .build()); default: throw new IllegalArgumentException("Unknown MetadataColumnType: " + colSpec); } } //////////////////////////////////////////////////////////////////////////////// // Instance Fields / Methods //////////////////////////////////////////////////////////////////////////////// private final String name; private final DataType dataType; private final boolean nullable; private final FieldMetadata metadata; private final List typeChanges; public StructField(String name, DataType dataType, boolean nullable) { this(name, dataType, nullable, FieldMetadata.empty()); } public StructField(String name, DataType dataType, boolean nullable, FieldMetadata metadata) { this(name, dataType, nullable, metadata, Collections.emptyList()); } /* * N.B. Type changes should be entirely managed by the Delta Kernel, users are not expected to * maintain this field, and therefore should not be using this constructor. */ StructField( String name, DataType dataType, boolean nullable, FieldMetadata metadata, List typeChanges) { this.name = name; this.dataType = dataType; this.nullable = nullable; this.typeChanges = typeChanges == null ? Collections.emptyList() : typeChanges; FieldMetadata nestedMetadata = collectNestedMapArrayTypeMetadata(); this.metadata = new FieldMetadata.Builder().fromMetadata(metadata).fromMetadata(nestedMetadata).build(); if (!this.typeChanges.isEmpty() && (dataType instanceof MapType || dataType instanceof StructType || dataType instanceof ArrayType)) { throw new KernelException("Type changes are not supported on nested types."); } } /** @return the name of this field */ public String getName() { return name; } /** @return the data type of this field */ public DataType getDataType() { return dataType; } /** @return the metadata for this field */ public FieldMetadata getMetadata() { return metadata; } /** @return whether this field allows to have a {@code null} value. */ public boolean isNullable() { return nullable; } /** * Returns the list of type changes for this field. A field can go through multiple type changes * (e.g. {@code int->long->decimal}). Changes are ordered from least recent to most recent in the * list (index 0 is the oldest change). * *

N.B. Type changes should be entirely managed by the Delta Kernel, users are not expected to * maintain this field. */ public List getTypeChanges() { return Collections.unmodifiableList(typeChanges); } public boolean isMetadataColumn() { return metadata != null && metadata.contains(METADATA_SPEC_KEY); } public boolean isDataColumn() { return !isMetadataColumn(); } /** Returns the type of metadata column if this is a metadata column, otherwise returns null. */ public MetadataColumnSpec getMetadataColumnSpec() { return metadata.getMetadataColumnSpec(METADATA_SPEC_KEY); } public boolean isInternalColumn() { return Optional.ofNullable(metadata.getBoolean(IS_INTERNAL_COLUMN_KEY)).orElse(false); } @Override public String toString() { return String.format( "StructField(name=%s,type=%s,nullable=%s,metadata=%s,typeChanges=%s)", name, dataType, nullable, metadata, typeChanges); } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } StructField that = (StructField) o; return nullable == that.nullable && name.equals(that.name) && dataType.equals(that.dataType) && metadata.equals(that.metadata) && Objects.equals(typeChanges, that.typeChanges); } /** * Checks whether the given {@code other} is compatible with this {@code StructField} when writing * data. Collation differences are ignored. */ public boolean isWriteCompatible(StructField other) { if (this == other) { return true; } if (other == null) { return false; } return nullable == other.nullable && name.equals(other.name) && dataType.isWriteCompatible(other.dataType) // Compare metadata while ignoring collation metadata differences && metadata.equalsIgnoreKeys(other.metadata, Collections.singleton(COLLATIONS_METADATA_KEY)) && Objects.equals(typeChanges, other.typeChanges); } @Override public int hashCode() { return Objects.hash(name, dataType, nullable, metadata, typeChanges); } public StructField withNewMetadata(FieldMetadata metadata) { return new StructField(name, dataType, nullable, metadata, typeChanges); } /** * Creates a copy of this StructField with the specified type changes. * *

N.B. Type changes should be entirely managed by the Delta Kernel, users are not expected to * maintain this field. * * @param typeChanges The list of type changes to set * @return A new StructField with the same properties but with the specified type changes */ public StructField withTypeChanges(List typeChanges) { return new StructField(name, dataType, nullable, metadata, typeChanges); } /** * Creates a copy of this StructField with the specified data type. * *

TypeChanges are NOT updated as part of this call. * * @param newType The new type to use in the StructField. * @return A new StructField with the same properties but with the specified data type. */ public StructField withDataType(DataType newType) { return new StructField(name, newType, nullable, metadata, typeChanges); } /** Fetches collation and type changes metadata from nested fields. */ private FieldMetadata collectNestedMapArrayTypeMetadata() { FieldMetadata.Builder collationBuilder = FieldMetadata.builder(); List typeChangesBuilder = new ArrayList<>(); // This is a little risky since this isn't fully initialized but should be fine since all fields // we needed are initialized. // StructTypes children would already have their own collation metadata, so skip them here. SchemaIterable iterable = SchemaIterable.newSchemaIterableWithIgnoredRecursion( new StructType().add(this), new Class[] {StructType.class}); for (SchemaIterable.SchemaElement element : iterable) { DataType type = element.getField().getDataType(); if (type instanceof StringType) { StringType stringType = (StringType) type; if (!stringType .getCollationIdentifier() .equals(CollationIdentifier.fromString("SPARK.UTF8_BINARY"))) { // TODO: Should this account for column mapping? String path = element.getPathFromNearestStructFieldAncestor( element.getNearestStructFieldAncestor().name); collationBuilder.putString( path, stringType.getCollationIdentifier().toStringWithoutVersion()); } } StructField field = element.getField(); if (!field.getTypeChanges().isEmpty()) { for (TypeChange typeChange : field.getTypeChanges()) { FieldMetadata.Builder typeChangeBuilder = FieldMetadata.builder(); typeChangeBuilder.putString(FROM_TYPE_KEY, typeAsString(typeChange.getFrom())); typeChangeBuilder.putString(TO_TYPE_KEY, typeAsString(typeChange.getTo())); if (!element.isStructField()) { // For type changes the field name the field name is not a prefix. typeChangeBuilder.putString( FIELD_PATH_KEY, element.getPathFromNearestStructFieldAncestor("")); } typeChangesBuilder.add(typeChangeBuilder.build()); } } } FieldMetadata.Builder finalBuilder = FieldMetadata.builder(); FieldMetadata collationMetadata = collationBuilder.build(); if (!collationMetadata.getEntries().isEmpty()) { finalBuilder.putFieldMetadata(COLLATIONS_METADATA_KEY, collationMetadata); } if (!typeChangesBuilder.isEmpty()) { finalBuilder.putFieldMetadataArray( DELTA_TYPE_CHANGES_KEY, typeChangesBuilder.toArray(new FieldMetadata[0])); } return finalBuilder.build(); } private static String typeAsString(DataType dt) { String jsonString = DataTypeJsonSerDe.serializeDataType(dt); // Remove leading/trailing quotes. return jsonString.substring(1, jsonString.length() - 1); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/StructType.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.expressions.Column; import io.delta.kernel.internal.types.DataTypeJsonSerDe; import io.delta.kernel.internal.util.Tuple2; import java.util.*; import java.util.stream.Collectors; import java.util.stream.IntStream; /** * Struct type which contains one or more columns. * * @since 3.0.0 */ @Evolving public final class StructType extends DataType { private final Map> nameToFieldAndOrdinal; private final List fields; private final List fieldNames; public StructType() { this(new ArrayList<>()); } public StructType(List fields) { // Extract all nested fields and ensure that they do not contain metadata columns validateNoMetadataColumns( fields.stream() .filter(f -> !(f.getDataType() instanceof BasePrimitiveType)) .collect(Collectors.toList())); // Ensure that there are no duplicate metadata columns at the top level Set seenMetadataCols = new HashSet<>(); for (StructField field : fields) { if (field.isMetadataColumn()) { MetadataColumnSpec colType = field.getMetadataColumnSpec(); if (seenMetadataCols.contains(colType)) { throw new IllegalArgumentException( String.format("Duplicate metadata column %s found in struct type", colType)); } seenMetadataCols.add(colType); } } this.fields = fields; this.fieldNames = fields.stream().map(f -> f.getName()).collect(Collectors.toList()); this.nameToFieldAndOrdinal = new HashMap<>(); for (int i = 0; i < fields.size(); i++) { nameToFieldAndOrdinal.put(fields.get(i).getName(), new Tuple2<>(fields.get(i), i)); } } public StructType add(StructField field) { final List fieldsCopy = new ArrayList<>(fields); fieldsCopy.add(field); return new StructType(fieldsCopy); } public StructType add(String name, DataType dataType) { return add(new StructField(name, dataType, true /* nullable */)); } public StructType add(String name, DataType dataType, boolean nullable) { return add(new StructField(name, dataType, nullable)); } public StructType add(String name, DataType dataType, FieldMetadata metadata) { return add(new StructField(name, dataType, true /* nullable */, metadata)); } public StructType add(String name, DataType dataType, boolean nullable, FieldMetadata metadata) { return add(new StructField(name, dataType, nullable, metadata)); } /** Add a predefined metadata column of {@link MetadataColumnSpec} to the struct type. */ public StructType addMetadataColumn(String name, MetadataColumnSpec colType) { return add(StructField.createMetadataColumn(name, colType)); } /** @return array of fields */ public List fields() { return Collections.unmodifiableList(fields); } /** @return array of field names */ public List fieldNames() { return fieldNames; } /** @return the number of fields */ public int length() { return fields.size(); } /** @return the index of the field with the given name, or -1 if not found */ public int indexOf(String fieldName) { Tuple2 fieldAndOrdinal = nameToFieldAndOrdinal.get(fieldName); return fieldAndOrdinal != null ? fieldAndOrdinal._2 : -1; } /** @return the index of the metadata column of the given spec, or -1 if not found */ public int indexOf(MetadataColumnSpec spec) { // We only allow each metadata column type to appear at most once in the schema and only at top // level (i.e., not nested). for (int i = 0; i < fields.size(); i++) { if (spec.equals(fields.get(i).getMetadataColumnSpec())) { return i; } } return -1; // Not found } /** @return true if the struct type contains a metadata column of the given spec */ public boolean contains(MetadataColumnSpec spec) { return indexOf(spec) >= 0; } public StructField get(String fieldName) { return nameToFieldAndOrdinal.get(fieldName)._1; } public StructField at(int index) { return fields.get(index); } /** * Creates a {@link Column} expression for the field at the given {@code ordinal} * * @param ordinal the ordinal of the {@link StructField} to create a column for * @return a {@link Column} expression for the {@link StructField} with ordinal {@code ordinal} */ public Column column(int ordinal) { final StructField field = at(ordinal); return new Column(field.getName()); } /** * Convert the struct type to Delta protocol specified serialization format. * * @return serialized in JSON format. */ public String toJson() { return DataTypeJsonSerDe.serializeStructType(this); } @Override public boolean equivalent(DataType dataType) { if (!(dataType instanceof StructType)) { return false; } StructType otherType = ((StructType) dataType); return otherType.length() == length() && IntStream.range(0, length()) .mapToObj(i -> otherType.at(i).getDataType().equivalent(at(i).getDataType())) .allMatch(result -> result); } /** * Checks whether the given {@code dataType} is compatible with this type when writing data. * Collation differences are ignored. * *

This method is intended to be used during the write path to validate that an input type * matches the expected schema before data is written. * *

It should not be used in other cases, such as the read path. * * @param dataType the input data type being written * @return {@code true} if the input type is compatible with this type. */ @Override public boolean isWriteCompatible(DataType dataType) { if (this == dataType) { return true; } if (dataType == null || getClass() != dataType.getClass()) { return false; } StructType structType = (StructType) dataType; return this.length() == structType.length() && fieldNames.equals(structType.fieldNames) && IntStream.range(0, this.length()) .mapToObj( i -> { StructField thisField = this.at(i); StructField otherField = structType.at(i); return (thisField == null && otherField == null) || (thisField != null && thisField.isWriteCompatible(otherField)); }) .allMatch(result -> result); } @Override public boolean isNested() { return true; } @Override public String toString() { return String.format( "struct(%s)", fields.stream().map(StructField::toString).collect(Collectors.joining(", "))); } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } StructType that = (StructType) o; return nameToFieldAndOrdinal.equals(that.nameToFieldAndOrdinal) && fields.equals(that.fields) && fieldNames.equals(that.fieldNames); } @Override public int hashCode() { return Objects.hash(nameToFieldAndOrdinal, fields, fieldNames); } /** * Validates that there are no metadata columns in a list of StructFields. * * @param fields The list of fields to validate * @throws IllegalArgumentException if any nested metadata columns are found */ private void validateNoMetadataColumns(List fields) { for (StructField field : fields) { DataType dataType = field.getDataType(); if (dataType instanceof StructType) { StructType structType = (StructType) dataType; // We filter out nested StructTypes since they have already been validated at their creation validateNoMetadataColumns( structType.fields().stream() .filter(f -> !(f.getDataType() instanceof StructType)) .collect(Collectors.toList())); } else if (dataType instanceof MapType) { MapType mapType = (MapType) dataType; validateNoMetadataColumns(Arrays.asList(mapType.getKeyField(), mapType.getValueField())); } else if (dataType instanceof ArrayType) { ArrayType arrayType = (ArrayType) dataType; validateNoMetadataColumns(Collections.singletonList(arrayType.getElementField())); } else if (field.isMetadataColumn()) { throw new IllegalArgumentException( "Metadata columns are only allowed at the top level of a schema."); } } } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/TimestampNTZType.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import io.delta.kernel.annotation.Evolving; /** * The timestamp without time zone type represents a local time in microsecond precision, which is * independent of time zone. Its valid range is [0001-01-01T00:00:00.000000, * 9999-12-31T23:59:59.999999]. To represent an absolute point in time, use {@link TimestampType} * instead. * * @since 3.2.0 */ @Evolving public class TimestampNTZType extends BasePrimitiveType { public static final TimestampNTZType TIMESTAMP_NTZ = new TimestampNTZType(); private TimestampNTZType() { super("timestamp_ntz"); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/TimestampType.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import io.delta.kernel.annotation.Evolving; /** * A timestamp type, supporting [0001-01-01T00:00:00.000000Z, 9999-12-31T23:59:59.999999Z] where the * left/right-bound is a date and time of the proleptic Gregorian calendar in UTC+00:00. Internally, * this is represented as the number of microseconds since the Unix epoch, 1970-01-01 00:00:00 UTC. * *

Due to historical reasons timestamp partition columns do not store timezone information. * Kernel interprets all timestamp partition columns in UTC. * * @since 3.0.0 */ @Evolving public class TimestampType extends BasePrimitiveType { public static final TimestampType TIMESTAMP = new TimestampType(); private TimestampType() { super("timestamp"); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/TypeChange.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import java.util.Objects; /** * Represents a type change for a field, containing the original and new primitive types. * *

Type changes are actually persisted in metadata attached to StructFields but the rules for * where the metadata is attached depend on if the change is for nested arrays/maps or primitive * types. */ public class TypeChange { private final DataType from; private final DataType to; public TypeChange(DataType from, DataType to) { this.from = Objects.requireNonNull(from, "from type cannot be null"); this.to = Objects.requireNonNull(to, "to type cannot be null"); } public DataType getFrom() { return from; } public DataType getTo() { return to; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } TypeChange that = (TypeChange) o; return Objects.equals(from, that.from) && Objects.equals(to, that.to); } @Override public int hashCode() { return Objects.hash(from, to); } @Override public String toString() { return String.format("TypeChange(from=%s,to=%s)", from, to); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/VariantType.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types; import io.delta.kernel.annotation.Evolving; /** * A logical variant type. * *

The RFC for the variant data type can be found at * https://github.com/delta-io/delta/blob/master/protocol_rfcs/variant-type.md. * * @since 3.3.0 */ @Evolving public class VariantType extends BasePrimitiveType { public static final VariantType VARIANT = new VariantType(); private VariantType() { super("variant"); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/types/package-info.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /** * Data types defined by the Delta Kernel to exchange the type info between the Delta Kernel and the * connectors. */ package io.delta.kernel.types; ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/utils/CloseableIterable.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.utils; import static io.delta.kernel.internal.util.Utils.toCloseableIterator; import io.delta.kernel.exceptions.KernelException; import java.io.Closeable; import java.io.IOException; import java.util.*; import java.util.function.Consumer; /** * Extend the Java {@link Iterable} interface to provide a way to close the iterator. * * @param the type of elements returned by this iterator */ public interface CloseableIterable extends Iterable, Closeable { /** * Overrides the default iterator method to return a {@link CloseableIterator}. * * @return a {@link CloseableIterator} instance. */ @Override CloseableIterator iterator(); @Override default void forEach(Consumer action) { try (CloseableIterator iterator = iterator()) { iterator.forEachRemaining(action); } catch (Exception e) { throw new RuntimeException(e); } } @Override default Spliterator spliterator() { // We need a way to close the iterator and is not used in Kernel, so for now // make the default implementation throw an exception. throw new UnsupportedOperationException("spliterator is not supported"); } /** * Return an {@link CloseableIterable} object that is backed by an in-memory collection of given * {@link CloseableIterator}. Users should aware that the returned {@link CloseableIterable} will * hold the data in memory. * * @param iterator the iterator to be converted to a {@link CloseableIterable}. It will be closed * by this callee. * @param the type of elements returned by the iterator * @return a {@link CloseableIterable} instance. */ static CloseableIterable inMemoryIterable(CloseableIterator iterator) { final ArrayList elements = new ArrayList<>(); try (CloseableIterator iter = iterator) { while (iter.hasNext()) { elements.add(iter.next()); } } catch (Exception e) { // TODO: we may need utility methods to throw the KernelException as is // without wrapping in RuntimeException. if (e instanceof KernelException) { throw (KernelException) e; } else { throw new RuntimeException(e); } } return new CloseableIterable() { @Override public void close() throws IOException { // nothing to close } @Override public CloseableIterator iterator() { return toCloseableIterator(elements.iterator()); } }; } /** * Return an {@link CloseableIterable} object for an empty collection. * * @return a {@link CloseableIterable} instance. * @param the type of elements returned by the iterator */ static CloseableIterable emptyIterable() { final CloseableIterator EMPTY_ITERATOR = toCloseableIterator(Collections.emptyList().iterator()); return new CloseableIterable() { @Override public void close() throws IOException { // nothing to close } @Override public CloseableIterator iterator() { return EMPTY_ITERATOR; } }; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/utils/CloseableIterator.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.utils; import io.delta.kernel.annotation.Evolving; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.KernelEngineException; import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.internal.util.Utils; import java.io.Closeable; import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; import java.util.function.Function; /** * Closeable extension of {@link Iterator} * * @param the type of elements returned by this iterator * @since 3.0.0 */ @Evolving public interface CloseableIterator extends Iterator, Closeable { /** * Represents the result of applying the filter condition in the {@link * #breakableFilter(Function)} method of a {@link CloseableIterator}. This enum determines how * each element in the iterator should be handled. */ enum BreakableFilterResult { /** * Indicates that the current element should be included in the resulting iterator produced by * {@link #breakableFilter(Function)}. */ INCLUDE, /** * Indicates that the current element should be excluded from the resulting iterator produced by * {@link #breakableFilter(Function)}. */ EXCLUDE, /** * Indicates that the iteration should stop immediately and that no further elements should be * processed by {@link #breakableFilter(Function)}. */ BREAK } /** * Returns true if the iteration has more elements. (In other words, returns true if next would * return an element rather than throwing an exception.) * * @return true if the iteration has more elements * @throws KernelEngineException For any underlying exception occurs in {@link Engine} while * trying to execute the operation. The original exception is (if any) wrapped in this * exception as cause. E.g. {@link IOException} thrown while trying to read from a Delta log * file. It will be wrapped in this exception as cause. * @throws KernelException When encountered an operation or state that is invalid or unsupported. */ @Override boolean hasNext(); /** * Returns the next element in the iteration. * * @return the next element in the iteration * @throws NoSuchElementException if the iteration has no more elements * @throws KernelEngineException For any underlying exception occurs in {@link Engine} while * trying to execute the operation. The original exception is (if any) wrapped in this * exception as cause. E.g. {@link IOException} thrown while trying to read from a Delta log * file. It will be wrapped in this exception as cause. * @throws KernelException When encountered an operation or state that is invalid or unsupported * in Kernel. For example, trying to read from a Delta table that has advanced features which * are not yet supported by Kernel. */ @Override T next(); default CloseableIterator map(Function mapper) { CloseableIterator delegate = this; return new CloseableIterator() { @Override public void remove() { delegate.remove(); } @Override public boolean hasNext() { return delegate.hasNext(); } @Override public U next() { return mapper.apply(delegate.next()); } @Override public void close() throws IOException { delegate.close(); } }; } /** * Returns a new {@link CloseableIterator} that applies a function to each element of this * iterator, where each element is transformed into another iterator, and the results are * flattened into a single iterator. * *

Example: * * 

{@code
   * // [1, 2, 3].flatMap(x -> [x, x]) => [1, 1, 2, 2, 3, 3]
   * iterator.flatMap(commit -> processCommit(commit))
   * }
* * @param mapper A function that transforms each element into a {@link CloseableIterator} * @param The type of elements in the resulting iterator * @return A flattened {@link CloseableIterator} over all elements from all inner iterators */ default CloseableIterator flatMap(Function> mapper) { CloseableIterator delegate = this; return new CloseableIterator() { private CloseableIterator currentInner = null; @Override public boolean hasNext() { while (true) { if (currentInner != null && currentInner.hasNext()) { return true; } if (currentInner != null) { Utils.closeCloseables(currentInner); currentInner = null; } if (!delegate.hasNext()) { return false; } currentInner = mapper.apply(delegate.next()); } } @Override public U next() { if (!hasNext()) { throw new NoSuchElementException(); } return currentInner.next(); } @Override public void close() throws IOException { Utils.closeCloseables(currentInner, delegate); currentInner = null; } }; } /** * Returns a new {@link CloseableIterator} that includes only the elements of this iterator for * which the given {@code mapper} function returns {@code true}. * * @param mapper A function that determines whether an element should be included in the resulting * iterator. * @return A {@link CloseableIterator} that includes only the filtered the elements of this * iterator. */ default CloseableIterator filter(Function mapper) { return breakableFilter( t -> { if (mapper.apply(t)) { return BreakableFilterResult.INCLUDE; } else { return BreakableFilterResult.EXCLUDE; } }); } /** * Returns a new {@link CloseableIterator} that includes elements from this iterator as long as * the given {@code mapper} function returns {@code true}. Once the mapper function returns {@code * false}, the iteration is terminated. * * @param mapper A function that determines whether to include an element in the resulting * iterator. * @return A {@link CloseableIterator} that stops iteration when the condition is not met. */ default CloseableIterator takeWhile(Function mapper) { return breakableFilter( t -> { if (mapper.apply(t)) { return BreakableFilterResult.INCLUDE; } else { return BreakableFilterResult.BREAK; } }); } /** * Returns a new {@link CloseableIterator} that applies a {@link BreakableFilterResult}-based * filtering function to determine whether elements of this iterator should be included or * excluded, or whether the iteration should terminate. * * @param mapper A function that determines the filtering action for each element: include, * exclude, or break. * @return A {@link CloseableIterator} that applies the specified {@link * BreakableFilterResult}-based logic. */ default CloseableIterator breakableFilter(Function mapper) { CloseableIterator delegate = this; return new CloseableIterator() { T next; boolean hasLoadedNext; boolean shouldBreak = false; @Override public boolean hasNext() { if (shouldBreak) { return false; } if (hasLoadedNext) { return true; } while (delegate.hasNext()) { final T potentialNext = delegate.next(); final BreakableFilterResult result = mapper.apply(potentialNext); if (result == BreakableFilterResult.INCLUDE) { next = potentialNext; hasLoadedNext = true; return true; } else if (result == BreakableFilterResult.BREAK) { shouldBreak = true; return false; } } return false; } @Override public T next() { if (!hasNext()) { throw new NoSuchElementException(); } hasLoadedNext = false; return next; } @Override public void close() throws IOException { delegate.close(); } }; } /** * Combine the current iterator with another iterator. The resulting iterator will return all * elements from the current iterator followed by all elements from the other iterator. * * @param other the other iterator to combine with * @return a new iterator that combines the current iterator with the other iterator */ default CloseableIterator combine(CloseableIterator other) { CloseableIterator delegate = this; return new CloseableIterator() { @Override public boolean hasNext() { return delegate.hasNext() || other.hasNext(); } @Override public T next() { if (delegate.hasNext()) { return delegate.next(); } else { return other.next(); } } @Override public void close() throws IOException { Utils.closeCloseables(delegate, other); } }; } /** * Collects all elements from this {@link CloseableIterator} into a {@link List}. * *

This method iterates through all elements of the iterator, storing them in an in-memory * list. Once iteration is complete, the iterator is automatically closed to release any * underlying resources. * * @return A {@link List} containing all elements from this iterator. * @throws UncheckedIOException If an {@link IOException} occurs while closing the iterator. */ default List toInMemoryList() { final List result = new ArrayList<>(); try (CloseableIterator iterator = this) { while (iterator.hasNext()) { result.add(iterator.next()); } } catch (IOException e) { throw new UncheckedIOException("Failed to close the CloseableIterator", e); } return result; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/utils/DataFileStatus.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.utils; import io.delta.kernel.statistics.DataFileStatistics; import java.util.Optional; /** * Extends {@link FileStatus} to include additional details such as column level statistics of the * data file in the Delta Lake table. */ public class DataFileStatus extends FileStatus { private final Optional statistics; /** * Create a new instance of {@link DataFileStatus}. * * @param path Fully qualified file path. * @param size File size in bytes. * @param modificationTime Last modification time of the file in epoch milliseconds. * @param statistics Optional column and file level statistics in the data file. */ public DataFileStatus( String path, long size, long modificationTime, Optional statistics) { super(path, size, modificationTime); this.statistics = statistics; } /** * Get the statistics of the data file encapsulated in this object. * * @return Statistics of the file. */ public Optional getStatistics() { return statistics; } @Override public String toString() { return "DataFileStatus{" + "path='" + getPath() + '\'' + ", size=" + getSize() + ", modificationTime=" + getModificationTime() + ", statistics=" + statistics + '}'; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/utils/FileStatus.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.utils; import io.delta.kernel.annotation.Evolving; import java.util.Objects; /** * Class for encapsulating metadata about a file in Delta Lake table. * * @since 3.0.0 */ @Evolving public class FileStatus { ////////////////////////////////// // Static variables and methods // ////////////////////////////////// /** * Create a {@link FileStatus} with the given path, size and modification time. * * @param path Fully qualified file path. * @param size File size in bytes * @param modificationTime Modification time of the file in epoch millis */ public static FileStatus of(String path, long size, long modificationTime) { return new FileStatus(path, size, modificationTime); } ////////////////////////////////// // Member variables and methods // ////////////////////////////////// private final String path; private final long size; private final long modificationTime; // TODO add further documentation about the expected format for modificationTime? protected FileStatus(String path, long size, long modificationTime) { this.path = Objects.requireNonNull(path, "path is null"); this.size = size; // TODO: validation this.modificationTime = modificationTime; // TODO: validation } /** * Get the path to the file. * * @return Fully qualified file path */ public String getPath() { return path; } /** * Get the size of the file in bytes. * * @return File size in bytes. */ public long getSize() { return size; } /** * Get the modification time of the file in epoch millis. * * @return Modification time in epoch millis */ public long getModificationTime() { return modificationTime; } @Override public String toString() { return String.format( "FileStatus{path='%s', size=%d, modificationTime=%d}", path, size, modificationTime); } /** * Create a {@link FileStatus} with the given path with size and modification time set to 0. * * @param path Fully qualified file path. * @return {@link FileStatus} object */ public static FileStatus of(String path) { return new FileStatus(path, 0 /* size */, 0 /* modTime */); } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } FileStatus that = (FileStatus) o; return Objects.equals(this.path, that.path) && Objects.equals(this.size, that.size) && Objects.equals(this.modificationTime, that.modificationTime); } @Override public int hashCode() { return Objects.hash(path, size, modificationTime); } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/utils/PartitionUtils.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.utils; import static java.util.Objects.requireNonNull; import io.delta.kernel.Scan; import io.delta.kernel.Snapshot; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; import io.delta.kernel.expressions.Predicate; import java.io.IOException; import java.io.UncheckedIOException; import java.util.HashSet; import java.util.Set; public class PartitionUtils { private PartitionUtils() {} /** * Check if a partition exists (i.e. actually has data) in the given {@link Snapshot} based on the * given {@link Predicate}. * * @param engine the {@link Engine} to use for scanning the partition. * @param snapshot the {@link Snapshot} to scan. * @param partitionPredicate the {@link Predicate} to use for filtering the partition. * @return true if the partition exists, false otherwise. * @throws IllegalArgumentException if the predicate does not reference any partition columns or * if it references any data columns */ public static boolean partitionExists( Engine engine, Snapshot snapshot, Predicate partitionPredicate) { requireNonNull(engine, "engine is null"); requireNonNull(snapshot, "snapshot is null"); requireNonNull(partitionPredicate, "partitionPredicate is null"); final Set snapshotPartColNames = new HashSet<>(snapshot.getPartitionColumnNames()); io.delta.kernel.internal.util.PartitionUtils.validatePredicateOnlyOnPartitionColumns( partitionPredicate, snapshotPartColNames); final Scan scan = snapshot.getScanBuilder().withFilter(partitionPredicate).build(); try (CloseableIterator columnarBatchIter = scan.getScanFiles(engine)) { while (columnarBatchIter.hasNext()) { try (CloseableIterator selectedRowsIter = columnarBatchIter.next().getRows()) { if (selectedRowsIter.hasNext()) { return true; } } } } catch (IOException e) { throw new UncheckedIOException(e); } return false; } } ================================================ FILE: kernel/kernel-api/src/main/java/io/delta/kernel/utils/package-info.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /** Utilities. */ package io.delta.kernel.utils; ================================================ FILE: kernel/kernel-api/src/test/resources/log4j2.properties ================================================ # # Copyright (2025) The Delta Lake Project Authors. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Set everything to be logged to the file target/unit-tests.log rootLogger.level = warn rootLogger.appenderRef.file.ref = ${sys:test.appender:-File} appender.file.type = File appender.file.name = File appender.file.fileName = target/unit-tests.log appender.file.append = true appender.file.layout.type = PatternLayout appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n # Tests that launch java subprocesses can set the "test.appender" system property to # "console" to avoid having the child process's logs overwrite the unit test's # log file. appender.console.type = Console appender.console.name = console appender.console.target = SYSTEM_ERR appender.console.layout.type = PatternLayout appender.console.layout.pattern = %t: %m%n ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/CloseableIteratorSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel import scala.collection.JavaConverters._ import scala.util.Using import io.delta.kernel.internal.util.Utils import io.delta.kernel.utils.CloseableIterator import io.delta.kernel.utils.CloseableIterator.BreakableFilterResult import org.scalatest.funsuite.AnyFunSuite class CloseableIteratorSuite extends AnyFunSuite { private def toCloseableIter[T](elems: Seq[T]): CloseableIterator[T] = { Utils.toCloseableIterator(elems.iterator.asJava) } private def toList[T](iter: CloseableIterator[T]): List[T] = { iter.toInMemoryList.asScala.toList } private def normalDataIter = toCloseableIter(Seq(1, 2, 3, 4, 5)) private def throwingDataIter = toCloseableIter(Seq(1, 2, 3, 4, 5)).map { x => if (x > 4) { throw new RuntimeException("Underlying data evaluated at element > 4") } x } /** * A CloseableIterator wrapper that tracks whether close() was called. * Used for testing resource cleanup. */ private class TrackingCloseableIterator( elems: Seq[Int], onClose: () => Unit) extends CloseableIterator[Int] { private val iter = elems.iterator private var closed = false override def hasNext(): Boolean = { assert(!closed) iter.hasNext } override def next(): Int = iter.next() override def close(): Unit = { if (!closed) { onClose() closed = true } } } test("CloseableIterator::filter -- returns filtered result") { val result = normalDataIter.filter(x => x <= 3 || x == 5) assert(toList(result) === List(1, 2, 3, 5)) } test("CloseableIterator::filter -- iterates over all elements") { intercept[RuntimeException] { toList(throwingDataIter.filter(x => x <= 3)) } } test("CloseableIterator::takeWhile -- stops iteration at first false condition") { // we expect it to evaluate 1, 2, 3, 4; break when it sees x == 4; and only return 1, 2, 3 val result = throwingDataIter.takeWhile(x => x <= 3) assert(toList(result) === List(1, 2, 3)) } test("CloseableIterator::breakableFilter -- correctly filters and breaks iteration") { val result = throwingDataIter.breakableFilter { x => if (x <= 1 || x == 3) { BreakableFilterResult.INCLUDE } else if (x == 2) { BreakableFilterResult.EXCLUDE } else if (x == 4) { BreakableFilterResult.BREAK } else { throw new RuntimeException("This should never be reached") } } // we except it to include 1; exclude 2; include 3; and break at 4, thus never seeing 5 assert(toList(result) === List(1, 3)) } test("flatten -- flattens nested iterators") { // Create an iterator of iterators val nestedIter = toCloseableIter( Seq( toCloseableIter(Seq(1, 2)), toCloseableIter(Seq(3, 4, 5)), toCloseableIter(Seq(6)))) val result = Utils.flatten(nestedIter) assert(toList(result) === List(1, 2, 3, 4, 5, 6)) } test("flatten -- handles empty inner iterators") { val nestedIter = toCloseableIter( Seq( toCloseableIter(Seq(1, 2)), toCloseableIter(Seq[Int]()), toCloseableIter(Seq(3, 4)), toCloseableIter(Seq[Int]()), toCloseableIter(Seq(5)))) val result = Utils.flatten(nestedIter) assert(toList(result) === List(1, 2, 3, 4, 5)) } test("flatten -- handles empty outer iterator") { val nestedIter = toCloseableIter(Seq[CloseableIterator[Int]]()) val result = Utils.flatten(nestedIter) assert(toList(result) === List()) } test("flatten -- properly closes inner iterators") { var innerClosedCount = 0 var outerClosed = false val nestedIter = new CloseableIterator[CloseableIterator[Int]] { private val iter = Seq( new TrackingCloseableIterator(Seq(1, 2), () => innerClosedCount += 1), new TrackingCloseableIterator(Seq(3, 4), () => innerClosedCount += 1)).iterator override def hasNext(): Boolean = iter.hasNext override def next(): CloseableIterator[Int] = iter.next() override def close(): Unit = { outerClosed = true } } val result = Utils.flatten(nestedIter) // Consume the iterator fully toList(result) // All inner iterators should have been closed (2 inner iterators) assert(innerClosedCount === 2) // Outer iterator should also be closed assert(outerClosed === true) } test("flatten -- closes iterators even when not fully consumed") { var innerClosedCount = 0 var outerClosed = false val nestedIter = new CloseableIterator[CloseableIterator[Int]] { private val iter = Seq( new TrackingCloseableIterator(Seq(1, 2), () => innerClosedCount += 1), new TrackingCloseableIterator(Seq(3, 4), () => innerClosedCount += 1), new TrackingCloseableIterator(Seq(5, 6), () => innerClosedCount += 1)).iterator override def hasNext(): Boolean = iter.hasNext override def next(): CloseableIterator[Int] = iter.next() override def close(): Unit = { outerClosed = true } } val result = Utils.flatten(nestedIter) // Only consume first 3 elements (from first 2 inner iterators) assert(result.hasNext === true) assert(result.next() === 1) assert(result.next() === 2) assert(result.next() === 3) // Explicitly close without consuming all result.close() // First two are closed. assert(innerClosedCount === 2) assert(outerClosed === true) } test("flatten -- handles exception during iteration and cleans up") { var innerClosedCount = 0 var outerClosed = false val nestedIter = new CloseableIterator[CloseableIterator[Int]] { private var count = 0 override def hasNext(): Boolean = count < 3 override def next(): CloseableIterator[Int] = { count += 1 if (count == 2) { throw new RuntimeException("Test exception during next()") } new TrackingCloseableIterator(Seq(1, 2), () => innerClosedCount += 1) } override def close(): Unit = { outerClosed = true } } val result = Utils.flatten(nestedIter) // Consume first inner iterator completely assert(result.hasNext === true) assert(result.next() === 1) assert(result.next() === 2) // This should trigger the exception when trying to get the next inner iterator val exception = intercept[RuntimeException] { result.hasNext } assert(exception.getMessage === "Test exception during next()") // Verify that the outer iterator was closed due to exception assert(outerClosed === true) } test("iteratorLast -- returns empty for empty iterator") { val result = Utils.iteratorLast(toCloseableIter(Seq[Int]())) assert(!result.isPresent) } test("iteratorLast -- returns last element for single element iterator") { val result = Utils.iteratorLast(toCloseableIter(Seq(42))) assert(result.isPresent) assert(result.get() === 42) } test("iteratorLast -- returns last element for multiple element iterator") { val result = Utils.iteratorLast(toCloseableIter(Seq(1, 2, 3, 4, 5))) assert(result.isPresent) assert(result.get() === 5) } test("iteratorLast -- properly closes iterator after consumption") { var closed = false val iter = new TrackingCloseableIterator(Seq(1, 2, 3), () => closed = true) val result = Utils.iteratorLast(iter) assert(result.isPresent) assert(result.get() === 3) assert(closed === true) } test("flatMap -- basic functionality") { val input = toCloseableIter(Seq(1, 2, 3)) val result = input.flatMap { x: Int => toCloseableIter(Seq(x, x * 10)) } assert(toList(result) === List(1, 10, 2, 20, 3, 30)) } test("flatMap -- properly closes all resources") { var outerClosed = false var innerClosedCount = 0 val outer = new TrackingCloseableIterator(Seq(1, 2, 3), () => outerClosed = true) val result = outer.flatMap { x: Int => new TrackingCloseableIterator( Seq(x, x * 10), () => innerClosedCount += 1): CloseableIterator[Int] } assert(toList(result) === List(1, 10, 2, 20, 3, 30)) // 3 inner iterators, one per outer element assert(innerClosedCount === 3) assert(outerClosed === true) } test("flatMap -- closes resources when mapper function throws") { var outerClosed = false val outer = new TrackingCloseableIterator(Seq(1, 2, 3), () => outerClosed = true) val result = outer.flatMap { x: Int => (throw new RuntimeException("Error in mapper")): CloseableIterator[Int] } // Use scala's equivalent of java's try-with-resources val exception = intercept[RuntimeException] { Using.resource(result) { r => r.hasNext } } assert(exception.getMessage === "Error in mapper") assert(outerClosed === true) } test("flatMap -- closes resources when inner iterator throws during consumption") { var outerClosed = false var innerClosedCount = 0 val outer = new TrackingCloseableIterator(Seq(1, 2, 3), () => outerClosed = true) val result = outer.flatMap { x: Int => (new TrackingCloseableIterator(Seq(x, x * 10), () => innerClosedCount += 1) { override def next(): Int = { val value = super.next() if (value == 20) { throw new RuntimeException("Error reading value 20") } value } }): CloseableIterator[Int] } // Use scala's equivalent of java's try-with-resources val exception = intercept[RuntimeException] { Using.resource(result) { r => // First inner iterator assert(r.next() === 1) assert(r.next() === 10) // Second inner iterator assert(r.next() === 2) // Second inner iterator -- throws on next value (20) r.next() } } assert(exception.getMessage === "Error reading value 20") assert(outerClosed === true) assert(innerClosedCount === 2) // Both inner iterators that were created should be closed } test("flatMap -- mapper returns null") { var outerClosed = false val outer = new TrackingCloseableIterator(Seq(1, 2, 3), () => outerClosed = true) val result = outer.flatMap { x: Int => if (x % 2 == 0) { null.asInstanceOf[CloseableIterator[Int]] } else { toCloseableIter(Seq(x, x * 10)) } } assert(toList(result) === List(1, 10, 3, 30)) assert(outerClosed === true) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/TransactionSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel import java.lang.{Long => JLong} import java.util import java.util.Optional import scala.collection.JavaConverters._ import io.delta.kernel.Transaction.{generateAppendActions, getWriteContext, transformLogicalData} import io.delta.kernel.data._ import io.delta.kernel.exceptions.KernelException import io.delta.kernel.expressions.{Column, Literal} import io.delta.kernel.internal.{DataWriteContextImpl, TableConfig, TransactionImpl} import io.delta.kernel.internal.TableConfig.{COLUMN_MAPPING_MODE, ICEBERG_COMPAT_V2_ENABLED, ICEBERG_COMPAT_V3_ENABLED} import io.delta.kernel.internal.actions.{Format, Metadata, Protocol} import io.delta.kernel.internal.data.TransactionStateRow import io.delta.kernel.internal.tablefeatures.TableFeatures import io.delta.kernel.internal.types.DataTypeJsonSerDe import io.delta.kernel.internal.util.Utils.toCloseableIterator import io.delta.kernel.internal.util.VectorUtils import io.delta.kernel.internal.util.VectorUtils.stringStringMapValue import io.delta.kernel.statistics.DataFileStatistics import io.delta.kernel.test.{MockEngineUtils, VectorTestUtils} import io.delta.kernel.types.{DoubleType, FloatType, IntegerType, LongType, StringType, StructField, StructType, TimestampType, VariantType} import io.delta.kernel.utils.{CloseableIterator, DataFileStatus} import org.scalatest.funsuite.AnyFunSuite class TransactionSuite extends AnyFunSuite with VectorTestUtils with MockEngineUtils { import io.delta.kernel.TransactionSuite._ def withIcebergCompatVersions(testNamePrefix: String)( body: (Boolean, Boolean) => Unit): Unit = { Seq((false, false), (true, false), (false, true)).foreach { case (v2, v3) => test(s"$testNamePrefix, icebergCompatV2Enabled=$v2 icebergCompatV3Enabled=$v3") { body(v2, v3) } } } withIcebergCompatVersions("transformLogicalData: un-partitioned table") { case (icebergCompatV2Enabled, icebergCompatV3Enabled) => val transformedDateIter = transformLogicalData( mockEngine(), testTxnState( testSchema, enableIcebergCompatV2 = icebergCompatV2Enabled, enableIcebergCompatV3 = icebergCompatV3Enabled), testData(includePartitionCols = false), Map.empty[String, Literal].asJava /* partition values */ ) transformedDateIter.map(_.getData).forEachRemaining(batch => { assert(batch.getSchema === testSchema) }) } withIcebergCompatVersions("transformLogicalData: partitioned table") { case (icebergCompatV2Enabled, icebergCompatV3Enabled) => val transformedDateIter = transformLogicalData( mockEngine(), testTxnState( testSchemaWithPartitions, testPartitionColNames, enableIcebergCompatV2 = icebergCompatV2Enabled, enableIcebergCompatV3 = icebergCompatV3Enabled), testData(includePartitionCols = true), /* partition values */ Map("state" -> Literal.ofString("CA"), "country" -> Literal.ofString("USA")).asJava) transformedDateIter.map(_.getData).forEachRemaining { batch => if (icebergCompatV2Enabled || icebergCompatV3Enabled) { // when icebergCompatV2Enabled is true, the partition columns included in the output assert(batch.getSchema === testSchemaWithPartitions) } else { assert(batch.getSchema === testSchema) } } } test("transformLogicalData: partitioned table with MaterializePartitionColumns feature") { val transformedDateIter = transformLogicalData( mockEngine(), testTxnState( testSchemaWithPartitions, testPartitionColNames, enableMaterializePartitionColumns = true), testData(includePartitionCols = true), /* partition values */ Map("state" -> Literal.ofString("CA"), "country" -> Literal.ofString("USA")).asJava) transformedDateIter.map(_.getData).forEachRemaining { batch => // when MaterializePartitionColumns feature is enabled, partition columns are included assert(batch.getSchema === testSchemaWithPartitions) } } test("transformLogicalData: partitioned table without MaterializePartitionColumns feature") { val transformedDateIter = transformLogicalData( mockEngine(), testTxnState( testSchemaWithPartitions, testPartitionColNames, enableMaterializePartitionColumns = false), testData(includePartitionCols = true), /* partition values */ Map("state" -> Literal.ofString("CA"), "country" -> Literal.ofString("USA")).asJava) transformedDateIter.map(_.getData).forEachRemaining { batch => // when MaterializePartitionColumns feature is disabled, partition columns are filtered out assert(batch.getSchema === testSchema) } } withIcebergCompatVersions("generateAppendActions: iceberg comaptibily checks") { case (icebergCompatV2Enabled, icebergCompatV3Enabled) => val txnState = testTxnState( testSchema, enableIcebergCompatV2 = icebergCompatV2Enabled, enableIcebergCompatV3 = icebergCompatV3Enabled) val engine = mockEngine() Seq( // missing stats ( testDataFileStatuses( "file1" -> testStats(Some(10)), // valid stats "file2" -> None // missing stats ), "compatibility requires 'numRecords' statistic." // expected error message )).foreach { case (actionRows, expectedErrorMsg) => if (icebergCompatV2Enabled || icebergCompatV3Enabled) { val ex = intercept[KernelException] { generateAppendActions(engine, txnState, actionRows, testDataWriteContext()) .forEachRemaining(_ => ()) // consume the iterator } assert(ex.getMessage.contains(expectedErrorMsg)) } else { // when icebergCompatV2Enabled is disabled, no exception should be thrown generateAppendActions(engine, txnState, actionRows, testDataWriteContext()) .forEachRemaining(_ => ()) // consume the iterator } } // valid stats val dataFileStatuses = testDataFileStatuses( "file1" -> testStats(Some(10)), "file2" -> testStats(Some(20))) var actStats: Seq[String] = Seq.empty generateAppendActions(engine, txnState, dataFileStatuses, testDataWriteContext()) .forEachRemaining { addActionRow => val addOrdinal = addActionRow.getSchema.indexOf("add") val add = addActionRow.getStruct(addOrdinal) val statsOrdinal = add.getSchema.indexOf("stats") actStats = actStats :+ add.getString(statsOrdinal) } assert(actStats === Seq( "{\"numRecords\":10,\"minValues\":{},\"maxValues\":{},\"nullCount\":{}}", "{\"numRecords\":20,\"minValues\":{},\"maxValues\":{},\"nullCount\":{}}")) } Seq(0, -1).foreach { numIndexedCols => test(s"stats: validate DATA_SKIPPING_NUM_INDEXED_COLS limit" + s" is respected when set to: $numIndexedCols") { // Create schema with simple and nested columns val schema = new StructType() .add("id", IntegerType.INTEGER) .add("name", StringType.STRING) .add( "metrics", new StructType() .add("temperature", DoubleType.DOUBLE) .add("humidity", FloatType.FLOAT)) .add("timestamp", TimestampType.TIMESTAMP) // Create transaction state with specified numIndexedCols val configMap = Map(TableConfig .DATA_SKIPPING_NUM_INDEXED_COLS.getKey -> numIndexedCols.toString) val metadata = new Metadata( "id", Optional.empty(), Optional.empty(), new Format(), DataTypeJsonSerDe.serializeDataType(schema), schema, VectorUtils.buildArrayValue(Seq.empty.asJava, StringType.STRING), Optional.empty(), stringStringMapValue(configMap.asJava)) val protocol = new Protocol(1, 1) // simple protocol for this test val txnState = TransactionStateRow.of( metadata, protocol, "table path", 200 /* maxRetries */ ) // Get statistics columns and define expected result val statsColumns = TransactionImpl.getStatisticsColumns(txnState) if (numIndexedCols == -1) { // For -1, all leaf columns should be included val expectedColumns = Set( new Column("id"), new Column("name"), new Column(Array("metrics", "temperature")), new Column(Array("metrics", "humidity")), new Column("timestamp")) assert( statsColumns.size == 5, s"With numIndexedCols=$numIndexedCols, expected 5 columns but got ${statsColumns.size}") // Verify the expected columns are present val statsColumnsSet = statsColumns.asScala.toSet assert(statsColumnsSet == expectedColumns, s"Expected columns do not match actual columns") } else if (numIndexedCols == 0) { // For 0, no columns should be included assert( statsColumns.isEmpty, s"With numIndexedCols=$numIndexedCols," + s" expected no columns but got ${statsColumns.size} columns") } } } Seq("name", "id").foreach { cmMode => test(s"transformLogicalData: CM tables are blocked: cmMode=$cmMode") { val txnState = testTxnState(new StructType(), cmMode = cmMode) val engine = mockEngine() val ex = intercept[UnsupportedOperationException] { transformLogicalData( engine, txnState, testData(includePartitionCols = false), Map.empty[String, Literal].asJava /* partition values */ ) .forEachRemaining(_ => ()) // consume the iterator } assert(ex.getMessage.contains( "Writing into column mapping enabled table is not supported yet.")) } } Seq("name", "id").foreach { cmMode => test(s"getWriteContext: CM tables are blocked: $cmMode") { val txnState = testTxnState(new StructType(), cmMode = cmMode) val engine = mockEngine() val ex = intercept[UnsupportedOperationException] { getWriteContext( engine, txnState, Map.empty[String, Literal].asJava /* partition values */ ) } assert(ex.getMessage.contains( "Writing into column mapping enabled table is not supported yet.")) } } test("transformLogicalData: Writing to tables with variant is blocked") { val txnState = testTxnState(new StructType().add("variant", VariantType.VARIANT)) val engine = mockEngine() val ex = intercept[UnsupportedOperationException] { transformLogicalData( engine, txnState, testData(includePartitionCols = false), Map.empty[String, Literal].asJava /* partition values */ ) .forEachRemaining(_ => ()) // consume the iterator } assert(ex.getMessage.contains( "Transforming logical data with variant data is currently unsupported")) } test("transformLogicalData: Writing to tables with nested variant is blocked") { val txnState = testTxnState(new StructType().add( "nested", new StructType().add("nested_variant", VariantType.VARIANT))) val engine = mockEngine() val ex = intercept[UnsupportedOperationException] { transformLogicalData( engine, txnState, testData(includePartitionCols = false), Map.empty[String, Literal].asJava /* partition values */ ) .forEachRemaining(_ => ()) // consume the iterator } assert(ex.getMessage.contains( "Transforming logical data with variant data is currently unsupported")) } } object TransactionSuite extends VectorTestUtils with MockEngineUtils { def testData(includePartitionCols: Boolean): CloseableIterator[FilteredColumnarBatch] = { toCloseableIterator( Seq.range(0, 5).map(_ => testBatch(includePartitionCols)).asJava.iterator()).map(batch => new FilteredColumnarBatch(batch, Optional.empty())) } def testBatch(includePartitionCols: Boolean): ColumnarBatch = { val testColumnVectors = Seq( stringVector(Seq("Alice", "Bob", "Charlie", "David", "Eve")), // name longVector(Seq(20L, 30L, 40L, 50L, 60L)), // id stringVector(Seq("Campbell", "Roanoke", "Dallas", "Monte Sereno", "Minneapolis")) // city ) ++ { if (includePartitionCols) { Seq( stringVector(Seq("CA", "TX", "NC", "CA", "MN")), // state stringVector(Seq("USA", "USA", "USA", "USA", "USA")) // country ) } else Seq.empty } columnarBatch( schema = if (includePartitionCols) testSchemaWithPartitions else testSchema, testColumnVectors) } val testSchema: StructType = new StructType() .add("name", StringType.STRING) .add("id", LongType.LONG) .add("city", StringType.STRING) val testSchemaWithPartitions: StructType = new StructType(testSchema.fields()) .add("state", StringType.STRING) // partition column .add("country", StringType.STRING) // partition column val testPartitionColNames = Seq("state", "country") def columnarBatch(schema: StructType, vectors: Seq[ColumnVector]): ColumnarBatch = { new ColumnarBatch { override def getSchema: StructType = schema override def getColumnVector(ordinal: Int): ColumnVector = vectors(ordinal) override def withDeletedColumnAt(ordinal: Int): ColumnarBatch = { // Update the schema val newStructFields = new util.ArrayList(schema.fields) newStructFields.remove(ordinal) val newSchema: StructType = new StructType(newStructFields) // Update the vectors val newColumnVectors = vectors.toBuffer newColumnVectors.remove(ordinal) columnarBatch(newSchema, newColumnVectors.toSeq) } override def withNewColumn( ordinal: Int, columnSchema: StructField, columnVector: ColumnVector): ColumnarBatch = { // Update the schema val newStructFields = new util.ArrayList(schema.fields) newStructFields.add(ordinal, columnSchema) val newSchema: StructType = new StructType(newStructFields) // Update the vectors val newColumnVectors = vectors.toBuffer newColumnVectors.insert(ordinal, columnVector) columnarBatch(newSchema, newColumnVectors.toSeq) } override def getSize: Int = vectors.head.getSize } } def testTxnState( schema: StructType, partitionCols: Seq[String] = Seq.empty, cmMode: String = "none", enableIcebergCompatV2: Boolean = false, enableIcebergCompatV3: Boolean = false, enableMaterializePartitionColumns: Boolean = false): Row = { val configurationMap = Map( ICEBERG_COMPAT_V2_ENABLED.getKey -> enableIcebergCompatV2.toString, ICEBERG_COMPAT_V3_ENABLED.getKey -> enableIcebergCompatV3.toString, COLUMN_MAPPING_MODE.getKey -> cmMode) val metadata = new Metadata( "id", Optional.empty(), /* name */ Optional.empty(), /* description */ new Format(), DataTypeJsonSerDe.serializeDataType(schema), schema, VectorUtils.buildArrayValue(partitionCols.asJava, StringType.STRING), // partitionColumns Optional.empty(), // createdTime stringStringMapValue(configurationMap.asJava) // configurationMap ) // Create protocol with appropriate features val writerFeatures: java.util.Set[String] = if (enableMaterializePartitionColumns) { Set(TableFeatures.MATERIALIZE_PARTITION_COLUMNS_W_FEATURE.featureName()).asJava } else { java.util.Collections.emptySet[String]() } val protocol = new Protocol( 3, // minReaderVersion 7, // minWriterVersion to support table features java.util.Collections.emptySet[String](), // readerFeatures writerFeatures) TransactionStateRow.of(metadata, protocol, "table path", 200 /* maxRetries */ ) } def testStats(numRowsOpt: Option[Long]): Option[DataFileStatistics] = { numRowsOpt.map(numRows => { new DataFileStatistics( numRows, Map.empty[Column, Literal].asJava, // minValues - empty value as this is just for tests. Map.empty[Column, Literal].asJava, // maxValues - empty value as this is just for tests. Map.empty[Column, JLong].asJava, // nullCount - empty value as this is just for tests. Optional.empty() // tightBounds is unspecified ) }) } def testDataFileStatuses(fileNameStatsPairs: (String, Option[DataFileStatistics])*) : CloseableIterator[DataFileStatus] = { toCloseableIterator( fileNameStatsPairs.map { case (fileName, statsOpt) => new DataFileStatus( fileName, 23L, // size - arbitrary value as this is just for tests. 23L, // modificationTime - arbitrary value as this is just for tests. Optional.ofNullable(statsOpt.orNull)) }.asJava.iterator()) } /** Test [[DataWriteContext]]. As of now we don't need any custom values in this suite. */ def testDataWriteContext(): DataWriteContext = { new DataWriteContextImpl("targetDir", Map.empty[String, Literal].asJava, Seq.empty.asJava) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/commit/CatalogCommitterUtilsSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.commit import scala.collection.JavaConverters._ import io.delta.kernel.internal.actions.Protocol import org.scalatest.funsuite.AnyFunSuite class CatalogCommitterUtilsSuite extends AnyFunSuite { test("extractProtocolProperties - legacy protocol (1, 2)") { // ===== GIVEN ===== val protocol = new Protocol(1, 2) // ===== WHEN ===== val properties = CatalogCommitterUtils.extractProtocolProperties(protocol).asScala // ===== THEN ===== assert(properties.size === 2) assert(properties("delta.minReaderVersion") === "1") assert(properties("delta.minWriterVersion") === "2") } test("extractProtocolProperties - protocol with overlapping reader and writer features") { // ===== GIVEN ===== val readerFeatures = Set("columnMapping", "deletionVectors") val writerFeatures = Set("columnMapping", "appendOnly") // Note: columnMapping overlaps val protocol = new Protocol(3, 7, readerFeatures.asJava, writerFeatures.asJava) // ===== WHEN ===== val properties = CatalogCommitterUtils.extractProtocolProperties(protocol).asScala // ===== THEN ===== assert(properties.size === 2 + 3) // minReader + minWriter + 3 unique features assert(properties("delta.minReaderVersion") === "3") assert(properties("delta.minWriterVersion") === "7") assert(properties("delta.feature.columnMapping") === "supported") assert(properties("delta.feature.deletionVectors") === "supported") assert(properties("delta.feature.appendOnly") === "supported") } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/deletionvectors/Base85CodecSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.deletionvectors import java.nio.charset.StandardCharsets.US_ASCII import java.util.UUID import scala.util.Random import io.delta.kernel.internal.deletionvectors.Base85Codec import org.scalatest.funsuite.AnyFunSuite class Base85CodecSuite extends AnyFunSuite { // Z85 reference strings are generated by https://cryptii.com/pipes/z85-encoder val testUuids = Seq[(UUID, String)]( new UUID(0L, 0L) -> "00000000000000000000", new UUID(Long.MinValue, Long.MinValue) -> "Fb/MH00000Fb/MH00000", new UUID(-1L, -1L) -> "%nSc0%nSc0%nSc0%nSc0", new UUID(0L, Long.MinValue) -> "0000000000Fb/MH00000", new UUID(0L, -1L) -> "0000000000%nSc0%nSc0", new UUID(0L, Long.MaxValue) -> "0000000000Fb/MG%nSc0", new UUID(Long.MinValue, 0L) -> "Fb/MH000000000000000", new UUID(-1L, 0L) -> "%nSc0%nSc00000000000", new UUID(Long.MaxValue, 0L) -> "Fb/MG%nSc00000000000", new UUID(0L, 1L) -> "00000000000000000001", // Just a few random ones, using literals for test determinism new UUID(-4124158004264678669L, -6032951921472435211L) -> "-(5oirYA.yTvx6v@H:L>", new UUID(6453181356142382984L, 8208554093199893996L) -> "s=Mlx-0Pp@AQ6uw@k6=D", new UUID(6453181356142382984L, -8208554093199893996L) -> "s=Mlx-0Pp@JUL=R13LuL", new UUID(-4124158004264678669L, 8208554093199893996L) -> "-(5oirYA.yAQ6uw@k6=D") // From https://rfc.zeromq.org/spec/32/ - Test Case test("Z85 spec reference value") { val inputBytes: Array[Byte] = Array(0x86, 0x4F, 0xD2, 0x6F, 0xB5, 0x59, 0xF7, 0x5B).map(_.toByte) val expectedEncodedString = "HelloWorld" val actualEncodedString = Base85Codec.encodeBytes(inputBytes) assert(actualEncodedString === expectedEncodedString) val outputBytes = Base85Codec.decodeAlignedBytes(expectedEncodedString) assert(outputBytes sameElements inputBytes) } test("Z85 reference implementation values") { for ((id, expectedEncodedString) <- testUuids) { val actualEncodedString = Base85Codec.encodeUUID(id) assert(actualEncodedString === expectedEncodedString) } } test("Z85 spec character map") { assert(Base85Codec.ENCODE_MAP.length === 85) val referenceBytes = Seq( 0x00, 0x09, 0x98, 0x62, 0x0F, 0xC7, 0x99, 0x43, 0x1F, 0x85, 0x9A, 0x24, 0x2F, 0x43, 0x9B, 0x05, 0x3F, 0x01, 0x9B, 0xE6, 0x4E, 0xBF, 0x9C, 0xC7, 0x5E, 0x7D, 0x9D, 0xA8, 0x6E, 0x3B, 0x9E, 0x89, 0x7D, 0xF9, 0x9F, 0x6A, 0x8D, 0xB7, 0xA0, 0x4B, 0x9D, 0x75, 0xA1, 0x2C, 0xAD, 0x33, 0xA2, 0x0D, 0xBC, 0xF1, 0xA2, 0xEE, 0xCC, 0xAF, 0xA3, 0xCF, 0xDC, 0x6D, 0xA4, 0xB0, 0xEC, 0x2B, 0xA5, 0x91, 0xFB, 0xE9, 0xA6, 0x72) .map(_.toByte).toArray val referenceString = new String(Base85Codec.ENCODE_MAP, US_ASCII) val encodedString = Base85Codec.encodeBytes(referenceBytes) assert(encodedString === referenceString) val decodedBytes = Base85Codec.decodeAlignedBytes(encodedString) assert(decodedBytes sameElements referenceBytes) } test("Reject illegal Z85 input - unaligned string") { // Minimum string should 5 characters val illegalEncodedString = "abc" assertThrows[IllegalArgumentException] { Base85Codec.decodeBytes( illegalEncodedString, // This value is irrelevant, any value should cause the failure. 3 ) // outputLength } } // scalastyle:off nonascii test(s"Reject illegal Z85 input - illegal character") { for (char <- Seq[Char]('î', 'π', '"', 0x7F)) { val illegalEncodedString = String.valueOf(Array[Char]('a', 'b', char, 'd', 'e')) val ex = intercept[IllegalArgumentException] { Base85Codec.decodeAlignedBytes(illegalEncodedString) } assert(ex.getMessage.contains("Input is not valid Z85")) } } // scalastyle:on nonascii test("base85 codec uuid roundtrips") { for ((id, _) <- testUuids) { val encodedString = Base85Codec.encodeUUID(id) // 16 bytes always get encoded into 20 bytes with Base85. assert(encodedString.length === Base85Codec.ENCODED_UUID_LENGTH) val decodedId = Base85Codec.decodeUUID(encodedString) assert(id === decodedId, s"encodedString = $encodedString") } } test("base85 codec empty byte array") { val empty = Array.empty[Byte] val encodedString = Base85Codec.encodeBytes(empty) assert(encodedString === "") val decodedArray = Base85Codec.decodeAlignedBytes(encodedString) assert(decodedArray.isEmpty) val decodedArray2 = Base85Codec.decodeBytes(encodedString, 0) assert(decodedArray2.isEmpty) } test("base85 codec byte array random roundtrips") { val rand = new Random(1L) // Fixed seed for determinism val arrayLengths = (1 to 20) ++ Seq(32, 56, 64, 128, 1022, 11 * 1024 * 1024) for (len <- arrayLengths) { val inputArray: Array[Byte] = Array.ofDim(len) rand.nextBytes(inputArray) val encodedString = Base85Codec.encodeBytes(inputArray) val decodedArray = Base85Codec.decodeBytes(encodedString, len) assert(decodedArray === inputArray, s"encodedString = $encodedString") } } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/deletionvectors/RoaringBitmapArraySuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.deletionvectors import io.delta.kernel.internal.deletionvectors.RoaringBitmapArray import org.scalatest.funsuite.AnyFunSuite class RoaringBitmapArraySuite extends AnyFunSuite { test("RoaringBitmapArray create empty map") { val bitmap = RoaringBitmapArray.create() assert(bitmap.toArray.isEmpty) } test("RoaringBitmapArray create map with values only in first bitmap") { // Values <= max unsigned int (4,294,967,295) will be in the first bitmap val bitmap = RoaringBitmapArray.create(1L, 100L) assert(bitmap.contains(1L)) assert(bitmap.contains(100L)) assert(!bitmap.contains(2L)) assert(!bitmap.contains(99L)) assert(bitmap.toArray sameElements Array(1L, 100L)) } test("RoaringBitmapArray create map with values only in second bitmap") { // Values between max unsigned int and 2*(max unsigned int) will be in the second bitmap val bitmap = RoaringBitmapArray.create(5000000000L, 5000000100L) assert(bitmap.contains(5000000000L)) assert(bitmap.contains(5000000100L)) assert(!bitmap.contains(5000000001L)) assert(!bitmap.contains(5000000099L)) assert(bitmap.toArray sameElements Array(5000000000L, 5000000100L)) } test("RoaringBitmapArray create map with values in first and third bitmap") { // Values between 2*(max unsigned int) and 3*(max unsigned int) will be in the third bitmap val bitmap = RoaringBitmapArray.create(100L, 10000000000L) assert(bitmap.contains(100L)) assert(bitmap.contains(10000000000L)) assert(!bitmap.contains(101L)) assert(!bitmap.contains(10000000001L)) assert(bitmap.toArray sameElements Array(100L, 10000000000L)) } // TODO need to implement serialize to copy over tests /* final val BITMAP2_NUMBER = Int.MaxValue.toLong * 3L for (serializationFormat <- RoaringBitmapArrayFormat.values) { test(s"serialization - $serializationFormat") { checkSerializeDeserialize(RoaringBitmapArray.create(), serializationFormat) checkSerializeDeserialize(RoaringBitmapArray.create(1L), serializationFormat) checkSerializeDeserialize(RoaringBitmapArray.create(BITMAP2_NUMBER), serializationFormat) checkSerializeDeserialize(RoaringBitmapArray.create(1L, BITMAP2_NUMBER), serializationFormat) // checkSerializeDeserialize(allContainerTypesBitmap, serializationFormat) } } private def checkSerializeDeserialize( input: RoaringBitmapArray, format: RoaringBitmapArrayFormat.Value): Unit = { val serializedSize = Ints.checkedCast(input.serializedSizeInBytes(format)) val buffer = ByteBuffer.allocate(serializedSize).order(ByteOrder.LITTLE_ENDIAN) input.serialize(buffer, format) val output = RoaringBitmapArray() buffer.flip() output.deserialize(buffer) assert(input === output) } */ } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/exceptions/ExceptionSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.exceptions import java.util.Optional import scala.collection.JavaConverters._ import io.delta.kernel.exceptions.UnsupportedProtocolVersionException.ProtocolVersionType import io.delta.kernel.internal.DeltaErrors import org.scalatest.funsuite.AnyFunSuite /** * Unit tests for Kernel exception types. */ class ExceptionSuite extends AnyFunSuite { test("UnsupportedReaderFeatureException - basic functionality") { val tablePath = "/path/to/table" val features = Set("feature1", "feature2").asJava val ex = DeltaErrors.unsupportedReaderFeatures(tablePath, features) assert(ex.getTablePath == tablePath) assert(ex.getUnsupportedFeatures.asScala == Set("feature1", "feature2")) assert(ex.getMessage.contains("reader table features")) assert(ex.getMessage.contains("feature1")) assert(ex.getMessage.contains("feature2")) assert(ex.isInstanceOf[UnsupportedTableFeatureException]) } test("UnsupportedWriterFeatureException - basic functionality") { val tablePath = "/path/to/table" val features = Set("writerFeature").asJava val ex = DeltaErrors.unsupportedWriterFeatures(tablePath, features) assert(ex.getTablePath == tablePath) assert(ex.getUnsupportedFeatures.asScala == Set("writerFeature")) assert(ex.getMessage.contains("writer table features")) assert(ex.getMessage.contains("writerFeature")) assert(ex.isInstanceOf[UnsupportedTableFeatureException]) } test("UnsupportedProtocolVersionException - reader version") { val tablePath = "/path/to/table" val version = 3 val ex = DeltaErrors.unsupportedReaderProtocol(tablePath, version) assert(ex.getTablePath == tablePath) assert(ex.getVersion == version) assert(ex.getVersionType == ProtocolVersionType.READER) assert(ex.getMessage.contains("reader")) assert(ex.getMessage.contains("version 3")) } test("UnsupportedProtocolVersionException - writer version") { val tablePath = "/path/to/table" val version = 7 val ex = DeltaErrors.unsupportedWriterProtocol(tablePath, version) assert(ex.getTablePath == tablePath) assert(ex.getVersion == version) assert(ex.getVersionType == ProtocolVersionType.WRITER) assert(ex.getMessage.contains("writer")) assert(ex.getMessage.contains("version 7")) } test("CommitRangeNotFoundException - with start and end version") { val tablePath = "/path/to/table" val startVersion = 5L val endVersion = Optional.of(java.lang.Long.valueOf(10L)) val ex = DeltaErrors.noCommitFilesFoundForVersionRange(tablePath, startVersion, endVersion) assert(ex.getTablePath == tablePath) assert(ex.getStartVersion == startVersion) assert(ex.getEndVersion == endVersion) assert(ex.getMessage.contains("Requested table changes between [5, Optional[10]]")) assert(ex.getMessage.contains("no log files found")) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/expressions/ExpressionsSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.expressions import io.delta.kernel.types._ import org.scalatest.funsuite.AnyFunSuite class ExpressionsSuite extends AnyFunSuite { test("expressions: unsupported literal data types") { val ex1 = intercept[IllegalArgumentException] { Literal.ofNull(new ArrayType(IntegerType.INTEGER, true)) } assert(ex1.getMessage.contains("array[integer] is an invalid data type for Literal.")) val ex2 = intercept[IllegalArgumentException] { Literal.ofNull(new MapType(IntegerType.INTEGER, IntegerType.INTEGER, true)) } assert(ex2.getMessage.contains("map[integer, integer] is an invalid data type for Literal.")) val ex3 = intercept[IllegalArgumentException] { Literal.ofNull(new StructType().add("s1", BooleanType.BOOLEAN)) } assert(ex3.getMessage.matches("struct.* is an invalid data type for Literal.")) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/expressions/PredicateSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.expressions import java.util.Locale import io.delta.kernel.types.CollationIdentifier import org.scalatest.funsuite.AnyFunSuite class PredicateSuite extends AnyFunSuite { test("Check invalid collation operations") { Seq( "anD", "oR", "ELEMENT_AT", "SUBstring").foreach { operationName => val e = intercept[IllegalArgumentException] { new Predicate( operationName, new Column("c1"), new Column("c2"), CollationIdentifier.fromString("SPARK.UTF8_LCASE")) } assert(e.getMessage.contains(s"Collation is not supported for operator" + s" ${operationName.toUpperCase(Locale.ENGLISH)}.")) } } test("Check toString with collation") { Seq( ( new Predicate( "<", new Column("c1"), new Column("c2"), CollationIdentifier.fromString("SPARK.UTF8_LCASE")), "(column(`c1`) < column(`c2`) COLLATE SPARK.UTF8_LCASE)"), ( new Predicate( ">=", Literal.ofString("a"), new Column("c1"), CollationIdentifier.fromString("ICU.sr_Cyrl_SRB.75.1")), "(a >= column(`c1`) COLLATE ICU.SR_CYRL_SRB.75.1)"), ( new Predicate( "stARtS_wiTh", new Column("c1"), Literal.ofString("a"), CollationIdentifier.fromString("ICU.en_US")), "(column(`c1`) STARTS_WITH a COLLATE ICU.EN_US)")).foreach { case (predicate, expectedToString) => assert(predicate.toString == expectedToString) } } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/CommitRangeBuilderSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal import java.util.{Collections, Optional} import scala.collection.JavaConverters._ import io.delta.kernel.{CommitRange, TableManager} import io.delta.kernel.CommitRangeBuilder.CommitBoundary import io.delta.kernel.engine.Engine import io.delta.kernel.exceptions.{InvalidTableException, KernelException} import io.delta.kernel.internal.commitrange.{CommitRangeBuilderImpl, CommitRangeImpl} import io.delta.kernel.internal.files.{ParsedCatalogCommitData, ParsedDeltaData, ParsedLogData} import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.util.FileNames import io.delta.kernel.test.{MockFileSystemClientUtils, MockListFromFileSystemClient, MockReadICTFileJsonHandler, MockSnapshotUtils, VectorTestUtils} import io.delta.kernel.test.MockSnapshotUtils.getMockSnapshot import io.delta.kernel.utils.FileStatus import junit.runner.Version import org.scalatest.funsuite.AnyFunSuite class CommitRangeBuilderSuite extends AnyFunSuite with MockFileSystemClientUtils with VectorTestUtils { private def checkQueryBoundaries( commitRange: CommitRange, startBoundary: RequiredBoundaryDef, endBoundary: BoundaryDef): Unit = { def assertBoundaryVersion(boundary: CommitBoundary, version: Long) = { assert(boundary.isVersion && boundary.getVersion == version) } def assertBoundaryTimestamp(boundary: CommitBoundary, timestamp: Long) = { assert(boundary.isTimestamp && boundary.getTimestamp == timestamp) } if (startBoundary.version.nonEmpty) { assertBoundaryVersion(commitRange.getQueryStartBoundary, startBoundary.version.get) } else if (startBoundary.timestamp.nonEmpty) { assertBoundaryTimestamp(commitRange.getQueryStartBoundary, startBoundary.timestamp.get) } else { throw new IllegalStateException("RequiredBoundaryDef must have either timestamp or version") } if (endBoundary.version.nonEmpty) { assert(commitRange.getQueryEndBoundary.isPresent) assertBoundaryVersion(commitRange.getQueryEndBoundary.get, endBoundary.version.get) } else if (endBoundary.timestamp.nonEmpty) { assert(commitRange.getQueryEndBoundary.isPresent) assertBoundaryTimestamp(commitRange.getQueryEndBoundary.get, endBoundary.timestamp.get) } else { assert(!commitRange.getQueryEndBoundary.isPresent) } } private def buildCommitRange( engine: Engine, fileList: Seq[FileStatus], startBoundary: RequiredBoundaryDef, endBoundary: BoundaryDef, logData: Option[Seq[ParsedLogData]] = None, ictEnablementInfo: Option[(Long, Long)] = None): CommitRange = { def getVersionFromFS(fs: FileStatus): Long = FileNames.getFileVersion(new Path(fs.getPath)) val latestVersion = fileList.map(getVersionFromFS).max // If we have a ratified commit file at the end version, we want to use this in the log segment // for our mockLatestSnapshot, so we get the ICT from that file val deltaFileAtEndVersion = fileList .filter(fs => FileNames.isStagedDeltaFile(fs.getPath)) .find(getVersionFromFS(_) == latestVersion) lazy val mockLatestSnapshot = getMockSnapshot( dataPath, latestVersion, ictEnablementInfoOpt = ictEnablementInfo, deltaFileAtEndVersion = deltaFileAtEndVersion) // Determine the start boundary val startBound = if (startBoundary.version.isDefined) { CommitBoundary.atVersion(startBoundary.version.get) } else if (startBoundary.timestamp.isDefined) { CommitBoundary.atTimestamp(startBoundary.timestamp.get, mockLatestSnapshot) } else { throw new IllegalStateException("RequiredBoundaryDef must have either timestamp or version") } var commitRangeBuilder = TableManager.loadCommitRange(dataPath.toString, startBound) endBoundary.version.foreach { v => commitRangeBuilder = commitRangeBuilder.withEndBoundary(CommitBoundary.atVersion(v)) } endBoundary.timestamp.foreach { v => commitRangeBuilder = commitRangeBuilder.withEndBoundary( CommitBoundary.atTimestamp(v, mockLatestSnapshot)) } logData.foreach { l => commitRangeBuilder = commitRangeBuilder.withLogData(l.asJava) } commitRangeBuilder.build(engine) } private def checkCommitRange( fileList: Seq[FileStatus], expectedStartVersion: Long, expectedEndVersion: Long, startBoundary: RequiredBoundaryDef, endBoundary: BoundaryDef): Unit = { val commitRange = buildCommitRange( createMockFSListFromEngine(fileList), fileList, startBoundary, endBoundary) assert(commitRange.getStartVersion == expectedStartVersion) assert(commitRange.getEndVersion == expectedEndVersion) checkQueryBoundaries(commitRange, startBoundary, endBoundary) val expectedFileList = fileList .filter(fs => { val version = FileNames.getFileVersion(new Path(fs.getPath)) version >= expectedStartVersion && version <= expectedEndVersion }).filter(fs => FileNames.isCommitFile(fs.getPath)) assert(expectedFileList.toSet == commitRange.asInstanceOf[CommitRangeImpl].getDeltaFiles.asScala.toSet) } /** * Base class for boundary definitions used in testing. * @param expectedVersion the expected version this def will be resolved to * @param expectError whether we expect this def to inherently fail for the corresponding listing */ private abstract class BoundaryDef( val expectedVersion: Long, val expectError: Boolean = false) { def version: Option[Long] = None def timestamp: Option[Long] = None } /** * Base class for boundary definitions that are NOT the default (i.e. are provided). * * At least one of `version` or `timestamp` must be defined in this case. */ private abstract class RequiredBoundaryDef( expectedVersion: Long, expectError: Boolean = false) extends BoundaryDef(expectedVersion, expectError) { assert(version.isDefined || timestamp.isDefined) } /** * Version-based boundary definition. * @param versionValue the version to use as boundary * @param expectsError whether we expect this def to inherently fail */ private case class VersionBoundaryDef( versionValue: Long, expectsError: Boolean = false) extends RequiredBoundaryDef(versionValue, expectsError) { override def version: Option[Long] = Some(versionValue) } /** * Timestamp-based boundary definition. * @param timestampValue the timestamp to use as boundary * @param resolvedVersion the expected version this timestamp will resolve to * @param expectsError whether we expect this def to inherently fail */ private case class TimestampBoundaryDef( timestampValue: Long, resolvedVersion: Long, expectsError: Boolean = false) extends RequiredBoundaryDef(resolvedVersion, expectsError) { override def timestamp: Option[Long] = Some(timestampValue) } /** * Default boundary definition (no specific version or timestamp). * @param resolvedVersion the expected version this will resolve to * @param expectsError whether we expect this def to inherently fail */ private case class DefaultBoundaryDef( resolvedVersion: Long, expectsError: Boolean = false) extends BoundaryDef(resolvedVersion, expectsError) def getExpectedException( startBoundary: RequiredBoundaryDef, endBoundary: BoundaryDef, fileStatuses: Seq[FileStatus]): Option[(Class[_ <: Throwable], String)] = { // These two cases fail on CommitRangeBuilderImpl.validateInputOnBuild if (startBoundary.version.isDefined && endBoundary.version.isDefined) { if (startBoundary.version.get > endBoundary.version.get) { return Some(classOf[IllegalArgumentException], "startVersion must be <= endVersion") } } if (startBoundary.timestamp.isDefined && endBoundary.timestamp.isDefined) { if (startBoundary.timestamp.get > endBoundary.timestamp.get) { return Some(classOf[IllegalArgumentException], "startTimestamp must be <= endTimestamp") } } if (endBoundary.version.isDefined) { val stagedCommits = fileStatuses .filter(fs => FileNames.isStagedDeltaFile(fs.getPath)) if (stagedCommits.nonEmpty) { val tailStagedCommit = stagedCommits(stagedCommits.length - 1) if (endBoundary.version.get > FileNames.deltaVersion(tailStagedCommit.getPath)) { return Some( classOf[IllegalArgumentException], "When endVersion is specified with logData, the last logData version") } } } // We try to resolve any timestamps, first startVersion then endVersion (CommitRangeFactory) if (startBoundary.expectError && startBoundary.timestamp.isDefined) { return Some(classOf[KernelException], "is after the latest available version") } if (endBoundary.expectError && endBoundary.timestamp.isDefined) { return Some(classOf[KernelException], "is before the earliest available version") } // Now we hit an exception if resolved startVersion > resolvedEndVersion (CommitRangeFactory) // (endVersion is only resolved before listing if either TS or Version was provided) if ( startBoundary.expectedVersion > endBoundary.expectedVersion && (endBoundary.timestamp.isDefined || endBoundary.version.isDefined) ) { return Some( classOf[KernelException], s"startVersion=${startBoundary.expectedVersion} > " + s"endVersion=${endBoundary.expectedVersion}") } // Now we query the file list, this is where we fail if the provided versions do not exist if (startBoundary.expectError) { // These either hit DeltaErrors.noCommitFilesFoundForVersionRange or // DeltaErrors.startVersionNotFound return Some(classOf[KernelException], s"no log file") } if (endBoundary.expectError) { // These either hit DeltaErrors.noCommitFilesFoundForVersionRange or // DeltaErrors.endVersionNotFound return Some(classOf[KernelException], s"no log file") } None } def testStartAndEndBoundaryCombinations( description: String, fileStatuses: Seq[FileStatus], startBoundaries: Seq[RequiredBoundaryDef], endBoundaries: Seq[BoundaryDef]): Unit = { startBoundaries.foreach { startBound => endBoundaries.foreach { endBound => test(s"$description: build CommitRange with startBound=$startBound endBound=$endBound") { val expectedException = getExpectedException(startBound, endBound, fileStatuses) if (expectedException.isDefined) { val e = intercept[Throwable] { buildCommitRange( createMockFSListFromEngine(fileStatuses), fileList = fileStatuses, startBoundary = startBound, endBoundary = endBound) } assert( expectedException.get._1.isInstance(e), s"Expected exception of ${expectedException.get._1} but found $e") assert(e.getMessage.contains(expectedException.get._2)) } else { checkCommitRange( fileList = fileStatuses, expectedStartVersion = startBound.expectedVersion, expectedEndVersion = endBound.expectedVersion, startBoundary = startBound, endBoundary = endBound) } } } } } /* --------------- Without catalog commits --------------- */ // Test with negative timestamps - manually create FileStatus with negative timestamps testStartAndEndBoundaryCombinations( description = "deltaFiles=(0, 1) with negative timestamps", // v0 -> -100, v1 -> -50 fileStatuses = Seq( FileStatus.of(FileNames.deltaFile(logPath, 0L), 0L, -100L), FileStatus.of(FileNames.deltaFile(logPath, 1L), 1L, -50L)), startBoundaries = Seq( VersionBoundaryDef(0L), VersionBoundaryDef(1L), TimestampBoundaryDef(-150, resolvedVersion = 0L), // before v0 TimestampBoundaryDef(-100, resolvedVersion = 0L), // at v0 TimestampBoundaryDef(-75, resolvedVersion = 1), // between v0, v1 TimestampBoundaryDef(-50, resolvedVersion = 1), // at v1 TimestampBoundaryDef(-40, resolvedVersion = -1, expectsError = true), // after v1 VersionBoundaryDef(2L, expectsError = true) // version DNE ), endBoundaries = Seq( VersionBoundaryDef(0L), VersionBoundaryDef(1L), TimestampBoundaryDef(-150, resolvedVersion = -1, expectsError = true), // before v0 TimestampBoundaryDef(-100, resolvedVersion = 0L), // at v0 TimestampBoundaryDef(-75, resolvedVersion = 0), // between v0, v1 TimestampBoundaryDef(-50, resolvedVersion = 1), // at v1 TimestampBoundaryDef(-40, resolvedVersion = 1), // after v1 DefaultBoundaryDef(resolvedVersion = 1), // default to latest VersionBoundaryDef(2L, expectsError = true) // version DNE )) // The below test cases mimic the cases in TableImplSuite for the timestamp-resolution testStartAndEndBoundaryCombinations( description = "deltaFiles=(0, 1)", // (version -> timestamp) = v0 -> 0, v1 -> 10 fileStatuses = deltaFileStatuses(Seq(0L, 1L)), startBoundaries = Seq( VersionBoundaryDef(0L), VersionBoundaryDef(1L), TimestampBoundaryDef(-5, resolvedVersion = 0L), // before v0, negative timestamp TimestampBoundaryDef(0, resolvedVersion = 0L), // at v0 TimestampBoundaryDef(5, resolvedVersion = 1), // between v0, v1 TimestampBoundaryDef(10, resolvedVersion = 1), // at v1 TimestampBoundaryDef(11, resolvedVersion = -1, expectsError = true), // after v1 VersionBoundaryDef(2L, expectsError = true) // version DNE ), endBoundaries = Seq( VersionBoundaryDef(0L), VersionBoundaryDef(1L), TimestampBoundaryDef(-5, resolvedVersion = -1, expectsError = true), // before v0, negative TimestampBoundaryDef(0, resolvedVersion = 0L), // at v0 TimestampBoundaryDef(5, resolvedVersion = 0), // between v0, v1 TimestampBoundaryDef(10, resolvedVersion = 1), // at v1 TimestampBoundaryDef(11, resolvedVersion = 1), // after v1 DefaultBoundaryDef(resolvedVersion = 1), // default to latest VersionBoundaryDef(2L, expectsError = true) // version DNE )) testStartAndEndBoundaryCombinations( // (version -> timestamp) = v10 -> 100, v11 -> 110, v12 -> 120 description = "deltaFiles=(10, 11, 12)", fileStatuses = deltaFileStatuses(Seq(10L, 11L, 12L)) ++ singularCheckpointFileStatuses(Seq(10L)), startBoundaries = Seq( VersionBoundaryDef(10L), VersionBoundaryDef(11L), VersionBoundaryDef(12L), TimestampBoundaryDef(99, resolvedVersion = 10), // before v10 TimestampBoundaryDef(100, resolvedVersion = 10L), // at v10 TimestampBoundaryDef(105, resolvedVersion = 11L), // between v10, v11 TimestampBoundaryDef(110, resolvedVersion = 11L), // at v11 TimestampBoundaryDef(115, resolvedVersion = 12L), // between v11, v12 TimestampBoundaryDef(120, resolvedVersion = 12L), // at v12 TimestampBoundaryDef(125, resolvedVersion = -1, expectsError = true), // after v12 VersionBoundaryDef(9L, expectsError = true), // version DNE VersionBoundaryDef(13L, expectsError = true) // version DNE ), endBoundaries = Seq( VersionBoundaryDef(10L), VersionBoundaryDef(11L), VersionBoundaryDef(12L), TimestampBoundaryDef(100, resolvedVersion = 10), // at v10 TimestampBoundaryDef(105, resolvedVersion = 10), // between v10, v11 TimestampBoundaryDef(110, resolvedVersion = 11), // at v11 TimestampBoundaryDef(115, resolvedVersion = 11), // between v11, v12 TimestampBoundaryDef(120, resolvedVersion = 12), // at v12 TimestampBoundaryDef(125, resolvedVersion = 12), // after v12 DefaultBoundaryDef(resolvedVersion = 12), // default to latest TimestampBoundaryDef(99, resolvedVersion = -1, expectsError = true), // before V10 VersionBoundaryDef(9L, expectsError = true), // version DNE VersionBoundaryDef(13L, expectsError = true) // version DNE )) // Check case with only 1 delta file testStartAndEndBoundaryCombinations( description = "deltaFiles=(10)", // (version -> timestamp) = v10 -> 100 fileStatuses = deltaFileStatuses(Seq(10L)) ++ singularCheckpointFileStatuses(Seq(10L)), startBoundaries = Seq( VersionBoundaryDef(10L), TimestampBoundaryDef(99L, resolvedVersion = 10L), // before v10 TimestampBoundaryDef(100L, resolvedVersion = 10L), // at v10 TimestampBoundaryDef(101L, resolvedVersion = -1, expectsError = true), // after v10 VersionBoundaryDef(1L, expectsError = true) // version DNE ), endBoundaries = Seq( VersionBoundaryDef(10L), TimestampBoundaryDef(100L, resolvedVersion = 10L), // at v10 TimestampBoundaryDef(101L, resolvedVersion = 10L), // after v10 DefaultBoundaryDef(resolvedVersion = 10L), // default to latest TimestampBoundaryDef(99L, resolvedVersion = -1, expectsError = true), // before v10 VersionBoundaryDef(1L, expectsError = true) // version DNE )) /* --------------- With catalog commits --------------- */ private def checkCommitRangeWithCatalogCommits( fileList: Seq[FileStatus], logData: Seq[ParsedLogData], versionToICT: Map[Long, Long], startBound: RequiredBoundaryDef, endBound: BoundaryDef, expectedFileList: Seq[FileStatus]): Unit = { // Create mock engine with ICT reading support val commitRange = buildCommitRange( createMockFSAndJsonEngineForICT(fileList, versionToICT), fileList, startBoundary = startBound, endBoundary = endBound, Some(logData), ictEnablementInfo = Some((0, 0))) assert(commitRange.getStartVersion == startBound.expectedVersion) assert(commitRange.getEndVersion == endBound.expectedVersion) checkQueryBoundaries( commitRange, startBound, endBound) assert(expectedFileList.toSet == commitRange.asInstanceOf[CommitRangeImpl].getDeltaFiles.asScala.toSet) } /** * @param expectedFileList takes in a tuple (startVersion, endVersion) of a range and returns the * expected file list for that version range */ private def testStartAndEndBoundaryCombinationsWithCatalogCommits( description: String, fileStatuses: Seq[FileStatus], expectedFileList: (Long, Long) => Seq[FileStatus], logData: Seq[ParsedLogData], versionToICT: Map[Long, Long], startBoundaries: Seq[RequiredBoundaryDef], endBoundaries: Seq[BoundaryDef]): Unit = { startBoundaries.foreach { startBound => endBoundaries.foreach { endBound => test(s"$description: build CommitRange with startBound=$startBound endBound=$endBound") { val expectedException = getExpectedException(startBound, endBound, fileStatuses) if (expectedException.isDefined) { val e = intercept[Throwable] { buildCommitRange( createMockFSAndJsonEngineForICT(fileStatuses, versionToICT), fileStatuses, startBoundary = startBound, endBoundary = endBound, Some(logData), ictEnablementInfo = Some((0, 0))) } assert( expectedException.get._1.isInstance(e), s"Expected exception of ${expectedException.get._1} but found $e") assert(e.getMessage.contains(expectedException.get._2)) } else { checkCommitRangeWithCatalogCommits( fileStatuses, logData, versionToICT, startBound, endBound, expectedFileList(startBound.expectedVersion, endBound.expectedVersion)) } } } } } // Basic case with no overlap: P0, P1, C2, C3, C4 { val catalogCommitFiles = Seq(stagedCommitFile(2), stagedCommitFile(3), stagedCommitFile(4)) val parsedLogData = catalogCommitFiles.map(ParsedCatalogCommitData.forFileStatus(_)) val fileList = Seq(deltaFileStatus(0), deltaFileStatus(1)) ++ catalogCommitFiles val versionToICT = Map(0L -> 50L, 1L -> 1050L, 2L -> 2050L, 3L -> 3050L, 4L -> 4050L) val startBoundaries = Seq( // V0 (first published commit) VersionBoundaryDef(0), TimestampBoundaryDef(5L, 0), // before V0 TimestampBoundaryDef(50L, 0L), // exactly at V0 // V1 (last published commit) VersionBoundaryDef(1), TimestampBoundaryDef(1000L, 1), // between V0 and V1 // V2 (first catalog commit) VersionBoundaryDef(2), TimestampBoundaryDef(1500, 2), // between V1 and V2 TimestampBoundaryDef(2050, 2), // exactly at V2 // V3 (middle catalog commit) VersionBoundaryDef(3L), // V4 (last catalog commit) VersionBoundaryDef(4L), TimestampBoundaryDef(3500L, 4L), // between V3 and V4 TimestampBoundaryDef(4050L, 4), // exactly at V4 // Some error cases TimestampBoundaryDef(4500, 4L, expectsError = true), // after V4 VersionBoundaryDef(5, expectsError = true) // version DNE ) val endBoundaries = Seq( // V0 (first published commit) VersionBoundaryDef(0), TimestampBoundaryDef(50L, 0L), // exactly at V0 TimestampBoundaryDef(500L, 0L), // between V0 and V1 // V1 (last published commit) VersionBoundaryDef(1), TimestampBoundaryDef(1500L, 1), // between V1 and V2 // V2 (first catalog commit) VersionBoundaryDef(2), TimestampBoundaryDef(2500, 2), // between V2 and V3 TimestampBoundaryDef(2050, 2), // exactly at V2 // V3 (middle catalog commit) VersionBoundaryDef(3L), TimestampBoundaryDef(3500L, 3L), // between V3 and V4 // V4 (last catalog commit) VersionBoundaryDef(4L), TimestampBoundaryDef(4050L, 4), // exactly at V4 DefaultBoundaryDef(4L), TimestampBoundaryDef(4500, 4L), // after V4 // Some error cases TimestampBoundaryDef(5L, 0, expectsError = true), // before V0 VersionBoundaryDef(5, expectsError = true) // version DNE ) testStartAndEndBoundaryCombinationsWithCatalogCommits( "catalog commits basic case no overlap", fileList, (startV, endV) => fileList.slice(startV.toInt, endV.toInt + 1), parsedLogData, versionToICT, startBoundaries, endBoundaries) } // Basic case with overlap: P0, P1, P2, R1, R2, R3 (+ prioritize catalog commits) { val catalogCommitFiles = Seq(stagedCommitFile(1), stagedCommitFile(2), stagedCommitFile(3)) val parsedLogData = catalogCommitFiles.map(ParsedCatalogCommitData.forFileStatus(_)) val publishedDeltaFiles = Seq(deltaFileStatus(0), deltaFileStatus(1), deltaFileStatus(2)) val fileList = publishedDeltaFiles ++ catalogCommitFiles val versionToICT = Map(0L -> 50L, 1L -> 1050L, 2L -> 2050L, 3L -> 3050L) // We expect catalog commits to take precedence over published deltas val expectedFileList = publishedDeltaFiles.slice(0, 1) ++ catalogCommitFiles val startBoundaries = Seq( // V0 VersionBoundaryDef(0), TimestampBoundaryDef(5L, 0), // before V0 TimestampBoundaryDef(50L, 0L), // exactly at V0 // V1 VersionBoundaryDef(1), TimestampBoundaryDef(1050L, 1), // at V1 TimestampBoundaryDef(1000L, 1), // between V0 and V1 // V2 VersionBoundaryDef(2), TimestampBoundaryDef(1500, 2), // between V1 and V2 TimestampBoundaryDef(2050, 2), // exactly at V2 // V3 VersionBoundaryDef(3L), TimestampBoundaryDef(2500, 3), // between V2 and V3 TimestampBoundaryDef(3050, 3), // exactly at V3 // Some error cases TimestampBoundaryDef(4500, 3L, expectsError = true), // after V4 VersionBoundaryDef(4, expectsError = true) // version DNE ) val endBoundaries = Seq( // V0 VersionBoundaryDef(0), TimestampBoundaryDef(50L, 0L), // exactly at V0 TimestampBoundaryDef(500L, 0L), // between V0 and V1 // V1 VersionBoundaryDef(1), TimestampBoundaryDef(1500L, 1), // between V1 and V2 // V2 VersionBoundaryDef(2), TimestampBoundaryDef(2500, 2), // between V2 and V3 TimestampBoundaryDef(2050, 2), // exactly at V2 // V3 VersionBoundaryDef(3L), TimestampBoundaryDef(3050L, 3), // exactly at V3 DefaultBoundaryDef(3L), TimestampBoundaryDef(3500, 3L), // after V3 // Some error cases TimestampBoundaryDef(5L, 0, expectsError = true), // before V0 VersionBoundaryDef(5, expectsError = true) // version DNE ) testStartAndEndBoundaryCombinationsWithCatalogCommits( "catalog commits basic case with overlap", fileList, (startV, endV) => expectedFileList.slice(startV.toInt, endV.toInt + 1), parsedLogData, versionToICT, startBoundaries, endBoundaries) } // Only catalog commits: C0, C1 { val catalogCommitFiles = Seq(stagedCommitFile(0), stagedCommitFile(1)) val parsedLogData = catalogCommitFiles.map(ParsedCatalogCommitData.forFileStatus(_)) val versionToICT = Map(0L -> 50L, 1L -> 1050L) val startBoundaries = Seq( // V0 VersionBoundaryDef(0), TimestampBoundaryDef(5L, 0), // before V0 TimestampBoundaryDef(50L, 0L), // exactly at V0 // V1 VersionBoundaryDef(1), TimestampBoundaryDef(1050L, 1), // at V1 TimestampBoundaryDef(1000L, 1), // between V0 and V1 // Some error cases TimestampBoundaryDef(4500, 1L, expectsError = true), // after V1 VersionBoundaryDef(4, expectsError = true) // version DNE ) val endBoundaries = Seq( // V0 VersionBoundaryDef(0), TimestampBoundaryDef(50L, 0L), // exactly at V0 TimestampBoundaryDef(500L, 0L), // between V0 and V1 // V1 VersionBoundaryDef(1), TimestampBoundaryDef(1050L, 1), // exactly at V1 TimestampBoundaryDef(1500L, 1), // after V1 DefaultBoundaryDef(1), // Some error cases TimestampBoundaryDef(5L, 0, expectsError = true), // before V0 VersionBoundaryDef(5, expectsError = true) // version DNE ) testStartAndEndBoundaryCombinationsWithCatalogCommits( "catalog commits no published deltas", catalogCommitFiles, (startV, endV) => catalogCommitFiles.slice(startV.toInt, endV.toInt + 1), parsedLogData, versionToICT, startBoundaries, endBoundaries) } // Single published commit + single catalog commit: P0, C1 { val catalogCommitFiles = Seq(stagedCommitFile(1)) val parsedLogData = catalogCommitFiles.map(ParsedCatalogCommitData.forFileStatus(_)) val publishedDeltaFiles = Seq(deltaFileStatus(0)) val fileList = publishedDeltaFiles ++ catalogCommitFiles val versionToICT = Map(0L -> 50L, 1L -> 1050L) val startBoundaries = Seq( // V0 VersionBoundaryDef(0), TimestampBoundaryDef(5L, 0), // before V0 TimestampBoundaryDef(50L, 0L), // exactly at V0 // V1 VersionBoundaryDef(1), TimestampBoundaryDef(1050L, 1), // at V1 TimestampBoundaryDef(1000L, 1), // between V0 and V1 // Some error cases TimestampBoundaryDef(4500, 1L, expectsError = true), // after V1 VersionBoundaryDef(4, expectsError = true) // version DNE ) val endBoundaries = Seq( // V0 VersionBoundaryDef(0), TimestampBoundaryDef(50L, 0L), // exactly at V0 TimestampBoundaryDef(500L, 0L), // between V0 and V1 // V1 VersionBoundaryDef(1), TimestampBoundaryDef(1050L, 1), // exactly at V1 TimestampBoundaryDef(1500L, 1), // after V1 DefaultBoundaryDef(1), // Some error cases TimestampBoundaryDef(5L, 0, expectsError = true), // before V0 VersionBoundaryDef(5, expectsError = true) // version DNE ) testStartAndEndBoundaryCombinationsWithCatalogCommits( "catalog commits single catalog commit single published commit", fileList, (startV, endV) => fileList.slice(startV.toInt, endV.toInt + 1), parsedLogData, versionToICT, startBoundaries, endBoundaries) } // Overlap by just 1: P0, P1, R1, R2 { val catalogCommitFiles = Seq(stagedCommitFile(1), stagedCommitFile(2)) val parsedLogData = catalogCommitFiles.map(ParsedCatalogCommitData.forFileStatus(_)) val publishedDeltaFiles = Seq(deltaFileStatus(0), deltaFileStatus(1)) val fileList = publishedDeltaFiles ++ catalogCommitFiles val versionToICT = Map(0L -> 50L, 1L -> 1050L, 2L -> 2050L) // We expect catalog commits to take precedence over published deltas val expectedFileList = publishedDeltaFiles.slice(0, 1) ++ catalogCommitFiles val startBoundaries = Seq( // V0 VersionBoundaryDef(0), TimestampBoundaryDef(5L, 0), // before V0 TimestampBoundaryDef(50L, 0L), // exactly at V0 // V1 VersionBoundaryDef(1), TimestampBoundaryDef(1050L, 1), // at V1 TimestampBoundaryDef(1000L, 1), // between V0 and V1 // V2 VersionBoundaryDef(2L), TimestampBoundaryDef(1500, 2), // between V1 and V2 TimestampBoundaryDef(2050, 2), // exactly at V2 // Some error cases TimestampBoundaryDef(4500, 3L, expectsError = true), // after V2 VersionBoundaryDef(4, expectsError = true) // version DNE ) val endBoundaries = Seq( // V0 VersionBoundaryDef(0), TimestampBoundaryDef(50L, 0L), // exactly at V0 TimestampBoundaryDef(500L, 0L), // between V0 and V1 // V1 VersionBoundaryDef(1), TimestampBoundaryDef(1500L, 1), // between V1 and V2 // V2 VersionBoundaryDef(2L), TimestampBoundaryDef(2050L, 2), // exactly at V2 DefaultBoundaryDef(2L), TimestampBoundaryDef(2500, 2), // after V2 // Some error cases TimestampBoundaryDef(5L, 0, expectsError = true), // before V0 VersionBoundaryDef(5, expectsError = true) // version DNE ) testStartAndEndBoundaryCombinationsWithCatalogCommits( "catalog commits single commit overlap", fileList, (startV, endV) => expectedFileList.slice(startV.toInt, endV.toInt + 1), parsedLogData, versionToICT, startBoundaries, endBoundaries) } // Full overlap: P0, P1, P2 + C0, C1, C2 { val catalogCommitFiles = Seq(stagedCommitFile(0), stagedCommitFile(1), stagedCommitFile(2)) val parsedLogData = catalogCommitFiles.map(ParsedCatalogCommitData.forFileStatus(_)) val publishedDeltaFiles = Seq(deltaFileStatus(0), deltaFileStatus(1), deltaFileStatus(2)) val fileList = publishedDeltaFiles ++ catalogCommitFiles val versionToICT = Map(0L -> 50L, 1L -> 1050L, 2L -> 2050L) val startBoundaries = Seq( // V0 VersionBoundaryDef(0), TimestampBoundaryDef(5L, 0), // before V0 TimestampBoundaryDef(50L, 0L), // exactly at V0 // V1 VersionBoundaryDef(1), TimestampBoundaryDef(1050L, 1), // at V1 TimestampBoundaryDef(1000L, 1), // between V0 and V1 // V2 VersionBoundaryDef(2L), TimestampBoundaryDef(1500, 2), // between V1 and V2 TimestampBoundaryDef(2050, 2), // exactly at V2 // Some error cases TimestampBoundaryDef(4500, 3L, expectsError = true), // after V2 VersionBoundaryDef(4, expectsError = true) // version DNE ) val endBoundaries = Seq( // V0 VersionBoundaryDef(0), TimestampBoundaryDef(50L, 0L), // exactly at V0 TimestampBoundaryDef(500L, 0L), // between V0 and V1 // V1 VersionBoundaryDef(1), TimestampBoundaryDef(1500L, 1), // between V1 and V2 // V2 VersionBoundaryDef(2L), TimestampBoundaryDef(2050L, 2), // exactly at V2 DefaultBoundaryDef(2L), TimestampBoundaryDef(2500, 2), // after V2 // Some error cases TimestampBoundaryDef(5L, 0, expectsError = true), // before V0 VersionBoundaryDef(5, expectsError = true) // version DNE ) testStartAndEndBoundaryCombinationsWithCatalogCommits( "catalog commits full overlap", fileList, (startV, endV) => catalogCommitFiles.slice(startV.toInt, endV.toInt + 1), parsedLogData, versionToICT, startBoundaries, endBoundaries) } test("build CommitRange fails if catalog commits pre-curse published commits") { val catalogCommitFiles = Seq(stagedCommitFile(0), stagedCommitFile(1), stagedCommitFile(2)) val parsedLogData = catalogCommitFiles.map(ParsedCatalogCommitData.forFileStatus(_)) val publishedDeltaFiles = Seq(deltaFileStatus(1), deltaFileStatus(2), deltaFileStatus(3)) val fileList = publishedDeltaFiles ++ catalogCommitFiles val versionToICT = Map(0L -> 50L, 1L -> 1050L, 2L -> 2050L) val e = intercept[InvalidTableException] { buildCommitRange( createMockFSAndJsonEngineForICT(fileList, versionToICT), fileList, startBoundary = VersionBoundaryDef(0), endBoundary = VersionBoundaryDef(2), logData = Some(parsedLogData), ictEnablementInfo = Some((0, 0))) } assert(e.getMessage.contains( "Missing delta file: found staged ratified commit for version 0 but no published " + "delta file. Found published deltas for later versions: [1, 2]")) } test("build CommitRange fails if published commits and catalog commits are not contiguous") { val catalogCommitFiles = Seq(stagedCommitFile(2)) val parsedLogData = catalogCommitFiles.map(ParsedCatalogCommitData.forFileStatus(_)) val publishedDeltaFiles = Seq(deltaFileStatus(0)) val fileList = publishedDeltaFiles ++ catalogCommitFiles val versionToICT = Map(0L -> 50L, 1L -> 1050L, 2L -> 2050L) val e = intercept[InvalidTableException] { buildCommitRange( createMockFSAndJsonEngineForICT(fileList, versionToICT), fileList, startBoundary = VersionBoundaryDef(0), endBoundary = VersionBoundaryDef(2), logData = Some(parsedLogData), ictEnablementInfo = Some((0, 0))) } assert(e.getMessage.contains( "Missing delta files: found published delta files for versions [0] and staged " + "ratified commits for versions [2]")) } test("build CommitRange fails if published deltas are not contiguous") { val publishedDeltaFiles = Seq(deltaFileStatus(0), deltaFileStatus(2), deltaFileStatus(3)) val e = intercept[InvalidTableException] { buildCommitRange( createMockFSListFromEngine(publishedDeltaFiles), publishedDeltaFiles, startBoundary = VersionBoundaryDef(0), endBoundary = VersionBoundaryDef(3)) } assert(e.getMessage.contains( "Missing delta files: versions are not contiguous: ([0, 2, 3])")) } Seq( ParsedCatalogCommitData.forInlineData(1, emptyColumnarBatch), ParsedLogData.forFileStatus(logCompactionStatus(0, 1))).foreach { parsedLogData => val suffix = s"- type=${parsedLogData.getGroupByCategoryClass.toString}" test(s"withLogData: non-staged-ratified-commit throws IllegalArgumentException $suffix") { val builder = TableManager .loadCommitRange(dataPath.toString, CommitBoundary.atVersion(0)) .withLogData(Collections.singletonList(parsedLogData)) val exMsg = intercept[IllegalArgumentException] { builder.build(mockEngine()) }.getMessage assert(exMsg.contains("Only staged ratified commits are supported")) } } test("withLogData: non-contiguous input throws IllegalArgumentException") { val exMsg = intercept[IllegalArgumentException] { TableManager.loadCommitRange(dataPath.toString, CommitBoundary.atVersion(0)) .withLogData(parsedRatifiedStagedCommits(Seq(0, 2)).toList.asJava) .build(mockEngine()) }.getMessage assert(exMsg.contains("Log data must be sorted and contiguous")) } test("withLogData: non-sorted input throws IllegalArgumentException") { val exMsg = intercept[IllegalArgumentException] { TableManager.loadCommitRange(dataPath.toString, CommitBoundary.atVersion(0)) .withLogData(parsedRatifiedStagedCommits(Seq(2, 1, 0)).toList.asJava) .build(mockEngine()) }.getMessage assert(exMsg.contains("Log data must be sorted and contiguous")) } ////////////////////////////////////////////// // withMaxCatalogVersion Tests ////////////////////////////////////////////// test("withMaxCatalogVersion: negative version throws IllegalArgumentException") { val exMsg = intercept[IllegalArgumentException] { TableManager .loadCommitRange(dataPath.toString, CommitBoundary.atVersion(0)) .withMaxCatalogVersion(-1) }.getMessage assert(exMsg.contains("maxCatalogVersion must be >= 0")) } test("withMaxCatalogVersion: zero is valid") { val fileList = deltaFileStatuses(Seq(0, 1, 2)) val builder = TableManager .loadCommitRange(dataPath.toString, CommitBoundary.atVersion(0)) .withMaxCatalogVersion(0) val commitRange = builder.build(createMockFSListFromEngine(fileList)) assert(commitRange.getStartVersion == 0) assert(commitRange.getEndVersion == 0) } test("withMaxCatalogVersion: positive version is valid") { val fileList = deltaFileStatuses(0L to 10L) val builder = TableManager .loadCommitRange(dataPath.toString, CommitBoundary.atVersion(0)) .withMaxCatalogVersion(5) val commitRange = builder.build(createMockFSListFromEngine(fileList)) assert(commitRange.getStartVersion == 0) assert(commitRange.getEndVersion == 5) } test("withMaxCatalogVersion: start version must be <= maxCatalogVersion") { val fileList = deltaFileStatuses(0L to 10L) val exMsg = intercept[IllegalArgumentException] { TableManager .loadCommitRange(dataPath.toString, CommitBoundary.atVersion(6)) .withMaxCatalogVersion(5) .build(createMockFSListFromEngine(fileList)) }.getMessage assert(exMsg.contains("startVersion (6) must be <= maxCatalogVersion (5)")) } test("withMaxCatalogVersion: start version equal to maxCatalogVersion is valid") { val fileList = deltaFileStatuses(0L to 10L) val builder = TableManager .loadCommitRange(dataPath.toString, CommitBoundary.atVersion(5)) .withMaxCatalogVersion(5) val commitRange = builder.build(createMockFSListFromEngine(fileList)) assert(commitRange.getStartVersion == 5) assert(commitRange.getEndVersion == 5) } test("withMaxCatalogVersion: end version must be <= maxCatalogVersion") { val fileList = deltaFileStatuses(0L to 10L) val exMsg = intercept[IllegalArgumentException] { TableManager .loadCommitRange(dataPath.toString, CommitBoundary.atVersion(0)) .withMaxCatalogVersion(5) .withEndBoundary(CommitBoundary.atVersion(6)) .build(createMockFSListFromEngine(fileList)) }.getMessage assert(exMsg.contains("endVersion (6) must be <= maxCatalogVersion (5)")) } test("withMaxCatalogVersion: end version equal to maxCatalogVersion is valid") { val fileList = deltaFileStatuses(0L to 10L) val builder = TableManager .loadCommitRange(dataPath.toString, CommitBoundary.atVersion(0)) .withMaxCatalogVersion(5) .withEndBoundary(CommitBoundary.atVersion(5)) val commitRange = builder.build(createMockFSListFromEngine(fileList)) assert(commitRange.getStartVersion == 0) assert(commitRange.getEndVersion == 5) } test( "withMaxCatalogVersion: start timestamp boundary requires snapshot version = " + "maxCatalogVersion") { val fileList = deltaFileStatuses(0L to 10L) val mockLatestSnapshot = getMockSnapshot(dataPath, 10) val exMsg = intercept[IllegalArgumentException] { TableManager .loadCommitRange( dataPath.toString, CommitBoundary.atTimestamp(50, mockLatestSnapshot)) .withMaxCatalogVersion(5) .build(createMockFSListFromEngine(fileList)) }.getMessage assert(exMsg.contains("the provided snapshot version (10) must equal maxCatalogVersion (5)")) } test("withMaxCatalogVersion: start timestamp boundary with matching snapshot version is valid") { val fileList = deltaFileStatuses(0L to 5L) val mockLatestSnapshot = getMockSnapshot(dataPath, 5) val builder = TableManager .loadCommitRange( dataPath.toString, CommitBoundary.atTimestamp(30, mockLatestSnapshot)) .withMaxCatalogVersion(5) val commitRange = builder.build(createMockFSListFromEngine(fileList)) assert(commitRange.getStartVersion == 3) // timestamp 30 maps to version 3 assert(commitRange.getEndVersion == 5) } test( "withMaxCatalogVersion: end timestamp boundary requires snapshot version = maxCatalogVersion") { val fileList = deltaFileStatuses(0L to 10L) val mockLatestSnapshot = getMockSnapshot(dataPath, 10) val exMsg = intercept[IllegalArgumentException] { TableManager .loadCommitRange(dataPath.toString, CommitBoundary.atVersion(0)) .withMaxCatalogVersion(5) .withEndBoundary(CommitBoundary.atTimestamp(50, mockLatestSnapshot)) .build(createMockFSListFromEngine(fileList)) }.getMessage assert(exMsg.contains("the provided snapshot version (10) must equal maxCatalogVersion (5)")) } test("withMaxCatalogVersion: end timestamp boundary with matching snapshot version is valid") { val fileList = deltaFileStatuses(0L to 5L) val mockLatestSnapshot = getMockSnapshot(dataPath, 5) val builder = TableManager .loadCommitRange(dataPath.toString, CommitBoundary.atVersion(0)) .withMaxCatalogVersion(5) .withEndBoundary(CommitBoundary.atTimestamp(30, mockLatestSnapshot)) val commitRange = builder.build(createMockFSListFromEngine(fileList)) assert(commitRange.getStartVersion == 0) assert(commitRange.getEndVersion == 3) // timestamp 30 maps to version 3 } test("withMaxCatalogVersion: without end boundary, logData must end with maxCatalogVersion") { val logData = parsedRatifiedStagedCommits(Seq(0, 1, 2, 3, 4)) val exMsg = intercept[IllegalArgumentException] { TableManager .loadCommitRange(dataPath.toString, CommitBoundary.atVersion(0)) .withMaxCatalogVersion(5) .withLogData(logData.toList.asJava) .build(mockEngine()) }.getMessage assert(exMsg.contains("the last logData version (4) must equal maxCatalogVersion (5)")) } test( "withMaxCatalogVersion: without end boundary, logData ending with maxCatalogVersion is valid") { val fileList = deltaFileStatuses(0L to 5L) val logData = parsedRatifiedStagedCommits(Seq(0, 1, 2, 3, 4, 5)) val builder = TableManager .loadCommitRange(dataPath.toString, CommitBoundary.atVersion(0)) .withMaxCatalogVersion(5) .withLogData(logData.toList.asJava) val commitRange = builder.build(createMockFSListFromEngine(fileList)) assert(commitRange.getStartVersion == 0) assert(commitRange.getEndVersion == 5) } test("withMaxCatalogVersion: empty logData with maxCatalogVersion is valid") { val fileList = deltaFileStatuses(0L to 10L) val builder = TableManager .loadCommitRange(dataPath.toString, CommitBoundary.atVersion(0)) .withMaxCatalogVersion(5) .withLogData(Collections.emptyList()) val commitRange = builder.build(createMockFSListFromEngine(fileList)) assert(commitRange.getStartVersion == 0) assert(commitRange.getEndVersion == 5) } test("withMaxCatalogVersion: with end boundary and logData is valid") { val fileList = deltaFileStatuses(0L to 10L) val logData = parsedRatifiedStagedCommits(Seq(0, 1, 2, 3)) val builder = TableManager .loadCommitRange(dataPath.toString, CommitBoundary.atVersion(0)) .withMaxCatalogVersion(10) .withEndBoundary(CommitBoundary.atVersion(3)) .withLogData(logData.toList.asJava) val commitRange = builder.build(createMockFSListFromEngine(fileList)) assert(commitRange.getStartVersion == 0) assert(commitRange.getEndVersion == 3) } ////////////////////////////////////////////// // withLogData + endVersion validation tests ////////////////////////////////////////////// test("withLogData: with endVersion, logData must cover the requested range") { val logData = parsedRatifiedStagedCommits(Seq(0, 1, 2)) val fileList = deltaFileStatuses(0L to 3L) val exMsg = intercept[IllegalArgumentException] { TableManager .loadCommitRange(dataPath.toString, CommitBoundary.atVersion(0)) .withEndBoundary(CommitBoundary.atVersion(3)) .withLogData(logData.toList.asJava) .build(createMockFSListFromEngine(fileList)) }.getMessage assert(exMsg.contains("the last logData version (2) must be >= endVersion (3)")) } test("withLogData: with endVersion equal to last logData version is valid") { val logData = parsedRatifiedStagedCommits(Seq(0, 1, 2, 3)) val fileList = deltaFileStatuses(0L to 3L) val builder = TableManager .loadCommitRange(dataPath.toString, CommitBoundary.atVersion(0)) .withEndBoundary(CommitBoundary.atVersion(3)) .withLogData(logData.toList.asJava) val commitRange = builder.build(createMockFSListFromEngine(fileList)) assert(commitRange.getStartVersion == 0) assert(commitRange.getEndVersion == 3) } test("withLogData: with endVersion less than last logData version is valid") { val logData = parsedRatifiedStagedCommits(Seq(0, 1, 2, 3, 4, 5)) val fileList = deltaFileStatuses(0L to 5L) val builder = TableManager .loadCommitRange(dataPath.toString, CommitBoundary.atVersion(0)) .withEndBoundary(CommitBoundary.atVersion(3)) .withLogData(logData.toList.asJava) val commitRange = builder.build(createMockFSListFromEngine(fileList)) assert(commitRange.getStartVersion == 0) assert(commitRange.getEndVersion == 3) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/DeltaHistoryManagerSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal import java.io.FileNotFoundException import java.util import java.util.Optional import scala.collection.JavaConverters._ import scala.reflect.ClassTag import io.delta.kernel.TransactionSuite.testSchema import io.delta.kernel.data.{ColumnarBatch, ColumnVector} import io.delta.kernel.engine.Engine import io.delta.kernel.exceptions.KernelException import io.delta.kernel.exceptions.TableNotFoundException import io.delta.kernel.internal.actions.{Format, Metadata, Protocol} import io.delta.kernel.internal.commit.DefaultFileSystemManagedTableOnlyCommitter import io.delta.kernel.internal.files.{ParsedCatalogCommitData, ParsedPublishedDeltaData} import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.lang.Lazy import io.delta.kernel.internal.metrics.SnapshotQueryContext import io.delta.kernel.internal.snapshot.LogSegment import io.delta.kernel.internal.util.{FileNames, VectorUtils} import io.delta.kernel.internal.util.InCommitTimestampUtils import io.delta.kernel.internal.util.VectorUtils.{buildArrayValue, stringStringMapValue} import io.delta.kernel.test.{MockFileSystemClientUtils, MockListFromFileSystemClient, MockReadICTFileJsonHandler} import io.delta.kernel.test.MockSnapshotUtils.getMockSnapshot import io.delta.kernel.types.StringType import io.delta.kernel.types.StructType import io.delta.kernel.utils.FileStatus import org.scalatest.funsuite.AnyFunSuite class DeltaHistoryManagerSuite extends AnyFunSuite with MockFileSystemClientUtils { // Helper function for non-catalog-managed tables (no staged commits) private def getActiveCommitAtTimestamp( engine: Engine, latestSnapshot: SnapshotImpl, logPath: Path, timestamp: Long, mustBeRecreatable: Boolean = true, canReturnLastCommit: Boolean = false, canReturnEarliestCommit: Boolean = false): DeltaHistoryManager.Commit = { DeltaHistoryManager.getActiveCommitAtTimestamp( engine, latestSnapshot, logPath, timestamp, mustBeRecreatable, canReturnLastCommit, canReturnEarliestCommit, Seq.empty.asJava /* parsedLogDelta */ ) } def checkGetActiveCommitAtTimestamp( fileList: Seq[FileStatus], timestamp: Long, expectedVersion: Long, mustBeRecreatable: Boolean = true, canReturnLastCommit: Boolean = false, canReturnEarliestCommit: Boolean = false): Unit = { val lastDelta = fileList.map(_.getPath).filter(FileNames.isCommitFile).last val latestVersion = FileNames.getFileVersion(new Path(lastDelta)) val activeCommit = getActiveCommitAtTimestamp( createMockFSListFromEngine(fileList), getMockSnapshot(dataPath, latestVersion = latestVersion), logPath, timestamp, mustBeRecreatable, canReturnLastCommit, canReturnEarliestCommit) assert( activeCommit.getVersion == expectedVersion, s"Expected version $expectedVersion but got $activeCommit for timestamp=$timestamp") if (mustBeRecreatable) { // When mustBeRecreatable=true, we should have the same answer as mustBeRecreatable=false // for valid queries that do not throw an error val activeCommit = getActiveCommitAtTimestamp( createMockFSListFromEngine(fileList), getMockSnapshot(dataPath, latestVersion), logPath, timestamp, false, // mustBeRecreatable canReturnLastCommit, canReturnEarliestCommit) assert( activeCommit.getVersion == expectedVersion, s"Expected version $expectedVersion but got $activeCommit for timestamp=$timestamp") } } def checkGetActiveCommitAtTimestampError[T <: Throwable]( fileList: Seq[FileStatus], latestVersion: Long, timestamp: Long, expectedErrorMessageContains: String, mustBeRecreatable: Boolean = true, canReturnLastCommit: Boolean = false, canReturnEarliestCommit: Boolean = false)(implicit classTag: ClassTag[T]): Unit = { val e = intercept[T] { getActiveCommitAtTimestamp( createMockFSListFromEngine(fileList), getMockSnapshot(dataPath, latestVersion = latestVersion), logPath, timestamp, mustBeRecreatable, canReturnLastCommit, canReturnEarliestCommit) } assert(e.getMessage.contains(expectedErrorMessageContains)) } test("getActiveCommitAtTimestamp: basic listing from 0 with no checkpoints") { val deltaFiles = deltaFileStatuses(Seq(0L, 1L, 2L)) // Valid queries checkGetActiveCommitAtTimestamp(deltaFiles, 0, 0) checkGetActiveCommitAtTimestamp(deltaFiles, 1, 0) checkGetActiveCommitAtTimestamp(deltaFiles, 10, 1) checkGetActiveCommitAtTimestamp(deltaFiles, 11, 1) checkGetActiveCommitAtTimestamp(deltaFiles, 20, 2) // Invalid queries checkGetActiveCommitAtTimestampError[RuntimeException]( deltaFiles, latestVersion = 2L, -1, DeltaErrors.timestampBeforeFirstAvailableCommit(dataPath.toString, -1, 0, 0).getMessage) checkGetActiveCommitAtTimestampError[RuntimeException]( deltaFiles, latestVersion = 2L, 21, DeltaErrors.timestampAfterLatestCommit(dataPath.toString, 21, 20, 2).getMessage) // Valid queries with canReturnLastCommit=true and canReturnEarliestCommit=true checkGetActiveCommitAtTimestamp(deltaFiles, -1, 0, canReturnEarliestCommit = true) checkGetActiveCommitAtTimestamp(deltaFiles, 21, 2, canReturnLastCommit = true) } test("getActiveCommitAtTimestamp: basic listing from 0 with a checkpoint") { val deltaFiles = deltaFileStatuses(Seq(0L, 1L, 2L)) ++ singularCheckpointFileStatuses(Seq(2L)) // Valid queries checkGetActiveCommitAtTimestamp(deltaFiles, 0, 0) checkGetActiveCommitAtTimestamp(deltaFiles, 1, 0) checkGetActiveCommitAtTimestamp(deltaFiles, 10, 1) checkGetActiveCommitAtTimestamp(deltaFiles, 11, 1) checkGetActiveCommitAtTimestamp(deltaFiles, 20, 2) // Invalid queries checkGetActiveCommitAtTimestampError[RuntimeException]( deltaFiles, latestVersion = 2L, -1, DeltaErrors.timestampBeforeFirstAvailableCommit(dataPath.toString, -1, 0, 0).getMessage) checkGetActiveCommitAtTimestampError[RuntimeException]( deltaFiles, latestVersion = 2L, 21, DeltaErrors.timestampAfterLatestCommit(dataPath.toString, 21, 20, 2).getMessage) // Valid queries with canReturnLastCommit=true and canReturnEarliestCommit=true checkGetActiveCommitAtTimestamp(deltaFiles, -1, 0, canReturnEarliestCommit = true) checkGetActiveCommitAtTimestamp(deltaFiles, 21, 2, canReturnLastCommit = true) } test("getActiveCommitAtTimestamp: truncated delta log") { val deltaFiles = deltaFileStatuses(Seq(2L, 3L)) ++ singularCheckpointFileStatuses(Seq(2L)) // Valid queries checkGetActiveCommitAtTimestamp(deltaFiles, 20, 2) checkGetActiveCommitAtTimestamp(deltaFiles, 25, 2) checkGetActiveCommitAtTimestamp(deltaFiles, 30, 3) // Invalid queries checkGetActiveCommitAtTimestampError[RuntimeException]( deltaFiles, latestVersion = 3L, 8, DeltaErrors.timestampBeforeFirstAvailableCommit(dataPath.toString, 8, 20, 2).getMessage) checkGetActiveCommitAtTimestampError[RuntimeException]( deltaFiles, latestVersion = 3L, 31, DeltaErrors.timestampAfterLatestCommit(dataPath.toString, 31, 30, 3).getMessage) // Valid queries with canReturnLastCommit=true and canReturnEarliestCommit=true checkGetActiveCommitAtTimestamp(deltaFiles, 8, 2, canReturnEarliestCommit = true) checkGetActiveCommitAtTimestamp(deltaFiles, 31, 3, canReturnLastCommit = true) } test("getActiveCommitAtTimestamp: truncated delta log only checkpoint version") { val deltaFiles = deltaFileStatuses(Seq(2L)) ++ singularCheckpointFileStatuses(Seq(2L)) // Valid queries checkGetActiveCommitAtTimestamp(deltaFiles, 20, 2) // Invalid queries checkGetActiveCommitAtTimestampError[RuntimeException]( deltaFiles, latestVersion = 2L, 8, DeltaErrors.timestampBeforeFirstAvailableCommit(dataPath.toString, 8, 20, 2).getMessage) checkGetActiveCommitAtTimestampError[RuntimeException]( deltaFiles, latestVersion = 2L, 21, DeltaErrors.timestampAfterLatestCommit(dataPath.toString, 21, 20, 2).getMessage) // Valid queries with canReturnLastCommit=true and canReturnEarliestCommit=true checkGetActiveCommitAtTimestamp(deltaFiles, 8, 2, canReturnEarliestCommit = true) checkGetActiveCommitAtTimestamp(deltaFiles, 21, 2, canReturnLastCommit = true) } test("getActiveCommitAtTimestamp: truncated delta log with multi-part checkpoint") { val deltaFiles = deltaFileStatuses(Seq(2L, 3L)) ++ multiCheckpointFileStatuses(Seq(2L), 2) // Valid queries checkGetActiveCommitAtTimestamp(deltaFiles, 20, 2) checkGetActiveCommitAtTimestamp(deltaFiles, 25, 2) checkGetActiveCommitAtTimestamp(deltaFiles, 30, 3) // Invalid queries checkGetActiveCommitAtTimestampError[RuntimeException]( deltaFiles, latestVersion = 3L, 8, DeltaErrors.timestampBeforeFirstAvailableCommit(dataPath.toString, 8, 20, 2).getMessage) checkGetActiveCommitAtTimestampError[RuntimeException]( deltaFiles, latestVersion = 3L, 31, DeltaErrors.timestampAfterLatestCommit(dataPath.toString, 31, 30, 3).getMessage) // Valid queries with canReturnLastCommit=true and canReturnEarliestCommit=true checkGetActiveCommitAtTimestamp(deltaFiles, 8, 2, canReturnEarliestCommit = true) checkGetActiveCommitAtTimestamp(deltaFiles, 31, 3, canReturnLastCommit = true) } test("getActiveCommitAtTimestamp: throws table not found exception") { // Non-existent path intercept[TableNotFoundException]( getActiveCommitAtTimestamp( createMockFSListFromEngine(p => throw new FileNotFoundException(p)), getMockSnapshot(dataPath, latestVersion = 1L), logPath, timestamp = 0)) // Empty _delta_log directory intercept[TableNotFoundException]( getActiveCommitAtTimestamp( createMockFSListFromEngine(p => Seq()), getMockSnapshot(dataPath, latestVersion = 1L), logPath, timestamp = 0)) } // TODO: corrects commit timestamps for increasing commits (monotonizeCommitTimestamps)? // (see test "getCommits should monotonize timestamps" in DeltaTimeTravelSuite)? test("getActiveCommitAtTimestamp: corrupt listings") { // No checkpoint or 000.json present checkGetActiveCommitAtTimestampError[RuntimeException]( deltaFileStatuses(Seq(1L, 2L, 3L)), latestVersion = 3L, 25, "No recreatable commits found") // Must have corresponding delta file for a checkpoint checkGetActiveCommitAtTimestampError[RuntimeException]( singularCheckpointFileStatuses(Seq(1L)) ++ deltaFileStatuses(Seq(2L, 3L)), latestVersion = 3L, 25, "No recreatable commits found") // No commit files at all (only checkpoint files) checkGetActiveCommitAtTimestampError[RuntimeException]( singularCheckpointFileStatuses(Seq(1L)), latestVersion = 1L, 25, "No commits found") // No delta files checkGetActiveCommitAtTimestampError[RuntimeException]( Seq("foo", "notdelta.parquet", "foo.json", "001.checkpoint.00f.oo0.parquet") .map(new Path(logPath, _)) .map(path => FileStatus.of(path.toString, 10, 10)), latestVersion = 1L, 25, "No delta files found in the directory") // No complete checkpoint checkGetActiveCommitAtTimestampError[RuntimeException]( deltaFileStatuses(Seq(2L, 3L)) ++ multiCheckpointFileStatuses(Seq(2L), 3).take(2), latestVersion = 3L, 25, "No recreatable commits found") } test("getActiveCommitAtTimestamp: when mustBeRecreatable=false") { Seq( deltaFileStatuses(Seq(1L, 2L, 3L)), // w/o checkpoint singularCheckpointFileStatuses(Seq(2L)) ++ deltaFileStatuses(Seq(1L, 2L, 3L)) // w/checkpoint ).foreach { deltaFiles => // Valid queries checkGetActiveCommitAtTimestamp(deltaFiles, 10, 1, mustBeRecreatable = false) checkGetActiveCommitAtTimestamp(deltaFiles, 11, 1, mustBeRecreatable = false) checkGetActiveCommitAtTimestamp(deltaFiles, 20, 2, mustBeRecreatable = false) checkGetActiveCommitAtTimestamp(deltaFiles, 21, 2, mustBeRecreatable = false) checkGetActiveCommitAtTimestamp(deltaFiles, 30, 3, mustBeRecreatable = false) // Invalid queries checkGetActiveCommitAtTimestampError[RuntimeException]( deltaFiles, latestVersion = 3L, -1, DeltaErrors.timestampBeforeFirstAvailableCommit(dataPath.toString, -1, 10, 1).getMessage, mustBeRecreatable = false) checkGetActiveCommitAtTimestampError[RuntimeException]( deltaFiles, latestVersion = 3L, 31, DeltaErrors.timestampAfterLatestCommit(dataPath.toString, 31, 30, 3).getMessage, mustBeRecreatable = false) // Valid queries with canReturnLastCommit=true and canReturnEarliestCommit=true checkGetActiveCommitAtTimestamp( deltaFiles, 0, 1, mustBeRecreatable = false, canReturnEarliestCommit = true) checkGetActiveCommitAtTimestamp( deltaFiles, 31, 3, mustBeRecreatable = false, canReturnLastCommit = true) } } test("getActiveCommitAtTimestamp: mustBeRecreatable=false error cases") { /* ---------- TABLE NOT FOUND --------- */ // Non-existent path intercept[TableNotFoundException]( getActiveCommitAtTimestamp( createMockFSListFromEngine(p => throw new FileNotFoundException(p)), getMockSnapshot(dataPath, latestVersion = 1L), logPath, timestamp = 0, mustBeRecreatable = false)) // Empty _delta_log directory intercept[TableNotFoundException]( getActiveCommitAtTimestamp( createMockFSListFromEngine(p => Seq()), getMockSnapshot(dataPath, latestVersion = 1L), logPath, timestamp = 0)) /* ---------- CORRUPT LISTINGS --------- */ // No commit files at all (only checkpoint files) checkGetActiveCommitAtTimestampError[RuntimeException]( singularCheckpointFileStatuses(Seq(1L)), latestVersion = 1L, 25, "No delta files found in the directory", mustBeRecreatable = false) // No delta files checkGetActiveCommitAtTimestampError[RuntimeException]( Seq("foo", "notdelta.parquet", "foo.json", "001.checkpoint.00f.oo0.parquet") .map(new Path(logPath, _)) .map(path => FileStatus.of(path.toString, 10, 10)), latestVersion = 1L, 25, "No delta files found in the directory", mustBeRecreatable = false) } // ========== ICT TIME TRAVEL TESTS ========== /** * Common function to test getActiveCommitAtTimestamp with all combinations of boolean flags. * This reduces duplication and ensures comprehensive testing of flag combinations. */ def checkGetActiveCommitAtTimestampWithAllFlags( fileList: Seq[FileStatus], timestamp: Long, expectedVersion: Long, ictEnablementInfoOpt: Option[(Long, Long)] = None, shouldSucceed: Boolean = true, expectedErrorMessageContains: String = ""): Unit = { val lastDelta = fileList.map(_.getPath).filter(FileNames.isCommitFile).last val latestVersion = FileNames.getFileVersion(new Path(lastDelta)) // Test all combinations of boolean flags val flagCombinations = for { mustBeRecreatable <- Seq(true, false) canReturnLastCommit <- Seq(true, false) canReturnEarliestCommit <- Seq(true, false) } yield (mustBeRecreatable, canReturnLastCommit, canReturnEarliestCommit) flagCombinations.foreach { case (mustBeRecreatable, canReturnLastCommit, canReturnEarliestCommit) => if (shouldSucceed) { val activeCommit = getActiveCommitAtTimestamp( createMockFSListFromEngine(fileList), getMockSnapshot(dataPath, latestVersion = latestVersion, ictEnablementInfoOpt), logPath, timestamp, mustBeRecreatable, canReturnLastCommit, canReturnEarliestCommit) assert( activeCommit.getVersion == expectedVersion, s"Expected version $expectedVersion but got ${activeCommit.getVersion} " + s"for timestamp=$timestamp with flags: " + s"mustBeRecreatable=$mustBeRecreatable, " + s"canReturnLastCommit=$canReturnLastCommit, " + s"canReturnEarliestCommit=$canReturnEarliestCommit") } else { val e = intercept[Exception] { getActiveCommitAtTimestamp( createMockFSListFromEngine(fileList), getMockSnapshot(dataPath, latestVersion = latestVersion, ictEnablementInfoOpt), logPath, timestamp, mustBeRecreatable, canReturnLastCommit, canReturnEarliestCommit) } assert( e.getMessage.contains(expectedErrorMessageContains), s"Expected error message to contain " + s"'$expectedErrorMessageContains' but got '${e.getMessage}' " + s"with flags: " + s"mustBeRecreatable=$mustBeRecreatable, " + s"canReturnLastCommit=$canReturnLastCommit, " + s"canReturnEarliestCommit=$canReturnEarliestCommit") } } } /** * Common function to test ICT time travel scenarios. */ def testICTTimeTravelScenario( icts: Seq[Long], modTimes: Seq[Long], ictEnablementVersion: Long, testCases: Seq[(Long, Long, String)] // (searchTimestamp, expectedVersion, description) ): Unit = { val deltasWithModTimes = modTimes.zipWithIndex.map { case (ts, v) => FileStatus.of( FileNames.deltaFile(logPath, v), 1, /* size */ ts) } val deltaToICTMap = icts.zipWithIndex.map { case (ts, v) => v.toLong -> ts }.toMap val engine = createMockFSAndJsonEngineForICT(deltasWithModTimes, deltaToICTMap) val mockSnapshot = getMockSnapshot( dataPath, latestVersion = icts.size - 1, Some((ictEnablementVersion, deltaToICTMap(ictEnablementVersion)))) testCases.foreach { case (timestamp, expectedVersion, description) => val activeCommit = getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp) assert( activeCommit.getVersion == expectedVersion, s"$description: Expected version $expectedVersion " + s"but got ${activeCommit.getVersion} for timestamp=$timestamp") // Verify timestamp is correct based on ICT enablement val expectedTimestamp = if (expectedVersion >= ictEnablementVersion) { icts(expectedVersion.toInt) } else { modTimes(expectedVersion.toInt) } assert( activeCommit.getTimestamp == expectedTimestamp, s"$description: Expected timestamp $expectedTimestamp but got ${activeCommit.getTimestamp}") } } test("ICT time travel: comprehensive enablement scenarios") { val icts = Seq(1L, 11L, 21L, 31L, 50L, 60L) val modTimes = Seq(4L, 14L, 24L, 34L, 54L, 64L) // Test enablement at version 0 (entire history has ICT) testICTTimeTravelScenario( icts, modTimes, ictEnablementVersion = 0L, Seq( (1L, 0L, "Exact match at first ICT"), (5L, 0L, "Between first and second ICT"), (11L, 1L, "Exact match at second ICT"), (25L, 2L, "Between third and fourth ICT"), (60L, 5L, "Exact match at last ICT"))) // Test enablement at version 1 (mixed ICT/non-ICT) testICTTimeTravelScenario( icts, modTimes, ictEnablementVersion = 1L, Seq( (4L, 0L, "Non-ICT commit using modification time"), (11L, 1L, "First ICT commit"), (25L, 2L, "Between ICT commits"), (31L, 3L, "Exact ICT match"), (60L, 5L, "Last ICT commit"))) // Test enablement at version 3 (mixed ICT/non-ICT) testICTTimeTravelScenario( icts, modTimes, ictEnablementVersion = 3L, Seq( (4L, 0L, "Non-ICT commit"), (14L, 1L, "Non-ICT commit"), (24L, 2L, "Non-ICT commit before enablement"), (31L, 3L, "First ICT commit"), (50L, 4L, "ICT commit"), (60L, 5L, "Last ICT commit"))) // Test enablement at last version testICTTimeTravelScenario( icts, modTimes, ictEnablementVersion = 5L, Seq( (4L, 0L, "Non-ICT commit"), (54L, 4L, "Non-ICT commit before enablement"), (60L, 5L, "Only ICT commit"))) } test("ICT time travel: boundary conditions and edge cases") { val icts = Seq(10L, 20L, 30L, 40L, 50L) val modTimes = Seq(5L, 15L, 25L, 35L, 45L) val deltasWithModTimes = modTimes.zipWithIndex.map { case (ts, v) => FileStatus.of(FileNames.deltaFile(logPath, v), 1, ts) } val deltaToICTMap = icts.zipWithIndex.map { case (ts, v) => v.toLong -> ts }.toMap val engine = createMockFSAndJsonEngineForICT(deltasWithModTimes, deltaToICTMap) // Test with ICT enabled from version 0 val mockSnapshot = getMockSnapshot( dataPath, latestVersion = icts.size - 1, Some((0L, deltaToICTMap(0L)))) // Test timestamp exactly at ICT enablement val activeCommit1 = getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 10L) assert(activeCommit1.getVersion == 0L) assert(activeCommit1.getTimestamp == 10L) // Test timestamp just before first ICT val activeCommit2 = getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 9L, canReturnEarliestCommit = true) assert(activeCommit2.getVersion == 0L) // Should return earliest commit assert(activeCommit2.getTimestamp == 10L) // Test timestamp just after last ICT intercept[io.delta.kernel.exceptions.KernelException] { getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 51L) } // Test with canReturnLastCommit=true val activeCommit3 = getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 51L, canReturnLastCommit = true) assert(activeCommit3.getVersion == 4L) assert(activeCommit3.getTimestamp == 50L) } test("ICT time travel: latest snapshot timestamp optimization") { val icts = Seq(10L, 20L, 30L) val modTimes = Seq(5L, 15L, 25L) val deltasWithModTimes = modTimes.zipWithIndex.map { case (ts, v) => FileStatus.of(FileNames.deltaFile(logPath, v), 1, ts) } val deltaToICTMap = icts.zipWithIndex.map { case (ts, v) => v.toLong -> ts }.toMap val engine = createMockFSAndJsonEngineForICT(deltasWithModTimes, deltaToICTMap) val mockSnapshot = getMockSnapshot( dataPath, latestVersion = icts.size - 1, Some((0L, deltaToICTMap(0L)))) // Test timestamp equal to latest snapshot timestamp val activeCommit1 = getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 30L) assert(activeCommit1.getVersion == 2L) assert(activeCommit1.getTimestamp == 30L) // Test timestamp greater than latest snapshot timestamp intercept[io.delta.kernel.exceptions.KernelException] { getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 35L) } // Test with canReturnLastCommit=true val activeCommit2 = getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 35L, canReturnLastCommit = true) assert(activeCommit2.getVersion == 2L) assert(activeCommit2.getTimestamp == 30L) } test("ICT time travel: mixed ICT and non-ICT commits with truncated log") { val icts = Seq(100L, 200L, 300L, 400L) // ICT enabled from version 2 val modTimes = Seq(50L, 150L, 250L, 350L) // Simulate truncated log starting from version 1 val deltasWithModTimes = modTimes.drop(1).zipWithIndex.map { case (ts, v) => FileStatus.of(FileNames.deltaFile(logPath, v + 1), 1, ts) } // Add a checkpoint file at version 1 to make the table recreatable val checkpointFile = FileStatus.of( FileNames.checkpointFileSingular(logPath, 1L).toString, 1, 150L) val allFiles = checkpointFile +: deltasWithModTimes val deltaToICTMap = icts.zipWithIndex.map { case (ts, v) => v.toLong -> ts }.toMap val engine = createMockFSAndJsonEngineForICT(allFiles, deltaToICTMap) val mockSnapshot = getMockSnapshot( dataPath, latestVersion = 3L, Some((2L, deltaToICTMap(2L))) ) // ICT enabled at version 2 // Test timestamp before ICT enablement (should use modification time) val activeCommit1 = getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 150L) assert(activeCommit1.getVersion == 1L) assert(activeCommit1.getTimestamp == 150L) // modification time // Test timestamp after ICT enablement (should use ICT) val activeCommit2 = getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 300L) assert(activeCommit2.getVersion == 2L) assert(activeCommit2.getTimestamp == 300L) // ICT // Test timestamp between ICT commits val activeCommit3 = getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 350L) assert(activeCommit3.getVersion == 2L) assert(activeCommit3.getTimestamp == 300L) // Should return previous ICT commit } test("ICT time travel: non-ICT commits missing scenario") { val icts = Seq(100L, 200L, 300L) val modTimes = Seq(50L, 150L, 250L) // Simulate scenario where ICT enablement version <= earliest available version val deltasWithModTimes = modTimes.drop(2).zipWithIndex.map { case (ts, v) => FileStatus.of(FileNames.deltaFile(logPath, v + 2), 1, ts) } // Add a checkpoint file at version 2 to make the table recreatable val checkpointFile = FileStatus.of( FileNames.checkpointFileSingular(logPath, 2L).toString, 1, 250L) val allFiles = checkpointFile +: deltasWithModTimes val deltaToICTMap = icts.zipWithIndex.map { case (ts, v) => v.toLong -> ts }.toMap val engine = createMockFSAndJsonEngineForICT(allFiles, deltaToICTMap) val mockSnapshot = getMockSnapshot( dataPath, latestVersion = 2L, Some((1L, deltaToICTMap(1L))) ) // ICT enabled at version 1, but earliest available is 2 // Test timestamp before ICT enablement but non-ICT commits are missing // Should return earliest available commit with its ICT val activeCommit1 = getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 50L, canReturnEarliestCommit = true) assert(activeCommit1.getVersion == 2L) assert(activeCommit1.getTimestamp == 300L) // ICT of earliest available commit // Test error case when canReturnEarliestCommit=false intercept[io.delta.kernel.exceptions.KernelException] { getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 50L) } } test("ICT time travel: binary search edge cases") { // Test with odd number of commits val icts = Seq(1L, 11L, 21L, 31L, 50L, 60L, 70L) val modTimes = Seq(4L, 14L, 24L, 34L, 54L, 64L, 74L) val deltasWithModTimes = modTimes.zipWithIndex.map { case (ts, v) => FileStatus.of(FileNames.deltaFile(logPath, v), 1, ts) } val deltaToICTMap = icts.zipWithIndex.map { case (ts, v) => v.toLong -> ts }.toMap val engine = createMockFSAndJsonEngineForICT(deltasWithModTimes, deltaToICTMap) val mockSnapshot = getMockSnapshot( dataPath, latestVersion = icts.size - 1, Some((0L, deltaToICTMap(0L)))) // Test searchTimestamp is the exact match with the middle commit val activeCommit1 = getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 31L // Exact match with version 3 ) assert(activeCommit1.getVersion == 3L) // Test searchTimestamp = start case val activeCommit2 = getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 1L // First ICT ) assert(activeCommit2.getVersion == 0L) // Test searchTimestamp = end case val activeCommit3 = getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 70L // Last ICT ) assert(activeCommit3.getVersion == 6L) // Test with even number of commits val ictsEven = Seq(1L, 11L, 21L, 31L, 50L, 60L) val modTimesEven = Seq(4L, 14L, 24L, 34L, 54L, 64L) val deltasWithModTimesEven = modTimesEven.zipWithIndex.map { case (ts, v) => FileStatus.of(FileNames.deltaFile(logPath, v), 1, ts) } val deltaToICTMapEven = ictsEven.zipWithIndex.map { case (ts, v) => v.toLong -> ts }.toMap val engineEven = createMockFSAndJsonEngineForICT(deltasWithModTimesEven, deltaToICTMapEven) val mockSnapshotEven = getMockSnapshot( dataPath, latestVersion = ictsEven.size - 1, Some((0L, deltaToICTMapEven(0L)))) val activeCommit4 = getActiveCommitAtTimestamp( engineEven, mockSnapshotEven, logPath, timestamp = 25L // Between version 2 and 3 ) assert(activeCommit4.getVersion == 2L) assert(activeCommit4.getTimestamp == 21L) } test("ICT time travel: single commit scenario") { val icts = Seq(100L) val modTimes = Seq(50L) val deltasWithModTimes = modTimes.zipWithIndex.map { case (ts, v) => FileStatus.of(FileNames.deltaFile(logPath, v), 1, ts) } val deltaToICTMap = icts.zipWithIndex.map { case (ts, v) => v.toLong -> ts }.toMap val engine = createMockFSAndJsonEngineForICT(deltasWithModTimes, deltaToICTMap) val mockSnapshot = getMockSnapshot( dataPath, latestVersion = 0L, Some((0L, deltaToICTMap(0L)))) // Test exact match val activeCommit1 = getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 100L) assert(activeCommit1.getVersion == 0L) assert(activeCommit1.getTimestamp == 100L) // Test timestamp before single commit val activeCommit2 = getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 50L, canReturnEarliestCommit = true) assert(activeCommit2.getVersion == 0L) assert(activeCommit2.getTimestamp == 100L) // Test timestamp after single commit val activeCommit3 = getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 150L, canReturnLastCommit = true) assert(activeCommit3.getVersion == 0L) assert(activeCommit3.getTimestamp == 100L) } test("ICT time travel: modification times out of order") { val icts = Seq(10L, 20L, 30L, 40L) val modTimes = Seq(40L, 30L, 20L, 10L) // Reversed modification times testICTTimeTravelScenario( icts, modTimes, ictEnablementVersion = 0L, Seq( (10L, 0L, "First ICT despite reversed mod times"), (15L, 0L, "Between first and second ICT"), (20L, 1L, "Second ICT"), (40L, 3L, "Last ICT"))) } test("ICT time travel: comprehensive boolean flag combinations") { val icts = Seq(10L, 20L, 30L) val modTimes = Seq(5L, 15L, 25L) val deltasWithModTimes = modTimes.zipWithIndex.map { case (ts, v) => FileStatus.of(FileNames.deltaFile(logPath, v), 1, ts) } val deltaToICTMap = icts.zipWithIndex.map { case (ts, v) => v.toLong -> ts }.toMap val engine = createMockFSAndJsonEngineForICT(deltasWithModTimes, deltaToICTMap) val mockSnapshot = getMockSnapshot( dataPath, latestVersion = 2L, Some((0L, deltaToICTMap(0L)))) // Test all flag combinations for valid timestamp val flagCombinations = for { mustBeRecreatable <- Seq(true, false) canReturnLastCommit <- Seq(true, false) canReturnEarliestCommit <- Seq(true, false) } yield (mustBeRecreatable, canReturnLastCommit, canReturnEarliestCommit) flagCombinations.foreach { case (mustBeRecreatable, canReturnLastCommit, canReturnEarliestCommit) => val activeCommit = getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 20L, mustBeRecreatable, canReturnLastCommit, canReturnEarliestCommit) assert(activeCommit.getVersion == 1L) assert(activeCommit.getTimestamp == 20L) } // Test edge cases with different flag combinations // Timestamp before earliest commit flagCombinations.foreach { case (mustBeRecreatable, canReturnLastCommit, canReturnEarliestCommit) => if (canReturnEarliestCommit) { val activeCommit = getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 5L, mustBeRecreatable, canReturnLastCommit, canReturnEarliestCommit) assert(activeCommit.getVersion == 0L) } else { intercept[io.delta.kernel.exceptions.KernelException] { getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 5L, mustBeRecreatable, canReturnLastCommit, canReturnEarliestCommit) } } } // Timestamp after latest commit flagCombinations.foreach { case (mustBeRecreatable, canReturnLastCommit, canReturnEarliestCommit) => if (canReturnLastCommit) { val activeCommit = getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 35L, mustBeRecreatable, canReturnLastCommit, canReturnEarliestCommit) assert(activeCommit.getVersion == 2L) } else { intercept[io.delta.kernel.exceptions.KernelException] { getActiveCommitAtTimestamp( engine, mockSnapshot, logPath, timestamp = 35L, mustBeRecreatable, canReturnLastCommit, canReturnEarliestCommit) } } } } test("ICT time travel: error handling and edge cases") { val icts = Seq(10L, 20L, 30L) val modTimes = Seq(5L, 15L, 25L) val deltasWithModTimes = modTimes.zipWithIndex.map { case (ts, v) => FileStatus.of(FileNames.deltaFile(logPath, v), 1, ts) } val deltaToICTMap = icts.zipWithIndex.map { case (ts, v) => v.toLong -> ts }.toMap val engine = createMockFSAndJsonEngineForICT(deltasWithModTimes, deltaToICTMap) // Test with ICT not enabled val nonICTSnapshot = getMockSnapshot(dataPath, latestVersion = 2L, None) val activeCommit1 = getActiveCommitAtTimestamp( createMockFSListFromEngine(deltasWithModTimes), nonICTSnapshot, logPath, timestamp = 15L) assert(activeCommit1.getVersion == 1L) assert(activeCommit1.getTimestamp == 15L) // Should use modification time // Test with malformed ICT enablement info (only version set) val malformedConfig = Map( TableConfig.IN_COMMIT_TIMESTAMPS_ENABLED.getKey -> "true", TableConfig.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.getKey -> "1" // Missing enablement timestamp ) val malformedMetadata = new Metadata( "id", Optional.empty(), Optional.empty(), new Format(), testSchema.toJson, testSchema, buildArrayValue(java.util.Arrays.asList("c3"), StringType.STRING), Optional.of(123), stringStringMapValue(malformedConfig.asJava)) val malformedSnapshot = new SnapshotImpl( dataPath, 2L, new Lazy(() => new LogSegment( logPath, 2L, Seq(deltaFileStatus(2L)).asJava, Seq.empty.asJava, Seq.empty.asJava, deltaFileStatus(2L), Optional.empty(), /* lastSeenChecksum */ Optional.empty() /* maxPublishedDeltaVersion */ )), null, /* logReplay */ new Protocol(1, 2), malformedMetadata, DefaultFileSystemManagedTableOnlyCommitter.INSTANCE, SnapshotQueryContext.forLatestSnapshot(dataPath.toString), Optional.empty() /* inCommitTimestampOpt */ ) intercept[IllegalStateException] { getActiveCommitAtTimestamp( engine, malformedSnapshot, logPath, timestamp = 15L) } } test("greatestLowerBound: basic functionality") { // Test with a simple sequence of timestamps val timestamps = Seq(1L, 3L, 5L, 7L, 9L) val indexToValueMapper = new java.util.function.Function[java.lang.Long, java.lang.Long] { override def apply(index: java.lang.Long): java.lang.Long = timestamps(index.toInt) } // Test exact match (should return index 2, value 5) val result1 = InCommitTimestampUtils.greatestLowerBound(5L, 0L, 4L, indexToValueMapper) assert(result1.isPresent) assert(result1.get._1 == 2L) assert(result1.get._2 == 5L) // Test between values (should return index 1, value 3) val result2 = InCommitTimestampUtils.greatestLowerBound(4L, 0L, 4L, indexToValueMapper) assert(result2.isPresent) assert(result2.get._1 == 1L) assert(result2.get._2 == 3L) // Test before first value (should not be present) val result3 = InCommitTimestampUtils.greatestLowerBound(0L, 0L, 4L, indexToValueMapper) assert(!result3.isPresent) // Test after last value (should return last index/value) val result4 = InCommitTimestampUtils.greatestLowerBound(10L, 0L, 4L, indexToValueMapper) assert(result4.isPresent) assert(result4.get._1 == 4L) assert(result4.get._2 == 9L) } test("greatestLowerBound: single element in search range") { // Test with single element val singleElement = Seq(5L) val singleElementMapper = new java.util.function.Function[java.lang.Long, java.lang.Long] { override def apply(index: java.lang.Long): java.lang.Long = singleElement(index.toInt) } // Target equals the element val result1 = InCommitTimestampUtils.greatestLowerBound(5L, 0L, 0L, singleElementMapper) assert(result1.isPresent) assert(result1.get._1 == 0L) assert(result1.get._2 == 5L) // Target less than the element val result2 = InCommitTimestampUtils.greatestLowerBound(4L, 0L, 0L, singleElementMapper) assert(!result2.isPresent) // Target greater than the element val result3 = InCommitTimestampUtils.greatestLowerBound(6L, 0L, 0L, singleElementMapper) assert(result3.isPresent) assert(result3.get._1 == 0L) assert(result3.get._2 == 5L) // Test with empty range (should not be present) val result4 = InCommitTimestampUtils.greatestLowerBound(5L, 1L, 0L, singleElementMapper) assert(!result4.isPresent) } test("greatestLowerBound: binary search correctness") { // Test with a larger sequence to verify binary search correctness val timestamps = (0L until 100L by 2).toSeq // 0, 2, 4, ..., 98 val indexToValueMapper = new java.util.function.Function[java.lang.Long, java.lang.Long] { override def apply(index: java.lang.Long): java.lang.Long = timestamps(index.toInt) } // Test various positions in the sequence (exact matches) for (i <- 0 until 50) { val target = i * 2L val result = InCommitTimestampUtils.greatestLowerBound(target, 0L, 49L, indexToValueMapper) assert(result.isPresent) assert(result.get._1 == i) assert(result.get._2 == target) } // Test values between elements (should return the lower index/value) for (i <- 1 until 50) { val target = i * 2L - 1 val result = InCommitTimestampUtils.greatestLowerBound(target, 0L, 49L, indexToValueMapper) assert(result.isPresent) assert(result.get._1 == i - 1) assert(result.get._2 == (i - 1) * 2L) } // Test value less than all (should not be present) val resultLow = InCommitTimestampUtils.greatestLowerBound(-1L, 0L, 49L, indexToValueMapper) assert(!resultLow.isPresent) // Test value greater than all (should return last index/value) val resultHigh = InCommitTimestampUtils.greatestLowerBound(100L, 0L, 49L, indexToValueMapper) assert(resultHigh.isPresent) assert(resultHigh.get._1 == 49L) assert(resultHigh.get._2 == 98L) } // ============== Tests for Staged Commits Support =============== private def checkGetActiveCommitAtTimestampWithParsedLogData( fileList: Seq[FileStatus], catalogCommits: Seq[ParsedCatalogCommitData], versionToICT: Map[Long, Long], timestampToQuery: Long, expectedVersion: Long, canReturnLastCommit: Boolean = false, canReturnEarliestCommit: Boolean = false, add10ToICTForStagedFiles: Boolean = false, ictEnablementInfo: (Long, Long) = (0, 0)): Unit = { // Create mock engine with ICT reading support val mockJsonHandler = new MockReadICTFileJsonHandler(versionToICT, add10ToICTForStagedFiles) val mockedEngine = mockEngine( fileSystemClient = new MockListFromFileSystemClient(listFromProvider(fileList)), jsonHandler = mockJsonHandler) def getVersionFromFS(fs: FileStatus): Long = FileNames.getFileVersion(new Path(fs.getPath)) val latestVersion = fileList.map(getVersionFromFS(_)).max // If we have a ratified commit file at the end version, we want to use this in the log segment // for our mockLatestSnapshot, so we get the ICT from that file val deltaFileAtEndVersion = fileList .filter(fs => FileNames.isStagedDeltaFile(fs.getPath)) .find(getVersionFromFS(_) == latestVersion) val mockLatestSnapshot = getMockSnapshot( dataPath, latestVersion = latestVersion, ictEnablementInfoOpt = Some(ictEnablementInfo), deltaFileAtEndVersion = deltaFileAtEndVersion) val activeCommit = DeltaHistoryManager.getActiveCommitAtTimestamp( mockedEngine, mockLatestSnapshot, logPath, timestampToQuery, false, canReturnLastCommit, canReturnEarliestCommit, catalogCommits.asJava) assert( activeCommit.getVersion == expectedVersion, s"Expected version $expectedVersion but got ${activeCommit.getVersion} " + s"for timestamp=$timestampToQuery") } test("getActiveCommitAtTimestamp with catalog commits: empty log, 1 ratified commit") { // Published commits: _ // Ratified commits: V0 val catalogCommitFiles = Seq(stagedCommitFile(0L)) val parsedLogData = catalogCommitFiles.map(ParsedCatalogCommitData.forFileStatus(_)) val versionToICT = Map(0L -> 180L) // Query the exact timestamp checkGetActiveCommitAtTimestampWithParsedLogData( catalogCommitFiles, parsedLogData, versionToICT, timestampToQuery = versionToICT(0), expectedVersion = 0) // Querying before without canReturnEarliestCommit results in error intercept[KernelException] { checkGetActiveCommitAtTimestampWithParsedLogData( catalogCommitFiles, parsedLogData, versionToICT, timestampToQuery = versionToICT(0) - 10, expectedVersion = 0) } // Querying before with canReturnEarliestCommit passes checkGetActiveCommitAtTimestampWithParsedLogData( catalogCommitFiles, parsedLogData, versionToICT, timestampToQuery = versionToICT(0) - 10, expectedVersion = 0, canReturnEarliestCommit = true) // Querying after without canReturnLatestCommit results in error intercept[KernelException] { checkGetActiveCommitAtTimestampWithParsedLogData( catalogCommitFiles, parsedLogData, versionToICT, timestampToQuery = versionToICT(0) + 10, expectedVersion = 0) } // Querying after with canReturnLatestCommit passes checkGetActiveCommitAtTimestampWithParsedLogData( catalogCommitFiles, parsedLogData, versionToICT, timestampToQuery = versionToICT(0) + 10, expectedVersion = 0, canReturnLastCommit = true) } test("getActiveCommitAtTimestamp with catalog commits: empty log, 2 ratified commit") { // Published commits: _ // Ratified commits: V0, V1 val catalogCommits = Seq(stagedCommitFile(0L), stagedCommitFile(1L)) val parsedLogData = catalogCommits.map(ParsedCatalogCommitData.forFileStatus(_)) val versionToICT = Map(0L -> 180L, 1L -> 280L) // Query the exact timestamp of V0 checkGetActiveCommitAtTimestampWithParsedLogData( catalogCommits, parsedLogData, versionToICT, timestampToQuery = 180L, expectedVersion = 0) // Query in between V0 and V1 checkGetActiveCommitAtTimestampWithParsedLogData( catalogCommits, parsedLogData, versionToICT, timestampToQuery = 200L, expectedVersion = 0) // Query the exact timestamp of V1 checkGetActiveCommitAtTimestampWithParsedLogData( catalogCommits, parsedLogData, versionToICT, timestampToQuery = 280L, expectedVersion = 1) } test("getActiveCommitAtTimestamp with catalog commits: no overlap") { // Published commits: V0, V1 // Ratified commits: V2, V3 val catalogCommitFiles = Seq(stagedCommitFile(2L), stagedCommitFile(3L)) val fileList = Seq( deltaFileStatus(0), deltaFileStatus(1)) ++ catalogCommitFiles val versionToICT = Map(0L -> 180L, 1L -> 280L, 2L -> 380L, 3L -> 480L) val parsedLogData = catalogCommitFiles.map(ParsedCatalogCommitData.forFileStatus(_)) // Query the exact timestamp of V1 checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 280L, expectedVersion = 1) // Query in between V1 and V2 checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 300L, expectedVersion = 1) // Query the exact timestamp of V2 checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 380L, expectedVersion = 2) // Query in between V2 and V3 checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 400L, expectedVersion = 2) // Query the exact timestamp of V3 checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 480L, expectedVersion = 3) } test("getActiveCommitAtTimestamp with catalog commits: " + "v0 published and ratified => prefer ratified") { // Published commits: V0 // Ratified commits: V0 val catalogCommitFiles = Seq(stagedCommitFile(0L)) val parsedLogData = catalogCommitFiles.map(ParsedCatalogCommitData.forFileStatus(_)) val fileList = Seq(deltaFileStatus(0)) ++ catalogCommitFiles val versionToICT = Map(0L -> 200L) // If we read from the published file, we should get ICT=200 // If we read from the ratified file, we should get ICT=210 (correct behavior!) checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 210L, expectedVersion = 0, add10ToICTForStagedFiles = true) intercept[KernelException] { checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 200L, expectedVersion = 0, add10ToICTForStagedFiles = true) } } test("getActiveCommitAtTimestamp with catalog commits: overlap => prefer ratified") { // Published commits: V10, V11 // Ratified commits: V11, V12 val catalogCommitFiles = Seq(stagedCommitFile(11), stagedCommitFile(12)) val parsedLogData = catalogCommitFiles.map(ParsedCatalogCommitData.forFileStatus(_)) val fileList = Seq( classicCheckpointFileStatus(10), deltaFileStatus(10), deltaFileStatus(11)) ++ catalogCommitFiles val versionToICT = Map(10L -> 1000L, 11L -> 1100L, 12L -> 1200L) // We have v10=1000, v11=1110 (if we use the ratified commit), v12=1210 // Read at v10 checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 1000L, expectedVersion = 10, add10ToICTForStagedFiles = true) // Read between v10 and v11 (if we incorrectly use the published file this will fail!) checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 1101L, expectedVersion = 10, add10ToICTForStagedFiles = true) // Read at v11 checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 1110L, expectedVersion = 11, add10ToICTForStagedFiles = true) // Read between v11 and v12 checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 1150, expectedVersion = 11, add10ToICTForStagedFiles = true) // Read at v12 checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 1210, expectedVersion = 12, add10ToICTForStagedFiles = true) } test("getActiveCommitAtTimestamp with catalog commits: " + "discontinuous catalog commits => prefer ratified") { // Published commits: V0, V1, V2 // Ratified commits: V0, V2 val catalogCommitFiles = Seq(stagedCommitFile(0), stagedCommitFile(2)) val parsedLogData = catalogCommitFiles.map(ParsedCatalogCommitData.forFileStatus(_)) val fileList = Seq( deltaFileStatus(0), deltaFileStatus(1), deltaFileStatus(2)) ++ catalogCommitFiles val versionToICT = Map(0L -> 1000L, 1L -> 2000L, 2L -> 3000L) // We have v0=1010, v1=2000, v2=3010 assuming we use the ratified commits > published commits // Read at published file ICT for v0 should fail intercept[KernelException] { checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 1000L, expectedVersion = 0, add10ToICTForStagedFiles = true) } // Read at correct version for v0 if using staged commit checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 1010L, expectedVersion = 0L, add10ToICTForStagedFiles = true) // Read between v0 and v1 checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 1500L, expectedVersion = 0, add10ToICTForStagedFiles = true) // Read at v1 checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 2000L, expectedVersion = 1, add10ToICTForStagedFiles = true) // Read between v1 and v2 (this will fail if we don't use the ratified commit) checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 3000L, expectedVersion = 1, add10ToICTForStagedFiles = true) // Read at v2 checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 3010L, expectedVersion = 2, add10ToICTForStagedFiles = true) } test("getActiveCommitAtTimestamp with catalog commits: ICT enabled after v0") { // Published commits: V0 (non-ICT), V1 (enables ICT) // Ratified commits: V2 val catalogCommitFiles = Seq(stagedCommitFile(2)) val parsedLogData = catalogCommitFiles.map(ParsedCatalogCommitData.forFileStatus(_)) val fileList = Seq( deltaFileStatus(0), deltaFileStatus(1)) ++ catalogCommitFiles val versionToICT = Map(1L -> 2000L, 2L -> 3000L) val ictEnablementInfo = (1L, 2000L) // (version, timestamp) // Query exact timestamp of v0 (no ICT) checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 0L, expectedVersion = 0, ictEnablementInfo = ictEnablementInfo) // Query between v0 and v1 checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 8L, expectedVersion = 0, ictEnablementInfo = ictEnablementInfo) // TODO: this fails due to an existing bug when querying a timestamp between // (ictEnablementVersionFsTs, ictEnablementTs) -- re-enable this once it's fixed // Query between v0 and v1 - ICT based /* checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, 500L, 0, ictEnablementInfo = ictEnablementInfo) */ // Query exact timestamp of v1 (ICT) checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 2000L, expectedVersion = 1, ictEnablementInfo = ictEnablementInfo) // Query between v1 and v2 checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 2500L, expectedVersion = 1, ictEnablementInfo = ictEnablementInfo) // Query exact v2 checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 3000L, expectedVersion = 2, ictEnablementInfo = ictEnablementInfo) } test("getActiveCommitAtTimestamp with catalog commits: ICT enabled after v0 and " + "only ICT commits available") { // This tests the scenario where we are searching for a pre-ICT time but all the non-ICT commits // are missing. This throws an error based on `canReturnEarliestCommit`. // Published commits: v10 // Ratified commits: V11 val catalogCommitFiles = Seq(stagedCommitFile(11)) val parsedLogData = catalogCommitFiles.map(ParsedCatalogCommitData.forFileStatus(_)) val fileList = Seq( classicCheckpointFileStatus(10), deltaFileStatus(10)) ++ catalogCommitFiles val versionToICT = Map(10L -> 1000L, 11L -> 1100L) val ictEnablementInfo = (5L, 500L) // (version, timestamp) // If we have canReturnEarliestCommit=false should fail // Querying after without canReturnLatestCommit results in error val e = intercept[KernelException] { checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 400, expectedVersion = 0, ictEnablementInfo = ictEnablementInfo) } assert(e.getMessage.contains("is before the earliest available version 10. Please use a " + "timestamp greater than or equal to 1000 ms")) // Query with canReturnEarliestCommit=true should pass checkGetActiveCommitAtTimestampWithParsedLogData( fileList, parsedLogData, versionToICT, timestampToQuery = 400, expectedVersion = 10, canReturnEarliestCommit = true, ictEnablementInfo = ictEnablementInfo) } test("getActiveCommitAtTimestamp rejects non-ratified staged commits") { val fileList = Seq( classicCheckpointFileStatus(0), deltaFileStatus(0)) // Test 1: Inline commits (non-materialized) should be rejected val mockColumnarBatch = new ColumnarBatch { override def getSchema: StructType = null override def getColumnVector(ordinal: Int): ColumnVector = null override def getSize: Int = 1 } val inlineCommit = ParsedCatalogCommitData.forInlineData(1L, mockColumnarBatch) val inlineData = Seq(inlineCommit).asJava assertThrows[IllegalArgumentException] { // Args don't matter as validation should fail immediately DeltaHistoryManager.getActiveCommitAtTimestamp( createMockFSListFromEngine(fileList), getMockSnapshot(dataPath, latestVersion = 0), logPath, 10, false, false, false, inlineData) } } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/DeltaLogActionUtilsSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal import java.io.FileNotFoundException import java.util.{Collections, Optional} import scala.collection.JavaConverters._ import scala.reflect.ClassTag import io.delta.kernel.exceptions.{CommitRangeNotFoundException, InvalidTableException, KernelException, TableNotFoundException} import io.delta.kernel.internal.DeltaLogActionUtils.{getCommitFilesForVersionRange, listDeltaLogFilesAsIter, verifyDeltaVersions} import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.util.FileNames import io.delta.kernel.test.MockFileSystemClientUtils import io.delta.kernel.utils.FileStatus import org.scalatest.funsuite.AnyFunSuite class DeltaLogActionUtilsSuite extends AnyFunSuite with MockFileSystemClientUtils { /////////////////////////////// // verifyDeltaVersions tests // /////////////////////////////// def getCommitFiles(versions: Seq[Long]): java.util.List[FileStatus] = { versions .map(v => FileStatus.of(FileNames.deltaFile(logPath, v), 0, 0)) .asJava } test("verifyDeltaVersions") { // Basic correct use case verifyDeltaVersions( getCommitFiles(Seq(1, 2, 3)), 1, Optional.of(3), dataPath) // Only one version provided verifyDeltaVersions( getCommitFiles(Seq(1)), 1, Optional.of(1), dataPath) // No end version provided verifyDeltaVersions( getCommitFiles(Seq(1, 2, 3, 4)), 1, Optional.empty(), dataPath) // Non-contiguous versions intercept[InvalidTableException] { verifyDeltaVersions( getCommitFiles(Seq(1, 3, 4)), 1, Optional.of(4L), dataPath) } // End-version or start-version not right intercept[KernelException] { verifyDeltaVersions( getCommitFiles(Seq(1, 2, 3)), 0, Optional.of(3L), dataPath) } intercept[KernelException] { verifyDeltaVersions( getCommitFiles(Seq(1, 2, 3)), 1, Optional.of(4L), dataPath) } // Empty versions intercept[KernelException] { verifyDeltaVersions( getCommitFiles(Seq()), 1, Optional.of(4L), dataPath) } // Unsorted or duplicates (shouldn't be possible) intercept[InvalidTableException] { verifyDeltaVersions( getCommitFiles(Seq(1, 1, 2)), 1, Optional.of(4L), dataPath) } intercept[InvalidTableException] { verifyDeltaVersions( getCommitFiles(Seq(1, 4, 3, 2)), 1, Optional.of(2L), dataPath) } } ///////////////////////////////////////// // getCommitFilesForVersionRange tests // ///////////////////////////////////////// test("getCommitFilesForVersionRange: directory does not exist") { intercept[TableNotFoundException] { getCommitFilesForVersionRange( createMockFSListFromEngine(_ => throw new FileNotFoundException()), dataPath, 0, Optional.of(1L)) } } def testGetCommitFilesExpectedError[T <: Throwable]( testName: String, files: Seq[FileStatus], startVersion: Long = 1, endVersion: Optional[java.lang.Long] = Optional.of(3L), expectedErrorMessageContains: String)(implicit classTag: ClassTag[T]): Unit = { test("getCommitFilesForVersionRange: " + testName) { val e = intercept[T] { getCommitFilesForVersionRange( createMockFSListFromEngine(files), dataPath, startVersion, endVersion) } assert(e.getMessage.contains(expectedErrorMessageContains)) } } testGetCommitFilesExpectedError[CommitRangeNotFoundException]( testName = "empty directory", files = Seq(), expectedErrorMessageContains = "no log files found in the requested version range") testGetCommitFilesExpectedError[CommitRangeNotFoundException]( testName = "all versions less than startVersion", files = deltaFileStatuses(Seq(0)), expectedErrorMessageContains = "no log files found in the requested version range") testGetCommitFilesExpectedError[CommitRangeNotFoundException]( testName = "all versions greater than endVersion", files = deltaFileStatuses(Seq(4, 5, 6)), expectedErrorMessageContains = "no log files found in the requested version range") testGetCommitFilesExpectedError[CommitRangeNotFoundException]( testName = "all versions less than startVersion no endVersion", files = deltaFileStatuses(Seq(0)), endVersion = Optional.empty(), expectedErrorMessageContains = "no log files found in the requested version range") testGetCommitFilesExpectedError[InvalidTableException]( testName = "missing log files", files = deltaFileStatuses(Seq(1, 3)), expectedErrorMessageContains = "versions are not contiguous") testGetCommitFilesExpectedError[KernelException]( testName = "start version not available", files = deltaFileStatuses(Seq(2, 3, 4, 5)), expectedErrorMessageContains = "no log file found for version 1") testGetCommitFilesExpectedError[KernelException]( testName = "end version not available", files = deltaFileStatuses(Seq(0, 1, 2)), expectedErrorMessageContains = "no log file found for version 3") testGetCommitFilesExpectedError[KernelException]( testName = "invalid start version", files = deltaFileStatuses(Seq(0, 1, 2)), startVersion = -1, expectedErrorMessageContains = "Invalid version range") testGetCommitFilesExpectedError[KernelException]( testName = "invalid end version", files = deltaFileStatuses(Seq(0, 1, 2)), startVersion = 3, endVersion = Optional.of(2L), expectedErrorMessageContains = "Invalid version range") def testGetCommitFiles( testName: String, files: Seq[FileStatus], startVersion: Long = 1, endVersion: Optional[java.lang.Long] = Optional.of(3L), expectedCommitFiles: Seq[FileStatus]): Unit = { test("getCommitFilesForVersionRange: " + testName) { assert( getCommitFilesForVersionRange( createMockFSListFromEngine(files), dataPath, startVersion, endVersion).asScala sameElements expectedCommitFiles) } } testGetCommitFiles( testName = "basic case", files = deltaFileStatuses(Seq(0, 1, 2, 3, 4, 5)), expectedCommitFiles = deltaFileStatuses(Seq(1, 2, 3))) testGetCommitFiles( testName = "basic case with checkpoint file", files = deltaFileStatuses(Seq(0, 1, 2, 3, 4, 5)) ++ singularCheckpointFileStatuses(Seq(2)), expectedCommitFiles = deltaFileStatuses(Seq(1, 2, 3))) testGetCommitFiles( testName = "basic case with non-log files", files = deltaFileStatuses(Seq(0, 1, 2, 3, 4, 5)) ++ deltaFileStatuses(Seq(2)) .map(fs => FileStatus.of(fs.getPath + ".crc", fs.getSize, fs.getModificationTime)), expectedCommitFiles = deltaFileStatuses(Seq(1, 2, 3))) testGetCommitFiles( testName = "version range size 1", files = deltaFileStatuses(Seq(0, 1, 2, 3, 4, 5)), startVersion = 0, endVersion = Optional.of(0L), expectedCommitFiles = deltaFileStatuses(Seq(0))) testGetCommitFiles( testName = "no end version provided - should read to latest available", files = deltaFileStatuses(Seq(0, 1, 2, 3, 4, 5)), endVersion = Optional.empty(), expectedCommitFiles = deltaFileStatuses(Seq(1, 2, 3, 4, 5))) testGetCommitFiles( testName = "no end version provided - single version from start", files = deltaFileStatuses(Seq(2)), startVersion = 2, endVersion = Optional.empty(), expectedCommitFiles = deltaFileStatuses(Seq(2))) ///////////////////////////// // listDeltaLogFiles tests // ///////////////////////////// private val checkpointsAndDeltas = singularCheckpointFileStatuses(Seq(10)) ++ deltaFileStatuses(Seq(10, 11, 12, 13, 14)) ++ Seq(FileStatus.of(s"$logPath/00000000000000000014.crc", 0, 0)) ++ multiCheckpointFileStatuses(Seq(14), 2) ++ deltaFileStatuses(Seq(15, 16, 17)) ++ v2CheckpointFileStatuses(Seq((17, false, 2)), "json").map(_._1) private def extractVersions(files: Seq[FileStatus]): Seq[Long] = { files.map(fs => FileNames.getFileVersion(new Path(fs.getPath))) } test("listDeltaLogFiles: no fileTypes provided") { intercept[IllegalArgumentException] { listDeltaLogFilesAsIter( createMockFSListFromEngine(deltaFileStatuses(Seq(1, 2, 3))), Collections.emptySet(), // No fileTypes provided! dataPath, 1, Optional.empty(), false /* mustBeRecreatable */ ).toInMemoryList } } test("listDeltaLogFiles: returns requested file type only") { val commitFiles = listDeltaLogFilesAsIter( createMockFSListFromEngine(checkpointsAndDeltas), Set(FileNames.DeltaLogFileType.COMMIT).asJava, dataPath, 10, Optional.empty(), false /* mustBeRecreatable */ ).toInMemoryList.asScala assert(commitFiles.forall(fs => FileNames.isCommitFile(fs.getPath))) assert(extractVersions(commitFiles.toSeq) == Seq(10, 11, 12, 13, 14, 15, 16, 17)) val checkpointFiles = listDeltaLogFilesAsIter( createMockFSListFromEngine(checkpointsAndDeltas), Set(FileNames.DeltaLogFileType.CHECKPOINT).asJava, dataPath, 10, Optional.empty(), false /* mustBeRecreatable */ ).toInMemoryList.asScala assert(checkpointFiles.forall(fs => FileNames.isCheckpointFile(fs.getPath))) assert(extractVersions(checkpointFiles.toSeq) == Seq(10, 14, 14, 17)) } test("listDeltaLogFiles: mustBeRecreatable") { val exMsg = intercept[KernelException] { listDeltaLogFilesAsIter( createMockFSListFromEngine(checkpointsAndDeltas), Set(FileNames.DeltaLogFileType.COMMIT, FileNames.DeltaLogFileType.CHECKPOINT).asJava, dataPath, 0, Optional.of(4), true /* mustBeRecreatable */ ).toInMemoryList }.getMessage assert(exMsg.contains("Cannot load table version 4 as the transaction log has been " + "truncated due to manual deletion or the log/checkpoint retention policy. The earliest " + "available version is 10")) } def testListWithCompactions( testName: String, files: Seq[FileStatus], startVersion: Long, endVersion: Optional[java.lang.Long], expectedListedFiles: Seq[FileStatus]): Unit = { test("testListWithCompactions: " + testName) { val listed = listDeltaLogFilesAsIter( createMockFSListFromEngine(files), Set( FileNames.DeltaLogFileType.COMMIT, FileNames.DeltaLogFileType.CHECKPOINT, FileNames.DeltaLogFileType.LOG_COMPACTION).asJava, dataPath, startVersion, endVersion, false /* mustBeRecreatable */ ).toInMemoryList.asScala assert(listed sameElements expectedListedFiles) } } testListWithCompactions( "compaction at start, no endVersion", files = deltaFileStatuses(0L to 4L) ++ compactedFileStatuses(Seq((0, 4))), startVersion = 0, endVersion = Optional.empty(), expectedListedFiles = compactedFileStatuses(Seq((0, 4))) ++ deltaFileStatuses(0L to 4L)) testListWithCompactions( "compaction at end, no endVersion", files = deltaFileStatuses(0L to 4L) ++ compactedFileStatuses(Seq((3, 4))), startVersion = 0, endVersion = Optional.empty(), expectedListedFiles = deltaFileStatuses(0L to 2L) ++ compactedFileStatuses(Seq((3, 4))) ++ deltaFileStatuses(3L to 4L)) testListWithCompactions( "compaction at end, with endVersion", files = deltaFileStatuses(0L to 4L) ++ compactedFileStatuses(Seq((3, 4))), startVersion = 0, endVersion = Optional.of(4), expectedListedFiles = deltaFileStatuses(0L to 2L) ++ compactedFileStatuses(Seq((3, 4))) ++ deltaFileStatuses(3L to 4L)) testListWithCompactions( "compaction in middle, no endVersion", files = deltaFileStatuses(0L to 4L) ++ compactedFileStatuses(Seq((2, 4))), startVersion = 0, endVersion = Optional.empty(), expectedListedFiles = deltaFileStatuses(0L to 1L) ++ compactedFileStatuses(Seq((2, 4))) ++ deltaFileStatuses(2L to 4L)) testListWithCompactions( "compaction over end, with endVersion", files = deltaFileStatuses(0L to 6L) ++ compactedFileStatuses(Seq((3, 7))), startVersion = 0, endVersion = Optional.of(5), expectedListedFiles = deltaFileStatuses(0L to 5L)) testListWithCompactions( "compaction before start, no endVersion", files = deltaFileStatuses(0L to 6L) ++ compactedFileStatuses(Seq((2, 4))), startVersion = 3, endVersion = Optional.empty(), expectedListedFiles = deltaFileStatuses(3L to 6L)) testListWithCompactions( "compaction before start, with endVersion", files = deltaFileStatuses(0L to 6L) ++ compactedFileStatuses(Seq((2, 4))), startVersion = 3, endVersion = Optional.of(5), expectedListedFiles = deltaFileStatuses(3L to 5L)) testListWithCompactions( "multiple compactions, no endVersion", files = deltaFileStatuses(0L to 7L) ++ compactedFileStatuses(Seq((2, 4), (5, 7))), startVersion = 0, endVersion = Optional.empty(), expectedListedFiles = deltaFileStatuses(0L to 1L) ++ compactedFileStatuses(Seq((2, 4))) ++ deltaFileStatuses(2L to 4L) ++ compactedFileStatuses(Seq((5, 7))) ++ deltaFileStatuses(5L to 7L)) testListWithCompactions( "multiple compactions, with endVersion, don't return second compaction", files = deltaFileStatuses(0L to 7L) ++ compactedFileStatuses(Seq((2, 4), (5, 7))), startVersion = 0, endVersion = Optional.of(6), expectedListedFiles = deltaFileStatuses(0L to 1L) ++ compactedFileStatuses(Seq((2, 4))) ++ deltaFileStatuses(2L to 4L) ++ deltaFileStatuses(5L to 6L)) testListWithCompactions( "multiple compactions, no endVersion, start after first compaction", files = deltaFileStatuses(0L to 7L) ++ compactedFileStatuses(Seq((2, 4), (5, 7))), startVersion = 3, endVersion = Optional.empty(), expectedListedFiles = deltaFileStatuses(3L to 4L) ++ compactedFileStatuses(Seq((5, 7))) ++ deltaFileStatuses(5L to 7L)) testListWithCompactions( "multiple compactions, with endVersion, return no compactions", files = deltaFileStatuses(0L to 7L) ++ compactedFileStatuses(Seq((2, 4), (5, 7))), startVersion = 3, endVersion = Optional.of(6), expectedListedFiles = deltaFileStatuses(3L to 6L)) } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/FilteredColumnarBatchSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import java.util.Optional import io.delta.kernel.TransactionSuite.columnarBatch import io.delta.kernel.data.{ColumnarBatch, ColumnVector} import io.delta.kernel.data.FilteredColumnarBatch import io.delta.kernel.test.VectorTestUtils import io.delta.kernel.types.{LongType, StructField, StructType} import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers class FilteredColumnarBatchSuite extends AnyFunSuite with VectorTestUtils with Matchers { private val testSchema = new StructType().add("id", LongType.LONG) test("constructor should succeed when selectionVector is present and numSelectedRows is valid") { val data = columnarBatch(testSchema, Seq(longVector(Seq(0L, 1L, 2L, 3L, 4L)))) val selectionVector = Optional.of(booleanVector(Seq(true, false, true, false, true))) val batch = new FilteredColumnarBatch(data, selectionVector, "/test/path", 3) assert(batch.getFilePath == Optional.of("/test/path")) assert(batch.getPreComputedNumSelectedRows == Optional.of(3)) assert(batch.getData == data) assert(batch.getSelectionVector == selectionVector) } test( "constructor should succeed when selectionVector is empty and numSelectedRows " + "equals batch size") { val data = columnarBatch(testSchema, Seq(longVector(Seq(0L, 1L, 2L, 3L, 4L)))) val selectionVector = Optional.empty[ColumnVector]() val batch = new FilteredColumnarBatch(data, selectionVector, "/test/path", 5) assert(batch.getFilePath == Optional.of("/test/path")) assert(batch.getPreComputedNumSelectedRows == Optional.of(5)) assert(batch.getData == data) assert(batch.getSelectionVector == selectionVector) } test("constructor should throw IllegalArgumentException " + "when selectionVector is empty and numSelectedRows != batch size") { val data = columnarBatch(testSchema, Seq(longVector(Seq(0L, 1L, 2L, 3L, 4L)))) val selectionVector = Optional.empty[ColumnVector]() val exMsg = intercept[IllegalArgumentException] { new FilteredColumnarBatch(data, selectionVector, "/test/path", 3) }.getMessage assert(exMsg.contains("Invalid precomputedNumSelectedRows")) assert(exMsg.contains("must be equal to batch size when selectionVector is empty")) } test("constructor should throw IllegalArgumentException " + "when selectionVector is present and numSelectedRows > batch size") { val data = columnarBatch(testSchema, Seq(longVector(Seq(0L, 1L, 2L, 3L)))) val selectionVector = Optional.of(booleanVector(Seq(true, false, true, false))) val exMsg = intercept[IllegalArgumentException] { new FilteredColumnarBatch(data, selectionVector, "/test/path", 5) }.getMessage assert(exMsg.contains("Invalid precomputedNumSelectedRows")) assert(exMsg.contains("no larger than batch size")) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/PageTokenSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal import java.util import java.util.{HashMap, Map} import java.util.Optional import scala.collection.JavaConverters._ import io.delta.kernel.data.Row import io.delta.kernel.internal.annotation.VisibleForTesting import io.delta.kernel.internal.data.GenericRow import io.delta.kernel.internal.replay.PageToken import io.delta.kernel.test.MockFileSystemClientUtils import io.delta.kernel.types._ import org.scalatest.funsuite.AnyFunSuite class PageTokenSuite extends AnyFunSuite with MockFileSystemClientUtils { private val TEST_FILE_NAME = "/path/to/table/test_file.json" private val TEST_ROW_INDEX = 42L private val TEST_SIDECAR_INDEX = Optional.of(java.lang.Long.valueOf(5L)) private val TEST_KERNEL_VERSION = "4.0.0" private val TEST_TABLE_PATH = "/path/to/table" private val TEST_TABLE_VERSION = 5L private val TEST_PREDICATE_HASH = 123 private val TEST_LOG_SEGMENT_HASH = 456 private val expectedPageToken = new PageToken( TEST_FILE_NAME, TEST_ROW_INDEX, TEST_SIDECAR_INDEX, TEST_KERNEL_VERSION, TEST_TABLE_PATH, TEST_TABLE_VERSION, TEST_PREDICATE_HASH, TEST_LOG_SEGMENT_HASH) private val rowData: Map[Integer, Object] = new HashMap() rowData.put(0, TEST_FILE_NAME) rowData.put(1, TEST_ROW_INDEX.asInstanceOf[Object]) rowData.put(2, TEST_SIDECAR_INDEX.get()) rowData.put(3, TEST_KERNEL_VERSION) rowData.put(4, TEST_TABLE_PATH) rowData.put(5, TEST_TABLE_VERSION.asInstanceOf[Object]) rowData.put(6, TEST_PREDICATE_HASH.asInstanceOf[Object]) rowData.put(7, TEST_LOG_SEGMENT_HASH.asInstanceOf[Object]) val expectedRow = new GenericRow(PageToken.PAGE_TOKEN_SCHEMA, rowData) test("PageToken.fromRow with valid data") { val pageToken = PageToken.fromRow(expectedRow) assert(pageToken.equals(expectedPageToken)) } test("PageToken.toRow with valid data") { val row = expectedPageToken.toRow assert(row.getSchema.equals(PageToken.PAGE_TOKEN_SCHEMA)) assert(row.getString(0) == TEST_FILE_NAME) assert(row.getLong(1) == TEST_ROW_INDEX) assert(Optional.of(row.getLong(2)) == TEST_SIDECAR_INDEX) assert(row.getString(3) == TEST_KERNEL_VERSION) assert(row.getString(4) == TEST_TABLE_PATH) assert(row.getLong(5) == TEST_TABLE_VERSION) assert(row.getInt(6) == TEST_PREDICATE_HASH) assert(row.getInt(7) == TEST_LOG_SEGMENT_HASH) } test("E2E: PageToken round-trip: toRow -> fromRow") { val row = expectedPageToken.toRow val reconstructedPageToken = PageToken.fromRow(row) assert(reconstructedPageToken.equals(expectedPageToken)) } test("PageToken.fromRow throws exception when input row schema has invalid field name") { val invalidSchema = new StructType() .add("wrongFieldName", StringType.STRING) .add("lastReturnedRowIndex", LongType.LONG) .add("lastReadSidecarFileIdx", LongType.LONG) .add("kernelVersion", StringType.STRING) .add("tablePath", StringType.STRING) .add("tableVersion", LongType.LONG) .add("predicateHash", LongType.LONG) .add("logSegmentHash", LongType.LONG) val invalidRowData: Map[Integer, Object] = new HashMap() invalidRowData.put(0, TEST_FILE_NAME) invalidRowData.put(1, TEST_ROW_INDEX.asInstanceOf[Object]) invalidRowData.put(2, TEST_SIDECAR_INDEX) invalidRowData.put(3, TEST_KERNEL_VERSION) invalidRowData.put(4, TEST_TABLE_PATH) invalidRowData.put(5, TEST_TABLE_VERSION.asInstanceOf[Object]) invalidRowData.put(6, TEST_PREDICATE_HASH.asInstanceOf[Object]) invalidRowData.put(7, TEST_LOG_SEGMENT_HASH.asInstanceOf[Object]) val row = new GenericRow(invalidSchema, invalidRowData) val exception = intercept[IllegalArgumentException] { PageToken.fromRow(row) } assert(exception.getMessage.contains( "Invalid Page Token: input row schema does not match expected PageToken schema")) } test("PageToken.fromRow throws exception when input row schema has wrong data type") { val invalidSchema = new StructType() .add("lastReadLogFilePath", StringType.STRING) .add("lastReturnedRowIndex", LongType.LONG) .add("lastReadSidecarFileIdx", StringType.STRING) // should be long type .add("kernelVersion", StringType.STRING) .add("tablePath", StringType.STRING) .add("tableVersion", LongType.LONG) .add("predicateHash", LongType.LONG) .add("logSegmentHash", LongType.LONG) val invalidRowData: Map[Integer, Object] = new HashMap() invalidRowData.put(0, TEST_FILE_NAME) invalidRowData.put(1, TEST_ROW_INDEX.asInstanceOf[Object]) invalidRowData.put(2, TEST_SIDECAR_INDEX) invalidRowData.put(3, TEST_KERNEL_VERSION) invalidRowData.put(4, TEST_TABLE_PATH) invalidRowData.put(5, TEST_TABLE_VERSION.asInstanceOf[Object]) invalidRowData.put(6, TEST_PREDICATE_HASH.asInstanceOf[Object]) invalidRowData.put(7, TEST_LOG_SEGMENT_HASH.asInstanceOf[Object]) val row = new GenericRow(invalidSchema, invalidRowData) val exception = intercept[IllegalArgumentException] { PageToken.fromRow(row) } assert(exception.getMessage.contains( "Invalid Page Token: input row schema does not match expected PageToken schema")) } test("PageToken.fromRow accepts the case sidecar field is null") { val nullSidecarData: Map[Integer, Object] = new HashMap(rowData) nullSidecarData.put(2, null) val nullSidecarRow = new GenericRow(PageToken.PAGE_TOKEN_SCHEMA, nullSidecarData) val pageToken = PageToken.fromRow(nullSidecarRow) assert(pageToken.getLastReadSidecarFileIdx == Optional.empty()) } test("PageToken.fromRow throws exception when required field is null") { val invalidData: Map[Integer, Object] = new HashMap(rowData) invalidData.put(3, null) val invalidRow = new GenericRow(PageToken.PAGE_TOKEN_SCHEMA, invalidData) val exception = intercept[IllegalArgumentException] { PageToken.fromRow(invalidRow) } assert(exception.getMessage.contains( "Invalid Page Token: required field")) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/PaginationContextSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal import java.util.Optional import io.delta.kernel.Meta import io.delta.kernel.internal.replay.{PageToken, PaginationContext} import org.scalatest.funsuite.AnyFunSuite class PaginationContextSuite extends AnyFunSuite { private val TEST_FILE_NAME = "test_file.json" private val TEST_ROW_INDEX = 42L private val TEST_SIDECAR_INDEX = Optional.of(java.lang.Long.valueOf(5L)) private val TEST_INVALID_KERNEL_VERSION = "300.0.0" private val TEST_VALID_KERNEL_VERSION = Meta.KERNEL_VERSION private val TEST_TABLE_PATH = "/path/to/table" private val TEST_WRONG_TABLE_PATH = "/wrong/path/to/table" private val TEST_TABLE_VERSION = 5L private val TEST_WRONG_TABLE_VERSION = 5000L private val TEST_PREDICATE_HASH = 123 private val TEST_WRONG_PREDICATE_HASH = 321 private val TEST_LOG_SEGMENT_HASH = 456 private val TEST_WRONG_LOG_SEGMENT_HASH = 654 private val validPageToken = new PageToken( TEST_FILE_NAME, TEST_ROW_INDEX, TEST_SIDECAR_INDEX, TEST_VALID_KERNEL_VERSION, TEST_TABLE_PATH, TEST_TABLE_VERSION, TEST_PREDICATE_HASH, TEST_LOG_SEGMENT_HASH) private val invalidKernelVersionPageToken = new PageToken( TEST_FILE_NAME, TEST_ROW_INDEX, TEST_SIDECAR_INDEX, TEST_INVALID_KERNEL_VERSION, TEST_TABLE_PATH, TEST_TABLE_VERSION, TEST_PREDICATE_HASH, TEST_LOG_SEGMENT_HASH) test("forFirstPage should create context with empty optionals and specified page size") { val pageSize = 100L val context = PaginationContext.forFirstPage( TEST_TABLE_PATH, TEST_TABLE_VERSION, TEST_LOG_SEGMENT_HASH, TEST_PREDICATE_HASH, pageSize) assert(!context.getLastReadLogFilePath().isPresent) assert(!context.getLastReturnedRowIndex().isPresent) assert(!context.getLastReadSidecarFileIdx().isPresent) assert(context.getPageSize() === pageSize) } test("forPageWithPageToken should create context with provided values") { val pageSize = 50L val context = PaginationContext.forPageWithPageToken( TEST_TABLE_PATH, TEST_TABLE_VERSION, TEST_LOG_SEGMENT_HASH, TEST_PREDICATE_HASH, pageSize, validPageToken) assert(context.getLastReadLogFilePath() === Optional.of(TEST_FILE_NAME)) assert(context.getLastReturnedRowIndex() === Optional.of(TEST_ROW_INDEX)) assert(context.getLastReadSidecarFileIdx() === TEST_SIDECAR_INDEX) assert(context.getPageSize() === pageSize) } test("forPageWithPageToken should throw exception when page token is null") { val pageSize = 50L val e = intercept[NullPointerException] { PaginationContext.forPageWithPageToken( TEST_TABLE_PATH, TEST_TABLE_VERSION, TEST_LOG_SEGMENT_HASH, TEST_PREDICATE_HASH, pageSize, null /* page token */ ) } assert(e.getMessage === "page token is null") } test("should throw exception for zero page size") { val e = intercept[IllegalArgumentException] { PaginationContext.forFirstPage( TEST_TABLE_PATH, TEST_TABLE_VERSION, TEST_LOG_SEGMENT_HASH, TEST_PREDICATE_HASH, 0L) } assert(e.getMessage === "Page size must be greater than zero!") } test("should throw exception for negative page size") { val negativePageSize = -10L val e = intercept[IllegalArgumentException] { PaginationContext.forFirstPage( TEST_TABLE_PATH, TEST_TABLE_VERSION, TEST_LOG_SEGMENT_HASH, TEST_PREDICATE_HASH, negativePageSize) } assert(e.getMessage === "Page size must be greater than zero!") } test("should throw exception for negative page size with page token") { val negativePageSize = -5L val e = intercept[IllegalArgumentException] { PaginationContext.forPageWithPageToken( TEST_TABLE_PATH, TEST_TABLE_VERSION, TEST_LOG_SEGMENT_HASH, TEST_PREDICATE_HASH, negativePageSize, validPageToken) } assert(e.getMessage === "Page size must be greater than zero!") } test("should throw exception when the requested kernel version doesn't " + "match the value in page token") { val pageSize = 50L val e = intercept[IllegalArgumentException] { PaginationContext.forPageWithPageToken( TEST_TABLE_PATH, TEST_TABLE_VERSION, TEST_LOG_SEGMENT_HASH, TEST_PREDICATE_HASH, pageSize, invalidKernelVersionPageToken) } assert(e.getMessage.contains("Invalid page token: token kernel version")) } test("should throw exception for when the requested table path doesn't " + "match the value in page token") { val pageSize = 50L val e = intercept[IllegalArgumentException] { PaginationContext.forPageWithPageToken( TEST_WRONG_TABLE_PATH, TEST_TABLE_VERSION, TEST_LOG_SEGMENT_HASH, TEST_PREDICATE_HASH, pageSize, validPageToken) } assert(e.getMessage.contains("Invalid page token: token table path")) } test("should throw exception for when the requested table version doesn't " + "match the value in page token") { val pageSize = 50L val e = intercept[IllegalArgumentException] { PaginationContext.forPageWithPageToken( TEST_TABLE_PATH, TEST_WRONG_TABLE_VERSION, TEST_LOG_SEGMENT_HASH, TEST_PREDICATE_HASH, pageSize, validPageToken) } assert(e.getMessage.contains("Invalid page token: token table version")) } test("should throw exception for when the requested predicate doesn't " + "match the value in page token") { val pageSize = 50L val e = intercept[IllegalArgumentException] { PaginationContext.forPageWithPageToken( TEST_TABLE_PATH, TEST_TABLE_VERSION, TEST_LOG_SEGMENT_HASH, TEST_WRONG_PREDICATE_HASH, pageSize, validPageToken) } assert(e.getMessage.contains("Invalid page token: token predicate")) } test("should throw exception for when the requested log segment doesn't " + "match the value in page token") { val pageSize = 50L val e = intercept[IllegalArgumentException] { PaginationContext.forPageWithPageToken( TEST_TABLE_PATH, TEST_TABLE_VERSION, TEST_WRONG_LOG_SEGMENT_HASH, TEST_PREDICATE_HASH, pageSize, validPageToken) } assert(e.getMessage.contains("Invalid page token: token log segment")) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/SnapshotManagerSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal import java.lang.{Long => JLong} import java.util.{Arrays, Collections, Optional} import scala.collection.JavaConverters._ import scala.reflect.ClassTag import io.delta.kernel.data.{ColumnarBatch, ColumnVector} import io.delta.kernel.engine.FileReadResult import io.delta.kernel.exceptions.{InvalidTableException, TableNotFoundException} import io.delta.kernel.expressions.Predicate import io.delta.kernel.internal.checkpoints.{CheckpointInstance, CheckpointMetaData, SidecarFile} import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.snapshot.{LogSegment, SnapshotManager} import io.delta.kernel.internal.util.{FileNames, Utils} import io.delta.kernel.test.{BaseMockJsonHandler, BaseMockParquetHandler, MockFileSystemClientUtils, MockListFromFileSystemClient, VectorTestUtils} import io.delta.kernel.types.StructType import io.delta.kernel.utils.{CloseableIterator, FileStatus} import org.scalatest.funsuite.AnyFunSuite class SnapshotManagerSuite extends AnyFunSuite with MockFileSystemClientUtils { test("verifyDeltaVersionsContiguous") { val path = new Path("/path/to/table") // empty array SnapshotManager.verifyDeltaVersionsContiguous(Collections.emptyList(), path) // array of size 1 SnapshotManager.verifyDeltaVersionsContiguous(Collections.singletonList(1), path) // contiguous versions SnapshotManager.verifyDeltaVersionsContiguous(Arrays.asList(1, 2, 3), path) // non-contiguous versions intercept[InvalidTableException] { SnapshotManager.verifyDeltaVersionsContiguous(Arrays.asList(1, 3), path) } // duplicates in versions intercept[InvalidTableException] { SnapshotManager.verifyDeltaVersionsContiguous(Arrays.asList(1, 2, 2, 3), path) } // unsorted versions intercept[InvalidTableException] { SnapshotManager.verifyDeltaVersionsContiguous(Arrays.asList(3, 2, 1), path) } } ////////////////////////////////////////////////////////////////////////////////// // getLogSegmentForVersion tests ////////////////////////////////////////////////////////////////////////////////// private val snapshotManager = new SnapshotManager(dataPath) /* ------------------HELPER METHODS------------------ */ private def checkLogSegment( logSegment: LogSegment, expectedVersion: Long, expectedDeltas: Seq[FileStatus], expectedCompactions: Seq[FileStatus], expectedCheckpoints: Seq[FileStatus], expectedCheckpointVersion: Option[Long], expectedLastCommitTimestamp: Long): Unit = { assert(logSegment.getLogPath == logPath) assert(logSegment.getVersion == expectedVersion) assert(expectedDeltas.map(f => (f.getPath, f.getSize, f.getModificationTime)) sameElements logSegment.getDeltas.asScala.map(f => (f.getPath, f.getSize, f.getModificationTime))) assert(expectedCompactions.map(f => (f.getPath, f.getSize, f.getModificationTime)) sameElements logSegment.getCompactions.asScala.map(f => (f.getPath, f.getSize, f.getModificationTime))) val expectedCheckpointStatuses = expectedCheckpoints .map(f => (f.getPath, f.getSize, f.getModificationTime)).sortBy(_._1) val actualCheckpointStatuses = logSegment.getCheckpoints.asScala .map(f => (f.getPath, f.getSize, f.getModificationTime)).sortBy(_._1) assert( expectedCheckpointStatuses sameElements actualCheckpointStatuses, s"expected:\n$expectedCheckpointStatuses\nactual:\n$actualCheckpointStatuses") expectedCheckpointVersion match { case Some(v) => assert(logSegment.getCheckpointVersionOpt.isPresent() && logSegment.getCheckpointVersionOpt.get == v) case None => assert(!logSegment.getCheckpointVersionOpt.isPresent()) } assert(expectedLastCommitTimestamp == logSegment.getDeltaFileAtEndVersion.getModificationTime) } /** * Test `getLogSegmentForVersion` for a given set of delta versions, singular checkpoint versions, * and multi-part checkpoint versions with a given _last_checkpoint starting checkpoint and * a versionToLoad. * * @param deltaVersions versions of the delta JSON files in the delta log * @param checkpointVersions version of the singular checkpoint parquet files in the delta log * @param multiCheckpointVersions versions of the multi-part checkpoint files in the delta log * @param numParts number of parts for the multi-part checkpoints if applicable * @param startCheckpoint starting checkpoint to list from, in practice provided by the * _last_checkpoint file; if not provided list from 0 * @param versionToLoad specific version to load; if not provided load the latest * @param v2CheckpointSpec Versions of V2 checkpoints to be included along with the number of * sidecars in each checkpoint and the naming scheme for each checkpoint. */ def testWithCheckpoints( deltaVersions: Seq[Long], checkpointVersions: Seq[Long], multiCheckpointVersions: Seq[Long], numParts: Int = -1, startCheckpoint: Optional[java.lang.Long] = Optional.empty(), versionToLoad: Optional[java.lang.Long] = Optional.empty(), v2CheckpointSpec: Seq[(Long, Boolean, Int)] = Seq.empty, compactionVersions: Seq[(Long, Long)] = Seq.empty): Unit = { val deltas = deltaFileStatuses(deltaVersions) val singularCheckpoints = singularCheckpointFileStatuses(checkpointVersions) val multiCheckpoints = multiCheckpointFileStatuses(multiCheckpointVersions, numParts) // Only test both filetypes if we have to read the checkpoint top-level file. val topLevelFileTypes = if (v2CheckpointSpec.nonEmpty) { Seq("parquet", "json") } else { Seq("parquet") } topLevelFileTypes.foreach { topLevelFileType => val v2Checkpoints = v2CheckpointFileStatuses(v2CheckpointSpec, topLevelFileType) val checkpointFiles = v2Checkpoints.flatMap { case (topLevelCheckpointFile, sidecars) => Seq(topLevelCheckpointFile) ++ sidecars } ++ singularCheckpoints ++ multiCheckpoints val expectedCheckpointVersion = (checkpointVersions ++ multiCheckpointVersions ++ v2CheckpointSpec.map(_._1)) .filter(_ <= versionToLoad.orElse(Long.MaxValue)) .sorted .lastOption val (expectedV2Checkpoint, expectedSidecars) = expectedCheckpointVersion.map { v => val matchingCheckpoints = v2Checkpoints.filter { case (topLevelFile, _) => FileNames.checkpointVersion(topLevelFile.getPath) == v } if (matchingCheckpoints.nonEmpty) { matchingCheckpoints.maxBy(f => new CheckpointInstance(f._1.getPath)) match { case (c, sidecars) => (Seq(c), sidecars) } } else { (Seq.empty, Seq.empty) } }.getOrElse((Seq.empty, Seq.empty)) val compactions = compactedFileStatuses(compactionVersions) val mockSidecarParquetHandler = if (expectedSidecars.nonEmpty) { new MockSidecarParquetHandler(expectedSidecars, expectedV2Checkpoint.head.getPath) } else { new BaseMockParquetHandler {} } val logSegment = snapshotManager.getLogSegmentForVersion( createMockFSListFromEngine( deltas ++ compactions ++ checkpointFiles, mockSidecarParquetHandler, new MockSidecarJsonHandler(expectedSidecars)), versionToLoad) val expectedDeltas = deltaFileStatuses( deltaVersions.filter { v => v > expectedCheckpointVersion.getOrElse(-1L) && v <= versionToLoad.orElse(Long.MaxValue) }) val expectedCheckpoints = expectedCheckpointVersion.map { v => if (expectedV2Checkpoint.nonEmpty) { expectedV2Checkpoint } else if (checkpointVersions.toSet.contains(v)) { singularCheckpointFileStatuses(Seq(v)) } else { multiCheckpointFileStatuses(Seq(v), numParts) } }.getOrElse(Seq.empty) val expectedCompactions = compactedFileStatuses( compactionVersions.filter { case (s, e) => // we can only use a compaction if it starts after the checkpoint and ends at or before // the version we're trying to load s > expectedCheckpointVersion.getOrElse(-1L) && e <= versionToLoad.orElse(Long.MaxValue) }) checkLogSegment( logSegment, expectedVersion = versionToLoad.orElse(deltaVersions.max), expectedDeltas = expectedDeltas, expectedCompactions = expectedCompactions, expectedCheckpoints = expectedCheckpoints, expectedCheckpointVersion = expectedCheckpointVersion, expectedLastCommitTimestamp = versionToLoad.orElse(deltaVersions.max) * 10) } } /** Simple test for a log with only JSON files and no checkpoints */ def testNoCheckpoint( deltaVersions: Seq[Long], versionToLoad: Optional[java.lang.Long] = Optional.empty()): Unit = { testWithCheckpoints( deltaVersions, checkpointVersions = Seq.empty, multiCheckpointVersions = Seq.empty, versionToLoad = versionToLoad) } /** Simple test with only json and compactions */ def testWithCompactionsNoCheckpoint( deltaVersions: Seq[Long], compactionVersions: Seq[(Long, Long)], versionToLoad: Optional[java.lang.Long] = Optional.empty()): Unit = { testWithCheckpoints( deltaVersions, checkpointVersions = Seq.empty, multiCheckpointVersions = Seq.empty, versionToLoad = versionToLoad, compactionVersions = compactionVersions) } /** * Test `getLogSegmentForVersion` for a set of delta versions and checkpoint versions. Tests * with (1) singular checkpoint (2) multi-part checkpoints with 5 parts * (3) multi-part checkpoints with 1 part */ def testWithSingularAndMultipartCheckpoint( deltaVersions: Seq[Long], checkpointVersions: Seq[Long], startCheckpoint: Optional[java.lang.Long] = Optional.empty(), versionToLoad: Optional[java.lang.Long] = Optional.empty()): Unit = { // test with singular checkpoint testWithCheckpoints( deltaVersions = deltaVersions, checkpointVersions = checkpointVersions, multiCheckpointVersions = Seq.empty, startCheckpoint = startCheckpoint, versionToLoad = versionToLoad) // test with multi-part checkpoint numParts=5 testWithCheckpoints( deltaVersions = deltaVersions, checkpointVersions = Seq.empty, multiCheckpointVersions = checkpointVersions, numParts = 5, startCheckpoint = startCheckpoint, versionToLoad = versionToLoad) // test with multi-part checkpoint numParts=1 testWithCheckpoints( deltaVersions = deltaVersions, checkpointVersions = Seq.empty, multiCheckpointVersions = checkpointVersions, numParts = 1, startCheckpoint = startCheckpoint, versionToLoad = versionToLoad) } /** * For a given set of _delta_log files check for error. */ def testExpectedError[T <: Throwable]( files: Seq[FileStatus], lastCheckpointVersion: Optional[java.lang.Long] = Optional.empty(), versionToLoad: Optional[java.lang.Long] = Optional.empty(), expectedErrorMessageContains: String = "")(implicit classTag: ClassTag[T]): Unit = { val e = intercept[T] { snapshotManager.getLogSegmentForVersion( createMockFSAndJsonEngineForLastCheckpoint(files, lastCheckpointVersion), versionToLoad) } assert(e.getMessage.contains(expectedErrorMessageContains)) } /* ------------------- VALID DELTA LOG FILE LISTINGS ----------------------- */ test("getLogSegmentForVersion: 000.json only") { testNoCheckpoint(Seq(0)) testNoCheckpoint(Seq(0), Optional.of(0)) } test("getLogSegmentForVersion: 000.json .. 009.json") { testNoCheckpoint(0L until 10L) testNoCheckpoint(0L until 10L, Optional.of(9)) testNoCheckpoint(0L until 10L, Optional.of(5)) } test("getLogSegmentForVersion: 000.json..010.json + checkpoint(10)") { testWithSingularAndMultipartCheckpoint( deltaVersions = (0L to 10L), checkpointVersions = Seq(10)) testWithSingularAndMultipartCheckpoint( deltaVersions = (0L to 10L), checkpointVersions = Seq(10), startCheckpoint = Optional.of(10)) testWithSingularAndMultipartCheckpoint( deltaVersions = (0L to 10L), checkpointVersions = Seq(10), versionToLoad = Optional.of(10)) testWithSingularAndMultipartCheckpoint( deltaVersions = (0L to 10L), checkpointVersions = Seq(10), startCheckpoint = Optional.of(10), versionToLoad = Optional.of(10)) testWithSingularAndMultipartCheckpoint( deltaVersions = (0L to 10L), checkpointVersions = Seq(10), versionToLoad = Optional.of(6)) testWithSingularAndMultipartCheckpoint( deltaVersions = (0L to 10L), checkpointVersions = Seq(10), startCheckpoint = Optional.of(10), versionToLoad = Optional.of(6)) } test("getLogSegmentForVersion: 000.json...20.json + checkpoint(10) + checkpoint(20)") { testWithSingularAndMultipartCheckpoint( deltaVersions = (0L to 20L), checkpointVersions = Seq(10, 20)) testWithSingularAndMultipartCheckpoint( deltaVersions = (0L to 20L), checkpointVersions = Seq(10, 20), startCheckpoint = Optional.of(20)) // _last_checkpoint hasn't been updated yet testWithSingularAndMultipartCheckpoint( deltaVersions = (0L to 20L), checkpointVersions = Seq(10, 20), startCheckpoint = Optional.of(10)) testWithSingularAndMultipartCheckpoint( deltaVersions = (0L to 20L), checkpointVersions = Seq(10, 20), versionToLoad = Optional.of(15)) testWithSingularAndMultipartCheckpoint( deltaVersions = (0L to 20L), checkpointVersions = Seq(10, 20), startCheckpoint = Optional.of(10), versionToLoad = Optional.of(15)) testWithSingularAndMultipartCheckpoint( deltaVersions = (0L to 20L), checkpointVersions = Seq(10, 20), startCheckpoint = Optional.of(20), versionToLoad = Optional.of(15)) } test("getLogSegmentForVersion: outdated _last_checkpoint that does not exist") { testWithSingularAndMultipartCheckpoint( deltaVersions = (20L until 25L), checkpointVersions = Seq(20), startCheckpoint = Optional.of(10)) testWithSingularAndMultipartCheckpoint( deltaVersions = (20L until 25L), checkpointVersions = Seq(20), startCheckpoint = Optional.of(10), versionToLoad = Optional.of(20)) } test("getLogSegmentForVersion: 20.json...25.json + checkpoint(20)") { testWithSingularAndMultipartCheckpoint( deltaVersions = (20L to 25L), checkpointVersions = Seq(20)) testWithSingularAndMultipartCheckpoint( deltaVersions = (20L to 25L), checkpointVersions = Seq(20), startCheckpoint = Optional.of(20)) testWithSingularAndMultipartCheckpoint( deltaVersions = (20L to 25L), checkpointVersions = Seq(20), versionToLoad = Optional.of(23)) } test("getLogSegmentForVersion: empty delta log") { val exMsg = intercept[TableNotFoundException] { snapshotManager.getLogSegmentForVersion( createMockFSListFromEngine(Seq.empty), Optional.empty() /* versionToLoad */ ) }.getMessage assert(exMsg.contains("No delta files found in the directory")) } test("getLogSegmentForVersion: no delta files in the delta log") { // listDeltaAndCheckpointFiles = Optional.of(EmptyList) val files = Seq("foo", "notdelta.parquet", "foo.json", "001.checkpoint.00f.oo0.parquet") .map(FileStatus.of(_, 10, 10)) testExpectedError[TableNotFoundException]( files, expectedErrorMessageContains = "No delta files found in the directory: /fake/path/to/table/_delta_log") testExpectedError[TableNotFoundException]( files, versionToLoad = Optional.of(5), expectedErrorMessageContains = "No delta files found in the directory: /fake/path/to/table/_delta_log") } test("getLogSegmentForVersion: versionToLoad higher than possible") { testExpectedError[RuntimeException]( files = deltaFileStatuses(Seq(0L)), versionToLoad = Optional.of(15), expectedErrorMessageContains = "Cannot load table version 15 as it does not exist. The latest available version is 0") testExpectedError[RuntimeException]( files = deltaFileStatuses((10L until 13L)) ++ singularCheckpointFileStatuses(Seq(10L)), versionToLoad = Optional.of(15), expectedErrorMessageContains = "Cannot load table version 15 as it does not exist. The latest available version is 12") } test("getLogSegmentForVersion: start listing from _last_checkpoint when it is provided") { val deltas = deltaFileStatuses(0L to 24) val checkpoints = singularCheckpointFileStatuses(Seq(10, 20)) for (lastCheckpointVersion <- Seq(10, 20)) { val lastCheckpointFileStatus = FileStatus.of(s"$logPath/_last_checkpoint", 2, 2) val files = deltas ++ checkpoints ++ Seq(lastCheckpointFileStatus) def listFrom(filePath: String): Seq[FileStatus] = { if (filePath < FileNames.listingPrefix(logPath, lastCheckpointVersion)) { throw new RuntimeException( s"Listing from before the checkpoint version referenced by _last_checkpoint. " + s"Last checkpoint version: $lastCheckpointVersion. Listing from: $filePath") } listFromProvider(files)(filePath) } val logSegment = snapshotManager.getLogSegmentForVersion( mockEngine( jsonHandler = new MockReadLastCheckpointFileJsonHandler( lastCheckpointFileStatus.getPath, lastCheckpointVersion), fileSystemClient = new MockListFromFileSystemClient(listFrom)), Optional.empty() /* versionToLoad */ ) checkLogSegment( logSegment, expectedVersion = 24, expectedDeltas = deltaFileStatuses(21L until 25L), expectedCompactions = Seq.empty, expectedCheckpoints = singularCheckpointFileStatuses(Seq(20L)), expectedCheckpointVersion = Some(20), expectedLastCommitTimestamp = 240L) } } test("getLogSegmentForVersion: multi-part and single-part checkpoints in same log") { testWithCheckpoints( (0L to 50L), Seq(10, 30, 50), Seq(20, 40), numParts = 5) testWithCheckpoints( (0L to 50L), Seq(10, 30, 50), Seq(20, 40), numParts = 5, startCheckpoint = Optional.of(40)) } test("getLogSegmentForVersion: versionToLoad not constructable from history") { testExpectedError[RuntimeException]( deltaFileStatuses(20L until 25L) ++ singularCheckpointFileStatuses(Seq(20L)), versionToLoad = Optional.of(15), expectedErrorMessageContains = "Cannot load table version 15") } /* ------------------- V2 CHECKPOINT TESTS ------------------ */ test("v2 checkpoint exists at version") { testWithCheckpoints( deltaVersions = (0L to 5L), checkpointVersions = Seq.empty, multiCheckpointVersions = Seq.empty, versionToLoad = Optional.of(5L), v2CheckpointSpec = Seq((0L, true, 2), (5L, true, 2))) } test("multiple v2 checkpoint exist at version") { testWithCheckpoints( deltaVersions = (0L to 5L), checkpointVersions = Seq.empty, multiCheckpointVersions = Seq.empty, versionToLoad = Optional.of(5L), v2CheckpointSpec = Seq((5L, true, 2), (5L, true, 2))) } test("v2 checkpoint exists before version") { testWithCheckpoints( deltaVersions = (0L to 7L), checkpointVersions = Seq.empty, multiCheckpointVersions = Seq.empty, versionToLoad = Optional.of(6L), v2CheckpointSpec = Seq((0L, true, 2), (5L, true, 2))) } test("v1 and v2 checkpoints in table") { testWithCheckpoints( deltaVersions = (0L to 12L), checkpointVersions = Seq(0L, 10L), multiCheckpointVersions = Seq.empty, versionToLoad = Optional.of(8L), v2CheckpointSpec = Seq((5L, true, 2))) testWithCheckpoints( (0L to 12L), checkpointVersions = Seq(0L, 10L), multiCheckpointVersions = Seq.empty, versionToLoad = Optional.of(12L), v2CheckpointSpec = Seq((5L, true, 2))) } test("multipart and v2 checkpoints in table") { testWithCheckpoints( deltaVersions = (0L to 12L), checkpointVersions = Seq.empty, multiCheckpointVersions = Seq(0L, 10L), numParts = 5, versionToLoad = Optional.of(8L), v2CheckpointSpec = Seq((5L, true, 2))) testWithCheckpoints( deltaVersions = (0L to 12L), checkpointVersions = Seq.empty, multiCheckpointVersions = Seq(0L, 10L), numParts = 5, versionToLoad = Optional.of(12L), v2CheckpointSpec = Seq((5L, true, 2))) } test("no checkpoint prior to version") { testWithCheckpoints( deltaVersions = (0L to 5L), checkpointVersions = Seq.empty, multiCheckpointVersions = Seq.empty, versionToLoad = Optional.of(3L), v2CheckpointSpec = Seq((5L, true, 2))) } test("read from compatibility checkpoint") { testWithCheckpoints( deltaVersions = (0L to 5L), checkpointVersions = Seq.empty, multiCheckpointVersions = Seq.empty, versionToLoad = Optional.of(5L), v2CheckpointSpec = Seq((5L, false, 5))) testWithCheckpoints( deltaVersions = (0L to 5L), checkpointVersions = Seq.empty, multiCheckpointVersions = Seq.empty, versionToLoad = Optional.of(5L), v2CheckpointSpec = Seq((0L, true, 5), (5L, false, 5))) } test("read from V2 checkpoint with compatibility checkpoint at same version") { testWithCheckpoints( deltaVersions = (0L to 5L), checkpointVersions = Seq.empty, multiCheckpointVersions = Seq.empty, versionToLoad = Optional.of(5L), v2CheckpointSpec = Seq((5L, true, 5), (5L, false, 5))) } test("read from V2 checkpoint with compatibility checkpoint at previous version") { testWithCheckpoints( deltaVersions = (0L to 5L), checkpointVersions = Seq.empty, multiCheckpointVersions = Seq.empty, versionToLoad = Optional.of(5L), v2CheckpointSpec = Seq((3L, false, 5), (5L, true, 5))) } /* ------------------- CORRUPT DELTA LOG FILE LISTINGS ------------------ */ test("getLogSegmentForVersion: corrupt listing with only checkpoint file") { Seq(Optional.empty(), Optional.of(10L)).foreach { lastCheckpointVersion => Seq(Optional.empty(), Optional.of(10L)).foreach { versionToLoad => testExpectedError[InvalidTableException]( files = singularCheckpointFileStatuses(Seq(10L)), lastCheckpointVersion.map(Long.box), versionToLoad.map(Long.box), expectedErrorMessageContains = "Missing delta file for version 10") } } } test("getLogSegmentForVersion: corrupt listing with missing log files") { // checkpoint(10), 010.json, 011.json, 013.json val fileList = deltaFileStatuses(Seq(10L, 11L)) ++ deltaFileStatuses(Seq(13L)) ++ singularCheckpointFileStatuses(Seq(10L)) Seq(Optional.empty(), Optional.of(10L)).foreach { lastCheckpointVersion => Seq(Optional.empty(), Optional.of(13L)).foreach { versionToLoad => testExpectedError[InvalidTableException]( fileList, lastCheckpointVersion.map(Long.box), versionToLoad.map(Long.box), expectedErrorMessageContains = "versions are not contiguous: ([11, 13])") } } } test("getLogSegmentForVersion: corrupt listing 000.json...009.json + checkpoint(10)") { val fileList = deltaFileStatuses((0L until 10L)) ++ singularCheckpointFileStatuses(Seq(10L)) Seq(Optional.empty(), Optional.of(10L)).foreach { lastCheckpointVersion => Seq(Optional.empty(), Optional.of(15L)).foreach { versionToLoad => testExpectedError[InvalidTableException]( fileList, lastCheckpointVersion.map(Long.box), versionToLoad.map(Long.box), expectedErrorMessageContains = "Missing delta file for version 10") } } } test("getLogSegmentForVersion: corrupt listing: checkpoint(10); 11 to 14.json; no 10.json") { val fileList = singularCheckpointFileStatuses(Seq(10L)) ++ deltaFileStatuses((11L until 15L)) Seq(Optional.empty(), Optional.of(10L)).foreach { lastCheckpointVersion => Seq(Optional.empty(), Optional.of(10L)).foreach { versionToLoad => testExpectedError[InvalidTableException]( fileList, lastCheckpointVersion.map(Long.box), versionToLoad.map(Long.box), expectedErrorMessageContains = "Missing delta file for version 10") } } } test("getLogSegmentForVersion: corrupted log missing json files / no way to construct history") { testExpectedError[InvalidTableException]( deltaFileStatuses(1L until 10L), expectedErrorMessageContains = "Cannot compute snapshot. Missing delta file version 0.") testExpectedError[InvalidTableException]( deltaFileStatuses(15L until 25L) ++ singularCheckpointFileStatuses(Seq(20L)), versionToLoad = Optional.of(17), expectedErrorMessageContains = "Cannot compute snapshot. Missing delta file version 0.") testExpectedError[InvalidTableException]( deltaFileStatuses((0L until 5L) ++ (6L until 9L)), expectedErrorMessageContains = "are not contiguous") // corrupt incomplete multi-part checkpoint val corruptedCheckpointStatuses = FileNames.checkpointFileWithParts(logPath, 10, 5).asScala .map(p => FileStatus.of(p.toString, 10, 10)) .take(4) val deltas = deltaFileStatuses(10L to 13L) testExpectedError[InvalidTableException]( corruptedCheckpointStatuses.toSeq ++ deltas, expectedErrorMessageContains = "Cannot compute snapshot. Missing delta file version 0.") } test("getLogSegmentForVersion: corrupt log but reading outside corrupted range") { testNoCheckpoint( deltaVersions = (0L until 5L) ++ (6L until 9L), versionToLoad = Optional.of(4)) testWithSingularAndMultipartCheckpoint( deltaVersions = 15L until 25L, checkpointVersions = Seq(20), versionToLoad = Optional.of(22)) testWithSingularAndMultipartCheckpoint( deltaVersions = 15L until 25L, checkpointVersions = Seq(20), startCheckpoint = Optional.of(20), versionToLoad = Optional.of(22)) } test("getLogSegmentForVersion: corrupt _last_checkpoint (is after existing versions)") { // in the case of a corrupted _last_checkpoint we revert to listing from version 0 // (on first run newFiles.isEmpty() but since startingCheckpointOpt.isPresent() re-list from 0) testWithSingularAndMultipartCheckpoint( (0L until 25L), Seq(10L, 20L), startCheckpoint = Optional.of(30)) } test("getLogSegmentForVersion: corrupt _last_checkpoint refers to in range version " + "but no valid checkpoint") { // _last_checkpoint refers to a v1 checkpoint at version 20 that is missing testExpectedError[RuntimeException]( deltaFileStatuses(0L until 25L) ++ singularCheckpointFileStatuses(Seq(10L)), lastCheckpointVersion = Optional.of(20), expectedErrorMessageContains = "Missing checkpoint at version 20") // _last_checkpoint refers to incomplete multi-part checkpoint at version 20 that is missing val corruptedCheckpointStatuses = FileNames.checkpointFileWithParts(logPath, 20, 5).asScala .map(p => FileStatus.of(p.toString, 10, 10)) .take(4) testExpectedError[RuntimeException]( files = corruptedCheckpointStatuses.toSeq ++ deltaFileStatuses(10L to 20L) ++ singularCheckpointFileStatuses(Seq(10L)), lastCheckpointVersion = Optional.of(20), expectedErrorMessageContains = "Missing checkpoint at version 20") } test("getLogSegmentForVersion: corrupted incomplete multi-part checkpoint with no" + "_last_checkpoint or a valid _last_checkpoint provided") { val cases: Seq[(Long, Seq[Long], Seq[Long], Optional[java.lang.Long])] = Seq( /* (corruptedCheckpointVersion, validCheckpointVersions, deltaVersions, lastCheckpointV) */ (20, Seq(10), (10L to 20L), Optional.empty()), (20, Seq(10), (10L to 20L), Optional.of(10)), (10, Seq.empty, (0L to 10L), Optional.empty())) cases.foreach { case (corruptedVersion, validVersions, deltaVersions, lastCheckpointVersion) => val corruptedCheckpoint = FileNames.checkpointFileWithParts(logPath, corruptedVersion, 5) .asScala .map(p => FileStatus.of(p.toString, 10, 10)) .take(4) val checkpoints = singularCheckpointFileStatuses(validVersions) val deltas = deltaFileStatuses(deltaVersions) val allFiles = deltas ++ corruptedCheckpoint ++ checkpoints val logSegment = snapshotManager.getLogSegmentForVersion( createMockFSAndJsonEngineForLastCheckpoint(allFiles, lastCheckpointVersion), Optional.empty()) val checkpointVersion = validVersions.sorted.lastOption checkLogSegment( logSegment, expectedVersion = deltaVersions.max, expectedDeltas = deltaFileStatuses( deltaVersions.filter(_ > checkpointVersion.getOrElse(-1L))), expectedCompactions = Seq.empty, expectedCheckpoints = checkpoints, expectedCheckpointVersion = checkpointVersion, expectedLastCommitTimestamp = deltaVersions.max * 10) } } test("getLogSegmentForVersion: corrupt _last_checkpoint with empty delta log") { val exMsg = intercept[InvalidTableException] { snapshotManager.getLogSegmentForVersion( createMockFSAndJsonEngineForLastCheckpoint(Seq.empty, Optional.of(1)), Optional.empty()) }.getMessage assert(exMsg.contains("Missing checkpoint at version 1")) } /* ------------------- CATALOG MANAGED TABLE TESTS ------------------ */ test("catalog managed: latest query, we load the maxCatalogVersion even if other deltas exist") { val deltas = deltaFileStatuses(0L to 20) val checkpoints = singularCheckpointFileStatuses(Seq(10)) val logSegment = snapshotManager.getLogSegmentForVersion( createMockFSListFromEngine(deltas ++ checkpoints), Optional.empty(), // timeTravelVersionOpt Collections.emptyList(), // parsedLogDatas Optional.of(15L) // maxCatalogVersionOpt ) checkLogSegment( logSegment, expectedVersion = 15, expectedDeltas = deltaFileStatuses(11L to 15), expectedCompactions = Seq.empty, expectedCheckpoints = singularCheckpointFileStatuses(Seq(10)), expectedCheckpointVersion = Some(10), expectedLastCommitTimestamp = 150L) } test("catalog managed: latest query, _last_checkpoint does not exist") { val deltas = deltaFileStatuses(0L to 20) val checkpoints = singularCheckpointFileStatuses(Seq(10)) val logSegment = snapshotManager.getLogSegmentForVersion( createMockFSListFromEngine(deltas ++ checkpoints), Optional.empty(), // timeTravelVersionOpt Collections.emptyList(), // parsedLogDatas Optional.of(20L) // maxCatalogVersionOpt ) // Should find checkpoint at version 10 by searching backwards from version 20 checkLogSegment( logSegment, expectedVersion = 20, expectedDeltas = deltaFileStatuses(11L to 20), expectedCompactions = Seq.empty, expectedCheckpoints = singularCheckpointFileStatuses(Seq(10)), expectedCheckpointVersion = Some(10), expectedLastCommitTimestamp = 200L) } test("catalog managed: latest query, when _last_checkpoint exists and " + "is <= maxCatalogVersion we use it") { val deltas = deltaFileStatuses(0L to 30) val checkpoints = singularCheckpointFileStatuses(Seq(10, 20, 25)) val lastCheckpointFileStatus = FileStatus.of(s"$logPath/_last_checkpoint", 2, 2) val files = deltas ++ checkpoints ++ Seq(lastCheckpointFileStatus) // Create mocked engine that fails if we try to list before the version stored in // _last_checkpoint def listFrom(filePath: String): Seq[FileStatus] = { if (filePath < FileNames.listingPrefix(logPath, 25)) { throw new RuntimeException( s"Listing from before the checkpoint version referenced by _last_checkpoint.") } listFromProvider(files)(filePath) } val mockedEngine = mockEngine( jsonHandler = new MockReadLastCheckpointFileJsonHandler( lastCheckpointFileStatus.getPath, 25 ), // _last_checkpoint points to version 25 fileSystemClient = new MockListFromFileSystemClient(listFrom)) // Latest query with catalog managed table (maxCatalogVersion = 30) val logSegment = snapshotManager.getLogSegmentForVersion( mockedEngine, Optional.empty(), // timeTravelVersionOpt Collections.emptyList(), // parsedLogDatas Optional.of(30L) // maxCatalogVersionOpt ) checkLogSegment( logSegment, expectedVersion = 30, expectedDeltas = deltaFileStatuses(26L to 30), expectedCompactions = Seq.empty, expectedCheckpoints = singularCheckpointFileStatuses(Seq(25)), expectedCheckpointVersion = Some(25), expectedLastCommitTimestamp = 300L) } test("catalog managed:" + "latest query, ignore _last_checkpoint if it's newer than maxCatalogVersion") { val deltas = deltaFileStatuses(0L to 26) val checkpoints = singularCheckpointFileStatuses(Seq(10, 20, 25)) val lastCheckpointFileStatus = FileStatus.of(s"$logPath/_last_checkpoint", 2, 2) val files = deltas ++ checkpoints ++ Seq(lastCheckpointFileStatus) // Latest query with catalog managed table where maxCatalogVersion < _last_checkpoint version val logSegment = snapshotManager.getLogSegmentForVersion( mockEngine( jsonHandler = new MockReadLastCheckpointFileJsonHandler( lastCheckpointFileStatus.getPath, 25 ), // _last_checkpoint points to version 25 fileSystemClient = new MockListFromFileSystemClient(listFromProvider(files))), Optional.empty(), // timeTravelVersionOpt Collections.emptyList(), // parsedLogDatas Optional.of(24L) // maxCatalogVersionOpt is 24, which is < 25 ) // Should use checkpoint at version 20 not 25, and should load maxCatalogVersion checkLogSegment( logSegment, expectedVersion = 24, expectedDeltas = deltaFileStatuses(21L to 24), expectedCompactions = Seq.empty, expectedCheckpoints = singularCheckpointFileStatuses(Seq(20)), expectedCheckpointVersion = Some(20), expectedLastCommitTimestamp = 240L) } test("catalog managed: time travel query ignores _last_checkpoint") { val deltas = deltaFileStatuses(0L to 30) val checkpoints = singularCheckpointFileStatuses(Seq(10, 20, 25)) val lastCheckpointFileStatus = FileStatus.of(s"$logPath/_last_checkpoint", 2, 2) val files = deltas ++ checkpoints ++ Seq(lastCheckpointFileStatus) val jsonHandler = new BaseMockJsonHandler { override def readJsonFiles( fileIter: CloseableIterator[FileStatus], physicalSchema: StructType, predicate: Optional[Predicate]): CloseableIterator[ColumnarBatch] = { assert(fileIter.hasNext) if (fileIter.next.getPath == lastCheckpointFileStatus.getPath) { throw new RuntimeException( "We should not be reading the _last_checkpoint file for time-travel queries") } else { throw new RuntimeException("We should not be reading JSON files besides " + "_last_checkpoint during log segment construction") } } } // Time travel query with catalog managed table val logSegment = snapshotManager.getLogSegmentForVersion( mockEngine( jsonHandler = jsonHandler, fileSystemClient = new MockListFromFileSystemClient(listFromProvider(files))), Optional.of(15L), // timeTravelVersionOpt = 15 Collections.emptyList(), // parsedLogDatas Optional.of(30L) // maxCatalogVersionOpt ) // Should use checkpoint at version 10 for time travel to version 15 checkLogSegment( logSegment, expectedVersion = 15, expectedDeltas = deltaFileStatuses(11L to 15), expectedCompactions = Seq.empty, expectedCheckpoints = singularCheckpointFileStatuses(Seq(10)), expectedCheckpointVersion = Some(10), expectedLastCommitTimestamp = 150L) } /* ------------------- Compaction tests ------------------ */ test("One compaction") { testWithCompactionsNoCheckpoint( deltaVersions = 0L until 5L, compactionVersions = Seq((0, 4))) testWithCompactionsNoCheckpoint( deltaVersions = 0L until 5L, compactionVersions = Seq((0, 4)), versionToLoad = Optional.of(4)) } test("Compaction extends too far") { testWithCompactionsNoCheckpoint( deltaVersions = 0L until 5L, compactionVersions = Seq((3, 5)), versionToLoad = Optional.of(4)) } test("Compaction after checkpoint") { testWithCheckpoints( deltaVersions = 0L until 6L, checkpointVersions = Seq(2), multiCheckpointVersions = Seq.empty, compactionVersions = Seq((3, 5))) } test("Compaction starting before checkpoint") { testWithCheckpoints( deltaVersions = 0L until 6L, checkpointVersions = Seq(2), multiCheckpointVersions = Seq.empty, compactionVersions = Seq((1, 5))) } test("Compaction starting same as checkpoint") { testWithCheckpoints( deltaVersions = 0L until 5L, checkpointVersions = Seq(2), multiCheckpointVersions = Seq.empty, compactionVersions = Seq((2, 5))) } } trait SidecarIteratorProvider extends VectorTestUtils { private def buildSidecarBatch(sidecars: Seq[FileStatus]): ColumnarBatch = new ColumnarBatch { override def getSchema: StructType = SidecarFile.READ_SCHEMA override def getColumnVector(ordinal: Int): ColumnVector = ordinal match { case 0 => stringVector(sidecars.map(_.getPath)) // path case 1 => longVector(sidecars.map(_.getSize).map(JLong.valueOf)) // size case 2 => longVector(sidecars.map(_.getModificationTime).map(JLong.valueOf)) // modification time } override def getSize: Int = sidecars.length } def singletonSidecarParquetIterator(sidecars: Seq[FileStatus], v2CheckpointFileName: String) : CloseableIterator[FileReadResult] = { val batch = buildSidecarBatch(sidecars) Utils.singletonCloseableIterator(new FileReadResult(batch, v2CheckpointFileName)) } // TODO: [delta-io/delta#4849] extend FileReadResult for JSON read result def singletonSidecarJsonIterator(sidecars: Seq[FileStatus]): CloseableIterator[ColumnarBatch] = { val batch = buildSidecarBatch(sidecars) Utils.singletonCloseableIterator(batch) } } class MockSidecarParquetHandler(sidecars: Seq[FileStatus], v2CheckpointFileName: String) extends BaseMockParquetHandler with SidecarIteratorProvider { override def readParquetFiles( fileIter: CloseableIterator[FileStatus], physicalSchema: StructType, predicate: Optional[Predicate]): CloseableIterator[FileReadResult] = singletonSidecarParquetIterator(sidecars, v2CheckpointFileName) } class MockSidecarJsonHandler(sidecars: Seq[FileStatus]) extends BaseMockJsonHandler with SidecarIteratorProvider { override def readJsonFiles( fileIter: CloseableIterator[FileStatus], physicalSchema: StructType, predicate: Optional[Predicate]): CloseableIterator[ColumnarBatch] = singletonSidecarJsonIterator(sidecars) } class MockReadLastCheckpointFileJsonHandler( lastCheckpointPath: String, lastCheckpointVersion: Long) extends BaseMockJsonHandler with VectorTestUtils { override def readJsonFiles( fileIter: CloseableIterator[FileStatus], physicalSchema: StructType, predicate: Optional[Predicate]): CloseableIterator[ColumnarBatch] = { assert(fileIter.hasNext) assert(fileIter.next.getPath == lastCheckpointPath) Utils.singletonCloseableIterator( new ColumnarBatch { override def getSchema: StructType = CheckpointMetaData.READ_SCHEMA override def getColumnVector(ordinal: Int): ColumnVector = { ordinal match { case 0 => longVector(Seq(lastCheckpointVersion)) /* version */ case 1 => longVector(Seq(100)) /* size */ case 2 => longVector(Seq(1)) /* parts */ case 3 => mapTypeVector(Seq(Map.empty[String, String])) } } override def getSize: Int = 1 }) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/TableConfigSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal import scala.collection.JavaConverters._ import io.delta.kernel.exceptions.KernelException import org.scalatest.funsuite.AnyFunSuite class TableConfigSuite extends AnyFunSuite { test("check TableConfig.editable is true") { TableConfig.validateAndNormalizeDeltaProperties( Map( TableConfig.TOMBSTONE_RETENTION.getKey -> "interval 2 week", TableConfig.CHECKPOINT_INTERVAL.getKey -> "20", TableConfig.IN_COMMIT_TIMESTAMPS_ENABLED.getKey -> "true", TableConfig.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.getKey -> "1", TableConfig.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.getKey -> "1", TableConfig.COLUMN_MAPPING_MODE.getKey -> "name", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true", TableConfig.UNIVERSAL_FORMAT_ENABLED_FORMATS.getKey -> "iceberg").asJava) } test("check TableConfig.MAX_COLUMN_ID.editable is false") { val e = intercept[KernelException] { TableConfig.validateAndNormalizeDeltaProperties( Map( TableConfig.TOMBSTONE_RETENTION.getKey -> "interval 2 week", TableConfig.CHECKPOINT_INTERVAL.getKey -> "20", TableConfig.IN_COMMIT_TIMESTAMPS_ENABLED.getKey -> "true", TableConfig.COLUMN_MAPPING_MAX_COLUMN_ID.getKey -> "10").asJava) } assert(e.isInstanceOf[KernelException]) assert(e.getMessage === s"The Delta table property " + s"'${TableConfig.COLUMN_MAPPING_MAX_COLUMN_ID.getKey}'" + s" is an internal property and cannot be updated.") } Seq( Map[String, String](), Map(TableConfig.UNIVERSAL_FORMAT_ENABLED_FORMATS.getKey -> "")).foreach { config => { test( s"Parsing UNIVERSAL_ENABLED formats returns empty set when key is not present $config") { val formats = TableConfig.UNIVERSAL_FORMAT_ENABLED_FORMATS.fromMetadata(config.asJava) assert(formats.isEmpty) } } } test("Parsing UNIVERSAL_ENABLED_FORMATS can parse spaces") { val FORMATS_KEY = TableConfig.UNIVERSAL_FORMAT_ENABLED_FORMATS.getKey val config = Map(FORMATS_KEY -> "iceberg, hudi ").asJava val formats = TableConfig.UNIVERSAL_FORMAT_ENABLED_FORMATS.fromMetadata(config) assert(formats == Set("iceberg", "hudi").asJava) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/TableImplSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal import io.delta.kernel.Snapshot import io.delta.kernel.engine.Engine import io.delta.kernel.exceptions.KernelException import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.util.{Clock, FileNames, ManualClock} import io.delta.kernel.test.{MockFileSystemClientUtils, MockListFromResolvePathFileSystemClient} import io.delta.kernel.test.MockSnapshotUtils.getMockSnapshot import io.delta.kernel.utils.FileStatus import org.scalatest.funsuite.AnyFunSuite class TableImplSuite extends AnyFunSuite with MockFileSystemClientUtils { /** * Both timestamp-based travel methods need to be able to construct the latest snapshot * internally. This class overrides getLatestSnapshot to return a mocked snapshot. */ class TableImplWithMockedLatestSnapshot(tablePath: String, clock: Clock, latestVersion: Long) extends TableImpl(tablePath, clock) { override def getLatestSnapshot(engine: Engine): SnapshotImpl = { getMockSnapshot( new Path(tablePath), latestVersion) } } def checkGetVersionBeforeOrAtTimestamp( fileList: Seq[FileStatus], timestamp: Long, expectedVersion: Option[Long] = None, expectedErrorMessageContains: Option[String] = None): Unit = { // Check our inputs are as expected assert(expectedVersion.isEmpty || expectedErrorMessageContains.isEmpty) assert(expectedVersion.nonEmpty || expectedErrorMessageContains.nonEmpty) val engine = mockEngine(fileSystemClient = new MockListFromResolvePathFileSystemClient(listFromProvider(fileList))) val latestVersion = fileList.map(fs => FileNames.getFileVersion(new Path(fs.getPath))).max val table = new TableImplWithMockedLatestSnapshot(dataPath.toString, new ManualClock(0), latestVersion) expectedVersion.foreach { v => assert(table.asInstanceOf[TableImpl].getVersionBeforeOrAtTimestamp(engine, timestamp) == v) } expectedErrorMessageContains.foreach { s => assert(intercept[KernelException] { table.asInstanceOf[TableImpl].getVersionBeforeOrAtTimestamp(engine, timestamp) }.getMessage.contains(s)) } } def checkGetVersionAtOrAfterTimestamp( fileList: Seq[FileStatus], timestamp: Long, expectedVersion: Option[Long] = None, expectedErrorMessageContains: Option[String] = None): Unit = { // Check our inputs are as expected assert(expectedVersion.isEmpty || expectedErrorMessageContains.isEmpty) assert(expectedVersion.nonEmpty || expectedErrorMessageContains.nonEmpty) val engine = mockEngine(fileSystemClient = new MockListFromResolvePathFileSystemClient(listFromProvider(fileList))) val latestVersion = fileList.map(fs => FileNames.getFileVersion(new Path(fs.getPath))).max val table = new TableImplWithMockedLatestSnapshot(dataPath.toString, new ManualClock(0), latestVersion) expectedVersion.foreach { v => assert(table.asInstanceOf[TableImpl].getVersionAtOrAfterTimestamp(engine, timestamp) == v) } expectedErrorMessageContains.foreach { s => assert(intercept[KernelException] { table.asInstanceOf[TableImpl].getVersionAtOrAfterTimestamp(engine, timestamp) }.getMessage.contains(s)) } } test("getVersionBeforeOrAtTimestamp: basic case from 0") { val deltaFiles = deltaFileStatuses(Seq(0L, 1L)) checkGetVersionBeforeOrAtTimestamp( deltaFiles, -1, expectedErrorMessageContains = Some("is before the earliest available version 0") ) // before 0 checkGetVersionBeforeOrAtTimestamp(deltaFiles, 0, expectedVersion = Some(0)) // at 0 checkGetVersionBeforeOrAtTimestamp(deltaFiles, 5, expectedVersion = Some(0)) // btw 0, 1 checkGetVersionBeforeOrAtTimestamp(deltaFiles, 10, expectedVersion = Some(1)) // at 1 checkGetVersionBeforeOrAtTimestamp(deltaFiles, 11, expectedVersion = Some(1)) // after 1 } test("getVersionAtOrAfterTimestamp: basic case from 0") { val deltaFiles = deltaFileStatuses(Seq(0L, 1L)) checkGetVersionAtOrAfterTimestamp(deltaFiles, -1, expectedVersion = Some(0)) // before 0 checkGetVersionAtOrAfterTimestamp(deltaFiles, 0, expectedVersion = Some(0)) // at 0 checkGetVersionAtOrAfterTimestamp(deltaFiles, 5, expectedVersion = Some(1)) // btw 0, 1 checkGetVersionAtOrAfterTimestamp(deltaFiles, 10, expectedVersion = Some(1)) // at 1 checkGetVersionAtOrAfterTimestamp( deltaFiles, 11, expectedErrorMessageContains = Some("is after the latest available version 1") ) // after 1 } test("getVersionBeforeOrAtTimestamp: w/ checkpoint + w/o checkpoint") { Seq( deltaFileStatuses(Seq(10L, 11L, 12L)) ++ singularCheckpointFileStatuses(Seq(10L)), deltaFileStatuses(Seq(10L, 11L, 12L)) // checks that does not need to be recreatable ).foreach { deltaFiles => checkGetVersionBeforeOrAtTimestamp( deltaFiles, 99, // before 10 expectedErrorMessageContains = Some("is before the earliest available version 10")) checkGetVersionBeforeOrAtTimestamp(deltaFiles, 100, expectedVersion = Some(10)) // at 10 checkGetVersionBeforeOrAtTimestamp(deltaFiles, 105, expectedVersion = Some(10)) // btw 10, 11 checkGetVersionBeforeOrAtTimestamp(deltaFiles, 110, expectedVersion = Some(11)) // at 11 checkGetVersionBeforeOrAtTimestamp(deltaFiles, 115, expectedVersion = Some(11)) // btw 11, 12 checkGetVersionBeforeOrAtTimestamp(deltaFiles, 120, expectedVersion = Some(12)) // at 12 checkGetVersionBeforeOrAtTimestamp(deltaFiles, 125, expectedVersion = Some(12)) // after 12 } } test("getVersionAtOrAfterTimestamp: w/ checkpoint + w/o checkpoint") { Seq( deltaFileStatuses(Seq(10L, 11L, 12L)) ++ singularCheckpointFileStatuses(Seq(10L)), deltaFileStatuses(Seq(10L, 11L, 12L)) // checks that does not need to be recreatable ).foreach { deltaFiles => checkGetVersionAtOrAfterTimestamp(deltaFiles, 99, expectedVersion = Some(10)) // before 10 checkGetVersionAtOrAfterTimestamp(deltaFiles, 100, expectedVersion = Some(10)) // at 10 checkGetVersionAtOrAfterTimestamp(deltaFiles, 105, expectedVersion = Some(11)) // btw 10, 11 checkGetVersionAtOrAfterTimestamp(deltaFiles, 110, expectedVersion = Some(11)) // at 11 checkGetVersionAtOrAfterTimestamp(deltaFiles, 115, expectedVersion = Some(12)) // btw 11, 12 checkGetVersionAtOrAfterTimestamp(deltaFiles, 120, expectedVersion = Some(12)) // at 12 checkGetVersionAtOrAfterTimestamp( deltaFiles, 125, expectedErrorMessageContains = Some("is after the latest available version 12") ) // after 12 } } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/TransactionBuilderImplSuite.scala ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal import java.util.Optional import scala.collection.JavaConverters._ import io.delta.kernel.Operation import io.delta.kernel.internal.actions.{Format, Metadata, Protocol} import io.delta.kernel.internal.checksum.CRCInfo import io.delta.kernel.internal.commit.DefaultFileSystemManagedTableOnlyCommitter import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.lang.Lazy import io.delta.kernel.internal.metrics.SnapshotQueryContext import io.delta.kernel.internal.snapshot.LogSegment import io.delta.kernel.internal.util.FileNames import io.delta.kernel.internal.util.VectorUtils.{buildArrayValue, stringStringMapValue} import io.delta.kernel.test.MockEngineUtils import io.delta.kernel.types.{IntegerType, StringType, StructType} import io.delta.kernel.utils.FileStatus import org.scalatest.funsuite.AnyFunSuite class TransactionBuilderImplSuite extends AnyFunSuite with MockEngineUtils { /** * Creates a mock snapshot whose metadata includes a `delta.feature.` property that * TransactionMetadataFactory would attempt to process. This is used to verify the early-return * path: if the code falls through to the factory, processing the unknown feature throws. */ private def createMockSnapshot( dataPath: Path, version: Long, extraConfig: Map[String, String] = Map.empty): SnapshotImpl = { val schema = new StructType().add("id", IntegerType.INTEGER) val metadata = new Metadata( "id", Optional.empty(), Optional.empty(), new Format(), schema.toJson, schema, buildArrayValue(java.util.Arrays.asList(), StringType.STRING), Optional.of(123), stringStringMapValue(extraConfig.asJava)) val logPath = new Path(dataPath, "_delta_log") val fs = FileStatus.of(FileNames.deltaFile(logPath, version), 1, 1) val logSegment = new LogSegment( logPath, version, Seq(fs).asJava, Seq.empty.asJava, Seq.empty.asJava, fs, Optional.empty(), Optional.empty()) val snapshotQueryContext = SnapshotQueryContext.forLatestSnapshot(dataPath.toString) new SnapshotImpl( dataPath, logSegment.getVersion, new Lazy(() => logSegment), null, // logReplay - not needed; getCurrentCrcInfo is overridden below new Protocol(1, 2), metadata, DefaultFileSystemManagedTableOnlyCommitter.INSTANCE, snapshotQueryContext, Optional.empty()) { override def getCurrentCrcInfo: Optional[CRCInfo] = Optional.empty() } } test("early return when no metadata or protocol update is needed") { // The snapshot metadata includes an unrecognized delta.feature.* property. If the code // falls through to TransactionMetadataFactory (the bug), extractFeaturePropertyOverrides // will try to resolve this feature and throw. The early-return path skips the factory // entirely, so no exception is thrown. val dataPath = new Path("/tmp/test-table") val tableImpl = new TableImpl(dataPath.toString, () => System.currentTimeMillis()) val snapshot = createMockSnapshot( dataPath, version = 0L, extraConfig = Map("delta.feature.fakeFeatureForEarlyReturnTest" -> "supported")) val builder = new TransactionBuilderImpl(tableImpl, "test-engine", Operation.WRITE) // With the fix this returns immediately; without the fix this would throw val txn = builder.buildTransactionInternal( mockEngine(), false, // isCreateOrReplace Optional.of(snapshot)) // Verify the returned transaction does not mark protocol/metadata for update assert(txn.getSchema(mockEngine()) === snapshot.getMetadata().getSchema()) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/TransactionMetadataFactorySuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal import java.util.Optional import scala.collection.JavaConverters._ import io.delta.kernel.expressions.Column import io.delta.kernel.internal.fs.Path import io.delta.kernel.test.MockSnapshotUtils import io.delta.kernel.types.{IntegerType, StringType, StructType} import org.scalatest.funsuite.AnyFunSuite class TransactionMetadataFactorySuite extends AnyFunSuite { // Test schema for metadata creation val testSchema: StructType = new StructType() .add("id", IntegerType.INTEGER) .add("name", StringType.STRING) .add("value", IntegerType.INTEGER) val testTablePath = "/test/table/path" // ===================================================================== // buildCreateTableMetadata tests // ===================================================================== test("buildCreateTableMetadata - basic schema without partitions or clustering") { val tableProperties = Map("key1" -> "value1", "key2" -> "value2").asJava val output = TransactionMetadataFactory.buildCreateTableMetadata( testTablePath, testSchema, tableProperties, Optional.empty(), // no partition columns Optional.empty(), // no clustering columns Optional.empty() // no custom committer ) // Verify both metadata and protocol are present for create table assert(output.newMetadata.isPresent, "New metadata should be present for create table") assert(output.newProtocol.isPresent, "New protocol should be present for create table") assert( !output.physicalNewClusteringColumns.isPresent, "No clustering columns should be present") val metadata = output.newMetadata.get() assert(metadata.getSchema === testSchema) assert(metadata.getConfiguration.asScala === tableProperties.asScala) } test("buildCreateTableMetadata - with partition columns") { val tableProperties = Map("delta.autoOptimize" -> "true").asJava val partitionCols = Optional.of(List("name").asJava) val output = TransactionMetadataFactory.buildCreateTableMetadata( testTablePath, testSchema, tableProperties, partitionCols, Optional.empty(), Optional.empty() /* committerOpt */ ) assert(output.newMetadata.isPresent) assert(output.newProtocol.isPresent) assert(!output.physicalNewClusteringColumns.isPresent) val metadata = output.newMetadata.get() assert(metadata.getSchema === testSchema) // Verify partition columns are set correctly val partitionColumns = metadata.getPartitionColumns assert(partitionColumns.getSize === 1) } test("buildCreateTableMetadata - with clustering columns") { val tableProperties = Map("delta.feature.clustering" -> "supported").asJava val clusteringCols = Optional.of(List(new Column("name")).asJava) val output = TransactionMetadataFactory.buildCreateTableMetadata( testTablePath, testSchema, tableProperties, Optional.empty(), // no partition columns clusteringCols, Optional.empty() /* committerOpt */ ) assert(output.newMetadata.isPresent) assert(output.newProtocol.isPresent) assert(output.physicalNewClusteringColumns.isPresent && output.physicalNewClusteringColumns.get.size == 1) } test("buildCreateTableMetadata - should reject both partition and clustering columns") { val tableProperties = Map.empty[String, String].asJava val partitionCols = Optional.of(List("name").asJava) val clusteringCols = Optional.of(List(new Column("value")).asJava) assertThrows[IllegalArgumentException] { TransactionMetadataFactory.buildCreateTableMetadata( testTablePath, testSchema, tableProperties, partitionCols, clusteringCols, Optional.empty() /* committerOpt */ ) } } // ===================================================================== // buildReplaceTableMetadata tests // ===================================================================== test("buildReplaceTableMetadata - basic replacement") { val newTableProperties = Map("newKey" -> "newValue").asJava // Create a mock snapshot for the existing table val mockSnapshot = MockSnapshotUtils.getMockSnapshot( new Path(testTablePath), latestVersion = 1L) val output = TransactionMetadataFactory.buildReplaceTableMetadata( testTablePath, mockSnapshot, testSchema, newTableProperties, Optional.empty(), // no partition columns Optional.empty() // no clustering columns ) assert(output.newMetadata.isPresent, "New metadata should be present for replace table") assert(output.newProtocol.isPresent, "New protocol should be present for replace table") assert(!output.physicalNewClusteringColumns.isPresent) val metadata = output.newMetadata.get() assert(metadata.getSchema === testSchema) } test("buildReplaceTableMetadata - with partition columns") { val newTableProperties = Map("delta.autoOptimize" -> "false").asJava val partitionCols = Optional.of(List("id").asJava) val mockSnapshot = MockSnapshotUtils.getMockSnapshot( new Path(testTablePath), latestVersion = 2L) val output = TransactionMetadataFactory.buildReplaceTableMetadata( testTablePath, mockSnapshot, testSchema, newTableProperties, partitionCols, Optional.empty()) assert(output.newMetadata.isPresent) assert(output.newProtocol.isPresent) val metadata = output.newMetadata.get() assert(metadata.getSchema === testSchema) } test("buildReplaceTableMetadata - with clustering columns") { val newTableProperties = Map("delta.feature.clustering" -> "supported").asJava val clusteringCols = Optional.of(List(new Column("name"), new Column("value")).asJava) val mockSnapshot = MockSnapshotUtils.getMockSnapshot( new Path(testTablePath), latestVersion = 3L) val output = TransactionMetadataFactory.buildReplaceTableMetadata( testTablePath, mockSnapshot, testSchema, newTableProperties, Optional.empty(), // no partition columns clusteringCols) assert(output.newMetadata.isPresent, "New metadata should be present for replace table") assert(output.newProtocol.isPresent, "New protocol should be present for replace table") assert(output.physicalNewClusteringColumns.isPresent, "Clustering columns should be resolved") val metadata = output.newMetadata.get() assert(metadata.getSchema === testSchema) // Verify clustering columns are resolved val clusteringColumns = output.physicalNewClusteringColumns.get() assert(clusteringColumns.size() === 2, "Should have 2 clustering columns") } test("buildReplaceTableMetadata - should reject both partition and clustering columns") { val newTableProperties = Map.empty[String, String].asJava val partitionCols = Optional.of(List("name").asJava) val clusteringCols = Optional.of(List(new Column("value")).asJava) val mockSnapshot = MockSnapshotUtils.getMockSnapshot( new Path(testTablePath), latestVersion = 1L) assertThrows[IllegalArgumentException] { TransactionMetadataFactory.buildReplaceTableMetadata( testTablePath, mockSnapshot, testSchema, newTableProperties, partitionCols, clusteringCols) } } // ===================================================================== // buildUpdateTableMetadata tests // ===================================================================== test("buildUpdateTableMetadata - no changes") { val mockSnapshot = MockSnapshotUtils.getMockSnapshot( new Path(testTablePath), latestVersion = 3L) val output = TransactionMetadataFactory.buildUpdateTableMetadata( testTablePath, mockSnapshot, Optional.empty(), // no properties added Optional.empty(), // no properties removed Optional.empty(), // no schema change Optional.empty() // no clustering columns ) // With no changes, no new metadata or protocol should be present assert(!output.newMetadata.isPresent, "No new metadata should be present when no changes") assert(!output.newProtocol.isPresent, "No new protocol should be present when no changes") assert(!output.physicalNewClusteringColumns.isPresent) } test("buildUpdateTableMetadata - add table properties") { val mockSnapshot = MockSnapshotUtils.getMockSnapshot( new Path(testTablePath), latestVersion = 4L) val newProperties = Map("newKey" -> "newValue", "anotherKey" -> "anotherValue").asJava val output = TransactionMetadataFactory.buildUpdateTableMetadata( testTablePath, mockSnapshot, Optional.of(newProperties), Optional.empty(), Optional.empty(), Optional.empty()) // Properties change should trigger new metadata assert(output.newMetadata.isPresent, "New metadata should be present when properties change") val metadata = output.newMetadata.get() // Verify the new properties are merged assert(metadata.getConfiguration.containsKey("newKey")) assert(metadata.getConfiguration.get("newKey") === "newValue") } test("buildUpdateTableMetadata - with clustering columns") { val mockSnapshot = MockSnapshotUtils.getMockSnapshot( new Path(testTablePath), latestVersion = 9L) val clusteringColumns = Optional.of(List(new Column("name")).asJava) val output = TransactionMetadataFactory.buildUpdateTableMetadata( testTablePath, mockSnapshot, Optional.empty(), Optional.empty(), Optional.empty(), clusteringColumns) assert(output.physicalNewClusteringColumns.isPresent && output.physicalNewClusteringColumns.get.size == 1) } test("buildUpdateTableMetadata - overlapping set and unset properties should fail") { val mockSnapshot = MockSnapshotUtils.getMockSnapshot( new Path(testTablePath), latestVersion = 7L) val propertiesToAdd = Map("conflictKey" -> "newValue").asJava val propertyKeysToRemove = Set("conflictKey").asJava assertThrows[Exception] { TransactionMetadataFactory.buildUpdateTableMetadata( testTablePath, mockSnapshot, Optional.of(propertiesToAdd), Optional.of(propertyKeysToRemove), Optional.empty(), Optional.empty()) } } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/AddFileSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.actions import java.lang.{Boolean => JBoolean, Long => JLong} import java.util.Optional import scala.collection.JavaConverters._ import io.delta.kernel.data.Row import io.delta.kernel.internal.util.VectorUtils import io.delta.kernel.internal.util.VectorUtils.stringStringMapValue import io.delta.kernel.statistics.DataFileStatistics import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.must.Matchers class AddFileSuite extends AnyFunSuite with Matchers { /** * Generate a Row representing an AddFile action with provided fields. */ private def generateTestAddFileRow( path: String = "path", partitionValues: Map[String, String] = Map.empty, size: Long = 10L, modificationTime: Long = 20L, dataChange: Boolean = true, deletionVector: Option[DeletionVectorDescriptor] = Option.empty, tags: Option[Map[String, String]] = Option.empty, baseRowId: Option[Long] = Option.empty, defaultRowCommitVersion: Option[Long] = Option.empty, stats: Option[String] = Option.empty): Row = { def toJavaOptional[T](option: Option[T]): Optional[T] = option match { case Some(value) => Optional.of(value) case None => Optional.empty() } AddFile.createAddFileRow( null, path, stringStringMapValue(partitionValues.asJava), size.asInstanceOf[JLong], modificationTime.asInstanceOf[JLong], dataChange.asInstanceOf[JBoolean], toJavaOptional(deletionVector), toJavaOptional(tags.map(_.asJava).map(stringStringMapValue)), toJavaOptional(baseRowId.asInstanceOf[Option[JLong]]), toJavaOptional(defaultRowCommitVersion.asInstanceOf[Option[JLong]]), DataFileStatistics.deserializeFromJson(stats.getOrElse(""), null)) } test("getters can read AddFile's fields from the backing row") { val addFileRow = generateTestAddFileRow( path = "test/path", partitionValues = Map("a" -> "1"), size = 1L, modificationTime = 10L, dataChange = false, deletionVector = Option.empty, tags = Option(Map("tag1" -> "value1")), baseRowId = Option(30L), defaultRowCommitVersion = Option(40L), stats = Option("{\"numRecords\":100}")) val addFile = new AddFile(addFileRow) assert(addFile.getPath === "test/path") assert(VectorUtils.toJavaMap(addFile.getPartitionValues).asScala.equals(Map("a" -> "1"))) assert(addFile.getSize === 1L) assert(addFile.getModificationTime === 10L) assert(addFile.getDataChange === false) assert(addFile.getDeletionVector === Optional.empty()) assert(VectorUtils.toJavaMap(addFile.getTags.get()).asScala.equals(Map("tag1" -> "value1"))) assert(addFile.getBaseRowId === Optional.of(30L)) assert(addFile.getDefaultRowCommitVersion === Optional.of(40L)) // DataFileStatistics doesn't have an equals() override, so we need to compare the string assert(addFile.getStats(null).get().serializeAsJson(null) === "{\"numRecords\":100}") assert(addFile.getNumRecords === Optional.of(100L)) } test("update a single field of an AddFile") { val addFileRow = generateTestAddFileRow(baseRowId = Option(1L)) val addFileAction = new AddFile(addFileRow) val updatedAddFileAction = addFileAction.withNewBaseRowId(2L) assert(updatedAddFileAction.getBaseRowId === Optional.of(2L)) val updatedAddFileRow = updatedAddFileAction.toRow assert(new AddFile(updatedAddFileRow).getBaseRowId === Optional.of(2L)) } test("update multiple fields of an AddFile multiple times") { val baseAddFileRow = generateTestAddFileRow( path = "test/path", baseRowId = Option(0L), defaultRowCommitVersion = Option(0L)) var addFileAction = new AddFile(baseAddFileRow) (1L until 10L).foreach { i => addFileAction = addFileAction .withNewBaseRowId(i) .withNewDefaultRowCommitVersion(i * 10) assert(addFileAction.getPath === "test/path") assert(addFileAction.getBaseRowId === Optional.of(i)) assert(addFileAction.getDefaultRowCommitVersion === Optional.of(i * 10)) } } test("toString() prints all fields of AddFile") { Seq(true, false).foreach { dvPresent => val deletionVector = if (dvPresent) { Some(new DeletionVectorDescriptor( "storage", "s", Optional.of(1), 25, 35)) } else { None } val addFileRow = generateTestAddFileRow( path = "test/path", partitionValues = Map("col1" -> "val1"), size = 100L, modificationTime = 1234L, dataChange = false, tags = Option(Map("tag1" -> "value1")), baseRowId = Option(12345L), defaultRowCommitVersion = Option(67890L), stats = Option("{\"numRecords\":10000}"), deletionVector = deletionVector) val addFile = new AddFile(addFileRow) val deletionVectorString = if (dvPresent) { "Optional[DeletionVectorDescriptor(storageType=storage," + " pathOrInlineDv=s, offset=Optional[1], sizeInBytes=25, cardinality=35)]" } else { "Optional.empty" } val expectedString = "AddFile{" + "path='test/path', " + "partitionValues={col1=val1}, " + "size=100, " + "modificationTime=1234, " + "dataChange=false, " + s"deletionVector=$deletionVectorString, " + "tags=Optional[{tag1=value1}], " + "baseRowId=Optional[12345], " + "defaultRowCommitVersion=Optional[67890], " + "stats={\"numRecords\":10000}}" assert(addFile.toString == expectedString) } } test("equals() compares AddFile instances correctly") { val addFileRow1 = generateTestAddFileRow( path = "test/path", size = 100L, partitionValues = Map("a" -> "1"), baseRowId = Option(12345L), stats = Option("{\"numRecords\":100}")) // Create an identical AddFile val addFileRow2 = generateTestAddFileRow( path = "test/path", size = 100L, partitionValues = Map("a" -> "1"), baseRowId = Option(12345L), stats = Option("{\"numRecords\":100}")) // Create a AddFile with different path val addFileRowDiffPath = generateTestAddFileRow( path = "different/path", size = 100L, partitionValues = Map("a" -> "1"), baseRowId = Option(12345L), stats = Option("{\"numRecords\":100}")) // Create a AddFile with different partition values, which is handled specially in equals() val addFileRowDiffPartition = generateTestAddFileRow( path = "test/path", size = 100L, partitionValues = Map("x" -> "0"), baseRowId = Option(12345L), stats = Option("{\"numRecords\":100}")) // Create a AddFile with deletion vector value val addFileRowDeletionVector = generateTestAddFileRow( path = "test/path", size = 100L, partitionValues = Map("x" -> "0"), baseRowId = Option(12345L), deletionVector = Some( new DeletionVectorDescriptor( "storage", "s", Optional.of(1), 25, 35)), stats = Option("{\"numRecords\":100}")) val addFile1 = new AddFile(addFileRow1) val addFile2 = new AddFile(addFileRow2) val addFileDiffPath = new AddFile(addFileRowDiffPath) val addFileDiffPartition = new AddFile(addFileRowDiffPartition) val addFileDeletionVector = new AddFile(addFileRowDeletionVector) // Test equality assert(addFile1 === addFile2) assert(addFile1 != addFileDiffPath) assert(addFile1 != addFileDiffPartition) assert(addFile2 != addFileDiffPath) assert(addFile2 != addFileDiffPartition) assert(addFileDiffPath != addFileDiffPartition) assert(addFileDeletionVector != addFileDiffPartition) // Test null and different type assert(!addFile1.equals(null)) assert(!addFile1.equals(new DomainMetadata("domain", "config", false))) } test("hashCode is consistent with equals") { val addFileRow1 = generateTestAddFileRow( path = "test/path", size = 100L, partitionValues = Map("a" -> "1"), baseRowId = Option(12345L), stats = Option("{\"numRecords\":100}")) val addFileRow2 = generateTestAddFileRow( path = "test/path", size = 100L, partitionValues = Map("a" -> "1"), baseRowId = Option(12345L), stats = Option("{\"numRecords\":100}")) val addFile1 = new AddFile(addFileRow1) val addFile2 = new AddFile(addFileRow2) // Equal objects should have equal hash codes assert(addFile1.hashCode === addFile2.hashCode) // Hash code should be consistent across multiple calls assert(addFile1.hashCode === addFile1.hashCode) } // Tests for toRemoveFileRow test("toRemoveFileRow: handles AddFile with all required fields") { val addFile = new AddFile(generateTestAddFileRow( path = "/path/to/file", dataChange = false)) def verify( result: RemoveFile, expDataChange: Boolean, expDeletionTimestamp: Option[Long]): Unit = { assert(result.getPath === "/path/to/file") if (expDeletionTimestamp.isDefined) { assert(result.getDeletionTimestamp.get() === expDeletionTimestamp.get) } assert(result.getDataChange === expDataChange) assert(result.getExtendedFileMetadata === Optional.of(true)) assert(VectorUtils.toJavaMap[String, String](result.getPartitionValues.get()).asScala === Map.empty[String, String]) assert(result.getSize === Optional.of(10L)) assert(result.getStatsJson === Optional.empty()) assert(result.getTags === Optional.empty()) assert(result.getBaseRowId === Optional.empty()) assert(result.getDefaultRowCommitVersion === Optional.empty()) } val result1 = new RemoveFile(addFile.toRemoveFileRow(true, Optional.empty())) verify(result1, expDataChange = true, expDeletionTimestamp = None) val result2 = new RemoveFile(addFile.toRemoveFileRow(false, Optional.empty())) verify(result2, expDataChange = false, expDeletionTimestamp = None) val result3 = new RemoveFile(addFile.toRemoveFileRow(true, Optional.of(100L))) verify(result3, expDataChange = true, expDeletionTimestamp = Some(100L)) } test("toRemoveFileRow: handles AddFile with optional fields present") { val addFile = new AddFile(generateTestAddFileRow( path = "/path/to/file", partitionValues = Map("a" -> "1"), size = 100L, modificationTime = 200L, dataChange = true, deletionVector = None, tags = Some(Map("tag1" -> "value1")), baseRowId = Some(67890L), defaultRowCommitVersion = Some(2823L), stats = Some("{\"numRecords\":100}"))) val result = new RemoveFile(addFile.toRemoveFileRow(false, Optional.of(200L))) assert(result.getPath === "/path/to/file") assert(VectorUtils.toJavaMap[String, String](result.getPartitionValues.get()).asScala === Map("a" -> "1")) assert(result.getSize === Optional.of(100L)) assert(result.getDeletionTimestamp === Optional.of(200L)) assert(result.getDataChange === false) assert(result.getDeletionVector === Optional.empty()) assert(VectorUtils.toJavaMap[String, String](result.getTags.get()).asScala === Map[String, String]("tag1" -> "value1")) assert(result.getBaseRowId === Optional.of(67890L)) assert(result.getDefaultRowCommitVersion === Optional.of(2823L)) assert(result.getStatsJson === Optional.of("{\"numRecords\":100}")) } test("toRemoveFileRow: DV is converted properly") { val addFile = new AddFile(generateTestAddFileRow( path = "/path/to/file", partitionValues = Map("a" -> "1"), size = 100L, modificationTime = 200L, dataChange = true, deletionVector = Some( new DeletionVectorDescriptor( "storage", "s", Optional.of(1), 25, 35)), tags = Some(Map("tag1" -> "value1")), baseRowId = Some(67890L), defaultRowCommitVersion = Some(2823L), stats = Some("{\"numRecords\":100}"))) val result = new RemoveFile(addFile.toRemoveFileRow(true, Optional.of(200L))) assert(result.getPath === "/path/to/file") assert(VectorUtils.toJavaMap[String, String](result.getPartitionValues.get()).asScala === Map[String, String]("a" -> "1")) assert(result.getSize.get() === 100L) assert(result.getDeletionTimestamp.get() === 200L) assert(result.getDataChange === true) assert(result.getDeletionVector === Optional.of( new DeletionVectorDescriptor("storage", "s", Optional.of(1), 25, 35))) assert(VectorUtils.toJavaMap[String, String](result.getTags.get()).asScala === Map[String, String]("tag1" -> "value1")) assert(result.getBaseRowId === Optional.of(67890L)) assert(result.getDefaultRowCommitVersion === Optional.of(2823L)) assert(result.getStatsJson === Optional.of("{\"numRecords\":100}")) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/DeletionVectorDescriptorSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.actions import java.io.{ByteArrayInputStream, DataInputStream} import java.util.{Base64, Optional} import org.scalatest.funsuite.AnyFunSuite /** * Tests for DeletionVectorDescriptor. */ class DeletionVectorDescriptorSuite extends AnyFunSuite { // Test cases: (storageType, pathOrInlineDv, offset, sizeInBytes, cardinality) private val testCases = Seq( ("u", "ab^-aqEH.-t@S}K{vb[*k^", Some(4), 40, 2L), ("p", "path/to/dv.bin", Some(100), 1024, 50L), ("i", "inline_data_here", None, 16, 3L)) testCases.foreach { case (storageType, pathOrInlineDv, offset, sizeInBytes, cardinality) => test(s"serializeToBase64 - $storageType storage type") { val dv = new DeletionVectorDescriptor( storageType, pathOrInlineDv, offset.map(Integer.valueOf).map(Optional.of[Integer]).getOrElse(Optional.empty[Integer]()), sizeInBytes, cardinality) val base64Result = dv.serializeToBase64() // Decode and verify the serialization format val bytes = Base64.getDecoder.decode(base64Result) val dis = new DataInputStream(new ByteArrayInputStream(bytes)) assert(dis.readLong() === cardinality) assert(dis.readInt() === sizeInBytes) assert(dis.readByte().toChar.toString === storageType) if (storageType != "i") { assert(dis.readInt() === offset.get) } assert(dis.readUTF() === pathOrInlineDv) dis.close() } } // Regression test: isInline() must use .equals() not == for String comparison. // Using `new String(...)` creates non-interned Strings that would fail with ==. testCases.foreach { case (storageType, pathOrInlineDv, offset, sizeInBytes, cardinality) => test(s"isInline with non-interned string - $storageType storage type") { val dv = new DeletionVectorDescriptor( new String(storageType), // deliberately non-interned pathOrInlineDv, offset.map(Integer.valueOf).map(Optional.of[Integer]).getOrElse(Optional.empty[Integer]()), sizeInBytes, cardinality) assert(dv.isInline() === (storageType == "i")) } } test("serializeToBase64 throws for non-inline DV without offset") { val ex = intercept[IllegalArgumentException] { val dv = new DeletionVectorDescriptor( "u", "ab^-aqEH.-t@S}K{vb[*k^", Optional.empty[Integer](), 40, 2L) dv.serializeToBase64() } assert(ex.getMessage.contains("Non-inline DV must have offset")) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/GenerateIcebergCompatActionUtilsSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.actions import java.util.{Collections, Optional} import scala.collection.JavaConverters._ import io.delta.kernel.data.Row import io.delta.kernel.exceptions.KernelException import io.delta.kernel.expressions.{Column, Literal} import io.delta.kernel.internal.TableConfig import io.delta.kernel.internal.data.TransactionStateRow import io.delta.kernel.internal.util.{ColumnMapping, VectorUtils} import io.delta.kernel.statistics.DataFileStatistics import io.delta.kernel.types.{IntegerType, StringType, StructType} import io.delta.kernel.utils.DataFileStatus import org.scalatest.funsuite.AnyFunSuite class GenerateIcebergCompatActionUtilsSuite extends AnyFunSuite { import GenerateIcebergCompatActionUtilsSuite._ private def getTestTransactionStateRow( tblProperties: Map[String, String], maxRetries: Int = 0, partitionColumns: Seq[String] = Seq.empty): Row = { val metadata = new Metadata( "id", Optional.empty(), /* name */ Optional.empty(), /* description */ new Format(), testSchema.toJson, testSchema, VectorUtils.buildArrayValue(partitionColumns.asJava, StringType.STRING), Optional.empty(), /* createdTime */ VectorUtils.stringStringMapValue(tblProperties.asJava)) val protocol = new Protocol( 3, // minReaderVersion 7, // minWriterVersion to support table features java.util.Collections.emptySet[String](), // readerFeatures java.util.Collections.emptySet[String]() // writerFeatures ) TransactionStateRow.of( ColumnMapping.updateColumnMappingMetadataIfNeeded(metadata, true).orElse(metadata), protocol, testTablePath, maxRetries) } /* ----- Error cases ----- */ private def testErrorAddAndRemove( txnStateRow: Row, dataFileStatus: DataFileStatus, partitionValues: java.util.Map[String, Literal], dataChange: Boolean, expectedErrorMessageContains: String): Unit = { assert( intercept[UnsupportedOperationException] { GenerateIcebergCompatActionUtils.generateIcebergCompatWriterV1AddAction( txnStateRow, dataFileStatus, partitionValues, dataChange, Optional.empty() /* Pre-parsed physicalSchema if present */ ) }.getMessage.contains(expectedErrorMessageContains)) assert( intercept[UnsupportedOperationException] { GenerateIcebergCompatActionUtils.generateIcebergCompatWriterV1RemoveAction( txnStateRow, dataFileStatus, partitionValues, dataChange, Optional.empty() /* Pre-parsed physicalSchema if present */ ) }.getMessage.contains(expectedErrorMessageContains)) } test("GenerateIcebergCompatActionUtils requires maxRetries=0") { testErrorAddAndRemove( getTestTransactionStateRow(compatibleTableProperties, maxRetries = 1), testDataFileStatusWithStatistics, partitionValues = Collections.emptyMap(), dataChange = true, "GenerateIcebergCompatActionUtils requires maxRetries=0") } test("GenerateIcebergCompatActionUtils requires icebergWriterCompatV1") { // Not set at all testErrorAddAndRemove( getTestTransactionStateRow(tblProperties = Map()), testDataFileStatusWithStatistics, partitionValues = Collections.emptyMap(), dataChange = true, "only supported on tables with 'delta.enableIcebergWriterCompatV1' set to true") // Set to false testErrorAddAndRemove( getTestTransactionStateRow(tblProperties = Map( TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.getKey -> "FALSE", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true", TableConfig.COLUMN_MAPPING_MODE.getKey -> "id")), testDataFileStatusWithStatistics, partitionValues = Collections.emptyMap(), dataChange = true, "only supported on tables with 'delta.enableIcebergWriterCompatV1' set to true") } test("GenerateIcebergCompatActionUtils doesn't support partitioned tables") { testErrorAddAndRemove( getTestTransactionStateRow(compatibleTableProperties, partitionColumns = Seq("id")), testDataFileStatusWithStatistics, partitionValues = Map("id" -> Literal.ofInt(1)).asJava, dataChange = true, "GenerateIcebergCompatActionUtils is not supported for partitioned tables") } test("GenerateIcebergCompatActionUtils requires statistics in add files") { assert( intercept[KernelException] { GenerateIcebergCompatActionUtils.generateIcebergCompatWriterV1AddAction( getTestTransactionStateRow(compatibleTableProperties), testDataFileStatusWithoutStatistics, Collections.emptyMap(), // partitionValues true, // dataChange Optional.empty() // Pre-parsed physical schema ) }.getMessage.contains("icebergCompatV2 compatibility requires 'numRecords' statistic")) } test("GenerateIcebergCompatActionUtils cannot create remove with dataChange=true " + "for append-only table") { val txnStateRow = getTestTransactionStateRow( compatibleTableProperties ++ Map(TableConfig.APPEND_ONLY_ENABLED.getKey -> "true")) assert( intercept[KernelException] { GenerateIcebergCompatActionUtils.generateIcebergCompatWriterV1RemoveAction( txnStateRow, testDataFileStatusWithStatistics, Collections.emptyMap(), // partitionValues true, // dataChange Optional.of(TransactionStateRow.getPhysicalSchema(txnStateRow))) }.getMessage.contains("Cannot modify append-only table")) } /* ----- Valid cases ----- */ private def validateAddAction( row: Row, expectedPath: String, // expectedPartitionValues - for now this is not supported as anything other than empty expectedSize: Long, expectedModificationTime: Long, expectedDataChange: Boolean, expectedStatsString: String): Unit = { assert(row.getSchema == SingleAction.FULL_SCHEMA) (0 until SingleAction.FULL_SCHEMA.length()).foreach { idx => if (idx == SingleAction.ADD_FILE_ORDINAL) { assert(!row.isNullAt(idx)) } else { assert(row.isNullAt(idx)) } } val addRow = row.getStruct(SingleAction.ADD_FILE_ORDINAL) assert(addRow.getSchema == AddFile.FULL_SCHEMA) val addFile = new AddFile(addRow) assert(addFile.getPath == expectedPath) assert(addFile.getPartitionValues.getSize == 0) assert(addFile.getSize == expectedSize) assert(addFile.getModificationTime == expectedModificationTime) assert(addFile.getDataChange == expectedDataChange) assert(!addFile.getTags.isPresent) assert(!addFile.getBaseRowId.isPresent) assert(!addFile.getDefaultRowCommitVersion.isPresent) assert(!addFile.getDeletionVector.isPresent) // We have to do our stats check differently since the AddFile::getStats API does not fully // deserialize the statistics (only grabs the numRecords field) assert(addRow.getString(AddFile.FULL_SCHEMA.indexOf("stats")) == expectedStatsString) } private def validateRemoveAction( row: Row, expectedPath: String, // expectedPartitionValues - for now this is not supported as anything other than empty expectedSize: Long, expectedDeletionTimestamp: Long, expectedDataChange: Boolean, expectedStatsString: Option[String]): Unit = { assert(row.getSchema == SingleAction.FULL_SCHEMA) (0 until SingleAction.FULL_SCHEMA.length()).foreach { idx => if (idx == SingleAction.REMOVE_FILE_ORDINAL) { assert(!row.isNullAt(idx)) } else { assert(row.isNullAt(idx)) } } val removeRow = row.getStruct(SingleAction.REMOVE_FILE_ORDINAL) assert(removeRow.getSchema == RemoveFile.FULL_SCHEMA) val removeFile = new RemoveFile(removeRow) assert(removeFile.getPath == expectedPath) assert(removeFile.getDeletionTimestamp.isPresent && removeFile.getDeletionTimestamp.get == expectedDeletionTimestamp) assert(removeFile.getDataChange == expectedDataChange) assert(removeFile.getExtendedFileMetadata.isPresent && removeFile.getExtendedFileMetadata.get) assert( removeFile.getPartitionValues.isPresent && removeFile.getPartitionValues.get.getSize == 0) assert(removeFile.getSize.isPresent && removeFile.getSize.get == expectedSize) if (expectedStatsString.nonEmpty) { // We have to do our stats check differently since the RemoveFile::getStats API does not fully // deserialize the statistics (only grabs the numRecords field) assert( removeRow.getString(RemoveFile.FULL_SCHEMA.indexOf("stats")) == expectedStatsString.get) } else { assert(removeRow.isNullAt(RemoveFile.FULL_SCHEMA.indexOf("stats"))) } assert(!removeFile.getTags.isPresent) assert(!removeFile.getDeletionVector.isPresent) assert(!removeFile.getBaseRowId.isPresent) assert(!removeFile.getDefaultRowCommitVersion.isPresent) } test("generateIcebergCompatWriterV1AddAction creates correct add row") { Seq(true, false).foreach { dataChange => val txnRow = getTestTransactionStateRow(compatibleTableProperties) val statsString = testDataFileStatusWithStatistics.getStatistics.get .serializeAsJson(TransactionStateRow.getPhysicalSchema(txnRow)) validateAddAction( GenerateIcebergCompatActionUtils.generateIcebergCompatWriterV1AddAction( txnRow, testDataFileStatusWithStatistics, Collections.emptyMap(), // partitionValues dataChange, Optional.empty() /* Pre-parsed physicalSchema if present */ ), expectedPath = "file1.parquet", expectedSize = 1000, expectedModificationTime = 10, expectedDataChange = dataChange, expectedStatsString = statsString) } } test("generateIcebergCompatWriterV1AddAction creates correct remove row") { Seq(true, false).foreach { dataChange => // RemoveFiles are allowed to be missing statistics (where as AddFiles are not) Seq(testDataFileStatusWithStatistics, testDataFileStatusWithoutStatistics).foreach { fileStatus => val txnRow = getTestTransactionStateRow(compatibleTableProperties) val statsString = if (fileStatus.getStatistics.isPresent) { Some(fileStatus.getStatistics.get .serializeAsJson(TransactionStateRow.getPhysicalSchema(txnRow))) } else { None } validateRemoveAction( GenerateIcebergCompatActionUtils.generateIcebergCompatWriterV1RemoveAction( txnRow, fileStatus, Collections.emptyMap(), // partitionValues dataChange, Optional.empty() /* Pre-parsed physicalSchema if present */ ), expectedPath = "file1.parquet", expectedSize = 1000, expectedDeletionTimestamp = 10, expectedDataChange = dataChange, expectedStatsString = statsString) } } } } object GenerateIcebergCompatActionUtilsSuite { private val testDataFileStatusWithStatistics = new DataFileStatus( "/test/table/path/file1.parquet", 1000, 10, Optional.of( new DataFileStatistics( 100, Map(new Column("id") -> Literal.ofInt(0)).asJava, Map(new Column("id") -> Literal.ofInt(10)).asJava, Map(new Column("id") -> java.lang.Long.valueOf(0)).asJava, Optional.empty()))) private val testDataFileStatusWithoutStatistics = new DataFileStatus( "/test/table/path/file1.parquet", 1000, 10, Optional.empty()) private val testSchema = new StructType() .add("id", IntegerType.INTEGER) .add("comment", StringType.STRING) private val testTablePath = "/test/table/path" private val compatibleTableProperties = Map( TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.getKey -> "true", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true", TableConfig.COLUMN_MAPPING_MODE.getKey -> "id") } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/MetadataSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.actions import java.util.{Collections, Optional} import scala.collection.JavaConverters._ import io.delta.kernel.data.{ArrayValue, ColumnVector, MapValue} import io.delta.kernel.exceptions.KernelException import io.delta.kernel.internal.data.GenericRow import io.delta.kernel.internal.util.InternalUtils.singletonStringColumnVector import io.delta.kernel.internal.util.VectorUtils.buildColumnVector import io.delta.kernel.test.TestUtils import io.delta.kernel.types.{IntegerType, StringType, StructType} import org.scalatest.funsuite.AnyFunSuite class MetadataSuite extends AnyFunSuite with TestUtils { test("withMergedConfig upserts values") { val metadata = testMetadata(Map("a" -> "b", "f" -> "g")) val newMetadata = metadata.withMergedConfiguration(Map("a" -> "c", "d" -> "f").asJava) assert(newMetadata.getConfiguration.equals(Map("a" -> "c", "d" -> "f", "f" -> "g").asJava)) } test("withReplacedConfiguration replaces values") { val metadata = testMetadata(Map("a" -> "b", "f" -> "g")) val newMetadata = metadata.withReplacedConfiguration(Map("a" -> "c", "d" -> "f").asJava) assert(newMetadata.getConfiguration.equals(Map("a" -> "c", "d" -> "f").asJava)) } private val defaultTestSchema = new StructType() .add("c1", IntegerType.INTEGER) .add("c2", StringType.STRING) def testMetadata( tblProps: Map[String, String] = Map.empty, schemaString: String = defaultTestSchema.toJson): Metadata = { val partitionCols = new ArrayValue() { override def getSize = 1 override def getElements: ColumnVector = singletonStringColumnVector("c3") } val conf = new MapValue() { override def getSize = tblProps.size override def getKeys: ColumnVector = buildColumnVector(tblProps.toSeq.map(_._1).asJava, StringType.STRING) override def getValues: ColumnVector = buildColumnVector(tblProps.toSeq.map(_._2).asJava, StringType.STRING) } val values = new java.util.HashMap[Integer, Object]() values.put(0, "id") values.put(1, "name") values.put(2, "description") values.put(3, new Format("parquet", Collections.emptyMap()).toRow) values.put(4, schemaString) values.put(5, partitionCols) values.put(6, null) // createdTime values.put(7, conf) Metadata.fromRow(new GenericRow(Metadata.FULL_SCHEMA, values)) } test("schema parsing is lazy - void type does not block non-schema access") { val voidSchemaJson = """{"type":"struct","fields":[""" + """{"name":"x","type":"integer","nullable":true,"metadata":{}},""" + """{"name":"y","type":"void","nullable":true,"metadata":{}}]}""" val metadata = testMetadata(schemaString = voidSchemaJson) // Non-schema methods should work without triggering schema parsing assert(metadata.getId === "id") assert(metadata.getConfiguration.isEmpty) assert(metadata.getSchemaString === voidSchemaJson) // Accessing the schema should throw KernelException due to VOID type val e = intercept[KernelException] { metadata.getSchema } assert(e.getMessage.contains("VOID")) } test("Metadata serialization round trip") { val source = testMetadata(Map("key1" -> "value1", "key2" -> "value2")) val deserialized = roundTripSerialize(source) // Verify all public methods return the same values assert(deserialized.getId === source.getId) assert(deserialized.getName === source.getName) assert(deserialized.getDescription === source.getDescription) assert(deserialized.getFormat === source.getFormat) assert(deserialized.getSchemaString === source.getSchemaString) assert(deserialized.getSchema === source.getSchema) assert(deserialized.getCreatedTime === source.getCreatedTime) assert(deserialized.getConfiguration === source.getConfiguration) assert(deserialized.getPartitionColNames === source.getPartitionColNames) assert(deserialized.getDataSchema === source.getDataSchema) assert(deserialized.getPhysicalSchema === source.getPhysicalSchema) // Verify equals and hashCode assert(deserialized === source) assert(deserialized.hashCode() === source.hashCode()) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/ProtocolSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.actions import scala.collection.JavaConverters._ import io.delta.kernel.internal.data.GenericRow import io.delta.kernel.internal.tablefeatures.TableFeatures import io.delta.kernel.internal.util.VectorUtils import io.delta.kernel.test.TestUtils import io.delta.kernel.types.{ArrayType, IntegerType, StringType, StructType} import org.scalatest.funsuite.AnyFunSuite class ProtocolSuite extends AnyFunSuite with TestUtils { ///////////////////////////////////////////////////////////////////////////////////////////////// // Tests for TableFeature related methods on Protocol // ///////////////////////////////////////////////////////////////////////////////////////////////// // Invalid protocol versions/features throw validation errors Seq( // Test format: // minReaderVersion, minWriterVersion, readerFeatures, writerFeatures, expectedErrorMsg (0, 1, Set(), Set(), "minReaderVersion should be at least 1"), (1, 0, Set(), Set(), "minWriterVersion should be at least 1"), ( // writer version doesn't support writer features 1, 2, Set("columnMapping"), Set(), "Reader features are not supported for the reader version: 1"), ( // writer version doesn't support writer features 1, 2, Set(), Set("columnMapping"), "Writer features are not supported for the writer version: 2"), // you can't have reader version with feature support, but not the writer version (3, 5, Set(), Set(), "writer version doesn't support writer features: 5"), // columnMapping feature is not supported for reader version 1 (1, 5, Set(), Set(), "Reader version 1 does not support readerWriter feature columnMapping"), ( // readerWriter feature columnMapping is missing from the readerFeatures set 3, 7, Set(), Set("columnMapping"), "ReaderWriter feature columnMapping is not present in readerFeatures"), // minReaderVersion doesn't support readerWriter feature columnMapping requirement ( 1, 7, Set(), Set("columnMapping"), "Reader version 1 does not support readerWriter feature columnMapping")).foreach { case ( readerVersion, writerVersion, readerFeatures: Set[String], writerFeatures: Set[String], expectedError) => test(s"Invalid protocol versions " + s"($readerVersion, $writerVersion, $readerFeatures, $writerFeatures)") { val protocol = new Protocol(readerVersion, writerVersion, readerFeatures.asJava, writerFeatures.asJava) val e = intercept[IllegalArgumentException] { protocol.validate() } assert(e.getMessage === expectedError) } } // Tests for getImplicitlySupportedFeatures, getExplicitlySupportedFeatures and // getImplicitlyAndExplicitlySupportedFeatures Seq( // Test format: // (minReaderVersion, minWriterVersion, expected features) (1, 1, Set()), (1, 2, Set("appendOnly", "invariants")), (1, 3, Set("appendOnly", "invariants", "checkConstraints")), ( 1, 4, Set("appendOnly", "invariants", "checkConstraints", "changeDataFeed", "generatedColumns")), ( 2, 5, Set( "appendOnly", "invariants", "checkConstraints", "changeDataFeed", "generatedColumns", "columnMapping")), ( 2, 6, Set( "appendOnly", "invariants", "checkConstraints", "changeDataFeed", "generatedColumns", "columnMapping", "identityColumns"))).foreach { case (minReaderVersion, minWriterVersion, expectedFeatures) => test(s"getImplicitlySupportedFeatures with minReaderVersion $minReaderVersion and " + s"minWriterVersion $minWriterVersion") { val protocol = new Protocol(minReaderVersion, minWriterVersion) assert( protocol.getImplicitlySupportedFeatures.asScala.map(_.featureName()) === expectedFeatures) assert( protocol.getImplicitlyAndExplicitlySupportedFeatures.asScala.map(_.featureName()) === expectedFeatures) assert( protocol.getExplicitlySupportedFeatures.asScala.map(_.featureName()) === Set()) } } Seq( // Test format: readerFeatures, writerFeatures, expected set (Set(), Set(), Set()), (Set(), Set("rowTracking"), Set("rowTracking")), (Set(), Set("checkConstraints", "rowTracking"), Set("checkConstraints", "rowTracking")), ( Set("columnMapping"), Set("columnMapping", "domainMetadata"), Set("columnMapping", "domainMetadata"))).foreach { case ( readerFeatures: Set[String], writerFeatures: Set[String], expectedFeatureSet: Set[String]) => test(s"getExplicitlySupportedFeatures $readerFeatures $writerFeatures") { val protocol = new Protocol(3, 7, readerFeatures.asJava, writerFeatures.asJava) assert( protocol.getExplicitlySupportedFeatures.asScala.map(_.featureName()) === expectedFeatureSet) assert( protocol.getImplicitlyAndExplicitlySupportedFeatures.asScala.map(_.featureName()) === expectedFeatureSet) assert(protocol.getImplicitlySupportedFeatures.asScala.map(_.featureName()) === Set()) } } // Tests for `normalized Seq( // Test format: input, expected output out of the `normalized` // If the protocol has no table features, then the normalized shouldn't change (1, 1, Set[String](), Set[String]()) -> (1, 1, Set[String](), Set[String]()), (1, 2, Set[String](), Set[String]()) -> (1, 2, Set[String](), Set[String]()), (2, 5, Set[String](), Set[String]()) -> (2, 5, Set[String](), Set[String]()), // If the protocol has table features, then the normalized may or // may not have the table features (3, 7, Set[String](), Set("appendOnly", "invariants")) -> (1, 2, Set[String](), Set[String]()), (3, 7, Set[String](), Set("appendOnly", "invariants", "checkConstraints")) -> (1, 3, Set[String](), Set[String]()), ( 3, 7, Set[String](), Set("appendOnly", "invariants", "checkConstraints", "changeDataFeed", "generatedColumns")) -> (1, 4, Set[String](), Set[String]()), ( 3, 7, Set("columnMapping"), Set( "appendOnly", "invariants", "checkConstraints", "changeDataFeed", "generatedColumns", "columnMapping")) -> (2, 5, Set[String](), Set[String]()), // reader version is downgraded // can't downgrade the writer version, because version 2 (appendOnly) also has support for // invariants which is not supported in the writer features in the input (1, 7, Set[String](), Set("appendOnly")) -> (1, 7, Set[String](), Set[String]("appendOnly")), (3, 7, Set("columnMapping"), Set("columnMapping")) -> (2, 7, Set[String](), Set("columnMapping")), (3, 7, Set("columnMapping"), Set("columnMapping", "domainMetadata")) -> (2, 7, Set[String](), Set("columnMapping", "domainMetadata"))).foreach { case ( (readerVersion, writerVersion, readerFeatures, writerFeatures), ( expReaderVersion, expWriterVersion, expReaderFeatures, expWriterFeatures)) => test(s"normalized $readerVersion $writerVersion $readerFeatures $writerFeatures") { val protocol = new Protocol(readerVersion, writerVersion, readerFeatures.asJava, writerFeatures.asJava) val normalized = protocol.normalized() assert(normalized.getMinReaderVersion === expReaderVersion) assert(normalized.getMinWriterVersion === expWriterVersion) assert(normalized.getReaderFeatures.asScala === expReaderFeatures) assert(normalized.getWriterFeatures.asScala === expWriterFeatures) } } // Tests for `denormalized` Seq( // Test format: input, expected output out of the `denormalized` (1, 1, Set[String](), Set[String]()) -> (1, 7, Set[String](), Set[String]()), (1, 2, Set[String](), Set[String]()) -> (1, 7, Set[String](), Set("appendOnly", "invariants")), (2, 5, Set[String](), Set[String]()) -> ( 2, 7, Set[String](), Set( "appendOnly", "invariants", "checkConstraints", "changeDataFeed", "generatedColumns", "columnMapping")), // invalid protocol versions (2, 3) (2, 3, Set[String](), Set[String]()) -> ( 1, 7, Set[String](), Set("appendOnly", "invariants", "checkConstraints")), // shouldn't change the protocol already has the table feature set support (3, 7, Set[String](), Set("appendOnly", "invariants")) -> (3, 7, Set[String](), Set("appendOnly", "invariants")), (3, 7, Set[String](), Set("appendOnly", "invariants", "checkConstraints")) -> (3, 7, Set[String](), Set("appendOnly", "invariants", "checkConstraints"))).foreach { case ( (readerVersion, writerVersion, readerFeatures, writerFeatures), ( expReaderVersion, expWriterVersion, expReaderFeatures, expWriterFeatures)) => test(s"denormalized $readerVersion $writerVersion $readerFeatures $writerFeatures") { val protocol = new Protocol(readerVersion, writerVersion, readerFeatures.asJava, writerFeatures.asJava) val denormalized = protocol.denormalized() assert(denormalized.getMinReaderVersion === expReaderVersion) assert(denormalized.getMinWriterVersion === expWriterVersion) assert(denormalized.getReaderFeatures.asScala === expReaderFeatures) assert(denormalized.getWriterFeatures.asScala === expWriterFeatures) } } // Tests for `withFeature` and `normalized` Seq( // can't downgrade the writer version, because version 2 (appendOnly) also has support for // invariants which is not supported in the writer features in the input Set("appendOnly") -> (1, 7, Set[String](), Set("appendOnly")), Set("invariants") -> (1, 7, Set[String](), Set[String]("invariants")), Set("appendOnly", "invariants") -> (1, 2, Set[String](), Set[String]()), Set("checkConstraints") -> (1, 7, Set[String](), Set("checkConstraints")), Set("changeDataFeed") -> (1, 7, Set[String](), Set("changeDataFeed")), Set("appendOnly", "invariants", "checkConstraints") -> (1, 3, Set[String](), Set[String]()), Set("generatedColumns") -> (1, 7, Set[String](), Set("generatedColumns")), Set("columnMapping") -> (2, 7, Set(), Set("columnMapping")), Set("identityColumns") -> (1, 7, Set[String](), Set[String]("identityColumns")), // expect the dependency features also to be supported Set("icebergCompatV2") -> (2, 7, Set[String](), Set[String]("icebergCompatV2", "columnMapping")), Set("variantShredding-preview") -> ( 3, 7, Set[String]("variantType", "variantShredding-preview"), Set[String]("variantType", "variantShredding-preview")), Set("variantShredding") -> ( 3, 7, Set[String]("variantType", "variantShredding"), Set[String]("variantType", "variantShredding")), Set("rowTracking") -> ( 1, 7, Set[String](), Set[String]("rowTracking", "domainMetadata"))).foreach { case (features, (expReaderVersion, expWriterVersion, expReaderFeatures, expWriterFeatures)) => test(s"withFeature $features") { val protocol = new Protocol(3, 7) val updated = protocol .withFeatures(features.map(TableFeatures.getTableFeature).asJava) .normalized() assert(updated.getMinReaderVersion === expReaderVersion) assert(updated.getMinWriterVersion === expWriterVersion) assert(updated.getReaderFeatures.asScala === expReaderFeatures) assert(updated.getWriterFeatures.asScala === expWriterFeatures) } } test("withFeature - can't add a feature at the current version") { val protocol = new Protocol(1, 2) val e = intercept[UnsupportedOperationException] { protocol.withFeatures(Set(TableFeatures.getTableFeature("columnMapping")).asJava) } assert(e.getMessage === "TableFeature requires higher reader protocol version") } // Tests for `merge` (also tests denormalized and normalized) Seq( // Test format: (protocol1, protocol2) -> expected merged protocol ( (1, 1, Set[String](), Set[String]()), (1, 2, Set[String](), Set[String]())) -> (1, 2, Set[String](), Set[String]()), ((1, 2, Set[String](), Set[String]()), (1, 3, Set[String](), Set[String]())) -> (1, 3, Set[String](), Set[String]()), ((1, 4, Set[String](), Set[String]()), (2, 5, Set[String](), Set[String]())) -> (2, 5, Set[String](), Set[String]()), ((1, 4, Set[String](), Set[String]()), (2, 6, Set[String](), Set[String]())) -> (2, 6, Set[String](), Set[String]()), ((1, 2, Set[String](), Set[String]()), (1, 7, Set[String](), Set("invariants"))) -> (1, 2, Set[String](), Set[String]()), ((1, 2, Set[String](), Set[String]()), (3, 7, Set("columnMapping"), Set("columnMapping"))) -> (2, 7, Set[String](), Set("columnMapping", "invariants", "appendOnly")), ( (1, 2, Set[String](), Set[String]()), (3, 7, Set("columnMapping"), Set("columnMapping", "domainMetadata"))) -> (2, 7, Set[String](), Set("domainMetadata", "columnMapping", "invariants", "appendOnly")), ( (2, 5, Set[String](), Set[String]()), (3, 7, Set("v2Checkpoint"), Set("v2Checkpoint", "domainMetadata"))) -> ( 3, 7, Set("columnMapping", "v2Checkpoint"), Set( "domainMetadata", "columnMapping", "v2Checkpoint", "invariants", "appendOnly", "checkConstraints", "changeDataFeed", "generatedColumns"))).foreach({ case ( ( (readerVersion1, writerVersion1, readerFeatures1, writerFeatures1), (readerVersion2, writerVersion2, readerFeatures2, writerFeatures2)), (expReaderVersion, expWriterVersion, expReaderFeatures, expWriterFeatures)) => test(s"merge $readerVersion1 $writerVersion1 $readerFeatures1 $writerFeatures1 " + s"$readerVersion2 $writerVersion2 $readerFeatures2 $writerFeatures2") { val protocol1 = new Protocol( readerVersion1, writerVersion1, readerFeatures1.asJava, writerFeatures1.asJava) val protocol2 = new Protocol( readerVersion2, writerVersion2, readerFeatures2.asJava, writerFeatures2.asJava) val merged = protocol1.merge(protocol2) assert(merged.getMinReaderVersion === expReaderVersion) assert(merged.getMinWriterVersion === expWriterVersion) assert(merged.getReaderFeatures.asScala === expReaderFeatures) assert(merged.getWriterFeatures.asScala === expWriterFeatures) } }) test("extract protocol from the row representation") { val ordinalToValue: Map[Integer, Object] = Map( Integer.valueOf(0) -> Integer.valueOf(42), Integer.valueOf(1) -> Integer.valueOf(43), Integer.valueOf(2) -> VectorUtils.buildArrayValue( List("foo").asJava, StringType.STRING).asInstanceOf[Object], Integer.valueOf(3) -> VectorUtils.buildArrayValue( List("bar").asJava, StringType.STRING).asInstanceOf[Object]) val row = new GenericRow( new StructType().add("minReaderVersion", IntegerType.INTEGER) .add("minWriterVersion", IntegerType.INTEGER) .add("readerFeatures", new ArrayType(StringType.STRING, true)) .add("writerFeatures", new ArrayType(StringType.STRING, true)), ordinalToValue.asJava) val expected = new Protocol(42, 43, Set("foo").asJava, Set("bar").asJava) assert(Protocol.fromRow(row) === expected) } test("Protocol serialization round trip") { val source = new Protocol( 3, 7, Set("columnMapping", "v2Checkpoint").asJava, Set("columnMapping", "domainMetadata").asJava) val deserialized = roundTripSerialize(source) // Verify all public methods return the same values assert(deserialized.getMinReaderVersion === source.getMinReaderVersion) assert(deserialized.getMinWriterVersion === source.getMinWriterVersion) assert(deserialized.getReaderFeatures === source.getReaderFeatures) assert(deserialized.getWriterFeatures === source.getWriterFeatures) assert(deserialized.getReaderAndWriterFeatures === source.getReaderAndWriterFeatures) assert(deserialized.supportsReaderFeatures() === source.supportsReaderFeatures()) assert(deserialized.supportsWriterFeatures() === source.supportsWriterFeatures()) assert(deserialized.getImplicitlySupportedFeatures === source.getImplicitlySupportedFeatures) assert(deserialized.getExplicitlySupportedFeatures === source.getExplicitlySupportedFeatures) assert(deserialized.getImplicitlyAndExplicitlySupportedFeatures === source.getImplicitlyAndExplicitlySupportedFeatures) // Verify equals and hashCode assert(deserialized === source) assert(deserialized.hashCode() === source.hashCode()) } ///////////////////////////////////////////////////////////////////////////////////////////////// // Tests for supportsFeature // ///////////////////////////////////////////////////////////////////////////////////////////////// test("supportsFeature - legacy protocol with readerVersion=1, writerVersion=2") { // Protocol (1, 2) implicitly supports appendOnly and invariants val protocol = new Protocol(1, 2) // appendOnly is a writer-only feature with minWriterVersion = 2 assert(protocol.supportsFeature(TableFeatures.APPEND_ONLY_W_FEATURE)) // invariants is a writer-only feature with minWriterVersion = 2 assert(protocol.supportsFeature(TableFeatures.INVARIANTS_W_FEATURE)) // checkConstraints is a writer-only feature with minWriterVersion = 3 assert(!protocol.supportsFeature(TableFeatures.CONSTRAINTS_W_FEATURE)) // columnMapping is a reader-writer feature with minReaderVersion = 2, minWriterVersion = 5 assert(!protocol.supportsFeature(TableFeatures.COLUMN_MAPPING_RW_FEATURE)) } test("supportsFeature - legacy protocol with readerVersion=2, writerVersion=5") { // Protocol (2, 5) implicitly supports columnMapping (and other legacy writer features) val protocol = new Protocol(2, 5) // columnMapping is a reader-writer feature with minReaderVersion = 2, minWriterVersion = 5 assert(protocol.supportsFeature(TableFeatures.COLUMN_MAPPING_RW_FEATURE)) // changeDataFeed is a writer feature with minWriterVersion = 4 assert(protocol.supportsFeature(TableFeatures.CHANGE_DATA_FEED_W_FEATURE)) // identityColumns is a writer-only feature with minWriterVersion = 6 assert(!protocol.supportsFeature(TableFeatures.IDENTITY_COLUMNS_W_FEATURE)) } test("supportsFeature - protocol with table features support") { // Protocol (3, 7) with explicit features val protocol = new Protocol( 3, 7, Set("columnMapping", "v2Checkpoint").asJava, Set("columnMapping", "domainMetadata", "rowTracking").asJava) // Features explicitly listed assert(protocol.supportsFeature(TableFeatures.COLUMN_MAPPING_RW_FEATURE)) assert(protocol.supportsFeature(TableFeatures.CHECKPOINT_V2_RW_FEATURE)) assert(protocol.supportsFeature(TableFeatures.DOMAIN_METADATA_W_FEATURE)) assert(protocol.supportsFeature(TableFeatures.ROW_TRACKING_W_FEATURE)) // Features not listed assert(!protocol.supportsFeature(TableFeatures.APPEND_ONLY_W_FEATURE)) assert(!protocol.supportsFeature(TableFeatures.DELETION_VECTORS_RW_FEATURE)) } test("supportsFeature - protocol with only writer features (and legacy reader version)") { // Protocol (1, 7) with only writer features val protocol = new Protocol( 1, 7, Set().asJava, Set("appendOnly", "invariants", "domainMetadata").asJava) // Writer features listed assert(protocol.supportsFeature(TableFeatures.APPEND_ONLY_W_FEATURE)) assert(protocol.supportsFeature(TableFeatures.INVARIANTS_W_FEATURE)) assert(protocol.supportsFeature(TableFeatures.DOMAIN_METADATA_W_FEATURE)) // Writer features not listed assert(!protocol.supportsFeature(TableFeatures.ROW_TRACKING_W_FEATURE)) // Reader-writer features not listed (reader version too low) assert(!protocol.supportsFeature(TableFeatures.COLUMN_MAPPING_RW_FEATURE)) assert(!protocol.supportsFeature(TableFeatures.DELETION_VECTORS_RW_FEATURE)) } test("supportsFeature - doesn't throw on unknown writer feature when checking reader feature") { // Protocol with unknown writer features in the set val protocol = new Protocol( 3, 7, Set("columnMapping").asJava, Set("columnMapping", "unknownWriterFeature").asJava) // Check a reader-writer feature that is present - should not throw assert(protocol.supportsFeature(TableFeatures.COLUMN_MAPPING_RW_FEATURE)) // Check a reader-writer feature that is not present - should not throw, just return false assert(!protocol.supportsFeature(TableFeatures.CHECKPOINT_V2_RW_FEATURE)) } test("supportsFeature - doesn't throw on unknown features in reader or writer list") { // Protocol with unknown features in both reader and writer feature sets val protocol = new Protocol( 3, 7, Set("columnMapping", "unknownReaderWriterFeature").asJava, Set( "columnMapping", "domainMetadata", "unknownReaderWriterFeature", "unknownWriterFeature").asJava) // Check reader-writer features - should not throw assert(protocol.supportsFeature(TableFeatures.COLUMN_MAPPING_RW_FEATURE)) assert(!protocol.supportsFeature(TableFeatures.CHECKPOINT_V2_RW_FEATURE)) // Check writer features - should not throw assert(protocol.supportsFeature(TableFeatures.DOMAIN_METADATA_W_FEATURE)) assert(!protocol.supportsFeature(TableFeatures.ROW_TRACKING_W_FEATURE)) assert(!protocol.supportsFeature(TableFeatures.APPEND_ONLY_W_FEATURE)) } test("supportsFeature - empty feature sets") { // Protocol (3, 7) with empty feature sets val protocol = new Protocol(3, 7, Set().asJava, Set().asJava) // No features should be supported assert(!protocol.supportsFeature(TableFeatures.APPEND_ONLY_W_FEATURE)) assert(!protocol.supportsFeature(TableFeatures.COLUMN_MAPPING_RW_FEATURE)) assert(!protocol.supportsFeature(TableFeatures.DOMAIN_METADATA_W_FEATURE)) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/RemoveFileSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.actions import java.lang.{Long => JLong} import java.util.Optional import scala.collection.JavaConverters._ import io.delta.kernel.data.Row import io.delta.kernel.internal.util.VectorUtils import io.delta.kernel.internal.util.VectorUtils.stringStringMapValue import io.delta.kernel.statistics.DataFileStatistics import org.scalatest.funsuite.AnyFunSuite class RemoveFileSuite extends AnyFunSuite { // For now we use GenerateIcebergCompatActionUtils::createRemoveFileRowWithExtendedFileMetadata // because this is the only path we support creating RemoveFile rows currently. In the future when // we implement broader support for RemoveFiles we should use the more generic methods to create // the test rows private def createTestRemoveFileRow( path: String, deletionTimestamp: Long, dataChange: Boolean, partitionValues: Map[String, String], size: Long, stats: Option[String], baseRowId: Option[Long] = Option.empty, defaultRowCommitVersion: Option[Long] = Option.empty, deletionVectorDescriptor: Option[DeletionVectorDescriptor]): Row = { def toJavaOptional[T](option: Option[T]): Optional[T] = option match { case Some(value) => Optional.of(value) case None => Optional.empty() } GenerateIcebergCompatActionUtils.createRemoveFileRowWithExtendedFileMetadata( path, deletionTimestamp, dataChange, stringStringMapValue(partitionValues.asJava), size, DataFileStatistics.deserializeFromJson(stats.getOrElse(""), null), null, // physicalSchema toJavaOptional(baseRowId.asInstanceOf[Option[JLong]]), toJavaOptional(defaultRowCommitVersion.asInstanceOf[Option[JLong]]), deletionVectorDescriptor match { case Some(dvd) => Optional.of(dvd) case None => Optional.empty[DeletionVectorDescriptor]() }) } test("getters can read RemoveFile's fields from the backing row") { val deletionVectorDescriptor = new DeletionVectorDescriptor( "storage", "s", Optional.of(1), 25, 35) val removeFileRow = createTestRemoveFileRow( path = "test/path", deletionTimestamp = 1000L, dataChange = true, partitionValues = Map("a" -> "1"), size = 55555L, stats = Option("{\"numRecords\":100}"), deletionVectorDescriptor = Some(deletionVectorDescriptor)) val removeFile = new RemoveFile(removeFileRow) assert(removeFile.getPath === "test/path") assert(removeFile.getDeletionTimestamp == Optional.of(1000L)) assert(removeFile.getDataChange) assert(removeFile.getExtendedFileMetadata == Optional.of(true)) assert(removeFile.getPartitionValues.isPresent && VectorUtils.toJavaMap(removeFile.getPartitionValues.get).asScala.equals(Map("a" -> "1"))) assert(removeFile.getSize == Optional.of(55555L)) assert(removeFile.getStats(null).isPresent && removeFile.getStats(null).get.serializeAsJson(null) == "{\"numRecords\":100}") assert(!removeFile.getTags.isPresent) assert(removeFile.getDeletionVector.isPresent) assert(removeFile.getDeletionVector.get == deletionVectorDescriptor) assert(!removeFile.getBaseRowId.isPresent) assert(!removeFile.getDefaultRowCommitVersion.isPresent) } test("getters can read RemoveFile's fields from the backing row with row tracking") { val deletionVectorDescriptor = new DeletionVectorDescriptor( "storage", "s", Optional.of(1), 25, 35) val removeFileRow = createTestRemoveFileRow( path = "test/path", deletionTimestamp = 1000L, dataChange = true, partitionValues = Map("a" -> "1"), size = 55555L, stats = Option("{\"numRecords\":100}"), baseRowId = Option(30L), defaultRowCommitVersion = Option(40L), deletionVectorDescriptor = Some(deletionVectorDescriptor)) val removeFile = new RemoveFile(removeFileRow) assert(removeFile.getPath === "test/path") assert(removeFile.getDeletionTimestamp == Optional.of(1000L)) assert(removeFile.getDataChange) assert(removeFile.getExtendedFileMetadata == Optional.of(true)) assert(removeFile.getPartitionValues.isPresent && VectorUtils.toJavaMap(removeFile.getPartitionValues.get).asScala.equals(Map("a" -> "1"))) assert(removeFile.getSize == Optional.of(55555L)) assert(removeFile.getStats(null).isPresent && removeFile.getStats( null).get.serializeAsJson(null) == "{\"numRecords\":100}") assert(removeFile.getBaseRowId === Optional.of(30L)) assert(removeFile.getDefaultRowCommitVersion === Optional.of(40L)) assert(removeFile.getDeletionVector.get == deletionVectorDescriptor) assert(!removeFile.getTags.isPresent) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/catalogManaged/CatalogManagedLogSegmentSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.catalogManaged import java.util.Optional import scala.collection.JavaConverters._ import io.delta.kernel.TableManager import io.delta.kernel.internal.actions.Protocol import io.delta.kernel.internal.table.SnapshotBuilderImpl import io.delta.kernel.internal.util.FileNames import io.delta.kernel.test.{ActionUtils, MockFileSystemClientUtils} import io.delta.kernel.types.{IntegerType, StructType} import org.scalatest.funsuite.AnyFunSuite class CatalogManagedLogSegmentSuite extends AnyFunSuite with MockFileSystemClientUtils with ActionUtils { implicit class OptionOps[T](option: Option[T]) { def asJava: Optional[T] = option match { case Some(value) => Optional.of(value) case None => Optional.empty() } } private def testLogSegment( testName: String, versionToLoad: Long, checkpointVersionOpt: Option[Long], deltaVersions: Seq[Long], ratifiedCommitVersions: Seq[Long], crcVersions: Seq[Long] = Seq.empty, expectedDeltaAndCommitVersionsOpt: Option[Seq[Long]] = None, expectedExceptionClassOpt: Option[Class[_ <: Exception]] = None): Unit = { // TODO: test with ratified=inline test(testName + " - ratified=materialized") { val checkpointFile = checkpointVersionOpt.map(v => classicCheckpointFileStatus(v)).toSeq val deltaFiles = deltaFileStatuses(deltaVersions) val crcFiles = crcVersions.map(checksumFileStatus) val ratifiedCommitParsedLogDatas = parsedRatifiedStagedCommits(ratifiedCommitVersions) val engine = createMockFSListFromEngine(checkpointFile ++ deltaFiles ++ crcFiles) val testSchema = new StructType().add("c1", IntegerType.INTEGER); val builder = TableManager .loadSnapshot(dataPath.toString) .asInstanceOf[SnapshotBuilderImpl] .atVersion(versionToLoad) .withProtocolAndMetadata(new Protocol(1, 2), testMetadata(testSchema)) .withLogData(ratifiedCommitParsedLogDatas.toList.asJava) if (expectedExceptionClassOpt.isDefined) { val exception = intercept[Throwable] { // Ensure we load the LogSegment to identify any gaps/issues builder.build(engine).getLogSegment } assert(expectedExceptionClassOpt.get.isInstance(exception)) } else { val snapshot = builder.build(engine) val logSegment = snapshot.getLogSegment val actualDeltaAndCommitFileStatuses = logSegment.getDeltas.asScala // Check: we got the expected versions val actualDeltaAndCommitVersions = actualDeltaAndCommitFileStatuses.map(x => FileNames.deltaVersion(x.getPath)) assert(actualDeltaAndCommitVersions sameElements expectedDeltaAndCommitVersionsOpt.get) // Check: ratified commits take priority over published deltas when versions overlap val expectedRatifiedVersions = ratifiedCommitVersions.toSet.intersect(actualDeltaAndCommitVersions.toSet) actualDeltaAndCommitFileStatuses.map(_.getPath).foreach { path => val version = FileNames.deltaVersion(path) if (expectedRatifiedVersions.contains(version)) { assert(FileNames.isStagedDeltaFile(path)) } else { assert(FileNames.isPublishedDeltaFile(path)) } } // Check: maxPublishedDeltaVersion val expectedMaxPublishedDeltaVersion = deltaVersions .filter(_ <= versionToLoad).reduceOption(_ max _).asJava assert(logSegment.getMaxPublishedDeltaVersion === expectedMaxPublishedDeltaVersion) // Check: lastSeenChecksum val expectedLastSeenChecksumVersion = crcVersions .filter(v => v <= versionToLoad && checkpointVersionOpt.forall(v >= _)) .lastOption expectedLastSeenChecksumVersion match { case Some(expectedVersion) => val checksumPath = logSegment.getLastSeenChecksum.get.getPath val actualVersion = FileNames.checksumVersion(checksumPath) assert(actualVersion === expectedVersion) case None => assert(!logSegment.getLastSeenChecksum.isPresent) } } } } ///////////////////////////////////////////////////// // LogSegment construction tests -- positive cases // ///////////////////////////////////////////////////// // _delta_log: [ 10.checkpoint+json, 11.json, 12.json] // catalog: [8.uuid.json, 9.uuid.json ] testLogSegment( testName = "Build RT with ratified commits that are before first checkpoint", versionToLoad = 12L, checkpointVersionOpt = Some(10L), deltaVersions = 10L to 12L, ratifiedCommitVersions = 8L to 9L, expectedDeltaAndCommitVersionsOpt = Some(11L to 12L)) // _delta_log: [ 10.checkpoint+json, 11.json, 12.json, 13.json] // catalog: [9.uuid.json, 10.uuid.json, 11.uuid.json ] testLogSegment( testName = "Build RT with ratified commits that overlap w first checkpoint + deltas", versionToLoad = 13L, checkpointVersionOpt = Some(10L), deltaVersions = 10L to 13L, ratifiedCommitVersions = 9L to 11L, expectedDeltaAndCommitVersionsOpt = Some(11L to 13L)) // _delta_log: [10.checkpoint+json, 11.json, 12.json, 13.json, 14.json, 15.json] // catalog: [ 11.uuid.json, 12.uuid.json, 13.uuid.json ] testLogSegment( testName = "Build RT with ratified commits that are contained within first checkpoint + deltas", versionToLoad = 15L, checkpointVersionOpt = Some(10L), deltaVersions = 10L to 15L, ratifiedCommitVersions = 11L to 13L, expectedDeltaAndCommitVersionsOpt = Some(11L to 15L)) // _delta_log: [ 10.checkpoint+json, 11.json, 12.json ] // catalog: [9.uuid.json, 10.uuid.json 11.uuid.json, 12.uuid.json, 13.uuid.json ] testLogSegment( testName = "Build RT with ratified commits that supersets the first checkpoint + deltas", versionToLoad = 13L, checkpointVersionOpt = Some(10L), deltaVersions = 10L to 12L, ratifiedCommitVersions = 9L to 13L, expectedDeltaAndCommitVersionsOpt = Some(11L to 13L)) // _delta_log: [10.checkpoint+json, 11.json, 12.json ] // catalog: [ 12.uuid.json, 13.uuid.json, 14.uuid.json] testLogSegment( testName = "Build RT with ratified commits that overlap with end of deltas", versionToLoad = 14L, checkpointVersionOpt = Some(10L), deltaVersions = 10L to 12L, ratifiedCommitVersions = 12L to 14L, expectedDeltaAndCommitVersionsOpt = Some(11L to 14L)) // _delta_log: [10.checkpoint+json, 11.json, 12.json ] // catalog: [ 13.uuid.json, 14.uuid.json] testLogSegment( testName = "Build RT with ratified commits that are after (no gap) the deltas", versionToLoad = 14L, checkpointVersionOpt = Some(10L), deltaVersions = 10L to 12L, ratifiedCommitVersions = 13L to 14L, expectedDeltaAndCommitVersionsOpt = Some(11L to 14L)) // versionToLoad: V // _delta_log: [10.checkpoint+json, 11.json, 12.json ] // catalog: [ 13.uuid.json, 14.uuid.json] testLogSegment( testName = "Build RT with commit versions > versionToLoad - versionToLoad = checkpoint version", versionToLoad = 10L, checkpointVersionOpt = Some(10L), deltaVersions = 10L to 12L, ratifiedCommitVersions = 13L to 14L, expectedDeltaAndCommitVersionsOpt = Some(Nil)) // versionToLoad: V // _delta_log: [10.checkpoint+json, 11.json, 12.json ] // catalog: [ 13.uuid.json, 14.uuid.json] testLogSegment( testName = "Build RT with commit versions > versionToLoad - versionToLoad = delta version", versionToLoad = 12L, checkpointVersionOpt = Some(10L), deltaVersions = 10L to 12L, ratifiedCommitVersions = 13L to 14L, expectedDeltaAndCommitVersionsOpt = Some(11L to 12L)) // _delta_log: [0.json, ] // catalog: [ 1.uuid.json, 2.uuid.json, 3.uuid.json] testLogSegment( testName = "Build RT with only deltas and ratified commits (no checkpoint)", versionToLoad = 3L, checkpointVersionOpt = None, deltaVersions = Seq(0L), ratifiedCommitVersions = 1L to 3L, expectedDeltaAndCommitVersionsOpt = Some(0L to 3L)) // _delta_log: [10.checkpoint+json, ] // catalog: [ 11.uuid.json] testLogSegment( testName = "Build RT when checkpoint version is the last version from the filesystem", versionToLoad = 11L, checkpointVersionOpt = Some(10L), deltaVersions = Seq(10L), ratifiedCommitVersions = Seq(11L), expectedDeltaAndCommitVersionsOpt = Some(Seq(11L))) // scalastyle:off line.size.limit // _delta_log: [10.checkpoint+json, 11.json+crc, 12.json, 13.crc, 15.crc] // catalog: [ 13.uuid.json, 14.uuid.json, 15.uuid.json, 16.uuid.json] // scalastyle:on line.size.limit testLogSegment( testName = "Build LogSegment with CRC files for unpublished versions", versionToLoad = 16L, checkpointVersionOpt = Some(10L), deltaVersions = Seq(10L, 11L, 12L), ratifiedCommitVersions = 13L to 16L, crcVersions = Seq(11L, 13L, 15L), expectedDeltaAndCommitVersionsOpt = Some(11L to 16L)) // TODO: Support this case in a followup PR // _delta_log: [ ] // catalog: [0.uuid.json, 1.uuid.json, 2.uuid.json, 3.uuid.json] /* testLogSegment( testName = "Build RT with only ratified commits", versionToLoad = 3L, checkpointVersionOpt = None, deltaVersions = Seq(), ratifiedCommitVersions = 0L to 3L, expectedDeltaAndCommitVersionsOpt = Some(0L to 3L)) */ ///////////////////////////////////////////////////// // LogSegment construction tests -- negative cases // ///////////////////////////////////////////////////// // _delta_log: [10.checkpoint+json, 11.json, 12.json ] // catalog: [ 14.uuid.json, 15.uuid.json] testLogSegment( testName = "Build RT with ratified commits that are after (with gap) the deltas => ERROR", versionToLoad = 15L, checkpointVersionOpt = Some(10L), deltaVersions = 10L to 12L, ratifiedCommitVersions = 14L to 15L, expectedExceptionClassOpt = Some(classOf[io.delta.kernel.exceptions.InvalidTableException])) } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/catalogManaged/SnapshotBuilderSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.catalogManaged import java.util.Collections import scala.collection.JavaConverters._ import io.delta.kernel.TableManager import io.delta.kernel.commit.{CommitMetadata, CommitResponse, Committer} import io.delta.kernel.data.Row import io.delta.kernel.engine.Engine import io.delta.kernel.exceptions.{KernelException, UnsupportedProtocolVersionException, UnsupportedTableFeatureException} import io.delta.kernel.internal.actions.Protocol import io.delta.kernel.internal.commit.DefaultFileSystemManagedTableOnlyCommitter import io.delta.kernel.internal.files.{ParsedCatalogCommitData, ParsedLogData, ParsedPublishedDeltaData} import io.delta.kernel.internal.table.SnapshotBuilderImpl import io.delta.kernel.test.{ActionUtils, MockFileSystemClientUtils, MockSnapshotUtils, VectorTestUtils} import io.delta.kernel.types.{IntegerType, StructType} import io.delta.kernel.utils.CloseableIterator import org.scalatest.funsuite.AnyFunSuite class SnapshotBuilderSuite extends AnyFunSuite with MockFileSystemClientUtils with ActionUtils with VectorTestUtils with MockSnapshotUtils { private val emptyMockEngine = createMockFSListFromEngine(Nil) private val protocol = new Protocol(1, 2) private val metadata = testMetadata(new StructType().add("c1", IntegerType.INTEGER)) private val mockSnapshotAtTimestamp0 = getMockSnapshot(dataPath, latestVersion = 0L, timestamp = 0L) /////////////////////////////////////// // Builder Validation Tests -- START // /////////////////////////////////////// test("loadTable: null path throws NullPointerException") { assertThrows[NullPointerException] { TableManager.loadSnapshot(null) } } // ===== Version Tests ===== // test("atVersion: negative version throws IllegalArgumentException") { val exMsg = intercept[IllegalArgumentException] { TableManager.loadSnapshot(dataPath.toString).atVersion(-1) }.getMessage assert(exMsg === "version must be >= 0") } // ===== Timestamp Tests ===== // test("atTimestamp: null latestSnapshot throws NullPointerException") { assertThrows[NullPointerException] { TableManager.loadSnapshot(dataPath.toString).atTimestamp(1000L, null) } } test("atTimestamp: timestamp greater than latest snapshot throws IllegalArgumentException") { val builder = TableManager.loadSnapshot(dataPath.toString).atTimestamp(99, mockSnapshotAtTimestamp0) val exMsg = intercept[KernelException] { builder.build(emptyMockEngine) }.getMessage assert(exMsg.contains("The provided timestamp 99 ms")) assert(exMsg.contains("is after the latest available version")) } test("atTimestamp: timestamp and version both provided throws IllegalArgumentException") { val builder = TableManager.loadSnapshot(dataPath.toString) .atVersion(1) .atTimestamp(0L, mockSnapshotAtTimestamp0) val exMsg = intercept[IllegalArgumentException] { builder.build(emptyMockEngine) }.getMessage assert(exMsg === "timestamp and version cannot be provided together") } test("atTimestamp: protocol and metadata with timestamp throws IllegalArgumentException") { val builder = TableManager.loadSnapshot(dataPath.toString) .atTimestamp(0L, mockSnapshotAtTimestamp0) .withProtocolAndMetadata(protocol, metadata) val exMsg = intercept[IllegalArgumentException] { builder.build(emptyMockEngine) }.getMessage assert(exMsg === "protocol and metadata can only be provided if a version is provided") } // ===== Committer Tests ===== // test("withCommitter: null committer throws NullPointerException") { assertThrows[NullPointerException] { TableManager.loadSnapshot(dataPath.toString).withCommitter(null) } } test("when no committer is provided, the default committer is created") { val committer = TableManager.loadSnapshot(dataPath.toString) .asInstanceOf[SnapshotBuilderImpl] .atVersion(1) .withProtocolAndMetadata(protocol, metadata) // avoid trying to use engine to load log segment .build(emptyMockEngine) .getCommitter assert(committer.isInstanceOf[DefaultFileSystemManagedTableOnlyCommitter]) } test("custom committer is correctly propagated") { class CustomCommitter extends Committer { override def commit( engine: Engine, finalizedActions: CloseableIterator[Row], commitMetadata: CommitMetadata): CommitResponse = { throw new UnsupportedOperationException("Not implemented") } } val committer = TableManager.loadSnapshot(dataPath.toString) .asInstanceOf[SnapshotBuilderImpl] .atVersion(1) .withCommitter(new CustomCommitter()) .withProtocolAndMetadata(protocol, metadata) // avoid trying to use engine to load log segment .build(emptyMockEngine) .getCommitter assert(committer.isInstanceOf[CustomCommitter]) } // ===== Protocol and Metadata Tests ===== // test("withProtocolAndMetadata: null protocol throws NullPointerException") { assertThrows[NullPointerException] { TableManager.loadSnapshot(dataPath.toString) .withProtocolAndMetadata(null, metadata) } assertThrows[NullPointerException] { TableManager.loadSnapshot(dataPath.toString) .withProtocolAndMetadata(protocol, null) } } test("withProtocolAndMetadata: only if version is provided") { val exMsg = intercept[IllegalArgumentException] { TableManager.loadSnapshot(dataPath.toString) .withProtocolAndMetadata(protocol, metadata) .build(emptyMockEngine) }.getMessage assert(exMsg === "protocol and metadata can only be provided if a version is provided") } test("withProtocolAndMetadata: invalid readerVersion throws KernelException") { val ex = intercept[UnsupportedProtocolVersionException] { TableManager.loadSnapshot(dataPath.toString) .atVersion(10) .withProtocolAndMetadata(new Protocol(999, 2), metadata) .build(emptyMockEngine) } assert(ex.getVersionType === UnsupportedProtocolVersionException.ProtocolVersionType.READER) assert(ex.getMessage.contains("Unsupported Delta protocol reader version")) } test("withProtocolAndMetadata: unknown reader feature throws KernelException") { val exMsg = intercept[UnsupportedTableFeatureException] { TableManager.loadSnapshot(dataPath.toString) .atVersion(10) .withProtocolAndMetadata( new Protocol(3, 7, Set("unknownReaderFeature").asJava, Collections.emptySet()), metadata) .build(emptyMockEngine) }.getMessage assert(exMsg.contains("Unsupported Delta table feature")) } // ===== LogData Tests ===== // test("withLogData: null input throws NullPointerException") { assertThrows[NullPointerException] { TableManager.loadSnapshot(dataPath.toString).withLogData(null) } } Seq( ParsedCatalogCommitData.forInlineData(1, emptyColumnarBatch), ParsedPublishedDeltaData.forFileStatus(deltaFileStatus(1, logPath)), ParsedLogData.forFileStatus(logCompactionStatus(0, 1))).foreach { parsedLogData => val suffix = s"- type=${parsedLogData.getClass.getSimpleName}" test(s"withLogData: non-staged-ratified-commit throws IllegalArgumentException $suffix") { val builder = TableManager .loadSnapshot(dataPath.toString) .atVersion(1) .withLogData(Collections.singletonList(parsedLogData)) val exMsg = intercept[IllegalArgumentException] { builder.build(emptyMockEngine) }.getMessage assert(exMsg.contains("Only staged ratified commits are supported")) } } test("withLogData: non-contiguous input throws IllegalArgumentException") { val exMsg = intercept[IllegalArgumentException] { TableManager.loadSnapshot(dataPath.toString) .atVersion(2) .withLogData(parsedRatifiedStagedCommits(Seq(0, 2)).toList.asJava) .build(emptyMockEngine) }.getMessage assert(exMsg.contains("Log data must be sorted and contiguous")) } test("withLogData: non-sorted input throws IllegalArgumentException") { val exMsg = intercept[IllegalArgumentException] { TableManager.loadSnapshot(dataPath.toString) .atVersion(2) .withLogData(parsedRatifiedStagedCommits(Seq(2, 1, 0)).toList.asJava) .build(emptyMockEngine) }.getMessage assert(exMsg.contains("Log data must be sorted and contiguous")) } ///////////////////////////////////// // Builder Validation Tests -- END // ///////////////////////////////////// test("if P & M are provided then LogSegment is not loaded") { val snapshot = TableManager .loadSnapshot(dataPath.toString) .asInstanceOf[SnapshotBuilderImpl] .atVersion(13) .withProtocolAndMetadata(protocol, metadata) .withLogData(Collections.emptyList()) .build(emptyMockEngine) assert(!snapshot.getLazyLogSegment.isPresent) } // ===== MaxCatalogVersion Tests ===== // test("withMaxCatalogVersion: negative version throws IllegalArgumentException") { val exMsg = intercept[IllegalArgumentException] { TableManager.loadSnapshot(dataPath.toString).withMaxCatalogVersion(-1) }.getMessage assert(exMsg === "A valid version must be >= 0") } test("withMaxCatalogVersion: zero is valid") { // Should not throw TableManager.loadSnapshot(dataPath.toString) .atVersion(0) .withProtocolAndMetadata(protocolWithCatalogManagedSupport, metadata) .withMaxCatalogVersion(0) .build(emptyMockEngine) } test("withMaxCatalogVersion: positive version is valid") { // Should not throw TableManager.loadSnapshot(dataPath.toString) .atVersion(10) .withProtocolAndMetadata(protocolWithCatalogManagedSupport, metadata) .withMaxCatalogVersion(10) .build(emptyMockEngine) } test("withMaxCatalogVersion: version time-travel must be <= maxCatalogVersion") { val exMsg = intercept[IllegalArgumentException] { TableManager.loadSnapshot(dataPath.toString) .atVersion(15) .withMaxCatalogVersion(10) .build(emptyMockEngine) }.getMessage assert(exMsg === "Cannot time-travel to version 15 after the max catalog version 10") } test("withMaxCatalogVersion: version time-travel equal to maxCatalogVersion is valid") { // Should not throw TableManager.loadSnapshot(dataPath.toString) .atVersion(10) .withProtocolAndMetadata(protocolWithCatalogManagedSupport, metadata) .withMaxCatalogVersion(10) .build(emptyMockEngine) } test("withMaxCatalogVersion: version time-travel less than maxCatalogVersion is valid") { // Should not throw TableManager.loadSnapshot(dataPath.toString) .atVersion(5) .withProtocolAndMetadata(protocolWithCatalogManagedSupport, metadata) .withMaxCatalogVersion(10) .build(emptyMockEngine) } test("withMaxCatalogVersion: timestamp time-travel latestSnapshot must have version = " + "maxCatalogVersion") { val mockSnapshotAtVersion5 = getMockSnapshot(dataPath, latestVersion = 5L, timestamp = 1000L) val exMsg = intercept[IllegalArgumentException] { TableManager.loadSnapshot(dataPath.toString) .atTimestamp(0L, mockSnapshotAtVersion5) .withMaxCatalogVersion(10) .build(emptyMockEngine) }.getMessage assert(exMsg === "The latestSnapshot provided for timestamp-based time-travel queries must " + "have version = maxCatalogVersion") } test( "withMaxCatalogVersion: timestamp time-travel with matching latestSnapshot version is valid") { val mockSnapshotAtVersion10 = getMockSnapshot(dataPath, latestVersion = 10L, timestamp = 1000L) // Input validation should not throw (but will throw later when trying to construct log segment) val exMsg = intercept[Exception] { TableManager.loadSnapshot(dataPath.toString) .atTimestamp(500L, mockSnapshotAtVersion10) .withMaxCatalogVersion(10) .build(emptyMockEngine) }.getMessage // Should fail on log segment loading, not on validation assert(!exMsg.contains("latestSnapshot provided for timestamp-based time-travel")) } test("withMaxCatalogVersion: without version, logData must end with maxCatalogVersion") { val exMsg = intercept[IllegalArgumentException] { TableManager.loadSnapshot(dataPath.toString) .withLogData(parsedRatifiedStagedCommits(Seq(0, 1, 2)).toList.asJava) .withMaxCatalogVersion(5) .build(emptyMockEngine) }.getMessage assert(exMsg === "Provided catalog commits must end with max catalog version") } test("withMaxCatalogVersion: without version, logData ending with maxCatalogVersion is valid") { // Input validation should not throw (but will throw later when trying to construct log segment) val exMsg = intercept[Exception] { TableManager.loadSnapshot(dataPath.toString) .withLogData(parsedRatifiedStagedCommits(Seq(0, 1, 2, 3, 4, 5)).toList.asJava) .withMaxCatalogVersion(5) .build(emptyMockEngine) }.getMessage // Should fail on log segment loading, not on validation assert(!exMsg.contains("Provided catalog commits must end with max catalog version")) } test("withMaxCatalogVersion: empty logData with maxCatalogVersion is valid") { // Should not throw - empty logData is allowed TableManager.loadSnapshot(dataPath.toString) .atVersion(5) .withProtocolAndMetadata(protocolWithCatalogManagedSupport, metadata) .withLogData(Collections.emptyList()) .withMaxCatalogVersion(5) .build(emptyMockEngine) } test("withMaxCatalogVersion: version time-travel with logData not including version fails") { val exMsg = intercept[IllegalArgumentException] { TableManager.loadSnapshot(dataPath.toString) .atVersion(5) .withLogData(parsedRatifiedStagedCommits(Seq(0, 1, 2, 3)).toList.asJava) .withMaxCatalogVersion(10) .build(emptyMockEngine) }.getMessage assert(exMsg === "Provided catalog commits must include versionToLoad for time-travel queries") } test("withMaxCatalogVersion: version time-travel with logData ending at version is valid") { // Should not throw - logData ends exactly at requested version TableManager.loadSnapshot(dataPath.toString) .atVersion(5) .withProtocolAndMetadata(protocolWithCatalogManagedSupport, metadata) .withLogData(parsedRatifiedStagedCommits(Seq(0, 1, 2, 3, 4, 5)).toList.asJava) .withMaxCatalogVersion(10) .build(emptyMockEngine) } test("withMaxCatalogVersion: version time-travel with logData beyond version is valid") { // Should not throw - logData extends beyond requested version TableManager.loadSnapshot(dataPath.toString) .atVersion(5) .withProtocolAndMetadata(protocolWithCatalogManagedSupport, metadata) .withLogData(parsedRatifiedStagedCommits(Seq(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10)).toList.asJava) .withMaxCatalogVersion(10) .build(emptyMockEngine) } test("validateMaxCatalogVersionPresence: catalogManaged table requires maxCatalogVersion") { val exMsg = intercept[IllegalArgumentException] { TableManager.loadSnapshot(dataPath.toString) .atVersion(1) .withProtocolAndMetadata(protocolWithCatalogManagedSupport, metadata) .build(emptyMockEngine) }.getMessage assert(exMsg === "Must provide maxCatalogVersion for catalogManaged tables") } test( "validateMaxCatalogVersionPresence: non-catalogManaged table cannot have maxCatalogVersion") { val exMsg = intercept[IllegalArgumentException] { TableManager.loadSnapshot(dataPath.toString) .atVersion(1) .withProtocolAndMetadata(protocol, metadata) // protocol without catalogManaged .withMaxCatalogVersion(1) .build(emptyMockEngine) }.getMessage assert(exMsg === "Should not provide maxCatalogVersion for file-system managed tables") } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/checkpoints/CheckpointInstanceSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.checkpoints import java.util.Optional import scala.collection.JavaConverters._ import io.delta.kernel.internal.fs.Path import org.scalatest.funsuite.AnyFunSuite class CheckpointInstanceSuite extends AnyFunSuite { private val FAKE_DELTA_LOG_PATH = new Path("/path/to/delta/log") test("checkpoint instance comparisons") { val ci1_single_1 = new CheckpointInstance(1, Optional.empty()) val ci1_withparts_2 = new CheckpointInstance(1, Optional.of(2)) val ci1_v2_1 = new CheckpointInstance("01.checkpoint.abc.parquet") val ci2_single_1 = new CheckpointInstance(2, Optional.empty()) val ci2_withparts_4 = new CheckpointInstance(2, Optional.of(4)) val ci2_v2_1 = new CheckpointInstance("02.checkpoint.abc.parquet") val ci2_v2_2 = new CheckpointInstance("02.checkpoint.def.parquet") val ci3_single_1 = new CheckpointInstance(3, Optional.empty()) val ci3_withparts_2 = new CheckpointInstance(3, Optional.of(2)) // version takes priority assert(ci1_single_1.compareTo(ci2_single_1) < 0) assert(ci1_v2_1.compareTo(ci2_single_1) < 0) // v2 takes priority over v1 and multipart assert(ci2_single_1.compareTo(ci2_v2_1) < 0) assert(ci2_withparts_4.compareTo(ci2_v2_1) < 0) // parts takes priority when versions are same assert(ci1_single_1.compareTo(ci1_withparts_2) < 0) // version takes priority over parts or v2 assert(ci2_withparts_4.compareTo(ci3_withparts_2) < 0) assert(ci2_single_1.compareTo(ci3_withparts_2) < 0) // For v2, filepath is tiebreaker. assert(ci2_v2_2.compareTo(ci2_v2_1) > 0) // Everything is less than CheckpointInstance.MAX_VALUE Seq( ci1_single_1, ci1_withparts_2, ci2_single_1, ci2_withparts_4, ci3_single_1, ci3_withparts_2, ci1_v2_1, ci2_v2_1).foreach(ci => assert(ci.compareTo(CheckpointInstance.MAX_VALUE) < 0)) } test("checkpoint instance equality") { val single = new CheckpointInstance("01.checkpoint.parquet") val multipartPart1 = new CheckpointInstance("01.checkpoint.01.02.parquet") val multipartPart2 = new CheckpointInstance("01.checkpoint.02.02.parquet") val v2Checkpoint1 = new CheckpointInstance("01.checkpoint.abc-def.parquet") val v2Checkpoint2 = new CheckpointInstance("01.checkpoint.ghi-klm.parquet") // Single checkpoint is not equal to any other checkpoints at the same version. Seq(multipartPart1, multipartPart2, v2Checkpoint1, v2Checkpoint2).foreach { ci => assert(!single.equals(ci)) assert(single.hashCode() != ci.hashCode()) } // Multipart checkpoints at the same version are equal if they have the same number of parts. Seq(single, v2Checkpoint1, v2Checkpoint2).foreach { ci => assert(!multipartPart1.equals(ci)) assert(multipartPart1.hashCode() != ci.hashCode()) } assert(multipartPart1.equals(multipartPart2)) assert(multipartPart1.hashCode() == multipartPart2.hashCode()) // V2 checkpoints at the same version are equal only if they have the same UUID. Seq(single, multipartPart1, multipartPart2, v2Checkpoint2).foreach { ci => assert(!v2Checkpoint1.equals(ci)) assert(v2Checkpoint1.hashCode() != ci.hashCode()) } } test("checkpoint instance instantiation") { // classic checkpoint val classicCheckpoint = new CheckpointInstance( new Path(FAKE_DELTA_LOG_PATH, "00000000000000000010.checkpoint.parquet").toString) assert(classicCheckpoint.version == 10) assert(!classicCheckpoint.numParts.isPresent()) assert(classicCheckpoint.format == CheckpointInstance.CheckpointFormat.CLASSIC) assert(classicCheckpoint.format.usesSidecars()) // multi-part checkpoint val multipartCheckpoint = new CheckpointInstance( new Path( FAKE_DELTA_LOG_PATH, "00000000000000000010.checkpoint.0000000002.0000000003.parquet").toString) assert(multipartCheckpoint.version == 10) assert(multipartCheckpoint.numParts.isPresent() && multipartCheckpoint.numParts.get() == 3) assert(multipartCheckpoint.format == CheckpointInstance.CheckpointFormat.MULTI_PART) assert(!multipartCheckpoint.format.usesSidecars()) // V2 checkpoint val v2Checkpoint = new CheckpointInstance( new Path( FAKE_DELTA_LOG_PATH, "00000000000000000010.checkpoint.abcda-bacbac.parquet").toString) assert(v2Checkpoint.version == 10) assert(!v2Checkpoint.numParts.isPresent()) assert(v2Checkpoint.format == CheckpointInstance.CheckpointFormat.V2) assert(v2Checkpoint.format.usesSidecars()) // invalid checkpoints intercept[RuntimeException] { new CheckpointInstance( new Path(FAKE_DELTA_LOG_PATH, "00000000000000000010.checkpoint.000000.a.parquet").toString) } intercept[RuntimeException] { new CheckpointInstance( new Path(FAKE_DELTA_LOG_PATH, "00000000000000000010.parquet").toString) } } test("checkpoint instance getCorrespondingFiles") { // classic checkpoint val classicCheckpoint0 = new CheckpointInstance(0) assert(classicCheckpoint0.getCorrespondingFiles(FAKE_DELTA_LOG_PATH).equals( Seq(new Path(FAKE_DELTA_LOG_PATH, "00000000000000000000.checkpoint.parquet")).asJava)) val classicCheckpoint10 = new CheckpointInstance(10) assert(classicCheckpoint10.getCorrespondingFiles(FAKE_DELTA_LOG_PATH).equals( Seq(new Path(FAKE_DELTA_LOG_PATH, "00000000000000000010.checkpoint.parquet")).asJava)) // multi-part checkpoint val multipartCheckpoint = new CheckpointInstance(10, Optional.of(3)) val expectedResult = Seq( "00000000000000000010.checkpoint.0000000001.0000000003.parquet", "00000000000000000010.checkpoint.0000000002.0000000003.parquet", "00000000000000000010.checkpoint.0000000003.0000000003.parquet").map(new Path( FAKE_DELTA_LOG_PATH, _)) assert(multipartCheckpoint.getCorrespondingFiles(FAKE_DELTA_LOG_PATH).equals( expectedResult.asJava)) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/checkpoints/CheckpointerSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.checkpoints import java.io.{FileNotFoundException, IOException} import java.util.Optional import scala.util.control.NonFatal import io.delta.kernel.data.{ColumnarBatch, ColumnVector} import io.delta.kernel.exceptions.KernelEngineException import io.delta.kernel.expressions.Predicate import io.delta.kernel.internal.checkpoints.Checkpointer.findLastCompleteCheckpointBeforeHelper import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.util.FileNames.checkpointFileSingular import io.delta.kernel.internal.util.Utils import io.delta.kernel.test.{BaseMockJsonHandler, MockFileSystemClientUtils, VectorTestUtils} import io.delta.kernel.types.StructType import io.delta.kernel.utils.{CloseableIterator, FileStatus} import org.scalatest.funsuite.AnyFunSuite class CheckpointerSuite extends AnyFunSuite with MockFileSystemClientUtils { import CheckpointerSuite._ ////////////////////////////////////////////////////////////////////////////////// // readLastCheckpointFile tests ////////////////////////////////////////////////////////////////////////////////// test("load a valid last checkpoint metadata file") { val jsonHandler = new MockLastCheckpointMetadataFileReader(maxFailures = 0) val lastCheckpoint = new Checkpointer(VALID_LAST_CHECKPOINT_FILE_TABLE) .readLastCheckpointFile(mockEngine(jsonHandler = jsonHandler)) assertValidCheckpointMetadata(lastCheckpoint) assert(jsonHandler.currentFailCount == 0) } test("load a zero-sized last checkpoint metadata file") { val jsonHandler = new MockLastCheckpointMetadataFileReader(maxFailures = 0) val lastCheckpoint = new Checkpointer(ZERO_SIZED_LAST_CHECKPOINT_FILE_TABLE) .readLastCheckpointFile(mockEngine(jsonHandler = jsonHandler)) assert(!lastCheckpoint.isPresent) assert(jsonHandler.currentFailCount == 0) } test("load an invalid last checkpoint metadata file") { val jsonHandler = new MockLastCheckpointMetadataFileReader(maxFailures = 0) val lastCheckpoint = new Checkpointer(INVALID_LAST_CHECKPOINT_FILE_TABLE) .readLastCheckpointFile(mockEngine(jsonHandler = jsonHandler)) assert(!lastCheckpoint.isPresent) assert(jsonHandler.currentFailCount == 0) } test("retry last checkpoint metadata loading - succeeds at third attempt") { val jsonHandler = new MockLastCheckpointMetadataFileReader(maxFailures = 2) val lastCheckpoint = new Checkpointer(VALID_LAST_CHECKPOINT_FILE_TABLE) .readLastCheckpointFile(mockEngine(jsonHandler = jsonHandler)) assertValidCheckpointMetadata(lastCheckpoint) assert(jsonHandler.currentFailCount == 2) } test("retry last checkpoint metadata loading - exceeds max failures") { val jsonHandler = new MockLastCheckpointMetadataFileReader(maxFailures = 4) val lastCheckpoint = new Checkpointer(VALID_LAST_CHECKPOINT_FILE_TABLE) .readLastCheckpointFile(mockEngine(jsonHandler = jsonHandler)) assert(!lastCheckpoint.isPresent) assert(jsonHandler.currentFailCount == 3) // 3 is the max retries } test("try to load last checkpoint metadata when the file is missing") { val jsonHandler = new MockLastCheckpointMetadataFileReader(maxFailures = 0) val lastCheckpoint = new Checkpointer(LAST_CHECKPOINT_FILE_NOT_FOUND_TABLE) .readLastCheckpointFile(mockEngine(jsonHandler = jsonHandler)) assert(!lastCheckpoint.isPresent) assert(jsonHandler.currentFailCount == 0) } ////////////////////////////////////////////////////////////////////////////////// // findLastCompleteCheckpointBefore tests ////////////////////////////////////////////////////////////////////////////////// test("findLastCompleteCheckpointBefore - no checkpoints") { val files = deltaFileStatuses(Seq.range(0, 25)) Seq((0, 0), (10, 10), (20, 20), (27, 25 /* no delta log files after version 24 */ )).foreach { case (beforeVersion, expNumFilesListed) => assertNoLastCheckpoint(files, beforeVersion, expNumFilesListed) } } test("findLastCompleteCheckpointBefore - single checkpoint") { // 25 delta files and 1 checkpoint file = total 26 files. val files = deltaFileStatuses(Seq.range(0, 25)) ++ singularCheckpointFileStatuses(Seq(10)) Seq((0, 0), (4, 4), (9, 9), (10, 10)).foreach { case (beforeVersion, expNumFilesListed) => assertNoLastCheckpoint(files, beforeVersion, expNumFilesListed) } Seq((14, 10, 15), (25, 10, 26), (27, 10, 26)).foreach { case (beforeVersion, expCheckpointVersion, expNumFilesListed) => assertLastCheckpoint(files, beforeVersion, expCheckpointVersion, expNumFilesListed) } } test("findLastCompleteCheckpointBefore - multi-part checkpoint") { // 25 delta files and 20 checkpoint files = total 45 files. val files = deltaFileStatuses(Seq.range(0, 25)) ++ multiCheckpointFileStatuses(Seq(10), 20) Seq((0, 0), (4, 4), (9, 9), (10, 10)).foreach { case (beforeVersion, expNumFilesListed) => assertNoLastCheckpoint(files, beforeVersion, expNumFilesListed) } Seq((14, 10, 14 + 20), (25, 10, 25 + 20), (27, 10, 25 + 20)).foreach { case (beforeVersion, expCheckpointVersion, expNumFilesListed) => assertLastCheckpoint(files, beforeVersion, expCheckpointVersion, expNumFilesListed) } } test("findLastCompleteCheckpointBefore - multi-part checkpoint per 10commits - 10K commits") { // 10K delta files and 1K checkpoints * 20 files for each checkpoint = total 30K files. val files = deltaFileStatuses(Seq.range(0, 10000)) ++ multiCheckpointFileStatuses(Seq.range(10, 10000, 10), 20) Seq(0, 4, 9, 10).foreach { beforeVersion => val expNumFilesListed = beforeVersion assertNoLastCheckpoint(files, beforeVersion, expNumFilesListed) } Seq(789, 1005, 5787, 9999).foreach { beforeVersion => val expCheckpointVersion = (beforeVersion / 10) * 10 // Listing size is 1000 delta versions (i.e list _delta_log/0001000* to _delta_log/0001999*) val versionsListed = Math.min(beforeVersion, 1000) val expNumFilesListed = versionsListed /* delta files */ + (versionsListed / 10) * 20 /* checkpoints */ assertLastCheckpoint(files, beforeVersion, expCheckpointVersion, expNumFilesListed) } } test("findLastCompleteCheckpointBefore - multi-part checkpoint per 2.5K commits - 10K commits") { // 10K delta files and 4 checkpoints * 50 files for each checkpoint = total 10,080 files. val files = deltaFileStatuses(Seq.range(0, 10000)) ++ multiCheckpointFileStatuses(Seq.range(2500, 10000, 2500), 50) Seq((0, 0), (889, 889), (1001, 1002), (2400, 2402)).foreach { case (beforeVersion, expNumFilesListed) => assertNoLastCheckpoint(files, beforeVersion, expNumFilesListed) } Seq(2600, 5002, 7980, 9999).foreach { beforeVersion => val expCheckpointVersion = (beforeVersion / 2500) * 2500 // Listing size is 1000 delta versions (i.e list _delta_log/0001000* to _delta_log/0001999*) // We list until the checkpoint is encounters in increments of 1000 versions at a time val numListCalls = ((beforeVersion - expCheckpointVersion) / 1000) + 1 val versionsListed = 1000 * numListCalls val expNumFilesListed = numListCalls - 1 /* last file scanned that fails the search and stops */ + versionsListed /* delta files */ + 50 /* one multi-part checkpoint */ assertLastCheckpoint(files, beforeVersion, expCheckpointVersion, expNumFilesListed) } } test("findLastCompleteCheckpointBefore - two checkpoints (one is zero-sized - not valid)") { // 25 delta files and 2 checkpoint file = total 27 files. val files = deltaFileStatuses(Seq.range(0, 25)) ++ singularCheckpointFileStatuses(Seq(10)) ++ Seq(FileStatus.of( checkpointFileSingular(logPath, 20).toString, 0, 0 )) // zero-sized CP Seq((0, 0), (4, 4), (9, 9), (10, 10)).foreach { case (beforeVersion, expNumFilesListed) => assertNoLastCheckpoint(files, beforeVersion, expNumFilesListed) } Seq((14, 10, 15), (25, 10, 27), (27, 10, 27)).foreach { case (beforeVersion, expCheckpointVersion, expNumFilesListed) => assertLastCheckpoint(files, beforeVersion, expCheckpointVersion, expNumFilesListed) } } /** Assert that the checkpoint metadata is same as [[SAMPLE_LAST_CHECKPOINT_FILE_CONTENT]] */ def assertValidCheckpointMetadata(actual: Optional[CheckpointMetaData]): Unit = { assert(actual.isPresent) val metadata = actual.get() assert(metadata.version == 40L) assert(metadata.size == 44L) assert(metadata.parts == Optional.of(20L)) } def assertLastCheckpoint( deltaLogFiles: Seq[FileStatus], beforeVersion: Long, expCheckpointVersion: Long, expNumFilesListed: Long): Unit = { val engine = createMockFSListFromEngine(deltaLogFiles) val result = findLastCompleteCheckpointBeforeHelper(engine, logPath, beforeVersion) assert(result._1.isPresent, s"Checkpoint should be found for version=$beforeVersion") assert( result._1.get().version === expCheckpointVersion, s"Incorrect checkpoint version before version=$beforeVersion") assert(result._2 === expNumFilesListed, s"Invalid number of files listed: $beforeVersion") } def assertNoLastCheckpoint( deltaLogFiles: Seq[FileStatus], beforeVersion: Long, expNumFilesListed: Long): Unit = { val engine = createMockFSListFromEngine(deltaLogFiles) val result = findLastCompleteCheckpointBeforeHelper(engine, logPath, beforeVersion) assert(!result._1.isPresent, s"No checkpoint should be found for version=$beforeVersion") assert(result._2 == expNumFilesListed, s"Invalid number of files listed: $beforeVersion") } } object CheckpointerSuite extends VectorTestUtils { val SAMPLE_LAST_CHECKPOINT_FILE_CONTENT: ColumnarBatch = new ColumnarBatch { override def getSchema: StructType = CheckpointMetaData.READ_SCHEMA override def getColumnVector(ordinal: Int): ColumnVector = { ordinal match { case 0 => longVector(Seq(40)) // version case 1 => longVector(Seq(44)) // size case 2 => longVector(Seq(20)); // parts case 3 => mapTypeVector(Seq(Map.empty[String, String])) // tags } } override def getSize: Int = 1 } val ZERO_ENTRIES_COLUMNAR_BATCH: ColumnarBatch = new ColumnarBatch { override def getSchema: StructType = CheckpointMetaData.READ_SCHEMA // empty vector for all columns override def getColumnVector(ordinal: Int): ColumnVector = longVector(Seq.empty) override def getSize: Int = 0 } val VALID_LAST_CHECKPOINT_FILE_TABLE = new Path("/valid") val ZERO_SIZED_LAST_CHECKPOINT_FILE_TABLE = new Path("/zero_sized") val INVALID_LAST_CHECKPOINT_FILE_TABLE = new Path("/invalid") val LAST_CHECKPOINT_FILE_NOT_FOUND_TABLE = new Path("/filenotfoundtable") } /** `maxFailures` allows how many times to fail before returning the valid data */ class MockLastCheckpointMetadataFileReader(maxFailures: Int) extends BaseMockJsonHandler { import CheckpointerSuite._ var currentFailCount = 0 override def readJsonFiles( fileIter: CloseableIterator[FileStatus], physicalSchema: StructType, predicate: Optional[Predicate]): CloseableIterator[ColumnarBatch] = { val file = fileIter.next() val path = new Path(file.getPath) Utils.singletonCloseableIterator( try { if (currentFailCount < maxFailures) { currentFailCount += 1 throw new IOException("Retryable exception") } path.getParent match { case VALID_LAST_CHECKPOINT_FILE_TABLE => SAMPLE_LAST_CHECKPOINT_FILE_CONTENT case ZERO_SIZED_LAST_CHECKPOINT_FILE_TABLE => ZERO_ENTRIES_COLUMNAR_BATCH case INVALID_LAST_CHECKPOINT_FILE_TABLE => throw new IOException("Invalid last checkpoint file") case LAST_CHECKPOINT_FILE_NOT_FOUND_TABLE => throw new FileNotFoundException("File not found") case _ => throw new IOException("Unknown table") } } catch { case NonFatal(e) => throw new KernelEngineException("Failed to read last checkpoint", e); }) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/checksum/CRCInfoReadCompatSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.checksum import java.util import java.util.{Collections, Optional} import scala.collection.JavaConverters._ import io.delta.kernel.data.{ColumnarBatch, ColumnVector, Row} import io.delta.kernel.internal.actions.{DomainMetadata, Format, Metadata, Protocol} import io.delta.kernel.internal.checksum.CRCInfo.{CRC_FILE_READ_SCHEMA, CRC_FILE_SCHEMA} import io.delta.kernel.internal.data.GenericColumnVector import io.delta.kernel.internal.stats.FileSizeHistogram import io.delta.kernel.internal.types.DataTypeJsonSerDe import io.delta.kernel.internal.util.VectorUtils import io.delta.kernel.internal.util.VectorUtils.{buildArrayValue, stringStringMapValue} import io.delta.kernel.test.VectorTestUtils import io.delta.kernel.types.{DataType, StringType, StructType} import org.scalatest.funsuite.AnyFunSuite /** * Tests that CRCInfo.fromColumnarBatch correctly reads the file size histogram from CRC files * written using the legacy field name "histogramOpt" or the spec-compliant "fileSizeHistogram". */ class CRCInfoReadCompatSuite extends AnyFunSuite with VectorTestUtils { private val testProtocol = new Protocol(1, 2, Collections.emptySet(), Collections.emptySet()) private val testMetadata = new Metadata( "id", Optional.of("name"), Optional.of("description"), new Format("parquet", Collections.emptyMap()), DataTypeJsonSerDe.serializeDataType(new StructType()), new StructType(), buildArrayValue(util.Arrays.asList("c3"), StringType.STRING), Optional.of(123), stringStringMapValue(new util.HashMap[String, String]() { put("delta.appendOnly", "true") })) /** Creates a simple histogram with distinct values for identification in tests. */ private def createTestHistogram(fileCount: Long): FileSizeHistogram = { val boundaries = Array(0L, 1024L) val counts = Array(fileCount, 0L) val bytes = Array(fileCount * 100, 0L) new FileSizeHistogram(boundaries, counts, bytes) } /** * Builds a ColumnVector for a FileSizeHistogram struct field. If histogram is None, the vector * is null at row 0. */ private def histogramColumnVector( histogram: Option[FileSizeHistogram]): ColumnVector = { val rowValue: Row = histogram.map(_.toRow()).orNull new GenericColumnVector( util.Arrays.asList(rowValue), FileSizeHistogram.FULL_SCHEMA) } /** * Build a ColumnarBatch with the given schema and histogram column vectors at the appropriate * positions. */ private def buildBatch( schema: StructType, fileSizeHistogram: Option[FileSizeHistogram], histogramOpt: Option[FileSizeHistogram]): ColumnarBatch = { val protocolColVector = new GenericColumnVector( util.Arrays.asList(testProtocol.toRow()), Protocol.FULL_SCHEMA) val metadataColVector = new GenericColumnVector( util.Arrays.asList(testMetadata.toRow()), Metadata.FULL_SCHEMA) new ColumnarBatch { override def getSchema: StructType = schema override def getSize: Int = 1 override def getColumnVector(ordinal: Int): ColumnVector = { val fieldName = schema.at(ordinal).getName fieldName match { case "tableSizeBytes" => longVector(Seq(1000L)) case "numFiles" => longVector(Seq(10L)) case "numMetadata" => longVector(Seq(1L)) case "numProtocol" => longVector(Seq(1L)) case "metadata" => metadataColVector case "protocol" => protocolColVector case "txnId" => stringVector(Seq(null)) case "domainMetadata" => nullColumnVector( CRC_FILE_SCHEMA.get("domainMetadata").getDataType) case "fileSizeHistogram" => histogramColumnVector(fileSizeHistogram) case "histogramOpt" => histogramColumnVector(histogramOpt) case _ => throw new IllegalArgumentException(s"Unknown field: $fieldName") } } } } /** Creates a column vector that is null at every row. */ private def nullColumnVector(dataType: DataType): ColumnVector = { new ColumnVector { override def getDataType: DataType = dataType override def getSize: Int = 1 override def close(): Unit = {} override def isNullAt(rowId: Int): Boolean = true } } test("reads fileSizeHistogram when only spec-compliant field is present") { val histogram = createTestHistogram(fileCount = 42) val batch = buildBatch( CRC_FILE_READ_SCHEMA, fileSizeHistogram = Some(histogram), histogramOpt = None) val crcInfo = CRCInfo.fromColumnarBatch(1L, batch, 0, "test.crc") assert(crcInfo.isPresent) assert(crcInfo.get().getFileSizeHistogram.isPresent) assert(crcInfo.get().getFileSizeHistogram.get() === histogram) } test("reads histogramOpt when only legacy field is present") { val histogram = createTestHistogram(fileCount = 99) val batch = buildBatch( CRC_FILE_READ_SCHEMA, fileSizeHistogram = None, histogramOpt = Some(histogram)) val crcInfo = CRCInfo.fromColumnarBatch(1L, batch, 0, "test.crc") assert(crcInfo.isPresent) assert(crcInfo.get().getFileSizeHistogram.isPresent) assert(crcInfo.get().getFileSizeHistogram.get() === histogram) } test("prefers fileSizeHistogram when both fields are present") { val specHistogram = createTestHistogram(fileCount = 10) val legacyHistogram = createTestHistogram(fileCount = 20) val batch = buildBatch( CRC_FILE_READ_SCHEMA, fileSizeHistogram = Some(specHistogram), histogramOpt = Some(legacyHistogram)) val crcInfo = CRCInfo.fromColumnarBatch(1L, batch, 0, "test.crc") assert(crcInfo.isPresent) assert(crcInfo.get().getFileSizeHistogram.isPresent) assert(crcInfo.get().getFileSizeHistogram.get() === specHistogram) } test("returns empty histogram when neither field is present") { val batch = buildBatch( CRC_FILE_READ_SCHEMA, fileSizeHistogram = None, histogramOpt = None) val crcInfo = CRCInfo.fromColumnarBatch(1L, batch, 0, "test.crc") assert(crcInfo.isPresent) assert(!crcInfo.get().getFileSizeHistogram.isPresent) } test("safely skips fallback when batch uses original CRC_FILE_SCHEMA") { // When fromColumnarBatch is called with a batch using the original schema // (without histogramOpt), the fallback should be safely skipped. val protocolColVector = new GenericColumnVector( util.Arrays.asList(testProtocol.toRow()), Protocol.FULL_SCHEMA) val metadataColVector = new GenericColumnVector( util.Arrays.asList(testMetadata.toRow()), Metadata.FULL_SCHEMA) val batch = new ColumnarBatch { override def getSchema: StructType = CRC_FILE_SCHEMA override def getSize: Int = 1 override def getColumnVector(ordinal: Int): ColumnVector = { val fieldName = CRC_FILE_SCHEMA.at(ordinal).getName fieldName match { case "tableSizeBytes" => longVector(Seq(1000L)) case "numFiles" => longVector(Seq(10L)) case "numMetadata" => longVector(Seq(1L)) case "numProtocol" => longVector(Seq(1L)) case "metadata" => metadataColVector case "protocol" => protocolColVector case "txnId" => stringVector(Seq(null)) case "domainMetadata" => nullColumnVector( CRC_FILE_SCHEMA.get("domainMetadata").getDataType) case "fileSizeHistogram" => histogramColumnVector(None) case _ => throw new IllegalArgumentException(s"Unknown field: $fieldName") } } } val crcInfo = CRCInfo.fromColumnarBatch(1L, batch, 0, "test.crc") assert(crcInfo.isPresent) assert(!crcInfo.get().getFileSizeHistogram.isPresent) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/checksum/ChecksumWriterSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.checksum import java.util import java.util.{Collections, Optional} import scala.collection.JavaConverters.{asScalaBufferConverter, asScalaSetConverter, seqAsJavaListConverter, setAsJavaSetConverter} import io.delta.kernel.data.Row import io.delta.kernel.exceptions.TableNotFoundException import io.delta.kernel.internal.actions.{DomainMetadata, Format, Metadata, Protocol} import io.delta.kernel.internal.checksum.CRCInfo.CRC_FILE_SCHEMA import io.delta.kernel.internal.data.{GenericRow, StructRow} import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.types.DataTypeJsonSerDe import io.delta.kernel.internal.util.VectorUtils import io.delta.kernel.internal.util.VectorUtils.{buildArrayValue, buildColumnVector, stringStringMapValue} import io.delta.kernel.test.{BaseMockJsonHandler, MockEngineUtils} import io.delta.kernel.types.{StringType, StructType} import io.delta.kernel.utils.CloseableIterator import org.scalatest.funsuite.AnyFunSuite /** * Test suite for ChecksumWriter functionality. */ class ChecksumWriterSuite extends AnyFunSuite with MockEngineUtils { private val FAKE_DELTA_LOG_PATH = new Path("/path/to/delta/log") // Schema field indices in crc file private val TABLE_SIZE_BYTES_IDX = CRC_FILE_SCHEMA.indexOf("tableSizeBytes") private val NUM_FILES_IDX = CRC_FILE_SCHEMA.indexOf("numFiles") private val NUM_METADATA_IDX = CRC_FILE_SCHEMA.indexOf("numMetadata") private val NUM_PROTOCOL_IDX = CRC_FILE_SCHEMA.indexOf("numProtocol") private val TXN_ID_IDX = CRC_FILE_SCHEMA.indexOf("txnId") private val DOMAIN_METADATA_IDX = CRC_FILE_SCHEMA.indexOf("domainMetadata") private val METADATA_IDX = CRC_FILE_SCHEMA.indexOf("metadata") private val PROTOCOL_IDX = CRC_FILE_SCHEMA.indexOf("protocol") private val FILE_SIZE_HISTOGRAM_IDX = CRC_FILE_SCHEMA.indexOf("fileSizeHistogram") test("write checksum") { val jsonHandler = new MockCheckSumFileJsonWriter() val checksumWriter = new ChecksumWriter(FAKE_DELTA_LOG_PATH) val protocol = createTestProtocol() val metadata = createTestMetadata() def testChecksumWrite( txn: Optional[String], domainMetadata: Optional[util.Set[DomainMetadata]]): Unit = { val version = 1L val tableSizeBytes = 100L val numFiles = 1L // TODO when we support writing fileSizeHistogram as part of CRC update this to be non-empty checksumWriter.writeCheckSum( mockEngine(jsonHandler = jsonHandler), new CRCInfo( version, metadata, protocol, tableSizeBytes, numFiles, txn, domainMetadata, Optional.empty())) verifyChecksumFile(jsonHandler, version) verifyChecksumContent( jsonHandler.capturedCrcRow.get, tableSizeBytes, numFiles, metadata, protocol, txn, domainMetadata) } // Test with and without transaction ID, domain metadata testChecksumWrite(Optional.of("txn"), Optional.empty()) testChecksumWrite(Optional.empty(), Optional.empty()) testChecksumWrite( Optional.empty(), Optional.of(Seq( new DomainMetadata("domain1", "", false /* removed */ ), new DomainMetadata("domain2", "", false /* removed */ )).toSet.asJava)) // Per protocol, domain metadata list should exclude tombstone. val exception = intercept[IllegalArgumentException] { testChecksumWrite( Optional.empty(), Optional.of(Seq( new DomainMetadata("domain1", "", true /* removed */ ), new DomainMetadata("domain2", "", false /* removed */ )).toSet.asJava)) } assert(exception.getMessage.contains("Domain metadata in CRC should exclude tombstones")) } private def verifyChecksumFile(jsonHandler: MockCheckSumFileJsonWriter, version: Long): Unit = { assert(jsonHandler.checksumFilePath == s"$FAKE_DELTA_LOG_PATH/${"%020d".format(version)}.crc") assert(jsonHandler.capturedCrcRow.isDefined) assert(jsonHandler.capturedCrcRow.get.getSchema == CRC_FILE_SCHEMA) } private def verifyChecksumContent( actualCheckSumRow: Row, expectedTableSizeBytes: Long, expectedNumFiles: Long, expectedMetadata: Metadata, expectedProtocol: Protocol, expectedTxnId: Optional[String], expectedDomainMetadata: Optional[util.Set[DomainMetadata]]): Unit = { assert(!actualCheckSumRow.isNullAt(TABLE_SIZE_BYTES_IDX) && actualCheckSumRow.getLong( TABLE_SIZE_BYTES_IDX) == expectedTableSizeBytes) assert(!actualCheckSumRow.isNullAt( NUM_FILES_IDX) && actualCheckSumRow.getLong(NUM_FILES_IDX) == expectedNumFiles) assert(!actualCheckSumRow.isNullAt( NUM_METADATA_IDX) && actualCheckSumRow.getLong(NUM_METADATA_IDX) == 1L) assert(!actualCheckSumRow.isNullAt( NUM_PROTOCOL_IDX) && actualCheckSumRow.getLong(NUM_PROTOCOL_IDX) == 1L) assert(expectedProtocol === Protocol.fromRow(actualCheckSumRow.getStruct(PROTOCOL_IDX))) assert(expectedMetadata === Metadata.fromRow(actualCheckSumRow.getStruct(METADATA_IDX))) if (expectedTxnId.isPresent) { assert(actualCheckSumRow.getString(TXN_ID_IDX) == expectedTxnId.get()) } else { assert(actualCheckSumRow.isNullAt(TXN_ID_IDX)) } if (expectedDomainMetadata.isPresent) { assert(VectorUtils.toJavaList[Row](actualCheckSumRow.getArray(DOMAIN_METADATA_IDX)).asScala .map(DomainMetadata.fromRow).toSet === expectedDomainMetadata.get().asScala) } else { assert(actualCheckSumRow.isNullAt(DOMAIN_METADATA_IDX)) } // TODO once we support writing fileSizeHistogram as part of CRC check it here assert(actualCheckSumRow.isNullAt(FILE_SIZE_HISTOGRAM_IDX)) } private def createTestMetadata(): Metadata = { new Metadata( "id", Optional.of("name"), Optional.of("description"), new Format("parquet", Collections.emptyMap()), DataTypeJsonSerDe.serializeDataType(new StructType()), new StructType(), buildArrayValue(util.Arrays.asList("c3"), StringType.STRING), Optional.of(123), stringStringMapValue(new util.HashMap[String, String]() { put("delta.appendOnly", "true") })) } private def createTestProtocol(): Protocol = { new Protocol( /* minReaderVersion= */ 1, /* minWriterVersion= */ 2, Collections.emptySet(), Collections.emptySet()) } } /** * Mock implementation of JsonHandler for testing checksum file writing. */ class MockCheckSumFileJsonWriter extends BaseMockJsonHandler { var capturedCrcRow: Option[Row] = None var checksumFilePath: String = "" override def writeJsonFileAtomically( filePath: String, data: CloseableIterator[Row], overwrite: Boolean): Unit = { checksumFilePath = filePath assert(data.hasNext, "Expected data iterator to contain exactly one row") capturedCrcRow = Some(data.next()) assert(!data.hasNext, "Expected data iterator to contain exactly one row") } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/clustering/ClusteringMetadataDomainSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.clustering import scala.collection.JavaConverters._ import io.delta.kernel.expressions.Column import io.delta.kernel.internal.util.{ColumnMapping, ColumnMappingSuiteBase} import io.delta.kernel.types._ import org.scalatest.funsuite.AnyFunSuite class ClusteringMetadataDomainSuite extends AnyFunSuite with ColumnMappingSuiteBase { private def convertToPhysicalColumn( logicalColumns: List[Column], schema: StructType): List[Column] = { logicalColumns.map { column => ColumnMapping.getPhysicalColumnNameAndDataType(schema, column)._1 } } test("ClusteringDomainMetadata can be serialized") { val clusteringColumns = List(new Column(Array("col1", "`col2,col3`", "`col4.col5`,col6"))) val clusteringMetadataDomain = ClusteringMetadataDomain.fromClusteringColumns( clusteringColumns.asJava) val serializedString = clusteringMetadataDomain.toDomainMetadata.toString assert(serializedString === """|DomainMetadata{domain='delta.clustering', configuration= |'{"clusteringColumns":[["col1","`col2,col3`","`col4.col5`,col6"]]}', | removed='false'}""".stripMargin.replace("\n", "")) } test("ClusteringDomainMetadata can be deserialized") { val configJson = """{"clusteringColumns":[["col1","`col2,col3`","`col4.col5`,col6"]]}""" val clusteringMD = ClusteringMetadataDomain.fromJsonConfiguration(configJson) assert(clusteringMD.getClusteringColumns === List(new Column(Array( "col1", "`col2,col3`", "`col4.col5`,col6"))).asJava) } test("Successfully get DomainMetadata for non-nested columns") { val schema = new StructType() .add("id", IntegerType.INTEGER, true) .add("name", IntegerType.INTEGER, true) .add("age", IntegerType.INTEGER, true) val clusterColumns = List(new Column("name"), new Column("age")) val physicalColumns = convertToPhysicalColumn(clusterColumns, schema) val clusteringMetadataDomain = ClusteringMetadataDomain.fromClusteringColumns( physicalColumns.asJava) val clusteringDomainMetadata = clusteringMetadataDomain.toDomainMetadata assert(clusteringMetadataDomain.getClusteringColumns == clusterColumns.asJava) assert(clusteringDomainMetadata.getDomain == "delta.clustering") assert(clusteringDomainMetadata.getConfiguration == """{"clusteringColumns":[["name"],["age"]]}""") } test("Successfully get DomainMetadata for nested columns") { val schema = new StructType() .add("id", IntegerType.INTEGER, true) .add( "user", new StructType() .add( "address", new StructType() .add("city", StringType.STRING, true))) val clusterColumns = List(new Column(Array("user", "address", "city"))) val physicalColumns = convertToPhysicalColumn(clusterColumns, schema) val clusteringMetadataDomain = ClusteringMetadataDomain.fromClusteringColumns( physicalColumns.asJava) val clusteringDomainMetadata = clusteringMetadataDomain.toDomainMetadata assert(clusteringMetadataDomain.getClusteringColumns == List(new Column(Array("user", "address", "city"))).asJava) assert(clusteringDomainMetadata.getDomain == "delta.clustering") assert(clusteringDomainMetadata.getConfiguration == """{"clusteringColumns":[["user","address","city"]]}""") } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/columndefaults/ColumnDefaultsSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.columndefaults import java.util.Optional import scala.collection.JavaConverters._ import io.delta.kernel.exceptions.KernelException import io.delta.kernel.internal.actions.{Metadata, Protocol} import io.delta.kernel.internal.tablefeatures.TableFeatures import io.delta.kernel.test.ActionUtils import io.delta.kernel.types._ import org.scalatest.funsuite.AnyFunSuite class ColumnDefaultsSuite extends AnyFunSuite with ActionUtils { val validProtocol = new Protocol( TableFeatures.TABLE_FEATURES_MIN_READER_VERSION, TableFeatures.TABLE_FEATURES_MIN_WRITER_VERSION, Set.empty[String].asJava, Set( TableFeatures.ALLOW_COLUMN_DEFAULTS_W_FEATURE.featureName(), TableFeatures.ICEBERG_COMPAT_V3_W_FEATURE.featureName()).asJava) def metadataForDefault(value: String): FieldMetadata = FieldMetadata.builder().putString("CURRENT_DEFAULT", value).build() test("validate schema with valid literals") { val correctSchema = new StructType() .add("id", IntegerType.INTEGER) .add("int", IntegerType.INTEGER, metadataForDefault("\"123\"")) .add("short", ShortType.SHORT, metadataForDefault("123")) .add("long", LongType.LONG, metadataForDefault("1231341")) .add("decimal", new DecimalType(10, 5), metadataForDefault("1231.34155")) .add("float", FloatType.FLOAT, metadataForDefault("123.1341")) .add("double", DoubleType.DOUBLE, metadataForDefault("123.7774")) .add("double2", DoubleType.DOUBLE, metadataForDefault("'123.7774'")) .add("name", StringType.STRING, metadataForDefault("\"tom\"")) .add("name2", StringType.STRING, metadataForDefault("'tom'")) .add("date", DateType.DATE, metadataForDefault("'2025-01-01'")) .add("date2", DateType.DATE, metadataForDefault("2025-01-01")) .add("ts", TimestampType.TIMESTAMP, metadataForDefault("\"2025-01-01T00:00:00Z\"")) .add("ts2", TimestampType.TIMESTAMP, metadataForDefault("\"2025-01-01T00:00:00+01:00\"")) .add("ts3", TimestampType.TIMESTAMP, metadataForDefault("2025-01-01T00:00:00Z")) .add("tsntz", TimestampNTZType.TIMESTAMP_NTZ, metadataForDefault("'2025-01-01T00:00:00'")) .add("tsntz2", TimestampNTZType.TIMESTAMP_NTZ, metadataForDefault("2025-01-01T00:00:00")) .add( "childStruct", new StructType() .add("childId", IntegerType.INTEGER, metadataForDefault("100")) .add( "grandChildList", new ArrayType( new StructType().add("nestedId", IntegerType.INTEGER, metadataForDefault("120")), false))) .add( "grandChildMap", new MapType( new StructType().add("mapKeyId", IntegerType.INTEGER, metadataForDefault("220")), new StructType().add("mapValueId", IntegerType.INTEGER, metadataForDefault("330")), false)) .add( "childList", new ArrayType( new StructType().add("clid", IntegerType.INTEGER, metadataForDefault("300")), false)) ColumnDefaults.validateSchema(correctSchema, true, true) var e = intercept[KernelException] { ColumnDefaults.validateSchema(correctSchema, true, false) } assert(e.getMessage == "In Delta Kernel, default values table feature requires IcebergCompatV3 to be enabled.") // Validate column default requires v3 even if schema has no defaults e = intercept[KernelException] { ColumnDefaults.validateSchema(new StructType().add("key", IntegerType.INTEGER), true, false) } assert(e.getMessage == "In Delta Kernel, default values table feature requires IcebergCompatV3 to be enabled.") e = intercept[KernelException] { ColumnDefaults.validateSchema(correctSchema, false, true) } assert(e.getMessage == "Found column defaults in the schema but the table does not support" + " the columnDefaults table feature.") } test("validate schema with unsupported types") { val unsupportedCases = Seq( new StructType().add("sub", IntegerType.INTEGER), new ArrayType(IntegerType.INTEGER, false), new MapType(IntegerType.INTEGER, IntegerType.INTEGER, false), VariantType.VARIANT) unsupportedCases.foreach { dataType => val schemaWithUnsupportedType = new StructType() .add("id", IntegerType.INTEGER) .add("col1", dataType, metadataForDefault("120")) val e = intercept[KernelException] { ColumnDefaults.validateSchema(schemaWithUnsupportedType, true, true) } assert(e.getMessage.contains("Kernel does not support default value for data type")) } } test("validate schema with invalid literal values") { val badCases = Seq( (StringType.STRING, "string with no quotes"), (BinaryType.BINARY, "string with no quotes"), (ShortType.SHORT, "1248.995"), (IntegerType.INTEGER, "1248.995"), (LongType.LONG, "1248.995"), (FloatType.FLOAT, "michael"), (DoubleType.DOUBLE, "jordan"), (new DecimalType(10, 0), "1248.995"), (new DecimalType(10, 5), "12480031341.995"), (new DecimalType(10, 5), "1248.995031"), (DateType.DATE, "1248.995031"), (DateType.DATE, "\"2025/09/01\""), (DateType.DATE, "09/01/2025"), (TimestampType.TIMESTAMP, "2025-01-01"), (TimestampType.TIMESTAMP, "2025-01-01"), (TimestampType.TIMESTAMP, "2025-01-01 00:00:00"), (TimestampType.TIMESTAMP, "'2025-01-01T00:00:00'"), (TimestampType.TIMESTAMP, "2025-01-01T00:00:00+2:00"), (TimestampNTZType.TIMESTAMP_NTZ, "2025-01-01"), (TimestampNTZType.TIMESTAMP_NTZ, "'2025-01-01 00:00:00'")) badCases.foreach { case (dataType, defaultValue) => val badSchema1 = new StructType() .add("id", IntegerType.INTEGER) .add("col", dataType, metadataForDefault(defaultValue)) val badSchema2 = new StructType() .add("id", IntegerType.INTEGER) .add( "childStruct", new StructType() .add("childId", IntegerType.INTEGER, metadataForDefault("100")) .add("badcol", dataType, metadataForDefault(defaultValue))) val badSchema3 = new StructType() .add("id", IntegerType.INTEGER) .add("name", StringType.STRING, metadataForDefault("tom")) .add( "childList", new ArrayType( new StructType() .add("clid", IntegerType.INTEGER, metadataForDefault("300")) .add("badcol", dataType, metadataForDefault(defaultValue)), false)) val badSchema4 = new StructType() .add("id", IntegerType.INTEGER) .add("name", StringType.STRING, metadataForDefault("tom")) .add( "grandChildMap", new MapType( new StructType().add("mapKeyId", IntegerType.INTEGER, metadataForDefault("220")), new StructType().add("mapValueId", dataType, metadataForDefault(defaultValue)), false)) Seq(badSchema1, badSchema2, badSchema3, badSchema4).foreach(schema => { val e = intercept[KernelException] { ColumnDefaults.validateSchema(schema, true, true) } assert(e.getMessage.contains( "currently only literal values are supported for default values in Kernel.")) }) } } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/commit/CommitMetadataSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.commit import java.util.Optional import scala.collection.JavaConverters._ import io.delta.kernel.commit.CommitMetadata.CommitType import io.delta.kernel.internal.actions.{DomainMetadata, Metadata, Protocol} import io.delta.kernel.internal.util.{Tuple2 => KernelTuple2} import io.delta.kernel.test.{TestFixtures, VectorTestUtils} import io.delta.kernel.types.{IntegerType, StructType} import org.scalatest.funsuite.AnyFunSuite class CommitMetadataSuite extends AnyFunSuite with TestFixtures with VectorTestUtils { private val protocol12 = new Protocol(1, 2) private val logPath = "/fake/_delta_log" private val createVersion0 = 0 private val updateVersionNonZero = 1 test("constructor validates non-negative version") { val ex = intercept[IllegalArgumentException] { createCommitMetadata(version = -1L) } assert(ex.getMessage.contains("version must be non-negative")) } test("constructor validates null parameters") { intercept[NullPointerException] { createCommitMetadata( version = updateVersionNonZero, logPath = null, readPandMOpt = Optional.of(new KernelTuple2(protocol12, basicPartitionedMetadata))) } intercept[NullPointerException] { createCommitMetadata( version = updateVersionNonZero, commitInfo = null, readPandMOpt = Optional.of(new KernelTuple2(protocol12, basicPartitionedMetadata))) } intercept[NullPointerException] { createCommitMetadata( version = createVersion0, commitDomainMetadatas = null, newProtocolOpt = Optional.of(protocol12), newMetadataOpt = Optional.of(basicPartitionedMetadata)) } intercept[NullPointerException] { createCommitMetadata( version = updateVersionNonZero, readPandMOpt = Optional.of(new KernelTuple2(protocol12, basicPartitionedMetadata)), committerProperties = null) } intercept[NullPointerException] { createCommitMetadata( version = updateVersionNonZero, readPandMOpt = Optional.of(new KernelTuple2(protocol12, basicPartitionedMetadata)), maxKnownPublishedDeltaVersion = null) } } test("constructor validates readProtocol and readMetadata consistency") { // Both present is valid createCommitMetadata( version = updateVersionNonZero, readPandMOpt = Optional.of(new KernelTuple2(protocol12, basicPartitionedMetadata))) // Both absent is valid if new ones are present createCommitMetadata( version = createVersion0, newProtocolOpt = Optional.of(protocol12), newMetadataOpt = Optional.of(basicPartitionedMetadata)) } test("constructor validates at least one protocol must be present") { intercept[IllegalArgumentException] { createCommitMetadata( version = createVersion0, newMetadataOpt = Optional.of(basicPartitionedMetadata)) } } test("constructor validates at least one metadata must be present") { intercept[IllegalArgumentException] { createCommitMetadata( version = createVersion0, newProtocolOpt = Optional.of(protocol12)) } } test("constructor validates ICT present if catalogManaged enabled") { val exMsg = intercept[IllegalArgumentException] { createCommitMetadata( version = createVersion0, commitInfo = testCommitInfo(ictEnabled = false), newProtocolOpt = Optional.of(protocolWithCatalogManagedSupport), newMetadataOpt = Optional.of(basicPartitionedMetadata)) }.getMessage assert(exMsg.contains("InCommitTimestamp must be present for commits to catalogManaged tables")) } test("getNewDomainMetadatas returns provided domain metadata") { val domainMetadata1 = new DomainMetadata("domain1", """{"key":"value"}""", false) val domainMetadata2 = new DomainMetadata("domain2", "", false) val domainMetadatas = List(domainMetadata1, domainMetadata2) val commitMetadata = createCommitMetadata( version = createVersion0, commitDomainMetadatas = domainMetadatas, newProtocolOpt = Optional.of(protocol12), newMetadataOpt = Optional.of(basicPartitionedMetadata)) val returnedMetadatas = commitMetadata.getCommitDomainMetadatas assert(returnedMetadatas.size() == 2) assert(returnedMetadatas.contains(domainMetadata1)) assert(returnedMetadatas.contains(domainMetadata2)) } test("getCommitterProperties returns provided supplier") { val props = Map("key1" -> "value1", "key2" -> "value2").asJava val commitMetadata = createCommitMetadata( version = updateVersionNonZero, readPandMOpt = Optional.of(new KernelTuple2(protocol12, basicPartitionedMetadata)), committerProperties = () => props) assert(commitMetadata.getCommitterProperties.get() == props) } test("getEffectiveProtocol returns new protocol when present") { val newProtocol = new Protocol(2, 3) val commitMetadata = createCommitMetadata( version = updateVersionNonZero, readPandMOpt = Optional.of(new KernelTuple2(protocol12, basicPartitionedMetadata)), newProtocolOpt = Optional.of(newProtocol)) assert(commitMetadata.getEffectiveProtocol == newProtocol) } test("getEffectiveProtocol returns read protocol when new protocol absent") { val commitMetadata = createCommitMetadata( version = updateVersionNonZero, readPandMOpt = Optional.of(new KernelTuple2(protocol12, basicPartitionedMetadata))) assert(commitMetadata.getEffectiveProtocol == protocol12) } test("getEffectiveMetadata returns new metadata when present") { val newMetadata = testMetadata(new StructType().add("newCol", IntegerType.INTEGER)) val commitMetadata = createCommitMetadata( version = updateVersionNonZero, readPandMOpt = Optional.of(new KernelTuple2(protocol12, basicPartitionedMetadata)), newMetadataOpt = Optional.of(newMetadata)) assert(commitMetadata.getEffectiveMetadata == newMetadata) } test("getEffectiveMetadata returns read metadata when new metadata absent") { val commitMetadata = createCommitMetadata( version = updateVersionNonZero, readPandMOpt = Optional.of(new KernelTuple2(protocol12, basicPartitionedMetadata))) assert(commitMetadata.getEffectiveMetadata == basicPartitionedMetadata) } // ========== CommitType Tests START ========== case class CommitTypeTestCase( readPandMOpt: Optional[KernelTuple2[Protocol, Metadata]] = Optional.empty(), newProtocolOpt: Optional[Protocol] = Optional.empty(), newMetadataOpt: Optional[Metadata] = Optional.empty(), expectedCommitType: CommitType) private val commitTypeTestCases = Seq( CommitTypeTestCase( readPandMOpt = Optional.empty(), // No read state for table creation newProtocolOpt = Optional.of(protocol12), newMetadataOpt = Optional.of(basicPartitionedMetadata), expectedCommitType = CommitType.FILESYSTEM_CREATE), CommitTypeTestCase( readPandMOpt = Optional.empty(), // No read state for table creation newProtocolOpt = Optional.of(protocolWithCatalogManagedSupport), newMetadataOpt = Optional.of(basicPartitionedMetadata), expectedCommitType = CommitType.CATALOG_CREATE), CommitTypeTestCase( readPandMOpt = Optional.of(new KernelTuple2(protocol12, basicPartitionedMetadata)), expectedCommitType = CommitType.FILESYSTEM_WRITE), CommitTypeTestCase( readPandMOpt = Optional.of( new KernelTuple2(protocolWithCatalogManagedSupport, basicPartitionedMetadata)), expectedCommitType = CommitType.CATALOG_WRITE), CommitTypeTestCase( readPandMOpt = Optional.of(new KernelTuple2(protocol12, basicPartitionedMetadata)), newProtocolOpt = Optional.of(protocolWithCatalogManagedSupport), expectedCommitType = CommitType.FILESYSTEM_UPGRADE_TO_CATALOG), CommitTypeTestCase( readPandMOpt = Optional.of( new KernelTuple2(protocolWithCatalogManagedSupport, basicPartitionedMetadata)), newProtocolOpt = Optional.of(protocol12), expectedCommitType = CommitType.CATALOG_DOWNGRADE_TO_FILESYSTEM)) commitTypeTestCases.foreach { testCase => test(s"getCommitType returns ${testCase.expectedCommitType}") { // version > 0 for writes, version 0 for create val version = if (testCase.readPandMOpt.isPresent) 1L else 0L val commitMetadata = createCommitMetadata( version = version, logPath = logPath, readPandMOpt = testCase.readPandMOpt, newProtocolOpt = testCase.newProtocolOpt, newMetadataOpt = testCase.newMetadataOpt) assert(commitMetadata.getCommitType == testCase.expectedCommitType) } } // ========== CommitType Tests END ========== test("checkReadStateAbsentIfAndOnlyIfVersion0 - version 0 with absent readState should pass") { // This should pass: version 0 (table creation) with absent readPandMOpt createCommitMetadata( version = createVersion0, newProtocolOpt = Optional.of(protocol12), newMetadataOpt = Optional.of(basicPartitionedMetadata)) } test("checkReadStateAbsentIfAndOnlyIfVersion0 - version 0 with present readState should fail") { // This should fail: version 0 (table creation) with present readPandMOpt val exMsg = intercept[IllegalArgumentException] { createCommitMetadata( version = createVersion0, readPandMOpt = Optional.of(new KernelTuple2(protocol12, basicPartitionedMetadata))) }.getMessage assert(exMsg.contains("Table creation (version 0) requires absent readPandMOpt")) } test("checkReadStateAbsentIfAndOnlyIfVersion0 - version > 0 with present readState should pass") { // This should pass: version > 0 (existing table) with present readPandMOpt createCommitMetadata( version = updateVersionNonZero, readPandMOpt = Optional.of(new KernelTuple2(protocol12, basicPartitionedMetadata))) } test("checkReadStateAbsentIfAndOnlyIfVersion0 - version > 0 with absent readState should fail") { // This should fail: version > 0 (existing table) with absent readPandMOpt val exMsg = intercept[IllegalArgumentException] { createCommitMetadata( version = updateVersionNonZero, newProtocolOpt = Optional.of(protocol12), newMetadataOpt = Optional.of(basicPartitionedMetadata)) }.getMessage assert(exMsg.contains("existing table writes (version > 0) require present readPandMOpt")) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/commit/DefaultCommitterSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.commit import java.io.IOException import java.nio.file.FileAlreadyExistsException import java.util.Optional import io.delta.kernel.TableManager import io.delta.kernel.commit.{CommitFailedException, CommitMetadata} import io.delta.kernel.data.Row import io.delta.kernel.exceptions.KernelEngineException import io.delta.kernel.internal.actions.Protocol import io.delta.kernel.internal.table.SnapshotBuilderImpl import io.delta.kernel.internal.tablefeatures.TableFeatures import io.delta.kernel.internal.util.{FileNames, Tuple2 => KernelTuple2} import io.delta.kernel.test.{ActionUtils, BaseMockFileSystemClient, BaseMockJsonHandler, MockFileSystemClientUtils, TestFixtures, VectorTestUtils} import io.delta.kernel.types.{IntegerType, StructType} import io.delta.kernel.utils.{CloseableIterator, FileStatus} import org.scalatest.funsuite.AnyFunSuite class DefaultCommitterSuite extends AnyFunSuite with MockFileSystemClientUtils with TestFixtures with VectorTestUtils { private val protocol12 = new Protocol(1, 2) private val basicFileSystemCommitMetadataNoPMChange = createCommitMetadata( version = 1L, commitInfo = testCommitInfo(ictEnabled = false), readPandMOpt = Optional.of(new KernelTuple2(protocol12, basicPartitionedMetadata))) Seq( (protocol12, protocolWithCatalogManagedSupport, "Upgrade"), (protocolWithCatalogManagedSupport, protocol12, "Downgrade"), ( protocolWithCatalogManagedSupport, protocolWithCatalogManagedSupport, "CatalogManagedWrite")).foreach { case (readProtocol, newProtocol, testCase) => test(s"default committer does not support committing to catalog-managed tables -- $testCase") { val emptyMockEngine = createMockFSListFromEngine(Nil) val schema = new StructType().add("col1", IntegerType.INTEGER) val metadata = testMetadata(schema, Seq[String]()) val committer = TableManager.loadSnapshot(dataPath.toString) .asInstanceOf[SnapshotBuilderImpl] .withProtocolAndMetadata(readProtocol, metadata) .atVersion(1) .withMaxCatalogVersionIfApplicable( readProtocol.supportsFeature(TableFeatures.CATALOG_MANAGED_RW_FEATURE), 1).build(emptyMockEngine) .getCommitter assert(committer.isInstanceOf[DefaultFileSystemManagedTableOnlyCommitter]) val exMsg = intercept[UnsupportedOperationException] { committer.commit( emptyMockEngine, emptyActionsIterator, createCommitMetadata( version = 3L, readPandMOpt = Optional.of(new KernelTuple2(readProtocol, metadata)), newProtocolOpt = Optional.of(newProtocol), newMetadataOpt = Optional.of(metadata))) }.getMessage assert(exMsg.contains("No io.delta.kernel.commit.Committer has been provided to Kernel, so " + "Kernel is using a default Committer that only supports committing to " + "filesystem-managed Delta tables, not catalog-managed Delta tables. Since this table " + "is catalog-managed, this commit operation is unsupported")) } } //////////////////////////////////////////////////////// // DefaultCommitter exception handling tests -- START // //////////////////////////////////////////////////////// case class ExceptionTestCase( description: String, exceptionToThrow: Exception, expectedRetryableOpt: Option[Boolean], expectedConflictOpt: Option[Boolean], expectedThrownType: Class[_], expectedCauseType: Class[_]) private val exceptionTestCases = Seq( ExceptionTestCase( description = "FileAlreadyExistsException -> CFE(true, true)", exceptionToThrow = new FileAlreadyExistsException("_delta_log/001.json"), expectedRetryableOpt = Some(true), expectedConflictOpt = Some(true), expectedThrownType = classOf[CommitFailedException], expectedCauseType = classOf[FileAlreadyExistsException]), ExceptionTestCase( description = "IOException -> CFE(true, false)", exceptionToThrow = new IOException("Network timeout writing to _delta_log/001.json"), expectedRetryableOpt = Some(true), expectedConflictOpt = Some(false), expectedThrownType = classOf[CommitFailedException], expectedCauseType = classOf[IOException]), ExceptionTestCase( description = "RuntimeException wrapped and thrown as KernelEngineException", exceptionToThrow = new RuntimeException("Some runtime error"), expectedRetryableOpt = None, expectedConflictOpt = None, expectedThrownType = classOf[KernelEngineException], expectedCauseType = classOf[RuntimeException])) exceptionTestCases.foreach { testCase => test(s"default committer handles ${testCase.description} correctly") { val throwingEngine = mockEngine(jsonHandler = new BaseMockJsonHandler { override def writeJsonFileAtomically( filePath: String, data: CloseableIterator[Row], overwrite: Boolean): Unit = { throw testCase.exceptionToThrow } }) val ex = intercept[Exception] { DefaultFileSystemManagedTableOnlyCommitter.INSTANCE.commit( throwingEngine, emptyActionsIterator, basicFileSystemCommitMetadataNoPMChange) } assert(ex.getClass == testCase.expectedThrownType) assert(ex.getCause.getClass == testCase.expectedCauseType) testCase.expectedRetryableOpt.foreach { expectedRetryable => assert(ex.isInstanceOf[CommitFailedException]) val commitEx = ex.asInstanceOf[CommitFailedException] assert(commitEx.isRetryable == expectedRetryable) } testCase.expectedConflictOpt.foreach { expectedConflict => assert(ex.isInstanceOf[CommitFailedException]) val commitEx = ex.asInstanceOf[CommitFailedException] assert(commitEx.isConflict == expectedConflict) } } } ////////////////////////////////////////////////////// // DefaultCommitter exception handling tests -- END // ////////////////////////////////////////////////////// test("success commit returns ParsedLogData containing FileStatus for that commit file") { var writtenFileStatus = Option.empty[FileStatus] val fakeWriteReadJsonEngine = mockEngine( jsonHandler = new BaseMockJsonHandler { override def writeJsonFileAtomically( filePath: String, data: CloseableIterator[Row], overwrite: Boolean): Unit = { writtenFileStatus = Some(FileStatus.of(filePath, 1234L, 4567L)) // (path, size, modTime) } }, fileSystemClient = new BaseMockFileSystemClient { override def getFileStatus(path: String): FileStatus = writtenFileStatus.get }) val commitResult = DefaultFileSystemManagedTableOnlyCommitter.INSTANCE.commit( fakeWriteReadJsonEngine, emptyActionsIterator, basicFileSystemCommitMetadataNoPMChange) val commit = commitResult.getCommitLogData assert(commit.isFile) assert(commit.getVersion === basicFileSystemCommitMetadataNoPMChange.getVersion) assert(commit.getFileStatus === writtenFileStatus.get) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/commit/PublishMetadataSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.commit import scala.collection.JavaConverters._ import io.delta.kernel.commit.PublishMetadata import io.delta.kernel.test.TestFixtures import org.scalatest.funsuite.AnyFunSuite class PublishMetadataSuite extends AnyFunSuite with TestFixtures { private val logPath = "/fake/_delta_log" //////////////////// // Negative Tests // //////////////////// test("constructor validates null logPath") { val commits = List(createStagedCatalogCommit(1)).asJava intercept[NullPointerException] { new PublishMetadata(1, null, commits) } } test("constructor validates null ascendingCatalogCommits") { intercept[NullPointerException] { new PublishMetadata(1, logPath, null) } } test("constructor validates non-empty commits") { val ex = intercept[IllegalArgumentException] { new PublishMetadata(1, logPath, List.empty.asJava) } assert(ex.getMessage.contains("ascendingCatalogCommits must be non-empty")) } test("constructor validates contiguous commits - gap in sequence") { val commits = List( createStagedCatalogCommit(1), createStagedCatalogCommit(2), createStagedCatalogCommit(4) // Gap: missing version 3 ).asJava val ex = intercept[IllegalArgumentException] { new PublishMetadata(4, logPath, commits) } assert(ex.getMessage.contains("must be sorted and contiguous")) } test("constructor validates sorted commits - out of order") { val commits = List( createStagedCatalogCommit(2), createStagedCatalogCommit(1), // Out of order createStagedCatalogCommit(3)).asJava val ex = intercept[IllegalArgumentException] { new PublishMetadata(3, logPath, commits) } assert(ex.getMessage.contains("must be sorted and contiguous")) } test("constructor validates last commit matches snapshot version") { val commits = List( createStagedCatalogCommit(1), createStagedCatalogCommit(2), createStagedCatalogCommit(3)).asJava val ex = intercept[IllegalArgumentException] { new PublishMetadata(5, logPath, commits) // Snapshot is 5, but last commit is 3 } assert(ex.getMessage.contains("Last catalog commit version 3 must equal snapshot version 5")) } //////////////////// // Positive Tests // //////////////////// test("valid construction with single commit") { val commits = List(createStagedCatalogCommit(5)).asJava val publishMetadata = new PublishMetadata(5, logPath, commits) assert(publishMetadata.getSnapshotVersion == 5) assert(publishMetadata.getLogPath == logPath) assert(publishMetadata.getAscendingCatalogCommits == commits) } test("valid construction with multiple contiguous commits") { val commits = List( createStagedCatalogCommit(3), createStagedCatalogCommit(4), createStagedCatalogCommit(5)).asJava val publishMetadata = new PublishMetadata(5, logPath, commits) assert(publishMetadata.getSnapshotVersion == 5) assert(publishMetadata.getAscendingCatalogCommits.size() == 3) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/files/LogDataUtilsSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.files import scala.collection.JavaConverters._ import io.delta.kernel.test.{MockFileSystemClientUtils, VectorTestUtils} import org.scalatest.funsuite.AnyFunSuite class LogDataUtilsSuite extends AnyFunSuite with MockFileSystemClientUtils with VectorTestUtils { private val emptyInlineData = emptyColumnarBatch ////////////////////////////////////////////////////// // validateLogDataContainsOnlyRatifiedStagedCommits // ////////////////////////////////////////////////////// test("validateLogDataContainsOnlyRatifiedStagedCommits: empty list passes") { // Should not throw any exception LogDataUtils.validateLogDataContainsOnlyRatifiedStagedCommits(Seq.empty.asJava) } test("validateLogDataContainsOnlyRatifiedStagedCommits: valid list passes") { val logDatas = Seq( ParsedCatalogCommitData.forFileStatus(stagedCommitFile(1)), ParsedCatalogCommitData.forFileStatus(stagedCommitFile(2))) // Should not throw any exception LogDataUtils.validateLogDataContainsOnlyRatifiedStagedCommits(logDatas.asJava) } test("validateLogDataContainsOnlyRatifiedStagedCommits: inline delta fails") { intercept[IllegalArgumentException] { LogDataUtils.validateLogDataContainsOnlyRatifiedStagedCommits( Seq(ParsedCatalogCommitData.forInlineData(3, emptyInlineData)).asJava) } } test("validateLogDataContainsOnlyRatifiedStagedCommits: published delta fails") { intercept[IllegalArgumentException] { LogDataUtils.validateLogDataContainsOnlyRatifiedStagedCommits( Seq(ParsedPublishedDeltaData.forFileStatus(deltaFileStatus(1))).asJava) } } test("validateLogDataContainsOnlyRatifiedStagedCommits: checkpoint data fails") { intercept[IllegalArgumentException] { LogDataUtils.validateLogDataContainsOnlyRatifiedStagedCommits( Seq(ParsedClassicCheckpointData.forFileStatus(classicCheckpointFileStatus(0))).asJava) } } /////////////////////////////////////// // validateLogDataIsSortedContiguous // /////////////////////////////////////// test("validateLogDataIsSortedContiguous: empty list should pass") { // Should not throw any exception LogDataUtils.validateLogDataIsSortedContiguous(Seq.empty.asJava) } test("validateLogDataIsSortedContiguous: single element should pass") { val singleElement = Seq(ParsedDeltaData.forFileStatus(deltaFileStatus(1))) // Should not throw any exception LogDataUtils.validateLogDataIsSortedContiguous(singleElement.asJava) } test("validateLogDataIsSortedContiguous: contiguous versions should pass") { val contiguousData = Seq( ParsedDeltaData.forFileStatus(deltaFileStatus(1)), ParsedDeltaData.forFileStatus(deltaFileStatus(2)), ParsedDeltaData.forFileStatus(deltaFileStatus(3))) // Should not throw any exception LogDataUtils.validateLogDataIsSortedContiguous(contiguousData.asJava) } test("validateLogDataIsSortedContiguous: non-contiguous versions should fail") { val nonContiguousData = Seq( ParsedDeltaData.forFileStatus(deltaFileStatus(1)), ParsedDeltaData.forFileStatus(deltaFileStatus(3)) // Missing version 2 ) intercept[IllegalArgumentException] { LogDataUtils.validateLogDataIsSortedContiguous(nonContiguousData.asJava) } } test("validateLogDataIsSortedContiguous: unsorted versions should fail") { val unsortedData = Seq( ParsedDeltaData.forFileStatus(deltaFileStatus(2)), ParsedDeltaData.forFileStatus(deltaFileStatus(1))) intercept[IllegalArgumentException] { LogDataUtils.validateLogDataIsSortedContiguous(unsortedData.asJava) } } test("validateLogDataIsSortedContiguous: duplicate versions should fail") { val duplicateData = Seq( ParsedDeltaData.forFileStatus(deltaFileStatus(1)), ParsedDeltaData.forFileStatus(deltaFileStatus(1))) intercept[IllegalArgumentException] { LogDataUtils.validateLogDataIsSortedContiguous(duplicateData.asJava) } } test("validateLogDataIsSortedContiguous: mixed log data types should work") { val mixedData = Seq( ParsedDeltaData.forFileStatus(deltaFileStatus(1)), ParsedLogData.forFileStatus(checksumFileStatus(2)), ParsedLogData.forFileStatus(classicCheckpointFileStatus(3))) // Should not throw any exception LogDataUtils.validateLogDataIsSortedContiguous(mixedData.asJava) } ////////////////////////////////////////////////////////// // combinePublishedAndRatifiedDeltasWithCatalogPriority // ////////////////////////////////////////////////////////// test("combinePublishedAndRatifiedDeltasWithCatalogPriority: empty published, empty ratified") { val result = LogDataUtils.combinePublishedAndRatifiedDeltasWithCatalogPriority( Seq.empty.asJava, Seq.empty.asJava) assert(result.isEmpty) } test( "combinePublishedAndRatifiedDeltasWithCatalogPriority: empty published, non-empty ratified") { val ratifiedDeltas = Seq( ParsedDeltaData.forFileStatus(stagedCommitFile(1)), ParsedDeltaData.forFileStatus(stagedCommitFile(2))) val result = LogDataUtils.combinePublishedAndRatifiedDeltasWithCatalogPriority( Seq.empty.asJava, ratifiedDeltas.asJava) assert(result.asScala === ratifiedDeltas) } test( "combinePublishedAndRatifiedDeltasWithCatalogPriority: non-empty published, empty ratified") { val publishedDeltas = Seq( ParsedDeltaData.forFileStatus(deltaFileStatus(1)), ParsedDeltaData.forFileStatus(deltaFileStatus(2))) val result = LogDataUtils.combinePublishedAndRatifiedDeltasWithCatalogPriority( publishedDeltas.asJava, Seq.empty.asJava) assert(result.asScala === publishedDeltas) } test("combinePublishedAndRatifiedDeltasWithCatalogPriority: non-overlapping ranges") { val publishedDeltas = Seq( ParsedDeltaData.forFileStatus(deltaFileStatus(1)), ParsedDeltaData.forFileStatus(deltaFileStatus(2))) val ratifiedDeltas = Seq( ParsedDeltaData.forFileStatus(stagedCommitFile(3)), ParsedDeltaData.forFileStatus(stagedCommitFile(4))) val result = LogDataUtils.combinePublishedAndRatifiedDeltasWithCatalogPriority( publishedDeltas.asJava, ratifiedDeltas.asJava) assert(result.asScala === publishedDeltas ++ ratifiedDeltas) } test( "combinePublishedAndRatifiedDeltasWithCatalogPriority: " + "overlapping ranges - ratified priority") { val publishedDeltas = Seq( ParsedDeltaData.forFileStatus(deltaFileStatus(1)), ParsedDeltaData.forFileStatus(deltaFileStatus(2))) val ratifiedDeltas = Seq( ParsedDeltaData.forFileStatus(stagedCommitFile(2)), ParsedDeltaData.forFileStatus(stagedCommitFile(3))) val result = LogDataUtils.combinePublishedAndRatifiedDeltasWithCatalogPriority( publishedDeltas.asJava, ratifiedDeltas.asJava) val expected = Seq(publishedDeltas.head) ++ ratifiedDeltas assert(result.asScala === expected) } test("combinePublishedAndRatifiedDeltasWithCatalogPriority: ratified in middle of published") { val publishedDeltas = Seq( ParsedDeltaData.forFileStatus(deltaFileStatus(1)), ParsedDeltaData.forFileStatus(deltaFileStatus(2)), ParsedDeltaData.forFileStatus(deltaFileStatus(3)), ParsedDeltaData.forFileStatus(deltaFileStatus(4))) val ratifiedDeltas = Seq( ParsedDeltaData.forFileStatus(stagedCommitFile(2)), ParsedDeltaData.forFileStatus(stagedCommitFile(3))) val result = LogDataUtils.combinePublishedAndRatifiedDeltasWithCatalogPriority( publishedDeltas.asJava, ratifiedDeltas.asJava) val expected = Seq(publishedDeltas.head) ++ ratifiedDeltas ++ Seq(publishedDeltas(3)) assert(result.asScala === expected) } test("combinePublishedAndRatifiedDeltasWithCatalogPriority: single ratified version") { val publishedDeltas = Seq( ParsedDeltaData.forFileStatus(deltaFileStatus(1)), ParsedDeltaData.forFileStatus(deltaFileStatus(2)), ParsedDeltaData.forFileStatus(deltaFileStatus(3))) val ratifiedDeltas = Seq( ParsedDeltaData.forFileStatus(stagedCommitFile(3))) val result = LogDataUtils.combinePublishedAndRatifiedDeltasWithCatalogPriority( publishedDeltas.asJava, ratifiedDeltas.asJava) val expected = Seq(publishedDeltas(0), publishedDeltas(1)) ++ ratifiedDeltas assert(result.asScala === expected) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/files/ParsedLogDataSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.files import io.delta.kernel.internal.util.FileNames import io.delta.kernel.test.{MockFileSystemClientUtils, VectorTestUtils} import io.delta.kernel.utils.FileStatus import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.must.Matchers.be import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper class ParsedLogDataSuite extends AnyFunSuite with MockFileSystemClientUtils with VectorTestUtils { private val emptyInlineData = emptyColumnarBatch ///////////// // General // ///////////// test("ParsedLogData throws on unknown log file") { val fileStatus = FileStatus.of("unknown", 0, 0) val exMsg = intercept[IllegalArgumentException] { ParsedLogData.forFileStatus(fileStatus) }.getMessage assert(exMsg.contains("Unknown log file type")) } test("ParsedLogData (super) throws on version < 0") { val exMsg = intercept[IllegalArgumentException] { ParsedCatalogCommitData.forInlineData(-1, emptyInlineData) }.getMessage assert(exMsg === "version must be non-negative") } test("ParsedLogData: different types are not equal") { val delta = ParsedLogData.forFileStatus(deltaFileStatus(5)) val checksum = ParsedLogData.forFileStatus(checksumFileStatus(5)) val checkpoint = ParsedLogData.forFileStatus(classicCheckpointFileStatus(5)) val logCompaction = ParsedLogData.forFileStatus(logCompactionStatus(5, 10)) assert(delta != checksum) assert(delta != checkpoint) assert(delta != logCompaction) assert(checksum != checkpoint) assert(checksum != logCompaction) assert(checkpoint != logCompaction) } ////////////////////////////// // ParsedPublishedDeltaData // ////////////////////////////// test("ParsedLogData.forFileStatus(publishedDelta) creates a ParsedPublishedDeltaData") { val fileStatus = deltaFileStatus(5) val parsed = ParsedLogData.forFileStatus(fileStatus) assert(parsed.isInstanceOf[ParsedPublishedDeltaData]) } test("ParsedPublishedDeltaData: correctly parses published delta file") { val fileStatus = deltaFileStatus(5) val parsed = ParsedPublishedDeltaData.forFileStatus(fileStatus) assert(parsed.isInstanceOf[ParsedDeltaData]) assert(parsed.getVersion == 5) assert(parsed.isFile) assert(!parsed.isInline) assert(parsed.getFileStatus == fileStatus) } test("ParsedPublishedDeltaData: throws on staged commit file") { val fileStatus = stagedCommitFile(5) val exMsg = intercept[IllegalArgumentException] { ParsedPublishedDeltaData.forFileStatus(fileStatus) }.getMessage assert(exMsg.contains("Expected a published Delta file but got")) } test("ParsedDeltaData: equality") { val fileStatus1 = deltaFileStatus(5) val fileStatus2 = deltaFileStatus(5) val fileStatus3 = deltaFileStatus(6) val delta1 = ParsedPublishedDeltaData.forFileStatus(fileStatus1) val delta2 = ParsedPublishedDeltaData.forFileStatus(fileStatus2) val delta3 = ParsedPublishedDeltaData.forFileStatus(fileStatus3) assert(delta1 == delta1) assert(delta1 == delta2) assert(delta1 != delta3) } ///////////////////////////// // ParsedCatalogCommitData // ///////////////////////////// test("ParsedLogData.forFileStatus(stagedCommit) creates a ParsedCatalogCommitData") { val fileStatus = stagedCommitFile(5) val parsed = ParsedLogData.forFileStatus(fileStatus) assert(parsed.isInstanceOf[ParsedCatalogCommitData]) } test("ParsedCatalogCommitData: correctly parses staged commit file") { val fileStatus = stagedCommitFile(5) val parsed = ParsedCatalogCommitData.forFileStatus(fileStatus) assert(parsed.isInstanceOf[ParsedDeltaData]) assert(parsed.getVersion == 5) assert(parsed.isFile) assert(!parsed.isInline) assert(parsed.getFileStatus == fileStatus) } test("ParsedCatalogCommitData: can construct inline data") { val parsed = ParsedCatalogCommitData.forInlineData(10, emptyInlineData) assert(parsed.getVersion == 10) assert(parsed.isInline) assert(!parsed.isFile) assert(parsed.getInlineData == emptyInlineData) } test("ParsedCatalogCommitData: throws on published delta file") { val fileStatus = deltaFileStatus(5) val exMsg = intercept[IllegalArgumentException] { ParsedCatalogCommitData.forFileStatus(fileStatus) }.getMessage assert(exMsg.contains("Expected a staged commit file but got")) } test("ParsedCatalogCommitData: equality") { val fileStatus1 = stagedCommitFile(5) val fileStatus3 = stagedCommitFile(6) val catalogCommit1 = ParsedCatalogCommitData.forFileStatus(fileStatus1) val catalogCommit2 = ParsedCatalogCommitData.forFileStatus(fileStatus1) val catalogCommit3 = ParsedCatalogCommitData.forFileStatus(fileStatus3) assert(catalogCommit1 == catalogCommit1) assert(catalogCommit1 == catalogCommit2) assert(catalogCommit1 != catalogCommit3) } ////////////////////////// // ParsedCheckpointData // ////////////////////////// test("ParsedClassicCheckpointData: throws on non-classic checkpoint file") { val fileStatus = deltaFileStatus(5) val exMsg = intercept[IllegalArgumentException] { ParsedClassicCheckpointData.forFileStatus(fileStatus) }.getMessage assert(exMsg.contains("Expected a classic checkpoint file but got")) } test("ParsedClassicCheckpointData: correctly parses classic checkpoint file") { val fileStatus = classicCheckpointFileStatus(10) val parsed = ParsedLogData.forFileStatus(fileStatus) assert(parsed.isInstanceOf[ParsedClassicCheckpointData]) assert(parsed.getVersion == 10) assert(parsed.getGroupByCategoryClass == classOf[ParsedCheckpointData]) assert(parsed.isFile) assert(!parsed.isInline) assert(parsed.getFileStatus == fileStatus) } test("ParsedClassicCheckpointData: equality") { val cp1 = ParsedLogData.forFileStatus(classicCheckpointFileStatus(10)) val cp2 = ParsedLogData.forFileStatus(classicCheckpointFileStatus(10)) val cp3 = ParsedLogData.forFileStatus(classicCheckpointFileStatus(11)) assert(cp1 == cp1) assert(cp1 == cp2) assert(cp1 != cp3) } test("ParsedV2CheckpointData: throws on non-V2 checkpoint file") { val fileStatus = deltaFileStatus(5) val exMsg = intercept[IllegalArgumentException] { ParsedV2CheckpointData.forFileStatus(fileStatus) }.getMessage assert(exMsg.contains("Expected a V2 checkpoint file but got")) } test("ParsedV2CheckpointData: correctly parses V2 checkpoint file") { val fileStatus = v2CheckpointFileStatus(20) val parsed = ParsedLogData.forFileStatus(fileStatus) assert(parsed.isInstanceOf[ParsedCheckpointData]) assert(parsed.getVersion == 20) assert(parsed.getGroupByCategoryClass == classOf[ParsedCheckpointData]) assert(parsed.isFile) assert(!parsed.isInline) assert(parsed.getFileStatus == fileStatus) } test("ParsedV2CheckpointData: equality") { val parsed1 = ParsedLogData.forFileStatus(v2CheckpointFileStatus(20, useUUID = false)) val parsed2 = ParsedLogData.forFileStatus(v2CheckpointFileStatus(20, useUUID = false)) val parsed3 = ParsedLogData.forFileStatus(v2CheckpointFileStatus(21)) assert(parsed1 == parsed1) assert(parsed1 == parsed2) assert(parsed1 != parsed3) } /////////////////////////////////// // ParsedMultiPartCheckpointData // /////////////////////////////////// test("ParsedMultiPartCheckpointData: throws on non-multi-part checkpoint file") { val fileStatus = deltaFileStatus(5) val exMsg = intercept[IllegalArgumentException] { ParsedMultiPartCheckpointData.forFileStatus(fileStatus) }.getMessage assert(exMsg.contains("Expected a multi-part checkpoint file but got")) } test("ParsedMultiPartCheckpointData: correctly parses multi-part checkpoint file") { val chkpt_15_1_3 = multiPartCheckpointFileStatus(15, 1, 3) val parsed = ParsedLogData.forFileStatus(chkpt_15_1_3) assert(parsed.isInstanceOf[ParsedMultiPartCheckpointData]) assert(parsed.getVersion == 15) assert(parsed.getGroupByCategoryClass == classOf[ParsedCheckpointData]) assert(parsed.isFile) assert(!parsed.isInline) assert(parsed.getFileStatus == chkpt_15_1_3) val casted = parsed.asInstanceOf[ParsedMultiPartCheckpointData] assert(casted.part == 1) assert(casted.numParts == 3) } test("ParsedMultiPartCheckpointData: throws on part > numParts") { val path = FileNames.multiPartCheckpointFile(logPath, 10, 5, 3) // part = 5, numParts = 3 val exMsg = intercept[IllegalArgumentException] { ParsedLogData.forFileStatus(FileStatus.of(path.toString)) }.getMessage assert(exMsg === "part must be between 1 and numParts") } test("ParsedMultiPartCheckpointData: throws on numParts = 0") { val path = FileNames.multiPartCheckpointFile(logPath, 10, 0, 0) // part = 0, numParts = 0 val exMsg = intercept[IllegalArgumentException] { ParsedLogData.forFileStatus(FileStatus.of(path.toString)) }.getMessage assert(exMsg === "numParts must be greater than 0") } test("ParsedMultiPartCheckpointData: throws on part = 0") { val path = FileNames.multiPartCheckpointFile(logPath, 10, 0, 3) // part = 0, numParts = 3 val exMsg = intercept[IllegalArgumentException] { ParsedLogData.forFileStatus(FileStatus.of(path.toString)) }.getMessage assert(exMsg === "part must be between 1 and numParts") } test("ParsedMultiPartCheckpointData: equality") { val parsed_15_1_3_a = ParsedLogData.forFileStatus(multiPartCheckpointFileStatus(15, 1, 3)) val parsed_15_1_3_b = ParsedLogData.forFileStatus(multiPartCheckpointFileStatus(15, 1, 3)) val parsed_15_2_3 = ParsedLogData.forFileStatus(multiPartCheckpointFileStatus(15, 2, 3)) val parsed_15_1_4 = ParsedLogData.forFileStatus(multiPartCheckpointFileStatus(15, 1, 4)) val parsed_16_1_3 = ParsedLogData.forFileStatus(multiPartCheckpointFileStatus(16, 1, 3)) assert(parsed_15_1_3_a == parsed_15_1_3_a) assert(parsed_15_1_3_a == parsed_15_1_3_b) assert(parsed_15_1_3_a != parsed_15_2_3) assert(parsed_15_1_3_a != parsed_15_1_4) assert(parsed_15_1_3_a != parsed_16_1_3) } ///////////////////////// // Checkpoint ordering // ///////////////////////// test("checkpoint ordering") { // _m means materialized, _i means inline val classic_12_m: ParsedCheckpointData = ParsedClassicCheckpointData.forFileStatus(classicCheckpointFileStatus(12)) val multi_11_3_m: ParsedCheckpointData = ParsedMultiPartCheckpointData.forFileStatus(multiPartCheckpointFileStatus(11, 1, 3)) val v2_10_m: ParsedCheckpointData = ParsedV2CheckpointData.forFileStatus(v2CheckpointFileStatus(10)) val multi_12_3_m: ParsedCheckpointData = ParsedMultiPartCheckpointData.forFileStatus(multiPartCheckpointFileStatus(12, 1, 3)) val v2_12_m: ParsedCheckpointData = ParsedV2CheckpointData.forFileStatus(v2CheckpointFileStatus(12)) val multi_12_4_m: ParsedCheckpointData = ParsedMultiPartCheckpointData.forFileStatus(multiPartCheckpointFileStatus(12, 1, 4)) val v2_aaa: ParsedCheckpointData = ParsedV2CheckpointData.forFileStatus( FileStatus.of(FileNames.topLevelV2CheckpointFile(logPath, 10, "aaa", "json").toString)) val v2_bbb: ParsedCheckpointData = ParsedV2CheckpointData.forFileStatus( FileStatus.of(FileNames.topLevelV2CheckpointFile(logPath, 10, "bbb", "json").toString)) // Case 1: Version priority classic_12_m should be > multi_11_3_m classic_12_m should be > v2_10_m multi_11_3_m should be > v2_10_m // Case 2: Type priority, when version is tied v2_12_m should be > classic_12_m v2_12_m should be > multi_12_3_m // Case 3: Inline priority, when version and type are tied (and parts are tied for multi) // TODO: Test this when we allow creating checkpoints with inline data // Case 4: Multi-part checkpoint with more parts has higher priority multi_12_4_m should be > multi_12_3_m // Case 5: For tied v2, filepath is tie-breaker v2_bbb should be > v2_aaa } //////////////////////////// // ParsedLogCompactionData // //////////////////////////// test("ParsedLogCompactionData: throws on non-log-compaction file") { val fileStatus = deltaFileStatus(5) val exMsg = intercept[IllegalArgumentException] { ParsedLogCompactionData.forFileStatus(fileStatus) }.getMessage assert(exMsg.contains("Expected a log compaction file but got")) } test("ParsedLogCompactionData: correctly parses log compaction file") { val fileStatus = logCompactionStatus(25, 30) val parsed = ParsedLogData.forFileStatus(fileStatus) assert(parsed.isInstanceOf[ParsedLogCompactionData]) assert(parsed.getVersion == 30) assert(parsed.getGroupByCategoryClass == classOf[ParsedLogCompactionData]) assert(parsed.isFile) assert(!parsed.isInline) assert(parsed.getFileStatus == fileStatus) val casted = parsed.asInstanceOf[ParsedLogCompactionData] assert(casted.startVersion == 25) assert(casted.endVersion == 30) } test("ParsedLogCompactionData: throws on startVersion > endVersion") { val exMsg = intercept[IllegalArgumentException] { val invalidFilePath = "00000000000000000003.00000000000000000001.compacted.json" ParsedLogCompactionData.forFileStatus(FileStatus.of(invalidFilePath)) }.getMessage assert(exMsg === "startVersion must be less than endVersion") } test("ParsedLogCompactionData: equality") { val fileStatus1 = logCompactionStatus(25, 30) val fileStatus2 = logCompactionStatus(25, 30) val fileStatus3 = logCompactionStatus(31, 32) val parsed1 = ParsedLogData.forFileStatus(fileStatus1) val parsed2 = ParsedLogData.forFileStatus(fileStatus2) val parsed3 = ParsedLogData.forFileStatus(fileStatus3) assert(parsed1 == parsed1) assert(parsed1 == parsed2) assert(parsed1 != parsed3) } //////////////////////// // ParsedChecksumData // //////////////////////// test("ParsedChecksumData: throws on non-checksum file") { val fileStatus = deltaFileStatus(5) val exMsg = intercept[IllegalArgumentException] { ParsedChecksumData.forFileStatus(fileStatus) }.getMessage assert(exMsg.contains("Expected a checksum file but got")) } test("ParsedChecksumData: correctly parses checksum file") { val fileStatus = checksumFileStatus(5) val parsed = ParsedLogData.forFileStatus(fileStatus) assert(parsed.isInstanceOf[ParsedChecksumData]) assert(parsed.getVersion == 5) assert(parsed.getGroupByCategoryClass == classOf[ParsedChecksumData]) assert(parsed.getFileStatus == fileStatus) } test("ParsedChecksumData: equality") { val fileStatus1 = checksumFileStatus(5) val fileStatus2 = checksumFileStatus(5) val fileStatus3 = checksumFileStatus(6) val parsed1 = ParsedLogData.forFileStatus(fileStatus1) val parsed2 = ParsedLogData.forFileStatus(fileStatus2) val parsed3 = ParsedLogData.forFileStatus(fileStatus3) assert(parsed1 == parsed1) assert(parsed1 == parsed2) assert(parsed1 != parsed3) } ////////////// // toString // ////////////// // scalastyle:off line.size.limit test("ParsedPublishedDeltaData: toString") { val parsed = ParsedPublishedDeltaData.forFileStatus(deltaFileStatus(5)) val expected = "ParsedPublishedDeltaData{version=5, source=FileStatus{path='/fake/path/to/table/_delta_log/00000000000000000005.json', size=5, modificationTime=50}}" assert(parsed.toString === expected) } test("ParsedCatalogCommitData: toString") { val parsed = ParsedCatalogCommitData.forFileStatus(stagedCommitFile(5)) val expectedPattern = """ParsedCatalogCommitData\{version=5, source=FileStatus\{path='/fake/path/to/table/_delta_log/_staged_commits/00000000000000000005\.[^']+\.json', size=5, modificationTime=50\}\}""".r assert(expectedPattern.findFirstIn(parsed.toString).isDefined) } test("ParsedLogCompactionData: toString") { val fileStatus = logCompactionStatus(10, 20) val parsed = ParsedLogCompactionData.forFileStatus(fileStatus) val expected = "ParsedLogCompactionData{version=20, source=FileStatus{path='/fake/path/to/table/_delta_log/00000000000000000010.00000000000000000020.compacted.json', size=10, modificationTime=100}, startVersion=10}" assert(parsed.toString === expected) } test("ParsedChecksumData: toString") { val parsed = ParsedLogData.forFileStatus(checksumFileStatus(5)) val expected = "ParsedChecksumData{version=5, source=FileStatus{path='/fake/path/to/table/_delta_log/00000000000000000005.crc', size=10, modificationTime=10}}" assert(parsed.toString === expected) } test("ParsedClassicCheckpointData: toString") { val parsed = ParsedLogData.forFileStatus(classicCheckpointFileStatus(10)) val expected = "ParsedClassicCheckpointData{version=10, source=FileStatus{path='/fake/path/to/table/_delta_log/00000000000000000010.checkpoint.parquet', size=10, modificationTime=100}}" assert(parsed.toString === expected) } test("ParsedMultiPartCheckpointData: toString") { val fileStatus = multiPartCheckpointFileStatus(10, 1, 3) val parsed = ParsedMultiPartCheckpointData.forFileStatus(fileStatus) val expected = "ParsedMultiPartCheckpointData{version=10, source=FileStatus{path='/fake/path/to/table/_delta_log/00000000000000000010.checkpoint.0000000001.0000000003.parquet', size=10, modificationTime=100}, part=1, numParts=3}" assert(parsed.toString === expected) } test("ParsedV2CheckpointData: toString") { val parsed = ParsedLogData.forFileStatus(v2CheckpointFileStatus(20)) val expectedPattern = """ParsedV2CheckpointData\{version=20, source=FileStatus\{path='/fake/path/to/table/_delta_log/00000000000000000020\.checkpoint\.[a-f0-9-]+\.json', size=20, modificationTime=200\}\}""".r assert(expectedPattern.findFirstIn(parsed.toString).isDefined) } // scalastyle:on line.size.limit } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/fs/PathSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.fs import java.net.URI import org.scalatest.funsuite.AnyFunSuite class PathSuite extends AnyFunSuite { test("Path construction from String") { // Basic path construction val path1 = new Path("/user/data") assert(path1.toString === "/user/data") // Path with scheme val path2 = new Path("file:/user/data") assert(path2.toString === "file:/user/data") // Path with authority val path3 = new Path("hdfs://localhost:9000/user/data") assert(path3.toString === "hdfs://localhost:9000/user/data") // Relative path val path4 = new Path("relative/path") assert(path4.toString === "relative/path") // Empty path should throw exception val exception1 = intercept[IllegalArgumentException] { new Path("") } assert(exception1.getMessage.contains("empty string")) // Null path should throw exception val exception2 = intercept[IllegalArgumentException] { new Path(null: String) } assert(exception2.getMessage.contains("null string")) } test("Path construction from parent and child") { // String parent, String child val path1 = new Path("/user", "data") assert(path1.toString === "/user/data") // Path parent, String child val path2 = new Path(new Path("/user"), "data") assert(path2.toString === "/user/data") // String parent, Path child val path3 = new Path("/user", new Path("data")) assert(path3.toString === "/user/data") // Path parent, Path child val path4 = new Path(new Path("/user"), new Path("data")) assert(path4.toString === "/user/data") // Parent with trailing slash val path5 = new Path("/user/", "data") assert(path5.toString === "/user/data") // Parent is root val path6 = new Path("/", "data") assert(path6.toString === "/data") // Parent with scheme val path7 = new Path("file:/user", "data") assert(path7.toString === "file:/user/data") } test("Path construction from URI") { val uri1 = new URI("file:/user/data") val path1 = new Path(uri1) assert(path1.toString === "file:/user/data") val uri2 = new URI("hdfs", "localhost:9000", "/user/data", null, null) val path2 = new Path(uri2) assert(path2.toString === "hdfs://localhost:9000/user/data") } test("Path construction from scheme, authority, path") { val path1 = new Path("file", null, "/user/data") assert(path1.toString === "file:/user/data") val path2 = new Path("hdfs", "localhost:9000", "/user/data") assert(path2.toString === "hdfs://localhost:9000/user/data") val path3 = new Path(null, null, "/user/data") assert(path3.toString === "/user/data") // Skip the test for relative path with scheme as it's not supported } test("Path normalization") { // Remove duplicate slashes val path1 = new Path("/user//data///file") assert(path1.toString === "/user/data/file") // Remove trailing slash val path2 = new Path("/user/data/") assert(path2.toString === "/user/data") val path3 = new Path("/user/data//") assert(path3.toString === "/user/data") // Don't remove trailing slash from root val path4 = new Path("/") assert(path4.toString === "/") } test("Path.getName") { val path1 = new Path("/user/data/file.txt") assert(path1.getName === "file.txt") val path2 = new Path("/user/data/") assert(path2.getName === "data") val path3 = new Path("/") assert(path3.getName === "") val path4 = new Path("file.txt") assert(path4.getName === "file.txt") } test("Path.getParent") { val path1 = new Path("/user/data/file.txt") assert(path1.getParent.toString === "/user/data") val path2 = new Path("/user/data") assert(path2.getParent.toString === "/user") val path3 = new Path("/user") assert(path3.getParent.toString === "/") val path4 = new Path("/") assert(path4.getParent === null) val path5 = new Path("file.txt") assert(path5.getParent.toString === "") val path6 = new Path("dir/file.txt") assert(path6.getParent.toString === "dir") } test("Path.isAbsolute and Path.isUriPathAbsolute") { val path1 = new Path("/user/data") assert(path1.isAbsolute === true) assert(path1.isUriPathAbsolute === true) val path2 = new Path("user/data") assert(path2.isAbsolute === false) assert(path2.isUriPathAbsolute === false) // Skip the tests with scheme and relative paths as they cause exceptions } test("Path.isRoot") { val path1 = new Path("/") assert(path1.isRoot === true) val path2 = new Path("/user") assert(path2.isRoot === false) val path3 = new Path("file:/") assert(path3.isRoot === true) val path4 = new Path("file:/user") assert(path4.isRoot === false) } test("Path.toUri") { val path1 = new Path("/user/data") val uri1 = path1.toUri assert(uri1.getScheme === null) assert(uri1.getAuthority === null) assert(uri1.getPath === "/user/data") val path2 = new Path("file:/user/data") val uri2 = path2.toUri assert(uri2.getScheme === "file") assert(uri2.getAuthority === null) assert(uri2.getPath === "/user/data") val path3 = new Path("hdfs://localhost:9000/user/data") val uri3 = path3.toUri assert(uri3.getScheme === "hdfs") assert(uri3.getAuthority === "localhost:9000") assert(uri3.getPath === "/user/data") } test("Path equality and comparison") { val path1 = new Path("/user/data") val path2 = new Path("/user/data") val path3 = new Path("/user/other") // Test equals assert(path1 === path2) assert(path1 !== path3) // Test hashCode assert(path1.hashCode === path2.hashCode) // Test compareTo assert(path1.compareTo(path2) === 0) assert(path1.compareTo(path3) < 0) // "data" comes before "other" alphabetically assert(path3.compareTo(path1) > 0) } test("Path.getName static method") { assert(Path.getName("/user/data/file.txt") === "file.txt") assert(Path.getName("file.txt") === "file.txt") assert(Path.getName("/") === "") } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/fs/benchmarks/PathNormalizationBenchmarks.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.fs.benchmarks; import io.delta.kernel.internal.fs.Path; import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.infra.Blackhole; import java.util.concurrent.TimeUnit; /** * Benchmark to measure the performance of initializing/normalizing path objects. * *

    *
  • * 
    {@code
 * build/sbt sbt:delta> project kernel
 * sbt:delta> set fork in run := true sbt:delta>
 * sbt:delta> test:runMain \
 *   io.delta.kernel.internal.fs.benchmarks.PathNormalizationBenchmarks.
 *
 * }
    *
* * } */ @State(Scope.Benchmark) @OutputTimeUnit(TimeUnit.MICROSECONDS) @Warmup(iterations = 3) @Measurement(iterations = 5, time = 10, timeUnit = TimeUnit.SECONDS) @Fork(1) public class PathNormalizationBenchmarks { @Benchmark @BenchmarkMode(Mode.AverageTime) public void benchmarkNoNormalizationNeeded(Blackhole blackhole) throws Exception { blackhole.consume(new Path("s3://bucket-name/table-path/metadata/data/some_file.parquet")); } @Benchmark @BenchmarkMode(Mode.AverageTime) public void benchmarkNormalizationNeeded(Blackhole blackhole) throws Exception { blackhole.consume(new Path("s3://bucket-name/table-path/metadata/data//some_file.parquet")); } public static void main(String[] args) throws Exception { org.openjdk.jmh.Main.main(args); } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergCompatMetadataValidatorAndUpdaterSuiteBase.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.icebergcompat import java.util.Optional import scala.collection.JavaConverters._ import io.delta.kernel.exceptions.KernelException import io.delta.kernel.internal.actions.{Metadata, Protocol} import io.delta.kernel.internal.tablefeatures.TableFeature import io.delta.kernel.internal.tablefeatures.TableFeatures.{COLUMN_MAPPING_RW_FEATURE, DELETION_VECTORS_RW_FEATURE, ICEBERG_COMPAT_V2_W_FEATURE, TYPE_WIDENING_RW_FEATURE, TYPE_WIDENING_RW_PREVIEW_FEATURE} import io.delta.kernel.internal.util.ColumnMappingSuiteBase import io.delta.kernel.test.{TestFixtures, VectorTestUtils} import io.delta.kernel.types._ import org.scalatest.funsuite.AnyFunSuite /** * Base trait for testing Iceberg compatibility metadata validation and updates. * This trait provides common functionality and test cases * that can be used by both writer and compat test suites. */ trait IcebergCompatMetadataValidatorAndUpdaterSuiteBase extends AnyFunSuite with VectorTestUtils with ColumnMappingSuiteBase with TestFixtures { /** The version of Iceberg compatibility being tested (e.g., "V2" or "V3") */ def icebergCompatVersion: String /** When testing supported simple column types skip any types defined here */ def simpleTypesToSkip: Set[DataType] /** Get a metadata with the given schema and partCols with the desired icebergCompat enabled */ def getCompatEnabledMetadata( schema: StructType, columnMappingMode: String = "name", partCols: Seq[String] = Seq.empty): Metadata /** Get a protocol with features needed for the desired icebergCompat plus the `tableFeatures` */ def getCompatEnabledProtocol(tableFeatures: TableFeature*): Protocol /** Run the desired validate and update metadata method that triggers icebergCompat checks */ def validateAndUpdateIcebergCompatMetadata( isNewTable: Boolean, metadata: Metadata, protocol: Protocol): Optional[Metadata] /** Returns a [[Metadata]] instance with IcebergCompat feature and column mapping mode enabled */ def withIcebergCompatAndCMEnabled( schema: StructType, columnMappingMode: String, partCols: Seq[String]): Metadata /** Get the set of supported data column types */ def supportedDataColumnTypes: Set[DataType] /** Get the set of unsupported data column types */ def unsupportedDataColumnTypes: Set[DataType] /** Get the set of unsupported partition column types */ def unsupportedPartitionColumnTypes: Set[DataType] /** Whether deletion vectors are supported */ def isDeletionVectorsSupported: Boolean // Common test cases that apply to both writer and compat versions supportedDataColumnTypes.diff(simpleTypesToSkip).foreach { dataType: DataType => Seq(true, false).foreach { isNewTable => test(s"allowed data column types: $dataType, new table = $isNewTable") { val schema = new StructType().add("col", dataType) val metadata = getCompatEnabledMetadata(schema) val protocol = getCompatEnabledProtocol() validateAndUpdateIcebergCompatMetadata(isNewTable, metadata, protocol) } } } PRIMITIVE_TYPES.diff(simpleTypesToSkip).foreach { dataType: DataType => Seq(true, false).foreach { isNewTable => test(s"allowed partition column types: $dataType, new table = $isNewTable") { val schema = new StructType().add("col", dataType) val metadata = getCompatEnabledMetadata(schema, partCols = Seq("col")) val protocol = getCompatEnabledProtocol() validateAndUpdateIcebergCompatMetadata(isNewTable, metadata, protocol) } } } unsupportedDataColumnTypes.foreach { dataType: DataType => Seq(true, false).foreach { isNewTable => test(s"disallowed data column types: $dataType, new table = $isNewTable") { val schema = new StructType().add("col", dataType) val metadata = getCompatEnabledMetadata(schema) val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { validateAndUpdateIcebergCompatMetadata(isNewTable, metadata, protocol) } assert(e.getMessage.contains( s"icebergCompat$icebergCompatVersion does not support the data types: ")) } } } unsupportedPartitionColumnTypes.foreach { dataType: DataType => Seq(true, false).foreach { isNewTable => test(s"disallowed partition column types: $dataType, new table = $isNewTable") { val schema = new StructType().add("col", dataType) val metadata = getCompatEnabledMetadata(schema, partCols = Seq("col")) val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { validateAndUpdateIcebergCompatMetadata(isNewTable, metadata, protocol) } assert(e.getMessage.matches( s"icebergCompat$icebergCompatVersion does not support" + s" the data type .* for a partition column.")) } } } Seq(true, false).foreach { isNewTable => test(s"deletion vectors support behavior, isNewTable $isNewTable") { val schema = new StructType().add("col", BooleanType.BOOLEAN) val metadata = getCompatEnabledMetadata(schema) val protocol = getCompatEnabledProtocol(DELETION_VECTORS_RW_FEATURE) if (isDeletionVectorsSupported) { // Should not throw an exception validateAndUpdateIcebergCompatMetadata(isNewTable, metadata, protocol) } else { val e = intercept[KernelException] { validateAndUpdateIcebergCompatMetadata(isNewTable, metadata, protocol) } assert(e.getMessage.contains( s"Table features [deletionVectors] are incompatible " + s"with icebergCompat$icebergCompatVersion")) } } } // Compat-specific test cases test("compatible type widening is allowed") { val schema = new StructType() .add( new StructField( "intToLong", IntegerType.INTEGER, true, FieldMetadata.empty()).withTypeChanges( Seq(new TypeChange(IntegerType.INTEGER, LongType.LONG)).asJava)) .add( new StructField( "decimalToDecimal", new DecimalType(10, 2), true, FieldMetadata.empty()).withTypeChanges( Seq(new TypeChange(new DecimalType(5, 2), new DecimalType(10, 2))).asJava)) val metadata = getCompatEnabledMetadata(schema) val protocol = getCompatEnabledProtocol(TYPE_WIDENING_RW_FEATURE) // This should not throw an exception validateAndUpdateIcebergCompatMetadata(false, metadata, protocol) } test("incompatible type widening throws exception") { val schema = new StructType() .add( new StructField( "dateToTimestamp", TimestampNTZType.TIMESTAMP_NTZ, true, FieldMetadata.empty()).withTypeChanges( Seq(new TypeChange(DateType.DATE, TimestampNTZType.TIMESTAMP_NTZ)).asJava)) val metadata = getCompatEnabledMetadata(schema) val protocol = getCompatEnabledProtocol(TYPE_WIDENING_RW_FEATURE) val e = intercept[KernelException] { validateAndUpdateIcebergCompatMetadata(false, metadata, protocol) } assert(e.getMessage.contains( s"icebergCompat$icebergCompatVersion does not support type widening present in table")) } Seq(true, false).foreach { isNewTable => test( s"can't enable icebergCompat$icebergCompatVersion on a table with icebergCompatV1 enabled, " + s"isNewTable = $isNewTable") { val schema = new StructType().add("col", BooleanType.BOOLEAN) val metadata = getCompatEnabledMetadata(schema) .withMergedConfiguration( Map("delta.enableIcebergCompatV1" -> "true").asJava) val protocol = getCompatEnabledProtocol() val ex = intercept[KernelException] { validateAndUpdateIcebergCompatMetadata(isNewTable, metadata, protocol) } assert(ex.getMessage.contains( s"icebergCompat$icebergCompatVersion: Only one IcebergCompat version can be enabled. " + "Incompatible version enabled: delta.enableIcebergCompatV1")) } } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergCompatV2MetadataValidatorAndUpdaterSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.icebergcompat import java.util.Optional import scala.collection.JavaConverters._ import io.delta.kernel.exceptions.KernelException import io.delta.kernel.internal.actions.{Metadata, Protocol} import io.delta.kernel.internal.icebergcompat.IcebergCompatV2MetadataValidatorAndUpdater.validateAndUpdateIcebergCompatV2Metadata import io.delta.kernel.internal.tablefeatures.TableFeature import io.delta.kernel.internal.tablefeatures.TableFeatures.{COLUMN_MAPPING_RW_FEATURE, DELETION_VECTORS_RW_FEATURE, ICEBERG_COMPAT_V2_W_FEATURE} import io.delta.kernel.test.TestFixtures import io.delta.kernel.types._ trait IcebergCompatV2MetadataValidatorAndUpdaterSuiteBase extends IcebergCompatMetadataValidatorAndUpdaterSuiteBase with TestFixtures { override def icebergCompatVersion: String = "V2" override def supportedDataColumnTypes: Set[DataType] = ALL_TYPES override def unsupportedDataColumnTypes: Set[DataType] = Set(VariantType.VARIANT) override def unsupportedPartitionColumnTypes: Set[DataType] = NESTED_TYPES override def isDeletionVectorsSupported: Boolean = false override def withIcebergCompatAndCMEnabled( schema: StructType, columnMappingMode: String = "name", partCols: Seq[String] = Seq.empty): Metadata = { testMetadata(schema, partCols).withIcebergCompatV2AndCMEnabled(columnMappingMode) } } class IcebergCompatV2MetadataValidatorAndUpdaterSuite extends IcebergCompatV2MetadataValidatorAndUpdaterSuiteBase { override def simpleTypesToSkip: Set[DataType] = Set.empty override def getCompatEnabledMetadata( schema: StructType, columnMappingMode: String = "name", partCols: Seq[String] = Seq.empty): Metadata = { testMetadata(schema, partCols) .withIcebergCompatV2AndCMEnabled(columnMappingMode) } override def getCompatEnabledProtocol(tableFeatures: TableFeature*): Protocol = { testProtocol(tableFeatures ++ Seq(ICEBERG_COMPAT_V2_W_FEATURE, COLUMN_MAPPING_RW_FEATURE): _*) } override def validateAndUpdateIcebergCompatMetadata( isNewTable: Boolean, metadata: Metadata, protocol: Protocol): Optional[Metadata] = { validateAndUpdateIcebergCompatV2Metadata(isNewTable, metadata, protocol, Optional.empty()) } Seq(true, false).foreach { isNewTable => test(s"protocol is missing required column mapping feature, isNewTable $isNewTable") { val schema = new StructType().add("col", BooleanType.BOOLEAN) val metadata = withIcebergCompatAndCMEnabled(schema, partCols = Seq.empty) val protocol = new Protocol(3, 7, Set.empty.asJava, Set(s"icebergCompat$icebergCompatVersion").asJava) val e = intercept[KernelException] { validateAndUpdateIcebergCompatMetadata(isNewTable, metadata, protocol) } assert(e.getMessage.contains( s"icebergCompat$icebergCompatVersion: requires the feature 'columnMapping' to be enabled.")) } } Seq("id", "name").foreach { existingCMMode => Seq(true, false).foreach { isNewTable => test(s"existing column mapping mode `$existingCMMode` is preserved " + s"when icebergCompat is enabled, isNewTable = $isNewTable") { val metadata = getCompatEnabledMetadata(cmTestSchema(), columnMappingMode = existingCMMode) val protocol = getCompatEnabledProtocol() assert(metadata.getConfiguration.get("delta.columnMapping.mode") === existingCMMode) val updatedMetadata = validateAndUpdateIcebergCompatMetadata(isNewTable, metadata, protocol) // No metadata update is needed since already compatible column mapping mode assert(!updatedMetadata.isPresent) } } } Seq(true, false).foreach { isNewTable => test(s"column mapping mode `name` is auto enabled when icebergCompat is enabled, " + s"isNewTable = $isNewTable") { val metadata = testMetadata(cmTestSchema()) .withMergedConfiguration( Map(s"delta.enableIcebergCompat$icebergCompatVersion" -> "true").asJava) val protocol = getCompatEnabledProtocol() assert(!metadata.getConfiguration.containsKey("delta.columnMapping.mode")) if (isNewTable) { val updatedMetadata = validateAndUpdateIcebergCompatMetadata(isNewTable, metadata, protocol) assert(updatedMetadata.isPresent) assert(updatedMetadata.get().getConfiguration.get("delta.columnMapping.mode") == "name") } else { val e = intercept[KernelException] { validateAndUpdateIcebergCompatMetadata(isNewTable, metadata, protocol) } assert(e.getMessage.contains( "The value 'none' for the property 'delta.columnMapping.mode' is" + s" not compatible with icebergCompat$icebergCompatVersion requirements")) } } } /* --- prevProtocol DV check (concurrent writer protection) --- */ test("DV check catches deletion vectors enabled in previous protocol") { val schema = new StructType().add("col", BooleanType.BOOLEAN) val metadata = getCompatEnabledMetadata(schema) // newProtocol does NOT support DVs val newProtocol = getCompatEnabledProtocol() // prevProtocol DOES support DVs (simulating a concurrent writer that enabled DVs) val prevProtocol = Optional.of(getCompatEnabledProtocol(DELETION_VECTORS_RW_FEATURE)) val e = intercept[KernelException] { validateAndUpdateIcebergCompatV2Metadata( false, /* isCreatingNewTable */ metadata, newProtocol, prevProtocol) } assert(e.getMessage.contains( "Table features [deletionVectors] are incompatible with icebergCompatV2")) } test("DV check passes when neither new nor previous protocol has DVs") { val schema = new StructType().add("col", BooleanType.BOOLEAN) val metadata = getCompatEnabledMetadata(schema) val newProtocol = getCompatEnabledProtocol() val prevProtocol = Optional.of(getCompatEnabledProtocol()) // Should not throw validateAndUpdateIcebergCompatV2Metadata( false, /* isCreatingNewTable */ metadata, newProtocol, prevProtocol) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergCompatV3MetadataValidatorAndUpdateSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.icebergcompat import java.util.Optional import scala.collection.JavaConverters._ import io.delta.kernel.exceptions.KernelException import io.delta.kernel.internal.TableConfig import io.delta.kernel.internal.actions.{Metadata, Protocol} import io.delta.kernel.internal.icebergcompat.IcebergCompatV3MetadataValidatorAndUpdater.validateAndUpdateIcebergCompatV3Metadata import io.delta.kernel.internal.tablefeatures.TableFeature import io.delta.kernel.internal.tablefeatures.TableFeatures.{ALLOW_COLUMN_DEFAULTS_W_FEATURE, COLUMN_MAPPING_RW_FEATURE, ICEBERG_COMPAT_V3_W_FEATURE, ROW_TRACKING_W_FEATURE, TYPE_WIDENING_RW_FEATURE} import io.delta.kernel.test.TestFixtures import io.delta.kernel.types._ import org.assertj.core.util.Maps trait IcebergCompatV3MetadataValidatorAndUpdaterSuiteBase extends IcebergCompatMetadataValidatorAndUpdaterSuiteBase with TestFixtures { override def icebergCompatVersion: String = "V3" override def supportedDataColumnTypes: Set[DataType] = ALL_TYPES + VariantType.VARIANT override def unsupportedDataColumnTypes: Set[DataType] = Set.empty override def unsupportedPartitionColumnTypes: Set[DataType] = NESTED_TYPES override def isDeletionVectorsSupported: Boolean = true override def withIcebergCompatAndCMEnabled( schema: StructType, columnMappingMode: String = "name", partCols: Seq[String] = Seq.empty): Metadata = { testMetadata( schema, partCols).withIcebergCompatV3AndCMEnabled(columnMappingMode).withMergedConfiguration( Maps.newHashMap(TableConfig.ROW_TRACKING_ENABLED.getKey, "true")) } } class IcebergCompatV3MetadataValidatorAndUpdaterSuite extends IcebergCompatV3MetadataValidatorAndUpdaterSuiteBase { override def simpleTypesToSkip: Set[DataType] = Set.empty override def getCompatEnabledMetadata( schema: StructType, columnMappingMode: String = "name", partCols: Seq[String] = Seq.empty): Metadata = { testMetadata(schema, partCols) .withIcebergCompatV3AndCMEnabled(columnMappingMode).withMergedConfiguration( Maps.newHashMap(TableConfig.ROW_TRACKING_ENABLED.getKey, "true")) } override def getCompatEnabledProtocol(tableFeatures: TableFeature*): Protocol = { testProtocol(tableFeatures ++ Seq( ICEBERG_COMPAT_V3_W_FEATURE, COLUMN_MAPPING_RW_FEATURE, ROW_TRACKING_W_FEATURE): _*) } override def validateAndUpdateIcebergCompatMetadata( isNewTable: Boolean, metadata: Metadata, protocol: Protocol): Optional[Metadata] = { validateAndUpdateIcebergCompatV3Metadata(isNewTable, metadata, protocol, Optional.empty()) } Seq(true, false).foreach { isNewTable => test(s"protocol is missing required column mapping feature, isNewTable $isNewTable") { val schema = new StructType().add("col", BooleanType.BOOLEAN) val metadata = getCompatEnabledMetadata(schema) val protocol = new Protocol(3, 7, Set.empty.asJava, Set("icebergCompatV3", "rowTracking").asJava) val e = intercept[KernelException] { validateAndUpdateIcebergCompatV3Metadata(isNewTable, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains( "icebergCompatV3: requires the feature 'columnMapping' to be enabled.")) } } Seq("id", "name").foreach { existingCMMode => Seq(true, false).foreach { isNewTable => test(s"existing column mapping mode `$existingCMMode` is preserved " + s"when icebergCompat is enabled, isNewTable = $isNewTable") { val metadata = getCompatEnabledMetadata(cmTestSchema(), columnMappingMode = existingCMMode) val protocol = getCompatEnabledProtocol() assert(metadata.getConfiguration.get("delta.columnMapping.mode") === existingCMMode) val updatedMetadata = validateAndUpdateIcebergCompatMetadata(isNewTable, metadata, protocol) // No metadata update is needed since already compatible column mapping mode assert(!updatedMetadata.isPresent) } } } Seq(true, false).foreach { isNewTable => test(s"column mapping and row tracking are auto enabled when icebergCompatV3 is enabled, " + s"isNewTable = $isNewTable") { val metadata = testMetadata(cmTestSchema()).withIcebergCompatV3Enabled val protocol = testProtocol(ICEBERG_COMPAT_V3_W_FEATURE, COLUMN_MAPPING_RW_FEATURE, ROW_TRACKING_W_FEATURE) assert(!metadata.getConfiguration.containsKey("delta.columnMapping.mode")) assert(!metadata.getConfiguration.containsKey("delta.rowTracking.enabled")) if (isNewTable) { val updatedMetadata = validateAndUpdateIcebergCompatV3Metadata(isNewTable, metadata, protocol, Optional.empty()) assert(updatedMetadata.isPresent) assert(updatedMetadata.get().getConfiguration.get("delta.columnMapping.mode") == "name") assert(TableConfig.ROW_TRACKING_ENABLED.fromMetadata(updatedMetadata.get())) } else { val e = intercept[KernelException] { validateAndUpdateIcebergCompatV3Metadata(isNewTable, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains( "The value 'none' for the property 'delta.columnMapping.mode' is" + " not compatible with icebergCompatV3 requirements")) } } } Seq(true, false).foreach { isNewTable => test( s"can't enable icebergCompatV3 on a table with icebergCompatV2 enabled, " + s"isNewTable = $isNewTable") { val schema = new StructType().add("col", BooleanType.BOOLEAN) val metadata = getCompatEnabledMetadata(schema) .withMergedConfiguration( Map("delta.enableIcebergCompatV2" -> "true").asJava) val protocol = getCompatEnabledProtocol() val ex = intercept[KernelException] { validateAndUpdateIcebergCompatMetadata(isNewTable, metadata, protocol) } assert(ex.getMessage.contains( s"icebergCompat$icebergCompatVersion: Only one IcebergCompat version can be enabled. " + "Incompatible version enabled: delta.enableIcebergCompatV2")) } } Seq(true, false).foreach { isNewTable => test( s"icebergCompatV3 requires column default to be literal, " + s"isNewTable = $isNewTable") { val schema = new StructType().add( "col", IntegerType.INTEGER, new FieldMetadata.Builder().putString("CURRENT_DEFAULT", "CURRENT_TIMESTAMP()").build()) val metadata = getCompatEnabledMetadata(schema) val protocol = getCompatEnabledProtocol(ALLOW_COLUMN_DEFAULTS_W_FEATURE) val ex = intercept[KernelException] { validateAndUpdateIcebergCompatMetadata(isNewTable, metadata, protocol) } assert(ex.getMessage.contains("icebergCompatV3 requires the default value to be literal " + "with correct data types for a column. 'integer: CURRENT_TIMESTAMP()' is invalid.")) } } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergUniversalFormatMetadataValidatorAndUpdaterSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.icebergcompat import io.delta.kernel.exceptions.{InvalidConfigurationValueException, KernelException} import io.delta.kernel.internal.TableConfig import io.delta.kernel.internal.actions.Metadata import io.delta.kernel.internal.util.ColumnMappingSuiteBase import io.delta.kernel.types.IntegerType import io.delta.kernel.types.StructType import org.scalatest.funsuite.AnyFunSuiteLike class IcebergUniversalFormatMetadataValidatorAndUpdaterSuite extends AnyFunSuiteLike with ColumnMappingSuiteBase { test("validateAndUpdate shouldn't throw when when no config is set") { val metadata = createMetadata(Map("unrelated_key" -> "unrelated_value")) IcebergUniversalFormatMetadataValidatorAndUpdater.validate( metadata) } test( "validate shouldn't throw with valid Hudi enabled.") { val metadata = createMetadata(Map( TableConfig.UNIVERSAL_FORMAT_ENABLED_FORMATS.getKey -> "hudi", "unrelated_key" -> "unrelated_value")) IcebergUniversalFormatMetadataValidatorAndUpdater.validate(metadata) } test( "validate can enable iceberg universal compat is enabled and icebergCompatV2 is enabled") { val metadata = createMetadata(Map( TableConfig.UNIVERSAL_FORMAT_ENABLED_FORMATS.getKey -> "iceberg,hudi", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true", "unrelated_key" -> "unrelated_value")) IcebergUniversalFormatMetadataValidatorAndUpdater.validate(metadata) } Seq( Map(TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "false"), Map(TableConfig.ICEBERG_COMPAT_V3_ENABLED.getKey -> "false"), Map[String, String]()).foreach { disableIcebergCompat => test( "validate should throw when iceberg universal format is enabled but " + s"no IcebergCompat version is enabled: $disableIcebergCompat") { val metadata = createMetadata(Map( TableConfig.UNIVERSAL_FORMAT_ENABLED_FORMATS.getKey -> "iceberg", "unrelated_key" -> "unrelated_value") ++ disableIcebergCompat) val exc = intercept[InvalidConfigurationValueException] { IcebergUniversalFormatMetadataValidatorAndUpdater.validate(metadata) } assert(exc.getMessage == "Invalid value for table property " + "'delta.universalFormat.enabledFormats': 'iceberg'. " + "One of delta.enableIcebergCompatV2 or delta.enableIcebergCompatV3 " + "must be set to \"true\" to enable iceberg uniform format.") } } test("validate should throw when both IcebergCompatV2 and V3 are enabled") { val metadata = createMetadata(Map( TableConfig.UNIVERSAL_FORMAT_ENABLED_FORMATS.getKey -> "iceberg", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true", TableConfig.ICEBERG_COMPAT_V3_ENABLED.getKey -> "true")) val exc = intercept[InvalidConfigurationValueException] { IcebergUniversalFormatMetadataValidatorAndUpdater.validate(metadata) } assert(exc.getMessage.contains( "'delta.enableIcebergCompatV2' and 'delta.enableIcebergCompatV3' " + "cannot be enabled at the same time.")) } def createMetadata(tblProps: Map[String, String] = Map.empty): Metadata = { val schema = new StructType() .add("c1", IntegerType.INTEGER) testMetadata(schema, tblProps = tblProps) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.icebergcompat import java.util.Optional import scala.collection.JavaConverters._ import io.delta.kernel.exceptions.KernelException import io.delta.kernel.internal.TableConfig import io.delta.kernel.internal.actions.{Metadata, Protocol} import io.delta.kernel.internal.icebergcompat.IcebergWriterCompatV1MetadataValidatorAndUpdater.validateAndUpdateIcebergWriterCompatV1Metadata import io.delta.kernel.internal.tablefeatures.TableFeature import io.delta.kernel.internal.tablefeatures.TableFeatures.{COLUMN_MAPPING_RW_FEATURE, DELETION_VECTORS_RW_FEATURE, ICEBERG_COMPAT_V2_W_FEATURE, ICEBERG_WRITER_COMPAT_V1, TYPE_WIDENING_RW_FEATURE} import io.delta.kernel.internal.util.ColumnMapping import io.delta.kernel.types.{BooleanType, ByteType, DataType, DecimalType, FieldMetadata, IntegerType, LongType, ShortType, StructField, StructType, TypeChange} class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite extends IcebergCompatV2MetadataValidatorAndUpdaterSuiteBase { override def validateAndUpdateIcebergCompatMetadata( isNewTable: Boolean, metadata: Metadata, protocol: Protocol): Optional[Metadata] = { validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol, Optional.empty()) } val icebergWriterCompatV1EnabledProps = Map( TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.getKey -> "true") val icebergCompatV2EnabledProps = Map( TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true") val columnMappingIdModeProps = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id") /* icebergWriterCompatV1 restricts additional types allowed by icebergCompatV2 */ override def simpleTypesToSkip: Set[DataType] = Set(ByteType.BYTE, ShortType.SHORT) override def getCompatEnabledMetadata( schema: StructType, columnMappingMode: String = "id", partCols: Seq[String] = Seq.empty): Metadata = { testMetadata(schema, partCols) .withMergedConfiguration(( icebergWriterCompatV1EnabledProps ++ icebergCompatV2EnabledProps ++ columnMappingIdModeProps).asJava) } override def getCompatEnabledProtocol(tableFeatures: TableFeature*): Protocol = { testProtocol(tableFeatures ++ Seq(ICEBERG_WRITER_COMPAT_V1, ICEBERG_COMPAT_V2_W_FEATURE, COLUMN_MAPPING_RW_FEATURE): _*) } test("checks are not enforced when table property is not enabled") { // Violate check by including BYTE type column val schema = new StructType().add("col", ByteType.BYTE) val metadata = testMetadata(schema) assert(!TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.fromMetadata(metadata)) validateAndUpdateIcebergWriterCompatV1Metadata( true, /* isNewTable */ metadata, getCompatEnabledProtocol(), Optional.empty()) } /* --- UNSUPPORTED_TYPES_CHECK tests --- */ Seq(ByteType.BYTE, ShortType.SHORT).foreach { unsupportedType => Seq(true, false).foreach { isNewTable => test(s"disallowed data types: $unsupportedType, new table = $isNewTable") { val schema = new StructType().add("col", unsupportedType) val metadata = getCompatEnabledMetadata(schema) val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { validateAndUpdateIcebergWriterCompatV1Metadata( isNewTable, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains( s"icebergWriterCompatV1 does not support the data types: ")) } } } /* --- CM_ID_MODE_ENABLED and PHYSICAL_NAMES_MATCH_FIELD_IDS_CHECK tests --- */ Seq(true, false).foreach { isNewTable => Seq(true, false).foreach { icebergCompatV2Enabled => test(s"column mapping mode `id` is auto enabled when icebergWriterCompatV1 is enabled, " + s"isNewTable = $isNewTable, icebergCompatV2Enabled = $icebergCompatV2Enabled") { val tblProperties = icebergWriterCompatV1EnabledProps ++ (if (icebergCompatV2Enabled) { icebergCompatV2EnabledProps } else { Map() }) val metadata = testMetadata(cmTestSchema(), tblProps = tblProperties) val protocol = getCompatEnabledProtocol() assert(!metadata.getConfiguration.containsKey("delta.columnMapping.mode")) if (isNewTable) { val updatedMetadata = validateAndUpdateIcebergWriterCompatV1Metadata( isNewTable, metadata, protocol, Optional.empty()) assert(updatedMetadata.isPresent) assert(updatedMetadata.get().getConfiguration.get("delta.columnMapping.mode") == "id") // We correctly populate the column mapping metadata verifyCMTestSchemaHasValidColumnMappingInfo( updatedMetadata.get(), isNewTable, enableIcebergCompatV2 = true, enableIcebergWriterCompatV1 = true) } else { val e = intercept[KernelException] { validateAndUpdateIcebergWriterCompatV1Metadata( isNewTable, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains( "The value 'none' for the property 'delta.columnMapping.mode' is" + " not compatible with icebergWriterCompatV1 requirements")) } } } } Seq("name", "none").foreach { cmMode => Seq(true, false).foreach { isNewTable => test(s"cannot enable icebergWriterCompatV1 with incompatible column mapping mode " + s"`$cmMode`, isNewTable = $isNewTable") { val tblProperties = icebergWriterCompatV1EnabledProps ++ Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> cmMode) val metadata = testMetadata(cmTestSchema(), tblProps = tblProperties) val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { validateAndUpdateIcebergWriterCompatV1Metadata( isNewTable, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains( s"The value '$cmMode' for the property 'delta.columnMapping.mode' is" + " not compatible with icebergWriterCompatV1 requirements")) } } } Seq(true, false).foreach { isNewTable => test(s"cannot set physicalName to anything other than col-{fieldId}, isNewTable=$isNewTable") { val schema = new StructType() .add( "c1", IntegerType.INTEGER, FieldMetadata.builder() .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, 1) .putString(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "c1") .build()) val metadata = getCompatEnabledMetadata(schema) val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { validateAndUpdateIcebergWriterCompatV1Metadata( isNewTable, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains( "IcebergWriterCompatV1 requires column mapping field physical names be equal to " + "'col-[fieldId]', but this is not true for the following fields " + "[c1(physicalName='c1', columnId=1)]")) } } Seq(true, false).foreach { isNewTable => test(s"can provide correct physicalName=col-{fieldId}, isNewTable=$isNewTable") { val schema = new StructType() .add( "c1", IntegerType.INTEGER, FieldMetadata.builder() .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, 1) .putString(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "col-1") .build()) val metadata = getCompatEnabledMetadata(schema) .withMergedConfiguration(Map(ColumnMapping.COLUMN_MAPPING_MAX_COLUMN_ID_KEY -> "1").asJava) val protocol = getCompatEnabledProtocol() val updatedMetadata = validateAndUpdateIcebergWriterCompatV1Metadata( isNewTable, metadata, protocol, Optional.empty()) // No metadata update happens assert(!updatedMetadata.isPresent) } } /* --- ICEBERG_COMPAT_V2_ENABLED tests --- */ Seq(true, false).foreach { isNewTable => test(s"icebergCompatV2 is auto enabled when icebergWriterCompatV1 is enabled, " + s"isNewTable = $isNewTable") { val metadata = testMetadata( cmTestSchema(), tblProps = icebergWriterCompatV1EnabledProps ++ columnMappingIdModeProps) val protocol = getCompatEnabledProtocol() assert(!TableConfig.ICEBERG_COMPAT_V2_ENABLED.fromMetadata(metadata)) if (isNewTable) { val updatedMetadata = validateAndUpdateIcebergWriterCompatV1Metadata( isNewTable, metadata, protocol, Optional.empty()) assert(updatedMetadata.isPresent) assert(TableConfig.ICEBERG_COMPAT_V2_ENABLED.fromMetadata(updatedMetadata.get)) } else { val e = intercept[KernelException] { validateAndUpdateIcebergWriterCompatV1Metadata( isNewTable, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains( "The value 'false' for the property 'delta.enableIcebergCompatV2' is" + " not compatible with icebergWriterCompatV1 requirements")) } } } Seq(true, false).foreach { isNewTable => test(s"cannot enable icebergWriterCompatV1 with icebergCompatV2 explicitly disabled, " + s"isNewTable = $isNewTable") { val tblProperties = icebergWriterCompatV1EnabledProps ++ columnMappingIdModeProps ++ Map(TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "false") val metadata = testMetadata(cmTestSchema(), tblProps = tblProperties) val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { validateAndUpdateIcebergWriterCompatV1Metadata( isNewTable, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains( "The value 'false' for the property 'delta.enableIcebergCompatV2' is" + " not compatible with icebergWriterCompatV1 requirements")) } } /* --- UNSUPPORTED_FEATURES_CHECK tests --- */ test("all supported features are allowed") { val readerFeatures = Set("columnMapping", "timestampNtz", "v2Checkpoint", "vacuumProtocolCheck") // TODO add typeWidening and typeWidening-preview here once it's no longer blocked // icebergCompatV2 val writerFeatures = Set( // Legacy incompatible features (allowed as long as they are inactive) "invariants", "checkConstraints", "changeDataFeed", "identityColumns", "generatedColumns", // Compatible table features "appendOnly", "columnMapping", "icebergCompatV2", "icebergWriterCompatV1", "domainMetadata", "vacuumProtocolCheck", "v2Checkpoint", "checkpointProtection", "inCommitTimestamp", "clustering", "typeWidening", "typeWidening-preview", "timestampNtz", "catalogManaged") val protocol = new Protocol(3, 7, readerFeatures.asJava, writerFeatures.asJava) val metadata = getCompatEnabledMetadata(cmTestSchema()) validateAndUpdateIcebergWriterCompatV1Metadata(true, metadata, protocol, Optional.empty()) validateAndUpdateIcebergWriterCompatV1Metadata(false, metadata, protocol, Optional.empty()) } test("compatible type widening is allowed with icebergWriterCompatV1") { val schema = new StructType() .add( new StructField( "intToLong", IntegerType.INTEGER, true, FieldMetadata.builder() .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, 1) .putString(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "col-1") .build()).withTypeChanges(Seq(new TypeChange( IntegerType.INTEGER, LongType.LONG)).asJava)) val metadata = getCompatEnabledMetadata(schema) .withMergedConfiguration(Map(ColumnMapping.COLUMN_MAPPING_MAX_COLUMN_ID_KEY -> "1").asJava) val protocol = getCompatEnabledProtocol(TYPE_WIDENING_RW_FEATURE) // This should not throw an exception validateAndUpdateIcebergWriterCompatV1Metadata(false, metadata, protocol, Optional.empty()) } test("incompatible type widening throws exception with icebergWriterCompatV1") { val schema = new StructType() .add( new StructField( "intToLong", IntegerType.INTEGER, true, FieldMetadata.builder() .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, 1) .putString(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "col-1") .build()).withTypeChanges( Seq(new TypeChange(ByteType.BYTE, new DecimalType(10, 0))).asJava)) val metadata = getCompatEnabledMetadata(schema) .withMergedConfiguration(Map(ColumnMapping.COLUMN_MAPPING_MAX_COLUMN_ID_KEY -> "1").asJava) val protocol = getCompatEnabledProtocol(TYPE_WIDENING_RW_FEATURE) val e = intercept[KernelException] { validateAndUpdateIcebergWriterCompatV1Metadata(false, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains("icebergCompatV2 does not support type widening present in table")) } private def checkUnsupportedOrIncompatibleFeature( tableFeature: String, expectedErrorMessageContains: String): Unit = { val protocol = new Protocol( 3, 7, Set("columnMapping").asJava, Set( "columnMapping", "icebergCompatV2", "icebergWriterCompatV1", tableFeature).asJava) val metadata = getCompatEnabledMetadata(cmTestSchema()) Seq(true, false).foreach { isNewTable => val e = intercept[KernelException] { validateAndUpdateIcebergWriterCompatV1Metadata( isNewTable, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains(expectedErrorMessageContains)) } } Seq( // "defaultColumns", add this to this test once we support defaultColumns // "collations", add this to this test once we support collations "variantType").foreach { incompatibleFeature => test(s"cannot enable with incompatible feature $incompatibleFeature") { checkUnsupportedOrIncompatibleFeature( incompatibleFeature, s"Table features [$incompatibleFeature] are incompatible with " + s"icebergWriterCompatV1") } } Seq("collations", "defaultColumns").foreach { unsupportedIncompatibleFeature => test(s"cannot enable with incompatible UNSUPPORTED feature $unsupportedIncompatibleFeature") { // We add this test here so that it will fail when we add Kernel support for these features // When this happens -> add the feature to the test above checkUnsupportedOrIncompatibleFeature( unsupportedIncompatibleFeature, "Unsupported Delta table feature: table requires feature " + s""""$unsupportedIncompatibleFeature" which is unsupported by this version of """ + "Delta Kernel") } } private def testIncompatibleActiveLegacyFeature( activeFeatureMetadata: Metadata, tableFeature: String): Unit = { Seq(true, false).foreach { isNewTable => test(s"cannot enable with $tableFeature active, isNewTable = $isNewTable") { val e = intercept[KernelException] { validateAndUpdateIcebergWriterCompatV1Metadata( isNewTable, activeFeatureMetadata, getCompatEnabledProtocol(), Optional.empty()) } assert(e.getMessage.contains( s"Table features [$tableFeature] are incompatible with icebergWriterCompatV1")) } } } /* --- INVARIANTS_INACTIVE_CHECK tests --- */ testIncompatibleActiveLegacyFeature( getCompatEnabledMetadata(new StructType() .add("c1", IntegerType.INTEGER) .add( "c2", IntegerType.INTEGER, FieldMetadata.builder() .putString("delta.invariants", "{\"expression\": { \"expression\": \"x > 3\"} }") .build())), "invariants") /* --- CHANGE_DATA_FEED_INACTIVE_CHECK tests --- */ testIncompatibleActiveLegacyFeature( getCompatEnabledMetadata(cmTestSchema()) .withMergedConfiguration(Map(TableConfig.CHANGE_DATA_FEED_ENABLED.getKey -> "true").asJava), "changeDataFeed") /* --- CHECK_CONSTRAINTS_INACTIVE_CHECK tests --- */ testIncompatibleActiveLegacyFeature( getCompatEnabledMetadata(cmTestSchema()) .withMergedConfiguration(Map("delta.constraints.a" -> "a = b").asJava), "checkConstraints") /* --- ROW_TRACKING_INACTIVE_CHECK tests --- */ testIncompatibleActiveLegacyFeature( getCompatEnabledMetadata(cmTestSchema()) .withMergedConfiguration(Map(TableConfig.ROW_TRACKING_ENABLED.getKey -> "true").asJava), "rowTracking") /* --- IDENTITY_COLUMNS_INACTIVE_CHECK tests --- */ testIncompatibleActiveLegacyFeature( getCompatEnabledMetadata(new StructType() .add("c1", IntegerType.INTEGER) .add( "c2", IntegerType.INTEGER, FieldMetadata.builder() .putLong("delta.identity.start", 1L) .putLong("delta.identity.step", 2L) .putBoolean("delta.identity.allowExplicitInsert", true) .build())), "identityColumns") /* --- GENERATED_COLUMNS_INACTIVE_CHECK tests --- */ testIncompatibleActiveLegacyFeature( getCompatEnabledMetadata(new StructType() .add("c1", IntegerType.INTEGER) .add( "c2", IntegerType.INTEGER, FieldMetadata.builder() .putString("delta.generationExpression", "{\"expression\": \"c1 + 1\"}") .build())), "generatedColumns") /* --- requiredDependencyTableFeatures tests --- */ Seq( ("columnMapping", "icebergCompatV2"), ("icebergCompatV2", "columnMapping")).foreach { case (featureToIncludeStr, missingFeatureStr) => Seq(true, false).foreach { isNewTable => test(s"protocol is missing required feature $missingFeatureStr, isNewTable = $isNewTable") { val metadata = getCompatEnabledMetadata(cmTestSchema()) val readerFeatures: Set[String] = if ("columnMapping" == featureToIncludeStr) { Set("columnMapping") } else Set.empty val writerFeatures = Set("icebergWriterCompatV1", featureToIncludeStr) val protocol = new Protocol(3, 7, readerFeatures.asJava, writerFeatures.asJava) val e = intercept[KernelException] { validateAndUpdateIcebergWriterCompatV1Metadata( isNewTable, metadata, protocol, Optional.empty()) } // Since we run icebergCompatV2 validation as part of // ICEBERG_COMPAT_V2_ENABLED.postProcess we actually hit the missing feature error in the // icebergCompatV2 checks first assert(e.getMessage.contains( s"icebergCompatV2: requires the feature '$missingFeatureStr' to be enabled")) } } } /* --- prevProtocol DV check flows through WriterCompatV1 -> CompatV2 delegation --- */ test("prevProtocol DV check flows through WriterCompatV1 delegation to CompatV2") { val schema = new StructType().add("col", BooleanType.BOOLEAN) val metadata = getCompatEnabledMetadata(schema) val newProtocol = getCompatEnabledProtocol() // prevProtocol has DVs enabled, simulating a table that had DVs before this transaction val prevProtocol = Optional.of(getCompatEnabledProtocol(DELETION_VECTORS_RW_FEATURE)) val e = intercept[KernelException] { validateAndUpdateIcebergWriterCompatV1Metadata( false, metadata, newProtocol, prevProtocol) } // The error comes from icebergCompatV2 since WriterCompatV1 delegates DV checking to V2 assert(e.getMessage.contains( "Table features [deletionVectors] are incompatible with icebergCompatV2")) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.icebergcompat import java.util.Optional import scala.collection.JavaConverters._ import io.delta.kernel.exceptions.KernelException import io.delta.kernel.internal.TableConfig import io.delta.kernel.internal.actions.{Metadata, Protocol} import io.delta.kernel.internal.icebergcompat.IcebergWriterCompatV3MetadataValidatorAndUpdater.validateAndUpdateIcebergWriterCompatV3Metadata import io.delta.kernel.internal.tablefeatures.TableFeature import io.delta.kernel.internal.tablefeatures.TableFeatures._ import io.delta.kernel.internal.util.ColumnMapping import io.delta.kernel.types.{ByteType, DataType, DateType, FieldMetadata, IntegerType, LongType, ShortType, StructField, StructType, TimestampNTZType, TypeChange} class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite extends IcebergCompatV3MetadataValidatorAndUpdaterSuiteBase { override def validateAndUpdateIcebergCompatMetadata( isNewTable: Boolean, metadata: Metadata, protocol: Protocol): Optional[Metadata] = { validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol, Optional.empty()) } val icebergWriterCompatV3EnabledProps = Map( TableConfig.ICEBERG_WRITER_COMPAT_V3_ENABLED.getKey -> "true") val icebergCompatV3EnabledProps = Map( TableConfig.ICEBERG_COMPAT_V3_ENABLED.getKey -> "true") val deletionVectorsEnabledProps = Map( TableConfig.DELETION_VECTORS_CREATION_ENABLED.getKey -> "true") val variantShreddingEnabledProps = Map( TableConfig.VARIANT_SHREDDING_ENABLED.getKey -> "true") val columnMappingIdModeProps = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id") val rowTrackingEnabledProps = Map( TableConfig.ROW_TRACKING_ENABLED.getKey -> "true") override def getCompatEnabledMetadata( schema: StructType, columnMappingMode: String = "id", partCols: Seq[String] = Seq.empty): Metadata = { val result = testMetadata(schema, partCols) .withMergedConfiguration(( icebergWriterCompatV3EnabledProps ++ icebergCompatV3EnabledProps ++ columnMappingIdModeProps ++ rowTrackingEnabledProps).asJava) result } override def getCompatEnabledProtocol(tableFeatures: TableFeature*): Protocol = { testProtocol(tableFeatures ++ Seq( ICEBERG_WRITER_COMPAT_V3, ICEBERG_COMPAT_V3_W_FEATURE, DELETION_VECTORS_RW_FEATURE, VARIANT_RW_FEATURE, VARIANT_SHREDDING_RW_FEATURE, VARIANT_SHREDDING_PREVIEW_RW_FEATURE, VARIANT_RW_PREVIEW_FEATURE, ROW_TRACKING_W_FEATURE, COLUMN_MAPPING_RW_FEATURE): _*) } /* icebergWriterCompatV3 restricts additional types allowed by icebergCompatV3 */ override def simpleTypesToSkip: Set[DataType] = Set(ByteType.BYTE, ShortType.SHORT) private def checkUnsupportedOrIncompatibleFeature( tableFeature: String, expectedErrorMessageContains: String): Unit = { val protocol = new Protocol( 3, 7, Set("columnMapping", "rowTracking").asJava, Set( "columnMapping", "icebergCompatV3", "icebergWriterCompatV3", "deletionVectors", "rowTracking", "variantType", "variantType-preview", "variantShredding", "variantShredding-preview", tableFeature).asJava) val metadata = getCompatEnabledMetadata(cmTestSchema()) Seq(true, false).foreach { isNewTable => val e = intercept[KernelException] { validateAndUpdateIcebergWriterCompatV3Metadata( isNewTable, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains(expectedErrorMessageContains)) } } private def testIncompatibleActiveLegacyFeature( activeFeatureMetadata: Metadata, tableFeature: String): Unit = { Seq(true, false).foreach { isNewTable => test(s"cannot enable with $tableFeature active, isNewTable = $isNewTable") { val e = intercept[KernelException] { validateAndUpdateIcebergWriterCompatV3Metadata( isNewTable, activeFeatureMetadata, getCompatEnabledProtocol(), Optional.empty()) } assert(e.getMessage.contains( s"Table features [$tableFeature] are incompatible with icebergWriterCompatV3")) } } } /* --- CM_ID_MODE_ENABLED and PHYSICAL_NAMES_MATCH_FIELD_IDS_CHECK tests --- */ Seq(true, false).foreach { isNewTable => Seq(true, false).foreach { icebergCompatV3Enabled => test(s"column mapping mode `id` is auto enabled when icebergWriterCompatV3 is enabled, " + s"isNewTable = $isNewTable, icebergCompatV3Enabled = $icebergCompatV3Enabled") { val tblProperties = icebergWriterCompatV3EnabledProps ++ (if (icebergCompatV3Enabled) { icebergCompatV3EnabledProps } else { Map() }) val metadata = testMetadata(cmTestSchema(), tblProps = tblProperties) val protocol = getCompatEnabledProtocol() assert(!metadata.getConfiguration.containsKey("delta.columnMapping.mode")) if (isNewTable) { val updatedMetadata = validateAndUpdateIcebergWriterCompatV3Metadata( isNewTable, metadata, protocol, Optional.empty()) assert(updatedMetadata.isPresent) assert(updatedMetadata.get().getConfiguration.get("delta.columnMapping.mode") == "id") // We correctly populate the column mapping metadata verifyCMTestSchemaHasValidColumnMappingInfo( updatedMetadata.get(), isNewTable, true, true) } else { val e = intercept[KernelException] { validateAndUpdateIcebergWriterCompatV3Metadata( isNewTable, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains( "The value 'none' for the property 'delta.columnMapping.mode' is" + " not compatible with icebergWriterCompatV3 requirements")) } } } } test("checks are not enforced when table property is not enabled") { // Violate check by including BYTE type column val schema = new StructType().add("col", ByteType.BYTE) val metadata = testMetadata(schema) assert(!TableConfig.ICEBERG_WRITER_COMPAT_V3_ENABLED.fromMetadata(metadata)) validateAndUpdateIcebergWriterCompatV3Metadata( true, /* isNewTable */ metadata, getCompatEnabledProtocol(), Optional.empty()) } Seq("name", "none").foreach { cmMode => Seq(true, false).foreach { isNewTable => test(s"cannot enable icebergWriterCompatV3 with incompatible column mapping mode " + s"`$cmMode`, isNewTable = $isNewTable") { val tblProperties = icebergWriterCompatV3EnabledProps ++ Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> cmMode) val metadata = testMetadata(cmTestSchema(), tblProps = tblProperties) val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { validateAndUpdateIcebergWriterCompatV3Metadata( isNewTable, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains( s"The value '$cmMode' for the property 'delta.columnMapping.mode' is" + " not compatible with icebergWriterCompatV3 requirements")) } } } Seq(true, false).foreach { isNewTable => test(s"cannot set physicalName to anything other than col-{fieldId}, isNewTable=$isNewTable") { val schema = new StructType() .add( "c1", IntegerType.INTEGER, FieldMetadata.builder() .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, 1) .putString(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "c1") .build()) val metadata = getCompatEnabledMetadata(schema) val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { validateAndUpdateIcebergWriterCompatV3Metadata( isNewTable, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains( "requires column mapping field physical names be equal to " + "'col-[fieldId]', but this is not true for the following fields " + "[c1(physicalName='c1', columnId=1)]")) } } Seq(true, false).foreach { isNewTable => test(s"can provide correct physicalName=col-{fieldId}, isNewTable=$isNewTable") { val schema = new StructType() .add( "c1", IntegerType.INTEGER, FieldMetadata.builder() .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, 1) .putString(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "col-1") .build()) val metadata = getCompatEnabledMetadata(schema) .withMergedConfiguration(Map(ColumnMapping.COLUMN_MAPPING_MAX_COLUMN_ID_KEY -> "1").asJava) val protocol = getCompatEnabledProtocol() val updatedMetadata = validateAndUpdateIcebergWriterCompatV3Metadata( isNewTable, metadata, protocol, Optional.empty()) // No metadata update happens assert(!updatedMetadata.isPresent) } } /* --- UNSUPPORTED_TYPES_CHECK tests --- */ Seq(ByteType.BYTE, ShortType.SHORT).foreach { unsupportedType => Seq(true, false).foreach { isNewTable => test(s"disallowed data types: $unsupportedType, new table = $isNewTable") { val schema = new StructType().add("col", unsupportedType) val metadata = getCompatEnabledMetadata(schema) val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { validateAndUpdateIcebergWriterCompatV3Metadata( isNewTable, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains( s"icebergWriterCompatV3 does not support the data types: ")) } } } test("compatible type widening is allowed with icebergWriterCompatV3") { val schema = new StructType() .add( new StructField( "intToLong", IntegerType.INTEGER, true, FieldMetadata.builder() .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, 1) .putString(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "col-1") .build()).withTypeChanges(Seq(new TypeChange( IntegerType.INTEGER, LongType.LONG)).asJava)) val metadata = getCompatEnabledMetadata(schema) .withMergedConfiguration(Map(ColumnMapping.COLUMN_MAPPING_MAX_COLUMN_ID_KEY -> "1").asJava) val protocol = getCompatEnabledProtocol(TYPE_WIDENING_RW_FEATURE) validateAndUpdateIcebergCompatMetadata(false, metadata, protocol) } test("incompatible type widening throws exception with icebergWriterCompatV3") { val schema = new StructType() .add( new StructField( "dateToTimestamp", TimestampNTZType.TIMESTAMP_NTZ, true, FieldMetadata.builder() .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, 1) .putString(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "col-1") .build()).withTypeChanges( Seq(new TypeChange(DateType.DATE, TimestampNTZType.TIMESTAMP_NTZ)).asJava)) val metadata = getCompatEnabledMetadata(schema) .withMergedConfiguration(Map(ColumnMapping.COLUMN_MAPPING_MAX_COLUMN_ID_KEY -> "1").asJava) val protocol = getCompatEnabledProtocol(TYPE_WIDENING_RW_FEATURE, ROW_TRACKING_W_FEATURE) val e = intercept[KernelException] { validateAndUpdateIcebergCompatMetadata(false, metadata, protocol) } assert(e.getMessage.contains("icebergCompatV3 does not support type widening present in table")) } /* --- ICEBERG_COMPAT_V3_ENABLED tests --- */ Seq(true, false).foreach { isNewTable => test(s"icebergCompatV3 is auto enabled when icebergWriterCompatV3 is enabled, " + s"isNewTable = $isNewTable") { val metadata = testMetadata( cmTestSchema(), tblProps = icebergWriterCompatV3EnabledProps ++ columnMappingIdModeProps ++ rowTrackingEnabledProps) val protocol = getCompatEnabledProtocol() assert(!TableConfig.ICEBERG_COMPAT_V3_ENABLED.fromMetadata(metadata)) if (isNewTable) { val updatedMetadata = validateAndUpdateIcebergWriterCompatV3Metadata( isNewTable, metadata, protocol, Optional.empty()) assert(updatedMetadata.isPresent) assert(TableConfig.ICEBERG_COMPAT_V3_ENABLED.fromMetadata(updatedMetadata.get)) } else { val e = intercept[KernelException] { validateAndUpdateIcebergWriterCompatV3Metadata( isNewTable, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains( "The value 'false' for the property 'delta.enableIcebergCompatV3' is" + " not compatible with icebergWriterCompatV3 requirements")) } } } Seq(true, false).foreach { isNewTable => test(s"cannot enable icebergWriterCompatV3 with icebergCompatV3 explicitly disabled, " + s"isNewTable = $isNewTable") { val tblProperties = icebergWriterCompatV3EnabledProps ++ columnMappingIdModeProps ++ rowTrackingEnabledProps ++ Map(TableConfig.ICEBERG_COMPAT_V3_ENABLED.getKey -> "false") val metadata = testMetadata(cmTestSchema(), tblProps = tblProperties) val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { validateAndUpdateIcebergWriterCompatV3Metadata( isNewTable, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains( "The value 'false' for the property 'delta.enableIcebergCompatV3' is" + " not compatible with icebergWriterCompatV3 requirements")) } } /* --- UNSUPPORTED_FEATURES_CHECK tests --- */ test("all supported features are allowed") { val readerFeatures = Set("columnMapping", "timestampNtz", "v2Checkpoint", "vacuumProtocolCheck", "rowTracking") val writerFeatures = Set( // Legacy incompatible features (allowed as long as they are inactive) "invariants", "checkConstraints", "changeDataFeed", "identityColumns", "generatedColumns", // Compatible table features "appendOnly", "columnMapping", "icebergCompatV3", "icebergWriterCompatV3", "domainMetadata", "vacuumProtocolCheck", "v2Checkpoint", "checkpointProtection", "inCommitTimestamp", "clustering", "typeWidening", "typeWidening-preview", "timestampNtz", "deletionVectors", "rowTracking", "variantType", "variantType-preview", "variantShredding", "variantShredding-preview", "icebergCompatV2", "icebergWriterCompatV1", "catalogManaged") val protocol = new Protocol(3, 7, readerFeatures.asJava, writerFeatures.asJava) val metadata = getCompatEnabledMetadata(cmTestSchema()) validateAndUpdateIcebergWriterCompatV3Metadata(true, metadata, protocol, Optional.empty()) validateAndUpdateIcebergWriterCompatV3Metadata(false, metadata, protocol, Optional.empty()) } Seq("collations", "defaultColumns").foreach { unsupportedIncompatibleFeature => test(s"cannot enable with incompatible UNSUPPORTED feature $unsupportedIncompatibleFeature") { // We add this test here so that it will fail when we add Kernel support for these features // When this happens -> add the feature to the test above checkUnsupportedOrIncompatibleFeature( unsupportedIncompatibleFeature, "Unsupported Delta table feature: table requires feature " + s""""$unsupportedIncompatibleFeature" which is unsupported by this version of """ + "Delta Kernel") } } /* --- INVARIANTS_INACTIVE_CHECK tests --- */ testIncompatibleActiveLegacyFeature( getCompatEnabledMetadata(new StructType() .add("c1", IntegerType.INTEGER) .add( "c2", IntegerType.INTEGER, FieldMetadata.builder() .putString("delta.invariants", "{\"expression\": { \"expression\": \"x > 3\"} }") .build())), "invariants") /* --- CHANGE_DATA_FEED_INACTIVE_CHECK tests --- */ testIncompatibleActiveLegacyFeature( getCompatEnabledMetadata(cmTestSchema()) .withMergedConfiguration(Map(TableConfig.CHANGE_DATA_FEED_ENABLED.getKey -> "true").asJava), "changeDataFeed") /* --- CHECK_CONSTRAINTS_INACTIVE_CHECK tests --- */ testIncompatibleActiveLegacyFeature( getCompatEnabledMetadata(cmTestSchema()) .withMergedConfiguration(Map("delta.constraints.a" -> "a = b").asJava), "checkConstraints") /* --- IDENTITY_COLUMNS_INACTIVE_CHECK tests --- */ testIncompatibleActiveLegacyFeature( getCompatEnabledMetadata(new StructType() .add("c1", IntegerType.INTEGER) .add( "c2", IntegerType.INTEGER, FieldMetadata.builder() .putLong("delta.identity.start", 1L) .putLong("delta.identity.step", 2L) .putBoolean("delta.identity.allowExplicitInsert", true) .build())), "identityColumns") /* --- GENERATED_COLUMNS_INACTIVE_CHECK tests --- */ testIncompatibleActiveLegacyFeature( getCompatEnabledMetadata(new StructType() .add("c1", IntegerType.INTEGER) .add( "c2", IntegerType.INTEGER, FieldMetadata.builder() .putString("delta.generationExpression", "{\"expression\": \"c1 + 1\"}") .build())), "generatedColumns") /* --- requiredDependencyTableFeatures tests --- */ Seq( ("columnMapping", "icebergCompatV3"), ("icebergCompatV3", "columnMapping"), ("rowTracking", "icebergCompatV3"), ("deletionVectors", "icebergCompatV3"), ("variantType", "icebergCompatV3")).foreach { case (featureToIncludeStr, missingFeatureStr) => Seq(true, false).foreach { isNewTable => test( s"protocol is missing required feature $missingFeatureStr when " + s"only $featureToIncludeStr present, isNewTable = $isNewTable") { val metadata = getCompatEnabledMetadata(cmTestSchema()) val readerFeatures: Set[String] = if (Set("columnMapping", "rowTracking").contains(featureToIncludeStr)) { Set(featureToIncludeStr) } else Set.empty val writerFeatures = Set("icebergWriterCompatV3", featureToIncludeStr) val protocol = new Protocol(3, 7, readerFeatures.asJava, writerFeatures.asJava) val e = intercept[KernelException] { validateAndUpdateIcebergWriterCompatV3Metadata( isNewTable, metadata, protocol, Optional.empty()) } // Since we run icebergCompatV3 validation as part of // ICEBERG_COMPAT_V3_ENABLED.postProcess we actually hit the missing feature error in the // icebergCompatV3 checks first assert(e.getMessage.contains( s"icebergCompatV3: requires the feature '$missingFeatureStr' to be enabled")) } } } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/metadatadomain/JsonMetadataDomainSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.metadatadomain import java.util.Optional import io.delta.kernel.internal.rowtracking.RowTrackingMetadataDomain import org.scalatest.funsuite.AnyFunSuite class JsonMetadataDomainSuite extends AnyFunSuite { test("JsonMetadataDomain can be serialized/deserialized - TestJsonMetadataDomain") { val testMetadataDomain = new TestJsonMetadataDomain(Optional.of("value1"), Optional.empty(), 10) // Test the serialization, empty Optional fields should be omitted val config = testMetadataDomain.toJsonConfiguration assert(config === """{"field1":"value1","field3":10}""") // Test the deserialization, missing Optional fields should be initialized as empty val deserializedDomain = TestJsonMetadataDomain.fromJsonConfiguration(config) assert(deserializedDomain.getField1.isPresent && deserializedDomain.getField1.get === "value1") assert(!deserializedDomain.getField2.isPresent) assert(deserializedDomain.getField3 === 10) assert(deserializedDomain.equals(testMetadataDomain)) } test("JsonMetadataDomain can be serialized/deserialized - RowTrackingMetadataDomain") { // RowTrackingMetadataDomain is an actual production class with one field 'rowIdHighWaterMark' val rowTrackingMetadataDomain = new RowTrackingMetadataDomain(10) // Test the serialization val config = rowTrackingMetadataDomain.toJsonConfiguration assert(config === """{"rowIdHighWaterMark":10}""") // Test the deserialization val deserializedDomain = RowTrackingMetadataDomain.fromJsonConfiguration(config) assert(deserializedDomain.getRowIdHighWaterMark === 10) assert(deserializedDomain.equals(rowTrackingMetadataDomain)) } test("JsonMetadataDomain deserialization can handle the extra 'domainName' field") { // Delta Spark has a bug where the serialized JSON includes an unintended 'domainName' field. // Delta Kernel can gracefully handle this because 'domainName' is annotated with @JsonIgnore, // so this field is ignored if encountered without throwing exception. // This test explicitly verifies that the deserialization can handle input JSON both // with and without the 'domainName' field. // Test with TestJsonMetadataDomain val testJson1 = """{"field3":10}""" val testJson2 = """{"domainName":"testDomain","field3":10}""" val testMD1 = TestJsonMetadataDomain.fromJsonConfiguration(testJson1) val testMD2 = TestJsonMetadataDomain.fromJsonConfiguration(testJson2) assert(!testMD1.getField1.isPresent) assert(!testMD1.getField2.isPresent) assert(testMD1.getField3 === 10) assert(testMD1 === testMD2) // Also test with an actual production class - RowTrackingMetadataDomain val rowTrackingJson1 = """{"rowIdHighWaterMark":10}""" val rowTrackingJson2 = """{"domainName":"delta.rowTracking","rowIdHighWaterMark":10}""" val rowTrackingMD1 = RowTrackingMetadataDomain.fromJsonConfiguration(rowTrackingJson1) val rowTrackingMD2 = RowTrackingMetadataDomain.fromJsonConfiguration(rowTrackingJson2) assert(rowTrackingMD1.getRowIdHighWaterMark === 10) assert(rowTrackingMD1 === rowTrackingMD2) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/metadatadomain/TestJsonMetadataDomain.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.metadatadomain; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Optional; /** * A test implementation of {@link JsonMetadataDomain} for testing purposes. It has two Optional * fields and one primitive field. */ public final class TestJsonMetadataDomain extends JsonMetadataDomain { private final Optional field1; private final Optional field2; private final int field3; @JsonCreator public TestJsonMetadataDomain( @JsonProperty("field1") Optional field1, @JsonProperty("field2") Optional field2, @JsonProperty("field3") int field3) { this.field1 = field1; this.field2 = field2; this.field3 = field3; } @Override public String getDomainName() { return "testDomain"; } public Optional getField1() { return field1; } public Optional getField2() { return field2; } public int getField3() { return field3; } public static TestJsonMetadataDomain fromJsonConfiguration(String json) { return JsonMetadataDomain.fromJsonConfiguration(json, TestJsonMetadataDomain.class); } @Override public boolean equals(Object obj) { if (obj instanceof TestJsonMetadataDomain) { TestJsonMetadataDomain other = (TestJsonMetadataDomain) obj; return field1.equals(other.field1) && field2.equals(other.field2) && field3 == other.field3; } return false; } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/metrics/CounterSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.metrics import org.scalatest.funsuite.AnyFunSuite class CounterSuite extends AnyFunSuite { test("Counter class") { val counter = new Counter() assert(counter.value == 0) counter.increment(0) assert(counter.value == 0) counter.increment() assert(counter.value == 1) counter.increment() assert(counter.value == 2) counter.increment(10) assert(counter.value == 12) counter.reset() assert(counter.value == 0) counter.increment() assert(counter.value == 1) } test("Counter toString representation") { val counter = new Counter() counter.increment(42) val stringRepresentation = counter.toString() assert(stringRepresentation === "Counter(42)") } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/metrics/MetricsReportSerializerSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.metrics import java.util.{Collections, Optional, UUID} import scala.collection.JavaConverters._ import io.delta.kernel.expressions.{Column, Literal, Predicate} import io.delta.kernel.metrics.{ScanReport, SnapshotReport, TransactionReport} import io.delta.kernel.types.{FieldMetadata, IntegerType, StringType, StructType} import org.scalatest.funsuite.AnyFunSuite class MetricsReportSerializerSuite extends AnyFunSuite { private def optionToString[T](option: Optional[T]): String = { if (option.isPresent) { if (option.get().isInstanceOf[String]) { s""""${option.get()}"""" // For string objects wrap with quotes } else { option.get().toString } } else { "null" } } private def testSnapshotReport(snapshotReport: SnapshotReport): Unit = { val computeTimestampToVersionTotalDuration = optionToString( snapshotReport.getSnapshotMetrics().getComputeTimestampToVersionTotalDurationNs()) val loadSnapshotTotalDuration = snapshotReport.getSnapshotMetrics().getLoadSnapshotTotalDurationNs() val loadProtocolAndMetadataDuration = snapshotReport.getSnapshotMetrics().getLoadProtocolMetadataTotalDurationNs() val buildLogSegmentDuration = snapshotReport.getSnapshotMetrics().getLoadLogSegmentTotalDurationNs() val loadCrcTotalDuration = snapshotReport.getSnapshotMetrics().getLoadCrcTotalDurationNs() val exception: Optional[String] = snapshotReport.getException().map(_.toString) val expectedJson = s""" |{"tablePath":"${snapshotReport.getTablePath()}", |"operationType":"Snapshot", |"reportUUID":"${snapshotReport.getReportUUID()}", |"exception":${optionToString(exception)}, |"version":${optionToString(snapshotReport.getVersion())}, |"checkpointVersion":${optionToString(snapshotReport.getCheckpointVersion())}, |"providedTimestamp":${optionToString(snapshotReport.getProvidedTimestamp())}, |"snapshotMetrics":{ |"computeTimestampToVersionTotalDurationNs":${computeTimestampToVersionTotalDuration}, |"loadSnapshotTotalDurationNs":${loadSnapshotTotalDuration}, |"loadProtocolMetadataTotalDurationNs":${loadProtocolAndMetadataDuration}, |"loadLogSegmentTotalDurationNs":${buildLogSegmentDuration}, |"loadCrcTotalDurationNs":${loadCrcTotalDuration} |} |} |""".stripMargin.replaceAll("\n", "") assert(expectedJson == snapshotReport.toJson()) } test("SnapshotReport serializer") { val snapshotContext1 = SnapshotQueryContext.forTimestampSnapshot("/table/path", 0) snapshotContext1.getSnapshotMetrics.computeTimestampToVersionTotalDurationTimer.record(10) snapshotContext1.getSnapshotMetrics.loadSnapshotTotalTimer.record(2000) snapshotContext1.getSnapshotMetrics.loadProtocolMetadataTotalDurationTimer.record(1000) snapshotContext1.getSnapshotMetrics.loadLogSegmentTotalDurationTimer.record(500) snapshotContext1.getSnapshotMetrics.loadCrcTotalDurationTimer.record(250) snapshotContext1.setResolvedVersion(25) snapshotContext1.setCheckpointVersion(Optional.of(20)) val exception = new RuntimeException("something something failed") val snapshotReport1 = SnapshotReportImpl.forError( snapshotContext1, exception) // Manually check expected JSON val expectedJson = s""" |{"tablePath":"/table/path", |"operationType":"Snapshot", |"reportUUID":"${snapshotReport1.getReportUUID()}", |"exception":"$exception", |"version":25, |"checkpointVersion":20, |"providedTimestamp":0, |"snapshotMetrics":{ |"computeTimestampToVersionTotalDurationNs":10, |"loadSnapshotTotalDurationNs":2000, |"loadProtocolMetadataTotalDurationNs":1000, |"loadLogSegmentTotalDurationNs":500, |"loadCrcTotalDurationNs":250 |} |} |""".stripMargin.replaceAll("\n", "") assert(expectedJson == snapshotReport1.toJson()) // Check with test function testSnapshotReport(snapshotReport1) // Empty options for all possible fields (version, providedTimestamp and exception) val snapshotContext2 = SnapshotQueryContext.forLatestSnapshot("/table/path") val snapshotReport2 = SnapshotReportImpl.forSuccess(snapshotContext2) testSnapshotReport(snapshotReport2) } private def testTransactionReport(transactionReport: TransactionReport): Unit = { val exception: Optional[String] = transactionReport.getException().map(_.toString) val snapshotReportUUID: Optional[String] = transactionReport.getSnapshotReportUUID().map(_.toString) val transactionMetrics = transactionReport.getTransactionMetrics val clusterColString = transactionReport.getClusteringColumns .asScala .map(col => col.getNames.map(s => s""""$s"""").mkString("[", ",", "]")) .mkString("[", ",", "]") val expectedJson = s""" |{"tablePath":"${transactionReport.getTablePath()}", |"operationType":"Transaction", |"reportUUID":"${transactionReport.getReportUUID()}", |"exception":${optionToString(exception)}, |"operation":"${transactionReport.getOperation()}", |"engineInfo":"${transactionReport.getEngineInfo()}", |"baseSnapshotVersion":${transactionReport.getBaseSnapshotVersion()}, |"snapshotReportUUID":${optionToString(snapshotReportUUID)}, |"committedVersion":${optionToString(transactionReport.getCommittedVersion())}, |"clusteringColumns":$clusterColString, |"transactionMetrics":{ |"totalCommitDurationNs":${transactionMetrics.getTotalCommitDurationNs}, |"numCommitAttempts":${transactionMetrics.getNumCommitAttempts}, |"numAddFiles":${transactionMetrics.getNumAddFiles}, |"numRemoveFiles":${transactionMetrics.getNumRemoveFiles}, |"numTotalActions":${transactionMetrics.getNumTotalActions}, |"totalAddFilesSizeInBytes":${transactionMetrics.getTotalAddFilesSizeInBytes}, |"totalRemoveFilesSizeInBytes":${transactionMetrics.getTotalRemoveFilesSizeInBytes} |} |} |""".stripMargin.replaceAll("\n", "") assert(expectedJson == transactionReport.toJson()) } test("TransactionReport serializer") { val snapshotReport1 = SnapshotReportImpl.forSuccess( SnapshotQueryContext.forVersionSnapshot("/table/path", 1)) val exception = new RuntimeException("something something failed") // Initialize transaction metrics and record some values val transactionMetrics1 = TransactionMetrics.forNewTable(); transactionMetrics1.totalCommitTimer.record(200) transactionMetrics1.commitAttemptsCounter.increment(2) transactionMetrics1.updateForAddFile(1000) transactionMetrics1.updateForAddFile(100) transactionMetrics1.updateForRemoveFile(1000) transactionMetrics1.totalActionsCounter.increment(90) val transactionReport1 = new TransactionReportImpl( "/table/path", "test-operation", "test-engine", Optional.of(2), /* committedVersion */ Optional.of(Collections.singletonList( new Column(Array[String]("test-clustering-col1", "nested")))), transactionMetrics1, Optional.of(snapshotReport1), Optional.of(exception)) // Manually check expected JSON val expectedJson = s""" |{"tablePath":"/table/path", |"operationType":"Transaction", |"reportUUID":"${transactionReport1.getReportUUID()}", |"exception":"$exception", |"operation":"test-operation", |"engineInfo":"test-engine", |"baseSnapshotVersion":1, |"snapshotReportUUID":"${snapshotReport1.getReportUUID}", |"committedVersion":2, |"clusteringColumns":[["test-clustering-col1","nested"]], |"transactionMetrics":{ |"totalCommitDurationNs":200, |"numCommitAttempts":2, |"numAddFiles":2, |"numRemoveFiles":1, |"numTotalActions":90, |"totalAddFilesSizeInBytes":1100, |"totalRemoveFilesSizeInBytes":1000 |} |} |""".stripMargin.replaceAll("\n", "") assert(expectedJson == transactionReport1.toJson()) // Check with test function testTransactionReport(transactionReport1) // Initialize snapshot report for the empty table case val snapshotReport2 = SnapshotReportImpl.forSuccess( SnapshotQueryContext.forVersionSnapshot("/table/path", -1)) // Empty option for all possible fields (committedVersion, exception) val transactionReport2 = new TransactionReportImpl( "/table/path", "test-operation-2", "test-engine-2", Optional.empty(), /* committedVersion */ Optional.of(Collections.singletonList(new Column("test-clustering-col1"))), // empty/un-incremented transaction metrics TransactionMetrics.withExistingTableFileSizeHistogram(Optional.empty()), Optional.of(snapshotReport2), Optional.empty() /* exception */ ) testTransactionReport(transactionReport2) } private def testScanReport(scanReport: ScanReport): Unit = { val exception: Optional[String] = scanReport.getException().map(_.toString) val filter: Optional[String] = scanReport.getFilter.map(_.toString) val partitionPredicate: Optional[String] = scanReport.getPartitionPredicate().map(_.toString) val dataSkippingFilter: Optional[String] = scanReport.getDataSkippingFilter().map(_.toString) val scanMetrics = scanReport.getScanMetrics val expectedJson = s""" |{"tablePath":"${scanReport.getTablePath()}", |"operationType":"Scan", |"reportUUID":"${scanReport.getReportUUID()}", |"exception":${optionToString(exception)}, |"tableVersion":${scanReport.getTableVersion()}, |"tableSchema":"${scanReport.getTableSchema()}", |"snapshotReportUUID":"${scanReport.getSnapshotReportUUID}", |"filter":${optionToString(filter)}, |"readSchema":"${scanReport.getReadSchema}", |"partitionPredicate":${optionToString(partitionPredicate)}, |"dataSkippingFilter":${optionToString(dataSkippingFilter)}, |"isFullyConsumed":${scanReport.getIsFullyConsumed}, |"scanMetrics":{ |"totalPlanningDurationNs":${scanMetrics.getTotalPlanningDurationNs}, |"numAddFilesSeen":${scanMetrics.getNumAddFilesSeen}, |"numAddFilesSeenFromDeltaFiles":${scanMetrics.getNumAddFilesSeenFromDeltaFiles}, |"numActiveAddFiles":${scanMetrics.getNumActiveAddFiles}, |"numDuplicateAddFiles":${scanMetrics.getNumDuplicateAddFiles}, |"numRemoveFilesSeenFromDeltaFiles":${scanMetrics.getNumRemoveFilesSeenFromDeltaFiles} |} |} |""".stripMargin.replaceAll("\n", "") assert(expectedJson == scanReport.toJson()) } test("ScanReport serializer") { val snapshotReportUUID = java.util.UUID.randomUUID() // tableSchema now includes FieldMetadata with a null value. val fmNull = FieldMetadata.builder().putString("kNull", null).build() val fmArray = FieldMetadata.builder() .putStringArray("arr", Array[String]("x", null, "z")) .build() val tableSchema = new StructType() .add("part", IntegerType.INTEGER, fmNull) .add("id", IntegerType.INTEGER, fmArray) val partitionPredicate = new Predicate(">", new Column("part"), Literal.ofInt(1)) val exception = new RuntimeException("something something failed") // Initialize transaction metrics and record some values val scanMetrics = new ScanMetrics() scanMetrics.totalPlanningTimer.record(200) scanMetrics.addFilesCounter.increment(100) scanMetrics.addFilesFromDeltaFilesCounter.increment(90) scanMetrics.activeAddFilesCounter.increment(10) scanMetrics.removeFilesFromDeltaFilesCounter.increment(10) val scanReport1 = new ScanReportImpl( "/table/path", 1, tableSchema, snapshotReportUUID, Optional.of(partitionPredicate), new StructType().add("id", IntegerType.INTEGER), Optional.of(partitionPredicate), Optional.empty(), true, scanMetrics, Optional.of(exception)) // Manually check expected JSON including field metadata val tableSchemaStr = "struct(StructField(name=part,type=integer,nullable=true,metadata={kNull=null}," + "typeChanges=[]), " + "StructField(name=id,type=integer,nullable=true,metadata={arr=[x, null, z]}," + "typeChanges=[]))" val readSchemaStr = "struct(StructField(name=id,type=integer,nullable=true,metadata={},typeChanges=[]))" val expectedJson = s""" |{"tablePath":"/table/path", |"operationType":"Scan", |"reportUUID":"${scanReport1.getReportUUID}", |"exception":"$exception", |"tableVersion":1, |"tableSchema":"$tableSchemaStr", |"snapshotReportUUID":"$snapshotReportUUID", |"filter":"(column(`part`) > 1)", |"readSchema":"$readSchemaStr", |"partitionPredicate":"(column(`part`) > 1)", |"dataSkippingFilter":null, |"isFullyConsumed":true, |"scanMetrics":{ |"totalPlanningDurationNs":200, |"numAddFilesSeen":100, |"numAddFilesSeenFromDeltaFiles":90, |"numActiveAddFiles":10, |"numDuplicateAddFiles":0, |"numRemoveFilesSeenFromDeltaFiles":10 |} |} |""".stripMargin.replaceAll("\n", "") assert(expectedJson == scanReport1.toJson()) // Check with test function testScanReport(scanReport1) // Empty options for all possible fields (version, providedTimestamp and exception) val scanReport2 = new ScanReportImpl( "/table/path", 1, tableSchema, snapshotReportUUID, Optional.empty(), tableSchema, Optional.empty(), Optional.empty(), false, // isFullyConsumed new ScanMetrics(), Optional.empty()) testScanReport(scanReport2) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/metrics/TimerSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.metrics import java.util.concurrent.Callable import java.util.function.Supplier import org.scalatest.funsuite.AnyFunSuite class TimerSuite extends AnyFunSuite { val NANOSECONDS_PER_MILLISECOND = 1000000 def millisToNanos(millis: Long): Long = { millis * NANOSECONDS_PER_MILLISECOND } /** * @param incrementFx Function given (duration, timer) increments the timer by approximately * duration ms */ def testTimer(incrementFx: (Long, Timer) => Unit): Unit = { val timer = new Timer() // Verify initial values assert(timer.count == 0) assert(timer.totalDurationNs == 0) def incrementAndCheck(amtMillis: Long): Unit = { val initialCount = timer.count() val initialDuration = timer.totalDurationNs() // in nanoseconds val startTime = System.currentTimeMillis() incrementFx(amtMillis, timer) // upperLimitDuration is in milliseconds; we take the max of time elapsed vs the incrementAmt val upperLimitDuration = Math.max( // we pad by 1 due to rounding of nanoseconds to milliseconds for system time System.currentTimeMillis() - startTime + 1, amtMillis) // check count assert(timer.count == initialCount + 1) // check lowerbound assert(timer.totalDurationNs >= initialDuration + millisToNanos(amtMillis)) // check upperbound assert(timer.totalDurationNs <= initialDuration + millisToNanos(upperLimitDuration)) } incrementAndCheck(0) incrementAndCheck(20) incrementAndCheck(50) } test("Timer class") { // Using Timer.record() testTimer((amount, timer) => timer.record(millisToNanos(amount))) // Using Timer.start() testTimer((amount, timer) => { val timed = timer.start() Thread.sleep(amount) timed.stop() }) // Using Timer.time(supplier) def supplier(amount: Long): Supplier[Long] = { () => { Thread.sleep(amount) amount } } testTimer((amount, timer) => { timer.time(supplier(amount)) }) // Using Timer.timeCallable def callable(amount: Long): Callable[Long] = { () => { Thread.sleep(amount) amount } } testTimer((amount, timer) => { timer.timeCallable(callable(amount)) }) // Using Timer.time(runnable) def runnable(amount: Long): Runnable = { () => Thread.sleep(amount) } testTimer((amount, timer) => { timer.time(runnable(amount)) }) } test("Timer class with exceptions") { // We catch the exception outside of the functional interfaces def catchException(fx: () => Any): Unit = { try { fx.apply() } catch { case _: Exception => } } // Using Timer.time(supplier) def supplier(amount: Long): Supplier[Long] = { () => { Thread.sleep(amount) throw new RuntimeException() } } testTimer((amount, timer) => { catchException(() => timer.time(supplier(amount))) }) // Using Timer.timeCallable def callable(amount: Long): Callable[Long] = { () => { Thread.sleep(amount) throw new RuntimeException() } } testTimer((amount, timer) => { catchException(() => timer.timeCallable(callable(amount))) }) // Using Timer.time(runnable) def runnable(amount: Long): Runnable = { () => { Thread.sleep(amount) throw new RuntimeException() } } testTimer((amount, timer) => { catchException(() => timer.time(runnable(amount))) }) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/replay/ActionsIteratorSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.replay import java.util.{Collections, Optional} import scala.collection.JavaConverters._ import io.delta.kernel.data.{ColumnarBatch, ColumnVector, Row} import io.delta.kernel.engine._ import io.delta.kernel.expressions.Predicate import io.delta.kernel.test.BaseMockJsonHandler import io.delta.kernel.test.MockEngineUtils import io.delta.kernel.types.StructType import io.delta.kernel.utils.{CloseableIterator, FileStatus} import org.scalatest.funsuite.AnyFunSuite class ActionsIteratorSuite extends AnyFunSuite with MockEngineUtils { /** * Test for ActionsIterator resource leak fix validation * * This test validates that the fix applied in ActionsIterator.java prevents resource * leaks by ensuring that CloseableIterators are properly closed when exceptions occur. * * The specific fix being tested: Utils.closeCloseablesSilently(dataIter) in the catch block of * readCommitOrCompactionFile method. */ test("ActionsIterator readCommitOrCompactionFile resource cleanup") { var iteratorClosed = false val engine = mockEngine(jsonHandler = new BaseMockJsonHandler { override def readJsonFiles( fileIter: CloseableIterator[FileStatus], physicalSchema: StructType, predicate: Optional[Predicate]): CloseableIterator[ColumnarBatch] = { // Return an empty iterator that tracks closure new CloseableIterator[ColumnarBatch] { override def hasNext(): Boolean = throw new NoSuchElementException("This is a test exception") override def next(): ColumnarBatch = throw new UnsupportedOperationException("Not needed for this test") override def close(): Unit = iteratorClosed = true } } }) val testFile = FileStatus.of( "/path/to/00000000000000000000.json", 100L, System.currentTimeMillis()) val files = Collections.singletonList(testFile) val schema = new StructType() val actionsIterator = new ActionsIterator(engine, files, schema, Optional.empty[Predicate]()) assertThrows[NoSuchElementException] { actionsIterator.hasNext() } // Verify that resources were cleaned up assert(iteratorClosed, "Internal iterator should be closed after exception in ActionsIterator") } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/skipping/DataSkippingUtilsSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.skipping import java.util.Optional import scala.collection.JavaConverters._ import io.delta.kernel.expressions.{Column, Expression, Predicate} import io.delta.kernel.internal.skipping.DataSkippingUtils.constructDataSkippingFilter import io.delta.kernel.internal.skipping.StatsSchemaHelper.{MAX, MIN, STATS_WITH_COLLATION} import io.delta.kernel.internal.util.ExpressionUtils.createPredicate import io.delta.kernel.test.TestUtils import io.delta.kernel.types._ import io.delta.kernel.types.IntegerType.INTEGER import org.scalatest.funsuite.AnyFunSuite class DataSkippingUtilsSuite extends AnyFunSuite with TestUtils { def dataSkippingPredicate( operator: String, children: Seq[Expression], referencedColumns: Set[Column]): DataSkippingPredicate = { new DataSkippingPredicate(operator, children.asJava, referencedColumns.asJava) } def dataSkippingPredicate( operator: String, left: DataSkippingPredicate, right: DataSkippingPredicate): DataSkippingPredicate = { new DataSkippingPredicate(operator, left, right) } def dataSkippingPredicateWithCollation( operator: String, children: Seq[Expression], collation: CollationIdentifier, referencedColumns: Set[Column]): DataSkippingPredicate = { new DataSkippingPredicate(operator, children.asJava, collation, referencedColumns.asJava) } /* For struct type checks for equality based on field names & data type only */ def compareDataTypeUnordered(type1: DataType, type2: DataType): Boolean = (type1, type2) match { case (schema1: StructType, schema2: StructType) => val fields1 = schema1.fields().asScala.sortBy(_.getName) val fields2 = schema2.fields().asScala.sortBy(_.getName) if (fields1.length != fields2.length) { false } else { fields1.zip(fields2).forall { case (field1: StructField, field2: StructField) => field1.getName == field2.getName && compareDataTypeUnordered(field1.getDataType, field2.getDataType) } } case _ => type1 == type2 } def checkPruneStatsSchema( inputSchema: StructType, referencedCols: Set[Column], expectedSchema: StructType): Unit = { val prunedSchema = DataSkippingUtils.pruneStatsSchema(inputSchema, referencedCols.asJava) assert( compareDataTypeUnordered(expectedSchema, prunedSchema), s"expected=$expectedSchema\nfound=$prunedSchema") } test("pruneStatsSchema - multiple basic cases one level of nesting") { val nestedField = new StructField( "nested", new StructType() .add("col1", INTEGER) .add("col2", INTEGER), true) val testSchema = new StructType() .add(nestedField) .add("top_level_col", INTEGER) // no columns pruned checkPruneStatsSchema( testSchema, Set(col("top_level_col"), nestedCol("nested.col1"), nestedCol("nested.col2")), testSchema) // top level column pruned checkPruneStatsSchema( testSchema, Set(nestedCol("nested.col1"), nestedCol("nested.col2")), new StructType().add(nestedField)) // nested column only one field pruned checkPruneStatsSchema( testSchema, Set(nestedCol("top_level_col"), nestedCol("nested.col1")), new StructType() .add("nested", new StructType().add("col1", INTEGER)) .add("top_level_col", INTEGER)) // nested column completely pruned checkPruneStatsSchema( testSchema, Set(nestedCol("top_level_col")), new StructType().add("top_level_col", INTEGER)) // prune all columns checkPruneStatsSchema( testSchema, Set(), new StructType()) } test("pruneStatsSchema - 3 levels of nesting") { /* |--level1: struct | |--level2: struct | |--level3: struct | |--level_4_col: int | |--level_3_col: int | |--level_2_col: int */ val testSchema = new StructType() .add( "level1", new StructType() .add( "level2", new StructType() .add( "level3", new StructType().add("level_4_col", INTEGER)) .add("level_3_col", INTEGER)) .add("level_2_col", INTEGER)) // prune only 4th level col checkPruneStatsSchema( testSchema, Set(nestedCol("level1.level2.level_3_col"), nestedCol("level1.level_2_col")), new StructType() .add( "level1", new StructType() .add("level2", new StructType().add("level_3_col", INTEGER)) .add("level_2_col", INTEGER))) // prune only 3rd level column checkPruneStatsSchema( testSchema, Set(nestedCol("level1.level2.level3.level_4_col"), nestedCol("level1.level_2_col")), new StructType() .add( "level1", new StructType() .add( "level2", new StructType() .add( "level3", new StructType().add("level_4_col", INTEGER))) .add("level_2_col", INTEGER))) // prune 4th and 3rd level column checkPruneStatsSchema( testSchema, Set(nestedCol("level1.level_2_col")), new StructType() .add( "level1", new StructType() .add("level_2_col", INTEGER))) // prune all columns checkPruneStatsSchema( testSchema, Set(), new StructType()) } test("pruneStatsSchema - collated statistics") { val utf8Lcase = CollationIdentifier.fromString("SPARK.UTF8_LCASE.75") val unicode = CollationIdentifier.fromString("ICU.UNICODE.74.1") val unicodeString = new StringType(unicode) val ab = new StructType() .add("a", StringType.STRING) .add("b", unicodeString) val statsSchema = new StructType() .add(MIN, ab) .add(MAX, ab) .add( STATS_WITH_COLLATION, new StructType() .add( utf8Lcase.toString, new StructType() .add(MIN, ab) .add(MAX, ab)) .add( unicode.toString, new StructType() .add(MIN, ab) .add(MAX, ab))) val referenced = Set( nestedCol(s"$MAX.b"), collatedStatsCol(utf8Lcase, MIN, "a"), collatedStatsCol(unicode, MAX, "b")) val expected = new StructType() .add(MAX, new StructType().add("b", unicodeString)) .add( STATS_WITH_COLLATION, new StructType() .add( utf8Lcase.toString, new StructType() .add(MIN, new StructType().add("a", StringType.STRING))) .add( unicode.toString, new StructType() .add(MAX, new StructType().add("b", unicodeString)))) checkPruneStatsSchema(statsSchema, referenced, expected) } // TODO: add tests for remaining operators test("check constructDataSkippingFilter") { val testCases = Seq( // (schema, predicate, expectedDataSkippingPredicateOpt) ( new StructType() .add("a", StringType.STRING) .add("b", StringType.STRING), createPredicate("<", col("a"), col("b"), Optional.empty[CollationIdentifier]), None), ( new StructType() .add("a", IntegerType.INTEGER) .add("b", StringType.STRING), createPredicate("<", col("a"), literal("x"), Optional.empty[CollationIdentifier]), Some(dataSkippingPredicate( "<", Seq(nestedCol(s"$MIN.a"), literal("x")), Set(nestedCol(s"$MIN.a"))))), ( new StructType() .add("a", IntegerType.INTEGER) .add("b", StringType.STRING), createPredicate("<", literal("x"), col("a"), Optional.empty[CollationIdentifier]), Some(dataSkippingPredicate( ">", Seq(nestedCol(s"$MAX.a"), literal("x")), Set(nestedCol(s"$MAX.a"))))), ( new StructType() .add("a", IntegerType.INTEGER) .add("b", StringType.STRING), createPredicate(">", col("a"), literal("x"), Optional.empty[CollationIdentifier]), Some(dataSkippingPredicate( ">", Seq(nestedCol(s"$MAX.a"), literal("x")), Set(nestedCol(s"$MAX.a"))))), ( new StructType() .add("a", IntegerType.INTEGER), createPredicate("=", col("a"), literal(10), Optional.empty[CollationIdentifier]), Some(dataSkippingPredicate( "AND", dataSkippingPredicate( "<=", Seq(nestedCol(s"$MIN.a"), literal(10)), Set(nestedCol(s"$MIN.a"))), dataSkippingPredicate( ">=", Seq(nestedCol(s"$MAX.a"), literal(10)), Set(nestedCol(s"$MAX.a")))))), ( new StructType() .add("a", IntegerType.INTEGER), new Predicate( "NOT", createPredicate("<", col("a"), literal(10), Optional.empty[CollationIdentifier])), Some(dataSkippingPredicate( ">=", Seq(nestedCol(s"$MAX.a"), literal(10)), Set(nestedCol(s"$MAX.a"))))), // NOT over AND: NOT(a < 5 AND a > 10) => (max.a >= 5) OR (min.a <= 10) ( new StructType() .add("a", IntegerType.INTEGER), new Predicate( "NOT", createPredicate( "AND", createPredicate("<", col("a"), literal(5), Optional.empty[CollationIdentifier]), createPredicate(">", col("a"), literal(10), Optional.empty[CollationIdentifier]), Optional.empty[CollationIdentifier])), Some(dataSkippingPredicate( "OR", dataSkippingPredicate( ">=", Seq(nestedCol(s"$MAX.a"), literal(5)), Set(nestedCol(s"$MAX.a"))), dataSkippingPredicate( "<=", Seq(nestedCol(s"$MIN.a"), literal(10)), Set(nestedCol(s"$MIN.a")))))), // NOT over OR: NOT(a < 5 OR a > 10) => (max.a >= 5) AND (min.a <= 10) ( new StructType() .add("a", IntegerType.INTEGER), new Predicate( "NOT", createPredicate( "OR", createPredicate("<", col("a"), literal(5), Optional.empty[CollationIdentifier]), createPredicate(">", col("a"), literal(10), Optional.empty[CollationIdentifier]), Optional.empty[CollationIdentifier])), Some(dataSkippingPredicate( "AND", dataSkippingPredicate( ">=", Seq(nestedCol(s"$MAX.a"), literal(5)), Set(nestedCol(s"$MAX.a"))), dataSkippingPredicate( "<=", Seq(nestedCol(s"$MIN.a"), literal(10)), Set(nestedCol(s"$MIN.a")))))), // NOT over OR with one ineligible leg: NOT(a < b OR a < 5) => NOT(a < b) AND NOT(a < 5) // The first leg is ineligible; AND with single leg should return that leg only ( new StructType() .add("a", IntegerType.INTEGER) .add("b", IntegerType.INTEGER), new Predicate( "NOT", createPredicate( "OR", createPredicate("<", col("a"), col("b"), Optional.empty[CollationIdentifier]), createPredicate("<", col("a"), literal(5), Optional.empty[CollationIdentifier]), Optional.empty[CollationIdentifier])), Some(dataSkippingPredicate( ">=", Seq(nestedCol(s"$MAX.a"), literal(5)), Set(nestedCol(s"$MAX.a"))))), // NOT over AND with one ineligible leg: NOT(a < 5 AND a < b) // => NOT(a < 5) OR NOT(a < b); since OR needs both legs, expect None ( new StructType() .add("a", IntegerType.INTEGER) .add("b", IntegerType.INTEGER), new Predicate( "NOT", createPredicate( "AND", createPredicate("<", col("a"), literal(5), Optional.empty[CollationIdentifier]), createPredicate("<", col("a"), col("b"), Optional.empty[CollationIdentifier]), Optional.empty[CollationIdentifier])), None), // Double NOT elimination: NOT(NOT(a < 5)) => a < 5 => min.a < 5 ( new StructType() .add("a", IntegerType.INTEGER), new Predicate( "NOT", new Predicate( "NOT", createPredicate("<", col("a"), literal(5), Optional.empty[CollationIdentifier]))), Some(dataSkippingPredicate( "<", Seq(nestedCol(s"$MIN.a"), literal(5)), Set(nestedCol(s"$MIN.a"))))), // Cross-column case: NOT(a < 5 OR b > 7) => (max.a >= 5) AND (min.b <= 7) ( new StructType() .add("a", IntegerType.INTEGER) .add("b", IntegerType.INTEGER), new Predicate( "NOT", createPredicate( "OR", createPredicate("<", col("a"), literal(5), Optional.empty[CollationIdentifier]), createPredicate(">", col("b"), literal(7), Optional.empty[CollationIdentifier]), Optional.empty[CollationIdentifier])), Some(dataSkippingPredicate( "AND", dataSkippingPredicate( ">=", Seq(nestedCol(s"$MAX.a"), literal(5)), Set(nestedCol(s"$MAX.a"))), dataSkippingPredicate( "<=", Seq(nestedCol(s"$MIN.b"), literal(7)), Set(nestedCol(s"$MIN.b"))))))) testCases.foreach { case (schema, predicate, expectedDataSkippingPredicateOpt) => val dataSkippingPredicateOpt = JavaOptionalOps(constructDataSkippingFilter(predicate, schema)).toScala (dataSkippingPredicateOpt, expectedDataSkippingPredicateOpt) match { case (Some(dataSkippingPredicate), Some(expectedDataSkippingPredicate)) => assert(dataSkippingPredicate == expectedDataSkippingPredicate) case (None, None) => // pass case _ => fail(s"Expected $expectedDataSkippingPredicateOpt, found $dataSkippingPredicateOpt") } } } test("check constructDataSkippingFilter with collations") { val utf8Lcase = CollationIdentifier.fromString("SPARK.UTF8_LCASE.75") val unicode = CollationIdentifier.fromString("ICU.UNICODE.74.1") val testCases = Seq( // (schema, predicate, expectedDataSkippingPredicateOpt) // Ineligible: both sides are columns ( new StructType() .add("a", StringType.STRING) .add("b", StringType.STRING), createPredicate("<", col("a"), col("b"), Optional.of(utf8Lcase)), None), // Eligible: a < "m" with collation -> min(a, collation) < "m" ( new StructType() .add("a", StringType.STRING) .add("b", StringType.STRING), createPredicate("<", col("a"), literal("m"), Optional.of(utf8Lcase)), { val minA = collatedStatsCol(utf8Lcase, MIN, "a") Some(dataSkippingPredicateWithCollation( "<", Seq(minA, literal("m")), utf8Lcase, Set(minA))) }), // Reversed comparator: "m" < a -> max(a, collation) > "m" ( new StructType() .add("a", StringType.STRING), createPredicate("<", literal("m"), col("a"), Optional.of(utf8Lcase)), { val maxA = collatedStatsCol(utf8Lcase, MAX, "a") Some(dataSkippingPredicateWithCollation( ">", Seq(maxA, literal("m")), utf8Lcase, Set(maxA))) }), // Direct ">": a > "m" -> max(a, collation) > "m" ( new StructType() .add("a", StringType.STRING), createPredicate(">", col("a"), literal("m"), Optional.of(utf8Lcase)), { val maxA = collatedStatsCol(utf8Lcase, MAX, "a") Some(dataSkippingPredicateWithCollation( ">", Seq(maxA, literal("m")), utf8Lcase, Set(maxA))) }), // Equality ( new StructType() .add("a", StringType.STRING), createPredicate("=", col("a"), literal("abc"), Optional.of(unicode)), { val minA = collatedStatsCol(unicode, MIN, "a") val maxA = collatedStatsCol(unicode, MAX, "a") Some(dataSkippingPredicate( "AND", dataSkippingPredicateWithCollation("<=", Seq(minA, literal("abc")), unicode, Set(minA)), dataSkippingPredicateWithCollation( ">=", Seq(maxA, literal("abc")), unicode, Set(maxA)))) }), // NOT over comparator: NOT(a < "m") -> max(a, collation) >= "m" ( new StructType() .add("a", StringType.STRING), new Predicate( "NOT", createPredicate("<", col("a"), literal("m"), Optional.of(utf8Lcase))), { val maxA = collatedStatsCol(utf8Lcase, MAX, "a") Some(dataSkippingPredicateWithCollation( ">=", Seq(maxA, literal("m")), utf8Lcase, Set(maxA))) }), // NOT over AND // NOT(a < "m" AND a > "t") => (max.a >= "m") OR (min.a <= "t") ( new StructType() .add("a", StringType.STRING), new Predicate( "NOT", createPredicate( "AND", createPredicate("<", col("a"), literal("m"), Optional.of(unicode)), createPredicate(">", col("a"), literal("t"), Optional.of(utf8Lcase)), Optional.empty[CollationIdentifier])), { val unicodeMaxA = collatedStatsCol(unicode, MAX, "a") val utf8LcaseMinA = collatedStatsCol(utf8Lcase, MIN, "a") Some(dataSkippingPredicate( "OR", dataSkippingPredicateWithCollation( ">=", Seq(unicodeMaxA, literal("m")), unicode, Set(unicodeMaxA)), dataSkippingPredicateWithCollation( "<=", Seq(utf8LcaseMinA, literal("t")), utf8Lcase, Set(utf8LcaseMinA)))) }), // AND(a < "m" COLLATE UTF8_LCASE, b < 1) ( new StructType() .add("a", StringType.STRING) .add("b", IntegerType.INTEGER), createPredicate( "AND", createPredicate("<", col("a"), literal("m"), Optional.of(utf8Lcase)), createPredicate("<", col("b"), literal(1), Optional.empty[CollationIdentifier]), Optional.empty[CollationIdentifier]), { val minA = collatedStatsCol(utf8Lcase, MIN, "a") val minB = nestedCol(s"$MIN.b") Some(dataSkippingPredicate( "AND", dataSkippingPredicateWithCollation("<", Seq(minA, literal("m")), utf8Lcase, Set(minA)), dataSkippingPredicate("<", Seq(minB, literal(1)), Set(minB)))) }), ( new StructType() .add("a", IntegerType.INTEGER), createPredicate("<", col("a"), literal(1), Optional.of(utf8Lcase)), { val minA = collatedStatsCol(utf8Lcase, MIN, "a") Some(dataSkippingPredicateWithCollation( "<", Seq(minA, literal(1)), utf8Lcase, Set(minA))) })) testCases.foreach { case (schema, predicate, expectedDataSkippingPredicateOpt) => val dataSkippingPredicateOpt = JavaOptionalOps(constructDataSkippingFilter(predicate, schema)).toScala (dataSkippingPredicateOpt, expectedDataSkippingPredicateOpt) match { case (Some(dataSkippingPredicate), Some(expectedDataSkippingPredicate)) => assert(dataSkippingPredicate == expectedDataSkippingPredicate) case (None, None) => // pass case _ => fail(s"Expected $expectedDataSkippingPredicateOpt, found $dataSkippingPredicateOpt") } } } test("check constructDataSkippingFilter with collations (no version in collation)") { val utf8Lcase = CollationIdentifier.fromString("SPARK.UTF8_LCASE") val unicodeWithoutVersion = CollationIdentifier.fromString("ICU.UNICODE") val unicodeWithVersion = CollationIdentifier.fromString("ICU.UNICODE.74.1") val testCases = Seq( ( new StructType() .add("a", StringType.STRING) .add("b", StringType.STRING), createPredicate("<", col("a"), literal("m"), Optional.of(unicodeWithoutVersion)), Optional.empty[DataSkippingPredicate]), ( new StructType() .add("a", StringType.STRING), createPredicate( "AND", Seq[Expression]( createPredicate( "<", col("a"), literal("m"), Optional.of(utf8Lcase)), createPredicate( ">", col("a"), literal("t"), Optional.of(unicodeWithVersion))).asJava, Optional.empty[CollationIdentifier]), { val minAUnicode = collatedStatsCol(unicodeWithVersion, MAX, "a") Optional.of(dataSkippingPredicateWithCollation( ">", Seq(minAUnicode, literal("t")), unicodeWithVersion, Set(minAUnicode))) })) testCases.foreach { case (schema, predicate, expectedDataSkippingPredicate) => val dataSkippingPredicateOpt = constructDataSkippingFilter(predicate, schema) assert(dataSkippingPredicateOpt == expectedDataSkippingPredicate) } } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/skipping/StatsSchemaHelperSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.skipping import scala.collection.JavaConverters.setAsJavaSetConverter import io.delta.kernel.types.{ArrayType, BinaryType, BooleanType, ByteType, CollationIdentifier, DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType, MapType, ShortType, StringType, StructType, TimestampNTZType, TimestampType} import org.scalatest.funsuite.AnyFunSuite class StatsSchemaHelperSuite extends AnyFunSuite { val utf8Lcase = CollationIdentifier.fromString("SPARK.UTF8_LCASE.74") val unicode = CollationIdentifier.fromString("ICU.UNICODE.75.1") val utf8LcaseString = new StringType(utf8Lcase) val unicodeString = new StringType(unicode) test("check getStatsSchema for supported data types") { val testCases = Seq( ( new StructType().add("a", IntegerType.INTEGER), new StructType() .add(StatsSchemaHelper.NUM_RECORDS, LongType.LONG, true) .add(StatsSchemaHelper.MIN, new StructType().add("a", IntegerType.INTEGER, true), true) .add(StatsSchemaHelper.MAX, new StructType().add("a", IntegerType.INTEGER, true), true) .add(StatsSchemaHelper.NULL_COUNT, new StructType().add("a", LongType.LONG, true), true) .add(StatsSchemaHelper.TIGHT_BOUNDS, BooleanType.BOOLEAN, true)), ( new StructType().add("b", StringType.STRING), new StructType() .add(StatsSchemaHelper.NUM_RECORDS, LongType.LONG, true) .add(StatsSchemaHelper.MIN, new StructType().add("b", StringType.STRING, true), true) .add(StatsSchemaHelper.MAX, new StructType().add("b", StringType.STRING, true), true) .add(StatsSchemaHelper.NULL_COUNT, new StructType().add("b", LongType.LONG, true), true) .add(StatsSchemaHelper.TIGHT_BOUNDS, BooleanType.BOOLEAN, true)), ( new StructType().add("c", ByteType.BYTE), new StructType() .add(StatsSchemaHelper.NUM_RECORDS, LongType.LONG, true) .add(StatsSchemaHelper.MIN, new StructType().add("c", ByteType.BYTE, true), true) .add(StatsSchemaHelper.MAX, new StructType().add("c", ByteType.BYTE, true), true) .add(StatsSchemaHelper.NULL_COUNT, new StructType().add("c", LongType.LONG, true), true) .add(StatsSchemaHelper.TIGHT_BOUNDS, BooleanType.BOOLEAN, true)), ( new StructType().add("d", ShortType.SHORT), new StructType() .add(StatsSchemaHelper.NUM_RECORDS, LongType.LONG, true) .add(StatsSchemaHelper.MIN, new StructType().add("d", ShortType.SHORT, true), true) .add(StatsSchemaHelper.MAX, new StructType().add("d", ShortType.SHORT, true), true) .add(StatsSchemaHelper.NULL_COUNT, new StructType().add("d", LongType.LONG, true), true) .add(StatsSchemaHelper.TIGHT_BOUNDS, BooleanType.BOOLEAN, true)), ( new StructType().add("e", LongType.LONG), new StructType() .add(StatsSchemaHelper.NUM_RECORDS, LongType.LONG, true) .add(StatsSchemaHelper.MIN, new StructType().add("e", LongType.LONG, true), true) .add(StatsSchemaHelper.MAX, new StructType().add("e", LongType.LONG, true), true) .add(StatsSchemaHelper.NULL_COUNT, new StructType().add("e", LongType.LONG, true), true) .add(StatsSchemaHelper.TIGHT_BOUNDS, BooleanType.BOOLEAN, true)), ( new StructType().add("f", FloatType.FLOAT), new StructType() .add(StatsSchemaHelper.NUM_RECORDS, LongType.LONG, true) .add(StatsSchemaHelper.MIN, new StructType().add("f", FloatType.FLOAT, true), true) .add(StatsSchemaHelper.MAX, new StructType().add("f", FloatType.FLOAT, true), true) .add(StatsSchemaHelper.NULL_COUNT, new StructType().add("f", LongType.LONG, true), true) .add(StatsSchemaHelper.TIGHT_BOUNDS, BooleanType.BOOLEAN, true)), ( new StructType().add("g", DoubleType.DOUBLE), new StructType() .add(StatsSchemaHelper.NUM_RECORDS, LongType.LONG, true) .add(StatsSchemaHelper.MIN, new StructType().add("g", DoubleType.DOUBLE, true), true) .add(StatsSchemaHelper.MAX, new StructType().add("g", DoubleType.DOUBLE, true), true) .add(StatsSchemaHelper.NULL_COUNT, new StructType().add("g", LongType.LONG, true), true) .add(StatsSchemaHelper.TIGHT_BOUNDS, BooleanType.BOOLEAN, true)), ( new StructType().add("h", DateType.DATE), new StructType() .add(StatsSchemaHelper.NUM_RECORDS, LongType.LONG, true) .add(StatsSchemaHelper.MIN, new StructType().add("h", DateType.DATE, true), true) .add(StatsSchemaHelper.MAX, new StructType().add("h", DateType.DATE, true), true) .add(StatsSchemaHelper.NULL_COUNT, new StructType().add("h", LongType.LONG, true), true) .add(StatsSchemaHelper.TIGHT_BOUNDS, BooleanType.BOOLEAN, true)), ( new StructType().add("i", TimestampType.TIMESTAMP), new StructType() .add(StatsSchemaHelper.NUM_RECORDS, LongType.LONG, true) .add( StatsSchemaHelper.MIN, new StructType().add("i", TimestampType.TIMESTAMP, true), true) .add( StatsSchemaHelper.MAX, new StructType().add("i", TimestampType.TIMESTAMP, true), true) .add(StatsSchemaHelper.NULL_COUNT, new StructType().add("i", LongType.LONG, true), true) .add(StatsSchemaHelper.TIGHT_BOUNDS, BooleanType.BOOLEAN, true)), ( new StructType().add("j", TimestampNTZType.TIMESTAMP_NTZ), new StructType() .add(StatsSchemaHelper.NUM_RECORDS, LongType.LONG, true) .add( StatsSchemaHelper.MIN, new StructType().add("j", TimestampNTZType.TIMESTAMP_NTZ, true), true) .add( StatsSchemaHelper.MAX, new StructType().add("j", TimestampNTZType.TIMESTAMP_NTZ, true), true) .add(StatsSchemaHelper.NULL_COUNT, new StructType().add("j", LongType.LONG, true), true) .add(StatsSchemaHelper.TIGHT_BOUNDS, BooleanType.BOOLEAN, true)), ( new StructType().add("k", new DecimalType(20, 5)), new StructType() .add(StatsSchemaHelper.NUM_RECORDS, LongType.LONG, true) .add(StatsSchemaHelper.MIN, new StructType().add("k", new DecimalType(20, 5), true), true) .add(StatsSchemaHelper.MAX, new StructType().add("k", new DecimalType(20, 5), true), true) .add(StatsSchemaHelper.NULL_COUNT, new StructType().add("k", LongType.LONG, true), true) .add(StatsSchemaHelper.TIGHT_BOUNDS, BooleanType.BOOLEAN, true)), ( new StructType().add("l", utf8LcaseString), new StructType() .add(StatsSchemaHelper.NUM_RECORDS, LongType.LONG, true) .add(StatsSchemaHelper.MIN, new StructType().add("l", utf8LcaseString, true), true) .add(StatsSchemaHelper.MAX, new StructType().add("l", utf8LcaseString, true), true) .add(StatsSchemaHelper.NULL_COUNT, new StructType().add("l", LongType.LONG, true), true) .add(StatsSchemaHelper.TIGHT_BOUNDS, BooleanType.BOOLEAN, true))) testCases.foreach { case (dataSchema, expectedStatsSchema) => val statsSchema = StatsSchemaHelper.getStatsSchema( dataSchema, Set.empty[CollationIdentifier].asJava) assert(statsSchema == expectedStatsSchema) } } test("check getStatsSchema with mix of supported and unsupported data types") { val testCases = Seq( ( new StructType() .add("a", IntegerType.INTEGER) .add("b", BinaryType.BINARY) .add("c", new ArrayType(LongType.LONG, true)), new StructType() .add(StatsSchemaHelper.NUM_RECORDS, LongType.LONG, true) .add(StatsSchemaHelper.MIN, new StructType().add("a", IntegerType.INTEGER, true), true) .add(StatsSchemaHelper.MAX, new StructType().add("a", IntegerType.INTEGER, true), true) .add( StatsSchemaHelper.NULL_COUNT, new StructType() .add("a", LongType.LONG, true) .add("b", LongType.LONG, true) .add("c", LongType.LONG, true), true) .add(StatsSchemaHelper.TIGHT_BOUNDS, BooleanType.BOOLEAN, true)), ( new StructType() .add( "s", new StructType() .add("s1", StringType.STRING) .add("s2", BooleanType.BOOLEAN)), new StructType() .add(StatsSchemaHelper.NUM_RECORDS, LongType.LONG, true) .add( StatsSchemaHelper.MIN, new StructType() .add("s", new StructType().add("s1", StringType.STRING, true), true), true) .add( StatsSchemaHelper.MAX, new StructType() .add("s", new StructType().add("s1", StringType.STRING, true), true), true) .add( StatsSchemaHelper.NULL_COUNT, new StructType() .add( "s", new StructType() .add("s1", LongType.LONG, true) .add("s2", LongType.LONG, true), true), true) .add(StatsSchemaHelper.TIGHT_BOUNDS, BooleanType.BOOLEAN, true)), // Un-nested array/map alongside a supported type ( new StructType() .add("arr", new ArrayType(IntegerType.INTEGER, true)) .add("mp", new MapType(StringType.STRING, LongType.LONG, true)) .add("z", DoubleType.DOUBLE), new StructType() .add(StatsSchemaHelper.NUM_RECORDS, LongType.LONG, true) .add( StatsSchemaHelper.MIN, new StructType().add("z", DoubleType.DOUBLE, true), true) .add( StatsSchemaHelper.MAX, new StructType().add("z", DoubleType.DOUBLE, true), true) .add( StatsSchemaHelper.NULL_COUNT, new StructType() .add("arr", LongType.LONG, true) .add("mp", LongType.LONG, true) .add("z", LongType.LONG, true), true) .add(StatsSchemaHelper.TIGHT_BOUNDS, BooleanType.BOOLEAN, true)), // Nested array/map inside a struct; empty struct preserved in min/max ( new StructType() .add( "s", new StructType() .add("arr", new ArrayType(StringType.STRING, true)) .add("mp", new MapType(IntegerType.INTEGER, StringType.STRING, true))) .add("k", StringType.STRING), new StructType() .add(StatsSchemaHelper.NUM_RECORDS, LongType.LONG, true) .add( StatsSchemaHelper.MIN, new StructType() .add("s", new StructType(), true) .add("k", StringType.STRING, true), true) .add( StatsSchemaHelper.MAX, new StructType() .add("s", new StructType(), true) .add("k", StringType.STRING, true), true) .add( StatsSchemaHelper.NULL_COUNT, new StructType() .add( "s", new StructType() .add("arr", LongType.LONG, true) .add("mp", LongType.LONG, true), true) .add("k", LongType.LONG, true), true) .add(StatsSchemaHelper.TIGHT_BOUNDS, BooleanType.BOOLEAN, true))) testCases.foreach { case (dataSchema, expectedStatsSchema) => val statsSchema = StatsSchemaHelper.getStatsSchema( dataSchema, Set.empty[CollationIdentifier].asJava) assert( statsSchema == expectedStatsSchema, s"Stats schema mismatch for data schema: $dataSchema") } } test("check getStatsSchema with collations - un-nested mix") { val dataSchema = new StructType() .add("a", StringType.STRING) .add("b", IntegerType.INTEGER) .add("c", BinaryType.BINARY) .add("d", unicodeString) val skippableFields = new StructType() .add("a", StringType.STRING) .add("b", IntegerType.INTEGER) .add("d", unicodeString) val collations = Set(utf8Lcase, unicode, CollationIdentifier.SPARK_UTF8_BINARY) val expectedCollatedMinMax = new StructType() .add("a", StringType.STRING, true).add("d", unicodeString, true) val expectedStatsSchema = new StructType() .add(StatsSchemaHelper.NUM_RECORDS, LongType.LONG, true) .add( StatsSchemaHelper.MIN, skippableFields, true) .add( StatsSchemaHelper.MAX, skippableFields, true) .add( StatsSchemaHelper.NULL_COUNT, new StructType() .add("a", LongType.LONG, true) .add("b", LongType.LONG, true) .add("c", LongType.LONG, true) .add("d", LongType.LONG, true), true) .add(StatsSchemaHelper.TIGHT_BOUNDS, BooleanType.BOOLEAN, true) .add( StatsSchemaHelper.STATS_WITH_COLLATION, new StructType() .add( utf8Lcase.toString, new StructType() .add(StatsSchemaHelper.MIN, expectedCollatedMinMax, true) .add(StatsSchemaHelper.MAX, expectedCollatedMinMax, true), true) .add( unicode.toString, new StructType() .add(StatsSchemaHelper.MIN, expectedCollatedMinMax, true) .add(StatsSchemaHelper.MAX, expectedCollatedMinMax, true), true), true) val statsSchema = StatsSchemaHelper.getStatsSchema(dataSchema, collations.asJava) assert(statsSchema == expectedStatsSchema) } test("check getStatsSchema with collations - nested mix and multiple collations") { val dataSchema = new StructType() .add( "s", new StructType() .add("x", StringType.STRING) .add("y", IntegerType.INTEGER) .add("z", new StructType().add("p", StringType.STRING).add("q", DoubleType.DOUBLE))) .add("arr", new ArrayType(StringType.STRING, true)) .add("mp", new MapType(StringType.STRING, StringType.STRING, true)) val skippableFields = new StructType() .add( "s", new StructType() .add("x", StringType.STRING) .add("y", IntegerType.INTEGER) .add("z", new StructType().add("p", StringType.STRING).add("q", DoubleType.DOUBLE))) val collations = Set(utf8Lcase, CollationIdentifier.SPARK_UTF8_BINARY) val expectedCollatedNested = new StructType() .add( "s", new StructType() .add("x", StringType.STRING, true) .add("z", new StructType().add("p", StringType.STRING, true), true), true) val expectedStatsSchema = new StructType() .add(StatsSchemaHelper.NUM_RECORDS, LongType.LONG, true) .add( StatsSchemaHelper.MIN, skippableFields, true) .add( StatsSchemaHelper.MAX, skippableFields, true) .add( StatsSchemaHelper.NULL_COUNT, new StructType() .add( "s", new StructType() .add("x", LongType.LONG, true) .add("y", LongType.LONG, true) .add( "z", new StructType().add("p", LongType.LONG, true).add("q", LongType.LONG, true), true), true) .add("arr", LongType.LONG, true) .add("mp", LongType.LONG, true), true) .add(StatsSchemaHelper.TIGHT_BOUNDS, BooleanType.BOOLEAN, true) .add( StatsSchemaHelper.STATS_WITH_COLLATION, new StructType() .add( utf8Lcase.toString, new StructType() .add(StatsSchemaHelper.MIN, expectedCollatedNested, true) .add(StatsSchemaHelper.MAX, expectedCollatedNested, true), true), true) val statsSchema = StatsSchemaHelper.getStatsSchema(dataSchema, collations.asJava) assert(statsSchema == expectedStatsSchema) } test("check getStatsSchema with collations - no eligible string columns") { val dataSchema = new StructType() .add("a", IntegerType.INTEGER) .add("b", new ArrayType(StringType.STRING, true)) .add("c", new MapType(StringType.STRING, LongType.LONG, true)) val a = new StructType().add("a", IntegerType.INTEGER, true) val collations = Set(utf8Lcase, unicode, CollationIdentifier.SPARK_UTF8_BINARY) val expectedStatsSchema = new StructType() .add(StatsSchemaHelper.NUM_RECORDS, LongType.LONG, true) .add( StatsSchemaHelper.MIN, a, true) .add( StatsSchemaHelper.MAX, a, true) .add( StatsSchemaHelper.NULL_COUNT, new StructType() .add("a", LongType.LONG, true) .add("b", LongType.LONG, true) .add("c", LongType.LONG, true), true) .add(StatsSchemaHelper.TIGHT_BOUNDS, BooleanType.BOOLEAN, true) val statsSchema = StatsSchemaHelper.getStatsSchema(dataSchema, collations.asJava) assert(statsSchema == expectedStatsSchema) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/snapshot/LogSegmentSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.snapshot import java.lang.{Long => JLong} import java.util.{Collections, List => JList, Optional} import scala.collection.JavaConverters._ import io.delta.kernel.internal.files.{ParsedCatalogCommitData, ParsedDeltaData} import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.util.FileNames import io.delta.kernel.test.{MockFileSystemClientUtils, VectorTestUtils} import io.delta.kernel.utils.FileStatus import org.scalatest.funsuite.AnyFunSuite class LogSegmentSuite extends AnyFunSuite with MockFileSystemClientUtils with VectorTestUtils { private val checkpointFs10List = singularCheckpointFileStatuses(Seq(10)).toList.asJava private val checksumAtVersion10 = checksumFileStatus(10) private val deltaFs11List = deltaFileStatuses(Seq(11)).toList.asJava private val deltaFs12List = deltaFileStatuses(Seq(12)).toList.asJava private val deltasFs11To12List = deltaFileStatuses(Seq(11, 12)).toList.asJava private val parsedRatifiedCommits11To12List = Seq(11, 12).map(v => ParsedDeltaData.forFileStatus(stagedCommitFile(v))).asJava private val compactionFs11To12List = compactedFileStatuses(Seq((11, 12))).toList.asJava private val badJsonsList = Collections.singletonList( FileStatus.of(s"${logPath.toString}/gibberish.json", 1, 1)) private val badCheckpointsList = Collections.singletonList( FileStatus.of(s"${logPath.toString}/gibberish.checkpoint.parquet", 1, 1)) private val logPath2 = new Path("/another/fake/path/to/table/", "_delta_log") private def createLogSegmentForTest( logPath: Path = this.logPath, version: Long, deltas: JList[FileStatus] = Collections.emptyList(), compactions: JList[FileStatus] = Collections.emptyList(), checkpoints: JList[FileStatus] = Collections.emptyList(), deltaAtEndVersion: Option[FileStatus] = None, lastSeenChecksum: Optional[FileStatus] = Optional.empty(), maxPublishedDeltaVersion: Optional[JLong] = Optional.empty()): LogSegment = { val finalDeltaAtEndVersion = deltaAtEndVersion.getOrElse { if (!deltas.isEmpty()) { // If we have deltas, use the last delta deltas.get(deltas.size() - 1) } else if (!checkpoints.isEmpty()) { // If we only have checkpoints, create a delta file for the checkpoint version val checkpointVersion = io.delta.kernel.internal.util.FileNames.checkpointVersion( new Path(checkpoints.get(0).getPath())) deltaFileStatus(checkpointVersion) } else { // If neither deltas nor checkpoints are provided, create a delta for the target version deltaFileStatus(version) } } new LogSegment( logPath, version, deltas, compactions, checkpoints, finalDeltaAtEndVersion, lastSeenChecksum, maxPublishedDeltaVersion) } test("constructor -- valid case (non-empty)") { createLogSegmentForTest( version = 12, deltas = deltasFs11To12List, compactions = compactionFs11To12List, checkpoints = checkpointFs10List) } test("constructor -- null arguments => throw") { // logPath is null intercept[NullPointerException] { createLogSegmentForTest( logPath = null, version = 1) } // deltas is null intercept[NullPointerException] { createLogSegmentForTest( version = 1, deltas = null) } // compactions is null intercept[NullPointerException] { createLogSegmentForTest( version = 1, compactions = null) } // checkpoints is null intercept[NullPointerException] { createLogSegmentForTest( version = 1, checkpoints = null) } // deltaAtEndVersion is null intercept[NullPointerException] { createLogSegmentForTest( version = 1, deltas = Collections.singletonList(deltaFileStatus(1)), deltaAtEndVersion = null) } // lastSeenChecksum is null intercept[NullPointerException] { createLogSegmentForTest( version = 1, deltas = Collections.singletonList(deltaFileStatus(1)), lastSeenChecksum = null) } } test("constructor -- version must be >= 0") { val exMsg = intercept[IllegalArgumentException] { createLogSegmentForTest( version = -1, deltaAtEndVersion = Some(deltaFileStatus(0)) ) // dummy value }.getMessage assert(exMsg === "version must be >= 0") } test("constructor -- all deltas must be actual delta files") { val exMsg = intercept[IllegalArgumentException] { createLogSegmentForTest( version = 12, deltas = badJsonsList, checkpoints = checkpointFs10List) }.getMessage assert(exMsg.startsWith("deltas must all be actual delta (commit) files")) } test("constructor -- all checkpoints must be actual checkpoint files") { val exMsg = intercept[IllegalArgumentException] { createLogSegmentForTest( version = 12, deltas = deltasFs11To12List, checkpoints = badCheckpointsList) }.getMessage assert(exMsg.startsWith("checkpoints must all be actual checkpoint files")) } test("constructor -- deltas and checkpoints cannot be empty") { val exMsg = intercept[IllegalArgumentException] { createLogSegmentForTest(version = 12) }.getMessage assert(exMsg === "No files to read") } test("constructor -- checksum version must be <= LogSegment version") { val checksumAtVersion13 = checksumFileStatus(13) val exMsg = intercept[IllegalArgumentException] { createLogSegmentForTest( version = 12, // LogSegment version is 12 deltas = deltasFs11To12List, checkpoints = checkpointFs10List, lastSeenChecksum = Optional.of(checksumAtVersion13) ) // Checksum version is 13 }.getMessage assert(exMsg.contains( "checksum version (13) should be less than or equal to LogSegment version (12)")) } test("constructor -- deltaAtEndVersion must match version (checkpoint only)") { val exMsg = intercept[IllegalArgumentException] { createLogSegmentForTest( version = 10, checkpoints = checkpointFs10List, deltaAtEndVersion = Some(deltaFileStatus(9)) // Wrong version - should be 10 ) }.getMessage assert(exMsg.contains( "deltaAtEndVersion (9) must be equal to LogSegment version (10)")) } test("constructor -- deltaAtEndVersion must match version (checkpoint + deltas)") { val exMsg = intercept[IllegalArgumentException] { createLogSegmentForTest( version = 12, deltas = deltasFs11To12List, checkpoints = checkpointFs10List, deltaAtEndVersion = Some(deltaFileStatus(11)) // Wrong version - should be 12 ) }.getMessage assert(exMsg.contains( "deltaAtEndVersion (11) must be equal to LogSegment version (12)")) } test("constructor -- checksum version must be >= checkpoint version") { val checksumAtVersion9 = checksumFileStatus(9) val exMsg = intercept[IllegalArgumentException] { createLogSegmentForTest( version = 12, deltas = deltasFs11To12List, checkpoints = checkpointFs10List, // Checkpoint version is 10 lastSeenChecksum = Optional.of(checksumAtVersion9) ) // Checksum version is 9 }.getMessage assert(exMsg.contains( "checksum version (9) should be greater than or equal to checkpoint version (10)")) } test("constructor -- if deltas non-empty then first delta must equal checkpointVersion + 1") { val exMsg = intercept[IllegalArgumentException] { createLogSegmentForTest( version = 12, deltas = deltaFs12List, checkpoints = checkpointFs10List) }.getMessage assert(exMsg.contains( "First delta file version (12) must equal checkpointVersion + 1 (11)")) } test("constructor -- if deltas non-empty then last delta must equal version") { val exMsg = intercept[IllegalArgumentException] { createLogSegmentForTest( version = 12, deltas = deltaFs11List, checkpoints = checkpointFs10List) }.getMessage assert(exMsg.contains( "Last delta file version (11) must equal LogSegment version (12)")) } test("constructor -- if no deltas then checkpointVersion must equal version") { val exMsg = intercept[IllegalArgumentException] { createLogSegmentForTest( version = 11, checkpoints = checkpointFs10List) }.getMessage assert(exMsg.contains( "If no deltas, then checkpointVersion (10) must equal LogSegment version (11)")) } test("constructor -- deltas not contiguous") { val deltas = deltaFileStatuses(Seq(11, 13)).toList.asJava val exMsg = intercept[IllegalArgumentException] { createLogSegmentForTest( version = 13, deltas = deltas, checkpoints = checkpointFs10List) }.getMessage assert(exMsg === "Delta versions must be contiguous: [11, 13]") } test("constructor -- delta commit files (JSON) outside of log path") { val deltasForDifferentTable = deltaFileStatuses(Seq(11, 12), logPath2).toList.asJava val ex = intercept[RuntimeException] { createLogSegmentForTest( version = 12, deltas = deltasForDifferentTable, checkpoints = checkpointFs10List) } assert(ex.getMessage.contains("doesn't belong in the transaction log")) } test("constructor -- compaction log files outside of log path") { val compactionsForDifferentTable = compactedFileStatuses(Seq((11, 12)), logPath2).toList.asJava val ex = intercept[RuntimeException] { createLogSegmentForTest( version = 12, deltas = deltasFs11To12List, compactions = compactionsForDifferentTable, checkpoints = checkpointFs10List) } assert(ex.getMessage.contains("doesn't belong in the transaction log")) } test("constructor -- checkpoint files (parquet) outside of log path") { val checkpointsForDifferentTable = singularCheckpointFileStatuses(Seq(10), logPath2).toList.asJava val ex = intercept[RuntimeException] { createLogSegmentForTest( version = 12, deltas = deltasFs11To12List, checkpoints = checkpointsForDifferentTable) } assert(ex.getMessage.contains("doesn't belong in the transaction log")) } test("toString") { val logSegment = createLogSegmentForTest( version = 12, deltas = deltasFs11To12List, checkpoints = checkpointFs10List, lastSeenChecksum = Optional.of(checksumAtVersion10), maxPublishedDeltaVersion = Optional.of(12L)) // scalastyle:off line.size.limit val expectedToString = """LogSegment { | logPath='/fake/path/to/table/_delta_log', | version=12, | deltas=[ | FileStatus{path='/fake/path/to/table/_delta_log/00000000000000000011.json', size=11, modificationTime=110}, | FileStatus{path='/fake/path/to/table/_delta_log/00000000000000000012.json', size=12, modificationTime=120} | ], | checkpoints=[ | FileStatus{path='/fake/path/to/table/_delta_log/00000000000000000010.checkpoint.parquet', size=10, modificationTime=100} | ], | deltaAtEndVersion=FileStatus{path='/fake/path/to/table/_delta_log/00000000000000000012.json', size=12, modificationTime=120}, | lastSeenChecksum=FileStatus{path='/fake/path/to/table/_delta_log/00000000000000000010.crc', size=10, modificationTime=10}, | checkpointVersion=10, | maxPublishedDeltaVersion=12 |}""".stripMargin // scalastyle:on line.size.limit assert(logSegment.toString === expectedToString) } private def parseExpectedString(expected: String): JList[FileStatus] = { expected.split(",").map(_.trim).map { item => if (item.contains("-")) { // compaction file contains a - val parts = item.split("-").map(_.trim.toLong) logCompactionStatus(parts(0), parts(1)) } else { // delta file does not deltaFileStatus(item.toLong) } }.toList.asJava } private def testCompactionCase( deltas: Seq[Long], compactions: Seq[(Long, Long)], expected: String): Unit = { val version = deltas.max val deltas_list = deltaFileStatuses(deltas).toList.asJava val compactions_list = compactedFileStatuses(compactions).toList.asJava val segment = createLogSegmentForTest( version = version, deltas = deltas_list, compactions = compactions_list) val expectedFiles = parseExpectedString(expected) assert(segment.allFilesWithCompactionsReversed() === expectedFiles) } test("allFilesWithCompactionsReversed -- 3 - 5 in middle") { testCompactionCase( Seq.range(0, 7), Seq((3, 5)), "6, 3-5, 2, 1, 0") } test("allFilesWithCompactionsReversed -- 3 - 5 at start") { testCompactionCase( Seq.range(3, 8), Seq((3, 5)), "7, 6, 3-5") } test("allFilesWithCompactionsReversed -- 3 - 5 at end") { testCompactionCase( Seq.range(0, 6), Seq((3, 5)), "3-5, 2, 1, 0") } test("allFilesWithCompactionsReversed -- 3 - 5 at second to last") { testCompactionCase( Seq.range(2, 7), Seq((3, 5)), "6, 3-5, 2") } test("allFilesWithCompactionsReversed -- 3 - 5, and 7 - 9") { testCompactionCase( Seq.range(1, 11), Seq((3, 5), (7, 9)), "10, 7-9, 6, 3-5, 2, 1") } test("allFilesWithCompactionsReversed -- 3 - 5, and 4 - 8 (overlap)") { testCompactionCase( Seq.range(2, 11), Seq((3, 5), (4, 8)), "10, 9, 4-8, 3, 2") } test("allFilesWithCompactionsReversed -- 3 - 5, whole range") { testCompactionCase( Seq.range(3, 6), Seq((3, 5)), "3-5") } test("allFilesWithCompactionsReversed -- consecutive compactions") { testCompactionCase( Seq.range(0, 13), Seq((3, 5), (6, 8), (9, 11)), "12, 9-11, 6-8, 3-5, 2, 1, 0") } test("allFilesWithCompactionsReversed -- contained range") { testCompactionCase( Seq.range(1, 12), Seq((2, 10), (4, 8)), "11, 2-10, 1") } test("allFilesWithCompactionsReversed -- complex ranges") { testCompactionCase( Seq.range(0, 21), Seq((1, 3), (1, 5), (7, 10), (11, 14), (11, 12), (16, 20), (18, 20)), "16-20, 15, 11-14, 7-10, 6, 1-5, 0") } test("assertLogFilesBelongToTable should pass for correct log paths") { val tablePath = new Path("s3://bucket/logPath") val logFiles = List( FileStatus.of("s3://bucket/logPath/deltafile1", 0L, 0L), FileStatus.of("s3://bucket/logPath/deltafile2", 0L, 0L), FileStatus.of("s3://bucket/logPath/checkpointfile1", 0L, 0L), FileStatus.of("s3://bucket/logPath/checkpointfile2", 0L, 0L)).asJava LogSegment.assertLogFilesBelongToTable(tablePath, logFiles) } test("assertLogFilesBelongToTable should fail for incorrect log paths") { val tablePath = new Path("s3://bucket/logPath") val logFiles = List( FileStatus.of("s3://bucket/logPath/deltafile1", 0L, 0L), FileStatus.of("s3://bucket/invalidLogPath/deltafile2", 0L, 0L), FileStatus.of("s3://bucket/logPath/checkpointfile1", 0L, 0L), FileStatus.of("s3://bucket/invalidLogPath/checkpointfile2", 0L, 0L)).asJava // Test that files with incorrect log paths trigger the assertion val ex = intercept[RuntimeException] { LogSegment.assertLogFilesBelongToTable(tablePath, logFiles) } assert(ex.getMessage.contains("File (s3://bucket/invalidLogPath/deltafile2) " + s"doesn't belong in the transaction log at $tablePath")) } //////////////////////////////////// // copyWithAdditionalDeltas tests // //////////////////////////////// test("copyWithAdditionalDeltas: single additional delta") { val baseSegment = createLogSegmentForTest( version = 10, checkpoints = checkpointFs10List) val updated = baseSegment.newWithAddedDeltas(parsedRatifiedCommits11To12List.subList(0, 1)) assert(updated.getVersion === 11) assert(updated.getDeltas.size() === 1) } test("copyWithAdditionalDeltas: multiple additional deltas") { val baseSegment = createLogSegmentForTest( version = 10, checkpoints = checkpointFs10List) val updated = baseSegment.newWithAddedDeltas(parsedRatifiedCommits11To12List) assert(updated.getVersion === 12) assert(updated.getDeltas.size() === 2) } test("copyWithAdditionalDeltas: empty list returns same segment") { val baseSegment = createLogSegmentForTest( version = 10, checkpoints = checkpointFs10List) val updated = baseSegment.newWithAddedDeltas(Collections.emptyList()) assert(updated eq baseSegment) } test("copyWithAdditionalDeltas: first delta must be version + 1") { val baseSegment = createLogSegmentForTest( version = 10, checkpoints = checkpointFs10List) val wrongVersionDeltas = List(ParsedDeltaData.forFileStatus(stagedCommitFile(12))).asJava val exMsg = intercept[IllegalArgumentException] { baseSegment.newWithAddedDeltas(wrongVersionDeltas) }.getMessage assert(exMsg.contains("Expected 11 but got 12")) } test("copyWithAdditionalDeltas: deltas must be contiguous") { val baseSegment = createLogSegmentForTest( version = 10, checkpoints = checkpointFs10List) val nonContiguousDeltas = List( ParsedDeltaData.forFileStatus(stagedCommitFile(11)), ParsedDeltaData.forFileStatus(stagedCommitFile(13))).asJava val exMsg = intercept[IllegalArgumentException] { baseSegment.newWithAddedDeltas(nonContiguousDeltas) }.getMessage assert(exMsg.contains("Delta versions must be contiguous. Expected 12 but got 13")) } test("copyWithAdditionalDeltas: inline delta fails") { val baseSegment = createLogSegmentForTest( version = 10, checkpoints = checkpointFs10List) val inlineDelta = ParsedCatalogCommitData.forInlineData(11, emptyColumnarBatch) val inlineDeltas = List[ParsedDeltaData](inlineDelta).asJava val exMsg = intercept[IllegalArgumentException] { baseSegment.newWithAddedDeltas(inlineDeltas) }.getMessage assert(exMsg.contains("Currently, only file-based deltas are supported")) } /////////////////////////// // fromSingleDelta tests // /////////////////////////// test("fromSingleDelta -- creates valid LogSegment") { val deltaData = ParsedDeltaData.forFileStatus(deltaFileStatus(0)) val logSegment = LogSegment.createForNewTable(logPath, deltaData) assert(logSegment.getVersion === 0) assert(logSegment.getDeltas.size() === 1) assert(logSegment.getCheckpoints.isEmpty) assert(logSegment.getCompactions.isEmpty) assert(logSegment.getLastSeenChecksum === Optional.empty()) } test("fromSingleDelta -- non-zero version fails") { val deltaData = ParsedDeltaData.forFileStatus(deltaFileStatus(1)) val exMsg = intercept[IllegalArgumentException] { LogSegment.createForNewTable(logPath, deltaData) }.getMessage assert(exMsg.contains("Version must be 0 for a LogSegment with only a single delta")) } test("fromSingleDelta -- inline delta fails") { val inlineDelta = ParsedCatalogCommitData.forInlineData(0, emptyColumnarBatch) val exMsg = intercept[IllegalArgumentException] { LogSegment.createForNewTable(logPath, inlineDelta) }.getMessage assert(exMsg.contains("Currently, only file-based deltas are supported")) } //////////////////////////////// // newAsPublished tests // //////////////////////////////// test("newAsPublished: list all files when there's no checkpoint") { val commits = (0 to 10).map(i => FileStatus.of(s"$logPath/$i.json")) ++ (11 to 20).map(i => FileStatus.of(s"$logPath/$i.${java.util.UUID.randomUUID.toString}.json")) val baseSegment = createLogSegmentForTest( version = 20, deltas = commits.asJava, deltaAtEndVersion = Some(commits.last), maxPublishedDeltaVersion = Optional.of(10)) val updated = baseSegment.newAsPublished() assert(updated.getVersion === 20) assert(updated.getDeltas.size() === 21) updated.getDeltas.asScala.zipWithIndex.foreach { case (fs, i) => assert(fs.getPath == FileNames.deltaFile(logPath, i)) } assert(updated.getDeltaFileAtEndVersion.getPath == FileNames.deltaFile(logPath, 20)) assert(updated.getMaxPublishedDeltaVersion == Optional.of(20L)) } test("newAsPublished: list all files starting from checkpoint") { val commits = (11 until 15).map(i => FileStatus.of(s"$logPath/$i.json")) ++ (15 to 20).map(i => FileStatus.of(s"$logPath/$i.${java.util.UUID.randomUUID.toString}.json")) val baseSegment = createLogSegmentForTest( version = 20, deltas = commits.asJava, deltaAtEndVersion = Some(commits.last), checkpoints = checkpointFs10List, maxPublishedDeltaVersion = Optional.of(15)) val updated = baseSegment.newAsPublished() assert(updated.getVersion === 20) assert(updated.getDeltas.size() === 10) updated.getDeltas.asScala.zipWithIndex.foreach { case (fs, i) => assert(fs.getPath == FileNames.deltaFile(logPath, i + 11)) } assert(updated.getDeltaFileAtEndVersion.getPath == FileNames.deltaFile(logPath, 20)) assert(updated.getMaxPublishedDeltaVersion == Optional.of(20L)) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/snapshot/MetadataCleanupSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.snapshot import io.delta.kernel.internal.snapshot.MetadataCleanup.cleanupExpiredLogs import io.delta.kernel.internal.util.ManualClock import io.delta.kernel.test.{MockFileSystemClientUtils, MockListFromDeleteFileSystemClient} import io.delta.kernel.utils.FileStatus import org.scalatest.funsuite.AnyFunSuite /** * Test suite for the metadata cleanup logic in the Delta log directory. It mocks the * `FileSystemClient` to test the cleanup logic for various combinations of delta files and * checkpoint files. Utility methods in `MockFileSystemClientUtils` are used to generate the * log file statuses which usually have modification time as the `version * 10`. */ class MetadataCleanupSuite extends AnyFunSuite with MockFileSystemClientUtils { import MetadataCleanupSuite._ /* ------------------- TESTS ------------------ */ // Simple case where the Delta log directory contains only delta files and no checkpoint files Seq( ( "no files should be deleted even some of them are expired", DeletedFileList(), // expected deleted files - none of them should be deleted 70, // current time 30 // retention period ), ( "no files should be deleted as none of them are expired", DeletedFileList(), // expected deleted files - none of them should be deleted 200, // current time 200 // retention period ), ( "no files should be deleted as none of them are expired", DeletedFileList(), // expected deleted files - none of them should be deleted 200, // current time 0 // retention period )).foreach { case (testName, expectedDeletedFiles, currentTime, retentionPeriod) => // _deltalog directory contents - contains only delta files val logFiles = deltaFileStatuses(Seq(0, 1, 2, 3, 4, 5, 6)) test(s"metadataCleanup: $testName: $currentTime, $retentionPeriod") { cleanupAndVerify(logFiles, expectedDeletedFiles.fileList(), currentTime, retentionPeriod) } } // with various checkpoint types Seq("classic", "multi-part", "v2", "hybrid").foreach { checkpointType => // _deltalog directory contains a combination of delta files and checkpoint files val logFiles = deltaFileStatuses(Seq(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12)) ++ (checkpointType match { case "classic" => singularCheckpointFileStatuses(Seq(3, 6, 9, 12)) case "multi-part" => multiCheckpointFileStatuses(Seq(3, 6, 9, 12), multiPartCheckpointPartsSize) case "v2" => v2CPFileStatuses(Seq[Long](3, 6, 9, 12)) case "hybrid" => singularCheckpointFileStatuses(Seq(3)) ++ multiCheckpointFileStatuses(Seq(6), numParts = multiPartCheckpointPartsSize) ++ v2CPFileStatuses(Seq[Long](9)) ++ singularCheckpointFileStatuses(Seq(12)) }) // test cases Seq( ( "delete expired delta files up to the checkpoint version, " + "not all expired delta files are deleted", Seq(0L, 1L, 2L), // expDeletedDeltaVersions, Seq(), // expDeletedCheckpointVersions, 130, // current time 80 // retention period ), ( "expired delta files + expired checkpoint should be deleted", Seq(0L, 1L, 2L, 3L, 4L, 5L), // expDeletedDeltaVersions, Seq(3L), // expDeletedCheckpointVersions, 130, // current time 60 // retention period ), ( "expired delta files + expired checkpoints should be deleted", Seq(0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L), // expDeletedDeltaVersions, Seq(3L, 6L), // expDeletedCheckpointVersions, 130, // current time 40 // retention period ), ( "all delta/checkpoint files should be except the last checkpoint file", Seq(0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L), // expDeletedDeltaVersions, Seq(3L, 6L, 9L), // expDeletedCheckpointVersions, 130, // current time 0 // retention period ), ( "no delta/checkpoint files should be deleted as none expired", Seq(), // expDeletedDeltaVersions Seq(), // expDeletedDeltaVersions 200, // current time 200 // retention period )).foreach { case ( testName, expDeletedDeltaVersions, expDeletedCheckpointVersions, currentTime, retentionPeriod) => val expectedDeletedFiles = DeletedFileList( deltaVersions = expDeletedDeltaVersions, classicCheckpointVersions = checkpointType match { case "classic" => expDeletedCheckpointVersions case "hybrid" => expDeletedCheckpointVersions.filter(Seq(3, 12).contains(_)) case _ => Seq.empty }, multipartCheckpointVersions = checkpointType match { case "multi-part" => expDeletedCheckpointVersions case "hybrid" => expDeletedCheckpointVersions.filter(_ == 6) case _ => Seq.empty }, v2CheckpointVersions = checkpointType match { case "v2" => expDeletedCheckpointVersions case "hybrid" => expDeletedCheckpointVersions.filter(_ == 9) case _ => Seq.empty }) test(s"metadataCleanup: $checkpointType: $testName: $currentTime, $retentionPeriod") { cleanupAndVerify(logFiles, expectedDeletedFiles.fileList(), currentTime, retentionPeriod) } } } test("first log entry is a checkpoint") { val logFiles = multiCheckpointFileStatuses(Seq(25), multiPartCheckpointPartsSize) ++ singularCheckpointFileStatuses(Seq(29)) ++ deltaFileStatuses(Seq(25, 26, 27, 28, 29, 30, 31, 32)) Seq( ( 330, // current time 50, // retention period DeletedFileList() // expected deleted files - none of them should be deleted ), ( 330, // current time 30, // retention period DeletedFileList( deltaVersions = Seq(25, 26, 27, 28), multipartCheckpointVersions = Seq(25))), ( 330, // current time 10, // retention period DeletedFileList( deltaVersions = Seq(25, 26, 27, 28), multipartCheckpointVersions = Seq(25)))).foreach { case (currentTime, retentionPeriod, expectedDeletedFiles) => cleanupAndVerify(logFiles, expectedDeletedFiles.fileList(), currentTime, retentionPeriod) } } /* ------------------- NEGATIVE TESTS ------------------ */ test("metadataCleanup: invalid retention period") { val e = intercept[IllegalArgumentException] { cleanupExpiredLogs( mockEngine(mockFsClient(Seq.empty)), new ManualClock(100), logPath, -1 /* retentionPeriod */ ) } assert(e.getMessage.contains("Retention period must be non-negative")) } test("incomplete checkpoints should not be considered") { val logFiles = deltaFileStatuses(Seq(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12)) ++ multiCheckpointFileStatuses(Seq(3), multiPartCheckpointPartsSize) // delete the third part of the checkpoint .filterNot(_.getPath.contains(s"%010d.%010d".format(2, 4))) ++ multiCheckpointFileStatuses(Seq(6), multiPartCheckpointPartsSize) ++ v2CPFileStatuses(Seq(9)) // test cases Seq( ( Seq[Long](), // expDeletedDeltaVersions, Seq[Long](), // expDeletedCheckpointVersions, 130, // current time 80 // retention period ), ( Seq(0L, 1L, 2L, 3L, 4L, 5L), // expDeletedDeltaVersions, Seq(3L), // expDeletedCheckpointVersions, 130, // current time 60 // retention period ), ( Seq(0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L), // expDeletedDeltaVersions, Seq(3L, 6L), // expDeletedCheckpointVersions, 130, // current time 20 // retention period )).foreach { case (expDeletedDeltaVersions, expDeletedCheckpointVersions, currentTime, retentionPeriod) => val expectedDeletedFiles = (deltaFileStatuses(expDeletedDeltaVersions) ++ expDeletedCheckpointVersions.flatMap { case v @ 3 => multiCheckpointFileStatuses(Seq(v), multiPartCheckpointPartsSize) .filterNot(_.getPath.contains(s"%010d.%010d".format(2, 4))) case v @ 6 => multiCheckpointFileStatuses(Seq(v), multiPartCheckpointPartsSize) case v @ 9 => v2CPFileStatuses(Seq(v)) }).map(_.getPath) cleanupAndVerify(logFiles, expectedDeletedFiles, currentTime, retentionPeriod) } } /* ------------------- HELPER UTILITIES/CONSTANTS ------------------ */ /** * Cleanup the metadata log files and verify the expected deleted files. * * @param logFiles List of log files in the _delta_log directory * @param expectedDeletedFiles List of expected deleted file paths * @param currentTimeMillis Current time in millis * @param retentionPeriodMillis Retention period in millis */ def cleanupAndVerify( logFiles: Seq[FileStatus], expectedDeletedFiles: Seq[String], currentTimeMillis: Long, retentionPeriodMillis: Long): Unit = { val fsClient = mockFsClient(logFiles) val resultDeletedCount = cleanupExpiredLogs( mockEngine(fsClient), new ManualClock(currentTimeMillis), logPath, retentionPeriodMillis) assert(resultDeletedCount === expectedDeletedFiles.size) assert(fsClient.getDeleteCalls.toSet === expectedDeletedFiles.toSet) } } object MetadataCleanupSuite extends MockFileSystemClientUtils { /* ------------------- HELPER UTILITIES/CONSTANTS ------------------ */ private val multiPartCheckpointPartsSize = 4 /** Case class containing the list of expected files in the deleted metadata log file list */ case class DeletedFileList( deltaVersions: Seq[Long] = Seq.empty, classicCheckpointVersions: Seq[Long] = Seq.empty, multipartCheckpointVersions: Seq[Long] = Seq.empty, v2CheckpointVersions: Seq[Long] = Seq.empty) { def fileList(): Seq[String] = { (deltaFileStatuses(deltaVersions) ++ singularCheckpointFileStatuses(classicCheckpointVersions) ++ multiCheckpointFileStatuses(multipartCheckpointVersions, multiPartCheckpointPartsSize) ++ v2CPFileStatuses(v2CheckpointVersions)).sortBy(_.getPath).map(_.getPath) } } def mockFsClient(logFiles: Seq[FileStatus]): MockListFromDeleteFileSystemClient = { new MockListFromDeleteFileSystemClient(logFiles) } def v2CPFileStatuses(versions: Seq[Long]): Seq[FileStatus] = { // Replace the UUID with a standard UUID to make the test deterministic val standardUUID = "123e4567-e89b-12d3-a456-426614174000" val uuidPattern = "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}".r v2CheckpointFileStatuses( versions.map(v => (v, true, 20)), // to (version, useUUID, numSidecars) "parquet").map(_._1) .map(f => FileStatus.of( uuidPattern.replaceAllIn(f.getPath, standardUUID), f.getSize, f.getModificationTime)) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/stats/FileSizeHistogramSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.stats import scala.collection.JavaConverters._ import scala.collection.Searching.search import io.delta.kernel.internal.util.VectorUtils import io.delta.kernel.internal.util.VectorUtils.toJavaList import org.scalatest.funsuite.AnyFunSuite class FileSizeHistogramSuite extends AnyFunSuite { implicit class HistogramOps(histogram: FileSizeHistogram) { def getSortedBinBoundaries: List[Long] = { toJavaList(histogram.toRow().getArray( FileSizeHistogram.FULL_SCHEMA.indexOf("sortedBinBoundaries"))).asScala.toList } def getFileCounts: List[Long] = { toJavaList(histogram.toRow().getArray( FileSizeHistogram.FULL_SCHEMA.indexOf("fileCounts"))).asScala.toList } def getTotalBytes: List[Long] = { toJavaList(histogram.toRow().getArray( FileSizeHistogram.FULL_SCHEMA.indexOf("totalBytes"))).asScala.toList } } private val KB = 1024L private val MB = KB * 1024 private val GB = MB * 1024 test("createDefaultHistogram should create histogram with zero counts and bytes") { val histogram = FileSizeHistogram.createDefaultHistogram() assert(histogram.getFileCounts.forall(_ == 0L)) assert(histogram.getTotalBytes.forall(_ == 0L)) assert(histogram.getSortedBinBoundaries == List( 0L, // Power of 2 till 4 MB 8 * KB, 16 * KB, 32 * KB, 64 * KB, 128 * KB, 256 * KB, 512 * KB, 1 * MB, 2 * MB, 4 * MB, // 4 MB jumps till 40 MB 8 * MB, 12 * MB, 16 * MB, 20 * MB, 24 * MB, 28 * MB, 32 * MB, 36 * MB, 40 * MB, // 8 MB jumps till 120 MB 48 * MB, 56 * MB, 64 * MB, 72 * MB, 80 * MB, 88 * MB, 96 * MB, 104 * MB, 112 * MB, 120 * MB, // 4 MB jumps till 144 MB 124 * MB, 128 * MB, 132 * MB, 136 * MB, 140 * MB, 144 * MB, // 16 MB jumps till 576 MB 160 * MB, 176 * MB, 192 * MB, 208 * MB, 224 * MB, 240 * MB, 256 * MB, 272 * MB, 288 * MB, 304 * MB, 320 * MB, 336 * MB, 352 * MB, 368 * MB, 384 * MB, 400 * MB, 416 * MB, 432 * MB, 448 * MB, 464 * MB, 480 * MB, 496 * MB, 512 * MB, 528 * MB, 544 * MB, 560 * MB, 576 * MB, // 64 MB jumps till 1408 MB 640 * MB, 704 * MB, 768 * MB, 832 * MB, 896 * MB, 960 * MB, 1024 * MB, 1088 * MB, 1152 * MB, 1216 * MB, 1280 * MB, 1344 * MB, 1408 * MB, // 128 MB jumps till 2 GB 1536 * MB, 1664 * MB, 1792 * MB, 1920 * MB, 2048 * MB, // 256 MB jumps till 4 GB 2304 * MB, 2560 * MB, 2816 * MB, 3072 * MB, 3328 * MB, 3584 * MB, 3840 * MB, 4 * GB, // power of 2 till 256 GB 8 * GB, 16 * GB, 32 * GB, 64 * GB, 128 * GB, 256 * GB)) } test("basic insert") { val histogram = FileSizeHistogram.createDefaultHistogram() val testSizes = List( 512L, // Small file 8L * KB, // 8KB 1L * MB, // 1MB 128L * MB, // 128MB 1L * GB, // 1GB 10L * GB // 10GB ) // Test single insertions with different bins testSizes.foreach { size => histogram.insert(size) val index = getBinIndexForTesting(histogram.getSortedBinBoundaries, size) assert(histogram.getFileCounts(index) == 1L) assert(histogram.getTotalBytes(index) == size) } // Test multiple insertions in same bin val sizeForMultiple = 1L * MB val numFiles = 5 val index = getBinIndexForTesting(histogram.getSortedBinBoundaries, sizeForMultiple) val initialCount = histogram.getFileCounts(index) val initialBytes = histogram.getTotalBytes(index) (1 to numFiles).foreach { i => histogram.insert(sizeForMultiple) assert(histogram.getFileCounts(index) == initialCount + i) assert(histogram.getTotalBytes(index) == initialBytes + (i * sizeForMultiple)) } // Test negative file size intercept[IllegalArgumentException] { histogram.insert(-1) } } test("insert the boundary") { val histogram = FileSizeHistogram.createDefaultHistogram() val boundaries = histogram.getSortedBinBoundaries boundaries.foreach { boundary => histogram.insert(boundary) val index = getBinIndexForTesting(boundaries, boundary) assert(histogram.getFileCounts(index) == 1) } } test("insert very large file") { val histogram = FileSizeHistogram.createDefaultHistogram() val veryLargeSize = 256L * GB histogram.insert(veryLargeSize) val lastIndex = histogram.getSortedBinBoundaries.size - 1 assert(histogram.getFileCounts(lastIndex) == 1) } test("remove should handle file sizes correctly") { val histogram = FileSizeHistogram.createDefaultHistogram() val fileSize = 1L * MB val index = getBinIndexForTesting(histogram.getSortedBinBoundaries, fileSize) // Test multiple inserts and removes val numFiles = 3 (1 to numFiles).foreach(_ => histogram.insert(fileSize)) (1 to numFiles).foreach { i => histogram.remove(fileSize) val remainingFiles = numFiles - i assert(histogram.getFileCounts(index) == remainingFiles) assert(histogram.getTotalBytes(index) == fileSize * remainingFiles) } assert(histogram.getTotalBytes( getBinIndexForTesting(histogram.getSortedBinBoundaries, fileSize)) === 0) // Test error cases intercept[IllegalArgumentException] { histogram.remove(fileSize) // Try to remove from empty bin } histogram.insert(fileSize) assert( getBinIndexForTesting(histogram.getSortedBinBoundaries, fileSize) === getBinIndexForTesting(histogram.getSortedBinBoundaries, fileSize + 1)) intercept[IllegalArgumentException] { histogram.remove(fileSize + 1) // Try to remove more bytes than available } intercept[IllegalArgumentException] { histogram.remove(-1) // Try to remove negative size } } test("histogram should be identical after serialization round trip") { val histogram = FileSizeHistogram.createDefaultHistogram() List( (512L, 5), // 5 files of 512B (1 * MB, 3), // 3 files of 1MB (10 * MB, 2) // 2 files of 10MB ).foreach { case (size, count) => (1 to count).foreach(_ => histogram.insert(size)) } val reconstructedHistogram = FileSizeHistogram.fromColumnVector( VectorUtils.buildColumnVector( Seq(histogram.toRow).toList.asJava, FileSizeHistogram.FULL_SCHEMA), 0) assert(reconstructedHistogram.isPresent) assert(histogram === reconstructedHistogram.get()) } test("plus should combine histograms correctly") { // Create two histograms val histogram1 = FileSizeHistogram.createDefaultHistogram() val histogram2 = FileSizeHistogram.createDefaultHistogram() // Add data to first histogram histogram1.insert(1 * MB) histogram1.insert(1 * MB) histogram1.insert(10 * MB) // Add data to second histogram histogram2.insert(2 * MB) histogram2.insert(10 * MB) histogram2.insert(10 * MB) // Combine histograms val combined = histogram1.plus(histogram2) // Check results val mbBinIndex1 = getBinIndexForTesting(combined.getSortedBinBoundaries, 1 * MB) val mbBinIndex2 = getBinIndexForTesting(combined.getSortedBinBoundaries, 2 * MB) val mbBinIndex10 = getBinIndexForTesting(combined.getSortedBinBoundaries, 10 * MB) assert(combined.getFileCounts(mbBinIndex1) == 2) assert(combined.getTotalBytes(mbBinIndex1) == 2 * MB) assert(combined.getFileCounts(mbBinIndex2) == 1) assert(combined.getTotalBytes(mbBinIndex2) == 2 * MB) assert(combined.getFileCounts(mbBinIndex10) == 3) assert(combined.getTotalBytes(mbBinIndex10) == 30 * MB) // check commutative. assert(combined === histogram2.plus(histogram1)) // Test error case - different bin boundaries val customBoundaries = Array(0L, 1L * KB, 1L * MB) val customCounts = new Array[Long](customBoundaries.length) val customBytes = new Array[Long](customBoundaries.length) val customHistogram = new FileSizeHistogram(customBoundaries, customCounts, customBytes) intercept[IllegalArgumentException] { histogram1.plus(customHistogram) } } test("minus should subtract histograms correctly") { // Create two histograms val histogram1 = FileSizeHistogram.createDefaultHistogram() val histogram2 = FileSizeHistogram.createDefaultHistogram() // Add data to first histogram histogram1.insert(1 * MB) histogram1.insert(1 * MB) histogram1.insert(1 * MB) histogram1.insert(10 * MB) histogram1.insert(10 * MB) // Add subset of data to second histogram histogram2.insert(1 * MB) histogram2.insert(10 * MB) // Check plus and minus are opposite operation. assert(histogram1 === histogram1.plus(histogram2).minus(histogram2)) // Subtract histograms val result = histogram1.minus(histogram2) // Check results val mbBinIndex1 = getBinIndexForTesting(result.getSortedBinBoundaries, 1 * MB) val mbBinIndex10 = getBinIndexForTesting(result.getSortedBinBoundaries, 10 * MB) assert(result.getFileCounts(mbBinIndex1) == 2) assert(result.getTotalBytes(mbBinIndex1) == 2 * MB) assert(result.getFileCounts(mbBinIndex10) == 1) assert(result.getTotalBytes(mbBinIndex10) == 10 * MB) // Test error cases // different bin boundaries val customBoundaries = Array(0L, 1L * KB, 1L * MB) val customCounts = new Array[Long](customBoundaries.length) val customBytes = new Array[Long](customBoundaries.length) val customHistogram = new FileSizeHistogram(customBoundaries, customCounts, customBytes) intercept[IllegalArgumentException] { histogram1.minus(customHistogram) } // Negative result scenario val largerHistogram = FileSizeHistogram.createDefaultHistogram() largerHistogram.insert(1 * MB) largerHistogram.insert(1 * MB) intercept[IllegalArgumentException] { // Try to subtract more than what exists FileSizeHistogram.createDefaultHistogram().minus(largerHistogram) } } /** * Determines the bin index for a given file size using binary search. * Returns the index of the largest bin boundary that is less than or equal to the file size. */ private def getBinIndexForTesting(boundaries: List[Long], fileSize: Long): Int = { import scala.collection.Searching.{Found, InsertionPoint} boundaries.search(fileSize) match { // Exact match found, return the matching index case Found(index) => index // Not found and got insert point. // Return the index of the bin boundary just below the file size (insert point - 1) case InsertionPoint(index) => index - 1 } } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.tablefeatures import java.util.{Collections, Optional} import java.util.Collections.{emptySet, singleton} import java.util.stream.Collectors.toList import scala.collection.JavaConverters._ import io.delta.kernel.data.{ArrayValue, ColumnVector, MapValue} import io.delta.kernel.exceptions.KernelException import io.delta.kernel.internal.actions.{Format, Metadata, Protocol} import io.delta.kernel.internal.tablefeatures.TableFeatures.{validateKernelCanReadTheTable, validateKernelCanWriteToTable, TABLE_FEATURES} import io.delta.kernel.internal.util.InternalUtils.singletonStringColumnVector import io.delta.kernel.internal.util.VectorUtils.buildColumnVector import io.delta.kernel.types._ import org.scalatest.funsuite.AnyFunSuite /** * Suite that tests Kernel throws error when it receives a unsupported protocol and metadata */ class TableFeaturesSuite extends AnyFunSuite { ///////////////////////////////////////////////////////////////////////////////////////////////// // Tests for [[TableFeature]] implementations // ///////////////////////////////////////////////////////////////////////////////////////////////// val readerWriterFeatures = Seq( "catalogManaged", "columnMapping", "deletionVectors", "timestampNtz", "typeWidening", "typeWidening-preview", "v2Checkpoint", "vacuumProtocolCheck", "variantType", "variantType-preview", "variantShredding", "variantShredding-preview") val writerOnlyFeatures = Seq( "allowColumnDefaults", "appendOnly", "checkpointProtection", "invariants", "checkConstraints", "generatedColumns", "changeDataFeed", "identityColumns", "rowTracking", "domainMetadata", "icebergCompatV2", "icebergCompatV3", "inCommitTimestamp", "icebergWriterCompatV1", "icebergWriterCompatV3", "clustering", "materializePartitionColumns") val legacyFeatures = Seq( "appendOnly", "invariants", "checkConstraints", "generatedColumns", "changeDataFeed", "identityColumns", "columnMapping") test("basic properties checks") { // Check that all features are correctly classified as reader-writer or writer-only readerWriterFeatures.foreach { feature => assert(TableFeatures.getTableFeature(feature).isReaderWriterFeature) } writerOnlyFeatures.foreach { feature => assert(!TableFeatures.getTableFeature(feature).isReaderWriterFeature) } // Check that legacy features are correctly classified as legacy features (readerWriterFeatures ++ writerOnlyFeatures) foreach { feature => if (legacyFeatures.contains(feature)) { assert(TableFeatures.getTableFeature(feature).isLegacyFeature) } else { assert(!TableFeatures.getTableFeature(feature).isLegacyFeature) } } // all expected features are present in list. Just make sure we don't miss any // adding to the list. This is the list used to iterate over all features assert( TableFeatures.TABLE_FEATURES.size() == readerWriterFeatures.size + writerOnlyFeatures.size) } val testProtocol = new Protocol(1, 2, emptySet(), emptySet()) Seq( // Test feature, metadata, expected result ("appendOnly", testMetadata(tblProps = Map("delta.appendOnly" -> "true")), true), ("appendOnly", testMetadata(tblProps = Map("delta.appendOnly" -> "false")), false), ("invariants", testMetadata(includeInvariant = true), true), ("invariants", testMetadata(includeInvariant = false), false), ("checkConstraints", testMetadata(tblProps = Map("delta.constraints.a" -> "a = b")), true), ("checkConstraints", testMetadata(), false), ("generatedColumns", testMetadata(includeGeneratedColumn = true), true), ("generatedColumns", testMetadata(includeGeneratedColumn = false), false), ("changeDataFeed", testMetadata(tblProps = Map("delta.enableChangeDataFeed" -> "true")), true), ( "changeDataFeed", testMetadata(tblProps = Map("delta.enableChangeDataFeed" -> "false")), false), ("identityColumns", testMetadata(includeIdentityColumn = true), true), ("identityColumns", testMetadata(includeIdentityColumn = false), false), ("columnMapping", testMetadata(tblProps = Map("delta.columnMapping.mode" -> "id")), true), ("columnMapping", testMetadata(tblProps = Map("delta.columnMapping.mode" -> "none")), false), ("typeWidening", testMetadata(tblProps = Map("delta.enableTypeWidening" -> "true")), true), ("typeWidening", testMetadata(tblProps = Map("delta.enableTypeWidening" -> "false")), false), ("variantType", testMetadata(includeVariantTypeCol = true), true), ("variantType", testMetadata(includeVariantTypeCol = false), false), ("rowTracking", testMetadata(tblProps = Map("delta.enableRowTracking" -> "true")), true), ("rowTracking", testMetadata(tblProps = Map("delta.enableRowTracking" -> "false")), false), ( "variantShredding", testMetadata(tblProps = Map("delta.enableVariantShredding" -> "true")), true), ( "variantShredding", testMetadata(tblProps = Map("delta.enableVariantShredding" -> "false")), false), ( "deletionVectors", testMetadata(tblProps = Map("delta.enableDeletionVectors" -> "true")), true), ( "deletionVectors", testMetadata(tblProps = Map("delta.enableDeletionVectors" -> "false")), false), ("timestampNtz", testMetadata(includeTimestampNtzTypeCol = true), true), ("timestampNtz", testMetadata(includeTimestampNtzTypeCol = false), false), ("v2Checkpoint", testMetadata(tblProps = Map("delta.checkpointPolicy" -> "v2")), true), ("v2Checkpoint", testMetadata(tblProps = Map("delta.checkpointPolicy" -> "classic")), false), ( "icebergCompatV2", testMetadata(tblProps = Map("delta.enableIcebergCompatV2" -> "true")), true), ( "icebergCompatV2", testMetadata(tblProps = Map("delta.enableIcebergCompatV2" -> "false")), false), ( "icebergCompatV3", testMetadata(tblProps = Map("delta.enableIcebergCompatV3" -> "true")), true), ( "icebergCompatV3", testMetadata(tblProps = Map("delta.enableIcebergCompatV3" -> "false")), false), ( "inCommitTimestamp", testMetadata(tblProps = Map("delta.enableInCommitTimestamps" -> "true")), true), ( "inCommitTimestamp", testMetadata(tblProps = Map("delta.enableInCommitTimestamps" -> "false")), false), ( "icebergWriterCompatV1", testMetadata(tblProps = Map("delta.enableIcebergWriterCompatV1" -> "true")), true), ( "icebergWriterCompatV1", testMetadata(tblProps = Map("delta.enableIcebergWriterCompatV1" -> "false")), false), ( "icebergWriterCompatV3", testMetadata(tblProps = Map("delta.enableIcebergWriterCompatV3" -> "true")), true), ( "icebergWriterCompatV3", testMetadata(tblProps = Map("delta.enableIcebergWriterCompatV3" -> "false")), false)).foreach({ case (feature, metadata, expected) => test(s"metadataRequiresFeatureToBeEnabled - $feature - $metadata") { val tableFeature = TableFeatures.getTableFeature(feature) assert(tableFeature.isInstanceOf[FeatureAutoEnabledByMetadata]) assert(tableFeature.asInstanceOf[FeatureAutoEnabledByMetadata] .metadataRequiresFeatureToBeEnabled(testProtocol, metadata) == expected) } }) Seq( "checkpointProtection", "domainMetadata", "vacuumProtocolCheck", "clustering", "catalogManaged", "allowColumnDefaults", "variantShredding-preview").foreach { feature => test(s"doesn't support auto enable by metadata: $feature") { val tableFeature = TableFeatures.getTableFeature(feature) assert(!tableFeature.isInstanceOf[FeatureAutoEnabledByMetadata]) } } Seq( ("variantType", testMetadata(includeVariantTypeCol = true)), ("typeWidening", testMetadata(tblProps = Map("delta.enableTypeWidening" -> "true"))), ( "variantShredding", testMetadata(tblProps = Map("delta.enableVariantShredding" -> "true")))).foreach { case (feature, metadataEnablingFeature) => test("special handling of tables containing preview features: " + feature) { val protocolWithPreviewFeature = new Protocol(3, 7) .withFeature(TableFeatures.getTableFeature(s"$feature-preview")) val enable = TableFeatures.getTableFeature(feature) .asInstanceOf[FeatureAutoEnabledByMetadata] .metadataRequiresFeatureToBeEnabled( protocolWithPreviewFeature, metadataEnablingFeature) assert(!enable, "shouldn't enable non-preview feature") } } test("hasKernelReadSupport expected to be true") { val results = TABLE_FEATURES.stream() .filter(_.isReaderWriterFeature) .filter(_.hasKernelReadSupport()) .collect(toList()).asScala val expected = Seq( "catalogManaged", "columnMapping", "v2Checkpoint", "variantType", "variantType-preview", "variantShredding", "variantShredding-preview", "typeWidening", "typeWidening-preview", "deletionVectors", "timestampNtz", "vacuumProtocolCheck") assert(results.map(_.featureName()).toSet == expected.toSet) } test("hasKernelWriteSupport expected to be true") { val results = TABLE_FEATURES.stream() .filter(_.hasKernelWriteSupport(testMetadata())) .collect(toList()).asScala // checkConstraints, generatedColumns, identityColumns, invariants and changeDataFeed // are writable because the metadata has not been set the info that // these features are enabled val expected = Seq( "catalogManaged", "checkpointProtection", "columnMapping", "allowColumnDefaults", "v2Checkpoint", "deletionVectors", "vacuumProtocolCheck", "rowTracking", "domainMetadata", "icebergCompatV2", "icebergCompatV3", "inCommitTimestamp", "appendOnly", "invariants", "checkConstraints", "generatedColumns", "changeDataFeed", "timestampNtz", "identityColumns", "typeWidening-preview", "typeWidening", "icebergWriterCompatV1", "icebergWriterCompatV3", "clustering", "variantType-preview", "variantType", "variantShredding", "variantShredding-preview", "materializePartitionColumns") assert(results.map(_.featureName()).toSet == expected.toSet) } Seq( // Test format: feature, metadata, expected value ("invariants", testMetadata(includeInvariant = true), false), ("checkConstraints", testMetadata(tblProps = Map("delta.constraints.a" -> "a = b")), false), ("generatedColumns", testMetadata(includeGeneratedColumn = true), false), ("identityColumns", testMetadata(includeIdentityColumn = true), false)).foreach({ case (feature, metadata, expected) => test(s"hasKernelWriteSupport - $feature has metadata") { val tableFeature = TableFeatures.getTableFeature(feature) assert(tableFeature.hasKernelWriteSupport(metadata) == expected) } }) ///////////////////////////////////////////////////////////////////////////////////////////////// // Tests for validateKernelCanReadTheTable and validateKernelCanWriteToTable // ///////////////////////////////////////////////////////////////////////////////////////////////// // Reads: All legacy protocols should be readable by Kernel Seq( // Test format: protocol (minReaderVersion, minWriterVersion) (1, 1), (1, 2), (1, 3), (1, 4), (2, 5), (2, 6)).foreach { case (minReaderVersion, minWriterVersion) => test(s"validateKernelCanReadTheTable: protocol ($minReaderVersion, $minWriterVersion)") { val protocol = new Protocol(minReaderVersion, minWriterVersion) validateKernelCanReadTheTable(protocol, "/test/table") } } // Reads: Supported table features represented as readerFeatures in the protocol Seq( "catalogManaged", "variantType", "variantType-preview", "variantShredding", "variantShredding-preview", "deletionVectors", "typeWidening", "typeWidening-preview", "timestampNtz", "v2Checkpoint", "vacuumProtocolCheck", "allowColumnDefaults", "columnMapping").foreach { feature => test(s"validateKernelCanReadTheTable: protocol 3 with $feature") { val protocol = new Protocol(3, 1, singleton(feature), Set().asJava) validateKernelCanReadTheTable(protocol, "/test/table") } } // Read is supported when all table readerWriter features are supported by the Kernel, // but the table has writeOnly table feature unknown to Kernel test("validateKernelCanReadTheTable: with writeOnly feature unknown to Kernel") { // legacy reader protocol version val protocol1 = new Protocol(1, 7, emptySet(), singleton("unknownFeature")) validateKernelCanReadTheTable(protocol1, "/test/table") // table feature supported reader version val protocol2 = new Protocol( 3, 7, Set("columnMapping", "timestampNtz").asJava, Set("columnMapping", "timestampNtz", "unknownFeature").asJava) validateKernelCanReadTheTable(protocol2, "/test/table") } test("validateKernelCanReadTheTable: unknown readerWriter feature to Kernel") { val protocol = new Protocol(3, 7, singleton("unknownFeature"), singleton("unknownFeature")) val ex = intercept[KernelException] { validateKernelCanReadTheTable(protocol, "/test/table") } assert(ex.getMessage.contains( "requires feature \"unknownFeature\" which is unsupported by this version of Delta Kernel")) } test("validateKernelCanReadTheTable: reader version > 3") { val protocol = new Protocol(4, 7, emptySet(), singleton("unknownFeature")) val ex = intercept[KernelException] { validateKernelCanReadTheTable(protocol, "/test/table") } assert(ex.getMessage.contains( "requires reader version 4 which is unsupported by this version of Delta Kernel")) } // Writes checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with catalogManaged", new Protocol(3, 7, singleton("catalogManaged"), singleton("catalogManaged")), testMetadata()) checkWriteUnsupported( "validateKernelCanWriteToTable: protocol 8", // beyond the table feature writer version new Protocol(3, 8)) checkWriteUnsupported( // beyond the table feature reader/writer version "validateKernelCanWriteToTable: protocol 4, 8", new Protocol(4, 8)) checkWriteSupported( "validateKernelCanWriteToTable: protocol 1", new Protocol(1, 1), testMetadata()) checkWriteSupported( "validateKernelCanWriteToTable: protocol 2", new Protocol(1, 2), testMetadata()) checkWriteSupported( "validateKernelCanWriteToTable: protocol 2 with appendOnly", new Protocol(1, 2), testMetadata(tblProps = Map("delta.appendOnly" -> "true"))) checkWriteUnsupported( "validateKernelCanWriteToTable: protocol 2 with invariants", new Protocol(1, 2), testMetadata(includeInvariant = true)) checkWriteUnsupported( "validateKernelCanWriteToTable: protocol 2, with appendOnly and invariants", new Protocol(1, 2), testMetadata(includeInvariant = true, tblProps = Map("delta.appendOnly" -> "true"))) checkWriteSupported( "validateKernelCanWriteToTable: protocol 3", new Protocol(1, 3)) checkWriteUnsupported( "validateKernelCanWriteToTable: protocol 3 with checkConstraints", new Protocol(1, 3), testMetadata(tblProps = Map("delta.constraints.a" -> "a = b"))) checkWriteSupported( "validateKernelCanWriteToTable: protocol 4", new Protocol(1, 4)) checkWriteUnsupported( "validateKernelCanWriteToTable: protocol 4 with generatedColumns", new Protocol(1, 4), testMetadata(includeGeneratedColumn = true)) checkWriteUnsupported( "validateKernelCanWriteToTable: protocol 4 with changeDataFeed", new Protocol(1, 4), testMetadata(tblProps = Map("delta.enableChangeDataFeed" -> "true"))) checkWriteSupported( "validateKernelCanWriteToTable: protocol 5 with columnMapping", new Protocol(2, 5), testMetadata(tblProps = Map("delta.columnMapping.mode" -> "id"))) checkWriteSupported( "validateKernelCanWriteToTable: protocol 6", new Protocol(2, 6), testMetadata(tblProps = Map("delta.columnMapping.mode" -> "none"))) checkWriteUnsupported( "validateKernelCanWriteToTable: protocol 6 with identityColumns", new Protocol(2, 6), testMetadata(includeIdentityColumn = true)) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with appendOnly supported", new Protocol(1, 7, Set().asJava, singleton("appendOnly")), testMetadata(tblProps = Map("delta.appendOnly" -> "true"))) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with invariants, " + "schema doesn't contain invariants", new Protocol(1, 7, Set().asJava, singleton("invariants")), testMetadata(includeInvariant = false)) checkWriteUnsupported( "validateKernelCanWriteToTable: protocol 7 with invariants, " + "schema contains invariants", new Protocol(1, 7, Set().asJava, singleton("invariants")), testMetadata(includeInvariant = true)) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with checkConstraints, " + "metadata doesn't contains any constraints", new Protocol(1, 7, Set().asJava, singleton("checkConstraints")), testMetadata()) checkWriteUnsupported( "validateKernelCanWriteToTable: protocol 7 with checkConstraints, " + "metadata contains constraints", new Protocol(1, 7, Set().asJava, singleton("checkConstraints")), testMetadata(tblProps = Map("delta.constraints.a" -> "a = b"))) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with generatedColumns, " + "metadata doesn't contains any generated columns", new Protocol(1, 7, Set().asJava, singleton("generatedColumns")), testMetadata()) checkWriteUnsupported( "validateKernelCanWriteToTable: protocol 7 with generatedColumns, " + "metadata contains generated columns", new Protocol(1, 7, Set().asJava, singleton("generatedColumns")), testMetadata(includeGeneratedColumn = true)) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with changeDataFeed, " + "metadata doesn't contains changeDataFeed", new Protocol(1, 7, Set().asJava, singleton("changeDataFeed")), testMetadata()) checkWriteUnsupported( "validateKernelCanWriteToTable: protocol 7 with changeDataFeed, " + "metadata contains changeDataFeed", new Protocol(1, 7, Set().asJava, singleton("changeDataFeed")), testMetadata(tblProps = Map("delta.enableChangeDataFeed" -> "true"))) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with columnMapping, " + "metadata doesn't contains columnMapping", new Protocol(2, 7, Set().asJava, singleton("columnMapping")), testMetadata()) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with columnMapping, " + "metadata contains columnMapping", new Protocol(2, 7, Set().asJava, singleton("columnMapping")), testMetadata(tblProps = Map("delta.columnMapping.mode" -> "id"))) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with allowColumnDefaults, " + "metadata contains allowColumnDefaults", new Protocol(2, 7, Set().asJava, singleton("allowColumnDefaults")), testMetadata(tblProps = Map("delta.feature.allowColumnDefaults" -> "supported"))) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with identityColumns, " + "schema doesn't contains identity columns", new Protocol(2, 7, Set().asJava, singleton("identityColumns")), testMetadata()) checkWriteUnsupported( "validateKernelCanWriteToTable: protocol 7 with identityColumns, " + "schema contains identity columns", new Protocol(2, 7, Set().asJava, singleton("identityColumns")), testMetadata(includeIdentityColumn = true)) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with deletionVectors, " + "metadata doesn't contains deletionVectors", new Protocol(2, 7, Set().asJava, singleton("deletionVectors")), testMetadata()) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with deletionVectors, " + "metadata contains deletionVectors", new Protocol(2, 7, Set().asJava, singleton("deletionVectors")), testMetadata(tblProps = Map("delta.enableDeletionVectors" -> "true"))) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with timestampNtz, " + "schema doesn't contains timestampNtz", new Protocol(3, 7, singleton("timestampNtz"), singleton("timestampNtz")), testMetadata()) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with timestampNtz, " + "schema contains timestampNtz", new Protocol(3, 7, singleton("timestampNtz"), singleton("timestampNtz")), testMetadata(includeTimestampNtzTypeCol = true)) Seq("typeWidening", "typeWidening-preview").foreach { feature => checkWriteSupported( s"validateKernelCanWriteToTable: protocol 7 with $feature, " + s"metadata doesn't contains $feature", new Protocol(3, 7, singleton(feature), singleton(feature)), testMetadata()) checkWriteSupported( s"validateKernelCanWriteToTable: protocol 7 with $feature, " + s"metadata contains $feature", new Protocol(3, 7, singleton(feature), singleton(feature)), testMetadata(tblProps = Map("delta.enableTypeWidening" -> "true"))) } Seq("variantType", "variantType-preview", "variantShredding", "variantShredding-preview") .foreach { feature => checkWriteSupported( s"validateKernelCanWriteToTable: protocol 7 with $feature, " + s"metadata doesn't contains $feature", new Protocol(3, 7, singleton(feature), singleton(feature)), testMetadata()) checkWriteSupported( s"validateKernelCanWriteToTable: protocol 7 with $feature, " + s"metadata contains $feature", new Protocol(3, 7, singleton(feature), singleton(feature)), testMetadata(includeVariantTypeCol = true)) } checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with vacuumProtocolCheck, " + "metadata doesn't contains vacuumProtocolCheck", new Protocol(3, 7, singleton("vacuumProtocolCheck"), singleton("vacuumProtocolCheck")), testMetadata()) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with domainMetadata", new Protocol(3, 7, emptySet(), singleton("domainMetadata")), testMetadata()) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with rowTracking", new Protocol(3, 7, emptySet(), singleton("rowTracking")), testMetadata()) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with inCommitTimestamp", new Protocol(3, 7, emptySet(), singleton("inCommitTimestamp")), testMetadata()) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with icebergCompatV2", new Protocol(3, 7, emptySet(), singleton("icebergCompatV2")), testMetadata(tblProps = Map("delta.enableIcebergCompatV2" -> "true"))) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with icebergCompatV3", new Protocol(3, 7, emptySet(), singleton("icebergCompatV3")), testMetadata(tblProps = Map("delta.enableIcebergCompatV3" -> "true"))) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with v2Checkpoint, " + "metadata enables v2Checkpoint", new Protocol(3, 7, singleton("v2Checkpoint"), singleton("v2Checkpoint")), testMetadata(tblProps = Map("delta.checkpointPolicy" -> "v2"))) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with icebergWriterCompatV1", new Protocol(3, 7, emptySet(), singleton("icebergWriterCompatV1")), testMetadata(tblProps = Map("delta.enableIcebergWriterCompatV1" -> "true"))) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with icebergWriterCompatV3", new Protocol(3, 7, emptySet(), singleton("icebergWriterCompatV3")), testMetadata(tblProps = Map("delta.enableIcebergWriterCompatV3" -> "true"))) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with clustering", new Protocol(3, 7, emptySet(), singleton("clustering")), testMetadata()) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with materializePartitionColumns", new Protocol(3, 7, emptySet(), singleton("materializePartitionColumns")), testMetadata()) checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with multiple features supported", new Protocol( 3, 7, Set("v2Checkpoint", "columnMapping").asJava, Set("v2Checkpoint", "columnMapping", "rowTracking", "domainMetadata").asJava), testMetadata(tblProps = Map( "delta.checkpointPolicy" -> "v2", "delta.columnMapping.mode" -> "id", "delta.enableRowTracking" -> "true"))) checkWriteUnsupported( "validateKernelCanWriteToTable: protocol 7 with multiple features supported, " + "with one of the features not supported by Kernel for writing", new Protocol( 3, 7, Set("v2Checkpoint", "columnMapping", "invariants").asJava, Set("v2Checkpoint", "columnMapping", "invariants").asJava), testMetadata( includeInvariant = true, // unsupported feature tblProps = Map( "delta.checkpointPolicy" -> "v2", "delta.enableRowTracking" -> "true"))) checkWriteUnsupported( "validateKernelCanWriteToTable: protocol 7 with unknown writerOnly feature", new Protocol(1, 7, emptySet(), singleton("unknownWriterOnlyFeature")), testMetadata()) checkWriteUnsupported( "validateKernelCanWriteToTable: protocol 7 with unknown readerWriter feature", new Protocol( 3, 7, singleton("unknownWriterOnlyFeature"), singleton("unknownWriterOnlyFeature")), testMetadata()) ///////////////////////////////////////////////////////////////////////////////////////////////// // Tests for autoUpgradeProtocolBasedOnMetadata // ///////////////////////////////////////////////////////////////////////////////////////////////// Seq( // Test format: // new metadata, // current protocol, // expected protocol, // expected new features added ( testMetadata(tblProps = Map("delta.appendOnly" -> "true")), new Protocol(1, 1), new Protocol(1, 7, emptySet(), set("appendOnly")), set("appendOnly")), ( testMetadata(includeInvariant = true), new Protocol(1, 1), new Protocol(1, 7, emptySet(), set("invariants")), set("invariants")), ( testMetadata(includeInvariant = true, tblProps = Map("delta.appendOnly" -> "true")), new Protocol(1, 1), new Protocol(1, 2), // (1, 2) covers both appendOnly and invariants Set("invariants", "appendOnly").asJava), ( testMetadata(tblProps = Map("delta.constraints.a" -> "a = b")), new Protocol(1, 1), new Protocol(1, 7, emptySet(), set("checkConstraints")), set("checkConstraints")), ( testMetadata( includeInvariant = true, tblProps = Map("delta.appendOnly" -> "true", "delta.constraints.a" -> "a = b")), new Protocol(1, 1), new Protocol(1, 3), set("appendOnly", "checkConstraints", "invariants")), ( testMetadata(tblProps = Map("delta.constraints.a" -> "a = b")), new Protocol(1, 2), new Protocol(1, 3), // (1, 3) covers all: appendOnly, invariants, checkConstraints set("checkConstraints")), ( testMetadata(tblProps = Map("delta.enableChangeDataFeed" -> "true")), new Protocol(1, 1), new Protocol(1, 7, emptySet(), set("changeDataFeed")), set("changeDataFeed")), ( testMetadata(includeGeneratedColumn = true), new Protocol(1, 1), new Protocol(1, 7, emptySet(), set("generatedColumns")), set("generatedColumns")), ( testMetadata( includeGeneratedColumn = true, tblProps = Map("delta.enableChangeDataFeed" -> "true")), new Protocol(1, 1), new Protocol(1, 7, emptySet(), set("generatedColumns", "changeDataFeed")), set("generatedColumns", "changeDataFeed")), ( testMetadata( includeGeneratedColumn = true, tblProps = Map("delta.enableChangeDataFeed" -> "true")), new Protocol(1, 2), new Protocol( 1, 7, set(), set("generatedColumns", "changeDataFeed", "appendOnly", "invariants")), set("generatedColumns", "changeDataFeed")), ( testMetadata( includeGeneratedColumn = true, tblProps = Map("delta.enableChangeDataFeed" -> "true")), new Protocol(1, 3), new Protocol(1, 4), // 4 - implicitly supports all features set("generatedColumns", "changeDataFeed")), ( testMetadata(tblProps = Map("delta.columnMapping.mode" -> "name")), new Protocol(1, 1), new Protocol( 2, 7, set(), set("columnMapping")), set("columnMapping")), ( testMetadata(tblProps = Map("delta.columnMapping.mode" -> "name")), new Protocol(1, 2), new Protocol( 2, 7, set(), set("columnMapping", "appendOnly", "invariants")), set("columnMapping")), ( testMetadata(tblProps = Map("delta.columnMapping.mode" -> "name")), new Protocol(1, 3), new Protocol( 2, 7, set(), set("columnMapping", "appendOnly", "invariants", "checkConstraints")), set("columnMapping")), ( testMetadata(tblProps = Map("delta.columnMapping.mode" -> "name")), new Protocol(1, 4), new Protocol(2, 5), // implicitly supports all features set("columnMapping")), ( testMetadata(includeIdentityColumn = true), new Protocol(1, 1), new Protocol( 1, 7, set(), set("identityColumns")), set("identityColumns")), ( testMetadata(includeIdentityColumn = true), new Protocol(1, 2), new Protocol( 1, 7, set(), set("identityColumns", "appendOnly", "invariants")), set("identityColumns")), ( testMetadata(includeIdentityColumn = true), new Protocol(1, 3), new Protocol( 1, 7, set(), set("identityColumns", "appendOnly", "invariants", "checkConstraints")), set("identityColumns")), ( testMetadata(includeIdentityColumn = true), new Protocol(2, 5), new Protocol(2, 6), // implicitly supports all features set("identityColumns")), ( testMetadata(includeTimestampNtzTypeCol = true), new Protocol(1, 1), new Protocol( 3, 7, set("timestampNtz"), set("timestampNtz")), set("timestampNtz")), ( testMetadata(includeTimestampNtzTypeCol = true), new Protocol(1, 2), new Protocol( 3, 7, set("timestampNtz"), set("timestampNtz", "appendOnly", "invariants")), set("timestampNtz")), ( testMetadata(tblProps = Map("delta.enableIcebergCompatV2" -> "true")), new Protocol(1, 2), new Protocol( 2, 7, set(), set("columnMapping", "appendOnly", "invariants", "icebergCompatV2")), set("icebergCompatV2", "columnMapping")), ( testMetadata(tblProps = Map("delta.enableIcebergCompatV3" -> "true")), new Protocol(1, 2), new Protocol( 2, 7, set(), set( "columnMapping", "appendOnly", "invariants", "icebergCompatV3", "domainMetadata", "rowTracking")), set("icebergCompatV3", "domainMetadata", "columnMapping", "rowTracking")), ( testMetadata(tblProps = Map("delta.enableIcebergCompatV2" -> "true", "delta.enableDeletionVectors" -> "true")), new Protocol(2, 5), new Protocol( 3, 7, set("columnMapping", "deletionVectors"), set( "columnMapping", "appendOnly", "invariants", "icebergCompatV2", "checkConstraints", "deletionVectors", "generatedColumns", "changeDataFeed")), set("icebergCompatV2", "deletionVectors")), ( testMetadata(tblProps = Map("delta.enableIcebergCompatV2" -> "true")), new Protocol(3, 7, set("columnMapping", "deletionVectors"), set("columnMapping")), new Protocol( 3, 7, set("columnMapping", "deletionVectors"), set("columnMapping", "icebergCompatV2", "deletionVectors")), set("icebergCompatV2")), ( testMetadata(tblProps = Map("delta.enableIcebergCompatV3" -> "true")), new Protocol(3, 7, set("columnMapping", "deletionVectors"), set("columnMapping")), new Protocol( 3, 7, set("columnMapping", "deletionVectors"), set( "columnMapping", "icebergCompatV3", "deletionVectors", "domainMetadata", "rowTracking")), set("icebergCompatV3", "domainMetadata", "rowTracking")), ( testMetadata(tblProps = Map("delta.enableIcebergWriterCompatV1" -> "true")), new Protocol(1, 2), new Protocol( 2, 7, set(), set( "columnMapping", "appendOnly", "invariants", "icebergCompatV2", "icebergWriterCompatV1")), set("icebergCompatV2", "columnMapping", "icebergWriterCompatV1")), ( testMetadata(tblProps = Map("delta.enableIcebergWriterCompatV3" -> "true")), new Protocol(1, 2), new Protocol( 2, 7, set(), set( "columnMapping", "appendOnly", "invariants", "icebergCompatV3", "icebergWriterCompatV3", "domainMetadata", "rowTracking")), set( "icebergCompatV3", "columnMapping", "icebergWriterCompatV3", "domainMetadata", "rowTracking")), ( testMetadata(tblProps = Map( "delta.enableIcebergWriterCompatV3" -> "true", "delta.enableDeletionVectors" -> "true")), new Protocol(2, 5), new Protocol( 3, 7, set("columnMapping", "deletionVectors"), set( "columnMapping", "appendOnly", "deletionVectors", "invariants", "icebergCompatV3", "icebergWriterCompatV3", "checkConstraints", "generatedColumns", "changeDataFeed", "domainMetadata", "rowTracking")), set( "icebergCompatV3", "icebergWriterCompatV3", "deletionVectors", "domainMetadata", "rowTracking")), ( testMetadata(tblProps = Map("delta.enableIcebergWriterCompatV3" -> "true")), new Protocol(1, 1), // Minimal starting protocol with no features new Protocol( 2, 7, set(), set( "columnMapping", "icebergCompatV3", // Added as dependency "icebergWriterCompatV3", "domainMetadata", "rowTracking")), set( "icebergCompatV3", "columnMapping", "icebergWriterCompatV3", "domainMetadata", "rowTracking")), ( testMetadata(tblProps = Map( "delta.enableIcebergWriterCompatV1" -> "true", "delta.enableDeletionVectors" -> "true")), new Protocol(2, 5), new Protocol( 3, 7, set("columnMapping", "deletionVectors"), set( "columnMapping", "appendOnly", "deletionVectors", "invariants", "icebergCompatV2", "icebergWriterCompatV1", "checkConstraints", "generatedColumns", "changeDataFeed")), set("icebergCompatV2", "icebergWriterCompatV1", "deletionVectors")), ( testMetadata(tblProps = Map("delta.enableIcebergWriterCompatV1" -> "true")), new Protocol(3, 7, set("columnMapping", "deletionVectors"), set("columnMapping")), new Protocol( 3, 7, set("columnMapping", "deletionVectors"), set("columnMapping", "icebergCompatV2", "deletionVectors", "icebergWriterCompatV1")), set("icebergCompatV2", "icebergWriterCompatV1")), ( testMetadata( tblProps = Map("delta.enableVariantShredding" -> "true"), includeVariantTypeCol = true), new Protocol( 3, 7, set("columnMapping", "deletionVectors"), set("columnMapping")), new Protocol( 3, 7, set("columnMapping", "deletionVectors"), set( "columnMapping", "deletionVectors", "variantShredding", "variantType")), set("variantType", "variantShredding"))).foreach { case (newMetadata, currentProtocol, expectedProtocol, expectedNewFeatures) => test(s"autoUpgradeProtocolBasedOnMetadata:" + s"$currentProtocol -> $expectedProtocol, $expectedNewFeatures") { for ( (manualFeatures) <- Seq( Set[TableFeature](), Set(TableFeatures.DOMAIN_METADATA_W_FEATURE), Set(TableFeatures.CLUSTERING_W_FEATURE), Set(TableFeatures.CLUSTERING_W_FEATURE, TableFeatures.DOMAIN_METADATA_W_FEATURE)) ) { val newProtocolAndNewFeaturesEnabled = TableFeatures.autoUpgradeProtocolBasedOnMetadata( newMetadata, manualFeatures.asJava, currentProtocol) assert(newProtocolAndNewFeaturesEnabled.isPresent, "expected protocol upgrade") val newProtocol = newProtocolAndNewFeaturesEnabled.get()._1 val newFeaturesEnabled = newProtocolAndNewFeaturesEnabled.get()._2 // Reader version should remain the same assert(newProtocol.getMinReaderVersion == expectedProtocol.getMinReaderVersion) // Writer version: upgrade to 7 if domain metadata or clustering feature is enabled val expectedWriterVersion = if ( !(manualFeatures & Set( TableFeatures.CLUSTERING_W_FEATURE, TableFeatures.DOMAIN_METADATA_W_FEATURE)).isEmpty ) { 7 } else expectedProtocol.getMinWriterVersion assert(newProtocol.getMinWriterVersion == expectedWriterVersion) // Expected enabled features val expectedEnabledFeatures = expectedNewFeatures.asScala ++ manualFeatures.map(_.featureName()).toSet ++ ( if (manualFeatures.contains(TableFeatures.CLUSTERING_W_FEATURE)) Set("domainMetadata") else Set.empty ) assert(newFeaturesEnabled.asScala.map(_.featureName()).toSet == expectedEnabledFeatures) // Expected supported features val implicitAndExplicitFeatures = expectedProtocol.getImplicitlyAndExplicitlySupportedFeatures.asScala val expectedSupportedFeatures = implicitAndExplicitFeatures ++ manualFeatures ++ ( if (manualFeatures.contains(TableFeatures.CLUSTERING_W_FEATURE)) { Set(TableFeatures.DOMAIN_METADATA_W_FEATURE) } else { Set.empty } ) assert(newProtocol.getImplicitlyAndExplicitlySupportedFeatures.asScala == expectedSupportedFeatures) } } } // No-op upgrade Seq( // Test format: new metadata, current protocol (testMetadata(), new Protocol(1, 1)), ( // try to enable the writer that is already supported on a protocol // that is of legacy protocol testMetadata(tblProps = Map("delta.appendOnly" -> "true")), new Protocol(1, 7, emptySet(), set("appendOnly"))), ( // try to enable the writer that is already supported on a protocol // that is of legacy protocol testMetadata(tblProps = Map("delta.appendOnly" -> "true", "delta.constraints.a" -> "a = b")), new Protocol(1, 3)), ( // try to enable the reader writer feature that is already supported on a protocol // that is of legacy protocol testMetadata(tblProps = Map("delta.columnMapping.mode" -> "name")), new Protocol(2, 5)), ( // try to enable the feature that is already supported on a protocol // that is of partial (writer only) table feature support testMetadata(tblProps = Map("delta.enableIcebergCompatV2" -> "true")), new Protocol(2, 7, set(), set("columnMapping", "icebergCompatV2"))), ( // try to enable the feature that is already supported on a protocol // that is of partial (writer only) table feature support testMetadata(tblProps = Map("delta.enableIcebergCompatV3" -> "true")), new Protocol( 2, 7, set(), set("columnMapping", "icebergCompatV3", "domainMetadata", "rowTracking"))), ( // try to enable the feature that is already supported on a protocol // that is of table feature support testMetadata(tblProps = Map("delta.enableIcebergCompatV2" -> "true")), new Protocol( 3, 7, set("columnMapping", "deletionVectors"), set("columnMapping", "deletionVectors", "icebergCompatV2"))), ( // Enable the variantShredding GA feature when the preview feature is already enabled. // The GA feature should not be auto-enabled. testMetadata( tblProps = Map("delta.enableVariantShredding" -> "true"), includeVariantTypeCol = true), new Protocol( 3, 7, set("variantType", "variantShredding-preview"), set("variantType", "variantShredding-preview")))).foreach { case (newMetadata, currentProtocol) => test(s"autoUpgradeProtocolBasedOnMetadata: no-op upgrade: $currentProtocol") { val newProtocolAndNewFeaturesEnabled = TableFeatures.autoUpgradeProtocolBasedOnMetadata( newMetadata, Set.empty.asJava, currentProtocol) assert(!newProtocolAndNewFeaturesEnabled.isPresent, "expected no-op upgrade") } } test( "extractFeaturePropertyOverrides returns feature options and removes from them from metadata") { val metadata = testMetadata(tblProps = Map( "delta.feature.deletionVectors" -> "supported", "delta.feature.appendOnly" -> "supported", "anotherkey" -> "some_value", "delta.enableRowTracking" -> "true")) val tableFeaturesAndMetadata = TableFeatures.extractFeaturePropertyOverrides(metadata) val newFeatures = tableFeaturesAndMetadata._1 assert(tableFeaturesAndMetadata._2.isPresent) val newMetadata = tableFeaturesAndMetadata._2.get assert( newFeatures.equals(Set( TableFeatures.APPEND_ONLY_W_FEATURE, TableFeatures.DELETION_VECTORS_RW_FEATURE).asJava), s"Explicit features: ${newFeatures}") val tableConfig = newMetadata.getConfiguration val expectedMap = Map("anotherkey" -> "some_value", "delta.enableRowTracking" -> "true") assert(expectedMap.asJava.equals(tableConfig), s"$tableConfig != $expectedMap") } test( "extractFeaturePropertyOverrides returns empty metadata with no change") { val metadata = testMetadata(tblProps = Map( "anotherkey" -> "some_value", "delta.enableRowTracking" -> "true")) val tableFeaturesAndMetadata = TableFeatures.extractFeaturePropertyOverrides(metadata) assert(tableFeaturesAndMetadata._1.isEmpty) assert(!tableFeaturesAndMetadata._2.isPresent) } Seq( Map("delta.feature.deletionVectors" -> "not_valid_value"), Map("delta.feature.invalidFeatureName" -> "supported")).foreach { properties => test(s"extractFeaturePropertyOverrides throws: $properties") { intercept[KernelException] { TableFeatures.extractFeaturePropertyOverrides( testMetadata(tblProps = properties)) } } } ///////////////////////////////////////////////////////////////////////////////////////////////// // Test utility methods. // ///////////////////////////////////////////////////////////////////////////////////////////////// def checkWriteSupported( testDesc: String, protocol: Protocol, metadata: Metadata = testMetadata()): Unit = { test(testDesc) { validateKernelCanWriteToTable(protocol, metadata, "/test/table") } } def checkWriteUnsupported( testDesc: String, protocol: Protocol, metadata: Metadata = testMetadata()): Unit = { test(testDesc) { intercept[KernelException] { validateKernelCanWriteToTable(protocol, metadata, "/test/table") } } } def testMetadata( includeInvariant: Boolean = false, includeTimestampNtzTypeCol: Boolean = false, includeVariantTypeCol: Boolean = false, includeGeneratedColumn: Boolean = false, includeIdentityColumn: Boolean = false, tblProps: Map[String, String] = Map.empty): Metadata = { val testSchema = createTestSchema( includeInvariant, includeTimestampNtzTypeCol, includeVariantTypeCol, includeGeneratedColumn, includeIdentityColumn) new Metadata( "id", Optional.of("name"), Optional.of("description"), new Format("parquet", Collections.emptyMap()), testSchema.toJson, testSchema, new ArrayValue() { // partitionColumns override def getSize = 1 override def getElements: ColumnVector = singletonStringColumnVector("c3") }, Optional.empty(), new MapValue() { // conf override def getSize = tblProps.size override def getKeys: ColumnVector = buildColumnVector(tblProps.toSeq.map(_._1).asJava, StringType.STRING) override def getValues: ColumnVector = buildColumnVector(tblProps.toSeq.map(_._2).asJava, StringType.STRING) }) } def createTestSchema( includeInvariant: Boolean = false, includeTimestampNtzTypeCol: Boolean = false, includeVariantTypeCol: Boolean = false, includeGeneratedColumn: Boolean = false, includeIdentityColumn: Boolean = false): StructType = { var structType = new StructType() .add("c1", IntegerType.INTEGER) .add("c2", StringType.STRING) if (includeInvariant) { structType = structType.add( "c3", TimestampType.TIMESTAMP, FieldMetadata.builder() .putString("delta.invariants", "{\"expression\": { \"expression\": \"x > 3\"} }") .build()) } if (includeTimestampNtzTypeCol) { structType = structType.add("c4", TimestampNTZType.TIMESTAMP_NTZ) } if (includeVariantTypeCol) { structType = structType.add("c5", VariantType.VARIANT) } if (includeGeneratedColumn) { structType = structType.add( "c6", IntegerType.INTEGER, FieldMetadata.builder() .putString("delta.generationExpression", "{\"expression\": \"c1 + 1\"}") .build()) } if (includeIdentityColumn) { structType = structType.add( "c7", IntegerType.INTEGER, FieldMetadata.builder() .putLong("delta.identity.start", 1L) .putLong("delta.identity.step", 2L) .putBoolean("delta.identity.allowExplicitInsert", true) .build()) } structType } private def set(elements: String*): java.util.Set[String] = { Set(elements: _*).asJava } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/types/DataTypeJsonSerDeSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.types import java.util.HashMap import scala.collection.JavaConverters._ import scala.reflect.ClassTag import io.delta.kernel.types._ import StructField.COLLATIONS_METADATA_KEY import com.fasterxml.jackson.databind.ObjectMapper import org.scalatest.funsuite.AnyFunSuite class DataTypeJsonSerDeSuite extends AnyFunSuite { import DataTypeJsonSerDeSuite._ private val objectMapper = new ObjectMapper() private def parse(json: String): DataType = { DataTypeJsonSerDe.parseDataType(objectMapper.readTree(json)) } private def serialize(dataType: DataType): String = { DataTypeJsonSerDe.serializeDataType(dataType) } private def testRoundTrip(dataTypeJsonString: String, dataType: DataType): Unit = { // test deserialization assert(parse(dataTypeJsonString) === dataType) // test serialization val serializedJson = serialize(dataType) assert(parse(serializedJson) === dataType) } private def checkError[T <: Throwable](json: String, expectedErrorContains: String)(implicit classTag: ClassTag[T]): Unit = { val e = intercept[T] { parse(json) } assert(e.getMessage.contains(expectedErrorContains)) } /* --------------- Primitive data types (stored as a string) ----------------- */ Seq( ("\"string\"", StringType.STRING), ("\"long\"", LongType.LONG), ("\"short\"", ShortType.SHORT), ("\"integer\"", IntegerType.INTEGER), ("\"boolean\"", BooleanType.BOOLEAN), ("\"byte\"", ByteType.BYTE), ("\"float\"", FloatType.FLOAT), ("\"double\"", DoubleType.DOUBLE), ("\"binary\"", BinaryType.BINARY), ("\"date\"", DateType.DATE), ("\"timestamp\"", TimestampType.TIMESTAMP), ("\"decimal\"", DecimalType.USER_DEFAULT), ("\"decimal(10, 5)\"", new DecimalType(10, 5)), ("\"variant\"", VariantType.VARIANT), ("\"geometry\"", GeometryType.ofDefault()), ("\"geometry(EPSG:0)\"", GeometryType.ofSRID("EPSG:0")), ("\"geography\"", GeographyType.ofDefault()), ("\"geography(EPSG:4326)\"", new GeographyType("EPSG:4326", "spherical")), ("\"geography(vincenty)\"", new GeographyType("OGC:CRS84", "vincenty")), ("\"geography(EPSG:3857, vincenty)\"", new GeographyType("EPSG:3857", "vincenty"))).foreach { case (json, dataType) => test("serialize/deserialize: " + dataType) { testRoundTrip(json, dataType) } } test("parseDataType: invalid primitive string data type") { checkError[IllegalArgumentException]("\"foo\"", "foo is not a supported delta data type") } test("parseDataType: mis-formatted decimal data type") { checkError[IllegalArgumentException]( "\"decimal(1)\"", "decimal(1) is not a supported delta data type") } test("deserialize: geometry with default SRID") { // Parsing "geometry" should use default SRID (OGC:CRS84) assert(parse("\"geometry\"") === GeometryType.ofDefault()) // But serialization always writes full form assert(serialize(GeometryType.ofDefault()) === "\"geometry(OGC:CRS84)\"") } test("serialize/deserialize: geometry with various SRID formats") { // Kernel accepts any SRID format; engine validates compatibility testRoundTrip("\"geometry(OGC:CRS84)\"", GeometryType.ofSRID("OGC:CRS84")) testRoundTrip("\"geometry(EPSG:4326)\"", GeometryType.ofSRID("EPSG:4326")) testRoundTrip("\"geometry(EPSG:0)\"", GeometryType.ofSRID("EPSG:0")) testRoundTrip("\"geometry(epsg:4326)\"", GeometryType.ofSRID("epsg:4326")) } test("deserialize: geography with default SRID and algorithm") { // Parsing "geography" should use defaults (OGC:CRS84, spherical) assert(parse("\"geography\"") === GeographyType.ofDefault()) // But serialization always writes full form assert(serialize(GeographyType.ofDefault()) === "\"geography(OGC:CRS84, spherical)\"") } test("deserialize: geography with SRID and default algorithm") { // Parsing "geography()" should use default algorithm (spherical) assert(parse("\"geography(EPSG:4326)\"") === GeographyType.ofSRID("EPSG:4326")) assert(parse("\"geography(EPSG:3857)\"") === GeographyType.ofSRID("EPSG:3857")) assert(parse("\"geography(OGC:CRS84)\"") === GeographyType.ofSRID("OGC:CRS84")) // But serialization always writes full form assert(serialize(GeographyType.ofSRID("EPSG:4326")) === "\"geography(EPSG:4326, spherical)\"") } test("deserialize: geography with default SRID and specified algorithm") { // Parsing "geography()" should use default SRID (OGC:CRS84) assert(parse("\"geography(spherical)\"") === GeographyType.ofAlgorithm("spherical")) assert(parse("\"geography(vincenty)\"") === GeographyType.ofAlgorithm("vincenty")) assert(parse("\"geography(andoyer)\"") === GeographyType.ofAlgorithm("andoyer")) // But serialization always writes full form assert( serialize(GeographyType.ofAlgorithm("vincenty")) === "\"geography(OGC:CRS84, vincenty)\"") } test("serialize/deserialize: geography with both SRID and algorithm") { // Both SRID and algorithm specified - round trips correctly testRoundTrip( "\"geography(EPSG:4326, spherical)\"", new GeographyType("EPSG:4326", "spherical")) testRoundTrip( "\"geography(EPSG:3857, vincenty)\"", new GeographyType("EPSG:3857", "vincenty")) testRoundTrip( "\"geography(OGC:CRS84, andoyer)\"", new GeographyType("OGC:CRS84", "andoyer")) assert( serialize(new GeographyType("EPSG:4326", "vincenty")) === "\"geography(EPSG:4326, vincenty)\"") } test("parseDataType: invalid geometry formats") { // Missing SRID parameter checkError[IllegalArgumentException]( "\"geometry()\"", "geometry() is not a supported delta data type") // Invalid format with multiple parameters checkError[IllegalArgumentException]( "\"geometry(EPSG:4326, extra)\"", "geometry(EPSG:4326, extra) is not a supported delta data type") checkError[IllegalArgumentException]( "\"geometry(noCollon)\"", "geometry(noCollon) is not a supported delta data type") } test("parseDataType: invalid geography formats") { // Empty parameters checkError[IllegalArgumentException]( "\"geography()\"", "geography() is not a supported delta data type") // Too many parameters checkError[IllegalArgumentException]( "\"geography(EPSG:4326, spherical, extra)\"", "geography(EPSG:4326, spherical, extra) is not a supported delta data type") // Invalid format checkError[IllegalArgumentException]( "\"geography(EPSG:4326,)\"", "geography(EPSG:4326,) is not a supported delta data type") } test("parseDataType: invalid geography algorithm") { val expectedMsg = "Algorithm must be one of: spherical, vincenty, thomas, andoyer, karney" checkError[IllegalArgumentException]( "\"geography(EPSG:4326, planar)\"", expectedMsg) checkError[IllegalArgumentException]( "\"geography(haversine)\"", expectedMsg) } /* --------------- Complex types ----------------- */ test("serialize/deserialize: array type") { for (containsNull <- Seq(true, false)) { for ((elementJson, elementType) <- SAMPLE_JSON_TO_TYPES) { // test deserialization val json = arrayTypeJson(elementJson, containsNull) val expectedType = new ArrayType(elementType, containsNull) testRoundTrip(json, expectedType) } } } test("serialize/deserialize: map type") { for (valueContainsNull <- Seq(true, false)) { for ((keyJson, keyType) <- SAMPLE_JSON_TO_TYPES) { for ((valueJson, valueType) <- SAMPLE_JSON_TO_TYPES) { val json = mapTypeJson(keyJson, valueJson, valueContainsNull) val expectedType = new MapType(keyType, valueType, valueContainsNull) testRoundTrip(json, expectedType) } } } } test("serialize/deserialize: struct type") { for ((col1Json, col1Type) <- SAMPLE_JSON_TO_TYPES) { for ((col2Json, col2Type) <- SAMPLE_JSON_TO_TYPES) { val fieldsJson = Seq( structFieldJson("col1", col1Json, false), structFieldJson("col2", col2Json, true, Some("{ \"int\" : 0 }"))) val json = structTypeJson(fieldsJson) val expectedType = new StructType() .add("col1", col1Type, false) .add("col2", col2Type, true, FieldMetadata.builder().putLong("int", 0).build()) testRoundTrip(json, expectedType) } } } test("serialize/deserialize: complex types") { SAMPLE_COMPLEX_JSON_TO_TYPES .foreach { case (json, dataType) => testRoundTrip(json, dataType) } } test("serialize/deserialize: types with collated strings") { SAMPLE_JSON_TO_TYPES_WITH_COLLATION .foreach { case (json, dataType) => testRoundTrip(json, dataType) } } test("deserialize: schema with collated map key throws IllegalArgumentException") { // A JSON schema encoding a MapType with a collated StringType key should fail // deserialization because MapType rejects collated keys. val json = structTypeJson(Seq( structFieldJson( "m", mapTypeJson("\"string\"", "\"integer\"", false), true, metadataJson = Some( s"""{"$COLLATIONS_METADATA_KEY" : {"m.key" : "ICU.UNICODE_CI"}}""")))) intercept[IllegalArgumentException] { parse(json) } } test("serialize/deserialize: parsed and original struct" + " type differing just in StringType collation") { val json = structTypeJson(Seq( structFieldJson( "a1", "\"string\"", true, metadataJson = Some( s"""{"$COLLATIONS_METADATA_KEY" : | {"a1" : "SPARK.UTF8_LCASE"}}""".stripMargin)))) val dataType = new StructType() .add("a1", new StringType("ICU.UNICODE"), true) assert(!(parse(json) === dataType)) } test("serialize/deserialize: round-trip type changes metadata") { // Test cases for various type changes based on Delta Protocol specification // Case 1: Simple type widening (short -> integer -> long) val simpleTypeChangeJson = structTypeJson(Seq( structFieldJson( "e", "\"long\"", true, metadataJson = Some( """{ |"delta.typeChanges": [ | { | "fromType": "short", | "toType": "integer" | }, | { | "fromType": "integer", | "toType": "long" | } |] |}""".stripMargin)))) val simpleTypeChangeDataType = new StructType() .add(new StructField("e", LongType.LONG, true).withTypeChanges( Seq( new TypeChange(ShortType.SHORT, IntegerType.INTEGER), new TypeChange(IntegerType.INTEGER, LongType.LONG)).asJava)) testRoundTrip(simpleTypeChangeJson, simpleTypeChangeDataType) // Case 2: Map key type change (float -> double) val mapKeyTypeChangeJson = structTypeJson(Seq( structFieldJson( "e", mapTypeJson("\"double\"", "\"integer\"", true), true, metadataJson = Some( """{ |"delta.typeChanges": [ | { | "fromType": "float", | "toType": "double", | "fieldPath": "key" | } |] |}""".stripMargin)))) val mapKeyTypeChangeDataType = new StructType() .add(new StructField( "e", new MapType( new StructField("key", DoubleType.DOUBLE, false) .withTypeChanges(Seq(new TypeChange(FloatType.FLOAT, DoubleType.DOUBLE)).asJava), new StructField("value", IntegerType.INTEGER, true)), true)) testRoundTrip(mapKeyTypeChangeJson, mapKeyTypeChangeDataType) // Case 3: Nested map value in array type change (decimal scale change) val nestedTypeChangeJson = structTypeJson(Seq( structFieldJson( "e", arrayTypeJson( mapTypeJson("\"string\"", "\"decimal(10,4)\"", true), true), true, metadataJson = Some( """{ |"delta.typeChanges": [ | { | "fromType": "decimal(6,2)", | "toType": "decimal(10,4)", | "fieldPath": "element.value" | } |] |}""".stripMargin)))) val nestedTypeChangeDataType = new StructType() .add( "e", new ArrayType( new MapType( new StructField("key", StringType.STRING, false), new StructField("value", new DecimalType(10, 4), true) .withTypeChanges( Seq(new TypeChange(new DecimalType(6, 2), new DecimalType(10, 4))).asJava)), true), true) testRoundTrip(nestedTypeChangeJson, nestedTypeChangeDataType) // Case 4: Combined type changes and collations val combinedJson = structTypeJson(Seq( structFieldJson( "tags", mapTypeJson("\"string\"", "\"string\"", false), true, metadataJson = Some( s"""{ |"$COLLATIONS_METADATA_KEY": { | "tags.value": "ICU.UNICODE" |}, |"delta.typeChanges": [ | { | "fromType": "binary", | "toType": "string", | "fieldPath": "value" | } |] |}""".stripMargin)))) val combinedDataType = new StructType() .add( "tags", new MapType( new StructField("key", StringType.STRING, false), new StructField("value", new StringType("ICU.UNICODE"), false) .withTypeChanges(Seq(new TypeChange(BinaryType.BINARY, StringType.STRING)).asJava)), true) testRoundTrip(combinedJson, combinedDataType) // Case 5: Deeply nested maps val deeplyNestedMapJson = structTypeJson(Seq( structFieldJson( "tags", mapTypeJson( /* key= */ mapTypeJson("\"integer\"", "\"integer\"", false), /* value= */ mapTypeJson("\"long\"", "\"long\"", false), false), true, metadataJson = Some( s"""{ |"delta.typeChanges": [ | { | "fromType": "byte", | "toType": "integer", | "fieldPath": "key.key" | }, | { | "fromType": "byte", | "toType": "short", | "fieldPath": "key.value" | }, | { | "fromType": "short", | "toType": "integer", | "fieldPath": "key.value" | }, | { | "fromType": "short", | "toType": "long", | "fieldPath": "value.key" | }, | { | "fromType": "byte", | "toType": "long", | "fieldPath": "value.value" | }] |}""".stripMargin)))) val deeplyNestedMapType = new StructType() .add( "tags", new MapType( new StructField( "key", new MapType( new StructField("key", IntegerType.INTEGER, false) .withTypeChanges(Seq(new TypeChange(ByteType.BYTE, IntegerType.INTEGER)).asJava), new StructField("value", IntegerType.INTEGER, false) .withTypeChanges( Seq( new TypeChange(ByteType.BYTE, ShortType.SHORT), new TypeChange(ShortType.SHORT, IntegerType.INTEGER)).asJava)), false), new StructField( "value", new MapType( new StructField("key", LongType.LONG, false) .withTypeChanges(Seq(new TypeChange(ShortType.SHORT, LongType.LONG)).asJava), new StructField("value", LongType.LONG, false) .withTypeChanges(Seq(new TypeChange(ByteType.BYTE, LongType.LONG)).asJava)), false)), true); testRoundTrip(deeplyNestedMapJson, deeplyNestedMapType) } test("serialize/deserialize: geometry and geography as nested types") { // Array of geometry testRoundTrip( arrayTypeJson("\"geometry(OGC:CRS84)\"", false), new ArrayType(GeometryType.ofDefault(), false)) // Array of geography with non-default SRID and algorithm testRoundTrip( arrayTypeJson("\"geography(EPSG:4326, vincenty)\"", true), new ArrayType(new GeographyType("EPSG:4326", "vincenty"), true)) // Struct with geometry and geography fields testRoundTrip( structTypeJson(Seq( structFieldJson("geom", "\"geometry(EPSG:4326)\"", false), structFieldJson("geog", "\"geography(EPSG:3857, spherical)\"", true))), new StructType() .add("geom", GeometryType.ofSRID("EPSG:4326"), false) .add("geog", new GeographyType("EPSG:3857", "spherical"), true)) // Struct containing arrays of geometry and geography testRoundTrip( structTypeJson(Seq( structFieldJson("geoms", arrayTypeJson("\"geometry(OGC:CRS84)\"", false), true), structFieldJson( "geogs", arrayTypeJson("\"geography(OGC:CRS84, vincenty)\"", true), false))), new StructType() .add("geoms", new ArrayType(GeometryType.ofDefault(), false), true) .add("geogs", new ArrayType(new GeographyType("OGC:CRS84", "vincenty"), true), false)) } test("serialize/deserialize: special characters for column name") { val json = structTypeJson(Seq( structFieldJson("@_! *c", "\"string\"", true))) val expectedType = new StructType() .add("@_! *c", StringType.STRING, true) testRoundTrip(json, expectedType) } test("serialize/deserialize: empty struct type") { val str = """ |{ | "type" : "struct", | "fields": [] |} |""".stripMargin testRoundTrip(str, new StructType()) } test("serialize/deserialize: parsing FieldMetadata") { def testFieldMetadata(fieldMetadataJson: String, expectedFieldMetadata: FieldMetadata): Unit = { val json = structTypeJson(Seq( structFieldJson("testCol", "\"string\"", true, Some(fieldMetadataJson)))) val dataType = new StructType().add("testCol", StringType.STRING, true, expectedFieldMetadata) testRoundTrip(json, dataType) } val fieldMetadataAllTypesJson = """ |{ | "null" : null, | "int" : 10, | "long-1" : -16070400023423400, | "long-2" : 16070400023423400, | "double" : 2.22, | "boolean" : true, | "string" : "10\"@", | "metadata" : { "nestedInt" : 200 }, | "empty_arr" : [], | "int_arr" : [1, 2, 0], | "double_arr" : [1.0, 2.0, 3.0], | "boolean_arr" : [true], | "string_arr" : ["one", "two"], | "metadata_arr" : [{ "one" : 1, "two" : true }, {}] |} |""".stripMargin val expectedFieldMetadataAllTypes = FieldMetadata.builder() .putNull("null") .putLong("int", 10) .putLong("long-1", -16070400023423400L) .putLong("long-2", 16070400023423400L) .putDouble("double", 2.22) .putBoolean("boolean", true) .putString("string", "10\"@") // special characters .putFieldMetadata("metadata", FieldMetadata.builder().putLong("nestedInt", 200).build()) .putLongArray("empty_arr", Array()) .putLongArray("int_arr", Array(1, 2, 0)) .putDoubleArray("double_arr", Array(1.0, 2.0, 3.0)) .putBooleanArray("boolean_arr", Array(true)) .putStringArray("string_arr", Array("one", "two")) .putFieldMetadataArray( "metadata_arr", Array( FieldMetadata.builder().putLong("one", 1).putBoolean("two", true).build(), FieldMetadata.empty())) .build() testFieldMetadata(fieldMetadataAllTypesJson, expectedFieldMetadataAllTypes) testFieldMetadata("{}", FieldMetadata.empty()) } test("parseDataType: invalid field for type") { checkError[IllegalArgumentException]( """ |{ | "type" : "foo", | "two" : "val2" |} |""".stripMargin, "Could not parse the following JSON as a valid Delta data type") } test("parseDataType: not a valid JSON node (not a string or object)") { checkError[IllegalArgumentException]( "0", "Could not parse the following JSON as a valid Delta data type") } } object DataTypeJsonSerDeSuite { val SAMPLE_JSON_TO_TYPES = Seq( ("\"string\"", StringType.STRING), ("\"integer\"", IntegerType.INTEGER), ("\"variant\"", VariantType.VARIANT), (arrayTypeJson("\"string\"", true), new ArrayType(StringType.STRING, true)), ( mapTypeJson("\"integer\"", "\"string\"", true), new MapType(IntegerType.INTEGER, StringType.STRING, true)), ( structTypeJson(Seq( structFieldJson("col1", "\"string\"", true), structFieldJson("col2", "\"string\"", false, Some("{ \"int\" : 0 }")), structFieldJson("col3", "\"variant\"", false))), new StructType() .add("col1", StringType.STRING, true) .add("col2", StringType.STRING, false, FieldMetadata.builder().putLong("int", 0).build()) .add("col3", VariantType.VARIANT, false))) val SAMPLE_COMPLEX_JSON_TO_TYPES = Seq( ( structTypeJson(Seq( structFieldJson("c1", "\"binary\"", true), structFieldJson("c2", "\"boolean\"", false), structFieldJson("c3", "\"byte\"", false), structFieldJson("c4", "\"date\"", true), structFieldJson("c5", "\"decimal(10,0)\"", false), structFieldJson("c6", "\"double\"", false), structFieldJson("c7", "\"float\"", false), structFieldJson("c8", "\"integer\"", true), structFieldJson("c9", "\"long\"", true), structFieldJson("c10", "\"short\"", true), structFieldJson("c11", "\"string\"", true), structFieldJson("c12", "\"timestamp_ntz\"", false), structFieldJson("c13", "\"timestamp\"", false), structFieldJson("c14", "\"variant\"", false))), new StructType() .add("c1", BinaryType.BINARY, true) .add("c2", BooleanType.BOOLEAN, false) .add("c3", ByteType.BYTE, false) .add("c4", DateType.DATE, true) .add("c5", DecimalType.USER_DEFAULT, false) .add("c6", DoubleType.DOUBLE, false) .add("c7", FloatType.FLOAT, false) .add("c8", IntegerType.INTEGER, true) .add("c9", LongType.LONG, true) .add("c10", ShortType.SHORT, true) .add("c11", StringType.STRING, true) .add("c12", TimestampNTZType.TIMESTAMP_NTZ, false) .add("c13", TimestampType.TIMESTAMP, false) .add("c14", VariantType.VARIANT, false)), ( structTypeJson(Seq( structFieldJson("a1", "\"string\"", true), structFieldJson( "a2", structTypeJson(Seq( structFieldJson( "b1", mapTypeJson( arrayTypeJson( arrayTypeJson( "\"string\"", true), true), structTypeJson(Seq( structFieldJson("c1", "\"string\"", false), structFieldJson("c2", "\"string\"", true))), true), true), structFieldJson("b2", "\"long\"", true))), true), structFieldJson( "a3", arrayTypeJson( mapTypeJson( "\"string\"", structTypeJson(Seq( structFieldJson("b1", "\"date\"", false))), false), false), true))), new StructType() .add("a1", StringType.STRING, true) .add( "a2", new StructType() .add( "b1", new MapType( new ArrayType( new ArrayType(StringType.STRING, true), true), new StructType() .add("c1", StringType.STRING, false) .add("c2", StringType.STRING, true), true)) .add("b2", LongType.LONG), true) .add( "a3", new ArrayType( new MapType( StringType.STRING, new StructType() .add("b1", DateType.DATE, false), false), false), true))) val SAMPLE_JSON_TO_TYPES_WITH_COLLATION = Seq( ( structTypeJson(Seq( structFieldJson( "a1", "\"string\"", true, metadataJson = Some(s"""{"$COLLATIONS_METADATA_KEY" : {"a1" : "ICU.UNICODE"}}""")), structFieldJson("a2", "\"integer\"", false), structFieldJson( "a3", "\"string\"", false, metadataJson = Some(s"""{"$COLLATIONS_METADATA_KEY" : {"a3" : "SPARK.UTF8_LCASE"}}""")), structFieldJson("a4", "\"string\"", true))), new StructType() .add("a1", new StringType("ICU.UNICODE"), true) .add("a2", IntegerType.INTEGER, false) .add("a3", new StringType("SPARK.UTF8_LCASE"), false) .add("a4", StringType.STRING, true)), ( structTypeJson(Seq( structFieldJson( "a1", structTypeJson(Seq( structFieldJson( "b1", "\"string\"", true, metadataJson = Some( s"""{"$COLLATIONS_METADATA_KEY" | : {"b1" : "ICU.UNICODE"}}""".stripMargin)))), true), structFieldJson( "a2", structTypeJson(Seq( structFieldJson( "b1", arrayTypeJson("\"string\"", false), true, metadataJson = Some( s"""{"$COLLATIONS_METADATA_KEY" | : {"b1.element" : "SPARK.UTF8_LCASE"}}""".stripMargin)), structFieldJson( "b2", mapTypeJson("\"string\"", "\"string\"", true), false, metadataJson = Some( s"""{"$COLLATIONS_METADATA_KEY" | : {"b2.value" : "SPARK.UTF8_LCASE"}}""".stripMargin)), structFieldJson("b3", arrayTypeJson("\"string\"", false), true), structFieldJson("b4", mapTypeJson("\"string\"", "\"string\"", false), false))), true), structFieldJson( "a3", structTypeJson(Seq( structFieldJson("b1", "\"string\"", false), structFieldJson("b2", arrayTypeJson("\"integer\"", false), true))), false, metadataJson = Some( s"""{"$COLLATIONS_METADATA_KEY" | : {"b1" : "SPARK.UTF8_LCASE"}}""".stripMargin)))), new StructType() .add( "a1", new StructType() .add("b1", new StringType("ICU.UNICODE")), true) .add( "a2", new StructType() .add("b1", new ArrayType(new StringType("SPARK.UTF8_LCASE"), false)) .add( "b2", new MapType( StringType.STRING, new StringType("SPARK.UTF8_LCASE"), true), false) .add("b3", new ArrayType(StringType.STRING, false)) .add( "b4", new MapType( StringType.STRING, StringType.STRING, false), false), true) .add( "a3", new StructType() .add("b1", StringType.STRING, false) .add("b2", new ArrayType(IntegerType.INTEGER, false), true), false)), ( structTypeJson(Seq( structFieldJson("a1", "\"string\"", true), structFieldJson( "a2", structTypeJson(Seq( structFieldJson( "b1", mapTypeJson( arrayTypeJson(arrayTypeJson("\"string\"", true), true), structTypeJson(Seq( structFieldJson( "c1", "\"string\"", false, metadataJson = Some( s"""{"$COLLATIONS_METADATA_KEY" | : {"c1" : "SPARK.UTF8_LCASE"}}""".stripMargin)), structFieldJson( "c2", "\"string\"", true, metadataJson = Some( s"""{"$COLLATIONS_METADATA_KEY" | : {\"c2\" : \"ICU.UNICODE\"}}""".stripMargin)), structFieldJson("c3", "\"string\"", true))), true), true, metadataJson = Some( s"""{"$COLLATIONS_METADATA_KEY" | : {"b1.key.element.element" : "SPARK.UTF8_LCASE"}}""".stripMargin)), structFieldJson("b2", "\"long\"", true))), true), structFieldJson( "a3", arrayTypeJson( mapTypeJson( "\"string\"", structTypeJson(Seq( structFieldJson( "b1", "\"string\"", false, metadataJson = Some( s"""{"$COLLATIONS_METADATA_KEY" | : {"b1" : "SPARK.UTF8_LCASE"}}""".stripMargin)))), false), false), true), structFieldJson( "a4", arrayTypeJson( structTypeJson(Seq( structFieldJson( "b1", "\"string\"", false, metadataJson = Some( s"""{"$COLLATIONS_METADATA_KEY" | : {"b1" : "SPARK.UTF8_LCASE"}}""".stripMargin)))), false), false), structFieldJson( "a5", mapTypeJson( structTypeJson(Seq( structFieldJson( "b1", "\"string\"", false, metadataJson = Some( s"""{"$COLLATIONS_METADATA_KEY" | : {"b1" : "SPARK.UTF8_LCASE"}}""".stripMargin)))), "\"string\"", false), false))), new StructType() .add("a1", StringType.STRING, true) .add( "a2", new StructType() .add( "b1", new MapType( new ArrayType( new ArrayType( new StringType("SPARK.UTF8_LCASE"), true), true), new StructType() .add("c1", new StringType("SPARK.UTF8_LCASE"), false) .add("c2", new StringType("ICU.UNICODE"), true) .add("c3", StringType.STRING), true)) .add("b2", LongType.LONG), true) .add( "a3", new ArrayType( new MapType( StringType.STRING, new StructType() .add("b1", new StringType("SPARK.UTF8_LCASE"), false), false), false), true) .add( "a4", new ArrayType( new StructType() .add("b1", new StringType("SPARK.UTF8_LCASE"), false), false), false) .add( "a5", new MapType( new StructType() .add("b1", new StringType("SPARK.UTF8_LCASE"), false), StringType.STRING, false), false))) def arrayTypeJson(elementJson: String, containsNull: Boolean): String = { s""" |{ | "type" : "array", | "elementType" : $elementJson, | "containsNull" : $containsNull |} |""".stripMargin } def mapTypeJson(keyJson: String, valueJson: String, valueContainsNull: Boolean): String = { s""" |{ | "type" : "map", | "keyType" : $keyJson, | "valueType" : $valueJson, | "valueContainsNull" : $valueContainsNull |} |""".stripMargin } def structFieldJson( name: String, typeJson: String, nullable: Boolean, metadataJson: Option[String] = None): String = { metadataJson match { case Some(metadata) => s""" |{ | "name" : "$name", | "type" : $typeJson, | "nullable" : $nullable, | "metadata" : $metadata |} |""".stripMargin case None => s""" |{ | "name" : "$name", | "type" : $typeJson, | "nullable" : $nullable |} |""".stripMargin } } def structTypeJson(fieldsJsons: Seq[String]): String = { s""" |{ | "type" : "struct", | "fields": ${fieldsJsons.mkString("[\n", ",\n", "]\n")} |} |""".stripMargin } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/types/TypeWideningCheckerSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.types import io.delta.kernel.types._ import org.scalatest.funsuite.AnyFunSuite /** Test suite for the TypeWideningChecker class. */ class TypeWideningCheckerSuite extends AnyFunSuite { test("same type is allowed") { // Same types should not be considered widening assert(TypeWideningChecker.isWideningSupported(IntegerType.INTEGER, IntegerType.INTEGER)) assert(TypeWideningChecker.isWideningSupported(StringType.STRING, StringType.STRING)) assert(TypeWideningChecker.isWideningSupported( new DecimalType(10, 2), new DecimalType(10, 2))) } test("integer widening is supported") { assert(TypeWideningChecker.isWideningSupported(ByteType.BYTE, ShortType.SHORT)) assert(TypeWideningChecker.isWideningSupported(ByteType.BYTE, IntegerType.INTEGER)) assert(TypeWideningChecker.isWideningSupported(ByteType.BYTE, LongType.LONG)) assert(TypeWideningChecker.isWideningSupported(ShortType.SHORT, IntegerType.INTEGER)) assert(TypeWideningChecker.isWideningSupported(ShortType.SHORT, LongType.LONG)) assert(TypeWideningChecker.isWideningSupported(IntegerType.INTEGER, LongType.LONG)) } test("integer type narrowing is not supported") { assert(!TypeWideningChecker.isWideningSupported(LongType.LONG, IntegerType.INTEGER)) assert(!TypeWideningChecker.isWideningSupported(IntegerType.INTEGER, ShortType.SHORT)) assert(!TypeWideningChecker.isWideningSupported(ShortType.SHORT, ByteType.BYTE)) } test("float to double widening") { assert(TypeWideningChecker.isWideningSupported(FloatType.FLOAT, DoubleType.DOUBLE)) } test("double to float is not supported") { assert(!TypeWideningChecker.isWideningSupported(DoubleType.DOUBLE, FloatType.FLOAT)) } test("integer to double widening supported") { Seq(ByteType.BYTE, ShortType.SHORT, IntegerType.INTEGER) foreach { t => assert(TypeWideningChecker.isWideningSupported(t, DoubleType.DOUBLE)) } } test("unsupported integral to floating point no supported") { // Test invalid integer to double widening assert(!TypeWideningChecker.isWideningSupported(LongType.LONG, DoubleType.DOUBLE)) Seq(ByteType.BYTE, ShortType.SHORT, IntegerType.INTEGER) foreach { t => assert(!TypeWideningChecker.isWideningSupported(t, FloatType.FLOAT)) } } test("date to timestamp NTZ widening") { // Test Date -> TimestampNTZ widening assert(TypeWideningChecker.isWideningSupported(DateType.DATE, TimestampNTZType.TIMESTAMP_NTZ)) } test("decimal widening supported") { // Test Decimal(p, s) -> Decimal(p + k1, s + k2) where k1 >= k2 >= 0 // Same scale, increased precision assert(TypeWideningChecker.isWideningSupported(new DecimalType(5, 2), new DecimalType(10, 2))) // Increased scale and precision, with precision increase >= scale increase assert(TypeWideningChecker.isWideningSupported(new DecimalType(5, 2), new DecimalType(10, 5))) assert(TypeWideningChecker.isWideningSupported(new DecimalType(5, 2), new DecimalType(10, 5))) assert(TypeWideningChecker.isWideningSupported(new DecimalType(5, 2), new DecimalType(8, 4))) } test("decimal widening unsupported") { // Invalid decimal widening assert(!TypeWideningChecker.isWideningSupported(new DecimalType(10, 2), new DecimalType(5, 2))) assert(!TypeWideningChecker.isWideningSupported(new DecimalType(10, 2), new DecimalType(10, 1))) assert(!TypeWideningChecker.isWideningSupported(new DecimalType(10, 2), new DecimalType(9, 2))) assert(!TypeWideningChecker.isWideningSupported( new DecimalType(10, 5), new DecimalType(12, 8) )) // k1 < k2 assert(!TypeWideningChecker.isWideningSupported( new DecimalType(10, 5), new DecimalType(10, 3) )) // scale decrease } test("integer to decimal supported widening") { // Test Byte, Short, Int -> Decimal(10 + k1, k2) where k1 >= k2 >= 0 Seq(ByteType.BYTE, ShortType.SHORT, IntegerType.INTEGER) foreach { t => assert(TypeWideningChecker.isWideningSupported(t, new DecimalType(10, 0))) assert(TypeWideningChecker.isWideningSupported(t, new DecimalType(11, 0))) assert(TypeWideningChecker.isWideningSupported(t, new DecimalType(12, 2))) assert(TypeWideningChecker.isWideningSupported(t, new DecimalType(15, 3))) } // Test Long -> Decimal(20 + k1, k2) where k1 >= k2 >= 0 assert(TypeWideningChecker.isWideningSupported(LongType.LONG, new DecimalType(20, 0))) assert(TypeWideningChecker.isWideningSupported(LongType.LONG, new DecimalType(25, 5))) } test("integer to Decimal unsupported widening") { Seq(ByteType.BYTE, ShortType.SHORT, IntegerType.INTEGER) foreach { t => assert(!TypeWideningChecker.isWideningSupported( t, new DecimalType(9, 0) )) // precision < 10 assert(!TypeWideningChecker.isWideningSupported( t, new DecimalType(12, 3) )) // k1 < k2 } assert(!TypeWideningChecker.isWideningSupported( LongType.LONG, new DecimalType(19, 0) )) // precision < 20 } test("unsupported widening") { // Test unsupported widening operations assert(!TypeWideningChecker.isWideningSupported(StringType.STRING, BinaryType.BINARY)) assert(!TypeWideningChecker.isWideningSupported(IntegerType.INTEGER, StringType.STRING)) assert(!TypeWideningChecker.isWideningSupported(DateType.DATE, StringType.STRING)) assert(!TypeWideningChecker.isWideningSupported(DoubleType.DOUBLE, new DecimalType(10, 2))) // Test invalid date widening assert(!TypeWideningChecker.isWideningSupported(DateType.DATE, TimestampType.TIMESTAMP)) assert(!TypeWideningChecker.isWideningSupported(TimestampNTZType.TIMESTAMP_NTZ, DateType.DATE)) } test("Iceberg V2 compatible widening") { // Test Iceberg V2 compatible widening // Integer widening assert(TypeWideningChecker.isIcebergV2Compatible(ByteType.BYTE, ShortType.SHORT)) assert(TypeWideningChecker.isIcebergV2Compatible(ShortType.SHORT, IntegerType.INTEGER)) assert(TypeWideningChecker.isIcebergV2Compatible(IntegerType.INTEGER, LongType.LONG)) // Float -> Double widening assert(TypeWideningChecker.isIcebergV2Compatible(FloatType.FLOAT, DoubleType.DOUBLE)) // Decimal precision increase (without scale increase) assert(TypeWideningChecker.isIcebergV2Compatible( new DecimalType(5, 2), new DecimalType(10, 2))) } test("iceberg V2 unsupported type widening") { ///////////////////////////////////////////////////////////////////////////////////// // Test generally unsupported widening operations ///////////////////////////////////////////////////////////////////////////////////// assert(!TypeWideningChecker.isIcebergV2Compatible(StringType.STRING, BinaryType.BINARY)) assert(!TypeWideningChecker.isIcebergV2Compatible(IntegerType.INTEGER, StringType.STRING)) assert(!TypeWideningChecker.isIcebergV2Compatible(DateType.DATE, StringType.STRING)) assert(!TypeWideningChecker.isIcebergV2Compatible(DoubleType.DOUBLE, new DecimalType(10, 2))) assert(!TypeWideningChecker.isIcebergV2Compatible(DateType.DATE, TimestampType.TIMESTAMP)) assert(!TypeWideningChecker.isIcebergV2Compatible( TimestampNTZType.TIMESTAMP_NTZ, DateType.DATE)) //////////////////////////////////////////////////////////////////////////////////// // Test invalid widening that are generally supported by Delta but not by Iceberg V2 //////////////////////////////////////////////////////////////////////////////////// // Integer to Double widening (not supported by Iceberg) assert(!TypeWideningChecker.isIcebergV2Compatible(ByteType.BYTE, DoubleType.DOUBLE)) assert(!TypeWideningChecker.isIcebergV2Compatible(ShortType.SHORT, DoubleType.DOUBLE)) assert(!TypeWideningChecker.isIcebergV2Compatible(IntegerType.INTEGER, DoubleType.DOUBLE)) // Date to TimestampNTZ widening (not supported by Iceberg) assert(!TypeWideningChecker.isIcebergV2Compatible( DateType.DATE, TimestampNTZType.TIMESTAMP_NTZ)) // Decimal scale increase (not supported by Iceberg) assert(!TypeWideningChecker.isIcebergV2Compatible( new DecimalType(5, 2), new DecimalType(10, 5))) // Integer to Decimal widening (not supported by Iceberg) assert(!TypeWideningChecker.isIcebergV2Compatible(ByteType.BYTE, new DecimalType(10, 0))) assert(!TypeWideningChecker.isIcebergV2Compatible(ShortType.SHORT, new DecimalType(12, 2))) assert(!TypeWideningChecker.isIcebergV2Compatible( IntegerType.INTEGER, new DecimalType(15, 3))) assert(!TypeWideningChecker.isIcebergV2Compatible(LongType.LONG, new DecimalType(20, 0))) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/ColumnMappingSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util import java.util import scala.collection.JavaConverters.mapAsJavaMapConverter import io.delta.kernel.exceptions.KernelException import io.delta.kernel.expressions.Column import io.delta.kernel.internal.actions.Metadata import io.delta.kernel.internal.util.ColumnMapping._ import io.delta.kernel.internal.util.ColumnMapping.ColumnMappingMode._ import io.delta.kernel.types._ import org.assertj.core.api.Assertions.{assertThat, assertThatNoException, assertThatThrownBy} import org.assertj.core.util.Maps import org.scalatest.funsuite.AnyFunSuite class ColumnMappingSuite extends AnyFunSuite with ColumnMappingSuiteBase { test("column mapping is only enabled on known mapping modes") { assertThat(ColumnMapping.isColumnMappingModeEnabled(null)).isFalse assertThat(ColumnMapping.isColumnMappingModeEnabled(NONE)).isFalse assertThat(ColumnMapping.isColumnMappingModeEnabled(NAME)).isTrue assertThat(ColumnMapping.isColumnMappingModeEnabled(ID)).isTrue } test("column mapping change with empty config") { assertThatNoException.isThrownBy(() => ColumnMapping.verifyColumnMappingChange( new util.HashMap(), new util.HashMap())) } test("column mapping mode change not allowed on existing table") { assertThatThrownBy(() => ColumnMapping.verifyColumnMappingChange( Maps.newHashMap(COLUMN_MAPPING_MODE_KEY, NAME.toString), Maps.newHashMap(COLUMN_MAPPING_MODE_KEY, ID.toString))) .isInstanceOf(classOf[IllegalArgumentException]) .hasMessage("Changing column mapping mode from 'name' to 'id' is not supported") assertThatThrownBy(() => ColumnMapping.verifyColumnMappingChange( Maps.newHashMap(COLUMN_MAPPING_MODE_KEY, ID.toString), Maps.newHashMap(COLUMN_MAPPING_MODE_KEY, NAME.toString))) .isInstanceOf(classOf[IllegalArgumentException]) .hasMessage("Changing column mapping mode from 'id' to 'name' is not supported") assertThatThrownBy(() => ColumnMapping.verifyColumnMappingChange( new util.HashMap(), Maps.newHashMap(COLUMN_MAPPING_MODE_KEY, ID.toString))) .isInstanceOf(classOf[IllegalArgumentException]) .hasMessage("Changing column mapping mode from 'none' to 'id' is not supported") } test("finding max column id with different schemas") { assertThat(ColumnMapping.findMaxColumnId(new StructType)).isEqualTo(0) assertThat(ColumnMapping.findMaxColumnId( new StructType() .add("a", StringType.STRING, true) .add("b", IntegerType.INTEGER, true))) .isEqualTo(0) assertThat(ColumnMapping.findMaxColumnId( new StructType() .add("a", StringType.STRING, createMetadataWithFieldId(14)) .add("b", IntegerType.INTEGER, createMetadataWithFieldId(17)) .add("c", IntegerType.INTEGER, createMetadataWithFieldId(3)))) .isEqualTo(17) // nested columns are currently not supported assertThat(ColumnMapping.findMaxColumnId( new StructType().add("a", StringType.STRING, createMetadataWithFieldId(14)) .add( "b", new StructType() .add("d", IntegerType.INTEGER, true) .add("e", IntegerType.INTEGER, true), createMetadataWithFieldId(15)) .add("c", IntegerType.INTEGER, createMetadataWithFieldId(7)))) .isEqualTo(15) } test("finding max column id with nested struct type") { val nestedStruct = new StructType() .add("d", IntegerType.INTEGER, createMetadataWithFieldId(3)) .add("e", IntegerType.INTEGER, createMetadataWithFieldId(4)) val schema = new StructType() .add("a", StringType.STRING, createMetadataWithFieldId(1)) .add("b", nestedStruct, createMetadataWithFieldId(2)) .add("c", IntegerType.INTEGER, createMetadataWithFieldId(5)) assertThat(ColumnMapping.findMaxColumnId(schema)).isEqualTo(5) } test("finding max column id with nested struct type and random ids") { val nestedStruct = new StructType() .add("d", IntegerType.INTEGER, createMetadataWithFieldId(2)) .add("e", IntegerType.INTEGER, createMetadataWithFieldId(1)) val schema = new StructType() .add("a", StringType.STRING, createMetadataWithFieldId(3)) .add("b", nestedStruct, createMetadataWithFieldId(4)) .add("c", IntegerType.INTEGER, createMetadataWithFieldId(5)) assertThat(ColumnMapping.findMaxColumnId(schema)).isEqualTo(5) } test("finding max column id with nested array type") { val nestedStruct = new StructType() .add("e", IntegerType.INTEGER, createMetadataWithFieldId(4)) .add("f", IntegerType.INTEGER, createMetadataWithFieldId(5)) val nestedMeta = FieldMetadata.builder() .putLong(COLUMN_MAPPING_ID_KEY, 2) .putFieldMetadata( COLUMN_MAPPING_NESTED_IDS_KEY, FieldMetadata.builder().putLong("b.element", 6).build()) .build() val schema = new StructType() .add("a", StringType.STRING, createMetadataWithFieldId(1)) .add("b", new ArrayType(new StructField("d", nestedStruct, false)), nestedMeta) .add("c", IntegerType.INTEGER, createMetadataWithFieldId(3)) assertThat(ColumnMapping.findMaxColumnId(schema)).isEqualTo(6) } test("finding max column id with nested map type") { val nestedStruct = new StructType() .add("e", IntegerType.INTEGER, createMetadataWithFieldId(4)) .add("f", IntegerType.INTEGER, createMetadataWithFieldId(5)) val nestedMeta = FieldMetadata.builder() .putLong(COLUMN_MAPPING_ID_KEY, 2) .putFieldMetadata( COLUMN_MAPPING_NESTED_IDS_KEY, FieldMetadata.builder() .putLong("b.key", 11) .putLong("b.value", 12).build()) .build() val schema = new StructType() .add("a", StringType.STRING, createMetadataWithFieldId(1)) .add( "b", new MapType( IntegerType.INTEGER, new StructField("d", nestedStruct, false).getDataType, false), nestedMeta) .add("c", IntegerType.INTEGER, createMetadataWithFieldId(3)) assertThat(ColumnMapping.findMaxColumnId(schema)).isEqualTo(12) } private val testingSchema = new StructType() .add("a", StringType.STRING) .add( "b", new StructType() .add("c", DoubleType.DOUBLE) .add("d", DateType.DATE)) .add("e", FloatType.FLOAT) .add( "f", new StructType() .add( "g", new StructType() .add("h", TimestampNTZType.TIMESTAMP_NTZ))) .add("i", new MapType(StringType.STRING, DoubleType.DOUBLE, false)) .add("j", new ArrayType(StringType.STRING, false)) Seq( (Array("a"), StringType.STRING), (Array("b", "c"), DoubleType.DOUBLE), (Array("b", "d"), DateType.DATE), (Array("e"), FloatType.FLOAT), (Array("f", "g", "h"), TimestampNTZType.TIMESTAMP_NTZ), (Array("i"), new MapType(StringType.STRING, DoubleType.DOUBLE, false)), (Array("j"), new ArrayType(StringType.STRING, false))).foreach { case (columnName, expectedType) => test(s"get physical column name and dataType for ${columnName.mkString(".")}") { // case 1: column mapping disabled val column = new Column(columnName) val resultTuple = ColumnMapping.getPhysicalColumnNameAndDataType(testingSchema, column) val actualColumn = resultTuple._1 val actualType = resultTuple._2 assert(actualColumn == column) assert(actualType == expectedType) // case 2: column mapping enabled val metadata: Metadata = updateColumnMappingMetadataIfNeeded( testMetadata(testingSchema).withColumnMappingEnabled("id"), true).orElseGet(() => fail("Metadata should not be empty")) val physicalResultTuple = ColumnMapping.getPhysicalColumnNameAndDataType( metadata.getSchema, column) val actualPhysicalColumn = physicalResultTuple._1 val actualPhysicalType = physicalResultTuple._2 assert(actualPhysicalColumn.getNames.length == columnName.length) assert(actualPhysicalType == expectedType) // case 3: round-trip test - physical back to logical val logicalResultTuple = ColumnMapping.getLogicalColumnNameAndDataType( metadata.getSchema, actualPhysicalColumn) assert(logicalResultTuple._1 == column) assert(logicalResultTuple._2 == expectedType) } } Seq( (Array("A"), Array("a"), StringType.STRING), (Array("B", "C"), Array("b", "c"), DoubleType.DOUBLE), (Array("B", "D"), Array("b", "d"), DateType.DATE), (Array("E"), Array("e"), FloatType.FLOAT), (Array("F", "G", "H"), Array("f", "g", "h"), TimestampNTZType.TIMESTAMP_NTZ), (Array("I"), Array("i"), new MapType(StringType.STRING, DoubleType.DOUBLE, false)), (Array("J"), Array("j"), new ArrayType(StringType.STRING, false))).foreach { case (inputColumnName, expectedColumnName, expectedType) => val inputColumnNameStr = inputColumnName.mkString(".") test(s"get physical column name should respect case of table schema, $inputColumnNameStr") { val column = new Column(inputColumnName) val resultTuple = ColumnMapping.getPhysicalColumnNameAndDataType(testingSchema, column) val actualColumn = resultTuple._1 val actualType = resultTuple._2 assert(actualColumn == new Column(expectedColumnName)) assert(actualType == expectedType) } } test("getPhysicalColumnNameAndDataType: exception expected when column does not exist") { val ex = intercept[KernelException] { ColumnMapping.getPhysicalColumnNameAndDataType( new StructType() .add("A", StringType.STRING) .add("b", IntegerType.INTEGER), new Column("abc")) } assert(ex.getMessage.contains("Column 'column(`abc`)' was not found in the table schema")) val ex1 = intercept[KernelException] { ColumnMapping.getPhysicalColumnNameAndDataType( new StructType().add("a", StringType.STRING) .add( "b", new StructType() .add("D", IntegerType.INTEGER) .add("e", IntegerType.INTEGER)) .add("c", IntegerType.INTEGER), new Column(Array("Bbb", "d"))) } assert(ex1.getMessage.contains("Column 'column(`Bbb`.`d`)' was not found in the table schema")) } test("getLogicalColumnNameAndDataType: without column mapping") { val logicalTuple = ColumnMapping.getLogicalColumnNameAndDataType( testingSchema, new Column(Array("b", "c"))) assert(logicalTuple._1 === new Column(Array("b", "c"))) assert(logicalTuple._2 === DoubleType.DOUBLE) } test("getLogicalColumnNameAndDataType: with column mapping") { val metadata = updateColumnMappingMetadataIfNeeded( testMetadata(testingSchema).withColumnMappingEnabled("id"), true).orElseGet(() => fail("Metadata should not be empty")) // Test simple column lookup { val physicalTuple = ColumnMapping.getPhysicalColumnNameAndDataType( metadata.getSchema, new Column("a")) val physicalColumn = physicalTuple._1 val logicalTuple = ColumnMapping.getLogicalColumnNameAndDataType( metadata.getSchema, physicalColumn) assert(logicalTuple._1 === new Column("a")) assert(logicalTuple._2 === StringType.STRING) } // Test nested column lookup { val physicalTuple = ColumnMapping.getPhysicalColumnNameAndDataType( metadata.getSchema, new Column(Array("b", "c"))) val physicalColumn = physicalTuple._1 val logicalTuple = ColumnMapping.getLogicalColumnNameAndDataType( metadata.getSchema, physicalColumn) assert(logicalTuple._1 === new Column(Array("b", "c"))) assert(logicalTuple._2 === DoubleType.DOUBLE) } } test("getLogicalColumnNameAndDataType: with column mapping + explicit physical schema") { // Create a simple schema with explicit physical column names val schemaWithPhysicalNames = new StructType() .add(new StructField( "logicalCol", StringType.STRING, true, FieldMetadata.builder() .putLong(COLUMN_MAPPING_ID_KEY, 1L) .putString(COLUMN_MAPPING_PHYSICAL_NAME_KEY, "col-uuid-123") .build())) .add(new StructField( "nestedStruct", new StructType() .add(new StructField( "innerCol", IntegerType.INTEGER, true, FieldMetadata.builder() .putLong(COLUMN_MAPPING_ID_KEY, 3L) .putString(COLUMN_MAPPING_PHYSICAL_NAME_KEY, "col-uuid-456") .build())), true, FieldMetadata.builder() .putLong(COLUMN_MAPPING_ID_KEY, 2L) .putString(COLUMN_MAPPING_PHYSICAL_NAME_KEY, "col-uuid-789") .build())) // Test simple column lookup val simpleResult = ColumnMapping.getLogicalColumnNameAndDataType( schemaWithPhysicalNames, new Column("col-uuid-123")) assert(simpleResult._1 === new Column("logicalCol")) assert(simpleResult._2 === StringType.STRING) // Test nested column lookup val nestedResult = ColumnMapping.getLogicalColumnNameAndDataType( schemaWithPhysicalNames, new Column(Array("col-uuid-789", "col-uuid-456"))) assert(nestedResult._1 === new Column(Array("nestedStruct", "innerCol"))) assert(nestedResult._2 === IntegerType.INTEGER) } test("getLogicalColumnNameAndDataType: exception when physical column not found") { val ex = intercept[KernelException] { ColumnMapping.getLogicalColumnNameAndDataType( testingSchema, new Column("foo")) } assert(ex.getMessage.contains("Column 'column(`foo`)' was not found in the table schema")) } Seq(true, false).foreach { isNewTable => test(s"assign id and physical name to new table: $isNewTable") { val schema: StructType = new StructType() .add("a", StringType.STRING, true) .add("b", StringType.STRING, true) val metadata: Metadata = updateColumnMappingMetadataIfNeeded( testMetadata(schema).withColumnMappingEnabled("id"), isNewTable).orElseGet(() => fail("Metadata should not be empty")) assertColumnMapping(metadata.getSchema.get("a"), 1L, if (isNewTable) "UUID" else "a") assertColumnMapping(metadata.getSchema.get("b"), 2L, if (isNewTable) "UUID" else "b") assertThat(metadata.getConfiguration) .containsEntry(ColumnMapping.COLUMN_MAPPING_MAX_COLUMN_ID_KEY, "2") // Requesting the same operation on the same schema shouldn't change anything // as the schema already has the necessary column mapping info assertNoOpOnUpdateColumnMappingMetadataRequest( metadata.getSchema, enableIcebergCompatV2 = false, isNewTable) } } test("none mapping mode returns original schema") { val schema = new StructType().add("a", StringType.STRING, true) assertThat(updateColumnMappingMetadataIfNeeded(testMetadata(schema), true)).isEmpty } test("assigning id and physical name preserves field metadata") { val schema = new StructType() .add( "a", StringType.STRING, FieldMetadata.builder.putString("key1", "val1").putString("key2", "val2").build) val metadata = updateColumnMappingMetadataIfNeeded( testMetadata(schema).withColumnMappingEnabled(), true).orElseGet(() => fail("Metadata should not be empty")) val fieldMetadata = metadata.getSchema.get("a").getMetadata.getEntries assertThat(fieldMetadata) .containsEntry("key1", "val1") .containsEntry("key2", "val2") .containsEntry(ColumnMapping.COLUMN_MAPPING_ID_KEY, (1L).asInstanceOf[AnyRef]) .hasEntrySatisfying( ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, (k: AnyRef) => assertThat(k).asString.startsWith("col-")) } runWithIcebergCompatComboForNewAndExistingTables( "assign id and physical name to schema with nested struct type") { (isNewTable, enableIcebergCompatV2, enableIcebergWriterCompatV1) => val schema: StructType = new StructType() .add("a", StringType.STRING) .add( "b", new StructType() .add("d", IntegerType.INTEGER) .add("e", IntegerType.INTEGER)) .add("c", IntegerType.INTEGER) var inputMetadata = testMetadata(schema).withColumnMappingEnabled("id") if (enableIcebergCompatV2) { inputMetadata = inputMetadata.withIcebergCompatV2Enabled } if (enableIcebergWriterCompatV1) { inputMetadata = inputMetadata.withIcebergWriterCompatV1Enabled } val metadata = updateColumnMappingMetadataIfNeeded(inputMetadata, isNewTable) .orElseGet(() => fail("Metadata should not be empty")) assertColumnMapping(metadata.getSchema.get("a"), 1L, isNewTable, enableIcebergWriterCompatV1) assertColumnMapping(metadata.getSchema.get("b"), 2L, isNewTable, enableIcebergWriterCompatV1) val innerStruct = metadata.getSchema.get("b").getDataType.asInstanceOf[StructType] assertColumnMapping(innerStruct.get("d"), 3L, isNewTable, enableIcebergWriterCompatV1) assertColumnMapping(innerStruct.get("e"), 4L, isNewTable, enableIcebergWriterCompatV1) assertColumnMapping(metadata.getSchema.get("c"), 5L, isNewTable, enableIcebergWriterCompatV1) assertThat(metadata.getConfiguration) .containsEntry(ColumnMapping.COLUMN_MAPPING_MAX_COLUMN_ID_KEY, "5") // Requesting the same operation on the same schema shouldn't change anything // as the schema already has the necessary column mapping info assertNoOpOnUpdateColumnMappingMetadataRequest( metadata.getSchema, enableIcebergCompatV2, isNewTable) } runWithIcebergCompatComboForNewAndExistingTables( "assign id and physical name to schema with array type") { (isNewTable, enableIcebergCompatV2, enableIcebergWriterCompatV1) => val schema: StructType = new StructType() .add("a", StringType.STRING) .add("b", new ArrayType(IntegerType.INTEGER, false)) .add("c", IntegerType.INTEGER) var inputMetadata = testMetadata(schema).withColumnMappingEnabled("id") if (enableIcebergCompatV2) { inputMetadata = inputMetadata.withIcebergCompatV2Enabled } if (enableIcebergWriterCompatV1) { inputMetadata = inputMetadata.withIcebergWriterCompatV1Enabled } val metadata = updateColumnMappingMetadataIfNeeded(inputMetadata, isNewTable) .orElseGet(() => fail("Metadata should not be empty")) assertColumnMapping(metadata.getSchema.get("a"), 1L, isNewTable, enableIcebergWriterCompatV1) assertColumnMapping(metadata.getSchema.get("b"), 2L, isNewTable, enableIcebergWriterCompatV1) assertColumnMapping(metadata.getSchema.get("c"), 3L, isNewTable, enableIcebergWriterCompatV1) if (enableIcebergCompatV2) { val colPrefix = if (enableIcebergWriterCompatV1) { "col-2." } else if (isNewTable) { "col-" } else { "b." } // verify nested ids assertThat(metadata.getSchema.get("b").getMetadata.getEntries .get(COLUMN_MAPPING_NESTED_IDS_KEY).asInstanceOf[FieldMetadata].getEntries) .hasSize(1) .anySatisfy((k: AnyRef, v: AnyRef) => { assertThat(k).asString.startsWith(colPrefix) assertThat(k).asString.endsWith(".element") assertThat(v).isEqualTo(4L) }) assertThat(metadata.getConfiguration) .containsEntry(ColumnMapping.COLUMN_MAPPING_MAX_COLUMN_ID_KEY, "4") } else { assertThat(metadata.getSchema.get("b").getMetadata.getEntries) .doesNotContainKey(COLUMN_MAPPING_NESTED_IDS_KEY) assertThat(metadata.getConfiguration) .containsEntry(ColumnMapping.COLUMN_MAPPING_MAX_COLUMN_ID_KEY, "3") } // Requesting the same operation on the same schema shouldn't change anything // as the schema already has the necessary column mapping info assertNoOpOnUpdateColumnMappingMetadataRequest( metadata.getSchema, enableIcebergCompatV2, isNewTable) } runWithIcebergCompatComboForNewAndExistingTables( "assign id and physical name to schema with map type") { (isNewTable, enableIcebergCompatV2, enableIcebergWriterCompatV1) => val schema: StructType = new StructType() .add("a", StringType.STRING) .add("b", new MapType(IntegerType.INTEGER, StringType.STRING, false)) .add("c", IntegerType.INTEGER) var inputMetadata = testMetadata(schema).withColumnMappingEnabled("id") if (enableIcebergCompatV2) { inputMetadata = inputMetadata.withIcebergCompatV2Enabled } if (enableIcebergWriterCompatV1) { inputMetadata = inputMetadata.withIcebergWriterCompatV1Enabled } val metadata = updateColumnMappingMetadataIfNeeded(inputMetadata, isNewTable) .orElseGet(() => fail("Metadata should not be empty")) assertColumnMapping(metadata.getSchema.get("a"), 1L, isNewTable, enableIcebergWriterCompatV1) assertColumnMapping(metadata.getSchema.get("b"), 2L, isNewTable, enableIcebergWriterCompatV1) assertColumnMapping(metadata.getSchema.get("c"), 3L, isNewTable, enableIcebergWriterCompatV1) if (enableIcebergCompatV2) { val colPrefix = if (enableIcebergWriterCompatV1) { "col-2." } else if (isNewTable) { "col-" } else { "b." } assert( metadata.getSchema.get( "b").getMetadata.getMetadata(COLUMN_MAPPING_NESTED_IDS_KEY) != null, s"${metadata.getSchema}") // verify nested ids assertThat(metadata.getSchema.get("b").getMetadata.getEntries .get(COLUMN_MAPPING_NESTED_IDS_KEY).asInstanceOf[FieldMetadata].getEntries) .hasSize(2) .anySatisfy((k: AnyRef, v: AnyRef) => { assertThat(k).asString.startsWith(colPrefix) assertThat(k).asString.endsWith(".key") assertThat(v).isEqualTo(4L) }) .anySatisfy((k: AnyRef, v: AnyRef) => { assertThat(k).asString.startsWith(colPrefix) assertThat(k).asString.endsWith(".value") assertThat(v).isEqualTo(5L) }) } else { assertThat(metadata.getSchema.get("b").getMetadata.getEntries) .doesNotContainKey(COLUMN_MAPPING_NESTED_IDS_KEY) } assertThat(metadata.getConfiguration).containsEntry( ColumnMapping.COLUMN_MAPPING_MAX_COLUMN_ID_KEY, if (enableIcebergCompatV2) "5" else "3") // Requesting the same operation on the same schema shouldn't change anything // as the schema already has the necessary column mapping info assertNoOpOnUpdateColumnMappingMetadataRequest( metadata.getSchema, enableIcebergCompatV2, isNewTable) } Seq(false, true).foreach { isNewTable => val baseSchema: StructType = new StructType() .add( "l", new ArrayType( new ArrayType( new MapType( new ArrayType( StringType.STRING, /* nullable= */ false), new MapType( StringType.STRING, new StructType().add( "leaf", StringType.STRING, false, FieldMetadata.builder().putBoolean("k1", true).build()), /* valuesContainNull= */ false), /* valuesContainNull= */ false), /* nullableElements= */ false), /* nullableElements= */ false), /* nullable= */ false, FieldMetadata.builder().putBoolean("k2", true).build()) Seq( (baseSchema, 1, (md: Metadata) => md.getSchema.get("l")), ( new StructType().add( "p", baseSchema, /* nullable= */ false), 2, (md: Metadata) => md.getSchema.get("p").getDataType.asInstanceOf[StructType].get("l"))).foreach { case (schemaToTest, base, getter) => test(s"Deeply nested values don't assign more field IDs then necessary $isNewTable $base") { var inputMetadata = testMetadata(schemaToTest).withColumnMappingEnabled("id") inputMetadata = inputMetadata.withIcebergCompatV2Enabled inputMetadata = inputMetadata.withIcebergWriterCompatV1Enabled val metadata = updateColumnMappingMetadataIfNeeded(inputMetadata, isNewTable) .orElseGet(() => fail("Metadata should not be empty")) assertThat(metadata.getConfiguration).containsEntry( ColumnMapping.COLUMN_MAPPING_MAX_COLUMN_ID_KEY, (base + 8).toString) val prefix = s"col-$base" // Values are offset by base. All Ids are assigned in depth first order to // StructField's first and then intermediate nested fields are added after. val nestedColumnMappingValues = FieldMetadata.builder() .putLong(s"$prefix.element", base + 2) .putLong(s"$prefix.element.element", base + 3) .putLong(s"$prefix.element.element.key", base + 4) .putLong(s"$prefix.element.element.key.element", base + 5) .putLong(s"$prefix.element.element.value", base + 6) .putLong(s"$prefix.element.element.value.key", base + 7) .putLong(s"$prefix.element.element.value.value", base + 8).build() val expectedMetadata = FieldMetadata.builder().putFieldMetadata( COLUMN_MAPPING_NESTED_IDS_KEY, nestedColumnMappingValues) .putString("delta.columnMapping.physicalName", prefix) .putLong("delta.columnMapping.id", base) .putBoolean("k2", true).build() val firstColumnMetadata = getter(metadata).getMetadata assertThat(firstColumnMetadata.getMetadata( COLUMN_MAPPING_NESTED_IDS_KEY).getEntries).containsExactlyInAnyOrderEntriesOf( nestedColumnMappingValues.getEntries) assertThat(firstColumnMetadata.getEntries).containsExactlyInAnyOrderEntriesOf( expectedMetadata.getEntries) // TODO: It would be nice to have visitor pattern on schema so we can assert // all metadata for nested fields // are empty but this at least provides a sanity check. assert(getter(metadata).getDataType.asInstanceOf[ ArrayType].getElementField.getMetadata == FieldMetadata.empty()) // Requesting the same operation on the same schema shouldn't change anything // as the schema already has the necessary column mapping info. This includes both // IDs and maxFieldId. assertNoOpOnUpdateColumnMappingMetadataRequest( metadata.getSchema, /* enableIcebergCompatV2= */ true, isNewTable) } } } runWithIcebergCompatComboForNewAndExistingTables( "assign id and physical name to schema with nested schema") { (isNewTable, enableIcebergCompatV2, enableIcebergWriterCompatV1) => val schema: StructType = cmTestSchema() var inputMetadata = testMetadata(schema).withColumnMappingEnabled("id") if (enableIcebergCompatV2) { inputMetadata = inputMetadata.withIcebergCompatV2Enabled } if (enableIcebergWriterCompatV1) { inputMetadata = inputMetadata.withIcebergWriterCompatV1Enabled } val metadata = updateColumnMappingMetadataIfNeeded(inputMetadata, isNewTable) .orElseGet(() => fail("Metadata should not be empty")) verifyCMTestSchemaHasValidColumnMappingInfo( metadata, isNewTable, enableIcebergCompatV2, enableIcebergWriterCompatV1) // Requesting the same operation on the same schema shouldn't change anything // as the schema already has the necessary column mapping info assertNoOpOnUpdateColumnMappingMetadataRequest( metadata.getSchema, enableIcebergCompatV2, isNewTable) } Seq(true, false).foreach { icebergCompatV2Enabled => test(s"assign id and physical name to only to the fields that don't have " + s"with icebergCompatV2=$icebergCompatV2Enabled") { val schema: StructType = new StructType().add("a", StringType.STRING) val inputMetadata = testMetadata(schema).withColumnMappingEnabled("id") val updatedMetadata = updateColumnMappingMetadataIfNeeded( if (icebergCompatV2Enabled) inputMetadata.withIcebergCompatV2Enabled else inputMetadata, true).orElseGet(() => fail("Metadata should not be empty")) assertColumnMapping(updatedMetadata.getSchema.get("a"), 1L) // Now add few more fields to the same schema val updateSchema = updatedMetadata.getSchema .add("b", StringType.STRING) .add("c", new ArrayType(IntegerType.INTEGER, false)) .add("d", new MapType(IntegerType.INTEGER, StringType.STRING, false)) .add("e", new StructType().add("h", IntegerType.INTEGER)) val inputMetadata2 = testMetadata(updateSchema).withColumnMappingEnabled("id") val updatedMetadata2 = updateColumnMappingMetadataIfNeeded( if (icebergCompatV2Enabled) inputMetadata2.withIcebergCompatV2Enabled else inputMetadata2, false).orElseGet(() => fail("Metadata should not be empty")) var fieldId = 0L def nextFieldId: Long = { fieldId += 1 fieldId } assertColumnMapping(updatedMetadata2.getSchema.get("a"), nextFieldId) // newly added columns will have the physical names same as logical names assertColumnMapping(updatedMetadata2.getSchema.get("b"), nextFieldId, "b") assertColumnMapping(updatedMetadata2.getSchema.get("c"), nextFieldId, "c") assertColumnMapping(updatedMetadata2.getSchema.get("d"), nextFieldId, "d") assertColumnMapping(updatedMetadata2.getSchema.get("e"), nextFieldId, "e") assertColumnMapping( updatedMetadata2.getSchema.get("e") .getDataType.asInstanceOf[StructType].get("h"), nextFieldId, "h") if (icebergCompatV2Enabled) { // verify nested ids assertThat(updatedMetadata2.getSchema.get("c").getMetadata.getEntries .get(COLUMN_MAPPING_NESTED_IDS_KEY).asInstanceOf[FieldMetadata].getEntries) .hasSize(1) .anySatisfy((k: AnyRef, v: AnyRef) => { assertThat(k).asString.startsWith("c.") assertThat(k).asString.endsWith(".element") assertThat(v).isEqualTo(nextFieldId) }) assertThat(updatedMetadata2.getSchema.get("d").getMetadata.getEntries .get(COLUMN_MAPPING_NESTED_IDS_KEY).asInstanceOf[FieldMetadata].getEntries) .hasSize(2) .anySatisfy((k: AnyRef, v: AnyRef) => { assertThat(k).asString.startsWith("d.") assertThat(k).asString.endsWith(".key") assertThat(v).isEqualTo(nextFieldId) }) .anySatisfy((k: AnyRef, v: AnyRef) => { assertThat(k).asString.startsWith("d.") assertThat(k).asString.endsWith(".value") assertThat(v).isEqualTo(nextFieldId) }) } else { assertThat(updatedMetadata2.getSchema.get("c").getMetadata.getEntries) .doesNotContainKey(COLUMN_MAPPING_NESTED_IDS_KEY) assertThat(updatedMetadata2.getSchema.get("d").getMetadata.getEntries) .doesNotContainKey(COLUMN_MAPPING_NESTED_IDS_KEY) } assertNoOpOnUpdateColumnMappingMetadataRequest( updatedMetadata2.getSchema, icebergCompatV2Enabled, isNewTable = false) } } test("both id and physical name must be provided if one is provided") { val schemaWithoutPhysicalName = new StructType() .add( new StructField( "col1", StringType.STRING, true, FieldMetadata.builder() .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, 0) .build())) val schemaWithoutId = new StructType() .add( new StructField( "col1", StringType.STRING, true, FieldMetadata.builder() .putString(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "physical-name-col1") .build())) Seq(schemaWithoutId, schemaWithoutPhysicalName).foreach { schema => val e = intercept[IllegalArgumentException] { updateColumnMappingMetadataIfNeeded(testMetadata(schema).withColumnMappingEnabled(), true) } assert(e.getMessage.contains( "Both columnId and physicalName must be present if one is present")) } } /** * A struct type with all necessary CM info won't cause metadata change by * [[updateColumnMappingMetadataIfNeeded]] */ def assertNoOpOnUpdateColumnMappingMetadataRequest( schemaWithCMInfo: StructType, enableIcebergCompatV2: Boolean, isNewTable: Boolean): Unit = { var metadata = testMetadata(schemaWithCMInfo).withColumnMappingEnabled("id") if (enableIcebergCompatV2) { metadata = metadata.withIcebergCompatV2Enabled } if (!metadata.getConfiguration.containsKey(COLUMN_MAPPING_MAX_COLUMN_ID_KEY)) { // A hack, if the metadata doesn't have max column ID in it, // then new metadata is always returned. metadata = metadata.withMergedConfiguration(Map(COLUMN_MAPPING_MAX_COLUMN_ID_KEY -> "100").asJava) } assertThat(updateColumnMappingMetadataIfNeeded(metadata, isNewTable)).isEmpty } def runWithIcebergCompatComboForNewAndExistingTables(testName: String)(f: ( Boolean, Boolean, Boolean) => Unit): Unit = { for { isNewTable <- Seq(true, false) enableIcebergCompatV2 <- Seq(true, false) } { // We only test icebergWriterCompatV1 when icebergCompatV2 is enabled val icebergWriterCompatV1Modes = if (enableIcebergCompatV2) { Seq(true, false) } else { Seq(false) } icebergWriterCompatV1Modes.foreach { enableIcebergWriterCompatV1 => test(s"$testName, enableIcebergCompatV2=$enableIcebergCompatV2, " + s"isNewTable=$isNewTable, enableIcebergWriterCompatV1=$enableIcebergWriterCompatV1") { f(isNewTable, enableIcebergCompatV2, enableIcebergWriterCompatV1) } } } } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/ColumnMappingSuiteBase.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util import scala.collection.JavaConverters._ import io.delta.kernel.internal.TableConfig import io.delta.kernel.internal.actions.{Metadata, Protocol} import io.delta.kernel.internal.tablefeatures.TableFeature import io.delta.kernel.internal.util.ColumnMapping.{COLUMN_MAPPING_ID_KEY, COLUMN_MAPPING_NESTED_IDS_KEY} import io.delta.kernel.test.ActionUtils import io.delta.kernel.types.{ArrayType, FieldMetadata, IntegerType, MapType, StringType, StructField, StructType} import org.assertj.core.api.Assertions.assertThat import org.assertj.core.util.Maps /** * Common utilities for column mapping and iceberg compat v2 related nested column mapping * functionality */ trait ColumnMappingSuiteBase extends ActionUtils { /* Asserts that the given field has the expected column mapping info */ def assertColumnMapping( field: StructField, expId: Long, isNewTable: Boolean, isIcebergWriterCompatV1: Boolean): Unit = { val logicalName = field.getName val expPhysicalName = if (isIcebergWriterCompatV1) { s"col-$expId" } else { if (isNewTable) { "UUID" } else { logicalName } } assertColumnMapping(field, expId, expPhysicalName) } /* Asserts that the given field has the expected column mapping info */ def assertColumnMapping( field: StructField, expId: Long, expPhysicalName: String = "UUID"): Unit = { assertThat(field.getMetadata.getEntries) .containsEntry(ColumnMapping.COLUMN_MAPPING_ID_KEY, expId.asInstanceOf[AnyRef]) .hasEntrySatisfying( ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, (k: AnyRef) => { if (expPhysicalName == "UUID") { assertThat(k).asString.startsWith("col-") } else { assertThat(k).asString.isEqualTo(expPhysicalName) } }) } implicit class MetadataImplicits(metadata: Metadata) { def withIcebergCompatV2Enabled: Metadata = { metadata.withMergedConfiguration( Maps.newHashMap(TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey, "true")) } def withIcebergCompatV3Enabled: Metadata = { metadata.withMergedConfiguration( Maps.newHashMap(TableConfig.ICEBERG_COMPAT_V3_ENABLED.getKey, "true")) } def withColumnMappingEnabled(mode: String = "name"): Metadata = { metadata.withMergedConfiguration( Maps.newHashMap(TableConfig.COLUMN_MAPPING_MODE.getKey, mode)) } def withIcebergCompatV2AndCMEnabled(columnMappingMode: String = "name"): Metadata = { metadata.withIcebergCompatV2Enabled.withColumnMappingEnabled(columnMappingMode) } def withIcebergCompatV3AndCMEnabled(columnMappingMode: String = "name"): Metadata = { metadata.withIcebergCompatV3Enabled.withColumnMappingEnabled(columnMappingMode) } def withIcebergWriterCompatV1Enabled: Metadata = { metadata.withMergedConfiguration( Maps.newHashMap(TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.getKey, "true")) } } def createMetadataWithFieldId(fieldId: Int): FieldMetadata = { FieldMetadata.builder.putLong(COLUMN_MAPPING_ID_KEY, fieldId).build() } /** Test schema containing various different types that test the nested column mapping info */ def cmTestSchema(): StructType = { new StructType() .add("a", StringType.STRING) .add( "b", new MapType( IntegerType.INTEGER, new StructType() .add("d", IntegerType.INTEGER) .add("e", IntegerType.INTEGER) .add( "f", new ArrayType( new StructType() .add("g", IntegerType.INTEGER) .add("h", IntegerType.INTEGER), false), false), false)) .add("c", IntegerType.INTEGER) } /** * Verify the schema returned by [[cmTestSchema()]] has correct column mapping (including nested) * info assigned */ def verifyCMTestSchemaHasValidColumnMappingInfo( metadata: Metadata, isNewTable: Boolean = true, enableIcebergCompatV2: Boolean = true, enableIcebergWriterCompatV1: Boolean = false, initialFieldId: Long = 0L): Unit = { var fieldId: Long = initialFieldId def nextFieldId: Long = { fieldId += 1 fieldId } assertColumnMapping( metadata.getSchema.get("a"), nextFieldId, isNewTable, enableIcebergWriterCompatV1) assertColumnMapping( metadata.getSchema.get("b"), nextFieldId, isNewTable, enableIcebergWriterCompatV1) val mapType = metadata.getSchema.get("b").getDataType.asInstanceOf[MapType] val innerStruct = mapType.getValueField.getDataType.asInstanceOf[StructType] assertColumnMapping(innerStruct.get("d"), nextFieldId, isNewTable, enableIcebergWriterCompatV1) assertColumnMapping(innerStruct.get("e"), nextFieldId, isNewTable, enableIcebergWriterCompatV1) assertColumnMapping(innerStruct.get("f"), nextFieldId, isNewTable, enableIcebergWriterCompatV1) val innerArray = innerStruct.get("f").getDataType.asInstanceOf[ArrayType] val structInArray = innerArray.getElementField.getDataType.asInstanceOf[StructType] assertColumnMapping( structInArray.get("g"), nextFieldId, isNewTable, enableIcebergWriterCompatV1) assertColumnMapping( structInArray.get("h"), nextFieldId, isNewTable, enableIcebergWriterCompatV1) assertColumnMapping( metadata.getSchema.get("c"), nextFieldId, isNewTable, enableIcebergWriterCompatV1) // verify nested ids if (enableIcebergCompatV2) { val colBPrefix = if (enableIcebergWriterCompatV1) { "col-2." } else if (isNewTable) { "col-" } else { "b." } assertThat(metadata.getSchema.get("b").getMetadata.getEntries .get(COLUMN_MAPPING_NESTED_IDS_KEY).asInstanceOf[FieldMetadata].getEntries) .hasSize(2) .anySatisfy((k: AnyRef, v: AnyRef) => { assertThat(k).asString.startsWith(colBPrefix) assertThat(k).asString.endsWith(".key") assert(k.asInstanceOf[String].count(_ == '.') == 1) assertThat(v).isEqualTo(nextFieldId) }) .anySatisfy((k: AnyRef, v: AnyRef) => { assertThat(k).asString.startsWith(colBPrefix) assertThat(k).asString.endsWith(".value") assert(k.asInstanceOf[String].count(_ == '.') == 1) assertThat(v).isEqualTo(nextFieldId) }) assertThat(mapType.getKeyField.getMetadata.getEntries) .doesNotContainKey(ColumnMapping.COLUMN_MAPPING_ID_KEY) .doesNotContainKey(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY) assertThat(mapType.getValueField.getMetadata.getEntries) .doesNotContainKey(ColumnMapping.COLUMN_MAPPING_ID_KEY) .doesNotContainKey(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY) // verify nested ids val colFPrefix = if (enableIcebergWriterCompatV1) { "col-5." } else if (isNewTable) { "col-" } else { "f." } assert( innerStruct.get("f").getMetadata.getEntries .get(COLUMN_MAPPING_NESTED_IDS_KEY) != null, s"${metadata.getSchema}") assertThat(innerStruct.get("f").getMetadata.getEntries .get(COLUMN_MAPPING_NESTED_IDS_KEY).asInstanceOf[FieldMetadata].getEntries) .hasSize(1) .anySatisfy((k: AnyRef, v: AnyRef) => { assertThat(k).asString.startsWith(colFPrefix) assertThat(k).asString.endsWith(".element") assert(k.asInstanceOf[String].count(_ == '.') == 1) assertThat(v).isEqualTo(nextFieldId) }) } assertThat(metadata.getConfiguration) .containsEntry(ColumnMapping.COLUMN_MAPPING_MAX_COLUMN_ID_KEY, fieldId.toString) } def testProtocol(tableFeatures: TableFeature*): Protocol = { val protocol = new Protocol(3, 7) protocol.withFeatures(tableFeatures.asJava).normalized() } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/DataFileStatisticsSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util import java.util.{Collections, Optional} import scala.collection.JavaConverters.mapAsJavaMapConverter import io.delta.kernel.exceptions.KernelException import io.delta.kernel.expressions.{Column, Literal} import io.delta.kernel.statistics.DataFileStatistics import io.delta.kernel.types._ import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.must.Matchers class DataFileStatisticsSuite extends AnyFunSuite with Matchers { val objectMapper = new ObjectMapper() def jsonToNode(json: String): JsonNode = { objectMapper.readTree(json) } def areJsonNodesEqual(json1: String, json2: String): Boolean = { val node1 = jsonToNode(json1) val node2 = jsonToNode(json2) node1 == node2 } test("DataFileStatistics serialization with all types") { val nestedStructType = new StructType() .add("aa", StringType.STRING) .add("ac", new StructType().add("aca", IntegerType.INTEGER)) .add("nested_variant", VariantType.VARIANT) val schema = new StructType() .add("ByteType", ByteType.BYTE) .add("ShortType", ShortType.SHORT) .add("IntegerType", IntegerType.INTEGER) .add("LongType", LongType.LONG) .add("FloatType", FloatType.FLOAT) .add("DoubleType", DoubleType.DOUBLE) .add("DecimalType", new DecimalType(10, 2)) .add("StringType", StringType.STRING) .add("DateType", DateType.DATE) .add("TimestampType", TimestampType.TIMESTAMP) .add("TimestampNTZType", TimestampNTZType.TIMESTAMP_NTZ) .add("BinaryType", BinaryType.BINARY) .add("NestedStruct", nestedStructType) .add("VariantType", VariantType.VARIANT) val minValues = Map( new Column("ByteType") -> Literal.ofByte(1.toByte), new Column("ShortType") -> Literal.ofShort(1.toShort), new Column("IntegerType") -> Literal.ofInt(1), new Column("LongType") -> Literal.ofLong(1L), new Column("FloatType") -> Literal.ofFloat(0.1f), new Column("DoubleType") -> Literal.ofDouble(0.1), new Column("DecimalType") -> Literal.ofDecimal(new java.math.BigDecimal("123.45"), 10, 2), new Column("StringType") -> Literal.ofString("a"), new Column("DateType") -> Literal.ofDate(1), new Column("TimestampType") -> Literal.ofTimestamp(1L), new Column("TimestampNTZType") -> Literal.ofTimestampNtz(1L), new Column("BinaryType") -> Literal.ofBinary("a".getBytes), new Column(Array("NestedStruct", "aa")) -> Literal.ofString("a"), new Column(Array("NestedStruct", "ac", "aca")) -> Literal.ofInt(1), new Column("VariantType") -> Literal.ofString( "0S&u501fk+ze0(tB98CpzF6vU0rJl95HpNdvjbtatpi(cu0wW^cTu"), new Column(Array("NestedStruct", "nested_variant")) -> Literal.ofString( "0S&u501fk+ze0(tB98CpzF6vU0rJl95HpNdvjbtatpi(cu0wW^cTu")).asJava val maxValues = Map( new Column("ByteType") -> Literal.ofByte(10.toByte), new Column("ShortType") -> Literal.ofShort(10.toShort), new Column("IntegerType") -> Literal.ofInt(10), new Column("LongType") -> Literal.ofLong(10L), new Column("FloatType") -> Literal.ofFloat(10.1f), new Column("DoubleType") -> Literal.ofDouble(10.1), new Column("DecimalType") -> Literal.ofDecimal(new java.math.BigDecimal("456.78"), 10, 2), new Column("StringType") -> Literal.ofString("z"), new Column("DateType") -> Literal.ofDate(10), new Column("TimestampType") -> Literal.ofTimestamp(10L), new Column("TimestampNTZType") -> Literal.ofTimestampNtz(10L), new Column("BinaryType") -> Literal.ofBinary("z".getBytes), new Column(Array("NestedStruct", "aa")) -> Literal.ofString("z"), new Column(Array("NestedStruct", "ac", "aca")) -> Literal.ofInt(10), new Column("VariantType") -> Literal.ofString( "0S&u500&]LC42A9vqZe}wb#-i1}-a+cT!xdbWhT9cTx}7v<+K"), new Column(Array("NestedStruct", "nested_variant")) -> Literal.ofString( "0S&u500&]LC42A9vqZe}wb#-i1}-a+cT!xdbWhT9cTx}7v<+K")).asJava val nullCount = Map( new Column("ByteType") -> 1L, new Column("ShortType") -> 1L, new Column("IntegerType") -> 1L, new Column("LongType") -> 1L, new Column("FloatType") -> 1L, new Column("DoubleType") -> 1L, new Column("DecimalType") -> 1L, new Column("StringType") -> 1L, new Column("DateType") -> 1L, new Column("TimestampType") -> 1L, new Column("TimestampNTZType") -> 1L, new Column("BinaryType") -> 1L, new Column(Array("NestedStruct", "aa")) -> 1L, new Column(Array("NestedStruct", "ac", "aca")) -> 1L, new Column("VariantType") -> 1L, new Column(Array("NestedStruct", "nested_variant")) -> 1L) val tightBounds = false val stats = new DataFileStatistics( 100, minValues, maxValues, nullCount.map { case (k, v) => (k, java.lang.Long.valueOf(v)) }.asJava, Optional.of(tightBounds)) val expectedJson = """{ | "numRecords": 100, | "minValues": { | "ByteType": 1, | "ShortType": 1, | "IntegerType": 1, | "LongType": 1, | "FloatType": 0.1, | "DoubleType": 0.1, | "DecimalType": 123.45, | "StringType": "a", | "DateType": "1970-01-02", | "TimestampType": "1970-01-01T00:00:00.000Z", | "TimestampNTZType": "1970-01-01T00:00:00", | "BinaryType": "a", | "NestedStruct": { | "aa": "a", | "ac": { | "aca": 1 | }, | "nested_variant": "0S&u501fk+ze0(tB98CpzF6vU0rJl95HpNdvjbtatpi(cu0wW^cTu" | }, | "VariantType": "0S&u501fk+ze0(tB98CpzF6vU0rJl95HpNdvjbtatpi(cu0wW^cTu" | }, | "maxValues": { | "ByteType": 10, | "ShortType": 10, | "IntegerType": 10, | "LongType": 10, | "FloatType": 10.1, | "DoubleType": 10.1, | "DecimalType": 456.78, | "StringType": "z", | "DateType": "1970-01-11", | "TimestampType": "1970-01-01T00:00:00.000Z", | "TimestampNTZType": "1970-01-01T00:00:00", | "BinaryType": "z", | "NestedStruct": { | "aa": "z", | "ac": { | "aca": 10 | }, | "nested_variant": "0S&u500&]LC42A9vqZe}wb#-i1}-a+cT!xdbWhT9cTx}7v<+K" | }, | "VariantType": "0S&u500&]LC42A9vqZe}wb#-i1}-a+cT!xdbWhT9cTx}7v<+K" | }, | "nullCount": { | "ByteType": 1, | "ShortType": 1, | "IntegerType": 1, | "LongType": 1, | "FloatType": 1, | "DoubleType": 1, | "DecimalType": 1, | "StringType": 1, | "DateType": 1, | "TimestampType": 1, | "TimestampNTZType": 1, | "BinaryType": 1, | "NestedStruct": { | "aa": 1, | "ac": { | "aca": 1 | }, | "nested_variant": 1 | }, | "VariantType": 1 |}, |"tightBounds": false |}""".stripMargin val json = stats.serializeAsJson(schema) assert(areJsonNodesEqual(json, expectedJson)) } test("serializeAsJson handles NaN and Infinity correctly") { val schema = new StructType() .add("FloatType", FloatType.FLOAT) .add("DoubleType", DoubleType.DOUBLE) val minValues = Map( new Column("FloatType") -> Literal.ofFloat(Float.NaN), new Column("DoubleType") -> Literal.ofDouble(Double.NegativeInfinity)).asJava val maxValues = Map( new Column("FloatType") -> Literal.ofFloat(Float.PositiveInfinity), new Column("DoubleType") -> Literal.ofDouble(Double.NaN)).asJava val stats = new DataFileStatistics( 1L, minValues, maxValues, Collections.emptyMap[Column, java.lang.Long](), Optional.empty()) val json = stats.serializeAsJson(schema) val expectedJson = """{ | "numRecords": 1, | "minValues": { | "FloatType": "NaN", | "DoubleType": "-Infinity" | }, | "maxValues": { | "FloatType": "Infinity", | "DoubleType": "NaN" | }, | "nullCount": {} |}""".stripMargin assert(areJsonNodesEqual(json, expectedJson)) } test("serializeAsJson handles null values and null literals correctly") { val schema = new StructType() .add("col1", IntegerType.INTEGER) .add("col2", StringType.STRING) .add("col3", DoubleType.DOUBLE) .add( "nested", new StructType() .add("nestedCol1", IntegerType.INTEGER) .add("nestedCol2", StringType.STRING)) val minValues = Map[Column, Literal]( new Column("col1") -> Literal.ofInt(1), new Column("col2") -> null, new Column("col3") -> Literal.ofNull(DoubleType.DOUBLE), new Column(Array("nested", "nestedCol1")) -> Literal.ofInt(5), new Column(Array("nested", "nestedCol2")) -> Literal.ofNull(StringType.STRING)).asJava val maxValues = Map[Column, Literal]( new Column("col2") -> Literal.ofString("z"), new Column("col3") -> null, new Column(Array("nested", "nestedCol1")) -> null, new Column(Array("nested", "nestedCol2")) -> Literal.ofString("zzz")).asJava val nullCount = Map( new Column("col1") -> 5L, new Column("col2") -> 0L, new Column(Array("nested", "nestedCol1")) -> 2L).map { case (k, v) => (k, java.lang.Long.valueOf(v)) }.asJava val tightBounds = true val stats = new DataFileStatistics( 100, minValues, maxValues, nullCount, Optional.of(tightBounds)) val expectedJson = """{ | "numRecords": 100, | "minValues": { | "col1": 1, | "col3": null, | "nested": { | "nestedCol1": 5, | "nestedCol2": null | } | }, | "maxValues": { | "col2": "z", | "nested": { | "nestedCol2": "zzz" | } | }, | "nullCount": { | "col1": 5, | "col2": 0, | "nested": { | "nestedCol1": 2 | } | }, | "tightBounds": true |}""".stripMargin val json = stats.serializeAsJson(schema) assert(areJsonNodesEqual(json, expectedJson)) } test("serializeAsJson returns empty nested objects when nested map is empty") { val nestedSchema = new StructType() .add("field1", IntegerType.INTEGER) .add("field2", StringType.STRING) val schema = new StructType().add("nested", nestedSchema) val stats = new DataFileStatistics( 50L, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Optional.empty()) val expectedJson = """{ | "numRecords": 50, | "minValues": {"nested": {}}, | "maxValues": {"nested": {}}, | "nullCount": {"nested": {}} |}""".stripMargin val json = stats.serializeAsJson(schema) assert(areJsonNodesEqual(json, expectedJson)) } test("serializeAsJson handles partially populated nested values") { val nestedSchema = new StructType() .add("field1", IntegerType.INTEGER) .add("field2", StringType.STRING) val schema = new StructType().add("nested", nestedSchema) val minValues = Map( new Column(Array("nested", "field1")) -> Literal.ofInt(10)).asJava val stats = new DataFileStatistics( 75L, minValues, Collections.emptyMap(), Collections.emptyMap(), Optional.empty()) val expectedJson = """{ | "numRecords": 75, | "minValues": { | "nested": { | "field1": 10 | } | }, | "maxValues": {"nested":{}}, | "nullCount": {"nested":{}} |}""".stripMargin val json = stats.serializeAsJson(schema) assert(areJsonNodesEqual(json, expectedJson)) } test("deserialize invalid JSON structure throws KernelException") { val malformedJson = """{ | "numRecords": "invalid_value", |}""".stripMargin val exception = intercept[KernelException] { DataFileStatistics.deserializeFromJson(malformedJson, null) } assert(exception.getMessage.contains("Failed to parse JSON string")) } test("serialization and deserialization of stats") { val numRecords = 123L val dataSchema = new StructType().add("a", IntegerType.INTEGER) val stats = new DataFileStatistics( numRecords, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Optional.empty()) val json = stats.serializeAsJson(dataSchema) val deserialized = DataFileStatistics.deserializeFromJson(json, null) assert(deserialized.get().getNumRecords == stats.getNumRecords) } test("test equals and hashCode work correctly for DataFileStatistics") { // Setup common test data val col1 = new Column("col1") val nestedField = new Column(Array("nested", "field")) // Create two identical stats objects val commonMaps = () => { val min = Map(col1 -> Literal.ofInt(10), nestedField -> Literal.ofString("value")).asJava val max = Map(col1 -> Literal.ofInt(100), nestedField -> Literal.ofString("zzzz")).asJava val nulls = Map(col1 -> java.lang.Long.valueOf(5L), nestedField -> java.lang.Long.valueOf(2L)).asJava (min, max, nulls) } val (min1, max1, nulls1) = commonMaps() val (min2, max2, nulls2) = commonMaps() val stats1 = new DataFileStatistics(100L, min1, max1, nulls1, Optional.empty()) val stats2 = new DataFileStatistics(100L, min2, max2, nulls2, Optional.empty()) // Stats with different value val differentMin = Map(col1 -> Literal.ofInt(20), nestedField -> Literal.ofString("value")).asJava val stats3 = new DataFileStatistics(100L, differentMin, max1, nulls1, Optional.empty()) // Stats with different structure val differentCol = new Column("col2") val structureMaps = Map(col1 -> Literal.ofInt(10), differentCol -> Literal.ofString("new")).asJava val stats4 = new DataFileStatistics(100L, structureMaps, structureMaps, nulls1, Optional.empty()) // Empty stats val emptyStats1 = new DataFileStatistics( 50L, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Optional.empty()) val emptyStats2 = new DataFileStatistics( 50L, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Optional.empty()) val emptyStats3 = new DataFileStatistics( 60L, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Optional.empty()) // Equality tests assert( stats1 == stats2 && stats1.hashCode() == stats2.hashCode(), "Identical stats should be equal with same hash") assert(stats1 != stats3, "Stats with different values should not be equal") assert(stats1 != stats4, "Stats with different structure should not be equal") assert(stats1 != null && stats1 != "string", "Stats should not equal null or different types") // Empty stats tests assert( emptyStats1 == emptyStats2 && emptyStats1.hashCode() == emptyStats2.hashCode(), "Empty stats with same records should be equal with same hash") assert(emptyStats1 != emptyStats3, "Empty stats with different records should not be equal") } test("serializeAsJson throws exception when literal type doesn't match schema data type") { val schema = new StructType() .add("intCol", IntegerType.INTEGER) .add("doubleCol", DoubleType.DOUBLE) .add( "nested", new StructType() .add("stringCol", StringType.STRING)) val minValues = Map[Column, Literal]( new Column("intCol") -> Literal.ofString("not an int"), new Column("doubleCol") -> Literal.ofDouble(1.23), new Column(Array("nested", "stringCol")) -> Literal.ofInt(42)).asJava val stats = new DataFileStatistics( 100, minValues, Collections.emptyMap[Column, Literal](), Collections.emptyMap[Column, java.lang.Long](), Optional.empty()) val exception = intercept[KernelException] { stats.serializeAsJson(schema) } val expectedMessage = "Type mismatch for field 'intCol' when writing statistics" + ": expected integer, but found string" assert(exception.getMessage === expectedMessage) } test("deserializeFromJson handles all data types correctly") { val schema = new StructType() .add("ByteType", ByteType.BYTE) .add("ShortType", ShortType.SHORT) .add("IntegerType", IntegerType.INTEGER) .add("LongType", LongType.LONG) .add("FloatType", FloatType.FLOAT) .add("DoubleType", DoubleType.DOUBLE) .add("DecimalType", new DecimalType(10, 2)) .add("StringType", StringType.STRING) .add("DateType", DateType.DATE) .add("TimestampType", TimestampType.TIMESTAMP) .add("TimestampNTZType", TimestampNTZType.TIMESTAMP_NTZ) .add("BinaryType", BinaryType.BINARY) .add("BooleanType", BooleanType.BOOLEAN) .add("VariantType", VariantType.VARIANT) val json = """{ | "numRecords": 100, | "minValues": { | "ByteType": 1, | "ShortType": 1, | "IntegerType": 1, | "LongType": 1, | "FloatType": 0.1, | "DoubleType": 0.1, | "DecimalType": 123.45, | "StringType": "a", | "DateType": "1970-01-02", | "TimestampType": "1970-01-01T00:00:00.001Z", | "TimestampNTZType": "1970-01-01T00:00:00.001", | "BinaryType": "a", | "BooleanType": true, | "VariantType": "0S&u501fk+ze0(tB98CpzF6vU0rJl95HpNdvjbtatpi(cu0wW^cTu" | }, | "maxValues": { | "ByteType": 10, | "ShortType": 10, | "IntegerType": 10, | "LongType": 10, | "FloatType": 10.1, | "DoubleType": 10.1, | "DecimalType": 456.78, | "StringType": "z", | "DateType": "1970-01-11", | "TimestampType": "1970-01-01T00:00:00.010Z", | "TimestampNTZType": "1970-01-01T00:00:00.010", | "BinaryType": "z", | "BooleanType": false, | "VariantType": "0S&u500&]LC42A9vqZe}wb#-i1}-a+cT!xdbWhT9cTx}7v<+K" | }, | "nullCount": { | "ByteType": 1, | "StringType": 2, | "DecimalType": 0, | "BooleanType": 5 | }, | "tightBounds": true |}""".stripMargin val result = DataFileStatistics.deserializeFromJson(json, schema) assert(result.isPresent) val stats = result.get() assert(stats.getNumRecords == 100) val minValues = stats.getMinValues assert(minValues.get(new Column("ByteType")).getValue == 1.toByte) assert(minValues.get(new Column("IntegerType")).getValue == 1) assert(minValues.get(new Column("FloatType")).getValue == 0.1f) assert(minValues.get(new Column("StringType")).getValue == "a") assert(minValues.get(new Column("BooleanType")).getValue == true) assert(minValues.get(new Column("VariantType")).getValue == "0S&u501fk+ze0(tB98CpzF6vU0rJl95HpNdvjbtatpi(cu0wW^cTu") val maxValues = stats.getMaxValues assert(maxValues.get(new Column("LongType")).getValue == 10L) assert(maxValues.get(new Column("DoubleType")).getValue == 10.1) assert(maxValues.get(new Column("BooleanType")).getValue == false) assert(maxValues.get(new Column("VariantType")).getValue == "0S&u500&]LC42A9vqZe}wb#-i1}-a+cT!xdbWhT9cTx}7v<+K") val nullCount = stats.getNullCount assert(nullCount.get(new Column("ByteType")) == 1L) assert(nullCount.get(new Column("StringType")) == 2L) assert(nullCount.get(new Column("DecimalType")) == 0L) assert(stats.getTightBounds.isPresent && stats.getTightBounds.get) } test("deserializeFromJson handles nested structures correctly") { val schema = new StructType() .add("simple", StringType.STRING) .add( "nested", new StructType() .add("field1", IntegerType.INTEGER) .add( "deep", new StructType() .add("field2", StringType.STRING) .add( "deeper", new StructType() .add("field3", IntegerType.INTEGER)))) val json = """{ | "numRecords": 50, | "minValues": { | "simple": "value1", | "nested": { | "field1": 10, | "deep": { | "field2": "nested_value", | "deeper": { | "field3": 42 | } | } | } | }, | "maxValues": { | "simple": "value2", | "nested": { | "field1": 100, | "deep": { | "field2": "zzz_value" | } | } | }, | "nullCount": { | "simple": 0, | "nested": { | "field1": 5, | "deep": { | "field2": 2, | "deeper": { | "field3": 1 | } | } | } | }, | "tightBounds": true |}""".stripMargin val result = DataFileStatistics.deserializeFromJson(json, schema) assert(result.isPresent) val stats = result.get() assert(stats.getNumRecords == 50) // Test simple column val minValues = stats.getMinValues assert(minValues.get(new Column("simple")).getValue == "value1") // Test nested columns with different path depths assert(minValues.get(new Column(Array("nested", "field1"))).getValue == 10) assert(minValues.get(new Column(Array("nested", "deep", "field2"))).getValue == "nested_value") assert(minValues.get(new Column(Array("nested", "deep", "deeper", "field3"))).getValue == 42) // Test that max values work for nested too val maxValues = stats.getMaxValues assert(maxValues.get(new Column(Array("nested", "field1"))).getValue == 100) assert(maxValues.get(new Column(Array("nested", "deep", "field2"))).getValue == "zzz_value") // Test null counts for nested val nullCount = stats.getNullCount assert(nullCount.get(new Column(Array("nested", "field1"))) == 5L) assert(nullCount.get(new Column(Array("nested", "deep", "deeper", "field3"))) == 1L) // Test tight bounds for nested columns assert(stats.getTightBounds.isPresent && stats.getTightBounds.get) } test("round-trip serialization and deserialization consistency") { val nestedStructType = new StructType() .add("aa", StringType.STRING) .add("ac", new StructType().add("aca", IntegerType.INTEGER)) val schema = new StructType() .add("IntegerType", IntegerType.INTEGER) .add("StringType", StringType.STRING) .add("DoubleType", DoubleType.DOUBLE) .add("NestedStruct", nestedStructType) val minValues = Map( new Column("IntegerType") -> Literal.ofInt(1), new Column("StringType") -> Literal.ofString("a"), new Column("DoubleType") -> Literal.ofDouble(0.1), new Column(Array("NestedStruct", "aa")) -> Literal.ofString("nested_a"), new Column(Array("NestedStruct", "ac", "aca")) -> Literal.ofInt(5)).asJava val maxValues = Map( new Column("IntegerType") -> Literal.ofInt(100), new Column("StringType") -> Literal.ofString("z"), new Column("DoubleType") -> Literal.ofDouble(99.9), new Column(Array("NestedStruct", "aa")) -> Literal.ofString("nested_z"), new Column(Array("NestedStruct", "ac", "aca")) -> Literal.ofInt(50)).asJava val nullCount = Map( new Column("IntegerType") -> 2L, new Column("StringType") -> 0L, new Column(Array("NestedStruct", "aa")) -> 1L).map { case (k, v) => (k, java.lang.Long.valueOf(v)) }.asJava val tightBounds = false val originalStats = new DataFileStatistics( 123L, minValues, maxValues, nullCount, Optional.of(tightBounds)) // Serialize then deserialize val json = originalStats.serializeAsJson(schema) val deserializedOpt = DataFileStatistics.deserializeFromJson(json, schema) assert(deserializedOpt.isPresent) val deserializedStats = deserializedOpt.get() // Verify they are equal assert(deserializedStats.getNumRecords == originalStats.getNumRecords) assert(deserializedStats.getMinValues.size() == originalStats.getMinValues.size()) assert(deserializedStats.getMaxValues.size() == originalStats.getMaxValues.size()) assert(deserializedStats.getNullCount.size() == originalStats.getNullCount.size()) // Verify specific values match assert(deserializedStats.getMinValues.get(new Column("IntegerType")).getValue == 1) assert(deserializedStats.getMaxValues.get(new Column(Array( "NestedStruct", "ac", "aca"))).getValue == 50) assert(deserializedStats.getNullCount.get(new Column("StringType")) == 0L) assert(deserializedStats.getTightBounds == originalStats.getTightBounds) } test("deserializeFromJson handles NaN and Infinity correctly") { val schema = new StructType() .add("FloatType", FloatType.FLOAT) .add("DoubleType", DoubleType.DOUBLE) val json = """{ | "numRecords": 10, | "minValues": { | "FloatType": "NaN", | "DoubleType": "-Infinity" | }, | "maxValues": { | "FloatType": "Infinity", | "DoubleType": "NaN" | }, | "nullCount": { | "FloatType": 1, | "DoubleType": 2 | }, | "tightBounds": true |}""".stripMargin val result = DataFileStatistics.deserializeFromJson(json, schema) assert(result.isPresent) val stats = result.get() assert(stats.getNumRecords == 10) val minValues = stats.getMinValues val maxValues = stats.getMaxValues // Test NaN and Infinity values - Note: Float values will be stored as Float, not Double assert( java.lang.Float.isNaN(minValues.get(new Column("FloatType")).getValue.asInstanceOf[Float])) assert(minValues.get(new Column("DoubleType")).getValue == Double.NegativeInfinity) assert(maxValues.get(new Column("FloatType")).getValue == Float.PositiveInfinity) assert( java.lang.Double.isNaN(maxValues.get(new Column("DoubleType")).getValue.asInstanceOf[Double])) val nullCount = stats.getNullCount assert(nullCount.get(new Column("FloatType")) == 1L) assert(nullCount.get(new Column("DoubleType")) == 2L) assert(stats.getTightBounds.isPresent && stats.getTightBounds.get) } test("deserializeFromJson handles empty stats correctly") { val schema = new StructType() // Empty schema for empty stats val json = """{ | "numRecords": 42, | "minValues": {}, | "maxValues": {}, | "nullCount": {} |}""".stripMargin val result = DataFileStatistics.deserializeFromJson(json, schema) assert(result.isPresent) val stats = result.get() assert(stats.getNumRecords == 42) assert(stats.getMinValues.isEmpty) assert(stats.getMaxValues.isEmpty) assert(stats.getNullCount.isEmpty) assert(!stats.getTightBounds.isPresent) } test("deserializeFromJson handles partial nested objects correctly") { // Schema should include all possible fields that appear in the JSON val schema = new StructType() .add("simple", StringType.STRING) .add( "nested", new StructType() .add("field1", IntegerType.INTEGER) .add("field2", StringType.STRING)) .add("other", IntegerType.INTEGER) val json = """{ | "numRecords": 25, | "minValues": { | "simple": "value", | "nested": { | "field1": 10 | } | }, | "maxValues": { | "nested": { | "field2": "different_field" | }, | "other": 99 | }, | "nullCount": { | "simple": 1, | "nested": { | "field1": 0, | "field2": 5 | } | }, |"tightBounds": true |}""".stripMargin val result = DataFileStatistics.deserializeFromJson(json, schema) assert(result.isPresent) val stats = result.get() assert(stats.getNumRecords == 25) val minValues = stats.getMinValues val maxValues = stats.getMaxValues val nullCount = stats.getNullCount // minValues has simple + nested.field1 assert(minValues.get(new Column("simple")).getValue == "value") assert(minValues.get(new Column(Array("nested", "field1"))).getValue == 10) assert(minValues.get(new Column(Array("nested", "field2"))) == null) // not present in minValues // maxValues has nested.field2 + other (different structure) assert(maxValues.get(new Column("simple")) == null) // not present in maxValues assert(maxValues.get(new Column(Array("nested", "field2"))).getValue == "different_field") assert(maxValues.get(new Column("other")).getValue == 99) // nullCount has both fields under nested assert(nullCount.get(new Column("simple")) == 1L) assert(nullCount.get(new Column(Array("nested", "field1"))) == 0L) assert(nullCount.get(new Column(Array("nested", "field2"))) == 5L) // tightBounds has simple + nested.field2 + other assert(stats.getTightBounds.isPresent && stats.getTightBounds.get) } test("withoutTightBounds removes tight bounds from DataFileStatistics") { val schema = new StructType() .add("col1", IntegerType.INTEGER) .add("col2", StringType.STRING) .add( "nested", new StructType() .add("field1", IntegerType.INTEGER) .add("field2", StringType.STRING)) // stats with a mix of true and false tight bounds val minValues = Map( new Column("col1") -> Literal.ofInt(1), new Column("col2") -> Literal.ofString("a"), new Column(Array("nested", "field1")) -> Literal.ofInt(10), new Column(Array("nested", "field2")) -> Literal.ofString("nested_a")).asJava val maxValues = Map( new Column("col1") -> Literal.ofInt(100), new Column("col2") -> Literal.ofString("z"), new Column(Array("nested", "field1")) -> Literal.ofInt(200), new Column(Array("nested", "field2")) -> Literal.ofString("nested_z")).asJava val nullCount = Map( new Column("col1") -> 5L, new Column("col2") -> 0L, new Column(Array("nested", "field1")) -> 2L, new Column(Array("nested", "field2")) -> 3L).map { case (k, v) => (k, java.lang.Long.valueOf(v)) }.asJava val originalTightBounds = true val originalStats = new DataFileStatistics( 100L, minValues, maxValues, nullCount, Optional.of(originalTightBounds)) // Test that original stats has tight bounds assert(originalStats.getTightBounds.isPresent && originalStats.getTightBounds.get) // Apply withoutTightBounds val statsWithoutTightBounds = originalStats.withoutTightBounds() // Verify all other fields remain unchanged assert(statsWithoutTightBounds.getNumRecords == originalStats.getNumRecords) assert(statsWithoutTightBounds.getMinValues == originalStats.getMinValues) assert(statsWithoutTightBounds.getMaxValues == originalStats.getMaxValues) assert(statsWithoutTightBounds.getNullCount == originalStats.getNullCount) // Verify tight bounds is now false assert(statsWithoutTightBounds.getTightBounds.isPresent && !statsWithoutTightBounds.getTightBounds.get) // Verify serialization reflects the change val jsonAfter = statsWithoutTightBounds.serializeAsJson(schema) val expectedJsonWithFalseTightBounds = """{ | "numRecords": 100, | "minValues": { | "col1": 1, | "col2": "a", | "nested": { | "field1": 10, | "field2": "nested_a" | } | }, | "maxValues": { | "col1": 100, | "col2": "z", | "nested": { | "field1": 200, | "field2": "nested_z" | } | }, | "nullCount": { | "col1": 5, | "col2": 0, | "nested": { | "field1": 2, | "field2": 3 | } | }, | "tightBounds": false |}""".stripMargin assert(areJsonNodesEqual(jsonAfter, expectedJsonWithFalseTightBounds)) // Test edge case: stats with already false tight bounds val statsAlreadyFalse = new DataFileStatistics( 50L, Map(new Column("col1") -> Literal.ofInt(1)).asJava, Map(new Column("col1") -> Literal.ofInt(10)).asJava, Map(new Column("col1") -> java.lang.Long.valueOf(0L)).asJava, Optional.of(false)) val resultAlreadyFalse = statsAlreadyFalse.withoutTightBounds() assert(resultAlreadyFalse.getTightBounds.isPresent && !resultAlreadyFalse.getTightBounds.get) // Test edge case: stats with empty tight bounds val emptyTightBoundsStats = new DataFileStatistics( 25L, minValues, maxValues, nullCount, Optional.empty()) val resultFromEmpty = emptyTightBoundsStats.withoutTightBounds() assert(resultFromEmpty.getTightBounds.isPresent && !resultFromEmpty.getTightBounds.get) } test("deserializing invalid variant stats throws KernelException") { val schema = new StructType().add("VariantType", VariantType.VARIANT); val invalidVariantStats = """|{ | "numRecords": 100, | "minValues": { | "VariantType": 1234 | }, | "maxValues": { | "VariantType": 5678 | } |}""".stripMargin val exception = intercept[KernelException] { DataFileStatistics.deserializeFromJson(invalidVariantStats, schema) } assert(exception.getMessage.contains("Expected variant as string value")) } test("serializeAsJson throws exception when literal type for variant is not string") { val schema = new StructType() .add("variant", VariantType.VARIANT) val minValues = Map[Column, Literal]( new Column("variant") -> Literal.ofInt(1)).asJava val stats = new DataFileStatistics( 100, minValues, Collections.emptyMap[Column, Literal](), Collections.emptyMap[Column, java.lang.Long](), Optional.empty()) val exception = intercept[KernelException] { stats.serializeAsJson(schema) } val expectedMessage = "Type mismatch for field 'variant' when writing statistics" + ": expected string, but found integer" assert(exception.getMessage === expectedMessage) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/FileNamesSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util import scala.collection.JavaConverters._ import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.util.FileNames._ import io.delta.kernel.utils.FileStatus import org.scalatest.funsuite.AnyFunSuite class FileNamesSuite extends AnyFunSuite { private val checkpointV1 = "/a/123.checkpoint.parquet" private val checkpointMultiPart = "/a/123.checkpoint.0000000001.0000000087.parquet" private val checkpointV2Json = "/a/000000010.checkpoint.80a083e8-7026.json" private val checkpointV2Parquet = "/a/000000010.checkpoint.80a083e8-7026.parquet" private val commitNormal = "/a/0000000088.json" private val commitUUID = "/a/00000022.dc0f9f58-a1a0.json" ///////////////////////////// // Version extractor tests // ///////////////////////////// test("checkpointVersion") { assert(checkpointVersion(new Path(checkpointV1)) == 123) assert(checkpointVersion(new Path(checkpointMultiPart)) == 123) assert(checkpointVersion(new Path(checkpointV2Json)) == 10) assert(checkpointVersion(new Path(checkpointV2Parquet)) == 10) } test("deltaVersion") { assert(deltaVersion(new Path(commitNormal)) == 88) assert(deltaVersion(new Path(commitUUID)) == 22) } test("getFileVersion") { assert(getFileVersion(new Path(checkpointV1)) == 123) assert(getFileVersion(new Path(checkpointMultiPart)) == 123) assert(getFileVersion(new Path(checkpointV2Json)) == 10) assert(getFileVersion(new Path(checkpointV2Parquet)) == 10) assert(getFileVersion(new Path(commitNormal)) == 88) assert(getFileVersion(new Path(commitUUID)) == 22) } ///////////////////////////////////////// // File path and prefix builders tests // ///////////////////////////////////////// test("deltaFile") { assert(deltaFile(new Path("/a"), 1234) == "/a/00000000000000001234.json") } test("sidecarFile") { assert(sidecarFile(new Path("/a"), "7d17ac10.parquet") == "/a/_sidecars/7d17ac10.parquet") } test("listingPrefix") { assert(listingPrefix(new Path("/a"), 1234) == "/a/00000000000000001234.") } test("checkpointFileSingular") { assert( checkpointFileSingular(new Path("/a"), 1234).toString == "/a/00000000000000001234.checkpoint.parquet") } test("topLevelV2CheckpointFile") { assert( topLevelV2CheckpointFile(new Path("/a"), 1234, "7d17ac10", "json").toString == "/a/00000000000000001234.checkpoint.7d17ac10.json") assert( topLevelV2CheckpointFile(new Path("/a"), 1234, "7d17ac10", "parquet").toString == "/a/00000000000000001234.checkpoint.7d17ac10.parquet") } test("v2CheckpointSidecarFile") { assert( v2CheckpointSidecarFile(new Path("/a"), "7d17ac10").toString == "/a/_sidecars/7d17ac10.parquet") } test("checkpointFileWithParts") { assert(checkpointFileWithParts(new Path("/a"), 1, 1).asScala == Seq( new Path("/a/00000000000000000001.checkpoint.0000000001.0000000001.parquet"))) assert(checkpointFileWithParts(new Path("/a"), 1, 2).asScala == Seq( new Path("/a/00000000000000000001.checkpoint.0000000001.0000000002.parquet"), new Path("/a/00000000000000000001.checkpoint.0000000002.0000000002.parquet"))) assert(checkpointFileWithParts(new Path("/a"), 1, 5).asScala == Seq( new Path("/a/00000000000000000001.checkpoint.0000000001.0000000005.parquet"), new Path("/a/00000000000000000001.checkpoint.0000000002.0000000005.parquet"), new Path("/a/00000000000000000001.checkpoint.0000000003.0000000005.parquet"), new Path("/a/00000000000000000001.checkpoint.0000000004.0000000005.parquet"), new Path("/a/00000000000000000001.checkpoint.0000000005.0000000005.parquet"))) } test("logCompactionPath") { assert(logCompactionPath(new Path("/a"), 1, 3) == new Path("/a/00000000000000000001.00000000000000000003.compacted.json")) assert(logCompactionPath(new Path("/a/b"), 11, 300) == new Path("/a/b/00000000000000000011.00000000000000000300.compacted.json")) } /////////////////////////////////// // Is file checkers tests // /////////////////////////////////// test("is checkpoint file") { // ===== V1 checkpoint ===== // Positive cases assert(isCheckpointFile(checkpointV1)) assert(isCheckpointFile(new Path(checkpointV1).getName)) assert(isClassicCheckpointFile(checkpointV1)) assert(isClassicCheckpointFile(new Path(checkpointV1).getName)) // Negative cases assert(!isMultiPartCheckpointFile(checkpointV1)) assert(!isV2CheckpointFile(checkpointV1)) assert(!isCommitFile(checkpointV1)) // ===== Multipart checkpoint ===== // Positive cases assert(isCheckpointFile(checkpointMultiPart)) assert(isCheckpointFile(new Path(checkpointMultiPart).getName)) assert(isMultiPartCheckpointFile(checkpointMultiPart)) assert(isMultiPartCheckpointFile(new Path(checkpointMultiPart).getName)) // Negative cases assert(!isClassicCheckpointFile(checkpointMultiPart)) assert(!isV2CheckpointFile(checkpointMultiPart)) assert(!isCommitFile(checkpointMultiPart)) // ===== V2 checkpoint ===== // Positive cases assert(isCheckpointFile(checkpointV2Json)) assert(isCheckpointFile(new Path(checkpointV2Json).getName)) assert(isV2CheckpointFile(checkpointV2Json)) assert(isV2CheckpointFile(new Path(checkpointV2Json).getName)) assert(isCheckpointFile(checkpointV2Parquet)) assert(isCheckpointFile(new Path(checkpointV2Parquet).getName)) assert(isV2CheckpointFile(checkpointV2Parquet)) assert(isV2CheckpointFile(new Path(checkpointV2Parquet).getName)) // Negative cases assert(!isClassicCheckpointFile(checkpointV2Json)) assert(!isClassicCheckpointFile(checkpointV2Parquet)) assert(!isMultiPartCheckpointFile(checkpointV2Json)) assert(!isMultiPartCheckpointFile(checkpointV2Parquet)) assert(!isCommitFile(checkpointV2Json)) assert(!isCommitFile(checkpointV2Parquet)) // ===== Others ===== assert(!isCheckpointFile("/a/123.json")) assert(!isCommitFile("/a/123.checkpoint.3.json")) } test("is commit file") { assert(isCommitFile(commitNormal)) assert(isCommitFile(commitUUID)) } test("determineFileType correctly identifies delta log file types") { // Test commit file detection val commitFile = FileStatus.of("/path/00000000000000000001.json", 100, 1000) assert(FileNames.determineFileType(commitFile) === DeltaLogFileType.COMMIT) // Test checkpoint file detection val checkpointFile = FileStatus.of("/path/00000000000000000002.checkpoint.parquet", 100, 1000) assert(FileNames.determineFileType(checkpointFile) === DeltaLogFileType.CHECKPOINT) // Test checksum file detection val checksumFile = FileStatus.of("/path/00000000000000000003.crc", 100, 1000) assert(FileNames.determineFileType(checksumFile) === DeltaLogFileType.CHECKSUM) // Test exception for unknown file type val unknownFile = FileStatus.of("/path/unknown_file.txt", 100, 1000) val exception = intercept[IllegalStateException] { FileNames.determineFileType(unknownFile) } assert(exception.getMessage.contains("Unexpected file type")) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/IntervalParserUtilsSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util import io.delta.kernel.internal.util.DateTimeConstants._ import io.delta.kernel.internal.util.IntervalParserUtils.parseIntervalAsMicros import org.scalatest.funsuite.AnyFunSuite /** * Subset of tests from Apache Spark's `org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala` */ class IntervalParserUtilsSuite extends AnyFunSuite { test("string to interval: basic") { testSingleUnit("Week", 3, 21, 0) testSingleUnit("DAY", 3, 3, 0) testSingleUnit("HouR", 3, 0, 3 * MICROS_PER_HOUR) testSingleUnit("MiNuTe", 3, 0, 3 * MICROS_PER_MINUTE) testSingleUnit("Second", 3, 0, 3 * MICROS_PER_SECOND) testSingleUnit("MilliSecond", 3, 0, 3 * MICROS_PER_MILLIS) testSingleUnit("MicroSecond", 3, 0, 3) checkFromInvalidString(null, "cannot be null") Seq( "", "interval", "foo", "foo 1 day", "month 3", "year 3").foreach { input => checkFromInvalidString(input, "Error parsing") } } test("string to interval: interval with dangling parts should not results null") { checkFromInvalidString("+", "expect a number after '+' but hit EOL") checkFromInvalidString("-", "expect a number after '-' but hit EOL") checkFromInvalidString("+ 2", "expect a unit name after '2' but hit EOL") checkFromInvalidString("- 1", "expect a unit name after '1' but hit EOL") checkFromInvalidString("1", "expect a unit name after '1' but hit EOL") checkFromInvalidString("1.2", "expect a unit name after '1.2' but hit EOL") checkFromInvalidString("1 day 2", "expect a unit name after '2' but hit EOL") checkFromInvalidString("1 day 2.2", "expect a unit name after '2.2' but hit EOL") checkFromInvalidString("1 day -", "expect a number after '-' but hit EOL") checkFromInvalidString("-.", "expect a unit name after '-.' but hit EOL") } test("string to interval: multiple units") { Seq( "interval -1 day +3 Microseconds" -> micros(-1, 3), "interval - 1 day + 3 Microseconds" -> micros(-1, 3), " interval 123 weeks -1 day " + "23 hours -22 minutes 1 second -123 millisecond 567 microseconds " -> micros(860, 81480877567L)).foreach { case (input, expected) => checkFromString(input, expected) } } test("string to interval: special cases") { // Support any order of interval units checkFromString("1 microsecond 1 day", micros(1, 1)) // Allow duplicated units and summarize their values checkFromString("1 day 10 day", micros(11, 0)) // Only the seconds units can have the fractional part checkFromInvalidString("1.5 days", "'days' cannot have fractional part") checkFromInvalidString("1. hour", "'hour' cannot have fractional part") checkFromInvalidString("1 hourX", "invalid unit 'hourx'") checkFromInvalidString("~1 hour", "unrecognized number '~1'") checkFromInvalidString("1 Mour", "invalid unit 'mour'") checkFromInvalidString("1 aour", "invalid unit 'aour'") checkFromInvalidString("1a1 hour", "invalid value '1a1'") checkFromInvalidString("1.1a1 seconds", "invalid value '1.1a1'") checkFromInvalidString("2234567890 days", "integer overflow") checkFromInvalidString(". seconds", "invalid value '.'") } test("string to interval: whitespaces") { checkFromInvalidString(" ", "Error parsing ' ' to interval") checkFromInvalidString("\n", "Error parsing '\n' to interval") checkFromInvalidString("\t", "Error parsing '\t' to interval") checkFromString("1 \t day \n 2 \r hour", micros(1, 2 * MICROS_PER_HOUR)) checkFromInvalidString("interval1 \t day \n 2 \r hour", "invalid interval prefix interval1") checkFromString("interval\r1\tday", micros(1, 0)) // scalastyle:off nonascii checkFromInvalidString("中国 interval 1 day", "unrecognized number '中国'") checkFromInvalidString("interval浙江 1 day", "invalid interval prefix interval浙江") checkFromInvalidString("interval 1杭州 day", "invalid value '1杭州'") checkFromInvalidString("interval 1 滨江day", "invalid unit '滨江day'") checkFromInvalidString("interval 1 day长河", "invalid unit 'day长河'") checkFromInvalidString("interval 1 day 网商路", "unrecognized number '网商路'") // scalastyle:on nonascii } test("string to interval: seconds with fractional part") { checkFromString("0.1 seconds", micros(0, 100000)) checkFromString("1. seconds", micros(0, 1000000)) checkFromString("123.001 seconds", micros(0, 123001000)) checkFromString("1.001001 seconds", micros(0, 1001001)) checkFromString("1 minute 1.001001 seconds", micros(0, 61001001)) checkFromString("-1.5 seconds", micros(0, -1500000)) // truncate nanoseconds to microseconds checkFromString("0.999999999 seconds", micros(0, 999999)) checkFromString(".999999999 seconds", micros(0, 999999)) checkFromInvalidString("0.123456789123 seconds", "'0.123456789123' is out of range") } private def testSingleUnit(unit: String, number: Int, days: Int, microseconds: Long): Unit = { for (prefix <- Seq("interval ", "")) { val input1 = prefix + number + " " + unit val input2 = prefix + number + " " + unit + "s" val result = micros(days, microseconds) checkFromString(input1, result) checkFromString(input2, result) } } private def checkFromString(input: String, expected: Long): Unit = { assert(parseIntervalAsMicros(input) === expected) } private def checkFromInvalidString(input: String, errorMsg: String): Unit = { failFuncWithInvalidInput(input, errorMsg, s => parseIntervalAsMicros(s)) } private def failFuncWithInvalidInput( input: String, errorMsg: String, converter: String => Long): Unit = { withClue(s"Expected to throw an exception for the invalid input: $input") { val e = intercept[IllegalArgumentException](converter(input)) assert(e.getMessage.contains(errorMsg)) } } private def micros(days: Long, microseconds: Long): Long = { days * MICROS_PER_DAY + microseconds } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/JsonUtilsSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util import scala.collection.JavaConverters._ import io.delta.kernel.exceptions.KernelException import org.scalatest.funsuite.AnyFunSuite class JsonUtilsSuite extends AnyFunSuite { test("Parse Map[String, String] JSON - positive case") { val expMap = Map("key1" -> "string_value", "key2Int" -> "2", "key3ComplexStr" -> "\"hello\"") val input = """{"key1": "string_value", "key2Int": "2", "key3ComplexStr": "\"hello\""}""" assert(JsonUtils.parseJSONKeyValueMap(input) === expMap.asJava) assert(JsonUtils.parseJSONKeyValueMap("").isEmpty) assert(JsonUtils.parseJSONKeyValueMap(null).isEmpty) } test("Parse Map[String, String] JSON - negative case") { val e = intercept[KernelException] { JsonUtils.parseJSONKeyValueMap("""{"key1": "string_value", asdf"}""") } assert(e.getMessage.contains("Failed to parse JSON string:")) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/PartitionUtilsSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util import java.util import scala.collection.JavaConverters._ import io.delta.kernel.expressions._ import io.delta.kernel.expressions.Literal._ import io.delta.kernel.internal.util.PartitionUtils._ import io.delta.kernel.types._ import org.scalatest.funsuite.AnyFunSuite class PartitionUtilsSuite extends AnyFunSuite { private val utf8Lcase = CollationIdentifier.fromString("SPARK.UTF8_LCASE") private val unicode = CollationIdentifier.fromString("ICU.UNICODE") // Table schema // Data columns: data1: int, data2: string, date3: struct(data31: boolean, data32: long) // Partition columns: part1: int, part2: date, part3: string val tableSchema = new StructType() .add("data1", IntegerType.INTEGER) .add("data2", StringType.STRING) .add( "data3", new StructType() .add("data31", BooleanType.BOOLEAN) .add("data32", LongType.LONG)) .add("part1", IntegerType.INTEGER) .add("part2", DateType.DATE) .add("part3", StringType.STRING) private val partitionColsMetadata = new util.HashMap[String, StructField]() { { put("part1", tableSchema.get("part1")) put("part2", tableSchema.get("part2")) put("part3", tableSchema.get("part3")) } } private val partitionCols: java.util.Set[String] = partitionColsMetadata.keySet() // Test cases for verifying partition of predicate into data and partition predicates // Map entry format (predicate -> (partition predicate, data predicate) val partitionTestCases = Map[Predicate, (String, String)]( // single predicate on a data column predicate("=", col("data1"), ofInt(12)) -> ("ALWAYS_TRUE()", "(column(`data1`) = 12)"), // single predicate with default collation on a data column predicate("=", col("data2"), ofString("12"), CollationIdentifier.SPARK_UTF8_BINARY) -> ("ALWAYS_TRUE()", "(column(`data2`) = 12 COLLATE SPARK.UTF8_BINARY)"), // single predicate with non-default collation on a data column predicate("=", col("data2"), ofString("12"), utf8Lcase) -> ("ALWAYS_TRUE()", "(column(`data2`) = 12 COLLATE SPARK.UTF8_LCASE)"), predicate("=", col("data2"), ofString("12"), unicode) -> ("ALWAYS_TRUE()", "(column(`data2`) = 12 COLLATE ICU.UNICODE)"), // multiple predicates on data columns joined with AND predicate( "AND", predicate("=", col("data1"), ofInt(12)), predicate(">=", col("data2"), ofString("sss"))) -> ("ALWAYS_TRUE()", "((column(`data1`) = 12) AND (column(`data2`) >= sss))"), // multiple predicates with collation on data columns joined with AND predicate( "AND", predicate("=", col("data2"), ofString("12")), predicate(">=", col("data2"), ofString("sss"), utf8Lcase)) -> ( "ALWAYS_TRUE()", "((column(`data2`) = 12) AND (column(`data2`) >= sss COLLATE SPARK.UTF8_LCASE))"), // multiple predicates with collation on data columns joined with AND predicate( "AND", predicate("=", col("data2"), ofString("12"), utf8Lcase), predicate(">=", col("data2"), ofString("sss"), unicode)) -> ( "ALWAYS_TRUE()", """((column(`data2`) = 12 COLLATE SPARK.UTF8_LCASE) AND |(column(`data2`) >= sss COLLATE ICU.UNICODE))""".stripMargin.replaceAll("\n", " ")), // multiple predicates on data columns joined with OR predicate( "OR", predicate("<=", col("data2"), ofString("sss")), predicate("=", col("data3", "data31"), ofBoolean(true))) -> ("ALWAYS_TRUE()", "((column(`data2`) <= sss) OR (column(`data3`.`data31`) = true))"), predicate( "OR", predicate("<=", col("data2"), ofString("sss"), utf8Lcase), predicate("=", col("data3", "data31"), ofBoolean(true))) -> ( "ALWAYS_TRUE()", "((column(`data2`) <= sss COLLATE SPARK.UTF8_LCASE) OR (column(`data3`.`data31`) = true))"), // single predicate on a partition column predicate("=", col("part1"), ofInt(12)) -> ("(column(`part1`) = 12)", "ALWAYS_TRUE()"), // single predicate with default collation on partition column predicate("=", col("part3"), ofString("12"), CollationIdentifier.SPARK_UTF8_BINARY) -> ("(column(`part3`) = 12 COLLATE SPARK.UTF8_BINARY)", "ALWAYS_TRUE()"), // single predicate with non-default collation on partition column predicate("=", col("part3"), ofString("12"), utf8Lcase) -> ("(column(`part3`) = 12 COLLATE SPARK.UTF8_LCASE)", "ALWAYS_TRUE()"), predicate("=", col("part3"), ofString("12"), unicode) -> ("(column(`part3`) = 12 COLLATE ICU.UNICODE)", "ALWAYS_TRUE()"), // multiple predicates on partition columns joined with AND predicate( "AND", predicate("=", col("part1"), ofInt(12)), predicate(">=", col("part3"), ofString("sss"))) -> ("((column(`part1`) = 12) AND (column(`part3`) >= sss))", "ALWAYS_TRUE()"), // multiple predicates with collation on partition columns joined with AND predicate( "AND", predicate("=", col("part3"), ofString("sss"), utf8Lcase), predicate(">=", col("part3"), ofString("sss"), CollationIdentifier.SPARK_UTF8_BINARY)) -> ( """((column(`part3`) = sss COLLATE SPARK.UTF8_LCASE) AND (column(`part3`) |>= sss COLLATE SPARK.UTF8_BINARY))""".stripMargin.replaceAll("\n", " "), "ALWAYS_TRUE()"), // multiple predicates on partition columns joined with OR predicate( "OR", predicate("<=", col("part3"), ofString("sss")), predicate("=", col("part1"), ofInt(2781))) -> ("((column(`part3`) <= sss) OR (column(`part1`) = 2781))", "ALWAYS_TRUE()"), // predicates (each on data and partition column) joined with AND predicate( "AND", predicate("=", col("data1"), ofInt(12)), predicate(">=", col("part3"), ofString("sss"))) -> ("(column(`part3`) >= sss)", "(column(`data1`) = 12)"), // predicates with collation (each on data and partition column) joined with AND predicate( "AND", predicate("=", col("data2"), ofString("12"), utf8Lcase), predicate(">=", col("part3"), ofString("sss"), unicode)) -> ( "(column(`part3`) >= sss COLLATE ICU.UNICODE)", "(column(`data2`) = 12 COLLATE SPARK.UTF8_LCASE)"), // predicates (each on data and partition column) joined with OR predicate( "OR", predicate("=", col("data1"), ofInt(12)), predicate(">=", col("part3"), ofString("sss"))) -> ("ALWAYS_TRUE()", "((column(`data1`) = 12) OR (column(`part3`) >= sss))"), // predicates with collation (each on data and partition column) joined with OR predicate( "OR", predicate("=", col("data2"), ofString("12"), unicode), predicate(">=", col("part3"), ofString("sss"), unicode)) -> ( "ALWAYS_TRUE()", """((column(`data2`) = 12 COLLATE ICU.UNICODE) OR (column(`part3`) |>= sss COLLATE ICU.UNICODE))""".stripMargin.replaceAll("\n", " ")), // predicates (multiple on data and partition columns) joined with AND predicate( "AND", predicate( "AND", predicate("=", col("data1"), ofInt(12)), predicate(">=", col("data2"), ofString("sss"))), predicate( "AND", predicate("=", col("part1"), ofInt(12)), predicate(">=", col("part3"), ofString("sss")))) -> ( "((column(`part1`) = 12) AND (column(`part3`) >= sss))", "((column(`data1`) = 12) AND (column(`data2`) >= sss))"), // predicates (multiple on data and partition columns joined with OR) joined with AND predicate( "AND", predicate( "OR", predicate("=", col("data1"), ofInt(12)), predicate(">=", col("data2"), ofString("sss"))), predicate( "OR", predicate("=", col("part1"), ofInt(12)), predicate(">=", col("part3"), ofString("sss")))) -> ( "((column(`part1`) = 12) OR (column(`part3`) >= sss))", "((column(`data1`) = 12) OR (column(`data2`) >= sss))"), // predicates (multiple on data and partition columns joined with OR) joined with OR predicate( "OR", predicate( "OR", predicate("=", col("data1"), ofInt(12)), predicate(">=", col("data2"), ofString("sss"))), predicate( "OR", predicate("=", col("part1"), ofInt(12)), predicate(">=", col("part3"), ofString("sss")))) -> ( "ALWAYS_TRUE()", "(((column(`data1`) = 12) OR (column(`data2`) >= sss)) OR " + "((column(`part1`) = 12) OR (column(`part3`) >= sss)))"), // predicates (data and partitions compared in the same expression) predicate( "AND", predicate("=", col("data1"), col("part1")), predicate(">=", col("part3"), ofString("sss"))) -> ( "(column(`part3`) >= sss)", "(column(`data1`) = column(`part1`))"), // predicates with collation (data and partitions compared in the same expression) predicate( "AND", predicate("=", col("data2"), col("part3"), utf8Lcase), predicate(">=", col("part3"), ofString("sss"), unicode)) -> ( "(column(`part3`) >= sss COLLATE ICU.UNICODE)", "(column(`data2`) = column(`part3`) COLLATE SPARK.UTF8_LCASE)"), // predicate only on data column but reverse order of literal and column predicate("=", ofInt(12), col("data1")) -> ("ALWAYS_TRUE()", "(12 = column(`data1`))"), // predicate with collation only on data column but reverse order of literal and column predicate("=", ofString("12"), col("data2"), utf8Lcase) -> ("ALWAYS_TRUE()", "(12 = column(`data2`) COLLATE SPARK.UTF8_LCASE)")) partitionTestCases.foreach { case (predicate, (partitionPredicate, dataPredicate)) => test(s"split predicate into data and partition predicates: $predicate") { val metadataAndDataPredicates = splitMetadataAndDataPredicates(predicate, partitionCols) assert(metadataAndDataPredicates._1.toString === partitionPredicate) assert(metadataAndDataPredicates._2.toString === dataPredicate) } } // Map entry format: (given predicate -> \ // (exp predicate for partition pruning, exp predicate for checkpoint reader pushdown)) val rewriteTestCases = Map( // single predicate on a partition column predicate("=", col("part2"), ofTimestamp(12)) -> ( // exp predicate for partition pruning "(partition_value(ELEMENT_AT(column(`add`.`partitionValues`), part2), date) = 12)", // exp predicate for checkpoint reader pushdown "(column(`add`.`partitionValues_parsed`.`part2`) = 12)"), // single predicate with collation on a partition column predicate("=", col("part3"), ofString("sss"), utf8Lcase) -> ( // exp predicate for partition pruning "(ELEMENT_AT(column(`add`.`partitionValues`), part3) = sss COLLATE SPARK.UTF8_LCASE)", // exp predicate for checkpoint reader pushdown "(column(`add`.`partitionValues_parsed`.`part3`) = sss COLLATE SPARK.UTF8_LCASE)"), // multiple predicates on partition columns joined with AND predicate( "AND", predicate("=", col("part1"), ofInt(12)), predicate(">=", col("part3"), ofString("sss"))) -> ( // exp predicate for partition pruning """((partition_value(ELEMENT_AT(column(`add`.`partitionValues`), part1), integer) = 12) AND |(ELEMENT_AT(column(`add`.`partitionValues`), part3) >= sss))""" .stripMargin.replaceAll("\n", " "), // exp predicate for checkpoint reader pushdown """((column(`add`.`partitionValues_parsed`.`part1`) = 12) AND |(column(`add`.`partitionValues_parsed`.`part3`) >= sss))""" .stripMargin.replaceAll("\n", " ")), // multiple predicates with collation on partition columns joined with AND predicate( "AND", predicate("=", col("part3"), ofString("sss"), utf8Lcase), predicate(">=", col("part3"), ofString("sss"), CollationIdentifier.SPARK_UTF8_BINARY)) -> ( // exp predicate for partition pruning """((ELEMENT_AT(column(`add`.`partitionValues`), part3) = sss COLLATE SPARK.UTF8_LCASE) AND |(ELEMENT_AT(column(`add`.`partitionValues`), part3) >= sss COLLATE SPARK.UTF8_BINARY))""" .stripMargin.replaceAll("\n", " "), // exp predicate for checkpoint reader pushdown """((column(`add`.`partitionValues_parsed`.`part3`) = sss COLLATE SPARK.UTF8_LCASE) AND |(column(`add`.`partitionValues_parsed`.`part3`) >= sss COLLATE SPARK.UTF8_BINARY))""" .stripMargin.replaceAll("\n", " ")), // multiple predicates on partition columns joined with OR predicate( "OR", predicate("<=", col("part3"), ofString("sss")), predicate("=", col("part1"), ofInt(2781))) -> ( // exp predicate for partition pruning """((ELEMENT_AT(column(`add`.`partitionValues`), part3) <= sss) OR |(partition_value(ELEMENT_AT(column(`add`.`partitionValues`), part1), integer) = 2781))""" .stripMargin.replaceAll("\n", " "), // exp predicate for checkpoint reader pushdown """((column(`add`.`partitionValues_parsed`.`part3`) <= sss) OR |(column(`add`.`partitionValues_parsed`.`part1`) = 2781))""" .stripMargin.replaceAll("\n", " "))) rewriteTestCases.foreach { case (predicate, (expPartitionPruningPredicate, expCheckpointReaderPushdownPredicate)) => test(s"rewrite partition predicate on scan file schema: $predicate") { val actPartitionPruningPredicate = rewritePartitionPredicateOnScanFileSchema(predicate, partitionColsMetadata) assert(actPartitionPruningPredicate.toString === expPartitionPruningPredicate) val actCheckpointReaderPushdownPredicate = rewritePartitionPredicateOnCheckpointFileSchema(predicate, partitionColsMetadata) assert(actCheckpointReaderPushdownPredicate.toString === expCheckpointReaderPushdownPredicate) } } private val nullFileName = "__HIVE_DEFAULT_PARTITION__" Seq( ofBoolean(true) -> ("true", "true"), ofBoolean(false) -> ("false", "false"), ofNull(BooleanType.BOOLEAN) -> (null, nullFileName), ofByte(24.toByte) -> ("24", "24"), ofNull(ByteType.BYTE) -> (null, nullFileName), ofShort(876.toShort) -> ("876", "876"), ofNull(ShortType.SHORT) -> (null, nullFileName), ofInt(2342342) -> ("2342342", "2342342"), ofNull(IntegerType.INTEGER) -> (null, nullFileName), ofLong(234234223L) -> ("234234223", "234234223"), ofNull(LongType.LONG) -> (null, nullFileName), ofFloat(23423.4223f) -> ("23423.422", "23423.422"), ofNull(FloatType.FLOAT) -> (null, nullFileName), ofDouble(23423.422233d) -> ("23423.422233", "23423.422233"), ofNull(DoubleType.DOUBLE) -> (null, nullFileName), ofString("string_val") -> ("string_val", "string_val"), ofString("string_\nval") -> ("string_\nval", "string_%0Aval"), ofString("str=ing_\u0001val") -> ("str=ing_\u0001val", "str%3Ding_%01val"), ofNull(StringType.STRING) -> (null, nullFileName), ofDecimal(new java.math.BigDecimal("23423.234234"), 15, 7) -> ("23423.2342340", "23423.2342340"), ofNull(new DecimalType(15, 7)) -> (null, nullFileName), ofBinary("binary_val".getBytes) -> ("binary_val", "binary_val"), ofNull(BinaryType.BINARY) -> (null, nullFileName), ofDate(4234) -> ("1981-08-05", "1981-08-05"), ofNull(DateType.DATE) -> (null, nullFileName), ofTimestamp(2342342342232L) -> ("1970-01-28 02:39:02.342232", "1970-01-28 02%3A39%3A02.342232"), ofNull(TimestampType.TIMESTAMP) -> (null, nullFileName), ofTimestampNtz(-2342342342L) -> ("1969-12-31 23:20:58.657658", "1969-12-31 23%3A20%3A58.657658"), ofNull(TimestampNTZType.TIMESTAMP_NTZ) -> (null, nullFileName)).foreach { case (literal, (expSerializedValue, expFileName)) => test(s"serialize partition value literal as string: ${literal.getDataType}($literal)") { val result = serializePartitionValue(literal) assert(result === expSerializedValue) } test(s"construct partition data output directory: ${literal.getDataType}($literal)") { val result = getTargetDirectory( "/tmp/root", Seq("part1").asJava, Map("part1" -> literal).asJava) assert(result === s"/tmp/root/part1=$expFileName") } } test("construct partition data output directory with multiple partition columns") { val result = getTargetDirectory( "/tmp/root", Seq("part1", "part2", "part3").asJava, Map( "part1" -> ofInt(12), "part3" -> ofTimestamp(234234234L), "part2" -> ofString("sss")).asJava) assert(result === "/tmp/root/part1=12/part2=sss/part3=1970-01-01 00%3A03%3A54.234234") } // Test cases for verifying if timestamp can be parsed correctly. test("parse valid standard timestamp") { val result1 = PartitionUtils.tryParseTimestamp("2024-01-01 10:00:00") assert(result1 == 1704103200000000L) val result2 = PartitionUtils.tryParseTimestamp("2024-01-01 10:00:00.123456") assert(result2 == 1704103200123456L) } test("parse valid ISO8601 timestamp") { val result = PartitionUtils.tryParseTimestamp("2024-01-01T10:00:00Z") assert(result == 1704103200000000L) } test("parse valid ISO8601 timestamp with microsecond precision") { val result = PartitionUtils.tryParseTimestamp("2025-01-01T00:00:00.123456Z") assert(result == 1735689600123456L) } test("throw on invalid timestamp") { val thrown = intercept[IllegalStateException] { PartitionUtils.tryParseTimestamp("not-a-timestamp") } assert(thrown.getMessage.contains("Invalid timestamp format for value")) } private def col(names: String*): Column = { new Column(names.toArray) } private def predicate(name: String, children: Expression*): Predicate = { new Predicate(name, children.asJava) } private def predicate( name: String, left: Expression, right: Expression, collationIdentifier: CollationIdentifier) = { new Predicate(name, left, right, collationIdentifier) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/SchemaIterableSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util import java.util.Optional import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer import io.delta.kernel.types._ import org.scalatest.funsuite.AnyFunSuite class SchemaIterableSuite extends AnyFunSuite { test("depth first traversal works with deeply nested types") { val schema: StructType = getDeeplyNestedSchema val iterable = new SchemaIterable(schema) def parentStructFieldAndPathToString( parent: Optional[SchemaIterable.ParentStructFieldInfo]): String = { if (parent.isPresent) { val parentInfo = parent.get val parentField = parentInfo.getParentField val parentPath = parentInfo.getPathFromParent s"Some(${parentField.getName}, $parentPath)" } else { "None" } } // Track the path stack during traversal val fieldInfo = iterable.asScala.map { element => ( parentStructFieldAndPathToString(element.getParentStructFieldAndPath()), element.getNamePath(), element.getPathFromNearestStructFieldAncestor( element.getNearestStructFieldAncestor.getName), element.getPathFromNearestStructFieldAncestor(""), element.getField.getDataType.getClass.getSimpleName) } .toList // The expected traversal order with field types, showing the complete depth-first traversal val expectedOrder = List( // First branch: nested_array ("Some(nested_array, element)", "nested_array.element.id", "id", "", "IntegerType"), ( "Some(tags, )", "nested_array.element.tags.element", "tags.element", "element", "StringType"), ("Some(nested_array, element)", "nested_array.element.tags", "tags", "", "ArrayType"), ( "Some(nested_array, )", "nested_array.element", "nested_array.element", "element", "StructType"), ("None", "nested_array", "nested_array", "", "ArrayType"), // Second branch: nested_map ( "Some(nested_map, key)", "nested_map.key.element", "nested_map.key.element", "key.element", "StringType"), ("Some(nested_map, )", "nested_map.key", "nested_map.key", "key", "ArrayType"), ( "Some(points, element)", "nested_map.value.points.element.x", "x", "", "DoubleType"), ( "Some(points, element)", "nested_map.value.points.element.y", "y", "", "DoubleType"), ( "Some(points, )", "nested_map.value.points.element", "points.element", "element", "StructType"), ( "Some(nested_map, value)", "nested_map.value.points", "points", "", "ArrayType"), ( "Some(metadata, )", "nested_map.value.metadata.key", "metadata.key", "key", "StringType"), ( "Some(metadata, )", "nested_map.value.metadata.value", "metadata.value", "value", "IntegerType"), ( "Some(nested_map, value)", "nested_map.value.metadata", "metadata", "", "MapType"), ("Some(nested_map, )", "nested_map.value", "nested_map.value", "value", "StructType"), ("None", "nested_map", "nested_map", "", "MapType"), // Third branch ( "Some(double_nested, element.element.key.key)", "double_nested.element.element.key.key.element", "double_nested.element.element.key.key.element", "element.element.key.key.element", "IntegerType"), ( "Some(double_nested, element.element.key)", "double_nested.element.element.key.key", "double_nested.element.element.key.key", "element.element.key.key", "ArrayType"), ( "Some(double_nested, element.element.key)", "double_nested.element.element.key.value", "double_nested.element.element.key.value", "element.element.key.value", "StringType"), ( "Some(double_nested, element.element)", "double_nested.element.element.key", "double_nested.element.element.key", "element.element.key", "MapType"), ( "Some(double_nested, element.element.value)", "double_nested.element.element.value.key", "double_nested.element.element.value.key", "element.element.value.key", "StringType"), ( "Some(double_nested, element.element.value)", "double_nested.element.element.value.value", "double_nested.element.element.value.value", "element.element.value.value", "StringType"), ( "Some(double_nested, element.element)", "double_nested.element.element.value", "double_nested.element.element.value", "element.element.value", "MapType"), ( "Some(double_nested, element)", "double_nested.element.element", "double_nested.element.element", "element.element", "MapType"), ( "Some(double_nested, )", "double_nested.element", "double_nested.element", "element", "ArrayType"), ("None", "double_nested", "double_nested", "", "ArrayType"), // fourth branch ("None", "empty_struct", "empty_struct", "", "StructType"), // fifth branch ( "Some(empty_struct_array, )", "empty_struct_array.element", "empty_struct_array.element", "element", "StructType"), ("None", "empty_struct_array", "empty_struct_array", "", "ArrayType"), // sixth branch ( "Some(empty_map_struct, )", "empty_map_struct.key", "empty_map_struct.key", "key", "StructType"), ( "Some(empty_map_struct, )", "empty_map_struct.value", "empty_map_struct.value", "value", "StructType"), ("None", "empty_map_struct", "empty_map_struct", "", "MapType")) fieldInfo.zip(expectedOrder).foreach { case (actual, expected) => assert(actual == expected) } assert(fieldInfo == expectedOrder) } Seq( (new StructType(), List()), (new StructType().add("empty", new StructType()), List("empty")), ( new StructType().add("f1", new StructType().add("f2", IntegerType.INTEGER)), List("f1.f2", "f1")), ( new StructType().add("f1", IntegerType.INTEGER).add("f2", IntegerType.INTEGER), List("f1", "f2")), ( new StructType() .add("f1", IntegerType.INTEGER) .add( "s1", new StructType() .add("f1", IntegerType.INTEGER) .add("f2", IntegerType.INTEGER)) .add("s2", new StructType()) .add("f2", IntegerType.INTEGER), List("f1", "s1.f1", "s1.f2", "s1", "s2", "f2"))).foreach { case (schema, expected) => test(s"check basic iteration ${schema.toString}") { val iterable = new SchemaIterable(schema) val fieldInfo = iterable.asScala.map { field => (field.getNamePath) } .toList assert(fieldInfo === expected) } } test("test update schema") { val schema: StructType = getDeeplyNestedSchema val iterable = new SchemaIterable(schema) val fieldMetadata = FieldMetadata.builder() .putString("k1", "v1") .build() val newTypes = Map( "nested_array.element.tags.element" -> IntegerType.INTEGER, "nested_map.value.metadata.value" -> StringType.STRING, "nested_map.value.points.element" -> new StructType().add( "x", DoubleType.DOUBLE, false).add("y", DoubleType.DOUBLE, false) .add("z", LongType.LONG, false)) val newMetadata = Map("nested_array" -> fieldMetadata) iterable.newMutableIterator().asScala.foreach { element => newTypes.get(element.getNamePath).foreach { t => element.updateField(element.getField.withDataType(t)) } newMetadata.get(element.getNamePath).foreach { fm => element.updateField(element.getField.withNewMetadata(fm)) } } val fieldInfo = iterable.asScala.map { element => (element.getNamePath, element.getField.getDataType.getClass.getSimpleName) }.toList // The expected traversal order with field types, showing the complete depth-first traversal val expectedOrder = List( // First branch: nested_array ("nested_array.element.id", "IntegerType"), ("nested_array.element.tags.element", "IntegerType"), ("nested_array.element.tags", "ArrayType"), ("nested_array.element", "StructType"), ("nested_array", "ArrayType"), // Second branch: nested_map ("nested_map.key.element", "StringType"), ("nested_map.key", "ArrayType"), ("nested_map.value.points.element.x", "DoubleType"), ("nested_map.value.points.element.y", "DoubleType"), ("nested_map.value.points.element.z", "LongType"), ("nested_map.value.points.element", "StructType"), ("nested_map.value.points", "ArrayType"), ("nested_map.value.metadata.key", "StringType"), ("nested_map.value.metadata.value", "StringType"), ("nested_map.value.metadata", "MapType"), ("nested_map.value", "StructType"), ("nested_map", "MapType"), // Third branch ("double_nested.element.element.key.key.element", "IntegerType"), ("double_nested.element.element.key.key", "ArrayType"), ("double_nested.element.element.key.value", "StringType"), ("double_nested.element.element.key", "MapType"), ("double_nested.element.element.value.key", "StringType"), ("double_nested.element.element.value.value", "StringType"), ("double_nested.element.element.value", "MapType"), ("double_nested.element.element", "MapType"), ("double_nested.element", "ArrayType"), ("double_nested", "ArrayType"), // fourth branch ("empty_struct", "StructType"), // fifth branch ("empty_struct_array.element", "StructType"), ("empty_struct_array", "ArrayType"), // sixth branch ("empty_map_struct.key", "StructType"), ("empty_map_struct.value", "StructType"), ("empty_map_struct", "MapType")) fieldInfo.zip(expectedOrder).foreach { case (actual, expected) => assert(actual == expected) } assert(iterable.getSchema.get("nested_array").getMetadata == fieldMetadata) } test("test set nearest ancestor field metadata") { val schema: StructType = getDeeplyNestedSchema val iterable = new SchemaIterable(schema) val newMetadata = Map( "nested_array.element.tags.element" -> newFieldMetadata("v1"), "nested_map.value.metadata" -> newFieldMetadata("v2"), "nested_map.value.metadata.value" -> newFieldMetadata("v3"), "nested_array" -> newFieldMetadata("v4")) val expected = Map( "nested_array.element.tags" -> newFieldMetadata("v1"), "nested_array.element.tags.element" -> FieldMetadata.empty(), "nested_map.value.metadata" -> FieldMetadata.builder .fromMetadata(newFieldMetadata("v2")) .fromMetadata(newFieldMetadata("v3")).build(), "nested_map.value.metadata.value" -> FieldMetadata.empty(), "nested_array" -> newFieldMetadata("v4")) val originalCount = iterable.asScala.count(_ => true) iterable.newMutableIterator().asScala.foreach { element => newMetadata.get(element.getNamePath).foreach { fm => val ancestorField = element.getNearestStructFieldAncestor val metadataBuilder = FieldMetadata.builder() .fromMetadata(ancestorField.getMetadata).fromMetadata(fm) element.setMetadataOnNearestStructFieldAncestor(metadataBuilder.build()) } } iterable.asScala.foreach { element => expected.get(element.getNamePath).foreach { fm => assert( fm == element.getField.getMetadata, s"Path: ${element.getNamePath} ${iterable.getSchema} ") } } val newCount = iterable.asScala.count(_ => true) assert(newCount > 0) assert(originalCount == newCount) } val testCases = Seq( // Test case 1: Skip ArrayType ( Seq(classOf[ArrayType]), List( "nested_array", "nested_map.key", "nested_map.value.points", "nested_map.value.metadata.key", "nested_map.value.metadata.value", "nested_map.value.metadata", "nested_map.value", "nested_map", "double_nested", "empty_struct", "empty_struct_array", "empty_map_struct.key", "empty_map_struct.value", "empty_map_struct")), // Test case 2: Skip MapType ( Seq(classOf[MapType]), List( "nested_array.element.id", "nested_array.element.tags.element", "nested_array.element.tags", "nested_array.element", "nested_array", "nested_map", "double_nested.element.element", "double_nested.element", "double_nested", "empty_struct", "empty_struct_array.element", "empty_struct_array", "empty_map_struct")), // Test case 3: Skip StructType ( Seq(classOf[StructType]), List( "nested_array.element", "nested_array", "nested_map.key.element", "nested_map.key", "nested_map.value", "nested_map", "double_nested.element.element.key.key.element", "double_nested.element.element.key.key", "double_nested.element.element.key.value", "double_nested.element.element.key", "double_nested.element.element.value.key", "double_nested.element.element.value.value", "double_nested.element.element.value", "double_nested.element.element", "double_nested.element", "double_nested", "empty_struct", "empty_struct_array.element", "empty_struct_array", "empty_map_struct.key", "empty_map_struct.value", "empty_map_struct")), // Test case 4: Skip multiple types (ArrayType and MapType) ( Seq(classOf[ArrayType], classOf[MapType]), List( "nested_array", "nested_map", "double_nested", "empty_struct", "empty_struct_array", "empty_map_struct"))).foreach { case (typesToSkip, expectedFields) => test(s"skip recursion for specified types $typesToSkip") { val schema: StructType = getDeeplyNestedSchema // Define test cases as a sequence of (types to skip, expected output) pairs val iterable = SchemaIterable.newSchemaIterableWithIgnoredRecursion(schema, typesToSkip.toArray) val visitedPaths = iterable.asScala.map(_.getNamePath).toList // Assert the results match expected output assert( visitedPaths === expectedFields, s"Failed for types: ${typesToSkip.map(_.getSimpleName).mkString(", ")}") } } test("update schema with type recursion skipping") { val schema: StructType = getDeeplyNestedSchema val iterable = SchemaIterable.newSchemaIterableWithIgnoredRecursion(schema, Array(classOf[ArrayType])) // Create a modified schema by skipping recursion into ArrayType // and modifying only the top-level fields iterable.newMutableIterator.asScala.foreach { element => if (element.getNamePath.contains("nested_array")) { // Add metadata to the array field but don't recurse into it val fieldMetadata = FieldMetadata.builder() .putString("array_skipped", "true") .build() element.updateField(element.getField.withNewMetadata(fieldMetadata)) } } // Verify the metadata was added to the array field assert(iterable.getSchema.get("nested_array").getMetadata .getString("array_skipped") == "true") // Verify that the array elements were not modified (recursion was skipped) val newIterable = new SchemaIterable(iterable.getSchema) var visited_count = 0 newIterable.asScala.foreach(element => { if (element.getNamePath.startsWith("nested_array.")) { visited_count += 1 assert(element.getField.getMetadata == FieldMetadata.empty()) } }) assert(visited_count > 0) } private def newFieldMetadata(v: String) = FieldMetadata.builder().putString(v, v).build() private def getDeeplyNestedSchema = { val intType = IntegerType.INTEGER val stringType = StringType.STRING val doubleType = DoubleType.DOUBLE // Create a deeply nested schema: // struct< // nested_array: array< // struct< // id: int, // tags: array // > // >, // nested_map: map< // array, // struct< // points: array< // struct // >, // metadata: map // > // > // double_nested: // array, string>, map>> // empty_struct: struct<> // empty_struct_array: // array>> // empty_map_struct: // map, struct<>> // > // Define the point struct inside the array val pointStruct = new StructType().add("x", doubleType).add("y", doubleType); // Define the inner struct containing tags array val innerStruct = new StructType().add("id", intType).add( "tags", new ArrayType(stringType, true)); // Define the value struct for the nested map val valueStruct = new StructType().add("points", new ArrayType(pointStruct, false)).add( "metadata", new MapType( stringType, intType, /* valuesContainsNull = */ false)); // Create the root schema val schema = new StructType().add("nested_array", new ArrayType(innerStruct, false)) .add("nested_map", new MapType(new ArrayType(stringType, false), valueStruct, true)) .add( "double_nested", new ArrayType( new ArrayType( new MapType( new MapType(new ArrayType(IntegerType.INTEGER, true), StringType.STRING, true), new MapType(StringType.STRING, StringType.STRING, true), true), false), false), false) .add("empty_struct", new StructType(), false) .add("empty_struct_array", new ArrayType(new StructType(), true), false) .add("empty_map_struct", new MapType(new StructType(), new StructType(), true), false) schema } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/SchemaUtilsSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util import java.util import java.util.{Locale, Optional} import java.util.Collections.emptySet import scala.collection.JavaConverters._ import scala.collection.JavaConverters.mapAsJavaMapConverter import scala.reflect.ClassTag import io.delta.kernel.exceptions.KernelException import io.delta.kernel.internal.TableConfig import io.delta.kernel.internal.actions.{Format, Metadata, Protocol} import io.delta.kernel.internal.tablefeatures.{TableFeature, TableFeatures} import io.delta.kernel.internal.types.DataTypeJsonSerDe import io.delta.kernel.internal.util.ColumnMapping.{COLUMN_MAPPING_ID_KEY, COLUMN_MAPPING_MODE_KEY, COLUMN_MAPPING_PHYSICAL_NAME_KEY} import io.delta.kernel.internal.util.SchemaUtils.{computeSchemaChangesById, validateUpdatedSchemaAndGetUpdatedSchema} import io.delta.kernel.internal.util.VectorUtils.stringStringMapValue import io.delta.kernel.types.{ArrayType, ByteType, DataType, DoubleType, FieldMetadata, IntegerType, LongType, MapType, StringType, StructField, StructType, TypeChange, VariantType} import io.delta.kernel.types.IntegerType.INTEGER import io.delta.kernel.types.LongType.LONG import io.delta.kernel.types.TimestampType.TIMESTAMP import org.scalatest.funsuite.AnyFunSuite import org.scalatest.prop.TableDrivenPropertyChecks.forAll import org.scalatest.prop.TableFor2 import org.scalatest.prop.Tables.Table class SchemaUtilsSuite extends AnyFunSuite { val dummyProtocol = new Protocol(0, 0) private def expectFailure(shouldContain: String*)(f: => Unit): Unit = { val e = intercept[KernelException] { f } val msg = e.getMessage.toLowerCase(Locale.ROOT) assert( shouldContain.map(_.toLowerCase(Locale.ROOT)).forall(msg.contains), s"Error message '$msg' didn't contain: $shouldContain") } private def validateSchema( schema: StructType, isColumnMappingEnabled: Boolean = false, isColumnDefaultEnabled: Boolean = false, isIcebergCompatV3Enabled: Boolean = false): Unit = SchemaUtils.validateSchema( schema, isColumnMappingEnabled, isColumnDefaultEnabled, isIcebergCompatV3Enabled) /////////////////////////////////////////////////////////////////////////// // Duplicate Column Checks /////////////////////////////////////////////////////////////////////////// test("duplicate column name in top level") { val schema = new StructType() .add("dupColName", INTEGER) .add("b", INTEGER) .add("dupColName", StringType.STRING) expectFailure("dupColName") { validateSchema(schema) } } test("duplicate column name in top level - case sensitivity") { val schema = new StructType() .add("dupColName", INTEGER) .add("b", INTEGER) .add("dupCOLNAME", StringType.STRING) expectFailure("dupColName") { validateSchema(schema) } } test("duplicate column name for nested column + non-nested column") { val schema = new StructType() .add( "dupColName", new StructType() .add("a", INTEGER) .add("b", INTEGER)) .add("dupColName", INTEGER) expectFailure("dupColName") { validateSchema(schema) } } test("duplicate column name for nested column + non-nested column - case sensitivity") { val schema = new StructType() .add( "dupColName", new StructType() .add("a", INTEGER) .add("b", INTEGER)) .add("dupCOLNAME", INTEGER) expectFailure("dupCOLNAME") { validateSchema(schema) } } test("duplicate column name in nested level") { val schema = new StructType() .add( "top", new StructType() .add("dupColName", INTEGER) .add("b", INTEGER) .add("dupColName", StringType.STRING)) expectFailure("top.dupColName") { validateSchema(schema) } } test("duplicate column name in nested level - case sensitivity") { val schema = new StructType() .add( "top", new StructType() .add("dupColName", INTEGER) .add("b", INTEGER) .add("dupCOLNAME", StringType.STRING)) expectFailure("top.dupColName") { validateSchema(schema) } } test("duplicate column name in double nested level") { val schema = new StructType() .add( "top", new StructType() .add( "b", new StructType() .add("dupColName", StringType.STRING) .add("c", INTEGER) .add("dupColName", StringType.STRING)) .add("d", INTEGER)) expectFailure("top.b.dupColName") { validateSchema(schema) } } test("duplicate column name in double nested array") { val schema = new StructType() .add( "top", new StructType() .add( "b", new ArrayType( new ArrayType( new StructType() .add("dupColName", StringType.STRING) .add("c", INTEGER) .add("dupColName", StringType.STRING), true), true)) .add("d", INTEGER)) expectFailure("top.b.element.element.dupColName") { validateSchema(schema) } } test("only duplicate columns are listed in the error message") { val schema = new StructType() .add("top", new StructType().add("a", INTEGER).add("b", INTEGER).add("c", INTEGER)).add( "top", new StructType().add("b", INTEGER).add("c", INTEGER).add("d", INTEGER)).add( "bottom", new StructType().add("b", INTEGER).add("c", INTEGER).add("d", INTEGER)) val e = intercept[KernelException] { validateSchema(schema) } assert(e.getMessage.contains("Schema contains duplicate columns: top, top.b, top.c")) } test("duplicate column name in double nested map") { val keyType = new StructType() .add("dupColName", INTEGER) .add("d", StringType.STRING) expectFailure("top.b.key.dupColName") { val schema = new StructType() .add( "top", new StructType() .add("b", new MapType(keyType.add("dupColName", StringType.STRING), keyType, true))) validateSchema(schema) } expectFailure("top.b.value.dupColName") { val schema = new StructType() .add( "top", new StructType() .add("b", new MapType(keyType, keyType.add("dupColName", StringType.STRING), true))) validateSchema(schema) } // This is okay val schema = new StructType() .add( "top", new StructType() .add("b", new MapType(keyType, keyType, true))) validateSchema(schema) } test("duplicate column name in nested array") { val schema = new StructType() .add( "top", new ArrayType( new StructType() .add("dupColName", INTEGER) .add("b", INTEGER) .add("dupColName", StringType.STRING), true)) expectFailure("top.element.dupColName") { validateSchema(schema) } } test("duplicate column name in nested array - case sensitivity") { val schema = new StructType() .add( "top", new ArrayType( new StructType() .add("dupColName", INTEGER) .add("b", INTEGER) .add("dupCOLNAME", StringType.STRING), true)) expectFailure("top.element.dupColName") { validateSchema(schema) } } test("non duplicate column because of back tick") { val schema = new StructType() .add( "top", new StructType() .add("a", INTEGER) .add("b", INTEGER)) .add("top.a", INTEGER) validateSchema(schema) } test("non duplicate column because of back tick - nested") { val schema = new StructType() .add( "first", new StructType() .add( "top", new StructType() .add("a", INTEGER) .add("b", INTEGER)) .add("top.a", INTEGER)) validateSchema(schema) } test("variant") { val schema = new StructType() .add( "variant", VariantType.VARIANT) validateSchema(schema) } test("variant - nested") { val schema = new StructType() .add( "first", new StructType() .add("variant", VariantType.VARIANT)) validateSchema(schema) } test("duplicate column with back ticks - nested") { val schema = new StructType() .add( "first", new StructType() .add("top.a", StringType.STRING) .add("b", INTEGER) .add("top.a", INTEGER)) expectFailure("first.`top.a`") { validateSchema(schema) } } test("duplicate column with back ticks - nested and case sensitivity") { val schema = new StructType() .add( "first", new StructType() .add("TOP.a", StringType.STRING) .add("b", INTEGER) .add("top.a", INTEGER)) expectFailure("first.`top.a`") { validateSchema(schema) } } /////////////////////////////////////////////////////////////////////////// // check default columns check is invoked /////////////////////////////////////////////////////////////////////////// test("check default columns checks are invoked") { val schema = new StructType() .add( "first", new StructType() .add("TOP.a", StringType.STRING) .add("b", INTEGER) .add("top.a", INTEGER)) expectFailure("first.`top.a`") { validateSchema( schema, isColumnMappingEnabled = false, isColumnDefaultEnabled = true, isIcebergCompatV3Enabled = false) } } /////////////////////////////////////////////////////////////////////////// // checkFieldNames /////////////////////////////////////////////////////////////////////////// test("check non alphanumeric column characters") { val badCharacters = " ,;{}()\n\t=" val goodCharacters = "#.`!@$%^&*~_<>?/:" badCharacters.foreach { char => Seq(s"a${char}b", s"${char}ab", s"ab${char}", char.toString).foreach { name => val schema = new StructType().add(name, INTEGER) val e = intercept[KernelException] { validateSchema(schema) } if (char != '\n') { // with column mapping disabled this should be a valid name validateSchema( schema, isColumnMappingEnabled = true, isColumnDefaultEnabled = false, isIcebergCompatV3Enabled = false) } assert(e.getMessage.contains("contains one of the unsupported")) } } goodCharacters.foreach { char => // no issues here Seq(s"a${char}b", s"${char}ab", s"ab${char}", char.toString).foreach { name => val schema = new StructType().add(name, INTEGER); validateSchema(schema) validateSchema( schema, /* isColumnMappingEnabled= */ true, /* isColumnDefaultEnabled= */ false, /* isIcebergCompatV3Enabled= */ false) } } } /////////////////////////////////////////////////////////////////////////// // computeSchemaChangesById /////////////////////////////////////////////////////////////////////////// test("Compute schema changes with added columns") { val fieldMappingBefore = newSchema((1, new StructField("id", IntegerType.INTEGER, true))) val fieldMappingAfter = newSchema( (1, new StructField("id", IntegerType.INTEGER, true)), (2, new StructField("data", StringType.STRING, true))) val schemaChanges = computeSchemaChangesById(fieldMappingBefore, fieldMappingAfter) assert(schemaChanges.removedFields().isEmpty) assert(schemaChanges.updatedFields().isEmpty) assert(schemaChanges.addedFields().size() == 1) assert(schemaChanges.addedFields().get(0) == fieldMappingAfter.get("data")) } test("Compute schema changes with renamed fields") { val fieldMappingBefore = newSchema( (1, new StructField("id", IntegerType.INTEGER, true))) val fieldMappingAfter = newSchema( (1, new StructField("renamed_id", IntegerType.INTEGER, true))) val schemaChanges = computeSchemaChangesById(fieldMappingBefore, fieldMappingAfter) assert(schemaChanges.addedFields().isEmpty) assert(schemaChanges.removedFields().isEmpty) assert(schemaChanges.updatedFields().size() == 1) val schemaUpdate = schemaChanges.updatedFields().get(0) assert(schemaUpdate.before == fieldMappingBefore.get("id")) assert(schemaUpdate.after == fieldMappingAfter.get("renamed_id")) } test("Compute schema changes with type changed columns") { val fieldMappingBefore = newSchema((1, new StructField("id", IntegerType.INTEGER, true))) val fieldMappingAfter = newSchema((1, new StructField("promoted_to_long", LongType.LONG, true))); val schemaChanges = computeSchemaChangesById(fieldMappingBefore, fieldMappingAfter) assert(schemaChanges.addedFields().isEmpty) assert(schemaChanges.removedFields().isEmpty) assert(schemaChanges.updatedFields().size() == 1) val schemaUpdate = schemaChanges.updatedFields().get(0) assert(schemaUpdate.before == fieldMappingBefore.get("id")) assert(schemaUpdate.after == fieldMappingAfter.get( "promoted_to_long")) } test("Compute schema changes with dropped fields") { val fieldMappingBefore = newSchema( (1, new StructField("id", IntegerType.INTEGER, true)), (2, new StructField("data", StringType.STRING, true))) val fieldMappingAfter = newSchema(( 2 -> new StructField("data", StringType.STRING, true))) val schemaChanges = computeSchemaChangesById(fieldMappingBefore, fieldMappingAfter) assert(schemaChanges.addedFields().isEmpty) assert(schemaChanges.updatedFields().isEmpty) assert(schemaChanges.removedFields().size() == 1) assert(schemaChanges.removedFields().get(0) == fieldMappingBefore.get("id")) } test("Compute schema changes with nullability change") { val fieldMappingBefore = newSchema( (1, new StructField("id", IntegerType.INTEGER, true)), (2, new StructField("data", StringType.STRING, true))) val fieldMappingAfter = newSchema( (1, new StructField("id", IntegerType.INTEGER, true)), (2, new StructField("required_data", StringType.STRING, false))) val schemaChanges = computeSchemaChangesById(fieldMappingBefore, fieldMappingAfter) assert(schemaChanges.addedFields().isEmpty) assert(schemaChanges.removedFields().isEmpty) assert(schemaChanges.updatedFields().size() == 1) val schemaUpdate = schemaChanges.updatedFields().get(0) assert(schemaUpdate.before == fieldMappingBefore.get("data")) assert( schemaUpdate.after == fieldMappingAfter.get("required_data")) } test("Compute schema changes with moved fields") { val fieldMappingBefore = newSchema( ( 1, new StructField( "struct", newSchema( (4, new StructField("id", IntegerType.INTEGER, true)), (5, new StructField("data", StringType.STRING, true))), true)), (2, new StructField("id", IntegerType.INTEGER, true)), (3, new StructField("data", StringType.STRING, true))) val fieldMappingAfter = newSchema( ( 1, new StructField( "struct", newSchema( (5, new StructField("data", StringType.STRING, true)), (4, new StructField("id", IntegerType.INTEGER, true))), true)), (2, new StructField("id", IntegerType.INTEGER, true)), (3, new StructField("data", StringType.STRING, true))) val schemaChanges = computeSchemaChangesById(fieldMappingBefore, fieldMappingAfter) assert(schemaChanges.addedFields().isEmpty) assert(schemaChanges.removedFields().isEmpty) assert( schemaChanges.updatedFields().size() == 1, s"${schemaChanges.updatedFields.get(0).before}") val schemaUpdate = schemaChanges.updatedFields().get(0) assert(schemaUpdate.before == fieldMappingBefore.get("struct")) assert(schemaUpdate.after == fieldMappingAfter.get("struct")) } test("Compute schema changes with field metadata changes") { val fieldMappingBefore = newSchema( ( 1, new StructField( "id", IntegerType.INTEGER, true, FieldMetadata.builder().putString( "metadata_col", "metadata_val").build()))) val fieldMappingAfter = newSchema( ( 1, new StructField( "id", IntegerType.INTEGER, true, FieldMetadata.builder().putString( "metadata_col", "updated_metadata_val").build()))) val schemaChanges = computeSchemaChangesById(fieldMappingBefore, fieldMappingAfter) assert(schemaChanges.addedFields().isEmpty) assert(schemaChanges.removedFields().isEmpty) assert(schemaChanges.updatedFields().size() == 1) val schemaUpdate = schemaChanges.updatedFields().get(0) assert(schemaUpdate.before == fieldMappingBefore.get("id")) assert(schemaUpdate.after == fieldMappingAfter.get("id")) } test("Compute schema changes fails for invalid field moves - basic") { val map_field_type = new MapType( new StructType() .add("c2", StringType.STRING, fieldMetadata(1)) .add( "c3", new StructType() .add("c4", IntegerType.INTEGER, fieldMetadata(3)), fieldMetadata(2)), new StructType() .add("c5", IntegerType.INTEGER, fieldMetadata(4)), true /* valueContainsNull */ ) val array_field_type = new ArrayType( new ArrayType( new StructType() .add("c6", IntegerType.INTEGER, fieldMetadata(6)) .add("c7", IntegerType.INTEGER, fieldMetadata(7)), true /* containsNull */ ), true /* containsNull */ ) val beforeSchema = new StructType() .add("nested_map", map_field_type, fieldMetadata(0)) .add("nested_array", array_field_type, fieldMetadata(5)) .add("c8", StringType.STRING, fieldMetadata(8)) Seq( // New struct column with existing field under it new StructType() .add( "nested_struct", new StructType() .add("c8", StringType.STRING, fieldMetadata(8)), fieldMetadata(100)), // Un-nest a nested struct all within a Map key new StructType() .add( "nested_map", new MapType( new StructType() .add("c2", StringType.STRING, fieldMetadata(1)) .add("c4", IntegerType.INTEGER, fieldMetadata(3)), map_field_type.getValueType, true /* valueContainsNull */ ), fieldMetadata(0)), // Nested column within Map key moved to top-level new StructType() .add("c2", StringType.STRING, fieldMetadata(1)), // Move struct field from key to value within map type new StructType() .add( "nested_map", new MapType( new StructType() .add( "c3", new StructType() .add("c4", IntegerType.INTEGER, fieldMetadata(3)), fieldMetadata(2)), new StructType() .add("c5", IntegerType.INTEGER, fieldMetadata(4)) .add("c2", StringType.STRING, fieldMetadata(1)), true /* valueContainsNull */ ), fieldMetadata(0)), // Move existing field into double-nested array new StructType() .add( "nested_array", new ArrayType( new ArrayType( new StructType() .add("c8", StringType.STRING, fieldMetadata(8)) .add("c6", IntegerType.INTEGER, fieldMetadata(6)) .add("c7", IntegerType.INTEGER, fieldMetadata(7)), true /* containsNull */ ), true /* containsNull */ ), fieldMetadata(5)), // Move field out of double-nested array new StructType() .add( "nested_array", new ArrayType( new ArrayType( new StructType() .add("c7", IntegerType.INTEGER, fieldMetadata(7)), true /* containsNull */ ), true /* containsNull */ ), fieldMetadata(5)).add("c6", IntegerType.INTEGER, fieldMetadata(6))).foreach { afterSchema => val e = intercept[KernelException] { computeSchemaChangesById(beforeSchema, afterSchema) } assert(e.getMessage.contains("Cannot move fields between different levels of nesting")) } } test("Compute schema changes fails for invalid field moves - map value operations") { val map_field_type = new MapType( new StructType() .add("k1", StringType.STRING, fieldMetadata(1)), new StructType() .add("v1", IntegerType.INTEGER, fieldMetadata(2)) .add( "v2", new StructType() .add("v3", StringType.STRING, fieldMetadata(4)), fieldMetadata(3)), true /* valueContainsNull */ ) val beforeSchema = new StructType() .add("mymap", map_field_type, fieldMetadata(0)) .add("c1", StringType.STRING, fieldMetadata(5)) Seq( // Move field from Map value to top-level new StructType() .add( "mymap", new MapType( map_field_type.getKeyType, new StructType() .add( "v2", new StructType() .add("v3", StringType.STRING, fieldMetadata(4)), fieldMetadata(3)), true /* valueContainsNull */ ), fieldMetadata(0)) .add("v1", IntegerType.INTEGER, fieldMetadata(2)) .add("c1", StringType.STRING, fieldMetadata(5)), // Move field from Map value to Map key new StructType() .add( "mymap", new MapType( new StructType() .add("k1", StringType.STRING, fieldMetadata(1)) .add("v1", IntegerType.INTEGER, fieldMetadata(2)), new StructType() .add( "v2", new StructType() .add("v3", StringType.STRING, fieldMetadata(4)), fieldMetadata(3)), true /* valueContainsNull */ ), fieldMetadata(0)) .add("c1", StringType.STRING, fieldMetadata(5)), // Un-nest a nested struct within Map value new StructType() .add( "mymap", new MapType( map_field_type.getKeyType, new StructType() .add("v1", IntegerType.INTEGER, fieldMetadata(2)) .add("v3", StringType.STRING, fieldMetadata(4)), true /* valueContainsNull */ ), fieldMetadata(0)) .add("c1", StringType.STRING, fieldMetadata(5)), // Move deeply nested field from Map value to top-level new StructType() .add( "mymap", new MapType( map_field_type.getKeyType, new StructType() .add("v1", IntegerType.INTEGER, fieldMetadata(2)) .add( "v2", new StructType(), fieldMetadata(3)), true /* valueContainsNull */ ), fieldMetadata(0)) .add("c1", StringType.STRING, fieldMetadata(5)) .add("v3", StringType.STRING, fieldMetadata(4))).foreach { afterSchema => val e = intercept[KernelException] { computeSchemaChangesById(beforeSchema, afterSchema) } assert(e.getMessage.contains("Cannot move fields between different levels of nesting")) } } test("Compute schema changes fails for invalid field moves - between sibling structs") { val beforeSchema = new StructType() .add( "struct1", new StructType() .add("a", IntegerType.INTEGER, fieldMetadata(1)) .add("b", StringType.STRING, fieldMetadata(2)), fieldMetadata(0)) .add( "struct2", new StructType() .add("c", IntegerType.INTEGER, fieldMetadata(4)) .add("d", StringType.STRING, fieldMetadata(5)), fieldMetadata(3)) .add("e", IntegerType.INTEGER, fieldMetadata(6)) Seq( // Move field from struct1 to struct2 new StructType() .add( "struct1", new StructType() .add("b", StringType.STRING, fieldMetadata(2)), fieldMetadata(0)) .add( "struct2", new StructType() .add("a", IntegerType.INTEGER, fieldMetadata(1)) .add("c", IntegerType.INTEGER, fieldMetadata(4)) .add("d", StringType.STRING, fieldMetadata(5)), fieldMetadata(3)) .add("e", IntegerType.INTEGER, fieldMetadata(6)), // Move top-level field into a struct new StructType() .add( "struct1", new StructType() .add("a", IntegerType.INTEGER, fieldMetadata(1)) .add("b", StringType.STRING, fieldMetadata(2)) .add("e", IntegerType.INTEGER, fieldMetadata(6)), fieldMetadata(0)) .add( "struct2", new StructType() .add("c", IntegerType.INTEGER, fieldMetadata(4)) .add("d", StringType.STRING, fieldMetadata(5)), fieldMetadata(3))).foreach { afterSchema => val e = intercept[KernelException] { computeSchemaChangesById(beforeSchema, afterSchema) } assert(e.getMessage.contains("Cannot move fields between different levels of nesting")) } } test("Compute schema changes fails for invalid field moves - deeply nested structures") { val beforeSchema = new StructType() .add( "level1", new StructType() .add( "level2", new StructType() .add( "level3", new StructType() .add("deep_field", StringType.STRING, fieldMetadata(3)), fieldMetadata(2)), fieldMetadata(1)), fieldMetadata(0)) .add("top_field", IntegerType.INTEGER, fieldMetadata(4)) Seq( // Move deeply nested field to top-level new StructType() .add( "level1", new StructType() .add( "level2", new StructType() .add( "level3", new StructType(), fieldMetadata(2)), fieldMetadata(1)), fieldMetadata(0)) .add("top_field", IntegerType.INTEGER, fieldMetadata(4)) .add("deep_field", StringType.STRING, fieldMetadata(3)), // Move deeply nested field to level2 new StructType() .add( "level1", new StructType() .add( "level2", new StructType() .add( "level3", new StructType(), fieldMetadata(2)) .add("deep_field", StringType.STRING, fieldMetadata(3)), fieldMetadata(1)), fieldMetadata(0)) .add("top_field", IntegerType.INTEGER, fieldMetadata(4)), // Move deeply nested field to level1 new StructType() .add( "level1", new StructType() .add( "level2", new StructType() .add( "level3", new StructType(), fieldMetadata(2)), fieldMetadata(1)) .add("deep_field", StringType.STRING, fieldMetadata(3)), fieldMetadata(0)) .add("top_field", IntegerType.INTEGER, fieldMetadata(4)), // Move top-level field deep into structure new StructType() .add( "level1", new StructType() .add( "level2", new StructType() .add( "level3", new StructType() .add("deep_field", StringType.STRING, fieldMetadata(3)) .add("top_field", IntegerType.INTEGER, fieldMetadata(4)), fieldMetadata(2)), fieldMetadata(1)), fieldMetadata(0))).foreach { afterSchema => val e = intercept[KernelException] { computeSchemaChangesById(beforeSchema, afterSchema) } assert(e.getMessage.contains("Cannot move fields between different levels of nesting")) } } test("Compute schema changes fails for invalid field moves - array and struct combinations") { val beforeSchema = new StructType() .add( "my_array", new ArrayType( new StructType() .add("arr_field1", IntegerType.INTEGER, fieldMetadata(1)) .add("arr_field2", StringType.STRING, fieldMetadata(2)), true /* containsNull */ ), fieldMetadata(0)) .add( "my_struct", new StructType() .add("struct_field", IntegerType.INTEGER, fieldMetadata(4)), fieldMetadata(3)) .add("top_field", StringType.STRING, fieldMetadata(5)) Seq( // Move field from array element to regular struct new StructType() .add( "my_array", new ArrayType( new StructType() .add("arr_field2", StringType.STRING, fieldMetadata(2)), true /* containsNull */ ), fieldMetadata(0)) .add( "my_struct", new StructType() .add("struct_field", IntegerType.INTEGER, fieldMetadata(4)) .add("arr_field1", IntegerType.INTEGER, fieldMetadata(1)), fieldMetadata(3)) .add("top_field", StringType.STRING, fieldMetadata(5)), // Move field from regular struct to array element new StructType() .add( "my_array", new ArrayType( new StructType() .add("arr_field1", IntegerType.INTEGER, fieldMetadata(1)) .add("arr_field2", StringType.STRING, fieldMetadata(2)) .add("struct_field", IntegerType.INTEGER, fieldMetadata(4)), true /* containsNull */ ), fieldMetadata(0)) .add( "my_struct", new StructType(), fieldMetadata(3)) .add("top_field", StringType.STRING, fieldMetadata(5)), // Move field from array element to top-level new StructType() .add( "my_array", new ArrayType( new StructType() .add("arr_field2", StringType.STRING, fieldMetadata(2)), true /* containsNull */ ), fieldMetadata(0)) .add( "my_struct", new StructType() .add("struct_field", IntegerType.INTEGER, fieldMetadata(4)), fieldMetadata(3)) .add("top_field", StringType.STRING, fieldMetadata(5)) .add("arr_field1", IntegerType.INTEGER, fieldMetadata(1))).foreach { afterSchema => val e = intercept[KernelException] { computeSchemaChangesById(beforeSchema, afterSchema) } assert(e.getMessage.contains("Cannot move fields between different levels of nesting")) } } test("Compute schema changes fails for invalid field moves - complex nested maps") { val beforeSchema = new StructType() .add( "outer_map", new MapType( StringType.STRING, new MapType( StringType.STRING, new StructType() .add("inner_field", IntegerType.INTEGER, fieldMetadata(2)), true /* valueContainsNull */ ), true /* valueContainsNull */ ), fieldMetadata(0)) .add("other_field", StringType.STRING, fieldMetadata(3)) Seq( // Move field from nested map value to outer map value new StructType() .add( "outer_map", new MapType( StringType.STRING, new MapType( StringType.STRING, new StructType(), true /* valueContainsNull */ ), true /* valueContainsNull */ ), fieldMetadata(0)) .add("other_field", StringType.STRING, fieldMetadata(3)) .add("inner_field", IntegerType.INTEGER, fieldMetadata(2)), // Move top-level field into nested map value new StructType() .add( "outer_map", new MapType( StringType.STRING, new MapType( StringType.STRING, new StructType() .add("inner_field", IntegerType.INTEGER, fieldMetadata(2)) .add("other_field", StringType.STRING, fieldMetadata(3)), true /* valueContainsNull */ ), true /* valueContainsNull */ ), fieldMetadata(0))).foreach { afterSchema => val e = intercept[KernelException] { computeSchemaChangesById(beforeSchema, afterSchema) } assert(e.getMessage.contains("Cannot move fields between different levels of nesting")) } } /////////////////////////////////////////////////////////////////////////// // validateUpdatedSchema /////////////////////////////////////////////////////////////////////////// test("validateUpdatedSchema fails when column mapping is disabled") { val current = new StructType().add(new StructField("id", IntegerType.INTEGER, true)) val updated = current.add(new StructField("data", StringType.STRING, true)) val e = intercept[IllegalArgumentException] { val tblProperties = Map(COLUMN_MAPPING_MODE_KEY -> "none") validateUpdatedSchemaAndGetUpdatedSchema( metadata(current, properties = tblProperties), metadata(updated, properties = tblProperties), dummyProtocol, emptySet(), false // allowNewRequiredFields ) } assert(e.getMessage == "Cannot validate updated schema when column mapping is disabled") } private val primitiveSchema = new StructType() .add( "id", IntegerType.INTEGER, true, fieldMetadata(id = 1, physicalName = "id")) private val mapWithStructKey = new StructType() .add( "map", new MapType( new StructType() .add("id", IntegerType.INTEGER, true, fieldMetadata(id = 2, physicalName = "id")), IntegerType.INTEGER, false), true, fieldMetadata(id = 1, physicalName = "map")) private val structWithArrayOfStructs = new StructType() .add( "top_level_struct", new StructType().add( "array", new ArrayType( new StructType().add("id", IntegerType.INTEGER, true, fieldMetadata(4, "id")), false), false, fieldMetadata(2, "array_field")), fieldMetadata(1, "top_level_struct")) private val updatedSchemasWithInconsistentPhysicalNames = Table( ("schemaBefore", "updatedSchemaWithInconsistentPhysicalNames"), // Top level primitive has inconsistent physical name ( primitiveSchema, new StructType() .add( "renamed_id", IntegerType.INTEGER, true, fieldMetadata(id = 1, physicalName = "inconsistent_name"))), // Map with struct key has inconsistent physical name ( mapWithStructKey, new StructType() .add( "map", new MapType( new StructType() .add( "renamed_id", IntegerType.INTEGER, false, fieldMetadata(id = 2, physicalName = "inconsistent_name")), IntegerType.INTEGER, false), true, fieldMetadata(id = 1, physicalName = "map"))), // Struct with array of struct field where inner struct field has inconsistent physical name ( structWithArrayOfStructs, structWithArrayOfStructs(arrayType = new ArrayType( new StructType().add( "renamed_id", IntegerType.INTEGER, fieldMetadata(4, "inconsistent_name")), false)))) test("validateUpdatedSchema fails when physical names are not consistent across ids") { assertSchemaEvolutionFailure[IllegalArgumentException]( updatedSchemasWithInconsistentPhysicalNames, "Existing field with id .* in current schema" + " has physical name id which is different from inconsistent_name") } private val updatedSchemasMissingId = Table( ("schemaBefore", "updatedSchemaWithMissingId"), // Top level primitive missing field ID ( primitiveSchema, new StructType() .add( "renamed_id", IntegerType.INTEGER, true, FieldMetadata.builder().putString(COLUMN_MAPPING_PHYSICAL_NAME_KEY, "id").build())), // Map with struct key missing field ID ( mapWithStructKey, mapWithStructKey(mapType = new MapType( new StructType() .add( "renamed_id", IntegerType.INTEGER, false, FieldMetadata.builder().putString(COLUMN_MAPPING_PHYSICAL_NAME_KEY, "id").build()), IntegerType.INTEGER, false))), // Struct with array of struct field where inner struct is missing ID ( structWithArrayOfStructs, structWithArrayOfStructs(new ArrayType( new StructType().add( "renamed_id", IntegerType.INTEGER, FieldMetadata.builder() .putString(COLUMN_MAPPING_PHYSICAL_NAME_KEY, "id").build()), false)))) test("validateUpdatedSchema fails when field is missing ID") { assertSchemaEvolutionFailure[IllegalArgumentException]( updatedSchemasMissingId, "Field renamed_id is missing column id") } private val updatedSchemasMissingPhysicalName = Table( ("schemaBefore", "updatedSchemaWithMissingPhysicalName"), // Top level primitive missing physical name ( primitiveSchema, new StructType() .add( "renamed_id", IntegerType.INTEGER, true, FieldMetadata.builder().putLong(COLUMN_MAPPING_ID_KEY, 1).build())), // Map with struct key missing physical name ( mapWithStructKey, mapWithStructKey(mapType = new MapType( new StructType() .add( "renamed_id", IntegerType.INTEGER, false, FieldMetadata.builder().putLong(COLUMN_MAPPING_ID_KEY, 1).build()), IntegerType.INTEGER, false))), // Struct with array of struct field where inner struct is missing physical name ( structWithArrayOfStructs, structWithArrayOfStructs(arrayType = new ArrayType( new StructType().add( "renamed_id", IntegerType.INTEGER, FieldMetadata.builder() .putLong(COLUMN_MAPPING_ID_KEY, 4L).build()), false)))) test("validateUpdatedSchema fails when field is missing physical name") { assertSchemaEvolutionFailure[IllegalArgumentException]( updatedSchemasMissingPhysicalName, "Field renamed_id is missing physical name") } private val updatedSchemaHasDuplicateColumnId = Table( ("schemaBefore", "updatedSchemaWithMissingPhysicalName"), // Top level primitive has duplicate id ( primitiveSchema, primitiveSchema .add( "duplicate_id", IntegerType.INTEGER, true, fieldMetadata(id = 1, physicalName = "duplicate_id"))), // Map with struct key adds duplicate field ( mapWithStructKey, mapWithStructKey .add( "duplicate_id", IntegerType.INTEGER, fieldMetadata(id = 2, physicalName = "duplicate_id"))), // Struct with array of struct field where field with duplicate ID is added ( structWithArrayOfStructs, structWithArrayOfStructs .add( "duplicate_id", IntegerType.INTEGER, fieldMetadata(4, "duplicate_id")))) test("validateUpdatedSchema fails with schema with duplicate column ID") { forAll(updatedSchemaHasDuplicateColumnId) { (schemaBefore, schemaAfter) => val e = intercept[IllegalArgumentException] { validateUpdatedSchemaAndGetUpdatedSchema( metadata(schemaBefore), metadata(schemaAfter), dummyProtocol, emptySet(), false /* allowNewRequiredFields */ ) } assert(e.getMessage.matches("Field duplicate_id with id .* already exists")) } } private val validUpdatedSchemas = Table( ("schemaBefore", "updatedSchemaWithRenamedColumns"), // Top level primitive missing physical name ( primitiveSchema, new StructType() .add( "renamed_id", IntegerType.INTEGER, true, fieldMetadata(id = 1, physicalName = "id"))), // Map with struct key renamed ( mapWithStructKey, mapWithStructKey( new MapType( new StructType() .add( "renamed_id", IntegerType.INTEGER, true, fieldMetadata(id = 2, physicalName = "id")), IntegerType.INTEGER, false), fieldMetadata(id = 1, physicalName = "map"))), // Struct with array of struct field where inner struct field is renamed ( structWithArrayOfStructs, new StructType() .add( "top_level_struct", new StructType().add( "array", new ArrayType( new StructType().add("renamed_id", IntegerType.INTEGER, fieldMetadata(4, "id")), false), false, fieldMetadata(2, "array_field")), fieldMetadata(1, "top_level_struct")))) test("validateUpdatedSchema succeeds with valid ID and physical name") { forAll(validUpdatedSchemas) { (schemaBefore, schemaAfter) => validateUpdatedSchemaAndGetUpdatedSchema( metadata(schemaBefore), metadata(schemaAfter), dummyProtocol, emptySet(), false /* allowNewRequiredFields */ ) } } private val nonNullableFieldAdded = Table( ("schemaBefore", "schemaWithNonNullableFieldAdded"), ( primitiveSchema, primitiveSchema.add( "required_field", IntegerType.INTEGER, false, fieldMetadata(3, "required_field"))), // Map with struct key where non-nullable field is added to struct ( mapWithStructKey, mapWithStructKey( new MapType( new StructType() .add( "renamed_id", IntegerType.INTEGER, true, fieldMetadata(id = 2, physicalName = "id")) .add( "required_field", IntegerType.INTEGER, false, fieldMetadata(id = 3, physicalName = "required_field")), IntegerType.INTEGER, false), fieldMetadata(id = 1, physicalName = "map"))), // Struct of array of structs where non-nullable field is added to struct ( structWithArrayOfStructs, structWithArrayOfStructs( arrayType = new ArrayType( new StructType().add( "renamed_id", IntegerType.INTEGER, fieldMetadata(4, "id")) .add("required_field", IntegerType.INTEGER, false, fieldMetadata(5, "required_field")), false), arrayName = "renamed_array"))) test("validateUpdatedSchema fails when non-nullable field is added with " + "allowNewRequiredFields=false") { assertSchemaEvolutionFailure[KernelException]( nonNullableFieldAdded, "Cannot add non-nullable field required_field", allowNewRequiredFields = false) } test("validateUpdatedSchema succeeds when non-nullable field is added with " + "allowNewRequiredFields=true") { forAll(nonNullableFieldAdded) { (schemaBefore, schemaAfter) => validateUpdatedSchemaAndGetUpdatedSchema( metadata(schemaBefore), metadata(schemaAfter), dummyProtocol, emptySet(), true /* allowNewRequiredFields */ ) } } private val existingFieldNullabilityTightened = Table( ("schemaBefore", "schemaWithFieldNullabilityTightened"), ( primitiveSchema, new StructType() .add( "id", IntegerType.INTEGER, false, fieldMetadata(id = 1, physicalName = "id"))), // Map with struct key where existing id field has nullability tightened ( mapWithStructKey, mapWithStructKey( new MapType( new StructType() .add( "renamed_id", IntegerType.INTEGER, false, fieldMetadata(id = 2, physicalName = "id")), IntegerType.INTEGER, false), fieldMetadata(id = 1, physicalName = "map"))), // Struct of array of structs where id field in inner struct has nullability tightened ( structWithArrayOfStructs, structWithArrayOfStructs( arrayType = new ArrayType( new StructType().add( "renamed_id", IntegerType.INTEGER, false, fieldMetadata(4, "id")), false), arrayName = "renamed_array"))) test("validateUpdatedSchema fails when existing nullability is tightened with " + "allowNewRequiredFields=false") { assertSchemaEvolutionFailure[KernelException]( existingFieldNullabilityTightened, "Cannot tighten the nullability of existing field .*id") } test("validateUpdatedSchema fails when existing nullability is tightened with " + "allowNewRequiredFields=true") { assertSchemaEvolutionFailure[KernelException]( existingFieldNullabilityTightened, "Cannot tighten the nullability of existing field .*id", allowNewRequiredFields = true) } private val invalidTypeChange = Table( ("schemaBefore", "schemaWithInvalidTypeChange"), ( primitiveSchema, new StructType() .add( "id", StringType.STRING, true, fieldMetadata(id = 1, physicalName = "id"))), // Map with struct key where id is changed to string ( mapWithStructKey, mapWithStructKey( new MapType( new StructType() .add( "renamed_id_as_string", StringType.STRING, true, fieldMetadata(id = 2, physicalName = "id")), IntegerType.INTEGER, false), fieldMetadata(id = 1, physicalName = "map"))), // Struct of array of struct where inner id field is changed to string ( structWithArrayOfStructs, structWithArrayOfStructs( arrayType = new ArrayType( new StructType().add( "renamed_id_as_string", StringType.STRING, true, fieldMetadata(4, "id")), false), arrayName = "renamed_array"))) test("validateUpdatedSchema fails when invalid type change is performed") { assertSchemaEvolutionFailure[KernelException]( invalidTypeChange, "Cannot change the type of existing field id from integer to string") } private val invalidTypeChangesNested = Table( ("schemaBefore", "schemaWithInvalidTypeChange"), // Array to Map ( newSchema(( 1, new StructField( "array", new ArrayType( IntegerType.INTEGER, false), true))), newSchema(( 1, new StructField( "to_map", new MapType( IntegerType.INTEGER, IntegerType.INTEGER, false), true)))), // Array to Map ( newSchema(( 1, new StructField( "map", new MapType( IntegerType.INTEGER, IntegerType.INTEGER, false), true))), newSchema(( 1, new StructField( "to_array", new ArrayType( IntegerType.INTEGER, false), true)))), // nested array change ( newSchema(( 1, new StructField( "array", new ArrayType( new ArrayType(IntegerType.INTEGER, false), false), true))), newSchema(( 1, new StructField( "to_map", new ArrayType( new MapType(IntegerType.INTEGER, IntegerType.INTEGER, false), false), true)))), // nested map change ( newSchema(( 1, new StructField( "map", new MapType( IntegerType.INTEGER, new ArrayType(IntegerType.INTEGER, false), false), true))), newSchema(( 1, new StructField( "to_nested_array_to_primitive", new MapType( IntegerType.INTEGER, IntegerType.INTEGER, false), false))))) test("validateUpdatedSchema fails when invalid type change is performed on nested fields") { assertSchemaEvolutionFailure[KernelException]( invalidTypeChangesNested, "Cannot change the type of existing field.*") } private val validateAddedFields = Table( ("schemaBefore", "schemaWithAddedField"), ( primitiveSchema, primitiveSchema .add( "data", StringType.STRING, true, fieldMetadata(id = 2, physicalName = "data"))), // Map with struct key where data field is added ( mapWithStructKey, mapWithStructKey( new MapType( new StructType() .add( "renamed_id", IntegerType.INTEGER, true, fieldMetadata(id = 2, physicalName = "id")) .add( "data", StringType.STRING, true, fieldMetadata(id = 3, physicalName = "data")), IntegerType.INTEGER, false), fieldMetadata(id = 1, physicalName = "map"))), // Struct of array of struct where inner struct has added string data field ( structWithArrayOfStructs, structWithArrayOfStructs( arrayType = new ArrayType( new StructType().add( "renamed_id", IntegerType.INTEGER, true, fieldMetadata(4, "id")) .add( "data", StringType.STRING, true, fieldMetadata(5, "data")), false), arrayName = "renamed_array"))) test("validateUpdatedSchema succeeds when adding field") { forAll(validateAddedFields) { (schemaBefore, schemaAfter) => validateUpdatedSchemaAndGetUpdatedSchema( metadata(schemaBefore), metadata(schemaAfter), dummyProtocol, emptySet(), false /* allowNewRequiredFields */ ) } } private val validateMetadataChange = Table( ("schemaBefore", "schemaWithAddedField"), // Adding column comment to id column ( primitiveSchema, new StructType() .add( "id", IntegerType.INTEGER, true, FieldMetadata.builder().fromMetadata(fieldMetadata( id = 1, physicalName = "id")).putString("comment", "id comment").build())), // Struct of array of struct where inner struct has added column comment to metadata ( structWithArrayOfStructs, structWithArrayOfStructs( arrayType = new ArrayType( new StructType().add( "id", IntegerType.INTEGER, true, FieldMetadata.builder().fromMetadata(fieldMetadata(4, "id")).putString( "comment", "id comment").build()), false), arrayName = "renamed_array"))) test("validateUpdatedSchema succeeds when updating field metadata") { forAll(validateMetadataChange) { (schemaBefore, schemaAfter) => validateUpdatedSchemaAndGetUpdatedSchema( metadata(schemaBefore), metadata(schemaAfter), dummyProtocol, emptySet(), false /* allowNewRequiredFields */ ) } } private val underMaxColIdFieldAdded = Table( ("schemaBefore", "schemaWithUnderMaxFieldIdAdded"), ( primitiveSchema, primitiveSchema.add( "too_low_field_id_field", IntegerType.INTEGER, true, fieldMetadata(0, "too_low_field_id_field"))), // Map with struct key where under max col-id field is added ( mapWithStructKey, mapWithStructKey( new MapType( new StructType() .add( "renamed_id", IntegerType.INTEGER, true, fieldMetadata(id = 2, physicalName = "id")) .add( "too_low_field_id_field", IntegerType.INTEGER, true, fieldMetadata(id = 0, physicalName = "too_low_field_id_field")), IntegerType.INTEGER, true), fieldMetadata(id = 1, physicalName = "map"))), // Struct of array of structs where under max col-id field is added ( structWithArrayOfStructs, structWithArrayOfStructs( arrayType = new ArrayType( new StructType().add( "renamed_id", IntegerType.INTEGER, fieldMetadata(4, "id")) .add( "too_low_field_id_field", IntegerType.INTEGER, true, fieldMetadata(0, "too_low_field_id_field")), true), arrayName = "renamed_array"))) test("validateUpdatedSchema fails when a new field with a fieldId < maxColId is added") { assertSchemaEvolutionFailure[IllegalArgumentException]( underMaxColIdFieldAdded, "Cannot add a new column with a fieldId <= maxFieldId. Found field: .* with" + "fieldId=0. Current maxFieldId in the table is: .*") } private def mapWithStructKey( mapType: DataType = mapWithStructKey.get("map").getDataType, mapFieldMetadata: FieldMetadata = mapWithStructKey.get("map").getMetadata): StructType = { new StructType() .add( "map", mapType, true, mapFieldMetadata) } private def structWithArrayOfStructs( arrayType: ArrayType = structWithArrayOfStructs .get("top_level_struct") .getDataType.asInstanceOf[StructType] .get("array").getDataType.asInstanceOf[ArrayType], arrayMetadata: FieldMetadata = structWithArrayOfStructs .get("top_level_struct") .getDataType.asInstanceOf[StructType] .get("array").getMetadata, arrayName: String = "array") = { new StructType() .add( "top_level_struct", new StructType().add( arrayName, arrayType, false, arrayMetadata), fieldMetadata(1, "top_level_struct")) } private def assertSchemaEvolutionFailure[T <: Throwable]( evolutionCases: TableFor2[StructType, StructType], expectedMessage: String, tableProperties: Map[String, String] = Map(ColumnMapping.COLUMN_MAPPING_MODE_KEY -> "id"), allowNewRequiredFields: Boolean = false)(implicit classTag: ClassTag[T]) { forAll(evolutionCases) { (schemaBefore, schemaAfter) => val e = intercept[T] { validateUpdatedSchemaAndGetUpdatedSchema( metadata(schemaBefore, tableProperties), metadata(schemaAfter, tableProperties), dummyProtocol, emptySet(), allowNewRequiredFields) } assert(e.getMessage.matches(expectedMessage), s"${e.getMessage} ~= $expectedMessage") } } private def metadata( schema: StructType, properties: Map[String, String] = Map(ColumnMapping.COLUMN_MAPPING_MODE_KEY -> "id")): Metadata = { val tblProperties = properties ++ Map(ColumnMapping.COLUMN_MAPPING_MAX_COLUMN_ID_KEY -> ColumnMapping.findMaxColumnId(schema).toString) new Metadata( "id", Optional.empty(), /* name */ Optional.empty(), /* description */ new Format(), DataTypeJsonSerDe.serializeDataType(schema), schema, VectorUtils.buildArrayValue(util.Arrays.asList(), StringType.STRING), // partitionColumns Optional.empty(), // createdTime stringStringMapValue(tblProperties.asJava) // configurationMap ) } private def fieldMetadata(id: Integer): FieldMetadata = { fieldMetadata(id, s"col-$id") } private def fieldMetadata(id: Integer, physicalName: String): FieldMetadata = { FieldMetadata.builder() .putLong(COLUMN_MAPPING_ID_KEY, id.longValue()) .putString(COLUMN_MAPPING_PHYSICAL_NAME_KEY, physicalName) .build() } private def newSchema(tuple: (Int, StructField)*): StructType = { val fields = tuple.map { case (id, field) => addFieldId(id, field) } val schemaWithIds = new StructType(fields.asJava) schemaWithIds } private def addFieldId(id: Int, field: StructField) = { val metadataWithFieldIds = FieldMetadata.builder().fromMetadata(field.getMetadata) .putLong("delta.columnMapping.id", id) .putString("delta.columnMapping.physicalName", s"col-$id") .build() field.withNewMetadata(metadataWithFieldIds) } /////////////////////////////////////////////////////////////////////////// // Type Changes Tests /////////////////////////////////////////////////////////////////////////// test("Compute schema changes adds schema change") { val currentSchema = newSchema((1, new StructField("id", IntegerType.INTEGER, true))) val updatedSchema = newSchema((1, new StructField("id", LongType.LONG, true))) val schemaChanges = computeSchemaChangesById(currentSchema, updatedSchema) // Verify that we have one updated field assert(schemaChanges.updatedFields().size() == 1) val schemaUpdate = schemaChanges.updatedFields().get(0) // Verify that the updated field has the expected before and after values assert(schemaUpdate.before == currentSchema.get("id")) assert(schemaUpdate.after == updatedSchema.get("id")) // Verify that the updated schema has a TypeChange recorded assert(schemaChanges.updatedSchema().isPresent) val updatedField = schemaChanges.updatedSchema().get().get("id") val typeChanges = updatedField.getTypeChanges assert(typeChanges.size() == 1) val typeChange = typeChanges.get(0) assert(typeChange.getFrom == IntegerType.INTEGER) assert(typeChange.getTo == LongType.LONG) } test("Type changes are preserved across schema updates") { // Create a field with integer type and an existing type change from byte to int val byteType = ByteType.BYTE val intType = IntegerType.INTEGER val existingTypeChange = new TypeChange(byteType, intType) val fieldWithTypeChange = new StructField( "id", intType, true, fieldMetadata(1, "id")).withTypeChanges(List(existingTypeChange).asJava) val currentSchema = new StructType(List(fieldWithTypeChange).asJava) // Change the type to long without specifying the previous type change val updatedSchema = newSchema((1, new StructField("id", intType, true))) val schemaChanges = computeSchemaChangesById(currentSchema, updatedSchema) // Verify that the updated schema has the TypeChange preserved and a new one added assert(schemaChanges.updatedSchema().isPresent) val updatedField = schemaChanges.updatedSchema().get().get("id") // Check that both type changes are recorded (the original and the new one) val typeChanges = updatedField.getTypeChanges assert(typeChanges.size() == 1) assert(typeChanges.get(0).getFrom == byteType) assert(typeChanges.get(0).getTo == intType) } test("Type changes are preserved across schema updates and new ones are added") { // Create a field with integer type and an existing type change from byte to int val byteType = ByteType.BYTE val intType = IntegerType.INTEGER val longType = LongType.LONG val existingTypeChange = new TypeChange(byteType, intType) val fieldWithTypeChange = new StructField( "id", intType, true, fieldMetadata(1, "id")).withTypeChanges(List(existingTypeChange).asJava) val currentSchema = new StructType(List(fieldWithTypeChange).asJava) // Change the type to long without specifying the previous type change val updatedSchema = newSchema((1, new StructField("id", longType, true))) val schemaChanges = computeSchemaChangesById(currentSchema, updatedSchema) // Verify that the updated schema has the TypeChange preserved and a new one added assert(schemaChanges.updatedSchema().isPresent) val updatedField = schemaChanges.updatedSchema().get().get("id") // Check that both type changes are recorded (the original and the new one) val typeChanges = updatedField.getTypeChanges assert(typeChanges.size() == 2) val typeChange = typeChanges.get(0) assert(typeChange.getFrom == byteType) assert(typeChange.getTo == intType) val newTypeChange = typeChanges.get(1) assert(newTypeChange.getFrom == intType) assert(newTypeChange.getTo == longType) } test("Throws exception when type changes are inconsistent") { // Create a field with integer type and an existing type change from byte to int val byteType = ByteType.BYTE val intType = IntegerType.INTEGER val longType = LongType.LONG val existingTypeChange = new TypeChange(byteType, intType) val fieldWithTypeChange = new StructField( "id", intType, true, fieldMetadata(1, "id")).withTypeChanges(List(existingTypeChange).asJava) val currentSchema = new StructType(List(fieldWithTypeChange).asJava) // Create a new field with a different type change history val differentTypeChange = new TypeChange(longType, intType) val fieldWithDifferentTypeChange = new StructField( "id", intType, true, fieldMetadata(1, "id")).withTypeChanges(List(differentTypeChange).asJava) val updatedSchema = new StructType(List(fieldWithDifferentTypeChange).asJava) // This should throw an exception because the type changes are not equal expectFailure( "detected a modified type changes field at 'id'", "typechange(from=byte,to=integer)", "typechange(from=long,to=integer)") { computeSchemaChangesById(currentSchema, updatedSchema) } } private val validTypeWideningSchemas = Table( ( "schemaBefore", "schemaAfter", "typeWideningEnabled", "icebergWriterCompatV1Enabled", "shouldSucceed"), // Integer widening: Int -> Long (always allowed with type widening) ( primitiveSchema, new StructType().add( "id", LongType.LONG, true, fieldMetadata(id = 1, physicalName = "id")), /* typeWideningEnabled= */ true, /* icebergWriterCompatv1enabled= */ false, /* shouldSucceed= */ true), // Integer widening: Int -> Long (not allowed without type widening) ( primitiveSchema, new StructType().add( "id", LongType.LONG, true, fieldMetadata(id = 1, physicalName = "id")), /* typeWideningEnabled= */ false, /* icebergWriterCompatv1enabled= */ false, /* shouldSucceed= */ false), // Integer to Double widening (allowed with type widening but not with Iceberg compat) ( primitiveSchema, new StructType().add( "id", DoubleType.DOUBLE, true, fieldMetadata(id = 1, physicalName = "id")), /* typeWideningEnabled= */ true, /* icebergWriterCompatv1enabled= */ false, /* shouldSucceed= */ true), // Integer to Double widening (not allowed with Iceberg compat) ( primitiveSchema, new StructType().add( "id", DoubleType.DOUBLE, true, fieldMetadata(id = 1, physicalName = "id")), /* typeWideningEnabled= */ true, /* icebergwritercompatv1enabled= */ true, /* shouldSucceed= */ false), // Invalid type change: Int -> String (never allowed) ( primitiveSchema, new StructType().add( "id", StringType.STRING, true, fieldMetadata(id = 1, physicalName = "id")), /* typeWideningEnabled= */ true, /* typeWideningEnabled= */ false, /* shouldSucceed= */ false)) forAll(validTypeWideningSchemas) { ( schemaBefore, schemaAfter, typeWideningEnabled, icebergWriterCompatV1Enabled, shouldSucceed) => test("validateUpdatedSchema with type widening " + s"$schemaBefore $schemaAfter $typeWideningEnabled $icebergWriterCompatV1Enabled") { val tblProperties = Map( ColumnMapping.COLUMN_MAPPING_MODE_KEY -> "id", TableConfig.TYPE_WIDENING_ENABLED.getKey -> typeWideningEnabled.toString, TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.getKey -> icebergWriterCompatV1Enabled.toString) if (shouldSucceed) { // Should not throw an exception validateUpdatedSchemaAndGetUpdatedSchema( metadata(schemaBefore, tblProperties), metadata(schemaAfter, tblProperties), dummyProtocol, emptySet(), false /* allowNewRequiredFields */ ) } else { // Should throw an exception val e = intercept[KernelException] { validateUpdatedSchemaAndGetUpdatedSchema( metadata(schemaBefore, tblProperties), metadata(schemaAfter, tblProperties), dummyProtocol, emptySet(), false /* allowNewRequiredFields */ ) } assert(e.getMessage.contains("Cannot change the type of existing field")) } } } /////////////////////////////////////////////////////////////////////////// // validateNoMapStructKeyChanges /////////////////////////////////////////////////////////////////////////// private val updatedSchemasWithChangedMaps = Table( ("schemaBefore", "updatedSchemaWithChangedMapKey"), // add a col ( mapWithStructKey, new StructType() .add( "map", new MapType( new StructType() .add("id", IntegerType.INTEGER, true, fieldMetadata(id = 2, physicalName = "id")) .add("id2", IntegerType.INTEGER, true, fieldMetadata(id = 3, physicalName = "id2")), IntegerType.INTEGER, false), true, fieldMetadata(id = 1, physicalName = "map"))), ( new StructType() .add( "map", new MapType( new StructType() .add("id", IntegerType.INTEGER, true, fieldMetadata(id = 2, physicalName = "id")) .add("id2", IntegerType.INTEGER, true, fieldMetadata(id = 3, physicalName = "id2")), IntegerType.INTEGER, false), true, fieldMetadata(id = 1, physicalName = "map")), mapWithStructKey), ( new StructType() .add( "top_level_struct", new StructType().add( "map", new MapType( new StructType() .add("id", IntegerType.INTEGER, true, fieldMetadata(id = 3, physicalName = "id")), IntegerType.INTEGER, false), true, fieldMetadata(2, "map")), fieldMetadata(1, "top_level_struct")), new StructType() .add( "top_level_struct", new StructType().add( "map", new MapType( new StructType() .add("id", IntegerType.INTEGER, true, fieldMetadata(id = 3, physicalName = "id")) .add("id2", IntegerType.INTEGER, true, fieldMetadata(id = 4, physicalName = "id")), IntegerType.INTEGER, false), true, fieldMetadata(2, "map")), fieldMetadata(1, "top_level_struct")))) test("validateNoMapStructKeyChanges fails when map struct changes") { val tblProperties = Map( TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.getKey -> "true", ColumnMapping.COLUMN_MAPPING_MODE_KEY -> "id") assertSchemaEvolutionFailure[KernelException]( updatedSchemasWithChangedMaps, "Cannot change the type key of Map field map from .*", tableProperties = tblProperties) } test("Validate succeeds when adding variant column") { val tableProperties = Map(ColumnMapping.COLUMN_MAPPING_MODE_KEY -> "id") val before = new StructType().add( "id", IntegerType.INTEGER, false, fieldMetadata(id = 1, physicalName = "id")) val schemaWithVariant = before.add( "variant", VariantType.VARIANT, true, fieldMetadata(id = 2, physicalName = "variant")) validateUpdatedSchemaAndGetUpdatedSchema( metadata(before, tableProperties), metadata(schemaWithVariant, tableProperties), dummyProtocol, emptySet(), false) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/TimestampUtilsSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util import java.time.{Instant, LocalDateTime, OffsetDateTime, ZoneOffset} import java.time.temporal.ChronoUnit import org.scalatest.funsuite.AnyFunSuite class TimestampUtilsSuite extends AnyFunSuite { // Expected micros for 9999-12-31T23:59:59Z private val FAR_FUTURE_MICROS = 253402300799000000L test("toEpochMicros(Instant) - epoch") { assert(TimestampUtils.toEpochMicros(Instant.EPOCH) === 0L) } test("toEpochMicros(Instant) - far future timestamp does not overflow") { val instant = Instant.parse("9999-12-31T23:59:59Z") assert(TimestampUtils.toEpochMicros(instant) === FAR_FUTURE_MICROS) } test("toEpochMicros(Instant) - preserves microsecond precision") { // 1000 seconds + 123456 microseconds (123456000 nanoseconds) val instant = Instant.ofEpochSecond(1000, 123456000) assert(TimestampUtils.toEpochMicros(instant) === 1000123456L) } test("toEpochMicros(OffsetDateTime) - epoch") { val dt = OffsetDateTime.ofInstant(Instant.EPOCH, ZoneOffset.UTC) assert(TimestampUtils.toEpochMicros(dt) === 0L) } test("toEpochMicros(OffsetDateTime) - far future timestamp does not overflow") { val dt = OffsetDateTime.parse("9999-12-31T23:59:59Z") assert(TimestampUtils.toEpochMicros(dt) === FAR_FUTURE_MICROS) } test("toEpochMicros(LocalDateTime) - epoch") { val dt = LocalDateTime.ofEpochSecond(0, 0, ZoneOffset.UTC) assert(TimestampUtils.toEpochMicros(dt) === 0L) } test("toEpochMicros(LocalDateTime) - far future timestamp does not overflow") { val dt = LocalDateTime.parse("9999-12-31T23:59:59") assert(TimestampUtils.toEpochMicros(dt) === FAR_FUTURE_MICROS) } test("toEpochMicros(LocalDateTime) - preserves microsecond precision") { // 1000 seconds + 123456 microseconds (123456000 nanoseconds) val dt = LocalDateTime.ofEpochSecond(1000, 123456000, ZoneOffset.UTC) assert(TimestampUtils.toEpochMicros(dt) === 1000123456L) } test("ChronoUnit.MICROS.between() throws for far-future timestamps") { // This test documents why we need TimestampUtils instead of ChronoUnit.MICROS.between(). // ChronoUnit.MICROS.between() internally computes (seconds * 1_000_000_000) / 1000, // where the intermediate nanoseconds value overflows for timestamps beyond ~292 years // from epoch. val farFutureInstant = Instant.parse("9999-12-31T23:59:59Z") // ChronoUnit throws ArithmeticException due to overflow intercept[ArithmeticException] { ChronoUnit.MICROS.between(Instant.EPOCH, farFutureInstant) } // TimestampUtils returns correct value assert(TimestampUtils.toEpochMicros(farFutureInstant) === FAR_FUTURE_MICROS) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/VectorUtilsSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.internal.util import java.lang.{Boolean => BooleanJ, Byte => ByteJ, Double => DoubleJ, Float => FloatJ, Integer => IntegerJ, Long => LongJ, Short => ShortJ} import java.math.BigDecimal import java.sql.{Date, Timestamp} import java.util import scala.collection.JavaConverters._ import io.delta.kernel.data.{ArrayValue, ColumnVector, MapValue, Row} import io.delta.kernel.internal.data.GenericRow import io.delta.kernel.test.VectorTestUtils import io.delta.kernel.types.{ArrayType, BinaryType, BooleanType, ByteType, DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType, MapType, ShortType, StringType, StructType, TimestampNTZType, TimestampType} import org.scalatest.funsuite.AnyFunSuite import org.scalatest.prop.Tables.Table class VectorUtilsSuite extends AnyFunSuite with VectorTestUtils { Table( ("values", "dataType"), (List[ByteJ](1.toByte, 2.toByte, 3.toByte, null), ByteType.BYTE), (List[ShortJ](1.toShort, 2.toShort, 3.toShort, null), ShortType.SHORT), (List[IntegerJ](1, 2, 3, null), IntegerType.INTEGER), (List[LongJ](1L, 2L, 3L, null), LongType.LONG), (List[FloatJ](1.0f, 2.0f, 3.0f, null), FloatType.FLOAT), (List[DoubleJ](1.0, 2.0, 3.0, null), DoubleType.DOUBLE), (List[Array[Byte]]("one".getBytes, "two".getBytes, "three".getBytes, null), BinaryType.BINARY), (List[BooleanJ](true, false, false, null), BooleanType.BOOLEAN), ( List[BigDecimal](new BigDecimal("1"), new BigDecimal("2"), new BigDecimal("3"), null), new DecimalType(10, 2)), (List[String]("one", "two", "three", null), StringType.STRING), ( List[IntegerJ](10, 20, 30, null), DateType.DATE), ( List[LongJ]( Timestamp.valueOf("2023-01-01 00:00:00").getTime, Timestamp.valueOf("2023-01-02 00:00:00").getTime, Timestamp.valueOf("2023-01-03 00:00:00").getTime, null), TimestampType.TIMESTAMP), ( List[LongJ]( Timestamp.valueOf("2023-01-01 00:00:00").getTime, Timestamp.valueOf("2023-01-02 00:00:00").getTime, Timestamp.valueOf("2023-01-03 00:00:00").getTime, null), TimestampNTZType.TIMESTAMP_NTZ)).foreach(testCase => test(s"handle ${testCase._2} array correctly") { val values = testCase._1 val dataType = testCase._2 val columnVector = VectorUtils.buildColumnVector(values.asJava, dataType) assert(columnVector.getSize == 4) dataType match { case ByteType.BYTE => assert(columnVector.getByte(0) == 1.toByte) assert(columnVector.getByte(1) == 2.toByte) assert(columnVector.getByte(2) == 3.toByte) case ShortType.SHORT => assert(columnVector.getShort(0) == 1.toShort) assert(columnVector.getShort(1) == 2.toShort) assert(columnVector.getShort(2) == 3.toShort) case IntegerType.INTEGER => assert(columnVector.getInt(0) == 1) assert(columnVector.getInt(1) == 2) assert(columnVector.getInt(2) == 3) case LongType.LONG => assert(columnVector.getLong(0) == 1L) assert(columnVector.getLong(1) == 2L) assert(columnVector.getLong(2) == 3L) case FloatType.FLOAT => assert(columnVector.getFloat(0) == 1.0f) assert(columnVector.getFloat(1) == 2.0f) assert(columnVector.getFloat(2) == 3.0f) case DoubleType.DOUBLE => assert(columnVector.getDouble(0) == 1.0) assert(columnVector.getDouble(1) == 2.0) assert(columnVector.getDouble(2) == 3.0) case BooleanType.BOOLEAN => assert(columnVector.getBoolean(0)) assert(!columnVector.getBoolean(1)) assert(!columnVector.getBoolean(2)) case _: DecimalType => assert(columnVector.getDecimal(0) == new BigDecimal("1")) assert(columnVector.getDecimal(1) == new BigDecimal("2")) assert(columnVector.getDecimal(2) == new BigDecimal("3")) case BinaryType.BINARY => assert(columnVector.getBinary(0) sameElements "one".getBytes) assert(columnVector.getBinary(1) sameElements "two".getBytes) assert(columnVector.getBinary(2) sameElements "three".getBytes) case StringType.STRING => assert(columnVector.getString(0) == "one") assert(columnVector.getString(1) == "two") assert(columnVector.getString(2) == "three") case DateType.DATE => assert(columnVector.getInt(0) == 10) assert(columnVector.getInt(1) == 20) assert(columnVector.getInt(2) == 30) case TimestampType.TIMESTAMP => assert( columnVector.getLong(0) == Timestamp.valueOf("2023-01-01 00:00:00").getTime) assert( columnVector.getLong(1) == Timestamp.valueOf("2023-01-02 00:00:00").getTime) assert( columnVector.getLong(2) == Timestamp.valueOf("2023-01-03 00:00:00").getTime) case TimestampNTZType.TIMESTAMP_NTZ => assert( columnVector.getLong(0) == Timestamp.valueOf("2023-01-01 00:00:00").getTime) assert( columnVector.getLong(1) == Timestamp.valueOf("2023-01-02 00:00:00").getTime) assert( columnVector.getLong(2) == Timestamp.valueOf("2023-01-03 00:00:00").getTime) } assert(columnVector.isNullAt(3)) }) test(s"handle array of struct correctly") { val structType = new StructType().add("name", StringType.STRING).add("value", IntegerType.INTEGER) val arrayType = new ArrayType(structType, true) def row(name: String, value: Integer): Row = { val map = new util.HashMap[Integer, AnyRef] map.put(0, name) map.put(1, value) new GenericRow(structType, map) } val values = List[ArrayValue]( new ArrayValue { override def getSize: Int = 2 override def getElements: ColumnVector = VectorUtils.buildColumnVector( List[Row]( row("a1", 1), row("a2", 2)).asJava, structType) }, new ArrayValue { override def getSize: Int = 2 override def getElements: ColumnVector = VectorUtils.buildColumnVector( List[Row]( row("b1", 3), row("b2", 4)).asJava, structType) }, new ArrayValue { override def getSize: Int = 2 override def getElements: ColumnVector = VectorUtils.buildColumnVector( List[Row]( row("c1", 5), row("c2", 6)).asJava, structType) }, null) val columnVector = VectorUtils.buildColumnVector(values.asJava, arrayType) // Test size assert(columnVector.getSize == 4) // Test first array val array0 = columnVector.getArray(0) val struct0 = array0.getElements assert(struct0.getSize == 2) val nameVector0 = struct0.getChild(0) val valueVector0 = struct0.getChild(1) assert(nameVector0.getString(0) == "a1") assert(valueVector0.getInt(0) == 1) assert(nameVector0.getString(1) == "a2") assert(valueVector0.getInt(1) == 2) // Test second array val array1 = columnVector.getArray(1) val struct1 = array1.getElements assert(struct1.getSize == 2) val nameVector1 = struct1.getChild(0) val valueVector1 = struct1.getChild(1) assert(nameVector1.getString(0) == "b1") assert(valueVector1.getInt(0) == 3) assert(nameVector1.getString(1) == "b2") assert(valueVector1.getInt(1) == 4) // Test third array val array2 = columnVector.getArray(2) val struct2 = array2.getElements assert(struct2.getSize == 2) val nameVector2 = struct2.getChild(0) val valueVector2 = struct2.getChild(1) assert(nameVector2.getString(0) == "c1") assert(valueVector2.getInt(0) == 5) assert(nameVector2.getString(1) == "c2") assert(valueVector2.getInt(1) == 6) // Test null value assert(columnVector.isNullAt(3)) } test(s"handle array of map correctly") { val mapType = new MapType(StringType.STRING, IntegerType.INTEGER, true) val arrayType = new ArrayType(mapType, true) val values = List[ArrayValue]( new ArrayValue { override def getSize: Int = 2 override def getElements: ColumnVector = VectorUtils.buildColumnVector( List[MapValue]( new MapValue { override def getSize: Int = 2 override def getKeys: ColumnVector = VectorUtils.buildColumnVector(List("a1", "a2").asJava, StringType.STRING) override def getValues: ColumnVector = VectorUtils.buildColumnVector(List[IntegerJ](1, 2).asJava, IntegerType.INTEGER) }, new MapValue { override def getSize: Int = 2 override def getKeys: ColumnVector = VectorUtils.buildColumnVector(List("a3", "a4").asJava, StringType.STRING) override def getValues: ColumnVector = VectorUtils.buildColumnVector(List[IntegerJ](3, 4).asJava, IntegerType.INTEGER) }).asJava, mapType) }, new ArrayValue { override def getSize: Int = 2 override def getElements: ColumnVector = VectorUtils.buildColumnVector( List[MapValue]( new MapValue { override def getSize: Int = 2 override def getKeys: ColumnVector = VectorUtils.buildColumnVector(List("b1", "b2").asJava, StringType.STRING) override def getValues: ColumnVector = VectorUtils.buildColumnVector(List[IntegerJ](5, 6).asJava, IntegerType.INTEGER) }, new MapValue { override def getSize: Int = 2 override def getKeys: ColumnVector = VectorUtils.buildColumnVector(List("b3", "b4").asJava, StringType.STRING) override def getValues: ColumnVector = VectorUtils.buildColumnVector(List[IntegerJ](7, 8).asJava, IntegerType.INTEGER) }).asJava, mapType) }, null) val columnVector = VectorUtils.buildColumnVector(values.asJava, arrayType) // Test size assert(columnVector.getSize == 3) // Test first array val firstArray = columnVector.getArray(0) val firstArrayMaps = firstArray.getElements assert(firstArrayMaps.getSize == 2) val firstArrayFirstMap = firstArrayMaps.getMap(0) assert(firstArrayFirstMap.getKeys.getString(0) == "a1") assert(firstArrayFirstMap.getKeys.getString(1) == "a2") assert(firstArrayFirstMap.getValues.getInt(0) == 1) assert(firstArrayFirstMap.getValues.getInt(1) == 2) val firstArraySecondMap = firstArrayMaps.getMap(1) assert(firstArraySecondMap.getKeys.getString(0) == "a3") assert(firstArraySecondMap.getKeys.getString(1) == "a4") assert(firstArraySecondMap.getValues.getInt(0) == 3) assert(firstArraySecondMap.getValues.getInt(1) == 4) // Test second array val secondArray = columnVector.getArray(1) val secondArrayMaps = secondArray.getElements assert(secondArrayMaps.getSize == 2) val secondArrayFirstMap = secondArrayMaps.getMap(0) assert(secondArrayFirstMap.getKeys.getString(0) == "b1") assert(secondArrayFirstMap.getKeys.getString(1) == "b2") assert(secondArrayFirstMap.getValues.getInt(0) == 5) assert(secondArrayFirstMap.getValues.getInt(1) == 6) val secondArraySecondMap = secondArrayMaps.getMap(1) assert(secondArraySecondMap.getKeys.getString(0) == "b3") assert(secondArraySecondMap.getKeys.getString(1) == "b4") assert(secondArraySecondMap.getValues.getInt(0) == 7) assert(secondArraySecondMap.getValues.getInt(1) == 8) // Test null value assert(columnVector.isNullAt(2)) } test(s"handle array of array correctly") { val innerArrayType = new ArrayType(IntegerType.INTEGER, true) val outerArrayType = new ArrayType(innerArrayType, true) val values = List[ArrayValue]( new ArrayValue { override def getSize: Int = 2 override def getElements: ColumnVector = VectorUtils.buildColumnVector( List[ArrayValue]( new ArrayValue { override def getSize: Int = 2 override def getElements: ColumnVector = VectorUtils.buildColumnVector(List[IntegerJ](1, 2).asJava, IntegerType.INTEGER) }, new ArrayValue { override def getSize: Int = 2 override def getElements: ColumnVector = VectorUtils.buildColumnVector(List[IntegerJ](3, 4).asJava, IntegerType.INTEGER) }).asJava, innerArrayType) }, new ArrayValue { override def getSize: Int = 2 override def getElements: ColumnVector = VectorUtils.buildColumnVector( List[ArrayValue]( new ArrayValue { override def getSize: Int = 2 override def getElements: ColumnVector = VectorUtils.buildColumnVector(List[IntegerJ](5, 6).asJava, IntegerType.INTEGER) }, new ArrayValue { override def getSize: Int = 2 override def getElements: ColumnVector = VectorUtils.buildColumnVector(List[IntegerJ](7, 8).asJava, IntegerType.INTEGER) }).asJava, innerArrayType) }, null) val columnVector = VectorUtils.buildColumnVector(values.asJava, outerArrayType) // Test size assert(columnVector.getSize == 3) // Test first outer array val firstOuterArray = columnVector.getArray(0) val firstOuterArrayElements = firstOuterArray.getElements assert(firstOuterArrayElements.getSize == 2) val firstOuterArrayFirstInner = firstOuterArrayElements.getArray(0) val firstOuterArrayFirstInnerElements = firstOuterArrayFirstInner.getElements assert(firstOuterArrayFirstInnerElements.getInt(0) == 1) assert(firstOuterArrayFirstInnerElements.getInt(1) == 2) val firstOuterArraySecondInner = firstOuterArrayElements.getArray(1) val firstOuterArraySecondInnerElements = firstOuterArraySecondInner.getElements assert(firstOuterArraySecondInnerElements.getInt(0) == 3) assert(firstOuterArraySecondInnerElements.getInt(1) == 4) // Test second outer array val secondOuterArray = columnVector.getArray(1) val secondOuterArrayElements = secondOuterArray.getElements assert(secondOuterArrayElements.getSize == 2) val secondOuterArrayFirstInner = secondOuterArrayElements.getArray(0) val secondOuterArrayFirstInnerElements = secondOuterArrayFirstInner.getElements assert(secondOuterArrayFirstInnerElements.getInt(0) == 5) assert(secondOuterArrayFirstInnerElements.getInt(1) == 6) val secondOuterArraySecondInner = secondOuterArrayElements.getArray(1) val secondOuterArraySecondInnerElements = secondOuterArraySecondInner.getElements assert(secondOuterArraySecondInnerElements.getInt(0) == 7) assert(secondOuterArraySecondInnerElements.getInt(1) == 8) // Test null value assert(columnVector.isNullAt(2)) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/test/ActionUtils.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.test import java.util.{Collections, Optional} import scala.collection.JavaConverters._ import io.delta.kernel.data.{ArrayValue, ColumnVector, MapValue} import io.delta.kernel.internal.actions.{CommitInfo, Format, Metadata, Protocol} import io.delta.kernel.internal.tablefeatures.TableFeatures import io.delta.kernel.types.{IntegerType, StructType} trait ActionUtils extends VectorTestUtils { val protocolWithCatalogManagedSupport: Protocol = new Protocol( TableFeatures.TABLE_FEATURES_MIN_READER_VERSION, TableFeatures.TABLE_FEATURES_MIN_WRITER_VERSION, Set( TableFeatures.CATALOG_MANAGED_RW_FEATURE.featureName()).asJava, Set( TableFeatures.CATALOG_MANAGED_RW_FEATURE.featureName(), TableFeatures.IN_COMMIT_TIMESTAMP_W_FEATURE.featureName()).asJava) val basicPartitionedMetadata = testMetadata( schema = new StructType() .add("part1", IntegerType.INTEGER).add("col1", IntegerType.INTEGER), partitionCols = Seq("part1")) def testCommitInfo(ictEnabled: Boolean = true): CommitInfo = { new CommitInfo( if (ictEnabled) Optional.of(1L) else Optional.empty(), // ICT 1L, // timestamp Optional.of("engineInfo"), Optional.of("operation"), Collections.emptyMap(), // operationParameters Optional.of(false), // isBlindAppend Optional.of("txnId"), Collections.emptyMap() // operationMetrics ) } def testMetadata( schema: StructType, partitionCols: Seq[String] = Seq.empty, tblProps: Map[String, String] = Map.empty): Metadata = { new Metadata( "id", Optional.of("name"), Optional.of("description"), new Format("parquet", Collections.emptyMap()), schema.toJson, schema, new ArrayValue() { // partitionColumns override def getSize: Int = partitionCols.size override def getElements: ColumnVector = stringVector(partitionCols) }, Optional.empty(), new MapValue() { // conf override def getSize: Int = tblProps.size override def getKeys: ColumnVector = stringVector(tblProps.toSeq.map(_._1)) override def getValues: ColumnVector = stringVector(tblProps.toSeq.map(_._2)) }) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/test/MockEngineUtils.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.test import java.io.ByteArrayInputStream import java.util import java.util.Optional import io.delta.kernel.data.{ColumnarBatch, ColumnVector, FilteredColumnarBatch, Row} import io.delta.kernel.engine._ import io.delta.kernel.expressions.{Column, Expression, ExpressionEvaluator, Predicate, PredicateEvaluator} import io.delta.kernel.internal.actions.CommitInfo import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.util.{FileNames, Utils} import io.delta.kernel.types.{DataType, StructType} import io.delta.kernel.utils.{CloseableIterator, DataFileStatus, FileStatus} /** * Contains broiler plate code for mocking [[Engine]] and its sub-interfaces. * * A concrete class is created for each sub-interface (e.g. [[FileSystemClient]]) with * default implementation (unsupported). Test suites can override a specific API(s) * in the sub-interfaces to mock the behavior as desired. * * Example: * {{{ * val myMockFileSystemClient = new BaseMockFileSystemClient() { * override def listFrom(filePath: String): CloseableIterator[FileStatus] = { * .. my mock code to return specific values for given file path ... * } * } * * val myMockEngine = mockEngine(fileSystemClient = myMockFileSystemClient) * }}} */ trait MockEngineUtils { /** * Create a mock Engine with the given components. If a component is not provided, it will * throw an exception when accessed. */ def mockEngine( fileSystemClient: FileSystemClient = null, jsonHandler: JsonHandler = null, parquetHandler: ParquetHandler = null, expressionHandler: ExpressionHandler = null): Engine = { new Engine() { override def getExpressionHandler: ExpressionHandler = Option(expressionHandler).getOrElse( throw new UnsupportedOperationException("not supported in this test suite")) override def getJsonHandler: JsonHandler = Option(jsonHandler).getOrElse( throw new UnsupportedOperationException("not supported in this test suite")) override def getFileSystemClient: FileSystemClient = Option(fileSystemClient).getOrElse( throw new UnsupportedOperationException("not supported in this test suite")) override def getParquetHandler: ParquetHandler = Option(parquetHandler).getOrElse( throw new UnsupportedOperationException("not supported in this test suite")) } } } /** * Base class for mocking [[JsonHandler]] */ trait BaseMockJsonHandler extends JsonHandler { override def parseJson( jsonStringVector: ColumnVector, outputSchema: StructType, selectionVector: Optional[ColumnVector]): ColumnarBatch = throw new UnsupportedOperationException("not supported in this test suite") override def readJsonFiles( fileIter: CloseableIterator[FileStatus], physicalSchema: StructType, predicate: Optional[Predicate]): CloseableIterator[ColumnarBatch] = throw new UnsupportedOperationException("not supported in this test suite") override def writeJsonFileAtomically( filePath: String, data: CloseableIterator[Row], overwrite: Boolean): Unit = throw new UnsupportedOperationException("not supported in this test suite") } /** * Base class for mocking [[ParquetHandler]] */ trait BaseMockParquetHandler extends ParquetHandler with MockEngineUtils { override def readParquetFiles( fileIter: CloseableIterator[FileStatus], physicalSchema: StructType, predicate: Optional[Predicate]): CloseableIterator[FileReadResult] = throw new UnsupportedOperationException("not supported in this test suite") override def writeParquetFiles( directoryPath: String, dataIter: CloseableIterator[FilteredColumnarBatch], statsColumns: util.List[Column]): CloseableIterator[DataFileStatus] = throw new UnsupportedOperationException("not supported in this test suite") override def writeParquetFileAtomically( filePath: String, data: CloseableIterator[FilteredColumnarBatch]): Unit = throw new UnsupportedOperationException("not supported in this test suite") } /** * Base class for mocking [[ExpressionHandler]] */ trait BaseMockExpressionHandler extends ExpressionHandler { override def getPredicateEvaluator( inputSchema: StructType, predicate: Predicate): PredicateEvaluator = throw new UnsupportedOperationException("not supported in this test suite") override def getEvaluator( inputSchema: StructType, expression: Expression, outputType: DataType): ExpressionEvaluator = throw new UnsupportedOperationException("not supported in this test suite") override def createSelectionVector(values: Array[Boolean], from: Int, to: Int): ColumnVector = throw new UnsupportedOperationException("not supported in this test suite") } /** * Base class for [[FileSystemClient]] */ trait BaseMockFileSystemClient extends FileSystemClient { override def listFrom(filePath: String): CloseableIterator[FileStatus] = throw new UnsupportedOperationException("not supported in this test suite") override def resolvePath(path: String): String = throw new UnsupportedOperationException("not supported in this test suite") override def readFiles( readRequests: CloseableIterator[FileReadRequest]): CloseableIterator[ByteArrayInputStream] = throw new UnsupportedOperationException("not supported in this test suite") override def mkdirs(path: String): Boolean = throw new UnsupportedOperationException("not supported in this test suite") override def delete(path: String): Boolean = throw new UnsupportedOperationException("not supported in this test suite") override def getFileStatus(path: String): FileStatus = throw new UnsupportedOperationException("not supported in this test suite") override def copyFileAtomically(srcPath: String, destPath: String, overwrite: Boolean): Unit = throw new UnsupportedOperationException("not supported in this test suite") } /** * A mock [[JsonHandler]] that reads a single file and returns a single [[ColumnarBatch]]. * The columnar batch only contains the [[CommitInfo]] action with the `inCommitTimestamp` * column set to the value in the mapping. * * @param deltaVersionToICTMapping A mapping from delta version to inCommitTimestamp. */ class MockReadICTFileJsonHandler( deltaVersionToICTMapping: Map[Long, Long], add10ForStagedFiles: Boolean = false) extends BaseMockJsonHandler with VectorTestUtils { override def readJsonFiles( fileIter: CloseableIterator[FileStatus], physicalSchema: StructType, predicate: Optional[Predicate]): CloseableIterator[ColumnarBatch] = { assert(fileIter.hasNext) val filePathStr = fileIter.next.getPath assert(FileNames.isCommitFile(filePathStr)) val deltaVersion = FileNames.getFileVersion(new Path(filePathStr)) assert(deltaVersionToICTMapping.contains(deltaVersion)) var ict = deltaVersionToICTMapping(deltaVersion) // This enables us to have different ICT times for staged vs published delta files, which lets // us test that we use the correct file when both exist for a specific version if (add10ForStagedFiles && FileNames.isStagedDeltaFile(filePathStr)) { ict += 10 } val schema = new StructType().add("commitInfo", CommitInfo.FULL_SCHEMA); Utils.singletonCloseableIterator( new ColumnarBatch { override def getSchema: StructType = schema override def getColumnVector(ordinal: Int): ColumnVector = { val struct = Seq( longVector(Seq(ict)), /* inCommitTimestamp */ longVector(Seq(-1L)), /* timestamp */ stringVector(Seq("engine")), /* engineInfo */ stringVector(Seq("operation")), /* operation */ mapTypeVector(Seq(Map("operationParameter" -> ""))), /* operationParameters */ booleanVector(Seq(false)), /* isBlindAppend */ stringVector(Seq("txnId")), /* txnId */ mapTypeVector(Seq(Map("operationMetrics" -> ""))) /* operationMetrics */ ) ordinal match { case 0 => new ColumnVector { override def getDataType: DataType = schema override def getSize: Int = struct.head.getSize override def close(): Unit = {} override def isNullAt(rowId: Int): Boolean = false override def getChild(ordinal: Int): ColumnVector = struct(ordinal) } } } override def getSize: Int = 1 }) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/test/MockFileSystemClientUtils.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.test import java.util.{Optional, UUID} import scala.collection.JavaConverters._ import io.delta.kernel.engine._ import io.delta.kernel.internal.MockReadLastCheckpointFileJsonHandler import io.delta.kernel.internal.files.ParsedLogData import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.util.FileNames import io.delta.kernel.internal.util.Utils.toCloseableIterator import io.delta.kernel.utils.{CloseableIterator, FileStatus} object MockFileSystemClientUtils extends MockFileSystemClientUtils /** * This is an extension to [[BaseMockFileSystemClient]] containing specific mock implementations * [[FileSystemClient]] which are shared across multiple test suite. * * [[MockListFromFileSystemClient]] - mocks the `listFrom` API within [[FileSystemClient]]. */ trait MockFileSystemClientUtils extends MockEngineUtils { val dataPath = new Path("/fake/path/to/table/") val logPath = new Path(dataPath, "_delta_log") def parsedRatifiedStagedCommit(version: Long): ParsedLogData = { ParsedLogData.forFileStatus(stagedCommitFile(version)) } def parsedRatifiedStagedCommits(versions: Seq[Long]): Seq[ParsedLogData] = { versions.map(parsedRatifiedStagedCommit) } /** Staged commit file status where the timestamp = 10*version */ def stagedCommitFile(v: Long): FileStatus = FileStatus.of(FileNames.stagedCommitFile(logPath, v), v, v * 10) /** Delta file status where the timestamp = 10*version */ def deltaFileStatus(v: Long, path: Path = logPath): FileStatus = FileStatus.of(FileNames.deltaFile(path, v), v, v * 10) /** Compaction file status where the timestamp = 10*startVersion */ def logCompactionStatus(s: Long, e: Long, path: Path = logPath): FileStatus = FileStatus.of(FileNames.logCompactionPath(path, s, e).toString, s, s * 10) /** Delta file statuses where the timestamp = 10*version */ def deltaFileStatuses(deltaVersions: Seq[Long], path: Path = logPath): Seq[FileStatus] = { assert(deltaVersions.size == deltaVersions.toSet.size) deltaVersions.map(v => deltaFileStatus(v, path)) } /** Compaction file statuses where the timestamp = 10*startVersion */ def compactedFileStatuses( compactedVersions: Seq[(Long, Long)], path: Path = logPath): Seq[FileStatus] = { compactedVersions.map { case (s, e) => logCompactionStatus(s, e, path) } } /** Checksum file status for given a version */ def checksumFileStatus(deltaVersion: Long): FileStatus = { FileStatus.of(FileNames.checksumFile(logPath, deltaVersion).toString, 10, 10) } /** Classic checkpoint file status where the timestamp = 10*version */ def classicCheckpointFileStatus(v: Long): FileStatus = { FileStatus.of(FileNames.checkpointFileSingular(logPath, v).toString, v, v * 10) } /** Checkpoint file statuses where the timestamp = 10*version */ def singularCheckpointFileStatuses( checkpointVersions: Seq[Long], path: Path = logPath): Seq[FileStatus] = { assert(checkpointVersions.size == checkpointVersions.toSet.size) checkpointVersions.map(v => FileStatus.of(FileNames.checkpointFileSingular(path, v).toString, v, v * 10)) } /** Multi-part checkpoint file status where the timestamp = 10*version */ def multiPartCheckpointFileStatus(version: Long, part: Integer, numParts: Integer): FileStatus = { val path = FileNames.multiPartCheckpointFile(logPath, version, part, numParts) FileStatus.of(path.toString, version, version * 10) } /** Checkpoint file statuses where the timestamp = 10*version */ def multiCheckpointFileStatuses( checkpointVersions: Seq[Long], numParts: Int): Seq[FileStatus] = { assert(checkpointVersions.size == checkpointVersions.toSet.size) checkpointVersions.flatMap(v => FileNames.checkpointFileWithParts(logPath, v, numParts).asScala .map(p => FileStatus.of(p.toString, v, v * 10))) } /** Checkpoint file status for a top-level V2 checkpoint file. */ def v2CheckpointFileStatus( version: Long, useUUID: Boolean = true, fileType: String = "json"): FileStatus = { val path = if (useUUID) { val uuid = UUID.randomUUID().toString FileNames.topLevelV2CheckpointFile(logPath, version, uuid, fileType).toString } else { FileNames.checkpointFileSingular(logPath, version).toString } FileStatus.of(path, version, version * 10) } /** * Checkpoint file status for a top-level V2 checkpoint file. * * @param checkpointVersions List of checkpoint versions, given as Seq(version, whether to use * UUID naming scheme, number of sidecars). * Returns top-level checkpoint file and sidecar files for each checkpoint version. */ def v2CheckpointFileStatuses( checkpointVersions: Seq[(Long, Boolean, Int)], fileType: String): Seq[(FileStatus, Seq[FileStatus])] = { checkpointVersions.map { case (v, useUUID, numSidecars) => val topLevelFile = v2CheckpointFileStatus(v, useUUID, fileType) val sidecars = (0 until numSidecars).map { _ => FileStatus.of( FileNames.v2CheckpointSidecarFile(logPath, UUID.randomUUID().toString).toString, v, v * 10) } (topLevelFile, sidecars) } } /* Create input function for createMockEngine to implement listFrom from a list of * file statuses. */ def listFromProvider(files: Seq[FileStatus])(filePath: String): Seq[FileStatus] = { val parentPath = new Path(filePath).getParent files // This currently excludes listing nested directories, we can fix this if needed .filter(fs => new Path(fs.getPath).getParent == parentPath) .filter(_.getPath.compareTo(filePath) >= 0) .sortBy(_.getPath) } /** * Create a mock [[Engine]] to mock the [[FileSystemClient.listFrom]] calls using * the given contents. The contents are filtered depending upon the list from path prefix. */ def createMockFSListFromEngine( contents: Seq[FileStatus], parquetHandler: ParquetHandler, jsonHandler: JsonHandler): Engine = { mockEngine( fileSystemClient = new MockListFromFileSystemClient(listFromProvider(contents)), parquetHandler = parquetHandler, jsonHandler = jsonHandler) } def createMockFSAndJsonEngineForLastCheckpoint( contents: Seq[FileStatus], lastCheckpointVersion: Optional[java.lang.Long]): Engine = { mockEngine( fileSystemClient = new MockListFromFileSystemClient(listFromProvider(contents)), jsonHandler = if (lastCheckpointVersion.isPresent) { new MockReadLastCheckpointFileJsonHandler( s"$logPath/_last_checkpoint", lastCheckpointVersion.get()) } else { null }) } /** * Create a mock [[Engine]] to mock the [[FileSystemClient.listFrom]] calls using * the given list of delta file statuses. When read, each file status will return * a single `commitInfo` action with the an inCommitTimestamp set as per * `deltaToICTMap`. */ def createMockFSAndJsonEngineForICT( contents: Seq[FileStatus], deltaToICTMap: Map[Long, Long]): Engine = { mockEngine( fileSystemClient = new MockListFromFileSystemClient(listFromProvider(contents)), jsonHandler = new MockReadICTFileJsonHandler(deltaToICTMap)) } /** * Create a mock [[Engine]] to mock the [[FileSystemClient.listFrom]] calls using * the given contents. The contents are filtered depending upon the list from path prefix. */ def createMockFSListFromEngine(contents: Seq[FileStatus]): Engine = { mockEngine(fileSystemClient = new MockListFromFileSystemClient(listFromProvider(contents))) } /** * Create a mock [[Engine]] to mock the [[FileSystemClient.listFrom]] calls using * [[MockListFromFileSystemClient]]. */ def createMockFSListFromEngine(listFromProvider: String => Seq[FileStatus]): Engine = { mockEngine(fileSystemClient = new MockListFromFileSystemClient(listFromProvider)) } } /** * A mock [[FileSystemClient]] that answers `listFrom` calls from a given content provider. * * It also maintains metrics on number of times `listFrom` is called and arguments for each call. */ class MockListFromFileSystemClient(listFromProvider: String => Seq[FileStatus]) extends BaseMockFileSystemClient { private var listFromCalls: Seq[String] = Seq.empty override def listFrom(filePath: String): CloseableIterator[FileStatus] = { listFromCalls = listFromCalls :+ filePath toCloseableIterator(listFromProvider(filePath).iterator.asJava) } override def resolvePath(path: String): String = path def getListFromCalls: Seq[String] = listFromCalls } /** * A mock [[FileSystemClient]] that answers `listFrom` calls from a given content provider and * implements the identity function for `resolvePath` calls. * * It also maintains metrics on number of times `listFrom` is called and arguments for each call. */ class MockListFromResolvePathFileSystemClient(listFromProvider: String => Seq[FileStatus]) extends BaseMockFileSystemClient { private var listFromCalls: Seq[String] = Seq.empty override def listFrom(filePath: String): CloseableIterator[FileStatus] = { listFromCalls = listFromCalls :+ filePath toCloseableIterator(listFromProvider(filePath).iterator.asJava) } override def resolvePath(path: String): String = path def getListFromCalls: Seq[String] = listFromCalls } /** * A mock [[FileSystemClient]] that answers `listFrom` call from the given list of file statuses * and tracks the delete calls. * @param listContents List of file statuses to be returned by `listFrom` call. */ class MockListFromDeleteFileSystemClient(listContents: Seq[FileStatus]) extends BaseMockFileSystemClient { private val listOfFiles: Seq[String] = listContents.map(_.getPath).toSeq private var isListFromAlreadyCalled = false private var deleteCalls: Seq[String] = Seq.empty override def listFrom(filePath: String): CloseableIterator[FileStatus] = { assert(!isListFromAlreadyCalled, "listFrom should be called only once") isListFromAlreadyCalled = true toCloseableIterator(listContents.sortBy(_.getPath).asJava.iterator()) } override def delete(path: String): Boolean = { deleteCalls = deleteCalls :+ path listOfFiles.contains(path) } def getDeleteCalls: Seq[String] = deleteCalls } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/test/MockSnapshotUtils.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.test import java.util.Optional import scala.collection.JavaConverters._ import io.delta.kernel.TransactionSuite.testSchema import io.delta.kernel.internal.{SnapshotImpl, TableConfig} import io.delta.kernel.internal.actions.{Format, Metadata, Protocol} import io.delta.kernel.internal.commit.DefaultFileSystemManagedTableOnlyCommitter import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.lang.Lazy import io.delta.kernel.internal.metrics.SnapshotQueryContext import io.delta.kernel.internal.snapshot.LogSegment import io.delta.kernel.internal.util.FileNames import io.delta.kernel.internal.util.VectorUtils.{buildArrayValue, stringStringMapValue} import io.delta.kernel.types.StringType import io.delta.kernel.utils.FileStatus object MockSnapshotUtils extends MockSnapshotUtils trait MockSnapshotUtils { /** * Creates a mock snapshot with valid metadata at the given version. * @param ictEnablementInfoOpt Controls the enablement state of in-commit timestamps. */ def getMockSnapshot( dataPath: Path, latestVersion: Long, ictEnablementInfoOpt: Option[(Long, Long)] = None, timestamp: Long = 0L, deltaFileAtEndVersion: Option[FileStatus] = None): SnapshotImpl = { val configuration = ictEnablementInfoOpt match { case Some((version, _)) if version == 0L => Map(TableConfig.IN_COMMIT_TIMESTAMPS_ENABLED.getKey -> "true") case Some((version, ts)) => Map( TableConfig.IN_COMMIT_TIMESTAMPS_ENABLED.getKey -> "true", TableConfig.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.getKey -> version.toString, TableConfig.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.getKey -> ts.toString) case None => Map[String, String]() } val metadata = new Metadata( "id", Optional.empty(), /* name */ Optional.empty(), /* description */ new Format(), testSchema.toJson, testSchema, buildArrayValue(java.util.Arrays.asList("c3"), StringType.STRING), Optional.of(123), stringStringMapValue(configuration.asJava)); val logPath = new Path(dataPath, "_delta_log") val fs = deltaFileAtEndVersion.getOrElse(FileStatus.of( FileNames.deltaFile(logPath, latestVersion), 1, /* size */ 1 /* modificationTime */ )) val logSegment = new LogSegment( logPath, /* logPath */ latestVersion, Seq(fs).asJava, /* deltas */ Seq.empty.asJava, /* compactions */ Seq.empty.asJava, /* checkpoints */ fs, /* deltaAtEndVersion */ Optional.empty(), /* lastSeenChecksum */ Optional.empty() /* maxPublishedDeltaVersion */ ) val snapshotQueryContext = SnapshotQueryContext.forLatestSnapshot(dataPath.toString) new SnapshotImpl( dataPath, /* dataPath */ logSegment.getVersion, /* version */ new Lazy(() => logSegment), /* logSegment */ null, /* logReplay */ new Protocol(1, 2), /* protocol */ metadata, DefaultFileSystemManagedTableOnlyCommitter.INSTANCE, snapshotQueryContext, /* snapshotContext */ Optional.empty() /* inCommitTimestampOpt */ ) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/test/TestFixtures.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.test import java.lang.{Long => JLong} import java.util.{Collections, Map => JMap, Optional} import java.util.function.Supplier import scala.collection.JavaConverters._ import io.delta.kernel.SnapshotBuilder import io.delta.kernel.commit.CommitMetadata import io.delta.kernel.internal.actions.{CommitInfo, DomainMetadata, Metadata, Protocol} import io.delta.kernel.internal.files.ParsedCatalogCommitData import io.delta.kernel.internal.util.{FileNames, Tuple2} import io.delta.kernel.types.{ArrayType, BinaryType, BooleanType, ByteType, DataType, DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType, MapType, ShortType, StringType, StructType, TimestampNTZType, TimestampType} import io.delta.kernel.utils.FileStatus /** * Test fixtures including factory methods and constants for creating test objects with sensible * defaults. */ trait TestFixtures extends ActionUtils { /** All simple data type used in parameterized tests where type is one of the test dimensions. */ val PRIMITIVE_TYPES = Set( BooleanType.BOOLEAN, ByteType.BYTE, ShortType.SHORT, IntegerType.INTEGER, LongType.LONG, FloatType.FLOAT, DoubleType.DOUBLE, DateType.DATE, TimestampType.TIMESTAMP, TimestampNTZType.TIMESTAMP_NTZ, StringType.STRING, BinaryType.BINARY, new DecimalType(10, 5)) val NESTED_TYPES: Set[DataType] = Set( new ArrayType(BooleanType.BOOLEAN, true), new MapType(IntegerType.INTEGER, LongType.LONG, true), new StructType().add("s1", BooleanType.BOOLEAN).add("s2", IntegerType.INTEGER)) /** All types. Used in parameterized tests where type is one of the test dimensions. */ val ALL_TYPES: Set[DataType] = PRIMITIVE_TYPES ++ NESTED_TYPES def createCommitMetadata( version: Long, logPath: String = "/fake/_delta_log", commitInfo: CommitInfo = testCommitInfo(), commitDomainMetadatas: List[DomainMetadata] = List.empty, committerProperties: Supplier[JMap[String, String]] = () => Collections.emptyMap(), readPandMOpt: Optional[Tuple2[Protocol, Metadata]] = Optional.empty(), newProtocolOpt: Optional[Protocol] = Optional.empty(), newMetadataOpt: Optional[Metadata] = Optional.empty(), maxKnownPublishedDeltaVersion: Optional[JLong] = Optional.empty()): CommitMetadata = { new CommitMetadata( version, logPath, commitInfo, commitDomainMetadatas.asJava, committerProperties, readPandMOpt, newProtocolOpt, newMetadataOpt, maxKnownPublishedDeltaVersion) } def createStagedCatalogCommit( v: Long, logPath: String = "/fake/_delta_log"): ParsedCatalogCommitData = { val fileStatus = FileStatus.of(FileNames.stagedCommitFile(logPath, v), v, v * 10) ParsedCatalogCommitData.forFileStatus(fileStatus) } implicit class SnapshotBuilderCatalogVersionOps[T <: SnapshotBuilder](snapshotBuilder: T) { def withMaxCatalogVersionIfApplicable( isCatalogManaged: Boolean, maxCatalogVersion: Long): T = { if (isCatalogManaged) { snapshotBuilder .withMaxCatalogVersion(maxCatalogVersion) .asInstanceOf[T] } else { snapshotBuilder } } } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/test/TestUtils.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.test import java.io.{ByteArrayInputStream, ByteArrayOutputStream, ObjectInputStream, ObjectOutputStream} import java.util.Optional import io.delta.kernel.expressions.{Column, Literal} import io.delta.kernel.internal.skipping.StatsSchemaHelper.STATS_WITH_COLLATION import io.delta.kernel.types.CollationIdentifier /** Utility functions for tests. */ trait TestUtils { def col(name: String): Column = new Column(name) def nestedCol(name: String): Column = { new Column(name.split("\\.")) } def collatedStatsCol( collation: CollationIdentifier, statName: String, fieldName: String): Column = { val columnPath = Array(STATS_WITH_COLLATION, collation.toString, statName) ++ fieldName.split('.') new Column(columnPath) } def literal(value: Any): Literal = { value match { case v: String => Literal.ofString(v) case v: Int => Literal.ofInt(v) case v: Long => Literal.ofLong(v) case v: Float => Literal.ofFloat(v) case v: Double => Literal.ofDouble(v) case v: Boolean => Literal.ofBoolean(v) case _ => throw new IllegalArgumentException(s"Unsupported literal type: ${value}") } } implicit class ScalaOptionOps[T](option: Option[T]) { def toJava: Optional[T] = option match { case Some(value) => Optional.of(value) case None => Optional.empty() } } implicit class JavaOptionalOps[T](optional: Optional[T]) { def toScala: Option[T] = if (optional.isPresent) Some(optional.get()) else None } /** * Helper to test Java serialization by performing a round-trip serialize/deserialize. * * @param obj The object to serialize (must be Serializable) * @return The deserialized object */ def roundTripSerialize[T](obj: T): T = { val baos = new ByteArrayOutputStream() val oos = new ObjectOutputStream(baos) try { oos.writeObject(obj) oos.flush() } finally { oos.close() } val bais = new ByteArrayInputStream(baos.toByteArray) val ois = new ObjectInputStream(bais) try { ois.readObject().asInstanceOf[T] } finally { ois.close() } } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/test/VectorTestUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.test import java.lang.{Boolean => BooleanJ, Double => DoubleJ, Float => FloatJ, Integer => IntegerJ, Long => LongJ} import scala.collection.JavaConverters._ import io.delta.kernel.data.{ColumnarBatch, ColumnVector, MapValue, Row} import io.delta.kernel.internal.util.VectorUtils import io.delta.kernel.types._ import io.delta.kernel.utils.CloseableIterator trait VectorTestUtils { protected def emptyActionsIterator = new CloseableIterator[Row] { override def hasNext: Boolean = false override def next(): Row = throw new NoSuchElementException("No more elements") override def close(): Unit = {} } protected def emptyColumnarBatch = new ColumnarBatch { override def getSchema: StructType = null override def getColumnVector(ordinal: Int): ColumnVector = null override def getSize: Int = 0 } protected def booleanVector(values: Seq[BooleanJ]): ColumnVector = { new ColumnVector { override def getDataType: DataType = BooleanType.BOOLEAN override def getSize: Int = values.length override def close(): Unit = {} override def isNullAt(rowId: Int): Boolean = values(rowId) == null override def getBoolean(rowId: Int): Boolean = values(rowId) } } protected def timestampVector(values: Seq[LongJ]): ColumnVector = { new ColumnVector { override def getDataType: DataType = TimestampType.TIMESTAMP override def getSize: Int = values.length override def close(): Unit = {} override def isNullAt(rowId: Int): Boolean = values(rowId) == null || values(rowId) == -1 // Values are stored as Longs representing milliseconds since epoch override def getLong(rowId: Int): Long = values(rowId) } } protected def stringVector(values: Seq[String]): ColumnVector = { new ColumnVector { override def getDataType: DataType = StringType.STRING override def getSize: Int = values.length override def close(): Unit = {} override def isNullAt(rowId: Int): Boolean = values(rowId) == null override def getString(rowId: Int): String = values(rowId) } } protected def mapTypeVector(values: Seq[Map[String, String]]): ColumnVector = { new ColumnVector { override def getDataType: DataType = new MapType(StringType.STRING, StringType.STRING, true) override def getSize: Int = values.length override def close(): Unit = {} override def isNullAt(rowId: Int): Boolean = values(rowId) == null override def getMap(rowId: Int): MapValue = VectorUtils.stringStringMapValue(values(rowId).asJava) } } protected def byteVector(values: Seq[java.lang.Byte]): ColumnVector = { new ColumnVector { override def getDataType: DataType = ByteType.BYTE override def getSize: Int = values.length override def close(): Unit = {} override def isNullAt(rowId: Int): Boolean = values(rowId) == null override def getByte(rowId: Int): Byte = values(rowId) } } protected def intVector(values: Seq[IntegerJ]): ColumnVector = { new ColumnVector { override def getDataType: DataType = IntegerType.INTEGER override def getSize: Int = values.length override def close(): Unit = {} override def isNullAt(rowId: Int): Boolean = values(rowId) == null override def getInt(rowId: Int): Int = values(rowId) } } protected def floatVector(values: Seq[FloatJ]): ColumnVector = { new ColumnVector { override def getDataType: DataType = FloatType.FLOAT override def getSize: Int = values.length override def close(): Unit = {} override def isNullAt(rowId: Int): Boolean = values(rowId) == null override def getFloat(rowId: Int): Float = values(rowId) } } protected def doubleVector(values: Seq[DoubleJ]): ColumnVector = { new ColumnVector { override def getDataType: DataType = DoubleType.DOUBLE override def getSize: Int = values.length override def close(): Unit = {} override def isNullAt(rowId: Int): Boolean = values(rowId) == null override def getDouble(rowId: Int): Double = values(rowId) } } def longVector(values: Seq[LongJ]): ColumnVector = new ColumnVector { override def getDataType: DataType = LongType.LONG override def getSize: Int = values.length override def close(): Unit = {} override def isNullAt(rowId: Int): Boolean = values(rowId) == null override def getLong(rowId: Int): Long = values(rowId) } def selectSingleElement(size: Int, selectRowId: Int): ColumnVector = new ColumnVector { override def getDataType: DataType = BooleanType.BOOLEAN override def getSize: Int = size override def close(): Unit = {} override def isNullAt(rowId: Int): Boolean = false override def getBoolean(rowId: Int): Boolean = rowId == selectRowId } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/transaction/DataLayoutSpecSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.transaction import scala.collection.JavaConverters._ import io.delta.kernel.expressions.Column import org.scalatest.funsuite.AnyFunSuite /** * Test suite for [[DataLayoutSpec]]. */ class DataLayoutSpecSuite extends AnyFunSuite { // Helper methods for creating test columns private def cols(names: String*): java.util.List[Column] = names.map(new Column(_)).asJava test("noDataLayout creates spec with no special layout") { val spec = DataLayoutSpec.noDataLayout() assert(spec.hasNoDataLayoutSpec()) assert(!spec.hasPartitioning()) assert(!spec.hasClustering()) } test("partitioned creates spec with partition columns") { val partitionCols = cols("year", "month", "day") val spec = DataLayoutSpec.partitioned(partitionCols) assert(!spec.hasNoDataLayoutSpec()) assert(spec.hasPartitioning()) assert(!spec.hasClustering()) assert(spec.getPartitionColumns() == partitionCols) assert(spec.getPartitionColumnsAsStrings().asScala == Seq("year", "month", "day")) } test("partitioned throws exception for null columns") { val exception = intercept[IllegalArgumentException] { DataLayoutSpec.partitioned(null) } assert(exception.getMessage.contains("Partition columns cannot be null or empty")) } test("partitioned throws exception for empty columns") { val exception = intercept[IllegalArgumentException] { DataLayoutSpec.partitioned(List.empty[Column].asJava) } assert(exception.getMessage.contains("Partition columns cannot be null or empty")) } test("partitioned throws exception for nested columns") { val nestedCol = new Column(Array("struct_col", "nested_field")) val exception = intercept[IllegalArgumentException] { DataLayoutSpec.partitioned(List(nestedCol).asJava) } assert(exception.getMessage.contains("Partition columns must be only top-level columns")) } test("clustered creates spec with clustering columns") { val clusteringCols = cols("user_id", "timestamp") val spec = DataLayoutSpec.clustered(clusteringCols) assert(!spec.hasNoDataLayoutSpec()) assert(!spec.hasPartitioning()) assert(spec.hasClustering()) assert(spec.getClusteringColumns() == clusteringCols) } test("clustered with empty columns list") { val spec = DataLayoutSpec.clustered(List.empty[Column].asJava) assert(!spec.hasNoDataLayoutSpec()) assert(!spec.hasPartitioning()) assert(spec.hasClustering()) assert(spec.getClusteringColumns().isEmpty) } test("clustered throws exception for null columns") { val exception = intercept[IllegalArgumentException] { DataLayoutSpec.clustered(null) } assert(exception.getMessage.contains("Clustering columns cannot be null (but can be empty)")) } test("getPartitionColumns throws exception when partitioning not enabled") { val spec = DataLayoutSpec.noDataLayout() val exception = intercept[IllegalStateException] { spec.getPartitionColumns() } assert(exception.getMessage.contains( "Cannot get partition columns: partitioning is not enabled on this layout")) } test("getPartitionColumnsAsStrings throws exception when partitioning not enabled") { val spec = DataLayoutSpec.noDataLayout() val exception = intercept[IllegalStateException] { spec.getPartitionColumnsAsStrings() } assert(exception.getMessage.contains( "Cannot get partition columns: partitioning is not enabled on this layout")) } test("getClusteringColumns throws exception when clustering not enabled") { val spec = DataLayoutSpec.noDataLayout() val exception = intercept[IllegalStateException] { spec.getClusteringColumns() } assert(exception.getMessage.contains( "Cannot get clustering columns: clustering is not enabled on this layout")) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/types/CollationIdentifierSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types import java.util.Optional import org.scalatest.funsuite.AnyFunSuite class CollationIdentifierSuite extends AnyFunSuite { val PROVIDER_SPARK = "SPARK" val PROVIDER_ICU = "ICU" val DEFAULT_COLLATION_NAME = "UTF8_BINARY" val DEFAULT_COLLATION_IDENTIFIER = CollationIdentifier.fromString("SPARK.UTF8_BINARY") test("check fromString with valid string") { Seq( ( s"$PROVIDER_SPARK.$DEFAULT_COLLATION_NAME", DEFAULT_COLLATION_IDENTIFIER), ( s"$PROVIDER_ICU.sr_Cyrl_SRB", CollationIdentifier.fromString(s"$PROVIDER_ICU.sr_Cyrl_SRB")), ( s"$PROVIDER_ICU.sr_Cyrl_SRB.75.1", CollationIdentifier.fromString(s"$PROVIDER_ICU.sr_Cyrl_SRB.75.1"))).foreach { case (stringIdentifier, collationIdentifier) => assert(CollationIdentifier.fromString(stringIdentifier).equals(collationIdentifier)) } } test("check fromString with invalid string") { Seq( PROVIDER_SPARK, s"${PROVIDER_SPARK}_sr_Cyrl_SRB").foreach { stringIdentifier => val e = intercept[IllegalArgumentException] { val collationIdentifier = CollationIdentifier.fromString(stringIdentifier) } assert(e.getMessage == String.format("Invalid collation identifier: %s", stringIdentifier)) } } test("check toStringWithoutVersion") { Seq( ( DEFAULT_COLLATION_IDENTIFIER, s"$PROVIDER_SPARK.$DEFAULT_COLLATION_NAME"), ( CollationIdentifier.fromString(s"$PROVIDER_ICU.sr_Cyrl_SRB"), s"$PROVIDER_ICU.SR_CYRL_SRB"), ( CollationIdentifier.fromString(s"$PROVIDER_ICU.sr_Cyrl_SRB.75.1"), s"$PROVIDER_ICU.SR_CYRL_SRB")).foreach { case (collationIdentifier, toStringWithoutVersion) => assert(collationIdentifier.toStringWithoutVersion == toStringWithoutVersion) } } test("check toString") { Seq( ( DEFAULT_COLLATION_IDENTIFIER, s"$PROVIDER_SPARK.$DEFAULT_COLLATION_NAME"), ( CollationIdentifier.fromString(s"$PROVIDER_ICU.sr_Cyrl_SRB"), s"$PROVIDER_ICU.SR_CYRL_SRB"), ( CollationIdentifier.fromString(s"$PROVIDER_ICU.sr_Cyrl_SRB.75.1"), s"$PROVIDER_ICU.SR_CYRL_SRB.75.1")).foreach { case (collationIdentifier, toString) => assert(collationIdentifier.toString == toString) } } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/types/DataTypeSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types import org.scalatest.funsuite.AnyFunSuite class DataTypeSuite extends AnyFunSuite { val utf8LcaseString = new StringType("SPARK.UTF8_LCASE") val unicodeString = new StringType("ICU.UNICODE") test("isWriteCompatible") { val testCases = Seq( (StringType.STRING, StringType.STRING, true), (StringType.STRING, utf8LcaseString, true), (IntegerType.INTEGER, StringType.STRING, false), (utf8LcaseString, unicodeString, true), ( new ArrayType(StringType.STRING, true), new ArrayType(utf8LcaseString, true), true), ( new ArrayType(unicodeString, false), new ArrayType(StringType.STRING, false), true), ( new ArrayType(StringType.STRING, true), new ArrayType(utf8LcaseString, false), false), ( new MapType(StringType.STRING, utf8LcaseString, false), new MapType(StringType.STRING, unicodeString, false), true), ( new MapType(StringType.STRING, utf8LcaseString, false), new MapType(StringType.STRING, StringType.STRING, false), true), ( new MapType(StringType.STRING, IntegerType.INTEGER, false), new MapType(StringType.STRING, IntegerType.INTEGER, true), false), ( new StructType() .add("name", StringType.STRING) .add("age", IntegerType.INTEGER), new StructType() .add("name", utf8LcaseString) .add("age", IntegerType.INTEGER), true), ( new StructType() .add("name", StringType.STRING) .add("details", new StructType().add("address", StringType.STRING)), new StructType() .add("name", unicodeString) .add("details", new StructType().add("address", utf8LcaseString)), true), ( new StructType() .add("c1", new ArrayType(unicodeString, true)) .add("c2", new MapType(StringType.STRING, utf8LcaseString, false)), new StructType() .add("c1", new ArrayType(StringType.STRING, true)) .add("c2", new MapType(StringType.STRING, unicodeString, false)), true), ( new StructType() .add("c1", new ArrayType(unicodeString, false)) .add("c2", new MapType(StringType.STRING, utf8LcaseString, false)), new StructType() .add("c1", new ArrayType(StringType.STRING, true)) .add("c2", new MapType(StringType.STRING, unicodeString, false)), false), ( new StructType() .add("c1", new ArrayType(IntegerType.INTEGER, true)) .add("c2", new MapType(StringType.STRING, utf8LcaseString, false)), new StructType() .add("c1", new ArrayType(StringType.STRING, true)) .add("c2", new MapType(StringType.STRING, unicodeString, false)), false), ( new ArrayType( new StructType().add("c1", new MapType(StringType.STRING, StringType.STRING, true), true), true), new ArrayType( new StructType().add("c1", new MapType(StringType.STRING, utf8LcaseString, true), true), true), true), ( new ArrayType( new StructType().add("c1", new MapType(StringType.STRING, StringType.STRING, true), true), true), new ArrayType( new StructType().add("c2", new MapType(StringType.STRING, unicodeString, true), true), true), false), ( new ArrayType( new StructType().add( "c1", new MapType(StringType.STRING, StringType.STRING, true), false), true), new ArrayType( new StructType().add("c1", new MapType(StringType.STRING, utf8LcaseString, true), true), true), false), ( new MapType( new StructType().add("c1", StringType.STRING), new ArrayType(utf8LcaseString, false), true), new MapType( new StructType().add("c1", StringType.STRING), new ArrayType(utf8LcaseString, false), true), true), ( new MapType( new StructType().add("c1", StringType.STRING), new ArrayType(utf8LcaseString, false), false), new MapType( new StructType().add("c1", StringType.STRING), new ArrayType(utf8LcaseString, false), true), false), ( new MapType(new StructType().add("c1", StringType.STRING), StringType.STRING, false), new MapType(new StructType().add("c1", StringType.STRING), utf8LcaseString, true), false)) testCases.foreach { case (dt1, dt2, expected) => assert(dt1.isWriteCompatible(dt2) == expected) } } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/types/FieldMetadataSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types import scala.jdk.CollectionConverters._ import io.delta.kernel.exceptions.KernelException import org.assertj.core.api.Assertions.{assertThat, assertThatThrownBy} import org.scalatest.funsuite.AnyFunSuite class FieldMetadataSuite extends AnyFunSuite { test("retrieving non-existing key returns null") { assertThat(FieldMetadata.builder().build().get("non-existing")).isNull() assertThat(FieldMetadata.builder().putBoolean("key", false).build() .getLong("non-existing")).isNull() } test("retrieving key with null value should never throw") { val meta = FieldMetadata.builder().putNull("nullKey").build() assertThat(meta.getLong("nullKey")).isNull() assertThat(meta.getBoolean("nullKey")).isNull() } test("retrieving key with wrong type throws exception") { val longs: Seq[java.lang.Long] = Seq(1L, 2L, 3L) val doubles: Seq[java.lang.Double] = Seq(1.0, 2.0, 3.0) val booleans: Seq[java.lang.Boolean] = Seq(true, false, true) val strings: Seq[java.lang.String] = Seq("a", "b", "c") val innerMeta = FieldMetadata.builder().putBoolean("key", true).build() val meta = FieldMetadata.builder() .putLong("longKey", 23L) .putDouble("doubleKey", 23.0) .putBoolean("booleanKey", true) .putString("stringKey", "random") .putFieldMetadata("fieldMetadataKey", innerMeta) .putLongArray("longArrayKey", longs.toArray) .putDoubleArray("doubleArrayKey", doubles.toArray) .putBooleanArray("booleanArrayKey", booleans.toArray) .putStringArray("stringArrayKey", strings.toArray) .putFieldMetadataArray("fieldMetadataArrayKey", Seq(innerMeta).toArray) .build assertThatThrownBy(() => meta.getLongArray("longKey")) .isInstanceOf(classOf[IllegalArgumentException]) assertThatThrownBy(() => meta.getDoubleArray("doubleKey")) .isInstanceOf(classOf[IllegalArgumentException]) assertThatThrownBy(() => meta.getBooleanArray("booleanKey")) .isInstanceOf(classOf[IllegalArgumentException]) assertThatThrownBy(() => meta.getStringArray("stringKey")) .isInstanceOf(classOf[IllegalArgumentException]) assertThatThrownBy(() => meta.getMetadataArray("fieldMetadataKey")) .isInstanceOf(classOf[IllegalArgumentException]) assertThatThrownBy(() => meta.getLong("longArrayKey")) .isInstanceOf(classOf[IllegalArgumentException]) assertThatThrownBy(() => meta.getDouble("doubleArrayKey")) .isInstanceOf(classOf[IllegalArgumentException]) assertThatThrownBy(() => meta.getBoolean("booleanArrayKey")) .isInstanceOf(classOf[IllegalArgumentException]) assertThatThrownBy(() => meta.getString("stringArrayKey")) .isInstanceOf(classOf[IllegalArgumentException]) assertThatThrownBy(() => meta.getMetadata("fieldMetadataArrayKey")) .isInstanceOf(classOf[IllegalArgumentException]) } test("retrieving key with correct type returns value") { val longs: Seq[java.lang.Long] = Seq(1L, 2L, 3L) val doubles: Seq[java.lang.Double] = Seq(1.0, 2.0, 3.0) val booleans: Seq[java.lang.Boolean] = Seq(true, false, true) val strings: Seq[java.lang.String] = Seq("a", "b", "c") val innerMeta = FieldMetadata.builder().putBoolean("key", true).build() val meta = FieldMetadata.builder() .putLong("longKey", 23L) .putDouble("doubleKey", 23.0) .putBoolean("booleanKey", true) .putString("stringKey", "random") .putFieldMetadata("fieldMetadataKey", innerMeta) .putLongArray("longArrayKey", longs.toArray) .putDoubleArray("doubleArrayKey", doubles.toArray) .putBooleanArray("booleanArrayKey", booleans.toArray) .putStringArray("stringArrayKey", strings.toArray) .putFieldMetadataArray("fieldMetadataArrayKey", Seq(innerMeta).toArray) .build assertThat(meta.getLong("longKey")).isEqualTo(23L) assertThat(meta.getDouble("doubleKey")).isEqualTo(23.0) assertThat(meta.getBoolean("booleanKey")).isTrue assertThat(meta.getString("stringKey")).isEqualTo("random") assertThat(meta.getMetadata("fieldMetadataKey")).isEqualTo(innerMeta) assertThat(meta.getLongArray("longArrayKey")).isEqualTo(longs.toArray) assertThat(meta.getDoubleArray("doubleArrayKey")).isEqualTo(doubles.toArray) assertThat(meta.getBooleanArray("booleanArrayKey")).isEqualTo(booleans.toArray) assertThat(meta.getStringArray("stringArrayKey")).isEqualTo(strings.toArray) assertThat(meta.getMetadataArray("fieldMetadataArrayKey")) .isEqualTo(Seq(innerMeta).toArray) } test("builder.getMetadata handles null correctly") { val builder = FieldMetadata.builder() assertThat(builder.getMetadata("non-existing")).isNull() } test("builder.getMetadata with wrong type throws KernelException") { val builder = FieldMetadata.builder() .putLong("longKey", 23L) val err = intercept[KernelException] { builder.getMetadata("longKey") } assert(err.getMessage.contains("Expected '23' to be of type 'FieldMetadata'")) } test("builder.getMetadata with correct type returns value") { val innerMeta = FieldMetadata.builder().putBoolean("key", true).build() val builder = FieldMetadata.builder() .putFieldMetadata("fieldMetadataKey", innerMeta) assertThat(builder.getMetadata("fieldMetadataKey")).isEqualTo(innerMeta) } test("toString handles empty metadata") { val fieldMetadata = FieldMetadata.builder().build() val result = fieldMetadata.toString assertThat(result).isEqualTo("{}") } test("toString handles null values and arrays with null elements") { val fieldMetadata = FieldMetadata.builder() .putString("nullValueKey", null) .putStringArray("arrayWithNulls", Array("a", null, "b")) .putString("validValue", "test") .putStringArray("validArray", Array("x", "y", "z")) .build() val result = fieldMetadata.toString assertThat(result).contains("nullValueKey=null") assertThat(result).contains("[a, null, b]") assertThat(result).contains("validValue=test") assertThat(result).contains("[x, y, z]") } test("equalsIgnoreKeys ignores specified keys while validating others") { val meta1 = FieldMetadata.builder() .putString("collation", "UTF8_BINARY") .putString("otherKey", "same") .putLongArray("arr", Seq[java.lang.Long](1L, 2L).toArray) .build() val meta2 = FieldMetadata.builder() .putString("collation", "EN_CI") // different but should be ignored .putString("otherKey", "same") .putLongArray("arr", Seq[java.lang.Long](1L, 2L).toArray) .build() val ignoreCollation: java.util.Set[String] = Set("collation").asJava val emptyIgnore: java.util.Set[String] = new java.util.HashSet[String]() assertThat(meta1.equalsIgnoreKeys(meta2, ignoreCollation)).isTrue assertThat(meta1.equalsIgnoreKeys(meta2, emptyIgnore)).isFalse } test("equalsIgnoreKeys deepEquals handles arrays properly") { val longs: Seq[java.lang.Long] = Seq(1L, 2L, 3L) val doubles: Seq[java.lang.Double] = Seq(1.0, 2.0, 3.0) val booleans: Seq[java.lang.Boolean] = Seq(true, false, true) val strings: Seq[java.lang.String] = Seq("x", "y", "z") val stringsWithNulls: Seq[java.lang.String] = Seq("a", null, "b") val inner1 = FieldMetadata.builder().putBoolean("k", true).build() val inner2 = FieldMetadata.builder().putBoolean("k", true).build() val meta1 = FieldMetadata.builder() .putLongArray("longArrayKey", longs.toArray) .putDoubleArray("doubleArrayKey", doubles.toArray) .putBooleanArray("booleanArrayKey", booleans.toArray) .putStringArray("stringArrayKey", strings.toArray) .putStringArray("stringArrayWithNulls", stringsWithNulls.toArray) .putFieldMetadataArray("fieldMetadataArrayKey", Array(inner1)) .build() val meta2 = FieldMetadata.builder() .putLongArray("longArrayKey", longs.toArray) .putDoubleArray("doubleArrayKey", doubles.toArray) .putBooleanArray("booleanArrayKey", booleans.toArray) .putStringArray("stringArrayKey", strings.toArray) .putStringArray("stringArrayWithNulls", stringsWithNulls.toArray) .putFieldMetadataArray("fieldMetadataArrayKey", Array(inner2)) .build() val emptyIgnore: java.util.Set[String] = new java.util.HashSet[String]() assertThat(meta1.equalsIgnoreKeys(meta2, emptyIgnore)).isTrue // Change one array element to ensure inequality is detected val meta3 = FieldMetadata.builder() .putLongArray("longArrayKey", Seq[java.lang.Long](1L, 2L, 99L).toArray) .putDoubleArray("doubleArrayKey", doubles.toArray) .putBooleanArray("booleanArrayKey", booleans.toArray) .putStringArray("stringArrayKey", strings.toArray) .putStringArray("stringArrayWithNulls", stringsWithNulls.toArray) .putFieldMetadataArray("fieldMetadataArrayKey", Array(inner2)) .build() val ignoreLongArrayKey: java.util.Set[String] = Set("longArrayKey").asJava assertThat(meta1.equalsIgnoreKeys(meta3, emptyIgnore)).isFalse assertThat(meta1.equalsIgnoreKeys(meta3, ignoreLongArrayKey)).isTrue } test("equalsIgnoreKeys handles case where only one side has the ignored key") { val meta1 = FieldMetadata.builder() .putString("collation", "EN_CI") .putString("common", "v") .build() val meta2 = FieldMetadata.builder() .putString("common", "v") .build() val ignoreCollation: java.util.Set[String] = Set("collation").asJava val emptyIgnore: java.util.Set[String] = new java.util.HashSet[String]() assertThat(meta1.equalsIgnoreKeys(meta2, ignoreCollation)).isTrue assertThat(meta1.equalsIgnoreKeys(meta2, emptyIgnore)).isFalse } test("equalsIgnoreKeys handles entries with null value") { val meta1 = FieldMetadata.builder() .putString("nullableKey", null) .putString("same", "x") .build() val meta2 = FieldMetadata.builder() .putString("nullableKey", null) .putString("same", "x") .build() val meta3 = FieldMetadata.builder() .putString("nullableKey", "value") .putString("same", "x") .build() val emptyIgnore: java.util.Set[String] = new java.util.HashSet[String]() val ignoreNullable: java.util.Set[String] = Set("nullableKey").asJava assertThat(meta1.equalsIgnoreKeys(meta2, emptyIgnore)).isTrue assertThat(meta1.equalsIgnoreKeys(meta2, ignoreNullable)).isTrue assertThat(meta1.equalsIgnoreKeys(meta3, emptyIgnore)).isFalse assertThat(meta1.equalsIgnoreKeys(meta3, ignoreNullable)).isTrue } test("equalsIgnoreKeys handles entries with null key") { val meta1 = FieldMetadata.builder() .putString(null, "A") .putString("same", "x") .build() val meta2 = FieldMetadata.builder() .putString(null, "A") .putString("same", "x") .build() val meta3 = FieldMetadata.builder() .putString(null, "B") .putString("same", "x") .build() val meta4 = FieldMetadata.builder() .putString("same", "x") .build() val emptyIgnore: java.util.Set[String] = new java.util.HashSet[String]() val ignoreNullKey: java.util.Set[String] = Set[String](null).asJava // same key/value pairs -> equal assertThat(meta1.equalsIgnoreKeys(meta2, emptyIgnore)).isTrue // different values under null key -> unequal unless null key is ignored assertThat(meta1.equalsIgnoreKeys(meta3, emptyIgnore)).isFalse assertThat(meta1.equalsIgnoreKeys(meta3, ignoreNullKey)).isTrue // one side missing the null key -> unequal unless null key is ignored assertThat(meta1.equalsIgnoreKeys(meta4, emptyIgnore)).isFalse assertThat(meta1.equalsIgnoreKeys(meta4, ignoreNullKey)).isTrue } test("equalsIgnoreKeys handles entry with null key and null value") { val meta1 = FieldMetadata.builder() .putString(null, null) .putString("same", "x") .build() val meta2 = FieldMetadata.builder() .putString(null, null) .putString("same", "x") .build() val emptyIgnore: java.util.Set[String] = new java.util.HashSet[String]() val ignoreNullKey: java.util.Set[String] = Set[String](null).asJava assertThat(meta1.equalsIgnoreKeys(meta2, emptyIgnore)).isTrue assertThat(meta1.equalsIgnoreKeys(meta2, ignoreNullKey)).isTrue } test("equalsIgnoreKeys throws when keys is null") { val meta1 = FieldMetadata.builder().putString("k", "v").build() val meta2 = FieldMetadata.builder().putString("k", "v").build() val e = intercept[IllegalArgumentException] { meta1.equalsIgnoreKeys(meta2, null.asInstanceOf[java.util.Set[String]]) } assert(e.getMessage == "keys must not be null") } test("equals validates all keys and values") { val meta1 = FieldMetadata.builder() .putString("collation", "UTF8_BINARY") .putString("otherKey", "same") .putLongArray("arr", Seq[java.lang.Long](1L, 2L).toArray) .build() val meta2 = FieldMetadata.builder() .putString("collation", "UTF8_BINARY") .putString("otherKey", "same") .putLongArray("arr", Seq[java.lang.Long](1L, 2L).toArray) .build() val meta3 = FieldMetadata.builder() .putString("collation", "EN_CI") // different -> should not be equal .putString("otherKey", "same") .putLongArray("arr", Seq[java.lang.Long](1L, 2L).toArray) .build() assertThat(meta1.equals(meta2)).isTrue assertThat(meta1.equals(meta3)).isFalse } test("equals handles arrays properly") { val longs: Seq[java.lang.Long] = Seq(1L, 2L, 3L) val doubles: Seq[java.lang.Double] = Seq(1.0, 2.0, 3.0) val booleans: Seq[java.lang.Boolean] = Seq(true, false, true) val strings: Seq[java.lang.String] = Seq("x", "y", "z") val stringsWithNulls: Seq[java.lang.String] = Seq("a", null, "b") val inner1 = FieldMetadata.builder().putBoolean("k", true).build() val inner2 = FieldMetadata.builder().putBoolean("k", true).build() val meta1 = FieldMetadata.builder() .putLongArray("longArrayKey", longs.toArray) .putDoubleArray("doubleArrayKey", doubles.toArray) .putBooleanArray("booleanArrayKey", booleans.toArray) .putStringArray("stringArrayKey", strings.toArray) .putStringArray("stringArrayWithNulls", stringsWithNulls.toArray) .putFieldMetadataArray("fieldMetadataArrayKey", Array(inner1)) .build() val meta2 = FieldMetadata.builder() .putLongArray("longArrayKey", longs.toArray) .putDoubleArray("doubleArrayKey", doubles.toArray) .putBooleanArray("booleanArrayKey", booleans.toArray) .putStringArray("stringArrayKey", strings.toArray) .putStringArray("stringArrayWithNulls", stringsWithNulls.toArray) .putFieldMetadataArray("fieldMetadataArrayKey", Array(inner2)) .build() assertThat(meta1.equals(meta2)).isTrue // Change one array element to ensure inequality is detected val meta3 = FieldMetadata.builder() .putLongArray("longArrayKey", Seq[java.lang.Long](1L, 2L, 99L).toArray) .putDoubleArray("doubleArrayKey", doubles.toArray) .putBooleanArray("booleanArrayKey", booleans.toArray) .putStringArray("stringArrayKey", strings.toArray) .putStringArray("stringArrayWithNulls", stringsWithNulls.toArray) .putFieldMetadataArray("fieldMetadataArrayKey", Array(inner2)) .build() assertThat(meta1.equals(meta3)).isFalse } test("equals handles case where only one side has the key") { val meta1 = FieldMetadata.builder() .putString("collation", "EN_CI") .putString("common", "v") .build() val meta2 = FieldMetadata.builder() .putString("common", "v") .build() assertThat(meta1.equals(meta2)).isFalse assertThat(meta2.equals(meta1)).isFalse } test("equals handles entries with null value") { val meta1 = FieldMetadata.builder() .putString("nullableKey", null) .putString("same", "x") .build() val meta2 = FieldMetadata.builder() .putString("nullableKey", null) .putString("same", "x") .build() val meta3 = FieldMetadata.builder() .putString("nullableKey", "value") .putString("same", "x") .build() assertThat(meta1.equals(meta2)).isTrue assertThat(meta1.equals(meta3)).isFalse } test("equals handles entries with null key") { val meta1 = FieldMetadata.builder() .putString(null, "A") .putString("same", "x") .build() val meta2 = FieldMetadata.builder() .putString(null, "A") .putString("same", "x") .build() val meta3 = FieldMetadata.builder() .putString(null, "B") .putString("same", "x") .build() val meta4 = FieldMetadata.builder() .putString("same", "x") .build() // same key/value pairs -> equal assertThat(meta1.equals(meta2)).isTrue // different values under null key -> unequal assertThat(meta1.equals(meta3)).isFalse // one side missing the null key -> unequal assertThat(meta1.equals(meta4)).isFalse } test("equals handles entry with null key and null value") { val meta1 = FieldMetadata.builder() .putString(null, null) .putString("same", "x") .build() val meta2 = FieldMetadata.builder() .putString(null, null) .putString("same", "x") .build() assertThat(meta1.equals(meta2)).isTrue } test("equals returns false for null or different class") { val meta = FieldMetadata.builder().putString("k", "v").build() assertThat(meta.equals(null)).isFalse assertThat(meta.equals("not-metadata")).isFalse } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/types/MetadataColumnSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types import io.delta.kernel.utils.MetadataColumnTestUtils import org.scalatest.funsuite.AnyFunSuite class MetadataColumnSuite extends AnyFunSuite with MetadataColumnTestUtils { test("add metadata columns to schema") { val schema = new StructType() .add("number", IntegerType.INTEGER) .add("name", StringType.STRING) .addMetadataColumn("_metadata.row_index", MetadataColumnSpec.ROW_INDEX) // We compare using addMetadataColumn() against manually adding the expected metadata columns // as provided by MetadataColumnTestUtils val expected = new StructType() .add("number", IntegerType.INTEGER) .add("name", StringType.STRING) .add(ROW_INDEX) assert(schema.equals(expected)) } test("fail if metadata column already exists in schema") { val schema = new StructType() .add("number", IntegerType.INTEGER) .add("name", StringType.STRING) .addMetadataColumn("_metadata.row_index", MetadataColumnSpec.ROW_INDEX) // Adding the same metadata column should fail val e = intercept[IllegalArgumentException] { schema.addMetadataColumn("some other name", MetadataColumnSpec.ROW_INDEX) } assert(e.getMessage.contains("Duplicate metadata column row_index found in struct type")) // Adding a different metadata column should not fail val updated = schema.addMetadataColumn("_metadata.row_id", MetadataColumnSpec.ROW_ID) val expected = new StructType() .add("number", IntegerType.INTEGER) .add("name", StringType.STRING) .add(ROW_INDEX) .add(ROW_ID) assert(updated.equals(expected)) } test("fail if metadata column is nested") { val schema = new StructType() // Adding a nested metadata column should fail val e1 = intercept[IllegalArgumentException] { schema.add(new StructField( "struct", new StructType().addMetadataColumn("row_index", MetadataColumnSpec.ROW_INDEX), false)) } assert( e1.getMessage.contains("Metadata columns are only allowed at the top level of a schema.")) // Verify two-level nesting fails val e2 = intercept[IllegalArgumentException] { schema.add(new StructField( "struct", new StructType().add(new StructField( "inner_struct", new StructType().addMetadataColumn("row_index", MetadataColumnSpec.ROW_INDEX), false)), false)) } assert( e2.getMessage.contains("Metadata columns are only allowed at the top level of a schema.")) // Verify metadata in map type fails val e3 = intercept[IllegalArgumentException] { schema.add(new StructField( "map", new MapType( StringType.STRING, new StructType().addMetadataColumn("row_index", MetadataColumnSpec.ROW_INDEX), false), false)) } assert( e3.getMessage.contains("Metadata columns are only allowed at the top level of a schema.")) // Verify metadata in array type fails val e4 = intercept[IllegalArgumentException] { schema.add(new StructField( "array", new ArrayType( new StructType().addMetadataColumn("row_index", MetadataColumnSpec.ROW_INDEX), false), false)) } assert( e4.getMessage.contains("Metadata columns are only allowed at the top level of a schema.")) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/types/StringTypeSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types import org.scalatest.funsuite.AnyFunSuite class StringTypeSuite extends AnyFunSuite { test("check equals") { // Testcase: (instance1, instance2, expected value for `instance1 == instance2`) Seq( ( StringType.STRING, StringType.STRING, true), ( StringType.STRING, new StringType("sPark.UTF8_bINary"), true), ( StringType.STRING, new StringType("SPARK.UTF8_LCASE"), false), ( new StringType("ICU.UNICODE"), new StringType("SPARK.UTF8_LCASE"), false), ( new StringType("ICU.UNICODE"), new StringType("ICU.UNICODE_CI"), false), ( new StringType("ICU.UNICODE_CI"), new StringType("icU.uniCODe_Ci"), true)).foreach { case (st1, st2, expResult) => assert(st1.equals(st2) == expResult) } } test("isUTF8BinaryCollated") { assert(StringType.STRING.isUTF8BinaryCollated) assert(new StringType("sPark.UTF8_bINary").isUTF8BinaryCollated) assert(!new StringType("SPARK.UTF8_LCASE").isUTF8BinaryCollated) assert(!new StringType("ICU.UNICODE.72.2").isUTF8BinaryCollated) assert(!new StringType("ICU.UNICODE_CI").isUTF8BinaryCollated) } test("toString") { assert(StringType.STRING.toString == "string") assert(new StringType("sPark.UTF8_bINary").toString == "string") assert(new StringType("SPARK.UTF8_LCASE").toString == "string collate UTF8_LCASE") assert(new StringType("ICU.uNICoDE.72.2").toString == "string collate UNICODE") assert(new StringType("ICU.UNICODE_CI").toString == "string collate UNICODE_CI") } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/types/StructFieldSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types import java.util.ArrayList import io.delta.kernel.exceptions.KernelException import io.delta.kernel.types.StructField.{COLLATIONS_METADATA_KEY, DELTA_TYPE_CHANGES_KEY, FIELD_PATH_KEY, FROM_TYPE_KEY, TO_TYPE_KEY} import collection.JavaConverters._ import org.scalatest.funsuite.AnyFunSuite /** * Test suite for [[StructField]] class. */ class StructFieldSuite extends AnyFunSuite { // Test equality and hashcode test("equality and hashcode") { val field1 = new StructField( "field", LongType.LONG, true, FieldMetadata.empty(), Seq(new TypeChange(IntegerType.INTEGER, LongType.LONG)).asJava) val field2 = new StructField( "field", LongType.LONG, true, FieldMetadata.empty(), Seq(new TypeChange(IntegerType.INTEGER, LongType.LONG)).asJava) val field3 = new StructField("differentField", IntegerType.INTEGER, true) val field4 = new StructField("field", StringType.STRING, true) val field5 = new StructField("field", IntegerType.INTEGER, false) val field6 = new StructField( "field", IntegerType.INTEGER, true, FieldMetadata.builder().putBoolean("a", true).build(), Seq(new TypeChange(IntegerType.INTEGER, LongType.LONG)).asJava) val field7 = new StructField( "field", LongType.LONG, true, FieldMetadata.empty(), Seq(new TypeChange(IntegerType.INTEGER, StringType.STRING)).asJava) assert(field1 == field2) assert(field1.hashCode() == field2.hashCode()) assert(field1 != field3) assert(field1 != field4) assert(field1 != field5) assert(field1 != field6) assert(field1 != field7) } Seq( new StructType(), new ArrayType(LongType.LONG, false), new MapType(LongType.LONG, LongType.LONG, false)).foreach { dataType => test(s"withType should throw exception with change types for nested types $dataType") { val field = new StructField( "field", dataType, true) assertThrows[KernelException] { field.withTypeChanges(Seq(new TypeChange(IntegerType.INTEGER, LongType.LONG)).asJava) } } test(s"Constructor should throw exception with change types for nested types $dataType") { assertThrows[KernelException] { new StructField( "field", dataType, true, FieldMetadata.empty(), Seq(new TypeChange(IntegerType.INTEGER, LongType.LONG)).asJava) } } } // Test metadata column detection test("metadata column detection") { val regularField = new StructField("regularField", IntegerType.INTEGER, true) assert(!regularField.isMetadataColumn) assert(regularField.isDataColumn) // Create a metadata field val metadataFieldName = "_metadata.custom" val metadataBuilder = FieldMetadata.builder() metadataBuilder.putMetadataColumnSpec( StructField.METADATA_SPEC_KEY, MetadataColumnSpec.ROW_INDEX) val metadataField = new StructField(metadataFieldName, LongType.LONG, false, metadataBuilder.build()) assert(metadataField.isMetadataColumn) assert(!metadataField.isDataColumn) } // Test withNewMetadata method test("withNewMetadata") { val originalField = new StructField("field", IntegerType.INTEGER, true) assert(originalField.getMetadata() == FieldMetadata.empty()) val newMetadataBuilder = FieldMetadata.builder() newMetadataBuilder.putString("key", "value") val newMetadata = newMetadataBuilder.build() val updatedField = originalField.withNewMetadata(newMetadata) assert(updatedField.getName == originalField.getName) assert(updatedField.getDataType == originalField.getDataType) assert(updatedField.isNullable == originalField.isNullable) assert(updatedField.getMetadata == newMetadata) assert(updatedField.getMetadata.getString("key") == "value") } // Test type changes test("type changes") { val originalField = new StructField( "field", IntegerType.INTEGER, true, FieldMetadata.builder().putString("a", "b").build()) assert(originalField.getTypeChanges.isEmpty) val typeChanges = new ArrayList[TypeChange]() typeChanges.add(new TypeChange(IntegerType.INTEGER, LongType.LONG)) val updatedField = originalField.withTypeChanges(typeChanges) assert(updatedField.getName == originalField.getName) assert(updatedField.getDataType == originalField.getDataType) assert(updatedField.isNullable == originalField.isNullable) assert(updatedField.getMetadata == FieldMetadata.builder() .putString("a", "b") .putFieldMetadataArray( "delta.typeChanges", Array(FieldMetadata.builder() .putString("fromType", "integer") .putString("toType", "long").build())) .build()) assert(updatedField.getTypeChanges.size() == 1) val typeChange = updatedField.getTypeChanges.get(0) assert(typeChange.getFrom == IntegerType.INTEGER) assert(typeChange.getTo == LongType.LONG) } // Test TypeChange class test("TypeChange class") { val from = IntegerType.INTEGER val to = LongType.LONG val typeChange = new TypeChange(from, to) assert(typeChange.getFrom == from) assert(typeChange.getTo == to) // Test equals and hashCode val sameTypeChange = new TypeChange(IntegerType.INTEGER, LongType.LONG) val differentTypeChange = new TypeChange(IntegerType.INTEGER, StringType.STRING) assert(typeChange == sameTypeChange) assert(typeChange.hashCode() == sameTypeChange.hashCode()) assert(typeChange != differentTypeChange) } // Sequence of tuples containing StructFields with type changes and their expected FieldMetadata Seq( // Simple primitive type change: Integer -> Long ( new StructField( "intToLongField", LongType.LONG, true, FieldMetadata.empty(), Seq(new TypeChange(IntegerType.INTEGER, LongType.LONG)).asJava), FieldMetadata.builder() .putFieldMetadataArray( DELTA_TYPE_CHANGES_KEY, Array( FieldMetadata.builder() .putString(FROM_TYPE_KEY, "integer") .putString(TO_TYPE_KEY, "long") .build())) .build()), // Multiple type changes: Integer -> Long -> Decimal ( new StructField( "multiTypeChangeField", new DecimalType(10, 2), true, FieldMetadata.empty(), Seq( new TypeChange(IntegerType.INTEGER, LongType.LONG), new TypeChange(LongType.LONG, new DecimalType(10, 2))).asJava), FieldMetadata.builder() .putFieldMetadataArray( DELTA_TYPE_CHANGES_KEY, Array( FieldMetadata.builder() .putString(FROM_TYPE_KEY, "integer") .putString(TO_TYPE_KEY, "long") .build(), FieldMetadata.builder() .putString(FROM_TYPE_KEY, "long") .putString(TO_TYPE_KEY, "decimal(10,2)") .build())) .build()), // Float -> Double type change with additional metadata ( new StructField( "floatToDoubleField", DoubleType.DOUBLE, true, FieldMetadata.builder().putString("description", "A field with type change").build(), Seq(new TypeChange(FloatType.FLOAT, DoubleType.DOUBLE)).asJava), FieldMetadata.builder() .putString("description", "A field with type change") .putFieldMetadataArray( DELTA_TYPE_CHANGES_KEY, Array( FieldMetadata.builder() .putString(FROM_TYPE_KEY, "float") .putString(TO_TYPE_KEY, "double") .build())) .build()), // Type change in array element type ( new StructField( "arrayField", new ArrayType(new StructField("element", LongType.LONG, true) .withTypeChanges(Seq(new TypeChange(ShortType.SHORT, LongType.LONG)).asJava)), true), FieldMetadata.builder() .putFieldMetadataArray( DELTA_TYPE_CHANGES_KEY, Array( FieldMetadata.builder() .putString(FROM_TYPE_KEY, "short") .putString(TO_TYPE_KEY, "long") .putString(FIELD_PATH_KEY, "element") .build())) .build()), // Type change in map value type ( new StructField( "mapField", new MapType( new StructField("key", StringType.STRING, false), new StructField("value", LongType.LONG, true).withTypeChanges( Seq( new TypeChange(ShortType.SHORT, IntegerType.INTEGER), new TypeChange(IntegerType.INTEGER, LongType.LONG)).asJava)), false), FieldMetadata.builder() .putFieldMetadataArray( DELTA_TYPE_CHANGES_KEY, Array( FieldMetadata.builder() .putString(FROM_TYPE_KEY, "short") .putString(TO_TYPE_KEY, "integer") .putString(FIELD_PATH_KEY, "value") .build(), FieldMetadata.builder() .putString(FROM_TYPE_KEY, "integer") .putString(TO_TYPE_KEY, "long") .putString(FIELD_PATH_KEY, "value") .build())) .build()), // Complex nested type with multiple type changes ( new StructField( "complexField", new ArrayType(new StructField( "element", new MapType( new StructField( "key", new ArrayType(new StructField("element", LongType.LONG, false) .withTypeChanges(Seq(new TypeChange(ShortType.SHORT, LongType.LONG)).asJava)), false), new StructField( "value", new MapType( new StructField("key", ShortType.SHORT, false), new StructField("value", LongType.LONG, true) .withTypeChanges(Seq(new TypeChange(ByteType.BYTE, LongType.LONG)).asJava)), false)), false)), false), FieldMetadata.builder() .putFieldMetadataArray( DELTA_TYPE_CHANGES_KEY, Array( FieldMetadata.builder() .putString(FROM_TYPE_KEY, "short") .putString(TO_TYPE_KEY, "long") .putString(FIELD_PATH_KEY, "element.key.element") .build(), FieldMetadata.builder() .putString(FROM_TYPE_KEY, "byte") .putString(TO_TYPE_KEY, "long") .putString(FIELD_PATH_KEY, "element.value.value") .build())) .build())).foreach { case (structField, expectedMetadata) => test(s"$structField has expected metadata") { assert(structField.getMetadata == expectedMetadata) } test(s"$structField does not leak field metadata if it is a child struct field.") { // Field metadata for type changes is stored at the nearest ancestor of the type // sho it shouldn't leak up. assert(new StructField("parent", new StructType().add(structField), false).getMetadata == FieldMetadata.empty()) assert(new StructField( "parent", new ArrayType(new StructType().add(structField), false), false).getMetadata == FieldMetadata.empty()) assert(new StructField( "parent", new MapType(new StructType().add(structField), new StructType().add(structField), false), false).getMetadata == FieldMetadata.empty()) } } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/types/TypesSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.types import java.util.Arrays import org.scalatest.funsuite.AnyFunSuite class TypesSuite extends AnyFunSuite { test("isNested - false") { // All primitive types should return false for isNested val primitiveTypes = Seq( BinaryType.BINARY, BooleanType.BOOLEAN, ByteType.BYTE, DateType.DATE, new DecimalType(10, 2), DoubleType.DOUBLE, FloatType.FLOAT, IntegerType.INTEGER, LongType.LONG, ShortType.SHORT, StringType.STRING, TimestampType.TIMESTAMP, TimestampNTZType.TIMESTAMP_NTZ, VariantType.VARIANT) primitiveTypes.foreach { dataType => assert(!dataType.isNested(), s"Expected $dataType to not be nested") } } test("isNested - nested types") { // Create instances of nested types val structFields = Arrays.asList( new StructField("field1", IntegerType.INTEGER, true), new StructField("field2", StringType.STRING, true)) val structType = new StructType(structFields) val arrayType = new ArrayType( new StructField("element", IntegerType.INTEGER, true)) val mapType = new MapType( new StructField("key", StringType.STRING, false), new StructField("value", IntegerType.INTEGER, true)) // All nested types should return true for isNested val nestedTypes = Seq(structType, arrayType, mapType) nestedTypes.foreach { dataType => assert(dataType.isNested(), s"Expected $dataType to be nested") } } test("MapType constructor throws IllegalArgumentException for collated StringType keys") { // Test multiple collation providers to ensure all non-default collations are rejected Seq("SPARK.UTF8_LCASE", "ICU.UNICODE_CI").foreach { collation => val collatedString = new StringType(collation) // 3-arg constructor val ex1 = intercept[IllegalArgumentException] { new MapType(collatedString, IntegerType.INTEGER, false) } assert(ex1.getMessage.contains("does not support collated string types as keys")) assert(ex1.getMessage.contains("UTF8_BINARY")) assert( ex1.getMessage.contains(collatedString.toString), s"Error message should include the found type but was: ${ex1.getMessage}") // 2-arg StructField constructor val ex2 = intercept[IllegalArgumentException] { new MapType( new StructField("key", collatedString, false), new StructField("value", IntegerType.INTEGER, true)) } assert(ex2.getMessage.contains("does not support collated string types as keys")) assert( ex2.getMessage.contains(collatedString.toString), s"Error message should include the found type but was: ${ex2.getMessage}") } } test("MapType allows default UTF8_BINARY StringType keys and non-string keys") { val map1 = new MapType(StringType.STRING, IntegerType.INTEGER, false) assert(map1.getKeyType === StringType.STRING) val utf8BinaryString = new StringType("SPARK.UTF8_BINARY") val map2 = new MapType(utf8BinaryString, IntegerType.INTEGER, false) assert(map2.getKeyType === utf8BinaryString) val map3 = new MapType(IntegerType.INTEGER, StringType.STRING, true) assert(map3.getKeyType === IntegerType.INTEGER) } test("MapType allows collated StringType as values") { val collatedString = new StringType("SPARK.UTF8_LCASE") val map = new MapType(StringType.STRING, collatedString, true) assert(map.getValueType === collatedString) } } ================================================ FILE: kernel/kernel-api/src/test/scala/io/delta/kernel/utils/MetadataColumnTestUtils.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.utils import io.delta.kernel.types.{MetadataColumnSpec, StructField} import io.delta.kernel.types.StructField.createMetadataColumn trait MetadataColumnTestUtils { val ROW_INDEX: StructField = createMetadataColumn("_metadata.row_index", MetadataColumnSpec.ROW_INDEX) val ROW_ID: StructField = createMetadataColumn("_metadata.row_id", MetadataColumnSpec.ROW_ID) val ROW_COMMIT_VERSION: StructField = createMetadataColumn("_metadata.row_commit_version", MetadataColumnSpec.ROW_COMMIT_VERSION) } ================================================ FILE: kernel/kernel-benchmarks/src/test/java/io/delta/kernel/benchmarks/AbstractBenchmarkState.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.benchmarks; import io.delta.kernel.benchmarks.models.WorkloadSpec; import io.delta.kernel.benchmarks.workloadrunners.WorkloadRunner; import io.delta.kernel.engine.*; import org.openjdk.jmh.annotations.*; /** * Base state class for all benchmark state. This class is responsible for setting up the {@link * WorkloadRunner} based on the {@link WorkloadSpec} and engine parameters provided by JMH. * *

To add support for a new engine, extend this class and implement the {@link * #getEngine(String)} method to return an instance of the desired engine based on the provided * engine name. * * @see WorkloadRunner * @see WorkloadSpec */ @State(Scope.Thread) public abstract class AbstractBenchmarkState { /** * The json representation of the workload specification. Note: This parameter will be set * dynamically by JMH. The value is set in the main method. */ @Param({}) private String workloadSpecJson; /** * The engine to use for this benchmark. Note: This parameter will be set dynamically by JMH. The * value is set in the main method. */ @Param({}) private String engineName; /** The workload runner initialized for this benchmark invocation. */ private WorkloadRunner runner; /** * Parses the workload specification from JSON and initializes the benchmarking engine. This also * sets up the workload runner. * * @throws Exception If any error occurs during setup. */ @Setup(Level.Trial) public void setupTrial() throws Exception { WorkloadSpec spec = WorkloadSpec.fromJsonString(workloadSpecJson); Engine engine = KernelMetricsProfiler.BenchmarkingEngine.wrapEngine(getEngine(engineName)); runner = spec.getRunner(engine); } /** * Setup method that runs before each benchmark invocation. This calls the {@link * WorkloadRunner#setup()} to set up the workload runner. * * @throws Exception If any error occurs during setup. */ @Setup(Level.Invocation) public void setupInvocation() throws Exception { runner.setup(); } /** * Teardown method that runs after each benchmark invocation. This calls the {@link * WorkloadRunner#cleanup()} to clean up any state created during execution. * * @throws Exception If any error occurs during cleanup. */ @TearDown(Level.Invocation) public void teardownInvocation() throws Exception { runner.cleanup(); } /** * Returns an instance of the desired engine based on the provided engine name. * * @param engineName The name of the engine to instantiate. * @return An instance of the specified engine. */ protected abstract Engine getEngine(String engineName); /** @return The workload specification for this benchmark invocation. */ public WorkloadSpec getWorkloadSpecification() { return getRunner().getWorkloadSpec(); } /** @return The workload runner initialized for this benchmark invocation. */ public WorkloadRunner getRunner() { return runner; } } ================================================ FILE: kernel/kernel-benchmarks/src/test/java/io/delta/kernel/benchmarks/BenchmarkParallelCheckpointReading.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.benchmarks; import static java.util.concurrent.Executors.newFixedThreadPool; import io.delta.kernel.*; import io.delta.kernel.data.*; import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.defaults.engine.DefaultParquetHandler; import io.delta.kernel.defaults.engine.fileio.FileIO; import io.delta.kernel.defaults.engine.hadoopio.HadoopFileIO; import io.delta.kernel.defaults.internal.parquet.ParquetFileReader; import io.delta.kernel.engine.Engine; import io.delta.kernel.engine.FileReadResult; import io.delta.kernel.engine.ParquetHandler; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import java.io.IOException; import java.util.*; import java.util.concurrent.*; import org.apache.hadoop.conf.Configuration; import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.infra.Blackhole; /** * Benchmark to measure the performance of reading multi-part checkpoint files, using a custom * ParquetHandler that reads the files in parallel. To run this benchmark (from delta repo root): * *

    *
  • Generate the test table by following the instructions at `testTablePath` member variable. *
  • * 
    {@code
 * build/sbt sbt:delta> project kernelDefaults
 * sbt:delta> set fork in run := true sbt:delta>
 * sbt:delta> test:runMain \
 *   io.delta.kernel.benchmarks.BenchmarkParallelCheckpointReading.
 *
 * }
    *
* *

Sample benchmarks on a table with checkpoint (13) parts containing total of 1.3mil actions on * Macbook Pro M2 Max with table stored locally. * * 

{@code
 * Benchmark  (parallelReaderCount)  Mode  Cnt Score Error  Units
 * benchmark                      0  avgt    5  1565.520 ±  20.551  ms/op
 * benchmark                      1  avgt    5  1064.850 ±  19.699  ms/op
 * benchmark                      2  avgt    5   785.918 ± 176.285  ms/op
 * benchmark                      4  avgt    5   729.487 ±  51.470  ms/op
 * benchmark                     10  avgt    5   693.757 ±  41.252  ms/op
 * benchmark                     20  avgt    5   702.656 ±  19.145  ms/op
 * }
*/ @State(Scope.Benchmark) @OutputTimeUnit(TimeUnit.MILLISECONDS) @Warmup(iterations = 3) @Fork(1) public class BenchmarkParallelCheckpointReading { /** * Following are the steps to generate a simple large table with multi-part checkpoint files * * 
{@code
   * bin/spark-shell --packages io.delta:delta-spark_2.12:3.1.0 \
   *   --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" \
   *   --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" \
   *   --conf "spark.databricks.delta.checkpoint.partSize=100000"
   *
   * # Within the Spark shell, run the following commands
   * scala> spark.range(0, 100000) .withColumn("pCol", 'id % 100000) .repartition(10)
   *   .write.format("delta") .partitionBy("pCol") .mode("append")
   *   .save("~/test-tables/large-table")
   *
   * # Repeat the above steps for each of the next ranges # 100000 to 200000, 200000 to 300000 etc
   * until enough log entries are reached.
   *
   * # Then create a checkpoint
   * # This step create a multi-part checkpoint with each checkpoint containing 100K records.
   * scala> import org.apache.spark.sql.delta.DeltaLog
   * scala> DeltaLog.forTable(spark, "~/test-tables/large-table").checkpoint()
   * }
*/ public static final String testTablePath = " fill the path here"; @State(Scope.Benchmark) public static class BenchmarkData { // Variations of number of threads to read the multi-part checkpoint files // When thread count is 0, we read using the current default parquet handler implementation // In all other cases we use the custom parallel parquet handler implementation defined // in this benchmark @Param({"0", "1", "2", "4", "10", "20"}) private int parallelReaderCount = 0; } @Benchmark @BenchmarkMode(Mode.AverageTime) public void benchmark(BenchmarkData benchmarkData, Blackhole blackhole) throws Exception { Engine engine = createEngine(benchmarkData.parallelReaderCount); Table table = Table.forPath(engine, testTablePath); Snapshot snapshot = table.getLatestSnapshot(engine); ScanBuilder scanBuilder = snapshot.getScanBuilder(); Scan scan = scanBuilder.build(); // Scan state is not used, but get it so that we simulate the real use case. Row row = scan.getScanState(engine); blackhole.consume(row); // To avoid dead code elimination by the JIT compiler long fileSize = 0; try (CloseableIterator batchIter = scan.getScanFiles(engine)) { while (batchIter.hasNext()) { FilteredColumnarBatch batch = batchIter.next(); try (CloseableIterator rowIter = batch.getRows()) { while (rowIter.hasNext()) { Row r = rowIter.next(); long size = r.getStruct(0).getLong(2); fileSize += size; } } } } // Consume the result to avoid dead code elimination by the JIT compiler blackhole.consume(fileSize); } public static void main(String[] args) throws Exception { org.openjdk.jmh.Main.main(args); } private static Engine createEngine(int numberOfParallelThreads) { FileIO fileIO = new HadoopFileIO(new Configuration()); if (numberOfParallelThreads <= 0) { return DefaultEngine.create(fileIO); } return new DefaultEngine(fileIO) { @Override public ParquetHandler getParquetHandler() { return new ParallelParquetHandler(fileIO, numberOfParallelThreads); } }; } /** * Custom implementation of {@link ParquetHandler} to read the Parquet files in parallel. Reason * for this not being in the {@link DefaultParquetHandler} is that this implementation keeps the * contents of the Parquet files in memory, which is not suitable for default implementation * without a proper design that allows limits on the memory usage. If the parallel reading of * checkpoint becomes a common in connectors, we can look at adding the functionality in the * default implementation. */ static class ParallelParquetHandler extends DefaultParquetHandler { private final FileIO fileIO; private final int numberOfParallelThreads; ParallelParquetHandler(FileIO fileIO, int numberOfParallelThreads) { super(fileIO); this.fileIO = fileIO; this.numberOfParallelThreads = numberOfParallelThreads; } @Override public CloseableIterator readParquetFiles( CloseableIterator fileIter, StructType physicalSchema, Optional predicate) throws IOException { return new CloseableIterator() { // Executor service will be closed as part of the returned `CloseableIterator`'s // close method. private final ExecutorService executorService = newFixedThreadPool(numberOfParallelThreads); private Iterator>> futuresIter; private Iterator currentBatchIter; @Override public void close() throws IOException { Utils.closeCloseables(fileIter, () -> executorService.shutdown()); } @Override public boolean hasNext() { submitReadRequestsIfNotDone(); if (currentBatchIter != null && currentBatchIter.hasNext()) { return true; } if (futuresIter.hasNext()) { try { currentBatchIter = futuresIter.next().get().iterator(); return hasNext(); } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); } } return false; } @Override public FileReadResult next() { return currentBatchIter.next(); } private void submitReadRequestsIfNotDone() { if (futuresIter != null) { return; } List>> futures = new ArrayList<>(); while (fileIter.hasNext()) { futures.add( executorService.submit(() -> parquetFileReader(fileIter.next(), physicalSchema))); } futuresIter = futures.iterator(); } }; } List parquetFileReader(FileStatus fileStatus, StructType readSchema) { ParquetFileReader reader = new ParquetFileReader(fileIO); try (CloseableIterator batchIter = reader.read(fileStatus, readSchema, Optional.empty())) { List batches = new ArrayList<>(); while (batchIter.hasNext()) { batches.add(new FileReadResult(batchIter.next(), fileStatus.getPath())); } return batches; } catch (IOException e) { throw new RuntimeException(e); } } } } ================================================ FILE: kernel/kernel-benchmarks/src/test/java/io/delta/kernel/benchmarks/BenchmarkUtils.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.benchmarks; import io.delta.kernel.benchmarks.models.TableInfo; import io.delta.kernel.benchmarks.models.WorkloadSpec; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; /** Useful utilities and values for benchmarks. */ public class BenchmarkUtils { public static final Path RESOURCES_DIR = getResourcesDirectory(); public static final Path WORKLOAD_SPECS_DIR = RESOURCES_DIR.resolve("workload_specs"); private static final String DELTA_DIR_NAME = "delta"; private static final String SPECS_DIR_NAME = "specs"; private static final String SPEC_FILE_NAME = "spec.json"; private static final String TABLE_INFO_FILE_NAME = "table_info.json"; /** * Gets the resources directory, ensuring user.dir system property is set. * * @return the path to the test resources directory * @throws IllegalStateException if user.dir system property is not set */ private static Path getResourcesDirectory() { String userDir = System.getProperty("user.dir"); if (userDir == null || userDir.trim().isEmpty()) { throw new IllegalStateException( "System property 'user.dir' is not set. This is required to locate test resources."); } return Paths.get(userDir + "/src/test/resources"); } /** * Scans the workloads directory and loads all JSON workload specifications. * *

This method: * *

    *
  1. Finds all table directories in the workload specs directory *
  2. Loads table_info.json from each table directory *
  3. Loads all spec.json files from specs/ subdirectories *
  4. Enriches each spec with tableInfo and caseName *
  5. Returns loaded specs (not yet expanded into variants) *
* *

Note: Variant generation happens later via {@link WorkloadSpec#getWorkloadVariants()}. * * @param specDirPath Path to the directory containing workload specifications * @return List of loaded workload specifications (base specs, not variants) * @throws WorkloadLoadException if workloads cannot be loaded */ public static List loadAllWorkloads(Path specDirPath) { validateWorkloadDirectory(specDirPath); List tableDirectories = findTableDirectories(specDirPath); return tableDirectories.stream() .flatMap(tableDir -> loadSpecsFromTable(tableDir).stream()) .collect(Collectors.toList()); } /** Validates that the workload directory exists and is accessible. */ private static void validateWorkloadDirectory(Path specDirPath) { if (!Files.exists(specDirPath)) { throw new WorkloadLoadException("Workload directory does not exist: " + specDirPath); } if (!Files.isDirectory(specDirPath)) { throw new WorkloadLoadException("Path is not a directory: " + specDirPath); } if (!Files.isReadable(specDirPath)) { throw new WorkloadLoadException("Cannot read workload directory: " + specDirPath); } } /** Finds all table directories within the workload specifications directory. */ private static List findTableDirectories(Path specDirPath) { try (Stream files = Files.list(specDirPath)) { List tableDirectories = files.filter(Files::isDirectory).collect(Collectors.toList()); if (tableDirectories.isEmpty()) { throw new WorkloadLoadException("No table directories found in " + specDirPath); } return tableDirectories; } catch (IOException e) { throw new WorkloadLoadException("Failed to list table directories in " + specDirPath, e); } } /** Loads all workload specifications from a single table directory. */ private static List loadSpecsFromTable(Path tableDir) { validateTableStructure(tableDir); Path tableInfoPath = tableDir.resolve(TABLE_INFO_FILE_NAME); Path specsDir = tableDir.resolve(SPECS_DIR_NAME); TableInfo tableInfo = TableInfo.fromJsonPath(tableInfoPath.toString(), tableDir.toAbsolutePath().toString()); return findSpecDirectories(specsDir).stream() .map(specDir -> loadSingleSpec(specDir, tableInfo)) .collect(Collectors.toList()); } /** Validates that a table directory has the required structure. */ private static void validateTableStructure(Path tableDir) { Path deltaDir = tableDir.resolve(DELTA_DIR_NAME); Path specsDir = tableDir.resolve(SPECS_DIR_NAME); if (!Files.exists(deltaDir) || !Files.isDirectory(deltaDir)) { throw new WorkloadLoadException("Delta directory not found: " + deltaDir); } if (!Files.exists(specsDir) || !Files.isDirectory(specsDir)) { throw new WorkloadLoadException("Specs directory not found: " + specsDir); } } /** Finds all specification directories within the specs directory. */ private static List findSpecDirectories(Path specsDir) { try (Stream specDirs = Files.list(specsDir)) { List specDirectories = specDirs.filter(Files::isDirectory).collect(Collectors.toList()); if (specDirectories.isEmpty()) { throw new WorkloadLoadException("No spec directories found in " + specsDir); } return specDirectories; } catch (IOException e) { throw new WorkloadLoadException("Failed to list spec directories in " + specsDir, e); } } /** Loads a single workload specification from a spec directory. */ private static WorkloadSpec loadSingleSpec(Path specDir, TableInfo tableInfo) { Path specFile = specDir.resolve(SPEC_FILE_NAME); if (!Files.exists(specFile) || !Files.isRegularFile(specFile)) { throw new WorkloadLoadException("Spec file not found: " + specFile); } try { String specName = specDir.getFileName().toString(); WorkloadSpec workloadSpec = WorkloadSpec.fromJsonPath(specFile.toString(), specName, tableInfo); return workloadSpec; } catch (Exception e) { throw new WorkloadLoadException("Failed to parse spec file: " + specFile, e); } } /** Custom exception for workload loading errors. */ public static class WorkloadLoadException extends RuntimeException { public WorkloadLoadException(String message) { super(message); } public WorkloadLoadException(String message, Throwable cause) { super(message, cause); } } } ================================================ FILE: kernel/kernel-benchmarks/src/test/java/io/delta/kernel/benchmarks/KernelMetricsProfiler.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.benchmarks; import io.delta.kernel.engine.*; import io.delta.kernel.metrics.MetricsReport; import io.delta.kernel.metrics.ScanMetricsResult; import io.delta.kernel.metrics.ScanReport; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; import org.openjdk.jmh.infra.BenchmarkParams; import org.openjdk.jmh.infra.IterationParams; import org.openjdk.jmh.profile.InternalProfiler; import org.openjdk.jmh.results.*; import org.openjdk.jmh.util.SampleBuffer; /** * JMH profiler that extracts and reports Delta Kernel metrics during benchmark execution. * *

This profiler collects metrics reports from the Delta Kernel during benchmark runs and * converts them into JMH secondary results for analysis. It works by wrapping the benchmark engine * with a {@link BenchmarkingEngine} that captures all metrics reports. * *

The profiler extracts various scan metrics including planning duration, file counts, and other * performance-related measurements that can be analyzed alongside the primary benchmark timing * results. */ public class KernelMetricsProfiler implements InternalProfiler { /** * Creates a new KernelMetricsProfiler instance. * *

This constructor is called by JMH when the profiler is registered via {@code * addProfiler(KernelMetricsProfiler.class)}. No initialization is needed as the profiler uses * static state to collect metrics across all benchmark iterations. */ public KernelMetricsProfiler() {} public static final List reports = new ArrayList<>(); /** * Adds a metrics report to the collection for processing during benchmark execution. * * @param newReport the metrics report to add */ public static void addReport(MetricsReport newReport) { reports.add(newReport); } @Override public String getDescription() { return "Extracts metrics from the Delta Kernel metrics reports."; } @Override public void beforeIteration(BenchmarkParams benchmarkParams, IterationParams iterationParams) { reports.clear(); } /** * Generates a secondary scalar result with average aggregation policy for count metrics. * * @param name the name of the metric * @param value the metric value * @return a ScalarResult configured as a secondary metric for count data with average aggregation */ private static ScalarResult generateAvgScalarCount(String name, double value) { return new ScalarResult(name, value, "count", AggregationPolicy.AVG); } /** * Generates a secondary time sample result for timing metrics. * * @param name the name of the timing metric * @param value the timing value * @param unit the time unit for the value * @return a SampleTimeResult configured as a secondary metric for timing data */ private static SampleTimeResult generateTimeSample(String name, long value, TimeUnit unit) { SampleBuffer buf = new SampleBuffer(); buf.add(value); return new SampleTimeResult(ResultRole.SECONDARY, name, buf, unit); } /** * Extracts scan metrics from a scan report and converts them to JMH secondary results. * * @param report the scan report containing metrics to extract * @return a stream of JMH secondary Result objects representing the extracted scan metrics */ private Stream extractSecondaryScanMetrics(ScanReport report) { ScanMetricsResult scanReport = report.getScanMetrics(); Stream out = Stream.of( generateTimeSample( "scan.scan_metrics.total_planning_duration_ns", scanReport.getTotalPlanningDurationNs(), TimeUnit.NANOSECONDS), generateAvgScalarCount( "scan.scan_metrics.num_active_add_files", scanReport.getNumActiveAddFiles()), generateAvgScalarCount( "scan.scan_metrics.num_add_files_seen", scanReport.getNumAddFilesSeen()), generateAvgScalarCount( "scan.scan_metrics.num_add_files_seen_from_delta_files", scanReport.getNumAddFilesSeenFromDeltaFiles()), generateAvgScalarCount( "scan.scan_metrics.num_duplicate_add_files", scanReport.getNumDuplicateAddFiles()), generateAvgScalarCount( "scan.scan_metrics.num_remove_files_seen_from_delta_files", scanReport.getNumRemoveFilesSeenFromDeltaFiles())); return out; } @Override public Collection afterIteration( BenchmarkParams benchmarkParams, IterationParams iterationParams, IterationResult result) { Stream out = Stream.empty(); for (MetricsReport report : reports) { if (report instanceof ScanReport) { out = Stream.concat(out, extractSecondaryScanMetrics((ScanReport) report)); } } return out.collect(Collectors.toList()); } /** * An {@link Engine} implementation that wraps an existing engine and delegates all engine tasks. * The BenchmarkingEngine sends all metrics reports to the KernelMetricsProfiler for collection * during benchmarks. The metrics reports can then be extracted and reported by the * KernelMetricsProfiler. */ public static class BenchmarkingEngine implements Engine { private final Engine delegate; static final BenchmarkMetricsReporter BENCHMARK_METRICS_REPORTER = new BenchmarkMetricsReporter(); /** * Creates a new BenchmarkingEngine that wraps the provided engine. * * @param delegate the engine to wrap for metrics collection */ BenchmarkingEngine(Engine delegate) { this.delegate = delegate; } @Override public ExpressionHandler getExpressionHandler() { return delegate.getExpressionHandler(); } @Override public JsonHandler getJsonHandler() { return delegate.getJsonHandler(); } @Override public FileSystemClient getFileSystemClient() { return delegate.getFileSystemClient(); } @Override public ParquetHandler getParquetHandler() { return delegate.getParquetHandler(); } @Override public List getMetricsReporters() { return Collections.singletonList(BENCHMARK_METRICS_REPORTER); } /** * Wraps an engine with benchmarking metrics collection capabilities. * * @param engine the engine to wrap * @return a BenchmarkingEngine that collects metrics from the wrapped engine */ public static BenchmarkingEngine wrapEngine(Engine engine) { return new BenchmarkingEngine(engine); } /** Metrics reporter implementation that forwards all metrics reports to the profiler. */ private static final class BenchmarkMetricsReporter implements MetricsReporter { @Override public void report(MetricsReport report) { addReport(report); } } } } ================================================ FILE: kernel/kernel-benchmarks/src/test/java/io/delta/kernel/benchmarks/WorkloadBenchmark.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.benchmarks; import static io.delta.kernel.benchmarks.BenchmarkUtils.*; import io.delta.kernel.benchmarks.models.WorkloadSpec; import io.delta.kernel.benchmarks.workloadrunners.WorkloadRunner; import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.engine.*; import java.io.IOException; import java.util.*; import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.infra.Blackhole; import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; import org.openjdk.jmh.runner.options.Options; import org.openjdk.jmh.runner.options.OptionsBuilder; import org.openjdk.jmh.runner.options.TimeValue; /** * Generic JMH benchmark for all workload types. Automatically loads and runs benchmarks based on * JSON workload specifications. */ @BenchmarkMode(Mode.SampleTime) @OutputTimeUnit(TimeUnit.MILLISECONDS) @Fork(value = 1, warmups = 1) @Warmup(iterations = 3, time = 1) @Measurement(iterations = 5, time = 1) public class WorkloadBenchmark { /** Default implementation of BenchmarkState that supports only the "default" engine. */ public static class DefaultBenchmarkState extends AbstractBenchmarkState { @Override protected Engine getEngine(String engineName) { if (engineName.equals("default")) { return DefaultEngine.create(new Configuration()); } else { throw new IllegalArgumentException("Unsupported engine: " + engineName); } } } /** * Benchmark method that executes the workload runner specified in the state as a benchmark. * * @param state The benchmark state containing the workload runner to execute. * @param blackhole The Blackhole provided by JMH to consume results and prevent dead code * elimination. * @throws Exception If any error occurs during workload execution. */ @Benchmark public void benchmarkWorkload(DefaultBenchmarkState state, Blackhole blackhole) throws Exception { WorkloadRunner runner = state.getRunner(); runner.executeAsBenchmark(blackhole); } /** * TODO: In the future, this can be extracted so that new benchmarks with custom BenchmarkStates * can be easily constructed. */ public static void main(String[] args) throws RunnerException, IOException { // Get workload specs from the workloads directory List workloadSpecs = BenchmarkUtils.loadAllWorkloads(WORKLOAD_SPECS_DIR); if (workloadSpecs.isEmpty()) { throw new RunnerException( "No workloads found. Please add workload specs to the workloads directory."); } // Parse the Json specs from the json paths List filteredSpecs = new ArrayList<>(); for (WorkloadSpec spec : workloadSpecs) { // TODO(#5420): In the future, we can filter specific workloads using command line args here. filteredSpecs.addAll(spec.getWorkloadVariants()); } // Convert paths into a String array for JMH. JMH requires that parameters be of type String[]. String[] workloadSpecsArray = filteredSpecs.stream().map(WorkloadSpec::toJsonString).toArray(String[]::new); // Configure and run JMH benchmark with the loaded workload specs Options opt = new OptionsBuilder() .include(WorkloadBenchmark.class.getSimpleName()) .shouldFailOnError(true) .param("workloadSpecJson", workloadSpecsArray) // TODO: In the future, this can be extended to support multiple engines. .param("engineName", "default") // TODO(#5420): Allow configuring forks, warmup, and measurement via command line args. .forks(1) .warmupIterations(3) // Proper warmup for production benchmarks .measurementIterations(5) // Proper measurement iterations for production benchmarks .warmupTime(TimeValue.seconds(1)) .measurementTime(TimeValue.seconds(1)) .addProfiler(KernelMetricsProfiler.class) .build(); new Runner(opt, new WorkloadOutputFormat()).run(); } } ================================================ FILE: kernel/kernel-benchmarks/src/test/java/io/delta/kernel/benchmarks/WorkloadOutputFormat.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.benchmarks; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import io.delta.kernel.benchmarks.models.WorkloadSpec; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Collection; import java.util.HashMap; import org.openjdk.jmh.infra.BenchmarkParams; import org.openjdk.jmh.infra.IterationParams; import org.openjdk.jmh.results.*; import org.openjdk.jmh.results.BenchmarkResult; import org.openjdk.jmh.results.IterationResult; import org.openjdk.jmh.results.Result; import org.openjdk.jmh.results.RunResult; import org.openjdk.jmh.runner.format.OutputFormat; import org.openjdk.jmh.runner.format.OutputFormatFactory; import org.openjdk.jmh.runner.options.VerboseMode; import org.openjdk.jmh.util.Statistics; /** * Custom JMH output format that generates structured JSON benchmark reports. * *

This output format captures benchmark results and generates a comprehensive JSON report * containing execution environment details, benchmark configuration, timing metrics, and secondary * metrics. The report includes detailed percentile analysis and is written to the working directory * as {@code benchmark_report.json}. * *

This format also delegates to JMH's standard text output format to print progress during * benchmark execution. * *

The generated report structure includes: * *

    *
  • Report metadata (generation time, JMH version, etc.) *
  • Execution environment (JVM, OS, hardware details) *
  • Benchmark configuration and parameters *
  • Benchmark details (spec, additional params, timing metrics, secondary metrics) *
*/ public class WorkloadOutputFormat implements OutputFormat { private final OutputFormat delegate = OutputFormatFactory.createFormatInstance(System.out, VerboseMode.NORMAL); private final Path outputPath = Paths.get(System.getProperty("user.dir"), "benchmark_report.json"); private static final double[] PERCENTILES = {0.5, 0.9, 0.95, 0.99, 0.999, 0.9999, 1.0}; /** Metadata about the benchmark report itself. Json formatted */ private static class ReportMetadata { @JsonProperty("generated_at") private final String generated_at; @JsonProperty("jmh_version") private final String jmh_version; @JsonProperty("report_version") private final String report_version; @JsonProperty("benchmark_suite") private final String benchmark_suite; ReportMetadata( String generated_at, String jmh_version, String report_version, String benchmark_suite) { this.generated_at = generated_at; this.jmh_version = jmh_version; this.report_version = report_version; this.benchmark_suite = benchmark_suite; } public String toString() { return String.format( "ReportMetadata{generated_at='%s', jmh_version='%s'," + " report_version='%s', benchmark_suite='%s'}", generated_at, jmh_version, report_version, benchmark_suite); } } public static class ExecutionEnvironment { @JsonProperty("jvm") private final String jvm; @JsonProperty("heap_size_mb") private final String heapSizeMB; @JsonProperty("jdk_version") private final String jdk_version; @JsonProperty("vm_name") private final String vm_name; @JsonProperty("vm_version") private final String vm_version; @JsonProperty("cpu_model") private final String cpuModel; @JsonProperty("cpu_arch") private final String cpuArch; @JsonProperty("cpu_cores") private final Long cpuCores; @JsonProperty("os_name") private final String osName; @JsonProperty("os_version") private final String osVersion; @JsonProperty("max_memory_mb") private final Long maxMemoryMb; public ExecutionEnvironment() { this.jvm = System.getProperty("java.vm.name"); this.heapSizeMB = Runtime.getRuntime().maxMemory() / (1024 * 1024) + " MB"; this.jdk_version = System.getProperty("java.version"); this.vm_name = System.getProperty("java.vm.name"); this.vm_version = System.getProperty("java.vm.version"); this.cpuModel = System.getProperty("os.arch"); this.cpuArch = System.getProperty("os.arch"); this.cpuCores = (long) Runtime.getRuntime().availableProcessors(); this.osName = System.getProperty("os.name"); this.osVersion = System.getProperty("os.version"); this.maxMemoryMb = Runtime.getRuntime().maxMemory() / (1024 * 1024); } public String toString() { return String.format( "ExecutionEnvironment{jvm='%s', heapSizeMB='%s', jdk_version='%s'," + " vm_name='%s', vm_version='%s', cpuModel='%s', cpuArch='%s'," + " cpuCores=%d, osName='%s', osVersion='%s'}", jvm, heapSizeMB, jdk_version, vm_name, vm_version, cpuModel, cpuArch, cpuCores, osName, osVersion); } } private static class BenchmarkDetails { @JsonProperty("spec") private WorkloadSpec spec; @JsonProperty("additional_params") private HashMap additionalParams; @JsonProperty("time") private TimingMetric time; @JsonProperty("secondary_metrics") private HashMap secondary_metrics; BenchmarkDetails( WorkloadSpec spec, HashMap additionalParams, TimingMetric time, HashMap secondary_metrics) { this.spec = spec; this.additionalParams = additionalParams; this.time = time; this.secondary_metrics = secondary_metrics; } public String toString() { return String.format( "BenchmarkDetails{spec=%s, additionalParams=%s, time=%s, secondary_metrics=%s}", spec, additionalParams.toString(), time.toString(), secondary_metrics.toString()); } } private static class BenchmarkReport { @JsonProperty("report_metadata") private ReportMetadata reportMetadata; @JsonProperty("execution_environment") private ExecutionEnvironment executionEnvironment; @JsonProperty("benchmark_configuration") private HashMap benchmarkConfiguration; @JsonProperty("benchmarks") private HashMap benchmarks; BenchmarkReport( ReportMetadata reportMetadata, ExecutionEnvironment executionEnvironment, HashMap benchmarkConfiguration, HashMap benchmarks) { this.reportMetadata = reportMetadata; this.executionEnvironment = executionEnvironment; this.benchmarkConfiguration = benchmarkConfiguration; this.benchmarks = benchmarks; } public String toString() { return String.format( "BenchmarkReport{reportMetadata=%s, executionEnvironment=%s," + " benchmarkConfiguration=%s, benchmarks=%s}", reportMetadata.toString(), executionEnvironment.toString(), benchmarkConfiguration.toString(), benchmarks.toString()); } } private static class TimingMetric { @JsonProperty("score") private final double score; @JsonProperty("score_unit") private final String score_unit; @JsonProperty("score_error") private final double score_error; @JsonProperty("score_confidence") private final double[] score_confidence; @JsonProperty("sample_count") private final long sample_count; @JsonProperty("percentiles") private final HashMap percentiles; TimingMetric( double score, String score_unit, double score_error, double[] score_confidence, long sample_count, HashMap percentiles) { this.score = score; this.score_unit = score_unit; this.score_error = score_error; this.score_confidence = score_confidence; this.sample_count = sample_count; this.percentiles = percentiles; } public static TimingMetric fromResult(Result result) { HashMap percentiles = new HashMap<>(); Statistics stats = result.getStatistics(); for (double p : PERCENTILES) { String key = String.format("p%.2f", p); percentiles.put(key, stats.getPercentile(p)); } return new TimingMetric( result.getScore(), result.getScoreUnit(), result.getScoreError(), result.getScoreConfidence(), result.getSampleCount(), percentiles); } public String toString() { return String.format( "TimingMetric{score=%f, score_unit='%s', score_error=%f," + " score_confidence=[%s], sample_count=%d, percentiles=%s}", score, score_unit, score_error, String.join( ", ", new String[] { String.valueOf(score_confidence[0]), String.valueOf(score_confidence[1]) }), sample_count, percentiles.toString()); } } @Override public void iteration(BenchmarkParams benchParams, IterationParams params, int iteration) { delegate.iteration(benchParams, params, iteration); } @Override public void iterationResult( BenchmarkParams benchParams, IterationParams params, int iteration, IterationResult data) { delegate.iterationResult(benchParams, params, iteration, data); } @Override public void startBenchmark(BenchmarkParams benchParams) { delegate.startBenchmark(benchParams); } @Override public void endBenchmark(BenchmarkResult result) { delegate.endBenchmark(result); } @Override public void startRun() { delegate.startRun(); } @Override public void endRun(Collection result) { println("\n=== Generating JSON Benchmark Report ==="); ReportMetadata metadata = new ReportMetadata( String.valueOf(System.currentTimeMillis()), org.openjdk.jmh.Main.class.getPackage().getImplementationVersion(), "1.0", "Delta Kernel Benchmarks"); ExecutionEnvironment env = new ExecutionEnvironment(); HashMap benchConfig = new HashMap<>(); HashMap benchmarks = new HashMap<>(); for (RunResult res : result) { BenchmarkResult br = res.getAggregatedResult(); try { WorkloadSpec spec = WorkloadSpec.fromJsonString(br.getParams().getParam("workloadSpecJson")); HashMap additionalParams = new HashMap<>(); additionalParams.put("engine", br.getParams().getParam("engineName")); HashMap secondaryMetrics = new HashMap<>(); for (String resultKey : br.getSecondaryResults().keySet()) { Result r = br.getSecondaryResults().get(resultKey); if (r instanceof org.openjdk.jmh.results.SampleTimeResult) { secondaryMetrics.put(r.getLabel(), TimingMetric.fromResult(r)); } else if (r instanceof org.openjdk.jmh.results.ScalarResult) { ScalarResult scalarResult = (ScalarResult) r; if (scalarResult.getScoreUnit().equals("count")) { // Convert count metrics to long integers to avoid decimal representation in JSON // output (e.g., report "42" instead of "42.0" for file counts) secondaryMetrics.put(r.getLabel(), (long) r.getScore()); } else { secondaryMetrics.put(r.getLabel(), r.getScore()); } } } BenchmarkDetails details = new BenchmarkDetails( spec, additionalParams, TimingMetric.fromResult(br.getPrimaryResult()), secondaryMetrics); benchmarks.put(spec.getFullName(), details); } catch (IOException e) { throw new RuntimeException(e); } } BenchmarkReport report = new BenchmarkReport(metadata, env, benchConfig, benchmarks); // Write report to user.dir ObjectMapper mapper = new ObjectMapper(); try { String jsonReport = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(report); println("Generated benchmark report:\n" + jsonReport); println("Writing benchmark report to " + outputPath); Files.write(outputPath, jsonReport.getBytes()); } catch (IOException e) { throw new RuntimeException("Failed to serialize benchmark report to JSON", e); } } @Override public void print(String s) { delegate.print(s); } @Override public void println(String s) { delegate.println(s); } @Override public void flush() { delegate.flush(); } @Override public void close() { delegate.close(); } @Override public void verbosePrintln(String s) { delegate.verbosePrintln(s); } @Override public void write(int b) { delegate.write(b); } @Override public void write(byte[] b) throws IOException { delegate.write(b); } } ================================================ FILE: kernel/kernel-benchmarks/src/test/java/io/delta/kernel/benchmarks/models/ReadSpec.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.benchmarks.models; import com.fasterxml.jackson.annotation.JsonProperty; import io.delta.kernel.benchmarks.workloadrunners.ReadMetadataRunner; import io.delta.kernel.benchmarks.workloadrunners.WorkloadRunner; import io.delta.kernel.engine.Engine; import java.util.ArrayList; import java.util.List; /** * Workload specification for read benchmarks. Defines test cases for reading Delta tables at * specific versions or latest, with different operation types. * *

Usage

* *

To run this workload, use {@link WorkloadSpec#getRunner(Engine)} to get the appropriate {@link * ReadMetadataRunner} or ReadDataRunner based on the operation type. * * @see ReadMetadataRunner */ public class ReadSpec extends WorkloadSpec { /** The snapshot version to read. If null, the latest version will be read. From spec file. */ @JsonProperty("version") private Long version; /** * The operation type for this variant ("read_metadata" or "read_data"). Set during variant * generation via getWorkloadVariants(), not present in spec files. */ @JsonProperty("operation_type") private String operationType; // Default constructor for Jackson public ReadSpec() { super("read"); } // Copy constructor public ReadSpec(TableInfo tableInfo, String caseName, Long version, String operationType) { super("read"); this.tableInfo = tableInfo; this.version = version; this.caseName = caseName; this.operationType = operationType; } /** @return the snapshot version to read, or null if the latest version should be read. */ public Long getVersion() { return version; } /** * @return the full name of this workload, derived from table name, case name, and operation type. */ @Override public String getFullName() { return this.tableInfo.name + "/" + caseName + "/read/" + operationType; } @Override public WorkloadRunner getRunner(Engine engine) { if (operationType.equals("read_metadata")) { return new ReadMetadataRunner(this, engine); } else { throw new IllegalArgumentException("Unsupported operation for ReadSpec: " + operationType); } } /** * Generates workload variants from this test case specification. * *

A single ReadSpec test case can generate multiple workload variants, one for each operation * type. Each variant is a complete ReadSpec with the operation type set. * * @return list of ReadSpec variants, each representing a separate workload execution */ @Override public List getWorkloadVariants() { // TODO: In the future, we will support the read_data operation as well. String[] operationTypes = {"read_metadata"}; List out = new ArrayList<>(); for (String opType : operationTypes) { ReadSpec specVariant = new ReadSpec(tableInfo, caseName, version, opType); out.add(specVariant); } return out; } /** @return the operation type ("read_metadata" or "read_data") */ public String getOperationType() { return operationType; } @Override public String toString() { return String.format( "Read{caseName='%s', operationType='%s', snapshotVersion=%s, tableInfo='%s'}", caseName, operationType, version, tableInfo); } } ================================================ FILE: kernel/kernel-benchmarks/src/test/java/io/delta/kernel/benchmarks/models/SnapshotConstructionSpec.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.benchmarks.models; import com.fasterxml.jackson.annotation.JsonProperty; import io.delta.kernel.benchmarks.workloadrunners.SnapshotConstructionRunner; import io.delta.kernel.benchmarks.workloadrunners.WorkloadRunner; import io.delta.kernel.engine.Engine; public class SnapshotConstructionSpec extends WorkloadSpec { /** The snapshot version to read. If null, the latest version will be read. From spec file. */ @JsonProperty("version") private Long version; // Default constructor for Jackson public SnapshotConstructionSpec() { super("snapshot_construction"); } /** @return the snapshot version to read, or null if the latest version should be read. */ public Long getVersion() { return version; } @Override public WorkloadRunner getRunner(Engine engine) { return new SnapshotConstructionRunner(this, engine); } } ================================================ FILE: kernel/kernel-benchmarks/src/test/java/io/delta/kernel/benchmarks/models/TableInfo.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.benchmarks.models; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; import java.io.IOException; import java.nio.file.Paths; import java.util.Optional; /** * Represents metadata about a Delta table used in benchmark workloads. * *

This class contains information about a Delta table that is used by benchmark workloads to * locate and access the table data. It includes the table name, description, the root path where * the table is stored, and optional engine information. * *

TableInfo instances are typically loaded from JSON files in the workload specifications * directory structure. Each table directory should contain a {@code table_info.json} file with the * table metadata and a {@code delta} subdirectory containing the actual table data. The table root * path is the absolute path to the root of the table and is provided separately in {@link * WorkloadSpec#fromJsonPath(String, String, TableInfo)}. * *

Example JSON structure: * * 

{@code
 * {
 *   "name": "large-table",
 *   "description": "A large Delta table with multi-part checkpoints for performance testing",
 *   "engineInfo": "Apache-Spark/3.5.1 Delta-Lake/3.1.0"
 * }
 * }
*/ public class TableInfo { /** The name of the table, used for identification in benchmark reports. */ @JsonProperty("name") public String name; /** A human-readable description of the table and its purpose. */ @JsonProperty("description") public String description; /** * Information about the engine used to create this table (e.g., "Apache-Spark/3.5.1 * Delta-Lake/3.1.0"). Optional field to track which engine/version created the table data. */ @JsonProperty("engine_info") public String engineInfo; /** The path to the table_info directory */ @JsonProperty("table_info_path") private String tableInfoPath; /** * Whether this table is a Unity Catalog managed table. If true, the UC Catalog info is loaded * from a fixed path: catalog_managed_info.json in the same directory as table_info.json. */ @JsonProperty("is_catalog_managed") private boolean isCatalogManaged; /** * Lazily loaded Unity Catalog information. This is populated when {@link #getUcCatalogInfo()} is * called for the first time. */ @JsonIgnore private Optional ucCatalogInfo = Optional.empty(); /** * Default constructor for Jackson deserialization. * *

This constructor is required for Jackson to deserialize JSON into TableInfo objects. All * fields should be set via Jackson annotations or setter methods. */ public TableInfo() {} /** Resolves the table root path based on the table type and location configuration. */ @JsonIgnore public String getResolvedTableRoot() { return Paths.get(tableInfoPath, "delta").toAbsolutePath().toString(); } public String getTableInfoPath() { return tableInfoPath; } public void setTableInfoPath(String tableInfoDirectory) { this.tableInfoPath = tableInfoDirectory; } /** * Checks if this table is a Unity Catalog managed table. * * @return true if is_catalog_managed is true, false otherwise */ @JsonIgnore public boolean isCatalogManaged() { return isCatalogManaged; } /** * Gets the Unity Catalog information for this table. Lazily loads the UcCatalogInfo from * catalog_managed_info.json in the same directory as table_info.json if not already loaded. * * @return the UcCatalogInfo for this table * @throws IllegalStateException if this is not a Unity Catalog managed table * @throws RuntimeException if there is an error loading the UcCatalogInfo */ @JsonIgnore public UcCatalogInfo getUcCatalogInfo() { if (!isCatalogManaged()) { throw new IllegalStateException( "This is not a Unity Catalog managed table. is_catalog_managed is not set to true in table_info.json"); } // If ucCatalogInfo is not cached, load it from catalog_managed_info.json if (!ucCatalogInfo.isPresent()) { String catalogManagedInfoFullPath = Paths.get(tableInfoPath, "catalog_managed_info.json").toString(); try { ucCatalogInfo = Optional.of(UcCatalogInfo.fromJsonPath(catalogManagedInfoFullPath)); } catch (java.io.IOException e) { throw new RuntimeException( "Failed to load UcCatalogInfo from: " + catalogManagedInfoFullPath, e); } } return ucCatalogInfo.get(); } /** * Creates a TableInfo instance by reading from a JSON file at the specified path. * *

This method loads table metadata from a JSON file and sets the table root path. The JSON * file should contain the table name and description, while the table root path is provided * separately with the absolute path. * * @param jsonPath the path to the JSON file containing the TableInfo metadata * @param tableInfoPath the directory containing the table_info.json file * @return a TableInfo instance populated from the JSON file and table root path * @throws RuntimeException if there is an error reading or parsing the JSON file */ public static TableInfo fromJsonPath(String jsonPath, String tableInfoPath) { ObjectMapper mapper = new ObjectMapper(); try { TableInfo info = mapper.readValue(new File(jsonPath), TableInfo.class); info.setTableInfoPath(tableInfoPath); return info; } catch (IOException e) { throw new RuntimeException("Failed to read TableInfo from JSON file: " + jsonPath, e); } } /** * Returns a string representation of this TableInfo. * *

The string includes the table name, description, and engine info, but excludes the table * root path for security reasons (as it may contain sensitive path information). * * @return a string representation of this TableInfo */ @Override public String toString() { return "TableInfo{name='" + name + "', description='" + description + "', engineInfo='" + engineInfo + "'}"; } } ================================================ FILE: kernel/kernel-benchmarks/src/test/java/io/delta/kernel/benchmarks/models/UcCatalogInfo.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.benchmarks.models; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.unitycatalog.InMemoryUCClient; import io.delta.kernel.unitycatalog.UCCatalogManagedCommitter; import io.delta.kernel.utils.FileStatus; import io.delta.storage.commit.Commit; import java.io.File; import java.io.IOException; import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; import scala.collection.mutable.ArrayBuffer; /** * Configuration for Unity catalog managed tables used in benchmarks. * *

Contains information about staged commits in the `_delta_log/_staged_commits/` directory. * *

Example JSON structure: * * 

{@code
 * {
 *   "uc_table_id": "12345678-1234-1234-1234-123456789abc",
 *   "max_ratified_version": 2,
 *   "log_tail": [
 *     {
 *       "staged_commit_file_name": "00000000000000000002.a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d.json"
 *     }
 *   ]
 * }
 * }
*/ public class UcCatalogInfo { /** * A single staged commit in the log tail. * *

Each staged commit is a JSON file in `_delta_log/_staged_commits/` containing Delta log * actions. */ public static class StagedCommit { /** * The filename of the staged commit, relative to `_delta_log/_staged_commits/`. * *

Example: "00000000000000000002.a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d.json" */ @JsonProperty("staged_commit_file_name") private String stagedCommitFileName; /** Default constructor for Jackson deserialization. */ public StagedCommit() {} /** * Constructor for creating a StagedCommit. * * @param stagedCommitFileName the filename of the staged commit */ public StagedCommit(String stagedCommitFileName) { this.stagedCommitFileName = stagedCommitFileName; } /** * Gets the version number from the filename. * * @return the version number */ @JsonIgnore public long getVersion() { return FileNames.deltaVersion(stagedCommitFileName); } /** @return the filename of the staged commit */ public String getStagedCommitFileName() { return stagedCommitFileName; } /** * Gets the full path to the staged commit file. * * @param tableRoot the root path of the Delta table * @return the full path */ public String getFullPath(String tableRoot) { Path logPath = new Path(tableRoot, "_delta_log"); Path stagedCommitsDir = new Path(logPath, FileNames.STAGED_COMMIT_DIRECTORY); return new Path(stagedCommitsDir, stagedCommitFileName).toUri().toString(); } @Override public String toString() { return String.format( "StagedCommit{version=%d, stagedCommitFileName='%s'}", getVersion(), stagedCommitFileName); } public Commit toCommit(Engine engine, String tableRoot) throws IOException { String stagedCommitPath = getFullPath(tableRoot); FileStatus fileStatus = engine.getFileSystemClient().getFileStatus(stagedCommitPath); // While catalog managed tables expect to use inCommitTimestamp, we use // modification time here for simplicity. long commitTimestamp = fileStatus.getModificationTime(); org.apache.hadoop.fs.FileStatus hadoopFileStatus = UCCatalogManagedCommitter.kernelFileStatusToHadoopFileStatus(fileStatus); return new Commit(getVersion(), hadoopFileStatus, commitTimestamp); } } /** * The list of staged commits for this table. * *

These commits are not yet backfilled to the regular Delta log. */ @JsonProperty(value = "log_tail", required = true) private List logTail; /** The Unity Catalog table ID. */ @JsonProperty(value = "uc_table_id", required = true) private String ucTableId; /** The maximum ratified version for this table in Unity Catalog. */ @JsonProperty(value = "max_ratified_version", required = true) private long maxRatifiedVersion; /** Default constructor for Jackson deserialization. */ public UcCatalogInfo() {} /** * Creates a UcCatalogInfo with the given staged commits. * * @param logTail the list of staged commits */ public UcCatalogInfo(List logTail) { this.logTail = logTail; } /** @return the list of staged commits, or an empty list if none */ public List getLogTail() { return logTail; } /** @return the Unity Catalog table ID */ public String getUcTableId() { return ucTableId; } /** * Creates an InMemoryUCClient for this table with the staged commits pre-loaded. * * @param engine the Engine to use for reading staged commit files * @param tableRoot the root path of the Delta table * @return an initialized InMemoryUCClient * @throws IOException if there's an error reading staged commit files */ public InMemoryUCClient createUCClient(Engine engine, String tableRoot) throws IOException { ArrayBuffer commits = new ArrayBuffer<>(); if (!logTail.isEmpty()) { // Commits must be added to TableData in order of version. We sort staged commits by version. List sortedLogTail = logTail.stream() .sorted(Comparator.comparingLong(StagedCommit::getVersion)) .collect(Collectors.toList()); for (StagedCommit stagedCommit : sortedLogTail) { commits.addOne(stagedCommit.toCommit(engine, tableRoot)); } } InMemoryUCClient ucClient = new InMemoryUCClient("benchmark-metastore"); InMemoryUCClient.TableData tableData = new InMemoryUCClient.TableData(maxRatifiedVersion, commits); ucClient.insertTableData(ucTableId, tableData); return ucClient; } /** * Loads a UcCatalogInfo from a JSON file. * * @param jsonPath the path to the JSON file * @return the parsed UcCatalogInfo * @throws IOException if there is an error reading the file */ public static UcCatalogInfo fromJsonPath(String jsonPath) throws IOException { ObjectMapper mapper = new ObjectMapper(); return mapper.readValue(new File(jsonPath), UcCatalogInfo.class); } @Override public String toString() { return String.format("UcCatalogInfo{logTail=%s}", logTail); } } ================================================ FILE: kernel/kernel-benchmarks/src/test/java/io/delta/kernel/benchmarks/models/WorkloadSpec.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.benchmarks.models; import com.fasterxml.jackson.annotation.*; import com.fasterxml.jackson.databind.ObjectMapper; import io.delta.kernel.benchmarks.workloadrunners.WorkloadRunner; import io.delta.kernel.engine.Engine; import java.io.File; import java.io.IOException; import java.util.Collections; import java.util.List; /** * Base class for all workload specifications. Workload specifications define test cases and their * parameters that can be executed as benchmarks using the corresponding {@link WorkloadRunner}. * *

This class uses Jackson annotations to support polymorphic deserialization based on the "type" * field in the JSON. */ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") @JsonSubTypes({ @JsonSubTypes.Type(value = ReadSpec.class, name = "read"), @JsonSubTypes.Type(value = SnapshotConstructionSpec.class, name = "snapshot_construction"), @JsonSubTypes.Type(value = WriteSpec.class, name = "write") }) public abstract class WorkloadSpec { /** * The type of workload (e.g., "read"). This is used by Jackson's polymorphic deserialization to * automatically instantiate the correct subclass based on the "type" field in the JSON. */ protected String type; @JsonProperty("table_info") protected TableInfo tableInfo; @JsonProperty("case_name") protected String caseName; private static final ObjectMapper objectMapper = new ObjectMapper(); /** Default constructor for Jackson */ protected WorkloadSpec() {} protected WorkloadSpec(String type) { this.type = type; } /** @return the type of this workload. */ @JsonIgnore public String getType() { return type; } /** @return the case name of this workload. */ public String getCaseName() { return caseName; } @JsonProperty(value = "full_name", access = JsonProperty.Access.READ_ONLY) public String getFullName() { return tableInfo.name + "/" + caseName + "/" + type; } public void setCaseName(String caseName) { this.caseName = caseName; } public TableInfo getTableInfo() { return tableInfo; } @JsonIgnore public String getSpecDirectoryPath() { return tableInfo.getTableInfoPath() + "/specs/" + caseName; } /** * Sets the table information for this workload specification. * * @param tableInfo the table information containing name, description, and root path */ public void setTableInfo(TableInfo tableInfo) { this.tableInfo = tableInfo; } /** * Creates a WorkloadRunner for this workload specification. * * @param engine The engine to use for executing the workload. * @return the WorkloadRunner instance for this workload specification. */ public abstract WorkloadRunner getRunner(Engine engine); /** * Loads a WorkloadSpec from the given JSON file path. * * @param workloadPath the path to the JSON file containing the workload specification. * @param caseName the name of the test case for this workload * @param tableInfo the table information to associate with this workload * @return the WorkloadSpec instance parsed from the JSON file. * @throws IOException if there is an error reading or parsing the file. */ public static WorkloadSpec fromJsonPath(String workloadPath, String caseName, TableInfo tableInfo) throws IOException { WorkloadSpec spec = objectMapper.readValue(new File(workloadPath), WorkloadSpec.class); spec.setTableInfo(tableInfo); spec.setCaseName(caseName); return spec; } /** * Generates workload variants from this test case specification. * *

A single WorkloadSpec can generate multiple workload variants. For example, a read spec * might generate both read_metadata and read_data variants. Each variant can be executed as a * benchmark or test. * *

The default implementation returns a single variant (itself). Subclasses should override to * generate multiple variants if needed. * * @return list of WorkloadSpec variants, each representing a separate workload execution */ @JsonIgnore public List getWorkloadVariants() { return Collections.singletonList(this); } /** * Loads a WorkloadSpec from the given JSON string. * * @param json the JSON string containing the workload specification. * @return the WorkloadSpec instance parsed from the JSON string. * @throws IOException if there is an error parsing the JSON. */ public static WorkloadSpec fromJsonString(String json) throws IOException { return objectMapper.readValue(json, WorkloadSpec.class); } /** * Serializes this WorkloadSpec to a pretty-printed JSON string. * * @return the JSON string representation of this WorkloadSpec. */ public String toJsonString() { try { return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(this); } catch (IOException e) { throw new RuntimeException("Failed to serialize WorkloadSpec to JSON", e); } } } ================================================ FILE: kernel/kernel-benchmarks/src/test/java/io/delta/kernel/benchmarks/models/WriteSpec.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.benchmarks.models; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import io.delta.kernel.benchmarks.workloadrunners.WorkloadRunner; import io.delta.kernel.benchmarks.workloadrunners.WriteRunner; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.statistics.DataFileStatistics; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.DataFileStatus; import java.io.File; import java.io.IOException; import java.util.*; import java.util.stream.Collectors; /** * Workload specification for write benchmarks. Defines test cases for writing to Delta tables with * one or more commits containing add/remove actions. * *

Usage

* *

To run this workload, use {@link WorkloadSpec#getRunner(Engine)} to get the appropriate {@link * WriteRunner}. * * @see WriteRunner */ public class WriteSpec extends WorkloadSpec { /** * Container for data file actions (adds) from a commit specification file. * *

This class is used to deserialize the JSON structure containing the list of files to add in * a commit. */ private static class DataFileOperations { @JsonProperty("adds") private ArrayList adds; /** @return the list of added data files in this commit. */ @JsonIgnore public ArrayList getAdds() { return adds; } } /** * Serialization/deserialization wrapper for {@link DataFileStatus}. * *

This class represents the JSON structure for data file metadata, including path, size, * modification time, and optional statistics. It can be converted to a {@link DataFileStatus} * instance for use in Delta Kernel APIs. */ private static class DataFilesStatusSerde { @JsonProperty("path") private String path; @JsonProperty("size") private long size; @JsonProperty("modification_time") private long modificationTime; @JsonProperty("stats") private String stats; /** * Converts this serialization object to a {@link DataFileStatus} instance. * * @param schema the table schema used to parse statistics * @return a DataFileStatus instance with the file metadata */ public DataFileStatus toDataFileStatus(StructType schema) { Optional parsedStats = Optional.empty(); if (stats != null) { parsedStats = DataFileStatistics.deserializeFromJson(stats, schema); } return new DataFileStatus(path, size, modificationTime, parsedStats); } } /** * Container for a single commit's configuration. * *

Each commit references a file containing the Delta log JSON actions (add/remove files) to be * committed. */ public static class CommitSpec { /** * Path to the commit file containing Delta log JSON actions. The path is relative to the spec * directory (where spec.json is located). * *

Example: "commit_a.json" */ @JsonProperty("data_files_path") private String dataFilesPath; /** Default constructor for Jackson. */ public CommitSpec() {} public String getDataFilesPath() { return dataFilesPath; } /** * Parses the data_files file and returns the list of added and removed data files. * * @param specPath the base path where the commit file is located * @throws IOException if there's an error reading or parsing the file */ public List readDataFiles(String specPath, StructType schema) throws IOException { ObjectMapper mapper = new ObjectMapper(); String commitFilePath = new Path(specPath, getDataFilesPath()).toString(); DataFileOperations dataFileOps = mapper.readValue(new File(commitFilePath), DataFileOperations.class); return dataFileOps.adds.stream() .map(file -> file.toDataFileStatus(schema)) .collect(Collectors.toList()); } } /** * List of commits to execute in sequence. Each commit contains a reference to a file with Delta * log JSON actions. All commits are executed as part of the timed benchmark. */ @JsonProperty("commits") private List commits; // Default constructor for Jackson public WriteSpec() { super("write"); } /** * Gets the list of commits to execute. * * @return list of commit specifications */ public List getCommits() { return commits != null ? commits : Collections.emptyList(); } /** @return the full name of this workload, derived from table name, case name, and type. */ @Override public String getFullName() { return this.tableInfo.name + "/" + caseName + "/write"; } @Override public WorkloadRunner getRunner(Engine engine) { return new WriteRunner(this, engine); } /** * Generates workload variants from this test case specification. * *

Currently, WriteSpec generates a single variant (itself). In the future, this could be * extended to generate variants for different write patterns or configurations. * * @return list of WriteSpec variants, each representing a separate workload execution */ @Override public List getWorkloadVariants() { return Collections.singletonList(this); } @Override public String toString() { return String.format( "Write{caseName='%s', commits=%d, tableInfo='%s'}", caseName, getCommits().size(), tableInfo); } } ================================================ FILE: kernel/kernel-benchmarks/src/test/java/io/delta/kernel/benchmarks/workloadrunners/ReadMetadataRunner.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.benchmarks.workloadrunners; import io.delta.kernel.*; import io.delta.kernel.benchmarks.models.ReadSpec; import io.delta.kernel.benchmarks.models.WorkloadSpec; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.engine.Engine; import io.delta.kernel.utils.CloseableIterator; import java.util.Optional; import org.openjdk.jmh.infra.Blackhole; /** * A WorkloadRunner that can execute the read_metadata workload as a benchmark. This runner is * created from a {@link ReadSpec}. The workload performs a scan of the table's metadata, at the * specified snapshot version (or latest if not specified). * *

If run as a benchmark using {@link #executeAsBenchmark(Blackhole)}, this measures the time to * perform the metadata scan and consume all results. It does not include the time to load the * snapshot or set up the scan, which is done in {@link #setup()}. */ public class ReadMetadataRunner extends WorkloadRunner { private Scan scan; private final Engine engine; private final ReadSpec workloadSpec; /** * Constructs the ReadMetadataRunner from the workload spec and engine. * * @param workloadSpec The read_metadata workload specification. * @param engine The engine to use for executing the workload. * @throws IllegalArgumentException if the operation type is not "read_metadata" */ public ReadMetadataRunner(ReadSpec workloadSpec, Engine engine) { // ensure the operation type is read_metadata if (!workloadSpec.getOperationType().equals("read_metadata")) { throw new IllegalArgumentException( "ReadMetadataRunner can only be used for read_metadata workloads"); } this.workloadSpec = workloadSpec; this.engine = engine; } @Override public void setup() throws Exception { Optional versionOpt = Optional.ofNullable(workloadSpec.getVersion()); Snapshot snapshot = loadSnapshot(engine, workloadSpec.getTableInfo(), versionOpt); scan = snapshot.getScanBuilder().build(); } /** @return the name of this workload derived from the workload specification. */ @Override public String getName() { return "read_metadata"; } /** @return The workload specification used to create this runner. */ @Override public WorkloadSpec getWorkloadSpec() { return workloadSpec; } /** * Executes the read_metadata workload as a benchmark, consuming results via the provided * Blackhole. * * @param blackhole The Blackhole to consume results and avoid dead code elimination. */ @Override public void executeAsBenchmark(Blackhole blackhole) { // Run the actual metadata reading workload try (CloseableIterator iterator = execute()) { // Consume the iterator to measure the actual work while (iterator.hasNext()) { FilteredColumnarBatch batch = iterator.next(); blackhole.consume(batch); } } catch (Exception e) { throw new RuntimeException("Error during benchmark execution", e); } } @Override public void cleanup() throws Exception { /* This is a read-only workload; no cleanup necessary. */ } /** * Executes the read_metadata workload, returning an iterator over the results. This must be fully * consumed by the caller to ensure the workload is fully executed. * * @return Iterator of results from the read_metadata workload. */ private CloseableIterator execute() { if (scan == null) { throw new IllegalStateException( "ReadMetadataRunner not initialized. Call setup() before executing."); } return scan.getScanFiles(engine); } } ================================================ FILE: kernel/kernel-benchmarks/src/test/java/io/delta/kernel/benchmarks/workloadrunners/SnapshotConstructionRunner.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.benchmarks.workloadrunners; import io.delta.kernel.Snapshot; import io.delta.kernel.SnapshotBuilder; import io.delta.kernel.TableManager; import io.delta.kernel.benchmarks.models.SnapshotConstructionSpec; import io.delta.kernel.benchmarks.models.WorkloadSpec; import io.delta.kernel.engine.Engine; import org.openjdk.jmh.infra.Blackhole; public class SnapshotConstructionRunner extends WorkloadRunner { private final SnapshotConstructionSpec workloadSpec; private final Engine engine; /** * Construct a SnapshotConstructionRunner from the workload spec and engine. * * @param workloadSpec The snapshot_construction workload specification. * @param engine The engine to use for executing the workload. */ public SnapshotConstructionRunner(SnapshotConstructionSpec workloadSpec, Engine engine) { this.workloadSpec = workloadSpec; this.engine = engine; } @Override public String getName() { return "snapshot_construction"; } @Override public WorkloadSpec getWorkloadSpec() { return this.workloadSpec; } @Override public void setup() throws Exception { /* No setup needed for snapshot construction */ } @Override public void executeAsBenchmark(Blackhole blackhole) throws Exception { blackhole.consume(this.execute()); } @Override public void cleanup() throws Exception { /* No cleanup needed for snapshot construction */ } private Snapshot execute() { String workloadTableRoot = workloadSpec.getTableInfo().getResolvedTableRoot(); SnapshotBuilder builder = TableManager.loadSnapshot(workloadTableRoot); if (workloadSpec.getVersion() != null) { builder.atVersion(workloadSpec.getVersion()); } return builder.build(engine); } } ================================================ FILE: kernel/kernel-benchmarks/src/test/java/io/delta/kernel/benchmarks/workloadrunners/WorkloadRunner.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.benchmarks.workloadrunners; import io.delta.kernel.Snapshot; import io.delta.kernel.SnapshotBuilder; import io.delta.kernel.TableManager; import io.delta.kernel.benchmarks.models.TableInfo; import io.delta.kernel.benchmarks.models.UcCatalogInfo; import io.delta.kernel.benchmarks.models.WorkloadSpec; import io.delta.kernel.engine.Engine; import io.delta.kernel.unitycatalog.InMemoryUCClient; import io.delta.kernel.unitycatalog.UCCatalogManagedClient; import java.net.URI; import java.nio.file.Paths; import java.util.Optional; import org.openjdk.jmh.infra.Blackhole; /** * A runner that can execute a specific workload as a benchmark or test. A WorkloadRunner is created * from a {@link WorkloadSpec} and is responsible for setting up any state necessary to execute the * workload using {@link WorkloadRunner#setup()}, as well as executing the workload itself. * *

Execution Modes

* *
    *
  • Benchmark: Execute via {@link #executeAsBenchmark(Blackhole)} for JMH performance * measurements *
  • Test: Execute via executeAsTest() for correctness validation (future work) *
* *

The {@link #setup()} method must be called before any execution method. */ public abstract class WorkloadRunner { public WorkloadRunner() {} /** @return the name of this workload derived from the contents of the workload specification. */ public abstract String getName(); /** @return The workload specification used to create this runner. */ public abstract WorkloadSpec getWorkloadSpec(); /** * Sets up any state necessary to execute this workload. This method must be called before * executing the workload as a benchmark or test. * * @throws Exception if any error occurs during setup. */ public abstract void setup() throws Exception; /** * Executes the workload as a benchmark, consuming any output via the provided Blackhole to * prevent dead code elimination by the JIT compiler. The {@link #setup()} method must be called * before invoking this method. * * @param blackhole the Blackhole provided by JMH to consume output. * @throws Exception if any error occurs during execution. */ public abstract void executeAsBenchmark(Blackhole blackhole) throws Exception; /** * Cleans up any state created during benchmark execution. For write workloads, this removes added * files and reverts table state. For read workloads, this is typically a no-op. * *

This method is called after each benchmark invocation to ensure a clean state for the next * run. * * @throws Exception if any error occurs during cleanup. */ public abstract void cleanup() throws Exception; /** * Loads a snapshot for the table. * *

For Unity Catalog managed tables, uses {@link UCCatalogManagedClient} to handle staged * commits. For regular tables, uses {@link TableManager}. * * @param engine the engine to use * @param tableInfo the table information * @param versionOpt optional version to load (if empty, loads latest) * @return a Snapshot for the table * @throws Exception if there's an error loading the snapshot */ protected Snapshot loadSnapshot(Engine engine, TableInfo tableInfo, Optional versionOpt) throws Exception { String tableRoot = tableInfo.getResolvedTableRoot(); if (tableInfo.isCatalogManaged()) { UcCatalogInfo ucCatalogInfo = tableInfo.getUcCatalogInfo(); InMemoryUCClient ucClient = ucCatalogInfo.createUCClient(engine, tableRoot); UCCatalogManagedClient ucCatalogManagedClient = new UCCatalogManagedClient(ucClient); // Use Paths.get().toUri() to get properly formatted file:// URI URI tableUri = Paths.get(tableRoot).toUri(); return ucCatalogManagedClient.loadSnapshot( engine, ucCatalogInfo.getUcTableId(), tableUri.toString(), versionOpt, Optional.empty() /* timestampOpt */); } else { // Use direct TableManager for regular filesystem tables SnapshotBuilder builder = TableManager.loadSnapshot(tableRoot); if (versionOpt.isPresent()) { builder = builder.atVersion(versionOpt.get()); } return builder.build(engine); } } // TODO: Add executeAsTest() method for correctness validation // public abstract void executeAsTest() throws Exception; } ================================================ FILE: kernel/kernel-benchmarks/src/test/java/io/delta/kernel/benchmarks/workloadrunners/WriteRunner.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.benchmarks.workloadrunners; import static io.delta.kernel.internal.util.Utils.toCloseableIterator; import io.delta.kernel.*; import io.delta.kernel.benchmarks.models.WorkloadSpec; import io.delta.kernel.benchmarks.models.WriteSpec; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.transaction.UpdateTableTransactionBuilder; import io.delta.kernel.utils.CloseableIterable; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.DataFileStatus; import io.delta.kernel.utils.FileStatus; import java.io.FileNotFoundException; import java.io.IOException; import java.util.*; import org.openjdk.jmh.infra.Blackhole; /** * A WorkloadRunner that executes write workloads as benchmarks. This runner performs one or more * commits to a Delta table and measures the performance of those commits. * *

The runner executes commits specified in the {@link WriteSpec}, where each commit contains a * set of Delta log actions (add/remove files) defined in external JSON files. * *

If run as a benchmark using {@link #executeAsBenchmark(Blackhole)}, this measures the time to * execute all commits. Setup (loading commit files) and cleanup (reverting changes) are not * included in the benchmark timing. */ public class WriteRunner extends WorkloadRunner { private final Engine engine; private final WriteSpec workloadSpec; private final List> commitContents; private Snapshot snapshot; private Optional> initialDeltaLogFiles = Optional.empty(); /** * Constructs the WriteRunner from the workload spec and engine. * * @param workloadSpec The write workload specification. * @param engine The engine to use for executing the workload. */ public WriteRunner(WriteSpec workloadSpec, Engine engine) { this.workloadSpec = workloadSpec; this.engine = engine; this.commitContents = new ArrayList<>(); } @Override public void setup() throws Exception { String tableRoot = workloadSpec.getTableInfo().getResolvedTableRoot(); // Capture initial listing of delta log files. This is used during cleanup to revert changes. if (!initialDeltaLogFiles.isPresent()) { initialDeltaLogFiles = Optional.of(captureFileListing()); } // Load the initial snapshot of the table. This will be used as the starting point for commits // and will be updated after each commit using the post-commit snapshot. snapshot = loadSnapshot(engine, workloadSpec.getTableInfo(), Optional.empty()); if (commitContents.isEmpty()) { for (WriteSpec.CommitSpec commitSpec : workloadSpec.getCommits()) { commitContents.add( commitSpec.readDataFiles(workloadSpec.getSpecDirectoryPath(), snapshot.getSchema())); } } } /** @return the name of this workload. */ @Override public String getName() { return "write"; } /** @return The workload specification used to create this runner. */ @Override public WorkloadSpec getWorkloadSpec() { return workloadSpec; } /** * Executes the write workload as a benchmark, consuming results via the provided Blackhole. * *

This method executes all commits specified in the workload spec in sequence. The timing * includes only the commit execution, not the setup or cleanup. We reuse the post-commit snapshot * from each transaction to avoid reloading from disk, which makes the benchmark more efficient * and realistic. * * @param blackhole The Blackhole to consume results and avoid dead code elimination. */ @Override public void executeAsBenchmark(Blackhole blackhole) throws Exception { // Execute all commits in sequence for (List actions : commitContents) { UpdateTableTransactionBuilder txnBuilder = snapshot.buildUpdateTableTransaction("Delta-Kernel-Benchmarks", Operation.WRITE); Transaction txn = txnBuilder.build(engine); Row txnState = txn.getTransactionState(engine); DataWriteContext writeContext = Transaction.getWriteContext(engine, txnState, new HashMap<>() /* partitionValues */); CloseableIterator add_actions = Transaction.generateAppendActions( engine, txnState, toCloseableIterator(actions.iterator()), writeContext); TransactionCommitResult result = txn.commit(engine, CloseableIterable.inMemoryIterable(add_actions)); long version = result.getVersion(); blackhole.consume(version); // Use the post-commit snapshot for the next transaction // Post-commit snapshot should always be present unless there was a conflict snapshot = result .getPostCommitSnapshot() .orElseThrow( () -> new IllegalStateException( "Post-commit snapshot not available. This indicates a conflict " + "occurred during the benchmark, which should not happen. " + "Ensure no other processes are writing to the table: " + workloadSpec.getTableInfo().getResolvedTableRoot())); } } /** Cleans up the state created during benchmark execution by reverting all committed changes. */ @Override public void cleanup() throws Exception { if (!initialDeltaLogFiles.isPresent()) { throw new RuntimeException("Cannot cleanup before setup is called."); } // Delete any files that weren't present initially Set currentFiles = captureFileListing(); Set initialFiles = initialDeltaLogFiles.orElseThrow( () -> new RuntimeException("Cannot cleanup before setup is called.")); for (String filePath : currentFiles) { if (!initialFiles.contains(filePath)) { engine.getFileSystemClient().delete(filePath); } } } /** * @return all file paths in the `_delta_log/` directory (including `_staged_commits/` if Unity * Catalog managed and `_sidecars/` if it exists) */ private Set captureFileListing() throws IOException { List prefixes = new ArrayList<>( Arrays.asList("_delta_log", "_delta_log/_sidecars", "_delta_log/_staged_commits")); Set files = new HashSet<>(); for (String prefix : prefixes) { // Construct path prefix for all files in `_delta_log/`. The prefix is for file with name `0` // because the filesystem client lists all _sibling_ files in the directory with a path // greater than `0`. String deltaLogPathPrefix = new Path(workloadSpec.getTableInfo().getResolvedTableRoot(), new Path(prefix, "0")) .toUri() .getPath(); // List from the lowest version in the prefix try (CloseableIterator filesIter = engine.getFileSystemClient().listFrom(deltaLogPathPrefix)) { while (filesIter.hasNext()) { files.add(filesIter.next().getPath()); } } catch (FileNotFoundException e) { // Ignore if the directory does not exist } } return files; } } ================================================ FILE: kernel/kernel-benchmarks/src/test/resources/workload_specs/basic_append/delta/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1712091396253,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"3","numOutputBytes":"996"},"engineInfo":"Apache-Spark/3.5.1 Delta-Lake/3.1.0","txnId":"5df7dc20-b980-4207-a5fd-b69cb4541b2e"}} {"metaData":{"id":"2a1e618f-d92a-4c94-bb06-a2808f8b39f3","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"letter\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"number\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"a_float\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1712091393302}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part-00000-c9f44819-b06d-45dd-b33d-ae9aa1b96909-c000.snappy.parquet","partitionValues":{},"size":996,"modificationTime":1712091396057,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"letter\":\"a\",\"number\":1,\"a_float\":1.1},\"maxValues\":{\"letter\":\"c\",\"number\":3,\"a_float\":3.3},\"nullCount\":{\"letter\":0,\"number\":0,\"a_float\":0}}"}} ================================================ FILE: kernel/kernel-benchmarks/src/test/resources/workload_specs/basic_append/delta/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1712091404556,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"2","numOutputBytes":"984"},"engineInfo":"Apache-Spark/3.5.1 Delta-Lake/3.1.0","txnId":"64562965-a4c4-48a7-84dd-e68ee934f467"}} {"add":{"path":"part-00000-a9daef62-5a40-43c5-ac63-3ad4a7d749ae-c000.snappy.parquet","partitionValues":{},"size":984,"modificationTime":1712091404545,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"letter\":\"d\",\"number\":4,\"a_float\":4.4},\"maxValues\":{\"letter\":\"e\",\"number\":5,\"a_float\":5.5},\"nullCount\":{\"letter\":0,\"number\":0,\"a_float\":0}}"}} ================================================ FILE: kernel/kernel-benchmarks/src/test/resources/workload_specs/basic_append/specs/read_latest/spec.json ================================================ { "type": "read" } ================================================ FILE: kernel/kernel-benchmarks/src/test/resources/workload_specs/basic_append/specs/read_v0/spec.json ================================================ { "type": "read", "version": 0 } ================================================ FILE: kernel/kernel-benchmarks/src/test/resources/workload_specs/basic_append/specs/snapshot_latest/spec.json ================================================ { "type": "snapshot_construction" } ================================================ FILE: kernel/kernel-benchmarks/src/test/resources/workload_specs/basic_append/specs/snapshot_v0/spec.json ================================================ { "type": "snapshot_construction", "version": 0 } ================================================ FILE: kernel/kernel-benchmarks/src/test/resources/workload_specs/basic_append/specs/write_appends/commit_2_adds.json ================================================ { "adds": [ { "path": "dummy_data_b.parquet", "size": 1024, "modification_time": 1712091404545, "stats": "{\"numRecords\":10,\"minValues\":{\"letter\":\"a\",\"number\":1,\"a_float\":1.1},\"maxValues\":{\"letter\":\"j\",\"number\":10,\"a_float\":10.10},\"nullCount\":{\"letter\":0,\"number\":0,\"a_float\":0}}" }, { "path": "dummy_data_c.parquet", "size": 927, "modification_time": 1712091405000, "stats": "{\"numRecords\":5,\"minValues\":{\"letter\":\"b\",\"number\":2,\"a_float\":1.3},\"maxValues\":{\"letter\":\"j\",\"number\":10,\"a_float\":10.10},\"nullCount\":{\"letter\":0,\"number\":0,\"a_float\":0}}" } ] } ================================================ FILE: kernel/kernel-benchmarks/src/test/resources/workload_specs/basic_append/specs/write_appends/commit_add.json ================================================ { "adds": [ { "path": "dummy_data_a.parquet", "size": 1024, "modification_time": 1712091404545, "stats": "{\"numRecords\":10,\"minValues\":{\"letter\":\"a\",\"number\":1,\"a_float\":1.1},\"maxValues\":{\"letter\":\"j\",\"number\":10,\"a_float\":10.10},\"nullCount\":{\"letter\":0,\"number\":0,\"a_float\":0}}" } ] } ================================================ FILE: kernel/kernel-benchmarks/src/test/resources/workload_specs/basic_append/specs/write_appends/spec.json ================================================ { "type": "write", "commits": [ {"data_files_path": "commit_add.json"}, {"data_files_path": "commit_2_adds.json"} ] } ================================================ FILE: kernel/kernel-benchmarks/src/test/resources/workload_specs/basic_append/table_info.json ================================================ { "name": "basic_append", "description": "A basic table with two append writes.", "engine_info": "Apache-Spark/3.5.1 Delta-Lake/3.1.0" } ================================================ FILE: kernel/kernel-benchmarks/src/test/resources/workload_specs/basic_catalog_managed/catalog_managed_info.json ================================================ { "uc_table_id": "12345678-1234-1234-1234-123456789abc", "max_ratified_version": 3, "log_tail": [ { "staged_commit_file_name": "00000000000000000002.a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d.json" }, { "staged_commit_file_name": "00000000000000000003.f7e8d9c0-b1a2-4536-9748-5a6b7c8d9e0f.json" } ] } ================================================ FILE: kernel/kernel-benchmarks/src/test/resources/workload_specs/basic_catalog_managed/delta/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"inCommitTimestamp":1712091396253,"timestamp":1712091396253,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"3","numOutputBytes":"996"},"engineInfo":"Apache-Spark/3.5.1 Delta-Lake/3.1.0","txnId":"5df7dc20-b980-4207-a5fd-b69cb4541b2e"}} {"metaData":{"id":"2a1e618f-d92a-4c94-bb06-a2808f8b39f3","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"letter\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"number\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"a_float\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableInCommitTimestamps":"true","catalogManaged.unityCatalog.tableId":"12345678-1234-1234-1234-123456789abc"},"createdTime":1712091393302}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["catalogManaged"],"writerFeatures":["catalogManaged","inCommitTimestamp"]}} {"add":{"path":"part-00000-c9f44819-b06d-45dd-b33d-ae9aa1b96909-c000.snappy.parquet","partitionValues":{},"size":996,"modificationTime":1712091396057,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"letter\":\"a\",\"number\":1,\"a_float\":1.1},\"maxValues\":{\"letter\":\"c\",\"number\":3,\"a_float\":3.3},\"nullCount\":{\"letter\":0,\"number\":0,\"a_float\":0}}"}} ================================================ FILE: kernel/kernel-benchmarks/src/test/resources/workload_specs/basic_catalog_managed/delta/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1712091404556,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"2","numOutputBytes":"984"},"engineInfo":"Apache-Spark/3.5.1 Delta-Lake/3.1.0","txnId":"64562965-a4c4-48a7-84dd-e68ee934f467"}} {"add":{"path":"part-00000-a9daef62-5a40-43c5-ac63-3ad4a7d749ae-c000.snappy.parquet","partitionValues":{},"size":984,"modificationTime":1712091404545,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"letter\":\"d\",\"number\":4,\"a_float\":4.4},\"maxValues\":{\"letter\":\"e\",\"number\":5,\"a_float\":5.5},\"nullCount\":{\"letter\":0,\"number\":0,\"a_float\":0}}"}} ================================================ FILE: kernel/kernel-benchmarks/src/test/resources/workload_specs/basic_catalog_managed/delta/_delta_log/_staged_commits/00000000000000000002.a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d.json ================================================ {"commitInfo":{"inCommitTimestamp":1712091410000,"timestamp":1712091410000,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"2","numOutputBytes":"984"},"engineInfo":"Apache-Spark/3.5.1 Delta-Lake/3.1.0","txnId":"catalog-managed-test-txn-0002"}} {"remove":{"path":"part-00000-a9daef62-5a40-43c5-ac63-3ad4a7d749ae-c000.snappy.parquet","partitionValues":{},"size":984,"modificationTime":1712091404545,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"letter\":\"d\",\"number\":4,\"a_float\":4.4},\"maxValues\":{\"letter\":\"e\",\"number\":5,\"a_float\":5.5},\"nullCount\":{\"letter\":0,\"number\":0,\"a_float\":0}}"}} ================================================ FILE: kernel/kernel-benchmarks/src/test/resources/workload_specs/basic_catalog_managed/delta/_delta_log/_staged_commits/00000000000000000003.f7e8d9c0-b1a2-4536-9748-5a6b7c8d9e0f.json ================================================ {"commitInfo":{"inCommitTimestamp":1712091415000,"timestamp":1712091415000,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"3","numOutputBytes":"996"},"engineInfo":"Apache-Spark/3.5.1 Delta-Lake/3.1.0","txnId":"catalog-managed-test-txn-0003"}} {"remove":{"path":"part-00000-c9f44819-b06d-45dd-b33d-ae9aa1b96909-c000.snappy.parquet","partitionValues":{},"size":996,"modificationTime":1712091396057,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"letter\":\"a\",\"number\":1,\"a_float\":1.1},\"maxValues\":{\"letter\":\"c\",\"number\":3,\"a_float\":3.3},\"nullCount\":{\"letter\":0,\"number\":0,\"a_float\":0}}"}} ================================================ FILE: kernel/kernel-benchmarks/src/test/resources/workload_specs/basic_catalog_managed/specs/read_with_staged/spec.json ================================================ { "type": "read", "operation_type": "read_metadata" } ================================================ FILE: kernel/kernel-benchmarks/src/test/resources/workload_specs/basic_catalog_managed/specs/write_with_staged/commit_2.json ================================================ { "adds": [ { "path": "part-00000-c9f44819-b06d-45dd-b33d-ae9aa1b96909-c000.snappy.parquet", "size": 996, "modification_time": 1712091396057, "stats": "{\"numRecords\":3,\"minValues\":{\"letter\":\"a\",\"number\":1,\"a_float\":1.1},\"maxValues\":{\"letter\":\"c\",\"number\":3,\"a_float\":3.3},\"nullCount\":{\"letter\":0,\"number\":0,\"a_float\":0}}" }, { "path": "part-00000-a9daef62-5a40-43c5-ac63-3ad4a7d749ae-c000.snappy.parquet", "size": 984, "modification_time": 1712091404545, "stats": "{\"numRecords\":2,\"minValues\":{\"letter\":\"d\",\"number\":4,\"a_float\":4.4},\"maxValues\":{\"letter\":\"e\",\"number\":5,\"a_float\":5.5},\"nullCount\":{\"letter\":0,\"number\":0,\"a_float\":0}}" } ] } ================================================ FILE: kernel/kernel-benchmarks/src/test/resources/workload_specs/basic_catalog_managed/specs/write_with_staged/spec.json ================================================ { "type": "write", "commits": [ {"data_files_path": "commit_2.json"} ] } ================================================ FILE: kernel/kernel-benchmarks/src/test/resources/workload_specs/basic_catalog_managed/table_info.json ================================================ { "name": "basic_catalog_managed", "description": "A basic Unity Catalog managed table with 2 backfilled commits (v0-1) and 2 staged commits (v2-3).", "engine_info": "Apache-Spark/3.5.1 Delta-Lake/3.1.0", "is_catalog_managed": true } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/DefaultEngine.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.engine; import io.delta.kernel.defaults.engine.fileio.FileIO; import io.delta.kernel.defaults.engine.hadoopio.HadoopFileIO; import io.delta.kernel.engine.*; import java.util.Collections; import java.util.List; import org.apache.hadoop.conf.Configuration; /** Default implementation of {@link Engine} based on Hadoop APIs. */ public class DefaultEngine implements Engine { private final FileIO fileIO; protected DefaultEngine(FileIO fileIO) { this.fileIO = fileIO; } @Override public ExpressionHandler getExpressionHandler() { return new DefaultExpressionHandler(); } @Override public JsonHandler getJsonHandler() { return new DefaultJsonHandler(fileIO); } @Override public FileSystemClient getFileSystemClient() { return new DefaultFileSystemClient(fileIO); } @Override public ParquetHandler getParquetHandler() { return new DefaultParquetHandler(fileIO); } @Override public List getMetricsReporters() { return Collections.singletonList(new LoggingMetricsReporter()); }; /** * Create an instance of {@link DefaultEngine}. * * @param hadoopConf Hadoop configuration to use. * @return an instance of {@link DefaultEngine}. */ public static DefaultEngine create(Configuration hadoopConf) { return new DefaultEngine(new HadoopFileIO(hadoopConf)); } /** * Create an instance of {@link DefaultEngine}. It takes {@link FileIO} as an argument which is * used for I/O related operations. * * @param fileIO File IO implementation to use for reading and writing files. * @return an instance of {@link DefaultEngine}. */ public static DefaultEngine create(FileIO fileIO) { return new DefaultEngine(fileIO); } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/DefaultExpressionHandler.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.engine; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.defaults.internal.data.vector.DefaultBooleanVector; import io.delta.kernel.defaults.internal.expressions.DefaultExpressionEvaluator; import io.delta.kernel.defaults.internal.expressions.DefaultPredicateEvaluator; import io.delta.kernel.engine.ExpressionHandler; import io.delta.kernel.expressions.Expression; import io.delta.kernel.expressions.ExpressionEvaluator; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.expressions.PredicateEvaluator; import io.delta.kernel.types.DataType; import io.delta.kernel.types.StructType; import java.util.Arrays; import java.util.Optional; /** Default implementation of {@link ExpressionHandler} */ public class DefaultExpressionHandler implements ExpressionHandler { @Override public ExpressionEvaluator getEvaluator( StructType inputSchema, Expression expression, DataType outputType) { return new DefaultExpressionEvaluator(inputSchema, expression, outputType); } @Override public PredicateEvaluator getPredicateEvaluator(StructType inputSchema, Predicate predicate) { return new DefaultPredicateEvaluator(inputSchema, predicate); } @Override public ColumnVector createSelectionVector(boolean[] values, int from, int to) { requireNonNull(values, "values is null"); int length = to - from; checkArgument( length >= 0 && values.length > from && values.length >= to, "invalid range from=%s, to=%s, values length=%s", from, to, values.length); // Make a copy of the `values` array. boolean[] valuesCopy = Arrays.copyOfRange(values, from, to); return new DefaultBooleanVector(length, Optional.empty(), valuesCopy); } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/DefaultFileSystemClient.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.engine; import io.delta.kernel.defaults.engine.fileio.FileIO; import io.delta.kernel.defaults.engine.fileio.InputFile; import io.delta.kernel.defaults.engine.fileio.SeekableInputStream; import io.delta.kernel.engine.FileReadRequest; import io.delta.kernel.engine.FileSystemClient; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import io.delta.storage.LogStore; import java.io.*; import java.util.Objects; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; /** * Default implementation of {@link FileSystemClient} based on Hadoop APIs. It takes a Hadoop {@link * Configuration} object to interact with the file system. The following optional configurations can * be set to customize the behavior of the client: * *

    *
  • {@code io.delta.kernel.logStore..impl} - The class name of the custom {@link * LogStore} implementation to use for operations on storage systems with the specified {@code * scheme}. For example, to use a custom {@link LogStore} for S3 storage objects: * 
    {@code
 * 
 *   io.delta.kernel.logStore.s3.impl
 *   com.example.S3LogStore
 * 
 *
 * }
    * If not set, the default LogStore implementation for the scheme will be used. *
  • {@code delta.enableFastS3AListFrom} - Set to {@code true} to enable fast listing * functionality when using a {@link LogStore} created for S3 storage objects. *
* * The above list of options is not exhaustive. For a complete list of options, refer to the * specific implementation of {@link FileSystem}. */ public class DefaultFileSystemClient implements FileSystemClient { private final FileIO fileIO; /** * Create an instance of the default {@link FileSystemClient} implementation. * * @param fileIO The {@link FileIO} implementation to use for file operations. */ public DefaultFileSystemClient(FileIO fileIO) { this.fileIO = Objects.requireNonNull(fileIO, "fileIO is null"); } @Override public CloseableIterator listFrom(String filePath) throws IOException { return fileIO.listFrom(filePath); } @Override public String resolvePath(String path) throws IOException { return fileIO.resolvePath(path); } @Override public CloseableIterator readFiles( CloseableIterator readRequests) throws IOException { return readRequests.map( elem -> getStream(elem.getPath(), elem.getStartOffset(), elem.getReadLength())); } @Override public boolean mkdirs(String path) throws IOException { return fileIO.mkdirs(path); } @Override public boolean delete(String path) throws IOException { return fileIO.delete(path); } @Override public FileStatus getFileStatus(String path) throws IOException { return fileIO.getFileStatus(path); } @Override public void copyFileAtomically(String srcPath, String destPath, boolean overwrite) throws IOException { fileIO.copyFileAtomically(srcPath, destPath, overwrite); } private ByteArrayInputStream getStream(String filePath, int offset, int size) { InputFile inputFile = this.fileIO.newInputFile(filePath, /* fileSize */ -1); try (SeekableInputStream stream = inputFile.newStream()) { stream.seek(offset); byte[] buff = new byte[size]; stream.readFully(buff, 0, size); return new ByteArrayInputStream(buff); } catch (IOException ex) { throw new UncheckedIOException( String.format( "IOException reading from file %s at offset %s size %s", filePath, offset, size), ex); } } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/DefaultJsonHandler.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.engine; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.lang.String.format; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.*; import com.fasterxml.jackson.databind.node.ObjectNode; import io.delta.kernel.data.*; import io.delta.kernel.defaults.engine.fileio.FileIO; import io.delta.kernel.defaults.engine.fileio.SeekableInputStream; import io.delta.kernel.defaults.internal.data.DefaultJsonRow; import io.delta.kernel.defaults.internal.data.DefaultRowBasedColumnarBatch; import io.delta.kernel.defaults.internal.json.JsonUtils; import io.delta.kernel.engine.JsonHandler; import io.delta.kernel.exceptions.KernelEngineException; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.types.*; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import io.delta.storage.LogStore; import java.io.*; import java.nio.charset.StandardCharsets; import java.util.*; /** Default implementation of {@link JsonHandler} based on Hadoop APIs. */ public class DefaultJsonHandler implements JsonHandler { private static final ObjectMapper mapper = new ObjectMapper(); // by default BigDecimals are truncated and read as floats private static final ObjectReader objectReaderReadBigDecimals = new ObjectMapper().reader(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); private final FileIO fileIO; private final int maxBatchSize; public DefaultJsonHandler(FileIO fileIO) { this.fileIO = fileIO; this.maxBatchSize = fileIO .getConf("delta.kernel.default.json.reader.batch-size") .map(Integer::valueOf) .orElse(1024); checkArgument(maxBatchSize > 0, "invalid JSON reader batch size: %d", maxBatchSize); } @Override public ColumnarBatch parseJson( ColumnVector jsonStringVector, StructType outputSchema, Optional selectionVector) { List rows = new ArrayList<>(); for (int i = 0; i < jsonStringVector.getSize(); i++) { boolean isSelected = !selectionVector.isPresent() || (!selectionVector.get().isNullAt(i) && selectionVector.get().getBoolean(i)); if (isSelected && !jsonStringVector.isNullAt(i)) { rows.add(parseJson(jsonStringVector.getString(i), outputSchema)); } else { rows.add(null); } } return new DefaultRowBasedColumnarBatch(outputSchema, rows); } @Override public CloseableIterator readJsonFiles( CloseableIterator scanFileIter, StructType physicalSchema, Optional predicate) throws IOException { return new CloseableIterator() { private FileStatus currentFile; private BufferedReader currentFileReader; private String nextLine; @Override public void close() throws IOException { Utils.closeCloseables(currentFileReader, scanFileIter); } @Override public boolean hasNext() { if (nextLine != null) { return true; // we have un-consumed last read line } // There is no file in reading or the current file being read has no more data // initialize the next file reader or return false if there are no more files to // read. try { if (currentFileReader == null || (nextLine = currentFileReader.readLine()) == null) { // `nextLine` will initially be null because `currentFileReader` is guaranteed // to be null if (tryOpenNextFile()) { return hasNext(); } } return nextLine != null; } catch (IOException ex) { throw new KernelEngineException( format("Error reading JSON file: %s", currentFile.getPath()), ex); } } @Override public ColumnarBatch next() { if (nextLine == null) { throw new NoSuchElementException(); } List rows = new ArrayList<>(); int currentBatchSize = 0; do { // hasNext already reads the next one and keeps it in member variable `nextLine` rows.add(parseJson(nextLine, physicalSchema)); nextLine = null; currentBatchSize++; } while (currentBatchSize < maxBatchSize && hasNext()); return new DefaultRowBasedColumnarBatch(physicalSchema, rows); } private boolean tryOpenNextFile() throws IOException { Utils.closeCloseables(currentFileReader); // close the current opened file currentFileReader = null; if (scanFileIter.hasNext()) { currentFile = scanFileIter.next(); SeekableInputStream stream = null; try { stream = fileIO.newInputFile(currentFile.getPath(), currentFile.getSize()).newStream(); currentFileReader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)); } catch (Exception e) { Utils.closeCloseablesSilently(stream); // close it avoid leaking resources throw e; } } return currentFileReader != null; } }; } /** * Makes use of {@link LogStore} implementations in `delta-storage` to atomically write the data * to a file depending upon the destination filesystem. * * @param filePath Destination file path * @param data Data to write as Json * @throws IOException */ @Override public void writeJsonFileAtomically( String filePath, CloseableIterator data, boolean overwrite) throws IOException { fileIO.newOutputFile(filePath).writeAtomically(data.map(JsonUtils::rowToJson), overwrite); } private Row parseJson(String json, StructType readSchema) { try { final JsonNode jsonNode = objectReaderReadBigDecimals.readTree(json); return new DefaultJsonRow((ObjectNode) jsonNode, readSchema); } catch (JsonProcessingException ex) { throw new KernelEngineException(format("Could not parse JSON: %s", json), ex); } } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/DefaultParquetHandler.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.engine; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.defaults.engine.fileio.FileIO; import io.delta.kernel.defaults.internal.parquet.ParquetFileReader; import io.delta.kernel.defaults.internal.parquet.ParquetFileWriter; import io.delta.kernel.engine.FileReadResult; import io.delta.kernel.engine.ParquetHandler; import io.delta.kernel.expressions.Column; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.*; import io.delta.kernel.utils.FileStatus; import io.delta.storage.LogStore; import java.io.IOException; import java.io.UncheckedIOException; import java.util.*; /** Default implementation of {@link ParquetHandler} based on Hadoop APIs. */ public class DefaultParquetHandler implements ParquetHandler { private final FileIO fileIO; /** * Create an instance of default {@link ParquetHandler} implementation. * * @param fileIO File IO implementation to use for reading and writing files. */ public DefaultParquetHandler(FileIO fileIO) { this.fileIO = Objects.requireNonNull(fileIO, "fileIO is null"); } @Override public CloseableIterator readParquetFiles( CloseableIterator fileIter, StructType physicalSchema, Optional predicate) throws IOException { return new CloseableIterator() { private final ParquetFileReader batchReader = new ParquetFileReader(fileIO); private CloseableIterator currentFileReader; private String currentFilePath; @Override public void close() throws IOException { Utils.closeCloseables(currentFileReader, fileIter); } @Override public boolean hasNext() { if (currentFileReader != null && currentFileReader.hasNext()) { return true; } else { // There is no file in reading or the current file being read has no more data. // Initialize the next file reader or return false if there are no more files to // read. Utils.closeCloseables(currentFileReader); currentFileReader = null; currentFilePath = null; if (fileIter.hasNext()) { FileStatus fileStatus = fileIter.next(); currentFileReader = batchReader.read(fileStatus, physicalSchema, predicate); currentFilePath = fileStatus.getPath(); return hasNext(); // recurse since it's possible the loaded file is empty } else { return false; } } } @Override public FileReadResult next() { return new FileReadResult(currentFileReader.next(), currentFilePath); } }; } @Override public CloseableIterator writeParquetFiles( String directoryPath, CloseableIterator dataIter, List statsColumns) throws IOException { ParquetFileWriter batchWriter = ParquetFileWriter.multiFileWriter(fileIO, directoryPath, statsColumns); return batchWriter.write(dataIter); } /** * Makes use of {@link LogStore} implementations in `delta-storage` to atomically write the data * to a file depending upon the destination filesystem. * * @param filePath Fully qualified destination file path * @param data Iterator of {@link FilteredColumnarBatch} * @throws IOException */ @Override public void writeParquetFileAtomically( String filePath, CloseableIterator data) throws IOException { try { ParquetFileWriter fileWriter = ParquetFileWriter.singleFileWriter( fileIO, filePath, /* atomicWrite= */ true, /* statsColumns= */ Collections.emptyList()); fileWriter.write(data).next(); // TODO: fix this } catch (UncheckedIOException e) { throw e.getCause(); } finally { Utils.closeCloseables(data); } } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/LoggingMetricsReporter.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.engine; import io.delta.kernel.engine.MetricsReporter; import io.delta.kernel.metrics.MetricsReport; import io.delta.kernel.shaded.com.fasterxml.jackson.core.JsonProcessingException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * An implementation of {@link MetricsReporter} that logs the reports (as JSON) to Log4J at the info * level. */ public class LoggingMetricsReporter implements MetricsReporter { private static final Logger logger = LoggerFactory.getLogger(LoggingMetricsReporter.class); @Override public void report(MetricsReport report) { try { logger.info("{} = {}", report.getClass().getName(), report.toJson()); } catch (JsonProcessingException e) { logger.warn("Serialization issue while logging metrics report {}: {}", report, e.toString()); } catch (Exception e) { logger.warn("Unexpected error while logging metrics report {}:", report, e); } } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/fileio/FileIO.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.engine.fileio; import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import java.io.FileNotFoundException; import java.io.IOException; import java.util.Optional; /** * Interface for file IO operations. Connectors can implement their own version of the {@link * FileIO} depending upon their environment. The {@link DefaultEngine} takes {@link FileIO} instance * as input and all I/O operations from the default engine are done using the passed in {@link * FileIO} instance. */ public interface FileIO { /** * List the paths in the same directory that are lexicographically greater or equal to (UTF-8 * sorting) the given `path`. The result should also be sorted by the file name. * * @param filePath Fully qualified path to a file * @return Closeable iterator of files. It is the responsibility of the caller to close the * iterator. * @throws FileNotFoundException if the file at the given path is not found * @throws IOException for any other IO error. */ CloseableIterator listFrom(String filePath) throws IOException; /** * Get the metadata of the file at the given path. * * @param path Fully qualified path to the file. * @return Metadata of the file. * @throws IOException for any IO error. */ FileStatus getFileStatus(String path) throws IOException; /** * Resolve the given path to a fully qualified path. * * @param path Input path * @return Fully qualified path. * @throws FileNotFoundException If the given path doesn't exist. * @throws IOException for any other IO error. */ String resolvePath(String path) throws IOException; /** * Create a directory at the given path including parent directories. This mimics the behavior of * `mkdir -p` in Unix. * * @param path Full qualified path to create a directory at. * @return true if the directory was created successfully, false otherwise. * @throws IOException for any IO error. */ boolean mkdirs(String path) throws IOException; /** * Get an {@link InputFile} for file at given path which can be used to read the file from any * arbitrary position in the file. * * @param path Fully qualified path to the file. * @param fileSize Size of the file in bytes. * @return {@link InputFile} instance. */ InputFile newInputFile(String path, long fileSize); /** * Create a {@link OutputFile} to write new file at the given path. * * @param path Fully qualified path to the file. * @return {@link OutputFile} instance which can be used to write to the file. */ OutputFile newOutputFile(String path); /** * Delete the file at given path. * * @param path the path to delete. If path is a directory throws an exception. * @return true if delete is successful else false. * @throws IOException for any IO error. */ boolean delete(String path) throws IOException; /** * Get the configuration value for the given key. * *

TODO: should be in a separate interface? may be called ConfigurationProvider? * * @param confKey configuration key name * @return If no such value is present, an {@link Optional#empty()} is returned. */ Optional getConf(String confKey); /** * Atomically copy a file from source path to destination path. The copy operation should be * atomic to ensure that the destination file is either fully copied or not present at all. * * @param srcPath Fully qualified path to the source file to copy * @param destPath Fully qualified path to the destination where the file will be copied * @param overwrite If true, overwrite the destination file if it already exists. If false, throw * an exception if the destination exists. * @throws java.nio.file.FileAlreadyExistsException if the destination file already exists and * {@code overwrite} is false. * @throws FileNotFoundException if the source file does not exist * @throws IOException for any other IO error */ void copyFileAtomically(String srcPath, String destPath, boolean overwrite) throws IOException; } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/fileio/InputFile.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.engine.fileio; import java.io.IOException; /** Interface for reading a file and getting metadata about it. */ public interface InputFile { /** * Get the size of the file. * * @return the size of the file. */ long length() throws IOException; /** * Get the path of the file. * * @return the path of the file. */ String path(); /** * Get the input stream to read the file. * * @return the input stream to read the file. It is the responsibility of the caller to close the * stream. */ SeekableInputStream newStream() throws IOException; } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/fileio/OutputFile.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.engine.fileio; import io.delta.kernel.utils.CloseableIterator; import java.io.IOException; /** Interface for writing to a file and getting metadata about it. */ public interface OutputFile { /** * Get the path of the file. * * @return the path of the file. */ String path(); /** * Get the output stream to write to the file. * *

    *
  • If the file already exists, (either at the time of creating the {@link * PositionOutputStream} or at the time of closing it), it will be overwritten. *
  • if {@code atomicWrite} is set, then the entire content is written or none, but won't * create a file with the partial contents. *
* * @return the output stream to write to the file. It is the responsibility of the caller to close * the stream. * @throws IOException if an I/O error occurs. */ PositionOutputStream create(boolean atomicWrite) throws IOException; /** * Atomically write (either write is completely or don't write all - i.e. don't leave file with * partial content) the data to a file at the given path. If the file already exists do not * replace it if {@code replace} is false. If {@code replace} is true, then replace the file with * the new data. * *

TODO: the semantics are very loose here, see if there is a better API name. One of the * reasons why the data is passed as an iterator is because of the existing LogStore interface * which are used in the Hadoop implementation of the {@link FileIO} * * @param data the data to write. Each element in the iterator is a line in the file. * @param overwrite if true, overwrite the file with the new data. If false, do not overwrite the * file. * @throws java.nio.file.FileAlreadyExistsException if the file already exists and replace is * false. * @throws IOException for any IO error. */ void writeAtomically(CloseableIterator data, boolean overwrite) throws IOException; } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/fileio/PositionOutputStream.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.engine.fileio; import java.io.IOException; import java.io.OutputStream; /** * Extends {@link OutputStream} to provide the current position in the stream. This stream is used * to write data into the file. */ public abstract class PositionOutputStream extends OutputStream { /** * Get the current position in the stream. * * @return the current position in bytes from the start of the stream */ public abstract long getPos() throws IOException; } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/fileio/SeekableInputStream.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.engine.fileio; import java.io.IOException; import java.io.InputStream; /** * Extends {@link InputStream} to provide the current position in the stream and seek to a new * position. Also provides additional utility methods such as {@link #readFully(byte[], int, int)}. */ public abstract class SeekableInputStream extends InputStream { /** * Get the current position in the stream. * * @return the current position in bytes from the start of the stream * @throws IOException if the underlying stream throws an IOException */ public abstract long getPos() throws IOException; /** * Seek to a new position in the stream. * * @param newPos the new position to seek to * @throws IOException if the underlying stream throws an IOException */ public abstract void seek(long newPos) throws IOException; /** * Read fully len bytes into the buffer b. * * @param b byte array * @param off offset in the byte array * @param len number of bytes to read * @throws java.io.EOFException – if this input stream reaches the end before reading all the * bytes. * @throws IOException – the stream has been closed and the contained input stream does not * support reading after close, or another I/ O error occurs. */ public abstract void readFully(byte[] b, int off, int len) throws IOException; } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/hadoopio/HadoopFileIO.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.engine.hadoopio; import io.delta.kernel.defaults.engine.fileio.FileIO; import io.delta.kernel.defaults.engine.fileio.InputFile; import io.delta.kernel.defaults.engine.fileio.OutputFile; import io.delta.kernel.defaults.internal.logstore.LogStoreProvider; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import io.delta.storage.LogStore; import java.io.IOException; import java.io.UncheckedIOException; import java.util.Objects; import java.util.Optional; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; /** Implementation of {@link FileIO} based on Hadoop APIs. */ public class HadoopFileIO implements FileIO { private final Configuration hadoopConf; public HadoopFileIO(Configuration hadoopConf) { this.hadoopConf = Objects.requireNonNull(hadoopConf, "hadoopConf is null"); } @Override public CloseableIterator listFrom(String filePath) throws IOException { Path path = new Path(filePath); LogStore logStore = LogStoreProvider.getLogStore(hadoopConf, path.toUri().getScheme()); return Utils.toCloseableIterator(logStore.listFrom(path, hadoopConf)) .map( hadoopFileStatus -> FileStatus.of( hadoopFileStatus.getPath().toString(), hadoopFileStatus.getLen(), hadoopFileStatus.getModificationTime())); } @Override public FileStatus getFileStatus(String path) throws IOException { Path pathObject = new Path(path); FileSystem fs = pathObject.getFileSystem(hadoopConf); org.apache.hadoop.fs.FileStatus hadoopFileStatus = fs.getFileStatus(pathObject); return FileStatus.of( hadoopFileStatus.getPath().toString(), hadoopFileStatus.getLen(), hadoopFileStatus.getModificationTime()); } @Override public String resolvePath(String path) throws IOException { Path pathObject = new Path(path); FileSystem fs = pathObject.getFileSystem(hadoopConf); return fs.makeQualified(pathObject).toString(); } @Override public boolean mkdirs(String path) throws IOException { Path pathObject = new Path(path); FileSystem fs = pathObject.getFileSystem(hadoopConf); return fs.mkdirs(pathObject); } @Override public InputFile newInputFile(String path, long fileSize) { return new HadoopInputFile(getFs(path), new Path(path), fileSize); } @Override public OutputFile newOutputFile(String path) { return new HadoopOutputFile(hadoopConf, path); } @Override public boolean delete(String path) throws IOException { FileSystem fs = getFs(path); return fs.delete(new Path(path), false); } @Override public Optional getConf(String confKey) { return Optional.ofNullable(hadoopConf.get(confKey)); } @Override public void copyFileAtomically(String srcPath, String destPath, boolean overwrite) throws IOException { Path parsedSrcPath = new Path(srcPath); Path parsedDestPath = new Path(destPath); LogStore logStore = LogStoreProvider.getLogStore(hadoopConf, parsedSrcPath.toUri().getScheme()); try (io.delta.storage.CloseableIterator srcContents = logStore.read(parsedSrcPath, hadoopConf)) { logStore.write(parsedDestPath, srcContents, overwrite, hadoopConf); } } private FileSystem getFs(String path) { try { Path pathObject = new Path(path); return pathObject.getFileSystem(hadoopConf); } catch (IOException e) { throw new UncheckedIOException("Could not resolve the FileSystem", e); } } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/hadoopio/HadoopInputFile.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.engine.hadoopio; import io.delta.kernel.defaults.engine.fileio.InputFile; import io.delta.kernel.defaults.engine.fileio.SeekableInputStream; import java.io.IOException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; public class HadoopInputFile implements InputFile { private final FileSystem fs; private final Path path; private final long fileSize; public HadoopInputFile(FileSystem fs, Path path, long fileSize) { this.fs = fs; this.path = path; this.fileSize = fileSize; } @Override public long length() throws IOException { return fileSize; } @Override public String path() { return path.toString(); } @Override public SeekableInputStream newStream() throws IOException { return new HadoopSeekableInputStream(fs.open(path)); } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/hadoopio/HadoopOutputFile.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.engine.hadoopio; import static java.lang.String.format; import io.delta.kernel.defaults.engine.fileio.OutputFile; import io.delta.kernel.defaults.engine.fileio.PositionOutputStream; import io.delta.kernel.defaults.internal.logstore.LogStoreProvider; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.utils.CloseableIterator; import io.delta.storage.LogStore; import java.io.IOException; import java.util.Objects; import java.util.UUID; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; public class HadoopOutputFile implements OutputFile { private final Configuration hadoopConf; private final String path; public HadoopOutputFile(Configuration hadoopConf, String path) { this.hadoopConf = Objects.requireNonNull(hadoopConf, "fs is null"); this.path = Objects.requireNonNull(path, "path is null"); } @Override public String path() { return path; } @Override public PositionOutputStream create(boolean putIfAbsent) throws IOException { Path targetPath = new Path(path); FileSystem fs = targetPath.getFileSystem(hadoopConf); if (!putIfAbsent) { return new HadoopPositionOutputStream(fs.create(targetPath)); } LogStore logStore = LogStoreProvider.getLogStore(hadoopConf, targetPath.toUri().getScheme()); boolean useRename = logStore.isPartialWriteVisible(targetPath, hadoopConf); final Path writePath; if (useRename) { // In order to atomically write the file, write to a temp file and rename // to target path String tempFileName = format(".%s.%s.tmp", targetPath.getName(), UUID.randomUUID()); writePath = new Path(targetPath.getParent(), tempFileName); } else { writePath = targetPath; } return new HadoopPositionOutputStream(fs.create(writePath)) { @Override public void close() throws IOException { super.close(); if (useRename) { boolean renameDone = false; try { renameDone = fs.rename(writePath, targetPath); if (!renameDone) { if (fs.exists(targetPath)) { throw new java.nio.file.FileAlreadyExistsException( "target file already exists: " + targetPath); } throw new IOException("Failed to rename the file"); } } finally { if (!renameDone) { fs.delete(writePath, false /* recursive */); } } } } }; } @Override public void writeAtomically(CloseableIterator data, boolean overwrite) throws IOException { Path pathObj = new Path(path); try { LogStore logStore = LogStoreProvider.getLogStore(hadoopConf, pathObj.toUri().getScheme()); logStore.write(pathObj, data, overwrite, hadoopConf); } finally { Utils.closeCloseables(data); } } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/hadoopio/HadoopPositionOutputStream.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.engine.hadoopio; import io.delta.kernel.defaults.engine.fileio.PositionOutputStream; import java.io.IOException; import org.apache.hadoop.fs.FSDataOutputStream; public class HadoopPositionOutputStream extends PositionOutputStream { private final FSDataOutputStream delegateStream; public HadoopPositionOutputStream(FSDataOutputStream delegateStream) { this.delegateStream = delegateStream; } @Override public void write(int b) throws IOException { delegateStream.write(b); } @Override public void write(byte[] b) throws IOException { delegateStream.write(b); } @Override public void write(byte[] b, int off, int len) throws IOException { delegateStream.write(b, off, len); } @Override public void flush() throws IOException { delegateStream.flush(); } @Override public void close() throws IOException { delegateStream.close(); } @Override public long getPos() throws IOException { return delegateStream.getPos(); } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/hadoopio/HadoopSeekableInputStream.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.engine.hadoopio; import io.delta.kernel.defaults.engine.fileio.SeekableInputStream; import java.io.IOException; import java.util.Objects; import org.apache.hadoop.fs.FSDataInputStream; public class HadoopSeekableInputStream extends SeekableInputStream { private final FSDataInputStream delegateStream; public HadoopSeekableInputStream(FSDataInputStream delegateStream) { this.delegateStream = Objects.requireNonNull(delegateStream, "delegateStream is null"); } @Override public int read() throws java.io.IOException { return delegateStream.read(); } @Override public int read(byte[] b) throws java.io.IOException { return delegateStream.read(b); } @Override public int read(byte[] b, int off, int len) throws java.io.IOException { return delegateStream.read(b, off, len); } @Override public long skip(long n) throws java.io.IOException { return delegateStream.skip(n); } @Override public int available() throws java.io.IOException { return delegateStream.available(); } @Override public void close() throws java.io.IOException { delegateStream.close(); } @Override public void seek(long pos) throws java.io.IOException { delegateStream.seek(pos); } @Override public long getPos() throws java.io.IOException { return delegateStream.getPos(); } @Override public void readFully(byte[] b, int off, int len) throws IOException { delegateStream.readFully(b, off, len); } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/package-info.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /** * Default implementation of {@link io.delta.kernel.engine.Engine} interface and the sub-interfaces * exposed by the {@link io.delta.kernel.engine.Engine}. */ package io.delta.kernel.defaults.engine; ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/DefaultEngineErrors.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal; import static java.lang.String.format; import io.delta.kernel.expressions.Expression; public class DefaultEngineErrors { public static IllegalArgumentException canNotInstantiateLogStore( String logStoreClassName, String context, Exception cause) { String msg = format("Can not instantiate `LogStore` class (%s): %s", context, logStoreClassName); return new IllegalArgumentException(msg, cause); } /** * Exception for when the default expression evaluator cannot evaluate an expression. * * @param expression the unsupported expression * @param reason reason for why the expression is not supported/cannot be evaluated */ public static UnsupportedOperationException unsupportedExpressionException( Expression expression, String reason) { String message = format( "Default expression evaluator cannot evaluate the expression: %s. Reason: %s", expression, reason); return new UnsupportedOperationException(message); } /** * Exception class for invalid escape sequence used in input for LIKE expressions * * @param pattern the invalid pattern * @param index character index of occurrence of the offending escape in the pattern */ public static IllegalArgumentException invalidEscapeSequence(String pattern, int index) { return new IllegalArgumentException( format("LIKE expression has invalid escape sequence '%s' at index %d", pattern, index)); } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/DefaultKernelUtils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal; import io.delta.kernel.expressions.Column; import io.delta.kernel.internal.util.DateTimeConstants; import io.delta.kernel.internal.util.TimestampUtils; import io.delta.kernel.internal.util.Tuple2; import io.delta.kernel.types.DataType; import io.delta.kernel.types.StructType; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; import java.time.temporal.ChronoField; import java.util.concurrent.TimeUnit; public class DefaultKernelUtils { private static final DateTimeFormatter DEFAULT_JSON_TIMESTAMPNTZ_FORMATTER = new DateTimeFormatterBuilder() .appendPattern("yyyy-MM-dd'T'HH:mm:ss") .optionalStart() .appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true) .optionalEnd() .toFormatter(); private DefaultKernelUtils() {} ////////////////////////////////////////////////////////////////////////////////// // Below utils are adapted from org.apache.spark.sql.catalyst.util.DateTimeUtils ////////////////////////////////////////////////////////////////////////////////// // See http://stackoverflow.com/questions/466321/convert-unix-timestamp-to-julian // It's 2440587.5, rounding up to be compatible with Hive. static int JULIAN_DAY_OF_EPOCH = 2440588; /** Returns the number of microseconds since epoch from Julian day and nanoseconds in a day. */ public static long fromJulianDay(int days, long nanos) { // use Long to avoid rounding errors return ((long) (days - JULIAN_DAY_OF_EPOCH)) * DateTimeConstants.MICROS_PER_DAY + nanos / DateTimeConstants.NANOS_PER_MICROS; } /** * Returns Julian day and remaining nanoseconds from the number of microseconds * *

Note: support timestamp since 4717 BC (without negative nanoseconds, compatible with Hive). */ public static Tuple2 toJulianDay(long micros) { long julianUs = micros + JULIAN_DAY_OF_EPOCH * DateTimeConstants.MICROS_PER_DAY; long days = julianUs / DateTimeConstants.MICROS_PER_DAY; long us = julianUs % DateTimeConstants.MICROS_PER_DAY; return new Tuple2<>((int) days, TimeUnit.MICROSECONDS.toNanos(us)); } public static long millisToMicros(long millis) { return Math.multiplyExact(millis, DateTimeConstants.MICROS_PER_MILLIS); } /** * Converts a number of days since epoch (1970-01-01 00:00:00 UTC) to microseconds between epoch * and start of the day in the given timezone. */ public static long daysToMicros(int days, ZoneOffset timezone) { long seconds = LocalDate.ofEpochDay(days).atStartOfDay(timezone).toEpochSecond(); return seconds * DateTimeConstants.MICROS_PER_SECOND; } /** * Parses a TimestampNTZ string in UTC format, supporting milliseconds and microseconds, to * microseconds since the Unix epoch. * * @param timestampString the timestamp string to parse. * @return the number of microseconds since epoch. */ public static long parseTimestampNTZ(String timestampString) { LocalDateTime time = LocalDateTime.parse(timestampString, DEFAULT_JSON_TIMESTAMPNTZ_FORMATTER); return TimestampUtils.toEpochMicros(time); } /** * Search for the data type of the given column in the schema. * * @param schema the schema to search * @param column the column whose data type is to be found * @return the data type of the column * @throws IllegalArgumentException if the column is not found in the schema */ public static DataType getDataType(StructType schema, Column column) { DataType dataType = schema; for (String part : column.getNames()) { if (!(dataType instanceof StructType)) { throw new IllegalArgumentException( String.format("Cannot resolve column (%s) in schema: %s", column, schema)); } StructType structType = (StructType) dataType; if (structType.fieldNames().contains(part)) { dataType = structType.get(part).getDataType(); } else { throw new IllegalArgumentException( String.format("Cannot resolve column (%s) in schema: %s", column, schema)); } } return dataType; } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/DefaultColumnarBatch.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.data; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.types.StructField; import io.delta.kernel.types.StructType; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; public class DefaultColumnarBatch implements ColumnarBatch { private final int size; private final StructType schema; private final List columnVectors; public DefaultColumnarBatch(int size, StructType schema, ColumnVector[] columnVectors) { this.schema = schema; this.size = size; this.columnVectors = Collections.unmodifiableList(Arrays.asList(columnVectors)); } @Override public StructType getSchema() { return schema; } @Override public ColumnVector getColumnVector(int ordinal) { checkColumnOrdinal(ordinal); return columnVectors.get(ordinal); } @Override public ColumnarBatch withNewColumn( int ordinal, StructField structField, ColumnVector columnVector) { if (ordinal < 0 || ordinal > columnVectors.size()) { throw new IllegalArgumentException("Invalid ordinal: " + ordinal); } if (columnVector == null || columnVector.getSize() != size) { throw new IllegalArgumentException( "given vector size is not matching the current batch size"); } // Update the schema ArrayList newStructFields = new ArrayList<>(schema.fields()); newStructFields.ensureCapacity(schema.length() + 1); newStructFields.add(ordinal, structField); StructType newSchema = new StructType(newStructFields); // Update the vectors ArrayList newColumnVectors = new ArrayList<>(columnVectors); newColumnVectors.ensureCapacity(columnVectors.size() + 1); newColumnVectors.add(ordinal, columnVector); return new DefaultColumnarBatch(size, newSchema, newColumnVectors.toArray(new ColumnVector[0])); } @Override public ColumnarBatch withDeletedColumnAt(int ordinal) { if (ordinal < 0 || ordinal >= columnVectors.size()) { throw new IllegalArgumentException("Invalid ordinal: " + ordinal); } // Update the schema ArrayList newStructFields = new ArrayList<>(schema.fields()); newStructFields.remove(ordinal); StructType newSchema = new StructType(newStructFields); // Update the vectors ArrayList newColumnVectors = new ArrayList<>(columnVectors); newColumnVectors.remove(ordinal); return new DefaultColumnarBatch(size, newSchema, newColumnVectors.toArray(new ColumnVector[0])); } @Override public ColumnarBatch withNewSchema(StructType newSchema) { if (!schema.equivalent(newSchema)) { throw new IllegalArgumentException( "Given new schema data type is not same as the existing schema"); } return new DefaultColumnarBatch(size, newSchema, columnVectors.toArray(new ColumnVector[0])); } @Override public int getSize() { return size; } private void checkColumnOrdinal(int ordinal) { if (ordinal < 0 || ordinal >= columnVectors.size()) { throw new IllegalArgumentException("invalid column ordinal: " + ordinal); } } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/DefaultJsonRow.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.data; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.delta.kernel.data.ArrayValue; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.MapValue; import io.delta.kernel.data.Row; import io.delta.kernel.defaults.internal.DefaultKernelUtils; import io.delta.kernel.defaults.internal.data.vector.DefaultGenericVector; import io.delta.kernel.internal.util.InternalUtils; import io.delta.kernel.internal.util.TimestampUtils; import io.delta.kernel.types.*; import java.math.BigDecimal; import java.sql.Date; import java.time.Instant; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; public class DefaultJsonRow implements Row { private final Object[] parsedValues; private final StructType readSchema; public DefaultJsonRow(ObjectNode rootNode, StructType readSchema) { this.readSchema = readSchema; this.parsedValues = new Object[readSchema.length()]; for (int i = 0; i < readSchema.length(); i++) { final StructField field = readSchema.at(i); final Object parsedValue = decodeField(rootNode, field); parsedValues[i] = parsedValue; } } @Override public StructType getSchema() { return readSchema; } @Override public boolean isNullAt(int ordinal) { return parsedValues[ordinal] == null; } @Override public boolean getBoolean(int ordinal) { return (boolean) parsedValues[ordinal]; } @Override public byte getByte(int ordinal) { return (byte) parsedValues[ordinal]; } @Override public short getShort(int ordinal) { return (short) parsedValues[ordinal]; } @Override public int getInt(int ordinal) { return (int) parsedValues[ordinal]; } @Override public long getLong(int ordinal) { return (long) parsedValues[ordinal]; } @Override public float getFloat(int ordinal) { return (float) parsedValues[ordinal]; } @Override public double getDouble(int ordinal) { return (double) parsedValues[ordinal]; } @Override public String getString(int ordinal) { return (String) parsedValues[ordinal]; } @Override public BigDecimal getDecimal(int ordinal) { return (BigDecimal) parsedValues[ordinal]; } @Override public byte[] getBinary(int ordinal) { throw new UnsupportedOperationException("not yet implemented"); } @Override public Row getStruct(int ordinal) { return (DefaultJsonRow) parsedValues[ordinal]; } @Override public ArrayValue getArray(int ordinal) { return (ArrayValue) parsedValues[ordinal]; } @Override public MapValue getMap(int ordinal) { return (MapValue) parsedValues[ordinal]; } private static void throwIfTypeMismatch(String expType, boolean hasExpType, JsonNode jsonNode) { if (!hasExpType) { throw new RuntimeException( String.format("Couldn't decode %s, expected a %s", jsonNode, expType)); } } private static Object decodeElement(JsonNode jsonValue, DataType dataType) { if (jsonValue.isNull()) { return null; } if (dataType instanceof BooleanType) { throwIfTypeMismatch("boolean", jsonValue.isBoolean(), jsonValue); return jsonValue.booleanValue(); } if (dataType instanceof ByteType) { throwIfTypeMismatch( "byte", jsonValue.canConvertToExactIntegral() && jsonValue.canConvertToInt() && jsonValue.intValue() <= Byte.MAX_VALUE && jsonValue.canConvertToInt() && jsonValue.intValue() >= Byte.MIN_VALUE, jsonValue); return jsonValue.numberValue().byteValue(); } if (dataType instanceof ShortType) { throwIfTypeMismatch( "short", jsonValue.canConvertToExactIntegral() && jsonValue.canConvertToInt() && jsonValue.intValue() <= Short.MAX_VALUE && jsonValue.canConvertToInt() && jsonValue.intValue() >= Short.MIN_VALUE, jsonValue); return jsonValue.numberValue().shortValue(); } if (dataType instanceof IntegerType) { throwIfTypeMismatch( "integer", jsonValue.isIntegralNumber() && jsonValue.canConvertToInt(), jsonValue); return jsonValue.intValue(); } if (dataType instanceof LongType) { throwIfTypeMismatch( "long", jsonValue.isIntegralNumber() && jsonValue.canConvertToLong(), jsonValue); return jsonValue.numberValue().longValue(); } if (dataType instanceof FloatType) { switch (jsonValue.getNodeType()) { case NUMBER: throwIfTypeMismatch( "float", // floatValue() will be converted to +/-INF if it cannot be represented // by a float // Note it is still possible to lose precision in this conversion but // checking for that requires converting to a float and back to BigDecimal !Float.isInfinite(jsonValue.floatValue()), jsonValue); return jsonValue.floatValue(); case STRING: switch (jsonValue.asText()) { case "NaN": return Float.NaN; case "+INF": case "+Infinity": case "Infinity": return Float.POSITIVE_INFINITY; case "-INF": case "-Infinity": return Float.NEGATIVE_INFINITY; } default: throwIfTypeMismatch("float", false, jsonValue); } } if (dataType instanceof DoubleType) { switch (jsonValue.getNodeType()) { case NUMBER: throwIfTypeMismatch( "double", // doubleValue() will be converted to +/-INF if it cannot be represented by // a double // Note it is still possible to lose precision in this conversion but // checking for that requires converting to a double and back to BigDecimal !Double.isInfinite(jsonValue.doubleValue()), jsonValue); return jsonValue.doubleValue(); case STRING: switch (jsonValue.asText()) { case "NaN": return Double.NaN; case "+INF": case "+Infinity": case "Infinity": return Double.POSITIVE_INFINITY; case "-INF": case "-Infinity": return Double.NEGATIVE_INFINITY; } default: throwIfTypeMismatch("double", false, jsonValue); } } if (dataType instanceof StringType) { throwIfTypeMismatch("string", jsonValue.isTextual(), jsonValue); return jsonValue.asText(); } if (dataType instanceof DecimalType) { throwIfTypeMismatch("decimal", jsonValue.isNumber(), jsonValue); return jsonValue.decimalValue(); } if (dataType instanceof DateType) { throwIfTypeMismatch("date", jsonValue.isTextual(), jsonValue); return InternalUtils.daysSinceEpoch(Date.valueOf(jsonValue.textValue())); } if (dataType instanceof TimestampType) { throwIfTypeMismatch("timestamp", jsonValue.isTextual(), jsonValue); Instant time = OffsetDateTime.parse(jsonValue.textValue()).toInstant(); return TimestampUtils.toEpochMicros(time); } if (dataType instanceof TimestampNTZType) { throwIfTypeMismatch("timestamp_ntz", jsonValue.isTextual(), jsonValue); return DefaultKernelUtils.parseTimestampNTZ(jsonValue.textValue()); } if (dataType instanceof StructType) { throwIfTypeMismatch("object", jsonValue.isObject(), jsonValue); return new DefaultJsonRow((ObjectNode) jsonValue, (StructType) dataType); } if (dataType instanceof ArrayType) { throwIfTypeMismatch("array", jsonValue.isArray(), jsonValue); final ArrayType arrayType = ((ArrayType) dataType); final ArrayNode jsonArray = (ArrayNode) jsonValue; final Object[] elements = new Object[jsonArray.size()]; for (int i = 0; i < jsonArray.size(); i++) { final JsonNode element = jsonArray.get(i); final Object parsedElement = decodeElement(element, arrayType.getElementType()); if (parsedElement == null && !arrayType.containsNull()) { throw new RuntimeException( "Array type expects no nulls as elements, but " + "received `null` as array element"); } elements[i] = parsedElement; } return new ArrayValue() { @Override public int getSize() { return elements.length; } @Override public ColumnVector getElements() { return DefaultGenericVector.fromArray(arrayType.getElementType(), elements); } }; } if (dataType instanceof MapType) { throwIfTypeMismatch("map", jsonValue.isObject(), jsonValue); final MapType mapType = (MapType) dataType; if (!(mapType.getKeyType() instanceof StringType)) { throw new RuntimeException( "MapType with a key type of `String` is supported, " + "received a key type: " + mapType.getKeyType()); } List keys = new ArrayList<>(jsonValue.size()); List values = new ArrayList<>(jsonValue.size()); final Iterator> iter = jsonValue.fields(); boolean isValueOfStringType = mapType.getValueType() instanceof StringType; while (iter.hasNext()) { Map.Entry entry = iter.next(); String keyParsed = entry.getKey(); Object valueParsed = null; if (isValueOfStringType) { // Special handling for value which is of type string. Delta tables generated by // Delta-Spark ended up having serializing values as their original type and not // as string in the Delta commit files. // Ex. {"key": true} instead of {"key": "true"} if (!entry.getValue().isNull()) { valueParsed = entry.getValue().asText(); } } else { valueParsed = decodeElement(entry.getValue(), mapType.getValueType()); } if (valueParsed == null && !mapType.isValueContainsNull()) { throw new RuntimeException( "Map type expects no nulls in values, but " + "received `null` as value"); } keys.add(keyParsed); values.add(valueParsed); } return new MapValue() { @Override public int getSize() { return jsonValue.size(); } @Override public ColumnVector getKeys() { return DefaultGenericVector.fromList(mapType.getKeyType(), keys); } @Override public ColumnVector getValues() { return DefaultGenericVector.fromList(mapType.getValueType(), values); } }; } throw new UnsupportedOperationException( String.format("Unsupported DataType %s for RootNode %s", dataType, jsonValue)); } private static Object decodeField(ObjectNode rootNode, StructField field) { if (rootNode.get(field.getName()) == null || rootNode.get(field.getName()).isNull()) { if (field.isNullable()) { return null; } throw new RuntimeException( String.format( "Root node at key %s is null but field isn't nullable. Root node: %s", field.getName(), rootNode)); } return decodeElement(rootNode.get(field.getName()), field.getDataType()); } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/DefaultRowBasedColumnarBatch.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.data; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.defaults.internal.data.vector.DefaultSubFieldVector; import io.delta.kernel.types.StructField; import io.delta.kernel.types.StructType; import java.util.ArrayList; import java.util.List; import java.util.Optional; /** * {@link ColumnarBatch} wrapper around list of {@link Row} objects. TODO: We should change the * {@link io.delta.kernel.defaults.engine.DefaultJsonHandler} to generate data in true columnar * format than wrapping a set of rows with a columnar batch interface. */ public class DefaultRowBasedColumnarBatch implements ColumnarBatch { private final StructType schema; private final List rows; /** * Holds the actual ColumnVectors, once the rows have been parsed for that column. * *

Uses lazy initialization, i.e. a value of Optional.empty() at an ordinal means we have not * parsed the rows for that column yet. */ private final List> columnVectors; public DefaultRowBasedColumnarBatch(StructType schema, List rows) { this.schema = schema; this.rows = rows; this.columnVectors = new ArrayList<>(schema.length()); for (int i = 0; i < schema.length(); i++) { columnVectors.add(Optional.empty()); } } @Override public StructType getSchema() { return schema; } @Override public int getSize() { return rows.size(); } @Override public ColumnVector getColumnVector(int ordinal) { if (ordinal < 0 || ordinal >= columnVectors.size()) { throw new IllegalArgumentException("Invalid ordinal: " + ordinal); } if (!columnVectors.get(ordinal).isPresent()) { final StructField field = schema.at(ordinal); final ColumnVector vector = new DefaultSubFieldVector( getSize(), field.getDataType(), ordinal, (rowId) -> rows.get(rowId)); columnVectors.set(ordinal, Optional.of(vector)); } return columnVectors.get(ordinal).get(); } @Override public ColumnarBatch withNewColumn( int ordinal, StructField columnSchema, ColumnVector columnVector) { if (ordinal < 0 || ordinal >= columnVectors.size() + 1) { throw new IllegalArgumentException("Invalid ordinal: " + ordinal); } // Update the schema final List newStructFields = new ArrayList<>(schema.fields()); newStructFields.add(ordinal, columnSchema); final StructType newSchema = new StructType(newStructFields); for (int i = 0; i < columnVectors.size(); i++) { getColumnVector(i); } // Add the vector at the target ordinal final List> newColumnVectors = new ArrayList<>(columnVectors); newColumnVectors.add(ordinal, Optional.of(columnVector)); // Fill the new array ColumnVector[] newColumnVectorArr = new ColumnVector[newColumnVectors.size()]; for (int i = 0; i < newColumnVectorArr.length; i++) { newColumnVectorArr[i] = newColumnVectors.get(i).get(); } return new DefaultColumnarBatch( getSize(), // # of rows hasn't changed newSchema, newColumnVectorArr); } /** TODO this implementation sucks */ @Override public ColumnarBatch withDeletedColumnAt(int ordinal) { if (ordinal < 0 || ordinal >= columnVectors.size()) { throw new IllegalArgumentException("Invalid ordinal: " + ordinal); } // Update the schema final List newStructFields = new ArrayList<>(schema.fields()); newStructFields.remove(ordinal); final StructType newSchema = new StructType(newStructFields); // Fill all the vectors, except the one being deleted for (int i = 0; i < columnVectors.size(); i++) { if (i == ordinal) { continue; } getColumnVector(i); } // Delete the vector at the target ordinal final List> newColumnVectors = new ArrayList<>(columnVectors); newColumnVectors.remove(ordinal); // Fill the new array ColumnVector[] newColumnVectorArr = new ColumnVector[newColumnVectors.size()]; for (int i = 0; i < newColumnVectorArr.length; i++) { newColumnVectorArr[i] = newColumnVectors.get(i).get(); } return new DefaultColumnarBatch( getSize(), // # of rows hasn't changed newSchema, newColumnVectorArr); } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/AbstractColumnVector.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.data.vector; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.data.ArrayValue; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.MapValue; import io.delta.kernel.types.DataType; import java.math.BigDecimal; import java.util.Optional; /** * Abstract implementation of {@link ColumnVector} that provides the default functionality common to * most of the specific data type {@link ColumnVector} implementations. */ public abstract class AbstractColumnVector implements ColumnVector { private final int size; private final DataType dataType; private final Optional nullability; protected AbstractColumnVector(int size, DataType dataType, Optional nullability) { checkArgument(size >= 0, "invalid size: %s", size); nullability.ifPresent( array -> checkArgument( array.length >= size, "invalid number of values (%s) for given size (%s)", array.length, size)); this.size = size; this.dataType = requireNonNull(dataType); this.nullability = requireNonNull(nullability); } @Override public DataType getDataType() { return dataType; } @Override public int getSize() { return size; } @Override public void close() { // By default, nothing to close, if the implementation has any resources to release // it can override it } /** * Is the value at given {@code rowId} index is null? * * @param rowId * @return */ @Override public boolean isNullAt(int rowId) { checkValidRowId(rowId); if (!nullability.isPresent()) { return false; // if there is no-nullability vector, every value is a non-null value } return nullability.get()[rowId]; } @Override public boolean getBoolean(int rowId) { throw unsupportedDataAccessException("boolean"); } @Override public byte getByte(int rowId) { throw unsupportedDataAccessException("byte"); } @Override public short getShort(int rowId) { throw unsupportedDataAccessException("short"); } @Override public int getInt(int rowId) { throw unsupportedDataAccessException("int"); } @Override public long getLong(int rowId) { throw unsupportedDataAccessException("long"); } @Override public float getFloat(int rowId) { throw unsupportedDataAccessException("float"); } @Override public double getDouble(int rowId) { throw unsupportedDataAccessException("double"); } @Override public byte[] getBinary(int rowId) { throw unsupportedDataAccessException("binary"); } @Override public String getString(int rowId) { throw unsupportedDataAccessException("string"); } @Override public BigDecimal getDecimal(int rowId) { throw unsupportedDataAccessException("decimal"); } @Override public MapValue getMap(int rowId) { throw unsupportedDataAccessException("map"); } @Override public ArrayValue getArray(int rowId) { throw unsupportedDataAccessException("array"); } // TODO no need to override these here; update default implementations in `ColumnVector` // to have a more informative exception message protected UnsupportedOperationException unsupportedDataAccessException(String accessType) { String msg = String.format( "Trying to access a `%s` value from vector of type `%s`", accessType, getDataType()); throw new UnsupportedOperationException(msg); } /** * Helper method that make sure the given {@code rowId} position is valid in this vector * * @param rowId */ protected void checkValidRowId(int rowId) { if (rowId < 0 || rowId >= size) { throw new IllegalArgumentException("invalid row access: " + rowId); } } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultArrayVector.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.data.vector; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.data.ArrayValue; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.types.DataType; import java.util.Optional; /** {@link io.delta.kernel.data.ColumnVector} implementation for array type data. */ public class DefaultArrayVector extends AbstractColumnVector { private final int[] offsets; private final ColumnVector elementVector; /** * Create an instance of {@link io.delta.kernel.data.ColumnVector} for array type. * * @param size number of elements in the vector. * @param nullability Optional array of nullability value for each element in the vector. All * values in the vector are considered non-null when parameter is empty. * @param offsets Offsets into element vector on where the index of particular row values start * and end. * @param elementVector Vector containing the array elements. */ public DefaultArrayVector( int size, DataType type, Optional nullability, int[] offsets, ColumnVector elementVector) { super(size, type, nullability); checkArgument(offsets.length >= size + 1, "invalid offset array size"); this.offsets = requireNonNull(offsets, "offsets is null"); this.elementVector = requireNonNull(elementVector, "elementVector is null"); } /** * Get the value at given {@code rowId}. The return value is undefined and can be anything, if the * slot for {@code rowId} is null. * * @param rowId * @return */ @Override public ArrayValue getArray(int rowId) { checkValidRowId(rowId); if (isNullAt(rowId)) { return null; } // use the offsets array to find the starting and ending index in the underlying vector // for this rowId int start = offsets[rowId]; int end = offsets[rowId + 1]; return new ArrayValue() { // create a view over the elements for this rowId private final ColumnVector elements = new DefaultViewVector(elementVector, start, end); @Override public int getSize() { return elements.getSize(); } @Override public ColumnVector getElements() { return elements; } }; } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultBinaryVector.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.data.vector; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.types.BinaryType; import io.delta.kernel.types.DataType; import io.delta.kernel.types.StringType; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Optional; /** {@link io.delta.kernel.data.ColumnVector} implementation for binary type data. */ public class DefaultBinaryVector extends AbstractColumnVector { private final byte[][] values; /** * Create an instance of {@link io.delta.kernel.data.ColumnVector} for binary type. * * @param size number of elements in the vector. * @param values column vector values. */ public DefaultBinaryVector(DataType dataType, int size, byte[][] values) { super(size, dataType, Optional.empty()); checkArgument( dataType instanceof StringType || dataType instanceof BinaryType, "invalid type for binary vector: %s", dataType); this.values = requireNonNull(values, "values is null"); checkArgument( values.length >= size, "invalid number of values (%s) for given size (%s)", values.length, size); } @Override public boolean isNullAt(int rowId) { checkValidRowId(rowId); return values[rowId] == null; } /** * Get the value at given {@code rowId}. The return value is undefined and can be anything, if the * slot for {@code rowId} is null. The error check on {@code rowId} explicitly skipped for * performance reasons. * * @param rowId * @return */ @Override public String getString(int rowId) { if (!(getDataType() instanceof StringType)) { throw unsupportedDataAccessException("string"); } checkValidRowId(rowId); byte[] value = values[rowId]; if (value == null) { return null; } return StandardCharsets.UTF_8.decode(ByteBuffer.wrap(value)).toString(); } /** * Get the value at given {@code rowId}. The return value is undefined and can be anything, if the * slot for {@code rowId} is null. The error check on {@code rowId} explicitly skipped for * performance reasons. * * @param rowId * @return */ @Override public byte[] getBinary(int rowId) { if (!(getDataType() instanceof BinaryType)) { throw unsupportedDataAccessException("binary"); } checkValidRowId(rowId); return values[rowId]; } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultBooleanVector.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.data.vector; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.types.BooleanType; import java.util.Optional; /** {@link io.delta.kernel.data.ColumnVector} implementation for boolean type data. */ public class DefaultBooleanVector extends AbstractColumnVector { private final boolean[] values; /** * Create an instance of {@link io.delta.kernel.data.ColumnVector} for boolean type. * * @param size number of elements in the vector. * @param nullability Optional array of nullability value for each element in the vector. All * values in the vector are considered non-null when parameter is empty. * @param values column vector values. */ public DefaultBooleanVector(int size, Optional nullability, boolean[] values) { super(size, BooleanType.BOOLEAN, nullability); this.values = requireNonNull(values, "values is null"); checkArgument( values.length >= size, "invalid number of values (%s) for given size (%s)", values.length, size); } /** * Get the value at given {@code rowId}. The return value is undefined and can be anything, if the * slot for {@code rowId} is null. * * @param rowId * @return */ @Override public boolean getBoolean(int rowId) { checkValidRowId(rowId); return values[rowId]; } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultByteVector.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.data.vector; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.types.ByteType; import java.util.Optional; /** {@link io.delta.kernel.data.ColumnVector} implementation for byte type data. */ public class DefaultByteVector extends AbstractColumnVector { private final byte[] values; /** * Create an instance of {@link io.delta.kernel.data.ColumnVector} for byte type. * * @param size number of elements in the vector. * @param nullability Optional array of nullability value for each element in the vector. All * values in the vector are considered non-null when parameter is empty. * @param values column vector values. */ public DefaultByteVector(int size, Optional nullability, byte[] values) { super(size, ByteType.BYTE, nullability); this.values = requireNonNull(values, "values is null"); checkArgument( values.length >= size, "invalid number of values (%s) for given size (%s)", values.length, size); } /** * Get the value at given {@code rowId}. The return value is undefined and can be anything, if the * slot for {@code rowId} is null. * * @param rowId * @return */ @Override public byte getByte(int rowId) { checkValidRowId(rowId); return values[rowId]; } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultConstantVector.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.data.vector; import io.delta.kernel.types.DataType; public class DefaultConstantVector extends DefaultGenericVector { public DefaultConstantVector(DataType dataType, int numRows, Object value) { // TODO: Validate datatype and value object type super(numRows, dataType, (rowId) -> value); } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultDecimalVector.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.data.vector; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.types.DataType; import io.delta.kernel.types.DecimalType; import java.math.BigDecimal; import java.util.Optional; /** {@link io.delta.kernel.data.ColumnVector} implementation for decimal type data. */ public class DefaultDecimalVector extends AbstractColumnVector { private final BigDecimal[] values; /** * Create an instance of {@link io.delta.kernel.data.ColumnVector} for decimal type. * * @param size number of elements in the vector. * @param values column vector values. */ public DefaultDecimalVector(DataType dataType, int size, BigDecimal[] values) { super(size, dataType, Optional.empty()); checkArgument(dataType instanceof DecimalType, "invalid type for decimal vector: %s", dataType); this.values = requireNonNull(values, "values is null"); checkArgument( values.length >= size, "invalid number of values (%s) for given size (%s)", values.length, size); } @Override public boolean isNullAt(int rowId) { checkValidRowId(rowId); return values[rowId] == null; } @Override public BigDecimal getDecimal(int rowId) { checkValidRowId(rowId); return values[rowId]; } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultDoubleVector.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.data.vector; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.types.DoubleType; import java.util.Optional; /** {@link io.delta.kernel.data.ColumnVector} implementation for double type data. */ public class DefaultDoubleVector extends AbstractColumnVector { private final double[] values; /** * Create an instance of {@link io.delta.kernel.data.ColumnVector} for float type. * * @param size number of elements in the vector. * @param nullability Optional array of nullability value for each element in the vector. All * values in the vector are considered non-null when parameter is empty. * @param values column vector values. */ public DefaultDoubleVector(int size, Optional nullability, double[] values) { super(size, DoubleType.DOUBLE, nullability); this.values = requireNonNull(values, "values is null"); checkArgument( values.length >= size, "invalid number of values (%s) for given size (%s)", values.length, size); } /** * Get the value at given {@code rowId}. The return value is undefined and can be anything, if the * slot for {@code rowId} is null. * * @param rowId * @return */ @Override public double getDouble(int rowId) { checkValidRowId(rowId); return values[rowId]; } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultFloatVector.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.data.vector; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.types.FloatType; import java.util.Optional; /** {@link io.delta.kernel.data.ColumnVector} implementation for float type data. */ public class DefaultFloatVector extends AbstractColumnVector { private final float[] values; /** * Create an instance of {@link io.delta.kernel.data.ColumnVector} for float type. * * @param size number of elements in the vector. * @param nullability Optional array of nullability value for each element in the vector. All * values in the vector are considered non-null when parameter is empty. * @param values column vector values. */ public DefaultFloatVector(int size, Optional nullability, float[] values) { super(size, FloatType.FLOAT, nullability); this.values = requireNonNull(values, "values is null"); checkArgument( values.length >= size, "invalid number of values (%s) for given size (%s)", values.length, size); } /** * Get the value at given {@code rowId}. The return value is undefined and can be anything, if the * slot for {@code rowId} is null. * * @param rowId * @return */ @Override public float getFloat(int rowId) { checkValidRowId(rowId); return values[rowId]; } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultGenericVector.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.data.vector; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.*; import io.delta.kernel.types.*; import java.math.BigDecimal; import java.util.List; import java.util.function.Function; /** Generic column vector implementation to expose an array of objects as a column vector. */ public class DefaultGenericVector implements ColumnVector { public static DefaultGenericVector fromArray(DataType dataType, Object[] elements) { return new DefaultGenericVector(elements.length, dataType, rowId -> elements[rowId]); } public static DefaultGenericVector fromList(DataType dataType, List elements) { return new DefaultGenericVector(elements.size(), dataType, rowId -> elements.get(rowId)); } private final int size; private final DataType dataType; private final Function rowIdToValueAccessor; protected DefaultGenericVector( int size, DataType dataType, Function rowIdToValueAccessor) { this.size = size; this.dataType = dataType; this.rowIdToValueAccessor = rowIdToValueAccessor; } @Override public DataType getDataType() { return dataType; } @Override public int getSize() { return size; } @Override public void close() {} @Override public boolean isNullAt(int rowId) { assertValidRowId(rowId); return rowIdToValueAccessor.apply(rowId) == null; } @Override public boolean getBoolean(int rowId) { assertValidRowId(rowId); throwIfUnsafeAccess(BooleanType.class, "boolean"); return (boolean) rowIdToValueAccessor.apply(rowId); } @Override public byte getByte(int rowId) { assertValidRowId(rowId); throwIfUnsafeAccess(ByteType.class, "byte"); return (byte) rowIdToValueAccessor.apply(rowId); } @Override public short getShort(int rowId) { assertValidRowId(rowId); throwIfUnsafeAccess(ShortType.class, "short"); return (short) rowIdToValueAccessor.apply(rowId); } @Override public int getInt(int rowId) { assertValidRowId(rowId); throwIfUnsafeAccess(IntegerType.class, DateType.class, dataType.toString()); return (int) rowIdToValueAccessor.apply(rowId); } @Override public long getLong(int rowId) { assertValidRowId(rowId); throwIfUnsafeAccess( LongType.class, TimestampType.class, TimestampNTZType.class, dataType.toString()); return (long) rowIdToValueAccessor.apply(rowId); } @Override public float getFloat(int rowId) { assertValidRowId(rowId); throwIfUnsafeAccess(FloatType.class, "float"); return (float) rowIdToValueAccessor.apply(rowId); } @Override public double getDouble(int rowId) { assertValidRowId(rowId); throwIfUnsafeAccess(DoubleType.class, "double"); return (double) rowIdToValueAccessor.apply(rowId); } @Override public String getString(int rowId) { assertValidRowId(rowId); throwIfUnsafeAccess(StringType.class, "string"); return (String) rowIdToValueAccessor.apply(rowId); } @Override public BigDecimal getDecimal(int rowId) { assertValidRowId(rowId); throwIfUnsafeAccess(DecimalType.class, "decimal"); return (BigDecimal) rowIdToValueAccessor.apply(rowId); } @Override public byte[] getBinary(int rowId) { assertValidRowId(rowId); throwIfUnsafeAccess(BinaryType.class, "binary"); return (byte[]) rowIdToValueAccessor.apply(rowId); } @Override public ArrayValue getArray(int rowId) { assertValidRowId(rowId); // TODO: not sufficient check, also need to check the element type throwIfUnsafeAccess(ArrayType.class, "array"); return (ArrayValue) rowIdToValueAccessor.apply(rowId); } @Override public MapValue getMap(int rowId) { assertValidRowId(rowId); // TODO: not sufficient check, also need to check the element types throwIfUnsafeAccess(MapType.class, "map"); return (MapValue) rowIdToValueAccessor.apply(rowId); } @Override public ColumnVector getChild(int ordinal) { throwIfUnsafeAccess(StructType.class, "struct"); StructType structType = (StructType) dataType; return new DefaultSubFieldVector( getSize(), structType.at(ordinal).getDataType(), ordinal, (rowId) -> (Row) rowIdToValueAccessor.apply(rowId)); } private void throwIfUnsafeAccess(Class expDataType, String accessType) { if (!expDataType.isAssignableFrom(dataType.getClass())) { String msg = String.format( "Trying to access a `%s` value from vector of type `%s`", accessType, dataType); throw new UnsupportedOperationException(msg); } } private void throwIfUnsafeAccess( Class expDataType1, Class expDataType2, String accessType) { if (!(expDataType1.isAssignableFrom(dataType.getClass()) || expDataType2.isAssignableFrom(dataType.getClass()))) { String msg = String.format( "Trying to access a `%s` value from vector of type `%s`", accessType, dataType); throw new UnsupportedOperationException(msg); } } private void throwIfUnsafeAccess( Class expDataType1, Class expDataType2, Class expDataType3, String accessType) { if (!(expDataType1.isAssignableFrom(dataType.getClass()) || expDataType2.isAssignableFrom(dataType.getClass()) || expDataType3.isAssignableFrom(dataType.getClass()))) { String msg = String.format( "Trying to access a `%s` value from vector of type `%s`", accessType, dataType); throw new UnsupportedOperationException(msg); } } private void assertValidRowId(int rowId) { checkArgument(rowId < size, "Invalid rowId: %s, max allowed rowId is: %s", rowId, (size - 1)); } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultIntVector.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.data.vector; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.types.DataType; import io.delta.kernel.types.DateType; import io.delta.kernel.types.IntegerType; import java.util.Optional; /** {@link io.delta.kernel.data.ColumnVector} implementation for integer type data. */ public class DefaultIntVector extends AbstractColumnVector { private final int[] values; /** * Create an instance of {@link io.delta.kernel.data.ColumnVector} for integer type. * * @param size number of elements in the vector. * @param nullability Optional array of nullability value for each element in the vector. All * values in the vector are considered non-null when parameter is empty. * @param values column vector values. */ public DefaultIntVector( DataType dataType, int size, Optional nullability, int[] values) { super(size, dataType, nullability); checkArgument(dataType instanceof IntegerType || dataType instanceof DateType); this.values = requireNonNull(values, "values is null"); checkArgument( values.length >= size, "invalid number of values (%s) for given size (%s)", values.length, size); } /** * Get the value at given {@code rowId}. The return value is undefined and can be anything, if the * slot for {@code rowId} is null. * * @param rowId * @return */ @Override public int getInt(int rowId) { checkValidRowId(rowId); return values[rowId]; } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultLongVector.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.data.vector; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.types.*; import java.util.Optional; /** {@link ColumnVector} implementation for long, timestamp or timestamp_ntz type data. */ public class DefaultLongVector extends AbstractColumnVector { private final long[] values; /** * Create an instance of {@link ColumnVector} for long type. * * @param size number of elements in the vector. * @param nullability Optional array of nullability value for each element in the vector. All * values in the vector are considered non-null when parameter is empty. * @param values column vector values. */ public DefaultLongVector( DataType dataType, int size, Optional nullability, long[] values) { super(size, dataType, nullability); checkArgument( dataType instanceof LongType || dataType instanceof TimestampType || dataType instanceof TimestampNTZType); this.values = requireNonNull(values, "values is null"); checkArgument( values.length >= size, "invalid number of values (%s) for given size (%s)", values.length, size); } /** * Get the value at given {@code rowId}. The return value is undefined and can be anything, if the * slot for {@code rowId} is null. * * @param rowId * @return */ @Override public long getLong(int rowId) { checkValidRowId(rowId); return values[rowId]; } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultMapVector.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.data.vector; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.MapValue; import io.delta.kernel.types.DataType; import java.util.Optional; /** {@link io.delta.kernel.data.ColumnVector} implementation for map type data. */ public class DefaultMapVector extends AbstractColumnVector { private final int[] offsets; private final ColumnVector keyVector; private final ColumnVector valueVector; /** * Create an instance of {@link io.delta.kernel.data.ColumnVector} for map type. * * @param size number of elements in the vector. * @param nullability Optional array of nullability value for each element in the vector. All * values in the vector are considered non-null when parameter is empty. * @param offsets Offsets into key and value column vectors on where the index of particular row * values start and end. * @param keyVector Vector containing the `key` values from the kv map. * @param valueVector Vector containing the `value` values from the kv map. */ public DefaultMapVector( int size, DataType type, Optional nullability, int[] offsets, ColumnVector keyVector, ColumnVector valueVector) { super(size, type, nullability); checkArgument(offsets.length >= size + 1, "invalid offset array size"); this.offsets = requireNonNull(offsets, "offsets is null"); this.keyVector = requireNonNull(keyVector, "keyVector is null"); this.valueVector = requireNonNull(valueVector, "valueVector is null"); } /** * Get the value at given {@code rowId}. The return value is undefined and can be anything, if the * slot for {@code rowId} is null. * * @param rowId * @return */ @Override public MapValue getMap(int rowId) { checkValidRowId(rowId); if (isNullAt(rowId)) { return null; } // use the offsets array to find the starting and ending index in the underlying vectors // for this rowId int start = offsets[rowId]; int end = offsets[rowId + 1]; return new MapValue() { // create a view over the keys and values for this rowId private final ColumnVector keys = new DefaultViewVector(keyVector, start, end); private final ColumnVector values = new DefaultViewVector(valueVector, start, end); @Override public int getSize() { return keys.getSize(); } @Override public ColumnVector getKeys() { return keys; } @Override public ColumnVector getValues() { return values; } }; } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultShortVector.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.data.vector; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.types.ShortType; import java.util.Optional; /** {@link io.delta.kernel.data.ColumnVector} implementation for short type data. */ public class DefaultShortVector extends AbstractColumnVector { private final short[] values; /** * Create an instance of {@link io.delta.kernel.data.ColumnVector} for short type. * * @param size number of elements in the vector. * @param nullability Optional array of nullability value for each element in the vector. All * values in the vector are considered non-null when parameter is empty. * @param values column vector values. */ public DefaultShortVector(int size, Optional nullability, short[] values) { super(size, ShortType.SHORT, nullability); this.values = requireNonNull(values, "values is null"); checkArgument( values.length >= size, "invalid number of values (%s) for given size (%s)", values.length, size); } /** * Get the value at given {@code rowId}. The return value is undefined and can be anything, if the * slot for {@code rowId} is null. * * @param rowId * @return */ @Override public short getShort(int rowId) { checkValidRowId(rowId); return values[rowId]; } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultStructVector.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.data.vector; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.types.DataType; import io.delta.kernel.types.StructType; import java.util.Optional; /** {@link io.delta.kernel.data.ColumnVector} implementation for struct type data. */ public class DefaultStructVector extends AbstractColumnVector { private final ColumnVector[] memberVectors; /** * Create an instance of {@link ColumnVector} for {@code struct} type. * * @param size number of elements in the vector. * @param dataType {@code struct} datatype definition. * @param nullability Optional array of nullability value for each element in the vector. All * values in the vector are considered non-null when parameter is empty. * @param memberVectors column vectors for each member of the struct. */ public DefaultStructVector( int size, DataType dataType, Optional nullability, ColumnVector[] memberVectors) { super(size, dataType, nullability); checkArgument(dataType instanceof StructType, "not a struct type"); StructType structType = (StructType) dataType; checkArgument( structType.length() == memberVectors.length, "expected a one column vector for each member"); this.memberVectors = memberVectors; } @Override public ColumnVector getChild(int ordinal) { checkArgument(ordinal >= 0 && ordinal < memberVectors.length, "Invalid ordinal %s", ordinal); return memberVectors[ordinal]; } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultSubFieldVector.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.data.vector; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.data.ArrayValue; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.MapValue; import io.delta.kernel.data.Row; import io.delta.kernel.types.DataType; import io.delta.kernel.types.StructField; import io.delta.kernel.types.StructType; import java.math.BigDecimal; import java.util.function.Function; /** * {@link ColumnVector} wrapper on top of {@link Row} objects. This wrapper allows referencing any * nested level column vector from a set of rows. */ public class DefaultSubFieldVector implements ColumnVector { private final int size; private final DataType dataType; private final int columnOrdinal; private final Function rowIdToRowAccessor; /** * Create an instance of {@link DefaultSubFieldVector} * * @param size Number of elements in the vector * @param dataType Datatype of the vector * @param columnOrdinal Ordinal of the column represented by this vector in the rows returned by * {@link #rowIdToRowAccessor} * @param rowIdToRowAccessor {@link Function} that returns a {@link Row} object for given rowId */ public DefaultSubFieldVector( int size, DataType dataType, int columnOrdinal, Function rowIdToRowAccessor) { checkArgument(size >= 0, "invalid size: %s", size); this.size = size; checkArgument(columnOrdinal >= 0, "invalid column ordinal: %s", columnOrdinal); this.columnOrdinal = columnOrdinal; this.rowIdToRowAccessor = requireNonNull(rowIdToRowAccessor, "rowIdToRowAccessor is null"); this.dataType = requireNonNull(dataType, "dataType is null"); } @Override public DataType getDataType() { return dataType; } @Override public int getSize() { return size; } @Override public void close() { /* nothing to close */ } @Override public boolean isNullAt(int rowId) { assertValidRowId(rowId); Row row = rowIdToRowAccessor.apply(rowId); return row == null || row.isNullAt(columnOrdinal); } @Override public boolean getBoolean(int rowId) { assertValidRowId(rowId); return rowIdToRowAccessor.apply(rowId).getBoolean(columnOrdinal); } @Override public byte getByte(int rowId) { assertValidRowId(rowId); return rowIdToRowAccessor.apply(rowId).getByte(columnOrdinal); } @Override public short getShort(int rowId) { assertValidRowId(rowId); return rowIdToRowAccessor.apply(rowId).getShort(columnOrdinal); } @Override public int getInt(int rowId) { assertValidRowId(rowId); return rowIdToRowAccessor.apply(rowId).getInt(columnOrdinal); } @Override public long getLong(int rowId) { assertValidRowId(rowId); return rowIdToRowAccessor.apply(rowId).getLong(columnOrdinal); } @Override public float getFloat(int rowId) { assertValidRowId(rowId); return rowIdToRowAccessor.apply(rowId).getFloat(columnOrdinal); } @Override public double getDouble(int rowId) { assertValidRowId(rowId); return rowIdToRowAccessor.apply(rowId).getDouble(columnOrdinal); } @Override public byte[] getBinary(int rowId) { assertValidRowId(rowId); return rowIdToRowAccessor.apply(rowId).getBinary(columnOrdinal); } @Override public String getString(int rowId) { assertValidRowId(rowId); return rowIdToRowAccessor.apply(rowId).getString(columnOrdinal); } @Override public BigDecimal getDecimal(int rowId) { assertValidRowId(rowId); return rowIdToRowAccessor.apply(rowId).getDecimal(columnOrdinal); } @Override public MapValue getMap(int rowId) { assertValidRowId(rowId); return rowIdToRowAccessor.apply(rowId).getMap(columnOrdinal); } @Override public ArrayValue getArray(int rowId) { assertValidRowId(rowId); return rowIdToRowAccessor.apply(rowId).getArray(columnOrdinal); } @Override public ColumnVector getChild(int childOrdinal) { StructType structType = (StructType) dataType; StructField childField = structType.at(childOrdinal); return new DefaultSubFieldVector( size, childField.getDataType(), childOrdinal, (rowId) -> { if (isNullAt(rowId)) { return null; } else { return rowIdToRowAccessor.apply(rowId).getStruct(columnOrdinal); } }); } private void assertValidRowId(int rowId) { checkArgument(rowId < size, "Invalid rowId: %s, max allowed rowId is: %s", rowId, (size - 1)); } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultViewVector.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.data.vector; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.ArrayValue; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.MapValue; import io.delta.kernel.types.DataType; import java.math.BigDecimal; /** Provides a restricted view on an underlying column vector. */ public class DefaultViewVector implements ColumnVector { private final ColumnVector underlyingVector; private final int offset; private final int size; /** * @param underlyingVector the underlying column vector to read * @param start the row index of the underlyingVector where we want this vector to start * @param end the row index of the underlyingVector where we want this vector to end (exclusive) */ public DefaultViewVector(ColumnVector underlyingVector, int start, int end) { this.underlyingVector = underlyingVector; this.offset = start; this.size = end - start; } @Override public DataType getDataType() { return underlyingVector.getDataType(); } @Override public int getSize() { return size; } @Override public void close() { // Don't close the underlying vector as it may still be used } @Override public boolean isNullAt(int rowId) { checkValidRowId(rowId); return underlyingVector.isNullAt(offset + rowId); } @Override public boolean getBoolean(int rowId) { checkValidRowId(rowId); return underlyingVector.getBoolean(offset + rowId); } @Override public byte getByte(int rowId) { checkValidRowId(rowId); return underlyingVector.getByte(offset + rowId); } @Override public short getShort(int rowId) { checkValidRowId(rowId); return underlyingVector.getShort(offset + rowId); } @Override public int getInt(int rowId) { checkValidRowId(rowId); return underlyingVector.getInt(offset + rowId); } @Override public long getLong(int rowId) { checkValidRowId(rowId); return underlyingVector.getLong(offset + rowId); } @Override public float getFloat(int rowId) { checkValidRowId(rowId); return underlyingVector.getFloat(offset + rowId); } @Override public double getDouble(int rowId) { checkValidRowId(rowId); return underlyingVector.getDouble(offset + rowId); } @Override public byte[] getBinary(int rowId) { checkValidRowId(rowId); return underlyingVector.getBinary(offset + rowId); } @Override public String getString(int rowId) { checkValidRowId(rowId); return underlyingVector.getString(offset + rowId); } @Override public BigDecimal getDecimal(int rowId) { checkValidRowId(rowId); return underlyingVector.getDecimal(offset + rowId); } @Override public MapValue getMap(int rowId) { checkValidRowId(rowId); return underlyingVector.getMap(offset + rowId); } @Override public ArrayValue getArray(int rowId) { checkValidRowId(rowId); return underlyingVector.getArray(offset + rowId); } @Override public ColumnVector getChild(int ordinal) { return new DefaultViewVector(underlyingVector.getChild(ordinal), offset, offset + size); } private void checkValidRowId(int rowId) { checkArgument(rowId >= 0 && rowId < size, "Invalid rowId=%s for size=%s", rowId, size); } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.expressions; import static io.delta.kernel.defaults.internal.DefaultEngineErrors.unsupportedExpressionException; import static io.delta.kernel.defaults.internal.expressions.DefaultExpressionUtils.*; import static io.delta.kernel.defaults.internal.expressions.ImplicitCastExpression.canCastTo; import static io.delta.kernel.internal.util.ExpressionUtils.*; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toList; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.defaults.internal.data.vector.DefaultBooleanVector; import io.delta.kernel.defaults.internal.data.vector.DefaultConstantVector; import io.delta.kernel.engine.ExpressionHandler; import io.delta.kernel.expressions.*; import io.delta.kernel.types.*; import java.util.*; import java.util.stream.Collectors; /** * Implementation of {@link ExpressionEvaluator} for default {@link ExpressionHandler}. It takes * care of validating, adding necessary implicit casts and evaluating the {@link Expression} on * given {@link ColumnarBatch}. */ public class DefaultExpressionEvaluator implements ExpressionEvaluator { private final Expression expression; /** * Create a {@link DefaultExpressionEvaluator} instance bound to the given expression and * inputSchem. * * @param inputSchema Input data schema * @param expression Expression to evaluate. * @param outputType Expected result data type. */ public DefaultExpressionEvaluator( StructType inputSchema, Expression expression, DataType outputType) { ExpressionTransformResult transformResult = new ExpressionTransformer(inputSchema).visit(expression); if (!transformResult.outputType.equivalent(outputType)) { String reason = String.format( "Expression %s does not match expected output type %s", expression, outputType); throw unsupportedExpressionException(expression, reason); } this.expression = transformResult.expression; } @Override public ColumnVector eval(ColumnarBatch input) { return new ExpressionEvalVisitor(input).visit(expression); } @Override public void close() { /* nothing to close */ } /** Encapsulates the result of {@link ExpressionTransformer} */ private static class ExpressionTransformResult { public final Expression expression; // transformed expression public final DataType outputType; // output type of the expression ExpressionTransformResult(Expression expression, DataType outputType) { this.expression = expression; this.outputType = outputType; } } /** * Implementation of {@link ExpressionVisitor} to validate the given expression as follows. * *
    *
  • given input column is part of the input data schema *
  • expression inputs are of supported types. Insert cast according to the rules in {@link * ImplicitCastExpression} to make the types compatible for evaluation by {@link * ExpressionEvalVisitor} *
* *

Return type of each expression visit is a tuple of new rewritten expression and its result * data type. */ private static class ExpressionTransformer extends ExpressionVisitor { private StructType inputDataSchema; ExpressionTransformer(StructType inputDataSchema) { this.inputDataSchema = requireNonNull(inputDataSchema, "inputDataSchema is null"); } @Override ExpressionTransformResult visitAnd(And and) { Predicate left = validateIsPredicate(and, visit(and.getLeft())); Predicate right = validateIsPredicate(and, visit(and.getRight())); return new ExpressionTransformResult(new And(left, right), BooleanType.BOOLEAN); } @Override ExpressionTransformResult visitOr(Or or) { Predicate left = validateIsPredicate(or, visit(or.getLeft())); Predicate right = validateIsPredicate(or, visit(or.getRight())); return new ExpressionTransformResult(new Or(left, right), BooleanType.BOOLEAN); } @Override ExpressionTransformResult visitAlwaysTrue(AlwaysTrue alwaysTrue) { // nothing to validate or rewrite. return new ExpressionTransformResult(alwaysTrue, BooleanType.BOOLEAN); } @Override ExpressionTransformResult visitAlwaysFalse(AlwaysFalse alwaysFalse) { // nothing to validate or rewrite. return new ExpressionTransformResult(alwaysFalse, BooleanType.BOOLEAN); } @Override ExpressionTransformResult visitComparator(Predicate predicate) { switch (predicate.getName()) { case "=": case ">": case ">=": case "<": case "<=": case "IS NOT DISTINCT FROM": return new ExpressionTransformResult( transformBinaryComparator(predicate), BooleanType.BOOLEAN); default: // We should never reach this based on the ExpressionVisitor throw new IllegalStateException( String.format("%s is not a recognized comparator", predicate.getName())); } } @Override ExpressionTransformResult visitLiteral(Literal literal) { // nothing to validate or rewrite return new ExpressionTransformResult(literal, literal.getDataType()); } @Override ExpressionTransformResult visitColumn(Column column) { String[] names = column.getNames(); DataType currentType = inputDataSchema; for (int level = 0; level < names.length; level++) { assertColumnExists(currentType instanceof StructType, inputDataSchema, column); StructType structSchema = ((StructType) currentType); int ordinal = structSchema.indexOf(names[level]); assertColumnExists(ordinal != -1, inputDataSchema, column); currentType = structSchema.at(ordinal).getDataType(); } assertColumnExists(currentType != null, inputDataSchema, column); return new ExpressionTransformResult(column, currentType); } @Override ExpressionTransformResult visitCast(ImplicitCastExpression cast) { throw new UnsupportedOperationException("CAST expression is not expected."); } @Override ExpressionTransformResult visitPartitionValue(PartitionValueExpression partitionValue) { ExpressionTransformResult serializedPartValueInput = visit(partitionValue.getInput()); checkArgument( serializedPartValueInput.outputType instanceof StringType, "%s: expected string input, but got %s", partitionValue, serializedPartValueInput.outputType); DataType partitionColType = partitionValue.getDataType(); if (partitionColType instanceof StructType || partitionColType instanceof ArrayType || partitionColType instanceof MapType) { throw unsupportedExpressionException( partitionValue, "unsupported partition data type: " + partitionColType); } return new ExpressionTransformResult( new PartitionValueExpression(serializedPartValueInput.expression, partitionColType), partitionColType); } @Override ExpressionTransformResult visitElementAt(ScalarExpression elementAt) { ExpressionTransformResult transformedMapInput = visit(childAt(elementAt, 0)); ExpressionTransformResult transformedLookupKey = visit(childAt(elementAt, 1)); ScalarExpression transformedExpression = ElementAtEvaluator.validateAndTransform( elementAt, transformedMapInput.expression, transformedMapInput.outputType, transformedLookupKey.expression, transformedLookupKey.outputType); return new ExpressionTransformResult( transformedExpression, ((MapType) transformedMapInput.outputType).getValueType()); } @Override ExpressionTransformResult visitNot(Predicate predicate) { Predicate child = validateIsPredicate(predicate, visit(predicate.getChildren().get(0))); return new ExpressionTransformResult( new Predicate(predicate.getName(), child), BooleanType.BOOLEAN); } @Override ExpressionTransformResult visitIsNotNull(Predicate predicate) { Expression child = visit(predicate.getChildren().get(0)).expression; return new ExpressionTransformResult( new Predicate(predicate.getName(), child), BooleanType.BOOLEAN); } @Override ExpressionTransformResult visitIsNull(Predicate predicate) { Expression child = visit(getUnaryChild(predicate)).expression; return new ExpressionTransformResult( new Predicate(predicate.getName(), child), BooleanType.BOOLEAN); } @Override ExpressionTransformResult visitCoalesce(ScalarExpression coalesce) { List children = coalesce.getChildren().stream().map(this::visit).collect(Collectors.toList()); if (children.isEmpty()) { throw unsupportedExpressionException(coalesce, "Coalesce requires at least one expression"); } // TODO support least-common-type resolution long numDistinctTypes = children.stream().map(e -> e.outputType).distinct().count(); if (numDistinctTypes > 1) { throw unsupportedExpressionException( coalesce, "Coalesce is only supported for arguments of the same type"); } return new ExpressionTransformResult( new ScalarExpression( "COALESCE", children.stream().map(e -> e.expression).collect(Collectors.toList())), children.get(0).outputType); } @Override ExpressionTransformResult visitAdd(ScalarExpression add) { List children = add.getChildren().stream().map(this::visit).collect(Collectors.toList()); if (children.size() != 2) { throw unsupportedExpressionException( add, "ADD requires exactly two arguments: left and right operands"); } if (!children.get(0).outputType.equivalent(children.get(1).outputType)) { throw unsupportedExpressionException( add, "ADD is only supported for arguments of the same type"); } if (!(children.get(0).outputType instanceof ByteType || children.get(0).outputType instanceof ShortType || children.get(0).outputType instanceof IntegerType || children.get(0).outputType instanceof LongType || children.get(0).outputType instanceof FloatType || children.get(0).outputType instanceof DoubleType)) { throw unsupportedExpressionException( add, "ADD is only supported for numeric types: byte, short, int, long, float, double"); } return new ExpressionTransformResult( new ScalarExpression( "ADD", Arrays.asList(children.get(0).expression, children.get(1).expression)), children.get(0).outputType); } @Override ExpressionTransformResult visitTimeAdd(ScalarExpression timeAdd) { List children = timeAdd.getChildren().stream().map(this::visit).collect(Collectors.toList()); if (children.size() != 2) { throw unsupportedExpressionException( timeAdd, "TIMEADD requires exactly two arguments: timestamp column and milliseconds"); } Expression timestampColumn = children.get(0).expression; Expression durationMilliseconds = children.get(1).expression; DataType timestampColumnType = children.get(0).outputType; DataType literalColumnType = children.get(1).outputType; // Ensure the first child is either a TimestampType or a TimestampNTZType, // and the second is a LongType. if (!((timestampColumnType instanceof TimestampType || timestampColumnType instanceof TimestampNTZType) && (literalColumnType instanceof LongType))) { throw new IllegalArgumentException( "TIMEADD requires a timestamp and a Long (milliseconds) to add to it"); } return new ExpressionTransformResult( new ScalarExpression("TIMEADD", Arrays.asList(timestampColumn, durationMilliseconds)), timestampColumnType // Result is also a timestamp ); } @Override ExpressionTransformResult visitSubstring(ScalarExpression substring) { List children = substring.getChildren().stream().map(this::visit).collect(toList()); ScalarExpression transformedExpression = SubstringEvaluator.validateAndTransform( substring, children.stream().map(e -> e.expression).collect(toList()), children.stream().map(e -> e.outputType).collect(toList())); return new ExpressionTransformResult(transformedExpression, StringType.STRING); } @Override ExpressionTransformResult visitLike(final Predicate like) { List children = like.getChildren().stream().map(this::visit).collect(toList()); Predicate transformedExpression = LikeExpressionEvaluator.validateAndTransform( like, children.stream().map(e -> e.expression).collect(toList()), children.stream().map(e -> e.outputType).collect(toList())); return new ExpressionTransformResult(transformedExpression, BooleanType.BOOLEAN); } @Override ExpressionTransformResult visitStartsWith(Predicate startsWith) { List children = startsWith.getChildren().stream().map(this::visit).collect(toList()); Predicate transformedExpression = StartsWithExpressionEvaluator.validateAndTransform( startsWith, children.stream().map(e -> e.expression).collect(toList()), children.stream().map(e -> e.outputType).collect(toList())); return new ExpressionTransformResult(transformedExpression, BooleanType.BOOLEAN); } @Override ExpressionTransformResult visitIn(In in) { ExpressionTransformResult visitedValue = visit(in.getValueExpression()); List visitedInList = in.getInListElements().stream().map(this::visit).collect(toList()); In transformedExpression = InExpressionEvaluator.validateAndTransform( in, visitedValue.expression, visitedValue.outputType, visitedInList.stream().map(e -> e.expression).collect(toList()), visitedInList.stream().map(e -> e.outputType).collect(toList())); return new ExpressionTransformResult(transformedExpression, BooleanType.BOOLEAN); } private Predicate validateIsPredicate( Expression baseExpression, ExpressionTransformResult result) { checkArgument( result.outputType instanceof BooleanType && result.expression instanceof Predicate, "%s: expected a predicate expression but got %s with output type %s.", baseExpression, result.expression, result.outputType); return (Predicate) result.expression; } private Expression transformBinaryComparator(Predicate predicate) { ExpressionTransformResult leftResult = visit(getLeft(predicate)); ExpressionTransformResult rightResult = visit(getRight(predicate)); Expression left = leftResult.expression; Expression right = rightResult.expression; if (predicate.getCollationIdentifier().isPresent()) { CollationIdentifier collationIdentifier = predicate.getCollationIdentifier().get(); checkIsUTF8BinaryCollation(predicate, collationIdentifier); for (DataType dataType : Arrays.asList(leftResult.outputType, rightResult.outputType)) { checkIsStringType( dataType, predicate, format("Predicate %s expects STRING type inputs", predicate.getName())); } return new Predicate(predicate.getName(), left, right, collationIdentifier); } if (!leftResult.outputType.equivalent(rightResult.outputType)) { if (canCastTo(leftResult.outputType, rightResult.outputType)) { left = new ImplicitCastExpression(left, rightResult.outputType); } else if (canCastTo(rightResult.outputType, leftResult.outputType)) { right = new ImplicitCastExpression(right, leftResult.outputType); } else { String msg = format( "operands are of different types which are not " + "comparable: left type=%s, right type=%s", leftResult.outputType, rightResult.outputType); throw unsupportedExpressionException(predicate, msg); } } return new Predicate(predicate.getName(), left, right); } } /** * Implementation of {@link ExpressionVisitor} to evaluate expression on a {@link ColumnarBatch}. */ private static class ExpressionEvalVisitor extends ExpressionVisitor { private final ColumnarBatch input; ExpressionEvalVisitor(ColumnarBatch input) { this.input = input; } /* | Operand 1 | Operand 2 | `AND` | `OR` | |-----------|-----------|------------|------------| | True | True | True | True | | True | False | False | True | | True | NULL | NULL | True | | False | True | False | True | | False | False | False | False | | False | NULL | False | NULL | | NULL | True | NULL | True | | NULL | False | False | NULL | | NULL | NULL | NULL | NULL | */ @Override ColumnVector visitAnd(And and) { PredicateChildrenEvalResult argResults = evalBinaryExpressionChildren(and); ColumnVector left = argResults.leftResult; ColumnVector right = argResults.rightResult; int numRows = argResults.rowCount; boolean[] result = new boolean[numRows]; boolean[] nullability = new boolean[numRows]; for (int rowId = 0; rowId < numRows; rowId++) { boolean leftIsTrue = !left.isNullAt(rowId) && left.getBoolean(rowId); boolean rightIsTrue = !right.isNullAt(rowId) && right.getBoolean(rowId); boolean leftIsFalse = !left.isNullAt(rowId) && !left.getBoolean(rowId); boolean rightIsFalse = !right.isNullAt(rowId) && !right.getBoolean(rowId); if (leftIsFalse || rightIsFalse) { nullability[rowId] = false; result[rowId] = false; } else if (leftIsTrue && rightIsTrue) { nullability[rowId] = false; result[rowId] = true; } else { nullability[rowId] = true; // result[rowId] is undefined when nullability[rowId] = true } } return new DefaultBooleanVector(numRows, Optional.of(nullability), result); } @Override ColumnVector visitOr(Or or) { PredicateChildrenEvalResult argResults = evalBinaryExpressionChildren(or); ColumnVector left = argResults.leftResult; ColumnVector right = argResults.rightResult; int numRows = argResults.rowCount; boolean[] result = new boolean[numRows]; boolean[] nullability = new boolean[numRows]; for (int rowId = 0; rowId < numRows; rowId++) { boolean leftIsTrue = !left.isNullAt(rowId) && left.getBoolean(rowId); boolean rightIsTrue = !right.isNullAt(rowId) && right.getBoolean(rowId); boolean leftIsFalse = !left.isNullAt(rowId) && !left.getBoolean(rowId); boolean rightIsFalse = !right.isNullAt(rowId) && !right.getBoolean(rowId); if (leftIsTrue || rightIsTrue) { nullability[rowId] = false; result[rowId] = true; } else if (leftIsFalse && rightIsFalse) { nullability[rowId] = false; result[rowId] = false; } else { nullability[rowId] = true; // result[rowId] is undefined when nullability[rowId] = true } } return new DefaultBooleanVector(numRows, Optional.of(nullability), result); } @Override ColumnVector visitAlwaysTrue(AlwaysTrue alwaysTrue) { return new DefaultConstantVector(BooleanType.BOOLEAN, input.getSize(), true); } @Override ColumnVector visitAlwaysFalse(AlwaysFalse alwaysFalse) { return new DefaultConstantVector(BooleanType.BOOLEAN, input.getSize(), false); } @Override ColumnVector visitComparator(Predicate predicate) { PredicateChildrenEvalResult argResults = evalBinaryExpressionChildren(predicate); switch (predicate.getName()) { case "=": return comparatorVector( argResults.leftResult, argResults.rightResult, (compareResult) -> (compareResult == 0)); case ">": return comparatorVector( argResults.leftResult, argResults.rightResult, (compareResult) -> (compareResult > 0)); case ">=": return comparatorVector( argResults.leftResult, argResults.rightResult, (compareResult) -> (compareResult >= 0)); case "<": return comparatorVector( argResults.leftResult, argResults.rightResult, (compareResult) -> (compareResult < 0)); case "<=": return comparatorVector( argResults.leftResult, argResults.rightResult, (compareResult) -> (compareResult <= 0)); case "IS NOT DISTINCT FROM": return nullSafeComparatorVector( argResults.leftResult, argResults.rightResult, (compareResult) -> (compareResult == 0)); default: // We should never reach this based on the ExpressionVisitor throw new IllegalStateException( String.format("%s is not a recognized comparator", predicate.getName())); } } @Override ColumnVector visitLiteral(Literal literal) { DataType dataType = literal.getDataType(); if (dataType instanceof BooleanType || dataType instanceof ByteType || dataType instanceof ShortType || dataType instanceof IntegerType || dataType instanceof LongType || dataType instanceof FloatType || dataType instanceof DoubleType || dataType instanceof StringType || dataType instanceof BinaryType || dataType instanceof DecimalType || dataType instanceof DateType || dataType instanceof TimestampType || dataType instanceof TimestampNTZType) { return new DefaultConstantVector(dataType, input.getSize(), literal.getValue()); } throw new UnsupportedOperationException("unsupported expression encountered: " + literal); } @Override ColumnVector visitColumn(Column column) { String[] names = column.getNames(); DataType currentType = input.getSchema(); ColumnVector columnVector = null; for (int level = 0; level < names.length; level++) { assertColumnExists(currentType instanceof StructType, input.getSchema(), column); StructType structSchema = ((StructType) currentType); int ordinal = structSchema.indexOf(names[level]); assertColumnExists(ordinal != -1, input.getSchema(), column); currentType = structSchema.at(ordinal).getDataType(); if (level == 0) { columnVector = input.getColumnVector(ordinal); } else { columnVector = columnVector.getChild(ordinal); } } assertColumnExists(columnVector != null, input.getSchema(), column); return columnVector; } @Override ColumnVector visitCast(ImplicitCastExpression cast) { ColumnVector inputResult = visit(cast.getInput()); return cast.eval(inputResult); } @Override ColumnVector visitPartitionValue(PartitionValueExpression partitionValue) { ColumnVector input = visit(partitionValue.getInput()); return PartitionValueEvaluator.eval(input, partitionValue.getDataType()); } @Override ColumnVector visitElementAt(ScalarExpression elementAt) { ColumnVector map = visit(childAt(elementAt, 0)); ColumnVector lookupKey = visit(childAt(elementAt, 1)); return ElementAtEvaluator.eval(map, lookupKey); } @Override ColumnVector visitNot(Predicate predicate) { ColumnVector childResult = visit(childAt(predicate, 0)); return booleanWrapperVector( childResult, rowId -> !childResult.getBoolean(rowId), rowId -> childResult.isNullAt(rowId)); } @Override ColumnVector visitIsNotNull(Predicate predicate) { ColumnVector childResult = visit(childAt(predicate, 0)); return booleanWrapperVector( childResult, rowId -> !childResult.isNullAt(rowId), rowId -> false); } @Override ColumnVector visitIsNull(Predicate predicate) { ColumnVector childResult = visit(getUnaryChild(predicate)); return booleanWrapperVector( childResult, rowId -> childResult.isNullAt(rowId), rowId -> false); } @Override ColumnVector visitCoalesce(ScalarExpression coalesce) { List childResults = coalesce.getChildren().stream().map(this::visit).collect(Collectors.toList()); return DefaultExpressionUtils.combinationVector( childResults, rowId -> { for (int idx = 0; idx < childResults.size(); idx++) { if (!childResults.get(idx).isNullAt(rowId)) { return idx; } } return 0; // If all are null then any idx suffices }); } @Override ColumnVector visitAdd(ScalarExpression add) { List childResults = add.getChildren().stream().map(this::visit).collect(toList()); // NOTE: The current implementation only supports operands of the same type, and it does not // check for overflows (i.e., values will wrap around when overflowing). return DefaultExpressionUtils.arithmeticVector( childResults.get(0), childResults.get(1), new ArithmeticOperator() { @Override public byte apply(byte a, byte b) { return (byte) (a + b); } @Override public short apply(short a, short b) { return (short) (a + b); } @Override public int apply(int a, int b) { return a + b; } @Override public long apply(long a, long b) { return a + b; } @Override public float apply(float a, float b) { return a + b; } @Override public double apply(double a, double b) { return a + b; } }); } @Override ColumnVector visitTimeAdd(ScalarExpression timeAdd) { ColumnVector timestampColumn = visit(timeAdd.getChildren().get(0)); ColumnVector durationVector = visit(timeAdd.getChildren().get(1)); return new ColumnVector() { @Override public DataType getDataType() { return timestampColumn.getDataType(); } @Override public int getSize() { return timestampColumn.getSize(); } @Override public void close() { timestampColumn.close(); durationVector.close(); } @Override public boolean isNullAt(int rowId) { return timestampColumn.isNullAt(rowId) || durationVector.isNullAt(rowId); } @Override public long getLong(int rowId) { if (isNullAt(rowId)) { return 0; } long durationMicros = durationVector.getLong(rowId) * 1000L; return timestampColumn.getLong(rowId) + durationMicros; } }; } @Override ColumnVector visitSubstring(ScalarExpression subString) { return SubstringEvaluator.eval( subString.getChildren().stream().map(this::visit).collect(toList())); } @Override ColumnVector visitLike(final Predicate like) { List children = like.getChildren(); return LikeExpressionEvaluator.eval( children, children.stream().map(this::visit).collect(toList())); } @Override ColumnVector visitStartsWith(Predicate startsWith) { return StartsWithExpressionEvaluator.eval( startsWith.getChildren().stream().map(this::visit).collect(toList())); } @Override ColumnVector visitIn(In in) { return InExpressionEvaluator.eval( in.getChildren().stream().map(this::visit).collect(toList())); } /** * Utility method to evaluate inputs to the binary input expression. Also validates the * evaluated expression result {@link ColumnVector}s are of the same size. * * @param predicate * @return Triplet of (result vector size, left operand result, left operand result) */ private PredicateChildrenEvalResult evalBinaryExpressionChildren(Predicate predicate) { ColumnVector left = visit(getLeft(predicate)); ColumnVector right = visit(getRight(predicate)); checkArgument( left.getSize() == right.getSize(), "Left and right operand returned different results: left=%d, right=d", left.getSize(), right.getSize()); return new PredicateChildrenEvalResult(left.getSize(), left, right); } } /** Encapsulates children expression result of binary input predicate */ private static class PredicateChildrenEvalResult { public final int rowCount; public final ColumnVector leftResult; public final ColumnVector rightResult; PredicateChildrenEvalResult(int rowCount, ColumnVector leftResult, ColumnVector rightResult) { this.rowCount = rowCount; this.leftResult = leftResult; this.rightResult = rightResult; } } private static void assertColumnExists(boolean condition, StructType schema, Column column) { if (!condition) { throw new IllegalArgumentException( format("%s doesn't exist in input data schema: %s", column, schema)); } } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.expressions; import static io.delta.kernel.defaults.internal.DefaultEngineErrors.unsupportedExpressionException; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.lang.String.format; import io.delta.kernel.data.ArrayValue; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.MapValue; import io.delta.kernel.expressions.Expression; import io.delta.kernel.expressions.Literal; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.types.*; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; import java.util.Comparator; import java.util.List; import java.util.function.Function; import java.util.function.IntPredicate; import java.util.stream.Collectors; /** Utility methods used by the default expression evaluator. */ class DefaultExpressionUtils { static final Comparator BIGDECIMAL_COMPARATOR = Comparator.naturalOrder(); static final Comparator BINARY_COMPARTOR = (leftOp, rightOp) -> { int i = 0; while (i < leftOp.length && i < rightOp.length) { if (leftOp[i] != rightOp[i]) { return Byte.toUnsignedInt(leftOp[i]) - Byte.toUnsignedInt(rightOp[i]); } i++; } return Integer.compare(leftOp.length, rightOp.length); }; static final Comparator STRING_COMPARATOR = (leftOp, rightOp) -> { byte[] leftBytes = leftOp.getBytes(StandardCharsets.UTF_8); byte[] rightBytes = rightOp.getBytes(StandardCharsets.UTF_8); return BINARY_COMPARTOR.compare(leftBytes, rightBytes); }; private DefaultExpressionUtils() {} /** * Utility method that calculates the nullability result from given two vectors. Result is null if * at least one side is a null. */ static boolean[] evalNullability(ColumnVector left, ColumnVector right) { int numRows = left.getSize(); boolean[] nullability = new boolean[numRows]; for (int rowId = 0; rowId < numRows; rowId++) { nullability[rowId] = left.isNullAt(rowId) || right.isNullAt(rowId); } return nullability; } /** * Wraps a child vector as a boolean {@link ColumnVector} with the given value and nullability * accessors. */ static ColumnVector booleanWrapperVector( ColumnVector childVector, Function valueAccessor, Function nullabilityAccessor) { return new ColumnVector() { @Override public DataType getDataType() { return BooleanType.BOOLEAN; } @Override public int getSize() { return childVector.getSize(); } @Override public void close() { childVector.close(); } @Override public boolean isNullAt(int rowId) { return nullabilityAccessor.apply(rowId); } @Override public boolean getBoolean(int rowId) { return valueAccessor.apply(rowId); } }; } /** * Utility method for getting value comparator * * @param left * @param right * @param booleanComparator * @return */ static IntPredicate getComparator( ColumnVector left, ColumnVector right, IntPredicate booleanComparator) { checkArgument( left.getSize() == right.getSize(), "Left and right operand have different vector sizes."); DataType dataType = left.getDataType(); IntPredicate vectorValueComparator; if (dataType instanceof BooleanType) { vectorValueComparator = rowId -> booleanComparator.test( Boolean.compare(left.getBoolean(rowId), right.getBoolean(rowId))); } else if (dataType instanceof ByteType) { vectorValueComparator = rowId -> booleanComparator.test(Byte.compare(left.getByte(rowId), right.getByte(rowId))); } else if (dataType instanceof ShortType) { vectorValueComparator = rowId -> booleanComparator.test(Short.compare(left.getShort(rowId), right.getShort(rowId))); } else if (dataType instanceof IntegerType || dataType instanceof DateType) { vectorValueComparator = rowId -> booleanComparator.test(Integer.compare(left.getInt(rowId), right.getInt(rowId))); } else if (dataType instanceof LongType || dataType instanceof TimestampType || dataType instanceof TimestampNTZType) { vectorValueComparator = rowId -> booleanComparator.test(Long.compare(left.getLong(rowId), right.getLong(rowId))); } else if (dataType instanceof FloatType) { vectorValueComparator = rowId -> booleanComparator.test(Float.compare(left.getFloat(rowId), right.getFloat(rowId))); } else if (dataType instanceof DoubleType) { vectorValueComparator = rowId -> booleanComparator.test(Double.compare(left.getDouble(rowId), right.getDouble(rowId))); } else if (dataType instanceof DecimalType) { vectorValueComparator = rowId -> booleanComparator.test( BIGDECIMAL_COMPARATOR.compare(left.getDecimal(rowId), right.getDecimal(rowId))); } else if (dataType instanceof StringType) { vectorValueComparator = rowId -> booleanComparator.test( STRING_COMPARATOR.compare(left.getString(rowId), right.getString(rowId))); } else if (dataType instanceof BinaryType) { vectorValueComparator = rowId -> booleanComparator.test( BINARY_COMPARTOR.compare(left.getBinary(rowId), right.getBinary(rowId))); } else { throw new UnsupportedOperationException(dataType + " can not be compared."); } return vectorValueComparator; } /** * Utility method to create a column vector that lazily evaluate the comparator ex. (ie. ==, >=, * <=......) for left and right column vector according to the natural ordering of numbers * *

Only primitive data types are supported. */ static ColumnVector comparatorVector( ColumnVector left, ColumnVector right, IntPredicate booleanComparator) { IntPredicate vectorValueComparator = getComparator(left, right, booleanComparator); return new ColumnVector() { @Override public DataType getDataType() { return BooleanType.BOOLEAN; } @Override public void close() { Utils.closeCloseables(left, right); } @Override public int getSize() { return left.getSize(); } @Override public boolean isNullAt(int rowId) { return left.isNullAt(rowId) || right.isNullAt(rowId); } @Override public boolean getBoolean(int rowId) { if (isNullAt(rowId)) { return false; } return vectorValueComparator.test(rowId); } }; } /** * Utility method to create a null safe column vector that lazily evaluate the comparator ex. (ie. * <=>) for left and right column vector according to the natural ordering of numbers * *

Only primitive data types are supported. */ static ColumnVector nullSafeComparatorVector( ColumnVector left, ColumnVector right, IntPredicate booleanComparator) { IntPredicate vectorValueComparator = getComparator(left, right, booleanComparator); return new ColumnVector() { @Override public DataType getDataType() { return BooleanType.BOOLEAN; } @Override public void close() { Utils.closeCloseables(left, right); } @Override public int getSize() { return left.getSize(); } @Override public boolean isNullAt(int rowId) { // Nullsafe comparator can never return null return false; } /** * Null safe comparator follows the truth table in Comparison Operators part of following link * https://spark.apache.org/docs/latest/sql-ref-null-semantics.html * *

If either left or right is null, return false If both left and right is null, return * true else compare the non null value of left and right * * @param rowId * @return */ @Override public boolean getBoolean(int rowId) { if (left.isNullAt(rowId) && right.isNullAt(rowId)) { return true; } else if (left.isNullAt(rowId) || right.isNullAt(rowId)) { return false; } return vectorValueComparator.test(rowId); } }; } static Expression childAt(Expression expression, int index) { return expression.getChildren().get(index); } /** * Combines a list of column vectors into one column vector based on the resolution of idxToReturn * * @param vectors List of ColumnVectors of the same data type with length >= 1 * @param idxToReturn Function that takes in a rowId and returns the index of the column vector to * use as the return value */ static ColumnVector combinationVector( List vectors, Function idxToReturn) { return new ColumnVector() { // Store the last lookup value to avoid multiple looks up for same rowId. // The general pattern is call `isNullAt(rowId)` followed by `getBoolean(rowId)` or // some other value accessor. So the cache of one value is enough. private int lastLookupRowId = -1; private ColumnVector lastLookupVector = null; @Override public DataType getDataType() { return vectors.get(0).getDataType(); } @Override public int getSize() { return vectors.get(0).getSize(); } @Override public void close() { Utils.closeCloseables(vectors.toArray(new ColumnVector[0])); } @Override public boolean isNullAt(int rowId) { return getVector(rowId).isNullAt(rowId); } @Override public boolean getBoolean(int rowId) { return getVector(rowId).getBoolean(rowId); } @Override public byte getByte(int rowId) { return getVector(rowId).getByte(rowId); } @Override public short getShort(int rowId) { return getVector(rowId).getShort(rowId); } @Override public int getInt(int rowId) { return getVector(rowId).getInt(rowId); } @Override public long getLong(int rowId) { return getVector(rowId).getLong(rowId); } @Override public float getFloat(int rowId) { return getVector(rowId).getFloat(rowId); } @Override public double getDouble(int rowId) { return getVector(rowId).getDouble(rowId); } @Override public byte[] getBinary(int rowId) { return getVector(rowId).getBinary(rowId); } @Override public String getString(int rowId) { return getVector(rowId).getString(rowId); } @Override public BigDecimal getDecimal(int rowId) { return getVector(rowId).getDecimal(rowId); } @Override public MapValue getMap(int rowId) { return getVector(rowId).getMap(rowId); } @Override public ArrayValue getArray(int rowId) { return getVector(rowId).getArray(rowId); } @Override public ColumnVector getChild(int ordinal) { return combinationVector( vectors.stream().map(v -> v.getChild(ordinal)).collect(Collectors.toList()), idxToReturn); } private ColumnVector getVector(int rowId) { if (rowId == lastLookupRowId) { return lastLookupVector; } lastLookupRowId = rowId; lastLookupVector = vectors.get(idxToReturn.apply(rowId)); return lastLookupVector; } }; } /** Represents an arithmetic operator that can be applied to two numeric values. */ public interface ArithmeticOperator { byte apply(byte a, byte b); short apply(short a, short b); int apply(int a, int b); long apply(long a, long b); float apply(float a, float b); double apply(double a, double b); } /** * Creates a column vector that lazily evaluates an arithmetic operation between two column * vectors. * *

Only numeric data types are supported. * * @param left the left operand column vector * @param right the right operand column vector * @param operator the arithmetic operator to apply * @return a new column vector representing the result of the arithmetic operation */ static ColumnVector arithmeticVector( ColumnVector left, ColumnVector right, ArithmeticOperator operator) { checkArgument( left.getSize() == right.getSize(), "Left and right operand have different vector sizes."); checkArgument( left.getDataType().equals(right.getDataType()), "Left and right operand have different data types."); return new ColumnVector() { @Override public DataType getDataType() { return left.getDataType(); } @Override public int getSize() { return left.getSize(); } @Override public void close() { Utils.closeCloseables(left, right); } @Override public boolean isNullAt(int rowId) { return left.isNullAt(rowId) || right.isNullAt(rowId); } @Override public byte getByte(int rowId) { return operator.apply(left.getByte(rowId), right.getByte(rowId)); } @Override public short getShort(int rowId) { return operator.apply(left.getShort(rowId), right.getShort(rowId)); } @Override public int getInt(int rowId) { return operator.apply(left.getInt(rowId), right.getInt(rowId)); } @Override public long getLong(int rowId) { return operator.apply(left.getLong(rowId), right.getLong(rowId)); } @Override public float getFloat(int rowId) { return operator.apply(left.getFloat(rowId), right.getFloat(rowId)); } @Override public double getDouble(int rowId) { return operator.apply(left.getDouble(rowId), right.getDouble(rowId)); } }; } /** * Checks if the specific expression is an integer literal, throws {@link * UnsupportedOperationException} if not. * * @param expr, expression to be checked. * @param context string describing the context, used for constructing error message. * @param baseExpression expression whose evaluation triggers this check. Used for constructing * error message. */ static void checkIntegerLiteral(Expression expr, String context, Expression baseExpression) { if (!(expr instanceof Literal) || !IntegerType.INTEGER.equals(((Literal) expr).getDataType())) { throw unsupportedExpressionException( baseExpression, String.format("%s, expects an integral numeric", context)); } } /** * Checks the argument count of an expression. throws {@code unsupportedExpressionException} if * argument count mismatched. */ static void checkArgsCount(Expression expr, int expectedCount, String exprName, String context) { if (expr.getChildren().size() != expectedCount) { throw unsupportedExpressionException( expr, String.format("Invalid number of inputs of %s expression, %s", exprName, context)); } } static void checkIsStringType(DataType dataType, Expression parentExpr, String errorMessage) { if (dataType instanceof StringType) { return; } throw unsupportedExpressionException(parentExpr, errorMessage); } static void checkIsLiteral(Expression expr, Expression parentExpr, String errorMessage) { if (!(expr instanceof Literal)) { throw unsupportedExpressionException(parentExpr, errorMessage); } } /** * Checks if the collation is `UTF8_BINARY`, since this is the only collation the default engine * can evaluate. */ static void checkIsUTF8BinaryCollation( Predicate predicate, CollationIdentifier collationIdentifier) { if (!collationIdentifier.isSparkUTF8BinaryCollation()) { String msg = format( "Unsupported collation: \"%s\". Default Engine supports just" + " \"%s\" collation.", collationIdentifier, CollationIdentifier.SPARK_UTF8_BINARY); throw unsupportedExpressionException(predicate, msg); } } /** * Checks if the given expression is a null literal. * * @param expression The expression to check * @return true if the expression is a Literal with null value, false otherwise */ static boolean isNullLiteral(Expression expression) { return expression instanceof Literal && ((Literal) expression).getValue() == null; } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultPredicateEvaluator.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.expressions; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.defaults.internal.data.vector.DefaultConstantVector; import io.delta.kernel.expressions.*; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.types.BooleanType; import io.delta.kernel.types.StructField; import io.delta.kernel.types.StructType; import java.util.Optional; /** * Default implementation of {@link PredicateEvaluator}. It makes use of the {@link * DefaultExpressionEvaluator} with some modification of the given predicate. Refer to {@link * #DefaultPredicateEvaluator(StructType, Predicate)} and {@link #eval(ColumnarBatch, Optional)} for * details. */ public class DefaultPredicateEvaluator implements PredicateEvaluator { private static final String EXISTING_SEL_VECTOR_COL_NAME = "____existing_selection_vector_value____"; private static final StructField EXISTING_SEL_VECTOR_FIELD = new StructField(EXISTING_SEL_VECTOR_COL_NAME, BooleanType.BOOLEAN, false); private final ExpressionEvaluator expressionEvaluator; public DefaultPredicateEvaluator(StructType inputSchema, Predicate predicate) { // Create a predicate that takes into account of the selection value in existing selection // vector in addition to the given predicate. This is needed to make a row remain // unselected in the final vector when it is unselected in existing selection vector. Predicate rewrittenPredicate = new And( new Predicate("=", new Column(EXISTING_SEL_VECTOR_COL_NAME), Literal.ofBoolean(true)), predicate); StructType rewrittenInputSchema = inputSchema.add(EXISTING_SEL_VECTOR_FIELD); this.expressionEvaluator = new DefaultExpressionEvaluator( rewrittenInputSchema, rewrittenPredicate, BooleanType.BOOLEAN); } @Override public ColumnVector eval( ColumnarBatch inputData, Optional existingSelectionVector) { try { ColumnVector newVector = existingSelectionVector.orElse( new DefaultConstantVector(BooleanType.BOOLEAN, inputData.getSize(), true)); ColumnarBatch withExistingSelVector = inputData.withNewColumn( inputData.getSchema().length(), EXISTING_SEL_VECTOR_FIELD, newVector); return expressionEvaluator.eval(withExistingSelVector); } finally { // release the existing selection vector. Utils.closeCloseables(existingSelectionVector.orElse(null)); } } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ElementAtEvaluator.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.expressions; import static io.delta.kernel.defaults.internal.DefaultEngineErrors.unsupportedExpressionException; import static io.delta.kernel.defaults.internal.expressions.ImplicitCastExpression.canCastTo; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.lang.String.format; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.MapValue; import io.delta.kernel.expressions.Expression; import io.delta.kernel.expressions.ScalarExpression; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.types.DataType; import io.delta.kernel.types.MapType; import io.delta.kernel.types.StringType; import java.util.Arrays; /** Utility methods to evaluate {@code element_at} expression. */ class ElementAtEvaluator { private ElementAtEvaluator() {} /** * Validate and transform the {@code element_at} expression with given validated and transformed * inputs. */ static ScalarExpression validateAndTransform( ScalarExpression elementAt, Expression mapInput, DataType mapInputType, Expression lookupKey, DataType lookupKeyType) { MapType asMapType = validateSupportedMapType(elementAt, mapInputType); DataType keyTypeFromMapInput = asMapType.getKeyType(); if (!keyTypeFromMapInput.equivalent(lookupKeyType)) { if (canCastTo(lookupKeyType, keyTypeFromMapInput)) { lookupKey = new ImplicitCastExpression(lookupKey, keyTypeFromMapInput); } else { String reason = format( "lookup key type (%s) is different from the map key type (%s)", lookupKeyType, asMapType.getKeyType()); throw unsupportedExpressionException(elementAt, reason); } } return new ScalarExpression(elementAt.getName(), Arrays.asList(mapInput, lookupKey)); } /** * Utility method to evaluate the {@code element_at} on given map and key vectors. * * @param map {@link ColumnVector} of {@code map(string, string)} type. * @param lookupKey {@link ColumnVector} of {@code string} type. * @return result {@link ColumnVector} containing the lookup values. */ static ColumnVector eval(ColumnVector map, ColumnVector lookupKey) { return new ColumnVector() { // Store the last lookup value to avoid multiple looks up for same row id. // The general pattern is call `isNullAt(rowId)` followed by `getString`. // So the cache of one value is enough. private int lastLookupRowId = -1; private String lastLookupValue = null; @Override public DataType getDataType() { return ((MapType) map.getDataType()).getValueType(); } @Override public int getSize() { return map.getSize(); } @Override public void close() { Utils.closeCloseables(map, lookupKey); } @Override public boolean isNullAt(int rowId) { if (rowId == lastLookupRowId) { return lastLookupValue == null; } return map.isNullAt(rowId) || lookupValue(rowId) == null; } @Override public String getString(int rowId) { lookupValue(rowId); return lastLookupValue == null ? null : lastLookupValue; } private Object lookupValue(int rowId) { if (rowId == lastLookupRowId) { return lastLookupValue; } lastLookupRowId = rowId; String keyValue = lookupKey.getString(rowId); lastLookupValue = findValueForKey(map.getMap(rowId), keyValue); return lastLookupValue; } /** * Given a {@link MapValue} and string {@code key} find the corresponding value. Returns null * if the key is not in the map. * * @param mapValue String->String map to search * @param key the key to look up the value for; may be null */ private String findValueForKey(MapValue mapValue, String key) { ColumnVector keyVector = mapValue.getKeys(); for (int i = 0; i < mapValue.getSize(); i++) { if ((keyVector.isNullAt(i) && key == null) || (!keyVector.isNullAt(i) && keyVector.getString(i).equals(key))) { return mapValue.getValues().isNullAt(i) ? null : mapValue.getValues().getString(i); } } // If the key is not in the map return null return null; } }; } private static MapType validateSupportedMapType(Expression elementAt, DataType mapInputType) { checkArgument( mapInputType instanceof MapType, "expected a map type input as first argument: %s", elementAt); MapType asMapType = (MapType) mapInputType; // For now we only need to support lookup in columns of type `map(string -> string)`. // Additional type support may be added later if (asMapType.getKeyType().equivalent(StringType.STRING) && asMapType.getValueType().equivalent(StringType.STRING)) { return asMapType; } throw new UnsupportedOperationException( format("%s: Supported only on type map(string, string) input data", elementAt)); } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ExpressionVisitor.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.expressions; import static io.delta.kernel.expressions.AlwaysFalse.ALWAYS_FALSE; import static io.delta.kernel.expressions.AlwaysTrue.ALWAYS_TRUE; import static io.delta.kernel.internal.util.ExpressionUtils.createPredicate; import static java.util.stream.Collectors.joining; import io.delta.kernel.expressions.*; import io.delta.kernel.types.CollationIdentifier; import java.util.List; import java.util.Locale; import java.util.Optional; /** * Interface to allow visiting an expression tree and implementing handling for each specific * expression type. * * @param Return type of result of visit expression methods. */ abstract class ExpressionVisitor { abstract R visitAnd(And and); abstract R visitOr(Or or); abstract R visitAlwaysTrue(AlwaysTrue alwaysTrue); abstract R visitAlwaysFalse(AlwaysFalse alwaysFalse); abstract R visitComparator(Predicate predicate); abstract R visitLiteral(Literal literal); abstract R visitColumn(Column column); abstract R visitCast(ImplicitCastExpression cast); abstract R visitPartitionValue(PartitionValueExpression partitionValue); abstract R visitElementAt(ScalarExpression elementAt); abstract R visitNot(Predicate predicate); abstract R visitIsNotNull(Predicate predicate); abstract R visitIsNull(Predicate predicate); abstract R visitCoalesce(ScalarExpression ifNull); abstract R visitTimeAdd(ScalarExpression timeAdd); abstract R visitSubstring(ScalarExpression subString); abstract R visitAdd(ScalarExpression add); abstract R visitLike(Predicate predicate); abstract R visitStartsWith(Predicate predicate); abstract R visitIn(In in); final R visit(Expression expression) { if (expression instanceof PartitionValueExpression) { return visitPartitionValue((PartitionValueExpression) expression); } else if (expression instanceof ScalarExpression) { return visitScalarExpression((ScalarExpression) expression); } else if (expression instanceof Literal) { return visitLiteral((Literal) expression); } else if (expression instanceof Column) { return visitColumn((Column) expression); } else if (expression instanceof ImplicitCastExpression) { return visitCast((ImplicitCastExpression) expression); } throw new UnsupportedOperationException( String.format("Expression %s is not supported.", expression)); } private R visitScalarExpression(ScalarExpression expression) { List children = expression.getChildren(); String name = expression.getName().toUpperCase(Locale.ENGLISH); Optional collationIdentifier = Optional.empty(); if (expression instanceof Predicate) { collationIdentifier = ((Predicate) expression).getCollationIdentifier(); } switch (name) { case "ALWAYS_TRUE": return visitAlwaysTrue(ALWAYS_TRUE); case "ALWAYS_FALSE": return visitAlwaysFalse(ALWAYS_FALSE); case "AND": return visitAnd(new And(elemAsPredicate(children, 0), elemAsPredicate(children, 1))); case "OR": return visitOr(new Or(elemAsPredicate(children, 0), elemAsPredicate(children, 1))); case "=": case "<": case "<=": case ">": case ">=": case "IS NOT DISTINCT FROM": return visitComparator(createPredicate(name, children, collationIdentifier)); case "ELEMENT_AT": return visitElementAt(expression); case "NOT": return visitNot(createPredicate(name, children, collationIdentifier)); case "IS_NOT_NULL": return visitIsNotNull(createPredicate(name, children, collationIdentifier)); case "IS_NULL": return visitIsNull(createPredicate(name, children, collationIdentifier)); case "COALESCE": return visitCoalesce(expression); case "ADD": return visitAdd(expression); case "TIMEADD": return visitTimeAdd(expression); case "SUBSTRING": return visitSubstring(expression); case "LIKE": return visitLike(createPredicate(name, children, collationIdentifier)); case "STARTS_WITH": return visitStartsWith(createPredicate(name, children, collationIdentifier)); case "IN": if (collationIdentifier.isPresent()) { return visitIn( new In( children.get(0), children.subList(1, children.size()), collationIdentifier.get())); } else { return visitIn(new In(children.get(0), children.subList(1, children.size()))); } default: throw new UnsupportedOperationException( String.format("Scalar expression `%s` is not supported.", name)); } } private static Predicate elemAsPredicate(List expressions, int index) { if (expressions.size() <= index) { throw new RuntimeException( String.format( "Trying to access invalid entry (%d) in list %s", index, expressions.stream().map(Object::toString).collect(joining(",")))); } Expression elemExpression = expressions.get(index); if (!(elemExpression instanceof Predicate)) { throw new RuntimeException("Expected a predicate, but got " + elemExpression); } return (Predicate) expressions.get(index); } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ImplicitCastExpression.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.expressions; import static java.lang.String.format; import static java.util.Collections.unmodifiableMap; import static java.util.Objects.requireNonNull; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.defaults.engine.DefaultExpressionHandler; import io.delta.kernel.expressions.Expression; import io.delta.kernel.types.DataType; import java.util.*; /** * An implicit cast expression to convert the input type to another given type. Here is the valid * list of casts * *

* *

    *
  • {@code byte} to {@code short, int, long, float, double} *
  • {@code short} to {@code int, long, float, double} *
  • {@code int} to {@code long, float, double} *
  • {@code long} to {@code float, double} *
  • {@code float} to {@code double} *
* *

The above list is not exhaustive. Based on the need, we can add more casts. * *

In {@link DefaultExpressionHandler} this is used when the operands of an expression are not of * the same type, but the evaluator expects same type inputs. There could be more use cases, but for * now this is the only use case. */ final class ImplicitCastExpression implements Expression { private final Expression input; private final DataType outputType; /** * Create a cast around the given input expression to specified output data type. It is the * responsibility of the caller to validate the input expression can be cast to the new type using * {@link #canCastTo(DataType, DataType)} */ ImplicitCastExpression(Expression input, DataType outputType) { this.input = requireNonNull(input, "input is null"); this.outputType = requireNonNull(outputType, "outputType is null"); } public Expression getInput() { return input; } public DataType getOutputType() { return outputType; } @Override public List getChildren() { return Collections.singletonList(input); } /** * Evaluate the given column expression on the input {@link ColumnVector}. * * @param input {@link ColumnVector} data of the input to the cast expression. * @return {@link ColumnVector} result applying target type casting on every element in the input * {@link ColumnVector}. */ ColumnVector eval(ColumnVector input) { String fromTypeStr = input.getDataType().toString(); switch (fromTypeStr) { case "byte": return new ByteUpConverter(outputType, input); case "short": return new ShortUpConverter(outputType, input); case "integer": return new IntUpConverter(outputType, input); case "long": return new LongUpConverter(outputType, input); case "float": return new FloatUpConverter(outputType, input); default: throw new UnsupportedOperationException( format("Cast from %s is not supported", fromTypeStr)); } } /** Map containing for each type what are the target cast types can be. */ private static final Map> UP_CASTABLE_TYPE_TABLE = unmodifiableMap( new HashMap>() { { this.put("byte", Arrays.asList("short", "integer", "long", "float", "double")); this.put("short", Arrays.asList("integer", "long", "float", "double")); this.put("integer", Arrays.asList("long", "float", "double")); this.put("long", Arrays.asList("float", "double")); this.put("float", Arrays.asList("double")); } }); /** * Utility method which returns whether the given {@code from} type can be cast to {@code to} * type. */ static boolean canCastTo(DataType from, DataType to) { // TODO: The type name should be a first class method on `DataType` instead of getting it // using the `toString`. String fromStr = from.toString(); String toStr = to.toString(); return UP_CASTABLE_TYPE_TABLE.containsKey(fromStr) && UP_CASTABLE_TYPE_TABLE.get(fromStr).contains(toStr); } /** Base class for up casting {@link ColumnVector} data. */ private abstract static class UpConverter implements ColumnVector { protected final DataType targetType; protected final ColumnVector inputVector; UpConverter(DataType targetType, ColumnVector inputVector) { this.targetType = targetType; this.inputVector = inputVector; } @Override public DataType getDataType() { return targetType; } @Override public boolean isNullAt(int rowId) { return inputVector.isNullAt(rowId); } @Override public int getSize() { return inputVector.getSize(); } @Override public void close() { inputVector.close(); } } private static class ByteUpConverter extends UpConverter { ByteUpConverter(DataType targetType, ColumnVector inputVector) { super(targetType, inputVector); } @Override public short getShort(int rowId) { return inputVector.getByte(rowId); } @Override public int getInt(int rowId) { return inputVector.getByte(rowId); } @Override public long getLong(int rowId) { return inputVector.getByte(rowId); } @Override public float getFloat(int rowId) { return inputVector.getByte(rowId); } @Override public double getDouble(int rowId) { return inputVector.getByte(rowId); } } private static class ShortUpConverter extends UpConverter { ShortUpConverter(DataType targetType, ColumnVector inputVector) { super(targetType, inputVector); } @Override public int getInt(int rowId) { return inputVector.getShort(rowId); } @Override public long getLong(int rowId) { return inputVector.getShort(rowId); } @Override public float getFloat(int rowId) { return inputVector.getShort(rowId); } @Override public double getDouble(int rowId) { return inputVector.getShort(rowId); } } private static class IntUpConverter extends UpConverter { IntUpConverter(DataType targetType, ColumnVector inputVector) { super(targetType, inputVector); } @Override public long getLong(int rowId) { return inputVector.getInt(rowId); } @Override public float getFloat(int rowId) { return inputVector.getInt(rowId); } @Override public double getDouble(int rowId) { return inputVector.getInt(rowId); } } private static class LongUpConverter extends UpConverter { LongUpConverter(DataType targetType, ColumnVector inputVector) { super(targetType, inputVector); } @Override public float getFloat(int rowId) { return inputVector.getLong(rowId); } @Override public double getDouble(int rowId) { return inputVector.getLong(rowId); } } private static class FloatUpConverter extends UpConverter { FloatUpConverter(DataType targetType, ColumnVector inputVector) { super(targetType, inputVector); } @Override public double getDouble(int rowId) { return inputVector.getFloat(rowId); } } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/InExpressionEvaluator.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.expressions; import static io.delta.kernel.defaults.internal.DefaultEngineErrors.unsupportedExpressionException; import static io.delta.kernel.defaults.internal.expressions.DefaultExpressionUtils.*; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.expressions.Expression; import io.delta.kernel.expressions.In; import io.delta.kernel.expressions.Literal; import io.delta.kernel.internal.util.Preconditions; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.internal.util.VectorUtils; import io.delta.kernel.types.*; import io.delta.kernel.types.CollationIdentifier; import java.math.BigDecimal; import java.util.*; import java.util.function.BiFunction; /** Utility methods to evaluate {@code IN} expression. */ public class InExpressionEvaluator { private static final Map, BiFunction> COMPARATORS = createComparatorMap(); private static Map, BiFunction> createComparatorMap() { Map, BiFunction> map = new HashMap<>(); map.put(BooleanType.class, (v1, v2) -> Boolean.compare((Boolean) v1, (Boolean) v2)); map.put( ByteType.class, (v1, v2) -> Byte.compare(((Number) v1).byteValue(), ((Number) v2).byteValue())); map.put( ShortType.class, (v1, v2) -> Short.compare(((Number) v1).shortValue(), ((Number) v2).shortValue())); map.put( IntegerType.class, (v1, v2) -> Integer.compare(((Number) v1).intValue(), ((Number) v2).intValue())); map.put( DateType.class, (v1, v2) -> Integer.compare(((Number) v1).intValue(), ((Number) v2).intValue())); map.put( LongType.class, (v1, v2) -> Long.compare(((Number) v1).longValue(), ((Number) v2).longValue())); map.put( TimestampType.class, (v1, v2) -> Long.compare(((Number) v1).longValue(), ((Number) v2).longValue())); map.put( TimestampNTZType.class, (v1, v2) -> Long.compare(((Number) v1).longValue(), ((Number) v2).longValue())); map.put( FloatType.class, (v1, v2) -> Float.compare(((Number) v1).floatValue(), ((Number) v2).floatValue())); map.put( DoubleType.class, (v1, v2) -> Double.compare(((Number) v1).doubleValue(), ((Number) v2).doubleValue())); map.put( DecimalType.class, (v1, v2) -> BIGDECIMAL_COMPARATOR.compare((BigDecimal) v1, (BigDecimal) v2)); map.put(StringType.class, (v1, v2) -> STRING_COMPARATOR.compare((String) v1, (String) v2)); map.put(BinaryType.class, (v1, v2) -> BINARY_COMPARTOR.compare((byte[]) v1, (byte[]) v2)); return Collections.unmodifiableMap(map); } /** Validates and transforms the {@code IN} expression. */ static In validateAndTransform( In in, Expression valueExpression, DataType valueDataType, List inListExpressions, List inListDataTypes) { // TODO: [delta-io/delta#5227] Try to reuse Implicit cast and simplify comparison logic validateArgumentCount(in, inListExpressions); validateInListElementsAreLiterals(in, inListExpressions); validateTypeCompatibility(in, valueDataType, inListDataTypes); validateCollation(in, valueDataType, inListExpressions, inListDataTypes); if (in.getCollationIdentifier().isPresent()) { return new In(valueExpression, inListExpressions, in.getCollationIdentifier().get()); } else { return new In(valueExpression, inListExpressions); } } /** Evaluates the IN expression on the given column vectors. */ static ColumnVector eval(List childrenVectors) { return new InColumnVector(childrenVectors); } //////////////////// // Private Helper // //////////////////// private static void validateArgumentCount(In in, List inListExpressions) { Objects.requireNonNull(inListExpressions); if (inListExpressions.isEmpty()) { throw unsupportedExpressionException( in, "IN expression requires at least 1 element in the IN list. " + "Example usage: column IN (value1, value2, ...)"); } } private static void validateInListElementsAreLiterals(In in, List inListExpressions) { for (int i = 0; i < inListExpressions.size(); i++) { Expression child = inListExpressions.get(i); if (!(child instanceof Literal)) { throw unsupportedExpressionException( in, String.format( "IN expression requires all list elements to be literals. " + "Non-literal expression found at position %d: %s. " + "Only constant values are currently supported in IN lists.", i + 1, child.getClass().getSimpleName())); } } } private static void validateTypeCompatibility( In in, DataType valueDataType, List inListDataTypes) { // Check for nested types which are not supported if (valueDataType.isNested()) { throw unsupportedExpressionException( in, String.format("IN expression does not support nested types.", valueDataType)); } for (int i = 0; i < inListDataTypes.size(); i++) { DataType listElementType = inListDataTypes.get(i); if (!valueDataType.equivalent(listElementType)) { throw unsupportedExpressionException( in, String.format( "IN expression requires all list elements to match the value type. " + "Value type: %s, but found incompatible element type at position %d: %s. " + "Consider casting the incompatible element to the value type.", valueDataType, i + 1, listElementType)); } } } /** Validates that collation is only used with string types and the collation is UTF8Binary. */ private static void validateCollation( In in, DataType valueDataType, List inListExpressions, List inListDataTypes) { in.getCollationIdentifier() .ifPresent( collationIdentifier -> { checkIsUTF8BinaryCollation(in, collationIdentifier); validateStringTypesForCollation( in, valueDataType, inListExpressions, inListDataTypes); }); } private static void validateStringTypesForCollation( In in, DataType valueDataType, List inListExpressions, List inListDataTypes) { checkIsStringType( valueDataType, in, "'IN' with collation expects STRING type for the value expression"); for (int i = 0; i < inListDataTypes.size(); i++) { if (!isNullLiteral(inListExpressions.get(i))) { checkIsStringType( inListDataTypes.get(i), in, "'IN' with collation expects STRING type for all list elements"); } } } private static void checkIsUTF8BinaryCollation(In in, CollationIdentifier collationIdentifier) { if (!"SPARK.UTF8_BINARY".equals(collationIdentifier.toString())) { throw unsupportedExpressionException( in, String.format( "Unsupported collation: \"%s\". " + "Default Engine supports just \"SPARK.UTF8_BINARY\" collation.", collationIdentifier)); } } private static void checkIsStringType(DataType dataType, In in, String message) { if (!(dataType instanceof StringType)) { throw unsupportedExpressionException(in, message); } } private static boolean compareValues(Object value1, Object value2, DataType valueType) { Preconditions.checkArgument(value1 != null || value2 != null); if (value1 == null || value2 == null) { return false; } return getComparator(valueType).apply(value1, value2) == 0; } private static BiFunction getComparator(DataType dataType) { BiFunction comparator = COMPARATORS.get(dataType.getClass()); if (comparator == null) { throw new UnsupportedOperationException( "No comparator available for data type: " + dataType.getClass().getSimpleName()); } return comparator; } /** Column vector implementation for IN expression evaluation. */ private static class InColumnVector implements ColumnVector { private final ColumnVector valueVector; private final List inListVectors; InColumnVector(List childrenVectors) { this.valueVector = childrenVectors.get(0); this.inListVectors = childrenVectors.subList(1, childrenVectors.size()); // Validate type compatibility once during construction rather than for each row DataType valueType = valueVector.getDataType(); for (ColumnVector inListVector : inListVectors) { Preconditions.checkArgument( valueType.equivalent(inListVector.getDataType()), String.format( "Type mismatch in IN expression: value type %s is not equivalent to " + "list element type %s", valueType, inListVector.getDataType())); } } @Override public DataType getDataType() { return BooleanType.BOOLEAN; } @Override public int getSize() { return valueVector.getSize(); } @Override public void close() { Utils.closeCloseables(valueVector); inListVectors.forEach(Utils::closeCloseables); } @Override public boolean getBoolean(int rowId) { Optional result = evaluateInLogic(rowId); Preconditions.checkArgument( result.isPresent(), "This method is expected to be called only when isNullAt is false"); return result.get(); } @Override public boolean isNullAt(int rowId) { return !evaluateInLogic(rowId).isPresent(); } private Optional evaluateInLogic(int rowId) { if (valueVector.isNullAt(rowId)) { return Optional.empty(); } Object valueToFind = VectorUtils.getValueAsObject(valueVector, valueVector.getDataType(), rowId); // Track if we encounter any null values in the IN list // SQL semantics: // - If value matches any element, return true (e.g., 5 IN {0, 4, null, 5} = true) // - If value is null OR (no matches found AND any null in list), return null // (e.g., null IN {1, 2} = null, 3 IN {1, null, 2} = null) // - If value is not null AND no matches found AND no nulls in list, return false // (e.g., 3 IN {1, 2} = false) boolean foundNull = false; for (ColumnVector inListVector : inListVectors) { if (inListVector.isNullAt(rowId)) { foundNull = true; } else { Object inListValue = VectorUtils.getValueAsObject(inListVector, inListVector.getDataType(), rowId); if (compareValues(valueToFind, inListValue, valueVector.getDataType())) { return Optional.of(true); } } } return foundNull ? Optional.empty() : Optional.of(false); } } private InExpressionEvaluator() {} } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/LikeExpressionEvaluator.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.expressions; import static io.delta.kernel.defaults.internal.DefaultEngineErrors.invalidEscapeSequence; import static io.delta.kernel.defaults.internal.DefaultEngineErrors.unsupportedExpressionException; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.expressions.Expression; import io.delta.kernel.expressions.Literal; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.types.BooleanType; import io.delta.kernel.types.DataType; import io.delta.kernel.types.StringType; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.regex.Pattern; /** Utility methods to evaluate {@code like} expression. */ public class LikeExpressionEvaluator { private LikeExpressionEvaluator() {} static Predicate validateAndTransform( Predicate like, List childrenExpressions, List childrenOutputTypes) { int size = childrenExpressions.size(); if (size < 2 || size > 3) { throw unsupportedExpressionException( like, "Invalid number of inputs to LIKE expression. " + "Example usage: LIKE(column, 'test%'), LIKE(column, 'test\\[%', '\\')"); } Expression left = childrenExpressions.get(0); DataType leftOutputType = childrenOutputTypes.get(0); Expression right = childrenExpressions.get(1); DataType rightOutputType = childrenOutputTypes.get(1); Expression escapeCharExpr = size == 3 ? childrenExpressions.get(2) : null; DataType escapeCharOutputType = size == 3 ? childrenOutputTypes.get(2) : null; if (!(StringType.STRING.equivalent(leftOutputType) && StringType.STRING.equivalent(rightOutputType))) { throw unsupportedExpressionException( like, "LIKE is only supported for string type expressions"); } if (escapeCharExpr != null && (!(escapeCharExpr instanceof Literal && StringType.STRING.equivalent(escapeCharOutputType)))) { throw unsupportedExpressionException( like, "LIKE expects escape token expression to be a literal of String type"); } Literal literal = (Literal) escapeCharExpr; if (literal != null && literal.getValue().toString().length() != 1) { throw unsupportedExpressionException( like, "LIKE expects escape token to be a single character"); } List children = new ArrayList<>(Arrays.asList(left, right)); if (Objects.nonNull(escapeCharExpr)) { children.add(escapeCharExpr); } return new Predicate(like.getName(), children); } static ColumnVector eval( List childrenExpressions, List childrenVectors) { final char DEFAULT_ESCAPE_CHAR = '\\'; final boolean isPatternLiteralType = childrenExpressions.get(1) instanceof Literal; return new ColumnVector() { final ColumnVector escapeCharVector = childrenVectors.size() == 3 ? childrenVectors.get(2) : null; final ColumnVector left = childrenVectors.get(0); final ColumnVector right = childrenVectors.get(1); Character escapeChar = null; String regexCache = null; public void initEscapeCharIfRequired() { if (escapeChar == null) { escapeChar = escapeCharVector != null && !escapeCharVector.getString(0).isEmpty() ? escapeCharVector.getString(0).charAt(0) : DEFAULT_ESCAPE_CHAR; } } @Override public DataType getDataType() { return BooleanType.BOOLEAN; } @Override public int getSize() { return left.getSize(); } @Override public void close() { Utils.closeCloseables(left, right); } @Override public boolean getBoolean(int rowId) { initEscapeCharIfRequired(); return isLike(left.getString(rowId), right.getString(rowId), escapeChar); } @Override public boolean isNullAt(int rowId) { return left.isNullAt(rowId) || right.isNullAt(rowId); } public boolean isLike(String input, String pattern, char escape) { if (!Objects.isNull(input) && !Objects.isNull(pattern)) { String regex = getRegexFromCacheOrEval(pattern, escape); return input.matches(regex); } return false; } public String getRegexFromCacheOrEval(String pattern, char escape) { if (regexCache != null) { return regexCache; } String regex = escapeLikeRegex(pattern, escape); if (isPatternLiteralType) { // set cache only for literals to avoid re-computation regexCache = regex; } return regex; } }; } /** * utility method to convert a predicate pattern to a java regex * * @param pattern the pattern used in the expression * @param escape escape character to use * @return java regex */ private static String escapeLikeRegex(String pattern, char escape) { final int len = pattern.length(); final StringBuilder javaPattern = new StringBuilder(len + len); for (int i = 0; i < len; i++) { char c = pattern.charAt(i); if (c == escape) { if (i == (pattern.length() - 1)) { throw invalidEscapeSequence(pattern, i); } char nextChar = pattern.charAt(i + 1); if ((nextChar == '_') || (nextChar == '%') || (nextChar == escape)) { javaPattern.append(Pattern.quote(Character.toString(nextChar))); i++; } else { throw invalidEscapeSequence(pattern, i); } } else if (c == '_') { javaPattern.append('.'); } else if (c == '%') { javaPattern.append(".*"); } else { javaPattern.append(Pattern.quote(Character.toString(c))); } } return "(?s)" + javaPattern; } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/PartitionValueEvaluator.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.expressions; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.internal.util.InternalUtils; import io.delta.kernel.internal.util.PartitionUtils; import io.delta.kernel.types.*; import java.math.BigDecimal; import java.sql.Date; import java.sql.Timestamp; /** Utility methods to evaluate {@code partition_value} expression */ class PartitionValueEvaluator { /** * Evaluate the {@code partition_value} expression for given input column vector and generate a * column vector with decoded values according to the given partition type. */ static ColumnVector eval(ColumnVector input, DataType partitionType) { return new ColumnVector() { @Override public DataType getDataType() { return partitionType; } @Override public int getSize() { return input.getSize(); } @Override public void close() { input.close(); } @Override public boolean isNullAt(int rowId) { return input.isNullAt(rowId); } @Override public boolean getBoolean(int rowId) { return Boolean.parseBoolean(input.getString(rowId)); } @Override public byte getByte(int rowId) { return Byte.parseByte(input.getString(rowId)); } @Override public short getShort(int rowId) { return Short.parseShort(input.getString(rowId)); } @Override public int getInt(int rowId) { if (partitionType.equivalent(IntegerType.INTEGER)) { return Integer.parseInt(input.getString(rowId)); } else if (partitionType.equivalent(DateType.DATE)) { return InternalUtils.daysSinceEpoch(Date.valueOf(input.getString(rowId))); } throw new UnsupportedOperationException("Invalid value request for data type"); } @Override public long getLong(int rowId) { if (partitionType.equivalent(LongType.LONG)) { return Long.parseLong(input.getString(rowId)); } else if (partitionType.equivalent(TimestampType.TIMESTAMP)) { // For TIMESTAMP type the format could be standard format or ISO8601 return PartitionUtils.tryParseTimestamp(input.getString(rowId)); } else if (partitionType.equivalent(TimestampNTZType.TIMESTAMP_NTZ)) { // For TIMESTAMP_NTZ the format should never have timezone info return InternalUtils.microsSinceEpoch(Timestamp.valueOf(input.getString(rowId))); } throw new UnsupportedOperationException("Invalid value request for data type"); } @Override public float getFloat(int rowId) { return Float.parseFloat(input.getString(rowId)); } @Override public double getDouble(int rowId) { return Double.parseDouble(input.getString(rowId)); } @Override public byte[] getBinary(int rowId) { return input.isNullAt(rowId) ? null : input.getString(rowId).getBytes(); } @Override public String getString(int rowId) { return input.getString(rowId); } @Override public BigDecimal getDecimal(int rowId) { return input.isNullAt(rowId) ? null : new BigDecimal(input.getString(rowId)); } }; } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/StartsWithExpressionEvaluator.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.expressions; import static io.delta.kernel.defaults.internal.expressions.DefaultExpressionUtils.*; import static io.delta.kernel.internal.util.ExpressionUtils.createPredicate; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.expressions.Expression; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.types.BooleanType; import io.delta.kernel.types.CollationIdentifier; import io.delta.kernel.types.DataType; import java.util.List; public class StartsWithExpressionEvaluator { /** Validates and transforms the {@code starts_with} expression. */ static Predicate validateAndTransform( Predicate startsWith, List childrenExpressions, List childrenOutputTypes) { checkArgsCount( startsWith, /* expectedCount= */ 2, startsWith.getName(), "Example usage: STARTS_WITH(column, 'test')"); for (DataType dataType : childrenOutputTypes) { checkIsStringType(dataType, startsWith, "'STARTS_WITH' expects STRING type inputs"); } // TODO: support non literal as the second input of starts with. checkIsLiteral( childrenExpressions.get(1), startsWith, "'STARTS_WITH' expects literal as the second input"); if (startsWith.getCollationIdentifier().isPresent()) { CollationIdentifier collationIdentifier = startsWith.getCollationIdentifier().get(); checkIsUTF8BinaryCollation(startsWith, collationIdentifier); } return createPredicate( startsWith.getName(), startsWith.getChildren(), startsWith.getCollationIdentifier()); } static ColumnVector eval(List childrenVectors) { return new ColumnVector() { final ColumnVector left = childrenVectors.get(0); final ColumnVector right = childrenVectors.get(1); @Override public DataType getDataType() { return BooleanType.BOOLEAN; } @Override public int getSize() { return left.getSize(); } @Override public void close() { Utils.closeCloseables(left, right); } @Override public boolean getBoolean(int rowId) { if (isNullAt(rowId)) { // The return value is undefined and can be anything, if the slot for rowId is null. return false; } return left.getString(rowId).startsWith(right.getString(rowId)); } @Override public boolean isNullAt(int rowId) { return left.isNullAt(rowId) || right.isNullAt(rowId); } }; } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/SubstringEvaluator.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.expressions; import static io.delta.kernel.defaults.internal.DefaultEngineErrors.unsupportedExpressionException; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.expressions.Expression; import io.delta.kernel.expressions.ScalarExpression; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.types.*; import java.util.*; /** Utility methods to evaluate {@code substring} expression. */ public class SubstringEvaluator { private SubstringEvaluator() {} /** Validates and transforms the {@code substring} expression. */ static ScalarExpression validateAndTransform( ScalarExpression substring, List childrenExpressions, List childrenOutputTypes) { int childrenSize = substring.getChildren().size(); if (childrenSize < 2 || childrenSize > 3) { throw unsupportedExpressionException( substring, "Invalid number of inputs to SUBSTRING expression. " + "Example usage: SUBSTRING(column, pos), SUBSTRING(column, pos, len)"); } // TODO: support binary type. if (!StringType.STRING.equals(childrenOutputTypes.get(0))) { throw unsupportedExpressionException( substring, "Invalid type of first input of SUBSTRING: expects STRING"); } Expression posExpression = childrenExpressions.get(1); DefaultExpressionUtils.checkIntegerLiteral( posExpression, /* context= */ "Invalid `pos` argument type for SUBSTRING", substring); if (childrenSize == 3) { Expression lengthExpression = childrenExpressions.get(2); DefaultExpressionUtils.checkIntegerLiteral( lengthExpression, /* context= */ "Invalid `len` argument type for SUBSTRING", substring); } return new ScalarExpression(substring.getName(), childrenExpressions); } /** * Evaluates the {@code substring} expression for given input column vector, builds a column * vector with substring applied to each row. */ static ColumnVector eval(List childrenVectors) { return new ColumnVector() { final ColumnVector input = childrenVectors.get(0); final ColumnVector positionVector = childrenVectors.get(1); final Optional lengthVector = childrenVectors.size() > 2 ? Optional.of(childrenVectors.get(2)) : Optional.empty(); @Override public DataType getDataType() { return StringType.STRING; } @Override public int getSize() { return input.getSize(); } @Override public void close() { // Utils.closeCloseables method will ignore the null element. Utils.closeCloseables(input, positionVector, lengthVector.orElse(null)); } @Override public boolean isNullAt(int rowId) { if (rowId < 0 || rowId >= getSize()) { throw new IllegalArgumentException( String.format( "Unexpected rowId %d, expected between 0 and the size of the column vector", rowId)); } return input.isNullAt(rowId); } @Override public String getString(int rowId) { if (isNullAt(rowId)) { return null; } String inputString = input.getString(rowId); int position = positionVector.getInt(rowId); Optional length = lengthVector.map(columnVector -> columnVector.getInt(rowId)); if (position > getStringLength(inputString) || (length.isPresent() && length.get() < 1)) { return ""; } int startPosition = buildStartPosition(inputString, position); int startIndex = Math.max(startPosition, 0); return length .map( len -> { // endIndex should be less than the length of input string, but positive. // e.g. Substring("aaa", -100, 95), should be read as Substring("aaa", 0, 0) int endIndex = Math.min(getStringLength(inputString), Math.max(startPosition + len, 0)); return getSubstring(inputString, startIndex, Optional.of(endIndex)); }) .orElse(getSubstring(inputString, startIndex, Optional.empty())); } }; } /** * Computes the start position following Hive and SQL's one-based indexing for substring. * * @param pos, pos can be positive, in which case the startIndex is computed from the left end of * the string. Otherwise, the startIndex is computed from the right end of the string. * @return the position of inputString to compute the substring. The returned value can fall * beyond the input index valid range. For example, pos could be larger than the inputString's * length or inputString.length() + pos could be negative. */ private static int buildStartPosition(String inputString, int pos) { // Handles the negative position (substring("abc", -2, 1), the start position should be 1("b")) if (pos < 0) { return getStringLength(inputString) + pos; } // Pos is 1 based and pos = 0 is treated as 1. return Math.max(pos - 1, 0); } /** Returns code point based string length for handling surrogate pairs. */ private static int getStringLength(String s) { return s.codePointCount(/* beginIndex = */ 0, s.length()); } /** Returns code point based substring for handling surrogate pairs. */ private static String getSubstring(String s, int start, Optional end) { int startIndex = s.offsetByCodePoints(/* beginIndex = */ 0, start); return end.map(e -> s.substring(startIndex, s.offsetByCodePoints(0, e))) .orElse(s.substring(startIndex)); } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/json/JsonUtils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.json; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.*; import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.ser.std.StdSerializer; import io.delta.kernel.data.*; import io.delta.kernel.defaults.internal.data.DefaultJsonRow; import io.delta.kernel.types.*; import java.io.IOException; /** * Utilities method to serialize and deserialize {@link Row} objects with a limited set of data type * values. * *

Following are the supported data types: {@code boolean}, {@code byte}, {@code short}, {@code * int}, {@code long}, {@code float}, {@code double}, {@code string}, {@code StructType} (containing * any of the supported subtypes), {@code ArrayType}, {@code MapType} (only a map with string keys * is supported). * *

At a high-level, the JSON serialization is similar to that of Jackson's {@link ObjectMapper}. */ public class JsonUtils { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); static { OBJECT_MAPPER.registerModule(new SimpleModule().addSerializer(Row.class, new RowSerializer())); } private JsonUtils() {} /** * Converts a {@link Row} to a single line JSON string. This is currently used just in tests. Wll * be used as part of the refactoring planned in #2929 * * @param row the row to convert * @return JSON string */ public static String rowToJson(Row row) { try { return OBJECT_MAPPER.writeValueAsString(row); } catch (JsonProcessingException ex) { throw new RuntimeException("Could not serialize row object to JSON", ex); } } /** * Converts a JSON string to a {@link Row}. * * @param json JSON string * @param schema to read the JSON according the schema * @return {@link Row} instance with given schema. */ public static Row rowFromJson(String json, StructType schema) { try { final JsonNode jsonNode = OBJECT_MAPPER.readTree(json); return new DefaultJsonRow((ObjectNode) jsonNode, schema); } catch (JsonProcessingException ex) { throw new RuntimeException(String.format("Could not parse JSON: %s", json), ex); } } public static class RowSerializer extends StdSerializer { public RowSerializer() { super(Row.class); } @Override public void serialize(Row row, JsonGenerator gen, SerializerProvider provider) throws IOException { writeRow(gen, row, row.getSchema()); } private void writeRow(JsonGenerator gen, Row row, StructType schema) throws IOException { gen.writeStartObject(); for (int columnOrdinal = 0; columnOrdinal < schema.length(); columnOrdinal++) { StructField field = schema.at(columnOrdinal); if (!row.isNullAt(columnOrdinal)) { gen.writeFieldName(field.getName()); writeValue(gen, row, columnOrdinal, field.getDataType()); } } gen.writeEndObject(); } private void writeStruct(JsonGenerator gen, ColumnVector vector, StructType type, int rowId) throws IOException { gen.writeStartObject(); for (int columnOrdinal = 0; columnOrdinal < type.length(); columnOrdinal++) { StructField field = type.at(columnOrdinal); ColumnVector childVector = vector.getChild(columnOrdinal); if (!childVector.isNullAt(rowId)) { gen.writeFieldName(field.getName()); writeValue(gen, childVector, rowId, field.getDataType()); } } gen.writeEndObject(); } private void writeArrayValue(JsonGenerator gen, ArrayValue arrayValue, ArrayType arrayType) throws IOException { gen.writeStartArray(); ColumnVector arrayElems = arrayValue.getElements(); for (int i = 0; i < arrayValue.getSize(); i++) { if (arrayElems.isNullAt(i)) { // Jackson serializes the null values in the array, but not in the map gen.writeNull(); } else { writeValue(gen, arrayValue.getElements(), i, arrayType.getElementType()); } } gen.writeEndArray(); } private void writeMapValue(JsonGenerator gen, MapValue mapValue, MapType mapType) throws IOException { assertSupportedMapType(mapType); gen.writeStartObject(); ColumnVector keys = mapValue.getKeys(); ColumnVector values = mapValue.getValues(); for (int i = 0; i < mapValue.getSize(); i++) { gen.writeFieldName(keys.getString(i)); if (!values.isNullAt(i)) { writeValue(gen, values, i, mapType.getValueType()); } else { gen.writeNull(); } } gen.writeEndObject(); } private void writeValue(JsonGenerator gen, Row row, int columnOrdinal, DataType type) throws IOException { checkArgument(!row.isNullAt(columnOrdinal), "value should not be null"); if (type instanceof BooleanType) { gen.writeBoolean(row.getBoolean(columnOrdinal)); } else if (type instanceof ByteType) { gen.writeNumber(row.getByte(columnOrdinal)); } else if (type instanceof ShortType) { gen.writeNumber(row.getShort(columnOrdinal)); } else if (type instanceof IntegerType) { gen.writeNumber(row.getInt(columnOrdinal)); } else if (type instanceof LongType) { gen.writeNumber(row.getLong(columnOrdinal)); } else if (type instanceof FloatType) { gen.writeNumber(row.getFloat(columnOrdinal)); } else if (type instanceof DoubleType) { gen.writeNumber(row.getDouble(columnOrdinal)); } else if (type instanceof StringType) { gen.writeString(row.getString(columnOrdinal)); } else if (type instanceof StructType) { writeRow(gen, row.getStruct(columnOrdinal), (StructType) type); } else if (type instanceof ArrayType) { writeArrayValue(gen, row.getArray(columnOrdinal), (ArrayType) type); } else if (type instanceof MapType) { writeMapValue(gen, row.getMap(columnOrdinal), (MapType) type); } else { // `binary` type is not supported according the Delta Protocol throw new UnsupportedOperationException("unsupported data type: " + type); } } private void writeValue(JsonGenerator gen, ColumnVector vector, int rowId, DataType type) throws IOException { checkArgument(!vector.isNullAt(rowId), "value should not be null"); if (type instanceof BooleanType) { gen.writeBoolean(vector.getBoolean(rowId)); } else if (type instanceof ByteType) { gen.writeNumber(vector.getByte(rowId)); } else if (type instanceof ShortType) { gen.writeNumber(vector.getShort(rowId)); } else if (type instanceof IntegerType) { gen.writeNumber(vector.getInt(rowId)); } else if (type instanceof LongType) { gen.writeNumber(vector.getLong(rowId)); } else if (type instanceof FloatType) { gen.writeNumber(vector.getFloat(rowId)); } else if (type instanceof DoubleType) { gen.writeNumber(vector.getDouble(rowId)); } else if (type instanceof StringType) { gen.writeString(vector.getString(rowId)); } else if (type instanceof StructType) { writeStruct(gen, vector, (StructType) type, rowId); } else if (type instanceof ArrayType) { writeArrayValue(gen, vector.getArray(rowId), (ArrayType) type); } else if (type instanceof MapType) { writeMapValue(gen, vector.getMap(rowId), (MapType) type); } else { throw new UnsupportedOperationException("unsupported data type: " + type); } } } private static void assertSupportedMapType(MapType keyType) { checkArgument( keyType.getKeyType() instanceof StringType, "Only STRING type keys are supported in MAP type in JSON serialization"); } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/logstore/LogStoreProvider.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.logstore; import static io.delta.kernel.defaults.internal.DefaultEngineErrors.canNotInstantiateLogStore; import io.delta.storage.*; import java.util.*; import org.apache.hadoop.conf.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Utility class to provide the correct {@link LogStore} based on the scheme of the path. */ public class LogStoreProvider { private static final Logger logger = LoggerFactory.getLogger(LogStoreProvider.class); // Supported schemes per storage system. private static final Set S3_SCHEMES = unmodifiableSet("s3", "s3a", "s3n"); private static final Set AZURE_SCHEMES = unmodifiableSet("abfs", "abfss", "adl", "wasb", "wasbs"); private static final Set GCS_SCHEMES = unmodifiableSet("gs"); /** * Get the {@link LogStore} instance for the given scheme and configuration. Callers can set * {@code io.delta.kernel.logStore..impl} to specify the {@link LogStore} implementation * to use for {@code scheme}. * *

If not set, the default {@link LogStore} implementation (given below) for the scheme will be * used. * *

    *
  • {@code s3, s3a, s3n}: {@link S3SingleDriverLogStore} *
  • {@code abfs, abfss, adl, wasb, wasbs}: {@link AzureLogStore} *
  • {@code gs}: {@link GCSLogStore} *
  • {@code hdfs, file}: {@link HDFSLogStore} *
  • remaining: {@link HDFSLogStore} *
* * @param hadoopConf {@link Configuration} to use for creating the LogStore. * @param scheme Scheme of the path. * @return {@link LogStore} instance. * @throws IllegalArgumentException if the LogStore implementation is not found or can not be * instantiated. */ public static LogStore getLogStore(Configuration hadoopConf, String scheme) { String schemeLower = Optional.ofNullable(scheme).map(String::toLowerCase).orElse(null); // Check if the LogStore implementation is set in the configuration. String classNameFromConfig = hadoopConf.get(getLogStoreSchemeConfKey(schemeLower)); if (classNameFromConfig != null) { return createLogStore(classNameFromConfig, hadoopConf, "from config"); } // Create default LogStore based on the scheme. String defaultClassName = HDFSLogStore.class.getName(); if (S3_SCHEMES.contains(schemeLower)) { defaultClassName = S3SingleDriverLogStore.class.getName(); } else if (AZURE_SCHEMES.contains(schemeLower)) { defaultClassName = AzureLogStore.class.getName(); } else if (GCS_SCHEMES.contains(schemeLower)) { defaultClassName = GCSLogStore.class.getName(); } return createLogStore(defaultClassName, hadoopConf, "(default for file scheme)"); } /** * Configuration key for setting the LogStore implementation for a scheme. ex: * `io.delta.kernel.logStore.s3.impl` -> `io.delta.storage.S3SingleDriverLogStore` */ static String getLogStoreSchemeConfKey(String scheme) { return "io.delta.kernel.logStore." + scheme + ".impl"; } /** Utility method to get the LogStore class from the class name. */ private static Class getLogStoreClass(String logStoreClassName) throws ClassNotFoundException { return Class.forName(logStoreClassName).asSubclass(LogStore.class); } private static LogStore createLogStore( String className, Configuration hadoopConf, String context) { try { return getLogStoreClass(className) .getConstructor(Configuration.class) .newInstance(hadoopConf); } catch (Exception e) { String msgTemplate = "Failed to instantiate LogStore class ({}): {}"; logger.error(msgTemplate, context, className, e); throw canNotInstantiateLogStore(className, context, e); } } /** Remove this method once we start supporting JDK9+ */ private static Set unmodifiableSet(String... elements) { return Collections.unmodifiableSet(new HashSet<>(Arrays.asList(elements))); } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ArrayColumnReader.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.parquet; import static io.delta.kernel.defaults.internal.parquet.ParquetColumnReaders.createConverter; import static io.delta.kernel.defaults.internal.parquet.ParquetSchemaUtils.validateAndGetThreeLevelParquetArrayElementType; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.defaults.internal.data.vector.DefaultArrayVector; import io.delta.kernel.types.ArrayType; import java.util.Optional; import org.apache.parquet.io.api.Converter; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.Type; /** * Array column reader for materializing the column values from Parquet files into Kernels {@link * ColumnVector}. */ class ArrayColumnReader extends RepeatedValueConverter { private final ArrayType typeFromClient; ArrayColumnReader(int initialBatchSize, ArrayType typeFromClient, GroupType typeFromFile) { super(initialBatchSize, createElementConverter(initialBatchSize, typeFromClient, typeFromFile)); this.typeFromClient = typeFromClient; } @Override public ColumnVector getDataColumnVector(int batchSize) { ColumnVector arrayVector = new DefaultArrayVector( batchSize, typeFromClient, Optional.of(getNullability()), getOffsets(), getElementDataVectors()[0]); resetWorkingState(); return arrayVector; } /** * Currently, support for 3-level nested arrays only. * *

optional group readerFeatures (LIST) { repeated group list { optional binary element * (STRING); } } * *

optional group readerFeatures (LIST) { repeated group bag { optional binary array (STRING); * } } * *

TODO: Add support for 2-level nested arrays. */ private static Converter createElementConverter( int initialBatchSize, ArrayType typeFromClient, GroupType typeFromFile) { Type elementType = validateAndGetThreeLevelParquetArrayElementType(typeFromFile); return createConverter(initialBatchSize, typeFromClient.getElementType(), elementType); } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/DecimalColumnReader.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.parquet; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.defaults.internal.data.vector.DefaultDecimalVector; import io.delta.kernel.defaults.internal.parquet.ParquetColumnReaders.BasePrimitiveColumnReader; import io.delta.kernel.types.DataType; import io.delta.kernel.types.DecimalType; import java.math.BigDecimal; import java.math.BigInteger; import java.math.RoundingMode; import java.util.Arrays; import org.apache.parquet.column.Dictionary; import org.apache.parquet.io.api.Binary; import org.apache.parquet.io.api.Converter; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.PrimitiveType; import org.apache.parquet.schema.Type; /** * Decimal column readers for materializing the column values from Parquet files into Kernels {@link * ColumnVector}. */ public class DecimalColumnReader { public static Converter createDecimalConverter( int initialBatchSize, DecimalType typeFromClient, Type typeFromFile) { PrimitiveType primType = typeFromFile.asPrimitiveType(); LogicalTypeAnnotation typeAnnotation = primType.getLogicalTypeAnnotation(); if (primType.getPrimitiveTypeName() == INT32) { // For INT32 backed decimals if (typeAnnotation instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) { LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimalType = (LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) typeAnnotation; return new IntDictionaryAwareDecimalColumnReader( typeFromClient, decimalType.getPrecision(), decimalType.getScale(), initialBatchSize); } else { // If the column is a plain INT32, we should pick the precision that can host // the largest INT32 value. return new IntDictionaryAwareDecimalColumnReader(typeFromClient, 10, 0, initialBatchSize); } } else if (primType.getPrimitiveTypeName() == INT64) { // For INT64 backed decimals if (typeAnnotation instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) { LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimalType = (LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) typeAnnotation; return new LongDictionaryAwareDecimalColumnReader( typeFromClient, decimalType.getPrecision(), decimalType.getScale(), initialBatchSize); } else { // If the column is a plain INT64, we should pick the precision that can host // the largest INT64 value. return new LongDictionaryAwareDecimalColumnReader(typeFromClient, 20, 0, initialBatchSize); } } else if (primType.getPrimitiveTypeName() == FIXED_LEN_BYTE_ARRAY || primType.getPrimitiveTypeName() == BINARY) { // For BINARY and FIXED_LEN_BYTE_ARRAY backed decimals if (typeAnnotation instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) { LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimalType = (LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) typeAnnotation; return new BinaryDictionaryAwareDecimalColumnReader( typeFromClient, decimalType.getPrecision(), decimalType.getScale(), initialBatchSize); } else { throw new RuntimeException( String.format( "Unable to create Parquet converter for DecimalType whose parquet " + "type is %s without decimal metadata.", typeFromFile)); } } else { throw new RuntimeException( String.format( "Unable to create Parquet converter for DecimalType whose Parquet type " + "is %s. Parquet DECIMAL type can only be backed by INT32, INT64, " + "FIXED_LEN_BYTE_ARRAY, or BINARY", typeFromFile)); } } public abstract static class BaseDecimalColumnReader extends BasePrimitiveColumnReader { // working state private BigDecimal[] values; private final DecimalType decimalType; private final int scale; protected BigDecimal[] expandedDictionary; BaseDecimalColumnReader(DataType dataType, int precision, int scale, int initialBatchSize) { super(initialBatchSize); DecimalType decimalType = (DecimalType) dataType; int scaleIncrease = decimalType.getScale() - scale; int precisionIncrease = decimalType.getPrecision() - precision; checkArgument( scaleIncrease >= 0 && precisionIncrease >= scaleIncrease, "Found Delta type %s but Parquet type has precision=%s and scale=%s", decimalType, precision, scale); this.scale = scale; this.decimalType = decimalType; this.values = new BigDecimal[initialBatchSize]; } /** * Dictionary support is an optional optimization to reduce BigDecimal instantiation and binary * decoding (for Binary backed decimals). */ @Override public boolean hasDictionarySupport() { return true; } protected void addDecimal(BigDecimal value) { resizeIfNeeded(); this.nullability[currentRowIndex] = false; if (decimalType.getScale() != scale) { value = value.setScale(decimalType.getScale(), RoundingMode.UNNECESSARY); } this.values[currentRowIndex] = value; } @Override public void addValueFromDictionary(int dictionaryId) { addDecimal(expandedDictionary[dictionaryId]); } @Override public ColumnVector getDataColumnVector(int batchSize) { ColumnVector vector = new DefaultDecimalVector(decimalType, batchSize, values); // re-initialize the working space this.nullability = ParquetColumnReaders.initNullabilityVector(nullability.length); this.values = new BigDecimal[values.length]; this.currentRowIndex = 0; return vector; } @Override public void resizeIfNeeded() { if (values.length == currentRowIndex) { int newSize = values.length * 2; this.values = Arrays.copyOf(this.values, newSize); this.nullability = Arrays.copyOf(this.nullability, newSize); ParquetColumnReaders.setNullabilityToTrue(this.nullability, newSize / 2, newSize); } } protected BigDecimal decimalFromLong(long value) { return BigDecimal.valueOf(value, scale); } protected BigDecimal decimalFromBinary(Binary value) { return new BigDecimal(new BigInteger(value.getBytes()), scale); } } public static class IntDictionaryAwareDecimalColumnReader extends BaseDecimalColumnReader { IntDictionaryAwareDecimalColumnReader( DataType dataType, int precision, int scale, int initialBatchSize) { super(dataType, precision, scale, initialBatchSize); } @Override public void setDictionary(Dictionary dictionary) { this.expandedDictionary = new BigDecimal[dictionary.getMaxId() + 1]; for (int id = 0; id < dictionary.getMaxId() + 1; id++) { this.expandedDictionary[id] = decimalFromLong(dictionary.decodeToInt(id)); } } @Override // Converts decimals stored as INT32 public void addInt(int value) { addDecimal(decimalFromLong(value)); } } public static class LongDictionaryAwareDecimalColumnReader extends BaseDecimalColumnReader { LongDictionaryAwareDecimalColumnReader( DataType dataType, int precision, int scale, int initialBatchSize) { super(dataType, precision, scale, initialBatchSize); } @Override public void setDictionary(Dictionary dictionary) { this.expandedDictionary = new BigDecimal[dictionary.getMaxId() + 1]; for (int id = 0; id < dictionary.getMaxId() + 1; id++) { this.expandedDictionary[id] = decimalFromLong(dictionary.decodeToLong(id)); } } @Override // Converts decimals stored as INT64 public void addLong(long value) { addDecimal(decimalFromLong(value)); } } public static class BinaryDictionaryAwareDecimalColumnReader extends BaseDecimalColumnReader { BinaryDictionaryAwareDecimalColumnReader( DataType dataType, int precision, int scale, int initialBatchSize) { super(dataType, precision, scale, initialBatchSize); } @Override public void setDictionary(Dictionary dictionary) { this.expandedDictionary = new BigDecimal[dictionary.getMaxId() + 1]; for (int id = 0; id < dictionary.getMaxId() + 1; id++) { this.expandedDictionary[id] = decimalFromBinary(dictionary.decodeToBinary(id)); } } @Override // Converts decimals stored as either FIXED_LENGTH_BYTE_ARRAY or BINARY public void addBinary(Binary value) { addDecimal(decimalFromBinary(value)); } } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/MapColumnReader.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.parquet; import static io.delta.kernel.defaults.internal.parquet.ParquetColumnReaders.createConverter; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.defaults.internal.data.vector.DefaultMapVector; import io.delta.kernel.types.MapType; import java.util.Optional; import org.apache.parquet.io.api.Converter; import org.apache.parquet.schema.GroupType; /** * Map column readers for materializing the column values from Parquet files into Kernels {@link * ColumnVector}. */ class MapColumnReader extends RepeatedValueConverter { private final MapType typeFromClient; MapColumnReader(int initialBatchSize, MapType typeFromClient, GroupType typeFromFile) { super( initialBatchSize, createElementConverters(initialBatchSize, typeFromClient, typeFromFile)); this.typeFromClient = typeFromClient; } @Override public ColumnVector getDataColumnVector(int batchSize) { ColumnVector[] elementVectors = getElementDataVectors(); ColumnVector mapVector = new DefaultMapVector( batchSize, typeFromClient, Optional.of(getNullability()), getOffsets(), elementVectors[0], elementVectors[1]); resetWorkingState(); return mapVector; } private static Converter[] createElementConverters( int initialBatchSize, MapType typeFromClient, GroupType typeFromFile) { // Repeated element can be any name. Latest Parquet versions use "key_value" as the name, // but legacy versions can use any arbitrary name for the repeated group. // See https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#maps for details checkArgument( typeFromFile.getFieldCount() == 1, "Expected exactly one repeated field in the map type, but got: %s", typeFromFile); GroupType innerMapType = typeFromFile.getType(0).asGroupType(); Converter[] elemConverters = new Converter[2]; elemConverters[0] = createConverter(initialBatchSize, typeFromClient.getKeyType(), innerMapType.getType("key")); elemConverters[1] = createConverter( initialBatchSize, typeFromClient.getValueType(), innerMapType.getType("value")); return elemConverters; } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetColumnReaders.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.parquet; import static io.delta.kernel.defaults.internal.parquet.TimestampConverters.createTimestampConverter; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.defaults.internal.data.vector.*; import io.delta.kernel.types.*; import java.util.Arrays; import java.util.Objects; import java.util.Optional; import org.apache.parquet.io.api.Binary; import org.apache.parquet.io.api.Converter; import org.apache.parquet.io.api.PrimitiveConverter; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.Type; /** * Parquet column readers for materializing the column values from Parquet files into Kernels {@link * ColumnVector}. */ class ParquetColumnReaders { public static Converter createConverter( int initialBatchSize, DataType typeFromClient, Type typeFromFile) { if (typeFromClient instanceof StructType) { checkArgument(typeFromFile instanceof GroupType, "cannot be cast to GroupType"); return new RowColumnReader( initialBatchSize, (StructType) typeFromClient, (GroupType) typeFromFile); } else if (typeFromClient instanceof ArrayType) { checkArgument(typeFromFile instanceof GroupType, "cannot be cast to GroupType"); return new ArrayColumnReader( initialBatchSize, (ArrayType) typeFromClient, (GroupType) typeFromFile); } else if (typeFromClient instanceof MapType) { checkArgument(typeFromFile instanceof GroupType, "cannot be cast to GroupType"); return new MapColumnReader( initialBatchSize, (MapType) typeFromClient, (GroupType) typeFromFile); } else if (typeFromClient instanceof StringType || typeFromClient instanceof BinaryType) { return new BinaryColumnReader(typeFromClient, initialBatchSize); } else if (typeFromClient instanceof BooleanType) { return new BooleanColumnReader(initialBatchSize); } else if (typeFromClient instanceof IntegerType || typeFromClient instanceof DateType) { return new IntColumnReader(typeFromClient, initialBatchSize); } else if (typeFromClient instanceof ByteType) { return new ByteColumnReader(initialBatchSize); } else if (typeFromClient instanceof ShortType) { return new ShortColumnReader(initialBatchSize); } else if (typeFromClient instanceof LongType) { return new LongColumnReader(typeFromClient, initialBatchSize); } else if (typeFromClient instanceof FloatType) { return new FloatColumnReader(initialBatchSize); } else if (typeFromClient instanceof DoubleType) { return new DoubleColumnReader(initialBatchSize); } else if (typeFromClient instanceof DecimalType) { return DecimalColumnReader.createDecimalConverter( initialBatchSize, (DecimalType) typeFromClient, typeFromFile); } else if (typeFromClient instanceof TimestampType) { return createTimestampConverter(initialBatchSize, typeFromFile, TimestampType.TIMESTAMP); } else if (typeFromClient instanceof TimestampNTZType) { return createTimestampConverter( initialBatchSize, typeFromFile, TimestampNTZType.TIMESTAMP_NTZ); } throw new UnsupportedOperationException(typeFromClient + " is not supported"); } static boolean[] initNullabilityVector(int size) { boolean[] nullability = new boolean[size]; // Initialize all values as null. As Parquet calls this converter only for non-null // values, make the corresponding value to false. Arrays.fill(nullability, true); return nullability; } static void setNullabilityToTrue(boolean[] nullability, int start, int end) { // Initialize all values as null. As Parquet calls this converter only for non-null // values, make the corresponding value to false. Arrays.fill(nullability, start, end, true); } /** * Base column reader for all implementations of Parquet {@link Converter} to return data in * columnar batch. General operation flow is: - each reader implementation allocates state to * receive a fixed number of column values - before accepting a new value the state is resized if * it is not of sufficient size - after each row, {@link #finalizeCurrentRow(long)} is called to * finalize the state of the last read row column value. */ public interface BaseColumnReader { ColumnVector getDataColumnVector(int batchSize); /** * Finalize the current row: - close the state of the row that was read most recently. - * reallocate the state to be of sufficient size for the current batch size. Generally the state * value arrays are resized as part of setting the value, but method doesn't get called for null * values which results in the state value arrays are not sufficient size for the current batch * size. * * @param currentRowIndex Row index of the current row in the Parquet file. */ void finalizeCurrentRow(long currentRowIndex); default void resizeIfNeeded() {} default void resetWorkingState() {} } public static class NonExistentColumnReader extends PrimitiveConverter implements BaseColumnReader { private final DataType dataType; NonExistentColumnReader(DataType dataType) { this.dataType = Objects.requireNonNull(dataType, "dataType is null"); } @Override public ColumnVector getDataColumnVector(int batchSize) { return new DefaultConstantVector(dataType, batchSize, null); } @Override public void finalizeCurrentRow(long currentRowIndex) {} } public abstract static class BasePrimitiveColumnReader extends PrimitiveConverter implements BaseColumnReader { // working state protected int currentRowIndex; protected boolean[] nullability; BasePrimitiveColumnReader(int initialBatchSize) { checkArgument(initialBatchSize > 0, "invalid initialBatchSize: %s", initialBatchSize); // Initialize the working state this.nullability = initNullabilityVector(initialBatchSize); } @Override public void finalizeCurrentRow(long currentRowIndex) { resizeIfNeeded(); this.currentRowIndex++; } } public static class BooleanColumnReader extends BasePrimitiveColumnReader { // working state private boolean[] values; BooleanColumnReader(int initialBatchSize) { super(initialBatchSize); this.values = new boolean[initialBatchSize]; } @Override public void addBoolean(boolean value) { resizeIfNeeded(); this.nullability[currentRowIndex] = false; this.values[currentRowIndex] = value; } @Override public ColumnVector getDataColumnVector(int batchSize) { ColumnVector vector = new DefaultBooleanVector(batchSize, Optional.of(nullability), values); this.nullability = initNullabilityVector(nullability.length); this.values = new boolean[values.length]; this.currentRowIndex = 0; return vector; } @Override public void resizeIfNeeded() { if (values.length == currentRowIndex) { int newSize = values.length * 2; this.values = Arrays.copyOf(this.values, newSize); this.nullability = Arrays.copyOf(this.nullability, newSize); setNullabilityToTrue(this.nullability, newSize / 2, newSize); } } } public static class ByteColumnReader extends BasePrimitiveColumnReader { // working state private byte[] values; ByteColumnReader(int initialBatchSize) { super(initialBatchSize); this.values = new byte[initialBatchSize]; } @Override public void addInt(int value) { resizeIfNeeded(); this.nullability[currentRowIndex] = false; this.values[currentRowIndex] = (byte) value; } @Override public ColumnVector getDataColumnVector(int batchSize) { ColumnVector vector = new DefaultByteVector(batchSize, Optional.of(nullability), values); this.nullability = initNullabilityVector(nullability.length); this.values = new byte[values.length]; this.currentRowIndex = 0; return vector; } @Override public void resizeIfNeeded() { if (values.length == currentRowIndex) { int newSize = values.length * 2; this.values = Arrays.copyOf(this.values, newSize); this.nullability = Arrays.copyOf(this.nullability, newSize); setNullabilityToTrue(this.nullability, newSize / 2, newSize); } } } public static class ShortColumnReader extends BasePrimitiveColumnReader { // working state private short[] values; ShortColumnReader(int initialBatchSize) { super(initialBatchSize); this.values = new short[initialBatchSize]; } @Override public void addInt(int value) { resizeIfNeeded(); this.nullability[currentRowIndex] = false; this.values[currentRowIndex] = (short) value; } @Override public ColumnVector getDataColumnVector(int batchSize) { ColumnVector vector = new DefaultShortVector(batchSize, Optional.of(nullability), values); this.nullability = initNullabilityVector(nullability.length); this.values = new short[values.length]; this.currentRowIndex = 0; return vector; } @Override public void resizeIfNeeded() { if (values.length == currentRowIndex) { int newSize = values.length * 2; this.values = Arrays.copyOf(this.values, newSize); this.nullability = Arrays.copyOf(this.nullability, newSize); setNullabilityToTrue(this.nullability, newSize / 2, newSize); } } } public static class IntColumnReader extends BasePrimitiveColumnReader { private final DataType dataType; // working state private int[] values; IntColumnReader(DataType dataType, int initialBatchSize) { super(initialBatchSize); checkArgument(dataType instanceof IntegerType || dataType instanceof DateType); this.dataType = dataType; this.values = new int[initialBatchSize]; } @Override public void addInt(int value) { resizeIfNeeded(); this.nullability[currentRowIndex] = false; this.values[currentRowIndex] = value; } @Override public ColumnVector getDataColumnVector(int batchSize) { ColumnVector vector = new DefaultIntVector(dataType, batchSize, Optional.of(nullability), values); this.nullability = initNullabilityVector(nullability.length); this.values = new int[values.length]; this.currentRowIndex = 0; return vector; } @Override public void resizeIfNeeded() { if (values.length == currentRowIndex) { int newSize = values.length * 2; this.values = Arrays.copyOf(this.values, newSize); this.nullability = Arrays.copyOf(this.nullability, newSize); setNullabilityToTrue(this.nullability, newSize / 2, newSize); } } } public static class LongColumnReader extends BasePrimitiveColumnReader { private final DataType dataType; // working state private long[] values; LongColumnReader(DataType dataType, int initialBatchSize) { super(initialBatchSize); checkArgument( dataType instanceof LongType || dataType instanceof TimestampType || dataType instanceof TimestampNTZType); this.dataType = dataType; this.values = new long[initialBatchSize]; } @Override public void addInt(int value) { resizeIfNeeded(); this.nullability[currentRowIndex] = false; this.values[currentRowIndex] = value; } @Override public void addLong(long value) { resizeIfNeeded(); this.nullability[currentRowIndex] = false; this.values[currentRowIndex] = value; } @Override public ColumnVector getDataColumnVector(int batchSize) { ColumnVector vector = new DefaultLongVector(dataType, batchSize, Optional.of(nullability), values); this.nullability = initNullabilityVector(nullability.length); this.values = new long[values.length]; this.currentRowIndex = 0; return vector; } @Override public void resizeIfNeeded() { if (values.length == currentRowIndex) { int newSize = values.length * 2; this.values = Arrays.copyOf(this.values, newSize); this.nullability = Arrays.copyOf(this.nullability, newSize); setNullabilityToTrue(this.nullability, newSize / 2, newSize); } } } public static class FloatColumnReader extends BasePrimitiveColumnReader { // working state private float[] values; FloatColumnReader(int initialBatchSize) { super(initialBatchSize); this.values = new float[initialBatchSize]; } @Override public void addFloat(float value) { resizeIfNeeded(); this.nullability[currentRowIndex] = false; this.values[currentRowIndex] = value; } @Override public ColumnVector getDataColumnVector(int batchSize) { ColumnVector vector = new DefaultFloatVector(batchSize, Optional.of(nullability), values); this.nullability = initNullabilityVector(nullability.length); this.values = new float[values.length]; this.currentRowIndex = 0; return vector; } @Override public void resizeIfNeeded() { if (values.length == currentRowIndex) { int newSize = values.length * 2; this.values = Arrays.copyOf(this.values, newSize); this.nullability = Arrays.copyOf(this.nullability, newSize); setNullabilityToTrue(this.nullability, newSize / 2, newSize); } } } public static class DoubleColumnReader extends BasePrimitiveColumnReader { // working state private double[] values; DoubleColumnReader(int initialBatchSize) { super(initialBatchSize); this.values = new double[initialBatchSize]; } @Override public void addInt(int value) { resizeIfNeeded(); this.nullability[currentRowIndex] = false; this.values[currentRowIndex] = value; } @Override public void addFloat(float value) { resizeIfNeeded(); this.nullability[currentRowIndex] = false; this.values[currentRowIndex] = value; } @Override public void addDouble(double value) { resizeIfNeeded(); this.nullability[currentRowIndex] = false; this.values[currentRowIndex] = value; } @Override public ColumnVector getDataColumnVector(int batchSize) { ColumnVector vector = new DefaultDoubleVector(batchSize, Optional.of(nullability), values); // re-initialize the working space this.nullability = initNullabilityVector(nullability.length); this.values = new double[values.length]; this.currentRowIndex = 0; return vector; } @Override public void resizeIfNeeded() { if (values.length == currentRowIndex) { int newSize = values.length * 2; this.values = Arrays.copyOf(this.values, newSize); this.nullability = Arrays.copyOf(this.nullability, newSize); setNullabilityToTrue(this.nullability, newSize / 2, newSize); } } } public static class BinaryColumnReader extends BasePrimitiveColumnReader { private final DataType dataType; // working state private byte[][] values; BinaryColumnReader(DataType dataType, int initialBatchSize) { super(initialBatchSize); this.dataType = dataType; this.values = new byte[initialBatchSize][]; } @Override public void addBinary(Binary value) { resizeIfNeeded(); this.nullability[currentRowIndex] = false; this.values[currentRowIndex] = value.getBytes(); } @Override public ColumnVector getDataColumnVector(int batchSize) { ColumnVector vector = new DefaultBinaryVector(dataType, batchSize, values); // re-initialize the working space this.nullability = initNullabilityVector(nullability.length); this.values = new byte[values.length][]; this.currentRowIndex = 0; return vector; } @Override public void resizeIfNeeded() { if (values.length == currentRowIndex) { int newSize = values.length * 2; this.values = Arrays.copyOf(this.values, newSize); this.nullability = Arrays.copyOf(this.nullability, newSize); setNullabilityToTrue(this.nullability, newSize / 2, newSize); } } } public static class FileRowIndexColumnReader extends LongColumnReader { FileRowIndexColumnReader(int initialBatchSize) { super(LongType.LONG, initialBatchSize); } @Override public void addLong(long value) { throw new UnsupportedOperationException("cannot add long to metadata column"); } @Override public void finalizeCurrentRow(long currentRowIndex) { // Set the previous row index value as the value super.addLong(currentRowIndex); super.finalizeCurrentRow(currentRowIndex); } } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetColumnWriters.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.parquet; import static io.delta.kernel.defaults.internal.parquet.ParquetSchemaUtils.MAX_BYTES_PER_PRECISION; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.data.*; import io.delta.kernel.types.*; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; import java.util.Arrays; import org.apache.parquet.io.api.Binary; import org.apache.parquet.io.api.RecordConsumer; /** * Parquet column writers for writing columnar vectors to Parquet files using the {@link * RecordConsumer} interface. */ class ParquetColumnWriters { private ParquetColumnWriters() {} /** * Create column vector writers for the given columnar batch. * * @param batch the columnar batch * @return an array of column vector writers */ static ColumnWriter[] createColumnVectorWriters(ColumnarBatch batch) { requireNonNull(batch, "batch is null"); StructType schema = batch.getSchema(); ColumnVector[] columnVectors = new ColumnVector[schema.length()]; for (int fieldIndex = 0; fieldIndex < schema.length(); fieldIndex++) { columnVectors[fieldIndex] = batch.getColumnVector(fieldIndex); } return createColumnVectorWritersHelper(schema, columnVectors); } /** * Create column vector writers for the given struct column vector. TODO: Having the ColumnarBatch * as separate interface complicates the code. ColumnarBatch is also a ColumnVector of type * STRUCT. * * @param structColumnVector the column vector * @return an array of column vector writers */ static ColumnWriter[] createColumnVectorWriters(ColumnVector structColumnVector) { requireNonNull(structColumnVector, "batch is null"); checkArgument( structColumnVector.getDataType() instanceof StructType, "ColumnVector is not a struct type"); StructType schema = (StructType) structColumnVector.getDataType(); ColumnVector[] columnVectors = new ColumnVector[schema.length()]; for (int fieldIndex = 0; fieldIndex < schema.length(); fieldIndex++) { columnVectors[fieldIndex] = structColumnVector.getChild(fieldIndex); } return createColumnVectorWritersHelper(schema, columnVectors); } private static ColumnWriter[] createColumnVectorWritersHelper( StructType schema, ColumnVector[] columnVectors) { int numCols = schema.length(); checkArgument( numCols == columnVectors.length, "Number of columns in schema does not match number of column vectors"); ColumnWriter[] columnWriters = new ColumnWriter[numCols]; for (int fieldIndex = 0; fieldIndex < numCols; fieldIndex++) { String colName = schema.at(fieldIndex).getName(); ColumnVector columnVector = columnVectors[fieldIndex]; columnWriters[fieldIndex] = createColumnWriter(colName, fieldIndex, columnVector); } return columnWriters; } private static ColumnWriter createColumnWriter( String colName, int fieldIndex, ColumnVector columnVector) { DataType dataType = columnVector.getDataType(); if (dataType instanceof BooleanType) { return new BooleanWriter(colName, fieldIndex, columnVector); } else if (dataType instanceof ByteType) { return new ByteWriter(colName, fieldIndex, columnVector); } else if (dataType instanceof ShortType) { return new ShortWriter(colName, fieldIndex, columnVector); } else if (dataType instanceof IntegerType) { return new IntWriter(colName, fieldIndex, columnVector); } else if (dataType instanceof LongType) { return new LongWriter(colName, fieldIndex, columnVector); } else if (dataType instanceof FloatType) { return new FloatWriter(colName, fieldIndex, columnVector); } else if (dataType instanceof DoubleType) { return new DoubleWriter(colName, fieldIndex, columnVector); } else if (dataType instanceof StringType) { return new StringWriter(colName, fieldIndex, columnVector); } else if (dataType instanceof BinaryType) { return new BinaryWriter(colName, fieldIndex, columnVector); } else if (dataType instanceof DecimalType) { int precision = ((DecimalType) dataType).getPrecision(); if (precision <= ParquetSchemaUtils.DECIMAL_MAX_DIGITS_IN_INT) { return new DecimalIntWriter(colName, fieldIndex, columnVector); } else if (precision <= ParquetSchemaUtils.DECIMAL_MAX_DIGITS_IN_LONG) { return new DecimalLongWriter(colName, fieldIndex, columnVector); } // TODO: Need to support legacy mode where all decimals are written as binary return new DecimalFixedBinaryWriter(colName, fieldIndex, columnVector); } else if (dataType instanceof DateType) { return new DateWriter(colName, fieldIndex, columnVector); } else if (dataType instanceof TimestampType || dataType instanceof TimestampNTZType) { // for both get the input as long type from column vector and write to file as INT64 return new TimestampWriter(colName, fieldIndex, columnVector); } else if (dataType instanceof ArrayType) { return new ArrayWriter(colName, fieldIndex, columnVector); } else if (dataType instanceof MapType) { return new MapWriter(colName, fieldIndex, columnVector); } else if (dataType instanceof StructType) { return new StructWriter(colName, fieldIndex, columnVector); } throw new IllegalArgumentException("Unsupported column vector type: " + dataType); } /** * Base class for column writers. Handles the common stuff such as null check, start/stop of field * and delegating the actual writing of non-null values to the subclass. */ abstract static class ColumnWriter { protected final String colName; protected final int fieldIndex; protected final ColumnVector columnVector; ColumnWriter(String colName, int fieldIndex, ColumnVector columnVector) { this.colName = colName; this.fieldIndex = fieldIndex; this.columnVector = columnVector; } void writeRowValue(RecordConsumer recordConsumer, int rowId) { if (!columnVector.isNullAt(rowId)) { recordConsumer.startField(colName, fieldIndex); writeNonNullRowValue(recordConsumer, rowId); recordConsumer.endField(colName, fieldIndex); } } /** * Each specific column writer for data type, will implement to call appropriate methods on the * {@link RecordConsumer} to write the non-null value. */ abstract void writeNonNullRowValue(RecordConsumer recordConsumer, int rowId); } static class BooleanWriter extends ColumnWriter { BooleanWriter(String name, int fieldId, ColumnVector columnVector) { super(name, fieldId, columnVector); } @Override void writeNonNullRowValue(RecordConsumer recordConsumer, int rowId) { recordConsumer.addBoolean(columnVector.getBoolean(rowId)); } } static class ByteWriter extends ColumnWriter { ByteWriter(String name, int fieldId, ColumnVector columnVector) { super(name, fieldId, columnVector); } @Override void writeNonNullRowValue(RecordConsumer recordConsumer, int rowId) { recordConsumer.addInteger(columnVector.getByte(rowId)); } } static class ShortWriter extends ColumnWriter { ShortWriter(String name, int fieldId, ColumnVector columnVector) { super(name, fieldId, columnVector); } @Override void writeNonNullRowValue(RecordConsumer recordConsumer, int rowId) { recordConsumer.addInteger(columnVector.getShort(rowId)); } } static class IntWriter extends ColumnWriter { IntWriter(String name, int fieldId, ColumnVector columnVector) { super(name, fieldId, columnVector); } @Override void writeNonNullRowValue(RecordConsumer recordConsumer, int rowId) { recordConsumer.addInteger(columnVector.getInt(rowId)); } } static class LongWriter extends ColumnWriter { LongWriter(String name, int fieldId, ColumnVector columnVector) { super(name, fieldId, columnVector); } @Override void writeNonNullRowValue(RecordConsumer recordConsumer, int rowId) { recordConsumer.addLong(columnVector.getLong(rowId)); } } static class FloatWriter extends ColumnWriter { FloatWriter(String name, int fieldId, ColumnVector columnVector) { super(name, fieldId, columnVector); } @Override void writeNonNullRowValue(RecordConsumer recordConsumer, int rowId) { recordConsumer.addFloat(columnVector.getFloat(rowId)); } } static class DoubleWriter extends ColumnWriter { DoubleWriter(String name, int fieldId, ColumnVector columnVector) { super(name, fieldId, columnVector); } @Override void writeNonNullRowValue(RecordConsumer recordConsumer, int rowId) { recordConsumer.addDouble(columnVector.getDouble(rowId)); } } static class DecimalIntWriter extends ColumnWriter { private final int scale; DecimalIntWriter(String name, int fieldId, ColumnVector columnVector) { super(name, fieldId, columnVector); this.scale = ((DecimalType) columnVector.getDataType()).getScale(); } @Override void writeNonNullRowValue(RecordConsumer recordConsumer, int rowId) { BigDecimal decimal = columnVector.getDecimal(rowId).movePointRight(scale); recordConsumer.addInteger(decimal.intValue()); } } static class DecimalLongWriter extends ColumnWriter { private final int scale; DecimalLongWriter(String name, int fieldId, ColumnVector columnVector) { super(name, fieldId, columnVector); this.scale = ((DecimalType) columnVector.getDataType()).getScale(); } @Override void writeNonNullRowValue(RecordConsumer recordConsumer, int rowId) { BigDecimal decimal = columnVector.getDecimal(rowId).movePointRight(scale); recordConsumer.addLong(decimal.longValue()); } } static class DecimalFixedBinaryWriter extends ColumnWriter { private final int precision; private final int scale; private final int numBytes; private final byte[] reusedBuffer; DecimalFixedBinaryWriter(String name, int fieldId, ColumnVector columnVector) { super(name, fieldId, columnVector); DecimalType decimalType = (DecimalType) columnVector.getDataType(); this.precision = decimalType.getPrecision(); this.scale = decimalType.getScale(); this.numBytes = MAX_BYTES_PER_PRECISION.get(precision); this.reusedBuffer = new byte[numBytes]; } @Override void writeNonNullRowValue(RecordConsumer recordConsumer, int rowId) { byte[] bytes = columnVector.getDecimal(rowId).unscaledValue().toByteArray(); Binary binary; if (bytes.length == numBytes) { // If the length of the underlying byte array of the unscaled `BigInteger` // happens to be `numBytes`, just reuse it, so that we don't bother // copying it to `reusedBuffer`. binary = Binary.fromReusedByteArray(bytes); } else { // Otherwise, the length must be less than `numBytes`. In this case we copy // contents of the underlying bytes with padding sign bytes to `decimalBuffer` // to form the result fixed-length byte array. byte signByte = (bytes[0] < 0) ? (byte) -1 : (byte) 0; Arrays.fill(reusedBuffer, 0, numBytes - bytes.length, signByte); System.arraycopy(bytes, 0, reusedBuffer, numBytes - bytes.length, bytes.length); binary = Binary.fromReusedByteArray(reusedBuffer); } recordConsumer.addBinary(binary); } } static class DateWriter extends ColumnWriter { DateWriter(String name, int fieldId, ColumnVector columnVector) { super(name, fieldId, columnVector); } @Override void writeNonNullRowValue(RecordConsumer recordConsumer, int rowId) { // TODO: Spark has various handling mode for DateType, need to check if it is needed // for Delta Kernel. recordConsumer.addInteger(columnVector.getInt(rowId)); // dates are stores as epoch days } } /** Writer for both timestamp and timestamp with time zone. */ static class TimestampWriter extends ColumnWriter { TimestampWriter(String name, int fieldId, ColumnVector columnVector) { super(name, fieldId, columnVector); } @Override void writeNonNullRowValue(RecordConsumer recordConsumer, int rowId) { long microsSinceEpochUTC = columnVector.getLong(rowId); recordConsumer.addLong(microsSinceEpochUTC); } } static class StringWriter extends ColumnWriter { StringWriter(String name, int fieldId, ColumnVector columnVector) { super(name, fieldId, columnVector); } @Override void writeNonNullRowValue(RecordConsumer recordConsumer, int rowId) { Binary binary = Binary.fromConstantByteArray( columnVector.getString(rowId).getBytes(StandardCharsets.UTF_8)); recordConsumer.addBinary(binary); } } static class BinaryWriter extends ColumnWriter { BinaryWriter(String name, int fieldId, ColumnVector columnVector) { super(name, fieldId, columnVector); } @Override void writeNonNullRowValue(RecordConsumer recordConsumer, int rowId) { Binary binary = Binary.fromConstantByteArray(columnVector.getBinary(rowId)); recordConsumer.addBinary(binary); } } static class ArrayWriter extends ColumnWriter { ArrayWriter(String name, int fieldId, ColumnVector columnVector) { super(name, fieldId, columnVector); } @Override void writeNonNullRowValue(RecordConsumer recordConsumer, int rowId) { // Write as 3-level representation. Later on, depending upon the config, // we can write either as 2-level or 3-level representation. recordConsumer.startGroup(); ArrayValue arrayValue = columnVector.getArray(rowId); if (arrayValue.getSize() > 0) { // Use the fieldIndex as zero. Once we support Uniform compatible Parquet files, // the field index will come from the Delta schema. recordConsumer.startField("list", 0 /* fieldIndex */); ColumnVector elementVector = arrayValue.getElements(); ColumnWriter elementWriter = createColumnWriter("element", 0 /* fieldIndex */, elementVector); for (int i = 0; i < arrayValue.getSize(); i++) { recordConsumer.startGroup(); if (!elementVector.isNullAt(i)) { elementWriter.writeRowValue(recordConsumer, i); } recordConsumer.endGroup(); } recordConsumer.endField("list", 0 /* fieldIndex */); } recordConsumer.endGroup(); } } static class MapWriter extends ColumnWriter { MapWriter(String name, int fieldId, ColumnVector columnVector) { super(name, fieldId, columnVector); } @Override void writeNonNullRowValue(RecordConsumer recordConsumer, int rowId) { // Write as 3-level representation. Later on, depending upon the config, // we can write either as 2-level or 3-level representation. recordConsumer.startGroup(); MapValue mapValue = columnVector.getMap(rowId); if (mapValue.getSize() > 0) { recordConsumer.startField("key_value", 0 /* fieldIndex */); // Use the fieldIndex as zero. Once we support Uniform compatible Parquet files, // the field index will come from the Delta schema. ColumnVector keyVector = mapValue.getKeys(); ColumnWriter keyWriter = createColumnWriter("key", 0 /* fieldIndex */, keyVector); ColumnVector valueVector = mapValue.getValues(); ColumnWriter valueWriter = createColumnWriter("value", 1 /* fieldIndex */, valueVector); for (int i = 0; i < mapValue.getSize(); i++) { recordConsumer.startGroup(); keyWriter.writeRowValue(recordConsumer, i); if (!valueVector.isNullAt(i)) { valueWriter.writeRowValue(recordConsumer, i); } recordConsumer.endGroup(); } recordConsumer.endField("key_value", 0 /* fieldIndex */); } recordConsumer.endGroup(); } } static class StructWriter extends ColumnWriter { private final ColumnWriter[] fieldWriters; StructWriter(String name, int fieldId, ColumnVector columnVector) { super(name, fieldId, columnVector); fieldWriters = createColumnVectorWriters(columnVector); } @Override void writeNonNullRowValue(RecordConsumer recordConsumer, int rowId) { recordConsumer.startGroup(); for (ColumnWriter fieldWriter : fieldWriters) { fieldWriter.writeRowValue(recordConsumer, rowId); } recordConsumer.endGroup(); } } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetFileReader.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.parquet; import static io.delta.kernel.defaults.internal.parquet.ParquetFilterUtils.toParquetFilter; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.defaults.engine.fileio.FileIO; import io.delta.kernel.defaults.engine.fileio.InputFile; import io.delta.kernel.exceptions.KernelEngineException; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.types.MetadataColumnSpec; import io.delta.kernel.types.StructField; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import java.io.IOException; import java.util.*; import org.apache.hadoop.conf.Configuration; import org.apache.parquet.filter2.compat.FilterCompat; import org.apache.parquet.filter2.predicate.FilterPredicate; import org.apache.parquet.format.converter.ParquetMetadataConverter; import org.apache.parquet.hadoop.ParquetReader; import org.apache.parquet.hadoop.api.InitContext; import org.apache.parquet.hadoop.api.ReadSupport; import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.io.api.GroupConverter; import org.apache.parquet.io.api.RecordMaterializer; import org.apache.parquet.schema.MessageType; public class ParquetFileReader { private final FileIO fileIO; private final int maxBatchSize; public ParquetFileReader(FileIO fileIO) { this.fileIO = requireNonNull(fileIO, "fileIO is null"); this.maxBatchSize = fileIO .getConf("delta.kernel.default.parquet.reader.batch-size") .map(Integer::valueOf) .orElse(1024); checkArgument(maxBatchSize > 0, "invalid Parquet reader batch size: %s", maxBatchSize); } public CloseableIterator read( FileStatus fileStatus, StructType schema, Optional predicate) { if (schema.fields().stream() .filter(StructField::isMetadataColumn) .anyMatch(col -> col.getMetadataColumnSpec() != MetadataColumnSpec.ROW_INDEX)) { throw new IllegalArgumentException( "The parquet reader does not support metadata columns other than ROW_INDEX"); } final boolean hasRowIndexCol = schema.contains(MetadataColumnSpec.ROW_INDEX); return new CloseableIterator() { private final BatchReadSupport readSupport = new BatchReadSupport(maxBatchSize, schema); private ParquetReader reader; private boolean hasNotConsumedNextElement; @Override public void close() throws IOException { Utils.closeCloseables(reader); } @Override public boolean hasNext() { initParquetReaderIfRequired(); try { if (hasNotConsumedNextElement) { return true; } Object next = reader.read(); hasNotConsumedNextElement = next != null; return hasNotConsumedNextElement; } catch (IOException ex) { throw new KernelEngineException( "Error reading Parquet file: " + fileStatus.getPath(), ex); } } @Override public ColumnarBatch next() { if (!hasNotConsumedNextElement) { throw new NoSuchElementException(); } int batchSize = 0; do { hasNotConsumedNextElement = false; // hasNext reads to row to confirm there is a next element. // get the row index only if required by the read schema long rowIndex = hasRowIndexCol ? reader.getCurrentRowIndex() : -1; readSupport.finalizeCurrentRow(rowIndex); batchSize++; } while (batchSize < maxBatchSize && hasNext()); return readSupport.getDataAsColumnarBatch(batchSize); } private void initParquetReaderIfRequired() { if (reader == null) { org.apache.parquet.hadoop.ParquetFileReader fileReader = null; try { InputFile inputFile = fileIO.newInputFile(fileStatus.getPath(), fileStatus.getSize()); // We need physical schema in order to construct a filter that can be // pushed into the `parquet-mr` reader. For that reason read the footer // in advance. org.apache.parquet.io.InputFile parquetInputFile = ParquetIOUtils.createParquetInputFile(inputFile); ParquetMetadata footer = org.apache.parquet.hadoop.ParquetFileReader.readFooter( parquetInputFile, ParquetMetadataConverter.NO_FILTER); MessageType parquetSchema = footer.getFileMetaData().getSchema(); Optional parquetPredicate = predicate.flatMap(predicate -> toParquetFilter(parquetSchema, predicate)); // TODO: We can avoid reading the footer again if we can pass the footer, but there is // no API to do that in the current version of parquet-mr which takes InputFile // as input. reader = new ParquetReader.Builder(parquetInputFile) { @Override protected ReadSupport getReadSupport() { return readSupport; } }.withFilter(parquetPredicate.map(FilterCompat::get).orElse(FilterCompat.NOOP)) // Disable the record level filtering as the `parquet-mr` evaluates // the filter once the entire record has been materialized. Instead, // we use the predicate to prune the row groups which is more efficient. // In the future, we can consider using the record level filtering if a // native Parquet reader is implemented in Kernel default module. .useRecordFilter(false) .useStatsFilter(true) // only enable the row group level filtering .useBloomFilter(false) .useDictionaryFilter(false) .useColumnIndexFilter(false) .build(); } catch (IOException e) { Utils.closeCloseablesSilently(fileReader, reader); throw new KernelEngineException( "Error reading Parquet file: " + fileStatus.getPath(), e); } } } }; } /** * Implement a {@link ReadSupport} that will collect the data for each row and return as a {@link * ColumnarBatch}. */ public static class BatchReadSupport extends ReadSupport { private final int maxBatchSize; private final StructType readSchema; private RowRecordCollector rowRecordCollector; public BatchReadSupport(int maxBatchSize, StructType readSchema) { this.maxBatchSize = maxBatchSize; this.readSchema = requireNonNull(readSchema, "readSchema is not null"); } @Override public ReadContext init(InitContext context) { return new ReadContext(ParquetSchemaUtils.pruneSchema(context.getFileSchema(), readSchema)); } @Override public RecordMaterializer prepareForRead( Configuration configuration, Map keyValueMetaData, MessageType fileSchema, ReadContext readContext) { rowRecordCollector = new RowRecordCollector(maxBatchSize, readSchema, fileSchema); return rowRecordCollector; } public ColumnarBatch getDataAsColumnarBatch(int batchSize) { return rowRecordCollector.getDataAsColumnarBatch(batchSize); } /** @param fileRowIndex the file row index of the row just processed. */ public void finalizeCurrentRow(long fileRowIndex) { rowRecordCollector.finalizeCurrentRow(fileRowIndex); } } /** * Collects the records given by the Parquet reader as columnar data. Parquet reader allows * reading data row by row, but {@link ParquetFileReader} wants to expose the data as a columnar * batch. Parquet reader takes an implementation of {@link RecordMaterializer} to which it gives * data for each column one row at a time. This {@link RecordMaterializer} implementation collects * the column values for multiple rows and returns a {@link ColumnarBatch} at the end. */ public static class RowRecordCollector extends RecordMaterializer { private static final Object FAKE_ROW_RECORD = new Object(); private final RowColumnReader rowRecordGroupConverter; public RowRecordCollector(int maxBatchSize, StructType readSchema, MessageType fileSchema) { this.rowRecordGroupConverter = new RowColumnReader(maxBatchSize, readSchema, fileSchema); } @Override public void skipCurrentRecord() { super.skipCurrentRecord(); } /** * Return a fake object. This is not used by {@link ParquetFileReader}, instead {@link * #getDataAsColumnarBatch}} once a sufficient number of rows are collected. */ @Override public Object getCurrentRecord() { return FAKE_ROW_RECORD; } @Override public GroupConverter getRootConverter() { return rowRecordGroupConverter; } /** Return the data collected so far as a {@link ColumnarBatch}. */ public ColumnarBatch getDataAsColumnarBatch(int batchSize) { return rowRecordGroupConverter.getDataAsColumnarBatch(batchSize); } /** * Finalize the current row. * * @param fileRowIndex the file row index of the row just processed */ public void finalizeCurrentRow(long fileRowIndex) { rowRecordGroupConverter.finalizeCurrentRow(fileRowIndex); } } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetFileWriter.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.parquet; import static io.delta.kernel.defaults.internal.parquet.ParquetIOUtils.createParquetOutputFile; import static io.delta.kernel.defaults.internal.parquet.ParquetStatsReader.readDataFileStatistics; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Collections.emptyMap; import static java.util.Objects.requireNonNull; import static org.apache.parquet.hadoop.ParquetOutputFormat.*; import io.delta.kernel.Meta; import io.delta.kernel.data.*; import io.delta.kernel.defaults.engine.fileio.FileIO; import io.delta.kernel.defaults.engine.fileio.OutputFile; import io.delta.kernel.defaults.internal.parquet.ParquetColumnWriters.ColumnWriter; import io.delta.kernel.expressions.Column; import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.statistics.DataFileStatistics; import io.delta.kernel.types.StructType; import io.delta.kernel.utils.*; import java.io.IOException; import java.io.UncheckedIOException; import java.util.*; import org.apache.hadoop.conf.Configuration; import org.apache.parquet.column.ParquetProperties.WriterVersion; import org.apache.parquet.hadoop.ParquetOutputFormat; import org.apache.parquet.hadoop.ParquetWriter; import org.apache.parquet.hadoop.api.WriteSupport; import org.apache.parquet.hadoop.metadata.CompressionCodecName; import org.apache.parquet.io.api.RecordConsumer; import org.apache.parquet.schema.MessageType; /** * Implements writing data given as {@link FilteredColumnarBatch} to Parquet files. * *

It makes use of the `parquet-mr` library to write the data in Parquet format. The main class * used is {@link ParquetWriter} which is used to write the data row by row to the Parquet file. * Supporting interface for this writer is {@link WriteSupport} (in this writer implementation, it * is {@link BatchWriteSupport}). {@link BatchWriteSupport}, on call back from {@link * ParquetWriter}, reads the contents of {@link ColumnarBatch} and passes the contents to {@link * ParquetWriter} through {@link RecordConsumer}. */ public class ParquetFileWriter { public static final String TARGET_FILE_SIZE_CONF = "delta.kernel.default.parquet.writer.targetMaxFileSize"; public static final long DEFAULT_TARGET_FILE_SIZE = 128 * 1024 * 1024; // 128MB private final FileIO fileIO; private final boolean writeAsSingleFile; private final String location; private final boolean atomicWrite; private final long targetMaxFileSize; private final List statsColumns; private long currentFileNumber; // used to generate the unique file names. /** * Create writer to write data into one or more files depending upon the {@code * delta.kernel.default.parquet.writer.targetMaxFileSize} value and the given data. * * @param fileIO File IO implementation to use for reading and writing files. * @param location Location to write the data. Should be a directory. * @param statsColumns List of columns to collect statistics for. The statistics collection is * optional. */ public static ParquetFileWriter multiFileWriter( FileIO fileIO, String location, List statsColumns) { return new ParquetFileWriter( fileIO, location, /* writeAsSingleFile = */ false, /* atomicWrite = */ false, statsColumns); } /** * Create writer to write the data exactly into one file. * * @param fileIO File IO implementation to use for reading and writing files. * @param location Location to write the data. Shouldn't be a directory. * @param atomicWrite If true, write the file is written atomically (i.e. either the entire * content is written or none, but won't create a file with the partial contents). * @param statsColumns List of columns to collect statistics for. The statistics collection is * optional. */ public static ParquetFileWriter singleFileWriter( FileIO fileIO, String location, boolean atomicWrite, List statsColumns) { return new ParquetFileWriter( fileIO, location, /* writeAsSingleFile = */ true, atomicWrite, statsColumns); } /** * Private constructor to create the writer. Use {@link #multiFileWriter} or {@link * #singleFileWriter} to create the writer. */ private ParquetFileWriter( FileIO fileIO, String location, boolean writeAsSingleFile, boolean atomicWrite, List statsColumns) { this.fileIO = requireNonNull(fileIO, "fileIO is null"); this.writeAsSingleFile = writeAsSingleFile; this.location = requireNonNull(location, "location is null"); this.atomicWrite = atomicWrite; this.statsColumns = requireNonNull(statsColumns, "statsColumns is null"); this.targetMaxFileSize = fileIO.getConf(TARGET_FILE_SIZE_CONF).map(Long::valueOf).orElse(DEFAULT_TARGET_FILE_SIZE); checkArgument(targetMaxFileSize > 0, "Invalid target Parquet file size: %s", targetMaxFileSize); } /** * Write the given data to Parquet files. * * @param dataIter Iterator of data to write. * @return an iterator of {@link DataFileStatus} where each entry contains the metadata of the * data file written. It is the responsibility of the caller to close the iterator. */ public CloseableIterator write( CloseableIterator dataIter) { return new CloseableIterator() { // Last written file output. private Optional lastWrittenFileOutput = Optional.empty(); // Current batch of data that is being written, updated in {@link #hasNextRow()}. private FilteredColumnarBatch currentBatch = null; // Which record in the `currentBatch` is being written, // initialized in {@link #hasNextRow()} and updated in {@link #consumeNextRow}. private int currentBatchCursor = 0; // BatchWriteSupport is initialized when the first batch is read and reused for // subsequent batches with the same schema. `ParquetWriter` can use this write support // to consume data from `ColumnarBatch` and write it to Parquet files. private BatchWriteSupport batchWriteSupport = null; private StructType dataSchema = null; @Override public void close() { Utils.closeCloseables(dataIter); } @Override public boolean hasNext() { if (lastWrittenFileOutput.isPresent()) { return true; } lastWrittenFileOutput = writeNextFile(); return lastWrittenFileOutput.isPresent(); } @Override public DataFileStatus next() { if (!hasNext()) { throw new NoSuchElementException(); } DataFileStatus toReturn = lastWrittenFileOutput.get(); lastWrittenFileOutput = Optional.empty(); return toReturn; } private Optional writeNextFile() { if (!hasNextRow()) { return Optional.empty(); } org.apache.parquet.io.OutputFile parquetOutputFile = createParquetOutputFile(generateNextOutputFile(), atomicWrite); assert batchWriteSupport != null : "batchWriteSupport is not initialized"; long currentFileRowCount = 0; // tracks the number of rows written to the current file try (ParquetWriter writer = createWriter(parquetOutputFile, batchWriteSupport)) { boolean maxFileSizeReached; do { if (consumeNextRow(writer)) { // If the row was written, increment the row count currentFileRowCount++; } // If we are writing a single file, then don't need to check for the current // file size. Otherwise see if the current file size reached the target file // size. maxFileSizeReached = !writeAsSingleFile && writer.getDataSize() >= targetMaxFileSize; // Keep writing until max file is reached or no more data to write } while (!maxFileSizeReached && hasNextRow()); } catch (IOException e) { throw new UncheckedIOException( "Failed to write the Parquet file: " + parquetOutputFile.getPath(), e); } return Optional.of( constructDataFileStatus(parquetOutputFile.getPath(), dataSchema, currentFileRowCount)); } /** * Returns true if there is data to write. * *

Internally it traverses the rows in one batch after the other. Whenever a batch is fully * consumed, moves to the next input batch and updates the column writers in * `batchWriteSupport`. */ boolean hasNextRow() { boolean hasNextRowInCurrentBatch = currentBatch != null && // Is current batch is fully read? currentBatchCursor < currentBatch.getData().getSize(); if (hasNextRowInCurrentBatch) { return true; } // loop until we find a non-empty batch or there are no more batches do { if (!dataIter.hasNext()) { return false; } currentBatch = dataIter.next(); currentBatchCursor = 0; } while (currentBatch.getData().getSize() == 0); // skip empty batches // Initialize the batch support and create writers for each column ColumnarBatch inputBatch = currentBatch.getData(); dataSchema = inputBatch.getSchema(); BatchWriteSupport writeSupport = createOrGetWriteSupport(dataSchema); ColumnWriter[] columnWriters = ParquetColumnWriters.createColumnVectorWriters(inputBatch); writeSupport.setColumnVectorWriters(columnWriters); return true; } /** * Consume the next row of data to write. If the row is selected, write it. Otherwise, skip * it. At the end move the cursor to the next row. * * @return true if the row was written, false if it was skipped */ boolean consumeNextRow(ParquetWriter writer) throws IOException { Optional selectionVector = currentBatch.getSelectionVector(); boolean isRowSelected = !selectionVector.isPresent() || (!selectionVector.get().isNullAt(currentBatchCursor) && selectionVector.get().getBoolean(currentBatchCursor)); if (isRowSelected) { writer.write(currentBatchCursor); } currentBatchCursor++; return isRowSelected; } /** * Create a {@link BatchWriteSupport} if it does not exist or return the existing one for * given schema. */ BatchWriteSupport createOrGetWriteSupport(StructType inputSchema) { if (batchWriteSupport == null) { MessageType parquetSchema = ParquetSchemaUtils.toParquetSchema(inputSchema); batchWriteSupport = new BatchWriteSupport(inputSchema, parquetSchema); return batchWriteSupport; } // Ensure the new input schema matches the one used to create the write support if (!batchWriteSupport.inputSchema.equals(inputSchema)) { throw new IllegalArgumentException( "Input data has columnar batches with " + "different schemas:\n schema 1: " + batchWriteSupport.inputSchema + "\n schema 2: " + inputSchema); } return batchWriteSupport; } }; } /** * Implementation of {@link WriteSupport} to write the {@link ColumnarBatch} to Parquet files. * {@link ParquetWriter} makes use of this interface to consume the data row by row and write to * the Parquet file. Call backs from the {@link ParquetWriter} includes: * *

    *
  • {@link #init(Configuration)}: Called once to init and get {@link WriteContext} which * includes the schema and extra properties. *
  • {@link #prepareForWrite(RecordConsumer)}: Called once to prepare for writing the data. * {@link RecordConsumer} is a way for this batch support to write data for each column in * the current row. *
  • {@link #write(Integer)}: Called for each row to write the data. In this method, column * values are passed to the {@link RecordConsumer} through series of calls. *
*/ private static class BatchWriteSupport extends WriteSupport { final StructType inputSchema; final MessageType parquetSchema; private ColumnWriter[] columnWriters; private RecordConsumer recordConsumer; BatchWriteSupport( StructType inputSchema, // WriteSupport created for this specific schema MessageType parquetSchema) { // Parquet equivalent schema this.inputSchema = requireNonNull(inputSchema, "inputSchema is null"); this.parquetSchema = requireNonNull(parquetSchema, "parquetSchema is null"); } void setColumnVectorWriters(ColumnWriter[] columnWriters) { this.columnWriters = requireNonNull(columnWriters, "columnVectorWriters is null"); } @Override public String getName() { return "delta-kernel-default-parquet-writer"; } @Override public WriteContext init(Configuration configuration) { Map extraProps = Collections.singletonMap( "io.delta.kernel.default-parquet-writer", "Kernel-Defaults-" + Meta.KERNEL_VERSION); return new WriteContext(parquetSchema, extraProps); } @Override public void prepareForWrite(RecordConsumer recordConsumer) { this.recordConsumer = recordConsumer; } @Override public void write(Integer rowId) { // Use java asserts which are disabled in prod to reduce the overhead // and enabled in tests with `-ea` argument. assert (recordConsumer != null) : "Parquet record consumer is null"; assert (columnWriters != null) : "Column writers are not set"; recordConsumer.startMessage(); for (int i = 0; i < columnWriters.length; i++) { columnWriters[i].writeRowValue(recordConsumer, rowId); } recordConsumer.endMessage(); } } /** Generate the next file path to write the data. */ private OutputFile generateNextOutputFile() { if (writeAsSingleFile) { checkArgument(currentFileNumber++ == 0, "expected to write just one file"); return fileIO.newOutputFile(location); } String fileName = String.format("%s-%03d.parquet", UUID.randomUUID(), currentFileNumber++); String filePath = new Path(location, fileName).toString(); return fileIO.newOutputFile(filePath); } /** * Helper method to create {@link ParquetWriter} for given file path and write support. It makes * use of configuration options in `configuration` to configure the writer. Different available * configuration options are defined in {@link ParquetOutputFormat}. */ private ParquetWriter createWriter( org.apache.parquet.io.OutputFile outputFile, WriteSupport writeSupport) throws IOException { ParquetRowDataBuilder rowDataBuilder = new ParquetRowDataBuilder(outputFile, writeSupport); fileIO .getConf(COMPRESSION) .ifPresent( compression -> rowDataBuilder.withCompressionCodec(CompressionCodecName.fromConf(compression))); fileIO.getConf(BLOCK_SIZE).map(Long::parseLong).ifPresent(rowDataBuilder::withRowGroupSize); fileIO.getConf(PAGE_SIZE).map(Integer::parseInt).ifPresent(rowDataBuilder::withPageSize); fileIO .getConf(DICTIONARY_PAGE_SIZE) .map(Integer::parseInt) .ifPresent(rowDataBuilder::withDictionaryPageSize); fileIO .getConf(MAX_PADDING_BYTES) .map(Integer::parseInt) .ifPresent(rowDataBuilder::withMaxPaddingSize); fileIO .getConf(ENABLE_DICTIONARY) .map(Boolean::parseBoolean) .ifPresent(rowDataBuilder::withDictionaryEncoding); fileIO.getConf(VALIDATION).map(Boolean::parseBoolean).ifPresent(rowDataBuilder::withValidation); fileIO .getConf(WRITER_VERSION) .map(WriterVersion::fromString) .ifPresent(rowDataBuilder::withWriterVersion); return rowDataBuilder.build(); } private static class ParquetRowDataBuilder extends ParquetWriter.Builder { private final WriteSupport writeSupport; protected ParquetRowDataBuilder( org.apache.parquet.io.OutputFile outputFile, WriteSupport writeSupport) { super(outputFile); this.writeSupport = requireNonNull(writeSupport, "writeSupport is null"); } @Override protected ParquetRowDataBuilder self() { return this; } @Override protected WriteSupport getWriteSupport(Configuration conf) { return writeSupport; } } /** * Construct the {@link DataFileStatus} for the given file path. It reads the file status and * Parquet footer to compute the statistics for the file. * *

Potential improvement in future to directly compute the statistics while writing the file if * this becomes a sufficiently large part of the write operation time. * * @param path the path of the file * @param dataSchema the schema of the data in the file * @param numRows the number of rows in the file. If no column stats are required, this is used to * construct the {@link DataFileStatistics}. Otherwise, the stats are read from the file. * @return the {@link DataFileStatus} for the file */ private DataFileStatus constructDataFileStatus(String path, StructType dataSchema, long numRows) { try { // Get the FileStatus to figure out the file size and modification time FileStatus fileStatus = fileIO.getFileStatus(path); String resolvedPath = fileIO.resolvePath(path); DataFileStatistics stats; if (statsColumns.isEmpty()) { stats = new DataFileStatistics( numRows, emptyMap() /* minValues */, emptyMap() /* maxValues */, emptyMap() /* nullCount */, Optional.empty() /* tightBounds */); } else { stats = readDataFileStatistics( fileIO.newInputFile(resolvedPath, fileStatus.getSize()), dataSchema, statsColumns); } return new DataFileStatus( resolvedPath, fileStatus.getSize(), fileStatus.getModificationTime(), Optional.ofNullable(stats)); } catch (IOException ioe) { throw new UncheckedIOException("Failed to read the stats for: " + path, ioe); } } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetFilterUtils.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.parquet; import static io.delta.kernel.internal.util.ExpressionUtils.*; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static org.apache.parquet.filter2.predicate.FilterApi.*; import io.delta.kernel.expressions.*; import io.delta.kernel.expressions.Column; import io.delta.kernel.types.*; import java.util.*; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.filter2.compat.FilterCompat.Filter; import org.apache.parquet.filter2.predicate.*; import org.apache.parquet.filter2.predicate.Operators.*; import org.apache.parquet.hadoop.metadata.ColumnPath; import org.apache.parquet.io.api.Binary; import org.apache.parquet.schema.*; import org.apache.parquet.schema.LogicalTypeAnnotation.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Utilities to convert the Kernel {@link Predicate} into `parquet-mr` {@link FilterPredicate}. */ public class ParquetFilterUtils { private static final Logger logger = LoggerFactory.getLogger(ParquetFilterUtils.class); private ParquetFilterUtils() {} /** * Convert the given Kernel predicate {@code kernelPredicate} into `parquet-mr` predicate. * * @param parquetFileSchema Schema of the Parquet file. We need it to find what columns exists in * the Parquet file in order to remove predicates on columns that do not exist in the file. * There is no clear way to handle the predicate on columns that don't exist in the Parquet * file. * @param kernelPredicate Kernel predicate to convert. * @return instance of {@link Filter} (`parquet-mr` filter) */ public static Optional toParquetFilter( MessageType parquetFileSchema, Predicate kernelPredicate) { // Construct a map of field names to field metadata objects Map parquetFieldMap = extractParquetFields(parquetFileSchema); return convertToParquetFilter(parquetFieldMap, kernelPredicate); } private static class ParquetField { final LogicalTypeAnnotation logicalType; final PrimitiveType primitiveType; private ParquetField(LogicalTypeAnnotation logicalType, PrimitiveType primitiveType) { this.logicalType = logicalType; this.primitiveType = primitiveType; } static ParquetField of(LogicalTypeAnnotation logicalType, PrimitiveType primitiveType) { return new ParquetField(logicalType, primitiveType); } } /** * Create a mapping of column to ParquetField for each non-repeated leaf-level column in the given * parquet schema. * * @param parquetSchema Schema of the Parquet file * @return Mapping of column to ParquetField */ private static Map extractParquetFields(MessageType parquetSchema) { Map parquetFields = new HashMap<>(); for (ColumnDescriptor columnDescriptor : parquetSchema.getColumns()) { String[] columnPath = columnDescriptor.getPath(); Type type = parquetSchema.getType(columnPath); if (type.getRepetition() == Type.Repetition.REPEATED) { // `parquet-mr` doesn't support applying filter on a repeated column continue; } assert type.isPrimitive() : "Only primitive types are expected from .getColumns()"; PrimitiveType primitiveType = type.asPrimitiveType(); parquetFields.put( new Column(columnPath), ParquetField.of(type.getLogicalTypeAnnotation(), primitiveType)); } return parquetFields; } private static boolean canUseLiteral(Literal literal, PrimitiveType parquetType) { DataType litType = literal.getDataType(); LogicalTypeAnnotation logicalType = parquetType.getLogicalTypeAnnotation(); switch (parquetType.getPrimitiveTypeName()) { case BOOLEAN: return litType instanceof BooleanType; case INT32: if (!isInteger(literal)) { return false; } return logicalType == null || // no logical type when the type is int32 or int64 (logicalType instanceof IntLogicalTypeAnnotation && ((IntLogicalTypeAnnotation) logicalType).getBitWidth() <= 32) || logicalType instanceof DateLogicalTypeAnnotation; case INT64: if (!isLong(literal)) { return false; } return logicalType == null || // no logical type when the type is int32 or int64 (logicalType instanceof IntLogicalTypeAnnotation && ((IntLogicalTypeAnnotation) logicalType).getBitWidth() <= 64); case FLOAT: return isFloat(literal); case DOUBLE: return isDouble(literal); case BINARY: { return isBinary(literal) && // logical type should be binary (null) or string (logicalType == null || logicalType instanceof StringLogicalTypeAnnotation); } default: return false; } } private static Optional convertToParquetFilter( Map parquetFieldMap, Predicate deltaPredicate) { String name = deltaPredicate.getName().toLowerCase(Locale.ROOT); switch (name) { case "=": case "<": case "<=": case ">": case ">=": return convertComparatorToParquetFilter(parquetFieldMap, deltaPredicate); case "not": return convertNotToParquetFilter(parquetFieldMap, deltaPredicate); case "and": return convertAndToParquetFilter(parquetFieldMap, deltaPredicate); case "or": return convertOrToParquetFilter(parquetFieldMap, deltaPredicate); case "is_null": return convertIsNullIsNotNull(parquetFieldMap, deltaPredicate, false /* isNotNull */); case "is_not_null": return convertIsNullIsNotNull(parquetFieldMap, deltaPredicate, true /* isNotNull */); default: return visitUnsupported(deltaPredicate, name + " is not a supported predicate."); } } private static Optional convertComparatorToParquetFilter( Map parquetFieldMap, Predicate deltaPredicate) { Expression child0 = getLeft(deltaPredicate); Expression child1 = getRight(deltaPredicate); if (child0 instanceof Literal && child1 instanceof Column) { Expression temp = child0; child0 = child1; child1 = temp; } if (!(child0 instanceof Column) || !(child1 instanceof Literal)) { return visitUnsupported( deltaPredicate, "Comparison predicate must have a column and a literal."); } Column column = (Column) child0; Literal literal = (Literal) child1; ParquetField parquetField = parquetFieldMap.get(column); if (parquetField == null) { return visitUnsupported( deltaPredicate, "Column used in predicate does not exist in the parquet file."); } if (literal.getValue() == null) { return visitUnsupported( deltaPredicate, "Literal value is null for a comparator operator. Comparator is not " + "supported for null values as the Parquet comparator is not null safe"); } if (!canUseLiteral(literal, parquetField.primitiveType)) { return visitUnsupported( deltaPredicate, "Literal type is not compatible with the column type: " + literal.getDataType()); } PrimitiveType parquetType = parquetField.primitiveType; String columnPath = ColumnPath.get(column.getNames()).toDotString(); String comparator = deltaPredicate.getName(); switch (parquetType.getPrimitiveTypeName()) { case BOOLEAN: BooleanColumn booleanColumn = booleanColumn(columnPath); if ("=".equals(comparator)) { // Only = is supported for boolean return Optional.of(FilterApi.eq(booleanColumn, getBoolean(literal))); } break; case INT32: IntColumn intColumn = intColumn(columnPath); switch (comparator) { case "=": return Optional.of(FilterApi.eq(intColumn, getInt(literal))); case "<": return Optional.of(FilterApi.lt(intColumn, getInt(literal))); case "<=": return Optional.of(FilterApi.ltEq(intColumn, getInt(literal))); case ">": return Optional.of(FilterApi.gt(intColumn, getInt(literal))); case ">=": return Optional.of(FilterApi.gtEq(intColumn, getInt(literal))); } break; case INT64: LongColumn longColumn = longColumn(columnPath); switch (comparator) { case "=": return Optional.of(FilterApi.eq(longColumn, getLong(literal))); case "<": return Optional.of(FilterApi.lt(longColumn, getLong(literal))); case "<=": return Optional.of(FilterApi.ltEq(longColumn, getLong(literal))); case ">": return Optional.of(FilterApi.gt(longColumn, getLong(literal))); case ">=": return Optional.of(FilterApi.gtEq(longColumn, getLong(literal))); } break; case FLOAT: FloatColumn floatColumn = floatColumn(columnPath); switch (comparator) { case "=": return Optional.of(FilterApi.eq(floatColumn, getFloat(literal))); case "<": return Optional.of(FilterApi.lt(floatColumn, getFloat(literal))); case "<=": return Optional.of(FilterApi.ltEq(floatColumn, getFloat(literal))); case ">": return Optional.of(FilterApi.gt(floatColumn, getFloat(literal))); case ">=": return Optional.of(FilterApi.gtEq(floatColumn, getFloat(literal))); } break; case DOUBLE: DoubleColumn doubleColumn = doubleColumn(columnPath); switch (comparator) { case "=": return Optional.of(FilterApi.eq(doubleColumn, getDouble(literal))); case "<": return Optional.of(FilterApi.lt(doubleColumn, getDouble(literal))); case "<=": return Optional.of(FilterApi.ltEq(doubleColumn, getDouble(literal))); case ">": return Optional.of(FilterApi.gt(doubleColumn, getDouble(literal))); case ">=": return Optional.of(FilterApi.gtEq(doubleColumn, getDouble(literal))); } break; case BINARY: BinaryColumn binaryColumn = binaryColumn(columnPath); Binary binary = getBinary(literal); switch (comparator) { case "=": return Optional.of(FilterApi.eq(binaryColumn, binary)); case "<": return Optional.of(FilterApi.lt(binaryColumn, binary)); case "<=": return Optional.of(FilterApi.ltEq(binaryColumn, binary)); case ">": return Optional.of(FilterApi.gt(binaryColumn, binary)); case ">=": return Optional.of(FilterApi.gtEq(binaryColumn, binary)); } break; } return visitUnsupported( deltaPredicate, String.format( "Unsupported column type (%s) with comparator (%s): ", parquetType, comparator)); } private static Optional convertNotToParquetFilter( Map parquetFieldMap, Predicate deltaPredicate) { Optional childFilter = convertToParquetFilter(parquetFieldMap, (Predicate) getUnaryChild(deltaPredicate)); return childFilter.map(FilterApi::not); } private static Optional convertOrToParquetFilter( Map parquetFieldMap, Predicate deltaPredicate) { Optional leftFilter = convertToParquetFilter(parquetFieldMap, asPredicate(getLeft(deltaPredicate))); Optional rightFilter = convertToParquetFilter(parquetFieldMap, asPredicate(getRight(deltaPredicate))); if (leftFilter.isPresent() && rightFilter.isPresent()) { return Optional.of(FilterApi.or(leftFilter.get(), rightFilter.get())); } return Optional.empty(); } private static Optional convertAndToParquetFilter( Map parquetFieldMap, Predicate deltaPredicate) { Optional leftFilter = convertToParquetFilter(parquetFieldMap, asPredicate(getLeft(deltaPredicate))); Optional rightFilter = convertToParquetFilter(parquetFieldMap, asPredicate(getRight(deltaPredicate))); if (leftFilter.isPresent() && rightFilter.isPresent()) { return Optional.of(FilterApi.and(leftFilter.get(), rightFilter.get())); } if (leftFilter.isPresent()) { return leftFilter; } return rightFilter; } private static Optional convertIsNullIsNotNull( Map parquetFieldMap, Predicate deltaPredicate, boolean isNotNull) { Expression child = getUnaryChild(deltaPredicate); if (!(child instanceof Column)) { return visitUnsupported(deltaPredicate, "IS NULL predicate must have a column input."); } Column column = (Column) child; ParquetField parquetField = parquetFieldMap.get(column); if (parquetField == null) { return visitUnsupported( deltaPredicate, "Column used in predicate does not exist in the parquet file."); } String columnPath = ColumnPath.get(column.getNames()).toDotString(); // Parquet filter keeps records if their value is equal to the provided value. // Nulls are treated the same way the java programming language does. // For example: eq(column, null) will keep all records whose value is null. eq(column, 7) // will keep all records whose value is 7, and will drop records whose value is null // NOTE: this is different from how some query languages handle null. switch (parquetField.primitiveType.getPrimitiveTypeName()) { case BOOLEAN: return createIsNullOrIsNotNullPredicate(booleanColumn(columnPath), isNotNull); case INT32: return createIsNullOrIsNotNullPredicate(intColumn(columnPath), isNotNull); case INT64: return createIsNullOrIsNotNullPredicate(longColumn(columnPath), isNotNull); case FLOAT: return createIsNullOrIsNotNullPredicate(floatColumn(columnPath), isNotNull); case DOUBLE: return createIsNullOrIsNotNullPredicate(doubleColumn(columnPath), isNotNull); case BINARY: return createIsNullOrIsNotNullPredicate(binaryColumn(columnPath), isNotNull); default: return visitUnsupported( deltaPredicate, "Unsupported column type: " + parquetField.primitiveType); } } private static , C extends Operators.Column & SupportsEqNotEq> Optional createIsNullOrIsNotNullPredicate(C column, boolean isNotNull) { return Optional.of(isNotNull ? FilterApi.notEq(column, null) : FilterApi.eq(column, null)); } private static Optional visitUnsupported(Predicate predicate, String message) { logger.info("Unsupported predicate: {}. Reason: {}", predicate, message); // Filtering is a best effort. If an unsupported predicate expression is received, // do not consider it for filtering. return Optional.empty(); } private static boolean isBoolean(Literal literal) { return literal.getDataType() instanceof BooleanType; } private static boolean getBoolean(Literal literal) { checkArgument(isBoolean(literal), "Literal is not a boolean: %s", literal); return (boolean) literal.getValue(); } private static boolean isInteger(Literal literal) { DataType dataType = literal.getDataType(); if (dataType instanceof LongType) { // Check if the long value can be represented as an integer return ((Long) literal.getValue()).intValue() == (Long) literal.getValue(); } return dataType instanceof ByteType || dataType instanceof ShortType || dataType instanceof IntegerType || dataType instanceof DateType; } private static int getInt(Literal literal) { checkArgument(isInteger(literal), "Literal is not an integer: %s", literal); DataType dataType = literal.getDataType(); if (dataType instanceof LongType) { return ((Long) literal.getValue()).intValue(); } return ((Number) literal.getValue()).intValue(); } private static boolean isLong(Literal literal) { DataType dataType = literal.getDataType(); return dataType instanceof LongType || dataType instanceof ByteType || dataType instanceof ShortType || dataType instanceof IntegerType || dataType instanceof DateType; } private static long getLong(Literal literal) { checkArgument(isLong(literal), "Literal is not a long: %s", literal); DataType dataType = literal.getDataType(); if (dataType instanceof LongType) { return (long) literal.getValue(); } return ((Number) literal.getValue()).longValue(); } private static boolean isFloat(Literal literal) { return literal.getDataType() instanceof FloatType; } private static float getFloat(Literal literal) { checkArgument(isFloat(literal), "Literal is not a float: %s", literal); return ((Number) literal.getValue()).floatValue(); } private static boolean isDouble(Literal literal) { return literal.getDataType() instanceof DoubleType; } private static double getDouble(Literal literal) { checkArgument(isDouble(literal), "Literal is not a double: %s", literal); return ((Number) literal.getValue()).doubleValue(); } private static boolean isBinary(Literal literal) { DataType type = literal.getDataType(); return type instanceof BinaryType || type instanceof StringType; } private static Binary getBinary(Literal literal) { checkArgument(isBinary(literal), "Literal is not a binary: %s", literal); DataType type = literal.getDataType(); if (type instanceof BinaryType) { return Binary.fromConstantByteArray((byte[]) literal.getValue()); } return Binary.fromString((String) literal.getValue()); } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetIOUtils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.parquet; import io.delta.kernel.defaults.engine.fileio.InputFile; import io.delta.kernel.defaults.engine.fileio.OutputFile; import io.delta.kernel.defaults.engine.fileio.PositionOutputStream; import io.delta.kernel.defaults.engine.fileio.SeekableInputStream; import java.io.IOException; import org.apache.parquet.io.DelegatingPositionOutputStream; import org.apache.parquet.io.DelegatingSeekableInputStream; /** * Utilities related to Parquet I/O. These utilities bridge the gap between Kernel's {@link * io.delta.kernel.defaults.engine.fileio.FileIO} and the Parquet I/O classes. */ public class ParquetIOUtils { private ParquetIOUtils() {} /** Create a Parquet {@link org.apache.parquet.io.InputFile} from a Kernel's {@link InputFile}. */ static org.apache.parquet.io.InputFile createParquetInputFile(InputFile inputFile) { return new org.apache.parquet.io.InputFile() { @Override public long getLength() throws IOException { return inputFile.length(); } @Override public org.apache.parquet.io.SeekableInputStream newStream() throws IOException { SeekableInputStream seekableStream = inputFile.newStream(); return new DelegatingSeekableInputStream(seekableStream) { @Override public void seek(long newPos) throws IOException { seekableStream.seek(newPos); } @Override public long getPos() throws IOException { return seekableStream.getPos(); } }; } }; } /** * Create a Parquet {@link org.apache.parquet.io.OutputFile} from a Kernel's {@link OutputFile}. */ static org.apache.parquet.io.OutputFile createParquetOutputFile( OutputFile kernelOutputFile, boolean atomicWrite) { return new org.apache.parquet.io.OutputFile() { @Override public org.apache.parquet.io.PositionOutputStream create(long blockSizeHint) throws IOException { // blockSizeHint is hint used in HDFS compliant file systems. In cloud storage systems // it is irrelevant. So, we ignore it. PositionOutputStream posOutputStream = kernelOutputFile.create(atomicWrite); return new DelegatingPositionOutputStream(posOutputStream) { @Override public long getPos() throws IOException { return posOutputStream.getPos(); } }; } @Override public org.apache.parquet.io.PositionOutputStream createOrOverwrite(long blockSizeHint) throws IOException { // In Kernel we never overwrite files, so this method is not used. throw new UnsupportedOperationException("createOrOverwrite is not supported in Kernel"); } @Override public boolean supportsBlockSize() { return false; } @Override public long defaultBlockSize() { // blockSizeHint is hint used in HDFS compliant file systems. In cloud storage systems // it is irrelevant. So, return some default value. return 128 * 1024 * 1024; // 128MB } @Override public String getPath() { return kernelOutputFile.path(); } }; } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetSchemaUtils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.parquet; import static io.delta.kernel.internal.util.ColumnMapping.PARQUET_FIELD_NESTED_IDS_METADATA_KEY; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.lang.String.format; import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.MICROS; import static org.apache.parquet.schema.LogicalTypeAnnotation.decimalType; import static org.apache.parquet.schema.LogicalTypeAnnotation.timestampType; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BOOLEAN; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64; import static org.apache.parquet.schema.Type.Repetition.OPTIONAL; import static org.apache.parquet.schema.Type.Repetition.REQUIRED; import static org.apache.parquet.schema.Types.primitive; import io.delta.kernel.internal.util.ColumnMapping; import io.delta.kernel.types.*; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiFunction; import java.util.stream.Collectors; import org.apache.parquet.schema.*; import org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation; import org.apache.parquet.schema.Type.Repetition; /** Utility methods for Delta schema to Parquet schema conversion. */ class ParquetSchemaUtils { /** * Constants that help if a Decimal type can be stored as INT32 or INT64 based on the precision. * The maximum precision that can be stored in INT32 is 9 and in INT64 is 18. If the precision * exceeds these values, then the DecimalType is stored as FIXED_LEN_BYTE_ARRAY. */ public static final int DECIMAL_MAX_DIGITS_IN_INT = 9; public static final int DECIMAL_MAX_DIGITS_IN_LONG = 18; /** * Maximum number of bytes required to store a decimal of a given precision as * FIXED_LEN_BYTE_ARRAY in Parquet. */ public static final List MAX_BYTES_PER_PRECISION; static { List maxBytesPerPrecision = new ArrayList<>(); for (int i = 0; i <= 38; i++) { int numBytes = 1; while (Math.pow(2.0, 8 * numBytes - 1) < Math.pow(10.0, i)) { numBytes += 1; } maxBytesPerPrecision.add(numBytes); } MAX_BYTES_PER_PRECISION = Collections.unmodifiableList(maxBytesPerPrecision); } private ParquetSchemaUtils() {} /** * Given the file schema in Parquet file and selected columns by Delta, return a subschema of the * file schema. * * @param fileSchema * @param deltaType * @return */ static MessageType pruneSchema( GroupType fileSchema /* parquet */, StructType deltaType /* delta-kernel */) { return new MessageType("fileSchema", pruneFields(fileSchema, deltaType)); } /** * Search for the Parquet type in {@code groupType} of subfield which is equivalent to given * {@code field}. * * @param groupType Parquet group type coming from the file schema. * @param field Sub field given as Delta Kernel's {@link StructField} * @return {@link Type} of the Parquet field. Returns {@code null}, if not found. */ static Type findSubFieldType( GroupType groupType, StructField field, Map parquetFieldIdToTypeMap) { // First search by the field id. If not found, search by case-sensitive name. Finally // by the case-insensitive name. // For Delta readers, no need to use the nested field ids added as part of icebergCompatV2 Optional fieldId = getFieldId(field.getMetadata()); if (fieldId.isPresent()) { Type subType = parquetFieldIdToTypeMap.get(fieldId.get()); if (subType != null) { return subType; } } final String columnName = field.getName(); if (groupType.containsField(columnName)) { return groupType.getType(columnName); } // Parquet is case-sensitive, but the engine that generated the parquet file may not be. // Check for direct match above but if no match found, try case-insensitive match. for (Type type : groupType.getFields()) { if (type.getName().equalsIgnoreCase(columnName)) { return type; } } return null; } /** Returns a map from field id to Parquet type for fields that have the field id set. */ static Map getParquetFieldToTypeMap(GroupType parquetGroupType) { // Generate the field id to Parquet type map only if the read schema has field ids. return parquetGroupType.getFields().stream() .filter(subFieldType -> subFieldType.getId() != null) .collect( Collectors.toMap( subFieldType -> subFieldType.getId().intValue(), subFieldType -> subFieldType, (u, v) -> { throw new IllegalStateException( format( "Parquet file contains multiple columns " + "(%s, %s) with the same field id", u, v)); })); } /** * Convert the given Kernel schema to Parquet's schema * * @param structType Kernel schema object * @return {@link MessageType} representing the schema in Parquet format. */ static MessageType toParquetSchema(StructType structType) { BiFunction, Boolean, Optional> fieldIdValidator = createFieldIdValidator(structType); List types = new ArrayList<>(); for (StructField structField : structType.fields()) { Optional fieldId = fieldIdValidator.apply( getFieldId(structField.getMetadata()), false /* isNestedFieldId */); types.add( toParquetType( structField /* nearestAncestor with struct field */, structField.getName() /* relativePath to nearestAncestor */, structField.getDataType(), structField.getName(), structField.isNullable() ? OPTIONAL : REQUIRED, fieldId, fieldIdValidator)); } return new MessageType("DefaultKernelSchema", types); } private static List pruneFields(GroupType type, StructType deltaDataType) { // prune fields including nested pruning like in pruneSchema final Map parquetFieldIdToTypeMap = getParquetFieldToTypeMap(type); return deltaDataType.fields().stream() .map( column -> { Type subType = findSubFieldType(type, column, parquetFieldIdToTypeMap); if (subType != null) { return prunedType(subType, column.getDataType()); } else { return null; } }) .filter(Objects::nonNull) .collect(Collectors.toList()); } private static Type prunedType(Type type, DataType deltaType) { if (type instanceof GroupType && deltaType instanceof StructType) { GroupType groupType = (GroupType) type; StructType structType = (StructType) deltaType; return groupType.withNewFields(pruneFields(groupType, structType)); } else if (type instanceof GroupType && deltaType instanceof ArrayType) { GroupType arrayGroupType = (GroupType) type; ArrayType deltaArrayType = (ArrayType) deltaType; // For standard 3-level arrays, extract the element type and recursively prune it Type elementType = validateAndGetThreeLevelParquetArrayElementType(arrayGroupType); Type listField = arrayGroupType.getType(0); GroupType listGroup = (GroupType) listField; GroupType newListGroup = listGroup.withNewFields( Collections.singletonList(prunedType(elementType, deltaArrayType.getElementType()))); return arrayGroupType.withNewFields(Collections.singletonList(newListGroup)); // TODO: check if we need to fix map type. } else { return type; } } /** * Validates and extracts the element type from a 3-level Parquet array type. * *

According to Parquet specification, all arrays (including primitive arrays) should use the * standard 3-level structure: * * 

   * optional group array_field (LIST) {
   *   repeated group list {
   *     optional  element;
   *   }
   * }
   *
* * Examples: - For array of int: * * 
   * optional group array_field (LIST) {
   *   repeated group list {
   *     optional int32 element;
   *   }
   * }
   *
* * - For array of struct{x:int, y:string};: * * 
   * optional group array_field (LIST) {
   *   repeated group list {
   *     optional group element {
   *       optional int32 x;
   *       optional binary y (STRING);
   *     }
   *   }
   * }
   *
* * @param arrayGroupType The group type of the array, which must be a LIST. * @return The Parquet type of the array element. * @throws IllegalArgumentException if the structure doesn't follow 3-level format. */ public static Type validateAndGetThreeLevelParquetArrayElementType(GroupType arrayGroupType) { checkArgument( arrayGroupType.getFieldCount() == 1, "In Parquet's 3-level structure, group type must only contain one sub field: %s", arrayGroupType); Type listField = arrayGroupType.getType(0); checkArgument( listField.isRepetition(Type.Repetition.REPEATED), "Array list field must be repeated: %s", listField); // Ensure this is a proper 3-level structure checkArgument( listField instanceof GroupType, "Expected 3-level array structure with repeated group, but got: %s", listField); // array_field.list GroupType listGroup = (GroupType) listField; checkArgument( listGroup.getFieldCount() == 1, "Array list group must have exactly one element: %s", listGroup); // array_field.list.element return listGroup.getType(0); } /** * Converts a Delta type {@code dataType} to a Parquet type. * * @param nearestAncestor The nearest ancestor with a {@link StructField}. This ancestor * represents the current node or nearest node on the path to the current node from root of * the schema. * @param relativePath The relative path to this element from {@code nearestAncestor}. For * example, consider a column type {@code col1 STRUCT(a INT, b STRUCT(c INT, d ARRAY(INT)))}. * The absolute path to the nested {@code element} field of the list is col1.b.d.element, * while the relative path is d.element, i.e., relative to the nearest ancestor with a struct * field. * @param dataType The Delta type to be converted to a Parquet type. * @param name The name of the field. * @param repetition The {@link Repetition} of the field. * @param fieldId The field ID of the field. If present, the field ID is added to the Parquet * type. * @return The Parquet type representing the given Delta type. */ private static Type toParquetType( StructField nearestAncestor, String relativePath, DataType dataType, String name, Repetition repetition, Optional fieldId, BiFunction, Boolean, Optional> fieldIdValidator) { Type type; if (dataType instanceof BooleanType) { type = primitive(BOOLEAN, repetition).named(name); } else if (dataType instanceof ByteType || dataType instanceof ShortType || dataType instanceof IntegerType) { type = primitive(INT32, repetition).named(name); } else if (dataType instanceof LongType) { type = primitive(INT64, repetition).named(name); } else if (dataType instanceof FloatType) { type = primitive(FLOAT, repetition).named(name); } else if (dataType instanceof DoubleType) { type = primitive(DOUBLE, repetition).named(name); } else if (dataType instanceof DecimalType) { DecimalType decimalType = (DecimalType) dataType; int precision = decimalType.getPrecision(); int scale = decimalType.getScale(); // DecimalType constructor already has checks to make sure the precision and scale are // within the valid range. No need to check them again. DecimalLogicalTypeAnnotation decimalAnnotation = decimalType(scale, precision); if (precision <= DECIMAL_MAX_DIGITS_IN_INT) { type = primitive(INT32, repetition).as(decimalAnnotation).named(name); } else if (precision <= DECIMAL_MAX_DIGITS_IN_LONG) { type = primitive(INT64, repetition).as(decimalAnnotation).named(name); } else { type = primitive(FIXED_LEN_BYTE_ARRAY, repetition) .as(decimalAnnotation) .length(MAX_BYTES_PER_PRECISION.get(precision)) .named(name); } } else if (dataType instanceof StringType) { type = primitive(BINARY, repetition).as(LogicalTypeAnnotation.stringType()).named(name); } else if (dataType instanceof BinaryType) { type = primitive(BINARY, repetition).named(name); } else if (dataType instanceof DateType) { type = primitive(INT32, repetition).as(LogicalTypeAnnotation.dateType()).named(name); } else if (dataType instanceof TimestampType) { // Kernel is by default going to write as INT64 with isAdjustedToUTC set to true // Delta-Spark writes as INT96 for legacy reasons (maintaining compatibility with // unknown consumers with very, very old versions of Parquet reader). Kernel is a new // project, and we are ok if it breaks readers (we use this opportunity to find such // readers and ask them to upgrade). type = primitive(INT64, repetition) .as(timestampType(true /* isAdjustedToUTC */, MICROS)) .named(name); } else if (dataType instanceof TimestampNTZType) { // Write as INT64 with isAdjustedToUTC set to false type = primitive(INT64, repetition) .as(timestampType(false /* isAdjustedToUTC */, MICROS)) .named(name); } else if (dataType instanceof ArrayType) { type = toParquetArrayType( nearestAncestor, relativePath, (ArrayType) dataType, name, repetition, fieldIdValidator); } else if (dataType instanceof MapType) { type = toParquetMapType( nearestAncestor, relativePath, (MapType) dataType, name, repetition, fieldIdValidator); } else if (dataType instanceof StructType) { type = toParquetStructType((StructType) dataType, name, repetition, fieldIdValidator); } else { throw new UnsupportedOperationException( "Writing given type data to Parquet is not supported: " + dataType); } if (fieldId.isPresent()) { // Add field id to the type. type = type.withId(fieldId.get()); } return type; } private static Type toParquetArrayType( StructField nearestAncestor, String relativePath, ArrayType arrayType, String name, Repetition rep, BiFunction, Boolean, Optional> fieldIdValidator) { // We will be supporting the 3-level array structure only. 2-level array structures are // a very old legacy versions of Parquet which Kernel doesn't support writing as. String elementRelativePath = relativePath + ".element"; Optional fieldId = fieldIdValidator.apply( getNestedFieldId(nearestAncestor, elementRelativePath), true /* isNestedFieldId */); return Types.buildGroup(rep) .as(LogicalTypeAnnotation.listType()) .addField( Types.repeatedGroup() .addField( toParquetType( nearestAncestor, elementRelativePath, arrayType.getElementType(), "element", /* name */ arrayType.containsNull() ? OPTIONAL : REQUIRED, fieldId, fieldIdValidator)) .named("list")) .named(name); } private static Type toParquetMapType( StructField nearestAncestor, String relativePath, MapType mapType, String name, Repetition repetition, BiFunction, Boolean, Optional> fieldIdValidator) { // We will be supporting the 3-level map structure only. 2-level map structures are // a very old legacy versions of Parquet which Kernel doesn't support writing as. String keyRelativePath = relativePath + ".key"; String valueRelativePath = relativePath + ".value"; Optional keyFieldId = fieldIdValidator.apply( getNestedFieldId(nearestAncestor, keyRelativePath), true /* isNestedFieldId */); Optional valueFieldId = fieldIdValidator.apply( getNestedFieldId(nearestAncestor, valueRelativePath), true /* isNestedFieldId */); return Types.buildGroup(repetition) .as(LogicalTypeAnnotation.mapType()) .addField( Types.repeatedGroup() .addField( toParquetType( nearestAncestor, keyRelativePath, mapType.getKeyType(), "key", /* name */ REQUIRED, /* repetition */ keyFieldId, fieldIdValidator)) .addField( toParquetType( nearestAncestor, valueRelativePath, mapType.getValueType(), "value", /* name */ mapType.isValueContainsNull() ? OPTIONAL : REQUIRED, valueFieldId, fieldIdValidator)) .named("key_value")) .named(name); } private static Type toParquetStructType( StructType structType, String name, Repetition repetition, BiFunction, Boolean, Optional> fieldIdValidator) { List fields = new ArrayList<>(); for (StructField field : structType.fields()) { Optional fieldId = fieldIdValidator.apply(getFieldId(field.getMetadata()), false /* isNestedFieldId */); fields.add( toParquetType( field, /* nearestAncestor with struct field */ field.getName(), /* relativePath to nearestAncestor */ field.getDataType(), field.getName(), field.isNullable() ? OPTIONAL : REQUIRED, fieldId, fieldIdValidator)); } return new GroupType(repetition, name, fields); } private static Optional getFieldId(FieldMetadata fieldMetadata) { return getFieldId(fieldMetadata, ColumnMapping.PARQUET_FIELD_ID_KEY); } private static Optional getNestedFieldId( StructField field, String nestedFieldRelativePath) { FieldMetadata nestedFieldIDMetadata = field.getMetadata().getMetadata(PARQUET_FIELD_NESTED_IDS_METADATA_KEY); if (nestedFieldIDMetadata != null) { return getFieldId(nestedFieldIDMetadata, nestedFieldRelativePath); } return Optional.empty(); } private static Optional getFieldId(FieldMetadata fieldMetadata, String fieldIdKey) { // Field id delta schema metadata is deserialized as long, but the range should always // be within integer range. return Optional.ofNullable(fieldMetadata.getLong(fieldIdKey)).map(Math::toIntExact); } /** * Validator for checking that the field ids in the schema are valid. Any visitor that recursively * visits the fields in the schema {@link StructType} can call this validator to check: * *
    *
  • Field ids should be unique within the schema. *
  • Field ids should be non-negative. *
  • All {@link StructField} should have a field id or none should have it *
  • All nested elements of {@link ArrayType} and {@link MapType} should have the nested field * id or none of them should have. *
* * @param structType The schema to validate. Used for error messages. * @return A lambda that can be used to validate the field ids. It takes the field id and a * boolean (true for nested field ids). Since the lamda is stateful, it should be used only * once for each unique schema and traversal. */ private static BiFunction, Boolean, Optional> createFieldIdValidator( StructType structType) { Set fieldIds = new HashSet<>(); // include nested field ids too. AtomicBoolean seenFieldWithNoId = new AtomicBoolean(); AtomicBoolean seenFieldWithId = new AtomicBoolean(); AtomicBoolean seenNestedFieldWithId = new AtomicBoolean(); AtomicBoolean seemNestedWithNoId = new AtomicBoolean(); return (fieldIdOpt, isNestedFieldId) -> { if (fieldIdOpt.isPresent()) { checkArgument(fieldIdOpt.get() >= 0, "Field id should be non-negative.\n%s", structType); checkArgument(fieldIds.add(fieldIdOpt.get()), "Field id should be unique.\n%s", structType); if (isNestedFieldId) { seenNestedFieldWithId.set(true); checkArgument( !seemNestedWithNoId.get() && !seenFieldWithNoId.get(), "Some of the fields are missing field ids.\n%s", structType); } else { seenFieldWithId.set(true); checkArgument( !seenFieldWithNoId.get(), "Some of the fields are missing field ids.\n%s", structType); } } else { if (isNestedFieldId) { seemNestedWithNoId.set(true); checkArgument( !seenNestedFieldWithId.get() && !seenFieldWithId.get(), "Some of the fields are missing field ids.\n%s", structType); } else { seenFieldWithNoId.set(true); checkArgument( !seenFieldWithId.get(), "Some of the fields are missing field ids.\n%s", structType); } } return fieldIdOpt; }; } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetStatsReader.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.parquet; import static io.delta.kernel.defaults.internal.DefaultKernelUtils.getDataType; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.function.UnaryOperator.identity; import static org.apache.hadoop.shaded.com.google.common.collect.ImmutableMap.toImmutableMap; import io.delta.kernel.defaults.engine.fileio.InputFile; import io.delta.kernel.expressions.Column; import io.delta.kernel.expressions.Literal; import io.delta.kernel.statistics.DataFileStatistics; import io.delta.kernel.types.*; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.util.*; import org.apache.hadoop.shaded.com.google.common.collect.ImmutableMultimap; import org.apache.hadoop.shaded.com.google.common.collect.Multimap; import org.apache.parquet.column.statistics.*; import org.apache.parquet.format.converter.ParquetMetadataConverter; import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.hadoop.metadata.BlockMetaData; import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation; /** Helper class to read statistics from Parquet files. */ public class ParquetStatsReader { /** * Read the statistics for the given Parquet file. * * @param kernelInputFile {@link InputFile} representing the Parquet file. * @param dataSchema The schema of the Parquet file. Type info is used to decode statistics. * @param statsColumns The columns for which statistics should be collected and returned. * @return File/column level statistics as {@link DataFileStatistics} instance. */ public static DataFileStatistics readDataFileStatistics( InputFile kernelInputFile, StructType dataSchema, List statsColumns) throws IOException { // Read the Parquet footer to compute the statistics org.apache.parquet.io.InputFile parquetFile = ParquetIOUtils.createParquetInputFile(kernelInputFile); ParquetMetadata footer = ParquetFileReader.readFooter(parquetFile, ParquetMetadataConverter.NO_FILTER); ImmutableMultimap.Builder metadataForColumn = ImmutableMultimap.builder(); long rowCount = 0; for (BlockMetaData blockMetaData : footer.getBlocks()) { rowCount += blockMetaData.getRowCount(); for (ColumnChunkMetaData columnChunkMetaData : blockMetaData.getColumns()) { Column column = new Column(columnChunkMetaData.getPath().toArray()); metadataForColumn.put(column, columnChunkMetaData); } } return constructFileStats(metadataForColumn.build(), dataSchema, statsColumns, rowCount); } /** * Merge statistics from multiple rowgroups into a single set of statistics for each column. * * @return Stats for each column in the file as {@link DataFileStatistics}. */ private static DataFileStatistics constructFileStats( Multimap metadataForColumn, StructType dataSchema, List statsColumns, long rowCount) { Map>> statsForColumn = metadataForColumn.keySet().stream() .collect( toImmutableMap(identity(), key -> mergeMetadataList(metadataForColumn.get(key)))); Map minValues = new HashMap<>(); Map maxValues = new HashMap<>(); Map nullCounts = new HashMap<>(); for (Column statsColumn : statsColumns) { Optional> stats = statsForColumn.get(statsColumn); DataType columnType = getDataType(dataSchema, statsColumn); if (stats == null || !stats.isPresent() || !isStatsSupportedDataType(columnType)) { continue; } Statistics statistics = stats.get(); Long numNulls = statistics.isNumNullsSet() ? statistics.getNumNulls() : null; nullCounts.put(statsColumn, numNulls); if (numNulls != null && rowCount == numNulls) { // If all values are null, then min and max are also null minValues.put(statsColumn, Literal.ofNull(columnType)); maxValues.put(statsColumn, Literal.ofNull(columnType)); continue; } Literal minValue = decodeMinMaxStat(columnType, statistics, true /* decodeMin */); minValues.put(statsColumn, minValue); Literal maxValue = decodeMinMaxStat(columnType, statistics, false /* decodeMin */); maxValues.put(statsColumn, maxValue); } return new DataFileStatistics(rowCount, minValues, maxValues, nullCounts, Optional.empty()); } private static Literal decodeMinMaxStat( DataType dataType, Statistics statistics, boolean decodeMin) { Object statValue = decodeMin ? statistics.genericGetMin() : statistics.genericGetMax(); if (statValue == null) { return null; } if (dataType instanceof BooleanType) { return Literal.ofBoolean((Boolean) statValue); } else if (dataType instanceof ByteType) { return Literal.ofByte(((Number) statValue).byteValue()); } else if (dataType instanceof ShortType) { return Literal.ofShort(((Number) statValue).shortValue()); } else if (dataType instanceof IntegerType) { return Literal.ofInt(((Number) statValue).intValue()); } else if (dataType instanceof LongType) { return Literal.ofLong(((Number) statValue).longValue()); } else if (dataType instanceof FloatType) { return Literal.ofFloat(((Number) statValue).floatValue()); } else if (dataType instanceof DoubleType) { return Literal.ofDouble(((Number) statValue).doubleValue()); } else if (dataType instanceof DecimalType) { LogicalTypeAnnotation logicalType = statistics.type().getLogicalTypeAnnotation(); checkArgument( logicalType instanceof DecimalLogicalTypeAnnotation, "Physical decimal column has invalid Parquet Logical Type: %s", logicalType); int scale = ((DecimalLogicalTypeAnnotation) logicalType).getScale(); DecimalType decimalType = (DecimalType) dataType; // Check the scale is same in both the Delta data type and the Parquet Logical Type checkArgument( scale == decimalType.getScale(), "Physical decimal type has different scale than the logical type: %s", scale); // Decimal is stored either as int, long or binary. Decode the stats accordingly. BigDecimal decimalStatValue; if (statistics instanceof IntStatistics) { decimalStatValue = BigDecimal.valueOf((Integer) statValue).movePointLeft(scale); } else if (statistics instanceof LongStatistics) { decimalStatValue = BigDecimal.valueOf((Long) statValue).movePointLeft(scale); } else if (statistics instanceof BinaryStatistics) { BigInteger base = new BigInteger(getBinaryStat(statistics, decodeMin)); decimalStatValue = new BigDecimal(base, scale); } else { throw new UnsupportedOperationException( "Unsupported stats type for Decimal: " + statistics.getClass()); } return Literal.ofDecimal( decimalStatValue, decimalType.getPrecision(), decimalType.getScale()); } else if (dataType instanceof DateType) { checkArgument( statistics instanceof IntStatistics, "Column with DATE type contained invalid statistics: %s", statistics); return Literal.ofDate((Integer) statValue); // stats are stored as epoch days in Parquet } else if (dataType instanceof TimestampType) { // Kernel Parquet writer always writes timestamps in INT64 format checkArgument( statistics instanceof LongStatistics, "Column with TIMESTAMP type contained invalid statistics: %s", statistics); return Literal.ofTimestamp((Long) statValue); } else if (dataType instanceof TimestampNTZType) { checkArgument( statistics instanceof LongStatistics, "Column with TIMESTAMP_NTZ type contained invalid statistics: %s", statistics); return Literal.ofTimestampNtz((Long) statValue); } else if (dataType instanceof StringType) { byte[] binaryStat = getBinaryStat(statistics, decodeMin); return Literal.ofString(new String(binaryStat, UTF_8)); } else if (dataType instanceof BinaryType) { return Literal.ofBinary(getBinaryStat(statistics, decodeMin)); } throw new IllegalArgumentException("Unsupported stats data type: " + statValue); } private static Optional> mergeMetadataList( Collection metadataList) { if (hasInvalidStatistics(metadataList)) { return Optional.empty(); } return metadataList.stream() .>map(ColumnChunkMetaData::getStatistics) .reduce( (statsA, statsB) -> { statsA.mergeStatistics(statsB); return statsA; }); } private static boolean hasInvalidStatistics(Collection metadataList) { // If any row group does not have stats collected, stats for the file will not be valid return metadataList.stream() .anyMatch( metadata -> { Statistics stats = metadata.getStatistics(); if (stats == null || stats.isEmpty() || !stats.isNumNullsSet()) { return true; } // Columns with NaN values are marked by `hasNonNullValue` = false by the Parquet // reader // See issue: https://issues.apache.org/jira/browse/PARQUET-1246 return !stats.hasNonNullValue() && stats.getNumNulls() != metadata.getValueCount(); }); } private static boolean isStatsSupportedDataType(DataType dataType) { return dataType instanceof BooleanType || dataType instanceof ByteType || dataType instanceof ShortType || dataType instanceof IntegerType || dataType instanceof LongType || dataType instanceof FloatType || dataType instanceof DoubleType || dataType instanceof DecimalType || dataType instanceof DateType || dataType instanceof TimestampType || dataType instanceof TimestampNTZType || dataType instanceof StringType || dataType instanceof BinaryType; } private static byte[] getBinaryStat(Statistics statistics, boolean decodeMin) { return decodeMin ? statistics.getMinBytes() : statistics.getMaxBytes(); } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/RepeatedValueConverter.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.parquet; import static io.delta.kernel.defaults.internal.parquet.ParquetColumnReaders.initNullabilityVector; import static io.delta.kernel.defaults.internal.parquet.ParquetColumnReaders.setNullabilityToTrue; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.defaults.internal.parquet.ParquetColumnReaders.BaseColumnReader; import java.util.Arrays; import org.apache.parquet.io.api.Converter; import org.apache.parquet.io.api.GroupConverter; /** * Abstract implementation of Parquet converters for capturing the repeated types such as list or * map. */ abstract class RepeatedValueConverter extends GroupConverter implements BaseColumnReader { private final Collector collector; // working state private int currentRowIndex; private boolean[] nullability; private int[] offsets; // If the repeated value is null, start/end never get called which is a signal for null // Set the initial state to true and when start() is called set it to false. private boolean isCurrentValueNull = true; /** * Create instance. * * @param initialBatchSize Starting batch output size. * @param elementConverters List of converters that are part of the repeated type. */ RepeatedValueConverter(int initialBatchSize, Converter... elementConverters) { checkArgument(initialBatchSize > 0, "invalid initialBatchSize: %s", initialBatchSize); this.collector = new Collector(elementConverters); // initialize working state this.nullability = initNullabilityVector(initialBatchSize); this.offsets = new int[initialBatchSize + 1]; } @Override public Converter getConverter(int fieldIndex) { if (fieldIndex != 0) { throw new IllegalArgumentException("Invalid field index: " + fieldIndex); } return collector; } @Override public void start() { this.isCurrentValueNull = false; } @Override public void end() {} @Override public void finalizeCurrentRow(long currentRowIndex) { resizeIfNeeded(); this.offsets[this.currentRowIndex + 1] = collector.currentEntryIndex; this.nullability[this.currentRowIndex] = isCurrentValueNull; this.isCurrentValueNull = true; this.currentRowIndex++; } @Override public void resizeIfNeeded() { if (nullability.length == currentRowIndex) { int newSize = nullability.length * 2; this.nullability = Arrays.copyOf(this.nullability, newSize); setNullabilityToTrue(this.nullability, newSize / 2, newSize); this.offsets = Arrays.copyOf(this.offsets, newSize + 1); } } @Override public void resetWorkingState() { this.currentRowIndex = 0; this.isCurrentValueNull = true; this.nullability = initNullabilityVector(nullability.length); this.offsets = new int[offsets.length]; } protected boolean[] getNullability() { return nullability; } protected int[] getOffsets() { return offsets; } /** * @return the {@link ColumnVector}s from the underlying element vectors. Once retrieved the * converters are reset and requires {@link #resetWorkingState()} before using this repeated * converter again. */ protected ColumnVector[] getElementDataVectors() { return collector.getDataVectors(); } /** * GroupConverter to collect repeated elements. For each repeated element value set, the call * pattern from the Parquet reader: start(), followed by value read for each element converter and * end(). */ private static class Collector extends GroupConverter { private final Converter[] elementConverters; // working state private int currentEntryIndex; Collector(Converter... elementConverters) { this.elementConverters = elementConverters; } @Override public Converter getConverter(int fieldIndex) { if (fieldIndex < 0 || fieldIndex >= elementConverters.length) { throw new IllegalArgumentException("Invalid field index: " + fieldIndex); } return elementConverters[fieldIndex]; } @Override public void start() {} @Override public void end() { for (Converter converter : elementConverters) { long prevRowIndex = -1; // Row indexes are not needed for nested columns ((BaseColumnReader) converter).finalizeCurrentRow(prevRowIndex); } currentEntryIndex++; } ColumnVector[] getDataVectors() { ColumnVector[] dataVectors = new ColumnVector[elementConverters.length]; for (int i = 0; i < elementConverters.length; i++) { dataVectors[i] = ((BaseColumnReader) elementConverters[i]).getDataColumnVector(currentEntryIndex); } currentEntryIndex = 0; return dataVectors; } } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/RowColumnReader.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.parquet; import static io.delta.kernel.defaults.internal.parquet.ParquetSchemaUtils.findSubFieldType; import static io.delta.kernel.defaults.internal.parquet.ParquetSchemaUtils.getParquetFieldToTypeMap; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.defaults.internal.data.DefaultColumnarBatch; import io.delta.kernel.defaults.internal.data.vector.DefaultStructVector; import io.delta.kernel.types.*; import java.util.*; import org.apache.parquet.io.api.Converter; import org.apache.parquet.io.api.GroupConverter; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.Type; /** * Row column readers for materializing the column values from Parquet files into Kernels {@link * ColumnVector}. */ class RowColumnReader extends GroupConverter implements ParquetColumnReaders.BaseColumnReader { private final StructType readSchema; private final Converter[] converters; // The delta may request columns that don't exists in Parquet // This map is to track the ordinal known to Parquet reader to the converter array ordinal. // If a column is missing, a dummy converter added to the `converters` array and which // generates all null vector at the end. private final Map parquetOrdinalToConverterOrdinal; // Working state private boolean isCurrentValueNull = true; private int currentRowIndex; private boolean[] nullability; /** * Create converter for {@link StructType} column. * * @param initialBatchSize Estimate of initial row batch size. Used in memory allocations. * @param readSchema Schem of the columns to read from the file. * @param fileSchema Schema of the pruned columns from the file schema We have some necessary * requirements here: (1) the fields in fileSchema are a subset of readSchema (parquet schema * has been pruned). (2) the fields in fileSchema are in the same order as the corresponding * fields in readSchema. */ RowColumnReader(int initialBatchSize, StructType readSchema, GroupType fileSchema) { checkArgument(initialBatchSize > 0, "invalid initialBatchSize: %s", initialBatchSize); this.readSchema = requireNonNull(readSchema, "readSchema is not null"); List fields = readSchema.fields(); this.converters = new Converter[fields.size()]; this.parquetOrdinalToConverterOrdinal = new HashMap<>(); // Initialize the working state this.nullability = ParquetColumnReaders.initNullabilityVector(initialBatchSize); int parquetOrdinal = 0; for (int i = 0; i < converters.length; i++) { final StructField field = fields.get(i); final DataType typeFromClient = field.getDataType(); final Map parquetFieldIdToTypeMap = getParquetFieldToTypeMap(fileSchema); final Type typeFromFile = field.isDataColumn() ? findSubFieldType(fileSchema, field, parquetFieldIdToTypeMap) : null; if (typeFromFile == null) { if (MetadataColumnSpec.ROW_INDEX.equals(field.getMetadataColumnSpec())) { checkArgument( field.getDataType() instanceof LongType, "row index metadata column must be type long"); converters[i] = new ParquetColumnReaders.FileRowIndexColumnReader(initialBatchSize); } else { converters[i] = new ParquetColumnReaders.NonExistentColumnReader(typeFromClient); } } else { converters[i] = ParquetColumnReaders.createConverter(initialBatchSize, typeFromClient, typeFromFile); parquetOrdinalToConverterOrdinal.put(parquetOrdinal, i); parquetOrdinal++; } } } @Override public Converter getConverter(int fieldIndex) { return converters[parquetOrdinalToConverterOrdinal.get(fieldIndex)]; } @Override public void start() { isCurrentValueNull = false; } @Override public void end() {} public ColumnarBatch getDataAsColumnarBatch(int batchSize) { ColumnVector[] memberVectors = collectMemberVectors(batchSize); ColumnarBatch batch = new DefaultColumnarBatch(batchSize, readSchema, memberVectors); resetWorkingState(); return batch; } @Override public void finalizeCurrentRow(long currentRowIndex) { resizeIfNeeded(); finalizeLastRowInConverters(currentRowIndex); nullability[this.currentRowIndex] = isCurrentValueNull; isCurrentValueNull = true; this.currentRowIndex++; } public ColumnVector getDataColumnVector(int batchSize) { ColumnVector[] memberVectors = collectMemberVectors(batchSize); ColumnVector vector = new DefaultStructVector(batchSize, readSchema, Optional.of(nullability), memberVectors); resetWorkingState(); return vector; } @Override public void resizeIfNeeded() { if (nullability.length == currentRowIndex) { int newSize = nullability.length * 2; this.nullability = Arrays.copyOf(this.nullability, newSize); ParquetColumnReaders.setNullabilityToTrue(this.nullability, newSize / 2, newSize); } } @Override public void resetWorkingState() { this.currentRowIndex = 0; this.isCurrentValueNull = true; this.nullability = ParquetColumnReaders.initNullabilityVector(this.nullability.length); } private void finalizeLastRowInConverters(long prevRowIndex) { for (int i = 0; i < converters.length; i++) { ((ParquetColumnReaders.BaseColumnReader) converters[i]).finalizeCurrentRow(prevRowIndex); } } private ColumnVector[] collectMemberVectors(int batchSize) { final ColumnVector[] output = new ColumnVector[converters.length]; for (int i = 0; i < converters.length; i++) { output[i] = ((ParquetColumnReaders.BaseColumnReader) converters[i]).getDataColumnVector(batchSize); } return output; } } ================================================ FILE: kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/TimestampConverters.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.parquet; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT96; import io.delta.kernel.defaults.internal.DefaultKernelUtils; import io.delta.kernel.types.*; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.time.ZoneOffset; import org.apache.parquet.io.api.Binary; import org.apache.parquet.io.api.Converter; import org.apache.parquet.schema.*; import org.apache.parquet.schema.LogicalTypeAnnotation.TimestampLogicalTypeAnnotation; /** * Column readers for columns of types {@code timestmap, timestamp_ntz}. These both data types share * the same encoding methods except the logical type. */ public class TimestampConverters { /** * Create a {@code timestamp} column type reader * * @param initialBatchSize Initial batch size of the generated column vector * @param typeFromFile Column type metadata from Parquet file * @param typeFromClient Column type from client * @return instance of {@link Converter} */ public static Converter createTimestampConverter( int initialBatchSize, Type typeFromFile, DataType typeFromClient) { PrimitiveType primType = typeFromFile.asPrimitiveType(); LogicalTypeAnnotation typeAnnotation = primType.getLogicalTypeAnnotation(); boolean isTimestampTz = (typeFromClient instanceof TimestampType); if (primType.getPrimitiveTypeName() == INT96) { // INT96 does not have a logical type in both TIMESTAMP and TIMESTAMP_NTZ // Also, TimestampNTZ type does not require rebasing // due to its lack of time zone context. return new TimestampBinaryConverter(typeFromClient, initialBatchSize); } else if (primType.getPrimitiveTypeName() == INT64 && typeAnnotation instanceof TimestampLogicalTypeAnnotation) { TimestampLogicalTypeAnnotation timestamp = (TimestampLogicalTypeAnnotation) typeAnnotation; boolean isAdjustedUtc = timestamp.isAdjustedToUTC(); if (!((isTimestampTz && isAdjustedUtc) || (!isTimestampTz && !isAdjustedUtc))) { throw new RuntimeException( String.format( "Incompatible Utc adjustment for timestamp column. " + "Client type: %s, File type: %s, isAdjustedUtc: %s", typeFromClient, typeFromFile, isAdjustedUtc)); } switch (timestamp.getUnit()) { case MICROS: return new ParquetColumnReaders.LongColumnReader(typeFromClient, initialBatchSize); case MILLIS: return new TimestampMillisConverter(typeFromClient, initialBatchSize); default: throw new UnsupportedOperationException( String.format("Unsupported Parquet TimeType unit=%s", timestamp.getUnit())); } } else if (typeFromClient == TimestampNTZType.TIMESTAMP_NTZ && primType.getPrimitiveTypeName() == INT32 && typeAnnotation instanceof LogicalTypeAnnotation.DateLogicalTypeAnnotation) { return new DateToTimestampNTZConverter(typeFromClient, initialBatchSize); } else { throw new RuntimeException( String.format("Unsupported timestamp column with Parquet type %s.", typeFromFile)); } } public static class TimestampMillisConverter extends ParquetColumnReaders.LongColumnReader { TimestampMillisConverter(DataType dataType, int initialBatchSize) { super(validTimestampType(dataType), initialBatchSize); } @Override public void addLong(long value) { super.addLong(DefaultKernelUtils.millisToMicros(value)); } } public static class TimestampBinaryConverter extends ParquetColumnReaders.LongColumnReader { TimestampBinaryConverter(DataType dataType, int initialBatchSize) { super(validTimestampType(dataType), initialBatchSize); } private long binaryToSQLTimestamp(Binary binary) { checkArgument( binary.length() == 12, "Timestamps (with nanoseconds) are expected to be stored in 12-byte long " + "binaries. Found a %s-byte binary instead.", binary.length()); ByteBuffer buffer = binary.toByteBuffer().order(ByteOrder.LITTLE_ENDIAN); long timeOfDayNanos = buffer.getLong(); int julianDay = buffer.getInt(); return DefaultKernelUtils.fromJulianDay(julianDay, timeOfDayNanos); } @Override public void addBinary(Binary value) { long julianMicros = binaryToSQLTimestamp(value); // we do not rebase timestamps long gregorianMicros = julianMicros; super.addLong(gregorianMicros); } @Override public void addLong(long value) { throw new UnsupportedOperationException(getClass().getName()); } } public static class DateToTimestampNTZConverter extends ParquetColumnReaders.LongColumnReader { DateToTimestampNTZConverter(DataType dataType, int initialBatchSize) { super(validTimestampType(dataType), initialBatchSize); } @Override public void addInt(int value) { super.addLong(DefaultKernelUtils.daysToMicros(value, ZoneOffset.UTC)); } } private static DataType validTimestampType(DataType dataType) { checkArgument(dataType instanceof TimestampType || dataType instanceof TimestampNTZType); return dataType; } } ================================================ FILE: kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/integration/DataBuilderUtils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.integration; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.data.ArrayValue; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.data.MapValue; import io.delta.kernel.data.Row; import io.delta.kernel.defaults.internal.data.DefaultRowBasedColumnarBatch; import io.delta.kernel.types.StructType; import java.math.BigDecimal; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.IntStream; public class DataBuilderUtils { public static TestColumnBatchBuilder builder(StructType schema) { return new TestColumnBatchBuilder(schema); } public static Row row(StructType structType, Object... values) { return new TestRow(structType, values); } public static Row row(StructType structType) { return new TestRow(structType); } public static class TestColumnBatchBuilder { private StructType schema; private List rows = new ArrayList<>(); private TestColumnBatchBuilder(StructType schema) { this.schema = schema; } public TestColumnBatchBuilder addRow(Object... values) { checkArgument(values.length == schema.length(), "Invalid columns length"); rows.add(row(schema, values)); return this; } public TestColumnBatchBuilder addAllNullsRow() { rows.add(row(schema)); return this; } public ColumnarBatch build() { return new DefaultRowBasedColumnarBatch(schema, rows); } } private static class TestRow implements Row { private final StructType schema; private final Map values; private TestRow(StructType schema, Object... values) { this.schema = schema; this.values = new HashMap<>(); for (int i = 0; i < values.length; i++) { // lamdas + streams don't work well with null values this.values.put(i, values[i]); } } private TestRow(StructType schema) { Map values = new HashMap<>(); IntStream.range(0, schema.length()).forEach(idx -> values.put(idx, null)); this.schema = schema; this.values = values; } @Override public StructType getSchema() { return schema; } @Override public boolean isNullAt(int ordinal) { return values.get(ordinal) == null; } @Override public boolean getBoolean(int ordinal) { return (boolean) values.get(ordinal); } @Override public byte getByte(int ordinal) { return (byte) values.get(ordinal); } @Override public short getShort(int ordinal) { return (short) values.get(ordinal); } @Override public int getInt(int ordinal) { return (int) values.get(ordinal); } @Override public long getLong(int ordinal) { return (long) values.get(ordinal); } @Override public float getFloat(int ordinal) { return (float) values.get(ordinal); } @Override public double getDouble(int ordinal) { return (double) values.get(ordinal); } @Override public String getString(int ordinal) { return (String) values.get(ordinal); } @Override public BigDecimal getDecimal(int ordinal) { return (BigDecimal) values.get(ordinal); } @Override public byte[] getBinary(int ordinal) { return (byte[]) values.get(ordinal); } @Override public Row getStruct(int ordinal) { return (Row) values.get(ordinal); } @Override public ArrayValue getArray(int ordinal) { throw new UnsupportedOperationException( "array type unsupported for TestColumnBatchBuilder; use scala test utilities"); } @Override public MapValue getMap(int ordinal) { throw new UnsupportedOperationException( "map type unsupported for TestColumnBatchBuilder; use scala test utilities"); } } } ================================================ FILE: kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/utils/DefaultKernelTestUtils.java ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.utils; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.Row; import io.delta.kernel.types.*; public class DefaultKernelTestUtils { private DefaultKernelTestUtils() {} /** Returns a URI encoded path of the resource. */ public static String getTestResourceFilePath(String resourcePath) { return DefaultKernelTestUtils.class.getClassLoader().getResource(resourcePath).getFile(); } // This will no longer be needed once all tests have been moved to Scala public static Object getValueAsObject(Row row, int columnOrdinal) { final DataType dataType = row.getSchema().at(columnOrdinal).getDataType(); if (row.isNullAt(columnOrdinal)) { return null; } if (dataType instanceof BooleanType) { return row.getBoolean(columnOrdinal); } else if (dataType instanceof ByteType) { return row.getByte(columnOrdinal); } else if (dataType instanceof ShortType) { return row.getShort(columnOrdinal); } else if (dataType instanceof IntegerType || dataType instanceof DateType) { return row.getInt(columnOrdinal); } else if (dataType instanceof LongType || dataType instanceof TimestampType) { return row.getLong(columnOrdinal); } else if (dataType instanceof FloatType) { return row.getFloat(columnOrdinal); } else if (dataType instanceof DoubleType) { return row.getDouble(columnOrdinal); } else if (dataType instanceof StringType) { return row.getString(columnOrdinal); } else if (dataType instanceof BinaryType) { return row.getBinary(columnOrdinal); } else if (dataType instanceof StructType) { return row.getStruct(columnOrdinal); } throw new UnsupportedOperationException(dataType + " is not supported yet"); } /** * Get the value at given {@code rowId} from the column vector. The type of the value object * depends on the data type of the {@code vector}. */ public static Object getValueAsObject(ColumnVector vector, int rowId) { final DataType dataType = vector.getDataType(); if (vector.isNullAt(rowId)) { return null; } if (dataType instanceof BooleanType) { return vector.getBoolean(rowId); } else if (dataType instanceof ByteType) { return vector.getByte(rowId); } else if (dataType instanceof ShortType) { return vector.getShort(rowId); } else if (dataType instanceof IntegerType || dataType instanceof DateType) { return vector.getInt(rowId); } else if (dataType instanceof LongType || dataType instanceof TimestampType || dataType instanceof TimestampNTZType) { return vector.getLong(rowId); } else if (dataType instanceof FloatType) { return vector.getFloat(rowId); } else if (dataType instanceof DoubleType) { return vector.getDouble(rowId); } else if (dataType instanceof StringType) { return vector.getString(rowId); } else if (dataType instanceof BinaryType) { return vector.getBinary(rowId); } else if (dataType instanceof DecimalType) { return vector.getDecimal(rowId); } throw new UnsupportedOperationException(dataType + " is not supported yet"); } } ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-no-checkpoint/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1686191546018,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1003"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"90476688-fac4-4af7-9ea1-debb0c965333"}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["deletionVectors"],"writerFeatures":["deletionVectors"]}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableDeletionVectors":"true"},"createdTime":1686191541734}} {"add":{"path":"part-00000-a489737f-d477-4d9a-8b4a-bd6a6536df5b-c000.snappy.parquet","partitionValues":{},"size":500,"modificationTime":1686191545000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0},\"tightBounds\":true}"}} {"add":{"path":"part-00001-1c9b5e60-ab86-4017-9ec9-a6fe4150cdd5-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1686191545000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":5},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0},\"tightBounds\":true}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-no-checkpoint/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1686191563139,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#378L < 2)\"]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"6958","numDeletedRows":"2","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"c4ab9bb3-c0af-4e68-8eac-4b6c3d141492"}} {"add":{"path":"part-00000-a489737f-d477-4d9a-8b4a-bd6a6536df5b-c000.snappy.parquet","partitionValues":{},"size":500,"modificationTime":1686191545000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"IjB3V2d3#qUP%s94R0WF","offset":1,"sizeInBytes":36,"cardinality":2}}} {"remove":{"path":"part-00000-a489737f-d477-4d9a-8b4a-bd6a6536df5b-c000.snappy.parquet","deletionTimestamp":1686191562047,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":500}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1687493352168,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"500","numOutputBytes":"3714"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"39f72778-9262-4a53-9179-a3386be311cb"}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["deletionVectors"],"writerFeatures":["deletionVectors"]}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableDeletionVectors":"true"},"createdTime":1687493346028}} {"add":{"path":"part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet","partitionValues":{},"size":2219,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":249},\"nullCount\":{\"id\":0},\"tightBounds\":true}"}} {"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":true}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1687493376134,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#378L = 0)\"]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"8909","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"5e6e12bc-1b7f-4776-b953-c76f358f85bd"}} {"add":{"path":"part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet","partitionValues":{},"size":2219,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":249},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"2FtLtJDE!.VZ.+udLGa0","offset":1,"sizeInBytes":34,"cardinality":1}}} {"remove":{"path":"part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet","deletionTimestamp":1687493374011,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":2219}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1687493383677,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#1630L = 11)\"]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"5074","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"b6abb0cc-20ed-402b-aaa4-da99395282f7"}} {"add":{"path":"part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet","partitionValues":{},"size":2219,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":249},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"u+2?H{A@KdHac(G*R3bY","offset":1,"sizeInBytes":36,"cardinality":2}}} {"remove":{"path":"part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet","deletionTimestamp":1687493382707,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":2219,"deletionVector":{"storageType":"u","pathOrInlineDv":"2FtLtJDE!.VZ.+udLGa0","offset":1,"sizeInBytes":34,"cardinality":1}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000003.json ================================================ {"commitInfo":{"timestamp":1687493388280,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#2728L = 22)\"]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"2789","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"6f872525-30c3-4f2e-b159-00e5dc79d883"}} {"add":{"path":"part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet","partitionValues":{},"size":2219,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":249},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"Z8-vWlG.o0}xSJ&Dj^:+rNR","offset":1,"sizeInBytes":48,"cardinality":8}}} {"remove":{"path":"part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet","deletionTimestamp":1687493404691,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":2219,"deletionVector":{"storageType":"u","pathOrInlineDv":"dTnVvWLQ^#NuWWk/:&lG.o0}xSJ&Dj^:+rNR","offset":1,"sizeInBytes":48,"cardinality":8}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000010.json ================================================ {"commitInfo":{"timestamp":1687493411379,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#10400L = 99)\"]"},"readVersion":9,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"2186","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"1d3a5961-02f7-43df-9e94-f685bd8aa4a9"}} {"add":{"path":"part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet","partitionValues":{},"size":2219,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":249},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"(w{TGxkf!oV>34U26WTB","offset":1,"sizeInBytes":52,"cardinality":10}}} {"remove":{"path":"part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet","deletionTimestamp":1687493410867,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":2219,"deletionVector":{"storageType":"u","pathOrInlineDv":"F}lqo34U26WTB","offset":1,"sizeInBytes":52,"cardinality":10}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000012.json ================================================ {"commitInfo":{"timestamp":1687493419030,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#12986L = 121)\"]"},"readVersion":11,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1553","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"448849f9-e20c-4012-8aa2-e52ed4d85c82"}} {"add":{"path":"part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet","partitionValues":{},"size":2219,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":249},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"7>Z9H2%z4LIuqwRlf+)>","offset":1,"sizeInBytes":56,"cardinality":12}}} {"remove":{"path":"part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet","deletionTimestamp":1687493418827,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":2219,"deletionVector":{"storageType":"u","pathOrInlineDv":"YK?8=aIQnrWC%eK+:syC","offset":1,"sizeInBytes":54,"cardinality":11}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000013.json ================================================ {"commitInfo":{"timestamp":1687493421483,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#14107L = 132)\"]"},"readVersion":12,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1378","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"43e8665e-173a-415a-ab30-20ce4308c8b0"}} {"add":{"path":"part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet","partitionValues":{},"size":2219,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":249},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"2(ocO7^QM/Msat&JfpAR","offset":1,"sizeInBytes":58,"cardinality":13}}} {"remove":{"path":"part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet","deletionTimestamp":1687493421320,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":2219,"deletionVector":{"storageType":"u","pathOrInlineDv":"7>Z9H2%z4LIuqwRlf+)>","offset":1,"sizeInBytes":56,"cardinality":12}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000014.json ================================================ {"commitInfo":{"timestamp":1687493423908,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#15228L = 143)\"]"},"readVersion":13,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1144","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"2319e0d7-eea8-4beb-820e-12b7f28ef060"}} {"add":{"path":"part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet","partitionValues":{},"size":2219,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":249},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"Tl?MJB%Y[oJiQ>B+@/*:","offset":1,"sizeInBytes":60,"cardinality":14}}} {"remove":{"path":"part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet","deletionTimestamp":1687493423676,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":2219,"deletionVector":{"storageType":"u","pathOrInlineDv":"2(ocO7^QM/Msat&JfpAR","offset":1,"sizeInBytes":58,"cardinality":13}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000015.json ================================================ {"commitInfo":{"timestamp":1687493426612,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#16349L = 154)\"]"},"readVersion":14,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1295","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"2881078a-fc97-42fc-9b6c-295aee92f3eb"}} {"add":{"path":"part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet","partitionValues":{},"size":2219,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":249},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"V./QH]:sH4Q]00QmNsI4","offset":1,"sizeInBytes":62,"cardinality":15}}} {"remove":{"path":"part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet","deletionTimestamp":1687493426448,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":2219,"deletionVector":{"storageType":"u","pathOrInlineDv":"Tl?MJB%Y[oJiQ>B+@/*:","offset":1,"sizeInBytes":60,"cardinality":14}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000016.json ================================================ {"commitInfo":{"timestamp":1687493429640,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#17470L = 165)\"]"},"readVersion":15,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"2021","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"b3e8c5e4-5230-43ab-a1a3-44dc928d7431"}} {"add":{"path":"part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet","partitionValues":{},"size":2219,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":249},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"egQo!fD<28G{F(Ci=@L$","offset":1,"sizeInBytes":64,"cardinality":16}}} {"remove":{"path":"part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet","deletionTimestamp":1687493429383,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":2219,"deletionVector":{"storageType":"u","pathOrInlineDv":"V./QH]:sH4Q]00QmNsI4","offset":1,"sizeInBytes":62,"cardinality":15}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000017.json ================================================ {"commitInfo":{"timestamp":1687493432150,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#18591L = 176)\"]"},"readVersion":16,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1523","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"0517b48f-02d2-4ea1-81c0-58cc248a881d"}} {"add":{"path":"part-00000-87eec267-9acd-4e9a-a216-ec596132203d-c000.snappy.parquet","partitionValues":{},"size":2219,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":249},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"ez0pRYlnGp#D/X2&/I","offset":1,"sizeInBytes":34,"cardinality":1}}} {"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493453617,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000025.json ================================================ {"commitInfo":{"timestamp":1687493456787,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#28067L = 264)\"]"},"readVersion":24,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1987","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"c4ae3482-f3d5-4e8c-972f-5dfe6c6d16e9"}} {"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"/PV+vOr=NRJZUZc>4izq","offset":1,"sizeInBytes":36,"cardinality":2}}} {"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493456593,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"0pRYlnGp#D/X2&/I","offset":1,"sizeInBytes":34,"cardinality":1}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000026.json ================================================ {"commitInfo":{"timestamp":1687493459673,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#29188L = 275)\"]"},"readVersion":25,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1523","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"7387b8d6-320c-48a1-b274-e08bdb09f365"}} {"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"[OrxMF<&j1KIzaYw>L1$","offset":1,"sizeInBytes":38,"cardinality":3}}} {"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493459510,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"/PV+vOr=NRJZUZc>4izq","offset":1,"sizeInBytes":36,"cardinality":2}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000027.json ================================================ {"commitInfo":{"timestamp":1687493462246,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#30309L = 286)\"]"},"readVersion":26,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1628","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"5bd58d9b-61a8-43e1-9f54-adb944e5bb51"}} {"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"JX$#0D<-MnJ9i[$BYwYD","offset":1,"sizeInBytes":40,"cardinality":4}}} {"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493461861,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"[OrxMF<&j1KIzaYw>L1$","offset":1,"sizeInBytes":38,"cardinality":3}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000028.json ================================================ {"commitInfo":{"timestamp":1687493465572,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#31430L = 297)\"]"},"readVersion":27,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"2070","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"67246fbb-8e42-471e-8a6e-911ee02289eb"}} {"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"?i@EYwa-J5RbtEIPJ.Gl","offset":1,"sizeInBytes":42,"cardinality":5}}} {"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493465351,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"JX$#0D<-MnJ9i[$BYwYD","offset":1,"sizeInBytes":40,"cardinality":4}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000029.json ================================================ {"commitInfo":{"timestamp":1687493467801,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#32551L = 308)\"]"},"readVersion":28,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1410","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"51e5335a-cd2c-467c-93b3-44b1731e03a6"}} {"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"l$m[rf}7BENh@zObPYMw","offset":1,"sizeInBytes":44,"cardinality":6}}} {"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493467650,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"?i@EYwa-J5RbtEIPJ.Gl","offset":1,"sizeInBytes":42,"cardinality":5}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000030.json ================================================ {"commitInfo":{"timestamp":1687493470112,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#33672L = 319)\"]"},"readVersion":29,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1399","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"852d93d5-998e-4210-871c-6f86b6fa9d9b"}} {"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":">i1<2fv@(-V::TBC{X^$","offset":1,"sizeInBytes":46,"cardinality":7}}} {"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493469924,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"l$m[rf}7BENh@zObPYMw","offset":1,"sizeInBytes":44,"cardinality":6}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000031.json ================================================ {"commitInfo":{"timestamp":1687493476087,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#34882L = 330)\"]"},"readVersion":30,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"3310","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"982792fe-4bd7-404a-a7bd-19113e7ba7da"}} {"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":")y]bjYT7n0T&3ZI7fsnR","offset":1,"sizeInBytes":48,"cardinality":8}}} {"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493475923,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":">i1<2fv@(-V::TBC{X^$","offset":1,"sizeInBytes":46,"cardinality":7}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000032.json ================================================ {"commitInfo":{"timestamp":1687493478659,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#36283L = 341)\"]"},"readVersion":31,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1489","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"19b580df-8671-46f3-bedf-f0ba70b45a64"}} {"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"B(bfMgm}tEXY:Biw>Sff","offset":1,"sizeInBytes":50,"cardinality":9}}} {"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493478493,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":")y]bjYT7n0T&3ZI7fsnR","offset":1,"sizeInBytes":48,"cardinality":8}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000033.json ================================================ {"commitInfo":{"timestamp":1687493481125,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#37404L = 352)\"]"},"readVersion":32,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1406","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"88f23969-ecda-4300-ad0a-0522b3c50277"}} {"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"o0UC@Ov}pLUd!8XlnsU/","offset":1,"sizeInBytes":52,"cardinality":10}}} {"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493480901,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"B(bfMgm}tEXY:Biw>Sff","offset":1,"sizeInBytes":50,"cardinality":9}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000034.json ================================================ {"commitInfo":{"timestamp":1687493483736,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#38525L = 363)\"]"},"readVersion":33,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1555","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"c4c917d0-94d2-4eab-9e3b-c38a00d8d30f"}} {"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"[W5]y}RT#DZ1e^jIu+&k","offset":1,"sizeInBytes":54,"cardinality":11}}} {"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493483543,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"o0UC@Ov}pLUd!8XlnsU/","offset":1,"sizeInBytes":52,"cardinality":10}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000035.json ================================================ {"commitInfo":{"timestamp":1687493486709,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#39646L = 374)\"]"},"readVersion":34,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"2087","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"ae9e26f7-5c42-4a11-ae83-28913ac79aaf"}} {"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"}Vi)6cLx4bSSkUi37-Sb","offset":1,"sizeInBytes":56,"cardinality":12}}} {"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493486280,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"[W5]y}RT#DZ1e^jIu+&k","offset":1,"sizeInBytes":54,"cardinality":11}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000036.json ================================================ {"commitInfo":{"timestamp":1687493488949,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#40767L = 385)\"]"},"readVersion":35,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1387","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"f8cfc704-8c61-470b-a00b-675db4744a4d"}} {"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"QV![s>Cg)KJc8r+c2","offset":1,"sizeInBytes":58,"cardinality":13}}} {"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493488645,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"}Vi)6cLx4bSSkUi37-Sb","offset":1,"sizeInBytes":56,"cardinality":12}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000037.json ================================================ {"commitInfo":{"timestamp":1687493491174,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#41888L = 396)\"]"},"readVersion":36,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1432","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"60dd6745-b035-4fd4-9c74-cae1ad56c998"}} {"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"=pfvKB2VCg)KJc8r+c2","offset":1,"sizeInBytes":58,"cardinality":13}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000038.json ================================================ {"commitInfo":{"timestamp":1687493493743,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#43009L = 407)\"]"},"readVersion":37,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1747","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"68f77edd-106f-4a66-a2fd-2cc53a1e8e2c"}} {"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"188rU5d7$uQnz8Cw*&JR","offset":1,"sizeInBytes":62,"cardinality":15}}} {"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493493431,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"=pfvKB2VckU","offset":1,"sizeInBytes":68,"cardinality":18}}} {"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493502653,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"^wgAJ=4X/TIzz&%qDElT","offset":1,"sizeInBytes":66,"cardinality":17}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000042.json ================================================ {"commitInfo":{"timestamp":1687493505010,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#47862L = 451)\"]"},"readVersion":41,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1328","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"c9eadb2b-ac9c-4b31-9b38-923978074394"}} {"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"1ENM$^ul]}VH<&ub7J5y","offset":1,"sizeInBytes":70,"cardinality":19}}} {"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493504829,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"5TLR.gkXvkTLoklq>ckU","offset":1,"sizeInBytes":68,"cardinality":18}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000043.json ================================================ {"commitInfo":{"timestamp":1687493507835,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#48983L = 462)\"]"},"readVersion":42,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1977","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"81b2bee3-440a-4051-92ca-c7c9df8fcd02"}} {"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"1lI0]tO]*HV].E0P<8T2","offset":1,"sizeInBytes":72,"cardinality":20}}} {"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493507663,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"1ENM$^ul]}VH<&ub7J5y","offset":1,"sizeInBytes":70,"cardinality":19}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000044.json ================================================ {"commitInfo":{"timestamp":1687493510227,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#50104L = 473)\"]"},"readVersion":43,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1453","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"5096139c-7d93-40aa-a573-37088d859c0c"}} {"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"8RfaCZ!EkOFD(^Syd?cC","offset":1,"sizeInBytes":74,"cardinality":21}}} {"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493510052,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"1lI0]tO]*HV].E0P<8T2","offset":1,"sizeInBytes":72,"cardinality":20}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000045.json ================================================ {"commitInfo":{"timestamp":1687493513120,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#51225L = 484)\"]"},"readVersion":44,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1945","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"5efa6fb6-29e3-46c9-8a4a-2d82df02da68"}} {"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"Yq}%xtpq&YTHdjxg?Osr","offset":1,"sizeInBytes":76,"cardinality":22}}} {"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493512712,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"8RfaCZ!EkOFD(^Syd?cC","offset":1,"sizeInBytes":74,"cardinality":21}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000046.json ================================================ {"commitInfo":{"timestamp":1687493515976,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#52346L = 495)\"]"},"readVersion":45,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1419","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"deeed50a-1264-4c7e-9e26-b3217e7779dd"}} {"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"+Ry*Wio9zqLW9MIvM6Su","offset":1,"sizeInBytes":78,"cardinality":23}}} {"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493515806,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"Yq}%xtpq&YTHdjxg?Osr","offset":1,"sizeInBytes":76,"cardinality":22}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/_last_checkpoint ================================================ {"version":40,"size":44,"sizeInBytes":17200,"numOfAddFiles":2,"checkpointSchema":{"type":"struct","fields":[{"name":"txn","type":{"type":"struct","fields":[{"name":"appId","type":"string","nullable":true,"metadata":{}},{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"lastUpdated","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"add","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"stats","type":"string","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"remove","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"deletionTimestamp","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"extendedFileMetadata","type":"boolean","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"metaData","type":{"type":"struct","fields":[{"name":"id","type":"string","nullable":true,"metadata":{}},{"name":"name","type":"string","nullable":true,"metadata":{}},{"name":"description","type":"string","nullable":true,"metadata":{}},{"name":"format","type":{"type":"struct","fields":[{"name":"provider","type":"string","nullable":true,"metadata":{}},{"name":"options","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"schemaString","type":"string","nullable":true,"metadata":{}},{"name":"partitionColumns","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"configuration","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"createdTime","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"protocol","type":{"type":"struct","fields":[{"name":"minReaderVersion","type":"integer","nullable":true,"metadata":{}},{"name":"minWriterVersion","type":"integer","nullable":true,"metadata":{}},{"name":"readerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"writerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"checksum":"1db7d2ee97496873124d8c27e4d28019"} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-with-checkpoint/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1679943471996,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1003"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"fa272a31-18c1-4c57-ae5c-6b52fbe83e92"}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1679943471575}} {"add":{"path":"part-00000-a65ab59f-72fd-44c9-a73e-e2d09459f836-c000.snappy.parquet","partitionValues":{},"size":500,"modificationTime":1679943471000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-39e6196f-2259-4ba4-b1d6-005712cd7784-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943471000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":5},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-with-checkpoint/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1679943473096,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1006"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"f9689ad3-1179-4682-bd00-4a635b48cba8"}} {"add":{"path":"part-00000-5c99dc53-38c0-420f-a91b-6df7a4c27a2b-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943473000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":10},\"maxValues\":{\"id\":14},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-8be0e82d-ce51-43a6-92eb-df71a9088173-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943473000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":15},\"maxValues\":{\"id\":19},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-with-checkpoint/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1679943474078,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1006"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"33a8226f-dfcc-4ec1-82a7-fb4f82014006"}} {"add":{"path":"part-00000-3317387d-183d-4db7-ac3e-596901d90de0-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943474000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":20},\"maxValues\":{\"id\":24},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-a65a81c6-292a-49f2-8eea-82c0299cdfb3-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943474000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":25},\"maxValues\":{\"id\":29},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-with-checkpoint/_delta_log/00000000000000000003.json ================================================ {"commitInfo":{"timestamp":1679943475370,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1006"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"1fc575df-b4bf-452f-bf30-6715cb63ae21"}} {"add":{"path":"part-00000-14b8a37a-107b-455f-ab94-8f55e44c004b-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943475000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":30},\"maxValues\":{\"id\":34},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-fdaa71cc-84b2-43b1-b049-7cd36dbaa0de-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943475000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":35},\"maxValues\":{\"id\":39},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-with-checkpoint/_delta_log/00000000000000000004.json ================================================ {"commitInfo":{"timestamp":1679943476409,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":3,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1006"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"896a19db-a5c0-45e1-9265-7d6745b0860e"}} {"add":{"path":"part-00000-100e4547-5ff3-4735-9550-7757ca23c61d-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943476000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":40},\"maxValues\":{\"id\":44},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-4e30e0a7-63d2-4d2f-a028-b92058c3c8cf-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943476000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":45},\"maxValues\":{\"id\":49},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-with-checkpoint/_delta_log/00000000000000000005.json ================================================ {"commitInfo":{"timestamp":1679943477339,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":4,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1006"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"a9a4c094-2e27-40f2-bdef-6eead53d535c"}} {"add":{"path":"part-00000-eb1dae3f-8c89-46c3-818f-491cc673936a-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943477000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":50},\"maxValues\":{\"id\":54},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-a9a33a7f-26fa-447d-8b34-863b5f695f06-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943477000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":55},\"maxValues\":{\"id\":59},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-with-checkpoint/_delta_log/00000000000000000006.json ================================================ {"commitInfo":{"timestamp":1679943478349,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":5,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1006"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"e60661af-beaf-444d-9b7e-df34d0ef119e"}} {"add":{"path":"part-00000-1bbb3853-04b4-4539-a112-be7140314153-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943478000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":60},\"maxValues\":{\"id\":64},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-30432f6b-710f-440c-8145-adbaed187c63-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943478000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":65},\"maxValues\":{\"id\":69},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-with-checkpoint/_delta_log/00000000000000000007.json ================================================ {"commitInfo":{"timestamp":1679943479247,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":6,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1006"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"1545252b-0e40-4ac8-9fb9-2a165a524c61"}} {"add":{"path":"part-00000-e51a2d2a-d1a3-437e-a428-f5afe93d4619-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943479000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":70},\"maxValues\":{\"id\":74},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-ed427b16-f597-432a-a49e-135b126d38a8-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943479000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":75},\"maxValues\":{\"id\":79},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-with-checkpoint/_delta_log/00000000000000000008.json ================================================ {"commitInfo":{"timestamp":1679943480075,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":7,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1006"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"43e883c7-8afd-4ef4-9d4d-0be79a3a6b9e"}} {"add":{"path":"part-00000-5e9186c7-c7b0-4c4d-9f22-1c0cd403142c-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943480000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":80},\"maxValues\":{\"id\":84},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-3cd0b397-0ac3-48ac-88ab-3cc21542e303-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943480000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":85},\"maxValues\":{\"id\":89},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-with-checkpoint/_delta_log/00000000000000000009.json ================================================ {"commitInfo":{"timestamp":1679943480946,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":8,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1006"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"c63d10cf-f665-4b99-a620-abbf9518d520"}} {"add":{"path":"part-00000-8d5f08ff-261b-4315-99cb-d289a3191368-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943480000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":90},\"maxValues\":{\"id\":94},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-51925029-c591-4366-b3e9-aeea97594037-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943480000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":95},\"maxValues\":{\"id\":99},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-with-checkpoint/_delta_log/00000000000000000010.json ================================================ {"commitInfo":{"timestamp":1679943481745,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":9,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1005"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"cdda48e4-68d6-4416-9eca-a23f5c3a59cd"}} {"add":{"path":"part-00000-2a210d80-a4e6-4a1c-8716-ee0b542aee08-c000.snappy.parquet","partitionValues":{},"size":502,"modificationTime":1679943481000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":100},\"maxValues\":{\"id\":104},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-43e387db-3e56-44f3-8965-9187a80fce9a-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943481000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":105},\"maxValues\":{\"id\":109},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-with-checkpoint/_delta_log/00000000000000000011.json ================================================ {"commitInfo":{"timestamp":1679943484145,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":10,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1006"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"8cdfb467-8be9-4fed-b3e4-d1c2ad6f8b46"}} {"add":{"path":"part-00000-326010e2-01f4-4dfb-90a7-98cbc04a60d1-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943484000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":110},\"maxValues\":{\"id\":114},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-ba1ceb1e-6a37-4e2e-8e97-a17b9b1bb33d-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943484000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":115},\"maxValues\":{\"id\":119},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-with-checkpoint/_delta_log/00000000000000000012.json ================================================ {"commitInfo":{"timestamp":1679943485143,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":11,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1006"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"d19bd23c-dc4a-430f-8882-657f3bb9aa20"}} {"add":{"path":"part-00000-6e367682-7cd1-48e6-bc2f-bc94aa94d1a3-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943485000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":120},\"maxValues\":{\"id\":124},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-63b224e2-ba72-4b95-af02-5af2367d4130-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943485000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":125},\"maxValues\":{\"id\":129},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-with-checkpoint/_delta_log/00000000000000000013.json ================================================ {"commitInfo":{"timestamp":1679943486048,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":12,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1005"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"8388e2cc-f11a-4b4b-98f1-a1ad5e45d9ee"}} {"add":{"path":"part-00000-0d9c05f4-8afc-4325-b1e0-ea32e4eff918-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943486000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":130},\"maxValues\":{\"id\":134},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-d64b05c7-d80d-4eff-8c58-d209003ee4c0-c000.snappy.parquet","partitionValues":{},"size":502,"modificationTime":1679943486000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":135},\"maxValues\":{\"id\":139},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-with-checkpoint/_delta_log/00000000000000000014.json ================================================ {"commitInfo":{"timestamp":1679943486941,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":13,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1005"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"9687712e-b9dd-4a48-bede-acb00101133f"}} {"add":{"path":"part-00000-ce6aed75-3e85-4d7c-90de-9d465e9acc04-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1679943486000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":140},\"maxValues\":{\"id\":144},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-4707caa4-d293-4b4a-aeef-fa4d5815e732-c000.snappy.parquet","partitionValues":{},"size":502,"modificationTime":1679943486000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":145},\"maxValues\":{\"id\":149},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/basic-with-checkpoint/_delta_log/_last_checkpoint ================================================ {"version":10,"size":24,"sizeInBytes":11658,"numOfAddFiles":22,"checkpointSchema":{"type":"struct","fields":[{"name":"txn","type":{"type":"struct","fields":[{"name":"appId","type":"string","nullable":true,"metadata":{}},{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"lastUpdated","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"add","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"stats","type":"string","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"remove","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"deletionTimestamp","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"extendedFileMetadata","type":"boolean","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"metaData","type":{"type":"struct","fields":[{"name":"id","type":"string","nullable":true,"metadata":{}},{"name":"name","type":"string","nullable":true,"metadata":{}},{"name":"description","type":"string","nullable":true,"metadata":{}},{"name":"format","type":{"type":"struct","fields":[{"name":"provider","type":"string","nullable":true,"metadata":{}},{"name":"options","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"schemaString","type":"string","nullable":true,"metadata":{}},{"name":"partitionColumns","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"configuration","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"createdTime","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"protocol","type":{"type":"struct","fields":[{"name":"minReaderVersion","type":"integer","nullable":true,"metadata":{}},{"name":"minWriterVersion","type":"integer","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"checksum":"d4dd43c87695abaede4556e73b008658"} ================================================ FILE: kernel/kernel-defaults/src/test/resources/catalog-owned-preview/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"inCommitTimestamp":1749830855993,"timestamp":1749830855992,"operation":"CREATE TABLE","operationParameters":{"partitionBy":"[\"part1\"]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{}"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/4.0.0 Delta-Lake/4.0.0","txnId":"d108f896-9662-4eda-b4de-444a99850aa8"}} {"metaData":{"id":"64dcd182-b3b4-4ee0-88e0-63c159a4121c","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"part1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"col1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["part1"],"configuration":{"delta.enableInCommitTimestamps":"true"},"createdTime":1749830855646}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["catalogManaged"],"writerFeatures":["catalogManaged","inCommitTimestamp","invariants","appendOnly"]}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/catalog-owned-preview/_delta_log/_staged_commits/00000000000000000001.4cb9708e-b478-44de-b203-53f9ba9b2876.json ================================================ {"commitInfo":{"inCommitTimestamp":1749830871085,"timestamp":1749830871084,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"100","numOutputBytes":"889"},"engineInfo":"Apache-Spark/4.0.0 Delta-Lake/4.0.0","txnId":"4cb9708e-b478-44de-b203-53f9ba9b2876"}} {"add":{"path":"part1=0/part-00000-13fefaba-8ec2-4762-b17e-aeda657451c5.c000.snappy.parquet","partitionValues":{"part1":"0"},"size":889,"modificationTime":1749830870833,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"col1\":0},\"maxValues\":{\"col1\":99},\"nullCount\":{\"col1\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/catalog-owned-preview/_delta_log/_staged_commits/00000000000000000002.5b9bba4a-0085-430d-a65e-b0d38c1afbe9.json ================================================ {"commitInfo":{"inCommitTimestamp":1749830881799,"timestamp":1749830881798,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"100","numOutputBytes":"891"},"engineInfo":"Apache-Spark/4.0.0 Delta-Lake/4.0.0","txnId":"5b9bba4a-0085-430d-a65e-b0d38c1afbe9"}} {"add":{"path":"part1=1/part-00000-8afb1c56-2018-4af2-aa4f-4336c1b39efd.c000.snappy.parquet","partitionValues":{"part1":"1"},"size":891,"modificationTime":1749830881779,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"col1\":100},\"maxValues\":{\"col1\":199},\"nullCount\":{\"col1\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/catalog-owned-preview/info.txt ================================================ # Below are the commands and instructions to create the `catalog-owned-preview` table. # Note that delta-spark:4.0.0 does not yet support *creating* catalogManaged tables. # So, for now, we create a normal table with ICT enabled and then # (a) manually add the `catalogManaged` # (b) manually move and rename the published delta files into the _staged_commits directory. pyspark --packages io.delta:delta-spark_2.13:4.0.0 \ --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" \ --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" table_path = # Commit 0: Create the table spark.sql(f""" CREATE TABLE delta.`{table_path}` ( part1 INT, col1 INT ) USING DELTA PARTITIONED BY (part1) """) # Commit 1: Insert 100 rows into part1=0 spark.sql(f""" INSERT INTO delta.`{table_path}` SELECT col1 DIV 100 as part1, col1 FROM ( SELECT explode(sequence(0, 99)) as col1 ) """) # Commit 2: Insert 100 rows into part1=1 spark.sql(f""" INSERT INTO delta.`{table_path}` SELECT col1 DIV 100 as part1, col1 FROM ( SELECT explode(sequence(100, 199)) as col1 ) """) # Then, add `"readerFeatures":["catalogManaged"]` to the _delta_log/001.json protocol # Then, for commits version $v in [1, 2] move _delta_log/$v.json into # _delta_log/_staged_commits/$v.$uuid.json, where $uuid is taken from the commitInfo.txnId in # $v.json ================================================ FILE: kernel/kernel-defaults/src/test/resources/column-mapping-id/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1681169404146,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"20","numOutputBytes":"1470"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.3.0-SNAPSHOT","txnId":"21c53b30-edd8-481b-9e07-7d37b04fd514"}} {"protocol":{"minReaderVersion":2,"minWriterVersion":5}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"value\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":1,\"delta.columnMapping.physicalName\":\"col-f6363b34-a18d-4117-a4f5-cd443e9d4fda\"}}]}","partitionColumns":[],"configuration":{"delta.columnMapping.mode":"id","delta.columnMapping.maxColumnId":"1"},"createdTime":1681169403930}} {"add":{"path":"part-00000-7f7d554f-a8f2-459f-aaca-9a3b7e8af2dc-c000.snappy.parquet","partitionValues":{},"size":737,"modificationTime":1681169404000,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"col-f6363b34-a18d-4117-a4f5-cd443e9d4fda\":1},\"maxValues\":{\"col-f6363b34-a18d-4117-a4f5-cd443e9d4fda\":18},\"nullCount\":{\"col-f6363b34-a18d-4117-a4f5-cd443e9d4fda\":0}}"}} {"add":{"path":"part-00001-85082a62-baeb-46c5-8970-c9c6c23dc33c-c000.snappy.parquet","partitionValues":{},"size":733,"modificationTime":1681169404000,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"col-f6363b34-a18d-4117-a4f5-cd443e9d4fda\":0},\"maxValues\":{\"col-f6363b34-a18d-4117-a4f5-cd443e9d4fda\":19},\"nullCount\":{\"col-f6363b34-a18d-4117-a4f5-cd443e9d4fda\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/data-reader-partition-values-column-mapping-name/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1687761154342,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"as_int\",\"as_long\",\"as_byte\",\"as_short\",\"as_boolean\",\"as_float\",\"as_double\",\"as_string\",\"as_date\",\"as_timestamp\",\"as_big_decimal\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"3","numOutputRows":"3","numOutputBytes":"1896"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.3.0","txnId":"8ddbc378-38bd-4992-b394-c73162a776ec"}} {"protocol":{"minReaderVersion":2,"minWriterVersion":5}} {"metaData":{"id":"85380a11-828f-4831-b58f-219ffc825181","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"as_int\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":1,\"delta.columnMapping.physicalName\":\"col-25948c99-9f51-4e05-9f9e-b4f7042f75ed\"}},{\"name\":\"as_long\",\"type\":\"long\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":2,\"delta.columnMapping.physicalName\":\"col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa\"}},{\"name\":\"as_byte\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":3,\"delta.columnMapping.physicalName\":\"col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08\"}},{\"name\":\"as_short\",\"type\":\"short\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":4,\"delta.columnMapping.physicalName\":\"col-29f826c0-7fff-4e5f-bc11-44a6975c7708\"}},{\"name\":\"as_boolean\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":5,\"delta.columnMapping.physicalName\":\"col-7781d665-6951-4244-b9bc-a28e477e2d57\"}},{\"name\":\"as_float\",\"type\":\"float\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":6,\"delta.columnMapping.physicalName\":\"col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae\"}},{\"name\":\"as_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":7,\"delta.columnMapping.physicalName\":\"col-3463c48b-4b94-4500-b14f-4a554284b94f\"}},{\"name\":\"as_string\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":8,\"delta.columnMapping.physicalName\":\"col-05f332c4-ebdb-4437-9e80-e23f92bee4a2\"}},{\"name\":\"as_date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":9,\"delta.columnMapping.physicalName\":\"col-c025b8f8-481c-4db2-8932-f37129146ceb\"}},{\"name\":\"as_timestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":10,\"delta.columnMapping.physicalName\":\"col-bd963d5f-2199-4700-b5d6-0759bd7a9d90\"}},{\"name\":\"as_big_decimal\",\"type\":\"decimal(1,0)\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":11,\"delta.columnMapping.physicalName\":\"col-01ec4063-ed54-41db-805e-ebfd9b9a6e67\"}},{\"name\":\"value\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":12,\"delta.columnMapping.physicalName\":\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\"}}]}","partitionColumns":["as_int","as_long","as_byte","as_short","as_boolean","as_float","as_double","as_string","as_date","as_timestamp","as_big_decimal"],"configuration":{"delta.columnMapping.mode":"name","delta.columnMapping.maxColumnId":"12"},"createdTime":1687761153419}} {"add":{"path":"col-25948c99-9f51-4e05-9f9e-b4f7042f75ed=1/col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa=1/col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08=1/col-29f826c0-7fff-4e5f-bc11-44a6975c7708=1/col-7781d665-6951-4244-b9bc-a28e477e2d57=false/col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae=1.0/col-3463c48b-4b94-4500-b14f-4a554284b94f=1.0/col-05f332c4-ebdb-4437-9e80-e23f92bee4a2=1/col-c025b8f8-481c-4db2-8932-f37129146ceb=2021-09-08/col-bd963d5f-2199-4700-b5d6-0759bd7a9d90=2021-09-08%2011%253A11%253A11/col-01ec4063-ed54-41db-805e-ebfd9b9a6e67=1/part-00000-c9d9ab23-0f5c-4a12-837f-b709c5037905.c000.snappy.parquet","partitionValues":{"col-25948c99-9f51-4e05-9f9e-b4f7042f75ed":"1","col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa":"1","col-29f826c0-7fff-4e5f-bc11-44a6975c7708":"1","col-01ec4063-ed54-41db-805e-ebfd9b9a6e67":"1","col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08":"1","col-3463c48b-4b94-4500-b14f-4a554284b94f":"1.0","col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae":"1.0","col-7781d665-6951-4244-b9bc-a28e477e2d57":"false","col-05f332c4-ebdb-4437-9e80-e23f92bee4a2":"1","col-c025b8f8-481c-4db2-8932-f37129146ceb":"2021-09-08","col-bd963d5f-2199-4700-b5d6-0759bd7a9d90":"2021-09-08 11:11:11"},"size":632,"modificationTime":1687761154332,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":\"1\"},\"maxValues\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":\"1\"},\"nullCount\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":0}}"}} {"add":{"path":"col-25948c99-9f51-4e05-9f9e-b4f7042f75ed=__HIVE_DEFAULT_PARTITION__/col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa=__HIVE_DEFAULT_PARTITION__/col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08=__HIVE_DEFAULT_PARTITION__/col-29f826c0-7fff-4e5f-bc11-44a6975c7708=__HIVE_DEFAULT_PARTITION__/col-7781d665-6951-4244-b9bc-a28e477e2d57=__HIVE_DEFAULT_PARTITION__/col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae=__HIVE_DEFAULT_PARTITION__/col-3463c48b-4b94-4500-b14f-4a554284b94f=__HIVE_DEFAULT_PARTITION__/col-05f332c4-ebdb-4437-9e80-e23f92bee4a2=__HIVE_DEFAULT_PARTITION__/col-c025b8f8-481c-4db2-8932-f37129146ceb=__HIVE_DEFAULT_PARTITION__/col-bd963d5f-2199-4700-b5d6-0759bd7a9d90=__HIVE_DEFAULT_PARTITION__/col-01ec4063-ed54-41db-805e-ebfd9b9a6e67=__HIVE_DEFAULT_PARTITION__/part-00001-dac9e981-94cc-4dc5-8d01-2cbae8ef69c6.c000.snappy.parquet","partitionValues":{"col-25948c99-9f51-4e05-9f9e-b4f7042f75ed":null,"col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa":null,"col-29f826c0-7fff-4e5f-bc11-44a6975c7708":null,"col-01ec4063-ed54-41db-805e-ebfd9b9a6e67":null,"col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08":null,"col-3463c48b-4b94-4500-b14f-4a554284b94f":null,"col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae":null,"col-7781d665-6951-4244-b9bc-a28e477e2d57":null,"col-05f332c4-ebdb-4437-9e80-e23f92bee4a2":null,"col-c025b8f8-481c-4db2-8932-f37129146ceb":null,"col-bd963d5f-2199-4700-b5d6-0759bd7a9d90":null},"size":632,"modificationTime":1687761154335,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":\"2\"},\"maxValues\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":\"2\"},\"nullCount\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":0}}"}} {"add":{"path":"col-25948c99-9f51-4e05-9f9e-b4f7042f75ed=0/col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa=0/col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08=0/col-29f826c0-7fff-4e5f-bc11-44a6975c7708=0/col-7781d665-6951-4244-b9bc-a28e477e2d57=true/col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae=0.0/col-3463c48b-4b94-4500-b14f-4a554284b94f=0.0/col-05f332c4-ebdb-4437-9e80-e23f92bee4a2=0/col-c025b8f8-481c-4db2-8932-f37129146ceb=2021-09-08/col-bd963d5f-2199-4700-b5d6-0759bd7a9d90=2021-09-08%2011%253A11%253A11/col-01ec4063-ed54-41db-805e-ebfd9b9a6e67=0/part-00002-e0842c02-93d2-4c38-b041-fc88b581688b.c000.snappy.parquet","partitionValues":{"col-25948c99-9f51-4e05-9f9e-b4f7042f75ed":"0","col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa":"0","col-29f826c0-7fff-4e5f-bc11-44a6975c7708":"0","col-01ec4063-ed54-41db-805e-ebfd9b9a6e67":"0","col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08":"0","col-3463c48b-4b94-4500-b14f-4a554284b94f":"0.0","col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae":"0.0","col-7781d665-6951-4244-b9bc-a28e477e2d57":"true","col-05f332c4-ebdb-4437-9e80-e23f92bee4a2":"0","col-c025b8f8-481c-4db2-8932-f37129146ceb":"2021-09-08","col-bd963d5f-2199-4700-b5d6-0759bd7a9d90":"2021-09-08 11:11:11"},"size":632,"modificationTime":1687761154336,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":\"0\"},\"maxValues\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":\"0\"},\"nullCount\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/data-reader-primitives-column-mapping-name/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1687757789720,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"11","numOutputBytes":"8996"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.3.0","txnId":"95e57353-d8fc-4e4e-a7ee-0c56559054d9"}} {"protocol":{"minReaderVersion":2,"minWriterVersion":5}} {"metaData":{"id":"02a552b7-5f9f-4fef-a992-ddc436e735cf","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"as_int\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":1,\"delta.columnMapping.physicalName\":\"col-41726333-d452-47f3-b9ed-0adff6ffeabf\"}},{\"name\":\"as_long\",\"type\":\"long\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":2,\"delta.columnMapping.physicalName\":\"col-668f7731-ddc9-4523-83cd-9c33d739c7d4\"}},{\"name\":\"as_byte\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":3,\"delta.columnMapping.physicalName\":\"col-6fbaa34c-ff0e-45c3-ab66-efdf765dd606\"}},{\"name\":\"as_short\",\"type\":\"short\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":4,\"delta.columnMapping.physicalName\":\"col-bbfb738d-7298-4c00-a465-44b80ca3c97a\"}},{\"name\":\"as_boolean\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":5,\"delta.columnMapping.physicalName\":\"col-eded3bff-704e-4046-97e6-1395b1e38f2a\"}},{\"name\":\"as_float\",\"type\":\"float\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":6,\"delta.columnMapping.physicalName\":\"col-97d3b9ba-c7e6-4f97-8e64-b4ffd3c836ca\"}},{\"name\":\"as_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":7,\"delta.columnMapping.physicalName\":\"col-438778fc-0408-4a26-9aca-9ce1d7d3b63d\"}},{\"name\":\"as_string\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":8,\"delta.columnMapping.physicalName\":\"col-ce49aefa-a5a0-4f8f-80c1-ae3ae9918ffa\"}},{\"name\":\"as_binary\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":9,\"delta.columnMapping.physicalName\":\"col-7441db29-eefe-4fed-b11c-a0886325267e\"}},{\"name\":\"as_big_decimal\",\"type\":\"decimal(1,0)\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":10,\"delta.columnMapping.physicalName\":\"col-1dbc7e88-d12e-4e0c-b35d-1d504f71207a\"}}]}","partitionColumns":[],"configuration":{"delta.columnMapping.mode":"name","delta.columnMapping.maxColumnId":"10"},"createdTime":1687757788998}} {"add":{"path":"part-00000-dedd3195-6cd1-451d-83b8-fe0028f9b2b6-c000.snappy.parquet","partitionValues":{},"size":4542,"modificationTime":1687757789686,"dataChange":true,"stats":"{\"numRecords\":6,\"minValues\":{\"col-41726333-d452-47f3-b9ed-0adff6ffeabf\":4,\"col-668f7731-ddc9-4523-83cd-9c33d739c7d4\":4,\"col-6fbaa34c-ff0e-45c3-ab66-efdf765dd606\":4,\"col-bbfb738d-7298-4c00-a465-44b80ca3c97a\":4,\"col-97d3b9ba-c7e6-4f97-8e64-b4ffd3c836ca\":4.0,\"col-438778fc-0408-4a26-9aca-9ce1d7d3b63d\":4.0,\"col-ce49aefa-a5a0-4f8f-80c1-ae3ae9918ffa\":\"4\",\"col-1dbc7e88-d12e-4e0c-b35d-1d504f71207a\":4},\"maxValues\":{\"col-41726333-d452-47f3-b9ed-0adff6ffeabf\":9,\"col-668f7731-ddc9-4523-83cd-9c33d739c7d4\":9,\"col-6fbaa34c-ff0e-45c3-ab66-efdf765dd606\":9,\"col-bbfb738d-7298-4c00-a465-44b80ca3c97a\":9,\"col-97d3b9ba-c7e6-4f97-8e64-b4ffd3c836ca\":9.0,\"col-438778fc-0408-4a26-9aca-9ce1d7d3b63d\":9.0,\"col-ce49aefa-a5a0-4f8f-80c1-ae3ae9918ffa\":\"9\",\"col-1dbc7e88-d12e-4e0c-b35d-1d504f71207a\":9},\"nullCount\":{\"col-41726333-d452-47f3-b9ed-0adff6ffeabf\":0,\"col-668f7731-ddc9-4523-83cd-9c33d739c7d4\":0,\"col-6fbaa34c-ff0e-45c3-ab66-efdf765dd606\":0,\"col-bbfb738d-7298-4c00-a465-44b80ca3c97a\":0,\"col-eded3bff-704e-4046-97e6-1395b1e38f2a\":0,\"col-97d3b9ba-c7e6-4f97-8e64-b4ffd3c836ca\":0,\"col-438778fc-0408-4a26-9aca-9ce1d7d3b63d\":0,\"col-ce49aefa-a5a0-4f8f-80c1-ae3ae9918ffa\":0,\"col-7441db29-eefe-4fed-b11c-a0886325267e\":0,\"col-1dbc7e88-d12e-4e0c-b35d-1d504f71207a\":0}}"}} {"add":{"path":"part-00001-d8bdfc55-29fe-40bc-bfe4-f7732d559aa9-c000.snappy.parquet","partitionValues":{},"size":4454,"modificationTime":1687757789686,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"col-41726333-d452-47f3-b9ed-0adff6ffeabf\":0,\"col-668f7731-ddc9-4523-83cd-9c33d739c7d4\":0,\"col-6fbaa34c-ff0e-45c3-ab66-efdf765dd606\":0,\"col-bbfb738d-7298-4c00-a465-44b80ca3c97a\":0,\"col-97d3b9ba-c7e6-4f97-8e64-b4ffd3c836ca\":0.0,\"col-438778fc-0408-4a26-9aca-9ce1d7d3b63d\":0.0,\"col-ce49aefa-a5a0-4f8f-80c1-ae3ae9918ffa\":\"0\",\"col-1dbc7e88-d12e-4e0c-b35d-1d504f71207a\":0},\"maxValues\":{\"col-41726333-d452-47f3-b9ed-0adff6ffeabf\":3,\"col-668f7731-ddc9-4523-83cd-9c33d739c7d4\":3,\"col-6fbaa34c-ff0e-45c3-ab66-efdf765dd606\":3,\"col-bbfb738d-7298-4c00-a465-44b80ca3c97a\":3,\"col-97d3b9ba-c7e6-4f97-8e64-b4ffd3c836ca\":3.0,\"col-438778fc-0408-4a26-9aca-9ce1d7d3b63d\":3.0,\"col-ce49aefa-a5a0-4f8f-80c1-ae3ae9918ffa\":\"3\",\"col-1dbc7e88-d12e-4e0c-b35d-1d504f71207a\":3},\"nullCount\":{\"col-41726333-d452-47f3-b9ed-0adff6ffeabf\":1,\"col-668f7731-ddc9-4523-83cd-9c33d739c7d4\":1,\"col-6fbaa34c-ff0e-45c3-ab66-efdf765dd606\":1,\"col-bbfb738d-7298-4c00-a465-44b80ca3c97a\":1,\"col-eded3bff-704e-4046-97e6-1395b1e38f2a\":1,\"col-97d3b9ba-c7e6-4f97-8e64-b4ffd3c836ca\":1,\"col-438778fc-0408-4a26-9aca-9ce1d7d3b63d\":1,\"col-ce49aefa-a5a0-4f8f-80c1-ae3ae9918ffa\":1,\"col-7441db29-eefe-4fed-b11c-a0886325267e\":1,\"col-1dbc7e88-d12e-4e0c-b35d-1d504f71207a\":1}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/json-files/1.json ================================================ {"path":"part-00000-d83dafd8-c344-49f0-ab1c-acd944e32493-c000.snappy.parquet","partitionValues":{},"size":348,"modificationTime":1603723974000,"dataChange":true} {"path":"part-00000-cb078bc1-0aeb-46ed-9cf8-74a843b32c8c-c000.snappy.parquet","partitionValues":{},"size":687,"modificationTime":1603723972000,"dataChange":true} {"path":"part-00001-9bf4b8f8-1b95-411b-bf10-28dc03aa9d2f-c000.snappy.parquet","partitionValues":{},"size":705,"modificationTime":1603723972000,"dataChange":true} ================================================ FILE: kernel/kernel-defaults/src/test/resources/json-files/2.json ================================================ {"path":"part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet","partitionValues":{},"size":650,"modificationTime":1603723967000,"dataChange":true} {"path":"part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet","partitionValues":{},"size":650,"modificationTime":1603723967000,"dataChange":true} ================================================ FILE: kernel/kernel-defaults/src/test/resources/json-files/3.json ================================================ {"path":"part-00000-842017c2-3e02-44b5-a3d6-5b9ae1745045-c000.snappy.parquet","partitionValues":{},"size":649,"modificationTime":1603723970000,"dataChange":true} {"path":"part-00001-e62ca5a1-923c-4ee6-998b-c61d1cfb0b1c-c000.snappy.parquet","partitionValues":{},"size":649,"modificationTime":1603723970000,"dataChange":true} ================================================ FILE: kernel/kernel-defaults/src/test/resources/json-files-all-empty/1.json ================================================ ================================================ FILE: kernel/kernel-defaults/src/test/resources/json-files-all-empty/2.json ================================================ ================================================ FILE: kernel/kernel-defaults/src/test/resources/json-files-all-empty/3.json ================================================ ================================================ FILE: kernel/kernel-defaults/src/test/resources/json-files-with-empty/1.json ================================================ ================================================ FILE: kernel/kernel-defaults/src/test/resources/json-files-with-empty/2.json ================================================ {"path":"part-00000-d83dafd8-c344-49f0-ab1c-acd944e32493-c000.snappy.parquet","partitionValues":{},"size":348,"modificationTime":1603723974000,"dataChange":true} {"path":"part-00000-cb078bc1-0aeb-46ed-9cf8-74a843b32c8c-c000.snappy.parquet","partitionValues":{},"size":687,"modificationTime":1603723972000,"dataChange":true} {"path":"part-00001-9bf4b8f8-1b95-411b-bf10-28dc03aa9d2f-c000.snappy.parquet","partitionValues":{},"size":705,"modificationTime":1603723972000,"dataChange":true} ================================================ FILE: kernel/kernel-defaults/src/test/resources/json-files-with-empty/3.json ================================================ {"path":"part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet","partitionValues":{},"size":650,"modificationTime":1603723967000,"dataChange":true} {"path":"part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet","partitionValues":{},"size":650,"modificationTime":1603723967000,"dataChange":true} ================================================ FILE: kernel/kernel-defaults/src/test/resources/json-files-with-empty/4.json ================================================ ================================================ FILE: kernel/kernel-defaults/src/test/resources/json-files-with-empty/5.json ================================================ ================================================ FILE: kernel/kernel-defaults/src/test/resources/json-files-with-empty/6.json ================================================ ================================================ FILE: kernel/kernel-defaults/src/test/resources/json-files-with-empty/7.json ================================================ {"path":"part-00000-842017c2-3e02-44b5-a3d6-5b9ae1745045-c000.snappy.parquet","partitionValues":{},"size":649,"modificationTime":1603723970000,"dataChange":true} {"path":"part-00001-e62ca5a1-923c-4ee6-998b-c61d1cfb0b1c-c000.snappy.parquet","partitionValues":{},"size":649,"modificationTime":1603723970000,"dataChange":true} ================================================ FILE: kernel/kernel-defaults/src/test/resources/json-files-with-empty/8.json ================================================ ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-all-jsons/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1752013250800,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"5","numOutputRows":"50","numOutputBytes":"2636"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"42c3eb4a-9e6e-4c19-9893-96106670b540"}} {"metaData":{"id":"b1624866-6060-42ab-9bbc-831735240dd3","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1752013249002}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part-00000-5aead402-bb3a-4c76-9d78-67c09cfcfa8a-c000.snappy.parquet","partitionValues":{},"size":528,"modificationTime":1752013250522,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-f7ad6879-5e8e-4ac2-817f-2dc95477d7d7-c000.snappy.parquet","partitionValues":{},"size":527,"modificationTime":1752013250614,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":10},\"maxValues\":{\"id\":19},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00002-021db285-9b02-4973-a282-afcfd8afd007-c000.snappy.parquet","partitionValues":{},"size":527,"modificationTime":1752013250650,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":20},\"maxValues\":{\"id\":29},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00003-a2307510-6cac-4772-8ad1-94e874534491-c000.snappy.parquet","partitionValues":{},"size":527,"modificationTime":1752013250686,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":30},\"maxValues\":{\"id\":39},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00004-39753157-19bf-4a6d-b65d-04505516762d-c000.snappy.parquet","partitionValues":{},"size":527,"modificationTime":1752013250722,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":40},\"maxValues\":{\"id\":49},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-all-jsons/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1752013252241,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"5","numOutputRows":"50","numOutputBytes":"2636"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"f460664f-8dbf-4775-8a06-0ed95a058c6e"}} {"add":{"path":"part-00000-520b63c8-7004-4fb3-9a7d-6b1ee913d1ac-c000.snappy.parquet","partitionValues":{},"size":527,"modificationTime":1752013252130,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":50},\"maxValues\":{\"id\":59},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-5236d99c-44d0-4eb3-b9a6-c84066e73742-c000.snappy.parquet","partitionValues":{},"size":527,"modificationTime":1752013252154,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":60},\"maxValues\":{\"id\":69},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00002-113a09fa-8d92-4285-a192-75d9ef68ccdf-c000.snappy.parquet","partitionValues":{},"size":527,"modificationTime":1752013252182,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":70},\"maxValues\":{\"id\":79},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00003-d6736222-7d56-4b90-ad5e-abea2a47353d-c000.snappy.parquet","partitionValues":{},"size":528,"modificationTime":1752013252206,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":80},\"maxValues\":{\"id\":89},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00004-b0a130bc-96cd-4b89-8d24-253f9fea21cb-c000.snappy.parquet","partitionValues":{},"size":527,"modificationTime":1752013252230,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":90},\"maxValues\":{\"id\":99},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-all-jsons/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1752013252497,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"5","numOutputRows":"50","numOutputBytes":"2640"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"b8cd7f95-9f09-467c-a151-73ca0bbfa40f"}} {"add":{"path":"part-00000-8dc5e78e-a25c-47ed-8025-e171c85ace7a-c000.snappy.parquet","partitionValues":{},"size":529,"modificationTime":1752013252394,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":100},\"maxValues\":{\"id\":109},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-5109896f-2a5e-4e79-b445-8966c6c82ef0-c000.snappy.parquet","partitionValues":{},"size":529,"modificationTime":1752013252418,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":110},\"maxValues\":{\"id\":119},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00002-38299dc0-eb52-4e16-a1df-f3c6bb37eaf4-c000.snappy.parquet","partitionValues":{},"size":526,"modificationTime":1752013252442,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":120},\"maxValues\":{\"id\":129},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00003-90bfacb7-34aa-4dab-bc3d-9367d9045103-c000.snappy.parquet","partitionValues":{},"size":527,"modificationTime":1752013252462,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":130},\"maxValues\":{\"id\":139},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00004-cdc9fd11-6e89-45d0-8941-a74a104fde75-c000.snappy.parquet","partitionValues":{},"size":529,"modificationTime":1752013252486,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":140},\"maxValues\":{\"id\":149},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-all-jsons/info.txt ================================================ # Below are scala codes used to create the `kernel-pagination-all-jsons` table. // First commit: files 0-4 (5 files) spark.range(0, 50, 1, 5).write.format("delta").save(tablePath) // Second commit: files 5-9 (5 more files) spark.range(50, 100, 1, 5).write.format("delta").mode("append").save(tablePath) // Third commit: files 10-14 (5 more files) spark.range(100, 150, 1, 5).write.format("delta").mode("append").save(tablePath) ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-multi-part-checkpoints/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1752557234741,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"18","numRemovedFiles":"0","numRemovedBytes":"0","numOutputRows":"1800","numOutputBytes":"16351"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"c49abce0-3b91-4c04-8a24-bb38cf9bb311"}} {"metaData":{"id":"3a943196-646c-42a6-b7ac-e8a27c0f231a","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1752557231235}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part-00000-78e9541a-f565-4548-a474-a675a4aab37f-c000.snappy.parquet","partitionValues":{},"size":906,"modificationTime":1752557233141,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":4},\"maxValues\":{\"id\":1785},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-54a44075-ed30-4dc5-b4a2-59a91f28fd37-c000.snappy.parquet","partitionValues":{},"size":911,"modificationTime":1752557233225,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":20},\"maxValues\":{\"id\":1786},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00002-9f4f85c0-5694-4b0c-9cfb-3b616460786c-c000.snappy.parquet","partitionValues":{},"size":911,"modificationTime":1752557233265,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":16},\"maxValues\":{\"id\":1789},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00003-e33eff7b-6300-4cc5-8612-27f4959d10eb-c000.snappy.parquet","partitionValues":{},"size":908,"modificationTime":1752557233305,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":12},\"maxValues\":{\"id\":1769},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00004-a5f332bd-bf7f-4552-a0ae-011cc6888787-c000.snappy.parquet","partitionValues":{},"size":912,"modificationTime":1752557233337,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":10},\"maxValues\":{\"id\":1759},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00005-4d98fede-302c-4db9-817e-ab226e024e63-c000.snappy.parquet","partitionValues":{},"size":902,"modificationTime":1752557233369,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":59},\"maxValues\":{\"id\":1771},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00006-06a29b13-6f35-42fa-a937-1403c16c5d9f-c000.snappy.parquet","partitionValues":{},"size":912,"modificationTime":1752557233405,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":48},\"maxValues\":{\"id\":1798},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00007-6c15fb5f-ece5-4067-bff3-fe2b4481066e-c000.snappy.parquet","partitionValues":{},"size":909,"modificationTime":1752557233441,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":1797},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00008-2e216ef2-8209-4c29-a775-d1ff656485a1-c000.snappy.parquet","partitionValues":{},"size":913,"modificationTime":1752557233473,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":2},\"maxValues\":{\"id\":1799},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00009-ba5132c6-d400-4260-9b1e-59d575d34e45-c000.snappy.parquet","partitionValues":{},"size":900,"modificationTime":1752557233505,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":1753},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00010-1b506159-7065-4f32-886b-b7ac88b9f25a-c000.snappy.parquet","partitionValues":{},"size":905,"modificationTime":1752557233533,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":13},\"maxValues\":{\"id\":1777},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00011-679b09b3-3872-4ead-98a3-97ad55c83560-c000.snappy.parquet","partitionValues":{},"size":914,"modificationTime":1752557233565,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":27},\"maxValues\":{\"id\":1792},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00012-7773a168-219f-46fe-a977-807219b9facb-c000.snappy.parquet","partitionValues":{},"size":916,"modificationTime":1752557233593,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":19},\"maxValues\":{\"id\":1779},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00013-340640ec-f6d0-4ca8-a015-8b6be2045627-c000.snappy.parquet","partitionValues":{},"size":905,"modificationTime":1752557233617,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":6},\"maxValues\":{\"id\":1766},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00014-61a01309-3035-43dd-ab57-722dc7a1d6c7-c000.snappy.parquet","partitionValues":{},"size":913,"modificationTime":1752557233649,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":3},\"maxValues\":{\"id\":1796},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00015-c4272dab-2228-43fc-9a33-f778a584e0f8-c000.snappy.parquet","partitionValues":{},"size":905,"modificationTime":1752557233677,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":5},\"maxValues\":{\"id\":1788},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00016-e5846569-bace-41ec-9652-81e1a5b01b31-c000.snappy.parquet","partitionValues":{},"size":900,"modificationTime":1752557233701,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":28},\"maxValues\":{\"id\":1743},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00017-2057bacd-70e6-43b9-be0b-eb6787dfb990-c000.snappy.parquet","partitionValues":{},"size":909,"modificationTime":1752557233729,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":11},\"maxValues\":{\"id\":1795},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-multi-part-checkpoints/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1752557240968,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"100","numOutputBytes":"898"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"b94a28fc-9c29-4f85-b27d-a3b6127dab7c"}} {"add":{"path":"part-00000-99790b5f-5683-4194-8d08-b3c87c854589-c000.snappy.parquet","partitionValues":{},"size":898,"modificationTime":1752557240957,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":1000},\"maxValues\":{\"id\":1099},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-multi-part-checkpoints/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1752557241128,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"100","numOutputBytes":"896"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"4be32135-c242-4d3e-b0fd-9d0ef5e088ea"}} {"add":{"path":"part-00000-3ae77526-4e2f-4408-b0fd-769dfed78395-c000.snappy.parquet","partitionValues":{},"size":896,"modificationTime":1752557241121,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":2000},\"maxValues\":{\"id\":2099},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-multi-part-checkpoints/_delta_log/_last_checkpoint ================================================ {"version":0,"size":20,"parts":3,"sizeInBytes":46186,"numOfAddFiles":18,"checkpointSchema":{"type":"struct","fields":[{"name":"txn","type":{"type":"struct","fields":[{"name":"appId","type":"string","nullable":true,"metadata":{}},{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"lastUpdated","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"add","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}},{"name":"clusteringProvider","type":"string","nullable":true,"metadata":{}},{"name":"stats","type":"string","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"remove","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"deletionTimestamp","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"extendedFileMetadata","type":"boolean","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"metaData","type":{"type":"struct","fields":[{"name":"id","type":"string","nullable":true,"metadata":{}},{"name":"name","type":"string","nullable":true,"metadata":{}},{"name":"description","type":"string","nullable":true,"metadata":{}},{"name":"format","type":{"type":"struct","fields":[{"name":"provider","type":"string","nullable":true,"metadata":{}},{"name":"options","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"schemaString","type":"string","nullable":true,"metadata":{}},{"name":"partitionColumns","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"configuration","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"createdTime","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"protocol","type":{"type":"struct","fields":[{"name":"minReaderVersion","type":"integer","nullable":true,"metadata":{}},{"name":"minWriterVersion","type":"integer","nullable":true,"metadata":{}},{"name":"readerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"writerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"domainMetadata","type":{"type":"struct","fields":[{"name":"domain","type":"string","nullable":true,"metadata":{}},{"name":"configuration","type":"string","nullable":true,"metadata":{}},{"name":"removed","type":"boolean","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"checksum":"30afa70634ba9cf813dc0270c5c5ad03"} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-multi-part-checkpoints/info.txt ================================================ # Below are scala codes used to create the `kernel-pagination-multi-part-checkpoints` table. // Create one commit with 10 files (10 AddFile actions) spark.range(0, 1800) .repartition(18) // 10 files = 10 AddFile actions .write.format("delta").mode("overwrite").save(tablePath) // Force multi-part checkpoint creation with small part size withSQLConf( "spark.databricks.delta.checkpoint.partSize" -> "6" // 10 AddFiles → 3 checkpoint parts ) { val deltaLog = DeltaLog.forTable(spark, tablePath) deltaLog.checkpoint() // multi-part checkpoint at version 0 } // Commits 1 and 2: Add 1 file each for (i <- 1 to 2) { spark.range(i * 1000, i * 1000 + 100) // small data .coalesce(1) // 1 file .write.format("delta").mode("append").save(tablePath) } ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-single-checkpoint/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1752013810389,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"2","numRemovedFiles":"0","numRemovedBytes":"0","numOutputRows":"10","numOutputBytes":"1003"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"36c63f66-f7e7-44cb-a4cc-2a51a2bdd83d"}} {"metaData":{"id":"a0079aa7-9f6d-435b-979c-39075ebef610","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1752013807695}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"add":{"path":"part-00000-677492ad-40aa-40c6-a1f0-bf9c7dd641e2-c000.snappy.parquet","partitionValues":{},"size":500,"modificationTime":1752013809237,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-c8077b5a-da61-4f74-a958-fa885076c35b-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013809337,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":5},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-single-checkpoint/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1752013811028,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1006"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"f77d1887-7e69-45a1-8093-08081b90f025"}} {"add":{"path":"part-00000-c5b4b509-af7c-42b7-bbf5-8797cdb5eeaa-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013810989,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":10},\"maxValues\":{\"id\":14},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-7c0e1634-1491-4f2d-9eec-15950d9d900d-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013811017,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":15},\"maxValues\":{\"id\":19},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-single-checkpoint/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1752013811224,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1006"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"c0447f48-f289-4bf6-a914-0cb5fc080e02"}} {"add":{"path":"part-00000-852814f7-34c8-43da-9c22-8e5f8bd0d571-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013811189,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":20},\"maxValues\":{\"id\":24},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-8410fa8f-df52-445b-82ea-6d03366945d0-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013811217,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":25},\"maxValues\":{\"id\":29},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-single-checkpoint/_delta_log/00000000000000000003.json ================================================ {"commitInfo":{"timestamp":1752013811405,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1006"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"71adef37-5fe3-4b27-9fe7-ce19b28db68c"}} {"add":{"path":"part-00000-ed129b33-0211-4af7-bdce-432cf823c5b7-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013811373,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":30},\"maxValues\":{\"id\":34},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-7d8b44fb-445c-4e16-a1ff-75da579203db-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013811397,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":35},\"maxValues\":{\"id\":39},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-single-checkpoint/_delta_log/00000000000000000004.json ================================================ {"commitInfo":{"timestamp":1752013811603,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":3,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1006"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"39db7b90-7aa6-4500-a0bb-367a0853ca29"}} {"add":{"path":"part-00000-f54d5bbb-fe06-4c91-a1ed-500abce87546-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013811569,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":40},\"maxValues\":{\"id\":44},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-eec9d516-0606-439e-9067-4edd54ce97c6-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013811597,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":45},\"maxValues\":{\"id\":49},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-single-checkpoint/_delta_log/00000000000000000005.json ================================================ {"commitInfo":{"timestamp":1752013811776,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":4,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1006"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"09a1fec1-8c16-44e2-8f4d-7ce2e8fa256d"}} {"add":{"path":"part-00000-0c720b4a-672d-4bcc-a267-d79c3c6dbc8f-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013811745,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":50},\"maxValues\":{\"id\":54},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-827b6732-c283-437a-a9e0-95458ded5340-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013811769,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":55},\"maxValues\":{\"id\":59},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-single-checkpoint/_delta_log/00000000000000000006.json ================================================ {"commitInfo":{"timestamp":1752013811948,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":5,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1006"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"6a44dc2d-88f3-4c0e-9eef-e7211ff4aa1f"}} {"add":{"path":"part-00000-3e44afef-2600-426f-9f47-771f7ace1868-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013811917,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":60},\"maxValues\":{\"id\":64},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-af89282e-3e0f-4589-9862-274a3e343245-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013811941,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":65},\"maxValues\":{\"id\":69},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-single-checkpoint/_delta_log/00000000000000000007.json ================================================ {"commitInfo":{"timestamp":1752013812128,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":6,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1006"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"885285be-5392-414f-a196-44ca35943841"}} {"add":{"path":"part-00000-b29b2c0c-a58e-43d8-880b-9a113c27035e-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013812097,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":70},\"maxValues\":{\"id\":74},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-0cea8033-06b0-4cbf-b32b-f14cc03fae77-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013812121,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":75},\"maxValues\":{\"id\":79},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-single-checkpoint/_delta_log/00000000000000000008.json ================================================ {"commitInfo":{"timestamp":1752013812293,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":7,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1007"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"52f0e2e3-71bb-4d3a-816a-40fcbccf5326"}} {"add":{"path":"part-00000-24d7f48b-c0c4-4363-a208-2a6417e19022-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013812265,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":80},\"maxValues\":{\"id\":84},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-4014c89a-e2c8-44ea-8421-86d80922ed3d-c000.snappy.parquet","partitionValues":{},"size":504,"modificationTime":1752013812285,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":85},\"maxValues\":{\"id\":89},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-single-checkpoint/_delta_log/00000000000000000009.json ================================================ {"commitInfo":{"timestamp":1752013812464,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":8,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1006"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"b814cdd8-7a8a-4463-b581-0de0176c023d"}} {"add":{"path":"part-00000-016be37f-0c8d-470d-b7b1-160515f14dc2-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013812433,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":90},\"maxValues\":{\"id\":94},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-81d82b44-4bce-405c-b483-ff741b45d3ef-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013812457,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":95},\"maxValues\":{\"id\":99},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-single-checkpoint/_delta_log/00000000000000000010.json ================================================ {"commitInfo":{"timestamp":1752013817910,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":9,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1005"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"e8bef162-9e06-44e2-8ba5-cc8b1358061a"}} {"add":{"path":"part-00000-70b4a723-351c-4495-b4ea-088d5637d901-c000.snappy.parquet","partitionValues":{},"size":502,"modificationTime":1752013817881,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":100},\"maxValues\":{\"id\":104},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-d723c7f5-e27d-4a89-b4a9-586a6c960721-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013817901,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":105},\"maxValues\":{\"id\":109},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-single-checkpoint/_delta_log/00000000000000000011.json ================================================ {"commitInfo":{"timestamp":1752013821726,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":10,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1007"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"8feb199d-27a1-4859-bf04-cd3b69c1db17"}} {"add":{"path":"part-00000-2ae23645-7b78-491f-aa29-01ba885bba82-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013821701,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":110},\"maxValues\":{\"id\":114},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-2605f1b1-7042-411e-850a-a969e55275ae-c000.snappy.parquet","partitionValues":{},"size":504,"modificationTime":1752013821721,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":115},\"maxValues\":{\"id\":119},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-single-checkpoint/_delta_log/00000000000000000012.json ================================================ {"commitInfo":{"timestamp":1752013821874,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":11,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10","numOutputBytes":"1006"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"bcabca67-f668-4e33-a51f-59f8d73f63ae"}} {"add":{"path":"part-00000-39319b4a-4a46-4698-a3f0-0b9f42ff9fad-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013821849,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":120},\"maxValues\":{\"id\":124},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-d25a786d-5ecc-4d53-98e3-b82dae5627da-c000.snappy.parquet","partitionValues":{},"size":503,"modificationTime":1752013821869,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":125},\"maxValues\":{\"id\":129},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-single-checkpoint/_delta_log/_last_checkpoint ================================================ {"version":10,"size":24,"sizeInBytes":16991,"numOfAddFiles":22,"checkpointSchema":{"type":"struct","fields":[{"name":"txn","type":{"type":"struct","fields":[{"name":"appId","type":"string","nullable":true,"metadata":{}},{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"lastUpdated","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"add","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}},{"name":"clusteringProvider","type":"string","nullable":true,"metadata":{}},{"name":"stats","type":"string","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"remove","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"deletionTimestamp","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"extendedFileMetadata","type":"boolean","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"metaData","type":{"type":"struct","fields":[{"name":"id","type":"string","nullable":true,"metadata":{}},{"name":"name","type":"string","nullable":true,"metadata":{}},{"name":"description","type":"string","nullable":true,"metadata":{}},{"name":"format","type":{"type":"struct","fields":[{"name":"provider","type":"string","nullable":true,"metadata":{}},{"name":"options","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"schemaString","type":"string","nullable":true,"metadata":{}},{"name":"partitionColumns","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"configuration","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"createdTime","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"protocol","type":{"type":"struct","fields":[{"name":"minReaderVersion","type":"integer","nullable":true,"metadata":{}},{"name":"minWriterVersion","type":"integer","nullable":true,"metadata":{}},{"name":"readerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"writerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"domainMetadata","type":{"type":"struct","fields":[{"name":"domain","type":"string","nullable":true,"metadata":{}},{"name":"configuration","type":"string","nullable":true,"metadata":{}},{"name":"removed","type":"boolean","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"checksum":"581fbce1d7a7e20ce1931e12ff67de8a"} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-single-checkpoint/info.txt ================================================ # Below are scala codes used to create the `kernel-pagination-single-checkpoint` table. // First, create 10 commits for (i <- 0 until 10) { val mode = if (i == 0) "overwrite" else "append" spark.range(i * 10, (i + 1) * 10, 1, 2) .write.format("delta").mode(mode).save(tablePath) } // Force checkpoint creation val deltaLog = DeltaLog.forTable(spark, tablePath) deltaLog.checkpoint() // Add a few more commits after checkpoint to create additional JSON files for (i <- 10 until 13) { spark.range(i * 10, (i + 1) * 10, 1, 2) .write.format("delta").mode("append").save(tablePath) } ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-v2-checkpoint-json/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1714496114594,"operation":"CREATE TABLE","operationParameters":{"partitionBy":"[]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{\"delta.checkpointInterval\":\"2\"}"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"f6282e54-afc6-4669-939b-0f8ba73062a0"}} {"metaData":{"id":"8a390218-e4ee-4341-b6de-4920e27d3f78","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2"},"createdTime":1714496114564}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-v2-checkpoint-json/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1714496114748,"operation":"SET TBLPROPERTIES","operationParameters":{"properties":"{\"delta.checkpointPolicy\":\"v2\"}"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"fddb3112-ca9b-48af-bf19-be23f1c36c22"}} {"metaData":{"id":"8a390218-e4ee-4341-b6de-4920e27d3f78","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2","delta.checkpointPolicy":"v2"},"createdTime":1714496114564}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["v2Checkpoint"],"writerFeatures":["v2Checkpoint","appendOnly","invariants"]}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-v2-checkpoint-json/_delta_log/00000000000000000002.checkpoint.6374b053-df23-479b-b2cf-c9c550132b49.json ================================================ {"checkpointMetadata":{"version":2}} {"sidecar":{"path":"00000000000000000002.checkpoint.0000000001.0000000002.bd1885fd-6ec0-4370-b0f5-43b5162fd4de.parquet","sizeInBytes":9367,"modificationTime":1714496115780}} {"sidecar":{"path":"00000000000000000002.checkpoint.0000000002.0000000002.0a8d73ee-aa83-49d0-9583-c99db75b89b2.parquet","sizeInBytes":9296,"modificationTime":1714496115788}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["v2Checkpoint"],"writerFeatures":["v2Checkpoint","appendOnly","invariants"]}} {"metaData":{"id":"8a390218-e4ee-4341-b6de-4920e27d3f78","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2","delta.checkpointPolicy":"v2"},"createdTime":1714496114564}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-v2-checkpoint-json/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1714496115090,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"4","numOutputRows":"10","numOutputBytes":"1952"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"a76e8fca-8bab-42cc-9618-77f8c536968c"}} {"add":{"path":"part-00000-240b5dd6-323b-4f74-b6bc-ab9fdcacc630-c000.snappy.parquet","partitionValues":{},"size":485,"modificationTime":1714496115046,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"id\":4},\"maxValues\":{\"id\":8},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-534ea355-2edd-4046-8d49-d932469170c7-c000.snappy.parquet","partitionValues":{},"size":496,"modificationTime":1714496115048,"dataChange":true,"stats":"{\"numRecords\":4,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00002-4438bc9d-9c60-4dd2-9343-574743ea4ca8-c000.snappy.parquet","partitionValues":{},"size":486,"modificationTime":1714496115087,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":5},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00003-ae431d66-23d5-4dc7-b961-136ce33e63da-c000.snappy.parquet","partitionValues":{},"size":485,"modificationTime":1714496115087,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"id\":2},\"maxValues\":{\"id\":6},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-v2-checkpoint-json/_delta_log/00000000000000000003.json ================================================ {"commitInfo":{"timestamp":1752616665239,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"100","numOutputBytes":"898"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"2507ea29-2720-4e92-98f9-4a251850513c"}} {"add":{"path":"part-00000-813a3813-84c9-4251-bbe6-f6502a32b833-c000.snappy.parquet","partitionValues":{},"size":898,"modificationTime":1752616665166,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":1000},\"maxValues\":{\"id\":1099},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-v2-checkpoint-json/_delta_log/00000000000000000004.checkpoint.a2670232-dd52-4e21-8ba7-1f70fe762bce.json ================================================ {"checkpointMetadata":{"version":4}} {"sidecar":{"path":"00000000000000000004.checkpoint.0000000001.0000000001.019924f2-3318-4cca-a460-b7d0b75f0d0f.parquet","sizeInBytes":9511,"modificationTime":1752616673806}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["v2Checkpoint"],"writerFeatures":["v2Checkpoint","appendOnly","invariants"]}} {"metaData":{"id":"8a390218-e4ee-4341-b6de-4920e27d3f78","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2","delta.checkpointPolicy":"v2"},"createdTime":1714496114564}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-v2-checkpoint-json/_delta_log/00000000000000000004.json ================================================ {"commitInfo":{"timestamp":1752616670215,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":3,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"100","numOutputBytes":"896"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"124835df-5eac-4b42-8ce6-969f94e839e8"}} {"add":{"path":"part-00000-9a247ca4-22bf-4173-bd69-66401dad2178-c000.snappy.parquet","partitionValues":{},"size":896,"modificationTime":1752616670206,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":2000},\"maxValues\":{\"id\":2099},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-v2-checkpoint-json/_delta_log/_last_checkpoint ================================================ {"version":4,"size":10,"sizeInBytes":10228,"numOfAddFiles":6,"v2Checkpoint":{"path":"00000000000000000004.checkpoint.a2670232-dd52-4e21-8ba7-1f70fe762bce.json","sizeInBytes":717,"modificationTime":1752616673818,"nonFileActions":[{"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["v2Checkpoint"],"writerFeatures":["v2Checkpoint","appendOnly","invariants"]}},{"metaData":{"id":"8a390218-e4ee-4341-b6de-4920e27d3f78","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2","delta.checkpointPolicy":"v2"},"createdTime":1714496114564}},{"checkpointMetadata":{"version":4}}],"sidecarFiles":[{"path":"00000000000000000004.checkpoint.0000000001.0000000001.019924f2-3318-4cca-a460-b7d0b75f0d0f.parquet","sizeInBytes":9511,"modificationTime":1752616673806}]},"checksum":"e0e16b97d85501a7f67b00c24aaa07f2"} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-v2-checkpoint-parquet/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1714496109365,"operation":"CREATE TABLE","operationParameters":{"partitionBy":"[]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{\"delta.checkpointInterval\":\"2\"}"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"7517176e-cff7-46ac-b133-3cf096e2620d"}} {"metaData":{"id":"7e2a1106-198b-4653-a612-2aa44685cb27","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2"},"createdTime":1714496109258}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-v2-checkpoint-parquet/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1714496110834,"operation":"SET TBLPROPERTIES","operationParameters":{"properties":"{\"delta.checkpointPolicy\":\"v2\"}"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"12ea26b9-c620-4104-95f6-654bcaabdda6"}} {"metaData":{"id":"7e2a1106-198b-4653-a612-2aa44685cb27","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2","delta.checkpointPolicy":"v2"},"createdTime":1714496109258}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["v2Checkpoint"],"writerFeatures":["v2Checkpoint","appendOnly","invariants"]}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-v2-checkpoint-parquet/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1714496112086,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"4","numOutputRows":"10","numOutputBytes":"1952"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"c9f86c17-1b30-44e7-873d-1e2102f54b0f"}} {"add":{"path":"part-00000-485b0fff-1c7b-4f14-92e9-a72300fcdf88-c000.snappy.parquet","partitionValues":{},"size":485,"modificationTime":1714496111974,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"id\":4},\"maxValues\":{\"id\":8},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00001-f7a80035-0622-431e-832e-a756c65cb2a5-c000.snappy.parquet","partitionValues":{},"size":496,"modificationTime":1714496111974,"dataChange":true,"stats":"{\"numRecords\":4,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00002-5754df9c-5a25-43a6-947b-f27840fddb1a-c000.snappy.parquet","partitionValues":{},"size":486,"modificationTime":1714496112068,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":5},\"nullCount\":{\"id\":0}}"}} {"add":{"path":"part-00003-6ab7bbbb-e14d-4fa3-8767-06b509e0a666-c000.snappy.parquet","partitionValues":{},"size":485,"modificationTime":1714496112071,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"id\":2},\"maxValues\":{\"id\":6},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-v2-checkpoint-parquet/_delta_log/00000000000000000003.json ================================================ {"commitInfo":{"timestamp":1752616904697,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"100","numOutputBytes":"898"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"afeb48b0-dc63-4af9-b6b9-45796b6043b9"}} {"add":{"path":"part-00000-4f78beda-ea4d-4ab3-95fc-ab68e40b3fce-c000.snappy.parquet","partitionValues":{},"size":898,"modificationTime":1752616904619,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":1000},\"maxValues\":{\"id\":1099},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-v2-checkpoint-parquet/_delta_log/00000000000000000004.checkpoint.1391a262-4df6-494d-8166-dcd139a6ba46.json ================================================ {"checkpointMetadata":{"version":4}} {"sidecar":{"path":"00000000000000000004.checkpoint.0000000001.0000000001.87b3aafd-6627-401d-b9aa-83f6b2450f0a.parquet","sizeInBytes":9518,"modificationTime":1752616913834}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["v2Checkpoint"],"writerFeatures":["v2Checkpoint","appendOnly","invariants"]}} {"metaData":{"id":"7e2a1106-198b-4653-a612-2aa44685cb27","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2","delta.checkpointPolicy":"v2"},"createdTime":1714496109258}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-v2-checkpoint-parquet/_delta_log/00000000000000000004.json ================================================ {"commitInfo":{"timestamp":1752616909723,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":3,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"100","numOutputBytes":"896"},"engineInfo":"Apache-Spark/3.5.3 Delta-Lake/3.4.0-SNAPSHOT","txnId":"b487a2c8-bf55-4773-be11-506b2d150450"}} {"add":{"path":"part-00000-38f3b7ca-0e92-449a-a2ff-0d4b7c7908f3-c000.snappy.parquet","partitionValues":{},"size":896,"modificationTime":1752616909714,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"id\":2000},\"maxValues\":{\"id\":2099},\"nullCount\":{\"id\":0}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/kernel-pagination-v2-checkpoint-parquet/_delta_log/_last_checkpoint ================================================ {"version":4,"size":10,"sizeInBytes":10235,"numOfAddFiles":6,"v2Checkpoint":{"path":"00000000000000000004.checkpoint.1391a262-4df6-494d-8166-dcd139a6ba46.json","sizeInBytes":717,"modificationTime":1752616913846,"nonFileActions":[{"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["v2Checkpoint"],"writerFeatures":["v2Checkpoint","appendOnly","invariants"]}},{"metaData":{"id":"7e2a1106-198b-4653-a612-2aa44685cb27","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2","delta.checkpointPolicy":"v2"},"createdTime":1714496109258}},{"checkpointMetadata":{"version":4}}],"sidecarFiles":[{"path":"00000000000000000004.checkpoint.0000000001.0000000001.87b3aafd-6627-401d-b9aa-83f6b2450f0a.parquet","sizeInBytes":9518,"modificationTime":1752616913834}]},"checksum":"88938ad4c49a232427f870052fb92743"} ================================================ FILE: kernel/kernel-defaults/src/test/resources/log4j2.properties ================================================ # # Copyright (2025) The Delta Lake Project Authors. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Set everything to be logged to the file target/unit-tests.log rootLogger.level = warn rootLogger.appenderRef.file.ref = ${sys:test.appender:-File} appender.file.type = File appender.file.name = File appender.file.fileName = target/unit-tests.log appender.file.append = true appender.file.layout.type = PatternLayout appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n # Tests that launch java subprocesses can set the "test.appender" system property to # "console" to avoid having the child process's logs overwrite the unit test's # log file. appender.console.type = Console appender.console.name = console appender.console.target = SYSTEM_ERR appender.console.layout.type = PatternLayout appender.console.layout.pattern = %t: %m%n ================================================ FILE: kernel/kernel-defaults/src/test/resources/spark-shredded-variant-preview-delta/_delta_log/00000000000000000000.crc ================================================ {"txnId":"e7ea145a-a509-48d3-b233-b1333e7ddb17","tableSizeBytes":14741,"numFiles":2,"numMetadata":1,"numProtocol":1,"setTransactions":[],"domainMetadata":[],"metadata":{"id":"757b2255-cffd-4165-90b4-b491beb21ba1","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_variants\",\"type\":{\"type\":\"array\",\"elementType\":\"variant\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_of_variants\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_of_variants\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"variant\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_struct_of_variants\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_of_array_of_variants\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":{\"type\":\"array\",\"elementType\":\"variant\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableVariantShredding":"true","delta.checkpointInterval":"2"},"createdTime":1747170231449},"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["variantShredding-preview","variantType"],"writerFeatures":["variantShredding-preview","variantType","appendOnly","invariants"]},"allFiles":[{"path":"test%25file%25prefix-part-00001-0ffdbb7b-af76-4d0b-9cbe-08bb2091c1c9-c000.snappy.parquet","partitionValues":{},"size":7311,"modificationTime":1747170233554,"dataChange":false,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":50},\"maxValues\":{\"id\":99},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}"},{"path":"test%25file%25prefix-part-00000-24104cdb-691b-4410-a2a1-afc84fe2ea18-c000.snappy.parquet","partitionValues":{},"size":7430,"modificationTime":1747170233554,"dataChange":false,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":49},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}"}]} ================================================ FILE: kernel/kernel-defaults/src/test/resources/spark-shredded-variant-preview-delta/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1747170234128,"operation":"CREATE OR REPLACE TABLE AS SELECT","operationParameters":{"partitionBy":"[]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{\"delta.enableVariantShredding\":\"true\",\"delta.checkpointInterval\":\"2\"}"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"2","numRemovedFiles":"0","numRemovedBytes":"0","numOutputRows":"100","numOutputBytes":"14741"},"engineInfo":"Apache-Spark/4.0.0-SNAPSHOT Delta-Lake/3.4.0-SNAPSHOT","txnId":"e7ea145a-a509-48d3-b233-b1333e7ddb17"}} {"metaData":{"id":"757b2255-cffd-4165-90b4-b491beb21ba1","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_variants\",\"type\":{\"type\":\"array\",\"elementType\":\"variant\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_of_variants\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_of_variants\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"variant\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_struct_of_variants\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_of_array_of_variants\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":{\"type\":\"array\",\"elementType\":\"variant\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableVariantShredding":"true","delta.checkpointInterval":"2"},"createdTime":1747170231449}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["variantShredding-preview","variantType"],"writerFeatures":["variantShredding-preview","variantType","appendOnly","invariants"]}} {"add":{"path":"test%25file%25prefix-part-00000-24104cdb-691b-4410-a2a1-afc84fe2ea18-c000.snappy.parquet","partitionValues":{},"size":7430,"modificationTime":1747170233554,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":49},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}"}} {"add":{"path":"test%25file%25prefix-part-00001-0ffdbb7b-af76-4d0b-9cbe-08bb2091c1c9-c000.snappy.parquet","partitionValues":{},"size":7311,"modificationTime":1747170233554,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":50},\"maxValues\":{\"id\":99},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/spark-shredded-variant-preview-delta/_delta_log/00000000000000000001.crc ================================================ {"txnId":"913a574c-1aeb-4834-82ba-2dc334dfb584","tableSizeBytes":19801,"numFiles":3,"numMetadata":1,"numProtocol":1,"setTransactions":[],"domainMetadata":[],"metadata":{"id":"757b2255-cffd-4165-90b4-b491beb21ba1","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_variants\",\"type\":{\"type\":\"array\",\"elementType\":\"variant\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_of_variants\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_of_variants\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"variant\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_struct_of_variants\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_of_array_of_variants\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":{\"type\":\"array\",\"elementType\":\"variant\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableVariantShredding":"true","delta.checkpointInterval":"2"},"createdTime":1747170231449},"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["variantShredding-preview","variantType"],"writerFeatures":["variantShredding-preview","variantType","appendOnly","invariants"]},"allFiles":[{"path":"test%25file%25prefix-part-00000-5ed80cd3-35e4-419e-bf56-e685f8634cbf-c000.snappy.parquet","partitionValues":{},"size":5060,"modificationTime":1747170236539,"dataChange":false,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":0},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}"},{"path":"test%25file%25prefix-part-00001-0ffdbb7b-af76-4d0b-9cbe-08bb2091c1c9-c000.snappy.parquet","partitionValues":{},"size":7311,"modificationTime":1747170233554,"dataChange":false,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":50},\"maxValues\":{\"id\":99},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}"},{"path":"test%25file%25prefix-part-00000-24104cdb-691b-4410-a2a1-afc84fe2ea18-c000.snappy.parquet","partitionValues":{},"size":7430,"modificationTime":1747170233554,"dataChange":false,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":49},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}"}]} ================================================ FILE: kernel/kernel-defaults/src/test/resources/spark-shredded-variant-preview-delta/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1747170236545,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"5060"},"engineInfo":"Apache-Spark/4.0.0-SNAPSHOT Delta-Lake/3.4.0-SNAPSHOT","txnId":"913a574c-1aeb-4834-82ba-2dc334dfb584"}} {"add":{"path":"test%25file%25prefix-part-00000-5ed80cd3-35e4-419e-bf56-e685f8634cbf-c000.snappy.parquet","partitionValues":{},"size":5060,"modificationTime":1747170236539,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":0},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/spark-shredded-variant-preview-delta/_delta_log/00000000000000000002.crc ================================================ {"txnId":"b08424c1-854d-443b-b615-b356164f37c5","tableSizeBytes":24861,"numFiles":4,"numMetadata":1,"numProtocol":1,"setTransactions":[],"domainMetadata":[],"metadata":{"id":"757b2255-cffd-4165-90b4-b491beb21ba1","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_variants\",\"type\":{\"type\":\"array\",\"elementType\":\"variant\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_of_variants\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_of_variants\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"variant\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_struct_of_variants\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_of_array_of_variants\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":{\"type\":\"array\",\"elementType\":\"variant\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableVariantShredding":"true","delta.checkpointInterval":"2"},"createdTime":1747170231449},"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["variantShredding-preview","variantType"],"writerFeatures":["variantShredding-preview","variantType","appendOnly","invariants"]},"allFiles":[{"path":"test%25file%25prefix-part-00000-bda6fee1-d8d4-4a8b-a1fb-eb171758ef40-c000.snappy.parquet","partitionValues":{},"size":5060,"modificationTime":1747170237486,"dataChange":false,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":1},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}"},{"path":"test%25file%25prefix-part-00000-5ed80cd3-35e4-419e-bf56-e685f8634cbf-c000.snappy.parquet","partitionValues":{},"size":5060,"modificationTime":1747170236539,"dataChange":false,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":0},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}"},{"path":"test%25file%25prefix-part-00001-0ffdbb7b-af76-4d0b-9cbe-08bb2091c1c9-c000.snappy.parquet","partitionValues":{},"size":7311,"modificationTime":1747170233554,"dataChange":false,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":50},\"maxValues\":{\"id\":99},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}"},{"path":"test%25file%25prefix-part-00000-24104cdb-691b-4410-a2a1-afc84fe2ea18-c000.snappy.parquet","partitionValues":{},"size":7430,"modificationTime":1747170233554,"dataChange":false,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":49},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}"}]} ================================================ FILE: kernel/kernel-defaults/src/test/resources/spark-shredded-variant-preview-delta/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1747170237490,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"5060"},"engineInfo":"Apache-Spark/4.0.0-SNAPSHOT Delta-Lake/3.4.0-SNAPSHOT","txnId":"b08424c1-854d-443b-b615-b356164f37c5"}} {"add":{"path":"test%25file%25prefix-part-00000-bda6fee1-d8d4-4a8b-a1fb-eb171758ef40-c000.snappy.parquet","partitionValues":{},"size":5060,"modificationTime":1747170237486,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":1},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/spark-shredded-variant-preview-delta/_delta_log/_last_checkpoint ================================================ {"version":2,"size":6,"sizeInBytes":21895,"numOfAddFiles":4,"checkpointSchema":{"type":"struct","fields":[{"name":"txn","type":{"type":"struct","fields":[{"name":"appId","type":"string","nullable":true,"metadata":{}},{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"lastUpdated","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"add","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}},{"name":"clusteringProvider","type":"string","nullable":true,"metadata":{}},{"name":"stats","type":"string","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"remove","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"deletionTimestamp","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"extendedFileMetadata","type":"boolean","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"metaData","type":{"type":"struct","fields":[{"name":"id","type":"string","nullable":true,"metadata":{}},{"name":"name","type":"string","nullable":true,"metadata":{}},{"name":"description","type":"string","nullable":true,"metadata":{}},{"name":"format","type":{"type":"struct","fields":[{"name":"provider","type":"string","nullable":true,"metadata":{}},{"name":"options","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"schemaString","type":"string","nullable":true,"metadata":{}},{"name":"partitionColumns","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"configuration","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"createdTime","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"protocol","type":{"type":"struct","fields":[{"name":"minReaderVersion","type":"integer","nullable":true,"metadata":{}},{"name":"minWriterVersion","type":"integer","nullable":true,"metadata":{}},{"name":"readerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"writerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"domainMetadata","type":{"type":"struct","fields":[{"name":"domain","type":"string","nullable":true,"metadata":{}},{"name":"configuration","type":"string","nullable":true,"metadata":{}},{"name":"removed","type":"boolean","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"checksum":"49753c4b48895f36efac7342b5db921b"} ================================================ FILE: kernel/kernel-defaults/src/test/resources/spark-shredded-variant-preview-delta/info.txt ================================================ This file contains the code used to generate this golden table "spark-variant-checkpoint" Using delta-spark 4.0, run the following scala script: val tableName = "" val query = """ with jsonStrings as ( select id, format_string('{"key": %s}', id) as jsonString from range(0, 100) ) select id, parse_json(jsonString) as v, array( parse_json(jsonString), null, parse_json(jsonString), null, parse_json(jsonString) ) as array_of_variants, named_struct('v', parse_json(jsonString)) as struct_of_variants, map( cast(id as string), parse_json(jsonString), 'nullKey', null ) as map_of_variants, array( named_struct('v', parse_json(jsonString)), named_struct('v', null), null, named_struct( 'v', parse_json(jsonString) ), null, named_struct( 'v', parse_json(jsonString) ) ) as array_of_struct_of_variants, named_struct( 'v', array( null, parse_json(jsonString) ) ) as struct_of_array_of_variants from jsonStrings """ val writeToTableSql = s""" create or replace table $tableName USING DELTA TBLPROPERTIES (delta.checkpointInterval = 2, '${DeltaConfigs.ENABLE_VARIANT_SHREDDING.key}' = 'true') """ spark.sql(s"${writeToTableSql}\n${query}") // Write two additional rows to create a checkpoint. (0 until 2).foreach { v => spark .sql(query) .where(s"id = $v") .write .format("delta") .mode("append") .insertInto(tableName) } ================================================ FILE: kernel/kernel-defaults/src/test/resources/spark-variant-checkpoint/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1723768497710,"operation":"CREATE OR REPLACE TABLE AS SELECT","operationParameters":{"partitionBy":"[]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{\"delta.checkpointInterval\":\"2\"}"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"2","numOutputRows":"100","numOutputBytes":"14767"},"engineInfo":"Apache-Spark/4.0.0-SNAPSHOT Delta-Lake/3.3.0-SNAPSHOT","txnId":"2cc10429-f586-4c74-805c-8d19fd180c87"}} {"metaData":{"id":"d7eb0848-b002-4e0b-9d8d-dd335c90946f","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_variants\",\"type\":{\"type\":\"array\",\"elementType\":\"variant\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_of_variants\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_of_variants\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"variant\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_struct_of_variants\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_of_array_of_variants\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":{\"type\":\"array\",\"elementType\":\"variant\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2"},"createdTime":1723768495302}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["variantType-preview"],"writerFeatures":["variantType-preview","appendOnly","invariants"]}} {"add":{"path":"part-00000-16c852df-ba66-4080-be25-530a05922422-c000.snappy.parquet","partitionValues":{},"size":7443,"modificationTime":1723768496908,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":49},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}"}} {"add":{"path":"part-00001-664313d3-14b4-4dbf-8110-77001b877182-c000.snappy.parquet","partitionValues":{},"size":7324,"modificationTime":1723768496908,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":50},\"maxValues\":{\"id\":99},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/spark-variant-checkpoint/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1723768498557,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"5072"},"engineInfo":"Apache-Spark/4.0.0-SNAPSHOT Delta-Lake/3.3.0-SNAPSHOT","txnId":"78417efa-a13f-45df-add0-f96aa113fd68"}} {"add":{"path":"part-00000-9a9c570c-ee32-4322-ad2f-8c837a77d398-c000.snappy.parquet","partitionValues":{},"size":5072,"modificationTime":1723768498551,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":0},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/spark-variant-checkpoint/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1723768498990,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"5072"},"engineInfo":"Apache-Spark/4.0.0-SNAPSHOT Delta-Lake/3.3.0-SNAPSHOT","txnId":"d90393d5-9cdd-40f1-8861-121f2169808b"}} {"add":{"path":"part-00000-1e14ba22-3114-46d1-96fb-48b4912507ce-c000.snappy.parquet","partitionValues":{},"size":5072,"modificationTime":1723768498986,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":1},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}"}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/spark-variant-checkpoint/_delta_log/_last_checkpoint ================================================ {"version":2,"size":6,"sizeInBytes":21929,"numOfAddFiles":4,"checkpointSchema":{"type":"struct","fields":[{"name":"txn","type":{"type":"struct","fields":[{"name":"appId","type":"string","nullable":true,"metadata":{}},{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"lastUpdated","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"add","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}},{"name":"clusteringProvider","type":"string","nullable":true,"metadata":{}},{"name":"stats","type":"string","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"remove","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"deletionTimestamp","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"extendedFileMetadata","type":"boolean","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"metaData","type":{"type":"struct","fields":[{"name":"id","type":"string","nullable":true,"metadata":{}},{"name":"name","type":"string","nullable":true,"metadata":{}},{"name":"description","type":"string","nullable":true,"metadata":{}},{"name":"format","type":{"type":"struct","fields":[{"name":"provider","type":"string","nullable":true,"metadata":{}},{"name":"options","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"schemaString","type":"string","nullable":true,"metadata":{}},{"name":"partitionColumns","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"configuration","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"createdTime","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"protocol","type":{"type":"struct","fields":[{"name":"minReaderVersion","type":"integer","nullable":true,"metadata":{}},{"name":"minWriterVersion","type":"integer","nullable":true,"metadata":{}},{"name":"readerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"writerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"domainMetadata","type":{"type":"struct","fields":[{"name":"domain","type":"string","nullable":true,"metadata":{}},{"name":"configuration","type":"string","nullable":true,"metadata":{}},{"name":"removed","type":"boolean","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"checksum":"a8d400a03ead8a86dbb412f2a693e26e"} ================================================ FILE: kernel/kernel-defaults/src/test/resources/spark-variant-checkpoint/info.txt ================================================ This file contains the code used to generate this golden table "spark-variant-checkpoint" Using delta-spark 4.0, run the following scala script: val tableName = "" val query = """ with jsonStrings as ( select id, format_string('{"key": %s}', id) as jsonString from range(0, 100) ) select id, parse_json(jsonString) as v, array( parse_json(jsonString), null, parse_json(jsonString), null, parse_json(jsonString) ) as array_of_variants, named_struct('v', parse_json(jsonString)) as struct_of_variants, map( cast(id as string), parse_json(jsonString), 'nullKey', null ) as map_of_variants, array( named_struct('v', parse_json(jsonString)), named_struct('v', null), null, named_struct( 'v', parse_json(jsonString) ), null, named_struct( 'v', parse_json(jsonString) ) ) as array_of_struct_of_variants, named_struct( 'v', array( null, parse_json(jsonString) ) ) as struct_of_array_of_variants from jsonStrings """ val writeToTableSql = s""" create or replace table $tableName USING DELTA TBLPROPERTIES (delta.checkpointInterval = 2) """ spark.sql(s"${writeToTableSql}\n${query}") // Write two additional rows to create a checkpoint. (0 until 2).foreach { v => spark .sql(query) .where(s"id = $v") .write .format("delta") .mode("append") .insertInto(tableName) } ================================================ FILE: kernel/kernel-defaults/src/test/resources/spark-variant-stable-feature-checkpoint/_delta_log/00000000000000000000.crc ================================================ {"txnId":"7dddb463-9062-4c74-a5e6-2b0866c16b00","tableSizeBytes":333867,"numFiles":2,"numMetadata":1,"numProtocol":1,"setTransactions":[],"domainMetadata":[],"metadata":{"id":"f1448d1b-cd82-48a2-ba5e-cffcb9fa9239","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_variants\",\"type\":{\"type\":\"array\",\"elementType\":\"variant\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_of_variants\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_of_variants\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"variant\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_struct_of_variants\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_of_array_of_variants\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":{\"type\":\"array\",\"elementType\":\"variant\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1734924468826},"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["variantType"],"writerFeatures":["variantType","appendOnly","invariants"]},"histogramOpt":{"sortedBinBoundaries":[0,8192,16384,32768,65536,131072,262144,524288,1048576,2097152,4194304,8388608,12582912,16777216,20971520,25165824,29360128,33554432,37748736,41943040,50331648,58720256,67108864,75497472,83886080,92274688,100663296,109051904,117440512,125829120,130023424,134217728,138412032,142606336,146800640,150994944,167772160,184549376,201326592,218103808,234881024,251658240,268435456,285212672,301989888,318767104,335544320,352321536,369098752,385875968,402653184,419430400,436207616,452984832,469762048,486539264,503316480,520093696,536870912,553648128,570425344,587202560,603979776,671088640,738197504,805306368,872415232,939524096,1006632960,1073741824,1140850688,1207959552,1275068416,1342177280,1409286144,1476395008,1610612736,1744830464,1879048192,2013265920,2147483648,2415919104,2684354560,2952790016,3221225472,3489660928,3758096384,4026531840,4294967296,8589934592,17179869184,34359738368,68719476736,137438953472,274877906944],"fileCounts":[0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"totalBytes":[0,0,0,0,0,333867,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"allFiles":[{"path":"test%25file%25prefix-part-00001-c7ee7ba3-625c-495b-95df-06f44ffb72c9-c000.snappy.parquet","partitionValues":{},"size":166740,"modificationTime":1734924470884,"dataChange":false,"stats":"{\"numRecords\":5000,\"minValues\":{\"id\":5000},\"maxValues\":{\"id\":9999},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}","tags":{"INSERTION_TIME":"1734924470884001","MIN_INSERTION_TIME":"1734924470884001","MAX_INSERTION_TIME":"1734924470884001","OPTIMIZE_TARGET_SIZE":"268435456"}},{"path":"test%25file%25prefix-part-00000-5f6f82ed-28c5-4f4e-b358-93904826c84d-c000.snappy.parquet","partitionValues":{},"size":167127,"modificationTime":1734924470884,"dataChange":false,"stats":"{\"numRecords\":5000,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":4999},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}","tags":{"INSERTION_TIME":"1734924470884000","MIN_INSERTION_TIME":"1734924470884000","MAX_INSERTION_TIME":"1734924470884000","OPTIMIZE_TARGET_SIZE":"268435456"}}]} ================================================ FILE: kernel/kernel-defaults/src/test/resources/spark-variant-stable-feature-checkpoint/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1734924472549,"operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"2","numOutputRows":"10000","numOutputBytes":"333867"},"tags":{"noRowsCopied":"true","restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/","txnId":"7dddb463-9062-4c74-a5e6-2b0866c16b00"}} {"metaData":{"id":"f1448d1b-cd82-48a2-ba5e-cffcb9fa9239","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_variants\",\"type\":{\"type\":\"array\",\"elementType\":\"variant\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_of_variants\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_of_variants\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"variant\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_struct_of_variants\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_of_array_of_variants\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":{\"type\":\"array\",\"elementType\":\"variant\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1734924468826}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["variantType"],"writerFeatures":["variantType","appendOnly","invariants"]}} {"add":{"path":"test%25file%25prefix-part-00000-5f6f82ed-28c5-4f4e-b358-93904826c84d-c000.snappy.parquet","partitionValues":{},"size":167127,"modificationTime":1734924470884,"dataChange":true,"stats":"{\"numRecords\":5000,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":4999},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}","tags":{"INSERTION_TIME":"1734924470884000","MIN_INSERTION_TIME":"1734924470884000","MAX_INSERTION_TIME":"1734924470884000","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"test%25file%25prefix-part-00001-c7ee7ba3-625c-495b-95df-06f44ffb72c9-c000.snappy.parquet","partitionValues":{},"size":166740,"modificationTime":1734924470884,"dataChange":true,"stats":"{\"numRecords\":5000,\"minValues\":{\"id\":5000},\"maxValues\":{\"id\":9999},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}","tags":{"INSERTION_TIME":"1734924470884001","MIN_INSERTION_TIME":"1734924470884001","MAX_INSERTION_TIME":"1734924470884001","OPTIMIZE_TARGET_SIZE":"268435456"}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/spark-variant-stable-feature-checkpoint/_delta_log/00000000000000000001.crc ================================================ {"txnId":"4004e1eb-034f-411d-9d98-742f1553ade2","tableSizeBytes":667559,"numFiles":4,"numMetadata":1,"numProtocol":1,"setTransactions":[],"domainMetadata":[],"metadata":{"id":"f1448d1b-cd82-48a2-ba5e-cffcb9fa9239","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_variants\",\"type\":{\"type\":\"array\",\"elementType\":\"variant\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_of_variants\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_of_variants\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"variant\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_struct_of_variants\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_of_array_of_variants\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":{\"type\":\"array\",\"elementType\":\"variant\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1734924468826},"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["variantType"],"writerFeatures":["variantType","appendOnly","invariants"]},"histogramOpt":{"sortedBinBoundaries":[0,8192,16384,32768,65536,131072,262144,524288,1048576,2097152,4194304,8388608,12582912,16777216,20971520,25165824,29360128,33554432,37748736,41943040,50331648,58720256,67108864,75497472,83886080,92274688,100663296,109051904,117440512,125829120,130023424,134217728,138412032,142606336,146800640,150994944,167772160,184549376,201326592,218103808,234881024,251658240,268435456,285212672,301989888,318767104,335544320,352321536,369098752,385875968,402653184,419430400,436207616,452984832,469762048,486539264,503316480,520093696,536870912,553648128,570425344,587202560,603979776,671088640,738197504,805306368,872415232,939524096,1006632960,1073741824,1140850688,1207959552,1275068416,1342177280,1409286144,1476395008,1610612736,1744830464,1879048192,2013265920,2147483648,2415919104,2684354560,2952790016,3221225472,3489660928,3758096384,4026531840,4294967296,8589934592,17179869184,34359738368,68719476736,137438953472,274877906944],"fileCounts":[0,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"totalBytes":[0,0,0,0,0,667559,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"allFiles":[{"path":"test%25file%25prefix-part-00001-95062e44-13fa-4917-b169-d289cd21c717-c000.snappy.parquet","partitionValues":{},"size":166770,"modificationTime":1734924475576,"dataChange":false,"stats":"{\"numRecords\":5000,\"minValues\":{\"id\":15000},\"maxValues\":{\"id\":19999},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}","tags":{"INSERTION_TIME":"1734924475576001","MIN_INSERTION_TIME":"1734924475576001","MAX_INSERTION_TIME":"1734924475576001","OPTIMIZE_TARGET_SIZE":"268435456"}},{"path":"test%25file%25prefix-part-00000-5f6f82ed-28c5-4f4e-b358-93904826c84d-c000.snappy.parquet","partitionValues":{},"size":167127,"modificationTime":1734924470884,"dataChange":false,"stats":"{\"numRecords\":5000,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":4999},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}","tags":{"INSERTION_TIME":"1734924470884000","MIN_INSERTION_TIME":"1734924470884000","MAX_INSERTION_TIME":"1734924470884000","OPTIMIZE_TARGET_SIZE":"268435456"}},{"path":"test%25file%25prefix-part-00001-c7ee7ba3-625c-495b-95df-06f44ffb72c9-c000.snappy.parquet","partitionValues":{},"size":166740,"modificationTime":1734924470884,"dataChange":false,"stats":"{\"numRecords\":5000,\"minValues\":{\"id\":5000},\"maxValues\":{\"id\":9999},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}","tags":{"INSERTION_TIME":"1734924470884001","MIN_INSERTION_TIME":"1734924470884001","MAX_INSERTION_TIME":"1734924470884001","OPTIMIZE_TARGET_SIZE":"268435456"}},{"path":"test%25file%25prefix-part-00000-c98a0433-2bfc-4903-9b2e-0fb34243f552-c000.snappy.parquet","partitionValues":{},"size":166922,"modificationTime":1734924475588,"dataChange":false,"stats":"{\"numRecords\":5000,\"minValues\":{\"id\":10000},\"maxValues\":{\"id\":14999},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}","tags":{"INSERTION_TIME":"1734924475576000","MIN_INSERTION_TIME":"1734924475576000","MAX_INSERTION_TIME":"1734924475576000","OPTIMIZE_TARGET_SIZE":"268435456"}}]} ================================================ FILE: kernel/kernel-defaults/src/test/resources/spark-variant-stable-feature-checkpoint/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1734924475736,"operation":"WRITE","operationParameters":{"mode":"Append","statsOnLoad":false,"partitionBy":"[]"},"readVersion":0,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"10000","numOutputBytes":"333692"},"tags":{"noRowsCopied":"true","restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/","txnId":"4004e1eb-034f-411d-9d98-742f1553ade2"}} {"add":{"path":"test%25file%25prefix-part-00000-c98a0433-2bfc-4903-9b2e-0fb34243f552-c000.snappy.parquet","partitionValues":{},"size":166922,"modificationTime":1734924475588,"dataChange":true,"stats":"{\"numRecords\":5000,\"minValues\":{\"id\":10000},\"maxValues\":{\"id\":14999},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}","tags":{"INSERTION_TIME":"1734924475576000","MIN_INSERTION_TIME":"1734924475576000","MAX_INSERTION_TIME":"1734924475576000","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"test%25file%25prefix-part-00001-95062e44-13fa-4917-b169-d289cd21c717-c000.snappy.parquet","partitionValues":{},"size":166770,"modificationTime":1734924475576,"dataChange":true,"stats":"{\"numRecords\":5000,\"minValues\":{\"id\":15000},\"maxValues\":{\"id\":19999},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}","tags":{"INSERTION_TIME":"1734924475576001","MIN_INSERTION_TIME":"1734924475576001","MAX_INSERTION_TIME":"1734924475576001","OPTIMIZE_TARGET_SIZE":"268435456"}}} ================================================ FILE: kernel/kernel-defaults/src/test/resources/spark-variant-stable-feature-checkpoint/_delta_log/info.txt ================================================ This file contains the code used to generate this golden table "spark-variant-stable-feature-checkpoint" Using delta-spark 4.0, run the following scala script: val tableName = "" val query = """ with jsonStrings as ( select id, format_string('{"key": %s}', id) as jsonString from range(0, 100) ) select id, parse_json(jsonString) as v, array( parse_json(jsonString), null, parse_json(jsonString), null, parse_json(jsonString) ) as array_of_variants, named_struct('v', parse_json(jsonString)) as struct_of_variants, map( cast(id as string), parse_json(jsonString), 'nullKey', null ) as map_of_variants, array( named_struct('v', parse_json(jsonString)), named_struct('v', null), null, named_struct( 'v', parse_json(jsonString) ), null, named_struct( 'v', parse_json(jsonString) ) ) as array_of_struct_of_variants, named_struct( 'v', array( null, parse_json(jsonString) ) ) as struct_of_array_of_variants from jsonStrings """ val writeToTableSql = s""" create or replace table $tableName USING DELTA TBLPROPERTIES (delta.checkpointInterval = 2) """ spark.sql(s"${writeToTableSql}\n${query}") // Write two additional rows to create a checkpoint. (0 until 2).foreach { v => spark .sql(query) .where(s"id = $v") .write .format("delta") .mode("append") .insertInto(tableName) } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/CheckpointV2ReadSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.io.File import scala.collection.JavaConverters._ import io.delta.kernel.defaults.engine.DefaultEngine import io.delta.kernel.defaults.utils.{AbstractTestUtils, ExpressionTestUtils, TestRow, TestUtils, TestUtilsWithLegacyKernelAPIs, TestUtilsWithTableManagerAPIs} import io.delta.kernel.expressions.Literal import io.delta.kernel.internal.{InternalScanFileUtils, SnapshotImpl} import io.delta.kernel.internal.checkpoints.CheckpointInstance import io.delta.tables.DeltaTable import org.apache.spark.sql.delta.{DeltaLog, Snapshot} import org.apache.spark.sql.delta.actions.{AddFile, Metadata, Protocol} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.FileNames import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.sql.Row import org.apache.spark.sql.types.{BooleanType, IntegerType, LongType, MapType, StringType, StructType} import org.scalatest.funsuite.AnyFunSuite class LegacyCheckpointV2ReadSuite extends AbstractCheckpointV2ReadSuite with TestUtilsWithLegacyKernelAPIs { override lazy val defaultEngine = defaultEngineBatchSize2 } class CheckpointV2ReadSuite extends AbstractCheckpointV2ReadSuite with TestUtilsWithTableManagerAPIs { override lazy val defaultEngine = defaultEngineBatchSize2 } trait AbstractCheckpointV2ReadSuite extends AnyFunSuite with ExpressionTestUtils { self: AbstractTestUtils => private final val supportedFileFormats = Seq("json", "parquet") def createSourceTable( tbl: String, path: String, partitionOrClusteringSpec: String): Unit = { spark.sql(s"CREATE TABLE $tbl (a INT, b STRING) USING delta " + s"$partitionOrClusteringSpec BY (a) LOCATION '$path' " + s"TBLPROPERTIES ('delta.checkpointInterval' = '2', 'delta.checkpointPolicy'='v2')") spark.sql(s"INSERT INTO $tbl VALUES (1, 'a'), (2, 'b')") spark.sql(s"INSERT INTO $tbl VALUES (3, 'c'), (4, 'd')") spark.sql(s"INSERT INTO $tbl VALUES (5, 'e'), (6, 'f')") } def validateSnapshot( path: String, snapshotFromSpark: Snapshot, strictFileValidation: Boolean = true, ckptVersionExpected: Option[Int] = None, expectV2CheckpointFormat: Boolean = true): Unit = { // Create a snapshot from Spark connector and from kernel. val snapshotImpl = getTableManagerAdapter.getSnapshotAtLatest(defaultEngine, path) // Validate metadata/protocol loaded correctly from top-level v2 checkpoint file. val expectedMetadataId = DeltaTable.forPath(path).detail().select("id").collect().head.getString(0) assert(snapshotImpl.getMetadata.getId == expectedMetadataId) assert(snapshotImpl.getProtocol.getMinReaderVersion == snapshotFromSpark.protocol.minReaderVersion) assert(snapshotImpl.getProtocol.getMinWriterVersion == snapshotFromSpark.protocol.minWriterVersion) assert(snapshotImpl.getProtocol.getReaderFeatures.asScala.toSet == snapshotFromSpark.protocol.readerFeatureNames) assert(snapshotImpl.getProtocol.getWriterFeatures.asScala.toSet == snapshotFromSpark.protocol.writerFeatureNames) assert(snapshotImpl.getVersion() == snapshotFromSpark.version) // Validate that snapshot read from most recent checkpoint. For most cases, given a checkpoint // interval of 2, this will be the most recent even version. val expectedV2CkptToRead = ckptVersionExpected.getOrElse(snapshotFromSpark.version - (snapshotFromSpark.version % 2)) assert(snapshotImpl.getLogSegment.getCheckpoints.asScala.map(f => FileNames.checkpointVersion(new Path(f.getPath))) .contains(expectedV2CkptToRead)) assert(snapshotImpl.getLogSegment.getCheckpoints.asScala.map(f => new CheckpointInstance(f.getPath).format == CheckpointInstance.CheckpointFormat.V2) .contains(expectV2CheckpointFormat)) // Validate AddFiles from sidecars found against Spark connector. val scan = snapshotImpl.getScanBuilder().build() val foundFiles = collectScanFileRows(scan).map(InternalScanFileUtils.getAddFileStatus).map( _.getPath.split('/').last).toSet val expectedFiles = snapshotFromSpark.allFiles.collect().map(_.toPath.toString).toSet if (strictFileValidation) { assert(foundFiles == expectedFiles) } else { assert(foundFiles.subsetOf(expectedFiles)) } } test("v2 checkpoint support") { supportedFileFormats.foreach { format => withTempDir { path => withTempTable { tbl => // Create table. withSQLConf( DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key -> format, "spark.databricks.delta.clusteredTable.enableClusteringTablePreview" -> "true") { createSourceTable(tbl, path.toString, "CLUSTER") // Insert more data to ensure multiple ColumnarBatches created. spark.createDataFrame( spark.sparkContext.parallelize(10 to 110).map(i => Row(i, i.toString)), new StructType().add("a", IntegerType).add("b", StringType)) .repartition(10) .write.format("delta").mode("append").saveAsTable(tbl) } // Validate snapshot and data. validateSnapshot(path.toString, DeltaLog.forTable(spark, path.toString).update()) checkTable( path = path.toString, expectedAnswer = spark.sql(s"SELECT * FROM $tbl").collect().map(TestRow(_))) // Remove some files from the table, then add a new one. spark.sql(s"DELETE FROM $tbl WHERE a=1 OR a=2") spark.sql(s"INSERT INTO $tbl VALUES (7, 'g'), (8, 'h')") // Validate snapshot and data. validateSnapshot(path.toString, DeltaLog.forTable(spark, path.toString).update()) checkTable( path = path.toString, expectedAnswer = spark.sql(s"SELECT * FROM $tbl").collect().map(TestRow(_))) } } } } test("v2 checkpoint support with multiple sidecars") { supportedFileFormats.foreach { format => withTempDir { path => withTempTable { tbl => // Create table. withSQLConf( DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key -> format, DeltaSQLConf.DELTA_CHECKPOINT_PART_SIZE.key -> "1", // Ensure 1 action per checkpoint. "spark.databricks.delta.clusteredTable.enableClusteringTablePreview" -> "true") { createSourceTable(tbl, path.toString, "CLUSTER") } // Validate snapshot and data. validateSnapshot(path.toString, DeltaLog.forTable(spark, path.toString).update()) checkTable( path = path.toString, expectedAnswer = (1 to 6).map(i => TestRow(i, (i - 1 + 'a').toChar.toString))) // Remove some files from the table, then add a new one. spark.sql(s"DELETE FROM $tbl WHERE a=1 OR a=2") spark.sql(s"INSERT INTO $tbl VALUES (7, 'g'), (8, 'h')") // Validate snapshot and data. validateSnapshot(path.toString, DeltaLog.forTable(spark, path.toString).update()) checkTable( path = path.toString, expectedAnswer = (3 to 8).map(i => TestRow(i, (i - 1 + 'a').toChar.toString))) } } } } test("UUID named checkpoint with actions") { withTempDir { path => // Create Delta log and a checkpoint file with actions in it. val log = DeltaLog.forTable(spark, new Path(path.toString)) new File(log.logPath.toUri).mkdirs() val metadata = Metadata( "testId", schemaString = "{\"type\":\"struct\",\"fields\":[" + "{\"name\":\"a\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}") val supportedFeatures = Set("v2Checkpoint", "appendOnly", "invariants") val protocol = Protocol(3, 7, Some(Set("v2Checkpoint")), Some(supportedFeatures)) val add = AddFile(new Path("addfile").toUri.toString, Map.empty, 100L, 10L, dataChange = true) log.startTransaction().commitManuallyWithValidation(metadata, add) log.upgradeProtocol(None, log.update(), protocol) log.checkpoint(log.update()) // Spark snapshot and files must be evaluated before renaming the checkpoint file. // This is because this checkpoint file (technically) becomes invalid, as there is no // CheckpointManifest action in it. However, because the Spark connector will place all // Add and Remove actions in the sidecar files, we must use this hack to test this // scenario. val snapshotFromSpark = DeltaLog.forTable(spark, path.toString).update() snapshotFromSpark.allFiles.collect() // Rename to UUID. val ckptPath = new Path(new File(log.logPath.toUri).listFiles().filter(f => FileNames.isCheckpointFile(new Path(f.getPath))).head.toURI) new File(ckptPath.toUri).renameTo(new File(new Path( ckptPath.getParent, ckptPath.getName .replace("checkpoint.parquet", "checkpoint.abc-def.parquet")).toUri)) // Validate snapshot. validateSnapshot(path.toString, snapshotFromSpark, ckptVersionExpected = Some(1)) } } test("compatibility checkpoint with sidecar files") { withTempDir { path => withTempTable { tbl => // Create checkpoint with sidecars. withSQLConf( DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key -> "parquet", "spark.databricks.delta.clusteredTable.enableClusteringTablePreview" -> "true") { spark.conf.set(DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key, "parquet") createSourceTable(tbl, path.toString, "CLUSTER") } // Spark snapshot and files must be evaluated before renaming the checkpoint file. val snapshotFromSpark = DeltaLog.forTable(spark, path.toString).update() snapshotFromSpark.allFiles.collect() // Rename from UUID. val ckptPath = new Path( new File(DeltaLog.forTable(spark, path.toString).logPath.toUri).listFiles() .filter(f => FileNames.isCheckpointFile(new Path(f.getPath))).head.toURI) new File(ckptPath.toUri).renameTo(new File( FileNames.checkpointFileSingular(ckptPath.getParent, 2).toUri)) // Validate snapshot and data. validateSnapshot(path.toString, snapshotFromSpark, expectV2CheckpointFormat = false) checkTable( path = path.toString, expectedAnswer = (1 to 6).map(i => TestRow(i, (i - 1 + 'a').toChar.toString))) } } } test("read from table with partition predicates") { withTempDir { path => withTempTable { tbl => // Create source table with schema (a INT, b STRING) partitioned by a. createSourceTable(tbl, path.toString, "PARTITIONED") // Read from the source table with a partition predicate and validate the results. val result = readSnapshot( latestSnapshot(path.toString), filter = greaterThan(col("a"), Literal.ofInt(3))) checkAnswer(result, Seq(TestRow(4, "d"), TestRow(5, "e"), TestRow(6, "f"))) } } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/ChecksumLogReplayMetricsTestBase.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import io.delta.kernel.defaults.utils.{AbstractTestUtils, TestUtilsWithLegacyKernelAPIs, TestUtilsWithTableManagerAPIs} /** Suite to test engine metrics when loading Protocol and Metadata through checksum files. */ class PandMCheckSumLogReplayMetricsSuite extends ChecksumLogReplayMetricsTestBase with TestUtilsWithTableManagerAPIs /** Suite to test engine metrics when loading Protocol and Metadata through checksum files. */ class LegacyPandMCheckSumLogReplayMetricsSuite extends ChecksumLogReplayMetricsTestBase with TestUtilsWithLegacyKernelAPIs /** * Base trait for testing log replay optimizations when reading tables with checksum files. * This trait contains common test setup and test cases but allows specific metadata types * to customize how they load and verify data. * * Test subclasses implement specialized behavior for loading different items: * - PandMCheckSumLogReplayMetricsSuite - tests Protocol and Metadata loading * - DomainMetadataCheckSumReplayMetricsSuite - tests Domain Metadata loading */ trait ChecksumLogReplayMetricsTestBase extends LogReplayBaseSuite { self: AbstractTestUtils => ///////////////////////// // Test Helper Methods // ///////////////////////// // Method to adjust list of versions of checkpoint file read. // For example, if crc is missing and P&M is loaded from checkpoint. // Domain metadata will load from checkpoint as well. protected def getExpectedCheckpointReadSize(size: Seq[Long]): Seq[Long] = size // When loading from a CRC at an earlier version, domain metadata and P&M use different // replay strategies. P&M replay checks the CRC eagerly at (crcVersion + 1) and returns // without opening files at or below the CRC version. Domain metadata replay instead scans // all files in reverse and breaks when version < minLogVersion, which requires reading one // batch from the file just below minLogVersion to discover its version. protected def isDomainMetadataReplay: Boolean = false /////////// // Tests // /////////// Seq(-1L, 0L, 3L, 4L).foreach { readVersion => // -1 means latest version test( s"checksum found at the read version: ${if (readVersion == -1) "latest" else readVersion}") { withTableWithCrc { (tablePath, engine) => loadPandMCheckMetrics( tablePath, engine, // shouldn't need to read commit or checkpoint files as P&M/DM are found through checksum expJsonVersionsRead = Nil, expParquetVersionsRead = Nil, expParquetReadSetSizes = Nil, expChecksumReadSet = Seq(if (readVersion == -1) 11 else readVersion), readVersion) } } } test( "checksum not found at read version and checkpoint exists at read version => use checkpoint") { withTableWithCrc { (tablePath, engine) => val checkpointVersion = 10 deleteChecksumFileForTable(tablePath, Seq(checkpointVersion)) loadPandMCheckMetrics( tablePath, engine, // 10.crc missing, 10.checkpoint.parquet exists. // Attempt to read 10.crc fails and read 10.checkpoint.parquet succeeds. expJsonVersionsRead = Nil, expParquetVersionsRead = Seq(10), expParquetReadSetSizes = getExpectedCheckpointReadSize(Seq(1)), expChecksumReadSet = Nil, version = 10) } } test( "checksum not found at read version but before and after version => use previous version") { withTableWithCrc { (tablePath, engine) => deleteChecksumFileForTable(tablePath, Seq(8)) loadPandMCheckMetrics( tablePath, engine, expJsonVersionsRead = if (isDomainMetadataReplay) Seq(8, 7) else Seq(8), expParquetVersionsRead = Nil, expParquetReadSetSizes = Nil, expChecksumReadSet = Seq(7), version = 8) } } test( "checksum missing read version & the previous version, " + "checkpoint exists the read version and the previous version => use checkpoint") { withTableWithCrc { (tablePath, engine) => val checkpointVersion = 10 deleteChecksumFileForTable(tablePath, Seq(checkpointVersion, checkpointVersion + 1)) // 11.crc, 10.crc missing, 10.checkpoint.parquet exists. // Attempt to read 11.crc fails and read 10.checkpoint.parquet and 11.json succeeds. loadPandMCheckMetrics( tablePath, engine, expJsonVersionsRead = Seq(11), expParquetVersionsRead = Seq(10), expParquetReadSetSizes = getExpectedCheckpointReadSize(Seq(1)), expChecksumReadSet = Nil, version = 11) } } test("crc found at read version and checkpoint at read version => use checksum") { withTableWithCrc { (tablePath, engine) => loadPandMCheckMetrics( tablePath, engine, expJsonVersionsRead = Nil, expParquetVersionsRead = Nil, expParquetReadSetSizes = Nil, expChecksumReadSet = Seq(10), version = 10) } } test("checksum not found at the read version, but found at a previous version") { withTableWithCrc { (tablePath, engine) => deleteChecksumFileForTable(tablePath, Seq(10, 11, 5, 6)) loadPandMCheckMetrics( tablePath, engine, expJsonVersionsRead = Seq(11), expParquetVersionsRead = Seq(10), expParquetReadSetSizes = getExpectedCheckpointReadSize(Seq(1)), expChecksumReadSet = Nil) loadPandMCheckMetrics( tablePath, engine, // We find the checksum from crc at version 4, but still read commit files 5 and 6 // to find the P&M which could have been updated in version 5 and 6. expJsonVersionsRead = if (isDomainMetadataReplay) Seq(6, 5, 4) else Seq(6, 5), expParquetVersionsRead = Nil, expParquetReadSetSizes = Nil, expChecksumReadSet = Seq(4), version = 6) // now try to load version 3 and it should get P&M from checksum files only loadPandMCheckMetrics( tablePath, engine, // We find the checksum from crc at version 3, so shouldn't read anything else expJsonVersionsRead = Nil, expParquetVersionsRead = Nil, expParquetReadSetSizes = Nil, expChecksumReadSet = Seq(3), version = 3) } } test( "checksum missing read version, " + "both checksum and checkpoint exist the read version the previous version => use checksum") { withTableWithCrc { (tablePath, engine) => val checkpointVersion = 10 val readVersion = checkpointVersion + 1 deleteChecksumFileForTable(tablePath, Seq(checkpointVersion + 1)) // 11.crc missing, 10.crc and 10.checkpoint.parquet exist. // read 10.crc and 11.json. loadPandMCheckMetrics( tablePath, engine, expJsonVersionsRead = Seq(readVersion), expParquetVersionsRead = if (isDomainMetadataReplay) Seq(checkpointVersion.toLong) else Nil, expParquetReadSetSizes = if (isDomainMetadataReplay) Seq(1L) else Nil, expChecksumReadSet = Seq(checkpointVersion), version = readVersion) } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/ChecksumSimpleComparisonSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.io.File import java.nio.file.Files import java.util import java.util.Optional import scala.collection.immutable.Seq import scala.jdk.CollectionConverters._ import io.delta.kernel.{Operation, Table} import io.delta.kernel.data.Row import io.delta.kernel.defaults.utils.{TestUtils, WriteUtils} import io.delta.kernel.engine.Engine import io.delta.kernel.hook.PostCommitHook.PostCommitHookType import io.delta.kernel.internal.DeltaLogActionUtils.DeltaAction import io.delta.kernel.internal.TableImpl import io.delta.kernel.internal.actions.{AddFile, Metadata, SingleAction} import io.delta.kernel.internal.checksum.{ChecksumReader, CRCInfo} import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.util.FileNames.checksumFile import io.delta.kernel.internal.util.Utils.toCloseableIterator import io.delta.kernel.types.LongType.LONG import io.delta.kernel.types.StructType import io.delta.kernel.utils.CloseableIterable.{emptyIterable, inMemoryIterable} import io.delta.kernel.utils.FileStatus import org.apache.spark.sql.functions.col import org.scalatest.funsuite.AnyFunSuite /** * Test suite to verify checksum file correctness by comparing * Delta Spark and Delta Kernel generated checksum files. * This suite ensures that both implementations generate consistent checksums * for various table operations. */ trait ChecksumComparisonSuiteBase extends AnyFunSuite with WriteUtils with TestUtils { private val PARTITION_COLUMN = "part" protected def getPostCommitHookType: PostCommitHookType test("create table, insert data and verify checksum") { withTempDirAndEngine { (tablePath, engine) => val sparkTablePath = tablePath + "spark" val kernelTablePath = tablePath + "kernel" getCreateTxn( engine, kernelTablePath, schema = new StructType().add("id", LONG), partCols = Seq.empty).commit(engine, emptyIterable()) .getPostCommitHooks .forEach(hook => hook.threadSafeInvoke(engine)) spark.sql(s"CREATE OR REPLACE TABLE delta.`${sparkTablePath}` (id LONG) USING DELTA") assertChecksumEquals(engine, sparkTablePath, kernelTablePath, 0) (1 to 10).foreach { version => spark.range(0, version).write.format("delta").mode("append").save(sparkTablePath) commitSparkChangeToKernel(kernelTablePath, engine, sparkTablePath, version) assertChecksumEquals(engine, sparkTablePath, kernelTablePath, version) } } } test("create partitioned table, insert and verify checksum") { withTempDirAndEngine { (tablePath, engine) => val sparkTablePath = tablePath + "spark" val kernelTablePath = tablePath + "kernel" getCreateTxn( engine, kernelTablePath, schema = new StructType().add("id", LONG).add(PARTITION_COLUMN, LONG), partCols = Seq(PARTITION_COLUMN)).commit(engine, emptyIterable()) .getPostCommitHooks .forEach(hook => hook.threadSafeInvoke(engine)) spark.sql( s"CREATE OR REPLACE TABLE delta.`${sparkTablePath}` " + s"(id LONG, part LONG) USING DELTA PARTITIONED BY (part)") assertChecksumEquals(engine, sparkTablePath, kernelTablePath, 0) (1 to 10).foreach { version => spark.range(0, version).withColumn(PARTITION_COLUMN, col("id") % 2) .write.format("delta").mode("append").save(sparkTablePath) commitSparkChangeToKernel(kernelTablePath, engine, sparkTablePath, version) assertChecksumEquals(engine, sparkTablePath, kernelTablePath, version) } } } implicit class MetadataOpt(private val metadata: Metadata) { def withDeterministicIdAndCreateTime: Metadata = { new Metadata( "id", metadata.getName, metadata.getDescription, metadata.getFormat, metadata.getSchemaString, metadata.getSchema, metadata.getPartitionColumns, Optional.empty(), metadata.getConfigurationMapValue) } } implicit class CrcInfoOpt(private val crcInfo: CRCInfo) { def withoutTransactionId: CRCInfo = { new CRCInfo( crcInfo.getVersion, crcInfo.getMetadata.withDeterministicIdAndCreateTime, crcInfo.getProtocol, crcInfo.getTableSizeBytes, crcInfo.getNumFiles, Optional.empty(), // TODO: check domain metadata. Optional.empty(), // TODO: check file size histogram once https://github.com/delta-io/delta/pull/3907 merged. Optional.empty()) } } private def assertChecksumEquals( engine: Engine, sparkTablePath: String, kernelTablePath: String, version: Long): Unit = { val sparkCrcPath = buildCrcPath(sparkTablePath, version) val kernelCrcPath = buildCrcPath(kernelTablePath, version) assert( Files.exists(sparkCrcPath) && Files.exists(kernelCrcPath), s"CRC files not found for version $version") val sparkCrc = readCrcInfo(engine, sparkTablePath, version) val kernelCrc = readCrcInfo(engine, kernelTablePath, version) // Remove the randomly generated TxnId assert(sparkCrc.withoutTransactionId === kernelCrc.withoutTransactionId) } private def readCrcInfo(engine: Engine, path: String, version: Long): CRCInfo = { ChecksumReader .tryReadChecksumFile( engine, FileStatus.of(checksumFile(new Path(f"$path/_delta_log/"), version).toString)) .orElseThrow(() => new IllegalStateException(s"CRC info not found for version $version")) } // Extracts the changes from spark table and commit the exactly same change to kernel table protected def commitSparkChangeToKernel( path: String, engine: Engine, sparkTablePath: String, versionToConvert: Long): Unit = { val txn = getUpdateTxn(engine, path, logCompactionInterval = 0) // disable compaction val tableChange = Table.forPath(engine, sparkTablePath).asInstanceOf[TableImpl].getChanges( engine, versionToConvert, versionToConvert, // TODO include REMOVE action as well once we support it Set(DeltaAction.ADD).asJava) val addFilesRows = new util.ArrayList[Row]() tableChange.forEach(batch => batch.getRows.forEach(row => { val addIndex = row.getSchema.indexOf("add") if (!row.isNullAt(addIndex)) { addFilesRows.add( SingleAction.createAddFileSingleAction(new AddFile(row.getStruct(addIndex)).toRow)) } })) txn .commit(engine, inMemoryIterable(toCloseableIterator(addFilesRows.iterator()))) .getPostCommitHooks .stream().filter(_.getType == getPostCommitHookType) .forEach(_.threadSafeInvoke(engine)) } } class ChecksumSimpleComparisonSuite extends ChecksumComparisonSuiteBase { override def getPostCommitHookType : PostCommitHookType = PostCommitHookType.CHECKSUM_SIMPLE } class ChecksumFullComparisonSuite extends ChecksumComparisonSuiteBase { override def getPostCommitHookType : PostCommitHookType = PostCommitHookType.CHECKSUM_FULL override def commitSparkChangeToKernel( kernelTablePath: String, engine: Engine, sparkTablePath: String, versionToConvert: Long): Unit = { // Delete previous version's checksum to force CHECKSUM_FULL for next commit if (versionToConvert > 0) { deleteChecksumFileForTable(kernelTablePath, Seq((versionToConvert - 1).toInt)) } super.commitSparkChangeToKernel(kernelTablePath, engine, sparkTablePath, versionToConvert) } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/ChecksumStatsSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.util.{Collections, Optional} import scala.jdk.CollectionConverters._ import io.delta.kernel.{Table, Transaction, TransactionCommitResult} import io.delta.kernel.data.Row import io.delta.kernel.defaults.utils.WriteUtils import io.delta.kernel.engine.Engine import io.delta.kernel.hook.PostCommitHook.PostCommitHookType import io.delta.kernel.internal.{InternalScanFileUtils, SnapshotImpl, TableConfig, TableImpl} import io.delta.kernel.internal.actions.{AddFile, GenerateIcebergCompatActionUtils, RemoveFile} import io.delta.kernel.internal.checksum.ChecksumReader import io.delta.kernel.internal.data.TransactionStateRow import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.stats.FileSizeHistogram import io.delta.kernel.internal.util.FileNames.checksumFile import io.delta.kernel.internal.util.Utils.toCloseableIterator import io.delta.kernel.utils.{CloseableIterable, DataFileStatus, FileStatus} import io.delta.kernel.utils.CloseableIterable.inMemoryIterable import org.scalatest.funsuite.AnyFunSuite /** * Functional e2e test suite for verifying file stats collection in CRC are correct. */ trait ChecksumStatsSuiteBase extends AnyFunSuite with WriteUtils { test("Check stats in checksum are correct") { withTempDirAndEngine { (tablePath, engine) => // Currently only table with IcebergWriterCompatV1 could easily // support both add/remove files. val tableProperties = Map( TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.getKey -> "true") createEmptyTable(engine, tablePath, testSchema, tableProperties = tableProperties) val expectedFileSizeHistogram = FileSizeHistogram.createDefaultHistogram() val dataFiles = Map("file1.parquet" -> 100L, "file2.parquet" -> 100802L) addFiles(engine, tablePath, dataFiles, expectedFileSizeHistogram) checkCrcCorrect( engine, tablePath, version = 1, expectedFileCount = 2, expectedTableSize = 100902, expectedFileSizeHistogram = expectedFileSizeHistogram) removeFiles( engine, tablePath, Map("file1.parquet" -> 100), expectedFileSizeHistogram) checkCrcCorrect( engine, tablePath, version = 2, expectedFileCount = 1, expectedTableSize = 100902 - 100, expectedFileSizeHistogram = expectedFileSizeHistogram) } } /** * Verifies that the CRC information at the given version matches expectations. * * @param engine The Delta Kernel engine * @param tablePath Path to the Delta table * @param version The table version to check * @param expectedFileCount Expected number of files in the table * @param expectedTableSize Expected total size of all files in bytes * @param expectedFileSizeHistogram Expected file size histogram */ def checkCrcCorrect( engine: Engine, tablePath: String, version: Long, expectedFileCount: Long, expectedTableSize: Long, expectedFileSizeHistogram: FileSizeHistogram): Unit = { def verifyCrcExistsAndCorrect(): Unit = { val crcInfo = ChecksumReader.tryReadChecksumFile( engine, FileStatus.of(checksumFile( new Path(tablePath + "/_delta_log"), version).toString)) .orElseThrow(() => new AssertionError("CRC information should be present")) assert(crcInfo.getNumFiles === expectedFileCount) assert(crcInfo.getTableSizeBytes === expectedTableSize) assert(crcInfo.getFileSizeHistogram === Optional.of(expectedFileSizeHistogram)) } verifyCrcExistsAndCorrect() // Delete existing CRC to regenerate a new one from state construction. engine.getFileSystemClient.delete(buildCrcPath(tablePath, version).toString) Table.forPath(engine, tablePath).checksum(engine, version) verifyCrcExistsAndCorrect() } /** * Adds files to the table and updates the expected histogram. * * @param engine The Delta Kernel engine * @param tablePath Path to the Delta table * @param filesToAdd Map of file paths to their sizes * @param histogram The histogram to update with new file sizes */ protected def addFiles( engine: Engine, tablePath: String, filesToAdd: Map[String, Long], histogram: FileSizeHistogram): Unit = { val txn = getUpdateTxn(engine, tablePath, maxRetries = 0) val actionsToCommit = filesToAdd.map { case (path, size) => histogram.insert(size) GenerateIcebergCompatActionUtils.generateIcebergCompatWriterV1AddAction( txn.getTransactionState(engine), generateDataFileStatus(tablePath, path, fileSize = size), Collections.emptyMap(), true, /* dataChange */ Optional.empty()) }.toSeq commitTransaction( txn, engine, inMemoryIterable(toCloseableIterator(actionsToCommit.asJava.iterator()))) } /** * Removes files from the table and updates the expected histogram. * * @param engine The Delta Kernel engine * @param tablePath Path to the Delta table * @param filesToRemove Map of file paths to their sizes * @param histogram The histogram to update by removing file sizes */ protected def removeFiles( engine: Engine, tablePath: String, filesToRemove: Map[String, Long], histogram: FileSizeHistogram): Unit = { val txn = getUpdateTxn(engine, tablePath, maxRetries = 0) val actionsToCommit = filesToRemove.map { case (path, size) => histogram.remove(size) GenerateIcebergCompatActionUtils.generateIcebergCompatWriterV1RemoveAction( txn.getTransactionState(engine), generateDataFileStatus(tablePath, path, fileSize = size), Collections.emptyMap(), true, /* dataChange */ Optional.of(TransactionStateRow.getPhysicalSchema(txn.getTransactionState(engine)))) }.toSeq commitTransaction( txn, engine, inMemoryIterable(toCloseableIterator(actionsToCommit.asJava.iterator()))) } override def commitTransaction( txn: Transaction, engine: Engine, dataActions: CloseableIterable[Row]): TransactionCommitResult = { val result = txn.commit(engine, dataActions) // Verify that we don't have both checksum hook types val simpleHooks = result.getPostCommitHooks.stream() .filter(hook => hook.getType == PostCommitHookType.CHECKSUM_SIMPLE) .count() val fullHooks = result.getPostCommitHooks.stream() .filter(hook => hook.getType == PostCommitHookType.CHECKSUM_FULL) .count() assert( simpleHooks == 0 || fullHooks == 0, "Both CHECKSUM_SIMPLE and CHECKSUM_FULL hooks should not be present") val checksumHook = result.getPostCommitHooks.stream().filter(hook => hook.getType == getPostCommitHookType).findFirst() if (getPostCommitHookType == PostCommitHookType.CHECKSUM_SIMPLE) { assert(checksumHook.isPresent, "CHECKSUM_SIMPLE hook should be present") // When result.getVersion is 0, there will only no CHECKSUM_FULL. } else if (result.getVersion > 0) { assert(checksumHook.isPresent, "CHECKSUM_FULL hook should be present for version > 0") } checksumHook.ifPresent(_.threadSafeInvoke(engine)) result } protected def getPostCommitHookType: PostCommitHookType } class ChecksumSimpleStatsSuite extends ChecksumStatsSuiteBase { override def getPostCommitHookType: PostCommitHookType = PostCommitHookType.CHECKSUM_SIMPLE } class ChecksumFullStatsSuite extends ChecksumStatsSuiteBase { override def getPostCommitHookType: PostCommitHookType = PostCommitHookType.CHECKSUM_FULL // Delete the checksum, so that the subsequent commit will generate CHECKSUM_FULL hook. override def addFiles( engine: Engine, tablePath: String, filesToAdd: Map[String, Long], histogram: FileSizeHistogram): Unit = { val previousVersion = Table.forPath(engine, tablePath).asInstanceOf[TableImpl] .getLatestSnapshot(engine).getVersion deleteChecksumFileForTable(tablePath.stripPrefix("file:"), Seq(previousVersion.toInt)) super.addFiles(engine, tablePath, filesToAdd, histogram) } override def removeFiles( engine: Engine, tablePath: String, filesToRemove: Map[String, Long], histogram: FileSizeHistogram): Unit = { val previousVersion = Table.forPath(engine, tablePath).asInstanceOf[TableImpl].getLatestSnapshot(engine).getVersion deleteChecksumFileForTable(tablePath.stripPrefix("file:"), Seq(previousVersion.toInt)) super.removeFiles(engine, tablePath, filesToRemove, histogram) } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/ChecksumUtilsSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import scala.collection.immutable.Seq import scala.jdk.CollectionConverters._ import io.delta.kernel.Table import io.delta.kernel.data.Row import io.delta.kernel.defaults.utils.WriteUtils import io.delta.kernel.engine.Engine import io.delta.kernel.expressions.{Column, Literal} import io.delta.kernel.internal.{SnapshotImpl, TableImpl} import io.delta.kernel.internal.checksum.ChecksumUtils import io.delta.kernel.internal.util.ManualClock import io.delta.kernel.types.{StringType, StructType} import io.delta.kernel.utils.CloseableIterable.emptyIterable import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.actions.CommitInfo import org.apache.hadoop.fs.Path import org.scalatest.funsuite.AnyFunSuite /** * Test suite for io.delta.kernel.internal.checksum.ChecksumUtils */ class ChecksumUtilsSuite extends AnyFunSuite with WriteUtils with LogReplayBaseSuite { private def initialTestTable(tablePath: String, engine: Engine): Unit = { createEmptyTable(engine, tablePath, testSchema, clock = new ManualClock(0)) appendData( engine, tablePath, isNewTable = false, data = Seq(Map.empty[String, Literal] -> dataBatches1)) } test("Create checksum for different version") { withTempDirAndEngine { (tablePath, engine) => initialTestTable(tablePath, engine) val snapshot0 = Table.forPath( engine, tablePath).getSnapshotAsOfVersion(engine, 0).asInstanceOf[SnapshotImpl] ChecksumUtils.computeStateAndWriteChecksum(engine, snapshot0.getLogSegment) verifyChecksumForSnapshot(snapshot0) val snapshot1 = Table.forPath( engine, tablePath).getSnapshotAsOfVersion(engine, 1).asInstanceOf[SnapshotImpl] ChecksumUtils.computeStateAndWriteChecksum(engine, snapshot1.getLogSegment) verifyChecksumForSnapshot(snapshot1) } } test("Create checksum is idempotent") { withTempDirAndEngine { (tablePath, engine) => initialTestTable(tablePath, engine) val snapshot = Table.forPath( engine, tablePath).getSnapshotAsOfVersion(engine, 0).asInstanceOf[SnapshotImpl] // First call should create the checksum file ChecksumUtils.computeStateAndWriteChecksum(engine, snapshot.getLogSegment) verifyChecksumForSnapshot(snapshot) // Second call should be a no-op (no exception thrown) ChecksumUtils.computeStateAndWriteChecksum(engine, snapshot.getLogSegment) verifyChecksumForSnapshot(snapshot) } } test("test checksum -- no checksum, with checkpoint") { withTableWithCrc { (table, _, engine) => // Need to use HadoopFs to delete file to avoid fs throwing checksum mismatch on read. deleteChecksumFileForTableUsingHadoopFs(table.getPath(engine).stripPrefix("file:"), (0 to 11)) engine.resetMetrics() table.checksum(engine, 11) assertMetrics( engine, Seq(11), Seq(10), Seq(1), expChecksumReadSet = Nil) verifyChecksumForSnapshot(table.getSnapshotAsOfVersion(engine, 11)) } } test("test checksum -- stale checksum without file size histogram" + ", no checkpoint => fall back to full state construction") { withTableWithCrc { (table, _, engine) => deleteChecksumFileForTableUsingHadoopFs(table.getPath(engine), (5 to 8)) engine.resetMetrics() table.checksum(engine, 8) assertMetrics( engine, Seq(8, 7, 6, 5, 4, 3, 2, 1, 0), Nil, Nil, expChecksumReadSet = Seq(4)) verifyChecksumForSnapshot(table.getSnapshotAsOfVersion(engine, 8)) } } test("test checksum -- stale checksum, no checkpoint => incrementally load from checksum") { withTableWithCrc { (table, _, engine) => deleteChecksumFileForTableUsingHadoopFs(table.getPath(engine).stripPrefix("file:"), (5 to 8)) // Spark generated CRC from Spark doesn't include file size histogram, regenerate it. table.checksum(engine, 5) engine.resetMetrics() table.checksum(engine, 8) assertMetrics( engine, Seq(8, 7, 6), Nil, Nil, expChecksumReadSet = Seq(5)) verifyChecksumForSnapshot(table.getSnapshotAsOfVersion(engine, 8)) } } test("test checksum -- stale checksum, checkpoint after checksum " + "=> checkpoint with log replay") { withTableWithCrc { (table, _, engine) => deleteChecksumFileForTableUsingHadoopFs(table.getPath(engine), (5 to 11)) engine.resetMetrics() table.checksum(engine, 11) assertMetrics( engine, Seq(11), Seq(10), Seq(1), expChecksumReadSet = Nil) verifyChecksumForSnapshot(table.getSnapshotAsOfVersion(engine, 11)) } } test("test checksum -- not allowlisted operation => fallback with full state construction") { withTableWithCrc { (table, path, engine) => val deltaLog = DeltaLog.forTable(spark, new Path(path)) deltaLog .startTransaction() .commitManuallyWithValidation( CommitInfo( time = 12345, operation = "MANUAL UPDATE", inCommitTimestamp = Some(12345), operationParameters = Map.empty, commandContext = Map.empty, readVersion = Some(11), isolationLevel = None, isBlindAppend = None, operationMetrics = None, userMetadata = None, tags = None, txnId = None), deltaLog.getSnapshotAt(11).allFiles.head().copy(dataChange = false).wrap.unwrap) deleteChecksumFileForTableUsingHadoopFs( table.getPath(engine).stripPrefix("file:"), Seq(11, 12)) table.checksum(engine, 11) engine.resetMetrics() table.checksum(engine, 12) assertMetrics( engine, Seq(12, 11), Seq(10), Seq(1), // Tries to incrementally load CRC but fall back with unable to handle // Add file without data change(compute stats). expChecksumReadSet = Seq(11)) verifyChecksumForSnapshot(table.getSnapshotAsOfVersion(engine, 12)) } } test("test checksum -- removeFile without Stats => fallback with full state construction") { withTableWithCrc { (table, path, engine) => val deltaLog = DeltaLog.forTable(spark, new Path(path)) deltaLog .startTransaction() .commitManuallyWithValidation( CommitInfo( time = 12345, operation = "REPLACE TABLE", inCommitTimestamp = Some(12345), operationParameters = Map.empty, commandContext = Map.empty, readVersion = Some(11), isolationLevel = None, isBlindAppend = None, operationMetrics = None, userMetadata = None, tags = None, txnId = None), deltaLog.getSnapshotAt(11).allFiles.head().remove.copy(size = None).wrap.unwrap) // Spark generated CRC from Spark doesn't include file size histogram deleteChecksumFileForTableUsingHadoopFs( table.getPath(engine).stripPrefix("file:"), Seq(11, 12)) table.checksum(engine, 11) engine.resetMetrics() table.checksum(engine, 12) assertMetrics( engine, Seq(12, 11), Seq(10), Seq(1), // Tries to incrementally load CRC but fall back with unable to handle // Remove file without stats. expChecksumReadSet = Seq(11)) verifyChecksumForSnapshot(table.getSnapshotAsOfVersion(engine, 12)) } } test("test checksum -- Optimize => incrementally build from CRC") { withTableWithCrc { (table, path, engine) => spark.sql(s"OPTIMIZE delta.`$path`") // Spark generated CRC from Spark doesn't include file size histogram deleteChecksumFileForTableUsingHadoopFs( table.getPath(engine).stripPrefix("file:"), Seq(11, 12)) table.checksum(engine, 11) engine.resetMetrics() table.checksum(engine, 12) assertMetrics( engine, Seq(12), Nil, Nil, expChecksumReadSet = Seq(11)) verifyChecksumForSnapshot(table.getSnapshotAsOfVersion(engine, 12)) } } test("test checksum -- replace table updating p&m, domain metadata is picked up") { withTableWithCrc { (table, path, engine) => // Spark generated CRC from Spark doesn't include file size histogram deleteChecksumFileForTableUsingHadoopFs( table.getPath(engine).stripPrefix("file:"), Seq(11)) table.checksum(engine, 11) getReplaceTxn( engine, path, new StructType().add( "a", StringType.STRING), clusteringColsOpt = Some(Seq(new Column("a"))), withDomainMetadataSupported = true).commit(engine, emptyIterable[Row]) engine.resetMetrics() table.checksum(engine, 12) assertMetrics( engine, Seq(12), Nil, Nil, expChecksumReadSet = Seq(11)) verifyChecksumForSnapshot(table.getSnapshotAsOfVersion(engine, 12)) } } test("test checksum -- commit info not in the first action => " + "fallback with full state construction") { withTableWithCrc { (table, path, engine) => val deltaLog = DeltaLog.forTable(spark, new Path(path)) deltaLog .startTransaction() .commitManuallyWithValidation( deltaLog.getSnapshotAt(11).allFiles.head().remove.wrap.unwrap, CommitInfo( time = 12345, operation = "REPLACE TABLE", inCommitTimestamp = Some(12345), operationParameters = Map.empty, commandContext = Map.empty, readVersion = Some(11), isolationLevel = None, isBlindAppend = None, operationMetrics = None, userMetadata = None, tags = None, txnId = None).wrap.unwrap) // Spark generated CRC from Spark doesn't include file size histogram deleteChecksumFileForTableUsingHadoopFs( table.getPath(engine).stripPrefix("file:"), Seq(11, 12)) table.checksum(engine, 11) engine.resetMetrics() table.checksum(engine, 12) assertMetrics( engine, Seq(12, 11), Seq(10), Seq(1), expChecksumReadSet = Seq(11)) verifyChecksumForSnapshot(table.getSnapshotAsOfVersion(engine, 12)) } } test("test checksum -- commit info missing => fallback with full state construction") { withTableWithCrc { (table, path, engine) => val deltaLog = DeltaLog.forTable(spark, new Path(path)) deltaLog .startTransaction() .commitManuallyWithValidation( deltaLog.getSnapshotAt(11).allFiles.head().remove.wrap.unwrap) // Spark generated CRC from Spark doesn't include file size histogram deleteChecksumFileForTableUsingHadoopFs( table.getPath(engine).stripPrefix("file:"), Seq(11, 12)) table.checksum(engine, 11) engine.resetMetrics() table.checksum(engine, 12) assertMetrics( engine, Seq(12, 11), Seq(10), Seq(1), expChecksumReadSet = Seq(11)) verifyChecksumForSnapshot(table.getSnapshotAsOfVersion(engine, 12)) } } test("test checksum -- crc missing domain metadata => fallback with full state construction") { withTableWithCrc { (table, path, engine) => // Spark generated CRC from Spark doesn't include file size histogram deleteChecksumFileForTableUsingHadoopFs( table.getPath(engine).stripPrefix("file:"), Seq(11)) rewriteChecksumFileToExcludeDomainMetadata(engine, path, 10) engine.resetMetrics() table.checksum(engine, 11) assertMetrics( engine, Seq(11), Seq(10), Seq(1), expChecksumReadSet = Seq(10)) verifyChecksumForSnapshot(table.getSnapshotAsOfVersion(engine, 11)) } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/ColumnDefaultsSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import scala.collection.immutable.Seq import io.delta.kernel.TransactionCommitResult import io.delta.kernel.data.Row import io.delta.kernel.defaults.utils.WriteUtils import io.delta.kernel.engine.Engine import io.delta.kernel.exceptions.KernelException import io.delta.kernel.expressions.{Column, Literal} import io.delta.kernel.internal.util.Clock import io.delta.kernel.types._ import io.delta.kernel.utils.CloseableIterable.emptyIterable import org.scalatest.funsuite.AnyFunSuite class ColumnDefaultsSuite extends AnyFunSuite with WriteUtils { private val schemasWithDefaults = Seq( ( "plain", new StructType() .add("a", StringType.STRING, true, fieldMeta(1, null)) .add("b", IntegerType.INTEGER, true, fieldMeta(2, "127"))), ( "nested", new StructType() .add("a", StringType.STRING, true, fieldMeta(1, null)) .add( "nested", new StructType() .add("nested_a", IntegerType.INTEGER, true, fieldMeta(2, "100")), fieldMeta(3, null)))) private def fieldMeta(fieldId: Int, defaultVal: String) = { var builder = FieldMetadata.builder() if (fieldId != -1) { builder = builder .putLong("delta.columnMapping.id", fieldId) .putString("delta.columnMapping.physicalName", s"col-$fieldId") } if (defaultVal != null) { builder = builder.putString("CURRENT_DEFAULT", defaultVal) } builder.build() } val goodTblProperties = Map( "delta.feature.allowColumnDefaults" -> "supported", "delta.enableIcebergCompatV3" -> "true", "delta.columnMapping.mode" -> "id") schemasWithDefaults.foreach { case (schemaName, schemaWithDefault) => test(s"allow default value in schema when the table feature is enabled - $schemaName") { val goodTblProperties2 = Map( "delta.feature.allowColumnDefaults" -> "supported", "delta.enableIcebergCompatV3" -> "true") for (tableProperties <- Seq(goodTblProperties, goodTblProperties2)) { // Create table withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, schemaWithDefault, tableProperties = tableProperties) } } } test(s"block default value in schema when table features are not properly set - $schemaName") { withTempDirAndEngine { (tablePath, engine) => val e = intercept[KernelException] { createEmptyTable( engine, tablePath, schemaWithDefault, tableProperties = Map( "delta.feature.allowColumnDefaults" -> "supported")) } assert(e.getMessage == "In Delta Kernel, default values table feature requires IcebergCompatV3 to be enabled.") } withTempDirAndEngine { (tablePath, engine) => val e = intercept[KernelException] { createEmptyTable(engine, tablePath, schemaWithDefault) } assert(e.getMessage == "Found column defaults in the schema but the table does not support the " + "columnDefaults table feature.") } } test(s"block writing to tables with default values - $schemaName") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, schemaWithDefault, tableProperties = goodTblProperties) val e = intercept[UnsupportedOperationException] { appendData(engine, tablePath, data = Seq(Map.empty[String, Literal] -> dataBatches1)) } assert(e.getMessage == "Writing into column mapping enabled table is not supported yet.") } } Seq( (StringType.STRING, "CURRENT_TIMESTAMP()"), (IntegerType.INTEGER, "313.55"), (DoubleType.DOUBLE, "Good boy"), (new DecimalType(10, 5), "234243243243243234.234"), (DateType.DATE, "2022/01/05"), (TimestampType.TIMESTAMP, "2025-01-01"), (TimestampNTZType.TIMESTAMP_NTZ, "2025-01-01T00:00:00+02:00")).foreach { case (dataType, value) => test(s"block invalid default values - ($schemaName, $value)") { // Create tables val schema = new StructType().add("col1", dataType, true, fieldMeta(100, value)) withTempDirAndEngine { (tablePath, engine) => intercept[KernelException] { createEmptyTable( engine, tablePath, schema, tableProperties = goodTblProperties) } } // Schema Evolutions -> change type withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, schemaWithDefault, tableProperties = goodTblProperties) intercept[KernelException] { val schema = new StructType() .add("a", StringType.STRING, true, fieldMeta(1, null)) .add("b", IntegerType.INTEGER, true, fieldMeta(2, null)) .add("col1", dataType, true, fieldMeta(100, value)) updateTableMetadata(engine, tablePath, schema) } intercept[KernelException] { val schema = new StructType().add("col1", dataType, true, fieldMeta(3, value)) updateTableMetadata(engine, tablePath, schema) } } } } } Seq( ("remove col", new StructType().add("a", StringType.STRING, true, fieldMeta(1, null))), ( "add col", new StructType() .add("a", StringType.STRING, true, fieldMeta(1, null)) .add("add1", StringType.STRING, true, fieldMeta(7, "'Tom'")) .add("add2", DateType.DATE, true, fieldMeta(4, "2025-01-01")) .add("add3", DoubleType.DOUBLE, true, fieldMeta(5, "'3.21'")) .add("b", IntegerType.INTEGER, true, fieldMeta(6, "127"))), ( "update value", new StructType() .add("a", StringType.STRING, true, fieldMeta(1, null)) .add("b", IntegerType.INTEGER, true, fieldMeta(2, "350"))), ( "rename column", new StructType() .add("a", StringType.STRING, true, fieldMeta(1, null)) .add("xxx", IntegerType.INTEGER, true, fieldMeta(2, "350"))), ( "add renamed column", new StructType() .add("a", StringType.STRING, true, fieldMeta(1, null)) .add("b", LongType.LONG, true, fieldMeta(220, "350")))).foreach { case (name, schema) => test(s"allow valid default values - for new table & $name schema evolve") { // Create tables // TODO: reconsider removing this part, not sure what exactly the point of this test is... // (schema evolve?) withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, schema, tableProperties = goodTblProperties) } // Schema Evolutions withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, // Test this with just the plain unnested schema // TODO: in the future reconsider the point of running all the above tests with both these // schemas schemasWithDefaults(0)._2, tableProperties = goodTblProperties) updateTableMetadata(engine, tablePath, schema) } } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/CommitIcebergActionSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.util.{Collections, Optional} import java.util.Collections.emptyMap import scala.collection.JavaConverters._ import scala.collection.immutable.Seq import io.delta.kernel.{Table, Transaction} import io.delta.kernel.data.Row import io.delta.kernel.defaults.utils.{AbstractWriteUtils, WriteUtilsWithV1Builders, WriteUtilsWithV2Builders} import io.delta.kernel.engine.Engine import io.delta.kernel.exceptions.KernelException import io.delta.kernel.internal.{TableConfig, TableImpl} import io.delta.kernel.internal.DeltaLogActionUtils.DeltaAction import io.delta.kernel.internal.actions.{AddFile, DeletionVectorDescriptor, GenerateIcebergCompatActionUtils, RemoveFile} import io.delta.kernel.internal.data.TransactionStateRow import io.delta.kernel.internal.util.Utils.toCloseableIterator import io.delta.kernel.internal.util.VectorUtils import io.delta.kernel.statistics.DataFileStatistics import io.delta.kernel.utils.CloseableIterable.{emptyIterable, inMemoryIterable} import io.delta.kernel.utils.DataFileStatus import org.apache.spark.sql.delta.DeltaLog import org.scalatest.funsuite.AnyFunSuite class CommitIcebergActionTransactionBuilderV1Suite extends AbstractCommitIcebergActionSuite with WriteUtilsWithV1Builders {} class CommitIcebergActionTransactionBuilderV2Suite extends AbstractCommitIcebergActionSuite with WriteUtilsWithV2Builders {} trait AbstractCommitIcebergActionSuite extends AnyFunSuite { self: AbstractWriteUtils => private val tblPropertiesIcebergWriterCompatV1Enabled = Map( TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.getKey -> "true") private val tblPropertiesIcebergWriterCompatV3Enabled = Map( TableConfig.ICEBERG_WRITER_COMPAT_V3_ENABLED.getKey -> "true") /** Helper to create a transaction, generate a single action, and commit it. */ private def commitSingleAction( engine: Engine, tablePath: String, actionFn: Transaction => Row): Unit = { val txn = getUpdateTxn(engine, tablePath, maxRetries = 0) val action = actionFn(txn) commitTransaction( txn, engine, inMemoryIterable(toCloseableIterator(Seq(action).asJava.iterator()))) } private def createIcebergCompatAction( actionType: String, // "ADD" or "REMOVE" version: String, txn: Transaction, engine: Engine, fileStatus: DataFileStatus, dataChange: Boolean, tags: Map[String, String] = Map.empty, // ignored for REMOVE deletionVector: Optional[DeletionVectorDescriptor] = Optional.empty() // ignored for V1 ): Row = { (actionType, version) match { case ("ADD", "V1") => GenerateIcebergCompatActionUtils.generateIcebergCompatWriterV1AddAction( txn.getTransactionState(engine), fileStatus, Collections.emptyMap(), dataChange, tags.asJava, Optional.of(TransactionStateRow.getPhysicalSchema(txn.getTransactionState(engine)))) case ("ADD", "V3") => GenerateIcebergCompatActionUtils.generateIcebergCompatWriterV3AddAction( txn.getTransactionState(engine), fileStatus, Collections.emptyMap(), dataChange, tags.asJava, Optional.empty[java.lang.Long](), Optional.empty[java.lang.Long](), deletionVector, Optional.of(TransactionStateRow.getPhysicalSchema(txn.getTransactionState(engine)))) case ("REMOVE", "V1") => GenerateIcebergCompatActionUtils.generateIcebergCompatWriterV1RemoveAction( txn.getTransactionState(engine), fileStatus, Collections.emptyMap(), dataChange, Optional.of(TransactionStateRow.getPhysicalSchema(txn.getTransactionState(engine)))) case ("REMOVE", "V3") => GenerateIcebergCompatActionUtils.generateIcebergCompatWriterV3RemoveAction( txn.getTransactionState(engine), fileStatus, Collections.emptyMap(), dataChange, Optional.empty[java.lang.Long](), Optional.empty[java.lang.Long](), deletionVector, Optional.of(TransactionStateRow.getPhysicalSchema(txn.getTransactionState(engine)))) case _ => throw new IllegalArgumentException( s"Unsupported actionType: $actionType or version: $version") } } /* ----- Error cases ----- */ Seq("V1", "V3").foreach { version => test(s"$version: requires that maxRetries = 0") { withTempDirAndEngine { (tablePath, engine) => val properties = if (version == "V1") tblPropertiesIcebergWriterCompatV1Enabled else tblPropertiesIcebergWriterCompatV3Enabled val txn = getCreateTxn(engine, tablePath, testSchema, tableProperties = properties) intercept[UnsupportedOperationException] { createIcebergCompatAction( "ADD", version, txn, engine, generateDataFileStatus(tablePath, "file1.parquet"), dataChange = true) } intercept[UnsupportedOperationException] { createIcebergCompatAction( "REMOVE", version, txn, engine, generateDataFileStatus(tablePath, "file1.parquet"), dataChange = true) } } } test(s"$version: requires that icebergWriterCompat${version} enabled") { withTempDirAndEngine { (tablePath, engine) => val txn = getCreateTxn(engine, tablePath, testSchema, maxRetries = 0) intercept[UnsupportedOperationException] { createIcebergCompatAction( "ADD", version, txn, engine, generateDataFileStatus(tablePath, "file1.parquet"), dataChange = true) } intercept[UnsupportedOperationException] { createIcebergCompatAction( "REMOVE", version, txn, engine, generateDataFileStatus(tablePath, "file1.parquet"), dataChange = true) } } } test(s"$version: partitioned tables unsupported") { withTempDirAndEngine { (tablePath, engine) => val properties = if (version == "V1") tblPropertiesIcebergWriterCompatV1Enabled else tblPropertiesIcebergWriterCompatV3Enabled val txn = getCreateTxn( engine, tablePath, testPartitionSchema, testPartitionColumns, properties, maxRetries = 0) intercept[UnsupportedOperationException] { createIcebergCompatAction( "ADD", version, txn, engine, generateDataFileStatus(tablePath, "file1.parquet"), dataChange = true) } intercept[UnsupportedOperationException] { createIcebergCompatAction( "REMOVE", version, txn, engine, generateDataFileStatus(tablePath, "file1.parquet"), dataChange = true) } } } test(s"$version: cannot create add without stats present") { withTempDirAndEngine { (tablePath, engine) => val properties = if (version == "V1") tblPropertiesIcebergWriterCompatV1Enabled else tblPropertiesIcebergWriterCompatV3Enabled val txn = getCreateTxn(engine, tablePath, testSchema, tableProperties = properties, maxRetries = 0) intercept[KernelException] { createIcebergCompatAction( "ADD", version, txn, engine, generateDataFileStatus(tablePath, "file1.parquet", includeStats = false), dataChange = true) } } } } /* ----- E2E commit tests + read back with Spark ----- */ // Note - since we don't fully support column mapping writes (i.e. transformLogicalData doesn't // support id mode) we cannot really test these APIs with actual tables with data since // we cannot write column-mapping-id-based data // For now - we write the actions and check that they are correct in Spark // After we have full column mapping support we will add E2E tests with data that can be read trait ExpectedFileAction case class ExpectedAdd(path: String, size: Long, modificationTime: Long, dataChange: Boolean) extends ExpectedFileAction case class ExpectedRemove(path: String, size: Long, deletionTimestamp: Long, dataChange: Boolean) extends ExpectedFileAction private def checkActionsWrittenInJson( engine: Engine, tablePath: String, version: Long, expectedFileActions: Set[ExpectedFileAction], icebergCompatWriterVersion: String = "V1"): Unit = { val rows = Table.forPath(engine, tablePath).asInstanceOf[TableImpl] .getChanges(engine, version, version, Set(DeltaAction.ADD, DeltaAction.REMOVE).asJava) .toSeq .flatMap(_.getRows.toSeq) val fileActions = rows.flatMap { row => if (!row.isNullAt(row.getSchema.indexOf("add"))) { val addFile = new AddFile(row.getStruct(row.getSchema.indexOf("add"))) assert(addFile.getPartitionValues.getSize == 0) assert(!addFile.getTags.isPresent) if (icebergCompatWriterVersion == "V1") { assert(!addFile.getBaseRowId.isPresent) assert(!addFile.getDefaultRowCommitVersion.isPresent) } else { // V3 // In V3, baseRowId and defaultRowCommitVersion are required for row lineage assert(addFile.getBaseRowId.isPresent) assert(addFile.getDefaultRowCommitVersion.isPresent) } assert(!addFile.getDeletionVector.isPresent) assert(addFile.getStats(null).isPresent) Some(ExpectedAdd( addFile.getPath, addFile.getSize, addFile.getModificationTime, addFile.getDataChange)) } else if (!row.isNullAt(row.getSchema.indexOf("remove"))) { val removeFile = new RemoveFile(row.getStruct(row.getSchema.indexOf("remove"))) assert(removeFile.getDeletionTimestamp.isPresent) assert(removeFile.getExtendedFileMetadata.toScala.contains(true)) assert(removeFile.getPartitionValues.toScala.exists(_.getSize == 0)) assert(removeFile.getSize.isPresent) assert(removeFile.getStats(null).isPresent) assert(!removeFile.getTags.isPresent) assert(!removeFile.getDeletionVector.isPresent) assert(!removeFile.getBaseRowId.isPresent) assert(!removeFile.getDefaultRowCommitVersion.isPresent) Some(ExpectedRemove( removeFile.getPath, removeFile.getSize.get, removeFile.getDeletionTimestamp.get, removeFile.getDataChange)) } else { None } } assert(fileActions.size == expectedFileActions.size) assert(fileActions.toSet == expectedFileActions) } private def checkSparkLogReplay( tablePath: String, version: Long, expectedAdds: Set[ExpectedAdd], icebergCompatWriterVersion: String = "V1"): Unit = { val snapshot = DeltaLog.forTable(spark, tablePath).getSnapshotAt(version) assert(snapshot.allFiles.count() == expectedAdds.size) val addsFoundSet = snapshot.allFiles.collect().map { add => assert(add.partitionValues.isEmpty) assert(!add.dataChange) // Row lineage fields - different behavior for V1 vs V3 if (icebergCompatWriterVersion == "V1") { assert(add.baseRowId.isEmpty) assert(add.defaultRowCommitVersion.isEmpty) } else { // V3 assert(add.baseRowId.nonEmpty) assert(add.defaultRowCommitVersion.nonEmpty) } assert(add.deletionVector == null) assert(add.stats != null) assert(add.clusteringProvider.isEmpty) ExpectedAdd( add.path, add.size, add.modificationTime, // Always false because Delta Spark copies add with dataChange=false during log replay add.dataChange) }.toSet // We must "hack" all the expectedAdds to have dataChange=false since Delta Spark does this // in log replay assert(addsFoundSet == expectedAdds.map(_.copy(dataChange = false))) } Seq("V1", "V3").foreach { version => test(s"$version: Correctly commits adds to table and compat with Spark") { withTempDirAndEngine { (tablePath, engine) => val properties = if (version == "V1") tblPropertiesIcebergWriterCompatV1Enabled else tblPropertiesIcebergWriterCompatV3Enabled // Create table createEmptyTable( engine, tablePath, testSchema, tableProperties = properties) // Append 1 add with dataChange = true { val txn = getUpdateTxn(engine, tablePath, maxRetries = 0) val actionsToCommit = Seq( createIcebergCompatAction( "ADD", version, txn, engine, generateDataFileStatus(tablePath, "file1.parquet"), dataChange = true)) commitTransaction( txn, engine, inMemoryIterable(toCloseableIterator(actionsToCommit.asJava.iterator()))) } // Append 1 add with dataChange = false (in theory this could involve updating stats but // once we support remove add a case that looks like optimize/compaction) { val txn = getUpdateTxn(engine, tablePath, maxRetries = 0) val actionsToCommit = Seq( createIcebergCompatAction( "ADD", version, txn, engine, generateDataFileStatus(tablePath, "file1.parquet"), dataChange = false)) commitTransaction( txn, engine, inMemoryIterable(toCloseableIterator(actionsToCommit.asJava.iterator()))) } // Verify we wrote the adds we expected into the JSON files using Kernel's getChanges checkActionsWrittenInJson(engine, tablePath, 0, Set(), version) checkActionsWrittenInJson( engine, tablePath, 1, Set(ExpectedAdd("file1.parquet", 1000, 10, true)), version) checkActionsWrittenInJson( engine, tablePath, 2, Set(ExpectedAdd("file1.parquet", 1000, 10, false)), version) // Verify that Spark can read the actions written via log replay checkSparkLogReplay(tablePath, 0, Set(), version) checkSparkLogReplay( tablePath, 1, Set(ExpectedAdd("file1.parquet", 1000, 10, true)), version) // We added the same path twice so only the second remains after log replay checkSparkLogReplay( tablePath, 2, Set(ExpectedAdd("file1.parquet", 1000, 10, false)), version) } } test(s"$version: Correctly commits adds and removes to table and compat with Spark") { withTempDirAndEngine { (tablePath, engine) => val properties = if (version == "V1") tblPropertiesIcebergWriterCompatV1Enabled else tblPropertiesIcebergWriterCompatV3Enabled // Create table createEmptyTable( engine, tablePath, testSchema, tableProperties = properties) // Append 1 add with dataChange = true { val txn = getUpdateTxn(engine, tablePath, maxRetries = 0) val actionsToCommit = Seq( createIcebergCompatAction( "ADD", version, txn, engine, generateDataFileStatus(tablePath, "file1.parquet"), dataChange = true)) commitTransaction( txn, engine, inMemoryIterable(toCloseableIterator(actionsToCommit.asJava.iterator()))) } // Re-arrange data by removing that Add and adding a new Add { val txn = getUpdateTxn(engine, tablePath, maxRetries = 0) val actionsToCommit = Seq( createIcebergCompatAction( "REMOVE", version, txn, engine, generateDataFileStatus(tablePath, "file1.parquet"), dataChange = false), createIcebergCompatAction( "ADD", version, txn, engine, generateDataFileStatus(tablePath, "file2.parquet"), dataChange = false)) commitTransaction( txn, engine, inMemoryIterable(toCloseableIterator(actionsToCommit.asJava.iterator()))) } // Remove that add so that the table is empty { val txn = getUpdateTxn(engine, tablePath, maxRetries = 0) val actionsToCommit = Seq( createIcebergCompatAction( "REMOVE", version, txn, engine, generateDataFileStatus(tablePath, "file2.parquet"), dataChange = true)) commitTransaction( txn, engine, inMemoryIterable(toCloseableIterator(actionsToCommit.asJava.iterator()))) } // Verify we wrote the adds we expected into the JSON files using Kernel's getChanges checkActionsWrittenInJson(engine, tablePath, 0, Set(), version) checkActionsWrittenInJson( engine, tablePath, 1, Set(ExpectedAdd("file1.parquet", 1000, 10, true)), version) checkActionsWrittenInJson( engine, tablePath, 2, Set( ExpectedAdd("file2.parquet", 1000, 10, false), ExpectedRemove("file1.parquet", 1000, 10, false)), version) checkActionsWrittenInJson( engine, tablePath, 3, Set(ExpectedRemove("file2.parquet", 1000, 10, true)), version) // Verify that Spark can read the actions written via log replay checkSparkLogReplay(tablePath, 0, Set(), version) checkSparkLogReplay( tablePath, 1, Set(ExpectedAdd("file1.parquet", 1000, 10, true)), version) checkSparkLogReplay( tablePath, 2, Set(ExpectedAdd("file2.parquet", 1000, 10, false)), version) checkSparkLogReplay(tablePath, 3, Set(), version) } } test(s"$version: append-only configuration is observed when committing removes") { withTempDirAndEngine { (tablePath, engine) => val properties = if (version == "V1") tblPropertiesIcebergWriterCompatV1Enabled else tblPropertiesIcebergWriterCompatV3Enabled // Create table createEmptyTable( engine, tablePath, testSchema, tableProperties = properties ++ Map( TableConfig.APPEND_ONLY_ENABLED.getKey -> "true")) // Append 1 add with dataChange = true { val txn = getUpdateTxn(engine, tablePath, maxRetries = 0) val actionsToCommit = Seq( createIcebergCompatAction( "ADD", version, txn, engine, generateDataFileStatus(tablePath, "file1.parquet"), dataChange = true)) commitTransaction( txn, engine, inMemoryIterable(toCloseableIterator(actionsToCommit.asJava.iterator()))) } // Re-arrange data by removing that Add and adding a new Add // (can commit remove with dataChange=false) { val txn = getUpdateTxn(engine, tablePath, maxRetries = 0) val actionsToCommit = Seq( createIcebergCompatAction( "REMOVE", version, txn, engine, generateDataFileStatus(tablePath, "file1.parquet"), dataChange = false), createIcebergCompatAction( "ADD", version, txn, engine, generateDataFileStatus(tablePath, "file2.parquet"), dataChange = true)) commitTransaction( txn, engine, inMemoryIterable(toCloseableIterator(actionsToCommit.asJava.iterator()))) } // Cannot create remove with dataChange=true { val txn = getUpdateTxn(engine, tablePath, maxRetries = 0) intercept[KernelException] { createIcebergCompatAction( "REMOVE", version, txn, engine, generateDataFileStatus(tablePath, "file1.parquet"), dataChange = true) } } } } test(s"$version: Tags can be successfully passed for generating addFile") { withTempDirAndEngine { (tablePath, engine) => val properties = if (version == "V1") tblPropertiesIcebergWriterCompatV1Enabled else tblPropertiesIcebergWriterCompatV3Enabled // Create table createEmptyTable( engine, tablePath, testSchema, tableProperties = properties) // Commit one add file with tags val tags = Map("tag1" -> "abc", "tag2" -> "def") { val txn = getUpdateTxn(engine, tablePath, maxRetries = 0) val actionsToCommit = Seq( createIcebergCompatAction( "ADD", version, txn, engine, generateDataFileStatus(tablePath, "file1.parquet"), dataChange = true, tags = tags)) commitTransaction( txn, engine, inMemoryIterable(toCloseableIterator(actionsToCommit.asJava.iterator()))) } // Read back committed ADD actions val tableVersion = 1 val rows = Table.forPath(engine, tablePath).asInstanceOf[TableImpl] .getChanges(engine, tableVersion, tableVersion, Set(DeltaAction.ADD).asJava) .toSeq .flatMap(_.getRows.toSeq) .filterNot(row => row.isNullAt(row.getSchema.indexOf("add"))) assert(rows.size == 1) val addFile = new AddFile(rows.head.getStruct(rows.head.getSchema.indexOf("add"))) assert(addFile.getTags.isPresent) assert(VectorUtils.toJavaMap(addFile.getTags.get()).asScala.equals(tags)) } } } test("V3: baseRowId and defaultRowCommitVersion are forwarded in remove actions") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, testSchema, tableProperties = tblPropertiesIcebergWriterCompatV3Enabled) // Append 1 add so we have something to remove commitSingleAction( engine, tablePath, createIcebergCompatAction( "ADD", "V3", _, engine, generateDataFileStatus(tablePath, "file1.parquet"), true)) // Remove with explicit baseRowId and defaultRowCommitVersion val expectedBaseRowId = 42L val expectedDefaultRowCommitVersion = 7L commitSingleAction( engine, tablePath, txn => GenerateIcebergCompatActionUtils.generateIcebergCompatWriterV3RemoveAction( txn.getTransactionState(engine), generateDataFileStatus(tablePath, "file1.parquet"), Collections.emptyMap(), true, // dataChange Optional.of[java.lang.Long](expectedBaseRowId), Optional.of[java.lang.Long](expectedDefaultRowCommitVersion), Optional.empty(), // deletionVectorDescriptor Optional.of( TransactionStateRow.getPhysicalSchema(txn.getTransactionState(engine))))) // Read back the remove action and verify the values are preserved val rows = Table.forPath(engine, tablePath).asInstanceOf[TableImpl] .getChanges(engine, 2, 2, Set(DeltaAction.REMOVE).asJava) .toSeq .flatMap(_.getRows.toSeq) .filterNot(row => row.isNullAt(row.getSchema.indexOf("remove"))) assert(rows.size == 1) val removeFile = new RemoveFile(rows.head.getStruct(rows.head.getSchema.indexOf("remove"))) assert( removeFile.getBaseRowId.isPresent, "baseRowId should be present when caller provides it") assert(removeFile.getBaseRowId.get == expectedBaseRowId) assert( removeFile.getDefaultRowCommitVersion.isPresent, "defaultRowCommitVersion should be present when caller provides it") assert(removeFile.getDefaultRowCommitVersion.get == expectedDefaultRowCommitVersion) } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/CommitMetadataE2ESuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer import io.delta.kernel._ import io.delta.kernel.commit.{CommitMetadata, CommitResponse, Committer} import io.delta.kernel.data.Row import io.delta.kernel.defaults.internal.parquet.ParquetSuiteBase import io.delta.kernel.defaults.utils.{TestCommitterUtils, WriteUtilsWithV2Builders} import io.delta.kernel.engine.Engine import io.delta.kernel.utils.CloseableIterable.emptyIterable import io.delta.kernel.utils.CloseableIterator import org.scalatest.funsuite.AnyFunSuite class CommitMetadataE2ESuite extends AnyFunSuite with WriteUtilsWithV2Builders with ParquetSuiteBase with TestCommitterUtils { private class CapturingCommitter extends Committer { var latestCommitMetadata: Option[CommitMetadata] = None override def commit( engine: Engine, finalizedActions: CloseableIterator[Row], commitMetadata: CommitMetadata): CommitResponse = { latestCommitMetadata = Some(commitMetadata) committerUsingPutIfAbsent.commit(engine, finalizedActions, commitMetadata) } } test("transaction passes added and removed (and not existing) domain metadatas to committer") { withTempDirAndEngine { (tablePath, engine) => // ===== TEST HELPER SETUP ===== val capturingCommitter = new CapturingCommitter() def createTxnAtLatest(): Transaction = TableManager .loadSnapshot(tablePath) .withCommitter(capturingCommitter) .build(engine) .buildUpdateTableTransaction("engineInfo", Operation.WRITE) .build(engine) // ===== GIVEN ===== val txn0 = TableManager.buildCreateTableTransaction(tablePath, testSchema, "engineInfo") .withTableProperties(Map("delta.feature.domainMetadata" -> "supported").asJava) .build(engine) txn0.addDomainMetadata("foo", "bar") commitTransaction(txn0, engine, emptyIterable()) // ===== WHEN (Case 1: No domain metadata change on table with existing domain metadata) ===== val txn1 = createTxnAtLatest() commitTransaction(txn1, engine, emptyIterable()) // ===== THEN (Case 1) ===== { val commitMetadata = capturingCommitter.latestCommitMetadata.get assert(commitMetadata.getVersion === 1) assert(commitMetadata.getCommitDomainMetadatas.isEmpty) } // ===== WHEN (Case 2: Add domain metadata) ===== val txn2 = createTxnAtLatest() txn2.addDomainMetadata("zip", "zap") commitTransaction(txn2, engine, emptyIterable()) // ===== THEN (Case 2) ===== { val commitMetadata = capturingCommitter.latestCommitMetadata.get assert(commitMetadata.getVersion === 2) val commitDomainMetadatas = commitMetadata.getCommitDomainMetadatas assert(commitDomainMetadatas.size() === 1) val dm = commitDomainMetadatas.asScala.head assert(dm.getDomain === "zip") assert(dm.getConfiguration === "zap") assert(!dm.isRemoved) } // ===== WHEN (Case 3: Remove domain metadata) ===== val txn3 = createTxnAtLatest() txn3.removeDomainMetadata("zip") commitTransaction(txn3, engine, emptyIterable()) // ===== THEN (Case 3: Remove domain metadata) ===== { val commitMetadata = capturingCommitter.latestCommitMetadata.get assert(commitMetadata.getVersion === 3) val commitDomainMetadatas = commitMetadata.getCommitDomainMetadatas assert(commitDomainMetadatas.size() === 1) val dm = commitDomainMetadatas.get(0) assert(dm.getDomain === "zip") assert(dm.isRemoved) } } } test("maxKnownPublishedDeltaVersion is -1 for CREATE operation") { withTempDirAndEngine { (tablePath, engine) => val capturingCommitter = new CapturingCommitter() // Create the table TableManager .buildCreateTableTransaction(tablePath, testSchema, "engineInfo") .withCommitter(capturingCommitter) .build(engine) .commit(engine, emptyIterable()) // Verify val commitMetadata = capturingCommitter.latestCommitMetadata.get assert(commitMetadata.getVersion === 0) assert(commitMetadata.getMaxKnownPublishedDeltaVersion.get === -1L) } } test("maxKnownPublishedDeltaVersion is correctly passed for UPDATE operation") { withTempDirAndEngine { (tablePath, engine) => val capturingCommitter = new CapturingCommitter() // Create the table TableManager .buildCreateTableTransaction(tablePath, testSchema, "engineInfo") .build(engine) .commit(engine, emptyIterable()) // Update the table TableManager .loadSnapshot(tablePath) .withCommitter(capturingCommitter) .build(engine) .buildUpdateTableTransaction("engineInfo", Operation.WRITE) .build(engine) .commit(engine, emptyIterable()) // Verify val commitMetadata = capturingCommitter.latestCommitMetadata.get assert(commitMetadata.getVersion === 1) assert(commitMetadata.getMaxKnownPublishedDeltaVersion.get === 0L) } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/CreateCheckpointSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.io.File import io.delta.golden.GoldenTableUtils.goldenTablePath import io.delta.kernel.{Table, TableManager} import io.delta.kernel.defaults.engine.DefaultEngine import io.delta.kernel.defaults.utils.{TestRow, TestUtils, WriteUtils} import io.delta.kernel.engine.Engine import io.delta.kernel.exceptions.{CheckpointAlreadyExistsException, TableNotFoundException} import io.delta.kernel.expressions.Literal import io.delta.kernel.internal.SnapshotImpl import org.apache.spark.sql.delta.{DeltaLog, VersionNotFoundException} import org.apache.spark.sql.delta.DeltaOperations.ManualUpdate import org.apache.spark.sql.delta.actions.{AddFile, Metadata, RemoveFile} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.commons.io.FileUtils import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.sql.types.{IntegerType, StructType} import org.scalatest.funsuite.AnyFunSuite /** * Test suite for `io.delta.kernel.Table.checkpoint(engine, version)` */ class CreateCheckpointSuite extends CheckpointBase { /////////// // Tests // /////////// /** * Helper for tests. * * Creates a new table at version 0, then appends {@code commits} additional commits. * Returns the `_delta_log` directory. */ private def setupTestTable( engine: Engine, tablePath: String, tableProperties: Map[String, String], commits: Int): File = { val data = Seq(Map.empty[String, Literal] -> generateData(testSchema, Seq.empty, Map.empty, batchSize = 1, numBatches = 1)) // Create table (version 0) and add commits (version 1..commits) appendData( engine, tablePath, isNewTable = true, schema = testSchema, data = data, tableProperties = tableProperties) for (_ <- 1 to commits) { appendData(engine, tablePath, data = data) } val deltaLogDir = new File(tablePath, "_delta_log") assert(deltaLogDir.listFiles().count(_.getName.endsWith(".json")) === commits + 1) deltaLogDir } Seq(true, false).foreach { includeRemoves => val testMsgUpdate = if (includeRemoves) " and removes" else "" test(s"commits containing adds$testMsgUpdate, and no previous checkpoint") { withTempDirAndEngine { (tablePath, tc) => addData(tablePath, alternateBetweenAddsAndRemoves = includeRemoves, numberIter = 10) // before creating checkpoint, read and save the expected results using Spark val expResults = readUsingSpark(tablePath) assert(expResults.size === (if (includeRemoves) 45 else 100)) val checkpointVersion = 9 kernelCheckpoint(tc, tablePath, checkpointVersion) verifyResults(tablePath, expResults, checkpointVersion) verifyLastCheckpointMetadata( tablePath, checkpointVersion, expSize = if (includeRemoves) 5 else 10) // add few more commits and verify the read still works appendCommit(tablePath) val newExpResults = expResults ++ Seq.range(0, 10).map(_.longValue()).map(TestRow(_)) verifyResults(tablePath, newExpResults, checkpointVersion) } } } Seq(true, false).foreach { includeRemoves => Seq( // Create a checkpoint using Spark (either classic or multi-part checkpoint) 1000000, // use large number of actions per file to make Spark create a classic checkpoint 3 // use small number of actions per file to make Spark create a multi-part checkpoint ).foreach { sparkCheckpointActionPerFile => val testMsgUpdate = if (includeRemoves) " and removes" else "" test(s"commits containing adds$testMsgUpdate, and a previous checkpoint " + s"created using Spark (actions/perfile): $sparkCheckpointActionPerFile") { withTempDirAndEngine { (tablePath, tc) => addData(tablePath, includeRemoves, numberIter = 6) // checkpoint using Spark sparkCheckpoint(tablePath, actionsPerFile = sparkCheckpointActionPerFile) addData(tablePath, includeRemoves, numberIter = 6) // add some more data // before creating checkpoint, read and save the expected results using Spark val expResults = readUsingSpark(tablePath) assert(expResults.size === (if (includeRemoves) 54 else 120)) val checkpointVersion = 11 kernelCheckpoint(tc, tablePath, checkpointVersion) verifyResults(tablePath, expResults, checkpointVersion) verifyLastCheckpointMetadata( tablePath, checkpointVersion, expSize = if (includeRemoves) 6 else 12) // add few more commits and verify the read still works appendCommit(tablePath) val newExpResults = expResults ++ Seq.range(0, 10).map(_.longValue()).map(TestRow(_)) verifyResults(tablePath, newExpResults, checkpointVersion) } } } } test("commits with metadata updates") { withTempDirAndEngine { (tablePath, tc) => addData(path = tablePath, alternateBetweenAddsAndRemoves = true, numberIter = 16) // makes the latest table version 16 spark.sql( s"""ALTER TABLE delta.`$tablePath` SET TBLPROPERTIES ('delta.appendOnly' = 'true')""") // before creating checkpoint, read and save the expected results using Spark val expResults = readUsingSpark(tablePath) assert(expResults.size === 72) val checkpointVersion = 16 kernelCheckpoint(tc, tablePath, checkpointVersion) verifyResults(tablePath, expResults, checkpointVersion) verifyLastCheckpointMetadata(tablePath, checkpointVersion, expSize = 8) // verify there is only one metadata entry in the checkpoint and it has the // configuration with `delta.appendOnly` = `true` val result = spark.read.format("parquet") .load(checkpointFilePath(tablePath, checkpointVersion)) .filter("metaData is not null") .select("metaData.configuration") .collect().toSeq.map(TestRow(_)) val expected = Seq(TestRow(Map("delta.appendOnly" -> "true"))) checkAnswer(result, expected) } } test("commits with protocol updates") { withTempDirAndEngine { (tablePath, tc) => addData(path = tablePath, alternateBetweenAddsAndRemoves = true, numberIter = 16) spark.sql( s""" |ALTER TABLE delta.`$tablePath` SET TBLPROPERTIES ( | 'delta.minReaderVersion' = '1', | 'delta.minWriterVersion' = '2' |) |""".stripMargin ) // makes the latest table version 16 // before creating checkpoint, read and save the expected results using Spark val expResults = readUsingSpark(tablePath) assert(expResults.size === 72) val checkpointVersion = 16 kernelCheckpoint(tc, tablePath, checkpointVersion) verifyResults(tablePath, expResults, checkpointVersion) // verify there is only one protocol entry in the checkpoint and it has the // expected minReaderVersion and minWriterVersion val result = spark.read.format("parquet") .load(checkpointFilePath(tablePath, checkpointVersion)) .filter("protocol is not null") .select("protocol.minReaderVersion", "protocol.minWriterVersion") .collect().toSeq.map(TestRow(_)) val expected = Seq(TestRow(1, 2)) checkAnswer(result, expected) } } test("commits with set transactions") { withTempDirAndEngine { (tablePath, tc) => def idempotentAppend(appId: String, version: Int): Unit = { spark.range(end = 10).repartition(2).write.format("delta") .option("txnAppId", appId) .option("txnVersion", version) .mode("append").save(tablePath) } idempotentAppend("appId1", 0) // version 0 idempotentAppend("appId1", 2) // version 1 idempotentAppend("appId1", 3) // version 2 deleteCommit(tablePath) // version 3 idempotentAppend("appId2", 7) // version 4 idempotentAppend("appId2", 25) // version 5 idempotentAppend("appId3", 7908) // version 6 appendCommit(tablePath) // version 7, no txn identifiers idempotentAppend("appId4", 12312312) // version 8 // before creating checkpoint, read and save the expected results using Spark val expResults = readUsingSpark(tablePath) assert(expResults.size === 77) val checkpointVersion = 8 kernelCheckpoint(tc, tablePath, checkpointVersion); verifyResults(tablePath, expResults, checkpointVersion) // Load the checkpoint and verify that only the last txn identifier for each appId is stored def verifyTxnIdInCheckpoint(appId: String, expVersion: Long): Unit = { val result = spark.read.format("parquet") .load(checkpointFilePath(tablePath, checkpointVersion)) .filter(s"txn is not null and txn.appId='$appId'") .select("txn.appId", "txn.version") .collect().toSeq.map(TestRow(_)) checkAnswer(result, Seq(TestRow(appId, expVersion))) } verifyTxnIdInCheckpoint("appId1", 3) verifyTxnIdInCheckpoint("appId2", 25) verifyTxnIdInCheckpoint("appId3", 7908) verifyTxnIdInCheckpoint("appId4", 12312312) } } Seq(None, Some("2 days"), Some("0 days")).foreach { retentionInterval => test(s"checkpoint contains all not expired tombstones: $retentionInterval") { withTempDirAndEngine { (tablePath, tc) => def addFile(path: String): AddFile = AddFile( path = path, partitionValues = Map.empty, size = 0, modificationTime = 0L, dataChange = true) def removeFile(path: String, deletionTimestamp: Long): Unit = { val remove = RemoveFile(path = path, deletionTimestamp = Some(deletionTimestamp)) val deltaLog = DeltaLog.forTable(spark, tablePath) val txn = deltaLog.startTransaction() txn.commit(Seq(remove), ManualUpdate) } def addFiles(addFiles: String*): Unit = { val deltaLog = DeltaLog.forTable(spark, tablePath) val txn = deltaLog.startTransaction() val configuration = retentionInterval.map(interval => Map("delta.deletedFileRetentionDuration" -> interval)).getOrElse(Map.empty) txn.updateMetadata(Metadata( schemaString = new StructType().add("c1", IntegerType).json, configuration = configuration)) txn.commit(addFiles.map(addFile(_)), ManualUpdate) } def millisPerDays(days: Int): Long = days * 24 * 60 * 60 * 1000 // version 0 addFiles( "file1", "file2", "file3", "file4", "file5", "file6", "file7", "file8", "file9") val now = System.currentTimeMillis() removeFile("file8", deletionTimestamp = 1) // set delete time very old removeFile("file7", deletionTimestamp = now - millisPerDays(8)) removeFile("file6", deletionTimestamp = now - millisPerDays(3)) removeFile("file5", deletionTimestamp = now - 1000) // set delete time 1 second ago // end version 4 // add few more files - version 5 addFiles( "file10", "file11", "file12", "file13", "file14", "file15", "file16", "file17", "file18") // delete some files again removeFile("file3", deletionTimestamp = now - millisPerDays(9)) removeFile("file2", deletionTimestamp = now - millisPerDays(1)) // end version 7 val expected = if (retentionInterval.isEmpty) { // Given the default retention interval is 1 week, the tombstones file8, file 7 and file 3 // should be expired and not included in the checkpoint Seq("file6", "file5", "file2").map(TestRow(_)) } else if (retentionInterval.get.equals("2 days")) { // Given the retention interval is 2 days, the tombstones file8, file 7, file 6, file 3 // should be expired and not included in the checkpoint Seq("file5", "file2").map(TestRow(_)) } else { // All tombstones should be excluded in the checkpoint Seq.empty } val checkpointVersion = 7 kernelCheckpoint(tc, tablePath, checkpointVersion) val result = spark.read.format("parquet") .load(checkpointFilePath(tablePath, checkpointVersion)) .filter("remove is not null") .select("remove.path") .collect().toSeq.map(TestRow(_)) checkAnswer(result, expected) } } } test("try creating checkpoint on a non-existent table") { withTempDirAndEngine { (path, tc) => Seq(0, 1, 2).foreach { checkpointVersion => val ex = intercept[TableNotFoundException] { kernelCheckpoint(tc, path, checkpointVersion) } assert(ex.getMessage.contains("not found")) } } } test("try creating checkpoint at version that already has a " + "checkpoint or a version that doesn't exist") { withTempDirAndEngine { (path, tc) => for (_ <- 0 to 3) { appendCommit(path) } val table = Table.forPath(tc, path) table.checkpoint(tc, 3) val ex = intercept[CheckpointAlreadyExistsException] { kernelCheckpoint(tc, path, 3) } assert(ex.getMessage.contains("Checkpoint for given version 3 already exists in the table")) val ex2 = intercept[Exception] { kernelCheckpoint(tc, path, checkpointVersion = 5) } assert(ex2.getMessage.contains("Cannot load table version 5 as it does not exist")) } } test("create a checkpoint on a existing table") { withTempDirAndEngine { (tablePath, tc) => copyTable("time-travel-start-start20-start40", tablePath) // before creating checkpoint, read and save the expected results using Spark val expResults = readUsingSpark(tablePath) assert(expResults.size === 30) val checkpointVersion = 2 kernelCheckpoint(tc, tablePath, checkpointVersion) verifyResults(tablePath, expResults, checkpointVersion) } } test("try create a checkpoint on a unsupported table feature table") { withTempDirAndEngine { (tablePath, tc) => spark.sql(s"CREATE TABLE delta.`$tablePath` (name STRING, age INT) USING delta " + "TBLPROPERTIES ('delta.constraints.checks' = 'name IS NOT NULL')") for (_ <- 0 to 3) { spark.sql(s"INSERT INTO delta.`$tablePath` VALUES ('John Doe', 30), ('Bob Johnson', 35)") } val ex2 = intercept[Exception] { kernelCheckpoint(tc, tablePath, checkpointVersion = 4) } assert(ex2.getMessage.contains("requires writer table features [checkConstraints] " + "which is unsupported by this version of Delta Kernel")) } } test("create a checkpoint on a table with deletion vectors") { withTempDirAndEngine { (tablePath, tc) => copyTable("dv-with-columnmapping", tablePath) // before creating checkpoint, read and save the expected results using Spark val expResults = readUsingSpark(tablePath) assert(expResults.size === 35) val checkpointVersion = 15 kernelCheckpoint(tc, tablePath, checkpointVersion) verifyResults(tablePath, expResults, checkpointVersion) } } test("log cleanup: non-latest snapshot can NOT trigger log cleanup") { withTempDirAndEngine { (tablePath, engine) => val commits = 3 val tableProperties = Map( "delta.logRetentionDuration" -> "interval 0 seconds", "delta.enableExpiredLogCleanup" -> "true") val deltaLogDir = setupTestTable(engine, tablePath, tableProperties, commits) // Checkpoint at version 2 using SnapshotBuilder.atVersion() - wasBuiltAsLatest=false val snapshot = TableManager.loadSnapshot(tablePath) .atVersion(2) .build(engine) .asInstanceOf[SnapshotImpl] snapshot.writeCheckpoint(engine) // Verify no log cleanup happened assert( deltaLogDir.listFiles().count(_.getName.endsWith(".json")) === commits + 1, "Checkpoint on snapshot built with specific version should NOT trigger log cleanup") } } test("log cleanup: latest snapshot can trigger log cleanup") { withTempDirAndEngine { (tablePath, engine) => val commits = 3 val tableProperties = Map( "delta.logRetentionDuration" -> "interval 0 seconds", "delta.enableExpiredLogCleanup" -> "true") val deltaLogDir = setupTestTable(engine, tablePath, tableProperties, commits) // Get latest snapshot (version == commits) using builder without atVersion() - // wasBuiltAsLatest=true val latestSnapshot = TableManager.loadSnapshot(tablePath) .build(engine) .asInstanceOf[SnapshotImpl] assert(latestSnapshot.wasBuiltAsLatest()) latestSnapshot.writeCheckpoint(engine) // Verify log cleanup happened assert( deltaLogDir.listFiles().count(_.getName.endsWith(".json")) < commits + 1, "Checkpoint on snapshot built without specific version should trigger log cleanup") } } test( "log cleanup: checkpointProtection enabled prevents log cleanup, " + "even snapshot is built as latest") { withTempDirAndEngine { (tablePath, engine) => val commits = 3 val tableProperties = Map( "delta.logRetentionDuration" -> "interval 0 seconds", "delta.enableExpiredLogCleanup" -> "true", "delta.feature.checkpointProtection" -> "supported") val deltaLogDir = setupTestTable(engine, tablePath, tableProperties, commits) // Get latest snapshot using builder without atVersion() - wasBuiltAsLatest=true val latestSnapshot = TableManager.loadSnapshot(tablePath) .build(engine) .asInstanceOf[SnapshotImpl] assert(latestSnapshot.wasBuiltAsLatest()) assert(latestSnapshot.getProtocol.getWriterFeatures.contains("checkpointProtection")) latestSnapshot.writeCheckpoint(engine) // Verify no log cleanup (checkpointProtection prevents it) assert( deltaLogDir.listFiles().count(_.getName.endsWith(".json")) === commits + 1, "Log cleanup should NOT happen with checkpointProtection enabled") } } } /** * Helper methods for suites that do checkpoint operations */ trait CheckpointBase extends AnyFunSuite with WriteUtils { def addData(path: String, alternateBetweenAddsAndRemoves: Boolean, numberIter: Int): Unit = { Seq.range(0, numberIter).foreach { version => if (version % 2 == 1 && alternateBetweenAddsAndRemoves) { deleteCommit(path) // removes one file and adds a new one } else { appendCommit(path) // add one new file } } } def appendCommit(path: String): Unit = spark.range(end = 10).write.format("delta").mode("append").save(path) def deleteCommit(path: String): Unit = { spark.sql(s"DELETE FROM delta.`${path}` WHERE id = 5") } def sparkCheckpoint(path: String, actionsPerFile: Int = 10000000): Unit = { withSQLConf(DeltaSQLConf.DELTA_CHECKPOINT_PART_SIZE.key -> actionsPerFile.toString) { DeltaLog.forTable(spark, path).checkpoint() } } def kernelCheckpoint(tc: Engine, tablePath: String, checkpointVersion: Long): Unit = { Table.forPath(tc, tablePath).checkpoint(tc, checkpointVersion) } def readUsingSpark(tablePath: String): Seq[TestRow] = { spark.read.format("delta").load(tablePath).collect().map(TestRow(_)) } def verifyResults( tablePath: String, expResults: Seq[TestRow], checkpointVersion: Long): Unit = { // before verifying delete the delta commits before the checkpoint to make sure // the state is constructed using the table path deleteDeltaFilesBefore(tablePath, checkpointVersion) // verify using Spark reader checkAnswer(readUsingSpark(tablePath), expResults) // verify using Kernel reader checkTable(tablePath, expResults) } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DataSkippingDeltaTestsUtils.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import scala.collection.immutable.Seq import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.{Expression, PredicateHelper} /** * Encapsulates a few Delta-Spark DataSkipping Utils. * E.g A helper to get files in a deltaScan after applying a predicate. */ trait DataSkippingDeltaTestsUtils extends PredicateHelper { /** Parses a predicate string into Spark expressions by analyzing the optimized query plan. */ def parse(spark: SparkSession, deltaLog: DeltaLog, predicate: String): Seq[Expression] = { if (predicate == "True") { Seq(org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral) } else { val filtered = spark.read.format("delta").load(deltaLog.dataPath.toString).where(predicate) filtered .queryExecution .optimizedPlan .expressions .flatMap(splitConjunctivePredicates) .toList } } /** * Returns the number of files that would be read when applying * the given predicate (for data skipping validation). */ def filesReadCount( spark: SparkSession, deltaLog: DeltaLog, predicate: String): Int = { getFilesRead(spark, deltaLog, predicate, checkEmptyUnusedFilters = false).size } /** * Returns the files that should be included in a scan after applying the given predicate on * a snapshot of the Delta log. * * @param deltaLog Delta log for a table. * @param predicate Predicate to run on the Delta table. * @param checkEmptyUnusedFilters If true, check if there were no unused filters, meaning * the given predicate was used as data or partition filters. * @return The files that should be included in a scan after applying the predicate. */ def getFilesRead( spark: SparkSession, deltaLog: DeltaLog, predicate: String, checkEmptyUnusedFilters: Boolean): Seq[AddFile] = { val parsed: Seq[Expression] = parse(spark, deltaLog, predicate) val res = deltaLog.snapshot.filesForScan(parsed) assert(res.total.files.get == deltaLog.snapshot.numOfFiles) assert(res.total.bytesCompressed.get == deltaLog.snapshot.sizeInBytes) assert(res.scanned.files.get == res.files.size) assert(res.scanned.bytesCompressed.get == res.files.map(_.size).sum) assert(!checkEmptyUnusedFilters || res.unusedFilters.isEmpty) res.files.toList } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeletionVectorSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import io.delta.golden.GoldenTableUtils.goldenTablePath import io.delta.kernel.defaults.utils.{TestRow, TestUtils} import org.apache.hadoop.conf.Configuration import org.scalatest.funsuite.AnyFunSuite class DeletionVectorSuite extends AnyFunSuite with TestUtils { test("end-to-end usage: reading a table with dv") { checkTable( path = getTestResourceFilePath("basic-dv-no-checkpoint"), expectedAnswer = (2L until 10L).map(TestRow(_))) } test("end-to-end usage: reading a table with dv with space in the root path") { withTempDir { tempDir => val target = tempDir.getCanonicalPath + "spark test" spark.sql(s"""CREATE TABLE tbl ( id int ) USING delta LOCATION '$target' TBLPROPERTIES ('delta.enableDeletionVectors' = true) """) spark.sql("INSERT INTO tbl VALUES (1),(2),(3),(4),(5)") spark.sql("DELETE FROM tbl WHERE id = 1") checkTable( path = target, expectedAnswer = Seq(TestRow(2), TestRow(3), TestRow(4), TestRow(5))) } } test("end-to-end usage: reading a table with dv with checkpoint") { checkTable( path = getTestResourceFilePath("basic-dv-with-checkpoint"), expectedAnswer = (0L until 500L).filter(_ % 11 != 0).map(TestRow(_))) } test("end-to-end usage: reading partitioned dv table with checkpoint") { val conf = new Configuration() // Set the batch size small enough so there will be multiple batches conf.setInt("delta.kernel.default.parquet.reader.batch-size", 2) val expectedResult = (0 until 50).map(x => (x % 10, x, s"foo${x % 5}")) .filter { case (_, col1, _) => !(col1 % 2 == 0 && col1 < 30) } checkTable( path = goldenTablePath("dv-partitioned-with-checkpoint"), expectedAnswer = expectedResult.map(TestRow.fromTuple(_)), engine = defaultEngine) } test( "end-to-end usage: reading partitioned dv table with checkpoint with columnMappingMode=name") { val expectedResult = (0 until 50).map(x => (x % 10, x, s"foo${x % 5}")) .filter { case (_, col1, _) => !(col1 % 2 == 0 && col1 < 30) } checkTable( path = goldenTablePath("dv-with-columnmapping"), expectedAnswer = expectedResult.map(TestRow.fromTuple(_))) } // TODO detect corrupted DV checksum // TODO detect corrupted dv size // TODO multiple dvs in one file } object DeletionVectorsSuite { // TODO: test using this once we support reading by version val table1Path = "src/test/resources/delta/table-with-dv-large" // Table at version 0: contains [0, 2000) val expectedTable1DataV0 = Seq.range(0, 2000) // Table at version 1: removes rows with id = 0, 180, 300, 700, 1800 val v1Removed = Set(0, 180, 300, 700, 1800) val expectedTable1DataV1 = expectedTable1DataV0.filterNot(e => v1Removed.contains(e)) // Table at version 2: inserts rows with id = 300, 700 val v2Added = Set(300, 700) val expectedTable1DataV2 = expectedTable1DataV1 ++ v2Added // Table at version 3: removes rows with id = 300, 250, 350, 900, 1353, 1567, 1800 val v3Removed = Set(300, 250, 350, 900, 1353, 1567, 1800) val expectedTable1DataV3 = expectedTable1DataV2.filterNot(e => v3Removed.contains(e)) // Table at version 4: inserts rows with id = 900, 1567 val v4Added = Set(900, 1567) val expectedTable1DataV4 = expectedTable1DataV3 ++ v4Added } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaColumnMappingSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import scala.collection.JavaConverters._ import scala.collection.immutable.Seq import io.delta.kernel.Table import io.delta.kernel.defaults.utils.{AbstractWriteUtils, WriteUtils, WriteUtilsWithV2Builders} import io.delta.kernel.exceptions.InvalidConfigurationValueException import io.delta.kernel.expressions.Literal import io.delta.kernel.internal.TableConfig import io.delta.kernel.internal.util.{ColumnMapping, ColumnMappingSuiteBase} import io.delta.kernel.types.{FieldMetadata, IntegerType, StringType, StructField, StructType} import org.scalatest.funsuite.AnyFunSuite class DeltaColumnMappingTransactionBuilderV1Suite extends DeltaColumnMappingSuiteBase with WriteUtils {} class DeltaColumnMappingTransactionBuilderV2Suite extends DeltaColumnMappingSuiteBase with WriteUtilsWithV2Builders {} trait DeltaColumnMappingSuiteBase extends AnyFunSuite with AbstractWriteUtils with ColumnMappingSuiteBase { val simpleTestSchema = new StructType() .add("a", StringType.STRING, true) .add("b", IntegerType.INTEGER, true) test("create table with unsupported column mapping mode") { withTempDirAndEngine { (tablePath, engine) => val ex = intercept[InvalidConfigurationValueException] { val props = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> "invalid") createEmptyTable(engine, tablePath, simpleTestSchema, tableProperties = props) } assert(ex.getMessage.contains("Invalid value for table property " + "'delta.columnMapping.mode': 'invalid'. Needs to be one of: [none, id, name].")) } } test("create table with column mapping mode = none") { withTempDirAndEngine { (tablePath, engine) => val props = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> "none") createEmptyTable(engine, tablePath, simpleTestSchema, tableProperties = props) assert(getMetadata(engine, tablePath).getSchema.equals(simpleTestSchema)) } } test("cannot update table with unsupported column mapping mode") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable(engine, tablePath, simpleTestSchema) val ex = intercept[InvalidConfigurationValueException] { val props = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> "invalid") updateTableMetadata(engine, tablePath, tableProperties = props) } assert(ex.getMessage.contains("Invalid value for table property " + "'delta.columnMapping.mode': 'invalid'. Needs to be one of: [none, id, name].")) } } test("new table with column mapping mode = name") { withTempDirAndEngine { (tablePath, engine) => val props = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> "name") createEmptyTable(engine, tablePath, simpleTestSchema, tableProperties = props) val structType = getMetadata(engine, tablePath).getSchema assertColumnMapping(structType.get("a"), 1) assertColumnMapping(structType.get("b"), 2) val protocol = getProtocol(engine, tablePath) assert(protocol.getMinReaderVersion == 2 && protocol.getMinWriterVersion == 7) } } test("new table with column mapping mode = id") { withTempDirAndEngine { (tablePath, engine) => val props = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> "id") createEmptyTable(engine, tablePath, simpleTestSchema, tableProperties = props) val structType = getMetadata(engine, tablePath).getSchema assertColumnMapping(structType.get("a"), 1) assertColumnMapping(structType.get("b"), 2) assert(TableConfig.COLUMN_MAPPING_MAX_COLUMN_ID.fromMetadata(getMetadata( engine, tablePath)) == 2) val protocol = getProtocol(engine, tablePath) assert(protocol.getMinReaderVersion == 2 && protocol.getMinWriterVersion == 7) } } test("new table with existing column mappings in schema writes COLUMN_MAPPING_MAX_COLUMN_ID") { withTempDirAndEngine { (tablePath, engine) => val props = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> "id") val fieldMetadata = FieldMetadata.builder() .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, 1) .putString(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "col-0").build() val structField = new StructField("col_name", IntegerType.INTEGER, false, fieldMetadata) val schema = new StructType(Seq(structField).asJava) createEmptyTable(engine, tablePath, schema, tableProperties = props) val structtype = getMetadata(engine, tablePath).getSchema assertColumnMapping(structtype.get("col_name"), 1) assert(TableConfig.COLUMN_MAPPING_MAX_COLUMN_ID.fromMetadata(getMetadata( engine, tablePath)) == 1) } } test("can update existing table to column mapping mode = name") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable(engine, tablePath, simpleTestSchema) val structType = getMetadata(engine, tablePath).getSchema assert(structType.equals(simpleTestSchema)) val props = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> "name") updateTableMetadata(engine, tablePath, tableProperties = props) val updatedSchema = getMetadata(engine, tablePath).getSchema assertColumnMapping(updatedSchema.get("a"), 1, "a") assertColumnMapping(updatedSchema.get("b"), 2, "b") } } Seq("name", "id").foreach { startingCMMode => test(s"cannot update table with unsupported column mapping mode change: $startingCMMode") { withTempDirAndEngine { (tablePath, engine) => val props = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> startingCMMode) createEmptyTable(engine, tablePath, simpleTestSchema, tableProperties = props) val structType = getMetadata(engine, tablePath).getSchema assertColumnMapping(structType.get("a"), 1) assertColumnMapping(structType.get("b"), 2) val ex = intercept[IllegalArgumentException] { val props = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> "none") updateTableMetadata(engine, tablePath, tableProperties = props) } assert(ex.getMessage.contains(s"Changing column mapping mode " + s"from '$startingCMMode' to 'none' is not supported")) } } } test("cannot update column mapping mode from name to id on existing table") { withTempDirAndEngine { (tablePath, engine) => val props = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> "name") createEmptyTable(engine, tablePath, simpleTestSchema, tableProperties = props) val structType = getMetadata(engine, tablePath).getSchema assertColumnMapping(structType.get("a"), 1) assertColumnMapping(structType.get("b"), 2) val ex = intercept[IllegalArgumentException] { val props = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> "id") updateTableMetadata(engine, tablePath, tableProperties = props) } assert(ex.getMessage.contains("Changing column mapping mode " + "from 'name' to 'id' is not supported")) } } test("cannot update column mapping mode from none to id on existing table") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable(engine, tablePath, simpleTestSchema) val structType = getMetadata(engine, tablePath).getSchema assert(structType.equals(simpleTestSchema)) val ex = intercept[IllegalArgumentException] { val props = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> "id") updateTableMetadata(engine, tablePath, tableProperties = props) } assert(ex.getMessage.contains("Changing column mapping mode " + "from 'none' to 'id' is not supported")) } } test("update table properties on a column mapping enabled table") { withTempDirAndEngine { (tablePath, engine) => val props = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> "name") createEmptyTable(engine, tablePath, simpleTestSchema, tableProperties = props) val metadata = getMetadata(engine, tablePath) assertColumnMapping(metadata.getSchema.get("a"), 1) assertColumnMapping(metadata.getSchema.get("b"), 2) val newProps = Map("key" -> "value") updateTableMetadata(engine, tablePath, tableProperties = newProps) assert(getMetadata(engine, tablePath).getConfiguration.get("key") == "value") } } Seq(true, false).foreach { withIcebergCompatV2 => test(s"new table with column mapping mode = name and nested schema, " + s"enableIcebergCompatV2 = $withIcebergCompatV2") { withTempDirAndEngine { (tablePath, engine) => val props = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "name", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> withIcebergCompatV2.toString) createEmptyTable(engine, tablePath, cmTestSchema(), tableProperties = props) verifyCMTestSchemaHasValidColumnMappingInfo( getMetadata(engine, tablePath), isNewTable = true, enableIcebergCompatV2 = withIcebergCompatV2) } } } test("subsequent updates don't update the metadata again when there is no change") { withTempDirAndEngine { (tablePath, engine) => val props = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "name", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true") createEmptyTable(engine, tablePath, testSchema, tableProperties = props) appendData(engine, tablePath, data = Seq.empty) // version 1 appendData(engine, tablePath, data = Seq.empty) // version 2 val table = Table.forPath(engine, tablePath) assert(getMetadataActionFromCommit(engine, table, version = 0).isDefined) assert(getMetadataActionFromCommit(engine, table, version = 1).isEmpty) assert(getMetadataActionFromCommit(engine, table, version = 2).isEmpty) } } Seq("name", "id").foreach { cmMode => test(s"test writing data into a column mapping enabled table is blocked: $cmMode") { withTempDirAndEngine { (tablePath, engine) => val props = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> cmMode) createEmptyTable(engine, tablePath, testSchema, tableProperties = props) val ex = intercept[UnsupportedOperationException] { appendData(engine, tablePath, data = Seq(Map.empty[String, Literal] -> dataBatches1)) } assert(ex.getMessage.contains( "Writing into column mapping enabled table is not supported yet.")) } } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaIcebergCompatBaseSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import scala.collection.JavaConverters._ import scala.collection.immutable.Seq import scala.reflect.ClassTag import io.delta.kernel.defaults.utils.AbstractWriteUtils import io.delta.kernel.exceptions.KernelException import io.delta.kernel.internal.TableConfig import io.delta.kernel.internal.tablefeatures.{TableFeature, TableFeatures} import io.delta.kernel.internal.util.ColumnMappingSuiteBase import io.delta.kernel.types.{DataType, DateType, IntegerType, LongType, StructField, StructType, TimestampNTZType, TypeChange} import org.scalatest.funsuite.AnyFunSuite /** * Base suite containing common test cases for Delta Iceberg compatibility features. * This includes tests that apply to both V2 and V3 compatibility modes. */ trait DeltaIcebergCompatBaseSuite extends AnyFunSuite with AbstractWriteUtils with ColumnMappingSuiteBase { /** The name of the iceberg compatibility version for display in test names */ def icebergCompatVersion: String /** The table property key for enabling the specific iceberg compatibility version */ def icebergCompatEnabledKey: String /** The table feature that should be enabled for this compatibility version */ def expectedTableFeatures: Seq[TableFeature] def supportedDataColumnTypes: Seq[DataType] def supportedPartitionColumnTypes: Seq[DataType] supportedDataColumnTypes.foreach { dataType: DataType => test(s"allowed data column types: $dataType on creating table") { withTempDirAndEngine { (tablePath, engine) => val schema = new StructType().add("col", dataType) val tblProps = Map(icebergCompatEnabledKey -> "true") createEmptyTable(engine, tablePath, schema, tableProperties = tblProps) } } } supportedPartitionColumnTypes.foreach { dataType: DataType => test(s"allowed partition column types: $dataType on creating table") { withTempDirAndEngine { (tablePath, engine) => val schema = new StructType().add("col", dataType) val partitionCols = Seq("col") val tblProps = Map(icebergCompatEnabledKey -> "true") createEmptyTable(engine, tablePath, schema, partitionCols, tableProperties = tblProps) } } } test(s"enable $icebergCompatVersion on creating table") { withTempDirAndEngine { (tablePath, engine) => val tblProps = Map(icebergCompatEnabledKey -> "true") createEmptyTable(engine, tablePath, cmTestSchema(), tableProperties = tblProps) val protocol = getProtocol(engine, tablePath) expectedTableFeatures.foreach { feature => assert(protocol.supportsFeature(feature)) } val metadata = getMetadata(engine, tablePath) val actualCMMode = metadata.getConfiguration.get(TableConfig.COLUMN_MAPPING_MODE.getKey) assert(actualCMMode === "name") verifyCMTestSchemaHasValidColumnMappingInfo(metadata) } } test(s"compatible type widening is allowed with $icebergCompatVersion") { withTempDirAndEngine { (tablePath, engine) => // Create a table with icebergCompat and type widening enabled val schema = new StructType() .add(new StructField( "intToLong", LongType.LONG, false).withTypeChanges(Seq(new TypeChange(IntegerType.INTEGER, LongType.LONG)).asJava)) val tblProps = Map( icebergCompatEnabledKey -> "true", TableConfig.TYPE_WIDENING_ENABLED.getKey -> "true") // This should not throw an exception createEmptyTable(engine, tablePath, schema, tableProperties = tblProps) appendData(engine, tablePath, data = Seq.empty) val protocol = getProtocol(engine, tablePath) assert(protocol.supportsFeature(TableFeatures.TYPE_WIDENING_RW_FEATURE)) val metadata = getMetadata(engine, tablePath) assert(metadata.getSchema.get("intToLong").getTypeChanges.asScala == schema.get( "intToLong").getTypeChanges.asScala) } } test(s"incompatible type widening throws exception with $icebergCompatVersion") { withTempDirAndEngine { (tablePath, engine) => // Try to create a table with icebergCompat and incompatible type widening val schema = new StructType() .add( new StructField( "dateToTimestamp", TimestampNTZType.TIMESTAMP_NTZ, false).withTypeChanges(Seq( new TypeChange(DateType.DATE, TimestampNTZType.TIMESTAMP_NTZ)).asJava)) val tblProps = Map( icebergCompatEnabledKey -> "true", TableConfig.TYPE_WIDENING_ENABLED.getKey -> "true") val e = intercept[KernelException] { createEmptyTable(engine, tablePath, schema, tableProperties = tblProps) } assert( e.getMessage.contains( s"$icebergCompatVersion does not support type widening present in table")) } } /** * Utility that checks after executing given fn gets the given exception and error message. * [[ClassTag]] is used to preserve the type information during the runtime. */ def checkError[T <: Throwable: ClassTag](expectedMessage: String)(fn: => Unit): Unit = { val e = intercept[T] { fn } assert(e.getMessage.contains(expectedMessage)) } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaIcebergCompatV2Suite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import scala.collection.JavaConverters._ import scala.collection.immutable.Seq import scala.reflect.ClassTag import io.delta.kernel.Table import io.delta.kernel.data.Row import io.delta.kernel.defaults.utils.{WriteUtils, WriteUtilsWithV2Builders} import io.delta.kernel.exceptions.KernelException import io.delta.kernel.internal.TableConfig import io.delta.kernel.internal.tablefeatures.{TableFeature, TableFeatures} import io.delta.kernel.internal.util.{ColumnMapping, VectorUtils} import io.delta.kernel.types.{DataType, DateType, FieldMetadata, StructField, StructType, TimestampNTZType, TypeChange} class DeltaIcebergCompatV2TransactionBuilderV1Suite extends DeltaIcebergCompatV2SuiteBase with WriteUtils {} class DeltaIcebergCompatV2TransactionBuilderV2Suite extends DeltaIcebergCompatV2SuiteBase with WriteUtilsWithV2Builders {} /** This suite tests reading or writing into Delta table that have `icebergCompatV2` enabled. */ trait DeltaIcebergCompatV2SuiteBase extends DeltaIcebergCompatBaseSuite { override def icebergCompatVersion: String = "icebergCompatV2" override def icebergCompatEnabledKey: String = TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey override def expectedTableFeatures: Seq[TableFeature] = Seq( TableFeatures.ICEBERG_COMPAT_V2_W_FEATURE, TableFeatures.COLUMN_MAPPING_RW_FEATURE) override def supportedDataColumnTypes: Seq[DataType] = ALL_TYPES.toList override def supportedPartitionColumnTypes: Seq[DataType] = PRIMITIVE_TYPES.toList ignore("can't enable icebergCompatV2 on a table with icebergCompatv1 enabled") { // We can't test this as Kernel throws error when enabling icebergCompatV1 // as there is no support it in the current version. // This is covered in unittests in [[IcebergCompatV2MetadataValidatorAndUpdaterSuite]] } ignore("test unsupported data types") { // Can't test this now as the only unsupported data type in Iceberg is VariantType, // and it also has no write support in Kernel. // Unit test for this is covered in the respective MetadataValidatorAndUpdaterSuite } Seq("id", "name").foreach { existingCMMode => // also tests enabling icebergCompat on an existing table test(s"existing column mapping mode `$existingCMMode` is " + s"preserved after $icebergCompatVersion is enabled") { withTempDirAndEngine { (tablePath, engine) => val tblProps = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> existingCMMode) createEmptyTable(engine, tablePath, cmTestSchema(), tableProperties = tblProps) val newTblProps = Map(icebergCompatEnabledKey -> "true") updateTableMetadata(engine, tablePath, tableProperties = newTblProps) val protocol = getProtocol(engine, tablePath) assert(protocol.supportsFeature(TableFeatures.COLUMN_MAPPING_RW_FEATURE)) val metadata = getMetadata(engine, tablePath) val actualCMMode = metadata.getConfiguration.get(TableConfig.COLUMN_MAPPING_MODE.getKey) assert(actualCMMode === existingCMMode) verifyCMTestSchemaHasValidColumnMappingInfo(metadata) } } } Seq("id", "name").foreach { existingCMMode => test(s"existing column mapping mode `$existingCMMode` is " + s"preserved after $icebergCompatVersion is enabled for new table") { withTempDirAndEngine { (tablePath, engine) => val tblProps = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> existingCMMode, icebergCompatEnabledKey -> "true") createEmptyTable(engine, tablePath, cmTestSchema(), tableProperties = tblProps) val protocol = getProtocol(engine, tablePath) assert(protocol.supportsFeature(TableFeatures.COLUMN_MAPPING_RW_FEATURE)) val metadata = getMetadata(engine, tablePath) val actualCMMode = metadata.getConfiguration.get(TableConfig.COLUMN_MAPPING_MODE.getKey) assert(actualCMMode === existingCMMode) verifyCMTestSchemaHasValidColumnMappingInfo(metadata) } } } test(s"when column mapping mode is set to 'none`, should fail enabling $icebergCompatVersion") { withTempDirAndEngine { (tablePath, engine) => val tblProps = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> "none") createEmptyTable(engine, tablePath, testSchema, tableProperties = tblProps) checkError[KernelException]( s"The value 'none' for the property 'delta.columnMapping.mode' is not " + s"compatible with $icebergCompatVersion requirements") { val newTblProps = Map(icebergCompatEnabledKey -> "true") updateTableMetadata(engine, tablePath, tableProperties = newTblProps) } } } test(s"can't enable $icebergCompatVersion on an existing table with no column mapping enabled") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable(engine, tablePath, testSchema) checkError[KernelException]( s"The value 'none' for the property 'delta.columnMapping.mode' is not " + s"compatible with $icebergCompatVersion requirements") { val tblProps = Map(icebergCompatEnabledKey -> "true") updateTableMetadata(engine, tablePath, testSchema, tableProperties = tblProps) } } } test("subsequent writes to icebergCompatV2 enabled tables doesn't update metadata") { // we want to make sure the [[IcebergCompatV2MetadataValidatorAndUpdater]] doesn't // make unneeded metadata updates withTempDirAndEngine { (tablePath, engine) => val tblProps = Map(TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true") createEmptyTable(engine, tablePath, testSchema, tableProperties = tblProps) val metadata = getMetadata(engine, tablePath) val actualCMMode = metadata.getConfiguration.get(TableConfig.COLUMN_MAPPING_MODE.getKey) assert(actualCMMode === "name") appendData(engine, tablePath, data = Seq.empty) // version 1 appendData(engine, tablePath, data = Seq.empty) // version 2 val table = Table.forPath(engine, tablePath) assert(getMetadataActionFromCommit(engine, table, version = 0).isDefined) assert(getMetadataActionFromCommit(engine, table, version = 1).isEmpty) assert(getMetadataActionFromCommit(engine, table, version = 2).isEmpty) // make a metadata update and see it is reflected in the table val newProps = Map("key" -> "value") updateTableMetadata(engine, tablePath, tableProperties = newProps) // version 3 val ver3Metadata: Row = getMetadataActionFromCommit(engine, table, version = 3) .getOrElse(fail("Metadata action not found in version 3")) // TODO: ugly, find a better utilities val result = VectorUtils.toJavaMap[String, String]( ver3Metadata.getMap(ver3Metadata.getSchema.indexOf("configuration"))) .get("key") assert(result === "value") } } test(s"can't be enabled on a new table with deletion vectors supported") { withTempDirAndEngine { (tablePath, engine) => checkError[KernelException]( s"Table features [deletionVectors] are " + s"incompatible with $icebergCompatVersion") { val tblProps = Map( TableConfig.DELETION_VECTORS_CREATION_ENABLED.getKey -> "true", icebergCompatEnabledKey -> "true") createEmptyTable(engine, tablePath, schema = testSchema, tableProperties = tblProps) } } } test(s"can't update an existing table with DVs supported to have $icebergCompatVersion") { withTempDirAndEngine { (tablePath, engine) => val tblProps = Map( TableConfig.DELETION_VECTORS_CREATION_ENABLED.getKey -> "true", // without CM on existing table, you can't update to icebergCompat TableConfig.COLUMN_MAPPING_MODE.getKey -> "name") createEmptyTable(engine, tablePath, schema = testSchema, tableProperties = tblProps) checkError[KernelException]( s"Table features [deletionVectors] are " + s"incompatible with $icebergCompatVersion") { val newTblProps = Map(icebergCompatEnabledKey -> "true") updateTableMetadata(engine, tablePath, tableProperties = newTblProps) } } } test(s"can't enable deletion vectors on a table with $icebergCompatVersion enabled") { withTempDirAndEngine { (tablePath, engine) => val tblProps = Map(icebergCompatEnabledKey -> "true") createEmptyTable(engine, tablePath, schema = testSchema, tableProperties = tblProps) checkError[KernelException]( s"Table features [deletionVectors] are incompatible with $icebergCompatVersion") { val tblProps = Map(TableConfig.DELETION_VECTORS_CREATION_ENABLED.getKey -> "true") updateTableMetadata(engine, tablePath, schema = testSchema, tableProperties = tblProps) } } } test( s"incompatible type widening throws exception with" + s" $icebergCompatVersion enabled on existing table") { withTempDirAndEngine { (tablePath, engine) => val schema = new StructType() .add(new StructField( "dateToTimestamp", TimestampNTZType.TIMESTAMP_NTZ, false, FieldMetadata.builder() .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, 1) .putString( ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "col-1").build()).withTypeChanges( Seq(new TypeChange(DateType.DATE, TimestampNTZType.TIMESTAMP_NTZ)).asJava)) val tblProps = Map(TableConfig.TYPE_WIDENING_ENABLED.getKey -> "true") createEmptyTable(engine, tablePath, schema, tableProperties = tblProps) val e = intercept[KernelException] { updateTableMetadata( engine, tablePath, tableProperties = Map( icebergCompatEnabledKey -> "true", TableConfig.COLUMN_MAPPING_MODE.getKey -> "ID")) } assert( e.getMessage.contains( s"$icebergCompatVersion does not support type widening present in table")) } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaIcebergCompatV3Suite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import scala.collection.JavaConverters._ import scala.collection.immutable.Seq import io.delta.kernel.Table import io.delta.kernel.data.Row import io.delta.kernel.defaults.utils.{WriteUtils, WriteUtilsWithV2Builders} import io.delta.kernel.exceptions.KernelException import io.delta.kernel.internal.TableConfig import io.delta.kernel.internal.tablefeatures.{TableFeature, TableFeatures} import io.delta.kernel.internal.util.{ColumnMapping, VectorUtils} import io.delta.kernel.types.{DataType, DateType, FieldMetadata, IntegerType, LongType, StructField, StructType, TimestampNTZType, TypeChange, VariantType} class DeltaIcebergCompatV3TransactionBuilderV1Suite extends DeltaIcebergCompatV3SuiteBase with WriteUtils {} class DeltaIcebergCompatV3TransactionBuilderV2Suite extends DeltaIcebergCompatV3SuiteBase with WriteUtilsWithV2Builders {} /** This suite tests reading or writing into Delta table that have `icebergCompatV3` enabled. */ trait DeltaIcebergCompatV3SuiteBase extends DeltaIcebergCompatBaseSuite { override def icebergCompatVersion: String = "icebergCompatV3" override def icebergCompatEnabledKey: String = TableConfig.ICEBERG_COMPAT_V3_ENABLED.getKey override def expectedTableFeatures: Seq[TableFeature] = Seq( TableFeatures.ICEBERG_COMPAT_V3_W_FEATURE, TableFeatures.COLUMN_MAPPING_RW_FEATURE, TableFeatures.ROW_TRACKING_W_FEATURE) override def supportedDataColumnTypes: Seq[DataType] = // TODO add VARIANT_TYPE once it is supported (PRIMITIVE_TYPES.toList ++ NESTED_TYPES.toList) // ++ Seq(VariantType.VARIANT)) override def supportedPartitionColumnTypes: Seq[DataType] = PRIMITIVE_TYPES.toList test(s"enable $icebergCompatVersion on a new table - verify row tracking is enabled") { withTempDirAndEngine { (tablePath, engine) => val tblProps = Map(icebergCompatEnabledKey -> "true") createEmptyTable(engine, tablePath, cmTestSchema(), tableProperties = tblProps) val protocol = getProtocol(engine, tablePath) expectedTableFeatures.foreach { feature => assert(protocol.supportsFeature(feature)) } val metadata = getMetadata(engine, tablePath) assert(metadata.getConfiguration.get(TableConfig.ROW_TRACKING_ENABLED.getKey) === "true") assert( metadata.getConfiguration.get(TableConfig.MATERIALIZED_ROW_ID_COLUMN_NAME.getKey).nonEmpty) assert(metadata.getConfiguration.get( TableConfig.MATERIALIZED_ROW_COMMIT_VERSION_COLUMN_NAME.getKey).nonEmpty) } } test(s"enable $icebergCompatVersion on a new table with deletion vectors") { withTempDirAndEngine { (tablePath, engine) => val tblProps = Map( TableConfig.DELETION_VECTORS_CREATION_ENABLED.getKey -> "true", icebergCompatEnabledKey -> "true") createEmptyTable(engine, tablePath, schema = testSchema, tableProperties = tblProps) } } test("can't enable icebergCompatV3 on a existing table") { withTempDirAndEngine { (tablePath, engine) => val tblProps = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> "name") createEmptyTable(engine, tablePath, schema = testSchema, tableProperties = tblProps) checkError[KernelException]( "Cannot enable delta.enableIcebergCompatV3 on an existing table") { val newTblProps = Map(TableConfig.ICEBERG_COMPAT_V3_ENABLED.getKey -> "true") updateTableMetadata(engine, tablePath, tableProperties = newTblProps) } } } test("can't disable icebergCompatV3 on a existing icebergCompatV3 enabled table") { withTempDirAndEngine { (tablePath, engine) => val tblProps = Map(TableConfig.ICEBERG_COMPAT_V3_ENABLED.getKey -> "true") createEmptyTable(engine, tablePath, schema = testSchema, tableProperties = tblProps) checkError[KernelException]( "Disabling delta.enableIcebergCompatV3 on an existing table is not allowed") { val newTblProps = Map(TableConfig.ICEBERG_COMPAT_V3_ENABLED.getKey -> "false") updateTableMetadata(engine, tablePath, tableProperties = newTblProps) } } } test("subsequent writes to icebergCompatV3 enabled tables doesn't update metadata") { // we want to make sure the [[IcebergCompatV3MetadataValidatorAndUpdater]] doesn't // make unneeded metadata updates withTempDirAndEngine { (tablePath, engine) => val tblProps = Map( TableConfig.ICEBERG_COMPAT_V3_ENABLED.getKey -> "true") createEmptyTable(engine, tablePath, testSchema, tableProperties = tblProps) val metadata = getMetadata(engine, tablePath) val actualCMMode = metadata.getConfiguration.get(TableConfig.COLUMN_MAPPING_MODE.getKey) assert(actualCMMode === "name") appendData(engine, tablePath, data = Seq.empty) // version 1 appendData(engine, tablePath, data = Seq.empty) // version 2 val table = Table.forPath(engine, tablePath) assert(getMetadataActionFromCommit(engine, table, version = 0).isDefined) assert(getMetadataActionFromCommit(engine, table, version = 1).isEmpty) assert(getMetadataActionFromCommit(engine, table, version = 2).isEmpty) // make a metadata update and see it is reflected in the table val newProps = Map("key" -> "value") updateTableMetadata(engine, tablePath, tableProperties = newProps) // version 3 val ver3Metadata: Row = getMetadataActionFromCommit(engine, table, version = 3) .getOrElse(fail("Metadata action not found in version 3")) // TODO: ugly, find a better utilities val result = VectorUtils.toJavaMap[String, String]( ver3Metadata.getMap(ver3Metadata.getSchema.indexOf("configuration"))) .get("key") assert(result === "value") } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaLogActionUtilsE2ESuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.io.File import java.util.Optional import scala.collection.JavaConverters._ import io.delta.kernel.defaults.utils.TestUtils import io.delta.kernel.exceptions.TableNotFoundException import io.delta.kernel.internal.DeltaLogActionUtils.listDeltaLogFilesAsIter import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.util.FileNames import org.scalatest.funsuite.AnyFunSuite /** Test suite for end-to-end cases. See also the mocked unit tests in DeltaLogActionUtilsSuite. */ class DeltaLogActionUtilsE2ESuite extends AnyFunSuite with TestUtils { test("listDeltaLogFiles: throws TableNotFoundException if _delta_log does not exist") { withTempDir { tableDir => intercept[TableNotFoundException] { listDeltaLogFilesAsIter( defaultEngine, Set(FileNames.DeltaLogFileType.COMMIT, FileNames.DeltaLogFileType.CHECKPOINT).asJava, new Path(tableDir.getAbsolutePath), 0, Optional.empty(), true /* mustBeRecreatable */ ).toInMemoryList } } } test("listDeltaLogFiles: returns empty list if _delta_log is empty") { withTempDir { tableDir => val logDir = new File(tableDir, "_delta_log") assert(logDir.mkdirs() && logDir.isDirectory && logDir.listFiles().isEmpty) val result = listDeltaLogFilesAsIter( defaultEngine, Set(FileNames.DeltaLogFileType.COMMIT, FileNames.DeltaLogFileType.CHECKPOINT).asJava, new Path(tableDir.getAbsolutePath), 0, Optional.empty(), true /* mustBeRecreatable */ ).toInMemoryList assert(result.isEmpty) } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaReplaceTableColumnMappingSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import scala.collection.immutable.Seq import scala.reflect.ClassTag import io.delta.kernel.defaults.utils.{WriteUtils, WriteUtilsWithV2Builders} import io.delta.kernel.exceptions.KernelException import io.delta.kernel.internal.TableConfig import io.delta.kernel.internal.util.{ColumnMapping, ColumnMappingSuiteBase} import io.delta.kernel.internal.util.ColumnMapping.ColumnMappingMode import io.delta.kernel.types.{ArrayType, DataType, FieldMetadata, IntegerType, LongType, MapType, StringType, StructField, StructType} class DeltaReplaceTableColumnMappingNameModeTransactionBuilderV1Suite extends DeltaReplaceTableColumnMappingNameModeSuite with WriteUtils class DeltaReplaceTableColumnMappingNameModeTransactionBuilderV2Suite extends DeltaReplaceTableColumnMappingNameModeSuite with WriteUtilsWithV2Builders class DeltaReplaceTableColumnMappingIdModeTransactionBuilderV1Suite extends DeltaReplaceTableColumnMappingIdModeSuite with WriteUtils class DeltaReplaceTableColumnMappingIdModeTransactionBuilderV2Suite extends DeltaReplaceTableColumnMappingIdModeSuite with WriteUtilsWithV2Builders abstract class DeltaReplaceTableColumnMappingNameModeSuite extends DeltaReplaceTableColumnMappingSuiteBase { override def tblPropertiesCmEnabled: Map[String, String] = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> "name") } abstract class DeltaReplaceTableColumnMappingIdModeSuite extends DeltaReplaceTableColumnMappingSuiteBase { override def tblPropertiesCmEnabled: Map[String, String] = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> "id") // We only need to run the below tests once since they check combos of id and name mode, put them // in this suite for this reason ColumnMapping.ColumnMappingMode.values().foreach { initialCmMode => ColumnMapping.ColumnMappingMode.values().foreach { replaceCmMode => if (initialCmMode != replaceCmMode) { test(s"Cannot change CM mode from $initialCmMode to $replaceCmMode") { withTempDirAndEngine { (tablePath, engine) => createInitialTable( engine, tablePath, tableProperties = cmModeTblProperties(initialCmMode), includeData = false) assert(intercept[UnsupportedOperationException] { commitReplaceTable( engine, tablePath, tableProperties = cmModeTblProperties(replaceCmMode)) }.getMessage.contains( s"Changing column mapping mode from $initialCmMode to $replaceCmMode is not " + s"currently supported in Kernel during REPLACE TABLE")) } } } else if (initialCmMode != ColumnMappingMode.NONE) { test(s"Replace with entirely new schema for cmMode=$initialCmMode assigns CM info") { withTempDirAndEngine { (tablePath, engine) => createInitialTable( engine, tablePath, schema = new StructType().add("col1", StringType.STRING), tableProperties = cmModeTblProperties(initialCmMode), includeData = false) commitReplaceTable( engine, tablePath, cmTestSchema(), tableProperties = cmModeTblProperties(replaceCmMode)) verifyCMTestSchemaHasValidColumnMappingInfo( getMetadata(engine, tablePath), enableIcebergCompatV2 = false, initialFieldId = 1) } } } } } } trait DeltaReplaceTableColumnMappingSuiteBase extends DeltaReplaceTableSuiteBase with ColumnMappingSuiteBase { // Child suites override this to run tests with either id or name based column mapping def tblPropertiesCmEnabled: Map[String, String] /* ------ Test helpers ------- */ def cmModeTblProperties(mode: ColumnMappingMode): Map[String, String] = { Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> mode.value) } implicit class StructFieldOps(field: StructField) { def withCMMetadata(physicalName: String, fieldId: Long): StructField = { field.withNewMetadata( FieldMetadata.builder() .fromMetadata(field.getMetadata) .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, fieldId) .putString(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, physicalName) .build()) } } def singletonSchema(colName: String, dataType: DataType): StructType = { val topLevelCol = new StructField(colName, dataType, true) .withCMMetadata(colName + "-physicalName", 4) new StructType().add(topLevelCol) } def nestedStructSchema(nestedField: StructField): StructType = { singletonSchema("top-struct", new StructType().add(nestedField)) } def nestedArraySchema(nestedField: StructField): StructType = { singletonSchema( "array-col", new ArrayType(new StructType().add(nestedField), true)) } def nestedMapKeySchema(nestedField: StructField): StructType = { singletonSchema( "map-col", new MapType(new StructType().add(nestedField), StringType.STRING, true)) } def nestedMapValueSchema(nestedField: StructField): StructType = { singletonSchema( "map-col", new MapType(StringType.STRING, new StructType().add(nestedField), true)) } implicit val exceptionType = ClassTag(classOf[KernelException]) def checkReplaceThrowsException[T <: Throwable]( initialSchema: StructType, replaceSchema: StructType, expectedErrorMessageContains: String)(implicit exceptionType: ClassTag[T]): Unit = { withTempDirAndEngine { (tablePath, engine) => createInitialTable( engine, tablePath, schema = initialSchema, includeData = false, tableProperties = tblPropertiesCmEnabled) val e = intercept[T] { commitReplaceTable( engine, tablePath, schema = replaceSchema, tableProperties = tblPropertiesCmEnabled) } assert(e.getMessage.contains(expectedErrorMessageContains)) } } def checkReplaceSucceeds( initialSchema: StructType, replaceSchema: StructType): Unit = { withTempDirAndEngine { (tablePath, engine) => createInitialTable( engine, tablePath, schema = initialSchema, includeData = false, tableProperties = tblPropertiesCmEnabled) commitReplaceTable( engine, tablePath, schema = replaceSchema, tableProperties = tblPropertiesCmEnabled) assert(getMetadata(engine, tablePath).getSchema == replaceSchema) } } def testCompatibleFieldIdReuseDifferentNestings( testDescription: String, initialField: StructField, replaceField: StructField): Unit = { val initialFieldComplete = initialField.withCMMetadata("col-1", 1) val replaceFieldComplete = replaceField.withCMMetadata("col-1", 1) test(s"$testDescription - top-level field") { val initialSchema = new StructType().add(initialFieldComplete) val replaceSchema = new StructType().add(replaceFieldComplete) checkReplaceSucceeds(initialSchema, replaceSchema) } test(s"$testDescription - nested within a struct, with struct fieldIdReuse") { val initialSchema = nestedStructSchema(initialFieldComplete) val replaceSchema = nestedStructSchema(replaceFieldComplete) checkReplaceSucceeds(initialSchema, replaceSchema) } test(s"$testDescription - nested within a struct in an array, with array fieldIdReuse") { val initialSchema = nestedArraySchema(initialFieldComplete) val replaceSchema = nestedArraySchema(replaceFieldComplete) checkReplaceSucceeds(initialSchema, replaceSchema) } test(s"$testDescription - nested within a struct in an map (key), with array fieldIdReuse") { val initialSchema = nestedMapKeySchema(initialFieldComplete) val replaceSchema = nestedMapKeySchema(replaceFieldComplete) checkReplaceSucceeds(initialSchema, replaceSchema) } test(s"$testDescription - nested within a struct in an map (value), with array fieldIdReuse") { val initialSchema = nestedMapValueSchema(initialFieldComplete) val replaceSchema = nestedMapValueSchema(replaceFieldComplete) checkReplaceSucceeds(initialSchema, replaceSchema) } } def testIncompatibleFieldIdReuseDifferentNestings[T <: Throwable]( testDescription: String, initialField: StructField, replaceField: StructField, expectedErrorMessageContains: String, initialPhysicalName: String = "col-1", replacePhysicalName: String = "col-1")(implicit exceptionType: ClassTag[T]): Unit = { val initialFieldComplete = initialField.withCMMetadata(initialPhysicalName, 1) val replaceFieldComplete = replaceField.withCMMetadata(replacePhysicalName, 1) test(s"$testDescription - top-level field") { val initialSchema = new StructType().add(initialFieldComplete) val replaceSchema = new StructType().add(replaceFieldComplete) checkReplaceThrowsException[T]( initialSchema, replaceSchema, expectedErrorMessageContains) } test(s"$testDescription - nested within a struct, with struct fieldIdReuse") { val initialSchema = nestedStructSchema(initialFieldComplete) val replaceSchema = nestedStructSchema(replaceFieldComplete) checkReplaceThrowsException[T]( initialSchema, replaceSchema, expectedErrorMessageContains) } test(s"$testDescription - nested within a struct in an array, with array fieldIdReuse") { val initialSchema = nestedArraySchema(initialFieldComplete) val replaceSchema = nestedArraySchema(replaceFieldComplete) checkReplaceThrowsException[T]( initialSchema, replaceSchema, expectedErrorMessageContains) } test(s"$testDescription - nested within a struct in a map (key), with array fieldIdReuse") { val initialSchema = nestedMapKeySchema(initialFieldComplete) val replaceSchema = nestedMapKeySchema(replaceFieldComplete) checkReplaceThrowsException[T]( initialSchema, replaceSchema, expectedErrorMessageContains) } test(s"$testDescription - nested within a struct in a map (value), with array fieldIdReuse") { val initialSchema = nestedMapKeySchema(initialFieldComplete) val replaceSchema = nestedMapKeySchema(replaceFieldComplete) checkReplaceThrowsException[T]( initialSchema, replaceSchema, expectedErrorMessageContains) } } /* ---------------- TEST CASES ---------------- */ testIncompatibleFieldIdReuseDifferentNestings( "Cannot reuse fieldId incompatible primitive type", new StructField("col1", StringType.STRING, true), new StructField("col1", IntegerType.INTEGER, true), "Cannot change the type of existing field") testIncompatibleFieldIdReuseDifferentNestings( "Cannot reuse fieldId incompatible primitive type w/logical name change", new StructField("col1", StringType.STRING, true), new StructField("col2", IntegerType.INTEGER, true), "Cannot change the type of existing field") testIncompatibleFieldIdReuseDifferentNestings( "Cannot reuse fieldId incompatible nullability", new StructField("col1", StringType.STRING, true), new StructField("col1", StringType.STRING, false), "Cannot tighten the nullability of existing field") testIncompatibleFieldIdReuseDifferentNestings( "Cannot reuse fieldId incompatible nullability for array-type field", new StructField("col1", new ArrayType(StringType.STRING, true), true), new StructField("col1", new ArrayType(StringType.STRING, false), true), "Cannot tighten the nullability of existing field") testIncompatibleFieldIdReuseDifferentNestings( "Cannot reuse fieldId incompatible nullability for map-type field", new StructField("col1", new MapType(StringType.STRING, StringType.STRING, true), true), new StructField("col1", new MapType(StringType.STRING, StringType.STRING, false), true), "Cannot tighten the nullability of existing field") testIncompatibleFieldIdReuseDifferentNestings( "Cannot reuse fieldId incompatible nullability for struct of struct", new StructField( "col1", new StructType() .add("col2", StringType.STRING, true), true), new StructField( "col1", new StructType() .add("col2", StringType.STRING, true), false), "Cannot tighten the nullability of existing field") testIncompatibleFieldIdReuseDifferentNestings( "Cannot reuse fieldId incompatible type for array-type field", new StructField("col1", new ArrayType(StringType.STRING, true), true), new StructField("col1", new ArrayType(IntegerType.INTEGER, true), true), "Cannot change the type of existing field") testIncompatibleFieldIdReuseDifferentNestings( "Cannot reuse fieldId incompatible key-type for map-type field", new StructField("col1", new MapType(StringType.STRING, StringType.STRING, true), true), new StructField("col1", new MapType(IntegerType.INTEGER, StringType.STRING, true), true), "Cannot change the type of existing field") testIncompatibleFieldIdReuseDifferentNestings( "Cannot reuse fieldId incompatible value-type for map-type field", new StructField("col1", new MapType(StringType.STRING, StringType.STRING, true), true), new StructField("col1", new MapType(StringType.STRING, IntegerType.INTEGER, true), true), "Cannot change the type of existing field") testIncompatibleFieldIdReuseDifferentNestings( "Cannot reuse fieldId incompatible primitive type, type-widening supported change", new StructField("col1", IntegerType.INTEGER, true), new StructField("col1", LongType.LONG, true), "Cannot change the type of existing field") test("Cannot add a new field with a fieldId <= maxColId") { val initialSchema = new StructType() .add(new StructField("col1", StringType.STRING, true).withCMMetadata("col-200", 200)) val replaceSchema = new StructType() .add(new StructField("col1", StringType.STRING, true).withCMMetadata("col-200", 200)) .add(new StructField("col2", StringType.STRING, true).withCMMetadata("col-1", 1)) checkReplaceThrowsException[IllegalArgumentException]( initialSchema, replaceSchema, "Cannot add a new column with a fieldId <= maxFieldId") } testIncompatibleFieldIdReuseDifferentNestings[IllegalArgumentException]( "Cannot reuse fieldId with change in physical name", new StructField("col1", StringType.STRING, true), new StructField("col1", StringType.STRING, true), "Existing field with id 1 in current schema has physical name " + "col1-physical-name which is different", initialPhysicalName = "col1-physical-name", replacePhysicalName = "0001111023383922") val validNullabilityChanges = Seq( (true, true), (false, false), (false, true)) validNullabilityChanges.foreach { case (initialNullable, replaceNullable) => testCompatibleFieldIdReuseDifferentNestings( s"Valid nullability + type change: initialNullable=$initialNullable to " + s"replaceNullable$replaceNullable - primitive type", new StructField("col1", StringType.STRING, initialNullable), new StructField("col1", StringType.STRING, replaceNullable)) testCompatibleFieldIdReuseDifferentNestings( s"Valid nullability + type change: initialNullable=$initialNullable to " + s"replaceNullable$replaceNullable - array type", new StructField("col1", new ArrayType(StringType.STRING, initialNullable), true), new StructField("col1", new ArrayType(StringType.STRING, replaceNullable), true)) testCompatibleFieldIdReuseDifferentNestings( s"Valid nullability + type change: initialNullable=$initialNullable to " + s"replaceNullable$replaceNullable - map type", new StructField( "col1", new MapType(StringType.STRING, StringType.STRING, initialNullable), true), new StructField( "col1", new MapType(StringType.STRING, StringType.STRING, replaceNullable), true)) testCompatibleFieldIdReuseDifferentNestings( s"Valid nullability + type change: initialNullable=$initialNullable to " + s"replaceNullable$replaceNullable - struct of struct", new StructField( "col1", new StructType() .add(new StructField("col2", StringType.STRING, true) // we must set the CM metadata for nested field so it doesn't update for replace .withCMMetadata("col-200", 200)), initialNullable), new StructField( "col1", new StructType() .add(new StructField("col2", StringType.STRING, true) .withCMMetadata("col-200", 200)), replaceNullable)) } testCompatibleFieldIdReuseDifferentNestings( "No type change with logical name change - primitive type", new StructField("col1", StringType.STRING, true), new StructField("col2", StringType.STRING, true)) test("Can add a new non-nullable column - top level primitive") { val initialSchema = new StructType() .add(new StructField("col1", StringType.STRING, true) .withCMMetadata("col-1", 1)) val replaceSchema = new StructType() .add(new StructField("col2", StringType.STRING, false) .withCMMetadata("col-2", 2)) checkReplaceSucceeds(initialSchema, replaceSchema) } testCompatibleFieldIdReuseDifferentNestings( "Can add a new non-nullable column - nested within a struct", new StructField( "struct", new StructType() .add(new StructField("col1", StringType.STRING, true) .withCMMetadata("col-100", 100)), true), new StructField( "struct", new StructType() .add(new StructField("col1", StringType.STRING, true) .withCMMetadata("col-100", 100)) .add(new StructField("col2", StringType.STRING, false) .withCMMetadata("col-200", 200)), true)) test("Cannot provide just a colId without physicalName") { val initialSchema = new StructType() .add(new StructField("col1", StringType.STRING, true).withCMMetadata("col-0", 0)) val replaceSchema = new StructType() .add( new StructField( "col1", StringType.STRING, true, FieldMetadata.builder() .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, 0) .build())) checkReplaceThrowsException[IllegalArgumentException]( initialSchema, replaceSchema, "Both columnId and physicalName must be present if one is present") } test("Assigns colId to new fields correctly based on previous maxFieldId with partial" + " fieldId reuse") { withTempDirAndEngine { (tablePath, engine) => val baseSchema = new StructType() .add(new StructField( "col1", StringType.STRING, true).withCMMetadata("col1-physical-name", 0)) val initialSchema = baseSchema .add("col2", StringType.STRING, true) .add("col3", StringType.STRING, true) createInitialTable( engine, tablePath, schema = initialSchema, tableProperties = tblPropertiesCmEnabled, includeData = false) assert(TableConfig.COLUMN_MAPPING_MAX_COLUMN_ID.fromMetadata(getMetadata( engine, tablePath)) == 2) // Update the schema such that the only present fieldId is 0, but the max should still be 2 updateTableMetadata( engine, tablePath, baseSchema) assert(TableConfig.COLUMN_MAPPING_MAX_COLUMN_ID.fromMetadata(getMetadata( engine, tablePath)) == 2) // Replace the table with a schema with some fieldId re-use, but also some new columns without // a fieldId val replaceSchema = baseSchema .add("col2", StringType.STRING, true) .add("col4", StringType.STRING, true) commitReplaceTable( engine, tablePath, schema = replaceSchema, tableProperties = tblPropertiesCmEnabled) assert(TableConfig.COLUMN_MAPPING_MAX_COLUMN_ID.fromMetadata(getMetadata( engine, tablePath)) == 4) val resultSchema = getMetadata(engine, tablePath).getSchema assert(ColumnMapping.getColumnId(resultSchema.get("col1")) == 0) assert(ColumnMapping.getColumnId(resultSchema.get("col2")) == 3) assert(ColumnMapping.getColumnId(resultSchema.get("col4")) == 4) assert(ColumnMapping.getPhysicalName(resultSchema.get("col1")) == "col1-physical-name") // These should have UUID physical names which start with "col-" assert(ColumnMapping.getPhysicalName(resultSchema.get("col2")).startsWith("col-")) assert(ColumnMapping.getPhysicalName(resultSchema.get("col4")).startsWith("col-")) } } test("Replace correctly updates the maxFieldId when providing fieldId in the replace schema") { withTempDirAndEngine { (tablePath, engine) => val initialSchema = new StructType() .add(new StructField( "col1", StringType.STRING, true).withCMMetadata("col1-physical-name", 0)) createInitialTable( engine, tablePath, schema = initialSchema, tableProperties = tblPropertiesCmEnabled, includeData = false) assert(TableConfig.COLUMN_MAPPING_MAX_COLUMN_ID.fromMetadata(getMetadata( engine, tablePath)) == 0) // Replace the table with a schema with new column with provided fieldId val replaceSchema = initialSchema .add(new StructField( "new-col", StringType.STRING, true).withCMMetadata("col-200", 200)) commitReplaceTable( engine, tablePath, schema = replaceSchema, tableProperties = tblPropertiesCmEnabled) assert(TableConfig.COLUMN_MAPPING_MAX_COLUMN_ID.fromMetadata(getMetadata( engine, tablePath)) == 200) } } // E2E tests that we disallow fieldId reuse when fields are moved out of their prior parent. // This validation is thoroughly unit tested in SchemaUtilsSuite. test("Cannot reuse fieldId when moving field out of parent struct to top-level") { val initialSchema = new StructType() .add(new StructField( "parent-struct", new StructType() .add(new StructField("nested-col", StringType.STRING, true) .withCMMetadata("nested-col-physical", 100)), true).withCMMetadata("parent-struct-physical", 1)) val replaceSchema = new StructType() .add(new StructField("nested-col", StringType.STRING, true) .withCMMetadata("nested-col-physical", 100)) checkReplaceThrowsException[KernelException]( initialSchema, replaceSchema, "Cannot move fields between different levels of nesting") } test("Cannot reuse fieldId when moving field from one parent struct to another") { // Initial: nested_struct (fieldId=0) with col1 (fieldId=1) inside val initialSchema = new StructType() .add(new StructField( "nested_struct", new StructType() .add(new StructField("col1", StringType.STRING, true) .withCMMetadata("col1-physical", 1)), true).withCMMetadata("nested_struct-physical", 0)) // Replace: nested_struct_new (fieldId=2) with col1 (fieldId=1) inside val replaceSchema = new StructType() .add(new StructField( "nested_struct_new", new StructType() .add(new StructField("col1", StringType.STRING, true) .withCMMetadata("col1-physical", 1)), true).withCMMetadata("nested_struct_new-physical", 2)) checkReplaceThrowsException[KernelException]( initialSchema, replaceSchema, "Cannot move fields between different levels of nesting") } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaReplaceTableSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import scala.collection.JavaConverters._ import scala.collection.immutable.Seq import io.delta.kernel.{Operation, Table, TableManager} import io.delta.kernel.commit.{CommitMetadata, CommitResponse, Committer} import io.delta.kernel.data.Row import io.delta.kernel.defaults.utils.{TestRow, WriteUtils, WriteUtilsWithV2Builders} import io.delta.kernel.engine.Engine import io.delta.kernel.exceptions.{KernelException, MaxCommitRetryLimitReachedException, TableNotFoundException} import io.delta.kernel.expressions.{Column, Literal} import io.delta.kernel.internal.{SnapshotImpl, TableConfig, TableImpl} import io.delta.kernel.internal.TableConfig._ import io.delta.kernel.internal.tablefeatures.TableFeatures import io.delta.kernel.types.{IntegerType, StringType, StructType} import io.delta.kernel.utils.CloseableIterable.emptyIterable import io.delta.kernel.utils.CloseableIterator class DeltaReplaceTableTransactionBuilderV1Suite extends DeltaReplaceTableSuite with WriteUtils class DeltaReplaceTableTransactionBuilderV2Suite extends DeltaReplaceTableSuite with WriteUtilsWithV2Builders { test("ReplaceTableTransactionBuilder uses the committer provided during snapshot building") { withTempDirAndEngine { (tablePath, engine) => class FakeCommitter extends Committer { override def commit( engine: Engine, finalizedActions: CloseableIterator[Row], commitMetadata: CommitMetadata): CommitResponse = { throw new RuntimeException("This is a fake committer") } } createEmptyTable( engine, tablePath, testSchema) // Build snapshot with committer and start txn val txn = TableManager.loadSnapshot(tablePath) .withCommitter(new FakeCommitter()) .build(engine).asInstanceOf[SnapshotImpl] .buildReplaceTableTransaction(testSchema, "test-engine") .build(engine) // Check the txn returns the correct committer assert(txn.getCommitter.isInstanceOf[FakeCommitter]) // Check that the txn invokes the provided committer upon commit val e = intercept[RuntimeException] { txn.commit(engine, emptyIterable()) } assert(e.getMessage.contains("This is a fake committer")) } } } abstract class DeltaReplaceTableSuite extends DeltaReplaceTableSuiteBase { /* ----- ERROR CASES ------ */ test("Conflict resolution is disabled for replace table") { withTempDirAndEngine { (tablePath, engine) => createInitialTable(engine, tablePath) // Start replace transaction - use default maxRetries val txn1 = getReplaceTxn(engine, tablePath, testSchema) // Start replace transaction - explicitly set maxRetries > 0 val txn2 = getReplaceTxn(engine, tablePath, testSchema, maxRetries = 100) // Commit a simple blind append as a conflicting txn appendData( engine, tablePath, data = Seq(Map.empty[String, Literal] -> (dataBatches2))) // Try to commit replace table and intercept conflicting txn (no conflict resolution) intercept[MaxCommitRetryLimitReachedException] { commitTransaction(txn1, engine, emptyIterable()) } intercept[MaxCommitRetryLimitReachedException] { commitTransaction(txn2, engine, emptyIterable()) } } } test("Table::createTransactionBuilder does not allow REPLACE TABLE") { withTempDirAndEngine { (tablePath, engine) => createInitialTable(engine, tablePath) assert(intercept[UnsupportedOperationException] { Table.forPath(engine, tablePath) .createTransactionBuilder(engine, testEngineInfo, Operation.REPLACE_TABLE) .build(engine) }.getMessage.contains("REPLACE TABLE is not yet supported")) } } test("Cannot replace a table that does not exist") { withTempDirAndEngine { (tablePath, engine) => assert( intercept[TableNotFoundException] { // This is not possible on an API level for V2 builders since building is from a Snapshot Table.forPath(engine, tablePath).asInstanceOf[TableImpl] .createReplaceTableTransactionBuilder(engine, "test-engine") .withSchema(engine, testSchema) .build(engine) }.getMessage.contains("Trying to replace a table that does not exist")) } } test("Cannot enable a feature that Kernel does not support") { withTempDirAndEngine { (tablePath, engine) => createInitialTable(engine, tablePath) assert( intercept[KernelException] { commitReplaceTable( engine, tablePath, tableProperties = Map(TableConfig.CHANGE_DATA_FEED_ENABLED.getKey -> "true")) }.getMessage.contains("Unsupported Delta writer feature")) } } test("Cannot replace a table with a protocol Kernel does not support") { withTempDirAndEngine { (tablePath, engine) => spark.sql( s""" |CREATE TABLE delta.`$tablePath` (id INT) USING DELTA |TBLPROPERTIES('delta.enableChangeDataFeed' = 'true') |""".stripMargin) assert( intercept[KernelException] { commitReplaceTable( engine, tablePath) }.getMessage.contains("Unsupported Delta writer feature")) } } test("Must provide a schema for replace table transaction") { withTempDirAndEngine { (tablePath, engine) => createInitialTable(engine, tablePath) assert(intercept[KernelException] { Table.forPath(engine, tablePath).asInstanceOf[TableImpl] .createReplaceTableTransactionBuilder(engine, "test-engine") .build(engine) }.getMessage.contains("Must provide a new schema for REPLACE TABLE")) } } test("Cannot define both partition and clustering columns at the same time") { withTempDirAndEngine { (tablePath, engine) => createInitialTable(engine, tablePath) assert(intercept[IllegalArgumentException] { // Setting both is not possible on an API level for v2 builders Table.forPath(engine, tablePath).asInstanceOf[TableImpl] .createReplaceTableTransactionBuilder(engine, "test-engine") .withSchema(engine, testPartitionSchema) .withPartitionColumns(engine, testPartitionColumns.asJava) .withClusteringColumns(engine, testClusteringColumns.asJava) .build(engine) }.getMessage.contains( "Partition Columns and Clustering Columns cannot be set at the same time")) } } test("Schema provided must be valid") { withTempDirAndEngine { (tablePath, engine) => createInitialTable(engine, tablePath) assert(intercept[KernelException] { getReplaceTxn( engine, tablePath, schema = new StructType().add("col", IntegerType.INTEGER).add("col", IntegerType.INTEGER)) }.getMessage.contains( "Schema contains duplicate columns")) } } test("Partition columns provided must be valid") { withTempDirAndEngine { (tablePath, engine) => createInitialTable(engine, tablePath) assert(intercept[IllegalArgumentException] { getReplaceTxn( engine, tablePath, schema = testSchema, partCols = Seq("foo")) }.getMessage.contains( "Partition column foo not found in the schema")) } } test("Clustering columns provided must be valid") { withTempDirAndEngine { (tablePath, engine) => createInitialTable(engine, tablePath) assert(intercept[KernelException] { getReplaceTxn( engine, tablePath, schema = testSchema, clusteringColsOpt = Some(Seq(new Column("foo")))) }.getMessage.contains( "Column 'column(`foo`)' was not found in the table schema")) } } test("icebergWriterCompatV1 checks are enforced") { withTempDirAndEngine { (tablePath, engine) => createInitialTable(engine, tablePath) assert( intercept[KernelException] { commitReplaceTable( engine, tablePath, tableProperties = Map( TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.getKey -> "true", TableConfig.COLUMN_MAPPING_MODE.getKey -> "name")) }.getMessage.contains("The value 'name' for the property 'delta.columnMapping.mode' is " + "not compatible with icebergWriterCompatV1 requirements")) } } test("icebergCompatV2 checks are enforced") { withTempDirAndEngine { (tablePath, engine) => createInitialTable(engine, tablePath) assert( intercept[KernelException] { commitReplaceTable( engine, tablePath, tableProperties = Map( TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true", TableConfig.DELETION_VECTORS_CREATION_ENABLED.getKey -> "true")) }.getMessage.contains( "Table features [deletionVectors] are incompatible with icebergCompatV2")) } } test("REPLACE is not supported on existing table with icebergCompatV3 feature") { withTempDirAndEngine { (tablePath, engine) => createInitialTable( engine, tablePath, tableProperties = Map(TableConfig.ICEBERG_COMPAT_V3_ENABLED.getKey -> "true"), includeData = false // To avoid writing data with correct CM schema ) assert( intercept[UnsupportedOperationException] { commitReplaceTable( engine, tablePath, tableProperties = Map(TableConfig.ICEBERG_COMPAT_V3_ENABLED.getKey -> "true")) }.getMessage.contains("REPLACE TABLE is not yet supported on IcebergCompatV3 tables")) } } test("REPLACE is not supported when enabling icebergCompatV3 feature") { withTempDirAndEngine { (tablePath, engine) => createInitialTable( engine, tablePath, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "name"), includeData = false // To avoid writing data with correct CM schema ) assert( intercept[UnsupportedOperationException] { commitReplaceTable( engine, tablePath, tableProperties = Map( TableConfig.ICEBERG_COMPAT_V3_ENABLED.getKey -> "true", TableConfig.COLUMN_MAPPING_MODE.getKey -> "name")) }.getMessage.contains("REPLACE TABLE is not yet supported on IcebergCompatV3 tables")) } } /* ----------------- POSITIVE CASES ----------------- */ // TODO can we refactor other suites to run with both create + replace? Seq(Seq(), Seq(Map.empty[String, Literal] -> (dataBatches1))).foreach { replaceData => test(s"Basic case with no metadata changes, insertData=${replaceData.nonEmpty}") { withTempDirAndEngine { (tablePath, engine) => createInitialTable(engine, tablePath) checkReplaceTable(engine, tablePath, data = replaceData) } } test(s"Basic case with initial empty table, insertData=${replaceData.nonEmpty}") { withTempDirAndEngine { (tablePath, engine) => createInitialTable(engine, tablePath) checkReplaceTable(engine, tablePath, data = replaceData) } } } // Note, these tests cover things like transitioning between unpartitioned, partitioned, and // clustered tables. This means it includes removing existing clustering domains when the initial // table was clustered. validSchemaDefs.foreach { case (initialSchemaDef, initialData) => validSchemaDefs.foreach { case (replaceSchemaDef, replaceData) => Seq(true, false).foreach { initialTableEmpty => Seq(true, false).foreach { insertDataInReplace => test(s"Schema change from $initialSchemaDef to $replaceSchemaDef; " + s"initialTableEmpty=$initialTableEmpty, insertDataInReplace=$insertDataInReplace") { withTempDirAndEngine { (tablePath, engine) => createInitialTable( engine, tablePath, schema = initialSchemaDef.schema, partitionColumns = initialSchemaDef.partitionColumns, clusteringColumns = initialSchemaDef.clusteringColumns, includeData = !initialTableEmpty, data = initialData) checkReplaceTable( engine, tablePath, schema = replaceSchemaDef.schema, partitionColumns = replaceSchemaDef.partitionColumns, clusteringColumns = replaceSchemaDef.clusteringColumns, data = if (insertDataInReplace) replaceData else Seq.empty) } } } } test(s"Schema change from $initialSchemaDef to $replaceSchemaDef") { withTempDirAndEngine { (tablePath, engine) => createInitialTable( engine, tablePath, schema = initialSchemaDef.schema, partitionColumns = initialSchemaDef.partitionColumns, clusteringColumns = initialSchemaDef.clusteringColumns, includeData = false) checkReplaceTable( engine, tablePath, schema = replaceSchemaDef.schema, partitionColumns = replaceSchemaDef.partitionColumns, clusteringColumns = replaceSchemaDef.clusteringColumns) } } } } test("Case with DVs in the initial table") { withTempDirAndEngine { (tablePath, engine) => spark.sql( s""" |CREATE TABLE delta.`$tablePath` (id INT) USING DELTA |TBLPROPERTIES('delta.enableDeletionVectors' = 'true') |""".stripMargin) spark.sql( s""" |INSERT INTO delta.`$tablePath` VALUES (0), (1), (2), (3) |""".stripMargin) spark.sql( s""" |DELETE FROM delta.`$tablePath` WHERE id > 1 |""".stripMargin) checkTable(tablePath, Seq(TestRow(0), TestRow(1))) checkReplaceTable(engine, tablePath) // check it is empty after (also DVs no longer enabled) } } test("Existing table properties are removed") { withTempDirAndEngine { (tablePath, engine) => createInitialTable( engine, tablePath, tableProperties = Map( TableConfig.APPEND_ONLY_ENABLED.getKey -> "true", "user.facing.prop" -> "existing_prop")) checkReplaceTable(engine, tablePath) } } test("New table features are correctly enabled") { // This also validates that withDomainMetadataSupported works withTempDirAndEngine { (tablePath, engine) => createInitialTable(engine, tablePath) checkReplaceTable( engine, tablePath, tableProperties = Map( TableConfig.DELETION_VECTORS_CREATION_ENABLED.getKey -> "true"), domainsToAdd = Seq(("domain-name", "some-config")), expectedTableFeaturesSupported = Seq(TableFeatures.DELETION_VECTORS_RW_FEATURE, TableFeatures.DOMAIN_METADATA_W_FEATURE)) } } test("Domain metadata are reset (user-facing)") { // (1) checks that we correctly override an existing domain with the new config if set in the // replace txn // (2) checks we remove stale ones that are not set in the replace txn withTempDirAndEngine { (tablePath, engine) => // Create initial table with 2 domains val txn = getCreateTxn(engine, tablePath, testSchema, withDomainMetadataSupported = true) txn.addDomainMetadata("domainToOverride", "check1") txn.addDomainMetadata("domainToRemove", "check2") commitTransaction(txn, engine, emptyIterable()) // Validate the 2 domains are present val snapshot = Table.forPath(engine, tablePath).getLatestSnapshot(engine) assert(snapshot.getDomainMetadata("domainToOverride").toScala.contains("check1")) assert(snapshot.getDomainMetadata("domainToRemove").toScala.contains("check2")) // Replace table and override 1/2 of the domains checkReplaceTable( engine, tablePath, domainsToAdd = Seq(("domainToOverride", "overridden-config"))) } } test("Column mapping maxFieldId is preserved during REPLACE TABLE " + "- turning off column mapping mode") { // Note: DeltaReplaceTableColumnMappingSuite already tests that we preserve it correctly for the // column mapping case // TODO: once we support Id -> None mode during replace update this test // We should preserve maxFieldId regardless of column mapping mode (if a future replace // operation re-enables id mode we should not start our fieldIds from 0) withTempDirAndEngine { (tablePath, engine) => createInitialTable( engine, tablePath, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id"), includeData = false // To avoid writing data with correct CM schema ) intercept[UnsupportedOperationException] { checkReplaceTable( engine, tablePath, expectedTableProperties = Some(Map(TableConfig.COLUMN_MAPPING_MAX_COLUMN_ID.getKey -> "1"))) } } } test("icebergCompatV2 checks are executed and properties updated/auto-enabled") { withTempDirAndEngine { (tablePath, engine) => createInitialTable(engine, tablePath) // TODO once we support column mapping update this test intercept[UnsupportedOperationException] { checkReplaceTable( engine, tablePath, tableProperties = Map(TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true"), expectedTableProperties = Some(Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "name", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")), expectedTableFeaturesSupported = Seq( TableFeatures.ICEBERG_COMPAT_V2_W_FEATURE, TableFeatures.COLUMN_MAPPING_RW_FEATURE)) } } } // TODO - can we reuse the tests in IcebergWriterCompatV1Suite to run with both create table and // replace table? test("icebergWriterCompatV1 checks are executed and properties updated/auto-enabled") { // This also validates you can enable icebergWriterCompatV1 on an existing table during replace withTempDirAndEngine { (tablePath, engine) => createInitialTable(engine, tablePath) // TODO once we support column mapping update this test intercept[UnsupportedOperationException] { checkReplaceTable( engine, tablePath, tableProperties = Map(TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.getKey -> "true"), expectedTableProperties = Some(Map( TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.getKey -> "true", TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")), expectedTableFeaturesSupported = Seq( TableFeatures.ICEBERG_COMPAT_V2_W_FEATURE, TableFeatures.ICEBERG_WRITER_COMPAT_V1, TableFeatures.COLUMN_MAPPING_RW_FEATURE)) } } } test("When cmMode=None it is possible to have column with same name different type") { withTempDirAndEngine { (tablePath, engine) => createInitialTable( engine, tablePath, schema = new StructType() .add("col1", StringType.STRING), includeData = false) checkReplaceTable( engine, tablePath, schema = new StructType() .add("col1", IntegerType.INTEGER)) } } test("REPLACE TABLE preserves ICT enablement tracking properties") { withTempDirAndEngine { (tablePath, engine) => val snapshotV1 = createTableThenEnableIctAndVerify(engine, tablePath) val ictEnablementTimestamp = snapshotV1.getTimestamp(engine) checkReplaceTable( engine, tablePath, expectedTableProperties = Some(Map( IN_COMMIT_TIMESTAMPS_ENABLED.getKey -> "true", IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.getKey -> ictEnablementTimestamp.toString, IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.getKey -> "1"))) } } test("REPLACE TABLE removes ICT enablement tracking properties when explicitly disabling ICT") { withTempDirAndEngine { (tablePath, engine) => createTableThenEnableIctAndVerify(engine, tablePath) checkReplaceTable( engine, tablePath, tableProperties = Map(IN_COMMIT_TIMESTAMPS_ENABLED.getKey -> "false"), expectedTableProperties = Some(Map( IN_COMMIT_TIMESTAMPS_ENABLED.getKey -> "false"))) } } test("REPLACE TABLE can enable ICT") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable(engine, tablePath, testSchema) checkReplaceTable( engine, tablePath, tableProperties = Map(IN_COMMIT_TIMESTAMPS_ENABLED.getKey -> "true"), expectedTableProperties = Some(Map( IN_COMMIT_TIMESTAMPS_ENABLED.getKey -> "true", IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.getKey -> "__check_exists__", IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.getKey -> "1"))) } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaReplaceTableSuiteBase.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import scala.collection.JavaConverters._ import scala.collection.immutable.Seq import io.delta.kernel.TransactionCommitResult import io.delta.kernel.data.FilteredColumnarBatch import io.delta.kernel.defaults.utils.AbstractWriteUtils import io.delta.kernel.engine.Engine import io.delta.kernel.expressions.{Column, Literal} import io.delta.kernel.expressions.Literal.{ofInt, ofString} import io.delta.kernel.internal.SnapshotImpl import io.delta.kernel.internal.clustering.ClusteringMetadataDomain import io.delta.kernel.internal.tablefeatures.{TableFeature, TableFeatures} import io.delta.kernel.types.{IntegerType, StringType, StructType} import org.scalatest.funsuite.AnyFunSuite trait DeltaReplaceTableSuiteBase extends AnyFunSuite with AbstractWriteUtils { /* -------- Test values to use -------- */ case class SchemaDef( schema: StructType, partitionColumns: Seq[String] = null, clusteringColumns: Option[List[Column]] = None) { override def toString: String = { s"Schema=$schema, partCols=$partitionColumns, " + s"clusteringColumns=${clusteringColumns.map(_.toString).getOrElse(List.empty)}" } } val schemaA = new StructType() .add("col1", IntegerType.INTEGER) .add("col2", IntegerType.INTEGER) val schemaB = new StructType() .add("col4", StringType.STRING) .add("col5", StringType.STRING) val unpartitionedSchemaDefA = SchemaDef(schemaA) val unpartitionedSchemaDefB = SchemaDef(schemaB) val unpartitionedSchemaDefA_dataBatches = generateData(schemaA, Seq.empty, Map.empty, 200, 3) val unpartitionedSchemaDefB_dataBatches = generateData(schemaB, Seq.empty, Map.empty, 200, 3) val partitionedSchemaDefA_1 = SchemaDef(schemaA, partitionColumns = Seq("col1")) val partitionedSchemaDefA_2 = SchemaDef(schemaA, partitionColumns = Seq("col2")) val partitionedSchemaDefA_1_dataBatches = generateData( schemaA, partitionedSchemaDefA_1.partitionColumns, Map("col1" -> ofInt(1)), batchSize = 237, numBatches = 3) val partitionedSchemaDefA_2_dataBatches = generateData( schemaA, partitionedSchemaDefA_2.partitionColumns, Map("col2" -> ofInt(5)), batchSize = 400, numBatches = 1) val partitionedSchemaDefB = SchemaDef(schemaB, partitionColumns = Seq("col4")) val partitionedSchemaDefB_dataBatches = generateData( schemaB, partitionedSchemaDefB.partitionColumns, Map("col4" -> ofString("foo")), batchSize = 100, numBatches = 1) val clusteredSchemaDefA_1 = SchemaDef( schemaA, clusteringColumns = Some(List(new Column("col1")))) val clusteredSchemaDefA_2 = SchemaDef( schemaA, clusteringColumns = Some(List(new Column("col2")))) val clusteredSchemaDefA_1_dataBatches = generateData( schemaA, partitionCols = Seq.empty, partitionValues = Map.empty, batchSize = 237, numBatches = 3) val clusteredSchemaDefA_2_dataBatches = generateData( schemaA, partitionCols = Seq.empty, partitionValues = Map.empty, batchSize = 100, numBatches = 3) val clusteredSchemaDefB = SchemaDef( schemaB, clusteringColumns = Some(List(new Column("col4")))) val clusteredSchemaDefB_dataBatches = generateData( schemaB, partitionCols = Seq.empty, partitionValues = Map.empty, batchSize = 2, numBatches = 1) val validSchemaDefs = Map( unpartitionedSchemaDefA -> Seq(Map.empty[String, Literal] -> unpartitionedSchemaDefA_dataBatches), unpartitionedSchemaDefB -> Seq(Map.empty[String, Literal] -> unpartitionedSchemaDefB_dataBatches), partitionedSchemaDefA_1 -> Seq(Map("col1" -> ofInt(1)) -> partitionedSchemaDefA_1_dataBatches), partitionedSchemaDefA_2 -> Seq(Map("col2" -> ofInt(5)) -> partitionedSchemaDefA_2_dataBatches), partitionedSchemaDefB -> Seq(Map("col4" -> ofString("foo")) -> partitionedSchemaDefB_dataBatches), clusteredSchemaDefA_1 -> Seq(Map.empty[String, Literal] -> clusteredSchemaDefA_1_dataBatches), clusteredSchemaDefA_2 -> Seq(Map.empty[String, Literal] -> clusteredSchemaDefA_2_dataBatches), clusteredSchemaDefB -> Seq(Map.empty[String, Literal] -> clusteredSchemaDefB_dataBatches)) /* -------- Test methods -------- */ protected def createInitialTable( engine: Engine, tablePath: String, schema: StructType = testSchema, partitionColumns: Seq[String] = null, clusteringColumns: Option[List[Column]] = None, tableProperties: Map[String, String] = null, includeData: Boolean = true, data: Seq[(Map[String, Literal], Seq[FilteredColumnarBatch])] = Seq(Map.empty[String, Literal] -> (dataBatches1))): Unit = { val dataToWrite = if (includeData) { data } else { Seq.empty } appendData( engine, tablePath, isNewTable = true, schema, partCols = partitionColumns, clusteringColsOpt = clusteringColumns, tableProperties = tableProperties, data = dataToWrite) checkTable(tablePath, dataToWrite.flatMap(_._2).flatMap(_.toTestRows)) } protected def commitReplaceTable( engine: Engine, tablePath: String, schema: StructType = testSchema, partitionColumns: Seq[String] = null, clusteringColumns: Option[Seq[Column]] = None, tableProperties: Map[String, String] = Map.empty, domainsToAdd: Seq[(String, String)] = Seq.empty, data: Seq[(Map[String, Literal], Seq[FilteredColumnarBatch])] = Seq.empty) : TransactionCommitResult = { val txn = getReplaceTxn( engine, tablePath, schema, partitionColumns, clusteringColumns, tableProperties, domainsToAdd.nonEmpty) domainsToAdd.foreach { case (domainName, config) => txn.addDomainMetadata(domainName, config) } commitTransaction(txn, engine, getAppendActions(txn, data)) } // scalastyle:off argcount protected def checkReplaceTable( engine: Engine, tablePath: String, schema: StructType = testSchema, partitionColumns: Seq[String] = null, clusteringColumns: Option[Seq[Column]] = None, tableProperties: Map[String, String] = Map.empty, domainsToAdd: Seq[(String, String)] = Seq.empty, data: Seq[(Map[String, Literal], Seq[FilteredColumnarBatch])] = Seq.empty, expectedTableProperties: Option[Map[String, String]] = None, expectedTableFeaturesSupported: Seq[TableFeature] = Seq.empty): Unit = { // scalastyle:on argcount val oldProtocol = getProtocol(engine, tablePath) val wasClusteredTable = oldProtocol.supportsFeature(TableFeatures.CLUSTERING_W_FEATURE) val commitResult = commitReplaceTable( engine, tablePath, schema, partitionColumns, clusteringColumns, tableProperties, domainsToAdd, data) assertCommitResultHasClusteringCols(commitResult, clusteringColumns.getOrElse(Seq.empty)) verifyWrittenContent( tablePath, schema, data.flatMap(_._2).flatMap(_.toTestRows)) val snapshot = latestSnapshot(tablePath).asInstanceOf[SnapshotImpl] // Check partition columns val expectedPartitionColumns = if (partitionColumns == null) Seq() else partitionColumns assert(snapshot.getPartitionColumnNames.asScala == expectedPartitionColumns) // Check clustering columns clusteringColumns match { case Some(clusteringCols) => // Check clustering table feature is supported assertHasWriterFeature(snapshot, "clustering") assertHasWriterFeature(snapshot, "domainMetadata") // Validate clustering columns are correct // TODO when we support column mapping we will need to convert to physical-name here assert(snapshot.getPhysicalClusteringColumns.toScala .exists(_.asScala == clusteringCols)) case None => if (wasClusteredTable) { // If the table was previously clustered we expect the table feature to remain and for // there to be a clustering domain metadata with clusteringColumns=[] assertHasWriterFeature(snapshot, "clustering") assert(snapshot.getPhysicalClusteringColumns.toScala .exists(_.isEmpty)) } else { // Otherwise there should be no table feature and no clustering domain metadata assertHasNoWriterFeature(snapshot, "clustering") assert(!ClusteringMetadataDomain.fromSnapshot(snapshot).isPresent) } } // Check table properties val actualProperties = snapshot.getMetadata.getConfiguration.asScala val expectedProperties = expectedTableProperties.getOrElse(tableProperties) expectedProperties.foreach { case (key, expectedValue) => if (expectedValue == "__check_exists__") { assert(actualProperties.contains(key), s"Expected property $key to exist") } else { assert( actualProperties.get(key).contains(expectedValue), s"Property $key: expected $expectedValue, got ${actualProperties.get(key)}") } } // Check other domain metadata val nonClusteringActiveDomains = snapshot.getActiveDomainMetadataMap.asScala .filter { case (domainName, _) => domainName != ClusteringMetadataDomain.DOMAIN_NAME }.map { case (domainName, domainMetadata) => (domainName, domainMetadata.getConfiguration) } assert(nonClusteringActiveDomains.toSet == domainsToAdd.toSet) // Check protocol. In particular, check that we never downgrade the protocol val newProtocol = getProtocol(engine, tablePath) assert(oldProtocol.canUpgradeTo(newProtocol)) assert(expectedTableFeaturesSupported.forall(newProtocol.supportsFeature)) // Check CommitInfo.operation val row = spark.sql(s"DESCRIBE HISTORY delta.`$tablePath`") .filter(s"version = ${snapshot.getVersion}") .select("operation") .collect().last assert(row.getAs[String]("operation") == "REPLACE TABLE") } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableClusteringSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import scala.collection.JavaConverters._ import scala.collection.immutable.Seq import io.delta.kernel.{Table, Transaction, TransactionCommitResult} import io.delta.kernel.Operation.{CREATE_TABLE, WRITE} import io.delta.kernel.data.Row import io.delta.kernel.defaults.utils.{AbstractWriteUtils, WriteUtils, WriteUtilsWithV2Builders} import io.delta.kernel.engine.Engine import io.delta.kernel.exceptions.{KernelException, TableAlreadyExistsException} import io.delta.kernel.expressions.{Column, Literal} import io.delta.kernel.expressions.Literal.ofInt import io.delta.kernel.internal.SnapshotImpl import io.delta.kernel.internal.actions.DomainMetadata import io.delta.kernel.internal.clustering.ClusteringMetadataDomain import io.delta.kernel.internal.util.ColumnMapping import io.delta.kernel.internal.util.ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY import io.delta.kernel.types.{MapType, StructType} import io.delta.kernel.types.IntegerType.INTEGER import io.delta.kernel.utils.CloseableIterable import io.delta.kernel.utils.CloseableIterable.emptyIterable import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.clustering.{ClusteringMetadataDomain => SparkClusteringMetadataDomain} import org.apache.hadoop.fs.Path import org.scalatest.funsuite.AnyFunSuite class DeltaTableClusteringTransactionBuilderV1Suite extends DeltaTableClusteringSuiteBase with WriteUtils { // It is not possible on an API level to set both clustering and partition columns in V2 builders test("build table txn: " + "clustering column and partition column cannot be set at same time") { withTempDirAndEngine { (tablePath, engine) => val ex = intercept[IllegalArgumentException] { getCreateTxn( engine, tablePath, testPartitionSchema, partCols = Seq("part1"), clusteringColsOpt = Some(List(new Column("PART1"), new Column("part2")))) } assert( ex.getMessage .contains("Partition Columns and Clustering Columns cannot be set at the same time")) } } } class DeltaTableClusteringTransactionBuilderV2Suite extends DeltaTableClusteringSuiteBase with WriteUtilsWithV2Builders {} trait DeltaTableClusteringSuiteBase extends AnyFunSuite with AbstractWriteUtils { private val testingDomainMetadata = new DomainMetadata( "delta.clustering", """{"clusteringColumns":[["part1"],["part2"]]}""", false) override def commitTransaction( txn: Transaction, engine: Engine, dataActions: CloseableIterable[Row]): TransactionCommitResult = { executeCrcSimple(txn.commit(engine, dataActions), engine) } private def verifyClusteringDMAndCRC( snapshot: SnapshotImpl, expectedDomainMetadata: DomainMetadata): Unit = { verifyClusteringDomainMetadata(snapshot, expectedDomainMetadata) // verifyChecksum will check the domain metadata in CRC against the latest snapshot. verifyChecksum(snapshot.getDataPath.toString) } test("build table txn: clustering column should be part of the schema") { withTempDirAndEngine { (tablePath, engine) => val ex = intercept[KernelException] { getCreateTxn( engine, tablePath, testPartitionSchema, clusteringColsOpt = Some(List(new Column("PART1"), new Column("part3")))) } assert(ex.getMessage.contains("Column 'column(`part3`)' was not found in the table schema")) } } test("build table txn: clustering column should be data skipping supported data type") { withTempDirAndEngine { (tablePath, engine) => val testPartitionSchema = new StructType() .add("id", INTEGER) .add("part1", INTEGER) // partition column .add("mapType", new MapType(INTEGER, INTEGER, false)); val ex = intercept[KernelException] { getCreateTxn( engine, tablePath, testPartitionSchema, clusteringColsOpt = Some(List(new Column("mapType")))) } assert(ex.getMessage.contains("Clustering is not supported because the following column(s)")) } } test("create a clustered table should succeed") { withTempDirAndEngine { (tablePath, engine) => val commitResult = createEmptyTable( engine, tablePath, testPartitionSchema, clusteringColsOpt = Some(testClusteringColumns)) assertCommitResultHasClusteringCols( commitResult, expectedClusteringCols = testClusteringColumns) val table = Table.forPath(engine, tablePath) // Verify the clustering feature is included in the protocol val snapshot = table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl] assertHasWriterFeature(snapshot, "clustering") // Verify the clustering domain metadata is written verifyClusteringDMAndCRC(snapshot, testingDomainMetadata) // Use Spark to read the table's clustering metadata domain and verify the result val deltaLog = DeltaLog.forTable(spark, new Path(tablePath)) val clusteringMetadataDomainRead = SparkClusteringMetadataDomain.fromSnapshot(deltaLog.snapshot) assert(clusteringMetadataDomainRead.exists(_.clusteringColumns === Seq( Seq("part1"), Seq("part2")))) } } test("clustering column should store as physical name with column mapping") { withTempDirAndEngine { (tablePath, engine) => val commitResult = createEmptyTable( engine, tablePath, testPartitionSchema, clusteringColsOpt = Some(testClusteringColumns), tableProperties = Map(ColumnMapping.COLUMN_MAPPING_MODE_KEY -> "id")) val table = Table.forPath(engine, tablePath) // Verify the clustering feature is included in the protocol val snapshot = table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl] assertHasWriterFeature(snapshot, "clustering") // Verify the clustering domain metadata is written val schema = table.getLatestSnapshot(engine).getSchema val col1 = schema.get("part1").getMetadata.getString(COLUMN_MAPPING_PHYSICAL_NAME_KEY) val col2 = schema.get("part2").getMetadata.getString(COLUMN_MAPPING_PHYSICAL_NAME_KEY) val expectedDomainMetadata = new DomainMetadata( "delta.clustering", s"""{"clusteringColumns":[["$col1"],["$col2"]]}""", false) verifyClusteringDMAndCRC(snapshot, expectedDomainMetadata) assertCommitResultHasClusteringCols( commitResult, expectedClusteringCols = Seq(new Column(col1), new Column(col2))) } } test("create a clustered table should succeed with column case matches schema") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, testPartitionSchema, clusteringColsOpt = Some(List(new Column("pArT1"), new Column("PaRt2")))) val table = Table.forPath(engine, tablePath) // Verify the clustering feature is included in the protocol val snapshot = table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl] assertHasWriterFeature(snapshot, "clustering") // Verify the clustering domain metadata is written verifyClusteringDMAndCRC(snapshot, testingDomainMetadata) } } test("update a non-clustered table with clustering columns should succeed") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable(engine, tablePath, testPartitionSchema) val table = Table.forPath(engine, tablePath) updateTableMetadata(engine, tablePath, clusteringColsOpt = Some(testClusteringColumns)) val snapshot = table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl] assertHasWriterFeature(snapshot, "clustering") verifyClusteringDMAndCRC(snapshot, testingDomainMetadata) } } test("update a clustered table with subset of previous clustering columns should succeed") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, testPartitionSchema, clusteringColsOpt = Some(testClusteringColumns)) val table = Table.forPath(engine, tablePath) updateTableMetadata(engine, tablePath, clusteringColsOpt = Some(List(new Column("part1")))) val snapshot = table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl] assertHasWriterFeature(snapshot, "clustering") val expectedDomainMetadata = new DomainMetadata( "delta.clustering", """{"clusteringColumns":[["part1"]]}""", false) verifyClusteringDMAndCRC(snapshot, expectedDomainMetadata) } } test("update a clustered table with a overlap clustering columns should succeed") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, testPartitionSchema, clusteringColsOpt = Some(testClusteringColumns) ) // Seq("part1", "part2") val table = Table.forPath(engine, tablePath) updateTableMetadata( engine, tablePath, clusteringColsOpt = Some(List(new Column("part2"), new Column("id")))) val snapshot = table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl] assertHasWriterFeature(snapshot, "clustering") val expectedDomainMetadata = new DomainMetadata( "delta.clustering", """{"clusteringColumns":[["part2"],["id"]]}""", false) verifyClusteringDMAndCRC(snapshot, expectedDomainMetadata) } } test("update a clustered table with a non-overlap clustering columns should succeed") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, testPartitionSchema, clusteringColsOpt = Some(List(new Column("part1")))) val table = Table.forPath(engine, tablePath) updateTableMetadata(engine, tablePath, clusteringColsOpt = Some(List(new Column("part2")))) val snapshot = table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl] val expectedDomainMetadata = new DomainMetadata( "delta.clustering", """{"clusteringColumns":[["part2"]]}""", false) assertHasWriterFeature(snapshot, "clustering") verifyClusteringDMAndCRC(snapshot, expectedDomainMetadata) } } test("update a clustered table with empty clustering columns should succeed") { withTempDirAndEngine { (tablePath, engine) => val commitResult0 = createEmptyTable( engine, tablePath, testPartitionSchema, clusteringColsOpt = Some(testClusteringColumns)) assertCommitResultHasClusteringCols( commitResult0, expectedClusteringCols = testClusteringColumns) val table = Table.forPath(engine, tablePath) val commitResult1 = updateTableMetadata(engine, tablePath, clusteringColsOpt = Some(List())) assertCommitResultHasClusteringCols(commitResult1, expectedClusteringCols = Seq.empty) val snapshot = table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl] val expectedDomainMetadata = new DomainMetadata( "delta.clustering", """{"clusteringColumns":[]}""", false) assertHasWriterFeature(snapshot, "clustering") verifyClusteringDMAndCRC(snapshot, expectedDomainMetadata) } } test("update a table with clustering columns doesn't exist in the table should fail") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable(engine, tablePath, testPartitionSchema) val ex = intercept[KernelException] { updateTableMetadata( engine, tablePath, clusteringColsOpt = Some(List(new Column("non-exist")))) } assert( ex.getMessage.contains("Column 'column(`non-exist`)' was not found in the table schema")) } } test("update a partitioned table with clustering columns should fail") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable(engine, tablePath, testPartitionSchema, partCols = testPartitionColumns) // test case 1: update with non-empty clustering columns val ex1 = intercept[KernelException] { updateTableMetadata( engine, tablePath, clusteringColsOpt = Some(List(new Column("non-exist")))) } assert( ex1.getMessage.contains("Cannot enable clustering on a partitioned table")) // test case 2: update with empty clustering columns, // this would still be regarded as enabling clustering val ex2 = intercept[KernelException] { updateTableMetadata( engine, tablePath, clusteringColsOpt = Some(List())) } assert( ex2.getMessage.contains("Cannot enable clustering on a partitioned table")) } } test("insert into clustered table - table create from scratch") { withTempDirAndEngine { (tablePath, engine) => val testData = Seq(Map.empty[String, Literal] -> dataClusteringBatches1) val commitResult = appendData( engine, tablePath, isNewTable = true, testPartitionSchema, clusteringColsOpt = Some(testClusteringColumns), data = testData) verifyCommitResult(commitResult, expVersion = 0, expIsReadyForCheckpoint = false) verifyCommitInfo(tablePath, version = 0) verifyWrittenContent( tablePath, testPartitionSchema, dataClusteringBatches1.flatMap(_.toTestRows)) val table = Table.forPath(engine, tablePath) verifyClusteringDMAndCRC( table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl], testingDomainMetadata) } } test("insert into clustered table - already existing table") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) { val commitResult0 = appendData( engine, tablePath, isNewTable = true, testPartitionSchema, clusteringColsOpt = Some(testClusteringColumns), data = Seq(Map.empty[String, Literal] -> dataClusteringBatches1)) assertCommitResultHasClusteringCols( commitResult0, expectedClusteringCols = testClusteringColumns) val expData = dataClusteringBatches1.flatMap(_.toTestRows) verifyCommitResult(commitResult0, expVersion = 0, expIsReadyForCheckpoint = false) verifyCommitInfo(tablePath, version = 0) verifyWrittenContent(tablePath, testPartitionSchema, expData) verifyClusteringDMAndCRC( table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl], testingDomainMetadata) } { val commitResult1 = appendData( engine, tablePath, data = Seq(Map.empty[String, Literal] -> dataClusteringBatches2)) assertCommitResultHasClusteringCols( commitResult1, expectedClusteringCols = testClusteringColumns) val expData = dataClusteringBatches1.flatMap(_.toTestRows) ++ dataClusteringBatches2.flatMap(_.toTestRows) verifyCommitResult(commitResult1, expVersion = 1, expIsReadyForCheckpoint = false) verifyCommitInfo(tablePath, version = 1, partitionCols = null) verifyWrittenContent(tablePath, testPartitionSchema, expData) verifyClusteringDMAndCRC( table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl], testingDomainMetadata) } } } test("insert into clustered table after update clusteringColumns should still work") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val expectedDomainMetadataAfterUpdate = new DomainMetadata( "delta.clustering", """{"clusteringColumns":[["id"],["part1"]]}""", false) val newClusteringCols = List(new Column("id"), new Column("part1")) // will be updated in v1 { val commitResult0 = appendData( engine, tablePath, isNewTable = true, testPartitionSchema, clusteringColsOpt = Some(testClusteringColumns), data = Seq(Map.empty[String, Literal] -> dataClusteringBatches1)) assertCommitResultHasClusteringCols( commitResult0, expectedClusteringCols = testClusteringColumns) val expData = dataClusteringBatches1.flatMap(_.toTestRows) verifyCommitResult(commitResult0, expVersion = 0, expIsReadyForCheckpoint = false) verifyCommitInfo(tablePath, version = 0) verifyWrittenContent(tablePath, testPartitionSchema, expData) verifyClusteringDMAndCRC( table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl], testingDomainMetadata) } { val commitResult1 = updateTableMetadata( engine, tablePath, clusteringColsOpt = Some(newClusteringCols)) assertCommitResultHasClusteringCols( commitResult1, expectedClusteringCols = newClusteringCols) verifyCommitResult(commitResult1, expVersion = 1, expIsReadyForCheckpoint = false) verifyClusteringDMAndCRC( table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl], expectedDomainMetadataAfterUpdate) } { val commitResult2 = appendData( engine, tablePath, data = Seq(Map.empty[String, Literal] -> dataClusteringBatches2)) assertCommitResultHasClusteringCols( commitResult2, expectedClusteringCols = newClusteringCols) val expData = dataClusteringBatches1.flatMap(_.toTestRows) ++ dataClusteringBatches2.flatMap(_.toTestRows) verifyCommitResult(commitResult2, expVersion = 2, expIsReadyForCheckpoint = false) verifyCommitInfo(tablePath, version = 2, partitionCols = null) verifyWrittenContent(tablePath, testPartitionSchema, expData) verifyClusteringDMAndCRC( table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl], expectedDomainMetadataAfterUpdate) } } } test("can convert physical clustering columns to logical on column-mapping-enabled table") { withTempDirAndEngine { (tablePath, engine) => // ===== GIVEN ===== val tableProperties = Map(ColumnMapping.COLUMN_MAPPING_MODE_KEY -> "id") val clusteringColumns = List(new Column("part1"), new Column("part2")) createEmptyTable( engine, tablePath, testPartitionSchema, tableProperties = tableProperties, clusteringColsOpt = Some(clusteringColumns)) // ===== WHEN ===== val snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath) val physicalClusteringColumns = snapshot.getPhysicalClusteringColumns.get().asScala // ===== THEN ===== assert(physicalClusteringColumns.size == 2) physicalClusteringColumns.foreach { c => assert(c.getNames()(0).startsWith("col-")) } val schema = snapshot.getSchema physicalClusteringColumns.zipWithIndex.foreach { case (physicalColumn, idx) => val logicalColumn = ColumnMapping.getLogicalColumnNameAndDataType(schema, physicalColumn)._1 val expectedLogicalName = if (idx == 0) "part1" else "part2" assert(logicalColumn.getNames.length == 1) assert(logicalColumn.getNames()(0) == expectedLogicalName) } } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableFeaturesSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.nio.file.{Files, Paths} import java.util.Collections import scala.collection.immutable.Seq import scala.jdk.CollectionConverters._ import io.delta.kernel.{Operation, Table} import io.delta.kernel.Operation.CREATE_TABLE import io.delta.kernel.defaults.utils.{AbstractWriteUtils, WriteUtils, WriteUtilsWithV2Builders} import io.delta.kernel.engine.Engine import io.delta.kernel.exceptions.{InvalidConfigurationValueException, KernelException} import io.delta.kernel.expressions.Literal import io.delta.kernel.internal.{SnapshotImpl, TableConfig} import io.delta.kernel.internal.TableConfig.UniversalFormats import io.delta.kernel.internal.actions.{Protocol => KernelProtocol} import io.delta.kernel.internal.tablefeatures.TableFeatures import io.delta.kernel.types.{StructType, TimestampNTZType} import io.delta.kernel.types.IntegerType.INTEGER import io.delta.kernel.utils.CloseableIterable.emptyIterable import org.apache.spark.sql.delta.{DeltaLog, DeltaTableFeatureException} import org.apache.spark.sql.delta.actions.Protocol import org.apache.hadoop.fs.Path import org.apache.parquet.hadoop.ParquetFileReader import org.scalatest.funsuite.AnyFunSuite class DeltaTableFeaturesTransactionBuilderV1Suite extends DeltaTableFeaturesSuiteBase with WriteUtils {} class DeltaTableFeaturesTransactionBuilderV2Suite extends DeltaTableFeaturesSuiteBase with WriteUtilsWithV2Builders {} /** * Integration test suite for Delta table features. */ trait DeltaTableFeaturesSuiteBase extends AnyFunSuite with AbstractWriteUtils { /////////////////////////////////////////////////////////////////////////// // Tests for deletionVector, v2Checkpoint table features /////////////////////////////////////////////////////////////////////////// Seq( // Test format: feature (readerWriter type), table property to enable the feature // For each feature, we test the following scenarios: // 1. able to write to an existing Delta table with the feature supported // 2. create a table with the feature supported and append data // 3. update an existing table with the feature supported ("deletionVectors", "delta.enableDeletionVectors", "true"), ("v2Checkpoint", "delta.checkpointPolicy", "v2")).foreach { case (feature, tblProp, propValue) => test(s"able to write to an existing Delta table with $feature supported") { withTempDirAndEngine { (tablePath, engine) => // Create a table with the feature supported spark.sql(s"CREATE TABLE delta.`$tablePath` (id INTEGER) USING delta " + s"TBLPROPERTIES ('$tblProp' = '$propValue')") checkReaderWriterFeaturesSupported(tablePath, feature) // Write data to the table using Kernel val testData = Seq(Map.empty[String, Literal] -> dataBatches1) appendData( engine, tablePath, data = testData) // Check the data using Kernel and Delta-Spark readers verifyWrittenContent(tablePath, testSchema, dataBatches1.flatMap(_.toTestRows)) } } test(s"create a table with $feature supported") { withTempDirAndEngine { (tablePath, engine) => val testData = Seq(Map.empty[String, Literal] -> dataBatches1) // create a table with the feature supported and append testData appendData( engine, tablePath, isNewTable = true, testSchema, data = testData, tableProperties = Map(tblProp -> propValue)) checkReaderWriterFeaturesSupported(tablePath, feature) // insert more data appendData( engine, tablePath, data = testData) // Check the data using Kernel and Delta-Spark readers verifyWrittenContent( tablePath, testSchema, dataBatches1.flatMap(_.toTestRows) ++ dataBatches1.flatMap(_.toTestRows)) } } test(s"update an existing table with $feature support") { withTempDirAndEngine { (tablePath, engine) => val testData = Seq(Map.empty[String, Literal] -> dataBatches1) // create a table without the table feature supported appendData( engine, tablePath, isNewTable = true, testSchema, data = testData) checkNoReaderWriterFeaturesSupported(tablePath, feature) // insert more data and enable the feature appendData( engine, tablePath, data = testData, tableProperties = Map(tblProp -> propValue)) checkReaderWriterFeaturesSupported(tablePath, feature) // Check the data using Kernel and Delta-Spark readers verifyWrittenContent( tablePath, testSchema, dataBatches1.flatMap(_.toTestRows) ++ dataBatches1.flatMap(_.toTestRows)) } } } // Test format: isTimestampNtzEnabled, expected protocol. Seq( (true, new KernelProtocol(3, 7, Set("timestampNtz").asJava, Set("timestampNtz").asJava)), (false, new KernelProtocol(1, 2, Collections.emptySet(), Collections.emptySet()))) .foreach({ case (isTimestampNtzEnabled, expectedProtocol) => test(s"Create table with timestampNtz enabled: $isTimestampNtzEnabled") { withTempDirAndEngine { (tablePath, engine) => val schema = if (isTimestampNtzEnabled) { new StructType().add("tz", TimestampNTZType.TIMESTAMP_NTZ) } else { new StructType().add("id", INTEGER) } val txn = getCreateTxn(engine, tablePath, schema) assert(txn.getSchema(engine) === schema) assert(txn.getPartitionColumns(engine).isEmpty) val txnResult = commitTransaction(txn, engine, emptyIterable()) assert(txnResult.getVersion === 0) val protocolRow = getProtocolActionFromCommit(engine, tablePath, 0) assert(protocolRow.isDefined) val protocol = KernelProtocol.fromRow(protocolRow.get) assert(protocol.getMinReaderVersion === expectedProtocol.getMinReaderVersion) assert(protocol.getMinWriterVersion === expectedProtocol.getMinWriterVersion) assert(protocol.getReaderFeatures.containsAll(expectedProtocol.getReaderFeatures)) assert(protocol.getWriterFeatures.containsAll(expectedProtocol.getWriterFeatures)) } } }) test("schema evolution from Spark to add TIMESTAMP_NTZ type on a table created with kernel") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val txn = getCreateTxn(engine, tablePath, testSchema) val txnResult = commitTransaction(txn, engine, emptyIterable()) assert(txnResult.getVersion === 0) assertThrows[DeltaTableFeatureException] { spark.sql("ALTER TABLE delta.`" + tablePath + "` ADD COLUMN newCol TIMESTAMP_NTZ") } spark.sql("ALTER TABLE delta.`" + tablePath + "` SET TBLPROPERTIES ('delta.feature.timestampNtz' = 'supported')") spark.sql("ALTER TABLE delta.`" + tablePath + "` ADD COLUMN newCol TIMESTAMP_NTZ") } } test("feature can be enabled via delta.feature prefix") { withTempDirAndEngine { (tablePath, engine) => val domainMetadataKey = ( TableFeatures.SET_TABLE_FEATURE_SUPPORTED_PREFIX + TableFeatures.DOMAIN_METADATA_W_FEATURE.featureName) val properties = Map( "delta.feature.vacuumProtocolCheck" -> "supported", domainMetadataKey -> "supported") createEmptyTable(engine, tablePath, testSchema, tableProperties = properties) val table = Table.forPath(engine, tablePath) val writtenSnapshot = latestSnapshot(table, engine) assert(writtenSnapshot.getMetadata.getConfiguration.isEmpty) assert(writtenSnapshot.getProtocol.getExplicitlySupportedFeatures.containsAll(Set( TableFeatures.VACUUM_PROTOCOL_CHECK_RW_FEATURE, TableFeatures.DOMAIN_METADATA_W_FEATURE).asJava)) } } test("withDomainMetadata adds corresponding feature option") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val txn = getCreateTxn(engine, tablePath, testSchema, withDomainMetadataSupported = true) commitTransaction(txn, engine, emptyIterable()) assert(latestSnapshot(table, engine).getProtocol.getExplicitlySupportedFeatures.contains( TableFeatures.DOMAIN_METADATA_W_FEATURE)) } } test("delta.feature prefixed keys are removed even if property is already present on protocol") { withTempDirAndEngine { (tablePath, engine) => val properties = Map("delta.feature.vacuumProtocolCheck" -> "supported") createEmptyTable(engine, tablePath, testSchema, tableProperties = properties) val table = Table.forPath(engine, tablePath) assert(latestSnapshot(table, engine).getMetadata.getConfiguration.isEmpty) // Update table with the same feature override set. val updateTxn = getUpdateTxn(engine, tablePath, tableProperties = properties) commitTransaction(updateTxn, engine, emptyIterable()) assert(latestSnapshot(table, engine).getMetadata.getConfiguration.isEmpty) } } test("delta.feature override populate dependent features") { withTempDirAndEngine { (tablePath, engine) => val properties = Map("delta.feature.clustering" -> "supported") createEmptyTable(engine, tablePath, testSchema, tableProperties = properties) val table = Table.forPath(engine, tablePath) val writtenSnapshot = latestSnapshot(table, engine) assert( writtenSnapshot.getProtocol.getExplicitlySupportedFeatures.containsAll(Set( TableFeatures.CLUSTERING_W_FEATURE, TableFeatures.DOMAIN_METADATA_W_FEATURE).asJava), s"${writtenSnapshot.getProtocol.getExplicitlySupportedFeatures}") } } test("delta.feature override and TableConfig populate necessary features") { withTempDirAndEngine { (tablePath, engine) => val properties = Map("delta.feature.clustering" -> "supported", "delta.enableDeletionVectors" -> "true") createEmptyTable(engine, tablePath, testSchema, tableProperties = properties) val table = Table.forPath(engine, tablePath) val writtenSnapshot = latestSnapshot(table, engine) assert( writtenSnapshot.getProtocol.getExplicitlySupportedFeatures.containsAll(Set( TableFeatures.CLUSTERING_W_FEATURE, TableFeatures.DOMAIN_METADATA_W_FEATURE, TableFeatures.DELETION_VECTORS_RW_FEATURE).asJava), s"${writtenSnapshot.getProtocol.getExplicitlySupportedFeatures}") assert(writtenSnapshot.getMetadata.getConfiguration == Map( "delta.enableDeletionVectors" -> "true").asJava) } } test("UNIVERSAL_FORMAT feature can be populated") { withTempDirAndEngine { (tablePath, engine) => val properties = Map( TableConfig.UNIVERSAL_FORMAT_ENABLED_FORMATS.getKey -> "iceberg", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true") createEmptyTable(engine, tablePath, testSchema, tableProperties = properties) val table = Table.forPath(engine, tablePath) val writtenSnapshot = latestSnapshot(table, engine) assert(TableConfig.UNIVERSAL_FORMAT_ENABLED_FORMATS.fromMetadata( writtenSnapshot.getMetadata).contains(UniversalFormats.FORMAT_ICEBERG)) } } test("UNIVERSAL_FORMAT feature will throw if icebergCompatV2 was not enabled") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable(engine, tablePath, testSchema) intercept[InvalidConfigurationValueException] { getUpdateTxn( engine, tablePath, tableProperties = Map(TableConfig.UNIVERSAL_FORMAT_ENABLED_FORMATS.getKey -> "iceberg")) } } } test("read throws if the table contains unsupported table feature") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable(engine, tablePath, testSchema) appendData( engine, tablePath, isNewTable = false, data = Seq(Map.empty[String, Literal] -> dataBatches1)) checkTable(tablePath, expectedAnswer = dataBatches1.flatMap(_.toTestRows)) // If test is running in intelliJ, set DELTA_TESTING=1 in env variables. // This will enable the testReaderWriter feature in delta-spark. In CI jobs, // build.sbt already has set and effective. spark.sql("ALTER TABLE delta.`" + tablePath + "` SET TBLPROPERTIES ('delta.feature.testReaderWriter' = 'supported')") // try to read the table val ex = intercept[KernelException] { checkTable( tablePath, expectedAnswer = Seq.empty /* it doesn't matter as expect failure in reading */ ) } assert(ex.getMessage.contains( "feature \"testReaderWriter\" which is unsupported by this version of Delta Kernel")) } } test("read succeeds with unrecognized writer-only feature") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable(engine, tablePath, testSchema) // Add an unknown writer feature to the protocol // When DELTA_TESTING=1 (set in build.sbt) this test writer feature is allowed spark.sql( "ALTER TABLE delta.`" + tablePath + "` SET TBLPROPERTIES ('delta.feature.testWriter' = 'supported')") // Read should succeed - writer-only features don't affect readers getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath) assert(getProtocol(engine, tablePath).getWriterFeatures().contains("testWriter")) } } /* ---- Start: type widening tests ---- */ test("only typeWidening feature is enabled when metadata supports it: new table") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath = tablePath, schema = testSchema, tableProperties = Map("delta.enableTypeWidening" -> "true")) val protocolV0 = getProtocol(engine, tablePath) assert(!protocolV0.supportsFeature(TableFeatures.TYPE_WIDENING_RW_PREVIEW_FEATURE)) assert(protocolV0.supportsFeature(TableFeatures.TYPE_WIDENING_RW_FEATURE)) // try enabling type widening again and expect no change in protocol updateTableMetadata( engine = engine, tablePath = tablePath, tableProperties = Map("delta.enableTypeWidening" -> "true")) val protocolV1 = getProtocol(engine, tablePath) assert(protocolV1 === protocolV0) } } test("only typeWidening feature is enabled when new metadata supports it: existing table") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable(engine, tablePath = tablePath, schema = testSchema) val protocolV0 = getProtocol(engine, tablePath) assert(!protocolV0.supportsFeature(TableFeatures.TYPE_WIDENING_RW_PREVIEW_FEATURE)) assert(!protocolV0.supportsFeature(TableFeatures.TYPE_WIDENING_RW_FEATURE)) // try enabling type widening and expect change in protocol updateTableMetadata( engine = engine, tablePath = tablePath, tableProperties = Map("delta.enableTypeWidening" -> "true")) val protocolV1 = getProtocol(engine, tablePath) assert(!protocolV1.supportsFeature(TableFeatures.TYPE_WIDENING_RW_PREVIEW_FEATURE)) assert(protocolV1.supportsFeature(TableFeatures.TYPE_WIDENING_RW_FEATURE)) } } test("typeWidening-preview in existing table is respected") { withTempDirAndEngine { (tablePath, engine) => spark.sql(s"CREATE TABLE delta.`$tablePath`(id INT) USING delta " + s"TBLPROPERTIES ('delta.feature.typeWidening-preview' = 'supported')") val protocolV0 = getProtocol(engine, tablePath) require(protocolV0.supportsFeature(TableFeatures.TYPE_WIDENING_RW_PREVIEW_FEATURE)) require(!protocolV0.supportsFeature(TableFeatures.TYPE_WIDENING_RW_FEATURE)) // now through Kernel type enabling the type widening through table property updateTableMetadata( engine = engine, tablePath = tablePath, tableProperties = Map("delta.enableTypeWidening" -> "true")) val protocolV1 = getProtocol(engine, tablePath) assert(protocolV1.supportsFeature(TableFeatures.TYPE_WIDENING_RW_PREVIEW_FEATURE)) assert(!protocolV1.supportsFeature(TableFeatures.TYPE_WIDENING_RW_FEATURE)) } } /* ---- End: type widening tests ---- */ /* ---- Start: variant shredding tests ---- */ test("only variantShredding feature is enabled when metadata supports it: new table") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath = tablePath, schema = testSchema, tableProperties = Map("delta.enableVariantShredding" -> "true")) val protocolV0 = getProtocol(engine, tablePath) assert(!protocolV0.supportsFeature(TableFeatures.VARIANT_SHREDDING_PREVIEW_RW_FEATURE)) assert(protocolV0.supportsFeature(TableFeatures.VARIANT_SHREDDING_RW_FEATURE)) updateTableMetadata( engine = engine, tablePath = tablePath, tableProperties = Map("delta.enableVariantShredding" -> "true")) val protocolV1 = getProtocol(engine, tablePath) assert(protocolV1 === protocolV0) } } test("only variantShredding feature is enabled when new metadata supports it: existing table") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable(engine, tablePath = tablePath, schema = testSchema) val protocolV0 = getProtocol(engine, tablePath) assert(!protocolV0.supportsFeature(TableFeatures.VARIANT_SHREDDING_PREVIEW_RW_FEATURE)) assert(!protocolV0.supportsFeature(TableFeatures.VARIANT_SHREDDING_RW_FEATURE)) updateTableMetadata( engine = engine, tablePath = tablePath, tableProperties = Map("delta.enableVariantShredding" -> "true")) val protocolV1 = getProtocol(engine, tablePath) assert(!protocolV1.supportsFeature(TableFeatures.VARIANT_SHREDDING_PREVIEW_RW_FEATURE)) assert(protocolV1.supportsFeature(TableFeatures.VARIANT_SHREDDING_RW_FEATURE)) } } test("variantShredding-preview in existing table is respected") { withTempDirAndEngine { (tablePath, engine) => spark.sql(s"CREATE TABLE delta.`$tablePath`(id INT) USING delta " + s"TBLPROPERTIES ('delta.feature.variantShredding-preview' = 'supported')") val protocolV0 = getProtocol(engine, tablePath) require(protocolV0.supportsFeature(TableFeatures.VARIANT_SHREDDING_PREVIEW_RW_FEATURE)) require(!protocolV0.supportsFeature(TableFeatures.VARIANT_SHREDDING_RW_FEATURE)) updateTableMetadata( engine = engine, tablePath = tablePath, tableProperties = Map("delta.enableVariantShredding" -> "true")) val protocolV1 = getProtocol(engine, tablePath) assert(protocolV1.supportsFeature(TableFeatures.VARIANT_SHREDDING_PREVIEW_RW_FEATURE)) assert(!protocolV1.supportsFeature(TableFeatures.VARIANT_SHREDDING_RW_FEATURE)) } } test("both variantShredding and variantShredding-preview can coexist") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath = tablePath, schema = testSchema, tableProperties = Map( "delta.enableVariantShredding" -> "true", "delta.feature.variantShredding-preview" -> "supported")) val protocol = getProtocol(engine, tablePath) assert(protocol.supportsFeature(TableFeatures.VARIANT_SHREDDING_RW_FEATURE)) assert(protocol.supportsFeature(TableFeatures.VARIANT_SHREDDING_PREVIEW_RW_FEATURE)) } } /* ---- End: variant shredding tests ---- */ /* ---- Start: materialize partition columns tests ---- */ test("materialize partition columns in parquet schema when feature is enabled") { withTempDirAndEngine { (tablePath, engine) => // Create a partitioned table with materialize partition columns feature enabled appendData( engine, tablePath, isNewTable = true, testPartitionSchema, partCols = testPartitionColumns, data = Seq(Map("part1" -> Literal.ofInt(1), "part2" -> Literal.ofInt(2)) -> dataPartitionBatches1), tableProperties = Map("delta.feature.materializePartitionColumns" -> "supported")) // Verify the feature is enabled in protocol val protocol = getProtocol(engine, tablePath) assert(protocol.supportsFeature(TableFeatures.MATERIALIZE_PARTITION_COLUMNS_W_FEATURE)) // Find data parquet files (excluding checkpoints) val dataFiles = Files.walk(Paths.get(tablePath)).iterator().asScala .filter(path => path.toString.endsWith(".parquet") && !path.toString.contains("_delta_log")) .toSeq assert(dataFiles.nonEmpty, "Expected at least one data file to be written") // Read parquet schema from the first data file val parquetMetadata = ParquetFileReader.readFooter( configuration, new Path(dataFiles.head.toString)) val parquetSchema = parquetMetadata.getFileMetaData.getSchema // Verify that partition columns are present in the parquet schema val fieldNames: Set[String] = (0 until parquetSchema.getFieldCount).map(i => parquetSchema.getType(i).getName).toSet assert( fieldNames.contains("part1"), s"Partition column 'part1' should be present in parquet schema. Found fields: $fieldNames") assert( fieldNames.contains("part2"), s"Partition column 'part2' should be present in parquet schema. Found fields: $fieldNames") assert( fieldNames.contains("id"), s"Data column 'id' should be present in parquet schema. Found fields: $fieldNames") // Verify data using Kernel and Spark readers verifyWrittenContent( tablePath, testPartitionSchema, dataPartitionBatches1.flatMap(_.toTestRows)) } } test("partition columns NOT in parquet schema when materializePartitionColumns is disabled") { withTempDirAndEngine { (tablePath, engine) => // Create a partitioned table without the feature (default behavior) appendData( engine, tablePath, isNewTable = true, testPartitionSchema, partCols = testPartitionColumns, data = Seq(Map("part1" -> Literal.ofInt(1), "part2" -> Literal.ofInt(2)) -> dataPartitionBatches1)) // Verify the feature is NOT enabled in protocol val protocol = getProtocol(engine, tablePath) assert(!protocol.supportsFeature(TableFeatures.MATERIALIZE_PARTITION_COLUMNS_W_FEATURE)) // Find data parquet files (excluding checkpoints) val dataFiles = Files.walk(Paths.get(tablePath)).iterator().asScala .filter(path => path.toString.endsWith(".parquet") && !path.toString.contains("_delta_log")) .toSeq assert(dataFiles.nonEmpty, "Expected at least one data file to be written") // Read parquet schema from the first data file. val parquetMetadata = ParquetFileReader.readFooter( configuration, new Path(dataFiles.head.toString)) val parquetSchema = parquetMetadata.getFileMetaData.getSchema // Verify that partition columns are NOT present in the parquet schema. val fieldNames: Set[String] = (0 until parquetSchema.getFieldCount).map(i => parquetSchema.getType(i).getName).toSet assert( !fieldNames.contains("part1"), s"Partition column 'part1' should NOT be present in parquet schema. Found fields: " + fieldNames.toString) assert( !fieldNames.contains("part2"), s"Partition column 'part2' should NOT be present in parquet schema. Found fields: " + fieldNames.toString) assert( fieldNames.contains("id"), s"Data column 'id' should be present in parquet schema. Found fields: $fieldNames") // Verify data using Kernel and Spark readers. verifyWrittenContent( tablePath, testPartitionSchema, dataPartitionBatches1.flatMap(_.toTestRows)) } } /* ---- End: materialize partition columns tests ---- */ /////////////////////////////////////////////////////////////////////////// // Helper methods /////////////////////////////////////////////////////////////////////////// def latestSnapshot(table: Table, engine: Engine): SnapshotImpl = { table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl] } def checkWriterFeaturesSupported( tblPath: String, expWriterOnlyFeatures: String*): Unit = { val protocol = getLatestProtocol(tblPath) val missingFeatures = expWriterOnlyFeatures.toSet -- protocol.writerFeatures.getOrElse(Set.empty) assert( missingFeatures.isEmpty, s"The following expected writer features are not supported: " + s"${missingFeatures.mkString(", ")}") } def checkNoWriterFeaturesSupported(tblPath: String, notExpWriterOnlyFeatures: String*): Unit = { val protocol = getLatestProtocol(tblPath) assert(protocol.writerFeatures.getOrElse(Set.empty) .intersect(notExpWriterOnlyFeatures.toSet).isEmpty) } def checkReaderWriterFeaturesSupported( tblPath: String, expectedReaderWriterFeatures: String*): Unit = { val protocol = getLatestProtocol(tblPath) val missingInWriterSet = expectedReaderWriterFeatures.toSet -- protocol.writerFeatures.getOrElse(Set.empty) assert( missingInWriterSet.isEmpty, s"The following expected readerWriter features are not supported in writerFeatures set: " + s"${missingInWriterSet.mkString(", ")}") val missingInReaderSet = expectedReaderWriterFeatures.toSet -- protocol.readerFeatures.getOrElse(Set.empty) assert( missingInReaderSet.isEmpty, s"The following expected readerWriter features are not supported in readerFeatures set: " + s"${missingInReaderSet.mkString(", ")}") } def checkNoReaderWriterFeaturesSupported( tblPath: String, notExpReaderWriterFeatures: String*): Unit = { val protocol = getLatestProtocol(tblPath) assert(protocol.readerFeatures.getOrElse(Set.empty) .intersect(notExpReaderWriterFeatures.toSet).isEmpty) assert(protocol.writerFeatures.getOrElse(Set.empty) .intersect(notExpReaderWriterFeatures.toSet).isEmpty) } def getLatestProtocol(tblPath: String): Protocol = { val deltaLog = DeltaLog.forTable(spark, tblPath) deltaLog.update() deltaLog.snapshot.protocol } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableReadsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.io.File import java.math.BigDecimal import java.sql.Date import java.time.Instant import scala.jdk.CollectionConverters._ import io.delta.golden.GoldenTableUtils.goldenTablePath import io.delta.kernel.Table import io.delta.kernel.defaults.utils.{AbstractTestUtils, TestRow, TestUtils, TestUtilsWithLegacyKernelAPIs, TestUtilsWithTableManagerAPIs} import io.delta.kernel.exceptions.{InvalidTableException, KernelException, TableNotFoundException, UnsupportedProtocolVersionException} import io.delta.kernel.expressions.{Column, Literal, Predicate} import io.delta.kernel.internal.TableImpl import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.util.{DateTimeConstants, FileNames} import io.delta.kernel.internal.util.InternalUtils.daysSinceEpoch import io.delta.kernel.types.{LongType, StructType} import org.apache.spark.sql.delta.{DeltaLog, DeltaOperations} import org.apache.spark.sql.delta.actions.AddFile import org.apache.hadoop.shaded.org.apache.commons.io.FileUtils import org.apache.spark.sql.functions.col import org.scalatest.funsuite.AnyFunSuite class LegacyDeltaTableReadsSuite extends AbstractDeltaTableReadsSuite with TestUtilsWithLegacyKernelAPIs { // Loading a `Table` at a path and then loading a `Snapshot` is only applicable to the Legacy API. test("table deleted after the `Table` creation") { withTempDir { temp => val source = new File(goldenTablePath("data-reader-primitives")) val target = new File(temp.getCanonicalPath) FileUtils.copyDirectory(source, target) val table = Table.forPath(defaultEngine, target.getCanonicalPath) // delete the table and try to get the snapshot. Expect a failure. FileUtils.deleteDirectory(target) val ex = intercept[TableNotFoundException] { table.getLatestSnapshot(defaultEngine) } assert(ex.getMessage.contains( s"Delta table at path `file:${target.getCanonicalPath}` is not found")) } } } class DeltaTableReadsSuite extends AbstractDeltaTableReadsSuite with TestUtilsWithTableManagerAPIs trait AbstractDeltaTableReadsSuite extends AnyFunSuite { self: AbstractTestUtils => ////////////////////////////////////////////////////////////////////////////////// // Timestamp type tests ////////////////////////////////////////////////////////////////////////////////// // Below table is written in either UTC or PDT for the golden tables // Kernel always interprets partition timestamp columns in UTC /* id: int | Part (TZ agnostic): timestamp | time : timestamp ------------------------------------------------------------------------ 0 | 2020-01-01 08:09:10.001 | 2020-02-01 08:09:10 1 | 2021-10-01 08:09:20 | 1999-01-01 09:00:00 2 | 2021-10-01 08:09:20 | 2000-01-01 09:00:00 3 | 1969-01-01 00:00:00 | 1969-01-01 00:00:00 4 | null | null */ def row0: TestRow = TestRow( 0, 1577866150001000L, // 2020-01-01 08:09:10.001 UTC to micros since the epoch 1580544550000000L // 2020-02-01 08:09:10 UTC to micros since the epoch ) def row1: TestRow = TestRow( 1, 1633075760000000L, // 2021-10-01 08:09:20 UTC to micros since the epoch 915181200000000L // 1999-01-01 09:00:00 UTC to micros since the epoch ) def row2: TestRow = TestRow( 2, 1633075760000000L, // 2021-10-01 08:09:20 UTC to micros since the epoch 946717200000000L // 2000-01-01 09:00:00 UTC to micros since the epoch ) def row3: TestRow = TestRow( 3, -31536000000000L, // 1969-01-01 00:00:00 UTC to micros since the epoch -31536000000000L // 1969-01-01 00:00:00 UTC to micros since the epoch ) def row4: TestRow = TestRow( 4, null, null) def utcTableExpectedResult: Seq[TestRow] = Seq(row0, row1, row2, row3, row4) def testTimestampTable( goldenTableName: String, timeZone: String, expectedResult: Seq[TestRow]): Unit = { withTimeZone(timeZone) { checkTable( path = goldenTablePath(goldenTableName), expectedAnswer = expectedResult) } } for (timestampType <- Seq("INT96", "TIMESTAMP_MICROS", "TIMESTAMP_MILLIS")) { for (timeZone <- Seq("UTC", "Iceland", "PST", "America/Los_Angeles")) { test( s"end-to-end usage: timestamp table parquet timestamp format $timestampType tz $timeZone") { testTimestampTable("kernel-timestamp-" + timestampType, timeZone, utcTableExpectedResult) } } } // PST table - all the "time" col timestamps are + 8 hours def pstTableExpectedResult: Seq[TestRow] = utcTableExpectedResult.map { testRow => val values = testRow.toSeq TestRow( values(0), // Partition columns are written as the local date time without timezone information and then // interpreted by Kernel in UTC --> so the written partition value (& the read value) is the // same as the UTC table values(1), if (values(2) == null) { null } else { values(2).asInstanceOf[Long] + DateTimeConstants.MICROS_PER_HOUR * 8 }) } for (timeZone <- Seq("UTC", "Iceland", "PST", "America/Los_Angeles")) { test(s"end-to-end usage: timestamp in written in PST read in $timeZone") { testTimestampTable("kernel-timestamp-PST", timeZone, pstTableExpectedResult) } } test(s"end-to-end usage: table with partition column in ISO8601 timestamp format") { /* str: string | ts: timestamp (partition col) ------------------------------------------------------------------------ 2024-01-01 10:00:00 | 2024-01-01T10:00:00.000000Z 2024-01-02 12:30:00 | 2024-01-02T12:30:00.000000Z */ def row00: TestRow = TestRow( "2024-01-01 10:00:00", 1704103200000000L // 2024-01-01 10:00:00 UTC to micros since the epoch ) def row11: TestRow = TestRow( "2024-01-02 12:30:00", 1704198600000000L // 2024-01-02 12:30:00 UTC to micros since the epoch ) def ISO8601PartitionColTableExpectedResult: Seq[TestRow] = Seq(row00, row11) checkTable( goldenTablePath("kernel-timestamp-partition-col-ISO8601"), ISO8601PartitionColTableExpectedResult) } test(s"end-to-end usage: table with partition column in ISO8601 timestamp format with " + s"partition pruning") { /* str: string | ts: timestamp (partition col) ------------------------------------------------------------------------ 2024-01-01 10:00:00 | 2024-01-01T10:00:00.000000Z 2024-01-02 12:30:00 | 2024-01-02T12:30:00.000000Z */ def row00: TestRow = TestRow( "2024-01-01 10:00:00", 1704103200000000L // 2024-01-01 10:00:00 UTC to micros since the epoch ) val filter = new Predicate("=", new Column("ts"), Literal.ofTimestamp(1704103200000000L)) def ISO8601PartitionColTableExpectedResult: Seq[TestRow] = Seq(row00) checkTable( goldenTablePath("kernel-timestamp-partition-col-ISO8601"), ISO8601PartitionColTableExpectedResult, filter = filter) } test(s"end-to-end usage: spark-created table with partition column in ISO8601 timestamp " + s"format with microsecond precision and partition pruning") { // Set timezone to UTC so timestamps are interpreted consistently withTimeZone("UTC") { withTempDir { tempDir => val tablePath = tempDir.getCanonicalPath // Create table with Spark - timestamp partition with microsecond precision // Using spark.databricks.delta.write.utcTimestampPartitionValues=true /* str: string | ts: timestamp (partition col) ------------------------------------------------------------------------ 2024-01-01 10:00:00.123456 | 2024-01-01T10:00:00.123456Z 2024-01-02 12:30:00.654321 | 2024-01-02T12:30:00.654321Z */ withSQLConf("spark.databricks.delta.write.utcTimestampPartitionValues" -> "true") { spark.sql(s"""CREATE TABLE delta.`$tablePath` ( str string, ts timestamp ) USING delta PARTITIONED BY (ts)""") // Insert data with microsecond precision spark.sql(s"""INSERT INTO delta.`$tablePath` VALUES ('2024-01-01 10:00:00.123456', TIMESTAMP '2024-01-01 10:00:00.123456'), ('2024-01-02 12:30:00.654321', TIMESTAMP '2024-01-02 12:30:00.654321')""") } // Verify partition format is ISO8601 by checking the Delta log's partitionValues // (not the physical directory names, which may use a different format) val deltaLog = DeltaLog.forTable(spark, tablePath) val snapshot = deltaLog.update() val addFiles = snapshot.allFiles.collect() assert(addFiles.length == 2, s"Expected 2 AddFile entries, but found ${addFiles.length}") // Check that partitionValues in the Delta log use ISO8601 format val partitionValues = addFiles.map(_.partitionValues("ts")).toSeq assert( partitionValues.forall(_.contains("T")), s"Expected ISO8601 format with 'T' separator in partitionValues, but found: " + s"${partitionValues.mkString(", ")}") // Now verify reading with Kernel def row00: TestRow = TestRow( "2024-01-01 10:00:00.123456", 1704103200123456L // 2024-01-01 10:00:00.123456 UTC to micros since the epoch ) def row11: TestRow = TestRow( "2024-01-02 12:30:00.654321", 1704198600654321L // 2024-01-02 12:30:00.654321 UTC to micros since the epoch ) // Test reading all data checkTable(tablePath, Seq(row00, row11)) // Test partition pruning val filter = new Predicate( "=", new Column("ts"), Literal.ofTimestamp(1704103200123456L) ) // Only read row00 checkTable( tablePath, Seq(row00), filter = filter) } } } test("read table with far-future timestamp in stats") { withTempDir { tempDir => val path = tempDir.getCanonicalPath spark.sql(s"CREATE TABLE delta.`$path` (ts TIMESTAMP) USING DELTA") spark.sql(s"INSERT INTO delta.`$path` VALUES (TIMESTAMP'2020-01-01 00:00:00')") spark.sql(s"INSERT INTO delta.`$path` VALUES (TIMESTAMP'9999-12-31 23:59:59')") val filter = new Predicate("<", new Column("ts"), Literal.ofTimestamp(253402300799000000L)) checkTable( path, Seq(TestRow(1577836800000000L)), filter = filter, expectedRemainingFilter = filter) } } ////////////////////////////////////////////////////////////////////////////////// // Timestamp_NTZ tests ////////////////////////////////////////////////////////////////////////////////// // Below is the golden table used in test // (INTEGER id, TIMESTAMP_NTZ tsNtz, TIMESTAMP_NTZ tsNtzPartition) // (0, '2021-11-18 02:30:00.123456','2021-11-18 02:30:00.123456'), // (1, '2013-07-05 17:01:00.123456','2021-11-18 02:30:00.123456'), // (2, NULL, '2021-11-18 02:30:00.123456'), // (3, '2021-11-18 02:30:00.123456','2013-07-05 17:01:00.123456'), // (4, '2013-07-05 17:01:00.123456','2013-07-05 17:01:00.123456'), // (5, NULL, '2013-07-05 17:01:00.123456'), // (6, '2021-11-18 02:30:00.123456', NULL), // (7, '2013-07-05 17:01:00.123456', NULL), // (8, NULL, NULL) val expectedTimestampNtzTestRows = Seq( TestRow(0, 1637202600123456L, 1637202600123456L), TestRow(1, 1373043660123456L, 1637202600123456L), TestRow(2, null, 1637202600123456L), TestRow(3, 1637202600123456L, 1373043660123456L), TestRow(4, 1373043660123456L, 1373043660123456L), TestRow(5, null, 1373043660123456L), TestRow(6, 1637202600123456L, null), TestRow(7, 1373043660123456L, null), TestRow(8, null, null)) Seq("", "-name-mode", "-id-mode").foreach { cmMode => test(s"end-to-end: read table with timestamp_ntz columns (including partition): $cmMode") { checkTable( path = goldenTablePath(s"data-reader-timestamp_ntz$cmMode"), expectedAnswer = expectedTimestampNtzTestRows) } } ////////////////////////////////////////////////////////////////////////////////// // Decimal type tests ////////////////////////////////////////////////////////////////////////////////// for (tablePath <- Seq("basic-decimal-table", "basic-decimal-table-legacy")) { test(s"end to end: reading $tablePath") { val expectedResult = Seq( ("234.00000", "1.00", "2.00000", "3.0000000000"), ("2342222.23454", "111.11", "22222.22222", "3333333333.3333333333"), ("0.00004", "0.00", "0.00000", "0E-10"), ("-2342342.23423", "-999.99", "-99999.99999", "-9999999999.9999999999")).map { tup => ( new BigDecimal(tup._1), new BigDecimal(tup._2), new BigDecimal(tup._3), new BigDecimal(tup._4)) } checkTable( path = goldenTablePath(tablePath), expectedAnswer = expectedResult.map(TestRow.fromTuple(_))) } } test(s"end to end: reading decimal-various-scale-precision") { val tablePath = goldenTablePath("decimal-various-scale-precision") val expResults = spark.sql(s"SELECT * FROM delta.`$tablePath`") .collect() .map(TestRow(_)) checkTable( path = goldenTablePath("decimal-various-scale-precision"), expectedAnswer = expResults) } ////////////////////////////////////////////////////////////////////////////////// // Table/Snapshot tests ////////////////////////////////////////////////////////////////////////////////// test("invalid path") { val invalidPath = "/path/to/non-existent-directory" val tableManager = getTableManagerAdapter def expectTableNotFoundException(fn: () => Unit): Unit = { val ex = intercept[TableNotFoundException] { fn() } assert(ex.getMessage().contains(s"Delta table at path `file:$invalidPath` is not found")) } expectTableNotFoundException { () => tableManager.getSnapshotAtLatest(defaultEngine, invalidPath) } expectTableNotFoundException { () => tableManager.getSnapshotAtVersion(defaultEngine, invalidPath, 1) } expectTableNotFoundException { () => tableManager.getSnapshotAtTimestamp(defaultEngine, invalidPath, 1) } } // TODO for the below, when should we throw an exception? #2253 // - on Table creation? // - on Snapshot creation? test("empty _delta_log folder") { withTempDir { dir => new File(dir, "_delta_log").mkdirs() intercept[TableNotFoundException] { latestSnapshot(dir.getAbsolutePath) } } } test("empty folder with no _delta_log dir") { withTempDir { dir => intercept[TableNotFoundException] { latestSnapshot(dir.getAbsolutePath) } } } test("non-empty folder not a delta table") { intercept[TableNotFoundException] { latestSnapshot(goldenTablePath("no-delta-log-folder")) } } ////////////////////////////////////////////////////////////////////////////////// // Misc tests ////////////////////////////////////////////////////////////////////////////////// test("end to end: multi-part checkpoint") { checkTable( path = goldenTablePath("multi-part-checkpoint"), expectedAnswer = (Seq(0L) ++ (0L until 30L)).map(TestRow(_))) } test("read partitioned table") { val path = "file:" + goldenTablePath("data-reader-partition-values") // for now we don't support timestamp type partition columns so remove from read columns val readCols = getTableManagerAdapter.getSnapshotAtLatest(defaultEngine, path) .getSchema() .withoutField("as_timestamp") .fields() .asScala .map(_.getName) .toSeq val expectedAnswer = Seq(0, 1).map { i => TestRow( i, i.toLong, i.toByte, i.toShort, i % 2 == 0, i.toFloat, i.toDouble, i.toString, "null", daysSinceEpoch(Date.valueOf("2021-09-08")), new BigDecimal(i), Seq(TestRow(i), TestRow(i), TestRow(i)), TestRow(i.toString, i.toString, TestRow(i, i.toLong)), i.toString) } ++ (TestRow( null, null, null, null, null, null, null, null, null, null, null, Seq(TestRow(2), TestRow(2), TestRow(2)), TestRow("2", "2", TestRow(2, 2L)), "2") :: Nil) checkTable( path = path, expectedAnswer = expectedAnswer, readCols = readCols) } test("table with complex array types") { val path = "file:" + goldenTablePath("data-reader-array-complex-objects") val expectedAnswer = (0 until 10).map { i => TestRow( i, Seq(Seq(Seq(i, i, i), Seq(i, i, i)), Seq(Seq(i, i, i), Seq(i, i, i))), Seq( Seq(Seq(Seq(i, i, i), Seq(i, i, i)), Seq(Seq(i, i, i), Seq(i, i, i))), Seq(Seq(Seq(i, i, i), Seq(i, i, i)), Seq(Seq(i, i, i), Seq(i, i, i)))), Seq( Map[String, Long](i.toString -> i.toLong), Map[String, Long](i.toString -> i.toLong)), Seq(TestRow(i), TestRow(i), TestRow(i))) } checkTable( path = path, expectedAnswer = expectedAnswer) } Seq("name", "id").foreach { columnMappingMode => test(s"table with `$columnMappingMode` column mapping mode") { val path = goldenTablePath(s"table-with-columnmapping-mode-$columnMappingMode") val expectedAnswer = (0 until 5).map { i => TestRow( i.byteValue(), i.shortValue(), i, i.longValue(), i.floatValue(), i.doubleValue(), new java.math.BigDecimal(i), i % 2 == 0, i.toString, i.toString.getBytes, daysSinceEpoch(Date.valueOf("2021-11-18")), // date in days (i * 1000).longValue(), // timestamp in micros TestRow(i.toString, TestRow(i)), // nested_struct Seq(i, i + 1), // array_of_prims Seq(Seq(i, i + 1), Seq(i + 2, i + 3)), // array_of_arrays Seq(Map(i -> Seq(2, 3), i + 1 -> Seq(4, 5))), // array_of_map_of_arrays Seq(TestRow(i), TestRow(i)), // array_of_structs TestRow( // struct_of_arrays_maps_of_structs Seq(i, i + 1), Map(Seq(i, i + 1) -> TestRow(i + 2))), Map( i -> (i + 1).longValue(), (i + 2) -> (i + 3).longValue() ), // map_of_prims Map(i + 1 -> TestRow((i * 20).longValue())), // map_of_rows { val val1 = Seq(i, null, i + 1) val val2 = Seq[Integer]() Map( i.longValue() -> val1, (i + 1).longValue() -> val2 ) // map_of_arrays }, Map( // map_of_maps i.toLong -> Map(i -> i), (i + 1).toLong -> Map(i + 2 -> i))) } ++ Seq(TestRow(Seq.fill(22)(null): _*)) // all nulls row, 22 columns checkTable( path = path, expectedAnswer = expectedAnswer) } } Seq("name", "id").foreach { columnMappingMode => test(s"table with `$columnMappingMode` column mapping mode - read subset of columns") { val path = goldenTablePath(s"table-with-columnmapping-mode-$columnMappingMode") val expectedAnswer = (0 until 5).map { i => TestRow( i.byteValue(), new java.math.BigDecimal(i), TestRow(i.toString, TestRow(i)), // nested_struct Seq(i, i + 1), // array_of_prims Map( i -> (i + 1).longValue(), (i + 2) -> (i + 3).longValue() ) // map_of_prims ) } ++ (TestRow( null, null, null, null, null ) :: Nil) checkTable( path = path, expectedAnswer = expectedAnswer, readCols = Seq("ByteType", "decimal", "nested_struct", "array_of_prims", "map_of_prims")) } } test("read subfield of array of struct") { withTempDir { path => withTempTable { tbl => spark.sql(s"""CREATE TABLE $tbl ( id int, array_of_struct array> ) USING delta LOCATION '$path' """) spark.sql(s"""INSERT INTO $tbl VALUES (1, array(struct(2 as x, 3 as y))), (6, array(struct(7 as x, 8 as y))), (11, array(struct(null as x, 8 as y))), (12, array()), (13, null)""") // Test reading with pruned schema - only x field from struct val prunedSchema = new io.delta.kernel.types.StructType() .add( "array_of_struct", new io.delta.kernel.types.ArrayType( new io.delta.kernel.types.StructType() .add("x", io.delta.kernel.types.IntegerType.INTEGER), true)) val result = readSnapshot( latestSnapshot(path.toString), readSchema = prunedSchema) val expectedAnswer = Seq( TestRow(Seq(TestRow(2))), TestRow(Seq(TestRow(7))), TestRow(Seq(TestRow(null: Any))), TestRow(Seq()), TestRow(null: Any)) checkAnswer(result, expectedAnswer) } } } test("read subfield of array of array of struct") { withTempDir { path => withTempTable { tbl => spark.sql(s"""CREATE TABLE $tbl ( id int, array_of_array_of_struct array>> ) USING delta LOCATION '$path' """) spark.sql(s"""INSERT INTO $tbl VALUES (1, array(array(struct(4 as x, 5 as y)))), (6, array(array(struct(9 as x, 10 as y)))), (11, array(array(struct(null as x, 10 as y)))), (12, array(array())), (13, array(null))""") val prunedSchema = new io.delta.kernel.types.StructType() .add( "array_of_array_of_struct", new io.delta.kernel.types.ArrayType( new io.delta.kernel.types.ArrayType( new io.delta.kernel.types.StructType() .add("x", io.delta.kernel.types.IntegerType.INTEGER), true), true)) val result = readSnapshot( latestSnapshot(path.toString), readSchema = prunedSchema) val expectedAnswer = Seq( TestRow(Seq(Seq(TestRow(4)))), TestRow(Seq(Seq(TestRow(9)))), TestRow(Seq(Seq(TestRow(null: Any)))), TestRow(Seq(Seq())), TestRow(Seq(null))) checkAnswer(result, expectedAnswer) } } } test("read array of array of int") { withTempDir { path => withTempTable { tbl => spark.sql(s"""CREATE TABLE $tbl ( id int, array_of_array_of_int array> ) USING delta LOCATION '$path' """) spark.sql(s"""INSERT INTO $tbl VALUES (1, array(array(100, 101))), (6, array(array(102, 103))), (11, array(array(null, 104))), (12, array(array())), (13, array(null))""") checkTable( path = path.toString, expectedAnswer = Seq( TestRow(Seq(Seq(100, 101))), TestRow(Seq(Seq(102, 103))), TestRow(Seq(Seq(null, 104))), TestRow(Seq(Seq())), TestRow(Seq(null))), readCols = Seq("array_of_array_of_int")) } } } test("read array of int") { withTempDir { path => withTempTable { tbl => spark.sql(s"""CREATE TABLE $tbl ( id int, array_of_int array ) USING delta LOCATION '$path' """) spark.sql(s"""INSERT INTO $tbl VALUES (1, array(200, 201)), (6, array(202, 203)), (11, array(null, 204)), (12, array()), (13, null)""") checkTable( path = path.toString, expectedAnswer = Seq( TestRow(Seq(200, 201)), TestRow(Seq(202, 203)), TestRow(Seq(null, 204)), TestRow(Seq()), TestRow(null: Any)), readCols = Seq("array_of_int")) } } } test("table with type widening on basic types") { val path = goldenTablePath("type-widening") def timestampToMicros(timestamp: String): Long = { val instant = Instant.parse(timestamp) instant.getEpochSecond() * DateTimeConstants.MICROS_PER_SECOND + instant.getNano() / 1000 } val expectedAnswer = Seq( TestRow( 1L, 2L, 3.4.toFloat.toDouble, 5.0, 6.0, 7.0, timestampToMicros("2024-09-09T00:00:00Z")), TestRow( Long.MaxValue, Long.MaxValue, 1.234567890123, 1.234567890123, 1.234567890123, 1.234567890123, timestampToMicros("2024-09-09T12:34:56.123456Z"))) checkTable( path = path, expectedAnswer = expectedAnswer, readCols = Seq( "byte_long", "int_long", "float_double", "byte_double", "short_double", "int_double", "date_timestamp_ntz")) } test("table with type widening to decimal types") { val path = goldenTablePath("type-widening") val expectedAnswer = Seq( TestRow( BigDecimal.valueOf(12345L, 2), BigDecimal.valueOf(6789000L, 5), BigDecimal.valueOf(10L, 1), BigDecimal.valueOf(20L, 1), BigDecimal.valueOf(30L, 1), BigDecimal.valueOf(40L, 1)), TestRow( BigDecimal.valueOf(1234567890123456L, 2), BigDecimal.valueOf(1234567890123456L, 5), BigDecimal.valueOf(1234L, 1), BigDecimal.valueOf(123456L, 1), BigDecimal.valueOf(12345678901L, 1), BigDecimal.valueOf(1234567890123456789L, 1))) checkTable( path = path, expectedAnswer = expectedAnswer, readCols = Seq( "decimal_decimal_same_scale", "decimal_decimal_greater_scale", "byte_decimal", "short_decimal", "int_decimal", "long_decimal")) } test("table with type widening to nested types") { val path = goldenTablePath("type-widening-nested") val expectedAnswer = Seq( TestRow(TestRow(1L), Map(2L -> 3L), Seq(4L, 5L)), TestRow( TestRow(Long.MaxValue), Map(Long.MaxValue -> Long.MaxValue), Seq(Long.MaxValue, Long.MinValue))) checkTable( path = path, expectedAnswer = expectedAnswer, readCols = Seq("struct", "map", "array")) } test("table with complex map types") { val path = "file:" + goldenTablePath("data-reader-map") val expectedAnswer = (0 until 10).map { i => TestRow( i, Map(i -> i), Map(i.toLong -> i.toByte), Map(i.toShort -> (i % 2 == 0)), Map(i.toFloat -> i.toDouble), Map(i.toString -> new BigDecimal(i)), Map(i -> Seq(TestRow(i), TestRow(i), TestRow(i)))) } checkTable( path = path, expectedAnswer = expectedAnswer) } test("table with array of primitives") { val expectedAnswer = (0 until 10).map { i => TestRow( Seq(i), Seq(i.toLong), Seq(i.toByte), Seq(i.toShort), Seq(i % 2 == 0), Seq(i.toFloat), Seq(i.toDouble), Seq(i.toString), Seq(Array(i.toByte, i.toByte)), Seq(new BigDecimal(i))) } checkTable( path = goldenTablePath("data-reader-array-primitives"), expectedAnswer = expectedAnswer) } test("table primitives") { val expectedAnswer = (0 to 10).map { case 10 => TestRow(null, null, null, null, null, null, null, null, null, null) case i => TestRow( i, i.toLong, i.toByte, i.toShort, i % 2 == 0, i.toFloat, i.toDouble, i.toString, Array[Byte](i.toByte, i.toByte), new BigDecimal(i)) } checkTable( path = goldenTablePath("data-reader-primitives"), expectedAnswer = expectedAnswer) } test("table with checkpoint") { checkTable( path = getTestResourceFilePath("basic-with-checkpoint"), expectedAnswer = (0 until 150).map(i => TestRow(i.toLong))) } test(s"table with spaces in the table path") { withTempDir { tempDir => val target = tempDir.getCanonicalPath + s"/table- -path" spark.sql(s"CREATE TABLE delta.`$target` USING DELTA " + s"SELECT * FROM delta.`${getTestResourceFilePath("basic-with-checkpoint")}`") checkTable( path = target, expectedAnswer = (0 until 150).map(i => TestRow(i.toLong))) } } test("table with name column mapping mode") { val expectedAnswer = (0 to 10).map { case 10 => TestRow(null, null, null, null, null, null, null, null, null, null) case i => TestRow( i, i.toLong, i.toByte, i.toShort, i % 2 == 0, i.toFloat, i.toDouble, i.toString, Array[Byte](i.toByte, i.toByte), new BigDecimal(i)) } checkTable( path = getTestResourceFilePath("data-reader-primitives-column-mapping-name"), expectedAnswer = expectedAnswer) } test("partitioned table with column mapping") { val expectedAnswer = (0 to 2).map { case 2 => TestRow(null, null, "2") case i => TestRow(i, i.toDouble, i.toString) } val readCols = Seq( // partition fields "as_int", "as_double", // data fields "value") checkTable( path = getTestResourceFilePath("data-reader-partition-values-column-mapping-name"), readCols = readCols, expectedAnswer = expectedAnswer) } test("simple end to end with vacuum protocol check feature") { val expectedValues = (0 until 100).map(x => (x, s"val=$x")) checkTable( path = goldenTablePath("basic-with-vacuum-protocol-check-feature"), expectedAnswer = expectedValues.map(TestRow.fromTuple)) } test("table with nested struct") { val expectedAnswer = (0 until 10).map { i => TestRow(TestRow(i.toString, i.toString, TestRow(i, i.toLong)), i) } checkTable( path = goldenTablePath("data-reader-nested-struct"), expectedAnswer = expectedAnswer) } test("table with empty parquet files") { checkTable( path = goldenTablePath("125-iterator-bug"), expectedAnswer = (1 to 5).map(TestRow(_))) } test("handle corrupted '_last_checkpoint' file") { checkTable( path = goldenTablePath("corrupted-last-checkpoint-kernel"), expectedAnswer = (0L until 100L).map(TestRow(_))) } test("error - version not contiguous") { val e = intercept[InvalidTableException] { latestSnapshot(goldenTablePath("versions-not-contiguous")) } assert(e.getMessage.contains("versions are not contiguous: ([0, 2])")) } test("table protocol version greater than reader protocol version") { val e = intercept[UnsupportedProtocolVersionException] { latestSnapshot(goldenTablePath("deltalog-invalid-protocol-version")) .getScanBuilder() .build() } assert(e.getMessage.contains("Unsupported Delta protocol reader version")) } test("table with void type - schema parsing is lazy") { withTempDir { tempDir => val path = tempDir.getCanonicalPath spark.sql(s"CREATE TABLE delta.`${tempDir.getAbsolutePath}`(x INTEGER, y VOID) USING DELTA") // Snapshot loading should succeed since schema parsing is now lazy val snapshot = latestSnapshot(path) assert(snapshot.getVersion >= 0) // Accessing the schema should still throw KernelException due to VOID type val e = intercept[KernelException] { snapshot.getSchema } assert(e.getMessage.contains( "Failed to parse the schema. Encountered unsupported Delta data type: VOID")) } } test("read a shallow cloned table") { withTempDir { tempDir => val target = tempDir.getCanonicalPath val source = goldenTablePath("data-reader-partition-values") spark.sql(s"CREATE TABLE delta.`$target` SHALLOW CLONE delta.`$source`") withSparkTimeZone("UTC") { val expAnswer = spark.read.format("delta").load(source).collect().map(TestRow(_)).toSeq assert(expAnswer.size == 3) checkTable(target, expAnswer) } } } ////////////////////////////////////////////////////////////////////////////////// // getSnapshotAtVersion end-to-end tests (log segment tests in SnapshotManagerSuite) ////////////////////////////////////////////////////////////////////////////////// test("getSnapshotAtVersion: basic end-to-end read") { withTempDir { tempDir => val path = tempDir.getCanonicalPath (0 to 10).foreach { i => spark.range(i * 10, i * 10 + 10).write .format("delta") .mode("append") .save(path) } // Read a checkpoint version checkTable( path = path, expectedAnswer = (0L to 99L).map(TestRow(_)), version = Some(9), expectedVersion = Some(9)) // Read a JSON version checkTable( path = path, expectedAnswer = (0L to 89L).map(TestRow(_)), version = Some(8), expectedVersion = Some(8)) // Read the current version checkTable( path = path, expectedAnswer = (0L to 109L).map(TestRow(_)), version = Some(10), expectedVersion = Some(10)) // Cannot read a version that does not exist val e = intercept[RuntimeException] { getTableManagerAdapter.getSnapshotAtVersion(defaultEngine, path, 11) } assert(e.getMessage.contains( "Cannot load table version 11 as it does not exist. The latest available version is 10")) } } test("getSnapshotAtVersion: end-to-end test with truncated delta log") { withTempDir { tempDir => val tablePath = tempDir.getCanonicalPath // Write versions [0, 10] (inclusive) including a checkpoint (0 to 10).foreach { i => spark.range(i * 10, i * 10 + 10).write .format("delta") .mode("append") .save(tablePath) } val log = org.apache.spark.sql.delta.DeltaLog.forTable( spark, new org.apache.hadoop.fs.Path(tablePath)) val deltaCommitFileProvider = org.apache.spark.sql.delta.util.DeltaCommitFileProvider( log.unsafeVolatileSnapshot) // Delete the log files for versions 0-9, truncating the table history to version 10 (0 to 9).foreach { i => val jsonFile = deltaCommitFileProvider.deltaFile(i) new File(new org.apache.hadoop.fs.Path(log.logPath, jsonFile).toUri).delete() } // Create version 11 that overwrites the whole table spark.range(50).write .format("delta") .mode("overwrite") .save(tablePath) // Cannot read a version that has been truncated val e = intercept[RuntimeException] { getTableManagerAdapter.getSnapshotAtVersion(defaultEngine, tablePath, 9) } assert(e.getMessage.contains("Cannot load table version 9")) // Can read version 10 checkTable( path = tablePath, expectedAnswer = (0L to 109L).map(TestRow(_)), version = Some(10), expectedVersion = Some(10)) // Can read version 11 checkTable( path = tablePath, expectedAnswer = (0L until 50L).map(TestRow(_)), version = Some(11), expectedVersion = Some(11)) } } test("time travel with schema change") { withTempDir { tempDir => spark.range(10).write.format("delta").save(tempDir.getCanonicalPath) spark.range(10, 20).withColumn("part", col("id")) .write.format("delta").mode("append").option("mergeSchema", true) .save(tempDir.getCanonicalPath) checkTable( path = tempDir.getCanonicalPath, expectedAnswer = (0L until 10L).map(TestRow(_)), expectedSchema = new StructType().add("id", LongType.LONG), version = Some(0), expectedVersion = Some(0)) } } test("time travel with partition change") { withTempDir { tempDir => spark.range(10).withColumn("part5", col("id") % 5) .write.format("delta").partitionBy("part5").mode("append") .save(tempDir.getCanonicalPath) spark.range(10, 20).withColumn("part2", col("id") % 2) .write .format("delta") .partitionBy("part2") .mode("overwrite") .option("overwriteSchema", true) .save(tempDir.getCanonicalPath) checkTable( path = tempDir.getCanonicalPath, expectedAnswer = (0L until 10L).map(v => TestRow(v, v % 5)), expectedSchema = new StructType() .add("id", LongType.LONG) .add("part5", LongType.LONG), version = Some(0), expectedVersion = Some(0)) } } ////////////////////////////////////////////////////////////////////////////////// // getSnapshotAtTimestamp end-to-end tests (more tests in DeltaHistoryManagerSuite) ////////////////////////////////////////////////////////////////////////////////// private def generateCommits(path: String, commits: Long*): Unit = { commits.zipWithIndex.foreach { case (ts, i) => spark.range(i * 10, i * 10 + 10).write.format("delta").mode("append").save(path) val file = new File(FileNames.deltaFile(new Path(path, "_delta_log"), i)) file.setLastModified(ts) } } test("getSnapshotAtTimestamp: basic end-to-end read") { withTempDir { tempDir => val start = 1540415658000L val minuteInMilliseconds = 60000L generateCommits( tempDir.getCanonicalPath, start, start + 20 * minuteInMilliseconds, start + 40 * minuteInMilliseconds) // Exact timestamp for version 0 checkTable( path = tempDir.getCanonicalPath, expectedAnswer = (0L until 10L).map(TestRow(_)), timestamp = Some(start), expectedVersion = Some(0)) // Timestamp between version 0 and 1 should load version 0 checkTable( path = tempDir.getCanonicalPath, expectedAnswer = (0L until 10L).map(TestRow(_)), timestamp = Some(start + 10 * minuteInMilliseconds), expectedVersion = Some(0)) // Exact timestamp for version 1 checkTable( path = tempDir.getCanonicalPath, expectedAnswer = (0L until 20L).map(TestRow(_)), timestamp = Some(start + 20 * minuteInMilliseconds), expectedVersion = Some(1)) // Exact timestamp for the last version checkTable( path = tempDir.getCanonicalPath, expectedAnswer = (0L until 30L).map(TestRow(_)), timestamp = Some(start + 40 * minuteInMilliseconds), expectedVersion = Some(2)) // Timestamp after last commit fails val e1 = intercept[RuntimeException] { checkTable( path = tempDir.getCanonicalPath, expectedAnswer = Seq(), timestamp = Some(start + 50 * minuteInMilliseconds)) } assert(e1.getMessage.contains( s"The provided timestamp ${start + 50 * minuteInMilliseconds} ms " + s"(2018-10-24T22:04:18Z) is after the latest available version")) // Timestamp before the first commit fails val e2 = intercept[RuntimeException] { checkTable( path = tempDir.getCanonicalPath, expectedAnswer = Seq(), timestamp = Some(start - 1L)) } assert(e2.getMessage.contains( s"The provided timestamp ${start - 1L} ms (2018-10-24T21:14:17.999Z) is before " + s"the earliest available version")) } } test("getSnapshotAtTimestamp: empty _delta_log folder") { withTempDir { dir => new File(dir, "_delta_log").mkdirs() intercept[TableNotFoundException] { getTableManagerAdapter .getSnapshotAtTimestamp(defaultEngine, dir.getCanonicalPath, 0L) } } } test("getSnapshotAtTimestamp: empty folder no _delta_log dir") { withTempDir { dir => intercept[TableNotFoundException] { getTableManagerAdapter .getSnapshotAtTimestamp(defaultEngine, dir.getCanonicalPath, 0L) } } } test("getSnapshotAtTimestamp: non-empty folder not a delta table") { withTempDir { dir => spark.range(20).write.format("parquet").mode("overwrite").save(dir.getCanonicalPath) intercept[TableNotFoundException] { getTableManagerAdapter .getSnapshotAtTimestamp(defaultEngine, dir.getCanonicalPath, 0L) } } } /////////////////////////////////////////////////////////////////////////////////////////////// // getVersionBeforeOrAtTimestamp + getVersionAtOrAfterTimestamp tests // (more in TableImplSuite and DeltaHistoryManagerSuite) ////////////////////////////////////////////////////////////////////////////////////////////// // Copied from Standalone DeltaLogSuite test("getVersionBeforeOrAtTimestamp and getVersionAtOrAfterTimestamp") { // TODO: [delta-io/delta#4770] Implement the `getVersionBeforeOrAtTimestamp` and // `getVersionAtOrAfterTimestamp` APIs for CatalogManaged tables. assume(getTableManagerAdapter.supportsTimestampResolution, "Timestamp queries not supported") // Note: // - all Xa test cases will test getVersionBeforeOrAtTimestamp // - all Xb test cases will test getVersionAtOrAfterTimestamp withTempDir { dir => val log = DeltaLog.forTable(spark, dir.getCanonicalPath) val tableImpl = Table.forPath(defaultEngine, dir.getCanonicalPath).asInstanceOf[TableImpl] // ========== case 0: delta table does not exist ========== intercept[TableNotFoundException] { tableImpl.getVersionBeforeOrAtTimestamp(defaultEngine, System.currentTimeMillis()) } intercept[TableNotFoundException] { tableImpl.getVersionAtOrAfterTimestamp(defaultEngine, System.currentTimeMillis()) } // Setup part 1 of 2: create log files spark.range(10).write.format("delta").mode("overwrite").save(dir.getCanonicalPath) (1 to 2).foreach { i => val files = AddFile(i.toString, Map.empty, 1, 1, true) :: Nil log.startTransaction().commit(files, DeltaOperations.ManualUpdate) } // Setup part 2 of 2: edit lastModified times val logPath = new Path(dir.getCanonicalPath, "_delta_log") val delta0 = new File(FileNames.deltaFile(logPath, 0)) val delta1 = new File(FileNames.deltaFile(logPath, 1)) val delta2 = new File(FileNames.deltaFile(logPath, 2)) delta0.setLastModified(1000) delta1.setLastModified(2000) delta2.setLastModified(3000) // ========== case 1: before first commit ========== // case 1a val e1 = intercept[KernelException] { tableImpl.getVersionBeforeOrAtTimestamp(defaultEngine, 500) }.getMessage assert(e1.contains("is before the earliest available version 0")) // case 1b assert(tableImpl.getVersionAtOrAfterTimestamp(defaultEngine, 500) == 0) // ========== case 2: at first commit ========== // case 2a assert(tableImpl.getVersionBeforeOrAtTimestamp(defaultEngine, 1000) == 0) // case 2b assert(tableImpl.getVersionAtOrAfterTimestamp(defaultEngine, 1000) == 0) // ========== case 3: between two normal commits ========== // case 3a assert(tableImpl.getVersionBeforeOrAtTimestamp(defaultEngine, 1500) == 0) // round down to v0 // case 3b assert(tableImpl.getVersionAtOrAfterTimestamp(defaultEngine, 1500) == 1) // round up to v1 // ========== case 4: at last commit ========== // case 4a assert(tableImpl.getVersionBeforeOrAtTimestamp(defaultEngine, 3000) == 2) // case 4b assert(tableImpl.getVersionAtOrAfterTimestamp(defaultEngine, 3000) == 2) // ========== case 5: after last commit ========== // case 5a assert(tableImpl.getVersionBeforeOrAtTimestamp(defaultEngine, 4000) == 2) // case 5b val e2 = intercept[KernelException] { tableImpl.getVersionAtOrAfterTimestamp(defaultEngine, 4000) }.getMessage assert(e2.contains("is after the latest available version 2")) } } // Copied from Standalone DeltaLogSuite test("getVersionBeforeOrAtTimestamp and getVersionAtOrAfterTimestamp - recoverability") { // TODO: [delta-io/delta#4770] Implement the `getVersionBeforeOrAtTimestamp` and // `getVersionAtOrAfterTimestamp` APIs for CatalogManaged tables. assume(getTableManagerAdapter.supportsTimestampResolution, "Timestamp queries not supported") withTempDir { dir => // local file system truncates to seconds val nowEpochMs = System.currentTimeMillis() / 1000 * 1000 val logPath = new Path(dir.getCanonicalPath, "_delta_log") val log = DeltaLog.forTable(spark, dir.getCanonicalPath) val tableImpl = Table.forPath(defaultEngine, dir.getCanonicalPath).asInstanceOf[TableImpl] spark.range(10).write.format("delta").mode("overwrite").save(dir.getCanonicalPath) (1 to 35).foreach { i => val files = AddFile(i.toString, Map.empty, 1, 1, true) :: Nil log.startTransaction().commit(files, DeltaOperations.ManualUpdate) } (0 to 35).foreach { i => val delta = new File(FileNames.deltaFile(logPath, i)) if (i >= 25) { delta.setLastModified(nowEpochMs + i * 1000) } else { assert(delta.delete()) } } // A checkpoint exists at version 30, so all versions [30, 35] are recoverable. // Nonetheless, getVersionBeforeOrAtTimestamp and getVersionAtOrAfterTimestamp do not // require that the version is recoverable, so we should still be able to get back versions // [25-29] (25 to 34).foreach { i => if (i == 25) { assertThrows[KernelException] { tableImpl.getVersionBeforeOrAtTimestamp(defaultEngine, nowEpochMs + i * 1000 - 1) } } else { assert(tableImpl.getVersionBeforeOrAtTimestamp(defaultEngine, nowEpochMs + i * 1000 - 1) == i - 1) } assert( tableImpl.getVersionAtOrAfterTimestamp(defaultEngine, nowEpochMs + i * 1000 - 1) == i) assert(tableImpl.getVersionBeforeOrAtTimestamp(defaultEngine, nowEpochMs + i * 1000) == i) assert(tableImpl.getVersionAtOrAfterTimestamp(defaultEngine, nowEpochMs + i * 1000) == i) assert( tableImpl.getVersionBeforeOrAtTimestamp(defaultEngine, nowEpochMs + i * 1000 + 1) == i) if (i == 35) { tableImpl.getVersionAtOrAfterTimestamp(defaultEngine, nowEpochMs + i * 1000 + 1) } else { assert(tableImpl.getVersionAtOrAfterTimestamp(defaultEngine, nowEpochMs + i * 1000 + 1) == i + 1) } } } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableSchemaEvolutionSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.util.Collections.emptySet import scala.collection.JavaConverters._ import scala.collection.immutable.Seq import io.delta.kernel.{Operation, Table, Transaction, TransactionCommitResult} import io.delta.kernel.data.Row import io.delta.kernel.defaults.utils.{AbstractWriteUtils, WriteUtils, WriteUtilsWithV2Builders} import io.delta.kernel.engine.Engine import io.delta.kernel.exceptions.KernelException import io.delta.kernel.expressions.Column import io.delta.kernel.internal.{SnapshotImpl, TableConfig} import io.delta.kernel.internal.actions.DomainMetadata import io.delta.kernel.internal.clustering.ClusteringMetadataDomain import io.delta.kernel.internal.util.{ColumnMapping, ColumnMappingSuiteBase} import io.delta.kernel.types.{ArrayType, CollationIdentifier, DecimalType, FieldMetadata, IntegerType, LongType, MapType, StringType, StructType, TypeChange} import io.delta.kernel.utils.CloseableIterable import io.delta.kernel.utils.CloseableIterable.emptyIterable import org.scalatest.funsuite.AnyFunSuite import org.scalatest.prop.TableDrivenPropertyChecks.forAll import org.scalatest.prop.Tables class DeltaTableSchemaEvolutionTransactionBuilderV1Suite extends DeltaTableSchemaEvolutionSuiteBase with WriteUtils {} class DeltaTableSchemaEvolutionTransactionBuilderV2Suite extends DeltaTableSchemaEvolutionSuiteBase with WriteUtilsWithV2Builders {} /** * ToDo: Clean this up by moving some common schemas to fixtures and abstracting * the setup/run schema evolution/assert loop */ trait DeltaTableSchemaEvolutionSuiteBase extends AnyFunSuite with AbstractWriteUtils with ColumnMappingSuiteBase { val utf8Lcase = CollationIdentifier.fromString("SPARK.UTF8_LCASE") test("Add nullable column succeeds and correctly updates maxFieldId") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", StringType.STRING, true) .add("c", IntegerType.INTEGER, true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema() val newSchema = new StructType() .add("a", StringType.STRING, true, currentSchema.get("a").getMetadata) .add( "b", new StructType() .add("d", IntegerType.INTEGER, true, fieldMetadataForColumn(4, "d")) .add("e", IntegerType.INTEGER, true, fieldMetadataForColumn(5, "e")), true, fieldMetadataForColumn(3, "b")) .add("c", IntegerType.INTEGER, true, currentSchema.get("c").getMetadata) .add("f", new StringType(utf8Lcase), true, fieldMetadataForColumn(6, "f")) updateTableMetadata(engine, tablePath, newSchema) val structType = table.getLatestSnapshot(engine).getSchema assertColumnMapping(structType.get("a"), 1) assertColumnMapping(structType.get("f"), 6, "f") val updatedFType = structType.get("f").getDataType.asInstanceOf[StringType] assert(updatedFType.getCollationIdentifier == utf8Lcase) val innerStruct = structType.get("b").getDataType.asInstanceOf[StructType] assertColumnMapping(innerStruct.get("d"), 4, "d") assertColumnMapping(innerStruct.get("e"), 5, "e") assertColumnMapping(structType.get("c"), 2) assert(getMaxFieldId(engine, tablePath) == 6) } } test("Change collation of existing STRING field succeeds") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", StringType.STRING, true) .add("c", IntegerType.INTEGER, true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val newSchema = new StructType() .add("a", new StringType(utf8Lcase), true, currentSchema.get("a").getMetadata) .add("c", IntegerType.INTEGER, true, currentSchema.get("c").getMetadata) updateTableMetadata(engine, tablePath, newSchema) val updatedSchema = table.getLatestSnapshot(engine).getSchema val updatedAType = updatedSchema.get("a").getDataType.asInstanceOf[StringType] assert(updatedAType.getCollationIdentifier == utf8Lcase) // Ensure no type changes recorded for a pure collation change assert(updatedSchema.get("a").getTypeChanges.isEmpty) } } test("Add new STRING column with non-default collation succeeds") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", StringType.STRING, true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val newSchema = new StructType() .add("a", StringType.STRING, true, currentSchema.get("a").getMetadata) .add("b", new StringType(utf8Lcase), true, fieldMetadataForColumn(3, "b")) updateTableMetadata(engine, tablePath, newSchema) val updatedSchema = table.getLatestSnapshot(engine).getSchema assertColumnMapping(updatedSchema.get("a"), 1) assertColumnMapping(updatedSchema.get("b"), 3, "b") val updatedBType = updatedSchema.get("b").getDataType.asInstanceOf[StringType] assert(updatedBType.getCollationIdentifier == utf8Lcase) } } test("Change nested STRING collation inside struct succeeds") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add( "b", new StructType() .add("d", StringType.STRING, true, fieldMetadataForColumn(3, "d")), true, fieldMetadataForColumn(2, "b")) .add("a", StringType.STRING, true, fieldMetadataForColumn(1, "a")) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val innerStruct = currentSchema.get("b").getDataType.asInstanceOf[StructType] val newSchema = new StructType() .add("a", StringType.STRING, true, currentSchema.get("a").getMetadata) .add( "b", new StructType() .add("d", new StringType(utf8Lcase), true, innerStruct.get("d").getMetadata), true, currentSchema.get("b").getMetadata) updateTableMetadata(engine, tablePath, newSchema) val updatedSchema = table.getLatestSnapshot(engine).getSchema val updatedInnerStruct = updatedSchema.get("b").getDataType.asInstanceOf[StructType] val updatedDType = updatedInnerStruct.get("d").getDataType.asInstanceOf[StringType] assert(updatedDType.getCollationIdentifier == utf8Lcase) // Ensure IDs/physical names preserved assertColumnMapping(updatedInnerStruct.get("d"), 3, "d") } } test("Change collation of STRING partition column succeeds") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("partition1", StringType.STRING, true) .add("data", StringType.STRING, true) createEmptyTable( engine, tablePath, initialSchema, partCols = Seq("partition1"), tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val newSchema = new StructType() .add( "partition1", new StringType(utf8Lcase), true, currentSchema.get("partition1").getMetadata) .add("data", StringType.STRING, true, currentSchema.get("data").getMetadata) updateTableMetadata(engine, tablePath, newSchema) val updatedSchema = table.getLatestSnapshot(engine).getSchema val updatedPartitionType = updatedSchema.get("partition1").getDataType.asInstanceOf[StringType] assert(updatedPartitionType.getCollationIdentifier == utf8Lcase) // Verify ordering preserved and no unintended changes val topLevelFields = updatedSchema.fieldNames().asScala assert(topLevelFields == Array("partition1", "data").toSeq) } } test("Drop column succeeds") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", StringType.STRING, true) .add("c", IntegerType.INTEGER, true) .add("d", new StringType(utf8Lcase), true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) assertColumnMapping(table.getLatestSnapshot(engine).getSchema.get("c"), 2) assertColumnMapping(table.getLatestSnapshot(engine).getSchema.get("d"), 3) val currentSchema = table.getLatestSnapshot(engine).getSchema() val newSchema = new StructType() .add("a", StringType.STRING, true, currentSchema.get("a").getMetadata) updateTableMetadata(engine, tablePath, newSchema) val structType = table.getLatestSnapshot(engine).getSchema assertColumnMapping(structType.get("a"), 1) assert(getMaxFieldId(engine, tablePath) == 3) } } test("Rename fields") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", StringType.STRING, true) .add( "b", new StructType() .add("d", IntegerType.INTEGER, true) .add("e", IntegerType.INTEGER, true), true) .add("c", IntegerType.INTEGER, true) .add("s", new StringType(utf8Lcase), true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema() val innerStruct = currentSchema.get("b").getDataType.asInstanceOf[StructType] val newSchema = new StructType() .add("a", StringType.STRING, true, currentSchema.get("a").getMetadata) .add( "b", new StructType() .add("renamed-d", IntegerType.INTEGER, true, innerStruct.get("d").getMetadata) .add("e", IntegerType.INTEGER, true, innerStruct.get("e").getMetadata), true, currentSchema.get("b").getMetadata) .add("renamed-c", IntegerType.INTEGER, true, currentSchema.get("c").getMetadata) .add("renamed-s", new StringType(utf8Lcase), true, currentSchema.get("s").getMetadata) updateTableMetadata(engine, tablePath, newSchema) val updatedSchema = table.getLatestSnapshot(engine).getSchema assertColumnMapping(updatedSchema.get("a"), 1) val updatedInnerStruct = updatedSchema.get("b").getDataType.asInstanceOf[StructType] assertColumnMapping(updatedInnerStruct.get("renamed-d"), 3) assertColumnMapping(updatedInnerStruct.get("e"), 4) assertColumnMapping(updatedSchema.get("renamed-c"), 5) assertColumnMapping(updatedSchema.get("renamed-s"), 6) val renamedSType = updatedSchema.get("renamed-s").getDataType.asInstanceOf[StringType] assert(renamedSType.getCollationIdentifier == utf8Lcase) } } test("Move fields") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", StringType.STRING, true) .add( "b", new StructType() .add("d", IntegerType.INTEGER, true) .add("e", IntegerType.INTEGER, true) .add("s", new StringType(utf8Lcase), true), true) .add("c", IntegerType.INTEGER, true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema() val innerStruct = currentSchema.get("b").getDataType.asInstanceOf[StructType] val newSchema = new StructType() .add("a", StringType.STRING, true, currentSchema.get("a").getMetadata) .add("c", IntegerType.INTEGER, true, currentSchema.get("c").getMetadata) .add( "b", new StructType() .add("s", new StringType(utf8Lcase), true, innerStruct.get("s").getMetadata) .add("e", IntegerType.INTEGER, true, innerStruct.get("e").getMetadata) .add("d", IntegerType.INTEGER, true, innerStruct.get("d").getMetadata), true, currentSchema.get("b").getMetadata) updateTableMetadata(engine, tablePath, newSchema) val updatedSchema = table.getLatestSnapshot(engine).getSchema assertColumnMapping(updatedSchema.get("a"), 1) val updatedInnerStruct = updatedSchema.get("b").getDataType.asInstanceOf[StructType] assertColumnMapping(updatedInnerStruct.get("d"), 3) assertColumnMapping(updatedInnerStruct.get("e"), 4) assertColumnMapping(updatedInnerStruct.get("s"), 5) val nestedSType = updatedInnerStruct.get("s").getDataType.asInstanceOf[StringType] assert(nestedSType.getCollationIdentifier == utf8Lcase) assertColumnMapping(updatedSchema.get("c"), 6) // Verify the top level and nested field reordering is maintained val topLevelFields = updatedSchema.fieldNames().asScala assert(topLevelFields == Array("a", "c", "b").toSeq) val innerFields = updatedInnerStruct.fieldNames().asScala assert(innerFields == Array("s", "e", "d").toSeq) } } test("Updating schema with adding an array and map type") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", IntegerType.INTEGER, true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val newSchema = new StructType() .add("a", IntegerType.INTEGER, true, currentSchema.get("a").getMetadata) .add( "arr", new ArrayType(StringType.STRING, false), true, fieldMetadataForArrayColumn(2, "arr", "arr", 3)) .add( "map", new MapType(StringType.STRING, StringType.STRING, false), true, fieldMetadataForMapColumn(4, "map", "map", 5, 6)) updateTableMetadata(engine, tablePath, newSchema) val structType = table.getLatestSnapshot(engine).getSchema assertColumnMapping(structType.get("a"), 1) assertColumnMapping(structType.get("arr"), 2, "arr") assertColumnMapping(structType.get("map"), 4, "map") assert(structType.get("arr").getMetadata.get(ColumnMapping.COLUMN_MAPPING_NESTED_IDS_KEY) == FieldMetadata.builder().putLong("arr.element", 3).build()) assert(structType.get("map").getMetadata.get(ColumnMapping.COLUMN_MAPPING_NESTED_IDS_KEY) == FieldMetadata.builder().putLong("map.key", 5).putLong("map.value", 6).build()) } } test("Updating schema with adding an array and map type with collated strings") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", IntegerType.INTEGER, true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val newSchema = new StructType() .add("a", IntegerType.INTEGER, true, currentSchema.get("a").getMetadata) .add( "arr", new ArrayType(new StringType(utf8Lcase), false), true, fieldMetadataForArrayColumn(2, "arr", "arr", 3)) .add( "map", new MapType(StringType.STRING, new StringType(utf8Lcase), false), true, fieldMetadataForMapColumn(4, "map", "map", 5, 6)) updateTableMetadata(engine, tablePath, newSchema) val structType = table.getLatestSnapshot(engine).getSchema assertColumnMapping(structType.get("a"), 1) assertColumnMapping(structType.get("arr"), 2, "arr") assertColumnMapping(structType.get("map"), 4, "map") assert(structType.get("arr").getMetadata.get(ColumnMapping.COLUMN_MAPPING_NESTED_IDS_KEY) == FieldMetadata.builder().putLong("arr.element", 3).build()) assert(structType.get("map").getMetadata.get(ColumnMapping.COLUMN_MAPPING_NESTED_IDS_KEY) == FieldMetadata.builder().putLong("map.key", 5).putLong("map.value", 6).build()) } } test("Add map whose values are array of struct") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", IntegerType.INTEGER, true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val newSchema = new StructType() .add("a", IntegerType.INTEGER, true, currentSchema.get("a").getMetadata) .add( "map", new MapType( StringType.STRING, new ArrayType( new StructType().add( "nested_map_value", IntegerType.INTEGER, fieldMetadataForColumn(3, "some-physical-column")), true), false), true, fieldMetadataForMapColumn(4, "map", "map", 5, 6)) updateTableMetadata(engine, tablePath, newSchema) val latestSnapshot = table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl] val structType = latestSnapshot.getSchema assertColumnMapping(structType.get("a"), 1) assertColumnMapping(structType.get("map"), 4, "map") assert(structType.get("map").getMetadata.get(ColumnMapping.COLUMN_MAPPING_NESTED_IDS_KEY) == FieldMetadata.builder() .putLong("map.key", 5) .putLong("map.value", 6) .putLong("map.value.element", 7) .build()) val configuration = latestSnapshot.getMetadata.getConfiguration assert(configuration.get(ColumnMapping.COLUMN_MAPPING_MAX_COLUMN_ID_KEY) == "7") } } test("Drop nested struct field in map>") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", IntegerType.INTEGER, true) .add( "map", new MapType( StringType.STRING, new ArrayType( new StructType().add("field", IntegerType.INTEGER) .add("field_to_drop", IntegerType.INTEGER), true), false), true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val mapSchema = currentSchema.get("map").getDataType.asInstanceOf[MapType] val arrayValue = mapSchema.getValueType.asInstanceOf[ArrayType] val innerStruct = arrayValue.getElementType.asInstanceOf[StructType] val newSchema = new StructType() .add("a", IntegerType.INTEGER, true, currentSchema.get("a").getMetadata) .add( "map", new MapType( StringType.STRING, new ArrayType( new StructType() .add("field", IntegerType.INTEGER, innerStruct.get("field").getMetadata), true), false), true, currentSchema.get("map").getMetadata) updateTableMetadata(engine, tablePath, newSchema) val structType = table.getLatestSnapshot(engine).getSchema assertColumnMapping(structType.get("a"), 1) assertColumnMapping(structType.get("map"), 2) } } test("Add nested struct field to map>") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", IntegerType.INTEGER, true) .add( "map", new MapType( StringType.STRING, new ArrayType( new StructType().add("field", IntegerType.INTEGER), true), false), true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val mapSchema = currentSchema.get("map").getDataType.asInstanceOf[MapType] val arrayValue = mapSchema.getValueType.asInstanceOf[ArrayType] val innerStruct = arrayValue.getElementType.asInstanceOf[StructType] val newSchema = new StructType() .add("a", IntegerType.INTEGER, true, currentSchema.get("a").getMetadata) .add( "map", new MapType( StringType.STRING, new ArrayType( new StructType() .add("field", IntegerType.INTEGER, innerStruct.get("field").getMetadata) .add( "field_to_add", IntegerType.INTEGER, fieldMetadataForColumn(7, "field_to_add")), true), false), true, currentSchema.get("map").getMetadata) updateTableMetadata(engine, tablePath, newSchema) val structType = table.getLatestSnapshot(engine).getSchema assertColumnMapping(structType.get("a"), 1) assertColumnMapping(structType.get("map"), 2) val mapType = structType.get("map").getDataType.asInstanceOf[MapType] val updatedArrayValue = mapType.getValueField.getDataType.asInstanceOf[ArrayType] val updatedInnerStruct = updatedArrayValue.getElementType.asInstanceOf[StructType] assertColumnMapping(updatedInnerStruct.get("field_to_add"), 7, "field_to_add") } } test("Renaming clustering columns") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("clustering-col", StringType.STRING, true) createEmptyTable( engine, tablePath, initialSchema, clusteringColsOpt = Some(List(new Column("clustering-col"))), tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val expectedSchema = new StructType() .add( "renamed-clustering-col", StringType.STRING, true, currentSchema.get("clustering-col").getMetadata) updateTableMetadata(engine, tablePath, expectedSchema) val snapshot = table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl] val actualSchema = snapshot.getSchema assert(expectedSchema == actualSchema) } } test("Renaming collated clustering columns") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("clustering-col", new StringType(utf8Lcase), true) createEmptyTable( engine, tablePath, initialSchema, clusteringColsOpt = Some(List(new Column("clustering-col"))), tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val expectedSchema = new StructType() .add( "renamed-clustering-col", new StringType(utf8Lcase), true, currentSchema.get("clustering-col").getMetadata) updateTableMetadata(engine, tablePath, expectedSchema) val snapshot = table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl] val actualSchema = snapshot.getSchema assert(expectedSchema == actualSchema) val renamedType = actualSchema.get("renamed-clustering-col").getDataType.asInstanceOf[StringType] assert(renamedType.getCollationIdentifier == utf8Lcase) } } test("Add nested array field to map with already assigned IDs") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", IntegerType.INTEGER, true) .add( "map", new MapType( StringType.STRING, new StructType().add("field", IntegerType.INTEGER), false), true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val mapSchema = currentSchema.get("map").getDataType.asInstanceOf[MapType] val innerStruct = mapSchema.getValueType.asInstanceOf[StructType] val newSchema = new StructType() .add("a", IntegerType.INTEGER, true, currentSchema.get("a").getMetadata) .add( "map", new MapType( StringType.STRING, new StructType() .add("field", IntegerType.INTEGER, innerStruct.get("field").getMetadata) .add( "array_field_to_add", new ArrayType(IntegerType.INTEGER, true), FieldMetadata.builder() .putFieldMetadata( ColumnMapping.COLUMN_MAPPING_NESTED_IDS_KEY, FieldMetadata.builder().putLong("array_field_to_add", 7).build()) .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, 6) .putString(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "array_field_to_add") .build()), false), true, currentSchema.get("map").getMetadata) updateTableMetadata(engine, tablePath, newSchema) val structType = table.getLatestSnapshot(engine).getSchema assertColumnMapping(structType.get("a"), 1) assertColumnMapping(structType.get("map"), 2) val mapType = structType.get("map").getDataType.asInstanceOf[MapType] val updatedInnerStruct = mapType.getValueType.asInstanceOf[StructType] assertColumnMapping(updatedInnerStruct.get("array_field_to_add"), 6, "array_field_to_add") } } test("Add nested array field to map with fresh IDs") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", IntegerType.INTEGER, true) .add( "map", new MapType( StringType.STRING, new StructType().add("field", IntegerType.INTEGER), false), true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val mapSchema = currentSchema.get("map").getDataType.asInstanceOf[MapType] val innerStruct = mapSchema.getValueType.asInstanceOf[StructType] val newSchema = new StructType() .add( "map", new MapType( StringType.STRING, new StructType() .add( "array_field_to_add", new ArrayType(IntegerType.INTEGER, true)) .add("field", IntegerType.INTEGER, innerStruct.get("field").getMetadata), false), true, currentSchema.get("map").getMetadata) .add("a", IntegerType.INTEGER, true, currentSchema.get("a").getMetadata) updateTableMetadata(engine, tablePath, newSchema) val structType = table.getLatestSnapshot(engine).getSchema assertColumnMapping(structType.get("a"), 1) assertColumnMapping(structType.get("map"), 2) val mapType = structType.get("map").getDataType.asInstanceOf[MapType] val updatedInnerStruct = mapType.getValueType.asInstanceOf[StructType] assertColumnMapping(updatedInnerStruct.get("array_field_to_add"), 6, "array_field_to_add") } } test("Drop nested array field in map") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", IntegerType.INTEGER, true) .add( "map", new MapType( StringType.STRING, new StructType().add("field", IntegerType.INTEGER) .add("array_field_to_drop", new ArrayType(IntegerType.INTEGER, true)), false), true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val mapSchema = currentSchema.get("map").getDataType.asInstanceOf[MapType] val innerStruct = mapSchema.getValueType.asInstanceOf[StructType] val newSchema = new StructType() .add("a", IntegerType.INTEGER, true, currentSchema.get("a").getMetadata) .add( "map", new MapType( StringType.STRING, new StructType() .add("field", IntegerType.INTEGER, innerStruct.get("field").getMetadata), false), true, currentSchema.get("map").getMetadata) updateTableMetadata(engine, tablePath, newSchema) val structType = table.getLatestSnapshot(engine).getSchema assertColumnMapping(structType.get("a"), 1) assertColumnMapping(structType.get("map"), 2) val mapType = structType.get("map").getDataType.asInstanceOf[MapType] val updatedInnerStruct = mapType.getValueType.asInstanceOf[StructType] assert(updatedInnerStruct == innerStruct.withoutField("array_field_to_drop")) } } test("Rename nested array field in map") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", IntegerType.INTEGER, true) .add( "map", new MapType( StringType.STRING, new StructType().add("field", IntegerType.INTEGER) .add("array_field_to_rename", new ArrayType(IntegerType.INTEGER, true)), false), true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val mapSchema = currentSchema.get("map").getDataType.asInstanceOf[MapType] val innerStruct = mapSchema.getValueType.asInstanceOf[StructType] val newSchema = new StructType() .add("a", IntegerType.INTEGER, true, currentSchema.get("a").getMetadata) .add( "map", new MapType( StringType.STRING, new StructType() .add("field", IntegerType.INTEGER, innerStruct.get("field").getMetadata) .add( "renamed_array_field", new ArrayType(IntegerType.INTEGER, true), innerStruct.get("array_field_to_rename").getMetadata), false), true, currentSchema.get("map").getMetadata) updateTableMetadata(engine, tablePath, newSchema) val structType = table.getLatestSnapshot(engine).getSchema assertColumnMapping(structType.get("a"), 1) assertColumnMapping(structType.get("map"), 2) val mapType = structType.get("map").getDataType.asInstanceOf[MapType] val updatedInnerStruct = mapType.getValueType.asInstanceOf[StructType] assert(updatedInnerStruct.get("renamed_array_field").getDataType == innerStruct.get("array_field_to_rename").getDataType) assert(updatedInnerStruct.get("renamed_array_field").getMetadata == innerStruct.get("array_field_to_rename").getMetadata) } } test("Adding struct of structs") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", StringType.STRING, true) .add("c", IntegerType.INTEGER, true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "name", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema() val newSchema = new StructType() .add("a", StringType.STRING, true, currentSchema.get("a").getMetadata) .add( "b", new StructType() .add( "d", new StructType().add("e", IntegerType.INTEGER, fieldMetadataForColumn(5, "e")), true, fieldMetadataForColumn(4, "d")), true, fieldMetadataForColumn(3, "b")) .add("c", IntegerType.INTEGER, true, currentSchema.get("c").getMetadata) updateTableMetadata(engine, tablePath, newSchema) val structType = table.getLatestSnapshot(engine).getSchema assertColumnMapping(structType.get("a"), 1) val firstInnerStruct = structType.get("b").getDataType.asInstanceOf[StructType] assertColumnMapping(firstInnerStruct.get("d"), 4, "d") val secondInnerStruct = firstInnerStruct.get("d").getDataType.asInstanceOf[StructType] assertColumnMapping(secondInnerStruct.get("e"), 5, "e") } } test("Add array of arrays") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", StringType.STRING, true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema() val newSchema = new StructType() .add("a", StringType.STRING, true, currentSchema.get("a").getMetadata) .add( "array_of_arrays", new ArrayType(new ArrayType(IntegerType.INTEGER, true), true), true, FieldMetadata.builder() .putString(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "array_of_arrays") .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, 4L) .putFieldMetadata( ColumnMapping.COLUMN_MAPPING_NESTED_IDS_KEY, FieldMetadata.builder().putLong("array_of_arrays.element", 2L) .putLong("array_of_arrays.element.element", 3L).build()).build()) updateTableMetadata(engine, tablePath, newSchema) val updatedSchema = table.getLatestSnapshot(engine).getSchema() assertColumnMapping(updatedSchema.get("a"), 1) assertColumnMapping(updatedSchema.get("array_of_arrays"), 4L, "array_of_arrays") val arrayMetadata = updatedSchema.get("array_of_arrays").getMetadata assert(arrayMetadata.getMetadata(ColumnMapping.COLUMN_MAPPING_NESTED_IDS_KEY) .getLong("array_of_arrays.element") == 2L) assert(arrayMetadata.getMetadata(ColumnMapping.COLUMN_MAPPING_NESTED_IDS_KEY) .getLong("array_of_arrays.element.element") == 3L) } } test("Changing column mapping on table and evolve schema at same time fails") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", StringType.STRING, true) .add("c", IntegerType.INTEGER, true) // Create a table initially without column mapping createEmptyTable(engine, tablePath, initialSchema) val currentSchema = table.getLatestSnapshot(engine).getSchema() val newSchema = new StructType() .add("a", StringType.STRING, true, currentSchema.get("a").getMetadata) .add( "b", new StructType() .add("d", IntegerType.INTEGER, true, fieldMetadataForColumn(4, "d")) .add("e", IntegerType.INTEGER, true, fieldMetadataForColumn(5, "e")), true, fieldMetadataForColumn(3, "b")) .add("c", IntegerType.INTEGER, true, currentSchema.get("c").getMetadata) // Add a new collated STRING field .add("s", new StringType(utf8Lcase), true, fieldMetadataForColumn(6, "s")) assertSchemaEvolutionFails[IllegalArgumentException]( table, engine, newSchema, "Cannot update mapping mode and perform schema evolution", Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "name", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) } } test("Updating schema on table when column mapping disabled fails") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", StringType.STRING, true) .add("c", IntegerType.INTEGER, true) .add("s", new StringType(utf8Lcase), true) createEmptyTable(engine, tablePath, initialSchema, tableProperties = Map.empty) val currentSchema = table.getLatestSnapshot(engine).getSchema val newSchema = new StructType() .add("a", StringType.STRING, true, currentSchema.get("a").getMetadata) .add( "b", new StructType() .add("d", IntegerType.INTEGER, true, fieldMetadataForColumn(4, "d")), true, fieldMetadataForColumn(3, "b")) .add("c", IntegerType.INTEGER, true, currentSchema.get("c").getMetadata) assertSchemaEvolutionFails[KernelException]( table, engine, newSchema, "Cannot update schema for table when column mapping is disabled") } } test("Move partition columns") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("partition1", StringType.STRING, true) .add("partition2", IntegerType.INTEGER, true) .add("partition3", new StringType(utf8Lcase), true) .add("data", StringType.STRING, true) createEmptyTable( engine, tablePath, initialSchema, partCols = Seq("partition1", "partition2", "partition3"), tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "name", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val newSchema = new StructType() .add("partition2", IntegerType.INTEGER, true, currentSchema.get("partition2").getMetadata) .add( "partition3", new StringType(utf8Lcase), true, currentSchema.get("partition3").getMetadata) .add("partition1", StringType.STRING, true, currentSchema.get("partition1").getMetadata) .add("data", StringType.STRING, true, currentSchema.get("data").getMetadata) updateTableMetadata(engine, tablePath, newSchema) val updatedSchema = table.getLatestSnapshot(engine).getSchema // Verify the ordering is expected val topLevelFields = updatedSchema.fieldNames().asScala assert(topLevelFields == Array("partition2", "partition3", "partition1", "data").toSeq) val p3Type = updatedSchema.get("partition3").getDataType.asInstanceOf[StringType] assert(p3Type.getCollationIdentifier == utf8Lcase) } } test("Updating schema with duplicate field IDs fails") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", StringType.STRING, true) .add("c", IntegerType.INTEGER, true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val newSchema = new StructType() .add("a", StringType.STRING, true, currentSchema.get("a").getMetadata) .add( "b", new StructType() .add("duplicate_field_id", IntegerType.INTEGER, true, fieldMetadataForColumn(1, "d")) .add("e", IntegerType.INTEGER, true, fieldMetadataForColumn(5, "e")), true, fieldMetadataForColumn(3, "b")) .add("c", IntegerType.INTEGER, true, currentSchema.get("c").getMetadata) assertSchemaEvolutionFails[IllegalArgumentException]( table, engine, newSchema, "Field duplicate_field_id with id 1 already exists") } } test("Adding non-nullable field fails") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", StringType.STRING, true) .add("c", IntegerType.INTEGER, true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val newSchema = new StructType() .add("a", StringType.STRING, true, currentSchema.get("a").getMetadata) .add( "b", new StructType() .add("non_nullable_field", IntegerType.INTEGER, false, fieldMetadataForColumn(4, "d")) .add("e", IntegerType.INTEGER, true, fieldMetadataForColumn(5, "e")), true, fieldMetadataForColumn(3, "b")) .add("s", new StringType(utf8Lcase), true, fieldMetadataForColumn(6, "s")) .add("c", IntegerType.INTEGER, true, currentSchema.get("c").getMetadata) assertSchemaEvolutionFails[KernelException]( table, engine, newSchema, "Cannot add non-nullable field non_nullable_field") } } test("Adding non-nullable field to map value which is a struct fails") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", StringType.STRING, true) .add( "map", new MapType( StringType.STRING, new ArrayType( new StructType().add("nested_map_value", IntegerType.INTEGER), true), false)) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "name", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val mapSchema = currentSchema.get("map").getDataType.asInstanceOf[MapType] val arrayValue = mapSchema.getValueType.asInstanceOf[ArrayType] val innerStruct = arrayValue.getElementType.asInstanceOf[StructType] val newSchema = new StructType() .add("a", StringType.STRING, true, currentSchema.get("a").getMetadata) .add( "map", new MapType( StringType.STRING, new ArrayType( new StructType().add( "nested_map_value", IntegerType.INTEGER, innerStruct.get("nested_map_value").getMetadata) .add( "new_required_field", IntegerType.INTEGER, false, fieldMetadataForColumn(7, "7")), true), false), true, fieldMetadataForMapColumn( 2, ColumnMapping.getPhysicalName(currentSchema.get("map")), "map", 4, 5)) assertSchemaEvolutionFails[KernelException]( table, engine, newSchema, "Cannot add non-nullable field new_required_field") } } test("Cannot drop a partition column") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", StringType.STRING, true) .add("c", IntegerType.INTEGER, true) createEmptyTable( engine, tablePath, initialSchema, partCols = Seq("c"), tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val newSchema = new StructType() .add("a", StringType.STRING, true, currentSchema.get("a").getMetadata) .add( "b", new StructType() .add("d", IntegerType.INTEGER, true, fieldMetadataForColumn(4, "d")) .add("e", IntegerType.INTEGER, true, fieldMetadataForColumn(5, "e")), true, fieldMetadataForColumn(3, "b")) assertSchemaEvolutionFails[IllegalArgumentException]( table, engine, newSchema, "Partition column c not found in the schema") } } test("Cannot drop a collated partition column") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", StringType.STRING, true) .add("p", new StringType(utf8Lcase), true) createEmptyTable( engine, tablePath, initialSchema, partCols = Seq("p"), tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val newSchema = new StructType() .add("a", StringType.STRING, true, currentSchema.get("a").getMetadata) .add( "b", new StructType() .add("d", IntegerType.INTEGER, true, fieldMetadataForColumn(4, "d")) .add("e", IntegerType.INTEGER, true, fieldMetadataForColumn(5, "e")), true, fieldMetadataForColumn(3, "b")) assertSchemaEvolutionFails[IllegalArgumentException]( table, engine, newSchema, "Partition column p not found in the schema") } } test("Cannot rename a partition column") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", StringType.STRING, true) .add("c", IntegerType.INTEGER, true) createEmptyTable( engine, tablePath, initialSchema, partCols = Seq("c"), tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val newSchema = new StructType() .add("a", StringType.STRING, true, currentSchema.get("a").getMetadata) .add("e", IntegerType.INTEGER, true, currentSchema.get("c").getMetadata) assertSchemaEvolutionFails[IllegalArgumentException]( table, engine, newSchema, "Partition column c not found in the schema") } } test("Cannot rename a collated partition column") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", StringType.STRING, true) .add("p", new StringType(utf8Lcase), true) createEmptyTable( engine, tablePath, initialSchema, partCols = Seq("p"), tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val newSchema = new StructType() .add( "a", StringType.STRING, true, currentSchema.get("a").getMetadata ) // currentSchema.get("p").getMetadata .add("q", new StringType(utf8Lcase), true, currentSchema.get("p").getMetadata) assertSchemaEvolutionFails[IllegalArgumentException]( table, engine, newSchema, "Partition column p not found in the schema") } } test("Cannot change types") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", StringType.STRING, true) .add("c", IntegerType.INTEGER, true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val newSchema = new StructType() .add("a", StringType.STRING, true, currentSchema.get("a").getMetadata) .add("c", LongType.LONG, true, currentSchema.get("c").getMetadata) assertSchemaEvolutionFails[IllegalArgumentException]( table, engine, newSchema, "Cannot change the type of existing field c from integer to long") } } test("Cannot change clustering column type") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("clustering_col", StringType.STRING, true) createEmptyTable( engine, tablePath, initialSchema, clusteringColsOpt = Some(List(new Column("clustering_col"))), tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val newSchema = new StructType() .add("clustering_col", LongType.LONG, true, currentSchema.get("clustering_col").getMetadata) assertSchemaEvolutionFails[IllegalArgumentException]( table, engine, newSchema, "Cannot change the type of existing field clustering_col from string to long") } } test("Updating schema if physical columns are not preserved fails") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", StringType.STRING, true) .add("c", IntegerType.INTEGER, true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val newSchema = new StructType() .add( "a", StringType.STRING, true, fieldMetadataForColumn(1, "not-preserving-physical-column")) .add("c", LongType.LONG, true, currentSchema.get("c").getMetadata) assertSchemaEvolutionFails[IllegalArgumentException]( table, engine, newSchema, "Existing field with id 1 in current schema has physical name") } } test("Updating schema and tightening nullability on existing field fails") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", IntegerType.INTEGER, true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val newSchema = new StructType() .add("renamed_a", IntegerType.INTEGER, false, currentSchema.get("a").getMetadata) assertSchemaEvolutionFails[IllegalArgumentException]( table, engine, newSchema, "Cannot tighten the nullability of existing field renamed_a") } } test("Cannot tighten nullability on renamed array element") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", IntegerType.INTEGER, true) .add( "arr", new ArrayType(StringType.STRING, true)) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val newSchema = new StructType() .add("a", IntegerType.INTEGER, true, currentSchema.get("a").getMetadata) .add( "some_renamed_array", new ArrayType(StringType.STRING, false), currentSchema.get("arr").getMetadata) assertSchemaEvolutionFails[IllegalArgumentException]( table, engine, newSchema, "Cannot tighten the nullability of existing field") } } test("Cannot change a partition column type") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", StringType.STRING, true) .add("c", IntegerType.INTEGER, true) createEmptyTable( engine, tablePath, initialSchema, partCols = Seq("c"), tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema val newSchema = new StructType() .add("c", StringType.STRING, true, currentSchema.get("c").getMetadata) .add("a", StringType.STRING, true, currentSchema.get("a").getMetadata) assertSchemaEvolutionFails[IllegalArgumentException]( table, engine, newSchema, "Cannot change the type of existing field c from integer to string") } } val primitiveSchemaWithClusteringColumn = new StructType() .add( "clustering_col", IntegerType.INTEGER, fieldMetadataForColumn(1, "clustering_col_physical")) .add("data", IntegerType.INTEGER, fieldMetadataForColumn(2, "data_physical")) val nestedSchemaWithClusteringColumn = new StructType() .add( "struct", new StructType() .add( "clustering_col", IntegerType.INTEGER, fieldMetadataForColumn(1, "clustering_col_physical")) .add("data", IntegerType.INTEGER, fieldMetadataForColumn(2, "data_physical")), true, fieldMetadataForColumn(3, "struct_physical")) private val updatedSchemaWithDroppedClusteringColumn = Tables.Table( ("schemaBefore", "updatedSchemaWithDroppedClusteringColumn", "clusteringColumn"), ( primitiveSchemaWithClusteringColumn, new StructType() .add( "data", IntegerType.INTEGER, true, primitiveSchemaWithClusteringColumn.get("data").getMetadata), new Column("clustering_col")), ( nestedSchemaWithClusteringColumn, new StructType() .add( "struct", new StructType() .add( "data", IntegerType.INTEGER, nestedSchemaWithClusteringColumn.get("struct").getDataType .asInstanceOf[StructType].get("data").getMetadata), true, nestedSchemaWithClusteringColumn.get("struct").getMetadata), new Column(Array("struct", "clustering_col"))), ( nestedSchemaWithClusteringColumn, new StructType().add("id", IntegerType.INTEGER, fieldMetadataForColumn(4, "id")), new Column(Array("struct", "clustering_col")))) test("Cannot drop clustering column") { forAll(updatedSchemaWithDroppedClusteringColumn) { (schemaBefore, schemaAfter, clusteringColumn) => withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) createEmptyTable( engine, tablePath, schemaBefore, clusteringColsOpt = Some(List(clusteringColumn)), tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) assertSchemaEvolutionFails[KernelException]( table, engine, schemaAfter, "Cannot drop clustering column clustering_col") } } } test("Updating schema should use the new clustering columns if passed") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("a", StringType.STRING, true) .add("b", StringType.STRING, true) .add("c", IntegerType.INTEGER, true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true"), clusteringColsOpt = Some(List(new Column("b")))) assertColumnMapping(table.getLatestSnapshot(engine).getSchema.get("c"), 3) val newSchema = new StructType() .add("d", StringType.STRING, true) updateTableMetadata( engine, tablePath, newSchema, clusteringColsOpt = Some(List(new Column("d")))) val snapshot = table.getLatestSnapshot(engine) val structType = snapshot.getSchema assertColumnMapping(structType.get("d"), 4, "d") assert(getMaxFieldId(engine, tablePath) == 4) val physicalName = structType.get("d").getMetadata.get(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY) val expectedDomainMetadata = new DomainMetadata( "delta.clustering", s"""{"clusteringColumns":[["$physicalName"]]}""", false) verifyClusteringDomainMetadata(snapshot.asInstanceOf[SnapshotImpl], expectedDomainMetadata) } } test("Cannot move field from nested struct to top-level") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val nestedSchema = new StructType() .add("nestedCol1", StringType.STRING, fieldMetadataForColumn(1, "col-1")) .add("nestedCol2", StringType.STRING, fieldMetadataForColumn(2, "col-2")) val initialSchema = new StructType() .add("topCol1", nestedSchema, fieldMetadataForColumn(3, "col-3")) .add("topCol2", IntegerType.INTEGER, fieldMetadataForColumn(4, "col-4")) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val newNestedSchema = new StructType() .add("nestedCol1", StringType.STRING, fieldMetadataForColumn(1, "col-1")) val newSchema = new StructType() .add("topCol1", newNestedSchema, fieldMetadataForColumn(3, "col-3")) .add("topCol2", IntegerType.INTEGER, fieldMetadataForColumn(4, "col-4")) .add("nestedCol2", StringType.STRING, fieldMetadataForColumn(2, "col-2")) assertSchemaEvolutionFails[KernelException]( table, engine, newSchema, "Cannot move fields between different levels of nesting") } } test("Cannot move field between sibling structs") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val struct1 = new StructType() .add("field1", StringType.STRING, fieldMetadataForColumn(1, "col-1")) .add("field2", IntegerType.INTEGER, fieldMetadataForColumn(2, "col-2")) val struct2 = new StructType() .add("field3", StringType.STRING, fieldMetadataForColumn(3, "col-3")) val initialSchema = new StructType() .add("struct1", struct1, fieldMetadataForColumn(4, "col-4")) .add("struct2", struct2, fieldMetadataForColumn(5, "col-5")) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val newStruct1 = new StructType() .add("field2", IntegerType.INTEGER, fieldMetadataForColumn(2, "col-2")) val newStruct2 = new StructType() .add("field1", StringType.STRING, fieldMetadataForColumn(1, "col-1")) .add("field3", StringType.STRING, fieldMetadataForColumn(3, "col-3")) val newSchema = new StructType() .add("struct1", newStruct1, fieldMetadataForColumn(4, "col-4")) .add("struct2", newStruct2, fieldMetadataForColumn(5, "col-5")) assertSchemaEvolutionFails[KernelException]( table, engine, newSchema, "Cannot move fields between different levels of nesting") } } test("Cannot move field from array element to top-level") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val arrayElementStruct = new StructType() .add("elemField1", IntegerType.INTEGER, fieldMetadataForColumn(1, "col-1")) .add("elemField2", StringType.STRING, fieldMetadataForColumn(2, "col-2")) val initialSchema = new StructType() .add( "arrayCol", new ArrayType(arrayElementStruct, true), fieldMetadataForColumn(3, "col-3")) .add("topField", IntegerType.INTEGER, fieldMetadataForColumn(4, "col-4")) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val newArrayElementStruct = new StructType() .add("elemField2", StringType.STRING, fieldMetadataForColumn(2, "col-2")) val newSchema = new StructType() .add( "arrayCol", new ArrayType(newArrayElementStruct, true), fieldMetadataForColumn(3, "col-3")) .add("topField", IntegerType.INTEGER, fieldMetadataForColumn(4, "col-4")) .add("elemField1", IntegerType.INTEGER, fieldMetadataForColumn(1, "col-1")) assertSchemaEvolutionFails[KernelException]( table, engine, newSchema, "Cannot move fields between different levels of nesting") } } test("Cannot move field from map value to map key") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val mapKeyStruct = new StructType() .add("keyField1", StringType.STRING, fieldMetadataForColumn(1, "col-1")) val mapValueStruct = new StructType() .add("valueField1", IntegerType.INTEGER, fieldMetadataForColumn(2, "col-2")) .add("valueField2", StringType.STRING, fieldMetadataForColumn(3, "col-3")) val initialSchema = new StructType() .add( "mapCol", new MapType(mapKeyStruct, mapValueStruct, true), fieldMetadataForColumn(4, "col-4")) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val newMapKeyStruct = new StructType() .add("keyField1", StringType.STRING, fieldMetadataForColumn(1, "col-1")) .add("valueField1", IntegerType.INTEGER, fieldMetadataForColumn(2, "col-2")) val newMapValueStruct = new StructType() .add("valueField2", StringType.STRING, fieldMetadataForColumn(3, "col-3")) val newSchema = new StructType() .add( "mapCol", new MapType(newMapKeyStruct, newMapValueStruct, true), fieldMetadataForColumn(4, "col-4")) assertSchemaEvolutionFails[KernelException]( table, engine, newSchema, "Cannot move fields between different levels of nesting") } } private val typeWideningTestCases = Tables.Table( ( "testName", "initialType", "newType", "typeWideningEnabled", "icebergV1Enabled", "shouldSucceed", "errorMessageFragment"), ( "Integer widening (Int -> Long) with type widening enabled", IntegerType.INTEGER, LongType.LONG, /* typeWideningEnabled= */ true, /* icebergV1Enabled= */ false, /* shouldSucceed= */ true, ""), ( "Integer widening (Int -> Long) with type widening disabled", IntegerType.INTEGER, LongType.LONG, /* typeWideningEnabled= */ false, /* icebergV1Enabled= */ false, /* shouldSucceed= */ false, "Cannot change the type of existing field id from integer to long"), ( "Decimal precision and scale increase", new DecimalType(10, 2), new DecimalType(15, 5), /* typeWideningEnabled= */ true, /* icebergV1Enabled= */ false, /* shouldSucceed= */ true, ""), ( "Decimal precision and scale increase with Iceberg V1 compatibility", new DecimalType(10, 2), new DecimalType(15, 5), /* typeWideningEnabled= */ true, /* icebergV1Enabled= */ true, /* shouldSucceed= */ false, "Cannot change the type of existing field id")) forAll(typeWideningTestCases) { ( testName, initialType, newType, typeWideningEnabled, icebergV1Enabled, shouldSucceed, errorMessageFragment) => test(s"Type widening scenarios $testName") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val initialSchema = new StructType() .add("id", initialType, true) .add("data", StringType.STRING, true) createEmptyTable( engine, tablePath, initialSchema, tableProperties = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id", TableConfig.TYPE_WIDENING_ENABLED.getKey -> typeWideningEnabled.toString, TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.getKey -> icebergV1Enabled.toString, TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true")) val currentSchema = table.getLatestSnapshot(engine).getSchema() val newSchema = new StructType() .add("id", newType, true, currentSchema.get("id").getMetadata) .add("data", StringType.STRING, true, currentSchema.get("data").getMetadata) if (shouldSucceed) { // This should succeed because conditions allow type widening updateTableMetadata(engine, tablePath, newSchema) val updatedSchema = table.getLatestSnapshot(engine).getSchema assert(updatedSchema.get("id").getDataType == newType) assert(updatedSchema.get("id").getTypeChanges.asScala == List(new TypeChange(initialType, newType))) // Do an unrelated schema change. And ensure type change and type changes // are still present. updateTableMetadata( engine, tablePath, newSchema.add("newField", StringType.STRING, true)) val lastSchema = table.getLatestSnapshot(engine).getSchema assert(lastSchema.get("id").getDataType == newType) assert(lastSchema.get("id").getTypeChanges.asScala == List(new TypeChange(initialType, newType))) } else { // This should fail because conditions don't allow type widening assertSchemaEvolutionFails[KernelException]( table, engine, newSchema, errorMessageFragment) } } } } def fieldMetadataForColumn( columnId: Long, physicalColumnId: String): FieldMetadata = { FieldMetadata.builder() .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, columnId) .putString(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, physicalColumnId) .build() } def fieldMetadataForArrayColumn( columnId: Long, physicalColumnId: String, arrayFieldName: String, nestedElementId: Long): FieldMetadata = { FieldMetadata.builder() .fromMetadata(fieldMetadataForColumn(columnId, physicalColumnId)) .putFieldMetadata( ColumnMapping.COLUMN_MAPPING_NESTED_IDS_KEY, FieldMetadata.builder().putLong(s"$arrayFieldName.element", nestedElementId).build()) .build() } def fieldMetadataForMapColumn( columnId: Long, physicalColumnId: String, mapFieldName: String, keyId: Long, valueId: Long): FieldMetadata = { FieldMetadata.builder() .fromMetadata(fieldMetadataForColumn(columnId, physicalColumnId)) .putFieldMetadata( ColumnMapping.COLUMN_MAPPING_NESTED_IDS_KEY, FieldMetadata.builder().putLong(s"$mapFieldName.key", keyId) .putLong(s"$mapFieldName.value", valueId).build()) .build() } private def assertSchemaEvolutionFails[T <: Throwable]( table: Table, engine: Engine, newSchema: StructType, expectedMessageContained: String, tableProperties: Map[String, String] = Map.empty): Unit = { val e = intercept[Exception] { updateTableMetadata( engine, table.getPath(engine), newSchema, tableProperties = tableProperties) } assert(e.isInstanceOf[T]) assert(e.getMessage.contains(expectedMessageContained)) } private def getMaxFieldId(engine: Engine, tablePath: String): Long = { TableConfig.COLUMN_MAPPING_MAX_COLUMN_ID .fromMetadata(getMetadata(engine, tablePath)) } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableWriteWithCrcSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import scala.collection.immutable.Seq import scala.language.implicitConversions import io.delta.kernel.{Transaction, TransactionCommitResult} import io.delta.kernel.Snapshot.ChecksumWriteMode import io.delta.kernel.data.Row import io.delta.kernel.defaults.utils.{TestRow, WriteUtils} import io.delta.kernel.engine.Engine import io.delta.kernel.types.StructType import io.delta.kernel.utils.CloseableIterable import org.scalatest.funsuite.AnyFunSuite /** * Trait to mixin into a test suite that extends [[WriteUtils]] to run all the tests * with CRC file written after each commit and verify the written CRC files are valid. * Note, this requires the test suite uses [[commitTransaction]] and [[verifyWrittenContent]]. */ trait WriteUtilsWithCrc extends AnyFunSuite with WriteUtils { override def commitTransaction( txn: Transaction, engine: Engine, dataActions: CloseableIterable[Row]): TransactionCommitResult = { executeCrcSimple(txn.commit(engine, dataActions), engine) } override def verifyWrittenContent( path: String, expSchema: StructType, expData: Seq[TestRow]): Unit = { super.verifyWrittenContent(path, expSchema, expData) verifyChecksum(path, expectEmptyTable = expData.isEmpty) } } /** * Trait to mixin into a test suite that extends [[WriteUtils]] to use post-commit snapshots for * writing CRC files using the simple CRC write method. This ensures that the checksum write mode is * SIMPLE and uses the post-commit snapshot's writeChecksumSimple method. Note, this requires the * test suite uses [[commitTransaction]] and [[verifyWrittenContent]]. */ trait WriteUtilsWithPostCommitSnapshotCrcSimpleWrite extends AnyFunSuite with WriteUtils { override def commitTransaction( txn: Transaction, engine: Engine, dataActions: CloseableIterable[Row]): TransactionCommitResult = { val txnResult = txn.commit(engine, dataActions) val postCommitSnapshot = txnResult .getPostCommitSnapshot .orElseThrow(() => new IllegalStateException("Required post-commit snapshot is missing")) postCommitSnapshot.writeChecksum(engine, ChecksumWriteMode.SIMPLE) txnResult } override def verifyWrittenContent( path: String, expSchema: StructType, expData: Seq[TestRow]): Unit = { super.verifyWrittenContent(path, expSchema, expData) verifyChecksum(path, expectEmptyTable = expData.isEmpty) } } class DeltaTableWriteWithCrcSuite extends DeltaTableWritesSuite with WriteUtilsWithCrc {} class DeltaReplaceTableWithCrcSuite extends DeltaReplaceTableSuite with WriteUtilsWithCrc {} class DeltaTableWriteWithPostCommitSnapshotCrcSimpleSuite extends DeltaTableWritesSuite with WriteUtilsWithPostCommitSnapshotCrcSimpleWrite { // Tests to skip due to known limitation: post-commit snapshots are not yet built after conflicts, // so we cannot write CRC files in those cases. See TransactionImpl.buildPostCommitSnapshotOpt. // We use `lazy` due to ScalaTest's initialization order. lazy val testsToSkip = Set( "create table and configure properties with retries", "insert into table - idempotent writes", "conflicts - concurrent data append (1) after the losing txn has started", "conflicts - concurrent data append (5) after the losing txn has started", "conflicts - concurrent data append (12) after the losing txn has started") override protected def test( testName: String, testTags: org.scalatest.Tag*)( testFun: => Any)(implicit pos: org.scalactic.source.Position): Unit = { if (testsToSkip.contains(testName)) { ignore(testName, testTags: _*)(testFun)(pos) } else { super.test(testName, testTags: _*)(testFun)(pos) } } } class DeltaReplaceTableWithPostCommitSnapshotCrcSimpleSuite extends DeltaReplaceTableSuite with WriteUtilsWithPostCommitSnapshotCrcSimpleWrite {} ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableWritesSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.io.File import java.nio.file.Files import java.util.{Locale, Optional} import scala.collection.immutable.Seq import scala.jdk.CollectionConverters._ import io.delta.golden.GoldenTableUtils.goldenTablePath import io.delta.kernel._ import io.delta.kernel.Operation.{CREATE_TABLE, MANUAL_UPDATE, WRITE} import io.delta.kernel.data.{ColumnarBatch, ColumnVector, FilteredColumnarBatch, Row} import io.delta.kernel.defaults.internal.data.DefaultColumnarBatch import io.delta.kernel.defaults.internal.data.vector.DefaultGenericVector import io.delta.kernel.defaults.internal.data.vector.DefaultStructVector import io.delta.kernel.defaults.internal.parquet.ParquetSuiteBase import io.delta.kernel.defaults.utils.{AbstractWriteUtils, TestRow, WriteUtils} import io.delta.kernel.engine.Engine import io.delta.kernel.exceptions._ import io.delta.kernel.expressions.{Column, Literal} import io.delta.kernel.expressions.Literal._ import io.delta.kernel.internal.{ScanImpl, SnapshotImpl, TableConfig} import io.delta.kernel.internal.checkpoints.CheckpointerSuite.selectSingleElement import io.delta.kernel.internal.table.SnapshotBuilderImpl import io.delta.kernel.internal.types.DataTypeJsonSerDe import io.delta.kernel.internal.util.{Clock, JsonUtils} import io.delta.kernel.internal.util.SchemaUtils.casePreservingPartitionColNames import io.delta.kernel.shaded.com.fasterxml.jackson.databind.node.ObjectNode import io.delta.kernel.transaction.DataLayoutSpec import io.delta.kernel.types._ import io.delta.kernel.types.ByteType.BYTE import io.delta.kernel.types.DateType.DATE import io.delta.kernel.types.DecimalType import io.delta.kernel.types.DoubleType.DOUBLE import io.delta.kernel.types.FloatType.FLOAT import io.delta.kernel.types.IntegerType.INTEGER import io.delta.kernel.types.LongType.LONG import io.delta.kernel.types.ShortType.SHORT import io.delta.kernel.types.StringType.STRING import io.delta.kernel.types.StructType import io.delta.kernel.types.TimestampType.TIMESTAMP import io.delta.kernel.utils.CloseableIterable import io.delta.kernel.utils.CloseableIterable.{emptyIterable, inMemoryIterable} import io.delta.tables.DeltaTable import org.scalatest.funsuite.AnyFunSuite class DeltaTableWritesSuite extends AbstractDeltaTableWritesSuite with WriteUtils /** Transaction commit in this suite IS REQUIRED TO use commitTransaction than .commit */ abstract class AbstractDeltaTableWritesSuite extends AnyFunSuite with AbstractWriteUtils with ParquetSuiteBase { /////////////////////////////////////////////////////////////////////////// // Create table tests /////////////////////////////////////////////////////////////////////////// test("create table - provide no schema - expect failure") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val txnBuilder = table.createTransactionBuilder(engine, testEngineInfo, CREATE_TABLE) val ex = intercept[TableNotFoundException] { txnBuilder.build(engine) } assert(ex.getMessage.contains("Must provide a new schema to write to a new table")) } } test("create table - provide partition columns but no schema - expect failure") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val txnBuilder = table .createTransactionBuilder(engine, testEngineInfo, CREATE_TABLE) .withPartitionColumns(engine, Seq("part1", "part2").asJava) val ex = intercept[TableNotFoundException] { txnBuilder.build(engine) } assert(ex.getMessage.contains("Must provide a new schema to write to a new table")) } } test("create table - table already exists at the location") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val txn = getCreateTxn(engine, tablePath, testSchema) commitTransaction(txn, engine, emptyIterable()) { intercept[TableAlreadyExistsException] { table.createTransactionBuilder(engine, testEngineInfo, CREATE_TABLE) .build(engine) } } // Provide schema { intercept[TableAlreadyExistsException] { table.createTransactionBuilder(engine, testEngineInfo, CREATE_TABLE) .withSchema(engine, testSchema) .build(engine) } } // Provide partition columns { intercept[TableAlreadyExistsException] { table.createTransactionBuilder(engine, testEngineInfo, CREATE_TABLE) .withSchema(engine, testPartitionSchema) .withPartitionColumns(engine, testPartitionColumns.asJava) .build(engine) } } } } test("create table - table is concurrently created before txn commits") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val txn1 = getCreateTxn(engine, tablePath, testSchema) val txn2 = getCreateTxn(engine, tablePath, testSchema) commitTransaction(txn2, engine, emptyIterable()) intercept[ConcurrentWriteException] { commitTransaction(txn1, engine, emptyIterable()) } } } test("cannot provide partition columns for existing table") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val txn = getCreateTxn(engine, tablePath, testSchema) commitTransaction(txn, engine, emptyIterable()) val ex = intercept[TableAlreadyExistsException] { // Use operation != CREATE_TABLE since this fails earlier if the table already exists table.createTransactionBuilder(engine, testEngineInfo, WRITE) .withSchema(engine, testPartitionSchema) .withPartitionColumns(engine, testPartitionColumns.asJava) .build(engine) } assert(ex.getMessage.contains("Table already exists, but provided new partition columns." + " Partition columns can only be set on a new table.")) } } test("create table with metadata columns in the schema - expect failure") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val schemaWithMetadataCol = testSchema.addMetadataColumn("_metadata.row_index", MetadataColumnSpec.ROW_INDEX) val ex = intercept[IllegalArgumentException] { getCreateTxn(engine, tablePath, schemaWithMetadataCol) } assert(ex.getMessage.contains("Table schema cannot contain metadata columns")) } } test("create un-partitioned table") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val txn = getCreateTxn(engine, tablePath, testSchema) assert(txn.getSchema(engine) === testSchema) assert(txn.getPartitionColumns(engine) === Seq.empty.asJava) assert(txn.getReadTableVersion == -1) val txnResult = commitTransaction(txn, engine, emptyIterable()) assert(txnResult.getVersion === 0) assertCheckpointReadiness(txnResult, isReadyForCheckpoint = false) verifyCommitInfo(tablePath = tablePath, version = 0) verifyWrittenContent(tablePath, testSchema, Seq.empty) } } test("create table and set properties") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) val txn1 = getCreateTxn(engine, tablePath, testSchema) commitTransaction(txn1, engine, emptyIterable()) val ver0Snapshot = table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl] assertMetadataProp(ver0Snapshot, TableConfig.CHECKPOINT_INTERVAL, 10) setTablePropAndVerify( engine = engine, tablePath = tablePath, isNewTable = false, key = TableConfig.CHECKPOINT_INTERVAL, value = "2", expectedValue = 2) } } test("create table with properties and they should be retained") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) setTablePropAndVerify( engine = engine, tablePath = tablePath, key = TableConfig.CHECKPOINT_INTERVAL, value = "2", expectedValue = 2) appendData( engine, tablePath, data = Seq(Map.empty[String, Literal] -> dataBatches1)) val ver1Snapshot = table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl] assertMetadataProp(ver1Snapshot, TableConfig.CHECKPOINT_INTERVAL, 2) } } test("create table and configure properties with retries") { withTempDirAndEngine { (tablePath, engine) => // Create table val table = Table.forPath(engine, tablePath) val txn0 = getCreateTxn(engine, tablePath, testSchema) commitTransaction(txn0, engine, emptyIterable()) // Create txn1 with config changes val txn1 = getUpdateTxn( engine, tablePath, tableProperties = Map(TableConfig.CHECKPOINT_INTERVAL.getKey -> "2")) // Create and commit txn2 appendData( engine, tablePath, data = Seq(Map.empty[String, Literal] -> dataBatches1)) val ver1Snapshot = table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl] assertMetadataProp(ver1Snapshot, TableConfig.CHECKPOINT_INTERVAL, 10) // Try to commit txn1 txn1.commit(engine, emptyIterable()) val ver2Snapshot = table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl] assertMetadataProp(ver2Snapshot, TableConfig.CHECKPOINT_INTERVAL, 2) } } test("Setting retries to 0 disables retries") { withTempDirAndEngine { (tablePath, engine) => // Create table val table = Table.forPath(engine, tablePath) val txn0 = getCreateTxn(engine, tablePath, testSchema) commitTransaction(txn0, engine, emptyIterable()) // Create txn1 with config changes val txn1 = getUpdateTxn( engine, tablePath, tableProperties = Map(TableConfig.CHECKPOINT_INTERVAL.getKey -> "2"), maxRetries = 0) // Create and commit txn2 appendData( engine, tablePath, data = Seq(Map.empty[String, Literal] -> dataBatches1)) val ver1Snapshot = table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl] assertMetadataProp(ver1Snapshot, TableConfig.CHECKPOINT_INTERVAL, 10) // Try to commit txn1 but expect failure intercept[MaxCommitRetryLimitReachedException] { txn1.commit(engine, emptyIterable()) } // check that we're still set to 10 val ver2Snapshot = table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl] assertMetadataProp(ver2Snapshot, TableConfig.CHECKPOINT_INTERVAL, 10) } } test("create table and configure the same properties") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) setTablePropAndVerify( engine = engine, tablePath = tablePath, key = TableConfig.CHECKPOINT_INTERVAL, value = "2", expectedValue = 2) assert(getMetadataActionFromCommit(engine, table, 0).isDefined) appendData( engine, tablePath, data = Seq(Map.empty[String, Literal] -> dataBatches1), tableProperties = Map(TableConfig.CHECKPOINT_INTERVAL.getKey.toLowerCase(Locale.ROOT) -> "2")) val ver1Snapshot = table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl] assertMetadataProp(ver1Snapshot, TableConfig.CHECKPOINT_INTERVAL, 2) assert(getMetadataActionFromCommit(engine, table, 1).isEmpty) } } test("create table and configure verifying that the case of the property is same as the one in" + "TableConfig and not the one passed by the user.") { withTempDirAndEngine { (tablePath, engine) => val table = Table.forPath(engine, tablePath) appendData( engine, tablePath, isNewTable = true, testSchema, data = Seq(Map.empty[String, Literal] -> dataBatches1), tableProperties = Map(TableConfig.CHECKPOINT_INTERVAL.getKey.toLowerCase(Locale.ROOT) -> "2")) val ver0Snapshot = table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl] assertMetadataProp(ver0Snapshot, TableConfig.CHECKPOINT_INTERVAL, 2) val configurations = ver0Snapshot.getMetadata.getConfiguration assert(configurations.containsKey(TableConfig.CHECKPOINT_INTERVAL.getKey)) assert( !configurations.containsKey( TableConfig.CHECKPOINT_INTERVAL.getKey.toLowerCase(Locale.ROOT))) } } test("create partitioned table - partition column is not part of the schema") { withTempDirAndEngine { (tablePath, engine) => val ex = intercept[IllegalArgumentException] { getCreateTxn( engine, tablePath, schema = testPartitionSchema, partCols = Seq("PART1", "part3")) } assert(ex.getMessage.contains("Partition column part3 not found in the schema")) } } test("create partitioned table - partition column type is not supported") { withTempDirAndEngine { (tablePath, engine) => val schema = new StructType() .add("p1", new ArrayType(INTEGER, true)) .add("c1", DATE) .add("c2", new DecimalType(14, 2)) val ex = intercept[KernelException] { getCreateTxn(engine, tablePath, schema = schema, partCols = Seq("p1", "c1")) } assert(ex.getMessage.contains( "Kernel doesn't support writing data with partition column (p1) of type: array[integer]")) } } test("create a partitioned table") { withTempDirAndEngine { (tablePath, engine) => val schema = new StructType() .add("id", INTEGER) .add("Part1", INTEGER) // partition column .add("part2", INTEGER) // partition column val txn = getCreateTxn( engine, tablePath, schema = schema, partCols = Seq("part1", "PART2")) assert(txn.getSchema(engine) === schema) // Expect the partition column name is exactly same as the one in the schema assert(txn.getPartitionColumns(engine) === Seq("Part1", "part2").asJava) val txnResult = commitTransaction(txn, engine, emptyIterable()) assert(txnResult.getVersion === 0) assertCheckpointReadiness(txnResult, isReadyForCheckpoint = false) verifyCommitInfo(tablePath, version = 0, Seq("Part1", "part2")) verifyWrittenContent(tablePath, schema, Seq.empty) } } Seq(true, false).foreach { includeTimestampNtz => test(s"create table with all supported types - timestamp_ntz included=$includeTimestampNtz") { withTempDirAndEngine { (tablePath, engine) => val parquetAllTypes = goldenTablePath("parquet-all-types") val goldenTableSchema = tableSchema(parquetAllTypes) val schema = if (includeTimestampNtz) goldenTableSchema else removeTimestampNtzTypeColumns(goldenTableSchema) val txn = getCreateTxn(engine, tablePath, schema = schema) val txnResult = commitTransaction(txn, engine, emptyIterable()) assert(txnResult.getVersion === 0) assertCheckpointReadiness(txnResult, isReadyForCheckpoint = false) verifyCommitInfo(tablePath, version = 0) verifyWrittenContent(tablePath, schema, Seq.empty) } } } /////////////////////////////////////////////////////////////////////////// // Collation write tests /////////////////////////////////////////////////////////////////////////// test("insert into table - simple collated string column") { withTempDirAndEngine { (tblPath, engine) => val utf8Lcase = new StringType("SPARK.UTF8_LCASE") val unicode = new StringType("ICU.UNICODE") val serbianWithVersion = new StringType("ICU.SR_CYRL_SRB.75.1") val serbianWithoutVersion = new StringType("ICU.SR_CYRL_SRB") val commonSchema = new StructType() .add("c1", IntegerType.INTEGER) .add("c2", StringType.STRING) .add("c3", STRING) .add("c4", utf8Lcase) .add("c5", unicode) val schemaWithVersion = commonSchema.add("c6", serbianWithVersion) val schemaWithoutVersion = commonSchema.add("c6", serbianWithoutVersion) // First append val data1 = generateData(schemaWithVersion, Seq.empty, Map.empty, batchSize = 10, numBatches = 1) val commitResult0 = appendData( engine, tblPath, isNewTable = true, schemaWithVersion, data = Seq(Map.empty[String, Literal] -> data1)) verifyCommitResult(commitResult0, expVersion = 0, expIsReadyForCheckpoint = false) verifyCommitInfo(tblPath, version = 0) // we use schemaWithoutVersion to verify since the version info is not stored in the // schema serialization verifyWrittenContent(tblPath, schemaWithoutVersion, data1.flatMap(_.toTestRows)) // Second append val data2 = generateData(schemaWithVersion, Seq.empty, Map.empty, batchSize = 5, numBatches = 1) val commitResult1 = appendData( engine, tblPath, data = Seq(Map.empty[String, Literal] -> data2)) verifyCommitResult(commitResult1, expVersion = 1, expIsReadyForCheckpoint = false) verifyCommitInfo(tblPath, version = 1, partitionCols = null) verifyWrittenContent( tblPath, schemaWithoutVersion, (data1 ++ data2).flatMap(_.toTestRows)) val metadata = getMetadata(engine, tblPath) val parsed = DataTypeJsonSerDe.deserializeStructType(metadata.getSchemaString()) assert(parsed === schemaWithoutVersion) } } test("insert into table - collated string column with nulls") { withTempDirAndEngine { (tblPath, engine) => val unicode = new StringType("ICU.UNICODE.74.1") val schema = new StructType() .add("id", IntegerType.INTEGER) .add("name", unicode) val batchSize = 4 val idValues = Array[java.lang.Integer](1, 2, 3, 4).asInstanceOf[Array[AnyRef]] val nameValues = Array[AnyRef]("Alice", null, "Bob", null) val idVector = DefaultGenericVector.fromArray(IntegerType.INTEGER, idValues) val nameVector = DefaultGenericVector.fromArray(unicode, nameValues) val batch = new DefaultColumnarBatch( batchSize, schema, Array[ColumnVector](idVector, nameVector)) val fcb = new FilteredColumnarBatch(batch, Optional.empty()) val commit = appendData( engine, tblPath, isNewTable = true, schema, data = Seq(Map.empty[String, Literal] -> Seq(fcb))) verifyCommitResult(commit, expVersion = 0, expIsReadyForCheckpoint = false) verifyCommitInfo(tblPath, version = 0) verifyWrittenContent(tblPath, schema, fcb.toTestRows) } } test("insert into table - complex types with collated strings in nested/array/map") { withTempDirAndEngine { (tblPath, engine) => val utf8Lcase = new StringType("SPARK.UTF8_LCASE") val unicode = new StringType("ICU.UNICODE") val unicodeWithVersion = new StringType("ICU.UNICODE.74") val commonNested = new StructType() .add("s1", utf8Lcase) .add("n", INTEGER) val nestedWithVersion = commonNested.add("s2", unicodeWithVersion) val nestedWithoutVersion = commonNested.add("s2", unicode) val schemaWithVersion = new StructType() .add("nested", nestedWithVersion) .add("arr", new ArrayType(utf8Lcase, true)) .add("map", new MapType(STRING, unicode, true)) val schemaWithoutVersion = new StructType() .add("nested", nestedWithoutVersion) .add("arr", new ArrayType(utf8Lcase, true)) .add("map", new MapType(STRING, unicode, true)) val batchSize = 4 def buildBatch(seed: String): FilteredColumnarBatch = { val nestedVectors = Array[ColumnVector]( testColumnVector(batchSize, utf8Lcase), testColumnVector(batchSize, INTEGER), testColumnVector(batchSize, unicode)) val nestedVector = new DefaultStructVector( batchSize, nestedWithVersion, Optional.empty(), nestedVectors) val arrValues: Seq[Seq[AnyRef]] = (0 until batchSize).map { i => Seq(s"${seed}t$i", s"${seed}x$i").map(_.asInstanceOf[AnyRef]) } val arrVector = buildArrayVector(arrValues, utf8Lcase, containsNull = true) val mapType = new MapType(STRING, unicode, true) val mapValues: Seq[Map[AnyRef, AnyRef]] = (0 until batchSize).map { i => Map[AnyRef, AnyRef](s"${seed}k$i" -> s"${seed}v$i") } val mapVector = buildMapVector(mapValues, mapType) val vectors = Array[ColumnVector](nestedVector, arrVector, mapVector) val batch = new DefaultColumnarBatch(batchSize, schemaWithVersion, vectors) new FilteredColumnarBatch(batch, Optional.empty()) } val fcb1 = buildBatch("a-") val fcb2 = buildBatch("b-") val commitResult0 = appendData( engine, tblPath, isNewTable = true, schemaWithVersion, data = Seq(Map.empty[String, Literal] -> Seq(fcb1))) verifyCommitResult(commitResult0, expVersion = 0, expIsReadyForCheckpoint = false) verifyCommitInfo(tblPath, version = 0) val commitResult1 = appendData( engine, tblPath, data = Seq(Map.empty[String, Literal] -> Seq(fcb2))) verifyCommitResult(commitResult1, expVersion = 1, expIsReadyForCheckpoint = false) verifyCommitInfo(tblPath, version = 1, partitionCols = null) val expectedRows = Seq(fcb1, fcb2).flatMap(_.toTestRows) verifyWrittenContent(tblPath, schemaWithVersion, expectedRows) val metadata = getMetadata(engine, tblPath) val parsed = DataTypeJsonSerDe.deserializeStructType(metadata.getSchemaString()) assert(parsed === schemaWithoutVersion) } } test("insert into table - nested struct with collated string field") { withTempDirAndEngine { (tblPath, engine) => val utf8Lcase = new StringType("SPARK.UTF8_LCASE") val unicode = new StringType("ICU.UNICODE") val nested = new StructType() .add("c21", utf8Lcase) .add("c22", IntegerType.INTEGER) .add("c23", unicode) .add("c24", STRING) val schema = new StructType() .add("c1", LongType.LONG) .add("c2", nested) val data = generateData(schema, Seq.empty, Map.empty, batchSize = 8, numBatches = 2) val commitResult0 = appendData( engine, tblPath, isNewTable = true, schema, data = Seq(Map.empty[String, Literal] -> data)) verifyCommitResult(commitResult0, expVersion = 0, expIsReadyForCheckpoint = false) verifyCommitInfo(tblPath, version = 0) verifyWrittenContent(tblPath, schema, data.flatMap(_.toTestRows)) val metadata = getMetadata(engine, tblPath) val parsed = DataTypeJsonSerDe.deserializeStructType(metadata.getSchemaString()) assert(parsed === schema) } } test("insert into table - complex types with collated strings in nested fields") { withTempDirAndEngine { (tblPath, engine) => val utf8Lcase = new StringType("SPARK.UTF8_LCASE") val unicode = new StringType("ICU.UNICODE") val nested = new StructType() .add("c1", unicode) .add("c2", IntegerType.INTEGER) .add("c3", STRING) val schema = new StructType() .add("c1", IntegerType.INTEGER) .add("c2", nested) .add("c3", new ArrayType(utf8Lcase, true)) .add("c4", new MapType(STRING, unicode, true)) // Build vectors val batchSize = 5 val c1Vector = testColumnVector(batchSize, IntegerType.INTEGER) val nestedVectors = Array[ColumnVector]( testColumnVector(batchSize, unicode), testColumnVector(batchSize, IntegerType.INTEGER), testColumnVector(batchSize, STRING)) val c2Vector = new DefaultStructVector(batchSize, nested, Optional.empty(), nestedVectors) val c3Values: Seq[Seq[AnyRef]] = (0 until batchSize).map { i => Seq(s"t$i", s"x$i").map(_.asInstanceOf[AnyRef]) } val c3Vector = buildArrayVector(c3Values, utf8Lcase, containsNull = true) val c4Type = new MapType(STRING, unicode, true) val c4Values: Seq[Map[AnyRef, AnyRef]] = (0 until batchSize).map { i => Map[AnyRef, AnyRef](s"k$i" -> s"v$i") } val c4Vector = buildMapVector(c4Values, c4Type) val vectors = Array[ColumnVector](c1Vector, c2Vector, c3Vector, c4Vector) val batch = new DefaultColumnarBatch(batchSize, schema, vectors) val fcb = new FilteredColumnarBatch(batch, Optional.empty()) val commitResult0 = appendData( engine, tblPath, isNewTable = true, schema, data = Seq(Map.empty[String, Literal] -> Seq(fcb))) val metadata = getMetadata(engine, tblPath) val parsed = DataTypeJsonSerDe.deserializeStructType(metadata.getSchemaString) assert(parsed === schema) verifyCommitResult(commitResult0, expVersion = 0, expIsReadyForCheckpoint = false) verifyCommitInfo(tblPath, version = 0) val expectedRows = Seq(fcb).flatMap(_.toTestRows) verifyWrittenContent(tblPath, schema, expectedRows) } } test("insert into partitioned table - collated string partition columns") { val utf8Lcase = new StringType("SPARK.UTF8_LCASE") val unicode = new StringType("ICU.UNICODE.75.1") val serbian = new StringType("ICU.SR_CYRL_SRB") Seq( // (p1BatchType, p2BatchType, vBatchType) (utf8Lcase, unicode, serbian), (serbian, serbian, utf8Lcase), (utf8Lcase, serbian, unicode), (unicode, serbian, STRING), (STRING, serbian, STRING), (STRING, STRING, STRING), (utf8Lcase, STRING, utf8Lcase)).foreach { case (p1BatchType, p2BatchType, vBatchType) => withTempDirAndEngine { (tblPath, engine) => val schema = new StructType() .add("id", INTEGER) .add("p1", utf8Lcase) // partition column .add("p2", unicode) // partition column .add("v", serbian) val schemaWithoutVersion = new StructType() .add("id", INTEGER) .add("p1", utf8Lcase) .add("p2", new StringType("ICU.UNICODE")) .add("v", serbian) val dataSchema = new StructType() .add("id", INTEGER) .add("p1", p1BatchType) .add("p2", p2BatchType) .add("v", vBatchType) val vCollation = vBatchType.getCollationIdentifier val partCols = Seq("p1", "p2") val v0Part = Map("p1" -> ofString("a"), "p2" -> ofString("alpha", vCollation)) val v0Data = generateData(dataSchema, partCols, v0Part, batchSize = 8, numBatches = 1) val v1Part = Map("p1" -> ofString("B", vCollation), "p2" -> ofString("beta")) val v1Data = generateData(dataSchema, partCols, v1Part, batchSize = 5, numBatches = 1) val commitResult0 = appendData( engine, tblPath, isNewTable = true, schema, partCols, data = Seq(v0Part -> v0Data, v1Part -> v1Data)) verifyCommitResult(commitResult0, expVersion = 0, expIsReadyForCheckpoint = false) // Expect partition columns in the same case as in the schema verifyCommitInfo(tblPath, version = 0, partitionCols = partCols) val expectedRows0 = v0Data.flatMap(_.toTestRows) ++ v1Data.flatMap(_.toTestRows) verifyWrittenContent(tblPath, schema, expectedRows0) val v2Part = Map("p1" -> ofString("c"), "p2" -> ofString("gamma")) val v2Data = generateData(dataSchema, partCols, v2Part, batchSize = 4, numBatches = 3) val commitResult1 = appendData( engine, tblPath, data = Seq(v2Part -> v2Data)) verifyCommitResult(commitResult1, expVersion = 1, expIsReadyForCheckpoint = false) // For subsequent commits, partitionBy is not recorded in commit info verifyCommitInfo(tblPath, version = 1, partitionCols = null) val expectedRows1 = expectedRows0 ++ v2Data.flatMap(_.toTestRows) verifyWrittenContent(tblPath, schema, expectedRows1) val metadata = getMetadata(engine, tblPath) val parsed = DataTypeJsonSerDe.deserializeStructType(metadata.getSchemaString) assert(parsed === schemaWithoutVersion) } } } test("stats: default engine writes binary stats for collated string columns") { val utf8Lcase = new StringType("SPARK.UTF8_LCASE") val unicode = new StringType("ICU.UNICODE") val serbian = new StringType("ICU.SR_CYRL_SRB.74") Seq( (STRING, utf8Lcase, unicode), (serbian, serbian, serbian), (STRING, serbian, unicode), (STRING, STRING, STRING)).foreach { case (c1DataType, c2DataType, c3DataType) => withTempDirAndEngine { (tblPath, engine) => val schema = new StructType() .add("c1", STRING) .add("c2", utf8Lcase) .add("c3", unicode) val txn = getCreateTxn(engine, tblPath, schema) commitTransaction(txn, engine, emptyIterable()) val batchSize = 4 val values = Array("b", "A", "B", "a").map(_.asInstanceOf[AnyRef]) val c1 = DefaultGenericVector.fromArray(c1DataType, values) val c2 = DefaultGenericVector.fromArray(c2DataType, values) val c3 = DefaultGenericVector.fromArray(c3DataType, values) val batch = new DefaultColumnarBatch(batchSize, schema, Array[ColumnVector](c1, c2, c3)) val fcb = new FilteredColumnarBatch(batch, Optional.empty()) val commit = appendData(engine, tblPath, data = Seq(Map.empty[String, Literal] -> Seq(fcb))) verifyCommitResult(commit, expVersion = 1, expIsReadyForCheckpoint = false) // Read stats JSON val snapshot = Table.forPath(engine, tblPath).getLatestSnapshot(engine) val scan = snapshot.getScanBuilder().build() val scanFiles = scan.asInstanceOf[ScanImpl].getScanFiles(engine, true).toSeq .flatMap(_.getRows.toSeq) val statsJson = scanFiles.headOption.flatMap { row => val add = row.getStruct(row.getSchema.indexOf("add")) val idx = add.getSchema.indexOf("stats") if (idx >= 0 && !add.isNullAt(idx)) Some(add.getString(idx)) else None }.getOrElse(fail("Stats JSON not found")) // Default engine computes just non-collated stats; verify min/max values val mapper = JsonUtils.mapper() val statsNode = mapper.readTree(statsJson) val minValues = statsNode.get("minValues") val maxValues = statsNode.get("maxValues") // All columns: [b, A, B, a] -> min "A", max "b" assert(minValues.get("c1").asText() == "A") assert(maxValues.get("c1").asText() == "b") assert(minValues.get("c2").asText() == "A") assert(maxValues.get("c2").asText() == "b") assert(minValues.get("c3").asText() == "A") assert(maxValues.get("c3").asText() == "b") } } } test("stats: collated non-partition column in partitioned table") { val utf8Lcase = new StringType("SPARK.UTF8_LCASE") val unicode = new StringType("ICU.UNICODE") val serbian = new StringType("ICU.SR_CYRL_SRB.74") Seq( (utf8Lcase, unicode), (serbian, serbian), (utf8Lcase, serbian), (STRING, STRING), (STRING, utf8Lcase), (unicode, STRING)).foreach { case (pBatchType, dBatchType) => withTempDirAndEngine { (tblPath, engine) => val schema = new StructType() .add("p", utf8Lcase) // partition column .add("c", serbian) // non-partition, collated val dCollation = dBatchType.getCollationIdentifier val txn = getCreateTxn(engine, tblPath, schema, partCols = Seq("p")) commitTransaction(txn, engine, emptyIterable()) // Commit 1: p = "north", c values [b, A, B, a] val batchSize1 = 4 val cValues1 = Array("b", "A", "B", "a").map(_.asInstanceOf[AnyRef]) val pValues1 = Array.fill[AnyRef](batchSize1)("north") val pVec1 = DefaultGenericVector.fromArray(pBatchType, pValues1) val cVec1 = DefaultGenericVector.fromArray(dBatchType, cValues1) val batch1 = new DefaultColumnarBatch(batchSize1, schema, Array[ColumnVector](pVec1, cVec1)) val fcb1 = new FilteredColumnarBatch(batch1, Optional.empty()) val commit1 = appendData( engine, tblPath, data = Seq(Map("p" -> ofString("north", dCollation)) -> Seq(fcb1))) verifyCommitResult(commit1, expVersion = 1, expIsReadyForCheckpoint = false) verifyCommitInfo(tblPath, version = 1, partitionCols = null) // Commit 2: p = "south", c values [d, C] val batchSize2 = 2 val cValues2 = Array("d", "C", "a").map(_.asInstanceOf[AnyRef]) val pValues2 = Array.fill[AnyRef](batchSize2)("south") val pVec2 = DefaultGenericVector.fromArray(pBatchType, pValues2) val cVec2 = DefaultGenericVector.fromArray(dBatchType, cValues2) val batch2 = new DefaultColumnarBatch(batchSize2, schema, Array[ColumnVector](pVec2, cVec2)) val fcb2 = new FilteredColumnarBatch(batch2, Optional.empty()) val commit2 = appendData(engine, tblPath, data = Seq(Map("p" -> ofString("south")) -> Seq(fcb2))) verifyCommitResult(commit2, expVersion = 2, expIsReadyForCheckpoint = false) verifyCommitInfo(tblPath, version = 2, partitionCols = null) // Read stats JSON val snapshot = Table.forPath(engine, tblPath).getLatestSnapshot(engine) val scan = snapshot.getScanBuilder.build() val scanFiles = scan.asInstanceOf[ScanImpl].getScanFiles(engine, true).toSeq .flatMap(_.getRows.toSeq) val mapper = JsonUtils.mapper() assert(scanFiles.nonEmpty) scanFiles.foreach { row => val add = row.getStruct(row.getSchema.indexOf("add")) val path = add.getString(add.getSchema.indexOf("path")) val statsIdx = add.getSchema.indexOf("stats") assert(statsIdx >= 0 && !add.isNullAt(statsIdx)) val statsJson = add.getString(statsIdx) val statsNode = mapper.readTree(statsJson) val minValues = statsNode.get("minValues") val maxValues = statsNode.get("maxValues") val minC = minValues.get("c").asText() val maxC = maxValues.get("c").asText() if (path.contains("p=north")) { // For [b, A, B, a] -> min "A", max "b" assert(minC == "A") assert(maxC == "b") } else if (path.contains("p=south")) { // For [d, C] -> min "C", max "d" assert(minC == "C") assert(maxC == "d") } else { fail(s"Unexpected partition: $path") } } } } } test("stats: collect min/max for collated nested struct fields") { withTempDirAndEngine { (tblPath, engine) => val utf8Lcase = new StringType("SPARK.UTF8_LCASE") val nested = new StructType() .add("s1", utf8Lcase) .add("i1", INTEGER) val schema = new StructType() .add("nested", nested) val txn = getCreateTxn(engine, tblPath, schema) commitTransaction(txn, engine, emptyIterable()) val batchSize = 4 val s1Values = Array("b", "A", "B", "a").map(_.asInstanceOf[AnyRef]) val i1Values = Array[java.lang.Integer](3, -1, 10, 5) val s1 = DefaultGenericVector.fromArray(utf8Lcase, s1Values) val i1 = DefaultGenericVector.fromArray(INTEGER, i1Values.asInstanceOf[Array[AnyRef]]) val nestedVector = new DefaultStructVector( batchSize, nested, Optional.empty(), Array[ColumnVector](s1, i1)) val batch = new DefaultColumnarBatch( batchSize, schema, Array[ColumnVector](nestedVector)) val fcb = new FilteredColumnarBatch(batch, Optional.empty()) val commit = appendData(engine, tblPath, data = Seq(Map.empty[String, Literal] -> Seq(fcb))) verifyCommitResult(commit, expVersion = 1, expIsReadyForCheckpoint = false) // Read stats JSON val snapshot = Table.forPath(engine, tblPath).getLatestSnapshot(engine) val scan = snapshot.getScanBuilder().build() val scanFiles = scan.asInstanceOf[ScanImpl].getScanFiles(engine, true).toSeq .flatMap(_.getRows.toSeq) val statsJson = scanFiles.headOption.flatMap { row => val add = row.getStruct(row.getSchema.indexOf("add")) val idx = add.getSchema.indexOf("stats") if (idx >= 0 && !add.isNullAt(idx)) Some(add.getString(idx)) else None }.getOrElse(fail("Stats JSON not found")) val mapper = JsonUtils.mapper() val statsNode = mapper.readTree(statsJson) val minValues = statsNode.get("minValues") val maxValues = statsNode.get("maxValues") val minNested = minValues.get("nested") val maxNested = maxValues.get("nested") // For s1: [b, A, B, a] -> min "A", max "b" assert(minNested.get("s1").asText() == "A") assert(maxNested.get("s1").asText() == "b") // For i1: [3, -1, 10, 5] -> min -1, max 10 assert(minNested.get("i1").asInt() == -1) assert(maxNested.get("i1").asInt() == 10) } } test("stats: arrays and maps produce no stats; collated string field stats present") { withTempDirAndEngine { (tblPath, engine) => val unicode = new StringType("ICU.UNICODE") val utf8Lcase = new StringType("SPARK.UTF8_LCASE") val schema = new StructType() .add("name", unicode) .add("arr", new ArrayType(utf8Lcase, true)) .add("map", new MapType(STRING, utf8Lcase, true)) val txn = getCreateTxn(engine, tblPath, schema) commitTransaction(txn, engine, emptyIterable()) val batchSize = 4 val nameValues = Array("b", "A", "B", "a").map(_.asInstanceOf[AnyRef]) val nameVec = DefaultGenericVector.fromArray(unicode, nameValues) val arrValues: Seq[Seq[AnyRef]] = (0 until batchSize).map { i => Seq(s"x$i").map(_.asInstanceOf[AnyRef]) } val arrVec = buildArrayVector(arrValues, utf8Lcase, containsNull = true) val mapType = new MapType(STRING, utf8Lcase, true) val mapValues: Seq[Map[AnyRef, AnyRef]] = (0 until batchSize).map { i => Map[AnyRef, AnyRef](s"k$i" -> s"v$i") } val mapVec = buildMapVector(mapValues, mapType) val batch = new DefaultColumnarBatch( batchSize, schema, Array[ColumnVector](nameVec, arrVec, mapVec)) val fcb = new FilteredColumnarBatch(batch, Optional.empty()) val commit = appendData(engine, tblPath, data = Seq(Map.empty[String, Literal] -> Seq(fcb))) verifyCommitResult(commit, expVersion = 1, expIsReadyForCheckpoint = false) // Read stats JSON val snapshot = Table.forPath(engine, tblPath).getLatestSnapshot(engine) val scan = snapshot.getScanBuilder().build() val scanFiles = scan.asInstanceOf[ScanImpl].getScanFiles(engine, true).toSeq .flatMap(_.getRows.toSeq) val statsJson = scanFiles.headOption.flatMap { row => val add = row.getStruct(row.getSchema.indexOf("add")) val idx = add.getSchema.indexOf("stats") if (idx >= 0 && !add.isNullAt(idx)) Some(add.getString(idx)) else None }.getOrElse(fail("Stats JSON not found")) val mapper = JsonUtils.mapper() val statsNode = mapper.readTree(statsJson) val minValues = statsNode.get("minValues") val maxValues = statsNode.get("maxValues") // String column stats are present assert(minValues.get("name").asText() == "A") assert(maxValues.get("name").asText() == "b") // Array/Map columns should not have stats assert(!minValues.has("arr")) assert(!maxValues.has("arr")) assert(!minValues.has("map")) assert(!maxValues.has("map")) } } /////////////////////////////////////////////////////////////////////////// // Create table and insert data tests (CTAS & INSERT) /////////////////////////////////////////////////////////////////////////// test("cannot write to a table with an unsupported writer feature") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, testSchema) // Use your new commitUnsafe API to write an unsupported writer feature import org.apache.spark.sql.delta.{DeltaLog, OptimisticTransaction} import org.apache.spark.sql.delta.actions.Protocol val deltaLog = DeltaLog.forTable(spark, tablePath) val txn = deltaLog.startTransaction() // Create Protocol action with unsupported writer feature val protocolAction = Protocol( minReaderVersion = 3, minWriterVersion = 7, readerFeatures = Some(Set.empty), writerFeatures = Some(Set("testUnsupportedWriter"))) // Use your elegant API to commit directly to version 1 txn.commitUnsafe(tablePath, 1L, protocolAction) val e = intercept[KernelException] { getUpdateTxn(engine, tablePath) } assert(e.getMessage.contains("Unsupported Delta table feature")) } } test("insert into table - table created from scratch") { withTempDirAndEngine { (tblPath, engine) => val commitResult0 = appendData( engine, tblPath, isNewTable = true, testSchema, data = Seq(Map.empty[String, Literal] -> (dataBatches1 ++ dataBatches2))) val expectedAnswer = dataBatches1.flatMap(_.toTestRows) ++ dataBatches2.flatMap(_.toTestRows) verifyCommitResult(commitResult0, expVersion = 0, expIsReadyForCheckpoint = false) verifyCommitInfo(tblPath, version = 0) verifyWrittenContent(tblPath, testSchema, expectedAnswer) } } test("insert into table - already existing table") { withTempDirAndEngine { (tblPath, engine) => val commitResult0 = appendData( engine, tblPath, isNewTable = true, testSchema, data = Seq(Map.empty[String, Literal] -> dataBatches1)) verifyCommitResult(commitResult0, expVersion = 0, expIsReadyForCheckpoint = false) verifyCommitInfo(tblPath, version = 0, partitionCols = Seq.empty) verifyWrittenContent(tblPath, testSchema, dataBatches1.flatMap(_.toTestRows)) val txn = getUpdateTxn(engine, tblPath) assert(txn.getReadTableVersion == 0) val commitResult1 = commitAppendData(engine, txn, data = Seq(Map.empty[String, Literal] -> dataBatches2)) val expAnswer = dataBatches1.flatMap(_.toTestRows) ++ dataBatches2.flatMap(_.toTestRows) verifyCommitResult(commitResult1, expVersion = 1, expIsReadyForCheckpoint = false) verifyCommitInfo(tblPath, version = 1, partitionCols = null) verifyWrittenContent(tblPath, testSchema, expAnswer) } } test("insert into table - fails when committing the same txn twice") { withTempDirAndEngine { (tblPath, engine) => val table = Table.forPath(engine, tblPath) val txn = getCreateTxn(engine, tblPath, schema = testSchema) val txnState = txn.getTransactionState(engine) val stagedFiles = stageData(txnState, Map.empty, dataBatches1) val stagedActionsIterable = inMemoryIterable(stagedFiles) val commitResult = commitTransaction(txn, engine, stagedActionsIterable) assert(commitResult.getVersion == 0) // try to commit the same transaction and expect failure val ex = intercept[IllegalStateException] { commitTransaction(txn, engine, stagedActionsIterable) } assert(ex.getMessage.contains( "Transaction is already attempted to commit. Create a new transaction.")) } } test("insert into partitioned table - table created from scratch") { withTempDirAndEngine { (tblPath, engine) => val commitResult0 = appendData( engine, tblPath, isNewTable = true, testPartitionSchema, testPartitionColumns, Seq( Map("part1" -> ofInt(1), "part2" -> ofInt(2)) -> dataPartitionBatches1, Map("part1" -> ofInt(4), "part2" -> ofInt(5)) -> dataPartitionBatches2)) val expData = dataPartitionBatches1.flatMap(_.toTestRows) ++ dataPartitionBatches2.flatMap(_.toTestRows) verifyCommitResult(commitResult0, expVersion = 0, expIsReadyForCheckpoint = false) verifyCommitInfo(tblPath, version = 0, testPartitionColumns) verifyWrittenContent(tblPath, testPartitionSchema, expData) } } test("insert into partitioned table - already existing table") { withTempDirAndEngine { (tempTblPath, engine) => val tblPath = tempTblPath + "/table+ with special chars" val partitionCols = Seq("part1", "part2") { val commitResult0 = appendData( engine, tblPath, isNewTable = true, testPartitionSchema, testPartitionColumns, data = Seq(Map("part1" -> ofInt(1), "part2" -> ofInt(2)) -> dataPartitionBatches1)) val expData = dataPartitionBatches1.flatMap(_.toTestRows) verifyCommitResult(commitResult0, expVersion = 0, expIsReadyForCheckpoint = false) verifyCommitInfo(tblPath, version = 0, partitionCols) verifyWrittenContent(tblPath, testPartitionSchema, expData) } { val commitResult1 = appendData( engine, tblPath, data = Seq(Map("part1" -> ofInt(4), "part2" -> ofInt(5)) -> dataPartitionBatches2)) val expData = dataPartitionBatches1.flatMap(_.toTestRows) ++ dataPartitionBatches2.flatMap(_.toTestRows) verifyCommitResult(commitResult1, expVersion = 1, expIsReadyForCheckpoint = false) verifyCommitInfo(tblPath, version = 1, partitionCols = null) verifyWrittenContent(tblPath, testPartitionSchema, expData) } } } test("insert into partitioned table - handling case sensitivity of partition columns") { withTempDirAndEngine { (tblPath, engine) => val schema = new StructType() .add("id", INTEGER) .add("Name", STRING) .add("Part1", DOUBLE) // partition column .add("parT2", TIMESTAMP) // partition column val partCols = Seq("part1", "Part2") // given as input to the txn builder // expected partition cols in the commit info or elsewhere in the Delta log. // it is expected to contain the partition columns in the same case as the schema val expPartCols = Seq("Part1", "parT2") val v0Part0Values = Map( "PART1" -> ofDouble(1.0), "pART2" -> ofTimestamp(1231212L)) val v0Part0Data = generateData(schema, expPartCols, v0Part0Values, batchSize = 200, numBatches = 3) val v0Part1Values = Map( "Part1" -> ofDouble(7), "PARt2" -> ofTimestamp(123112L)) val v0Part1Data = generateData(schema, expPartCols, v0Part1Values, batchSize = 100, numBatches = 7) val v1Part0Values = Map( "PART1" -> ofNull(DOUBLE), "pART2" -> ofTimestamp(1231212L)) val v1Part0Data = generateData(schema, expPartCols, v1Part0Values, batchSize = 200, numBatches = 3) val v1Part1Values = Map( "Part1" -> ofDouble(7), "PARt2" -> ofNull(TIMESTAMP)) val v1Part1Data = generateData(schema, expPartCols, v1Part1Values, batchSize = 100, numBatches = 7) val dataPerVersion = Map( 0 -> Seq(v0Part0Values -> v0Part0Data, v0Part1Values -> v0Part1Data), 1 -> Seq(v1Part0Values -> v1Part0Data, v1Part1Values -> v1Part1Data)) val expV0Data = v0Part0Data.flatMap(_.toTestRows) ++ v0Part1Data.flatMap(_.toTestRows) val expV1Data = v1Part0Data.flatMap(_.toTestRows) ++ v1Part1Data.flatMap(_.toTestRows) for (i <- 0 until 2) { val commitResult = appendData( engine, tblPath, isNewTable = i == 0, if (i == 0) schema else null, partCols, dataPerVersion(i)) verifyCommitResult(commitResult, expVersion = i, expIsReadyForCheckpoint = false) // partition cols are not written in the commit info for inserts val partitionBy = if (i == 0) expPartCols else null val expectedOperation = if (i == 0) CREATE_TABLE else WRITE verifyCommitInfo(tblPath, version = i, partitionBy) verifyWrittenContent( tblPath, schema, if (i == 0) expV0Data else expV0Data ++ expV1Data) } } } Seq(10, 2).foreach { checkpointInterval => test(s"insert into partitioned table - isReadyForCheckpoint(interval=$checkpointInterval)") { withTempDirAndEngine { (tblPath, engine) => val schema = new StructType() .add("id", INTEGER) .add("Name", STRING) .add("Part1", DOUBLE) // partition column .add("parT2", TIMESTAMP) // partition column val partCols = Seq("Part1", "parT2") val partValues = Map("PART1" -> ofDouble(1.0), "pART2" -> ofTimestamp(1231212L)) val data = Seq( partValues -> generateData(schema, partCols, partValues, batchSize = 200, numBatches = 3)) // create a table first appendData(engine, tblPath, isNewTable = true, schema, partCols, data) // version 0 var expData = data.map(_._2).flatMap(_.flatMap(_.toTestRows)) var currentTableVersion = 0 if (checkpointInterval != 10) { // If it is not the default interval alter the table using Spark to set a // custom checkpoint interval setCheckpointInterval(tblPath, interval = checkpointInterval) // version 1 currentTableVersion = 1 } def isCheckpointExpected(version: Long): Boolean = { version != 0 && version % checkpointInterval == 0 } for (i <- currentTableVersion + 1 until 31) { val commitResult = appendData(engine, tblPath, data = data) val parquetFileCount = dataFileCount(tblPath) assert(parquetFileCount > 0) checkpointIfReady(engine, tblPath, commitResult, expSize = parquetFileCount) verifyCommitResult(commitResult, expVersion = i, isCheckpointExpected(i)) expData = expData ++ data.map(_._2).flatMap(_.flatMap(_.toTestRows)) } // expect the checkpoints created at expected versions Seq.range(0, 31).filter(isCheckpointExpected(_)).foreach { version => assertCheckpointExists(tblPath, atVersion = version) } // delete all commit files before version 30 in both cases and expect the read to pass as // there is a checkpoint at version 30 and should be used for state reconstruction. deleteDeltaFilesBefore(tblPath, beforeVersion = 30) verifyWrittenContent(tblPath, schema, expData) } } } Seq(true, false).foreach { includeTimestampNtz => test(s"insert into table - all supported types data - " + s"timestamp_ntz included = $includeTimestampNtz") { withTempDirAndEngine { (tblPath, engine) => val parquetAllTypes = goldenTablePath("parquet-all-types") val goldenTableSchema = tableSchema(parquetAllTypes) val schema = if (includeTimestampNtz) goldenTableSchema else removeTimestampNtzTypeColumns(goldenTableSchema) val data = readTableUsingKernel(engine, parquetAllTypes, schema).toSeq val dataWithPartInfo = Seq(Map.empty[String, Literal] -> data) appendData(engine, tblPath, isNewTable = true, schema, data = dataWithPartInfo) var expData = dataWithPartInfo.flatMap(_._2).flatMap(_.toTestRows) val checkpointInterval = 4 setCheckpointInterval(tblPath, checkpointInterval) for (i <- 2 until 5) { // insert until a checkpoint is required val commitResult = appendData(engine, tblPath, data = dataWithPartInfo) expData = expData ++ dataWithPartInfo.flatMap(_._2).flatMap(_.toTestRows) checkpointIfReady(engine, tblPath, commitResult, expSize = i /* one file per version */ ) verifyCommitResult(commitResult, expVersion = i, i % checkpointInterval == 0) verifyCommitInfo(tblPath, version = i, null) verifyWrittenContent(tblPath, schema, expData) } assertCheckpointExists(tblPath, atVersion = checkpointInterval) } } } Seq(true, false).foreach { includeTimestampNtz => test(s"insert into partitioned table - all supported partition column types data - " + s"timestamp_ntz included = $includeTimestampNtz") { withTempDirAndEngine { (tblPath, engine) => val parquetAllTypes = goldenTablePath("parquet-all-types") val goldenTableSchema = tableSchema(parquetAllTypes) val schema = if (includeTimestampNtz) goldenTableSchema else removeTimestampNtzTypeColumns(goldenTableSchema) val partCols = Seq( "byteType", "shortType", "integerType", "longType", "floatType", "doubleType", "decimal", "booleanType", "stringType", "binaryType", "dateType", "timestampType") ++ (if (includeTimestampNtz) Seq("timestampNtzType") else Seq.empty) val casePreservingPartCols = casePreservingPartitionColNames(schema, partCols.asJava).asScala.toSeq // get the partition values from the data batch at the given rowId def getPartitionValues(batch: ColumnarBatch, rowId: Int): Map[String, Literal] = { casePreservingPartCols.map { partCol => val colIndex = schema.indexOf(partCol) val vector = batch.getColumnVector(colIndex) val literal = if (vector.isNullAt(rowId)) { Literal.ofNull(vector.getDataType) } else { vector.getDataType match { case _: ByteType => Literal.ofByte(vector.getByte(rowId)) case _: ShortType => Literal.ofShort(vector.getShort(rowId)) case _: IntegerType => Literal.ofInt(vector.getInt(rowId)) case _: LongType => Literal.ofLong(vector.getLong(rowId)) case _: FloatType => Literal.ofFloat(vector.getFloat(rowId)) case _: DoubleType => Literal.ofDouble(vector.getDouble(rowId)) case dt: DecimalType => Literal.ofDecimal(vector.getDecimal(rowId), dt.getPrecision, dt.getScale) case _: BooleanType => Literal.ofBoolean(vector.getBoolean(rowId)) case _: StringType => Literal.ofString(vector.getString(rowId)) case _: BinaryType => Literal.ofBinary(vector.getBinary(rowId)) case _: DateType => Literal.ofDate(vector.getInt(rowId)) case _: TimestampType => Literal.ofTimestamp(vector.getLong(rowId)) case _: TimestampNTZType => Literal.ofTimestampNtz(vector.getLong(rowId)) case _ => throw new IllegalArgumentException(s"Unsupported type: ${vector.getDataType}") } } (partCol, literal) }.toMap } val data = readTableUsingKernel(engine, parquetAllTypes, schema).toSeq // From the above table read data, convert each row as a new batch with partition info // Take the values of the partitionCols from the data and create a new batch with the // selection vector to just select a single row. var dataWithPartInfo = Seq.empty[(Map[String, Literal], Seq[FilteredColumnarBatch])] data.foreach { filteredBatch => val batch = filteredBatch.getData Seq.range(0, batch.getSize).foreach { rowId => val partValues = getPartitionValues(batch, rowId) val filteredBatch = new FilteredColumnarBatch( batch, Optional.of(selectSingleElement(batch.getSize, rowId))) dataWithPartInfo = dataWithPartInfo :+ (partValues, Seq(filteredBatch)) } } appendData(engine, tblPath, isNewTable = true, schema, partCols, dataWithPartInfo) verifyCommitInfo(tblPath, version = 0, casePreservingPartCols) var expData = dataWithPartInfo.flatMap(_._2).flatMap(_.toTestRows) val checkpointInterval = 2 setCheckpointInterval(tblPath, checkpointInterval) // version 1 for (i <- 2 until 4) { // insert until a checkpoint is required val commitResult = appendData(engine, tblPath, data = dataWithPartInfo) expData = expData ++ dataWithPartInfo.flatMap(_._2).flatMap(_.toTestRows) val fileCount = dataFileCount(tblPath) checkpointIfReady(engine, tblPath, commitResult, expSize = fileCount) verifyCommitResult(commitResult, expVersion = i, i % checkpointInterval == 0) verifyCommitInfo(tblPath, version = i, partitionCols = null) verifyWrittenContent(tblPath, schema, expData) } assertCheckpointExists(tblPath, atVersion = checkpointInterval) } } } test("insert into table - given data schema mismatch") { withTempDirAndEngine { (tblPath, engine) => val ex = intercept[KernelException] { val data = Seq(Map.empty[String, Literal] -> dataPartitionBatches1) // data schema mismatch appendData(engine, tblPath, isNewTable = true, testSchema, data = data) } assert(ex.getMessage.contains("The schema of the data to be written to " + "the table doesn't match the table schema")) } } test("insert into table - missing partition column info") { withTempDirAndEngine { (tblPath, engine) => val ex = intercept[IllegalArgumentException] { appendData( engine, tblPath, isNewTable = true, testPartitionSchema, testPartitionColumns, data = Seq(Map("part1" -> ofInt(1)) -> dataPartitionBatches1) // missing part2 ) } assert(ex.getMessage.contains( "Partition values provided are not matching the partition columns.")) } } test("insert into partitioned table - invalid type of partition value") { withTempDirAndEngine { (tblPath, engine) => val ex = intercept[IllegalArgumentException] { // part2 type should be int, be giving a string value val data = Seq(Map("part1" -> ofInt(1), "part2" -> ofString("sdsd")) -> dataPartitionBatches1) appendData( engine, tblPath, isNewTable = true, testPartitionSchema, testPartitionColumns, data) } assert(ex.getMessage.contains( "Partition column part2 is of type integer but the value provided is of type string")) } } test("insert into table - idempotent writes") { withTempDirAndEngine { (tblPath, engine) => val data = Seq(Map("part1" -> ofInt(1), "part2" -> ofInt(2)) -> dataPartitionBatches1) var expData = Seq.empty[TestRow] // as the data in inserted, update this. def prepTxnAndActions(newTbl: Boolean, appId: String, txnVer: Long) : (Transaction, CloseableIterable[Row]) = { val txn = if (newTbl) { getCreateTxn( engine, tblPath, schema = testPartitionSchema, partCols = testPartitionColumns) } else { getUpdateTxn( engine, tblPath, txnId = if (appId != null) Some((appId, txnVer)) else None) } val combinedActions = inMemoryIterable( data.map { case (partValues, partData) => stageData(txn.getTransactionState(engine), partValues, partData) }.reduceLeft(_ combine _)) (txn, combinedActions) } def commitAndVerify( newTbl: Boolean, txn: Transaction, actions: CloseableIterable[Row], expTblVer: Long): Unit = { val commitResult = commitTransaction(txn, engine, actions) expData = expData ++ data.flatMap(_._2).flatMap(_.toTestRows) verifyCommitResult(commitResult, expVersion = expTblVer, expIsReadyForCheckpoint = false) val expPartCols = if (newTbl) testPartitionColumns else null val expOperation = if (newTbl) CREATE_TABLE else WRITE verifyCommitInfo(tblPath, version = expTblVer, expPartCols) verifyWrittenContent(tblPath, testPartitionSchema, expData) } def addDataWithTxnId(newTbl: Boolean, appId: String, txnVer: Long, expTblVer: Long): Unit = { val (txn, combinedActions) = prepTxnAndActions(newTbl, appId, txnVer) commitAndVerify(newTbl, txn, combinedActions, expTblVer) } def expFailure(appId: String, txnVer: Long, latestTxnVer: Long)(fn: => Any): Unit = { val ex = intercept[ConcurrentTransactionException] { fn } assert(ex.getMessage.contains(s"This error occurs when multiple updates are using the " + s"same transaction identifier to write into this table.\nApplication ID: $appId, " + s"Attempted version: $txnVer, Latest version in table: $latestTxnVer")) } // Create a transaction with id (txnAppId1, 0) and commit it addDataWithTxnId(newTbl = true, appId = "txnAppId1", txnVer = 0, expTblVer = 0) // Try to create a transaction with id (txnAppId1, 1) and commit it - should be valid addDataWithTxnId(newTbl = false, appId = "txnAppId1", txnVer = 1, expTblVer = 1) // Try to create a transaction with id (txnAppId1, 1) and try to commit it // Should fail the it is already committed above. expFailure("txnAppId1", txnVer = 1, latestTxnVer = 1) { addDataWithTxnId(newTbl = false, "txnAppId1", txnVer = 1, expTblVer = 2) } // append with no txn id addDataWithTxnId(newTbl = false, appId = null, txnVer = 0, expTblVer = 2) // Try to create a transaction with id (txnAppId2, 1) and commit it // Should be successful as the transaction app id is different addDataWithTxnId(newTbl = false, "txnAppId2", txnVer = 1, expTblVer = 3) // Try to create a transaction with id (txnAppId2, 0) and commit it // Should fail as the transaction app id is same but the version is less than the committed expFailure("txnAppId2", txnVer = 0, latestTxnVer = 1) { addDataWithTxnId(newTbl = false, "txnAppId2", txnVer = 0, expTblVer = 4) } // Start a transaction (txnAppId2, 2), but don't commit it yet val (txn, combinedActions) = prepTxnAndActions(newTbl = false, "txnAppId2", txnVer = 2) // Now start a new transaction with the same id (txnAppId2, 2) and commit it addDataWithTxnId(newTbl = false, "txnAppId2", txnVer = 2, expTblVer = 4) // Now try to commit the previous transaction (txnAppId2, 2) - should fail expFailure("txnAppId2", txnVer = 2, latestTxnVer = 2) { commitAndVerify(newTbl = false, txn, combinedActions, expTblVer = 5) } // Start a transaction (txnAppId2, 3), but don't commit it yet val (txn2, combinedActions2) = prepTxnAndActions(newTbl = false, "txnAppId2", txnVer = 3) // Now start a new transaction with the different id (txnAppId1, 10) and commit it addDataWithTxnId(newTbl = false, "txnAppId1", txnVer = 10, expTblVer = 5) // Now try to commit the previous transaction (txnAppId2, 3) - should pass commitAndVerify(newTbl = false, txn2, combinedActions2, expTblVer = 6) } } test("insert into table - write stats and validate they can be read by Spark ") { withTempDirAndEngine { (tblPath, engine) => // Configure the table property for stats collection via TableConfig. val numIndexedCols = 5 val tableProperties = Map(TableConfig. DATA_SKIPPING_NUM_INDEXED_COLS.getKey -> numIndexedCols.toString) // Schema of the table with some nested types val schema = new StructType() .add("id", INTEGER) .add("name", STRING) .add("height", DOUBLE) .add("timestamp", TIMESTAMP) .add( "metrics", new StructType() .add("temperature", DoubleType.DOUBLE) .add("humidity", FloatType.FLOAT)) // Create the table with the given schema and table properties. val txn = getCreateTxn(engine, tblPath, schema, tableProperties = tableProperties) commitTransaction(txn, engine, emptyIterable()) val dataBatches1 = generateData(schema, Seq.empty, Map.empty, batchSize = 10, numBatches = 1) val dataBatches2 = generateData(schema, Seq.empty, Map.empty, batchSize = 20, numBatches = 1) // Write initial data via Kernel. val commitResult0 = appendData( engine, tblPath, data = Seq(Map.empty[String, Literal] -> dataBatches1)) verifyCommitResult(commitResult0, expVersion = 1, expIsReadyForCheckpoint = false) verifyWrittenContent(tblPath, schema, dataBatches1.flatMap(_.getRows().toSeq).map(TestRow(_))) // Append additional data. val commitResult1 = appendData( engine, tblPath, data = Seq(Map.empty[String, Literal] -> dataBatches2)) val expectedRows = dataBatches1.flatMap(_.getRows().toSeq) ++ dataBatches2.flatMap(_.getRows().toSeq) verifyCommitResult(commitResult1, expVersion = 2, expIsReadyForCheckpoint = false) verifyWrittenContent(tblPath, schema, expectedRows.map(TestRow(_))) } } test("insert - validate DATA_SKIPPING_NUM_INDEXED_COLS is respected when collecting stats") { withTempDirAndEngine { (tblPath, engine) => val numIndexedCols = 2 val tableProps = Map(TableConfig.DATA_SKIPPING_NUM_INDEXED_COLS .getKey -> numIndexedCols.toString) val schema = new StructType() .add("id", INTEGER) .add( "name", new StructType() .add("height", DoubleType.DOUBLE) .add("timestamp", TimestampType.TIMESTAMP)) // Create table with stats collection enabled. val txn = getCreateTxn(engine, tblPath, schema, tableProperties = tableProps) commitTransaction(txn, engine, emptyIterable()) // Write one batch of data. val dataBatches = generateData(schema, Seq.empty, Map.empty, batchSize = 10, numBatches = 1) val commitResult = appendData(engine, tblPath, data = Seq(Map.empty[String, Literal] -> dataBatches)) verifyCommitResult(commitResult, expVersion = 1, expIsReadyForCheckpoint = false) // Retrieve the stats JSON from the file. val snapshot = Table.forPath(engine, tblPath).getLatestSnapshot(engine) val scan = snapshot.getScanBuilder().build() val scanFiles = scan.asInstanceOf[ScanImpl].getScanFiles(engine, true) .toSeq.flatMap(_.getRows.toSeq) val statsJson = scanFiles.headOption.flatMap { row => val addFile = row.getStruct(row.getSchema.indexOf("add")) val statsIdx = addFile.getSchema.indexOf("stats") if (statsIdx >= 0 && !addFile.isNullAt(statsIdx)) { Some(addFile.getString(statsIdx)) } else { None } }.getOrElse(fail("Stats JSON not found")) // With numIndexedCols = 2, we expect stats for id and name.height, but not for name.timestamp assert(statsJson.contains("\"id\""), "Stats should contain 'id' field") assert(statsJson.contains("\"height\""), "Stats should contain 'height' field") assert( !statsJson.contains("\"timestamp\""), "Stats should not contain 'timestamp' field, as it exceeds numIndexedCols") } } test("conflicts - creating new table - table created by other txn after current txn start") { withTempDirAndEngine { (tablePath, engine) => val losingTx = getCreateTxn(engine, tablePath, schema = testSchema) // don't commit losingTxn, instead create a new txn and commit it val winningTx = getCreateTxn(engine, tablePath, schema = testSchema) val winningTxResult = commitTransaction(winningTx, engine, emptyIterable()) // now attempt to commit the losingTxn val ex = intercept[ProtocolChangedException] { commitTransaction(losingTx, engine, emptyIterable()) } assert(ex.getMessage.contains( "Transaction has encountered a conflict and can not be committed.")) // helpful message for table creation conflict assert(ex.getMessage.contains("This happens when multiple writers are " + "writing to an empty directory. Creating the table ahead of time will avoid " + "this conflict.")) verifyCommitResult(winningTxResult, expVersion = 0, expIsReadyForCheckpoint = false) verifyCommitInfo(tablePath = tablePath, version = 0) verifyWrittenContent(tablePath, testSchema, Seq.empty) } } test("insert into table - validate serialized json stats equal Spark written stats") { withTempDirAndEngine { (dir, engine) => // Test with all Skipping eligible types. // TODO(Issue: 4284): Validate TIMESTAMP and TIMESTAMP_NTZ serialization // format. val schema = new StructType() .add("byteCol", BYTE) .add("shortCol", SHORT) .add("intCol", INTEGER) .add("longCol", LONG) .add("floatCol", FLOAT) .add("doubleCol", DOUBLE) .add("stringCol", STRING) .add("dateCol", DATE) .add( "structCol", new StructType() .add("nestedDecimal", DecimalType.USER_DEFAULT) .add("nestedDoubleCol", DOUBLE)) // Create "kernel" and "spark-copy" directories val kernelPath = new File(dir, "kernel").getAbsolutePath val sparkPath = new File(dir, "spark-copy").getAbsolutePath // Write a batch of data using the Kernel val batch = generateData(schema, Seq.empty, Map.empty, batchSize = 10, numBatches = 1) appendData( engine, kernelPath, isNewTable = true, schema, data = Seq(Map.empty[String, Literal] -> batch)) spark.read.format("delta").load(kernelPath) .write.format("delta").mode("overwrite").save(sparkPath) val mapper = JsonUtils.mapper() val kernelStats = collectStatsFromAddFiles(engine, kernelPath).map(mapper.readTree) val sparkStats = collectStatsFromAddFiles(engine, sparkPath).map(mapper.readTree) require( kernelStats.nonEmpty && sparkStats.nonEmpty, "stats collected from AddFiles should be non-empty") // Since Spark doesn't write tightBounds but Kernel now does, // we need to compare stats after removing the tightBounds field from Kernel stats val kernelStatsWithoutTightBounds = kernelStats.map { node => val objectNode = node.deepCopy().asInstanceOf[ObjectNode] objectNode.remove("tightBounds") objectNode } assert( kernelStatsWithoutTightBounds.toSet == sparkStats.toSet, s"\nKernel stats (without tightBounds):" + s"\n${kernelStatsWithoutTightBounds.mkString("\n")}\n" + s"Spark stats:\n${sparkStats.mkString("\n")}") } } test("conflicts - table metadata has changed after the losing txn has started") { withTempDirAndEngine { (tablePath, engine) => val testData = Seq(Map.empty[String, Literal] -> dataBatches1) // create a new table and commit it appendData(engine, tablePath, isNewTable = true, testSchema, data = testData) // start the losing transaction val losingTx = getUpdateTxn(engine, tablePath) // don't commit losingTxn, instead create a new txn (that changes metadata) and commit it spark.sql("ALTER TABLE delta.`" + tablePath + "` ADD COLUMN newCol INT") // now attempt to commit the losingTxn val ex = intercept[MetadataChangedException] { commitTransaction(losingTx, engine, emptyIterable()) } assert(ex.getMessage.contains("The metadata of the Delta table has been changed " + "by a concurrent update. Please try the operation again.")) } } // Different scenarios that have multiple winning txns and with a checkpoint in between. Seq(1, 5, 12).foreach { numWinningTxs => test(s"conflicts - concurrent data append ($numWinningTxs) after the losing txn has started") { withTempDirAndEngine { (tablePath, engine) => val testData = Seq(Map.empty[String, Literal] -> dataBatches1) var expData = Seq.empty[TestRow] // create a new table and commit it appendData(engine, tablePath, isNewTable = true, testSchema, data = testData) expData ++= testData.flatMap(_._2).flatMap(_.toTestRows) // start the losing transaction val txn1 = getUpdateTxn(engine, tablePath) // don't commit txn1 yet, instead commit nex txns (that appends data) and commit it Seq.range(0, numWinningTxs).foreach { i => appendData(engine, tablePath, data = Seq(Map.empty[String, Literal] -> dataBatches2)) expData ++= dataBatches2.flatMap(_.toTestRows) } // add data using the txn1 val txn1State = txn1.getTransactionState(engine) val actions = inMemoryIterable(stageData(txn1State, Map.empty, dataBatches2)) expData ++= dataBatches2.flatMap(_.toTestRows) val txn1Result = commitTransaction(txn1, engine, actions) verifyCommitResult( txn1Result, expVersion = numWinningTxs + 1, expIsReadyForCheckpoint = false) verifyCommitInfo(tablePath = tablePath, version = 0) verifyWrittenContent(tablePath, testSchema, expData) } } } def removeTimestampNtzTypeColumns(structType: StructType): StructType = { def process(dataType: DataType): Option[DataType] = dataType match { case a: ArrayType => val newElementType = process(a.getElementType) newElementType.map(new ArrayType(_, a.containsNull())) case m: MapType => val newKeyType = process(m.getKeyType) val newValueType = process(m.getValueType) (newKeyType, newValueType) match { case (Some(newKeyType), Some(newValueType)) => Some(new MapType(newKeyType, newValueType, m.isValueContainsNull)) case _ => None } case _: TimestampNTZType => None // ignore case s: StructType => val newType = removeTimestampNtzTypeColumns(s); if (newType.length() > 0) { Some(newType) } else { None } case _ => Some(dataType) } var newStructType = new StructType(); structType.fields().forEach { field => val newDataType = process(field.getDataType) if (newDataType.isDefined) { newStructType = newStructType.add(field.getName(), newDataType.get) } } newStructType } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableWritesTransactionBuilderV2Suite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import scala.collection.JavaConverters._ import scala.collection.immutable.Seq import io.delta.kernel.{Operation, TableManager} import io.delta.kernel.commit.{CommitMetadata, CommitResponse, Committer} import io.delta.kernel.data.Row import io.delta.kernel.defaults.engine.{DefaultEngine, DefaultJsonHandler} import io.delta.kernel.defaults.engine.hadoopio.HadoopFileIO import io.delta.kernel.defaults.utils.WriteUtilsWithV2Builders import io.delta.kernel.engine.{Engine, JsonHandler} import io.delta.kernel.exceptions.{KernelException, MaxCommitRetryLimitReachedException, TableAlreadyExistsException} import io.delta.kernel.expressions.Column import io.delta.kernel.internal.commit.DefaultFileSystemManagedTableOnlyCommitter import io.delta.kernel.internal.table.SnapshotBuilderImpl import io.delta.kernel.internal.tablefeatures.TableFeatures import io.delta.kernel.internal.util.ColumnMapping import io.delta.kernel.types.IntegerType import io.delta.kernel.utils.CloseableIterable.emptyIterable import io.delta.kernel.utils.CloseableIterator import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.clustering.{ClusteringMetadataDomain => SparkClusteringMetadataDomain} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path /** * Tests for the V2 transaction builders [[CreateTableTransactionBuilder]] and * [[UpdateTableTransactionBuilder]]. We don't cover the full scope of tests we have for * TransactionBuilderV1 (to do so, we would have to duplicate many, many of the existing suites). * Instead, we selectively run everything in [[DeltaTableWritesSuite]] as well as some additional * white-box-tests for the logic specific to the new builders. The main metadata validation + * update logic is shared by both V1 + V2 builders in * [[io.delta.kernel.internal.TransactionMetadataFactory]] and thus is covered by all the existing * tests we have for the V1 builder (ideally we would have unit tests for just * TransactionMetadataFactory in the future). * *

In the future, we should consider duplicating additional test suites with these builders * (requires sharding our Kernel CI tests first to avoid increasing CI runtime too much). */ class DeltaTableWritesTransactionBuilderV2Suite extends DeltaTableWritesSuite with WriteUtilsWithV2Builders { /////////////////////////////////////////////////// // Tests for code logic within the builder impls // /////////////////////////////////////////////////// // TablePropertiesTransactionBuilderV2Suite tests table property validation, normalization and // unset/set overlap for Create + Update // Tested in DeltaTableWritesSuite: setTxnOpt (covered by idempotent writes test) // Tested in DeltaTableWritesSuite: validateKernelCanWriteToTable (covered by unsupported // writer feature test) test("Cannot add clustering columns to a partitioned table") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, testPartitionSchema, partCols = testPartitionColumns) val e = intercept[KernelException] { TableManager.loadSnapshot(tablePath) .asInstanceOf[SnapshotBuilderImpl].build(engine) .buildUpdateTableTransaction(testEngineInfo, Operation.WRITE) .withClusteringColumns(List(new Column("part1")).asJava) } assert(e.getMessage.contains("Cannot enable clustering on a partitioned table")) } } test("Cannot use UpdateTableTransactionBuilder with incompatible operations") { Seq(Operation.CREATE_TABLE, Operation.REPLACE_TABLE).foreach { op => withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, testSchema) val e = intercept[IllegalArgumentException] { TableManager.loadSnapshot(tablePath) .build(engine) .buildUpdateTableTransaction(testEngineInfo, op) } assert(e.getMessage.contains( s"Operation $op is not compatible with Snapshot::buildUpdateTableTransaction")) } } } test("UpdateTableTransactionBuilder uses the committer provided during snapshot building") { withTempDirAndEngine { (tablePath, engine) => class FakeCommitter extends Committer { override def commit( engine: Engine, finalizedActions: CloseableIterator[Row], commitMetadata: CommitMetadata): CommitResponse = { throw new RuntimeException("This is a fake committer") } } createEmptyTable( engine, tablePath, testSchema) // Build snapshot with committer and start txn val txn = TableManager.loadSnapshot(tablePath) .withCommitter(new FakeCommitter()) .build(engine) .buildUpdateTableTransaction(testEngineInfo, Operation.WRITE) .build(engine) // Check the txn returns the correct committer assert(txn.getCommitter.isInstanceOf[FakeCommitter]) // Check that the txn invokes the provided committer upon commit val e = intercept[RuntimeException] { txn.commit(engine, emptyIterable()) } assert(e.getMessage.contains("This is a fake committer")) } } test("create table fails when the table already exists (non-catalog-managed)") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable(engine, tablePath, testSchema) intercept[TableAlreadyExistsException] { TableManager.buildCreateTableTransaction( tablePath, testSchema, testEngineInfo).build(engine) } } } test("CreateTableTransactionBuilderImpl::build does NOT check if the table already exists " + "when creating a catalogManaged table") { // The catalog is responsible for providing a valid, empty table location when creating // catalogManaged tables. CreateTableTransactionBuilderImpl::build does NOT check if the table // exists when creating a catalogManaged table. // // This test validates that the above logic is correct, by first creating a table at a given // path P, and then trying to create a catalogManaged table at the same path P. We expect that // CreateTableTransactionBuilderImpl::build should NOT throw. withTempDirAndEngine { (tablePath, engine) => // Create the table createEmptyTable(engine, tablePath, testSchema) // Now create it again but with catalogManaged supported. This should NOT throw. TableManager.buildCreateTableTransaction(tablePath, testSchema, testEngineInfo) .withTableProperties( Map( TableFeatures.CATALOG_MANAGED_RW_FEATURE.getTableFeatureSupportKey -> "supported").asJava) .build(engine) } } ////////////////////////////////////////////////////////////////////////////////////////////////// // Tests that builder impls correctly propagate inputs + outputs for TransactionMetadataFactory // ////////////////////////////////////////////////////////////////////////////////////////////////// // Table props are checked in TablePropertiesTransactionBuilderV2Suite + DeltaTableWritesSuite // Transaction Id is checked in DeltaTableWritesSuite // Max retries is checked DeltaTableWritesSuite for Update // Creating partitioned table is covered in DeltaTableWritesSuite test("Creating a table with clustering columns and updating the clustering columns") { withTempDirAndEngine { (tablePath, engine) => def checkClusteringColsWithSpark(expectedCols: Seq[Seq[String]]): Unit = { val deltaLog = DeltaLog.forTable(spark, new Path(tablePath)) val clusteringMetadataDomainRead = SparkClusteringMetadataDomain.fromSnapshot(deltaLog.update()) assert(clusteringMetadataDomainRead.exists(_.clusteringColumns === expectedCols)) } createEmptyTable( engine, tablePath, testPartitionSchema, clusteringColsOpt = Some(testClusteringColumns)) // Validate with Spark that the clustering columns are set checkClusteringColsWithSpark(Seq(Seq("part1"), Seq("part2"))) // Update clustering columns and check that they are updated updateTableMetadata(engine, tablePath, clusteringColsOpt = Some(List(new Column("part1")))) checkClusteringColsWithSpark(Seq(Seq("part1"))) } } test("Can evolve schema using withUpdatedSchema") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, testSchema, tableProperties = Map(ColumnMapping.COLUMN_MAPPING_MODE_KEY -> "id")) val currentSchema = getMetadata(engine, tablePath).getSchema assert(currentSchema.indexOf("newCol") == -1) val newSchema = currentSchema.add("newCol", IntegerType.INTEGER) updateTableMetadata(engine, tablePath, schema = newSchema) // Validate that the new column exits assert(getMetadata(engine, tablePath).getSchema.indexOf("newCol") >= 0) } } test("maxRetries is obeyed for Create table (error not a conflict)") { withTempDirAndEngine { (tablePath, engine) => val fileIO = new HadoopFileIO(new Configuration()) class CustomJsonHandler extends DefaultJsonHandler(fileIO) { var attemptCount = 0 override def writeJsonFileAtomically( filePath: String, data: CloseableIterator[Row], overwrite: Boolean): Unit = { attemptCount += 1 if (attemptCount == 1) { // The default committer will turn this into a CFE(isRetryable=true, isConflict=false) throw new java.io.IOException("Transient network error") } super.writeJsonFileAtomically(filePath, data, overwrite) } } class CustomEngine extends DefaultEngine(fileIO) { val jsonHandler = new CustomJsonHandler() override def getJsonHandler: JsonHandler = jsonHandler } // Commit fails when maxRetries = 0 { val transientErrorEngine = new CustomEngine() intercept[MaxCommitRetryLimitReachedException] { getCreateTxn( transientErrorEngine, tablePath, schema = testSchema, maxRetries = 0).commit(transientErrorEngine, emptyIterable()) } } // Commit succeeds when maxRetries > 1 { val transientErrorEngine = new CustomEngine() getCreateTxn( transientErrorEngine, tablePath, schema = testSchema, maxRetries = 10).commit(transientErrorEngine, emptyIterable()) } } } test("CreateTableTransactionBuilder uses the committer when provided") { withTempDirAndEngine { (tablePath, engine) => class FakeCommitter extends Committer { override def commit( engine: Engine, finalizedActions: CloseableIterator[Row], commitMetadata: CommitMetadata): CommitResponse = { throw new RuntimeException("This is a fake committer") } } val txn = TableManager.buildCreateTableTransaction(tablePath, testSchema, testEngineInfo) .withCommitter(new FakeCommitter()) .build(engine) // Check the txn returns the correct committer assert(txn.getCommitter.isInstanceOf[FakeCommitter]) // Check that the txn invokes the provided committer upon commit val e = intercept[RuntimeException] { txn.commit(engine, emptyIterable()) } assert(e.getMessage.contains("This is a fake committer")) } } test("CreateTableTransactionBuilder uses the default file system committer if none is provided") { withTempDirAndEngine { (tablePath, engine) => val txn = TableManager.buildCreateTableTransaction(tablePath, testSchema, testEngineInfo) .build(engine) // Check the txn returns the correct committer assert(txn.getCommitter == DefaultFileSystemManagedTableOnlyCommitter.INSTANCE) } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DirectoryCreationSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.io.File import java.util.Optional import scala.collection.JavaConverters._ import io.delta.kernel.{Operation, TableManager} import io.delta.kernel.defaults.utils.WriteUtils import io.delta.kernel.internal.actions.Protocol import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.tablefeatures.TableFeatures import io.delta.kernel.internal.util.DirectoryCreationUtils.createAllDeltaDirectoriesAsNeeded import io.delta.kernel.test.ActionUtils import io.delta.kernel.utils.CloseableIterable.emptyIterable import org.scalatest.funsuite.AnyFunSuite class DirectoryCreationSuite extends AnyFunSuite with WriteUtils with ActionUtils { /////////////// // E2E Tests // /////////////// test("create table with catalogManaged & v2Checkpoints supported: _delta_log, _staged_commits, " + "_sidecars directories are created") { withTempDirAndEngine { (tablePath, engine) => val logDir = new File(tablePath, "_delta_log") val stagedCommitsDir = new File(logDir.toString, "_staged_commits") val sidecarDir = new File(logDir.toString, "_sidecars") TableManager .buildCreateTableTransaction(tablePath, testSchema, "engineInfo") .withTableProperties(Map( TableFeatures.CATALOG_MANAGED_RW_FEATURE.getTableFeatureSupportKey -> "supported", "delta.checkpointPolicy" -> "v2").asJava) .withCommitter(committerUsingPutIfAbsent) .build(engine) .commit(engine, emptyIterable()) assert(logDir.exists() && logDir.isDirectory()) assert(stagedCommitsDir.exists() && stagedCommitsDir.isDirectory()) assert(sidecarDir.exists() && sidecarDir.isDirectory()) } } test("enable catalogManaged & v2Checkpoints on existing table: _staged_commits and _sidecars " + "directories are created") { withTempDirAndEngine { (tablePath, engine) => val logDir = new File(tablePath, "_delta_log") val stagedCommitsDir = new File(logDir.toString, "_staged_commits") val sidecarDir = new File(logDir.toString, "_sidecars") getCreateTxn(engine, tablePath, testSchema).commit(engine, emptyIterable()) assert(logDir.exists() && logDir.isDirectory()) assert(!sidecarDir.exists()) assert(!stagedCommitsDir.exists()) TableManager .loadSnapshot(tablePath) .withCommitter(committerUsingPutIfAbsent) .build(engine) .buildUpdateTableTransaction("engineInfo", Operation.MANUAL_UPDATE) .withTablePropertiesAdded(Map( TableFeatures.CATALOG_MANAGED_RW_FEATURE.getTableFeatureSupportKey -> "supported", "delta.checkpointPolicy" -> "v2").asJava) .build(engine) .commit(engine, emptyIterable()) assert(stagedCommitsDir.exists() && stagedCommitsDir.isDirectory()) assert(sidecarDir.exists() && sidecarDir.isDirectory()) } } //////////////// // Unit tests // //////////////// val basicProtocol = new Protocol(1, 2) val protocolWithCatalogManagedAndV2CheckpointSupport = new Protocol( TableFeatures.TABLE_FEATURES_MIN_READER_VERSION, TableFeatures.TABLE_FEATURES_MIN_WRITER_VERSION, Set( TableFeatures.CATALOG_MANAGED_RW_FEATURE.featureName(), TableFeatures.CHECKPOINT_V2_RW_FEATURE.featureName()).asJava, Set( TableFeatures.CATALOG_MANAGED_RW_FEATURE.featureName(), TableFeatures.CHECKPOINT_V2_RW_FEATURE.featureName(), TableFeatures.IN_COMMIT_TIMESTAMP_W_FEATURE.featureName()).asJava) test("_delta_log -- Case: CREATE") { withTempDirAndEngine { (tempDir, engine) => val logPath = new Path(tempDir, "_delta_log") createAllDeltaDirectoriesAsNeeded( engine, logPath, 0, Optional.empty[Protocol](), basicProtocol) val logDir = new File(logPath.toString) assert(logDir.exists() && logDir.isDirectory()) } } test("_sidecars && _staged_commits -- Case: CREATE") { withTempDirAndEngine { (tempDir, engine) => val logPath = new Path(tempDir, "_delta_log") createAllDeltaDirectoriesAsNeeded( engine, logPath, 0, Optional.empty[Protocol](), protocolWithCatalogManagedAndV2CheckpointSupport) // Check delta log directory val logDir = new File(logPath.toString) assert(logDir.exists() && logDir.isDirectory()) // Check staged commits directory val stagedCommitDir = new File(logPath.toString, "_staged_commits") assert(stagedCommitDir.exists() && stagedCommitDir.isDirectory()) // Check sidecar directory val sidecarDir = new File(logPath.toString, "_sidecars") assert(sidecarDir.exists() && sidecarDir.isDirectory()) } } test("_sidecars && _staged_commits -- Case: UPGRADE") { withTempDirAndEngine { (tempDir, engine) => val logPath = new Path(tempDir, "_delta_log") createAllDeltaDirectoriesAsNeeded( engine, logPath, 0, Optional.of(basicProtocol), protocolWithCatalogManagedAndV2CheckpointSupport) // All directories should be created assert(new File(logPath.toString).exists()) assert(new File(logPath.toString, "_staged_commits").exists()) assert(new File(logPath.toString, "_sidecars").exists()) } } test("handles existing directories gracefully") { withTempDirAndEngine { (tempDir, engine) => val logPath = new Path(tempDir, "_delta_log") // Pre-create all directories new File(logPath.toString).mkdirs() new File(logPath.toString, "_staged_commits").mkdirs() new File(logPath.toString, "_sidecars").mkdirs() // Should not throw exception when directories already exist createAllDeltaDirectoriesAsNeeded( engine, logPath, 0, Optional.empty[Protocol](), protocolWithCatalogManagedAndV2CheckpointSupport) // Directories should still exist assert(new File(logPath.toString).exists()) assert(new File(logPath.toString, "_staged_commits").exists()) assert(new File(logPath.toString, "_sidecars").exists()) } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DomainMetadataCheckSumReplayMetricsSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import io.delta.kernel.Table import io.delta.kernel.defaults.utils.{AbstractTestUtils, TestUtilsWithLegacyKernelAPIs, TestUtilsWithTableManagerAPIs} class LegacyDomainMetadataCheckSumReplayMetricsSuite extends AbstractDomainMetadataCheckSumReplayMetricsSuite with TestUtilsWithLegacyKernelAPIs { // SnapshotHint tests only apply for the legacy APIs since in the new APIs there is no persistent // Table instance test("read domain metadata fro checksum even if snapshot hint exists") { withTableWithCrc { (tablePath, engine) => val readVersion = 11 val table = Table.forPath(engine, tablePath) // Get snapshot to produce a snapshot hit at version 11. table.getLatestSnapshot(engine) engine.resetMetrics() table.getLatestSnapshot(engine).getDomainMetadata("foo") assertMetrics( engine, expJsonVersionsRead = Nil, expParquetVersionsRead = Nil, expParquetReadSetSizes = Seq(), expChecksumReadSet = Seq(readVersion)) } } } class DomainMetadataCheckSumReplayMetricsSuite extends AbstractDomainMetadataCheckSumReplayMetricsSuite with TestUtilsWithTableManagerAPIs /** * Suite to test the engine metrics when loading Domain Metadata through checksum files. */ trait AbstractDomainMetadataCheckSumReplayMetricsSuite extends ChecksumLogReplayMetricsTestBase { self: AbstractTestUtils => override protected def loadPandMCheckMetrics( tablePath: String, engine: MetricsEngine, expJsonVersionsRead: Seq[Long], expParquetVersionsRead: Seq[Long], expParquetReadSetSizes: Seq[Long], expChecksumReadSet: Seq[Long], readVersion: Long = -1): Unit = { engine.resetMetrics() readVersion match { case -1 => getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath).getDomainMetadata("foo") case ver => getTableManagerAdapter.getSnapshotAtVersion( engine, tablePath, ver).getDomainMetadata("foo") } assertMetrics( engine, expJsonVersionsRead, expParquetVersionsRead, expParquetReadSetSizes, expChecksumReadSet = expChecksumReadSet) } override protected def isDomainMetadataReplay: Boolean = true // Domain metadata requires reading checkpoint files twice: // 1. First read happens during loading Protocol & Metadata in snapshot construction. // 2. Second read happens specifically for domain metadata loading. override protected def getExpectedCheckpointReadSize(sizes: Seq[Long]): Seq[Long] = { // we read each checkpoint file twice: once for P&M and once for domain metadata sizes.flatMap(size => Seq(size, size)) } test("checksum doesn't contain domain metadata => read from logs") { withTableWithCrc { (tablePath, engine) => val readVersion = 5L rewriteChecksumFileToExcludeDomainMetadata(engine, tablePath, readVersion) loadPandMCheckMetrics( tablePath, engine, expJsonVersionsRead = Seq.range(readVersion, -1, -1), expParquetVersionsRead = Nil, expParquetReadSetSizes = Nil, expChecksumReadSet = Seq(readVersion), readVersion = readVersion) } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DomainMetadataSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import scala.collection.immutable.Seq import scala.jdk.CollectionConverters._ import io.delta.kernel._ import io.delta.kernel.data.Row import io.delta.kernel.defaults.internal.parquet.ParquetSuiteBase import io.delta.kernel.defaults.utils.{AbstractWriteUtils, WriteUtils, WriteUtilsWithV1Builders, WriteUtilsWithV2Builders} import io.delta.kernel.engine.Engine import io.delta.kernel.exceptions._ import io.delta.kernel.expressions.Literal import io.delta.kernel.internal.{SnapshotImpl, TableImpl, TransactionImpl} import io.delta.kernel.internal.actions.DomainMetadata import io.delta.kernel.internal.checksum.ChecksumReader import io.delta.kernel.internal.rowtracking.RowTrackingMetadataDomain import io.delta.kernel.utils.CloseableIterable import io.delta.kernel.utils.CloseableIterable.emptyIterable import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.RowId.{RowTrackingMetadataDomain => SparkRowTrackingMetadataDomain} import org.apache.spark.sql.delta.actions.{DomainMetadata => SparkDomainMetadata} import org.apache.hadoop.fs.Path import org.scalatest.funsuite.AnyFunSuite /** Runs domain metadata tests using the TableManager snapshot APIs and V2 transaction builders */ class DomainMetadataSuite extends AbstractDomainMetadataSuite with WriteUtilsWithV2Builders /** Runs domain metadata tests using the legacy Table snapshot APIs and V1 transaction builders */ class LegacyDomainMetadataSuite extends AbstractDomainMetadataSuite with WriteUtilsWithV1Builders trait AbstractDomainMetadataSuite extends AnyFunSuite with AbstractWriteUtils with ParquetSuiteBase { private def assertDomainMetadata( snapshot: SnapshotImpl, expectedValue: Map[String, DomainMetadata]): Unit = { // Check using internal API assert(expectedValue === snapshot.getActiveDomainMetadataMap.asScala) // Verify public API expectedValue.foreach { case (key, domainMetadata) => snapshot.getDomainMetadata(key).toScala match { case Some(config) => assert(!domainMetadata.isRemoved && config == domainMetadata.getConfiguration) case None => assert(domainMetadata.isRemoved) } } } private def assertDomainMetadata( tablePath: String, engine: Engine, expectedValue: Map[String, DomainMetadata]): Unit = { // Get the table and latest snapshot val snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath) assertDomainMetadata(snapshot, expectedValue) // verifyChecksum will check the domain metadata in CRC against the lastest snapshot. verifyChecksum(tablePath) // Delete CRC and reload snapshot from log. deleteChecksumFileForTable( tablePath.stripPrefix("file:"), versions = Seq(snapshot.getVersion.toInt)) // Rebuild table to avoid loading domain metadata from cached crc info. assertDomainMetadata( getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath), expectedValue) // Write CRC back so that subsequence operation could generate CRC incrementally. Table.forPath(engine, tablePath).checksum(engine, snapshot.getVersion) } private def commitDomainMetadataAndVerify( engine: Engine, tablePath: String, domainMetadatas: Seq[DomainMetadata], expectedValue: Map[String, DomainMetadata], useInternalApi: Boolean = false): Unit = { // Create the transaction with domain metadata and commit val txn = createTxnWithDomainMetadatas( engine, tablePath, domainMetadatas, useInternalApi) commitTransaction(txn, engine, emptyIterable()) // Verify the final state includes the expected domain metadata assertDomainMetadata(tablePath, engine, expectedValue) } private def createTableWithDomainMetadataSupported(engine: Engine, tablePath: String): Unit = { // Create an empty table commitTransaction( getCreateTxn( engine, tablePath, testSchema, withDomainMetadataSupported = true), engine, emptyIterable()) } private def validateDomainMetadataConflictResolution( engine: Engine, tablePath: String, currentTxn1DomainMetadatas: Seq[DomainMetadata], winningTxn2DomainMetadatas: Seq[DomainMetadata], winningTxn3DomainMetadatas: Seq[DomainMetadata], expectedConflict: Boolean): Unit = { // Create table with domain metadata support createTableWithDomainMetadataSupported(engine, tablePath) /** * Txn1: i.e. the current transaction that comes later than winning transactions. * Txn2: i.e. the winning transaction that was committed first. * Txn3: i.e. the winning transaction that was committed secondly. * * Note tx is the timestamp. * * t1 ------------------------ Txn1 starts. * t2 ------- Txn2 starts. * t3 ------- Txn2 commits. * t4 ------- Txn3 starts. * t5 ------- Txn3 commits. * t6 ------------------------ Txn1 commits (SUCCESS or FAIL). */ // For these txns, set enableDomainMetadata = false since it's already been enabled in the // initial table, and for V2 builders, re-enabling it will commit a new Metadata change (which // will always trigger a conflict!) val txn1 = createTxnWithDomainMetadatas( engine, tablePath, currentTxn1DomainMetadatas, enableDomainMetadata = false) val txn2 = createTxnWithDomainMetadatas( engine, tablePath, winningTxn2DomainMetadatas, enableDomainMetadata = false) commitTransaction(txn2, engine, emptyIterable()) val txn3 = createTxnWithDomainMetadatas( engine, tablePath, winningTxn3DomainMetadatas, enableDomainMetadata = false) commitTransaction(txn3, engine, emptyIterable()) if (expectedConflict) { // We expect the commit of txn1 to fail because of the conflicting DM actions val ex = intercept[KernelException] { commitTransaction(txn1, engine, emptyIterable()) } assert( ex.getMessage.contains( "A concurrent writer added a domainMetadata action for the same domain")) } else { // We expect the commit of txn1 to succeed commitTransaction(txn1, engine, emptyIterable()) // Verify the final state includes merged domain metadata val expectedMetadata = (winningTxn2DomainMetadatas ++ winningTxn3DomainMetadatas ++ currentTxn1DomainMetadatas) .groupBy(_.getDomain) .view.mapValues(_.last).toMap assertDomainMetadata(tablePath, engine, expectedMetadata) } } override def commitTransaction( txn: Transaction, engine: Engine, dataActions: CloseableIterable[Row]): TransactionCommitResult = { executeCrcSimple(txn.commit(engine, dataActions), engine) } test("create table w/o domain metadata") { withTempDirAndEngine { (tablePath, engine) => // Create an empty table commitTransaction( getCreateTxn(engine, tablePath, testSchema), engine, emptyIterable()) // Verify that the table doesn't have any domain metadata assertDomainMetadata(tablePath, engine, Map.empty) } } test("table w/o domain metadata support fails domain metadata commits") { withTempDirAndEngine { (tablePath, engine) => // Create an empty table // Its minWriterVersion is 2 and doesn't have 'domainMetadata' in its writerFeatures commitTransaction( getCreateTxn( engine, tablePath, testSchema), engine, emptyIterable()) val dm1 = new DomainMetadata("domain1", "", false) // We use the internal API because our public API will automatically upgrade the protocol val txn1 = createTxnWithDomainMetadatas(engine, tablePath, List(dm1), useInternalApi = true) // We expect the commit to fail because the table doesn't support domain metadata val e = intercept[KernelException] { commitTransaction(txn1, engine, emptyIterable()) } assert( e.getMessage .contains( "Cannot commit DomainMetadata action(s) because the feature 'domainMetadata' " + "is not supported on this table.")) // Commit domain metadata again and expect success commitDomainMetadataAndVerify( engine, tablePath, domainMetadatas = Seq(dm1), expectedValue = Map("domain1" -> dm1)) } } test("latest domain metadata overwriting existing ones") { withTempDirAndEngine { (tablePath, engine) => createTableWithDomainMetadataSupported(engine, tablePath) val dm1 = new DomainMetadata("domain1", """{"key1":"1"}, {"key2":"2"}""", false) val dm2 = new DomainMetadata("domain2", "", false) val dm3 = new DomainMetadata("domain3", """{"key3":"3"}""", false) val dm1_2 = new DomainMetadata("domain1", """{"key1":"10"}""", false) val dm3_2 = new DomainMetadata("domain3", """{"key3":"30"}""", false) Seq( (Seq(dm1), Map("domain1" -> dm1)), (Seq(dm2, dm3, dm1_2), Map("domain1" -> dm1_2, "domain2" -> dm2, "domain3" -> dm3)), (Seq(dm3_2), Map("domain1" -> dm1_2, "domain2" -> dm2, "domain3" -> dm3_2))).foreach { case (domainMetadatas, expectedValue) => commitDomainMetadataAndVerify(engine, tablePath, domainMetadatas, expectedValue) } } } test("domain metadata persistence across log replay") { withTempDirAndEngine { (tablePath, engine) => createTableWithDomainMetadataSupported(engine, tablePath) val dm1 = new DomainMetadata("domain1", """{"key1":"1"}, {"key2":"2"}""", false) val dm2 = new DomainMetadata("domain2", "", false) commitDomainMetadataAndVerify( engine, tablePath, domainMetadatas = Seq(dm1, dm2), expectedValue = Map("domain1" -> dm1, "domain2" -> dm2)) // Restart the table and verify the domain metadata assertDomainMetadata(tablePath, engine, Map("domain1" -> dm1, "domain2" -> dm2)) } } test("only the latest domain metadata per domain is stored in checkpoints") { withTempDirAndEngine { (tablePath, engine) => createTableWithDomainMetadataSupported(engine, tablePath) val dm1 = new DomainMetadata("domain1", """{"key1":"1"}, {"key2":"2"}""", false) val dm2 = new DomainMetadata("domain2", "", false) val dm3 = new DomainMetadata("domain3", """{"key3":"3"}""", false) val dm1_2 = new DomainMetadata("domain1", """{"key1":"10"}""", false) val dm3_2 = new DomainMetadata("domain3", """{"key3":"3"}""", true) Seq( (Seq(dm1), Map("domain1" -> dm1)), (Seq(dm2), Map("domain1" -> dm1, "domain2" -> dm2)), (Seq(dm3), Map("domain1" -> dm1, "domain2" -> dm2, "domain3" -> dm3)), ( Seq(dm1_2, dm3_2), Map("domain1" -> dm1_2, "domain2" -> dm2))).foreach { case (domainMetadatas, expectedValue) => commitDomainMetadataAndVerify(engine, tablePath, domainMetadatas, expectedValue) } // Checkpoint the table val latestVersion = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath).getVersion() Table.forPath(engine, tablePath).checkpoint(engine, latestVersion) // Verify that only the latest domain metadata is persisted in the checkpoint assertDomainMetadata( tablePath, engine, Map("domain1" -> dm1_2, "domain2" -> dm2)) } } test("Conflict resolution - one of three concurrent txns has DomainMetadata") { withTempDirAndEngine { (tablePath, engine) => /** * Txn1: include DomainMetadata action. * Txn2: does NOT include DomainMetadata action. * Txn3: does NOT include DomainMetadata action. * * t1 ------------------------ Txn1 starts. * t2 ------- Txn2 starts. * t3 ------- Txn2 commits. * t4 ------- Txn3 starts. * t5 ------- Txn3 commits. * t6 ------------------------ Txn1 commits (SUCCESS). */ val dm1 = new DomainMetadata("domain1", "", false) validateDomainMetadataConflictResolution( engine, tablePath, currentTxn1DomainMetadatas = Seq(dm1), winningTxn2DomainMetadatas = Seq.empty, winningTxn3DomainMetadatas = Seq.empty, expectedConflict = false) } } test( "Conflict resolution - three concurrent txns have DomainMetadata w/o conflicting domains") { withTempDirAndEngine { (tablePath, engine) => /** * Txn1: include DomainMetadata action for "domain1". * Txn2: include DomainMetadata action for "domain2". * Txn3: include DomainMetadata action for "domain3". * * t1 ------------------------ Txn1 starts. * t2 ------- Txn2 starts. * t3 ------- Txn2 commits. * t4 ------- Txn3 starts. * t5 ------- Txn3 commits. * t6 ------------------------ Txn1 commits (SUCCESS). */ val dm1 = new DomainMetadata("domain1", "", false) val dm2 = new DomainMetadata("domain2", "", false) val dm3 = new DomainMetadata("domain3", "", false) validateDomainMetadataConflictResolution( engine, tablePath, currentTxn1DomainMetadatas = Seq(dm1), winningTxn2DomainMetadatas = Seq(dm2), winningTxn3DomainMetadatas = Seq(dm3), expectedConflict = false) } } test( "Conflict resolution - three concurrent txns have DomainMetadata w/ conflicting domains") { withTempDirAndEngine { (tablePath, engine) => /** * Txn1: include DomainMetadata action for "domain1". * Txn2: include DomainMetadata action for "domain2". * Txn3: include DomainMetadata action for "domain1". * * t1 ------------------------ Txn1 starts. * t2 ------- Txn2 starts. * t3 ------- Txn2 commits. * t4 ------- Txn3 starts. * t5 ------- Txn3 commits. * t6 ------------------------ Txn1 commits (FAIL). */ val dm1 = new DomainMetadata("domain1", "", false) val dm2 = new DomainMetadata("domain2", "", false) val dm3 = new DomainMetadata("domain1", "", false) validateDomainMetadataConflictResolution( engine, tablePath, currentTxn1DomainMetadatas = Seq(dm1), winningTxn2DomainMetadatas = Seq(dm2), winningTxn3DomainMetadatas = Seq(dm3), expectedConflict = true) } } test( "Conflict resolution - three concurrent txns have DomainMetadata w/ conflict domains - 2") { withTempDirAndEngine { (tablePath, engine) => /** * Txn1: include DomainMetadata action for "domain1". * Txn2: include DomainMetadata action for "domain1". * Txn3: include DomainMetadata action for "domain2". * * t1 ------------------------ Txn1 starts. * t2 ------- Txn2 starts. * t3 ------- Txn2 commits. * t4 ------- Txn3 starts. * t5 ------- Txn3 commits. * t6 ------------------------ Txn1 commits (FAIL). */ val dm1 = new DomainMetadata("domain1", "", false) val dm2 = new DomainMetadata("domain1", "", false) val dm3 = new DomainMetadata("domain2", "", false) validateDomainMetadataConflictResolution( engine, tablePath, currentTxn1DomainMetadatas = Seq(dm1), winningTxn2DomainMetadatas = Seq(dm2), winningTxn3DomainMetadatas = Seq(dm3), expectedConflict = true) } } test("Integration test - create a table with Spark and read its domain metadata using Kernel") { withTempDir(dir => { withTempTable { tbl => val tablePath = dir.getCanonicalPath // Create table with domain metadata enabled spark.sql(s"CREATE TABLE $tbl (id LONG) USING delta LOCATION '$tablePath'") spark.sql( s"ALTER TABLE $tbl SET TBLPROPERTIES(" + s"'delta.feature.domainMetadata' = 'enabled'," + s"'delta.checkpointInterval' = '3')") // Manually commit domain metadata actions. This will create 02.json val deltaLog = DeltaLog.forTable(spark, new Path(tablePath)) deltaLog .startTransaction() .commitManuallyWithValidation( SparkDomainMetadata("testDomain1", "{\"key1\":\"1\"}", removed = false), SparkDomainMetadata("testDomain2", "", removed = false), SparkDomainMetadata("testDomain3", "", removed = false)) // This will create 03.json and 03.checkpoint spark.range(0, 2).write.format("delta").mode("append").save(tablePath) // Manually commit domain metadata actions. This will create 04.json deltaLog .startTransaction() .commitManuallyWithValidation( SparkDomainMetadata("testDomain1", "{\"key1\":\"10\"}", removed = false), SparkDomainMetadata("testDomain2", "", removed = true)) // Use Delta Kernel to read the table's domain metadata and verify the result. // We will need to read 1 checkpoint file and 1 log file to replay the table. // The state of the domain metadata should be: // testDomain1: "{\"key1\":\"10\"}", removed = false (from 03.checkpoint) // testDomain2: "", removed = true (from 03.checkpoint) // testDomain3: "", removed = false (from 04.json) val dm1 = new DomainMetadata("testDomain1", """{"key1":"10"}""", false) val dm3 = new DomainMetadata("testDomain3", "", false) assertDomainMetadata( tablePath, defaultEngine, Map("testDomain1" -> dm1, "testDomain3" -> dm3)) } }) } test("Integration test - create a table using Kernel and read its domain metadata using Spark") { withTempDirAndEngine { (tablePath, engine) => withTempTable { tbl => // Create table with domain metadata enabled createTableWithDomainMetadataSupported(engine, tablePath) // Manually commit three domain metadata actions val dm1 = new DomainMetadata("testDomain1", """{"key1":"1"}""", false) val dm2 = new DomainMetadata("testDomain2", "", false) val dm3 = new DomainMetadata("testDomain3", "", false) commitDomainMetadataAndVerify( engine, tablePath, domainMetadatas = Seq(dm1, dm2, dm3), expectedValue = Map("testDomain1" -> dm1, "testDomain2" -> dm2, "testDomain3" -> dm3)) appendData( engine, tablePath, data = Seq(Map.empty[String, Literal] -> dataBatches1)) // Checkpoint the table so domain metadata is distributed to both checkpoint and log files val latestVersion = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath).getVersion() Table.forPath(engine, tablePath).checkpoint(engine, latestVersion) // Manually commit two domain metadata actions val dm1_2 = new DomainMetadata("testDomain1", """{"key1":"10"}""", false) val dm2_2 = new DomainMetadata("testDomain2", "", true) commitDomainMetadataAndVerify( engine, tablePath, domainMetadatas = Seq(dm1_2, dm2_2), expectedValue = Map("testDomain1" -> dm1_2, "testDomain3" -> dm3)) // Use Spark to read the table's domain metadata and verify the result val deltaLog = DeltaLog.forTable(spark, new Path(tablePath)) val domainMetadata = deltaLog.snapshot.domainMetadata.groupBy(_.domain).map { case (name, domains) => assert(domains.size == 1) name -> domains.head } // Note that in Delta-Spark, the deltaLog.snapshot.domainMetadata does not include // domain metadata that are removed. assert( domainMetadata === Map( "testDomain1" -> SparkDomainMetadata( "testDomain1", """{"key1":"10"}""", removed = false), "testDomain3" -> SparkDomainMetadata("testDomain3", "", removed = false))) } } } test("RowTrackingMetadataDomain can be committed and read") { withTempDirAndEngine((tablePath, engine) => { val rowTrackingMetadataDomain = new RowTrackingMetadataDomain(10) val dmAction = rowTrackingMetadataDomain.toDomainMetadata // The configuration string should be a JSON serialization of the rowTrackingMetadataDomain assert(dmAction.getDomain === rowTrackingMetadataDomain.getDomainName) assert(dmAction.getConfiguration === """{"rowIdHighWaterMark":10}""") // Commit the DomainMetadata action and verify createTableWithDomainMetadataSupported(engine, tablePath) commitDomainMetadataAndVerify( engine, tablePath, domainMetadatas = Seq(dmAction), expectedValue = Map(rowTrackingMetadataDomain.getDomainName -> dmAction), useInternalApi = true // cannot commit system-controlled domains through public API ) // Read the RowTrackingMetadataDomain from the table and verify val snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath) val rowTrackingMetadataDomainFromSnapshot = RowTrackingMetadataDomain.fromSnapshot(snapshot) assert(rowTrackingMetadataDomainFromSnapshot.isPresent) assert(rowTrackingMetadataDomain === rowTrackingMetadataDomainFromSnapshot.get) }) } test("RowTrackingMetadataDomain Integration test - Write with Spark and read with Kernel") { withTempDirAndEngine((tablePath, engine) => { withTempTable { tbl => // Create table with domain metadata enabled using Spark spark.sql(s"CREATE TABLE $tbl (id LONG) USING delta LOCATION '$tablePath'") spark.sql( s"ALTER TABLE $tbl SET TBLPROPERTIES(" + s"'delta.feature.domainMetadata' = 'enabled'," + s"'delta.feature.rowTracking' = 'supported')") // Append 100 rows to the table, with fresh row IDs from 0 to 99 // The `delta.rowTracking.rowIdHighWaterMark` should be 99 spark.range(0, 20).write.format("delta").mode("append").save(tablePath) spark.range(20, 100).write.format("delta").mode("append").save(tablePath) // Read the RowTrackingMetadataDomain from the table using Kernel val snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath) val rowTrackingMetadataDomainRead = RowTrackingMetadataDomain.fromSnapshot(snapshot) assert(rowTrackingMetadataDomainRead.isPresent) assert(rowTrackingMetadataDomainRead.get.getRowIdHighWaterMark === 99) } }) } test("RowTrackingMetadataDomain Integration test - Write with Kernel and read with Spark") { withTempDirAndEngine { (tablePath, engine) => withTempTable { tbl => // Create table and manually make changes to the row tracking metadata domain using Kernel createTableWithDomainMetadataSupported(engine, tablePath) val dmAction = new RowTrackingMetadataDomain(10).toDomainMetadata commitDomainMetadataAndVerify( engine, tablePath, domainMetadatas = Seq(dmAction), expectedValue = Map(dmAction.getDomain -> dmAction), useInternalApi = true // cannot commit system-controlled domains through public API ) // Use Spark to read the table's row tracking metadata domain and verify the result val deltaLog = DeltaLog.forTable(spark, new Path(tablePath)) val rowTrackingMetadataDomainRead = SparkRowTrackingMetadataDomain.fromSnapshot(deltaLog.snapshot) assert(rowTrackingMetadataDomainRead.exists(_.rowIdHighWaterMark === 10)) } } } test("basic txn.addDomainMetadata API tests") { // addDomainMetadata is tested thoroughly elsewhere in this suite, here we just test API // specific behaviors withTempDirAndEngine { (tablePath, engine) => createTableWithDomainMetadataSupported(engine, tablePath) // Cannot set system-controlled domain metadata Seq("delta.foo", "DELTA.foo").foreach { domain => val e = intercept[IllegalArgumentException] { val txn = getUpdateTxn(engine, tablePath) txn.addDomainMetadata(domain, "misc config") } assert( e.getMessage.contains("Setting a non-supported system-controlled domain is not allowed")) } } // Setting the same domain more than once uses the latest pair withTempDirAndEngine { (tablePath, engine) => createTableWithDomainMetadataSupported(engine, tablePath) val dm1_1 = new DomainMetadata("domain1", """{"key1":"1"}""", false) val dm1_2 = new DomainMetadata("domain1", """{"key1":"10"}"""", false) commitDomainMetadataAndVerify(engine, tablePath, List(dm1_1, dm1_2), Map("domain1" -> dm1_2)) } } test("updating domain metadata fails after transaction committed") { withTempDirAndEngine { (tablePath, engine) => val txn = getCreateTxn(engine, tablePath, testSchema) commitTransaction(txn, engine, emptyIterable()) intercept[IllegalStateException] { txn.addDomainMetadata("domain", "config") } intercept[IllegalStateException] { txn.removeDomainMetadata("domain") } } } test("basic txn.removeDomainMetadata API tests") { // removeDomainMetadata is tested thoroughly elsewhere in this suite, here we just test API // specific behaviors withTempDirAndEngine { (tablePath, engine) => createTableWithDomainMetadataSupported(engine, tablePath) // Cannot remove system-controlled domain metadata Seq("delta.foo", "DELTA.foo").foreach { domain => val e = intercept[IllegalArgumentException] { val txn = getUpdateTxn(defaultEngine, tablePath) txn.removeDomainMetadata(domain) } assert(e.getMessage.contains("Removing a system-controlled domain is not allowed")) } } // Can remove same domain more than once in same txn withTempDirAndEngine { (tablePath, engine) => createTableWithDomainMetadataSupported(engine, tablePath) // Set up table with domain "domain1 val dm1 = new DomainMetadata("domain1", """{"key1":"1"}""", false) commitDomainMetadataAndVerify(engine, tablePath, List(dm1), Map("domain1" -> dm1)) val dm1_removed = dm1.removed() commitDomainMetadataAndVerify( engine, tablePath, List(dm1_removed, dm1_removed, dm1_removed), Map()) } } test("txn.removeDomainMetadata removing a non-existent domain") { // Remove domain that does not exist and has never existed withTempDirAndEngine { (tablePath, engine) => createTableWithDomainMetadataSupported(engine, tablePath) intercept[DomainDoesNotExistException] { val txn = getUpdateTxn(defaultEngine, tablePath) txn.removeDomainMetadata("foo") commitTransaction(txn, defaultEngine, emptyIterable()); } } // Remove domain that exists as a tombstone withTempDirAndEngine { (tablePath, engine) => createTableWithDomainMetadataSupported(engine, tablePath) // Set up table with domain "domain1" val dm1 = new DomainMetadata("domain1", """{"key1":"1"}""", false) commitDomainMetadataAndVerify(engine, tablePath, List(dm1), Map("domain1" -> dm1)) // Remove domain1 so it exists as a tombstone val dm1_removed = dm1.removed() commitDomainMetadataAndVerify( engine, tablePath, List(dm1_removed), Map()) // Removing it again should fail since it doesn't exist intercept[DomainDoesNotExistException] { commitDomainMetadataAndVerify( engine, tablePath, List(dm1_removed), Map()) } } } test("Using add and remove with the same domain in the same txn") { withTempDirAndEngine { (tablePath, engine) => createTableWithDomainMetadataSupported(engine, tablePath) // We forbid adding + removing a domain with the same identifier in a transaction to avoid // any ambiguous behavior // For example, is the expected behavior // a) we don't write any domain metadata, and it's a no-op (remove cancels out the add) // b) we remove the previous domain from the read snapshot, and add the new one as the current // domain metadata { val txn = getUpdateTxn(defaultEngine, tablePath) txn.addDomainMetadata("foo", "fake config") val e = intercept[IllegalArgumentException] { txn.removeDomainMetadata("foo") } assert(e.getMessage.contains("Cannot remove a domain that is added in this transaction")) } { val txn = getUpdateTxn(defaultEngine, tablePath) txn.removeDomainMetadata("foo") val e = intercept[IllegalArgumentException] { txn.addDomainMetadata("foo", "fake config") } assert(e.getMessage.contains("Cannot add a domain that is removed in this transaction")) } } } test("basic snapshot.getDomainMetadataConfiguration API tests") { // getDomainMetadataConfiguration is tested thoroughly elsewhere in this suite, here we just // test the API directly to be safe withTempDirAndEngine { (tablePath, engine) => createTableWithDomainMetadataSupported(engine, tablePath) // Non-existent domain is not returned assert(!latestSnapshot(tablePath).getDomainMetadata("foo").isPresent) // Commit domain foo val fooDm = new DomainMetadata("foo", "foo!", false) commitDomainMetadataAndVerify(engine, tablePath, List(fooDm), Map("foo" -> fooDm)) assert( // Check here even though already verified in commitDomainMetadataAndVerify latestSnapshot(tablePath).getDomainMetadata("foo").toScala.contains("foo!")) // Remove domain foo (so tombstone exists but should not be returned) val fooDm_removed = fooDm.removed() commitDomainMetadataAndVerify( engine, tablePath, List(fooDm_removed), Map()) // Already checked in commitDomainMetadataAndVerify but check again assert(!latestSnapshot(tablePath).getDomainMetadata("foo").isPresent) } } test("removing a domain on a table without DomainMetadata support") { withTempDirAndEngine { (tablePath, engine) => // Create table with legacy protocol commitTransaction( getCreateTxn( engine, tablePath = tablePath, schema = testSchema), engine, emptyIterable()) intercept[IllegalStateException] { val txn = getUpdateTxn(engine, tablePath) txn.removeDomainMetadata("foo") } } } test("all domain metadata batches are read when CRC exists at earlier version") { withTempDirAndEngine { (tablePath, engine) => // Create table with domain metadata support createTableWithDomainMetadataSupported(engine, tablePath) // Commit initial domain metadata so that the CRC has domain metadata entries val initDm = new DomainMetadata("init", """{"initial":"true"}""", false) commitDomainMetadataAndVerify( engine, tablePath, domainMetadatas = Seq(initDm), expectedValue = Map("init" -> initDm)) // Commit 3 domain metadata actions in a single transaction. // The commit JSON has 4 rows: commitInfo + 3 DomainMetadata actions. // With a JSON reader batch size of 2, this produces 2 batches: // Batch 1: [commitInfo, d1] // Batch 2: [d2, d3] val dm1 = new DomainMetadata("d1", """{"key":"v1"}""", false) val dm2 = new DomainMetadata("d2", """{"key":"v2"}""", false) val dm3 = new DomainMetadata("d3", """{"key":"v3"}""", false) val dmTxn = createTxnWithDomainMetadatas( engine, tablePath, Seq(dm1, dm2, dm3), enableDomainMetadata = false) val dmVersion = commitTransaction(dmTxn, engine, emptyIterable()).getVersion // Delete CRC for the new version to force Case 3 in loadDomainMetadataMap: // CRC at the earlier version (with "init" DM) exists, CRC at dmVersion is missing. // This triggers incremental log replay from dmVersion with minLogVersion = dmVersion. deleteChecksumFileForTable(tablePath, Seq(dmVersion.toInt)) // Load snapshot using an engine with batch size 2. Before the fix, the break // condition (minLogVersion == version) would exit after the first batch of // minLogVersion, missing d2 and d3 in the second batch. val snapshot = getTableManagerAdapter.getSnapshotAtLatest( defaultEngineBatchSize2, tablePath) assertDomainMetadata( snapshot, Map("init" -> initDm, "d1" -> dm1, "d2" -> dm2, "d3" -> dm3)) } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/IcebergWriterCompatV1Suite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import scala.collection.JavaConverters._ import scala.collection.immutable.Seq import io.delta.kernel.{Operation, Table, TableManager} import io.delta.kernel.data.Row import io.delta.kernel.defaults.utils.{AbstractWriteUtils, WriteUtils, WriteUtilsWithV2Builders} import io.delta.kernel.engine.Engine import io.delta.kernel.exceptions.{KernelException, UnsupportedTableFeatureException} import io.delta.kernel.internal.TableConfig import io.delta.kernel.internal.actions.{Metadata, Protocol} import io.delta.kernel.internal.table.SnapshotBuilderImpl import io.delta.kernel.internal.tablefeatures.TableFeatures import io.delta.kernel.internal.util.{ColumnMapping, ColumnMappingSuiteBase} import io.delta.kernel.internal.util.ColumnMapping.ColumnMappingMode import io.delta.kernel.test.TestFixtures import io.delta.kernel.types.{ByteType, DataType, DateType, FieldMetadata, IntegerType, LongType, ShortType, StructField, StructType, TimestampNTZType, TypeChange, VariantType} import io.delta.kernel.utils.CloseableIterable.emptyIterable import org.assertj.core.api.Assertions.assertThat import org.scalatest.funsuite.AnyFunSuite class IcebergWriterCompatV1TransactionBuilderV1Suite extends IcebergWriterCompatV1SuiteBase with WriteUtils {} class IcebergWriterCompatV1TransactionBuilderV2Suite extends IcebergWriterCompatV1SuiteBase with WriteUtilsWithV2Builders {} class CatalogManagedWithIcebergWriterCompatV1Suite extends AnyFunSuite with WriteUtilsWithV2Builders with IcebergWriterCompatV1TestUtils { test("can create a catalogManaged table with icebergWriterCompatV1") { withTempDirAndEngine { (tablePath, engine) => // ===== GIVEN ===== val createTxn = TableManager .buildCreateTableTransaction(tablePath, testSchema, "engineInfo") .withCommitter(committerUsingPutIfAbsent) .withTableProperties( Map( TableFeatures.CATALOG_MANAGED_RW_FEATURE.getTableFeatureSupportKey -> "supported", TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.getKey -> "true").asJava) .build(engine) // ===== WHEN ===== createTxn.commit(engine, emptyIterable[Row]) // ===== THEN ===== val snapshotImpl = TableManager .loadSnapshot(tablePath) .asInstanceOf[SnapshotBuilderImpl] .withMaxCatalogVersion(0) .build(engine) verifyIcebergWriterCompatV1Enabled(snapshotImpl.getProtocol, snapshotImpl.getMetadata) assert( snapshotImpl.getProtocol.supportsFeature(TableFeatures.CATALOG_MANAGED_RW_FEATURE)) } } } trait IcebergWriterCompatV1TestUtils { self: AbstractWriteUtils => def verifyIcebergWriterCompatV1Enabled(tablePath: String, engine: Engine): Unit = { val protocol = getProtocol(engine, tablePath) val metadata = getMetadata(engine, tablePath) verifyIcebergWriterCompatV1Enabled(protocol, metadata) } def verifyIcebergWriterCompatV1Enabled(protocol: Protocol, metadata: Metadata): Unit = { // Check expected protocol features are enabled assert(protocol.supportsFeature(TableFeatures.ICEBERG_COMPAT_V2_W_FEATURE)) assert(protocol.supportsFeature(TableFeatures.COLUMN_MAPPING_RW_FEATURE)) assert(protocol.supportsFeature(TableFeatures.ICEBERG_WRITER_COMPAT_V1)) // Check expected confs are present assert(TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.fromMetadata(metadata)) assert(TableConfig.ICEBERG_COMPAT_V2_ENABLED.fromMetadata(metadata)) assert(TableConfig.COLUMN_MAPPING_MODE.fromMetadata(metadata) == ColumnMappingMode.ID) } } trait IcebergWriterCompatV1SuiteBase extends AnyFunSuite with AbstractWriteUtils with TestFixtures with IcebergWriterCompatV1TestUtils with ColumnMappingSuiteBase { private val tblPropertiesIcebergWriterCompatV1Enabled = Map( TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.getKey -> "true") private val tblPropertiesIcebergCompatV2Enabled = Map( TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "true") private val tblPropertiesColumnMappingModeId = Map( TableConfig.COLUMN_MAPPING_MODE.getKey -> "id") Seq( (Map(), "no other properties"), (tblPropertiesIcebergCompatV2Enabled, "icebergCompatV2 enabled"), (tblPropertiesColumnMappingModeId, "column mapping mode set to id"), ( tblPropertiesIcebergCompatV2Enabled ++ tblPropertiesColumnMappingModeId, "icebergCompatV2 enabled and column mapping mode set to id")).foreach { case (tblProperties, description) => test(s"Basic enablement on new table with $description") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, cmTestSchema(), tableProperties = tblPropertiesIcebergWriterCompatV1Enabled ++ tblProperties) verifyIcebergWriterCompatV1Enabled(tablePath, engine) verifyCMTestSchemaHasValidColumnMappingInfo( getMetadata(engine, tablePath), enableIcebergWriterCompatV1 = true) } } } test("Cannot enable on an existing table") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, testSchema, tableProperties = tblPropertiesColumnMappingModeId ++ tblPropertiesIcebergCompatV2Enabled) val e = intercept[KernelException] { updateTableMetadata( engine, tablePath, tableProperties = tblPropertiesIcebergWriterCompatV1Enabled) } assert(e.getMessage.contains( "Cannot enable delta.enableIcebergWriterCompatV1 on an existing table")) } } test("Can enable on an existing table if already enabled") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, testSchema, tableProperties = tblPropertiesIcebergWriterCompatV1Enabled) verifyIcebergWriterCompatV1Enabled(tablePath, engine) updateTableMetadata( engine, tablePath, tableProperties = tblPropertiesIcebergWriterCompatV1Enabled) verifyIcebergWriterCompatV1Enabled(tablePath, engine) } } test("Cannot disable icebergWriterCompatV1 conf on existing table") { withTempDirAndEngine { (tablePath, engine) => // Create an empty table with icebergWriterCompatV1 enabled createEmptyTable( engine, tablePath, cmTestSchema(), tableProperties = tblPropertiesIcebergWriterCompatV1Enabled) verifyIcebergWriterCompatV1Enabled(tablePath, engine) val e = intercept[KernelException] { // Disable icebergWriterCompatV1 in the table properties updateTableMetadata( engine, tablePath, tableProperties = Map(TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.getKey -> "false")) } assert(e.getMessage.contains( "Disabling delta.enableIcebergWriterCompatV1 on an existing table is not allowed")) } } test("Cannot enable when column mapping mode explicitly set to name/none") { Seq("name", "none").foreach { cmMode => withTempDirAndEngine { (tablePath, engine) => val e = intercept[KernelException] { createEmptyTable( engine, tablePath, testSchema, tableProperties = tblPropertiesIcebergWriterCompatV1Enabled ++ Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> cmMode)) } assert(e.getMessage.contains(s"The value '$cmMode' for the property " + s"'delta.columnMapping.mode' is not compatible with icebergWriterCompatV1")) } } } Seq(true, false).foreach { cmInfoPopulated => test( s"Column mapping metadata set correctly when cmInfoPrePopulated=$cmInfoPopulated") { withTempDirAndEngine { (tablePath, engine) => // Create new table and verify column mapping info set correctly val initialSchema = if (cmInfoPopulated) { new StructType() .add( "c1", IntegerType.INTEGER, FieldMetadata.builder() .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, 1) .putString(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "col-1") .build()) } else { new StructType() .add("c1", IntegerType.INTEGER) } createEmptyTable( engine, tablePath, initialSchema, tableProperties = tblPropertiesIcebergWriterCompatV1Enabled) val initialMetadata = getMetadata(engine, tablePath) assertThat(initialMetadata.getConfiguration) .containsEntry(ColumnMapping.COLUMN_MAPPING_MAX_COLUMN_ID_KEY, "1") assertThat(initialMetadata.getSchema.get("c1").getMetadata.getEntries) .containsEntry(ColumnMapping.COLUMN_MAPPING_ID_KEY, 1L.asInstanceOf[AnyRef]) .containsEntry(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "col-1") // Add a new column and verify column mapping info set correctly val updatedSchema = if (cmInfoPopulated) { initialMetadata.getSchema .add( "c2", IntegerType.INTEGER, FieldMetadata.builder() .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, 2) .putString(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "col-2") .build()) } else { initialMetadata.getSchema .add("c2", IntegerType.INTEGER) } updateTableMetadata(engine, tablePath, schema = updatedSchema) val updatedMetadata = getMetadata(engine, tablePath) assertThat(updatedMetadata.getConfiguration) .containsEntry(ColumnMapping.COLUMN_MAPPING_MAX_COLUMN_ID_KEY, "2") assertThat(updatedMetadata.getSchema.get("c2").getMetadata.getEntries) .containsEntry(ColumnMapping.COLUMN_MAPPING_ID_KEY, 2L.asInstanceOf[AnyRef]) .containsEntry(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "col-2") } } } Seq(true, false).foreach { isNewTable => test(s"Cannot set physicalName to something other than col-{fieldId}, isNewTable=$isNewTable") { withTempDirAndEngine { (tablePath, engine) => if (!isNewTable) { createEmptyTable( engine, tablePath, new StructType().add("c1", IntegerType.INTEGER), tableProperties = tblPropertiesIcebergWriterCompatV1Enabled) } val schemaToCommit = if (isNewTable) { new StructType() .add( "c2", IntegerType.INTEGER, FieldMetadata.builder() .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, 1) .putString(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "c2") .build()) } else { getMetadata(engine, tablePath) .getSchema .add( "c2", IntegerType.INTEGER, FieldMetadata.builder() .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, 2) .putString(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "c2") .build()) } val e = intercept[KernelException] { appendData( engine, tablePath, isNewTable = isNewTable, schema = schemaToCommit, data = Seq.empty, tableProperties = tblPropertiesIcebergWriterCompatV1Enabled) } val expectedInvalidColumnId = if (isNewTable) 1 else 2 assert(e.getMessage.contains( "IcebergWriterCompatV1 requires column mapping field physical names be equal to " + "'col-[fieldId]', but this is not true for the following fields " + s"[c2(physicalName='c2', columnId=$expectedInvalidColumnId)]")) } } } test("Cannot enable when icebergCompatV2 explicitly disabled") { withTempDirAndEngine { (tablePath, engine) => val e = intercept[KernelException] { createEmptyTable( engine, tablePath, testSchema, tableProperties = tblPropertiesIcebergWriterCompatV1Enabled ++ Map(TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "false")) } assert(e.getMessage.contains("'false' for the property 'delta.enableIcebergCompatV2' is " + "not compatible with icebergWriterCompatV1")) } } test("Cannot disable icebergCompatV2 on an existing table") { withTempDirAndEngine { (tablePath, engine) => // Create an empty table with icebergWriterCompatV1 enabled createEmptyTable( engine, tablePath, cmTestSchema(), tableProperties = tblPropertiesIcebergWriterCompatV1Enabled) verifyIcebergWriterCompatV1Enabled(tablePath, engine) val e = intercept[KernelException] { // Disable icebergCompatV2 updateTableMetadata( engine, tablePath, tableProperties = Map(TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey -> "false")) } assert(e.getMessage.contains("'false' for the property 'delta.enableIcebergCompatV2' is " + "not compatible with icebergWriterCompatV1")) } } // TODO once we support schema evolution test adding columns of these types Seq(ByteType.BYTE, ShortType.SHORT).foreach { dataType => test(s"Cannot enable IcebergWriterCompatV2 on a table with datatype $dataType") { withTempDirAndEngine { (tablePath, engine) => val e = intercept[KernelException] { createEmptyTable( engine, tablePath, new StructType().add("col", dataType), tableProperties = tblPropertiesIcebergWriterCompatV1Enabled) } assert(e.getMessage.contains( s"icebergWriterCompatV1 does not support the data types: [${dataType.toString}]")) } } } test("subsequent writes to icebergWriterCompatV1 enabled tables doesn't update metadata") { // we want to make sure the [[IcebergWriterCompatV1MetadataValidatorAndUpdater]] doesn't // make unneeded metadata updates withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, testSchema, tableProperties = tblPropertiesIcebergWriterCompatV1Enabled) appendData( engine, tablePath, data = Seq.empty, tableProperties = tblPropertiesIcebergWriterCompatV1Enabled ++ tblPropertiesIcebergCompatV2Enabled ++ tblPropertiesColumnMappingModeId ) // version 1 appendData(engine, tablePath, data = Seq.empty) // version 2 val table = Table.forPath(engine, tablePath) assert(getMetadataActionFromCommit(engine, table, version = 0).isDefined) assert(getMetadataActionFromCommit(engine, table, version = 1).isEmpty) assert(getMetadataActionFromCommit(engine, table, version = 2).isEmpty) // make a metadata update and see it is reflected in the table val newProps = Map("key" -> "value") updateTableMetadata(engine, tablePath, tableProperties = newProps) // version 3 assert(getMetadataActionFromCommit(engine, table, version = 3).isDefined) val ver3Metadata = getMetadata(engine, tablePath) assert(ver3Metadata.getConfiguration().get("key") == "value") } } /* -------------------- Tests for blocked table features -------------------- */ def testIncompatibleTableFeature( featureName: String, tablePropertiesToEnable: Map[String, String] = Map.empty, schemaToEnable: StructType = testSchema, expectedErrorMessage: String, testOnExistingTable: Boolean = true // some features cannot be enabled for existing tables ): Unit = { if (testOnExistingTable) { test(s"Cannot enable feature $featureName on an existing table with " + s"icebergWriterCompatV1 enabled") { withTempDirAndEngine { (tablePath, engine) => // Create existing table with icebergWriterCompatV1 enabled createEmptyTable( engine, tablePath, testSchema, tableProperties = tblPropertiesIcebergWriterCompatV1Enabled) verifyIcebergWriterCompatV1Enabled(tablePath, engine) val e = intercept[KernelException] { // Update the table such that we enable the incompatible feature updateTableMetadata( engine, tablePath, schema = schemaToEnable, tableProperties = tablePropertiesToEnable) } assert(e.getMessage.contains(expectedErrorMessage)) } } } test(s"Cannot enable feature $featureName and icebergWriterCompatV1 on a new table") { withTempDirAndEngine { (tablePath, engine) => // Create table with IcebergCompatWriterV1 and the incompatible feature enabled val e = intercept[KernelException] { createEmptyTable( engine, tablePath, schema = schemaToEnable, tableProperties = tblPropertiesIcebergWriterCompatV1Enabled ++ tablePropertiesToEnable) } assert(e.getMessage.contains(expectedErrorMessage)) } } // Since we don't support enabling icebergWriterCompatV1 on an existing table we cannot test // the case of enabling icebergWriterCompatV1 on an existing table with the incompatible // feature enabled } // Features that don't have write support currently (once we add write support convert these // tests and update error intercepted) def testIncompatibleUnsupportedTableFeature( featureName: String, tablePropertiesToEnable: Map[String, String] = Map.empty, schemaToEnable: StructType = testSchema, expectedErrorMessage: String = "Unsupported Delta writer feature", testOnExistingTable: Boolean = true // some features cannot be enabled for existing tables ): Unit = { testIncompatibleTableFeature( featureName, tablePropertiesToEnable, schemaToEnable, expectedErrorMessage, testOnExistingTable) } /* ----- Incompatible features not supported when ACTIVE in the table ----- */ testIncompatibleUnsupportedTableFeature( "changeDataFeed", tablePropertiesToEnable = Map(TableConfig.CHANGE_DATA_FEED_ENABLED.getKey -> "true")) testIncompatibleUnsupportedTableFeature( "invariants", schemaToEnable = new StructType() .add("c1", IntegerType.INTEGER) .add( "c2", IntegerType.INTEGER, FieldMetadata.builder() .putString("delta.invariants", "{\"expression\": { \"expression\": \"x > 3\"} }") .build()), testOnExistingTable = false // we don't currently support schema updates ) testIncompatibleUnsupportedTableFeature( "checkConstraints", tablePropertiesToEnable = Map("delta.constraints.a" -> "a = b"), expectedErrorMessage = "Unknown configuration was specified: delta.constraints.a") testIncompatibleUnsupportedTableFeature( "generatedColumns", schemaToEnable = new StructType() .add("c1", IntegerType.INTEGER) .add( "c2", IntegerType.INTEGER, FieldMetadata.builder() .putString("delta.generationExpression", "{\"expression\": \"c1 + 1\"}") .build()), testOnExistingTable = false // we don't currently support schema updates ) testIncompatibleUnsupportedTableFeature( "identityColumns", schemaToEnable = new StructType() .add("c1", IntegerType.INTEGER) .add( "c2", IntegerType.INTEGER, FieldMetadata.builder() .putLong("delta.identity.start", 1L) .putLong("delta.identity.step", 2L) .putBoolean("delta.identity.allowExplicitInsert", true) .build()), testOnExistingTable = false // we don't currently support schema updates ) testIncompatibleUnsupportedTableFeature( "variantType", schemaToEnable = new StructType() .add("c1", IntegerType.INTEGER) .add("c2", VariantType.VARIANT), testOnExistingTable = false, // we don't currently support schema updates // We throw an error earlier for variant for some reason expectedErrorMessage = "icebergCompatV2 does not support the data types: [variant]") testIncompatibleTableFeature( "rowTracking", tablePropertiesToEnable = Map("delta.enableRowTracking" -> "true"), expectedErrorMessage = "Table features [rowTracking] are incompatible with icebergWriterCompatV1") // deletionVectors is blocked by both icebergCompatV2 and icebergWriterCompatV1; since the // icebergCompatV2 checks are executed first as part of ICEBERG_COMPAT_V2_ENABLED.postProcess we // hit that error message first testIncompatibleTableFeature( "deletionVectors", tablePropertiesToEnable = Map(TableConfig.DELETION_VECTORS_CREATION_ENABLED.getKey -> "true"), expectedErrorMessage = "Table features [deletionVectors] are incompatible with icebergCompatV2") /* ----- Non-legacy incompatible features not allowed even when inactive ----- */ testIncompatibleUnsupportedTableFeature( "variantType inactive", tablePropertiesToEnable = Map("delta.feature.variantType" -> "supported"), expectedErrorMessage = "Table features [variantType] are " + "incompatible with icebergWriterCompatV1") // deletionVectors is blocked by both icebergCompatV2 and icebergWriterCompatV1; since the // icebergCompatV2 checks are executed first as part of ICEBERG_COMPAT_V2_ENABLED.postProcess we // hit that error message first testIncompatibleTableFeature( "deletionVectors inactive", tablePropertiesToEnable = Map("delta.feature.deletionVectors" -> "supported"), expectedErrorMessage = "Table features [deletionVectors] are incompatible with icebergCompatV2") // defaultColumns is not added to Kernel yet --> throws an error on feature lookup testIncompatibleUnsupportedTableFeature( "defaultColumns inactive", tablePropertiesToEnable = Map("delta.feature.defaultColumns" -> "supported"), expectedErrorMessage = "Unsupported Delta table feature") // collations is not added to Kernel yet --> throws an error on feature lookup testIncompatibleUnsupportedTableFeature( "collations inactive", tablePropertiesToEnable = Map("delta.feature.collations" -> "supported"), expectedErrorMessage = "Unsupported Delta table feature") /* ----- Legacy incompatible features allowed if they are inactive ----- */ test("legacy table features allowed with icebergWriterCompatV1 if inactive") { val tblProperties = Seq( "invariants", "changeDataFeed", "checkConstraints", "identityColumns", "generatedColumns", "typeWidening", "typeWidening-preview", "rowTracking") .map(tableFeature => s"delta.feature.$tableFeature" -> "supported") .toMap // New table with these features + icebergWriterCompatV1 withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, cmTestSchema(), tableProperties = tblProperties ++ tblPropertiesIcebergWriterCompatV1Enabled) verifyIcebergWriterCompatV1Enabled(tablePath, engine) // Check all the features are supported val protocol = getProtocol(engine, tablePath) assert(protocol.supportsFeature(TableFeatures.GENERATED_COLUMNS_W_FEATURE)) assert(protocol.supportsFeature(TableFeatures.IDENTITY_COLUMNS_W_FEATURE)) assert(protocol.supportsFeature(TableFeatures.CONSTRAINTS_W_FEATURE)) assert(protocol.supportsFeature(TableFeatures.CHANGE_DATA_FEED_W_FEATURE)) assert(protocol.supportsFeature(TableFeatures.INVARIANTS_W_FEATURE)) } // Existing table with icebergWriterCompatV1 - enable these features withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, cmTestSchema(), tableProperties = tblPropertiesIcebergWriterCompatV1Enabled) verifyIcebergWriterCompatV1Enabled(tablePath, engine) updateTableMetadata( engine, tablePath, tableProperties = tblProperties) // Check all the features are supported val protocol = getProtocol(engine, tablePath) assert(protocol.supportsFeature(TableFeatures.GENERATED_COLUMNS_W_FEATURE)) assert(protocol.supportsFeature(TableFeatures.IDENTITY_COLUMNS_W_FEATURE)) assert(protocol.supportsFeature(TableFeatures.CONSTRAINTS_W_FEATURE)) assert(protocol.supportsFeature(TableFeatures.CHANGE_DATA_FEED_W_FEATURE)) assert(protocol.supportsFeature(TableFeatures.INVARIANTS_W_FEATURE)) assert(protocol.supportsFeature(TableFeatures.TYPE_WIDENING_RW_FEATURE)) } } /* ----- Compatible features allowed when active ----- */ test("All expected compatible features can be active with icebergWriterCompatV1") { val tblProperties = Map( TableConfig.APPEND_ONLY_ENABLED.getKey -> "true", // appendOnly TableConfig.CHECKPOINT_POLICY.getKey -> "v2", // checkpointV2 TableConfig.IN_COMMIT_TIMESTAMPS_ENABLED.getKey -> "true", // inCommitTimestamp TableConfig.ICEBERG_WRITER_COMPAT_V1_ENABLED.getKey -> "true", TableConfig.TYPE_WIDENING_ENABLED.getKey -> "true") val schema = new StructType() .add("c1", IntegerType.INTEGER) .add("c2", TimestampNTZType.TIMESTAMP_NTZ) // timestampNtz // New table with these features + icebergWriterCompatV1 withTempDirAndEngine { (tablePath, engine) => getCreateTxn( engine, tablePath, schema, tableProperties = tblProperties, withDomainMetadataSupported = true) .commit(engine, emptyIterable[Row]) verifyIcebergWriterCompatV1Enabled(tablePath, engine) // Check all the features are supported val protocol = getProtocol(engine, tablePath) assert(protocol.supportsFeature(TableFeatures.APPEND_ONLY_W_FEATURE)) assert(protocol.supportsFeature(TableFeatures.CHECKPOINT_V2_RW_FEATURE)) assert(protocol.supportsFeature(TableFeatures.IN_COMMIT_TIMESTAMP_W_FEATURE)) assert(protocol.supportsFeature(TableFeatures.TIMESTAMP_NTZ_RW_FEATURE)) assert(protocol.supportsFeature(TableFeatures.DOMAIN_METADATA_W_FEATURE)) assert(protocol.supportsFeature(TableFeatures.INVARIANTS_W_FEATURE)) // TODO in the future add typeWidening and clustering once they are supported } // Existing table with icebergWriterCompatV1 - enable these features withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, cmTestSchema(), tableProperties = tblPropertiesIcebergWriterCompatV1Enabled) verifyIcebergWriterCompatV1Enabled(tablePath, engine) getUpdateTxn( engine, tablePath, tableProperties = tblProperties, withDomainMetadataSupported = true) .commit(engine, emptyIterable[Row]) // Check all the features are supported val protocol = getProtocol(engine, tablePath) assert(protocol.supportsFeature(TableFeatures.APPEND_ONLY_W_FEATURE)) assert(protocol.supportsFeature(TableFeatures.CHECKPOINT_V2_RW_FEATURE)) assert(protocol.supportsFeature(TableFeatures.IN_COMMIT_TIMESTAMP_W_FEATURE)) // assert(protocol.supportsFeature(TableFeatures.TIMESTAMP_NTZ_RW_FEATURE)) assert(protocol.supportsFeature(TableFeatures.DOMAIN_METADATA_W_FEATURE)) assert(protocol.supportsFeature(TableFeatures.INVARIANTS_W_FEATURE)) assert(protocol.supportsFeature(TableFeatures.TYPE_WIDENING_RW_FEATURE)) // TODO in the future add clustering once they are supported } } test("compatible type widening is allowed with icebergWriterCompatV1") { withTempDirAndEngine { (tablePath, engine) => // Create a table with icebergWriterCompatV1 and type widening enabled val schema = new StructType() .add(new StructField( "intToLong", LongType.LONG, false, FieldMetadata.builder() .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, 1) .putString( ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "col-1").build()).withTypeChanges( Seq(new TypeChange(IntegerType.INTEGER, LongType.LONG)).asJava)) val tblProps = tblPropertiesIcebergWriterCompatV1Enabled ++ Map(TableConfig.TYPE_WIDENING_ENABLED.getKey -> "true") // This should not throw an exception createEmptyTable(engine, tablePath, schema, tableProperties = tblProps) val protocol = getProtocol(engine, tablePath) assert(protocol.supportsFeature(TableFeatures.TYPE_WIDENING_RW_FEATURE)) } } test("incompatible type widening throws exception with icebergWriterCompatV1 on new Table") { withTempDirAndEngine { (tablePath, engine) => // Try to create a table with icebergWriterCompatV1 and incompatible type widening val schema = new StructType() .add(new StructField( "dateToTimestamp", TimestampNTZType.TIMESTAMP_NTZ, false, FieldMetadata.builder() .putLong(ColumnMapping.COLUMN_MAPPING_ID_KEY, 1) .putString( ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "col-1").build()).withTypeChanges( Seq(new TypeChange(DateType.DATE, TimestampNTZType.TIMESTAMP_NTZ)).asJava)) val tblProps = tblPropertiesIcebergWriterCompatV1Enabled ++ Map(TableConfig.TYPE_WIDENING_ENABLED.getKey -> "true") val e = intercept[KernelException] { createEmptyTable(engine, tablePath, schema, tableProperties = tblProps) } assert( e.getMessage.contains("icebergCompatV2 does not support type widening present in table")) } } /* -------------------- Enforcements blocked by icebergCompatV2 -------------------- */ // We test the deletionVector checks above as part of blocked table feature tests // We cannot test enabling icebergCompatV1 since it is not a table feature in Kernel; This is // tested in the unit tests in IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite (PRIMITIVE_TYPES ++ NESTED_TYPES) // filter out the types unsupported by icebergWriterCompatV1 .filter(dataType => dataType != ByteType.BYTE && dataType != ShortType.SHORT) .foreach { dataType: DataType => test(s"allowed data column types: $dataType on a new table") { withTempDirAndEngine { (tablePath, engine) => val schema = new StructType().add("col", dataType) createEmptyTable( engine, tablePath, schema, tableProperties = tblPropertiesIcebergWriterCompatV1Enabled) } } } ignore("test unsupported data types") { // Can't test this now as the only unsupported data type in Iceberg is VariantType, // and it also has no write support in Kernel. // Unit test for this is covered in [[IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite]] } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/InCommitTimestampSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.util.{Locale, Optional} import scala.collection.JavaConverters._ import scala.collection.immutable.{ListMap, Seq} import scala.collection.mutable import io.delta.kernel._ import io.delta.kernel.Operation.{CREATE_TABLE, WRITE} import io.delta.kernel.defaults.utils.{AbstractWriteUtils, WriteUtilsWithV1Builders, WriteUtilsWithV2Builders} import io.delta.kernel.engine.Engine import io.delta.kernel.exceptions.{InvalidTableException, ProtocolChangedException} import io.delta.kernel.expressions.Literal import io.delta.kernel.internal.{DeltaHistoryManager, SnapshotImpl, TableImpl} import io.delta.kernel.internal.TableConfig._ import io.delta.kernel.internal.actions.{CommitInfo, SingleAction} import io.delta.kernel.internal.actions.SingleAction.createCommitInfoSingleAction import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.util.{FileNames, VectorUtils} import io.delta.kernel.internal.util.ManualClock import io.delta.kernel.internal.util.Utils.singletonCloseableIterator import io.delta.kernel.types._ import io.delta.kernel.types.IntegerType.INTEGER import io.delta.kernel.utils.CloseableIterable.{emptyIterable, inMemoryIterable} import io.delta.kernel.utils.FileStatus import org.scalatest.funsuite.AnyFunSuite /** * Runs in-commit timestamp tests using the TableManager snapshot APIs and V2 transaction builders */ class InCommitTimestampSuite extends AbstractInCommitTimestampSuite with WriteUtilsWithV2Builders /** * Runs in-commit timestamp tests using the legacy Table snapshot APIs and V1 transaction builders */ class LegacyInCommitTimestampSuite extends AbstractInCommitTimestampSuite with WriteUtilsWithV1Builders trait AbstractInCommitTimestampSuite extends AnyFunSuite { self: AbstractWriteUtils => private def getLogPath(engine: Engine, tablePath: String): Path = { val resolvedTablePath = engine.getFileSystemClient.resolvePath(tablePath) new Path(resolvedTablePath, "_delta_log") } private def removeCommitInfoFromCommit(engine: Engine, version: Long, logPath: Path): Unit = { val file = FileStatus.of(FileNames.deltaFile(logPath, version), 0, 0) val columnarBatches = engine.getJsonHandler.readJsonFiles( singletonCloseableIterator(file), SingleAction.FULL_SCHEMA, Optional.empty()) assert(columnarBatches.hasNext) val rows = columnarBatches.next().getRows val rowsWithoutCommitInfo = rows.filter(row => row.isNullAt(row.getSchema.indexOf("commitInfo"))) engine .getJsonHandler .writeJsonFileAtomically( FileNames.deltaFile(logPath, version), rowsWithoutCommitInfo, true /* overwrite */ ) } test("Enable ICT on commit 0") { withTempDirAndEngine { (tablePath, engine) => val beforeCommitAttemptStartTime = System.currentTimeMillis val clock = new ManualClock(beforeCommitAttemptStartTime + 1) setTablePropAndVerify( engine = engine, tablePath = tablePath, key = IN_COMMIT_TIMESTAMPS_ENABLED, value = "true", expectedValue = true, clock = clock) val ver0Snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath) assert(ver0Snapshot.getTimestamp(engine) === beforeCommitAttemptStartTime + 1) assert( getInCommitTimestamp( engine, tablePath, version = 0).get === ver0Snapshot.getTimestamp(engine)) assertHasWriterFeature(ver0Snapshot, "inCommitTimestamp") // Time travel should work val searchedSnapshot = getTableManagerAdapter.getSnapshotAtTimestamp( engine, tablePath, beforeCommitAttemptStartTime + 1) assert(searchedSnapshot.getVersion == 0) } } test("Create a non-inCommitTimestamp table and then enable ICT") { withTempDirAndEngine { (tablePath, engine) => val txn1 = getCreateTxn(engine, tablePath, testSchema) txn1.commit(engine, emptyIterable()) val ver0Snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath) assertMetadataProp(ver0Snapshot, IN_COMMIT_TIMESTAMPS_ENABLED, false) assertHasNoWriterFeature(ver0Snapshot, "inCommitTimestamp") assert(getInCommitTimestamp(engine, tablePath, version = 0).isEmpty) setTablePropAndVerify( engine = engine, tablePath = tablePath, isNewTable = false, key = IN_COMMIT_TIMESTAMPS_ENABLED, value = "true", expectedValue = true) val ver1Snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath) assertHasWriterFeature(ver1Snapshot, "inCommitTimestamp") assert(ver1Snapshot.getTimestamp(engine) > ver0Snapshot.getTimestamp(engine)) assert( getInCommitTimestamp( engine, tablePath, version = 1).get === ver1Snapshot.getTimestamp(engine)) // Time travel should work // Search timestamp = ICT enablement time - 1 val searchedSnapshot1 = getTableManagerAdapter.getSnapshotAtTimestamp( engine, tablePath, ver1Snapshot.getTimestamp(engine) - 1) assert(searchedSnapshot1.getVersion == 0) // Search timestamp = ICT enablement time val searchedSnapshot2 = getTableManagerAdapter.getSnapshotAtTimestamp( engine, tablePath, ver1Snapshot.getTimestamp(engine)) assert(searchedSnapshot2.getVersion == 1) } } test("InCommitTimestamps are monotonic even when the clock is skewed") { withTempDirAndEngine { (tablePath, engine) => val startTime = System.currentTimeMillis() val clock = new ManualClock(startTime) appendData( engine, tablePath, isNewTable = true, testSchema, data = Seq(Map.empty[String, Literal] -> dataBatches1), clock = clock, tableProperties = Map(IN_COMMIT_TIMESTAMPS_ENABLED.getKey -> "true")) val ver1Snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath) val ver1Timestamp = ver1Snapshot.getTimestamp(engine) assert(IN_COMMIT_TIMESTAMPS_ENABLED.fromMetadata(ver1Snapshot.getMetadata)) clock.setTime(startTime - 10000) appendData( engine, tablePath, data = Seq(Map.empty[String, Literal] -> dataBatches2), clock = clock) val ver2Snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath) val ver2Timestamp = ver2Snapshot.getTimestamp(engine) assert(ver2Timestamp === ver1Timestamp + 1) } } test("Missing CommitInfo should result in a DELTA_MISSING_COMMIT_INFO exception") { withTempDirAndEngine { (tablePath, engine) => setTablePropAndVerify( engine = engine, tablePath = tablePath, key = IN_COMMIT_TIMESTAMPS_ENABLED, value = "true", expectedValue = true) // Remove CommitInfo from the commit. removeCommitInfoFromCommit(engine, 0, getLogPath(engine, tablePath)) val ex = intercept[InvalidTableException] { getTableManagerAdapter.getSnapshotAtLatest( engine, tablePath).getTimestamp(engine) } assert(ex.getMessage.contains(String.format( "This table has the feature %s enabled which requires the presence of the " + "CommitInfo action in every commit. However, the CommitInfo action is " + "missing from commit version %s.", "inCommitTimestamp", "0"))) } } test("Missing CommitInfo.inCommitTimestamp should result in a " + "DELTA_MISSING_COMMIT_TIMESTAMP exception") { withTempDirAndEngine { (tablePath, engine) => setTablePropAndVerify( engine, tablePath, isNewTable = true, IN_COMMIT_TIMESTAMPS_ENABLED, "true", true) // Remove CommitInfo.inCommitTimestamp from the commit. val logPath = getLogPath(engine, tablePath) val file = FileStatus.of(FileNames.deltaFile(logPath, 0), 0, 0) val columnarBatches = engine.getJsonHandler.readJsonFiles( singletonCloseableIterator(file), SingleAction.FULL_SCHEMA, Optional.empty()) assert(columnarBatches.hasNext) val rows = columnarBatches.next().getRows val commitInfoOpt = CommitInfo.unsafeTryReadCommitInfoFromPublishedDeltaFile(engine, logPath, 0) assert(commitInfoOpt.isPresent) val commitInfo = commitInfoOpt.get commitInfo.setInCommitTimestamp(Optional.empty()) val rowsWithoutCommitInfoInCommitTimestamp = rows.map(row => { val commitInfoOrd = row.getSchema.indexOf("commitInfo") if (row.isNullAt(commitInfoOrd)) { row } else { createCommitInfoSingleAction(commitInfo.toRow) } }) engine .getJsonHandler .writeJsonFileAtomically( FileNames.deltaFile(logPath, 0), rowsWithoutCommitInfoInCommitTimestamp, true /* overwrite */ ) val ex = intercept[InvalidTableException] { getTableManagerAdapter.getSnapshotAtLatest( engine, tablePath).getTimestamp(engine) } assert(ex.getMessage.contains(String.format( "This table has the feature %s enabled which requires the presence of " + "inCommitTimestamp in the CommitInfo action. However, this field has not " + "been set in commit version %s.", "inCommitTimestamp", "0"))) } } test("Enablement tracking properties should not be added if ICT is enabled on commit 0") { withTempDirAndEngine { (tablePath, engine) => setTablePropAndVerify( engine = engine, tablePath = tablePath, key = IN_COMMIT_TIMESTAMPS_ENABLED, value = "true", expectedValue = true) val ver0Snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath) assertHasNoMetadataProp(ver0Snapshot, IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP) assertHasNoMetadataProp(ver0Snapshot, IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION) } } test("Enablement tracking works when ICT is enabled post commit 0") { withTempDirAndEngine { (tablePath, engine) => val txn = getCreateTxn(engine, tablePath, testSchema) txn.commit(engine, emptyIterable()) appendData( engine, tablePath, data = Seq(Map.empty[String, Literal] -> dataBatches1), tableProperties = Map(IN_COMMIT_TIMESTAMPS_ENABLED.getKey -> "true")) val ver1Snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath) assertMetadataProp(ver1Snapshot, IN_COMMIT_TIMESTAMPS_ENABLED, true) assertMetadataProp( ver1Snapshot, IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP, Optional.of(ver1Snapshot.getTimestamp(engine))) assertMetadataProp( ver1Snapshot, IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION, Optional.of(1L)) appendData( engine, tablePath, data = Seq(Map.empty[String, Literal] -> dataBatches2)) val ver2Snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath) assertMetadataProp(ver2Snapshot, IN_COMMIT_TIMESTAMPS_ENABLED, true) assertMetadataProp( ver2Snapshot, IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP, Optional.of(ver1Snapshot.getTimestamp(engine))) assertMetadataProp( ver2Snapshot, IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION, Optional.of(1L)) } } test("Update the protocol only if required") { withTempDirAndEngine { (tablePath, engine) => setTablePropAndVerify( engine = engine, tablePath = tablePath, key = IN_COMMIT_TIMESTAMPS_ENABLED, value = "true", expectedValue = true) val protocol = getProtocolActionFromCommit(engine, tablePath, 0) assert(protocol.isDefined) assert(VectorUtils.toJavaList(protocol.get.getArray(3)).contains("inCommitTimestamp")) setTablePropAndVerify( engine = engine, tablePath = tablePath, isNewTable = false, key = IN_COMMIT_TIMESTAMPS_ENABLED, value = "false", expectedValue = false) assert(getProtocolActionFromCommit(engine, tablePath, 1).isEmpty) setTablePropAndVerify( engine = engine, tablePath = tablePath, isNewTable = false, key = IN_COMMIT_TIMESTAMPS_ENABLED, value = "true", expectedValue = true) assert(getProtocolActionFromCommit(engine, tablePath, 2).isEmpty) } } test("Metadata toString should work with ICT enabled") { withTempDirAndEngine { (tablePath, engine) => val txn = getCreateTxn(engine, tablePath, testSchema) txn.commit(engine, emptyIterable()) appendData( engine, tablePath, data = Seq(Map.empty[String, Literal] -> dataBatches1), tableProperties = Map(IN_COMMIT_TIMESTAMPS_ENABLED.getKey -> "true")) val metadata = getTableManagerAdapter.getSnapshotAtLatest( engine, tablePath).getMetadata val inCommitTimestamp = getInCommitTimestamp(engine, tablePath, version = 1).get assert(metadata.toString == String.format( "Metadata{id='%s', name=Optional.empty, description=Optional.empty, " + "format=Format{provider='parquet', options={}}, " + "schemaString='{\"type\":\"struct\",\"fields\":[{" + "\"name\":\"id\",\"type\":\"integer\",\"nullable\":true," + "\"metadata\":{}}]}', " + "partitionColumns=List(), createdTime=Optional[%s], " + "configuration={delta.inCommitTimestampEnablementTimestamp=%s, " + "delta.enableInCommitTimestamps=true, " + "delta.inCommitTimestampEnablementVersion=1}}", metadata.getId, metadata.getCreatedTime.get, inCommitTimestamp.toString)) } } test("Table with ICT enabled is readable") { withTempDirAndEngine { (tablePath, engine) => val txn = getCreateTxn(engine, tablePath, testSchema) txn.commit(engine, emptyIterable()) val commitResult = appendData( engine, tablePath, data = Seq(Map.empty[String, Literal] -> dataBatches1), tableProperties = Map(IN_COMMIT_TIMESTAMPS_ENABLED.getKey -> "true")) val ver1Snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath) assertMetadataProp(ver1Snapshot, IN_COMMIT_TIMESTAMPS_ENABLED, true) assertMetadataProp( ver1Snapshot, IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP, Optional.of(getInCommitTimestamp(engine, tablePath, version = 1).get)) assertMetadataProp( ver1Snapshot, IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION, Optional.of(1L)) val expData = dataBatches1.flatMap(_.toTestRows) verifyCommitResult(commitResult, expVersion = 1, expIsReadyForCheckpoint = false) verifyCommitInfo(tablePath, version = 1, partitionCols = null) verifyWrittenContent(tablePath, testSchema, expData) verifyTableProperties( tablePath, ListMap( // appendOnly, invariants implicitly supported as the protocol is upgraded from 2 to 7 // These properties are not set in the table properties, but are generated by the // Spark describe IN_COMMIT_TIMESTAMPS_ENABLED.getKey -> true, "delta.feature.appendOnly" -> "supported", "delta.feature.inCommitTimestamp" -> "supported", "delta.feature.invariants" -> "supported", IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.getKey -> getInCommitTimestamp(engine, tablePath, version = 1).get, IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.getKey -> 1L), 1, 7) } } /** * Helper method to read the inCommitTimestamp from the commit file of the given version if it * is not null, otherwise return null. */ private def getInCommitTimestamp( engine: Engine, tablePath: String, version: Long): Option[Long] = { val commitInfoOpt = CommitInfo.unsafeTryReadCommitInfoFromPublishedDeltaFile( engine, getLogPath(engine, tablePath), version) if (commitInfoOpt.isPresent && commitInfoOpt.get.getInCommitTimestamp.isPresent) { Some(commitInfoOpt.get.getInCommitTimestamp.get) } else { Option.empty } } test("Conflict resolution of timestamps") { withTempDirAndEngine { (tablePath, engine) => setTablePropAndVerify( engine, tablePath, isNewTable = true, IN_COMMIT_TIMESTAMPS_ENABLED, "true", true) val startTime = System.currentTimeMillis() val clock = new ManualClock(startTime) val txn1 = getUpdateTxn( engine, tablePath, clock = clock) clock.setTime(startTime) appendData( engine, tablePath, data = Seq(Map.empty[String, Literal] -> dataBatches2), clock = clock) clock.setTime(startTime - 1000) commitAppendData(engine, txn1, Seq(Map.empty[String, Literal] -> dataBatches1)) assert( getInCommitTimestamp(engine, tablePath, version = 2).get === getInCommitTimestamp(engine, tablePath, version = 1).get + 1) } } Seq(10, 2).foreach { winningCommitCount => test(s"Conflict resolution of enablement version(Winning Commit Count=$winningCommitCount)") { withTempDirAndEngine { (tablePath, engine) => val txn = getCreateTxn(engine, tablePath, testSchema) txn.commit(engine, emptyIterable()) val startTime = System.currentTimeMillis() // we need to fix this now! val txn1 = getUpdateTxn( engine, tablePath, tableProperties = Map(IN_COMMIT_TIMESTAMPS_ENABLED.getKey -> "true"), clock = () => startTime) // Sleep for 1 second to ensure that due to file-system truncation, the timestamp for these // non-ict commits is not less than the fixed time for the txn above // If this happens, nothing incorrect happens, but the // ictEnablementTimestamp != prevVersion.timestamp + 1 since we will use the greater ts Thread.sleep(1000) for (_ <- 0 until winningCommitCount) { appendData( engine, tablePath, data = Seq(Map.empty[String, Literal] -> dataBatches2)) } commitAppendData(engine, txn1, Seq(Map.empty[String, Literal] -> dataBatches1)) val lastSnapshot = getTableManagerAdapter.getSnapshotAtVersion( engine, tablePath, winningCommitCount) val curSnapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath) val observedEnablementTimestamp = IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.fromMetadata(curSnapshot.getMetadata) val observedEnablementVersion = IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.fromMetadata(curSnapshot.getMetadata) assert(observedEnablementTimestamp.get === lastSnapshot.getTimestamp(engine) + 1) assert( observedEnablementTimestamp.get === getInCommitTimestamp(engine, tablePath, version = winningCommitCount + 1).get) assert(observedEnablementVersion.get === winningCommitCount + 1) } } } test("Missing CommitInfo in last winning commit in conflict resolution should result in a " + "DELTA_MISSING_COMMIT_INFO exception") { withTempDirAndEngine { (tablePath, engine) => setTablePropAndVerify( engine, tablePath, isNewTable = true, IN_COMMIT_TIMESTAMPS_ENABLED, "true", true) val startTime = System.currentTimeMillis() val clock = new ManualClock(startTime) val txn1 = getUpdateTxn( engine, tablePath, clock = clock) clock.setTime(startTime) appendData( engine, tablePath, data = Seq(Map.empty[String, Literal] -> dataBatches2), clock = clock) appendData( engine, tablePath, data = Seq(Map.empty[String, Literal] -> dataBatches2), clock = clock) // Remove CommitInfo from the commit. removeCommitInfoFromCommit(engine, 2, getLogPath(engine, tablePath)) clock.setTime(startTime - 1000) val ex = intercept[InvalidTableException] { commitAppendData(engine, txn1, Seq(Map.empty[String, Literal] -> dataBatches1)) } assert(ex.getMessage.contains(String.format( "This table has the feature %s enabled which requires the presence of the " + "CommitInfo action in every commit. However, the CommitInfo action is " + "missing from commit version %s.", "inCommitTimestamp", "2"))) } } test("Throw an error where the winning txn enables the ICT and losing txn prepares txn with " + "ICT enabled") { withTempDirAndEngine { (tablePath, engine) => val txn = getCreateTxn(engine, tablePath, testSchema) txn.commit(engine, emptyIterable()) val startTime = System.currentTimeMillis() val clock = new ManualClock(startTime) val txn1 = getUpdateTxn( engine, tablePath, tableProperties = Map(IN_COMMIT_TIMESTAMPS_ENABLED.getKey -> "true"), clock = clock) clock.setTime(startTime) appendData( engine, tablePath, data = Seq(Map.empty[String, Literal] -> dataBatches2), tableProperties = Map(IN_COMMIT_TIMESTAMPS_ENABLED.getKey -> "true"), clock = clock) clock.setTime(startTime - 1000) val ex = intercept[ProtocolChangedException] { commitAppendData(engine, txn1, Seq(Map.empty[String, Literal] -> dataBatches1)) } assert(ex.getMessage.contains(String.format("Transaction has encountered a conflict and " + "can not be committed. Query needs to be re-executed using the latest version of the " + "table."))) } } test("Disabling ICT removes enablement tracking properties") { withTempDirAndEngine { (tablePath, engine) => // ===== GIVEN ===== createTableThenEnableIctAndVerify(engine, tablePath) // ===== WHEN ===== // Disable ICT. This should remove enablement tracking properties. setTablePropAndVerify( engine = engine, tablePath = tablePath, isNewTable = false, key = IN_COMMIT_TIMESTAMPS_ENABLED, value = "false", expectedValue = false) // ===== THEN ===== val snapshotV2 = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath) assertMetadataProp(snapshotV2, IN_COMMIT_TIMESTAMPS_ENABLED, false) assertHasNoMetadataProp(snapshotV2, IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP) assertHasNoMetadataProp(snapshotV2, IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION) } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/LogCompactionSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults // import scala.collection.JavaConverters._ import scala.collection.immutable.Seq import io.delta.kernel.Table import io.delta.kernel.defaults.engine.hadoopio.HadoopFileIO import io.delta.kernel.defaults.utils.{AbstractTestUtils, TestRow, TestUtilsWithLegacyKernelAPIs, TestUtilsWithTableManagerAPIs, WriteUtils} import io.delta.kernel.expressions.Literal import io.delta.kernel.internal.SnapshotImpl import io.delta.kernel.internal.TableConfig import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.hook.LogCompactionHook import org.apache.hadoop.conf.Configuration import org.scalatest.funsuite.AnyFunSuite class LogCompactionSuite extends AbstractLogCompactionSuite with TestUtilsWithTableManagerAPIs class LegacyLogCompactionSuite extends AbstractLogCompactionSuite with TestUtilsWithLegacyKernelAPIs trait AbstractLogCompactionSuite extends AnyFunSuite with WriteUtils { self: AbstractTestUtils => test("Compaction containing different action types") { withTempDirAndEngine { (tblPath, engine) => // commit 0 - add data appendData( engine, tblPath, isNewTable = true, testSchema, data = Seq(Map.empty[String, Literal] -> dataBatches1)) // commit 1 - set a metadata prop val newTblProps = Map(TableConfig.CHECKPOINT_POLICY.getKey -> "v2") updateTableMetadata(engine, tblPath, tableProperties = newTblProps) // commit 2 - add domain metadata val dmTxn = getUpdateTxn( engine, tblPath, withDomainMetadataSupported = true) dmTxn.addDomainMetadata("testDomain", "testConfig") commitAppendData(engine, dmTxn, Seq.empty) // commit 3 - add more data appendData( engine, tblPath, data = Seq(Map.empty[String, Literal] -> dataBatches2)) // create the compaction file(s) val dataPath = new Path(s"file:${tblPath}") val logPath = new Path(s"file:${tblPath}", "_delta_log") val hook = new LogCompactionHook(dataPath, logPath, 0, 3, 0) hook.threadSafeInvoke(engine) val hadoopFileIO = new HadoopFileIO(new Configuration()) val metricEngine = new MetricsEngine(hadoopFileIO) val snapshot = getTableManagerAdapter.getSnapshotAtLatest(metricEngine, tblPath) val checkpointProp = snapshot.getMetadata().getConfiguration.get(TableConfig.CHECKPOINT_POLICY.getKey) assert(checkpointProp == "v2") // this is the read that the snapshot did val propCompactionsRead = metricEngine.getJsonHandler.getCompactionsRead assert(propCompactionsRead.toSet === Set((0, 3))) metricEngine.resetMetrics() val domainMetadata = snapshot.getDomainMetadata("testDomain") assert(domainMetadata.isPresent()) assert(domainMetadata.get == "testConfig") // getting domain metadata requires another log-reply, so check that this one also used the // compaction val dmCompactionsRead = metricEngine.getJsonHandler.getCompactionsRead assert(dmCompactionsRead.toSet === Set((0, 3))) // ensure the data is all there metricEngine.resetMetrics() val expectedAnswer = dataBatches1.flatMap(_.toTestRows) ++ dataBatches2.flatMap(_.toTestRows) checkTable(tblPath, expectedAnswer, engine = metricEngine) val readCompactionsRead = metricEngine.getJsonHandler.getCompactionsRead assert(readCompactionsRead.toSet === Set((0, 3))) } } def testWithCompactions( versionsToWrite: Seq[Int], // highest version MUST be last! versionToRead: Option[Long], doRemoves: Boolean, compactions: Seq[(Int, Int)], expectedDeltasToBeRead: Set[Int], expectedCompactionsToBeRead: Set[(Int, Int)]) { withTempDir { tmpDir => val tablePath = tmpDir.getCanonicalPath val hadoopFileIO = new HadoopFileIO(new Configuration() { { // Set the batch sizes to small so that we get to test the multiple batch scenarios. set("delta.kernel.default.parquet.reader.batch-size", "2"); set("delta.kernel.default.json.reader.batch-size", "2"); } }) val engine = new MetricsEngine(hadoopFileIO) var expectedRows: Set[Long] = Set() versionsToWrite.foreach { i => // if we're removing, then on odd commits, remove the lower 10 of the previous 20 rows added if (doRemoves && i % 2 == 1) { val prev = i - 1; val low = prev * 10 val high = prev * 10 + 10 val deleteQuery = "DELETE FROM delta.`%s` WHERE id >= %d AND id < %d".format( tablePath, low, high) spark.sql(deleteQuery) if (versionToRead.isEmpty || versionToRead.get >= i) { expectedRows --= (low until high).map(i => i.toLong) } // if (i == compactions(0).1) { // // ensure we put a DM in a compaction // } } else { val low = i * 10 // if we're removing, add 20 rows as the first 10 will be removed by the next version, // otherwise add 10 rows val high = if (doRemoves) low + 20 else low + 10 spark.range(low, high).write .format("delta") .mode("append") .save(tablePath) if (versionToRead.isEmpty || versionToRead.get >= i) { expectedRows ++= (low until high).map(i => i.toLong) } } } val dataPath = new Path(s"file:${tablePath}") val logPath = new Path(s"file:${tablePath}", "_delta_log") // create the compaction file(s) compactions.foreach { compaction => val hook = new LogCompactionHook( dataPath, logPath, compaction._1, compaction._2, 0) hook.threadSafeInvoke(engine) } engine.resetMetrics() checkTable( path = tablePath, expectedAnswer = expectedRows.toSeq.map(i => TestRow(i)), engine = engine, version = versionToRead) val actualJsonVersionsRead = engine.getJsonHandler.getVersionsRead val actualCompactionsRead = engine.getJsonHandler.getCompactionsRead assert(actualJsonVersionsRead.toSet == expectedDeltasToBeRead) assert(actualCompactionsRead.toSet == expectedCompactionsToBeRead) } } Seq(Seq((0, 3)), Seq((3, 5)), Seq((5, 9)), Seq((0, 3), (5, 8))).foreach { compactions => Seq(true, false).foreach { doRemoves => val compactionStr = compactions.mkString(", ") test(s"Compaction(s) at $compactionStr (no checkpoint, removes: $doRemoves)") { // for these tests, write 0 - 9 (inclusive) val versionsToWrite = (0 to 9) var expectedDeltasToBeRead = versionsToWrite.toSet compactions.foreach { compaction => // subtract out the compaction versions from the full set expectedDeltasToBeRead &~= (compaction._1 to compaction._2).toSet } testWithCompactions( versionsToWrite, versionToRead = None, doRemoves, compactions, expectedDeltasToBeRead, compactions.toSet) } } } Seq(Seq((3, 5)), Seq((8, 11)), Seq((8, 12), (11, 15)), Seq((11, 13), (15, 17))).foreach { compactions => Seq(true, false).foreach { doRemoves => val compactionStr = compactions.mkString(", ") test(s"Compaction(s) at $compactionStr (with checkpoint, removes: $doRemoves)") { // for these tests, write 0 - 19 (inclusive), will checkpoint at 10 val versionsToWrite = (0 to 19) val versionsAfterCheckpoint = (11 to 19) var expectedDeltasToBeRead = versionsAfterCheckpoint.toSet var expectedCompactionsToBeRead = Set[(Int, Int)]() compactions.foreach { compaction => if (compaction._1 > 10) { // only use if after checkpoint // subtract out the compaction versions from the full set expectedDeltasToBeRead &~= (compaction._1 to compaction._2).toSet // add to expected compactions expectedCompactionsToBeRead += compaction } } testWithCompactions( versionsToWrite, versionToRead = None, doRemoves, compactions, expectedDeltasToBeRead, expectedCompactionsToBeRead) } } } test("Compaction with overlap") { testWithCompactions( versionsToWrite = (0 to 9), versionToRead = None, doRemoves = true, compactions = Seq((0, 3), (2, 4)), expectedDeltasToBeRead = Set(0, 1, 5, 6, 7, 8, 9), expectedCompactionsToBeRead = Set((2, 4))) } test("Compaction is whole range") { testWithCompactions( versionsToWrite = (0 to 5), versionToRead = None, doRemoves = true, compactions = Seq((0, 5)), expectedDeltasToBeRead = Set(), expectedCompactionsToBeRead = Set((0, 5))) } test("Compaction out of range") { testWithCompactions( versionsToWrite = (0 to 9), versionToRead = Some(6), doRemoves = true, compactions = Seq((1, 3), (5, 8)), expectedDeltasToBeRead = Set(0, 4, 5, 6), expectedCompactionsToBeRead = Set((1, 3))) } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/LogCompactionWriterSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.util.{Collections, Optional} import scala.collection.JavaConverters._ import scala.collection.immutable.Seq import io.delta.kernel.Table import io.delta.kernel.data.Row import io.delta.kernel.defaults.utils.TestRow import io.delta.kernel.engine.Engine import io.delta.kernel.expressions.Literal import io.delta.kernel.hook.PostCommitHook import io.delta.kernel.internal.{DeltaLogActionUtils, SnapshotImpl} import io.delta.kernel.internal.TableConfig.TOMBSTONE_RETENTION import io.delta.kernel.internal.actions._ import io.delta.kernel.internal.compaction.LogCompactionWriter import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.hook.LogCompactionHook import io.delta.kernel.internal.replay.ActionsIterator import io.delta.kernel.internal.util.FileNames.DeltaLogFileType import io.delta.kernel.internal.util.ManualClock import io.delta.kernel.internal.util.Utils.singletonCloseableIterator import io.delta.kernel.types.{IntegerType, StructType} import io.delta.kernel.utils.FileStatus import org.apache.spark.sql.delta.{DeltaLog, DomainMetadataTableFeature} import org.apache.spark.sql.delta.DeltaOperations.Truncate import org.apache.spark.sql.delta.actions.{DomainMetadata => DeltaSparkDomainMetadata, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.sources.DeltaSQLConf /** * Test suite for io.delta.kernel.internal.compaction.LogCompactionWriter */ class LogCompactionWriterSuite extends CheckpointBase { val COMPACTED_SCHEMA = new StructType() .add("txn", SetTransaction.FULL_SCHEMA) .add("add", AddFile.FULL_SCHEMA) .add("remove", RemoveFile.FULL_SCHEMA) .add("metaData", Metadata.FULL_SCHEMA) .add("protocol", Protocol.FULL_SCHEMA) // .add("cdc", new StructType()) .add("domainMetadata", DomainMetadata.FULL_SCHEMA); val ADD_INDEX = 1 val REMOVE_INDEX = 2 val METADATA_INDEX = 3 val PROTOCOL_INDEX = 4 val DM_INDEX = 5 val ADD_REM_PATH_INDEX = 0 val DM_NAME_INDEX = 0 // check if a row is all null def rowIsNull(row: Row): Boolean = { val schema = row.getSchema() for (ordinal <- 0 until schema.length()) { if (!row.isNullAt(ordinal)) { return false } } true } // Get the expected actions from the log. We filter down to just the actions that end up in a // compacted log file, and also filter out adds that have been removed as well as duplicate // metadata/protocol actions def getActionsFromLog( tablePath: Path, engine: Engine, startVersion: Long, endVersion: Long): Seq[TestRow] = { val files = DeltaLogActionUtils.listDeltaLogFilesAsIter( engine, Collections.singleton(DeltaLogFileType.COMMIT), tablePath, startVersion, Optional.of(endVersion), false /* mustBeRecreatable */ ) .toInMemoryList() Collections.reverse(files) // we want things in reverse order val actions = new ActionsIterator(engine, files, COMPACTED_SCHEMA, Optional.empty()) val removed = scala.collection.mutable.HashSet.empty[String] val seenDomains = scala.collection.mutable.HashSet.empty[String] var seenMetadata = false var seenProtocol = false actions.toSeq.flatMap { wrapper => wrapper.getColumnarBatch().getRows.toSeq.flatMap { row => if (!row.isNullAt(REMOVE_INDEX)) { val removeRow = row.getStruct(REMOVE_INDEX) val path = removeRow.getString(ADD_REM_PATH_INDEX) removed += path } if (!row.isNullAt(ADD_INDEX)) { val addRow = row.getStruct(ADD_INDEX) val path = addRow.getString(ADD_REM_PATH_INDEX) if (!removed.contains(path)) { Some(TestRow(row)) } else { None } } else if (!row.isNullAt(METADATA_INDEX)) { if (!seenMetadata) { seenMetadata = true Some(TestRow(row)) } else { None } } else if (!row.isNullAt(PROTOCOL_INDEX)) { if (!seenProtocol) { seenProtocol = true Some(TestRow(row)) } else { None } } else if (!row.isNullAt(DM_INDEX)) { val dm = row.getStruct(DM_INDEX) val domain = dm.getString(DM_NAME_INDEX) if (!seenDomains.contains(domain)) { seenDomains += domain Some(TestRow(row)) } else { None } } else if (!rowIsNull(row)) { Some(TestRow(row)) } else { None } } } } def getActionsFromCompacted( compactedPath: String, engine: Engine): Seq[Row] = { val fileStatus = FileStatus.of(compactedPath, 0, 0) val batches = engine .getJsonHandler() .readJsonFiles( singletonCloseableIterator(fileStatus), COMPACTED_SCHEMA, Optional.empty()) batches.toSeq.flatMap(_.getRows().toSeq) } def addDomainMetadata(path: String, d1Val: String, d2Val: String): Unit = { spark.sql( s""" |ALTER TABLE delta.`$path` |SET TBLPROPERTIES |('${TableFeatureProtocolUtils.propertyKey(DomainMetadataTableFeature)}' = 'enabled') |""".stripMargin) val deltaLog = DeltaLog.forTable(spark, path) val domainMetadata = DeltaSparkDomainMetadata("testDomain1", d1Val, false) :: DeltaSparkDomainMetadata("testDomain2", d2Val, false) :: Nil deltaLog.startTransaction().commit(domainMetadata, Truncate()) } Seq(false, true).foreach { includeRemoves => Seq(false, true).foreach { includeDM => val removesMsg = if (includeRemoves) " and removes" else "" val dmMsg = if (includeDM) ", include multiple PandM and DomainMetadata" else "" test(s"commits containing adds${removesMsg}${dmMsg}") { withTempDirAndEngine { (tablePath, engine) => addData(tablePath, alternateBetweenAddsAndRemoves = includeRemoves, numberIter = 8) if (includeDM) { addDomainMetadata(tablePath, "", "{\"key1\":\"value1\"}") addDomainMetadata(tablePath, "here", "{\"key2\":\"value2\"}") } val expectedLastCommit = if (includeDM) { 11 // 0-7 for add/removes + 2 for enable+set DomainMetatdata } else { 7 // 0-7 for add/removes } val actionsFromCommits = getActionsFromLog(new Path(tablePath), engine, 0, expectedLastCommit) val dataPath = new Path(s"file:${tablePath}") val logPath = new Path(s"file:${tablePath}", "_delta_log") val hook = new LogCompactionHook( dataPath, logPath, 0, expectedLastCommit, 0) hook.threadSafeInvoke(engine) val endCommitStr = f"$expectedLastCommit%020d" val compactedPath = tablePath + s"/_delta_log/00000000000000000000.${endCommitStr}.compacted.json" val actionsFromCompacted = getActionsFromCompacted(compactedPath, engine) checkAnswer(actionsFromCompacted, actionsFromCommits) } } } } Seq(false, true).foreach { includeRemoves => val testMsgUpdate = if (includeRemoves) " and removes" else "" test(s"Read table with adds$testMsgUpdate") { withTempDirAndEngine { (tablePath, engine) => addData(tablePath, alternateBetweenAddsAndRemoves = includeRemoves, numberIter = 10) spark.conf.set(DeltaSQLConf.DELTALOG_MINOR_COMPACTION_USE_FOR_READS.key, "false") val withoutCompactionData = readUsingSpark(tablePath) val dataPath = new Path(s"file:${tablePath}") val logPath = new Path(s"file:${tablePath}", "_delta_log") val hook = new LogCompactionHook(dataPath, logPath, 0, 9, 0) hook.threadSafeInvoke(engine) spark.conf.set(DeltaSQLConf.DELTALOG_MINOR_COMPACTION_USE_FOR_READS.key, "true") val withCompactionData = readUsingSpark(tablePath) checkAnswer(withCompactionData, withoutCompactionData) } } } test(s"Error if not enough commits") { withTempDirAndEngine { (tablePath, engine) => addData(tablePath, alternateBetweenAddsAndRemoves = false, numberIter = 2) val dataPath = new Path(s"file:${tablePath}") val logPath = new Path(s"file:${tablePath}", "_delta_log") val hook = new LogCompactionHook(dataPath, logPath, 0, 5, 0) val ex = intercept[IllegalArgumentException] { hook.threadSafeInvoke(engine) } assert(ex.getMessage.contains( "Asked to compact between versions 0 and 5, but found 2 delta files")) } } test("Hook is generated correctly and when expected") { withTempDirAndEngine { (tablePath, engine) => val clock = new ManualClock(0) val schema = new StructType().add("col", IntegerType.INTEGER) val dataPath = new Path(s"file:${tablePath}") val logPath = new Path(s"file:${tablePath}", "_delta_log") createEmptyTable(engine, tablePath, schema, clock = clock) val table = Table.forPath(engine, tablePath) val metadata = table.getLatestSnapshot(engine).asInstanceOf[SnapshotImpl].getMetadata() val tombstoneRetention = TOMBSTONE_RETENTION.fromMetadata(metadata) clock.setTime(tombstoneRetention) // set to the retention time so (time - retention) == 0 val compactionInterval = 3 var hooksFound = 0 // start at 1 since the create of the table is 0 for (commitNum <- 1 to 5) { val txn = getUpdateTxn(engine, tablePath, clock = clock, logCompactionInterval = compactionInterval) val data = generateData( schema, Seq.empty, Map.empty[String, Literal], batchSize = 1, numBatches = 1) val commitResult = commitAppendData(engine, txn, data = Seq(Map.empty[String, Literal] -> data)) // expect every compactionInterval val expectHook = ((commitNum + 1) % compactionInterval == 0) assert(LogCompactionWriter.shouldCompact(commitNum, compactionInterval) == expectHook) var foundHook = false for (hook <- commitResult.getPostCommitHooks().asScala) { if (hook.getType() == PostCommitHook.PostCommitHookType.LOG_COMPACTION) { assert(!foundHook) // there should never be more than one foundHook = true hooksFound += 1 val logCompactionHook = hook.asInstanceOf[LogCompactionHook] assert(logCompactionHook.getDataPath() == dataPath) assert(logCompactionHook.getLogPath() == logPath) assert(logCompactionHook.getStartVersion() == commitNum + 1 - compactionInterval) assert(logCompactionHook.getCommitVersion() == commitNum) assert(logCompactionHook.getMinFileRetentionTimestampMillis() == 0) } hook.threadSafeInvoke(engine) } assert(foundHook == expectHook) } assert(hooksFound == 2) // expect 0<->2 and 3<->5 } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/LogReplayBaseSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.util.Optional import scala.collection.convert.ImplicitConversions._ import scala.collection.mutable.ArrayBuffer import io.delta.kernel.Table import io.delta.kernel.data.ColumnarBatch import io.delta.kernel.defaults.engine.{DefaultEngine, DefaultJsonHandler, DefaultParquetHandler} import io.delta.kernel.defaults.engine.fileio.FileIO import io.delta.kernel.defaults.engine.hadoopio.HadoopFileIO import io.delta.kernel.defaults.utils.AbstractTestUtils import io.delta.kernel.engine.{Engine, ExpressionHandler, FileSystemClient} import io.delta.kernel.engine.FileReadResult import io.delta.kernel.expressions.Predicate import io.delta.kernel.internal.checkpoints.Checkpointer import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.util.FileNames import io.delta.kernel.internal.util.Utils.toCloseableIterator import io.delta.kernel.types.StructType import io.delta.kernel.utils.{CloseableIterator, FileStatus} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.hadoop.conf.Configuration import org.scalatest.funsuite.AnyFunSuite /** * Base trait containing shared code for log replay metric testing. * * This trait provides common infrastructure for testing how Delta log files are read * during table operations, with utilities for metrics collection and verification. */ trait LogReplayBaseSuite extends AnyFunSuite { self: AbstractTestUtils => protected def withTempDirAndMetricsEngine(f: (String, MetricsEngine) => Unit): Unit = { val hadoopFileIO = new HadoopFileIO(new Configuration() { { // Set the batch sizes to small so that we get to test the multiple batch scenarios. set("delta.kernel.default.parquet.reader.batch-size", "2"); set("delta.kernel.default.json.reader.batch-size", "2"); } }) val engine = new MetricsEngine(hadoopFileIO) withTempDir { dir => f(dir.getAbsolutePath, engine) } } protected def loadPandMCheckMetrics( tablePath: String, engine: MetricsEngine, expJsonVersionsRead: Seq[Long], expParquetVersionsRead: Seq[Long], expParquetReadSetSizes: Seq[Long] = null, expChecksumReadSet: Seq[Long] = null, version: Long = -1): Unit = { engine.resetMetrics() version match { case -1 => getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath) case ver => getTableManagerAdapter.getSnapshotAtVersion(engine, tablePath, version) } assertMetrics( engine, expJsonVersionsRead, expParquetVersionsRead, expParquetReadSetSizes, expChecksumReadSet = expChecksumReadSet) } protected def assertMetrics( engine: MetricsEngine, expJsonVersionsRead: Seq[Long], expParquetVersionsRead: Seq[Long], expParquetReadSetSizes: Seq[Long] = null, expLastCheckpointReadCalls: Option[Int] = None, expChecksumReadSet: Seq[Long] = null): Unit = { val actualJsonVersionsRead = engine.getJsonHandler.getVersionsRead val actualParquetVersionsRead = engine.getParquetHandler.getVersionsRead assert( actualJsonVersionsRead === expJsonVersionsRead, s"Expected to read json versions " + s"$expJsonVersionsRead but read $actualJsonVersionsRead") assert( actualParquetVersionsRead === expParquetVersionsRead, s"Expected to read parquet " + s"versions $expParquetVersionsRead but read $actualParquetVersionsRead") if (expParquetReadSetSizes != null) { val actualParquetReadSetSizes = engine.getParquetHandler.checkpointReadRequestSizes assert( actualParquetReadSetSizes === expParquetReadSetSizes, s"Expected parquet read set sizes " + s"$expParquetReadSetSizes but read $actualParquetReadSetSizes") } expLastCheckpointReadCalls.foreach { expCalls => val actualCalls = engine.getJsonHandler.getLastCheckpointMetadataReadCalls assert( actualCalls === expCalls, s"Expected to read last checkpoint metadata $expCalls times but read $actualCalls times") } if (expChecksumReadSet != null) { val actualChecksumReadSet = engine.getJsonHandler.checksumsRead assert( actualChecksumReadSet === expChecksumReadSet, s"Expected checksum read set " + s"$expChecksumReadSet but read $actualChecksumReadSet") } } protected def appendCommit(path: String): Unit = spark.range(10).write.format("delta").mode("append").save(path) /** * Creates a temporary directory with a test engine and builds a table with CRC files. * Returns the created table path and engine for testing. * * @param f code to run with the table and engine */ protected def withTableWithCrc(f: (String, MetricsEngine) => Any): Unit = { withTempDirAndMetricsEngine { (path, engine) => // Produce a test table with 0 to 11 .json, 0 to 11.crc, 10.checkpoint.parquet withSQLConf(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> "true") { spark.sql( s"CREATE TABLE delta.`$path` USING DELTA AS " + s"SELECT 0L as id") for (_ <- 0 to 10) { appendCommit(path) } assert(checkpointFileExistsForTable(path, 10)) } f(path, engine) } } /** * Creates a temporary directory with a test engine and builds a table with CRC files. * Returns the created table and engine for testing. * * @param f code to run with the table and engine */ protected def withTableWithCrc(f: (Table, String, MetricsEngine) => Any): Unit = { withTableWithCrc((path, engine) => { val table = Table.forPath(engine, path) f(table, path, engine) }) } } //////////////////// // Helper Classes // //////////////////// /** An engine that records the Delta commit (.json) and checkpoint (.parquet) files read */ class MetricsEngine(fileIO: FileIO) extends Engine { private val impl = DefaultEngine.create(fileIO) private val jsonHandler = new MetricsJsonHandler(fileIO) private val parquetHandler = new MetricsParquetHandler(fileIO) def resetMetrics(): Unit = { jsonHandler.resetMetrics() parquetHandler.resetMetrics() } override def getExpressionHandler: ExpressionHandler = impl.getExpressionHandler override def getJsonHandler: MetricsJsonHandler = jsonHandler override def getFileSystemClient: FileSystemClient = impl.getFileSystemClient override def getParquetHandler: MetricsParquetHandler = parquetHandler } /** * Helper trait which wraps an underlying json/parquet read and collects the versions (e.g. 10.json, * 10.checkpoint.parquet) read */ trait FileReadMetrics { self: Object => // number of times read is requested on `_last_checkpoint` private var lastCheckpointMetadataReadCalls = 0 val checksumsRead = new ArrayBuffer[Long]() // versions of checksum files read private val versionsRead = ArrayBuffer[Long]() private val compactionVersionsRead = ArrayBuffer[(Long, Long)]() // Number of checkpoint files requested read in each readParquetFiles call val checkpointReadRequestSizes = new ArrayBuffer[Long]() private def updateVersionsRead(fileStatus: FileStatus): Unit = { val path = new Path(fileStatus.getPath) if (FileNames.isCommitFile(path.getName) || FileNames.isCheckpointFile(path.getName)) { val version = FileNames.getFileVersion(path) // We may split json/parquet reads, so don't record the same file multiple times if (!versionsRead.contains(version)) { versionsRead += version } } else if (Checkpointer.LAST_CHECKPOINT_FILE_NAME.equals(path.getName)) { lastCheckpointMetadataReadCalls += 1 } else if (FileNames.isChecksumFile(path.getName)) { checksumsRead += FileNames.getFileVersion(path) } else if (FileNames.isLogCompactionFile(path.getName)) { val versions = FileNames.logCompactionVersions(path) compactionVersionsRead += ((versions._1, versions._2)) } } def getVersionsRead: Seq[Long] = versionsRead.toSeq def getCompactionsRead: Seq[(Long, Long)] = compactionVersionsRead.toSeq def getLastCheckpointMetadataReadCalls: Int = lastCheckpointMetadataReadCalls def resetMetrics(): Unit = { lastCheckpointMetadataReadCalls = 0 versionsRead.clear() compactionVersionsRead.clear() checkpointReadRequestSizes.clear() checksumsRead.clear() } def collectReadFiles(fileIter: CloseableIterator[FileStatus]): CloseableIterator[FileStatus] = { fileIter.map(file => { updateVersionsRead(file) file }) } } /** A JsonHandler that collects metrics on the Delta commit (.json) files read */ class MetricsJsonHandler(fileIO: FileIO) extends DefaultJsonHandler(fileIO) with FileReadMetrics { override def readJsonFiles( fileIter: CloseableIterator[FileStatus], physicalSchema: StructType, predicate: Optional[Predicate]): CloseableIterator[ColumnarBatch] = { super.readJsonFiles(collectReadFiles(fileIter), physicalSchema, predicate) } } /** A ParquetHandler that collects metrics on the Delta checkpoint (.parquet) files read */ class MetricsParquetHandler(fileIO: FileIO) extends DefaultParquetHandler(fileIO) with FileReadMetrics { override def readParquetFiles( fileIter: CloseableIterator[FileStatus], physicalSchema: StructType, predicate: Optional[Predicate]): CloseableIterator[FileReadResult] = { val fileReadSet = fileIter.toSeq checkpointReadRequestSizes += fileReadSet.size super.readParquetFiles( collectReadFiles(toCloseableIterator(fileReadSet.iterator)), physicalSchema, predicate) } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/LogReplayEngineMetricsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.io.File import java.nio.file.Files import io.delta.kernel.defaults.utils.{AbstractTestUtils, TestUtilsWithLegacyKernelAPIs, TestUtilsWithTableManagerAPIs} import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.scalatest.BeforeAndAfterAll class LegacyLogReplayEngineMetricsSuite extends AbstractLogReplayEngineMetricsSuite with TestUtilsWithLegacyKernelAPIs class LogReplayEngineMetricsSuite extends AbstractLogReplayEngineMetricsSuite with TestUtilsWithTableManagerAPIs /** * Suite to test the engine metrics while replaying logs for getting the table protocol and * metadata (P&M) and scanning files. The metrics include how many files delta files, checkpoint * files read, size of checkpoint read set, and how many times `_last_checkpoint` is read etc. * * The goal is to test the behavior of calls to `readJsonFiles` and `readParquetFiles` that * Kernel makes. This calls determine the performance. */ trait AbstractLogReplayEngineMetricsSuite extends LogReplayBaseSuite with BeforeAndAfterAll { self: AbstractTestUtils => // Disable writing checksums for this test suite // This test suite checks the files read when loading the P&M, however, with the crc optimization // if crc are available, crc will be the only files read. // We want to test the P&M loading when CRC are not available in the tests. // Tests for tables with available CRC are included using resource test tables (and thus are // unaffected by changing our confs for writes). override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key, false) } override def afterAll(): Unit = { try { spark.conf.set(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key, true) } finally { super.afterAll() } } ///////////////////////// // Test Helper Methods // ///////////////////////// def loadScanFilesCheckMetrics( engine: MetricsEngine, tablePath: String, expJsonVersionsRead: Seq[Long], expParquetVersionsRead: Seq[Long], expParquetReadSetSizes: Seq[Long], expLastCheckpointReadCalls: Option[Int] = None): Unit = { engine.resetMetrics() val scan = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath) .getScanBuilder().build() // get all scan files and iterate through them to trigger the metrics collection val scanFiles = scan.getScanFiles(engine) while (scanFiles.hasNext) scanFiles.next() assertMetrics( engine, expJsonVersionsRead, expParquetVersionsRead, expParquetReadSetSizes, expLastCheckpointReadCalls) } def checkpoint(path: String, actionsPerFile: Int): Unit = { withSQLConf(DeltaSQLConf.DELTA_CHECKPOINT_PART_SIZE.key -> actionsPerFile.toString) { DeltaLog.forTable(spark, path).checkpoint() } } /////////// // Tests // /////////// test("no hint, no checkpoint, reads all files") { withTempDirAndMetricsEngine { (path, engine) => for (_ <- 0 to 9) { appendCommit(path) } loadPandMCheckMetrics( path, engine, expJsonVersionsRead = 9L to 0L by -1L, expParquetVersionsRead = Nil) } } test("no hint, existing checkpoint, reads all files up to that checkpoint") { withTempDirAndMetricsEngine { (path, engine) => for (_ <- 0 to 14) { appendCommit(path) } loadPandMCheckMetrics( path, engine, expJsonVersionsRead = 14L to 11L by -1L, expParquetVersionsRead = Seq(10), expParquetReadSetSizes = Seq(1)) } } test("no hint, existing checkpoint, newer P & M update, reads up to P & M commit") { withTempDirAndMetricsEngine { (path, engine) => for (_ <- 0 to 12) { appendCommit(path) } // v13 changes the protocol (which also updates the metadata) spark.sql(s""" |ALTER TABLE delta.`$path` SET TBLPROPERTIES ( | 'delta.minReaderVersion' = '2', | 'delta.minWriterVersion' = '5', | 'delta.columnMapping.mode' = 'name' |) |""".stripMargin) for (_ <- 14 to 16) { appendCommit(path) } loadPandMCheckMetrics( path, engine, expJsonVersionsRead = 16L to 13L by -1L, expParquetVersionsRead = Nil) } } test("read a table with multi-part checkpoint") { withTempDirAndMetricsEngine { (path, engine) => for (_ <- 0 to 14) { appendCommit(path) } // there should be one checkpoint file at version 10 loadScanFilesCheckMetrics( engine, path, expJsonVersionsRead = 14L to 11L by -1L, expParquetVersionsRead = Seq(10), // we read the checkpoint twice: once for the P &M and once for the scan files expParquetReadSetSizes = Seq(1, 1)) // create a multi-part checkpoint checkpoint(path, actionsPerFile = 2) // Reset metrics. engine.resetMetrics() // expect the Parquet read set to contain one request with size of 15 loadScanFilesCheckMetrics( engine, path, expJsonVersionsRead = Nil, expParquetVersionsRead = Seq(14), // we read the checkpoint twice: once for the P &M and once for the scan files expParquetReadSetSizes = Seq(8, 8)) } } Seq(true, false).foreach { deleteLastCheckpointMetadataFile => test("ensure `_last_checkpoint` is tried to read only once when " + s"""${if (deleteLastCheckpointMetadataFile) "not exists" else "valid file exists"}""") { withTempDirAndMetricsEngine { (path, engine) => for (_ <- 0 to 14) { appendCommit(path) } if (deleteLastCheckpointMetadataFile) { assert(Files.deleteIfExists(new File(path, "_delta_log/_last_checkpoint").toPath)) } // there should be one checkpoint file at version 10 loadScanFilesCheckMetrics( engine, path, expJsonVersionsRead = 14L to 11L by -1L, expParquetVersionsRead = Seq(10), // we read the checkpoint twice: once for the P &M and once for the scan files expParquetReadSetSizes = Seq(1, 1), // We try to read `_last_checkpoint` once. If it doesn't exist, we don't try reading // again. If it exists, we succeed reading in the first time expLastCheckpointReadCalls = Some(1)) } } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/LogReplaySuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.io.File import java.nio.file.Files import java.util.Optional import scala.collection.JavaConverters._ import io.delta.golden.GoldenTableUtils.goldenTablePath import io.delta.kernel.Table import io.delta.kernel.defaults.engine.DefaultEngine import io.delta.kernel.defaults.utils.{AbstractTestUtils, TestRow, TestUtils, TestUtilsWithLegacyKernelAPIs, TestUtilsWithTableManagerAPIs} import io.delta.kernel.internal.{InternalScanFileUtils, SnapshotImpl} import io.delta.kernel.internal.data.ScanStateRow import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.util.FileNames import io.delta.kernel.types.{LongType, StructType} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.hadoop.conf.Configuration import org.scalatest.funsuite.AnyFunSuite class LogReplaySuite extends AbstractLogReplaySuite with TestUtilsWithTableManagerAPIs { override lazy val defaultEngine = defaultEngineBatchSize2 } class LegacyLogReplaySuite extends AbstractLogReplaySuite with TestUtilsWithLegacyKernelAPIs { override lazy val defaultEngine = defaultEngineBatchSize2 } trait AbstractLogReplaySuite extends AnyFunSuite { self: AbstractTestUtils => test("simple end to end with inserts and deletes and checkpoint") { val expectedValues = (0L until 5L) ++ (10L until 15L) ++ (20L until 25L) ++ (30L until 35L) ++ (40L until 45L) ++ (50L to 65L) checkTable( path = goldenTablePath("basic-with-inserts-deletes-checkpoint"), expectedAnswer = expectedValues.map(TestRow(_)), expectedSchema = new StructType().add("id", LongType.LONG), expectedVersion = Some(13L)) } test("simple end to end with inserts and updates") { val expectedValues = (0 until 50).map((_, "N/A")) ++ (50 until 100).map(x => (x, s"val=$x")) checkTable( path = goldenTablePath("basic-with-inserts-updates"), expectedAnswer = expectedValues.map(TestRow.fromTuple)) } test("simple end to end with inserts and merge") { val expectedValues = (10 until 50).map(x => (x, s"val=$x")) ++ (50 until 100).map((_, "N/A")) ++ (100 until 150).map((_, "EXT")) checkTable( path = goldenTablePath("basic-with-inserts-merge"), expectedAnswer = expectedValues.map(TestRow.fromTuple)) } test("simple end to end with restore") { checkTable( path = goldenTablePath("basic-with-inserts-overwrite-restore"), expectedAnswer = (0L until 200L).map(TestRow(_)), expectedVersion = Some(3)) } test("end to end only checkpoint files") { val expectedValues = (5L until 10L) ++ (0L until 20L) checkTable( path = goldenTablePath("only-checkpoint-files"), expectedAnswer = expectedValues.map(TestRow(_))) } Seq("protocol", "metadata").foreach { action => test(s"missing $action should fail") { val path = goldenTablePath(s"deltalog-state-reconstruction-without-$action") val e = intercept[IllegalStateException] { latestSnapshot(path).getSchema() } assert(e.getMessage.contains(s"No $action found")) } } // TODO missing protocol should fail when missing from checkpoint // GoldenTable("deltalog-state-reconstruction-from-checkpoint-missing-protocol") // generation is broken and cannot be regenerated with a non-null schemaString until fixed Seq("metadata" /* , "protocol" */ ).foreach { action => test(s"missing $action should fail missing from checkpoint") { val path = goldenTablePath(s"deltalog-state-reconstruction-from-checkpoint-missing-$action") val e = intercept[IllegalStateException] { latestSnapshot(path).getSchema() } assert(e.getMessage.contains(s"No $action found")) } } test("fetches the latest protocol and metadata") { val path = goldenTablePath("log-replay-latest-metadata-protocol") val snapshot = latestSnapshot(path) val scanStateRow = snapshot.getScanBuilder().build() .getScanState(defaultEngine) // schema is updated assert(ScanStateRow.getLogicalSchema(scanStateRow) .fieldNames().asScala.toSet == Set("col1", "col2")) // check protocol version is upgraded val readerVersionOrd = scanStateRow.getSchema().indexOf("minReaderVersion") val writerVersionOrd = scanStateRow.getSchema().indexOf("minWriterVersion") assert(scanStateRow.getInt(readerVersionOrd) == 3 && scanStateRow.getInt(writerVersionOrd) == 7) } test("standalone DeltaLogSuite: 'checkpoint'") { val path = goldenTablePath("checkpoint") val snapshot = latestSnapshot(path) assert(snapshot.getVersion() == 14) val scan = snapshot.getScanBuilder().build() assert(collectScanFileRows(scan).length == 1) } test("standalone DeltaLogSuite: 'snapshot'") { def getDirDataFiles(tablePath: String): Array[File] = { val correctTablePath = if (tablePath.startsWith("file:")) tablePath.stripPrefix("file:") else tablePath val dir = new File(correctTablePath) dir.listFiles().filter(_.isFile).filter(_.getName.endsWith("snappy.parquet")) } def verifySnapshotScanFiles( tablePath: String, expectedFiles: Array[File], expectedVersion: Int): Unit = { val snapshot = latestSnapshot(tablePath) assert(snapshot.getVersion() == expectedVersion) val scanFileRows = collectScanFileRows( snapshot.getScanBuilder().build()) assert(scanFileRows.length == expectedFiles.length) val scanFilePaths = scanFileRows .map(InternalScanFileUtils.getAddFileStatus) .map(_.getPath) .map(new File(_).getName) // get the relative path to compare assert(scanFilePaths.toSet == expectedFiles.map(_.getName).toSet) } // Append data0 var data0_files: Array[File] = Array.empty withGoldenTable("snapshot-data0") { tablePath => data0_files = getDirDataFiles(tablePath) // data0 files verifySnapshotScanFiles(tablePath, data0_files, 0) } // Append data1 var data0_data1_files: Array[File] = Array.empty withGoldenTable("snapshot-data1") { tablePath => data0_data1_files = getDirDataFiles(tablePath) // data0 & data1 files verifySnapshotScanFiles(tablePath, data0_data1_files, 1) } // Overwrite with data2 var data2_files: Array[File] = Array.empty withGoldenTable("snapshot-data2") { tablePath => // we have overwritten files for data0 & data1; only data2 files should remain data2_files = getDirDataFiles(tablePath) .filterNot(f => data0_data1_files.exists(_.getName == f.getName)) verifySnapshotScanFiles(tablePath, data2_files, 2) } // Append data3 withGoldenTable("snapshot-data3") { tablePath => // we have overwritten files for data0 & data1; only data2 & data3 files should remain val data2_data3_files = getDirDataFiles(tablePath) .filterNot(f => data0_data1_files.exists(_.getName == f.getName)) verifySnapshotScanFiles(tablePath, data2_data3_files, 3) } // Delete data2 files withGoldenTable("snapshot-data2-deleted") { tablePath => // we have overwritten files for data0 & data1, and deleted data2 files; only data3 files // should remain val data3_files = getDirDataFiles(tablePath) .filterNot(f => data0_data1_files.exists(_.getName == f.getName)) .filterNot(f => data2_files.exists(_.getName == f.getName)) verifySnapshotScanFiles(tablePath, data3_files, 4) } // Repartition into 2 files withGoldenTable("snapshot-repartitioned") { tablePath => val snapshot = latestSnapshot(tablePath) assert(snapshot.getVersion() == 5) val scanFileRows = collectScanFileRows( snapshot.getScanBuilder().build()) assert(scanFileRows.length == 2) } // Vacuum withGoldenTable("snapshot-vacuumed") { tablePath => // all remaining dir data files should be needed for current snapshot version // vacuum doesn't change the snapshot version verifySnapshotScanFiles(tablePath, getDirDataFiles(tablePath), 5) } } test("DV cases with same path different DV keys") { val snapshot = latestSnapshot(goldenTablePath("log-replay-dv-key-cases")) val scanFileRows = collectScanFileRows( snapshot.getScanBuilder().build()) assert(scanFileRows.length == 1) // there should only be 1 add file val dv = InternalScanFileUtils.getDeletionVectorDescriptorFromRow(scanFileRows.head) assert(dv.getCardinality == 3) // dv cardinality should be 3 } test("special characters in path") { withGoldenTable("log-replay-special-characters-a") { path => val snapshot = latestSnapshot(path) val scanFileRows = collectScanFileRows( snapshot.getScanBuilder().build()) assert(scanFileRows.isEmpty) } withGoldenTable("log-replay-special-characters-b") { path => val snapshot = latestSnapshot(path) val scanFileRows = collectScanFileRows( snapshot.getScanBuilder().build()) assert(scanFileRows.length == 1) val addFileStatus = InternalScanFileUtils.getAddFileStatus(scanFileRows.head) // get the relative path to compare assert(new File(addFileStatus.getPath).getName == "special p@#h") } } // TODO we need to canonicalize path during log replay see issue #2213 ignore("path should be canonicalized - normal characters") { Seq("canonicalized-paths-normal-a", "canonicalized-paths-normal-b").foreach { path => val snapshot = latestSnapshot(goldenTablePath(path)) assert(snapshot.getVersion() == 1) val scanFileRows = collectScanFileRows(snapshot.getScanBuilder().build()) assert(scanFileRows.isEmpty) } } ignore("path should be canonicalized - special characters") { Seq("canonicalized-paths-special-a", "canonicalized-paths-special-b").foreach { path => val snapshot = latestSnapshot(goldenTablePath(path)) assert(snapshot.getVersion() == 1) val scanFileRows = collectScanFileRows(snapshot.getScanBuilder().build()) assert(scanFileRows.isEmpty) } } // from DeltaDataReaderSuite in standalone test("escaped chars sequences in path") { checkTable( path = goldenTablePath("data-reader-escaped-chars"), expectedAnswer = TestRow("foo1", "bar+%21") :: TestRow("foo2", "bar+%22") :: TestRow("foo3", "bar+%23") :: Nil) } test("delete and re-add same file in different transactions") { val path = goldenTablePath("delete-re-add-same-file-different-transactions") val snapshot = latestSnapshot(path) val scan = snapshot.getScanBuilder().build() val foundFiles = collectScanFileRows(scan).map(InternalScanFileUtils.getAddFileStatus) assert(foundFiles.length == 2) assert(foundFiles.map(_.getPath.split('/').last).toSet == Set("foo", "bar")) // We added two add files with the same path `foo`. The first should have been removed. // The second should remain, and should have a hard-coded modification time of 1700000000000L assert( foundFiles.find(_.getPath.endsWith("foo")).exists(_.getModificationTime == 1700000000000L)) } test("get the last transaction version for appID") { val unresolvedPath = goldenTablePath("deltalog-getChanges") val snapshot = getTableManagerAdapter.getSnapshotAtLatest(defaultEngine, unresolvedPath) assert(snapshot.isInstanceOf[SnapshotImpl]) assert(snapshot.getLatestTransactionVersion(defaultEngine, "fakeAppId") === Optional.of(3L)) assert(!snapshot.getLatestTransactionVersion(defaultEngine, "nonExistentAppId").isPresent) } test("current checksum read => snapshot provides crc info") { withTempDir { tempFile => val tablePath = tempFile.getAbsolutePath spark.sql( s"CREATE TABLE delta.`$tablePath` USING DELTA AS " + s"SELECT 0L as id") spark.sql( s"INSERT INTO delta.`$tablePath` SELECT 1L as id") val snapshot = getTableManagerAdapter.getSnapshotAtLatest(defaultEngine, tablePath) assert(snapshot.getCurrentCrcInfo.isPresent) val crcInfo = snapshot.getCurrentCrcInfo.get() assert(crcInfo.getVersion == 1) assert(crcInfo.getProtocol == snapshot.getProtocol) assert(crcInfo.getMetadata == snapshot.getMetadata) } } test("stale checksum read => snapshot doesn't provides crc info") { withTempDir { tempFile => val tablePath = tempFile.getAbsolutePath spark.sql( s"CREATE TABLE delta.`$tablePath` USING DELTA AS " + s"SELECT 0L as id") spark.sql( s"INSERT INTO delta.`$tablePath` SELECT 1L as id") deleteChecksumFileForTable(tablePath, versions = Seq(1)) val snapshot = getTableManagerAdapter.getSnapshotAtLatest(defaultEngine, tablePath) assert(!snapshot.getCurrentCrcInfo.isPresent) } } test("no checksum read => snapshot doesn't provides crc info") { withTempDir { tempFile => val tablePath = tempFile.getAbsolutePath spark.sql( s"CREATE TABLE delta.`$tablePath` USING DELTA AS " + s"SELECT 0L as id") spark.sql( s"INSERT INTO delta.`$tablePath` SELECT 1L as id") deleteChecksumFileForTable(tablePath, versions = Seq(0, 1)) val snapshot = getTableManagerAdapter.getSnapshotAtLatest(defaultEngine, tablePath) assert(!snapshot.getCurrentCrcInfo.isPresent) } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/PaginatedScanSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.util.Optional import io.delta.kernel.PaginatedScan import io.delta.kernel.PaginatedScanFilesIterator import io.delta.kernel.ScanBuilder import io.delta.kernel.data.FilteredColumnarBatch import io.delta.kernel.data.Row import io.delta.kernel.defaults.engine.{DefaultEngine, DefaultJsonHandler, DefaultParquetHandler} import io.delta.kernel.defaults.test.AbstractTableManagerAdapter import io.delta.kernel.defaults.utils.{ExpressionTestUtils, TestUtils, TestUtilsWithTableManagerAPIs, WriteUtils} import io.delta.kernel.expressions.Literal import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.hook.LogCompactionHook import io.delta.kernel.internal.replay.{PageToken, PaginatedScanFilesIteratorImpl} import io.delta.kernel.utils.CloseableIterator import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.catalyst.plans.SQLHelper import org.scalatest.funsuite.AnyFunSuite import org.slf4j.{Logger, LoggerFactory} class PaginatedScanSuite extends AnyFunSuite with TestUtilsWithTableManagerAPIs with ExpressionTestUtils with SQLHelper with WriteUtils { private val logger = LoggerFactory.getLogger(classOf[PaginatedScanSuite]) val tableManager: AbstractTableManagerAdapter = getTableManagerAdapter /** * Custom engine with customized batch size. This engine will be used by * all test cases. This number should not change, and it affects every single test. */ private val customEngine: DefaultEngine = { val hadoopConf = new org.apache.hadoop.conf.Configuration() hadoopConf.set("delta.kernel.default.json.reader.batch-size", "5") hadoopConf.set("delta.kernel.default.parquet.reader.batch-size", "5") DefaultEngine.create(hadoopConf) } // TODO: this can be a testUtil? private def getScanBuilder(tablePath: String, tableVersionOpt: Optional[Long]): ScanBuilder = { val snapshot = { if (tableVersionOpt.isPresent) { tableManager.getSnapshotAtVersion( customEngine, tablePath, tableVersionOpt.get()) } else { tableManager.getSnapshotAtLatest(customEngine, tablePath) } } snapshot.getScanBuilder() } private def createPaginatedScan( tablePath: String, tableVersionOpt: Optional[Long], pageSize: Long, pageTokenOpt: Optional[Row] = Optional.empty()): PaginatedScan = { getScanBuilder(tablePath, tableVersionOpt).buildPaginated(pageSize, pageTokenOpt) } case class FirstPageRequestTestContext( pageSize: Int, expScanFilesCnt: Int, expBatchCnt: Int, expLastReadLogFile: String, expLastReadRowIdx: Int) private def validateFirstPageResults( batches: Seq[FilteredColumnarBatch], expectedFileCount: Int, expectedBatchCount: Int): Unit = { assert(batches.nonEmpty) val fileCounts: Seq[Long] = batches.map(_.getPreComputedNumSelectedRows.get().toLong) val totalFileCountsReturned = fileCounts.sum assert(fileCounts.length == expectedBatchCount) assert(totalFileCountsReturned == expectedFileCount) logger.info(s"Total num batches returned in page one = ${fileCounts.length}") logger.info(s"Total num Parquet Files fetched in page one = " + s"$totalFileCountsReturned") } private def validateFirstPageToken( pageTokenRow: Row, expectedLogFileName: String, expectedRowIndex: Long): Unit = { val lastReadLogFilePath = PageToken.fromRow(pageTokenRow).getLastReadLogFilePath val lastReturnedRowIndex = PageToken.fromRow(pageTokenRow).getLastReturnedRowIndex assert(lastReadLogFilePath.endsWith(expectedLogFileName)) assert(lastReturnedRowIndex == expectedRowIndex) logger.info(s"New PageToken: lastReadLogFileName = $lastReadLogFilePath") logger.info(s"New PageToken: lastReadRowIndex = $lastReturnedRowIndex") } /** * Executes a single paginated scan request. * * 1. Constructs a paginated scan using the provided page size and page token (optional). * 2. Collects scan results for the current page. * 3. Returns the results along with the next page token. */ private def doSinglePageRequest( tablePath: String, tableVersionOpt: Optional[Long], pageTokenOpt: Optional[Row] = Optional.empty(), pageSize: Long): (Optional[Row], Seq[FilteredColumnarBatch]) = { val paginatedScan = createPaginatedScan( tablePath = tablePath, tableVersionOpt = tableVersionOpt, pageSize = pageSize, pageTokenOpt = pageTokenOpt) val paginatedIter = paginatedScan.getScanFiles(customEngine) val returnedBatchesInPage = paginatedIter.toSeq val nextPageToken = paginatedIter.getCurrentPageToken assert(returnedBatchesInPage.nonEmpty) val fileCounts: Seq[Long] = returnedBatchesInPage.map(_.getPreComputedNumSelectedRows .get().toLong) val totalFileCountsReturned = fileCounts.sum logger.info(s"number of batches = ${returnedBatchesInPage.length}") logger.info(s"number of AddFiles = ${totalFileCountsReturned}") if (nextPageToken.isPresent) { val lastReadLogFilePath = PageToken.fromRow(nextPageToken.get).getLastReadLogFilePath val lastReturnedRowIndex = PageToken.fromRow(nextPageToken.get).getLastReturnedRowIndex logger.info(s"New PageToken: lastReadLogFileName = $lastReadLogFilePath") logger.info(s"New PageToken: lastReadRowIndex = $lastReturnedRowIndex") } (nextPageToken, returnedBatchesInPage) } /** * Simulates the client's behavior of reading a full scan in a paginated manner * with a given page size. * * The client: * 1. Starts by requesting the first page (no page token). * 2. Receives a page of results along with a page token. * 3. Uses the page token to request the next page. * 4. Repeats until the returned page token is empty, indicating that all data has been consumed. */ private def runCompletePaginationTest( testCase: FirstPageRequestTestContext, tablePath: String, tableVersionOpt: Optional[Long] = Optional.empty()): Unit = { // ============ Request the first page ============== var (pageTokenOpt, returnedBatchesInPage) = doSinglePageRequest( tablePath = tablePath, tableVersionOpt = tableVersionOpt, pageSize = testCase.pageSize) validateFirstPageResults( returnedBatchesInPage, testCase.expScanFilesCnt, testCase.expBatchCnt) // When the scan is exhausted, returned page token should be empty. if (pageTokenOpt.isPresent) { validateFirstPageToken( pageTokenOpt.get, testCase.expLastReadLogFile, testCase.expLastReadRowIdx) } // ============ Request following pages ============== var allBatchesPaginationScan = returnedBatchesInPage while (pageTokenOpt.isPresent) { val (newPageTokenOpt, newReturnedBatchesInPage) = doSinglePageRequest( tablePath = tablePath, tableVersionOpt = tableVersionOpt, pageTokenOpt = pageTokenOpt, pageSize = testCase.pageSize) pageTokenOpt = newPageTokenOpt allBatchesPaginationScan ++= newReturnedBatchesInPage } val normalScan = getScanBuilder(tablePath = tablePath, tableVersionOpt = tableVersionOpt).build() val iter = normalScan.getScanFiles(customEngine) val allBatchesNormalScan = iter.toSeq // check no duplicate or missing batches in paginated scan assert(allBatchesNormalScan.size == allBatchesPaginationScan.size) for (i <- allBatchesNormalScan.indices) { val normalBatch = allBatchesNormalScan(i) val paginatedBatch = allBatchesPaginationScan(i) assert(normalBatch.getFilePath.equals(paginatedBatch.getFilePath)) assert(normalBatch.getData.getSize == paginatedBatch.getData.getSize) } } // ==== Test Paginated Iterator Behaviors ====== // TODO: test call hasNext() twice test("Calling getCurrentPageToken() without calling next() should throw Exception") { // Request first page withTempDir { tempDir => val tablePath = tempDir.getCanonicalPath // First commit: files 0-4 (5 files) spark.range(0, 50, 1, 5).write.format("delta").save(tablePath) val firstPageSize = 2L val firstPaginatedScan = createPaginatedScan( tablePath = tablePath, tableVersionOpt = Optional.empty(), pageSize = firstPageSize) val firstPaginatedIter = firstPaginatedScan.getScanFiles(customEngine) // throw exception var e = intercept[IllegalStateException] { firstPaginatedIter.getCurrentPageToken.get } assert(e.getMessage.contains("Can't call getCurrentPageToken()")) // throw exception e = intercept[IllegalStateException] { firstPaginatedIter.hasNext firstPaginatedIter.getCurrentPageToken.get } assert(e.getMessage.contains("Can't call getCurrentPageToken()")) firstPaginatedIter.close() } } test("getCurrentPageToken() is impacted only by next() calls, not hasNext() calls") { // Request first page withTempDir { tempDir => val tablePath = tempDir.getCanonicalPath // First commit: files 0-4 (5 files) spark.range(0, 50, 1, 5).write.format("delta").save(tablePath) // Second commit: files 5-9 (5 more files) spark.range(50, 100, 1, 5).write.format("delta").mode("append").save(tablePath) // Third commit: files 10-14 (5 more files) spark.range(100, 150, 1, 5).write.format("delta").mode("append").save(tablePath) val firstPageSize = 2L val firstPaginatedScan = createPaginatedScan( tablePath = tablePath, tableVersionOpt = Optional.empty(), pageSize = firstPageSize) val firstPaginatedIter = firstPaginatedScan.getScanFiles(customEngine) if (firstPaginatedIter.hasNext) firstPaginatedIter.next() val expectedPageToken = firstPaginatedIter.getCurrentPageToken.get firstPaginatedIter.hasNext // call hsaNext() again, should not affect page token val pageToken = firstPaginatedIter.getCurrentPageToken.get assert(PageToken.fromRow(pageToken).equals(PageToken.fromRow(expectedPageToken))) firstPaginatedIter.close() } } // ===== Data Integrity test cases===== // TODO: test predicate changes /** * Test case to verify pagination behavior when log segment changes between page requests. */ test("Throw exception when log segment changes between page requests") { withTempDir { tempDir => val tablePath = tempDir.getCanonicalPath // First commit: files 0-4 (5 files) spark.range(0, 50, 1, 5).write.format("delta").save(tablePath) // Second commit: files 5-9 (5 more files) spark.range(50, 100, 1, 5).write.format("delta").mode("append").save(tablePath) // Third commit: files 10-14 (5 more files) spark.range(100, 150, 1, 5).write.format("delta").mode("append").save(tablePath) // Fourth commit: files 15-19 (5 more files) spark.range(150, 200, 1, 5).write.format("delta").mode("append").save(tablePath) // Fifth commit: files 20-24 (5 more files) spark.range(200, 250, 1, 5).write.format("delta").mode("append").save(tablePath) // Request first page val firstPageSize = 2L val firstPaginatedScan = createPaginatedScan( tablePath = tablePath, tableVersionOpt = Optional.empty(), pageSize = firstPageSize) val firstPaginatedIter = firstPaginatedScan.getScanFiles(customEngine) if (firstPaginatedIter.hasNext) firstPaginatedIter.next() // call next() once val firstPageToken = firstPaginatedIter.getCurrentPageToken.get firstPaginatedIter.close() // Perform log compaction for versions 0-2; log segment should change val dataPath = new Path(s"file:${tablePath}") val logPath = new Path(s"file:${tablePath}", "_delta_log") val compactionHook = new LogCompactionHook(dataPath, logPath, 0, 2, 0) compactionHook.threadSafeInvoke(customEngine) logger.info("Log compaction completed for versions 0-2") // Request second page val secondPageSize = 5L val e = intercept[IllegalArgumentException] { createPaginatedScan( tablePath = tablePath, tableVersionOpt = Optional.empty(), pageSize = secondPageSize, pageTokenOpt = Optional.of(firstPageToken)) } assert(e.getMessage.contains("Invalid page token: token log segment")) } } // ==== Log File Name Variables ====== private val JSON_FILE_0 = "00000000000000000000.json" private val JSON_FILE_1 = "00000000000000000001.json" private val JSON_FILE_2 = "00000000000000000002.json" private val JSON_FILE_11 = "00000000000000000011.json" private val JSON_FILE_12 = "00000000000000000012.json" private val CHECKPOINT_FILE_10 = "00000000000000000010.checkpoint.parquet" // ===== Single JSON file test cases ===== /** * Log Segment List: * 00000000000000000000.json contains 2 batches, 5 active AddFiles in total * * Note: batch size is set to 5 * Batch 1: 5 rows, 2 selected AddFiles * Batch 2: 3 rows, 3 selected AddFiles */ Seq( // Kernel is asked to read the 1st page of size 1. Kernel reads the 1st // full batch, so returns 2 AddFiles and ends at the 5th row (index 4). // Note: Kernel should always return full batches, so return full batch one. FirstPageRequestTestContext( pageSize = 1, expScanFilesCnt = 2, expBatchCnt = 1, expLastReadLogFile = JSON_FILE_0, expLastReadRowIdx = 4), // Kernel is asked to read the 1st page of size 2. Kernel reads the 1st // full batch, so returns 2 AddFiles and ends at the 5th row (index 4) FirstPageRequestTestContext( pageSize = 2, expScanFilesCnt = 2, expBatchCnt = 1, expLastReadLogFile = JSON_FILE_0, expLastReadRowIdx = 4), // Kernel is asked to read the 1st page of size 4. Kernel reads batch 1 and // batch 2 in JSON_FILE_0, so returns 5 AddFiles and ends at the 8th row (index 7) // Note: Kernel should always return full batches, so return full 2 batches. FirstPageRequestTestContext( pageSize = 4, expScanFilesCnt = 5, expBatchCnt = 2, expLastReadLogFile = JSON_FILE_0, expLastReadRowIdx = 7), // Kernel is asked to read the 1st page of size 5. Kernel reads batch 1 and // batch 2 in JSON_FILE_0, so returns 5 AddFiles and ends at the 8th row (index 7) FirstPageRequestTestContext( pageSize = 5, expScanFilesCnt = 5, expBatchCnt = 2, expLastReadLogFile = JSON_FILE_0, expLastReadRowIdx = 7), // Kernel is asked to read the 1st page of size 20. Kernel reads batch 1 and // batch 2 in JSON_FILE_0, so returns 5 AddFiles and ends at the 8th row (index 7) // Note: page size won't be reached because there is only 5 data files in total. FirstPageRequestTestContext( pageSize = 20, expScanFilesCnt = 5, expBatchCnt = 2, expLastReadLogFile = JSON_FILE_0, expLastReadRowIdx = 7)).foreach { testCase => test(s"Single JSON file - page size ${testCase.pageSize}") { runCompletePaginationTest( testCase = testCase, tablePath = getTestResourceFilePath("kernel-pagination-all-jsons"), tableVersionOpt = Optional.of(0L)) } } // ===== Multiple JSON files test cases ===== /** * Log Segment List: * 00000000000000000000.json : 8 rows (5 AddFile row + 3 non-AddFile rows) * 00000000000000000001.json : 6 rows (5 AddFile row + 1 non-AddFile row) * 00000000000000000002.json : 6 rows (5 AddFile row + 1 non-AddFile row) * * Note: batch size is set to 5 * 00000000000000000002.json contains 2 batches, 5 active AddFiles in total * Batch 1: 5 rows, 4 selected AddFiles * Batch 2: 1 rows, 1 selected AddFiles * * 00000000000000000001.json contains 2 batches, 5 active AddFiles in total * Batch 1: 5 rows, 4 selected AddFiles * Batch 2: 1 rows, 1 selected AddFiles * * 00000000000000000000.json contains 2 batches, 5 active AddFiles in total * Batch 1: 5 rows, 2 selected AddFiles * Batch 2: 3 rows, 3 selected AddFiles */ Seq( // Kernel is asked to read the 1st page of size 1. Kernel reads batch 1 in JSON_FILE_2, // so returns 4 AddFiles and ends at the 5th row (index 4) in JSON_FILE_2. // Note: Kernel should return full batches, so return full one batch (and go over page limit). FirstPageRequestTestContext( pageSize = 1, expScanFilesCnt = 4, expBatchCnt = 1, expLastReadLogFile = JSON_FILE_2, expLastReadRowIdx = 4), // Kernel is asked to read the 1st page of size 4. Kernel reads batch 1 in JSON_FILE_2, // so returns 4 AddFiles and ends at the 5th row (index 4) in JSON_FILE_2. FirstPageRequestTestContext( pageSize = 4, expScanFilesCnt = 4, expBatchCnt = 1, expLastReadLogFile = JSON_FILE_2, expLastReadRowIdx = 4), // Kernel is asked to read the 1st page of size 5. Kernel reads batch 1 and 2 in JSON_FILE_2, // so returns 5 AddFiles and ends at the 6th row (index 5) in JSON_FILE_2. FirstPageRequestTestContext( pageSize = 5, expScanFilesCnt = 5, expBatchCnt = 2, expLastReadLogFile = JSON_FILE_2, expLastReadRowIdx = 5), // Kernel is asked to read the 1st page of size 7. Kernel reads all batches in JSON_FILE_2, // batch 1 in JSON_FILE_1, so returns 9 AddFiles and ends at the 5th row (index 4) // in JSON_FILE_1. // Note: Kernel should return full batches, so return 3 full batches (and go over page limit). FirstPageRequestTestContext( pageSize = 7, expScanFilesCnt = 9, expBatchCnt = 3, expLastReadLogFile = JSON_FILE_1, expLastReadRowIdx = 4), // Kernel is asked to read the 1st page of size 9. Kernel reads all batches in JSON_FILE_2, // batch 1 in JSON_FILE_1, so returns 9 AddFiles and ends at the 5th row (index 4) // in JSON_FILE_1. FirstPageRequestTestContext( pageSize = 9, expScanFilesCnt = 9, expBatchCnt = 3, expLastReadLogFile = JSON_FILE_1, expLastReadRowIdx = 4), // Kernel is asked to read the 1st page of size 18. Kernel reads all batches in JSON_FILE_2, // JSON_FILE_1 and SON_FILE_0, so returns 15 AddFiles and ends at the last row (index 7) // in JSON_FILE_0. // Note: page size won't be reached because there are 15 data files in total. FirstPageRequestTestContext( pageSize = 18, expScanFilesCnt = 15, expBatchCnt = 6, expLastReadLogFile = JSON_FILE_0, expLastReadRowIdx = 7)).foreach { testCase => test(s"Multiple JSON files - page size ${testCase.pageSize}") { runCompletePaginationTest( testCase = testCase, tablePath = getTestResourceFilePath("kernel-pagination-all-jsons")) } } // ===== Single checkpoint file test cases ===== /** * Log Segment List: * 00000000000000000010.checkpoint.parquet contains 5 batches, 22 active AddFiles, 24 rows * * Note: batch size is set to 5 * * Batch 1: 5 rows, 5 selected AddFiles * Batch 2: 5 rows, 5 selected AddFiles * Batch 3: 5 rows, 5 selected AddFiles * Batch 4: 5 rows, 3 selected AddFiles * Batch 5: 4 rows, 4 selected AddFiles */ Seq( // Kernel is asked to read the 1st page of size 1. Kernel reads batch 1 in 10.checkpoint, // so returns 5 AddFiles and ends at the 5th row (index 4) in 10.checkpoint. // Note: Kernel should return full batches, so return one full batch (and go over page limit). FirstPageRequestTestContext( pageSize = 1, expScanFilesCnt = 5, expBatchCnt = 1, expLastReadLogFile = CHECKPOINT_FILE_10, expLastReadRowIdx = 4), // Kernel is asked to read the 1st page of size 10. Kernel reads 2 batches in 10.checkpoint, // so returns 10 AddFiles and ends at the 10th row (index 9) in 10.checkpoint. FirstPageRequestTestContext( pageSize = 10, expScanFilesCnt = 10, expBatchCnt = 2, expLastReadLogFile = CHECKPOINT_FILE_10, expLastReadRowIdx = 9), // Kernel is asked to read the 1st page of size 12. Kernel reads 3 batches in 10.checkpoint, // so returns 15 AddFiles and ends at the 15th row (index 14) in 10.checkpoint. // Note: Kernel should return full batches, so return 3 full batches (and go over page limit). FirstPageRequestTestContext( pageSize = 12, expScanFilesCnt = 15, expBatchCnt = 3, expLastReadLogFile = CHECKPOINT_FILE_10, expLastReadRowIdx = 14), // Kernel is asked to read the 1st page of size 100. Kernel reads all 5 batches // in 10.checkpoint, so returns 22 AddFiles and ends at the 24th row (index 23) // in 10.checkpoint. Note: page size won't be reached in this test case. FirstPageRequestTestContext( pageSize = 100, expScanFilesCnt = 22, expBatchCnt = 5, expLastReadLogFile = CHECKPOINT_FILE_10, expLastReadRowIdx = 23)).foreach { testCase => test(s"Single checkpoint file - page size ${testCase.pageSize}") { runCompletePaginationTest( testCase = testCase, tableVersionOpt = Optional.of(10L), tablePath = getTestResourceFilePath("kernel-pagination-single-checkpoint")) } } // ===== Single checkpoint file and multiple JSON files test cases ===== /** * Log segment list: * 00000000000000000010.checkpoint.parquet * 00000000000000000011.json * 00000000000000000012.json * * Note: batch size is set to 5 * * 00000000000000000012.json contains 1 batch, 2 active AddFiles in total * Batch 1: 3 rows, 2 selected AddFiles * * 00000000000000000011.json contains 1 batch, 2 active AddFiles in total * Batch 1: 3 rows, 2 selected AddFiles * * 00000000000000000010.checkpoint.parquet contains 5 batches, 22 active AddFiles, 24 rows * Batch 1: 5 rows, 5 selected AddFiles * Batch 2: 5 rows, 5 selected AddFiles * Batch 3: 5 rows, 5 selected AddFiles * Batch 4: 5 rows, 3 selected AddFiles * Batch 5: 4 rows, 4 selected AddFiles */ Seq( // Kernel is asked to read the 1st page of size 1. Kernel reads 1 batches in 12.json, // so returns 2 AddFiles and ends at the 3rd row (index 2) in 10.checkpoint. // Note: Kernel should return full batches, so return one full batch (and go over page limit). FirstPageRequestTestContext( pageSize = 1, expScanFilesCnt = 2, expBatchCnt = 1, expLastReadLogFile = JSON_FILE_12, expLastReadRowIdx = 2), // Kernel is asked to read the 1st page of size 2. Kernel reads 1 batches in 12.json, // so returns 2 AddFiles and ends at the 3rd row (index 2) in 10.checkpoint. FirstPageRequestTestContext( pageSize = 2, expScanFilesCnt = 2, expBatchCnt = 1, expLastReadLogFile = JSON_FILE_12, expLastReadRowIdx = 2), // Kernel is asked to read the 1st page of size 1. Kernel reads one batch in 12.json, // and one batch in 11.json, so returns 4 AddFiles and ends at the 3rd row (index 2) in 11.json. // Note: Kernel should return full batches, so return 2 full batches (and go over page limit). FirstPageRequestTestContext( pageSize = 3, expScanFilesCnt = 4, expBatchCnt = 2, expLastReadLogFile = JSON_FILE_11, expLastReadRowIdx = 2), // Kernel is asked to read the 1st page of size 4. Kernel reads one batch in 12.json, // and one batch in 11.json, so returns 4 AddFiles and ends at the 3rd row (index 2) in 11.json. FirstPageRequestTestContext( pageSize = 4, expScanFilesCnt = 4, expBatchCnt = 2, expLastReadLogFile = JSON_FILE_11, expLastReadRowIdx = 2), // Kernel is asked to read the 1st page of size 8. Kernel reads one batch in 12.json, // one batch in 11.json, and one batch in 10.checkpoint, so returns 9 AddFiles and // ends at the 5th row (index 4) in 10.checkpoint. // Note: Kernel should return full batches, so return 3 full batches (and go over page limit). FirstPageRequestTestContext( pageSize = 8, expScanFilesCnt = 9, expBatchCnt = 3, expLastReadLogFile = CHECKPOINT_FILE_10, expLastReadRowIdx = 4), // Kernel is asked to read the 1st page of size 18. Kernel reads one batch in 12.json, // one batch in 11.json, and 3 batches in 10.checkpoint, so returns 19 AddFiles and // ends at the 15th row (index 14) in 10.checkpoint. // Note: Kernel should return full batches, so return 5 full batches (and go over page limit). FirstPageRequestTestContext( pageSize = 18, expScanFilesCnt = 19, expBatchCnt = 5, expLastReadLogFile = CHECKPOINT_FILE_10, expLastReadRowIdx = 14)).foreach { testCase => test(s"Single checkpoint and JSON files - page size ${testCase.pageSize}") { runCompletePaginationTest( testCase = testCase, tablePath = getTestResourceFilePath("kernel-pagination-single-checkpoint")) } } // ===== Multi-part checkpoint files test cases ===== /** * Log Segment List: * 00000000000000000000.checkpoint.0000000003.0000000003.parquet (6 AddFile) * Batch A: 5 AddFile, 5 rows * Batch B: 1 AddFile, 1 row * 00000000000000000000.checkpoint.0000000002.0000000003.parquet (7 AddFile) * Batch C: 5 AddFile, 5 row * Batch D: 2 AddFile, 2 row * 00000000000000000000.checkpoint.0000000001.0000000003.parquet (5 AddFile) * Batch E: 3 AddFile, 5 row * Batch F: 2 AddFile, 2 row */ val MULTI_CHECKPOINT_FILE_0_1 = "00000000000000000000.checkpoint.0000000001.0000000003.parquet" val MULTI_CHECKPOINT_FILE_0_2 = "00000000000000000000.checkpoint.0000000002.0000000003.parquet" val MULTI_CHECKPOINT_FILE_0_3 = "00000000000000000000.checkpoint.0000000003.0000000003.parquet" // Note: Kernel should return full batches (and may go over page limit). Seq( FirstPageRequestTestContext( pageSize = 1, expScanFilesCnt = 5, /* 5 AddFiles in Batch A */ expBatchCnt = 1, /* return only batch A */ expLastReadLogFile = MULTI_CHECKPOINT_FILE_0_3, /* Batch A from checkpoint file 3 */ expLastReadRowIdx = 4 /* Last Row index in Batch A is 4 in checkpoint file 3 */ ), FirstPageRequestTestContext( pageSize = 7, expScanFilesCnt = 11, /* 5 (batch A) + 1 (batch B) + 5 (batch C) */ expBatchCnt = 3, /* return batch A, B, C */ expLastReadLogFile = MULTI_CHECKPOINT_FILE_0_2, /* Batch C from checkpoint file 2 */ expLastReadRowIdx = 4 /* Last Row index in Batch C is 4 in checkpoint file 2 */ ), FirstPageRequestTestContext( pageSize = 13, expScanFilesCnt = 13, /* 5 (batch A) + 1 (batch B) + 5 (batch C) + 2 (batch D) */ expBatchCnt = 4, /* return batch A, B, C, D */ expLastReadLogFile = MULTI_CHECKPOINT_FILE_0_2, /* Batch D from checkpoint file 2 */ expLastReadRowIdx = 6 /* Last Row index in Batch D is 6 in checkpoint file 3 */ ), FirstPageRequestTestContext( pageSize = 100, expScanFilesCnt = 18, /* 5 (batch A) + 1 (batch B) + 5 (C) + 2 (D) + 3 (E) + 2 F) */ expBatchCnt = 6, /* return batch A, B, C, D, E, F */ expLastReadLogFile = MULTI_CHECKPOINT_FILE_0_1, expLastReadRowIdx = 1)).foreach { testCase => test(s"Multi-part checkpoints - page size ${testCase.pageSize}") { runCompletePaginationTest( testCase = testCase, tableVersionOpt = Optional.of(0L), tablePath = getTestResourceFilePath("kernel-pagination-multi-part-checkpoints")) } } // ===== Multi-part checkpoint files with JSON files test cases ===== /** * Log Segment List: * 00000000000000000002.json (1 AddFile) * Batch A: 1 AddFile, 2 rows * 00000000000000000001.json (1 AddFile) * Batch B: 1 AddFile, 2 rows * 00000000000000000000.checkpoint.0000000003.0000000003.parquet (6 AddFile) * Batch C: 5 AddFile, 5 rows * Batch D: 1 AddFile, 1 row * 00000000000000000000.checkpoint.0000000002.0000000003.parquet (7 AddFile) * Batch E: 5 AddFile, 5 row * Batch F: 2 AddFile, 2 row * 00000000000000000000.checkpoint.0000000001.0000000003.parquet (5 AddFile) * Batch G: 3 AddFile, 5 row * Batch H: 2 AddFile, 2 row */ Seq( FirstPageRequestTestContext( pageSize = 1, expScanFilesCnt = 1, /* 1 AddFile in Batch A */ expBatchCnt = 1, /* return only batch A */ expLastReadLogFile = JSON_FILE_2, /* Batch A from JSON file 2 */ expLastReadRowIdx = 1 ), /* Last Row index in Batch A is 1 in JSON file 2 */ FirstPageRequestTestContext( pageSize = 2, expScanFilesCnt = 2, /* 1 (batch A) + 1 (batch B) */ expBatchCnt = 2, /* return batch A, B */ expLastReadLogFile = JSON_FILE_1, /* Batch B from JSON file 1 */ expLastReadRowIdx = 1 ), /* Last Row index in Batch B is 1 in JSON file 1 */ FirstPageRequestTestContext( pageSize = 6, expScanFilesCnt = 7, /* 1 (batch A) + 1 (batch B) + 5 (batch C) */ expBatchCnt = 3, /* return batch A, B, C */ expLastReadLogFile = MULTI_CHECKPOINT_FILE_0_3, /* Batch C from checkpoint file 3 */ expLastReadRowIdx = 4 ), /* Last Row index in Batch C is 4 in checkpoint file 3 */ FirstPageRequestTestContext( pageSize = 100, expScanFilesCnt = 20, /* 1 (batch A) + 1 (batch B) + 6 (batch C+D) + 7 (batch E+F) + 5 (batch G+H) */ expBatchCnt = 8, /* return all batches A through H */ expLastReadLogFile = MULTI_CHECKPOINT_FILE_0_1, /* Batch H from checkpoint file 1 */ expLastReadRowIdx = 1 ) /* Last Row index in Batch H is 1 in checkpoint file 1 */ ).foreach { testCase => test(s"Multi-part checkpoints with jsons - page size ${testCase.pageSize}") { runCompletePaginationTest( testCase = testCase, tablePath = getTestResourceFilePath("kernel-pagination-multi-part-checkpoints")) } } // ===== V2 parquet checkpoint with sidecar files test cases ===== /** * 2.checkpoint.uuid.parquet: * - Batch A: 5 rows, 0 selected AddFiles * sidecar 1: * - Batch B: 3 rows, 3 selected AddFiles * sidecar 2: * - Batch C: 1 row, 1 selected AddFiles */ val PARQUET_MANIFEST_SIDECAR_1 = "00000000000000000002.checkpoint.0000000001.0000000002." + "055454d8-329c-4e0e-864d-7f867075af33.parquet" val PARQUET_MANIFEST_SIDECAR_2 = "00000000000000000002.checkpoint.0000000002.0000000002." + "33321cc1-9c55-4d1f-8511-fafe6d2e1133.parquet" Seq( FirstPageRequestTestContext( pageSize = 1, expScanFilesCnt = 3, /* 0 (batch A) + 3 (batch B) */ expBatchCnt = 2, /* return batch A, B */ expLastReadLogFile = PARQUET_MANIFEST_SIDECAR_1, /* Batch B from sidecar 1 */ expLastReadRowIdx = 2 ), /* Last Row index in Batch B is 2 in sidecar 1 */ FirstPageRequestTestContext( pageSize = 3, expScanFilesCnt = 3, /* 0 (batch A) + 3 (batch B) */ expBatchCnt = 2, /* return batch A, B */ expLastReadLogFile = PARQUET_MANIFEST_SIDECAR_1, /* Batch B from sidecar 1 */ expLastReadRowIdx = 2 ), /* Last Row index in Batch B is 2 in sidecar 1 */ FirstPageRequestTestContext( pageSize = 4, expScanFilesCnt = 4, /* 0 (batch A) + 3 (batch B) + 1 (batch C) */ expBatchCnt = 3, /* return batch A, B, C */ expLastReadLogFile = PARQUET_MANIFEST_SIDECAR_2, /* Batch C from sidecar 2 */ expLastReadRowIdx = 0 ), /* Last Row index in Batch C is 0 in sidecar 2 */ FirstPageRequestTestContext( pageSize = 10000, expScanFilesCnt = 4, /* 0 (batch A) + 3 (batch B) + 1 (batch C) */ expBatchCnt = 3, /* return batch A, B, C */ expLastReadLogFile = PARQUET_MANIFEST_SIDECAR_2, /* Batch C from sidecar 2 */ expLastReadRowIdx = 0 ) /* Last Row index in Batch C is 0 in sidecar 2 */ ).foreach { testCase => test(s"V2 parquet checkpoints (and sidecars) - page size ${testCase.pageSize}") { runCompletePaginationTest( testCase = testCase, tableVersionOpt = Optional.of(2L), tablePath = getTestResourceFilePath("kernel-pagination-v2-checkpoint-parquet")) } } // ===== V2 parquet checkpoint with sidecar files with jsons test cases ===== /** * 00000000000000000003.json * - Batch A: 2 rows, 1 selected AddFiles * 2.checkpoint.uuid.parquet: * - Batch B: 5 rows, 0 selected AddFiles * sidecar 1: * - Batch C: 3 rows, 3 selected AddFiles * sidecar 2: * - Batch D: 1 row, 1 selected AddFiles */ val JSON_FILE_3 = "00000000000000000003.json" Seq( FirstPageRequestTestContext( pageSize = 1, expScanFilesCnt = 1, /* 1 AddFile in Batch A */ expBatchCnt = 1, /* return only batch A */ expLastReadLogFile = JSON_FILE_3, /* Batch A from JSON file 3 */ expLastReadRowIdx = 1 ), /* Last Row index in Batch A is 1 in JSON file 3 */ FirstPageRequestTestContext( pageSize = 2, expScanFilesCnt = 4, /* 1 (batch A) + 0 (batch B) + 3 (batch C) */ expBatchCnt = 3, /* return batch A, B, C */ expLastReadLogFile = PARQUET_MANIFEST_SIDECAR_1, /* Batch C from sidecar 1 */ expLastReadRowIdx = 2 ), /* Last Row index in Batch C is 2 in sidecar 1 */ FirstPageRequestTestContext( pageSize = 3, expScanFilesCnt = 4, /* 1 (batch A) + 0 (batch B) + 3 (batch C) */ expBatchCnt = 3, /* return batch A, B, C */ expLastReadLogFile = PARQUET_MANIFEST_SIDECAR_1, /* Batch C from sidecar 1 */ expLastReadRowIdx = 2 ), /* Last Row index in Batch C is 2 in sidecar 1 */ FirstPageRequestTestContext( pageSize = 4, expScanFilesCnt = 4, /* 1 (batch A) + 0 (batch B) + 3 (batch C) */ expBatchCnt = 3, /* return batch A, B, C */ expLastReadLogFile = PARQUET_MANIFEST_SIDECAR_1, /* Batch C from sidecar 1 */ expLastReadRowIdx = 2 ), /* Last Row index in Batch C is 2 in sidecar 1 */ FirstPageRequestTestContext( pageSize = 10000, expScanFilesCnt = 5, /* 1 (batch A) + 0 (batch B) + 3 (batch C) + 1 (batch D) */ expBatchCnt = 4, /* return batch A, B, C, D */ expLastReadLogFile = PARQUET_MANIFEST_SIDECAR_2, /* Batch D from sidecar 2 */ expLastReadRowIdx = 0 ) /* Last Row index in Batch D is 0 in sidecar 2 */ ).foreach { testCase => test( s"v2 parquet checkpoints (and sidecars) " + s"with json delta commit files - page size ${testCase.pageSize}") { runCompletePaginationTest( testCase = testCase, tableVersionOpt = Optional.of(3L), tablePath = getTestResourceFilePath("kernel-pagination-v2-checkpoint-parquet")) } } // ===== V2 json checkpoint with sidecar files test cases ===== /** * 2.checkpoint.uuid.json: * - Batch A: 5 rows, 0 selected AddFiles * sidecar 1: * - Batch B: 1 rows, 1 selected AddFiles * sidecar 2: * - Batch C: 3 row, 3 selected AddFiles */ val JSON_MANIFEST = "00000000000000000002.checkpoint." + "6374b053-df23-479b-b2cf-c9c550132b49.json" val JSON_MANIFEST_SIDECAR_1 = "00000000000000000002.checkpoint.0000000001.0000000002." + "bd1885fd-6ec0-4370-b0f5-43b5162fd4de.parquet" val JSON_MANIFEST_SIDECAR_2 = "00000000000000000002.checkpoint.0000000002.0000000002." + "0a8d73ee-aa83-49d0-9583-c99db75b89b2.parquet" Seq( FirstPageRequestTestContext( pageSize = 1, expScanFilesCnt = 1, /* 0 (batch A) + 1 (batch B) */ expBatchCnt = 2, /* return batch A, B */ expLastReadLogFile = JSON_MANIFEST_SIDECAR_1, /* Batch B from sidecar 1 */ expLastReadRowIdx = 0 ), /* Last Row index in Batch B is 0 in sidecar 1 */ FirstPageRequestTestContext( pageSize = 2, expScanFilesCnt = 4, /* 0 (batch A) + 1 (batch B) + 3 (batch C) */ expBatchCnt = 3, /* return batch A, B, C */ expLastReadLogFile = JSON_MANIFEST_SIDECAR_2, /* Batch C from sidecar 2 */ expLastReadRowIdx = 2 ), /* Last Row index in Batch C is 2 in sidecar 2 */ FirstPageRequestTestContext( pageSize = 4, expScanFilesCnt = 4, /* 0 (batch A) + 1 (batch B) + 3 (batch C) */ expBatchCnt = 3, /* return batch A, B, C */ expLastReadLogFile = JSON_MANIFEST_SIDECAR_2, /* Batch C from sidecar 2 */ expLastReadRowIdx = 2 ), /* Last Row index in Batch C is 2 in sidecar 2 */ FirstPageRequestTestContext( pageSize = 10000, expScanFilesCnt = 4, /* 0 (batch A) + 1 (batch B) + 3 (batch C) */ expBatchCnt = 3, /* return batch A, B, C */ expLastReadLogFile = JSON_MANIFEST_SIDECAR_2, /* Batch C from sidecar 2 */ expLastReadRowIdx = 0 ) /* Last Row index in Batch C is 0 in sidecar 2 */ ).foreach { testCase => test(s"v2 json checkpoint (and sidecars) - page size ${testCase.pageSize}") { runCompletePaginationTest( testCase = testCase, tableVersionOpt = Optional.of(2L), tablePath = getTestResourceFilePath("kernel-pagination-v2-checkpoint-json")) } } // ===== V2 json checkpoint with sidecar files with jsons test cases ===== /** * 00000000000000000003.json * - Batch A: 2 rows, 1 selected AddFiles * 2.checkpoint.uuid.json: * - Batch B: 5 rows, 0 selected AddFiles * sidecar 1: * - Batch C: 1 rows, 1 selected AddFiles * sidecar 2: * - Batch D: 3 row, 3 selected AddFiles */ Seq( FirstPageRequestTestContext( pageSize = 1, expScanFilesCnt = 1, /* 1 AddFile in Batch A */ expBatchCnt = 1, /* return only batch A */ expLastReadLogFile = JSON_FILE_3, /* Batch A from JSON file 3 */ expLastReadRowIdx = 1 ), /* Last Row index in Batch A is 1 in JSON file 3 */ FirstPageRequestTestContext( pageSize = 3, expScanFilesCnt = 5, /* 1 (batch A) + 0 (batch B) + 1 (batch C) + 3 (batch D) */ expBatchCnt = 4, /* return batch A, B, C, D */ expLastReadLogFile = JSON_MANIFEST_SIDECAR_2, /* Batch D from sidecar 2 */ expLastReadRowIdx = 2 ), /* Last Row index in Batch D is 2 in sidecar 2 */ FirstPageRequestTestContext( pageSize = 4, expScanFilesCnt = 5, /* 1 (batch A) + 0 (batch B) + 1 (batch C) + 3 (batch D) */ expBatchCnt = 4, /* return batch A, B, C, D */ expLastReadLogFile = JSON_MANIFEST_SIDECAR_2, /* Batch D from sidecar 2 */ expLastReadRowIdx = 2 ), /* Last Row index in Batch D is 2 in sidecar 2 */ FirstPageRequestTestContext( pageSize = 10000, expScanFilesCnt = 5, /* 1 (batch A) + 0 (batch B) + 1 (batch C) + 3 (batch D) */ expBatchCnt = 4, /* return batch A, B, C, D */ expLastReadLogFile = JSON_MANIFEST_SIDECAR_2, /* Batch D from sidecar 2 */ expLastReadRowIdx = 2 ) /* Last Row index in Batch D is 2 in sidecar 2 */ ).foreach { testCase => test(s"v2 json checkpoint files (and sidecars) with json delta commit files - " + s"page size ${testCase.pageSize}") { runCompletePaginationTest( testCase = testCase, tableVersionOpt = Optional.of(3L), tablePath = getTestResourceFilePath("kernel-pagination-v2-checkpoint-json")) } } // TODO: tests for log compaction files } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/PartitionPruningSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.math.{BigDecimal => BigDecimalJ} import io.delta.golden.GoldenTableUtils.goldenTablePath import io.delta.kernel.defaults.utils.{ExpressionTestUtils, TestRow, TestUtils} import io.delta.kernel.expressions.{Column, Literal, Predicate} import io.delta.kernel.expressions.Literal._ import io.delta.kernel.types._ import io.delta.kernel.types.TimestampNTZType.TIMESTAMP_NTZ import org.scalatest.funsuite.AnyFunSuite class PartitionPruningSuite extends AnyFunSuite with TestUtils with ExpressionTestUtils { // scalastyle:off sparkimplicits // scalastyle:on sparkimplicits // Test golden table containing partition columns of all simple types val allTypesPartitionTable = goldenTablePath("data-reader-partition-values") // Test case to verify pruning on each partition column type works. // format: partition column reference -> (nonNullPartitionValues, nullPartitionValue) val testCasesAllTypes = Map( col("as_boolean") -> (ofBoolean(false), ofNull(BooleanType.BOOLEAN)), col("as_byte") -> (ofByte(1), ofNull(ByteType.BYTE)), col("as_short") -> (ofShort(1), ofNull(ShortType.SHORT)), col("as_int") -> (ofInt(1), ofNull(IntegerType.INTEGER)), col("as_long") -> (ofLong(1), ofNull(LongType.LONG)), col("as_float") -> (ofFloat(1), ofNull(FloatType.FLOAT)), col("as_double") -> (ofDouble(1), ofNull(DoubleType.DOUBLE)), // 2021-09-08 in days since epoch 18878 col("as_date") -> (ofDate(18878 /* daysSinceEpochUTC */ ), ofNull(DateType.DATE)), col("as_string") -> (ofString("1"), ofNull(StringType.STRING)), // 2021-09-08 11:11:11 in micros since epoch UTC col("as_timestamp") -> (ofTimestamp(1631099471000000L), ofNull(TimestampType.TIMESTAMP)), col("as_big_decimal") -> ( ofDecimal(new BigDecimalJ(1), 1, 0), ofNull(new DecimalType(1, 0)))) // Test for each partition column data type with partition value equal to non-null and null each // Try with or without selecting the partition column that has the predicate testCasesAllTypes.foreach { case (partitionCol, (nonNullLiteral, nullLiteral)) => Seq(nonNullLiteral, nullLiteral).foreach { literal => Seq(true, false).foreach { selectPredicatePartitionCol => test(s"partition pruning: simple filter `$partitionCol = $literal`, " + s"select partition predicate column = $selectPredicatePartitionCol") { val isPartitionColDateOrTimestampType = literal.getDataType.isInstanceOf[DateType] || literal.getDataType.isInstanceOf[TimestampType] val filter = predicate("=", partitionCol, literal) val expectedResult = if (literal.getValue == null) { Seq.empty // part1 == null should always return false - that means no results } else { if (selectPredicatePartitionCol) { if (isPartitionColDateOrTimestampType) { // Date and timestamp type has two partitions with the same value in golden table Seq((literal.getValue, 0L, "0"), (literal.getValue, 1L, "1")) } else { Seq((literal.getValue, 1L, "1")) } } else { if (isPartitionColDateOrTimestampType) { // Date and timestamp type has two partitions with the same value in golden table Seq((0L, "0"), (1L, "1")) } else { Seq((1L, "1")) } } } // "value" is a non-partition column val selectedColumns = if (selectPredicatePartitionCol) { Seq(partColName(partitionCol), "as_long", "value") } else { Seq("as_long", "value") } checkTable( path = allTypesPartitionTable, expectedAnswer = expectedResult.map(TestRow.fromTuple(_)), readCols = selectedColumns, filter = filter, expectedRemainingFilter = null) } } } } // Various combinations of predicate mix on partition and/or data columns mixes with AND or OR // test case format: (test_name, predicate) -> (remainingPredicate, expectedResults) // expected results is for query selecting `as_date` (partition column) and `value` (data column) val combinationTestCases = Map( ( "partition pruning: with predicate on two different partition col combined with AND", and( predicate(">=", col("as_float"), ofFloat(-200)), predicate("=", col("as_date"), ofDate(18878 /* daysSinceEpochUTC */ )))) -> ( null, Seq((18878, "0"), (18878, "1"))), ( "partition pruning: with predicate on two different partition col combined with OR", or( predicate("=", col("as_float"), ofFloat(0)), predicate("=", col("as_int"), ofInt(1)))) -> (null, Seq((18878, "0"), (18878, "1"))), ( "partition pruning: with predicate on data and partition column mix with AND", and( predicate("=", col("as_value"), ofString("1")), // data col filter predicate("=", col("as_float"), ofFloat(0)) // partition col filter )) -> ( predicate("=", col("as_value"), ofString("1")), Seq((18878, "0"))), ( "partition pruning: with predicate on data and partition column mix with OR", or( predicate("=", col("as_value"), ofString("1")), // data col filter predicate("=", col("as_float"), ofFloat(0)) // partition col filter )) -> ( or( predicate("=", col("as_value"), ofString("1")), // data col filter predicate("=", col("as_float"), ofFloat(0)) // partition col filter ), Seq((18878, "0"), (18878, "1"), (null, "2"))), ( "partition pruning: partition predicate prunes everything", and( predicate("=", col("as_value"), ofString("200")), // data col filter predicate("=", col("as_float"), ofFloat(234)) // partition col filter )) -> ( predicate("=", col("as_value"), ofString("200")), Seq())) combinationTestCases.foreach { case ((testTag, predicate), (expRemainingFilter, expResults)) => test(testTag) { checkTable( path = allTypesPartitionTable, expectedAnswer = expResults.map(TestRow.fromTuple(_)), readCols = Seq("as_date", "value"), filter = predicate, expectedRemainingFilter = expRemainingFilter) } } Seq("name", "id").foreach { mode => test(s"partition pruning on a column mapping enabled table: mode = $mode") { withTempDir { tempDir => val tablePath = tempDir.getCanonicalPath spark.sql( s"""CREATE TABLE delta.`$tablePath`(c1 long, c2 STRING, p1 STRING, p2 LONG) | USING delta PARTITIONED BY (p1, p2) | TBLPROPERTIES( |'delta.columnMapping.mode' = '$mode', |'delta.minReaderVersion' = '2', |'delta.minWriterVersion' = '5') |""".stripMargin) Seq.range(0, 5).foreach { i => spark.sql(s"insert into delta.`$tablePath` values ($i, '$i', '$i', $i)") } checkTable( tablePath, expectedAnswer = Seq((3L, "3"), (4L, "4")).map(TestRow.fromTuple(_)), readCols = Seq("p2", "c2"), filter = predicate(">=", col("p2"), ofLong(3)), expectedRemainingFilter = null) } } } Seq("", "-name-mode", "-id-mode").foreach { cmMode => // Below is the golden table used in test // (INTEGER id, TIMESTAMP_NTZ tsNtz, TIMESTAMP_NTZ tsNtzPartition) // (0, '2021-11-18 02:30:00.123456','2021-11-18 02:30:00.123456'), // (1, '2013-07-05 17:01:00.123456','2021-11-18 02:30:00.123456'), // (2, NULL, '2021-11-18 02:30:00.123456'), // (3, '2021-11-18 02:30:00.123456','2013-07-05 17:01:00.123456'), // (4, '2013-07-05 17:01:00.123456','2013-07-05 17:01:00.123456'), // (5, NULL, '2013-07-05 17:01:00.123456'), // (6, '2021-11-18 02:30:00.123456', NULL), // (7, '2013-07-05 17:01:00.123456', NULL), // (8, NULL, NULL) // test case (kernel predicate object, equivalent spark predicate as string, // expected row count, expected remaining filter) Seq( ( // 1637202600123456L in epoch micros for '2021-11-18 02:30:00.123456' predicate("=", col("tsNtzPartition"), ofTimestampNtz(1637202600123456L)), "tsNtzPartition = '2021-11-18 02:30:00.123456'", 3, // expected row count null.asInstanceOf[Predicate] // expected remaining filter ), ( predicate("=", col("tsNtzPartition"), Literal ofNull (TIMESTAMP_NTZ)), "tsNtzPartition = null", 0, // expected row count null.asInstanceOf[Predicate] // expected remaining filter ), ( // 1373043660123456L in epoch micros for '2013-07-05 17:01:00.123456' predicate(">=", col("tsNtzPartition"), ofTimestampNtz(1373043660123456L)), "tsNtzPartition >= '2013-07-05 17:01:00'", 6, // expected row count null.asInstanceOf[Predicate] // expected remaining filter ), ( predicate("IS_NULL", col("tsNtzPartition")), "tsNtzPartition IS NULL", 3, // expected row count null.asInstanceOf[Predicate] // expected remaining filter ), ( // Filter on just the data column // 1637202600123456L in epoch micros for '2021-11-18 02:30:00.123456' predicate( "OR", predicate("=", col("tsNtz"), ofTimestampNtz(1637202600123456L)), predicate("=", col("tsNtz"), ofTimestampNtz(1373043660123456L))), "", 9, // expected row count // expected remaining filter predicate( "OR", predicate("=", col("tsNtz"), ofTimestampNtz(1637202600123456L)), predicate("=", col("tsNtz"), ofTimestampNtz(1373043660123456L))))).foreach { case (kernelPredicate, sparkPredicate, expectedRowCount, expRemainingFilter) => test(s"partition pruning on timestamp_ntz columns: $cmMode ($kernelPredicate)") { val tablePath = goldenTablePath(s"data-reader-timestamp_ntz$cmMode") val expectedResult = readUsingSpark(tablePath, sparkPredicate) assert(expectedResult.size === expectedRowCount) checkTable( expectedAnswer = expectedResult, path = tablePath, expectedRemainingFilter = expRemainingFilter, filter = kernelPredicate) } } } test("partition pruning from checkpoint") { withTempDir { path => withTempTable { tbl => // Create partitioned table and insert some data, ensuring that a checkpoint is created // after the last insertion. spark.sql(s"CREATE TABLE $tbl (a INT, b STRING) USING delta " + s"PARTITIONED BY (a) LOCATION '$path' " + s"TBLPROPERTIES ('delta.checkpointInterval' = '2')") spark.sql(s"INSERT INTO $tbl VALUES (1, 'a'), (2, 'b')") spark.sql(s"INSERT INTO $tbl VALUES (3, 'c'), (4, 'd')") spark.sql(s"INSERT INTO $tbl VALUES (5, 'e'), (6, 'f')") // Read from the source table with a partition predicate and validate the results. val result = readSnapshot( latestSnapshot(path.toString), filter = greaterThan(col("a"), Literal.ofInt(3))) checkAnswer(result, Seq(TestRow(4, "d"), TestRow(5, "e"), TestRow(6, "f"))) } } } private def readUsingSpark(tablePath: String, predicate: String): Seq[TestRow] = { val where = if (predicate.isEmpty) "" else s"WHERE $predicate" spark.sql(s"SELECT * FROM delta.`$tablePath` $where") .collect() .map(TestRow(_)) } private def partColName(column: Column): String = { assert(column.getNames.length == 1) column.getNames()(0) } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/PartitionUtilsSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import io.delta.kernel.Table import io.delta.kernel.defaults.utils.{ExpressionTestUtils, TestRow, TestUtils} import io.delta.kernel.expressions.Literal.ofInt import io.delta.kernel.utils.PartitionUtils import org.apache.spark.sql.functions.{col => sparkCol} import org.scalatest.funsuite.AnyFunSuite class PartitionUtilsSuite extends AnyFunSuite with TestUtils with ExpressionTestUtils { private def createTableWithPartCols(path: String): Unit = { spark.range(100) .withColumn("part1", sparkCol("id") % 2) .withColumn("part2", sparkCol("id") % 5) .withColumn("col1", sparkCol("id")) .withColumn("col2", sparkCol("id")) .drop("id") .write .format("delta") .partitionBy("part1", "part2") .save(path) } Seq("name", "id").foreach { mode => test(s"withPartitionColumns - read subset of partition cols with column mapping mode = $mode") { withTempDir { tempDir => val tablePath = tempDir.getCanonicalPath spark.sql( s"""CREATE TABLE delta.`$tablePath`(c1 long, c2 STRING, p1 STRING, p2 LONG) | USING delta PARTITIONED BY (p1, p2) | TBLPROPERTIES( |'delta.columnMapping.mode' = '$mode') |""".stripMargin) Seq.range(0, 5).foreach { i => spark.sql(s"insert into delta.`$tablePath` values ($i, '$i', '$i', $i)") } checkTable( tablePath, expectedAnswer = Seq((0L, "0"), (1L, "1"), (2L, "2"), (3L, "3"), (4L, "4")).map(TestRow.fromTuple(_)), readCols = Seq("p2", "c2")) } } } test("partitionExists - input validation") { withTempDirAndEngine { (path, engine) => createTableWithPartCols(path) val snapshot = Table.forPath(engine, path).getLatestSnapshot(engine) { val badPredicate = and( predicate("=", col("part1"), ofInt(0)), predicate("=", col("col1"), ofInt(0))) val exMsg = intercept[IllegalArgumentException] { PartitionUtils.partitionExists(engine, snapshot, badPredicate) }.getMessage assert(exMsg.contains("Partition predicate must contain only partition columns")) } { val badPredicate = predicate("=", col("col1"), ofInt(0)) val exMsg = intercept[IllegalArgumentException] { PartitionUtils.partitionExists(engine, snapshot, badPredicate) }.getMessage assert(exMsg.contains("Partition predicate must contain at least one partition column")) } } } test("partitionExists - simple case using latest table snapshot") { withTempDirAndEngine { (path, engine) => createTableWithPartCols(path) val snapshot = Table.forPath(engine, path).getLatestSnapshot(engine) // ===== Simple Cases ===== { val partPredicate = predicate("=", col("part1"), ofInt(1)) // Yes assert(PartitionUtils.partitionExists(engine, snapshot, partPredicate)) } { val partPredicate = predicate(">=", col("part1"), ofInt(500)) // No assert(!PartitionUtils.partitionExists(engine, snapshot, partPredicate)) } { val partPredicate = predicate("<", col("part1"), ofInt(100)) // Yes assert(PartitionUtils.partitionExists(engine, snapshot, partPredicate)) } // ===== Conjunction and Disjunction Cases ===== { val partPredicate = and( predicate("=", col("part1"), ofInt(0)), // Yes predicate("=", col("part2"), ofInt(4)) // Yes ) assert(PartitionUtils.partitionExists(engine, snapshot, partPredicate)) } { val partPredicate = and( predicate("=", col("part1"), ofInt(0)), // Yes predicate("=", col("part2"), ofInt(500)) // No ) assert(!PartitionUtils.partitionExists(engine, snapshot, partPredicate)) } { val partPredicate = or( predicate("=", col("part1"), ofInt(500)), // No predicate("=", col("part2"), ofInt(3)) // N/A ) assert(PartitionUtils.partitionExists(engine, snapshot, partPredicate)) } } } test("partitionExists - some or all data in partition was removed") { withTempDirAndEngine { (path, engine) => createTableWithPartCols(path) spark.sql(s"DELETE FROM delta.`$path` WHERE part1 = 0") spark.sql(s"DELETE FROM delta.`$path` WHERE part1 = 1 AND part2 = 0") spark.sql(s"DELETE FROM delta.`$path` WHERE part1 = 1 AND part2 = 1 AND col1 < 50") val snapshot = Table.forPath(engine, path).getLatestSnapshot(engine) { val partPredicate = predicate("=", col("part1"), ofInt(0)) // No assert(!PartitionUtils.partitionExists(engine, snapshot, partPredicate)) } { val partPredicate = and( predicate("=", col("part1"), ofInt(1)), // Yes predicate("=", col("part2"), ofInt(0)) // No ) assert(!PartitionUtils.partitionExists(engine, snapshot, partPredicate)) } { val partPredicate = and( predicate("=", col("part1"), ofInt(1)), // Yes predicate("=", col("part2"), ofInt(1)) // Has some data ) assert(PartitionUtils.partitionExists(engine, snapshot, partPredicate)) } } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/PostCommitSnapshotSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import scala.collection.immutable.Seq import io.delta.kernel.{Operation, Snapshot, TransactionCommitResult} import io.delta.kernel.Snapshot.ChecksumWriteMode import io.delta.kernel.defaults.utils.{TestRow, WriteUtilsWithV2Builders} import io.delta.kernel.engine.Engine import io.delta.kernel.expressions.Literal import io.delta.kernel.internal.{InternalScanFileUtils, SnapshotImpl, TableConfig} import io.delta.kernel.internal.tablefeatures.TableFeatures import io.delta.kernel.test.MockEngineUtils import io.delta.kernel.types.IntegerType.INTEGER import io.delta.kernel.utils.CloseableIterable.inMemoryIterable import org.scalatest.funsuite.AnyFunSuite /** * Test suite for validating the behavior of postCommitSnapshot in various scenarios. * * Note that we use "PCS" in our test names for brevity. */ class PostCommitSnapshotSuite extends AnyFunSuite with WriteUtilsWithV2Builders with MockEngineUtils { ////////////////// // Test Helpers // ////////////////// private def assertAddFilesMatch( engine: Engine, actual: SnapshotImpl, expected: SnapshotImpl): Unit = { val actualFiles = collectScanFileRows(actual.getScanBuilder.build(), engine) .map(x => InternalScanFileUtils.getAddFileStatus(x).getPath) val expectedFiles = collectScanFileRows(expected.getScanBuilder.build(), engine) .map(x => InternalScanFileUtils.getAddFileStatus(x).getPath) assert(actualFiles === expectedFiles) } private def checkPostCommitSnapshot( engine: Engine, postCommitSnapshot: Snapshot, expectCrc: Boolean = false): Unit = { val actual = postCommitSnapshot.asInstanceOf[SnapshotImpl] val expected = latestSnapshot(actual.getPath, engine) if (expectCrc) { assert(actual.getCurrentCrcInfo.isPresent) } // TODO: We need better visibility into when the below information is loaded from the log, // loaded from CRC, or already stored in memory (i.e. injected during post-commit snapshot // creation) assert(actual.getVersion === expected.getVersion) assert(actual.getSchema === expected.getSchema) assert(actual.getProtocol === expected.getProtocol) assert(actual.getMetadata === expected.getMetadata) assert(actual.getPartitionColumnNames === expected.getPartitionColumnNames) assert(actual.getPhysicalClusteringColumns === expected.getPhysicalClusteringColumns) assert(actual.getTimestamp(engine) === expected.getTimestamp(engine)) assert(actual.getActiveDomainMetadataMap === expected.getActiveDomainMetadataMap) assertAddFilesMatch(engine, actual, expected) } //////////////////////////// // Create new table tests // //////////////////////////// test("creating a new empty table => yields a PCS") { withTempDirAndEngine { (tablePath, engine) => val result = createEmptyTable(engine, tablePath, testSchema) checkPostCommitSnapshot(engine, result.getPostCommitSnapshot.get(), expectCrc = true) } } test("creating a new table with data => yields a PCS") { withTempDirAndEngine { (tablePath, engine) => val result = appendData( engine, tablePath, isNewTable = true, schema = testSchema, data = seqOfUnpartitionedDataBatch1) checkPostCommitSnapshot(engine, result.getPostCommitSnapshot.get(), expectCrc = true) } } ///////////////////////// // CRC existence tests // ///////////////////////// test("commit at readVersion + 1 (*with* CRC at readVersion) => yields a PCS with CRC") { withTempDirAndEngine { (tablePath, engine) => val result0 = createEmptyTable(engine, tablePath, testSchema) result0.getPostCommitSnapshot.get().writeChecksum(engine, ChecksumWriteMode.SIMPLE) assert(latestSnapshot(tablePath, engine).getCurrentCrcInfo.isPresent) val result1 = appendData(engine, tablePath, data = seqOfUnpartitionedDataBatch1) checkPostCommitSnapshot(engine, result1.getPostCommitSnapshot.get(), expectCrc = true) } } test("commit at readVersion + 1 (*without* CRC at readVersion) => yields a PCS without CRC") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable(engine, tablePath, testSchema) val result1 = appendData(engine, tablePath, data = seqOfUnpartitionedDataBatch1) checkPostCommitSnapshot(engine, result1.getPostCommitSnapshot.get(), expectCrc = false) } } test("commit at readVersion + 2 => does NOT yield a PCS") { withTempDirAndEngine { (tablePath, engine) => // ===== GIVEN ===== createEmptyTable(engine, tablePath, testSchema) // V0 val txn = getUpdateTxn(engine, tablePath) // Create a transaction that reads at V0 assert(txn.getReadTableVersion == 0) // Create winning commits at V1 and V2 appendData(engine, tablePath, data = seqOfUnpartitionedDataBatch1) // V1 appendData(engine, tablePath, data = seqOfUnpartitionedDataBatch2) // V2 assert(latestSnapshot(tablePath, engine).getVersion == 2) // ===== WHEN ===== // Now commit the original txn that read at v0. This will commit at v3 val txnState = txn.getTransactionState(engine) val actions = inMemoryIterable(stageData(txnState, Map.empty[String, Literal], dataBatches1)) val result = commitTransaction(txn, engine, actions) // V3 // ===== THEN ===== assert(result.getVersion == 3) assert(!result.getPostCommitSnapshot.isPresent) } } //////////////////////////////////////////////////////////// // PostCommitSnapshot has certain fields pre-loaded tests // //////////////////////////////////////////////////////////// test("PCS has ICT pre-loaded") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, testSchema, tableProperties = Map(TableConfig.IN_COMMIT_TIMESTAMPS_ENABLED.getKey -> "true")) val result = appendData(engine, tablePath, data = seqOfUnpartitionedDataBatch1) val postCommitSnapshot = result.getPostCommitSnapshot.get().asInstanceOf[SnapshotImpl] val failingEngine = mockEngine() // should *not* use the engine to try and read ICT from delta file postCommitSnapshot.getTimestamp(failingEngine) } } // TODO: Test CRC is also pre-loaded. Requires // (1) SnapshotImpl::getCurrentCrcInfo to take in an engine param // (2) LogReplay to *not* be injected into SnapshotImpl constructor // (3) CRC to be injected into SnapshotImpl constructor // TODO: Test clusteringColumns are pre-loaded (when txn sets new clustering columns) /////////////////////////// // Metadata change tests // /////////////////////////// case class MetadataChangeTestCase( changeType: String, initTableProperties: Map[String, String] = Map.empty, updateFn: (Engine, String) => TransactionCommitResult) Seq( MetadataChangeTestCase( changeType = "schema", initTableProperties = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> "name"), updateFn = (engine, tablePath) => { val newSchema = latestSnapshot(tablePath, engine).getSchema.add("newCol", INTEGER) updateTableMetadata(engine, tablePath, schema = newSchema) }), MetadataChangeTestCase( changeType = "tbl property", updateFn = (engine, tablePath) => updateTableMetadata(engine, tablePath, tableProperties = Map("foo" -> "bar"))), MetadataChangeTestCase( changeType = "protocol", updateFn = (engine, tablePath) => { val snapshot = latestSnapshot(tablePath, engine) assert(!snapshot.getProtocol.getWriterFeatures.contains("deletionVectors")) updateTableMetadata( engine, tablePath, tableProperties = Map("delta.enableDeletionVectors" -> "true")) }), MetadataChangeTestCase( changeType = "clustering columns", updateFn = (engine, tablePath) => updateTableMetadata( engine, tablePath, clusteringColsOpt = Some(testClusteringColumns)))).foreach { case MetadataChangeTestCase(changeType, initTableProperties, updateFn) => test( s"commit $changeType change at readVersion + 1 => yields a PCS with updated $changeType") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable( engine, tablePath, schema = testPartitionSchema, tableProperties = initTableProperties) val result = updateFn(engine, tablePath) checkPostCommitSnapshot(engine, result.getPostCommitSnapshot.get()) } } } //////////////////////////////////////////////////////////////// // Using PostCommitSnapshot to read data and write data tests // //////////////////////////////////////////////////////////////// test("postCommitSnapshot can be used to read data") { withTempDirAndEngine { (tablePath, engine) => val result = appendData( engine, tablePath, isNewTable = true, schema = testSchema, data = seqOfUnpartitionedDataBatch1) val postCommitSnapshot = result.getPostCommitSnapshot.get() val expectedData = dataBatches1.flatMap(_.toTestRows) val dataFromPostCommit = readSnapshot(postCommitSnapshot, engine = engine).map(TestRow(_)) checkAnswer(dataFromPostCommit, expectedData) } } test("postCommitSnapshot can be used to start and commit a new transaction") { withTempDirAndEngine { (tablePath, engine) => // ===== GIVEN ===== val result0 = appendData( engine, tablePath, isNewTable = true, schema = testSchema, data = seqOfUnpartitionedDataBatch1) val postCommitSnapshot0 = result0.getPostCommitSnapshot.get() assert(postCommitSnapshot0.getVersion == 0) // ===== WHEN ===== // Use the postCommitSnapshot to start a new transaction and commit v1 val txn = postCommitSnapshot0 .buildUpdateTableTransaction(testEngineInfo, Operation.WRITE) .build(engine) val txnState = txn.getTransactionState(engine) val actions = inMemoryIterable(stageData(txnState, Map.empty[String, Literal], dataBatches2)) val result1 = commitTransaction(txn, engine, actions) // ===== THEN ===== checkPostCommitSnapshot(engine, result1.getPostCommitSnapshot.get()) } } //////////////// // Publishing // //////////////// test("publishing on a postCommitSnapshot for a filesystem-based table is a no-op") { withTempDirAndEngine { (tablePath, engine) => val result = appendData( engine, tablePath, isNewTable = true, schema = testSchema, data = seqOfUnpartitionedDataBatch1) val postCommitSnapshot = result.getPostCommitSnapshot.get().asInstanceOf[SnapshotImpl] assert(!TableFeatures.isCatalogManagedSupported(postCommitSnapshot.getProtocol)) postCommitSnapshot.publish(engine) // Should not throw -- there are no catalog commits! } } test("publishing on a postCommitSnapshot will return a snapshot") { withTempDirAndEngine { (tablePath, engine) => var result = appendData( engine, tablePath, isNewTable = true, schema = testSchema, data = seqOfUnpartitionedDataBatch1) var postCommitSnapshot = result.getPostCommitSnapshot.get().asInstanceOf[SnapshotImpl] var postPublishSnapshot = postCommitSnapshot.publish(engine) assert(postPublishSnapshot == postCommitSnapshot) result = appendData( engine, tablePath, isNewTable = false, data = seqOfUnpartitionedDataBatch1) postCommitSnapshot = result.getPostCommitSnapshot.get().asInstanceOf[SnapshotImpl] postPublishSnapshot = postCommitSnapshot.publish(engine) assert(postPublishSnapshot == postCommitSnapshot) } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/RowTrackingSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.util import java.util.Optional import scala.collection.JavaConverters._ import scala.collection.immutable.Seq import io.delta.kernel.Table import io.delta.kernel.data.{FilteredColumnarBatch, Row} import io.delta.kernel.defaults.internal.parquet.ParquetSuiteBase import io.delta.kernel.defaults.utils.{AbstractWriteUtils, TestRow, WriteUtilsWithV1Builders, WriteUtilsWithV2Builders} import io.delta.kernel.engine.Engine import io.delta.kernel.exceptions.{ConcurrentWriteException, InvalidTableException, KernelException, MaxCommitRetryLimitReachedException} import io.delta.kernel.expressions.Literal import io.delta.kernel.internal.{InternalScanFileUtils, SnapshotImpl, TableConfig, TableImpl} import io.delta.kernel.internal.actions.{AddFile, SingleAction} import io.delta.kernel.internal.rowtracking.{RowTracking, RowTrackingMetadataDomain} import io.delta.kernel.internal.rowtracking.MaterializedRowTrackingColumn.{MATERIALIZED_ROW_COMMIT_VERSION, MATERIALIZED_ROW_ID} import io.delta.kernel.internal.util.Utils.toCloseableIterator import io.delta.kernel.internal.util.VectorUtils import io.delta.kernel.types._ import io.delta.kernel.utils.{CloseableIterable, MetadataColumnTestUtils} import io.delta.kernel.utils.CloseableIterable.{emptyIterable, inMemoryIterable} import org.apache.spark.sql.delta.DeltaLog import org.apache.hadoop.fs.Path import org.scalatest.funsuite.AnyFunSuite /** Runs row tracking tests using the TableManager snapshot APIs and V2 transaction builders */ class RowTrackingSuite extends AbstractRowTrackingSuite with WriteUtilsWithV2Builders /** Runs row tracking tests using the legacy Table snapshot APIs and V1 transaction builders */ class LegacyRowTrackingSuite extends AbstractRowTrackingSuite with WriteUtilsWithV1Builders trait AbstractRowTrackingSuite extends AnyFunSuite with ParquetSuiteBase with MetadataColumnTestUtils { self: AbstractWriteUtils => private def prepareActionsForCommit(actions: Row*): CloseableIterable[Row] = { inMemoryIterable(toCloseableIterator(actions.asJava.iterator())) } private def createTableWithRowTracking( engine: Engine, tablePath: String, schema: StructType = testSchema, extraProps: Map[String, String] = Map.empty): Unit = { val tableProps = Map(TableConfig.ROW_TRACKING_ENABLED.getKey -> "true") ++ extraProps createEmptyTable(engine, tablePath, schema = schema, tableProperties = tableProps) } /** * Creates a table with row tracking enabled and inserts initial data, then performs a merge * operation to update some records and insert new ones. We use Spark SQL for this test table to * ensure that the result table has both materialized and not materialized row tracking columns. * * @param tablePath The path to the Delta table. * @param extraProps Additional table properties to set. */ private def createRowTrackingTableWithSpark( tablePath: String, extraProps: Map[String, String] = Map.empty): Unit = { val tblPropsStr = (extraProps + ("delta.enableRowTracking" -> "true")) .map { case (k, v) => s"'$k' = '$v'" }.mkString(", ") spark.sql( s""" |CREATE TABLE delta.`$tablePath` ( | id INT, | value STRING |) USING DELTA |TBLPROPERTIES ($tblPropsStr) |""".stripMargin) // Insert 5 records val initialData = Seq( (1, "A"), (2, "B"), (3, "C"), (4, "D"), (5, "E")) spark.createDataFrame(initialData).toDF( "id", "value").repartition(1).write.format("delta").mode("overwrite").save(tablePath) // Prepare source for merge val sourceData = Seq( (3, "C_updated"), // will update id=3 (6, "F") // will insert new id=6 ) spark.createDataFrame(sourceData).toDF("id", "value").createOrReplaceTempView("merge_source") // Merge: update id=3, insert id=6 spark.sql( s""" |MERGE INTO delta.`$tablePath` t |USING merge_source s |ON t.id = s.id |WHEN MATCHED THEN UPDATE SET t.value = s.value |WHEN NOT MATCHED THEN INSERT (id, value) VALUES (s.id, s.value) |""".stripMargin) } private def verifyBaseRowIDs( engine: Engine, tablePath: String, expectedValue: Seq[Long]): Unit = { val snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath).asInstanceOf[SnapshotImpl] val scanFileRows = collectScanFileRows(snapshot.getScanBuilder().build()) val sortedBaseRowIds = scanFileRows .map(InternalScanFileUtils.getBaseRowId) .map(_.orElse(-1)) .sorted assert(sortedBaseRowIds === expectedValue) } private def verifyDefaultRowCommitVersion( engine: Engine, tablePath: String, expectedValue: Seq[Long]) = { val snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath).asInstanceOf[SnapshotImpl] val scanFileRows = collectScanFileRows(snapshot.getScanBuilder().build()) val sortedAddFileDefaultRowCommitVersions = scanFileRows .map(InternalScanFileUtils.getDefaultRowCommitVersion) .map(_.orElse(-1)) .sorted assert(sortedAddFileDefaultRowCommitVersions === expectedValue) } private def verifyHighWatermark(engine: Engine, tablePath: String, expectedValue: Long): Unit = { val snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath).asInstanceOf[SnapshotImpl] val rowTrackingMetadataDomain = RowTrackingMetadataDomain.fromSnapshot(snapshot) assert(rowTrackingMetadataDomain.isPresent) assert(rowTrackingMetadataDomain.get().getRowIdHighWaterMark === expectedValue) } private def prepareDataForCommit(data: Seq[FilteredColumnarBatch]*) : Seq[(Map[String, Literal], Seq[FilteredColumnarBatch])] = { data.map(Map.empty[String, Literal] -> _).toIndexedSeq } test("RowTracking.isEnabled - returns false when row tracking is not enabled in metadata") { withTempDirAndEngine { (tablePath, engine) => // Create table with row tracking supported but not enabled createEmptyTable( engine, tablePath, testSchema, tableProperties = Map.empty) val snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath).asInstanceOf[SnapshotImpl] val protocol = snapshot.getProtocol val metadata = snapshot.getMetadata assert(!RowTracking.isEnabled(protocol, metadata)) } } test("RowTracking.isEnabled - returns true when row tracking is supported and enabled") { withTempDirAndEngine { (tablePath, engine) => // Create table with row tracking enabled createTableWithRowTracking(engine, tablePath) val snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath).asInstanceOf[SnapshotImpl] val protocol = snapshot.getProtocol val metadata = snapshot.getMetadata assert(RowTracking.isEnabled(protocol, metadata)) } } test( "RowTracking.isEnabled - throws exception when enabled in metadata but not " + "supported by protocol") { withTempDirAndEngine { (tablePath, engine) => // Create a table without row tracking support createEmptyTable(engine, tablePath, testSchema) // Get the current metadata and protocol val snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath).asInstanceOf[SnapshotImpl] val protocol = snapshot.getProtocol val originalMetadata = snapshot.getMetadata // Manually create metadata with row tracking enabled (bypassing validation) val configWithRowTracking = originalMetadata.getConfiguration.asScala.toMap + (TableConfig.ROW_TRACKING_ENABLED.getKey -> "true") val problematicMetadata = originalMetadata.withReplacedConfiguration(configWithRowTracking.asJava) // Verify that calling isEnabled throws IllegalStateException val e = intercept[IllegalStateException] { RowTracking.isEnabled(protocol, problematicMetadata) } assert(e.getMessage.contains( "Table property 'delta.enableRowTracking' is set on the table but this table version " + "doesn't support table feature 'delta.feature.rowTracking'")) } } test("Base row IDs/default row commit versions are assigned to AddFile actions") { withTempDirAndEngine { (tablePath, engine) => createTableWithRowTracking(engine, tablePath) val dataBatch1 = generateData(testSchema, Seq.empty, Map.empty, 100, 1) // 100 rows val dataBatch2 = generateData(testSchema, Seq.empty, Map.empty, 200, 1) // 200 rows val dataBatch3 = generateData(testSchema, Seq.empty, Map.empty, 400, 1) // 400 rows // Commit three files in one transaction val commitVersion = appendData( engine, tablePath, data = prepareDataForCommit(dataBatch1, dataBatch2, dataBatch3)).getVersion verifyBaseRowIDs(engine, tablePath, Seq(0, 100, 300)) verifyDefaultRowCommitVersion(engine, tablePath, Seq.fill(3)(commitVersion)) verifyHighWatermark(engine, tablePath, 699) } } test("Previous Row ID high watermark can be picked up to assign base row IDs") { withTempDirAndEngine { (tablePath, engine) => createTableWithRowTracking(engine, tablePath) val dataBatch1 = generateData(testSchema, Seq.empty, Map.empty, 100, 1) val commitVersion1 = appendData( engine, tablePath, data = Seq(dataBatch1).map(Map.empty[String, Literal] -> _)).getVersion verifyBaseRowIDs(engine, tablePath, Seq(0)) verifyDefaultRowCommitVersion(engine, tablePath, Seq(commitVersion1)) verifyHighWatermark(engine, tablePath, 99) val dataBatch2 = generateData(testSchema, Seq.empty, Map.empty, 200, 1) val commitVersion2 = appendData( engine, tablePath, data = prepareDataForCommit(dataBatch2)).getVersion verifyBaseRowIDs(engine, tablePath, Seq(0, 100)) verifyDefaultRowCommitVersion(engine, tablePath, Seq(commitVersion1, commitVersion2)) verifyHighWatermark(engine, tablePath, 299) } } test("Base row IDs/default row commit versions are preserved in checkpoint") { withTempDirAndEngine { (tablePath, engine) => createTableWithRowTracking(engine, tablePath) val dataBatch1 = generateData(testSchema, Seq.empty, Map.empty, 100, 1) val dataBatch2 = generateData(testSchema, Seq.empty, Map.empty, 200, 1) val dataBatch3 = generateData(testSchema, Seq.empty, Map.empty, 400, 1) val commitVersion1 = appendData( engine, tablePath, data = prepareDataForCommit(dataBatch1)).getVersion val commitVersion2 = appendData( engine, tablePath, data = prepareDataForCommit(dataBatch2)).getVersion // Checkpoint the table val latestVersion = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath).getVersion() TableImpl.forPath(engine, tablePath).checkpoint(engine, latestVersion) val commitVersion3 = appendData( engine, tablePath, data = prepareDataForCommit(dataBatch3)).getVersion verifyBaseRowIDs(engine, tablePath, Seq(0, 100, 300)) verifyDefaultRowCommitVersion( engine, tablePath, Seq(commitVersion1, commitVersion2, commitVersion3)) verifyHighWatermark(engine, tablePath, 699) } } test("Provided Row ID high watermark should be set in the txn") { withTempDirAndEngine { (tablePath, engine) => createTableWithRowTracking(engine, tablePath) val dataBatch1 = generateData(testSchema, Seq.empty, Map.empty, 100, 1) val commitVersion1 = appendData( engine, tablePath, data = Seq(dataBatch1).map(Map.empty[String, Literal] -> _)).getVersion verifyBaseRowIDs(engine, tablePath, Seq(0)) verifyDefaultRowCommitVersion(engine, tablePath, Seq(commitVersion1)) verifyHighWatermark(engine, tablePath, 99) val dataBatch2 = generateData(testSchema, Seq.empty, Map.empty, 200, 1) val rowTrackingMetadataDomain = new RowTrackingMetadataDomain(400) val txn2 = createTxnWithDomainMetadatas( engine, tablePath, List(rowTrackingMetadataDomain.toDomainMetadata)) val commitVersion2 = commitAppendData(engine, txn2, prepareDataForCommit(dataBatch2)).getVersion verifyBaseRowIDs(engine, tablePath, Seq(0, 100)) verifyDefaultRowCommitVersion(engine, tablePath, Seq(commitVersion1, commitVersion2)) verifyHighWatermark(engine, tablePath, 400) } } test("Fail if provided Row ID high watermark is smaller than the calculated high watermark") { withTempDirAndEngine { (tablePath, engine) => createTableWithRowTracking(engine, tablePath) val dataBatch1 = generateData(testSchema, Seq.empty, Map.empty, 100, 1) val commitVersion1 = appendData( engine, tablePath, data = Seq(dataBatch1).map(Map.empty[String, Literal] -> _)).getVersion verifyBaseRowIDs(engine, tablePath, Seq(0)) verifyDefaultRowCommitVersion(engine, tablePath, Seq(commitVersion1)) verifyHighWatermark(engine, tablePath, 99) val dataBatch2 = generateData(testSchema, Seq.empty, Map.empty, 200, 1) // Set a higher value than the calculated high watermark = 299 val rowTrackingMetadataDomain = new RowTrackingMetadataDomain(120) val txn2 = createTxnWithDomainMetadatas( engine, tablePath, List(rowTrackingMetadataDomain.toDomainMetadata)) val e = intercept[RuntimeException] { commitAppendData(engine, txn2, prepareDataForCommit(dataBatch2)).getVersion } assert( e.getMessage.contains( "The provided row ID high watermark (120) must be greater than " + "or equal to the calculated row ID high watermark (299) based " + "on the transaction's data actions.")) } } test("Fail if row tracking is supported but AddFile actions are missing stats") { withTempDirAndEngine { (tablePath, engine) => createTableWithRowTracking(engine, tablePath) val addFileRow = AddFile.createAddFileRow( null, "fakePath", VectorUtils.stringStringMapValue(new util.HashMap[String, String]()), 0L, 0L, false, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty() // No stats ) val action = SingleAction.createAddFileSingleAction(addFileRow) val txn = getUpdateTxn(engine, tablePath) // KernelException thrown inside a lambda is wrapped in a RuntimeException val e = intercept[RuntimeException] { txn.commit(engine, prepareActionsForCommit(action)) } assert( e.getMessage.contains( "Cannot write to a rowTracking-supported table without 'numRecords' statistics. " + "Connectors are expected to populate the number of records statistics when " + "writing to a Delta table with 'rowTracking' table feature supported.")) } } test("Fail if row tracking is not supported but client call withHighWatermark in txn") { withTempDirAndEngine { (tablePath, engine) => createEmptyTable(engine, tablePath, testSchema) val rowTrackingMetadataDomain = new RowTrackingMetadataDomain(30) val e = intercept[RuntimeException] { createTxnWithDomainMetadatas( engine, tablePath, List(rowTrackingMetadataDomain.toDomainMetadata)) } assert( e.getMessage.contains( "Cannot assign a row id high water mark")) } } test("Integration test - Write table with Kernel then write with Spark") { withTempDirAndEngine((tablePath, engine) => { withTempTable { tbl => val schema = new StructType().add("id", LongType.LONG) createTableWithRowTracking(engine, tablePath, schema) // Write table using Kernel val dataBatch1 = generateData(schema, Seq.empty, Map.empty, 100, 1) // 100 rows val dataBatch2 = generateData(schema, Seq.empty, Map.empty, 200, 1) // 200 rows val dataBatch3 = generateData(schema, Seq.empty, Map.empty, 400, 1) // 400 rows appendData( engine, tablePath, data = prepareDataForCommit(dataBatch1, dataBatch2, dataBatch3) ).getVersion // version 1 // Verify the table state verifyBaseRowIDs(engine, tablePath, Seq(0, 100, 300)) verifyDefaultRowCommitVersion(engine, tablePath, Seq(1, 1, 1)) verifyHighWatermark(engine, tablePath, 699) // Write 20, 80 rows to the table using Spark spark.range(0, 20).write.format("delta").mode("append").save(tablePath) // version 2 spark.range(20, 100).write.format("delta").mode("append").save(tablePath) // version 3 // Verify the table state verifyBaseRowIDs(engine, tablePath, Seq(0, 100, 300, 700, 720)) verifyDefaultRowCommitVersion(engine, tablePath, Seq(1, 1, 1, 2, 3)) verifyHighWatermark(engine, tablePath, 799) } }) } test("Integration test - Write table with Spark then write with Kernel") { withTempDirAndEngine((tablePath, engine) => { withTempTable { tbl => spark.sql( s"""CREATE TABLE $tbl (id LONG) USING delta |LOCATION '$tablePath' |TBLPROPERTIES ( | 'delta.feature.domainMetadata' = 'enabled', | 'delta.feature.rowTracking' = 'supported' |) |""".stripMargin) // Write to the table using delta-spark spark.range(0, 20).write.format("delta").mode("append").save(tablePath) // version 1 spark.range(20, 100).write.format("delta").mode("append").save(tablePath) // version 2 // Verify the table state verifyBaseRowIDs(engine, tablePath, Seq(0, 20)) verifyDefaultRowCommitVersion(engine, tablePath, Seq(1, 2)) verifyHighWatermark(engine, tablePath, 99) // Write to the table using Kernel val schema = new StructType().add("id", LongType.LONG) val dataBatch1 = generateData(schema, Seq.empty, Map.empty, 100, 1) // 100 rows val dataBatch2 = generateData(schema, Seq.empty, Map.empty, 200, 1) // 200 rows val dataBatch3 = generateData(schema, Seq.empty, Map.empty, 400, 1) // 400 rows appendData( engine, tablePath, data = prepareDataForCommit(dataBatch1, dataBatch2, dataBatch3) ) // version 3 // Verify the table state verifyBaseRowIDs(engine, tablePath, Seq(0, 20, 100, 200, 400)) verifyDefaultRowCommitVersion(engine, tablePath, Seq(1, 2, 3, 3, 3)) verifyHighWatermark(engine, tablePath, 799) } }) } /* -------- Test reading from tables with row tracking -------- */ test("Error when reading row tracking columns from a non-row-tracking table") { withTempDirAndEngine { (tablePath, engine) => // Create a new table without row tracking val wrongSchema = new StructType().add("id", IntegerType.INTEGER).add("_metadata.row_id", LongType.LONG) createEmptyTable(engine, tablePath, wrongSchema) // Try to read row tracking columns val e = intercept[KernelException] { checkTable( tablePath, expectedAnswer = Seq(), readCols = Seq("id", "_metadata.row_id"), metadataCols = Seq(ROW_ID, ROW_COMMIT_VERSION), engine = engine) } assert(e.getMessage.contains("Row tracking is not enabled, but row tracking column")) } } Seq("none", "name", "id").foreach(mode => { test(s"Read row tracking columns from delta-spark table with column mapping = $mode") { withTempDirAndEngine { (tablePath, _) => createRowTrackingTableWithSpark( tablePath, extraProps = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> mode)) val expectedAnswer = Seq( TestRow(1, "A", 0L, 1L), TestRow(2, "B", 1L, 1L), TestRow(3, "C_updated", 2L, 2L), TestRow(4, "D", 3L, 1L), TestRow(5, "E", 4L, 1L), TestRow(6, "F", 10L, 2L)) // We only check whether the delta-spark table schema is inferred correctly if column // mapping is disabled val expectedSchema = if (mode == "none") { new StructType() .add(new StructField("id", IntegerType.INTEGER, true)) .add(new StructField("value", StringType.STRING, true)) } else { null } checkTable( path = tablePath, expectedAnswer, metadataCols = Seq(ROW_ID, ROW_COMMIT_VERSION), expectedSchema = expectedSchema) } } }) test("Read subset of row tracking columns from delta-spark table") { withTempDirAndEngine { (tablePath, _) => createRowTrackingTableWithSpark(tablePath) val expectedAnswer = Seq( TestRow("A", 0L), TestRow("B", 1L), TestRow("C_updated", 2L), TestRow("D", 3L), TestRow("E", 4L), TestRow("F", 10L)) checkTable( path = tablePath, expectedAnswer, readCols = Seq("value"), metadataCols = Seq(ROW_ID)) } } test("Only read row tracking columns from delta-spark table") { withTempDirAndEngine { (tablePath, _) => createRowTrackingTableWithSpark(tablePath) val expectedAnswer = Seq( TestRow(1L, 0L), TestRow(1L, 1L), TestRow(2L, 2L), TestRow(1L, 3L), TestRow(1L, 4L), TestRow(2L, 10L)) // This test also checks a different ordering of the metadata columns checkTable( path = tablePath, expectedAnswer, readCols = Seq(), metadataCols = Seq(ROW_COMMIT_VERSION, ROW_ID)) } } test("Metadata columns are not read by default from delta-spark table") { withTempDirAndEngine { (tablePath, _) => createRowTrackingTableWithSpark(tablePath) val expectedAnswer = Seq( TestRow(1, "A"), TestRow(2, "B"), TestRow(3, "C_updated"), TestRow(4, "D"), TestRow(5, "E"), TestRow(6, "F")) checkTable( path = tablePath, expectedAnswer, expectedSchema = new StructType() .add(new StructField("id", IntegerType.INTEGER, true)) .add(new StructField("value", StringType.STRING, true))) } } /* -------- Conflict resolution tests -------- */ private def validateConflictResolution( engine: Engine, tablePath: String, dataSizeTxn1: Int, dataSizeTxn2: Int, useSparkTxn2: Boolean = false, dataSizeTxn3: Int, useSparkTxn3: Boolean = false): Unit = { /** * Txn1: the current transaction that commits later than winning transactions. * Txn2: the winning transaction that was committed first. * Txn3: the winning transaction that was committed second. * * Note tx is the timestamp. * * t1 ------------------------ Txn1 starts. * t2 ------- Txn2 starts. * t3 ------- Txn2 commits. * t4 ------- Txn3 starts. * t5 ------- Txn3 commits. * t6 ------------------------ Txn1 commits. */ val schema = new StructType().add("id", LongType.LONG) // Create a row-tracking-supported table and bump the row ID high watermark to the initial value createTableWithRowTracking(engine, tablePath, schema) val initDataSize = 100L val dataBatch = generateData(schema, Seq.empty, Map.empty, initDataSize.toInt, 1) val v0 = appendData(engine, tablePath, data = prepareDataForCommit(dataBatch)).getVersion var expectedBaseRowIDs = Seq(0L) var expectedDefaultRowCommitVersion = Seq(v0) var expectedHighWatermark = initDataSize - 1 def verifyRowTrackingStates(): Unit = { verifyBaseRowIDs(engine, tablePath, expectedBaseRowIDs) verifyDefaultRowCommitVersion(engine, tablePath, expectedDefaultRowCommitVersion) verifyHighWatermark(engine, tablePath, expectedHighWatermark) } verifyRowTrackingStates() // Create txn1 but don't commit it yet val txn1 = getUpdateTxn(engine, tablePath) // Create and commit txn2 if (dataSizeTxn2 > 0) { val v = if (useSparkTxn2) { spark.range(0, dataSizeTxn2).write.format("delta").mode("append").save(tablePath) DeltaLog.forTable(spark, new Path(tablePath)).snapshot.version } else { val dataBatchTxn2 = generateData(schema, Seq.empty, Map.empty, dataSizeTxn2, 1) appendData(engine, tablePath, data = prepareDataForCommit(dataBatchTxn2)).getVersion } expectedBaseRowIDs = expectedBaseRowIDs ++ Seq(initDataSize) expectedDefaultRowCommitVersion = expectedDefaultRowCommitVersion ++ Seq(v) expectedHighWatermark = initDataSize + dataSizeTxn2 - 1 } else { getUpdateTxn(engine, tablePath).commit(engine, emptyIterable()) } verifyRowTrackingStates() // Create and commit txn3 if (dataSizeTxn3 > 0) { val v = if (useSparkTxn3) { spark.range(0, dataSizeTxn3).write.format("delta").mode("append").save(tablePath) DeltaLog.forTable(spark, new Path(tablePath)).snapshot.version } else { val dataBatchTxn3 = generateData(schema, Seq.empty, Map.empty, dataSizeTxn3, 1) appendData(engine, tablePath, data = prepareDataForCommit(dataBatchTxn3)).getVersion } expectedBaseRowIDs = expectedBaseRowIDs ++ Seq(initDataSize + dataSizeTxn2) expectedDefaultRowCommitVersion = expectedDefaultRowCommitVersion ++ Seq(v) expectedHighWatermark = initDataSize + dataSizeTxn2 + dataSizeTxn3 - 1 } else { getUpdateTxn(engine, tablePath).commit(engine, emptyIterable()) } verifyRowTrackingStates() // Commit txn1 if (dataSizeTxn1 > 0) { val dataBatchTxn1 = generateData(schema, Seq.empty, Map.empty, dataSizeTxn1, 1) val v = commitAppendData(engine, txn1, prepareDataForCommit(dataBatchTxn1)).getVersion expectedBaseRowIDs = expectedBaseRowIDs ++ Seq(initDataSize + dataSizeTxn2 + dataSizeTxn3) expectedDefaultRowCommitVersion = expectedDefaultRowCommitVersion ++ Seq(v) expectedHighWatermark = initDataSize + dataSizeTxn2 + dataSizeTxn3 + dataSizeTxn1 - 1 } else { txn1.commit(engine, emptyIterable()) } verifyRowTrackingStates() } test("Conflict resolution - two concurrent txns both added new files") { withTempDirAndEngine((tablePath, engine) => { validateConflictResolution( engine, tablePath, dataSizeTxn1 = 200, dataSizeTxn2 = 300, dataSizeTxn3 = 400) }) } test("Conflict resolution - only one of the two concurrent txns added new files") { withTempDirAndEngine((tablePath, engine) => { validateConflictResolution( engine, tablePath, dataSizeTxn1 = 200, dataSizeTxn2 = 300, dataSizeTxn3 = 0) }) withTempDirAndEngine((tablePath, engine) => { validateConflictResolution( engine, tablePath, dataSizeTxn1 = 200, dataSizeTxn2 = 0, dataSizeTxn3 = 300) }) } test("Conflict resolution - none of the two concurrent txns added new files") { withTempDirAndEngine((tablePath, engine) => { validateConflictResolution( engine, tablePath, dataSizeTxn1 = 200, dataSizeTxn2 = 0, dataSizeTxn3 = 0) }) } test("Conflict resolution - the current txn didn't add new files") { withTempDirAndEngine((tablePath, engine) => { validateConflictResolution( engine, tablePath, dataSizeTxn1 = 0, dataSizeTxn2 = 200, dataSizeTxn3 = 300) }) } test( "Conflict resolution - two concurrent txns were commited by delta-spark " + "and both added new files") { withTempDirAndEngine((tablePath, engine) => { validateConflictResolution( engine, tablePath, dataSizeTxn1 = 200, dataSizeTxn2 = 300, useSparkTxn2 = true, dataSizeTxn3 = 400, useSparkTxn3 = true) }) } test("Conflict resolution - " + "conflict resolution is not supported when providedRowIdHighWatermark is set") { withTempDirAndEngine { (tablePath, engine) => createTableWithRowTracking(engine, tablePath) // Create txn1 but don't commit it yet val rowTrackingMetadataDomainTxn1 = new RowTrackingMetadataDomain(400) val txn1 = createTxnWithDomainMetadatas( engine, tablePath, List(rowTrackingMetadataDomainTxn1.toDomainMetadata)) // Create and commit txn2 val dataBatch2 = generateData(testSchema, Seq.empty, Map.empty, 100, 1) val commitVersion2 = appendData( engine, tablePath, data = Seq(dataBatch2).map(Map.empty[String, Literal] -> _)).getVersion verifyBaseRowIDs(engine, tablePath, Seq(0L)) verifyDefaultRowCommitVersion(engine, tablePath, Seq(commitVersion2)) verifyHighWatermark(engine, tablePath, 99) // Commit txn1 with a provided row ID high watermark would fail intercept[MaxCommitRetryLimitReachedException] { txn1.commit(engine, emptyIterable()) } } } private val ROW_TRACKING_ENABLED_PROP = Map(TableConfig.ROW_TRACKING_ENABLED.getKey -> "true") private val ROW_TRACKING_DISABLED_PROP = Map(TableConfig.ROW_TRACKING_ENABLED.getKey -> "false") test("row tracking can be enabled/disabled on new table") { withTempDirAndEngine { (tablePath, engine) => getCreateTxn( engine, tablePath, schema = testSchema, tableProperties = ROW_TRACKING_ENABLED_PROP).commit(engine, emptyIterable()) val snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath).asInstanceOf[SnapshotImpl] assertMetadataProp(snapshot, TableConfig.ROW_TRACKING_ENABLED, true) } withTempDirAndEngine { (tablePath, engine) => getCreateTxn( engine, tablePath, schema = testSchema, tableProperties = ROW_TRACKING_DISABLED_PROP).commit(engine, emptyIterable()) val snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath).asInstanceOf[SnapshotImpl] assertMetadataProp(snapshot, TableConfig.ROW_TRACKING_ENABLED, false) } } test("row tracking cannot be enabled on existing table") { withTempDirAndEngine { (tablePath, engine) => // Create a new table with row tracking disabled (it is disabled by default) getCreateTxn(engine, tablePath, testSchema, tableProperties = Map.empty) .commit(engine, emptyIterable()) // Fail if try to enable row tracking on an existing table val e = intercept[KernelException] { getUpdateTxn(engine, tablePath, tableProperties = ROW_TRACKING_ENABLED_PROP) .commit(engine, emptyIterable()) } assert( e.getMessage.contains("Row tracking support cannot be changed once the table is created")) // It's okay to continue setting it disabled on an existing table; it will be a no-op getUpdateTxn(engine, tablePath, tableProperties = ROW_TRACKING_DISABLED_PROP) .commit(engine, emptyIterable()) } } test("row tracking cannot be disabled on existing table") { withTempDirAndEngine { (tablePath, engine) => // Create a new table with row tracking enabled createTableWithRowTracking(engine, tablePath) // Fail if try to disable row tracking on an existing table val e = intercept[KernelException] { getUpdateTxn(engine, tablePath, tableProperties = ROW_TRACKING_DISABLED_PROP) .commit(engine, emptyIterable()) } assert( e.getMessage.contains("Row tracking support cannot be changed once the table is created")) // It's okay to continue setting it enabled on an existing table; it will be a no-op getUpdateTxn(engine, tablePath, tableProperties = ROW_TRACKING_ENABLED_PROP) .commit(engine, emptyIterable()) } } test("materialized row tracking column names are assigned when the feature is enabled") { withTempDirAndEngine { (tablePath, engine) => createTableWithRowTracking(engine, tablePath) val config = getMetadata(engine, tablePath).getConfiguration Seq(MATERIALIZED_ROW_ID, MATERIALIZED_ROW_COMMIT_VERSION).foreach { rowTrackingColumn => assert(config.containsKey(rowTrackingColumn.getMaterializedColumnNameProperty)) assert( config .get(rowTrackingColumn.getMaterializedColumnNameProperty) .startsWith(rowTrackingColumn.getMaterializedColumnNamePrefix)) } } } Seq("none", "name", "id").foreach(mode => { test( s"throw if materialized row tracking column name conflicts with schema, " + s"with column mapping = $mode") { withTempDirAndEngine { (tablePath, engine) => // Create a new table with row tracking and specified column mapping mode val columnMappingProp = Map(TableConfig.COLUMN_MAPPING_MODE.getKey -> mode) createTableWithRowTracking(engine, tablePath, extraProps = columnMappingProp) val config = getMetadata(engine, tablePath).getConfiguration Seq(MATERIALIZED_ROW_ID, MATERIALIZED_ROW_COMMIT_VERSION).foreach { rowTrackingColumn => val colName = config.get(rowTrackingColumn.getMaterializedColumnNameProperty) val newSchema = testSchema.add(colName, LongType.LONG) val e = intercept[KernelException] { updateTableMetadata(engine, tablePath, schema = newSchema) } if (mode == "none") { assert( e.getMessage .contains(s"Cannot update schema for table when column mapping is disabled")) } else { assert( e.getMessage.contains( s"Cannot use column name '$colName' because it is reserved for internal use")) } } } } }) test("manually setting materialized row tracking column names is not allowed - new table") { withTempDirAndEngine { (tablePath, engine) => Seq(MATERIALIZED_ROW_ID, MATERIALIZED_ROW_COMMIT_VERSION).foreach { rowTrackingColumn => val propName = rowTrackingColumn.getMaterializedColumnNameProperty val customTableProps = Map(propName -> "custom_name") val e = intercept[KernelException] { createTableWithRowTracking(engine, tablePath, extraProps = customTableProps) } assert(e.getMessage.contains( s"The Delta table property '$propName' is an internal property and cannot be updated")) } } } test("manually setting materialized row tracking column names is not allowed - existing table") { withTempDirAndEngine { (tablePath, engine) => createTableWithRowTracking(engine, tablePath) Seq(MATERIALIZED_ROW_ID, MATERIALIZED_ROW_COMMIT_VERSION).foreach { rowTrackingColumn => val propName = rowTrackingColumn.getMaterializedColumnNameProperty val customTableProps = Map(propName -> "custom_name") val e = intercept[KernelException] { getUpdateTxn(engine, tablePath, tableProperties = customTableProps) .commit(engine, emptyIterable()) } assert(e.getMessage.contains( s"The Delta table property '$propName' is an internal property and cannot be updated")) } } } test("throw if materialized row tracking column configs are missing on an existing table") { withTempDirAndEngine { (tablePath, engine) => // Create a normal table with row tracking enabled first createTableWithRowTracking(engine, tablePath) // Get the current metadata and manually remove row tracking materialized column configs val originalMetadata = getMetadata(engine, tablePath) val configWithoutMaterializedCols = originalMetadata.getConfiguration.asScala.toMap .filterNot { case (key, _) => key == MATERIALIZED_ROW_ID.getMaterializedColumnNameProperty || key == MATERIALIZED_ROW_COMMIT_VERSION.getMaterializedColumnNameProperty } // Create new metadata with row tracking enabled but configs missing val newMetadata = originalMetadata.withReplacedConfiguration(configWithoutMaterializedCols.asJava) // Manually commit this problematic metadata val txn = getUpdateTxn(engine, tablePath) val metadataAction = SingleAction.createMetadataSingleAction(newMetadata.toRow) commitTransaction( txn, engine, inMemoryIterable(toCloseableIterator(Seq(metadataAction).asJava.iterator()))) // Verify that row tracking is enabled but configs are missing val metadata = getMetadata(engine, tablePath) assert(TableConfig.ROW_TRACKING_ENABLED.fromMetadata(metadata) == true) assert(!metadata.getConfiguration.containsKey( MATERIALIZED_ROW_ID.getMaterializedColumnNameProperty)) assert( !metadata.getConfiguration .containsKey(MATERIALIZED_ROW_COMMIT_VERSION.getMaterializedColumnNameProperty)) // Now try to perform an append operation on this existing table with missing configs // This should trigger the validation and throw the expected exception val e = intercept[InvalidTableException] { val dataBatch = generateData(testSchema, Seq.empty, Map.empty, 10, 1) appendData(engine, tablePath, data = prepareDataForCommit(dataBatch)) } assert( e.getMessage.contains( s"Row tracking is enabled but the materialized column name " + s"`${MATERIALIZED_ROW_ID.getMaterializedColumnNameProperty}` is missing.")) } } /* -------- Test row tracking with replace table -------- */ val someData = Seq(Map.empty[String, Literal] -> dataBatches1) val otherData = Seq(Map.empty[String, Literal] -> dataBatches2) // Each tuple represents: (enableBefore, enableAfter, initialData, replaceData) val replaceTableTestCases = Seq( // Row tracking turned on (false, true, Seq(), Seq()), (false, true, Seq(), someData), (false, true, someData, Seq()), (false, true, someData, otherData), // Row tracking turned off (true, false, someData, otherData), (true, false, someData, Seq()), // Row tracking remains unchanged (true, true, someData, Seq()), (true, true, Seq(), someData), (true, true, someData, otherData), (true, true, Seq(), Seq())) for ((enableBefore, enableAfter, initialData, replaceData) <- replaceTableTestCases) { val testName = s"""Replace table with row tracking: |enableBefore=$enableBefore, |enableAfter=$enableAfter, |initialData=${initialData.nonEmpty}, |replaceData=${replaceData.nonEmpty}""" .stripMargin .replace("\n", " ") test(testName) { withTempDirAndEngine { (tablePath, engine) => // Create an empty table with row tracking enabled or disabled createEmptyTable( engine, tablePath, testSchema, tableProperties = Map(TableConfig.ROW_TRACKING_ENABLED.getKey -> enableBefore.toString)) // Optionally fill the table with initial data if provided if (initialData.nonEmpty) { appendData(engine, tablePath, data = initialData) } val beforeSnapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath).asInstanceOf[SnapshotImpl] // Create a REPLACE transaction and commit val replaceTableProps = Map(TableConfig.ROW_TRACKING_ENABLED.getKey -> enableAfter.toString) val txn = getReplaceTxn( engine, tablePath, testSchema, tableProperties = replaceTableProps) commitTransaction(txn, engine, getAppendActions(txn, replaceData)) // Get the latest snapshot of the table after the replace operation val afterSnapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath).asInstanceOf[SnapshotImpl] // Assert that row tracking is enabled/disabled as expected assertMetadataProp(afterSnapshot, TableConfig.ROW_TRACKING_ENABLED, enableAfter) // Assert that the high watermark is preserved or incremented based on the operations // This only applies if row tracking is enabled before and after the replace operation // and if there is initial data present if (enableBefore && enableAfter && initialData.nonEmpty) { val beforeHighWaterMark: Optional[Long] = RowTrackingMetadataDomain.fromSnapshot(beforeSnapshot).map(_.getRowIdHighWaterMark) val afterHighWaterMark: Optional[Long] = RowTrackingMetadataDomain.fromSnapshot(afterSnapshot).map(_.getRowIdHighWaterMark) val numInitialRows = initialData.head._2.map(_.getData.getSize).sum assert(beforeHighWaterMark.get() == numInitialRows - 1) if (replaceData.nonEmpty) { // If replace data is provided, the high watermark should be incremented val numReplaceRows = replaceData.head._2.map(_.getData.getSize).sum assert(afterHighWaterMark.get() == numInitialRows + numReplaceRows - 1) } else { // If no replace data, the high watermark should remain the same assert(beforeHighWaterMark.get() == afterHighWaterMark.get()) } } // Assert that metadata configurations are different before and after // Since REPLACE assigns new materialized column names, the configs should never match val beforeConfig = beforeSnapshot.getMetadata.getConfiguration val afterConfig = afterSnapshot.getMetadata.getConfiguration assert(!beforeConfig.equals(afterConfig)) // Assert that materialized row tracking columns are present when row tracking is enabled if (enableAfter) { Seq(MATERIALIZED_ROW_ID, MATERIALIZED_ROW_COMMIT_VERSION).foreach { rowTrackingColumn => assert(afterConfig.containsKey(rowTrackingColumn.getMaterializedColumnNameProperty)) assert( afterConfig .get(rowTrackingColumn.getMaterializedColumnNameProperty) .startsWith(rowTrackingColumn.getMaterializedColumnNamePrefix)) } } // Check that AddFile actions in the new table have a base row ID and default commit version if (replaceData.nonEmpty) { // Base row IDs do not start from 0 if we had initial data and row tracking was enabled val baseRowIds = if (initialData.nonEmpty && enableBefore) { initialData.head._2.map(_.getData.getSize).sum } else { 0L } // There was one more previous commit if initialData was present val defaultCommitVersion = if (initialData.nonEmpty) 2 else 1 verifyBaseRowIDs(engine, tablePath, Seq(baseRowIds)) verifyDefaultRowCommitVersion(engine, tablePath, Seq(defaultCommitVersion)) } } } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/ScanSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.math.{BigDecimal => JBigDecimal} import java.sql.Date import java.time.{Instant, OffsetDateTime} import java.time.temporal.ChronoUnit import java.util.Optional import scala.collection.JavaConverters._ import io.delta.golden.GoldenTableUtils.goldenTablePath import io.delta.kernel.{Scan, Snapshot, Table} import io.delta.kernel.data.{ColumnarBatch, ColumnVector, FilteredColumnarBatch, Row} import io.delta.kernel.defaults.engine.{DefaultEngine, DefaultJsonHandler, DefaultParquetHandler} import io.delta.kernel.defaults.engine.hadoopio.HadoopFileIO import io.delta.kernel.defaults.internal.data.DefaultColumnarBatch import io.delta.kernel.defaults.internal.data.vector.{DefaultGenericVector, DefaultStructVector} import io.delta.kernel.defaults.utils.{ExpressionTestUtils, TestUtils, WriteUtils} import io.delta.kernel.engine.{Engine, JsonHandler, ParquetHandler} import io.delta.kernel.engine.FileReadResult import io.delta.kernel.exceptions.KernelEngineException import io.delta.kernel.expressions._ import io.delta.kernel.expressions.Literal._ import io.delta.kernel.internal.{InternalScanFileUtils, ScanImpl, TableConfig} import io.delta.kernel.internal.util.InternalUtils import io.delta.kernel.types._ import io.delta.kernel.types.IntegerType.INTEGER import io.delta.kernel.types.StringType.STRING import io.delta.kernel.utils.{CloseableIterator, FileStatus} import io.delta.kernel.utils.CloseableIterable.emptyIterable import org.apache.spark.sql.delta.{DeltaConfigs, DeltaLog} import org.apache.hadoop.conf.Configuration import org.apache.spark.sql.{Row => SparkRow} import org.apache.spark.sql.catalyst.plans.SQLHelper import org.apache.spark.sql.types.{IntegerType => SparkIntegerType, StructField => SparkStructField, StructType => SparkStructType} import org.scalatest.funsuite.AnyFunSuite class ScanSuite extends AnyFunSuite with TestUtils with ExpressionTestUtils with SQLHelper with WriteUtils { import io.delta.kernel.defaults.ScanSuite._ // scalastyle:off sparkimplicits import spark.implicits._ // scalastyle:on sparkimplicits private def getDataSkippingConfs( indexedCols: Option[Int], deltaStatsColNamesOpt: Option[String]): Seq[(String, String)] = { val numIndexedColsConfOpt = indexedCols .map(DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.defaultTablePropertyKey -> _.toString) val indexedColNamesConfOpt = deltaStatsColNamesOpt .map(DeltaConfigs.DATA_SKIPPING_STATS_COLUMNS.defaultTablePropertyKey -> _) (numIndexedColsConfOpt ++ indexedColNamesConfOpt).toSeq } def writeDataSkippingTable( tablePath: String, data: String, schema: SparkStructType, indexedCols: Option[Int], deltaStatsColNamesOpt: Option[String]): Unit = { withSQLConf(getDataSkippingConfs(indexedCols, deltaStatsColNamesOpt): _*) { val jsonRecords = data.split("\n").toSeq val reader = spark.read if (schema != null) { reader.schema(schema) } val df = reader.json(jsonRecords.toDS()) val r = DeltaLog.forTable(spark, tablePath) df.coalesce(1).write.format("delta").save(r.dataPath.toString) } } private def getScanFileStats(scanFiles: Seq[Row]): Seq[String] = { scanFiles.map { scanFile => val addFile = scanFile.getStruct(scanFile.getSchema.indexOf("add")) if (scanFile.getSchema.indexOf("stats") >= 0) { addFile.getString(scanFile.getSchema.indexOf("stats")) } else { "[No stats read]" } } } /** * @param tablePath the table to scan * @param hits query filters that should yield at least one scan file * @param misses query filters that should yield no scan files */ def checkSkipping(tablePath: String, hits: Seq[Predicate], misses: Seq[Predicate]): Unit = { val snapshot = latestSnapshot(tablePath) hits.foreach { predicate => val scanFiles = collectScanFileRows( snapshot.getScanBuilder().withFilter(predicate).build()) assert(scanFiles.nonEmpty, s"Expected hit but got miss for $predicate") } misses.foreach { predicate => val scanFiles = collectScanFileRows( snapshot.getScanBuilder() .withFilter(predicate) .build()) assert( scanFiles.isEmpty, s"Expected miss but got hit for $predicate\n" + s"Returned scan files have stats: ${getScanFileStats(scanFiles)}") } } /** * @param tablePath the table to scan * @param filterToNumExpFiles map of {predicate -> number of expected scan files} */ def checkSkipping(tablePath: String, filterToNumExpFiles: Map[Predicate, Int]): Unit = { val snapshot = latestSnapshot(tablePath) filterToNumExpFiles.foreach { case (filter, numExpFiles) => val scanFiles = collectScanFileRows( snapshot.getScanBuilder().withFilter(filter).build()) assert( scanFiles.length == numExpFiles, s"Expected $numExpFiles but found ${scanFiles.length} for $filter") } } def testSkipping( testName: String, data: String, schema: SparkStructType = null, hits: Seq[Predicate], misses: Seq[Predicate], indexedCols: Option[Int] = None, deltaStatsColNamesOpt: Option[String] = None): Unit = { test(testName) { withTempDir { tempDir => writeDataSkippingTable( tempDir.getCanonicalPath, data, schema, indexedCols, deltaStatsColNamesOpt) checkSkipping( tempDir.getCanonicalPath, hits, misses) } } } /* Where timestampStr is in the format of "yyyy-MM-dd'T'HH:mm:ss.SSSXXX" */ def getTimestampPredicate( expr: String, col: Column, timestampStr: String, timeStampType: String): Predicate = { val time = OffsetDateTime.parse(timestampStr) new Predicate( expr, col, if (timeStampType.equalsIgnoreCase("timestamp")) { ofTimestamp(ChronoUnit.MICROS.between(Instant.EPOCH, time)) } else { ofTimestampNtz(ChronoUnit.MICROS.between(Instant.EPOCH, time)) }) } ////////////////////////////////////////////////////////////////////////////////// // Skipping tests from Spark's DataSkippingDeltaTests ////////////////////////////////////////////////////////////////////////////////// testSkipping( "data skipping - top level, single 1", """{"a": 1}""", hits = Seq( AlwaysTrue.ALWAYS_TRUE, // trivial base case equals(col("a"), ofInt(1)), // a = 1 equals(ofInt(1), col("a")), // 1 = a greaterThanOrEqual(col("a"), ofInt(1)), // a >= 1 lessThanOrEqual(col("a"), ofInt(1)), // a <= 1 lessThanOrEqual(col("a"), ofInt(2)), // a <= 2 greaterThanOrEqual(col("a"), ofInt(0)), // a >= 0 lessThanOrEqual(ofInt(1), col("a")), // 1 <= a greaterThanOrEqual(ofInt(1), col("a")), // 1 >= a greaterThanOrEqual(ofInt(2), col("a")), // 2 >= a lessThanOrEqual(ofInt(0), col("a")), // 0 <= a // note <=> is not supported yet but these should still be hits once supported nullSafeEquals(col("a"), ofInt(1)), // a <=> 1 nullSafeEquals(ofInt(1), col("a")), // 1 <=> a not(nullSafeEquals(col("a"), ofInt(2))), // NOT a <=> 2 // MOVE BELOW EXPRESSIONS TO MISSES ONCE SUPPORTED BY DATA SKIPPING notEquals(col("a"), ofInt(1)), // a != 1 notEquals(ofInt(1), col("a")) // 1 != a ), misses = Seq( equals(col("a"), ofInt(2)), // a = 2 equals(ofInt(2), col("a")), // 2 = a greaterThan(col("a"), ofInt(1)), // a > 1 lessThan(col("a"), ofInt(1)), // a < 1 greaterThanOrEqual(col("a"), ofInt(2)), // a >= 2 lessThanOrEqual(col("a"), ofInt(0)), // a <= 0 lessThan(ofInt(1), col("a")), // 1 < a greaterThan(ofInt(1), col("a")), // 1 > a lessThanOrEqual(ofInt(2), col("a")), // 2 <= a greaterThanOrEqual(ofInt(0), col("a")), // 0 >= a not(equals(col("a"), ofInt(1))), // NOT a = 1 not(equals(ofInt(1), col("a"))), // NOT 1 = a not(nullSafeEquals(col("a"), ofInt(1))), // NOT a <=> 1 not(nullSafeEquals(ofInt(1), col("a"))), // NOT 1 <=> a nullSafeEquals(ofInt(2), col("a")), // 2 <=> a nullSafeEquals(col("a"), ofInt(2)) // a <=> 2 )) testSkipping( "data skipping - nested, single 1", """{"a": {"b": 1}}""", hits = Seq( equals(nestedCol("a.b"), ofInt(1)), // a.b = 1 greaterThanOrEqual(nestedCol("a.b"), ofInt(1)), // a.b >= 1 lessThanOrEqual(nestedCol("a.b"), ofInt(1)), // a.b <= 1 lessThanOrEqual(nestedCol("a.b"), ofInt(2)), // a.b <= 2 greaterThanOrEqual(nestedCol("a.b"), ofInt(0)) // a.b >= 0 ), misses = Seq( equals(nestedCol("a.b"), ofInt(2)), // a.b = 2 greaterThan(nestedCol("a.b"), ofInt(1)), // a.b > 1 lessThan(nestedCol("a.b"), ofInt(1)) // a.b < 1 )) testSkipping( "data skipping - double nested, single 1", """{"a": {"b": {"c": 1}}}""", hits = Seq( equals(nestedCol("a.b.c"), ofInt(1)), // a.b.c = 1 greaterThanOrEqual(nestedCol("a.b.c"), ofInt(1)), // a.b.c >= 1 lessThanOrEqual(nestedCol("a.b.c"), ofInt(1)), // a.b.c <= 1 lessThanOrEqual(nestedCol("a.b.c"), ofInt(2)), // a.b.c <= 2 greaterThanOrEqual(nestedCol("a.b.c"), ofInt(0)) // a.b.c >= 0 ), misses = Seq( equals(nestedCol("a.b.c"), ofInt(2)), // a.b.c = 2 greaterThan(nestedCol("a.b.c"), ofInt(1)), // a.b.c > 1 lessThan(nestedCol("a.b.c"), ofInt(1)) // a.b.c < 1 )) private def longString(str: String) = str * 1000 testSkipping( "data skipping - long strings - long min", s""" {"a": '${longString("A")}'} {"a": 'B'} {"a": 'C'} """, hits = Seq( equals(col("a"), ofString(longString("A"))), greaterThan(col("a"), ofString("BA")), lessThan(col("a"), ofString("AB")), // note startsWith is not supported yet but these should still be hits once supported startsWith(col("a"), ofString("A")) // a like 'A%' ), misses = Seq( lessThan(col("a"), ofString("AA")), greaterThan(col("a"), ofString("CD")))) testSkipping( "data skipping - long strings - long max", s""" {"a": 'A'} {"a": 'B'} {"a": '${longString("C")}'} """, hits = Seq( equals(col("a"), ofString(longString("C"))), greaterThan(col("a"), ofString("BA")), lessThan(col("a"), ofString("AB")), greaterThan(col("a"), ofString("CC")), // note startsWith is not supported yet but these should still be hits once supported startsWith(col("a"), ofString("A")), // a like 'A%' startsWith(col("a"), ofString("C")) // a like 'C%' ), misses = Seq( greaterThanOrEqual(col("a"), ofString("D")), greaterThan(col("a"), ofString("CD")))) // Test:'starts with' Expression: like // Test:'starts with, nested' Expression: like testSkipping( "data skipping - and statements - simple", """ {"a": 1} {"a": 2} """, hits = Seq( new And( greaterThan(col("a"), ofInt(0)), lessThan(col("a"), ofInt(3))), new And( lessThanOrEqual(col("a"), ofInt(1)), greaterThan(col("a"), ofInt(-1)))), misses = Seq( new And( lessThan(col("a"), ofInt(0)), greaterThan(col("a"), ofInt(-2))))) testSkipping( "data skipping - and statements - two fields", """ {"a": 1, "b": "2017-09-01"} {"a": 2, "b": "2017-08-31"} """, hits = Seq( new And( greaterThan(col("a"), ofInt(0)), equals(col("b"), ofString("2017-09-01"))), new And( equals(col("a"), ofInt(2)), greaterThanOrEqual(col("b"), ofString("2017-08-30"))), // note startsWith is not supported yet but these should still be hits once supported new And( // a >= 2 AND b like '2017-08-%' greaterThanOrEqual(col("a"), ofInt(2)), startsWith(col("b"), ofString("2017-08-"))), // MOVE BELOW EXPRESSION TO MISSES ONCE SUPPORTED BY DATA SKIPPING new And( // a > 0 AND b like '2016-%' greaterThan(col("a"), ofInt(0)), startsWith(col("b"), ofString("2016-")))), misses = Seq()) private val aRem100 = new ScalarExpression("%", Seq(col("a"), ofInt(100)).asJava) private val bRem100 = new ScalarExpression("%", Seq(col("b"), ofInt(100)).asJava) testSkipping( "data skipping - and statements - one side unsupported", """ {"a": 10, "b": 10} {"a": 20: "b": 20} """, hits = Seq( // a % 100 < 10 AND b % 100 > 20 new And(lessThan(aRem100, ofInt(10)), greaterThan(bRem100, ofInt(20)))), misses = Seq( // a < 10 AND b % 100 > 20 new And(lessThan(col("a"), ofInt(10)), greaterThan(bRem100, ofInt(20))), // a % 100 < 10 AND b > 20 new And(lessThan(aRem100, ofInt(10)), greaterThan(col("b"), ofInt(20))))) testSkipping( "data skipping - or statements - simple", """ {"a": 1} {"a": 2} """, hits = Seq( // a > 0 or a < -3 new Or(greaterThan(col("a"), ofInt(0)), lessThan(col("a"), ofInt(-3))), // a >= 2 or a < -1 new Or(greaterThanOrEqual(col("a"), ofInt(2)), lessThan(col("a"), ofInt(-1)))), misses = Seq( // a > 5 or a < -2 new Or(greaterThan(col("a"), ofInt(5)), lessThan(col("a"), ofInt(-2))))) testSkipping( "data skipping - or statements - two fields", """ {"a": 1, "b": "2017-09-01"} {"a": 2, "b": "2017-08-31"} """, hits = Seq( new Or( lessThan(col("a"), ofInt(0)), equals(col("b"), ofString("2017-09-01"))), new Or( equals(col("a"), ofInt(2)), lessThan(col("b"), ofString("2017-08-30"))), // note startsWith is not supported yet but these should still be hits once supported new Or( // a < 2 or b like '2017-08-%' lessThan(col("a"), ofInt(2)), startsWith(col("b"), ofString("2017-08-"))), new Or( // a >= 2 or b like '2016-08-%' greaterThanOrEqual(col("a"), ofInt(2)), startsWith(col("b"), ofString("2016-08-"))), // MOVE BELOW EXPRESSION TO MISSES ONCE SUPPORTED BY DATA SKIPPING new Or( // a < 0 or b like '2016-%' lessThan(col("a"), ofInt(0)), startsWith(col("b"), ofString("2016-")))), misses = Seq()) // One side of OR by itself isn't powerful enough to prune any files. testSkipping( "data skipping - or statements - one side unsupported", """ {"a": 10, "b": 10} {"a": 20: "b": 20} """, hits = Seq( // a % 100 < 10 OR b > 20 new Or(lessThan(aRem100, ofInt(10)), greaterThan(col("b"), ofInt(20))), // a < 10 OR b % 100 > 20 new Or(lessThan(col("a"), ofInt(10)), greaterThan(bRem100, ofInt(20)))), misses = Seq( // a < 10 OR b > 20 new Or(lessThan(col("a"), ofInt(10)), greaterThan(col("b"), ofInt(20))))) testSkipping( "data skipping - not statements - simple", """ {"a": 1} {"a": 2} """, hits = Seq( not(lessThan(col("a"), ofInt(0)))), misses = Seq( not(greaterThan(col("a"), ofInt(0))), not(lessThan(col("a"), ofInt(3))), not(greaterThanOrEqual(col("a"), ofInt(1))), not(lessThanOrEqual(col("a"), ofInt(2))), not(not(lessThan(col("a"), ofInt(0)))), not(not(equals(col("a"), ofInt(3)))))) // NOT(AND(a, b)) === OR(NOT(a), NOT(b)) ==> One side by itself cannot prune. testSkipping( "data skipping - not statements - and", """ {"a": 10, "b": 10} {"a": 20: "b": 20} """, hits = Seq( not( new And( greaterThanOrEqual(aRem100, ofInt(10)), lessThanOrEqual(bRem100, ofInt(20)))), not( new And( greaterThanOrEqual(col("a"), ofInt(10)), lessThanOrEqual(bRem100, ofInt(20)))), not( new And( greaterThanOrEqual(aRem100, ofInt(10)), lessThanOrEqual(col("b"), ofInt(20))))), misses = Seq( not( new And( greaterThanOrEqual(col("a"), ofInt(10)), lessThanOrEqual(col("b"), ofInt(20)))))) // NOT(OR(a, b)) === AND(NOT(a), NOT(b)) => One side by itself is enough to prune. testSkipping( "data skipping - not statements - or", """ {"a": 1, "b": 10} {"a": 2, "b": 20} """, hits = Seq( // NOT(a < 1 OR b > 20), not(new Or(lessThan(col("a"), ofInt(1)), greaterThan(col("b"), ofInt(20)))), // NOT(a % 100 >= 1 OR b % 100 <= 20) not(new Or(greaterThanOrEqual(aRem100, ofInt(1)), lessThanOrEqual(bRem100, ofInt(20))))), misses = Seq( // NOT(a >= 1 OR b <= 20) not( new Or(greaterThanOrEqual(col("a"), ofInt(1)), lessThanOrEqual(col("b"), ofInt(20)))), // NOT(a % 100 >= 1 OR b <= 20), not( new Or(greaterThanOrEqual(aRem100, ofInt(1)), lessThanOrEqual(col("b"), ofInt(20)))), // NOT(a >= 1 OR b % 100 <= 20) not( new Or(greaterThanOrEqual(col("a"), ofInt(1)), lessThanOrEqual(bRem100, ofInt(20)))))) // If a column does not have stats, it does not participate in data skipping, which disqualifies // that leg of whatever conjunct it was part of. testSkipping( "data skipping - missing stats columns", """ {"a": 1, "b": 10} {"a": 2, "b": 20} """, indexedCols = Some(1), hits = Seq( lessThan(col("b"), ofInt(10)), // b < 10: disqualified // note OR is not supported yet but these should still be hits once supported new Or( // a < 1 OR b < 10: a disqualified by b (same conjunct) lessThan(col("a"), ofInt(1)), lessThan(col("b"), ofInt(10))), new Or( // a < 1 OR (a >= 1 AND b < 10): ==> a < 1 OR a >=1 ==> TRUE lessThan(col("a"), ofInt(1)), new And(greaterThanOrEqual(col("a"), ofInt(1)), lessThan(col("b"), ofInt(10))))), misses = Seq( new And( // a < 1 AND b < 10: ==> a < 1 ==> FALSE lessThan(col("a"), ofInt(1)), lessThan(col("b"), ofInt(10))), new Or( // a < 1 OR (a > 10 AND b < 10): ==> a < 1 OR a > 10 ==> FALSE lessThan(col("a"), ofInt(1)), new And(greaterThan(col("a"), ofInt(10)), lessThan(col("b"), ofInt(10)))))) private def generateJsonData(numCols: Int): String = { val fields = (0 until numCols).map(i => s""""col${"%02d".format(i)}":$i""".stripMargin) "{" + fields.mkString(",") + "}" } testSkipping( "data-skipping - more columns than indexed", generateJsonData(33), // defaultNumIndexedCols + 1 hits = Seq( equals(col("col00"), ofInt(0)), equals(col("col32"), ofInt(32)), equals(col("col32"), ofInt(-1))), misses = Seq( equals(col("col00"), ofInt(1)))) testSkipping( "data skipping - nested schema - # indexed column = 3", """{ "a": 1, "b": { "c": { "d": 2, "e": 3, "f": { "g": 4, "h": 5, "i": 6 }, "j": 7, "k": 8 }, "l": 9 }, "m": 10 }""".replace("\n", ""), indexedCols = Some(3), hits = Seq( equals(col("a"), ofInt(1)), // a = 1 equals(nestedCol("b.c.d"), ofInt(2)), // b.c.d = 2 equals(nestedCol("b.c.e"), ofInt(3)), // b.c.e = 3 // below matches due to missing stats lessThan(nestedCol("b.c.f.g"), ofInt(0)), // b.c.f.g < 0 lessThan(nestedCol("b.c.f.i"), ofInt(0)), // b.c.f.i < 0 lessThan(nestedCol("b.l"), ofInt(0)) // b.l < 0 ), misses = Seq( lessThan(col("a"), ofInt(0)), // a < 0 lessThan(nestedCol("b.c.d"), ofInt(0)), // b.c.d < 0 lessThan(nestedCol("b.c.e"), ofInt(0)) // b.c.e < 0 )) testSkipping( "data skipping - nested schema - # indexed column = 0", """{ "a": 1, "b": { "c": { "d": 2, "e": 3, "f": { "g": 4, "h": 5, "i": 6 }, "j": 7, "k": 8 }, "l": 9 }, "m": 10 }""".replace("\n", ""), indexedCols = Some(0), hits = Seq( // all included due to missing stats lessThan(col("a"), ofInt(0)), lessThan(nestedCol("b.c.d"), ofInt(0)), lessThan(nestedCol("b.c.f.i"), ofInt(0)), lessThan(nestedCol("b.l"), ofInt(0)), lessThan(col("m"), ofInt(0))), misses = Seq()) testSkipping( "data skipping - indexed column names - " + "naming a nested column indexes all leaf fields of that column", """{ "a": 1, "b": { "c": { "d": 2, "e": 3, "f": { "g": 4, "h": 5, "i": 6 }, "j": 7, "k": 8 }, "l": 9 }, "m": 10 }""".replace("\n", ""), indexedCols = Some(3), deltaStatsColNamesOpt = Some("b.c"), hits = Seq( // these all have missing stats lessThan(col("a"), ofInt(0)), lessThan(nestedCol("b.l"), ofInt(0)), lessThan(col("m"), ofInt(0))), misses = Seq( lessThan(nestedCol("b.c.d"), ofInt(0)), lessThan(nestedCol("b.c.e"), ofInt(0)), lessThan(nestedCol("b.c.f.g"), ofInt(0)), lessThan(nestedCol("b.c.f.h"), ofInt(0)), lessThan(nestedCol("b.c.f.i"), ofInt(0)), lessThan(nestedCol("b.c.j"), ofInt(0)), lessThan(nestedCol("b.c.k"), ofInt(0)))) testSkipping( "data skipping - indexed column names - index only a subset of leaf columns", """{ "a": 1, "b": { "c": { "d": 2, "e": 3, "f": { "g": 4, "h": 5, "i": 6 }, "j": 7, "k": 8 }, "l": 9 }, "m": 10 }""".replace("\n", ""), indexedCols = Some(3), deltaStatsColNamesOpt = Some("b.c.e, b.c.f.h, b.c.k, b.l"), hits = Seq( // these all have missing stats lessThan(col("a"), ofInt(0)), lessThan(nestedCol("b.c.d"), ofInt(0)), lessThan(nestedCol("b.c.f.g"), ofInt(0)), lessThan(nestedCol("b.c.f.i"), ofInt(0)), lessThan(nestedCol("b.c.j"), ofInt(0)), lessThan(col("m"), ofInt(0))), misses = Seq( lessThan(nestedCol("b.c.e"), ofInt(0)), lessThan(nestedCol("b.c.f.h"), ofInt(0)), lessThan(nestedCol("b.c.k"), ofInt(0)), lessThan(nestedCol("b.l"), ofInt(0)))) testSkipping( "data skipping - boolean comparisons", """{"a": false}""", hits = Seq( equals(col("a"), ofBoolean(false)), greaterThan(col("a"), ofBoolean(true)), lessThanOrEqual(col("a"), ofBoolean(false)), equals(ofBoolean(true), col("a")), lessThan(ofBoolean(true), col("a")), not(equals(col("a"), ofBoolean(false)))), misses = Seq()) // Data skipping by stats should still work even when the only data in file is null, in spite of // the NULL min/max stats that result -- this is different to having no stats at all. testSkipping( "data skipping - nulls - only null in file", """ {"a": null } """, schema = new SparkStructType().add(new SparkStructField("a", SparkIntegerType)), hits = Seq( AlwaysTrue.ALWAYS_TRUE, // Ideally this should not hit as it is always FALSE, but its correct to not skip equals(col("a"), ofNull(INTEGER)), not(equals(col("a"), ofNull(INTEGER))), // Same as previous case isNull(col("a")), // This is optimized to `IsNull(a)` by NullPropagation in Spark nullSafeEquals(col("a"), ofNull(INTEGER)), not(nullSafeEquals(col("a"), ofInt(1))), // In delta-spark we use verifyStatsForFilter to deal with missing stats instead of // converting all nulls ==> true (keep). For comparisons with null statistics we end up with // filter: dataFilter || !(verifyStatsForFilter) = null || false = null // When filtering on a DF nulls are counted as false and eliminated. Thus these are misses // in Delta-Spark. // Including them is not incorrect. To skip these filters for Kernel we could use // verifyStatsForFilter or some other solution like inserting a && isNotNull(a) expression. equals(col("a"), ofInt(1)), lessThan(col("a"), ofInt(1)), greaterThan(col("a"), ofInt(1)), not(equals(col("a"), ofInt(1))), notEquals(col("a"), ofInt(1))), misses = Seq( AlwaysFalse.ALWAYS_FALSE, nullSafeEquals(col("a"), ofInt(1)), not(nullSafeEquals(col("a"), ofNull(INTEGER))), isNotNull(col("a")))) testSkipping( "data skipping - nulls - null + not-null in same file", """ {"a": null } {"a": 1 } """, schema = new SparkStructType().add(new SparkStructField("a", SparkIntegerType)), hits = Seq( // Ideally this should not hit as it is always FALSE, but its correct to not skip equals(col("a"), ofNull(INTEGER)), equals(col("a"), ofInt(1)), AlwaysTrue.ALWAYS_TRUE, isNotNull(col("a")), // Note these expressions either aren't supported or aren't added to skipping yet // but should still be hits once supported isNull(col("a")), not(equals(col("a"), ofNull(INTEGER))), // This is optimized to `IsNull(a)` by NullPropagation in Spark nullSafeEquals(col("a"), ofNull(INTEGER)), // This is optimized to `IsNotNull(a)` by NullPropagation in Spark not(nullSafeEquals(col("a"), ofNull(INTEGER))), nullSafeEquals(col("a"), ofInt(1)), not(nullSafeEquals(col("a"), ofInt(1))), // MOVE BELOW EXPRESSIONS TO MISSES ONCE SUPPORTED BY DATA SKIPPING notEquals(col("a"), ofInt(1))), misses = Seq( AlwaysFalse.ALWAYS_FALSE, lessThan(col("a"), ofInt(1)), greaterThan(col("a"), ofInt(1)), not(equals(col("a"), ofInt(1))))) Seq("TIMESTAMP", "TIMESTAMP_NTZ").foreach { dataType => test(s"data skipping - on $dataType type") { withTempDir { tempDir => withSparkTimeZone("UTC") { val data = "2019-09-09 01:02:03.456789" val df = Seq(data).toDF("strTs") .selectExpr( s"CAST(strTs AS $dataType) AS ts", s"STRUCT(CAST(strTs AS $dataType) AS ts) AS nested") val r = DeltaLog.forTable(spark, tempDir.getCanonicalPath) df.coalesce(1).write.format("delta").save(r.dataPath.toString) } checkSkipping( tempDir.getCanonicalPath, hits = Seq( getTimestampPredicate("=", col("ts"), "2019-09-09T01:02:03.456789Z", dataType), getTimestampPredicate(">=", col("ts"), "2019-09-09T01:02:03.456789Z", dataType), getTimestampPredicate("<=", col("ts"), "2019-09-09T01:02:03.456789Z", dataType), getTimestampPredicate( ">=", nestedCol("nested.ts"), "2019-09-09T01:02:03.456789Z", dataType), getTimestampPredicate( "<=", nestedCol("nested.ts"), "2019-09-09T01:02:03.456789Z", dataType)), misses = Seq( getTimestampPredicate("=", col("ts"), "2019-09-09T01:02:03.457001Z", dataType), getTimestampPredicate(">=", col("ts"), "2019-09-09T01:02:03.457001Z", dataType), getTimestampPredicate("<=", col("ts"), "2019-09-09T01:02:03.455999Z", dataType), getTimestampPredicate( ">=", nestedCol("nested.ts"), "2019-09-09T01:02:03.457001Z", dataType), getTimestampPredicate( "<=", nestedCol("nested.ts"), "2019-09-09T01:02:03.455999Z", dataType))) } } } test("data skipping - Basic: Data skipping with delta statistic column") { withTempDir { tempDir => val tablePath = tempDir.getCanonicalPath val tableProperty = "TBLPROPERTIES('delta.dataSkippingStatsColumns' = 'c1,c2,c3,c4,c5,c6,c9')" spark.sql( s"""CREATE TABLE delta.`$tablePath`( |c1 long, c2 STRING, c3 FLOAT, c4 DOUBLE, c5 TIMESTAMP, c6 DATE, |c7 BINARY, c8 BOOLEAN, c9 DECIMAL(3, 2) |) USING delta $tableProperty""".stripMargin) spark.sql( s"""insert into delta.`$tablePath` values |(1, '1', 1.0, 1.0, TIMESTAMP'2001-01-01 01:00', DATE'2001-01-01', '1111', true, 1.0), |(2, '2', 2.0, 2.0, TIMESTAMP'2002-02-02 02:00', DATE'2002-02-02', '2222', false, 2.0) |""".stripMargin) checkSkipping( tablePath, hits = Seq( equals(col("c1"), ofInt(1)), equals(col("c2"), ofString("2")), lessThan(col("c3"), ofFloat(1.5f)), greaterThan(col("c4"), ofFloat(1.0f)), equals(col("c6"), ofDate(InternalUtils.daysSinceEpoch(Date.valueOf("2002-02-02")))), // Binary Column doesn't support delta statistics. equals(col("c7"), ofBinary("1111".getBytes)), equals(col("c7"), ofBinary("3333".getBytes)), equals(col("c8"), ofBoolean(true)), equals(col("c8"), ofBoolean(false)), greaterThan(col("c9"), ofDecimal(JBigDecimal.valueOf(1.5), 3, 2)), getTimestampPredicate(">=", col("c5"), "2001-01-01T01:00:00-07:00", "TIMESTAMP")), misses = Seq( equals(col("c1"), ofInt(10)), equals(col("c2"), ofString("4")), lessThan(col("c3"), ofFloat(0.5f)), greaterThan(col("c4"), ofFloat(5.0f)), equals(col("c6"), ofDate(InternalUtils.daysSinceEpoch(Date.valueOf("2003-02-02")))), greaterThan(col("c9"), ofDecimal(JBigDecimal.valueOf(2.5), 3, 2)), getTimestampPredicate(">=", col("c5"), "2003-01-01T01:00:00-07:00", "TIMESTAMP"))) } } test("data skipping - Data skipping with delta statistic column rename column") { withTempDir { tempDir => val tablePath = tempDir.getCanonicalPath spark.sql( s"""CREATE TABLE delta.`$tablePath`( |c1 long, c2 STRING, c3 FLOAT, c4 DOUBLE, c5 TIMESTAMP, c6 DATE, |c7 BINARY, c8 BOOLEAN, c9 DECIMAL(3, 2) |) USING delta |TBLPROPERTIES( |'delta.dataSkippingStatsColumns' = 'c1,c2,c3,c4,c5,c6,c9', |'delta.columnMapping.mode' = 'name', |'delta.minReaderVersion' = '2', |'delta.minWriterVersion' = '5' |) |""".stripMargin) (1 to 9).foreach { i => spark.sql(s"alter table delta.`$tablePath` RENAME COLUMN c$i to cc$i") } spark.sql( s"""insert into delta.`$tablePath` values |(1, '1', 1.0, 1.0, TIMESTAMP'2001-01-01 01:00', DATE'2001-01-01', '1111', true, 1.0), |(2, '2', 2.0, 2.0, TIMESTAMP'2002-02-02 02:00', DATE'2002-02-02', '2222', false, 2.0) |""".stripMargin) checkSkipping( tablePath, hits = Seq( equals(col("cc1"), ofInt(1)), equals(col("cc2"), ofString("2")), lessThan(col("cc3"), ofFloat(1.5f)), greaterThan(col("cc4"), ofFloat(1.0f)), equals(col("cc6"), ofDate(InternalUtils.daysSinceEpoch(Date.valueOf("2002-02-02")))), // Binary Column doesn't support delta statistics. equals(col("cc7"), ofBinary("1111".getBytes)), equals(col("cc7"), ofBinary("3333".getBytes)), equals(col("cc8"), ofBoolean(true)), equals(col("cc8"), ofBoolean(false)), greaterThan(col("cc9"), ofDecimal(JBigDecimal.valueOf(1.5), 3, 2)), getTimestampPredicate(">=", col("cc5"), "2001-01-01T01:00:00-07:00", "TIMESTAMP")), misses = Seq( equals(col("cc1"), ofInt(10)), equals(col("cc2"), ofString("4")), lessThan(col("cc3"), ofFloat(0.5f)), greaterThan(col("cc4"), ofFloat(5.0f)), equals(col("cc6"), ofDate(InternalUtils.daysSinceEpoch(Date.valueOf("2003-02-02")))), getTimestampPredicate(">=", col("cc5"), "2003-01-01T01:00:00-07:00", "TIMESTAMP"), greaterThan(col("cc9"), ofDecimal(JBigDecimal.valueOf(2.5), 3, 2)))) } } test("data skipping - Data skipping with delta statistic column drop column") { withTempDir { tempDir => val tablePath = tempDir.getCanonicalPath spark.sql( s"""CREATE TABLE delta.`$tablePath`( |c1 long, c2 STRING, c3 FLOAT, c4 DOUBLE, c5 TIMESTAMP, c6 DATE, |c7 BINARY, c8 BOOLEAN, c9 DECIMAL(3, 2), c10 TIMESTAMP_NTZ |) USING delta |TBLPROPERTIES( |'delta.dataSkippingStatsColumns' = 'c1,c2,c3,c4,c5,c6,c9,c10', |'delta.columnMapping.mode' = 'name' |) |""".stripMargin) spark.sql(s"alter table delta.`$tablePath` drop COLUMN c2") spark.sql(s"alter table delta.`$tablePath` drop COLUMN c7") spark.sql(s"alter table delta.`$tablePath` drop COLUMN c8") spark.sql( s"""insert into delta.`$tablePath` values |(1, 1.0, 1.0, TIMESTAMP'2001-01-01 01:00', DATE'2001-01-01', |1.0, TIMESTAMP_NTZ'2001-01-01 01:00'), |(2, 2.0, 2.0, TIMESTAMP'2002-02-02 02:00', DATE'2002-02-02', |2.0, TIMESTAMP_NTZ'2002-02-02 02:00') |""".stripMargin) checkSkipping( tablePath, hits = Seq( equals(col("c1"), ofInt(1)), lessThan(col("c3"), ofFloat(1.5f)), greaterThan(col("c4"), ofFloat(1.0f)), equals(col("c6"), ofDate(InternalUtils.daysSinceEpoch(Date.valueOf("2002-02-02")))), greaterThan(col("c9"), ofDecimal(JBigDecimal.valueOf(1.5), 3, 2)), getTimestampPredicate(">=", col("c5"), "2001-01-01T01:00:00-07:00", "TIMESTAMP"), getTimestampPredicate(">=", col("c10"), "2001-01-01T01:00:00-07:00", "TIMESTAMP_NTZ")), misses = Seq( equals(col("c1"), ofInt(10)), lessThan(col("c3"), ofFloat(0.5f)), greaterThan(col("c4"), ofFloat(5.0f)), equals(col("c6"), ofDate(InternalUtils.daysSinceEpoch(Date.valueOf("2003-02-02")))), greaterThan(col("c9"), ofDecimal(JBigDecimal.valueOf(2.5), 3, 2)), getTimestampPredicate(">=", col("c5"), "2003-01-01T01:00:00-07:00", "TIMESTAMP"), getTimestampPredicate(">=", col("c10"), "2003-01-01T01:00:00-07:00", "TIMESTAMP_NTZ"))) } } test("data skipping by partition and data values - nulls") { withTempDir { tableDir => val dataSeqs = Seq( // each sequence produce a single file Seq((null, null)), Seq((null, "a")), Seq((null, "b")), Seq(("a", "a"), ("a", null)), Seq(("b", null))) dataSeqs.foreach { seq => seq.toDF("key", "value").coalesce(1) .write.format("delta").partitionBy("key").mode("append").save(tableDir.getCanonicalPath) } def checkResults( predicate: Predicate, expNumPartitions: Int, expNumFiles: Long): Unit = { val snapshot = latestSnapshot(tableDir.getCanonicalPath) val scanFiles = collectScanFileRows( snapshot.getScanBuilder().withFilter(predicate).build()) assert( scanFiles.length == expNumFiles, s"Expected $expNumFiles but found ${scanFiles.length} for $predicate") val partitionValues = scanFiles.map { row => InternalScanFileUtils.getPartitionValues(row) }.distinct assert( partitionValues.length == expNumPartitions, s"Expected $expNumPartitions partitions but found ${partitionValues.length}") } // Trivial base case checkResults( predicate = AlwaysTrue.ALWAYS_TRUE, expNumPartitions = 3, expNumFiles = 5) // Conditions on partition key checkResults( predicate = isNotNull(col("key")), expNumPartitions = 2, expNumFiles = 2 ) // 2 files with key = 'a', and 1 file with key = 'b' checkResults( predicate = equals(col("key"), ofString("a")), expNumPartitions = 1, expNumFiles = 1 ) // 1 files with key = 'a' checkResults( predicate = equals(col("key"), ofString("b")), expNumPartitions = 1, expNumFiles = 1 ) // 1 files with key = 'b' // TODO shouldn't partition filters on unsupported expressions just not prune instead of fail? checkResults( predicate = isNull(col("key")), expNumPartitions = 1, expNumFiles = 3 ) // 3 files with key = null checkResults( predicate = nullSafeEquals(col("key"), ofNull(STRING)), expNumPartitions = 1, expNumFiles = 3 ) // 3 files with key = null checkResults( predicate = nullSafeEquals(col("key"), ofString("a")), expNumPartitions = 1, expNumFiles = 1 ) // 1 files with key <=> 'a' checkResults( predicate = nullSafeEquals(col("key"), ofString("b")), expNumPartitions = 1, expNumFiles = 1 ) // 1 files with key <=> 'b' // Conditions on partitions keys and values checkResults( predicate = isNull(col("value")), expNumPartitions = 3, expNumFiles = 3) checkResults( predicate = isNotNull(col("value")), expNumPartitions = 2, // one of the partitions has no files left after data skipping expNumFiles = 3 ) // files with all NULL values get skipped checkResults( predicate = nullSafeEquals(col("value"), ofNull(STRING)), expNumPartitions = 3, expNumFiles = 3) checkResults( predicate = nullSafeEquals(ofNull(STRING), col("value")), expNumPartitions = 3, expNumFiles = 3) checkResults( predicate = equals(col("value"), ofString("a")), expNumPartitions = 3, // should be 2 if we can correctly skip "value = 'a'" for nulls expNumFiles = 4 ) // should be 2 if we can correctly skip "value = 'a'" for nulls checkResults( predicate = nullSafeEquals(col("value"), ofString("a")), expNumPartitions = 2, expNumFiles = 2) checkResults( predicate = nullSafeEquals(ofString("a"), col("value")), expNumPartitions = 2, expNumFiles = 2) checkResults( predicate = notEquals(col("value"), ofString("a")), expNumPartitions = 3, // should be 1 once <> is supported expNumFiles = 5 ) // should be 1 once <> is supported checkResults( predicate = equals(col("value"), ofString("b")), expNumPartitions = 2, // should be 1 if we can correctly skip "value = 'b'" for nulls expNumFiles = 3 ) // should be 1 if we can correctly skip "value = 'a'" for nulls checkResults( predicate = nullSafeEquals(col("value"), ofString("b")), expNumPartitions = 1, expNumFiles = 1) // Conditions on both, partition keys and values /* NOT YET SUPPORTED EXPRESSIONS checkResults( predicate = new And(isNull(col("key")), equals(col("value"), ofString("a"))), expNumPartitions = 2, expNumFiles = 1) // only one file in the partition has (*, "a") checkResults( predicate = new And(nullSafeEquals(col("key"), ofNull(STRING)), nullSafeEquals(col("value"), ofNull(STRING))), expNumPartitions = 1, expNumFiles = 1) // 3 files with key = null, but only 1 with val = null. */ checkResults( predicate = new And(isNotNull(col("key")), isNotNull(col("value"))), expNumPartitions = 1, expNumFiles = 1 ) // 1 file with (*, a) checkResults( predicate = new Or( nullSafeEquals(col("key"), ofNull(STRING)), nullSafeEquals(col("value"), ofNull(STRING))), expNumPartitions = 3, expNumFiles = 5 ) // all 5 files } } ////////////////////////////////////////////////////////////////////////////////// // Kernel data skipping tests ////////////////////////////////////////////////////////////////////////////////// test("basic data skipping for all types - all CM modes + checkpoint") { // Map of column name to (value_in_table, smaller_value, bigger_value) val colToLits = Map( "as_int" -> (ofInt(0), ofInt(-1), ofInt(1)), "as_long" -> (ofLong(0), ofLong(-1), ofLong(1)), "as_byte" -> (ofByte(0), ofByte(-1), ofByte(1)), "as_short" -> (ofShort(0), ofShort(-1), ofShort(1)), "as_float" -> (ofFloat(0), ofFloat(-1), ofFloat(1)), "as_double" -> (ofDouble(0), ofDouble(-1), ofDouble(1)), "as_string" -> (ofString("0"), ofString("!"), ofString("1")), "as_date" -> ( ofDate(InternalUtils.daysSinceEpoch(Date.valueOf("2000-01-01"))), ofDate(InternalUtils.daysSinceEpoch(Date.valueOf("1999-01-01"))), ofDate(InternalUtils.daysSinceEpoch(Date.valueOf("2000-01-02")))), // TODO (delta-io/delta#2462) add Timestamp once we support skipping for TimestampType "as_big_decimal" -> ( ofDecimal(JBigDecimal.valueOf(0), 1, 0), ofDecimal(JBigDecimal.valueOf(-1), 1, 0), ofDecimal(JBigDecimal.valueOf(1), 1, 0))) val misses = colToLits.flatMap { case (colName, (value, small, big)) => Seq( equals(col(colName), small), greaterThan(col(colName), value), greaterThanOrEqual(col(colName), big), lessThan(col(colName), value), lessThanOrEqual(col(colName), small)) }.toSeq val hits = colToLits.flatMap { case (colName, (value, small, big)) => Seq( equals(col(colName), value), greaterThan(col(colName), small), greaterThanOrEqual(col(colName), value), lessThan(col(colName), big), lessThanOrEqual(col(colName), value)) }.toSeq Seq( "data-skipping-basic-stats-all-types", "data-skipping-basic-stats-all-types-columnmapping-name", "data-skipping-basic-stats-all-types-columnmapping-id", "data-skipping-basic-stats-all-types-checkpoint").foreach { goldenTable => checkSkipping( goldenTablePath(goldenTable), hits, misses) } } test("data skipping - implicit casting works") { checkSkipping( goldenTablePath("data-skipping-basic-stats-all-types"), hits = Seq( equals(col("as_short"), ofFloat(0f)), equals(col("as_float"), ofShort(0))), misses = Seq( equals(col("as_short"), ofFloat(1f)), equals(col("as_float"), ofShort(1)))) } test("data skipping - incompatible schema change doesn't break") { withTempDir { tempDir => val tablePath = tempDir.getPath // initially write with integer column value Seq(0, 1, 2).toDF.repartition(1).write.format("delta").save(tablePath) // overwrite with string column value Seq("0", "1", "2").toDF.repartition(1).write .format("delta").mode("overwrite").option("overwriteSchema", true).save(tablePath) checkSkipping( tablePath, hits = Seq( equals(col("value"), ofString("1"))), misses = Seq( equals(col("value"), ofString("3")))) } } test("data skipping - filter on non-existent column") { checkSkipping( goldenTablePath("data-skipping-basic-stats-all-types"), hits = Seq(equals(col("foo"), ofInt(1))), misses = Seq()) } // todo add a test with dvs where tightBounds=false test("data skipping - filter on partition AND data column") { checkSkipping( goldenTablePath("data-skipping-basic-stats-all-types"), filterToNumExpFiles = Map( new And( greaterThan(col("part"), ofInt(0)), greaterThan(col("id"), ofInt(0)) ) -> 1 // should prune 3 files from partition + data filter )) } test("data skipping - stats collected changing across versions") { checkSkipping( goldenTablePath("data-skipping-change-stats-collected-across-versions"), filterToNumExpFiles = Map( equals(col("col1"), ofInt(1)) -> 1, // should prune 2 files equals(col("col2"), ofInt(1)) -> 2, // should prune 1 file new And( equals(col("col1"), ofInt(1)), equals(col("col2"), ofInt(1)) ) -> 1 // should prune 2 files )) } test("data skipping - range of ints") { withTempDir { tempDir => spark.range(10).repartition(1).write.format("delta").save(tempDir.getCanonicalPath) // to test where MIN != MAX checkSkipping( tempDir.getCanonicalPath, hits = Seq( equals(col("id"), ofInt(5)), lessThan(col("id"), ofInt(7)), lessThan(col("id"), ofInt(15)), lessThanOrEqual(col("id"), ofInt(9)), greaterThan(col("id"), ofInt(3)), greaterThan(col("id"), ofInt(-1)), greaterThanOrEqual(col("id"), ofInt(0))), misses = Seq( equals(col("id"), ofInt(10)), lessThan(col("id"), ofInt(0)), lessThan(col("id"), ofInt(-1)), lessThanOrEqual(col("id"), ofInt(-1)), greaterThan(col("id"), ofInt(10)), greaterThan(col("id"), ofInt(11)), greaterThanOrEqual(col("id"), ofInt(11)))) } } test("data skipping - non-eligible min/max data skipping types") { withTempDir { tempDir => val schema = SparkStructType.fromDDL("`id` INT, `arr_col` ARRAY, " + "`map_col` MAP, `struct_col` STRUCT<`field1`: INT>") val data = SparkRow(0, Array(1, 2), Map("foo" -> 1), SparkRow(5)) :: Nil spark.createDataFrame(spark.sparkContext.parallelize(data), schema) .write.format("delta").save(tempDir.getCanonicalPath) checkSkipping( tempDir.getCanonicalPath, hits = Seq( equals(col("id"), ofInt(0)), // filter on the one eligible column isNotNull(col("id")), isNotNull(col("arr_col")), isNotNull(col("map_col")), isNotNull(col("struct_col")), isNotNull(nestedCol("struct_col.field1")), not(isNotNull(col("struct_col"))), // we don't skip on non-leaf columns not(isNull(col("id"))), not(isNull(col("arr_col"))), not(isNull(col("map_col"))), not(isNull(col("struct_col"))), not(isNull(nestedCol("struct_col.field1"))), isNull(col("struct_col"))), misses = Seq( equals(col("id"), ofInt(1)), not(isNotNull(col("id"))), not(isNotNull(col("arr_col"))), not(isNotNull(col("map_col"))), not(isNotNull(nestedCol("struct_col.field1"))), isNull(col("id")), isNull(col("arr_col")), isNull(col("map_col")), isNull(nestedCol("struct_col.field1")))) } } test("data skipping - non-eligible min/max data skipping types all nulls in file") { withTempDir { tempDir => val schema = SparkStructType.fromDDL("`id` INT, `arr_col` ARRAY, " + "`map_col` MAP, `struct_col` STRUCT<`field1`: INT>") val data = SparkRow(null, null, null, null) :: Nil spark.createDataFrame(spark.sparkContext.parallelize(data), schema).coalesce(1) .write.format("delta").save(tempDir.getCanonicalPath) checkSkipping( tempDir.getCanonicalPath, hits = Seq( // [not(is_not_null) is converted to is_null] not(isNotNull(col("id"))), not(isNotNull(col("arr_col"))), not(isNotNull(col("map_col"))), not(isNotNull(col("struct_col"))), not(isNotNull(nestedCol("struct_col.field1"))), isNotNull(col("struct_col")) // we don't skip on non-leaf columns ), misses = Seq( isNotNull(col("id")), isNotNull(col("arr_col")), isNotNull(col("map_col")), isNotNull(nestedCol("struct_col.field1")))) } } test("data skipping - non-eligible min/max data skipping types null +" + "non-null in same file") { withTempDir { tempDir => val schema = SparkStructType.fromDDL("`id` INT, `arr_col` ARRAY, " + "`map_col` MAP, `struct_col` STRUCT<`field1`: INT>") val data = SparkRow(0, Array(1, 2), Map("foo" -> 1), SparkRow(5)) :: SparkRow(null, null, null, null) :: Nil spark.createDataFrame(spark.sparkContext.parallelize(data), schema) .write.format("delta").save(tempDir.getCanonicalPath) checkSkipping( tempDir.getCanonicalPath, hits = Seq( // [not(is_not_null) is converted to is_null] not(isNotNull(col("id"))), not(isNotNull(col("arr_col"))), not(isNotNull(col("map_col"))), not(isNotNull(col("struct_col"))), not(isNotNull(nestedCol("struct_col.field1"))), isNotNull(col("id")), isNotNull(col("arr_col")), isNotNull(col("map_col")), isNotNull(col("struct_col")), isNotNull(nestedCol("struct_col.field1"))), misses = Seq()) } } test("data skipping - is not null with DVs in file with non-nulls") { withSQLConf(("spark.databricks.delta.properties.defaults.enableDeletionVectors", "true")) { withTempDir { tempDir => def overwriteTableAndPerformDelete(deleteCondition: String): Unit = { val data = SparkRow(0, 0) :: SparkRow(1, 1) :: SparkRow(2, null) :: SparkRow(3, null) :: Nil val schema = new SparkStructType() .add("col1", SparkIntegerType) .add("col2", SparkIntegerType) spark.createDataFrame(spark.sparkContext.parallelize(data), schema) .repartition(1) .write .format("delta") .mode("overwrite") .save(tempDir.getCanonicalPath) spark.sql(s"DELETE FROM delta.`${tempDir.getCanonicalPath}` WHERE $deleteCondition") } def checkNoSkipping(): Unit = { checkSkipping( tempDir.getCanonicalPath, hits = Seq( isNotNull(col("col2")), isNotNull(col("col1"))), misses = Seq()) } // remove no rows overwriteTableAndPerformDelete("false") checkNoSkipping() // remove all null rows overwriteTableAndPerformDelete("col2 IS NULL") checkNoSkipping() // remove all non-null rows overwriteTableAndPerformDelete("col2 IS NOT NULL") checkNoSkipping() // remove one null row overwriteTableAndPerformDelete("col1 = 2") checkNoSkipping() } } } test("data skipping - is not null with DVs in file with all nulls") { withSQLConf(("spark.databricks.delta.properties.defaults.enableDeletionVectors", "true")) { withTempDir { tempDir => def checkDoesSkipping(): Unit = { checkSkipping( tempDir.getCanonicalPath, hits = Seq( isNotNull(col("col1"))), misses = Seq( isNotNull(col("col2")))) } // write initial table with all nulls for col2 val data = SparkRow(0, null) :: SparkRow(1, null) :: Nil val schema = new SparkStructType() .add("col1", SparkIntegerType) .add("col2", SparkIntegerType) spark.createDataFrame(spark.sparkContext.parallelize(data), schema) .repartition(1) .write .format("delta") .save(tempDir.getCanonicalPath) checkDoesSkipping() // delete one of the nulls spark.sql(s"DELETE FROM delta.`${tempDir.getCanonicalPath}` WHERE col1 = 0") checkDoesSkipping() } } } test("don't read stats column when there is no usable data skipping filter") { val path = goldenTablePath("data-skipping-basic-stats-all-types") val engine = engineDisallowedStatsReads def snapshot(engine: Engine): Snapshot = { Table.forPath(engine, path).getLatestSnapshot(engine) } def verifyNoStatsColumn(scanFiles: CloseableIterator[FilteredColumnarBatch]): Unit = { scanFiles.forEach { batch => val addSchema = batch.getData.getSchema.get("add").getDataType.asInstanceOf[StructType] assert(addSchema.indexOf("stats") < 0) } } // no filter --> don't read stats verifyNoStatsColumn( snapshot(engineDisallowedStatsReads).getScanBuilder().build().getScanFiles(engine)) // partition filter only --> don't read stats val partFilter = equals(new Column("part"), ofInt(1)) verifyNoStatsColumn( snapshot(engineDisallowedStatsReads) .getScanBuilder().withFilter(partFilter).build() .getScanFiles(engine)) // no eligible data skipping filter --> don't read stats val nonEligibleFilter = lessThan( new ScalarExpression("%", Seq(col("as_int"), ofInt(10)).asJava), ofInt(1)) verifyNoStatsColumn( snapshot(engineDisallowedStatsReads) .getScanBuilder().withFilter(nonEligibleFilter).build() .getScanFiles(engine)) } test("data skipping - prune schema correctly for various predicates") { def structTypeToLeafColumns( schema: StructType, parentPath: Seq[String] = Seq()): Set[Column] = { schema.fields().asScala.flatMap { field => field.getDataType() match { case nestedSchema: StructType => assert( nestedSchema.fields().size() > 0, "Schema should not have field of type StructType with no child fields") structTypeToLeafColumns(nestedSchema, parentPath ++ Seq(field.getName())) case _ => Seq(new Column(parentPath.toArray :+ field.getName())) } }.toSet } def verifySchema(expectedReadCols: Set[Column]): StructType => Unit = { readSchema => assert(structTypeToLeafColumns(readSchema) == expectedReadCols) } val path = goldenTablePath("data-skipping-basic-stats-all-types") // Map of expression -> expected read columns Map( equals(col("as_int"), ofInt(0)) -> Set(nestedCol("minValues.as_int"), nestedCol("maxValues.as_int")), lessThan(col("as_int"), ofInt(0)) -> Set(nestedCol("minValues.as_int")), greaterThan(col("as_int"), ofInt(0)) -> Set(nestedCol("maxValues.as_int")), greaterThanOrEqual(col("as_int"), ofInt(0)) -> Set(nestedCol("maxValues.as_int")), lessThanOrEqual(col("as_int"), ofInt(0)) -> Set(nestedCol("minValues.as_int")), new And( lessThan(col("as_int"), ofInt(0)), greaterThan(col("as_long"), ofInt(0))) -> Set( nestedCol("minValues.as_int"), nestedCol("maxValues.as_long"))).foreach { case (predicate, expectedCols) => val engine = engineVerifyJsonParseSchema(verifySchema(expectedCols)) collectScanFileRows( Table.forPath(engine, path).getLatestSnapshot(engine) .getScanBuilder().withFilter(predicate).build(), engine = engine) } } test("data skipping - validate stats written by kernel can be read and used") { withTempDirAndEngine { (tablePath, engine) => val schema = new StructType() .add( "nested", new StructType() .add("byteCol", ByteType.BYTE) .add("intCol", IntegerType.INTEGER) .add("floatCol", FloatType.FLOAT) .add("normalDouble", DoubleType.DOUBLE) // used for filtering .add("weirdDouble", DoubleType.DOUBLE) // may contain NaN/Infinity .add("decimalCol", new DecimalType(10, 2))) val tableProps = Map(TableConfig.DATA_SKIPPING_NUM_INDEXED_COLS.getKey -> "10") val txn = getCreateTxn(engine, tablePath, schema, List.empty, tableProps) txn.commit(engine, emptyIterable()) // Build some rows with corner-case values val testRows = Seq( ( -128.toByte, Int.MinValue, Float.NaN, 1500.0, Double.PositiveInfinity, new java.math.BigDecimal("98765.43")), (0.toByte, 0, 1.23f, 10.0, -42.99, new java.math.BigDecimal("0.00")), ( 127.toByte, Int.MaxValue, Float.NegativeInfinity, 200.0, Double.NaN, new java.math.BigDecimal("9999999.99"))) testRows.zipWithIndex.foreach { case ((b, i, f, normalD, weirdD, dec), idx) => val singleRowBatch = buildSingleStructColumnRowBatch(schema, Array[Any](b, i, f, normalD, weirdD, dec)) val commitResult = appendData( engine, tablePath, data = List(Map.empty[String, Literal] -> List(singleRowBatch))) verifyCommitResult(commitResult, expVersion = idx + 1, expIsReadyForCheckpoint = false) } // Filter: select rows where nested.normalDouble > 50.0. // Expected: Row 0 (1500.0) and Row 2 (200.0) pass, Row 1 (10.0) is pruned. val skipFilter = greaterThan(nestedCol("nested.normalDouble"), ofDouble(50.0)) val snapshot = Table.forPath(engine, tablePath).getLatestSnapshot(engine) val scan = snapshot.getScanBuilder().withFilter(skipFilter).build() val scanFiles = collectScanFileRows(scan, engine) // Assert that exactly 2 files (rows) match the filter. assert( scanFiles.size == 2, s"Expected exactly 2 matching files (rows with normalDouble > 50.0: row 0 & 2)." + s" Found ${scanFiles.size}.") } } /** * Creates a single-row FilteredColumnarBatch assuming the schema has one top-level StructType * and `rowValues` align with its subfields. */ def buildSingleStructColumnRowBatch( schema: StructType, rowValues: Array[Any]): FilteredColumnarBatch = { require(schema.length() == 1, s"Expected 1 field, found ${schema.length()}") val nestedType = schema.get("nested").getDataType.asInstanceOf[StructType] require( nestedType.length() == rowValues.length, s"${nestedType.length()} vs ${rowValues.length}") // We zip each field with an index so we can pick rowValues(i) val childVectors: Array[ColumnVector] = nestedType.fields().asScala.zipWithIndex.map { case (field: StructField, i: Int) => DefaultGenericVector.fromArray(field.getDataType, Array(rowValues(i).asInstanceOf[AnyRef])) }.toArray val structVector = new DefaultStructVector(1, nestedType, java.util.Optional.empty(), childVectors) val batch = new DefaultColumnarBatch(1, schema, Array(structVector)) new FilteredColumnarBatch(batch, java.util.Optional.empty()) } ////////////////////////////////////////////////////////////////////////////////////////// // Check the includeStats parameter on ScanImpl.getScanFiles(engine, includeStats) ////////////////////////////////////////////////////////////////////////////////////////// test("check ScanImpl.getScanFiles for includeStats=true") { // When includeStats=true the JSON statistic should always be returned in the scan files withTempDir { tempDir => spark.range(10).write.format("delta").save(tempDir.getCanonicalPath) def checkStatsPresent(scan: Scan): Unit = { val scanFileBatches = scan.asInstanceOf[ScanImpl].getScanFiles(defaultEngine, true) scanFileBatches.forEach { batch => assert(batch.getData().getSchema() == InternalScanFileUtils.SCAN_FILE_SCHEMA_WITH_STATS) } } // No query filter checkStatsPresent( latestSnapshot(tempDir.getCanonicalPath).getScanBuilder().build()) // Query filter but no valid data skipping filter checkStatsPresent( latestSnapshot(tempDir.getCanonicalPath) .getScanBuilder() .withFilter( greaterThan( new ScalarExpression("+", Seq(col("id"), ofInt(10)).asJava), ofInt(100))).build()) // With valid data skipping filter present checkStatsPresent( latestSnapshot(tempDir.getCanonicalPath) .getScanBuilder() .withFilter(greaterThan(col("id"), ofInt(0))) .build()) } } ////////////////////////////////////////////////////////////////////////////////////////// // Tests for collation data skipping / partition pruning ////////////////////////////////////////////////////////////////////////////////////////// // Generic helpers for building batches with fixed column names c1, c2, c3 private def buildBatch(schema: StructType, v1: AnyRef, v2: AnyRef): FilteredColumnarBatch = { val c1Type = schema.get("c1").getDataType val c2Type = schema.get("c2").getDataType val c1Vec = DefaultGenericVector.fromArray(c1Type, Array(v1)) val c2Vec = DefaultGenericVector.fromArray(c2Type, Array(v2)) val batch = new DefaultColumnarBatch(1, schema, Array(c1Vec, c2Vec)) new FilteredColumnarBatch(batch, java.util.Optional.empty()) } private def buildBatch( schema: StructType, v1: AnyRef, v2: AnyRef, v3: AnyRef): FilteredColumnarBatch = { val c1Type = schema.get("c1").getDataType val c2Type = schema.get("c2").getDataType val c3Type = schema.get("c3").getDataType val c1Vec = DefaultGenericVector.fromArray(c1Type, Array(v1)) val c2Vec = DefaultGenericVector.fromArray(c2Type, Array(v2)) val c3Vec = DefaultGenericVector.fromArray(c3Type, Array(v3)) val batch = new DefaultColumnarBatch(1, schema, Array(c1Vec, c2Vec, c3Vec)) new FilteredColumnarBatch(batch, java.util.Optional.empty()) } private def buildNestedBatch( schema: StructType, v1: AnyRef, v2: AnyRef): FilteredColumnarBatch = { val sType = schema.get("s").getDataType.asInstanceOf[StructType] val c1Type = sType.get("c1").getDataType val c2Type = sType.get("c2").getDataType val c1Vec = DefaultGenericVector.fromArray(c1Type, Array(v1)) val c2Vec = DefaultGenericVector.fromArray(c2Type, Array(v2)) val structVec = new DefaultStructVector(1, sType, java.util.Optional.empty(), Array(c1Vec, c2Vec)) val batch = new DefaultColumnarBatch(1, schema, Array(structVec)) new FilteredColumnarBatch(batch, java.util.Optional.empty()) } val utf8Lcase = CollationIdentifier.fromString("SPARK.UTF8_LCASE.74") val utf8LcaseString = new StringType(utf8Lcase) val unicode = CollationIdentifier.fromString("ICU.UNICODE.75.1") val unicodeString = new StringType(unicode) test("partition pruning - predicates with SPARK.UTF8_BINARY on partition column") { Seq(true, false).foreach { createCheckpoint => Seq( (STRING, STRING), (utf8LcaseString, utf8LcaseString), (unicodeString, unicodeString), (utf8LcaseString, unicodeString), (STRING, utf8LcaseString), (unicodeString, STRING)).foreach { case (c1Type, c2Type) => withTempDir { tempDir => val tablePath = tempDir.getCanonicalPath val schema = new StructType() .add("c1", c1Type, true) .add("c2", c2Type, true) val c2Collation = c2Type.getCollationIdentifier getCreateTxn(defaultEngine, tablePath, schema, List("c1")).commit( defaultEngine, emptyIterable()) appendData( defaultEngine, tablePath, data = List( Map("c1" -> ofString("a")) -> List(buildBatch(schema, "a", "b")), Map("c1" -> ofString("c", c2Collation)) -> List(buildBatch(schema, "c", "d")), Map("c1" -> ofString("e")) -> List(buildBatch(schema, "e", "f")))) val snapshot = latestSnapshot(tablePath) val totalFiles = collectScanFileRows(snapshot.getScanBuilder().build()).length assert(totalFiles == 3) if (createCheckpoint) { // Create a checkpoint for the table val version = latestSnapshot(tempDir.getCanonicalPath).getVersion Table.forPath(defaultEngine, tempDir.getCanonicalPath) .checkpoint(defaultEngine, version) } val filterToFileNumber = Map( new Predicate( "<", col("c1"), ofString("a")) -> 0, new Predicate( "<", col("c1"), ofString("a", c2Collation)) -> 0, new Predicate( "<", col("c1"), ofString("a"), CollationIdentifier.SPARK_UTF8_BINARY) -> 0, new Predicate( "<", col("c1"), ofString("a", c2Collation), CollationIdentifier.SPARK_UTF8_BINARY) -> 0, new Predicate( "=", ofString("d"), col("c1"), CollationIdentifier.SPARK_UTF8_BINARY) -> 0, new Predicate( "=", col("c1"), ofString("a"), CollationIdentifier.SPARK_UTF8_BINARY) -> 1, new Predicate( "=", col("c1"), ofString("a", c2Collation), CollationIdentifier.SPARK_UTF8_BINARY) -> 1, new Predicate( "=", col("c1"), ofString("a")) -> 1, new Predicate( "=", col("c1"), ofString("a", c2Collation)) -> 1, new Predicate( ">=", col("c1"), ofString("a"), CollationIdentifier.SPARK_UTF8_BINARY) -> totalFiles, new Predicate( ">=", col("c1"), ofString("a", c2Collation), CollationIdentifier.SPARK_UTF8_BINARY) -> totalFiles, new Predicate( ">", col("c1"), ofString("e"), CollationIdentifier.SPARK_UTF8_BINARY) -> 0, new And( new Predicate( ">=", col("c1"), ofString("b"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( "<=", col("c1"), ofString("e"), CollationIdentifier.SPARK_UTF8_BINARY)) -> 2, new Or( new Predicate( "=", col("c1"), ofString("x"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( ">", col("c1"), ofString("d"), CollationIdentifier.SPARK_UTF8_BINARY)) -> 1, new And( new Predicate( ">=", col("c1"), ofString("a"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( "<=", col("c1"), ofString("z"), CollationIdentifier.SPARK_UTF8_BINARY)) -> totalFiles, new Predicate( "STARTS_WITH", col("c1"), ofString("a"), CollationIdentifier.SPARK_UTF8_BINARY) -> 1, new In( col("c1"), java.util.Arrays.asList(ofString("a"), ofString("x")), CollationIdentifier.SPARK_UTF8_BINARY) -> 1, new In( col("c1"), java.util.Arrays.asList(ofString("x"), ofString("y")), CollationIdentifier.SPARK_UTF8_BINARY) -> 0) checkSkipping(tempDir.getCanonicalPath, filterToFileNumber) } } } } test("partition pruning - predicates with non default collation on partition column") { Seq(true, false).foreach { createCheckpoint => Seq( (STRING, STRING), (utf8LcaseString, utf8LcaseString), (unicodeString, unicodeString), (utf8LcaseString, unicodeString), (STRING, utf8LcaseString), (unicodeString, STRING)).foreach { case (c1Type, c2Type) => withTempDir { tempDir => val tablePath = tempDir.getCanonicalPath val schema = new StructType() .add("c1", c1Type, true) .add("c2", c2Type, true) val c2Collation = c2Type.getCollationIdentifier getCreateTxn(defaultEngine, tablePath, schema, List("c1")).commit( defaultEngine, emptyIterable()) appendData( defaultEngine, tablePath, data = List( Map("c1" -> ofString("a")) -> List(buildBatch(schema, "a", "b")), Map("c1" -> ofString("c")) -> List(buildBatch(schema, "c", "d")), Map("c1" -> ofString("e", c2Collation)) -> List(buildBatch(schema, "e", "f")))) val snapshot = latestSnapshot(tablePath) val totalFiles = collectScanFileRows(snapshot.getScanBuilder().build()).length assert(totalFiles == 3) if (createCheckpoint) { // Create a checkpoint for the table val version = latestSnapshot(tempDir.getCanonicalPath).getVersion Table.forPath( defaultEngine, tempDir.getCanonicalPath).checkpoint(defaultEngine, version) } // Non-default collations are not supported by the default engine for predicate // evaluation. Assert that attempting to evaluate such predicates during partition // pruning throws. val failingPredicates = Seq( new Predicate("<", col("c1"), ofString("a"), utf8Lcase), new Predicate("<", col("c1"), ofString("a", c2Collation), utf8Lcase), new Predicate("=", ofString("d"), col("c1"), unicode), new And( new Predicate(">=", col("c1"), ofString("b"), utf8Lcase), new Predicate("<=", col("c1"), ofString("e"), unicode)), new Or( new Predicate("<", col("c1"), ofString("b"), utf8Lcase), new Predicate(">", col("c1"), ofString("a"), CollationIdentifier.SPARK_UTF8_BINARY)), new Predicate("STARTS_WITH", col("c1"), ofString("a"), utf8Lcase), new In( col("c1"), java.util.Arrays.asList(ofString("a"), ofString("c")), utf8Lcase), new In( col("c1"), java.util.Arrays.asList(ofString("a", c2Collation), ofString("c")), utf8Lcase), new Or( new In(col("c1"), java.util.Arrays.asList(ofString("x"), ofString("y")), unicode), new Predicate("=", col("c1"), ofString("z")))) failingPredicates.foreach { predicate => val ex = intercept[KernelEngineException] { collectScanFileRows(snapshot.getScanBuilder().withFilter(predicate).build()) } assert(ex.getMessage.contains("Unsupported collation")) assert(ex.getMessage.contains(CollationIdentifier.SPARK_UTF8_BINARY.toString)) assert(ex.getCause.isInstanceOf[UnsupportedOperationException]) } } } } } test("data skipping - predicates with SPARK.UTF8_BINARY on data column") { Seq(true, false).foreach { createCheckpoint => Seq( (STRING, STRING), (utf8LcaseString, utf8LcaseString), (unicodeString, unicodeString), (utf8LcaseString, unicodeString), (STRING, utf8LcaseString), (unicodeString, STRING)).foreach { case (c1Type, c2Type) => withTempDir { tempDir => // Create three files with values on non-partitioned STRING columns (c1, c2) // Files: ("a","x"), ("c","y"), ("e","z") val tablePath = tempDir.getCanonicalPath val schema = new StructType() .add("c1", c1Type, true) .add("c2", c2Type, true) val c2Collation = c2Type.getCollationIdentifier getCreateTxn(defaultEngine, tablePath, schema, List.empty).commit( defaultEngine, emptyIterable()) appendData( defaultEngine, tablePath, data = List( Map.empty[String, Literal] -> List(buildBatch(schema, "a", "x")), Map.empty[String, Literal] -> List(buildBatch(schema, "c", "y")), Map.empty[String, Literal] -> List(buildBatch(schema, "e", "z")))) val snapshot = latestSnapshot(tablePath) val totalFiles = collectScanFileRows(snapshot.getScanBuilder.build()).length assert(totalFiles == 3) if (createCheckpoint) { // Create a checkpoint for the table val version = latestSnapshot(tempDir.getCanonicalPath).getVersion Table.forPath( defaultEngine, tempDir.getCanonicalPath).checkpoint(defaultEngine, version) } val filterToFileNumber = Map( new Predicate( "<", col("c1"), ofString("a"), CollationIdentifier.SPARK_UTF8_BINARY) -> 0, new Predicate( "<", col("c1"), ofString("a", c2Collation), CollationIdentifier.SPARK_UTF8_BINARY) -> 0, new Predicate( "<", col("c1"), ofString("a")) -> 0, new Predicate( "<", col("c1"), ofString("a", c2Collation)) -> 0, new Predicate( "=", ofString("d"), col("c1"), CollationIdentifier.SPARK_UTF8_BINARY) -> 0, new Predicate( "=", ofString("a"), col("c1"), CollationIdentifier.SPARK_UTF8_BINARY) -> 1, new Predicate( "=", ofString("a", c2Collation), col("c1"), CollationIdentifier.SPARK_UTF8_BINARY) -> 1, new Predicate( "=", ofString("a"), col("c1")) -> 1, new Predicate( "=", ofString("a", c2Collation), col("c1")) -> 1, new Predicate( "<=", ofString("a"), col("c1"), CollationIdentifier.SPARK_UTF8_BINARY) -> totalFiles, new Predicate( "<=", ofString("a", c2Collation), col("c1"), CollationIdentifier.SPARK_UTF8_BINARY) -> totalFiles, new Predicate( "<=", ofString("a"), col("c1")) -> totalFiles, new Predicate( "<=", ofString("a", c2Collation), col("c1")) -> totalFiles, new Predicate( "<", ofString("e"), col("c1"), CollationIdentifier.SPARK_UTF8_BINARY) -> 0, new And( new Predicate( ">=", col("c1"), ofString("b"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( "<=", col("c1"), ofString("e"), CollationIdentifier.SPARK_UTF8_BINARY)) -> 2, new Or( new Predicate( "=", col("c1"), ofString("x"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( ">", col("c1"), ofString("d"), CollationIdentifier.SPARK_UTF8_BINARY)) -> 1, new And( new Predicate( ">=", col("c1"), ofString("a"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( "<=", col("c1"), ofString("z"), CollationIdentifier.SPARK_UTF8_BINARY)) -> totalFiles, new And( new Predicate( "<=", ofString("b"), col("c1"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( ">=", ofString("y"), col("c2"), CollationIdentifier.SPARK_UTF8_BINARY)) -> 1, new And( new Predicate( ">=", ofString("c"), col("c1"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( "<=", ofString("y"), col("c2"), CollationIdentifier.SPARK_UTF8_BINARY)) -> 1, new Or( new Predicate( "=", ofString("a"), col("c1"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( "=", ofString("z"), col("c2"), CollationIdentifier.SPARK_UTF8_BINARY)) -> 2, new Or( new Predicate( "<", ofString("d"), col("c1"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( ">", ofString("y"), col("c2"), CollationIdentifier.SPARK_UTF8_BINARY)) -> 2, new And( new Predicate( "<", ofString("e"), col("c1"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( ">", ofString("y"), col("c2"), CollationIdentifier.SPARK_UTF8_BINARY)) -> 0, new And( new Predicate( "<", ofString("e"), col("c1")), new Predicate( ">", ofString("y"), col("c2"), CollationIdentifier.SPARK_UTF8_BINARY)) -> 0, new And( new Predicate( "<", ofString("e"), col("c1"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( ">", ofString("y"), col("c2"))) -> 0) checkSkipping(tempDir.getCanonicalPath, filterToFileNumber) } } } } test("data skipping - predicate with collation without version on data column") { Seq(true, false).foreach { createCheckpoint => Seq( (STRING, STRING), (utf8LcaseString, utf8LcaseString), (unicodeString, unicodeString), (utf8LcaseString, unicodeString), (STRING, utf8LcaseString), (unicodeString, STRING)).foreach { case (c1Type, c2Type) => withTempDir { tempDir => val tablePath = tempDir.getCanonicalPath val schema = new StructType() .add("c1", c1Type, true) .add("c2", c2Type, true) getCreateTxn(defaultEngine, tablePath, schema, List("c1")).commit( defaultEngine, emptyIterable()) appendData( defaultEngine, tablePath, data = List( Map("c1" -> ofString("a")) -> List(buildBatch(schema, "a", "b")), Map("c1" -> ofString("c")) -> List(buildBatch(schema, "c", "d")), Map("c1" -> ofString("e")) -> List(buildBatch(schema, "e", "f")))) val snapshot = latestSnapshot(tablePath) val totalFiles = collectScanFileRows(snapshot.getScanBuilder().build()).length assert(totalFiles == 3) if (createCheckpoint) { // Create a checkpoint for the table val version = latestSnapshot(tempDir.getCanonicalPath).getVersion Table.forPath( defaultEngine, tempDir.getCanonicalPath).checkpoint(defaultEngine, version) } val filterToFileNumber = Map( new Predicate( "<", col("c2"), ofString("a"), CollationIdentifier.fromString("SPARK.UTF8_LCASE")) -> totalFiles) checkSkipping(tablePath, filterToFileNumber) } } } } test("data skipping - predicates with SPARK.UTF8_BINARY on nested data column") { Seq(true, false).foreach { createCheckpoint => Seq( (STRING, STRING), (utf8LcaseString, utf8LcaseString), (unicodeString, unicodeString), (utf8LcaseString, unicodeString), (STRING, utf8LcaseString), (unicodeString, STRING)).foreach { case (c1Type, c2Type) => withTempDir { tempDir => // Create three files with values on non-partitioned nested STRING columns (s.c1, s.c2) // Files: ("a","x"), ("c","y"), ("e","z") val tablePath = tempDir.getCanonicalPath val schema = new StructType() .add("s", new StructType().add("c1", c1Type, true).add("c2", c2Type, true), true) val c2Collation = c2Type.getCollationIdentifier getCreateTxn(defaultEngine, tablePath, schema, List.empty).commit( defaultEngine, emptyIterable()) appendData( defaultEngine, tablePath, data = List( Map.empty[String, Literal] -> List(buildNestedBatch(schema, "a", "x")), Map.empty[String, Literal] -> List(buildNestedBatch(schema, "c", "y")), Map.empty[String, Literal] -> List(buildNestedBatch(schema, "e", "z")))) val snapshot = latestSnapshot(tablePath) val totalFiles = collectScanFileRows(snapshot.getScanBuilder.build()).length assert(totalFiles == 3) if (createCheckpoint) { // Create a checkpoint for the table val version = latestSnapshot(tempDir.getCanonicalPath).getVersion Table.forPath( defaultEngine, tempDir.getCanonicalPath).checkpoint(defaultEngine, version) } val filterToFileNumber = Map( new Predicate( "<", nestedCol("s.c1"), ofString("a"), CollationIdentifier.SPARK_UTF8_BINARY) -> 0, new Predicate( "=", ofString("d"), nestedCol("s.c1"), CollationIdentifier.SPARK_UTF8_BINARY) -> 0, new Predicate( "=", ofString("a"), nestedCol("s.c1"), CollationIdentifier.SPARK_UTF8_BINARY) -> 1, new Predicate( "=", ofString("a", c2Collation), nestedCol("s.c1"), CollationIdentifier.SPARK_UTF8_BINARY) -> 1, new Predicate( "=", ofString("a"), nestedCol("s.c1")) -> 1, new Predicate( "=", ofString("a", c2Collation), nestedCol("s.c1")) -> 1, new Predicate( "<=", ofString("a"), nestedCol("s.c1"), CollationIdentifier.SPARK_UTF8_BINARY) -> totalFiles, new Predicate( "<", ofString("e"), nestedCol("s.c1"), CollationIdentifier.SPARK_UTF8_BINARY) -> 0, new And( new Predicate( ">=", nestedCol("s.c1"), ofString("b"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( "<=", nestedCol("s.c1"), ofString("e"), CollationIdentifier.SPARK_UTF8_BINARY)) -> 2, new Or( new Predicate( "=", nestedCol("s.c1"), ofString("x"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( ">", nestedCol("s.c1"), ofString("d"), CollationIdentifier.SPARK_UTF8_BINARY)) -> 1, new And( new Predicate( ">=", nestedCol("s.c1"), ofString("a"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( "<=", nestedCol("s.c1"), ofString("z"), CollationIdentifier.SPARK_UTF8_BINARY)) -> totalFiles, new And( new Predicate( "<=", ofString("b"), nestedCol("s.c1"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( ">=", ofString("y"), nestedCol("s.c2"), CollationIdentifier.SPARK_UTF8_BINARY)) -> 1, new And( new Predicate( ">=", ofString("c"), nestedCol("s.c1"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( "<=", ofString("y"), nestedCol("s.c2"), CollationIdentifier.SPARK_UTF8_BINARY)) -> 1, new Or( new Predicate( "=", ofString("a"), nestedCol("s.c1"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( "=", ofString("z"), nestedCol("s.c2"), CollationIdentifier.SPARK_UTF8_BINARY)) -> 2, new Or( new Predicate( "<", ofString("d"), nestedCol("s.c1"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( ">", ofString("y"), nestedCol("s.c2"), CollationIdentifier.SPARK_UTF8_BINARY)) -> 2, new And( new Predicate( "<", ofString("e"), nestedCol("s.c1"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( ">", ofString("y"), nestedCol("s.c2"), CollationIdentifier.SPARK_UTF8_BINARY)) -> 0) checkSkipping(tablePath, filterToFileNumber) } } } } test("data skipping - collated predicates not or partially convertible to skipping filter") { Seq(true, false).foreach { createCheckpoint => Seq( (STRING, STRING), (utf8LcaseString, utf8LcaseString), (unicodeString, unicodeString), (utf8LcaseString, unicodeString), (STRING, utf8LcaseString), (unicodeString, STRING)).foreach { case (c1Type, c2Type) => withTempDir { tempDir => val tablePath = tempDir.getCanonicalPath val schema = new StructType() .add("c1", c1Type, true) .add("c2", c2Type, true) getCreateTxn(defaultEngine, tablePath, schema, List.empty).commit( defaultEngine, emptyIterable()) appendData( defaultEngine, tablePath, data = List( Map.empty[String, Literal] -> List(buildBatch(schema, "a", "x")), Map.empty[String, Literal] -> List(buildBatch(schema, "c", "y")), Map.empty[String, Literal] -> List(buildBatch(schema, "e", "z")))) val snapshot = latestSnapshot(tablePath) val totalFiles = collectScanFileRows(snapshot.getScanBuilder().build()).length assert(totalFiles == 3) if (createCheckpoint) { val version = latestSnapshot(tempDir.getCanonicalPath).getVersion Table.forPath( defaultEngine, tempDir.getCanonicalPath).checkpoint(defaultEngine, version) } val filterToFileNumber = Map( new Predicate( "STARTS_WITH", col("c1"), ofString("a"), CollationIdentifier.SPARK_UTF8_BINARY) -> totalFiles, new Predicate( "STARTS_WITH", col("c1"), ofString("a"), utf8Lcase) -> totalFiles, new Predicate( "STARTS_WITH", col("c1"), ofString("z"), unicode) -> totalFiles, new In( col("c1"), java.util.Arrays.asList(ofString("a"), ofString("z")), CollationIdentifier.SPARK_UTF8_BINARY) -> totalFiles, new In( col("c2"), java.util.Arrays.asList(ofString("x"), ofString("zz")), utf8Lcase) -> totalFiles, new And( new Predicate( "<", col("c1"), ofString("d"), CollationIdentifier.SPARK_UTF8_BINARY), new In( col("c2"), java.util.Arrays.asList(ofString("x"), ofString("zz")), CollationIdentifier.SPARK_UTF8_BINARY)) -> 2, new Or( new Predicate("STARTS_WITH", col("c1"), ofString("a"), utf8Lcase), new In( col("c2"), java.util.Arrays.asList(ofString("x"), ofString("y")), unicode)) -> totalFiles, new And( new Predicate( "STARTS_WITH", col("c1"), ofString("a"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate("STARTS_WITH", col("c2"), ofString("x"), unicode)) -> totalFiles) checkSkipping(tempDir.getCanonicalPath, filterToFileNumber) } } } } test("data skipping - evaluation fails with non default collation on data column") { Seq(true, false).foreach { createCheckpoint => Seq( (STRING, STRING), (utf8LcaseString, utf8LcaseString), (unicodeString, unicodeString), (utf8LcaseString, unicodeString), (STRING, utf8LcaseString), (unicodeString, STRING)).foreach { case (c1Type, c2Type) => withTempDir { tempDir => val tablePath = tempDir.getCanonicalPath val schema = new StructType() .add("c1", c1Type, true) .add("c2", c2Type, true) val c2Collation = c2Type.getCollationIdentifier getCreateTxn(defaultEngine, tablePath, schema, List.empty).commit( defaultEngine, emptyIterable()) appendData( defaultEngine, tablePath, data = List( Map.empty[String, Literal] -> List(buildBatch(schema, "a", "x")), Map.empty[String, Literal] -> List(buildBatch(schema, "c", "y")), Map.empty[String, Literal] -> List(buildBatch(schema, "e", "z")))) val snapshot = latestSnapshot(tablePath) val totalFiles = collectScanFileRows(snapshot.getScanBuilder().build()).length assert(totalFiles == 3) if (createCheckpoint) { val version = latestSnapshot(tempDir.getCanonicalPath).getVersion Table.forPath( defaultEngine, tempDir.getCanonicalPath).checkpoint(defaultEngine, version) } val failingPredicates = Seq( new Predicate("<", col("c1"), ofString("a"), utf8Lcase), new Predicate("<", col("c1"), ofString("a", c2Collation), utf8Lcase), new Predicate("=", ofString("d"), col("c1"), unicode), new And( new Predicate(">=", col("c1"), ofString("b"), utf8Lcase), new Predicate("<=", col("c1"), ofString("e"), unicode)), new And( new Predicate(">=", col("c1"), ofString("b"), utf8Lcase), new Predicate("<=", col("c1"), ofString("e"))), new And( new Predicate(">=", col("c1"), ofString("b")), new Predicate("<=", col("c1"), ofString("e"), unicode)), new Or( new Predicate("<", col("c1"), ofString("b"), utf8Lcase), new Predicate(">", col("c1"), ofString("a"), CollationIdentifier.SPARK_UTF8_BINARY)), new Or( new Predicate("<", col("c1"), ofString("b", c2Collation), utf8Lcase), new Predicate( ">", col("c1"), ofString("a", c2Collation), CollationIdentifier.SPARK_UTF8_BINARY)), new And( new Predicate(">=", col("c1"), ofString("a"), utf8Lcase), new Predicate("<=", col("c1"), ofString("z"), unicode)), new Predicate("=", col("c1"), ofString("a"), utf8Lcase)) failingPredicates.foreach { predicate => val ex = intercept[KernelEngineException] { collectScanFileRows(snapshot.getScanBuilder.withFilter(predicate).build()) } assert(ex.getMessage.contains("Unsupported collation")) assert(ex.getMessage.contains(CollationIdentifier.SPARK_UTF8_BINARY.toString)) assert(ex.getCause.isInstanceOf[UnsupportedOperationException]) } } } } } test("partition and data skipping - combined pruning on collated partition and data columns") { Seq(true, false).foreach { createCheckpoint => Seq( (STRING, STRING, STRING), (utf8LcaseString, utf8LcaseString, utf8LcaseString), (utf8LcaseString, unicodeString, unicodeString), (STRING, utf8LcaseString, unicodeString), (STRING, utf8LcaseString, STRING), (unicodeString, STRING, utf8LcaseString)).foreach { case (c1Type, c2Type, c3Type) => withTempDir { tempDir => val tablePath = tempDir.getCanonicalPath val schema = new StructType() .add("c1", c1Type, true) .add("c2", c2Type, true) .add("c3", c3Type, true) getCreateTxn(defaultEngine, tablePath, schema, List("c1")).commit( defaultEngine, emptyIterable()) appendData( defaultEngine, tablePath, data = List( Map("c1" -> ofString("a")) -> List(buildBatch( schema, "a", "x", "u")), Map("c1" -> ofString("c")) -> List(buildBatch( schema, "c", "y", "v")), Map("c1" -> ofString("e")) -> List(buildBatch( schema, "e", "z", "w")))) val snapshot = latestSnapshot(tablePath) val totalFiles = collectScanFileRows(snapshot.getScanBuilder().build()).length assert(totalFiles == 3) if (createCheckpoint) { val version = latestSnapshot(tempDir.getCanonicalPath).getVersion Table.forPath( defaultEngine, tempDir.getCanonicalPath).checkpoint(defaultEngine, version) } val filterToFileNumber: Map[Predicate, Int] = Map( new And( new Predicate( "<=", col("c2"), ofString("y"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( ">=", col("c1"), ofString("b"), CollationIdentifier.SPARK_UTF8_BINARY)) -> 1, new And( new Predicate( "<=", col("c1"), ofString("c"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( "<=", col("c2"), ofString("z"), CollationIdentifier.SPARK_UTF8_BINARY)) -> 2, new And( new Predicate( ">=", col("c1"), ofString("a"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( "<", col("c2"), ofString("d"), CollationIdentifier.SPARK_UTF8_BINARY)) -> 0, new And( new Predicate( ">=", col("c1"), ofString("a"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( "<=", col("c2"), ofString("z"), CollationIdentifier.SPARK_UTF8_BINARY)) -> totalFiles) checkSkipping(tempDir.getCanonicalPath, filterToFileNumber) } } } } test("partition and data skipping - evaluation fails with non default collation on " + "combined filter") { Seq(true, false).foreach { createCheckpoint => Seq( (STRING, STRING, STRING), (utf8LcaseString, utf8LcaseString, utf8LcaseString), (utf8LcaseString, unicodeString, unicodeString), (STRING, utf8LcaseString, unicodeString), (STRING, utf8LcaseString, STRING), (unicodeString, STRING, utf8LcaseString)).foreach { case (c1Type, c2Type, c3Type) => withTempDir { tempDir => val tablePath = tempDir.getCanonicalPath val schema = new StructType() .add("c1", c1Type, true) .add("c2", c2Type, true) .add("c3", c3Type, true) getCreateTxn(defaultEngine, tablePath, schema, List("c1")).commit( defaultEngine, emptyIterable()) appendData( defaultEngine, tablePath, data = List( Map("c1" -> ofString("a")) -> List(buildBatch( schema, "a", "x", "u")), Map("c1" -> ofString("c")) -> List(buildBatch( schema, "c", "y", "v")), Map("c1" -> ofString("e")) -> List(buildBatch( schema, "e", "z", "w")))) val snapshot = latestSnapshot(tablePath) val totalFiles = collectScanFileRows(snapshot.getScanBuilder().build()).length assert(totalFiles == 3) if (createCheckpoint) { val version = latestSnapshot(tempDir.getCanonicalPath).getVersion Table.forPath( defaultEngine, tempDir.getCanonicalPath).checkpoint(defaultEngine, version) } val failingPredicates = Seq( new And( new Predicate( ">=", col("c1"), ofString("a"), CollationIdentifier.SPARK_UTF8_BINARY), new Predicate( "<=", col("c2"), ofString("z"), utf8Lcase)), new And( new Predicate( ">=", col("c1"), ofString("a"), unicode), new Predicate( "<=", col("c2"), ofString("z"), CollationIdentifier.SPARK_UTF8_BINARY)), new And( new Predicate( "<", col("c1"), ofString("z"), utf8Lcase), new Predicate( ">", col("c2"), ofString("a"), unicode))) failingPredicates.foreach { predicate => val ex = intercept[KernelEngineException] { collectScanFileRows(snapshot.getScanBuilder.withFilter(predicate).build()) } assert(ex.getMessage.contains("Unsupported collation")) assert(ex.getMessage.contains(CollationIdentifier.SPARK_UTF8_BINARY.toString)) assert(ex.getCause.isInstanceOf[UnsupportedOperationException]) } } } } } Seq( "spark-variant-checkpoint", "spark-variant-stable-feature-checkpoint", "spark-shredded-variant-preview-delta").foreach { tableName => Seq( ("version 0 no predicate", None, Some(0), 2), ("latest version (has checkpoint) no predicate", None, None, 4), ("version 0 with predicate", Some(equals(col("id"), ofLong(10))), Some(0), 1)).foreach { case (nameSuffix, predicate, snapshotVersion, expectedNumFiles) => test(s"read scan files with variant - $nameSuffix - $tableName") { val path = getTestResourceFilePath(tableName) val table = Table.forPath(defaultEngine, path) val snapshot = snapshotVersion match { case Some(version) => table.getSnapshotAsOfVersion(defaultEngine, version) case None => table.getLatestSnapshot(defaultEngine) } val snapshotSchema = snapshot.getSchema() val expectedSchema = new StructType() .add("id", LongType.LONG, true) .add("v", VariantType.VARIANT, true) .add("array_of_variants", new ArrayType(VariantType.VARIANT, true), true) .add("struct_of_variants", new StructType().add("v", VariantType.VARIANT, true)) .add("map_of_variants", new MapType(StringType.STRING, VariantType.VARIANT, true), true) .add( "array_of_struct_of_variants", new ArrayType(new StructType().add("v", VariantType.VARIANT, true), true), true) .add( "struct_of_array_of_variants", new StructType().add("v", new ArrayType(VariantType.VARIANT, true), true), true) assert(snapshotSchema == expectedSchema) val scanBuilder = snapshot.getScanBuilder() val scan = predicate match { case Some(pred) => scanBuilder.withFilter(pred).build() case None => scanBuilder.build() } val scanFiles = scan.asInstanceOf[ScanImpl].getScanFiles(defaultEngine, true) var numFiles: Int = 0 scanFiles.forEach { s => numFiles += s.getRows().toSeq.length } assert(numFiles == expectedNumFiles) } } } } object ScanSuite { private def throwErrorIfAddStatsInSchema(readSchema: StructType): Unit = { if (readSchema.indexOf("add") >= 0) { val addSchema = readSchema.get("add").getDataType.asInstanceOf[StructType] assert(addSchema.indexOf("stats") < 0, "reading column add.stats is not allowed"); } } /** * Returns a custom engine implementation that doesn't allow "add.stats" in the read schema * for parquet or json handlers. */ def engineDisallowedStatsReads: Engine = { val fileIO = new HadoopFileIO(new Configuration()) new DefaultEngine(fileIO) { override def getParquetHandler: ParquetHandler = { new DefaultParquetHandler(fileIO) { override def readParquetFiles( fileIter: CloseableIterator[FileStatus], physicalSchema: StructType, predicate: Optional[Predicate]): CloseableIterator[FileReadResult] = { throwErrorIfAddStatsInSchema(physicalSchema) super.readParquetFiles(fileIter, physicalSchema, predicate) } } } override def getJsonHandler: JsonHandler = { new DefaultJsonHandler(fileIO) { override def readJsonFiles( fileIter: CloseableIterator[FileStatus], physicalSchema: StructType, predicate: Optional[Predicate]): CloseableIterator[ColumnarBatch] = { throwErrorIfAddStatsInSchema(physicalSchema) super.readJsonFiles(fileIter, physicalSchema, predicate) } } } } } def engineVerifyJsonParseSchema(verifyFx: StructType => Unit): Engine = { val fileIO = new HadoopFileIO(new Configuration()) new DefaultEngine(fileIO) { override def getJsonHandler: JsonHandler = { new DefaultJsonHandler(fileIO) { override def parseJson( stringVector: ColumnVector, schema: StructType, selectionVector: Optional[ColumnVector]): ColumnarBatch = { verifyFx(schema) super.parseJson(stringVector, schema, selectionVector) } } } } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/SnapshotChecksumStatisticsAndWriteSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import io.delta.kernel.{Operation, TableManager} import io.delta.kernel.Snapshot.ChecksumWriteMode import io.delta.kernel.defaults.utils.TestUtils import io.delta.kernel.engine.Engine import io.delta.kernel.types.IntegerType.INTEGER import io.delta.kernel.types.StructType import io.delta.kernel.utils.CloseableIterable.emptyIterable import org.scalatest.funsuite.AnyFunSuite class SnapshotChecksumStatisticsAndWriteSuite extends AnyFunSuite with TestUtils { val testSchema = new StructType().add("id", INTEGER) private def assertCrcExistsAtLatest(engine: Engine, tablePath: String): Unit = { val latestSnapshot = TableManager.loadSnapshot(tablePath).build(engine) assert(latestSnapshot.getStatistics.getChecksumWriteMode.isEmpty) } test("getChecksumWriteMode: CRC already exists => empty (trivial case)") { withTempDirAndEngine { (tablePath, engine) => // GIVEN val txn0 = TableManager.buildCreateTableTransaction(tablePath, testSchema, "x").build(engine) val result0 = txn0.commit(engine, emptyIterable()) val snapshot0 = result0.getPostCommitSnapshot.get() snapshot0.writeChecksum(engine, ChecksumWriteMode.SIMPLE) // WHEN/THEN assertCrcExistsAtLatest(engine, tablePath) // this is what we are really testing. trivial. } } test("getChecksumWriteMode: created new table => SIMPLE") { withTempDirAndEngine { (tablePath, engine) => // ===== WHEN ===== val txn0 = TableManager.buildCreateTableTransaction(tablePath, testSchema, "xx").build(engine) val result0 = txn0.commit(engine, emptyIterable()) // ===== THEN ===== val snapshot0 = result0.getPostCommitSnapshot.get() assert(snapshot0.getStatistics.getChecksumWriteMode.get == ChecksumWriteMode.SIMPLE) snapshot0.writeChecksum(engine, ChecksumWriteMode.SIMPLE) // we can write it! assertCrcExistsAtLatest(engine, tablePath) // it exists now } } test("getChecksumWriteMode: CRC exists at N-1 => SIMPLE") { withTempDirAndEngine { (tablePath, engine) => // ===== GIVEN ===== val txn0 = TableManager.buildCreateTableTransaction(tablePath, testSchema, "xx").build(engine) val result0 = txn0.commit(engine, emptyIterable()) val snapshot0 = result0.getPostCommitSnapshot.get() snapshot0.writeChecksum(engine, ChecksumWriteMode.SIMPLE) assertCrcExistsAtLatest(engine, tablePath) // ===== WHEN ===== val txn1 = snapshot0.buildUpdateTableTransaction("xx", Operation.WRITE).build(engine) val result1 = txn1.commit(engine, emptyIterable()) // ===== THEN ===== val snapshot1 = result1.getPostCommitSnapshot.get() assert(snapshot1.getStatistics.getChecksumWriteMode.get == ChecksumWriteMode.SIMPLE) snapshot1.writeChecksum(engine, ChecksumWriteMode.SIMPLE) // we can write it! assertCrcExistsAtLatest(engine, tablePath) // it exists now } } test("getChecksumWriteMode: CRC gap exists (no CRC at N-1) with fresh Snapshot => FULL") { withTempDirAndEngine { (tablePath, engine) => // ===== GIVEN ===== val txn0 = TableManager.buildCreateTableTransaction(tablePath, testSchema, "xx").build(engine) txn0.commit(engine, emptyIterable()) // We do NOT write 00.crc // ===== WHEN ===== // We explicitly load a fresh Snapshot. If we used the post-commit Snapshot,the mode would be // SIMPLE! See the test below. val txn1 = TableManager .loadSnapshot(tablePath) .build(engine) .buildUpdateTableTransaction("xx", Operation.WRITE).build(engine) val result1 = txn1.commit(engine, emptyIterable()) val snapshot1 = result1.getPostCommitSnapshot.get() // ===== THEN ===== assert(snapshot1.getStatistics.getChecksumWriteMode.get == ChecksumWriteMode.FULL) snapshot1.writeChecksum(engine, ChecksumWriteMode.FULL) // we can write it! assertCrcExistsAtLatest(engine, tablePath) // it exists now } } // Some additional context: This tests that even if there is no physical CRC file, a post-commit // snapshot, and even the 20th post-commit snapshot in a continuous sequence of writes, will still // have the CRC info loaded in memory, and thus the mode is SIMPLE. test("getChecksumWriteMode: PostCommitSnapshot (starting from CREATE) => always SIMPLE") { withTempDirAndEngine { (tablePath, engine) => // ===== GIVEN ===== // Create the table and do NOT write 00.crc. var txn = TableManager.buildCreateTableTransaction(tablePath, testSchema, "xx").build(engine) var postCommitSnapshot = txn.commit(engine, emptyIterable()).getPostCommitSnapshot.get() assert(postCommitSnapshot.getStatistics.getChecksumWriteMode.get == ChecksumWriteMode.SIMPLE) // ===== WHEN ==== for (_ <- 1 to 20) { // NOTE: We do NOT write N.crc either! txn = postCommitSnapshot.buildUpdateTableTransaction("xx", Operation.WRITE).build(engine) postCommitSnapshot = txn.commit(engine, emptyIterable()).getPostCommitSnapshot.get() // Nonetheless, our post-commit snapshot (starting from CREATE) should have the CRC info // loaded into memory ==> SIMPLE assert( postCommitSnapshot.getStatistics.getChecksumWriteMode.get == ChecksumWriteMode.SIMPLE) } // ===== THEN ===== // We can now write 20.crc via the SIMPLE mode, even though 0 to 19.crc do not exist! postCommitSnapshot.writeChecksum(engine, ChecksumWriteMode.SIMPLE) assertCrcExistsAtLatest(engine, tablePath) } } // Some additional context: This tests that when starting from a fresh snapshot with an existing // CRC file (at version 10), all subsequent post-commit snapshots in a continuous sequence will // inherit and maintain the CRC info in memory, making the mode SIMPLE even without intermediate // CRC files being written. test("getChecksumWriteMode: PostCommitSnapshot (starting from N>0 with CRC) => always SIMPLE") { withTempDirAndEngine { (tablePath, engine) => // ===== GIVEN ===== // Create the table and do NOT write 00.crc. var txn = TableManager.buildCreateTableTransaction(tablePath, testSchema, "xx").build(engine) var postCommitSnapshot = txn.commit(engine, emptyIterable()).getPostCommitSnapshot.get() assert(postCommitSnapshot.getStatistics.getChecksumWriteMode.get == ChecksumWriteMode.SIMPLE) for (_ <- 1 to 10) { // Commit versions 1-10 without writing CRC files txn = postCommitSnapshot.buildUpdateTableTransaction("xx", Operation.WRITE).build(engine) postCommitSnapshot = txn.commit(engine, emptyIterable()).getPostCommitSnapshot.get() } // Versions 0 to 9 do NOT have CRCs. Now we write 10.crc. assert(postCommitSnapshot.getStatistics.getChecksumWriteMode.get == ChecksumWriteMode.SIMPLE) postCommitSnapshot.writeChecksum(engine, ChecksumWriteMode.SIMPLE) // Now, we restart our txn write loop, but using a FRESH Snapshot loaded from version 10. // It will see the 10.crc file. var postCommitSnapshot2 = TableManager.loadSnapshot(tablePath).build(engine) // ===== WHEN ===== for (_ <- 11 to 20) { // NOTE: We do NOT write N.crc either! txn = postCommitSnapshot2.buildUpdateTableTransaction("xx", Operation.WRITE).build(engine) postCommitSnapshot2 = txn.commit(engine, emptyIterable()).getPostCommitSnapshot.get() // Nonetheless, our post-commit snapshot (starting from a FRESH Snapshot at version 10) // should have the CRC info loaded into memory ==> SIMPLE assert( postCommitSnapshot2.getStatistics.getChecksumWriteMode.get == ChecksumWriteMode.SIMPLE) } // ===== THEN ===== // We can now write 20.crc via the SIMPLE mode, even though 11 to 19.crc do not exist! postCommitSnapshot2.writeChecksum(engine, ChecksumWriteMode.SIMPLE) assertCrcExistsAtLatest(engine, tablePath) } } test("invoking writeChecksum with SIMPLE mode when actual mode is FULL => throws") { withTempDirAndEngine { (tablePath, engine) => TableManager .buildCreateTableTransaction(tablePath, testSchema, "engineInfo") .build(engine) .commit(engine, emptyIterable()) val latestSnapshot = TableManager.loadSnapshot(tablePath).build(engine) assert(latestSnapshot.getStatistics.getChecksumWriteMode.get == ChecksumWriteMode.FULL) intercept[IllegalStateException] { latestSnapshot.writeChecksum(engine, ChecksumWriteMode.SIMPLE) } } } test("invoking writeChecksum when checksum already exists => no-op") { withTempDirAndEngine { (tablePath, engine) => val snapshot = TableManager.buildCreateTableTransaction(tablePath, testSchema, "engineInfo") .build(engine) .commit(engine, emptyIterable()) .getPostCommitSnapshot.get() snapshot.writeChecksum(engine, ChecksumWriteMode.SIMPLE) val latestSnapshot = TableManager.loadSnapshot(tablePath).build(engine) assert(latestSnapshot.getStatistics.getChecksumWriteMode.isEmpty) // Both SIMPLE and FULL should be no-op when checksum already exists latestSnapshot.writeChecksum(engine, ChecksumWriteMode.FULL) // no-op, should not throw latestSnapshot.writeChecksum(engine, ChecksumWriteMode.SIMPLE) // no-op, should not throw } } test("invoking writeChecksum with FULL mode when actual mode is SIMPLE => succeeds") { withTempDirAndEngine { (tablePath, engine) => val snapshot = TableManager.buildCreateTableTransaction(tablePath, testSchema, "x") .build(engine).commit(engine, emptyIterable()).getPostCommitSnapshot.get() assert(snapshot.getStatistics.getChecksumWriteMode.get == ChecksumWriteMode.SIMPLE) snapshot.writeChecksum(engine, ChecksumWriteMode.FULL) assertCrcExistsAtLatest(engine, tablePath) } } // Note that we can only use SIMPLE when starting from a post-commit snapshot whose transaction // started with a CRC file. Even if there's a CRC file at historical version N-1, we still need to // do a FULL replay to load the CRC file at version N to write it. test("write checksum at historical version => FULL mode") { withTempDirAndEngine { (tablePath, engine) => // Create version 0 without writing its CRC val txn0 = TableManager.buildCreateTableTransaction(tablePath, testSchema, "xx").build(engine) txn0.commit(engine, emptyIterable()) // Create version 1 without writing its CRC val snapshot0 = TableManager.loadSnapshot(tablePath).build(engine) val txn1 = snapshot0.buildUpdateTableTransaction("xx", Operation.WRITE).build(engine) txn1.commit(engine, emptyIterable()) // Create version 2 without writing its CRC val snapshot1 = TableManager.loadSnapshot(tablePath).build(engine) val txn2 = snapshot1.buildUpdateTableTransaction("xx", Operation.WRITE).build(engine) txn2.commit(engine, emptyIterable()) // Now load the historical snapshot at version 1 and check its mode val historicalSnapshot = TableManager.loadSnapshot(tablePath).atVersion(1).build(engine) assert(historicalSnapshot.getVersion == 1) assert(historicalSnapshot.getStatistics.getChecksumWriteMode.get == ChecksumWriteMode.FULL) // Write checksum for the historical version 1 historicalSnapshot.writeChecksum(engine, ChecksumWriteMode.FULL) // Verify CRC file exists for version 1 val snapshot1Again = TableManager.loadSnapshot(tablePath).atVersion(1).build(engine) assert(snapshot1Again.getStatistics.getChecksumWriteMode.isEmpty) } } test("concurrent checksum write => second write still returns successfully without error") { withTempDirAndEngine { (tablePath, engine) => // ===== GIVEN ===== // Step 1: Create a table (v0.json) and get the post-commit snapshot val txn = TableManager.buildCreateTableTransaction(tablePath, testSchema, "xx").build(engine) val result = txn.commit(engine, emptyIterable()) val postCommitSnapshot = result.getPostCommitSnapshot.get() // Step 2: Load a new snapshot to latest (v0) val freshSnapshot = TableManager.loadSnapshot(tablePath).build(engine) assert(freshSnapshot.getStatistics.getChecksumWriteMode.get == ChecksumWriteMode.FULL) // ===== WHEN ===== // Step 3: Use the fresh snapshot to write the checksum freshSnapshot.writeChecksum(engine, ChecksumWriteMode.FULL) assertCrcExistsAtLatest(engine, tablePath) // ===== THEN ===== // Step 4: Use the first (post-commit) snapshot to write the checksum -- should NOT fail // This simulates a concurrent write scenario where another writer already wrote the CRC assert(postCommitSnapshot.getStatistics.getChecksumWriteMode.get == ChecksumWriteMode.SIMPLE) postCommitSnapshot.writeChecksum(engine, ChecksumWriteMode.SIMPLE) // should be a no-op } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/SnapshotSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import scala.collection.JavaConverters._ import io.delta.kernel.{Operation, Table} import io.delta.kernel.defaults.utils.{AbstractTestUtils, TestUtilsWithLegacyKernelAPIs, TestUtilsWithTableManagerAPIs} import io.delta.kernel.types.{IntegerType, StructField, StructType} import io.delta.kernel.utils.CloseableIterable import org.scalatest.funsuite.AnyFunSuite class SnapshotSuite extends AbstractSnapshotSuite with TestUtilsWithTableManagerAPIs class LegacySnapshotSuite extends AbstractSnapshotSuite with TestUtilsWithLegacyKernelAPIs trait AbstractSnapshotSuite extends AnyFunSuite { self: AbstractTestUtils => Seq( Seq("part1"), // simple case Seq("part1", "part2", "part3"), // multiple partition columns Seq(), // non-partitioned Seq("PART1", "part2") // case-sensitive ).foreach { partCols => test(s"Snapshot getPartitionColumnNames - partCols=$partCols") { withTempDir { dir => // Step 1: Create a table with the given partition columns val table = Table.forPath(defaultEngine, dir.getCanonicalPath) val columns = (partCols ++ Seq("col1", "col2")).map { colName => new StructField(colName, IntegerType.INTEGER, true /* nullable */ ) } val schema = new StructType(columns.asJava) var txnBuilder = table .createTransactionBuilder(defaultEngine, "engineInfo", Operation.CREATE_TABLE) .withSchema(defaultEngine, schema) if (partCols.nonEmpty) { txnBuilder = txnBuilder.withPartitionColumns(defaultEngine, partCols.asJava) } txnBuilder.build(defaultEngine).commit(defaultEngine, CloseableIterable.emptyIterable()) // Step 2: Check the partition columns val tablePartCols = getTableManagerAdapter.getSnapshotAtLatest(defaultEngine, dir.getCanonicalPath) .getPartitionColumnNames() assert(partCols.asJava === tablePartCols) } } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/TableChangesSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.io.File import scala.collection.JavaConverters._ import scala.collection.immutable import io.delta.golden.GoldenTableUtils.goldenTablePath import io.delta.kernel.{Table, TableManager} import io.delta.kernel.CommitRangeBuilder.CommitBoundary import io.delta.kernel.data.{ColumnarBatch, ColumnVector} import io.delta.kernel.data.Row import io.delta.kernel.defaults.utils.{TestUtils, WriteUtils} import io.delta.kernel.engine.Engine import io.delta.kernel.exceptions.{CommitRangeNotFoundException, KernelException, TableNotFoundException, UnsupportedProtocolVersionException} import io.delta.kernel.expressions.Literal import io.delta.kernel.internal.DeltaLogActionUtils.DeltaAction import io.delta.kernel.internal.TableImpl import io.delta.kernel.internal.actions.{AddCDCFile, AddFile, CommitInfo, Metadata, Protocol, RemoveFile, SetTransaction} import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.util.{FileNames, ManualClock, VectorUtils} import io.delta.kernel.types.{DataType, LongType, StructField} import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.actions.{Action => SparkAction, AddCDCFile => SparkAddCDCFile, AddFile => SparkAddFile, CommitInfo => SparkCommitInfo, Metadata => SparkMetadata, Protocol => SparkProtocol, RemoveFile => SparkRemoveFile, SetTransaction => SparkSetTransaction} import org.apache.hadoop.fs.{Path => HadoopPath} import org.apache.spark.sql.functions.col import org.apache.spark.sql.types.{IntegerType, StructType} import org.scalatest.funsuite.AnyFunSuite class LegacyTableChangesSuite extends TableChangesSuite { override def getChanges( tablePath: String, startVersion: Long, endVersion: Long, actionSet: Set[DeltaAction]): Seq[ColumnarBatch] = { Table.forPath(defaultEngine, tablePath) .asInstanceOf[TableImpl] .getChanges(defaultEngine, startVersion, endVersion, actionSet.asJava) .toSeq } } class CommitRangeTableChangesSuite extends TableChangesSuite { override def getChanges( tablePath: String, startVersion: Long, endVersion: Long, actionSet: Set[DeltaAction]): Seq[ColumnarBatch] = { val commitRange = TableManager.loadCommitRange(tablePath, CommitBoundary.atVersion(startVersion)) .withEndBoundary(CommitBoundary.atVersion(endVersion)) .build(defaultEngine) commitRange.getActions( defaultEngine, getTableManagerAdapter.getSnapshotAtVersion(defaultEngine, tablePath, startVersion), actionSet.asJava).toSeq } test("Must provide startSnapshot with the correct version") { withTempDir { tempDir => (0 to 4).foreach { _ => spark.range(10).write.format("delta").mode("append").save(tempDir.getCanonicalPath) } val commitRange = TableManager.loadCommitRange(tempDir.getCanonicalPath, CommitBoundary.atVersion(0)) .withEndBoundary(CommitBoundary.atVersion(4)) .build(defaultEngine) val e = intercept[IllegalArgumentException] { commitRange.getActions( defaultEngine, getTableManagerAdapter.getSnapshotAtVersion(defaultEngine, tempDir.getCanonicalPath, 2), FULL_ACTION_SET.asJava).toSeq } assert(e.getMessage.contains("startSnapshot must have version = startVersion")) } } test("No end boundary provided defaults to latest") { withTempDir { tempDir => (0 to 4).foreach { _ => spark.range(10).write.format("delta").mode("append").save(tempDir.getCanonicalPath) } val commitRange = TableManager.loadCommitRange(tempDir.getCanonicalPath, CommitBoundary.atVersion(0)) .build(defaultEngine) assert(commitRange.getStartVersion == 0 && commitRange.getEndVersion == 4) // Just double check the changes are correct testGetChangesVsSpark(tempDir.getCanonicalPath, 0, 4, FULL_ACTION_SET) } } test("Basic timestamp resolution") { withTempDir { dir => val tablePath = dir.getCanonicalPath val log = DeltaLog.forTable(spark, dir.getCanonicalPath) // Setup part 1 of 2: create log files (0 to 2).foreach { i => spark.range(10).write.format("delta").mode("append").save(tablePath) } // Setup part 2 of 2: edit lastModified times val logPath = new Path(dir.getCanonicalPath, "_delta_log") val delta0 = new File(FileNames.deltaFile(logPath, 0)) val delta1 = new File(FileNames.deltaFile(logPath, 1)) val delta2 = new File(FileNames.deltaFile(logPath, 2)) delta0.setLastModified(1000) delta1.setLastModified(2000) delta2.setLastModified(3000) val latestSnapshot = getTableManagerAdapter.getSnapshotAtLatest(defaultEngine, tablePath) def checkStartBoundary(timestamp: Long, expectedVersion: Long): Unit = { assert(TableManager.loadCommitRange( tablePath, CommitBoundary.atTimestamp(timestamp, latestSnapshot)) .build(defaultEngine) .getStartVersion == expectedVersion) } def checkEndBoundary(timestamp: Long, expectedVersion: Long): Unit = { assert(TableManager.loadCommitRange(tablePath, CommitBoundary.atVersion(0)) .withEndBoundary(CommitBoundary.atTimestamp(timestamp, latestSnapshot)) .build(defaultEngine) .getEndVersion == expectedVersion) } // startTimestamp is before the earliest available version checkStartBoundary(500, 0) // endTimestamp is before the earliest available version intercept[KernelException] { checkEndBoundary(500, -1) } // startTimestamp is at first commit checkStartBoundary(1000, 0) // endTimestamp is at first commit checkEndBoundary(1000, 0) // startTimestamp is between two normal commits checkStartBoundary(1500, 1) // endTimestamp is between two normal commits checkEndBoundary(1500, 0) // startTimestamp is at last commit checkStartBoundary(3000, 2) // endTimestamp is at last commit checkEndBoundary(3000, 2) // startTimestamp is after the last commit intercept[KernelException] { checkStartBoundary(4000, -1) } // endTimestamp is after the last commit checkEndBoundary(4000, 2) } } test("ICT timestamp resolution") { withTempDirAndEngine { (tablePath, engine) => // Create a table with ICT enabled from the start and commits with specific custom-set ICTs val startTime = 1000L val clock = new ManualClock(startTime) // Version 0 has ICT=1000L, but modificationTime=approx current time (1757368326512+) appendData( engine, tablePath, isNewTable = true, testSchema, data = immutable.Seq(Map.empty[String, Literal] -> dataBatches1), clock = clock, tableProperties = Map("delta.enableInCommitTimestamps" -> "true")) // Version 1 has ICT=2000L clock.setTime(startTime + 1000) appendData( engine, tablePath, data = immutable.Seq(Map.empty[String, Literal] -> (dataBatches1 ++ dataBatches2)), clock = clock) // Version 2 has ICT=3000L clock.setTime(startTime + 2000) appendData( engine, tablePath, data = immutable.Seq(Map.empty[String, Literal] -> dataBatches1), clock = clock) val latestSnapshot = getTableManagerAdapter.getSnapshotAtLatest(defaultEngine, tablePath) def checkStartBoundary(timestamp: Long, expectedVersion: Long): Unit = { assert(TableManager.loadCommitRange( tablePath, CommitBoundary.atTimestamp(timestamp, latestSnapshot)) .build(defaultEngine) .getStartVersion == expectedVersion) } def checkEndBoundary(timestamp: Long, expectedVersion: Long): Unit = { assert(TableManager.loadCommitRange(tablePath, CommitBoundary.atVersion(0)) .withEndBoundary(CommitBoundary.atTimestamp(timestamp, latestSnapshot)) .build(defaultEngine) .getEndVersion == expectedVersion) } // Test that timestamp resolution is done using ICT. Since the file modification times for // this table should all be approx the current time, if we are able to correctly resolve // TS-to-version for the timestamp-range we custom set (~1000L-3000L), we know we are // correctly resolving with ICT. // startTimestamp is before the earliest available version checkStartBoundary(startTime - 500, 0) // endTimestamp is before the earliest available version intercept[KernelException] { checkEndBoundary(startTime - 500, -1) } // startTimestamp is at first commit (ICT enabled) checkStartBoundary(startTime, 0) // endTimestamp is at first commit checkEndBoundary(startTime, 0) // startTimestamp is between first and second commit checkStartBoundary(startTime + 500, 1) // endTimestamp is between first and second commit checkEndBoundary(startTime + 500, 0) // startTimestamp is at second commit checkStartBoundary(startTime + 1000, 1) // endTimestamp is at second commit checkEndBoundary(startTime + 1000, 1) // startTimestamp is between second and third commit checkStartBoundary(startTime + 1500, 2) // endTimestamp is between second and third commit checkEndBoundary(startTime + 1500, 1) // startTimestamp is at third commit checkStartBoundary(startTime + 2000, 2) // endTimestamp is at third commit checkEndBoundary(startTime + 2000, 2) // startTimestamp is after the last commit intercept[KernelException] { checkStartBoundary(startTime + 3000, -1) } // endTimestamp is after the last commit checkEndBoundary(startTime + 3000, 2) // Verify that the changes are correctly retrieved using ICT timestamps testGetChangesVsSpark(tablePath, 0, 2, FULL_ACTION_SET) } } test("getCommitActions returns CommitActions with correct version and timestamp") { withTempDir { dir => val tablePath = dir.getCanonicalPath // Create commits with known modification times (0 to 2).foreach { i => spark.range(10).write.format("delta").mode("append").save(tablePath) } // Set custom modification times on delta files val logPath = new Path(dir.getCanonicalPath, "_delta_log") val delta0 = new File(FileNames.deltaFile(logPath, 0)) val delta1 = new File(FileNames.deltaFile(logPath, 1)) val delta2 = new File(FileNames.deltaFile(logPath, 2)) delta0.setLastModified(1000) delta1.setLastModified(2000) delta2.setLastModified(3000) val commitRange = TableManager.loadCommitRange(tablePath, CommitBoundary.atVersion(0)) .withEndBoundary(CommitBoundary.atVersion(2)) .build(defaultEngine) val actionSet = Set( DeltaAction.ADD, DeltaAction.REMOVE, DeltaAction.METADATA, DeltaAction.PROTOCOL, DeltaAction.CDC) val commitsIter = commitRange.getCommitActions( defaultEngine, getTableManagerAdapter.getSnapshotAtVersion(defaultEngine, tablePath, 0), actionSet.asJava) val commits = commitsIter.toSeq assert(commits.size == 3) // Verify versions assert(commits(0).getVersion == 0) assert(commits(1).getVersion == 1) assert(commits(2).getVersion == 2) // Verify timestamps match file modification times (no ICT in this table) assert(commits(0).getTimestamp == 1000) assert(commits(1).getTimestamp == 2000) assert(commits(2).getTimestamp == 3000) // Get Spark's results for comparison val sparkChanges = DeltaLog.forTable(spark, tablePath) .getChanges(0) // Compare actions with Spark using the new compareCommitActions method compareCommitActions(commits, pruneSparkActionsByActionSet(sparkChanges, actionSet)) commitsIter.close() } } test("getCommitActions with ICT returns timestamp from inCommitTimestamp") { withTempDirAndEngine { (tablePath, engine) => val startTime = 5000L val clock = new ManualClock(startTime) // Version 0 with ICT=5000L appendData( engine, tablePath, isNewTable = true, testSchema, data = immutable.Seq(Map.empty[String, Literal] -> dataBatches1), clock = clock, tableProperties = Map("delta.enableInCommitTimestamps" -> "true")) // Version 1 with ICT=6000L clock.setTime(startTime + 1000) appendData( engine, tablePath, data = immutable.Seq(Map.empty[String, Literal] -> (dataBatches1 ++ dataBatches2)), clock = clock) // Version 2 with ICT=7000L clock.setTime(startTime + 2000) appendData( engine, tablePath, data = immutable.Seq(Map.empty[String, Literal] -> dataBatches1), clock = clock) val commitRange = TableManager.loadCommitRange(tablePath, CommitBoundary.atVersion(0)) .withEndBoundary(CommitBoundary.atVersion(2)) .build(defaultEngine) val actionSet = Set( DeltaAction.ADD, DeltaAction.REMOVE, DeltaAction.METADATA, DeltaAction.PROTOCOL, DeltaAction.CDC) val commitsIter = commitRange.getCommitActions( engine, getTableManagerAdapter.getSnapshotAtVersion(engine, tablePath, 0), actionSet.asJava) val commits = commitsIter.toSeq assert(commits.size == 3) // Verify versions assert(commits(0).getVersion == 0) assert(commits(1).getVersion == 1) assert(commits(2).getVersion == 2) // Verify timestamps come from ICT, not file modification times // The file modification times would be much larger (current epoch time) // but our ICT values are in the 5000-7000 range assert(commits(0).getTimestamp == 5000) assert(commits(1).getTimestamp == 6000) assert(commits(2).getTimestamp == 7000) // Get Spark's results for comparison val sparkChanges = DeltaLog.forTable(spark, tablePath) .getChanges(0) compareCommitActions(commits, pruneSparkActionsByActionSet(sparkChanges, actionSet)) commitsIter.close() } } test("getCommitActions can be called multiple times and returns same results") { withTempDir { dir => val tablePath = dir.getCanonicalPath // Create some commits (0 to 2).foreach { i => spark.range(10).write.format("delta").mode("append").save(tablePath) } val commitRange = TableManager.loadCommitRange(tablePath, CommitBoundary.atVersion(0)) .withEndBoundary(CommitBoundary.atVersion(2)) .build(defaultEngine) val actionSet = Set( DeltaAction.ADD, DeltaAction.REMOVE, DeltaAction.METADATA, DeltaAction.PROTOCOL, DeltaAction.CDC) // Call getCommitActions multiple times val commits1 = commitRange.getCommitActions( defaultEngine, getTableManagerAdapter.getSnapshotAtVersion(defaultEngine, tablePath, 0), actionSet.asJava).toSeq val commits2 = commitRange.getCommitActions( defaultEngine, getTableManagerAdapter.getSnapshotAtVersion(defaultEngine, tablePath, 0), actionSet.asJava).toSeq val commits3 = commitRange.getCommitActions( defaultEngine, getTableManagerAdapter.getSnapshotAtVersion(defaultEngine, tablePath, 0), actionSet.asJava).toSeq // Verify all calls return the same number of commits assert(commits1.size == 3) assert(commits2.size == 3) assert(commits3.size == 3) // Verify each call returns the same actions by comparing with Spark val sparkChanges = DeltaLog.forTable(spark, tablePath) .getChanges(0) .filter(_._1 <= 2) compareCommitActions(commits1, pruneSparkActionsByActionSet(sparkChanges, actionSet)) // For commits2 and commits3, we need fresh Spark iterators val sparkChanges2 = DeltaLog.forTable(spark, tablePath) .getChanges(0) .filter(_._1 <= 2) compareCommitActions(commits2, pruneSparkActionsByActionSet(sparkChanges2, actionSet)) val sparkChanges3 = DeltaLog.forTable(spark, tablePath) .getChanges(0) .filter(_._1 <= 2) compareCommitActions(commits3, pruneSparkActionsByActionSet(sparkChanges3, actionSet)) // Also verify that calling getActions on each CommitActions multiple times works commits1.foreach { commit => val actions1 = commit.getActions.toSeq val actions2 = commit.getActions.toSeq assert(actions1.size == actions2.size) // Verify the schemas are the same actions1.zip(actions2).foreach { case (batch1, batch2) => assert(batch1.getSchema.equals(batch2.getSchema)) assert(batch1.getSize == batch2.getSize) } } } } test("getCommitActions with empty commit file") { withTempDirAndEngine { (tablePath, engine) => // Create a table with an initial commit appendData( engine, tablePath, isNewTable = true, testSchema, data = immutable.Seq(Map.empty[String, Literal] -> dataBatches1)) // Create an empty commit file at version 1 using commitUnsafe with no actions import org.apache.spark.sql.delta.DeltaLog val deltaLog = DeltaLog.forTable(spark, tablePath) val txn = deltaLog.startTransaction() txn.commitUnsafe(tablePath, 1) val commitRange = TableManager.loadCommitRange(tablePath, CommitBoundary.atVersion(0)) .withEndBoundary(CommitBoundary.atVersion(1)) .build(defaultEngine) val actionSet = Set( DeltaAction.ADD, DeltaAction.REMOVE, DeltaAction.METADATA, DeltaAction.PROTOCOL, DeltaAction.CDC) val commitsIter = commitRange.getCommitActions( engine, getTableManagerAdapter.getSnapshotAtVersion(engine, tablePath, 0), actionSet.asJava) val commits = commitsIter.toSeq assert(commits.size == 2) // Version 0 and version 1 // Version 0 should have actions (normal commit) val v0Actions = commits(0).getActions.toSeq assert(v0Actions.nonEmpty) // Version 1 (empty commit) should return empty actions val v1Actions = commits(1).getActions.toSeq val totalRows = v1Actions.map(_.getSize).sum assert(totalRows == 0, s"Empty commit file should have no actions, but got $totalRows rows") // Can call getActions multiple times on empty commit val v1ActionsTwice = commits(1).getActions.toSeq assert(v1ActionsTwice.map(_.getSize).sum == 0) } } } abstract class TableChangesSuite extends AnyFunSuite with TestUtils with WriteUtils { /* actionSet including all currently supported actions */ val FULL_ACTION_SET: Set[DeltaAction] = DeltaAction.values().toSet def getChanges( tablePath: String, startVersion: Long, endVersion: Long, actionSet: Set[DeltaAction]): Seq[ColumnarBatch] ////////////////////////////////////////////////////////////////////////////////// // TableImpl.getChangesByVersion tests ////////////////////////////////////////////////////////////////////////////////// /** * For the given parameters, read the table changes from Kernel using * TableImpl.getChangesByVersion and compare results with Spark */ def testGetChangesVsSpark( tablePath: String, startVersion: Long, endVersion: Long, actionSet: Set[DeltaAction]): Unit = { val sparkChanges = DeltaLog.forTable(spark, tablePath) .getChanges(startVersion) .filter(_._1 <= endVersion) // Spark API does not have endVersion val kernelChanges = getChanges(tablePath, startVersion, endVersion, actionSet) // Check schema is as expected (version + timestamp column + the actions requested) kernelChanges.foreach { batch => batch.getSchema.fields().asScala sameElements (Seq("version", "timestamp") ++ actionSet.map(_.colName)) } compareActions(kernelChanges, pruneSparkActionsByActionSet(sparkChanges, actionSet)) } Seq(true, false).foreach { ictEnabled => test(s"getChanges should return the same results as Spark [ictEnabled: $ictEnabled]") { withTempDir { tempDir => val tablePath = tempDir.getCanonicalPath() // The code to create this table is copied from GoldenTables.scala. // The part that enables ICT is a new addition. val log = DeltaLog.forTable(spark, new HadoopPath(tablePath)) val schema = new StructType() .add("part", IntegerType) .add("id", IntegerType) val configuration = if (ictEnabled) { Map("delta.enableInCommitTimestamps" -> "true") } else { Map.empty[String, String] } val metadata = SparkMetadata(schemaString = schema.json, configuration = configuration) val add1 = SparkAddFile("fake/path/1", Map.empty, 1, 1, dataChange = true) val txn1 = log.startTransaction() txn1.commitManuallyWithValidation(metadata, add1) val addCDC2 = SparkAddCDCFile( "fake/path/2", Map("partition_foo" -> "partition_bar"), 1, Map("tag_foo" -> "tag_bar")) val remove2 = SparkRemoveFile("fake/path/1", Some(100), dataChange = true) val txn2 = log.startTransaction() txn2.commitManuallyWithValidation(addCDC2, remove2) val setTransaction3 = SparkSetTransaction("fakeAppId", 3L, Some(200)) val txn3 = log.startTransaction() val latestTableProtocol = log.snapshot.protocol txn3.commitManuallyWithValidation(latestTableProtocol, setTransaction3) // request subset of actions testGetChangesVsSpark( tablePath, 0, 2, Set(DeltaAction.REMOVE)) testGetChangesVsSpark( tablePath, 0, 2, Set(DeltaAction.ADD)) testGetChangesVsSpark( tablePath, 0, 2, Set(DeltaAction.ADD, DeltaAction.REMOVE, DeltaAction.METADATA, DeltaAction.PROTOCOL)) // request full actions, various versions testGetChangesVsSpark( tablePath, 0, 2, FULL_ACTION_SET) testGetChangesVsSpark( tablePath, 1, 2, FULL_ACTION_SET) testGetChangesVsSpark( tablePath, 0, 0, FULL_ACTION_SET) } } } Seq(Some(0), Some(1), None).foreach { ictEnablementVersion => test("getChanges - returns correct timestamps " + s"[ictEnablementVersion = ${ictEnablementVersion.getOrElse("None")}]") { withTempDirAndEngine { (tempDir, engine) => def generateCommits(tablePath: String, commits: Long*): Unit = { commits.zipWithIndex.foreach { case (ts, i) => val tableProperties = if (ictEnablementVersion.contains(i)) { Map("delta.enableInCommitTimestamps" -> "true") } else { Map.empty[String, String] } val clock = new ManualClock(ts) appendData( engine, tablePath, isNewTable = i == 0, schema = if (i == 0) testSchema else null, data = immutable.Seq(Map.empty[String, Literal] -> dataBatches2), clock = clock, tableProperties = tableProperties) // Only set the file modification time if ICT has not been enabled yet. if (!ictEnablementVersion.exists(_ <= i)) { val file = new File(FileNames.deltaFile(new Path(tablePath, "_delta_log"), i)) file.setLastModified(ts) } } } val start = 1540415658000L val minuteInMilliseconds = 60000L generateCommits( tempDir, start, start + 20 * minuteInMilliseconds, start + 40 * minuteInMilliseconds) val versionToTimestamp: Map[Long, Long] = Map( 0L -> start, 1L -> (start + 20 * minuteInMilliseconds), 2L -> (start + 40 * minuteInMilliseconds)) // Check the timestamps are returned correctly getChanges(tempDir, 0, 2, Set(DeltaAction.ADD)) .flatMap(_.getRows.toSeq) .foreach { row => val version = row.getLong(0) val timestamp = row.getLong(1) assert( timestamp == versionToTimestamp(version), f"Expected timestamp ${versionToTimestamp(version)} for version $version but" + f"Kernel returned timestamp $timestamp") } // Check contents as well testGetChangesVsSpark( tempDir, 0, 2, FULL_ACTION_SET) } } } test("getChanges - empty _delta_log folder") { withTempDir { tempDir => new File(tempDir, "delta_log").mkdirs() intercept[TableNotFoundException] { getChanges(tempDir.getCanonicalPath, 0, 2, FULL_ACTION_SET) } } } test("getChanges - empty folder no _delta_log dir") { withTempDir { tempDir => intercept[TableNotFoundException] { getChanges(tempDir.getCanonicalPath, 0, 2, FULL_ACTION_SET) } } } test("getChanges - non-empty folder not a delta table") { withTempDir { tempDir => spark.range(20).write.format("parquet").mode("overwrite").save(tempDir.getCanonicalPath) intercept[TableNotFoundException] { getChanges(tempDir.getCanonicalPath, 0, 2, FULL_ACTION_SET) } } } test("getChanges - directory does not exist") { intercept[TableNotFoundException] { getChanges("/fake/table/path", 0, 2, FULL_ACTION_SET) } } test("getChanges - golden table deltalog-getChanges invalid queries") { withGoldenTable("deltalog-getChanges") { tablePath => def getChangesByVersion( startVersion: Long, endVersion: Long): Seq[ColumnarBatch] = { getChanges(tablePath, startVersion, endVersion, FULL_ACTION_SET) } // startVersion after latest available version assert(intercept[CommitRangeNotFoundException] { getChangesByVersion(3, 8) }.getMessage.contains("no log files found in the requested version range")) // endVersion larger than latest available version assert(intercept[KernelException] { getChangesByVersion(0, 8) }.getMessage.contains("no log file found for version 8")) // invalid start version assert(intercept[IllegalArgumentException] { getChangesByVersion(-1, 2) }.getMessage.contains("must be >= 0")) // invalid end version assert(intercept[IllegalArgumentException] { getChangesByVersion(2, 1) }.getMessage.contains("startVersion must be <= endVersion")) } } test("getChanges - with truncated log") { withTempDir { tempDir => // PREPARE TEST TABLE val tablePath = tempDir.getCanonicalPath // Write versions [0, 10] (inclusive) including a checkpoint (0 to 10).foreach { i => spark.range(i * 10, i * 10 + 10).write .format("delta") .mode("append") .save(tablePath) } val log = org.apache.spark.sql.delta.DeltaLog.forTable( spark, new org.apache.hadoop.fs.Path(tablePath)) val deltaCommitFileProvider = org.apache.spark.sql.delta.util.DeltaCommitFileProvider( log.unsafeVolatileSnapshot) // Delete the log files for versions 0-9, truncating the table history to version 10 (0 to 9).foreach { i => val jsonFile = deltaCommitFileProvider.deltaFile(i) new File(new org.apache.hadoop.fs.Path(log.logPath, jsonFile).toUri).delete() } // Create version 11 that overwrites the whole table spark.range(50).write .format("delta") .mode("overwrite") .save(tablePath) // Create version 12 that appends new data spark.range(10).write .format("delta") .mode("append") .save(tablePath) // TEST ERRORS // endVersion before earliest available version assert(intercept[CommitRangeNotFoundException] { getChanges(tablePath, 0, 9, FULL_ACTION_SET) }.getMessage.contains("no log files found in the requested version range")) // startVersion less than the earliest available version assert(intercept[KernelException] { getChanges(tablePath, 5, 11, FULL_ACTION_SET) }.getMessage.contains("no log file found for version 5")) // TEST VALID CASES testGetChangesVsSpark( tablePath, 10, 12, FULL_ACTION_SET) testGetChangesVsSpark( tablePath, 11, 12, FULL_ACTION_SET) } } test("getChanges - table with a lot of changes") { withTempDir { tempDir => spark.sql( f""" |CREATE TABLE delta.`${tempDir.getCanonicalPath}` (id LONG, month LONG) |USING DELTA |PARTITIONED BY (month) |TBLPROPERTIES (delta.enableChangeDataFeed = true) |""".stripMargin) spark.range(100).withColumn("month", col("id") % 12 + 1) .write .format("delta") .mode("append") .save(tempDir.getCanonicalPath) spark.sql( // cdc actions f""" |UPDATE delta.`${tempDir.getCanonicalPath}` SET month = 1 WHERE id < 10 |""".stripMargin) spark.sql( f""" |DELETE FROM delta.`${tempDir.getCanonicalPath}` WHERE month = 12 |""".stripMargin) spark.sql( f""" |DELETE FROM delta.`${tempDir.getCanonicalPath}` WHERE id = 52 |""".stripMargin) spark.range(100, 150).withColumn("month", col("id") % 12) .write .format("delta") .mode("overwrite") .save(tempDir.getCanonicalPath) spark.sql( // change metadata f""" |ALTER TABLE delta.`${tempDir.getCanonicalPath}` |ADD CONSTRAINT validMonth CHECK (month <= 12) |""".stripMargin) // Check all actions are correctly retrieved testGetChangesVsSpark( tempDir.getCanonicalPath, 0, 6, FULL_ACTION_SET) // Check some subset of actions testGetChangesVsSpark( tempDir.getCanonicalPath, 0, 6, Set(DeltaAction.ADD)) } } test("getChanges - fails when protocol is not readable by Kernel") { // Existing tests suffice to check if the protocol column is present/dropped correctly // We test our protocol checks for table features in TableFeaturesSuite // Min reader version is too high assert(intercept[UnsupportedProtocolVersionException] { // Use toSeq because we need to consume the iterator to force the exception getChanges(goldenTablePath("deltalog-invalid-protocol-version"), 0, 0, FULL_ACTION_SET) }.getMessage.contains("Unsupported Delta protocol reader version")) // We still get an error if we don't request the protocol file action assert(intercept[UnsupportedProtocolVersionException] { getChanges(goldenTablePath("deltalog-invalid-protocol-version"), 0, 0, Set(DeltaAction.ADD)) }.getMessage.contains("Unsupported Delta protocol reader version")) } withGoldenTable("commit-info-containing-arbitrary-operationParams-types") { tablePath => test("getChanges - commit info with arbitrary operationParams types") { // Check all actions are correctly retrieved testGetChangesVsSpark( tablePath, 0, 2, FULL_ACTION_SET) } } ////////////////////////////////////////////////////////////////////////////////// // Helpers to compare actions returned between Kernel and Spark ////////////////////////////////////////////////////////////////////////////////// // Standardize actions with case classes, keeping just a few fields to compare trait StandardAction case class StandardRemove( path: String, dataChange: Boolean, partitionValues: Map[String, String]) extends StandardAction case class StandardAdd( path: String, partitionValues: Map[String, String], size: Long, modificationTime: Long, dataChange: Boolean) extends StandardAction case class StandardMetadata( id: String, schemaString: String, partitionColumns: Seq[String], configuration: Map[String, String]) extends StandardAction case class StandardProtocol( minReaderVersion: Int, minWriterVersion: Int, readerFeatures: Set[String], writerFeatures: Set[String]) extends StandardAction case class StandardCommitInfo( operation: String, operationMetrics: Map[String, String]) extends StandardAction case class StandardCdc( path: String, partitionValues: Map[String, String], size: Long, tags: Map[String, String]) extends StandardAction case class StandardTxn( appId: String, version: Long, lastUpdated: Option[Long]) extends StandardAction def standardizeKernelAction(row: Row, startIdx: Int = 2): Option[StandardAction] = { val actionIdx = (startIdx until row.getSchema.length()).find(!row.isNullAt(_)).getOrElse( return None) row.getSchema.at(actionIdx).getName match { case DeltaAction.REMOVE.colName => val removeRow = row.getStruct(actionIdx) val partitionValues: Map[String, String] = { // partitionValues is nullable for removes if (removeRow.isNullAt(RemoveFile.FULL_SCHEMA.indexOf("partitionValues"))) { null } else { VectorUtils.toJavaMap[String, String]( removeRow.getMap(RemoveFile.FULL_SCHEMA.indexOf("partitionValues"))).asScala.toMap } } Some(StandardRemove( removeRow.getString(RemoveFile.FULL_SCHEMA.indexOf("path")), removeRow.getBoolean(RemoveFile.FULL_SCHEMA.indexOf("dataChange")), partitionValues)) case DeltaAction.ADD.colName => val addRow = row.getStruct(actionIdx) Some(StandardAdd( addRow.getString(AddFile.FULL_SCHEMA.indexOf("path")), VectorUtils.toJavaMap[String, String]( addRow.getMap(AddFile.FULL_SCHEMA.indexOf("partitionValues"))).asScala.toMap, addRow.getLong(AddFile.FULL_SCHEMA.indexOf("size")), addRow.getLong(AddFile.FULL_SCHEMA.indexOf("modificationTime")), addRow.getBoolean(AddFile.FULL_SCHEMA.indexOf("dataChange")))) case DeltaAction.METADATA.colName => val metadataRow = row.getStruct(actionIdx) Some(StandardMetadata( metadataRow.getString(Metadata.FULL_SCHEMA.indexOf("id")), metadataRow.getString(Metadata.FULL_SCHEMA.indexOf("schemaString")), VectorUtils.toJavaList( metadataRow.getArray(Metadata.FULL_SCHEMA.indexOf("partitionColumns"))).asScala.toSeq, VectorUtils.toJavaMap[String, String]( metadataRow.getMap(Metadata.FULL_SCHEMA.indexOf("configuration"))).asScala.toMap)) case DeltaAction.PROTOCOL.colName => val protocolRow = row.getStruct(actionIdx) val readerFeatures = if (protocolRow.isNullAt(Protocol.FULL_SCHEMA.indexOf("readerFeatures"))) { Seq() } else { VectorUtils.toJavaList( protocolRow.getArray(Protocol.FULL_SCHEMA.indexOf("readerFeatures"))).asScala } val writerFeatures = if (protocolRow.isNullAt(Protocol.FULL_SCHEMA.indexOf("writerFeatures"))) { Seq() } else { VectorUtils.toJavaList( protocolRow.getArray(Protocol.FULL_SCHEMA.indexOf("writerFeatures"))).asScala } Some(StandardProtocol( protocolRow.getInt(Protocol.FULL_SCHEMA.indexOf("minReaderVersion")), protocolRow.getInt(Protocol.FULL_SCHEMA.indexOf("minWriterVersion")), readerFeatures.toSet, writerFeatures.toSet)) case DeltaAction.COMMITINFO.colName => val commitInfoRow = row.getStruct(actionIdx) val operationIdx = CommitInfo.FULL_SCHEMA.indexOf("operation") val operationMetricsIdx = CommitInfo.FULL_SCHEMA.indexOf("operationMetrics") Some(StandardCommitInfo( if (commitInfoRow.isNullAt(operationIdx)) null else commitInfoRow.getString(operationIdx), if (commitInfoRow.isNullAt(operationMetricsIdx)) { Map.empty } else { VectorUtils.toJavaMap[String, String]( commitInfoRow.getMap(operationMetricsIdx)).asScala.toMap })) case DeltaAction.CDC.colName => val cdcRow = row.getStruct(actionIdx) val tags: Map[String, String] = { if (cdcRow.isNullAt(AddCDCFile.FULL_SCHEMA.indexOf("tags"))) { null } else { VectorUtils.toJavaMap[String, String]( cdcRow.getMap(AddCDCFile.FULL_SCHEMA.indexOf("tags"))).asScala.toMap } } Some(StandardCdc( cdcRow.getString(AddCDCFile.FULL_SCHEMA.indexOf("path")), VectorUtils.toJavaMap[String, String]( cdcRow.getMap(AddCDCFile.FULL_SCHEMA.indexOf("partitionValues"))).asScala.toMap, cdcRow.getLong(AddCDCFile.FULL_SCHEMA.indexOf("size")), tags)) case DeltaAction.TXN.colName => val txnRow = row.getStruct(actionIdx) val lastUpdated = if (txnRow.isNullAt(SetTransaction.FULL_SCHEMA.indexOf("lastUpdated"))) { None } else { Some(txnRow.getLong(SetTransaction.FULL_SCHEMA.indexOf("lastUpdated"))) } Some(StandardTxn( txnRow.getString(SetTransaction.FULL_SCHEMA.indexOf("appId")), txnRow.getLong(SetTransaction.FULL_SCHEMA.indexOf("version")), lastUpdated)) case _ => throw new RuntimeException("Encountered an action that hasn't been added as an option yet") } } def standardizeSparkAction(action: SparkAction): Option[StandardAction] = action match { case remove: SparkRemoveFile => Some(StandardRemove(remove.path, remove.dataChange, remove.partitionValues)) case add: SparkAddFile => Some(StandardAdd( add.path, add.partitionValues, add.size, add.modificationTime, add.dataChange)) case metadata: SparkMetadata => Some(StandardMetadata( metadata.id, metadata.schemaString, metadata.partitionColumns, metadata.configuration)) case protocol: SparkProtocol => Some(StandardProtocol( protocol.minReaderVersion, protocol.minWriterVersion, protocol.readerFeatures.getOrElse(Set.empty), protocol.writerFeatures.getOrElse(Set.empty))) case commitInfo: SparkCommitInfo => Some(StandardCommitInfo( commitInfo.operation, commitInfo.operationMetrics.getOrElse(Map.empty))) case cdc: SparkAddCDCFile => Some(StandardCdc(cdc.path, cdc.partitionValues, cdc.size, cdc.tags)) case txn: SparkSetTransaction => Some(StandardTxn(txn.appId, txn.version, txn.lastUpdated)) case _ => None } /** * When we query the Spark actions using DeltaLog::getChanges ALL action types are returned. Since * Kernel only returns actions in the provided `actionSet` this FX prunes the Spark actions to * match `actionSet`. */ def pruneSparkActionsByActionSet( sparkActions: Iterator[(Long, Seq[SparkAction])], actionSet: Set[DeltaAction]): Iterator[(Long, Seq[SparkAction])] = { sparkActions.map { case (version, actions) => ( version, actions.filter { case _: SparkRemoveFile => actionSet.contains(DeltaAction.REMOVE) case _: SparkAddFile => actionSet.contains(DeltaAction.ADD) case _: SparkMetadata => actionSet.contains(DeltaAction.METADATA) case _: SparkProtocol => actionSet.contains(DeltaAction.PROTOCOL) case _: SparkCommitInfo => actionSet.contains(DeltaAction.COMMITINFO) case _: SparkAddCDCFile => actionSet.contains(DeltaAction.CDC) case _: SparkSetTransaction => actionSet.contains(DeltaAction.TXN) case _ => false }) } } /** * Compare actions from CommitActions objects directly with Spark actions. * This automatically extracts version from each commit and standardizes the batches. */ def compareCommitActions( commits: Seq[io.delta.kernel.CommitActions], sparkActions: Iterator[(Long, Seq[SparkAction])]): Unit = { // Directly convert CommitActions to StandardActions without adding columns val standardKernelActions: Seq[(Long, StandardAction)] = commits.flatMap { commit => val version = commit.getVersion commit.getActions.toSeq.flatMap { batch => batch.getRows.toSeq .map(row => (version, standardizeKernelAction(row, startIdx = 0))) .filter(_._2.nonEmpty) .map(t => (t._1, t._2.get)) } } val standardSparkActions: Seq[(Long, StandardAction)] = sparkActions.flatMap { case (version, actions) => actions.map(standardizeSparkAction(_)).flatten.map((version, _)) }.toSeq assert( standardKernelActions.sameElements(standardSparkActions), s"Kernel actions did not match Spark actions.\n" + s"Kernel actions: ${standardKernelActions.take(5)}\n" + s"Spark actions: ${standardSparkActions.take(5)}") } def compareActions( kernelActions: Seq[ColumnarBatch], sparkActions: Iterator[(Long, Seq[SparkAction])]): Unit = { val standardKernelActions: Seq[(Long, StandardAction)] = { kernelActions.flatMap(_.getRows.toSeq) .map(row => (row.getLong(0), standardizeKernelAction(row))) .filter(_._2.nonEmpty) .map(t => (t._1, t._2.get)) } val standardSparkActions: Seq[(Long, StandardAction)] = sparkActions.flatMap { case (version, actions) => actions.map(standardizeSparkAction(_)).flatten.map((version, _)) }.toSeq assert( standardKernelActions sameElements standardSparkActions, f"Kernel actions did not match Spark actions.\n" + f"Kernel actions: $standardKernelActions\n" + f"Spark actions: $standardSparkActions") } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/TablePropertiesSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import scala.collection.JavaConverters._ import io.delta.kernel.{Table, TableManager} import io.delta.kernel.defaults.utils.{AbstractWriteUtils, WriteUtils, WriteUtilsWithV2Builders} import io.delta.kernel.exceptions.{KernelException, UnknownConfigurationException} import io.delta.kernel.utils.CloseableIterable.emptyIterable import org.scalatest.funsuite.AnyFunSuite class TablePropertiesTransactionBuilderV1Suite extends TablePropertiesSuiteBase with WriteUtils {} class TablePropertiesTransactionBuilderV2Suite extends TablePropertiesSuiteBase with WriteUtilsWithV2Builders { test("create table (V2 only) - withTableProperties can be called multiple times") { withTempDir { tempFile => val tablePath = tempFile.getAbsolutePath TableManager .buildCreateTableTransaction(tablePath, testSchema, "engineInfo") .withTableProperties(Map("key1" -> "value1").asJava) .withTableProperties(Map("key2" -> "value2").asJava) .build(defaultEngine) .commit(defaultEngine, emptyIterable()) assertHasProp(tablePath, Map("key1" -> "value1", "key2" -> "value2")) } } test("create table (V2 only) - withTableProperties throws on same key with different value") { withTempDir { tempFile => val tablePath = tempFile.getAbsolutePath val createBuilder = TableManager .buildCreateTableTransaction(tablePath, testSchema, "engineInfo") .withTableProperties(Map("key1" -> "value1", "key2" -> "value2").asJava) val ex = intercept[IllegalArgumentException] { createBuilder.withTableProperties(Map("key2" -> "different_value").asJava) } assert(ex.getMessage.contains("Table property 'key2' has already been set")) } } test("create table (V2 only) - withTableProperties allows setting same key with same value") { withTempDir { tempFile => val tablePath = tempFile.getAbsolutePath TableManager .buildCreateTableTransaction(tablePath, testSchema, "engineInfo") .withTableProperties(Map("key1" -> "value1", "key2" -> "value2").asJava) .withTableProperties(Map("key2" -> "value2").asJava) // Same value, should not throw .build(defaultEngine) .commit(defaultEngine, emptyIterable()) assertHasProp(tablePath, Map("key1" -> "value1", "key2" -> "value2")) } } } /** * Suite to set or get table properties. */ trait TablePropertiesSuiteBase extends AnyFunSuite with AbstractWriteUtils { test("create/update/replace table - allow arbitrary properties") { withTempDir { tempFile => val tablePath = tempFile.getAbsolutePath // create table with arbitrary properties and check if they are set createUpdateTableWithProps( tablePath, createTable = true, propsAdded = Map("my key" -> "10", "my key2" -> "20")) assertHasProp(tablePath, expProps = Map("my key" -> "10", "my key2" -> "20")) // update table by modifying the arbitrary properties and check if they are updated createUpdateTableWithProps(tablePath, propsAdded = Map("my key" -> "30")) assertHasProp(tablePath, expProps = Map("my key" -> "30", "my key2" -> "20")) // update table without any new properties and check if the existing properties are retained createUpdateTableWithProps(tablePath) assertHasProp(tablePath, expProps = Map("my key" -> "30", "my key2" -> "20")) // update table by adding new arbitrary properties and check if they are set createUpdateTableWithProps(tablePath, propsAdded = Map("new key3" -> "str")) assertHasProp( tablePath, expProps = Map("my key" -> "30", "my key2" -> "20", "new key3" -> "str")) // replace table and set new arbitrary properties and check if they are set (and old ones are // removed) getReplaceTxn( defaultEngine, tablePath, testSchema, tableProperties = Map("my key" -> "40", "my replace key" -> "0")) .commit(defaultEngine, emptyIterable()) assertHasProp( tablePath, expProps = Map("my key" -> "40", "my replace key" -> "0")) assertPropsDNE(tablePath, Set("my key2", "my key3")) } } test("create/update/replace table - disallow unknown delta.* properties to Kernel") { withTempDir { tempFile => val tablePath = tempFile.getAbsolutePath val ex1 = intercept[UnknownConfigurationException] { createUpdateTableWithProps(tablePath, createTable = true, Map("delta.unknown" -> "str")) } assert(ex1.getMessage.contains("Unknown configuration was specified: delta.unknown")) // Try updating in an existing table createUpdateTableWithProps(tablePath, createTable = true) val ex2 = intercept[UnknownConfigurationException] { createUpdateTableWithProps(tablePath, propsAdded = Map("Delta.unknown" -> "str")) } assert(ex2.getMessage.contains("Unknown configuration was specified: Delta.unknown")) // Try replacing an existing table val ex3 = intercept[UnknownConfigurationException] { getReplaceTxn( defaultEngine, tablePath, testSchema, tableProperties = Map("Delta.unknown" -> "str")) } assert(ex3.getMessage.contains("Unknown configuration was specified: Delta.unknown")) } } test("create/update/replace table - delta configs are stored with same case as " + "defined in TableConfig") { withTempDir { tempFile => val tablePath = tempFile.getAbsolutePath createUpdateTableWithProps( tablePath, createTable = true, Map("delta.CHECKPOINTINTERVAL" -> "20")) assertHasProp(tablePath, expProps = Map("delta.checkpointInterval" -> "20")) // Try updating in an existing table createUpdateTableWithProps( tablePath, propsAdded = Map("DELTA.CHECKPOINTINTERVAL" -> "30")) assertHasProp(tablePath, expProps = Map("delta.checkpointInterval" -> "30")) // Try replacing an existing table getReplaceTxn( defaultEngine, tablePath, testSchema, tableProperties = Map("DELTA.CHECKPOINTINTERVAL" -> "30")) .commit(defaultEngine, emptyIterable()) assertHasProp(tablePath, expProps = Map("delta.checkpointInterval" -> "30")) } } test("Case is preserved for user properties and is case sensitive") { // This aligns with Spark's behavior withTempDir { tempFile => val tablePath = tempFile.getAbsolutePath createUpdateTableWithProps( tablePath, createTable = true, Map("user.facing.PROP" -> "20")) assertHasProp(tablePath, expProps = Map("user.facing.PROP" -> "20")) // Try updating in an existing table createUpdateTableWithProps( tablePath, propsAdded = Map("user.facing.prop" -> "30")) assertHasProp( tablePath, expProps = Map("user.facing.PROP" -> "20", "user.facing.prop" -> "30")) // Try replacing an existing table getReplaceTxn( defaultEngine, tablePath, testSchema, tableProperties = Map("user.facing.prop" -> "30", "user.facing.PROP" -> "20")) .commit(defaultEngine, emptyIterable()) assertHasProp( tablePath, expProps = Map("user.facing.PROP" -> "20", "user.facing.prop" -> "30")) } } test("Cannot unset delta table properties") { withTempDir { tablePath => // Create empty table with delta props createUpdateTableWithProps( tablePath.getAbsolutePath, createTable = true, propsAdded = Map("delta.checkpointInterval" -> "10")) Seq("delta.checkpointInterval", "DELTA.checkpointInterval").foreach { key => val e = intercept[IllegalArgumentException] { createUpdateTableWithProps( tablePath.getAbsolutePath, propsRemoved = Set(key)) } assert( e.getMessage.contains("Unsetting 'delta.' table properties is currently unsupported")) } } } test("Cannot set and unset the same table property in same txn - new property") { withTempDir { tempFile => val tablePath = tempFile.getAbsolutePath createEmptyTable(tablePath = tablePath, schema = testSchema) val e = intercept[KernelException] { createUpdateTableWithProps( tablePath, propsAdded = Map("foo.key" -> "value"), propsRemoved = Set("foo.key")) } assert(e.getMessage.contains( "Cannot set and unset the same table property in the same transaction. " + "Properties set and unset: [foo.key]")) } } test("Cannot set and unset the same table property in same txn - existing property") { // i.e. we don't only check against the new properties withTempDir { tempFile => val tablePath = tempFile.getAbsolutePath // Create initial table with the property createEmptyTable( tablePath = tablePath, schema = testSchema, tableProperties = Map("foo.key" -> "value")) // Try to set and unset the existing property val e = intercept[KernelException] { createUpdateTableWithProps( tablePath, propsAdded = Map("foo.key" -> "value"), propsRemoved = Set("foo.key")) } assert(e.getMessage.contains( "Cannot set and unset the same table property in the same transaction. " + "Properties set and unset: [foo.key]")) } } test("Unset valid cases - properties are removed from the table") { withTempDir { tempFile => val tablePath = tempFile.getAbsolutePath // Create initial table with properties // This test also validates the operation is case-sensitive createEmptyTable( tablePath = tablePath, schema = testSchema, tableProperties = Map("foo.key" -> "value", "FOO.KEY" -> "VALUE")) assertHasProp(tablePath, Map("foo.key" -> "value", "FOO.KEY" -> "VALUE")) // Remove 1 of the properties set createUpdateTableWithProps( tablePath, propsRemoved = Set("foo.key")) assertPropsDNE(tablePath, Set("foo.key")) // Check that the other property is not touched assertHasProp(tablePath, Map("FOO.KEY" -> "VALUE")) // Can unset a property that DNE createUpdateTableWithProps( tablePath, propsRemoved = Set("not.a.key")) assertPropsDNE(tablePath, Set("not.a.key")) // Check that the other property is not touched assertHasProp(tablePath, Map("FOO.KEY" -> "VALUE")) // Can be simultaneous with setTblProps as long as no overlap createUpdateTableWithProps( tablePath, propsAdded = Map("foo.key" -> "value-new"), propsRemoved = Set("FOO.KEY")) assertPropsDNE(tablePath, Set("FOO.KEY")) // Check that the other property is added successfully assertHasProp(tablePath, Map("foo.key" -> "value-new")) } } def createUpdateTableWithProps( tablePath: String, createTable: Boolean = false, propsAdded: Map[String, String] = null, propsRemoved: Set[String] = null): Unit = { val txn = if (createTable) { getCreateTxn( defaultEngine, tablePath, testSchema, tableProperties = propsAdded) } else { getUpdateTxn( defaultEngine, tablePath, tableProperties = propsAdded, tablePropertiesRemoved = propsRemoved) } txn.commit(defaultEngine, emptyIterable()) } def assertHasProp(tablePath: String, expProps: Map[String, String]): Unit = { val snapshot = Table.forPath(defaultEngine, tablePath) .getLatestSnapshot(defaultEngine) expProps.foreach { case (key, value) => assert(snapshot.getTableProperties.get(key) === value, key) } } def assertPropsDNE(tablePath: String, keys: Set[String]): Unit = { val metadata = getMetadata(defaultEngine, tablePath) assert(keys.forall(!metadata.getConfiguration.containsKey(_))) } val recognizedButUnimplementedProps = Seq( ("delta.dataSkippingStatsColumns", "col1,col2,nested.field")) recognizedButUnimplementedProps.foreach { case (propKey, value) => test(s"$propKey is allowed (but not implemented) - create table") { withTempDir { tempFile => val tablePath = tempFile.getAbsolutePath createUpdateTableWithProps( tablePath, createTable = true, propsAdded = Map(propKey -> value)) assertHasProp(tablePath, Map(propKey -> value)) } } test(s"$propKey is allowed (but not implemented) - update table") { withTempDir { tempFile => val tablePath = tempFile.getAbsolutePath createUpdateTableWithProps(tablePath, createTable = true) val updatedValue = s"${value}_updated" createUpdateTableWithProps( tablePath, propsAdded = Map(propKey -> updatedValue)) assertHasProp(tablePath, Map(propKey -> updatedValue)) } } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/TimestampStatsAndDataSkippingSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.io.File import java.time.{LocalDateTime, ZoneOffset} import scala.collection.immutable.Seq import scala.jdk.CollectionConverters._ import io.delta.kernel.data.{ColumnarBatch, ColumnVector} import io.delta.kernel.defaults.internal.data.DefaultColumnarBatch import io.delta.kernel.defaults.internal.data.vector.DefaultGenericVector import io.delta.kernel.defaults.internal.parquet.ParquetSuiteBase import io.delta.kernel.defaults.utils.WriteUtils import io.delta.kernel.expressions.Literal import io.delta.kernel.internal.util.JsonUtils import io.delta.kernel.types.{StructType, TimestampNTZType, TimestampType} import org.apache.spark.sql.delta.DeltaLog import org.apache.hadoop.fs.Path import org.scalatest.funsuite.AnyFunSuite /** * Tests timestamp statistics serialization and data skipping behavior * for TIMESTAMP and TIMESTAMP_NTZ types. */ class TimestampStatsAndDataSkippingSuite extends AnyFunSuite with WriteUtils with DataSkippingDeltaTestsUtils with ParquetSuiteBase { test("verify on-disk TIMESTAMP stats format is equal when writing through spark and kernel") { withTempDirAndEngine { (dir, engine) => // Test with TIMESTAMP and TIMESTAMP_NTZ to verify serialization format val schema = new StructType() .add("timestampCol", TimestampType.TIMESTAMP) .add("timestampNtzCol", TimestampNTZType.TIMESTAMP_NTZ) // Create different batches with different timestamp ranges to test multiple boundaries val timestampRanges = Seq( ("2019-06-09 01:02:04.123456", "2019-09-09 01:02:04.123999"), ("2019-06-09 01:02:04.123456", "2019-09-09 01:02:05.123999"), // Microsecond boundary ("2019-09-09 01:02:03.456789", "2019-09-09 01:02:03.456999"), // End-of-millisecond boundary ("2019-09-09 01:02:03.999000", "2019-09-09 01:02:03.999999"), ("2019-09-09 01:02:04.123456", "2019-09-09 01:02:04.123999")) // Create "kernel" and "spark-copy" directories val kernelPath = new File(dir, "kernel").getAbsolutePath val sparkTablePath = new File(dir, "spark-copy").getAbsolutePath // Write through Kernel timestampRanges.zipWithIndex.foreach { case ((minTs, maxTs), fileIndex) => val batch = createTimestampBatch(schema, minTs, maxTs, rowsPerFile = 10) appendData( engine, kernelPath, isNewTable = fileIndex == 0, schema = if (fileIndex == 0) schema else null, partCols = Seq.empty, data = Seq(Map.empty[String, Literal] -> Seq(batch.toFiltered(Option.empty)))) } val kernelDf = spark.read.format("delta").load(kernelPath) val kernelFiles = kernelDf.inputFiles log.info(s"Found ${kernelFiles.length} files from Kernel") withSparkTimeZone("UTC") { kernelFiles.zipWithIndex.foreach { case (filePath, fileIndex) => val singleFileDf = spark.read.parquet(filePath) if (fileIndex == 0) { singleFileDf.write.format("delta").mode("overwrite").save(sparkTablePath) } else { singleFileDf.write.format("delta").mode("append").save(sparkTablePath) } } } val mapper = JsonUtils.mapper() val kernelStats = collectStatsFromAddFiles(engine, kernelPath).map(mapper.readTree) val sparkStats = collectStatsFromAddFiles(engine, sparkTablePath).map(mapper.readTree) require( kernelStats.nonEmpty && sparkStats.nonEmpty, "stats collected from AddFiles should be non-empty") assert( kernelStats.toSet == sparkStats.toSet, s"\nKernel stats:\n${kernelStats.mkString("\n")}\n" + s"Spark stats:\n${sparkStats.mkString("\n")}") } } ////////////////////////////////////////////////////////////////////////////////// // Timestamp Data Skipping Tests that mirror those of Delta-Spark ////////////////////////////////////////////////////////////////////////////////// for (timestampType <- Seq(TimestampType.TIMESTAMP, TimestampNTZType.TIMESTAMP_NTZ)) { test(s"validate basic delta-spark data-skipping on ${timestampType.getClass.getSimpleName}") { withTempDirAndEngine { (kernelPath, engine) => val schema = new StructType().add("ts", timestampType) // Generate multiple files with different timestamp ranges. val timestampRanges = Seq( ("2019-01-01 12:00:00.123456", "2019-01-01 18:00:00.999999"), ("2019-09-09 01:02:03.456789", "2019-09-09 01:02:03.456789"), ("2019-12-31 20:00:00.100000", "2019-12-31 23:59:59.999999"), ("2020-06-15 10:30:45.555555", "2020-06-15 15:45:30.888888"), ("2021-03-20 08:15:22.777777", "2021-03-20 16:42:18.333333")) timestampRanges.zipWithIndex.foreach { case ((minTs, maxTs), fileIndex) => val batch = createTimestampBatch(schema, minTs, maxTs, rowsPerFile = 10) appendData( engine, kernelPath, isNewTable = fileIndex == 0, schema = if (fileIndex == 0) schema else null, partCols = Seq.empty, data = Seq(Map.empty[String, Literal] -> Seq(batch.toFiltered(Option.empty)))) } // Query with all predicates in UTC for this test. // We mainly want to validate that the scan results are correct. withSparkTimeZone("UTC") { val deltaLogPath = DeltaLog.forTable(spark, new Path(kernelPath)) val exactHits = Seq( // Files 1,2,3,4 (s"ts >= ${createTimestampLiteral(timestampType, "2019-09-09 01:02:03.456789")}", 4), // Files 1,2,3,4 (millisecond expansion) (s"ts <= ${createTimestampLiteral(timestampType, "2019-09-09 01:02:03.456789")}", 2), // Files 1,2,3,4 (tests millisecond expansion in MAX due to truncation) (s"ts >= ${createTimestampLiteral(timestampType, "2019-09-09 01:02:03.456790")}", 4), // File 1 only (s"ts = ${createTimestampLiteral(timestampType, "2019-09-09 01:02:03.456789")}", 1)) exactHits.foreach { case (predicate, expectedFiles) => val filesHit = filesReadCount(spark, deltaLogPath, predicate) assert( filesHit == expectedFiles, s"Expected exactly $expectedFiles files for: $predicate, but got $filesHit files") } // Test range queries (should hit multiple files) val rangeHits = Seq( (s"ts >= ${createTimestampLiteral(timestampType, "2019-01-01 00:00:00")}", 5), (s"ts >= ${createTimestampLiteral(timestampType, "2020-01-01 00:00:00")}", 3), (s"ts >= ${createTimestampLiteral(timestampType, "2019-06-01 00:00:00")}", 4), (s"ts <= ${createTimestampLiteral(timestampType, "2019-06-01 00:00:00")}", 1), ( s"ts BETWEEN ${createTimestampLiteral(timestampType, "2019-09-01 00:00:00")} " + s"AND ${createTimestampLiteral(timestampType, "2019-09-30 23:59:59")}", 1), ( s"ts BETWEEN ${createTimestampLiteral(timestampType, "2019-01-01 00:00:00")} " + s"AND ${createTimestampLiteral(timestampType, "2019-12-31 23:59:59")}", 3)) rangeHits.foreach { case (predicate, expectedFiles) => val filesHit = filesReadCount(spark, deltaLogPath, predicate) assert( filesHit == expectedFiles, s"Expected exactly $expectedFiles files for: $predicate, but got $filesHit files") } // Test precise misses (outside the millisecond range due to truncation ) val preciseCases = Seq( // Files 1, 2, 3. Next millisecond // ${createTimestampLiteral(timestampType, "2019-01-01 00:00:00")} (s"ts > ${createTimestampLiteral(timestampType, "2019-09-09 01:02:03.457000")}", 3), (s"ts < ${createTimestampLiteral(timestampType, "2019-09-09 01:02:03.456000")}", 1), // Year boundary (s"ts >= ${createTimestampLiteral(timestampType, "2020-01-01 00:00:00.000001")}", 2), // True misses (0 files) (s"ts >= ${createTimestampLiteral(timestampType, "2022-01-01 00:00:00")}", 0), (s"ts <= ${createTimestampLiteral(timestampType, "2018-01-01 00:00:00")}", 0)) preciseCases.foreach { case (predicate, expectedFiles) => val filesHit = filesReadCount(spark, deltaLogPath, predicate) assert( filesHit == expectedFiles, s"Expected exactly $expectedFiles files for: $predicate, but got $filesHit files") } // Test that we have the expected total number of files val totalFiles = filesReadCount(spark, deltaLogPath, "TRUE") assert(totalFiles == 5, s"Expected 5 total files, but got $totalFiles") } } } } // Test timezone boundary behavior with a single timestamp across multiple timezones test(s"kernel data skipping timezone boundary behavior for TIMESTAMP type") { withTempDirAndEngine { (kernelPath, engine) => val timestampType = TimestampType.TIMESTAMP val schema = new StructType().add("ts", timestampType) val testTimestamp = "2019-09-09 01:02:03.456789" val batch = createTimestampBatch(schema, testTimestamp, testTimestamp, rowsPerFile = 1) appendData( engine, kernelPath, isNewTable = true, schema, partCols = Seq.empty, data = Seq(Map.empty[String, Literal] -> Seq(batch.toFiltered(Option.empty)))) val sparkLog = DeltaLog.forTable(spark, new Path(kernelPath)) // Test UTC timezone-aware queries val utcHits = Seq( s"ts >= ${createTimestampLiteral(timestampType, "2019-09-09 01:02:03.456789+00:00")}", s"ts <= ${createTimestampLiteral(timestampType, "2019-09-09 01:02:03.456789+00:00")}", s"ts >= ${createTimestampLiteral(timestampType, "2019-09-09 01:02:03.456789 UTC")}", s"TS >= ${createTimestampLiteral(timestampType, "2019-09-09 01:02:03.456789+00:00")}") val utcMisses = Seq( s"ts >= ${createTimestampLiteral(timestampType, "2019-09-09 01:02:03.457001+00:00")}", s"ts <= ${createTimestampLiteral(timestampType, "2019-09-04 01:02:03.455999+00:00")}", s"TS >= ${createTimestampLiteral(timestampType, "2019-09-09 01:02:03.457001 UTC")}") // Test PST timezone-aware queries val pstHits = Seq( s"ts >= ${createTimestampLiteral(timestampType, "2019-09-08 17:02:03.456789-08:00")}", s"ts <= ${createTimestampLiteral(timestampType, "2019-09-08 17:02:03.456789-08:00")}", s"ts >= ${createTimestampLiteral(timestampType, "2019-09-08 17:02:03.456789 PST")}") val pstMisses = Seq( s"ts >= ${createTimestampLiteral(timestampType, "2019-09-08 17:02:03.457001-08:00")}", s"ts <= ${createTimestampLiteral(timestampType, "2019-09-08 17:02:03.455999-08:00")}") utcHits.foreach { predicate => val filesHit = filesReadCount(spark, sparkLog, predicate) assert(filesHit == 1, s"Expected UTC hit but got miss for $predicate") } utcMisses.foreach { predicate => val filesHit = filesReadCount(spark, sparkLog, predicate) assert(filesHit == 0, s"Expected UTC miss but got hit for $predicate") } pstHits.foreach { predicate => val filesHit = filesReadCount(spark, sparkLog, predicate) assert(filesHit == 1, s"Expected PST hit but got miss for $predicate") } pstMisses.foreach { predicate => val filesHit = filesReadCount(spark, sparkLog, predicate) assert(filesHit == 0, s"Expected PST miss but got hit for $predicate") } } } private def createTimestampBatch( schema: StructType, minTimestampStr: String, maxTimestampStr: String, rowsPerFile: Int): ColumnarBatch = { val minMicros = parseTimestampToMicros(minTimestampStr) val maxMicros = parseTimestampToMicros(maxTimestampStr) val timestampValues = (0 until rowsPerFile).map { rowIndex => if (rowIndex != 0 && rowIndex % 4 == 0) { null } else { val fraction = if (rowsPerFile == 1) 0.0 else rowIndex.toDouble / (rowsPerFile - 1) val interpolatedMicros = minMicros + ((maxMicros - minMicros) * fraction).toLong interpolatedMicros } }.toArray.asInstanceOf[Array[AnyRef]] val vectors = schema.fields().asScala.toSeq.map { field => DefaultGenericVector.fromArray(field.getDataType, timestampValues.toSeq.toArray) }.toArray.asInstanceOf[Array[ColumnVector]] new DefaultColumnarBatch(rowsPerFile, schema, vectors) } /** * Parse timestamp string to microseconds since epoch */ private def parseTimestampToMicros(timestampStr: String): Long = { // Parse "2019-09-09 01:02:03.456789" format val localDateTime = LocalDateTime.parse(timestampStr.replace(" ", "T")) val instant = localDateTime.toInstant(ZoneOffset.UTC) instant.getEpochSecond * 1000000L + instant.getNano / 1000L } // Create type-appropriate literal function def createTimestampLiteral( timestampType: io.delta.kernel.types.DataType, timestamp: String): String = { timestampType match { case _: TimestampType => s"TIMESTAMP'$timestamp'" case _: TimestampNTZType => s"TIMESTAMP_NTZ'$timestamp'" case _ => throw new IllegalArgumentException(s"Unsupported type: $timestampType") } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/TransactionCommitLoopSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults import java.nio.file.FileAlreadyExistsException import scala.collection.immutable.Seq import io.delta.kernel.Table import io.delta.kernel.data.Row import io.delta.kernel.defaults.engine.{DefaultEngine, DefaultJsonHandler} import io.delta.kernel.defaults.engine.hadoopio.HadoopFileIO import io.delta.kernel.defaults.utils.{AbstractWriteUtils, WriteUtilsWithV1Builders, WriteUtilsWithV2Builders} import io.delta.kernel.engine.JsonHandler import io.delta.kernel.exceptions.{CommitStateUnknownException, MaxCommitRetryLimitReachedException} import io.delta.kernel.expressions.Literal import io.delta.kernel.utils.CloseableIterable.emptyIterable import io.delta.kernel.utils.CloseableIterator import org.apache.hadoop.conf.Configuration import org.scalatest.funsuite.AnyFunSuite class TransactionCommitLoopTransactionBuilderV1Suite extends AbstractTransactionCommitLoopSuite with WriteUtilsWithV1Builders {} class TransactionCommitLoopTransactionBuilderV2Suite extends AbstractTransactionCommitLoopSuite with WriteUtilsWithV2Builders {} trait AbstractTransactionCommitLoopSuite extends AnyFunSuite { self: AbstractWriteUtils => private val fileIO = new HadoopFileIO(new Configuration()) test("Txn attempts to commit *next* version on CFE(isRetryable=true, isConflict=true)") { withTempDirAndEngine { (tablePath, engine) => val initialTxn = getCreateTxn(engine, tablePath, testSchema) commitTransaction(initialTxn, engine, emptyIterable()) // 000.json val kernelTxn = getUpdateTxn(engine, tablePath, maxRetries = 5) // Create 001.json. This will make the engine throw a FileAlreadyExistsException when trying // to write 001.json. The default committer will turn this into a // CFE(isRetryable=true, isConflict=true). appendData(engine, tablePath, data = Seq(Map.empty[String, Literal] -> dataBatches1)) val result = commitTransaction(kernelTxn, engine, emptyIterable()) assert(result.getVersion == 2) assert(result.getTransactionReport.getTransactionMetrics.getNumCommitAttempts == 2) } } test("Txn attempts to commit *same* version on CFE(isRetryable=true, isConflict=false)") { withTempDirAndEngine { (tablePath, engine) => val initialTxn = getCreateTxn(engine, tablePath, testSchema) commitTransaction(initialTxn, engine, emptyIterable()) // 000.json var attemptCount = 0 // Will be incremented when actual writeJson attempt occurs val attemptNumberToSucceedAt = 5 val attemptedFilePaths = scala.collection.mutable.Set[String]() class CustomJsonHandler extends DefaultJsonHandler(fileIO) { override def writeJsonFileAtomically( filePath: String, data: CloseableIterator[Row], overwrite: Boolean): Unit = { attemptCount += 1 attemptedFilePaths += filePath if (attemptCount < attemptNumberToSucceedAt) { // The default committer will turn this into a CFE(isRetryable=true, isConflict=false) throw new java.io.IOException("Transient network error") } super.writeJsonFileAtomically(filePath, data, overwrite) } } class CustomEngine extends DefaultEngine(fileIO) { val jsonHandler = new CustomJsonHandler() override def getJsonHandler: JsonHandler = jsonHandler } val transientErrorEngine = new CustomEngine() val txn = getUpdateTxn(transientErrorEngine, tablePath) val result = commitTransaction(txn, transientErrorEngine, emptyIterable()) assert(result.getVersion == 1) assert(attemptCount == attemptNumberToSucceedAt) assert(attemptedFilePaths.size == 1) // we should only be attempting to write 001.json assert(result.getTransactionReport.getTransactionMetrics.getNumCommitAttempts == attemptNumberToSucceedAt) } } test("Txn throws MaxCommitRetryLimitReachedException on too many retries") { withTempDirAndEngine { (tablePath, engine) => val initialTxn = getCreateTxn(engine, tablePath, testSchema) commitTransaction(initialTxn, engine, emptyIterable()) // 000.json class CustomJsonHandler extends DefaultJsonHandler(fileIO) { override def writeJsonFileAtomically( filePath: String, data: CloseableIterator[Row], overwrite: Boolean): Unit = { // The default committer will turn this into a CFE(isRetryable=true, isConflict=false) throw new java.io.IOException("Transient network error") } } class AlwaysFailingEngine extends DefaultEngine(fileIO) { val jsonHandler = new CustomJsonHandler() override def getJsonHandler: JsonHandler = jsonHandler } val alwaysFailingEngine = new AlwaysFailingEngine() val txn = getUpdateTxn(alwaysFailingEngine, tablePath, maxRetries = 10) val exMsg = intercept[MaxCommitRetryLimitReachedException] { commitTransaction(txn, alwaysFailingEngine, emptyIterable()) }.getMessage assert(exMsg.contains("Commit attempt for version 1 failed with a retryable exception but " + "will not be retried because the maximum number of retries (10) has been reached.")) } } test("Txn throws CommitStateUnknownException if it sees CFE(true,false) then CFE(true,true)") { withTempDirAndEngine { (tablePath, engine) => val initialTxn = getCreateTxn(engine, tablePath, testSchema) commitTransaction(initialTxn, engine, emptyIterable()) // 000.json // This tests the case of: // - first commit attempt: We succeed at writing 001.json, BUT a transient network error // occurs, so Kernel txn sees a failure. // - second commit attempt: We try again to write 001.json, but we see that it already exists! // For now, we just throw, but in the future we could try detecting if that 001.json was // written by us on the previous attempt, or written by another writer. class CustomJsonHandler extends DefaultJsonHandler(fileIO) { var attemptCount = 0 // Will be incremented when actual writeJson attempt occurs override def writeJsonFileAtomically( filePath: String, data: CloseableIterator[Row], overwrite: Boolean): Unit = { attemptCount += 1 if (attemptCount == 1) { // The default committer will turn this into a CFE(isRetryable=true, isConflict=false) throw new java.io.IOException("Transient network error") } else { // The default committer will turn this into a CFE(isRetryable=true, isConflict=true) throw new FileAlreadyExistsException("001.json already exists") } } } class CustomEngine extends DefaultEngine(fileIO) { private val jsonHandler = new CustomJsonHandler() override def getJsonHandler: JsonHandler = jsonHandler } val transientErrorEngine = new CustomEngine() val txn = getUpdateTxn(transientErrorEngine, tablePath) val exMsg = intercept[CommitStateUnknownException] { commitTransaction(txn, transientErrorEngine, emptyIterable()) }.getMessage assert(exMsg.contains("Commit attempt 2 for version 1 failed due to a concurrent write " + "conflict after a previous retry.")) } } // TODO: Transaction will fail on CFE(isRetryable=false, isConflict=true/false). The default // committer doesn't throw this error type. We could test this with a custom committer, but // currently our API to create transactions just use Table::getLatestSnapshot(), and is not // yet properly connected to the SnapshotBuilder.withCommitter code. test("Txn will *not* retry on non-IOException RuntimeException") { withTempDirAndEngine { (tablePath, engine) => val initialTxn = getCreateTxn(engine, tablePath, testSchema) commitTransaction(initialTxn, engine, emptyIterable()) // 000.json class CustomJsonHandler extends DefaultJsonHandler(fileIO) { override def writeJsonFileAtomically( filePath: String, data: CloseableIterator[Row], overwrite: Boolean): Unit = { // The default committer doesn't explicitly turn this into a CFE throw new RuntimeException("Non-retryable error") } } class CustomEngine extends DefaultEngine(fileIO) { val jsonHandler = new CustomJsonHandler() override def getJsonHandler: JsonHandler = jsonHandler } val alwaysFailingEngine = new CustomEngine() val txn = getUpdateTxn(alwaysFailingEngine, tablePath) val ex = intercept[RuntimeException] { commitTransaction(txn, alwaysFailingEngine, emptyIterable()) } assert(ex.getMessage.contains("Non-retryable error")) } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/catalogManaged/CatalogManagedE2EReadSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.catalogManaged import scala.collection.JavaConverters._ import io.delta.kernel.{SnapshotBuilder, TableManager} import io.delta.kernel.CommitRangeBuilder.CommitBoundary import io.delta.kernel.defaults.engine.hadoopio.HadoopFileIO import io.delta.kernel.defaults.utils.{TestRow, TestUtilsWithTableManagerAPIs, WriteUtilsWithV2Builders} import io.delta.kernel.exceptions.KernelException import io.delta.kernel.internal.DeltaHistoryManager import io.delta.kernel.internal.commitrange.CommitRangeImpl import io.delta.kernel.internal.files.{ParsedCatalogCommitData, ParsedLogData} import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.table.SnapshotBuilderImpl import io.delta.kernel.internal.tablefeatures.TableFeatures.{isCatalogManagedSupported, CATALOG_MANAGED_RW_FEATURE, IN_COMMIT_TIMESTAMP_W_FEATURE, TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION} import io.delta.kernel.internal.util.FileNames import io.delta.kernel.utils.FileStatus import org.apache.hadoop.conf.Configuration import org.scalatest.funsuite.AnyFunSuite /** * Test suite for end-to-end reads of catalog-managed Delta tables. * * The goal of this suite is to simulate how a real "Catalog-Managed-Client" would read a * catalog-managed Delta table, without introducing a full, or even partial (e.g. in-memory) * catalog client implementation. * * The catalog boundary is simulated by tests manually providing [[ParsedLogData]]. For example, * there can be X commits in the _staged_commits directory, and a given test can decide that Y * commits (subset of X) are in fact "ratified". The test can then turn those commits into * [[ParsedLogData]] and inject them into the [[SnapshotBuilder]]. This is, * in essence, doing exactly what we would expect a "Catalog-Managed-Client" to do. */ class CatalogManagedE2EReadSuite extends AnyFunSuite with TestUtilsWithTableManagerAPIs with WriteUtilsWithV2Builders { def withCatalogOwnedPreviewTestTable(testFx: (String, List[ParsedLogData]) => Unit): Unit = { val tablePath = getTestResourceFilePath("catalog-owned-preview") // Note: We need to *resolve* each test resource file path, because the table root file path // will itself be resolved when we create the Snapshot. If we resolved some paths but // not others, we would get an error like `File doesn't belong in the // transaction log at `. val parsedLogData = Seq( // scalastyle:off line.size.limit getTestResourceFilePath("catalog-owned-preview/_delta_log/_staged_commits/00000000000000000001.4cb9708e-b478-44de-b203-53f9ba9b2876.json"), getTestResourceFilePath("catalog-owned-preview/_delta_log/_staged_commits/00000000000000000002.5b9bba4a-0085-430d-a65e-b0d38c1afbe9.json")) // scalastyle:on line.size.limit .map { path => defaultEngine.getFileSystemClient.resolvePath(path) } .map { p => FileStatus.of(p) } .map { fs => ParsedLogData.forFileStatus(fs) } .toList testFx(tablePath, parsedLogData) } test("simple e2e read of catalogManaged table with staged ratified commits") { withCatalogOwnedPreviewTestTable { (tablePath, parsedLogData) => // ===== WHEN ===== val snapshot = TableManager .loadSnapshot(tablePath) .asInstanceOf[SnapshotBuilderImpl] .atVersion(2) .withLogData(parsedLogData.asJava) .withMaxCatalogVersion(2) .build(defaultEngine) // ===== THEN ===== assert(snapshot.getVersion === 2) assert(snapshot.getLogSegment.getDeltas.size() === 3) assert(snapshot.getTimestamp(defaultEngine) === 1749830881799L) val protocol = snapshot.getProtocol assert(protocol.getMinReaderVersion == TABLE_FEATURES_MIN_READER_VERSION) assert(protocol.getMinWriterVersion == TABLE_FEATURES_MIN_WRITER_VERSION) assert(protocol.getReaderFeatures.contains(CATALOG_MANAGED_RW_FEATURE.featureName())) assert(protocol.getWriterFeatures.contains(CATALOG_MANAGED_RW_FEATURE.featureName())) assert(protocol.getWriterFeatures.contains(IN_COMMIT_TIMESTAMP_W_FEATURE.featureName())) val actualResult = readSnapshot(snapshot) val expectedResult = (0 to 199).map { x => TestRow(x / 100, x) } checkAnswer(actualResult, expectedResult) } } test("e2e DeltaHistoryManager.getActiveCommitAtTimestamp with catalogManaged table " + "with staged ratified commits") { withCatalogOwnedPreviewTestTable { (tablePath, parsedLogData) => val logPath = new Path(tablePath, "_delta_log") val parsedRatifiedCatalogCommits = parsedLogData .filter(_.isInstanceOf[ParsedCatalogCommitData]) .map(_.asInstanceOf[ParsedCatalogCommitData]) val latestSnapshot = TableManager .loadSnapshot(tablePath) .asInstanceOf[SnapshotBuilderImpl] .withLogData(parsedLogData.asJava) .withMaxCatalogVersion(2) .build(defaultEngine) def checkGetActiveCommitAtTimestamp( timestamp: Long, expectedVersion: Long, canReturnLastCommit: Boolean = false, canReturnEarliestCommit: Boolean = false): Unit = { val activeCommit = DeltaHistoryManager.getActiveCommitAtTimestamp( defaultEngine, latestSnapshot, logPath, timestamp, true, /* mustBeRecreatable */ canReturnLastCommit, canReturnEarliestCommit, parsedRatifiedCatalogCommits.asJava) assert(activeCommit.getVersion == expectedVersion) } val v0Ts = 1749830855993L // published commit val v1Ts = 1749830871085L // staged commit val v2Ts = 1749830881799L // staged commit // Query a timestamp before V0 should fail if canReturnEarliestCommit = false val e1 = intercept[KernelException] { checkGetActiveCommitAtTimestamp(v0Ts - 1, 0) } assert(e1.getMessage.contains("before the earliest available version")) // Query a timestamp before V0 with canReturnEarliestCommit = true checkGetActiveCommitAtTimestamp(v0Ts - 1, 0, canReturnEarliestCommit = true) // Query @ V0 checkGetActiveCommitAtTimestamp(v0Ts, 0) // Query between V0 and V1 checkGetActiveCommitAtTimestamp(v0Ts + 1, 0) // Query at V1 checkGetActiveCommitAtTimestamp(v1Ts, 1) // Query between V1 and V2 checkGetActiveCommitAtTimestamp(v1Ts + 1, 1) // Query at V2 checkGetActiveCommitAtTimestamp(v2Ts, 2) // Query a timestamp after V2 should fail with canReturnLastCommit = false val e2 = intercept[KernelException] { checkGetActiveCommitAtTimestamp(v2Ts + 1, 2) } assert(e2.getMessage.contains("is after the latest available version")) // Query a timestamp after V2 with canReturnLastCommit = true checkGetActiveCommitAtTimestamp(v2Ts + 1, 2, canReturnLastCommit = true) } } test("time-travel by ts read of catalogManaged table with ratified commits") { withCatalogOwnedPreviewTestTable { (tablePath, parsedLogData) => val v0Ts = 1749830855993L // published commit val v1Ts = 1749830871085L // ratified staged commit val v2Ts = 1749830881799L // ratified staged commit val latestSnapshot = TableManager .loadSnapshot(tablePath) .asInstanceOf[SnapshotBuilderImpl] .withLogData(parsedLogData.asJava) .withMaxCatalogVersion(2) .build(defaultEngine) def checkTimeTravelByTimestamp( timestamp: Long, expectedVersion: Long, expectedSnapshotTimestamp: Long): Unit = { val snapshot = TableManager .loadSnapshot(tablePath) .atTimestamp(timestamp, latestSnapshot) .withMaxCatalogVersion(2) .withLogData(parsedLogData.asJava) .build(defaultEngine) assert(snapshot.getVersion == expectedVersion) assert(snapshot.getTimestamp(defaultEngine) == expectedSnapshotTimestamp) } // Between v0 and v1 should return v0 (between published & ratified) checkTimeTravelByTimestamp(v0Ts + 1, 0, v0Ts) // Exactly v1 should return v1 checkTimeTravelByTimestamp(v1Ts, 1, v1Ts) // Between v1 and v2 should return v1 (between 2 ratified commits) checkTimeTravelByTimestamp(v1Ts + 1, 1, v1Ts) // Exactly v2 should return v2 checkTimeTravelByTimestamp(v2Ts, 2, v2Ts) // After v2 should fail val e = intercept[KernelException] { checkTimeTravelByTimestamp(v2Ts + 1, 2, v2Ts) } assert(e.getMessage.contains("is after the latest available version")) } } test("e2e CommitRange test with catalogManaged table with staged ratified commits") { withCatalogOwnedPreviewTestTable { (tablePath, parsedLogData) => val v0Ts = 1749830855993L // published commit val v1Ts = 1749830871085L // staged commit val v2Ts = 1749830881799L // staged commit val latestSnapshot = TableManager .loadSnapshot(tablePath) .withLogData(parsedLogData.asJava) .withMaxCatalogVersion(2) .build(defaultEngine) def checkStartBoundary(timestamp: Long, expectedVersion: Long): Unit = { assert(TableManager.loadCommitRange( tablePath, CommitBoundary.atTimestamp(timestamp, latestSnapshot)) .withLogData(parsedLogData.asJava) .withMaxCatalogVersion(2) .build(defaultEngine) .getStartVersion == expectedVersion) } def checkEndBoundary(timestamp: Long, expectedVersion: Long): Unit = { assert(TableManager.loadCommitRange(tablePath, CommitBoundary.atVersion(0)) .withLogData(parsedLogData.asJava) .withMaxCatalogVersion(2) .withEndBoundary(CommitBoundary.atTimestamp(timestamp, latestSnapshot)) .build(defaultEngine) .getEndVersion == expectedVersion) } // startTimestamp is before V0 checkStartBoundary(v0Ts - 1, 0) // endTimestamp is before V0 intercept[KernelException] { checkEndBoundary(v0Ts - 1, -1) } // startTimestamp is at V0 checkStartBoundary(v0Ts, 0) // endTimestamp is at V0 checkEndBoundary(v0Ts, 0) // startTimestamp is between V0 and V1 checkStartBoundary(v0Ts + 100L, 1) // endTimestamp is between V0 and V1 checkEndBoundary(v0Ts + 100L, 0) // startTimestamp is at V1 checkStartBoundary(v1Ts, 1) // endTimestamp is at V1 checkEndBoundary(v1Ts, 1) // startTimestamp is between V1 and V2 checkStartBoundary(v1Ts + 100L, 2) // endTimestamp is between V1 and V2 checkEndBoundary(v1Ts + 100L, 1) // startTimestamp is at V2 checkStartBoundary(v2Ts, 2) // endTimestamp is at V2 checkEndBoundary(v2Ts, 2) // startTimestamp is after V2 intercept[KernelException] { checkStartBoundary(v2Ts + 10, -1) } // endTimestamp is after V2 checkEndBoundary(v2Ts + 10, 2) // Verify the fileList in the CommitRange val commitRange = TableManager .loadCommitRange(tablePath, CommitBoundary.atVersion(0)) .withLogData(parsedLogData.asJava) .withMaxCatalogVersion(2) .build(defaultEngine) val expectedFileList = Seq( // scalastyle:off line.size.limit getTestResourceFilePath("catalog-owned-preview/_delta_log/00000000000000000000.json"), getTestResourceFilePath("catalog-owned-preview/_delta_log/_staged_commits/00000000000000000001.4cb9708e-b478-44de-b203-53f9ba9b2876.json"), getTestResourceFilePath("catalog-owned-preview/_delta_log/_staged_commits/00000000000000000002.5b9bba4a-0085-430d-a65e-b0d38c1afbe9.json") // scalastyle:on line.size.limit ).map(path => defaultEngine.getFileSystemClient.resolvePath(path)) assert(commitRange.asInstanceOf[CommitRangeImpl].getDeltaFiles().asScala.map(_.getPath) == expectedFileList) } } // We test this in the unit tests as well, but since those use the withProtocolAndMetadata API // we also test it here with a real table where we load the P&M from the log test("reading a catalogManaged table without providing maxCatalogVersion fails") { withCatalogOwnedPreviewTestTable { (tablePath, parsedLogData) => // With logData intercept[IllegalArgumentException] { TableManager .loadSnapshot(tablePath) .withLogData(parsedLogData.asJava) .build(defaultEngine) } // Without logData (and with time-travel-version) intercept[IllegalArgumentException] { TableManager .loadSnapshot(tablePath) .atVersion(0) .build(defaultEngine) } } } test("reading a file-system managed table and providing maxCatalogVersion fails") { withTempDirAndEngine { (tablePath, engine) => // Create a basic file-system managed table createEmptyTable(tablePath = tablePath, schema = testSchema) // Try to read it and provide the maxCatalogVersion intercept[IllegalArgumentException] { TableManager .loadSnapshot(tablePath) .withMaxCatalogVersion(0) .build(engine) } } } test("for latest queries we do not load past the maxRatifiedVersion even if " + "later versions exist on the file-system") { withTempDir { tempDir => withCatalogOwnedPreviewTestTable { (resourceTablePath, resourceLogData) => // Copy the catalog-owned-preview test resource table to the temp directory org.apache.commons.io.FileUtils.copyDirectory( new java.io.File(resourceTablePath), tempDir) // "Publish" v1 and v2 (we do both to maintain ordered backfill) val deltaLogPath = new Path(tempDir.getPath, "_delta_log") val stagedCommitPath = new Path(deltaLogPath, "_staged_commits") resourceLogData.foreach { stagedCommit => val stagedCommitFile = new java.io.File( stagedCommitPath.toString, new Path(stagedCommit.getFileStatus.getPath).getName) val publishedCommitFile = new java.io.File( FileNames.deltaFile(deltaLogPath.toString, stagedCommit.getVersion)) org.apache.commons.io.FileUtils.copyFile(stagedCommitFile, publishedCommitFile) } def convertResourceLogData(logData: ParsedLogData): ParsedLogData = { val path = new Path(stagedCommitPath, new Path(logData.getFileStatus.getPath).getName) ParsedLogData.forFileStatus(FileStatus.of( defaultEngine.getFileSystemClient.resolvePath(path.toString))) } Seq(0, 1, 2).foreach { maxCatalogVersion => { // Try to read the table with no parsedLogData val snapshot = TableManager .loadSnapshot(tempDir.getPath) .withMaxCatalogVersion(maxCatalogVersion) .build(defaultEngine) assert(snapshot.getVersion == maxCatalogVersion) } { // Try to read the table with parsedLogData val parsedLogData = resourceLogData .filter(_.getVersion <= maxCatalogVersion) .map(convertResourceLogData) val snapshot = TableManager .loadSnapshot(tempDir.getPath) .withMaxCatalogVersion(maxCatalogVersion) .withLogData(parsedLogData.asJava) .build(defaultEngine) assert(snapshot.getVersion == maxCatalogVersion) } } } } } test("for latest queries if we cannot load the maxRatifiedVersion we fail") { withCatalogOwnedPreviewTestTable { (tablePath, _) => // We can only test this when no logData is provided. Otherwise we require logData to end // with maxRatifiedVersion ==> it should be able to be read. val e = intercept[KernelException] { TableManager .loadSnapshot(tablePath) .withMaxCatalogVersion(2) .build(defaultEngine) } assert(e.getMessage.contains("Cannot load table version 2")) } } test("for latest queries we read the _last_checkpoint file") { withCatalogOwnedPreviewTestTable { (resourceTablePath, resourceLogData) => // It doesn't matter if the checkpoint actually exists; we just want to check that during // log segment building we try to read _last_checkpoint import io.delta.kernel.defaults.MetricsEngine val engine = new MetricsEngine(new HadoopFileIO(new Configuration())) val snapshot = TableManager .loadSnapshot(resourceTablePath) .withMaxCatalogVersion(2) .withLogData(resourceLogData.asJava) .build(engine) assert(snapshot.getVersion == 2) assert(engine.getJsonHandler.getLastCheckpointMetadataReadCalls == 1) } } test("for commitRange queries with no end boundary we do not load past the maxRatifiedVersion " + "even if later versions exist on the file-system") { withTempDir { tempDir => withCatalogOwnedPreviewTestTable { (resourceTablePath, resourceLogData) => // Copy the catalog-owned-preview test resource table to the temp directory org.apache.commons.io.FileUtils.copyDirectory( new java.io.File(resourceTablePath), tempDir) // "Publish" v1 and v2 (we do both to maintain ordered backfill) val deltaLogPath = new Path(tempDir.getPath, "_delta_log") val stagedCommitPath = new Path(deltaLogPath, "_staged_commits") resourceLogData.foreach { stagedCommit => val stagedCommitFile = new java.io.File( stagedCommitPath.toString, new Path(stagedCommit.getFileStatus.getPath).getName) val publishedCommitFile = new java.io.File( FileNames.deltaFile(deltaLogPath.toString, stagedCommit.getVersion)) org.apache.commons.io.FileUtils.copyFile(stagedCommitFile, publishedCommitFile) } def convertResourceLogData(logData: ParsedLogData): ParsedLogData = { val path = new Path(stagedCommitPath, new Path(logData.getFileStatus.getPath).getName) ParsedLogData.forFileStatus(FileStatus.of( defaultEngine.getFileSystemClient.resolvePath(path.toString))) } Seq(0, 1, 2).foreach { maxCatalogVersion => { // Try to read the table with no parsedLogData val commitRange = TableManager .loadCommitRange(tempDir.getPath, CommitBoundary.atVersion(0)) .withMaxCatalogVersion(maxCatalogVersion) .build(defaultEngine) assert(commitRange.getEndVersion == maxCatalogVersion) } { // Try to read the table with parsedLogData val parsedLogData = resourceLogData .filter(_.getVersion <= maxCatalogVersion) .map(convertResourceLogData) val commitRange = TableManager .loadCommitRange(tempDir.getPath, CommitBoundary.atVersion(0)) .withMaxCatalogVersion(maxCatalogVersion) .withLogData(parsedLogData.asJava) .build(defaultEngine) assert(commitRange.getEndVersion == maxCatalogVersion) } } } } } test("for commitRange queries with no end boundary if we cannot load the maxRatifiedVersion we " + "fail") { withCatalogOwnedPreviewTestTable { (tablePath, _) => // We can only test this when no logData is provided. Otherwise we require logData to end // with maxRatifiedVersion ==> it should be able to be read. val e = intercept[KernelException] { TableManager .loadCommitRange(tablePath, CommitBoundary.atVersion(0)) .withMaxCatalogVersion(2) .build(defaultEngine) } assert(e.getMessage.contains( "Requested table changes ending with endVersion=2 but no log file found for version 2")) } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/catalogManaged/CatalogManagedPropertyValidationSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.catalogManaged import scala.collection.JavaConverters._ import io.delta.kernel.{Operation, TableManager, Transaction} import io.delta.kernel.commit.Committer import io.delta.kernel.data.Row import io.delta.kernel.defaults.utils.TestUtils import io.delta.kernel.engine.Engine import io.delta.kernel.internal.SnapshotImpl import io.delta.kernel.internal.tablefeatures.TableFeatures import io.delta.kernel.types.{IntegerType, StructType} import io.delta.kernel.utils.CloseableIterable.emptyIterable import org.scalatest.funsuite.AnyFunSuite class CatalogManagedPropertyValidationSuite extends AnyFunSuite with TestUtils { val catalogManagedFeaturePropMap = Map( TableFeatures.CATALOG_MANAGED_RW_FEATURE.getTableFeatureSupportKey -> "supported") val validRequiredCatalogPropMap = Map( customCatalogCommitter.REQUIRED_PROPERTY_KEY -> customCatalogCommitter.REQUIRED_PROPERTY_VALUE) val invalidRequiredCatalogPropMap = Map( customCatalogCommitter.REQUIRED_PROPERTY_KEY -> "invalid") case class CatalogManagedTestCase( testName: String, /** "CREATE", "UPDATE", or "REPLACE" */ operationType: String, initialTableProperties: Map[String, String] = Map.empty, transactionProperties: Map[String, String], /** only applicable to UPDATE */ removedPropertyKeys: Set[String] = Set.empty, /** create table for UPDATE/REPLACE */ createInitialTableCommitter: Committer = customCatalogCommitter, expectedSuccess: Boolean = true, expectedExceptionMessage: Option[String] = None, /** only applicable if SUCCESS */ expectedIctEnabled: Boolean = true, /** only applicable if SUCCESS */ expectedCatalogManagedSupported: Boolean = true) val catalogManagedTestCases = Seq( // ===== CREATE cases ===== CatalogManagedTestCase( testName = "CREATE: set catalogManaged=supported => enables catalogManaged and ICT", operationType = "CREATE", transactionProperties = catalogManagedFeaturePropMap), CatalogManagedTestCase( testName = "ILLEGAL CREATE: set catalogManaged=supported and explicitly disable ICT => THROW", operationType = "CREATE", transactionProperties = Map( TableFeatures.CATALOG_MANAGED_RW_FEATURE.getTableFeatureSupportKey -> "supported", "delta.enableInCommitTimestamps" -> "false"), expectedSuccess = false, expectedExceptionMessage = Some("Cannot disable inCommitTimestamp when enabling catalogManaged")), // ===== UPDATE cases ===== CatalogManagedTestCase( testName = "UPDATE: set catalogManaged=supported => enables catalogManaged and ICT", operationType = "UPDATE", initialTableProperties = Map.empty, // Start with basic table transactionProperties = catalogManagedFeaturePropMap), CatalogManagedTestCase( testName = "UPDATE: set catalogManaged=supported => enables ICT if previously disabled", operationType = "UPDATE", initialTableProperties = Map("delta.enableInCommitTimestamps" -> "false"), transactionProperties = catalogManagedFeaturePropMap), CatalogManagedTestCase( testName = "UPDATE: set catalogManaged=supported and ICT already enabled => Okay", operationType = "UPDATE", initialTableProperties = Map("delta.enableInCommitTimestamps" -> "true"), transactionProperties = catalogManagedFeaturePropMap), CatalogManagedTestCase( testName = "ILLEGAL UPDATE: set catalogManaged=supported and disable ICT => THROW", operationType = "UPDATE", initialTableProperties = Map.empty, transactionProperties = Map( TableFeatures.CATALOG_MANAGED_RW_FEATURE.getTableFeatureSupportKey -> "supported", "delta.enableInCommitTimestamps" -> "false"), expectedSuccess = false, expectedExceptionMessage = Some("Cannot disable inCommitTimestamp when enabling catalogManaged")), CatalogManagedTestCase( testName = "ILLEGAL UPDATE: catalogManaged already supported, then disable ICT => THROW", operationType = "UPDATE", initialTableProperties = catalogManagedFeaturePropMap, transactionProperties = Map("delta.enableInCommitTimestamps" -> "false"), expectedSuccess = false, expectedExceptionMessage = Some("Cannot disable inCommitTimestamp on a catalogManaged table")), CatalogManagedTestCase( testName = "NO-OP UPDATE: catalogManaged not being enabled should not affect ICT", operationType = "UPDATE", initialTableProperties = Map.empty, transactionProperties = Map(), expectedIctEnabled = false, expectedCatalogManagedSupported = false), // ===== REPLACE cases ===== CatalogManagedTestCase( testName = "REPLACE: normal replace should succeed on a catalogManaged table", operationType = "REPLACE", initialTableProperties = catalogManagedFeaturePropMap, transactionProperties = Map()), CatalogManagedTestCase( testName = "ILLEGAL REPLACE: set catalogManaged=supported => THROW", operationType = "REPLACE", initialTableProperties = Map.empty, transactionProperties = catalogManagedFeaturePropMap, expectedSuccess = false, expectedExceptionMessage = Some("Cannot enable the catalogManaged feature during a REPLACE command.")), CatalogManagedTestCase( testName = "ILLEGAL REPLACE: catalogManaged already supported, then disable ICT => THROW", operationType = "REPLACE", initialTableProperties = catalogManagedFeaturePropMap, transactionProperties = Map("delta.enableInCommitTimestamps" -> "false"), expectedSuccess = false, expectedExceptionMessage = Some("Cannot disable inCommitTimestamp on a catalogManaged table")), // ===== Required catalog table property cases: Txn allowed to not explicitly set value ===== CatalogManagedTestCase( testName = "CREATE: User does not explicitly set catalog property => auto-set", operationType = "CREATE", transactionProperties = catalogManagedFeaturePropMap ), // <-- Missing, will be auto-set CatalogManagedTestCase( testName = "REPLACE: User does not explicitly set catalog property => auto-set", operationType = "REPLACE", initialTableProperties = catalogManagedFeaturePropMap, transactionProperties = Map.empty ), // <-- Missing, will be auto-set CatalogManagedTestCase( testName = "UPDATE: Normal updates succeed", operationType = "UPDATE", initialTableProperties = catalogManagedFeaturePropMap ++ validRequiredCatalogPropMap, transactionProperties = Map("zip" -> "zap") ), // <-- Just testing that normal updates succee // ===== Required catalog table property cases: User can input correct value ===== CatalogManagedTestCase( testName = "CREATE: Can set required catalog property to correct value", operationType = "CREATE", transactionProperties = catalogManagedFeaturePropMap ++ validRequiredCatalogPropMap ), // <-- Set to valid CatalogManagedTestCase( testName = "REPLACE: Can set required catalog property to correct value", operationType = "REPLACE", initialTableProperties = catalogManagedFeaturePropMap, transactionProperties = validRequiredCatalogPropMap ), // <-- Set to valid CatalogManagedTestCase( testName = "UPDATE: Can set required catalog property to correct value", operationType = "UPDATE", initialTableProperties = catalogManagedFeaturePropMap, transactionProperties = validRequiredCatalogPropMap ), // <-- Set to valid // ===== Required catalog table property case: User cannot remove or input incorrect value ===== CatalogManagedTestCase( testName = "ILLEGAL CREATE: Set required catalog property to incorrect value => THROW", operationType = "CREATE", transactionProperties = catalogManagedFeaturePropMap ++ invalidRequiredCatalogPropMap, // <-- Set to invalid expectedSuccess = false), CatalogManagedTestCase( testName = "ILLEGAL REPLACE: Set required catalog property to incorrect value => THROW", operationType = "REPLACE", initialTableProperties = catalogManagedFeaturePropMap, transactionProperties = invalidRequiredCatalogPropMap, // <-- Set to invalid expectedSuccess = false, expectedExceptionMessage = Some("Metadata is missing or has incorrect values for required catalog properties")), CatalogManagedTestCase( testName = "ILLEGAL UPDATE: Set required catalog property to incorrect value => THROW", operationType = "UPDATE", initialTableProperties = catalogManagedFeaturePropMap, transactionProperties = invalidRequiredCatalogPropMap, // <-- Set to invalid expectedSuccess = false, expectedExceptionMessage = Some("Metadata is missing or has incorrect values for required catalog properties")), CatalogManagedTestCase( testName = "ILLEGAL UPDATE: Remove required catalog property => THROW", operationType = "UPDATE", initialTableProperties = catalogManagedFeaturePropMap ++ validRequiredCatalogPropMap, transactionProperties = Map.empty, removedPropertyKeys = Set(customCatalogCommitter.REQUIRED_PROPERTY_KEY), // <-- Removed! expectedSuccess = false, expectedExceptionMessage = Some("Metadata is missing or has incorrect values for required catalog properties")), // ===== Required catalog table property case: Existing table invalid ===== CatalogManagedTestCase( testName = "REPLACE: On existing table with incorrect required catalog property => sets it", operationType = "REPLACE", initialTableProperties = catalogManagedFeaturePropMap ++ invalidRequiredCatalogPropMap, // <-- Set to invalid createInitialTableCommitter = committerUsingPutIfAbsent, // allow creating the invalid table transactionProperties = Map.empty), CatalogManagedTestCase( testName = "UPDATE: On existing table with incorrect required catalog property => throws", operationType = "UPDATE", initialTableProperties = catalogManagedFeaturePropMap ++ invalidRequiredCatalogPropMap, // <-- Set to invalid createInitialTableCommitter = committerUsingPutIfAbsent, // allow creating the invalid table transactionProperties = Map.empty, expectedSuccess = false, expectedExceptionMessage = Some("Metadata is missing or has incorrect values for required catalog properties"))) catalogManagedTestCases.foreach { testCase => test(testCase.testName) { withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath val schema = new StructType().add("id", IntegerType.INTEGER) // Setup initial table if this is an UPDATE operation if (testCase.operationType == "UPDATE" || testCase.operationType == "REPLACE") { TableManager .buildCreateTableTransaction(tablePath, schema, "engineInfo") .withTableProperties(testCase.initialTableProperties.asJava) .withCommitter(testCase.createInitialTableCommitter) .build(defaultEngine) .commit(defaultEngine, emptyIterable[Row]) } // CREATE, UPDATE, and REPLACE txnBuilders don't share a common parent interface. So, we // treat the `txnBuilder` as a trait that has a `build(engine)` method. Scalastyle doesn't // like this, but it's valid. // // scalastyle:off val txnBuilder: { def build(engine: Engine): Transaction } = testCase.operationType match { case "CREATE" => TableManager .buildCreateTableTransaction(tablePath, schema, "engineInfo") .withTableProperties(testCase.transactionProperties.asJava) .withCommitter(customCatalogCommitter) case "UPDATE" => val updateBuilder = TableManager .loadSnapshot(tablePath) .withCommitter(customCatalogCommitter) .withMaxCatalogVersionIfApplicable( isCatalogManaged = TableFeatures.isPropertiesManuallySupportingTableFeature( testCase.initialTableProperties.asJava, TableFeatures.CATALOG_MANAGED_RW_FEATURE), maxCatalogVersion = 0) .build(defaultEngine) .buildUpdateTableTransaction("engineInfo", Operation.MANUAL_UPDATE) .withTablePropertiesAdded(testCase.transactionProperties.asJava) if (testCase.removedPropertyKeys.nonEmpty) { updateBuilder.withTablePropertiesRemoved(testCase.removedPropertyKeys.asJava) } else { updateBuilder } case "REPLACE" => val replaceSchema = schema.add("col2", IntegerType.INTEGER) TableManager .loadSnapshot(tablePath) .withCommitter(customCatalogCommitter) .withMaxCatalogVersionIfApplicable( isCatalogManaged = TableFeatures.isPropertiesManuallySupportingTableFeature( testCase.initialTableProperties.asJava, TableFeatures.CATALOG_MANAGED_RW_FEATURE), maxCatalogVersion = 0) .build(defaultEngine) .asInstanceOf[SnapshotImpl] .buildReplaceTableTransaction(replaceSchema, "engineInfo") .withTableProperties(testCase.transactionProperties.asJava) } // scalastyle:on if (testCase.expectedSuccess) { // Transaction building should succeed val result = txnBuilder.build(defaultEngine).commit(defaultEngine, emptyIterable[Row]) val postCommitSnapshot = result .getPostCommitSnapshot .orElseThrow(() => new RuntimeException("Expected post-commit snapshot when no concurrent writes")) .asInstanceOf[SnapshotImpl] // Verify the results val protocol = postCommitSnapshot.getProtocol // Check if catalogManaged feature is supported val catalogManagedSupported = protocol .supportsFeature(TableFeatures.CATALOG_MANAGED_RW_FEATURE) assert(catalogManagedSupported == testCase.expectedCatalogManagedSupported) // Check if ICT is enabled in metadata val ictEnabled = postCommitSnapshot.getMetadata.getConfiguration.asScala .get("delta.enableInCommitTimestamps") .contains("true") assert(ictEnabled == testCase.expectedIctEnabled) // If catalogManaged is supported, ICT feature should also be supported if (testCase.expectedCatalogManagedSupported) { assert(protocol.supportsFeature(TableFeatures.IN_COMMIT_TIMESTAMP_W_FEATURE)) assert( customCatalogCommitter .getRequiredTableProperties .asScala.toSet.subsetOf(postCommitSnapshot.getTableProperties.asScala.toSet)) } } else { // Transaction building should fail val exception = intercept[Exception] { txnBuilder.build(defaultEngine) } testCase.expectedExceptionMessage.foreach { expectedMsg => assert(exception.getMessage.contains(expectedMsg)) } } } } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/engine/DefaultExpressionHandlerSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.engine import io.delta.kernel.defaults.utils.ExpressionTestUtils import io.delta.kernel.types.BooleanType.BOOLEAN import io.delta.kernel.types.IntegerType.INTEGER import io.delta.kernel.types.LongType.LONG import io.delta.kernel.types.StringType.STRING import io.delta.kernel.types.StructType import org.scalatest.funsuite.AnyFunSuite class DefaultExpressionHandlerSuite extends AnyFunSuite with ExpressionTestUtils { test("create selection vector: single value") { Seq(true, false).foreach { testValue => val outputVector = selectionVector(Seq(testValue).toArray, 0, 1) assert(outputVector.getDataType === BOOLEAN) assert(outputVector.getSize == 1) assert(outputVector.isNullAt(0) == false) assert(outputVector.getBoolean(0) == testValue) } } test("create selection vector: multiple values array, partial array") { Seq((0, testValues.length), (0, 3), (2, 2), (2, 4), (3, testValues.length)).foreach { pair => val (from, to) = (pair._1, pair._2) val outputVector = selectionVector(testValues, from, to) assert(outputVector.getDataType === BOOLEAN) assert(outputVector.getSize == (to - from)) Seq.range(from, to).foreach { rowId => assert(outputVector.isNullAt(rowId - from) == false) assert(outputVector.getBoolean(rowId - from) == testValues(rowId)) } } } test("create selection vector: update values array and expect no changes in output") { val outputVector = selectionVector(testValues, 0, testValues.length) // update the input values array and assert the value is not changed in the returned vector val oldValue = testValues(2) assert(oldValue == false) testValues(2) = true assert(outputVector.isNullAt(2) == false) assert(outputVector.getBoolean(2) == oldValue) } test("create selection vector: invalid to and/or from offset") { Seq((3, 2), (2, testValues.length + 1), (testValues.length + 1, 100)).foreach { pair => val (from, to) = (pair._1, pair._2) val ex = intercept[IllegalArgumentException] { selectionVector(testValues, from, to) } assert(ex.getMessage.contains( s"invalid range from=$from, to=$to, values length=${testValues.length}")) } } test("create selection vector: null values array") { val ex = intercept[NullPointerException] { selectionVector(null, 0, 25) } assert(ex.getMessage.contains("values is null")) } private def selectionVector(values: Array[Boolean], from: Int, to: Int) = { new DefaultExpressionHandler().createSelectionVector(values, from, to) } private val testValues = Seq(false, true, false, false, true, true).toArray } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/engine/DefaultFileSystemClientSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.engine import java.io.FileNotFoundException import scala.collection.mutable.ArrayBuffer import io.delta.kernel.defaults.utils.TestUtils import org.apache.hadoop.fs.{FileSystem, Path} import org.scalatest.funsuite.AnyFunSuite class DefaultFileSystemClientSuite extends AnyFunSuite with TestUtils { val fsClient = defaultEngine.getFileSystemClient val fs = FileSystem.get(configuration) private def writeFile(path: String, content: String): Unit = { val out = fs.create(new Path(path)) try { out.write(content.getBytes("UTF-8")) } finally { out.close() } } private def readFile(path: String): String = { val fileStatus = fs.getFileStatus(new Path(path)) val buffer = new Array[Byte](fileStatus.getLen.toInt) val in = fs.open(new Path(path)) try { in.readFully(buffer) new String(buffer, "UTF-8") } finally { in.close() } } private def withTempSrcAndDestFiles(f: (String, String) => Unit): Unit = { withTempDir { tempDir => val src = tempDir + "/source.txt" val dest = tempDir + "/dest.txt" f(src, dest) } } test("list from file") { val basePath = fsClient.resolvePath(getTestResourceFilePath("json-files")) val listFrom = fsClient.resolvePath(getTestResourceFilePath("json-files/2.json")) val actListOutput = new ArrayBuffer[String]() val files = fsClient.listFrom(listFrom) try { fsClient.listFrom(listFrom).forEach(f => actListOutput += f.getPath) } finally if (files != null) { files.close() } val expListOutput = Seq(basePath + "/2.json", basePath + "/3.json") assert(expListOutput === actListOutput) } test("list from non-existent file") { intercept[FileNotFoundException] { fsClient.listFrom("file:/non-existentfileTable/01.json") } } test("resolve path") { val inputPath = getTestResourceFilePath("json-files") val resolvedPath = fsClient.resolvePath(inputPath) assert("file:" + inputPath === resolvedPath) } test("resolve path on non-existent file") { val inputPath = "/non-existentfileTable/01.json" val resolvedPath = fsClient.resolvePath(inputPath) assert("file:" + inputPath === resolvedPath) } test("mkdirs") { withTempDir { tempdir => val dir1 = tempdir + "/test" assert(fsClient.mkdirs(dir1)) assert(fs.exists(new Path(dir1))) val dir2 = tempdir + "/test1/test2" // nested assert(fsClient.mkdirs(dir2)) assert(fs.exists(new Path(dir2))) val dir3 = "/non-existentfileTable/sfdsd" assert(!fsClient.mkdirs(dir3)) assert(!fs.exists(new Path(dir3))) } } test("getFileStatus") { val filePath = getTestResourceFilePath("json-files/1.json") val fileStatus = fsClient.getFileStatus(filePath) assert(fileStatus.getPath == fsClient.resolvePath(filePath)) assert(fileStatus.getSize > 0) assert(fileStatus.getModificationTime > 0) } test("getFileStatus on non-existent file") { intercept[FileNotFoundException] { fsClient.getFileStatus("/non-existent-file.json") } } test("copyFileAtomically - overwrite=false, dest does not exist") { withTempSrcAndDestFiles { (src, dest) => writeFile(src, "test content") fsClient.copyFileAtomically(src, dest, false /* overwrite */ ) assert(fs.exists(new Path(dest))) assert(readFile(dest).trim == "test content") } } test("copyFileAtomically - overwrite=false, dest exists") { withTempSrcAndDestFiles { (src, dest) => writeFile(src, "source content") writeFile(dest, "existing content") intercept[java.nio.file.FileAlreadyExistsException] { fsClient.copyFileAtomically(src, dest, false /* overwrite */ ) } } } test("copyFileAtomically - overwrite=true") { withTempSrcAndDestFiles { (src, dest) => writeFile(src, "new content") writeFile(dest, "old content") fsClient.copyFileAtomically(src, dest, true /* overwrite */ ) assert(readFile(dest).trim == "new content") } } test("copyFileAtomically with non-existent source") { withTempSrcAndDestFiles { (src, dest) => intercept[FileNotFoundException] { fsClient.copyFileAtomically(src, dest, false /* overwrite */ ) } } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/engine/DefaultJsonHandlerSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.engine import java.math.{BigDecimal => JBigDecimal} import java.nio.file.FileAlreadyExistsException import java.util.{Collections, Optional} import scala.collection.JavaConverters._ import io.delta.kernel.data.ColumnVector import io.delta.kernel.defaults.engine.hadoopio.HadoopFileIO import io.delta.kernel.defaults.utils.{DefaultVectorTestUtils, TestRow, TestUtils} import io.delta.kernel.internal.actions.CommitInfo import io.delta.kernel.internal.util.InternalUtils.singletonStringColumnVector import io.delta.kernel.types._ import org.apache.hadoop.conf.Configuration import org.scalatest.funsuite.AnyFunSuite class DefaultJsonHandlerSuite extends AnyFunSuite with TestUtils with DefaultVectorTestUtils { val jsonHandler = new DefaultJsonHandler( new HadoopFileIO( new Configuration { set("delta.kernel.default.json.reader.batch-size", "1") })) val fsClient = defaultEngine.getFileSystemClient ///////////////////////////////////////////////////////////////////////////////////////////////// // Tests for parseJson for statistics eligible types (additional in TestDefaultJsonHandler.java) ///////////////////////////////////////////////////////////////////////////////////////////////// def testJsonParserWithSchema( jsonString: String, schema: StructType, expectedRow: TestRow): Unit = { val batchRows = jsonHandler.parseJson( singletonStringColumnVector(jsonString), schema, Optional.empty()).getRows.toSeq checkAnswer(batchRows, Seq(expectedRow)) } def testJsonParserForSingleType( jsonString: String, dataType: DataType, numColumns: Int, expectedRow: TestRow): Unit = { val schema = new StructType( (1 to numColumns).map(i => new StructField(s"col$i", dataType, true)).asJava) testJsonParserWithSchema(jsonString, schema, expectedRow) } def testOutOfRangeValue(stringValue: String, dataType: DataType): Unit = { val e = intercept[RuntimeException] { testJsonParserForSingleType( jsonString = s"""{"col1":$stringValue}""", dataType = dataType, numColumns = 1, expectedRow = TestRow()) } assert(e.getMessage.contains(s"Couldn't decode $stringValue")) } test("parse byte type") { testJsonParserForSingleType( jsonString = """{"col1":0,"col2":-127,"col3":127, "col4":null}""", dataType = ByteType.BYTE, 4, TestRow(0.toByte, -127.toByte, 127.toByte, null)) testOutOfRangeValue("128", ByteType.BYTE) testOutOfRangeValue("-129", ByteType.BYTE) testOutOfRangeValue("2147483648", ByteType.BYTE) } test("parse short type") { testJsonParserForSingleType( jsonString = """{"col1":-32767,"col2":8,"col3":32767, "col4":null}""", dataType = ShortType.SHORT, 4, TestRow(-32767.toShort, 8.toShort, 32767.toShort, null)) testOutOfRangeValue("32768", ShortType.SHORT) testOutOfRangeValue("-32769", ShortType.SHORT) testOutOfRangeValue("2147483648", ShortType.SHORT) } test("parse integer type") { testJsonParserForSingleType( jsonString = """{"col1":-2147483648,"col2":8,"col3":2147483647, "col4":null}""", dataType = IntegerType.INTEGER, 4, TestRow(-2147483648, 8, 2147483647, null)) testOutOfRangeValue("2147483648", IntegerType.INTEGER) testOutOfRangeValue("-2147483649", IntegerType.INTEGER) } test("parse long type") { testJsonParserForSingleType( jsonString = """{"col1":-9223372036854775808,"col2":8,"col3":9223372036854775807, "col4":null}""", dataType = LongType.LONG, 4, TestRow(-9223372036854775808L, 8L, 9223372036854775807L, null)) testOutOfRangeValue("9223372036854775808", LongType.LONG) testOutOfRangeValue("-9223372036854775809", LongType.LONG) } test("parse float type") { testJsonParserForSingleType( jsonString = """ |{"col1":-9223.33,"col2":0.4,"col3":1.2E8, |"col4":1.23E-7,"col5":0.004444444, "col6":null}""".stripMargin, dataType = FloatType.FLOAT, 6, TestRow(-9223.33f, 0.4f, 120000000.0f, 0.000000123f, 0.004444444f, null)) testOutOfRangeValue("3.4028235E+39", FloatType.FLOAT) } test("parse double type") { testJsonParserForSingleType( jsonString = """ |{"col1":-9.2233333333E8,"col2":0.4,"col3":1.2E8, |"col4":1.234444444E-7,"col5":0.0444444444, "col6":null}""".stripMargin, dataType = DoubleType.DOUBLE, 6, TestRow(-922333333.33d, 0.4d, 120000000.0d, 0.0000001234444444d, 0.0444444444d, null)) // For some reason out-of-range doubles are parsed initially as Double.INFINITY instead of // a BigDecimal val e = intercept[RuntimeException] { testJsonParserForSingleType( jsonString = s"""{"col1":1.7976931348623157E+309}""", dataType = DoubleType.DOUBLE, numColumns = 1, expectedRow = TestRow()) } assert(e.getMessage.contains(s"Couldn't decode")) } test("parse string type") { testJsonParserForSingleType( jsonString = """{"col1": "foo", "col2": "", "col3": null}""", dataType = StringType.STRING, 3, TestRow("foo", "", null)) } test("parse decimal type") { testJsonParserWithSchema( jsonString = """ |{ | "col1":0, | "col2":0.01234567891234567891234567891234567890, | "col3":123456789123456789123456789123456789, | "col4":1234567891234567891234567891.2345678900, | "col5":1.23, | "col6":null |} |""".stripMargin, schema = new StructType() .add("col1", DecimalType.USER_DEFAULT) .add("col2", new DecimalType(38, 38)) .add("col3", new DecimalType(38, 0)) .add("col4", new DecimalType(38, 10)) .add("col5", new DecimalType(5, 2)) .add("col6", new DecimalType(5, 2)), TestRow( new JBigDecimal(0), new JBigDecimal("0.01234567891234567891234567891234567890"), new JBigDecimal("123456789123456789123456789123456789"), new JBigDecimal("1234567891234567891234567891.2345678900"), new JBigDecimal("1.23"), null)) } test("parse date type") { testJsonParserForSingleType( jsonString = """{"col1":"2020-12-31", "col2":"1965-01-31", "col3": null}""", dataType = DateType.DATE, 3, TestRow(18627, -1796, null)) } test("parse timestamp type") { testJsonParserForSingleType( jsonString = """ |{ | "col1":"2050-01-01T00:00:00.000-08:00", | "col2":"1970-01-01T06:30:23.523Z", | "col3":"1960-01-01T10:00:00.000Z", | "col4":null | } | """.stripMargin, dataType = TimestampType.TIMESTAMP, numColumns = 4, TestRow(2524636800000000L, 23423523000L, -315583200000000L, null)) } test("parse timestamp type with large values") { // Timestamps far in the future should not cause overflow. // ChronoUnit.MICROS.between() internally computes nanoseconds first, which overflows // for timestamps more than ~292 years from epoch. testJsonParserForSingleType( jsonString = """{"col1":"9999-12-31T23:59:59.000+00:00"}""", dataType = TimestampType.TIMESTAMP, numColumns = 1, TestRow(253402300799000000L)) } test("parse null input") { val schema = new StructType() .add("nested_struct", new StructType().add("foo", IntegerType.INTEGER)) val batch = jsonHandler.parseJson( singletonStringColumnVector(null), schema, Optional.empty()) assert(batch.getColumnVector(0).getChild(0).isNullAt(0)) } test("parse NaN and INF for float and double") { def testSpecifiedString(json: String, output: TestRow): Unit = { testJsonParserWithSchema( jsonString = json, schema = new StructType() .add("col1", FloatType.FLOAT) .add("col2", DoubleType.DOUBLE), output) } testSpecifiedString("""{"col1":"NaN","col2":"NaN"}""", TestRow(Float.NaN, Double.NaN)) testSpecifiedString( """{"col1":"+INF","col2":"+INF"}""", TestRow(Float.PositiveInfinity, Double.PositiveInfinity)) testSpecifiedString( """{"col1":"+Infinity","col2":"+Infinity"}""", TestRow(Float.PositiveInfinity, Double.PositiveInfinity)) testSpecifiedString( """{"col1":"Infinity","col2":"Infinity"}""", TestRow(Float.PositiveInfinity, Double.PositiveInfinity)) testSpecifiedString( """{"col1":"-INF","col2":"-INF"}""", TestRow(Float.NegativeInfinity, Double.NegativeInfinity)) testSpecifiedString( """{"col1":"-Infinity","col2":"-Infinity"}""", TestRow(Float.NegativeInfinity, Double.NegativeInfinity)) } test("don't parse unselected rows") { val selectionVector = booleanVector(Seq(true, false, false)) val jsonVector = stringVector( Seq("""{"col1":1}""", """{"col1":"foo"}""", """{"col1":"foo"}""")) val batchRows = jsonHandler.parseJson( jsonVector, new StructType() .add("col1", IntegerType.INTEGER), Optional.of(selectionVector)).getRows.toSeq assert(!batchRows(0).isNullAt(0) && batchRows(0).getInt(0) == 1) assert(batchRows(1).isNullAt(0) && batchRows(2).isNullAt(0)) } test("read json files") { val expResults = Seq( TestRow("part-00000-d83dafd8-c344-49f0-ab1c-acd944e32493-c000.snappy.parquet", 348L, true), TestRow("part-00000-cb078bc1-0aeb-46ed-9cf8-74a843b32c8c-c000.snappy.parquet", 687L, true), TestRow("part-00001-9bf4b8f8-1b95-411b-bf10-28dc03aa9d2f-c000.snappy.parquet", 705L, true), TestRow("part-00000-0441e99a-c421-400e-83a1-212aa6c84c73-c000.snappy.parquet", 650L, true), TestRow("part-00001-34c8c673-3f44-4fa7-b94e-07357ec28a7d-c000.snappy.parquet", 650L, true), TestRow("part-00000-842017c2-3e02-44b5-a3d6-5b9ae1745045-c000.snappy.parquet", 649L, true), TestRow("part-00001-e62ca5a1-923c-4ee6-998b-c61d1cfb0b1c-c000.snappy.parquet", 649L, true)) Seq( ( fsClient.listFrom(getTestResourceFilePath("json-files/1.json")), expResults), ( fsClient.listFrom(getTestResourceFilePath("json-files-with-empty/1.json")), expResults), ( fsClient.listFrom(getTestResourceFilePath("json-files-with-empty/5.json")), expResults.takeRight(2)), ( fsClient.listFrom(getTestResourceFilePath("json-files-all-empty/1.json")), Seq())).foreach { case (testFiles, expResults) => val actResult = jsonHandler.readJsonFiles( testFiles, new StructType() .add("path", StringType.STRING) .add("size", LongType.LONG) .add("dataChange", BooleanType.BOOLEAN), Optional.empty()).toSeq.map(batch => TestRow(batch.getRows.next)) checkAnswer(actResult, expResults) } } test("parse json content") { val input = """ |{ | "path":"part-00000-d83dafd8-c344-49f0-ab1c-acd944e32493-c000.snappy.parquet", | "partitionValues":{"p1" : "0", "p2" : "str"}, | "size":348, | "modificationTime":1603723974000, | "dataChange":true |} |""".stripMargin val readSchema = new StructType() .add("path", StringType.STRING) .add("partitionValues", new MapType(StringType.STRING, StringType.STRING, false)) .add("size", LongType.LONG) .add("dataChange", BooleanType.BOOLEAN) val batch = jsonHandler.parseJson( singletonStringColumnVector(input), readSchema, Optional.empty[ColumnVector]()) assert(batch.getSize == 1) val actResult = Seq(TestRow(batch.getRows.next)) val expResult = Seq(TestRow( "part-00000-d83dafd8-c344-49f0-ab1c-acd944e32493-c000.snappy.parquet", Map("p1" -> "0", "p2" -> "str"), 348L, true)) checkAnswer(actResult, expResult) } test("parse nested complex types") { val json = """ |{ | "array": [0, 1, null], | "nested_array": [["a", "b"], ["c"], []], | "map": {"a": true, "b": false}, | "nested_map": {"a": {"one": [], "two": [1, 2, 3]}, "b": {}}, | "array_of_struct": [{"field1": "foo", "field2": 3}, {"field1": null}] |} |""".stripMargin val schema = new StructType() .add("array", new ArrayType(IntegerType.INTEGER, true)) .add("nested_array", new ArrayType(new ArrayType(StringType.STRING, true), true)) .add("map", new MapType(StringType.STRING, BooleanType.BOOLEAN, true)) .add( "nested_map", new MapType( StringType.STRING, new MapType(StringType.STRING, new ArrayType(IntegerType.INTEGER, true), true), true)) .add( "array_of_struct", new ArrayType( new StructType() .add("field1", StringType.STRING, true) .add("field2", IntegerType.INTEGER, true), true)) val batch = jsonHandler.parseJson( singletonStringColumnVector(json), schema, Optional.empty[ColumnVector]()) val actResult = Seq(TestRow(batch.getRows.next)) val expResult = Seq(TestRow( Vector(0, 1, null), Vector(Vector("a", "b"), Vector("c"), Vector()), Map("a" -> true, "b" -> false), Map( "a" -> Map( "one" -> Vector(), "two" -> Vector(1, 2, 3)), "b" -> Map()), Vector(TestRow.fromSeq(Seq("foo", 3)), TestRow.fromSeq(Seq(null, null))))) checkAnswer(actResult, expResult) } test("write rows as json") { withTempDir { tempDir => val input = Seq( """{ | "add": | { | "path":"part-00000-d83dafd8-c344-49f0-ab1c-acd944e32493-c000.snappy.parquet", | "partitionValues":{"p1" : "0", "p2" : "str"}, | "size":348, | "dataChange":true | } |} |""".stripMargin.linesIterator.mkString, """{ | "remove": | { | "path":"part-00000-d83dafd8-c344-49f0-ab1c-acd944e32493-c000.snappy.parquet", | "partitionValues":{"p1" : "0", "p2" : "str"}, | "size":348, | "dataChange":true | } |} |""".stripMargin.linesIterator.mkString) val addRemoveSchema = new StructType() .add("path", StringType.STRING) .add("partitionValues", new MapType(StringType.STRING, StringType.STRING, false)) .add("size", LongType.LONG) .add("dataChange", BooleanType.BOOLEAN) val readSchema = new StructType() .add("add", addRemoveSchema) .add("remove", addRemoveSchema) val batch = jsonHandler.parseJson(stringVector(input), readSchema, Optional.empty()) assert(batch.getSize == 2) val filePath = tempDir + "/1.json" def writeAndVerify(overwrite: Boolean): Unit = { jsonHandler.writeJsonFileAtomically(filePath, batch.getRows, overwrite) // read it back and verify the contents are correct val source = scala.io.Source.fromFile(filePath) val result = try source.getLines().mkString(",") finally source.close() // remove the whitespaces from the input to compare assert(input.map(_.replaceAll(" ", "")).mkString(",") === result) } writeAndVerify(overwrite = false) // Try to write as same file with overwrite as false and expect an error intercept[FileAlreadyExistsException] { jsonHandler.writeJsonFileAtomically(filePath, batch.getRows, false /* overwrite */ ) } // Try to write as file with overwrite set to true writeAndVerify(overwrite = true) } } test("parse diverse type values in a map[string, string]") { val input = """ |{ | "inCommitTimestamp":1740009523401, | "timestamp":1740009523401, | "engineInfo":"myengine.com", | "operation":"WRITE", | "operationParameters": | {"mode":"Append","statsOnLoad":false,"partitionBy":"[]"}, | "isBlindAppend":true, | "txnId":"cb009f42-5da1-4e7e-b4fa-09de3332f52a", | "operationMetrics": { | "numFiles":"1", | "serializedAsNumber":2, | "serializedAsBoolean":true | } |} |""".stripMargin val output = jsonHandler.parseJson( stringVector(Seq(input)), CommitInfo.FULL_SCHEMA, Optional.empty()) assert(output.getSize == 1) val actResult = TestRow(output.getRows.next) val expResult = TestRow( 1740009523401L, 1740009523401L, "myengine.com", "WRITE", Map("mode" -> "Append", "statsOnLoad" -> "false", "partitionBy" -> "[]"), true, "cb009f42-5da1-4e7e-b4fa-09de3332f52a", Map("numFiles" -> "1", "serializedAsNumber" -> "2", "serializedAsBoolean" -> "true")) checkAnswer(Seq(actResult), Seq(expResult)) } test("parse CommitInfo JSON with missing isBlindAppend field") { val input = """ |{ | "inCommitTimestamp":1740009523401, | "timestamp":1740009523401, | "engineInfo":"myengine.com", | "operation":"WRITE", | "operationParameters": | {"mode":"Append","partitionBy":"[]"}, | "txnId":"cb009f42-5da1-4e7e-b4fa-09de3332f52a", | "operationMetrics": { | "numFiles":"1" | } |} |""".stripMargin val output = jsonHandler.parseJson( stringVector(Seq(input)), CommitInfo.FULL_SCHEMA, Optional.empty()) assert(output.getSize == 1) val actResult = TestRow(output.getRows.next) val expResult = TestRow( 1740009523401L, 1740009523401L, "myengine.com", "WRITE", Map("mode" -> "Append", "partitionBy" -> "[]"), null, // isBlindAppend is missing from JSON, should be null "cb009f42-5da1-4e7e-b4fa-09de3332f52a", Map("numFiles" -> "1")) checkAnswer(Seq(actResult), Seq(expResult)) } test("fromColumnVector handles null isBlindAppend from parsed JSON without NPE") { val input = """ |{ | "timestamp":1740009523401, | "engineInfo":"myengine.com", | "operation":"WRITE", | "operationParameters":{}, | "txnId":"test-txn-id", | "operationMetrics":{} |} |""".stripMargin val readSchema = new StructType().add("commitInfo", CommitInfo.FULL_SCHEMA) val output = jsonHandler.parseJson( stringVector(Seq(s"""{"commitInfo":${input.trim}}""")), readSchema, Optional.empty()) assert(output.getSize == 1) val commitInfoVector = output.getColumnVector(0) val commitInfo = CommitInfo.fromColumnVector(commitInfoVector, 0) assert(commitInfo != null) assert(commitInfo.getIsBlindAppend === Optional.empty()) assert(commitInfo.getInCommitTimestamp === Optional.empty()) assert(commitInfo.getTimestamp === 1740009523401L) assert(commitInfo.getEngineInfo === Optional.of("myengine.com")) assert(commitInfo.getOperation === Optional.of("WRITE")) assert(commitInfo.getTxnId === Optional.of("test-txn-id")) } test("fromColumnVector handles null engineInfo, operation, and txnId without NPE") { // Simulates a commit written by an external engine that omits these optional fields val input = """ |{ | "timestamp":1740009523401, | "operationParameters":{}, | "operationMetrics":{} |} |""".stripMargin val readSchema = new StructType().add("commitInfo", CommitInfo.FULL_SCHEMA) val output = jsonHandler.parseJson( stringVector(Seq(s"""{"commitInfo":${input.trim}}""")), readSchema, Optional.empty()) assert(output.getSize == 1) val commitInfoVector = output.getColumnVector(0) val commitInfo = CommitInfo.fromColumnVector(commitInfoVector, 0) assert(commitInfo != null) assert(commitInfo.getEngineInfo === Optional.empty()) assert(commitInfo.getOperation === Optional.empty()) assert(commitInfo.getTxnId === Optional.empty()) assert(commitInfo.getIsBlindAppend === Optional.empty()) assert(commitInfo.getInCommitTimestamp === Optional.empty()) assert(commitInfo.getTimestamp === 1740009523401L) assert(commitInfo.getOperationParameters.isEmpty) assert(commitInfo.getOperationMetrics.isEmpty) } test("fromColumnVector with only timestamp field does not NPE") { // Minimal commit info - only the required timestamp field val input = """ |{ | "timestamp":1000 |} |""".stripMargin val readSchema = new StructType().add("commitInfo", CommitInfo.FULL_SCHEMA) val output = jsonHandler.parseJson( stringVector(Seq(s"""{"commitInfo":${input.trim}}""")), readSchema, Optional.empty()) assert(output.getSize == 1) val commitInfoVector = output.getColumnVector(0) val commitInfo = CommitInfo.fromColumnVector(commitInfoVector, 0) assert(commitInfo != null) assert(commitInfo.getTimestamp === 1000L) assert(commitInfo.getEngineInfo === Optional.empty()) assert(commitInfo.getOperation === Optional.empty()) assert(commitInfo.getTxnId === Optional.empty()) assert(commitInfo.getIsBlindAppend === Optional.empty()) assert(commitInfo.getInCommitTimestamp === Optional.empty()) assert(commitInfo.getOperationParameters.isEmpty) assert(commitInfo.getOperationMetrics.isEmpty) } test("CommitInfo round-trips through toRow with nullable fields") { val commitInfo = new CommitInfo( Optional.of(100L), 200L, Optional.empty(), // engineInfo Optional.empty(), // operation Collections.emptyMap(), Optional.empty(), // isBlindAppend Optional.empty(), // txnId Collections.emptyMap()) val row = commitInfo.toRow() assert(row.isNullAt(CommitInfo.FULL_SCHEMA.indexOf("engineInfo"))) assert(row.isNullAt(CommitInfo.FULL_SCHEMA.indexOf("operation"))) assert(row.isNullAt(CommitInfo.FULL_SCHEMA.indexOf("txnId"))) assert(row.isNullAt(CommitInfo.FULL_SCHEMA.indexOf("isBlindAppend"))) assert(row.getLong(CommitInfo.FULL_SCHEMA.indexOf("inCommitTimestamp")) === 100L) assert(row.getLong(CommitInfo.FULL_SCHEMA.indexOf("timestamp")) === 200L) } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/engine/DefaultParquetHandlerSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.engine import java.io.IOException import java.nio.file.FileAlreadyExistsException import scala.collection.JavaConverters._ import io.delta.golden.GoldenTableUtils.goldenTableFile import io.delta.kernel.defaults.engine.hadoopio.HadoopFileIO import io.delta.kernel.defaults.internal.parquet.ParquetSuiteBase import io.delta.kernel.internal.util.Utils.toCloseableIterator import org.apache.hadoop.conf.Configuration import org.scalatest.funsuite.AnyFunSuite class DefaultParquetHandlerSuite extends AnyFunSuite with ParquetSuiteBase { val parquetHandler = new DefaultParquetHandler( new HadoopFileIO( new Configuration { set("delta.kernel.default.parquet.reader.batch-size", "10") })) ///////////////////////////////////////////////////////////////////////////////////////////////// // Tests for `writeParquetFileAtomically`. Test for `writeParquetFiles` are covered in // `ParquetFileWriterSuite` as this API implementation by itself doesn't have any special logic. ///////////////////////////////////////////////////////////////////////////////////////////////// test("atomic write of a single Parquet file") { withTempDir { tempDir => val inputLocation = goldenTableFile("parquet-all-types").toString val dataToWrite = readParquetUsingKernelAsColumnarBatches(inputLocation, tableSchema(inputLocation)) .map(_.toFiltered) assert(dataToWrite.size === 1) assert(dataToWrite.head.getData.getSize === 200) val filePath = tempDir + "/1.parquet" def writeAndVerify(): Unit = { parquetHandler.writeParquetFileAtomically( filePath, toCloseableIterator(dataToWrite.asJava.iterator())) // Uses both Spark and Kernel to read and verify the content is same as the one written. verifyContent(tempDir.getAbsolutePath, dataToWrite) } writeAndVerify() // Try to write as same file and expect an error val e = intercept[IOException] { parquetHandler.writeParquetFileAtomically( filePath, toCloseableIterator(dataToWrite.asJava.iterator())) } assert(e.getCause.isInstanceOf[FileAlreadyExistsException]) } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluatorSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.expressions import java.lang.{Boolean => BooleanJ, Double => DoubleJ, Float => FloatJ, Integer => IntegerJ, Long => LongJ} import java.math.{BigDecimal => BigDecimalJ} import java.sql.{Date, Timestamp} import java.util import java.util.Optional import scala.jdk.CollectionConverters._ import io.delta.kernel.data.{ColumnarBatch, ColumnVector} import io.delta.kernel.defaults.internal.data.DefaultColumnarBatch import io.delta.kernel.defaults.internal.data.vector.{DefaultIntVector, DefaultStructVector} import io.delta.kernel.defaults.utils.DefaultKernelTestUtils.getValueAsObject import io.delta.kernel.expressions._ import io.delta.kernel.expressions.AlwaysFalse.ALWAYS_FALSE import io.delta.kernel.expressions.AlwaysTrue.ALWAYS_TRUE import io.delta.kernel.expressions.Literal._ import io.delta.kernel.internal.util.InternalUtils import io.delta.kernel.types._ import io.delta.kernel.types.CollationIdentifier.SPARK_UTF8_BINARY import org.scalatest.funsuite.AnyFunSuite class DefaultExpressionEvaluatorSuite extends AnyFunSuite with ExpressionSuiteBase { test("evaluate expression: literal") { val testLiterals = Seq( Literal.ofBoolean(true), Literal.ofBoolean(false), Literal.ofNull(BooleanType.BOOLEAN), ofByte(24.toByte), Literal.ofNull(ByteType.BYTE), Literal.ofShort(876.toShort), Literal.ofNull(ShortType.SHORT), Literal.ofInt(2342342), Literal.ofNull(IntegerType.INTEGER), Literal.ofLong(234234223L), Literal.ofNull(LongType.LONG), Literal.ofFloat(23423.4223f), Literal.ofNull(FloatType.FLOAT), Literal.ofDouble(23423.422233d), Literal.ofNull(DoubleType.DOUBLE), Literal.ofString("string_val"), Literal.ofNull(StringType.STRING), Literal.ofBinary("binary_val".getBytes), Literal.ofNull(BinaryType.BINARY), Literal.ofDate(4234), Literal.ofNull(DateType.DATE), Literal.ofTimestamp(2342342342232L), Literal.ofNull(TimestampType.TIMESTAMP), Literal.ofTimestampNtz(2342342342L), Literal.ofNull(TimestampNTZType.TIMESTAMP_NTZ)) val inputBatches: Seq[ColumnarBatch] = Seq[ColumnarBatch]( zeroColumnBatch(rowCount = 0), zeroColumnBatch(rowCount = 25), zeroColumnBatch(rowCount = 128)) for (literal <- testLiterals) { val outputDataType = literal.getDataType for (inputBatch <- inputBatches) { val outputVector: ColumnVector = evaluator(inputBatch.getSchema, literal, literal.getDataType) .eval(inputBatch) assert(inputBatch.getSize === outputVector.getSize) assert(outputDataType === outputVector.getDataType) for (rowId <- 0 until outputVector.getSize) { if (literal.getValue == null) { assert( outputVector.isNullAt(rowId), s"expected a null at $rowId for $literal expression") } else { assert( literal.getValue === getValueAsObject(outputVector, rowId), s"invalid value at $rowId for $literal expression") } } } } } PRIMITIVE_TYPES.foreach { dataType => test(s"evaluate expression: column of type $dataType") { val batchSize = 78; val batchSchema = new StructType().add("col1", dataType) val batch = new DefaultColumnarBatch( batchSize, batchSchema, Array[ColumnVector](testColumnVector(batchSize, dataType))) val outputVector = evaluator(batchSchema, new Column("col1"), dataType) .eval(batch) assert(batchSize === outputVector.getSize) assert(dataType === outputVector.getDataType) Seq.range(0, outputVector.getSize).foreach { rowId => assert( testIsNullValue(dataType, rowId) === outputVector.isNullAt(rowId), s"unexpected nullability at $rowId for $dataType type vector") if (!outputVector.isNullAt(rowId)) { assert( testColumnValue(dataType, rowId) === getValueAsObject(outputVector, rowId), s"unexpected value at $rowId for $dataType type vector") } } } } test("evaluate expression: nested column reference") { val col3Type = IntegerType.INTEGER val col2Type = new StructType().add("col3", col3Type) val col1Type = new StructType().add("col2", col2Type) val batchSchema = new StructType().add("col1", col1Type) val numRows = 5 val col3Nullability = Seq(false, true, false, true, false).toArray val col3Values = Seq(27, 24, 29, 100, 125).toArray val col3Vector = new DefaultIntVector(col3Type, numRows, Optional.of(col3Nullability), col3Values) val col2Nullability = Seq(false, true, true, true, false).toArray val col2Vector = new DefaultStructVector(numRows, col2Type, Optional.of(col2Nullability), Array(col3Vector)) val col1Nullability = Seq(false, false, false, true, false).toArray val col1Vector = new DefaultStructVector(numRows, col1Type, Optional.of(col1Nullability), Array(col2Vector)) val batch = new DefaultColumnarBatch(numRows, batchSchema, Array(col1Vector)) def assertTypeAndNullability( actVector: ColumnVector, expType: DataType, expNullability: Array[Boolean]): Unit = { assert(actVector.getDataType === expType) assert(actVector.getSize === numRows) Seq.range(0, numRows).foreach { rowId => assert(actVector.isNullAt(rowId) === expNullability(rowId)) } } val col3Ref = new Column(Array("col1", "col2", "col3")) val col3RefResult = evaluator(batchSchema, col3Ref, col3Type).eval(batch) assertTypeAndNullability(col3RefResult, col3Type, col3Nullability); Seq.range(0, numRows).foreach { rowId => assert(col3RefResult.getInt(rowId) === col3Values(rowId)) } val col2Ref = new Column(Array("col1", "col2")) val col2RefResult = evaluator(batchSchema, col2Ref, col2Type).eval(batch) assertTypeAndNullability(col2RefResult, col2Type, col2Nullability) val col1Ref = new Column(Array("col1")) val col1RefResult = evaluator(batchSchema, col1Ref, col1Type).eval(batch) assertTypeAndNullability(col1RefResult, col1Type, col1Nullability) // try to reference non-existent nested column val colNotValid = new Column(Array("col1", "colX`X")) val ex = intercept[IllegalArgumentException] { evaluator(batchSchema, colNotValid, col1Type).eval(batch) } assert(ex.getMessage.contains("column(`col1`.`colX``X`) doesn't exist in input data schema")) } test("evaluate expression: always true, always false") { Seq(ALWAYS_TRUE, ALWAYS_FALSE).foreach { expr => val batch = zeroColumnBatch(rowCount = 87) val outputVector = evaluator(batch.getSchema, expr, BooleanType.BOOLEAN).eval(batch) assert(outputVector.getSize === 87) assert(outputVector.getDataType === BooleanType.BOOLEAN) Seq.range(0, 87).foreach { rowId => assert(!outputVector.isNullAt(rowId)) assert(outputVector.getBoolean(rowId) == (expr == ALWAYS_TRUE)) } } } test("evaluate expression: and, or") { val leftColumn = booleanVector( Seq[BooleanJ](true, true, false, false, null, true, null, false, null)) val rightColumn = booleanVector( Seq[BooleanJ](true, false, false, true, true, null, false, null, null)) val expAndOutputVector = booleanVector( Seq[BooleanJ](true, false, false, false, null, null, false, false, null)) val expOrOutputVector = booleanVector( Seq[BooleanJ](true, true, false, true, true, true, null, null, null)) val schema = new StructType() .add("left", BooleanType.BOOLEAN) .add("right", BooleanType.BOOLEAN) val batch = new DefaultColumnarBatch(leftColumn.getSize, schema, Array(leftColumn, rightColumn)) val left = comparator("=", new Column("left"), Literal.ofBoolean(true)) val right = comparator("=", new Column("right"), Literal.ofBoolean(true)) // And val andExpression = and(left, right) val actAndOutputVector = evaluator(schema, andExpression, BooleanType.BOOLEAN).eval(batch) checkBooleanVectors(actAndOutputVector, expAndOutputVector) // Or val orExpression = or(left, right) val actOrOutputVector = evaluator(schema, orExpression, BooleanType.BOOLEAN).eval(batch) checkBooleanVectors(actOrOutputVector, expOrOutputVector) } test("evaluate expression: not") { val childColumn = booleanVector(Seq[BooleanJ](true, false, null)) val schema = new StructType().add("child", BooleanType.BOOLEAN) val batch = new DefaultColumnarBatch(childColumn.getSize, schema, Array(childColumn)) val notExpression = new Predicate( "NOT", comparator("=", new Column("child"), Literal.ofBoolean(true))) val expOutputVector = booleanVector(Seq[BooleanJ](false, true, null)) val actOutputVector = evaluator(schema, notExpression, BooleanType.BOOLEAN).eval(batch) checkBooleanVectors(actOutputVector, expOutputVector) } test("evaluate expression: is not null") { val childColumn = booleanVector(Seq[BooleanJ](true, false, null)) val schema = new StructType().add("child", BooleanType.BOOLEAN) val batch = new DefaultColumnarBatch(childColumn.getSize, schema, Array(childColumn)) val isNotNullExpression = new Predicate("IS_NOT_NULL", new Column("child")) val expOutputVector = booleanVector(Seq[BooleanJ](true, true, false)) val actOutputVector = evaluator(schema, isNotNullExpression, BooleanType.BOOLEAN).eval(batch) checkBooleanVectors(actOutputVector, expOutputVector) } test("evaluate expression: is null") { val childColumn = booleanVector(Seq[BooleanJ](true, false, null)) val schema = new StructType().add("child", BooleanType.BOOLEAN) val batch = new DefaultColumnarBatch(childColumn.getSize, schema, Array(childColumn)) val isNullExpression = new Predicate("IS_NULL", new Column("child")) val expOutputVector = booleanVector(Seq[BooleanJ](false, false, true)) val actOutputVector = evaluator(schema, isNullExpression, BooleanType.BOOLEAN).eval(batch) checkBooleanVectors(actOutputVector, expOutputVector) } test("evaluate expression: coalesce (boolean columns)") { val col1 = booleanVector(Seq[BooleanJ](true, null, null, null)) val col2 = booleanVector(Seq[BooleanJ](false, false, null, null)) val col3 = booleanVector(Seq[BooleanJ](true, true, true, null)) val schema = new StructType() .add("col1", BooleanType.BOOLEAN) .add("col2", BooleanType.BOOLEAN) .add("col3", BooleanType.BOOLEAN) val batch = new DefaultColumnarBatch(col1.getSize, schema, Array(col1, col2, col3)) val coalesceEpxr1 = new ScalarExpression( "COALESCE", util.Arrays.asList(new Column("col1"))) val expOutputVector1 = booleanVector(Seq[BooleanJ](true, null, null, null)) val actOutputVector1 = evaluator(schema, coalesceEpxr1, BooleanType.BOOLEAN).eval(batch) checkBooleanVectors(actOutputVector1, expOutputVector1) val coalesceEpxr3 = new ScalarExpression( "COALESCE", util.Arrays.asList( new Column("col1"), new Column("col2"), new Column("col3"))) val expOutputVector3 = booleanVector(Seq[BooleanJ](true, false, true, null)) val actOutputVector3 = evaluator(schema, coalesceEpxr3, BooleanType.BOOLEAN).eval(batch) checkBooleanVectors(actOutputVector3, expOutputVector3) } test("evaluate expression: coalesce (long columns)") { val longCol1 = longVector(Seq(1L, null, null, 4L)) val longCol2 = longVector(Seq(null, 2L, null, 5L)) val longCol3 = longVector(Seq(100L, null, 3L, null)) val longSchema = new StructType() .add("longCol1", LongType.LONG) .add("longCol2", LongType.LONG) .add("longCol3", LongType.LONG) val longBatch = new DefaultColumnarBatch(longCol1.getSize, longSchema, Array(longCol1, longCol2, longCol3)) val longCoalesceExpr = new ScalarExpression( "COALESCE", util.Arrays.asList(new Column("longCol1"), new Column("longCol2"), new Column("longCol3"))) val expLongOutput = longVector(Seq(1L, 2L, 3L, 4L)) val actLongOutput = evaluator(longSchema, longCoalesceExpr, LongType.LONG).eval(longBatch) checkLongVectors(actLongOutput, expLongOutput) } test("evaluate expression: coalesce (string columns)") { val strCol1 = stringVector(Seq("a", null, null, "d")) val strCol2 = stringVector(Seq("null", "b", null, null)) val strCol3 = stringVector(Seq(null, null, "c", "abc")) val strSchema = new StructType() .add("strCol1", StringType.STRING) .add("strCol2", StringType.STRING) .add("strCol3", StringType.STRING) val strBatch = new DefaultColumnarBatch(strCol1.getSize, strSchema, Array(strCol1, strCol2, strCol3)) val strCoalesceExpr = new ScalarExpression( "COALESCE", util.Arrays.asList(new Column("strCol1"), new Column("strCol2"), new Column("strCol3"))) val expStrOutput = stringVector(Seq("a", "b", "c", "d")) val actStrOutput = evaluator(strSchema, strCoalesceExpr, StringType.STRING).eval(strBatch) checkStringVectors(actStrOutput, expStrOutput) } test("evaluate expression: coalesce (timestamp columns)") { val tsCol1 = timestampVector(Seq(1000L, null, null, 4000L)) val tsCol2 = timestampVector(Seq(null, 2000L, null, 5000L)) val tsCol3 = timestampVector(Seq(10000L, null, 3000L, null)) val tsSchema = new StructType() .add("tsCol1", TimestampType.TIMESTAMP) .add("tsCol2", TimestampType.TIMESTAMP) .add("tsCol3", TimestampType.TIMESTAMP) val tsBatch = new DefaultColumnarBatch(tsCol1.getSize, tsSchema, Array(tsCol1, tsCol2, tsCol3)) val tsCoalesceExpr = new ScalarExpression( "COALESCE", util.Arrays.asList(new Column("tsCol1"), new Column("tsCol2"), new Column("tsCol3"))) val expTsOutput = timestampVector(Seq(1000L, 2000L, 3000L, 4000L)) val actTsOutput = evaluator(tsSchema, tsCoalesceExpr, TimestampType.TIMESTAMP).eval(tsBatch) checkLongVectors(actTsOutput, expTsOutput) } test("evaluate expression: coalesce (unequal column types)") { def checkUnsupportedTypes( col1Type: DataType, col2Type: DataType, messageContains: String): Unit = { val schema = new StructType() .add("col1", col1Type) .add("col2", col2Type) val batch = new DefaultColumnarBatch( 5, schema, Array(testColumnVector(5, col1Type), testColumnVector(5, col2Type))) val e = intercept[UnsupportedOperationException] { evaluator( schema, new ScalarExpression( "COALESCE", util.Arrays.asList(new Column("col1"), new Column("col2"))), col1Type).eval(batch) } assert(e.getMessage.contains(messageContains)) } // TODO support least-common-type resolution checkUnsupportedTypes( LongType.LONG, IntegerType.INTEGER, "Coalesce is only supported for arguments of the same type") } test("evaluate expression: ADD (column and literal)") { val col1 = longVector(Seq(1, 2, 3, 4, null)) val schema = new StructType().add("col1", LongType.LONG) val batch = new DefaultColumnarBatch(col1.getSize, schema, Array(col1)) // ADD with literal val addExpr = new ScalarExpression( "ADD", util.Arrays.asList(new Column("col1"), Literal.ofLong(10L))) val expOutputVector = longVector(Seq(11, 12, 13, 14, null)) val actOutputVector = evaluator(schema, addExpr, LongType.LONG).eval(batch) checkLongVectors(actOutputVector, expOutputVector) } test("evaluate expression: ADD (column and column)") { val col1 = longVector(Seq(1, 2, null, 4, null)) val col2 = longVector(Seq(null, 20, 30, 40, null)) val schema = new StructType() .add("col1", LongType.LONG) .add("col2", LongType.LONG) val batch = new DefaultColumnarBatch(col1.getSize, schema, Array(col1, col2)) // ADD with two columns val addExpr = new ScalarExpression( "ADD", util.Arrays.asList(new Column("col1"), new Column("col2"))) val expOutputVector = longVector(Seq(null, 22, null, 44, null)) val actOutputVector = evaluator(schema, addExpr, LongType.LONG).eval(batch) checkLongVectors(actOutputVector, expOutputVector) } test("evaluate expression: ADD (more than two operands)") { val col1 = longVector(Seq(1, 2, null, 4, null)) val col2 = longVector(Seq(null, 20, 30, 40, null)) val col3 = longVector(Seq(5, null, 15, null, 25)) val schema = new StructType() .add("col1", LongType.LONG) .add("col2", LongType.LONG) .add("col3", LongType.LONG) val batch = new DefaultColumnarBatch(col1.getSize, schema, Array(col1, col2, col3)) // ADD with three columns val addExpr = new ScalarExpression( "ADD", util.Arrays.asList(new Column("col1"), new Column("col2"), new Column("col3"))) val e = intercept[UnsupportedOperationException] { evaluator(schema, addExpr, LongType.LONG).eval(batch) } assert(e.getMessage.contains("ADD requires exactly two arguments: left and right operands")) } test("evaluate expression: ADD (unequal operand types") { val col1 = longVector(Seq(1, 2, null, 4, null)) val col2 = floatVector(Seq(1.0f, 2.0f, 3.0f, 4.0f, null)) val schema = new StructType() .add("col1", LongType.LONG) .add("col2", FloatType.FLOAT) val batch = new DefaultColumnarBatch(col1.getSize, schema, Array(col1, col2)) // ADD with two columns of different types val addExpr = new ScalarExpression( "ADD", util.Arrays.asList(new Column("col1"), new Column("col2"))) val e = intercept[UnsupportedOperationException] { evaluator(schema, addExpr, LongType.LONG).eval(batch) } assert(e.getMessage.contains("ADD is only supported for arguments of the same type")) } test("evaluate expression: ADD (unsupported types)") { val col1 = stringVector(Seq("a", "b", null, "d", null)) val col2 = stringVector(Seq("x", "y", "z", "w", null)) val schema = new StructType() .add("col1", StringType.STRING) .add("col2", StringType.STRING) val batch = new DefaultColumnarBatch(col1.getSize, schema, Array(col1, col2)) // ADD with two columns of unsupported types val addExpr = new ScalarExpression( "ADD", util.Arrays.asList(new Column("col1"), new Column("col2"))) val e = intercept[UnsupportedOperationException] { evaluator(schema, addExpr, StringType.STRING).eval(batch) } assert(e.getMessage.contains( "ADD is only supported for numeric types: byte, short, int, long, float, double")) } test("evaluate expression: TIMEADD with TIMESTAMP columns") { val timestampColumn = timestampVector(Seq( 1577836800000000L, // 2020-01-01 00:00:00.000 1577836800123456L, // 2020-01-01 00:00:00.123456 -1 // Representing null )) val durationColumn = longVector(Seq( 1000, // 1 second in milliseconds 100, // 0.1 second in milliseconds -1)) val schema = new StructType() .add("timestamp", TimestampType.TIMESTAMP) .add("duration", LongType.LONG) val batch = new DefaultColumnarBatch( timestampColumn.getSize, schema, Array(timestampColumn, durationColumn)) // TimeAdd expression adds milliseconds to timestamps val timeAddExpr = new ScalarExpression( "TIMEADD", util.Arrays.asList(new Column("timestamp"), new Column("duration"))) val expOutputVector = timestampVector(Seq( 1577836801000000L, // 2020-01-01 00:00:01.000 1577836800123456L + 100000, // 2020-01-01 00:00:00.123556 -1 // Null should propagate )) val actOutputVector = evaluator(schema, timeAddExpr, TimestampType.TIMESTAMP).eval(batch) checkTimestampVectors(actOutputVector, expOutputVector) } def checkUnsupportedTimeAddTypes( col1Type: DataType, col2Type: DataType): Unit = { val schema = new StructType() .add("timestamp", col1Type) .add("duration", col2Type) val batch = new DefaultColumnarBatch( 5, schema, Array(testColumnVector(5, col1Type), testColumnVector(5, col2Type))) val timeAddExpr = new ScalarExpression( "TIMEADD", util.Arrays.asList(new Column("timestamp"), new Column("duration"))) val e = intercept[IllegalArgumentException] { val evaluator = new DefaultExpressionEvaluator(schema, timeAddExpr, col1Type) evaluator.eval(batch) } assert(e.getMessage.contains("TIMEADD requires a timestamp and a Long")) } // Test to ensure TIMEADD requires the first argument to be a TimestampType // and the second to be a LongType test("TIMEADD with unsupported types") { // Check invalid timestamp column type checkUnsupportedTimeAddTypes( IntegerType.INTEGER, IntegerType.INTEGER) // Check invalid duration column type checkUnsupportedTimeAddTypes( TimestampType.TIMESTAMP, StringType.STRING) // Check valid type but with unsupported operations checkUnsupportedTimeAddTypes( TimestampType.TIMESTAMP, FloatType.FLOAT) } test("evaluate expression: like") { val col1 = stringVector(Seq[String]( null, "one", "two", "three", "four", null, null, "seven", "eight")) val col2 = stringVector(Seq[String]( null, "one", "Two", "thr%", "four%", "f", null, null, "%ght")) val schema = new StructType() .add("col1", StringType.STRING) .add("col2", StringType.STRING) val input = new DefaultColumnarBatch(col1.getSize, schema, Array(col1, col2)) def checkLike( input: DefaultColumnarBatch, likeExpression: Predicate, expOutputSeq: Seq[BooleanJ]): Unit = { val actOutputVector = new DefaultExpressionEvaluator( schema, likeExpression, BooleanType.BOOLEAN).eval(input) val expOutputVector = booleanVector(expOutputSeq); checkBooleanVectors(actOutputVector, expOutputVector) } // check column expressions on both sides checkLike( input, like(new Column("col1"), new Column("col2")), Seq[BooleanJ](null, true, false, true, true, null, null, null, true)) // check column expression against literal checkLike( input, like(new Column("col1"), Literal.ofString("t%")), Seq[BooleanJ](null, false, true, true, false, null, null, false, false)) // ends with checks checkLike( input, like(new Column("col1"), Literal.ofString("%t")), Seq[BooleanJ](null, false, false, false, false, null, null, false, true)) // contains checks checkLike( input, like(new Column("col1"), Literal.ofString("%t%")), Seq[BooleanJ](null, false, true, true, false, null, null, false, true)) val dummyInput = new DefaultColumnarBatch( 1, new StructType().add("dummy", StringType.STRING), Array(stringVector(Seq[String]("")))) def checkLikeLiteral( left: String, right: String, escape: Character = null, expOutput: BooleanJ): Unit = { val expression = like(Literal.ofString(left), Literal.ofString(right), Option(escape)) checkLike(dummyInput, expression, Seq[BooleanJ](expOutput)) } // null/empty checkLikeLiteral(null, "a", null, null) checkLikeLiteral("a", null, null, null) checkLikeLiteral(null, null, null, null) checkLikeLiteral("", "", null, true) checkLikeLiteral("a", "", null, false) checkLikeLiteral("", "a", null, false) Seq('!', '@', '#').foreach { escape => { // simple patterns checkLikeLiteral("abc", "abc", escape, true) checkLikeLiteral("a_%b", s"a${escape}__b", escape, true) checkLikeLiteral("abbc", "a_%c", escape, true) checkLikeLiteral("abbc", s"a${escape}__c", escape, false) checkLikeLiteral("abbc", s"a%${escape}%c", escape, false) checkLikeLiteral("a_%b", s"a%${escape}%b", escape, true) checkLikeLiteral("abbc", "a%", escape, true) checkLikeLiteral("abbc", "**", escape, false) checkLikeLiteral("abc", "a%", escape, true) checkLikeLiteral("abc", "b%", escape, false) checkLikeLiteral("abc", "bc%", escape, false) checkLikeLiteral("a\nb", "a_b", escape, true) checkLikeLiteral("ab", "a%b", escape, true) checkLikeLiteral("a\nb", "a%b", escape, true) checkLikeLiteral("a\nb", "ab", escape, false) checkLikeLiteral("a\nb", "a\nb", escape, true) checkLikeLiteral("a\n\nb", "a\nb", escape, false) checkLikeLiteral("a\n\nb", "a\n_b", escape, true) // case checkLikeLiteral("A", "a%", escape, false) checkLikeLiteral("a", "a%", escape, true) checkLikeLiteral("a", "A%", escape, false) checkLikeLiteral(s"aAa", s"aA_", escape, true) // regex checkLikeLiteral("a([a-b]{2,4})a", "_([a-b]{2,4})%", null, true) checkLikeLiteral("a([a-b]{2,4})a", "_([a-c]{2,6})_", null, false) // %/_ checkLikeLiteral("a%a", s"%${escape}%%", escape, true) checkLikeLiteral("a%", s"%${escape}%%", escape, true) checkLikeLiteral("a%a", s"_${escape}%_", escape, true) checkLikeLiteral("a_a", s"%${escape}_%", escape, true) checkLikeLiteral("a_", s"%${escape}_%", escape, true) checkLikeLiteral("a_a", s"_${escape}__", escape, true) // double-escaping checkLikeLiteral( s"$escape$escape$escape$escape", s"%${escape}${escape}%", escape, true) checkLikeLiteral("%%", "%%", escape, true) checkLikeLiteral(s"${escape}__", s"${escape}${escape}${escape}__", escape, true) checkLikeLiteral(s"${escape}__", s"%${escape}${escape}%${escape}%", escape, false) checkLikeLiteral(s"_${escape}${escape}${escape}%", s"%${escape}${escape}", escape, false) } } // check '_' for escape char checkLikeLiteral("abc", "abc", '_', true) checkLikeLiteral("a_%b", s"a__%%b", '_', true) checkLikeLiteral("abbc", "a__c", '_', false) checkLikeLiteral("abbc", "a%%c", '_', true) checkLikeLiteral("abbc", s"a___%c", '_', false) checkLikeLiteral("abbc", s"a%_%c", '_', false) // check '%' for escape char checkLikeLiteral("abc", "abc", '%', true) checkLikeLiteral("a_%b", s"a__%%b", '%', false) checkLikeLiteral("a_%b", s"a_%%b", '%', true) checkLikeLiteral("abbc", "a__c", '%', true) checkLikeLiteral("abbc", "a%%c", '%', false) checkLikeLiteral("abbc", s"a%__c", '%', false) checkLikeLiteral("abbc", s"a%_%_c", '%', false) def checkUnsupportedTypes( col1Type: DataType, col2Type: DataType): Unit = { val schema = new StructType() .add("col1", col1Type) .add("col2", col2Type) val expr = like(new Column("col1"), new Column("col2"), Option(null)) val input = new DefaultColumnarBatch( 5, schema, Array(testColumnVector(5, col1Type), testColumnVector(5, col2Type))) val e = intercept[UnsupportedOperationException] { new DefaultExpressionEvaluator( schema, expr, BooleanType.BOOLEAN).eval(input) } assert(e.getMessage.contains("LIKE is only supported for string type expressions")) } checkUnsupportedTypes(BooleanType.BOOLEAN, BooleanType.BOOLEAN) checkUnsupportedTypes(LongType.LONG, LongType.LONG) checkUnsupportedTypes(IntegerType.INTEGER, IntegerType.INTEGER) checkUnsupportedTypes(StringType.STRING, BooleanType.BOOLEAN) checkUnsupportedTypes(StringType.STRING, IntegerType.INTEGER) checkUnsupportedTypes(StringType.STRING, LongType.LONG) checkUnsupportedTypes(BooleanType.BOOLEAN, BooleanType.BOOLEAN) // input count checks val inputCountCheckUserMessage = "Invalid number of inputs to LIKE expression. Example usage:" val inputCountError1 = intercept[UnsupportedOperationException] { val expression = like(List(Literal.ofString("a"))) checkLike(dummyInput, expression, Seq[BooleanJ](null)) } assert(inputCountError1.getMessage.contains(inputCountCheckUserMessage)) val inputCountError2 = intercept[UnsupportedOperationException] { val expression = like(List( Literal.ofString("a"), Literal.ofString("b"), Literal.ofString("c"), Literal.ofString("d"))) checkLike(dummyInput, expression, Seq[BooleanJ](null)) } assert(inputCountError2.getMessage.contains(inputCountCheckUserMessage)) // additional escape token checks val escapeCharError1 = intercept[UnsupportedOperationException] { val expression = like(List(Literal.ofString("a"), Literal.ofString("b"), Literal.ofString("~~"))) checkLike(dummyInput, expression, Seq[BooleanJ](null)) } assert(escapeCharError1.getMessage.contains( "LIKE expects escape token to be a single character")) val escapeCharError2 = intercept[UnsupportedOperationException] { val expression = like(List(Literal.ofString("a"), Literal.ofString("b"), Literal.ofInt(1))) checkLike(dummyInput, expression, Seq[BooleanJ](null)) } assert(escapeCharError2.getMessage.contains( "LIKE expects escape token expression to be a literal of String type")) // empty input checks val emptyInput = new DefaultColumnarBatch( 0, new StructType().add("dummy", StringType.STRING), Array(stringVector(Seq[String]("")))) checkLike( emptyInput, like(Literal.ofString("abc"), Literal.ofString("abc"), Some('_')), Seq[BooleanJ]()) // invalid pattern check val invalidPatternError = intercept[IllegalArgumentException] { checkLikeLiteral("abbc", "a%%%c", '%', false) } assert(invalidPatternError.getMessage.contains( "LIKE expression has invalid escape sequence")) } private val SPARK_UTF8_LCASE = CollationIdentifier.fromString("SPARK.UTF8_LCASE") test("evaluate expression: starts with") { Seq( // collation None, Some(SPARK_UTF8_BINARY)).foreach { collationIdentifier => val col1 = stringVector(Seq[String]("one", "two", "t%hree", "four", null, null, "%")) val col2 = stringVector(Seq[String]("o", "t", "T", "4", "f", null, null)) val schema = new StructType() .add("col1", StringType.STRING) .add("col2", new StringType(SPARK_UTF8_LCASE)) val input = new DefaultColumnarBatch(col1.getSize, schema, Array(col1, col2)) val startsWithExpressionLiteral = startsWith(new Column("col1"), Literal.ofString("t%"), collationIdentifier) val expOutputVectorLiteral = booleanVector(Seq[BooleanJ](false, false, true, false, null, null, false)) checkBooleanVectors( new DefaultExpressionEvaluator( schema, startsWithExpressionLiteral, BooleanType.BOOLEAN).eval(input), expOutputVectorLiteral) val startsWithExpressionNullLiteral = startsWith(new Column("col1"), Literal.ofString(null), collationIdentifier) val allNullVector = booleanVector(Seq[BooleanJ](null, null, null, null, null, null, null)) checkBooleanVectors( new DefaultExpressionEvaluator( schema, startsWithExpressionNullLiteral, BooleanType.BOOLEAN).eval(input), allNullVector) // Two literal expressions on both sides val startsWithExpressionAlwaysTrue = startsWith(Literal.ofString("ABC"), Literal.ofString("A"), collationIdentifier) val allTrueVector = booleanVector(Seq[BooleanJ](true, true, true, true, true, true, true)) checkBooleanVectors( new DefaultExpressionEvaluator( schema, startsWithExpressionAlwaysTrue, BooleanType.BOOLEAN).eval(input), allTrueVector) val startsWithExpressionAlwaysFalse = startsWith(Literal.ofString("ABC"), Literal.ofString("_B%"), collationIdentifier) val allFalseVector = booleanVector(Seq[BooleanJ](false, false, false, false, false, false, false)) checkBooleanVectors( new DefaultExpressionEvaluator( schema, startsWithExpressionAlwaysFalse, BooleanType.BOOLEAN).eval(input), allFalseVector) // scalastyle:off nonascii val colUnicode = stringVector(Seq[String]("中文", "中", "文")) val schemaUnicode = new StructType().add("col", StringType.STRING) val inputUnicode = new DefaultColumnarBatch(colUnicode.getSize, schemaUnicode, Array(colUnicode)) val startsWithExpressionUnicode = startsWith(new Column("col"), Literal.ofString("中"), collationIdentifier) val expOutputVectorLiteralUnicode = booleanVector(Seq[BooleanJ](true, true, false)) checkBooleanVectors( new DefaultExpressionEvaluator( schemaUnicode, startsWithExpressionUnicode, BooleanType.BOOLEAN).eval(inputUnicode), expOutputVectorLiteralUnicode) // scalastyle:off nonascii val colSurrogatePair = stringVector(Seq[String]("💕😉💕", "😉💕", "💕")) val schemaSurrogatePair = new StructType().add("col", StringType.STRING) val inputSurrogatePair = new DefaultColumnarBatch(colSurrogatePair.getSize, schemaUnicode, Array(colSurrogatePair)) val startsWithExpressionSurrogatePair = startsWith(new Column("col"), Literal.ofString("💕"), collationIdentifier) val expOutputVectorLiteralSurrogatePair = booleanVector(Seq[BooleanJ](true, false, true)) checkBooleanVectors( new DefaultExpressionEvaluator( schemaSurrogatePair, startsWithExpressionSurrogatePair, BooleanType.BOOLEAN).eval(inputSurrogatePair), expOutputVectorLiteralSurrogatePair) val startsWithExpressionExpression = startsWith(new Column("col1"), new Column("col2"), collationIdentifier) val e = intercept[UnsupportedOperationException] { new DefaultExpressionEvaluator( schema, startsWithExpressionExpression, BooleanType.BOOLEAN).eval(input) } assert(e.getMessage.contains("'STARTS_WITH' expects literal as the second input")) def checkUnsupportedTypes(colType: DataType, literalType: DataType): Unit = { val schema = new StructType() .add("col", colType) val expr = startsWith(new Column("col"), Literal.ofNull(literalType), collationIdentifier) val input = new DefaultColumnarBatch(5, schema, Array(testColumnVector(5, colType))) val e = intercept[UnsupportedOperationException] { new DefaultExpressionEvaluator( schema, expr, BooleanType.BOOLEAN).eval(input) } assert(e.getMessage.contains("'STARTS_WITH' expects STRING type inputs")) } checkUnsupportedTypes(BooleanType.BOOLEAN, BooleanType.BOOLEAN) checkUnsupportedTypes(LongType.LONG, LongType.LONG) checkUnsupportedTypes(IntegerType.INTEGER, IntegerType.INTEGER) checkUnsupportedTypes(StringType.STRING, BooleanType.BOOLEAN) checkUnsupportedTypes(StringType.STRING, IntegerType.INTEGER) checkUnsupportedTypes(StringType.STRING, LongType.LONG) } } test("evaluate expression: starts with (unsupported collations)") { Seq( Some(SPARK_UTF8_LCASE), Some(CollationIdentifier.fromString("ICU.sr_Cyrl_SRB")), Some(CollationIdentifier.fromString("ICU.sr_Cyrl_SRB.75.1"))).foreach { collationIdentifier => val col1 = stringVector(Seq[String]("one", "two", "t%hree", "four", null, null, "%")) val col2 = stringVector(Seq[String]("o", "t", "T", "4", "f", null, null)) val schema = new StructType() .add("col1", new StringType(SPARK_UTF8_LCASE)) .add("col2", StringType.STRING) val input = new DefaultColumnarBatch(col1.getSize, schema, Array(col1, col2)) val startsWithExpressionLiteral = startsWith(new Column("col1"), Literal.ofString("t%"), collationIdentifier) checkUnsupportedCollation( schema, startsWithExpressionLiteral, input, collationIdentifier.get) val startsWithExpressionNullLiteral = startsWith(new Column("col1"), Literal.ofString(null), collationIdentifier) checkUnsupportedCollation( schema, startsWithExpressionNullLiteral, input, collationIdentifier.get) // Two literal expressions on both sides val startsWithExpressionAlwaysTrue = startsWith(Literal.ofString("ABC"), Literal.ofString("A"), collationIdentifier) checkUnsupportedCollation( schema, startsWithExpressionAlwaysTrue, input, collationIdentifier.get) val startsWithExpressionAlwaysFalse = startsWith(Literal.ofString("ABC"), Literal.ofString("_B%"), collationIdentifier) checkUnsupportedCollation( schema, startsWithExpressionAlwaysFalse, input, collationIdentifier.get) // scalastyle:off nonascii val colUnicode = stringVector(Seq[String]("中文", "中", "文")) val schemaUnicode = new StructType().add("col", StringType.STRING) val inputUnicode = new DefaultColumnarBatch(colUnicode.getSize, schemaUnicode, Array(colUnicode)) val startsWithExpressionUnicode = startsWith(new Column("col"), Literal.ofString("中"), collationIdentifier) checkUnsupportedCollation( schemaUnicode, startsWithExpressionUnicode, inputUnicode, collationIdentifier.get) // scalastyle:off nonascii val colSurrogatePair = stringVector(Seq[String]("💕😉💕", "😉💕", "💕")) val schemaSurrogatePair = new StructType().add("col", StringType.STRING) val inputSurrogatePair = new DefaultColumnarBatch( colSurrogatePair.getSize, schemaSurrogatePair, Array(colSurrogatePair)) val startsWithExpressionSurrogatePair = startsWith(new Column("col"), Literal.ofString("💕"), collationIdentifier) checkUnsupportedCollation( schemaSurrogatePair, startsWithExpressionSurrogatePair, inputSurrogatePair, collationIdentifier.get) } } test("evaluate expression: basic case for in expression") { // Test with string values val col1 = stringVector(Seq[String]("one", "two", "three", "four", null, "five")) val schema = new StructType().add("col1", StringType.STRING) val input = new DefaultColumnarBatch(col1.getSize, schema, Array(col1)) // Basic case for string: col1 IN ("one", "three", "five") val inExpressionBasic = in( new Column("col1"), Literal.ofString("one"), Literal.ofString("three"), Literal.ofString("five")) val expOutputBasic = booleanVector(Seq[BooleanJ](true, false, true, false, null, true)) checkBooleanVectors( new DefaultExpressionEvaluator( schema, inExpressionBasic, BooleanType.BOOLEAN).eval(input), expOutputBasic) // IN test with no matches: col1 IN ("six", "seven") val inExpressionNoMatch = in( new Column("col1"), Literal.ofString("six"), Literal.ofString("seven")) val expOutputNoMatch = booleanVector(Seq[BooleanJ](false, false, false, false, null, false)) checkBooleanVectors( new DefaultExpressionEvaluator( schema, inExpressionNoMatch, BooleanType.BOOLEAN).eval(input), expOutputNoMatch) // IN test with NULL in list: col1 IN ("one", NULL, "three"), returns null if no matches. val inExpressionWithNull = in( new Column("col1"), Literal.ofString("one"), Literal.ofString(null), Literal.ofString("three")) val expOutputWithNull = booleanVector(Seq[BooleanJ](true, null, true, null, null, null)) checkBooleanVectors( new DefaultExpressionEvaluator( schema, inExpressionWithNull, BooleanType.BOOLEAN).eval(input), expOutputWithNull) // Test with float values: col IN (1.5f, 2.5f, 3.5f) val floatCol = floatVector(Seq[FloatJ](1.5f, 2.5f, 3.5f, 4.5f, null)) val floatSchema = new StructType().add("floatCol", FloatType.FLOAT) val floatInput = new DefaultColumnarBatch(floatCol.getSize, floatSchema, Array(floatCol)) val inExpressionFloat = in( new Column("floatCol"), Literal.ofFloat(1.5f), Literal.ofFloat(2.5f), Literal.ofFloat(3.5f)) val expOutputFloat = booleanVector(Seq[BooleanJ](true, true, true, false, null)) checkBooleanVectors( new DefaultExpressionEvaluator( floatSchema, inExpressionFloat, BooleanType.BOOLEAN).eval(floatInput), expOutputFloat) // Test with double values: col IN (1.1, 2.2, null) val doubleCol = doubleVector(Seq[DoubleJ](1.1, 2.2, 3.3, 4.4, null)) val doubleSchema = new StructType().add("doubleCol", DoubleType.DOUBLE) val doubleInput = new DefaultColumnarBatch(doubleCol.getSize, doubleSchema, Array(doubleCol)) val inExpressionDouble = in( new Column("doubleCol"), Literal.ofDouble(1.1), Literal.ofDouble(2.2), Literal.ofNull(DoubleType.DOUBLE)) val expOutputDouble = booleanVector(Seq[BooleanJ](true, true, null, null, null)) checkBooleanVectors( new DefaultExpressionEvaluator( doubleSchema, inExpressionDouble, BooleanType.BOOLEAN).eval(doubleInput), expOutputDouble) // Test with byte values: col IN (0, 1) val byteCol = byteVector(Seq[java.lang.Byte](null, 0.toByte, 0.toByte, 1.toByte, 2.toByte)) val byteSchema = new StructType().add("byteCol", ByteType.BYTE) val byteInput = new DefaultColumnarBatch(byteCol.getSize, byteSchema, Array(byteCol)) val inExpressionByte = in( new Column("byteCol"), Literal.ofByte(0.toByte), Literal.ofByte(1.toByte)) // Expected: [null, true, true, true, false] for values [null, 0, 0, 1, 2] val expOutputByte = booleanVector(Seq[BooleanJ](null, true, true, true, false)) checkBooleanVectors( new DefaultExpressionEvaluator( byteSchema, inExpressionByte, BooleanType.BOOLEAN).eval(byteInput), expOutputByte) } test("evaluate expression: in with incompatible types") { // Test error cases - incompatible types def checkIncompatibleTypes(valueType: DataType, listElementType: DataType): Unit = { val valueSchema = new StructType().add("col", valueType) val valueVector = testColumnVector(3, valueType) val valueInput = new DefaultColumnarBatch(3, valueSchema, Array(valueVector)) val incompatibleInExpr = in( new Column("col"), Literal.ofNull(listElementType)) val e = intercept[UnsupportedOperationException] { new DefaultExpressionEvaluator( valueSchema, incompatibleInExpr, BooleanType.BOOLEAN).eval(valueInput) } assert( e.getMessage.contains("IN expression requires all list elements to match the value type")) } // Test incompatible type combinations checkIncompatibleTypes(StringType.STRING, IntegerType.INTEGER) checkIncompatibleTypes(IntegerType.INTEGER, StringType.STRING) checkIncompatibleTypes(BooleanType.BOOLEAN, StringType.STRING) } test("evaluate expression: in with collation") { Seq( None, // no collation Some(SPARK_UTF8_BINARY) // UTF8_BINARY collation ).foreach { collationIdentifier => val col1 = stringVector(Seq[String]("Test", "test", "TEST", null)) val schema = new StructType().add("col1", StringType.STRING) val input = new DefaultColumnarBatch(col1.getSize, schema, Array(col1)) // Test with basic case val inExpressionBasic = in( new Column("col1"), collationIdentifier, Literal.ofString("test"), Literal.ofString("other")) val expectedOutput = booleanVector(Seq[BooleanJ](false, true, false, null)) checkBooleanVectors( new DefaultExpressionEvaluator( schema, inExpressionBasic, BooleanType.BOOLEAN).eval(input), expectedOutput) // Test with NULL in list val inExpressionWithNull = in( new Column("col1"), collationIdentifier, Literal.ofString("Test"), Literal.ofString(null)) val expOutputWithNull = booleanVector(Seq[BooleanJ](true, null, null, null)) checkBooleanVectors( new DefaultExpressionEvaluator( schema, inExpressionWithNull, BooleanType.BOOLEAN).eval(input), expOutputWithNull) } } test("evaluate expression: in with unsupported collations") { val col1 = stringVector(Seq[String]("Test", "test")) val schema = new StructType().add("col1", StringType.STRING) val input = new DefaultColumnarBatch(col1.getSize, schema, Array(col1)) val inExpressionUnsupported = in( new Column("col1"), Some(SPARK_UTF8_LCASE), Literal.ofString("test")) checkUnsupportedCollation( schema, inExpressionUnsupported, input, SPARK_UTF8_LCASE) } test("evaluate expression: in with non-literal list elements") { val schema = new StructType().add("col1", IntegerType.INTEGER).add("col2", IntegerType.INTEGER) val input = new DefaultColumnarBatch( 2, schema, Array( testColumnVector(2, IntegerType.INTEGER), testColumnVector(2, IntegerType.INTEGER))) // Try to create IN with non-literal (Column) in the list val nonLiteralInExpr = new Predicate( "IN", List[Expression]( new Column("col1"), new Column("col2"), // This should cause an error Literal.ofInt(1)).asJava) val e = intercept[UnsupportedOperationException] { new DefaultExpressionEvaluator(schema, nonLiteralInExpr, BooleanType.BOOLEAN).eval(input) } assert(e.getMessage.contains("IN expression requires all list elements to be literals")) } test("evaluate expression: in expression handling null") { val col1 = testColumnVector(6, StringType.STRING) // [null, "1", null, "3", null, "5"] val schema = new StructType().add("col1", StringType.STRING) val input = new DefaultColumnarBatch(col1.getSize, schema, Array(col1)) // Test all null semantics scenarios: // 1. NULL value with non-null list -> NULL // 2. Non-null value matches -> TRUE // 3. Non-null value no match, no nulls in list -> FALSE // 4. Non-null value no match, but nulls in list -> NULL // Case: value IN (match, null) -> [null, true, null, null, null, null] val inExprMatchWithNull = in(new Column("col1"), Literal.ofString("1"), Literal.ofString(null)) val expectedMatchWithNull = booleanVector(Seq[BooleanJ](null, true, null, null, null, null)) checkBooleanVectors( new DefaultExpressionEvaluator(schema, inExprMatchWithNull, BooleanType.BOOLEAN).eval(input), expectedMatchWithNull) // Case: value IN (no_match1, no_match2) -> [null, false, null, false, null, false] val inExprNoMatch = in(new Column("col1"), Literal.ofString("x"), Literal.ofString("y")) val expectedNoMatch = booleanVector(Seq[BooleanJ](null, false, null, false, null, false)) checkBooleanVectors( new DefaultExpressionEvaluator(schema, inExprNoMatch, BooleanType.BOOLEAN).eval(input), expectedNoMatch) // Case: value IN (no_match, null) -> [null, null, null, null, null, null] val inExprNoMatchWithNull = in(new Column("col1"), Literal.ofString("x"), Literal.ofString(null)) val expectedNoMatchWithNull = booleanVector(Seq[BooleanJ](null, null, null, null, null, null)) checkBooleanVectors( new DefaultExpressionEvaluator( schema, inExprNoMatchWithNull, BooleanType.BOOLEAN).eval(input), expectedNoMatchWithNull) } test("evaluate expression: comparators (=, <, <=, >, >=, 'IS NOT DISTINCT FROM')") { val ASCII_MAX_CHARACTER = '\u007F' val UTF8_MAX_CHARACTER = new String(Character.toChars(Character.MAX_CODE_POINT)) // Literals for each data type from the data type value range, used as inputs to comparator // (small, big, small, null) val literals = Seq( (ofByte(1.toByte), ofByte(2.toByte), ofByte(1.toByte), ofNull(ByteType.BYTE)), (ofShort(1.toShort), ofShort(2.toShort), ofShort(1.toShort), ofNull(ShortType.SHORT)), (ofInt(1), ofInt(2), ofInt(1), ofNull(IntegerType.INTEGER)), (ofLong(1L), ofLong(2L), ofLong(1L), ofNull(LongType.LONG)), (ofFloat(1.0f), ofFloat(2.0f), ofFloat(1.0f), ofNull(FloatType.FLOAT)), (ofDouble(1.0), ofDouble(2.0), ofDouble(1.0), ofNull(DoubleType.DOUBLE)), (ofBoolean(false), ofBoolean(true), ofBoolean(false), ofNull(BooleanType.BOOLEAN)), ( ofTimestamp(343L), ofTimestamp(123212312L), ofTimestamp(343L), ofNull(TimestampType.TIMESTAMP)), ( ofTimestampNtz(323423L), ofTimestampNtz(1232123423312L), ofTimestampNtz(323423L), ofNull(TimestampNTZType.TIMESTAMP_NTZ)), (ofDate(-12123), ofDate(123123), ofDate(-12123), ofNull(DateType.DATE)), (ofString("apples"), ofString("oranges"), ofString("apples"), ofNull(StringType.STRING)), (ofString(""), ofString("a"), ofString(""), ofNull(StringType.STRING)), (ofString("abc"), ofString("abc0"), ofString("abc"), ofNull(StringType.STRING)), (ofString("abc"), ofString("abcd"), ofString("abc"), ofNull(StringType.STRING)), (ofString("abc"), ofString("abd"), ofString("abc"), ofNull(StringType.STRING)), ( ofString("Abcabcabc"), ofString("aBcabcabc"), ofString("Abcabcabc"), ofNull(StringType.STRING)), ( ofString("abcabcabC"), ofString("abcabcabc"), ofString("abcabcabC"), ofNull(StringType.STRING)), // scalastyle:off nonascii (ofString("abc"), ofString("世界"), ofString("abc"), ofNull(StringType.STRING)), (ofString("世界"), ofString("你好"), ofString("世界"), ofNull(StringType.STRING)), (ofString("你好122"), ofString("你好123"), ofString("你好122"), ofNull(StringType.STRING)), (ofString("A"), ofString("Ā"), ofString("A"), ofNull(StringType.STRING)), (ofString("»"), ofString("î"), ofString("»"), ofNull(StringType.STRING)), (ofString("�"), ofString("🌼"), ofString("�"), ofNull(StringType.STRING)), ( ofString("abcdef🚀"), ofString(s"abcdef$UTF8_MAX_CHARACTER"), ofString("abcdef🚀"), ofNull(StringType.STRING)), ( ofString("abcde�abcdef�abcdef�abcdef"), ofString(s"abcde�$ASCII_MAX_CHARACTER"), ofString("abcde�abcdef�abcdef�abcdef"), ofNull(StringType.STRING)), ( ofString("abcde�abcdef�abcdef�abcdef"), ofString(s"abcde�$ASCII_MAX_CHARACTER"), ofString("abcde�abcdef�abcdef�abcdef"), ofNull(StringType.STRING)), ( ofString("����"), ofString(s"��$UTF8_MAX_CHARACTER"), ofString("����"), ofNull(StringType.STRING)), ( ofString(s"a${UTF8_MAX_CHARACTER}d"), ofString(s"a$UTF8_MAX_CHARACTER$ASCII_MAX_CHARACTER"), ofString(s"a${UTF8_MAX_CHARACTER}d"), ofNull(StringType.STRING)), ( ofString("abcdefghijklm💞😉💕\n🥀🌹💐🌺🌷🌼🌻🌷🥀"), ofString(s"abcdefghijklm💞😉💕\n🥀🌹💐🌺🌷🌼$UTF8_MAX_CHARACTER"), ofString("abcdefghijklm💞😉💕\n🥀🌹💐🌺🌷🌼🌻🌷🥀"), ofNull(StringType.STRING)), // scalastyle:on nonascii ( ofBinary("apples".getBytes()), ofBinary("oranges".getBytes()), ofBinary("apples".getBytes()), ofNull(BinaryType.BINARY)), ( ofBinary(Array[Byte]()), ofBinary(Array[Byte](5.toByte)), ofBinary(Array[Byte]()), ofNull(BinaryType.BINARY)), ( ofBinary(Array[Byte](0.toByte)), // 00000000 ofBinary(Array[Byte](-1.toByte)), // 11111111 ofBinary(Array[Byte](0.toByte)), ofNull(BinaryType.BINARY)), ( ofBinary(Array[Byte](127.toByte)), // 01111111 ofBinary(Array[Byte](-1.toByte)), // 11111111 ofBinary(Array[Byte](127.toByte)), ofNull(BinaryType.BINARY)), ( ofBinary(Array[Byte](5.toByte, 10.toByte)), ofBinary(Array[Byte](6.toByte)), ofBinary(Array[Byte](5.toByte, 10.toByte)), ofNull(BinaryType.BINARY)), ( ofBinary(Array[Byte](5.toByte, 10.toByte)), ofBinary(Array[Byte](5.toByte, 100.toByte)), ofBinary(Array[Byte](5.toByte, 10.toByte)), ofNull(BinaryType.BINARY)), ( ofBinary(Array[Byte](5.toByte, 10.toByte, 5.toByte)), // 00000101 00001010 00000101 ofBinary(Array[Byte](5.toByte, -3.toByte)), // 00000101 11111101 ofBinary(Array[Byte](5.toByte, 10.toByte, 5.toByte)), ofNull(BinaryType.BINARY)), ( ofBinary(Array[Byte](5.toByte, -25.toByte, 5.toByte)), // 00000101 11100111 00000101 ofBinary(Array[Byte](5.toByte, -9.toByte)), // 00000101 11110111 ofBinary(Array[Byte](5.toByte, -25.toByte, 5.toByte)), ofNull(BinaryType.BINARY)), ( ofBinary(Array[Byte](5.toByte, 10.toByte)), ofBinary(Array[Byte](5.toByte, 10.toByte, 0.toByte)), ofBinary(Array[Byte](5.toByte, 10.toByte)), ofNull(BinaryType.BINARY)), ( ofDecimal(BigDecimalJ.valueOf(1.12), 7, 3), ofDecimal(BigDecimalJ.valueOf(5233.232), 7, 3), ofDecimal(BigDecimalJ.valueOf(1.12), 7, 3), ofNull(new DecimalType(7, 3)))) // Mapping of comparator to expected results for: // comparator(small, big) // comparator(big, small) // comparator(small, small) // comparator(small, null) // comparator(big, null) // comparator(null, null) val comparatorToExpResults = Map[String, Seq[BooleanJ]]( "<" -> Seq(true, false, false, null, null, null), "<=" -> Seq(true, false, true, null, null, null), ">" -> Seq(false, true, false, null, null, null), ">=" -> Seq(false, true, true, null, null, null), "=" -> Seq(false, false, true, null, null, null), "IS NOT DISTINCT FROM" -> Seq(false, false, true, false, false, true)) literals.foreach { case (small1, big, small2, nullLit) => comparatorToExpResults.foreach { case (comparator, expectedResults) => val testCases = Seq( (small1, big), (big, small1), (small1, small2), (small1, nullLit), (nullLit, big), (nullLit, nullLit)) testCases.zip(expectedResults).foreach { case ((left, right), expected) => testComparator(comparator, left, right, expected) } // Predicate with collation is supported only for comparisons between StringTypes val allStringTypes = Seq(small1, big, small2, nullLit).forall { literal => literal.getDataType.isInstanceOf[StringType] } if (allStringTypes) { Seq( SPARK_UTF8_BINARY, SPARK_UTF8_LCASE, CollationIdentifier.fromString("ICU.sr_Cyrl_SRB"), CollationIdentifier.fromString("ICU.sr_Cyrl_SRB.75.1")).foreach { collationIdentifier => testCases.zip(expectedResults).foreach { case ((left, right), expected) => testCollatedComparator(comparator, left, right, expected, collationIdentifier) } } } } } } test("check Predicate with collation comparing invalid types") { Seq( // predicateName "=", "<", "<=", ">", ">=", "IS NOT DISTINCT FROM", "STARTS_WITH").foreach { predicateName => Seq( // (expr1, expr2, schema) ( Literal.ofString("apple"), Literal.ofInt(1), new StructType()), ( Literal.ofString("apple"), Literal.ofLong(1L), new StructType()), ( Literal.ofFloat(2.3f), Literal.ofString("apple"), new StructType()), ( Literal.ofDouble(2.3), Literal.ofBoolean(false), new StructType()), ( new Column(Array("col1", "col11")), Literal.ofString("apple"), new StructType() .add( "col1", new StructType() .add("col11", IntegerType.INTEGER))), ( new Column(Array("col1", "col11")), Literal.ofBoolean(false), new StructType() .add( "col1", new StructType() .add("col11", StringType.STRING))), ( new Column(Array("col1", "col11")), Literal.ofBoolean(false), new StructType() .add( "col1", new StructType() .add("col11", new StringType(SPARK_UTF8_LCASE)))), ( new Column(Array("col1", "col11")), new Column(Array("col1", "col12")), new StructType() .add( "col1", new StructType() .add("col11", DoubleType.DOUBLE) .add("col12", FloatType.FLOAT)))).foreach { case (expr1: Expression, expr2: Expression, schema: StructType) => val expr = comparator( predicateName, expr1, expr2, Some(SPARK_UTF8_BINARY)) val input = zeroColumnBatch(rowCount = 1) val e = intercept[UnsupportedOperationException] { new DefaultExpressionEvaluator( schema, expr, BooleanType.BOOLEAN).eval(input) } assert(e.getMessage.contains("expects STRING type inputs")) } } } // Literals for each data type from the data type value range, used as inputs to comparator // (byte, short, int, float, double) val literals = Seq( ofByte(1.toByte), ofShort(223), ofInt(-234), ofLong(223L), ofFloat(-2423423.9f), ofNull(DoubleType.DOUBLE)) test("evaluate expression: substring") { // scalastyle:off nonascii val data = Seq[String]( null, "one", "two", "three", "four", null, null, "seven", "eight", "😉", "ë") val col = stringVector(data) val col_name = "str_col" val schema = new StructType().add(col_name, StringType.STRING) val input = new DefaultColumnarBatch(col.getSize, schema, Array(col)) def checkSubString( input: DefaultColumnarBatch, substringExpression: ScalarExpression, expOutputSeq: Seq[String]): Unit = { val actOutputVector = new DefaultExpressionEvaluator( schema, substringExpression, StringType.STRING).eval(input) val expOutputVector = stringVector(expOutputSeq); checkStringVectors(actOutputVector, expOutputVector) } checkSubString( input, substring(new Column(col_name), 0), // scalastyle:off nonascii Seq[String](null, "one", "two", "three", "four", null, null, "seven", "eight", "😉", "ë")) checkSubString( input, substring(new Column(col_name), 1), // scalastyle:off nonascii Seq[String](null, "one", "two", "three", "four", null, null, "seven", "eight", "😉", "ë")) checkSubString( input, substring(new Column(col_name), 2), Seq[String](null, "ne", "wo", "hree", "our", null, null, "even", "ight", "", "̈")) checkSubString( input, substring(new Column(col_name), -1), // scalastyle:off nonascii Seq[String](null, "e", "o", "e", "r", null, null, "n", "t", "😉", "̈")) checkSubString( input, substring(new Column(col_name), -1000), // scalastyle:off nonascii Seq[String](null, "one", "two", "three", "four", null, null, "seven", "eight", "😉", "ë")) checkSubString( input, substring(new Column(col_name), 0, Option(4)), // scalastyle:off nonascii Seq[String](null, "one", "two", "thre", "four", null, null, "seve", "eigh", "😉", "ë")) checkSubString( input, substring(new Column(col_name), 2, Option(0)), Seq[String](null, "", "", "", "", null, null, "", "", "", "")) checkSubString( input, substring(new Column(col_name), 1, Option(1)), // scalastyle:off nonascii Seq[String](null, "o", "t", "t", "f", null, null, "s", "e", "😉", "e")) checkSubString( input, substring(new Column(col_name), 2, Option(1)), Seq[String](null, "n", "w", "h", "o", null, null, "e", "i", "", "̈")) checkSubString( input, substring(new Column(col_name), 2, Option(10000)), Seq[String](null, "ne", "wo", "hree", "our", null, null, "even", "ight", "", "̈")) checkSubString( input, substring(new Column(col_name), 1000), Seq[String](null, "", "", "", "", null, null, "", "", "", "")) checkSubString( input, substring(new Column(col_name), 1000, Option(10000)), Seq[String](null, "", "", "", "", null, null, "", "", "", "")) checkSubString( input, substring(new Column(col_name), 2, Option(-10)), Seq[String](null, "", "", "", "", null, null, "", "", "", "")) checkSubString( input, substring(new Column(col_name), -2, Option(1)), Seq[String](null, "n", "w", "e", "u", null, null, "e", "h", "", "e")) checkSubString( input, substring(new Column(col_name), -2, Option(2)), // scalastyle:off nonascii Seq[String](null, "ne", "wo", "ee", "ur", null, null, "en", "ht", "😉", "ë")) checkSubString( input, substring(new Column(col_name), -4, Option(3)), Seq[String](null, "on", "tw", "hre", "fou", null, null, "eve", "igh", "", "e")) checkSubString( input, substring(new Column(col_name), -100, Option(95)), Seq[String](null, "", "", "", "", null, null, "", "", "", "")) checkSubString( input, substring(new Column(col_name), -100, Option(98)), Seq[String](null, "o", "t", "thr", "fo", null, null, "sev", "eig", "", "")) checkSubString( input, substring(new Column(col_name), -100, Option(108)), // scalastyle:off nonascii Seq[String](null, "one", "two", "three", "four", null, null, "seven", "eight", "😉", "ë")) checkSubString( input, substring(new Column(col_name), 2147483647, Option(10000)), Seq[String](null, "", "", "", "", null, null, "", "", "", "")) checkSubString( input, substring(new Column(col_name), 2147483647), Seq[String](null, "", "", "", "", null, null, "", "", "", "")) checkSubString( input, substring(new Column(col_name), -2147483648, Option(10000)), Seq[String](null, "", "", "", "", null, null, "", "", "", "")) checkSubString( input, substring(new Column(col_name), -2147483648), // scalastyle:off nonascii Seq[String](null, "one", "two", "three", "four", null, null, "seven", "eight", "😉", "ë")) val outputVectorForEmptyInput = evaluator( schema, new ScalarExpression( "SUBSTRING", util.Arrays.asList( new Column(col_name), Literal.ofInt(1), Literal.ofInt(1))), StringType.STRING).eval(new DefaultColumnarBatch( /* size= */ 0, schema, Array( testColumnVector( /* size= */ 0, StringType.STRING), testColumnVector( /* size= */ 0, BinaryType.BINARY)))) checkStringVectors(outputVectorForEmptyInput, stringVector(Seq[String]())) def checkUnsupportedColumnTypes(colType: DataType): Unit = { val schema = new StructType() .add(col_name, colType) val batch = new DefaultColumnarBatch(5, schema, Array(testColumnVector(5, colType))) val e = intercept[UnsupportedOperationException] { evaluator( schema, new ScalarExpression( "SUBSTRING", util.Arrays.asList(new Column(col_name), Literal.ofInt(1))), StringType.STRING).eval(batch) } assert( e.getMessage.contains("Invalid type of first input of SUBSTRING: expects STRING")) } checkUnsupportedColumnTypes(IntegerType.INTEGER) checkUnsupportedColumnTypes(ByteType.BYTE) checkUnsupportedColumnTypes(BooleanType.BOOLEAN) checkUnsupportedColumnTypes(BinaryType.BINARY) val badLiteralSize = intercept[UnsupportedOperationException] { evaluator( schema, new ScalarExpression( "SUBSTRING", util.Arrays.asList( new Column(col_name), Literal.ofInt(1), Literal.ofInt(1), Literal.ofInt(1))), StringType.STRING).eval(new DefaultColumnarBatch( /* size= */ 5, schema, Array(testColumnVector( /* size= */ 5, StringType.STRING)))) } assert( badLiteralSize.getMessage.contains( "Invalid number of inputs to SUBSTRING expression.")) val badPosType = intercept[UnsupportedOperationException] { evaluator( schema, new ScalarExpression( "SUBSTRING", util.Arrays.asList( new Column("str_col"), Literal.ofBoolean(true))), StringType.STRING).eval(new DefaultColumnarBatch( /* size= */ 5, schema, Array(testColumnVector( /* size= */ 5, StringType.STRING)))) } assert(badPosType.getMessage.contains("Invalid `pos` argument type for SUBSTRING")) val badLenType = intercept[UnsupportedOperationException] { evaluator( schema, new ScalarExpression( "SUBSTRING", util.Arrays.asList( new Column(col_name), Literal.ofInt(1), Literal.ofBoolean(true))), StringType.STRING).eval(new DefaultColumnarBatch( /* size= */ 5, schema, Array(testColumnVector( /* size= */ 5, StringType.STRING)))) } assert(badLenType.getMessage.contains("Invalid `len` argument type for SUBSTRING")) } test("evaluate expression: comparators `byte` with other implicit types") { // Mapping of comparator to expected results for: // (byte, short), (byte, int), (byte, long), (byte, float), (byte, double) val comparatorToExpResults = Map[String, Seq[BooleanJ]]( "<" -> Seq(true, false, true, false, null), "<=" -> Seq(true, false, true, false, null), ">" -> Seq(false, true, false, true, null), ">=" -> Seq(false, true, false, true, null), "=" -> Seq(false, false, false, false, null)) // Left operand is first literal in [[literal]] which a byte type // Right operands are the remaining literals to the left side of it in [[literal]] val right = literals(0) Seq.range(1, literals.length).foreach { idx => comparatorToExpResults.foreach { case (comparator, expectedResults) => testComparator(comparator, right, literals(idx), expectedResults(idx - 1)) } } } test("evaluate expression: comparators `short` with other implicit types") { // Mapping of comparator to expected results for: // (short, int), (short, long), (short, float), (short, double) val comparatorToExpResults = Map[String, Seq[BooleanJ]]( "<" -> Seq(false, false, false, null), "<=" -> Seq(false, true, false, null), ">" -> Seq(true, false, true, null), ">=" -> Seq(true, true, true, null), "=" -> Seq(false, true, false, null)) // Left operand is first literal in [[literal]] which a short type // Right operands are the remaining literals to the left side of it in [[literal]] val right = literals(1) Seq.range(2, literals.length).foreach { idx => comparatorToExpResults.foreach { case (comparator, expectedResults) => testComparator(comparator, right, literals(idx), expectedResults(idx - 2)) } } } test("evaluate expression: comparators `int` with other implicit types") { // Mapping of comparator to expected results for: (int, long), (int, float), (int, double) val comparatorToExpResults = Map[String, Seq[BooleanJ]]( "<" -> Seq(true, false, null), "<=" -> Seq(true, false, null), ">" -> Seq(false, true, null), ">=" -> Seq(false, true, null), "=" -> Seq(false, false, null)) // Left operand is first literal in [[literal]] which a int type // Right operands are the remaining literals to the left side of it in [[literal]] val right = literals(2) Seq.range(3, literals.length).foreach { idx => comparatorToExpResults.foreach { case (comparator, expectedResults) => testComparator(comparator, right, literals(idx), expectedResults(idx - 3)) } } } test("evaluate expression: comparators `long` with other implicit types") { // Mapping of comparator to expected results for: (long, float), (long, double) val comparatorToExpResults = Map[String, Seq[BooleanJ]]( "<" -> Seq(false, null), "<=" -> Seq(false, null), ">" -> Seq(true, null), ">=" -> Seq(true, null), "=" -> Seq(false, null)) // Left operand is fourth literal in [[literal]] which a long type // Right operands are the remaining literals to the left side of it in [[literal]] val right = literals(3) Seq.range(4, literals.length).foreach { idx => comparatorToExpResults.foreach { case (comparator, expectedResults) => testComparator(comparator, right, literals(idx), expectedResults(idx - 4)) } } } test("evaluate expression: unsupported implicit casts") { intercept[UnsupportedOperationException] { testComparator("<", ofInt(21), ofDate(123), null) } } test("evaluate expression: comparators `float` with other implicit types") { // Comparator results for: (float, double) is always null as one of the operands is null val comparatorToExpResults = Seq("<", "<=", ">", ">=", "=") // Left operand is fifth literal in [[literal]] which is a float type // Right operands are the remaining literals to the left side of it in [[literal]] val right = literals(4) Seq.range(5, literals.length).foreach { idx => comparatorToExpResults.foreach { comparator => testComparator(comparator, right, literals(idx), null) } } } test("evaluate expression: element_at") { val nullStr = null.asInstanceOf[String] val testMapValues: Seq[Map[AnyRef, AnyRef]] = Seq( Map("k0" -> "v00", "k1" -> "v01", "k3" -> nullStr, nullStr -> "v04"), Map("k0" -> "v10", "k1" -> nullStr, "k3" -> "v13", nullStr -> "v14"), Map("k0" -> nullStr, "k1" -> "v21", "k3" -> "v23", nullStr -> "v24"), null) val testMapVector = buildMapVector( testMapValues, new MapType(StringType.STRING, StringType.STRING, true)) val inputBatch = new DefaultColumnarBatch( testMapVector.getSize, new StructType().add("partitionValues", testMapVector.getDataType), Seq(testMapVector).toArray) Seq("k0", "k1", "k2", null).foreach { lookupKey => val expOutput = testMapValues.map(map => { if (map == null) null else map.getOrElse(lookupKey, null) }) val lookupKeyExpr = if (lookupKey == null) { Literal.ofNull(StringType.STRING) } else { Literal.ofString(lookupKey) } val elementAtExpr = new ScalarExpression( "element_at", util.Arrays.asList(new Column("partitionValues"), lookupKeyExpr)) val outputVector = evaluator(inputBatch.getSchema, elementAtExpr, StringType.STRING) .eval(inputBatch) assert(outputVector.getSize === testMapValues.size) assert(outputVector.getDataType === StringType.STRING) Seq.range(0, testMapValues.size).foreach { rowId => val expNull = expOutput(rowId) == null assert(outputVector.isNullAt(rowId) == expNull) if (!expNull) { assert(outputVector.getString(rowId) === expOutput(rowId)) } } } } test("evaluate expression: element_at - unsupported map type input") { val inputSchema = new StructType() .add("as_map", new MapType(IntegerType.INTEGER, BooleanType.BOOLEAN, true)) val elementAtExpr = new ScalarExpression( "element_at", util.Arrays.asList(new Column("as_map"), Literal.ofString("empty"))) val ex = intercept[UnsupportedOperationException] { evaluator(inputSchema, elementAtExpr, StringType.STRING) } assert(ex.getMessage.contains( "ELEMENT_AT(column(`as_map`), empty): Supported only on type map(string, string) input data")) } test("evaluate expression: element_at - unsupported lookup type input") { val inputSchema = new StructType() .add("as_map", new MapType(StringType.STRING, StringType.STRING, true)) val elementAtExpr = new ScalarExpression( "element_at", util.Arrays.asList(new Column("as_map"), Literal.ofShort(24))) val ex = intercept[UnsupportedOperationException] { evaluator(inputSchema, elementAtExpr, StringType.STRING) } assert(ex.getMessage.contains( "lookup key type (short) is different from the map key type (string)")) } test("evaluate expression: partition_value") { // (serialized partition value, partition col type, expected deserialized partition value) val testCases = Seq( ("true", BooleanType.BOOLEAN, true), ("false", BooleanType.BOOLEAN, false), (null, BooleanType.BOOLEAN, null), ("24", ByteType.BYTE, 24.toByte), ("null", ByteType.BYTE, null), ("876", ShortType.SHORT, 876.toShort), ("null", ShortType.SHORT, null), ("2342342", IntegerType.INTEGER, 2342342), ("null", IntegerType.INTEGER, null), ("234234223", LongType.LONG, 234234223L), ("null", LongType.LONG, null), ("23423.4223", FloatType.FLOAT, 23423.4223f), ("null", FloatType.FLOAT, null), ("23423.422233", DoubleType.DOUBLE, 23423.422233d), ("null", DoubleType.DOUBLE, null), ("234.422233", new DecimalType(10, 6), new BigDecimalJ("234.422233")), ("null", DoubleType.DOUBLE, null), ("string_val", StringType.STRING, "string_val"), ("null", StringType.STRING, null), ("binary_val", BinaryType.BINARY, "binary_val".getBytes()), ("null", BinaryType.BINARY, null), ("2021-11-18", DateType.DATE, InternalUtils.daysSinceEpoch(Date.valueOf("2021-11-18"))), ("null", DateType.DATE, null), ( "2020-02-18 22:00:10", TimestampType.TIMESTAMP, InternalUtils.microsSinceEpoch(Timestamp.valueOf("2020-02-18 22:00:10"))), ( "2020-02-18 00:00:10.023", TimestampType.TIMESTAMP, InternalUtils.microsSinceEpoch(Timestamp.valueOf("2020-02-18 00:00:10.023"))), ("null", TimestampType.TIMESTAMP, null), ( // ISO8601 format "2024-01-02T12:30:00.000000Z", TimestampType.TIMESTAMP, InternalUtils.microsSinceEpoch(Timestamp.valueOf("2024-01-02 12:30:00"))), ( // Test with microsecond precision (no seconds) "1970-01-01T00:00:00.123456Z", TimestampType.TIMESTAMP, InternalUtils.microsSinceEpoch(Timestamp.valueOf("1970-01-01 00:00:00.123456"))), ( // Test with microsecond precision (with seconds, current date) "2025-01-01T00:00:00.123456Z", TimestampType.TIMESTAMP, InternalUtils.microsSinceEpoch(Timestamp.valueOf("2025-01-01 00:00:00.123456")))) val inputBatch = zeroColumnBatch(rowCount = 1) testCases.foreach { testCase => val (serializedPartVal, partType, deserializedPartVal) = testCase val literalSerializedPartVal = if (serializedPartVal == "null") { Literal.ofNull(StringType.STRING) } else { Literal.ofString(serializedPartVal) } val expr = new PartitionValueExpression(literalSerializedPartVal, partType) val outputVector = evaluator(inputBatch.getSchema, expr, partType).eval(inputBatch) assert(outputVector.getSize === 1) assert(outputVector.getDataType === partType) assert(outputVector.isNullAt(0) === (deserializedPartVal == null)) if (deserializedPartVal != null) { assert(getValueAsObject(outputVector, 0) === deserializedPartVal) } } } test("evaluate expression: partition_value - invalid serialize value") { val inputBatch = zeroColumnBatch(rowCount = 1) val (serializedPartVal, partType) = ("23423sdfsdf", IntegerType.INTEGER) val expr = new PartitionValueExpression(Literal.ofString(serializedPartVal), partType) val ex = intercept[IllegalArgumentException] { val outputVector = evaluator(inputBatch.getSchema, expr, partType).eval(inputBatch) outputVector.getInt(0) } assert(ex.getMessage.contains(serializedPartVal)) } private def evaluator(inputSchema: StructType, expression: Expression, outputType: DataType) : DefaultExpressionEvaluator = { new DefaultExpressionEvaluator(inputSchema, expression, outputType) } private def checkUnsupportedCollation( schema: StructType, expression: Expression, input: ColumnarBatch, collationIdentifier: CollationIdentifier): Unit = { val e = intercept[UnsupportedOperationException] { evaluator(schema, expression, BooleanType.BOOLEAN).eval(input) } assert(e.getMessage.contains( s"""Unsupported collation: "$collationIdentifier". | Default Engine supports just "$SPARK_UTF8_BINARY" | collation.""".stripMargin.replace("\n", ""))) } private def testCollatedComparator( comparatorName: String, left: Expression, right: Expression, expResult: BooleanJ, collationIdentifier: CollationIdentifier): Unit = { val collatedPredicate = comparator(comparatorName, left, right, Some(collationIdentifier)) val batch = zeroColumnBatch(rowCount = 1) if (collationIdentifier.isSparkUTF8BinaryCollation) { val outputVector = evaluator( batch.getSchema, collatedPredicate, BooleanType.BOOLEAN).eval(batch) assert(outputVector.getSize === 1) assert(outputVector.getDataType === BooleanType.BOOLEAN) assert( outputVector.isNullAt(0) === (expResult == null), s"Unexpected null value for Predicate with collation: $collatedPredicate") if (expResult != null) { assert( outputVector.getBoolean(0) === expResult, s"""Unexpected value for Predicate with collation: $collatedPredicate, | expected: $expResult, actual: ${outputVector.getBoolean(0)}""".stripMargin) } } else { checkUnsupportedCollation(batch.getSchema, collatedPredicate, batch, collationIdentifier) } } private def testComparator( comparator: String, left: Expression, right: Expression, expResult: BooleanJ): Unit = { val expression = new Predicate(comparator, left, right) val batch = zeroColumnBatch(rowCount = 1) val outputVector = evaluator(batch.getSchema, expression, BooleanType.BOOLEAN).eval(batch) assert(outputVector.getSize === 1) assert(outputVector.getDataType === BooleanType.BOOLEAN) assert( outputVector.isNullAt(0) === (expResult == null), s"Unexpected null value: $comparator($left, $right)") if (expResult != null) { assert( outputVector.getBoolean(0) === expResult, s"Unexpected value: $comparator($left, $right)") } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/expressions/DefaultPredicateEvaluatorSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.expressions import java.lang.{Boolean => BooleanJ} import java.util.Optional import java.util.Optional.empty import io.delta.kernel.data.{ColumnarBatch, ColumnVector} import io.delta.kernel.defaults.internal.data.DefaultColumnarBatch import io.delta.kernel.expressions.{Column, Literal} import io.delta.kernel.types.{BooleanType, StructType} import org.scalatest.funsuite.AnyFunSuite /** * [[DefaultPredicateEvaluator]] internally uses [[DefaultExpressionEvaluator]]. In this suite * test the code specific to the [[DefaultPredicateEvaluator]] such as taking into consideration of * existing selection vector. */ class DefaultPredicateEvaluatorSuite extends AnyFunSuite with ExpressionSuiteBase { private val testLeftCol = booleanVector( Seq[BooleanJ](true, true, false, true, true, false, null, true, null, false, null)) private val testRightCol = booleanVector( Seq[BooleanJ](true, false, false, true, false, false, true, null, false, null, null)) private val testSchema = new StructType() .add("left", BooleanType.BOOLEAN) .add("right", BooleanType.BOOLEAN) private val batch = new DefaultColumnarBatch( testLeftCol.getSize, testSchema, Array(testLeftCol, testRightCol)) private val left = comparator("=", new Column("left"), Literal.ofBoolean(true)) private val right = comparator("=", new Column("right"), Literal.ofBoolean(true)) private val orPredicate = or(left, right) private val expOrOutput = booleanVector( Seq[BooleanJ](true, true, false, true, true, false, true, true, null, null, null)) test("evaluate predicate: with no starting selection vector") { val batch = new DefaultColumnarBatch( testLeftCol.getSize, testSchema, Array(testLeftCol, testRightCol)) val actOutputVector = evalOr(batch) checkBooleanVectors(actOutputVector, expOrOutput) } test("evaluate predicate: with existing selection vector") { val existingSelVector = booleanVector( Seq[BooleanJ](false, true, true, true, false, false, null, null, null, null, null)) val outputWithSelVector = booleanVector( Seq[BooleanJ](false, true, false, true, false, false, null, null, null, null, null)) val actOutputVector = evalOr(batch, Optional.of(existingSelVector)) checkBooleanVectors(actOutputVector, outputWithSelVector) } test("evaluate predicate: multiple rounds with selection vectors") { val output0 = evalOr(batch) checkBooleanVectors(output0, expOrOutput) val selVec1 = booleanVector( Seq[BooleanJ](false, true, false, true, false, false, null, null, null, null, null)) val expOutputWithSelVec1 = booleanVector( Seq[BooleanJ](false, true, false, true, false, false, null, null, null, null, null)) checkBooleanVectors(evalOr(batch, Optional.of(selVec1)), expOutputWithSelVec1) val selVec2 = booleanVector( Seq[BooleanJ](false, false, false, true, false, false, null, null, null, null, null)) val expOutputWithSelVec2 = booleanVector( Seq[BooleanJ](false, false, false, true, false, false, null, null, null, null, null)) checkBooleanVectors(evalOr(batch, Optional.of(selVec2)), expOutputWithSelVec2) } def evalOr( batch: ColumnarBatch, existingSelVector: Optional[ColumnVector] = empty()): ColumnVector = { val evaluator = new DefaultPredicateEvaluator(batch.getSchema, orPredicate) evaluator.eval(batch, existingSelVector) } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/expressions/ExpressionSuiteBase.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.expressions import scala.collection.JavaConverters._ import io.delta.kernel.data.{ColumnarBatch, ColumnVector} import io.delta.kernel.defaults.internal.data.DefaultColumnarBatch import io.delta.kernel.defaults.utils.{DefaultVectorTestUtils, TestUtils} import io.delta.kernel.defaults.utils.DefaultKernelTestUtils.getValueAsObject import io.delta.kernel.expressions._ import io.delta.kernel.internal.util.ExpressionUtils.createPredicate import io.delta.kernel.types._ trait ExpressionSuiteBase extends TestUtils with DefaultVectorTestUtils { /** create a columnar batch of given `size` with zero columns in it. */ protected def zeroColumnBatch(rowCount: Int): ColumnarBatch = { new DefaultColumnarBatch(rowCount, new StructType(), new Array[ColumnVector](0)) } protected def and(left: Predicate, right: Predicate): And = { new And(left, right) } protected def or(left: Predicate, right: Predicate): Or = { new Or(left, right) } protected def substring(expr: Expression, pos: Int, len: Option[Int] = None): ScalarExpression = { var children = List(expr, Literal.ofInt(pos)) if (len.isDefined) { children = children :+ Literal.ofInt(len.get) } new ScalarExpression("substring", children.asJava) } protected def like( left: Expression, right: Expression, escape: Option[Character] = None): Predicate = { if (escape.isDefined && escape.get != null) { like(List(left, right, Literal.ofString(escape.get.toString))) } else like(List(left, right)) } protected def like(children: List[Expression]): Predicate = { new Predicate("like", children.asJava) } protected def startsWith( left: Expression, right: Expression, collationIdentifier: Option[CollationIdentifier] = None): Predicate = { createPredicate( "starts_with", List(left, right).asJava, optionToJava(collationIdentifier)) } protected def comparator( symbol: String, left: Expression, right: Expression, collationIdentifier: Option[CollationIdentifier] = None): Predicate = { createPredicate(symbol, List(left, right).asJava, optionToJava(collationIdentifier)) } protected def in(value: Expression, inList: Expression*): In = { new In(value, inList.toList.asJava) } protected def in( value: Expression, collationIdentifier: Option[CollationIdentifier], inList: Expression*): In = { if (collationIdentifier.isDefined) { new In(value, inList.toList.asJava, collationIdentifier.get) } else { new In(value, inList.toList.asJava) } } protected def checkBooleanVectors(actual: ColumnVector, expected: ColumnVector): Unit = { assert(actual.getDataType === expected.getDataType) assert(actual.getSize === expected.getSize) Seq.range(0, actual.getSize).foreach { rowId => assert(actual.isNullAt(rowId) === expected.isNullAt(rowId)) if (!actual.isNullAt(rowId)) { assert( actual.getBoolean(rowId) === expected.getBoolean(rowId), s"unexpected value at $rowId") } } } protected def checkLongVectors(actual: ColumnVector, expected: ColumnVector): Unit = { assert(actual.getDataType === expected.getDataType) assert(actual.getSize === expected.getSize) Seq.range(0, actual.getSize).foreach { rowId => if (expected.isNullAt(rowId)) { assert(actual.isNullAt(rowId), s"Expected null at row $rowId") } else { assert(actual.getLong(rowId) === expected.getLong(rowId), s"Unexpected value at row $rowId") } } } protected def checkTimestampVectors(actual: ColumnVector, expected: ColumnVector): Unit = { assert(actual.getSize === expected.getSize) for (rowId <- 0 until actual.getSize) { if (expected.isNullAt(rowId)) { assert(actual.isNullAt(rowId), s"Expected null at row $rowId") } else { val expectedValue = getValueAsObject(expected, rowId).asInstanceOf[Long] val actualValue = getValueAsObject(actual, rowId).asInstanceOf[Long] assert(actualValue === expectedValue, s"Unexpected value at row $rowId") } } } protected def checkStringVectors(actual: ColumnVector, expected: ColumnVector): Unit = { assert(actual.getDataType === StringType.STRING) assert(actual.getDataType === expected.getDataType) assert(actual.getSize === expected.getSize) Seq.range(0, actual.getSize).foreach { rowId => assert(actual.isNullAt(rowId) === expected.isNullAt(rowId)) if (!actual.isNullAt(rowId)) { assert( actual.getString(rowId) === expected.getString(rowId), s"unexpected value at $rowId: " + s"expected: ${expected.getString(rowId)} " + s"actual: ${actual.getString(rowId)} ") } } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/expressions/ImplicitCastExpressionSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.expressions import io.delta.kernel.data.ColumnVector import io.delta.kernel.defaults.internal.expressions.ImplicitCastExpression.canCastTo import io.delta.kernel.defaults.utils.DefaultKernelTestUtils.getValueAsObject import io.delta.kernel.defaults.utils.TestUtils import io.delta.kernel.expressions.Column import io.delta.kernel.types._ import org.scalatest.funsuite.AnyFunSuite class ImplicitCastExpressionSuite extends AnyFunSuite with TestUtils { private val allowedCasts: Set[(DataType, DataType)] = Set( (ByteType.BYTE, ShortType.SHORT), (ByteType.BYTE, IntegerType.INTEGER), (ByteType.BYTE, LongType.LONG), (ByteType.BYTE, FloatType.FLOAT), (ByteType.BYTE, DoubleType.DOUBLE), (ShortType.SHORT, IntegerType.INTEGER), (ShortType.SHORT, LongType.LONG), (ShortType.SHORT, FloatType.FLOAT), (ShortType.SHORT, DoubleType.DOUBLE), (IntegerType.INTEGER, LongType.LONG), (IntegerType.INTEGER, FloatType.FLOAT), (IntegerType.INTEGER, DoubleType.DOUBLE), (LongType.LONG, FloatType.FLOAT), (LongType.LONG, DoubleType.DOUBLE), (FloatType.FLOAT, DoubleType.DOUBLE)) test("can cast to") { ALL_TYPES.foreach { fromType => ALL_TYPES.foreach { toType => assert(canCastTo(fromType, toType) === allowedCasts.contains((fromType, toType))) } } } allowedCasts.foreach { castPair => test(s"eval cast expression: ${castPair._1} -> ${castPair._2}") { val fromType = castPair._1 val toType = castPair._2 val inputVector = testData(87, fromType, (rowId) => rowId % 7 == 0) val outputVector = new ImplicitCastExpression(new Column("id"), toType) .eval(inputVector) checkCastOutput(inputVector, toType, outputVector) } } def testData(size: Int, dataType: DataType, nullability: (Int) => Boolean): ColumnVector = { new ColumnVector { override def getDataType: DataType = dataType override def getSize: Int = size override def close(): Unit = {} override def isNullAt(rowId: Int): Boolean = nullability(rowId) override def getByte(rowId: Int): Byte = { assert(dataType === ByteType.BYTE) generateValue(rowId).toByte } override def getShort(rowId: Int): Short = { assert(dataType === ShortType.SHORT) generateValue(rowId).toShort } override def getInt(rowId: Int): Int = { assert(dataType === IntegerType.INTEGER) generateValue(rowId).toInt } override def getLong(rowId: Int): Long = { assert(dataType === LongType.LONG) generateValue(rowId).toLong } override def getFloat(rowId: Int): Float = { assert(dataType === FloatType.FLOAT) generateValue(rowId).toFloat } override def getDouble(rowId: Int): Double = { assert(dataType === DoubleType.DOUBLE) generateValue(rowId) } } } // Utility method to generate a value based on the rowId. Returned value is a double // which the callers can cast to appropriate numerical type. private def generateValue(rowId: Int): Double = rowId * 2.76 + 7623 private def checkCastOutput(input: ColumnVector, toType: DataType, output: ColumnVector): Unit = { assert(input.getSize === output.getSize) assert(toType === output.getDataType) Seq.range(0, input.getSize).foreach { rowId => assert(input.isNullAt(rowId) === output.isNullAt(rowId)) assert(getValueAsObject(input, rowId) === getValueAsObject(output, rowId)) } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/json/JsonUtilsSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.json import scala.Double.NegativeInfinity import scala.collection.JavaConverters._ import io.delta.kernel.defaults.utils.{TestRow, TestUtils} import io.delta.kernel.test.VectorTestUtils import io.delta.kernel.types._ import org.scalatest.funsuite.AnyFunSuite class JsonUtilsSuite extends AnyFunSuite with TestUtils with VectorTestUtils { // Tests for round trip of each data type Seq( ( BooleanType.BOOLEAN, s"""{"c0":false,"c1":true,"c2":null,"c3":false}""", // test JSON TestRow(false, true, null, false), // expected decoded row // expected row serialized as JSON, null values won't be in output s"""{"c0":false,"c1":true,"c3":false}"""), ( ByteType.BYTE, s"""{"c0":${Byte.MinValue},"c1":${Byte.MaxValue},"c2":null,"c3":4}""", TestRow(Byte.MinValue, Byte.MaxValue, null, 4.toByte), s"""{"c0":${Byte.MinValue},"c1":${Byte.MaxValue},"c3":4}"""), ( ShortType.SHORT, s"""{"c0":${Short.MinValue},"c1":${Short.MaxValue},"c2":null,"c3":44}""", TestRow(Short.MinValue, Short.MaxValue, null, 44.toShort), s"""{"c0":${Short.MinValue},"c1":${Short.MaxValue},"c3":44}"""), ( IntegerType.INTEGER, s"""{"c0":${Integer.MIN_VALUE},"c1":${Integer.MAX_VALUE},"c2":null,"c3":423423}""", TestRow(Integer.MIN_VALUE, Integer.MAX_VALUE, null, 423423), s"""{"c0":${Integer.MIN_VALUE},"c1":${Integer.MAX_VALUE},"c3":423423}"""), ( LongType.LONG, s"""{"c0":${Long.MinValue},"c1":${Long.MaxValue},"c2":null,"c3":423423}""", TestRow(Long.MinValue, Long.MaxValue, null, 423423.toLong), s"""{"c0":${Long.MinValue},"c1":${Long.MaxValue},"c3":423423}"""), ( FloatType.FLOAT, s"""{"c0":${Float.MinValue},"c1":${Float.MaxValue},"c2":null,"c3":"${Float.NaN}"}""", TestRow(Float.MinValue, Float.MaxValue, null, Float.NaN), s"""{"c0":${Float.MinValue},"c1":${Float.MaxValue},"c3":"NaN"}"""), ( DoubleType.DOUBLE, s"""{"c0":${Double.MinValue},"c1":${Double.MaxValue},"c2":null,"c3":"${NegativeInfinity}"}""", TestRow(Double.MinValue, Double.MaxValue, null, NegativeInfinity), s"""{"c0":${Double.MinValue},"c1":${Double.MaxValue},"c3":"-Infinity"}"""), ( StringType.STRING, s"""{"c0":"","c1":"ssdfsdf","c2":null,"c3":"123sdsd"}""", TestRow("", "ssdfsdf", null, "123sdsd"), s"""{"c0":"","c1":"ssdfsdf","c3":"123sdsd"}"""), ( new ArrayType(IntegerType.INTEGER, true /* containsNull */ ), """{"c0":[23,23],"c1":[1212,null,2332],"c2":null,"c3":[]}""", TestRow(Seq(23, 23), Seq(1212, null, 2332), null, Seq()), """{"c0":[23,23],"c1":[1212,null,2332],"c3":[]}"""), ( // array with complex element types new ArrayType( new StructType() .add("cn0", IntegerType.INTEGER) .add("cn1", new ArrayType(LongType.LONG, true /* containsNull */ )), true /* containsNull */ ), """{ |"c0":[{"cn0":24,"cn1":[23,232]},{"cn0":25,"cn1":[24,237]}], |"c1":[{"cn0":32,"cn1":[37,null,2323]},{"cn0":29,"cn1":[200,111237]}], |"c2":null, |"c3":[]}""".stripMargin, TestRow( Seq(TestRow(24, Seq(23L, 232L)), TestRow(25, Seq(24L, 237L))), Seq(TestRow(32, Seq(37L, null, 2323L)), TestRow(29, Seq(200L, 111237L))), null, Seq()), """{ |"c0":[{"cn0":24,"cn1":[23,232]},{"cn0":25,"cn1":[24,237]}], |"c1":[{"cn0":32,"cn1":[37,null,2323]},{"cn0":29,"cn1":[200,111237]}], |"c3":[]}""".stripMargin), ( new MapType(StringType.STRING, IntegerType.INTEGER, true /* valueContainsNull */ ), """{ |"c0":{"24":200,"25":201}, |"c1":{"27":null,"25":203}, |"c2":null, |"c3":{} |}""".stripMargin, TestRow( Map("24" -> 200, "25" -> 201), Map("27" -> null, "25" -> 203), null, Map()), """{ |"c0":{"24":200,"25":201}, |"c1":{"27":null,"25":203}, |"c3":{} |}""".stripMargin), ( new StructType() .add("cn0", IntegerType.INTEGER) .add("cn1", new ArrayType(LongType.LONG, true /* containsNull */ )), """{ |"c0":{"cn0":24,"cn1":[23,232]}, |"c1":{"cn0":29,"cn1":[200,null,111237]}, |"c2":null, |"c3":{} |}""".stripMargin, TestRow( TestRow(24, Seq(23L, 232L)), TestRow(29, Seq(200L, null, 111237L)), null, TestRow(null, null)), """{ |"c0":{"cn0":24,"cn1":[23,232]}, |"c1":{"cn0":29,"cn1":[200,null,111237]}, |"c3":{} |}""".stripMargin)).foreach { case (dataType, testJson, expRow, expJson) => test(s"JsonUtils.RowSerializer: $dataType") { val schema = new StructType(Seq.range(0, 4).map(colOrdinal => new StructField(s"c$colOrdinal", dataType, true)).asJava) val actRow = JsonUtils.rowFromJson(testJson, schema) checkAnswer(Seq(actRow), Seq(expRow)) assert(JsonUtils.rowToJson(actRow) === expJson.linesIterator.mkString) } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/logstore/LogStoreProviderSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.logstore import io.delta.storage._ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, Path} import org.scalatest.funsuite.AnyFunSuite class LogStoreProviderSuite extends AnyFunSuite { private val customLogStoreClassName = classOf[UserDefinedLogStore].getName val hadoopConf = new Configuration() Seq( "s3" -> classOf[S3SingleDriverLogStore].getName, "s3a" -> classOf[S3SingleDriverLogStore].getName, "s3n" -> classOf[S3SingleDriverLogStore].getName, "hdfs" -> classOf[HDFSLogStore].getName, "file" -> classOf[HDFSLogStore].getName, "gs" -> classOf[GCSLogStore].getName, "abfss" -> classOf[AzureLogStore].getName, "abfs" -> classOf[AzureLogStore].getName, "adl" -> classOf[AzureLogStore].getName, "wasb" -> classOf[AzureLogStore].getName, "wasbs" -> classOf[AzureLogStore].getName).foreach { case (scheme, logStoreClass) => test(s"get the default LogStore for scheme $scheme") { val logStore = LogStoreProvider.getLogStore(hadoopConf, scheme) assert(logStore.getClass.getName === logStoreClass) } } test("override the default LogStore for a schema") { val hadoopConf = new Configuration() hadoopConf.set(LogStoreProvider.getLogStoreSchemeConfKey("s3"), customLogStoreClassName) val logStore = LogStoreProvider.getLogStore(hadoopConf, "s3") assert(logStore.getClass.getName === customLogStoreClassName) } test("set LogStore config for a custom scheme") { val hadoopConf = new Configuration() hadoopConf.set(LogStoreProvider.getLogStoreSchemeConfKey("fake"), customLogStoreClassName) val logStore = LogStoreProvider.getLogStore(hadoopConf, "fake") assert(logStore.getClass.getName === customLogStoreClassName) } test("set LogStore config to a class that doesn't extend LogStore") { val hadoopConf = new Configuration() hadoopConf.set(LogStoreProvider.getLogStoreSchemeConfKey("fake"), "java.lang.String") val e = intercept[IllegalArgumentException]( LogStoreProvider.getLogStore(hadoopConf, "fake")) assert(e.getMessage.contains( "Can not instantiate `LogStore` class (from config): %s".format("java.lang.String"))) } } /** * Sample user-defined log store implementing [[LogStore]]. */ class UserDefinedLogStore(override val initHadoopConf: Configuration) extends LogStore(initHadoopConf) { private val logStoreInternal = new HDFSLogStore(initHadoopConf) override def read(path: Path, hadoopConf: Configuration): CloseableIterator[String] = { logStoreInternal.read(path, hadoopConf) } override def write( path: Path, actions: java.util.Iterator[String], overwrite: java.lang.Boolean, hadoopConf: Configuration): Unit = { logStoreInternal.write(path, actions, overwrite, hadoopConf) } override def listFrom(path: Path, hadoopConf: Configuration): java.util.Iterator[FileStatus] = { logStoreInternal.listFrom(path, hadoopConf) } override def resolvePathOnPhysicalStorage(path: Path, hadoopConf: Configuration): Path = { logStoreInternal.resolvePathOnPhysicalStorage(path, hadoopConf) } override def isPartialWriteVisible(path: Path, hadoopConf: Configuration): java.lang.Boolean = { false } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/parquet/ParquetFileReaderSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.parquet import java.math.BigDecimal import java.util.TimeZone import io.delta.golden.GoldenTableUtils.{goldenTableFile, goldenTablePath} import io.delta.kernel.defaults.utils.{ExpressionTestUtils, TestRow} import io.delta.kernel.test.VectorTestUtils import io.delta.kernel.types._ import io.delta.kernel.utils.MetadataColumnTestUtils import org.apache.spark.sql.internal.SQLConf import org.scalatest.funsuite.AnyFunSuite import org.slf4j.LoggerFactory class ParquetFileReaderSuite extends AnyFunSuite with ParquetSuiteBase with VectorTestUtils with ExpressionTestUtils with MetadataColumnTestUtils { private val logger = LoggerFactory.getLogger(classOf[ParquetFileReaderSuite]) test("decimals encoded using dictionary encoding ") { // Below golden tables contains three decimal columns // each stored in a different physical format: int32, int64 and fixed binary val decimalDictFileV1 = goldenTableFile("parquet-decimal-dictionaries-v1").getAbsolutePath val decimalDictFileV2 = goldenTableFile("parquet-decimal-dictionaries-v2").getAbsolutePath val expResult = (0 until 1000000).map { i => TestRow(i, BigDecimal.valueOf(i % 5), BigDecimal.valueOf(i % 6), BigDecimal.valueOf(i % 2)) } val readSchema = tableSchema(decimalDictFileV1) for (file <- Seq(decimalDictFileV1, decimalDictFileV2)) { val actResult = readParquetFilesUsingKernel(file, readSchema) checkAnswer(actResult, expResult) } } test("large scale decimal type file") { val largeScaleDecimalTypesFile = goldenTableFile("parquet-decimal-type").getAbsolutePath def expand(n: BigDecimal): BigDecimal = { n.scaleByPowerOfTen(5).add(n) } val expResult = (0 until 99998).map { i => if (i % 85 == 0) { val n = BigDecimal.valueOf(i) TestRow(i, n.movePointLeft(1).setScale(1), n.setScale(5), n.setScale(5)) } else { val negation = if (i % 33 == 0) -1 else 1 val n = BigDecimal.valueOf(i * negation) TestRow( i, n.movePointLeft(1), expand(n).movePointLeft(5), expand(expand(expand(n))).movePointLeft(5)) } } val readSchema = tableSchema(largeScaleDecimalTypesFile) val actResult = readParquetFilesUsingKernel(largeScaleDecimalTypesFile, readSchema) checkAnswer(actResult, expResult) } Seq( "parquet-all-types", "parquet-all-types-legacy-format").foreach { allTypesTableName => test(s"read all types of data - $allTypesTableName") { val allTypesFile = goldenTableFile(allTypesTableName).getAbsolutePath val readSchema = tableSchema(allTypesFile) checkAnswer( readParquetFilesUsingKernel(allTypesFile, readSchema), /* actual */ readParquetFilesUsingSpark(allTypesFile, readSchema) /* expected */ ) } } ///////////////////////////////////////////////////////////////////////////////////////////////// // Tests covering reading parquet values into a wider column type // ///////////////////////////////////////////////////////////////////////////////////////////////// /** * Test case for reading a column using a given type. * @param columnName Column to read from the file * @param toType Read type to use. May be different from the actually Parquet type. * @param expectedExpr Expression returning the expected value for each row in the file. */ case class TestCase(columnName: String, toType: DataType, expectedExpr: Int => Any) private val supportedConversions: Seq[TestCase] = Seq( // 'ByteType' column was generated with overflowing values, we need to call i.toByte to also // wrap around here and generate the correct expected values. TestCase("ByteType", ShortType.SHORT, i => if (i % 72 != 0) i.toByte.toShort else null), TestCase("ByteType", IntegerType.INTEGER, i => if (i % 72 != 0) i.toByte.toInt else null), TestCase("ByteType", LongType.LONG, i => if (i % 72 != 0) i.toByte.toLong else null), TestCase("ByteType", DoubleType.DOUBLE, i => if (i % 72 != 0) i.toByte.toDouble else null), TestCase("ShortType", IntegerType.INTEGER, i => if (i % 56 != 0) i else null), TestCase("ShortType", LongType.LONG, i => if (i % 56 != 0) i.toLong else null), TestCase("ShortType", DoubleType.DOUBLE, i => if (i % 56 != 0) i.toDouble else null), TestCase("IntegerType", LongType.LONG, i => if (i % 23 != 0) i.toLong else null), TestCase("IntegerType", DoubleType.DOUBLE, i => if (i % 23 != 0) i.toDouble else null), TestCase( "FloatType", DoubleType.DOUBLE, i => if (i % 28 != 0) (i * 0.234).toFloat.toDouble else null), TestCase( "decimal", new DecimalType(12, 2), i => if (i % 67 != 0) java.math.BigDecimal.valueOf(i * 12352, 2) else null), TestCase( "decimal", new DecimalType(12, 4), i => if (i % 67 != 0) java.math.BigDecimal.valueOf(i * 1235200, 4) else null), TestCase( "decimal", new DecimalType(26, 10), i => if (i % 67 != 0) java.math.BigDecimal.valueOf(i * 12352, 2).setScale(10) else null), TestCase( "IntegerType", new DecimalType(10, 0), i => if (i % 23 != 0) new java.math.BigDecimal(i) else null), TestCase( "IntegerType", new DecimalType(16, 4), i => if (i % 23 != 0) new java.math.BigDecimal(i).setScale(4) else null), TestCase( "LongType", new DecimalType(20, 0), i => if (i % 25 != 0) new java.math.BigDecimal(i + 1) else null), TestCase( "LongType", new DecimalType(28, 6), i => if (i % 25 != 0) new java.math.BigDecimal(i + 1).setScale(6) else null), TestCase("BinaryType", StringType.STRING, i => if (i % 59 != 0) i.toString else null)) // The following conversions are supported by Kernel but not by Spark with parquet-mr. // TODO: We should properly reject these conversions, a lot of them produce wrong results. // Collecting them here to document the current behavior. private val kernelOnlyConversions: Seq[TestCase] = Seq( // This conversions will silently overflow. TestCase("ShortType", ByteType.BYTE, i => if (i % 56 != 0) i.toByte else null), TestCase("IntegerType", ByteType.BYTE, i => if (i % 23 != 0) i.toByte else null), TestCase("IntegerType", ShortType.SHORT, i => if (i % 23 != 0) i.toShort else null), // This is reading the unscaled decimal value as long which is wrong. TestCase("decimal", LongType.LONG, i => if (i % 67 != 0) i.toLong * 12352 else null), // The following conversions seem legit, although Spark rejects them. TestCase("ByteType", DateType.DATE, i => if (i % 72 != 0) i.toByte.toInt else null), TestCase("ShortType", DateType.DATE, i => if (i % 56 != 0) i else null), TestCase("IntegerType", DateType.DATE, i => if (i % 23 != 0) i else null), TestCase("StringType", BinaryType.BINARY, i => if (i % 57 != 0) i.toString.getBytes else null)) for (testCase <- supportedConversions ++ kernelOnlyConversions) test(s"parquet supported conversion - ${testCase.columnName} -> ${testCase.toType.toString}") { val inputLocation = goldenTablePath("parquet-all-types") val readSchema = new StructType().add(testCase.columnName, testCase.toType) val result = readParquetFilesUsingKernel(inputLocation, readSchema) val expected = (0 until 200) .map { i => TestRow(testCase.expectedExpr(i)) } checkAnswer(result, expected) if (!kernelOnlyConversions.contains(testCase)) { withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false") { val sparkResult = readParquetFilesUsingSpark(inputLocation, readSchema) checkAnswer(result, sparkResult) } } } test(s"parquet supported conversion - date -> timestamp_ntz") { val timezones = Seq("UTC", "Iceland", "PST", "America/Los_Angeles", "Etc/GMT+9", "Asia/Beirut", "JST") for (fromTimezone <- timezones; toTimezone <- timezones) { val inputLocation = goldenTablePath(s"data-reader-date-types-$fromTimezone") TimeZone.setDefault(TimeZone.getTimeZone(toTimezone)) val readSchema = new StructType().add("date", TimestampNTZType.TIMESTAMP_NTZ) val result = readParquetFilesUsingKernel(inputLocation, readSchema) // 1577836800000000L -> 2020-01-01 00:00:00 UTC checkAnswer(result, Seq(TestRow(1577836800000000L))) } } def checkParquetReadError(inputLocation: String, readSchema: StructType): Unit = { val ex = intercept[Throwable] { readParquetFilesUsingKernel(inputLocation, readSchema) } // We don't properly reject conversions and the error we get vary a lot, this checks various // error message we may get as result. // TODO(delta-io/delta#4493): Uniformize rejecting unsupported conversions. assert( ex.getMessage.contains("Can not read value") || ex.getMessage.contains("column with Parquet type") || ex.getMessage.contains("Unable to create Parquet converter for") || ex.getMessage.contains("Found Delta type Decimal") || ex.getMessage.contains("cannot be cast to")) } for ( column <- Seq( "BooleanType", "ByteType", "ShortType", "IntegerType", "LongType", "FloatType", "DoubleType", "StringType", "BinaryType") ) { test(s"parquet unsupported conversion from $column") { val inputLocation = goldenTablePath("parquet-all-types") val supportedTypes = (supportedConversions ++ kernelOnlyConversions) .filter(_.columnName == column) .map(_.toType) val unsupportedTypes = ALL_TYPES .filterNot(supportedTypes.contains) .filterNot(_.getClass.getSimpleName == column) for (toType <- unsupportedTypes) { val readSchema = new StructType().add(column, toType) withClue(s"Converting $column to $toType") { checkParquetReadError(inputLocation, readSchema) } } } } test(s"parquet unsupported conversion from decimal") { val inputLocation = goldenTablePath("parquet-all-types") // 'decimal' column is Decimal(10, 2) which fits into a long. for (toType <- ALL_TYPES.filterNot(_ == LongType.LONG)) { val readSchema = new StructType().add("decimal", toType) withClue(s"Converting decimal to $toType") { checkParquetReadError(inputLocation, readSchema) } } } test("read subset of columns") { val tablePath = goldenTableFile("parquet-all-types").getAbsolutePath val readSchema = new StructType() .add("byteType", ByteType.BYTE) .add("booleanType", BooleanType.BOOLEAN) .add("stringType", StringType.STRING) .add("dateType", DateType.DATE) .add( "nested_struct", new StructType() .add("aa", StringType.STRING) .add("ac", new StructType().add("aca", IntegerType.INTEGER))) .add("array_of_prims", new ArrayType(IntegerType.INTEGER, true)) checkAnswer( readParquetFilesUsingKernel(tablePath, readSchema), /* actual */ readParquetFilesUsingSpark(tablePath, readSchema) /* expected */ ) } test("read subset of columns with missing columns in file") { val tablePath = goldenTableFile("parquet-all-types").getAbsolutePath val readSchema = new StructType() .add("booleanType", BooleanType.BOOLEAN) .add("integerType", IntegerType.INTEGER) .add("missing_column_struct", new StructType().add("ab", IntegerType.INTEGER)) .add("longType", LongType.LONG) .add("missing_column_primitive", DateType.DATE) .add( "nested_struct", new StructType() .add("aa", StringType.STRING) .add("ac", new StructType().add("aca", IntegerType.INTEGER))) checkAnswer( readParquetFilesUsingKernel(tablePath, readSchema), /* actual */ readParquetFilesUsingSpark(tablePath, readSchema) /* expected */ ) } test("read columns with int96 timestamp_ntz") { // Spark doesn't support writing timestamp_NTZ as INT96 (although reads are) // So we're reusing a pre-written file directly. val filePath = getTestResourceFilePath("parquet/parquet-timestamp_ntz_int96.parquet") val readSchema = new StructType() .add("id", IntegerType.INTEGER) .add("time", TimestampNTZType.TIMESTAMP_NTZ) checkAnswer( readParquetFilesUsingKernel(filePath, readSchema), /* actual */ Seq(TestRow(1, 915181200000000L) /* expected */ )) } test("request row indices") { val readSchema = new StructType() .add("id", LongType.LONG) .add(ROW_INDEX) val path = getTestResourceFilePath("parquet-basic-row-indexes") val actResult1 = readParquetFilesUsingKernel(path, readSchema) val expResult1 = (0L until 30L) .map(i => TestRow(i, if (i < 10) i else if (i < 20) i - 10L else i - 20L)) checkAnswer(actResult1, expResult1) // File with multiple row-groups [0, 20000) where rowIndex = id val filePath = getTestResourceFilePath("parquet/row_index_multiple_row_groups.parquet") val actResult2 = readParquetFilesUsingKernel(filePath, readSchema) val expResult2 = (0L until 20000L).map(i => TestRow(i, i)) checkAnswer(actResult2, expResult2) } ///////////////////////////////////////////////////////////////////////////////////////////////// // Test compatibility with Parquet legacy format files // ///////////////////////////////////////////////////////////////////////////////////////////////// // Test and the test file are copied from Spark's `ParquetThriftCompatibilitySuite` test("read parquet file generated by parquet-thrift") { val parquetFilePath = getTestResourceFilePath("parquet/parquet-thrift-compat.snappy.parquet") val readSchema = new StructType() .add("boolColumn", BooleanType.BOOLEAN) .add("byteColumn", ByteType.BYTE) .add("shortColumn", ShortType.SHORT) .add("intColumn", IntegerType.INTEGER) .add("longColumn", LongType.LONG) .add("doubleColumn", DoubleType.DOUBLE) // Thrift `BINARY` values are actually unencoded `STRING` values, and thus are always // treated as `BINARY (UTF8)` in parquet-thrift, since parquet-thrift always assume // Thrift `STRING`s are encoded using UTF-8. .add("binaryColumn", StringType.STRING) .add("stringColumn", StringType.STRING) .add("enumColumn", StringType.STRING) // maybe indicates nullable columns, above ones are non-nullable .add("maybeBoolColumn", BooleanType.BOOLEAN) .add("maybeByteColumn", ByteType.BYTE) .add("maybeShortColumn", ShortType.SHORT) .add("maybeIntColumn", IntegerType.INTEGER) .add("maybeLongColumn", LongType.LONG) .add("maybeDoubleColumn", DoubleType.DOUBLE) // Thrift `BINARY` values are actually unencoded `STRING` values, and thus are always // treated as `BINARY (UTF8)` in parquet-thrift, since parquet-thrift always assume // Thrift `STRING`s are encoded using UTF-8. .add("maybeBinaryColumn", StringType.STRING) .add("maybeStringColumn", StringType.STRING) .add("maybeEnumColumn", StringType.STRING) // TODO: not working - separate PR to handle 2-level legacy lists // .add("stringsColumn", new ArrayType(StringType.STRING, true /* containsNull */)) // .add("intSetColumn", new ArrayType(IntegerType.INTEGER, true /* containsNull */)) .add( "intToStringColumn", new MapType(IntegerType.INTEGER, StringType.STRING, true /* valueContainsNull */ )) // TODO: not working - separate PR to handle 2-level legacy lists // .add("complexColumn", new MapType( // IntegerType.INTEGER, // new ArrayType( // new StructType() // .add("nestedIntsColumn", new ArrayType(IntegerType.INTEGER, true /* containsNull */)) // .add("nestedStringColumn", StringType.STRING) // .add("stringColumn", StringType.STRING), // true /* containsNull */), // true /* valueContainsNull */)) assert(parquetFileRowCount(parquetFilePath) === 10) checkAnswer( readParquetFilesUsingKernel(parquetFilePath, readSchema), /* actual */ readParquetFilesUsingSpark(parquetFilePath, readSchema) /* expected */ ) } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/parquet/ParquetFileWriterSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.parquet import java.lang.{Double => DoubleJ, Float => FloatJ} import io.delta.golden.GoldenTableUtils.{goldenTableFile, goldenTablePath} import io.delta.kernel.data.{ColumnarBatch, FilteredColumnarBatch} import io.delta.kernel.defaults.internal.DefaultKernelUtils import io.delta.kernel.defaults.utils.{DefaultVectorTestUtils, ExpressionTestUtils, TestRow} import io.delta.kernel.expressions.{Column, Literal, Predicate} import io.delta.kernel.internal.TableConfig import io.delta.kernel.internal.util.ColumnMapping import io.delta.kernel.internal.util.ColumnMapping.{convertToPhysicalSchema, ColumnMappingMode} import io.delta.kernel.types._ import io.delta.kernel.utils.DataFileStatus import org.apache.spark.sql.{functions => sparkfn} import org.scalatest.funsuite.AnyFunSuite /** * Test strategy for [[ParquetFileWriter]] *

* Golden tables already have Parquet files containing various supported * data types and variations (null, non-nulls, decimal types, nested nested types etc.). * We will use these files to simplify the tests for ParquetFileWriter. Alternative is to * generate the test data in the tests and try to write as Parquet files, but that would be a lot * of test code to cover all the combinations. *

* Using the golden Parquet files in combination with Kernel Parquet reader and Spark Parquet * reader we will reduce the test code and also test the inter-working of the Parquet writer with * the Parquet readers. *

* High level steps in the test: * 1) read data using the Kernel Parquet reader to generate the data in [[ColumnarBatch]]es * 2) Optional: filter the data from (1) and generate [[FilteredColumnarBatch]]es * 3) write the data back to new Parquet file(s) using the ParquetFileWriter that we are * testing. We will test the following variations: * 3.1) change target file size and stats collection columns etc. * 4) verification * 4.1) read the new Parquet file(s) using the Kernel Parquet reader and compare with (2) * 4.2) read the new Parquet file(s) using the Spark Parquet reader and compare with (2) * 4.3) verify the stats returned in (3) are correct using the Spark Parquet reader */ class ParquetFileWriterSuite extends AnyFunSuite with ParquetSuiteBase with DefaultVectorTestUtils with ExpressionTestUtils { Seq( // Test cases reading and writing all types of data with or without stats collection Seq((200, 67), (1024, 16), (1048576, 1)).map { case (targetFileSize, expParquetFileCount) => ( "write all types (no stats)", // test name "parquet-all-types", // input table where the data is read and written targetFileSize, expParquetFileCount, 200, /* expected number of rows written to Parquet files */ Option.empty[Predicate], // predicate for filtering what rows to write to parquet files Seq.empty[Column], // list of columns to collect stats as part of the Parquet file write 0 // how many columns have the stats collected from given list above ) }, // Test cases reading and writing decimal types data with different precisions // They trigger different paths in the Parquet writer as how decimal types are stored in Parquet // based on the precision and scale. Seq((1048576, 3), (2048576, 2)).map { case (targetFileSize, expParquetFileCount) => ( "write decimal all types (with stats)", // test name "parquet-decimal-type", targetFileSize, expParquetFileCount, 99998, /* expected number of rows written to Parquet files */ Option.empty[Predicate], // predicate for filtering what rows to write to parquet files leafLevelPrimitiveColumns( Seq.empty, tableSchema(goldenTablePath("parquet-decimal-type"))), 4 // how many columns have the stats collected from given list above ) }, // Test cases reading and writing data with field ids. This is for column mapping mode ID. Seq((1024, 1)).map { case (targetFileSize, expParquetFileCount) => ( "write data with field ids (no stats)", // test name "table-with-columnmapping-mode-id", targetFileSize, expParquetFileCount, 6, /* expected number of rows written to Parquet files */ Option.empty[Predicate], // predicate for filtering what rows to write to parquet files Seq.empty[Column], // list of columns to collect stats as part of the Parquet file write 0 // how many columns have the stats collected from given list above ) }, // Test cases reading and writing only a subset of data passing a predicate. Seq((200, 26), (1024, 6), (1048576, 1)).map { case (targetFileSize, expParquetFileCount) => ( "write filtered all types (no stats)", // test name "parquet-all-types", // input table where the data is read and written targetFileSize, expParquetFileCount, 77, /* expected number of rows written to Parquet files */ // predicate for filtering what input rows to write to parquet files Some(greaterThanOrEqual(col("ByteType"), Literal.ofInt(50))), Seq.empty[Column], // list of columns to collect stats as part of the Parquet file write 0 // how many columns have the stats collected from given list above ) }, // Test cases reading and writing all types of data WITH stats collection Seq((200, 67), (1024, 16), (1048576, 1)).map { case (targetFileSize, expParquetFileCount) => ( "write all types (with stats for all leaf-level columns)", // test name "parquet-all-types", // input table where the data is read and written targetFileSize, expParquetFileCount, 200, /* expected number of rows written to Parquet files */ Option.empty[Predicate], // predicate for filtering what rows to write to parquet files leafLevelPrimitiveColumns(Seq.empty, tableSchema(goldenTablePath("parquet-all-types"))), 15 // how many columns have the stats collected from given list above ) }, // Test cases reading and writing all types of data with a partial column set stats collection Seq((200, 67), (1024, 16), (1048576, 1)).map { case (targetFileSize, expParquetFileCount) => ( "write all types (with stats for a subset of leaf-level columns)", // test name "parquet-all-types", // input table where the data is read and written targetFileSize, expParquetFileCount, 200, /* expected number of rows written to Parquet files */ Option.empty[Predicate], // predicate for filtering what rows to write to parquet files Seq( new Column("ByteType"), new Column("DateType"), new Column(Array("nested_struct", "aa")), new Column(Array("nested_struct", "ac", "aca")), new Column(Array("nested_struct", "ac")), // stats are not collected for struct types new Column("nested_struct"), // stats are not collected for struct types new Column("array_of_prims"), // stats are not collected for array types new Column("map_of_prims") // stats are not collected for map types ), 4 // how many columns have the stats collected from given list above ) }, // Decimal types with various precision and scales Seq((10000, 1)).map { case (targetFileSize, expParquetFileCount) => ( "write decimal various scales and precision (with stats)", // test name "decimal-various-scale-precision", targetFileSize, expParquetFileCount, 3, /* expected number of rows written to Parquet files */ Option.empty[Predicate], // predicate for filtering what rows to write to parquet files leafLevelPrimitiveColumns( Seq.empty, tableSchema(goldenTablePath("decimal-various-scale-precision"))), 29 // how many columns have the stats collected from given list above ) }, // Read a iceberg compat v2 data with field ids and nested ids, and write it back Seq((200, 1)).map { case (targetFileSize, expParquetFileCount) => ( "write iceberg compat v2 data with field ids (no stats)", // test name "table-with-columnmapping-mode-id", // input table where the data is read targetFileSize, expParquetFileCount, 6, /* input table has 6 rows, exp these in output Parquet files */ Option.empty[Predicate], // predicate for filtering what rows to write to parquet files Seq.empty, // list of columns to collect statistics on 0 // how many columns have the stats collected from given list above ) }).flatten.foreach { case (name, input, fileSize, expFileCount, expRowCount, predicate, statsCols, expStatsColCnt) => test(s"$name: targetFileSize=$fileSize, predicate=$predicate") { withTempDir { tempPath => val targetDir = tempPath.getAbsolutePath val inputLocation = goldenTablePath(input) val schema = tableSchema(inputLocation) val hasColumnMappingId = hasTableProperty(inputLocation, TableConfig.COLUMN_MAPPING_MODE.getKey, "id") val hasIcebergCompatV2 = hasTableProperty(inputLocation, TableConfig.ICEBERG_COMPAT_V2_ENABLED.getKey, "true") val physicalSchema = if (hasColumnMappingId || hasIcebergCompatV2) { convertToPhysicalSchema(schema, schema, ColumnMappingMode.ID) } else { schema } val dataToWrite = readParquetUsingKernelAsColumnarBatches(inputLocation, physicalSchema) // read data // Convert the schema of the data to the physical schema with field ids .map(_.withNewSchema(physicalSchema)) // convert the data to filtered columnar batches .map(_.toFiltered(predicate)) val writeOutput = writeToParquetUsingKernel(dataToWrite, targetDir, fileSize, statsCols) assert(parquetFileCount(targetDir) === expFileCount) assert(parquetFileRowCount(targetDir) == expRowCount) verifyContent(targetDir, dataToWrite) if (hasIcebergCompatV2 || hasColumnMappingId) { verifyFieldIds(targetDir, physicalSchema, hasIcebergCompatV2) } verifyStatsUsingSpark(targetDir, writeOutput, schema, statsCols, expStatsColCnt) } } } test("columnar batches containing different schema") { withTempDir { tempPath => val targetDir = tempPath.getAbsolutePath // First batch with one column val batch1 = columnarBatch(testColumnVector(10, IntegerType.INTEGER)) // Batch with two columns val batch2 = columnarBatch( testColumnVector(10, IntegerType.INTEGER), testColumnVector(10, LongType.LONG)) // Batch with one column as first batch but different data type val batch3 = columnarBatch(testColumnVector(10, LongType.LONG)) Seq(Seq(batch1, batch2), Seq(batch1, batch3)).foreach { dataToWrite => val e = intercept[IllegalArgumentException] { writeToParquetUsingKernel(dataToWrite.map(_.toFiltered), targetDir) } assert(e.getMessage.contains("Input data has columnar batches with different schemas:")) } } } /** * Tests to cover floating point comparison special cases in Parquet. * - https://issues.apache.org/jira/browse/PARQUET-1222 * - Parquet doesn't collect stats if NaN is present in the column values * - Min is written as -0.0 instead of 0.0 and max is written as 0.0 instead of -0.0 */ test("float/double type column stats collection") { // Try writing different set of floating point values and verify the stats are correct // (float values, double values, exp rowCount in files, exp stats (min, max, nullCount) Seq( ( // no stats collection as NaN is present Seq( Float.NegativeInfinity, Float.MinValue, -1.0f, -0.0f, 0.0f, 1.0f, null, Float.MaxValue, Float.PositiveInfinity, Float.NaN), Seq( Double.NegativeInfinity, Double.MinValue, -1.0d, -0.0d, 0.0d, 1.0d, null, Double.MaxValue, Double.PositiveInfinity, Double.NaN), 10, (null, null, null), (null, null, null)), ( // Min and max are infinities Seq( Float.NegativeInfinity, Float.MinValue, -1.0f, -0.0f, 0.0f, 1.0f, null, Float.MaxValue, Float.PositiveInfinity), Seq( Double.NegativeInfinity, Double.MinValue, -1.0d, -0.0d, 0.0d, 1.0d, null, Double.MaxValue, Double.PositiveInfinity), 9, (Float.NegativeInfinity, Float.PositiveInfinity, 1L), (Double.NegativeInfinity, Double.PositiveInfinity, 1L)), ( // no infinities or NaN - expect stats collected Seq(Float.MinValue, -1.0f, -0.0f, 0.0f, 1.0f, null, Float.MaxValue), Seq(Double.MinValue, -1.0d, -0.0d, 0.0d, 1.0d, null, Double.MaxValue), 7, (Float.MinValue, Float.MaxValue, 1L), (Double.MinValue, Double.MaxValue, 1L)), ( // Only negative numbers. Max is 0.0 instead of -0.0 to avoid PARQUET-1222 Seq(Float.NegativeInfinity, Float.MinValue, -1.0f, -0.0f, null), Seq(Double.NegativeInfinity, Double.MinValue, -1.0d, -0.0d, null), 5, (Float.NegativeInfinity, 0.0f, 1L), (Double.NegativeInfinity, 0.0d, 1L)), ( // Only positive numbers. Min is -0.0 instead of 0.0 to avoid PARQUET-1222 Seq(0.0f, 1.0f, null, Float.MaxValue, Float.PositiveInfinity), Seq(0.0d, 1.0d, null, Double.MaxValue, Double.PositiveInfinity), 5, (-0.0f, Float.PositiveInfinity, 1L), (-0.0d, Double.PositiveInfinity, 1L))).foreach { case (floats: Seq[FloatJ], doubles: Seq[DoubleJ], expRowCount, expFltStats, expDblStats) => withTempDir { tempPath => val targetDir = tempPath.getAbsolutePath val testBatch = columnarBatch(floatVector(floats), doubleVector(doubles)) val dataToWrite = Seq(testBatch.toFiltered) val writeOutput = writeToParquetUsingKernel( dataToWrite, targetDir, statsColumns = Seq(col("col_0"), col("col_1"))) assert(parquetFileRowCount(targetDir) == expRowCount) verifyContent(targetDir, dataToWrite) val stats = writeOutput.head.getStatistics.get() def getStats(column: String): (Object, Object, Object) = ( Option(stats.getMinValues.get(col(column))).map(_.getValue).orNull, Option(stats.getMaxValues.get(col(column))).map(_.getValue).orNull, Option(stats.getNullCount.get(col(column))).orNull) assert(getStats("col_0") === expFltStats) assert(getStats("col_1") === expDblStats) } } } test(s"invalid target file size") { withTempDir { tempPath => val targetDir = tempPath.getAbsolutePath val inputLocation = goldenTableFile("parquet-all-types").toString val schema = tableSchema(inputLocation) val dataToWrite = readParquetUsingKernelAsColumnarBatches(inputLocation, schema) .map(_.toFiltered) Seq(-1, 0).foreach { targetFileSize => val e = intercept[IllegalArgumentException] { writeToParquetUsingKernel(dataToWrite, targetDir, targetFileSize) } assert(e.getMessage.contains("Invalid target Parquet file size: " + targetFileSize)) } } } def verifyStatsUsingSpark( actualFileDir: String, actualFileStatuses: Seq[DataFileStatus], fileDataSchema: StructType, statsColumns: Seq[Column], expStatsColCount: Int): Unit = { val actualStatsOutput = actualFileStatuses .map { fileStatus => // validate there are no more the expected number of stats columns assert(fileStatus.getStatistics.isPresent) assert(fileStatus.getStatistics.get().getMinValues.size() === expStatsColCount) assert(fileStatus.getStatistics.get().getMaxValues.size() === expStatsColCount) assert(fileStatus.getStatistics.get().getNullCount.size() === expStatsColCount) // Convert to TestRow for comparison with the actual values computing using Spark. fileStatus.toTestRow(statsColumns) } // Use spark to fetch the stats from the parquet files use them as the expected statistics // Compare them with the actual stats returned by the Kernel's Parquet writer. val df = spark.read .format("parquet") .parquet(actualFileDir) .to(fileDataSchema.toSpark) .select( sparkfn.col("*"), // select all columns from the parquet files sparkfn.col("_metadata.file_path").as("path"), // select file path sparkfn.col("_metadata.file_size").as("size"), // select file size // select mod time and convert to millis sparkfn.unix_timestamp( sparkfn.col("_metadata.file_modification_time")).as("modificationTime")) .groupBy("path", "size", "modificationTime") val nullStats = Seq(sparkfn.lit(null), sparkfn.lit(null), sparkfn.lit(null)) // Add the row count aggregation val aggs = Seq(sparkfn.count(sparkfn.col("*")).as("rowCount")) ++ // add agg for each stats column to get min, max and null count statsColumns .flatMap { statColumn => val dataType = DefaultKernelUtils.getDataType(fileDataSchema, statColumn) dataType match { case _: StructType => nullStats // no concept of stats for struct types case _: ArrayType => nullStats // no concept of stats for array types case _: MapType => nullStats // no concept of stats for map types case _ => // for all other types val colName = statColumn.toPath Seq( sparkfn.min(colName).as("min_" + colName), sparkfn.max(colName).as("max_" + colName), sparkfn.sum(sparkfn.when( sparkfn.col(colName).isNull, 1).otherwise(0)).as("nullCount_" + colName)) } } val expectedStatsOutput = df.agg(aggs.head, aggs.tail: _*).collect().map(TestRow(_)) checkAnswer(actualStatsOutput, expectedStatsOutput) } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/parquet/ParquetReaderPredicatePushdownSuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.parquet import java.nio.file.Files import java.sql.Date import java.util.Optional import io.delta.golden.GoldenTableUtils.goldenTablePath import io.delta.kernel.defaults.utils.{ExpressionTestUtils, TestRow} import io.delta.kernel.expressions._ import io.delta.kernel.expressions.Literal.{ofBinary, ofBoolean, ofDate, ofDouble, ofFloat, ofInt, ofLong, ofNull, ofString} import io.delta.kernel.internal.util.InternalUtils.daysSinceEpoch import io.delta.kernel.test.VectorTestUtils import io.delta.kernel.types.{IntegerType, StructType} import org.apache.spark.sql.{types => sparktypes, Row} import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite class ParquetReaderPredicatePushdownSuite extends AnyFunSuite with BeforeAndAfterAll with ParquetSuiteBase with VectorTestUtils with ExpressionTestUtils { ////////////////////////////////////////////////////////////////////////////////// // Test data generation and helper methods ////////////////////////////////////////////////////////////////////////////////// var testParquetTable: String = "" override def beforeAll(): Unit = { super.beforeAll() testParquetTable = Files.createTempDirectory("tempDir").toString // Generate a test Parquet file with 20 row groups. Each row group has 100 rows. // Parquet-mr checks whether the current row group has reached the limit or for every 100 rows. // We set the `parquet.block.size` to very low, so for every 100 rows, it will create a // new row group. val rows = Seq.range(0, 20).flatMap(i => generateRowsGroup(i)) val df = spark.createDataFrame(spark.sparkContext.parallelize(rows), testTableSchema) withSQLConf("parquet.block.size" -> 1.toString) { df.repartition(1) .orderBy("rowId") .write .format("delta") .mode("append") .save(testParquetTable) } } // test table schema val testTableSchema: sparktypes.StructType = { // These are the only supported column types in Parquet filter push down def allTypesSchema(): Array[sparktypes.StructField] = { Seq( sparktypes.StructField("byteCol", sparktypes.ByteType), sparktypes.StructField("shortCol", sparktypes.ShortType), sparktypes.StructField("intCol", sparktypes.IntegerType), sparktypes.StructField("longCol", sparktypes.LongType), sparktypes.StructField("floatCol", sparktypes.FloatType), sparktypes.StructField("doubleCol", sparktypes.DoubleType), sparktypes.StructField("stringCol", sparktypes.StringType), // column with values that are truncated in stats sparktypes.StructField("truncatedStringCol", sparktypes.StringType), sparktypes.StructField("binaryCol", sparktypes.BinaryType), sparktypes.StructField("truncatedBinaryCol", sparktypes.BinaryType), sparktypes.StructField("booleanCol", sparktypes.BooleanType), sparktypes.StructField("dateCol", sparktypes.DateType)).toArray } // supported data type columns as top level columns new sparktypes.StructType(allTypesSchema()) // supported data type columns as nested columns .add("nested", sparktypes.StructType(allTypesSchema())) // row id to help with the test results verification .add("rowId", sparktypes.IntegerType) } private def generateRowsGroup(rowGroupIdx: Int): Seq[Row] = { def values(rowId: Int): Seq[Any] = { // One of the columns in each row group is all nulls or all non-nulls depending on // the [[rowGroupIdx]]. This helps to verify the test results for `is null` and // `is not null` pushdown Seq( // byteCol if (rowGroupIdx == 0) null /* all nulls */ else if (rowGroupIdx == 11) rowId.byteValue() /* all non-nulls */ else (if (rowId % 72 != 0) rowId.byteValue() else null), /* mix of nulls and non-nulls */ // shortCol if (rowGroupIdx == 1) null else if (rowGroupIdx == 10) rowId.shortValue() else (if (rowId % 56 != 0) rowId.shortValue() else null), // intCol if (rowGroupIdx == 2) null else if (rowGroupIdx == 9) rowId else (if (rowId % 23 != 0) rowId else null), // longCol if (rowGroupIdx == 3) null else if (rowGroupIdx == 8) (rowId + 1).longValue() else (if (rowId % 25 != 0) (rowId + 1).longValue() else null), // floatCol if (rowGroupIdx == 4) null else if (rowGroupIdx == 7) (rowId + 0.125).floatValue() else (if (rowId % 28 != 0) (rowId + 0.125).floatValue() else null), // doubleCol if (rowGroupIdx == 5) null else if (rowGroupIdx == 6) (rowId + 0.000001).doubleValue() else (if (rowId % 54 != 0) (rowId + 0.000001).doubleValue() else null), // stringCol if (rowGroupIdx == 6) null else if (rowGroupIdx == 5) "%05d".format(rowId) else (if (rowId % 57 != 0) "%05d".format(rowId) else null), // truncatedStringCol - stats will be truncated as the value is too long if (rowGroupIdx == 7) null else if (rowGroupIdx == 4) "%050d".format(rowId) else (if (rowId % 57 != 0) "%050d".format(rowId) else null), // binaryCol if (rowGroupIdx == 8) null else if (rowGroupIdx == 3) "%06d".format(rowId).getBytes else (if (rowId % 59 != 0) "%06d".format(rowId).getBytes else null), // truncatedBinaryCol - stats will be truncated as the value is too long if (rowGroupIdx == 9) null else if (rowGroupIdx == 2) "%060d".format(rowId).getBytes else (if (rowId % 59 != 0) "%060d".format(rowId).getBytes else null), // booleanCol if (rowGroupIdx == 10) null else if (rowGroupIdx == 1) rowId % 2 == 0 // alternative between true and false for each row group else (if (rowId % 29 != 0) rowGroupIdx % 2 == 0 else null), // dateCol if (rowGroupIdx == 11) null else if (rowGroupIdx == 0) new Date(rowId * 86400000L) else (if (rowId % 61 != 0) new Date(rowId * 86400000L) else null)) } Seq.range(rowGroupIdx * 100, (rowGroupIdx + 1) * 100).map { rowId => Row.fromSeq( values(rowId) ++ // top-level column values Seq( Row.fromSeq(values(rowId)), // nested column values rowId // row id to help with the test results verification )) } } def generateExpData(rowGroupIndexes: Seq[Int]): Seq[TestRow] = { spark.createDataFrame( spark.sparkContext.parallelize(rowGroupIndexes.flatMap(i => generateRowsGroup(i))), testTableSchema) .collect .map(TestRow(_)) } private def readUsingKernel(tablePath: String, predicate: Predicate): Seq[TestRow] = { val readSchema: StructType = tableSchema(testParquetTable) readParquetFilesUsingKernel(tablePath, readSchema, Optional.of(predicate)) } private def assertConvertedFilterIsEmpty(predicate: Predicate, tablePath: String): Unit = { val parquetFileSchema = parquetFiles(tablePath).map(_.getPath).map(footer(_)).head.getFileMetaData.getSchema assert( !ParquetFilterUtils.toParquetFilter(parquetFileSchema, predicate).isPresent, "Predicate should not be converted to Parquet filter") } ////////////////////////////////////////////////////////////////////////////////// // End-2-end tests ////////////////////////////////////////////////////////////////////////////////// Seq( // filter on int type column ( eq(col("intCol"), ofInt(20)), // top-level column eq(col("nested", "intCol"), ofInt(20)), // nested column Seq(0) // expected row groups ), // filter on long type column ( gt(col("longCol"), ofLong(1600)), gt(col("nested", "longCol"), ofLong(1600)), Seq(16, 17, 18, 19) // expected row groups ), // filter on float type column ( lt(col("floatCol"), ofFloat(1000.0f)), lt(col("nested", "floatCol"), ofFloat(1000.0f)), Seq(0, 1, 2, 3, 5, 6, 7, 8, 9) // expected row groups - row group 4 has all nulls ), // filter on double type column ( gt(col("doubleCol"), ofDouble(1000.0)), gt(col("nested", "doubleCol"), ofDouble(1000.0)), Seq(10, 11, 12, 13, 14, 15, 16, 17, 18, 19) // expected row groups ), // filter on boolean type column ( eq(col("booleanCol"), ofBoolean(true)), eq(col("nested", "booleanCol"), ofBoolean(true)), // expected row groups // 1 has mix of true/false (included), 10 has all nulls (not included) Seq(0, 1, 2, 4, 6, 8, 12, 14, 16, 18)), // filter on date type column ( lte( col("dateCol"), ofDate( daysSinceEpoch(new Date(500 * 86400000L /* millis in a day */ )))), lte( col("nested", "dateCol"), ofDate( daysSinceEpoch(new Date(500 * 86400000L /* millis in a day */ )))), Seq(0, 1, 2, 3, 4, 5) // expected row groups ), // filter on string type column ( eq(col("stringCol"), ofString("%05d".format(300))), eq(col("nested", "stringCol"), ofString("%05d".format(300))), Seq(3) // expected row groups ), // filter on binary type column ( gte(col("binaryCol"), ofBinary("%06d".format(1700).getBytes)), gte(col("nested", "binaryCol"), ofBinary("%06d".format(1700).getBytes)), Seq(17, 18, 19) // expected row groups ), // filter on truncated stats string type column ( gte(col("truncatedStringCol"), ofString("%050d".format(300))), gte(col("nested", "truncatedStringCol"), ofString("%050d".format(300))), // expected row groups Seq(3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19) // 7 has all nulls ), // filter on truncated stats binary type column ( lte(col("truncatedBinaryCol"), ofBinary("%060d".format(600).getBytes)), lte(col("nested", "truncatedBinaryCol"), ofBinary("%060d".format(600).getBytes)), Seq(0, 1, 2, 3, 4, 5, 6) // expected row groups )).foreach { // boolean, int32, data, int64, float, double, binary, string // Test table has 20 row groups, each with 100 rows. case (predicateTopLevelCol, predicateNestedCol, expRowGroups) => Seq(predicateTopLevelCol, predicateNestedCol).foreach { predicate => test(s"filter pushdown: $predicate") { val actualData = readUsingKernel(testParquetTable, predicate) val expOutputRowCount = expRowGroups.length * 100 // 100 rows per row group assert(actualData.size === expOutputRowCount, s"predicate: $predicate") checkAnswer(actualData, generateExpData(expRowGroups)) } } } // IS NULL and IS NOT NULL tests Seq( // (columnName, row groups with all nulls, row groups with all non-nulls) ("byteCol", Seq(0), Seq(11)), // int type column ("shortCol", Seq(1), Seq(10)), // short type column ("intCol", Seq(2), Seq(9)), // int type column ("longCol", Seq(3), Seq(8)), // long type column ("floatCol", Seq(4), Seq(7)), // float type column ("doubleCol", Seq(5), Seq(6)), // double type column ("stringCol", Seq(6), Seq(5)), // string type column ("truncatedStringCol", Seq(7), Seq(4)), // truncatedStringCol type column ("binaryCol", Seq(8), Seq(3)), // binary type column ("truncatedBinaryCol", Seq(9), Seq(2)), // truncatedBinaryCol type column ("booleanCol", Seq(10), Seq(1)), // boolean type column ("dateCol", Seq(11), Seq(0)) // date type column ).foreach { // Test table has 20 row groups, each with 100 rows. case (colName, allNullsRowGroups, allNonNullsRowGroups) => // Test predicate on both top-level and nested columns Seq(col(colName), col("nested", colName)).foreach { column => val isNullFilter = isNull(column) test(s"filter pushdown: $isNullFilter") { val actualData = readUsingKernel(testParquetTable, isNullFilter) val expOutputRowCount = 100 * (20 - 1) // 100 rows per row group // we get everything expect the rowgroup that has all non-nulls val expRowGroups = (0 until 20).filter(!allNonNullsRowGroups.contains(_)) assert(actualData.size === expOutputRowCount, s"predicate: $isNullFilter") checkAnswer(actualData, generateExpData(expRowGroups)) // not (col is null) should return all row groups exception the one with all nulls assertNot(isNullFilter, (0 until 20).filter(!allNullsRowGroups.contains(_))) } val isNotNullFilter = isNotNull(column) test(s"filter pushdown: $isNotNullFilter") { val actualData = readUsingKernel(testParquetTable, isNotNullFilter) val expOutputRowCount = 100 * (20 - 1) // 100 rows per row group // we get everything expect the rowgroup that has all nulls val expRowGroups = (0 until 20).filter(!allNullsRowGroups.contains(_)) assert(actualData.size === expOutputRowCount, s"predicate: $isNotNullFilter") checkAnswer(actualData, generateExpData(expRowGroups)) // not (col is not null) should return all row groups exception the one with all non-nulls assertNot(isNotNullFilter, (0 until 20).filter(!allNonNullsRowGroups.contains(_))) } } } test("for a column that doesn't exist in the table") { val testPredicate = predicate("=", col("nonExistentCol"), ofInt(20)) assertConvertedFilterIsEmpty(testPredicate, testParquetTable) val actData = readUsingKernel(testParquetTable, testPredicate) // contains all the data in the table as the predicate is not pushed down checkAnswer(actData, generateExpData(Seq.range(0, 20))) } test("literal and column are swapped") { val testPredicate = predicate("=", ofInt(20), col("intCol")) val actData = readUsingKernel(testParquetTable, testPredicate) checkAnswer(actData, generateExpData(Seq(0))) } test("comparator literal value is null") { val testPredicate = predicate("=", col("intCol"), ofNull(IntegerType.INTEGER)) assertConvertedFilterIsEmpty(testPredicate, testParquetTable) val actData = readUsingKernel(testParquetTable, testPredicate) // contains all the data in the table as the predicate is not pushed down checkAnswer(actData, generateExpData(Seq.range(0, 20))) } test("comparator that compare column and column") { val testPredicate = predicate("=", col("intCol"), col("longCol")) assertConvertedFilterIsEmpty(testPredicate, testParquetTable) val actData = readUsingKernel(testParquetTable, testPredicate) // contains all the data in the table as the predicate is not pushed down checkAnswer(actData, generateExpData(Seq.range(0, 20))) } test("comparator that compare literal and literal") { val testPredicate = predicate("=", ofInt(20), ofInt(20)) assertConvertedFilterIsEmpty(testPredicate, testParquetTable) val actData = readUsingKernel(testParquetTable, testPredicate) // contains all the data in the table as the predicate is not pushed down checkAnswer(actData, generateExpData(Seq.range(0, 20))) } test("OR support") { val predicate = or( eq(col("intCol"), ofInt(20)), eq(col("longCol"), ofLong(1600))) val actData = readUsingKernel(testParquetTable, predicate) checkAnswer(actData, generateExpData(Seq(0, 15))) } test("one end of the OR is not convertible") { val predicate = or( eq(col("intCol"), ofInt(1599)), eq(col("nonExistentCol"), ofInt(1600))) assertConvertedFilterIsEmpty(predicate, testParquetTable) val actData = readUsingKernel(testParquetTable, predicate) // contains all the data in the table as the predicate is not pushed down checkAnswer(actData, generateExpData(Seq.range(0, 20))) } test("AND support") { val predicate = and( eq(col("intCol"), ofInt(1599)), eq(col("longCol"), ofLong(1600))) val actData = readUsingKernel(testParquetTable, predicate) checkAnswer(actData, generateExpData(Seq(15))) } test("one end of the AND is not convertible") { val predicate = and( eq(col("intCol"), ofInt(1599)), eq(col("nonExistentCol"), ofInt(1600))) val actData = readUsingKernel(testParquetTable, predicate) checkAnswer(actData, generateExpData(Seq(15))) } test("not support on gt") { val predicate = not(gt(col("intCol"), ofInt(950))) val actData = readUsingKernel(testParquetTable, predicate) // rowgroups until 9 could have values <= 950 // rowgroup 2 has all nulls, so it won't be included in the result val expRowGroups = Seq(0, 1, 3, 4, 5, 6, 7, 8, 9) val expOutputRowCount = expRowGroups.length * 100 // 100 rows per row group assert(actData.size === expOutputRowCount, s"predicate: $predicate") checkAnswer(actData, generateExpData(expRowGroups)) } test("not support on equality") { val predicate = not(eq(col("longCol"), ofLong(768))) val actData = readUsingKernel(testParquetTable, predicate) // rowgroup 3 has all nulls, so it will be included in the results as // Parquet equality filter is not null safe // every other group has value that is not 768 checkAnswer(actData, generateExpData(Seq.range(0, 20))) } test("doesn't work on the repeated columns") { val testTable = goldenTablePath("parquet-all-types") val readSchema = tableSchema(testTable) val predicate = eq(col("array_of_prims"), ofInt(20)) assertConvertedFilterIsEmpty(predicate, testTable) val actResult = readParquetFilesUsingKernel(testTable, readSchema, Optional.of(predicate)) val expResult = readParquetFilesUsingSpark(testTable, readSchema) checkAnswer(actResult, expResult) } /** Test the `not(predicate)` returns expected rowgroups */ private def assertNot(predicate: Predicate, expRowGroups: Seq[Int]): Unit = { val notPredicate = not(predicate) val actualData = readUsingKernel(testParquetTable, notPredicate) val expOutputRowCount = expRowGroups.length * 100 // 100 rows per row group assert(actualData.size === expOutputRowCount, s"predicate: $notPredicate") checkAnswer(actualData, generateExpData(expRowGroups)) } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/parquet/ParquetSchemaUtilsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.parquet import io.delta.kernel.defaults.internal.parquet.ParquetSchemaUtils.pruneSchema import io.delta.kernel.defaults.utils.TestUtils import io.delta.kernel.internal.util.ColumnMapping import io.delta.kernel.internal.util.ColumnMapping.PARQUET_FIELD_NESTED_IDS_METADATA_KEY import io.delta.kernel.types.{ArrayType, DoubleType, FieldMetadata, MapType, StructType} import io.delta.kernel.types.IntegerType.INTEGER import io.delta.kernel.types.LongType.LONG import org.apache.parquet.schema.MessageTypeParser import org.scalatest.funsuite.AnyFunSuite class ParquetSchemaUtilsSuite extends AnyFunSuite with TestUtils { // Test parquet schema type containing different types of columns with field ids private val testParquetFileSchema = MessageTypeParser.parseMessageType( """message fileSchema { | required group f0 = 1 { | optional int32 f00 = 2; | optional int64 f01 = 3; | } | optional group f1 = 4 { | repeated group list = 5 { | optional int32 element = 6; | } | } | required group f2 (MAP) = 7 { | repeated group key_value = 8 { | required group key = 9 { | required int32 key_f0 = 10; | required int64 key_f1 = 11; | } | required int32 value = 12; | } | } | optional double f3 = 13; |} """.stripMargin) // Delta schema corresponding to the above test [[parquetSchema]] private val testParquetFileDeltaSchema = new StructType() .add( "f0", new StructType() .add("f00", INTEGER, fieldMetadata(2)) .add("f01", LONG, fieldMetadata(3)), fieldMetadata(1)) .add("f1", new ArrayType(INTEGER, false), fieldMetadata(4)) .add( "f2", new MapType( new StructType() .add("key_f0", INTEGER, fieldMetadata(10)) .add("key_f1", INTEGER, fieldMetadata(11)), INTEGER, false), fieldMetadata(7)) .add("f3", DoubleType.DOUBLE, fieldMetadata(13)) test("id mapping mode - delta reads all columns in the parquet file") { val prunedParquetSchema = pruneSchema(testParquetFileSchema, testParquetFileDeltaSchema) assert(prunedParquetSchema === testParquetFileSchema) } test("id mapping mode - delta selects a subset of columns in the parquet file") { val readDeltaSchema = new StructType() .add(testParquetFileDeltaSchema.get("f1")) .add( // nested column pruning "f0", new StructType() .add("f00", INTEGER, fieldMetadata(2)), fieldMetadata(1)) val expectedParquetSchema = MessageTypeParser.parseMessageType( """message fileSchema { | optional group f1 = 4 { | repeated group list = 5 { | optional int32 element = 6; | } | } | required group f0 = 1 { | optional int32 f00 = 2; | } |} """.stripMargin) val prunedParquetSchema = pruneSchema(testParquetFileSchema, readDeltaSchema) assert(prunedParquetSchema === expectedParquetSchema) } test("id mapping mode - delta tries to read a column not present in the parquet file") { val readDeltaSchema = new StructType() .add(testParquetFileDeltaSchema.get("f1")) .add( // nested column has extra column that is not present in the file "f0", new StructType() .add("f00", INTEGER, fieldMetadata(2)) .add("f02", INTEGER, fieldMetadata(15)), fieldMetadata(1)) .add("f4", INTEGER, fieldMetadata(14)) // pruned parquet file schema shouldn't have the column "f4" val expectedParquetSchema = MessageTypeParser.parseMessageType( """message fileSchema { | optional group f1 = 4 { | repeated group list = 5 { | optional int32 element = 6; | } | } | required group f0 = 1 { | optional int32 f00 = 2; | } |} """.stripMargin) val prunedParquetSchema = pruneSchema(testParquetFileSchema, readDeltaSchema) assert(prunedParquetSchema === expectedParquetSchema) } test("id mapping mode - combination of columns with and w/o field ids in delta read schema") { val readDeltaSchema = new StructType() .add(testParquetFileDeltaSchema.get("f1")) // with field id .add( // nested column has extra column that is not present in the file "f0", new StructType() .add("F00", INTEGER) // no field id and with case-insensitive column name .add("f01", INTEGER, fieldMetadata(3)) // no field id for struct f0 ) val expectedParquetSchema = MessageTypeParser.parseMessageType( """message fileSchema { | optional group f1 = 4 { | repeated group list = 5 { | optional int32 element = 6; | } | } | required group f0 = 1 { | optional int32 f00 = 2; | optional int64 f01 = 3; | } |} """.stripMargin) val prunedParquetSchema = pruneSchema(testParquetFileSchema, readDeltaSchema) assert(prunedParquetSchema === expectedParquetSchema) } test("id mapping mode - field id matches but not the column name") { val readDeltaSchema = new StructType() // physical name in the file is f3, but the same field id .add("f3_new", DoubleType.DOUBLE, fieldMetadata(13)) .add( "f0", new StructType() // physical name in the file is f00, but the same field id .add("f00_new", INTEGER, fieldMetadata(2)), fieldMetadata(1)) val expectedParquetSchema = MessageTypeParser.parseMessageType( """message fileSchema { | optional double f3 = 13; | required group f0 = 1 { | optional int32 f00 = 2; | } |} """.stripMargin) val prunedParquetSchema = pruneSchema(testParquetFileSchema, readDeltaSchema) assert(prunedParquetSchema === expectedParquetSchema) } test("id mapping mode - duplicate id in file at the same level throws error") { val readDeltaSchema = new StructType() .add("f3", DoubleType.DOUBLE, fieldMetadata(13)) val testParquetFileSchema = MessageTypeParser.parseMessageType( """message fileSchema { | optional double f3 = 13; | optional double f4 = 13; |} """.stripMargin) val ex = intercept[Exception] { pruneSchema(testParquetFileSchema, readDeltaSchema) } assert(ex.getMessage.contains( "Parquet file contains multiple columns (optional double f3 = 13, " + "optional double f4 = 13) with the same field id")) } test("id mapping mode - duplicate id in file at the same nested level throws error") { val readDeltaSchema = new StructType() .add( "f0", new StructType() .add("f00", INTEGER, fieldMetadata(2)), fieldMetadata(1)) val testParquetFileSchema = MessageTypeParser.parseMessageType( """message fileSchema { | required group f0 = 1 { | optional int32 f00 = 2; | optional int64 f01 = 3; | optional int64 f02 = 2; | } |} """.stripMargin) val ex = intercept[Exception] { pruneSchema(testParquetFileSchema, readDeltaSchema) } assert(ex.getMessage.contains( "Parquet file contains multiple columns (optional int32 f00 = 2, " + "optional int64 f02 = 2) with the same field id")) } // icebergCompatV2 tests - nested field ids are converted correctly to parquet schema Seq( ( "struct with array and map", // Delta schema - input new StructType() .add( "f0", new StructType() .add("f00", new ArrayType(LONG, false), fieldMetadata(2, ("f00.element", 3))) .add( "f01", new MapType(INTEGER, INTEGER, true), fieldMetadata(4, ("f01.key", 5), ("f01.value", 6))), fieldMetadata(1)), // Expected parquet schema MessageTypeParser.parseMessageType( """message DefaultKernelSchema { | optional group f0 = 1 { | optional group f00 (LIST) = 2 { | repeated group list { | required int64 element = 3; | } | } | optional group f01 (MAP) = 4 { | repeated group key_value { | required int32 key = 5; | optional int32 value = 6; | } | } | } |}""".stripMargin)), ( "top-level array and map columns", // Delta schema - input new StructType() .add("f1", new ArrayType(INTEGER, true), fieldMetadata(1, ("f1.element", 2))) .add( "f2", new MapType( new StructType() .add("key_f0", INTEGER, fieldMetadata(6)) .add("key_f1", INTEGER, fieldMetadata(7)), INTEGER, true), fieldMetadata(3, ("f2.key", 4), ("f2.value", 5))), // Expected parquet schema MessageTypeParser.parseMessageType("""message DefaultKernelSchema { | optional group f1 (LIST) = 1 { | repeated group list { | optional int32 element = 2; | } | } | optional group f2 (MAP) = 3 { | repeated group key_value { | required group key = 4 { | optional int32 key_f0 = 6; | optional int32 key_f1 = 7; | } | optional int32 value = 5; | } | } |}""".stripMargin)), ( "array/map inside array/map", // Delta schema - input new StructType() .add( "f3", new ArrayType(new ArrayType(INTEGER, false), false), fieldMetadata(0, ("f3.element", 1), ("f3.element.element", 2))) .add( "f4", new MapType( new MapType( new StructType() .add("key_f0", INTEGER, fieldMetadata(3)) .add("key_f1", INTEGER, fieldMetadata(4)), INTEGER, false), INTEGER, false), fieldMetadata(5, ("f4.key", 6), ("f4.value", 7), ("f4.key.key", 8), ("f4.key.value", 9))), // Expected parquet schema MessageTypeParser.parseMessageType("""message DefaultKernelSchema { | optional group f3 (LIST) = 0 { | repeated group list { | required group element (LIST) = 1 { | repeated group list { | required int32 element = 2; | } | } | } | } | optional group f4 (MAP) = 5 { | repeated group key_value { | required group key (MAP) = 6 { | repeated group key_value { | required group key = 8 { | optional int32 key_f0 = 3; | optional int32 key_f1 = 4; | } | required int32 value = 9; | } | } | required int32 value = 7; | } | } |}""".stripMargin))).foreach { case (testName, deltaSchema, expectedParquetSchema) => test(s"icebergCompatV2 - nested fields are converted to parquet schema - $testName") { val actParquetSchema = ParquetSchemaUtils.toParquetSchema(deltaSchema) assert(actParquetSchema === expectedParquetSchema) } } Seq( ( "field id validation: no negative field id", // Delta schema - input new StructType() .add( "f0", new StructType() .add("f00", new ArrayType(LONG, false), fieldMetadata(-1)) .add("f01", new MapType(INTEGER, INTEGER, true), fieldMetadata(4)), fieldMetadata(1)), // Expected error message "Field id should be non-negative."), ( "field id validation: no negative nested field id", // Delta schema - input new StructType() .add( "f0", new StructType() .add("f00", new ArrayType(LONG, false), fieldMetadata(1, ("f00.element", -1))) .add("f01", new MapType(INTEGER, INTEGER, true), fieldMetadata(4)), fieldMetadata(0)), // Expected error message "Field id should be non-negative."), ( "field id validation: no duplicate field id", // Delta schema - input new StructType() .add( "f0", new StructType() .add("f00", new ArrayType(LONG, false), fieldMetadata(1, ("f00.element", 1))) .add("f01", new MapType(INTEGER, INTEGER, true), fieldMetadata(4)), fieldMetadata(1)), // Expected error message "Field id should be unique."), ( "field id validation: no duplicate nested field id", // Delta schema - input new StructType() .add( "f0", new StructType() .add("f00", new ArrayType(LONG, false), fieldMetadata(1, ("f00.element", 2))) .add("f01", new MapType(INTEGER, INTEGER, true), fieldMetadata(2)), fieldMetadata(1)), // Expected error message "Field id should be unique."), ( "field id validation: missing field ids", // Delta schema - input new StructType() .add( "f0", new StructType() .add("f00", new ArrayType(LONG, false)) .add("f01", new MapType(INTEGER, INTEGER, true), fieldMetadata(4)), fieldMetadata(1)), // Expected error message "Some of the fields are missing field ids."), ( "field id validation: missing nested field ids", // Delta schema - input new StructType() .add( "f0", new StructType() .add("f00", new ArrayType(LONG, false), fieldMetadata(1, ("f00.element", 2))) .add("f01", new MapType(INTEGER, INTEGER, true), fieldMetadata(4)), // missing nested id fieldMetadata(0)), // Expected error message "Some of the fields are missing field ids."), ( "field id validation: missing field ids but have nested fields", // Delta schema - input new StructType() .add( "f0", new StructType() .add("f00", new ArrayType(LONG, false), fieldMetadata(1, ("f00.element", 2))) .add("f01", new MapType(INTEGER, INTEGER, true), fieldMetadata(4, ("f01.key", 5))) ), // missing field id for f0 // Expected error message "Some of the fields are missing field ids.")).foreach { case (testName, deltaSchema, expectedErrorMsg) => test(testName) { val ex = intercept[IllegalArgumentException] { ParquetSchemaUtils.toParquetSchema(deltaSchema) } assert(ex.getMessage.contains(expectedErrorMsg)) } } private def fieldMetadata(id: Int, nestedFieldIds: (String, Int)*): FieldMetadata = { val builder = FieldMetadata.builder().putLong(ColumnMapping.PARQUET_FIELD_ID_KEY, id) val nestedFiledMetadata = FieldMetadata.builder() nestedFieldIds.foreach { case (nestedColPath, nestedId) => nestedFiledMetadata.putLong(nestedColPath, nestedId) } builder .putFieldMetadata(PARQUET_FIELD_NESTED_IDS_METADATA_KEY, nestedFiledMetadata.build()) .build() } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/parquet/ParquetSuiteBase.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.internal.parquet import java.nio.file.{Files, Paths} import java.util.Optional import scala.collection.JavaConverters._ import scala.util.control.NonFatal import io.delta.kernel.data.{ColumnarBatch, FilteredColumnarBatch} import io.delta.kernel.defaults.engine.hadoopio.HadoopFileIO import io.delta.kernel.defaults.utils.{TestRow, TestUtils} import io.delta.kernel.expressions.{Column, Predicate} import io.delta.kernel.internal.util.ColumnMapping import io.delta.kernel.internal.util.Utils.toCloseableIterator import io.delta.kernel.types.{ArrayType, DataType, MapType, StructField, StructType} import io.delta.kernel.utils.{DataFileStatus, FileStatus} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.parquet.hadoop.metadata.{ColumnPath, ParquetMetadata} trait ParquetSuiteBase extends TestUtils { implicit class DataFileStatusOps(dataFileStatus: DataFileStatus) { /** * Convert the [[DataFileStatus]] to a [[TestRow]]. * (path, size, modification time, numRecords, * min_col1, max_col1, nullCount_col1 (..repeated for every stats column) * ) */ def toTestRow(statsColumns: Seq[Column]): TestRow = { val statsOpt = dataFileStatus.getStatistics val record: Seq[Any] = { dataFileStatus.getPath +: dataFileStatus.getSize +: // convert to seconds, Spark returns in seconds and we can compare at second level (dataFileStatus.getModificationTime / 1000) +: // Add the row count to the stats literals (if (statsOpt.isPresent) statsOpt.get().getNumRecords else null) +: statsColumns.flatMap { column => if (statsOpt.isPresent) { val stats = statsOpt.get() Seq( Option(stats.getMinValues.get(column)).map(_.getValue).orNull, Option(stats.getMaxValues.get(column)).map(_.getValue).orNull, Option(stats.getNullCount.get(column)).orNull) } else { Seq(null, null, null) } } } TestRow(record: _*) } } /** * Verify the contents of the Parquet files located in `actualFileDir` matches the * `expected` data. Does two types of verifications. * 1) Verify the data using the Kernel Parquet reader * 2) Verify the data using the Spark Parquet reader */ def verifyContent(actualFileDir: String, expected: Seq[FilteredColumnarBatch]): Unit = { verifyFileMetadata(actualFileDir) verifyContentUsingKernelReader(actualFileDir, expected) verifyContentUsingSparkReader(actualFileDir, expected) } /** * Verify the metadata of the Parquet files in `targetDir` matches says it is written by Kernel. */ def verifyFileMetadata(targetDir: String): Unit = { parquetFiles(targetDir).foreach { file => footer(file.getPath).getFileMetaData .getKeyValueMetaData.containsKey("io.delta.kernel.default-parquet-writer") } } /** * Verify the data in the Parquet files located in `actualFileDir` matches the expected data. * Use Kernel Parquet reader to read the data from the Parquet files. */ def verifyContentUsingKernelReader( actualFileDir: String, expected: Seq[FilteredColumnarBatch]): Unit = { val dataSchema = expected.head.getData.getSchema val expectedTestRows = expected .map(fb => fb.getRows) .flatMap(_.toSeq) .map(TestRow(_)) val actualTestRows = readParquetFilesUsingKernel(actualFileDir, dataSchema) checkAnswer(actualTestRows, expectedTestRows) } /** * Verify the data in the Parquet files located in `actualFileDir` matches the expected data. * Use Spark Parquet reader to read the data from the Parquet files. */ def verifyContentUsingSparkReader( actualFileDir: String, expected: Seq[FilteredColumnarBatch]): Unit = { val dataSchema = expected.head.getData.getSchema; val expectedTestRows = expected .map(fb => fb.getRows) .flatMap(_.toSeq) .map(TestRow(_)) val actualTestRows = readParquetFilesUsingSpark(actualFileDir, dataSchema) checkAnswer(actualTestRows, expectedTestRows) } /** * Verify the field ids in Parquet files match the corresponding field ids in the Delta schema. * If [[expectListMapEntryIds]] is true, verifies the array and map elements also have field ids * the match the fields in nearest ancestor struct field (i.e array or map) */ def verifyFieldIds( targetDir: String, deltaSchema: StructType, expectNestedFiledIds: Boolean): Unit = { parquetFiles(targetDir).map(_.getPath).map(footer(_)).foreach { footer => val parquetSchema = footer.getFileMetaData.getSchema def verifyFieldId(deltaFieldId: Long, parquetColumnPath: Array[String]): Unit = { val parquetFieldId = parquetSchema.getType(parquetColumnPath: _*).getId assert(parquetFieldId != null) assert(deltaFieldId === parquetFieldId.intValue()) } def verifyNestedFieldId( nearestAncestorStructField: StructField, relativeNestedFieldPath: Array[String], // relative to the nearest ancestor struct field columnPath: Array[String]): Unit = { val deltaFieldId = nearestAncestorStructField.getMetadata .getMetadata(ColumnMapping.COLUMN_MAPPING_NESTED_IDS_KEY) .getLong(relativeNestedFieldPath.mkString(".")) .toInt val parquetFieldId = parquetSchema.getType(columnPath: _*).getId assert(parquetFieldId != null) assert(deltaFieldId === parquetFieldId.intValue()) } def visitDeltaType( basePathInParquet: Array[String], nearestAncestorStructField: StructField, baseRelativePathToAncestor: Array[String], deltaType: DataType): Unit = { deltaType match { case struct: StructType => visitStructType(basePathInParquet, struct) case array: ArrayType => // Arrays are stored as three-level structure in Parquet. There are two elements // between the array element and array itself. So in order to // search for the array element field id, we need to append "list, element" // to the path. // optional group col-b89fd303-7352-4044-842e-87f428ee80be (LIST) = 19 { // repeated group list { // optional group element { // optional int64 col-e983d1fc-d588-46a7-a0ad-2f63a6834ea6 = 20; // } // } // } val elemPathInParquet = basePathInParquet :+ "list" :+ "element" val relativePathToNearestAncestor = baseRelativePathToAncestor :+ "element" if (expectNestedFiledIds) { verifyNestedFieldId( nearestAncestorStructField, relativePathToNearestAncestor, elemPathInParquet) } visitDeltaType( elemPathInParquet, nearestAncestorStructField, relativePathToNearestAncestor, array.getElementType) case map: MapType => // reason for appending the "key_value" is same as the array type (see above) val keyPathInParquet = basePathInParquet :+ "key_value" :+ "key" val valuePathInParquet = basePathInParquet :+ "key_value" :+ "value" val keyRelativePathToNearestAncestor = baseRelativePathToAncestor :+ "key" val valueRelativePathToNearestAncestor = baseRelativePathToAncestor :+ "value" if (expectNestedFiledIds) { verifyNestedFieldId( nearestAncestorStructField, keyRelativePathToNearestAncestor, keyPathInParquet) verifyNestedFieldId( nearestAncestorStructField, valueRelativePathToNearestAncestor, valuePathInParquet) } visitDeltaType( keyPathInParquet, nearestAncestorStructField, keyRelativePathToNearestAncestor, map.getKeyType) visitDeltaType( valuePathInParquet, nearestAncestorStructField, valueRelativePathToNearestAncestor, map.getValueType) case _ => // Primitive type - continue } } def visitStructType(basePathInParquet: Array[String], structType: StructType): Unit = { structType.fields.forEach { field => val deltaFieldId = field.getMetadata .getLong(ColumnMapping.COLUMN_MAPPING_ID_KEY) val physicalName = field.getMetadata .getString(ColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY) verifyFieldId(deltaFieldId, basePathInParquet :+ physicalName) visitDeltaType( basePathInParquet :+ physicalName, nearestAncestorStructField = field, baseRelativePathToAncestor = Array(physicalName), field.getDataType) } } visitStructType(Array.empty, deltaSchema) } } /** * Write the [[FilteredColumnarBatch]]es to Parquet files using the ParquetFileWriter and * verify the data using the Kernel Parquet reader and Spark Parquet reader. */ def writeToParquetUsingKernel( filteredData: Seq[FilteredColumnarBatch], location: String, targetFileSize: Long = 1024 * 1024, statsColumns: Seq[Column] = Seq.empty): Seq[DataFileStatus] = { val conf = new Configuration(configuration); conf.setLong(ParquetFileWriter.TARGET_FILE_SIZE_CONF, targetFileSize) val fileIO = new HadoopFileIO(conf) val parquetWriter = ParquetFileWriter.multiFileWriter( fileIO, location, statsColumns.asJava) parquetWriter.write(toCloseableIterator(filteredData.asJava.iterator())).toSeq } def readParquetFilesUsingKernel( actualFileDir: String, readSchema: StructType, predicate: Optional[Predicate] = Optional.empty()): Seq[TestRow] = { val columnarBatches = readParquetUsingKernelAsColumnarBatches(actualFileDir, readSchema, predicate) columnarBatches.map(_.getRows).flatMap(_.toSeq).map(TestRow(_)) } def readParquetUsingKernelAsColumnarBatches( inputFileOrDir: String, readSchema: StructType, predicate: Optional[Predicate] = Optional.empty()): Seq[ColumnarBatch] = { val parquetFileList = parquetFiles(inputFileOrDir) val data = defaultEngine.getParquetHandler.readParquetFiles( toCloseableIterator(parquetFileList.asJava.iterator()), readSchema, predicate) data.asScala.toSeq.map(_.getData) } def parquetFileCount(fileOrDir: String): Long = parquetFiles(fileOrDir).size def parquetFileRowCount(fileOrDir: String): Long = { val files = parquetFiles(fileOrDir) var rowCount = 0L files.foreach { file => // read parquet file using spark and count. rowCount = rowCount + spark.read.parquet(file.getPath).count() } rowCount } def parquetFiles(fileOrDir: String): Seq[FileStatus] = { val fileOrDirPath = new Path(fileOrDir) val hadoopFs = new Path(fileOrDir).getFileSystem(configuration) hadoopFs.listStatus(fileOrDirPath) .iterator .filter(_.getPath.toString.endsWith(".parquet")) .map(status => FileStatus.of(status.getPath.toString, status.getLen, status.getModificationTime)) .toSeq } def footer(path: String): ParquetMetadata = { try { org.apache.parquet.hadoop.ParquetFileReader.readFooter(configuration, new Path(path)) } catch { case NonFatal(e) => fail(s"Failed to read footer for file: $path", e) } } // Read the parquet files in actionFileDir using Spark Parquet reader def readParquetFilesUsingSpark( actualFileDir: String, readSchema: StructType): Seq[TestRow] = { spark.read .format("parquet") .parquet(actualFileDir) .to(readSchema.toSpark) .collect() .map(TestRow(_)) } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/metrics/LoggingMetricsReporterSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.metrics import scala.collection.mutable.ArrayBuffer import io.delta.kernel.defaults.engine.LoggingMetricsReporter import io.delta.kernel.metrics.MetricsReport import io.delta.kernel.shaded.com.fasterxml.jackson.databind.exc.MismatchedInputException import io.delta.kernel.types.{FieldMetadata, StringType, StructType} import org.apache.logging.log4j.{Level, LogManager} import org.apache.logging.log4j.core.{LogEvent, Logger => Log4jLogger} import org.apache.logging.log4j.core.appender.AbstractAppender import org.apache.logging.log4j.core.config.Property import org.scalatest.funsuite.AnyFunSuite class LoggingMetricsReporterSuite extends AnyFunSuite { /** Captures logging events * */ private class BufferingAppender(name: String) extends AbstractAppender(name, null, null, true, Property.EMPTY_ARRAY) { val events: ArrayBuffer[LogEvent] = ArrayBuffer.empty[LogEvent] override def append(event: LogEvent): Unit = events += event.toImmutable } private def withCapturedReporterLogger[T](f: BufferingAppender => T): T = { val log = LogManager.getLogger("io.delta.kernel.defaults.engine.LoggingMetricsReporter") .asInstanceOf[Log4jLogger] val app = new BufferingAppender("test-appender") app.start() val oldLevel = log.getLevel try { log.addAppender(app) log.setLevel(Level.ALL) f(app) } finally { log.removeAppender(app) log.setLevel(oldLevel) app.stop() } } test("LoggingMetricsReporter successfully logs a metrics report") { val fmNull = FieldMetadata.builder().putString("kNull", null).build() val fmArray = FieldMetadata.builder().putStringArray("arr", Array[String]("x", null, "z")).build() val schema = new StructType() .add("c1", StringType.STRING, fmNull) .add("c2", StringType.STRING, fmArray) val schemaStr = schema.toString val report = new MetricsReport { override def toJson: String = { val s = schemaStr.replace("\\", "\\\\").replace("\"", "\\\"") s"""{"tableSchema":"$s"}""" } } withCapturedReporterLogger { app => new LoggingMetricsReporter().report(report) val msgs = app.events.map(e => (e.getLevel, e.getMessage.getFormattedMessage)) assert( msgs.exists { case (lvl, msg) => lvl == Level.INFO && msg.contains("tableSchema") }, s"Expected INFO log with 'tableSchema' but got: ${msgs.mkString("; ")}") assert( msgs.exists { case (lvl, msg) => lvl == Level.INFO && msg.contains("kNull=null") && msg.contains("arr=[x, null, z]") }, s"Expected schema with null values and arrays to be serialized correctly") } } test("LoggingMetricsReporter catches and logs kernel-api shaded JsonProcessingException") { val shadedThrow = new MetricsReport { override def toJson: String = { val ex = MismatchedInputException.from( null.asInstanceOf[io.delta.kernel.shaded.com.fasterxml.jackson.core.JsonParser], classOf[Object], "test exception") throw ex } } withCapturedReporterLogger { app => val reporter = new LoggingMetricsReporter() reporter.report(shadedThrow) val msgs = app.events.map(e => (e.getLevel, e.getMessage.getFormattedMessage)) assert( msgs.exists { case (lvl, msg) => lvl == Level.WARN && msg.contains("Serialization issue") }, s"Expected WARN with 'Serialization issue' but got: ${msgs.mkString("; ")}") } } test("LoggingMetricsReporter logs generic Exception at WARN level") { val genericThrow = new MetricsReport { override def toJson: String = throw new RuntimeException("generic boom") } withCapturedReporterLogger { app => val reporter = new LoggingMetricsReporter() reporter.report(genericThrow) val msgs = app.events.map(e => (e.getLevel, e.getMessage.getFormattedMessage)) assert( msgs.exists { case (lvl, msg) => lvl == Level.WARN && msg.contains("Unexpected error") }, s"Expected WARN with 'Unexpected error' but got: ${msgs.mkString("; ")}") } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/metrics/MetricsReportTestUtils.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.metrics import java.util import scala.collection.mutable.ArrayBuffer import io.delta.kernel.defaults.engine.DefaultEngine import io.delta.kernel.defaults.utils.TestUtils import io.delta.kernel.engine._ import io.delta.kernel.metrics.MetricsReport import org.apache.hadoop.conf.Configuration import org.slf4j.LoggerFactory /** * Test utilities for testing the Kernel-API created [[MetricsReports]]s. * * We test [[MetricsReport]]s in the defaults package so we can use real tables and avoid having * to mock both file listings AND file contents. */ trait MetricsReportTestUtils extends TestUtils { private val logger = LoggerFactory.getLogger(classOf[MetricsReportTestUtils]) override lazy val defaultEngine = DefaultEngine.create(new Configuration() { { // Set the batch sizes to small so that we get to test the multiple batch scenarios. set("delta.kernel.default.parquet.reader.batch-size", "2"); set("delta.kernel.default.json.reader.batch-size", "2"); } }) // For now this just uses the default engine since we have no need to override it, if we would // like to use a specific engine in the future for other tests we can simply add another arg here /** * Executes [[f]] using a special engine implementation to collect and return metrics reports. * If [[expectException]], catches any exception thrown by [[f]] and returns it with the reports. */ def collectMetricsReports( f: Engine => Unit, expectException: Boolean): (Seq[MetricsReport], Option[Exception]) = { // Initialize a buffer for any metric reports and wrap the engine so that they are recorded val reports = ArrayBuffer.empty[MetricsReport] if (expectException) { val e = intercept[Exception] { f(new EngineWithInMemoryMetricsReporter(reports, defaultEngine)) } logger.warn("Caught exception:", e) (reports.toSeq, Some(e)) } else { f(new EngineWithInMemoryMetricsReporter(reports, defaultEngine)) (reports.toSeq, Option.empty) } } /** * Wraps an {@link Engine} to implement the metrics reporter such that it appends any reports * to the provided in memory buffer. */ class EngineWithInMemoryMetricsReporter(buf: ArrayBuffer[MetricsReport], baseEngine: Engine) extends Engine { private val inMemoryMetricsReporter = new MetricsReporter { override def report(report: MetricsReport): Unit = buf.append(report) } private val metricsReporters = new util.ArrayList[MetricsReporter]() { { addAll(baseEngine.getMetricsReporters) add(inMemoryMetricsReporter) } } override def getExpressionHandler: ExpressionHandler = baseEngine.getExpressionHandler override def getJsonHandler: JsonHandler = baseEngine.getJsonHandler override def getFileSystemClient: FileSystemClient = baseEngine.getFileSystemClient override def getParquetHandler: ParquetHandler = baseEngine.getParquetHandler override def getMetricsReporters(): java.util.List[MetricsReporter] = { metricsReporters } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/metrics/ScanReportSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.metrics import java.util.Collections import io.delta.kernel._ import io.delta.kernel.data.FilteredColumnarBatch import io.delta.kernel.engine._ import io.delta.kernel.expressions.{Column, Literal, Predicate} import io.delta.kernel.internal.data.GenericRow import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.metrics.Timer import io.delta.kernel.internal.util.{FileNames, Utils} import io.delta.kernel.metrics.{ScanReport, SnapshotReport} import io.delta.kernel.types.{IntegerType, LongType, StructType} import io.delta.kernel.utils.CloseableIterator import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.StatisticsCollection import org.apache.spark.sql.functions.col import org.scalatest.funsuite.AnyFunSuite class ScanReportSuite extends AnyFunSuite with MetricsReportTestUtils { /** * Creates a [[Scan]] using `getScan` and then requests and consumes the scan files. Uses a custom * engine to collect emitted metrics reports (exactly 1 [[ScanReport]] and 1 [[SnapshotReport]] is * expected). Also times and returns the duration it takes to consume the scan files. * * @param getScan function to generate a [[Scan]] given an engine * @param expectException whether we expect consuming the scan files to throw an exception, which * if so, is caught and returned with the other results * @return (ScanReport, durationToConsumeScanFiles, SnapshotReport, ExceptionIfThrown) */ def getScanAndSnapshotReport( getScan: Engine => Scan, expectException: Boolean, consumeScanFiles: CloseableIterator[FilteredColumnarBatch] => Unit) : (ScanReport, Long, SnapshotReport, Option[Exception]) = { val timer = new Timer() val (metricsReports, exception) = collectMetricsReports( engine => { val scan = getScan(engine) // Time the actual operation timer.timeCallable(() => consumeScanFiles(scan.getScanFiles(engine))) }, expectException) val scanReports = metricsReports.filter(_.isInstanceOf[ScanReport]) assert(scanReports.length == 1, "Expected exactly 1 ScanReport") val snapshotReports = metricsReports.filter(_.isInstanceOf[SnapshotReport]) assert(snapshotReports.length == 1, "Expected exactly 1 SnapshotReport") ( scanReports.head.asInstanceOf[ScanReport], timer.totalDurationNs(), snapshotReports.head.asInstanceOf[SnapshotReport], exception) } /** * Given a table path, constructs the latest snapshot, and uses it to generate a Scan with the * provided filter and readSchema (if provided). Consumes the scan files from the scan and * collects the emitted [[ScanReport]] and checks that the report is as expected. * * @param path table path to query * @param expectException whether we expect consuming the scan files to throw an exception * @param expectedNumAddFiles expected number of add files seen * @param expectedNumAddFilesFromDeltaFiles expected number of add files seen from delta files * @param expectedNumActiveAddFiles expected number of active add files * @param expectedNumDuplicateAddFiles expected number of duplicate add files seen * @param expectedNumRemoveFilesSeenFromDeltaFiles expected number of remove files seen * @param expectedPartitionPredicate expected partition predicate * @param expectedDataSkippingFilter expected data skipping filter * @param filter filter to build the scan with * @param readSchema read schema to build the scan with * @param consumeScanFiles function to consume scan file iterator */ // scalastyle:off def checkScanReport( path: String, expectException: Boolean, expectedNumAddFiles: Long, expectedNumAddFilesFromDeltaFiles: Long, expectedNumActiveAddFiles: Long, expectedNumDuplicateAddFiles: Long = 0, expectedNumRemoveFilesSeenFromDeltaFiles: Long = 0, expectedPartitionPredicate: Option[Predicate] = None, expectedDataSkippingFilter: Option[Predicate] = None, expectedIsFullyConsumed: Boolean = true, filter: Option[Predicate] = None, readSchema: Option[StructType] = None, // toSeq triggers log replay, consumes the actions and closes the iterator consumeScanFiles: CloseableIterator[FilteredColumnarBatch] => Unit = iter => iter.toSeq): Unit = { // scalastyle:on // We need to save the snapshotSchema to check against the generated scan report // In order to use the utils to collect the reports, we need to generate the snapshot in a anon // fx, thus we save the snapshotSchema as a side-effect var snapshotSchema: StructType = null val (scanReport, durationNs, snapshotReport, exceptionOpt) = getScanAndSnapshotReport( engine => { val snapshot = Table.forPath(engine, path).getLatestSnapshot(engine) snapshotSchema = snapshot.getSchema() var scanBuilder = snapshot.getScanBuilder() if (filter.nonEmpty) { scanBuilder = scanBuilder.withFilter(filter.get) } if (readSchema.nonEmpty) { scanBuilder = scanBuilder.withReadSchema(readSchema.get) } scanBuilder.build() }, expectException, consumeScanFiles) // Verify contents assert(scanReport.getTablePath == defaultEngine.getFileSystemClient.resolvePath(path)) assert(scanReport.getOperationType == "Scan") exceptionOpt match { case Some(e) => assert(scanReport.getException().isPresent) assert(scanReport.getException().get().getClass == e.getClass) assert(scanReport.getException().get().getMessage == e.getMessage) case None => assert(!scanReport.getException().isPresent) } assert(scanReport.getReportUUID != null) assert( snapshotReport.getVersion.isPresent, "Version should be present for success SnapshotReport") assert(scanReport.getTableVersion() == snapshotReport.getVersion.get()) assert(scanReport.getTableSchema() == snapshotSchema) assert(scanReport.getSnapshotReportUUID == snapshotReport.getReportUUID) assert(scanReport.getFilter.toScala == filter) assert(scanReport.getReadSchema == readSchema.getOrElse(snapshotSchema)) assert(scanReport.getPartitionPredicate.toScala == expectedPartitionPredicate) assert(scanReport.getIsFullyConsumed == expectedIsFullyConsumed) (scanReport.getDataSkippingFilter.toScala, expectedDataSkippingFilter) match { case (Some(found), Some(expected)) => assert(found.getName == expected.getName && found.getChildren == expected.getChildren) case (found, expected) => assert(found == expected) } // Since we cannot know the actual duration of the scan we sanity check that they are > 0 and // less than the total operation duration assert(scanReport.getScanMetrics.getTotalPlanningDurationNs > 0) assert(scanReport.getScanMetrics.getTotalPlanningDurationNs < durationNs) assert(scanReport.getScanMetrics.getNumAddFilesSeen == expectedNumAddFiles) assert(scanReport.getScanMetrics.getNumAddFilesSeenFromDeltaFiles == expectedNumAddFilesFromDeltaFiles) assert(scanReport.getScanMetrics.getNumActiveAddFiles == expectedNumActiveAddFiles) assert(scanReport.getScanMetrics.getNumDuplicateAddFiles == expectedNumDuplicateAddFiles) assert(scanReport.getScanMetrics.getNumRemoveFilesSeenFromDeltaFiles == expectedNumRemoveFilesSeenFromDeltaFiles) } test("ScanReport: basic case with no extra parameters") { withTempDir { tempDir => val path = tempDir.getCanonicalPath // Set up delta table with 1 add file spark.range(10).write.format("delta").mode("append").save(path) checkScanReport( path, expectException = false, expectedNumAddFiles = 1, expectedNumAddFilesFromDeltaFiles = 1, expectedNumActiveAddFiles = 1) } } test("ScanReport: basic case with read schema") { withTempDir { tempDir => val path = tempDir.getCanonicalPath // Set up delta table with 1 add file spark.range(10).withColumn("c2", col("id") % 2) .write.format("delta").mode("append").save(path) checkScanReport( path, expectException = false, expectedNumAddFiles = 1, expectedNumAddFilesFromDeltaFiles = 1, expectedNumActiveAddFiles = 1, readSchema = Some(new StructType().add("id", LongType.LONG))) } } test("ScanReport: different filter scenarios") { withTempDir { tempDir => val path = tempDir.getCanonicalPath // Set up partitioned table spark.range(10).withColumn("part", col("id") % 2) .write.format("delta").partitionBy("part").save(path) val partFilter = new Predicate("=", new Column("part"), Literal.ofLong(0)) val dataFilter = new Predicate("<=", new Column("id"), Literal.ofLong(0)) val expectedSkippingFilter = new Predicate( "<=", new Column(Array("minValues", "id")), Literal.ofLong(0)) // The below metrics are incremented during log replay before any filtering happens and thus // should be the same for all of the following test cases val expectedNumAddFiles = 2 val expectedNumAddFilesFromDeltaFiles = 2 val expectedNumActiveAddFiles = 2 // No filter - 2 add files one for each partition checkScanReport( path, expectException = false, expectedNumAddFiles = expectedNumAddFiles, expectedNumAddFilesFromDeltaFiles = expectedNumAddFilesFromDeltaFiles, expectedNumActiveAddFiles = expectedNumActiveAddFiles) // With partition filter checkScanReport( path, expectException = false, expectedNumAddFiles = expectedNumAddFiles, expectedNumAddFilesFromDeltaFiles = expectedNumAddFilesFromDeltaFiles, expectedNumActiveAddFiles = expectedNumActiveAddFiles, filter = Some(partFilter), expectedPartitionPredicate = Some(partFilter)) // With data filter checkScanReport( path, expectException = false, expectedNumAddFiles = expectedNumAddFiles, expectedNumAddFilesFromDeltaFiles = expectedNumAddFilesFromDeltaFiles, expectedNumActiveAddFiles = expectedNumActiveAddFiles, filter = Some(dataFilter), expectedDataSkippingFilter = Some(expectedSkippingFilter)) // With data and partition filter checkScanReport( path, expectException = false, expectedNumAddFiles = expectedNumAddFiles, expectedNumAddFilesFromDeltaFiles = expectedNumAddFilesFromDeltaFiles, expectedNumActiveAddFiles = expectedNumActiveAddFiles, filter = Some(new Predicate("AND", partFilter, dataFilter)), expectedDataSkippingFilter = Some(expectedSkippingFilter), expectedPartitionPredicate = Some(partFilter)) } } test("ScanReport: close scan file iterator early") { withTempDir { tempDir => val path = tempDir.getCanonicalPath // Set up delta table with 2 add files spark.range(10).write.format("delta").mode("append").save(path) spark.range(10).write.format("delta").mode("append").save(path) checkScanReport( path, expectException = false, expectedNumAddFiles = 1, expectedNumAddFilesFromDeltaFiles = 1, expectedNumActiveAddFiles = 1, expectedIsFullyConsumed = false, consumeScanFiles = iter => iter.close() // Close iterator before consuming any scan files ) } } ////////////////// // Error cases /// ////////////////// test("ScanReport error case - unrecognized partition filter") { // Thrown during partition pruning when the expression handler cannot evaluate the filter // Because partition pruning happens within a `map` on the iterator, this is caught and reported // within `map` withTempDir { tempDir => val path = tempDir.getCanonicalPath // Set up partitioned table spark.range(10).withColumn("part", col("id") % 2) .write.format("delta").partitionBy("part").save(path) val partFilter = new Predicate("foo", new Column("part"), Literal.ofLong(0)) checkScanReport( path, expectException = true, expectedNumAddFiles = 0, expectedNumAddFilesFromDeltaFiles = 0, expectedNumActiveAddFiles = 0, expectedIsFullyConsumed = false, filter = Some(partFilter), expectedPartitionPredicate = Some(partFilter)) } } test("ScanReport error case - error reading the log files") { withTempDir { tempDir => val path = tempDir.getCanonicalPath // We set up a table with a giberish json file at version 0 and a valid json file at version 1 // that contains the P&M // This is so the snapshot loading will happen successful, and we will only fail when trying // to load up the scan files // This exception is thrown from within the `hasNext` method on the iterator since that is // when we load the actions from the log files. This exception is caught and reported within // `hasNext` spark.range(10).write.format("delta").save(path) // Update protocol and metadata (so that version 1 has both P&M present) spark.sql( s"ALTER TABLE delta.`$path` SET TBLPROPERTIES ('delta.columnMapping.mode' = 'name')") // Overwrite json file with giberish (this will have a schema mismatch issue for `add`) val giberishRow = new GenericRow( new StructType().add("add", IntegerType.INTEGER), Collections.singletonMap(0, Integer.valueOf(0))) defaultEngine.getJsonHandler.writeJsonFileAtomically( FileNames.deltaFile(new Path(tempDir.toString, "_delta_log"), 0), Utils.singletonCloseableIterator(giberishRow), true) checkScanReport( path, expectException = true, expectedNumAddFiles = 0, expectedNumAddFilesFromDeltaFiles = 0, expectedNumActiveAddFiles = 0, expectedIsFullyConsumed = false) } } /////////////////////////////// // Log replay metrics tests /// /////////////////////////////// test("active add files log replay metrics: only delta files") { withTempDir { tempDir => val path = tempDir.getCanonicalPath for (_ <- 0 to 9) { appendCommit(path) } checkScanReport( path, expectException = false, expectedNumAddFiles = 20, // each commit creates 2 files expectedNumAddFilesFromDeltaFiles = 20, expectedNumActiveAddFiles = 20) } } Seq(true, false).foreach { multipartCheckpoint => val checkpointStr = if (multipartCheckpoint) "multipart " else "" test(s"active add files log replay metrics: ${checkpointStr}checkpoint + delta files") { withTempDir { tempDir => val path = tempDir.getCanonicalPath for (_ <- 0 to 3) { appendCommit(path) } checkpoint(path, actionsPerFile = if (multipartCheckpoint) 2 else 1000000) for (_ <- 4 to 9) { appendCommit(path) } checkScanReport( path, expectException = false, expectedNumAddFiles = 20, // each commit creates 2 files expectedNumAddFilesFromDeltaFiles = 12, // checkpoint is created at version 3 expectedNumActiveAddFiles = 20) } } } Seq(true, false).foreach { multipartCheckpoint => val checkpointStr = if (multipartCheckpoint) "multipart " else "" test(s"active add files log replay metrics: ${checkpointStr}checkpoint + " + s"delta files + tombstones") { withTempDir { tempDir => val path = tempDir.getCanonicalPath for (_ <- 0 to 3) { appendCommit(path) } // has 8 add files deleteCommit(path) // version 4 - deletes 4 files and adds 1 file checkpoint(path, actionsPerFile = if (multipartCheckpoint) 2 else 1000000) // version 4 appendCommit(path) // version 5 - adds 2 files deleteCommit(path) // version 6 - deletes 1 file and adds 1 file appendCommit(path) // version 7 - adds 2 files appendCommit(path) // version 8 - adds 2 files deleteCommit(path) // version 9 - deletes 2 files and adds 1 file checkScanReport( path, expectException = false, expectedNumAddFiles = 5 /* checkpoint */ + 8, /* delta */ expectedNumAddFilesFromDeltaFiles = 8, expectedNumActiveAddFiles = 10, expectedNumRemoveFilesSeenFromDeltaFiles = 3) } } } Seq(true, false).foreach { multipartCheckpoint => val checkpointStr = if (multipartCheckpoint) "multipart " else "" test(s"active add files log replay metrics: ${checkpointStr}checkpoint + delta files +" + s" tombstones + duplicate adds") { withTempDir { tempDir => val path = tempDir.getCanonicalPath for (_ <- 0 to 1) { appendCommit(path) } // activeAdds = 4 deleteCommit(path) // ver 2 - deletes 2 files and adds 1 file, activeAdds = 3 checkpoint(path, actionsPerFile = if (multipartCheckpoint) 2 else 1000000) // version 2 appendCommit(path) // ver 3 - adds 2 files, activeAdds = 5 recomputeStats(path) // ver 4 - adds the same 5 add files again, activeAdds = 5, dupes = 5 deleteCommit(path) // ver 5 - removes 1 file and adds 1 file, activeAdds = 5, dupes = 5 appendCommit(path) // ver 6 - adds 2 files, activeAdds = 7, dupes = 4 recomputeStats(path) // ver 7 - adds the same 7 add files again, activeAdds = 7, dupes = 12 deleteCommit(path) // ver 8 - removes 1 file and adds 1 files, activeAdds = 7, dupes = 12 checkScanReport( path, expectException = false, expectedNumAddFiles = 3 /* checkpoint */ + 18, /* delta */ expectedNumAddFilesFromDeltaFiles = 18, expectedNumActiveAddFiles = 7, expectedNumDuplicateAddFiles = 12, expectedNumRemoveFilesSeenFromDeltaFiles = 2) } } } ///////////////////////////////////////////// // Helpers for testing log replay metrics /// ///////////////////////////////////////////// def appendCommit(path: String): Unit = spark.range(10).repartition(2).write.format("delta").mode("append").save(path) def deleteCommit(path: String): Unit = { spark.sql("DELETE FROM delta.`%s` WHERE id = 5".format(path)) } def recomputeStats(path: String): Unit = { val deltaLog = DeltaLog.forTable(spark, new org.apache.hadoop.fs.Path(path)) StatisticsCollection.recompute(spark, deltaLog, catalogTable = None) } def checkpoint(path: String, actionsPerFile: Int): Unit = { withSQLConf(DeltaSQLConf.DELTA_CHECKPOINT_PART_SIZE.key -> actionsPerFile.toString) { DeltaLog.forTable(spark, path).checkpoint() } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/metrics/SnapshotReportSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.metrics import java.io.File import java.util.{Objects, Optional} import io.delta.golden.GoldenTableUtils.goldenTablePath import io.delta.kernel._ import io.delta.kernel.defaults.test.{AbstractTableManagerAdapter, LegacyTableManagerAdapter} import io.delta.kernel.defaults.utils.WriteUtils import io.delta.kernel.engine._ import io.delta.kernel.expressions.Literal import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.metrics.Timer import io.delta.kernel.internal.util.FileNames import io.delta.kernel.metrics.SnapshotReport import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite /** * Concrete implementation for legacy [[Table.forPath]] API. This is being replaced by the * [[TableManager.loadSnapshot]] API. */ class LegacySnapshotReportSuite extends AbstractSnapshotReportSuite { import io.delta.kernel.defaults.test.LegacyTableManagerAdapter override def tableManager: AbstractTableManagerAdapter = new LegacyTableManagerAdapter() // This test is only applicable to the legacy API because the // SnapshotBuilder::atTimestamp(latestSnapshot, timestamp) API takes in the latest snapshot // directly (as opposed to generating it internally). This lets the builder validate eagerly if // the provided timestamp is after the latest snapshot, which happens before any metrics are // recorded. test("Snapshot report - invalid timestamp (timestamp too late)") { withTempDir { tempDir => val path = tempDir.getCanonicalPath appendData(tablePath = path, isNewTable = true, schema = testSchema, data = Nil) // Test getSnapshotAsOfTimestamp with timestamp=currentTime (does not exist) // This fails during timestamp -> version resolution val currentTimeMillis = System.currentTimeMillis checkSnapshotReport( (engine, path) => tableManager.getSnapshotAtTimestamp(engine, path, currentTimeMillis), path, SnapshotReportExpectations( expectedReportCount = 2, // latestSnapshot + timeTravelToTimestampSnapshot expectException = true, expectedVersion = Optional.empty(), expectedCheckpointVersion = Optional.empty(), expectedProvidedTimestamp = Optional.of(currentTimeMillis), expectNonEmptyTimestampToVersionResolutionDuration = true, expectNonZeroLoadProtocolAndMetadataDuration = false, expectNonZeroBuildLogSegmentDuration = false, expectNonZeroDurationToGetCrcInfo = false)) } } } /** Concrete implementation for [[TableManager.loadSnapshot]] API. */ class TableManagerSnapshotReportSuite extends AbstractSnapshotReportSuite { import io.delta.kernel.defaults.test.TableManagerAdapter override def tableManager: AbstractTableManagerAdapter = new TableManagerAdapter() } abstract class AbstractSnapshotReportSuite extends AnyFunSuite with MetricsReportTestUtils with WriteUtils with BeforeAndAfterAll { override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key, true) } def tableManager: AbstractTableManagerAdapter case class SnapshotReportExpectations( expectedReportCount: Int, expectException: Boolean, expectedVersion: Optional[Long], expectedCheckpointVersion: Optional[Long], expectedProvidedTimestamp: Optional[Long], expectNonEmptyTimestampToVersionResolutionDuration: Boolean = false, expectNonZeroLoadProtocolAndMetadataDuration: Boolean = true, expectNonZeroBuildLogSegmentDuration: Boolean = true, expectNonZeroDurationToGetCrcInfo: Boolean = true) /** * Given a function [[f]] that generates a snapshot from an engine and path, runs [[f]] and looks * for a generated [[SnapshotReport]]. Times and returns the duration it takes to run [[f]]. * Uses a custom engine to collect emitted metrics reports. If more than one report is * generated (e.g. during timestamp-based time travel), only the last one is returned. * * @param f function to generate a snapshot from an engine and path * @param path path of the table to query * @param expectedReportCount the expected number of [[SnapshotReport]]s to be generated. This * can be greater than 1 for timestamp-based time travel queries. * @param expectException whether we expect [[f]] to throw an exception, which if so, is caught * and returned with the other results * @returns (SnapshotReport, durationToRunF, ExceptionIfThrown) */ def getSnapshotReport( f: (Engine, String) => Snapshot, path: String, expectedReportCount: Int, expectException: Boolean): (SnapshotReport, Long, Option[Exception]) = { val timer = new Timer() val (metricsReports, exception) = collectMetricsReports( engine => { timer.time(() => f(engine, path)) // Time the actual operation }, expectException) val snapshotReports = metricsReports.filter(_.isInstanceOf[SnapshotReport]) assert( snapshotReports.length == expectedReportCount, s"Expected exactly $expectedReportCount SnapshotReport") (snapshotReports.last.asInstanceOf[SnapshotReport], timer.totalDurationNs(), exception) } /** * Given a table path and a function [[f]] to generate a snapshot, runs [[f]] and collects the * generated [[SnapshotReport]]. Checks that the report is as expected. * * @param f function to generate a snapshot from an engine and path * @param path table path to query from * @param expectations encapsulates all the expected values and behaviors for the snapshot report. * See [[SnapshotReportExpectations]] for detailed parameter descriptions. */ def checkSnapshotReport( f: (Engine, String) => Snapshot, path: String, expectations: SnapshotReportExpectations): Unit = { val (snapshotReport, duration, exception) = getSnapshotReport(f, path, expectations.expectedReportCount, expectations.expectException) // Verify contents assert(snapshotReport.getTablePath == defaultEngine.getFileSystemClient.resolvePath(path)) assert(snapshotReport.getOperationType == "Snapshot") exception match { case Some(e) => assert(snapshotReport.getException().isPresent && Objects.equals(snapshotReport.getException().get(), e)) case None => assert(!snapshotReport.getException().isPresent) } assert(snapshotReport.getReportUUID != null) assert( Objects.equals(snapshotReport.getVersion, expectations.expectedVersion), s"Expected version ${expectations.expectedVersion} found ${snapshotReport.getVersion}") assert( Objects.equals( snapshotReport.getCheckpointVersion, expectations.expectedCheckpointVersion), s"Expected checkpoint version ${expectations.expectedCheckpointVersion}, found " + s"${snapshotReport.getCheckpointVersion}") assert(Objects.equals( snapshotReport.getProvidedTimestamp, expectations.expectedProvidedTimestamp)) // Since we cannot know the actual durations of these we sanity check that they are > 0 and // less than the total operation duration whenever they are expected to be non-zero/non-empty val metrics = snapshotReport.getSnapshotMetrics // ===== Metric: getLoadSnapshotTotalDurationNs ===== if (!expectations.expectException) { assert(metrics.getLoadSnapshotTotalDurationNs > 0) assert(metrics.getLoadSnapshotTotalDurationNs <= duration) } else { assert(metrics.getLoadSnapshotTotalDurationNs >= 0) } // ===== Metric: getComputeTimestampToVersionTotalDurationNs ===== if (expectations.expectNonEmptyTimestampToVersionResolutionDuration) { assert(metrics.getComputeTimestampToVersionTotalDurationNs.isPresent) assert(metrics.getComputeTimestampToVersionTotalDurationNs.get > 0) assert(metrics.getComputeTimestampToVersionTotalDurationNs.get < duration) assert(metrics.getComputeTimestampToVersionTotalDurationNs.get <= metrics.getLoadSnapshotTotalDurationNs) } else { assert(!metrics.getComputeTimestampToVersionTotalDurationNs.isPresent) } // ===== Metric: getLoadProtocolMetadataTotalDurationNs ===== if (expectations.expectNonZeroLoadProtocolAndMetadataDuration) { assert(metrics.getLoadProtocolMetadataTotalDurationNs > 0) assert(metrics.getLoadProtocolMetadataTotalDurationNs < duration) assert( metrics.getLoadProtocolMetadataTotalDurationNs <= metrics.getLoadSnapshotTotalDurationNs) } else { assert(metrics.getLoadProtocolMetadataTotalDurationNs == 0) } // ===== Metric: getLoadLogSegmentTotalDurationNs ===== if (expectations.expectNonZeroBuildLogSegmentDuration) { assert(metrics.getLoadLogSegmentTotalDurationNs > 0) assert(metrics.getLoadLogSegmentTotalDurationNs < duration) assert(metrics.getLoadLogSegmentTotalDurationNs <= metrics.getLoadSnapshotTotalDurationNs) } else { assert(metrics.getLoadLogSegmentTotalDurationNs == 0) } // ===== Metric: getLoadCrcTotalDurationNs ===== if (expectations.expectNonZeroDurationToGetCrcInfo) { assert(metrics.getLoadCrcTotalDurationNs > 0) assert(metrics.getLoadCrcTotalDurationNs < duration) assert(metrics.getLoadCrcTotalDurationNs <= metrics.getLoadSnapshotTotalDurationNs) } else { assert(metrics.getLoadCrcTotalDurationNs == 0) } } /** * Wait for the CRC file to exist for the given version. This helps ensure that Delta-Spark's * [[ChecksumHook]] has finished writing the checksum file before running tests. */ private def waitForCrcFileToExistElseThrow(tablePath: String, version: Long): Unit = { val logPath = new Path(tablePath, "_delta_log") val maxWaitMs = 1000 // Wait up to 1 second val startTime = System.currentTimeMillis() val crcFile = new java.io.File(FileNames.checksumFile(logPath, version).toString) while (!crcFile.exists() && (System.currentTimeMillis() - startTime) < maxWaitMs) { Thread.sleep(100) } def getDeltaLogContents: String = { new java.io.File(tablePath, "_delta_log") .listFiles().map(_.getName).sorted.mkString("\n- ", "\n- ", "") } assert(crcFile.exists(), s"CRC file $crcFile does not exist. Delta Log:$getDeltaLogContents") } test("SnapshotReport valid queries - no checkpoint") { withTempDir { tempDir => val path = tempDir.getCanonicalPath // Set up delta table with version 0, 1 spark.range(10).write.format("delta").mode("append").save(path) val version0timestamp = System.currentTimeMillis // Since filesystem modification time might be truncated to the second, we sleep to make // sure the next commit is after this timestamp Thread.sleep(1000) spark.range(10).write.format("delta").mode("append").save(path) waitForCrcFileToExistElseThrow(path, 0L) waitForCrcFileToExistElseThrow(path, 1L) // Test getLatestSnapshot checkSnapshotReport( (engine, path) => tableManager.getSnapshotAtLatest(engine, path), path, SnapshotReportExpectations( expectedReportCount = 1, expectException = false, expectedVersion = Optional.of(1), expectedCheckpointVersion = Optional.empty(), expectedProvidedTimestamp = Optional.empty() // No time travel by timestamp )) // Test getSnapshotAsOfVersion checkSnapshotReport( (engine, path) => tableManager.getSnapshotAtVersion(engine, path, 0), path, SnapshotReportExpectations( expectedReportCount = 1, expectException = false, expectedVersion = Optional.of(0), expectedCheckpointVersion = Optional.empty(), expectedProvidedTimestamp = Optional.empty() // No time travel by timestamp )) // Test getSnapshotAsOfTimestamp checkSnapshotReport( (engine, path) => tableManager.getSnapshotAtTimestamp(engine, path, version0timestamp), path, SnapshotReportExpectations( expectedReportCount = 2, // latestSnapshot + timeTravelToTimestampSnapshot expectException = false, expectedVersion = Optional.of(0), expectedCheckpointVersion = Optional.empty(), expectedProvidedTimestamp = Optional.of(version0timestamp), expectNonEmptyTimestampToVersionResolutionDuration = true)) } } test("SnapshotReport valid queries - with checkpoint") { withTempDir { tempDir => val path = tempDir.getCanonicalPath // Set up delta table with version 0 to 11 with checkpoint at version 10 (0 until 11).foreach(_ => spark.range(10).write.format("delta").mode("append").save(path)) val version11timestamp = System.currentTimeMillis // Since filesystem modification time might be truncated to the second, we sleep to make // sure the next commit is after this timestamp Thread.sleep(1000) // create version 11 spark.range(10).write.format("delta").mode("append").save(path) waitForCrcFileToExistElseThrow(path, 10L) waitForCrcFileToExistElseThrow(path, 11L) // Test getLatestSnapshot checkSnapshotReport( (engine, path) => tableManager.getSnapshotAtLatest(engine, path), path, SnapshotReportExpectations( expectedReportCount = 1, expectException = false, expectedVersion = Optional.of(11), expectedCheckpointVersion = Optional.of(10), expectedProvidedTimestamp = Optional.empty() // No time travel by timestamp )) // Test getSnapshotAsOfVersion checkSnapshotReport( (engine, path) => tableManager.getSnapshotAtVersion(engine, path, 11), path, SnapshotReportExpectations( expectedReportCount = 1, expectException = false, expectedVersion = Optional.of(11), expectedCheckpointVersion = Optional.of(10), expectedProvidedTimestamp = Optional.empty() // No time travel by timestamp )) // Test getSnapshotAsOfTimestamp checkSnapshotReport( (engine, path) => tableManager.getSnapshotAtTimestamp(engine, path, version11timestamp), path, SnapshotReportExpectations( expectedReportCount = 2, // latestSnapshot + timeTravelToTimestampSnapshot expectException = false, expectedVersion = Optional.of(10), expectedCheckpointVersion = Optional.of(10), expectedProvidedTimestamp = Optional.of(version11timestamp), expectNonEmptyTimestampToVersionResolutionDuration = true)) } } test("Snapshot report - invalid version (version does not exist)") { withTempDir { tempDir => val path = tempDir.getCanonicalPath appendData(tablePath = path, isNewTable = true, schema = testSchema, data = Nil) // Test getSnapshotAsOfVersion with version 1 (does not exist) // This fails during log segment building checkSnapshotReport( (engine, path) => tableManager.getSnapshotAtVersion(engine, path, 1), path, SnapshotReportExpectations( expectedReportCount = 1, expectException = true, expectedVersion = Optional.of(1), expectedCheckpointVersion = Optional.empty(), expectedProvidedTimestamp = Optional.empty(), // No time travel by timestamp expectNonZeroLoadProtocolAndMetadataDuration = false, expectNonZeroDurationToGetCrcInfo = false)) } } test("Snapshot report - invalid timestamp (timestamp too early)") { withTempDir { tempDir => val path = tempDir.getCanonicalPath appendData(tablePath = path, isNewTable = true, schema = testSchema, data = Nil) // Test getSnapshotAsOfTimestamp with timestamp=0 (does not exist) // This fails during timestamp -> version resolution checkSnapshotReport( (engine, path) => tableManager.getSnapshotAtTimestamp(engine, path, 0), path, SnapshotReportExpectations( expectedReportCount = 2, // latestSnapshot + timeTravelToTimestampSnapshot expectException = true, expectedVersion = Optional.empty(), expectedCheckpointVersion = Optional.empty(), expectedProvidedTimestamp = Optional.of(0), expectNonEmptyTimestampToVersionResolutionDuration = true, expectNonZeroLoadProtocolAndMetadataDuration = false, expectNonZeroBuildLogSegmentDuration = false, expectNonZeroDurationToGetCrcInfo = false)) } } test("Snapshot report - table does not exist") { withTempDir { tempDir => // This fails during either log segment building or timestamp -> version resolution val path = tempDir.getCanonicalPath // Test getLatestSnapshot checkSnapshotReport( (engine, path) => tableManager.getSnapshotAtLatest(engine, path), path, SnapshotReportExpectations( expectedReportCount = 1, expectException = true, expectedVersion = Optional.empty(), expectedCheckpointVersion = Optional.empty(), expectedProvidedTimestamp = Optional.empty(), // No time travel by timestamp expectNonZeroLoadProtocolAndMetadataDuration = false, expectNonZeroDurationToGetCrcInfo = false)) // Test getSnapshotAsOfVersion checkSnapshotReport( (engine, path) => tableManager.getSnapshotAtVersion(engine, path, 0), path, SnapshotReportExpectations( expectedReportCount = 1, expectException = true, expectedVersion = Optional.of(0), expectedCheckpointVersion = Optional.empty(), expectedProvidedTimestamp = Optional.empty(), // No time travel by timestamp expectNonZeroLoadProtocolAndMetadataDuration = false, expectNonZeroDurationToGetCrcInfo = false)) // Test getSnapshotAsOfTimestamp checkSnapshotReport( (engine, path) => tableManager.getSnapshotAtTimestamp(engine, path, 1000), path, SnapshotReportExpectations( expectedReportCount = 1, expectException = true, expectedVersion = Optional.empty(), expectedCheckpointVersion = Optional.empty(), // Query will fail before timestamp -> version resolution. The failure // will happen when `getLatestSnapshot` is called. expectedProvidedTimestamp = Optional.empty(), expectNonZeroLoadProtocolAndMetadataDuration = false, // It will first build a lastest snapshot, and a logSegment is built there. expectNonZeroBuildLogSegmentDuration = true, expectNonZeroDurationToGetCrcInfo = false)) } } test("Snapshot report - log is corrupted") { withTempDir { tempDir => val path = tempDir.getCanonicalPath // Set up table with non-contiguous version (0, 2) which will fail during log segment building // for all the following queries (0 until 3).foreach(_ => spark.range(3).write.format("delta").mode("append").save(path)) assert( new File(FileNames.deltaFile(new Path(tempDir.getCanonicalPath, "_delta_log"), 1)).delete()) // Test getLatestSnapshot checkSnapshotReport( (engine, path) => tableManager.getSnapshotAtLatest(engine, path), path, SnapshotReportExpectations( expectedReportCount = 1, expectException = true, expectedVersion = Optional.empty(), expectedCheckpointVersion = Optional.empty(), expectedProvidedTimestamp = Optional.empty(), // No time travel by timestamp expectNonZeroLoadProtocolAndMetadataDuration = false, expectNonZeroDurationToGetCrcInfo = false)) // Test getSnapshotAsOfVersion checkSnapshotReport( (engine, path) => tableManager.getSnapshotAtVersion(engine, path, 2), path, SnapshotReportExpectations( expectedReportCount = 1, expectException = true, expectedVersion = Optional.of(2), expectedCheckpointVersion = Optional.empty(), expectedProvidedTimestamp = Optional.empty(), // No time travel by timestamp expectNonZeroLoadProtocolAndMetadataDuration = false, expectNonZeroDurationToGetCrcInfo = false)) // Test getSnapshotAsOfTimestamp val version2Timestamp = new File( FileNames.deltaFile(new Path(tempDir.getCanonicalPath, "_delta_log"), 2)).lastModified() checkSnapshotReport( (engine, path) => tableManager.getSnapshotAtTimestamp(engine, path, version2Timestamp), tempDir.getCanonicalPath, SnapshotReportExpectations( expectedReportCount = 1, expectException = true, // Query will fail before timestamp -> version resolution. The failure // will happen when `getLatestSnapshot` is called. expectedVersion = Optional.empty(), expectedCheckpointVersion = Optional.empty(), expectedProvidedTimestamp = Optional.empty(), expectNonZeroLoadProtocolAndMetadataDuration = false, expectNonZeroDurationToGetCrcInfo = false)) } } test("Snapshot report - missing metadata") { // This fails during P&M loading for all of the following queries val path = goldenTablePath("deltalog-state-reconstruction-without-metadata") // Test getLatestSnapshot checkSnapshotReport( (engine, path) => tableManager.getSnapshotAtLatest(engine, path), path, SnapshotReportExpectations( expectedReportCount = 1, expectException = true, expectedVersion = Optional.of(0), expectedCheckpointVersion = Optional.empty(), expectedProvidedTimestamp = Optional.empty(), // No time travel by timestamp // No CRC for golden table expectNonZeroDurationToGetCrcInfo = false)) // Test getSnapshotAsOfVersion checkSnapshotReport( (engine, path) => tableManager.getSnapshotAtVersion(engine, path, 0), path, SnapshotReportExpectations( expectedReportCount = 1, expectException = true, expectedVersion = Optional.of(0), expectedCheckpointVersion = Optional.empty(), expectedProvidedTimestamp = Optional.empty(), // No time travel by timestamp // No CRC for golden table expectNonZeroDurationToGetCrcInfo = false)) // Test getSnapshotAsOfTimestamp // We use the timestamp of version 0 val version0Timestamp = new File(FileNames.deltaFile(new Path(path, "_delta_log"), 0)) .lastModified() checkSnapshotReport( (engine, path) => tableManager.getSnapshotAtTimestamp(engine, path, version0Timestamp), path, SnapshotReportExpectations( expectedReportCount = 1, expectException = true, // Query will fail before timestamp -> version resolution. The failure // will happen when `getLatestSnapshot` is called. expectedVersion = Optional.of(0), expectedCheckpointVersion = Optional.empty(), expectedProvidedTimestamp = Optional.empty(), // This is due to the `getLatestSnapshot` call expectNonZeroLoadProtocolAndMetadataDuration = true, // No CRC for golden table expectNonZeroDurationToGetCrcInfo = false)) } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/metrics/TransactionReportSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.metrics import java.util.{Collections, Objects, Optional} import scala.collection.JavaConverters._ import scala.collection.immutable.Seq import io.delta.kernel._ import io.delta.kernel.data.Row import io.delta.kernel.defaults.utils.{AbstractWriteUtils, WriteUtilsWithV1Builders, WriteUtilsWithV2Builders} import io.delta.kernel.engine._ import io.delta.kernel.expressions.{Column, Literal} import io.delta.kernel.internal.{TableConfig, TableImpl} import io.delta.kernel.internal.actions.{GenerateIcebergCompatActionUtils, SingleAction} import io.delta.kernel.internal.data.TransactionStateRow import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.metrics.Timer import io.delta.kernel.internal.stats.FileSizeHistogram import io.delta.kernel.internal.util.Utils import io.delta.kernel.metrics.{FileSizeHistogramResult, SnapshotReport, TransactionMetricsResult, TransactionReport} import io.delta.kernel.types.{IntegerType, StructType} import io.delta.kernel.utils.{CloseableIterable, CloseableIterator, DataFileStatus} import io.delta.kernel.utils.CloseableIterable.{emptyIterable, inMemoryIterable} import org.scalatest.funsuite.AnyFunSuite class TransactionReportTransactionBuilderV1Suite extends AbstractTransactionReportSuite with WriteUtilsWithV1Builders class TransactionReportTransactionBuilderV2Suite extends AbstractTransactionReportSuite with WriteUtilsWithV2Builders trait AbstractTransactionReportSuite extends AnyFunSuite with MetricsReportTestUtils { self: AbstractWriteUtils => /** * Creates a [[Transaction]] using `getTransaction`, requests actions to commit using * `generateCommitActions`, and commits them to the transaction. Uses a custom engine for all * of these operations that collects any emitted metrics reports. Exactly 1 [[TransactionReport]] * is expected to be emitted, and at most one [[SnapshotReport]]. Also times and returns the * duration it takes for [[Transaction#commit]] to finish. * * @param createTransaction given an engine return a started [[Transaction]] * @param generateCommitActions given a [[Transaction]] and engine generates actions to commit * @param expectException whether we expect committing to throw an exception, which if so, is * caught and returned with the other results * @return (TransactionReport, durationToCommit, SnapshotReportIfPresent, ExceptionIfThrown) */ def getTransactionAndSnapshotReport( createTransaction: Engine => Transaction, generateCommitActions: (Transaction, Engine) => CloseableIterable[Row], expectException: Boolean, validateTransactionMetrics: (TransactionMetricsResult, Long) => Unit) : (TransactionReport, Long, Option[SnapshotReport], Option[Exception]) = { val timer = new Timer() val (metricsReports, exception) = collectMetricsReports( engine => { val transaction = createTransaction(engine) val actionsToCommit = generateCommitActions(transaction, engine) val txnCommitResult = timer.time(() => transaction.commit(engine, actionsToCommit)) // Time the actual operation // Validate the txn metrics returned in txnCommitResult validateTransactionMetrics( txnCommitResult.getTransactionReport.getTransactionMetrics, timer.totalDurationNs()) }, expectException) val transactionReports = metricsReports.filter(_.isInstanceOf[TransactionReport]) assert(transactionReports.length == 1, "Expected exactly 1 TransactionReport") val snapshotReports = metricsReports.filter(_.isInstanceOf[SnapshotReport]) assert(snapshotReports.length <= 1, "Expected at most 1 SnapshotReport") ( transactionReports.head.asInstanceOf[TransactionReport], timer.totalDurationNs(), snapshotReports.headOption.map(_.asInstanceOf[SnapshotReport]), exception) } /** * Builds a transaction using `getTransaction` for the table at the provided path. Commits * to the transaction the actions generated by `generateCommitActions` and collects any emitted * [[TransactionReport]]. Checks that the report is as expected * * @param generateCommitActions function to generate commit actions from a transaction and engine * @param path table path to commit to * @param expectException whether we expect committing to throw an exception * @param expectedBaseSnapshotVersion expected snapshot version for the transaction * @param expectedClusteringColumns expected clustering columns for the transaction * @param expectedNumAddFiles expected number of add files recorded in the metrics * @param expectedNumRemoveFiles expected number of remove files recorded in the metrics * @param expectedNumTotalActions expected number of total actions recorded in the metrics * @param expectedCommitVersion expected commit version if not `expectException` * @param expectedNumAttempts expected number of commit attempts * @param getTransaction function to build a transaction from a transaction builder * @param engineInfo engine info to create the transaction with * @param operation operation to create the transaction with */ // scalastyle:off def checkTransactionReport( generateCommitActions: (Transaction, Engine) => CloseableIterable[Row], path: String, expectException: Boolean, expectedBaseSnapshotVersion: Long, getTransaction: (Engine) => Transaction, expectedClusteringColumns: Seq[Column] = Seq.empty, expectedNumAddFiles: Long = 0, expectedNumRemoveFiles: Long = 0, expectedNumTotalActions: Long = 0, expectedCommitVersion: Option[Long] = None, expectedNumAttempts: Long = 1, expectedTotalAddFilesSizeInBytes: Long = 0, expectedTotalRemoveFilesSizeInBytes: Long = 0, expectedFileSizeHistogramResult: Option[FileSizeHistogramResult] = None, operation: Operation = Operation.WRITE): Unit = { // scalastyle:on assert(expectException == expectedCommitVersion.isEmpty) def validateTransactionMetrics(txnMetrics: TransactionMetricsResult, duration: Long): Unit = { // Since we cannot know the actual duration of commit we sanity check that they are > 0 and // less than the total operation duration assert(txnMetrics.getTotalCommitDurationNs > 0) assert(txnMetrics.getTotalCommitDurationNs < duration) assert(txnMetrics.getNumCommitAttempts == expectedNumAttempts) assert(txnMetrics.getNumAddFiles == expectedNumAddFiles) assert(txnMetrics.getTotalAddFilesSizeInBytes == expectedTotalAddFilesSizeInBytes) assert(txnMetrics.getNumRemoveFiles == expectedNumRemoveFiles) assert(txnMetrics.getNumTotalActions == expectedNumTotalActions) assert(txnMetrics.getTotalRemoveFilesSizeInBytes == expectedTotalRemoveFilesSizeInBytes) // For now since we don't support writing fileSizeHistogram yet we only expect this to be // present on the first write to a table. We will update these tests when we add write // support. expectedFileSizeHistogramResult match { case Some(expectedHistogram) => assert(txnMetrics.getTableFileSizeHistogram.isPresent) txnMetrics.getTableFileSizeHistogram.toScala.foreach { foundHistogram => assert(expectedHistogram.getSortedBinBoundaries sameElements foundHistogram.getSortedBinBoundaries) assert(expectedHistogram.getFileCounts sameElements foundHistogram.getFileCounts) assert(expectedHistogram.getTotalBytes sameElements foundHistogram.getTotalBytes) } case None => assert(!txnMetrics.getTableFileSizeHistogram.isPresent) } } val (transactionReport, duration, snapshotReportOpt, exception) = getTransactionAndSnapshotReport( getTransaction, generateCommitActions, expectException, validateTransactionMetrics) // Verify contents assert(transactionReport.getTablePath == defaultEngine.getFileSystemClient.resolvePath(path)) assert(transactionReport.getOperationType == "Transaction") exception match { case Some(e) => assert(transactionReport.getException().isPresent && Objects.equals(transactionReport.getException().get(), e)) case None => assert(!transactionReport.getException().isPresent) } assert(transactionReport.getReportUUID != null) assert(transactionReport.getOperation == operation.toString) assert(transactionReport.getEngineInfo == "test-engine") assert(transactionReport.getBaseSnapshotVersion == expectedBaseSnapshotVersion) if (expectedBaseSnapshotVersion < 0) { // This was for a new table, there is no corresponding SnapshotReport assert(!transactionReport.getSnapshotReportUUID.isPresent) } else { assert(snapshotReportOpt.exists { snapshotReport => snapshotReport.getVersion.toScala.contains(expectedBaseSnapshotVersion) && transactionReport.getSnapshotReportUUID.toScala.contains(snapshotReport.getReportUUID) }) } assert(transactionReport.getClusteringColumns.asScala == expectedClusteringColumns) assert(transactionReport.getCommittedVersion.toScala == expectedCommitVersion) validateTransactionMetrics(transactionReport.getTransactionMetrics, duration) } def generateAppendActions(fileStatusIter: CloseableIterator[DataFileStatus])( trans: Transaction, engine: Engine): CloseableIterable[Row] = { val transState = trans.getTransactionState(engine) CloseableIterable.inMemoryIterable( Transaction.generateAppendActions( engine, transState, fileStatusIter, Transaction.getWriteContext(engine, transState, Collections.emptyMap()))) } def generateRemoveActions(fileStatusIter: CloseableIterator[DataFileStatus])( trans: Transaction, engine: Engine): CloseableIterable[Row] = { // For now we use GenerateIcebergCompatActionUtils to generate the remove rows since this is the // only current API support in Kernel for generating removes; in the future when we support a // more general API for removes we should use that here inMemoryIterable(fileStatusIter.map { fileStatus => SingleAction.createRemoveFileSingleAction( GenerateIcebergCompatActionUtils.convertRemoveDataFileStatus( TransactionStateRow.getPhysicalSchema(trans.getTransactionState(engine)), new Path(TransactionStateRow.getTablePath(trans.getTransactionState(engine))).toUri, fileStatus, Collections.emptyMap(), // partitionValues true, // dataChange, Optional.empty(), // baseRowId Optional.empty(), // defaultRowCommitVersion Optional.empty() // deletionVectorDescriptor )) }) } def incrementFileSizeHistogram( histogram: FileSizeHistogram, fileStatusIter: CloseableIterator[DataFileStatus]): FileSizeHistogram = { fileStatusIter.forEach(fs => histogram.insert(fs.getSize)) histogram } test("TransactionReport: Basic append to existing table + update metadata") { withTempDir { tempDir => val path = tempDir.getCanonicalPath // Set up delta table with version 0 spark.range(10).write.format("delta").mode("append").save(path) // Commit 1 AddFiles checkTransactionReport( generateCommitActions = generateAppendActions(fileStatusIter1), path, expectException = false, expectedBaseSnapshotVersion = 0, getTransaction = (e) => getUpdateTxn(e, path), expectedNumAddFiles = 1, expectedNumTotalActions = 2, // commitInfo + addFile expectedCommitVersion = Some(1), expectedTotalAddFilesSizeInBytes = 100) // Commit 2 AddFiles checkTransactionReport( generateCommitActions = generateAppendActions(fileStatusIter2), path, expectException = false, expectedBaseSnapshotVersion = 1, getTransaction = (e) => getUpdateTxn(e, path), expectedNumAddFiles = 2, expectedNumTotalActions = 3, // commitInfo + addFile expectedCommitVersion = Some(2), expectedTotalAddFilesSizeInBytes = 200) // Update metadata only checkTransactionReport( generateCommitActions = (_, _) => CloseableIterable.emptyIterable(), path, expectException = false, expectedBaseSnapshotVersion = 2, getTransaction = engine => getUpdateTxn( engine, path, tableProperties = Map(TableConfig.CHECKPOINT_INTERVAL.getKey -> "2")), expectedNumTotalActions = 2, // metadata, commitInfo expectedCommitVersion = Some(3)) } } test("TransactionReport: Create new empty table and then append") { withTempDir { tempDir => val path = tempDir.getCanonicalPath checkTransactionReport( generateCommitActions = (_, _) => CloseableIterable.emptyIterable(), path, expectException = false, expectedBaseSnapshotVersion = -1, getTransaction = (engine) => getCreateTxn(engine, path, new StructType().add("id", IntegerType.INTEGER)), expectedNumTotalActions = 3, // protocol, metadata, commitInfo expectedCommitVersion = Some(0), expectedFileSizeHistogramResult = Some( FileSizeHistogram.createDefaultHistogram().captureFileSizeHistogramResult()), operation = Operation.CREATE_TABLE) // Commit 2 AddFiles checkTransactionReport( generateCommitActions = generateAppendActions(fileStatusIter2), path, expectException = false, expectedBaseSnapshotVersion = 0, getTransaction = (e) => getUpdateTxn(e, path), expectedNumAddFiles = 2, expectedNumTotalActions = 3, // commitInfo + addFile expectedCommitVersion = Some(1), expectedTotalAddFilesSizeInBytes = 200) } } test("TransactionReport: Create new non-empty table with insert") { withTempDir { tempDir => val path = tempDir.getCanonicalPath checkTransactionReport( generateCommitActions = generateAppendActions(fileStatusIter1), path, expectException = false, expectedBaseSnapshotVersion = -1, getTransaction = (engine) => getCreateTxn(engine, path, new StructType().add("id", IntegerType.INTEGER)), expectedNumAddFiles = 1, expectedNumTotalActions = 4, // protocol, metadata, commitInfo expectedCommitVersion = Some(0), expectedTotalAddFilesSizeInBytes = 100, expectedFileSizeHistogramResult = Some( incrementFileSizeHistogram( FileSizeHistogram.createDefaultHistogram(), fileStatusIter1).captureFileSizeHistogramResult()), operation = Operation.CREATE_TABLE) } } test("TransactionReport: remove files from a table") { withTempDir { tempDir => val path = tempDir.getCanonicalPath // Create a table and insert 1 file into it checkTransactionReport( generateCommitActions = generateAppendActions(fileStatusIter1), path, expectException = false, expectedBaseSnapshotVersion = -1, getTransaction = (engine) => getCreateTxn(engine, path, new StructType().add("id", IntegerType.INTEGER)), expectedNumAddFiles = 1, expectedNumTotalActions = 4, // protocol, metadata, commitInfo, addFile expectedCommitVersion = Some(0), expectedTotalAddFilesSizeInBytes = 100, expectedFileSizeHistogramResult = Some( incrementFileSizeHistogram( FileSizeHistogram.createDefaultHistogram(), fileStatusIter1).captureFileSizeHistogramResult()), operation = Operation.CREATE_TABLE) // Remove the 1 file and insert 2 new ones checkTransactionReport( generateCommitActions = (txn, engine) => inMemoryIterable(generateAppendActions(fileStatusIter2)(txn, engine).iterator().combine( generateRemoveActions(fileStatusIter1)(txn, engine).iterator())), path, expectException = false, expectedBaseSnapshotVersion = 0, expectedNumAddFiles = 2, expectedNumRemoveFiles = 1, expectedNumTotalActions = 4, // commitInfo, removeFile, 2 addFile getTransaction = (e) => getUpdateTxn(e, path), expectedCommitVersion = Some(1), expectedTotalAddFilesSizeInBytes = 200, expectedTotalRemoveFilesSizeInBytes = 100) // Remove the two files inserted checkTransactionReport( generateCommitActions = generateRemoveActions(fileStatusIter2), path, expectException = false, expectedBaseSnapshotVersion = 1, getTransaction = (e) => getUpdateTxn(e, path), expectedNumRemoveFiles = 2, expectedNumTotalActions = 3, // commitInfo, 2 removeFile expectedCommitVersion = Some(2), expectedTotalRemoveFilesSizeInBytes = 200) } } test("TransactionReport: retry with a concurrent append") { withTempDir { tempDir => val path = tempDir.getCanonicalPath // Set up delta table with version 0 spark.range(10).write.format("delta").mode("append").save(path) checkTransactionReport( generateCommitActions = (trans, engine) => { spark.range(10).write.format("delta").mode("append").save(path) generateAppendActions(fileStatusIter1)(trans, engine) }, path, expectException = false, expectedBaseSnapshotVersion = 0, expectedNumAddFiles = 1, expectedNumTotalActions = 2, // commitInfo + removeFile getTransaction = (e) => getUpdateTxn(e, path), expectedCommitVersion = Some(2), expectedNumAttempts = 2, expectedTotalAddFilesSizeInBytes = 100, // This should always be empty on retries until we support updating based on concurrent txn expectedFileSizeHistogramResult = None) } } test("TransactionReport: fail due to conflicting write") { withTempDir { tempDir => val path = tempDir.getCanonicalPath // Set up delta table with version 0 spark.range(10).write.format("delta").mode("append").save(path) checkTransactionReport( generateCommitActions = (trans, engine) => { spark.sql("ALTER TABLE delta.`" + path + "` ADD COLUMN newCol INT") generateAppendActions(fileStatusIter1)(trans, engine) }, path, expectException = true, expectedBaseSnapshotVersion = 0, getTransaction = (e) => getUpdateTxn(e, path)) } } test("TransactionReport: fail due to too many tries") { withTempDir { tempDir => val path = tempDir.getCanonicalPath // Set up delta table with version 0 spark.range(10).write.format("delta").mode("append").save(path) // This writes a concurrent append everytime the iterable is asked for an iterator. This means // there should be a conflicting transaction committed everytime Kernel tries to commit def actionsIterableWithConcurrentAppend( trans: Transaction, engine: Engine): CloseableIterable[Row] = { val transState = trans.getTransactionState(engine) val writeContext = Transaction.getWriteContext(engine, transState, Collections.emptyMap()) new CloseableIterable[Row] { override def iterator(): CloseableIterator[Row] = { spark.range(10).write.format("delta").mode("append").save(path) Transaction.generateAppendActions(engine, transState, fileStatusIter1, writeContext) } override def close(): Unit = () } } checkTransactionReport( generateCommitActions = actionsIterableWithConcurrentAppend, path, expectException = true, expectedBaseSnapshotVersion = 0, getTransaction = (engine) => getUpdateTxn(engine, path, maxRetries = 5), expectedNumAttempts = 6 ) // 1 first try + 6 retries } } Seq(true, false).foreach { includeData => test(s"TransactionReport: REPLACE a non-empty table, includeData=$includeData") { withTempDir { tempDir => val path = tempDir.getCanonicalPath // Set up a non-empty table at version 0 with add file size that we know val txn = getCreateTxn( defaultEngine, path, new StructType().add("col1", IntegerType.INTEGER), withDomainMetadataSupported = true) txn.addDomainMetadata("user-domain", "some config") val result = txn.commit( defaultEngine, generateAppendActions(fileStatusIter1)(txn, defaultEngine)) // Write out the CRC so that we will have fileSizeHistogram in the next commit result.getPostCommitHooks.asScala.foreach(_.threadSafeInvoke(defaultEngine)) def generateCommitActions: (Transaction, Engine) => CloseableIterable[Row] = if (!includeData) { case (_, _) => emptyIterable() } else { generateAppendActions(fileStatusIter1) } val numAddFiles = if (includeData) 1 else 0 val expectedFileSizeHistogram = if (includeData) { incrementFileSizeHistogram( FileSizeHistogram.createDefaultHistogram(), fileStatusIter1).captureFileSizeHistogramResult() } else { FileSizeHistogram.createDefaultHistogram().captureFileSizeHistogramResult() } // Check TransactionReport for REPLACE operation checkTransactionReport( generateCommitActions, path, expectException = false, expectedBaseSnapshotVersion = 0, expectedNumAddFiles = numAddFiles, expectedNumRemoveFiles = 1, // protocol, metadata, commitInfo, domainMetadata (tombstone) expectedNumTotalActions = numAddFiles + 5, getTransaction = (engine) => getReplaceTxn(engine, path, new StructType().add("id", IntegerType.INTEGER)), expectedCommitVersion = Some(1), expectedTotalRemoveFilesSizeInBytes = 100, expectedTotalAddFilesSizeInBytes = 100 * numAddFiles, expectedFileSizeHistogramResult = Some(expectedFileSizeHistogram), operation = Operation.REPLACE_TABLE) } } } test("TransactionReport: clustering columns are in transaction report") { withTempDirAndEngine { (tablePath, engine) => val testSchema = new StructType() .add("id", IntegerType.INTEGER) .add("name", IntegerType.INTEGER) .add( "nested", new StructType() .add("nestedId", IntegerType.INTEGER) .add("nestedName", IntegerType.INTEGER)) // create table checkTransactionReport( generateCommitActions = (_, _) => CloseableIterable.emptyIterable(), tablePath, expectException = false, expectedBaseSnapshotVersion = -1, expectedCommitVersion = Some(0), expectedNumTotalActions = 4, // protocol, metadata, commitInfo, domainMetadata getTransaction = (engine) => getCreateTxn( engine, tablePath, testSchema, clusteringColsOpt = Some(List( new Column("id"), new Column(Array[String]("nested", "nestedId"))))), expectedClusteringColumns = Seq( new Column("id"), new Column(Array[String]("nested", "nestedId"))), expectedFileSizeHistogramResult = Some( FileSizeHistogram.createDefaultHistogram().captureFileSizeHistogramResult()), operation = Operation.CREATE_TABLE) // update table (no clustering column change) checkTransactionReport( generateCommitActions = generateAppendActions(fileStatusIter1), tablePath, expectException = false, expectedBaseSnapshotVersion = 0, getTransaction = (engine) => getUpdateTxn(engine, tablePath), expectedCommitVersion = Some(1), expectedNumTotalActions = 2, // commitInfo, one add file expectedNumAddFiles = 1, expectedTotalAddFilesSizeInBytes = 100, expectedClusteringColumns = Seq( new Column("id"), new Column(Array[String]("nested", "nestedId")))) // update clustering columns checkTransactionReport( generateCommitActions = (_, _) => CloseableIterable.emptyIterable(), tablePath, expectException = false, expectedBaseSnapshotVersion = 1, expectedCommitVersion = Some(2), getTransaction = (engine) => getUpdateTxn( engine, tablePath, clusteringColsOpt = Some(List( new Column("name"), new Column(Array[String]("nested", "nestedName"))))), expectedNumTotalActions = 2, // commitInfo, domainMetadata expectedClusteringColumns = Seq( new Column("name"), new Column(Array[String]("nested", "nestedName")))) // replace table (with no new clustering columns) checkTransactionReport( generateCommitActions = (_, _) => CloseableIterable.emptyIterable(), tablePath, expectException = false, expectedBaseSnapshotVersion = 2, expectedNumAddFiles = 0, expectedNumRemoveFiles = 1, getTransaction = (engine) => getReplaceTxn(engine, tablePath, testSchema.add("id2", IntegerType.INTEGER)), // protocol, metadata, commitInfo, remove file, domainMetadata (tombstone) expectedNumTotalActions = 5, expectedCommitVersion = Some(3), expectedTotalRemoveFilesSizeInBytes = 100, expectedTotalAddFilesSizeInBytes = 0, operation = Operation.REPLACE_TABLE) // replace the table with new clustering columns checkTransactionReport( generateCommitActions = (_, _) => CloseableIterable.emptyIterable(), tablePath, expectException = false, expectedBaseSnapshotVersion = 3, expectedClusteringColumns = Seq( new Column("id3"), new Column(Array[String]("nested", "nestedName"))), expectedNumAddFiles = 0, expectedNumRemoveFiles = 0, getTransaction = (engine) => getReplaceTxn( engine, tablePath, testSchema.add("id3", IntegerType.INTEGER), clusteringColsOpt = Some(Seq( new Column("id3"), new Column(Array[String]("nested", "nestedName"))))), // protocol, metadata, commitInfo, domainMetadata (new one for clustering cols) expectedNumTotalActions = 4, expectedCommitVersion = Some(4), expectedTotalRemoveFilesSizeInBytes = 0, expectedTotalAddFilesSizeInBytes = 0, operation = Operation.REPLACE_TABLE) } } ///////////////////// // Test Constants // //////////////////// private def fileStatusIter1 = Utils.toCloseableIterator( Seq(new DataFileStatus("/path/to/file", 100, 100, Optional.empty())).iterator.asJava) private def fileStatusIter2 = Utils.toCloseableIterator( Seq( new DataFileStatus("/path/to/file1", 100, 100, Optional.empty()), new DataFileStatus("/path/to/file2", 100, 100, Optional.empty())).iterator.asJava) } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/test/AbstractTableManagerAdapter.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.test import io.delta.kernel.{Table, TableManager} import io.delta.kernel.engine.Engine import io.delta.kernel.internal.{SnapshotImpl, TableImpl} import io.delta.kernel.internal.table.SnapshotBuilderImpl /** * Test framework adapter that provides a unified interface for **loading** Delta tables. * * This trait enables test suites to be parameterized over different Kernel APIs via the * [[LegacyTableManagerAdapter]] and [[TableManagerAdapter]] child classes. * * By using this adapter pattern, the same test suite can verify both APIs work correctly, without * duplicating test logic. * * Tests can switch between implementations by mixing in either * [[io.delta.kernel.defaults.utils.TestUtilsWithLegacyKernelAPIs]] or * [[io.delta.kernel.defaults.utils.TestUtilsWithTableManagerAPIs]]. */ trait AbstractTableManagerAdapter { /** * Does this adapter support resolving a timestamp to a version? * * e.g. getVersionBeforeOrAtTimestamp and getVersionAtOrAfterTimestamp * * This is different from loading a snapshot at a specific timestamp, which is supported by all * adapter implementations. */ def supportsTimestampResolution: Boolean def getSnapshotAtLatest(engine: Engine, path: String): SnapshotImpl def getSnapshotAtVersion(engine: Engine, path: String, version: Long): SnapshotImpl def getSnapshotAtTimestamp(engine: Engine, path: String, timestamp: Long): SnapshotImpl } /** * Legacy implementation using the [[Table.forPath]] API. */ class LegacyTableManagerAdapter extends AbstractTableManagerAdapter { override def supportsTimestampResolution: Boolean = true override def getSnapshotAtLatest( engine: Engine, path: String): SnapshotImpl = { Table.forPath(engine, path).asInstanceOf[TableImpl].getLatestSnapshot(engine) } override def getSnapshotAtVersion( engine: Engine, path: String, version: Long): SnapshotImpl = { Table.forPath(engine, path).asInstanceOf[TableImpl].getSnapshotAsOfVersion(engine, version) } override def getSnapshotAtTimestamp( engine: Engine, path: String, timestamp: Long): SnapshotImpl = { Table.forPath(engine, path).asInstanceOf[TableImpl].getSnapshotAsOfTimestamp(engine, timestamp) } } /** * New implementation using the [[TableManager.loadSnapshot]] API. */ class TableManagerAdapter extends AbstractTableManagerAdapter { override def supportsTimestampResolution: Boolean = false override def getSnapshotAtLatest( engine: Engine, path: String): SnapshotImpl = { TableManager.loadSnapshot(path).asInstanceOf[SnapshotBuilderImpl].build(engine) } override def getSnapshotAtVersion( engine: Engine, path: String, version: Long): SnapshotImpl = { TableManager .loadSnapshot(path).asInstanceOf[SnapshotBuilderImpl].atVersion(version).build(engine) } override def getSnapshotAtTimestamp( engine: Engine, path: String, timestamp: Long): SnapshotImpl = { TableManager .loadSnapshot(path) .asInstanceOf[SnapshotBuilderImpl] .atTimestamp(timestamp, getSnapshotAtLatest(engine, path)) .build(engine) } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/utils/DefaultVectorTestUtils.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.utils import io.delta.kernel.data.{ColumnarBatch, ColumnVector} import io.delta.kernel.defaults.internal.data.DefaultColumnarBatch import io.delta.kernel.test.VectorTestUtils import io.delta.kernel.types._ trait DefaultVectorTestUtils extends VectorTestUtils { /** * Returns a [[ColumnarBatch]] with each given vector is a top-level column col_i where i is * the index of the vector in the input list. */ protected def columnarBatch(vectors: ColumnVector*): ColumnarBatch = { val numRows = vectors.head.getSize vectors.tail.foreach(v => require(v.getSize == numRows, "All vectors should have the same size")) val schema = (0 until vectors.length) .foldLeft(new StructType())((s, i) => s.add(s"col_$i", vectors(i).getDataType)) new DefaultColumnarBatch(numRows, schema, vectors.toArray) } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/utils/ExpressionTestUtils.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.utils import scala.collection.JavaConverters._ import io.delta.kernel.expressions._ /** Useful helper functions for creating expressions in tests */ trait ExpressionTestUtils { def eq(left: Expression, right: Expression): Predicate = predicate("=", left, right) def equals(e1: Expression, e2: Expression): Predicate = eq(e1, e2) def lt(e1: Expression, e2: Expression): Predicate = new Predicate("<", e1, e2) def lessThan(e1: Expression, e2: Expression): Predicate = lt(e1, e2) def gt(e1: Expression, e2: Expression): Predicate = new Predicate(">", e1, e2) def greaterThan(e1: Expression, e2: Expression): Predicate = gt(e1, e2) def gte(e1: Expression, e2: Expression): Predicate = predicate(">=", e1, e2) def greaterThanOrEqual(e1: Expression, e2: Expression): Predicate = gte(e1, e2) def lessThanOrEqual(e1: Expression, e2: Expression): Predicate = new Predicate("<=", e1, e2) def lte(column: Column, literal: Literal): Predicate = predicate("<=", column, literal) def not(pred: Predicate): Predicate = new Predicate("NOT", pred) def isNotNull(e1: Expression): Predicate = new Predicate("IS_NOT_NULL", e1) def col(names: String*): Column = new Column(names.toArray) def nestedCol(name: String): Column = { new Column(name.split("\\.")) } def predicate(name: String, children: Expression*): Predicate = { new Predicate(name, children.asJava) } def and(left: Predicate, right: Predicate): Predicate = predicate("AND", left, right) def or(left: Predicate, right: Predicate): Predicate = predicate("OR", left, right) def int(value: Int): Literal = Literal.ofInt(value) def str(value: String): Literal = Literal.ofString(value) def nullSafeEquals(e1: Expression, e2: Expression): Predicate = { new Predicate("IS NOT DISTINCT FROM", e1, e2) } def unsupported(colName: String): Predicate = predicate("UNSUPPORTED", col(colName)); /* ---------- NOT-YET SUPPORTED EXPRESSIONS ----------- */ /* These expressions are used in ScanSuite to test data skipping. For unsupported expressions no skipping filter will be generated and they should just be returned as part of the remaining predicate to evaluate. As we add support for these expressions we'll adjust the tests that use them to expect skipped files. If they are ever actually evaluated they will throw an exception. */ def notEquals(e1: Expression, e2: Expression): Predicate = new Predicate("<>", e1, e2) def startsWith(e1: Expression, e2: Expression): Predicate = new Predicate("STARTS_WITH", e1, e2) def isNull(e1: Expression): Predicate = new Predicate("IS_NULL", e1) } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/utils/TestCommitterUtils.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.utils import java.util.Collections import io.delta.kernel.commit.{CatalogCommitter, CommitMetadata, CommitResponse, Committer, PublishMetadata} import io.delta.kernel.data.Row import io.delta.kernel.engine.Engine import io.delta.kernel.internal.files.ParsedPublishedDeltaData import io.delta.kernel.internal.util.FileNames import io.delta.kernel.utils.{CloseableIterator, FileStatus} trait TestCommitterUtils { val committerUsingPutIfAbsent = new Committer { override def commit( engine: Engine, finalizedActions: CloseableIterator[Row], commitMetadata: CommitMetadata): CommitResponse = { val filePath = FileNames.deltaFile(commitMetadata.getDeltaLogDirPath, commitMetadata.getVersion) engine .getJsonHandler .writeJsonFileAtomically(filePath, finalizedActions, false) new CommitResponse(ParsedPublishedDeltaData.forFileStatus(FileStatus.of(filePath))) } } val customCatalogCommitter = new CatalogCommitter { val REQUIRED_PROPERTY_KEY = "test.committer.required.foo" val REQUIRED_PROPERTY_VALUE = "bar" override def commit( engine: Engine, finalizedActions: CloseableIterator[Row], commitMetadata: CommitMetadata): CommitResponse = { committerUsingPutIfAbsent.commit(engine, finalizedActions, commitMetadata) } override def getRequiredTableProperties: java.util.Map[String, String] = { Collections.singletonMap(REQUIRED_PROPERTY_KEY, REQUIRED_PROPERTY_VALUE) } override def publish(engine: Engine, publishMetadata: PublishMetadata): Unit = { // No-op } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/utils/TestRow.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.utils import java.sql.Timestamp import java.time.{Instant, LocalDate, LocalDateTime, ZoneOffset} import java.time.ZoneOffset.UTC import java.time.temporal.ChronoUnit import scala.collection.JavaConverters._ import io.delta.kernel.data.{ArrayValue, ColumnVector, MapValue, Row} import io.delta.kernel.types._ import org.apache.spark.sql.{Row => SparkRow} import org.apache.spark.sql.{types => sparktypes} /** * Corresponding Scala class for each Kernel data type: * - BooleanType --> boolean * - ByteType --> byte * - ShortType --> short * - IntegerType --> int * - LongType --> long * - FloatType --> float * - DoubleType --> double * - StringType --> String * - DateType --> int (number of days since the epoch) * - TimestampType --> long (number of microseconds since the unix epoch) * - TimestampNTZType --> long (number of microseconds in local time with no timezone) * - DecimalType --> java.math.BigDecimal * - BinaryType --> Array[Byte] * - ArrayType --> Seq[Any] * - MapType --> Map[Any, Any] * - StructType --> TestRow * * For complex types array and map, the inner elements types should align with this mapping. */ class TestRow(val values: Array[Any]) { def length: Int = values.length def get(i: Int): Any = values(i) def toSeq: Seq[Any] = values.clone() def mkString(start: String, sep: String, end: String): String = { val n = length val builder = new StringBuilder builder.append(start) if (n > 0) { builder.append(get(0)) var i = 1 while (i < n) { builder.append(sep) builder.append(get(i)) i += 1 } } builder.append(end) builder.toString() } override def toString: String = this.mkString("[", ",", "]") } object TestRow { /** * Construct a [[TestRow]] with the given values. See the docs for [[TestRow]] for * the scala type corresponding to each Kernel data type. */ def apply(values: Any*): TestRow = { new TestRow(values.toArray) } /** * Construct a [[TestRow]] with the same values as a Kernel [[Row]]. */ def apply(row: Row): TestRow = { TestRow.fromSeq(row.getSchema.fields().asScala.zipWithIndex.map { case (field, i) => field.getDataType match { case _ if row.isNullAt(i) => null case _: BooleanType => row.getBoolean(i) case _: ByteType => row.getByte(i) case _: IntegerType => row.getInt(i) case _: LongType => row.getLong(i) case _: ShortType => row.getShort(i) case _: DateType => row.getInt(i) case _: TimestampType => row.getLong(i) case _: TimestampNTZType => row.getLong(i) case _: FloatType => row.getFloat(i) case _: DoubleType => row.getDouble(i) case _: StringType => row.getString(i) case _: BinaryType => row.getBinary(i) case _: DecimalType => row.getDecimal(i) case _: ArrayType => arrayValueToScalaSeq(row.getArray(i)) case _: MapType => mapValueToScalaMap(row.getMap(i)) case _: StructType => TestRow(row.getStruct(i)) case _ => throw new UnsupportedOperationException("unrecognized data type") } }.toSeq) } def apply(row: SparkRow): TestRow = { def decodeCellValue(dataType: sparktypes.DataType, obj: Any): Any = { dataType match { case _ if obj == null => null case _: sparktypes.BooleanType => obj.asInstanceOf[Boolean] case _: sparktypes.ByteType => obj.asInstanceOf[Byte] case _: sparktypes.IntegerType => obj.asInstanceOf[Int] case _: sparktypes.LongType => obj.asInstanceOf[Long] case _: sparktypes.ShortType => obj.asInstanceOf[Short] case _: sparktypes.DateType => LocalDate.ofEpochDay(obj.asInstanceOf[Int]) case _: sparktypes.TimestampType => ChronoUnit.MICROS.between(Instant.EPOCH, obj.asInstanceOf[Timestamp].toInstant) case _: sparktypes.TimestampNTZType => ChronoUnit.MICROS.between(Instant.EPOCH, obj.asInstanceOf[LocalDateTime].toInstant(UTC)) case _: sparktypes.FloatType => obj.asInstanceOf[Float] case _: sparktypes.DoubleType => obj.asInstanceOf[Double] case _: sparktypes.StringType => obj.asInstanceOf[String] case _: sparktypes.BinaryType => obj.asInstanceOf[Array[Byte]] case _: sparktypes.DecimalType => obj.asInstanceOf[java.math.BigDecimal] case arrayType: sparktypes.ArrayType => obj.asInstanceOf[Seq[Any]] .map(decodeCellValue(arrayType.elementType, _)) case mapType: sparktypes.MapType => obj.asInstanceOf[Map[Any, Any]].map { case (k, v) => decodeCellValue(mapType.keyType, k) -> decodeCellValue(mapType.valueType, v) } case _: sparktypes.StructType => TestRow(obj.asInstanceOf[SparkRow]) case _ => throw new UnsupportedOperationException("unrecognized data type") } } TestRow.fromSeq(row.schema.fields.zipWithIndex.map { case (field, i) => field.dataType match { case _ if row.isNullAt(i) => null case _: sparktypes.BooleanType => row.getBoolean(i) case _: sparktypes.ByteType => row.getByte(i) case _: sparktypes.IntegerType => row.getInt(i) case _: sparktypes.LongType => row.getLong(i) case _: sparktypes.ShortType => row.getShort(i) case _: sparktypes.DateType => row.getDate(i).toLocalDate.toEpochDay.toInt case _: sparktypes.TimestampType => ChronoUnit.MICROS.between(Instant.EPOCH, row.getTimestamp(i).toInstant) case _: sparktypes.TimestampNTZType => ChronoUnit.MICROS.between(Instant.EPOCH, row.getAs[LocalDateTime](i).toInstant(UTC)) case _: sparktypes.FloatType => row.getFloat(i) case _: sparktypes.DoubleType => row.getDouble(i) case _: sparktypes.StringType => row.getString(i) case _: sparktypes.BinaryType => row(i) // return as byte[], there is no getBinary method case _: sparktypes.DecimalType => row.getDecimal(i) case arrayType: sparktypes.ArrayType => val arrayValue = row.getSeq[Any](i) arrayValue.indices.map { i => decodeCellValue(arrayType.elementType, arrayValue(i)); } case mapType: sparktypes.MapType => val mapValue = row.getMap[Any, Any](i) mapValue.map { case (k, v) => decodeCellValue(mapType.keyType, k) -> decodeCellValue(mapType.valueType, v) } case _: sparktypes.StructType => TestRow(row.getStruct(i)) case _ => throw new UnsupportedOperationException("unrecognized data type") } }) } /** * Retrieves the value at `rowId` in the column vector as it's corresponding scala type. * See the [[TestRow]] docs for details. */ private def getAsTestObject(vector: ColumnVector, rowId: Int): Any = { vector.getDataType match { case _ if vector.isNullAt(rowId) => null case _: BooleanType => vector.getBoolean(rowId) case _: ByteType => vector.getByte(rowId) case _: IntegerType => vector.getInt(rowId) case _: LongType => vector.getLong(rowId) case _: ShortType => vector.getShort(rowId) case _: DateType => vector.getInt(rowId) case _: TimestampType => vector.getLong(rowId) case _: TimestampNTZType => vector.getLong(rowId) case _: FloatType => vector.getFloat(rowId) case _: DoubleType => vector.getDouble(rowId) case _: StringType => vector.getString(rowId) case _: BinaryType => vector.getBinary(rowId) case _: DecimalType => vector.getDecimal(rowId) case _: ArrayType => arrayValueToScalaSeq(vector.getArray(rowId)) case _: MapType => mapValueToScalaMap(vector.getMap(rowId)) case dataType: StructType => TestRow.fromSeq(Seq.range(0, dataType.length()).map { ordinal => getAsTestObject(vector.getChild(ordinal), rowId) }) case _ => throw new UnsupportedOperationException("unrecognized data type") } } private def arrayValueToScalaSeq(arrayValue: ArrayValue): Seq[Any] = { val elemVector = arrayValue.getElements (0 until arrayValue.getSize).map { i => getAsTestObject(elemVector, i) } } private def mapValueToScalaMap(mapValue: MapValue): Map[Any, Any] = { val keyVector = mapValue.getKeys() val valueVector = mapValue.getValues() (0 until mapValue.getSize).map { i => getAsTestObject(keyVector, i) -> getAsTestObject(valueVector, i) }.toMap } /** * Construct a [[TestRow]] from the given seq of values. See the docs for [[TestRow]] for * the scala type corresponding to each Kernel data type. */ def fromSeq(values: Seq[Any]): TestRow = { new TestRow(values.toArray) } /** * Construct a [[TestRow]] with the elements of the given tuple. See the docs for * [[TestRow]] for the scala type corresponding to each Kernel data type. */ def fromTuple(tuple: Product): TestRow = fromSeq(tuple.productIterator.toSeq) } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/utils/TestUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.utils import java.io.{File, FileNotFoundException} import java.math.{BigDecimal => BigDecimalJ} import java.nio.charset.StandardCharsets.UTF_8 import java.nio.file.{Files, Paths} import java.util.{Optional, TimeZone, UUID} import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer import io.delta.golden.GoldenTableUtils import io.delta.kernel.{Scan, Snapshot, Table, TransactionCommitResult} import io.delta.kernel.data._ import io.delta.kernel.defaults.engine.DefaultEngine import io.delta.kernel.defaults.internal.data.vector.{DefaultGenericVector, DefaultStructVector} import io.delta.kernel.defaults.test.{AbstractTableManagerAdapter, LegacyTableManagerAdapter, TableManagerAdapter} import io.delta.kernel.engine.Engine import io.delta.kernel.expressions.{Column, Predicate} import io.delta.kernel.hook.PostCommitHook.PostCommitHookType import io.delta.kernel.internal.{InternalScanFileUtils, SnapshotImpl} import io.delta.kernel.internal.actions.DomainMetadata import io.delta.kernel.internal.checksum.{ChecksumReader, ChecksumWriter, CRCInfo} import io.delta.kernel.internal.clustering.ClusteringMetadataDomain import io.delta.kernel.internal.data.ScanStateRow import io.delta.kernel.internal.fs.Path import io.delta.kernel.internal.stats.FileSizeHistogram import io.delta.kernel.internal.util.{FileNames, Utils} import io.delta.kernel.internal.util.FileNames.checksumFile import io.delta.kernel.internal.util.Utils.singletonCloseableIterator import io.delta.kernel.test.TestFixtures import io.delta.kernel.types._ import io.delta.kernel.utils.{CloseableIterator, FileStatus} import org.apache.spark.sql.delta.{sources, OptimisticTransaction} import org.apache.spark.sql.delta.DeltaOperations.ManualUpdate import org.apache.spark.sql.delta.actions.Action import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.hadoop.conf.Configuration import org.apache.hadoop.shaded.org.apache.commons.io.FileUtils import org.apache.spark.sql.{types => sparktypes, SparkSession} import org.apache.spark.sql.catalyst.plans.SQLHelper import org.scalatest.Assertions trait TestUtils extends AbstractTestUtils { override def getTableManagerAdapter: AbstractTableManagerAdapter = new LegacyTableManagerAdapter() } /** * DO NOT MODIFY this trait -- this is just syntactic sugar to clearly indicate we are extending the * "default" TestUtils which happens to use the legacy Kernel APIs */ trait TestUtilsWithLegacyKernelAPIs extends TestUtils trait TestUtilsWithTableManagerAPIs extends AbstractTestUtils { override def getTableManagerAdapter: AbstractTableManagerAdapter = new TableManagerAdapter() } object TestUtilsWithTableManagerAPIs extends TestUtilsWithTableManagerAPIs trait AbstractTestUtils extends Assertions with SQLHelper with TestCommitterUtils with TestFixtures { def getTableManagerAdapter: AbstractTableManagerAdapter lazy val configuration = new Configuration() lazy val defaultEngine = DefaultEngine.create(configuration) // Used in child suites to override defaultEngine lazy val defaultEngineBatchSize2 = DefaultEngine.create(new Configuration() { { // Set the batch sizes to small so that we get to test the multiple batch scenarios. set("delta.kernel.default.parquet.reader.batch-size", "2"); set("delta.kernel.default.json.reader.batch-size", "2"); } }) lazy val spark = SparkSession .builder() .appName("Spark Test Writer for Delta Kernel") .config("spark.master", "local") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") // Set this conf to empty string so that the golden tables generated // using with the test-prefix (i.e. there is no DELTA_TESTING set) can still work .config(DeltaSQLConf.TEST_DV_NAME_PREFIX.key, "") .getOrCreate() implicit class CloseableIteratorOps[T](private val iter: CloseableIterator[T]) { def forEach(f: T => Unit): Unit = { try { while (iter.hasNext) { f(iter.next()) } } finally { iter.close() } } def toSeq: Seq[T] = { try { val result = new ArrayBuffer[T] while (iter.hasNext) { result.append(iter.next()) } result.toSeq } finally { iter.close() } } } implicit class StructTypeOps(schema: StructType) { def withoutField(name: String): StructType = { val newFields = schema.fields().asScala .filter(_.getName != name).asJava new StructType(newFields) } def toSpark: sparktypes.StructType = { toSparkSchema(schema) } } implicit class ColumnarBatchOps(batch: ColumnarBatch) { def toFiltered: FilteredColumnarBatch = { new FilteredColumnarBatch(batch, Optional.empty()) } def toFiltered(predicate: Option[Predicate]): FilteredColumnarBatch = { if (predicate.isEmpty) { new FilteredColumnarBatch(batch, Optional.empty()) } else { val predicateEvaluator = defaultEngine.getExpressionHandler .getPredicateEvaluator(batch.getSchema, predicate.get) val selVector = predicateEvaluator.eval(batch, Optional.empty()) new FilteredColumnarBatch(batch, Optional.of(selVector)) } } } implicit class FilteredColumnarBatchOps(batch: FilteredColumnarBatch) { def toTestRows: Seq[TestRow] = { batch.getRows.toSeq.map(TestRow(_)) } } implicit class ColumnOps(column: Column) { def toPath: String = column.getNames.mkString(".") } implicit class JavaOptionalOps[T](optional: Optional[T]) { def toScala: Option[T] = if (optional.isPresent) Some(optional.get()) else None } /** * Provides test-only apis to internal Delta Spark APIs. */ implicit class OptimisticTxnTestHelper(txn: OptimisticTransaction) { /** * Test only method to commit arbitrary actions to delta table. */ def commitManuallyWithValidation(actions: Action*): Unit = { txn.commit(actions.toSeq, ManualUpdate) } /** * Test only method to unsafe commit - writes actions directly to transaction log. * Note: This bypasses Delta Spark transaction logic. * * @param tablePath The path to the Delta table * @param version The commit version number * @param actions Sequence of Action objects to write */ def commitUnsafe(tablePath: String, version: Long, actions: Action*): Unit = { val logPath = new org.apache.hadoop.fs.Path(tablePath, "_delta_log") val commitFile = org.apache.spark.sql.delta.util.FileNames.unsafeDeltaFile(logPath, version) val commitContent = actions.map(_.json + "\n").mkString.getBytes(UTF_8) Files.write(Paths.get(commitFile.toString), commitContent) // Generate crc file for this commit version. Table.forPath(defaultEngine, tablePath).checksum(defaultEngine, version) } } implicit object ResourceLoader { lazy val classLoader: ClassLoader = ResourceLoader.getClass.getClassLoader } def withTempDirAndEngine( f: (String, Engine) => Unit, hadoopConf: Map[String, String] = Map.empty): Unit = { val engine = DefaultEngine.create(new Configuration() { { for ((key, value) <- hadoopConf) { set(key, value) } // Set the batch sizes to small so that we get to test the multiple batch/file scenarios. set("delta.kernel.default.parquet.reader.batch-size", "20"); set("delta.kernel.default.json.reader.batch-size", "20"); set("delta.kernel.default.parquet.writer.targetMaxFileSize", "20"); } }) withTempDir { dir => f(dir.getAbsolutePath, engine) } } def withGoldenTable(tableName: String)(testFunc: String => Unit): Unit = { val tablePath = GoldenTableUtils.goldenTablePath(tableName) testFunc(tablePath) } def latestSnapshot(path: String, engine: Engine = defaultEngine): SnapshotImpl = { getTableManagerAdapter.getSnapshotAtLatest(engine, path) } def tableSchema(path: String): StructType = { latestSnapshot(path).getSchema() } def hasTableProperty(tablePath: String, propertyKey: String, expValue: String): Boolean = { val schema = tableSchema(tablePath) schema.fields().asScala.exists { field => field.getMetadata.getString(propertyKey) == expValue } } /** Get the list of all leaf-level primitive column references in the given `structType` */ def leafLevelPrimitiveColumns(basePath: Seq[String], structType: StructType): Seq[Column] = { structType.fields.asScala.flatMap { case field if field.getDataType.isInstanceOf[StructType] => leafLevelPrimitiveColumns( basePath :+ field.getName, field.getDataType.asInstanceOf[StructType]) case field if !field.getDataType.isInstanceOf[ArrayType] && !field.getDataType.isInstanceOf[MapType] => // for all primitive types Seq(new Column((basePath :+ field.getName).asJava.toArray(new Array[String](0)))); case _ => Seq.empty }.toSeq } def collectScanFileRows(scan: Scan, engine: Engine = defaultEngine): Seq[Row] = { scan.getScanFiles(engine).toSeq .flatMap(_.getRows.toSeq) } def readSnapshot( snapshot: Snapshot, readSchema: StructType = null, filter: Predicate = null, expectedRemainingFilter: Predicate = null, engine: Engine = defaultEngine): Seq[Row] = { val result = ArrayBuffer[Row]() var scanBuilder = snapshot.getScanBuilder() if (readSchema != null) { scanBuilder = scanBuilder.withReadSchema(readSchema) } if (filter != null) { scanBuilder = scanBuilder.withFilter(filter) } val scan = scanBuilder.build() if (filter != null) { val actRemainingPredicate = scan.getRemainingFilter() assert( actRemainingPredicate.toString === Optional.ofNullable(expectedRemainingFilter).toString) } val scanState = scan.getScanState(engine); val fileIter = scan.getScanFiles(engine) val physicalDataReadSchema = ScanStateRow.getPhysicalDataReadSchema(scanState) fileIter.forEach { fileColumnarBatch => fileColumnarBatch.getRows().forEach { scanFileRow => val fileStatus = InternalScanFileUtils.getAddFileStatus(scanFileRow) val physicalDataIter = engine.getParquetHandler().readParquetFiles( singletonCloseableIterator(fileStatus), physicalDataReadSchema, Optional.empty()).map(_.getData) var dataBatches: CloseableIterator[FilteredColumnarBatch] = null try { dataBatches = Scan.transformPhysicalData( engine, scanState, scanFileRow, physicalDataIter) dataBatches.forEach { batch => val selectionVector = batch.getSelectionVector() val data = batch.getData() var i = 0 val rowIter = data.getRows() try { while (rowIter.hasNext) { val row = rowIter.next() if (!selectionVector.isPresent || selectionVector.get.getBoolean(i)) { // row is valid result.append(row) } i += 1 } } finally { rowIter.close() } } } finally { dataBatches.close() } } } result.toSeq } def readTableUsingKernel( engine: Engine, tablePath: String, readSchema: StructType): Seq[FilteredColumnarBatch] = { val scan = latestSnapshot(tablePath, engine) .getScanBuilder() .withReadSchema(readSchema) .build() val scanState = scan.getScanState(engine) val physicalDataReadSchema = ScanStateRow.getPhysicalDataReadSchema(scanState) var result: Seq[FilteredColumnarBatch] = Nil scan.getScanFiles(engine).forEach { fileColumnarBatch => fileColumnarBatch.getRows.forEach { scanFile => val fileStatus = InternalScanFileUtils.getAddFileStatus(scanFile) val physicalDataIter = engine.getParquetHandler.readParquetFiles( singletonCloseableIterator(fileStatus), physicalDataReadSchema, Optional.empty()) var dataBatches: CloseableIterator[FilteredColumnarBatch] = null try { dataBatches = Scan.transformPhysicalData(engine, scanState, scanFile, physicalDataIter.map(_.getData)) dataBatches.forEach { dataBatch => result = result :+ dataBatch } } finally { Utils.closeCloseables(dataBatches) } } } result } /** * Execute {@code f} with {@code TimeZone.getDefault()} set to the time zone provided. * * @param zoneId the ID for a TimeZone, either an abbreviation such as "PST", a full name such as * "America/Los_Angeles", or a custom ID such as "GMT-8:00". */ def withTimeZone(zoneId: String)(f: => Unit): Unit = { val currentDefault = TimeZone.getDefault try { TimeZone.setDefault(TimeZone.getTimeZone(zoneId)) f } finally { TimeZone.setDefault(currentDefault) } } /** * Compares the rows in the tables latest snapshot with the expected answer and fails if they * do not match. The comparison is order independent. If expectedSchema is provided, checks * that the latest snapshot's schema is equivalent. * * @param path fully qualified path of the table to check * @param expectedAnswer expected rows * @param readCols subset of columns to read; if null then all columns will be read * @param metadataCols set of metadata columns to read; if null then no metadata columns will * be read * @param engine engine to use to read the table * @param expectedSchema expected schema to check for (ignoring metadata columns); * if null then no check is performed * @param filter Filter to select a subset of rows form the table * @param expectedRemainingFilter Remaining predicate out of the `filter` that is not enforced * by Kernel. * @param expectedVersion expected version of the latest snapshot for the table */ // scalastyle:off argcount def checkTable( path: String, expectedAnswer: Seq[TestRow], readCols: Seq[String] = null, metadataCols: Seq[StructField] = null, engine: Engine = defaultEngine, expectedSchema: StructType = null, filter: Predicate = null, version: Option[Long] = None, timestamp: Option[Long] = None, expectedRemainingFilter: Predicate = null, expectedVersion: Option[Long] = None): Unit = { assert(version.isEmpty || timestamp.isEmpty, "Cannot provide both a version and timestamp") val snapshot = if (version.isDefined) { getTableManagerAdapter.getSnapshotAtVersion(engine, path, version.get) } else if (timestamp.isDefined) { getTableManagerAdapter.getSnapshotAtTimestamp(engine, path, timestamp.get) } else { getTableManagerAdapter.getSnapshotAtLatest(engine, path) } val readSchema = if (readCols == null && metadataCols == null) null else { val schema = snapshot.getSchema() val readFields = Option(readCols).map(_.map(schema.get)).getOrElse(schema.fields().asScala) val metadataFields = Option(metadataCols).getOrElse(Seq()) new StructType((readFields ++ metadataFields).asJava) } if (expectedSchema != null) { // We ignore metadata columns in this check because metadata columns are not part of the // public table schema. assert( expectedSchema == snapshot.getSchema(), s""" |Expected schema does not match actual schema: |Expected schema: $expectedSchema |Actual schema: ${snapshot.getSchema()} |""".stripMargin) } val actualVersion = snapshot.getVersion() expectedVersion.foreach { version => assert( version == actualVersion, s"Expected version $version does not match actual version $actualVersion}") } val result = readSnapshot( snapshot, readSchema, filter, expectedRemainingFilter, engine) checkAnswer(result, expectedAnswer) } // scalastyle:on argcount def checkAnswer(result: => Seq[Row], expectedAnswer: Seq[TestRow]): Unit = { checkAnswer(result.map(TestRow(_)), expectedAnswer) } def checkAnswer(result: Seq[TestRow], expectedAnswer: Seq[TestRow]): Unit = { if (!compare(prepareAnswer(result), prepareAnswer(expectedAnswer))) { fail(genErrorMessage(expectedAnswer, result)) } } private def prepareAnswer(answer: Seq[TestRow]): Seq[TestRow] = { // Converts data to types that we can do equality comparison using Scala collections. // For BigDecimal type, the Scala type has a better definition of equality test (similar to // Java's java.math.BigDecimal.compareTo). // For binary arrays, we convert it to Seq to avoid of calling java.util.Arrays.equals for // equality test. val converted = answer.map(prepareRow) converted.sortBy(_.toString()) } // We need to call prepareRow recursively to handle schemas with struct types. private def prepareRow(row: TestRow): TestRow = { TestRow.fromSeq(row.toSeq.map { case null => null case bd: java.math.BigDecimal => BigDecimal(bd) // Equality of WrappedArray differs for AnyVal and AnyRef in Scala 2.12.2+ case seq: Seq[_] => seq.map { case b: java.lang.Byte => b.byteValue case s: java.lang.Short => s.shortValue case i: java.lang.Integer => i.intValue case l: java.lang.Long => l.longValue case f: java.lang.Float => f.floatValue case d: java.lang.Double => d.doubleValue case x => x } // Convert array to Seq for easy equality check. case b: Array[_] => b.toSeq case r: TestRow => prepareRow(r) case o => o }) } private def compare(obj1: Any, obj2: Any): Boolean = (obj1, obj2) match { case (null, null) => true case (null, _) => false case (_, null) => false case (a: Array[_], b: Array[_]) => a.length == b.length && a.zip(b).forall { case (l, r) => compare(l, r) } case (a: Map[_, _], b: Map[_, _]) => a.size == b.size && a.keys.forall { aKey => b.keys.find(bKey => compare(aKey, bKey)).exists(bKey => compare(a(aKey), b(bKey))) } case (a: Iterable[_], b: Iterable[_]) => a.size == b.size && a.zip(b).forall { case (l, r) => compare(l, r) } case (a: Product, b: Product) => compare(a.productIterator.toSeq, b.productIterator.toSeq) case (a: TestRow, b: TestRow) => compare(a.toSeq, b.toSeq) // 0.0 == -0.0, turn float/double to bits before comparison, to distinguish 0.0 and -0.0. case (a: Double, b: Double) => java.lang.Double.doubleToRawLongBits(a) == java.lang.Double.doubleToRawLongBits(b) case (a: Float, b: Float) => java.lang.Float.floatToRawIntBits(a) == java.lang.Float.floatToRawIntBits(b) case (a, b) => if (!a.equals(b)) { val sds = 200; } a.equals(b) // In scala == does not call equals for boxed numeric classes? } private def genErrorMessage(expectedAnswer: Seq[TestRow], result: Seq[TestRow]): String = { // TODO: improve to include schema or Java type information to help debugging s""" |== Results == | |== Expected Answer - ${expectedAnswer.size} == |${prepareAnswer(expectedAnswer).map(_.toString()).mkString("(", ",", ")")} | |== Result - ${result.size} == |${prepareAnswer(result).map(_.toString()).mkString("(", ",", ")")} | |""".stripMargin } /** * Creates a temporary directory, which is then passed to `f` and will be deleted after `f` * returns. */ protected def withTempDir(f: File => Unit): Unit = { val tempDir = Files.createTempDirectory(UUID.randomUUID().toString).toFile try f(tempDir) finally { FileUtils.deleteDirectory(tempDir) } } /** * Creates a temporary directory with Delta log structure (_delta_log, _staged_commits, * _sidecars), passes (tablePath, logPath) to `f`, and deletes the directory after `f` returns. */ protected def withTempDirAndAllDeltaSubDirs(f: (String, String) => Unit): Unit = { val tempDir = Files.createTempDirectory(UUID.randomUUID().toString).toFile val deltaLogDir = new File(tempDir, "_delta_log") deltaLogDir.mkdirs() new File(deltaLogDir, FileNames.STAGED_COMMIT_DIRECTORY).mkdirs() new File(deltaLogDir, FileNames.SIDECAR_DIRECTORY).mkdirs() try f(tempDir.getAbsolutePath, deltaLogDir.getAbsolutePath) finally { FileUtils.deleteDirectory(tempDir) } } /** * Create a unique table name and drops it after completing `f` */ protected def withTempTable[T](f: String => T): T = { val tableName = s"temp_table_${UUID.randomUUID().toString.replace("-", "_")}" try { f(tableName) } finally { spark.sql(s"DROP TABLE IF EXISTS $tableName") } } def withSparkTimeZone(timeZone: String)(fn: => Unit): Unit = { val prevTimeZone = spark.conf.get("spark.sql.session.timeZone") try { spark.conf.set("spark.sql.session.timeZone", timeZone) fn } finally { spark.conf.set("spark.sql.session.timeZone", prevTimeZone) } } /** * Builds a MapType ColumnVector from a sequence of maps. */ def buildMapVector(mapValues: Seq[Map[AnyRef, AnyRef]], dataType: MapType): ColumnVector = { val keyType = dataType.getKeyType val valueType = dataType.getValueType def getMapValue(map: Map[AnyRef, AnyRef]): MapValue = { if (map == null) { null } else { val (keys, values) = map.unzip new MapValue() { override def getSize: Int = map.size override def getKeys = DefaultGenericVector.fromArray(keyType, keys.toArray) override def getValues = DefaultGenericVector.fromArray(valueType, values.toArray) } } } DefaultGenericVector.fromArray(dataType, mapValues.map(getMapValue).toArray) } /** * Builds an ArrayType ColumnVector from a sequence of per-row element sequences. */ def buildArrayVector( valuesPerRow: Seq[Seq[AnyRef]], elementType: DataType, containsNull: Boolean): ColumnVector = { val arrayType = new ArrayType(elementType, containsNull) val arrayValues: Array[ArrayValue] = valuesPerRow.map { elems => if (elems == null) null else new ArrayValue { override def getSize: Int = elems.size override def getElements: ColumnVector = DefaultGenericVector.fromArray(elementType, elems.toArray) } }.toArray DefaultGenericVector.fromArray(arrayType, arrayValues.asInstanceOf[Array[AnyRef]]) } /** * Utility method to generate a [[dataType]] column vector of given size. * The nullability of rows is determined by the [[testIsNullValue(dataType, rowId)]]. * The row values are determined by [[testColumnValue(dataType, rowId)]]. */ def testColumnVector(size: Int, dataType: DataType): ColumnVector = { dataType match { // Build a DefaultStructVector and recursively // build child vectors for each field. case structType: StructType => val memberVectors: Array[ColumnVector] = structType.fields().asScala.map { field => testColumnVector(size, field.getDataType) }.toArray new DefaultStructVector( size, structType, Optional.empty(), memberVectors) case _ => new ColumnVector { override def getDataType: DataType = dataType override def getSize: Int = size override def close(): Unit = {} override def isNullAt(rowId: Int): Boolean = testIsNullValue(dataType, rowId) override def getBoolean(rowId: Int): Boolean = testColumnValue(dataType, rowId).asInstanceOf[Boolean] override def getByte(rowId: Int): Byte = testColumnValue(dataType, rowId).asInstanceOf[Byte] override def getShort(rowId: Int): Short = testColumnValue(dataType, rowId).asInstanceOf[Short] override def getInt(rowId: Int): Int = testColumnValue(dataType, rowId).asInstanceOf[Int] override def getLong(rowId: Int): Long = testColumnValue(dataType, rowId).asInstanceOf[Long] override def getFloat(rowId: Int): Float = testColumnValue(dataType, rowId).asInstanceOf[Float] override def getDouble(rowId: Int): Double = testColumnValue(dataType, rowId).asInstanceOf[Double] override def getBinary(rowId: Int): Array[Byte] = testColumnValue(dataType, rowId).asInstanceOf[Array[Byte]] override def getString(rowId: Int): String = testColumnValue(dataType, rowId).asInstanceOf[String] override def getDecimal(rowId: Int): BigDecimalJ = testColumnValue(dataType, rowId).asInstanceOf[BigDecimalJ] } } } /** Utility method to generate a consistent `isNull` value for given column type and row id */ def testIsNullValue(dataType: DataType, rowId: Int): Boolean = { dataType match { case BooleanType.BOOLEAN => rowId % 4 == 0 case ByteType.BYTE => rowId % 8 == 0 case ShortType.SHORT => rowId % 12 == 0 case IntegerType.INTEGER => rowId % 20 == 0 case LongType.LONG => rowId % 25 == 0 case FloatType.FLOAT => rowId % 5 == 0 case DoubleType.DOUBLE => rowId % 10 == 0 case _: StringType => rowId % 2 == 0 case BinaryType.BINARY => rowId % 3 == 0 case DateType.DATE => rowId % 5 == 0 case TimestampType.TIMESTAMP => rowId % 3 == 0 case TimestampNTZType.TIMESTAMP_NTZ => rowId % 2 == 0 case _ => if (dataType.isInstanceOf[DecimalType]) rowId % 6 == 0 else throw new UnsupportedOperationException(s"$dataType is not supported") } } /** Utility method to generate a consistent column value for given column type and row id */ def testColumnValue(dataType: DataType, rowId: Int): Any = { dataType match { case BooleanType.BOOLEAN => rowId % 7 == 0 case ByteType.BYTE => (rowId * 7 / 17).toByte case ShortType.SHORT => (rowId * 9 / 87).toShort case IntegerType.INTEGER => rowId * 2876 / 176 case LongType.LONG => rowId * 287623L / 91 case FloatType.FLOAT => rowId * 7651.2323f / 91 case DoubleType.DOUBLE => rowId * 23423.23d / 17 case _: StringType => (rowId % 19).toString case BinaryType.BINARY => Array[Byte]((rowId % 21).toByte, (rowId % 7 - 1).toByte) case DateType.DATE => (rowId * 28234) % 2876 case TimestampType.TIMESTAMP => (rowId * 2342342L) % 23 case TimestampNTZType.TIMESTAMP_NTZ => (rowId * 523423L) % 29 case _ => if (dataType.isInstanceOf[DecimalType]) new BigDecimalJ(rowId * 22342.23) else throw new UnsupportedOperationException(s"$dataType is not supported") } } /** * Utility method to replicate the behavior of individual values when they are converted from * Row to TestRow. */ def testColumnNullableValue(dataType: DataType, rowId: Int): Any = { if (testIsNullValue(dataType, rowId)) { null } else { testColumnValue(dataType, rowId) } } def testSingleValueVector(dataType: DataType, size: Int, value: Any): ColumnVector = { new ColumnVector { override def getDataType: DataType = dataType override def getSize: Int = size override def close(): Unit = {} override def isNullAt(rowId: Int): Boolean = value == null override def getBoolean(rowId: Int): Boolean = value.asInstanceOf[Boolean] override def getByte(rowId: Int): Byte = value.asInstanceOf[Byte] override def getShort(rowId: Int): Short = value.asInstanceOf[Short] override def getInt(rowId: Int): Int = value.asInstanceOf[Int] override def getLong(rowId: Int): Long = value.asInstanceOf[Long] override def getFloat(rowId: Int): Float = value.asInstanceOf[Float] override def getDouble(rowId: Int): Double = value.asInstanceOf[Double] override def getBinary(rowId: Int): Array[Byte] = value.asInstanceOf[Array[Byte]] override def getString(rowId: Int): String = value.asInstanceOf[String] override def getDecimal(rowId: Int): BigDecimalJ = value.asInstanceOf[BigDecimalJ] } } /** * Converts a Delta Schema to a Spark Schema. */ private def toSparkSchema(deltaSchema: StructType): sparktypes.StructType = { toSparkType(deltaSchema).asInstanceOf[sparktypes.StructType] } /** * Converts a Delta DataType to a Spark DataType. */ private def toSparkType(deltaType: DataType): sparktypes.DataType = { deltaType match { case BooleanType.BOOLEAN => sparktypes.DataTypes.BooleanType case ByteType.BYTE => sparktypes.DataTypes.ByteType case ShortType.SHORT => sparktypes.DataTypes.ShortType case IntegerType.INTEGER => sparktypes.DataTypes.IntegerType case LongType.LONG => sparktypes.DataTypes.LongType case FloatType.FLOAT => sparktypes.DataTypes.FloatType case DoubleType.DOUBLE => sparktypes.DataTypes.DoubleType case _: StringType => sparktypes.DataTypes.StringType case BinaryType.BINARY => sparktypes.DataTypes.BinaryType case DateType.DATE => sparktypes.DataTypes.DateType case TimestampType.TIMESTAMP => sparktypes.DataTypes.TimestampType case TimestampNTZType.TIMESTAMP_NTZ => sparktypes.DataTypes.TimestampNTZType case dt: DecimalType => sparktypes.DecimalType(dt.getPrecision, dt.getScale) case at: ArrayType => sparktypes.ArrayType(toSparkType(at.getElementType), at.containsNull()) case mt: MapType => sparktypes.MapType( toSparkType(mt.getKeyType), toSparkType(mt.getValueType), mt.isValueContainsNull) case st: StructType => sparktypes.StructType(st.fields().asScala.map { field => sparktypes.StructField( field.getName, toSparkType(field.getDataType), field.isNullable) }.toSeq) } } /** * Returns a URI encoded path of the resource. */ def getTestResourceFilePath(resourcePath: String): String = { val resource = ResourceLoader.classLoader.getResource(resourcePath) if (resource == null) { throw new FileNotFoundException("resource not found") } resource.getFile } def checkpointFileExistsForTable(tablePath: String, versions: Int): Boolean = Files.exists( new File(FileNames.checkpointFileSingular( new Path(s"$tablePath/_delta_log"), versions).toString).toPath) def deleteChecksumFileForTable(tablePath: String, versions: Seq[Int]): Unit = versions.foreach(v => Files.deleteIfExists( new File(FileNames.checksumFile(new Path(s"$tablePath/_delta_log"), v).toString).toPath)) def deleteChecksumFileForTableUsingHadoopFs(tablePath: String, versions: Seq[Int]): Unit = versions.foreach(v => defaultEngine.getFileSystemClient.delete(FileNames.checksumFile( new Path(s"$tablePath/_delta_log"), v).toString)) def rewriteChecksumFileToExcludeDomainMetadata( engine: Engine, tablePath: String, version: Long): Unit = { val logPath = new Path(s"$tablePath/_delta_log"); val crcInfo = ChecksumReader.tryReadChecksumFile( engine, FileStatus.of(checksumFile( logPath, version).toString)).get() // Delete it in hdfs. engine.getFileSystemClient.delete(FileNames.checksumFile( new Path(s"$tablePath/_delta_log"), version).toString) val crcWriter = new ChecksumWriter(logPath) crcWriter.writeCheckSum( engine, new CRCInfo( crcInfo.getVersion, crcInfo.getMetadata, crcInfo.getProtocol, crcInfo.getTableSizeBytes, crcInfo.getNumFiles, crcInfo.getTxnId, /* domainMetadata */ Optional.empty(), crcInfo.getFileSizeHistogram)) } def executeCrcSimple(result: TransactionCommitResult, engine: Engine): TransactionCommitResult = { val crcSimpleHook = result .getPostCommitHooks .asScala .find(hook => hook.getType == PostCommitHookType.CHECKSUM_SIMPLE) .getOrElse(throw new IllegalStateException("CRC simple hook not found")) crcSimpleHook.threadSafeInvoke(engine) result } def verifyClusteringDomainMetadata( snapshot: SnapshotImpl, expectedDomainMetadata: DomainMetadata): Unit = { assert(snapshot.getActiveDomainMetadataMap.get(ClusteringMetadataDomain.DOMAIN_NAME) == expectedDomainMetadata) } /** * Verify checksum data matches the expected values in the snapshot. * @param snapshot Snapshot to verify the checksum against */ protected def verifyChecksumForSnapshot( snapshot: Snapshot, expectEmptyTable: Boolean = false): Unit = { val logPath = snapshot.asInstanceOf[SnapshotImpl].getLogPath val crcInfoOpt = ChecksumReader.tryReadChecksumFile( defaultEngine, FileStatus.of(checksumFile( logPath, snapshot.getVersion).toString)) assert( crcInfoOpt.isPresent, s"CRC information should be present for version ${snapshot.getVersion}") crcInfoOpt.toScala.foreach { crcInfo => // TODO: check file size. assert(crcInfo.getProtocol === snapshot.asInstanceOf[SnapshotImpl].getProtocol) assert(crcInfo.getMetadata.getSchema === snapshot.getSchema) assert( crcInfo.getNumFiles === collectScanFileRows(snapshot.getScanBuilder.build()).size, "Number of files in checksum should match snapshot") if (expectEmptyTable) { assert(crcInfo.getTableSizeBytes == 0) crcInfo.getFileSizeHistogram.toScala.foreach { fileSizeHistogram => assert(fileSizeHistogram == FileSizeHistogram.createDefaultHistogram) } } assert( crcInfo.getDomainMetadata === Optional.of( snapshot.asInstanceOf[SnapshotImpl].getActiveDomainMetadataMap.values().asScala .toSet .asJava), "Domain metadata in checksum should match snapshot") } } /** * Ensure checksum is readable by CRC reader, matches snapshot data, and can be regenerated. * This test verifies: * 1. The initial checksum exists and is correct * 2. After deleting the checksum file, it can be regenerated with the same content */ def verifyChecksum(tablePath: String, expectEmptyTable: Boolean = false): Unit = { val currentSnapshot = latestSnapshot(tablePath, defaultEngine) val checksumVersion = currentSnapshot.getVersion // Step 1: Verify initial checksum verifyChecksumForSnapshot(currentSnapshot) // Step 2: Delete and regenerate the checksum defaultEngine.getFileSystemClient.delete(buildCrcPath(tablePath, checksumVersion).toString) Table.forPath(defaultEngine, tablePath).checksum(defaultEngine, checksumVersion) // Step 3: Verify regenerated checksum verifyChecksumForSnapshot(currentSnapshot) } protected def buildCrcPath(basePath: String, version: Long): java.nio.file.Path = { new File(FileNames.checksumFile(new Path(f"$basePath/_delta_log"), version).toString).toPath } protected def optionToJava[T](option: Option[T]): Optional[T] = { option match { case Some(value) => Optional.of(value) case None => Optional.empty() } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/utils/TransactionBuilderSupport.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.utils import scala.collection.JavaConverters._ import io.delta.kernel.{Operation, Table, TableManager, Transaction} import io.delta.kernel.engine.Engine import io.delta.kernel.expressions.Column import io.delta.kernel.internal.{CreateTableTransactionBuilderImpl, UpdateTableTransactionBuilderImpl} import io.delta.kernel.internal.{SnapshotImpl, TableImpl} import io.delta.kernel.internal.tablefeatures.TableFeatures import io.delta.kernel.internal.util.Clock import io.delta.kernel.transaction.DataLayoutSpec import io.delta.kernel.types.StructType /** * Test helper contract for constructing and configuring Delta Kernel transactions. */ trait TransactionBuilderSupport { // scalastyle:off argcount def getCreateTxn( engine: Engine, tablePath: String, schema: StructType, partCols: Seq[String] = null, tableProperties: Map[String, String] = null, clock: Clock = () => System.currentTimeMillis, withDomainMetadataSupported: Boolean = false, maxRetries: Int = -1, clusteringColsOpt: Option[List[Column]] = None): Transaction def getUpdateTxn( engine: Engine, tablePath: String, schema: StructType = null, tableProperties: Map[String, String] = null, clock: Clock = () => System.currentTimeMillis, withDomainMetadataSupported: Boolean = false, maxRetries: Int = -1, clusteringColsOpt: Option[List[Column]] = None, logCompactionInterval: Int = 10, txnId: Option[(String, Long)] = None, tablePropertiesRemoved: Set[String] = null): Transaction // scalastyle:on argcount def getReplaceTxn( engine: Engine, tablePath: String, schema: StructType, partCols: Seq[String] = null, clusteringColsOpt: Option[Seq[Column]] = None, tableProperties: Map[String, String] = null, withDomainMetadataSupported: Boolean = false, maxRetries: Int = -1): Transaction } /** An implementation of [[TransactionBuilderSupport]] that uses the V1 transaction builder. */ trait TransactionBuilderV1Support extends TransactionBuilderSupport with TestUtils { // scalastyle:off argcount override def getCreateTxn( engine: Engine, tablePath: String, schema: StructType, partCols: Seq[String] = null, tableProperties: Map[String, String] = null, clock: Clock = () => System.currentTimeMillis, withDomainMetadataSupported: Boolean = false, maxRetries: Int = -1, clusteringColsOpt: Option[List[Column]] = None): Transaction = { // scalastyle:on argcount var txnBuilder = TableImpl.forPath(engine, tablePath, clock) .createTransactionBuilder(engine, "test-engine", Operation.CREATE_TABLE) .withSchema(engine, schema) if (partCols != null) { txnBuilder = txnBuilder.withPartitionColumns(engine, partCols.asJava) } if (tableProperties != null) { txnBuilder = txnBuilder.withTableProperties(engine, tableProperties.asJava) } if (withDomainMetadataSupported) { txnBuilder = txnBuilder.withDomainMetadataSupported() } if (maxRetries >= 0) { txnBuilder = txnBuilder.withMaxRetries(maxRetries) } if (clusteringColsOpt.isDefined) { txnBuilder = txnBuilder.withClusteringColumns(engine, clusteringColsOpt.get.asJava) } txnBuilder.build(engine) } // scalastyle:off argcount override def getUpdateTxn( engine: Engine, tablePath: String, schema: StructType = null, tableProperties: Map[String, String] = null, clock: Clock = () => System.currentTimeMillis, withDomainMetadataSupported: Boolean = false, maxRetries: Int = -1, clusteringColsOpt: Option[List[Column]] = None, logCompactionInterval: Int = 10, txnId: Option[(String, Long)] = None, tablePropertiesRemoved: Set[String] = null): Transaction = { // scalastyle:on argcount var txnBuilder = TableImpl.forPath(engine, tablePath, clock) .createTransactionBuilder(engine, "test-engine", Operation.WRITE) if (schema != null) { txnBuilder = txnBuilder.withSchema(engine, schema) } if (tableProperties != null) { txnBuilder = txnBuilder.withTableProperties(engine, tableProperties.asJava) } if (withDomainMetadataSupported) { txnBuilder = txnBuilder.withDomainMetadataSupported() } if (maxRetries >= 0) { txnBuilder = txnBuilder.withMaxRetries(maxRetries) } if (clusteringColsOpt.isDefined) { txnBuilder = txnBuilder.withClusteringColumns(engine, clusteringColsOpt.get.asJava) } txnBuilder = txnBuilder.withLogCompactionInverval(logCompactionInterval) txnId.foreach { case (appId, txnVer) => txnBuilder = txnBuilder.withTransactionId(engine, appId, txnVer) } if (tablePropertiesRemoved != null) { txnBuilder = txnBuilder.withTablePropertiesRemoved(tablePropertiesRemoved.asJava) } txnBuilder.build(engine) } override def getReplaceTxn( engine: Engine, tablePath: String, schema: StructType, partCols: Seq[String] = null, clusteringColsOpt: Option[Seq[Column]] = None, tableProperties: Map[String, String] = null, withDomainMetadataSupported: Boolean = false, maxRetries: Int = -1): Transaction = { var txnBuilder = Table.forPath(engine, tablePath).asInstanceOf[TableImpl] .createReplaceTableTransactionBuilder(engine, "test-engine") .withSchema(engine, schema) if (partCols != null) { txnBuilder = txnBuilder.withPartitionColumns(engine, partCols.asJava) } if (tableProperties != null) { txnBuilder = txnBuilder.withTableProperties(engine, tableProperties.asJava) } if (withDomainMetadataSupported) { txnBuilder = txnBuilder.withDomainMetadataSupported() } clusteringColsOpt.foreach { cols => txnBuilder = txnBuilder.withClusteringColumns(engine, cols.asJava) } if (maxRetries >= 0) { txnBuilder = txnBuilder.withMaxRetries(maxRetries) } txnBuilder.build(engine) } } /** An implementation of [[TransactionBuilderSupport]] that uses the V2 transaction builder. */ trait TransactionBuilderV2Support extends TransactionBuilderSupport with TestUtils { // scalastyle:off argcount override def getCreateTxn( engine: Engine, tablePath: String, schema: StructType, partCols: Seq[String] = null, tableProperties: Map[String, String] = null, clock: Clock = () => System.currentTimeMillis, withDomainMetadataSupported: Boolean = false, maxRetries: Int = -1, clusteringColsOpt: Option[List[Column]] = None): Transaction = { // scalastyle:on argcount var txnBuilder = TableManager.buildCreateTableTransaction( tablePath, schema, "test-engine") .asInstanceOf[CreateTableTransactionBuilderImpl] .withClock(clock) if (partCols != null) { txnBuilder = txnBuilder.withDataLayoutSpec( DataLayoutSpec.partitioned(partCols.map(new Column(_)).asJava)) } val completeTblProps = tblPropertiesWithDomainMetadata(tableProperties, withDomainMetadataSupported) if (completeTblProps != null) { txnBuilder = txnBuilder.withTableProperties(completeTblProps.asJava) } if (clusteringColsOpt.nonEmpty) { txnBuilder = txnBuilder.withDataLayoutSpec( DataLayoutSpec.clustered(clusteringColsOpt.get.asJava)) } if (maxRetries >= 0) { txnBuilder = txnBuilder.withMaxRetries(maxRetries) } txnBuilder.build(engine) } // scalastyle:off argcount override def getUpdateTxn( engine: Engine, tablePath: String, schema: StructType = null, tableProperties: Map[String, String] = null, clock: Clock = () => System.currentTimeMillis, withDomainMetadataSupported: Boolean = false, maxRetries: Int = -1, clusteringColsOpt: Option[List[Column]] = None, logCompactionInterval: Int = 10, txnId: Option[(String, Long)] = None, tablePropertiesRemoved: Set[String] = null): Transaction = { // scalastyle:on argcount var txnBuilder = TableManager.loadSnapshot(tablePath) .build(engine) .buildUpdateTableTransaction("test-engine", Operation.WRITE) .asInstanceOf[UpdateTableTransactionBuilderImpl] .withClock(clock) if (schema != null) { txnBuilder = txnBuilder.withUpdatedSchema(schema) } clusteringColsOpt.foreach { clusteringCols => txnBuilder = txnBuilder.withClusteringColumns(clusteringCols.asJava) } val completeTblProps = tblPropertiesWithDomainMetadata(tableProperties, withDomainMetadataSupported) if (completeTblProps != null) { txnBuilder = txnBuilder.withTablePropertiesAdded(completeTblProps.asJava) } if (maxRetries >= 0) { txnBuilder = txnBuilder.withMaxRetries(maxRetries) } txnBuilder = txnBuilder.withLogCompactionInterval(logCompactionInterval) txnId.foreach { case (appId, txnVer) => txnBuilder = txnBuilder.withTransactionId(appId, txnVer) } if (tablePropertiesRemoved != null) { txnBuilder = txnBuilder.withTablePropertiesRemoved(tablePropertiesRemoved.asJava) } txnBuilder.build(engine) } override def getReplaceTxn( engine: Engine, tablePath: String, schema: StructType, partCols: Seq[String] = null, clusteringColsOpt: Option[Seq[Column]] = None, tableProperties: Map[String, String] = null, withDomainMetadataSupported: Boolean = false, maxRetries: Int = -1): Transaction = { var txnBuilder = TableManager.loadSnapshot(tablePath) .build(engine).asInstanceOf[SnapshotImpl] .buildReplaceTableTransaction(schema, "test-engine") if (partCols != null) { txnBuilder = txnBuilder.withDataLayoutSpec( DataLayoutSpec.partitioned(partCols.map(new Column(_)).asJava)) } val completeTblProps = tblPropertiesWithDomainMetadata(tableProperties, withDomainMetadataSupported) if (completeTblProps != null) { txnBuilder = txnBuilder.withTableProperties(completeTblProps.asJava) } if (clusteringColsOpt.nonEmpty) { txnBuilder = txnBuilder.withDataLayoutSpec( DataLayoutSpec.clustered(clusteringColsOpt.get.asJava)) } if (maxRetries >= 0) { txnBuilder = txnBuilder.withMaxRetries(maxRetries) } txnBuilder.build(engine) } private def tblPropertiesWithDomainMetadata( tableProperties: Map[String, String], withDomainMetadataSupported: Boolean): Map[String, String] = { if (tableProperties == null && !withDomainMetadataSupported) { null } else { val origTblProps = if (tableProperties != null) tableProperties else Map() val dmTblProps = if (withDomainMetadataSupported) { Map(TableFeatures.SET_TABLE_FEATURE_SUPPORTED_PREFIX + "domainMetadata" -> "supported") } else { Map() } (origTblProps ++ dmTblProps).toMap } } } ================================================ FILE: kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/utils/WriteUtils.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.defaults.utils import java.io.File import java.nio.file.{Files, Paths} import java.util.Collections.emptyMap import java.util.Optional import scala.collection.JavaConverters._ import scala.collection.immutable.{ListMap, Seq} import io.delta.golden.GoldenTableUtils.goldenTablePath import io.delta.kernel._ import io.delta.kernel.data.{ColumnarBatch, ColumnVector, FilteredColumnarBatch, Row} import io.delta.kernel.defaults.internal.data.DefaultColumnarBatch import io.delta.kernel.engine.Engine import io.delta.kernel.expressions.{Column, Literal} import io.delta.kernel.expressions.Literal.ofInt import io.delta.kernel.hook.PostCommitHook.PostCommitHookType import io.delta.kernel.internal._ import io.delta.kernel.internal.actions.{DomainMetadata, Metadata, Protocol, SingleAction} import io.delta.kernel.internal.fs.{Path => DeltaPath} import io.delta.kernel.internal.util.{Clock, FileNames} import io.delta.kernel.internal.util.SchemaUtils.casePreservingPartitionColNames import io.delta.kernel.internal.util.Utils.{singletonCloseableIterator, toCloseableIterator} import io.delta.kernel.statistics.DataFileStatistics import io.delta.kernel.types.IntegerType.INTEGER import io.delta.kernel.types.StructType import io.delta.kernel.utils.{CloseableIterable, CloseableIterator, DataFileStatus, FileStatus} import io.delta.kernel.utils.CloseableIterable.{emptyIterable, inMemoryIterable} import org.apache.spark.sql.delta.VersionNotFoundException import com.fasterxml.jackson.databind.ObjectMapper import org.apache.commons.io.FileUtils import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path /** Default write utilities that use the V1 transaction builders and legacy Table Snapshot APIs */ trait WriteUtils extends AbstractWriteUtils with TransactionBuilderV1Support with TestUtilsWithLegacyKernelAPIs /** * DO NOT MODIFY this trait -- this is just syntactic sugar to clearly indicate we are extending the * "default" WriteUtils which happens to use the legacy Kernel APIs */ trait WriteUtilsWithV1Builders extends WriteUtils /** * Write utilities that use the V2 transaction builders to create transactions and TableManager * snapshot APIs */ trait WriteUtilsWithV2Builders extends AbstractWriteUtils with TransactionBuilderV2Support with TestUtilsWithTableManagerAPIs /** * Common utility methods for write test suites. For now, this includes mostly concrete * implementations for the utilities. As we improve our test structure, we should move concrete * implementations out of this class (like we have done for [[TransactionBuilderSupport]]). For * example, `commitTransaction` could go into a [[TransactionCommitSupport]] trait since it is * overridden in child suites. */ trait AbstractWriteUtils extends TestUtils with TransactionBuilderSupport { val OBJ_MAPPER = new ObjectMapper() val testEngineInfo = "test-engine" /** Test table schemas and test */ lazy val testSchema = new StructType().add("id", INTEGER) lazy val dataBatches1 = generateData(testSchema, Seq.empty, Map.empty, 200, 3) lazy val dataBatches2 = generateData(testSchema, Seq.empty, Map.empty, 400, 5) lazy val seqOfUnpartitionedDataBatch1 = Seq(Map.empty[String, Literal] -> dataBatches1) lazy val seqOfUnpartitionedDataBatch2 = Seq(Map.empty[String, Literal] -> dataBatches2) val testPartitionColumns = Seq("part1", "part2") val testPartitionSchema = new StructType() .add("id", INTEGER) .add("part1", INTEGER) // partition column .add("part2", INTEGER) // partition column val dataPartitionBatches1 = generateData( testPartitionSchema, testPartitionColumns, Map("part1" -> ofInt(1), "part2" -> ofInt(2)), batchSize = 237, numBatches = 3) val dataPartitionBatches2 = generateData( testPartitionSchema, testPartitionColumns, Map("part1" -> ofInt(4), "part2" -> ofInt(5)), batchSize = 876, numBatches = 7) val testClusteringColumns = List(new Column("part1"), new Column("part2")) val dataClusteringBatches1 = generateData( testPartitionSchema, partitionCols = Seq.empty, partitionValues = Map.empty, batchSize = 200, numBatches = 3) val dataClusteringBatches2 = generateData( testPartitionSchema, partitionCols = Seq.empty, partitionValues = Map.empty, batchSize = 456, numBatches = 5) def verifyLastCheckpointMetadata(tablePath: String, checkpointAt: Long, expSize: Long): Unit = { val filePath = f"$tablePath/_delta_log/_last_checkpoint" val source = scala.io.Source.fromFile(filePath) val result = try source.getLines().mkString(",") finally source.close() assert(result === s"""{"version":$checkpointAt,"size":$expSize}""") } /** * Helper method to remove the delta files before the given version, to make sure the read is * using a checkpoint as base for state reconstruction. */ def deleteDeltaFilesBefore(tablePath: String, beforeVersion: Long): Unit = { Seq.range(0, beforeVersion).foreach { version => val filePath = new Path(f"$tablePath/_delta_log/$version%020d.json") new Path(tablePath).getFileSystem(new Configuration()).delete( filePath, false /* recursive */ ) } // try to query a version < beforeVersion val ex = intercept[VersionNotFoundException] { spark.read.format("delta").option("versionAsOf", beforeVersion - 1).load(tablePath) } assert(ex.getMessage().contains( s"Cannot time travel Delta table to version ${beforeVersion - 1}")) } def setCheckpointInterval(tablePath: String, interval: Int): Unit = { spark.sql(s"ALTER TABLE delta.`$tablePath` " + s"SET TBLPROPERTIES ('delta.checkpointInterval' = '$interval')") } def dataFileCount(tablePath: String): Int = { Files.walk(Paths.get(tablePath)).iterator().asScala .count(path => path.toString.endsWith(".parquet") && !path.toString.contains("_delta_log")) } def checkpointFilePath(tablePath: String, checkpointVersion: Long): String = { f"$tablePath/_delta_log/$checkpointVersion%020d.checkpoint.parquet" } def assertCheckpointExists(tablePath: String, atVersion: Long): Unit = { val cpPath = checkpointFilePath(tablePath, checkpointVersion = atVersion) assert(new File(cpPath).exists()) } def copyTable(goldenTableName: String, targetLocation: String): Unit = { val source = new File(goldenTablePath(goldenTableName)) val target = new File(targetLocation) FileUtils.copyDirectory(source, target) } def checkpointIfReady( engine: Engine, tablePath: String, result: TransactionCommitResult, expSize: Long): Unit = { result.getPostCommitHooks.forEach(hook => { if (hook.getType == PostCommitHookType.CHECKPOINT) { hook.threadSafeInvoke(engine) verifyLastCheckpointMetadata(tablePath, checkpointAt = result.getVersion, expSize) } }) } /** * Helper method to read the commit file of the given version and return the value at the given * ordinal if it is not null and the consumer returns a value, otherwise return null. */ def readCommitFile( engine: Engine, tablePath: String, version: Long, consumer: Row => Option[Any]): Option[Any] = { val table = Table.forPath(engine, tablePath) val logPath = new DeltaPath(table.getPath(engine), "_delta_log") val file = FileStatus.of(FileNames.deltaFile(logPath, version), 0, 0) val columnarBatches = engine.getJsonHandler.readJsonFiles( singletonCloseableIterator(file), SingleAction.FULL_SCHEMA, Optional.empty()) while (columnarBatches.hasNext) { val batch = columnarBatches.next val rows = batch.getRows while (rows.hasNext) { val row = rows.next val ret = consumer(row) if (ret.isDefined) { return ret } } } Option.empty } def getMetadata(engine: Engine, tablePath: String): Metadata = { getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath).getMetadata } def getProtocol(engine: Engine, tablePath: String): Protocol = { getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath).getProtocol } /** * Helper method to read the Metadata from the commit file of the given version if it is not * null, otherwise return null. * TODO: get rid of this and use getMetadata instead */ def getMetadataActionFromCommit( engine: Engine, table: Table, version: Long): Option[Row] = { readCommitFile( engine, table.getPath(engine), version, (row) => { val ord = row.getSchema.indexOf("metaData") if (!row.isNullAt(ord)) { Option(row.getStruct(ord)) } else { Option.empty } }).map { case metadata: Row => Some(metadata) }.getOrElse(Option.empty) } /** * Helper method to read the Protocol from the commit file of the given version if it is not * null, otherwise return null. * TODO: get rid of this and use getProtocol instead */ def getProtocolActionFromCommit(engine: Engine, tablePath: String, version: Long): Option[Row] = { readCommitFile( engine, tablePath, version, (row) => { val ord = row.getSchema.indexOf("protocol") if (!row.isNullAt(ord)) { Some(row.getStruct(ord)) } else { Option.empty } }).map { case protocol: Row => Some(protocol) }.getOrElse(Option.empty) } def generateData( schema: StructType, partitionCols: Seq[String], partitionValues: Map[String, Literal], batchSize: Int, numBatches: Int): Seq[FilteredColumnarBatch] = { val partitionValuesSchemaCase = casePreservingPartitionColNames(partitionCols.asJava, partitionValues.asJava) var batches = Seq.empty[ColumnarBatch] for (_ <- 0 until numBatches) { var vectors = Seq.empty[ColumnVector] schema.fields().forEach { field => val colType = field.getDataType val partValue = partitionValuesSchemaCase.get(field.getName) if (partValue != null) { // handle the partition column by inserting a vector with single value val vector = testSingleValueVector(colType, batchSize, partValue.getValue) vectors = vectors :+ vector } else { // handle the regular columns val vector = testColumnVector(batchSize, colType) vectors = vectors :+ vector } } batches = batches :+ new DefaultColumnarBatch(batchSize, schema, vectors.toArray) } batches.map(batch => new FilteredColumnarBatch(batch, Optional.empty())) } def stageData( state: Row, partitionValues: Map[String, Literal], data: Seq[FilteredColumnarBatch]) : CloseableIterator[Row] = { val physicalDataIter = Transaction.transformLogicalData( defaultEngine, state, toCloseableIterator(data.toIterator.asJava), partitionValues.asJava) val writeContext = Transaction.getWriteContext(defaultEngine, state, partitionValues.asJava) val writeResultIter = defaultEngine .getParquetHandler .writeParquetFiles( writeContext.getTargetDirectory, physicalDataIter, writeContext.getStatisticsColumns) Transaction.generateAppendActions(defaultEngine, state, writeResultIter, writeContext) } def createTxnWithDomainMetadatas( engine: Engine, tablePath: String, domainMetadatas: Seq[DomainMetadata], useInternalApi: Boolean = false, enableDomainMetadata: Boolean = true): Transaction = { val txn = if (domainMetadatas.nonEmpty && !useInternalApi) { getUpdateTxn(engine, tablePath, withDomainMetadataSupported = enableDomainMetadata) .asInstanceOf[TransactionImpl] } else { getUpdateTxn(engine, tablePath).asInstanceOf[TransactionImpl] } domainMetadatas.foreach { dm => if (dm.isRemoved) { if (useInternalApi) { txn.removeDomainMetadataInternal(dm.getDomain) } else { txn.removeDomainMetadata(dm.getDomain) } } else { if (useInternalApi) { txn.addDomainMetadataInternal(dm.getDomain, dm.getConfiguration) } else { txn.addDomainMetadata(dm.getDomain, dm.getConfiguration) } } } txn } def getAppendActions( txn: Transaction, data: Seq[(Map[String, Literal], Seq[FilteredColumnarBatch])]): CloseableIterable[Row] = { val txnState = txn.getTransactionState(defaultEngine) val actions = data.map { case (partValues, partData) => stageData(txnState, partValues, partData) } actions.reduceLeftOption(_ combine _) match { case Some(combinedActions) => inMemoryIterable(combinedActions) case None => emptyIterable[Row] } } def commitAppendData( engine: Engine = defaultEngine, txn: Transaction, data: Seq[(Map[String, Literal], Seq[FilteredColumnarBatch])]): TransactionCommitResult = { commitTransaction(txn, engine, getAppendActions(txn, data)) } /** Utility to create table, with no data */ def createEmptyTable( engine: Engine = defaultEngine, tablePath: String, schema: StructType, partCols: Seq[String] = null, clock: Clock = () => System.currentTimeMillis, tableProperties: Map[String, String] = null, clusteringColsOpt: Option[List[Column]] = None): TransactionCommitResult = { appendData( engine, tablePath, isNewTable = true, schema, partCols, data = Seq.empty, clock, tableProperties, clusteringColsOpt) } /** Update an existing table - metadata only changes (no data changes) */ def updateTableMetadata( engine: Engine = defaultEngine, tablePath: String, schema: StructType = null, // non-null schema means schema change clock: Clock = () => System.currentTimeMillis, tableProperties: Map[String, String] = null, clusteringColsOpt: Option[List[Column]] = None): TransactionCommitResult = { appendData( engine, tablePath, isNewTable = false, schema, Seq.empty, data = Seq.empty, clock, tableProperties, clusteringColsOpt) } def appendData( engine: Engine = defaultEngine, tablePath: String, isNewTable: Boolean = false, schema: StructType = null, partCols: Seq[String] = null, data: Seq[(Map[String, Literal], Seq[FilteredColumnarBatch])], clock: Clock = () => System.currentTimeMillis, tableProperties: Map[String, String] = null, clusteringColsOpt: Option[List[Column]] = None): TransactionCommitResult = { val txn = if (isNewTable) { getCreateTxn( engine, tablePath, schema, partCols, tableProperties, clock, clusteringColsOpt = clusteringColsOpt) } else { getUpdateTxn( engine, tablePath, schema, tableProperties, clock, clusteringColsOpt = clusteringColsOpt) } commitAppendData(engine, txn, data) } def assertMetadataProp( snapshot: SnapshotImpl, key: TableConfig[_ <: Any], expectedValue: Any): Unit = { assert(key.fromMetadata(snapshot.getMetadata) == expectedValue) } def assertHasNoMetadataProp(snapshot: SnapshotImpl, key: TableConfig[_ <: Any]): Unit = { assertMetadataProp(snapshot, key, Optional.empty()) } def assertHasWriterFeature(snapshot: SnapshotImpl, writerFeature: String): Unit = { assert(snapshot.getProtocol.getWriterFeatures.contains(writerFeature)) } def assertHasNoWriterFeature(snapshot: SnapshotImpl, writerFeature: String): Unit = { assert(!snapshot.getProtocol.getWriterFeatures.contains(writerFeature)) } def setTablePropAndVerify( engine: Engine, tablePath: String, isNewTable: Boolean = true, key: TableConfig[_ <: Any], value: String, expectedValue: Any, clock: Clock = () => System.currentTimeMillis): Unit = { val txn = if (isNewTable) { getCreateTxn( engine, tablePath, testSchema, tableProperties = Map(key.getKey -> value), clock = clock) } else { getUpdateTxn( engine, tablePath, schema = if (isNewTable) testSchema else null, tableProperties = Map(key.getKey -> value), clock = clock) } commitTransaction(txn, engine, emptyIterable()) val snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath) assertMetadataProp(snapshot, key, expectedValue) } protected def verifyWrittenContent( path: String, expSchema: StructType, expData: Seq[TestRow]): Unit = { val actSchema = tableSchema(path) assert(expSchema.isWriteCompatible(actSchema)) // verify data using Kernel reader checkTable(path, expData) // verify data using Spark reader. // Spark reads the timestamp partition columns in local timezone vs. Kernel reads in UTC. We // need to set the timezone to UTC before reading the data using Spark to make the tests pass withSparkTimeZone("UTC") { val resultSpark = spark.sql(s"SELECT * FROM delta.`$path`").collect().map(TestRow(_)) checkAnswer(resultSpark, expData) } } def verifyCommitInfo( tablePath: String, version: Long, partitionCols: Seq[String] = Seq.empty): Unit = { val expectedOperation = if (version == 0) Operation.CREATE_TABLE else Operation.WRITE val row = spark.sql(s"DESCRIBE HISTORY delta.`$tablePath`") .filter(s"version = $version") .select( "version", "operationParameters.partitionBy", "isBlindAppend", "engineInfo", "operation") .collect().last assert(row.getAs[Long]("version") === version) assert(row.getAs[Long]("partitionBy") === (if (partitionCols == null) null else OBJ_MAPPER.writeValueAsString(partitionCols.asJava))) // For now we've hardcoded isBlindAppend=false, once we support more precise setting of this // field we should update this check assert(!row.getAs[Boolean]("isBlindAppend")) assert(row.getAs[Seq[String]]("engineInfo") === "Kernel-" + Meta.KERNEL_VERSION + "/" + testEngineInfo) assert(row.getAs[String]("operation") === expectedOperation.getDescription) } def verifyCommitResult( result: TransactionCommitResult, expVersion: Long, expIsReadyForCheckpoint: Boolean): Unit = { assert(result.getVersion === expVersion) assertCheckpointReadiness(result, expIsReadyForCheckpoint) } // TODO: Change this to use the table metadata and protocol and // not rely on DESCRIBE which adds some properties based on the protocol. def verifyTableProperties( tablePath: String, expProperties: ListMap[String, Any], minReaderVersion: Int, minWriterVersion: Int): Unit = { val resultProperties = spark.sql(s"DESCRIBE EXTENDED delta.`$tablePath`") .filter("col_name = 'Table Properties'") .select("data_type") .collect().map(TestRow(_)) val builder = new StringBuilder("[") expProperties.foreach { case (key, value) => builder.append(s"$key=$value,") } builder.append(s"delta.minReaderVersion=$minReaderVersion,") builder.append(s"delta.minWriterVersion=$minWriterVersion") builder.append("]") checkAnswer(resultProperties, Seq(builder.toString()).map(TestRow(_))) } def assertCheckpointReadiness( txnResult: TransactionCommitResult, isReadyForCheckpoint: Boolean): Unit = { assert( txnResult.getPostCommitHooks .stream() .anyMatch(hook => hook.getType == PostCommitHookType.CHECKPOINT) === isReadyForCheckpoint) } def collectStatsFromAddFiles(engine: Engine, path: String): Seq[String] = { val snapshot = getTableManagerAdapter.getSnapshotAtLatest(engine, path) val scan = snapshot.getScanBuilder.build() val scanFiles = scan.asInstanceOf[ScanImpl].getScanFiles(engine, true) scanFiles.asScala.toList.flatMap { scanFile => scanFile.getRows.asScala.toList.flatMap { row => val add = row.getStruct(row.getSchema.indexOf("add")) val idx = add.getSchema.indexOf("stats") if (idx >= 0 && !add.isNullAt(idx)) List(add.getString(idx)) else Nil } } } /** * Commit transaction, all child suites should use this instead of txn.commit * directly and could override it for specific test cases (e.g. commit and write CRC). */ protected def commitTransaction( txn: Transaction, engine: Engine, dataActions: CloseableIterable[Row]): TransactionCommitResult = { txn.commit(engine, dataActions) } protected def generateDataFileStatus( tablePath: String, fileName: String, fileSize: Long = 1000, includeStats: Boolean = true): DataFileStatus = { val filePath = defaultEngine.getFileSystemClient.resolvePath(tablePath + "/" + fileName) new DataFileStatus( filePath, fileSize, 10, if (includeStats) { Optional.of(new DataFileStatistics( 100, emptyMap(), emptyMap(), emptyMap(), Optional.empty())) } else Optional.empty()) } protected def assertCommitResultHasClusteringCols( commitResult: TransactionCommitResult, expectedClusteringCols: Seq[Column]): Unit = { val actualClusteringCols = commitResult.getTransactionReport.getClusteringColumns.asScala assert( actualClusteringCols === expectedClusteringCols, s"Expected clustering columns: $expectedClusteringCols, but got: $actualClusteringCols") } /** * A very particular utility that is used in both InCommitTimestampSuite and * DeltaReplaceTableSuite. */ protected def createTableThenEnableIctAndVerify( engine: Engine, tablePath: String): SnapshotImpl = { // Create table without ICT. Note that this does not add ICT enablement tracking properties. val txn1 = getCreateTxn(engine, tablePath, testSchema) commitTransaction(txn1, engine, emptyIterable()) // Enable ICT. This should add enablement tracking properties. setTablePropAndVerify( engine = engine, tablePath = tablePath, isNewTable = false, key = TableConfig.IN_COMMIT_TIMESTAMPS_ENABLED, value = "true", expectedValue = true) val snapshotV1 = getTableManagerAdapter.getSnapshotAtLatest(engine, tablePath) // Verify enablement properties are present assertMetadataProp(snapshotV1, TableConfig.IN_COMMIT_TIMESTAMPS_ENABLED, true) assertMetadataProp( snapshotV1, TableConfig.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP, Optional.of(snapshotV1.getTimestamp(engine))) assertMetadataProp( snapshotV1, TableConfig.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION, Optional.of(1L)) snapshotV1 } } ================================================ FILE: kernel/project/plugins.sbt ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ addSbtPlugin("com.github.sbt" % "sbt-release" % "1.1.0") addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.1.2") addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0") addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.3") addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "1.0.1") addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.15") addSbtPlugin("com.etsy" % "sbt-checkstyle-plugin" % "3.1.1") // By default, sbt-checkstyle-plugin uses checkstyle version 6.15, but we should set it to use the // same version as Spark dependencyOverrides += "com.puppycrawl.tools" % "checkstyle" % "8.43" ================================================ FILE: kernel/scalastyle-config.xml ================================================ Scalastyle standard configuration true true ARROW, EQUALS, ELSE, TRY, CATCH, FINALLY, LARROW, RARROW ARROW, EQUALS, COMMA, COLON, IF, ELSE, DO, WHILE, FOR, MATCH, TRY, CATCH, FINALLY, LARROW, RARROW ^FunSuite[A-Za-z]*$ Tests must extend org.apache.spark.SparkFunSuite instead. ^println$ spark(.sqlContext)?.sparkContext.hadoopConfiguration sessionState.newHadoopConf @VisibleForTesting Runtime\.getRuntime\.addShutdownHook mutable\.SynchronizedBuffer Class\.forName Await\.result Await\.ready (\.toUpperCase|\.toLowerCase)(?!(\(|\(Locale.ROOT\))) typed[lL]it spark(Session)?.implicits._ throw new \w+Error\( count\(" JavaConversions Instead of importing implicits in scala.collection.JavaConversions._, import scala.collection.JavaConverters._ and use .asScala / .asJava methods org\.apache\.commons\.lang\. Use Commons Lang 3 classes (package org.apache.commons.lang3.*) instead of Commons Lang 2 (package org.apache.commons.lang.*) extractOpt Use jsonOption(x).map(.extract[T]) instead of .extractOpt[T], as the latter is slower. COMMA \)\{ (?m)^(\s*)/[*][*].*$(\r|)\n^\1 [*] Use Javadoc style indentation for multiline comments case[^\n>]*=>\s*\{ Omit braces in case clauses. 800> 30 10 50 -1,0,1,2,3 ================================================ FILE: kernel/unitycatalog/src/main/java/io/delta/kernel/unitycatalog/UCCatalogManagedClient.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static io.delta.kernel.unitycatalog.utils.OperationTimer.timeUncheckedOperation; import io.delta.kernel.CommitRange; import io.delta.kernel.CommitRangeBuilder; import io.delta.kernel.Snapshot; import io.delta.kernel.SnapshotBuilder; import io.delta.kernel.TableManager; import io.delta.kernel.annotation.Experimental; import io.delta.kernel.commit.Committer; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.annotation.VisibleForTesting; import io.delta.kernel.internal.files.ParsedCatalogCommitData; import io.delta.kernel.internal.files.ParsedLogData; import io.delta.kernel.internal.lang.Lazy; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.transaction.CreateTableTransactionBuilder; import io.delta.kernel.types.StructType; import io.delta.kernel.unitycatalog.metrics.UcLoadSnapshotTelemetry; import io.delta.storage.commit.Commit; import io.delta.storage.commit.GetCommitsResponse; import io.delta.storage.commit.uccommitcoordinator.UCClient; import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorException; import java.io.IOException; import java.io.UncheckedIOException; import java.util.*; import java.util.function.BiConsumer; import java.util.stream.Collectors; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Client for interacting with Unity Catalog (UC) catalog-managed Delta tables. * * @see UCClient * @see Snapshot */ @Experimental public class UCCatalogManagedClient { private static final Logger logger = LoggerFactory.getLogger(UCCatalogManagedClient.class); public static final String UC_PROPERTY_NAMESPACE_PREFIX = "io.unitycatalog."; /** Key for identifying Unity Catalog table ID. */ public static final String UC_TABLE_ID_KEY = UC_PROPERTY_NAMESPACE_PREFIX + "tableId"; protected final UCClient ucClient; public UCCatalogManagedClient(UCClient ucClient) { this.ucClient = Objects.requireNonNull(ucClient, "ucClient is null"); } // TODO: [delta-io/delta#4817] loadSnapshot API that takes in a UC TableInfo object ///////////////// // Public APIs // ///////////////// /** * Loads a Kernel {@link Snapshot}. If no version is specified, the latest version of the table is * loaded. * * @param engine The Delta Kernel {@link Engine} to use for loading the table. * @param ucTableId The Unity Catalog table ID, which is a unique identifier for the table in UC. * @param tablePath The path to the Delta table in the underlying storage system. * @param versionOpt The optional version to time-travel to when loading the table. This must be * mutually exclusive with timestampOpt. * @param timestampOpt The optional timestamp to time-travel to when loading the table. This must * be mutually exclusive with versionOpt. * @throws IllegalArgumentException if a negative version or timestamp is provided * @throws IllegalArgumentException if both versionOpt and timestampOpt are defined */ public Snapshot loadSnapshot( Engine engine, String ucTableId, String tablePath, Optional versionOpt, Optional timestampOpt) { Objects.requireNonNull(engine, "engine is null"); Objects.requireNonNull(ucTableId, "ucTableId is null"); Objects.requireNonNull(tablePath, "tablePath is null"); Objects.requireNonNull(versionOpt, "versionOpt is null"); Objects.requireNonNull(timestampOpt, "timestampOpt is null"); versionOpt.ifPresent(version -> checkArgument(version >= 0, "version must be non-negative")); checkArgument( !timestampOpt.isPresent() || !versionOpt.isPresent(), "cannot provide both timestamp and version"); logger.info( "[{}] Loading Snapshot at {}", ucTableId, getVersionOrTimestampString(versionOpt, timestampOpt)); final UcLoadSnapshotTelemetry telemetry = new UcLoadSnapshotTelemetry(ucTableId, tablePath, versionOpt, timestampOpt); final UcLoadSnapshotTelemetry.MetricsCollector metricsCollector = telemetry.getMetricsCollector(); try { final Snapshot result = metricsCollector.totalSnapshotLoadTimer.timeChecked( () -> { final GetCommitsResponse response = metricsCollector.getCommitsTimer.timeChecked( () -> getRatifiedCommitsFromUC(ucTableId, tablePath, versionOpt)); metricsCollector.setNumCatalogCommits(response.getCommits().size()); final long maxUcTableVersion = response.getLatestTableVersion(); versionOpt.ifPresent( version -> validateTimeTravelVersionNotPastMax(ucTableId, version, maxUcTableVersion)); final List logData = getSortedKernelParsedDeltaDataFromRatifiedCommits( ucTableId, response.getCommits()); return metricsCollector.kernelSnapshotBuildTimer.timeChecked( () -> { SnapshotBuilder snapshotBuilder = TableManager.loadSnapshot(tablePath); if (versionOpt.isPresent()) { snapshotBuilder = snapshotBuilder.atVersion(versionOpt.get()); } if (timestampOpt.isPresent()) { // If timestampOpt is present, we know versionOpt is not present. This means // logData was not requested with an endVersion and thus it can be re-used // to load the latest snapshot Snapshot latestSnapshot = metricsCollector.loadLatestSnapshotForTimestampTimeTravelTimer .timeChecked( () -> loadLatestSnapshotForTimestampResolution( engine, ucTableId, tablePath, logData, maxUcTableVersion)); snapshotBuilder = snapshotBuilder.atTimestamp(timestampOpt.get(), latestSnapshot); } Snapshot snapshot = snapshotBuilder .withCommitter(createUCCommitter(ucClient, ucTableId, tablePath)) .withLogData(logData) .withMaxCatalogVersion(maxUcTableVersion) .build(engine); metricsCollector.setResolvedSnapshotVersion(snapshot.getVersion()); return snapshot; }); }); final UcLoadSnapshotTelemetry.Report successReport = telemetry.createSuccessReport(); engine.getMetricsReporters().forEach(r -> r.report(successReport)); return result; } catch (Exception e) { final UcLoadSnapshotTelemetry.Report failureReport = telemetry.createFailureReport(e); engine.getMetricsReporters().forEach(r -> r.report(failureReport)); throw e; } } /** * Builds a create table transaction for a Unity Catalog managed Delta table. * *

Configures the transaction with a {@link UCCatalogManagedCommitter} and required table * properties for catalog-managed table enablement. * *

This assumes the table is being created in a staging location as per UC semantics. Once this * transaction is built and committed, creating 000.json, you must call {@code * TablesApi::createTable} to inform Unity Catalog of the successful table creation. * * @param ucTableId The Unity Catalog table ID. * @param tablePath The staging path to the Delta table. * @param schema The table schema. * @param engineInfo Information about the creating engine. * @return A {@link CreateTableTransactionBuilder} configured for UC managed tables. */ public CreateTableTransactionBuilder buildCreateTableTransaction( String ucTableId, String tablePath, StructType schema, String engineInfo) { Objects.requireNonNull(ucTableId, "ucTableId is null"); Objects.requireNonNull(tablePath, "tablePath is null"); Objects.requireNonNull(schema, "schema is null"); Objects.requireNonNull(engineInfo, "engineInfo is null"); return TableManager.buildCreateTableTransaction(tablePath, schema, engineInfo) .withCommitter(createUCCommitter(ucClient, ucTableId, tablePath)) .withTableProperties(getRequiredTablePropertiesForCreate(ucTableId)); } /** * Loads a Kernel {@link CommitRange} for the provided boundaries. If no end boundary is provided, * defaults to the latest version. * *

A start boundary is required and must be specified using either {@code startVersionOpt} or * {@code startTimestampOpt}. These parameters are mutually exclusive and at least one must be * provided. * * @param engine The Delta Kernel {@link Engine} to use for loading the table. * @param ucTableId The Unity Catalog table ID, which is a unique identifier for the table in UC. * @param tablePath The path to the Delta table in the underlying storage system. * @param startVersionOpt The optional start version boundary. This must be mutually exclusive * with startTimestampOpt. Either this or startTimestampOpt must be provided. * @param startTimestampOpt The optional start timestamp boundary. This must be mutually exclusive * with startVersionOpt. Either this or startVersionOpt must be provided. * @param endVersionOpt The optional end version boundary. This must be mutually exclusive with * endTimestampOpt. * @param endTimestampOpt The optional end timestamp boundary. This must be mutually exclusive * with endVersionOpt. * @throws IllegalArgumentException if neither startVersionOpt nor startTimestampOpt is provided * @throws IllegalArgumentException if both startVersionOpt and startTimestampOpt are defined * @throws IllegalArgumentException if both endVersionOpt and endTimestampOpt are defined * @throws IllegalArgumentException if either startVersionOpt or endVersionOpt is provided and is * greater than the latest ratified version from UC */ public CommitRange loadCommitRange( Engine engine, String ucTableId, String tablePath, Optional startVersionOpt, Optional startTimestampOpt, Optional endVersionOpt, Optional endTimestampOpt) { Objects.requireNonNull(engine, "engine is null"); Objects.requireNonNull(ucTableId, "ucTableId is null"); Objects.requireNonNull(tablePath, "tablePath is null"); Objects.requireNonNull(startVersionOpt, "startVersionOpt is null"); Objects.requireNonNull(startTimestampOpt, "startTimestampOpt is null"); Objects.requireNonNull(endVersionOpt, "endVersionOpt is null"); Objects.requireNonNull(endTimestampOpt, "endTimestampOpt is null"); checkArgument( !startVersionOpt.isPresent() || !startTimestampOpt.isPresent(), "Cannot provide both a start timestamp and start version"); checkArgument( !endVersionOpt.isPresent() || !endTimestampOpt.isPresent(), "Cannot provide both an end timestamp and start version"); checkArgument( startVersionOpt.isPresent() || startTimestampOpt.isPresent(), "Must provide either a start timestamp or start version"); if (startVersionOpt.isPresent() && endVersionOpt.isPresent()) { checkArgument( startVersionOpt.get() <= endVersionOpt.get(), "Cannot provide a start version greater than the end version"); } if (startTimestampOpt.isPresent() && endTimestampOpt.isPresent()) { checkArgument( startTimestampOpt.get() <= endTimestampOpt.get(), "Cannot provide a start timestamp greater than the end timestamp"); } logger.info( "[{}] Loading CommitRange for {}", ucTableId, getCommitRangeBoundariesString( startVersionOpt, startTimestampOpt, endVersionOpt, endTimestampOpt)); // If we have a timestamp-based boundary we need to build the latest snapshot, don't provide // an endVersion Optional endVersionOptForCommitQuery = endVersionOpt.filter(v -> !startTimestampOpt.isPresent()); final GetCommitsResponse response = getRatifiedCommitsFromUC(ucTableId, tablePath, endVersionOptForCommitQuery); final long ucTableVersion = response.getLatestTableVersion(); validateVersionBoundariesExist(ucTableId, startVersionOpt, endVersionOpt, ucTableVersion); final List logData = getSortedKernelParsedDeltaDataFromRatifiedCommits(ucTableId, response.getCommits()); final Lazy latestSnapshot = new Lazy<>( () -> loadLatestSnapshotForTimestampResolution( engine, ucTableId, tablePath, logData, ucTableVersion)); return timeUncheckedOperation( logger, "TableManager.loadCommitRange", ucTableId, () -> { // Determine the start boundary (required - validated above) CommitRangeBuilder.CommitBoundary startBoundary; if (startVersionOpt.isPresent()) { startBoundary = CommitRangeBuilder.CommitBoundary.atVersion(startVersionOpt.get()); } else { // startTimestampOpt must be present due to validation above startBoundary = CommitRangeBuilder.CommitBoundary.atTimestamp( startTimestampOpt.get(), latestSnapshot.get()); } CommitRangeBuilder commitRangeBuilder = TableManager.loadCommitRange(tablePath, startBoundary) .withMaxCatalogVersion(ucTableVersion); if (endVersionOpt.isPresent()) { commitRangeBuilder = commitRangeBuilder.withEndBoundary( CommitRangeBuilder.CommitBoundary.atVersion(endVersionOpt.get())); } if (endTimestampOpt.isPresent()) { commitRangeBuilder = commitRangeBuilder.withEndBoundary( CommitRangeBuilder.CommitBoundary.atTimestamp( endTimestampOpt.get(), latestSnapshot.get())); } return commitRangeBuilder.withLogData(logData).build(engine); }); } ///////////////////////////////////////// // Protected Methods for Extensibility // ///////////////////////////////////////// /** * Creates a UC committer instance for the specified table. * *

This method allows subclasses to provide custom committer implementations for specialized * use cases. */ protected Committer createUCCommitter(UCClient ucClient, String ucTableId, String tablePath) { return new UCCatalogManagedCommitter(ucClient, ucTableId, tablePath); } //////////////////// // Helper Methods // //////////////////// private String getVersionString(Optional versionOpt) { return versionOpt.map(String::valueOf).orElse("latest"); } private String getVersionOrTimestampString( Optional versionOpt, Optional timestampOpt) { if (versionOpt.isPresent()) { return "version=" + versionOpt.get(); } else if (timestampOpt.isPresent()) { return "timestamp=" + timestampOpt.get(); } else { return "latest"; } } private String getCommitRangeBoundariesString( Optional startVersionOpt, Optional startTimestampOpt, Optional endVersionOpt, Optional endTimestampOpt) { String startBound; if (startVersionOpt.isPresent()) { startBound = startVersionOpt.get() + "(version)"; } else if (startTimestampOpt.isPresent()) { startBound = startTimestampOpt.get() + "(timestamp)"; } else { startBound = "0(default)"; } String endBound; if (endVersionOpt.isPresent()) { endBound = endVersionOpt.get() + "(version)"; } else if (endTimestampOpt.isPresent()) { endBound = endTimestampOpt.get() + "(timestamp)"; } else { endBound = "latestVersion(default)"; } return String.format("startBoundary=%s and endBoundary=%s", startBound, endBound); } private GetCommitsResponse getRatifiedCommitsFromUC( String ucTableId, String tablePath, Optional versionOpt) { logger.info( "[{}] Invoking the UCClient to get ratified commits at version {}", ucTableId, getVersionString(versionOpt)); // TODO: We can remove timeUncheckedOperation when the commitRange code integrates with metrics final GetCommitsResponse response = timeUncheckedOperation( logger, "UCClient.getCommits", ucTableId, () -> { try { return ucClient.getCommits( ucTableId, new Path(tablePath).toUri(), Optional.empty() /* startVersion */, versionOpt /* endVersion */); } catch (IOException ex) { throw new UncheckedIOException(ex); } catch (UCCommitCoordinatorException ex) { throw new RuntimeException(ex); } }); logger.info( "[{}] Number of ratified commits: {}, Max ratified version in UC: {}", ucTableId, response.getCommits().size(), response.getLatestTableVersion()); return response; } private void validateTimeTravelVersionNotPastMax( String ucTableId, long tableVersionToLoad, long maxRatifiedVersion) { if (tableVersionToLoad > maxRatifiedVersion) { throw new IllegalArgumentException( String.format( "[%s] Cannot load table version %s as the latest version ratified by UC is %s", ucTableId, tableVersionToLoad, maxRatifiedVersion)); } } private void validateVersionBoundariesExist( String ucTableId, Optional startVersion, Optional endVersion, long maxRatifiedVersion) { BiConsumer validateVersion = (version, type) -> { if (version > maxRatifiedVersion) { throw new IllegalArgumentException( String.format( "[%s] Cannot load commit range with %s version %d as the latest version " + "ratified by UC is %d", ucTableId, type, version, maxRatifiedVersion)); } }; startVersion.ifPresent(v -> validateVersion.accept(v, "start")); endVersion.ifPresent(v -> validateVersion.accept(v, "end")); } private Map getRequiredTablePropertiesForCreate(String ucTableId) { final Map requiredProperties = new HashMap<>(); requiredProperties.put( TableFeatures.CATALOG_MANAGED_RW_FEATURE.getTableFeatureSupportKey(), TableFeatures.SET_TABLE_FEATURE_SUPPORTED_VALUE); requiredProperties.put( TableFeatures.VACUUM_PROTOCOL_CHECK_RW_FEATURE.getTableFeatureSupportKey(), TableFeatures.SET_TABLE_FEATURE_SUPPORTED_VALUE); requiredProperties.put(UC_TABLE_ID_KEY, ucTableId); return requiredProperties; } /** * Converts a list of ratified commits into a sorted list of {@link ParsedLogData} for use in * loading a Delta table. */ @VisibleForTesting static List getSortedKernelParsedDeltaDataFromRatifiedCommits( String ucTableId, List commits) { final List result = timeUncheckedOperation( logger, "Sort and convert UC ratified commits into Kernel ParsedLogData", ucTableId, () -> commits.stream() .sorted(Comparator.comparingLong(Commit::getVersion)) .map( commit -> ParsedCatalogCommitData.forFileStatus( hadoopFileStatusToKernelFileStatus(commit.getFileStatus()))) .collect(Collectors.toList())); logger.debug("[{}] Created ParsedLogData from ratified commits: {}", ucTableId, result); return result; } private static io.delta.kernel.utils.FileStatus hadoopFileStatusToKernelFileStatus( org.apache.hadoop.fs.FileStatus hadoopFS) { return io.delta.kernel.utils.FileStatus.of( hadoopFS.getPath().toString(), hadoopFS.getLen(), hadoopFS.getModificationTime()); } /** * Helper method to load the latest snapshot and time the operation. This is used to load the * latest snapshot for timestamp resolution queries. Reuses existing logData that has already been * queried from the catalog (it is required that this includes the latest commits from the catalog * and were not queried with an endVersion). */ private Snapshot loadLatestSnapshotForTimestampResolution( Engine engine, String ucTableId, String tablePath, List logData, long ucTableVersion) { // TODO: We can remove timeUncheckedOperation when the commitRange code integrates with metrics return timeUncheckedOperation( logger, "TableManager.loadSnapshot at latest for time-travel query", ucTableId, () -> TableManager.loadSnapshot(tablePath) .withCommitter(new UCCatalogManagedCommitter(ucClient, ucTableId, tablePath)) .withLogData(logData) .withMaxCatalogVersion(ucTableVersion) .build(engine)); } } ================================================ FILE: kernel/unitycatalog/src/main/java/io/delta/kernel/unitycatalog/UCCatalogManagedCommitter.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static io.delta.kernel.internal.util.Preconditions.checkState; import static io.delta.kernel.unitycatalog.UCCatalogManagedClient.UC_TABLE_ID_KEY; import static java.util.Objects.requireNonNull; import io.delta.kernel.commit.*; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.annotation.VisibleForTesting; import io.delta.kernel.internal.files.ParsedCatalogCommitData; import io.delta.kernel.internal.files.ParsedPublishedDeltaData; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.unitycatalog.adapters.MetadataAdapter; import io.delta.kernel.unitycatalog.adapters.ProtocolAdapter; import io.delta.kernel.unitycatalog.adapters.UniformAdapter; import io.delta.kernel.unitycatalog.metrics.UcCommitTelemetry; import io.delta.kernel.unitycatalog.metrics.UcPublishTelemetry; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; import io.delta.storage.commit.Commit; import io.delta.storage.commit.uccommitcoordinator.UCClient; import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorException; import io.delta.storage.commit.uniform.UniformMetadata; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * An implementation of {@link Committer} that handles commits to Delta tables managed by Unity * Catalog. That is, these Delta tables must have the catalogManaged table feature supported. */ public class UCCatalogManagedCommitter implements Committer, CatalogCommitter { private static final Logger logger = LoggerFactory.getLogger(UCCatalogManagedCommitter.class); protected final UCClient ucClient; protected final String ucTableId; protected final Path tablePath; /** * Creates a new UCCatalogManagedCommitter for the specified Unity Catalog-managed Delta table. * * @param ucClient the Unity Catalog client to use for commit operations * @param ucTableId the unique Unity Catalog table identifier * @param tablePath the path to the Delta table in the underlying storage system */ public UCCatalogManagedCommitter(UCClient ucClient, String ucTableId, String tablePath) { this.ucClient = requireNonNull(ucClient, "ucClient is null"); this.ucTableId = requireNonNull(ucTableId, "ucTableId is null"); this.tablePath = new Path(requireNonNull(tablePath, "tablePath is null")); } ///////////////// // Public APIs // ///////////////// @Override public CommitResponse commit( Engine engine, CloseableIterator finalizedActions, CommitMetadata commitMetadata) throws CommitFailedException { requireNonNull(engine, "engine is null"); requireNonNull(finalizedActions, "finalizedActions is null"); requireNonNull(commitMetadata, "commitMetadata is null"); validateLogPathBelongsToThisUcTable(commitMetadata); final UcCommitTelemetry telemetry = new UcCommitTelemetry(ucTableId, tablePath.toString(), commitMetadata); final UcCommitTelemetry.MetricsCollector metricsCollector = telemetry.getMetricsCollector(); try { final CommitResponse response = metricsCollector.totalCommitTimer.timeChecked( () -> { final CommitMetadata.CommitType commitType = commitMetadata.getCommitType(); if (commitType == CommitMetadata.CommitType.CATALOG_CREATE) { return createImpl(engine, finalizedActions, commitMetadata, metricsCollector); } if (commitType == CommitMetadata.CommitType.CATALOG_WRITE) { return writeImpl(engine, finalizedActions, commitMetadata, metricsCollector); } throw new UnsupportedOperationException("Unsupported commit type: " + commitType); }); final UcCommitTelemetry.Report successfulReport = telemetry.createSuccessReport(); engine.getMetricsReporters().forEach(r -> r.report(successfulReport)); return response; } catch (CommitFailedException | RuntimeException e) { final UcCommitTelemetry.Report failureReport = telemetry.createFailureReport(e); engine.getMetricsReporters().forEach(r -> r.report(failureReport)); throw e; } } @Override public void publish(Engine engine, PublishMetadata publishMetadata) throws PublishFailedException { requireNonNull(engine, "engine is null"); requireNonNull(publishMetadata, "publishMetadata is null"); final List catalogCommits = publishMetadata.getAscendingCatalogCommits(); if (catalogCommits.isEmpty()) { return; } final String logPath = publishMetadata.getLogPath(); final long snapshotVersion = publishMetadata.getSnapshotVersion(); logger.info( "[{}] Publishing {} catalog commits up to version {}", ucTableId, catalogCommits.size(), snapshotVersion); final UcPublishTelemetry telemetry = new UcPublishTelemetry( ucTableId, tablePath.toString(), snapshotVersion, catalogCommits.size()); final UcPublishTelemetry.MetricsCollector metricsCollector = telemetry.getMetricsCollector(); try { metricsCollector.totalPublishTimer.time( () -> { for (ParsedCatalogCommitData catalogCommit : catalogCommits) { publishSingleCommit(engine, catalogCommit, logPath, metricsCollector); } return null; }); logger.info( "[{}] Successfully published all catalog commits up to version {}. {} were published by " + "this process, {} were already published by another process.", ucTableId, snapshotVersion, metricsCollector.getCommitsPublished(), metricsCollector.getCommitsAlreadyPublished()); final UcPublishTelemetry.Report successfulReport = telemetry.createSuccessReport(); engine.getMetricsReporters().forEach(r -> r.report(successfulReport)); } catch (RuntimeException e) { final UcPublishTelemetry.Report failureReport = telemetry.createFailureReport(e); engine.getMetricsReporters().forEach(r -> r.report(failureReport)); throw e; } } @Override public Map getRequiredTableProperties() { return Collections.singletonMap(UC_TABLE_ID_KEY, ucTableId); } /////////////////////////// // Commit helper methods // /////////////////////////// /** * Handles CATALOG_CREATE by writing the published delta file for version 0. * *

Note that this assumes that the table is being created within a staging location, and that * the Connector will post-commit inform UC of this 000.json file. */ // TODO: [delta-io/delta#5118] If UC changes CREATE semantics, update logic here. private CommitResponse createImpl( Engine engine, CloseableIterator finalizedActions, CommitMetadata commitMetadata, UcCommitTelemetry.MetricsCollector metricsCollector) throws CommitFailedException { checkArgument( commitMetadata.getVersion() == 0, "Expected version 0, but got %s", commitMetadata.getVersion()); final FileStatus kernelPublishedDeltaFileStatus = writeDeltaFile( engine, finalizedActions, commitMetadata.getPublishedDeltaFilePath(), metricsCollector); return new CommitResponse( ParsedPublishedDeltaData.forFileStatus(kernelPublishedDeltaFileStatus)); } /** * Handles CATALOG_WRITE by writing the staged commit file and then committing (e.g. REST or RPC * call) to UC server. */ private CommitResponse writeImpl( Engine engine, CloseableIterator finalizedActions, CommitMetadata commitMetadata, UcCommitTelemetry.MetricsCollector commitMetricsCollector) throws CommitFailedException { checkArgument( commitMetadata.getVersion() > 0, "Can only write staged commit files for versions > 0"); final FileStatus kernelStagedCommitFileStatus = writeDeltaFile( engine, finalizedActions, commitMetadata.generateNewStagedCommitFilePath(), commitMetricsCollector); commitToUC(commitMetadata, kernelStagedCommitFileStatus, commitMetricsCollector); return new CommitResponse(ParsedCatalogCommitData.forFileStatus(kernelStagedCommitFileStatus)); } //////////////////////////// // Publish helper methods // //////////////////////////// private void publishSingleCommit( Engine engine, ParsedCatalogCommitData catalogCommit, String logPath, UcPublishTelemetry.MetricsCollector publishMetricsCollector) throws PublishFailedException { final long commitVersion = catalogCommit.getVersion(); if (catalogCommit.isInline()) { throw new UnsupportedOperationException( "Publishing inline catalog commits is not yet supported"); } final String sourcePath = catalogCommit.getFileStatus().getPath(); final String targetPath = FileNames.deltaFile(logPath, commitVersion); try { logger.info("[{}] Publishing catalog commit: {} -> {}", ucTableId, sourcePath, targetPath); // Copy the staged commit file to the published delta file location. We use overwrite=false to // ensure PUT-if-absent semantics, since UC catalogManaged tables expect immutability of // published delta files (e.g. never want the e-tag to change). engine .getFileSystemClient() .copyFileAtomically(sourcePath, targetPath, false /* overwrite */); logger.info("[{}] Successfully published version {}", ucTableId, commitVersion); publishMetricsCollector.incrementCommitsPublished(); } catch (java.nio.file.FileAlreadyExistsException e) { // File already exists - this is okay, it means this version was already published logger.info("[{}] Version {} already published", ucTableId, commitVersion); publishMetricsCollector.incrementCommitsAlreadyPublished(); } catch (Exception ex) { throw new PublishFailedException( String.format( "Failed to publish version %d from %s to %s: %s", commitVersion, sourcePath, targetPath, ex.getMessage()), ex); } } ///////////////////////////////////////// // Protected Methods for Extensibility // ///////////////////////////////////////// /** * Generates the metadata payload for UC commit operations. * *

This method allows subclasses to customize or enhance metadata before sending to Unity * Catalog. */ protected Optional generateMetadataPayloadOpt(CommitMetadata commitMetadata) { return commitMetadata.getNewMetadataOpt(); } //////////////////// // Helper methods // //////////////////// private String normalize(Path path) { return path.toUri().normalize().toString(); } private void validateLogPathBelongsToThisUcTable(CommitMetadata cm) { final String expectedDeltaLogPathNormalized = normalize(new Path(tablePath, "_delta_log")); final String providedDeltaLogPathNormalized = normalize(new Path(cm.getDeltaLogDirPath())); checkArgument( expectedDeltaLogPathNormalized.equals(providedDeltaLogPathNormalized), "Delta log path '%s' does not match expected '%s'", expectedDeltaLogPathNormalized, providedDeltaLogPathNormalized); } /** * Writes either a published delta file (for CREATE) or a staged commit file (for WRITE). * *

For both cases, writes using {@code overwrite=true} since: * *

    *
  • For CREATE, we can assume we are the only writer writing to the staging location *
  • For WRITE, we are writing to a UUID commit file *
*/ private FileStatus writeDeltaFile( Engine engine, CloseableIterator finalizedActions, String filePath, UcCommitTelemetry.MetricsCollector metricsCollector) throws CommitFailedException { return metricsCollector.writeCommitFileTimer.timeChecked( () -> { try { logger.info("[{}] Writing file: {}", ucTableId, filePath); // Note: the engine is responsible for closing the actions iterator once it has been // fully consumed. engine .getJsonHandler() .writeJsonFileAtomically(filePath, finalizedActions, true /* overwrite */); return engine.getFileSystemClient().getFileStatus(filePath); } catch (IOException ex) { // Note that as per the JsonHandler::writeJsonFileAtomically API contract with // overwrite=true, FileAlreadyExistsException should not be possible here. throw new CommitFailedException( true /* retryable */, false /* conflict */, "Failed to write delta file due to: " + ex.getMessage(), ex); } }); } private void commitToUC( CommitMetadata commitMetadata, FileStatus kernelStagedCommitFileStatus, UcCommitTelemetry.MetricsCollector metricsCollector) throws CommitFailedException { metricsCollector.commitToUcServerTimer.timeChecked( () -> { logger.info( "[{}] Committing staged commit file to UC: {}", ucTableId, kernelStagedCommitFileStatus.getPath()); final CommitMetadata.CommitType commitType = commitMetadata.getCommitType(); // commitToUc is only for normal catalog WRITES, not for CREATE, or UPGRADE, or // DOWNGRADE, or anything filesystem related. checkState( commitType == CommitMetadata.CommitType.CATALOG_WRITE, "Only supported commit type is CATALOG_WRITE, but got: " + commitType); // Extract and validate Uniform metadata if present Optional uniformMetadataOpt = UniformAdapter.fromCommitterProperties(commitMetadata.getCommitterProperties().get()); // Validate that convertedDeltaVersion matches the current commit version uniformMetadataOpt.ifPresent( uniformMetadata -> { uniformMetadata .getIcebergMetadata() .ifPresent( icebergMetadata -> { long convertedVersion = icebergMetadata.getConvertedDeltaVersion(); long commitVersion = commitMetadata.getVersion(); checkState( convertedVersion == commitVersion, String.format( "Uniform convertedDeltaVersion (%d) must match " + "commit version (%d)", convertedVersion, commitVersion)); }); }); try { ucClient.commit( ucTableId, tablePath.toUri(), Optional.of(getUcCommitPayload(commitMetadata, kernelStagedCommitFileStatus)), commitMetadata.getMaxKnownPublishedDeltaVersion(), false /* isDisown */, generateMetadataPayloadOpt(commitMetadata).map(MetadataAdapter::new), commitMetadata.getNewProtocolOpt().map(ProtocolAdapter::new), uniformMetadataOpt); return null; } catch (io.delta.storage.commit.CommitFailedException cfe) { throw storageCFEtoKernelCFE(cfe); } catch (IOException ex) { throw new CommitFailedException( true /* retryable */, false /* conflict */, ex.getMessage(), ex); } catch (UCCommitCoordinatorException ucce) { // For now, this catches all UC exceptions such as: // - CommitLimitReachedException -> TODO: publish in this case // - InvalidTargetTableException // - UpgradeNotAllowedException // We can add specific catch statements for these exceptions if needed in the future. throw new CommitFailedException( false /* retryable */, false /* conflict */, ucce.getMessage(), ucce); } }); } private Commit getUcCommitPayload( CommitMetadata commitMetadata, FileStatus kernelStagedCommitFileStatus) { return new Commit( commitMetadata.getVersion(), kernelFileStatusToHadoopFileStatus(kernelStagedCommitFileStatus), // commitMetadata validates that the ICT is present if writing to a catalogManaged table commitMetadata.getCommitInfo().getInCommitTimestamp().get()); } @VisibleForTesting public static org.apache.hadoop.fs.FileStatus kernelFileStatusToHadoopFileStatus( io.delta.kernel.utils.FileStatus kernelFileStatus) { return new org.apache.hadoop.fs.FileStatus( kernelFileStatus.getSize() /* length */, false /* isDirectory */, 1 /* blockReplication */, 128 * 1024 * 1024 /* blockSize (128MB) */, kernelFileStatus.getModificationTime() /* modificationTime */, kernelFileStatus.getModificationTime() /* accessTime */, org.apache.hadoop.fs.permission.FsPermission.getFileDefault() /* permission */, "unknown" /* owner */, "unknown" /* group */, new org.apache.hadoop.fs.Path(kernelFileStatus.getPath()) /* path */); } private static CommitFailedException storageCFEtoKernelCFE( io.delta.storage.commit.CommitFailedException storageCFE) { return new CommitFailedException( storageCFE.getRetryable(), storageCFE.getConflict(), storageCFE.getMessage(), storageCFE.getCause()); } } ================================================ FILE: kernel/unitycatalog/src/main/java/io/delta/kernel/unitycatalog/UnityCatalogUtils.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog; import static io.delta.kernel.commit.CatalogCommitterUtils.*; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.delta.kernel.engine.Engine; import io.delta.kernel.expressions.Column; import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.util.ColumnMapping; import io.delta.kernel.internal.util.Tuple2; import io.delta.kernel.types.DataType; import java.util.*; import java.util.stream.Collectors; public class UnityCatalogUtils { private UnityCatalogUtils() {} private static final String UC_PROP_CLUSTERING_COLUMNS = "clusteringColumns"; private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); /** * Extract all properties that should be sent to Unity Catalog when creating a table (version 0). * *

This method extracts: * *

    *
  • All table properties from the metadata configuration *
  • Protocol-derived properties (e.g., delta.minReaderVersion=3, delta.feature.XXX=supported) *
  • UC-specific properties (delta.lastUpdateVersion, delta.lastCommitTimestamp) *
  • Clustering properties if a clustering domain metadata is present *
* * @param engine the engine to use for I/O operations (to retrieve the commit timestamp) * @param postCreateSnapshot the snapshot after version 0 has been written * @return a map of properties to send to Unity Catalog * @throws IllegalArgumentException if the snapshot is not version 0 */ public static Map getPropertiesForCreate( Engine engine, SnapshotImpl postCreateSnapshot) { if (postCreateSnapshot.getVersion() != 0) { throw new IllegalArgumentException( String.format( "Expected a snapshot at version 0, but got a snapshot at version %d", postCreateSnapshot.getVersion())); } final Map properties = new HashMap<>(); // Case 1: All table properties from metadata.configuration properties.putAll(postCreateSnapshot.getTableProperties()); // Case 2: Protocol-derived properties properties.putAll(extractProtocolProperties(postCreateSnapshot.getProtocol())); // Case 3: UC-specific properties properties.put(METASTORE_LAST_UPDATE_VERSION, String.valueOf(postCreateSnapshot.getVersion())); properties.put( METASTORE_LAST_COMMIT_TIMESTAMP, String.valueOf(postCreateSnapshot.getTimestamp(engine))); // Case 4: Clustering properties if present properties.putAll(extractClusteringProperties(postCreateSnapshot)); return properties; } /** * Extract clustering properties from the snapshot. * *

Converts physical clustering columns to logical column names and serializes them as a JSON * array of arrays for the "clusteringColumns" property. * *

Examples: * *

    *
  • Not clustered: returns empty map (no "clusteringColumns" property) *
  • Clustered with empty list: returns {"clusteringColumns": "[]"} *
  • Clustered with columns: physical column "col-abcd-1234" maps to nested logical column * "address.city" and is serialized as {"clusteringColumns": "[["address", "city"]]"} *
* * @return clustering properties if present, otherwise empty map */ private static Map extractClusteringProperties(SnapshotImpl snapshot) { return snapshot .getPhysicalClusteringColumns() .map( physicalClusteringCols -> { // Convert physical to logical column names final List> logicalClusteringCols = physicalClusteringCols.stream() .map( physicalCol -> { final Tuple2 logicalColumnAndType = ColumnMapping.getLogicalColumnNameAndDataType( snapshot.getSchema(), physicalCol); final Column logicalColumn = logicalColumnAndType._1; return Arrays.asList(logicalColumn.getNames()); }) .collect(Collectors.toList()); // Serialize to JSON try { final String clusteringColumnsJson = OBJECT_MAPPER.writeValueAsString(logicalClusteringCols); return Map.of(UC_PROP_CLUSTERING_COLUMNS, clusteringColumnsJson); } catch (JsonProcessingException ex) { throw new RuntimeException("Failed to serialize clustering columns to JSON", ex); } }) .orElse(Collections.emptyMap()); } } ================================================ FILE: kernel/unitycatalog/src/main/java/io/delta/kernel/unitycatalog/adapters/MetadataAdapter.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog.adapters; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.util.VectorUtils; import io.delta.storage.commit.actions.AbstractMetadata; import java.util.*; /** * Adapter from {@link io.delta.kernel.internal.actions.Metadata} to {@link * io.delta.storage.commit.actions.AbstractMetadata}. */ public class MetadataAdapter implements AbstractMetadata { private final Metadata kernelMetadata; public MetadataAdapter(Metadata kernelMetadata) { this.kernelMetadata = Objects.requireNonNull(kernelMetadata, "kernelMetadata is null"); } @Override public String getId() { return kernelMetadata.getId(); } @Override public String getName() { return kernelMetadata.getName().orElse(null); } @Override public String getDescription() { return kernelMetadata.getDescription().orElse(null); } @Override public String getProvider() { return kernelMetadata.getFormat().getProvider(); } @Override public Map getFormatOptions() { return Collections.unmodifiableMap(kernelMetadata.getFormat().getOptions()); } @Override public String getSchemaString() { return kernelMetadata.getSchemaString(); } @Override public List getPartitionColumns() { return Collections.unmodifiableList( VectorUtils.toJavaList(kernelMetadata.getPartitionColumns())); } @Override public Map getConfiguration() { return Collections.unmodifiableMap(kernelMetadata.getConfiguration()); } @Override public Long getCreatedTime() { return kernelMetadata.getCreatedTime().orElse(null); } } ================================================ FILE: kernel/unitycatalog/src/main/java/io/delta/kernel/unitycatalog/adapters/ProtocolAdapter.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog.adapters; import io.delta.kernel.internal.actions.Protocol; import io.delta.storage.commit.actions.AbstractProtocol; import java.util.Collections; import java.util.Objects; import java.util.Set; /** * Adapter from {@link io.delta.kernel.internal.actions.Protocol} to {@link * io.delta.storage.commit.actions.AbstractProtocol}. */ public class ProtocolAdapter implements AbstractProtocol { private final Protocol kernelProtocol; public ProtocolAdapter(Protocol kernelProtocol) { this.kernelProtocol = Objects.requireNonNull(kernelProtocol, "kernelProtocol is null"); } @Override public int getMinReaderVersion() { return kernelProtocol.getMinReaderVersion(); } @Override public int getMinWriterVersion() { return kernelProtocol.getMinWriterVersion(); } @Override public Set getReaderFeatures() { return Collections.unmodifiableSet(kernelProtocol.getReaderFeatures()); } @Override public Set getWriterFeatures() { return Collections.unmodifiableSet(kernelProtocol.getWriterFeatures()); } } ================================================ FILE: kernel/unitycatalog/src/main/java/io/delta/kernel/unitycatalog/adapters/UniformAdapter.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog.adapters; import io.delta.storage.commit.uniform.IcebergMetadata; import io.delta.storage.commit.uniform.UniformMetadata; import java.util.Map; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Adapter for Delta Uniform metadata to {@link UniformMetadata}. * *

This adapter extracts Uniform metadata (e.g., Iceberg) from committer properties and provides * it in the format expected by Unity Catalog. * *

The committer properties are provided by the connector which is responsible for computing the * Uniform metadata during write operations. The connector injects these properties into Kernel via * the CommitMetadata.withCommitterProperties, and Kernel propagates them to the * UCCatalogManagedCommitter, which then forwards them to Unity Catalog. */ public class UniformAdapter { private static final Logger logger = LoggerFactory.getLogger(UniformAdapter.class); // Keys for extracting Iceberg metadata from committer properties public static final String ICEBERG_METADATA_LOCATION_KEY = "delta.uniform.iceberg.metadataLocation"; public static final String ICEBERG_CONVERTED_DELTA_VERSION_KEY = "delta.uniform.iceberg.convertedDeltaVersion"; public static final String ICEBERG_CONVERTED_DELTA_TIMESTAMP_KEY = "delta.uniform.iceberg.convertedDeltaTimestamp"; private UniformAdapter() { // Private constructor to prevent instantiation } /** * Extracts Uniform metadata from committer properties. * * @param properties the committer properties map * @return an Optional containing the UniformMetadata if all required fields are present, * Optional.empty() otherwise */ public static Optional fromCommitterProperties(Map properties) { if (properties == null || properties.isEmpty()) { return Optional.empty(); } String metadataLocation = properties.get(ICEBERG_METADATA_LOCATION_KEY); String convertedVersionStr = properties.get(ICEBERG_CONVERTED_DELTA_VERSION_KEY); String convertedTimestamp = properties.get(ICEBERG_CONVERTED_DELTA_TIMESTAMP_KEY); // All three fields must be present if (metadataLocation == null || convertedVersionStr == null || convertedTimestamp == null) { return Optional.empty(); } try { long convertedVersion = Long.parseLong(convertedVersionStr); IcebergMetadata icebergMetadata = new IcebergMetadata(metadataLocation, convertedVersion, convertedTimestamp); return Optional.of(new UniformMetadata(icebergMetadata)); } catch (NumberFormatException e) { logger.warn( "Invalid converted delta version in committer properties: {}", convertedVersionStr, e); return Optional.empty(); } } } ================================================ FILE: kernel/unitycatalog/src/main/java/io/delta/kernel/unitycatalog/metrics/UcCommitTelemetry.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog.metrics; import io.delta.kernel.commit.CommitMetadata; import io.delta.kernel.internal.metrics.MetricsReportSerializer; import io.delta.kernel.internal.metrics.Timer; import io.delta.kernel.metrics.MetricsReport; import io.delta.kernel.shaded.com.fasterxml.jackson.annotation.JsonPropertyOrder; import io.delta.kernel.shaded.com.fasterxml.jackson.core.JsonProcessingException; import java.util.Optional; /** * Telemetry framework for Unity Catalog commit operations. * *

Collects timing metrics for commit operations and generates reports for successful and failed * commits. */ public class UcCommitTelemetry { private final String ucTableId; private final String ucTablePath; private final CommitMetadata commitMetadata; private final MetricsCollector metricsCollector; public UcCommitTelemetry(String ucTableId, String ucTablePath, CommitMetadata commitMetadata) { this.ucTableId = ucTableId; this.ucTablePath = ucTablePath; this.commitMetadata = commitMetadata; this.metricsCollector = new MetricsCollector(); } public MetricsCollector getMetricsCollector() { return metricsCollector; } public Report createSuccessReport() { return new Report(metricsCollector.capture(), Optional.empty()); } public Report createFailureReport(Exception error) { return new Report(metricsCollector.capture(), Optional.of(error)); } /** Mutable collector for gathering metrics during commit. */ public static class MetricsCollector { public final Timer totalCommitTimer = new Timer(); public final Timer writeCommitFileTimer = new Timer(); public final Timer commitToUcServerTimer = new Timer(); public MetricsResult capture() { return new MetricsResult(this); } } /** Immutable snapshot of collected metric results. */ @JsonPropertyOrder({ "totalCommitDurationNs", "writeCommitFileDurationNs", "commitToUcServerDurationNs" }) public static class MetricsResult { public final long totalCommitDurationNs; public final long writeCommitFileDurationNs; public final long commitToUcServerDurationNs; MetricsResult(MetricsCollector collector) { this.totalCommitDurationNs = collector.totalCommitTimer.totalDurationNs(); this.writeCommitFileDurationNs = collector.writeCommitFileTimer.totalDurationNs(); this.commitToUcServerDurationNs = collector.commitToUcServerTimer.totalDurationNs(); } } /** Complete UC commit report with metadata and metrics. */ @JsonPropertyOrder({ "operationType", "reportUUID", "ucTableId", "ucTablePath", "commitVersion", "commitType", "metrics", "exception" }) public class Report implements MetricsReport { public final String operationType = "UcCommit"; public final String reportUUID = java.util.UUID.randomUUID().toString(); public final String ucTableId = UcCommitTelemetry.this.ucTableId; public final String ucTablePath = UcCommitTelemetry.this.ucTablePath; public final long commitVersion = commitMetadata.getVersion(); public final CommitMetadata.CommitType commitType = commitMetadata.getCommitType(); public final MetricsResult metrics; public final Optional exception; public Report(MetricsResult metrics, Optional exception) { this.metrics = metrics; this.exception = exception; } @Override public String toJson() throws JsonProcessingException { return MetricsReportSerializer.OBJECT_MAPPER.writeValueAsString(this); } } } ================================================ FILE: kernel/unitycatalog/src/main/java/io/delta/kernel/unitycatalog/metrics/UcLoadSnapshotTelemetry.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog.metrics; import io.delta.kernel.internal.metrics.MetricsReportSerializer; import io.delta.kernel.internal.metrics.Timer; import io.delta.kernel.metrics.MetricsReport; import io.delta.kernel.shaded.com.fasterxml.jackson.annotation.JsonPropertyOrder; import io.delta.kernel.shaded.com.fasterxml.jackson.core.JsonProcessingException; import java.util.Optional; /** * Telemetry framework for Unity Catalog snapshot loading operations. * *

Collects timing metrics for snapshot loading and generates reports for successful and failed * loads. */ public class UcLoadSnapshotTelemetry { private final String ucTableId; private final String ucTablePath; private final Optional versionOpt; private final Optional timestampOpt; private final MetricsCollector metricsCollector; public UcLoadSnapshotTelemetry( String ucTableId, String ucTablePath, Optional versionOpt, Optional timestampOpt) { this.ucTableId = ucTableId; this.ucTablePath = ucTablePath; this.versionOpt = versionOpt; this.timestampOpt = timestampOpt; this.metricsCollector = new MetricsCollector(); } public MetricsCollector getMetricsCollector() { return metricsCollector; } public Report createSuccessReport() { return new Report(metricsCollector.capture(), Optional.empty()); } public Report createFailureReport(Exception error) { return new Report(metricsCollector.capture(), Optional.of(error)); } /** Mutable collector for gathering metrics during snapshot loading. */ public static class MetricsCollector { public final Timer totalSnapshotLoadTimer = new Timer(); public final Timer getCommitsTimer = new Timer(); public final Timer kernelSnapshotBuildTimer = new Timer(); public final Timer loadLatestSnapshotForTimestampTimeTravelTimer = new Timer(); private int numCatalogCommits = -1; private long resolvedSnapshotVersion = -1; public void setNumCatalogCommits(int count) { this.numCatalogCommits = count; } public void setResolvedSnapshotVersion(long version) { this.resolvedSnapshotVersion = version; } public MetricsResult capture() { return new MetricsResult(this); } } /** Immutable snapshot of collected metric results. */ @JsonPropertyOrder({ "totalLoadSnapshotDurationNs", "getCommitsDurationNs", "numCatalogCommits", "kernelSnapshotBuildDurationNs", "loadLatestSnapshotForTimestampTimeTravelDurationNs", "resolvedSnapshotVersion" }) public static class MetricsResult { public final long totalLoadSnapshotDurationNs; public final long getCommitsDurationNs; public final int numCatalogCommits; public final long kernelSnapshotBuildDurationNs; public final long loadLatestSnapshotForTimestampTimeTravelDurationNs; public final long resolvedSnapshotVersion; MetricsResult(MetricsCollector collector) { this.totalLoadSnapshotDurationNs = collector.totalSnapshotLoadTimer.totalDurationNs(); this.getCommitsDurationNs = collector.getCommitsTimer.totalDurationNs(); this.numCatalogCommits = collector.numCatalogCommits; this.kernelSnapshotBuildDurationNs = collector.kernelSnapshotBuildTimer.totalDurationNs(); this.loadLatestSnapshotForTimestampTimeTravelDurationNs = collector.loadLatestSnapshotForTimestampTimeTravelTimer.totalDurationNs(); this.resolvedSnapshotVersion = collector.resolvedSnapshotVersion; } } /** Complete UC snapshot loading report with metadata and metrics. */ @JsonPropertyOrder({ "operationType", "reportUUID", "ucTableId", "ucTablePath", "versionOpt", "timestampOpt", "metrics", "exception" }) public class Report implements MetricsReport { public final String operationType = "UcLoadSnapshot"; public final String reportUUID = java.util.UUID.randomUUID().toString(); public final String ucTableId = UcLoadSnapshotTelemetry.this.ucTableId; public final String ucTablePath = UcLoadSnapshotTelemetry.this.ucTablePath; public final Optional versionOpt = UcLoadSnapshotTelemetry.this.versionOpt; public final Optional timestampOpt = UcLoadSnapshotTelemetry.this.timestampOpt; public final MetricsResult metrics; public final Optional exception; public Report(MetricsResult metrics, Optional exception) { this.metrics = metrics; this.exception = exception; } @Override public String toJson() throws JsonProcessingException { return MetricsReportSerializer.OBJECT_MAPPER.writeValueAsString(this); } } } ================================================ FILE: kernel/unitycatalog/src/main/java/io/delta/kernel/unitycatalog/metrics/UcPublishTelemetry.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog.metrics; import io.delta.kernel.internal.metrics.MetricsReportSerializer; import io.delta.kernel.internal.metrics.Timer; import io.delta.kernel.metrics.MetricsReport; import io.delta.kernel.shaded.com.fasterxml.jackson.annotation.JsonPropertyOrder; import io.delta.kernel.shaded.com.fasterxml.jackson.core.JsonProcessingException; import java.util.Optional; /** * Telemetry framework for Unity Catalog publish operations. * *

Collects timing and counter metrics for publish operations and generates reports for * successful and failed publishes. */ public class UcPublishTelemetry { private final String ucTableId; private final String ucTablePath; private final long snapshotVersion; private final int numCommitsToPublish; private final MetricsCollector metricsCollector; public UcPublishTelemetry( String ucTableId, String ucTablePath, long snapshotVersion, int numCommitsToPublish) { this.ucTableId = ucTableId; this.ucTablePath = ucTablePath; this.snapshotVersion = snapshotVersion; this.numCommitsToPublish = numCommitsToPublish; this.metricsCollector = new MetricsCollector(); } public MetricsCollector getMetricsCollector() { return metricsCollector; } public Report createSuccessReport() { return new Report(metricsCollector.capture(), Optional.empty()); } public Report createFailureReport(Exception error) { return new Report(metricsCollector.capture(), Optional.of(error)); } /** Mutable collector for gathering metrics during publish. */ public static class MetricsCollector { public final Timer totalPublishTimer = new Timer(); private int commitsPublished = 0; private int commitsAlreadyPublished = 0; public void incrementCommitsPublished() { commitsPublished++; } /** * Increments the counter for commits already published by another process. Called when * FileAlreadyExistsException indicates the commit was previously published. */ public void incrementCommitsAlreadyPublished() { commitsAlreadyPublished++; } /** @return number of commits published */ public int getCommitsPublished() { return commitsPublished; } /** @return number of commits already published by another process */ public int getCommitsAlreadyPublished() { return commitsAlreadyPublished; } public MetricsResult capture() { return new MetricsResult(this); } } /** Immutable snapshot of collected metric results. */ @JsonPropertyOrder({ "totalPublishDurationNs", "numCommitsPublished", "numCommitsAlreadyPublished" }) public static class MetricsResult { public final long totalPublishDurationNs; public final int numCommitsPublished; public final int numCommitsAlreadyPublished; MetricsResult(MetricsCollector collector) { this.totalPublishDurationNs = collector.totalPublishTimer.totalDurationNs(); this.numCommitsPublished = collector.commitsPublished; this.numCommitsAlreadyPublished = collector.commitsAlreadyPublished; } } /** Complete UC publish report with metadata and metrics. */ @JsonPropertyOrder({ "operationType", "reportUUID", "ucTableId", "ucTablePath", "snapshotVersion", "numCommitsToPublish", "metrics", "exception" }) public class Report implements MetricsReport { public final String operationType = "UcPublish"; public final String reportUUID = java.util.UUID.randomUUID().toString(); public final String ucTableId = UcPublishTelemetry.this.ucTableId; public final String ucTablePath = UcPublishTelemetry.this.ucTablePath; public final long snapshotVersion = UcPublishTelemetry.this.snapshotVersion; public final int numCommitsToPublish = UcPublishTelemetry.this.numCommitsToPublish; public final MetricsResult metrics; public final Optional exception; public Report(MetricsResult metrics, Optional exception) { this.metrics = metrics; this.exception = exception; } @Override public String toJson() throws JsonProcessingException { return MetricsReportSerializer.OBJECT_MAPPER.writeValueAsString(this); } } } ================================================ FILE: kernel/unitycatalog/src/main/java/io/delta/kernel/unitycatalog/utils/OperationTimer.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog.utils; import java.util.function.Supplier; import org.slf4j.Logger; /** Utility class for timing operations and logging their execution duration. */ public class OperationTimer { private OperationTimer() {} @FunctionalInterface public interface ThrowingSupplier { T get() throws E; } /** Times an operation that throws a checked exception of type E and logs the duration. */ @SuppressWarnings("unchecked") public static T timeCheckedOperation( Logger logger, String operationName, String ucTableId, ThrowingSupplier operation) throws E { final long startTime = System.nanoTime(); try { final T result = operation.get(); final long durationMs = nanoToMs(System.nanoTime() - startTime); logger.info("[{}] {} completed in {} ms", ucTableId, operationName, durationMs); return result; } catch (Exception e) { final long durationMs = nanoToMs(System.nanoTime() - startTime); logger.warn( "[{}] {} failed after {} ms: {}", ucTableId, operationName, durationMs, e.getMessage()); throw (E) e; // Safe cast since operation can only throw E } } /** Times an operation and logs the duration. */ public static T timeUncheckedOperation( Logger logger, String operationName, String ucTableId, Supplier operation) { return timeCheckedOperation(logger, operationName, ucTableId, operation::get); } private static long nanoToMs(long nanoTime) { return nanoTime / 1_000_000; } } ================================================ FILE: kernel/unitycatalog/src/test/resources/log4j2.properties ================================================ # # Copyright (2025) The Delta Lake Project Authors. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Set everything to be logged to the file target/unit-tests.log rootLogger.level = warn rootLogger.appenderRef.file.ref = ${sys:test.appender:-File} appender.file.type = File appender.file.name = File appender.file.fileName = target/unit-tests.log appender.file.append = true appender.file.layout.type = PatternLayout appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n # Tests that launch java subprocesses can set the "test.appender" system property to # "console" to avoid having the child process's logs overwrite the unit test's # log file. appender.console.type = Console appender.console.name = console appender.console.target = SYSTEM_ERR appender.console.layout.type = PatternLayout appender.console.layout.pattern = %t: %m%n ================================================ FILE: kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/InMemoryUCClient.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog import java.lang.{Long => JLong} import java.net.URI import java.util.Optional import java.util.concurrent.ConcurrentHashMap import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer import io.delta.storage.commit.{Commit, CommitFailedException, GetCommitsResponse} import io.delta.storage.commit.actions.{AbstractMetadata, AbstractProtocol} import io.delta.storage.commit.uccommitcoordinator.{InvalidTargetTableException, UCClient} import io.delta.storage.commit.uniform.{IcebergMetadata, UniformMetadata} object InMemoryUCClient { /** * Internal data structure to track table state including commits and version information. * * Thread Safety: All public methods are synchronized to ensure thread-safe access to the * internal mutable state. This class is designed to be safely accessed by multiple threads * concurrently. */ class TableData( private var maxRatifiedVersion: Long, private val commits: ArrayBuffer[Commit]) { // For test only, since UC doesn't store these as top-level entities. private var currentProtocolOpt: Option[AbstractProtocol] = None private var currentMetadataOpt: Option[AbstractMetadata] = None private var currentIcebergOpt: Option[IcebergMetadata] = None /** @return the maximum ratified version. */ def getMaxRatifiedVersion: Long = synchronized { maxRatifiedVersion } /** @return An immutable list of all commits. */ def getCommits: List[Commit] = synchronized { commits.toList } /** @return commits filtered by version range. */ def getCommitsInRange( startVersion: Optional[JLong], endVersion: Optional[JLong]): List[Commit] = synchronized { commits .filter { commit => startVersion.orElse(0L) <= commit.getVersion && commit.getVersion <= endVersion.orElse(Long.MaxValue) } .toList } /** @return the current protocol. For test only. */ def getCurrentProtocolOpt: Option[AbstractProtocol] = synchronized { currentProtocolOpt } /** @return the current metadata. For test only. */ def getCurrentMetadataOpt: Option[AbstractMetadata] = synchronized { currentMetadataOpt } /** @return the current Iceberg metadata. For test only. */ def getCurrentIcebergOpt: Option[IcebergMetadata] = synchronized { currentIcebergOpt } /** Updates the Iceberg metadata. */ def updateIcebergMetadata(icebergMetadata: IcebergMetadata): Unit = synchronized { currentIcebergOpt = Some(icebergMetadata) } /** Appends a new commit to this table and atomically updates protocol/metadata. */ def appendCommit( commit: Commit, newProtocol: Optional[AbstractProtocol] = Optional.empty(), newMetadata: Optional[AbstractMetadata] = Optional.empty()): Unit = synchronized { val expectedCommitVersion = maxRatifiedVersion + 1 if (commit.getVersion != expectedCommitVersion) { throw new CommitFailedException( false, /* retryable */ false, /* conflict */ s"Expected commit version $expectedCommitVersion but got ${commit.getVersion}") } // Atomically update everything commits += commit maxRatifiedVersion = commit.getVersion if (newProtocol.isPresent) currentProtocolOpt = Some(newProtocol.get()) if (newMetadata.isPresent) currentMetadataOpt = Some(newMetadata.get()) } def forceRemoveCommitsUpToVersion(version: Long): Unit = synchronized { if (version < 0) { throw new IllegalArgumentException(s"Version must be non-negative, but got: $version") } val indexToRemove = commits.lastIndexWhere(_.getVersion <= version) if (indexToRemove >= 0) { commits.remove(0, indexToRemove + 1) } } } object TableData { def afterCreate(): TableData = new TableData(0, ArrayBuffer.empty[Commit]) } } /** * In-memory Unity Catalog client implementation for testing. * * Provides a mock implementation of UCClient that stores all table data in memory. This is useful * for unit tests that need to simulate Unity Catalog operations without connecting to an actual UC * service. * * Thread Safety: This implementation is thread-safe for concurrent access. Multiple threads can * safely perform operations on different tables simultaneously. Operations on the same table are * internally synchronized by the [[TableData]] class. */ class InMemoryUCClient(ucMetastoreId: String) extends UCClient { import InMemoryUCClient._ /** Map from UC_TABLE_ID to TABLE_DATA */ private val tables = new ConcurrentHashMap[String, TableData]() override def getMetastoreId: String = ucMetastoreId /** Convenience method for tests to commit with default parameters. */ def commitWithDefaults( tableId: String, tableUri: URI, commit: Optional[Commit], lastKnownBackfilledVersion: Optional[JLong] = Optional.empty(), disown: Boolean = false, newMetadata: Optional[AbstractMetadata] = Optional.empty(), newProtocol: Optional[AbstractProtocol] = Optional.empty()): Unit = { this.commit( tableId, tableUri, commit, lastKnownBackfilledVersion, disown, newMetadata, newProtocol, Optional.empty() /* uniform */ ) } override def commit( tableId: String, tableUri: URI, commitOpt: Optional[Commit] = Optional.empty(), lastKnownBackfilledVersionOpt: Optional[JLong], disown: Boolean, newMetadata: Optional[AbstractMetadata], newProtocol: Optional[AbstractProtocol], uniform: Optional[UniformMetadata]): Unit = { forceThrowInCommitMethod() if (disown) { throw new UnsupportedOperationException("disown not yet supported in InMemoryUCClient") } val tableData = getOrCreateTableIfNotExists(tableId) tableData.synchronized { commitOpt.ifPresent { commit => tableData.appendCommit(commit, newProtocol, newMetadata) } lastKnownBackfilledVersionOpt.ifPresent { lastKnownBackfilledVersion => tableData.forceRemoveCommitsUpToVersion(lastKnownBackfilledVersion) } // Update Iceberg metadata if provided in uniform uniform.ifPresent { u => u.getIcebergMetadata.ifPresent { iceberg => tableData.updateIcebergMetadata(iceberg) } } } } override def getCommits( tableId: String, tableUri: URI, startVersion: Optional[JLong], endVersion: Optional[JLong]): GetCommitsResponse = { val tableData = getTableDataElseThrow(tableId) val filteredCommits = tableData.getCommitsInRange(startVersion, endVersion) new GetCommitsResponse(filteredCommits.asJava, tableData.getMaxRatifiedVersion) } override def close(): Unit = {} /** Visible for testing. Can be overridden to force an exception in commit method. */ protected def forceThrowInCommitMethod(): Unit = {} private[unitycatalog] def insertTableDataAfterCreate(ucTableId: String): Unit = { Option(tables.putIfAbsent(ucTableId, TableData.afterCreate())) .foreach(_ => throw new IllegalArgumentException(s"Table $ucTableId already exists")) } private[unitycatalog] def insertTableData(ucTableId: String, tableData: TableData): Unit = { Option(tables.putIfAbsent(ucTableId, tableData)) .foreach(_ => throw new IllegalArgumentException(s"Table $ucTableId already exists")) } private[unitycatalog] def getTablesCopy: Map[String, TableData] = { tables.asScala.toMap } /** Retrieves table data for the given table ID or throws an exception if not found. */ private[unitycatalog] def getTableDataElseThrow(tableId: String): TableData = { Option(tables.get(tableId)) .getOrElse(throw new InvalidTargetTableException(s"Table not found: $tableId")) } /** Retrieves the table data for the given table ID, creating it if it does not exist. */ private def getOrCreateTableIfNotExists(tableId: String): TableData = { tables.computeIfAbsent(tableId, _ => TableData.afterCreate()) } } ================================================ FILE: kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/InMemoryUCClientSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog import java.lang.{Long => JLong} import java.net.URI import java.util.Optional import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer import io.delta.storage.commit.{Commit, CommitFailedException} import io.delta.storage.commit.uccommitcoordinator.InvalidTargetTableException import org.scalatest.funsuite.AnyFunSuite /** Unit tests for [[InMemoryUCClient]]. */ class InMemoryUCClientSuite extends AnyFunSuite with UCCatalogManagedTestUtils { private def testGetCommitsFiltering( allVersions: Seq[Long], startVersionOpt: Optional[JLong], endVersionOpt: Optional[JLong], expectedVersions: Seq[Long]): Unit = { val client = getInMemoryUCClientWithCommitsForTableId("tableId", allVersions) val response = client.getCommits("tableId", fakeURI, startVersionOpt, endVersionOpt) val actualVersions = response.getCommits.asScala.map(_.getVersion) assert(actualVersions == expectedVersions) } test("TableData::appendCommit handles commit version 1 (since CREATE does not go through UC)") { val tableData = InMemoryUCClient.TableData.afterCreate() assert(tableData.getMaxRatifiedVersion == 0L) tableData.appendCommit(createCommit(1L)) assert(tableData.getMaxRatifiedVersion == 1L) assert(tableData.getCommits.size == 1) assert(tableData.getCommits.head.getVersion == 1L) } test("TableData::appendCommit throws if commit version is not maxRatifiedVersion + 1") { val tableData = InMemoryUCClient.TableData.afterCreate() tableData.appendCommit(createCommit(1L)) val exMsg = intercept[CommitFailedException] { tableData.appendCommit(createCommit(99L)) }.getMessage assert(exMsg.contains("Expected commit version 2 but got 99")) } test("TableData::appendCommit appends the commit and updates the maxRatifiedVersion") { val tableData = InMemoryUCClient.TableData.afterCreate() tableData.appendCommit(createCommit(1L)) assert(tableData.getMaxRatifiedVersion == 1L) assert(tableData.getCommits.size == 1) assert(tableData.getCommits.head.getVersion == 1L) tableData.appendCommit(createCommit(2L)) assert(tableData.getMaxRatifiedVersion == 2L) assert(tableData.getCommits.size == 2) assert(tableData.getCommits.last.getVersion == 2L) } test("getCommits throws InvalidTargetTableException for non-existent table") { val client = new InMemoryUCClient("ucMetastoreId") val exception = intercept[InvalidTargetTableException] { client.getCommits("abcd", new URI("s3://bucket/table"), Optional.empty(), Optional.empty()) } assert(exception.getMessage.contains(s"Table not found: abcd")) } test("getCommits returns all commits if no startVersion or endVersion filter") { testGetCommitsFiltering( allVersions = 1L to 5L, startVersionOpt = Optional.empty(), endVersionOpt = Optional.empty(), expectedVersions = 1L to 5L) } test("getCommits filters by startVersion") { testGetCommitsFiltering( allVersions = 1L to 5L, startVersionOpt = Optional.of(2L), endVersionOpt = Optional.empty(), expectedVersions = 2L to 5L) } test("getCommits filters by endVersion") { testGetCommitsFiltering( allVersions = 1L to 5L, startVersionOpt = Optional.empty(), endVersionOpt = Optional.of(3L), expectedVersions = 1L to 3L) } test("getCommits filters by startVersion and endVersion") { testGetCommitsFiltering( allVersions = 1L to 5L, startVersionOpt = Optional.of(2L), endVersionOpt = Optional.of(4L), expectedVersions = 2L to 4L) } } ================================================ FILE: kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UCCatalogManagedClientCommitRangeSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog import java.util.Optional import io.delta.kernel.engine.Engine import io.delta.kernel.exceptions.KernelException import io.delta.storage.commit.uccommitcoordinator.{InvalidTargetTableException, UCClient} import org.scalatest.funsuite.AnyFunSuite class UCCatalogManagedClientCommitRangeSuite extends AnyFunSuite with UCCatalogManagedTestUtils { /** Helper method with reasonable defaults */ private def loadCommitRange( ucCatalogManagedClient: UCCatalogManagedClient, engine: Engine = defaultEngine, ucTableId: String = "testUcTableId", tablePath: String = "testUcTablePath", startVersionOpt: Optional[java.lang.Long] = emptyLongOpt, startTimestampOpt: Optional[java.lang.Long] = emptyLongOpt, endVersionOpt: Optional[java.lang.Long] = emptyLongOpt, endTimestampOpt: Optional[java.lang.Long] = emptyLongOpt) = { ucCatalogManagedClient.loadCommitRange( engine, ucTableId, tablePath, startVersionOpt, startTimestampOpt, endVersionOpt, endTimestampOpt) } private def testLoadCommitRange( expectedStartVersion: Long, expectedEndVersion: Long, startVersionOpt: Optional[java.lang.Long] = emptyLongOpt, startTimestampOpt: Optional[java.lang.Long] = emptyLongOpt, endVersionOpt: Optional[java.lang.Long] = emptyLongOpt, endTimestampOpt: Optional[java.lang.Long] = emptyLongOpt): Unit = { withUCClientAndTestTable { (ucClient, tablePath, _) => val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) val commitRange = loadCommitRange( ucCatalogManagedClient, tablePath = tablePath, startVersionOpt = startVersionOpt, startTimestampOpt = startTimestampOpt, endVersionOpt = endVersionOpt, endTimestampOpt = endTimestampOpt) assert(commitRange.getStartVersion == expectedStartVersion) assert(commitRange.getEndVersion == expectedEndVersion) assert(ucClient.getNumGetCommitCalls == 1) } } test("loadCommitRange throws on null input") { val ucClient = new InMemoryUCClient("ucMetastoreId") val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) assertThrows[NullPointerException] { // engine is null loadCommitRange(ucCatalogManagedClient, engine = null) } assertThrows[NullPointerException] { // ucTableId is null loadCommitRange(ucCatalogManagedClient, ucTableId = null) } assertThrows[NullPointerException] { // tablePath is null loadCommitRange(ucCatalogManagedClient, tablePath = null) } assertThrows[NullPointerException] { // startVersionOpt is null loadCommitRange(ucCatalogManagedClient, startVersionOpt = null) } assertThrows[NullPointerException] { // startTimestampOpt is null loadCommitRange(ucCatalogManagedClient, startTimestampOpt = null) } assertThrows[NullPointerException] { // endVersionOpt is null loadCommitRange(ucCatalogManagedClient, endVersionOpt = null) } assertThrows[NullPointerException] { // endTimestampOpt is null loadCommitRange(ucCatalogManagedClient, endTimestampOpt = null) } } test("loadCommitRange throws on invalid input - conflicting start boundaries") { val ucClient = new InMemoryUCClient("ucMetastoreId") val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) val ex = intercept[IllegalArgumentException] { loadCommitRange( ucCatalogManagedClient, startVersionOpt = Optional.of(1L), startTimestampOpt = Optional.of(100L)) } assert(ex.getMessage.contains("Cannot provide both a start timestamp and start version")) } test("loadCommitRange throws on invalid input - conflicting end boundaries") { val ucClient = new InMemoryUCClient("ucMetastoreId") val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) val ex = intercept[IllegalArgumentException] { loadCommitRange( ucCatalogManagedClient, startVersionOpt = Optional.of(0L), endVersionOpt = Optional.of(2L), endTimestampOpt = Optional.of(200L)) } assert(ex.getMessage.contains("Cannot provide both an end timestamp and start version")) } test("loadCommitRange throws on invalid input - start version > end version") { val ucClient = new InMemoryUCClient("ucMetastoreId") val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) val ex = intercept[IllegalArgumentException] { loadCommitRange( ucCatalogManagedClient, startVersionOpt = Optional.of(5L), endVersionOpt = Optional.of(2L)) } assert(ex.getMessage.contains("Cannot provide a start version greater than the end version")) } test("loadCommitRange throws on invalid input - start timestamp > end timestamp") { val ucClient = new InMemoryUCClient("ucMetastoreId") val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) val ex = intercept[IllegalArgumentException] { loadCommitRange( ucCatalogManagedClient, startTimestampOpt = Optional.of(500L), endTimestampOpt = Optional.of(200L)) } assert(ex.getMessage.contains( "Cannot provide a start timestamp greater than the end timestamp")) } test("loadCommitRange throws if startVersion is greater than max ratified version") { val ucClient = new InMemoryUCClient("ucMetastoreId") val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) val ex = intercept[IllegalArgumentException] { testLoadCommitRange( expectedStartVersion = 0, expectedEndVersion = 2, startVersionOpt = Optional.of(9L)) } assert(ex.getMessage.contains( "Cannot load commit range with start version 9 as the latest version ratified by UC is 2")) } test("loadCommitRange throws if endVersion is greater than max ratified version") { val ucClient = new InMemoryUCClient("ucMetastoreId") val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) val ex = intercept[IllegalArgumentException] { testLoadCommitRange( expectedStartVersion = 0, expectedEndVersion = 2, startVersionOpt = Optional.of(0L), endVersionOpt = Optional.of(9L)) } assert(ex.getMessage.contains( "Cannot load commit range with end version 9 as the latest version ratified by UC is 2")) } test("loadCommitRange throws when no start boundary is provided") { val ucClient = new InMemoryUCClient("ucMetastoreId") val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) val ex = intercept[IllegalArgumentException] { loadCommitRange(ucCatalogManagedClient) } assert(ex.getMessage.contains("Must provide either a start timestamp or start version")) } test("loadCommitRange loads with default end boundary -> latest") { testLoadCommitRange( expectedStartVersion = 0, expectedEndVersion = 2, startVersionOpt = Optional.of(0L)) } test("loadCommitRange loads with version boundaries") { testLoadCommitRange( expectedStartVersion = 1, expectedEndVersion = 2, startVersionOpt = Optional.of(1L), endVersionOpt = Optional.of(2L)) } test("loadCommitRange loads with timestamp boundaries") { testLoadCommitRange( expectedStartVersion = 0L, expectedEndVersion = 1L, startTimestampOpt = Optional.of(v0Ts), endTimestampOpt = Optional.of(v1Ts + 10)) } test("loadCommitRange loads with mixed start timestamp and end version") { testLoadCommitRange( expectedStartVersion = 0L, expectedEndVersion = 2L, startTimestampOpt = Optional.of(v0Ts), endVersionOpt = Optional.of(2L)) } test("loadCommitRange loads with mixed start version and end timestamp") { testLoadCommitRange( expectedStartVersion = 1L, expectedEndVersion = 2L, startVersionOpt = Optional.of(1L), endTimestampOpt = Optional.of(v2Ts)) } test("loadCommitRange loads single version range") { testLoadCommitRange( expectedStartVersion = 1L, expectedEndVersion = 1L, startVersionOpt = Optional.of(1L), endVersionOpt = Optional.of(1L)) } test("loadCommitRange loads single version range by timestamps") { testLoadCommitRange( expectedStartVersion = 1L, expectedEndVersion = 1L, startTimestampOpt = Optional.of(v1Ts - 50), endTimestampOpt = Optional.of(v1Ts + 50)) } test("loadCommitRange invalid timestamp bound") { intercept[KernelException] { testLoadCommitRange( expectedStartVersion = 1L, expectedEndVersion = 1L, startTimestampOpt = Optional.of(v2Ts + 10)) } intercept[KernelException] { testLoadCommitRange( expectedStartVersion = 1L, expectedEndVersion = 1L, startVersionOpt = Optional.of(0), endTimestampOpt = Optional.of(v0Ts - 10)) } } test("loadCommitRange throws when the table doesn't exist in catalog") { val ucClient = new InMemoryUCClient("ucMetastoreId") val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) val ex = intercept[RuntimeException] { loadCommitRange( ucCatalogManagedClient, ucTableId = "nonExistentTableId", startVersionOpt = Optional.of(0L)) } assert(ex.getCause.isInstanceOf[InvalidTargetTableException]) } test("loadCommitRange for new table when UC maxRatifiedVersion is 0") { val tablePath = getTestResourceFilePath("catalog-owned-preview") val ucCatalogManagedClient = createUCCatalogManagedClientForTableAfterCreate() val commitRange = loadCommitRange( ucCatalogManagedClient, tablePath = tablePath, startVersionOpt = Optional.of(0L)) assert(commitRange.getStartVersion == 0) assert(commitRange.getEndVersion == 0) } } ================================================ FILE: kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UCCatalogManagedClientSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog import java.util.Optional import scala.collection.JavaConverters._ import io.delta.kernel.exceptions.KernelException import io.delta.kernel.internal.CreateTableTransactionBuilderImpl import io.delta.kernel.internal.tablefeatures.TableFeatures import io.delta.kernel.internal.tablefeatures.TableFeatures.{CATALOG_MANAGED_RW_FEATURE, TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION} import io.delta.storage.commit.uccommitcoordinator.InvalidTargetTableException import org.scalatest.funsuite.AnyFunSuite /** Unit tests for [[UCCatalogManagedClient]]. */ class UCCatalogManagedClientSuite extends AnyFunSuite with UCCatalogManagedTestUtils { import UCCatalogManagedClientSuite._ private val testUcTableId = "testUcTableId" /** * If present, loads the given `versionToLoad`, else loads the maxRatifiedVersion of 2. * * Also asserts that the desired `versionToLoad` is, in fact, loaded. */ private def testCatalogManagedTable( versionToLoad: Optional[java.lang.Long] = emptyLongOpt, timestampToLoad: Optional[java.lang.Long] = emptyLongOpt, expectedVersion: Option[Long] = None): Unit = { require(!versionToLoad.isPresent || !timestampToLoad.isPresent) // If timestamp time-travel, must provide expected version require(!timestampToLoad.isPresent || expectedVersion.isDefined) withUCClientAndTestTable { (ucClient, tablePath, maxRatifiedVersion) => val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) val snapshot = loadSnapshot( ucCatalogManagedClient, tablePath = tablePath, versionToLoad = versionToLoad, timestampToLoad = timestampToLoad) val version = expectedVersion.getOrElse(versionToLoad.orElse(maxRatifiedVersion)) val protocol = snapshot.getProtocol assert(snapshot.getVersion == version) assert(protocol.getMinReaderVersion == TABLE_FEATURES_MIN_READER_VERSION) assert(protocol.getMinWriterVersion == TABLE_FEATURES_MIN_WRITER_VERSION) assert(protocol.getReaderFeatures.contains(CATALOG_MANAGED_RW_FEATURE.featureName())) assert(protocol.getWriterFeatures.contains(CATALOG_MANAGED_RW_FEATURE.featureName())) assert(ucClient.getNumGetCommitCalls == 1) } } test("constructor throws on invalid input") { assertThrows[NullPointerException] { new UCCatalogManagedClient(null) } } test("loadTable throws on invalid input") { val ucClient = new InMemoryUCClient("ucMetastoreId") val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) assertThrows[NullPointerException] { // engine is null loadSnapshot(ucCatalogManagedClient, engine = null) } assertThrows[NullPointerException] { // ucTableId is null loadSnapshot(ucCatalogManagedClient, ucTableId = null) } assertThrows[NullPointerException] { // tablePath is null loadSnapshot(ucCatalogManagedClient, tablePath = null) } assertThrows[NullPointerException] { // versionToLoad is null loadSnapshot(ucCatalogManagedClient, versionToLoad = null) } assertThrows[NullPointerException] { // timestampToLoad is null loadSnapshot(ucCatalogManagedClient, timestampToLoad = null) } assertThrows[IllegalArgumentException] { // version < 0 loadSnapshot(ucCatalogManagedClient, versionToLoad = Optional.of(-1L)) } assertThrows[IllegalArgumentException] { // cannot provide both timestamp and version loadSnapshot( ucCatalogManagedClient, versionToLoad = Optional.of(10L), timestampToLoad = Optional.of(10L)) } } Seq( (emptyLongOpt, emptyLongOpt, "latest (implicitly)"), (javaLongOpt(0L), emptyLongOpt, "v0 (explicitly by version)"), (emptyLongOpt, javaLongOpt(1749830855993L), "v0 (explicitly by timestamp")).foreach { case (versionToLoad, timestampToLoad, description) => test(s"loadTable throws when table doesn't exist in catalog -- $description") { val ucClient = new InMemoryUCClient("ucMetastoreId") val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) val ex = intercept[RuntimeException] { loadSnapshot( ucCatalogManagedClient, ucTableId = "nonExistentTableId", versionToLoad = versionToLoad, timestampToLoad = timestampToLoad) } assert(ex.getCause.isInstanceOf[InvalidTargetTableException]) } } Seq( (emptyLongOpt, emptyLongOpt, "latest (implicitly)"), (javaLongOpt(0L), emptyLongOpt, "v0 (explicitly by version)"), (emptyLongOpt, javaLongOpt(1749830855993L), "v0 (explicitly by timestamp")).foreach { case (versionToLoad, timestampToLoad, description) => test(s"table version 0 is loaded when UC maxRatifiedVersion is 0 -- $description") { val tablePath = getTestResourceFilePath("catalog-owned-preview") val ucCatalogManagedClient = createUCCatalogManagedClientForTableAfterCreate() val snapshot = loadSnapshot( ucCatalogManagedClient, tablePath = tablePath, versionToLoad = versionToLoad, timestampToLoad = timestampToLoad) assert(snapshot.getVersion == 0L) } } test("loadTable correctly loads a UC table -- versionToLoad is empty => load latest") { // Since versionToLoad is empty, it asserts that the latest version (2) is loaded testCatalogManagedTable() } /* ---- Time-travel-by-version tests --- */ test("loadTable correctly loads a UC table -- versionToLoad is a ratified commit (the max)") { testCatalogManagedTable(versionToLoad = Optional.of(2L)) } test("loadTable correctly loads a UC table -- versionToLoad is a ratified commit (not the max)") { testCatalogManagedTable(versionToLoad = Optional.of(1L)) } test("loadTable correctly loads a UC table -- versionToLoad is a published commit") { testCatalogManagedTable(versionToLoad = Optional.of(0L)) } test("loadTable throws if version to load is greater than max ratified version") { val exMsg = intercept[IllegalArgumentException] { testCatalogManagedTable(versionToLoad = Optional.of(9L)) }.getMessage assert(exMsg.contains("Cannot load table version 9 as the latest version ratified by UC is 2")) } /* ---- Time-travel-by-timestamp tests --- */ test("loadTable correctly loads a UC table -- " + "timestampToLoad is exactly a ratified commit (the max)") { testCatalogManagedTable(timestampToLoad = Optional.of(v2Ts), expectedVersion = Some(2L)) } test("loadTable correctly loads a UC table -- timestampToLoad is between ratified commits") { testCatalogManagedTable(timestampToLoad = Optional.of(v2Ts - 50L), expectedVersion = Some(1L)) } test("loadTable correctly loads a UC table -- " + "timestampToLoad is exactly a ratified commit (not the max)") { testCatalogManagedTable(timestampToLoad = Optional.of(v1Ts), expectedVersion = Some(1L)) } test("loadTable correctly loads a UC table -- " + "timestampToLoad is between ratified and published commits") { testCatalogManagedTable(timestampToLoad = Optional.of(v1Ts - 50L), expectedVersion = Some(0L)) } test("loadTable correctly loads a UC table -- timestampToLoad is exactly a published commit") { testCatalogManagedTable(timestampToLoad = Optional.of(v0Ts), expectedVersion = Some(0L)) } test("loadTable throws if timestampToLoad is before the earliest commit") { val exMsg = intercept[KernelException] { testCatalogManagedTable(timestampToLoad = Optional.of(v0Ts - 1), expectedVersion = Some(0)) }.getMessage assert(exMsg.contains("The provided timestamp 1749830855992 ms (2025-06-13T16:07:35.992Z) is " + "before the earliest available version 0")) } test("loadTable throws if timestampToLoad is after the latest commit") { val exMsg = intercept[KernelException] { testCatalogManagedTable(timestampToLoad = Optional.of(v2Ts + 1), expectedVersion = Some(2)) }.getMessage assert(exMsg.contains("The provided timestamp 1749830881800 ms (2025-06-13T16:08:01.800Z) is " + "after the latest available version 2")) } test("loadTable does not throw on negative timestamp in validation") { // This specifically tests that the validation logic in UCCatalogManagedClient.loadTable // does not reject negative timestamps val ucClient = new InMemoryUCClient("ucMetastoreId") val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) // Should not throw IllegalArgumentException for negative timestamp // (it will fail later when trying to find the table, but that's expected) val ex = intercept[RuntimeException] { loadSnapshot(ucCatalogManagedClient, timestampToLoad = Optional.of(-1L)) } // Verify it fails because the table doesn't exist, NOT because of timestamp validation assert(ex.getCause.isInstanceOf[InvalidTargetTableException]) } /* ---- end time-travel-by-timestamp tests ---- */ test("converts UC Commit into Kernel ParsedLogData.RATIFIED_STAGED_COMMIT") { val ucCommit = createCommit(1) val hadoopFS = ucCommit.getFileStatus val kernelParsedDeltaData = UCCatalogManagedClient .getSortedKernelParsedDeltaDataFromRatifiedCommits(testUcTableId, Seq(ucCommit).asJava) .get(0) val kernelFS = kernelParsedDeltaData.getFileStatus assert(kernelParsedDeltaData.isFile) assert(kernelFS.getPath == hadoopFS.getPath.toString) assert(kernelFS.getSize == hadoopFS.getLen) assert(kernelFS.getModificationTime == hadoopFS.getModificationTime) } test("sorts UC commits by version") { val ucCommitsUnsorted = Seq(createCommit(1), createCommit(2), createCommit(3)).asJava val kernelParsedLogData = UCCatalogManagedClient .getSortedKernelParsedDeltaDataFromRatifiedCommits(testUcTableId, ucCommitsUnsorted) assert(kernelParsedLogData.size() == 3) assert(kernelParsedLogData.get(0).getVersion == 1) assert(kernelParsedLogData.get(1).getVersion == 2) assert(kernelParsedLogData.get(2).getVersion == 3) } test("creates snapshot with UCCatalogManagedCommitter") { val tablePath = getTestResourceFilePath("catalog-owned-preview") val ucCatalogManagedClient = createUCCatalogManagedClientForTableAfterCreate() val snapshot = loadSnapshot(ucCatalogManagedClient, tablePath = tablePath, versionToLoad = Optional.of(0L)) assert(snapshot.getCommitter.isInstanceOf[UCCatalogManagedCommitter]) } test("buildCreateTableTransaction sets required properties and uses UC committer") { // ===== GIVEN ===== val ucClient = new InMemoryUCClient("ucMetastoreId") val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) // ===== WHEN ===== val createTableTxnBuilder = ucCatalogManagedClient .buildCreateTableTransaction(testUcTableId, baseTestTablePath, testSchema, "test-engine") .withTableProperties(Map("foo" -> "bar").asJava) .asInstanceOf[CreateTableTransactionBuilderImpl] // ===== THEN ===== val builderTableProperties = createTableTxnBuilder.getTablePropertiesOpt.get() assert(builderTableProperties .get(TableFeatures.CATALOG_MANAGED_RW_FEATURE.getTableFeatureSupportKey) == "supported") assert(builderTableProperties .get(TableFeatures.VACUUM_PROTOCOL_CHECK_RW_FEATURE.getTableFeatureSupportKey) == "supported") assert(builderTableProperties.get("io.unitycatalog.tableId") == testUcTableId) assert(builderTableProperties.get("foo") == "bar") val committerOpt = createTableTxnBuilder.getCommitterOpt assert(committerOpt.get().isInstanceOf[UCCatalogManagedCommitter]) } } object UCCatalogManagedClientSuite { private def javaLongOpt(value: Long): Optional[java.lang.Long] = { Optional.of(value) } } ================================================ FILE: kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UCCatalogManagedCommitterSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog import java.io.IOException import java.util.Optional import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer import io.delta.kernel.commit.{CommitFailedException, CommitMetadata} import io.delta.kernel.commit.CommitMetadata.CommitType import io.delta.kernel.data.Row import io.delta.kernel.internal.actions.{Metadata, Protocol} import io.delta.kernel.internal.tablefeatures.TableFeatures import io.delta.kernel.internal.util.{Tuple2 => KernelTuple2} import io.delta.kernel.test.{BaseMockJsonHandler, MockFileSystemClientUtils, TestFixtures, VectorTestUtils} import io.delta.kernel.unitycatalog.adapters.UniformAdapter import io.delta.kernel.utils.{CloseableIterator, FileStatus} import io.delta.storage.commit.Commit import io.delta.storage.commit.uccommitcoordinator.InvalidTargetTableException import InMemoryUCClient.TableData import org.scalatest.funsuite.AnyFunSuite class UCCatalogManagedCommitterSuite extends AnyFunSuite with UCCatalogManagedTestUtils with TestFixtures with VectorTestUtils with MockFileSystemClientUtils { private val testUcTableId = "testUcTableId" // ============================================================ // ===================== Misc. Unit Tests ===================== // ============================================================ test("constructor throws on null inputs") { val ucClient = new InMemoryUCClient("ucMetastoreId") assertThrows[NullPointerException] { new UCCatalogManagedCommitter(null, testUcTableId, baseTestTablePath) } assertThrows[NullPointerException] { new UCCatalogManagedCommitter(ucClient, null, baseTestTablePath) } assertThrows[NullPointerException] { new UCCatalogManagedCommitter(ucClient, testUcTableId, null) } } test("commit throws on null inputs") { val ucClient = new InMemoryUCClient("ucMetastoreId") val committer = new UCCatalogManagedCommitter(ucClient, testUcTableId, baseTestTablePath) // Null engine assertThrows[NullPointerException] { committer.commit(null, emptyActionsIterator, catalogManagedWriteCommitMetadata(version = 1)) } // Null finalizedActions assertThrows[NullPointerException] { committer.commit(defaultEngine, null, catalogManagedWriteCommitMetadata(version = 1)) } // Null commitMetadata assertThrows[NullPointerException] { committer.commit(defaultEngine, emptyActionsIterator, null) } } test("commit throws if CommitMetadata is for a different table") { val ucClient = new InMemoryUCClient("ucMetastoreId") val committer = new UCCatalogManagedCommitter(ucClient, testUcTableId, baseTestTablePath) val badCommitMetadata = catalogManagedWriteCommitMetadata( version = 1, "/path/to/different/table/_delta_log") val exMsg = intercept[IllegalArgumentException] { committer.commit(defaultEngine, emptyActionsIterator, badCommitMetadata) }.getMessage assert(exMsg.contains("Delta log path '/path/to/table/_delta_log' does not match expected " + "'/path/to/different/table/_delta_log'")) } // ========== CommitType Tests START ========== case class CommitTypeTestCase( readPandMOpt: Optional[KernelTuple2[Protocol, Metadata]] = Optional.empty(), newProtocolOpt: Optional[Protocol] = Optional.empty(), newMetadataOpt: Optional[Metadata] = Optional.empty(), expectedCommitType: CommitType) private val protocol12 = new Protocol(1, 2) private val unsupportedCommitTypesTestCases = Seq( CommitTypeTestCase( readPandMOpt = Optional.empty(), newProtocolOpt = Optional.of(protocol12), newMetadataOpt = Optional.of(basicPartitionedMetadata), expectedCommitType = CommitType.FILESYSTEM_CREATE), CommitTypeTestCase( readPandMOpt = Optional.of(new KernelTuple2(protocol12, basicPartitionedMetadata)), expectedCommitType = CommitType.FILESYSTEM_WRITE), CommitTypeTestCase( readPandMOpt = Optional.of(new KernelTuple2(protocol12, basicPartitionedMetadata)), newProtocolOpt = Optional.of(protocolWithCatalogManagedSupport), expectedCommitType = CommitType.FILESYSTEM_UPGRADE_TO_CATALOG), CommitTypeTestCase( readPandMOpt = Optional.of( new KernelTuple2(protocolWithCatalogManagedSupport, basicPartitionedMetadata)), newProtocolOpt = Optional.of(protocol12), expectedCommitType = CommitType.CATALOG_DOWNGRADE_TO_FILESYSTEM)) unsupportedCommitTypesTestCases.foreach { testCase => test(s"commit throws UnsupportedOperationException for ${testCase.expectedCommitType}") { val ucClient = new InMemoryUCClient("ucMetastoreId") val committer = new UCCatalogManagedCommitter(ucClient, testUcTableId, baseTestTablePath) // version > 0 for updates, version = 0 for creates val version = if (testCase.readPandMOpt.isPresent) 1L else 0L val commitMetadata = createCommitMetadata( version = version, logPath = baseTestLogPath, readPandMOpt = testCase.readPandMOpt, newProtocolOpt = testCase.newProtocolOpt, newMetadataOpt = testCase.newMetadataOpt) assert(commitMetadata.getCommitType == testCase.expectedCommitType) val exception = intercept[UnsupportedOperationException] { committer.commit(defaultEngine, emptyActionsIterator, commitMetadata) } assert(exception.getMessage == s"Unsupported commit type: ${testCase.expectedCommitType}") } } // ========== CommitType Tests END ========== test("kernelFileStatusToHadoopFileStatus converts kernel FileStatus to Hadoop FileStatus") { // ===== GIVEN ===== val kernelFileStatus = FileStatus.of("/path/to/file.json", 1024L, 1234567890L) // ===== WHEN ===== val hadoopFileStatus = UCCatalogManagedCommitter.kernelFileStatusToHadoopFileStatus(kernelFileStatus) // ===== THEN ===== // These are the fields that we care about, taken from the Kernel FileStatus assert(hadoopFileStatus.getPath.toString == "/path/to/file.json") assert(hadoopFileStatus.getLen == 1024L) assert(hadoopFileStatus.getModificationTime == 1234567890L) // These are defaults that we set assert(hadoopFileStatus.getAccessTime == 1234567890L) // same as modification time assert(!hadoopFileStatus.isDirectory) assert(hadoopFileStatus.getReplication == 1) assert(hadoopFileStatus.getBlockSize == 128 * 1024 * 1024) // 128MB assert(hadoopFileStatus.getOwner == "unknown") assert(hadoopFileStatus.getGroup == "unknown") assert(hadoopFileStatus.getPermission == org.apache.hadoop.fs.permission.FsPermission.getFileDefault) } test("writeDeltaFile returns real FileStatus") { withTempDirAndAllDeltaSubDirs { case (tablePath, logPath) => // ===== GIVEN ===== val ucClient = new InMemoryUCClient("ucMetastoreId") val committer = new UCCatalogManagedCommitter(ucClient, testUcTableId, tablePath) val testValue = "TEST_FILE_STATUS_DATA" val actionsIterator = getSingleElementRowIter(testValue) val commitMetadata = createCommitMetadata( version = 0, logPath = logPath, newProtocolOpt = Optional.of(protocolWithCatalogManagedSupport), newMetadataOpt = Optional.of(basicPartitionedMetadata)) // ===== WHEN ===== val response = committer.commit(defaultEngine, actionsIterator, commitMetadata) // ===== THEN ===== val fileStatus = response.getCommitLogData.getFileStatus assert(fileStatus.getSize > 0) assert(fileStatus.getModificationTime > 0) } } // =============================================================== // ===================== CATALOG_WRITE Tests ===================== // =============================================================== test("CATALOG_WRITE: protocol and metadata changes are passed to UC client") { withTempDirAndAllDeltaSubDirs { case (tablePath, logPath) => // ===== GIVEN ===== val ucClient = new InMemoryUCClient("ucMetastoreId") ucClient.insertTableDataAfterCreate(testUcTableId) val committer = new UCCatalogManagedCommitter(ucClient, testUcTableId, tablePath) // ===== WHEN ===== val protocolUpgrade = protocolWithCatalogManagedSupport .withFeature(TableFeatures.DELETION_VECTORS_RW_FEATURE) val metadataUpgrade = basicPartitionedMetadata .withMergedConfiguration(Map("foo" -> "bar").asJava) val commitMetadata = createCommitMetadata( version = 1, logPath = logPath, readPandMOpt = Optional.of( new KernelTuple2[Protocol, Metadata]( protocolWithCatalogManagedSupport, basicPartitionedMetadata)), newProtocolOpt = Optional.of(protocolUpgrade), newMetadataOpt = Optional.of(metadataUpgrade)) committer.commit(defaultEngine, emptyActionsIterator, commitMetadata) // ===== THEN ===== val updatedTableData = ucClient.getTablesCopy.get(testUcTableId).get val latestProtocol = updatedTableData.getCurrentProtocolOpt.get val latestMetadata = updatedTableData.getCurrentMetadataOpt.get assert(latestProtocol.getReaderFeatures === protocolUpgrade.getReaderFeatures) assert(latestProtocol.getWriterFeatures === protocolUpgrade.getWriterFeatures) assert(latestMetadata.getConfiguration === metadataUpgrade.getConfiguration) } } test("CATALOG_WRITE: Iceberg metadata is extracted and passed to UC client") { withTempDirAndAllDeltaSubDirs { case (tablePath, logPath) => // ===== GIVEN ===== val ucClient = new InMemoryUCClient("ucMetastoreId") ucClient.insertTableDataAfterCreate(testUcTableId) val committer = new UCCatalogManagedCommitter(ucClient, testUcTableId, tablePath) // ===== WHEN ===== val icebergProperties = Map( UniformAdapter.ICEBERG_METADATA_LOCATION_KEY -> "s3://bucket/table/metadata/v1.json", UniformAdapter.ICEBERG_CONVERTED_DELTA_VERSION_KEY -> "1", UniformAdapter.ICEBERG_CONVERTED_DELTA_TIMESTAMP_KEY -> "2025-01-04T03:13:11.423").asJava val commitMetadata = createCommitMetadata( version = 1, logPath = logPath, committerProperties = () => icebergProperties, readPandMOpt = Optional.of( new KernelTuple2[Protocol, Metadata]( protocolWithCatalogManagedSupport, basicPartitionedMetadata))) committer.commit(defaultEngine, emptyActionsIterator, commitMetadata) // ===== THEN ===== val updatedTableData = ucClient.getTablesCopy.get(testUcTableId).get val icebergOpt = updatedTableData.getCurrentIcebergOpt assert(icebergOpt.isDefined) val iceberg = icebergOpt.get assert(iceberg.getMetadataLocation === "s3://bucket/table/metadata/v1.json") assert(iceberg.getConvertedDeltaVersion === 1L) assert(iceberg.getConvertedDeltaTimestamp === "2025-01-04T03:13:11.423") } } test("CATALOG_WRITE: empty committer properties result in no Iceberg metadata") { withTempDirAndAllDeltaSubDirs { case (tablePath, logPath) => // ===== GIVEN ===== val ucClient = new InMemoryUCClient("ucMetastoreId") ucClient.insertTableDataAfterCreate(testUcTableId) val committer = new UCCatalogManagedCommitter(ucClient, testUcTableId, tablePath) // ===== WHEN ===== val emptyProperties = Map.empty[String, String].asJava val commitMetadata = createCommitMetadata( version = 1, logPath = logPath, committerProperties = () => emptyProperties, readPandMOpt = Optional.of( new KernelTuple2[Protocol, Metadata]( protocolWithCatalogManagedSupport, basicPartitionedMetadata))) committer.commit(defaultEngine, emptyActionsIterator, commitMetadata) // ===== THEN ===== val updatedTableData = ucClient.getTablesCopy.get(testUcTableId).get val icebergOpt = updatedTableData.getCurrentIcebergOpt assert(icebergOpt.isEmpty) } } test("CATALOG_WRITE: throws exception when convertedDeltaVersion " + "does not match commit version") { withTempDirAndAllDeltaSubDirs { case (tablePath, logPath) => // ===== GIVEN ===== val ucClient = new InMemoryUCClient("ucMetastoreId") ucClient.insertTableDataAfterCreate(testUcTableId) val committer = new UCCatalogManagedCommitter(ucClient, testUcTableId, tablePath) // ===== WHEN ===== // Commit version is 1, but convertedDeltaVersion is 2 (mismatch) val icebergProperties = Map( UniformAdapter.ICEBERG_METADATA_LOCATION_KEY -> "s3://bucket/table/metadata/v2.json", UniformAdapter.ICEBERG_CONVERTED_DELTA_VERSION_KEY -> "2", UniformAdapter.ICEBERG_CONVERTED_DELTA_TIMESTAMP_KEY -> "2025-01-04T03:13:11.423").asJava val commitMetadata = createCommitMetadata( version = 1, logPath = logPath, committerProperties = () => icebergProperties, readPandMOpt = Optional.of( new KernelTuple2[Protocol, Metadata]( protocolWithCatalogManagedSupport, basicPartitionedMetadata))) // ===== THEN ===== val exception = intercept[IllegalStateException] { committer.commit(defaultEngine, emptyActionsIterator, commitMetadata) } assert(exception.getMessage.contains( "Uniform convertedDeltaVersion (2) must match commit version (1)")) } } test("CATALOG_WRITE: writes staged commit file and invokes UC client commit API (no P&M change") { withTempDirAndAllDeltaSubDirs { case (tablePath, logPath) => // ===== GIVEN ===== val ucClient = new InMemoryUCClient("ucMetastoreId") ucClient.insertTableDataAfterCreate(testUcTableId) val testValue = "TEST_COMMIT_DATA_12345" val actionsIterator = getSingleElementRowIter(testValue) val committer = new UCCatalogManagedCommitter(ucClient, testUcTableId, tablePath) val commitMetadata = catalogManagedWriteCommitMetadata(version = 1, logPath = logPath) // ===== WHEN ===== val response = committer.commit(defaultEngine, actionsIterator, commitMetadata) // ===== THEN ===== val stagedCommitFilePath = response.getCommitLogData.getFileStatus.getPath // Verify the staged commit file actually exists on disk val file = new java.io.File(new java.net.URI(stagedCommitFilePath)) assert(file.exists()) assert(file.isFile()) // Read the file content and verify our test value was written val fileContent = scala.io.Source.fromFile(file).getLines().mkString("\n") assert(fileContent.contains(testValue)) // Verify the file is in the correct location val expectedPattern = s"^file:$tablePath/_delta_log/_staged_commits/00000000000000000001\\.[^.]+\\.json$$" assert(stagedCommitFilePath.matches(expectedPattern)) // Verify UC client was invoked and table was updated. val updatedTable = ucClient.getTablesCopy.get(testUcTableId).get assert(updatedTable.getMaxRatifiedVersion == 1) assert(updatedTable.getCommits.size == 1) // Assert that no P&M change in this txn => No P&M change sent to UC assert(updatedTable.getCurrentProtocolOpt.isEmpty) assert(updatedTable.getCurrentMetadataOpt.isEmpty) // Verify the new commit in UC has correct version val lastCommit = updatedTable.getCommits.last assert(lastCommit.getVersion == 1) assert(lastCommit.getFileStatus.getPath.toString == stagedCommitFilePath) } } test("CATALOG_WRITE: IOException writing staged commit => CFE(retryable=true, conflict=false)") { withTempDirAndAllDeltaSubDirs { case (tablePath, logPath) => // ===== GIVEN ===== val throwingEngine = mockEngine(jsonHandler = new BaseMockJsonHandler { override def writeJsonFileAtomically( path: String, data: CloseableIterator[Row], overwrite: Boolean): Unit = throw new IOException("Network error") }) val ucClient = new InMemoryUCClient("ucMetastoreId") val tableData = new TableData(maxRatifiedVersion = 1, commits = ArrayBuffer.empty[Commit]) ucClient.insertTableData(testUcTableId, tableData) val committer = new UCCatalogManagedCommitter(ucClient, testUcTableId, tablePath) val commitMetadata = catalogManagedWriteCommitMetadata(2, logPath = logPath) // ===== WHEN ===== val ex = intercept[CommitFailedException] { committer.commit(throwingEngine, emptyActionsIterator, commitMetadata) } // ===== THEN ===== assert(ex.isRetryable && !ex.isConflict) assert(ex.getMessage.contains("Failed to write delta file due to: Network error")) } } test("CATALOG_WRITE: i.d.s.c.CommitFailedException during UC commit => kernel CFE") { withTempDirAndAllDeltaSubDirs { case (tablePath, logPath) => // ===== GIVEN ===== val ucClient = new InMemoryUCClient("ucMetastoreId") { override def forceThrowInCommitMethod(): Unit = throw new io.delta.storage.commit.CommitFailedException( true, // retryable true, // conflict "Storage conflict", null) } val tableData = new TableData(maxRatifiedVersion = 1, commits = ArrayBuffer.empty[Commit]) ucClient.insertTableData(testUcTableId, tableData) val committer = new UCCatalogManagedCommitter(ucClient, testUcTableId, tablePath) val commitMetadata = catalogManagedWriteCommitMetadata(2, logPath = logPath) // ===== WHEN ===== val ex = intercept[CommitFailedException] { committer.commit(defaultEngine, emptyActionsIterator, commitMetadata) } // ===== THEN ===== assert(ex.isRetryable && ex.isConflict) assert(ex.getMessage.contains("Storage conflict")) } } test("CATALOG_WRITE: IOException during UC commit => CFE(retryable=true, conflict=false)") { withTempDirAndAllDeltaSubDirs { case (tablePath, logPath) => // ===== GIVEN ===== val ucClient = new InMemoryUCClient("ucMetastoreId") { override def forceThrowInCommitMethod(): Unit = throw new IOException("UC network error") } val tableData = new TableData(maxRatifiedVersion = 1, commits = ArrayBuffer.empty[Commit]) ucClient.insertTableData(testUcTableId, tableData) val committer = new UCCatalogManagedCommitter(ucClient, testUcTableId, tablePath) val commitMetadata = catalogManagedWriteCommitMetadata(2, logPath = logPath) // ===== WHEN ===== val ex = intercept[CommitFailedException] { committer.commit(defaultEngine, emptyActionsIterator, commitMetadata) } // ===== THEN ===== assert(ex.isRetryable && !ex.isConflict) assert(ex.getMessage.contains("UC network error")) } } test("CATALOG_WRITE: i.d.s.c.u.UCCCE during UC commit => CFE(retryable=false, conflict=false)") { withTempDirAndAllDeltaSubDirs { case (tablePath, logPath) => // ===== GIVEN ===== val ucClient = new InMemoryUCClient("ucMetastoreId") { override def forceThrowInCommitMethod(): Unit = { // A child type of UCCommitCoordinatorException throw new InvalidTargetTableException("Target table does not exist") } } val tableData = new TableData(maxRatifiedVersion = 1, commits = ArrayBuffer.empty[Commit]) ucClient.insertTableData(testUcTableId, tableData) val committer = new UCCatalogManagedCommitter(ucClient, "unknownTableId", tablePath) val commitMetadata = catalogManagedWriteCommitMetadata(2, logPath = logPath) // ===== WHEN ===== val ex = intercept[CommitFailedException] { committer.commit(defaultEngine, emptyActionsIterator, commitMetadata) } // ===== THEN ===== assert(ex.getCause.isInstanceOf[InvalidTargetTableException]) assert(!ex.isRetryable && !ex.isConflict) assert(ex.getMessage.contains("Target table does not exist")) } } // ================================================================ // ===================== CATALOG_CREATE Tests ===================== // ================================================================ test("CATALOG_CREATE: writes published delta file for version 0") { withTempDirAndAllDeltaSubDirs { case (tablePath, logPath) => // ===== GIVEN ===== val ucClient = new InMemoryUCClient("ucMetastoreId") val testValue = "CREATE_TABLE_DATA_12345" val actionsIterator = getSingleElementRowIter(testValue) val committer = new UCCatalogManagedCommitter(ucClient, testUcTableId, tablePath) val commitMetadata = createCommitMetadata( version = 0, logPath = logPath, newProtocolOpt = Optional.of(protocolWithCatalogManagedSupport), newMetadataOpt = Optional.of(basicPartitionedMetadata)) // ===== WHEN ===== val response = committer.commit(defaultEngine, actionsIterator, commitMetadata) // ===== THEN ===== val publishedDeltaFilePath = response.getCommitLogData.getFileStatus.getPath // Verify the published delta file exists and is version 0 val expectedFilePath = s"file:$logPath/00000000000000000000.json" assert(publishedDeltaFilePath == expectedFilePath) val file = new java.io.File(new java.net.URI(publishedDeltaFilePath)) assert(file.exists()) assert(file.isFile()) // Read the file content and verify our test value was written val fileContent = scala.io.Source.fromFile(file).getLines().mkString("\n") assert(fileContent.contains(testValue)) // Validate that UC was not updated for v0 // TODO: [delta-io/delta#5118] If UC changes CREATE semantics, update logic here. assert(!ucClient.getTablesCopy.contains(testUcTableId)) } } test("CATALOG_CREATE: IOException during write throws CFE(retryable=true, conflict=false)") { withTempDirAndAllDeltaSubDirs { case (tablePath, logPath) => // ===== GIVEN ===== val ucClient = new InMemoryUCClient("ucMetastoreId") val throwingEngine = mockEngine(jsonHandler = new BaseMockJsonHandler { override def writeJsonFileAtomically( path: String, data: CloseableIterator[Row], overwrite: Boolean): Unit = throw new IOException("Network hiccup") }) val committer = new UCCatalogManagedCommitter(ucClient, testUcTableId, tablePath) val commitMetadata = createCommitMetadata( version = 0, logPath = logPath, newProtocolOpt = Optional.of(protocolWithCatalogManagedSupport), newMetadataOpt = Optional.of(basicPartitionedMetadata)) // ===== WHEN ===== val ex = intercept[CommitFailedException] { committer.commit(throwingEngine, emptyActionsIterator, commitMetadata) } // ===== THEN ===== assert(ex.isRetryable && !ex.isConflict) assert(ex.getMessage.contains("Failed to write delta file due to: Network hiccup")) } } } ================================================ FILE: kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UCCatalogManagedTestUtils.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog import java.lang.{Long => JLong} import java.net.URI import java.util.Optional import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer import scala.reflect.ClassTag import io.delta.kernel.commit.{CommitMetadata, PublishMetadata} import io.delta.kernel.data.Row import io.delta.kernel.defaults.engine.DefaultEngine import io.delta.kernel.defaults.utils.{TestUtils, WriteUtils} import io.delta.kernel.engine.{Engine, MetricsReporter} import io.delta.kernel.internal.SnapshotImpl import io.delta.kernel.internal.actions.{Metadata, Protocol} import io.delta.kernel.internal.files.ParsedCatalogCommitData import io.delta.kernel.internal.util.{Tuple2 => KernelTuple2} import io.delta.kernel.internal.util.FileNames import io.delta.kernel.internal.util.Utils.singletonCloseableIterator import io.delta.kernel.metrics.MetricsReport import io.delta.kernel.test.{ActionUtils, TestFixtures} import io.delta.kernel.utils.CloseableIterator import io.delta.storage.commit.{Commit, GetCommitsResponse} import InMemoryUCClient.TableData import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus => HadoopFileStatus, FileSystem, Path} trait UCCatalogManagedTestUtils extends TestUtils with ActionUtils with TestFixtures with WriteUtils { val fakeURI = new URI("s3://bucket/table") val baseTestTablePath = "/path/to/table" val baseTestLogPath = "/path/to/table/_delta_log" val emptyLongOpt = Optional.empty[java.lang.Long]() /** * Generic MetricsReporter that captures specific types of MetricsReport instances. * This can be used for both UcCommitTelemetry.Report and UcPublishTelemetry.Report. * * @tparam T the type of MetricsReport to capture */ class CapturingMetricsReporter[T <: MetricsReport: ClassTag] extends MetricsReporter { val reports = ArrayBuffer[T]() override def report(report: MetricsReport): Unit = { report match { case r: T => reports.append(r) case _ => // Ignore other report types } } } /** Creates an Engine with a custom MetricsReporter for testing telemetry */ def createEngineWithMetricsCapture(reporter: MetricsReporter): Engine = { val hadoopConf = new Configuration() new DefaultEngine( new io.delta.kernel.defaults.engine.hadoopio.HadoopFileIO(hadoopConf)) { override def getMetricsReporters: java.util.List[MetricsReporter] = { val reporters = new java.util.ArrayList[MetricsReporter]() reporters.add(reporter) reporters } } } /** Helper method with reasonable defaults */ def loadSnapshot( ucCatalogManagedClient: UCCatalogManagedClient, engine: Engine = defaultEngine, ucTableId: String = "testUcTableId", tablePath: String = "testUcTablePath", versionToLoad: Optional[java.lang.Long] = emptyLongOpt, timestampToLoad: Optional[java.lang.Long] = emptyLongOpt): SnapshotImpl = { ucCatalogManagedClient.loadSnapshot( engine, ucTableId, tablePath, versionToLoad, timestampToLoad).asInstanceOf[SnapshotImpl] } def hadoopCommitFileStatus(version: Long): HadoopFileStatus = { val filePath = FileNames.stagedCommitFile(baseTestLogPath, version) new HadoopFileStatus( version, /* length */ false, /* isDir */ version.toInt, /* blockReplication */ version, /* blockSize */ version, /* modificationTime */ new Path(filePath)) } def createCommit(version: Long): Commit = { new Commit(version, hadoopCommitFileStatus(version), version) // version, fileStatus, timestamp } /** Creates an InMemoryUCClient with the given tableId and commits for the specified versions. */ def getInMemoryUCClientWithCommitsForTableId( tableId: String, versions: Seq[Long]): InMemoryUCClient = { val client = new InMemoryUCClient("ucMetastoreId") versions.foreach { v => client.commitWithDefaults(tableId, fakeURI, Optional.of(createCommit(v))) } client } def createPublishMetadata( snapshotVersion: Long, logPath: String, catalogCommits: List[ParsedCatalogCommitData]): PublishMetadata = { new PublishMetadata(snapshotVersion, logPath, catalogCommits.asJava) } def getSingleElementRowIter(elem: String): CloseableIterator[Row] = { import io.delta.kernel.defaults.integration.DataBuilderUtils import io.delta.kernel.types.{StringType, StructField, StructType} val schema = new StructType().add(new StructField("testColumn", StringType.STRING, true)) val simpleRow = DataBuilderUtils.row(schema, elem) singletonCloseableIterator(simpleRow) } /** Creates a UCCatalogManagedClient with an InMemoryUCClient for testing */ def createUCClientAndCatalogManagedClient( metastoreId: String = "ucMetastoreId"): (InMemoryUCClient, UCCatalogManagedClient) = { val ucClient = new InMemoryUCClient(metastoreId) val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) (ucClient, ucCatalogManagedClient) } /** Version TS for the test table used in [[withUCClientAndTestTable]] */ val v0Ts = 1749830855993L // published commit val v1Ts = 1749830871085L // ratified staged commit val v2Ts = 1749830881799L // ratified staged commit /** * @param textFx test function to run that takes input (ucClient, tablePath, maxRatifiedVersion) */ def withUCClientAndTestTable( textFx: (InMemoryUCClientWithMetrics, String, Long) => Unit): Unit = { val maxRatifiedVersion = 2L val tablePath = getTestResourceFilePath("catalog-owned-preview") val ucClient = new InMemoryUCClientWithMetrics("ucMetastoreId") val fs = FileSystem.get(new Configuration()) val catalogCommits = Seq( // scalastyle:off line.size.limit getTestResourceFilePath("catalog-owned-preview/_delta_log/_staged_commits/00000000000000000001.4cb9708e-b478-44de-b203-53f9ba9b2876.json"), getTestResourceFilePath("catalog-owned-preview/_delta_log/_staged_commits/00000000000000000002.5b9bba4a-0085-430d-a65e-b0d38c1afbe9.json")) // scalastyle:on line.size.limit .map { path => fs.getFileStatus(new Path(path)) } .map { fileStatus => new Commit( FileNames.deltaVersion(fileStatus.getPath.toString), fileStatus, fileStatus.getModificationTime) } val tableData = new TableData(maxRatifiedVersion, ArrayBuffer(catalogCommits: _*)) ucClient.insertTableData("testUcTableId", tableData) textFx(ucClient, tablePath, maxRatifiedVersion) } def createUCCatalogManagedClientForTableAfterCreate( ucTableId: String = "testUcTableId"): UCCatalogManagedClient = { val ucClient = new InMemoryUCClient("ucMetastoreId") ucClient.insertTableDataAfterCreate(ucTableId) new UCCatalogManagedClient(ucClient) } /** This should be used for WRITE operations (version >= 1), not for CREATE. */ def catalogManagedWriteCommitMetadata( version: Long, logPath: String = baseTestLogPath): CommitMetadata = createCommitMetadata( version = version, logPath = logPath, readPandMOpt = Optional.of( new KernelTuple2[Protocol, Metadata]( protocolWithCatalogManagedSupport, basicPartitionedMetadata))) /** Wrapper class around InMemoryUCClient that tracks number of getCommit calls made */ class InMemoryUCClientWithMetrics(ucMetastoreId: String) extends InMemoryUCClient(ucMetastoreId) { private var numGetCommitsCalls: Long = 0 override def getCommits( tableId: String, tableUri: URI, startVersion: Optional[JLong], endVersion: Optional[JLong]): GetCommitsResponse = { numGetCommitsCalls += 1 super.getCommits(tableId, tableUri, startVersion, endVersion) } def getNumGetCommitCalls: Long = numGetCommitsCalls } } ================================================ FILE: kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UCE2ESuite.scala ================================================ /* * Copyright (2023) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog import java.util.Optional import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer import io.delta.kernel.{CommitRange, Operation} import io.delta.kernel.Snapshot import io.delta.kernel.Snapshot.ChecksumWriteMode import io.delta.kernel.engine.Engine import io.delta.kernel.internal.SnapshotImpl import io.delta.kernel.internal.util.FileNames import io.delta.kernel.unitycatalog.UCCatalogManagedCommitter import io.delta.kernel.utils.CloseableIterable import io.delta.storage.commit.{Commit, GetCommitsResponse} import InMemoryUCClient.TableData import org.scalatest.funsuite.AnyFunSuite class UCE2ESuite extends AnyFunSuite with UCCatalogManagedTestUtils { import UCE2ESuite._ private val testUcTableId = "testUcTableId" /** Commits some data. Verifies UC is updated as expected. Returns the post-commit snapshot. */ private def writeDataAndVerify( engine: Engine, snapshot: Snapshot, ucClient: InMemoryUCClient, expCommitVersion: Long, expNumCatalogCommits: Long): Snapshot = { val txn = snapshot .buildUpdateTableTransaction("engineInfo", Operation.MANUAL_UPDATE) .build(engine) val result = commitAppendData(engine, txn, seqOfUnpartitionedDataBatch1) val tableData = ucClient.getTableDataElseThrow(testUcTableId) assert(tableData.getMaxRatifiedVersion === expCommitVersion) assert(tableData.getCommits.size === expNumCatalogCommits) result.getPostCommitSnapshot.get() } test("simple case: create, write, publish, load") { withTempDirAndEngine { case (tablePathUnresolved, engine) => val tablePath = engine.getFileSystemClient.resolvePath(tablePathUnresolved) val ucClient = new InMemoryUCClient("ucMetastoreId") val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) // Step 1: CREATE -- v0.json val result0 = ucCatalogManagedClient .buildCreateTableTransaction(testUcTableId, tablePath, testSchema, "test-engine") .build(engine) .commit(engine, CloseableIterable.emptyIterable() /* dataActions */ ) ucClient.insertTableDataAfterCreate(testUcTableId) result0.getPostCommitSnapshot.get().publish(engine) // Should be no-op! // Step 2: WRITE -- v1.uuid.json val postCommitSnapshot1 = writeDataAndVerify( engine, result0.getPostCommitSnapshot.get(), ucClient, expCommitVersion = 1, expNumCatalogCommits = 1) // Step 3: WRITE -- v2.uuid.json val postCommitSnapshot2 = writeDataAndVerify( engine, postCommitSnapshot1, ucClient, expCommitVersion = 2, expNumCatalogCommits = 2) // Step 4a: PUBLISH v1.json and v2.json -- Note that this does NOT update UC postCommitSnapshot2.publish(engine) // Step 4b: VERIFY UC is unchanged by the publish operation val tableData2 = ucClient.getTableDataElseThrow(testUcTableId) assert(tableData2.getMaxRatifiedVersion === 2) assert(tableData2.getCommits.size === 2) postCommitSnapshot2.publish(engine) // idempotent! shouldn't throw // Step 5: WRITE -- v3.uuid.json // Even though v1.json and v2.json are published, snapshotV2 will still have v1.uuid.json and // v2.uuid.json in its LogSegment (since catalog commits take priority). Nonetheless, it will // see that v2 is the maxKnownPublishedDeltaVersion. It will include this information in its // next commit, and UC will then clean up catalog commits v1.uuid.json and v2.uuid.json. val snapshotV2 = loadSnapshot(ucCatalogManagedClient, engine, testUcTableId, tablePath) val logSegmentV2 = snapshotV2.getLogSegment assert(logSegmentV2.getAllCatalogCommits.asScala.map(x => x.getVersion) === Seq(1, 2)) assert(logSegmentV2.getMaxPublishedDeltaVersion.get() === 2) writeDataAndVerify( engine, snapshotV2, ucClient, expCommitVersion = 3, expNumCatalogCommits = 1 // just v3.uuid.json, since v1 and v2 are cleaned up ) // Step 6: LOAD -- should read v0.json, v1.json, v2.json, and v3.uuid.json val snapshotV3 = loadSnapshot(ucCatalogManagedClient, engine, testUcTableId, tablePath) val logSegmentV3 = snapshotV3.getLogSegment assert(snapshotV3.getVersion === 3) assert(logSegmentV3.getAllCatalogCommits.asScala.map(x => x.getVersion) === Seq(3)) assert(logSegmentV3.getMaxPublishedDeltaVersion.get() === 2) } } test("post-publish snapshot is similar to the actual snapshot") { withTempDirAndEngine { case (tablePathUnresolved, engine) => val tablePath = engine.getFileSystemClient.resolvePath(tablePathUnresolved) val ucClient = new InMemoryUCClient("ucMetastoreId") val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) // Step 1: CREATE -- v0.json val result0 = ucCatalogManagedClient .buildCreateTableTransaction(testUcTableId, tablePath, testSchema, "test-engine") .build(engine) .commit(engine, CloseableIterable.emptyIterable() /* dataActions */ ) ucClient.insertTableDataAfterCreate(testUcTableId) result0.getPostCommitSnapshot.get().publish(engine) // Should be no-op! // Step 2: WRITE -- v1.uuid.json val postCommitSnapshot1 = writeDataAndVerify( engine, result0.getPostCommitSnapshot.get(), ucClient, expCommitVersion = 1, expNumCatalogCommits = 1) // Step 3: WRITE -- v2.uuid.json val postCommitSnapshot2 = writeDataAndVerify( engine, postCommitSnapshot1, ucClient, expCommitVersion = 2, expNumCatalogCommits = 2) // Step 4a: PUBLISH v1.json and v2.json -- Note that this does NOT update UC val postPublishSnapshot = postCommitSnapshot2.publish(engine).asInstanceOf[SnapshotImpl] assert(postCommitSnapshot2.getVersion == 2) assert(postCommitSnapshot2.asInstanceOf[SnapshotImpl] .getLogSegment.getMaxPublishedDeltaVersion == Optional.of(0L)) // All versions will be published in the post publish snapshot assert(postPublishSnapshot.getVersion == 2) assert(postPublishSnapshot.getLogSegment.getMaxPublishedDeltaVersion == Optional.of(2L)) // Step 5: Read the latest snapshot from disk. Post-publish snapshot should be similar to it val snapshotV2 = loadSnapshot(ucCatalogManagedClient, engine, testUcTableId, tablePath) assert(postPublishSnapshot.getVersion == snapshotV2.getVersion) assert(postPublishSnapshot.getPath == snapshotV2.getPath) assert(postPublishSnapshot.getLogPath == snapshotV2.getLogPath) assert(postPublishSnapshot.getTimestamp(engine) == snapshotV2.getTimestamp(engine)) assert(postPublishSnapshot.getCommitter.isInstanceOf[UCCatalogManagedCommitter]) assert(postPublishSnapshot.getActiveDomainMetadataMap == snapshotV2.getActiveDomainMetadataMap) assert(postPublishSnapshot.getSchema.equivalent(snapshotV2.getSchema)) assert(postPublishSnapshot.getMetadata == snapshotV2.getMetadata) assert(postPublishSnapshot.getProtocol == snapshotV2.getProtocol) assert(postPublishSnapshot.getPartitionColumnNames == snapshotV2.getPartitionColumnNames) val postPublishLogSegment = postPublishSnapshot.getLogSegment val logSegmentV2 = snapshotV2.getLogSegment assert(logSegmentV2.getAllCatalogCommits.asScala.map(x => x.getVersion) === Seq(1, 2)) assert(logSegmentV2.getMaxPublishedDeltaVersion.get() === 2) // Step 5: Use postPublish snapshot to write -- v3.uuid.json writeDataAndVerify( engine, postPublishSnapshot, ucClient, expCommitVersion = 3, expNumCatalogCommits = 1) // Step 6: LOAD -- should read v0.json, v1.json, v2.json, and v3.uuid.json val snapshotV3 = loadSnapshot(ucCatalogManagedClient, engine, testUcTableId, tablePath) val logSegmentV3 = snapshotV3.getLogSegment assert(snapshotV3.getVersion === 3) assert(logSegmentV3.getAllCatalogCommits.asScala.map(x => x.getVersion) === Seq(3)) assert(logSegmentV3.getMaxPublishedDeltaVersion.get() === 2) } } test("can load snapshot for table with CRC files for unpublished versions") { withTempDirAndEngine { case (tablePathUnresolved, engine) => // ===== GIVEN ===== val tablePath = engine.getFileSystemClient.resolvePath(tablePathUnresolved) val ucClient = new InMemoryUCClient("ucMetastoreId") val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) // CREATE -- v0.json val result0 = ucCatalogManagedClient .buildCreateTableTransaction(testUcTableId, tablePath, testSchema, "test-engine") .build(engine) .commit(engine, CloseableIterable.emptyIterable()) ucClient.insertTableDataAfterCreate(testUcTableId) var currentSnapshot = result0.getPostCommitSnapshot.get() // INSERT -- Empty commits with CRC generation for (_ <- 1 to 3) { val txn = currentSnapshot .buildUpdateTableTransaction("engineInfo", Operation.MANUAL_UPDATE) .build(engine) val result = txn.commit(engine, CloseableIterable.emptyIterable()) currentSnapshot = result.getPostCommitSnapshot.get() currentSnapshot.writeChecksum(engine, ChecksumWriteMode.SIMPLE) } // ===== WHEN ===== val freshSnapshot = loadSnapshot(ucCatalogManagedClient, engine, testUcTableId, tablePath) // ===== THEN ===== val logSegment = freshSnapshot.getLogSegment assert(freshSnapshot.getVersion === 3) assert(logSegment.getAllCatalogCommits.asScala.map(_.getVersion) === Seq(1, 2, 3)) assert(logSegment.getMaxPublishedDeltaVersion.get() === 0) val checksumVersion = FileNames.checksumVersion(logSegment.getLastSeenChecksum.get.getPath) assert(checksumVersion === 3) } } test("don't read versions past maxCatalogVersion even if they exist on filesystem") { withTempDirAndEngine { case (tablePathUnresolved, engine) => val tablePath = engine.getFileSystemClient.resolvePath(tablePathUnresolved) val ucClient = new ConfigurableMaxVersionUCClient() val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) // Step 1: CREATE -- v0.json val result0 = ucCatalogManagedClient .buildCreateTableTransaction(testUcTableId, tablePath, testSchema, "test-engine") .build(engine) .commit(engine, CloseableIterable.emptyIterable()) ucClient.insertTableDataAfterCreate(testUcTableId) // Step 2: WRITE and commit data up to version 2 val postCommitSnapshot1 = writeDataAndVerify( engine, result0.getPostCommitSnapshot.get(), ucClient, expCommitVersion = 1, expNumCatalogCommits = 1) val postCommitSnapshot2 = writeDataAndVerify( engine, postCommitSnapshot1, ucClient, expCommitVersion = 2, expNumCatalogCommits = 2) // Step 3: PUBLISH v1.json and v2.json to the filesystem postCommitSnapshot2.publish(engine) // Step 4: Configure the UC client to limit maxRatifiedVersion to 1 ucClient.setMaxVersionLimit(1) // Step 5: Load snapshot with UC client that limits maxRatifiedVersion to 1 val snapshot = loadSnapshot(ucCatalogManagedClient, engine, testUcTableId, tablePath) // Step 6: Verify that snapshot is at version 1, not version 2 assert( snapshot.getVersion === 1, "Snapshot should be at version 1, not reading beyond maxCatalogVersion") // Verify the log segment only contains commits up to version 1 assert( snapshot.getLogSegment.getMaxPublishedDeltaVersion.get() === 1, "Should recognize published version 1 but not go beyond it") } } test("CommitRange respects maxCatalogVersion") { withTempDirAndEngine { case (tablePathUnresolved, engine) => val tablePath = engine.getFileSystemClient.resolvePath(tablePathUnresolved) val ucClient = new ConfigurableMaxVersionUCClient() val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) // Step 1: CREATE -- v0.json val result0 = ucCatalogManagedClient .buildCreateTableTransaction(testUcTableId, tablePath, testSchema, "test-engine") .build(engine) .commit(engine, CloseableIterable.emptyIterable()) ucClient.insertTableDataAfterCreate(testUcTableId) // Step 2: WRITE multiple versions val postCommitSnapshot1 = writeDataAndVerify( engine, result0.getPostCommitSnapshot.get(), ucClient, expCommitVersion = 1, expNumCatalogCommits = 1) val postCommitSnapshot2 = writeDataAndVerify( engine, postCommitSnapshot1, ucClient, expCommitVersion = 2, expNumCatalogCommits = 2) val postCommitSnapshot3 = writeDataAndVerify( engine, postCommitSnapshot2, ucClient, expCommitVersion = 3, expNumCatalogCommits = 3) // Step 3: Publish all versions postCommitSnapshot3.publish(engine) // Step 4: Load CommitRange with end boundary (should go from 0 to 3) val commitRange1: CommitRange = ucCatalogManagedClient.loadCommitRange( engine, testUcTableId, tablePath, Optional.of(0), emptyLongOpt, emptyLongOpt, emptyLongOpt) assert(commitRange1.getStartVersion === 0) assert(commitRange1.getEndVersion === 3, "Should respect maxCatalogVersion of 3") // Step 5: Configure UC client to limit maxRatifiedVersion to 2 ucClient.setMaxVersionLimit(2) // Step 6: Load CommitRange again (should now be limited to version 2) val commitRange2: CommitRange = ucCatalogManagedClient.loadCommitRange( engine, testUcTableId, tablePath, Optional.of(0), emptyLongOpt, emptyLongOpt, emptyLongOpt) assert(commitRange2.getStartVersion === 0) assert(commitRange2.getEndVersion === 2, "Should respect maxCatalogVersion of 2") // Step 8: Load CommitRange with start version at maxCatalogVersion (should work) val commitRange3: CommitRange = ucCatalogManagedClient.loadCommitRange( engine, testUcTableId, tablePath, Optional.of(2), emptyLongOpt, emptyLongOpt, emptyLongOpt) assert(commitRange3.getStartVersion === 2) assert(commitRange3.getEndVersion === 2) } } } object UCE2ESuite { // Custom UCClient that can configure maxRatifiedVersion for testing withMaxCatalogVersion class ConfigurableMaxVersionUCClient extends InMemoryUCClient("ucMetastoreId") { @volatile private var maxVersionLimit: Option[Long] = None def setMaxVersionLimit(limit: Long): Unit = { maxVersionLimit = Some(limit) } override def getCommits( tableId: String, tableUri: java.net.URI, startVersion: Optional[java.lang.Long], endVersion: Optional[java.lang.Long]): GetCommitsResponse = { val response = super.getCommits(tableId, tableUri, startVersion, endVersion) maxVersionLimit match { case Some(limit) => // Filter commits and limit maxRatifiedVersion val filteredCommits = response.getCommits.asScala.filter(_.getVersion <= limit) new GetCommitsResponse(filteredCommits.asJava, limit) case None => response } } } } ================================================ FILE: kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UCPublishingSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog import java.io.IOException import io.delta.kernel.commit.PublishFailedException import io.delta.kernel.internal.files.ParsedCatalogCommitData import io.delta.kernel.internal.util.FileNames import io.delta.kernel.test.{BaseMockFileSystemClient, MockFileSystemClientUtils, TestFixtures, VectorTestUtils} import org.scalatest.funsuite.AnyFunSuite class UCPublishingSuite extends AnyFunSuite with UCCatalogManagedTestUtils with TestFixtures with VectorTestUtils with MockFileSystemClientUtils { private def createCommitter(tablePath: String): UCCatalogManagedCommitter = { val ucClient = new InMemoryUCClient("ucMetastoreId") new UCCatalogManagedCommitter(ucClient, "testUcTableId", tablePath) } private def toFile(path: String): java.io.File = { if (path.startsWith("file:")) { new java.io.File(new java.net.URI(path)) } else { new java.io.File(path) } } private def readFile(path: String): String = { scala.io.Source.fromFile(toFile(path)).getLines().mkString("\n") } private def assertFileExists(path: String): Unit = { assert(toFile(path).exists(), s"File should exist: $path") } /** * Helper to create a staged commit file and return its ParsedCatalogCommitData. * Just writes the file directly without using the committer. */ private def writeStagedCatalogCommit( logPath: String, version: Long, content: String = ""): ParsedCatalogCommitData = { val stagedPath = FileNames.stagedCommitFile(logPath, version) defaultEngine.getJsonHandler.writeJsonFileAtomically( stagedPath, getSingleElementRowIter(content), true /* overwrite */ ) val fileStatus = defaultEngine.getFileSystemClient.getFileStatus(stagedPath) ParsedCatalogCommitData.forFileStatus(fileStatus) } test("publish: throws on null inputs") { val committer = createCommitter(baseTestTablePath) val publishMetadata = createPublishMetadata( snapshotVersion = 1, logPath = baseTestLogPath, catalogCommits = List(createStagedCatalogCommit(1, baseTestLogPath))) assertThrows[NullPointerException] { committer.publish(null, publishMetadata) } assertThrows[NullPointerException] { committer.publish(defaultEngine, null) } } test("publish: throws UnsupportedOperationException for inline commits") { withTempDirAndAllDeltaSubDirs { case (tablePath, logPath) => // ===== GIVEN ===== val inlineCatalogCommit = ParsedCatalogCommitData.forInlineData( 1L, emptyColumnarBatch) val committer = createCommitter(tablePath) val publishMetadata = createPublishMetadata( snapshotVersion = 1, logPath = logPath, catalogCommits = List(inlineCatalogCommit)) // ===== WHEN ===== val ex = intercept[UnsupportedOperationException] { committer.publish(defaultEngine, publishMetadata) } // ===== THEN ===== assert(ex.getMessage.contains("Publishing inline catalog commits is not yet supported")) } } test("publish: multiple catalog commits successfully") { withTempDirAndAllDeltaSubDirs { case (tablePath, logPath) => // ===== GIVEN ===== val catalogCommits = List( writeStagedCatalogCommit(logPath, 1, "COMMIT_V1"), writeStagedCatalogCommit(logPath, 2, "COMMIT_V2"), writeStagedCatalogCommit(logPath, 3, "COMMIT_V3")) val committer = createCommitter(tablePath) val publishMetadata = createPublishMetadata( snapshotVersion = 3, logPath = logPath, catalogCommits = catalogCommits) // ===== WHEN ===== committer.publish(defaultEngine, publishMetadata) // ===== THEN ===== assert(readFile(FileNames.deltaFile(logPath, 1)).contains("COMMIT_V1")) assert(readFile(FileNames.deltaFile(logPath, 2)).contains("COMMIT_V2")) assert(readFile(FileNames.deltaFile(logPath, 3)).contains("COMMIT_V3")) } } test("publish: return a published snapshot") {} test("publish: does not overwrite existing published files") { withTempDirAndAllDeltaSubDirs { case (tablePath, logPath) => // ===== GIVEN ===== val catalogCommit = writeStagedCatalogCommit(logPath, 1, "TEST_IDEMPOTENT_PUBLISH") val committer = createCommitter(tablePath) val publishMetadata = createPublishMetadata( snapshotVersion = 1, logPath = logPath, catalogCommits = List(catalogCommit)) // ===== WHEN ===== // Publish once committer.publish(defaultEngine, publishMetadata) val publishedTimestamp1 = defaultEngine .getFileSystemClient.getFileStatus(FileNames.deltaFile(logPath, 1)).getModificationTime // Publish again - should succeed but not overwrite existing file committer.publish(defaultEngine, publishMetadata) val publishedTimestamp2 = defaultEngine .getFileSystemClient.getFileStatus(FileNames.deltaFile(logPath, 1)).getModificationTime // ===== THEN ===== assert(publishedTimestamp1 === publishedTimestamp2) } } test("publish: creates published file at correct location with identical content") { withTempDirAndAllDeltaSubDirs { case (tablePath, logPath) => // ===== GIVEN ===== val catalogCommit = writeStagedCatalogCommit(logPath, 1, "VERSION_1") val committer = createCommitter(tablePath) val publishMetadata = createPublishMetadata( snapshotVersion = 1, logPath = logPath, catalogCommits = List(catalogCommit)) // ===== WHEN ===== committer.publish(defaultEngine, publishMetadata) // ===== THEN ===== val stagedPath = catalogCommit.getFileStatus.getPath val publishedPath = FileNames.deltaFile(logPath, 1) // Verify staged file still exists (publish doesn't delete source) assertFileExists(stagedPath) // Verify published file exists at correct location assertFileExists(publishedPath) assert(FileNames.isPublishedDeltaFile(publishedPath)) // Verify content was copied correctly assert(readFile(stagedPath) === readFile(publishedPath)) } } test("publish: throws PublishFailedException on IOException") { withTempDirAndAllDeltaSubDirs { case (tablePath, logPath) => // ===== GIVEN ===== val catalogCommit = writeStagedCatalogCommit(logPath, 1, "TEST_EXCEPTION") val throwingEngine = mockEngine(fileSystemClient = new BaseMockFileSystemClient { override def copyFileAtomically( srcPath: String, destPath: String, overwrite: Boolean): Unit = { throw new IOException("Network failure during copy") } }) val committer = createCommitter(tablePath) val publishMetadata = createPublishMetadata( snapshotVersion = 1, logPath = logPath, catalogCommits = List(catalogCommit)) // ===== WHEN ===== val ex = intercept[PublishFailedException] { committer.publish(throwingEngine, publishMetadata) } // ===== THEN ===== assert(ex.getMessage.contains("Failed to publish version 1")) assert(ex.getMessage.contains("Network failure during copy")) assert(ex.getCause.isInstanceOf[IOException]) } } } ================================================ FILE: kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UcCommitTelemetrySuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog import java.util.Optional import scala.collection.mutable.ArrayBuffer import io.delta.kernel.Operation import io.delta.kernel.commit.{CommitFailedException, CommitMetadata} import io.delta.kernel.data.Row import io.delta.kernel.defaults.engine.DefaultEngine import io.delta.kernel.exceptions.MaxCommitRetryLimitReachedException import io.delta.kernel.test.{BaseMockJsonHandler, MockFileSystemClientUtils} import io.delta.kernel.unitycatalog.InMemoryUCClient.TableData import io.delta.kernel.unitycatalog.metrics.UcCommitTelemetry import io.delta.kernel.utils.{CloseableIterable, CloseableIterator} import io.delta.storage.commit.Commit import org.apache.hadoop.conf.Configuration import org.scalatest.funsuite.AnyFunSuite class UcCommitTelemetrySuite extends AnyFunSuite with UCCatalogManagedTestUtils with MockFileSystemClientUtils { test("commit metrics for CREATE and WRITE operations") { withTempDirAndEngine { case (tablePathUnresolved, _) => val reporter = new CapturingMetricsReporter[UcCommitTelemetry#Report] val engine = createEngineWithMetricsCapture(reporter) val tablePath = engine.getFileSystemClient.resolvePath(tablePathUnresolved) val (ucClient, ucCatalogManagedClient) = createUCClientAndCatalogManagedClient() // CREATE -- v0.json val result0 = ucCatalogManagedClient .buildCreateTableTransaction("testUcTableId", tablePath, testSchema, "test-engine") .build(engine) .commit(engine, CloseableIterable.emptyIterable() /* dataActions */ ) ucClient.insertTableDataAfterCreate("testUcTableId") // Verify CREATE metrics assert(reporter.reports.size === 1) val createReport = reporter.reports.head assert(createReport.operationType === "UcCommit") assert(createReport.ucTableId === "testUcTableId") assert(createReport.ucTablePath === tablePath) assert(createReport.commitVersion === 0) assert(createReport.commitType === CommitMetadata.CommitType.CATALOG_CREATE) assert(createReport.exception.isEmpty) val createMetrics = createReport.metrics assert(createMetrics.totalCommitDurationNs > 0) assert(createMetrics.writeCommitFileDurationNs > 0) assert(createMetrics.commitToUcServerDurationNs === 0) reporter.reports.clear() // WRITE -- v1.uuid.json result0 .getPostCommitSnapshot .get() .buildUpdateTableTransaction("engineInfo", Operation.MANUAL_UPDATE) .build(engine) .commit(engine, CloseableIterable.emptyIterable()) // Verify WRITE metrics assert(reporter.reports.size === 1) val writeReport = reporter.reports.head assert(writeReport.operationType === "UcCommit") assert(writeReport.ucTableId === "testUcTableId") assert(writeReport.ucTablePath === tablePath) assert(writeReport.commitVersion === 1) assert(writeReport.commitType === CommitMetadata.CommitType.CATALOG_WRITE) assert(writeReport.exception.isEmpty) val writeMetrics = writeReport.metrics assert(writeMetrics.totalCommitDurationNs > 0) assert(writeMetrics.writeCommitFileDurationNs > 0) assert(writeMetrics.commitToUcServerDurationNs > 0) assert( writeMetrics.totalCommitDurationNs >= writeMetrics.writeCommitFileDurationNs + writeMetrics.commitToUcServerDurationNs) assert(writeReport.reportUUID != createReport.reportUUID) } } test("telemetry captures exceptions during commit") { withTempDirAndEngine { case (tablePathUnresolved, engine) => // ===== GIVEN ===== val reporter = new CapturingMetricsReporter[UcCommitTelemetry#Report] val throwingJsonHandler = new BaseMockJsonHandler { override def writeJsonFileAtomically( path: String, data: CloseableIterator[Row], overwrite: Boolean): Unit = throw new java.io.IOException("Simulated network failure") } val throwingEngineWithReporter = new DefaultEngine( new io.delta.kernel.defaults.engine.hadoopio.HadoopFileIO(new Configuration())) { override def getJsonHandler = throwingJsonHandler override def getMetricsReporters = java.util.Arrays.asList(reporter) } val tablePath = engine.getFileSystemClient.resolvePath(tablePathUnresolved) val (_, ucCatalogManagedClient) = createUCClientAndCatalogManagedClient() // ===== WHEN ===== intercept[MaxCommitRetryLimitReachedException] { ucCatalogManagedClient .buildCreateTableTransaction("testUcTableId", tablePath, testSchema, "test-engine") .withMaxRetries(0) .build(throwingEngineWithReporter) .commit(throwingEngineWithReporter, CloseableIterable.emptyIterable()) } // ===== THEN ===== assert(reporter.reports.size === 1) val report = reporter.reports.head assert(report.operationType === "UcCommit") assert(report.ucTableId === "testUcTableId") assert(report.commitVersion === 0) assert(report.commitType === CommitMetadata.CommitType.CATALOG_CREATE) assert(report.exception.isPresent) val exceptionString = report.exception.get().toString assert(exceptionString.contains("CommitFailedException")) assert(exceptionString.contains("Simulated network failure")) } } test("JSON serialization: success + create (version == 0)") { val commitMetadata = createCommitMetadata( version = 0, logPath = baseTestLogPath, readPandMOpt = Optional.empty(), newProtocolOpt = Optional.of(protocolWithCatalogManagedSupport), newMetadataOpt = Optional.of(basicPartitionedMetadata)) val telemetry = new UcCommitTelemetry("testUcTableId", "ucTablePath", commitMetadata) telemetry.getMetricsCollector.totalCommitTimer.record(200) telemetry.getMetricsCollector.writeCommitFileTimer.record(200) // Note: commitToUcServerTimer is not invoked for CREATE operations val report = telemetry.createSuccessReport() // scalastyle:off line.size.limit val expectedJson = s""" |{"operationType":"UcCommit", |"reportUUID":"${report.reportUUID}", |"ucTableId":"testUcTableId", |"ucTablePath":"ucTablePath", |"commitVersion":0, |"commitType":"CATALOG_CREATE", |"metrics":{"totalCommitDurationNs":200,"writeCommitFileDurationNs":200,"commitToUcServerDurationNs":0}, |"exception":null} |""".stripMargin.replaceAll("\n", "") // scalastyle:on line.size.limit assert(report.toJson() === expectedJson) } test("JSON serialization: success + update (version >= 1)") { val commitMetadata = catalogManagedWriteCommitMetadata(version = 5) val telemetry = new UcCommitTelemetry("testUcTableId", "ucTablePath", commitMetadata) telemetry.getMetricsCollector.totalCommitTimer.record(300) telemetry.getMetricsCollector.writeCommitFileTimer.record(200) telemetry.getMetricsCollector.commitToUcServerTimer.record(100) val report = telemetry.createSuccessReport() // scalastyle:off line.size.limit val expectedJson = s""" |{"operationType":"UcCommit", |"reportUUID":"${report.reportUUID}", |"ucTableId":"testUcTableId", |"ucTablePath":"ucTablePath", |"commitVersion":5, |"commitType":"CATALOG_WRITE", |"metrics":{"totalCommitDurationNs":300,"writeCommitFileDurationNs":200,"commitToUcServerDurationNs":100}, |"exception":null} |""".stripMargin.replaceAll("\n", "") // scalastyle:on line.size.limit assert(report.toJson() === expectedJson) } test("JSON serialization: fail + update") { val commitMetadata = catalogManagedWriteCommitMetadata(version = 3) val telemetry = new UcCommitTelemetry("testUcTableId", "ucTablePath", commitMetadata) telemetry.getMetricsCollector.totalCommitTimer.record(300) telemetry.getMetricsCollector.writeCommitFileTimer.record(200) telemetry.getMetricsCollector.commitToUcServerTimer.record(100) val exception = new CommitFailedException(false, false, "errMsg") // notRetryable, notConflict val report = telemetry.createFailureReport(exception) // scalastyle:off line.size.limit val expectedJson = s""" |{"operationType":"UcCommit", |"reportUUID":"${report.reportUUID}", |"ucTableId":"testUcTableId", |"ucTablePath":"ucTablePath", |"commitVersion":3, |"commitType":"CATALOG_WRITE", |"metrics":{"totalCommitDurationNs":300,"writeCommitFileDurationNs":200,"commitToUcServerDurationNs":100}, |"exception":"io.delta.kernel.commit.CommitFailedException: retryable=false, conflict=false, msg=errMsg"} |""".stripMargin.replaceAll("\n", "") // scalastyle:on line.size.limit assert(report.toJson() === expectedJson) } } ================================================ FILE: kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UcLoadSnapshotTelemetrySuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog import java.util.Optional import io.delta.kernel.engine.Engine import io.delta.kernel.test.MockFileSystemClientUtils import io.delta.kernel.unitycatalog.metrics.UcLoadSnapshotTelemetry import io.delta.kernel.utils.CloseableIterable import org.scalatest.funsuite.AnyFunSuite class UcLoadSnapshotTelemetrySuite extends AnyFunSuite with UCCatalogManagedTestUtils with MockFileSystemClientUtils { /** * Helper to set up a table with v0 published and v1, v2 as staged commits. * * @return timestamp between v1 and v2 creation */ private def setupTableWithCommits( engine: Engine, tablePath: String, ucClient: InMemoryUCClient, ucCatalogManagedClient: UCCatalogManagedClient): Long = { val result0 = ucCatalogManagedClient .buildCreateTableTransaction("testUcTableId", tablePath, testSchema, "test-engine") .build(engine) .commit(engine, CloseableIterable.emptyIterable()) ucClient.insertTableDataAfterCreate("testUcTableId") val result1 = result0.getPostCommitSnapshot.get() .buildUpdateTableTransaction("engineInfo", io.delta.kernel.Operation.MANUAL_UPDATE) .build(engine) .commit(engine, CloseableIterable.emptyIterable()) val timestampBetweenV1AndV2 = System.currentTimeMillis() Thread.sleep(100) // Ensure v2 timestamp is sufficiently after our captured timestamp val result2 = result1.getPostCommitSnapshot.get() .buildUpdateTableTransaction("engineInfo", io.delta.kernel.Operation.MANUAL_UPDATE) .build(engine) .commit(engine, CloseableIterable.emptyIterable()) timestampBetweenV1AndV2 } test("snapshot loading metrics for latest version") { withTempDirAndEngine { case (tablePathUnresolved, _) => // ===== GIVEN ===== val reporter = new CapturingMetricsReporter[UcLoadSnapshotTelemetry#Report] val engine = createEngineWithMetricsCapture(reporter) val tablePath = engine.getFileSystemClient.resolvePath(tablePathUnresolved) val (ucClient, ucCatalogManagedClient) = createUCClientAndCatalogManagedClient() setupTableWithCommits(engine, tablePath, ucClient, ucCatalogManagedClient) reporter.reports.clear() // ===== WHEN ===== ucCatalogManagedClient.loadSnapshot( engine, "testUcTableId", tablePath, Optional.empty(), Optional.empty()) // ===== THEN ===== assert(reporter.reports.size === 1) val report = reporter.reports.head assert(report.operationType === "UcLoadSnapshot") assert(report.ucTableId === "testUcTableId") assert(report.ucTablePath === tablePath) assert(report.versionOpt.isEmpty) assert(report.timestampOpt.isEmpty) assert(!report.exception.isPresent) val metrics = report.metrics assert(metrics.totalLoadSnapshotDurationNs > 0) assert(metrics.getCommitsDurationNs > 0) assert(metrics.numCatalogCommits === 2) // v1.uuid.json and v2.uuid.json assert(metrics.kernelSnapshotBuildDurationNs > 0) assert(metrics.loadLatestSnapshotForTimestampTimeTravelDurationNs === 0) assert(metrics.resolvedSnapshotVersion === 2) // v2 is the latest assert( metrics.totalLoadSnapshotDurationNs >= metrics.getCommitsDurationNs + metrics.kernelSnapshotBuildDurationNs) } } test("snapshot loading metrics with timestamp") { withTempDirAndEngine { case (tablePathUnresolved, _) => // ===== GIVEN ===== val reporter = new CapturingMetricsReporter[UcLoadSnapshotTelemetry#Report] val engine = createEngineWithMetricsCapture(reporter) val tablePath = engine.getFileSystemClient.resolvePath(tablePathUnresolved) val (ucClient, ucCatalogManagedClient) = createUCClientAndCatalogManagedClient() val timestampBetweenV1AndV2 = setupTableWithCommits(engine, tablePath, ucClient, ucCatalogManagedClient) reporter.reports.clear() // Time travel to timestamp between v1 and v2 - should resolve to v1 ucCatalogManagedClient.loadSnapshot( engine, "testUcTableId", tablePath, Optional.empty(), Optional.of(timestampBetweenV1AndV2)) // Verify snapshot loading metrics assert(reporter.reports.size === 1) val report = reporter.reports.head assert(report.operationType === "UcLoadSnapshot") assert(report.ucTableId === "testUcTableId") assert(report.ucTablePath === tablePath) assert(report.versionOpt.isEmpty) assert(report.timestampOpt.isPresent) assert(report.timestampOpt.get() === timestampBetweenV1AndV2) assert(!report.exception.isPresent) val metrics = report.metrics assert(metrics.totalLoadSnapshotDurationNs > 0) assert(metrics.getCommitsDurationNs > 0) assert(metrics.numCatalogCommits === 2) // v1.uuid.json and v2.uuid.json from loading latest assert(metrics.kernelSnapshotBuildDurationNs > 0) assert(metrics.loadLatestSnapshotForTimestampTimeTravelDurationNs > 0) assert(metrics.resolvedSnapshotVersion === 1) // v1, since timestamp is between v1 and v2 } } test("telemetry captures exceptions during snapshot loading") { withTempDirAndEngine { case (tablePathUnresolved, engine) => val reporter = new CapturingMetricsReporter[UcLoadSnapshotTelemetry#Report] val engineWithReporter = new io.delta.kernel.defaults.engine.DefaultEngine( new io.delta.kernel.defaults.engine.hadoopio.HadoopFileIO( new org.apache.hadoop.conf.Configuration())) { override def getMetricsReporters = java.util.Arrays.asList(reporter) } val tablePath = engine.getFileSystemClient.resolvePath(tablePathUnresolved) val (_, ucCatalogManagedClient) = createUCClientAndCatalogManagedClient() intercept[RuntimeException] { // Try to load snapshot from non-existent table ucCatalogManagedClient.loadSnapshot( engineWithReporter, "nonExistentTableId", tablePath, Optional.empty(), Optional.empty()) } assert(reporter.reports.size === 1) val report = reporter.reports.head assert(report.operationType === "UcLoadSnapshot") assert(report.ucTableId === "nonExistentTableId") assert(report.exception.isPresent) assert(report.metrics.numCatalogCommits === -1) // Not set due to failure assert(report.metrics.resolvedSnapshotVersion === -1) // Not set due to failure } } test("JSON serialization: success report for latest version") { val telemetry = new UcLoadSnapshotTelemetry( "testUcTableId", "ucTablePath", Optional.empty(), // versionOpt Optional.empty() // timestampOpt ) telemetry.getMetricsCollector.totalSnapshotLoadTimer.record(500) telemetry.getMetricsCollector.getCommitsTimer.record(200) telemetry.getMetricsCollector.kernelSnapshotBuildTimer.record(250) telemetry.getMetricsCollector.setNumCatalogCommits(5) telemetry.getMetricsCollector.setResolvedSnapshotVersion(3) val report = telemetry.createSuccessReport() // scalastyle:off line.size.limit val expectedJson = s""" |{"operationType":"UcLoadSnapshot", |"reportUUID":"${report.reportUUID}", |"ucTableId":"testUcTableId", |"ucTablePath":"ucTablePath", |"versionOpt":null, |"timestampOpt":null, |"metrics":{"totalLoadSnapshotDurationNs":500,"getCommitsDurationNs":200,"numCatalogCommits":5,"kernelSnapshotBuildDurationNs":250,"loadLatestSnapshotForTimestampTimeTravelDurationNs":0,"resolvedSnapshotVersion":3}, |"exception":null} |""".stripMargin.replaceAll("\n", "") // scalastyle:on line.size.limit assert(report.toJson() === expectedJson) } test("JSON serialization: failure report") { val telemetry = new UcLoadSnapshotTelemetry( "testUcTableId", "ucTablePath", Optional.empty(), // versionOpt Optional.of(123456789L) // timestampOpt ) telemetry.getMetricsCollector.totalSnapshotLoadTimer.record(100) telemetry.getMetricsCollector.getCommitsTimer.record(100) val exception = new RuntimeException("Failed to load snapshot") val report = telemetry.createFailureReport(exception) // scalastyle:off line.size.limit val expectedJson = s""" |{"operationType":"UcLoadSnapshot", |"reportUUID":"${report.reportUUID}", |"ucTableId":"testUcTableId", |"ucTablePath":"ucTablePath", |"versionOpt":null, |"timestampOpt":123456789, |"metrics":{"totalLoadSnapshotDurationNs":100,"getCommitsDurationNs":100,"numCatalogCommits":-1,"kernelSnapshotBuildDurationNs":0,"loadLatestSnapshotForTimestampTimeTravelDurationNs":0,"resolvedSnapshotVersion":-1}, |"exception":"java.lang.RuntimeException: Failed to load snapshot"} |""".stripMargin.replaceAll("\n", "") // scalastyle:on line.size.limit assert(report.toJson() === expectedJson) } } ================================================ FILE: kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UcPublishTelemetrySuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog import scala.collection.mutable.ArrayBuffer import io.delta.kernel.Operation import io.delta.kernel.commit.PublishFailedException import io.delta.kernel.test.MockFileSystemClientUtils import io.delta.kernel.unitycatalog.InMemoryUCClient.TableData import io.delta.kernel.unitycatalog.metrics.UcPublishTelemetry import io.delta.kernel.utils.CloseableIterable import io.delta.storage.commit.Commit import org.scalatest.funsuite.AnyFunSuite class UcPublishTelemetrySuite extends AnyFunSuite with UCCatalogManagedTestUtils with MockFileSystemClientUtils { test("publish metrics for successful publish operations") { withTempDirAndEngine { case (tablePathUnresolved, _) => // ===== GIVEN ===== val reporter = new CapturingMetricsReporter[UcPublishTelemetry#Report] val engine = createEngineWithMetricsCapture(reporter) val tablePath = engine.getFileSystemClient.resolvePath(tablePathUnresolved) val (ucClient, ucCatalogManagedClient) = createUCClientAndCatalogManagedClient() val result0 = ucCatalogManagedClient .buildCreateTableTransaction("testUcTableId", tablePath, testSchema, "test-engine") .build(engine) .commit(engine, CloseableIterable.emptyIterable()) ucClient.insertTableDataAfterCreate("testUcTableId") val resultV1 = result0.getPostCommitSnapshot.get() .buildUpdateTableTransaction("engineInfo", Operation.MANUAL_UPDATE) .build(engine) .commit(engine, CloseableIterable.emptyIterable()) val resultV2 = resultV1.getPostCommitSnapshot.get() .buildUpdateTableTransaction("engineInfo", Operation.MANUAL_UPDATE) .build(engine) .commit(engine, CloseableIterable.emptyIterable()) reporter.reports.clear() // ===== WHEN ===== resultV1.getPostCommitSnapshot.get().publish(engine) // publishes 01.uuid.json -> 01.json resultV2.getPostCommitSnapshot.get().publish(engine) // publishes 02.uuid.json -> 02.json // ===== THEN ===== assert(reporter.reports.size === 2) val firstPublish = reporter.reports(0) assert(firstPublish.operationType === "UcPublish") assert(firstPublish.ucTableId === "testUcTableId") assert(firstPublish.snapshotVersion === 1) assert(firstPublish.numCommitsToPublish === 1) assert(firstPublish.metrics.numCommitsPublished === 1) assert(firstPublish.metrics.numCommitsAlreadyPublished === 0) val secondPublish = reporter.reports(1) assert(secondPublish.operationType === "UcPublish") assert(secondPublish.ucTableId === "testUcTableId") assert(secondPublish.snapshotVersion === 2) assert(secondPublish.numCommitsToPublish === 2) // Both 01.uuid.json and 02.uuid.json assert(secondPublish.metrics.numCommitsPublished === 1) // Only 02.uuid.json assert(secondPublish.metrics.numCommitsAlreadyPublished === 1) // 01.uuid.json was already! } } test("JSON serialization: success report") { val telemetry = new UcPublishTelemetry("testUcTableId", "ucTablePath", 5, 3) val collector = telemetry.getMetricsCollector collector.totalPublishTimer.record(500) collector.incrementCommitsPublished() collector.incrementCommitsPublished() collector.incrementCommitsAlreadyPublished() val report = telemetry.createSuccessReport() // scalastyle:off line.size.limit val expectedJson = s""" |{"operationType":"UcPublish", |"reportUUID":"${report.reportUUID}", |"ucTableId":"testUcTableId", |"ucTablePath":"ucTablePath", |"snapshotVersion":5, |"numCommitsToPublish":3, |"metrics":{"totalPublishDurationNs":500,"numCommitsPublished":2,"numCommitsAlreadyPublished":1}, |"exception":null} |""".stripMargin.replaceAll("\n", "") // scalastyle:on line.size.limit assert(report.toJson() === expectedJson) } test("JSON serialization: failure report") { val telemetry = new UcPublishTelemetry("testUcTableId", "ucTablePath", 3, 2) val collector = telemetry.getMetricsCollector collector.totalPublishTimer.record(300) collector.incrementCommitsPublished() val exception = new PublishFailedException("Failed to publish") val report = telemetry.createFailureReport(exception) // scalastyle:off line.size.limit val expectedJson = s""" |{"operationType":"UcPublish", |"reportUUID":"${report.reportUUID}", |"ucTableId":"testUcTableId", |"ucTablePath":"ucTablePath", |"snapshotVersion":3, |"numCommitsToPublish":2, |"metrics":{"totalPublishDurationNs":300,"numCommitsPublished":1,"numCommitsAlreadyPublished":0}, |"exception":"io.delta.kernel.commit.PublishFailedException: Failed to publish"} |""".stripMargin.replaceAll("\n", "") // scalastyle:on line.size.limit assert(report.toJson() === expectedJson) } } ================================================ FILE: kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/UnityCatalogUtilsSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog import scala.jdk.CollectionConverters._ import io.delta.kernel.expressions.Column import io.delta.kernel.internal.SnapshotImpl import io.delta.kernel.internal.fs.Path import io.delta.kernel.test.MockSnapshotUtils import io.delta.kernel.transaction.DataLayoutSpec import io.delta.kernel.types.{IntegerType, StringType, StructType} import io.delta.kernel.utils.CloseableIterable import org.scalatest.funsuite.AnyFunSuite class UnityCatalogUtilsSuite extends AnyFunSuite with UCCatalogManagedTestUtils with MockSnapshotUtils { private val testUcTableId = "testUcTableId" test("getPropertiesForCreate: throws when snapshot is not version 0") { val mockSnapshotV1 = getMockSnapshot(new Path("/fake/table/path"), latestVersion = 1) val exMsg = intercept[IllegalArgumentException] { UnityCatalogUtils.getPropertiesForCreate(defaultEngine, mockSnapshotV1) }.getMessage assert(exMsg.contains("Expected a snapshot at version 0, but got a snapshot at version 1")) } test("getPropertiesForCreate: handles all cases together") { withTempDirAndEngine { case (tablePathUnresolved, engine) => val ucClient = new InMemoryUCClient("ucMetastoreId") val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) val tablePath = engine.getFileSystemClient.resolvePath(tablePathUnresolved) val testSchema = new StructType() .add("id", IntegerType.INTEGER) .add( "address", new StructType() .add("city", StringType.STRING) .add("state", StringType.STRING)) .add("data", StringType.STRING) val clusteringColumns = List(new Column("id"), new Column(Array("address", "city"))) val snapshot = ucCatalogManagedClient .buildCreateTableTransaction(testUcTableId, tablePath, testSchema, "test-engine") .withTableProperties( Map( "foo" -> "bar", "delta.enableRowTracking" -> "true", "delta.columnMapping.mode" -> "name").asJava) .withDataLayoutSpec(DataLayoutSpec.clustered(clusteringColumns.asJava)) .build(engine) .commit(engine, CloseableIterable.emptyIterable()) .getPostCommitSnapshot .get() .asInstanceOf[SnapshotImpl] val snapshotTimestamp = snapshot.getTimestamp(engine) val actualProps = UnityCatalogUtils.getPropertiesForCreate(engine, snapshot).asScala val expectedProps = Map( // Case 0: Properties we expect to be injected by the UC-CatalogManaged-Client (and are // stored in the metadata.configuration) "io.unitycatalog.tableId" -> testUcTableId, // Case 1: Table properties from metadata.configuration "foo" -> "bar", "delta.enableRowTracking" -> "true", "delta.columnMapping.mode" -> "name", // Case 2: Protocol-derived properties "delta.minReaderVersion" -> "3", "delta.minWriterVersion" -> "7", "delta.feature.catalogManaged" -> "supported", "delta.feature.rowTracking" -> "supported", "delta.feature.columnMapping" -> "supported", "delta.feature.inCommitTimestamp" -> "supported", // Case 3: UC metastore properties "delta.lastUpdateVersion" -> "0", "delta.lastCommitTimestamp" -> s"$snapshotTimestamp", // Case 4: Clustering properties - these should be the LOGICAL names not the PHYSICAL names "clusteringColumns" -> """[["id"],["address","city"]]""") val failures = expectedProps.collect { case (k, v) if !actualProps.contains(k) => s"$k: MISSING (expected: $v)" case (k, v) if actualProps(k) != v => s"$k: expected '$v', got '${actualProps(k)}'" } assert(failures.isEmpty, failures.mkString("Property mismatches:\n", "\n", "")) } } test("getPropertiesForCreate: clustered table with empty clustering columns") { withTempDirAndEngine { case (tablePathUnresolved, engine) => val ucClient = new InMemoryUCClient("ucMetastoreId") val ucCatalogManagedClient = new UCCatalogManagedClient(ucClient) val tablePath = engine.getFileSystemClient.resolvePath(tablePathUnresolved) val snapshot = ucCatalogManagedClient .buildCreateTableTransaction(testUcTableId, tablePath, testSchema, "test-engine") .withDataLayoutSpec(DataLayoutSpec.clustered(List.empty.asJava)) .build(engine) .commit(engine, CloseableIterable.emptyIterable()) .getPostCommitSnapshot .get() .asInstanceOf[SnapshotImpl] val props = UnityCatalogUtils.getPropertiesForCreate(engine, snapshot).asScala assert(props("clusteringColumns") == "[]") } } } ================================================ FILE: kernel/unitycatalog/src/test/scala/io/delta/kernel/unitycatalog/adapters/ActionAdaptersSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.kernel.unitycatalog.adapters import scala.jdk.CollectionConverters._ import io.delta.kernel.data.{ArrayValue, ColumnVector} import io.delta.kernel.internal.actions.{Format, Metadata => KernelMetadata, Protocol => KernelProtocol} import io.delta.kernel.internal.util.InternalUtils.singletonStringColumnVector import io.delta.kernel.internal.util.VectorUtils import io.delta.kernel.types.{IntegerType, StructType} import org.scalatest.funsuite.AnyFunSuite class ActionAdaptersSuite extends AnyFunSuite { test("ProtocolAdapter") { // ===== GIVEN ===== val readerFeatures = Set("v2Checkpoint").asJava val writerFeatures = Set("v2Checkpoint", "rowTracking").asJava val kernelProtocol = new KernelProtocol(3, 7, readerFeatures, writerFeatures) // ===== WHEN ===== val adapterProtocol = new ProtocolAdapter(kernelProtocol) // ===== THEN ===== assert(adapterProtocol.getMinReaderVersion === 3) assert(adapterProtocol.getMinWriterVersion === 7) assert(adapterProtocol.getReaderFeatures.asScala == Set("v2Checkpoint")) assert(adapterProtocol.getWriterFeatures.asScala == Set("v2Checkpoint", "rowTracking")) } test("MetadataAdapter") { // ===== GIVEN ===== val partCols = new ArrayValue() { override def getSize = 1 override def getElements: ColumnVector = singletonStringColumnVector("part1") } val formatOptions = Map("foo" -> "bar").asJava val format = new Format("parquet", formatOptions) val configuration = Map("zip" -> "zap").asJava val kernelMetadata = new KernelMetadata( "id", java.util.Optional.of("name"), java.util.Optional.of("description"), format, "schemaStringJson", new StructType().add("part1", IntegerType.INTEGER).add("col1", IntegerType.INTEGER), partCols, java.util.Optional.of(42L), // createdTime VectorUtils.stringStringMapValue(configuration)) // ===== WHEN ===== val adapter = new MetadataAdapter(kernelMetadata) // ===== THEN ===== assert(adapter.getId === "id") assert(adapter.getName === "name") assert(adapter.getDescription === "description") assert(adapter.getProvider === "parquet") assert(adapter.getFormatOptions.asScala == Map("foo" -> "bar")) assert(adapter.getSchemaString === "schemaStringJson") assert(adapter.getPartitionColumns.asScala == Seq("part1")) assert(adapter.getConfiguration.asScala == Map("zip" -> "zap")) assert(adapter.getCreatedTime === 42L) } } ================================================ FILE: kernel/version.sbt ================================================ ThisBuild / version := "0.1.0-SNAPSHOT" ================================================ FILE: project/Checkstyle.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import com.etsy.sbt.checkstyle.CheckstylePlugin.autoImport._ import org.scalastyle.sbt.ScalastylePlugin.autoImport._ import sbt._ import sbt.Keys._ object Checkstyle { /* ***************************** * Scala checkstyle settings * ***************************** */ ThisBuild / scalastyleConfig := baseDirectory.value / "scalastyle-config.xml" private lazy val compileScalastyle = taskKey[Unit]("compileScalastyle") private lazy val testScalastyle = taskKey[Unit]("testScalastyle") lazy val scalaStyleSettings = Seq( compileScalastyle := (Compile / scalastyle).toTask("").value, Compile / compile := ((Compile / compile) dependsOn compileScalastyle).value, testScalastyle := (Test / scalastyle).toTask("").value, Test / test := ((Test / test) dependsOn testScalastyle).value ) /* **************************** * Java checkstyle settings * **************************** */ private lazy val compileJavastyle = taskKey[Unit]("compileJavastyle") private lazy val testJavastyle = taskKey[Unit]("testJavastyle") def javaCheckstyleSettings(checkstyleFile: String): Def.SettingsDefinition = { // Can be run explicitly via: build/sbt $module/checkstyle // Will automatically be run during compilation (e.g. build/sbt compile) // and during tests (e.g. build/sbt test) Seq( checkstyleConfigLocation := CheckstyleConfigLocation.File(checkstyleFile), // if we keep the Error severity, `build/sbt` will throw an error and immediately stop at // the `checkstyle` phase (if error) -> never execute the `check-report` phase of // `checkstyle-report.xml` and `checkstyle-test-report.xml`. We need to ignore and throw // error if exists when checking *report.xml. checkstyleSeverityLevel := CheckstyleSeverityLevel.Ignore, compileJavastyle := { (Compile / checkstyle).value javaCheckstyle(streams.value.log, checkstyleOutputFile.value) }, (Compile / compile) := ((Compile / compile) dependsOn compileJavastyle).value, testJavastyle := { (Test / checkstyle).value javaCheckstyle(streams.value.log, (Compile / target).value / "checkstyle-test-report.xml") }, (Test / test) := ((Test / test) dependsOn (Test / testJavastyle)).value ) } private def javaCheckstyle(log: Logger, reportFile: File): Unit = { val report = scala.xml.XML.loadFile(reportFile) val errors = (report \\ "file").flatMap { fileNode => val file = fileNode.attribute("name").get.head.text (fileNode \ "error").map { error => val line = error.attribute("line").get.head.text val message = error.attribute("message").get.head.text (file, line, message) } } if (errors.nonEmpty) { var errorMsg = "Found checkstyle errors" errors.foreach { case (file, line, message) => val lineError = s"File: $file, Line: $line, Message: $message" log.error(lineError) errorMsg += ("\n" + lineError) } sys.error(errorMsg + "\n") } } } ================================================ FILE: project/CrossSparkVersions.scala ================================================ import sbt._ import sbt.Keys._ import sbt.complete.DefaultParsers._ import com.simplytyped.Antlr4Plugin import com.simplytyped.Antlr4Plugin.autoImport._ import sbtrelease.ReleasePlugin.autoImport.ReleaseStep import Unidoc._ /** * ======================================================== * Cross-Spark Build and Publish System * ======================================================== * * This SBT plugin enables Delta Lake to be built and published for multiple Spark versions. * It provides version-specific configurations, artifact naming, and publishing workflows. * * ======================================================== * Spark Version Definitions * ======================================================== * * The Spark versions used for Delta is defined in the SparkVersionSpec object, and controlled by the sparkVersion property. * There are 2 keys labels assigned to the Spark versions: DEFAULT and MASTER. * - DEFAULT VERSION: This is the default when no sparkVersion property is specified. * * - MASTER VERSION: The Spark master/development branch version * This is optional and typically * - set in the Delta master branch to a Spark released or snapshot version . * - not set in the Delta release branches as we want to avoid building against Spark unreleased version. * If MASTER is defined, then it can be selected by setting the sparkVersion property to "master". * Spark-dependent artifacts for this version HAVE a Spark version suffix in their artifact names (e.g., delta-spark_4.0_2.13 if MASTER is defined as Spark 4.0 branch). * * - OTHER VERSIONS: Any non-default Spark version specified in ALL_SPECS. * Spark-dependent artifacts of all non-default versions get a Spark version suffix in their artifact names (e.g., delta-spark_4.1_2.13 if one of the other versions is defined as Spark 4.1 branch). * * To configure versions, update the SparkVersionSpec values (e.g., spark35, spark40, etc.) below. * * ======================================================== * The sparkVersion Property * ======================================================== * * The sparkVersion system property controls which Spark version to build against. * It accepts the following formats: * * 1. Full version string (e.g., "3.5.7", "4.0.2-SNAPSHOT") * 2. Short version string (e.g., "3.5", "4.0") * 3. Aliases: * - "default" -> maps to DEFAULT version (e.g., spark35) * - "master" -> maps to MASTER version (e.g., spark40), if configured * * If not specified, it defaults to the DEFAULT version. * * Examples: * build/sbt # Uses default version * build/sbt -DsparkVersion=4.0 # Uses Spark 4.0.x * build/sbt -DsparkVersion=4.0.1 # Uses Spark 4.0.1 only if this version is defined in ALL_SPECS * build/sbt -DsparkVersion=4.1 # Uses Spark 4.1.x whatever it is defined in ALL_SPECS * build/sbt -DsparkVersion=default # Uses default version * build/sbt -DsparkVersion=master # Uses master version (if defined) * * ======================================================== * Cross-Building for Development and Testing * ======================================================== * * To build/test against a specific Spark version: * build/sbt -DsparkVersion= compile * build/sbt -DsparkVersion= test * build/sbt -DsparkVersion=master compile test * * To publish to local Maven for testing: * # Publish all modules for default Spark version * build/sbt publishM2 * * # Publish only Spark-dependent modules for other versions * build/sbt -DsparkVersion=master "runOnlyForReleasableSparkModules publishM2" * * ======================================================== * Module Types * ======================================================== * * Modules are automatically classified based on their settings: * * 1. Spark-Dependent Published Modules: * - Use CrossSparkVersions.sparkDependentSettings(sparkVersion) * - Include releaseSettings (publishable) * - Examples: delta-spark, delta-connect-*, delta-sharing-spark, delta-iceberg, delta-hudi, delta-contribs * - These modules get version-specific artifact names for non-default Spark versions * - Automatically included in cross-Spark publishing * * 2. Spark-Dependent Internal Modules: * - Use CrossSparkVersions.sparkDependentSettings(sparkVersion) * - Include skipReleaseSettings (not published) * - Examples: sparkV1, sparkV2 * - These modules are built for each Spark version but not published * - Automatically excluded from cross-Spark publishing * * 3. Spark-Independent Modules: * - Do not use CrossSparkVersions settings * - Examples: delta-storage, delta-kernel-*, delta-standalone * - These modules are built once and work with all Spark versions * * ======================================================== * Artifact Naming Convention of Spark-dependent modules * ======================================================== * * By default, Spark-dependent modules ALWAYS include the Spark version suffix: * io.delta:delta-spark_4.0_2.13:4.1.0 * io.delta:delta-spark_4.1_2.13:4.1.0 * io.delta:delta-connect-server_4.0_2.13:4.1.0 * io.delta:delta-storage:4.1.0 (Spark-independent, no suffix) * * During release, backward-compatible artifacts are ALSO published (without suffix): * io.delta:delta-spark_2.13:4.1.0 (backward compatibility) * io.delta:delta-connect-server_2.13:4.1.0 * * This means during release, Spark-dependent modules are published TWICE: * - With suffix (e.g., delta-spark_4.1_2.13) - the default/normal name * - Without suffix (e.g., delta-spark_2.13) - for backward compatibility * * ======================================================== * Cross-Release Workflow * ======================================================== * * The cross-release workflow publishes artifacts for all Spark versions: * * Step 1: Publish ALL modules WITHOUT Spark suffix (backward compatibility) * build/sbt -DskipSparkSuffix=true publishSigned * # Publishes: delta-spark_2.13, delta-storage, delta-kernel-api, etc. * * Step 2: Publish Spark-dependent modules WITH suffix for each non-master Spark version * build/sbt -DsparkVersion=4.0 "runOnlyForReleasableSparkModules publishSigned" * build/sbt -DsparkVersion=4.1 "runOnlyForReleasableSparkModules publishSigned" * # Publishes: delta-spark_4.0_2.13, delta-spark_4.1_2.13, etc. * * This workflow is automated via crossSparkReleaseSteps() in the release process. * See releaseProcess in build.sbt for integration. * * Why this approach? * - Default behavior always includes Spark suffix for clarity * - Release also publishes without suffix for backward compatibility * - Spark-independent modules (kernel, storage) are built once * - Spark-dependent modules are built for each Spark version * * For manual testing during development: * build/sbt publishM2 # Publishes delta-spark_4.0_2.13 (default, with suffix) * * For manual release testing: * build/sbt -DskipSparkSuffix=true publishM2 # Without suffix (backward compat) * build/sbt -DsparkVersion=4.0 "runOnlyForReleasableSparkModules publishM2" * build/sbt -DsparkVersion=4.1 "runOnlyForReleasableSparkModules publishM2" * # Verify JARs in ~/.m2/repository/io/delta/ * * ======================================================== * Commands Provided * ======================================================== * * runOnlyForReleasableSparkModules * Runs the specified task only on publishable Spark-dependent modules. * Automatically detects modules that: * 1. Have the sparkVersion setting (use Spark-aware configuration) * 2. Are publishable (publish/skip is not true) * * Used for publishing Spark-dependent modules for non-default Spark versions. * * Example: * build/sbt -DsparkVersion=4.0 "runOnlyForReleasableSparkModules publishM2" * * showSparkVersions * Lists all configured Spark versions (for testing/debugging). * * Example: * build/sbt showSparkVersions * * exportSparkVersionsJson * Exports Spark version information to target/spark-versions.json. * This is the SINGLE SOURCE OF TRUTH for Spark versions used by: * - GitHub Actions workflows (for dynamic matrix generation) * - CI/CD scripts (for version-specific configuration) * * The JSON is an array where each element contains: * - fullVersion: Full version string (e.g., "4.0.1", "4.1.0") * - shortVersion: Short version string (e.g., "4.0", "4.1") * - isMaster: Whether this is the master/snapshot version * - isDefault: Whether this is the default Spark version * - targetJvm: Target JVM version (e.g., "17") * - packageSuffix: Maven artifact suffix for this version (e.g., "_4.0", "_4.1") * * Example: * build/sbt exportSparkVersionsJson * # Generates: target/spark-versions.json * # Output: [{"fullVersion": "4.0.1", "shortVersion": "4.0", "isMaster": false, "isDefault": true, "targetJvm": "17", "packageSuffix": "_4.0"}, ...] * * Use with Python utilities to extract specific fields: * python3 project/scripts/get_spark_version_info.py --all-spark-versions * # Output: ["4.0", "4.1"] or ["master", "4.0"] if master is present * python3 project/scripts/get_spark_version_info.py --get-field "4.0" targetJvm * python3 project/scripts/get_spark_version_info.py --get-field "master" targetJvm * * This ensures GitHub Actions always uses the versions defined here, * eliminating manual synchronization across multiple files. * * ======================================================== */ /** * Specification for a Spark version with all its build configuration. * * @param fullVersion The full Spark version (e.g., "3.5.7", "4.0.2-SNAPSHOT") * @param targetJvm Target JVM version (e.g., "11", "17") * @param additionalSourceDir Optional version-specific source directory suffix (e.g., "scala-spark-3.5") * @param antlr4Version ANTLR version to use (e.g., "4.9.3", "4.13.1") * @param additionalJavaOptions Additional JVM options for tests (e.g., Java 17 --add-opens flags) */ case class SparkVersionSpec( fullVersion: String, targetJvm: String, additionalSourceDir: Option[String] = None, supportIceberg: Boolean, supportHudi: Boolean = true, antlr4Version: String, additionalJavaOptions: Seq[String] = Seq.empty, jacksonVersion: String = "2.15.2", additionalResolvers: Seq[Resolver] = Seq.empty ) { /** Returns the Spark short version (e.g., "3.5", "4.0") */ def shortVersion: String = { Mima.getMajorMinorPatch(fullVersion) match { case (maj, min, _) => s"$maj.$min" } } /** Whether this is the default Spark version */ def isDefault: Boolean = this == SparkVersionSpec.DEFAULT /** Whether this is the master Spark version */ def isMaster: Boolean = SparkVersionSpec.MASTER.contains(this) /** Returns log4j config file */ def log4jConfig: String = "log4j2.properties" /** Whether to export JARs instead of class directories (needed for Spark Connect on master) */ def exportJars: Boolean = additionalSourceDir.exists(_.contains("master")) /** Whether to generate Javadoc/Scaladoc for this version */ def generateDocs: Boolean = isDefault } object SparkVersionSpec { private val java17TestSettings = Seq( // Copied from SparkBuild.scala to support Java 17 for unit tests (see apache/spark#34153) "--add-opens=java.base/java.lang=ALL-UNNAMED", "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", "--add-opens=java.base/java.io=ALL-UNNAMED", "--add-opens=java.base/java.net=ALL-UNNAMED", "--add-opens=java.base/java.nio=ALL-UNNAMED", "--add-opens=java.base/java.util=ALL-UNNAMED", "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED", "--add-opens=java.base/sun.security.action=ALL-UNNAMED", "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED" ) private val spark40 = SparkVersionSpec( fullVersion = "4.0.1", targetJvm = "17", additionalSourceDir = Some("scala-shims/spark-4.0"), supportIceberg = true, antlr4Version = "4.13.1", additionalJavaOptions = java17TestSettings, jacksonVersion = "2.18.2" ) private val spark41 = SparkVersionSpec( fullVersion = "4.1.0", targetJvm = "17", additionalSourceDir = Some("scala-shims/spark-4.1"), supportIceberg = false, supportHudi = false, antlr4Version = "4.13.1", additionalJavaOptions = java17TestSettings, jacksonVersion = "2.18.2" ) private val spark42Snapshot = SparkVersionSpec( fullVersion = "4.2.0-SNAPSHOT", targetJvm = "17", additionalSourceDir = Some("scala-shims/spark-4.2"), supportIceberg = false, supportHudi = false, antlr4Version = "4.13.1", additionalJavaOptions = java17TestSettings, jacksonVersion = "2.18.2", // Artifact updates in maven central for roaringbitmap stopped after 1.3.0. // Spark master uses 1.5.3. Relevant Spark PR here https://github.com/apache/spark/pull/52892 additionalResolvers = Seq("jitpack" at "https://jitpack.io") ) /** Default Spark version */ val DEFAULT = spark41 /** Spark master branch version (optional). Release branches should not build against master */ val MASTER: Option[SparkVersionSpec] = None /** All supported Spark versions - internal use only */ val ALL_SPECS = Seq(spark40, spark41) } /** See docs on top of this file */ object CrossSparkVersions extends AutoPlugin { override def trigger = allRequirements /** * Returns the current configured Spark version spec based on the `sparkVersion` property. */ def getSparkVersionSpec(): SparkVersionSpec = { val input = sys.props.getOrElse("sparkVersion", SparkVersionSpec.DEFAULT.fullVersion) // Resolve aliases first val resolvedInput = input match { case "default" => SparkVersionSpec.DEFAULT.fullVersion case "master" => SparkVersionSpec.MASTER match { case Some(masterSpec) => masterSpec.fullVersion case None => throw new IllegalArgumentException( "No master Spark version is configured. Available versions: " + SparkVersionSpec.ALL_SPECS.map(_.fullVersion).mkString(", ") ) } case other => other } // Find spec by full version or short version SparkVersionSpec.ALL_SPECS.find { spec => spec.fullVersion == resolvedInput || spec.shortVersion == resolvedInput }.getOrElse { val aliases = Seq("default") ++ SparkVersionSpec.MASTER.map(_ => "master").toSeq val validInputs = SparkVersionSpec.ALL_SPECS.flatMap { spec => Seq(spec.fullVersion, spec.shortVersion) } ++ aliases throw new IllegalArgumentException( s"Invalid sparkVersion: $input. Valid values: ${validInputs.mkString(", ")}" ) } } /** * Returns the current configured Spark version based on the `sparkVersion` property. */ def getSparkVersion(): String = getSparkVersionSpec().fullVersion /** * Returns module name with Spark version suffix. * * By default, ALL Spark-dependent modules include the Spark version suffix: * delta-spark_4.0_2.13, delta-spark_4.1_2.13, etc. * * During release, the `skipSparkSuffix=true` property is used to also publish * backward-compatible artifacts without the suffix (e.g., delta-spark_2.13). */ private def moduleName(baseName: String, sparkVer: String): String = { val spec = SparkVersionSpec.ALL_SPECS.find(_.fullVersion == sparkVer) .getOrElse(throw new IllegalArgumentException(s"Unknown Spark version: $sparkVer")) // skipSparkSuffix removes the suffix (used during release for backward compatibility) val skipSparkSuffix = sys.props.getOrElse("skipSparkSuffix", "false").toBoolean if (skipSparkSuffix) { baseName } else { s"${baseName}_${spec.shortVersion}" } } // Scala version constant (Scala 2.12 support was dropped) private val scala213 = "2.13.17" /** * Common Spark version-specific settings used by all Spark-aware modules. * Returns Scala version, source directories, ANTLR version, JVM options, etc. */ private def sparkVersionAwareSettings(sparkVersionKey: SettingKey[String]): Seq[Setting[_]] = { val spec = getSparkVersionSpec() val baseSettings = Seq( scalaVersion := scala213, crossScalaVersions := Seq(scala213), resolvers ++= spec.additionalResolvers, Antlr4 / antlr4Version := spec.antlr4Version, Test / javaOptions ++= (Seq(s"-Dlog4j.configurationFile=${spec.log4jConfig}") ++ spec.additionalJavaOptions) ) val additionalSourceDirSettings = spec.additionalSourceDir.map { dir => // Add both scala-shims and java-shims directories val javaShimsDir = dir.replace("scala-shims", "java-shims") Seq( Compile / unmanagedSourceDirectories += (Compile / baseDirectory).value / "src" / "main" / dir, Compile / unmanagedSourceDirectories += (Compile / baseDirectory).value / "src" / "main" / javaShimsDir, Test / unmanagedSourceDirectories += (Test / baseDirectory).value / "src" / "test" / dir ) }.getOrElse(Seq.empty) val conditionalSettings = Seq( if (spec.exportJars) Seq(exportJars := true) else Nil, if (spec.generateDocs) Seq(unidocSourceFilePatterns := Seq(SourceFilePattern("io/delta/tables/", "io/delta/exceptions/"))) else Nil ).flatten // Jackson dependency overrides to match Spark version and avoid conflicts val jacksonOverrides = Seq( dependencyOverrides ++= { val sparkVer = sparkVersionKey.value val jacksonVer = SparkVersionSpec.ALL_SPECS.find(_.fullVersion == sparkVer) .getOrElse(throw new IllegalArgumentException(s"Unknown Spark version: $sparkVer")) .jacksonVersion Seq( "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVer, "com.fasterxml.jackson.core" % "jackson-core" % jacksonVer, "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVer, "com.fasterxml.jackson.datatype" % "jackson-datatype-jdk8" % jacksonVer, "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVer ) } ) baseSettings ++ additionalSourceDirSettings ++ conditionalSettings ++ jacksonOverrides } /** * Just the module name setting for Spark-dependent modules that don't need full Spark integration. * Use this for modules that need versioned artifacts but use default Scala settings. * * @param sparkVersionKey The sparkVersion setting key for this project */ def sparkDependentModuleName(sparkVersionKey: SettingKey[String]): Seq[Setting[_]] = { Seq( sparkVersionKey := getSparkVersion(), // Dynamically modify moduleName to add Spark version suffix Keys.moduleName := moduleName(Keys.name.value, sparkVersionKey.value) ) } /** * Unified settings for Spark-dependent modules. * Use this for modules that need to be built for multiple Spark versions. * Works for both published modules and internal modules. * * @param sparkVersionKey The sparkVersion setting key for this project */ def sparkDependentSettings(sparkVersionKey: SettingKey[String]): Seq[Setting[_]] = { sparkDependentModuleName(sparkVersionKey) ++ sparkVersionAwareSettings(sparkVersionKey) } /** * Generates release steps for cross-Spark publishing. * * Returns a sequence of release steps that: * 1. Publishes all modules WITHOUT Spark suffix (backward compatibility) * 2. Publishes Spark-dependent modules WITH Spark suffix for each non-master version * * For example, with Spark versions 4.0 (default) and 4.1: * - Step 1 publishes: delta-spark_2.13, delta-storage, delta-kernel-api, etc. (no suffix) * - Step 2 publishes: delta-spark_4.0_2.13, delta-spark_4.1_2.13, etc. (with suffix) * * Each step runs as a separate SBT subprocess so the build reloads with * the correct sparkVersion/skipSparkSuffix settings (SBT settings like * moduleName are evaluated once at build load time and can't be changed * at runtime). * * Usage in build.sbt: * releaseProcess := Seq[ReleaseStep]( * ..., * ) ++ CrossSparkVersions.crossSparkReleaseSteps("publishSigned") ++ Seq( * ... * ) */ def crossSparkReleaseSteps(task: String): Seq[ReleaseStep] = { // SBT settings (like moduleName) are evaluated once at build load time. // To publish with different Spark versions or suffix modes, we must run // separate SBT processes so the build reloads with the correct settings. // The release version is already committed to version.sbt by prior steps, // so subprocess SBT instances will pick up the correct version. def runSbtSubprocess(state: State, sbtArgs: Seq[String], description: String): State = { val extracted = Project.extract(state) val baseDir = extracted.get(ThisBuild / Keys.baseDirectory) val cmd = Seq(s"${baseDir.getAbsolutePath}/build/sbt") ++ sbtArgs println(s"[info] ========================================") println(s"[info] $description") println(s"[info] Running: ${cmd.mkString(" ")}") println(s"[info] ========================================") val exitCode = scala.sys.process.Process(cmd, baseDir).! if (exitCode != 0) { sys.error(s"$description failed with exit code $exitCode") } state } // Step 1: Publish ALL modules WITHOUT Spark suffix (backward compatibility) // Uses skipSparkSuffix=true to get artifact names like delta-spark_2.13 val backwardCompatStep: ReleaseStep = { (state: State) => runSbtSubprocess( state, Seq("-DskipSparkSuffix=true", task), "Publishing all modules without Spark suffix (backward compat)" ) } // Step 2+: Publish Spark-dependent modules WITH suffix for each non-master Spark version // This gives users versioned artifacts like delta-spark_4.0_2.13, delta-spark_4.1_2.13 val suffixedSparkSteps: Seq[ReleaseStep] = SparkVersionSpec.ALL_SPECS .filterNot(_.isMaster) // Exclude master/snapshot versions .map { spec => { (state: State) => runSbtSubprocess( state, Seq(s"-DsparkVersion=${spec.fullVersion}", s"runOnlyForReleasableSparkModules $task"), s"Publishing Spark-dependent modules with suffix for Spark ${spec.fullVersion}" ) }: ReleaseStep } backwardCompatStep +: suffixedSparkSteps } override lazy val projectSettings = Seq( commands += Command.args("runOnlyForReleasableSparkModules", "") { (state, args) => // Used for cross-Spark publishing of Spark-dependent modules only. // Runs the specified task only on publishable Spark-dependent projects. if (args.isEmpty) { sys.error("Usage: runOnlyForReleasableSparkModules \nExample: build/sbt -DsparkVersion= \"runOnlyForReleasableSparkModules publishM2\"") } val task = args.mkString(" ") // Discover Spark-dependent projects dynamically // A project is Spark-dependent if: // 1. It has the sparkVersion setting (uses Spark-aware configuration) // 2. It is publishable (publishArtifact is not false) val extracted = sbt.Project.extract(state) val sparkVersionKey = SettingKey[String]("sparkVersion") val publishArtifactKey = SettingKey[Boolean]("publishArtifact") val sparkDependentProjects = extracted.structure.allProjectRefs.filter { projRef => val hasSparkVersion = (projRef / sparkVersionKey).get(extracted.structure.data).isDefined val isPublishable = (projRef / publishArtifactKey).get(extracted.structure.data).getOrElse(true) hasSparkVersion && isPublishable } if (sparkDependentProjects.isEmpty) { println(s"[warn] No publishable projects with sparkVersion setting found") state } else { val projectNames = sparkDependentProjects.map(_.project).mkString(", ") val sparkVer = getSparkVersion() println(s"[info] Running '$task' for Spark-dependent modules with Spark $sparkVer") println(s"[info] Spark-dependent projects: $projectNames") println(s"[info] ========================================") // Build scoped task for each Spark-dependent project sequentially sparkDependentProjects.foldLeft(state) { (currentState, projRef) => // Handle SBT cross-build prefix: "+publishSigned" must become // "+project/publishSigned", not "project/+publishSigned" val scopedTask = if (task.startsWith("+")) { s"+${projRef.project}/${task.stripPrefix("+")}" } else { s"${projRef.project}/$task" } Command.process(scopedTask, currentState) } } }, commands += Command.command("showSparkVersions") { state => // Used for testing the cross-Spark publish workflow SparkVersionSpec.ALL_SPECS.foreach { spec => println(spec.fullVersion) } state }, commands += Command.command("exportSparkVersionsJson") { state => // Export Spark version information as JSON for use by CI/CD and other tools import java.io.{File, PrintWriter} val outputFile = new File("target/spark-versions.json") outputFile.getParentFile.mkdirs() val writer = new PrintWriter(outputFile) // scalastyle:off try { writer.println("[") SparkVersionSpec.ALL_SPECS.zipWithIndex.foreach { case (spec, idx) => val comma = if (idx < SparkVersionSpec.ALL_SPECS.size - 1) "," else "" val isMaster = SparkVersionSpec.MASTER.contains(spec) val isDefault = spec == SparkVersionSpec.DEFAULT // Package suffix always includes Spark version (e.g., "_4.0", "_4.1") val packageSuffix = s"_${spec.shortVersion}" writer.println(s""" {""") writer.println(s""" "fullVersion": "${spec.fullVersion}",""") writer.println(s""" "shortVersion": "${spec.shortVersion}",""") writer.println(s""" "isMaster": $isMaster,""") writer.println(s""" "isDefault": $isDefault,""") writer.println(s""" "targetJvm": "${spec.targetJvm}",""") writer.println(s""" "packageSuffix": "$packageSuffix",""") writer.println(s""" "supportIceberg": "${spec.supportIceberg}",""") writer.println(s""" "supportHudi": "${spec.supportHudi}"""") writer.println(s""" }$comma""") } writer.println("]") println(s"[info] Spark version information exported to: ${outputFile.getAbsolutePath}") } finally { writer.close() } // scalastyle:on state } ) } ================================================ FILE: project/FlinkMimaExcludes.scala ================================================ /* * Copyright (2020-present) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import com.typesafe.tools.mima.core._ /** * The list of Mima errors to exclude in the Flink project. */ object FlinkMimaExcludes { // scalastyle:off line.size.limit val ignoredABIProblems = Seq( // We can ignore internal changes ProblemFilters.exclude[Problem]("io.delta.standalone.internal.*") ) } ================================================ FILE: project/Mima.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import com.typesafe.tools.mima.plugin.MimaPlugin.autoImport.{mimaBinaryIssueFilters, mimaPreviousArtifacts, mimaReportBinaryIssues} import sbt._ import sbt.Keys._ /** * Mima settings */ object Mima { /** * @return tuple of (major, minor, patch) versions extracted from a version string. * e.g. "1.2.3" would return (1, 2, 3) */ def getMajorMinorPatch(versionStr: String): (Int, Int, Int) = { implicit def extractInt(str: String): Int = { """\d+""".r.findFirstIn(str).map(java.lang.Integer.parseInt).getOrElse { throw new Exception(s"Could not extract version number from $str in $version") } } versionStr.split("\\.").toList match { case majorStr :: minorStr :: patchStr :: _ => (majorStr, minorStr, patchStr) case _ => throw new Exception(s"Could not parse version for $version.") } } def getPrevSparkName(currentVersion: String): String = { val (major, minor, patch) = getMajorMinorPatch(currentVersion) // name change in version 3.0.0, so versions > 3.0.0 should have delta-spark are prev version. if (major < 3 || (major == 3 && minor == 0 && patch == 0)) { "delta-core" } else { "delta-spark" } } def getPrevSparkVersion(currentVersion: String): String = { val (major, minor, patch) = getMajorMinorPatch(currentVersion) val lastVersionInMajorVersion = Map( 0 -> "0.8.0", 1 -> "1.2.1", 2 -> "2.4.0", 3 -> "3.3.1" ) if (minor == 0) { // 1.0.0 or 2.0.0 or 3.0.0 or 4.0.0 lastVersionInMajorVersion.getOrElse(major - 1, { throw new Exception(s"Last version of ${major - 1}.x.x not configured.") }) } else if (patch == 0) { s"$major.${minor - 1}.0" // 1.1.0 -> 1.0.0 } else { s"$major.$minor.${patch - 1}" // 1.1.1 -> 1.1.0 } } def getPrevConnectorVersion(currentVersion: String): String = { val (major, minor, patch) = getMajorMinorPatch(currentVersion) val majorToLastMinorVersions: Map[Int, String] = Map( // We skip from 0.6.0 to 3.0.0 when migrating connectors to the main delta repo 0 -> "0.6.0", 1 -> "0.6.0", 2 -> "0.6.0", 3 -> "3.3.1" ) if (minor == 0) { // 1.0.0 majorToLastMinorVersions.getOrElse(major - 1, { throw new Exception(s"Last minor version of ${major - 1}.x.x not configured.") }) } else if (patch == 0) { s"$major.${minor - 1}.0" // 1.1.0 -> 1.0.0 } else { s"$major.$minor.${patch - 1}" // 1.1.1 -> 1.1.0 } } lazy val sparkMimaSettings = Seq( Test / test := ((Test / test) dependsOn mimaReportBinaryIssues).value, mimaPreviousArtifacts := Set("io.delta" %% getPrevSparkName(version.value) % getPrevSparkVersion(version.value)), mimaBinaryIssueFilters ++= SparkMimaExcludes.ignoredABIProblems ) lazy val standaloneMimaSettings = Seq( Test / test := ((Test / test) dependsOn mimaReportBinaryIssues).value, mimaPreviousArtifacts := { Set("io.delta" %% "delta-standalone" % getPrevConnectorVersion(version.value)) }, mimaBinaryIssueFilters ++= StandaloneMimaExcludes.ignoredABIProblems ) lazy val flinkMimaSettings = Seq( Test / test := ((Test / test) dependsOn mimaReportBinaryIssues).value, mimaPreviousArtifacts := { Set("io.delta" % "delta-flink" % getPrevConnectorVersion(version.value)) }, mimaBinaryIssueFilters ++= FlinkMimaExcludes.ignoredABIProblems ) } ================================================ FILE: project/MultiShardMultiJVMTestParallelization.scala ================================================ import scala.util.hashing.MurmurHash3 import sbt.Keys._ import sbt._ // scalastyle:off println /** Provides SBT test settings for sharding and parallelizing multi-JVM tests. */ object MultiShardMultiJVMTestParallelization { /** * Total number of shards (machines) to split tests across. * E.g., NUM_SHARDS=4 means tests will be split across 4 machines. * Each test is assigned to exactly one shard based on hash(testName) % NUM_SHARDS. */ lazy val numShardsOpt = sys.env.get("NUM_SHARDS").map(_.toInt) /** * The ID of the current shard (0-indexed). * E.g., SHARD_ID=0 means this is shard 0 out of NUM_SHARDS total shards. * This shard will only run tests where hash(testName) % NUM_SHARDS == SHARD_ID. */ lazy val shardIdOpt = sys.env.get("SHARD_ID").map(_.toInt) /** * Number of parallel JVMs to use within this shard. * E.g., TEST_PARALLELISM_COUNT=2 means tests in this shard will run across 2 JVMs in parallel. * Tests are distributed across JVMs using hash(testName + "group") % TEST_PARALLELISM_COUNT. */ lazy val testParallelismOpt = sys.env.get("TEST_PARALLELISM_COUNT").map(_.toInt) lazy val settings = { println(s"numShardsOpt: $numShardsOpt") println(s"shardIdOpt: $shardIdOpt") println(s"testParallelismOpt: $testParallelismOpt") (numShardsOpt, shardIdOpt, testParallelismOpt) match { case (Some(numShards), Some(shardId), Some(testParallelism)) if numShards >= 1 && shardId >= 0 && testParallelism >= 1 => println("Test parallelization enabled.") Seq( Test / testGrouping := { val tests = (Test / definedTests).value // Create default fork options that inherit all the project's settings val defaultForkOptions = ForkOptions( javaHome = javaHome.value, outputStrategy = outputStrategy.value, bootJars = Vector.empty, workingDirectory = Some(baseDirectory.value), runJVMOptions = (Test / javaOptions).value.toVector, connectInput = connectInput.value, envVars = (Test / envVars).value ) // Filter tests for this shard val testsForThisShard = tests.filter { testDef => math.abs(MurmurHash3.stringHash(testDef.name) % numShards) == shardId } println(s"[Shard $shardId] # tests: ${testsForThisShard.size}") // Distribute tests across groups (JVMs) within this shard (0 until testParallelism).map { groupId => val testsForThisGroup = testsForThisShard.filter { testDef => // Add "group" suffix to create a different hash than shard assignment, // ensuring even distribution across groups independent of shard assignment val groupHash = MurmurHash3.stringHash(testDef.name + "group") math.abs(groupHash % testParallelism) == groupId } println(s"[Group $groupId] # tests: ${testsForThisGroup.size}") Tests.Group( name = s"Shard $shardId - Group $groupId", tests = testsForThisGroup, runPolicy = Tests.SubProcess(defaultForkOptions) ) } }, Test / parallelExecution := true, Global / concurrentRestrictions := Seq( Tags.limit(Tags.ForkedTestGroup, testParallelism) ) ) case _ => println("Test parallelization disabled.") Seq.empty[Setting[_]] // Run tests normally } } } ================================================ FILE: project/README.md ================================================ # Updating delta-spark TestParallelization Top 50 Slowest Test Suites List - Cherry-pick changes from https://github.com/delta-io/delta/pull/3694 - That PR adds a test report listener to delta-spark that will output csv files containing per-JVM, per-group (thread), and per-test runtimes - Run the CI and download the generated csv artifacts - You can use the following pyspark code to get the top 50 slowest test suites - You can copy and paste that into Chat GPT and ask it to format it as a Scala List ```python from pyspark.sql.functions import col, sum from pyspark.sql.types import StructType, StructField, StringType, LongType schema = StructType([ StructField("test_suite", StringType(), True), StructField("test_name", StringType(), True), StructField("execution_time_ms", LongType(), True), StructField("result", StringType(), True) ]) csv_dir = "..." spark.read.csv(csv_dir, schema=schema) \ .filter(col("execution_time_ms") != -1) \ .groupBy("test_suite") \ .agg((sum("execution_time_ms") / 60000).alias("execution_time_mins")) \ .orderBy(col("execution_time_mins").desc()) \ .limit(50) \ .select("test_suite", "execution_time_mins") \ .show(50, truncate=False) ``` ================================================ FILE: project/ShadedIcebergBuild.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import sbt._ import sbtassembly.* /** * Exclusion rules for not bringing in conflicting dependencies via Iceberg Jar */ object ShadedIcebergBuild { val icebergExclusionRules = List.apply( ExclusionRule("com.github.ben-manes.caffeine"), ExclusionRule("io.netty"), ExclusionRule("org.apache.httpcomponents.client5"), ExclusionRule("org.apache.httpcomponents.core5"), ExclusionRule("io.airlift"), ExclusionRule("org.apache.commons"), ExclusionRule("commons-io"), ExclusionRule("commons-compress"), ExclusionRule("commons-lang3"), ExclusionRule("commons-codec"), ExclusionRule("com.fasterxml.jackson.core"), ExclusionRule("com.fasterxml.jackson.databind"), ) val hadoopClientExclusionRules = List.apply( ExclusionRule("org.apache.avro"), ExclusionRule("org.slf4j"), ExclusionRule("commons-beanutils"), ExclusionRule("org.datanucleus"), ExclusionRule("io.netty") ) val hiveMetastoreExclusionRules = List.apply( ExclusionRule("org.apache.avro"), ExclusionRule("org.slf4j"), ExclusionRule("org.pentaho"), ExclusionRule("org.apache.hbase"), ExclusionRule("org.apache.logging.log4j"), ExclusionRule("co.cask.tephra"), ExclusionRule("com.google.code.findbugs"), ExclusionRule("org.eclipse.jetty.aggregate"), ExclusionRule("org.eclipse.jetty.orbit"), ExclusionRule("org.apache.parquet"), ExclusionRule("com.tdunning"), ExclusionRule("javax.transaction"), ExclusionRule("com.zaxxer"), ExclusionRule("org.apache.ant"), ExclusionRule("javax.servlet"), ExclusionRule("javax.jdo"), ExclusionRule("commons-beanutils"), ExclusionRule("org.datanucleus") ) /** * Replace those files with our customized version * Here's an overview: * PartitionSpec: sets checkConflicts to false to honor field ID assigned by Delta * HiveCatalog, HiveTableOperations: allow metadataUpdates to overwrite schema and partition spec * RESTFileScanTaskParser: fixes NoSuchElementException on empty delete-file-references arrays */ def updateMergeStrategy(prev: String => MergeStrategy): String => MergeStrategy = { case PathList("shadedForDelta", "org", "apache", "iceberg", s) if s.matches("TableMetadata(\\$.*)?\\.class") => MergeStrategy.first case PathList("shadedForDelta", "org", "apache", "iceberg", s) if s.matches("MetadataUpdate(\\$.*)?\\.class") => MergeStrategy.first case PathList("shadedForDelta", "org", "apache", "iceberg", "PartitionSpec$Builder.class") => MergeStrategy.first case PathList("shadedForDelta", "org", "apache", "iceberg", "PartitionSpec.class") => MergeStrategy.first case PathList("shadedForDelta", "org", "apache", "iceberg", "rest", "RESTFileScanTaskParser.class") => MergeStrategy.first case PathList("shadedForDelta", "org", "apache", "iceberg", "hive", "HiveCatalog.class") => MergeStrategy.first case PathList("shadedForDelta", "org", "apache", "iceberg", "hive", "HiveCatalog$1.class") => MergeStrategy.first case PathList( "shadedForDelta", "org", "apache", "iceberg", "hive", "HiveCatalog$ViewAwareTableBuilder.class" ) => MergeStrategy.first case PathList( "shadedForDelta", "org", "apache", "iceberg", "hive", "HiveCatalog$TableAwareViewBuilder.class" ) => MergeStrategy.first case PathList( "shadedForDelta", "org", "apache", "iceberg", "hive", "HiveTableOperations.class" ) => MergeStrategy.first case PathList( "shadedForDelta", "org", "apache", "iceberg", "hive", "HiveTableOperations$1.class" ) => MergeStrategy.first case PathList("org", "slf4j", xs @ _*) => // SLF4J is provided by Spark runtime, exclude from assembly MergeStrategy.discard case PathList("org", "jspecify", "annotations", xs @ _*) => MergeStrategy.discard case x => prev(x) } } ================================================ FILE: project/SparkMimaExcludes.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import com.typesafe.tools.mima.core._ import com.typesafe.tools.mima.core.ProblemFilters._ /** * The list of Mima errors to exclude. */ object SparkMimaExcludes { val ignoredABIProblems = Seq( // scalastyle:off line.size.limit ProblemFilters.exclude[Problem]("org.*"), ProblemFilters.exclude[Problem]("io.delta.sql.parser.*"), ProblemFilters.exclude[Problem]("io.delta.tables.execution.*"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaTable.apply"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaTable.executeGenerate"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaTable.executeHistory"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaTable.executeVacuum"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaTable.executeVacuum$default$3"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaTable.this"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaTable.deltaLog"), // Changes in 0.6.0 ProblemFilters.exclude[IncompatibleResultTypeProblem]("io.delta.tables.DeltaTable.makeUpdateTable"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("io.delta.tables.DeltaMergeBuilder.withClause"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("io.delta.tables.DeltaTable.this"), // ... removed unnecessarily public methods in DeltaMergeBuilder ProblemFilters.exclude[MissingTypesProblem]("io.delta.tables.DeltaMergeBuilder"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordUsage"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordUsage$default$3"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordOperation$default$7"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordUsage$default$6"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.logError"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.log"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordDeltaOperation$default$3"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordOperation$default$4"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordEvent$default$3"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.logName"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordDeltaEvent"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.withStatusCode$default$3"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordOperation"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.isTraceEnabled"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.withStatusCode"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordEvent"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordDeltaEvent$default$4"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.logDebug"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.logInfo"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.logInfo"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordUsage$default$5"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordOperation$default$6"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.logTrace"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.initializeLogIfNecessary"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordOperation$default$9"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordEvent$default$2"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordUsage$default$4"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.logWarning"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordUsage$default$7"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordDeltaEvent$default$3"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordOperation$default$2"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordDeltaOperation"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.logConsole"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordOperation$default$5"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordEvent$default$4"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.recordOperation$default$8"), ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaMergeBuilder.initializeLogIfNecessary$default$2"), // Changes in 0.7.0 ProblemFilters.exclude[DirectMissingMethodProblem]("io.delta.tables.DeltaTable.makeUpdateTable"), // Changes in 1.2.0 ProblemFilters.exclude[MissingClassProblem]("io.delta.storage.LogStore"), ProblemFilters.exclude[MissingClassProblem]("io.delta.storage.CloseableIterator"), // Changes in 4.0.0 ProblemFilters.exclude[IncompatibleResultTypeProblem]("io.delta.tables.DeltaTable.improveUnsupportedOpError"), ProblemFilters.exclude[IncompatibleResultTypeProblem]("io.delta.tables.DeltaMergeBuilder.improveUnsupportedOpError"), ProblemFilters.exclude[IncompatibleResultTypeProblem]("io.delta.tables.DeltaMergeBuilder.execute"), // Changes in 4.1.0 // TODO: change in type hierarchy due to removal of DeltaThrowableConditionShim ProblemFilters.exclude[MissingTypesProblem]("io.delta.exceptions.*") // scalastyle:on line.size.limit ) } ================================================ FILE: project/StandaloneMimaExcludes.scala ================================================ /* * Copyright (2020-present) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import com.typesafe.tools.mima.core._ /** * The list of Mima errors to exclude in the Standalone project. */ object StandaloneMimaExcludes { val ignoredABIProblems = Seq( // scalastyle:off line.size.limit // Ignore changes to internal Scala codes ProblemFilters.exclude[Problem]("io.delta.standalone.internal.*"), // Public API changes in 0.2.0 -> 0.3.0 ProblemFilters.exclude[ReversedMissingMethodProblem]("io.delta.standalone.DeltaLog.getChanges"), ProblemFilters.exclude[ReversedMissingMethodProblem]("io.delta.standalone.DeltaLog.startTransaction"), ProblemFilters.exclude[ReversedMissingMethodProblem]("io.delta.standalone.Snapshot.scan"), ProblemFilters.exclude[ReversedMissingMethodProblem]("io.delta.standalone.DeltaLog.tableExists"), // Switch to using delta-storage LogStore API in 0.4.0 -> 0.5.0 ProblemFilters.exclude[MissingClassProblem]("io.delta.standalone.storage.LogStore"), // Ignore missing shaded attributes ProblemFilters.exclude[Problem]("shadedelta.*"), // Public API changes in 0.4.0 -> 0.5.0 ProblemFilters.exclude[ReversedMissingMethodProblem]("io.delta.standalone.DeltaLog.getVersionBeforeOrAtTimestamp"), ProblemFilters.exclude[ReversedMissingMethodProblem]("io.delta.standalone.DeltaLog.getVersionAtOrAfterTimestamp"), // ParquetSchemaConverter etc. were moved to project standalone-parquet ProblemFilters.exclude[MissingClassProblem]("io.delta.standalone.util.ParquetSchemaConverter"), ProblemFilters.exclude[MissingClassProblem]("io.delta.standalone.util.ParquetSchemaConverter$ParquetOutputTimestampType"), // Public API changes in 0.5.0 -> 0.6.0 ProblemFilters.exclude[ReversedMissingMethodProblem]("io.delta.standalone.OptimisticTransaction.readVersion"), // scalastyle:on line.size.limit ) } ================================================ FILE: project/TestParallelization.scala ================================================ import scala.util.hashing.MurmurHash3 import sbt.Keys._ import sbt._ object TestParallelization { lazy val numShardsOpt = sys.env.get("NUM_SHARDS").map(_.toInt) lazy val shardIdOpt = sys.env.get("SHARD_ID").map(_.toInt) lazy val testParallelismOpt = sys.env.get("TEST_PARALLELISM_COUNT").map(_.toInt) lazy val settings = { println( s"Test parallelization settings: numShardsOpt=$numShardsOpt, " + s"shardIdOpt=$shardIdOpt, testParallelismOpt=$testParallelismOpt" ) if ((numShardsOpt.exists(_ > 1) && shardIdOpt.exists(_ >= 0)) || testParallelismOpt.exists(_ > 1)) { customTestGroupingSettings ++ simpleGroupingStrategySettings } else { Seq.empty[Setting[_]] } } /** * Replace the default value for Test / testGrouping settingKey and set it to a new value * calculated by using the custom Task [[testGroupingStrategy]]. * * Adding these settings to the build will require us to separately provide a value for the * TaskKey [[testGroupingStrategy]] */ lazy val customTestGroupingSettings = { Seq( Test / testGrouping := { val tests = (Test / definedTests).value val groupingStrategy = (Test / testGroupingStrategy).value val grouping = tests.foldLeft(groupingStrategy) { case (strategy, testDefinition) => strategy.add(testDefinition) } val logger = streams.value.log logger.info(s"Tests will be grouped in ${grouping.testGroups.size} groups") val groups = grouping.testGroups groups.foreach { group => logger.info(s"${group.name} contains ${group.tests.size} tests") } logger.info(groupingStrategy.toString) groups } ) } /** * Sets the Test / testGroupingStrategy Task to an instance of the MinShardGroupDurationStrategy */ lazy val simpleGroupingStrategySettings = Seq( Test / forkTestJVMCount := { testParallelismOpt.getOrElse(java.lang.Runtime.getRuntime.availableProcessors) }, Test / shardId := { shardIdOpt.getOrElse(0) }, Test / testGroupingStrategy := { val groupsCount = (Test / forkTestJVMCount).value val shard = (Test / shardId).value val baseJvmDir = baseDirectory.value MinShardGroupDurationStrategy(groupsCount, baseJvmDir, shard, defaultForkOptions.value) }, Test / parallelExecution := true, Global / concurrentRestrictions := { Seq(Tags.limit(Tags.ForkedTestGroup, (Test / forkTestJVMCount).value)) } ) val shardId = SettingKey[Int]("shard id", "The shard id assigned") val forkTestJVMCount = SettingKey[Int]( "fork test jvm count", "The number of separate JVM to use for tests" ) val testGroupingStrategy = TaskKey[GroupingStrategy]( "test grouping strategy", "The strategy to allocate different tests into groups," + "potentially using multiple JVMS for their execution" ) private val defaultForkOptions = Def.task { ForkOptions( javaHome = javaHome.value, outputStrategy = outputStrategy.value, bootJars = Vector.empty, // Use Test/baseDirectory instead of baseDirectory to support modules where these differ // (e.g. spark-combined module where Test/baseDirectory points to spark/ source directory) workingDirectory = Some((Test / baseDirectory).value), runJVMOptions = (Test / javaOptions).value.toVector, connectInput = connectInput.value, envVars = (Test / envVars).value ) } /** * Base trait to group tests. * * By default, SBT will run all tests as if they belong to a single group, but allows tests to be * grouped. Setting [[sbt.Keys.testGrouping]] to a list of groups replaces the default * single-group definition. * * When creating an instance of [[sbt.Tests.Group]] it is possible to specify an * [[sbt.Tests.TestRunPolicy]]: this parameter can be used to use multiple subprocesses for test * execution */ sealed trait GroupingStrategy { /** * Adds an [[sbt.TestDefinition]] to this GroupingStrategy and returns an updated Grouping * Strategy */ def add(testDefinition: TestDefinition): GroupingStrategy /** Returns the test groups built from this GroupingStrategy */ def testGroups: List[Tests.Group] } /** * GreedyHashStrategy is a grouping strategy used to distribute test suites across multiple shards * and groups (threads) based on their estimated duration. It aims to balance the test load across * the shards and groups by utilizing a greedy assignment algorithm that assigns test suites to * the group with the smallest estimated runtime. * * @param groups The initial mapping of group indices to their respective [[sbt.Tests.Group]] * objects, which hold test definitions. * @param shardId The shard ID that this instance is responsible for. * @param highDurationTestAssignment Precomputed assignments of high-duration test suites to * specific groups within the shard. * @param groupRuntimes Array holding the current total runtime for each group within the shard. */ class MinShardGroupDurationStrategy private( groups: scala.collection.mutable.Map[Int, Tests.Group], shardId: Int, highDurationTestAssignment: Array[Set[String]], var groupRuntimes: Array[Double] ) extends GroupingStrategy { import TestParallelization.MinShardGroupDurationStrategy._ if (shardId < 0 || shardId >= NUM_SHARDS) { throw new IllegalArgumentException( s"Assigned shard ID $shardId is not between 0 and ${NUM_SHARDS - 1} inclusive") } lazy val testGroups = groups.values.toList override def add(testDefinition: TestDefinition): GroupingStrategy = { val testSuiteName = testDefinition.name val isHighDurationTest = TOP_N_HIGH_DURATION_TEST_SUITES.exists(_._1 == testSuiteName) if (isHighDurationTest) { val highDurationTestGroupIndex = highDurationTestAssignment.indexWhere(_.contains(testSuiteName)) if (highDurationTestGroupIndex >= 0) { // Case 1: this is a high duration test that was pre-computed in the optimal assignment to // belong to this shard. Assign it. val duration = TOP_N_HIGH_DURATION_TEST_SUITES.find(_._1 == testSuiteName).get._2 val currentGroup = groups(highDurationTestGroupIndex) val updatedGroup = currentGroup.withTests(currentGroup.tests :+ testDefinition) groups(highDurationTestGroupIndex) = updatedGroup // Do NOT update groupRuntimes -- this was already included in the initial value of // groupRuntimes this } else { // Case 2: this is a high duration test that does NOT belong to this shard. Skip it. this } } else if (math.abs(MurmurHash3.stringHash(testDefinition.name) % NUM_SHARDS) == shardId) { // Case 3: this is a normal test that belongs to this shard. Assign it. val minDurationGroupIndex = groupRuntimes.zipWithIndex.minBy(_._1)._2 val currentGroup = groups(minDurationGroupIndex) val updatedGroup = currentGroup.withTests(currentGroup.tests :+ testDefinition) groups(minDurationGroupIndex) = updatedGroup groupRuntimes(minDurationGroupIndex) += AVG_TEST_SUITE_DURATION_EXCLUDING_TOP_N this } else { // Case 4: this is a normal test that does NOT belong to this shard. Skip it. this } } override def toString: String = { val actualDurationsStr = groupRuntimes.zipWithIndex.map { case (actualDuration, groupIndex) => f" Group $groupIndex: Estimated Duration = $actualDuration%.2f mins, " + f"Count = ${groups(groupIndex).tests.size}" }.mkString("\n") s""" |Shard ID: $shardId |Suite Group Assignments: |$actualDurationsStr """.stripMargin } } object MinShardGroupDurationStrategy { val NUM_SHARDS = numShardsOpt.getOrElse(1) val AVG_TEST_SUITE_DURATION_EXCLUDING_TOP_N = 0.83 /** * High-duration test suites loaded from project/test-durations.csv. * * To update, run: python3 project/scripts/collect_test_durations.py */ val TOP_N_HIGH_DURATION_TEST_SUITES: List[(String, Double)] = { val csvFile = new java.io.File("project/test-durations.csv") if (!csvFile.exists()) { println(s"Warning: ${csvFile.getPath} not found, using empty test durations") List.empty } else { val source = scala.io.Source.fromFile(csvFile) try { source.getLines().drop(1).filter(_.trim.nonEmpty).map { line => val idx = line.lastIndexOf(',') (line.substring(0, idx), line.substring(idx + 1).toDouble) }.toList } finally { source.close() } } } /** * Generates the optimal test assignment across shards and groups for high duration test suites. * * Will assign the high duration test suites in descending order, always assigning to the * group with the smallest total duration. In case of ties (e.g. early on when some group * durations are still 0, will assign to the shard with the smallest total duration). * * Here's a simple example using 3 shards and 2 groups per shard: * * Test 1: DeltaRetentionWithCatalogOwnedBatch1Suite (22.66 mins) --> Shard 0, Group 0 * - Shard 0: Group 0 = 22.66 mins, Group 1 = 0.0 mins * - Shard 1: Group 0 = 0.0 mins, Group 1 = 0.0 mins * - Shard 2: Group 0 = 0.0 mins, Group 1 = 0.0 mins * * Test 2: DeltaRetentionSuite (21.46 mins) --> Shard 1, Group 0 * - Shard 0: Group 0 = 22.66 mins, Group 1 = 0.0 mins * - Shard 1: Group 0 = 21.46 mins, Group 1 = 0.0 mins * - Shard 2: Group 0 = 0.0 mins, Group 1 = 0.0 mins * * Test 3: DeletionVectorsSuite (16.85 mins) --> Shard 2, Group 0 * - Shard 0: Group 0 = 22.66 mins, Group 1 = 0.0 mins * - Shard 1: Group 0 = 21.46 mins, Group 1 = 0.0 mins * - Shard 2: Group 0 = 16.85 mins, Group 1 = 0.0 mins * * Test 4: DataSkippingDeltaV1WithCatalogOwnedBatch100Suite (12.48 mins) --> Shard 2, Group 1 * - Shard 0: Group 0 = 22.66 mins, Group 1 = 0.0 mins * - Shard 1: Group 0 = 21.46 mins, Group 1 = 0.0 mins * - Shard 2: Group 0 = 16.85 mins, Group 1 = 12.48 mins * * Test 5: DataSkippingDeltaV1WithCatalogOwnedBatch2Suite (11.68 mins) --> Shard 1, Group 1 * - Shard 0: Group 0 = 22.66 mins, Group 1 = 0.0 mins * - Shard 1: Group 0 = 21.46 mins, Group 1 = 11.68 mins * - Shard 2: Group 0 = 16.85 mins, Group 1 = 12.48 mins * * Test 6: DeltaFastDropFeatureSuite (11.04 mins) --> Shard 0, Group 1 * - Shard 0: Group 0 = 22.66 mins, Group 1 = 11.04 mins * - Shard 1: Group 0 = 21.46 mins, Group 1 = 11.68 mins * - Shard 2: Group 0 = 16.85 mins, Group 1 = 12.48 mins */ def highDurationOptimalAssignment(numGroups: Int): (Array[Array[Set[String]]], Array[Array[Double]]) = { val assignment = Array.fill(NUM_SHARDS)(Array.fill(numGroups)(List.empty[String])) val groupDurations = Array.fill(NUM_SHARDS)(Array.fill(numGroups)(0.0)) val shardDurations = Array.fill(NUM_SHARDS)(0.0) val sortedTestSuites = TOP_N_HIGH_DURATION_TEST_SUITES.sortBy(-_._2) sortedTestSuites.foreach { case (testSuiteName, duration) => val (shardIdx, groupIdx) = findShardAndGroupWithLowestDuration(numGroups, shardDurations, groupDurations) assignment(shardIdx)(groupIdx) = assignment(shardIdx)(groupIdx) :+ testSuiteName groupDurations(shardIdx)(groupIdx) += duration shardDurations(shardIdx) += duration } (assignment.map(_.map(_.toSet)), groupDurations) } /** * Finds the best shard and group to assign the next test suite. * * Selects the group with the smallest total duration, and in case of ties, selects the shard * with the smallest total duration. * * @param numShards Number of shards * @param numGroups Number of groups per shard * @param shardDurations Total duration per shard * @param groupDurations Total duration per group in each shard * @return Tuple of (shard index, group index) for the optimal assignment */ private def findShardAndGroupWithLowestDuration( numGroups: Int, shardDurations: Array[Double], groupDurations: Array[Array[Double]]): (Int, Int) = { var bestShardIdx = -1 var bestGroupIdx = -1 var minGroupDuration = Double.MaxValue var minShardDuration = Double.MaxValue for (shardIdx <- 0 until NUM_SHARDS) { for (groupIdx <- 0 until numGroups) { val currentGroupDuration = groupDurations(shardIdx)(groupIdx) val currentShardDuration = shardDurations(shardIdx) if (currentGroupDuration < minGroupDuration || (currentGroupDuration == minGroupDuration && currentShardDuration < minShardDuration)) { minGroupDuration = currentGroupDuration minShardDuration = currentShardDuration bestShardIdx = shardIdx bestGroupIdx = groupIdx } } } (bestShardIdx, bestGroupIdx) } def apply( groupCount: Int, baseDir: File, shardId: Int, forkOptionsTemplate: ForkOptions): GroupingStrategy = { val testGroups = scala.collection.mutable.Map((0 until groupCount).map { groupIdx => val tmpDir = s"$baseDir/target/tmp/$groupIdx" java.nio.file.Files.createDirectories(java.nio.file.Paths.get(tmpDir)) val forkOptions = forkOptionsTemplate.withRunJVMOptions( runJVMOptions = forkOptionsTemplate.runJVMOptions ++ Seq(s"-Djava.io.tmpdir=$tmpDir") ) val group = Tests.Group( name = s"Test group $groupIdx", tests = Nil, runPolicy = Tests.SubProcess(forkOptions) ) groupIdx -> group }: _*) val (allShardsTestAssignments, allShardsGroupDurations) = highDurationOptimalAssignment(groupCount) new MinShardGroupDurationStrategy( testGroups, shardId, allShardsTestAssignments(shardId), allShardsGroupDurations(shardId) ) } } } ================================================ FILE: project/Unidoc.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import sbt._ import sbt.Keys._ import sbtunidoc._ import sbtunidoc.BaseUnidocPlugin.autoImport._ import sbtunidoc.ScalaUnidocPlugin.autoImport._ import sbtunidoc.JavaUnidocPlugin.autoImport._ object Unidoc { /** * Patterns are strings to do simple substring matches on the full path of every source file. */ case class SourceFilePattern(patterns: Seq[String], project: Option[Project] = None) object SourceFilePattern { def apply(patterns: String*): SourceFilePattern = SourceFilePattern(patterns.toSeq, None) } val unidocSourceFilePatterns = settingKey[Seq[SourceFilePattern]]( "Patterns to match (simple substring match) against full source file paths. " + "Matched files will be selected for generating API docs.") implicit class PatternsHelper(patterns: Seq[SourceFilePattern]) { def scopeToProject(projectToAdd: Project): Seq[SourceFilePattern] = { patterns.map(_.copy(project = Some(projectToAdd))) } } implicit class UnidocHelper(val projectToUpdate: Project) { def configureUnidoc( docTitle: String = null, generatedJavaDoc: Boolean = true, generateScalaDoc: Boolean = false, classPathToSkip: String = null ): Project = { if (sys.env.contains("DISABLE_UNIDOC")) return projectToUpdate if (!generatedJavaDoc && !generateScalaDoc) return projectToUpdate var updatedProject: Project = projectToUpdate if (generateScalaDoc) { updatedProject = updatedProject.enablePlugins(ScalaUnidocPlugin) } updatedProject .enablePlugins(GenJavadocPlugin, JavaUnidocPlugin) // TODO: Allows maven publishing to use unidoc doc jar, but it currently throws errors. // .enablePlugins(PublishJavadocPlugin) .settings( libraryDependencies ++= Seq( // Ensure genJavaDoc plugin is of the right version that works with Scala 2.13.16 compilerPlugin( "com.typesafe.genjavadoc" %% "genjavadoc-plugin" % "0.19" cross CrossVersion.full) ), generateUnidocSettings(docTitle, generateScalaDoc, classPathToSkip), // Ensure unidoc is run with tests. (Test / test) := ((Test / test) dependsOn (Compile / unidoc)).value, // hide package private types and methods in javadoc scalacOptions ++= Seq( "-P:genjavadoc:strictVisibility=true" ), ) } private def generateUnidocSettings( customDocTitle: String, generateScalaDoc: Boolean, classPathToSkip : String): Def.SettingsDefinition = { val internalFilePattern = Seq("/internal/", "/execution/", "$") // Generate the full doc title def fullDocTitle(projectName: String, version: String, isScalaDoc: Boolean): String = { val namePart = Option(customDocTitle).getOrElse { projectName.split("-").map(_.capitalize).mkString(" ") } val versionPart = version.replaceAll("-SNAPSHOT", "") val langPart = if (isScalaDoc) "Scala API Docs" else "Java API Docs" s"$namePart $versionPart - $langPart" } // Remove source files that does not match the pattern def ignoreUndocumentedSources( allSourceFiles: Seq[Seq[java.io.File]], sourceFilePatternsToKeep: Seq[SourceFilePattern] ): Seq[Seq[java.io.File]] = { if (sourceFilePatternsToKeep.isEmpty) return Nil val projectSrcDirToFilePatternsToKeep = sourceFilePatternsToKeep.map { case SourceFilePattern(dirs, projOption) => val projectPath = projOption.getOrElse(projectToUpdate).base.getCanonicalPath projectPath -> dirs }.toMap def shouldKeep(path: String): Boolean = { projectSrcDirToFilePatternsToKeep.foreach { case (projBaseDir, filePatterns) => def isInProjectSrcDir = path.contains(s"$projBaseDir/src") || path.contains(s"$projBaseDir/target/java/") def matchesFilePattern = filePatterns.exists(path.contains(_)) def matchesInternalFilePattern = internalFilePattern.exists(path.contains(_)) if (isInProjectSrcDir && matchesFilePattern && !matchesInternalFilePattern) return true } false } allSourceFiles.map {_.filter(f => shouldKeep(f.getCanonicalPath))} } val javaUnidocSettings = Seq( // Configure Java unidoc JavaUnidoc / unidoc / javacOptions := Seq( "-public", "-windowtitle", fullDocTitle((projectToUpdate / name).value, version.value, isScalaDoc = false), "-noqualifier", "java.lang", "-tag", "implNote:a:Implementation Note:", "-tag", "apiNote:a:API Note:", "-Xdoclint:none" ), JavaUnidoc / unidoc / unidocAllSources := { ignoreUndocumentedSources( allSourceFiles = (JavaUnidoc / unidoc / unidocAllSources).value, sourceFilePatternsToKeep = unidocSourceFilePatterns.value) }, // Settings for plain, old Java doc needed for successful doc generation during publishing. Compile / doc / javacOptions ++= Seq( "-public", "-noqualifier", "java.lang", "-tag", "implNote:a:Implementation Note:", "-tag", "apiNote:a:API Note:", "-Xdoclint:all") ) val scalaUnidocSettings = if (generateScalaDoc) Seq( // Configure Scala unidoc ScalaUnidoc / unidoc / scalacOptions ++= Seq( "-doc-title", fullDocTitle((projectToUpdate / name).value, version.value, isScalaDoc = true), ), ScalaUnidoc / unidoc / unidocAllSources := { ignoreUndocumentedSources( allSourceFiles = (ScalaUnidoc / unidoc / unidocAllSources).value, sourceFilePatternsToKeep = unidocSourceFilePatterns.value ) }, ScalaUnidoc / unidoc / fullClasspath := { (ScalaUnidoc / unidoc / fullClasspath).value .filter(f => classPathToSkip == null || !f.data.getCanonicalPath.contains(classPathToSkip)) } ) else Nil javaUnidocSettings ++ scalaUnidocSettings } } } ================================================ FILE: project/build.properties ================================================ # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # # This file contains code from the Apache Spark project (original license above). # It contains modifications, which are licensed as follows: # # # Copyright (2021) The Delta Lake Project Authors. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # sbt.version=1.9.9 ================================================ FILE: project/plugins.sbt ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ libraryDependencies += "org.apache.commons" % "commons-compress" % "1.0" addSbtPlugin("com.github.sbt" % "sbt-release" % "1.1.0") addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.1.2") addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0") addSbtPlugin("com.github.sbt" % "sbt-unidoc" % "0.5.0") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.0") addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "1.1.3") addSbtPlugin("com.simplytyped" % "sbt-antlr4" % "0.8.3") addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.11.3") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.4.0") //Upgrade sbt-scoverage to 2.0.3+ because 2.0.0 is not compatible to Scala 2.12.17: //sbt.librarymanagement.ResolveException: Error downloading org.scoverage:scalac-scoverage-plugin_2.12.17:2.0.0 //It caused a conflict issue: //[error] java.lang.RuntimeException: found version conflict(s) in library dependencies; some are suspected to be binary incompatible: //[error] //[error] * org.scala-lang.modules:scala-xml_2.12:2.1.0 (early-semver) is selected over 1.0.6 //[error] +- org.scoverage:scalac-scoverage-reporter_2.12:2.0.7 (depends on 2.1.0) //[error] +- org.scalariform:scalariform_2.12:0.2.0 (depends on 1.0.6) //The following fix the conflict: libraryDependencySchemes += "org.scala-lang.modules" %% "scala-xml" % VersionScheme.Always % "test" addSbtPlugin("com.github.sbt.junit" % "sbt-jupiter-interface" % "0.17.0") addSbtPlugin("software.purpledragon" % "sbt-checkstyle-plugin" % "4.0.1") // By default, sbt-checkstyle-plugin uses checkstyle version 6.15, but we should set it to use the // same version as Spark dependencyOverrides += "com.puppycrawl.tools" % "checkstyle" % "9.3" addSbtPlugin("com.thesamet" % "sbt-protoc" % "1.0.7") addSbtPlugin("com.lightbend.sbt" % "sbt-java-formatter" % "0.8.0") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.4") ================================================ FILE: project/project/plugins.sbt ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.9") ================================================ FILE: project/scripts/collect_test_durations.py ================================================ #!/usr/bin/env python3 """ Collect per-suite test durations from CI and write project/test-durations.csv. Usage: # Update test-durations.csv from last 30 successful runs on master (default): python3 project/scripts/collect_test_durations.py # Use more runs for averaging: python3 project/scripts/collect_test_durations.py --last-n-runs 50 # Time-bound the run to 5 minutes (stops downloading more artifacts after the limit): python3 project/scripts/collect_test_durations.py --max-minutes 5 # Disable the time limit (process all --last-n-runs runs unconditionally): python3 project/scripts/collect_test_durations.py --max-minutes 0 Requirements: - gh CLI authenticated with access to delta-io/delta - Python 3.6+ """ import argparse import json import os import re import subprocess import sys import tempfile import time import zipfile from collections import defaultdict from xml.etree import ElementTree REPO = "delta-io/delta" CSV_FILE = "project/test-durations.csv" DEFAULT_LAST_N_RUNS = 30 DEFAULT_TOP_N = 200 DEFAULT_MAX_MINUTES = 5 def run_gh(args): """Run a gh CLI command and return stdout.""" cmd = ["gh"] + args result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if result.returncode != 0: print(f"Error running: {' '.join(cmd)}", file=sys.stderr) print(result.stderr.decode(), file=sys.stderr) sys.exit(1) return result.stdout.decode() def get_run_ids(last_n): """Find the latest N successful workflow run IDs on master.""" runs_json = run_gh([ "run", "list", "--repo", REPO, "--branch", "master", "--workflow", "spark_test.yaml", "--status", "success", "--limit", str(last_n), "--json", "databaseId,headSha,createdAt" ]) runs = json.loads(runs_json) if not runs: print("No successful runs found on master", file=sys.stderr) sys.exit(1) for run in runs: print(f" Run {run['databaseId']} (commit: {run['headSha'][:8]}, " f"created: {run['createdAt']})") return [run["databaseId"] for run in runs] def list_artifacts(run_id): """List test-report artifacts from a workflow run.""" artifacts_json = run_gh([ "api", f"repos/{REPO}/actions/runs/{run_id}/artifacts", "--paginate", "--jq", ".artifacts[]" ]) artifacts = [] for line in artifacts_json.strip().split('\n'): if not line.strip(): continue art = json.loads(line) match = re.match(r"test-reports-spark([\d.]+)-shard(\d+)", art["name"]) if match: artifacts.append({ "id": art["id"], "spark_version": match.group(1), "shard": int(match.group(2)), }) artifacts.sort(key=lambda a: (a["spark_version"], a["shard"])) return artifacts def download_and_extract(artifact_id, dest_dir): """Download and extract a GitHub Actions artifact zip.""" zip_path = os.path.join(dest_dir, f"{artifact_id}.zip") cmd = ["gh", "api", f"repos/{REPO}/actions/artifacts/{artifact_id}/zip"] with open(zip_path, 'wb') as f: result = subprocess.run(cmd, stdout=f, stderr=subprocess.PIPE) if result.returncode != 0: print(f"Error downloading artifact {artifact_id}", file=sys.stderr) sys.exit(1) extract_dir = os.path.join(dest_dir, str(artifact_id)) os.makedirs(extract_dir, exist_ok=True) with zipfile.ZipFile(zip_path, 'r') as zf: zf.extractall(extract_dir) return extract_dir def parse_junit_xmls(directory): """Parse all JUnit XML files in a directory. Returns {suite_name: duration_minutes}.""" durations = {} for dirpath, _, filenames in os.walk(directory): for fname in filenames: if not fname.endswith('.xml'): continue try: tree = ElementTree.parse(os.path.join(dirpath, fname)) except ElementTree.ParseError: continue root = tree.getroot() suites = [root] if root.tag == "testsuite" else root.findall("testsuite") for suite in suites: name = suite.get("name", "") if not name: continue try: dur = round(float(suite.get("time", "0")) / 60, 2) except ValueError: continue if name not in durations or dur > durations[name]: durations[name] = dur return durations def collect_from_run(run_id, tmpdir): """Collect test durations from a single CI run.""" artifacts = list_artifacts(run_id) if not artifacts: print(f" No artifacts in run {run_id}", file=sys.stderr) return {} run_durations = {} for art in artifacts: print(f" Spark {art['spark_version']}, Shard {art['shard']}...") artifact_dir = download_and_extract(art["id"], tmpdir) for name, dur in parse_junit_xmls(artifact_dir).items(): if name not in run_durations or dur > run_durations[name]: run_durations[name] = dur return run_durations def main(): parser = argparse.ArgumentParser( description="Collect test durations from CI and update project/test-durations.csv" ) parser.add_argument("--last-n-runs", type=int, default=DEFAULT_LAST_N_RUNS, help=f"Number of runs to average (default: {DEFAULT_LAST_N_RUNS})") parser.add_argument("--top-n", type=int, default=DEFAULT_TOP_N, help=f"Number of slowest suites to keep (default: {DEFAULT_TOP_N})") parser.add_argument("--max-minutes", type=float, default=DEFAULT_MAX_MINUTES, help=f"Stop downloading after this many minutes (default: {DEFAULT_MAX_MINUTES}," " 0 = no limit)") args = parser.parse_args() # Fetch run IDs print("Finding recent successful runs on master...") run_ids = get_run_ids(args.last_n_runs) deadline = time.monotonic() + args.max_minutes * 60 if args.max_minutes > 0 else None # Collect durations from each run all_durations = defaultdict(list) with tempfile.TemporaryDirectory() as tmpdir: for i, run_id in enumerate(run_ids): if deadline is not None and time.monotonic() >= deadline: print(f"\nTime limit reached after {i} run(s); stopping early.") break print(f"\nRun {i + 1}/{len(run_ids)}: {run_id}") for name, dur in collect_from_run(run_id, tmpdir).items(): all_durations[name].append(dur) # Average and sort averaged = { name: round(sum(durs) / len(durs), 2) for name, durs in all_durations.items() } sorted_suites = sorted(averaged.items(), key=lambda x: -x[1])[:args.top_n] if not sorted_suites: print("\nNo test-report artifacts found. Has the JUnit XML upload been enabled on master?") print(f"{CSV_FILE} was NOT modified.") sys.exit(1) # Write CSV with open(CSV_FILE, 'w') as f: f.write("suite_name,duration_minutes\n") for name, dur in sorted_suites: f.write(f"{name},{dur}\n") print(f"\nWrote {len(sorted_suites)} suites to {CSV_FILE}") print(f"Total duration: {sum(d for _, d in sorted_suites):.1f} min") print(f"Slowest: {sorted_suites[0][0]} ({sorted_suites[0][1]} min)") print(f"Fastest included: {sorted_suites[-1][0]} ({sorted_suites[-1][1]} min)") if __name__ == "__main__": main() ================================================ FILE: project/scripts/get_spark_version_info.py ================================================ #!/usr/bin/env python3 """ Generate Spark version information for CI/CD from CrossSparkVersions.scala This script reads the JSON file generated by `build/sbt exportSparkVersionsJson` and provides utilities for GitHub Actions workflows. The script automatically generates the JSON file if it doesn't exist. Usage: # Get all Spark versions as JSON array python project/scripts/get_spark_version_info.py --all-spark-versions # Output: ["4.0", "4.1"] or ["master", "4.0"] if master is present # Get only released Spark versions (no snapshots) python project/scripts/get_spark_version_info.py --released-spark-versions # Output: ["4.0", "4.1"] (excludes versions with -SNAPSHOT) # Get a specific field for a Spark version (using short version or "master") python project/scripts/get_spark_version_info.py --get-field 4.0 targetJvm python project/scripts/get_spark_version_info.py --get-field master targetJvm # Output: "17" """ import argparse import json import subprocess import sys from pathlib import Path def generate_spark_versions_json(repo_root: Path) -> bool: """Generate the spark-versions.json file by running sbt exportSparkVersionsJson.""" try: print("Generating spark-versions.json...", file=sys.stderr) subprocess.run( ["build/sbt", "exportSparkVersionsJson"], cwd=repo_root, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE ) return True except subprocess.CalledProcessError as e: print(f"ERROR: Failed to generate spark-versions.json: {e}", file=sys.stderr) return False def load_spark_versions(json_path: Path, repo_root: Path): """Load Spark versions from JSON file, generating it if necessary.""" if not json_path.exists(): if not generate_spark_versions_json(repo_root): sys.exit(1) if not json_path.exists(): print( f"ERROR: spark-versions.json not found at {json_path} even after generation", file=sys.stderr ) sys.exit(1) with open(json_path, 'r') as f: return json.load(f) def main(): parser = argparse.ArgumentParser( description="Generate Spark version information from CrossSparkVersions.scala" ) parser.add_argument( "--all-spark-versions", action="store_true", help="Output all Spark versions as JSON array (e.g., [\"4.0\", \"4.1\"] or [\"master\", \"4.0\"])" ) parser.add_argument( "--released-spark-versions", action="store_true", help="Output only released Spark versions (excluding snapshots) as JSON array" ) parser.add_argument( "--get-field", nargs=2, metavar=("SPARK_VERSION", "FIELD"), help="Get a specific field for a Spark version (e.g., --get-field 4.0 targetJvm or --get-field master targetJvm)" ) args = parser.parse_args() # Determine JSON path (relative to repo root) script_dir = Path(__file__).parent repo_root = script_dir.parent.parent json_path = repo_root / "target" / "spark-versions.json" try: versions = load_spark_versions(json_path, repo_root) if args.all_spark_versions: # For master version, use "master"; for others, use short version matrix_versions = [] for v in versions: if v.get("isMaster", False): matrix_versions.append("master") else: matrix_versions.append(v["shortVersion"]) print(json.dumps(matrix_versions)) elif args.released_spark_versions: # Only include released versions (no -SNAPSHOT in fullVersion) matrix_versions = [] for v in versions: if "-SNAPSHOT" not in v["fullVersion"]: matrix_versions.append(v["shortVersion"]) print(json.dumps(matrix_versions)) elif args.get_field: spark_version, field = args.get_field # Find the version entry by matching: # - "master" matches isMaster=true # - short version like "4.0" matches shortVersion # - full version like "4.0.1" matches fullVersion version_entry = None for v in versions: if spark_version == "master" and v.get("isMaster", False): version_entry = v break elif spark_version == v["shortVersion"] or spark_version == v["fullVersion"]: version_entry = v break if not version_entry: print(f"ERROR: Spark version '{spark_version}' not found", file=sys.stderr) sys.exit(1) if field not in version_entry: print( f"ERROR: Field '{field}' not found for Spark version {spark_version}\n" f"Available fields: {', '.join(version_entry.keys())}", file=sys.stderr ) sys.exit(1) # Print as JSON for proper formatting print(json.dumps(version_entry[field])) else: parser.print_help() sys.exit(1) except Exception as e: print(f"ERROR: {e}", file=sys.stderr) import traceback traceback.print_exc() sys.exit(1) if __name__ == "__main__": main() ================================================ FILE: project/test-durations.csv ================================================ suite_name,duration_minutes org.apache.spark.sql.delta.DeltaRetentionWithCatalogOwnedBatch1Suite,22.66 org.apache.spark.sql.delta.DeltaRetentionSuite,21.46 org.apache.spark.sql.delta.deletionvectors.DeletionVectorsSuite,16.85 org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch100Suite,12.48 org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch2Suite,11.68 org.apache.spark.sql.delta.DeltaFastDropFeatureSuite,11.04 org.apache.spark.sql.delta.stats.DataSkippingDeltaV1ParquetCheckpointV2Suite,10.29 io.delta.sharing.spark.DeltaSharingDataSourceDeltaSuite,10.28 org.apache.spark.sql.delta.DeltaSourceLargeLogWithCoordinatedCommitsBatch1Suite,9.89 org.apache.spark.sql.delta.DeltaSourceWithCoordinatedCommitsBatch100Suite,9.2 org.apache.spark.sql.delta.DeltaSourceLargeLogSuite,8.86 org.apache.spark.sql.delta.DeltaInsertIntoSchemaEvolutionSuite,8.7 org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch1Suite,8.68 org.apache.spark.sql.delta.DeltaSourceWithCoordinatedCommitsBatch1Suite,8.64 org.apache.spark.sql.delta.DeltaSourceWithCoordinatedCommitsBatch10Suite,8.61 org.apache.spark.sql.delta.CheckpointsWithCatalogOwnedBatch1Suite,8.55 org.apache.spark.sql.delta.DeltaSourceLargeLogWithCoordinatedCommitsBatch100Suite,8.39 org.apache.spark.sql.delta.DeltaVacuumSuite,8.16 org.apache.spark.sql.delta.ImplicitMergeCastingSuite,8.12 org.apache.spark.sql.delta.stats.DataSkippingDeltaV1JsonCheckpointV2Suite,7.87 org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite,7.73 org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite,7.58 org.apache.spark.sql.delta.typewidening.TypeWideningInsertSchemaEvolutionExtendedSuite,7.3 org.apache.spark.sql.delta.DeltaCDCScalaWithCatalogOwnedBatch2Suite,7.24 org.apache.spark.sql.delta.DeltaInsertIntoMissingColumnSuite,7.08 org.apache.spark.sql.delta.DescribeDeltaHistorySuite,6.76 org.apache.spark.sql.delta.generatedsuites.DeltaInsertIntoImplicitCastSuite,6.63 io.delta.sharing.spark.DeltaSharingDataSourceCMSuite,6.43 org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceUpd7CQVRRQSuite,6.38 io.delta.sharing.spark.DeltaFormatSharingSourceSuite,6.26 org.apache.spark.sql.delta.DeltaCDCScalaWithCatalogOwnedBatch1Suite,6.14 org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite,6.04 org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite,5.99 io.delta.tables.DeltaTableSuite,5.93 org.apache.spark.sql.delta.DeltaWithCatalogOwnedBatch2Suite,5.89 org.apache.spark.sql.delta.stats.DataSkippingDeltaV1Suite,5.88 org.apache.spark.sql.delta.DeltaWithCatalogOwnedBatch1Suite,5.8 org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionInsertSQLPathBasedCDCOnDVsPredPushOffSuite,5.77 org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpR36OX5ISuite,5.7 org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscSQLPathBasedCDCOnDVsPredPushOnSuite,5.69 org.apache.spark.sql.delta.GenerateIdentityValuesSuite,5.59 org.apache.spark.sql.delta.generatedsuites.MergeIntoSchemaEvolutionBaseExistingColumnSQLPathBasedCDCOnDVsPredPushOffSuite,5.51 org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceU6MQ3SIISuite,5.51 org.apache.spark.sql.delta.stats.StatsCollectionSuite,5.42 org.apache.spark.sql.delta.DeltaInsertIntoColumnOrderSuite,5.38 org.apache.spark.sql.delta.columnmapping.RemoveColumnMappingCDCSuite,5.36 org.apache.spark.sql.delta.generatedsuites.MergeIntoSchemaEvolutionBaseNewColumnSQLPathBasedCDCOnDVsPredPushOffSuite,5.07 org.apache.spark.sql.delta.DeltaLiteVacuumSuite,5.05 org.apache.spark.sql.delta.DeltaProtocolVersionSuite,5.04 org.apache.spark.sql.delta.CheckpointsWithCatalogOwnedBatch2Suite,4.97 org.apache.spark.sql.delta.GeneratedColumnSuite,4.95 org.apache.spark.sql.delta.DeltaSourceSuite,4.87 org.apache.spark.sql.connect.delta.DeltaConnectPlannerSuite,4.86 org.apache.spark.sql.delta.DeltaSuite,4.83 org.apache.spark.sql.delta.deletionvectors.DeletionVectorsWithPredicatePushdownSuite,4.83 org.apache.spark.sql.delta.DeltaWithCatalogOwnedBatch100Suite,4.8 org.apache.spark.sql.delta.generatedsuites.MergeIntoSchemaEvolutionBaseNewColumnSQLPathBasedDVsPredPushOffSuite,4.68 org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpdaH76RPFYSuite,4.66 org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscSQLPathBasedDVsPredPushOffSuite,4.62 org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscSQLPathBasedCDCOnDVsPredPushOffSuite,4.59 org.apache.spark.sql.delta.CheckpointsSuite,4.55 org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpFQ7PINASuite,4.46 org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpda5FZ34QYSuite,4.46 org.apache.spark.sql.delta.test.DeltaV2SourceSuite,4.42 org.apache.spark.sql.delta.IdentityColumnSyncScalaSuite,4.39 org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite,4.33 org.apache.spark.sql.delta.typewidening.TypeWideningTableFeatureDropSuite,4.32 org.apache.spark.sql.delta.generatedsuites.MergeIntoSchemaEvolutionBaseNewColumnSQLPathBasedCDCOnSuite,4.32 org.apache.spark.sql.delta.DeltaSinkImplicitCastWithCoordinatedCommitsBatch100Suite,4.29 org.apache.spark.sql.delta.DeltaCDCScalaWithDeletionVectorsSuite,4.27 org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpdateStarOnSuite,4.25 org.apache.spark.sql.delta.CheckpointsWithCatalogOwnedBatch100Suite,4.24 org.apache.spark.sql.delta.generatedsuites.UpdateBaseMiscSQLPathBasedCDCOnSuite,4.23 org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionInsertSQLPathBasedCDCOnSuite,4.21 org.apache.spark.sql.delta.ChecksumDVMetricsSuite,4.21 org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceUpdateStarOffSuite,4.15 org.apache.spark.sql.delta.DeltaSourceNameColumnMappingSuite,4.14 org.apache.spark.sql.delta.DeltaInsertIntoSQLByPathSuite,4.12 org.apache.spark.sql.delta.hudi.ConvertToHudiSuite,4.07 org.apache.spark.sql.delta.generatedsuites.MergeIntoExtendedSyntaxSQLPathBasedDVsPredPushOnSuite,4.03 org.apache.spark.sql.delta.schema.InvariantEnforcementSuite,3.97 org.apache.spark.sql.delta.DeltaLogSuite,3.96 org.apache.spark.sql.delta.IdentityColumnIngestionScalaSuite,3.96 org.apache.spark.sql.delta.typewidening.TypeWideningAlterTableSuite,3.92 org.apache.spark.sql.delta.generatedsuites.UpdateBaseMiscSQLPathBasedCDCOnDVSuite,3.92 org.apache.spark.sql.delta.generatedsuites.RowTrackingMergeCommonNameBasedRowTrackingMergeDVSuite,3.91 org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceUpdateStarOnSuite,3.88 org.apache.spark.sql.delta.DeltaProtocolTransitionsSuite,3.85 org.apache.spark.sql.delta.stats.PartitionLikeDataSkippingSuite,3.8 org.apache.spark.sql.delta.generatedsuites.MergeIntoSchemaEvolutionBaseNewColumnScalaSuite,3.8 org.apache.spark.sql.delta.DeltaSourceIdColumnMappingSuite,3.78 org.apache.spark.sql.delta.generatedsuites.MergeIntoBasicSQLPathBasedCDCOnDVsPredPushOnSuite,3.71 org.apache.spark.sql.delta.DeltaTimeTravelWithCatalogOwnedBatch1Suite,3.64 org.apache.spark.sql.delta.ConvertToDeltaScalaSuite,3.63 org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionUpdateOnlySQLPathBasedSuite,3.63 org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionUpdateOnlyScalaSuite,3.59 org.apache.spark.sql.delta.InCommitTimestampWithCatalogOwnedBatch2Suite,3.58 org.apache.spark.sql.delta.generatedsuites.MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpdateStarOnSuite,3.57 org.apache.spark.sql.delta.generatedsuites.MergeIntoNotMatchedBySourceCDCPart1SQLPathBasedCDCOnDVsPredPushOnSuite,3.55 org.apache.spark.sql.delta.DeltaLogWithCatalogOwnedBatch1Suite,3.54 ================================================ FILE: project/tests/test_cross_spark_publish.py ================================================ #!/usr/bin/env python3 """ Cross-Spark Version Build Testing Tests the Delta Lake build system by validating JAR file names for: 1. Default publish (publishM2) - publishes ALL modules WITH Spark suffix 2. Backward-compat publish (skipSparkSuffix=true) - publishes WITHOUT suffix 3. Full cross-version workflow publishes both with and without suffix Usage: python project/tests/test_cross_spark_publish.py The script will: 1. Test default publishM2 command publishes all modules WITH Spark suffix 2. Test skipSparkSuffix=true publishes WITHOUT suffix (backward compatibility) 3. Test full cross-version build workflow (both with and without suffix) 4. Exit with status 0 on success, 1 on failure """ import json import subprocess import sys from dataclasses import dataclass from pathlib import Path from typing import List, Set, Dict # Spark-related modules (requiresCrossSparkBuild := true) # These modules get a Spark version suffix (e.g., _4.0) for non-default versions # Template format: {suffix} = short Spark version suffix (e.g., "", "_4.0") # {version} = full Delta version (e.g., "3.4.0-SNAPSHOT") SPARK_RELATED_JAR_TEMPLATES = [ "delta-spark{suffix}_2.13-{version}.jar", "delta-connect-common{suffix}_2.13-{version}.jar", "delta-connect-client{suffix}_2.13-{version}.jar", "delta-connect-server{suffix}_2.13-{version}.jar", "delta-sharing-spark{suffix}_2.13-{version}.jar", ] # Iceberg-related modules - only built for Spark versions with supportIceberg=true # delta-iceberg has no Spark suffix (always delta-iceberg_2.13) because it only supports Spark 4.0 DELTA_ICEBERG_JAR_TEMPLATES = [ "delta-iceberg_2.13-{version}.jar", ] # Hudi-related modules - only built for Spark versions with supportHudi=true # delta-hudi has no Spark suffix (always delta-hudi_2.13) DELTA_HUDI_JAR_TEMPLATES = [ "delta-hudi_2.13-{version}.jar", ] # Non-spark-related modules (built once, same for all Spark versions) # Template format: {version} = Delta version (e.g., "3.4.0-SNAPSHOT") NON_SPARK_RELATED_JAR_TEMPLATES = [ "delta-storage-{version}.jar", "delta-kernel-api-{version}.jar", "delta-kernel-defaults-{version}.jar", "delta-storage-s3-dynamodb-{version}.jar", "delta-kernel-unitycatalog-{version}.jar", "delta-contribs_2.13-{version}.jar", ] @dataclass class SparkVersionSpec: """Configuration for a specific Spark version. Mirrors the SparkVersionSpec in CrossSparkVersions.scala. """ suffix: str # e.g., "" for default, "_X.Y" for other versions support_iceberg: bool = False # Whether this Spark version supports iceberg integration support_hudi: bool = True # Whether this Spark version supports hudi integration def __post_init__(self): """Generate JAR templates with the suffix applied.""" # Generate Spark-related JAR templates with the suffix self.spark_related_jars = [ jar.format(suffix=self.suffix, version="{version}") for jar in SPARK_RELATED_JAR_TEMPLATES ] # Iceberg JARs have no Spark suffix (always delta-iceberg_2.13) if self.support_iceberg: self.iceberg_jars = list(DELTA_ICEBERG_JAR_TEMPLATES) else: self.iceberg_jars = [] # Hudi JARs have no Spark suffix (always delta-hudi_2.13) if self.support_hudi: self.hudi_jars = list(DELTA_HUDI_JAR_TEMPLATES) else: self.hudi_jars = [] # Non-Spark-related JAR templates are the same for all Spark versions self.non_spark_related_jars = list(NON_SPARK_RELATED_JAR_TEMPLATES) @property def all_jars(self) -> List[str]: """All JAR templates for this Spark version.""" return self.spark_related_jars + self.non_spark_related_jars + self.iceberg_jars + self.hudi_jars # Spark versions to test (key = full version string, value = spec with suffix) # By default, ALL versions get a Spark suffix (e.g., delta-spark_4.0_2.13) # skipSparkSuffix=true removes the suffix (used during release for backward compat) # These should mirror CrossSparkVersions.scala SPARK_VERSIONS: Dict[str, SparkVersionSpec] = { "4.0.1": SparkVersionSpec(suffix="_4.0", support_iceberg=True, support_hudi=True), "4.1.0": SparkVersionSpec(suffix="_4.1", support_iceberg=False, support_hudi=False) } # The default Spark version # This is intentionally hardcoded here to explicitly test the default version. DEFAULT_SPARK = "4.1.0" def substitute_xversion(jar_templates: List[str], delta_version: str) -> Set[str]: """ Substitutes {version} placeholder in JAR templates with actual Delta version. """ return {jar.format(version=delta_version) for jar in jar_templates} class CrossSparkPublishTest: """Tests cross-Spark version builds.""" def __init__(self, delta_root: Path): self.delta_root = delta_root self.delta_version = self._get_delta_version() self.scala_version = "2.13" def _get_delta_version(self) -> str: """Reads Delta version from version.sbt.""" with open(self.delta_root / "version.sbt", 'r') as f: for line in f: if 'version :=' in line: return line.split('"')[1] sys.exit("Error: Could not parse version from version.sbt") def clean_maven_cache(self) -> None: """Clears Maven local cache for io.delta artifacts.""" import shutil m2_repo = Path.home() / ".m2" / "repository" / "io" / "delta" if m2_repo.exists(): print(f"Cleaning Maven cache: {m2_repo}") shutil.rmtree(m2_repo) print("✓ Maven cache cleaned\n") else: print("Maven cache already clean\n") def find_all_jars(self) -> Set[str]: """Finds all JAR files from Maven local repository.""" m2_repo = Path.home() / ".m2" / "repository" / "io" / "delta" if not m2_repo.exists(): return set() found_jars = set() for version_dir in m2_repo.rglob(self.delta_version): for jar_file in version_dir.glob("*.jar"): # Exclude test/source/javadoc JARs if not any(x in jar_file.name for x in ["-tests", "-sources", "-javadoc"]): found_jars.add(jar_file.name) return found_jars def run_sbt_command(self, description: str, command: List[str]) -> bool: """Runs an SBT command and returns True if successful.""" print(f" {description}") try: subprocess.run(command, cwd=self.delta_root, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) return True except subprocess.CalledProcessError: print(f" ✗ Command failed: {' '.join(command)}") return False def validate_jars(self, expected: Set[str], test_name: str) -> bool: """Validates that found JARs match expected JARs exactly.""" found = self.find_all_jars() print(f"\n{test_name} - Found JARs ({len(found)} total):") for jar in sorted(found): print(f" {jar}") print(f"\n{test_name} - Expected JARs ({len(expected)} total):") for jar in sorted(expected): print(f" {jar}") missing = expected - found extra = found - expected print() if not missing and not extra: print(f"✓ {test_name} - All expected JARs found") return True if missing: print(f"✗ {test_name} - Missing JARs ({len(missing)}):") for jar in sorted(missing): print(f" ✗ {jar}") if extra: print(f"\n✗ {test_name} - Unexpected JARs ({len(extra)}):") for jar in sorted(extra): print(f" ✗ {jar}") return False def test_default_publish(self) -> bool: """Default publishM2 should publish ALL modules WITH Spark suffix.""" spark_spec = SPARK_VERSIONS[DEFAULT_SPARK] print("\n" + "="*70) print(f"TEST: Default publishM2 (should publish ALL modules WITH suffix for Spark {DEFAULT_SPARK})") print("="*70) self.clean_maven_cache() if not self.run_sbt_command( "Running: build/sbt publishM2", ["build/sbt", "publishM2"] ): return False # Default behavior: all Spark-dependent modules have suffix (e.g., delta-spark_4.0_2.13) expected = substitute_xversion(spark_spec.all_jars, self.delta_version) return self.validate_jars(expected, "Default publishM2 (with suffix)") def test_backward_compat_publish(self) -> bool: """skipSparkSuffix=true should publish ALL modules WITHOUT Spark suffix.""" # Create a spec without suffix for backward compatibility # Uses the same iceberg support as the default Spark version default_spark_spec = SPARK_VERSIONS[DEFAULT_SPARK] spark_spec_no_suffix = SparkVersionSpec(suffix="", support_iceberg=default_spark_spec.support_iceberg, support_hudi=default_spark_spec.support_hudi) print("\n" + "="*70) print(f"TEST: skipSparkSuffix=true (backward compatibility - no suffix)") print("="*70) self.clean_maven_cache() if not self.run_sbt_command( "Running: build/sbt -DskipSparkSuffix=true publishM2", ["build/sbt", "-DskipSparkSuffix=true", "publishM2"] ): return False # Expect artifacts WITHOUT suffix (e.g., delta-spark_2.13 instead of delta-spark_4.0_2.13) expected = substitute_xversion(spark_spec_no_suffix.all_jars, self.delta_version) return self.validate_jars(expected, "skipSparkSuffix=true (backward compat)") def test_cross_spark_workflow(self) -> bool: """Full cross-Spark workflow: backward-compat (no suffix) + all versions (with suffix).""" print("\n" + "="*70) print("TEST: Cross-Spark Workflow (backward-compat + all non-master with suffix)") print("="*70) self.clean_maven_cache() # Step 1: Publish all modules WITHOUT suffix (backward compatibility) if not self.run_sbt_command( "Step 1: build/sbt -DskipSparkSuffix=true publishM2 (backward compat, no suffix)", ["build/sbt", "-DskipSparkSuffix=true", "publishM2"] ): return False # Step 2: Publish Spark-dependent modules WITH suffix for each non-master version for spark_version, spark_spec in SPARK_VERSIONS.items(): # Skip master/snapshot versions if "SNAPSHOT" in spark_version: continue if not self.run_sbt_command( f"Step 2: build/sbt -DsparkVersion={spark_version} \"runOnlyForReleasableSparkModules publishM2\" (with suffix)", ["build/sbt", f"-DsparkVersion={spark_version}", "runOnlyForReleasableSparkModules publishM2"] ): return False # Build expected JARs: # 1. All modules WITHOUT suffix (from Step 1 - backward compat) # 2. Spark-dependent modules WITH suffix for each non-master version (from Step 2) # 3. Iceberg/Hudi JARs for supported versions (no Spark suffix) expected = set() # Step 1: All modules without suffix (uses default Spark version's iceberg support) default_spark_spec = SPARK_VERSIONS[DEFAULT_SPARK] no_suffix_spec = SparkVersionSpec(suffix="", support_iceberg=default_spark_spec.support_iceberg, support_hudi=default_spark_spec.support_hudi) expected.update(substitute_xversion(no_suffix_spec.all_jars, self.delta_version)) # Step 2: Spark-dependent modules WITH suffix for each non-master version for spark_version, spark_spec in SPARK_VERSIONS.items(): if "SNAPSHOT" in spark_version: continue # Skip master/snapshot expected.update(substitute_xversion(spark_spec.spark_related_jars, self.delta_version)) expected.update(substitute_xversion(spark_spec.iceberg_jars, self.delta_version)) expected.update(substitute_xversion(spark_spec.hudi_jars, self.delta_version)) return self.validate_jars(expected, "Cross-Spark Workflow") def validate_spark_versions(self) -> None: """ Validates that Spark versions in this test match those in CrossSparkVersions.scala. Uses 'build/sbt showSparkVersions' to query versions directly from the build. """ try: # Query Spark versions from SBT result = subprocess.run( ["build/sbt", "showSparkVersions"], cwd=self.delta_root, capture_output=True, text=True, check=True ) # Parse output - each line is a Spark version # Version format: X.Y.Z or X.Y.Z-SNAPSHOT import re version_pattern = re.compile(r'^\d+\.\d+\.\d+(-SNAPSHOT)?$') build_versions = set() for line in result.stdout.strip().split('\n'): line = line.strip() if version_pattern.match(line): build_versions.add(line) # Get Python test versions test_versions = set(SPARK_VERSIONS.keys()) # Compare versions if build_versions != test_versions: missing_in_test = build_versions - test_versions extra_in_test = test_versions - build_versions print("\n" + "="*70) print("ERROR: Spark version mismatch between test and build") print("="*70) if missing_in_test: print(f"\n✗ Build defines these versions, missing in test:") for v in sorted(missing_in_test): print(f" {v}") if extra_in_test: print(f"\n✗ Test defines these versions, missing in build:") for v in sorted(extra_in_test): print(f" {v}") print("\nPlease update SPARK_VERSIONS in this test to match build configuration.") print("="*70 + "\n") sys.exit(1) # Success - silent validation print(f"✓ Spark versions: {', '.join(sorted(build_versions))}\n") except subprocess.CalledProcessError as e: print(f"Warning: Could not validate Spark versions: {e}\n") class SparkVersionsScriptTest: """Tests for the get_spark_version_info.py script.""" def __init__(self, delta_root: Path): self.delta_root = delta_root self.json_path = delta_root / "target" / "spark-versions.json" self.script_path = delta_root / "project" / "scripts" / "get_spark_version_info.py" def ensure_json_exists(self) -> bool: """Ensure the JSON file exists by running exportSparkVersionsJson.""" if not self.json_path.exists(): print(" Generating spark-versions.json...") try: subprocess.run( ["build/sbt", "exportSparkVersionsJson"], cwd=self.delta_root, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT ) except subprocess.CalledProcessError: print(" ✗ Failed to generate spark-versions.json") return False return True def test_json_format(self) -> bool: """Test that the JSON file is well-formed with expected fields.""" if not self.ensure_json_exists(): return False try: with open(self.json_path, 'r') as f: data = json.load(f) # Validate it's an array if not isinstance(data, list) or len(data) == 0: print(" ✗ JSON must be a non-empty array") return False # Validate each entry has required fields required_fields = ["fullVersion", "shortVersion", "isMaster", "isDefault", "targetJvm", "packageSuffix"] for idx, entry in enumerate(data): for field in required_fields: if field not in entry: print(f" ✗ Entry {idx} missing required field: {field}") return False # Validate field types if not isinstance(entry["fullVersion"], str) or not isinstance(entry["shortVersion"], str) or \ not isinstance(entry["isMaster"], bool) or not isinstance(entry["isDefault"], bool) or \ not isinstance(entry["targetJvm"], str) or not isinstance(entry["packageSuffix"], str): print(f" ✗ Entry {idx}: Invalid field types") return False versions_str = ", ".join([entry.get("isMaster") and "master" or entry["shortVersion"] for entry in data]) print(f" ✓ JSON format valid: {len(data)} version(s) [{versions_str}]") return True except json.JSONDecodeError as e: print(f" ✗ Invalid JSON: {e}") return False except Exception as e: print(f" ✗ Unexpected error: {e}") return False def test_all_spark_versions(self) -> bool: """Test that --all-spark-versions produces valid JSON array.""" if not self.ensure_json_exists(): return False try: result = subprocess.run( ["python3", str(self.script_path), "--all-spark-versions"], cwd=self.delta_root, capture_output=True, text=True, check=True ) matrix_versions = json.loads(result.stdout.strip()) # Validate it's a non-empty array of strings if not isinstance(matrix_versions, list) or len(matrix_versions) == 0: print(" ✗ Must output a non-empty JSON array") return False if not all(isinstance(v, str) for v in matrix_versions): print(" ✗ All matrix entries must be strings") return False # Validate consistency with JSON with open(self.json_path, 'r') as f: data = json.load(f) if len(matrix_versions) != len(data): print(f" ✗ Matrix has {len(matrix_versions)} versions, JSON has {len(data)}") return False print(f" ✓ --all-spark-versions: {matrix_versions}") return True except (subprocess.CalledProcessError, json.JSONDecodeError) as e: print(f" ✗ Failed: {e}") return False def test_released_spark_versions(self) -> bool: """Test that --released-spark-versions excludes snapshots.""" if not self.ensure_json_exists(): return False try: result = subprocess.run( ["python3", str(self.script_path), "--released-spark-versions"], cwd=self.delta_root, capture_output=True, text=True, check=True ) released_versions = json.loads(result.stdout.strip()) # Validate it's an array of strings if not isinstance(released_versions, list): print(" ✗ Must output a JSON array") return False if not all(isinstance(v, str) for v in released_versions): print(" ✗ All entries must be strings") return False # Load JSON and verify snapshots are excluded with open(self.json_path, 'r') as f: data = json.load(f) expected_count = sum(1 for entry in data if "-SNAPSHOT" not in entry["fullVersion"]) if len(released_versions) != expected_count: print(f" ✗ Expected {expected_count} released versions, got {len(released_versions)}") return False # Verify no snapshot versions included for version in released_versions: if "SNAPSHOT" in version.upper(): print(f" ✗ Released versions should not include snapshots: {version}") return False print(f" ✓ --released-spark-versions: {released_versions} (snapshots excluded)") return True except (subprocess.CalledProcessError, json.JSONDecodeError) as e: print(f" ✗ Failed: {e}") return False def test_get_field(self) -> bool: """Test that --get-field works for various version formats.""" if not self.ensure_json_exists(): return False try: # Load the JSON to know what versions to test with open(self.json_path, 'r') as f: data = json.load(f) test_cases = [] for entry in data: # Test short version and full version test_cases.append((entry["shortVersion"], "targetJvm", entry["targetJvm"])) test_cases.append((entry["fullVersion"], "fullVersion", entry["fullVersion"])) # Test "master" if applicable if entry["isMaster"]: test_cases.append(("master", "targetJvm", entry["targetJvm"])) all_passed = True for version, field, expected in test_cases: result = subprocess.run( ["python3", str(self.script_path), "--get-field", version, field], cwd=self.delta_root, capture_output=True, text=True, check=True ) actual = json.loads(result.stdout.strip()) if actual != expected: print(f" ✗ --get-field {version} {field}: expected {expected}, got {actual}") all_passed = False if all_passed: print(f" ✓ --get-field: Tested {len(test_cases)} cases successfully") return all_passed except (subprocess.CalledProcessError, json.JSONDecodeError) as e: print(f" ✗ Failed: {e}") return False def main(): """Main entry point.""" try: delta_root = Path(__file__).parent.parent.parent if not (delta_root / "build.sbt").exists(): print("Error: build.sbt not found. Run from Delta repository root.") sys.exit(1) print("="*70) print("Cross-Spark Build Test Suite") print("="*70) print() # Test the get_spark_version_info.py script first print("\n" + "="*70) print("PART 1: Spark Versions Script Tests") print("="*70) script_test = SparkVersionsScriptTest(delta_root) script_test1_passed = script_test.test_json_format() script_test2_passed = script_test.test_all_spark_versions() script_test3_passed = script_test.test_released_spark_versions() script_test4_passed = script_test.test_get_field() # Test cross-Spark build workflow print("\n" + "="*70) print("PART 2: Cross-Spark Build Tests") print("="*70) build_test = CrossSparkPublishTest(delta_root) build_test.validate_spark_versions() # Run all build tests build_test1_passed = build_test.test_default_publish() build_test2_passed = build_test.test_backward_compat_publish() build_test3_passed = build_test.test_cross_spark_workflow() # Summary print("\n" + "="*70) print("TEST SUMMARY") print("="*70) print("\nPart 1: Spark Versions Script Tests") print(f" JSON Format: {'✓ PASSED' if script_test1_passed else '✗ FAILED'}") print(f" All Spark Versions Output: {'✓ PASSED' if script_test2_passed else '✗ FAILED'}") print(f" Released Spark Versions Output: {'✓ PASSED' if script_test3_passed else '✗ FAILED'}") print(f" Get Field Functionality: {'✓ PASSED' if script_test4_passed else '✗ FAILED'}") print("\nPart 2: Cross-Spark Build Tests") print(f" Default publishM2 (with suffix): {'✓ PASSED' if build_test1_passed else '✗ FAILED'}") print(f" skipSparkSuffix (backward compat): {'✓ PASSED' if build_test2_passed else '✗ FAILED'}") print(f" Cross-Spark Workflow (both): {'✓ PASSED' if build_test3_passed else '✗ FAILED'}") print("="*70) all_tests_passed = ( script_test1_passed and script_test2_passed and script_test3_passed and script_test4_passed and build_test1_passed and build_test2_passed and build_test3_passed ) if all_tests_passed: print("\n✓ ALL TESTS PASSED") sys.exit(0) else: print("\n✗ SOME TESTS FAILED") sys.exit(1) except Exception as e: print(f"\n✗ TEST EXECUTION FAILED WITH ERROR: {e}") import traceback traceback.print_exc() sys.exit(1) if __name__ == "__main__": main() ================================================ FILE: protocol_rfcs/README.md ================================================ # Protocol RFCs This directory contains information about the process of making Delta protocol changes via RFCs and all the RFCs that have been proposed since this process was adopted. - [Table of RFCs](#table-of-rfcs) - [Proposed RFCs](#proposed-rfcs) - [Accepted RFCs](#accepted-rfcs) - [Rejected RFCs](#rejected-rfcs) - [RFC Process](#rfc-process) ## Table of RFCs Here is the history of all the RFCs propose/accepted/rejected since Feb 6, 2024, when this process was introduced. ### Proposed RFCs | Date proposed | RFC file | Github issue | RFC title | |:--------------|:---------------------------------------------------------------------------------------------------------------------------------|:----------------------------------------------|:---------------------------------------| | 2023-02-26 | [column-mapping-usage.tracking.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/column-mapping-usage-tracking.md) | https://github.com/delta-io/delta/issues/2682 | Column Mapping Usage Tracking | | 2023-04-24 | [variant-type.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/variant-type.md) | https://github.com/delta-io/delta/issues/2864 | Variant Data Type | | 2024-04-30 | [collated-string-type.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/collated-string-type.md) | https://github.com/delta-io/delta/issues/2894 | Collated String Type | | 2025-03-13 | [checkpoint-protection.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/checkpoint-protection.md) | https://github.com/delta-io/delta/issues/4152 | Checkpoint Protection | | 2025-03-18 | [iceberg-writer-compat-v1.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/iceberg-writer-compat-v1.md) | https://github.com/delta-io/delta/issues/4284 | IcebergWriterCompatV1 | | 2025-05-06 | [variant-shredding.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/variant-shredding.md) | https://github.com/delta-io/delta/issues/4032 | Variant Shredding | | 2025-11-20 | [materialize-partition-columns.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/materialize-partition-columns.md) | https://github.com/delta-io/delta/issues/5555 | Materialize Partition Columns | ### Accepted RFCs | Date proposed | Date accepted | RFC file | Github issue | RFC title | |:-|:-|:-|:-|:-| | 2025-04-07 | 2026-02-17 |[catalog-managed.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/accepted/catalog-managed.md) | https://github.com/delta-io/delta/issues/4381 | Catalog-Managed Tables | | 2023-02-28 | 2023-03-26 |[vacuum-protocol-check.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/accepted/vacuum-protocol-check.md)| https://github.com/delta-io/delta/issues/2630 | Enforce Vacuum Protocol Check | | 2023-02-02 | 2023-07-24 |[in-commit-timestamps.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/accepted/in-commit-timestamps.md) | https://github.com/delta-io/delta/issues/2532 | In-Commit Timestamps | | 2023-02-09 | 2025-01-28 |[type-widening.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/accepted/type-widening.md) | https://github.com/delta-io/delta/issues/2623 | Type Widening | ### Rejected RFCs | Date proposed | Date rejected | RFC file | Github issue | RFC title | |:--------------|:--------------|:--------------------------------------------------------------------------------------------------------------|:----------------------------------------------|:-----------------| | 2023-02-14 | 2025-04-07 | [managed-commits.md](https://github.com/delta-io/delta/blob/master/protocol_rfcs/rejected/managed-commits.md) | https://github.com/delta-io/delta/issues/2598 | Managed Commits | ## RFC process ### **1. Make initial proposal** Create a Github issue of type [Protocol Change Request]. - The description of the issue may have links to design docs, etc. - This issue will serve as the central location for all discussions related to the protocol change. - If the proposal comes with a prototype or other pathfinding, the changes should be in an open PR. ### **2. Add the RFC doc** After creating the issue and discussing with the community, if a basic consensus is reached that this feature should be implemented, then create a PR to add the protocol RFC before merging code in master. - Clone the RFC template `template.md` and create a new RFC markdown doc. - Cross-link with the issue with "see #xxx". DONT USE "closes #xxx" or "fixes #xxx" or "resolves #xxx" because we don't want the issue to be closed when this RFC PR is merged. Note: - For table features, it is strongly recommended that any experimental support for the feature uses a temporary feature name with a suffix like `-dev`. This will communicate to the users that are about to use experimental feature with no future compatibility guarantee. - Code related to a proposed feature should not be merged into the main branch until the RFC attains "proposed" status (that is, the RFC PR has been through public review and merged). Until the RFC has been accepted (that is, the proposed changes have been merged into the Delta specification), any code changes should be isolated from production code behind feature flags, etc. so that existing users are not affected in any way. ### **3. Finally, accept or reject the RFC** For a RFC to be accepted, it must satisfy the following criteria: - There is a production implementation (for example, in delta-spark) of the feature that has been thoroughly well tested. - There is at least some discussion and/or prototype (preferred) that ensure the feasibility of the feature in Delta Kernel. When the success criteria are met, then the protocol can be finalized by making a PR to make the following changes: - Closely validate that the protocol spec changes are actually consistent with the production implementation. - Cross-link the PR with the original issue with "closes #xxx" as now we are ready to close the issue. In addition, update the title of the issue to say `[ACCEPTED]` to make it obvious how the proposal was resolved. - Update `protocol.md`. - Move the RFC doc to the `accepted` subdirectory, and update the state in index.md. - Remove the temporary/preview suffix like `-dev` in the table feature name from all the code. However, if the RFC is to be rejected, then make a PR to do the following changes: - Cross-link the PR with the original issue with "closes #xxx" as now we are ready to close the issue. In addition, update the title of the issue to say `[REJECTED]` to make it obvious how the proposal was resolved. - Move the RFC doc to the `rejected` subdirectory. - Update the state in `index.md`. - Remove any experimental/preview code related to the feature. ================================================ FILE: protocol_rfcs/accepted/catalog-managed.md ================================================ # Catalog-Managed Tables **Associated Github issue for discussions: https://github.com/delta-io/delta/issues/4381** This RFC proposes a new reader-writer table feature `catalogManaged` which changes the way Delta Lake discovers and accesses tables. Today’s Delta protocol relies entirely on the filesystem for read-time discovery as well as write-time commit atomicity. This feature request is to allow catalog-managed Delta tables whose discovery and commits go through the table's managing catalog instead of going directly to the filesystem (s3, abfs, etc). In particular, the catalog becomes the source of truth about whether a given commit attempt succeeded or not, instead of relying exclusively on filesystem PUT-if-absent primitives. Making the catalog the source of truth for commits to a table brings several important advantages: 1. Allows the catalog to broker all commits to the tables it manages, and to reject filesystem-based commits that would bypass the catalog. Otherwise, the catalog cannot reliably stay in sync with the table state, nor can it reject invalid commits, because it doesn’t even know about writes until they are already durable and visible to readers. For instance, a catalog might want to block low-privilege writers from modifying table metadata (e.g. schema, table features, or table properties) while still allowing normal reads and writes. Similarly, if a column is referenced by a foreign key, the catalog might want to prevent dropping its NOT NULL constraint. 2. Opens a clear path to transactions that could span multiple tables and/or involve non-table catalog updates. Otherwise, the catalog cannot participate in commit at all, because filesystem-based commits (i.e. using PUT-if-absent) do not admit any way to coordinate with other entities. 3. Allows the catalog to facilitate efficient writes of the table, e.g. by directly hosting the content of small commits instead of forcing clients to write them to cloud storage first. Otherwise, the catalog is not a source of truth, and at best it can only mirror stale copies of table state. 4. Allows the catalog to facilitate efficient reads of the table. Examples include vending storage credentials, as well as serving up the content of small commits and/or table state such as [version checksum file](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#version-checksum-file), so that clients do not have to read those files from cloud storage. 5. Allows the catalog to be the authoritative source of the latest table version, no longer requiring Delta clients to LIST the `_delta_log` to discover it. This saves time and can also allow implementations of Delta on file systems where LIST is not ordered, such as S3 Express One Zone. 6. Allows the catalog to trigger followup actions based on a commit, such as VACUUMing, data layout optimizations, automatic UniForm conversions, or triggering arbitrary listeners such as downstream ETL or streaming pipelines. -------- # Changes to existing sections ### Delta Log Entries > ***Change to [existing section](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#delta-log-entries)*** Delta Log Entries, also known as Delta files, are JSON files stored in the `_delta_log` directory at the root of the table. Together with checkpoints, they make up the log of all changes that have occurred to a table. Delta files are the unit of atomicity for a table, and are named using the next available version number, zero-padded to 20 digits. For example: ``` ./_delta_log/00000000000000000000.json ``` Delta files use newline-delimited JSON format, where every action is stored as a single-line JSON document. A Delta file, corresponding to version `v`, contains an atomic set of [_actions_](#Actions) that should be applied to the previous table state corresponding to version `v-1`, in order to construct the `v`th snapshot of the table. An action changes one aspect of the table's state, for example, adding or removing a file. **Note:** If the [`catalogManaged` table feature](#catalog-managed-tables) is enabled on the table, recently [ratified commits](#ratified-commit) may not yet be published to the `_delta_log` directory as normal Delta files - they may be stored directly by the catalog or reside in the `_delta_log/_staged_commits` directory. Delta clients must contact the table's managing catalog in order to find the information about these [ratified, potentially-unpublished commits](#publishing-commits). The `_delta_log/_staged_commits` directory is the staging area for [staged](#staged-commit) commits. Delta files in this directory have a UUID embedded into them and follow the pattern `..json`, where the version corresponds to the proposed commit version, zero-padded to 20 digits. For example: ``` ./_delta_log/_staged_commits/00000000000000000000.3a0d65cd-4056-49b8-937b-95f9e3ee90e5.json ./_delta_log/_staged_commits/00000000000000000001.7d17ac10-5cc3-401b-bd1a-9c82dd2ea032.json ./_delta_log/_staged_commits/00000000000000000001.016ae953-37a9-438e-8683-9a9a4a79a395.json ./_delta_log/_staged_commits/00000000000000000002.3ae45b72-24e1-865a-a211-34987ae02f2a.json ``` NOTE: The (proposed) version number of a staged commit is authoritative - file `00000000000000000100..json` always corresponds to a commit attempt for version 100. Besides simplifying implementations, it also acknowledges the fact that commit files cannot safely be reused for multiple commit attempts. For example, resolving conflicts in a table with [row tracking](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#row-tracking) enabled requires rewriting all file actions to update their `baseRowId` field. The [catalog](#terminology-catalogs) is the source of truth about which staged commit files in the `_delta_log/_staged_commits` directory correspond to ratified versions, and Delta clients should not attempt to directly interpret the contents of that directory. Refer to [catalog-managed tables](#catalog-managed-tables) for more details. ~~Delta files use new-line delimited JSON format, where every action is stored as a single line JSON document. A delta file, `n.json`, contains an atomic set of [_actions_](#actions) that should be applied to the previous table state, `n-1.json`, in order to construct the `n`th snapshot of the table. An action changes one aspect of the table's state, for example, adding or removing a file.~~ ### Commit Provenance Information > ***Change to [existing section](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#commit-provenance-information)*** When the `catalogManaged` table feature is enabled, the `commitInfo` action must have a field `txnId` that stores a unique transaction identifier string. ### Metadata Cleanup > ***Change to [existing section](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#metadata-cleanup)*** 2. Identify the newest checkpoint that is not newer than the `cutOffCommit`. A checkpoint at the `cutOffCommit` is ideal, but an older one will do. Let's call it `cutOffCheckpoint`. We need to preserve the `cutOffCheckpoint` and all published commits after it, because we need them to enable time travel for commits between `cutOffCheckpoint` and the next available checkpoint. - If no `cutOffCheckpoint` can be found, do not proceed with metadata cleanup as there is nothing to cleanup. 3. Delete all [delta log entries](#delta-log-entries), [checkpoint files](#checkpoints), and [version checksum files](#version-checksum-file) before the `cutOffCheckpoint` checkpoint. Also delete all the [log compaction files](#log-compaction-files) having startVersion <= `cutOffCheckpoint`'s version. - Also delete all the [staged commit files](#staged-commit) having version <= `cutOffCheckpoint`'s version from the `_delta_log/_staged_commits` directory. -------- > ***The next set of sections will be added to the existing spec just before [Iceberg Compatibility V1](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#iceberg-compatibility-v1) section*** # Catalog-managed tables With this feature enabled, the [catalog](#terminology-catalogs) that manages the table becomes the source of truth for whether a given commit attempt succeeded. The table feature defines the parts of the [commit protocol](#commit-protocol) that directly impact the Delta table (e.g. atomicity requirements, publishing, etc). The Delta client and catalog together are responsible for implementing the Delta-specific aspects of commit as defined by this spec, but are otherwise free to define their own APIs and protocols for communication with each other. **NOTE**: Filesystem-based access to catalog-managed tables is not supported. Delta clients are expected to discover and access catalog-managed tables through the managing catalog, not by direct listing in the filesystem. This feature is primarily designed to warn filesystem-based readers that might attempt to access a catalog-managed table's storage location without going through the catalog first, and to block filesystem-based writers who could otherwise corrupt both the table and the catalog by failing to commit through the catalog. Before we can go into details of this protocol feature, we must first align our terminology. ## Terminology: Commits A commit is a set of [actions](#actions) that transform a Delta table from version `v - 1` to `v`. It contains the same kind of content as is stored in a [Delta file](#delta-log-entries). A commit may be stored in the file system as a Delta file - either _published_ or _staged_ - or stored _inline_ in the managing catalog, using whatever format the catalog prefers. There are several types of commits: 1. **Proposed commit**: A commit that a Delta client has proposed for the next version of the table. It could be _staged_ or _inline_. It will either become _ratified_ or be rejected. 2. **Staged commit**: A commit that is written to disk at `_delta_log/_staged_commits/..json`. It has the same content and format as a published Delta file. - Here, the `uuid` is a random UUID that is generated for each commit and `v` is the version which is proposed to be committed, zero-padded to 20 digits. - The mere existence of a staged commit does not mean that the file has been ratified or even proposed. It might correspond to a failed or in-progress commit attempt. - The catalog is the source of truth around which staged commits are ratified. - The catalog stores only the location, not the content, of a staged (and ratified) commit. 3. **Inline commit**: A proposed commit that is not written to disk but rather has its content sent to the catalog for the catalog to store directly. 4. **Ratified commit**: A proposed commit that a catalog has determined has won the commit at the desired version of the table. - The catalog must store ratified commits (that is, the staged commit's location or the inline commit's content) until they are published to the `_delta_log` directory. - A ratified commit may or may not yet be published. - A ratified commit may or may not even be stored by the catalog at all - the catalog may have just atomically published it to the filesystem directly, relying on PUT-if-absent primitives to facilitate the ratification and publication all in one step. 5. **Published commit**: A ratified commit that has been copied into the `_delta_log` as a normal Delta file, i.e. `_delta_log/.json`. - Here, the `v` is the version which is being committed, zero-padded to 20 digits. - The existence of a `.json` file proves that the corresponding version `v` is ratified, regardless of whether the table is catalog-managed or filesystem-based. The catalog is allowed to return information about published commits, but Delta clients can also use filesystem listing operations to directly discover them. - Published commits do not need to be stored by the catalog. ## Terminology: Delta Client This is the component that implements support for reading and writing Delta tables, and implements the logic required by the `catalogManaged` table feature. Among other things, it - triggers the filesystem listing, if needed, to discover published commits - generates the commit content (the set of [actions](#actions)) - works together with the query engine to trigger the commit process and invoke the client-side catalog component with the commit content The Delta client is also responsible for defining the client-side API that catalogs should target. That is, there must be _some_ API that the [catalog client](#catalog-client) can use to communicate to the Delta client the subset of catalog-managed information that the Delta client cares about. This protocol feature is concerned with what information Delta cares about, but leaves to Delta clients the design of the API they use to obtain that information from catalog clients. ## Terminology: Catalogs 1. **Catalog**: A catalog is an entity which manages a Delta table, including its creation, writes, reads, and eventual deletion. - It could be backed by a database, a filesystem, or any other persistence mechanism. - Each catalog has its own spec around how catalog clients should interact with them, and how they perform a commit. 2. **Catalog Client**: The catalog always has a client-side component which the Delta client interacts with directly. This client-side component has two primary responsibilities: - implement any client-side catalog-specific logic (such as staging or [publishing](#publishing-commits) commits) - communicate with the Catalog Server, if any 3. **Catalog Server**: The catalog may also involve a server-side component which the client-side component would be responsible to communicate with. - This server is responsible for coordinating commits and potentially persisting table metadata and enforcing authorization policies. - Not all catalogs require a server; some may be entirely client-side, e.g. filesystem-backed catalogs, or they may make use of a generic database server and implement all of the catalog's business logic client-side. **NOTE**: This specification outlines the responsibilities and actions that catalogs must implement. This spec does its best not to assume any specific catalog _implementation_, though it does call out likely client-side and server-side responsibilities. Nonetheless, what a given catalog does client-side or server-side is up to each catalog implementation to decide for itself. ## Catalog Responsibilities When the `catalogManaged` table feature is enabled, a catalog performs commits to the table on behalf of the Delta client. As stated above, the Delta spec does not mandate any particular client-server design or API for catalogs that manage Delta tables. However, the catalog does need to provide certain capabilities for reading and writing Delta tables: - Atomically commit a version `v` with a given set of `actions`. This is explained in detail in the [commit protocol](#commit-protocol) section. - Retrieve information about recent ratified commits and the latest ratified version on the table. This is explained in detail in the [Getting Ratified Commits from the Catalog](#getting-ratified-commits-from-the-catalog) section. - Though not required, it is encouraged that catalogs also return the latest table-level metadata, such as the latest Protocol and Metadata actions, for the table. This can provide significant performance advantages to conforming Delta clients, who may forgo log replay and instead trust the information provided by the catalog during query planning. ## Reading Catalog-managed Tables A catalog-managed table can have a mix of (a) published and (b) ratified but non-published commits. The catalog is the source of truth for ratified commits. Also recall that ratified commits can be [staged commits](#staged-commit) that are persisted to the `_delta_log/_staged_commits` directory, or [inline commits](#inline-commit) whose content the catalog stores directly. For example, suppose the `_delta_log` directory contains the following files: ``` 00000000000000000000.json 00000000000000000001.json 00000000000000000002.checkpoint.parquet 00000000000000000002.json 00000000000000000003.00000000000000000005.compacted.json 00000000000000000003.json 00000000000000000004.json 00000000000000000005.json 00000000000000000006.json 00000000000000000007.json _staged_commits/00000000000000000007.016ae953-37a9-438e-8683-9a9a4a79a395.json // ratified and published _staged_commits/00000000000000000008.7d17ac10-5cc3-401b-bd1a-9c82dd2ea032.json // ratified _staged_commits/00000000000000000008.b91807ba-fe18-488c-a15e-c4807dbd2174.json // rejected _staged_commits/00000000000000000010.0f707846-cd18-4e01-b40e-84ee0ae987b0.json // not yet ratified _staged_commits/00000000000000000010.7a980438-cb67-4b89-82d2-86f73239b6d6.json // partial file ``` Further, suppose the catalog stores the following ratified commits: ``` { 7 -> "00000000000000000007.016ae953-37a9-438e-8683-9a9a4a79a395.json", 8 -> "00000000000000000008.7d17ac10-5cc3-401b-bd1a-9c82dd2ea032.json", 9 -> } ``` Some things to note are: - the catalog isn't aware that commit 7 was already published - perhaps the response from the filesystem was dropped - commit 9 is an inline commit - neither of the two staged commits for version 10 have been ratified To read such tables, Delta clients must first contact the catalog to get the ratified commits. This informs the Delta client of commits [7, 9] as well as the latest ratified version, 9. If this information is insufficient to construct a complete snapshot of the table, Delta clients must LIST the `_delta_log` directory to get information about the published commits. For commits that are both returned by the catalog and already published, Delta clients must treat the catalog's version as authoritative and read the commit returned by the catalog. Additionally, Delta clients must ignore any files with versions greater than the latest ratified commit version returned by the catalog. Combining these two sets of files and commits enables Delta clients to generate a snapshot at the latest version of the table. **NOTE**: This spec prescribes the _minimum_ required interactions between Delta clients and catalogs for commits. Catalogs may very well expose APIs and work with Delta clients to be informed of other non-commit [file types](#file-types), such as checkpoint, log compaction, and version checksum files. This would allow catalogs to return additional information to Delta clients during query and scan planning, potentially allowing Delta clients to avoid LISTing the filesystem altogether. ## Commit Protocol To start, Delta Clients send the desired actions to be committed to the client-side component of the catalog. This component then has several options for proposing, ratifying, and publishing the commit, detailed below. - Option 1: Write the actions (likely client-side) to a [staged commit file](#staged-commit) in the `_delta_log/_staged_commits` directory and then ratify the staged commit (likely server-side) by atomically recording (in persistent storage of some kind) that the file corresponds to version `v`. - Option 2: Treat this as an [inline commit](#inline-commit) (i.e. likely that the client-side component sends the contents to the server-side component) and atomically record (in persistent storage of some kind) the content of the commit as version `v` of the table. - Option 3: Catalog implementations that use PUT-if-absent (client- or server-side) can ratify and publish all-in-one by atomically writing a [published commit file](#published-commit) in the `_delta_log` directory. Note that this commit will be considered to have succeeded as soon as the file becomes visible in the filesystem, regardless of when or whether the catalog is made aware of the successful publish. The catalog does not need to store these files. A catalog must not ratify version `v` until it has ratified version `v - 1`, and it must ratify version `v` at most once. The catalog must store both flavors of ratified commits (staged or inline) and make them available to readers until they are [published](#publishing-commits). For performance reasons, Delta clients are encouraged to establish an API contract where the catalog provides the latest ratified commit information whenever a commit fails due to version conflict. ## Getting Ratified Commits from the Catalog Even after a commit is ratified, it is not discoverable through filesystem operations until it is [published](#publishing-commits). The catalog-client is responsible to implement an API (defined by the Delta client) that Delta clients can use to retrieve the latest ratified commit version (authoritative), as well as the set of ratified commits the catalog is still storing for the table. If some commits needed to complete the snapshot are not stored by the catalog, as they are already published, Delta clients can issue a filesystem LIST operation to retrieve them. Delta clients must establish an API contract where the catalog provides ratified commit information as part of the standard table resolution process performed at query planning time. ## Publishing Commits Publishing is the process of copying the ratified commit with version `` to `_delta_log/.json`. The ratified commit may be a staged commit located in `_delta_log/_staged_commits/..json`, or it may be an inline commit whose content the catalog stores itself. Because the content of a ratified commit is immutable, it does not matter whether the client-side, server-side, or both catalog components initiate publishing. Implementations are strongly encouraged to publish commits promptly. This reduces the number of commits the catalog needs to store internally (and serve up to readers). Commits must be published _in order_. That is, version `v - 1` must be published _before_ version `v`. **NOTE**: Because commit publishing can happen at any time after the commit succeeds, the file modification timestamp of the published file will not accurately reflect the original commit time. For this reason, catalog-managed tables must use [in-commit-timestamps](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#in-commit-timestamps) to ensure stability of time travel reads. Refer to [Writer Requirements for Catalog-managed Tables](#writer-requirements-for-catalog-managed-tables) section for more details. ## Maintenance Operations on Catalog-managed Tables [Checkpoints](#checkpoints-1) and [Log Compaction Files](#log-compaction-files) can only be created for versions that are already published in the `_delta_log`. In other words, in order to checkpoint version `v` or produce a log compaction file for commit range `x <= v <= y`, `_delta_log/.json` must exist. Notably, the [Version Checksum File](#version-checksum-file) for version `v` _can_ be created in the `_delta_log` even if the commit for version `v` is not published. By default, maintenance operations are prohibited unless the managing catalog explicitly permits the client to run them. The only exceptions are checkpoints, log compaction, and version checksum, as they are essential for all basic table operations (e.g. reads and writes) to operate reliably. All other maintenance operations such as the following are not allowed by default. - [Log and other metadata files clean up](#metadata-cleanup). - Data files cleanup, for example VACUUM. - Data layout changes, for example OPTIMIZE and REORG. ## Creating and Dropping Catalog-managed Tables The catalog and query engine ultimately dictate how to create and drop catalog-managed tables. As one example, table creation often works in three phases: 1. An initial catalog operation to obtain a unique storage location which serves as an unnamed "staging" table 2. A table operation that physically initializes a new `catalogManaged`-enabled table at the staging location. 3. A final catalog operation that registers the new table with its intended name. Delta clients would primarily be involved with the second step, but an implementation could choose to combine the second and third steps so that a single catalog call registers the table as part of the table's first commit. As another example, dropping a table can be as simple as removing its name from the catalog (a "soft delete"), followed at some later point by a "hard delete" that physically purges the data. The Delta client would not be involved at all in this process, because no commits are made to the table. ## Catalog-managed Table Enablement The `catalogManaged` table feature is supported and active when: - The table is on Reader Version 3 and Writer Version 7. - The table has a `protocol` action with `readerFeatures` and `writerFeatures` both containing the feature `catalogManaged`. ## Writer Requirements for Catalog-managed tables When supported and active: - Writers must discover and access the table using catalog calls, which happens _before_ the table's protocol is known. See [Table Discovery](#table-discovery) for more details. - The [in-commit-timestamps](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#in-commit-timestamps) table feature must be supported and active. - The `commitInfo` action must also contain a field `txnId` that stores a unique transaction identifier string - Writers must follow the catalog's [commit protocol](#commit-protocol) and must not perform ordinary filesystem-based commits against the table. - Writers must follow the catalog's [maintenance operation protocol](#maintenance-operations-on-catalog-managed-tables) ## Reader Requirements for Catalog-managed tables When supported and active: - Readers must discover the table using catalog calls, which happens before the table's protocol is known. See [Table Discovery](#table-discovery) for more details. - Readers must contact the catalog for information about unpublished ratified commits. - Readers must follow the rules described in the [Reading Catalog-managed Tables](#reading-catalog-managed-tables) section above. Notably - If the catalog said `v` is the latest version, clients must ignore any later versions that may have been published - When the catalog returns a ratified commit for version `v`, readers must use that catalog-supplied commit and ignore any published Delta file for version `v` that might also be present. ## Table Discovery The requirements above state that readers and writers must discover and access the table using catalog calls, which occurs _before_ the table's protocol is known. This raises an important question: how can a client discover a `catalogManaged` Delta table without first knowing that it _is_, in fact, `catalogManaged` (according to the protocol)? To solve this, first note that, in practice, catalog-integrated engines already ask the catalog to resolve a table name to its storage location during the name resolution step. This protocol therefore encourages that the same name resolution step also indicate whether the table is catalog-managed. Surfacing this at the very moment the catalog returns the path imposes no extra round-trips, yet it lets the client decide — early and unambiguously — whether to follow the `catalogManaged` read and write rules. ## Sample Catalog Client API The following is an example of a possible API which a Java-based Delta client might require catalog implementations to target: ```scala interface CatalogManagedTable { /** * Commits the given set of `actions` to the given commit `version`. * * @param version The version we want to commit. * @param actions Actions that need to be committed. * * @return CommitResponse which has details around the new committed delta file. */ def commit( version: Long, actions: Iterator[String]): CommitResponse /** * Retrieves a (possibly empty) suffix of ratified commits in the range [startVersion, * endVersion] for this table. * * Some of these ratified commits may already have been published. Some of them may be staged, * in which case the staged commit file path is returned; others may be inline, in which case * the inline commit content is returned. * * The returned commits are sorted in ascending version number and are contiguous. * * If neither start nor end version is specified, the catalog will return all available ratified * commits (possibly empty, if all commits have been published). * * In all cases, the response also includes the table's latest ratified commit version. * * @return GetCommitsResponse which contains an ordered list of ratified commits * stored by the catalog, as well as table's latest commit version. */ def getRatifiedCommits( startVersion: Option[Long], endVersion: Option[Long]): GetCommitsResponse } ``` Note that the above is only one example of a possible Catalog Client API. It is also _NOT_ a catalog API (no table discovery, ACL, create/drop, etc). The Delta protocol is agnostic to API details, and the API surface Delta clients define should only cover the specific catalog capabilities that Delta client needs to correctly read and write catalog-managed tables. ================================================ FILE: protocol_rfcs/accepted/in-commit-timestamps.md ================================================ # In-Commit Timestamps This RFC proposes a new Writer table feature called In-Commit Timestamps. When enabled, commit metadata includes a monotonically increasing timestamp that allows for reliable TIMESTAMP AS OF time travel even if filesystem operations change a commit file's modification timestamp. **For further discussions about this protocol change, please refer to the Github issue - https://github.com/delta-io/delta/issues/2532** -------- ### Commit Provenance Information > ***Change to existing section*** A delta file can optionally contain additional provenance information about what higher-level operation was being performed as well as who executed it. Implementations are free to store any valid JSON [object literal](https://www.w3schools.com/js/js_json_objects.asp) as the `commitInfo` action unless some table feature (e.g. [In-Commit Timestamps](#in-commit-timestamps)) imposes additional requirements on the data. When In-Commit Timestamp are enabled, writers are required to include a commitInfo action with every commit, which must include the `inCommitTimestamp` field. #### Reader Requirements for AddCDCFile > ***Change to existing section*** ... 3. Change data readers should return the following extra columns: Field Name | Data Type | Description -|-|- _commit_version|`Long`| The table version containing the change. This can be derived from the name of the Delta log file that contains actions. _commit_timestamp|`Timestamp`| The timestamp associated when the commit was created. ~~This can be derived from the file modification time of the Delta log file that contains actions.~~ Depending on whether [In-Commit Timestamps](#in-commit-timestamps) are enabled, this is derived from either the `inCommitTimestamp` field of the `commitInfo` action of the version's Delta log, or from the Delta log's file modification time. # In-Commit Timestamps > ***New Section after the [Clustered Table](#clustered-table) section*** The In-Commit Timestamps writer feature strongly associates a monotonically increasing timestamp with each commit by storing it in the commit's metadata. Enablement: - The table must be on Writer Version 7. - The feature `inCommitTimestamps` must exist in the table `protocol`'s `writerFeatures`. - The table property `delta.enableInCommitTimestamps` must be set to `true`. ## Writer Requirements for In-Commit Timestamps When In-Commit Timestamps is enabled, then: 1. Writers must write the `commitInfo` (see [Commit Provenance Information](#commit-provenance-information)) action in the commit. 2. The `commitInfo` action must be the first action in the commit. 3. The `commitInfo` action must include a field named `inCommitTimestamp`, of type `long` (see [Primitive Types](#primitive-types)), which represents the time (in milliseconds since the Unix epoch) when the commit is considered to have succeeded. It is the larger of two values: - The time, in milliseconds since the Unix epoch, at which the writer attempted the commit - One millisecond later than the previous commit's `inCommitTimestamp` 4. If the table has commits from a period when this feature was not enabled, provenance information around when this feature was enabled must be tracked in table properties: - The property `delta.inCommitTimestampEnablementVersion` must be used to track the version of the table when this feature was enabled. - The property `delta.inCommitTimestampEnablementTimestamp` must be the same as the `inCommitTimestamp` of the commit when this feature was enabled. 5. The `inCommitTimestamp` of the commit that enables this feature must be greater than the file modification time of the immediately preceding commit. ## Recommendations for Readers of Tables with In-Commit Timestamps For tables with In-Commit timestamps enabled, readers should use the `inCommitTimestamp` as the commit timestamp for operations like time travel and [`DESCRIBE HISTORY`](https://docs.delta.io/latest/delta-utility.html#retrieve-delta-table-history). If a table has commits from a period before In-Commit timestamps were enabled, the table properties `delta.inCommitTimestampEnablementVersion` and `delta.inCommitTimestampEnablementTimestamp` would be set and can be used to identify commits that don't have `inCommitTimestamp`. To correctly determine the commit timestamp for these tables, readers can use the following rules: 1. For commits with version >= `delta.inCommitTimestampEnablementVersion`, readers should use the `inCommitTimestamp` field of the `commitInfo` action. 2. For commits with version < `delta.inCommitTimestampEnablementVersion`, readers should use the file modification timestamp. Furthermore, when attempting timestamp-based time travel where table state must be fetched as of `timestamp X`, readers should use the following rules: 1. If `timestamp X` >= `delta.inCommitTimestampEnablementTimestamp`, only table versions >= `delta.inCommitTimestampEnablementVersion` should be considered for the query. 2. Otherwise, only table versions less than `delta.inCommitTimestampEnablementVersion` should be considered for the query. ================================================ FILE: protocol_rfcs/accepted/type-widening.md ================================================ # Type Widening **Associated Github issue for discussions: https://github.com/delta-io/delta/issues/2623** This protocol change introduces the Type Widening feature, which enables changing the type of a column or field in an existing Delta table to a wider type. -------- # Type Widening > ***New Section after the [Clustered Table](#clustered-table) section*** The Type Widening feature enables changing the type of a column or field in an existing Delta table to a wider type. The supported type changes are: - Integer widening: - `Byte` -> `Short` -> `Int` -> `Long` - Floating-point widening: - `Float` -> `Double` - `Byte`, `Short` or `Int` -> `Double` - Date widening: - `Date` -> `Timestamp without timezone` - Decimal widening - `p` and `s` denote the decimal precision and scale respectively. - `Decimal(p, s)` -> `Decimal(p + k1, s + k2)` where `k1 >= k2 >= 0`. - `Byte`, `Short` or `Int` -> `Decimal(10 + k1, k2)` where `k1 >= k2 >= 0`. - `Long` -> `Decimal(20 + k1, k2)` where `k1 >= k2 >= 0`. To support this feature: - The table must be on Reader version 3 and Writer Version 7. - The feature `typeWidening` must exist in the table `protocol`'s `readerFeatures` and `writerFeatures`, either during its creation or at a later stage. When supported: - A table may have a metadata property `delta.enableTypeWidening` in the Delta schema set to `true`. Writers must reject widening type changes when this property isn't set to `true`. - The `metadata` for a column or field in the table schema may contain the key `delta.typeChanges` storing a history of type changes for that column or field. ### Type Change Metadata Type changes applied to a table are recorded in the table schema and stored in the `metadata` of their nearest ancestor [StructField](#struct-field) using the key `delta.typeChanges`. The value for the key `delta.typeChanges` must be a JSON list of objects, where each object contains the following fields: Field Name | optional/required | Description -|-|- `fromType`| required | The type of the column or field before the type change. `toType`| required | The type of the column or field after the type change. `fieldPath`| optional | When updating the type of a map key/value or array element only: the path from the struct field holding the metadata to the map key/value or array element that was updated. The `fieldPath` value is "key", "value" and "element" when updating resp. the type of a map key, map value and array element. The `fieldPath` value for nested maps and nested arrays are prefixed by their parents's path, separated by dots. The following is an example for the definition of a column that went through two type changes: ```json { "name" : "e", "type" : "long", "nullable" : true, "metadata" : { "delta.typeChanges": [ { "fromType": "short", "toType": "integer" }, { "fromType": "integer", "toType": "long" } ] } } ``` The following is an example for the definition of a column after changing the type of a map key: ```json { "name" : "e", "type" : { "type": "map", "keyType": "double", "valueType": "integer", "valueContainsNull": true }, "nullable" : true, "metadata" : { "delta.typeChanges": [ { "fromType": "float", "toType": "double", "fieldPath": "key" } ] } } ``` The following is an example for the definition of a column after changing the type of a map value nested in an array: ```json { "name" : "e", "type" : { "type": "array", "elementType": { "type": "map", "keyType": "string", "valueType": "decimal(10, 4)", "valueContainsNull": true }, "containsNull": true }, "nullable" : true, "metadata" : { "delta.typeChanges": [ { "fromType": "decimal(6, 2)", "toType": "decimal(10, 4)", "fieldPath": "element.value" } ] } } ``` ## Writer Requirements for Type Widening When Type Widening is supported (when the `writerFeatures` field of a table's `protocol` action contains `typeWidening`), then: - Writers must reject applying any unsupported type change. - Writers must reject applying type changes not supported by [Iceberg V2](https://iceberg.apache.org/spec/#schema-evolution) when either the [Iceberg Compatibility V1](#iceberg-compatibility-v1) or [Iceberg Compatibility V2](#iceberg-compatibility-v2) table feature is supported: - `Byte`, `Short` or `Int` -> `Double` - `Date` -> `Timestamp without timezone` - Decimal scale increase - `Byte`, `Short`, `Int` or `Long` -> `Decimal` - Writers must record type change information in the `metadata` of the nearest ancestor [StructField](#struct-field). See [Type Change Metadata](#type-change-metadata). - Writers must preserve the `delta.typeChanges` field in the metadata fields in the schema when the table schema is updated. - Writers may remove the `delta.typeChanges` metadata in the table schema if all data files use the same field types as the table schema. When Type Widening is enabled (when the table property `delta.enableTypeWidening` is set to `true`), then: - Writers should allow updating the table schema to apply a supported type change to a column, struct field, map key/value or array element. When removing the Type Widening table feature from the table, in the version that removes `typeWidening` from the `writerFeatures` and `readerFeatures` fields of the table's `protocol` action: - Writers must ensure no `delta.typeChanges` metadata key is present in the table schema. This may require rewriting existing data files to ensure that all data files use the same field types as the table schema in order to fulfill the requirement to remove type widening metadata. - Writers must ensure that the table property `delta.enableTypeWidening` is not set. ## Reader Requirements for Type Widening When Type Widening is supported (when the `readerFeatures` field of a table's `protocol` action contains `typeWidening`), then: - Readers must allow reading data files written before the table underwent any supported type change, and must convert such values to the current, wider type. - Readers must validate that they support all type changes in the `delta.typeChanges` field in the table schema for the table version they are reading and fail when finding any unsupported type change. ## Writer Requirements for IcebergCompatV1 > ***Change to existing section (underlined)*** When supported and active, writers must: - Require that Column Mapping be enabled and set to either `name` or `id` mode - Require that Deletion Vectors are not supported (and, consequently, not active, either). i.e., the `deletionVectors` table feature is not present in the table `protocol`. - Require that partition column values are materialized into any Parquet data file that is present in the table, placed *after* the data columns in the parquet schema - Require that all `AddFile`s committed to the table have the `numRecords` statistic populated in their `stats` field - When the [Type Widening](#type-widening) table feature is supported, require that all type changes applied on the table are supported by [Iceberg V2](https://iceberg.apache.org/spec/#schema-evolution), based on the [Type Change Metadata](#type-change-metadata) recorded in the table schema. ## Writer Requirements for IcebergCompatV2 > ***Change to existing section (underlined)*** When this feature is supported and enabled, writers must: - Require that Column Mapping be enabled and set to either `name` or `id` mode - Require that the nested `element` field of ArrayTypes and the nested `key` and `value` fields of MapTypes be assigned 32 bit integer identifiers. These identifiers must be unique and different from those used in [Column Mapping](#column-mapping), and must be stored in the metadata of their nearest ancestor [StructField](#struct-field) of the Delta table schema. Identifiers belonging to the same `StructField` must be organized as a `Map[String, Long]` and stored in metadata with key `parquet.field.nested.ids`. The keys of the map are "element", "key", or "value", prefixed by the name of the nearest ancestor StructField, separated by dots. The values are the identifiers. The keys for fields in nested arrays or nested maps are prefixed by their parents' key, separated by dots. An [example](#example-of-storing-identifiers-for-nested-fields-in-arraytype-and-maptype) is provided below to demonstrate how the identifiers are stored. These identifiers must be also written to the `field_id` field of the `SchemaElement` struct in the [Parquet Thrift specification](https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift) when writing parquet files. - Require that IcebergCompatV1 is not active, which means either the `icebergCompatV1` table feature is not present in the table protocol or the table property `delta.enableIcebergCompatV1` is not set to `true` - Require that Deletion Vectors are not active, which means either the `deletionVectors` table feature is not present in the table protocol or the table property `delta.enableDeletionVectors` is not set to `true` - Require that partition column values be materialized when writing Parquet data files - Require that all new `AddFile`s committed to the table have the `numRecords` statistic populated in their `stats` field - Require writing timestamp columns as int64 - Require that the table schema contains only data types in the following allow-list: [`byte`, `short`, `integer`, `long`, `float`, `double`, `decimal`, `string`, `binary`, `boolean`, `timestamp`, `timestampNTZ`, `date`, `array`, `map`, `struct`]. - When the [Type Widening](#type-widening) table feature is supported, require that all type changes applied on the table are supported by [Iceberg V2](https://iceberg.apache.org/spec/#schema-evolution), based on the [Type Change Metadata](#type-change-metadata) recorded in the table schema. ### Column Metadata > ***Change to existing section (underlined)*** A column metadata stores various information about the column. For example, this MAY contain some keys like [`delta.columnMapping`](#column-mapping) or [`delta.generationExpression`](#generated-columns) or [`CURRENT_DEFAULT`](#default-columns). Field Name | Description -|- delta.columnMapping.*| These keys are used to store information about the mapping between the logical column name to the physical name. See [Column Mapping](#column-mapping) for details. delta.identity.*| These keys are for defining identity columns. See [Identity Columns](#identity-columns) for details. delta.invariants| JSON string contains SQL expression information. See [Column Invariants](#column-invariants) for details. delta.generationExpression| SQL expression string. See [Generated Columns](#generated-columns) for details. delta.typeChanges| JSON string containing information about previous type changes applied to this column. See [Type Change Metadata](#type-change-metadata) for details. ================================================ FILE: protocol_rfcs/accepted/vacuum-protocol-check.md ================================================ # Vacuum Protocol Check This RFC introduces a new ReaderWriter feature named `vacuumProtocolCheck`. This feature ensures that the Vacuum operation consistently performs both reader and writer protocol check. The motivation for this change is to address inconsistencies in Vacuum's behavior across different delta implementations, as some of them skip the writer protocol checks in practice. This omission blocks any protocol changes that might impact vacuum, including improvements to vacuum itself. The writer protocol check addresses an initial oversight in the original Delta specification where an older Delta Client executing a Vacuum command might incorrectly delete files that are still in use by newer versions, potentially leading to data corruption. **For further discussions about this protocol change, please refer to the Github issue - https://github.com/delta-io/delta/issues/2630** -------- > ***New Section*** # VACUUM Protocol Check The `vacuumProtocolCheck` ReaderWriter feature ensures consistent application of reader and writer protocol checks during `VACUUM` operations, addressing potential protocol discrepancies and mitigating the risk of data corruption due to skipped writer checks. Enablement: - The table must be on Writer Version 7 and Reader Version 3. - The feature `vacuumProtocolCheck` must exist in the table `protocol`'s `writerFeatures` and `readerFeatures`. ## Writer Requirements for Vacuum Protocol Check This feature affects only the VACUUM operations; standard commits remain unaffected. Before performing a VACUUM operation, writers must ensure that they check the table's write protocol. This is most easily implemented by adding an unconditional write protocol check for all tables, which removes the need to examine individual table properties. Writers that do not implement VACUUM do not need to change anything and can safely write to tables that enable the feature. ## Recommendations for Readers of Tables with Vacuum Protocol Check feature For tables with Vacuum Protocol Check enabled, readers don’t need to understand or change anything new; they just need to acknowledge the feature exists. ================================================ FILE: protocol_rfcs/accepted/variant-type.md ================================================ # Variant Data Type **Folded into [PROTOCOL.md](../../protocol.md#variant-data-type)** **Associated Github issue for discussions: https://github.com/delta-io/delta/issues/2864** This protocol change adds support for the Variant data type. The Variant data type is beneficial for storing and processing semi-structured data. -------- > ***New Section after the [Clustered Table](#clustered-table) section*** # Variant Data Type This feature enables support for the `variant` data type, which stores semi-structured data. The schema serialization method is described in [Schema Serialization Format](#schema-serialization-format). To support this feature: - The table must be on Reader Version 3 and Writer Version 7 - The feature `variantType` must exist in the table `protocol`'s `readerFeatures` and `writerFeatures`. ## Example JSON-Encoded Delta Table Schema with Variant types ``` { "type" : "struct", "fields" : [ { "name" : "raw_data", "type" : "variant", "nullable" : true, "metadata" : { } }, { "name" : "variant_array", "type" : { "type" : "array", "elementType" : { "type" : "variant" }, "containsNull" : false }, "nullable" : false, "metadata" : { } } ] } ``` ## Variant data in Parquet The Variant data type is represented as two binary encoded values, according to the [Spark Variant binary encoding specification](https://github.com/apache/spark/blob/master/common/variant/README.md). The two binary values are named `value` and `metadata`. When writing Variant data to parquet files, the Variant data is written as a single Parquet struct, with the following fields: Struct field name | Parquet primitive type | Description -|-|- value | binary | The binary-encoded Variant value, as described in [Variant binary encoding](https://github.com/apache/spark/blob/master/common/variant/README.md) metadata | binary | The binary-encoded Variant metadata, as described in [Variant binary encoding](https://github.com/apache/spark/blob/master/common/variant/README.md) The parquet struct must include the two struct fields `value` and `metadata`. Supported writers must write the two binary fields, and supported readers must read the two binary fields. [Variant shredding](https://github.com/apache/parquet-format/blob/master/VariantShredding.md) will be introduced in a separate `variantShredding` table feature. will be introduced later, as a separate `variantShredding` table feature. ## Writer Requirements for Variant Data Type When Variant type is supported (`writerFeatures` field of a table's `protocol` action contains `variantType`), writers: - must write a column of type `variant` to parquet as a struct containing the fields `value` and `metadata` and storing values that conform to the [Variant binary encoding specification](https://github.com/apache/spark/blob/master/common/variant/README.md) - must not write a parquet struct field named `typed_value` to avoid confusion with the field required by [Variant shredding](https://github.com/apache/parquet-format/blob/master/VariantShredding.md) with the same name. ## Reader Requirements for Variant Data Type When Variant type is supported (`readerFeatures` field of a table's `protocol` action contains `variantType`), readers: - must recognize and tolerate a `variant` data type in a Delta schema - must use the correct physical schema (struct-of-binary, with fields `value` and `metadata`) when reading a Variant data type from file - must make the column available to the engine: - [Recommended] Expose and interpret the struct-of-binary as a single Variant field in accordance with the [Spark Variant binary encoding specification](https://github.com/apache/spark/blob/master/common/variant/README.md). - [Alternate] Expose the raw physical struct-of-binary, e.g. if the engine does not support Variant. - [Alternate] Convert the struct-of-binary to a string, and expose the string representation, e.g. if the engine does not support Variant. ## Compatibility with other Delta Features Feature | Support for Variant Data Type -|- Partition Columns | **Supported:** A Variant column is allowed to be a non-partitioned column of a partitioned table.
**Unsupported:** Variant is not a comparable data type, so it cannot be included in a partition column. Clustered Tables | **Supported:** A Variant column is allowed to be a non-clustering column of a clustered table.
**Unsupported:** Variant is not a comparable data type, so it cannot be included in a clustering column. Delta Column Statistics | **Supported:** A Variant column supports the `nullCount` statistic.
**Unsupported:** Variant is not a comparable data type, so a Variant column does not support the `minValues` and `maxValues` statistics. Generated Columns | **Supported:** A Variant column is allowed to be used as a source in a generated column expression, as long as the Variant type is not the result type of the generated column expression.
**Unsupported:** The Variant data type is not allowed to be the result type of a generated column expression. Delta CHECK Constraints | **Supported:** A Variant column is allowed to be used for a CHECK constraint expression. Default Column Values | **Supported:** A Variant column is allowed to have a default column value. Change Data Feed | **Supported:** A table using the Variant data type is allowed to enable the Delta Change Data Feed. -------- > ***New Sub-Section after the [Map Type](#map-type) sub-section within the [Schema Serialization Format](#schema-serialization-format) section*** ### Variant Type Variant data uses the Delta type name `variant` for Delta schema serialization. Field Name | Description -|- type | Always the string "variant" ================================================ FILE: protocol_rfcs/checkpoint-protection.md ================================================ # Checkpoint Protection This RFC introduces a new Writer feature named `checkpointProtection`. When the feature is present in the protocol, no checkpoint removal/creation before `delta.requireCheckpointProtectionBeforeVersion` is allowed during metadata cleanup, unless everything is cleaned up in one go. The motivation is to improve the drop feature functionality. Today, dropping a feature requires truncating the history of a Delta table at the version boundary where the feature is removed from the protocol. This is necessary because the Delta protocol only safely supports table protocols that are monotonically increasing with table versions. And because it is unsafe to truncate the history of a Delta table while transactions are running, dropping a feature requires a 24-hour wait time to avoid corrupting the table. We can improve this process by setting up the table's history (including checkpoints) in such a way that older readers will be able to handle it correctly, i.e., to read correctly at versions for which they support the read protocol, and to reject reading of versions for which they do not support all features. The `checkpointProtection` feature is needed to ensure that this very specific setup of the history _stays in place_ until the feature removal is cleaned up from the retained version history. A key component of this solution is a special set of protected checkpoints at the DROP FEATURE boundary that are guaranteed to persist until all history is truncated up to the checkpoints in one go. These checkpoints act as barriers that hide unsupported commit records behind them. By "hiding", we mean that older readers will not need to replay those commits that they _don't_ support in order to reconstruct the table state at a later version that they _do_ support. With the `checkpointProtection`, we can guarantee these checkpoints will persist until history is truncated. Furthermore, with the new drop feature method, it is no longer guaranteed that protocols are monotonically increasing. This means that clients that validate against the latest protocol can no longer assume that they can also operate on earlier versions correctly. In particular, writers are allowed to create checkpoints at earlier versions, but if they do this without checking the protocol at that specific version, and then they may write corrupted checkpoints for table versions for which they do not support the protocol. The `checkpointProtection` table feature also protects against these cases by requiring writers to check the protocol versions at historical table versions before creating a new checkpoint. With these changes, we can drop table features without needing to truncate history. More importantly, they simplify the drop feature user journey by requiring a single execution of the DROP FEATURE command. **For further discussions about this protocol change, please refer to the Github issue - https://github.com/delta-io/delta/issues/4152** -------- > ***Add a new section at the [Table Features](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#table-features) section*** # Checkpoint Protection The `checkpointProtection` is a Writer feature that protects checkpoints before the version indicated by table property `delta.requireCheckpointProtectionBeforeVersion`, and that forbids writers from creating checkpoints before that version unless they confirm that they support the table protocol at that version. Enablement: - The table must be at least on Writer Version 7 and Reader Version 1. - The feature `checkpointProtection` must exist in the table `protocol`'s `writerFeatures`. ## Writer Requirements for Checkpoint Protection For tables with `checkpointProtection` supported in the protocol: a) Writers must not clean up any checkpoints for table versions before the version given by table property `delta.requireCheckpointProtectionBeforeVersion`. b) Writers must not create new checkpoints for table versions before the version given by table property `delta.requireCheckpointProtectionBeforeVersion` unless they support all of the features in the table protocol at that version. c) Writers must not clean up version history for table versions for which they do not support the protocol. A writer is allowed to clean up a range of versions if it supports all table features for every version that is being cleaned up. If a writer does not support the protocol for some of the versions that are being cleaned up, then the cleanup is allowed if and only if the cleanup includes _all_ table versions before the version given by `delta.requireCheckpointProtectionBeforeVersion`. In this case, a single cleanup operation should truncate the history up to that boundary version in one go as opposed to several cleanup operations truncating in chunks. d) In version history cleanup, writers must remove commits _before_ removing the associated checkpoints, so that requirement (a) is satisfied even during the cleanup. ## Recommendations for Readers of Tables with Checkpoint Protection feature For tables with `checkpointProtection` supported in the protocol, readers do not need to understand or change anything new; they just need to acknowledge the feature exists. ================================================ FILE: protocol_rfcs/collated-string-type.md ================================================ # Collated String Type **Associated Github issue for discussions: https://github.com/delta-io/delta/issues/2894** This protocol change adds support for collated strings. It consists of three changes to the protocol: * Collations in the table schema * Per-column statistics are annotated with the collation that was used to collect them * Domain metadata with active collation version -------- > *** Add New Section after the [Clustered Table](#clustered-table) section*** # Collations Table Feature To support this feature: * The table must have Writer Version 7. * The feature `collations` must exist in the table's `writerFeatures`. * The feature `domainMetadata` must exist in the table's `writerFeatures`. ## Reader Requirements for Collations: When Collations are supported (when the `writerFeatures` field of a table's protocol action contains `collations`), then: - Readers could do comparisons and sorting of strings based on the collation specified in the schema. - If the collation is not specified for a string type, then the reader must use the default comparison operators for the binary representation of strings under UTF-8 encoding. - Readers must only do file skipping based on column statistics for a collation if the filter operator used for the data skipping is specified to treat the column as having that same collation. For example, when filtering a string column using the string equality comparison operator that is configured with the collation `ICU.en_US.72`, the reader must not use file skipping statistics from the collation `spark.UTF8_LCASE.75.1`. It should also not use the statistics from `ICU.en_US.69` because the collation version number does not match. ## Writer Requirements for Collations: When Collations are supported (when the `writerFeatures` field of a table's protocol action contains `collations`), then: - Writers must write the collation identifier in the schema metadata for a column with non-default collation, i.e., any collation that is not comparing strings using their binary representations under UTF-8 encoding. - Writers must not write the collation identifier in the schema metadata for a column with default collation (comparisons using binary representation of the strings under UTF-8 encoding). - Writers could write per-file statistics for string columns with non-default collations in `statsWithCollation`. See [Per-file Statistics](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#per-file-statistics) for more details. - If a writer adds per-file statistics for a new version of a collation, the writer should also update the `domainMetadata` for the `collations` table feature to include the new collation versions that are used to collect statistics. - Writers could remove a collation version from the `domainMetadata` for the `collations` table feature if stats collection for the collation version is no longer desired. For example, the engine upgrades their ICU library and now desires a newer version for a collation. > ***Add a new section in front of the [Primitive Types](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#primitive-types) section.*** ### Collations Collations are a set of rules for how strings are compared. Collations do not affect how strings are stored. Collations are applied when comparing strings for equality or to determine the sort order of two strings. Case insensitive comparison is one example of a collation where case is ignored when string are compared for equality and the lower cased variant of a string is used to determine its sort order. Each string field can have a collation, which is specified in the table schema. It is also possible to store statistics per collation version. This is required because the min and max values of a column can differ based on the used collation or collation version. By default, all strings are collated using binary collation. That means that strings compare equal if their binary UTF-8 encoded representations are equal. The binary UTF-8 encoded representation is also used to sort them. Note that in Delta all strings are encoded in UTF-8. The `collations` table feature is a writer only feature and allows clients that do not support collations to read the table using UTF-8 binary collation. To support the table feature clients must preserve collations when they change the schema. Collecting collated statistics is optional and it is valid to store UTF-8 binary collated statistics for fields with a collation other than UTF-8 binary. The column level collation indicates the default collation that readers should use to operate on a column. However, readers are responsible for choosing what collation to actually apply on operations. An engine may apply a different collation than the schema collation based on the engine's collation precedence rules. However, an engine must take care to only use column statistics for file skipping from a collation that is identical to the one specified in the filtering operation in all aspects, including the collation version. #### Collation identifiers Collations can be referred to using collation identifiers. The Delta format does not specify any collation rules other than binary collation, but supports the concept of collation providers such that engines can use providers like [ICU](https://icu.unicode.org/) and mark statistics accordingly. A collation identifier consists of 3 parts, which are combined into one identifier using dots as separators. Dots are not allowed to be part of provider and collation names, but can be used in versions. Part | Description -|- Provider | Name of the provider. Must not contain dots Name | Name of the collation as provided by the provider. Must not contain dots Version | Version string. Is allowed to contain dots. This part is optional. Collations without a version are used in the schema because readers are not forced to use a specific version of the collation. Statistics are annotated with versioned collations to guarantee correctness. #### Specifying collations in the table schema Collations can be specified for any string type in a schema. This includes string fields, but also the key and value type of maps and the element type of arrays. Collations are stored in the `__COLLATIONS` key of the metadata of the nearest ancestor [StructField](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#struct-field) of the Delta table schema. Nested maps and arrays are encoded the same way as ids in [IcebergCompatV2](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#writer-requirements-for-icebergcompatv2). Collation identifiers are stored without key because the version of a collation is not enforced for reading. This example provides an overview of how collations are stored in the schema. Note that irrelevant fields have been stripped. Example schema ``` |-- col1: string |-- col2: array | |-- elementType: map | |-- keyType: string | |-- valueType: struct | |-- f1: string ``` Schema with collation information ``` { "type":"struct", "fields":[ { "name":"col1", "type":"string", "metadata":{ "__COLLATIONS":{ "col1":"ICU.de_DE" } } }, { "name":"col2", "type":{ "type":"array", "elementType":{ "type":"map", "keyType":"string", "valueType":{ "type":"struct", "fields":[ { "name":"f1", "type":"string", "metadata":{ "__COLLATIONS":{ "f1":"ICU.de_DE" } } } ] } } }, "metadata":{ "__COLLATIONS":{ "col2.element.key":"ICU.en_US" } } } ] } ``` #### Collation versions The [Domain Metadata](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#domain-metadata) for the `collations` table feature contains hints for which versions of a collations clients should produce statistics when writing to the table. The hints allow clients to choose a collation version without having to look at the statistics of all AddFiles first. Clients are allowed to ignore the hints. `collations` [Domain Metadata](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#domain-metadata) ``` { "writeVersions": { "ICU.en_US": ["72", "73"] } } ``` > ***Update the string row in the [Primitive Types](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#primitive-types) table.*** ### Primitive Types Type Name | Description -|- string| UTF-8 encoded string of characters. A collation can be specified in [Column Metadata](#specifying-collations-in-the-table-schema), otherwise binary collation is used as the default. > ***Add new rows to the [Column Metadata](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#column-metadata) table.*** Field Name | Description -|- __COLLATIONS | Collations for strings stored in the field or combinations of maps and arrays that are stored in this field and do not have nested structs. Refer to [Specifying collations in the table schema](#specifying-collations-in-the-table-schema) for more details. > ***Edit the [Per-file Statistics](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#per-file-statistics) section and change it from the "Per-column statistics" section onwards.*** Per-column statistics record information for each column in the file and they are encoded, mirroring the schema of the actual data. Statistic are optional and it is allowed to provide UTF-8 binary statistics for strings when the field has a different collation. For example, given the following data schema: ``` |-- a: struct | |-- b: struct | | |-- c: long |-- d: struct |-- e: string collate ICU.en_US.72 ``` Statistics could be stored with the following schema: ``` |-- stats: struct | |-- numRecords: long | |-- tightBounds: boolean | |-- minValues: struct | | |-- a: struct | | | |-- b: struct | | | | |-- c: long | |-- maxValues: struct | | |-- a: struct | | | |-- b: struct | | | | |-- c: long | |-- statsWithCollation: struct | | |-- ICU.en_US.72: struct | | | |-- minValues: struct | | | | |-- d: struct | | | | | | e: string | | | |-- maxValues: struct | | | | |-- d: struct | | | | | | e: string ``` The following per-column statistics are currently supported: Name | Description (`stats.tightBounds=true`) | Description (`stats.tightBounds=false`) -|-|- nullCount | The number of `null` values for this column |

If the `nullCount` for a column equals the physical number of records (`stats.numRecords`) then **all** valid rows for this column must have `null` values (the reverse is not necessarily true).

If the `nullCount` for a column equals 0 then **all** valid rows are non-`null` in this column (the reverse is not necessarily true).

If the `nullCount` for a column is any value other than these two special cases, the value carries no information and should be treated as if absent.

minValues | A value that is equal to the smallest valid value[^1] present in the file for this column. If all valid rows are null, this carries no information. | A value that is less than or equal to all valid values[^1] present in this file for this column. If all valid rows are null, this carries no information. maxValues | A value that is equal to the largest valid value[^1] present in the file for this column. If all valid rows are null, this carries no information. | A value that is greater than or equal to all valid values[^1] present in this file for this column. If all valid rows are null, this carries no information. statsWithCollation | minValues and maxValues for string columns that are not using binary collation. | Has the same semantics as the top level minValues and maxValues, but wraps both minValues and maxValues into an object keyed by the collation used the generate them. [^1]: String columns are cut off at a fixed prefix length. Timestamp columns are truncated down to milliseconds. ================================================ FILE: protocol_rfcs/column-mapping-usage-tracking.md ================================================ # Column Mapping Usage Tracking **Associated Github issue for discussions: https://github.com/delta-io/delta/issues/2682** This RFC proposes an extension for Column Mapping to track where columns have been dropped or renamed during the history of a table. This allows using the (logical) name of a column as the physical name of a column, while still ensuring that all physical names are unique. This helps with the disablement of Column Mapping proposed in [#2481](https://github.com/delta-io/delta/issues/2481), as in this case it is no longer required to rewrite the table, and it simply suffices to change the mode to none. -------- > New subsection at the end of the `Column Mapping` section ## Usage Tracking Column Mapping Usage Tracking is an extension of the column mapping feature that allows Delta to track whether a column has been dropped or renamed. This is tracked by the table property `delta.columnMapping.hasDroppedOrRenamed`. This table property is set to `false` when the table is created, and flipped to `true` when the first column is either dropped or renamed. The writer table feature `columnMappingUsageTracking` is added to the `writerFeatures` in the `protocol` to ensure that all writers correctly track when columns are dropped or renamed. -------- > Modification to the `Writer Requirements for Column Mapping` subsection - Assign a globally unique identifier as the physical name for each new column that is added to the schema. This is especially important for supporting cheap column deletions in `name` mode. In addition, column identifiers need to be assigned to each column. The maximum id that is assigned to a column is tracked as the table property `delta.columnMapping.maxColumnId`. This is an internal table property that cannot be configured by users. This value must increase monotonically as new columns are introduced and committed to the table alongside the introduction of the new columns to the schema. **is replaced by** - Assign a unique physical name to each column. - When enabling column mapping on existing table, the physical name of the column must be set to the (logical) name of the column. - If the feature `columnMappingUsageTracking` is supported, then when adding a new column to a table and `delta.columnMapping.hasDroppedOrRenamed` column property is `false` the (logical) name of the column should be used as the physical name. - Otherwise the physical column name must contain a universally unique identifier (UUID) to guarantee uniqueness. - Assign a column id to each column. The maximum id that is assigned to a column is tracked as the table property `delta.columnMapping.maxColumnId`. This is an internal table property that cannot be configured by users. This value must increase monotonically as new columns are introduced and committed to the table alongside the introduction of the new columns to the schema. -------- > New subsection at the end of the `Writer Requirements for Column Mapping` subsection ### Writer Requirements for Usage Tracking In order to support column mapping usage tracking, writers must: - Write `protocol` and `metaData` actions when Column Mapping Usage Tracking is turned on for the first time: - Write a `protocol` action with writer version 7 and the feature `columnMappingUsageTracking` in the `writerFeatures`. - Write a `metaData` action with the table property `delta.columnMapping.hasDroppedOrRenamed` set to `false` when creating a new table or enabling the feature on an existing table without column mapping enabled, and set to `true` when enabling usage tracking on an existing table with column mapping enabled. - When dropping or renaming a column `delta.columnMapping.hasDroppedOrRenamed` must be set to `true`. - After `delta.columnMapping.hasDroppedOrRenamed` is set to `true` it must never be set back to `false` again. ================================================ FILE: protocol_rfcs/iceberg-compat-v3.md ================================================ # IcebergCompatV3 This protocol change introduces a compatibility flag, which ensures that a delta table can be safely read and written as an Apache Iceberg™ format table, similar to [IcebergCompatV1](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#iceberg-compatibility-v1) and [IcebergCompatV2](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#iceberg-compatibility-v2). -------- # IcebergCompatV3 > ***New Section after [Iceberg Compatibility V2](#iceberg-compatibility-v2)*** # Iceberg Compatibility V3 This table feature (`icebergCompatV3`) ensures that Delta tables can be converted to Apache Iceberg™ format, though this table feature does not implement or specify that conversion. To support this feature: - Since this table feature depends on Column Mapping, the table must be on Reader Version = 2, or it must be on Reader Version >= 3 and the feature `columnMapping` must exist in the `protocol`'s `readerFeatures`. - The table must be on Writer Version 7. - The feature `icebergCompatV3` must exist in the table protocol's `writerFeatures`. This table feature is enabled when the table property `delta.enableIcebergCompatV3` is set to `true`. > **NOTE:** Unlike IcebergCompatV1 and IcebergCompatV2, this feature does _NOT_ forbid supporting and enabling Deletion Vectors on the table. ## Writer Requirements for IcebergCompatV3 When this feature is supported and enabled, writers must: - Require that Column Mapping be enabled and set to either `name` or `id` mode - Require that Row Tracking to be enabled on the table. - Materialized Row ID column must use field ID 2147483540 - Materialized Row Commit Version column must use field ID 2147483539 - Require that the nested `element` field of ArrayTypes and the nested `key` and `value` fields of MapTypes be assigned 32 bit integer identifiers. The requirement to ID allocation is the same as that in IcebergCompatV2. - Require that IcebergCompatV1 and IcebergCompatV2 are not active on the table - Require that partition column values be materialized when writing Parquet data files - Require that all new `AddFile`s committed to the table have the `numRecords` statistic populated in their `stats` field - Require writing timestamp columns as int64 - Block replacing partitioned tables with a differently-named partition spec - e.g. replacing a table partitioned by `part_a INT` with partition spec `part_b INT` must be blocked - e.g. replacing a table partitioned by `part_a INT` with partition spec `part_a LONG` is allowed ================================================ FILE: protocol_rfcs/iceberg-writer-compat-v1.md ================================================ # IcebergWriterCompatV1 **Associated Github issue for discussions: https://github.com/delta-io/delta/issues/4284 This protocol change introduces a compatibility flag, which ensures that a delta table can be safely read and written as an Apache Iceberg™ format table, similar to [IcebergCompatV1](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#iceberg-compatibility-v1) and [IcebergCompatV2](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#iceberg-compatibility-v2). -------- # IcebergWriterCompatV1 > ***New Section after [Iceberg Compatibility V2](#iceberg-compatibility-v2)*** This table feature (`icebergWriterCompatV1`) ensures that Delta tables can be converted to Apache Iceberg™ format, though this table feature does not implement or specify that conversion. To support this feature: - Since this table feature depends on Column Mapping, the table must be on Reader Version = 2, or it must be on Reader Version >= 3 and the feature `columnMapping` must exist in the `protocol`'s `readerFeatures`. - The table must be on Writer Version 7. - The feature `icebergCompatV2` must exist in the table protocol's `writerFeatures`. - The feature `icebergWriterCompatV1` must exist in the table protocol's `writerFeatures`. This table feature is enabled when the table property `delta.enableIcebergWriterCompatV1` is set to `true`. ## Writer Requirements for IcebergWriterCompatV1 For `IcebergWriterCompatV1` writers must ensure: - The table is using [Column Mapping](#column-mapping) and that it is set to `id` mode. - Note this is a tightening of the `IcebergCompatV2` requirement which supports `name` and `id` mode. - Each field _must_ have a column mapping physical name that is exactly `col-[column id]`. That is the `delta.columnMapping.physicalName` in the column metadata _must_ be equal to `col-[delta.columnMapping.id]`. The following is an example compliant schema definition: ```json { "type": "struct", "fields": [ { "name": "a", "type": "integer", "nullable": false, "metadata": { "delta.columnMapping.id": 1, "delta.columnMapping.physicalName": "col-1" } }, { "name": "b", "type": "string", "nullable": false, "metadata": { "delta.columnMapping.id": 2, "delta.columnMapping.physicalName": "col-2" } } ] } ``` - The table does not contain any columns with the type `byte` or `short` - Note that these types _are_ allowed by `IcebergCompatV2` - Therefore the list of allowed types for a table with `IcebergWriterCompatV1` enabled is: [`integer`, `long`, `float`, `double`, `decimal`, `string`, `binary`, `boolean`, `timestamp`, `timestampNTZ`, `date`, `array`, `map`, `struct`]. - [Iceberg Compatibility V2](#iceberg-compatibility-v2) is **enabled** on the table. - This means _all_ the conditions that [Iceberg Compatibility V2](#iceberg-compatibility-v2) imposes are met. - The writer **must** block *any* schema changes to a `struct` that is used as a `map` key. - For example, if the schema contains `map MAP, INT>`, then any schema change to `map.key` must be disallowed. - Changes to the schema of the value are allowed. - This matches Iceberg's behavior, which is documented [here](https://iceberg.apache.org/docs/nightly/spark-ddl/#alter-table-add-column). In practice Iceberg writers block any changes, not just column additions. - Any enabled features are in the [allowlist](#allowed-supported-list-of-features) - All [Disallowed features](#disallowed-features) are not supported and/or inactive (see below) ### Disallowed Features For this section, we use the specific meanings of "supported" and "active" from [Supported Features](#supported-features). All the following features must not be used in the table. For legacy features (any feature introduced before writer version 7), the feature _can_ be "supported", but must _not_ be "active". | Feature | Legacy | Can be ["supported"](#supported-features)? | Not Active Check | |---------------------------------------------------------------------------------------------------|--------|--------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | [column invariants](#column-invariants) | Yes | Yes, if not active | No column includes `delta.invariants` in its [Metadata] | | [Change Data Feed](#add-cdc-file) | Yes | Yes, if not active | The `delta.enableChangeDataFeed` configuration flag in the [Metadata] of the table does not exist (or is `disabled`?) | | [CHECK Constraints](#check-constraints) | Yes | Yes, if not active | No keys in the `configuration` field of [Metadata] start with `delta.constraints.`. | | [Identity Columns](#identity-columns) | Yes | Yes, if not active | No columns exist in the schema with any of the properties specified in [Identity Columns](#identity-columns) in the column metadata: `delta.identity.start`, `delta.identity.step`, `delta.identity.highWaterMark`, `delta.identity.allowExplicitInsert` | | [Generated Columns](#default-columns) | Yes | Yes, if not active | No column metadata contains the key `delta.generationExpression` | | [Default Columns](#default-columns) | No | No | N/A | | [Row Tracking](#row-tracking) | No | Yes, if not active | The delta.enableRowTracking configuration flag in the Metadata of the table does not exist (or has a value of false) | | [Collations](https://github.com/delta-io/delta/blob/master/protocol_rfcs/collated-string-type.md) | No | No | N/A | | [Variant Types](#variant-data-type) | No | No | N/A | ### Allowed Supported list of features To ensure that future features do not break tables with `IcebergWriterCompatV1` enabled, all enabled features must also be checked against an allowlist. Any enabled table features _must_ be in the list: [`appendOnly`, `columnMapping`, `icebergWriterCompatV1`, `icebergCompatV2`, `domainMetadata`, `vacuumProtocolCheck`, `v2Checkpoint`, `inCommitTimestamp`, `clustering`, `timestampNtz`, `typeWidening`] Additionally, the following features are allowed to be "supported", but must not be "active" (see [Disallowed Features](#disallowed-features)): [`invariants`, `changeDataFeed`, `checkConstraints`, `identityColumns`, `generatedColumns`, `rowTracking`]. These features, if supported, must be verified to be "inactive" via the checks specified above. We allow these legacy features to be "supported" because protocol updates can cause features to be carried over even though they are not in use. For example, if a table is on writer version 2, and then is updated to version 7, `invariants` can appear in the `writerFeatures` list because it was implicitly supported at version 2, even if it was not in use. [Metadata]: #change-metadata ================================================ FILE: protocol_rfcs/materialize-partition-columns.md ================================================ # Materialize Partition Columns **Associated Github issue for discussions: https://github.com/delta-io/delta/issues/5555** ## Overview Currently, Delta tables store partition column values primarily in the table metadata (specifically in the `partitionValues` field of `AddFile` actions), and by default these columns are not physically written into the Parquet data files themselves. This RFC proposes a new writer-only table feature called `materializePartitionColumns`. When supported, this feature requires partition columns to be physically materialized in Parquet data files alongside the data columns. ## Motivation This feature provides a mechanism to require partition column materialization at the protocol level, ensuring all writers to the table comply with this requirement during the period when the feature is supported. Materializing partition columns enhances compatibility with Parquet readers that access Parquet files directly and do not interpret Delta’s AddFile metadata, as well as with Iceberg readers, which expect partition columns to be stored within the data files. Additionally, having partition information embedded in the data files themselves enables more flexible data reorganization strategies. The same parquet files could be linked in future versions of a table that do not have the same (or any) partition columns. -------- > ***New Section after Identity Columns section*** ## Materialize Partition Columns When this feature is supported, partition columns are physically written to Parquet files alongside the data columns. To support this feature: - The table must be on Writer Version 7, and a feature name `materializePartitionColumns` must exist in the table `protocol`'s `writerFeatures`. When supported: - When the writer feature `materializePartitionColumns` is set in the protocol, writers must materialize partition columns into any newly created data file, placing them after the data columns in the parquet schema. This mimics the same partition column materialization requirement from [IcebergCompatV1](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#iceberg-compatibility-v1) and [IcebergCompatV2](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#iceberg-compatibility-v2). As such, the `materializePartitionColumns` feature can be seen as a subset of the requirements imposed by those features, providing the partition column materialization guarantee independently without requiring full Iceberg compatibility. - When the writer feature `materializePartitionColumns` is not set in the table protocol, writers are not required to write partition columns to data files. Note that other features might still require materialization of partition values, such as [IcebergCompatV1](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#iceberg-compatibility-v1) This feature does not impose any requirements on readers. All Delta readers must be able to read the table regardless of whether partition columns are materialized in the data files. If partition values are present in both parquet and AddFile metadata, Delta readers should continue to read partition values from AddFile metadata. ================================================ FILE: protocol_rfcs/rejected/managed-commits.md ================================================ # Managed Commits **Associated Github issue for discussions: https://github.com/delta-io/delta/issues/2598** This RFC proposes a new table feature `managedCommit` which changes the way Delta Lake performs commits. Today’s Delta commit protocol relies on the filesystem to provide commit atomicity. This feature request is to allow Delta tables which gets commit atomicity using an external commit-owner and not the filesystem (s3, abfs etc). This allows us to deal with various limitations of Delta: 1. No reliable way for the table's owner to participate in commits. - The table's owner (such as a catalog) cannot reliably stay in sync with the table state, nor reject commit attempts it wouldn’t like, because it doesn’t even know about writes until they are already durable (and visible to readers). - No clear path to transactions that could span multiple tables and/or involve catalog updates, because filesystem commits cannot be made conditionally or atomically. 2. No way to tie commit ownership to a table. - In general, Delta tables have no way to advertise that they are managed by catalog or LogStore X (at endpoint Y). - No way to express different commit owners for different tables. For example, Delta spark supports a notion of a "[log store](https://delta.io/blog/2022-05-18-multi-cluster-writes-to-delta-lake-storage-in-s3/)" or commit service for enforcing commit atomicity in S3, but it's a cluster-level setting that affects all tables indiscriminately, with no way to validate whether the mapping is even correct. - There is no central entity that needs to be contacted in order to commit to the table. So if the underlying file system is missing _putIfAbsent_ semantics, then there is no way to ensure that a commit is atomic, which could lead to lost writes when concurrent writers are writing to the table. -------- ### Delta Log Entries > ***Change to [existing section](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#delta-log-entries)*** Delta files are stored as JSON in a directory at the root of the table named `_delta_log`, and together with checkpoints make up the log of all changes that have occurred to a table. ~~Delta files are the unit of atomicity for a table, and are named using the next available version number, zero-padded to 20 digits.~~ They are the unit of atomicity for a table. **Note:** If [managed commits](#managed-commits) table feature is enabled on the table, recently committed delta files may reside in the `_delta_log/_commits` directory. Delta clients have to contact the corresponding commit-owner of the table in order to find the information about the [un-backfilled commits](#commit-backfills). The delta files in `_delta_log` directory are named using the next available version number, zero-padded to 20 digits. For example: ``` ./_delta_log/00000000000000000000.json ``` The delta files in the `_delta_log/_commits` directory have a UUID embedded into them and follow the pattern `..json`, where the version corresponds to the next attempt version zero-padded to 20 digits. For example: ``` ./_delta_log/_commits/00000000000000000000.3a0d65cd-4056-49b8-937b-95f9e3ee90e5.json ./_delta_log/_commits/00000000000000000001.7d17ac10-5cc3-401b-bd1a-9c82dd2ea032.json ./_delta_log/_commits/00000000000000000001.016ae953-37a9-438e-8683-9a9a4a79a395.json ./_delta_log/_commits/00000000000000000002.3ae45b72-24e1-865a-a211-34987ae02f2a.json ``` The `_delta_log/_commits` directory may contain uncommitted delta files. The [commit-owner](#commit-owner) is the source of truth about which of those delta files map to committed versions. Refer to [managed commits](#managed-commits) for more details. ~~Delta files use new-line delimited JSON format, where every action is stored as a single line JSON document. A delta file, `n.json`, contains an atomic set of [_actions_](#Actions) that should be applied to the previous table state, `n-1.json`, in order to the construct `n`th snapshot of the table. An action changes one aspect of the table's state, for example, adding or removing a file.~~ Delta files use newline-delimited JSON format, where every action is stored as a single line JSON document. A delta file, corresponding to version `n`, contains an atomic set of [_actions_](#Actions) that should be applied to the previous table state, corresponding to `n-1`, in order to construct the `n`th snapshot of the table. An action changes one aspect of the table's state, for example, adding or removing a file. ### Metadata Cleanup > ***Change to [existing section](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#metadata-cleanup)*** 2. Identify the newest checkpoint that is not newer than the `cutOffCommit`. A checkpoint at the `cutOffCommit` is ideal, but an older one will do. Lets call it `cutOffCheckpoint`. We need to preserve the `cutOffCheckpoint` and all commits after it, because we need them to enable time travel for commits between `cutOffCheckpoint` and the next available checkpoint. - If no `cutOffCheckpoint` can be found, do not proceed with metadata cleanup as there is nothing to cleanup. 3. Delete all [delta log entries](#delta-log-entries) and [checkpoint files](#checkpoints) before the `cutOffCheckpoint` checkpoint. Also delete all the [log compaction files](#log-compaction-files) having startVersion <= `cutOffCheckpoint`'s version. - Also delete all the [un-backfilled commit files](#commit-files) having version <= `cutOffCheckpoint`'s version from the `_delta_log/_commits` directory. ### Checkpoints > ***Change to [existing section](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#checkpoints)*** Checkpoints are also stored in the `_delta_log` directory, and can be created at any time, for any committed version of the table. For performance reasons, readers should prefer to use the newest complete checkpoint possible. **Note:** If [managed commits](#managed-commits) table feature is enabled on the table, a checkpoint can be created only for commit versions which are backfilled. Refer to [maintenance operations on managed-commit tables](#maintenance-operations-on-managed-commit-tables) section for more details ### Log Compaction Files > ***Change to [existing section](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#log-compaction-files)*** **Note:** If [managed commits](#managed-commits) table feature is enabled on the table, a log compaction file for commit range `[x, y]` i.e. `x.y.compacted.json` can be created only when commit `y` is already backfilled i.e. `_delta_log/.json` must exist. Refer to [maintenance operations on managed-commit tables](#maintenance-operations-on-managed-commit-tables) section for more details. > ***The next set of sections will be added to the existing spec just before [Iceberg Compatibility V1](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#iceberg-compatibility-v1) section*** # Managed Commits With this feature enabled: - The file system remains the source of truth for the _content_ of a (proposed) commit. - The [commit-owner](#commit-owner) becomes the source of truth for whether a given commit succeeded. The following is a high-level overview of how commits work in a table with managed-commits enabled: 1. Delta client passes the actions that need to be committed to the [commit-owner](#commit-owner). 2. The [commit-owner](#commit-owner) abstracts the commit process and defines the atomicity protocol for commits to that table. It writes the actions in a [delta file](#delta-log-entries) and atomically makes this file part of the table. Refer to [commit protocol](#commit protocol) section for details around how the commit-owner performs commits. 3. In case of no conflict, the [commit-owner](#commit-owner) responds with success to the delta client. 4. Delta clients could contact the commit-owner to get the information about the table's most recent commits. Essentially the [managed-commits](#managed-commits) table feature defines the overall [commit protocol](#commit-protocol) (e.g. atomicity requirements, backfills, etc), and the commit-owner is responsible to implement that protocol. ## Commit Owner A commit-owner is an external entity which manages the commits on a delta table. It could be backed by a database, a file system, or any other persistence mechanism. Each commit-owner has its own spec around how Delta clients should contact them, and how they perform a commit. ## Commit Files A commit file is a [delta file](#delta-log-entries) that contains the actions which are committed / need to be committed. There are two types of commit files: 1. **Un-backfilled commit files**: These reside in the `_delta_log/_commits` directory. - The filename must follow the pattern: `..json`. Here the `uuid` is a random UUID that is generated for each commit and `version` is the version `v` which is being committed, zero-padded to 20 digits. - Mere existence of these files does not mean that the file is a _valid_ commit. It might correspond to a failed or in-progress commit. The commit-owner is the source of truth around which un-backfilled commits are valid. - The commit-owner must track these files until they are backfilled to the `_delta_log` directory. 2. **Backfilled commit files**: These reside in the `_delta_log` directory. - The filename must follow the pattern: `.json`. Here the `version` is the version `v` which is being committed, zero-padded to 20 digits. - The existence of a `.json` file proves that the corresponding version `v` is committed, even for managed-commit tables. Filesystem based Delta clients can use filesystem listing operations to directly discover such commits. Without [managed-commits](#managed-commits), a delta client must always write commit files directly to the `_delta_log` directory, relying on filesystem atomicity to prevent lost writes when multiple writers attempt to commit the same version at the same time. With [managed-commits](#managed-commits), the delta client asks the [commit-owner](#commit-owner) to commit the version `v` and the commit-owner decides which type of commit file to write, based on the [managed commit protocol](#commit-protocol). ## Commit Owner API When managed commits are enabled, a `commit-owner` performs commits to the table on behalf of the Delta client. A commit-owner always has a client-side component (which the Delta client interacts with directly). It may also involve a server-side component (which the client-side component would be responsible to communicate with). The Delta client is responsible to define the client-side API that commit-owners should target, and commit-owners are responsible to define the commit atomicity and backfill protocols which the commit-owner client should implement. At a high level, the `commit-owner` needs to provide: - API to atomically commit a version `x` with given set of `actions`. This is explained in detail in the [commit protocol](#commit-protocol) section. - API to retrieve information about the recent commits and the latest ratified version on the table. This is explained in detail in the [getting un-backfilled commits from commit-owner](#getting-un-backfilled-commits-from-commit-owner) section. ### Commit Protocol When a `commit-owner` receives a request to commit version `v`, it must first verify that the previous version `v-1` already exists, and that version `v` does not yet exist. It then has following choices to publish the commit: 1. Write the actions to an 'un-backfilled' [commit file](#commit-files) in the `_delta_log/_commits` directory, and **atomically** record that the new file now corresponds to version `v`. 2. Atomically write a backfilled [commit file](#commit-files) in the `_delta_log` directory. Note that the commit will be considered to have succeeded as soon as the file becomes visible to other clients in the filesystem, regardless of when or whether the originating client receives a response. - A commit-owner must not write a backfilled commit until the previous commit has been backfilled. The commit-owner must track the un-backfilled commits until they are [backfilled](#commit-backfills). ### Getting Un-backfilled Commits from Commit Owner Even after a commit succeeds, Delta clients can only discover the commit through filesystem operations if the commit is [backfilled](#backfills). If the commit is not backfilled, then delta implementations have no way to determine which file in `_delta_log/_commits` directory corresponds to the actual commit `v`. The commit-owner is responsible to implement an API (defined by the Delta client) that Delta clients can use to retrieve information about un-backfilled commits maintained by the commit-owner. The API must also return the latest version of the table ratified by the commit-owner (if any). Providing the latest ratified table version helps address potential race conditions between listing commits and contacting the commit-owner. For example, if a client performs a listing before a recently ratified commit is backfilled, and then contacts the commit-owner after the backfill completes, the commit-owner may return an empty list of un-backfilled commits. Without knowing the latest ratified version, the client might incorrectly assume their listing was complete and read a stale snapshot. Delta clients who are unaware of the commit-owner (or unwilling to talk to it), may not see recent un-backfilled commits and thus may encounter stale reads. ## Sample Commit Owner API The following is an example of a possible commit-owner API which some Java-based Delta client might require commit-owner implementations to target: ```java interface CommitStore { /** * Commits the given set of `actions` to the given commit `version`. * * @param version The version we want to commit. * @param actions Actions that need to be committed. * * @return CommitResponse which has details around the new committed delta file. */ def commit( version: Long, actions: Iterator[String]): CommitResponse /** * API to get the un-backfilled commits for the table represented by the given `tablePath` where * `startVersion` <= version <= endVersion. * If endVersion is -1, then it means that we want to get all the commits starting from `startVersion` * till the latest version tracked by commit-owner. * The returned commits are contiguous and in ascending version order. * Note that the first version returned by this API may not be equal to the `startVersion`. This * happens when few versions starting from `startVersion` are already backfilled and so * CommitStore may have stopped tracking them. * The returned latestTableVersion is the maximum commit version ratified by the Commit-Owner. * Note that returning latestTableVersion as -1 is acceptable only if the commit-owner never * ratified any version i.e. it never accepted any un-backfilled commit. * * @return GetCommitsResponse which contains a list of `Commit`s and the latestTableVersion * tracked by the commit-owner. */ def getCommits( startVersion: Long, endVersion: Long): GetCommitsResponse /** * API to ask the commit-owner to backfill all commits <= given `version`. */ def backfillToVersion(version: Long): Unit } ``` ## Commit Backfills Backfilling is the process of copying the un-backfilled commits i.e. `_delta_log/_commits/..json` to `_delta_log/.json`. With the help of backfilling, the [delta files](#delta-log-entries) are visible even to the filesystem based Delta clients that do not understand `managed-commits`. Backfill also allows the commit-owner to reduce the number of commits it must track internally. Backfill must be sequential. In other words, a commit-owner must ensure that backfill of commit `v-1` is complete before initiating backfill of commit `v`. `commit-owner`s are encouraged to backfill the commits frequently. This has several advantages: 1. Filesystem-based Delta implementations may only understand backfilled commits, and frequent backfill allows them to access the most recent table snapshots. 2. Frequent backfilling minimizes the impact to readers in case the `commit-owner` is unavailable or loses state. 3. Some maintenance operations (such as checkpoints, log compaction, and metadata cleanup) can be performed only on the backfilled part of the table. Refer to the [Maintenance operations on managed-commit tables](#maintenance-operations-on-managed-commit-tables) section for more details. The commit-owner also needs to expose an API to backfill the commits. This will allow clients to ask the commit-owner to backfill the commits if needed in order to do some maintenance operations. Since commit backfills may happen at a later point in time, so the `file modification timestamp` of the backfilled file might be very different than the time of actual commit. For this reason, the `managed-commit` feature depends on another writer feature called [in-commit-timestamps](#TODO-Put-Relevant-Link) to make the commit timestamps more reliable. Refer to [Writer Requirements for Managed Commits](#writer-requirements-for-managed-commits) section for more details. ## Converting an existing filesystem based table to managed-commit table In order for a commit-owner to successfully take over an existing filesystem-based Delta table, the following invariants must hold: - The commit-owner must agree to take ownership of the table, by accepting a proposed commit that would install it. This essentially follows the normal commit protocol, except… - The commit-owner and client must both recognize that the ownership change only officially takes effect when the ownership-change is successfully backfilled. Unlike the backfill of a normal commit, this ownership-change backfill must be atomic because it is also a filesystem-based commit that potentially races with other filesystem-based commit attempts. Assuming the client follows the commit-owner’s protocol for ownership changes, the commit-owner MUST NOT refuse ownership after the backfill succeeds. Otherwise, the table would become permanently unusable, because the advertised commit-owner refuses to ratify the very commits that would repair the table by removing that commit-owner. Thus, the commit-owner and client effectively perform a two-phase commit, where the commit-owner persists its commitment to own the table, and the actual commit point is the PUT-if-absent. Notifying the commit-owner that backfill has completed becomes a post-commit cleanup operation. If the put-if-absent fails (because somebody else gets there first), the commit-owner forgets about the proposed ownership change. Once the backfill succeeds, clients will start contacting the commit-owner for any further commits. Meanwhile, any clients who were already attempting filesystem-based commits will encounter a physical conflict, see the protocol change, and either abort the commit or route it to the new owner. ## Creating a new managed-commit table Conceptually, creating a new managed-commit table is very similar to proposing an ownership change of an existing filesystem-based table that happens to not yet contain any commits. This means that, until commit 0 has been backfilled, there is a risk of multiple clients racing to create the same table with different commit-owners (or to create a filesystem-based table). To avoid such races, Commit-owners are encouraged to use a put-if-absent API (if available) to write the backfilled commit directly (i.e. `_delta_log/00000000000000000000.json`). If such put-if-absent is not available, then it is the responsibility of commit-owners to take whatever measures they deem appropriate to avoid or respond to such races. ## Converting a managed-commit table to filesystem table In order to convert a managed-commit table to a filesystem-based table, the Delta client needs to initiate a commit which tries to remove the commit-owner information from [change-metadata](#change-metadata) and also removes the table feature from the [protocol](#protocol-evolution) action. The commit-owner is not required to give up ownership, and may reject the request. If it chooses to honor such a request, it must: 1. Ensure that all prior commit files are backfilled. 2. Not accept any new commits on the table. 3. Write the commit which removes the ownership. - Either the commit-owner writes the backfilled commit file directly. - Or it writes an unbackfilled commit and ensures that it is backfilled reliably. Until the backfill is done, table will be in unusable state: - the filesystem based delta clients won't be able to write to such table as they still believe that table has managed-commit enabled. - the managed-commit aware delta clients won't be able to write to such table as the commit-owner won't accept any new commits. In such a scenario, they could backfill required commit themselves (preferably using PUT-if-absent) to unblock themselves. ## Reading managed-commit tables With `managed-commits` enabled, a table could have some part of table already backfilled and some part of the table yet-to-be-backfilled. The precise information about what are the valid un-backfilled commits is maintained by the commit-owner. E.g. ``` _delta_log/00000000000000000000.json _delta_log/00000000000000000001.json _delta_log/00000000000000000002.json _delta_log/00000000000000000002.checkpoint.parquet _delta_log/00000000000000000003.json _delta_log/00000000000000000003.00000000000000000005.compacted.json _delta_log/00000000000000000004.json _delta_log/00000000000000000005.json _delta_log/00000000000000000006.json _delta_log/00000000000000000007.json _delta_log/_commits/00000000000000000006.3a0d65cd-4056-49b8-937b-95f9e3ee90e5.json _delta_log/_commits/00000000000000000007.016ae953-37a9-438e-8683-9a9a4a79a395.json _delta_log/_commits/00000000000000000008.7d17ac10-5cc3-401b-bd1a-9c82dd2ea032.json _delta_log/_commits/00000000000000000008.b91807ba-fe18-488c-a15e-c4807dbd2174.json _delta_log/_commits/00000000000000000009.41bf693a-f5b9-4478-9434-af7475d5a9f0.json _delta_log/_commits/00000000000000000010.0f707846-cd18-4e01-b40e-84ee0ae987b0.json _delta_log/_commits/00000000000000000010.7a980438-cb67-4b89-82d2-86f73239b6d6.json ``` Suppose the commit-owner is tracking: ``` { 6 -> "00000000000000000006.3a0d65cd-4056-49b8-937b-95f9e3ee90e5.json", 7 -> "00000000000000000007.016ae953-37a9-438e-8683-9a9a4a79a395.json", 8 -> "00000000000000000008.7d17ac10-5cc3-401b-bd1a-9c82dd2ea032.json", 9 -> "00000000000000000009.41bf693a-f5b9-4478-9434-af7475d5a9f0.json" } ``` Delta clients have two choices to read such tables: 1. Any Delta client can read such table by listing the `_delta_log` directory and reading the delta/checkpoint/log-compaction files. Without contacting the commit owner, they cannot access recent un-backfilled commits in the `_delta_log/_commits` directory, and may construct a stale snapshot. - In the above example, such delta implementation will see version 7 as the latest snapshot. 2. A client can guarantee freshness by additionally requesting the set of recent un-backfilled commits from the commit-owner. - In the above example, a delta implementation could get information about versions 0 through 7 from `_delta_log` directory and get information about un-backfilled commits (v8, v9) from the commit-owner. ## Maintenance operations on managed-commit tables [Checkpoints](#checkpoints-1) and [log compaction files](#log-compaction-files) can only be created for commits in the `_delta_log` directory. In other words, in order to checkpoint version `v` or produce a compacted log file for commit range x <= v <= y, `_delta_log/.json` must exist. Otherwise, filesystem-based readers who encountered the seemingly-extra files might think the table metadata was corrupted. ## Managed Commit Enablement The managed-commit feature is supported and active when: - The table must be on Writer Version 7. - The table has a `protocol` action with `writerFeatures` containing the feature `managedCommit`. - The table has a metadata property `delta.managedCommit.commitOwner` in the [change-metadata](#change-metadata)'s configuration. - The table may have a metadata property `delta.managedCommit.commitOwnerConf` in the [change-metadata](#change-metadata)'s configuration. The value of this property is a json-coded string-to-string map. - A commit-owner can store additional information (e.g. configuration information such as service endpoints) in this field, for use by the commit-owner client (it is opaque to the Delta client). - This field should never include secrets such as auth tokens or credentials, because any reader with access to the table's storage location can see them. Note that a table is in invalid state if the change-metadata contains the `delta.managedCommit.commitOwner` property but the table does not have the `managedCommit` feature in the `protocol` action (or vice versa). E.g. ```json { "metaData":{ "id":"af23c9d7-fff1-4a5a-a2c8-55c59bd782aa", "format":{"provider":"parquet","options":{}}, "schemaString":"...", "partitionColumns":[], "configuration":{ "appendOnly": "true", "delta.managedCommit.commitOwner": "commit-owner-1", "delta.managedCommit.commitOwnerConf": "{\"endpoint\":\"http://sample-url.com/commit\", \"authenticationMode\":\"oauth2\"}" } } } ``` ## Writer Requirements for Managed Commits When supported and active: - The `inCommitTimestamp` table feature must also be supported and active. - Writer must follow the commit-owner's [commit protocol](#commit-protocol) and must not perform filesystem-based commits. - Writer must only create checkpoints or log compaction files for commits in the `_delta_log` directory. - Metadata cleanup must always preserve the newest k >= 1 backfilled commits. ## Reader Requirements for Managed Commits Managed commits is a writer feature. So it doesn't put any restrictions on the reader. - Filesystem-based delta readers which do not understand [managed commits](#managed-commits) may only be able to read the backfilled commits. They may see a stale snapshot of the table if the recent commits are not backfilled. - The [managed commits](#managed-commits) aware delta readers could additionally contact the commit-owner to get the information about the recent un-backfilled commits. This allows them to get the most recent snapshot of the table. ================================================ FILE: protocol_rfcs/template.md ================================================ # Table feature name / meaningful name **Associated Github issue for discussions: https://github.com/delta-io/delta/issues/XXXX** -------- ================================================ FILE: protocol_rfcs/variant-shredding.md ================================================ # Variant Shredding **Associated Github issue for discussions: https://github.com/delta-io/delta/issues/4032** This protocol change adds support for Variant shredding for the Variant data type. Shredding allows Variant data to be be more efficiently stored and queried. -------- > ***New Section after the `Variant Data Type` section*** # Variant Shredding This feature enables support for shredding of the Variant data type, to store and query Variant data more efficiently. Shredding a Variant value is taking paths from the Variant value, and storing them as a typed column in the file. The shredding does not duplicate data, so if a value is stored in the typed column, it is removed from the Variant binary. Storing Variant values as typed columns is faster to access, and enables data skipping with statistics. The `variantShredding` feature depends on the `variantType` feature. To support this feature: - The table must be on Reader Version 3 and Writer Version 7 - The feature `variantType` must exist in the table `protocol`'s `readerFeatures` and `writerFeatures`. - The feature `variantShredding` must exist in the table `protocol`'s `readerFeatures` and `writerFeatures`. ## Shredded Variant data in Parquet Shredded Variant data is stored according to the [Parquet Variant Shredding specification](https://github.com/apache/parquet-format/blob/master/VariantShredding.md) The shredded Variant data written to parquet files is written as a single Parquet struct, with the following fields: Struct field name | Parquet primitive type | Description -|-|- metadata | binary | (required) The binary-encoded Variant metadata, as described in [Parquet Variant binary encoding](https://github.com/apache/parquet-format/blob/master/VariantEncoding.md) value | binary | (optional) The binary-encoded Variant value, as described in [Parquet Variant binary encoding](https://github.com/apache/parquet-format/blob/master/VariantEncoding.md) typed_value | * | (optional) This can be any Parquet type, representing the data stored in the Variant. Details of the shredding scheme is found in the [Variant Shredding specification](https://github.com/apache/parquet-format/blob/master/VariantShredding.md) ## Writer Requirements for Variant Shredding When Variant Shredding is supported (`writerFeatures` field of a table's `protocol` action contains `variantShredding`), writers: - must respect the `delta.enableVariantShredding` table property configuration. If `delta.enableVariantShredding=false`, a column of type `variant` must not be written as a shredded Variant, but as an unshredded Variant. If `delta.enableVariantShredding=true`, the writer can choose to shred a Variant column according to the [Parquet Variant Shredding specification](https://github.com/apache/parquet-format/blob/master/VariantShredding.md) ## Reader Requirements for Variant Shredding When Variant type is supported (`readerFeatures` field of a table's `protocol` action contains `variantShredding`), readers: - must recognize and tolerate a `variant` data type in a Delta schema - must recognize and correctly process a parquet schema that is either unshredded (only `metadata` and `value` struct fields) or shredded (`metadata`, `value`, and `typed_value` struct fields) when reading a Variant data type from file. > ***Update the `Per-file Statistics` section*** > After the description and examples starting from: `Per-column statistics record information for each column in the file and they are encoded, mirroring the schema of the actual data. For example, given the following data schema:` ### Statistics for Variant Columns - The `nullCount` stat for a Variant column is a LONG representing the nullcount for the Variant column itself (nullcount stats are not captured for individual paths within the Variant). - The `minValues` and `maxValues` stats for a Variant column are Variant objects, where the object keys are [normalized JSON path expressions](https://www.rfc-editor.org/rfc/rfc9535.html#name-normalized-paths), and the object values are the primitive Variant values representing the lower and upper bound for that field. - In JSON, the `minValues` and `maxValues` stats for a Variant column are [binary-encoded](https://github.com/apache/parquet-format/blob/master/VariantEncoding.md) Variant values, concatenating the `metadata` and `value`, and serialized to strings using [z85](https://rfc.zeromq.org/spec/32/) encoding (see example below). - In Parquet, the `minValues` and `maxValues` stats for a Variant column are Parquet Variant columns, following the Parquet Variant [encoding](https://github.com/apache/parquet-format/blob/master/VariantEncoding.md) and [shredding](https://github.com/apache/parquet-format/blob/master/VariantShredding.md) specifications. - In Parquet, the Variant `minValues` and `maxValues` stats are allowed to be shredded, but it is not required. - Each path in the Variant `minValues` (`maxValues`) value is the independently computed min (max) stat for the corresponding path in the file's Variant data, so e.g. `minValues.v:a` and `minValues.v:b` could come from different rows in the file. - Min/max stats may only be written for primitive (leaf) values, packed into a Variant representation. - Min/max stats may only be written for a path if that path has the same data type in every row of the data file. - The paths and types inside `minValues` and `maxValues` must be the same within any one file, but can vary from file to file. - Subject to the above constraints, the writer of a given file determines which Variant leaf paths (if any) to emit statistics for. For a table with a single Variant column (`varCol: variant`) in its data schema, example statistics in JSON would look like: ``` "stats": { "nullCount": { "varCol": 2 } "minValues": { "varCol": "0S&u501fk+ze0(tB98CpzF6vU0rJl95HpNdvjbtatpi(cu0wW^cTu" }, "maxValues": { "varCol": "0S&u500&]LC42A9vqZe}wb#-i1}-a+cT!xdbWhT9cTx}7v<+K" } } ``` The corresponding human-readable form is: ``` "stats": { "nullCount": { "varCol": 2 } "minValues": { "varCol": { "$['a']" : "min-string", "$['b']['c']" : 1 } }, "maxValues": { "varCol": { "$['a']" : "variant", "$['b']['c']" : 100 } } } ``` ================================================ FILE: python/README.md ================================================ # Delta Lake [Delta Lake](https://delta.io) is an open source storage layer that brings reliability to data lakes. Delta Lake provides ACID transactions, scalable metadata handling, and unifies streaming and batch data processing. Delta Lake runs on top of your existing data lake and is fully compatible with Apache Spark APIs. This PyPi package contains the Python APIs for using Delta Lake with Apache Spark. ## Installation and usage 1. Install using `pip install delta-spark` 2. To use the Delta Lake with Apache Spark, you have to set additional configurations when creating the SparkSession. See the online [project web page](https://docs.delta.io/latest/delta-intro.html) for details. ## Documentation This README file only contains basic information related to pip installed Delta Lake. You can find the full documentation on the [project web page](https://docs.delta.io/latest/delta-intro.html) ================================================ FILE: python/delta/__init__.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from delta.tables import DeltaTable from delta.pip_utils import configure_spark_with_delta_pip from delta.version import __version__ __all__ = ['DeltaTable', 'configure_spark_with_delta_pip', '__version__'] ================================================ FILE: python/delta/_typing.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from typing import Dict, Optional, Union from pyspark.sql.column import Column ExpressionOrColumn = Union[str, Column] OptionalExpressionOrColumn = Optional[ExpressionOrColumn] ColumnMapping = Dict[str, ExpressionOrColumn] OptionalColumnMapping = Optional[ColumnMapping] ================================================ FILE: python/delta/connect/__init__.py ================================================ # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from delta.connect.tables import DeltaTable __all__ = ['DeltaTable'] ================================================ FILE: python/delta/connect/_typing.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from typing import Dict, Optional, Union from pyspark.sql.connect.column import Column ExpressionOrColumn = Union[str, Column] OptionalExpressionOrColumn = Optional[ExpressionOrColumn] ColumnMapping = Dict[str, ExpressionOrColumn] OptionalColumnMapping = Optional[ColumnMapping] ================================================ FILE: python/delta/connect/exceptions.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import json from typing import TYPE_CHECKING from pyspark.errors.exceptions.connect import SparkConnectException from delta.exceptions.base import ( DeltaConcurrentModificationException as BaseDeltaConcurrentModificationException, ConcurrentWriteException as BaseConcurrentWriteException, MetadataChangedException as BaseMetadataChangedException, ProtocolChangedException as BaseProtocolChangedException, ConcurrentAppendException as BaseConcurrentAppendException, ConcurrentDeleteReadException as BaseConcurrentDeleteReadException, ConcurrentDeleteDeleteException as BaseConcurrentDeleteDeleteException, ConcurrentTransactionException as BaseConcurrentTransactionException, ) if TYPE_CHECKING: from google.rpc.error_details_pb2 import ErrorInfo class DeltaConcurrentModificationException(SparkConnectException, BaseDeltaConcurrentModificationException): """ The basic class for all Delta commit conflict exceptions. .. versionadded:: 4.0 .. note:: Evolving """ class ConcurrentWriteException(SparkConnectException, BaseConcurrentWriteException): """ Thrown when a concurrent transaction has written data after the current transaction read the table. .. versionadded:: 4.0 .. note:: Evolving """ class MetadataChangedException(SparkConnectException, BaseMetadataChangedException): """ Thrown when the metadata of the Delta table has changed between the time of read and the time of commit. .. versionadded:: 4.0 .. note:: Evolving """ class ProtocolChangedException(SparkConnectException, BaseProtocolChangedException): """ Thrown when the protocol version has changed between the time of read and the time of commit. .. versionadded:: 4.0 .. note:: Evolving """ class ConcurrentAppendException(SparkConnectException, BaseConcurrentAppendException): """ Thrown when files are added that would have been read by the current transaction. .. versionadded:: 4.0 .. note:: Evolving """ class ConcurrentDeleteReadException(SparkConnectException, BaseConcurrentDeleteReadException): """ Thrown when the current transaction reads data that was deleted by a concurrent transaction. .. versionadded:: 4.0 .. note:: Evolving """ class ConcurrentDeleteDeleteException(SparkConnectException, BaseConcurrentDeleteDeleteException): """ Thrown when the current transaction deletes data that was deleted by a concurrent transaction. .. versionadded:: 4.0 .. note:: Evolving """ class ConcurrentTransactionException(SparkConnectException, BaseConcurrentTransactionException): """ Thrown when concurrent transaction both attempt to update the same idempotent transaction. .. versionadded:: 4.0 .. note:: Evolving """ def _convert_delta_exception(info: "ErrorInfo", message: str): classes = [] if "classes" in info.metadata: classes = json.loads(info.metadata["classes"]) if "io.delta.exceptions.ConcurrentWriteException" in classes: return ConcurrentWriteException(message) if "io.delta.exceptions.MetadataChangedException" in classes: return MetadataChangedException(message) if "io.delta.exceptions.ProtocolChangedException" in classes: return ProtocolChangedException(message) if "io.delta.exceptions.ConcurrentAppendException" in classes: return ConcurrentAppendException(message) if "io.delta.exceptions.ConcurrentDeleteReadException" in classes: return ConcurrentDeleteReadException(message) if "io.delta.exceptions.ConcurrentDeleteDeleteException" in classes: return ConcurrentDeleteDeleteException(message) if "io.delta.exceptions.ConcurrentTransactionException" in classes: return ConcurrentTransactionException(message) if "io.delta.exceptions.DeltaConcurrentModificationException" in classes: return DeltaConcurrentModificationException(message) return None ================================================ FILE: python/delta/connect/plan.py ================================================ # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from typing import cast, Dict, List, Optional, Union import delta.connect.proto as proto from pyspark.sql.connect.client import SparkConnectClient from pyspark.sql.connect.column import Column from pyspark.sql.connect.plan import LogicalPlan import pyspark.sql.connect.proto as spark_proto from pyspark.sql.connect.types import pyspark_types_to_proto_types from pyspark.sql.types import StructType class DeltaLogicalPlan(LogicalPlan): def __init__(self, child: Optional[LogicalPlan]) -> None: super().__init__(child) def plan(self, session: SparkConnectClient) -> spark_proto.Relation: plan = self._create_proto_relation() plan.extension.Pack(self.to_delta_relation(session)) return plan def to_delta_relation(self, session: SparkConnectClient) -> proto.DeltaRelation: ... def command(self, session: SparkConnectClient) -> spark_proto.Command: command = spark_proto.Command() command.extension.Pack(self.to_delta_command(session)) return command def to_delta_command(self, session: SparkConnectClient) -> proto.DeltaCommand: ... class DeltaScan(DeltaLogicalPlan): def __init__(self, table: proto.DeltaTable) -> None: super().__init__(None) self._table = table def to_delta_relation(self, client: SparkConnectClient) -> proto.DeltaRelation: relation = proto.DeltaRelation() relation.scan.table.CopyFrom(self._table) return relation class Generate(DeltaLogicalPlan): def __init__( self, table: proto.DeltaTable, mode: str ) -> None: super().__init__(None) self._mode = mode self._table = table def to_delta_command(self, client: SparkConnectClient) -> proto.DeltaCommand: command = proto.DeltaCommand() command.generate.table.CopyFrom(self._table) command.generate.mode = self._mode return command class DeleteFromTable(DeltaLogicalPlan): def __init__(self, target: Optional[LogicalPlan], condition: Optional[Column]) -> None: super().__init__(target) self._target = cast(LogicalPlan, target) self._condition = condition def to_delta_relation(self, session: SparkConnectClient) -> proto.DeltaRelation: relation = proto.DeltaRelation() relation.delete_from_table.target.CopyFrom(self._target.plan(session)) if self._condition is not None: relation.delete_from_table.condition.CopyFrom(self._condition.to_plan(session)) return relation class Assignment: def __init__(self, field: Column, value: Column) -> None: self._field = field self._value = value def to_proto(self, session: SparkConnectClient) -> proto.Assignment: assignment = proto.Assignment() assignment.field.CopyFrom(self._field.to_plan(session)) assignment.value.CopyFrom(self._value.to_plan(session)) return assignment class UpdateTable(DeltaLogicalPlan): def __init__( self, target: Optional[LogicalPlan], condition: Optional[Column], assignments: List[Assignment], ) -> None: super().__init__(target) self._target = cast(LogicalPlan, target) self._condition = condition self._assignments = assignments def to_delta_relation(self, session: SparkConnectClient) -> proto.DeltaRelation: relation = proto.DeltaRelation() relation.update_table.target.CopyFrom(self._target.plan(session)) if self._condition is not None: relation.update_table.condition.CopyFrom(self._condition.to_plan(session)) relation.update_table.assignments.extend( [assignment.to_proto(session) for assignment in self._assignments] ) return relation class MergeAction(object): def __init__(self, condition: Optional[Column]) -> None: self._condition = condition def to_proto(self, session: SparkConnectClient) -> proto.MergeIntoTable.Action: action = proto.MergeIntoTable.Action() if self._condition is not None: action.condition.CopyFrom(self._condition.to_plan(session)) return action class UpdateAction(MergeAction): def __init__( self, condition: Optional[Column], assignments: List[Assignment], ) -> None: super().__init__(condition) self._assignments = assignments def to_proto(self, session: SparkConnectClient) -> proto.MergeIntoTable.Action: action = super().to_proto(session) action.update_action.assignments.extend( [assignment.to_proto(session) for assignment in self._assignments] ) return action class UpdateStarAction(MergeAction): def __init__(self, condition: Optional[Column]) -> None: super().__init__(condition) def to_proto(self, session: SparkConnectClient) -> proto.MergeIntoTable.Action: action = super().to_proto(session) action.update_star_action.SetInParent() return action class DeleteAction(MergeAction): def __init__(self, condition: Optional[Column]) -> None: super().__init__(condition) def to_proto(self, session: SparkConnectClient) -> proto.MergeIntoTable.Action: action = super().to_proto(session) action.delete_action.SetInParent() return action class InsertAction(MergeAction): def __init__( self, condition: Optional[Column], assignments: List[Assignment], ) -> None: super().__init__(condition) self._assignments = assignments def to_proto(self, session: SparkConnectClient) -> proto.MergeIntoTable.Action: action = super().to_proto(session) action.insert_action.assignments.extend( [assignment.to_proto(session) for assignment in self._assignments] ) return action class InsertStarAction(MergeAction): def __init__(self, condition: Optional[Column]) -> None: super().__init__(condition) def to_proto(self, session: SparkConnectClient) -> proto.MergeIntoTable.Action: action = super().to_proto(session) action.insert_star_action.SetInParent() return action class MergeIntoTable(DeltaLogicalPlan): def __init__( self, target: Optional[LogicalPlan], source: LogicalPlan, condition: Column, matched_actions: List[MergeAction], not_matched_actions: List[MergeAction], not_matched_by_source_actions: List[MergeAction], with_schema_evolution: Optional[bool] ) -> None: super().__init__(target) self._target = cast(LogicalPlan, target) self._source = source self._condition = condition self._matched_actions = matched_actions self._not_matched_actions = not_matched_actions self._not_matched_by_source_actions = not_matched_by_source_actions self._with_schema_evolution = with_schema_evolution or False def to_delta_relation(self, session: SparkConnectClient) -> proto.DeltaRelation: relation = proto.DeltaRelation() relation.merge_into_table.target.CopyFrom(self._target.plan(session)) relation.merge_into_table.source.CopyFrom(self._source.plan(session)) relation.merge_into_table.condition.CopyFrom(self._condition.to_plan(session)) relation.merge_into_table.matched_actions.extend( [action.to_proto(session) for action in self._matched_actions] ) relation.merge_into_table.not_matched_actions.extend( [action.to_proto(session) for action in self._not_matched_actions] ) relation.merge_into_table.not_matched_by_source_actions.extend( [action.to_proto(session) for action in self._not_matched_by_source_actions] ) relation.merge_into_table.with_schema_evolution = self._with_schema_evolution return relation class Vacuum(DeltaLogicalPlan): def __init__( self, table: proto.DeltaTable, retentionHours: Optional[float] ) -> None: super().__init__(None) self._table = table self._retentionHours = retentionHours def to_delta_command(self, client: SparkConnectClient) -> proto.DeltaCommand: command = proto.DeltaCommand() command.vacuum_table.table.CopyFrom(self._table) if self._retentionHours is not None: command.vacuum_table.retention_hours = self._retentionHours return command class DescribeHistory(DeltaLogicalPlan): def __init__(self, table: proto.DeltaTable) -> None: super().__init__(None) self._table = table def to_delta_relation(self, session: SparkConnectClient) -> proto.DeltaRelation: relation = proto.DeltaRelation() relation.describe_history.table.CopyFrom(self._table) return relation class DescribeDetail(DeltaLogicalPlan): def __init__(self, table: proto.DeltaTable) -> None: super().__init__(None) self._table = table def to_delta_relation(self, client: SparkConnectClient) -> proto.DeltaRelation: relation = proto.DeltaRelation() relation.describe_detail.table.CopyFrom(self._table) return relation class ConvertToDelta(DeltaLogicalPlan): def __init__( self, identifier: str, partitionSchema: Optional[Union[str, StructType]] ) -> None: super().__init__(None) self._identifier = identifier self._partitionSchema = partitionSchema def to_delta_relation(self, client: SparkConnectClient) -> proto.DeltaRelation: relation = proto.DeltaRelation() relation.convert_to_delta.identifier = self._identifier if self._partitionSchema is not None: if isinstance(self._partitionSchema, str): relation.convert_to_delta.partition_schema_string = self._partitionSchema if isinstance(self._partitionSchema, StructType): relation.convert_to_delta.partition_schema_struct.CopyFrom( pyspark_types_to_proto_types(self._partitionSchema) ) return relation class IsDeltaTable(DeltaLogicalPlan): def __init__(self, path: str): super().__init__(None) self._path = path def to_delta_relation(self, session: SparkConnectClient) -> proto.DeltaRelation: relation = proto.DeltaRelation() relation.is_delta_table.path = self._path return relation class CreateDeltaTable(DeltaLogicalPlan): def __init__( self, mode: proto.CreateDeltaTable.Mode, tableName: Optional[str], location: Optional[str], comment: Optional[str], columns: List[proto.CreateDeltaTable.Column], partitioningColumns: List[str], properties: Dict[str, str], clusteringColumns: List[str] ) -> None: super().__init__(None) self._mode = mode self._tableName = tableName self._location = location self._comment = comment self._columns = columns self._partitioningColumns = partitioningColumns self._clusteringColumns = clusteringColumns self._properties = properties def to_delta_command(self, client: SparkConnectClient) -> proto.DeltaCommand: command = proto.DeltaCommand() command.create_delta_table.mode = self._mode if self._tableName is not None: command.create_delta_table.table_name = self._tableName if self._location is not None: command.create_delta_table.location = self._location if self._comment is not None: command.create_delta_table.comment = self._comment command.create_delta_table.columns.extend(self._columns) command.create_delta_table.partitioning_columns.extend(self._partitioningColumns) command.create_delta_table.clustering_columns.extend(self._clusteringColumns) for k, v in self._properties.items(): command.create_delta_table.properties[k] = v return command class UpgradeTableProtocol(DeltaLogicalPlan): def __init__( self, table: proto.DeltaTable, readerVersion: int, writerVersion: int ) -> None: super().__init__(None) self._table = table self._readerVersion = readerVersion self._writerVersion = writerVersion def to_delta_command(self, client: SparkConnectClient) -> proto.DeltaCommand: command = proto.DeltaCommand() command.upgrade_table_protocol.table.CopyFrom(self._table) command.upgrade_table_protocol.reader_version = self._readerVersion command.upgrade_table_protocol.writer_version = self._writerVersion return command class AddFeatureSupport(DeltaLogicalPlan): def __init__( self, table: proto.DeltaTable, featureName: str ) -> None: super().__init__(None) self._table = table self._featureName = featureName def to_delta_command(self, client: SparkConnectClient) -> proto.DeltaCommand: command = proto.DeltaCommand() command.add_feature_support.table.CopyFrom(self._table) command.add_feature_support.feature_name = self._featureName return command class DropFeatureSupport(DeltaLogicalPlan): def __init__( self, table: proto.DeltaTable, featureName: str, truncateHistory: Optional[bool] ) -> None: super().__init__(None) self._table = table self._featureName = featureName self._truncateHistory = truncateHistory def to_delta_command(self, client: SparkConnectClient) -> proto.DeltaCommand: command = proto.DeltaCommand() command.drop_feature_support.table.CopyFrom(self._table) command.drop_feature_support.feature_name = self._featureName if self._truncateHistory is not None: command.drop_feature_support.truncate_history = self._truncateHistory return command class RestoreTable(DeltaLogicalPlan): def __init__( self, table: proto.DeltaTable, version: Optional[int] = None, timestamp: Optional[str] = None ) -> None: super().__init__(None) self._table = table self._version = version self._timestamp = timestamp def to_delta_relation(self, client: SparkConnectClient) -> proto.DeltaRelation: relation = proto.DeltaRelation() relation.restore_table.table.CopyFrom(self._table) if self._version is not None: relation.restore_table.version = self._version if self._timestamp is not None: relation.restore_table.timestamp = self._timestamp return relation class OptimizeTable(DeltaLogicalPlan): def __init__( self, table: proto.DeltaTable, partitionFilters: List[str], zOrderCols: List[str] ) -> None: super().__init__(None) self._table = table self._partitionFilters = partitionFilters self._zOrderCols = zOrderCols def to_delta_relation(self, client: SparkConnectClient) -> proto.DeltaRelation: relation = proto.DeltaRelation() relation.optimize_table.table.CopyFrom(self._table) relation.optimize_table.partition_filters.extend(self._partitionFilters) relation.optimize_table.zorder_columns.extend(self._zOrderCols) return relation class CloneTable(DeltaLogicalPlan): def __init__( self, table: proto.DeltaTable, target: str, isShallow: bool, replace: bool, properties: Optional[Dict[str, str]], version: Optional[int] = None, timestamp: Optional[str] = None, ) -> None: super().__init__(None) self._table = table self._target = target self._isShallow = isShallow self._replace = replace self._properties = properties or {} self._version = version self._timestamp = timestamp def to_delta_command(self, client: SparkConnectClient) -> proto.DeltaCommand: command = proto.DeltaCommand() command.clone_table.table.CopyFrom(self._table) command.clone_table.target = self._target command.clone_table.is_shallow = self._isShallow command.clone_table.replace = self._replace for k, v in self._properties.items(): command.clone_table.properties[k] = v if self._version is not None: command.clone_table.version = self._version if self._timestamp is not None: command.clone_table.timestamp = self._timestamp return command ================================================ FILE: python/delta/connect/proto/__init__.py ================================================ # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from delta.connect.proto.base_pb2 import * from delta.connect.proto.commands_pb2 import * from delta.connect.proto.relations_pb2 import * ================================================ FILE: python/delta/connect/proto/base_pb2.py ================================================ # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: delta/connect/base.proto """Generated protocol buffer code.""" from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( b'\n\x18\x64\x65lta/connect/base.proto\x12\rdelta.connect"\xad\x02\n\nDeltaTable\x12\x34\n\x04path\x18\x01 \x01(\x0b\x32\x1e.delta.connect.DeltaTable.PathH\x00R\x04path\x12-\n\x12table_or_view_name\x18\x02 \x01(\tH\x00R\x0ftableOrViewName\x1a\xaa\x01\n\x04Path\x12\x12\n\x04path\x18\x01 \x01(\tR\x04path\x12O\n\x0bhadoop_conf\x18\x02 \x03(\x0b\x32..delta.connect.DeltaTable.Path.HadoopConfEntryR\nhadoopConf\x1a=\n\x0fHadoopConfEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\r\n\x0b\x61\x63\x63\x65ss_typeB\x1a\n\x16io.delta.connect.protoP\x01\x62\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "delta.connect.proto.base_pb2", globals()) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b"\n\026io.delta.connect.protoP\001" _DELTATABLE_PATH_HADOOPCONFENTRY._options = None _DELTATABLE_PATH_HADOOPCONFENTRY._serialized_options = b"8\001" _DELTATABLE._serialized_start = 44 _DELTATABLE._serialized_end = 345 _DELTATABLE_PATH._serialized_start = 160 _DELTATABLE_PATH._serialized_end = 330 _DELTATABLE_PATH_HADOOPCONFENTRY._serialized_start = 269 _DELTATABLE_PATH_HADOOPCONFENTRY._serialized_end = 330 # @@protoc_insertion_point(module_scope) ================================================ FILE: python/delta/connect/proto/base_pb2.pyi ================================================ # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """ @generated by mypy-protobuf. Do not edit manually! isort:skip_file Copyright (2024) The Delta Lake Project Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ import builtins import collections.abc import google.protobuf.descriptor import google.protobuf.internal.containers import google.protobuf.message import sys if sys.version_info >= (3, 8): import typing as typing_extensions else: import typing_extensions DESCRIPTOR: google.protobuf.descriptor.FileDescriptor class DeltaTable(google.protobuf.message.Message): """Information required to access a Delta table either by name or by path.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor class Path(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor class HadoopConfEntry(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor KEY_FIELD_NUMBER: builtins.int VALUE_FIELD_NUMBER: builtins.int key: builtins.str value: builtins.str def __init__( self, *, key: builtins.str = ..., value: builtins.str = ..., ) -> None: ... def ClearField( self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"] ) -> None: ... PATH_FIELD_NUMBER: builtins.int HADOOP_CONF_FIELD_NUMBER: builtins.int path: builtins.str """(Required) Path to the Delta table.""" @property def hadoop_conf( self, ) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: """(Optional) Hadoop configuration used to access the file system.""" def __init__( self, *, path: builtins.str = ..., hadoop_conf: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., ) -> None: ... def ClearField( self, field_name: typing_extensions.Literal["hadoop_conf", b"hadoop_conf", "path", b"path"], ) -> None: ... PATH_FIELD_NUMBER: builtins.int TABLE_OR_VIEW_NAME_FIELD_NUMBER: builtins.int @property def path(self) -> global___DeltaTable.Path: ... table_or_view_name: builtins.str def __init__( self, *, path: global___DeltaTable.Path | None = ..., table_or_view_name: builtins.str = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal[ "access_type", b"access_type", "path", b"path", "table_or_view_name", b"table_or_view_name", ], ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ "access_type", b"access_type", "path", b"path", "table_or_view_name", b"table_or_view_name", ], ) -> None: ... def WhichOneof( self, oneof_group: typing_extensions.Literal["access_type", b"access_type"] ) -> typing_extensions.Literal["path", "table_or_view_name"] | None: ... global___DeltaTable = DeltaTable ================================================ FILE: python/delta/connect/proto/commands_pb2.py ================================================ # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: delta/connect/commands.proto """Generated protocol buffer code.""" from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() from delta.connect.proto import base_pb2 as delta_dot_connect_dot_base__pb2 from pyspark.sql.connect.proto import types_pb2 as spark_dot_connect_dot_types__pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( b'\n\x1c\x64\x65lta/connect/commands.proto\x12\rdelta.connect\x1a\x18\x64\x65lta/connect/base.proto\x1a\x19spark/connect/types.proto"\xad\x04\n\x0c\x44\x65ltaCommand\x12<\n\x0b\x63lone_table\x18\x01 \x01(\x0b\x32\x19.delta.connect.CloneTableH\x00R\ncloneTable\x12?\n\x0cvacuum_table\x18\x02 \x01(\x0b\x32\x1a.delta.connect.VacuumTableH\x00R\x0bvacuumTable\x12[\n\x16upgrade_table_protocol\x18\x03 \x01(\x0b\x32#.delta.connect.UpgradeTableProtocolH\x00R\x14upgradeTableProtocol\x12\x35\n\x08generate\x18\x04 \x01(\x0b\x32\x17.delta.connect.GenerateH\x00R\x08generate\x12O\n\x12\x63reate_delta_table\x18\x05 \x01(\x0b\x32\x1f.delta.connect.CreateDeltaTableH\x00R\x10\x63reateDeltaTable\x12R\n\x13\x61\x64\x64_feature_support\x18\x06 \x01(\x0b\x32 .delta.connect.AddFeatureSupportH\x00R\x11\x61\x64\x64\x46\x65\x61tureSupport\x12U\n\x14\x64rop_feature_support\x18\x07 \x01(\x0b\x32!.delta.connect.DropFeatureSupportH\x00R\x12\x64ropFeatureSupportB\x0e\n\x0c\x63ommand_type"\xec\x02\n\nCloneTable\x12/\n\x05table\x18\x01 \x01(\x0b\x32\x19.delta.connect.DeltaTableR\x05table\x12\x16\n\x06target\x18\x02 \x01(\tR\x06target\x12\x1a\n\x07version\x18\x03 \x01(\x05H\x00R\x07version\x12\x1e\n\ttimestamp\x18\x04 \x01(\tH\x00R\ttimestamp\x12\x1d\n\nis_shallow\x18\x05 \x01(\x08R\tisShallow\x12\x18\n\x07replace\x18\x06 \x01(\x08R\x07replace\x12I\n\nproperties\x18\x07 \x03(\x0b\x32).delta.connect.CloneTable.PropertiesEntryR\nproperties\x1a=\n\x0fPropertiesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x16\n\x14version_or_timestamp"\x80\x01\n\x0bVacuumTable\x12/\n\x05table\x18\x01 \x01(\x0b\x32\x19.delta.connect.DeltaTableR\x05table\x12,\n\x0fretention_hours\x18\x02 \x01(\x01H\x00R\x0eretentionHours\x88\x01\x01\x42\x12\n\x10_retention_hours"\x95\x01\n\x14UpgradeTableProtocol\x12/\n\x05table\x18\x01 \x01(\x0b\x32\x19.delta.connect.DeltaTableR\x05table\x12%\n\x0ereader_version\x18\x02 \x01(\x05R\rreaderVersion\x12%\n\x0ewriter_version\x18\x03 \x01(\x05R\rwriterVersion"O\n\x08Generate\x12/\n\x05table\x18\x01 \x01(\x0b\x32\x19.delta.connect.DeltaTableR\x05table\x12\x12\n\x04mode\x18\x02 \x01(\tR\x04mode"\xd0\x08\n\x10\x43reateDeltaTable\x12\x38\n\x04mode\x18\x01 \x01(\x0e\x32$.delta.connect.CreateDeltaTable.ModeR\x04mode\x12"\n\ntable_name\x18\x02 \x01(\tH\x00R\ttableName\x88\x01\x01\x12\x1f\n\x08location\x18\x03 \x01(\tH\x01R\x08location\x88\x01\x01\x12\x1d\n\x07\x63omment\x18\x04 \x01(\tH\x02R\x07\x63omment\x88\x01\x01\x12@\n\x07\x63olumns\x18\x05 \x03(\x0b\x32&.delta.connect.CreateDeltaTable.ColumnR\x07\x63olumns\x12\x31\n\x14partitioning_columns\x18\x06 \x03(\tR\x13partitioningColumns\x12O\n\nproperties\x18\x07 \x03(\x0b\x32/.delta.connect.CreateDeltaTable.PropertiesEntryR\nproperties\x12-\n\x12\x63lustering_columns\x18\x08 \x03(\tR\x11\x63lusteringColumns\x1a\xc5\x03\n\x06\x43olumn\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x34\n\tdata_type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x08\x64\x61taType\x12\x1a\n\x08nullable\x18\x03 \x01(\x08R\x08nullable\x12\x33\n\x13generated_always_as\x18\x04 \x01(\tH\x00R\x11generatedAlwaysAs\x88\x01\x01\x12\x1d\n\x07\x63omment\x18\x05 \x01(\tH\x01R\x07\x63omment\x88\x01\x01\x12]\n\ridentity_info\x18\x06 \x01(\x0b\x32\x33.delta.connect.CreateDeltaTable.Column.IdentityInfoH\x02R\x0cidentityInfo\x88\x01\x01\x1al\n\x0cIdentityInfo\x12\x14\n\x05start\x18\x01 \x01(\x03R\x05start\x12\x12\n\x04step\x18\x02 \x01(\x03R\x04step\x12\x32\n\x15\x61llow_explicit_insert\x18\x03 \x01(\x08R\x13\x61llowExplicitInsertB\x16\n\x14_generated_always_asB\n\n\x08_commentB\x10\n\x0e_identity_info\x1a=\n\x0fPropertiesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01"z\n\x04Mode\x12\x14\n\x10MODE_UNSPECIFIED\x10\x00\x12\x0f\n\x0bMODE_CREATE\x10\x01\x12\x1d\n\x19MODE_CREATE_IF_NOT_EXISTS\x10\x02\x12\x10\n\x0cMODE_REPLACE\x10\x03\x12\x1a\n\x16MODE_CREATE_OR_REPLACE\x10\x04\x42\r\n\x0b_table_nameB\x0b\n\t_locationB\n\n\x08_comment"g\n\x11\x41\x64\x64\x46\x65\x61tureSupport\x12/\n\x05table\x18\x01 \x01(\x0b\x32\x19.delta.connect.DeltaTableR\x05table\x12!\n\x0c\x66\x65\x61ture_name\x18\x02 \x01(\tR\x0b\x66\x65\x61tureName"\xad\x01\n\x12\x44ropFeatureSupport\x12/\n\x05table\x18\x01 \x01(\x0b\x32\x19.delta.connect.DeltaTableR\x05table\x12!\n\x0c\x66\x65\x61ture_name\x18\x02 \x01(\tR\x0b\x66\x65\x61tureName\x12.\n\x10truncate_history\x18\x03 \x01(\x08H\x00R\x0ftruncateHistory\x88\x01\x01\x42\x13\n\x11_truncate_historyB\x1a\n\x16io.delta.connect.protoP\x01\x62\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "delta.connect.proto.commands_pb2", globals()) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b"\n\026io.delta.connect.protoP\001" _CLONETABLE_PROPERTIESENTRY._options = None _CLONETABLE_PROPERTIESENTRY._serialized_options = b"8\001" _CREATEDELTATABLE_PROPERTIESENTRY._options = None _CREATEDELTATABLE_PROPERTIESENTRY._serialized_options = b"8\001" _DELTACOMMAND._serialized_start = 101 _DELTACOMMAND._serialized_end = 658 _CLONETABLE._serialized_start = 661 _CLONETABLE._serialized_end = 1025 _CLONETABLE_PROPERTIESENTRY._serialized_start = 940 _CLONETABLE_PROPERTIESENTRY._serialized_end = 1001 _VACUUMTABLE._serialized_start = 1028 _VACUUMTABLE._serialized_end = 1156 _UPGRADETABLEPROTOCOL._serialized_start = 1159 _UPGRADETABLEPROTOCOL._serialized_end = 1308 _GENERATE._serialized_start = 1310 _GENERATE._serialized_end = 1389 _CREATEDELTATABLE._serialized_start = 1392 _CREATEDELTATABLE._serialized_end = 2496 _CREATEDELTATABLE_COLUMN._serialized_start = 1816 _CREATEDELTATABLE_COLUMN._serialized_end = 2269 _CREATEDELTATABLE_COLUMN_IDENTITYINFO._serialized_start = 2107 _CREATEDELTATABLE_COLUMN_IDENTITYINFO._serialized_end = 2215 _CREATEDELTATABLE_PROPERTIESENTRY._serialized_start = 940 _CREATEDELTATABLE_PROPERTIESENTRY._serialized_end = 1001 _CREATEDELTATABLE_MODE._serialized_start = 2334 _CREATEDELTATABLE_MODE._serialized_end = 2456 _ADDFEATURESUPPORT._serialized_start = 2498 _ADDFEATURESUPPORT._serialized_end = 2601 _DROPFEATURESUPPORT._serialized_start = 2604 _DROPFEATURESUPPORT._serialized_end = 2777 # @@protoc_insertion_point(module_scope) ================================================ FILE: python/delta/connect/proto/commands_pb2.pyi ================================================ # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """ @generated by mypy-protobuf. Do not edit manually! isort:skip_file Copyright (2024) The Delta Lake Project Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ import builtins import collections.abc import delta.connect.proto.proto.base_pb2 import google.protobuf.descriptor import google.protobuf.internal.containers import google.protobuf.internal.enum_type_wrapper import google.protobuf.message import pyspark.sql.connect.proto.types_pb2 import sys import typing if sys.version_info >= (3, 10): import typing as typing_extensions else: import typing_extensions DESCRIPTOR: google.protobuf.descriptor.FileDescriptor class DeltaCommand(google.protobuf.message.Message): """Message to hold all command extensions in Delta Connect.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor CLONE_TABLE_FIELD_NUMBER: builtins.int VACUUM_TABLE_FIELD_NUMBER: builtins.int UPGRADE_TABLE_PROTOCOL_FIELD_NUMBER: builtins.int GENERATE_FIELD_NUMBER: builtins.int CREATE_DELTA_TABLE_FIELD_NUMBER: builtins.int ADD_FEATURE_SUPPORT_FIELD_NUMBER: builtins.int DROP_FEATURE_SUPPORT_FIELD_NUMBER: builtins.int @property def clone_table(self) -> global___CloneTable: ... @property def vacuum_table(self) -> global___VacuumTable: ... @property def upgrade_table_protocol(self) -> global___UpgradeTableProtocol: ... @property def generate(self) -> global___Generate: ... @property def create_delta_table(self) -> global___CreateDeltaTable: ... @property def add_feature_support(self) -> global___AddFeatureSupport: ... @property def drop_feature_support(self) -> global___DropFeatureSupport: ... def __init__( self, *, clone_table: global___CloneTable | None = ..., vacuum_table: global___VacuumTable | None = ..., upgrade_table_protocol: global___UpgradeTableProtocol | None = ..., generate: global___Generate | None = ..., create_delta_table: global___CreateDeltaTable | None = ..., add_feature_support: global___AddFeatureSupport | None = ..., drop_feature_support: global___DropFeatureSupport | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal[ "add_feature_support", b"add_feature_support", "clone_table", b"clone_table", "command_type", b"command_type", "create_delta_table", b"create_delta_table", "drop_feature_support", b"drop_feature_support", "generate", b"generate", "upgrade_table_protocol", b"upgrade_table_protocol", "vacuum_table", b"vacuum_table", ], ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ "add_feature_support", b"add_feature_support", "clone_table", b"clone_table", "command_type", b"command_type", "create_delta_table", b"create_delta_table", "drop_feature_support", b"drop_feature_support", "generate", b"generate", "upgrade_table_protocol", b"upgrade_table_protocol", "vacuum_table", b"vacuum_table", ], ) -> None: ... def WhichOneof( self, oneof_group: typing_extensions.Literal["command_type", b"command_type"] ) -> ( typing_extensions.Literal[ "clone_table", "vacuum_table", "upgrade_table_protocol", "generate", "create_delta_table", "add_feature_support", "drop_feature_support", ] | None ): ... global___DeltaCommand = DeltaCommand class CloneTable(google.protobuf.message.Message): """Command that creates a copy of a DeltaTable in the specified target location.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor class PropertiesEntry(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor KEY_FIELD_NUMBER: builtins.int VALUE_FIELD_NUMBER: builtins.int key: builtins.str value: builtins.str def __init__( self, *, key: builtins.str = ..., value: builtins.str = ..., ) -> None: ... def ClearField( self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"] ) -> None: ... TABLE_FIELD_NUMBER: builtins.int TARGET_FIELD_NUMBER: builtins.int VERSION_FIELD_NUMBER: builtins.int TIMESTAMP_FIELD_NUMBER: builtins.int IS_SHALLOW_FIELD_NUMBER: builtins.int REPLACE_FIELD_NUMBER: builtins.int PROPERTIES_FIELD_NUMBER: builtins.int @property def table(self) -> delta.connect.proto.base_pb2.DeltaTable: """(Required) The source Delta table to clone.""" target: builtins.str """(Required) Path to the location where the cloned table should be stored.""" version: builtins.int """Clones the source table as of the provided version.""" timestamp: builtins.str """Clones the source table as of the provided timestamp.""" is_shallow: builtins.bool """(Required) Performs a clone when true, this field should always be set to true.""" replace: builtins.bool """(Required) Overwrites the target location when true.""" @property def properties( self, ) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: """(Required) User-defined table properties that override properties with the same key in the source table. """ def __init__( self, *, table: delta.connect.proto.base_pb2.DeltaTable | None = ..., target: builtins.str = ..., version: builtins.int = ..., timestamp: builtins.str = ..., is_shallow: builtins.bool = ..., replace: builtins.bool = ..., properties: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal[ "table", b"table", "timestamp", b"timestamp", "version", b"version", "version_or_timestamp", b"version_or_timestamp", ], ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ "is_shallow", b"is_shallow", "properties", b"properties", "replace", b"replace", "table", b"table", "target", b"target", "timestamp", b"timestamp", "version", b"version", "version_or_timestamp", b"version_or_timestamp", ], ) -> None: ... def WhichOneof( self, oneof_group: typing_extensions.Literal["version_or_timestamp", b"version_or_timestamp"], ) -> typing_extensions.Literal["version", "timestamp"] | None: ... global___CloneTable = CloneTable class VacuumTable(google.protobuf.message.Message): """Command that deletes files and directories in the table that are not needed by the table for maintaining older versions up to the given retention threshold. """ DESCRIPTOR: google.protobuf.descriptor.Descriptor TABLE_FIELD_NUMBER: builtins.int RETENTION_HOURS_FIELD_NUMBER: builtins.int @property def table(self) -> delta.connect.proto.base_pb2.DeltaTable: """(Required) The Delta table to vacuum.""" retention_hours: builtins.float """(Optional) Number of hours retain history for. If not specified, then the default retention period will be used. """ def __init__( self, *, table: delta.connect.proto.base_pb2.DeltaTable | None = ..., retention_hours: builtins.float | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal[ "_retention_hours", b"_retention_hours", "retention_hours", b"retention_hours", "table", b"table", ], ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ "_retention_hours", b"_retention_hours", "retention_hours", b"retention_hours", "table", b"table", ], ) -> None: ... def WhichOneof( self, oneof_group: typing_extensions.Literal["_retention_hours", b"_retention_hours"] ) -> typing_extensions.Literal["retention_hours"] | None: ... global___VacuumTable = VacuumTable class UpgradeTableProtocol(google.protobuf.message.Message): """Command to updates the protocol version of the table so that new features can be used.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor TABLE_FIELD_NUMBER: builtins.int READER_VERSION_FIELD_NUMBER: builtins.int WRITER_VERSION_FIELD_NUMBER: builtins.int @property def table(self) -> delta.connect.proto.base_pb2.DeltaTable: """(Required) The Delta table to upgrade the protocol of.""" reader_version: builtins.int """(Required) The minimum required reader protocol version.""" writer_version: builtins.int """(Required) The minimum required writer protocol version.""" def __init__( self, *, table: delta.connect.proto.base_pb2.DeltaTable | None = ..., reader_version: builtins.int = ..., writer_version: builtins.int = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal["table", b"table"] ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ "reader_version", b"reader_version", "table", b"table", "writer_version", b"writer_version", ], ) -> None: ... global___UpgradeTableProtocol = UpgradeTableProtocol class Generate(google.protobuf.message.Message): """Command that generates manifest files for a given Delta table.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor TABLE_FIELD_NUMBER: builtins.int MODE_FIELD_NUMBER: builtins.int @property def table(self) -> delta.connect.proto.base_pb2.DeltaTable: """(Required) The Delta table to generate the manifest files for.""" mode: builtins.str """(Required) The type of manifest file to be generated.""" def __init__( self, *, table: delta.connect.proto.base_pb2.DeltaTable | None = ..., mode: builtins.str = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal["table", b"table"] ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal["mode", b"mode", "table", b"table"] ) -> None: ... global___Generate = Generate class CreateDeltaTable(google.protobuf.message.Message): """Command that creates or replace a Delta table (depending on the mode).""" DESCRIPTOR: google.protobuf.descriptor.Descriptor class _Mode: ValueType = typing.NewType("ValueType", builtins.int) V: typing_extensions.TypeAlias = ValueType class _ModeEnumTypeWrapper( google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[ CreateDeltaTable._Mode.ValueType ], builtins.type, ): # noqa: F821 DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor MODE_UNSPECIFIED: CreateDeltaTable._Mode.ValueType # 0 MODE_CREATE: CreateDeltaTable._Mode.ValueType # 1 """Create the table if it does not exist, and throw an error otherwise.""" MODE_CREATE_IF_NOT_EXISTS: CreateDeltaTable._Mode.ValueType # 2 """Create the table if it does not exist, and do nothing otherwise.""" MODE_REPLACE: CreateDeltaTable._Mode.ValueType # 3 """Replace the table if it already exists, and throw an error otherwise.""" MODE_CREATE_OR_REPLACE: CreateDeltaTable._Mode.ValueType # 4 """Create the table if it does not exist, and replace it otherwise.""" class Mode(_Mode, metaclass=_ModeEnumTypeWrapper): ... MODE_UNSPECIFIED: CreateDeltaTable.Mode.ValueType # 0 MODE_CREATE: CreateDeltaTable.Mode.ValueType # 1 """Create the table if it does not exist, and throw an error otherwise.""" MODE_CREATE_IF_NOT_EXISTS: CreateDeltaTable.Mode.ValueType # 2 """Create the table if it does not exist, and do nothing otherwise.""" MODE_REPLACE: CreateDeltaTable.Mode.ValueType # 3 """Replace the table if it already exists, and throw an error otherwise.""" MODE_CREATE_OR_REPLACE: CreateDeltaTable.Mode.ValueType # 4 """Create the table if it does not exist, and replace it otherwise.""" class Column(google.protobuf.message.Message): """Column in the schema of the table.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor class IdentityInfo(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor START_FIELD_NUMBER: builtins.int STEP_FIELD_NUMBER: builtins.int ALLOW_EXPLICIT_INSERT_FIELD_NUMBER: builtins.int start: builtins.int """(Required) The start value of the identity column.""" step: builtins.int """(Required) The increment value of the identity column.""" allow_explicit_insert: builtins.bool """(Required) Whether the identity column is BY DEFAULT (true) or ALWAYS (false).""" def __init__( self, *, start: builtins.int = ..., step: builtins.int = ..., allow_explicit_insert: builtins.bool = ..., ) -> None: ... def ClearField( self, field_name: typing_extensions.Literal[ "allow_explicit_insert", b"allow_explicit_insert", "start", b"start", "step", b"step", ], ) -> None: ... NAME_FIELD_NUMBER: builtins.int DATA_TYPE_FIELD_NUMBER: builtins.int NULLABLE_FIELD_NUMBER: builtins.int GENERATED_ALWAYS_AS_FIELD_NUMBER: builtins.int COMMENT_FIELD_NUMBER: builtins.int IDENTITY_INFO_FIELD_NUMBER: builtins.int name: builtins.str """(Required) Name of the column.""" @property def data_type(self) -> pyspark.sql.connect.proto.types_pb2.DataType: """(Required) Data type of the column.""" nullable: builtins.bool """(Required) Whether the column is nullable.""" generated_always_as: builtins.str """(Optional) SQL Expression that is used to generate the values in the column.""" comment: builtins.str """(Optional) Comment to describe the column.""" @property def identity_info(self) -> global___CreateDeltaTable.Column.IdentityInfo: """(Optional) Identity information for the column.""" def __init__( self, *, name: builtins.str = ..., data_type: pyspark.sql.connect.proto.types_pb2.DataType | None = ..., nullable: builtins.bool = ..., generated_always_as: builtins.str | None = ..., comment: builtins.str | None = ..., identity_info: global___CreateDeltaTable.Column.IdentityInfo | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal[ "_comment", b"_comment", "_generated_always_as", b"_generated_always_as", "_identity_info", b"_identity_info", "comment", b"comment", "data_type", b"data_type", "generated_always_as", b"generated_always_as", "identity_info", b"identity_info", ], ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ "_comment", b"_comment", "_generated_always_as", b"_generated_always_as", "_identity_info", b"_identity_info", "comment", b"comment", "data_type", b"data_type", "generated_always_as", b"generated_always_as", "identity_info", b"identity_info", "name", b"name", "nullable", b"nullable", ], ) -> None: ... @typing.overload def WhichOneof( self, oneof_group: typing_extensions.Literal["_comment", b"_comment"] ) -> typing_extensions.Literal["comment"] | None: ... @typing.overload def WhichOneof( self, oneof_group: typing_extensions.Literal["_generated_always_as", b"_generated_always_as"], ) -> typing_extensions.Literal["generated_always_as"] | None: ... @typing.overload def WhichOneof( self, oneof_group: typing_extensions.Literal["_identity_info", b"_identity_info"] ) -> typing_extensions.Literal["identity_info"] | None: ... class PropertiesEntry(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor KEY_FIELD_NUMBER: builtins.int VALUE_FIELD_NUMBER: builtins.int key: builtins.str value: builtins.str def __init__( self, *, key: builtins.str = ..., value: builtins.str = ..., ) -> None: ... def ClearField( self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"] ) -> None: ... MODE_FIELD_NUMBER: builtins.int TABLE_NAME_FIELD_NUMBER: builtins.int LOCATION_FIELD_NUMBER: builtins.int COMMENT_FIELD_NUMBER: builtins.int COLUMNS_FIELD_NUMBER: builtins.int PARTITIONING_COLUMNS_FIELD_NUMBER: builtins.int PROPERTIES_FIELD_NUMBER: builtins.int CLUSTERING_COLUMNS_FIELD_NUMBER: builtins.int mode: global___CreateDeltaTable.Mode.ValueType """(Required) Mode that determines what to do when a table with the given name or location already exists. """ table_name: builtins.str """(Optional) Qualified name of the table.""" location: builtins.str """(Optional) Path to the directory where the table date is stored.""" comment: builtins.str """(Optional) Comment describing the table.""" @property def columns( self, ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ global___CreateDeltaTable.Column ]: """(Optional) Columns in the schema of the table.""" @property def partitioning_columns( self, ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: """(Optional) Columns used for partitioning the table.""" @property def properties( self, ) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: """(Optional) Properties of the table.""" @property def clustering_columns( self, ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: """(Optional) Columns used for clustering the table.""" def __init__( self, *, mode: global___CreateDeltaTable.Mode.ValueType = ..., table_name: builtins.str | None = ..., location: builtins.str | None = ..., comment: builtins.str | None = ..., columns: collections.abc.Iterable[global___CreateDeltaTable.Column] | None = ..., partitioning_columns: collections.abc.Iterable[builtins.str] | None = ..., properties: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., clustering_columns: collections.abc.Iterable[builtins.str] | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal[ "_comment", b"_comment", "_location", b"_location", "_table_name", b"_table_name", "comment", b"comment", "location", b"location", "table_name", b"table_name", ], ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ "_comment", b"_comment", "_location", b"_location", "_table_name", b"_table_name", "clustering_columns", b"clustering_columns", "columns", b"columns", "comment", b"comment", "location", b"location", "mode", b"mode", "partitioning_columns", b"partitioning_columns", "properties", b"properties", "table_name", b"table_name", ], ) -> None: ... @typing.overload def WhichOneof( self, oneof_group: typing_extensions.Literal["_comment", b"_comment"] ) -> typing_extensions.Literal["comment"] | None: ... @typing.overload def WhichOneof( self, oneof_group: typing_extensions.Literal["_location", b"_location"] ) -> typing_extensions.Literal["location"] | None: ... @typing.overload def WhichOneof( self, oneof_group: typing_extensions.Literal["_table_name", b"_table_name"] ) -> typing_extensions.Literal["table_name"] | None: ... global___CreateDeltaTable = CreateDeltaTable class AddFeatureSupport(google.protobuf.message.Message): """Command to add a supported feature to the table by modifying the protocol.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor TABLE_FIELD_NUMBER: builtins.int FEATURE_NAME_FIELD_NUMBER: builtins.int @property def table(self) -> delta.connect.proto.base_pb2.DeltaTable: """(Required) The Delta table to add the supported feature to.""" feature_name: builtins.str """(Required) The name of the supported feature to add.""" def __init__( self, *, table: delta.connect.proto.base_pb2.DeltaTable | None = ..., feature_name: builtins.str = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal["table", b"table"] ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal["feature_name", b"feature_name", "table", b"table"], ) -> None: ... global___AddFeatureSupport = AddFeatureSupport class DropFeatureSupport(google.protobuf.message.Message): """Command to drop a supported feature from the table by modifying the protocol.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor TABLE_FIELD_NUMBER: builtins.int FEATURE_NAME_FIELD_NUMBER: builtins.int TRUNCATE_HISTORY_FIELD_NUMBER: builtins.int @property def table(self) -> delta.connect.proto.base_pb2.DeltaTable: """(Required) The Delta table to drop the supported feature from.""" feature_name: builtins.str """(Required) The name of the supported feature to drop.""" truncate_history: builtins.bool """(optional) Whether to truncate history. When not specified, history is not truncated.""" def __init__( self, *, table: delta.connect.proto.base_pb2.DeltaTable | None = ..., feature_name: builtins.str = ..., truncate_history: builtins.bool | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal[ "_truncate_history", b"_truncate_history", "table", b"table", "truncate_history", b"truncate_history", ], ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ "_truncate_history", b"_truncate_history", "feature_name", b"feature_name", "table", b"table", "truncate_history", b"truncate_history", ], ) -> None: ... def WhichOneof( self, oneof_group: typing_extensions.Literal["_truncate_history", b"_truncate_history"] ) -> typing_extensions.Literal["truncate_history"] | None: ... global___DropFeatureSupport = DropFeatureSupport ================================================ FILE: python/delta/connect/proto/relations_pb2.py ================================================ # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: delta/connect/relations.proto """Generated protocol buffer code.""" from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() from delta.connect.proto import base_pb2 as delta_dot_connect_dot_base__pb2 from pyspark.sql.connect.proto import expressions_pb2 as spark_dot_connect_dot_expressions__pb2 from pyspark.sql.connect.proto import relations_pb2 as spark_dot_connect_dot_relations__pb2 from pyspark.sql.connect.proto import types_pb2 as spark_dot_connect_dot_types__pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( b'\n\x1d\x64\x65lta/connect/relations.proto\x12\rdelta.connect\x1a\x18\x64\x65lta/connect/base.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto"\xd7\x05\n\rDeltaRelation\x12)\n\x04scan\x18\x01 \x01(\x0b\x32\x13.delta.connect.ScanH\x00R\x04scan\x12K\n\x10\x64\x65scribe_history\x18\x02 \x01(\x0b\x32\x1e.delta.connect.DescribeHistoryH\x00R\x0f\x64\x65scribeHistory\x12H\n\x0f\x64\x65scribe_detail\x18\x03 \x01(\x0b\x32\x1d.delta.connect.DescribeDetailH\x00R\x0e\x64\x65scribeDetail\x12I\n\x10\x63onvert_to_delta\x18\x04 \x01(\x0b\x32\x1d.delta.connect.ConvertToDeltaH\x00R\x0e\x63onvertToDelta\x12\x42\n\rrestore_table\x18\x05 \x01(\x0b\x32\x1b.delta.connect.RestoreTableH\x00R\x0crestoreTable\x12\x43\n\x0eis_delta_table\x18\x06 \x01(\x0b\x32\x1b.delta.connect.IsDeltaTableH\x00R\x0cisDeltaTable\x12L\n\x11\x64\x65lete_from_table\x18\x07 \x01(\x0b\x32\x1e.delta.connect.DeleteFromTableH\x00R\x0f\x64\x65leteFromTable\x12?\n\x0cupdate_table\x18\x08 \x01(\x0b\x32\x1a.delta.connect.UpdateTableH\x00R\x0bupdateTable\x12I\n\x10merge_into_table\x18\t \x01(\x0b\x32\x1d.delta.connect.MergeIntoTableH\x00R\x0emergeIntoTable\x12\x45\n\x0eoptimize_table\x18\n \x01(\x0b\x32\x1c.delta.connect.OptimizeTableH\x00R\roptimizeTableB\x0f\n\rrelation_type"7\n\x04Scan\x12/\n\x05table\x18\x01 \x01(\x0b\x32\x19.delta.connect.DeltaTableR\x05table"B\n\x0f\x44\x65scribeHistory\x12/\n\x05table\x18\x01 \x01(\x0b\x32\x19.delta.connect.DeltaTableR\x05table"A\n\x0e\x44\x65scribeDetail\x12/\n\x05table\x18\x01 \x01(\x0b\x32\x19.delta.connect.DeltaTableR\x05table"\xd1\x01\n\x0e\x43onvertToDelta\x12\x1e\n\nidentifier\x18\x01 \x01(\tR\nidentifier\x12\x38\n\x17partition_schema_string\x18\x02 \x01(\tH\x00R\x15partitionSchemaString\x12Q\n\x17partition_schema_struct\x18\x03 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\x15partitionSchemaStructB\x12\n\x10partition_schema"\x93\x01\n\x0cRestoreTable\x12/\n\x05table\x18\x01 \x01(\x0b\x32\x19.delta.connect.DeltaTableR\x05table\x12\x1a\n\x07version\x18\x02 \x01(\x03H\x00R\x07version\x12\x1e\n\ttimestamp\x18\x03 \x01(\tH\x00R\ttimestampB\x16\n\x14version_or_timestamp""\n\x0cIsDeltaTable\x12\x12\n\x04path\x18\x01 \x01(\tR\x04path"{\n\x0f\x44\x65leteFromTable\x12/\n\x06target\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x06target\x12\x37\n\tcondition\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\tcondition"\xb4\x01\n\x0bUpdateTable\x12/\n\x06target\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x06target\x12\x37\n\tcondition\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\tcondition\x12;\n\x0b\x61ssignments\x18\x03 \x03(\x0b\x32\x19.delta.connect.AssignmentR\x0b\x61ssignments"\x8c\n\n\x0eMergeIntoTable\x12/\n\x06target\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x06target\x12/\n\x06source\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\x06source\x12\x37\n\tcondition\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\tcondition\x12M\n\x0fmatched_actions\x18\x04 \x03(\x0b\x32$.delta.connect.MergeIntoTable.ActionR\x0ematchedActions\x12T\n\x13not_matched_actions\x18\x05 \x03(\x0b\x32$.delta.connect.MergeIntoTable.ActionR\x11notMatchedActions\x12\x66\n\x1dnot_matched_by_source_actions\x18\x06 \x03(\x0b\x32$.delta.connect.MergeIntoTable.ActionR\x19notMatchedBySourceActions\x12\x37\n\x15with_schema_evolution\x18\x07 \x01(\x08H\x00R\x13withSchemaEvolution\x88\x01\x01\x1a\xfe\x05\n\x06\x41\x63tion\x12\x37\n\tcondition\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\tcondition\x12X\n\rdelete_action\x18\x02 \x01(\x0b\x32\x31.delta.connect.MergeIntoTable.Action.DeleteActionH\x00R\x0c\x64\x65leteAction\x12X\n\rupdate_action\x18\x03 \x01(\x0b\x32\x31.delta.connect.MergeIntoTable.Action.UpdateActionH\x00R\x0cupdateAction\x12\x65\n\x12update_star_action\x18\x04 \x01(\x0b\x32\x35.delta.connect.MergeIntoTable.Action.UpdateStarActionH\x00R\x10updateStarAction\x12X\n\rinsert_action\x18\x05 \x01(\x0b\x32\x31.delta.connect.MergeIntoTable.Action.InsertActionH\x00R\x0cinsertAction\x12\x65\n\x12insert_star_action\x18\x06 \x01(\x0b\x32\x35.delta.connect.MergeIntoTable.Action.InsertStarActionH\x00R\x10insertStarAction\x1a\x0e\n\x0c\x44\x65leteAction\x1aK\n\x0cUpdateAction\x12;\n\x0b\x61ssignments\x18\x01 \x03(\x0b\x32\x19.delta.connect.AssignmentR\x0b\x61ssignments\x1a\x12\n\x10UpdateStarAction\x1aK\n\x0cInsertAction\x12;\n\x0b\x61ssignments\x18\x01 \x03(\x0b\x32\x19.delta.connect.AssignmentR\x0b\x61ssignments\x1a\x12\n\x10InsertStarActionB\r\n\x0b\x61\x63tion_typeB\x18\n\x16_with_schema_evolution"n\n\nAssignment\x12/\n\x05\x66ield\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05\x66ield\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05value"\x94\x01\n\rOptimizeTable\x12/\n\x05table\x18\x01 \x01(\x0b\x32\x19.delta.connect.DeltaTableR\x05table\x12+\n\x11partition_filters\x18\x02 \x03(\tR\x10partitionFilters\x12%\n\x0ezorder_columns\x18\x03 \x03(\tR\rzorderColumnsB\x1a\n\x16io.delta.connect.protoP\x01\x62\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "delta.connect.proto.relations_pb2", globals()) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b"\n\026io.delta.connect.protoP\001" _DELTARELATION._serialized_start = 166 _DELTARELATION._serialized_end = 893 _SCAN._serialized_start = 895 _SCAN._serialized_end = 950 _DESCRIBEHISTORY._serialized_start = 952 _DESCRIBEHISTORY._serialized_end = 1018 _DESCRIBEDETAIL._serialized_start = 1020 _DESCRIBEDETAIL._serialized_end = 1085 _CONVERTTODELTA._serialized_start = 1088 _CONVERTTODELTA._serialized_end = 1297 _RESTORETABLE._serialized_start = 1300 _RESTORETABLE._serialized_end = 1447 _ISDELTATABLE._serialized_start = 1449 _ISDELTATABLE._serialized_end = 1483 _DELETEFROMTABLE._serialized_start = 1485 _DELETEFROMTABLE._serialized_end = 1608 _UPDATETABLE._serialized_start = 1611 _UPDATETABLE._serialized_end = 1791 _MERGEINTOTABLE._serialized_start = 1794 _MERGEINTOTABLE._serialized_end = 3086 _MERGEINTOTABLE_ACTION._serialized_start = 2294 _MERGEINTOTABLE_ACTION._serialized_end = 3060 _MERGEINTOTABLE_ACTION_DELETEACTION._serialized_start = 2837 _MERGEINTOTABLE_ACTION_DELETEACTION._serialized_end = 2851 _MERGEINTOTABLE_ACTION_UPDATEACTION._serialized_start = 2853 _MERGEINTOTABLE_ACTION_UPDATEACTION._serialized_end = 2928 _MERGEINTOTABLE_ACTION_UPDATESTARACTION._serialized_start = 2930 _MERGEINTOTABLE_ACTION_UPDATESTARACTION._serialized_end = 2948 _MERGEINTOTABLE_ACTION_INSERTACTION._serialized_start = 2950 _MERGEINTOTABLE_ACTION_INSERTACTION._serialized_end = 3025 _MERGEINTOTABLE_ACTION_INSERTSTARACTION._serialized_start = 3027 _MERGEINTOTABLE_ACTION_INSERTSTARACTION._serialized_end = 3045 _ASSIGNMENT._serialized_start = 3088 _ASSIGNMENT._serialized_end = 3198 _OPTIMIZETABLE._serialized_start = 3201 _OPTIMIZETABLE._serialized_end = 3349 # @@protoc_insertion_point(module_scope) ================================================ FILE: python/delta/connect/proto/relations_pb2.pyi ================================================ # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """ @generated by mypy-protobuf. Do not edit manually! isort:skip_file Copyright (2024) The Delta Lake Project Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ import builtins import collections.abc import delta.connect.proto.proto.base_pb2 import google.protobuf.descriptor import google.protobuf.internal.containers import google.protobuf.message import pyspark.sql.connect.proto.expressions_pb2 import pyspark.sql.connect.proto.relations_pb2 import pyspark.sql.connect.proto.types_pb2 import sys if sys.version_info >= (3, 8): import typing as typing_extensions else: import typing_extensions DESCRIPTOR: google.protobuf.descriptor.FileDescriptor class DeltaRelation(google.protobuf.message.Message): """Message to hold all relation extensions in Delta Connect.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor SCAN_FIELD_NUMBER: builtins.int DESCRIBE_HISTORY_FIELD_NUMBER: builtins.int DESCRIBE_DETAIL_FIELD_NUMBER: builtins.int CONVERT_TO_DELTA_FIELD_NUMBER: builtins.int RESTORE_TABLE_FIELD_NUMBER: builtins.int IS_DELTA_TABLE_FIELD_NUMBER: builtins.int DELETE_FROM_TABLE_FIELD_NUMBER: builtins.int UPDATE_TABLE_FIELD_NUMBER: builtins.int MERGE_INTO_TABLE_FIELD_NUMBER: builtins.int OPTIMIZE_TABLE_FIELD_NUMBER: builtins.int @property def scan(self) -> global___Scan: ... @property def describe_history(self) -> global___DescribeHistory: ... @property def describe_detail(self) -> global___DescribeDetail: ... @property def convert_to_delta(self) -> global___ConvertToDelta: ... @property def restore_table(self) -> global___RestoreTable: ... @property def is_delta_table(self) -> global___IsDeltaTable: ... @property def delete_from_table(self) -> global___DeleteFromTable: ... @property def update_table(self) -> global___UpdateTable: ... @property def merge_into_table(self) -> global___MergeIntoTable: ... @property def optimize_table(self) -> global___OptimizeTable: ... def __init__( self, *, scan: global___Scan | None = ..., describe_history: global___DescribeHistory | None = ..., describe_detail: global___DescribeDetail | None = ..., convert_to_delta: global___ConvertToDelta | None = ..., restore_table: global___RestoreTable | None = ..., is_delta_table: global___IsDeltaTable | None = ..., delete_from_table: global___DeleteFromTable | None = ..., update_table: global___UpdateTable | None = ..., merge_into_table: global___MergeIntoTable | None = ..., optimize_table: global___OptimizeTable | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal[ "convert_to_delta", b"convert_to_delta", "delete_from_table", b"delete_from_table", "describe_detail", b"describe_detail", "describe_history", b"describe_history", "is_delta_table", b"is_delta_table", "merge_into_table", b"merge_into_table", "optimize_table", b"optimize_table", "relation_type", b"relation_type", "restore_table", b"restore_table", "scan", b"scan", "update_table", b"update_table", ], ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ "convert_to_delta", b"convert_to_delta", "delete_from_table", b"delete_from_table", "describe_detail", b"describe_detail", "describe_history", b"describe_history", "is_delta_table", b"is_delta_table", "merge_into_table", b"merge_into_table", "optimize_table", b"optimize_table", "relation_type", b"relation_type", "restore_table", b"restore_table", "scan", b"scan", "update_table", b"update_table", ], ) -> None: ... def WhichOneof( self, oneof_group: typing_extensions.Literal["relation_type", b"relation_type"] ) -> ( typing_extensions.Literal[ "scan", "describe_history", "describe_detail", "convert_to_delta", "restore_table", "is_delta_table", "delete_from_table", "update_table", "merge_into_table", "optimize_table", ] | None ): ... global___DeltaRelation = DeltaRelation class Scan(google.protobuf.message.Message): """Relation that reads from a Delta table.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor TABLE_FIELD_NUMBER: builtins.int @property def table(self) -> delta.connect.proto.base_pb2.DeltaTable: """(Required) The Delta table to scan.""" def __init__( self, *, table: delta.connect.proto.base_pb2.DeltaTable | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal["table", b"table"] ) -> builtins.bool: ... def ClearField(self, field_name: typing_extensions.Literal["table", b"table"]) -> None: ... global___Scan = Scan class DescribeHistory(google.protobuf.message.Message): """Relation containing information of the latest commits on a Delta table. The information is in reverse chronological order. """ DESCRIPTOR: google.protobuf.descriptor.Descriptor TABLE_FIELD_NUMBER: builtins.int @property def table(self) -> delta.connect.proto.base_pb2.DeltaTable: """(Required) The Delta table to read the history of.""" def __init__( self, *, table: delta.connect.proto.base_pb2.DeltaTable | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal["table", b"table"] ) -> builtins.bool: ... def ClearField(self, field_name: typing_extensions.Literal["table", b"table"]) -> None: ... global___DescribeHistory = DescribeHistory class DescribeDetail(google.protobuf.message.Message): """Relation containing the details of a Delta table such as the format, name, and size.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor TABLE_FIELD_NUMBER: builtins.int @property def table(self) -> delta.connect.proto.base_pb2.DeltaTable: """(Required) The Delta table to describe the details of.""" def __init__( self, *, table: delta.connect.proto.base_pb2.DeltaTable | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal["table", b"table"] ) -> builtins.bool: ... def ClearField(self, field_name: typing_extensions.Literal["table", b"table"]) -> None: ... global___DescribeDetail = DescribeDetail class ConvertToDelta(google.protobuf.message.Message): """Command that turns a Parquet table into a Delta table. This needs to be a Relation as it returns the identifier of the resulting table. We cannot simply reuse the input identifier, as it could be a path-based identifier, and in that case we need to replace "parquet.`...`" with "delta.`...`". """ DESCRIPTOR: google.protobuf.descriptor.Descriptor IDENTIFIER_FIELD_NUMBER: builtins.int PARTITION_SCHEMA_STRING_FIELD_NUMBER: builtins.int PARTITION_SCHEMA_STRUCT_FIELD_NUMBER: builtins.int identifier: builtins.str """(Required) Parquet table identifier formatted as "parquet.`path`" """ partition_schema_string: builtins.str """Hive DDL formatted string""" @property def partition_schema_struct(self) -> pyspark.sql.connect.proto.types_pb2.DataType: """Struct with names and types of partitioning columns""" def __init__( self, *, identifier: builtins.str = ..., partition_schema_string: builtins.str = ..., partition_schema_struct: pyspark.sql.connect.proto.types_pb2.DataType | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal[ "partition_schema", b"partition_schema", "partition_schema_string", b"partition_schema_string", "partition_schema_struct", b"partition_schema_struct", ], ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ "identifier", b"identifier", "partition_schema", b"partition_schema", "partition_schema_string", b"partition_schema_string", "partition_schema_struct", b"partition_schema_struct", ], ) -> None: ... def WhichOneof( self, oneof_group: typing_extensions.Literal["partition_schema", b"partition_schema"] ) -> typing_extensions.Literal["partition_schema_string", "partition_schema_struct"] | None: ... global___ConvertToDelta = ConvertToDelta class RestoreTable(google.protobuf.message.Message): """Command that restores the DeltaTable to an older version of the table specified by either a version number or a timestamp. Needs to be a Relation, as it returns a row containing the execution metrics. """ DESCRIPTOR: google.protobuf.descriptor.Descriptor TABLE_FIELD_NUMBER: builtins.int VERSION_FIELD_NUMBER: builtins.int TIMESTAMP_FIELD_NUMBER: builtins.int @property def table(self) -> delta.connect.proto.base_pb2.DeltaTable: """(Required) The Delta table to restore to an earlier version.""" version: builtins.int """The version number to restore to.""" timestamp: builtins.str """The timestamp to restore to.""" def __init__( self, *, table: delta.connect.proto.base_pb2.DeltaTable | None = ..., version: builtins.int = ..., timestamp: builtins.str = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal[ "table", b"table", "timestamp", b"timestamp", "version", b"version", "version_or_timestamp", b"version_or_timestamp", ], ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ "table", b"table", "timestamp", b"timestamp", "version", b"version", "version_or_timestamp", b"version_or_timestamp", ], ) -> None: ... def WhichOneof( self, oneof_group: typing_extensions.Literal["version_or_timestamp", b"version_or_timestamp"], ) -> typing_extensions.Literal["version", "timestamp"] | None: ... global___RestoreTable = RestoreTable class IsDeltaTable(google.protobuf.message.Message): """Relation containing a single row containing a single boolean that indicates whether the provided path contains a Delta table. """ DESCRIPTOR: google.protobuf.descriptor.Descriptor PATH_FIELD_NUMBER: builtins.int path: builtins.str """(Required) The path to check.""" def __init__( self, *, path: builtins.str = ..., ) -> None: ... def ClearField(self, field_name: typing_extensions.Literal["path", b"path"]) -> None: ... global___IsDeltaTable = IsDeltaTable class DeleteFromTable(google.protobuf.message.Message): """Command that deletes data from the target table that matches the given condition. Needs to be a Relation, as it returns a row containing the execution metrics. """ DESCRIPTOR: google.protobuf.descriptor.Descriptor TARGET_FIELD_NUMBER: builtins.int CONDITION_FIELD_NUMBER: builtins.int @property def target(self) -> pyspark.sql.connect.proto.relations_pb2.Relation: """(Required) Target table to delete data from. Must either be a DeltaRelation containing a Scan or a SubqueryAlias with a DeltaRelation containing a Scan as its input. """ @property def condition(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression: """(Optional) Expression returning a boolean.""" def __init__( self, *, target: pyspark.sql.connect.proto.relations_pb2.Relation | None = ..., condition: pyspark.sql.connect.proto.expressions_pb2.Expression | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal["condition", b"condition", "target", b"target"] ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal["condition", b"condition", "target", b"target"] ) -> None: ... global___DeleteFromTable = DeleteFromTable class UpdateTable(google.protobuf.message.Message): """Command that updates data in the target table using the given assignments for rows that matches the given condition. Needs to be a Relation, as it returns a row containing the execution metrics. """ DESCRIPTOR: google.protobuf.descriptor.Descriptor TARGET_FIELD_NUMBER: builtins.int CONDITION_FIELD_NUMBER: builtins.int ASSIGNMENTS_FIELD_NUMBER: builtins.int @property def target(self) -> pyspark.sql.connect.proto.relations_pb2.Relation: """(Required) Target table to delete data from. Must either be a DeltaRelation containing a Scan or a SubqueryAlias with a DeltaRelation containing a Scan as its input. """ @property def condition(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression: """(Optional) Condition that determines which rows must be updated. Must be an expression returning a boolean. """ @property def assignments( self, ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Assignment]: """(Optional) Set of assignments to apply to the rows matching the condition.""" def __init__( self, *, target: pyspark.sql.connect.proto.relations_pb2.Relation | None = ..., condition: pyspark.sql.connect.proto.expressions_pb2.Expression | None = ..., assignments: collections.abc.Iterable[global___Assignment] | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal["condition", b"condition", "target", b"target"] ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ "assignments", b"assignments", "condition", b"condition", "target", b"target" ], ) -> None: ... global___UpdateTable = UpdateTable class MergeIntoTable(google.protobuf.message.Message): """Command that merges a source query/table into a Delta table, Needs to be a Relation, as it returns a row containing the execution metrics. """ DESCRIPTOR: google.protobuf.descriptor.Descriptor class Action(google.protobuf.message.Message): """Rule that specifies how the target table should be modified.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor class DeleteAction(google.protobuf.message.Message): """Action that deletes the target row.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor def __init__( self, ) -> None: ... class UpdateAction(google.protobuf.message.Message): """Action that updates the target row using a set of assignments.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor ASSIGNMENTS_FIELD_NUMBER: builtins.int @property def assignments( self, ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ global___Assignment ]: """(Optional) Set of assignments to apply.""" def __init__( self, *, assignments: collections.abc.Iterable[global___Assignment] | None = ..., ) -> None: ... def ClearField( self, field_name: typing_extensions.Literal["assignments", b"assignments"] ) -> None: ... class UpdateStarAction(google.protobuf.message.Message): """Action that updates the target row by overwriting all columns.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor def __init__( self, ) -> None: ... class InsertAction(google.protobuf.message.Message): """Action that inserts the source row into the target using a set of assignments.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor ASSIGNMENTS_FIELD_NUMBER: builtins.int @property def assignments( self, ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ global___Assignment ]: """(Optional) Set of assignments to apply.""" def __init__( self, *, assignments: collections.abc.Iterable[global___Assignment] | None = ..., ) -> None: ... def ClearField( self, field_name: typing_extensions.Literal["assignments", b"assignments"] ) -> None: ... class InsertStarAction(google.protobuf.message.Message): """Action that inserts the source row into the target by setting all columns.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor def __init__( self, ) -> None: ... CONDITION_FIELD_NUMBER: builtins.int DELETE_ACTION_FIELD_NUMBER: builtins.int UPDATE_ACTION_FIELD_NUMBER: builtins.int UPDATE_STAR_ACTION_FIELD_NUMBER: builtins.int INSERT_ACTION_FIELD_NUMBER: builtins.int INSERT_STAR_ACTION_FIELD_NUMBER: builtins.int @property def condition(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression: """(Optional) Condition for the action to be applied.""" @property def delete_action(self) -> global___MergeIntoTable.Action.DeleteAction: ... @property def update_action(self) -> global___MergeIntoTable.Action.UpdateAction: ... @property def update_star_action(self) -> global___MergeIntoTable.Action.UpdateStarAction: ... @property def insert_action(self) -> global___MergeIntoTable.Action.InsertAction: ... @property def insert_star_action(self) -> global___MergeIntoTable.Action.InsertStarAction: ... def __init__( self, *, condition: pyspark.sql.connect.proto.expressions_pb2.Expression | None = ..., delete_action: global___MergeIntoTable.Action.DeleteAction | None = ..., update_action: global___MergeIntoTable.Action.UpdateAction | None = ..., update_star_action: global___MergeIntoTable.Action.UpdateStarAction | None = ..., insert_action: global___MergeIntoTable.Action.InsertAction | None = ..., insert_star_action: global___MergeIntoTable.Action.InsertStarAction | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal[ "action_type", b"action_type", "condition", b"condition", "delete_action", b"delete_action", "insert_action", b"insert_action", "insert_star_action", b"insert_star_action", "update_action", b"update_action", "update_star_action", b"update_star_action", ], ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ "action_type", b"action_type", "condition", b"condition", "delete_action", b"delete_action", "insert_action", b"insert_action", "insert_star_action", b"insert_star_action", "update_action", b"update_action", "update_star_action", b"update_star_action", ], ) -> None: ... def WhichOneof( self, oneof_group: typing_extensions.Literal["action_type", b"action_type"] ) -> ( typing_extensions.Literal[ "delete_action", "update_action", "update_star_action", "insert_action", "insert_star_action", ] | None ): ... TARGET_FIELD_NUMBER: builtins.int SOURCE_FIELD_NUMBER: builtins.int CONDITION_FIELD_NUMBER: builtins.int MATCHED_ACTIONS_FIELD_NUMBER: builtins.int NOT_MATCHED_ACTIONS_FIELD_NUMBER: builtins.int NOT_MATCHED_BY_SOURCE_ACTIONS_FIELD_NUMBER: builtins.int WITH_SCHEMA_EVOLUTION_FIELD_NUMBER: builtins.int @property def target(self) -> pyspark.sql.connect.proto.relations_pb2.Relation: """(Required) Target table to merge into.""" @property def source(self) -> pyspark.sql.connect.proto.relations_pb2.Relation: """(Required) Source data to merge from.""" @property def condition(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression: """(Required) Condition for a source row to match with a target row.""" @property def matched_actions( self, ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ global___MergeIntoTable.Action ]: """(Optional) Actions to apply when a source row matches a target row.""" @property def not_matched_actions( self, ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ global___MergeIntoTable.Action ]: """(Optional) Actions to apply when a source row does not match a target row.""" @property def not_matched_by_source_actions( self, ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ global___MergeIntoTable.Action ]: """(Optional) Actions to apply when a target row does not match a source row.""" with_schema_evolution: builtins.bool """(Optional) Whether Schema Evolution is enabled for this command.""" def __init__( self, *, target: pyspark.sql.connect.proto.relations_pb2.Relation | None = ..., source: pyspark.sql.connect.proto.relations_pb2.Relation | None = ..., condition: pyspark.sql.connect.proto.expressions_pb2.Expression | None = ..., matched_actions: collections.abc.Iterable[global___MergeIntoTable.Action] | None = ..., not_matched_actions: collections.abc.Iterable[global___MergeIntoTable.Action] | None = ..., not_matched_by_source_actions: collections.abc.Iterable[global___MergeIntoTable.Action] | None = ..., with_schema_evolution: builtins.bool | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal[ "_with_schema_evolution", b"_with_schema_evolution", "condition", b"condition", "source", b"source", "target", b"target", "with_schema_evolution", b"with_schema_evolution", ], ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ "_with_schema_evolution", b"_with_schema_evolution", "condition", b"condition", "matched_actions", b"matched_actions", "not_matched_actions", b"not_matched_actions", "not_matched_by_source_actions", b"not_matched_by_source_actions", "source", b"source", "target", b"target", "with_schema_evolution", b"with_schema_evolution", ], ) -> None: ... def WhichOneof( self, oneof_group: typing_extensions.Literal["_with_schema_evolution", b"_with_schema_evolution"], ) -> typing_extensions.Literal["with_schema_evolution"] | None: ... global___MergeIntoTable = MergeIntoTable class Assignment(google.protobuf.message.Message): """Represents an assignment of a value to a field.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor FIELD_FIELD_NUMBER: builtins.int VALUE_FIELD_NUMBER: builtins.int @property def field(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression: """(Required) Expression identifying the (struct) field that is assigned a new value.""" @property def value(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression: """(Required) Expression that produces the value to assign to the field.""" def __init__( self, *, field: pyspark.sql.connect.proto.expressions_pb2.Expression | None = ..., value: pyspark.sql.connect.proto.expressions_pb2.Expression | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal["field", b"field", "value", b"value"] ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal["field", b"field", "value", b"value"] ) -> None: ... global___Assignment = Assignment class OptimizeTable(google.protobuf.message.Message): """Command that optimizes the layout of a Delta table by either compacting small files or by ordering the data. Allows specifying partition filters to limit the scope of the data reorganization. Needs to be a Relation, as it returns a row containing the execution metrics. """ DESCRIPTOR: google.protobuf.descriptor.Descriptor TABLE_FIELD_NUMBER: builtins.int PARTITION_FILTERS_FIELD_NUMBER: builtins.int ZORDER_COLUMNS_FIELD_NUMBER: builtins.int @property def table(self) -> delta.connect.proto.base_pb2.DeltaTable: """(Required) The Delta table to optimize.""" @property def partition_filters( self, ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: """(Optional) Partition filters that limit the operation to the files in the matched partitions.""" @property def zorder_columns( self, ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: """(Optional) Columns to z-order by. Compaction is performed when no z-order columns are provided.""" def __init__( self, *, table: delta.connect.proto.base_pb2.DeltaTable | None = ..., partition_filters: collections.abc.Iterable[builtins.str] | None = ..., zorder_columns: collections.abc.Iterable[builtins.str] | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal["table", b"table"] ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ "partition_filters", b"partition_filters", "table", b"table", "zorder_columns", b"zorder_columns", ], ) -> None: ... global___OptimizeTable = OptimizeTable ================================================ FILE: python/delta/connect/tables.py ================================================ # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from typing import ( Any, Dict, Iterable, List, NoReturn, Optional, Tuple, Union, overload ) from delta.connect._typing import ( ColumnMapping, OptionalColumnMapping, ExpressionOrColumn, OptionalExpressionOrColumn ) from delta.connect.plan import ( AddFeatureSupport, Assignment, CloneTable, ConvertToDelta, CreateDeltaTable, DeleteAction, DeleteFromTable, DeltaScan, DescribeHistory, DescribeDetail, DropFeatureSupport, Generate, InsertAction, InsertStarAction, IsDeltaTable, MergeIntoTable, OptimizeTable, RestoreTable, UpdateAction, UpdateStarAction, UpdateTable, UpgradeTableProtocol, Vacuum, ) import delta.connect.proto as proto from delta.tables import ( DeltaTable as LocalDeltaTable, DeltaTableBuilder as LocalDeltaTableBuilder, DeltaMergeBuilder as LocalDeltaMergeBuilder, DeltaOptimizeBuilder as LocalDeltaOptimizeBuilder, IdentityGenerator, ) from pyspark.sql.connect import functions from pyspark.sql.connect.column import Column from pyspark.sql.connect.dataframe import DataFrame from pyspark.sql.connect.plan import LogicalPlan, SubqueryAlias from pyspark.sql.connect.session import SparkSession from pyspark.sql.connect.types import pyspark_types_to_proto_types from pyspark.sql.types import DataType, StructField, StructType class DeltaTable(object): __doc__ = LocalDeltaTable.__doc__ def __init__( self, spark: SparkSession, path: Optional[str] = None, tableOrViewName: Optional[str] = None, hadoopConf: Dict[str, str] = dict(), plan: Optional[LogicalPlan] = None ) -> None: self._spark = spark self._path = path self._tableOrViewName = tableOrViewName self._hadoopConf = hadoopConf if plan is not None: self._plan = plan else: self._plan = DeltaScan(self._to_proto()) def toDF(self) -> DataFrame: return DataFrame(self._plan, session=self._spark) toDF.__doc__ = LocalDeltaTable.toDF.__doc__ def alias(self, aliasName: str) -> "DeltaTable": return DeltaTable( self._spark, self._path, self._tableOrViewName, self._hadoopConf, SubqueryAlias(self._plan, aliasName) ) alias.__doc__ = LocalDeltaTable.alias.__doc__ def generate(self, mode: str) -> None: command = Generate(self._to_proto(), mode).command(session=self._spark.client) self._spark.client.execute_command(command) generate.__doc__ = LocalDeltaTable.generate.__doc__ def delete(self, condition: OptionalExpressionOrColumn = None) -> DataFrame: plan = DeleteFromTable( self._plan, DeltaTable._condition_to_column(condition) ) df = DataFrame(plan, session=self._spark) return self._spark.createDataFrame(df.toPandas()) delete.__doc__ = LocalDeltaTable.delete.__doc__ @overload def update( self, condition: ExpressionOrColumn, set: ColumnMapping ) -> None: ... @overload def update(self, *, set: ColumnMapping) -> None: ... def update( self, condition: OptionalExpressionOrColumn = None, set: OptionalColumnMapping = None ) -> DataFrame: assignments = DeltaTable._dict_to_assignments(set, "'set'") condition = DeltaTable._condition_to_column(condition) plan = UpdateTable( self._plan, condition, assignments ) df = DataFrame(plan, session=self._spark) return self._spark.createDataFrame(df.toPandas()) update.__doc__ = LocalDeltaTable.update.__doc__ def merge( self, source: DataFrame, condition: ExpressionOrColumn ) -> "DeltaMergeBuilder": if source is None: raise ValueError("'source' in merge cannot be None") elif not isinstance(source, DataFrame): raise TypeError("Type of 'source' in merge must be DataFrame. {}".format(type(source))) if condition is None: raise ValueError("'condition' in merge cannot be None") return DeltaMergeBuilder( self._spark, self._plan, source._plan, DeltaTable._condition_to_column(condition)) merge.__doc__ = LocalDeltaTable.merge.__doc__ def vacuum(self, retentionHours: Optional[float] = None) -> DataFrame: command = Vacuum(self._to_proto(), retentionHours).command(session=self._spark.client) self._spark.client.execute_command(command) return None # TODO: Return empty DataFrame vacuum.__doc__ = LocalDeltaTable.vacuum.__doc__ def history(self, limit: Optional[int] = None) -> DataFrame: df = DataFrame(DescribeHistory(self._to_proto()), session=self._spark) if limit is not None: df = df.limit(limit) return df history.__doc__ = LocalDeltaTable.history.__doc__ def detail(self) -> DataFrame: return DataFrame(DescribeDetail(self._to_proto()), session=self._spark) detail.__doc__ = LocalDeltaTable.detail.__doc__ @classmethod def convertToDelta( cls, sparkSession: SparkSession, identifier: str, partitionSchema: Optional[Union[str, StructType]] = None, ) -> "DeltaTable": assert sparkSession is not None pdf = DataFrame( ConvertToDelta(identifier, partitionSchema), session=sparkSession ).toPandas() identifier = pdf.iloc[0].iloc[0] return DeltaTable.forName(sparkSession, identifier) convertToDelta.__func__.__doc__ = LocalDeltaTable.convertToDelta.__doc__ @classmethod def forPath( cls, sparkSession: SparkSession, path: str, hadoopConf: Dict[str, str] = dict() ) -> "DeltaTable": assert sparkSession is not None return DeltaTable(sparkSession, path=path, hadoopConf=hadoopConf) forPath.__func__.__doc__ = LocalDeltaTable.forPath.__doc__ @classmethod def forName( cls, sparkSession: SparkSession, tableOrViewName: str ) -> "DeltaTable": assert sparkSession is not None return DeltaTable(sparkSession, tableOrViewName=tableOrViewName) forName.__func__.__doc__ = LocalDeltaTable.forName.__doc__ @classmethod def create( cls, sparkSession: Optional[SparkSession] = None ) -> "DeltaTableBuilder": return DeltaTableBuilder( sparkSession, proto.CreateDeltaTable.Mode.MODE_CREATE) create.__func__.__doc__ = LocalDeltaTable.create.__doc__ @classmethod def createIfNotExists( cls, sparkSession: Optional[SparkSession] = None ) -> "DeltaTableBuilder": return DeltaTableBuilder( sparkSession, proto.CreateDeltaTable.Mode.MODE_CREATE_IF_NOT_EXISTS) createIfNotExists.__func__.__doc__ = LocalDeltaTable.createIfNotExists.__doc__ @classmethod def replace( cls, sparkSession: Optional[SparkSession] = None ) -> "DeltaTableBuilder": return DeltaTableBuilder( sparkSession, proto.CreateDeltaTable.Mode.MODE_REPLACE) replace.__func__.__doc__ = LocalDeltaTable.replace.__doc__ @classmethod def createOrReplace( cls, sparkSession: Optional[SparkSession] = None ) -> "DeltaTableBuilder": return DeltaTableBuilder( sparkSession, proto.CreateDeltaTable.Mode.MODE_CREATE_OR_REPLACE) createOrReplace.__func__.__doc__ = LocalDeltaTable.createOrReplace.__doc__ @classmethod def isDeltaTable(cls, sparkSession: SparkSession, identifier: str) -> bool: assert sparkSession is not None pdf = DataFrame( IsDeltaTable(identifier), session=sparkSession ).toPandas() return pdf.iloc[0].iloc[0] isDeltaTable.__func__.__doc__ = LocalDeltaTable.isDeltaTable.__doc__ def upgradeTableProtocol(self, readerVersion: int, writerVersion: int) -> None: if not isinstance(readerVersion, int): raise ValueError("The readerVersion needs to be an integer but got '%s'." % type(readerVersion)) if not isinstance(writerVersion, int): raise ValueError("The writerVersion needs to be an integer but got '%s'." % type(writerVersion)) command = UpgradeTableProtocol( self._to_proto(), readerVersion, writerVersion ).command(session=self._spark.client) self._spark.client.execute_command(command) upgradeTableProtocol.__doc__ = LocalDeltaTable.upgradeTableProtocol.__doc__ def addFeatureSupport(self, featureName: str) -> None: LocalDeltaTable._verify_type_str(featureName, "featureName") command = AddFeatureSupport( self._to_proto(), featureName ).command(session=self._spark.client) self._spark.client.execute_command(command) addFeatureSupport.__doc__ = LocalDeltaTable.addFeatureSupport.__doc__ def dropFeatureSupport(self, featureName: str, truncateHistory: Optional[bool] = None) -> None: LocalDeltaTable._verify_type_str(featureName, "featureName") if truncateHistory is not None: LocalDeltaTable._verify_type_bool(truncateHistory, "truncateHistory") command = DropFeatureSupport( self._to_proto(), featureName, truncateHistory ).command(session=self._spark.client) self._spark.client.execute_command(command) dropFeatureSupport.__doc__ = LocalDeltaTable.dropFeatureSupport.__doc__ def restoreToVersion(self, version: int) -> DataFrame: LocalDeltaTable._verify_type_int(version, "version") plan = RestoreTable(self._to_proto(), version=version) df = DataFrame(plan, session=self._spark) return self._spark.createDataFrame(df.toPandas()) restoreToVersion.__doc__ = LocalDeltaTable.restoreToVersion.__doc__ def restoreToTimestamp(self, timestamp: str) -> DataFrame: LocalDeltaTable._verify_type_str(timestamp, "timestamp") plan = RestoreTable(self._to_proto(), timestamp=timestamp) df = DataFrame(plan, session=self._spark) return self._spark.createDataFrame(df.toPandas()) restoreToTimestamp.__doc__ = LocalDeltaTable.restoreToTimestamp.__doc__ def optimize(self) -> "DeltaOptimizeBuilder": return DeltaOptimizeBuilder(self._spark, self) optimize.__doc__ = LocalDeltaTable.optimize.__doc__ def clone( self, target: str, isShallow: bool = False, replace: bool = False, properties: Optional[Dict[str, str]] = None ) -> "DeltaTable": LocalDeltaTable._verify_clone_types(target, isShallow, replace, properties) command = CloneTable( self._to_proto(), target, isShallow, replace, properties ).command(session=self._spark.client) self._spark.client.execute_command(command) return DeltaTable.forName(self._spark, target) clone.__doc__ = LocalDeltaTable.clone.__doc__ def cloneAtVersion( self, version: int, target: str, isShallow: bool = False, replace: bool = False, properties: Optional[Dict[str, str]] = None ) -> "DeltaTable": LocalDeltaTable._verify_clone_types(target, isShallow, replace, properties, version=version) command = CloneTable( self._to_proto(), target, isShallow, replace, properties, version=version ).command(session=self._spark.client) self._spark.client.execute_command(command) return DeltaTable.forName(self._spark, target) cloneAtVersion.__doc__ = LocalDeltaTable.cloneAtVersion.__doc__ def cloneAtTimestamp( self, timestamp: str, target: str, isShallow: bool = False, replace: bool = False, properties: Optional[Dict[str, str]] = None ) -> "DeltaTable": LocalDeltaTable._verify_clone_types(target, isShallow, replace, properties, timestamp) command = CloneTable( self._to_proto(), target, isShallow, replace, properties, timestamp=timestamp ).command(session=self._spark.client) self._spark.client.execute_command(command) return DeltaTable.forName(self._spark, target) cloneAtTimestamp.__doc__ = LocalDeltaTable.cloneAtTimestamp.__doc__ def _to_proto(self) -> proto.DeltaTable: result = proto.DeltaTable() if self._path is not None: result.path.path = self._path if self._tableOrViewName is not None: result.table_or_view_name = self._tableOrViewName return result @staticmethod def _dict_to_assignments( mapping: OptionalColumnMapping, argname: str, ) -> Optional[List[Assignment]]: if mapping is None: raise ValueError("%s cannot be None" % argname) elif type(mapping) is not dict: e = "%s must be a dict, found to be %s" % (argname, str(type(dict))) raise TypeError(e) result = [] for col, expr in mapping.items(): if type(col) is not str: e = ("Keys of dict in %s must contain only strings with column names" % argname) + \ (", found '%s' of type '%s" % (str(col), str(type(col)))) raise TypeError(e) field = functions.col(col) if isinstance(expr, Column): value = expr elif isinstance(expr, str): value = functions.expr(expr) else: e = ("Values of dict in %s must contain only Spark SQL Columns " % argname) + \ "or strings (expressions in SQL syntax) as values, " + \ ("found '%s' of type '%s'" % (str(expr), str(type(expr)))) raise TypeError(e) result.append(Assignment(field, value)) return result @staticmethod def _condition_to_column( condition: OptionalExpressionOrColumn, argname: str = "'condition'" ) -> Column: if condition is None: result = None elif isinstance(condition, Column): result = condition elif isinstance(condition, str): result = functions.expr(condition) else: e = ("%s must be a Spark SQL Column or a string (expression in SQL syntax)" % argname) \ + ", found to be of type %s" % str(type(condition)) raise TypeError(e) return result class DeltaMergeBuilder(object): __doc__ = LocalDeltaMergeBuilder.__doc__ def __init__( self, spark: SparkSession, target: LogicalPlan, source: LogicalPlan, condition: ExpressionOrColumn ) -> None: self._spark = spark self._target = target self._source = source self._condition = condition self._matchedActions = [] self._notMatchedActions = [] self._notMatchedBySourceActions = [] self._with_schema_evolution = False @overload def whenMatchedUpdate( self, condition: OptionalExpressionOrColumn, set: ColumnMapping ) -> "DeltaMergeBuilder": ... @overload def whenMatchedUpdate( self, *, set: ColumnMapping ) -> "DeltaMergeBuilder": ... def whenMatchedUpdate( self, condition: OptionalExpressionOrColumn = None, set: OptionalColumnMapping = None ) -> "DeltaMergeBuilder": assignments = DeltaTable._dict_to_assignments(set, "'set' in whenMatchedUpdate") condition = DeltaTable._condition_to_column(condition) self._matchedActions.append(UpdateAction(condition, assignments)) return self whenMatchedUpdate.__doc__ = LocalDeltaMergeBuilder.whenMatchedUpdate.__doc__ def whenMatchedUpdateAll( self, condition: OptionalExpressionOrColumn = None ) -> "DeltaMergeBuilder": self._matchedActions.append(UpdateStarAction(DeltaTable._condition_to_column(condition))) return self whenMatchedUpdateAll.__doc__ = LocalDeltaMergeBuilder.whenMatchedUpdateAll.__doc__ def whenMatchedDelete( self, condition: OptionalExpressionOrColumn = None ) -> "DeltaMergeBuilder": self._matchedActions.append(DeleteAction(DeltaTable._condition_to_column(condition))) return self whenMatchedDelete.__doc__ = LocalDeltaMergeBuilder.whenMatchedDelete.__doc__ @overload def whenNotMatchedInsert( self, condition: ExpressionOrColumn, values: ColumnMapping ) -> "DeltaMergeBuilder": ... @overload def whenNotMatchedInsert( self, *, values: ColumnMapping = ... ) -> "DeltaMergeBuilder": ... def whenNotMatchedInsert( self, condition: OptionalExpressionOrColumn = None, values: OptionalColumnMapping = None ) -> "DeltaMergeBuilder": assignments = DeltaTable._dict_to_assignments(values, "'values' in whenNotMatchedInsert") condition = DeltaTable._condition_to_column(condition) self._notMatchedActions.append(InsertAction(condition, assignments)) return self whenNotMatchedInsert.__doc__ = LocalDeltaMergeBuilder.whenNotMatchedInsert.__doc__ def whenNotMatchedInsertAll( self, condition: OptionalExpressionOrColumn = None ) -> "DeltaMergeBuilder": self._notMatchedActions.append( InsertStarAction(DeltaTable._condition_to_column(condition)) ) return self whenNotMatchedInsertAll.__doc__ = LocalDeltaMergeBuilder.whenNotMatchedInsertAll.__doc__ @overload def whenNotMatchedBySourceUpdate( self, condition: OptionalExpressionOrColumn, set: ColumnMapping ) -> "DeltaMergeBuilder": ... @overload def whenNotMatchedBySourceUpdate( self, *, set: ColumnMapping ) -> "DeltaMergeBuilder": ... def whenNotMatchedBySourceUpdate( self, condition: OptionalExpressionOrColumn = None, set: OptionalColumnMapping = None ) -> "DeltaMergeBuilder": assignments = DeltaTable._dict_to_assignments(set, "'set' in whenNotMatchedBySourceUpdate") condition = DeltaTable._condition_to_column(condition) self._notMatchedBySourceActions.append(UpdateAction(condition, assignments)) return self whenNotMatchedBySourceUpdate.__doc__ = LocalDeltaMergeBuilder.whenNotMatchedBySourceUpdate.__doc__ def whenNotMatchedBySourceDelete( self, condition: OptionalExpressionOrColumn = None ) -> "DeltaMergeBuilder": action = DeleteAction(DeltaTable._condition_to_column(condition)) self._notMatchedBySourceActions.append(action) return self whenNotMatchedBySourceDelete.__doc__ = LocalDeltaMergeBuilder.whenNotMatchedBySourceDelete.__doc__ def withSchemaEvolution(self) -> "DeltaMergeBuilder": self._with_schema_evolution = True return self def execute(self) -> DataFrame: plan = MergeIntoTable( self._target, self._source, self._condition, self._matchedActions, self._notMatchedActions, self._notMatchedBySourceActions, self._with_schema_evolution ) df = DataFrame(plan, session=self._spark) return self._spark.createDataFrame(df.toPandas()) execute.__doc__ = LocalDeltaMergeBuilder.execute.__doc__ class DeltaTableBuilder(object): __doc__ = LocalDeltaTableBuilder.__doc__ def __init__( self, spark: SparkSession, mode: proto.CreateDeltaTable.Mode ) -> None: self._spark = spark self._mode = mode self._tableName = None self._location = None self._comment = None self._columns = [] self._properties = {} self._partitioningColumns = [] self._clusteringColumns = [] def _raise_type_error(self, msg: str, objs: Iterable[Any]) -> NoReturn: errorMsg = msg for obj in objs: errorMsg += " Found %s with type %s" % ((str(obj)), str(type(obj))) raise TypeError(errorMsg) def _check_identity_column_spec(self, identityGenerator: IdentityGenerator) -> None: if identityGenerator.step == 0: raise ValueError("Column identity generation requires step to be non-zero.") def tableName(self, identifier: str) -> "DeltaTableBuilder": if type(identifier) is not str: self._raise_type_error("Identifier must be str.", [identifier]) self._tableName = identifier return self tableName.__doc__ = LocalDeltaTableBuilder.tableName.__doc__ def location(self, location: str) -> "DeltaTableBuilder": if type(location) is not str: self._raise_type_error("Location must be str.", [location]) self._location = location return self location.__doc__ = LocalDeltaTableBuilder.location.__doc__ def comment(self, comment: str) -> "DeltaTableBuilder": if type(comment) is not str: self._raise_type_error("Table comment must be str.", [comment]) self._comment = comment return self comment.__doc__ = LocalDeltaTableBuilder.comment.__doc__ def addColumn( self, colName: str, dataType: Union[str, DataType], nullable: bool = True, generatedAlwaysAs: Optional[Union[str, IdentityGenerator]] = None, generatedByDefaultAs: Optional[IdentityGenerator] = None, comment: Optional[str] = None, ) -> "DeltaTableBuilder": if type(colName) is not str: self._raise_type_error("Column name must be str.", [colName]) if type(dataType) is not str and not isinstance(dataType, DataType): self._raise_type_error( "Column data type must be str or DataType.", [dataType]) if type(nullable) is not bool: self._raise_type_error("Column nullable must be bool.", [nullable]) if generatedAlwaysAs is not None and generatedByDefaultAs is not None: raise ValueError( "generatedByDefaultAs and generatedAlwaysAs cannot both be set.", [generatedByDefaultAs, generatedAlwaysAs]) if generatedAlwaysAs is not None: if isinstance(generatedAlwaysAs, IdentityGenerator): self._check_identity_column_spec(generatedAlwaysAs) elif type(generatedAlwaysAs) is not str: self._raise_type_error( "Generated always as expression must be str or IdentityGenerator.", [generatedAlwaysAs]) elif generatedByDefaultAs is not None: if not isinstance(generatedByDefaultAs, IdentityGenerator): self._raise_type_error( "Generated by default expression must be IdentityGenerator.", [generatedByDefaultAs]) self._check_identity_column_spec(generatedByDefaultAs) if comment is not None and type(comment) is not str: self._raise_type_error("Comment must be str or None.", [colName]) column = proto.CreateDeltaTable.Column() column.name = colName if type(dataType) is str: column.data_type.unparsed.data_type_string = dataType elif isinstance(dataType, DataType): column.data_type.CopyFrom(pyspark_types_to_proto_types(dataType)) column.nullable = nullable if generatedAlwaysAs is not None: if type(generatedAlwaysAs) is str: column.generated_always_as = generatedAlwaysAs else: identity_info = proto.CreateDeltaTable.Column.IdentityInfo( start=generatedAlwaysAs.start, step=generatedAlwaysAs.step, allow_explicit_insert=False) column.identity_info.CopyFrom(identity_info) if generatedByDefaultAs is not None: identity_info = proto.CreateDeltaTable.Column.IdentityInfo( start=generatedByDefaultAs.start, step=generatedByDefaultAs.step, allow_explicit_insert=True) column.identity_info.CopyFrom(identity_info) if comment is not None: column.comment = comment self._columns.append(column) return self addColumn.__doc__ = LocalDeltaTableBuilder.addColumn.__doc__ def addColumns( self, cols: Union[StructType, List[StructField]] ) -> "DeltaTableBuilder": if isinstance(cols, list): for col in cols: if type(col) is not StructField: self._raise_type_error( "Column in existing schema must be StructField.", [col]) cols = StructType(cols) if type(cols) is not StructType: self._raise_type_error( "Schema must be StructType or a list of StructField.", [cols]) for col in cols: self.addColumn(col.name, col.dataType, col.nullable) return self addColumns.__doc__ = LocalDeltaTableBuilder.addColumns.__doc__ @overload def partitionedBy( self, *cols: str ) -> "DeltaTableBuilder": ... @overload def partitionedBy( self, __cols: Union[List[str], Tuple[str, ...]] ) -> "DeltaTableBuilder": ... def partitionedBy( self, *cols: Union[str, List[str], Tuple[str, ...]] ) -> "DeltaTableBuilder": if len(cols) == 1 and isinstance(cols[0], (list, tuple)): cols = cols[0] # type: ignore[assignment] for c in cols: if type(c) is not str: self._raise_type_error("Partitioning column must be str.", [c]) self._partitioningColumns.extend(cols) return self partitionedBy.__doc__ = LocalDeltaTableBuilder.partitionedBy.__doc__ @overload def clusterBy( self, *cols: str ) -> "DeltaTableBuilder": ... @overload def clusterBy( self, __cols: Union[List[str], Tuple[str, ...]] ) -> "DeltaTableBuilder": ... def clusterBy( self, *cols: Union[str, List[str], Tuple[str, ...]] ) -> "DeltaTableBuilder": if len(cols) == 1 and isinstance(cols[0], (list, tuple)): cols = cols[0] # type: ignore[assignment] for c in cols: if type(c) is not str: self._raise_type_error("Clustering column must be str.", [c]) self._clusteringColumns.extend(cols) return self clusterBy.__doc__ = LocalDeltaTableBuilder.clusterBy.__doc__ def property(self, key: str, value: str) -> "DeltaTableBuilder": if type(key) is not str or type(value) is not str: self._raise_type_error( "Key and value of property must be string.", [key, value]) self._properties[key] = value return self property.__doc__ = LocalDeltaTableBuilder.property.__doc__ def execute(self) -> DeltaTable: command = CreateDeltaTable( self._mode, self._tableName, self._location, self._comment, self._columns, self._partitioningColumns, self._properties, self._clusteringColumns ).command(session=self._spark.client) self._spark.client.execute_command(command) if self._tableName is not None: return DeltaTable.forName(self._spark, self._tableName) else: return DeltaTable.forPath(self._spark, self._location) execute.__doc__ = LocalDeltaTableBuilder.execute.__doc__ class DeltaOptimizeBuilder(object): __doc__ = LocalDeltaOptimizeBuilder.__doc__ def __init__(self, spark: SparkSession, table: "DeltaTable"): self._spark = spark self._table = table self._partitionFilters = [] def where(self, partitionFilter: str) -> "DeltaOptimizeBuilder": self._partitionFilters.append(partitionFilter) return self where.__doc__ = LocalDeltaOptimizeBuilder.where.__doc__ def executeCompaction(self) -> DataFrame: plan = OptimizeTable(self._table._to_proto(), self._partitionFilters, []) df = DataFrame(plan, session=self._spark) return self._spark.createDataFrame(df.toPandas()) executeCompaction.__doc__ = LocalDeltaOptimizeBuilder.executeCompaction.__doc__ def executeZOrderBy(self, *cols: Union[str, List[str], Tuple[str, ...]]) -> DataFrame: if len(cols) == 1 and isinstance(cols[0], (list, tuple)): cols = cols[0] # type: ignore[assignment] for c in cols: if type(c) is not str: errorMsg = "Z-order column must be str. " errorMsg += "Found %s with type %s" % ((str(c)), str(type(c))) raise TypeError(errorMsg) plan = OptimizeTable(self._table._to_proto(), self._partitionFilters, cols) df = DataFrame(plan, session=self._spark) return self._spark.createDataFrame(df.toPandas()) executeZOrderBy.__doc__ = LocalDeltaOptimizeBuilder.executeZOrderBy.__doc__ ================================================ FILE: python/delta/connect/testing/__init__.py ================================================ # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ================================================ FILE: python/delta/connect/testing/utils.py ================================================ # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import tempfile import shutil import os import uuid from contextlib import contextmanager from pyspark import SparkConf from pyspark.testing.connectutils import ReusedConnectTestCase from typing import Generator class DeltaTestCase(ReusedConnectTestCase): """ Test suite base for setting up a properly configured SparkSession for using Delta Connect. """ @classmethod def setUpClass(cls) -> None: # Spark Connect will set SPARK_CONNECT_TESTING_REMOTE, and it does not allow MASTER # to be set simultaneously, so we need to clear it. # TODO(long.vu): Find a cleaner way to clear "MASTER". if "MASTER" in os.environ: del os.environ["MASTER"] super(DeltaTestCase, cls).setUpClass() @classmethod def conf(cls) -> SparkConf: _conf = super(DeltaTestCase, cls).conf() _conf.set("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") _conf.set("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") _conf.set("spark.connect.extensions.relation.classes", "org.apache.spark.sql.connect.delta.DeltaRelationPlugin") _conf.set("spark.connect.extensions.command.classes", "org.apache.spark.sql.connect.delta.DeltaCommandPlugin") return _conf def setUp(self) -> None: super(DeltaTestCase, self).setUp() self.tempPath = tempfile.mkdtemp() self.tempFile = os.path.join(self.tempPath, "tempFile") def tearDown(self) -> None: super(DeltaTestCase, self).tearDown() shutil.rmtree(self.tempPath) @contextmanager def tempTable(self) -> Generator[str, None, None]: table_name = "table_" + str(uuid.uuid4()).replace("-", "_") with super(DeltaTestCase, self).table(table_name): yield table_name ================================================ FILE: python/delta/connect/tests/__init__.py ================================================ # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ================================================ FILE: python/delta/connect/tests/test_deltatable.py ================================================ # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os import unittest import sys from delta.connect.testing.utils import DeltaTestCase path_to_delta_connect_tests_folder = os.path.dirname(os.path.abspath(__file__)) path_to_delta_folder = os.path.dirname(os.path.dirname(path_to_delta_connect_tests_folder)) sys.path.append(path_to_delta_folder) from tests.test_deltatable import DeltaTableTestsMixin class DeltaTableTests(DeltaTableTestsMixin, DeltaTestCase): @unittest.skip("relies on jvm") def test_verify_paritionedBy_compatibility(self): pass if __name__ == "__main__": try: import xmlrunner testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=4) except ImportError: testRunner = None unittest.main(testRunner=testRunner, verbosity=4) ================================================ FILE: python/delta/exceptions/__init__.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from delta.exceptions.base import ( DeltaConcurrentModificationException, ConcurrentWriteException, MetadataChangedException, ProtocolChangedException, ConcurrentAppendException, ConcurrentDeleteReadException, ConcurrentDeleteDeleteException, ConcurrentTransactionException, ) __all__ = [ "DeltaConcurrentModificationException", "ConcurrentWriteException", "MetadataChangedException", "ProtocolChangedException", "ConcurrentAppendException", "ConcurrentDeleteReadException", "ConcurrentDeleteDeleteException", "ConcurrentTransactionException", ] ================================================ FILE: python/delta/exceptions/base.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from pyspark.errors.exceptions.base import PySparkException class DeltaConcurrentModificationException(PySparkException): """ The basic class for all Delta commit conflict exceptions. .. versionadded:: 1.0 .. note:: Evolving """ class ConcurrentWriteException(PySparkException): """ Thrown when a concurrent transaction has written data after the current transaction read the table. .. versionadded:: 1.0 .. note:: Evolving """ class MetadataChangedException(PySparkException): """ Thrown when the metadata of the Delta table has changed between the time of read and the time of commit. .. versionadded:: 1.0 .. note:: Evolving """ class ProtocolChangedException(PySparkException): """ Thrown when the protocol version has changed between the time of read and the time of commit. .. versionadded:: 1.0 .. note:: Evolving """ class ConcurrentAppendException(PySparkException): """ Thrown when files are added that would have been read by the current transaction. .. versionadded:: 1.0 .. note:: Evolving """ class ConcurrentDeleteReadException(PySparkException): """ Thrown when the current transaction reads data that was deleted by a concurrent transaction. .. versionadded:: 1.0 .. note:: Evolving """ class ConcurrentDeleteDeleteException(PySparkException): """ Thrown when the current transaction deletes data that was deleted by a concurrent transaction. .. versionadded:: 1.0 .. note:: Evolving """ class ConcurrentTransactionException(PySparkException): """ Thrown when concurrent transaction both attempt to update the same idempotent transaction. .. versionadded:: 1.0 .. note:: Evolving """ ================================================ FILE: python/delta/exceptions/captured.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from typing import TYPE_CHECKING, Optional from pyspark import SparkContext from pyspark.errors.exceptions import captured from pyspark.errors.exceptions.captured import CapturedException from delta.exceptions.base import ( DeltaConcurrentModificationException as BaseDeltaConcurrentModificationException, ConcurrentWriteException as BaseConcurrentWriteException, MetadataChangedException as BaseMetadataChangedException, ProtocolChangedException as BaseProtocolChangedException, ConcurrentAppendException as BaseConcurrentAppendException, ConcurrentDeleteReadException as BaseConcurrentDeleteReadException, ConcurrentDeleteDeleteException as BaseConcurrentDeleteDeleteException, ConcurrentTransactionException as BaseConcurrentTransactionException, ) if TYPE_CHECKING: from py4j.java_gateway import JavaObject, JVMView # type: ignore[import] class DeltaConcurrentModificationException( CapturedException, BaseDeltaConcurrentModificationException ): """ The basic class for all Delta commit conflict exceptions. .. versionadded:: 1.0 .. note:: Evolving """ class ConcurrentWriteException(CapturedException, BaseConcurrentWriteException): """ Thrown when a concurrent transaction has written data after the current transaction read the table. .. versionadded:: 1.0 .. note:: Evolving """ class MetadataChangedException(CapturedException, BaseMetadataChangedException): """ Thrown when the metadata of the Delta table has changed between the time of read and the time of commit. .. versionadded:: 1.0 .. note:: Evolving """ class ProtocolChangedException(CapturedException, BaseProtocolChangedException): """ Thrown when the protocol version has changed between the time of read and the time of commit. .. versionadded:: 1.0 .. note:: Evolving """ class ConcurrentAppendException(CapturedException, BaseConcurrentAppendException): """ Thrown when files are added that would have been read by the current transaction. .. versionadded:: 1.0 .. note:: Evolving """ class ConcurrentDeleteReadException(CapturedException, BaseConcurrentDeleteReadException): """ Thrown when the current transaction reads data that was deleted by a concurrent transaction. .. versionadded:: 1.0 .. note:: Evolving """ class ConcurrentDeleteDeleteException(CapturedException, BaseConcurrentDeleteDeleteException): """ Thrown when the current transaction deletes data that was deleted by a concurrent transaction. .. versionadded:: 1.0 .. note:: Evolving """ class ConcurrentTransactionException(CapturedException, BaseConcurrentTransactionException): """ Thrown when concurrent transaction both attempt to update the same idempotent transaction. .. versionadded:: 1.0 .. note:: Evolving """ _delta_exception_patched = False def _convert_delta_exception(e: "JavaObject") -> Optional[CapturedException]: """ Convert Delta's Scala concurrent exceptions to the corresponding Python exceptions. """ s: str = e.toString() c: "JavaObject" = e.getCause() jvm: "JVMView" = SparkContext._jvm # type: ignore[attr-defined] gw = SparkContext._gateway # type: ignore[attr-defined] stacktrace = jvm.org.apache.spark.util.Utils.exceptionString(e) if s.startswith('io.delta.exceptions.DeltaConcurrentModificationException: '): return DeltaConcurrentModificationException(s.split(': ', 1)[1], stacktrace, c) if s.startswith('io.delta.exceptions.ConcurrentWriteException: '): return ConcurrentWriteException(s.split(': ', 1)[1], stacktrace, c) if s.startswith('io.delta.exceptions.MetadataChangedException: '): return MetadataChangedException(s.split(': ', 1)[1], stacktrace, c) if s.startswith('io.delta.exceptions.ProtocolChangedException: '): return ProtocolChangedException(s.split(': ', 1)[1], stacktrace, c) if s.startswith('io.delta.exceptions.ConcurrentAppendException: '): return ConcurrentAppendException(s.split(': ', 1)[1], stacktrace, c) if s.startswith('io.delta.exceptions.ConcurrentDeleteReadException: '): return ConcurrentDeleteReadException(s.split(': ', 1)[1], stacktrace, c) if s.startswith('io.delta.exceptions.ConcurrentDeleteDeleteException: '): return ConcurrentDeleteDeleteException(s.split(': ', 1)[1], stacktrace, c) if s.startswith('io.delta.exceptions.ConcurrentTransactionException: '): return ConcurrentTransactionException(s.split(': ', 1)[1], stacktrace, c) return None def _patch_convert_exception() -> None: """ Patch PySpark's exception convert method to convert Delta's Scala concurrent exceptions to the corresponding Python exceptions. """ original_convert_sql_exception = captured.convert_exception def convert_delta_exception(e: "JavaObject") -> CapturedException: delta_exception = _convert_delta_exception(e) if delta_exception is not None: return delta_exception return original_convert_sql_exception(e) captured.convert_exception = convert_delta_exception if not _delta_exception_patched: _patch_convert_exception() _delta_exception_patched = True ================================================ FILE: python/delta/integration_tests/unity-catalog-commit-coordinator-integration-tests.py ================================================ # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import datetime import os import py4j import unittest from delta.tables import DeltaTable from pyspark.errors.exceptions.captured import AnalysisException, UnsupportedOperationException from pyspark.sql import SparkSession, DataFrame from pyspark.sql.functions import lit from pyspark.sql.types import IntegerType, StructType, StructField from pyspark.testing import assertDataFrameEqual """ Run this script in root dir of repository: ===== Mandatory input from user ===== export CATALOG_TOKEN=___ export CATALOG_URI=___ export CATALOG_NAME=___ export SCHEMA=___ export MANAGED_CC_TABLE=___ export MANAGED_NON_CC_TABLE=___ ./run-integration-tests.py --use-local --unity-catalog-commit-coordinator-integration-tests \ --packages \ io.unitycatalog:unitycatalog-spark_2.13:0.3.0,org.apache.spark:spark-hadoop-cloud_2.13:4.0.0 """ CATALOG_NAME = os.environ.get("CATALOG_NAME") CATALOG_TOKEN = os.environ.get("CATALOG_TOKEN") CATALOG_URI = os.environ.get("CATALOG_URI") MANAGED_CC_TABLE = os.environ.get("MANAGED_CC_TABLE") SCHEMA = os.environ.get("SCHEMA") MANAGED_NON_CC_TABLE = os.environ.get("MANAGED_NON_CC_TABLE") spark = SparkSession \ .builder \ .appName("coordinated_commit_tester") \ .master("local[*]") \ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \ .config(f"spark.sql.catalog.{CATALOG_NAME}", "io.unitycatalog.spark.UCSingleCatalog") \ .config(f"spark.sql.catalog.{CATALOG_NAME}.token", CATALOG_TOKEN) \ .config(f"spark.sql.catalog.{CATALOG_NAME}.uri", CATALOG_URI) \ .config("spark.databricks.delta.replaceWhere.constraintCheck.enabled", True) \ .config("spark.hadoop.fs.s3.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem") \ .getOrCreate() MANAGED_CATALOG_OWNED_TABLE_FULL_NAME = f"{CATALOG_NAME}.{SCHEMA}.{MANAGED_CC_TABLE}" MANAGED_NON_CATALOG_OWNED_TABLE_FULL_NAME = f"{CATALOG_NAME}.{SCHEMA}.{MANAGED_NON_CC_TABLE}" class UnityCatalogManagedTableTestBase(unittest.TestCase): """ Shared helpers and test setup for test suites below. """ setup_df = spark.createDataFrame([(1, ), (2, ), (3, )], schema=StructType([StructField("id", IntegerType(), True)])) def setUp(self) -> None: self.setup_df.write.mode("overwrite").insertInto(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME) # Helper methods def read(self, table_name: str) -> DataFrame: return spark.read.table(table_name) def current_version(self, table_name: str) -> int: # Access the delta table's max version. dt = DeltaTable.forName(spark, table_name) return dt.history().selectExpr("max(version)").collect()[0][0] def read_with_cdf_timestamp(self, timestamp: str, table_name: str) -> DataFrame: return spark.read.option('readChangeFeed', 'true').option( "startingTimestamp", timestamp).table(table_name) def read_with_cdf_version(self, version: int, table_name: str) -> DataFrame: return spark.read.option('readChangeFeed', 'true').option( "startingVersion", version).table(table_name) def create_df_with_rows(self, list_of_rows: list) -> DataFrame: return spark.createDataFrame(list_of_rows, schema=StructType([StructField("id", IntegerType(), True)])) def get_table_history(self, table_name: str) -> DataFrame: return spark.sql(f"DESCRIBE HISTORY {table_name}") def append(self, table_name: str) -> None: single_col_df = spark.createDataFrame( [(4, ), (5, )], schema=StructType([StructField("id", IntegerType(), True)])) single_col_df.writeTo(table_name).append() class UnityCatalogManagedTableBasicSuite(UnityCatalogManagedTableTestBase): """ Suite covering basic functionality of catalog owned tables. """ def test_read_from_managed_table_without_catalog_owned(self) -> None: self.read(MANAGED_NON_CATALOG_OWNED_TABLE_FULL_NAME) def test_write_to_managed_catalog_owned_table(self) -> None: self.append(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME) updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(1, ), (2, ), (3, ), (4, ), (5, )])) def test_read_from_managed_catalog_owned_table(self) -> None: self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME) updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.setup_df) # Writing to tables that are not catalog owned is not supported. def test_write_to_managed_table_without_catalog_owned(self) -> None: try: self.append(MANAGED_NON_CATALOG_OWNED_TABLE_FULL_NAME) except py4j.protocol.Py4JJavaError as error: assert("[TASK_WRITE_FAILED] Task failed while writing rows to s3" in str(error)) def test_unset_catalog_owned_feature(self) -> None: try: spark.sql(f"ALTER TABLE {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME} " f"UNSET TBLPROPERTIES ('delta.feature.catalogManaged')") except UnsupportedOperationException as error: assert("Altering a table is not supported yet" in str(error)) def test_drop_catalog_owned_property(self) -> None: try: spark.sql(f"ALTER TABLE {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME} " f"DROP FEATURE 'catalogManaged'") except UnsupportedOperationException as error: assert("Altering a table is not supported yet" in str(error)) class UnityCatalogManagedTableDMLSuite(UnityCatalogManagedTableTestBase): """ Suite covering DMLs (INSERT, MERGE, UPDATE, DELETE) on catalog owned tables. """ def test_update(self) -> None: dt = DeltaTable.forName(spark, MANAGED_CATALOG_OWNED_TABLE_FULL_NAME) dt.update(condition="id = 1", set={"id": "4"}) updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(4, ), (2, ), (3, )])) def test_sql_update(self) -> None: spark.sql(f"UPDATE {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME} SET id=4 WHERE id=1") updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(4, ), (2, ), (3, )])) def test_delete(self) -> None: dt = DeltaTable.forName(spark, MANAGED_CATALOG_OWNED_TABLE_FULL_NAME) dt.delete(condition="id = 1") updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(2, ), (3, )])) def test_sql_delete(self) -> None: spark.sql(f"DELETE FROM {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME} where id=1") updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(2, ), (3, )])) def test_merge(self) -> None: dt = DeltaTable.forName(spark, MANAGED_CATALOG_OWNED_TABLE_FULL_NAME) src = self.create_df_with_rows([(1, ), (2, ), (3, ), (4, ), (5, )]) dt.alias("target") \ .merge( source=src.alias("src"), condition="src.id = target.id") \ .whenNotMatchedInsertAll() \ .execute() updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(1, ), (2, ), (3, ), (4, ), (5, )])) def test_sql_merge(self) -> None: spark.sql(f"MERGE INTO {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME} AS target " f"USING (VALUES 2, 3, 4, 5 AS src(id)) AS src " f"ON src.id = target.id WHEN NOT MATCHED THEN INSERT *") updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(1, ), (2, ), (3, ), (4, ), (5, )])) def test_merge_schema_evolution(self) -> None: spark.conf.set("spark.databricks.delta.schema.autoMerge.enabled", "true") try: spark.sql(f"MERGE INTO {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME} AS target " f"USING (VALUES (2, 2), (3, 3), (4, 4), (5, 5) AS src(id, extra)) AS src " f"ON src.id = target.id WHEN NOT MATCHED THEN INSERT *") except py4j.protocol.Py4JJavaError as error: assert( "A table's Delta metadata can only be changed from a cluster or warehouse" in str(error) ) finally: spark.conf.unset("spark.databricks.delta.schema.autoMerge.enabled") updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.setup_df) def test_insert_schema_evolution(self) -> None: two_cols_df = spark.createDataFrame([(4, 4), (5, 5)], schema=["id, extra"]) try: two_cols_df\ .write.mode("append").option("mergeSchema", "true")\ .insertInto(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME) except py4j.protocol.Py4JJavaError as error: assert( "A table's Delta metadata can only be changed from a cluster or warehouse" in str(error) ) updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.setup_df) def test_sql_insert(self) -> None: spark.sql(f"INSERT INTO {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME} " f"VALUES (4), (5)") updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(1,), (2,), (3,), (4,), (5,)])) def test_sql_insert_overwrite(self) -> None: spark.sql(f"INSERT OVERWRITE {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME} " f"VALUES (2), (3), (4), (5)") updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(2,), (3,), (4,), (5,)])) def test_sql_insert_replace_where(self) -> None: spark.sql(f"INSERT INTO {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME} " f"REPLACE WHERE id = 1 " f"VALUES (1)") updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(1,), (2,), (3,)])) def test_sql_insert_dynamic_partition_overwrite(self) -> None: spark.conf.set("spark.databricks.delta.dynamicPartitionOverwrite.enabled", "true") try: spark.sql(f"INSERT INTO {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME} VALUES (5)") updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(1,), (2,), (3,), (5,)])) finally: spark.conf.unset("spark.databricks.delta.dynamicPartitionOverwrite.enabled") # Dataframe Writer V1 Tests # def test_insert_into_append(self) -> None: single_col_df = spark.createDataFrame([(4, ), (5, )], schema=["id"]) single_col_df.write.mode("append").insertInto(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME) updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(1, ), (2, ), (3, ), (4, ), (5, )])) def test_insert_into_overwrite(self) -> None: single_col_df = spark.createDataFrame([(5, )], schema=["id"]) single_col_df.write.mode("overwrite").insertInto( MANAGED_CATALOG_OWNED_TABLE_FULL_NAME, True) updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(5, )])) def test_insert_into_overwrite_replace_where(self) -> None: single_col_df = spark.createDataFrame([(5, )], schema=["id"]) single_col_df.write.mode("overwrite").option("replaceWhere", "id > 1").insertInto( f"{MANAGED_CATALOG_OWNED_TABLE_FULL_NAME}", True) updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(1, ), (5, )])) def test_insert_into_overwrite_partition_overwrite(self) -> None: single_col_df = spark.createDataFrame([(5,)], schema=["id"]) single_col_df.write.mode("overwrite").option( "partitionOverwriteMode", "dynamic").insertInto( MANAGED_CATALOG_OWNED_TABLE_FULL_NAME, True) updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(5,)])) def test_save_as_table_append_existing_table(self) -> None: single_col_df = spark.createDataFrame( [(4, ), (5, )], schema=StructType([StructField("id", IntegerType(), True)])) single_col_df.write.format("delta").mode("append").saveAsTable( MANAGED_CATALOG_OWNED_TABLE_FULL_NAME) updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(1, ), (2, ), (3, ), (4, ), (5, )])) # Setting mode to append should work, however cc tables do not allow path based access. def test_save_append_using_path(self) -> None: single_col_df = spark.createDataFrame([(4, ), (5, )]) # Fetch managed table path and attempt to side-step UC # and directly update table using path based access. tbl_path = spark.sql( f"DESCRIBE formatted {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME}").collect()[5].data_type try: single_col_df.write.format("delta").save(mode="append", path=tbl_path) except py4j.protocol.Py4JJavaError as error: assert("AccessDeniedException" in str(error)) updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.setup_df) # DataFrame V2 Tests # def test_append(self) -> None: self.append(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME) updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(1, ), (2, ), (3, ), (4, ), (5, )])) def test_overwrite(self) -> None: single_col_df = spark.createDataFrame( [(5,)], schema=StructType([StructField("id", IntegerType(), True)])) single_col_df.writeTo(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).overwrite(lit(True)) updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(5,)])) def test_overwrite_partitions(self) -> None: single_col_df = spark.createDataFrame( [(5,)], schema=StructType([StructField("id", IntegerType(), True)])) single_col_df.writeTo(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).overwritePartitions() updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(5,)])) class UnityCatalogManagedTableDDLSuite(UnityCatalogManagedTableTestBase): """ Suite covering DDLs (CREATE, REPLACE, CLONE, ALTER) on catalog owned tables. """ def test_create_non_delta(self) -> None: single_col_df = spark.createDataFrame( [(5,)], schema=StructType([StructField("id", IntegerType(), True)])) try: # CREATE TABLE is currently not supported by UC. single_col_df.writeTo(f"{CATALOG_NAME}.{SCHEMA}.created_table").create() except py4j.protocol.Py4JJavaError as error: assert("io.unitycatalog.spark.UCProxy.createTable" in str(error)) def test_create_delta(self) -> None: single_col_df = spark.createDataFrame( [(5,)], schema=StructType([StructField("id", IntegerType(), True)])) try: # CREATE TABLE is currently not supported by UC. single_col_df.writeTo(f"{CATALOG_NAME}.{SCHEMA}.created_table").using("delta").create() except AnalysisException as error: assert( f"[SCHEMA_NOT_FOUND] The schema `spark_catalog`.`{SCHEMA}` cannot be found" in str(error) ) def test_sql_create(self) -> None: try: # This ignores the catalog name passed and tries to create the table under # 'spark_catalog'. spark.sql(f"CREATE TABLE {CATALOG_NAME}.{SCHEMA}.created_table (a int) USING DELTA") except AnalysisException as error: assert( f"[SCHEMA_NOT_FOUND] The schema `spark_catalog`.`{SCHEMA}` cannot be found" in str(error) ) def test_create_non_catalog_owned(self) -> None: try: # This ignores the catalog name passed and tries to create the table under # 'spark_catalog'. spark.sql(f"CREATE TABLE {CATALOG_NAME}.{SCHEMA}.created_table (id int) USING DELTA") except AnalysisException as error: assert( f"[SCHEMA_NOT_FOUND] The schema `spark_catalog`.`{SCHEMA}` cannot be found" in str(error) ) def test_clone_into_catalog_owned(self) -> None: try: # CLONE fails with an assertion error in UCSingleCatalog spark.sql(f"CREATE TABLE {CATALOG_NAME}.{SCHEMA}.created_table" + f" SHALLOW CLONE {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME}") except py4j.protocol.Py4JJavaError as error: assert("java.lang.AssertionError: assertion failed" in str(error)) def test_clone_into_non_catalog_owned(self) -> None: try: # CLONE fails with an assertion error in UCSingleCatalog spark.sql(f"CREATE TABLE {CATALOG_NAME}.{SCHEMA}.created_table" + f" SHALLOW CLONE {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME} " f"TBLPROPERTIES ('delta.feature.catalogManaged' = 'false')") except py4j.protocol.Py4JJavaError as error: assert("java.lang.AssertionError: assertion failed" in str(error)) def test_alter_table_comment(self) -> None: try: spark.sql(f"ALTER TABLE {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME} " f"ALTER COLUMN id COMMENT 'comment'") except UnsupportedOperationException as error: assert("Altering a table is not supported yet" in str(error)) def test_alter_table_add_column(self) -> None: try: spark.sql(f"ALTER TABLE {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME} ADD COLUMN extra INT") except UnsupportedOperationException as error: assert("Altering a table is not supported yet" in str(error)) def test_alter_table_set_tbl_properties(self) -> None: try: spark.sql(f"ALTER TABLE {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME} " f"SET TBLPROPERTIES ('customProp' = 'customValue')") except UnsupportedOperationException as error: assert("Altering a table is not supported yet" in str(error)) description = spark.sql(f"DESCRIBE EXTENDED {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME}")\ .filter("col_name = 'Table Properties'").collect()[0][1] assert("customProp" not in description) class UnityCatalogManagedTableUtilitySuite(UnityCatalogManagedTableTestBase): """ Suite covering utility operations on a managed table in Unity Catalog: OPTIMIZE, ANALYZE, VACUUM, ... """ def test_optimize(self) -> None: spark.sql(f"OPTIMIZE {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME}") updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(1, ), (2, ), (3, )])) def test_optimize_sql(self) -> None: spark.sql(f"OPTIMIZE {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME}") updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(1, ), (2, ), (3, )])) def test_zorder_by(self) -> None: spark.sql(f"OPTIMIZE {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME} ZORDER BY (id)") updated_tbl = self.read(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).toDF("id") assertDataFrameEqual(updated_tbl, self.create_df_with_rows([(1, ), (2, ), (3, )])) def test_analyze(self) -> None: try: spark.sql(f"ANALYZE TABLE {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME} COMPUTE STATISTICS") except AnalysisException as error: assert( "[NOT_SUPPORTED_COMMAND_FOR_V2_TABLE] ANALYZE TABLE is not supported for v2 tables." in str(error) ) def test_describe_table(self) -> None: description = spark.sql(f"DESCRIBE {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME}").collect() expected = spark.createDataFrame([("id", "int", None)], "col_name string, data_type string, comment string") assertDataFrameEqual(description, expected) def test_history(self) -> None: try: # DESCRIBE HISTORY is currently unsupported on catalog owned tables. self.get_table_history(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).collect() except py4j.protocol.Py4JJavaError as error: assert("catalog-managed" in str(error).lower()) def test_vacuum(self) -> None: try: # VACUUM is currently unsupported on catalog owned tables. spark.sql(f"VACUUM {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME}") except UnsupportedOperationException as error: assert("DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION" in str(error)) def test_restore(self) -> None: # Intentionally add a new data change commit. self.append(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME) try: current_version = self.current_version(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME) # Restore is currently unsupported on catalog owned tables. spark.sql(f"RESTORE TABLE {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME} TO " f"VERSION AS OF {current_version-1}") except py4j.protocol.Py4JJavaError as error: assert("UPDATE_DELTA_METADATA" in str(error)) class UnityCatalogManagedTableReadSuite(UnityCatalogManagedTableTestBase): """ Suite covering reading from a managed table in Unity Catalog/ """ def test_time_travel_read(self) -> None: dt = DeltaTable.forName(spark, MANAGED_CATALOG_OWNED_TABLE_FULL_NAME) current_version = dt.history().selectExpr("max(version)").collect()[0][0] current_timestamp = str(datetime.datetime.now()) self.append(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME) result = spark.read.option("timestampAsOf", current_timestamp)\ .table(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME) assertDataFrameEqual(result, self.setup_df) result = spark.read.option("versionAsOf", current_version)\ .table(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME) assertDataFrameEqual(result, self.setup_df) result = spark.sql(f"SELECT * FROM {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME} " f"TIMESTAMP AS OF '{current_timestamp}'") assertDataFrameEqual(result, self.setup_df) result = spark.sql(f"SELECT * FROM {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME} " f"VERSION AS OF {current_version}") assertDataFrameEqual(result, self.setup_df) # CDC (Timestamps, Versions) are currently unsupported for Catalog owned tables. def test_change_data_feed_with_timestamp(self) -> None: timestamp = str(datetime.datetime.now()) self.append(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME) try: self.read_with_cdf_timestamp( timestamp, MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).select("id", "_change_type") except py4j.protocol.Py4JJavaError as error: assert("Path based access is not supported for Catalog-Owned table" in str(error)) def test_change_data_feed_with_version(self) -> None: # Intentionally add a new data change commit. self.append(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME) try: current_version = self.current_version(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME) self.read_with_cdf_version( current_version - 1, MANAGED_CATALOG_OWNED_TABLE_FULL_NAME).select("id", "_change_type") except py4j.protocol.Py4JJavaError as error: assert("UPDATE_DELTA_METADATA" in str(error)) def test_delta_table_for_path(self) -> None: tbl_path = spark.sql( f"DESCRIBE formatted {MANAGED_CATALOG_OWNED_TABLE_FULL_NAME}").collect()[5].data_type try: DeltaTable.forPath(spark, tbl_path) except py4j.protocol.Py4JJavaError as error: # Path-based access isn't supported. This could throw a better error than just # 'access denied' though. assert("AccessDeniedException" in str(error)) def test_streaming_read(self) -> None: try: spark.readStream\ .table(MANAGED_CATALOG_OWNED_TABLE_FULL_NAME)\ .writeStream\ .option("checkpointLocation", "test")\ .toTable("output_table") except py4j.protocol.Py4JJavaError as error: # Streaming from a catalog owned table fails as it attempts to access the table by path. # This could also throw a better error than jsut 'access denied'. assert("AccessDeniedException" in str(error)) if __name__ == "__main__": """ Change this to select tests to run, for example: - '__main__': all tests in this file. - '__main__.UnityCatalogManagedTableDMLSuite': all tests in that single suites. - '__main__.UnityCatalogManagedTableDDLSuite.test_sql_create': only that single test. """ test_name = "__main__" suite = unittest.TestLoader().loadTestsFromName(test_name) unittest.TextTestRunner(verbosity=2).run(suite) ================================================ FILE: python/delta/pip_utils.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from typing import List, Optional from pyspark.sql import SparkSession def configure_spark_with_delta_pip( spark_session_builder: SparkSession.Builder, extra_packages: Optional[List[str]] = None ) -> SparkSession.Builder: """ Utility function to configure a SparkSession builder such that the generated SparkSession will automatically download the required Delta Lake JARs from Maven. This function is required when you want to 1. Install Delta Lake locally using pip, and 2. Execute your Python code using Delta Lake + Pyspark directly, that is, not using `spark-submit --packages io.delta:...` or `pyspark --packages io.delta:...`. builder = SparkSession.builder \ .master("local[*]") \ .appName("test") spark = configure_spark_with_delta_pip(builder).getOrCreate() 3. If you would like to add more packages, use the `extra_packages` parameter. builder = SparkSession.builder \ .master("local[*]") \ .appName("test") my_packages = ["org.apache.spark:spark-sql-kafka-0-10_2.12:x.y.z"] spark = configure_spark_with_delta_pip(builder, extra_packages=my_packages).getOrCreate() :param spark_session_builder: SparkSession.Builder object being used to configure and create a SparkSession. :param extra_packages: Set other packages to add to Spark session besides Delta Lake. :return: Updated SparkSession.Builder object .. versionadded:: 1.0 .. note:: Evolving """ import importlib_metadata # load this library only when this function is called if type(spark_session_builder) is not SparkSession.Builder: msg = f''' This function must be called with a SparkSession builder as the argument. The argument found is of type {str(type(spark_session_builder))}. See the online documentation for the correct usage of this function. ''' raise TypeError(msg) try: delta_version = importlib_metadata.version("delta_spark") except Exception as e: msg = ''' This function can be used only when Delta Lake has been locally installed with pip. See the online documentation for the correct usage of this function. ''' raise Exception(msg) from e # Get Spark version from pyspark module import pyspark spark_version = pyspark.__version__ scala_version = "2.13" # Determine the Spark major.minor version for artifact name # Artifact names include Spark version suffix when spark_version is known # (e.g., delta-spark_4.0_2.13). Falls back to no suffix for backward compatibility. if spark_version: spark_major_minor = ".".join(spark_version.split(".")[:2]) # e.g., "4.0" or "4.1" artifact_name = f"delta-spark_{spark_major_minor}_{scala_version}" else: # Fallback to artifact without suffix for backward compatibility artifact_name = f"delta-spark_{scala_version}" maven_artifact = f"io.delta:{artifact_name}:{delta_version}" extra_packages = extra_packages if extra_packages is not None else [] all_artifacts = [maven_artifact] + extra_packages packages_str = ",".join(all_artifacts) return spark_session_builder.config("spark.jars.packages", packages_str) ================================================ FILE: python/delta/py.typed ================================================ ================================================ FILE: python/delta/tables.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from dataclasses import dataclass from typing import ( TYPE_CHECKING, cast, overload, Any, Dict, Iterable, Optional, Union, NoReturn, List, Tuple ) from delta._typing import ( ColumnMapping, OptionalColumnMapping, ExpressionOrColumn, OptionalExpressionOrColumn ) from pyspark import since from pyspark.sql import Column, DataFrame, functions, SparkSession from pyspark.sql.types import DataType, StructType, StructField from pyspark.sql.utils import is_remote if TYPE_CHECKING: from py4j.java_gateway import JavaObject, JVMView # type: ignore[import] from py4j.java_collections import JavaMap # type: ignore[import] class DeltaTable(object): """ Main class for programmatically interacting with Delta tables. You can create DeltaTable instances using the path of the Delta table.:: deltaTable = DeltaTable.forPath(spark, "/path/to/table") In addition, you can convert an existing Parquet table in place into a Delta table.:: deltaTable = DeltaTable.convertToDelta(spark, "parquet.`/path/to/table`") .. versionadded:: 0.4 """ def __init__(self, spark: SparkSession, jdt: "JavaObject"): self._spark = spark self._jdt = jdt @since(0.4) # type: ignore[arg-type] def toDF(self) -> DataFrame: """ Get a DataFrame representation of this Delta table. """ return DataFrame( self._jdt.toDF(), # Simple trick to avoid warnings from Spark 3.3.0. `_wrapped` # in SparkSession is removed in Spark 3.3.0, see also SPARK-38121. getattr(self._spark, "_wrapped", self._spark) # type: ignore[attr-defined] ) @since(0.4) # type: ignore[arg-type] def alias(self, aliasName: str) -> "DeltaTable": """ Apply an alias to the Delta table. """ jdt = self._jdt.alias(aliasName) return DeltaTable(self._spark, jdt) @since(0.5) # type: ignore[arg-type] def generate(self, mode: str) -> None: """ Generate manifest files for the given delta table. :param mode: mode for the type of manifest file to be generated The valid modes are as follows (not case sensitive): - "symlink_format_manifest": This will generate manifests in symlink format for Presto and Athena read support. See the online documentation for more information. """ self._jdt.generate(mode) @since(0.4) # type: ignore[arg-type] def delete(self, condition: OptionalExpressionOrColumn = None) -> None: """ Delete data from the table that match the given ``condition``. Example:: deltaTable.delete("date < '2017-01-01'") # predicate using SQL formatted string deltaTable.delete(col("date") < "2017-01-01") # predicate using Spark SQL functions :param condition: condition of the update :type condition: str or pyspark.sql.Column """ if condition is None: self._jdt.delete() else: self._jdt.delete(DeltaTable._condition_to_jcolumn(condition)) @overload def update( self, condition: ExpressionOrColumn, set: ColumnMapping ) -> None: ... @overload def update(self, *, set: ColumnMapping) -> None: ... def update( self, condition: OptionalExpressionOrColumn = None, set: OptionalColumnMapping = None ) -> None: """ Update data from the table on the rows that match the given ``condition``, which performs the rules defined by ``set``. Example:: # condition using SQL formatted string deltaTable.update( condition = "eventType = 'clck'", set = { "eventType": "'click'" } ) # condition using Spark SQL functions deltaTable.update( condition = col("eventType") == "clck", set = { "eventType": lit("click") } ) :param condition: Optional condition of the update :type condition: str or pyspark.sql.Column :param set: Defines the rules of setting the values of columns that need to be updated. *Note: This param is required.* Default value None is present to allow positional args in same order across languages. :type set: dict with str as keys and str or pyspark.sql.Column as values .. versionadded:: 0.4 """ jmap = DeltaTable._dict_to_jmap(self._spark, set, "'set'") jcolumn = DeltaTable._condition_to_jcolumn(condition) if condition is None: self._jdt.update(jmap) else: self._jdt.update(jcolumn, jmap) @since(0.4) # type: ignore[arg-type] def merge( self, source: DataFrame, condition: ExpressionOrColumn ) -> "DeltaMergeBuilder": """ Merge data from the `source` DataFrame based on the given merge `condition`. This returns a :class:`DeltaMergeBuilder` object that can be used to specify the update, delete, or insert actions to be performed on rows based on whether the rows matched the condition or not. See :class:`DeltaMergeBuilder` for a full description of this operation and what combinations of update, delete and insert operations are allowed. Example 1 with conditions and update expressions as SQL formatted string:: deltaTable.alias("events").merge( source = updatesDF.alias("updates"), condition = "events.eventId = updates.eventId" ).whenMatchedUpdate(set = { "data": "updates.data", "count": "events.count + 1" } ).whenNotMatchedInsert(values = { "date": "updates.date", "eventId": "updates.eventId", "data": "updates.data", "count": "1" } ).execute() Example 2 with conditions and update expressions as Spark SQL functions:: from pyspark.sql.functions import * deltaTable.alias("events").merge( source = updatesDF.alias("updates"), condition = expr("events.eventId = updates.eventId") ).whenMatchedUpdate(set = { "data" : col("updates.data"), "count": col("events.count") + 1 } ).whenNotMatchedInsert(values = { "date": col("updates.date"), "eventId": col("updates.eventId"), "data": col("updates.data"), "count": lit("1") } ).execute() :param source: Source DataFrame :type source: pyspark.sql.DataFrame :param condition: Condition to match sources rows with the Delta table rows. :type condition: str or pyspark.sql.Column :return: builder object to specify whether to update, delete or insert rows based on whether the condition matched or not :rtype: :py:class:`delta.tables.DeltaMergeBuilder` """ if source is None: raise ValueError("'source' in merge cannot be None") elif not isinstance(source, DataFrame): raise TypeError("Type of 'source' in merge must be DataFrame.") if condition is None: raise ValueError("'condition' in merge cannot be None") jbuilder = self._jdt.merge(source._jdf, DeltaTable._condition_to_jcolumn(condition)) return DeltaMergeBuilder(self._spark, jbuilder) @since(0.4) # type: ignore[arg-type] def vacuum(self, retentionHours: Optional[float] = None) -> DataFrame: """ Recursively delete files and directories in the table that are not needed by the table for maintaining older versions up to the given retention threshold. This method will return an empty DataFrame on successful completion. Example:: deltaTable.vacuum() # vacuum files not required by versions more than 7 days old deltaTable.vacuum(100) # vacuum files not required by versions more than 100 hours old :param retentionHours: Optional number of hours retain history. If not specified, then the default retention period of 168 hours (7 days) will be used. """ jdt = self._jdt if retentionHours is None: return DataFrame( jdt.vacuum(), getattr(self._spark, "_wrapped", self._spark) # type: ignore[attr-defined] ) else: return DataFrame( jdt.vacuum(float(retentionHours)), getattr(self._spark, "_wrapped", self._spark) # type: ignore[attr-defined] ) @since(0.4) # type: ignore[arg-type] def history(self, limit: Optional[int] = None) -> DataFrame: """ Get the information of the latest `limit` commits on this table as a Spark DataFrame. The information is in reverse chronological order. Example:: fullHistoryDF = deltaTable.history() # get the full history of the table lastOperationDF = deltaTable.history(1) # get the last operation :param limit: Optional, number of latest commits to returns in the history. :return: Table's commit history. See the online Delta Lake documentation for more details. :rtype: pyspark.sql.DataFrame """ jdt = self._jdt if limit is None: return DataFrame( jdt.history(), getattr(self._spark, "_wrapped", self._spark) # type: ignore[attr-defined] ) else: return DataFrame( jdt.history(limit), getattr(self._spark, "_wrapped", self._spark) # type: ignore[attr-defined] ) @since(2.1) # type: ignore[arg-type] def detail(self) -> DataFrame: """ Get the details of a Delta table such as the format, name, and size. Example:: detailDF = deltaTable.detail() # get the full details of the table :return Information of the table (format, name, size, etc.) :rtype: pyspark.sql.DataFrame .. note:: Evolving """ return DataFrame( self._jdt.detail(), getattr(self._spark, "_wrapped", self._spark) # type: ignore[attr-defined] ) @classmethod @since(0.4) # type: ignore[arg-type] def convertToDelta( cls, sparkSession: SparkSession, identifier: str, partitionSchema: Optional[Union[str, StructType]] = None ) -> "DeltaTable": """ Create a DeltaTable from the given parquet table. Takes an existing parquet table and constructs a delta transaction log in the base path of the table. Note: Any changes to the table during the conversion process may not result in a consistent state at the end of the conversion. Users should stop any changes to the table before the conversion is started. Example:: # Convert unpartitioned parquet table at path 'path/to/table' deltaTable = DeltaTable.convertToDelta( spark, "parquet.`path/to/table`") # Convert partitioned parquet table at path 'path/to/table' and partitioned by # integer column named 'part' partitionedDeltaTable = DeltaTable.convertToDelta( spark, "parquet.`path/to/table`", "part int") :param sparkSession: SparkSession to use for the conversion :type sparkSession: pyspark.sql.SparkSession :param identifier: Parquet table identifier formatted as "parquet.`path`" :type identifier: str :param partitionSchema: Hive DDL formatted string, or pyspark.sql.types.StructType :return: DeltaTable representing the converted Delta table :rtype: :py:class:`~delta.tables.DeltaTable` """ assert sparkSession is not None if is_remote(): from pyspark.sql.connect.session import SparkSession as RemoteSparkSession if isinstance(sparkSession, RemoteSparkSession): from delta.connect.tables import DeltaTable as RemoteDeltaTable return RemoteDeltaTable.convertToDelta(sparkSession, identifier, partitionSchema) jvm: "JVMView" = sparkSession._sc._jvm # type: ignore[attr-defined] jsparkSession: "JavaObject" = sparkSession._jsparkSession # type: ignore[attr-defined] if partitionSchema is None: jdt = jvm.io.delta.tables.DeltaTable.convertToDelta( jsparkSession, identifier ) else: if not isinstance(partitionSchema, str): partitionSchema = jsparkSession.parseDataType(partitionSchema.json()) jdt = jvm.io.delta.tables.DeltaTable.convertToDelta( jsparkSession, identifier, partitionSchema) return DeltaTable(sparkSession, jdt) @classmethod @since(0.4) # type: ignore[arg-type] def forPath( cls, sparkSession: SparkSession, path: str, hadoopConf: Dict[str, str] = dict() ) -> "DeltaTable": """ Instantiate a :class:`DeltaTable` object representing the data at the given path, If the given path is invalid (i.e. either no table exists or an existing table is not a Delta table), it throws a `not a Delta table` error. :param sparkSession: SparkSession to use for loading the table :type sparkSession: pyspark.sql.SparkSession :param hadoopConf: Hadoop configuration starting with "fs." or "dfs." will be picked up by `DeltaTable` to access the file system when executing queries. Other configurations will not be allowed. :type hadoopConf: optional dict with str as key and str as value. :return: loaded Delta table :rtype: :py:class:`~delta.tables.DeltaTable` Example:: hadoopConf = {"fs.s3a.access.key" : "", "fs.s3a.secret.key": "secret-key"} deltaTable = DeltaTable.forPath( spark, "/path/to/table", hadoopConf) """ assert sparkSession is not None if is_remote(): from pyspark.sql.connect.session import SparkSession as RemoteSparkSession if isinstance(sparkSession, RemoteSparkSession): from delta.connect.tables import DeltaTable as RemoteDeltaTable return RemoteDeltaTable.forPath(sparkSession, path, hadoopConf) jvm: "JVMView" = sparkSession._sc._jvm # type: ignore[attr-defined] jsparkSession: "JavaObject" = sparkSession._jsparkSession # type: ignore[attr-defined] jdt = jvm.io.delta.tables.DeltaTable.forPath(jsparkSession, path, hadoopConf) return DeltaTable(sparkSession, jdt) @classmethod @since(0.7) # type: ignore[arg-type] def forName( cls, sparkSession: SparkSession, tableOrViewName: str ) -> "DeltaTable": """ Instantiate a :class:`DeltaTable` object using the given table name. If the given tableOrViewName is invalid (i.e. either no table exists or an existing table is not a Delta table), it throws a `not a Delta table` error. Note: Passing a view name will also result in this error as views are not supported. The given tableOrViewName can also be the absolute path of a delta datasource (i.e. delta.`path`), If so, instantiate a :class:`DeltaTable` object representing the data at the given path (consistent with the `forPath`). :param sparkSession: SparkSession to use for loading the table :param tableOrViewName: name of the table or view :return: loaded Delta table :rtype: :py:class:`~delta.tables.DeltaTable` Example:: deltaTable = DeltaTable.forName(spark, "tblName") """ assert sparkSession is not None if is_remote(): from pyspark.sql.connect.session import SparkSession as RemoteSparkSession if isinstance(sparkSession, RemoteSparkSession): from delta.connect.tables import DeltaTable as RemoteDeltaTable return RemoteDeltaTable.forName(sparkSession, tableOrViewName) jvm: "JVMView" = sparkSession._sc._jvm # type: ignore[attr-defined] jsparkSession: "JavaObject" = sparkSession._jsparkSession # type: ignore[attr-defined] jdt = jvm.io.delta.tables.DeltaTable.forName(jsparkSession, tableOrViewName) return DeltaTable(sparkSession, jdt) @classmethod @since(1.0) # type: ignore[arg-type] def create( cls, sparkSession: Optional[SparkSession] = None ) -> "DeltaTableBuilder": """ Return :class:`DeltaTableBuilder` object that can be used to specify the table name, location, columns, partitioning columns, table comment, and table properties to create a Delta table, error if the table exists (the same as SQL `CREATE TABLE`). See :class:`DeltaTableBuilder` for a full description and examples of this operation. :param sparkSession: SparkSession to use for creating the table :return: an instance of DeltaTableBuilder :rtype: :py:class:`~delta.tables.DeltaTableBuilder` .. note:: Evolving """ if sparkSession is None: sparkSession = SparkSession.getActiveSession() assert sparkSession is not None if is_remote(): from pyspark.sql.connect.session import SparkSession as RemoteSparkSession if isinstance(sparkSession, RemoteSparkSession): from delta.connect.tables import DeltaTable as RemoteDeltaTable return RemoteDeltaTable.create(sparkSession) jvm: "JVMView" = sparkSession._sc._jvm # type: ignore[attr-defined] jsparkSession: "JavaObject" = sparkSession._jsparkSession # type: ignore[attr-defined] jdt = jvm.io.delta.tables.DeltaTable.create(jsparkSession) return DeltaTableBuilder(sparkSession, jdt) @classmethod @since(1.0) # type: ignore[arg-type] def createIfNotExists( cls, sparkSession: Optional[SparkSession] = None ) -> "DeltaTableBuilder": """ Return :class:`DeltaTableBuilder` object that can be used to specify the table name, location, columns, partitioning columns, table comment, and table properties to create a Delta table, if it does not exists (the same as SQL `CREATE TABLE IF NOT EXISTS`). See :class:`DeltaTableBuilder` for a full description and examples of this operation. :param sparkSession: SparkSession to use for creating the table :return: an instance of DeltaTableBuilder :rtype: :py:class:`~delta.tables.DeltaTableBuilder` .. note:: Evolving """ if sparkSession is None: sparkSession = SparkSession.getActiveSession() assert sparkSession is not None if is_remote(): from pyspark.sql.connect.session import SparkSession as RemoteSparkSession if isinstance(sparkSession, RemoteSparkSession): from delta.connect.tables import DeltaTable as RemoteDeltaTable return RemoteDeltaTable.createIfNotExists(sparkSession) jvm: "JVMView" = sparkSession._sc._jvm # type: ignore[attr-defined] jsparkSession: "JavaObject" = sparkSession._jsparkSession # type: ignore[attr-defined] jdt = jvm.io.delta.tables.DeltaTable.createIfNotExists(jsparkSession) return DeltaTableBuilder(sparkSession, jdt) @classmethod @since(1.0) # type: ignore[arg-type] def replace( cls, sparkSession: Optional[SparkSession] = None ) -> "DeltaTableBuilder": """ Return :class:`DeltaTableBuilder` object that can be used to specify the table name, location, columns, partitioning columns, table comment, and table properties to replace a Delta table, error if the table doesn't exist (the same as SQL `REPLACE TABLE`). See :class:`DeltaTableBuilder` for a full description and examples of this operation. :param sparkSession: SparkSession to use for creating the table :return: an instance of DeltaTableBuilder :rtype: :py:class:`~delta.tables.DeltaTableBuilder` .. note:: Evolving """ if sparkSession is None: sparkSession = SparkSession.getActiveSession() assert sparkSession is not None if is_remote(): from pyspark.sql.connect.session import SparkSession as RemoteSparkSession if isinstance(sparkSession, RemoteSparkSession): from delta.connect.tables import DeltaTable as RemoteDeltaTable return RemoteDeltaTable.replace(sparkSession) jvm: "JVMView" = sparkSession._sc._jvm # type: ignore[attr-defined] jsparkSession: "JavaObject" = sparkSession._jsparkSession # type: ignore[attr-defined] jdt = jvm.io.delta.tables.DeltaTable.replace(jsparkSession) return DeltaTableBuilder(sparkSession, jdt) @classmethod @since(1.0) # type: ignore[arg-type] def createOrReplace( cls, sparkSession: Optional[SparkSession] = None ) -> "DeltaTableBuilder": """ Return :class:`DeltaTableBuilder` object that can be used to specify the table name, location, columns, partitioning columns, table comment, and table properties replace a Delta table, error if the table doesn't exist (the same as SQL `REPLACE TABLE`). See :class:`DeltaTableBuilder` for a full description and examples of this operation. :param sparkSession: SparkSession to use for creating the table :return: an instance of DeltaTableBuilder :rtype: :py:class:`~delta.tables.DeltaTableBuilder` .. note:: Evolving """ if sparkSession is None: sparkSession = SparkSession.getActiveSession() assert sparkSession is not None if is_remote(): from pyspark.sql.connect.session import SparkSession as RemoteSparkSession if isinstance(sparkSession, RemoteSparkSession): from delta.connect.tables import DeltaTable as RemoteDeltaTable return RemoteDeltaTable.createOrReplace(sparkSession) jvm: "JVMView" = sparkSession._sc._jvm # type: ignore[attr-defined] jsparkSession: "JavaObject" = sparkSession._jsparkSession # type: ignore[attr-defined] jdt = jvm.io.delta.tables.DeltaTable.createOrReplace(jsparkSession) return DeltaTableBuilder(sparkSession, jdt) @classmethod @since(0.4) # type: ignore[arg-type] def isDeltaTable(cls, sparkSession: SparkSession, identifier: str) -> bool: """ Check if the provided `identifier` string, in this case a file path, is the root of a Delta table using the given SparkSession. :param sparkSession: SparkSession to use to perform the check :param path: location of the table :return: If the table is a delta table or not :rtype: bool Example:: DeltaTable.isDeltaTable(spark, "/path/to/table") """ assert sparkSession is not None if is_remote(): from pyspark.sql.connect.session import SparkSession as RemoteSparkSession if isinstance(sparkSession, RemoteSparkSession): from delta.connect.tables import DeltaTable as RemoteDeltaTable return RemoteDeltaTable.isDeltaTable(sparkSession, identifier) jvm: "JVMView" = sparkSession._sc._jvm # type: ignore[attr-defined] jsparkSession: "JavaObject" = sparkSession._jsparkSession # type: ignore[attr-defined] return jvm.io.delta.tables.DeltaTable.isDeltaTable(jsparkSession, identifier) @since(0.8) # type: ignore[arg-type] def upgradeTableProtocol(self, readerVersion: int, writerVersion: int) -> None: """ Updates the protocol version of the table to leverage new features. Upgrading the reader version will prevent all clients that have an older version of Delta Lake from accessing this table. Upgrading the writer version will prevent older versions of Delta Lake to write to this table. The reader or writer version cannot be downgraded. See online documentation and Delta's protocol specification at PROTOCOL.md for more details. """ jdt = self._jdt if not isinstance(readerVersion, int): raise ValueError("The readerVersion needs to be an integer but got '%s'." % type(readerVersion)) if not isinstance(writerVersion, int): raise ValueError("The writerVersion needs to be an integer but got '%s'." % type(writerVersion)) jdt.upgradeTableProtocol(readerVersion, writerVersion) @since(3.3) # type: ignore[arg-type] def addFeatureSupport(self, featureName: str) -> None: """ Modify the protocol to add a supported feature, and if the table does not support table features, upgrade the protocol automatically. In such a case when the provided feature is writer-only, the table's writer version will be upgraded to `7`, and when the provided feature is reader-writer, both reader and writer versions will be upgraded, to `(3, 7)`. See online documentation and Delta's protocol specification at PROTOCOL.md for more details. """ DeltaTable._verify_type_str(featureName, "featureName") self._jdt.addFeatureSupport(featureName) @since(3.4) # type: ignore[arg-type] def dropFeatureSupport(self, featureName: str, truncateHistory: Optional[bool] = None) -> None: """ Modify the protocol to drop a supported feature. The operation always normalizes the resulting protocol. Protocol normalization is the process of converting a table features protocol to the weakest possible form. This primarily refers to converting a table features protocol to a legacy protocol. A table features protocol can be represented with the legacy representation only when the feature set of the former exactly matches a legacy protocol. Normalization can also decrease the reader version of a table features protocol when it is higher than necessary. For example: (1, 7, None, {AppendOnly, Invariants, CheckConstraints}) -> (1, 3) (3, 7, None, {RowTracking}) -> (1, 7, RowTracking) The dropFeatureSupport method can be used as follows: delta.tables.DeltaTable.dropFeatureSupport("rowTracking") :param featureName: The name of the feature to drop. :param truncateHistory: Optional value whether to truncate history. If not specified, the history is not truncated. :return: None. """ DeltaTable._verify_type_str(featureName, "featureName") if truncateHistory is None: self._jdt.dropFeatureSupport(featureName) else: DeltaTable._verify_type_bool(truncateHistory, "truncateHistory") self._jdt.dropFeatureSupport(featureName, truncateHistory) @since(1.2) # type: ignore[arg-type] def restoreToVersion(self, version: int) -> DataFrame: """ Restore the DeltaTable to an older version of the table specified by version number. Example:: delta.tables.DeltaTable.restoreToVersion(1) :param version: target version of restored table :return: Dataframe with metrics of restore operation. :rtype: pyspark.sql.DataFrame """ DeltaTable._verify_type_int(version, "version") return DataFrame( self._jdt.restoreToVersion(version), getattr(self._spark, "_wrapped", self._spark) # type: ignore[attr-defined] ) @since(1.2) # type: ignore[arg-type] def restoreToTimestamp(self, timestamp: str) -> DataFrame: """ Restore the DeltaTable to an older version of the table specified by a timestamp. Timestamp can be of the format yyyy-MM-dd or yyyy-MM-dd HH:mm:ss Example:: delta.tables.DeltaTable.restoreToTimestamp('2021-01-01') delta.tables.DeltaTable.restoreToTimestamp('2021-01-01 01:01:01') :param timestamp: target timestamp of restored table :return: Dataframe with metrics of restore operation. :rtype: pyspark.sql.DataFrame """ DeltaTable._verify_type_str(timestamp, "timestamp") return DataFrame( self._jdt.restoreToTimestamp(timestamp), getattr(self._spark, "_wrapped", self._spark) # type: ignore[attr-defined] ) @since(2.0) # type: ignore[arg-type] def optimize(self) -> "DeltaOptimizeBuilder": """ Optimize the data layout of the table. This returns a :py:class:`~delta.tables.DeltaOptimizeBuilder` object that can be used to specify the partition filter to limit the scope of optimize and also execute different optimization techniques such as file compaction or order data using Z-Order curves. See the :py:class:`~delta.tables.DeltaOptimizeBuilder` for a full description of this operation. Example:: deltaTable.optimize().where("date='2021-11-18'").executeCompaction() :return: an instance of DeltaOptimizeBuilder. :rtype: :py:class:`~delta.tables.DeltaOptimizeBuilder` """ jbuilder = self._jdt.optimize() return DeltaOptimizeBuilder(self._spark, jbuilder) def clone( # type: ignore[no-untyped-def] self, target, isShallow=False, replace=False, properties=None ) -> "DeltaTable": """ Clone the latest state of a DeltaTable to a destination which mirrors the existing table's data and metadata at that version. Example:: # Shallow clone a table to path '/path/to/table' deltaTable = DeltaTable.clone("/path/to/table", False, True) :param self: The current instance :type self: :py:class:`~delta.tables.DeltaTable` :param target: Path where we should clone the Delta table :type target: str :param isShallow: True for shallow clones, false for deep clones :type isShallow: bool :param replace: True if the desired behavior is to overwrite the target table if one exists otherwise throw an error if table exists at the target :type replace: bool :param properties: user-defined table properties that should override any properties with the same key from the source table :type properties: dict :rtype: :py:class:`~delta.tables.DeltaTable` """ DeltaTable._verify_clone_types(target, isShallow, replace, properties) return self._jdt.clone(target, isShallow, replace, properties) def cloneAtVersion( # type: ignore[no-untyped-def] self, version, target, isShallow=False, replace=False, properties=None ) -> "DeltaTable": """ Clone a DeltaTable at the given version to a destination which mirrors the existing table's data and metadata at that version. Example:: # Shallow clone a table to path '/path/to/table' at version 1 deltaTable = DeltaTable.cloneAtVersion(1, "/path/to/table", False) :param self: The current instance :type self: :py:class:`~delta.tables.DeltaTable` :param version: Version at which to clone the source directory. Take the metadata at this version of the table as well. :type version: number :param target: Path where we should clone the Delta table :type target: str :param isShallow: True for shallow clones, false for deep clones :type isShallow: bool :param replace: True if the desired behavior is to overwrite the target table if one exists otherwise throw an error if table exists at the target :type replace: bool :param properties: user-defined table properties that should override any properties with the same key from the source table :type properties: dict :rtype: :py:class:`~delta.tables.DeltaTable` """ DeltaTable._verify_clone_types(target, isShallow, replace, properties, version=version) return self._jdt.cloneAtVersion(version, target, isShallow, replace, properties) def cloneAtTimestamp( # type: ignore[no-untyped-def] self, timestamp, target, isShallow=False, replace=False, properties=None ) -> "DeltaTable": """ Clone a DeltaTable at the given timestamp to a destination which mirrors the existing table's data and metadata at that timestamp. Example:: # Shallow clone a table to path '/path/to/table' at time of format yyyy-MM-dd'T'HH:mm:ss # or yyyy-MM-dd deltaTable = DeltaTable.cloneAtTimestamp( "2019-01-01", "/path/to/table", False) :param self: The current instance :type self: :py:class:`~delta.tables.DeltaTable` :param timestamp: Timestamp at which to clone the source directory. Take the metadata at this timestamp as well. :type timestamp: str :param target: Path where we should clone the Delta table :type target: str :param isShallow: True for shallow clones, false for deep clones :type isShallow: bool :param replace: True if the desired behavior is to overwrite the target table if one exists otherwise throw an error if table exists at the target :type replace: bool :param properties: user-defined table properties that should override any properties with the same key from the source table :type properties: dict :rtype: :py:class:`~delta.tables.DeltaTable` """ DeltaTable._verify_clone_types(target, isShallow, replace, properties, timestamp) return self._jdt.cloneAtTimestamp(timestamp, target, isShallow, replace, properties) @classmethod def _verify_clone_types( self, target: str, isShallow: bool, replace: bool, properties: dict, timestamp: str = "", version: int = 0 ) -> None: """ Throw an error if any of the types passed in to Clone do not adhere to the types that we expect """ DeltaTable._verify_type_str(timestamp, "timestamp") DeltaTable._verify_type_int(version, "version") DeltaTable._verify_type_str(target, "target") DeltaTable._verify_type_bool(isShallow, "isShallow") DeltaTable._verify_type_bool(replace, "replace") if properties is not None: DeltaTable._verify_type_dict(properties, "properties") for property, value in properties.items(): DeltaTable._verify_type_str(property, "All property keys including %s" % property) DeltaTable._verify_type_str(value, "All property values including %s" % value) @classmethod def _verify_type_dict(cls, variable: dict, name: str) -> None: if not isinstance(variable, dict): raise ValueError("%s needs to be a dict but got '%s'." % (name, type(variable))) @classmethod # type: ignore[arg-type] def _verify_type_bool(self, variable: bool, name: str) -> None: if not isinstance(variable, bool) or variable is None: raise ValueError("%s needs to be a boolean but got '%s'." % (name, type(variable))) @staticmethod # type: ignore[arg-type] def _verify_type_str(variable: str, name: str) -> None: if not isinstance(variable, str) or variable is None: raise ValueError("%s needs to be a string but got '%s'." % (name, type(variable))) @staticmethod # type: ignore[arg-type] def _verify_type_int(variable: int, name: str) -> None: if not isinstance(variable, int) or variable is None: raise ValueError("%s needs to be an int but got '%s'." % (name, type(variable))) @staticmethod def _dict_to_jmap( sparkSession: SparkSession, pydict: OptionalColumnMapping, argname: str, ) -> "JavaObject": """ convert dict to Map """ # Get the Java map for pydict if pydict is None: raise ValueError("%s cannot be None" % argname) elif type(pydict) is not dict: e = "%s must be a dict, found to be %s" % (argname, str(type(pydict))) raise TypeError(e) jvm: "JVMView" = sparkSession._sc._jvm # type: ignore[attr-defined] jmap: "JavaMap" = jvm.java.util.HashMap() for col, expr in pydict.items(): if type(col) is not str: e = ("Keys of dict in %s must contain only strings with column names" % argname) + \ (", found '%s' of type '%s" % (str(col), str(type(col)))) raise TypeError(e) if isinstance(expr, Column) and hasattr(expr, "_jc"): jmap.put(col, expr._jc) elif type(expr) is str: jmap.put(col, functions.expr(expr)._jc) else: e = ("Values of dict in %s must contain only Spark SQL Columns " % argname) + \ "or strings (expressions in SQL syntax) as values, " + \ ("found '%s' of type '%s'" % (str(expr), str(type(expr)))) raise TypeError(e) return jmap @staticmethod def _condition_to_jcolumn( condition: OptionalExpressionOrColumn, argname: str = "'condition'" ) -> "JavaObject": if condition is None: jcondition = None elif isinstance(condition, Column) and hasattr(condition, "_jc"): jcondition = condition._jc elif type(condition) is str: jcondition = functions.expr(condition)._jc else: e = ("%s must be a Spark SQL Column or a string (expression in SQL syntax)" % argname) \ + ", found to be of type %s" % str(type(condition)) raise TypeError(e) return jcondition class DeltaMergeBuilder(object): """ Builder to specify how to merge data from source DataFrame into the target Delta table. Use :py:meth:`delta.tables.DeltaTable.merge` to create an object of this class. Using this builder, you can specify any number of ``whenMatched``, ``whenNotMatched`` and ``whenNotMatchedBySource`` clauses. Here are the constraints on these clauses. - Constraints in the ``whenMatched`` clauses: - The condition in a ``whenMatched`` clause is optional. However, if there are multiple ``whenMatched`` clauses, then only the last one may omit the condition. - When there are more than one ``whenMatched`` clauses and there are conditions (or the lack of) such that a row satisfies multiple clauses, then the action for the first clause satisfied is executed. In other words, the order of the ``whenMatched`` clauses matters. - If none of the ``whenMatched`` clauses match a source-target row pair that satisfy the merge condition, then the target rows will not be updated or deleted. - If you want to update all the columns of the target Delta table with the corresponding column of the source DataFrame, then you can use the ``whenMatchedUpdateAll()``. This is equivalent to:: whenMatchedUpdate(set = { "col1": "source.col1", "col2": "source.col2", ... # for all columns in the delta table }) - Constraints in the ``whenNotMatched`` clauses: - The condition in a ``whenNotMatched`` clause is optional. However, if there are multiple ``whenNotMatched`` clauses, then only the last one may omit the condition. - When there are more than one ``whenNotMatched`` clauses and there are conditions (or the lack of) such that a row satisfies multiple clauses, then the action for the first clause satisfied is executed. In other words, the order of the ``whenNotMatched`` clauses matters. - If no ``whenNotMatched`` clause is present or if it is present but the non-matching source row does not satisfy the condition, then the source row is not inserted. - If you want to insert all the columns of the target Delta table with the corresponding column of the source DataFrame, then you can use ``whenNotMatchedInsertAll()``. This is equivalent to:: whenNotMatchedInsert(values = { "col1": "source.col1", "col2": "source.col2", ... # for all columns in the delta table }) - Constraints in the ``whenNotMatchedBySource`` clauses: - The condition in a ``whenNotMatchedBySource`` clause is optional. However, if there are multiple ``whenNotMatchedBySource`` clauses, then only the last ``whenNotMatchedBySource`` clause may omit the condition. - Conditions and update expressions in ``whenNotMatchedBySource`` clauses may only refer to columns from the target Delta table. - When there are more than one ``whenNotMatchedBySource`` clauses and there are conditions (or the lack of) such that a row satisfies multiple clauses, then the action for the first clause satisfied is executed. In other words, the order of the ``whenNotMatchedBySource`` clauses matters. - If no ``whenNotMatchedBySource`` clause is present or if it is present but the non-matching target row does not satisfy any of the ``whenNotMatchedBySource`` clause condition, then the target row will not be updated or deleted. Example 1 with conditions and update expressions as SQL formatted string:: deltaTable.alias("events").merge( source = updatesDF.alias("updates"), condition = "events.eventId = updates.eventId" ).whenMatchedUpdate(set = { "data": "updates.data", "count": "events.count + 1" } ).whenNotMatchedInsert(values = { "date": "updates.date", "eventId": "updates.eventId", "data": "updates.data", "count": "1", "missed_count": "0" } ).whenNotMatchedBySourceUpdate(set = { "missed_count": "events.missed_count + 1" } ).execute() Example 2 with conditions and update expressions as Spark SQL functions:: from pyspark.sql.functions import * deltaTable.alias("events").merge( source = updatesDF.alias("updates"), condition = expr("events.eventId = updates.eventId") ).whenMatchedUpdate(set = { "data" : col("updates.data"), "count": col("events.count") + 1 } ).whenNotMatchedInsert(values = { "date": col("updates.date"), "eventId": col("updates.eventId"), "data": col("updates.data"), "count": lit("1"), "missed_count": lit("0") } ).whenNotMatchedBySourceUpdate(set = { "missed_count": col("events.missed_count") + 1 } ).execute() .. versionadded:: 0.4 """ def __init__(self, spark: SparkSession, jbuilder: "JavaObject"): self._spark = spark self._jbuilder = jbuilder @overload def whenMatchedUpdate( self, condition: OptionalExpressionOrColumn, set: ColumnMapping ) -> "DeltaMergeBuilder": ... @overload def whenMatchedUpdate( self, *, set: ColumnMapping ) -> "DeltaMergeBuilder": ... def whenMatchedUpdate( self, condition: OptionalExpressionOrColumn = None, set: OptionalColumnMapping = None ) -> "DeltaMergeBuilder": """ Update a matched table row based on the rules defined by ``set``. If a ``condition`` is specified, then it must evaluate to true for the row to be updated. See :py:class:`~delta.tables.DeltaMergeBuilder` for complete usage details. :param condition: Optional condition of the update :type condition: str or pyspark.sql.Column :param set: Defines the rules of setting the values of columns that need to be updated. *Note: This param is required.* Default value None is present to allow positional args in same order across languages. :type set: dict with str as keys and str or pyspark.sql.Column as values :return: this builder .. versionadded:: 0.4 """ jset = DeltaTable._dict_to_jmap(self._spark, set, "'set' in whenMatchedUpdate") new_jbuilder = self.__getMatchedBuilder(condition).update(jset) return DeltaMergeBuilder(self._spark, new_jbuilder) @since(0.4) # type: ignore[arg-type] def whenMatchedUpdateAll( self, condition: OptionalExpressionOrColumn = None ) -> "DeltaMergeBuilder": """ Update all the columns of the matched table row with the values of the corresponding columns in the source row. If a ``condition`` is specified, then it must be true for the new row to be updated. See :py:class:`~delta.tables.DeltaMergeBuilder` for complete usage details. :param condition: Optional condition of the insert :type condition: str or pyspark.sql.Column :return: this builder """ new_jbuilder = self.__getMatchedBuilder(condition).updateAll() return DeltaMergeBuilder(self._spark, new_jbuilder) @since(0.4) # type: ignore[arg-type] def whenMatchedDelete( self, condition: OptionalExpressionOrColumn = None ) -> "DeltaMergeBuilder": """ Delete a matched row from the table only if the given ``condition`` (if specified) is true for the matched row. See :py:class:`~delta.tables.DeltaMergeBuilder` for complete usage details. :param condition: Optional condition of the delete :type condition: str or pyspark.sql.Column :return: this builder """ new_jbuilder = self.__getMatchedBuilder(condition).delete() return DeltaMergeBuilder(self._spark, new_jbuilder) @overload def whenNotMatchedInsert( self, condition: ExpressionOrColumn, values: ColumnMapping ) -> "DeltaMergeBuilder": ... @overload def whenNotMatchedInsert( self, *, values: ColumnMapping = ... ) -> "DeltaMergeBuilder": ... def whenNotMatchedInsert( self, condition: OptionalExpressionOrColumn = None, values: OptionalColumnMapping = None ) -> "DeltaMergeBuilder": """ Insert a new row to the target table based on the rules defined by ``values``. If a ``condition`` is specified, then it must evaluate to true for the new row to be inserted. See :py:class:`~delta.tables.DeltaMergeBuilder` for complete usage details. :param condition: Optional condition of the insert :type condition: str or pyspark.sql.Column :param values: Defines the rules of setting the values of columns that need to be updated. *Note: This param is required.* Default value None is present to allow positional args in same order across languages. :type values: dict with str as keys and str or pyspark.sql.Column as values :return: this builder .. versionadded:: 0.4 """ jvalues = DeltaTable._dict_to_jmap(self._spark, values, "'values' in whenNotMatchedInsert") new_jbuilder = self.__getNotMatchedBuilder(condition).insert(jvalues) return DeltaMergeBuilder(self._spark, new_jbuilder) @since(0.4) # type: ignore[arg-type] def whenNotMatchedInsertAll( self, condition: OptionalExpressionOrColumn = None ) -> "DeltaMergeBuilder": """ Insert a new target Delta table row by assigning the target columns to the values of the corresponding columns in the source row. If a ``condition`` is specified, then it must evaluate to true for the new row to be inserted. See :py:class:`~delta.tables.DeltaMergeBuilder` for complete usage details. :param condition: Optional condition of the insert :type condition: str or pyspark.sql.Column :return: this builder """ new_jbuilder = self.__getNotMatchedBuilder(condition).insertAll() return DeltaMergeBuilder(self._spark, new_jbuilder) @overload def whenNotMatchedBySourceUpdate( self, condition: OptionalExpressionOrColumn, set: ColumnMapping ) -> "DeltaMergeBuilder": ... @overload def whenNotMatchedBySourceUpdate( self, *, set: ColumnMapping ) -> "DeltaMergeBuilder": ... def whenNotMatchedBySourceUpdate( self, condition: OptionalExpressionOrColumn = None, set: OptionalColumnMapping = None ) -> "DeltaMergeBuilder": """ Update a target row that has no matches in the source based on the rules defined by ``set``. If a ``condition`` is specified, then it must evaluate to true for the row to be updated. See :py:class:`~delta.tables.DeltaMergeBuilder` for complete usage details. :param condition: Optional condition of the update :type condition: str or pyspark.sql.Column :param set: Defines the rules of setting the values of columns that need to be updated. *Note: This param is required.* Default value None is present to allow positional args in same order across languages. :type set: dict with str as keys and str or pyspark.sql.Column as values :return: this builder .. versionadded:: 2.3 """ jset = DeltaTable._dict_to_jmap(self._spark, set, "'set' in whenNotMatchedBySourceUpdate") new_jbuilder = self.__getNotMatchedBySourceBuilder(condition).update(jset) return DeltaMergeBuilder(self._spark, new_jbuilder) @since(2.3) # type: ignore[arg-type] def whenNotMatchedBySourceDelete( self, condition: OptionalExpressionOrColumn = None ) -> "DeltaMergeBuilder": """ Delete a target row that has no matches in the source from the table only if the given ``condition`` (if specified) is true for the target row. See :py:class:`~delta.tables.DeltaMergeBuilder` for complete usage details. :param condition: Optional condition of the delete :type condition: str or pyspark.sql.Column :return: this builder """ new_jbuilder = self.__getNotMatchedBySourceBuilder(condition).delete() return DeltaMergeBuilder(self._spark, new_jbuilder) @since(3.2) # type: ignore[arg-type] def withSchemaEvolution(self) -> "DeltaMergeBuilder": """ Enable schema evolution for the merge operation. This allows the target table schema to be automatically updated based on the schema of the source DataFrame. See :py:class:`~delta.tables.DeltaMergeBuilder` for complete usage details. :return: this builder """ new_jbuilder = self._jbuilder.withSchemaEvolution() return DeltaMergeBuilder(self._spark, new_jbuilder) @since(0.4) # type: ignore[arg-type] def execute(self) -> DataFrame: """ Execute the merge operation based on the built matched and not matched actions. See :py:class:`~delta.tables.DeltaMergeBuilder` for complete usage details. """ return DataFrame( self._jbuilder.execute(), getattr(self._spark, "_wrapped", self._spark)) # type: ignore[attr-defined] def __getMatchedBuilder( self, condition: OptionalExpressionOrColumn = None ) -> "JavaObject": if condition is None: return self._jbuilder.whenMatched() else: return self._jbuilder.whenMatched(DeltaTable._condition_to_jcolumn(condition)) def __getNotMatchedBuilder( self, condition: OptionalExpressionOrColumn = None ) -> "JavaObject": if condition is None: return self._jbuilder.whenNotMatched() else: return self._jbuilder.whenNotMatched(DeltaTable._condition_to_jcolumn(condition)) def __getNotMatchedBySourceBuilder( self, condition: OptionalExpressionOrColumn = None ) -> "JavaObject": if condition is None: return self._jbuilder.whenNotMatchedBySource() else: return self._jbuilder.whenNotMatchedBySource( DeltaTable._condition_to_jcolumn(condition)) @dataclass class IdentityGenerator: """ Identity generator specifications for the identity column in the Delta table. :param start: the start for the identity column. Default is 1. :type start: int :param step: the step for the identity column. Default is 1. :type step: int """ start: int = 1 step: int = 1 class DeltaTableBuilder(object): """ Builder to specify how to create / replace a Delta table. You must specify the table name or the path before executing the builder. You can specify the table columns, the partitioning columns, the location of the data, the table comment and the property, and how you want to create / replace the Delta table. After executing the builder, a :py:class:`~delta.tables.DeltaTable` object is returned. Use :py:meth:`delta.tables.DeltaTable.create`, :py:meth:`delta.tables.DeltaTable.createIfNotExists`, :py:meth:`delta.tables.DeltaTable.replace`, :py:meth:`delta.tables.DeltaTable.createOrReplace` to create an object of this class. Example 1 to create a Delta table with separate columns, using the table name:: deltaTable = DeltaTable.create(sparkSession) .tableName("testTable") .addColumn("c1", dataType = "INT", nullable = False) .addColumn("c2", dataType = IntegerType(), generatedAlwaysAs = "c1 + 1") .partitionedBy("c1") .execute() Example 2 to replace a Delta table with existing columns, using the location:: df = spark.createDataFrame([('a', 1), ('b', 2), ('c', 3)], ["key", "value"]) deltaTable = DeltaTable.replace(sparkSession) .tableName("testTable") .addColumns(df.schema) .execute() .. versionadded:: 1.0 .. note:: Evolving """ def __init__(self, spark: SparkSession, jbuilder: "JavaObject"): self._spark = spark self._jbuilder = jbuilder def _raise_type_error(self, msg: str, objs: Iterable[Any]) -> NoReturn: errorMsg = msg for obj in objs: errorMsg += " Found %s with type %s" % ((str(obj)), str(type(obj))) raise TypeError(errorMsg) def _check_identity_column_spec(self, identityGenerator: IdentityGenerator) -> None: if identityGenerator.step == 0: raise ValueError("Column identity generation requires step to be non-zero.") @since(1.0) # type: ignore[arg-type] def tableName(self, identifier: str) -> "DeltaTableBuilder": """ Specify the table name. Optionally qualified with a database name [database_name.] table_name. :param identifier: the table name :type identifier: str :return: this builder .. note:: Evolving """ if type(identifier) is not str: self._raise_type_error("Identifier must be str.", [identifier]) self._jbuilder = self._jbuilder.tableName(identifier) return self @since(1.0) # type: ignore[arg-type] def location(self, location: str) -> "DeltaTableBuilder": """ Specify the path to the directory where table data is stored, which could be a path on distributed storage. :param location: the data stored location :type location: str :return: this builder .. note:: Evolving """ if type(location) is not str: self._raise_type_error("Location must be str.", [location]) self._jbuilder = self._jbuilder.location(location) return self @since(1.0) # type: ignore[arg-type] def comment(self, comment: str) -> "DeltaTableBuilder": """ Comment to describe the table. :param comment: the table comment :type comment: str :return: this builder .. note:: Evolving """ if type(comment) is not str: self._raise_type_error("Table comment must be str.", [comment]) self._jbuilder = self._jbuilder.comment(comment) return self @since(1.0) # type: ignore[arg-type] def addColumn( self, colName: str, dataType: Union[str, DataType], nullable: bool = True, generatedAlwaysAs: Optional[Union[str, IdentityGenerator]] = None, generatedByDefaultAs: Optional[IdentityGenerator] = None, comment: Optional[str] = None, ) -> "DeltaTableBuilder": """ Specify a column in the table :param colName: the column name :type colName: str :param dataType: the column data type :type dataType: str or pyspark.sql.types.DataType :param nullable: whether column is nullable :type nullable: bool :param generatedAlwaysAs: a SQL expression if the column is always generated as a function of other columns; an IdentityGenerator object if the column is always generated using identity generator See online documentation for details on Generated Columns. :type generatedAlwaysAs: str or delta.tables.IdentityGenerator :param generatedByDefaultAs: an IdentityGenerator object to generate identity values if the user does not provide values for the column See online documentation for details on Generated Columns. :type generatedByDefaultAs: delta.tables.IdentityGenerator :param comment: the column comment :type comment: str :return: this builder .. note:: Evolving """ if type(colName) is not str: self._raise_type_error("Column name must be str.", [colName]) if type(dataType) is not str and not isinstance(dataType, DataType): self._raise_type_error("Column data type must be str or DataType.", [dataType]) jvm: "JVMView" = self._spark._sc._jvm # type: ignore[attr-defined] jsparkSession: "JavaObject" = self._spark._jsparkSession # type: ignore[attr-defined] _col_jbuilder = jvm.io.delta.tables.DeltaTable.columnBuilder(jsparkSession, colName) if isinstance(dataType, DataType): dataType = jsparkSession.parseDataType(dataType.json()) _col_jbuilder = _col_jbuilder.dataType(dataType) if type(nullable) is not bool: self._raise_type_error("Column nullable must be bool.", [nullable]) _col_jbuilder = _col_jbuilder.nullable(nullable) if generatedAlwaysAs is not None and generatedByDefaultAs is not None: raise ValueError( "generatedByDefaultAs and generatedAlwaysAs cannot both be set.", [generatedByDefaultAs, generatedAlwaysAs]) if generatedAlwaysAs is not None: if type(generatedAlwaysAs) is str: _col_jbuilder = _col_jbuilder.generatedAlwaysAs(generatedAlwaysAs) elif isinstance(generatedAlwaysAs, IdentityGenerator): self._check_identity_column_spec(generatedAlwaysAs) _col_jbuilder = _col_jbuilder.generatedAlwaysAsIdentity( generatedAlwaysAs.start, generatedAlwaysAs.step) else: self._raise_type_error( "Generated always as expression must be str or IdentityGenerator.", [generatedAlwaysAs]) elif generatedByDefaultAs is not None: if not isinstance(generatedByDefaultAs, IdentityGenerator): self._raise_type_error( "Generated by default expression must be IdentityGenerator.", [generatedByDefaultAs]) self._check_identity_column_spec(generatedByDefaultAs) _col_jbuilder = _col_jbuilder.generatedByDefaultAsIdentity( generatedByDefaultAs.start, generatedByDefaultAs.step) if comment is not None: if type(comment) is not str: self._raise_type_error("Column comment must be str.", [comment]) _col_jbuilder = _col_jbuilder.comment(comment) self._jbuilder = self._jbuilder.addColumn(_col_jbuilder.build()) return self @since(1.0) # type: ignore[arg-type] def addColumns( self, cols: Union[StructType, List[StructField]] ) -> "DeltaTableBuilder": """ Specify columns in the table using an existing schema :param cols: the columns in the existing schema :type cols: pyspark.sql.types.StructType or a list of pyspark.sql.types.StructType. :return: this builder .. note:: Evolving """ if isinstance(cols, list): for col in cols: if type(col) is not StructField: self._raise_type_error( "Column in existing schema must be StructField.", [col]) cols = StructType(cols) if type(cols) is not StructType: self._raise_type_error("Schema must be StructType " + "or a list of StructField.", [cols]) jsparkSession: "JavaObject" = self._spark._jsparkSession # type: ignore[attr-defined] scalaSchema = jsparkSession.parseDataType(cols.json()) self._jbuilder = self._jbuilder.addColumns(scalaSchema) return self @overload def partitionedBy( self, *cols: str ) -> "DeltaTableBuilder": ... @overload def partitionedBy( self, __cols: Union[List[str], Tuple[str, ...]] ) -> "DeltaTableBuilder": ... @since(1.0) # type: ignore[arg-type] def partitionedBy( self, *cols: Union[str, List[str], Tuple[str, ...]] ) -> "DeltaTableBuilder": """ Specify columns for partitioning :param cols: the partitioning cols :type cols: str or list name of columns :return: this builder .. note:: Evolving """ try: from pyspark.sql.column import _to_seq # type: ignore[attr-defined] except ImportError: # Spark 4 from pyspark.sql.classic.column import _to_seq # type: ignore[import, no-redef] if len(cols) == 1 and isinstance(cols[0], (list, tuple)): cols = cols[0] # type: ignore[assignment] for c in cols: if type(c) is not str: self._raise_type_error("Partitioning column must be str.", [c]) self._jbuilder = self._jbuilder.partitionedBy(_to_seq( self._spark._sc, # type: ignore[attr-defined] cast(Iterable[Union[Column, str]], cols) )) return self @overload def clusterBy( self, *cols: str ) -> "DeltaTableBuilder": ... @overload def clusterBy( self, __cols: Union[List[str], Tuple[str, ...]] ) -> "DeltaTableBuilder": ... @since(3.2) # type: ignore[arg-type] def clusterBy( self, *cols: Union[str, List[str], Tuple[str, ...]] ) -> "DeltaTableBuilder": """ Specify columns for clustering :param cols: the clustering cols :type cols: str or list name of columns :return: this builder .. note:: Evolving """ try: from pyspark.sql.column import _to_seq # type: ignore[attr-defined] except ImportError: # Spark 4 from pyspark.sql.classic.column import _to_seq # type: ignore[import, no-redef] if len(cols) == 1 and isinstance(cols[0], (list, tuple)): cols = cols[0] # type: ignore[assignment] for c in cols: if type(c) is not str: self._raise_type_error("Clustering column must be str.", [c]) self._jbuilder = self._jbuilder.clusterBy(_to_seq( self._spark._sc, # type: ignore[attr-defined] cast(Iterable[Union[Column, str]], cols) )) return self @since(1.0) # type: ignore[arg-type] def property(self, key: str, value: str) -> "DeltaTableBuilder": """ Specify a table property :param key: the table property key :type value: the table property value :return: this builder .. note:: Evolving """ if type(key) is not str or type(value) is not str: self._raise_type_error("Key and value of property must be string.", [key, value]) self._jbuilder = self._jbuilder.property(key, value) return self @since(1.0) # type: ignore[arg-type] def execute(self) -> DeltaTable: """ Execute Table Creation. :rtype: :py:class:`~delta.tables.DeltaTable` .. note:: Evolving """ jdt = self._jbuilder.execute() return DeltaTable(self._spark, jdt) class DeltaOptimizeBuilder(object): """ Builder class for constructing OPTIMIZE command and executing. Use :py:meth:`delta.tables.DeltaTable.optimize` to create an instance of this class. .. versionadded:: 2.0.0 """ def __init__(self, spark: SparkSession, jbuilder: "JavaObject"): self._spark = spark self._jbuilder = jbuilder @since(2.0) # type: ignore[arg-type] def where(self, partitionFilter: str) -> "DeltaOptimizeBuilder": """ Apply partition filter on this optimize command builder to limit the operation on selected partitions. :param partitionFilter: The partition filter to apply :type partitionFilter: str :return: DeltaOptimizeBuilder with partition filter applied :rtype: :py:class:`~delta.tables.DeltaOptimizeBuilder` """ self._jbuilder = self._jbuilder.where(partitionFilter) return self @since(2.0) # type: ignore[arg-type] def executeCompaction(self) -> DataFrame: """ Compact the small files in selected partitions. :return: DataFrame containing the OPTIMIZE execution metrics :rtype: pyspark.sql.DataFrame """ return DataFrame( self._jbuilder.executeCompaction(), getattr(self._spark, "_wrapped", self._spark) # type: ignore[attr-defined] ) @since(2.0) # type: ignore[arg-type] def executeZOrderBy(self, *cols: Union[str, List[str], Tuple[str, ...]]) -> DataFrame: """ Z-Order the data in selected partitions using the given columns. :param cols: the Z-Order cols :type cols: str or list name of columns :return: DataFrame containing the OPTIMIZE execution metrics :rtype: pyspark.sql.DataFrame """ try: from pyspark.sql.column import _to_seq # type: ignore[attr-defined] except ImportError: # Spark 4 from pyspark.sql.classic.column import _to_seq # type: ignore[import, no-redef] if len(cols) == 1 and isinstance(cols[0], (list, tuple)): cols = cols[0] # type: ignore[assignment] for c in cols: if type(c) is not str: errorMsg = "Z-order column must be str. " errorMsg += "Found %s with type %s" % ((str(c)), str(type(c))) raise TypeError(errorMsg) return DataFrame( self._jbuilder.executeZOrderBy(_to_seq( self._spark._sc, # type: ignore[attr-defined] cast(Iterable[Union[Column, str]], cols) )), getattr(self._spark, "_wrapped", self._spark) # type: ignore[attr-defined] ) ================================================ FILE: python/delta/testing/__init__.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # __all__ = ['utils'] ================================================ FILE: python/delta/testing/log4j2.properties ================================================ # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Set everything to be logged to the console rootLogger.level = warn rootLogger.appenderRef.stdout.ref = STDOUT appender.console.type = Console appender.console.name = STDOUT appender.console.target = SYSTEM_OUT appender.console.layout.type = PatternLayout appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n # Settings to quiet third party logs that are too verbose logger.jetty.name = org.sparkproject.jetty logger.jetty.level = warn logger.jetty2.name = org.sparkproject.jetty.util.component.AbstractLifeCycle logger.jetty2.level = error logger.repl1.name = org.apache.spark.repl.SparkIMain$exprTyper logger.repl1.level = info logger.repl2.name = org.apache.spark.repl.SparkILoop$SparkILoopInterpreter logger.repl2.level = info # Set the default spark-shell log level to WARN. When running the spark-shell, the # log level for this class is used to overwrite the root logger's log level, so that # the user can have different defaults for the shell and regular Spark apps. logger.repl.name = org.apache.spark.repl.Main logger.repl.level = warn # SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs # in SparkSQL with Hive support logger.metastore.name = org.apache.hadoop.hive.metastore.RetryingHMSHandler logger.metastore.level = fatal logger.hive_functionregistry.name = org.apache.hadoop.hive.ql.exec.FunctionRegistry logger.hive_functionregistry.level = error # Parquet related logging logger.parquet.name = org.apache.parquet.CorruptStatistics logger.parquet.level = error logger.parquet2.name = parquet.CorruptStatistics logger.parquet2.level = error ================================================ FILE: python/delta/testing/utils.py ================================================ # # Copyright (2023) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os import shutil import sys import tempfile import unittest import uuid from contextlib import contextmanager from pyspark import SparkConf from pyspark.testing.sqlutils import ReusedSQLTestCase # type: ignore[import] from typing import Generator class DeltaTestCase(ReusedSQLTestCase): """Test class base that sets up a correctly configured SparkSession for querying Delta tables. """ @classmethod def conf(cls) -> SparkConf: _conf = super(DeltaTestCase, cls).conf() _conf.set("spark.app.name", cls.__name__) _conf.set("spark.master", "local[4]") _conf.set("spark.ui.enabled", "false") _conf.set("spark.databricks.delta.snapshotPartitions", "2") _conf.set("spark.sql.shuffle.partitions", "5") _conf.set("delta.log.cacheSize", "3") _conf.set("spark.databricks.delta.delta.log.cacheSize", "3") _conf.set("spark.sql.sources.parallelPartitionDiscovery.parallelism", "5") _conf.set("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") _conf.set("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") return _conf def setUp(self) -> None: super(DeltaTestCase, self).setUp() self.tempPath = tempfile.mkdtemp() self.tempFile = os.path.join(self.tempPath, "tempFile") def tearDown(self) -> None: super(DeltaTestCase, self).tearDown() shutil.rmtree(self.tempPath, ignore_errors=True) @contextmanager def tempTable(self) -> Generator[str, None, None]: table_name = "table_" + str(uuid.uuid4()).replace("-", "_") with super(DeltaTestCase, self).table(table_name): yield table_name ================================================ FILE: python/delta/tests/__init__.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ================================================ FILE: python/delta/tests/test_deltatable.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # mypy: disable-error-code="union-attr, attr-defined" import unittest import os from multiprocessing.pool import ThreadPool from typing import List, Set, Dict, Optional, Any, Callable, Union, Tuple from py4j.java_gateway import JavaObject from py4j.protocol import Py4JJavaError from pyspark.errors.exceptions.base import UnsupportedOperationException from pyspark.sql import DataFrame, Row from pyspark.sql.functions import col, lit, expr, floor from pyspark.sql.types import StructType, StructField, StringType, IntegerType, LongType, DataType from pyspark.sql.utils import AnalysisException, ParseException from delta.tables import DeltaTable, DeltaTableBuilder, DeltaOptimizeBuilder, IdentityGenerator from delta.testing.utils import DeltaTestCase class DeltaTableTestsMixin: def test_forPath(self) -> None: self.__writeDeltaTable([('a', 1), ('b', 2), ('c', 3)]) dt = DeltaTable.forPath(self.spark, self.tempFile).toDF() self.__checkAnswer(dt, [('a', 1), ('b', 2), ('c', 3)]) def test_forPathWithOptions(self) -> None: path = self.tempFile fsOptions = {"fs.fake.impl": "org.apache.spark.sql.delta.FakeFileSystem", "fs.fake.impl.disable.cache": "true"} self.__writeDeltaTable([('a', 1), ('b', 2), ('c', 3)]) dt = DeltaTable.forPath(self.spark, path, fsOptions).toDF() self.__checkAnswer(dt, [('a', 1), ('b', 2), ('c', 3)]) def test_forName(self) -> None: with self.tempTable() as tableName: self.__writeAsTable([('a', 1), ('b', 2), ('c', 3)], tableName) df = DeltaTable.forName(self.spark, tableName).toDF() self.__checkAnswer(df, [('a', 1), ('b', 2), ('c', 3)]) def test_alias_and_toDF(self) -> None: self.__writeDeltaTable([('a', 1), ('b', 2), ('c', 3)]) dt = DeltaTable.forPath(self.spark, self.tempFile).toDF() self.__checkAnswer( dt.alias("myTable").select('myTable.key', 'myTable.value'), [('a', 1), ('b', 2), ('c', 3)]) def test_delete(self) -> None: self.__writeDeltaTable([('a', 1), ('b', 2), ('c', 3), ('d', 4)]) dt = DeltaTable.forPath(self.spark, self.tempFile) # delete with condition as str dt.delete("key = 'a'") self.__checkAnswer(dt.toDF(), [('b', 2), ('c', 3), ('d', 4)]) # delete with condition as Column dt.delete(col("key") == lit("b")) self.__checkAnswer(dt.toDF(), [('c', 3), ('d', 4)]) # delete without condition dt.delete() self.__checkAnswer(dt.toDF(), []) # bad args with self.assertRaises(TypeError): dt.delete(condition=1) # type: ignore[arg-type] def test_generate(self) -> None: # create a delta table numFiles = 10 self.spark.range(100).repartition(numFiles).write.format("delta").save(self.tempFile) dt = DeltaTable.forPath(self.spark, self.tempFile) # Generate the symlink format manifest dt.generate("symlink_format_manifest") # check the contents of the manifest # NOTE: this is not a correctness test, we are testing correctness in the scala suite manifestPath = os.path.join(self.tempFile, os.path.join("_symlink_format_manifest", "manifest")) files = [] with open(manifestPath) as f: files = f.readlines() # the number of files we write should equal the number of lines in the manifest self.assertEqual(len(files), numFiles) def test_update(self) -> None: self.__writeDeltaTable([('a', 1), ('b', 2), ('c', 3), ('d', 4)]) dt = DeltaTable.forPath(self.spark, self.tempFile) # update with condition as str and with set exprs as str dt.update("key = 'a' or key = 'b'", {"value": "1"}) self.__checkAnswer(dt.toDF(), [('a', 1), ('b', 1), ('c', 3), ('d', 4)]) # update with condition as Column and with set exprs as Columns dt.update(expr("key = 'a' or key = 'b'"), {"value": expr("0")}) self.__checkAnswer(dt.toDF(), [('a', 0), ('b', 0), ('c', 3), ('d', 4)]) # update without condition dt.update(set={"value": "200"}) self.__checkAnswer(dt.toDF(), [('a', 200), ('b', 200), ('c', 200), ('d', 200)]) # bad args with self.assertRaisesRegex(ValueError, "cannot be None"): dt.update({"value": "200"}) # type: ignore[call-overload] with self.assertRaisesRegex(ValueError, "cannot be None"): dt.update(condition='a') # type: ignore[call-overload] with self.assertRaisesRegex(TypeError, "must be a dict"): dt.update(set=1) # type: ignore[call-overload] with self.assertRaisesRegex(TypeError, "must be a Spark SQL Column or a string"): dt.update(1, {}) # type: ignore[call-overload] with self.assertRaisesRegex(TypeError, "Values of dict in .* must contain only"): dt.update(set={"value": 1}) # type: ignore[dict-item] with self.assertRaisesRegex(TypeError, "Keys of dict in .* must contain only"): dt.update(set={1: ""}) # type: ignore[dict-item] with self.assertRaises(TypeError): dt.update(set=1) # type: ignore[call-overload] def test_merge(self) -> None: self.__writeDeltaTable([('a', 1), ('b', 2), ('c', 3), ('d', 4)]) source = self.spark.createDataFrame([('a', -1), ('b', 0), ('e', -5), ('f', -6)], ["k", "v"]) def reset_table() -> None: self.__overwriteDeltaTable([('a', 1), ('b', 2), ('c', 3), ('d', 4)]) dt = DeltaTable.forPath(self.spark, self.tempFile) # ============== Test basic syntax ============== # String expressions in merge condition and dicts reset_table() merge_output = dt.merge(source, "key = k") \ .whenMatchedUpdate(set={"value": "v + 0"}) \ .whenNotMatchedInsert(values={"key": "k", "value": "v + 0"}) \ .whenNotMatchedBySourceUpdate(set={"value": "value + 0"}) \ .execute() self.__checkAnswer(merge_output, ([Row(6, # type: ignore[call-overload] 4, # updated rows (a and b in WHEN MATCHED # and c and d in WHEN NOT MATCHED BY SOURCE) 0, # deleted rows 2)]), # inserted rows (e and f) StructType([StructField('num_affected_rows', LongType(), False), StructField('num_updated_rows', LongType(), False), StructField('num_deleted_rows', LongType(), False), StructField('num_inserted_rows', LongType(), False)])) self.__checkAnswer(dt.toDF(), ([('a', -1), ('b', 0), ('c', 3), ('d', 4), ('e', -5), ('f', -6)])) # Column expressions in merge condition and dicts reset_table() merge_output = dt.merge(source, expr("key = k")) \ .whenMatchedUpdate(set={"value": col("v") + 0}) \ .whenNotMatchedInsert(values={"key": "k", "value": col("v") + 0}) \ .whenNotMatchedBySourceUpdate(set={"value": col("value") + 0}) \ .execute() self.__checkAnswer(dt.toDF(), ([('a', -1), ('b', 0), ('c', 3), ('d', 4), ('e', -5), ('f', -6)])) # Multiple matched update clauses reset_table() dt.merge(source, expr("key = k")) \ .whenMatchedUpdate(condition="key = 'a'", set={"value": "5"}) \ .whenMatchedUpdate(set={"value": "0"}) \ .execute() self.__checkAnswer(dt.toDF(), ([('a', 5), ('b', 0), ('c', 3), ('d', 4)])) # Multiple matched delete clauses reset_table() dt.merge(source, expr("key = k")) \ .whenMatchedDelete(condition="key = 'a'") \ .whenMatchedDelete() \ .execute() self.__checkAnswer(dt.toDF(), ([('c', 3), ('d', 4)])) # Redundant matched update and delete clauses reset_table() dt.merge(source, expr("key = k")) \ .whenMatchedUpdate(condition="key = 'a'", set={"value": "5"}) \ .whenMatchedUpdate(condition="key = 'a'", set={"value": "0"}) \ .whenMatchedUpdate(condition="key = 'b'", set={"value": "6"}) \ .whenMatchedDelete(condition="key = 'b'") \ .execute() self.__checkAnswer(dt.toDF(), ([('a', 5), ('b', 6), ('c', 3), ('d', 4)])) # Interleaved matched update and delete clauses reset_table() dt.merge(source, expr("key = k")) \ .whenMatchedDelete(condition="key = 'a'") \ .whenMatchedUpdate(condition="key = 'a'", set={"value": "5"}) \ .whenMatchedDelete(condition="key = 'b'") \ .whenMatchedUpdate(set={"value": "6"}) \ .execute() self.__checkAnswer(dt.toDF(), ([('c', 3), ('d', 4)])) # Multiple not matched insert clauses reset_table() dt.alias("t")\ .merge(source.toDF("key", "value").alias("s"), expr("t.key = s.key")) \ .whenNotMatchedInsert(condition="s.key = 'e'", values={"t.key": "s.key", "t.value": "5"}) \ .whenNotMatchedInsertAll() \ .execute() self.__checkAnswer(dt.toDF(), ([('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5), ('f', -6)])) # Redundant not matched update and delete clauses reset_table() dt.merge(source, expr("key = k")) \ .whenNotMatchedInsert(condition="k = 'e'", values={"key": "k", "value": "5"}) \ .whenNotMatchedInsert(condition="k = 'e'", values={"key": "k", "value": "6"}) \ .whenNotMatchedInsert(condition="k = 'f'", values={"key": "k", "value": "7"}) \ .whenNotMatchedInsert(condition="k = 'f'", values={"key": "k", "value": "8"}) \ .execute() self.__checkAnswer(dt.toDF(), ([('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5), ('f', 7)])) # Multiple not matched by source update clauses reset_table() dt.merge(source, expr("key = k")) \ .whenNotMatchedBySourceUpdate(condition="key = 'c'", set={"value": "5"}) \ .whenNotMatchedBySourceUpdate(set={"value": "0"}) \ .execute() self.__checkAnswer(dt.toDF(), ([('a', 1), ('b', 2), ('c', 5), ('d', 0)])) # Multiple not matched by source delete clauses reset_table() dt.merge(source, expr("key = k")) \ .whenNotMatchedBySourceDelete(condition="key = 'c'") \ .whenNotMatchedBySourceDelete() \ .execute() self.__checkAnswer(dt.toDF(), ([('a', 1), ('b', 2)])) # Redundant not matched by source update and delete clauses reset_table() dt.merge(source, expr("key = k")) \ .whenNotMatchedBySourceUpdate(condition="key = 'c'", set={"value": "5"}) \ .whenNotMatchedBySourceUpdate(condition="key = 'c'", set={"value": "0"}) \ .whenNotMatchedBySourceUpdate(condition="key = 'd'", set={"value": "6"}) \ .whenNotMatchedBySourceDelete(condition="key = 'd'") \ .execute() self.__checkAnswer(dt.toDF(), ([('a', 1), ('b', 2), ('c', 5), ('d', 6)])) # Interleaved update and delete clauses reset_table() dt.merge(source, expr("key = k")) \ .whenNotMatchedBySourceDelete(condition="key = 'c'") \ .whenNotMatchedBySourceUpdate(condition="key = 'c'", set={"value": "5"}) \ .whenNotMatchedBySourceDelete(condition="key = 'd'") \ .whenNotMatchedBySourceUpdate(set={"value": "6"}) \ .execute() self.__checkAnswer(dt.toDF(), ([('a', 1), ('b', 2)])) # ============== Test clause conditions ============== # String expressions in all conditions and dicts reset_table() dt.merge(source, "key = k") \ .whenMatchedUpdate(condition="k = 'a'", set={"value": "v + 0"}) \ .whenMatchedDelete(condition="k = 'b'") \ .whenNotMatchedInsert(condition="k = 'e'", values={"key": "k", "value": "v + 0"}) \ .whenNotMatchedBySourceUpdate(condition="key = 'c'", set={"value": col("value") + 0}) \ .whenNotMatchedBySourceDelete(condition="key = 'd'") \ .execute() self.__checkAnswer(dt.toDF(), ([('a', -1), ('c', 3), ('e', -5)])) # Column expressions in all conditions and dicts reset_table() dt.merge(source, expr("key = k")) \ .whenMatchedUpdate( condition=expr("k = 'a'"), set={"value": col("v") + 0}) \ .whenMatchedDelete(condition=expr("k = 'b'")) \ .whenNotMatchedInsert( condition=expr("k = 'e'"), values={"key": "k", "value": col("v") + 0}) \ .whenNotMatchedBySourceUpdate( condition=expr("key = 'c'"), set={"value": col("value") + 0}) \ .whenNotMatchedBySourceDelete(condition=expr("key = 'd'")) \ .execute() self.__checkAnswer(dt.toDF(), ([('a', -1), ('c', 3), ('e', -5)])) # Positional arguments reset_table() dt.merge(source, "key = k") \ .whenMatchedUpdate("k = 'a'", {"value": "v + 0"}) \ .whenMatchedDelete("k = 'b'") \ .whenNotMatchedInsert("k = 'e'", {"key": "k", "value": "v + 0"}) \ .whenNotMatchedBySourceUpdate("key = 'c'", {"value": "value + 0"}) \ .whenNotMatchedBySourceDelete("key = 'd'") \ .execute() self.__checkAnswer(dt.toDF(), ([('a', -1), ('c', 3), ('e', -5)])) # ============== Test updateAll/insertAll ============== # No clause conditions and insertAll/updateAll + aliases reset_table() dt.alias("t") \ .merge(source.toDF("key", "value").alias("s"), expr("t.key = s.key")) \ .whenMatchedUpdateAll() \ .whenNotMatchedInsertAll() \ .execute() self.__checkAnswer(dt.toDF(), ([('a', -1), ('b', 0), ('c', 3), ('d', 4), ('e', -5), ('f', -6)])) # String expressions in all clause conditions and insertAll/updateAll + aliases reset_table() dt.alias("t") \ .merge(source.toDF("key", "value").alias("s"), "s.key = t.key") \ .whenMatchedUpdateAll("s.key = 'a'") \ .whenNotMatchedInsertAll("s.key = 'e'") \ .execute() self.__checkAnswer(dt.toDF(), ([('a', -1), ('b', 2), ('c', 3), ('d', 4), ('e', -5)])) # Column expressions in all clause conditions and insertAll/updateAll + aliases reset_table() dt.alias("t") \ .merge(source.toDF("key", "value").alias("s"), expr("t.key = s.key")) \ .whenMatchedUpdateAll(expr("s.key = 'a'")) \ .whenNotMatchedInsertAll(expr("s.key = 'e'")) \ .execute() self.__checkAnswer(dt.toDF(), ([('a', -1), ('b', 2), ('c', 3), ('d', 4), ('e', -5)])) # Schema evolution reset_table() dt.alias("t") \ .merge(source.toDF("key", "extra").alias("s"), expr("t.key = s.key")) \ .whenMatchedUpdate(set={"extra": "-1"}) \ .whenNotMatchedInsertAll() \ .withSchemaEvolution() \ .execute() self.__checkAnswer( DeltaTable.forPath(self.spark, self.tempFile).toDF(), # reload the table ([('a', 1, -1), ('b', 2, -1), ('c', 3, None), ('d', 4, None), ('e', None, -5), ('f', None, -6)]), ["key", "value", "extra"]) # ============== Test bad args ============== # ---- bad args in merge() with self.assertRaisesRegex(TypeError, "must be DataFrame"): dt.merge(1, "key = k") # type: ignore[arg-type] with self.assertRaisesRegex(TypeError, "must be a Spark SQL Column or a string"): dt.merge(source, 1) # type: ignore[arg-type] # ---- bad args in whenMatchedUpdate() with self.assertRaisesRegex(ValueError, "cannot be None"): (dt # type: ignore[call-overload] .merge(source, "key = k") .whenMatchedUpdate({"value": "v"})) with self.assertRaisesRegex(ValueError, "cannot be None"): (dt # type: ignore[call-overload] .merge(source, "key = k") .whenMatchedUpdate(1)) with self.assertRaisesRegex(ValueError, "cannot be None"): (dt # type: ignore[call-overload] .merge(source, "key = k") .whenMatchedUpdate(condition="key = 'a'")) with self.assertRaisesRegex(TypeError, "must be a Spark SQL Column or a string"): (dt # type: ignore[call-overload] .merge(source, "key = k") .whenMatchedUpdate(1, {"value": "v"})) with self.assertRaisesRegex(TypeError, "must be a dict"): (dt # type: ignore[call-overload] .merge(source, "key = k") .whenMatchedUpdate("k = 'a'", 1)) with self.assertRaisesRegex(TypeError, "Values of dict in .* must contain only"): (dt .merge(source, "key = k") .whenMatchedUpdate(set={"value": 1})) # type: ignore[dict-item] with self.assertRaisesRegex(TypeError, "Keys of dict in .* must contain only"): (dt .merge(source, "key = k") .whenMatchedUpdate(set={1: ""})) # type: ignore[dict-item] with self.assertRaises(TypeError): (dt # type: ignore[call-overload] .merge(source, "key = k") .whenMatchedUpdate(set="k = 'a'", condition={"value": 1})) # bad args in whenMatchedDelete() with self.assertRaisesRegex(TypeError, "must be a Spark SQL Column or a string"): dt.merge(source, "key = k").whenMatchedDelete(1) # type: ignore[arg-type] # ---- bad args in whenNotMatchedInsert() with self.assertRaisesRegex(ValueError, "cannot be None"): (dt # type: ignore[call-overload] .merge(source, "key = k") .whenNotMatchedInsert({"value": "v"})) with self.assertRaisesRegex(ValueError, "cannot be None"): dt.merge(source, "key = k").whenNotMatchedInsert(1) # type: ignore[call-overload] with self.assertRaisesRegex(ValueError, "cannot be None"): (dt # type: ignore[call-overload] .merge(source, "key = k") .whenNotMatchedInsert(condition="key = 'a'")) with self.assertRaisesRegex(TypeError, "must be a Spark SQL Column or a string"): (dt # type: ignore[call-overload] .merge(source, "key = k") .whenNotMatchedInsert(1, {"value": "v"})) with self.assertRaisesRegex(TypeError, "must be a dict"): (dt # type: ignore[call-overload] .merge(source, "key = k") .whenNotMatchedInsert("k = 'a'", 1)) with self.assertRaisesRegex(TypeError, "Values of dict in .* must contain only"): (dt .merge(source, "key = k") .whenNotMatchedInsert(values={"value": 1})) # type: ignore[dict-item] with self.assertRaisesRegex(TypeError, "Keys of dict in .* must contain only"): (dt .merge(source, "key = k") .whenNotMatchedInsert(values={1: "value"})) # type: ignore[dict-item] with self.assertRaises(TypeError): (dt # type: ignore[call-overload] .merge(source, "key = k") .whenNotMatchedInsert(values="k = 'a'", condition={"value": 1})) # ---- bad args in whenNotMatchedBySourceUpdate() with self.assertRaisesRegex(ValueError, "cannot be None"): (dt # type: ignore[call-overload] .merge(source, "key = k") .whenNotMatchedBySourceUpdate({"value": "value"})) with self.assertRaisesRegex(ValueError, "cannot be None"): (dt # type: ignore[call-overload] .merge(source, "key = k") .whenNotMatchedBySourceUpdate(1)) with self.assertRaisesRegex(ValueError, "cannot be None"): (dt # type: ignore[call-overload] .merge(source, "key = k") .whenNotMatchedBySourceUpdate(condition="key = 'a'")) with self.assertRaisesRegex(TypeError, "must be a Spark SQL Column or a string"): (dt # type: ignore[call-overload] .merge(source, "key = k") .whenNotMatchedBySourceUpdate(1, {"value": "value"})) with self.assertRaisesRegex(TypeError, "must be a dict"): (dt # type: ignore[call-overload] .merge(source, "key = k") .whenNotMatchedBySourceUpdate("key = 'a'", 1)) with self.assertRaisesRegex(TypeError, "Values of dict in .* must contain only"): (dt .merge(source, "key = k") .whenNotMatchedBySourceUpdate(set={"value": 1})) # type: ignore[dict-item] with self.assertRaisesRegex(TypeError, "Keys of dict in .* must contain only"): (dt .merge(source, "key = k") .whenNotMatchedBySourceUpdate(set={1: ""})) # type: ignore[dict-item] with self.assertRaises(TypeError): (dt # type: ignore[call-overload] .merge(source, "key = k") .whenNotMatchedBySourceUpdate(set="key = 'a'", condition={"value": 1})) # bad args in whenNotMatchedBySourceDelete() with self.assertRaisesRegex(TypeError, "must be a Spark SQL Column or a string"): dt.merge(source, "key = k").whenNotMatchedBySourceDelete(1) # type: ignore[arg-type] def test_merge_with_inconsistent_sessions(self) -> None: source_path = os.path.join(self.tempFile, "source") target_path = os.path.join(self.tempFile, "target") spark = self.spark def f(spark): # type: ignore[no-untyped-def] spark.range(20) \ .withColumn("x", col("id")) \ .withColumn("y", col("id")) \ .write.mode("overwrite").format("delta").save(source_path) spark.range(1) \ .withColumn("x", col("id")) \ .write.mode("overwrite").format("delta").save(target_path) target = DeltaTable.forPath(spark, target_path) source = spark.read.format("delta").load(source_path).alias("s") target.alias("t") \ .merge(source, "t.id = s.id") \ .whenMatchedUpdate(set={"t.x": "t.x + 1"}) \ .whenNotMatchedInsertAll() \ .execute() assert(spark.read.format("delta").load(target_path).count() == 20) pool = ThreadPool(3) spark.conf.set("spark.databricks.delta.schema.autoMerge.enabled", "true") try: pool.starmap(f, [(spark,)]) finally: spark.conf.unset("spark.databricks.delta.schema.autoMerge.enabled") def test_history(self) -> None: self.__writeDeltaTable([('a', 1), ('b', 2), ('c', 3)]) self.__overwriteDeltaTable([('a', 3), ('b', 2), ('c', 1)]) dt = DeltaTable.forPath(self.spark, self.tempFile) operations = dt.history().select('operation') self.__checkAnswer(operations, [Row("WRITE"), Row("WRITE")], StructType([StructField( "operation", StringType(), True)])) lastMode = dt.history(1).select('operationParameters.mode') self.__checkAnswer( lastMode, [Row("Overwrite")], StructType([StructField("operationParameters.mode", StringType(), True)])) def test_cdc(self) -> None: self.spark.range(0, 5).write.format("delta").save(self.tempFile) deltaTable = DeltaTable.forPath(self.spark, self.tempFile) # Enable Change Data Feed self.spark.sql( "ALTER TABLE delta.`{}` SET TBLPROPERTIES (delta.enableChangeDataFeed = true)" .format(self.tempFile)) # Perform some operations deltaTable.update("id = 1", {"id": "10"}) deltaTable.delete("id = 2") self.spark.range(5, 10).write.format("delta").mode("append").save(self.tempFile) # Check the Change Data Feed expected = [ (1, "update_preimage"), (10, "update_postimage"), (2, "delete"), (5, "insert"), (6, "insert"), (7, "insert"), (8, "insert"), (9, "insert") ] # Read Change Data Feed # (Test handling of the option as boolean and string and with different cases) for option in [True, "true", "tRuE"]: cdf = self.spark.read.format("delta") \ .option("readChangeData", option) \ .option("startingVersion", "1") \ .load(self.tempFile) result = [(row.id, row._change_type) for row in cdf.collect()] self.assertEqual(sorted(result), sorted(expected)) def test_detail(self) -> None: self.__writeDeltaTable([('a', 1), ('b', 2), ('c', 3)]) dt = DeltaTable.forPath(self.spark, self.tempFile) details = dt.detail() self.__checkAnswer( details.select('format'), [Row('delta')], StructType([StructField('format', StringType(), True)]) ) def test_vacuum(self) -> None: self.__writeDeltaTable([('a', 1), ('b', 2), ('c', 3)]) dt = DeltaTable.forPath(self.spark, self.tempFile) self.__createFile('abc.txt', 'abcde') self.__createFile('bac.txt', 'abcdf') self.assertEqual(True, self.__checkFileExists('abc.txt')) dt.vacuum() # will not delete files as default retention is used. dt.vacuum(1000) # test whether integers work self.assertEqual(True, self.__checkFileExists('bac.txt')) retentionConf = "spark.databricks.delta.retentionDurationCheck.enabled" self.spark.conf.set(retentionConf, "false") dt.vacuum(0.0) self.spark.conf.set(retentionConf, "true") self.assertEqual(False, self.__checkFileExists('bac.txt')) self.assertEqual(False, self.__checkFileExists('abc.txt')) def test_convertToDelta(self) -> None: df = self.spark.createDataFrame([('a', 1), ('b', 2), ('c', 3)], ["key", "value"]) df.write.format("parquet").save(self.tempFile) dt = DeltaTable.convertToDelta(self.spark, "parquet.`%s`" % self.tempFile) self.__checkAnswer( self.spark.read.format("delta").load(self.tempFile), [('a', 1), ('b', 2), ('c', 3)]) # test if convert to delta with partition columns work tempFile2 = self.tempFile + "_2" df.write.partitionBy("value").format("parquet").save(tempFile2) schema = StructType() schema.add("value", IntegerType(), True) dt = DeltaTable.convertToDelta( self.spark, "parquet.`%s`" % tempFile2, schema) self.__checkAnswer( self.spark.read.format("delta").load(tempFile2), [('a', 1), ('b', 2), ('c', 3)]) self.assertEqual(type(dt), type(DeltaTable.forPath(self.spark, tempFile2))) # convert to delta with partition column provided as a string tempFile3 = self.tempFile + "_3" df.write.partitionBy("value").format("parquet").save(tempFile3) dt = DeltaTable.convertToDelta( self.spark, "parquet.`%s`" % tempFile3, "value int") self.__checkAnswer( self.spark.read.format("delta").load(tempFile3), [('a', 1), ('b', 2), ('c', 3)]) self.assertEqual(type(dt), type(DeltaTable.forPath(self.spark, tempFile3))) def test_isDeltaTable(self) -> None: df = self.spark.createDataFrame([('a', 1), ('b', 2), ('c', 3)], ["key", "value"]) df.write.format("parquet").save(self.tempFile) tempFile2 = self.tempFile + '_2' df.write.format("delta").save(tempFile2) self.assertEqual(DeltaTable.isDeltaTable(self.spark, self.tempFile), False) self.assertEqual(DeltaTable.isDeltaTable(self.spark, tempFile2), True) def __verify_table_schema(self, tableName: str, schema: StructType, cols: List[str], types: List[DataType], nullables: Set[str] = set(), comments: Dict[str, str] = {}, properties: Dict[str, str] = {}, partitioningColumns: List[str] = [], clusteringColumns: List[str] = [], tblComment: Optional[str] = None) -> None: fields = [] for i in range(len(cols)): col = cols[i] dataType = types[i] metadata = {} if col in comments: metadata["comment"] = comments[col] fields.append(StructField(col, dataType, col in nullables, metadata)) self.assertEqual(StructType(fields), schema) if len(properties) > 0: result = ( self.spark.sql( # type: ignore[assignment, misc] "SHOW TBLPROPERTIES {}".format(tableName) ) .collect()) tablePropertyMap = {row.key: row.value for row in result} for key in properties: self.assertIn(key, tablePropertyMap) self.assertEqual(tablePropertyMap[key], properties[key]) tableDetails = self.spark.sql("DESCRIBE DETAIL {}".format(tableName))\ .collect()[0] self.assertEqual(tableDetails.format, "delta") actualComment = tableDetails.description self.assertEqual(actualComment, tblComment) partitionCols = tableDetails.partitionColumns self.assertEqual(sorted(partitionCols), sorted((partitioningColumns))) clusterByCols = tableDetails.clusteringColumns self.assertEqual(sorted(clusterByCols), sorted(clusteringColumns)) def __verify_generated_column(self, tableName: str, deltaTable: DeltaTable) -> None: cmd = "INSERT INTO {table} (col1, col2) VALUES (1, 11)".format(table=tableName) self.spark.sql(cmd) deltaTable.update(expr("col2 = 11"), {"col1": expr("2")}) self.__checkAnswer(deltaTable.toDF(), [(2, 12)], schema=["col1", "col2"]) def __verify_identity_column(self, tableName: str, deltaTable: DeltaTable) -> None: for i in range(2): cmd = "INSERT INTO {table} (val) VALUES ({i})".format(table=tableName, i=i) self.spark.sql(cmd) cmd = "INSERT INTO {table} (id3, val) VALUES (8, 2)".format(table=tableName) self.spark.sql(cmd) self.__checkAnswer(deltaTable.toDF(), expectedAnswer=[(1, 2, 2, 0), (2, 3, 4, 1), (3, 4, 8, 2)], schema=["id1", "id2", "id3", "val"]) def __build_delta_table(self, builder: DeltaTableBuilder) -> DeltaTable: return builder.addColumn("col1", "int", comment="foo", nullable=False) \ .addColumn("col2", IntegerType(), generatedAlwaysAs="col1 + 10") \ .property("foo", "bar") \ .comment("comment") \ .partitionedBy("col1").execute() def __create_table(self, ifNotExists: bool, tableName: Optional[str] = None, location: Optional[str] = None) -> DeltaTable: builder = DeltaTable.createIfNotExists(self.spark) if ifNotExists \ else DeltaTable.create(self.spark) if tableName: builder = builder.tableName(tableName) if location: builder = builder.location(location) return self.__build_delta_table(builder) def __replace_table(self, orCreate: bool, tableName: Optional[str] = None, location: Optional[str] = None) -> DeltaTable: builder = DeltaTable.createOrReplace(self.spark) if orCreate \ else DeltaTable.replace(self.spark) if tableName: builder = builder.tableName(tableName) if location: builder = builder.location(location) return self.__build_delta_table(builder) def test_create_table_with_existing_schema(self) -> None: df = self.spark.createDataFrame([('a', 1), ('b', 2), ('c', 3)], ["key", "value"]) with self.tempTable() as tableName: deltaTable = DeltaTable.create(self.spark).tableName(tableName) \ .addColumns(df.schema) \ .addColumn("value2", dataType="int")\ .partitionedBy(["value2", "value"])\ .execute() self.__verify_table_schema(tableName, deltaTable.toDF().schema, ["key", "value", "value2"], [StringType(), LongType(), IntegerType()], nullables={"key", "value", "value2"}, partitioningColumns=["value", "value2"]) with self.tempTable() as tableName: # verify creating table with list of structFields deltaTable2 = DeltaTable.create(self.spark).tableName(tableName).addColumns( df.schema.fields) \ .addColumn("value2", dataType="int") \ .partitionedBy("value2", "value")\ .execute() self.__verify_table_schema(tableName, deltaTable2.toDF().schema, ["key", "value", "value2"], [StringType(), LongType(), IntegerType()], nullables={"key", "value", "value2"}, partitioningColumns=["value", "value2"]) def test_create_replace_table_with_cluster_by(self) -> None: df = self.spark.createDataFrame([('a', 1), ('b', 2), ('c', 3)], ["key", "value"]) with self.tempTable() as tableName: # verify creating table with list of structFields deltaTable = DeltaTable.create(self.spark).tableName(tableName).addColumns( df.schema.fields) \ .addColumn("value2", dataType="int") \ .clusterBy("value2", "value")\ .execute() self.__verify_table_schema(tableName, deltaTable.toDF().schema, ["key", "value", "value2"], [StringType(), LongType(), IntegerType()], nullables={"key", "value", "value2"}, clusteringColumns=["value2", "value"], partitioningColumns=[]) deltaTable = DeltaTable.replace(self.spark).tableName(tableName).addColumns( df.schema.fields) \ .addColumn("value2", dataType="int") \ .clusterBy("value2", "value")\ .execute() self.__verify_table_schema(tableName, deltaTable.toDF().schema, ["key", "value", "value2"], [StringType(), LongType(), IntegerType()], nullables={"key", "value", "value2"}, clusteringColumns=["value2", "value"], partitioningColumns=[]) def test_create_replace_table_with_no_spark_session_passed(self) -> None: with self.tempTable() as tableName: # create table. deltaTable = DeltaTable.create().tableName(tableName)\ .addColumn("value", dataType="int").execute() self.__verify_table_schema(tableName, deltaTable.toDF().schema, ["value"], [IntegerType()], nullables={"value"}) # ignore existence with createIfNotExists deltaTable = DeltaTable.createIfNotExists().tableName(tableName) \ .addColumn("value2", dataType="int").execute() self.__verify_table_schema(tableName, deltaTable.toDF().schema, ["value"], [IntegerType()], nullables={"value"}) # replace table with replace deltaTable = DeltaTable.replace().tableName(tableName) \ .addColumn("key", dataType="int").execute() self.__verify_table_schema(tableName, deltaTable.toDF().schema, ["key"], [IntegerType()], nullables={"key"}) # replace with a new column again deltaTable = DeltaTable.createOrReplace().tableName(tableName) \ .addColumn("col1", dataType="int").execute() self.__verify_table_schema(tableName, deltaTable.toDF().schema, ["col1"], [IntegerType()], nullables={"col1"}) def test_create_table_with_name_only(self) -> None: for ifNotExists in (False, True): with self.tempTable() as tableName: deltaTable = self.__create_table(ifNotExists, tableName=tableName) self.__verify_table_schema(tableName, deltaTable.toDF().schema, ["col1", "col2"], [IntegerType(), IntegerType()], nullables={"col2"}, comments={"col1": "foo"}, properties={"foo": "bar"}, partitioningColumns=["col1"], tblComment="comment") # verify generated columns. self.__verify_generated_column(tableName, deltaTable) def test_create_table_with_location_only(self) -> None: for ifNotExists in (False, True): path = self.tempFile + str(ifNotExists) deltaTable = self.__create_table(ifNotExists, location=path) self.__verify_table_schema("delta.`{}`".format(path), deltaTable.toDF().schema, ["col1", "col2"], [IntegerType(), IntegerType()], nullables={"col2"}, comments={"col1": "foo"}, partitioningColumns=["col1"], tblComment="comment") # verify generated columns. self.__verify_generated_column("delta.`{}`".format(path), deltaTable) def test_create_table_with_name_and_location(self) -> None: for ifNotExists in (False, True): path = self.tempFile + str(ifNotExists) with self.tempTable() as tableName: deltaTable = self.__create_table( ifNotExists, tableName=tableName, location=path) self.__verify_table_schema(tableName, deltaTable.toDF().schema, ["col1", "col2"], [IntegerType(), IntegerType()], nullables={"col2"}, comments={"col1": "foo"}, properties={"foo": "bar"}, partitioningColumns=["col1"], tblComment="comment") # verify generated columns. self.__verify_generated_column(tableName, deltaTable) def test_create_table_behavior(self) -> None: with self.tempTable() as tableName: self.spark.sql(f"CREATE TABLE {tableName} (c1 int) USING DELTA") # Errors out if doesn't ignore. with self.assertRaises(AnalysisException) as error_ctx: self.__create_table(False, tableName=tableName) msg = str(error_ctx.exception) assert (tableName in msg and "already exists" in msg) # ignore table creation. self.__create_table(True, tableName=tableName) schema = self.spark.read.format("delta").table(tableName).schema self.__verify_table_schema(tableName, schema, ["c1"], [IntegerType()], nullables={"c1"}) def test_replace_table_with_name_only(self) -> None: for orCreate in (False, True): with self.tempTable() as tableName: self.spark.sql("CREATE TABLE {} (c1 int) USING DELTA".format(tableName)) deltaTable = self.__replace_table(orCreate, tableName=tableName) self.__verify_table_schema(tableName, deltaTable.toDF().schema, ["col1", "col2"], [IntegerType(), IntegerType()], nullables={"col2"}, comments={"col1": "foo"}, properties={"foo": "bar"}, partitioningColumns=["col1"], tblComment="comment") # verify generated columns. self.__verify_generated_column(tableName, deltaTable) def test_replace_table_with_location_only(self) -> None: for orCreate in (False, True): path = self.tempFile + str(orCreate) self.__create_table(False, location=path) deltaTable = self.__replace_table(orCreate, location=path) self.__verify_table_schema("delta.`{}`".format(path), deltaTable.toDF().schema, ["col1", "col2"], [IntegerType(), IntegerType()], nullables={"col2"}, comments={"col1": "foo"}, properties={"foo": "bar"}, partitioningColumns=["col1"], tblComment="comment") # verify generated columns. self.__verify_generated_column("delta.`{}`".format(path), deltaTable) def test_replace_table_with_name_and_location(self) -> None: for orCreate in (False, True): path = self.tempFile + str(orCreate) with self.tempTable() as tableName: self.spark.sql("CREATE TABLE {} (col int) USING DELTA LOCATION '{}'" .format(tableName, path)) deltaTable = self.__replace_table( orCreate, tableName=tableName, location=path) self.__verify_table_schema(tableName, deltaTable.toDF().schema, ["col1", "col2"], [IntegerType(), IntegerType()], nullables={"col2"}, comments={"col1": "foo"}, properties={"foo": "bar"}, partitioningColumns=["col1"], tblComment="comment") # verify generated columns. self.__verify_generated_column(tableName, deltaTable) def test_replace_table_behavior(self) -> None: with self.tempTable() as tableName: with self.assertRaises(AnalysisException) as error_ctx: self.__replace_table(False, tableName=tableName) msg = str(error_ctx.exception) self.assertIn(tableName, msg.lower()) self.assertTrue("did not exist" in msg or "cannot be found" in msg) deltaTable = self.__replace_table(True, tableName=tableName) self.__verify_table_schema(tableName, deltaTable.toDF().schema, ["col1", "col2"], [IntegerType(), IntegerType()], nullables={"col2"}, comments={"col1": "foo"}, properties={"foo": "bar"}, partitioningColumns=["col1"], tblComment="comment") def test_verify_paritionedBy_compatibility(self) -> None: try: from pyspark.sql.column import _to_seq # type: ignore[attr-defined] except ImportError: # Spark 4 from pyspark.sql.classic.column import _to_seq # type: ignore with self.tempTable() as tableName: tableBuilder = DeltaTable.create(self.spark).tableName(tableName) \ .addColumn("col1", "int", comment="foo", nullable=False) \ .addColumn("col2", IntegerType(), generatedAlwaysAs="col1 + 10") \ .property("foo", "bar") \ .comment("comment") tableBuilder._jbuilder = tableBuilder._jbuilder.partitionedBy( _to_seq(self.spark._sc, ["col1"]) # type: ignore[attr-defined] ) deltaTable = tableBuilder.execute() self.__verify_table_schema(tableName, deltaTable.toDF().schema, ["col1", "col2"], [IntegerType(), IntegerType()], nullables={"col2"}, comments={"col1": "foo"}, properties={"foo": "bar"}, partitioningColumns=["col1"], tblComment="comment") def test_create_table_with_identity_column(self) -> None: for ifNotExists in (False, True): with self.tempTable() as tableName: try: self.spark.conf.set("spark.databricks.delta.identityColumn.enabled", "true") builder = ( DeltaTable.createIfNotExists(self.spark) if ifNotExists else DeltaTable.create(self.spark)) builder = builder.tableName(tableName) builder = ( builder.addColumn( "id1", LongType(), generatedAlwaysAs=IdentityGenerator()) .addColumn( "id2", "BIGINT", generatedAlwaysAs=IdentityGenerator(start=2)) .addColumn( "id3", "bigint", generatedByDefaultAs=IdentityGenerator(start=2, step=2)) .addColumn("val", "bigint", nullable=False)) deltaTable = builder.execute() self.__verify_table_schema( tableName, deltaTable.toDF().schema, ["id1", "id2", "id3", "val"], [LongType(), LongType(), LongType(), LongType()], nullables={"id1", "id2", "id3"}) self.__verify_identity_column(tableName, deltaTable) finally: self.spark.conf.unset("spark.databricks.delta.identityColumn.enabled") def test_delta_table_builder_with_bad_args(self) -> None: builder = DeltaTable.create(self.spark).location(self.tempFile) # bad table name with self.assertRaises(TypeError): builder.tableName(1) # type: ignore[arg-type] # bad location with self.assertRaises(TypeError): builder.location(1) # type: ignore[arg-type] # bad comment with self.assertRaises(TypeError): builder.comment(1) # type: ignore[arg-type] # bad column name with self.assertRaises(TypeError): builder.addColumn(1, "int") # type: ignore[arg-type] # bad datatype. with self.assertRaises(TypeError): builder.addColumn("a", 1) # type: ignore[arg-type] # bad column datatype - can't be parsed with self.assertRaises(ParseException): builder.addColumn("a", "1") builder.execute() # reset the builder builder = DeltaTable.create(self.spark).location(self.tempFile) # bad comment with self.assertRaises(TypeError): builder.addColumn("a", "int", comment=1) # type: ignore[arg-type] # bad generatedAlwaysAs with self.assertRaises(TypeError): builder.addColumn("a", "int", generatedAlwaysAs=1) # type: ignore[arg-type] # bad generatedAlwaysAs - identity column data type must be Long with self.assertRaises(UnsupportedOperationException): builder.addColumn( "a", "int", generatedAlwaysAs=IdentityGenerator() ) # type: ignore[arg-type] # exception is thrown in builder.execute() for delta connect builder.execute() # reset the builder builder = DeltaTable.create(self.spark).location(self.tempFile) # bad generatedAlwaysAs - step can't be 0 with self.assertRaises(ValueError): builder.addColumn( "a", "bigint", generatedAlwaysAs=IdentityGenerator(step=0) ) # type: ignore[arg-type] # bad generatedByDefaultAs - can't be set with generatedAlwaysAs with self.assertRaises(ValueError): builder.addColumn( "a", "bigint", generatedAlwaysAs="", generatedByDefaultAs=IdentityGenerator() ) # type: ignore[arg-type] # bad generatedByDefaultAs - argument type must be IdentityGenerator with self.assertRaises(TypeError): builder.addColumn( "a", "bigint", generatedByDefaultAs="" # type: ignore[arg-type] ) # bad generatedByDefaultAs - identity column data type must be Long with self.assertRaises(UnsupportedOperationException): builder.addColumn( "a", "int", generatedByDefaultAs=IdentityGenerator() ) # type: ignore[arg-type] # exception is thrown in builder.execute() for delta connect builder.execute() # reset the builder builder = DeltaTable.create(self.spark).location(self.tempFile) # bad generatedByDefaultAs - step can't be 0 with self.assertRaises(ValueError): builder.addColumn( "a", "bigint", generatedByDefaultAs=IdentityGenerator(step=0) ) # type: ignore[arg-type] # bad nullable with self.assertRaises(TypeError): builder.addColumn("a", "int", nullable=1) # type: ignore[arg-type] # bad existing schema with self.assertRaises(TypeError): builder.addColumns(1) # type: ignore[arg-type] # bad existing schema. with self.assertRaises(TypeError): builder.addColumns([StructField("1", IntegerType()), 1]) # type: ignore[list-item] # bad partitionedBy col name with self.assertRaises(TypeError): builder.partitionedBy(1) # type: ignore[call-overload] with self.assertRaises(TypeError): builder.partitionedBy(1, "1") # type: ignore[call-overload] with self.assertRaises(TypeError): builder.partitionedBy([1]) # type: ignore[list-item] # bad clusterBy col name with self.assertRaises(TypeError): builder.clusterBy(1) # type: ignore[call-overload] with self.assertRaises(TypeError): builder.clusterBy(1, "1") # type: ignore[call-overload] with self.assertRaises(TypeError): builder.clusterBy([1]) # type: ignore[list-item] # bad property key with self.assertRaises(TypeError): builder.property(1, "1") # type: ignore[arg-type] # bad property value with self.assertRaises(TypeError): builder.property("1", 1) # type: ignore[arg-type] def __create_df_for_feature_tests(self) -> DeltaTable: try: self.spark.conf.set('spark.databricks.delta.minReaderVersion', '1') self.spark.conf.set('spark.databricks.delta.minWriterVersion', '2') self.__writeDeltaTable([('a', 1), ('b', 2), ('c', 3), ('d', 4)]) return DeltaTable.forPath(self.spark, self.tempFile) finally: self.spark.conf.unset('spark.databricks.delta.minReaderVersion') self.spark.conf.unset('spark.databricks.delta.minWriterVersion') def test_protocolUpgrade(self) -> None: dt = self.__create_df_for_feature_tests() dt.upgradeTableProtocol(1, 3) # cannot downgrade once upgraded dt.upgradeTableProtocol(1, 2) dt_details = dt.detail().collect()[0].asDict() self.assertTrue(dt_details["minReaderVersion"] == 1, "The upgrade should be a no-op, because downgrades aren't allowed") self.assertTrue(dt_details["minWriterVersion"] == 3, "The upgrade should be a no-op, because downgrades aren't allowed") # bad args with self.assertRaisesRegex(ValueError, "readerVersion"): dt.upgradeTableProtocol("abc", 3) # type: ignore[arg-type] with self.assertRaisesRegex(ValueError, "readerVersion"): dt.upgradeTableProtocol([1], 3) # type: ignore[arg-type] with self.assertRaisesRegex(ValueError, "readerVersion"): dt.upgradeTableProtocol([], 3) # type: ignore[arg-type] with self.assertRaisesRegex(ValueError, "readerVersion"): dt.upgradeTableProtocol({}, 3) # type: ignore[arg-type] with self.assertRaisesRegex(ValueError, "writerVersion"): dt.upgradeTableProtocol(1, "abc") # type: ignore[arg-type] with self.assertRaisesRegex(ValueError, "writerVersion"): dt.upgradeTableProtocol(1, [3]) # type: ignore[arg-type] with self.assertRaisesRegex(ValueError, "writerVersion"): dt.upgradeTableProtocol(1, []) # type: ignore[arg-type] with self.assertRaisesRegex(ValueError, "writerVersion"): dt.upgradeTableProtocol(1, {}) # type: ignore[arg-type] def test_addFeatureSupport(self) -> None: dt = self.__create_df_for_feature_tests() # bad args with self.assertRaisesRegex(Exception, "DELTA_UNSUPPORTED_FEATURES_IN_CONFIG"): dt.addFeatureSupport("abc") with self.assertRaisesRegex(ValueError, "featureName needs to be a string"): dt.addFeatureSupport(12345) # type: ignore[arg-type] with self.assertRaisesRegex(ValueError, "featureName needs to be a string"): dt.addFeatureSupport([12345]) # type: ignore[arg-type] with self.assertRaisesRegex(ValueError, "featureName needs to be a string"): dt.addFeatureSupport({}) # type: ignore[arg-type] with self.assertRaisesRegex(ValueError, "featureName needs to be a string"): dt.addFeatureSupport([]) # type: ignore[arg-type] # good args dt.addFeatureSupport("appendOnly") dt_details = dt.detail().collect()[0].asDict() self.assertTrue(dt_details["minReaderVersion"] == 1, "The upgrade should be a no-op") self.assertTrue(dt_details["minWriterVersion"] == 2, "The upgrade should be a no-op") self.assertEqual(sorted(dt_details["tableFeatures"]), ["appendOnly", "invariants"]) dt.addFeatureSupport("deletionVectors") dt_details = dt.detail().collect()[0].asDict() self.assertTrue(dt_details["minReaderVersion"] == 3, "DV requires reader version 3") self.assertTrue(dt_details["minWriterVersion"] == 7, "DV requires writer version 7") self.assertEqual(sorted(dt_details["tableFeatures"]), ["appendOnly", "deletionVectors", "invariants"]) def test_dropFeatureSupport(self) -> None: # The expected results below are based on drop feature with history truncation. # Fast drop feature, adds a writer feature when dropped. The relevant behavior is tested # in the DeltaFastDropFeatureSuite. self.spark.conf.set('spark.databricks.delta.tableFeatures.fastDropFeature.enabled', 'false') dt = self.__create_df_for_feature_tests() dt.addFeatureSupport("testRemovableWriter") dt_details = dt.detail().collect()[0].asDict() self.assertTrue(dt_details["minReaderVersion"] == 1) self.assertTrue(dt_details["minWriterVersion"] == 7, "Should upgrade to table features") self.assertEqual(sorted(dt_details["tableFeatures"]), ["appendOnly", "invariants", "testRemovableWriter"]) # Attempt truncating the history when dropping a feature that is not required. # This verifies the truncateHistory option was correctly passed. with self.assertRaisesRegex(Exception, "DELTA_FEATURE_DROP_HISTORY_TRUNCATION_NOT_ALLOWED"): dt.dropFeatureSupport("testRemovableWriter", True) dt.dropFeatureSupport("testRemovableWriter") dt_details = dt.detail().collect()[0].asDict() self.assertTrue(dt_details["minReaderVersion"] == 1) self.assertTrue(dt_details["minWriterVersion"] == 2, "Should return to legacy protocol") dt.addFeatureSupport("testRemovableReaderWriter") dt_details = dt.detail().collect()[0].asDict() self.assertTrue(dt_details["minReaderVersion"] == 3, "Should upgrade to table features") self.assertTrue(dt_details["minWriterVersion"] == 7, "Should upgrade to table features") self.assertEqual(sorted(dt_details["tableFeatures"]), ["appendOnly", "invariants", "testRemovableReaderWriter"]) dt.dropFeatureSupport("testRemovableReaderWriter") dt_details = dt.detail().collect()[0].asDict() self.assertTrue(dt_details["minReaderVersion"] == 1, "Should return to legacy protocol") self.assertTrue(dt_details["minWriterVersion"] == 2, "Should return to legacy protocol") # Try to drop an unsupported feature. with self.assertRaisesRegex(Exception, "DELTA_FEATURE_DROP_UNSUPPORTED_CLIENT_FEATURE"): dt.dropFeatureSupport("__invalid_feature__") # Try to drop a feature that is not present in the protocol. with self.assertRaisesRegex(Exception, "DELTA_FEATURE_DROP_FEATURE_NOT_PRESENT"): dt.dropFeatureSupport("testRemovableReaderWriter") # Try to drop a non-removable feature. dt.addFeatureSupport("testReaderWriter") with self.assertRaisesRegex(Exception, "DELTA_FEATURE_DROP_NONREMOVABLE_FEATURE"): dt.dropFeatureSupport("testReaderWriter") with self.assertRaisesRegex(ValueError, "featureName needs to be a string"): dt.dropFeatureSupport(12345) # type: ignore[arg-type] with self.assertRaisesRegex(ValueError, "featureName needs to be a string"): dt.dropFeatureSupport([12345]) # type: ignore[arg-type] with self.assertRaisesRegex(ValueError, "featureName needs to be a string"): dt.dropFeatureSupport({}) # type: ignore[arg-type] with self.assertRaisesRegex(ValueError, "featureName needs to be a string"): dt.dropFeatureSupport([]) # type: ignore[arg-type] with self.assertRaisesRegex(ValueError, "truncateHistory needs to be a boolean"): dt.dropFeatureSupport("testRemovableWriter", 12345) # type: ignore[arg-type] with self.assertRaisesRegex(ValueError, "truncateHistory needs to be a boolean"): dt.dropFeatureSupport("testRemovableWriter", [12345]) # type: ignore[arg-type] with self.assertRaisesRegex(ValueError, "truncateHistory needs to be a boolean"): dt.dropFeatureSupport("testRemovableWriter", {}) # type: ignore[arg-type] with self.assertRaisesRegex(ValueError, "truncateHistory needs to be a boolean"): dt.dropFeatureSupport("testRemovableWriter", []) # type: ignore[arg-type] def test_restore_to_version(self) -> None: self.__writeDeltaTable([('a', 1), ('b', 2)]) self.__overwriteDeltaTable([('a', 3), ('b', 2)], schema=["key_new", "value_new"], overwriteSchema='true') overwritten = DeltaTable.forPath(self.spark, self.tempFile).toDF() self.__checkAnswer(overwritten, [Row(key_new='a', value_new=3), Row(key_new='b', value_new=2)]) DeltaTable.forPath(self.spark, self.tempFile).restoreToVersion(0) restored = DeltaTable.forPath(self.spark, self.tempFile).toDF() self.__checkAnswer(restored, [Row(key='a', value=1), Row(key='b', value=2)]) def test_restore_to_timestamp(self) -> None: self.__writeDeltaTable([('a', 1), ('b', 2)]) timestampToRestore = DeltaTable.forPath(self.spark, self.tempFile) \ .history() \ .head() \ .timestamp \ .strftime('%Y-%m-%d %H:%M:%S.%f') self.__overwriteDeltaTable([('a', 3), ('b', 2)], schema=["key_new", "value_new"], overwriteSchema='true') overwritten = DeltaTable.forPath(self.spark, self.tempFile).toDF() self.__checkAnswer(overwritten, [Row(key_new='a', value_new=3), Row(key_new='b', value_new=2)]) DeltaTable.forPath(self.spark, self.tempFile).restoreToTimestamp(timestampToRestore) restored = DeltaTable.forPath(self.spark, self.tempFile).toDF() self.__checkAnswer(restored, [Row(key='a', value=1), Row(key='b', value=2)]) # we cannot test the actual working of restore to timestamp here but we can make sure # that the api is being called at least def runRestore() -> None: DeltaTable.forPath(self.spark, self.tempFile).restoreToTimestamp('05/04/1999') self.__intercept(runRestore, "The provided timestamp ('05/04/1999') " "cannot be converted to a valid timestamp") def test_restore_invalid_inputs(self) -> None: df = self.spark.createDataFrame([('a', 1), ('b', 2), ('c', 3)], ["key", "value"]) df.write.format("delta").save(self.tempFile) dt = DeltaTable.forPath(self.spark, self.tempFile) def runRestoreToTimestamp() -> None: dt.restoreToTimestamp(12342323232) # type: ignore[arg-type] self.__intercept(runRestoreToTimestamp, "timestamp needs to be a string but got ''") def runRestoreToVersion() -> None: dt.restoreToVersion("0") # type: ignore[arg-type] self.__intercept(runRestoreToVersion, "version needs to be an int but got ''") def test_optimize(self) -> None: # write an unoptimized delta table df = self.spark.createDataFrame([("a", 1), ("a", 2)], ["key", "value"]).repartition(1) df.write.format("delta").save(self.tempFile) df = self.spark.createDataFrame([("a", 3), ("a", 4)], ["key", "value"]).repartition(1) df.write.format("delta").save(self.tempFile, mode="append") df = self.spark.createDataFrame([("b", 1), ("b", 2)], ["key", "value"]).repartition(1) df.write.format("delta").save(self.tempFile, mode="append") # create DeltaTable dt = DeltaTable.forPath(self.spark, self.tempFile) # execute bin compaction optimizer = dt.optimize() res = optimizer.executeCompaction() op_params = dt.history().first().operationParameters # assertions self.assertEqual(1, res.first().metrics.numFilesAdded) self.assertEqual(3, res.first().metrics.numFilesRemoved) self.assertEqual('[]', op_params['predicate']) # test non-partition column def optimize() -> None: dt.optimize().where("key = 'a'").executeCompaction() self.__intercept(optimize, "Predicate references non-partition column 'key'. " "Only the partition columns may be referenced: []") def test_optimize_w_partition_filter(self) -> None: # write an unoptimized delta table df = self.spark.createDataFrame([("a", 1), ("a", 2)], ["key", "value"]).repartition(1) df.write.partitionBy("key").format("delta").save(self.tempFile) df = self.spark.createDataFrame([("a", 3), ("a", 4)], ["key", "value"]).repartition(1) df.write.partitionBy("key").format("delta").save(self.tempFile, mode="append") df = self.spark.createDataFrame([("b", 1), ("b", 2)], ["key", "value"]).repartition(1) df.write.partitionBy("key").format("delta").save(self.tempFile, mode="append") # create DeltaTable dt = DeltaTable.forPath(self.spark, self.tempFile) # execute bin compaction optimizer = dt.optimize().where("key = 'a'") res = optimizer.executeCompaction() op_params = dt.history().first().operationParameters # assertions self.assertEqual(1, res.first().metrics.numFilesAdded) self.assertEqual(2, res.first().metrics.numFilesRemoved) self.assertEqual('''["('key = a)"]''', op_params['predicate']) # test non-partition column def optimize() -> None: dt.optimize().where("value = 1").executeCompaction() self.__intercept(optimize, "Predicate references non-partition column 'value'. " "Only the partition columns may be referenced: [key]") def test_optimize_zorder_by(self) -> None: # write an unoptimized delta table self.spark.createDataFrame([i for i in range(0, 100)], IntegerType()) \ .withColumn("col1", floor(col("value") % 7)) \ .withColumn("col2", floor(col("value") % 27)) \ .withColumn("p", floor(col("value") % 10)) \ .repartition(4).write.partitionBy("p").format("delta").save(self.tempFile) # get the number of data files in the current version numDataFilesPreZOrder = self.spark.read.format("delta").load(self.tempFile) \ .select("_metadata.file_path").distinct().count() # create DeltaTable dt = DeltaTable.forPath(self.spark, self.tempFile) # execute Z-Order Optimization optimizer = dt.optimize() result = optimizer.executeZOrderBy(["col1", "col2"]) metrics = result.select("metrics.*").head() # expect there is only one file after the Z-Order as Z-Order also # does the compaction implicitly and all small files are written to one file # for each partition. Ther are 10 partitions in the table, so expect 10 final files numDataFilesPostZOrder = 10 self.assertEqual(numDataFilesPostZOrder, metrics.numFilesAdded) self.assertEqual(numDataFilesPreZOrder, metrics.numFilesRemoved) self.assertEqual(0, metrics.totalFilesSkipped) self.assertEqual(numDataFilesPreZOrder, metrics.totalConsideredFiles) self.assertEqual('all', metrics.zOrderStats.strategyName) self.assertEqual(10, metrics.zOrderStats.numOutputCubes) # one for each partition # negative test: Z-Order on partition column def optimize() -> None: dt.optimize().where("p = 1").executeZOrderBy(["p"]) self.__intercept(optimize, "p is a partition column. " "Z-Ordering can only be performed on data columns") def test_optimize_zorder_by_w_partition_filter(self) -> None: # write an unoptimized delta table df = self.spark.createDataFrame([i for i in range(0, 100)], IntegerType()) \ .withColumn("col1", floor(col("value") % 7)) \ .withColumn("col2", floor(col("value") % 27)) \ .withColumn("p", floor(col("value") % 10)) \ .repartition(4).write.partitionBy("p") df.format("delta").save(self.tempFile) # get the number of data files in the current version in partition p = 2 numDataFilesPreZOrder = self.spark.read.format("delta").load(self.tempFile) \ .filter("p=2").select("_metadata.file_path").distinct().count() # create DeltaTable dt = DeltaTable.forPath(self.spark, self.tempFile) # execute Z-OrderBy optimizer = dt.optimize().where("p = 2") result = optimizer.executeZOrderBy(["col1", "col2"]) metrics = result.select("metrics.*").head() # expect there is only one file after the Z-Order as Z-Order also # does the compaction implicitly and all small files are written to one file numDataFilesPostZOrder = 1 self.assertEqual(numDataFilesPostZOrder, metrics.numFilesAdded) self.assertEqual(numDataFilesPreZOrder, metrics.numFilesRemoved) self.assertEqual(0, metrics.totalFilesSkipped) # expected to consider all input files for Z-Order self.assertEqual(numDataFilesPreZOrder, metrics.totalConsideredFiles) self.assertEqual('all', metrics.zOrderStats.strategyName) self.assertEqual(1, metrics.zOrderStats.numOutputCubes) # one per each affected partition def test_clone(self) -> None: # type: ignore[no-untyped-def] df = self.spark.createDataFrame([('a', 1), ('b', 2), ('c', 3)], ["key", "value"]) df2 = self.spark.createDataFrame([('d', 4), ('e', 5), ('f', 6)], ["key", "value"]) df.write.format("delta").save(self.tempFile) df2.write.format("delta").mode("overwrite").save(self.tempFile) # source dt = DeltaTable.forPath(self.spark, self.tempFile) tempFile2 = self.tempFile + "_2" tempFile3 = self.tempFile + "_3" dt.clone(tempFile2, True, False, {"foo": "bar"}) props = self.spark.sql('''SHOW TBLPROPERTIES delta.`{}`("foo") '''.format(tempFile2)) self.__checkAnswer(props, [("foo", "bar")]) self.__checkAnswer( self.spark.read.format("delta").load(tempFile2), [('d', 4), ('e', 5), ('f', 6)]) dt.cloneAtVersion(0, tempFile3, True) self.__checkAnswer( self.spark.read.format("delta").load(tempFile3), [('a', 1), ('b', 2), ('c', 3)]) # clone over tempFile3 with source at current version dt.clone(tempFile3, True, True) self.__checkAnswer( self.spark.read.format("delta").load(tempFile3), [('d', 4), ('e', 5), ('f', 6)]) def test_clone_invalid_inputs(self) -> None: # type: ignore[no-untyped-def] df = self.spark.createDataFrame([('a', 1), ('b', 2), ('c', 3)], ["key", "value"]) df.write.format("delta").save(self.tempFile) # source dt = DeltaTable.forPath(self.spark, self.tempFile) tempFile2 = self.tempFile + "_2" def incorrectTarget() -> "DeltaTable": return dt.clone(10) self.__intercept(incorrectTarget, "target needs to be a string but got int") def incorrectShallow() -> "DeltaTable": return dt.clone(tempFile2, isShallow=10) self.__intercept(incorrectShallow, "isShallow needs to be a boolean but got int") def incorrectReplace() -> "DeltaTable": return dt.clone(tempFile2, False, replace=10) self.__intercept(incorrectReplace, "replace needs to be a boolean but got int") def incorrectProperties() -> "DeltaTable": return dt.clone(tempFile2, False, False, properties=10) self.__intercept(incorrectProperties, "properties needs to be a dict but got int") def incorrectPropertyValue() -> "DeltaTable": return dt.clone(tempFile2, False, False, properties={"key": 10}) self.__intercept(incorrectPropertyValue, "All property values including 10" " needs to be a str but got int") def incorrectVersion() -> "DeltaTable": return dt.cloneAtVersion("0", tempFile2, False, False) self.__intercept(incorrectVersion, "version needs to be an int but got string") def incorrectTimestamp() -> "DeltaTable": return dt.cloneAtTimestamp(10, tempFile2, False, False) self.__intercept(incorrectTimestamp, "timestamp needs to be a string but got int") def test_create_table_with_cluster_by(self) -> None: with self.tempTable() as tableName: builder = DeltaTable.create(self.spark) self.__test_table_with_cluster_by( tableName, builder, lambda builder: builder.clusterBy(["value2", "value"]), expected=["value", "value2"]) def test_replace_table_with_cluster_by(self) -> None: with self.tempTable() as tableName: self.spark.sql(f"CREATE TABLE {tableName} (c1 int) USING DELTA") builder = DeltaTable.replace(self.spark) self.__test_table_with_cluster_by( tableName, builder, lambda builder: builder.clusterBy("value2", "value"), expected=["value", "value2"]) # type: ignore[arg-type] def __test_table_with_cluster_by(self, tableName: str, builder: "JavaObject", setClusterBy: Callable[["JavaObject"], None], expected: List[str]) -> None: df = self.spark.createDataFrame([('a', 1), ('b', 2), ('c', 3)], ["key", "value"]) builder = builder.tableName(tableName) \ .addColumns(df.schema) \ .addColumn("value2", dataType="int") setClusterBy(builder) deltaTable = builder.execute() self.__verify_table_schema(tableName, deltaTable.toDF().schema, ["key", "value", "value2"], [StringType(), LongType(), IntegerType()], nullables={"key", "value", "value2"}, clusteringColumns=expected) def test_cluster_by_bad_args(self) -> None: builder = DeltaTable.create(self.spark).location(self.tempFile) # bad clusterBy col name with self.assertRaises(TypeError): builder.clusterBy(1) # type: ignore[call-overload] with self.assertRaises(TypeError): builder.clusterBy(1, "1") # type: ignore[call-overload] with self.assertRaises(TypeError): builder.clusterBy([1]) # type: ignore[list-item] def __checkAnswer(self, df: DataFrame, expectedAnswer: List[Any], schema: Union[StructType, List[str]] = ["key", "value"]) -> None: if not expectedAnswer: self.assertEqual(df.count(), 0) return expectedDF = self.spark.createDataFrame(expectedAnswer, schema) try: self.assertEqual(df.count(), expectedDF.count()) self.assertEqual(len(df.columns), len(expectedDF.columns)) self.assertEqual([], df.subtract(expectedDF).take(1)) self.assertEqual([], expectedDF.subtract(df).take(1)) except AssertionError: print("Expected:") expectedDF.show() print("Found:") df.show() raise def __writeDeltaTable(self, datalist: List[Tuple[Any, Any]]) -> None: df = self.spark.createDataFrame(datalist, ["key", "value"]) df.write.format("delta").save(self.tempFile) def __writeAsTable(self, datalist: List[Tuple[Any, Any]], tblName: str) -> None: df = self.spark.createDataFrame(datalist, ["key", "value"]) df.write.format("delta").saveAsTable(tblName) def __overwriteDeltaTable(self, datalist: List[Tuple[Any, Any]], schema: Union[StructType, List[str]] = ["key", "value"], overwriteSchema: str = 'false') -> None: df = self.spark.createDataFrame(datalist, schema) df.write.format("delta") \ .option('overwriteSchema', overwriteSchema) \ .mode("overwrite") \ .save(self.tempFile) def __createFile(self, fileName: str, content: Any) -> None: with open(os.path.join(self.tempFile, fileName), 'w') as f: f.write(content) def __checkFileExists(self, fileName: str) -> bool: return os.path.exists(os.path.join(self.tempFile, fileName)) def __intercept(self, func: Callable[[], None], exceptionMsg: str) -> None: seenTheRightException = False try: func() except Exception as e: if exceptionMsg in str(e): seenTheRightException = True assert seenTheRightException, ("Did not catch expected Exception:" + exceptionMsg) class DeltaTableTests(DeltaTableTestsMixin, DeltaTestCase): pass if __name__ == "__main__": try: import xmlrunner testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=4) except ImportError: testRunner = None unittest.main(testRunner=testRunner, verbosity=4) ================================================ FILE: python/delta/tests/test_exceptions.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from typing import Any, Callable, TYPE_CHECKING import unittest import delta.exceptions.captured as exceptions from delta.testing.utils import DeltaTestCase from pyspark.sql.utils import AnalysisException, IllegalArgumentException if TYPE_CHECKING: from py4j.java_gateway import JVMView # type: ignore[import] class DeltaExceptionTests(DeltaTestCase): def setUp(self) -> None: super(DeltaExceptionTests, self).setUp() self.jvm: "JVMView" = self.spark.sparkContext._jvm # type: ignore[attr-defined] def _raise_concurrent_exception(self, exception_type: Callable[[Any], Any]) -> None: e = exception_type("") self.jvm.scala.util.Failure(e).get() def test_capture_concurrent_write_exception(self) -> None: e = self.jvm.io.delta.exceptions.ConcurrentWriteException self.assertRaises(exceptions.ConcurrentWriteException, lambda: self._raise_concurrent_exception(e)) def test_capture_metadata_changed_exception(self) -> None: e = self.jvm.io.delta.exceptions.MetadataChangedException self.assertRaises(exceptions.MetadataChangedException, lambda: self._raise_concurrent_exception(e)) def test_capture_protocol_changed_exception(self) -> None: e = self.jvm.io.delta.exceptions.ProtocolChangedException self.assertRaises(exceptions.ProtocolChangedException, lambda: self._raise_concurrent_exception(e)) def test_capture_concurrent_append_exception(self) -> None: e = self.jvm.io.delta.exceptions.ConcurrentAppendException self.assertRaises(exceptions.ConcurrentAppendException, lambda: self._raise_concurrent_exception(e)) def test_capture_concurrent_delete_read_exception(self) -> None: e = self.jvm.io.delta.exceptions.ConcurrentDeleteReadException self.assertRaises(exceptions.ConcurrentDeleteReadException, lambda: self._raise_concurrent_exception(e)) def test_capture_concurrent_delete_delete_exception(self) -> None: e = self.jvm.io.delta.exceptions.ConcurrentDeleteDeleteException self.assertRaises(exceptions.ConcurrentDeleteDeleteException, lambda: self._raise_concurrent_exception(e)) def test_capture_concurrent_transaction_exception(self) -> None: e = self.jvm.io.delta.exceptions.ConcurrentTransactionException self.assertRaises(exceptions.ConcurrentTransactionException, lambda: self._raise_concurrent_exception(e)) def test_capture_delta_analysis_exception(self) -> None: e = self.jvm.org.apache.spark.sql.delta.DeltaErrors.invalidColumnName self.assertRaises(AnalysisException, lambda: self.jvm.scala.util.Failure(e("invalid")).get()) def test_capture_delta_illegal_argument_exception(self) -> None: e = self.jvm.org.apache.spark.sql.delta.DeltaErrors method = e.throwDeltaIllegalArgumentException self.assertRaises(IllegalArgumentException, lambda: self.jvm.scala.util.Failure(method()).get()) if __name__ == "__main__": try: import xmlrunner testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=4) except ImportError: testRunner = None unittest.main(testRunner=testRunner, verbosity=4) ================================================ FILE: python/delta/tests/test_pip_utils.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os import shutil import tempfile import unittest from typing import List, Optional from pyspark.sql import SparkSession import delta class PipUtilsTests(unittest.TestCase): def setUp(self) -> None: builder = SparkSession.builder \ .appName("pip-test") \ .master("local[*]") \ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") self.spark = delta.configure_spark_with_delta_pip(builder).getOrCreate() self.tempPath = tempfile.mkdtemp() self.tempFile = os.path.join(self.tempPath, "tempFile") def tearDown(self) -> None: self.spark.stop() shutil.rmtree(self.tempPath) def test_maven_jar_loaded(self) -> None: # Read and write Delta table to check that the maven jars are loaded and Delta works. self.spark.range(0, 5).write.format("delta").save(self.tempFile) self.spark.read.format("delta").load(self.tempFile) class PipUtilsCustomJarsTests(unittest.TestCase): def setUp(self) -> None: builder = SparkSession.builder \ .appName("pip-test") \ .master("local[*]") \ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") import importlib_metadata scala_version = "2.12" delta_version = importlib_metadata.version("delta_spark") maven_artifacts = [f"io.delta:delta-spark_{scala_version}:{delta_version}"] # configure extra packages self.spark = delta.configure_spark_with_delta_pip(builder, maven_artifacts).getOrCreate() self.tempPath = tempfile.mkdtemp() self.tempFile = os.path.join(self.tempPath, "tempFile") def tearDown(self) -> None: self.spark.stop() shutil.rmtree(self.tempPath) def test_maven_jar_loaded(self) -> None: packagesConf: Optional[str] = self.spark.conf.get("spark.jars.packages") assert packagesConf is not None # mypi needs this to assign type str from Optional[str] packages: str = packagesConf packagesList: List[str] = packages.split(",") # Check `spark.jars.packages` contains `extra_packages` self.assertTrue(len(packagesList) == 2, "There should only be 2 packages") # Read and write Delta table to check that the maven jars are loaded and Delta works. self.spark.range(0, 5).write.format("delta").save(self.tempFile) self.spark.read.format("delta").load(self.tempFile) if __name__ == "__main__": try: import xmlrunner testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=4) except ImportError: testRunner = None unittest.main(testRunner=testRunner, verbosity=4) ================================================ FILE: python/delta/tests/test_sql.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # mypy: disable-error-code="union-attr" # mypy: disable-error-code="attr-defined" import unittest import tempfile import shutil import os from typing import List, Any from pyspark.sql import DataFrame from delta.testing.utils import DeltaTestCase class DeltaSqlTests(DeltaTestCase): def setUp(self) -> None: super(DeltaSqlTests, self).setUp() # Create a simple Delta table inside the temp directory to test SQL commands. df = self.spark.createDataFrame([('a', 1), ('b', 2), ('c', 3)], ["key", "value"]) df.write.format("delta").save(self.tempFile) df.write.mode("overwrite").format("delta").save(self.tempFile) def test_vacuum(self) -> None: self.spark.sql("set spark.databricks.delta.retentionDurationCheck.enabled = false") try: deleted_files = self.spark.sql("VACUUM '%s' RETAIN 0 HOURS" % self.tempFile).collect() # Verify `VACUUM` did delete some data files self.assertTrue(self.tempFile in deleted_files[0][0]) finally: self.spark.sql("set spark.databricks.delta.retentionDurationCheck.enabled = true") def test_describe_history(self) -> None: self.assertGreater( len(self.spark.sql("desc history delta.`%s`" % (self.tempFile)).collect()), 0) def test_generate(self) -> None: # create a delta table temp_path = tempfile.mkdtemp() temp_file = os.path.join(temp_path, "delta_sql_test_table") numFiles = 10 self.spark.range(100).repartition(numFiles).write.format("delta").save(temp_file) # Generate the symlink format manifest self.spark.sql("GENERATE SYMLINK_FORMAT_MANIFEST FOR TABLE delta.`{}`" .format(temp_file)) # check the contents of the manifest # NOTE: this is not a correctness test, we are testing correctness in the scala suite manifestPath = os.path.join(temp_file, os.path.join("_symlink_format_manifest", "manifest")) files = [] with open(manifestPath) as f: files = f.readlines() shutil.rmtree(temp_path) # the number of files we write should equal the number of lines in the manifest self.assertEqual(len(files), numFiles) def test_convert(self) -> None: df = self.spark.createDataFrame([('a', 1), ('b', 2), ('c', 3)], ["key", "value"]) temp_path2 = tempfile.mkdtemp() temp_path3 = tempfile.mkdtemp() temp_file2 = os.path.join(temp_path2, "delta_sql_test2") temp_file3 = os.path.join(temp_path3, "delta_sql_test3") df.write.format("parquet").save(temp_file2) self.spark.sql("CONVERT TO DELTA parquet.`" + temp_file2 + "`") self.__checkAnswer( self.spark.read.format("delta").load(temp_file2), [('a', 1), ('b', 2), ('c', 3)]) # test if convert to delta with partition columns work df.write.partitionBy("value").format("parquet").save(temp_file3) self.spark.sql("CONVERT TO DELTA parquet.`" + temp_file3 + "` PARTITIONED BY (value INT)") self.__checkAnswer( self.spark.read.format("delta").load(temp_file3), [('a', 1), ('b', 2), ('c', 3)]) shutil.rmtree(temp_path2) shutil.rmtree(temp_path3) def test_ddls(self) -> None: table = "deltaTable" table2 = "deltaTable2" with self.table(table, table + "_part", table2): def read_table() -> DataFrame: return self.spark.sql(f"SELECT * FROM {table}") self.spark.sql(f"DROP TABLE IF EXISTS {table}") self.spark.sql(f"DROP TABLE IF EXISTS {table}_part") self.spark.sql(f"DROP TABLE IF EXISTS {table2}") self.spark.sql(f"CREATE TABLE {table}(a LONG, b String NOT NULL) USING delta") self.assertEqual(read_table().count(), 0) self.spark.sql(f"CREATE TABLE {table}_part(a LONG, b String NOT NULL)" " USING delta PARTITIONED BY (a)") # Unpartitioned table does not include partitioning information in Spark 3.4+ answer = [("a", "bigint"), ("b", "string")] self.__checkAnswer( self.spark.sql(f"DESCRIBE TABLE {table}").select("col_name", "data_type"), answer, schema=["col_name", "data_type"]) answer_part = [("a", "bigint"), ("b", "string"), ("# Partition Information", ""), ("# col_name", "data_type"), ("a", "bigint")] self.__checkAnswer( self.spark.sql(f"DESCRIBE TABLE {table}_part").select("col_name", "data_type"), answer_part, schema=["col_name", "data_type"]) self.spark.sql(f"ALTER TABLE {table} CHANGE COLUMN a a LONG AFTER b") self.assertSequenceEqual(["b", "a"], [f.name for f in read_table().schema.fields]) self.spark.sql(f"ALTER TABLE {table} ALTER COLUMN b DROP NOT NULL") self.assertIn(True, [f.nullable for f in read_table().schema.fields if f.name == "b"]) self.spark.sql(f"ALTER TABLE {table} ADD COLUMNS (x LONG)") self.assertIn("x", [f.name for f in read_table().schema.fields]) self.spark.sql(f"ALTER TABLE {table} SET TBLPROPERTIES ('k' = 'v')") self.__checkAnswer(self.spark.sql(f"SHOW TBLPROPERTIES {table}"), [('k', 'v'), ('delta.minReaderVersion', '1'), ('delta.minWriterVersion', '2')]) self.spark.sql(f"ALTER TABLE {table} UNSET TBLPROPERTIES ('k')") self.__checkAnswer(self.spark.sql(f"SHOW TBLPROPERTIES {table}"), [('delta.minReaderVersion', '1'), ('delta.minWriterVersion', '2')]) self.spark.sql(f"ALTER TABLE {table} RENAME TO {table2}") self.assertEqual(self.spark.sql(f"SELECT * FROM {table2}").count(), 0) test_dir = os.path.join(tempfile.mkdtemp(), table2) self.spark.createDataFrame([("", 0, 0)], ["b", "a", "x"]) \ .write.format("delta").save(test_dir) self.spark.sql(f"ALTER TABLE {table2} SET LOCATION '{test_dir}'") self.assertEqual(self.spark.sql(f"SELECT * FROM {table2}").count(), 1) def __checkAnswer(self, df: DataFrame, expectedAnswer: List[Any], schema: List[str] = ["key", "value"]) -> None: if not expectedAnswer: self.assertEqual(df.count(), 0) return expectedDF = self.spark.createDataFrame(expectedAnswer, schema) self.assertEqual(df.count(), expectedDF.count()) self.assertEqual(len(df.columns), len(expectedDF.columns)) self.assertEqual([], df.subtract(expectedDF).take(1)) self.assertEqual([], expectedDF.subtract(df).take(1)) if __name__ == "__main__": try: import xmlrunner testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=4) except ImportError: testRunner = None unittest.main(testRunner=testRunner, verbosity=4) ================================================ FILE: python/delta/tests/test_version.py ================================================ # # Copyright (2026) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os import unittest from packaging.version import Version class VersionAPITests(unittest.TestCase): version_sbt_path = os.path.abspath(os.path.join( os.path.dirname(__file__), '..', '..', '..', 'version.sbt') ) def verify_version(self, version: str) -> None: self.assertIsNotNone(version) self.assertIsInstance(version, str) self.assertEqual(version.count("."), 2, "Version should have major.minor.patch format") # version should be parseable by packaging.version.Version Version(version) def test_version_import_from_module(self) -> None: """Test that __version__ can be imported from delta.version""" from delta.version import __version__ self.verify_version(__version__) def test_version_import_from_package(self) -> None: """Test that __version__ can be imported from delta package""" from delta import __version__ self.verify_version(__version__) def test_version_consistency_across_imports(self) -> None: """Test that version is consistent across import methods""" from delta.version import __version__ as version_from_module from delta import __version__ as version_from_package self.assertEqual(version_from_module, version_from_package) def test_version_sbt_exists(self) -> None: """Verify version.sbt exists""" self.assertTrue( os.path.exists(self.version_sbt_path), f"version.sbt not found at {self.version_sbt_path}" ) def test_version_sbt_and_version_py_consistency(self) -> None: with open(self.version_sbt_path) as f: sbt_content = f.read() # Extract version from: ThisBuild / version := "x.y.z-SNAPSHOT" -> "x.y.z" sbt_version = sbt_content.split('"')[1].removesuffix("-SNAPSHOT") from delta import __version__ self.assertEqual( __version__, sbt_version, f"version.py ({__version__}) does not match version.sbt ({sbt_version}). " f"Run: build/sbt sparkV1/generatePythonVersion" ) if __name__ == '__main__': unittest.main() ================================================ FILE: python/delta/version.py ================================================ # # Copyright (2026) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # This file is auto-generated by the build.sbt generatePythonVersion task. # Do not edit manually - edit version.sbt instead and run: # build/sbt sparkV1/generatePythonVersion __version__ = "4.1.0" ================================================ FILE: python/environment.yml ================================================ name: delta_python_tests channels: - defaults - https://repo.anaconda.com/pkgs/main - https://repo.anaconda.com/pkgs/r dependencies: - _libgcc_mutex=0.1=main - _openmp_mutex=5.1=1_gnu - ca-certificates=2023.08.22=h06a4308_0 - ld_impl_linux-64=2.40=h12ee557_0 - libcxx=14.0.6=h83ecd13_0 - libcxxabi=14.0.6=h06a4308_0 - libffi=3.4.4=h6a678d5_1 - libgcc-ng=11.2.0=h1234567_1 - libgomp=11.2.0=h1234567_1 - libstdcxx-ng=11.2.0=h1234567_1 - ncurses=6.4=h6a678d5_0 - openssl=3.0.11=h7f8727e_2 - python=3.8.18=h955ad1f_0 - readline=8.2=h5eee18b_0 - sqlite=3.41.2=h5eee18b_0 - tk=8.6.12=h1ccaba5_0 - xz=5.4.2=h5eee18b_0 - zlib=1.2.13=h5eee18b_1 - pip: - alabaster==0.7.13 - babel==2.13.0 - backports-tarfile==1.2.0 - black==23.9.1 - certifi==2023.7.22 - cffi==1.17.1 - charset-normalizer==3.3.0 - click==8.1.8 - colorama==0.4.6 - cryptography==37.0.4 - delta-spark==3.4.0-SNAPSHOT - docutils==0.15.2 - flake8==3.5.0 - idna==3.4 - imagesize==1.4.1 - importlib-metadata==8.5.0 - importlib-resources==6.4.5 - jaraco-classes==3.4.0 - jaraco-context==6.0.1 - jaraco-functools==4.1.0 - jeepney==0.9.0 - jinja2==2.11.3 - keyring==25.5.0 - livereload==2.6.3 - markdown-it-py==3.0.0 - markupsafe==2.0.0 - mccabe==0.6.1 - mdurl==0.1.2 - more-itertools==10.5.0 - mypy==0.982 - mypy-extensions==1.0.0 - mypy-protobuf==3.3.0 - nh3==0.2.21 - numpy==1.24.4 - packaging==23.2 - pandas==1.1.3 - pathspec==0.12.1 - pip==24.0 - pkginfo==1.12.1.2 - platformdirs==4.3.6 - protobuf==5.29.3 - py4j==0.10.9.7 - pyarrow==8.0.0 - pycodestyle==2.3.1 - pycparser==2.22 - pydocstyle==3.0.0 - pyflakes==1.6.0 - pygments==2.16.1 - pypandoc==1.3.3 - pyspark==3.5.3 - python-dateutil==2.9.0.post0 - pytz==2023.3.post1 - readme-renderer==43.0 - requests==2.31.0 - requests-toolbelt==1.0.0 - rfc3986==2.0.0 - rich==13.9.4 - secretstorage==3.3.3 - setuptools==41.1.0 - six==1.16.0 - snowballstemmer==2.2.0 - sphinx==2.0.1 - sphinx-autobuild==2021.3.14 - sphinxcontrib-applehelp==1.0.4 - sphinxcontrib-devhelp==1.0.2 - sphinxcontrib-htmlhelp==2.0.1 - sphinxcontrib-jsmath==1.0.1 - sphinxcontrib-qthelp==1.0.3 - sphinxcontrib-serializinghtml==1.1.5 - tomli==2.2.1 - tornado==6.3.3 - twine==4.0.1 - types-protobuf==5.29.1.20241207 - typing-extensions==4.12.2 - urllib3==2.0.6 - wheel==0.33.4 - zipp==3.20.2 prefix: /miniconda3/envs/delta_python_tests ================================================ FILE: python/mypy.ini ================================================ ; ; Copyright (2021) The Delta Lake Project Authors. ; ; Licensed under the Apache License, Version 2.0 (the "License"); ; you may not use this file except in compliance with the License. ; You may obtain a copy of the License at ; ; http://www.apache.org/licenses/LICENSE-2.0 ; ; Unless required by applicable law or agreed to in writing, software ; distributed under the License is distributed on an "AS IS" BASIS, ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ; See the License for the specific language governing permissions and ; limitations under the License. ; [mypy] strict_optional = True no_implicit_optional = True disallow_untyped_defs = True show_error_codes = True [mypy-xmlrunner.*] ignore_missing_imports = True [mypy-delta.connect.proto.proto.*] ignore_errors = True ignore_missing_imports = True [mypy-delta.connect.*] ignore_errors = True ignore_missing_imports = True [mypy-google.*] ignore_missing_imports = True [mypy-py4j.*] ignore_missing_imports = True ================================================ FILE: python/run-tests.py ================================================ #!/usr/bin/env python3 # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os import subprocess import shutil from os import path import json def test(root_dir, code_dir, packages): # Test the codes in the code_dir directory using its "tests" subdirectory, # each of them has main entry point to execute, which is python's unittest testing # framework. python_root_dir = path.join(root_dir, "python") test_dir = path.join(python_root_dir, path.join(code_dir, "tests")) test_files = [os.path.join(test_dir, f) for f in os.listdir(test_dir) if os.path.isfile(os.path.join(test_dir, f)) and f.endswith(".py") and not f.startswith("_")] extra_class_path = path.join(python_root_dir, path.join(code_dir, "testing")) # Include Maven local repository to resolve locally published Delta artifacts maven_local_repo = "file://" + os.path.expanduser("~/.m2/repository") for test_file in test_files: try: cmd = ["spark-submit", "--driver-class-path=%s" % extra_class_path, "--repositories", (f"{maven_local_repo}," "https://maven-central.storage-download.googleapis.com/maven2/," "https://repo1.maven.org/maven2/," "https://repository.apache.org/content/repositories/orgapachespark-1484"), "--packages", ",".join(packages), test_file] print("Running tests in %s\n=============" % test_file) print("Command: %s" % str(cmd)) run_cmd(cmd, stream_output=True) except: print("Failed tests in %s" % (test_file)) raise def delete_if_exists(path): # if path exists, delete it. if os.path.exists(path): shutil.rmtree(path) print("Deleted %s " % path) def prepare(root_dir, spark_version): print("##### Preparing python tests & building packages #####") # Build package with python files in it sbt_path = path.join(root_dir, path.join("build", "sbt")) ivy_caches_to_clear = [ filepath for filepath in os.listdir(os.path.expanduser("~")) if filepath.startswith(".ivy") ] print(f"Clearing Ivy caches in: {ivy_caches_to_clear}") for filepath in ivy_caches_to_clear: delete_if_exists(os.path.expanduser(f"~/{filepath}/cache/io.delta")) delete_if_exists(os.path.expanduser("~/.m2/repository/io/delta/")) sbt_command = [sbt_path] sbt_command = sbt_command + [f"-DsparkVersion={spark_version}"] run_cmd(sbt_command + ["clean", "publishM2"], stream_output=True) def get_local_package(package_name, spark_version, root_dir): """Get the Maven coordinates for a Delta package. Queries CrossSparkVersions for the packageSuffix (e.g., "", "_4.1"). Args: package_name: Name of the package (e.g., "delta-spark", "delta-connect-server") spark_version: Spark version string (e.g., "4.0", "4.1", or "default") root_dir: Root directory of the Delta repository Returns: Maven coordinates string (e.g., "io.delta:delta-spark_2.13:4.1.0-SNAPSHOT") """ # Get current release version version = '0.0.0' with open(os.path.join(root_dir, "version.sbt")) as fd: version = fd.readline().split('"')[1] # Get package suffix directly from CrossSparkVersions (single source of truth) script_path = os.path.join(root_dir, "project", "scripts", "get_spark_version_info.py") try: result = subprocess.run( ["python3", script_path, "--get-field", spark_version, "packageSuffix"], cwd=root_dir, capture_output=True, text=True, check=True ) package_name_suffix = json.loads(result.stdout.strip()) except Exception as e: print(f"Warning: Could not determine package suffix for Spark {spark_version}: {e}") print(f"Falling back to empty suffix") package_name_suffix = "" return f"io.delta:{package_name}{package_name_suffix}_2.13:" + version def run_cmd(cmd, throw_on_error=True, env=None, stream_output=False, print_cmd=True, **kwargs): if print_cmd: print("### Executing cmd: " + " ".join(cmd)) cmd_env = os.environ.copy() if env: cmd_env.update(env) if stream_output: child = subprocess.Popen(cmd, env=cmd_env, **kwargs) exit_code = child.wait() if throw_on_error and exit_code != 0: raise Exception("Non-zero exitcode: %s" % (exit_code)) return exit_code else: child = subprocess.Popen( cmd, env=cmd_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) (stdout, stderr) = child.communicate() exit_code = child.wait() if throw_on_error and exit_code != 0: raise Exception( "Non-zero exitcode: %s\n\nSTDOUT:\n%s\n\nSTDERR:%s" % (exit_code, stdout, stderr)) return (exit_code, stdout, stderr) def run_python_style_checks(root_dir): print("##### Running python style tests #####") run_cmd([os.path.join(root_dir, "dev", "lint-python")], stream_output=True) def run_mypy_tests(root_dir): print("##### Running mypy tests #####") python_package_root = path.join(root_dir, path.join("python", "delta")) mypy_config_path = path.join(root_dir, path.join("python", "mypy.ini")) run_cmd([ "mypy", "--config-file", mypy_config_path, python_package_root ], stream_output=True) def run_pypi_packaging_tests(root_dir): """ We want to test that the delta-spark PyPi artifact for this delta version can be generated, locally installed, and used in python tests. We will uninstall any existing local delta-spark PyPi artifact. We will generate a new local delta-spark PyPi artifact. We will install it into the local PyPi repository. And then we will run relevant python tests to ensure everything works as expected. """ print("##### Running PyPi Packaging tests #####") version = '0.0.0' with open(os.path.join(root_dir, "version.sbt")) as fd: version = fd.readline().split('"')[1] # uninstall packages if they exist run_cmd(["pip3", "uninstall", "--yes", "delta-spark"], stream_output=True) wheel_dist_dir = path.join(root_dir, "dist") print("### Deleting `dist` directory if it exists") delete_if_exists(wheel_dist_dir) # generate artifacts run_cmd( ["python3", "setup.py", "bdist_wheel"], stream_output=True, stderr=open('/dev/null', 'w')) run_cmd(["python3", "setup.py", "sdist"], stream_output=True) # we need, for example, 1.1.0_SNAPSHOT not 1.1.0-SNAPSHOT version_formatted = version.replace("-", "_") delta_whl_name = "delta_spark-" + version_formatted + "-py3-none-any.whl" # this will install delta-spark-$version install_whl_cmd = ["pip3", "install", path.join(wheel_dist_dir, delta_whl_name)] run_cmd(install_whl_cmd, stream_output=True) # run test python file directly with python and not with spark-submit test_file = path.join(root_dir, path.join("examples", "python", "using_with_pip.py")) test_cmd = ["python3", test_file] try: print("### Starting tests...") run_cmd(test_cmd, stream_output=True) except: print("Failed pip installation tests in %s" % (test_file)) raise def run_delta_connect_codegen_python(root_dir): print("##### Running generated Delta Connect Python protobuf codes syncing tests #####") test_file = os.path.join(root_dir, "dev", "check-delta-connect-codegen-python.py") test_cmd = ["python3", test_file] run_cmd(test_cmd, stream_output=True) if __name__ == "__main__": print("##### Running python tests #####") root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) spark_version = os.getenv("SPARK_VERSION") or "default" prepare(root_dir, spark_version) delta_spark_package = get_local_package("delta-spark", spark_version, root_dir) run_python_style_checks(root_dir) run_mypy_tests(root_dir) run_pypi_packaging_tests(root_dir) test(root_dir, "delta", [delta_spark_package]) # Run Delta Connect tests as well run_delta_connect_codegen_python(root_dir) # TODO: In the future, find a way to get these # packages locally instead of downloading from Maven. # Get the full Spark version for spark-connect artifact script_path = os.path.join(root_dir, "project", "scripts", "get_spark_version_info.py") result = subprocess.run( ["python3", script_path, "--get-field", spark_version, "fullVersion"], cwd=root_dir, capture_output=True, text=True, check=True ) spark_full_version = json.loads(result.stdout.strip()) delta_connect_packages = ["com.google.protobuf:protobuf-java:3.25.1", f"org.apache.spark:spark-connect_2.13:{spark_full_version}", get_local_package("delta-connect-server", spark_version, root_dir)] test(root_dir, path.join("delta", "connect"), delta_connect_packages) ================================================ FILE: run-integration-tests.py ================================================ #!/usr/bin/env python3 # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # # Integration test script for Delta Lake. Builds artifacts locally and runs # Scala, Python, and pip tests against them. # # Usage: # python run-integration-tests.py --use-local # Run all tests # python run-integration-tests.py --use-local --scala-only # Scala tests only # python run-integration-tests.py --use-local --python-only # Python tests only # # Setup: # With --use-local, tests run across all Spark versions defined in # CrossSparkVersions.scala. Each Spark version needs a local distribution at: # ~/spark-{version}-bin-hadoop3/ # # Download them from Apache: # wget https://archive.apache.org/dist/spark/spark-{version}/spark-{version}-bin-hadoop3.tgz # tar xzf spark-{version}-bin-hadoop3.tgz -C ~/ # import os import subprocess from os import path import shutil import argparse import json _original_path = os.environ.get("PATH", "") def set_spark_env(spark_version): """ Sets SPARK_HOME and prepends its bin/ to PATH for the given Spark version. Resets PATH to its original value first to avoid accumulation. This must override any existing SPARK_HOME because the multi-variant loop tests different Spark versions in sequence (e.g., 4.0.1 then 4.1.0). """ os.environ["PATH"] = _original_path # In non-local mode, spark_version is "" — tests resolve artifacts from Maven Central # and use whatever spark-submit is already on PATH. SNAPSHOT versions also have no # pre-built distribution to look up, so we fall back to PATH. if not spark_version or "-SNAPSHOT" in spark_version: print("Using spark-submit from PATH for version %s" % (spark_version or "unspecified")) return spark_home = os.path.expanduser("~/spark-%s-bin-hadoop3" % spark_version) if not os.path.isdir(spark_home): raise Exception( "Spark %s not found at %s. Please download it first:\n" " wget https://archive.apache.org/dist/spark/spark-%s/spark-%s-bin-hadoop3.tgz\n" " tar xzf spark-%s-bin-hadoop3.tgz -C ~/" % (spark_version, spark_home, spark_version, spark_version, spark_version)) os.environ["SPARK_HOME"] = spark_home spark_bin = os.path.join(spark_home, "bin") os.environ["PATH"] = spark_bin + os.pathsep + _original_path print("Using SPARK_HOME=%s" % spark_home) def delete_if_exists(path): # if path exists, delete it. if os.path.exists(path): shutil.rmtree(path) print("Deleted %s " % path) def load_spark_version_specs(root_dir): """ Loads Spark version specs from target/spark-versions.json (single source of truth). Runs `build/sbt exportSparkVersionsJson` if the file doesn't exist yet. Returns a list of dicts with keys: fullVersion, shortVersion, isMaster, isDefault, targetJvm, packageSuffix, supportIceberg, supportHudi. """ json_path = path.join(root_dir, "target", "spark-versions.json") if not path.exists(json_path): print("Generating %s via exportSparkVersionsJson..." % json_path) run_cmd(["build/sbt", "exportSparkVersionsJson"], stream_output=True) with open(json_path) as f: return json.load(f) def publish_all_variants(root_dir, spark_specs): """ Publishes all artifact variants once upfront (replaces per-function publishM2 calls). Step 1: Publish all modules WITHOUT Spark suffix (backward compatibility) Step 2: Publish Spark-dependent modules WITH suffix for each non-master Spark version """ # Step 1: unsuffixed (backward compat) print("\n##### Publishing all modules without Spark suffix (backward compat) #####") run_cmd(["build/sbt", "-DskipSparkSuffix=true", "publishM2"], stream_output=True) # Step 2: suffixed for each non-master Spark version # Clean between publishes to avoid stale class files from different Spark shims for spec in spark_specs: if spec.get("isMaster", False): continue spark_version = spec["fullVersion"] print("\n##### Publishing Spark-dependent modules for Spark %s #####" % spark_version) run_cmd( ["build/sbt", "-DsparkVersion=%s" % spark_version, "runOnlyForReleasableSparkModules clean", "runOnlyForReleasableSparkModules publishM2"], stream_output=True) def get_spark_variants(spark_specs): """ Builds the list of artifact variants to test from the Spark version specs. Each variant is a dict with: - suffix: Maven artifact suffix, e.g. "" (unsuffixed), "_4.0", "_4.1" - spark_version: full Spark version, e.g. "4.1.0", "4.0.1" - support_iceberg: "true" or "false" - support_hudi: "true" or "false" The first variant is always unsuffixed (backward compat) using the DEFAULT spec's metadata. Remaining variants are suffixed, one per non-master Spark version. Example return value (given Spark 4.0 and 4.1 specs, with 4.1 as default): [ {"suffix": "", "spark_version": "4.1.0", "support_iceberg": "false", "support_hudi": "false"}, {"suffix": "_4.0", "spark_version": "4.0.1", "support_iceberg": "true", "support_hudi": "true"}, {"suffix": "_4.1", "spark_version": "4.1.0", "support_iceberg": "false", "support_hudi": "false"}, ] """ variants = [] # Find the default spec for the unsuffixed backward-compat variant default_spec = None for spec in spark_specs: if spec.get("isDefault", False): default_spec = spec break if default_spec is None and spark_specs: default_spec = spark_specs[-1] # fallback to last spec # Unsuffixed variant (backward compat) - uses default's metadata if default_spec: variants.append({ "suffix": "", "spark_version": default_spec["fullVersion"], "support_iceberg": default_spec.get("supportIceberg", "false"), "support_hudi": default_spec.get("supportHudi", "false"), }) # Suffixed variants for each non-master spec for spec in spark_specs: if spec.get("isMaster", False): continue variants.append({ "suffix": spec["packageSuffix"], "spark_version": spec["fullVersion"], "support_iceberg": spec.get("supportIceberg", "false"), "support_hudi": spec.get("supportHudi", "false"), }) return variants def run_scala_integration_tests(root_dir, version, test_name, extra_maven_repo, scala_version, variant): """ Runs Scala integration tests for a single artifact variant. variant: dict with suffix, spark_version, support_iceberg, support_hudi. See get_spark_variants() for the format and example. """ suffix = variant["suffix"] spark_version = variant["spark_version"] support_iceberg = variant["support_iceberg"] label = " (suffix=%s, spark=%s)" % (suffix or "none", spark_version) if suffix or spark_version else "" print("\n\n##### Running Scala tests%s on delta %s, scala %s #####" % (label, str(version), scala_version)) test_dir = path.join(root_dir, "examples", "scala") test_src_dir = path.join(test_dir, "src", "main", "scala", "example") test_classes = [f.replace(".scala", "") for f in os.listdir(test_src_dir) if f.endswith(".scala") and not f.startswith("_")] # Set env vars that examples/scala/build.sbt reads to resolve dependencies: # SPARK_PACKAGE_SUFFIX -> artifact suffix (e.g., "_4.0") # SPARK_VERSION -> Spark version for spark-sql/spark-hive deps (e.g., "4.0.1") # SUPPORT_ICEBERG -> whether to include Iceberg deps and compile IcebergCompat examples env = {"DELTA_VERSION": str(version), "SCALA_VERSION": scala_version} if suffix: env["SPARK_PACKAGE_SUFFIX"] = suffix if spark_version: env["SPARK_VERSION"] = spark_version if support_iceberg == "true": env["SUPPORT_ICEBERG"] = "true" if extra_maven_repo: env["EXTRA_MAVEN_REPO"] = extra_maven_repo with WorkingDirectory(test_dir): for test_class in test_classes: if test_name is not None and test_name not in test_class: print("\nSkipping Scala tests in %s\n=====================" % test_class) continue # Skip Iceberg tests for variants that don't support Iceberg if "IcebergCompat" in test_class and support_iceberg != "true": print("\nSkipping %s (Iceberg not supported for this variant)\n=====================" % test_class) continue try: cmd = ["build/sbt", "runMain example.%s" % test_class] print("\nRunning Scala tests in %s%s\n=====================" % (test_class, label)) print("Command: %s" % " ".join(cmd)) run_cmd(cmd, stream_output=True, env=env) except: print("Failed Scala tests in %s%s" % (test_class, label)) raise def get_artifact_name(version): """ version: string representation, e.g. 2.3.0 or 3.0.0.rc1 return: either "core" or "spark" """ return "spark" if int(version[0]) >= 3 else "core" def run_python_integration_tests(root_dir, version, test_name, extra_maven_repo, variant): """ Runs Python integration tests for a single artifact variant. variant: dict with suffix, spark_version, support_iceberg, support_hudi. See get_spark_variants() for the format and example. """ suffix = variant["suffix"] label = " (suffix=%s)" % (suffix or "none") if suffix else "" print("\n\n##### Running Python tests%s on version %s #####" % (label, str(version))) test_dir = path.join(root_dir, path.join("examples", "python")) files_to_skip = {"using_with_pip.py", "missing_delta_storage_jar.py", "image_storage.py", "delta_connect.py"} test_files = [path.join(test_dir, f) for f in os.listdir(test_dir) if path.isfile(path.join(test_dir, f)) and f.endswith(".py") and not f.startswith("_") and f not in files_to_skip] python_root_dir = path.join(root_dir, "python") extra_class_path = path.join(python_root_dir, path.join("delta", "testing")) repo = extra_maven_repo if extra_maven_repo else "" # Build Maven coordinate with the variant's suffix # e.g., "io.delta:delta-spark_2.13:4.0.0" or "io.delta:delta-spark_4.0_2.13:4.0.0" artifact_name = get_artifact_name(version) package = "io.delta:delta-%s%s_2.13:%s" % (artifact_name, suffix, version) print("Package: %s" % package) for test_file in test_files: if test_name is not None and test_name not in test_file: print("\nSkipping Python tests in %s\n=====================" % test_file) continue try: cmd = ["spark-submit", "--driver-class-path=%s" % extra_class_path, # for less verbose logging "--packages", package, "--repositories", repo, test_file] print("\nRunning Python tests in %s%s\n=============" % (test_file, label)) print("Command: %s" % " ".join(cmd)) run_cmd(cmd, stream_output=True) except: print("Failed Python tests in %s%s" % (test_file, label)) raise def test_missing_delta_storage_jar(root_dir, version, use_local): if not use_local: print("Skipping 'missing_delta_storage_jar' - test should only run in local mode") return print("\n\n##### Running 'missing_delta_storage_jar' on version %s #####" % str(version)) # The unsuffixed artifact was published via publish_all_variants upfront. # Clear only the delta-storage artifact to test the missing JAR scenario. print("Clearing delta-storage artifact") delete_if_exists(os.path.expanduser("~/.m2/repository/io/delta/delta-storage")) delete_if_exists(os.path.expanduser("~/.ivy2/cache/io.delta/delta-storage")) delete_if_exists(os.path.expanduser("~/.ivy2/local/io.delta/delta-storage")) delete_if_exists(os.path.expanduser("~/.ivy2.5.2/local/io.delta/delta-storage")) delete_if_exists(os.path.expanduser("~/.ivy2.5.2/cache/io.delta/delta-storage")) python_root_dir = path.join(root_dir, "python") extra_class_path = path.join(python_root_dir, path.join("delta", "testing")) test_file = path.join(root_dir, path.join("examples", "python", "missing_delta_storage_jar.py")) artifact_name = get_artifact_name(version) # Uses unsuffixed artifact name (published via -DskipSparkSuffix=true) jar = path.join( os.path.expanduser("~/.m2/repository/io/delta/"), "delta-%s_2.13" % artifact_name, version, "delta-%s_2.13-%s.jar" % (artifact_name, str(version))) try: cmd = ["spark-submit", "--driver-class-path=%s" % extra_class_path, # for less verbose logging "--jars", jar, test_file] print("\nRunning Python tests in %s\n=============" % test_file) print("Command: %s" % " ".join(cmd)) run_cmd(cmd, stream_output=True) except: print("Failed Python tests in %s" % (test_file)) raise def run_dynamodb_logstore_integration_tests(root_dir, version, test_name, extra_maven_repo, extra_packages, conf, variant): """ Runs DynamoDB logstore integration tests for a single artifact variant. variant: dict with suffix, spark_version, support_iceberg, support_hudi. See get_spark_variants() for the format and example. """ suffix = variant["suffix"] label = " (suffix=%s)" % (suffix or "none") if suffix else "" print( "\n\n##### Running DynamoDB logstore integration tests%s on version %s #####" % (label, str(version)) ) test_dir = path.join(root_dir, path.join("storage-s3-dynamodb", "integration_tests")) test_files = [path.join(test_dir, f) for f in os.listdir(test_dir) if path.isfile(path.join(test_dir, f)) and f.endswith(".py") and not f.startswith("_")] python_root_dir = path.join(root_dir, "python") extra_class_path = path.join(python_root_dir, path.join("delta", "testing")) conf_args = [] if conf: for i in conf: conf_args.extend(["--conf", i]) repo_args = ["--repositories", extra_maven_repo] if extra_maven_repo else [] # Build package string: delta-spark with suffix + delta-storage-s3-dynamodb (Spark-independent, no suffix) artifact_name = get_artifact_name(version) packages = "io.delta:delta-%s%s_2.13:%s" % (artifact_name, suffix, version) packages += "," + "io.delta:delta-storage-s3-dynamodb:" + version if extra_packages: packages += "," + extra_packages for test_file in test_files: if test_name is not None and test_name not in test_file: print("\nSkipping DynamoDB logstore integration tests in %s\n============" % test_file) continue try: cmd = ["spark-submit", "--driver-class-path=%s" % extra_class_path, # for less verbose logging "--packages", packages] + repo_args + conf_args + [test_file] print("\nRunning DynamoDB logstore integration tests in %s%s\n=============" % (test_file, label)) print("Command: %s" % " ".join(cmd)) run_cmd(cmd, stream_output=True) except: print("Failed DynamoDB logstore integration tests tests in %s%s" % (test_file, label)) raise def run_dynamodb_commit_coordinator_integration_tests(root_dir, version, test_name, extra_maven_repo, extra_packages, conf, variant): """ Runs DynamoDB Commit Coordinator integration tests for a single artifact variant. variant: dict with suffix, spark_version, support_iceberg, support_hudi. See get_spark_variants() for the format and example. """ suffix = variant["suffix"] label = " (suffix=%s)" % (suffix or "none") if suffix else "" print( "\n\n##### Running DynamoDB Commit Coordinator integration tests%s on version %s #####" % (label, str(version)) ) test_dir = path.join(root_dir, \ path.join("spark", "src", "main", "java", "io", "delta", "dynamodbcommitcoordinator", "integration_tests")) test_files = [path.join(test_dir, f) for f in os.listdir(test_dir) if path.isfile(path.join(test_dir, f)) and f.endswith(".py") and not f.startswith("_")] python_root_dir = path.join(root_dir, "python") extra_class_path = path.join(python_root_dir, path.join("delta", "testing")) conf_args = [] if conf: for i in conf: conf_args.extend(["--conf", i]) repo_args = ["--repositories", extra_maven_repo] if extra_maven_repo else [] # Build package string with the variant's suffix artifact_name = get_artifact_name(version) packages = "io.delta:delta-%s%s_2.13:%s" % (artifact_name, suffix, version) if extra_packages: packages += "," + extra_packages for test_file in test_files: if test_name is not None and test_name not in test_file: print("\nSkipping DynamoDB Commit Coordinator integration tests in %s\n============" % test_file) continue try: cmd = ["spark-submit", "--driver-class-path=%s" % extra_class_path, # for less verbose logging "--packages", packages] + repo_args + conf_args + [test_file] print("\nRunning DynamoDB Commit Coordinator integration tests in %s%s\n=============" % (test_file, label)) print("Command: %s" % " ".join(cmd)) run_cmd(cmd, stream_output=True) except: print("Failed DynamoDB Commit Coordinator integration tests in %s%s" % (test_file, label)) raise def run_s3_log_store_util_integration_tests(): print("\n\n##### Running S3LogStoreUtil tests #####") env = { "S3_LOG_STORE_UTIL_TEST_ENABLED": "true" } assert os.environ.get("S3_LOG_STORE_UTIL_TEST_BUCKET") is not None, "S3_LOG_STORE_UTIL_TEST_BUCKET must be set" assert os.environ.get("S3_LOG_STORE_UTIL_TEST_RUN_UID") is not None, "S3_LOG_STORE_UTIL_TEST_RUN_UID must be set" try: cmd = ["build/sbt", "project storage", "testOnly -- -n IntegrationTest"] print("\nRunning IntegrationTests of storage\n=====================") print("Command: %s" % " ".join(cmd)) run_cmd(cmd, stream_output=True, env=env) except: print("Failed IntegrationTests") raise def run_iceberg_integration_tests(root_dir, version, iceberg_version, extra_maven_repo, variant): """ Runs Iceberg integration tests for a single artifact variant. variant: dict with suffix, spark_version, support_iceberg, support_hudi. See get_spark_variants() for the format and example. spark_version is used to derive the iceberg-spark-runtime artifact name (e.g., "4.0.1" -> iceberg-spark-runtime-4.0_2.13). """ suffix = variant["suffix"] spark_version = variant["spark_version"] label = " (suffix=%s)" % (suffix or "none") if suffix else "" print("\n\n##### Running Iceberg tests%s on version %s #####" % (label, str(version))) test_dir = path.join(root_dir, path.join("iceberg", "integration_tests")) # Add more Iceberg tests here if needed ... test_files_names = ["iceberg_converter.py"] test_files = [path.join(test_dir, f) for f in test_files_names] python_root_dir = path.join(root_dir, "python") extra_class_path = path.join(python_root_dir, path.join("delta", "testing")) repo = extra_maven_repo if extra_maven_repo else "" artifact_name = get_artifact_name(version) # Derive major.minor Spark version for iceberg-spark-runtime artifact name # e.g., "4.0.1" -> "4.0", or "4.0" stays "4.0" parts = spark_version.split(".") iceberg_spark_ver = "%s.%s" % (parts[0], parts[1]) if len(parts) >= 2 else spark_version # Build package string with suffixed Delta artifacts + Iceberg runtime package = ','.join([ "io.delta:delta-%s%s_2.13:%s" % (artifact_name, suffix, version), "io.delta:delta-iceberg_2.13:%s" % (version), "org.apache.iceberg:iceberg-spark-runtime-{}_2.13:{}".format(iceberg_spark_ver, iceberg_version)]) print("Package: %s" % package) for test_file in test_files: try: cmd = ["spark-submit", "--driver-class-path=%s" % extra_class_path, # for less verbose logging "--packages", package, "--repositories", repo, test_file] print("\nRunning Iceberg tests in %s%s\n=============" % (test_file, label)) print("Command: %s" % " ".join(cmd)) run_cmd(cmd, stream_output=True) except: print("Failed Iceberg tests in %s%s" % (test_file, label)) raise def run_uniform_hudi_integration_tests(root_dir, version, hudi_version, extra_maven_repo, variant): """ Runs Uniform Hudi integration tests for a single artifact variant. variant: dict with suffix, spark_version, support_iceberg, support_hudi. See get_spark_variants() for the format and example. spark_version is used to derive the hudi-spark-bundle artifact name (e.g., "4.0.1" -> hudi-spark4.0-bundle_2.13). """ suffix = variant["suffix"] spark_version = variant["spark_version"] label = " (suffix=%s)" % (suffix or "none") if suffix else "" print("\n\n##### Running Uniform hudi tests%s on version %s #####" % (label, str(version))) test_dir = path.join(root_dir, path.join("hudi", "integration_tests")) # Add more tests here if needed ... test_files_names = ["write_uniform_hudi.py"] test_files = [path.join(test_dir, f) for f in test_files_names] python_root_dir = path.join(root_dir, "python") extra_class_path = path.join(python_root_dir, path.join("delta", "testing")) # The hudi assembly JAR path uses name.value (no suffix), not moduleName jars = path.join(root_dir, "hudi/target/scala-2.13/delta-hudi-assembly_2.13-%s.jar" % (version)) repo = extra_maven_repo if extra_maven_repo else "" artifact_name = get_artifact_name(version) # Derive major.minor Spark version for hudi-spark-bundle artifact name # e.g., "4.0.1" -> "4.0", or "4.0" stays "4.0" parts = spark_version.split(".") hudi_spark_ver = "%s.%s" % (parts[0], parts[1]) if len(parts) >= 2 else spark_version # Build package string with suffixed Delta artifact + Hudi bundle package = ','.join([ "io.delta:delta-%s%s_2.13:%s" % (artifact_name, suffix, version), "org.apache.hudi:hudi-spark%s-bundle_2.13:%s" % (hudi_spark_ver, hudi_version) ]) print("Package: %s" % package) for test_file in test_files: try: cmd = ["spark-submit", "--driver-class-path=%s" % extra_class_path, # for less verbose logging "--packages", package, "--jars", jars, "--repositories", repo, test_file] print("\nRunning Uniform Hudi tests in %s%s\n=============" % (test_file, label)) print("Command: %s" % " ".join(cmd)) run_cmd(cmd, stream_output=True) except: print("Failed Uniform Hudi tests in %s%s" % (test_file, label)) raise def run_pip_installation_tests(root_dir, version, use_testpypi, use_localpypi, extra_maven_repo): print("\n\n##### Running pip installation tests on version %s #####" % str(version)) # Note: no clear_artifact_cache() here. Pip tests install from PyPI, not local M2. delta_pip_name = "delta-spark" # uninstall packages if they exist run_cmd(["pip", "uninstall", "--yes", delta_pip_name, "pyspark"], stream_output=True) # install packages delta_pip_name_with_version = "%s==%s" % (delta_pip_name, str(version)) if use_testpypi: install_cmd = ["pip", "install", "--extra-index-url", "https://test.pypi.org/simple/", delta_pip_name_with_version] elif use_localpypi: pip_wheel_file_name = "%s-%s-py3-none-any.whl" % \ (delta_pip_name.replace("-", "_"), str(version)) pip_wheel_file_path = os.path.join(use_localpypi, pip_wheel_file_name) install_cmd = ["pip", "install", pip_wheel_file_path] else: install_cmd = ["pip", "install", delta_pip_name_with_version] print("pip install command: %s" % str(install_cmd)) run_cmd(install_cmd, stream_output=True) # run test python file directly with python and not with spark-submit env = {} if extra_maven_repo: env["EXTRA_MAVEN_REPO"] = extra_maven_repo tests = ["image_storage.py", "using_with_pip.py"] for test in tests: test_file = path.join(root_dir, path.join("examples", "python", test)) print("\nRunning Python tests in %s\n=============" % test_file) test_cmd = ["python3", test_file] print("Test command: %s" % str(test_cmd)) try: run_cmd(test_cmd, stream_output=True, env=env) except: print("Failed pip installation tests in %s" % (test_file)) raise def run_unity_catalog_commit_coordinator_integration_tests(root_dir, version, test_name, variant, extra_packages): """ Runs Unity Catalog commit coordinator integration tests for a single artifact variant. variant: dict with suffix, spark_version, support_iceberg, support_hudi. See get_spark_variants() for the format and example. """ suffix = variant["suffix"] label = " (suffix=%s)" % (suffix or "none") if suffix else "" print( "\n\n##### Running Unity Catalog commit coordinator integration tests%s on version %s #####" % (label, str(version)) ) test_dir = path.join(root_dir, \ path.join("python", "delta", "integration_tests")) test_files = [path.join(test_dir, f) for f in os.listdir(test_dir) if path.isfile(path.join(test_dir, f)) and f.endswith(".py") and not f.startswith("_")] print("\n\nTests compiled\n\n") python_root_dir = path.join(root_dir, "python") extra_class_path = path.join(python_root_dir, path.join("delta", "testing")) # Build package string with the variant's suffix artifact_name = get_artifact_name(version) packages = "io.delta:delta-%s%s_2.13:%s" % (artifact_name, suffix, version) if extra_packages: packages += "," + extra_packages for test_file in test_files: if test_name is not None and test_name not in test_file: print("\nSkipping Unity Catalog commit coordinator integration tests in %s\n============" % test_file) continue try: cmd = ["spark-submit", "--driver-class-path=%s" % extra_class_path, # for less verbose logging "--packages", packages] + [test_file] print("\nRunning External uc managed tables integration tests in %s%s\n=============" % (test_file, label)) print("Command: %s" % " ".join(cmd)) run_cmd(cmd, stream_output=True) except: print("Failed Unity Catalog commit coordinator integration tests in %s%s" % (test_file, label)) raise def clear_artifact_cache(): print("Clearing Delta artifacts from ivy2 and mvn cache") ivy_caches_to_clear = [filepath for filepath in os.listdir(os.path.expanduser("~")) if filepath.startswith(".ivy")] print(f"Clearing Ivy caches in: {ivy_caches_to_clear}") for filepath in ivy_caches_to_clear: delete_if_exists(os.path.expanduser(f"~/{filepath}/cache/io.delta")) delete_if_exists(os.path.expanduser(f"~/{filepath}/local/io.delta")) delete_if_exists(os.path.expanduser("~/.m2/repository/io/delta/")) def run_cmd(cmd, throw_on_error=True, env=None, stream_output=False, **kwargs): cmd_env = os.environ.copy() if env: cmd_env.update(env) if stream_output: child = subprocess.Popen(cmd, env=cmd_env, **kwargs) exit_code = child.wait() if throw_on_error and exit_code != 0: raise Exception("Non-zero exitcode: %s" % (exit_code)) return exit_code else: child = subprocess.Popen( cmd, env=cmd_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) (stdout, stderr) = child.communicate() exit_code = child.wait() if throw_on_error and exit_code != 0: raise Exception( "Non-zero exitcode: %s\n\nSTDOUT:\n%s\n\nSTDERR:%s" % (exit_code, stdout, stderr)) return (exit_code, stdout, stderr) # pylint: disable=too-few-public-methods class WorkingDirectory(object): def __init__(self, working_directory): self.working_directory = working_directory self.old_workdir = os.getcwd() def __enter__(self): os.chdir(self.working_directory) def __exit__(self, tpe, value, traceback): os.chdir(self.old_workdir) if __name__ == "__main__": """ Script to run integration tests which are located in the examples directory. call this by running "python run-integration-tests.py" additionally the version can be provided as a command line argument. " """ # get the version of the package root_dir = path.dirname(__file__) with open(path.join(root_dir, "version.sbt")) as fd: default_version = fd.readline().split('"')[1] parser = argparse.ArgumentParser() parser.add_argument( "--version", required=False, default=default_version, help="Delta version to use to run the integration tests") parser.add_argument( "--python-only", required=False, default=False, action="store_true", help="Run only Python tests") parser.add_argument( "--scala-only", required=False, default=False, action="store_true", help="Run only Scala tests") parser.add_argument( "--s3-log-store-util-only", required=False, default=False, action="store_true", help="Run only S3LogStoreUtil tests") parser.add_argument( "--scala-version", required=False, default="2.13", help="Specify scala version for scala tests only, valid values are '2.13'") parser.add_argument( "--pip-only", required=False, default=False, action="store_true", help="Run only pip installation tests") parser.add_argument( "--no-pip", required=False, default=False, action="store_true", help="Do not run pip installation tests") parser.add_argument( "--test", required=False, default=None, help="Run a specific test by substring-match with Scala/Python file name") parser.add_argument( "--maven-repo", required=False, default=None, help="Additional Maven repo to resolve staged new release artifacts") parser.add_argument( "--use-testpypi", required=False, default=False, action="store_true", help="Use testpypi for testing pip installation") parser.add_argument( "--use-localpypiartifact", required=False, default=None, help="Directory path where the downloaded pypi artifacts are present. " + "It should have two files: e.g. delta_spark-3.1.0.tar.gz, delta_spark-3.1.0-py3-none-any.whl") parser.add_argument( "--use-local", required=False, default=False, action="store_true", help="Generate JARs from local source code and use to run tests") parser.add_argument( "--run-storage-s3-dynamodb-integration-tests", required=False, default=False, action="store_true", help="Run the DynamoDB integration tests (and only them)") parser.add_argument( "--packages", required=False, default=None, help="Additional packages required for integration tests") parser.add_argument( "--dbb-conf", required=False, default=None, nargs="+", help="All `--conf` values passed to `spark-submit` for DynamoDB logstore/commit-coordinator integration tests") parser.add_argument( "--run-dynamodb-commit-coordinator-integration-tests", required=False, default=False, action="store_true", help="Run the DynamoDB Commit Coordinator tests (and only them)") parser.add_argument( "--run-iceberg-integration-tests", required=False, default=False, action="store_true", help="Run the Iceberg integration tests (and only them)") parser.add_argument( "--run-uniform-hudi-integration-tests", required=False, default=False, action="store_true", help="Run the Uniform Hudi integration tests (and only them)") parser.add_argument( "--iceberg-spark-version", required=False, default="4.0", help="Spark version for the Iceberg library (used in non-local mode)") parser.add_argument( "--iceberg-lib-version", required=False, default="1.4.0", help="Iceberg Spark Runtime library version") parser.add_argument( "--hudi-spark-version", required=False, default="4.0", help="Spark version for the Hudi library (used in non-local mode)") parser.add_argument( "--hudi-version", required=False, default="0.15.0", help="Hudi library version" ) parser.add_argument( "--unity-catalog-commit-coordinator-integration-tests", required=False, default=False, action="store_true", help="Run the Unity Catalog Commit Coordinator tests (and only them)" ) args = parser.parse_args() if args.scala_version not in ["2.13"]: raise Exception("Scala version can only be specified as --scala-version 2.13") if args.pip_only and args.no_pip: raise Exception("Cannot specify both --pip-only and --no-pip") if args.use_local and (args.version != default_version): raise Exception("Cannot specify --use-local with a --version different than in version.sbt") # When --use-local, publish all artifact variants once upfront and build the variant list # from CrossSparkVersions.scala. In non-local mode, use a single default (unsuffixed) variant. default_variant = { "suffix": "", "spark_version": "", "support_iceberg": "false", "support_hudi": "false" } spark_specs = None variants = [default_variant] if args.use_local: spark_specs = load_spark_version_specs(root_dir) clear_artifact_cache() publish_all_variants(root_dir, spark_specs) variants = get_spark_variants(spark_specs) run_python = not args.scala_only and not args.pip_only run_scala = not args.python_only and not args.pip_only run_pip = not args.python_only and not args.scala_only and not args.no_pip if args.run_iceberg_integration_tests: # In local mode, only test variants that support Iceberg. # In non-local mode, run once with --iceberg-spark-version from CLI args. if spark_specs: iceberg_variants = [v for v in variants if v["support_iceberg"] == "true"] if not iceberg_variants: print("No Spark variants support Iceberg - skipping Iceberg integration tests") quit() else: iceberg_variants = [{ "suffix": "", "spark_version": args.iceberg_spark_version, "support_iceberg": "true", "support_hudi": "false" }] for variant in iceberg_variants: set_spark_env(variant["spark_version"]) run_iceberg_integration_tests( root_dir, args.version, args.iceberg_lib_version, args.maven_repo, variant) quit() if args.run_uniform_hudi_integration_tests: # Build hudi assembly once before running tests (needs specific Spark version) if args.use_local: hudi_spark_ver = None if spark_specs: for spec in spark_specs: if spec.get("supportHudi", "false") == "true": hudi_spark_ver = spec["fullVersion"] break if hudi_spark_ver: run_cmd(["build/sbt", "-DsparkVersion=%s" % hudi_spark_ver, "hudi/assembly"], stream_output=True) else: run_cmd(["build/sbt", "hudi/assembly"], stream_output=True) # In local mode, only test variants that support Hudi. # In non-local mode, run once with --hudi-spark-version from CLI args. if spark_specs: hudi_variants = [v for v in variants if v["support_hudi"] == "true"] if not hudi_variants: print("No Spark variants support Hudi - skipping Hudi integration tests") quit() else: hudi_variants = [{ "suffix": "", "spark_version": args.hudi_spark_version, "support_iceberg": "false", "support_hudi": "true" }] for variant in hudi_variants: set_spark_env(variant["spark_version"]) run_uniform_hudi_integration_tests( root_dir, args.version, args.hudi_version, args.maven_repo, variant) quit() if args.run_storage_s3_dynamodb_integration_tests: for variant in variants: set_spark_env(variant["spark_version"]) run_dynamodb_logstore_integration_tests(root_dir, args.version, args.test, args.maven_repo, args.packages, args.dbb_conf, variant) quit() if args.run_dynamodb_commit_coordinator_integration_tests: for variant in variants: set_spark_env(variant["spark_version"]) run_dynamodb_commit_coordinator_integration_tests(root_dir, args.version, args.test, args.maven_repo, args.packages, args.dbb_conf, variant) quit() if args.s3_log_store_util_only: run_s3_log_store_util_integration_tests() quit() if args.unity_catalog_commit_coordinator_integration_tests: for variant in variants: set_spark_env(variant["spark_version"]) run_unity_catalog_commit_coordinator_integration_tests(root_dir, args.version, args.test, variant, args.packages) quit() # Run the standard test suite: Scala, Python, pip # Each test function is called once per variant (the loop is here, not inside the functions) if run_scala: for variant in variants: set_spark_env(variant["spark_version"]) run_scala_integration_tests(root_dir, args.version, args.test, args.maven_repo, args.scala_version, variant) if run_python: for variant in variants: set_spark_env(variant["spark_version"]) run_python_integration_tests(root_dir, args.version, args.test, args.maven_repo, variant) test_missing_delta_storage_jar(root_dir, args.version, args.use_local) if run_pip: if args.use_testpypi and args.use_localpypiartifact is not None: raise Exception("Cannot specify both --use-testpypi and --use-localpypiartifact.") run_pip_installation_tests(root_dir, args.version, args.use_testpypi, args.use_localpypiartifact, args.maven_repo) ================================================ FILE: run-tests.py ================================================ #!/usr/bin/env python3 # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os import subprocess import shlex from os import path import argparse # Define groups of subprojects that can be tested separately from other groups. # As of now, we have only defined project groups in the SBT build, so these must match # the group names defined in build.sbt. valid_project_groups = ["spark", "iceberg", "kernel", "spark-python"] def get_args(): parser = argparse.ArgumentParser() parser.add_argument( "--group", required=False, default=None, choices=valid_project_groups, help="Run tests on a group of SBT projects" ) parser.add_argument( "--coverage", required=False, default=False, action="store_true", help="Enables test coverage and generates an aggregate report for all subprojects") parser.add_argument( "--shard", required=False, default=None, help="some shard") parser.add_argument( "--spark-version", required=False, default=None, help="Spark version to use (passed as -DsparkVersion to SBT)") return parser.parse_args() def run_sbt_tests(root_dir, test_group, coverage, scala_version=None, shard=None, spark_version=None): print("##### Running SBT tests #####") sbt_path = path.join(root_dir, path.join("build", "sbt")) cmd = [sbt_path] # Pass Spark version as system property to SBT (must come before commands) if spark_version: cmd.append(f"-DsparkVersion={spark_version}") cmd.append("clean") test_cmd = "test" if shard: os.environ["SHARD_ID"] = str(shard) if test_group: # if test group is specified, then run tests only on that test group test_cmd = "{}Group/test".format(test_group) if coverage: cmd += ["coverage"] if scala_version is None: # when no scala version is specified, run test with all scala versions cmd += ["+ %s" % test_cmd] # build/sbt ... "+ project/test" ... else: # when no scala version is specified, run test with only the specified scala version cmd += ["++ %s" % scala_version, test_cmd] # build/sbt ... "++ 2.13.16" "project/test" ... if coverage: cmd += ["coverageAggregate", "coverageOff"] cmd += ["-v"] # show java options used # https://docs.oracle.com/javase/7/docs/technotes/guides/vm/G1.html # a GC that is optimized for larger multiprocessor machines with large memory cmd += ["-J-XX:+UseG1GC"] # 6x the default heap size (set in delta/built.sbt) cmd += ["-J-Xmx6G"] run_cmd(cmd, stream_output=True) def run_python_tests(root_dir): print("##### Running Python tests #####") python_test_script = path.join(root_dir, path.join("python", "run-tests.py")) print("Calling script %s", python_test_script) run_cmd(["python3", python_test_script], env={'DELTA_TESTING': '1'}, stream_output=True) def run_cmd(cmd, throw_on_error=True, env=None, stream_output=False, **kwargs): if isinstance(cmd, str): old_cmd = cmd cmd = shlex.split(cmd) cmd_env = os.environ.copy() if env: cmd_env.update(env) print("Running command: " + str(cmd)) if stream_output: child = subprocess.Popen(cmd, env=cmd_env, **kwargs) exit_code = child.wait() if throw_on_error and exit_code != 0: raise Exception("Non-zero exitcode: %s" % (exit_code)) return exit_code else: child = subprocess.Popen( cmd, env=cmd_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) (stdout, stderr) = child.communicate() exit_code = child.wait() if not isinstance(stdout, str): # Python 3 produces bytes which needs to be converted to str stdout = stdout.decode("utf-8") stderr = stderr.decode("utf-8") if throw_on_error and exit_code != 0: raise Exception( "Non-zero exitcode: %s\n\nSTDOUT:\n%s\n\nSTDERR:%s" % (exit_code, stdout, stderr)) return (exit_code, stdout, stderr) def pull_or_build_docker_image(root_dir): """ This method prepare the docker image for running tests. It uses a hash of the Dockerfile to generate the image tag/name so that we reuse images until the Dockerfile has changed. Then it tries to prepare that image by either pulling from a Docker registry (if configured with environment variable DOCKER_REGISTRY) or by building it from scratch using the Dockerfile. If pulling from registry fails, then it will fallback to building it from scratch, but it will also attempt to push to the registry to avoid image builds in the future. """ dockerfile_path = os.path.join(root_dir, "Dockerfile") _, out, _ = run_cmd("md5sum %s" % dockerfile_path) dockerfile_hash = out.strip().split(" ")[0].strip() print("Dockerfile hash: %s" % dockerfile_hash) test_env_image_tag = "delta_test_env:%s" % dockerfile_hash print("Test env image: %s" % test_env_image_tag) docker_registry = os.getenv("DOCKER_REGISTRY") print("Docker registry set as " + str(docker_registry)) def build_image(): print("---\nBuilding image %s ..." % test_env_image_tag) run_cmd("docker build --tag=%s %s" % (test_env_image_tag, root_dir)) print("Built image %s" % test_env_image_tag) def pull_image(registry_image_tag): try: print("---\nPulling image %s ..." % registry_image_tag) run_cmd("docker pull %s" % registry_image_tag) run_cmd("docker tag %s %s" % (registry_image_tag, test_env_image_tag)) print("Pulling image %s succeeded" % registry_image_tag) return True except Exception as e: print("Pulling image %s failed: %s" % (registry_image_tag, repr(e))) return False def push_image(registry_image_tag): try: print("---\nPushing image %s ..." % registry_image_tag) run_cmd("docker tag %s %s" % (test_env_image_tag, registry_image_tag)) run_cmd("docker push %s" % registry_image_tag) print("Pushing image %s succeeded" % registry_image_tag) return True except Exception as e: print("Pushing image %s failed: %s" % (registry_image_tag, repr(e))) return False if docker_registry is not None: print("Attempting to use the docker registry") test_env_image_tag_with_registry = docker_registry + "/delta/" + test_env_image_tag success = pull_image(test_env_image_tag_with_registry) if not success: build_image() push_image(test_env_image_tag_with_registry) else: build_image() return test_env_image_tag def run_tests_in_docker(image_tag, test_group): """ Run the necessary tests in a docker container made from the given image. It starts the container with the delta repo mounted in it, and then executes this script. """ # Note: Pass only relevant env that the script needs to run in the docker container. # Do not pass docker related env variable as we want this script to run natively in # the container and not attempt to recursively another docker container. envs = "-e JENKINS_URL -e SBT_1_5_5_MIRROR_JAR_URL " scala_version = os.getenv("SCALA_VERSION") if scala_version is not None: envs = envs + "-e SCALA_VERSION=%s " % scala_version test_parallelism = os.getenv("TEST_PARALLELISM_COUNT") if test_parallelism is not None: envs = envs + "-e TEST_PARALLELISM_COUNT=%s " % test_parallelism disable_unidoc = os.getenv("DISABLE_UNIDOC") if disable_unidoc is not None: envs = envs + "-e DISABLE_UNIDOC=%s " % disable_unidoc cwd = os.getcwd() test_script = os.path.basename(__file__) test_script_args = "" if test_group: test_script_args += " --group %s" % test_group test_run_cmd = "docker run --rm -v %s:%s -w %s %s %s ./%s %s" % ( cwd, cwd, cwd, envs, image_tag, test_script, test_script_args ) run_cmd(test_run_cmd, stream_output=True) def print_configuration(args: argparse.Namespace) -> None: print("=" * 60) print("DELTA LAKE TEST RUNNER CONFIGURATION") print("=" * 60) # Print parsed arguments print("-" * 25) print("Command Line Arguments:") print("-" * 25) args_dict = vars(args) for key, value in args_dict.items(): if value is not None: print(f" {key:<12}: {value}") else: print(f" {key:<12}: ") # Print relevant environment variables print("-" * 25) print("Environment Variables:") print("-" * 22) env_vars = [ "USE_DOCKER", "SCALA_VERSION", "DISABLE_UNIDOC", "DOCKER_REGISTRY", "NUM_SHARDS", "SHARD_ID", "TEST_PARALLELISM_COUNT", "JENKINS_URL", "SBT_1_5_5_MIRROR_JAR_URL", "DELTA_TESTING", "SBT_OPTS" ] for var in env_vars: value = os.getenv(var) if value is not None: print(f" {var:<22}: {value}") else: print(f" {var:<22}: ") print("=" * 60) if __name__ == "__main__": root_dir = os.path.dirname(os.path.abspath(__file__)) args = get_args() print_configuration(args) if os.getenv("USE_DOCKER") is not None: test_env_image_tag = pull_or_build_docker_image(root_dir) run_tests_in_docker(test_env_image_tag, args.group) elif args.group == "spark-python": run_python_tests(root_dir) else: scala_version = os.getenv("SCALA_VERSION") spark_version = args.spark_version or os.getenv("SPARK_VERSION") run_sbt_tests(root_dir, args.group, args.coverage, scala_version, args.shard, spark_version) ================================================ FILE: scalastyle-config.xml ================================================ Scalastyle standard configuration true true ARROW, EQUALS, ELSE, TRY, CATCH, FINALLY, LARROW, RARROW ARROW, EQUALS, COMMA, COLON, IF, ELSE, DO, WHILE, FOR, MATCH, TRY, CATCH, FINALLY, LARROW, RARROW ^FunSuite[A-Za-z]*$ Tests must extend org.apache.spark.SparkFunSuite instead. ^println$ spark(.sqlContext)?.sparkContext.hadoopConfiguration sessionState.newHadoopConf @VisibleForTesting Runtime\.getRuntime\.addShutdownHook mutable\.SynchronizedBuffer Class\.forName Await\.result Await\.ready (\.toUpperCase|\.toLowerCase)(?!(\(|\(Locale.ROOT\))) typed[lL]it spark(Session)?.implicits._ throw new \w+Error\( count\(" JavaConversions Instead of importing implicits in scala.collection.JavaConversions._, import scala.collection.JavaConverters._ and use .asScala / .asJava methods org\.apache\.commons\.lang\. Use Commons Lang 3 classes (package org.apache.commons.lang3.*) instead of Commons Lang 2 (package org.apache.commons.lang.*) extractOpt Use jsonOption(x).map(.extract[T]) instead of .extractOpt[T], as the latter is slower. COMMA \)\{ (?m)^(\s*)/[*][*].*$(\r|)\n^\1 [*] Use Javadoc style indentation for multiline comments case[^\n>]*=>\s*\{ Omit braces in case clauses. 800> 30 10 50 -1,0,1,2,3 ================================================ FILE: setup.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import os import sys from setuptools import setup from setuptools.command.install import install # delta.io version def get_version_from_sbt(): with open("version.sbt") as fp: version = fp.read().strip() return version.split('"')[1] VERSION = get_version_from_sbt() class VerifyVersionCommand(install): """Custom command to verify that the git tag matches our version""" description = 'verify that the git tag matches our version' def run(self): tag = os.getenv('CIRCLE_TAG') if tag != VERSION: info = "Git tag: {0} does not match the version of this app: {1}".format( tag, VERSION ) sys.exit(info) with open("python/README.md", "r", encoding="utf-8") as fh: long_description = fh.read() install_requires_arg = ['pyspark>=4.0.1', 'importlib_metadata>=1.0.0'] python_requires_arg = '>=3.10' setup( name="delta_spark", version=VERSION, description="Python APIs for using Delta Lake with Apache Spark", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/delta-io/delta/", project_urls={ 'Source': 'https://github.com/delta-io/delta', 'Documentation': 'https://docs.delta.io/latest/index.html', 'Issues': 'https://github.com/delta-io/delta/issues' }, author="The Delta Lake Project Authors", author_email="delta-users@googlegroups.com", license="Apache-2.0", classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Topic :: Software Development :: Libraries :: Python Modules", "Programming Language :: Python :: 3", "Typing :: Typed", ], keywords='delta.io', package_dir={'': 'python'}, packages=['delta', 'delta.connect', 'delta.connect.proto', 'delta.exceptions'], package_data={ 'delta': ['py.typed'], }, install_requires=install_requires_arg, python_requires=python_requires_arg, cmdclass={ 'verify': VerifyVersionCommand, } ) ================================================ FILE: sharing/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister ================================================ io.delta.sharing.spark.DeltaSharingDataSource ================================================ FILE: sharing/src/main/scala/io/delta/sharing/spark/DeltaFormatSharingLimitPushDown.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark import io.delta.sharing.client.util.ConfUtils import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.IntegerLiteral import org.apache.spark.sql.catalyst.plans.logical.{LocalLimit, LogicalPlan} import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelationWithTable} // A spark rule that applies limit pushdown to DeltaSharingFileIndex, when the config is enabled. // To allow only fetching needed files from delta sharing server. object DeltaFormatSharingLimitPushDown extends Rule[LogicalPlan] { def setup(spark: SparkSession): Unit = synchronized { if (!spark.experimental.extraOptimizations.contains(DeltaFormatSharingLimitPushDown)) { spark.experimental.extraOptimizations ++= Seq(DeltaFormatSharingLimitPushDown) } } def apply(p: LogicalPlan): LogicalPlan = { p transform { case localLimit @ LocalLimit( literalExpr @ IntegerLiteral(limit), l @ LogicalRelationWithTable( r @ HadoopFsRelation(remoteIndex: DeltaSharingFileIndex, _, _, _, _, _), _ ) ) if (ConfUtils.limitPushdownEnabled(p.conf) && remoteIndex.limitHint.isEmpty) => val spark = SparkSession.active val newRel = r.copy(location = remoteIndex.copy(limitHint = Some(limit)))(spark) LocalLimit(literalExpr, l.copy(relation = newRel)) } } } ================================================ FILE: sharing/src/main/scala/io/delta/sharing/spark/DeltaFormatSharingSource.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark import java.lang.ref.WeakReference import java.util.UUID import java.util.concurrent.TimeUnit import org.apache.spark.sql.delta.{ DeltaErrors, DeltaLog, DeltaOptions, SnapshotDescriptor } import org.apache.spark.sql.delta.actions.{Metadata, Protocol} import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.{ DeltaDataSource, DeltaSource, DeltaSourceOffset } import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.JsonUtils import io.delta.sharing.client.DeltaSharingClient import io.delta.sharing.client.util.ConfUtils import io.delta.sharing.client.model.{Table => DeltaSharingTable} import org.apache.spark.delta.sharing.CachedTableManager import org.apache.spark.sql.{DataFrame, SparkSession} import org.apache.spark.sql.connector.read.streaming import org.apache.spark.sql.connector.read.streaming.{ReadLimit, SupportsAdmissionControl} import org.apache.spark.sql.execution.streaming.{Offset, Source} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType /** * A streaming source for a Delta Sharing table. * * This class wraps a DeltaSource to read data out of locally constructed delta log. * When a new stream is started, delta sharing starts by fetching delta log from the server side, * constructing a local delta log, and call delta source apis to compute offset or read data. * * TODO: Support CDC Streaming, SupportsTriggerAvailableNow and SupportsConcurrentExecution. */ case class DeltaFormatSharingSource( spark: SparkSession, client: DeltaSharingClient, table: DeltaSharingTable, options: DeltaSharingOptions, parameters: Map[String, String], sqlConf: SQLConf, metadataPath: String) extends Source with SupportsAdmissionControl with DeltaLogging { private val sourceId = Some(UUID.randomUUID().toString().split('-').head) private var tableId: String = "unset_table_id" private val tablePath = options.options.getOrElse( "path", throw DeltaSharingErrors.pathNotSpecifiedException ) // A unique string composed of a formatted timestamp and an uuid. // Used as a suffix for the table name and its delta log path of a delta sharing table in a // streaming job, to avoid overwriting the delta log from multiple references of the same delta // sharing table in one streaming job. private val timestampWithUUID = DeltaSharingUtils.getFormattedTimestampWithUUID() private val customTablePathWithUUIDSuffix = DeltaSharingUtils.getTablePathWithIdSuffix( client.getProfileProvider.getCustomTablePath(tablePath), timestampWithUUID ) private val deltaLogPath = s"${DeltaSharingLogFileSystem.encode(customTablePathWithUUIDSuffix).toString}/_delta_log" // The latest metadata of the shared table, fetched at the initialization time of the // DeltaFormatSharingSource, used to initialize the wrapped DeltaSource. private lazy val deltaSharingTableMetadata = DeltaSharingUtils.getDeltaSharingTableMetadata(client, table) private lazy val deltaSource = initDeltaSource() private def initDeltaSource(): DeltaSource = { val (localDeltaLog, snapshotDescriptor) = DeltaSharingUtils.getDeltaLogAndSnapshotDescriptor( spark, deltaSharingTableMetadata, customTablePathWithUUIDSuffix ) // Delta sharing delta log doesn't have catalog table and `localDeltaLog` is not binded to // catalog table. val schemaTrackingLogOpt = DeltaDataSource.getMetadataTrackingLogForDeltaSource( spark, snapshotDescriptor, catalogTableOpt = None, parameters, // Pass in the metadata path opt so we can use it for validation sourceMetadataPathOpt = Some(metadataPath) ) val readSchema = schemaTrackingLogOpt .flatMap(_.getCurrentTrackedMetadata.map(_.dataSchema)) .getOrElse(snapshotDescriptor.schema) if (readSchema.isEmpty) { throw DeltaErrors.schemaNotSetException } // Catalog table represents the table's catalog metadata and it's managed by Unity Catalog. // Delta sharing delta log doesn't have it and `localDeltaLog` is not bound to it. DeltaSource( spark = spark, deltaLog = localDeltaLog, catalogTableOpt = None, options = new DeltaOptions(parameters, sqlConf), snapshotAtSourceInit = snapshotDescriptor, metadataPath = metadataPath, metadataTrackingLog = schemaTrackingLogOpt ) } // schema of the streaming source, based on the latest metadata of the shared table. override val schema: StructType = { val schemaWithoutCDC = deltaSharingTableMetadata.metadata.schema tableId = deltaSharingTableMetadata.metadata.deltaMetadata.id if (options.readChangeFeed) { CDCReader.cdcReadSchema(schemaWithoutCDC) } else { schemaWithoutCDC } } // Latest endOffset of the getBatch call, used to compute startingOffset which will then be used // to compare with the the latest table version on server to decide whether to fetch new data. private var latestProcessedEndOffsetOption: Option[DeltaSourceOffset] = None // Latest table version for the data fetched from the delta sharing server, and stored in the // local delta log. Used to check whether all fetched files are processed by the DeltaSource. private var latestTableVersionInLocalDeltaLogOpt: Option[Long] = None // This is needed because DeltaSource is not advancing the offset to the next version // automatically when scanning through a snapshot, so DeltaFormatSharingSource needs to count the // number of files in the min version and advance the offset to the next version when the offset // is at the last index of the version. private var numFileActionsInStartingSnapshotOpt: Option[Int] = None // Latest timestamp for getTableVersion rpc from the server, used to compare with the current // timestamp, to ensure the gap QUERY_TABLE_VERSION_INTERVAL_MILLIS between two rpcs, to avoid // a high traffic load to the server. private var lastTimestampForGetVersionFromServer: Long = -1 // The minimum gap between two getTableVersion rpcs, to avoid a high traffic load to the server. private val QUERY_TABLE_VERSION_INTERVAL_MILLIS = { val intervalSeconds = ConfUtils.MINIMUM_TABLE_VERSION_INTERVAL_SECONDS.max( ConfUtils.streamingQueryTableVersionIntervalSeconds(spark.sessionState.conf) ) logInfo(s"Configured queryTableVersionIntervalSeconds:${intervalSeconds}," + getTableInfoForLogging) if (intervalSeconds < ConfUtils.MINIMUM_TABLE_VERSION_INTERVAL_SECONDS) { throw new IllegalArgumentException(s"QUERY_TABLE_VERSION_INTERVAL_MILLIS($intervalSeconds) " + s"must not be less than ${ConfUtils.MINIMUM_TABLE_VERSION_INTERVAL_SECONDS} seconds," + getTableInfoForLogging) } TimeUnit.SECONDS.toMillis(intervalSeconds) } // Maximum number of versions of getFiles() rpc when fetching files from the server. Used to // reduce the number of files returned to avoid timeout of the rpc on the server. private val maxVersionsPerRpc: Int = options.maxVersionsPerRpc.getOrElse( DeltaSharingOptions.MAX_VERSIONS_PER_RPC_DEFAULT ) private lazy val getTableInfoForLogging: String = s" for table(id:$tableId, name:${table.toString}, source:$sourceId)" private def getQueryIdForLogging: String = { s", with queryId(${client.getQueryId})" } // A variable to store the latest table version on server, returned from the getTableVersion rpc. // Used to store the latest table version for getOrUpdateLatestTableVersion when not getting // updates from the server. // For all other callers, please use getOrUpdateLatestTableVersion instead of this variable. private var latestTableVersionOnServer: Long = -1 /** * Check the latest table version from the delta sharing server through the client.getTableVersion * RPC. Adding a minimum interval of QUERY_TABLE_VERSION_INTERVAL_MILLIS between two consecutive * rpcs to avoid traffic jam on the delta sharing server. * * @return the latest table version on the server. */ private def getOrUpdateLatestTableVersion: Long = { val currentTimeMillis = System.currentTimeMillis() if ((currentTimeMillis - lastTimestampForGetVersionFromServer) >= QUERY_TABLE_VERSION_INTERVAL_MILLIS) { val serverVersion = client.getTableVersion(table) if (serverVersion < 0) { throw new IllegalStateException(s"Delta Sharing Server returning negative table version:" + s"$serverVersion," + getTableInfoForLogging) } else if (serverVersion < latestTableVersionOnServer) { logWarning( s"Delta Sharing Server returning smaller table version: $serverVersion < " + s"$latestTableVersionOnServer," + getTableInfoForLogging ) } logInfo(s"Got table version $serverVersion from Delta Sharing Server,$getTableInfoForLogging") latestTableVersionOnServer = serverVersion lastTimestampForGetVersionFromServer = currentTimeMillis } latestTableVersionOnServer } /** * NOTE: need to match with the logic in DeltaSource.extractStartingState(). * * Get the starting offset used to send rpc to delta sharing server, to fetch needed files. * Use input startOffset when it's defined, otherwise use user defined starting version, otherwise * use input endOffset if it's defined, the least option is the latest table version returned from * the delta sharing server (which is usually used when a streaming query starts from scratch). * * @param startOffsetOption optional start offset, return it if defined. It's empty when the * streaming query starts from scratch. It's set for following calls. * @param endOffsetOption optional end offset. It's set when the function is called from * getBatch and is empty when called from latestOffset. * @return The starting offset. */ private def getStartingOffset( startOffsetOption: Option[DeltaSourceOffset], endOffsetOption: Option[DeltaSourceOffset]): DeltaSourceOffset = { if (startOffsetOption.isEmpty) { val (version, isInitialSnapshot) = getStartingVersion match { case Some(v) => (v, false) case None => if (endOffsetOption.isDefined) { if (endOffsetOption.get.isInitialSnapshot) { (endOffsetOption.get.reservoirVersion, true) } else { assert( endOffsetOption.get.reservoirVersion > 0, s"invalid reservoirVersion in endOffset: ${endOffsetOption.get}" ) // Load from snapshot `endOffset.reservoirVersion - 1L` so that `index` in `endOffset` // is still valid. // It's OK to use the previous version as the updated initial snapshot, even if the // initial snapshot might have been different from the last time when this starting // offset was computed. (endOffsetOption.get.reservoirVersion - 1L, true) } } else { (getOrUpdateLatestTableVersion, true) } } // Constructed the same way as DeltaSource.buildOffsetFromIndexedFile DeltaSourceOffset( reservoirId = tableId, reservoirVersion = version, index = DeltaSourceOffset.BASE_INDEX, isInitialSnapshot = isInitialSnapshot ) } else { startOffsetOption.get } } /** * Converts an offset from the checkpoint to DeltaSourceOffset, and returns whether it was * converted from legacy format (DeltaSharingSourceOffset or legacy JSON tableId/tableVersion). * @return (DeltaSourceOffset, fromLegacy) * Visible for testing (private[spark]). */ private[spark] def forceToDeltaSourceOffset( offset: streaming.Offset): (DeltaSourceOffset, Boolean) = { if (offset == null) { throw new IllegalArgumentException("offset cannot be null") } offset match { case o: DeltaSourceOffset => (o, false) case o: DeltaSharingSourceOffset => (convertDeltaSharingSourceOffsetToDeltaSourceOffset(o), true) case _ => // For JSON (SerializedOffset): parse as DeltaSourceOffset first, // if that throws or yields empty reservoirId, parse as legacy DeltaSharingSourceOffset. try { val deltaOffset = deltaSource.toDeltaSourceOffset(offset) val reservoirIdEmpty = deltaOffset.reservoirId == null || deltaOffset.reservoirId.isEmpty if (reservoirIdEmpty) { // Throw to let the catcher handle the exception. throw new IllegalArgumentException(s"Invalid offset format: $offset") } else { logInfo("Offset JSON parsed as Delta format") (deltaOffset, false) } } catch { // Parsing legacy Offset JSON using DeltaSourceOffset // yields a null reservoirId, causing an exception // since toDeltaSourceOffset expects it to match tableId. case e: Exception => val autoResolve = sqlConf.getConf( DeltaSQLConf.DELTA_SHARING_STREAMING_AUTO_RESOLVE_RESPONSE_FORMAT) if (!autoResolve) { throw e } logInfo(s"Offset JSON not valid Delta format, parsing as legacy: ${e.getMessage}") (toDeltaSourceOffsetFromLegacyJson(offset), true) } } } /** Parse the given offset as DeltaSharingSourceOffset (tableId/tableVersion) * and convert to DeltaSourceOffset. */ private def toDeltaSourceOffsetFromLegacyJson(offset: streaming.Offset): DeltaSourceOffset = { val legacy = DeltaSharingSourceOffset(tableId, offset) convertDeltaSharingSourceOffsetToDeltaSourceOffset(legacy) } private def convertDeltaSharingSourceOffsetToDeltaSourceOffset( o: DeltaSharingSourceOffset): DeltaSourceOffset = { // Legacy DeltaSharingSourceOffset hardcodes -1 as the // version boundary index, but DeltaSourceOffset uses // BASE_INDEX (which varies across versions) and rejects // -1. Convert the index to BASE_INDEX for compatibility. val index = if (o.index == -1L) DeltaSourceOffset.BASE_INDEX else o.index logInfo(s"Converted DeltaSharingSourceOffset to DeltaSourceOffset: reservoirId=${o.tableId}, " + s"reservoirVersion=${o.tableVersion}, index=$index, isInitialSnapshot=${o.isStartingVersion}") DeltaSourceOffset( reservoirId = o.tableId, reservoirVersion = o.tableVersion, index = index, isInitialSnapshot = o.isStartingVersion ) } /** * The ending version used in rpc is restricted by both the latest table version and * maxVersionsPerRpc, to avoid loading too many files from the server to cause a timeout. * @param startingOffset The start offset used in the rpc. * @param latestTableVersion The latest table version at the server. * @return the ending version used in the rpc. */ private def getEndingVersionForRpc( startingOffset: DeltaSourceOffset, latestTableVersion: Long): Long = { if (startingOffset.isInitialSnapshot) { // ending version is the same as starting version for snapshot query. return startingOffset.reservoirVersion } // using "startVersion + maxVersionsPerRpc - 1" because the endingVersion is inclusive. val endingVersionForQuery = latestTableVersion.min( startingOffset.reservoirVersion + maxVersionsPerRpc - 1 ) if (endingVersionForQuery < latestTableVersion) { logInfo( s"Reducing ending version for delta sharing rpc from latestTableVersion(" + s"$latestTableVersion) to endingVersionForQuery($endingVersionForQuery), " + s"startVersion:${startingOffset.reservoirVersion}, maxVersionsPerRpc:$maxVersionsPerRpc, " + getTableInfoForLogging ) } endingVersionForQuery } override def getDefaultReadLimit: ReadLimit = { deltaSource.getDefaultReadLimit } override def latestOffset(startOffset: streaming.Offset, limit: ReadLimit): streaming.Offset = { logInfo(s"latestOffset with startOffset($startOffset), limit($limit)") // startOffset is null for initialSnapshot. val (startDeltaSourceOffsetOpt, wasConvertedFromLegacy) = Option(startOffset).map(forceToDeltaSourceOffset) match { case Some((offset, fromLegacy)) => (Some(offset), fromLegacy) case None => (None, false) } // When DVs are enabled on a shared table with an existing // LegacySource streaming query, the query fails. On restart // this Source is freshly instantiated so // latestProcessedEndOffsetOption is None. We must use the // legacy offset as the starting point to fetch files. val deltaSourceOffset = if (latestProcessedEndOffsetOption.isEmpty && startDeltaSourceOffsetOpt.nonEmpty && wasConvertedFromLegacy) { startDeltaSourceOffsetOpt.get } else { getStartingOffset(latestProcessedEndOffsetOption, None) } if (deltaSourceOffset.reservoirVersion < 0) { return null } maybeGetLatestFileChangesFromServer(deltaSourceOffset) maybeMoveToNextVersion(deltaSource.latestOffset(startDeltaSourceOffsetOpt.orNull, limit)) } // Advance the DeltaSourceOffset to the next version when the offset is at the last index of the // version. // This is because DeltaSource is not advancing the offset automatically when processing a // snapshot (isStartingVersion = true), and advancing the offset is necessary for delta sharing // streaming to fetch new files from the delta sharing server. private def maybeMoveToNextVersion( latestOffsetFromDeltaSource: streaming.Offset): DeltaSourceOffset = { val deltaLatestOffset = deltaSource.toDeltaSourceOffset(latestOffsetFromDeltaSource) if (deltaLatestOffset.isInitialSnapshot && (numFileActionsInStartingSnapshotOpt.exists(_ == deltaLatestOffset.index + 1))) { DeltaSourceOffset( reservoirId = deltaLatestOffset.reservoirId, reservoirVersion = deltaLatestOffset.reservoirVersion + 1, index = DeltaSourceOffset.BASE_INDEX, isInitialSnapshot = false ) } else { deltaLatestOffset } } /** * Whether need to fetch new files from the delta sharing server. * @param startingOffset the startingOffset of the next batch asked by spark streaming engine. * @param latestTableVersion the latest table version on the delta sharing server. * @return whether need to fetch new files from the delta sharing server, this is needed when all * files are processed in the local delta log, and there are new files on the delta * sharing server. * And we avoid fetching new files when files in the delta log are not fully processed. */ private def needNewFilesFromServer( startingOffset: DeltaSourceOffset, latestTableVersion: Long): Boolean = { if (latestTableVersionInLocalDeltaLogOpt.isEmpty) { return true } val allLocalFilesProcessed = latestTableVersionInLocalDeltaLogOpt.exists( _ < startingOffset.reservoirVersion ) val newChangesOnServer = latestTableVersionInLocalDeltaLogOpt.exists(_ < latestTableVersion) allLocalFilesProcessed && newChangesOnServer } /** * Check whether we need to fetch new files from the server and calls getTableFileChanges if true. * * @param startingOffset the starting offset used to fetch files, the 3 parameters will be useful: * - reservoirVersion: initially would be the startingVersion or the latest * table version. * - index: index of a file within the same version. * - isInitialSnapshot: If true, will load fromVersion as a table snapshot( * including files from previous versions). If false, will only load files * since fromVersion. * 2 usages: 1) used to compare with latestTableVersionInLocalDeltaLogOpt to * check whether new files are needed. 2) used for getTableFileChanges, * check more details in the function header. */ private def maybeGetLatestFileChangesFromServer(startingOffset: DeltaSourceOffset): Unit = { // Use a local variable to avoid a difference in the two usages below. val latestTableVersion = getOrUpdateLatestTableVersion if (needNewFilesFromServer(startingOffset, latestTableVersion)) { val endingVersionForQuery = getEndingVersionForRpc(startingOffset, latestTableVersion) if (startingOffset.isInitialSnapshot || !options.readChangeFeed) { getTableFileChanges(startingOffset, endingVersionForQuery) } else { throw new UnsupportedOperationException("CDF Streaming is not supported yet.") } } } /** * Fetch the table changes from delta sharing server starting from (version, index) of the * startingOffset, and store them in locally constructed delta log. * * @param startingOffset Includes a reservoirVersion, an index of a file within the same version, * and an isInitialSnapshot. * If isInitialSnapshot is true, will load startingOffset.reservoirVersion * as a table snapshot (including files from previous versions). If false, * it will only load files since startingOffset.reservoirVersion. * @param endingVersionForQuery The ending version used for the query, always smaller than * the latest table version on server. */ private def getTableFileChanges( startingOffset: DeltaSourceOffset, endingVersionForQuery: Long): Unit = { logInfo( s"Fetching files with table version(${startingOffset.reservoirVersion}), " + s"index(${startingOffset.index}), isInitialSnapshot(${startingOffset.isInitialSnapshot})," + s" endingVersionForQuery($endingVersionForQuery), server version" + s"($latestTableVersionOnServer)," + getTableInfoForLogging ) val (tableFiles, refreshFunc) = if (startingOffset.isInitialSnapshot) { // If isInitialSnapshot is true, it means to fetch the snapshot at the fromVersion, which may // include table changes from previous versions. val tableFiles = client.getFiles( table = table, predicates = Nil, limit = None, versionAsOf = Some(startingOffset.reservoirVersion), timestampAsOf = None, jsonPredicateHints = None, refreshToken = None, fileIdHash = None ) val refreshFunc = DeltaSharingUtils.getRefresherForGetFiles( client = client, table = table, predicates = Nil, limit = None, versionAsOf = Some(startingOffset.reservoirVersion), timestampAsOf = None, jsonPredicateHints = None, useRefreshToken = false ) logInfo( s"Fetched ${tableFiles.lines.size} lines for table version ${tableFiles.version} from" + " delta sharing server." + getTableInfoForLogging + getQueryIdForLogging ) (tableFiles, refreshFunc) } else { // If isStartingVersion is false, it means to fetch files for data changes since fromVersion, // not including files from previous versions. val tableFiles = client.getFiles( table = table, startingVersion = startingOffset.reservoirVersion, endingVersion = Some(endingVersionForQuery), fileIdHash = None ) val refreshFunc = DeltaSharingUtils.getRefresherForGetFilesWithStartingVersion( client = client, table = table, startingVersion = startingOffset.reservoirVersion, endingVersion = Some(endingVersionForQuery) ) logInfo( s"Fetched ${tableFiles.lines.size} lines from startingVersion " + s"${startingOffset.reservoirVersion} to enedingVersion ${endingVersionForQuery} from " + "delta sharing server," + getTableInfoForLogging + getQueryIdForLogging ) (tableFiles, refreshFunc) } val deltaLogMetadata = DeltaSharingLogFileSystem.constructLocalDeltaLogAcrossVersions( lines = tableFiles.lines, customTablePath = customTablePathWithUUIDSuffix, startingVersionOpt = Some(startingOffset.reservoirVersion), endingVersionOpt = Some(endingVersionForQuery) ) assert( deltaLogMetadata.maxVersion > 0, s"Invalid table version in delta sharing response: ${tableFiles.lines}." ) latestTableVersionInLocalDeltaLogOpt = Some(deltaLogMetadata.maxVersion) logInfo(s"Setting latestTableVersionInLocalDeltaLogOpt to ${deltaLogMetadata.maxVersion}" + getTableInfoForLogging) assert( deltaLogMetadata.numFileActionsInMinVersionOpt.isDefined, "numFileActionsInMinVersionOpt missing after constructed delta log." ) if (startingOffset.isInitialSnapshot) { numFileActionsInStartingSnapshotOpt = deltaLogMetadata.numFileActionsInMinVersionOpt } CachedTableManager.INSTANCE.register( tablePath = DeltaSharingUtils.getTablePathWithIdSuffix(tablePath, timestampWithUUID), idToUrl = deltaLogMetadata.idToUrl, refs = Seq(new WeakReference(this)), profileProvider = client.getProfileProvider, refresher = refreshFunc, expirationTimestamp = if (CachedTableManager.INSTANCE .isValidUrlExpirationTime(deltaLogMetadata.minUrlExpirationTimestamp)) { deltaLogMetadata.minUrlExpirationTimestamp.get } else { System.currentTimeMillis() + CachedTableManager.INSTANCE.preSignedUrlExpirationMs }, refreshToken = tableFiles.refreshToken ) } override def getBatch(startOffsetOption: Option[Offset], end: Offset): DataFrame = { logInfo(s"getBatch with startOffsetOption($startOffsetOption) and end($end)," + getTableInfoForLogging) // When DVs are enabled on a shared table with an existing // LegacySource streaming query still reading an initial // snapshot, startOffsetOption is None and endOffset // specifies the starting version. On restart, this Source // is instantiated, so we convert legacy offsets to // DeltaSourceOffset. val endOffset = forceToDeltaSourceOffset(end)._1 // When the query is past the initial snapshot, // startOffsetOption is defined and contains the starting // version. Convert from legacy offset if needed. val startDeltaOffsetOption = startOffsetOption.map(o => forceToDeltaSourceOffset(o)._1) val startingOffset = getStartingOffset(startDeltaOffsetOption, Some(endOffset)) // Files should already be fetched in latestOffset; this // is a safeguard in case files for startOffset are not // present. Whether startOffset was converted from legacy // does not matter here. maybeGetLatestFileChangesFromServer(startingOffset = startingOffset) // Reset latestProcessedEndOffsetOption only when endOffset is larger. // Because with microbatch pipelining, we may get getBatch requests out of order. if (latestProcessedEndOffsetOption.isEmpty || endOffset.reservoirVersion > latestProcessedEndOffsetOption.get.reservoirVersion || (endOffset.reservoirVersion == latestProcessedEndOffsetOption.get.reservoirVersion && endOffset.index > latestProcessedEndOffsetOption.get.index)) { latestProcessedEndOffsetOption = Some(endOffset) logInfo(s"Setting latestProcessedEndOffsetOption to $endOffset," + getTableInfoForLogging) } deltaSource.getBatch(startDeltaOffsetOption, endOffset) } override def getOffset: Option[Offset] = { throw new UnsupportedOperationException( "latestOffset(Offset, ReadLimit) should be called instead of this method." ) } /** * Extracts whether users provided the option to time travel a relation. If a query restarts from * a checkpoint and the checkpoint has recorded the offset, this method should never been called. */ private lazy val getStartingVersion: Option[Long] = { /** DeltaOption validates input and ensures that only one is provided. */ if (options.startingVersion.isDefined) { val v = options.startingVersion.get match { case StartingVersionLatest => getOrUpdateLatestTableVersion + 1 case StartingVersion(version) => version } Some(v) } else if (options.startingTimestamp.isDefined) { Some(client.getTableVersion(table, options.startingTimestamp)) } else { None } } override def stop(): Unit = { deltaSource.stop() DeltaSharingLogFileSystem.tryToCleanUpDeltaLog(deltaLogPath) } // Calls deltaSource.commit for checks related to column mapping. override def commit(end: Offset): Unit = { logInfo(s"Commit end offset: $end," + getTableInfoForLogging) val endOffset = forceToDeltaSourceOffset(end)._1 // If DeltaSource detects a metadata change at endOffset // version, deltaSource.commit throws an exception so the // stream restarts from the checkpoint with the new schema. deltaSource.commit(endOffset) // Clean up processed versions in block manager regardless // of whether endOffset is from legacy format. DeltaSharingLogFileSystem.tryToCleanUpPreviousBlocks( deltaLogPath, endOffset.reservoirVersion - 1 ) } override def toString(): String = s"DeltaFormatSharingSource[${table.toString}]" } ================================================ FILE: sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingCDFUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark import java.lang.ref.WeakReference import java.nio.charset.StandardCharsets.UTF_8 import org.apache.spark.sql.delta.catalog.DeltaTableV2 import com.google.common.hash.Hashing import io.delta.sharing.client.DeltaSharingClient import io.delta.sharing.client.model.{Table => DeltaSharingTable} import org.apache.hadoop.fs.Path import org.apache.spark.delta.sharing.CachedTableManager import org.apache.spark.internal.Logging import org.apache.spark.sql.SQLContext import org.apache.spark.sql.sources.BaseRelation object DeltaSharingCDFUtils extends Logging { private def getDuration(start: Long): Double = { (System.currentTimeMillis() - start) / 1000.0 } /** * Prepares the BaseRelation for cdf queries on a delta sharing table. Since there's no limit * pushdown or filter pushdown involved, it wiill firatly fetch all the files from the delta * sharing server, prepare the local delta log, and leverage DeltaTableV2 to produce the relation. */ private[sharing] def prepareCDFRelation( sqlContext: SQLContext, options: DeltaSharingOptions, table: DeltaSharingTable, client: DeltaSharingClient): BaseRelation = { val startTime = System.currentTimeMillis() // 1. Get all files with DeltaSharingClient. // includeHistoricalMetadata is always set to true, to get the metadata at the startingVersion // and also any metadata changes between [startingVersion, endingVersion], to put them in the // delta log. This is to allow delta library to check the metadata change and handle it // properly -- currently it throws error for column mapping changes. val deltaTableFiles = client.getCDFFiles( table, options.cdfOptions, includeHistoricalMetadata = true, fileIdHash = None ) logInfo( s"Fetched ${deltaTableFiles.lines.size} lines with cdf options ${options.cdfOptions} " + s"for table ${table} from delta sharing server, took ${getDuration(startTime)}s." ) val path = options.options.getOrElse("path", throw DeltaSharingErrors.pathNotSpecifiedException) // 2. Prepare local delta log val queryCustomTablePath = client.getProfileProvider.getCustomTablePath(path) val queryParamsHashId = DeltaSharingUtils.getQueryParamsHashId(options.cdfOptions) val tablePathWithHashIdSuffix = DeltaSharingUtils.getTablePathWithIdSuffix(queryCustomTablePath, queryParamsHashId) val deltaLogMetadata = DeltaSharingLogFileSystem.constructLocalDeltaLogAcrossVersions( lines = deltaTableFiles.lines, customTablePath = tablePathWithHashIdSuffix, startingVersionOpt = None, endingVersionOpt = None ) // 3. Register parquet file id to url mapping CachedTableManager.INSTANCE.register( // Using path instead of queryCustomTablePath because it will be customized within // CachedTableManager. tablePath = DeltaSharingUtils.getTablePathWithIdSuffix(path, queryParamsHashId), idToUrl = deltaLogMetadata.idToUrl, // A weak reference is needed by the CachedTableManager to decide whether the query is done // and it's ok to clean up the id to url mapping for this table. refs = Seq(new WeakReference(this)), profileProvider = client.getProfileProvider, refresher = DeltaSharingUtils.getRefresherForGetCDFFiles( client = client, table = table, cdfOptions = options.cdfOptions ), expirationTimestamp = if (CachedTableManager.INSTANCE .isValidUrlExpirationTime(deltaLogMetadata.minUrlExpirationTimestamp)) { deltaLogMetadata.minUrlExpirationTimestamp.get } else { System.currentTimeMillis() + CachedTableManager.INSTANCE.preSignedUrlExpirationMs }, refreshToken = None ) // 4. return Delta val localDeltaCdfOptions = Map( DeltaSharingOptions.CDF_START_VERSION -> deltaLogMetadata.minVersion.toString, DeltaSharingOptions.CDF_END_VERSION -> deltaLogMetadata.maxVersion.toString, DeltaSharingOptions.CDF_READ_OPTION -> "true" ) DeltaTableV2( spark = sqlContext.sparkSession, path = DeltaSharingLogFileSystem.encode(tablePathWithHashIdSuffix), options = localDeltaCdfOptions ).toBaseRelation } } ================================================ FILE: sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingDataSource.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.{ DeltaErrors, DeltaTableUtils => TahoeDeltaTableUtils } import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.{DeltaDataSource, DeltaSQLConf} import io.delta.sharing.client.{DeltaSharingClient, DeltaSharingRestClient, ParsedDeltaSharingTablePath} import io.delta.sharing.client.model.{DeltaTableMetadata, Table => DeltaSharingTable} import io.delta.sharing.client.util.{ConfUtils, JsonUtils} import org.apache.hadoop.fs.Path import org.apache.spark.SparkEnv import org.apache.spark.delta.sharing.PreSignedUrlCache import org.apache.spark.sql.{SparkSession, SQLContext} import org.apache.spark.sql.execution.datasources.HadoopFsRelation import org.apache.spark.sql.execution.streaming.Source import org.apache.spark.sql.sources.{ BaseRelation, DataSourceRegister, RelationProvider, StreamSourceProvider } import org.apache.spark.sql.types.StructType /** * A DataSource for Delta Sharing, used to support all types of queries on a delta sharing table: * batch, cdf, streaming, time travel, filters, etc. */ private[sharing] class DeltaSharingDataSource extends RelationProvider with StreamSourceProvider with DataSourceRegister with DeltaLogging { override def sourceSchema( sqlContext: SQLContext, schema: Option[StructType], providerName: String, parameters: Map[String, String]): (String, StructType) = { DeltaSharingDataSource.setupFileSystem(sqlContext) if (schema.nonEmpty && schema.get.nonEmpty) { throw DeltaErrors.specifySchemaAtReadTimeException } val options = new DeltaSharingOptions(parameters) if (options.isTimeTravel) { throw DeltaErrors.timeTravelNotSupportedException } val path = options.options.getOrElse("path", throw DeltaSharingErrors.pathNotSpecifiedException) val (responseFormat, parsedPath, deltaTableMetadataOpt) = autoResolveStreamingSource(sqlContext, path, options) if (responseFormat == DeltaSharingOptions.RESPONSE_FORMAT_PARQUET) { logInfo(s"sourceSchema with parquet format for table path:$path, parameters:$parameters") val deltaLog = RemoteDeltaLog( path, shareCredentialsOptions = options.shareCredentialsOptions, forStreaming = true, responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_PARQUET, callerOrg = options.callerOrg ) val schemaToUse = deltaLog.snapshot().schema if (schemaToUse.isEmpty) { throw DeltaSharingErrors.schemaNotSetException } if (options.readChangeFeed) { (shortName(), DeltaTableUtils.addCdcSchema(schemaToUse)) } else { (shortName(), schemaToUse) } } else if (responseFormat == DeltaSharingOptions.RESPONSE_FORMAT_DELTA) { logInfo(s"sourceSchema with delta format for table path:$path, parameters:$parameters") if (options.readChangeFeed) { throw new UnsupportedOperationException( s"Delta sharing cdc streaming is not supported when responseformat=delta." ) } // 1. create delta sharing client val client = DeltaSharingRestClient( profileFile = parsedPath.profileFile, shareCredentialsOptions = options.shareCredentialsOptions, forStreaming = true, responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_DELTA, // comma separated delta reader features, used to tell delta sharing server what delta // reader features the client is able to process. readerFeatures = DeltaSharingUtils.STREAMING_SUPPORTED_READER_FEATURES.mkString(","), callerOrg = options.callerOrg ) val dsTable = DeltaSharingTable( share = parsedPath.share, schema = parsedPath.schema, name = parsedPath.table ) // 2. getMetadata for schema to be used in the file index. val deltaSharingTableMetadata = deltaTableMetadataOpt match { case Some(metadata) => DeltaSharingUtils.getDeltaSharingTableMetadata( table = dsTable, deltaTableMetadata = metadata) case None => DeltaSharingUtils.getDeltaSharingTableMetadata(client = client, table = dsTable) } val customTablePathWithUUIDSuffix = DeltaSharingUtils.getTablePathWithIdSuffix( client.getProfileProvider.getCustomTablePath(path), DeltaSharingUtils.getFormattedTimestampWithUUID() ) val deltaLogPath = s"${DeltaSharingLogFileSystem.encode(customTablePathWithUUIDSuffix).toString}/_delta_log" val (_, snapshotDescriptor) = DeltaSharingUtils.getDeltaLogAndSnapshotDescriptor( sqlContext.sparkSession, deltaSharingTableMetadata, customTablePathWithUUIDSuffix ) // This is the analyzed schema for Delta streaming val readSchema = { // Check if we would like to merge consecutive schema changes, this would allow customers // to write queries based on their latest changes instead of an arbitrary schema in the // past. val shouldMergeConsecutiveSchemas = sqlContext.sparkSession.sessionState.conf.getConf( DeltaSQLConf.DELTA_STREAMING_ENABLE_SCHEMA_TRACKING_MERGE_CONSECUTIVE_CHANGES ) // This method is invoked during the analysis phase and would determine the schema for the // streaming dataframe. We only need to merge consecutive schema changes here because the // process would create a new entry in the schema log such that when the schema log is // looked up again in the execution phase, we would use the correct schema. // Delta sharing delta log doesn't have a catalog table, so we pass None here. DeltaDataSource .getMetadataTrackingLogForDeltaSource( sqlContext.sparkSession, snapshotDescriptor, catalogTableOpt = None, parameters, mergeConsecutiveSchemaChanges = shouldMergeConsecutiveSchemas ) .flatMap(_.getCurrentTrackedMetadata.map(_.dataSchema)) .getOrElse(snapshotDescriptor.schema) } val schemaToUse = TahoeDeltaTableUtils.removeInternalWriterMetadata( sqlContext.sparkSession, readSchema ) if (schemaToUse.isEmpty) { throw DeltaErrors.schemaNotSetException } DeltaSharingLogFileSystem.tryToCleanUpDeltaLog(deltaLogPath) (shortName(), schemaToUse) } else { throw new UnsupportedOperationException( s"responseformat(${responseFormat}) is not " + s"supported in delta sharing." ) } } override def createSource( sqlContext: SQLContext, metadataPath: String, schema: Option[StructType], providerName: String, parameters: Map[String, String]): Source = { DeltaSharingDataSource.setupFileSystem(sqlContext) if (schema.nonEmpty && schema.get.nonEmpty) { throw DeltaSharingErrors.specifySchemaAtReadTimeException } val options = new DeltaSharingOptions(parameters) val path = options.options.getOrElse("path", throw DeltaSharingErrors.pathNotSpecifiedException) val (responseFormat, parsedPath, _) = autoResolveStreamingSource(sqlContext, path, options) if (responseFormat == DeltaSharingOptions.RESPONSE_FORMAT_PARQUET) { logInfo(s"createSource with parquet format for table path:$path, parameters:$parameters") val deltaLog = RemoteDeltaLog( path, shareCredentialsOptions = options.shareCredentialsOptions, forStreaming = true, responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_PARQUET, callerOrg = options.callerOrg ) DeltaSharingSource(SparkSession.active, deltaLog, options) } else if (responseFormat == DeltaSharingOptions.RESPONSE_FORMAT_DELTA) { logInfo(s"createSource with delta format for table path:$path, parameters:$parameters") if (options.readChangeFeed) { throw new UnsupportedOperationException( s"Delta sharing cdc streaming is not supported when responseformat=delta." ) } // 1. create delta sharing client val client = DeltaSharingRestClient( profileFile = parsedPath.profileFile, shareCredentialsOptions = options.shareCredentialsOptions, forStreaming = true, responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_DELTA, // comma separated delta reader features, used to tell delta sharing server what delta // reader features the client is able to process. readerFeatures = DeltaSharingUtils.STREAMING_SUPPORTED_READER_FEATURES.mkString(","), callerOrg = options.callerOrg ) val dsTable = DeltaSharingTable( share = parsedPath.share, schema = parsedPath.schema, name = parsedPath.table ) DeltaFormatSharingSource( spark = sqlContext.sparkSession, client = client, table = dsTable, options = options, parameters = parameters, sqlConf = sqlContext.sparkSession.sessionState.conf, metadataPath = metadataPath ) } else { throw new UnsupportedOperationException( s"responseformat(${responseFormat}) is not " + s"supported in delta sharing." ) } } /** * Resolves the response format for streaming: when * DELTA_SHARING_STREAMING_AUTO_RESOLVE_RESPONSE_FORMAT is true, calls getMetadata on the * table and uses the server's responded format; otherwise uses the user's responseFormat option. * Returns (responseFormat, parsedPath, deltaTableMetadataOpt). When the conf is on, * deltaTableMetadataOpt is Some; sourceSchema's delta path reuses it to avoid a second RPC. */ private def autoResolveStreamingSource( sqlContext: SQLContext, path: String, options: DeltaSharingOptions ): (String, ParsedDeltaSharingTablePath, Option[DeltaTableMetadata]) = { val useGetMetadata = sqlContext.sparkSession.sessionState.conf.getConf( DeltaSQLConf.DELTA_SHARING_STREAMING_AUTO_RESOLVE_RESPONSE_FORMAT) val parsedPath = DeltaSharingRestClient.parsePath(path, options.shareCredentialsOptions) if (!useGetMetadata) { (options.responseFormat, parsedPath, None) } else if (options.responseFormat == DeltaSharingOptions.RESPONSE_FORMAT_DELTA) { // User explicitly requested delta; no need to call getMetadata. // Delta format can handle both parquet tables and delta tables. // Parquet format can only handle parquet tables. (DeltaSharingOptions.RESPONSE_FORMAT_DELTA, parsedPath, None) } else { val (_, deltaTableMetadata) = createClientAndQueryMetadata( sqlContext = sqlContext, parsedPath = parsedPath, shareCredentialsOptions = options.shareCredentialsOptions, forStreaming = true, versionAsOf = None, timestampAsOf = None, callerOrg = options.callerOrg ) logInfo(s"Streaming format resolved via getMetadata: ${deltaTableMetadata.respondedFormat} " + s"for path:$path") (deltaTableMetadata.respondedFormat, parsedPath, Some(deltaTableMetadata)) } } /** * Creates a Delta Sharing client (accepting parquet and/or delta per conf), a DeltaSharingTable, * and queries getMetadata. Used by streaming auto-resolve and batch auto-resolve. * Returns (dsTable, deltaTableMetadata); the client is not returned as callers either discard it * or create a format-specific client when needed. */ private def createClientAndQueryMetadata( sqlContext: SQLContext, parsedPath: ParsedDeltaSharingTablePath, shareCredentialsOptions: Map[String, String], forStreaming: Boolean, versionAsOf: Option[Long], timestampAsOf: Option[String], callerOrg: Option[String] = None): (DeltaSharingTable, DeltaTableMetadata) = { val responseFormat = { if (sqlContext.sparkSession.sessionState.conf.getConf( DeltaSQLConf.DELTA_SHARING_FORCE_DELTA_FORMAT)) { // If the Spark config is enabled, force the query to return results in Delta format. // This is primarily used for testing the Delta format code path, even when the source // table doesn't include advanced features like deletion vector. logInfo("Set delta sharing client to only accept delta format due to Spark config setting.") DeltaSharingOptions.RESPONSE_FORMAT_DELTA } else { s"${DeltaSharingOptions.RESPONSE_FORMAT_PARQUET}," + s"${DeltaSharingOptions.RESPONSE_FORMAT_DELTA}" } } // comma separated delta reader features, used to tell delta sharing server what delta // reader features the client is able to process. val readerFeatures = if (forStreaming) { DeltaSharingUtils.STREAMING_SUPPORTED_READER_FEATURES.mkString(",") } else { DeltaSharingUtils.SUPPORTED_READER_FEATURES.mkString(",") } val client = DeltaSharingRestClient( profileFile = parsedPath.profileFile, shareCredentialsOptions = shareCredentialsOptions, forStreaming = forStreaming, // Indicating that the client is able to process response format in both parquet and delta. responseFormat = responseFormat, readerFeatures = readerFeatures, callerOrg = callerOrg ) val dsTable = DeltaSharingTable( share = parsedPath.share, schema = parsedPath.schema, name = parsedPath.table ) val deltaTableMetadata = DeltaSharingUtils.queryDeltaTableMetadata( client = client, table = dsTable, versionAsOf = versionAsOf, timestampAsOf = timestampAsOf ) (dsTable, deltaTableMetadata) } override def createRelation( sqlContext: SQLContext, parameters: Map[String, String]): BaseRelation = { DeltaSharingDataSource.setupFileSystem(sqlContext) val options = new DeltaSharingOptions(parameters) val userInputResponseFormat = options.options.get(DeltaSharingOptions.RESPONSE_FORMAT) if (userInputResponseFormat.isEmpty && !options.readChangeFeed) { return autoResolveBaseRelationForSnapshotQuery(options, sqlContext) } val path = options.options.getOrElse("path", throw DeltaSharingErrors.pathNotSpecifiedException) if (options.responseFormat == DeltaSharingOptions.RESPONSE_FORMAT_PARQUET) { // When user explicitly set responseFormat=parquet, to query shared tables without advanced // delta features. logInfo(s"createRelation with parquet format for table path:$path, parameters:$parameters") val deltaLog = RemoteDeltaLog( path, shareCredentialsOptions = options.shareCredentialsOptions, forStreaming = false, responseFormat = options.responseFormat, callerOrg = options.callerOrg ) deltaLog.createRelation( options.versionAsOf, options.timestampAsOf, options.cdfOptions ) } else if (options.responseFormat == DeltaSharingOptions.RESPONSE_FORMAT_DELTA) { // When user explicitly set responseFormat=delta, to query shared tables with advanced // delta features. logInfo(s"createRelation with delta format for table path:$path, parameters:$parameters") // 1. create delta sharing client val parsedPath = DeltaSharingRestClient.parsePath(path, options.shareCredentialsOptions) val client = DeltaSharingRestClient( profileFile = parsedPath.profileFile, shareCredentialsOptions = options.shareCredentialsOptions, forStreaming = false, responseFormat = options.responseFormat, // comma separated delta reader features, used to tell delta sharing server what delta // reader features the client is able to process. readerFeatures = DeltaSharingUtils.SUPPORTED_READER_FEATURES.mkString(","), callerOrg = options.callerOrg ) val dsTable = DeltaSharingTable( share = parsedPath.share, schema = parsedPath.schema, name = parsedPath.table ) if (options.readChangeFeed) { return DeltaSharingCDFUtils.prepareCDFRelation(sqlContext, options, dsTable, client) } // 2. getMetadata for schema to be used in the file index. val deltaTableMetadata = DeltaSharingUtils.queryDeltaTableMetadata( client = client, table = dsTable, versionAsOf = options.versionAsOf, timestampAsOf = options.timestampAsOf ) val deltaSharingTableMetadata = DeltaSharingUtils.getDeltaSharingTableMetadata( table = dsTable, deltaTableMetadata = deltaTableMetadata ) // 3. Prepare HadoopFsRelation getHadoopFsRelationForDeltaSnapshotQuery( path = path, options = options, dsTable = dsTable, client = client, deltaSharingTableMetadata = deltaSharingTableMetadata ) } else { throw new UnsupportedOperationException( s"responseformat(${options.responseFormat}) is not supported in delta sharing." ) } } /** * "parquet format sharing" leverages the existing set of remote classes to directly handle the * list of presigned urls and read data. * "delta format sharing" instead constructs a local delta log and leverages the delta library to * read data. * Firstly we sends a getMetadata call to the delta sharing server the suggested response format * of the shared table by the server (based on whether there are advanced delta features in the * shared table), and then decide the code path on the client side. */ private def autoResolveBaseRelationForSnapshotQuery( options: DeltaSharingOptions, sqlContext: SQLContext): BaseRelation = { val path = options.options.getOrElse("path", throw DeltaSharingErrors.pathNotSpecifiedException) logInfo(s"autoResolving BaseRelation for path:${path}, " + s"with options:${DeltaSharingDataSource.redactOptions(options.options)}.") val parsedPath = DeltaSharingRestClient.parsePath(path, options.shareCredentialsOptions) val (dsTable, deltaTableMetadata) = createClientAndQueryMetadata( sqlContext = sqlContext, parsedPath = parsedPath, shareCredentialsOptions = options.shareCredentialsOptions, forStreaming = false, versionAsOf = options.versionAsOf, timestampAsOf = options.timestampAsOf, callerOrg = options.callerOrg ) if (deltaTableMetadata.respondedFormat == DeltaSharingOptions.RESPONSE_FORMAT_PARQUET) { logInfo(s"Resolved as parquet format for table path:$path, " + s"parameters:${DeltaSharingDataSource.redactOptions(options.options)}") val deltaLog = RemoteDeltaLog( path = path, options.shareCredentialsOptions, forStreaming = false, responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_PARQUET, initDeltaTableMetadata = Some(deltaTableMetadata), callerOrg = options.callerOrg ) deltaLog.createRelation(options.versionAsOf, options.timestampAsOf, options.cdfOptions) } else if (deltaTableMetadata.respondedFormat == DeltaSharingOptions.RESPONSE_FORMAT_DELTA) { logInfo(s"Resolved as delta format for table path:$path, " + s"parameters:${DeltaSharingDataSource.redactOptions(options.options)}") val deltaSharingTableMetadata = DeltaSharingUtils.getDeltaSharingTableMetadata( table = dsTable, deltaTableMetadata = deltaTableMetadata ) val deltaOnlyClient = DeltaSharingRestClient( profileFile = parsedPath.profileFile, shareCredentialsOptions = options.shareCredentialsOptions, forStreaming = false, // Indicating that the client request delta format in response. responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_DELTA, // comma separated delta reader features, used to tell delta sharing server what delta // reader features the client is able to process. readerFeatures = DeltaSharingUtils.SUPPORTED_READER_FEATURES.mkString(","), callerOrg = options.callerOrg ) getHadoopFsRelationForDeltaSnapshotQuery( path = path, options = options, dsTable = dsTable, client = deltaOnlyClient, deltaSharingTableMetadata = deltaSharingTableMetadata ) } else { throw new UnsupportedOperationException( s"Unexpected respondedFormat for getMetadata rpc:${deltaTableMetadata.respondedFormat}." ) } } /** * Prepare a HadoopFsRelation for the snapshot query on a delta sharing table. It will contain a * DeltaSharingFileIndex which is used to handle delta sharing rpc, and construct the local delta * log, and then build a TahoeFileIndex on top of the delta log. */ private def getHadoopFsRelationForDeltaSnapshotQuery( path: String, options: DeltaSharingOptions, dsTable: DeltaSharingTable, client: DeltaSharingClient, deltaSharingTableMetadata: DeltaSharingUtils.DeltaSharingTableMetadata): BaseRelation = { // Prepare DeltaSharingFileIndex val spark = SparkSession.active val params = new DeltaSharingFileIndexParams( new Path(path), spark, deltaSharingTableMetadata, options ) if (ConfUtils.limitPushdownEnabled(spark.sessionState.conf)) { DeltaFormatSharingLimitPushDown.setup(spark) } // limitHint is always None here and will be overridden in DeltaFormatSharingLimitPushDown. val fileIndex = DeltaSharingFileIndex( params = params, table = dsTable, client = client, limitHint = None ) // return HadoopFsRelation with the DeltaSharingFileIndex. HadoopFsRelation( location = fileIndex, // This is copied from DeltaLog.buildHadoopFsRelationWithFileIndex. // Dropping column mapping metadata because it is not relevant for partition schema. partitionSchema = TahoeDeltaTableUtils.removeInternalDeltaMetadata( spark, TahoeDeltaTableUtils.removeInternalWriterMetadata(spark, fileIndex.partitionSchema) ), // This is copied from DeltaLog.buildHadoopFsRelationWithFileIndex, original comment: // We pass all table columns as `dataSchema` so that Spark will preserve the partition // column locations. Otherwise, for any partition columns not in `dataSchema`, Spark would // just append them to the end of `dataSchema`. dataSchema = TahoeDeltaTableUtils.removeInternalDeltaMetadata( spark, TahoeDeltaTableUtils.removeInternalWriterMetadata( spark, SchemaUtils.dropNullTypeColumns(deltaSharingTableMetadata.metadata.schema) ) ), bucketSpec = None, // Handle column mapping metadata in schema. fileFormat = fileIndex.fileFormat( deltaSharingTableMetadata.protocol.deltaProtocol, deltaSharingTableMetadata.metadata.deltaMetadata ), options = Map.empty )(spark) } override def shortName(): String = "deltaSharing" } private[sharing] object DeltaSharingDataSource { def setupFileSystem(sqlContext: SQLContext): Unit = { sqlContext.sparkContext.hadoopConfiguration .setIfUnset("fs.delta-sharing.impl", "io.delta.sharing.client.DeltaSharingFileSystem") sqlContext.sparkContext.hadoopConfiguration .setIfUnset( "fs.delta-sharing-log.impl", "io.delta.sharing.spark.DeltaSharingLogFileSystem" ) PreSignedUrlCache.registerIfNeeded(SparkEnv.get) } def redactOptions(options: Map[String, String]): Map[String, String] = { options.map { case (k, _) if k.equalsIgnoreCase("bearerToken") => (k, "REDACTED") case (k, _) if k.equalsIgnoreCase("clientId") => (k, "REDACTED") case (k, _) if k.equalsIgnoreCase("clientSecret") => (k, "REDACTED") case (k, _) if k.equalsIgnoreCase("scope") => (k, "REDACTED") case (k, v) => (k, v) } } } ================================================ FILE: sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingFileIndex.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark import java.lang.ref.WeakReference import java.util.UUID import org.apache.spark.sql.delta.{DeltaFileFormat, DeltaLog} import org.apache.spark.sql.delta.files.{SupportsRowIndexFilters, TahoeLogFileIndex} import io.delta.sharing.client.DeltaSharingClient import io.delta.sharing.client.model.{Table => DeltaSharingTable} import io.delta.sharing.client.util.{ConfUtils, JsonUtils} import io.delta.sharing.filters.{AndOp, BaseOp, OpConverter} import org.apache.hadoop.fs.Path import org.apache.spark.delta.sharing.CachedTableManager import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.execution.datasources.{FileIndex, PartitionDirectory} import org.apache.spark.sql.types.StructType private[sharing] case class DeltaSharingFileIndexParams( path: Path, spark: SparkSession, deltaSharingTableMetadata: DeltaSharingUtils.DeltaSharingTableMetadata, options: DeltaSharingOptions) /** * A file index for delta sharing batch queries, that wraps a delta sharing table and client, which * is used to issue rpcs to delta sharing server to fetch pre-signed urls, then a local delta log is * constructed, and a TahoeFileIndex can be built on top of it. */ case class DeltaSharingFileIndex( params: DeltaSharingFileIndexParams, table: DeltaSharingTable, client: DeltaSharingClient, limitHint: Option[Long]) extends FileIndex with SupportsRowIndexFilters with DeltaFileFormat with Logging { // Use the head of an uuid as fileIndexId, which will be used in the key of the map from a delta // log path to a deltaLog class, to allow different queries with the same parameters on the // same fileIndex to reuse delta log. // Also, the uuid is used to differentiate different fileIndices on the same table in the same // cluster, the head of an uuid should be sufficient given the low collision chance and small // number of classes that could be created in the same computing environment. private val fileIndexId = UUID.randomUUID().toString().split('-').head override def spark: SparkSession = params.spark override def refresh(): Unit = {} override def sizeInBytes: Long = Option(params.deltaSharingTableMetadata.metadata.size).getOrElse { // Throw error if metadata.size is not returned, to urge the server to respond a table size. throw new IllegalStateException( "size is null in the metadata returned from the delta " + s"sharing server: ${params.deltaSharingTableMetadata.metadata}." ) } override def partitionSchema: StructType = params.deltaSharingTableMetadata.metadata.partitionSchema // Returns the partition columns of the shared delta table based on the returned metadata. def partitionColumns: Seq[String] = params.deltaSharingTableMetadata.metadata.deltaMetadata.partitionColumns override def rootPaths: Seq[Path] = params.path :: Nil override def inputFiles: Array[String] = { throw new UnsupportedOperationException("DeltaSharingFileIndex.inputFiles") } // A map that from queriedTableQueryId that we've issued delta sharing rpc, to the deltaLog // constructed with the response. // It is because this function will be called twice or more in a spark query, with this set, we // can avoid doing duplicated work of making expensive rpc and constructing the delta log. private val queriedTableQueryIdToDeltaLog = scala.collection.mutable.Map[String, DeltaLog]() def fetchFilesAndConstructDeltaLog( partitionFilters: Seq[Expression], dataFilters: Seq[Expression], overrideLimit: Option[Long]): DeltaLog = { val jsonPredicateHints = convertToJsonPredicate(partitionFilters, dataFilters) val queryParamsHashId = DeltaSharingUtils.getQueryParamsHashId( params.options, // Using .sql instead of toString because it doesn't include class pointer, which // keeps the string the same for the same filters. partitionFilters.map(_.sql).mkString(";"), dataFilters.map(_.sql).mkString(";"), jsonPredicateHints.getOrElse(""), overrideLimit.map(_.toString).getOrElse(""), params.deltaSharingTableMetadata.version ) // listFiles will be called twice or more in a spark query, with this check we can avoid // duplicated work of making expensive rpc and constructing the delta log. val tableKey = DeltaSharingUtils.getTablePathWithIdSuffix( fileIndexId + "." + params.path.toString, queryParamsHashId ) queriedTableQueryIdToDeltaLog.get(tableKey) match { case Some(deltaLog) => logInfo(s"Reusing deltaLog for tableKey:$tableKey.partitionFilters:$partitionFilters," + s"dataFilters:$dataFilters,overrideLimit:$overrideLimit.") deltaLog case None => val newDeltaLog = createDeltaLog( jsonPredicateHints, queryParamsHashId, overrideLimit ) // In theory there should only be one entry in this set since each query creates its own // FileIndex class. This is purged together with the FileIndex class when the query // finishes. queriedTableQueryIdToDeltaLog.put(tableKey, newDeltaLog) logInfo(s"Added new deltaLog for tableKey:$tableKey.partitionFilters:$partitionFilters," + s"dataFilters:$dataFilters,overrideLimit:$overrideLimit.") newDeltaLog } } private def createDeltaLog( jsonPredicateHints: Option[String], queryParamsHashId: String, overrideLimit: Option[Long]): DeltaLog = { // 1. Call client.getFiles. val startTime = System.currentTimeMillis() val deltaTableFiles = client.getFiles( table = table, predicates = Nil, limit = overrideLimit.orElse(limitHint), versionAsOf = params.options.versionAsOf, timestampAsOf = params.options.timestampAsOf, jsonPredicateHints = jsonPredicateHints, refreshToken = None, fileIdHash = None ) logInfo( s"Fetched ${deltaTableFiles.lines.size} lines for table $table with version " + s"${deltaTableFiles.version} from delta sharing server, took " + s"${(System.currentTimeMillis() - startTime) / 1000.0}s." ) // 2. Prepare a DeltaLog. val tablePathWithHashIdSuffix = DeltaSharingUtils.getTablePathWithIdSuffix( client.getProfileProvider.getCustomTablePath( params.path.toString ), queryParamsHashId ) val deltaLogMetadata = DeltaSharingLogFileSystem.constructLocalDeltaLogAtVersionZero( deltaTableFiles.lines, tablePathWithHashIdSuffix ) // 3. Register parquet file id to url mapping CachedTableManager.INSTANCE.register( // Using params.path directly because it will be customized within CachedTableManager. tablePath = DeltaSharingUtils.getTablePathWithIdSuffix( params.path.toString, queryParamsHashId ), idToUrl = deltaLogMetadata.idToUrl, refs = Seq(new WeakReference(this)), profileProvider = client.getProfileProvider, refresher = DeltaSharingUtils.getRefresherForGetFiles( client = client, table = table, predicates = Nil, limit = overrideLimit.orElse(limitHint), versionAsOf = params.options.versionAsOf, timestampAsOf = params.options.timestampAsOf, jsonPredicateHints = jsonPredicateHints, useRefreshToken = true ), expirationTimestamp = if (CachedTableManager.INSTANCE .isValidUrlExpirationTime(deltaLogMetadata.minUrlExpirationTimestamp)) { deltaLogMetadata.minUrlExpirationTimestamp.get } else { System.currentTimeMillis() + CachedTableManager.INSTANCE.preSignedUrlExpirationMs }, refreshToken = deltaTableFiles.refreshToken ) // 4. Create a local file index and call listFiles of this class. val deltaLog = DeltaLog.forTable( params.spark, DeltaSharingLogFileSystem.encode(tablePathWithHashIdSuffix) ) deltaLog } def asTahoeFileIndex( partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): TahoeLogFileIndex = { val deltaLog = fetchFilesAndConstructDeltaLog(partitionFilters, dataFilters, None) TahoeLogFileIndex(params.spark, deltaLog, catalogTableOpt = None) } override def listFiles( partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): Seq[PartitionDirectory] = { // NOTE: The server is not required to apply all filters, so we apply them client-side as well. asTahoeFileIndex(partitionFilters, dataFilters).listFiles(partitionFilters, dataFilters) } // Converts the specified SQL expressions to a json predicate. // // If jsonPredicatesV2 are enabled, converts both partition and data filters // and combines them using an AND. // // If the conversion fails, returns a None, which will imply that we will // not perform json predicate based filtering. private def convertToJsonPredicate( partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): Option[String] = { if (!ConfUtils.jsonPredicatesEnabled(params.spark.sessionState.conf)) { return None } // Convert the partition filters. val partitionOp = try { OpConverter.convert(partitionFilters) } catch { case e: Exception => log.error("Error while converting partition filters: " + e) None } // If V2 predicates are enabled, also convert the data filters. val dataOp = try { if (ConfUtils.jsonPredicatesV2Enabled(params.spark.sessionState.conf)) { log.info("Converting data filters") OpConverter.convert(dataFilters) } else { None } } catch { case e: Exception => log.error("Error while converting data filters: " + e) None } // Combine partition and data filters using an AND operation. val combinedOp = if (partitionOp.isDefined && dataOp.isDefined) { Some(AndOp(Seq(partitionOp.get, dataOp.get))) } else if (partitionOp.isDefined) { partitionOp } else { dataOp } log.info("Using combined predicate: " + combinedOp) if (combinedOp.isDefined) { Some(JsonUtils.toJson[BaseOp](combinedOp.get)) } else { None } } } ================================================ FILE: sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingLogFileSystem.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark import java.io.{ByteArrayInputStream, FileNotFoundException} import java.net.{URI, URLDecoder, URLEncoder} import java.nio.charset.StandardCharsets import scala.collection.JavaConverters._ import scala.collection.mutable.{ArrayBuffer, Builder} import scala.reflect.ClassTag import scala.util.control.NonFatal import org.apache.spark.sql.delta.actions.{AddCDCFile, AddFile, DeletionVectorDescriptor, RemoveFile, SingleAction} import org.apache.spark.sql.delta.util.FileNames import io.delta.sharing.client.util.JsonUtils import io.delta.sharing.spark.DeltaSharingUtils.{DeltaSharingTableMetadata, FAKE_CHECKPOINT_BYTE_ARRAY} import org.apache.hadoop.fs._ import org.apache.hadoop.fs.permission.FsPermission import org.apache.hadoop.util.Progressable import org.apache.spark.SparkEnv import org.apache.spark.internal.Logging import org.apache.spark.storage.BlockId /** Read-only file system for delta sharing log. * This is a faked file system to serve data under path delta-sharing-log:/. The delta log will be * prepared by DeltaSharingDataSource and its related classes, put in blockManager, and then serve * to DeltaLog with a path pointing to this file system. * In executor, when it tries to read data from the delta log, this file system class will return * the data fetched from the block manager. */ private[sharing] class DeltaSharingLogFileSystem extends FileSystem with Logging { import DeltaSharingLogFileSystem._ override def getScheme: String = SCHEME override def getUri(): URI = URI.create(s"$SCHEME:///") override def open(f: Path, bufferSize: Int): FSDataInputStream = { if (FileNames.isCheckpointFile(f)) { new FSDataInputStream( new SeekableByteArrayInputStream(DeltaSharingUtils.FAKE_CHECKPOINT_BYTE_ARRAY) ) } else if (FileNames.isDeltaFile(f)) { val iterator = SparkEnv.get.blockManager.get[String](getDeltaSharingLogBlockId(f.toString)) match { case Some(block) => block.data.asInstanceOf[Iterator[String]] case _ => throw new FileNotFoundException(s"Failed to open delta log file: $f.") } // Explicitly call hasNext to allow the reader lock on the block to be released. val arrayBuilder = Array.newBuilder[Byte] while (iterator.hasNext) { val actionJsonStr = iterator.next() arrayBuilder ++= actionJsonStr.getBytes(StandardCharsets.UTF_8) } // We still have to load the full content of a delta log file in memory to serve them. // This still exposes the risk of OOM. new FSDataInputStream(new SeekableByteArrayInputStream(arrayBuilder.result())) } else { val content = getBlockAndReleaseLockHelper[String](f, None, "open") new FSDataInputStream(new SeekableByteArrayInputStream( content.getBytes(StandardCharsets.UTF_8) )) } } override def exists(f: Path): Boolean = { // The reason of using the variable exists is to allow us to explicitly release the reader lock // on the blockId. val blockId = getDeltaSharingLogBlockId(f.toString) val exists = SparkEnv.get.blockManager.get(blockId).isDefined if (exists) { releaseLockHelper(blockId) } exists } // Delta sharing log file system serves checkpoint file with a CONSTANT value so we construct the // FileStatus when the function is being called. // For other files, they will be constructed and put into block manager when constructing the // delta log based on the rpc response from the server. override def getFileStatus(f: Path): FileStatus = { val status = if (FileNames.isCheckpointFile(f)) { DeltaSharingLogFileStatus( path = f.toString, size = FAKE_CHECKPOINT_BYTE_ARRAY.size, modificationTime = 0L ) } else { getBlockAndReleaseLockHelper[DeltaSharingLogFileStatus](f, Some("_status"), "getFileStatus") } new FileStatus( /* length */ status.size, /* isdir */ false, /* block_replication */ 0, /* blocksize */ 1, /* modification_time */ status.modificationTime, /* path */ new Path(status.path) ) } /** * @param f: a Path pointing to a delta log directory of a delta sharing table, example: * delta-sharing-log:/customized-delta-sharing-table/_delta_log * The iterator contains a list of tuple(json_file_path, json_file_size) which are * pre-prepared and set in the block manager by DeltaSharingDataSource and its related * classes. * @return the list of json files under the /_delta_log directory, if prepared. */ override def listStatus(f: Path): Array[FileStatus] = { val iterator = SparkEnv.get.blockManager .get[DeltaSharingLogFileStatus](getDeltaSharingLogBlockId(f.toString)) match { case Some(block) => block.data.asInstanceOf[Iterator[DeltaSharingLogFileStatus]] case _ => throw new FileNotFoundException(s"Failed to listStatus for path: $f.") } // Explicitly call hasNext to allow the reader lock on the block to be released. val arrayBuilder = Array.newBuilder[FileStatus] while (iterator.hasNext) { val fileStatus = iterator.next() arrayBuilder += new FileStatus( /* length */ fileStatus.size, /* isdir */ false, /* block_replication */ 0, /* blocksize */ 1, /* modification_time */ fileStatus.modificationTime, /* path */ new Path(fileStatus.path) ) } arrayBuilder.result() } override def create( f: Path, permission: FsPermission, overwrite: Boolean, bufferSize: Int, replication: Short, blockSize: Long, progress: Progressable): FSDataOutputStream = { throw new UnsupportedOperationException(s"create: $f") } override def append(f: Path, bufferSize: Int, progress: Progressable): FSDataOutputStream = { throw new UnsupportedOperationException(s"append: $f") } override def rename(src: Path, dst: Path): Boolean = { throw new UnsupportedOperationException(s"rename: src:$src, dst:$dst") } override def delete(f: Path, recursive: Boolean): Boolean = { throw new UnsupportedOperationException(s"delete: $f") } override def listStatusIterator(f: Path): RemoteIterator[FileStatus] = { throw new UnsupportedOperationException(s"listStatusIterator: $f") } override def setWorkingDirectory(newDir: Path): Unit = throw new UnsupportedOperationException(s"setWorkingDirectory: $newDir") override def getWorkingDirectory: Path = new Path(getUri) override def mkdirs(f: Path, permission: FsPermission): Boolean = { throw new UnsupportedOperationException(s"mkdirs: $f") } override def close(): Unit = { super.close() } private def getBlockAndReleaseLockHelper[T: ClassTag]( f: Path, suffix: Option[String], caller: String): T = { val blockId = getDeltaSharingLogBlockId(suffix.foldLeft(f.toString)(_ + _)) val result = SparkEnv.get.blockManager.getSingle[T](blockId).getOrElse { throw new FileNotFoundException(s"Failed to $caller for $f.") } releaseLockHelper(blockId) result } private def releaseLockHelper(blockId: BlockId): Unit = { try { SparkEnv.get.blockManager.releaseLock(blockId) } catch { // releaseLock may fail when the lock is not hold by this thread, we are not exactly sure // when it fails or not, but no need to fail the entire delta sharing query. case e: Throwable => logWarning(s"Error while releasing lock for blockId:$blockId: $e.") } } } /** * A case class including the metadata for the constructed delta log based on the delta sharing * rpc response. * @param idToUrl stores the id to url mapping, used to register to CachedTableManager * @param minUrlExpirationTimestamp used to indicate when to refresh urls in CachedTableManager * @param numFileActionsInMinVersionOpt This is needed because DeltaSource is not advancing the * offset to the next version automatically when scanning * through a snapshot, so DeltaSharingSource needs to count the * number of files in the min version and advance the offset to * the next version when the offset is at the last index of the * version. * @param minVersion minVersion of all the files returned from server * @param maxVersion maxVersion of all the files returned from server */ case class ConstructedDeltaLogMetadata( idToUrl: Map[String, String], minUrlExpirationTimestamp: Option[Long], numFileActionsInMinVersionOpt: Option[Int], minVersion: Long, maxVersion: Long) /** Public constants for DeltaSharingLogFileSystem accessible visible outside the package */ object DeltaSharingLogFileSystemConstants { /** The URI scheme used for delta-sharing fake delta-logs. */ final val SCHEME = "delta-sharing-log" } private[sharing] object DeltaSharingLogFileSystem extends Logging { val SCHEME = DeltaSharingLogFileSystemConstants.SCHEME // The constant added as prefix to all delta sharing block ids. private val BLOCK_ID_TEST_PREFIX = "test_" // It starts with test_ to match the prefix of TestBlockId. // In the meantime, we'll investigate in an option to add a general purposed BlockId subclass // and use it in delta sharing. val DELTA_SHARING_LOG_BLOCK_ID_PREFIX = "test_delta-sharing-log:" def getDeltaSharingLogBlockId(path: String): BlockId = { BlockId(BLOCK_ID_TEST_PREFIX + path) } /** * Encode `tablePath` to a `Path` in the following format: * * ``` * delta-sharing-log:/// * ``` * * This format can be decoded by `DeltaSharingLogFileSystem.decode`. * It will be used to: * 1) construct a DeltaLog class which points to a delta sharing table. * 2) construct a block id to look for commit files of the delta sharing table. */ def encode(tablePath: String): Path = { val encodedTablePath = URLEncoder.encode(tablePath, "UTF-8") new Path(s"$SCHEME:///$encodedTablePath") } def decode(path: Path): String = { val encodedTablePath = path.toString .stripPrefix(s"$SCHEME:///") .stripPrefix(s"$SCHEME:/") URLDecoder.decode(encodedTablePath, "UTF-8") } // Convert a deletion vector path to a delta sharing path. // Only paths needs to be converted since it's pre-signed url. Inline DV should be handled // in place. And UUID should throw error since it should be converted to pre-signed url when // returned from the server. private def getDeltaSharingDeletionVectorDescriptor( fileAction: model.DeltaSharingFileAction, customTablePath: String): DeletionVectorDescriptor = { if (fileAction.getDeletionVectorOpt.isEmpty) { null } else { val deletionVector = fileAction.getDeletionVectorOpt.get deletionVector.storageType match { case DeletionVectorDescriptor.PATH_DV_MARKER => deletionVector.copy( pathOrInlineDv = fileAction.getDeletionVectorDeltaSharingPath(customTablePath) ) case DeletionVectorDescriptor.INLINE_DV_MARKER => deletionVector case storageType => throw new IllegalStateException( s"Unexpected DV storage type:" + s"$storageType in the delta sharing response for ${fileAction.json}." ) } } } /** * Convert DeltaSharingFileAction with delta sharing file path and serialize as json to store in * the delta log. * * @param fileAction The DeltaSharingFileAction to convert. * @param customTablePath The table path used to construct action.path field. * @return json serialization of delta action. */ private def getActionWithDeltaSharingPath( fileAction: model.DeltaSharingFileAction, customTablePath: String): String = { val deltaSharingPath = fileAction.getDeltaSharingPath(customTablePath) val newSingleAction = fileAction.deltaSingleAction.unwrap match { case add: AddFile => add.copy( path = deltaSharingPath, deletionVector = getDeltaSharingDeletionVectorDescriptor(fileAction, customTablePath) ) case cdc: AddCDCFile => assert( cdc.deletionVector == null, "deletionVector not null in the AddCDCFile from delta" + s" sharing response: ${cdc.json}" ) cdc.copy(path = deltaSharingPath) case remove: RemoveFile => remove.copy( path = deltaSharingPath, deletionVector = getDeltaSharingDeletionVectorDescriptor(fileAction, customTablePath) ) case action => throw new IllegalStateException( s"unexpected action in delta sharing " + s"response: ${action.json}" ) } newSingleAction.json } // Sort by id to keep a stable order of the files within a version in the delta log. private def deltaSharingFileActionIncreaseOrderFunc( f1: model.DeltaSharingFileAction, f2: model.DeltaSharingFileAction): Boolean = { f1.id < f2.id } /** * Cleanup the delta log upon explicit stop of a query on a delta sharing table. * * @param deltaLogPath deltaLogPath is constructed per query with credential scope id as prefix * and a uuid as suffix, which is very unique to the query and won't interfere * with other queries. */ def tryToCleanUpDeltaLog(deltaLogPath: String): Unit = { def shouldCleanUp(blockId: BlockId): Boolean = { if (!blockId.name.startsWith(DELTA_SHARING_LOG_BLOCK_ID_PREFIX)) { return false } val blockName = blockId.name // deltaLogPath is constructed per query with credential scope id as prefix and a uuid as // suffix, which is very unique to the query and won't interfere with other queries. blockName.startsWith(BLOCK_ID_TEST_PREFIX + deltaLogPath) } val blockManager = SparkEnv.get.blockManager val matchingBlockIds = blockManager.getMatchingBlockIds(shouldCleanUp(_)) logInfo( s"Trying to clean up ${matchingBlockIds.size} blocks for $deltaLogPath." ) val problematicBlockIds = Seq.newBuilder[BlockId] matchingBlockIds.foreach { b => try { blockManager.removeBlock(b) } catch { case _: Throwable => problematicBlockIds += b } } val problematicBlockIdsSeq = problematicBlockIds.result().toSeq if (problematicBlockIdsSeq.size > 0) { logWarning( s"Done cleaning up ${matchingBlockIds.size} blocks for $deltaLogPath, but " + s"failed to remove: ${problematicBlockIdsSeq}." ) } else { logInfo( s"Done cleaning up ${matchingBlockIds.size} blocks for $deltaLogPath." ) } } private def prepareCheckpointFile( deltaLogPath: String, checkpointVersion: Long, fileSizeTsSeq: Builder[DeltaSharingLogFileStatus, Seq[DeltaSharingLogFileStatus]]): Unit = { // 1) store the checkpoint byte array in BlockManager for future read. val checkpointParquetFileName = FileNames.checkpointFileSingular(new Path(deltaLogPath), checkpointVersion).toString fileSizeTsSeq += DeltaSharingLogFileStatus( path = checkpointParquetFileName, size = FAKE_CHECKPOINT_BYTE_ARRAY.size, modificationTime = 0L ) // 2) Prepare the content for _last_checkpoint val lastCheckpointContent = s"""{"version":${checkpointVersion},"size":${FAKE_CHECKPOINT_BYTE_ARRAY.size}}""" val lastCheckpointPath = new Path(deltaLogPath, "_last_checkpoint").toString fileSizeTsSeq += DeltaSharingLogFileStatus( path = lastCheckpointPath, size = lastCheckpointContent.length, modificationTime = 0L ) DeltaSharingUtils.overrideSingleBlock[String]( blockId = getDeltaSharingLogBlockId(lastCheckpointPath), value = lastCheckpointContent ) } private def updateListingDeltaLog(deltaLogPath: String, checkpointVersion: Long): Unit = { val fileSizeTsSeq = Seq.newBuilder[DeltaSharingLogFileStatus] prepareCheckpointFile(deltaLogPath, checkpointVersion, fileSizeTsSeq) val iterator = SparkEnv.get.blockManager .get[DeltaSharingLogFileStatus](getDeltaSharingLogBlockId(deltaLogPath)) match { case Some(block) => block.data.asInstanceOf[Iterator[DeltaSharingLogFileStatus]] case _ => throw new FileNotFoundException(s"Failed to list files for path: $deltaLogPath.") } // Explicitly materialize iterator to allow the reader lock on the block to be released. val files = iterator.flatMap { deltaSharingLogFileStatus => val filePath = new Path(deltaSharingLogFileStatus.path) filePath match { case FileNames.CheckpointFile(_, version) if version > checkpointVersion => Some(deltaSharingLogFileStatus) case FileNames.DeltaFile(_, version) if version > checkpointVersion => Some(deltaSharingLogFileStatus) case _ => None } }.toIndexedSeq DeltaSharingUtils.overrideIteratorBlock[DeltaSharingLogFileStatus]( getDeltaSharingLogBlockId(deltaLogPath), (fileSizeTsSeq.result() ++ files).toIterator ) } /** * @param deltaLogPath The delta log directory to clean up. It is constructed per query with * credential scope id as prefix and a uuid as suffix, which is very unique * to the query and won't interfere with other queries. * @param maxVersion maxVersion of any checkpoint or delta file that needs clean up, inclusive. */ def tryToCleanUpPreviousBlocks(deltaLogPath: String, maxVersion: Long): Unit = { if (maxVersion < 0) { logInfo( s"Skipping clean up previous blocks for $deltaLogPath because maxVersion(" + s"$maxVersion) < 0." ) return } def shouldCleanUp(blockId: BlockId): Boolean = { if (!blockId.name.startsWith(DELTA_SHARING_LOG_BLOCK_ID_PREFIX)) { return false } val blockName = blockId.name blockName.startsWith(BLOCK_ID_TEST_PREFIX + deltaLogPath) && FileNames .getFileVersionOpt(new Path(blockName.stripPrefix(BLOCK_ID_TEST_PREFIX))) .exists(_ <= maxVersion) } val blockManager = SparkEnv.get.blockManager // 1) try to update the listing of the delta log files: // - add a new checkpoint at maxVersion. // - update the _last_checkpoint file. // - update the result of listStatus for deltaLogPath. try { updateListingDeltaLog(deltaLogPath, maxVersion) } catch { case NonFatal(e) => logWarning( s"Stopped cleaning up the delta log for [$deltaLogPath], because updating the " + s"listStatus of deltaLog() failed due to [${e.toString}]." ) return } // 2) try to clean up the delta log (.json) file for each version that has been read. val matchingBlockIds = blockManager.getMatchingBlockIds(shouldCleanUp(_)) logInfo( s"Trying to clean up ${matchingBlockIds.size} previous blocks for $deltaLogPath " + s"before version: $maxVersion." ) val problematicBlockIds = Seq.newBuilder[BlockId] matchingBlockIds.foreach { b => try { blockManager.removeBlock(b) } catch { case _: Throwable => problematicBlockIds += b } } val problematicBlockIdsSeq = problematicBlockIds.result().toSeq if (problematicBlockIdsSeq.size > 0) { logWarning( s"Done cleaning up ${matchingBlockIds.size} previous blocks for $deltaLogPath " + s"before version: $maxVersion, but failed to remove: ${problematicBlockIdsSeq}." ) } else { logInfo( s"Done cleaning up ${matchingBlockIds.size} previous blocks for $deltaLogPath " + s"before version: $maxVersion." ) } } /** * Construct local delta log based on delta log actions returned from delta sharing server. * * @param lines a list of delta actions, to be processed and put in the local delta log, * each action contains a version field to indicate the version of log to * put it in. * @param customTablePath query customized table path, used to construct action.path field for * DeltaSharingFileSystem * @param startingVersionOpt If set, used to construct the delta file (.json log file) from the * given startingVersion. This is needed by DeltaSharingSource to * construct the delta log for the rpc no matter if there are files in * that version or not, so DeltaSource can read delta actions from the * starting version (instead from checkpoint). * @param endingVersionOpt If set, used to construct the delta file (.json log file) until the * given endingVersion. This is needed by DeltaSharingSource to construct * the delta log for the rpc no matter if there are files in that version * or not. * NOTE: DeltaSource will not advance the offset if there are no files in * a version of the delta log, but we still create the delta log file for * that version to avoid missing delta log (json) files. * @return ConstructedDeltaLogMetadata, which contains 3 fields: * - idToUrl: mapping from file id to pre-signed url * - minUrlExpirationTimestamp timestamp indicating the when to refresh pre-signed urls. * Both are used to register to CachedTableManager. * - maxVersion: the max version returned in the http response, used by * DeltaSharingSource to quickly understand the progress of rpcs from the server. */ def constructLocalDeltaLogAcrossVersions( lines: Seq[String], customTablePath: String, startingVersionOpt: Option[Long], endingVersionOpt: Option[Long]): ConstructedDeltaLogMetadata = { val startTime = System.currentTimeMillis() assert( startingVersionOpt.isDefined == endingVersionOpt.isDefined, s"startingVersionOpt($startingVersionOpt) and endingVersionOpt($endingVersionOpt) should be" + " both defined or not." ) if (startingVersionOpt.isDefined) { assert( startingVersionOpt.get <= endingVersionOpt.get, s"startingVersionOpt($startingVersionOpt) must be smaller than " + s"endingVersionOpt($endingVersionOpt)." ) } var minVersion = Long.MaxValue var maxVersion = 0L var minUrlExpirationTimestamp: Option[Long] = None val idToUrl = scala.collection.mutable.Map[String, String]() val versionToDeltaSharingFileActions = scala.collection.mutable.Map[Long, ArrayBuffer[model.DeltaSharingFileAction]]() val versionToMetadata = scala.collection.mutable.Map[Long, model.DeltaSharingMetadata]() val versionToJsonLogBuilderMap = scala.collection.mutable.Map[Long, ArrayBuffer[String]]() val versionToJsonLogSize = scala.collection.mutable.Map[Long, Long]().withDefaultValue(0L) var numFileActionsInMinVersion = 0 val versionToTimestampMap = scala.collection.mutable.Map[Long, Long]() var startingMetadataLineOpt: Option[String] = None var startingProtocolLineOpt: Option[String] = None lines.foreach { line => val action = JsonUtils.fromJson[model.DeltaSharingSingleAction](line).unwrap action match { case fileAction: model.DeltaSharingFileAction => minVersion = minVersion.min(fileAction.version) maxVersion = maxVersion.max(fileAction.version) // Store file actions in an array to sort them based on id later. versionToDeltaSharingFileActions.getOrElseUpdate( fileAction.version, ArrayBuffer[model.DeltaSharingFileAction]() ) += fileAction case metadata: model.DeltaSharingMetadata => if (metadata.version != null) { // This is to handle the cdf and streaming query result. minVersion = minVersion.min(metadata.version) maxVersion = maxVersion.max(metadata.version) versionToMetadata(metadata.version) = metadata if (metadata.version == minVersion) { startingMetadataLineOpt = Some(metadata.deltaMetadata.json + "\n") } } else { // This is to handle the snapshot query result from DeltaSharingSource. startingMetadataLineOpt = Some(metadata.deltaMetadata.json + "\n") } case protocol: model.DeltaSharingProtocol => startingProtocolLineOpt = Some(protocol.deltaProtocol.json + "\n") case _ => // do nothing, ignore the line. } } if (startingVersionOpt.isDefined) { minVersion = minVersion.min(startingVersionOpt.get) } else if (minVersion == Long.MaxValue) { // This means there are no files returned from server for this cdf request. // A 0.json file will be prepared with metadata and protocol only. minVersion = 0 } if (endingVersionOpt.isDefined) { maxVersion = maxVersion.max(endingVersionOpt.get) } // Store the starting protocol and metadata in the minVersion.json. val protocolAndMetadataStr = startingMetadataLineOpt.getOrElse("") + startingProtocolLineOpt .getOrElse("") versionToJsonLogBuilderMap.getOrElseUpdate( minVersion, ArrayBuffer[String]() ) += protocolAndMetadataStr versionToJsonLogSize(minVersion) += protocolAndMetadataStr.getBytes( StandardCharsets.UTF_8 ).length numFileActionsInMinVersion = versionToDeltaSharingFileActions .getOrElseUpdate(minVersion, ArrayBuffer[model.DeltaSharingFileAction]()) .size // Write metadata to the delta log json file. versionToMetadata.foreach { case (version, metadata) => if (version != minVersion) { val metadataStr = metadata.deltaMetadata.json + "\n" versionToJsonLogBuilderMap.getOrElseUpdate( version, ArrayBuffer[String]() ) += metadataStr versionToJsonLogSize(version) += metadataStr.getBytes(StandardCharsets.UTF_8).length } } // Write file actions to the delta log json file. var previousIdOpt: Option[String] = None versionToDeltaSharingFileActions.foreach { case (version, actions) => previousIdOpt = None actions.toSeq.sortWith(deltaSharingFileActionIncreaseOrderFunc).foreach { fileAction => assert( // Using > instead of >= because there can be a removeFile and addFile pointing to the // same parquet file which result in the same file id, since id is a hash of file path. // This is ok because eventually it can read data out of the correct parquet file. !previousIdOpt.exists(_ > fileAction.id), s"fileActions must be in increasing order by id: ${previousIdOpt} is not smaller than" + s" ${fileAction.id}, in version:$version." ) previousIdOpt = Some(fileAction.id) // 1. build it to url mapping idToUrl(fileAction.id) = fileAction.path if (DeltaSharingUtils.requiresIdToUrlForDV(fileAction.getDeletionVectorOpt)) { idToUrl(fileAction.deletionVectorFileId) = fileAction.getDeletionVectorOpt.get.pathOrInlineDv } // 2. prepare json log content. versionToTimestampMap.getOrElseUpdate(version, fileAction.timestamp) val actionJsonStr = getActionWithDeltaSharingPath(fileAction, customTablePath) + "\n" versionToJsonLogBuilderMap.getOrElseUpdate( version, ArrayBuffer[String]() ) += actionJsonStr versionToJsonLogSize(version) += actionJsonStr.getBytes(StandardCharsets.UTF_8).length // 3. process expiration timestamp if (fileAction.expirationTimestamp != null) { minUrlExpirationTimestamp = minUrlExpirationTimestamp .filter(_ < fileAction.expirationTimestamp) .orElse(Some(fileAction.expirationTimestamp)) } } } val encodedTablePath = DeltaSharingLogFileSystem.encode(customTablePath) val deltaLogPath = s"${encodedTablePath.toString}/_delta_log" val fileSizeTsSeq = Seq.newBuilder[DeltaSharingLogFileStatus] if (minVersion > 0) { // If the minVersion is not 0 in the response, then prepare checkpoint at minVersion - 1: // need to prepare two files: 1) (minVersion-1).checkpoint.parquet 2) _last_checkpoint prepareCheckpointFile( deltaLogPath, checkpointVersion = minVersion - 1, fileSizeTsSeq = fileSizeTsSeq) } for (version <- minVersion to maxVersion) { val jsonFilePath = FileNames.unsafeDeltaFile(new Path(deltaLogPath), version).toString DeltaSharingUtils.overrideIteratorBlock[String]( getDeltaSharingLogBlockId(jsonFilePath), versionToJsonLogBuilderMap.getOrElse(version, Seq.empty).toIterator ) fileSizeTsSeq += DeltaSharingLogFileStatus( path = jsonFilePath, size = versionToJsonLogSize.getOrElse(version, 0), modificationTime = versionToTimestampMap.get(version).getOrElse(0L) ) } DeltaSharingUtils.overrideIteratorBlock[DeltaSharingLogFileStatus]( getDeltaSharingLogBlockId(deltaLogPath), fileSizeTsSeq.result().toIterator ) logInfo( s"It takes ${(System.currentTimeMillis() - startTime) / 1000.0}s to construct delta log" + s"for $customTablePath from $minVersion to $maxVersion, with ${idToUrl.toMap.size} urls." ) ConstructedDeltaLogMetadata( idToUrl = idToUrl.toMap, minUrlExpirationTimestamp = minUrlExpirationTimestamp, numFileActionsInMinVersionOpt = Some(numFileActionsInMinVersion), minVersion = minVersion, maxVersion = maxVersion ) } /** Set the modificationTime to zero, this is to align with the time returned from * DeltaSharingFileSystem.getFileStatus */ private def setModificationTimestampToZero(deltaSingleAction: SingleAction): SingleAction = { deltaSingleAction.unwrap match { case a: AddFile => a.copy(modificationTime = 0).wrap case _ => deltaSingleAction } } /** * Construct local delta log at version zero based on lines returned from delta sharing server, * to support latest snapshot or time travel queries. Storing both protocol/metadata and * the actual data actions in version 0 will simplify both the log construction and log reply. * * @param lines a list of delta actions, to be processed and put in the local delta log, * each action contains a version field to indicate the version of log to * put it in. * @param customTablePath query customized table path, used to construct action.path field for * DeltaSharingFileSystem * @return ConstructedDeltaLogMetadata, which contains 3 fields: * - idToUrl: mapping from file id to pre-signed url * - minUrlExpirationTimestamp timestamp indicating the when to refresh pre-signed urls. * Both are used to register to CachedTableManager. * - maxVersion: to be 0. */ def constructLocalDeltaLogAtVersionZero( lines: Seq[String], customTablePath: String): ConstructedDeltaLogMetadata = { val startTime = System.currentTimeMillis() val jsonLogSeq = Seq.newBuilder[String] var jsonLogSize = 0 var minUrlExpirationTimestamp: Option[Long] = None val fileActionsSeq = ArrayBuffer[model.DeltaSharingFileAction]() val idToUrl = scala.collection.mutable.Map[String, String]() lines.foreach { line => val action = JsonUtils.fromJson[model.DeltaSharingSingleAction](line).unwrap action match { case fileAction: model.DeltaSharingFileAction => // Store file actions in an array to sort them based on id later. fileActionsSeq += fileAction.copy( deltaSingleAction = setModificationTimestampToZero(fileAction.deltaSingleAction) ) case protocol: model.DeltaSharingProtocol => val protocolJsonStr = protocol.deltaProtocol.json + "\n" jsonLogSize += protocolJsonStr.getBytes(StandardCharsets.UTF_8).length jsonLogSeq += protocolJsonStr case metadata: model.DeltaSharingMetadata => val metadataJsonStr = metadata.deltaMetadata.json + "\n" jsonLogSize += metadataJsonStr.getBytes(StandardCharsets.UTF_8).length jsonLogSeq += metadataJsonStr case _ => throw new IllegalStateException( s"unknown action in the delta sharing " + s"response: $line" ) } } var previousIdOpt: Option[String] = None fileActionsSeq.toSeq.sortWith(deltaSharingFileActionIncreaseOrderFunc).foreach { fileAction => assert( // Using > instead of >= because there can be a removeFile and addFile pointing to the same // parquet file which result in the same file id, since id is a hash of file path. // This is ok because eventually it can read data out of the correct parquet file. !previousIdOpt.exists(_ > fileAction.id), s"fileActions must be in increasing order by id: ${previousIdOpt} is not smaller than" + s" ${fileAction.id}." ) previousIdOpt = Some(fileAction.id) // 1. build id to url mapping idToUrl(fileAction.id) = fileAction.path if (DeltaSharingUtils.requiresIdToUrlForDV(fileAction.getDeletionVectorOpt)) { idToUrl(fileAction.deletionVectorFileId) = fileAction.getDeletionVectorOpt.get.pathOrInlineDv } // 2. prepare json log content. val actionJsonStr = getActionWithDeltaSharingPath(fileAction, customTablePath) + "\n" jsonLogSize += actionJsonStr.getBytes(StandardCharsets.UTF_8).length jsonLogSeq += actionJsonStr // 3. process expiration timestamp if (fileAction.expirationTimestamp != null) { minUrlExpirationTimestamp = if (minUrlExpirationTimestamp.isDefined && minUrlExpirationTimestamp.get < fileAction.expirationTimestamp) { minUrlExpirationTimestamp } else { Some(fileAction.expirationTimestamp) } } } val encodedTablePath = DeltaSharingLogFileSystem.encode(customTablePath) // Always use 0.json for snapshot queries. val deltaLogPath = s"${encodedTablePath.toString}/_delta_log" val jsonFilePath = FileNames.unsafeDeltaFile(new Path(deltaLogPath), 0).toString DeltaSharingUtils.overrideIteratorBlock[String]( getDeltaSharingLogBlockId(jsonFilePath), jsonLogSeq.result().toIterator ) val fileStatusSeq = Seq( DeltaSharingLogFileStatus(path = jsonFilePath, size = jsonLogSize, modificationTime = 0L) ) DeltaSharingUtils.overrideIteratorBlock[DeltaSharingLogFileStatus]( getDeltaSharingLogBlockId(deltaLogPath), fileStatusSeq.toIterator ) logInfo( s"It takes ${(System.currentTimeMillis() - startTime) / 1000.0}s to construct delta" + s" log for $customTablePath with ${jsonLogSize} bytes for ${idToUrl.toMap.size} urls." ) ConstructedDeltaLogMetadata( idToUrl = idToUrl.toMap, minUrlExpirationTimestamp = minUrlExpirationTimestamp, numFileActionsInMinVersionOpt = None, minVersion = 0, maxVersion = 0 ) } // Create a delta log directory with protocol and metadata at version 0. // Used by DeltaSharingSource to initialize a DeltaLog class, which is then used to initialize // a DeltaSource class, also the metadata id will be used for schemaTrackingLocation. // There are no data files in the delta log because the DeltaSource class is initialized before // any rpcs to the delta sharing server, so no data files are available yet. def constructDeltaLogWithMetadataAtVersionZero( customTablePath: String, deltaSharingTableMetadata: DeltaSharingTableMetadata): Unit = { val encodedTablePath = DeltaSharingLogFileSystem.encode(customTablePath) val deltaLogPath = s"${encodedTablePath.toString}/_delta_log" // Always use 0.json for snapshot queries. val jsonLogStr = deltaSharingTableMetadata.protocol.deltaProtocol.json + "\n" + deltaSharingTableMetadata.metadata.deltaMetadata.json + "\n" val jsonFilePath = FileNames.unsafeDeltaFile(new Path(deltaLogPath), 0).toString DeltaSharingUtils.overrideIteratorBlock[String]( getDeltaSharingLogBlockId(jsonFilePath), Seq(jsonLogStr).toIterator ) val fileStatusSeq = Seq( DeltaSharingLogFileStatus( path = jsonFilePath, size = jsonLogStr.getBytes(StandardCharsets.UTF_8).length, modificationTime = 0L ) ) DeltaSharingUtils.overrideIteratorBlock[DeltaSharingLogFileStatus]( getDeltaSharingLogBlockId(deltaLogPath), fileStatusSeq.toIterator ) } } /** * A ByteArrayInputStream that implements interfaces required by FSDataInputStream, which is the * return type of DeltaSharingLogFileSystem.open. It will convert the string content as array of * bytes and allow caller to read data out of it. * The string content are list of json serializations of delta actions in a json delta log file. */ private[sharing] class SeekableByteArrayInputStream(bytes: Array[Byte]) extends ByteArrayInputStream(bytes) with Seekable with PositionedReadable { assert(available == bytes.length) override def seek(pos: Long): Unit = { if (mark != 0) { throw new IllegalStateException("Cannot seek if mark is set") } reset() skip(pos) } override def seekToNewSource(pos: Long): Boolean = { false // there aren't multiple sources available } override def getPos(): Long = { bytes.length - available } override def read(buffer: Array[Byte], offset: Int, length: Int): Int = { super.read(buffer, offset, length) } override def read(pos: Long, buffer: Array[Byte], offset: Int, length: Int): Int = { if (pos >= bytes.length) { return -1 } val readSize = math.min(length, bytes.length - pos).toInt System.arraycopy(bytes, pos.toInt, buffer, offset, readSize) readSize } override def readFully(pos: Long, buffer: Array[Byte], offset: Int, length: Int): Unit = { System.arraycopy(bytes, pos.toInt, buffer, offset, length) } override def readFully(pos: Long, buffer: Array[Byte]): Unit = { System.arraycopy(bytes, pos.toInt, buffer, 0, buffer.length) } } case class DeltaSharingLogFileStatus(path: String, size: Long, modificationTime: Long) ================================================ FILE: sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark import java.nio.charset.StandardCharsets.UTF_8 import java.text.SimpleDateFormat import java.util.{TimeZone, UUID} import scala.reflect.ClassTag import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.{DeletionVectorDescriptor, Metadata, Protocol} import com.google.common.hash.Hashing import io.delta.sharing.client.{DeltaSharingClient, DeltaSharingRestClient} import io.delta.sharing.client.model.{DeltaTableFiles, DeltaTableMetadata, Table} import io.delta.sharing.client.util.JsonUtils import org.apache.spark.SparkEnv import org.apache.spark.delta.sharing.TableRefreshResult import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.storage.{BlockId, StorageLevel} object DeltaSharingUtils extends Logging { val STREAMING_SUPPORTED_READER_FEATURES: Seq[String] = Seq( DeletionVectorsTableFeature.name, ColumnMappingTableFeature.name, TimestampNTZTableFeature.name, TypeWideningPreviewTableFeature.name, TypeWideningTableFeature.name, VariantTypePreviewTableFeature.name, VariantTypeTableFeature.name, VariantShreddingPreviewTableFeature.name ) val SUPPORTED_READER_FEATURES: Seq[String] = Seq( DeletionVectorsTableFeature.name, ColumnMappingTableFeature.name, TimestampNTZTableFeature.name, TypeWideningPreviewTableFeature.name, TypeWideningTableFeature.name, VariantTypePreviewTableFeature.name, VariantTypeTableFeature.name, VariantShreddingPreviewTableFeature.name ) // The prefix will be used for block ids of all blocks that store the delta log in BlockManager. // It's used to ensure delta sharing queries don't mess up with blocks with other applications. val DELTA_SHARING_BLOCK_ID_PREFIX = "test_delta-sharing" // Refresher function for CachedTableManager to use. // It takes refreshToken: Option[String] as a parameter and return TableRefreshResult. type RefresherFunction = Option[String] => TableRefreshResult case class DeltaSharingTableMetadata( version: Long, protocol: model.DeltaSharingProtocol, metadata: model.DeltaSharingMetadata ) // A wrapper function for streaming query to get the latest version/protocol/metadata of the // shared table. def getDeltaSharingTableMetadata( client: DeltaSharingClient, table: Table): DeltaSharingTableMetadata = { val deltaTableMetadata = client.getMetadata(table) getDeltaSharingTableMetadata(table, deltaTableMetadata) } def queryDeltaTableMetadata( client: DeltaSharingClient, table: Table, versionAsOf: Option[Long] = None, timestampAsOf: Option[String] = None): DeltaTableMetadata = { val deltaTableMetadata = client.getMetadata(table, versionAsOf, timestampAsOf) logInfo( s"getMetadata returned in ${deltaTableMetadata.respondedFormat} format for table " + s"$table with v_${versionAsOf.map(_.toString).getOrElse("None")} " + s"t_${timestampAsOf.getOrElse("None")} from delta sharing server." ) deltaTableMetadata } /** * parse the protocol and metadata from rpc response for getMetadata. */ def getDeltaSharingTableMetadata( table: Table, deltaTableMetadata: DeltaTableMetadata): DeltaSharingTableMetadata = { var metadataOption: Option[model.DeltaSharingMetadata] = None var protocolOption: Option[model.DeltaSharingProtocol] = None deltaTableMetadata.lines .map( JsonUtils.fromJson[model.DeltaSharingSingleAction](_).unwrap ) .foreach { case m: model.DeltaSharingMetadata => metadataOption = Some(m) case p: model.DeltaSharingProtocol => protocolOption = Some(p) case _ => // ignore other lines } DeltaSharingTableMetadata( version = deltaTableMetadata.version, protocol = protocolOption.getOrElse { throw new IllegalStateException( s"Failed to get Protocol for ${table.toString}, " + s"response from server:${deltaTableMetadata.lines}." ) }, metadata = metadataOption.getOrElse { throw new IllegalStateException( s"Failed to get Metadata for ${table.toString}, " + s"response from server:${deltaTableMetadata.lines}." ) } ) } // Only absolute path (which is pre-signed url) need to be put in IdToUrl mapping. // inline DV should be processed in place, and UUID should throw error. def requiresIdToUrlForDV(deletionVectorOpt: Option[DeletionVectorDescriptor]): Boolean = { deletionVectorOpt.isDefined && deletionVectorOpt.get.storageType == DeletionVectorDescriptor.PATH_DV_MARKER } private def getTableRefreshResult(tableFiles: DeltaTableFiles): TableRefreshResult = { var minUrlExpiration: Option[Long] = None // Collect the id to url mapping from the table files, which includes the file actions // and deletion vectors. val idToUrl = tableFiles.lines .map(JsonUtils.fromJson[model.DeltaSharingSingleAction](_).unwrap) .collect { case fileAction: model.DeltaSharingFileAction => val baseEntries = Seq(fileAction.id -> fileAction.path) val dvEntries = if (requiresIdToUrlForDV(fileAction.getDeletionVectorOpt)) { Seq( fileAction.deletionVectorFileId -> fileAction.getDeletionVectorOpt.get.pathOrInlineDv ) } else { Seq.empty } if (fileAction.expirationTimestamp != null) { minUrlExpiration = minUrlExpiration .filter(_ < fileAction.expirationTimestamp) .orElse(Some(fileAction.expirationTimestamp)) } baseEntries ++ dvEntries } .flatten .toMap TableRefreshResult(idToUrl, minUrlExpiration, tableFiles.refreshToken) } /** * Get the refresher function for a delta sharing table who calls client.getFiles with the * provided parameters. * * @return A refresher function used by the CachedTableManager to refresh urls. */ def getRefresherForGetFiles( client: DeltaSharingClient, table: Table, predicates: Seq[String], limit: Option[Long], versionAsOf: Option[Long], timestampAsOf: Option[String], jsonPredicateHints: Option[String], useRefreshToken: Boolean): RefresherFunction = { refreshTokenOpt => { // If versionAsOf is specified, ignore refresh token (e.g., in streaming queries) val tableFiles = client .getFiles( table = table, predicates = predicates, limit = limit, versionAsOf = versionAsOf, timestampAsOf = timestampAsOf, jsonPredicateHints = jsonPredicateHints, refreshToken = if (useRefreshToken) refreshTokenOpt else None, fileIdHash = None ) getTableRefreshResult(tableFiles) } } /** * Get the refresher function for a delta sharing table who calls client.getCDFFiles with the * provided parameters. * * @return A refresher function used by the CachedTableManager to refresh urls. */ def getRefresherForGetCDFFiles( client: DeltaSharingClient, table: Table, cdfOptions: Map[String, String]): RefresherFunction = { (_: Option[String]) => { val tableFiles = client.getCDFFiles( table = table, cdfOptions = cdfOptions, includeHistoricalMetadata = true, fileIdHash = None ) getTableRefreshResult(tableFiles) } } /** * Get the refresher function for a delta sharing table who calls client.getFiles with the * provided parameters. * * @return A refresher function used by the CachedTableManager to refresh urls. */ def getRefresherForGetFilesWithStartingVersion( client: DeltaSharingClient, table: Table, startingVersion: Long, endingVersion: Option[Long]): RefresherFunction = { (_: Option[String]) => { val tableFiles = client .getFiles( table = table, startingVersion = startingVersion, endingVersion = endingVersion, fileIdHash = None ) getTableRefreshResult(tableFiles) } } def overrideSingleBlock[T: ClassTag](blockId: BlockId, value: T): Unit = { assert( blockId.name.startsWith(DELTA_SHARING_BLOCK_ID_PREFIX), s"invalid delta sharing log block id: $blockId" ) removeBlockForJsonLogIfExists(blockId) SparkEnv.get.blockManager.putSingle[T]( blockId = blockId, value = value, level = StorageLevel.MEMORY_AND_DISK_SER, tellMaster = true ) } def overrideIteratorBlock[T: ClassTag](blockId: BlockId, values: Iterator[T]): Unit = { assert( blockId.name.startsWith(DELTA_SHARING_BLOCK_ID_PREFIX), s"invalid delta sharing log block id: $blockId" ) removeBlockForJsonLogIfExists(blockId) SparkEnv.get.blockManager.putIterator[T]( blockId = blockId, values = values, level = StorageLevel.MEMORY_AND_DISK_SER, tellMaster = true ) } // A helper function used by DeltaSharingSource and DeltaSharingDataSource to get // SnapshotDescriptor used for delta sharing streaming. def getDeltaLogAndSnapshotDescriptor( spark: SparkSession, deltaSharingTableMetadata: DeltaSharingTableMetadata, customTablePathWithUUIDSuffix: String): (DeltaLog, SnapshotDescriptor) = { // Create a delta log with metadata at version 0. // Used by DeltaSharingSource to initialize a DeltaLog class, which is then used to initialize // a DeltaSource class, also the metadata id will be used for schemaTrackingLocation. DeltaSharingLogFileSystem.constructDeltaLogWithMetadataAtVersionZero( customTablePathWithUUIDSuffix, deltaSharingTableMetadata ) val tablePath = DeltaSharingLogFileSystem.encode(customTablePathWithUUIDSuffix).toString val localDeltaLog = DeltaLog.forTable(spark, tablePath) ( localDeltaLog, new SnapshotDescriptor { val deltaLog: DeltaLog = localDeltaLog val metadata: Metadata = deltaSharingTableMetadata.metadata.deltaMetadata val protocol: Protocol = deltaSharingTableMetadata.protocol.deltaProtocol val version = deltaSharingTableMetadata.version val numOfFilesIfKnown = None val sizeInBytesIfKnown = None } ) } // Get a query hash id based on the query parameters: time travel options and filters. // The id concatenated with table name and used in local DeltaLog and CachedTableManager. // This is to uniquely identify the delta sharing table used twice in the same query but with // different query parameters, so we can differentiate their delta log and entries in the // CachedTableManager. private[sharing] def getQueryParamsHashId( options: DeltaSharingOptions, partitionFiltersString: String, dataFiltersString: String, jsonPredicateHints: String, limitHint: String, version: Long): String = { val fullQueryString = s"${options.versionAsOf}_${options.timestampAsOf}_" + s"${partitionFiltersString}_${dataFiltersString}_${jsonPredicateHints}_${limitHint}_" + s"${version}" Hashing.sha256().hashString(fullQueryString, UTF_8).toString } // Get a query hash id based on the query parameters: cdfOptions. // The id concatenated with table name and used in local DeltaLoc and CachedTableManager. // This is to uniquely identify the delta sharing table used twice in the same query but with // different query parameters, so we can differentiate their delta log and entries in the // CachedTableManager. private[sharing] def getQueryParamsHashId(cdfOptions: Map[String, String]): String = { Hashing.sha256().hashString(cdfOptions.toString, UTF_8).toString } // Concatenate table path with an id as a suffix, to uniquely identify a delta sharing table and // its corresponding delta log in a query. private[sharing] def getTablePathWithIdSuffix(customTablePath: String, id: String): String = { s"${customTablePath}_${id}" } // Get a unique string composed of a formatted timestamp and an uuid. // Used as a suffix for the table name and its delta log path of a delta sharing table in a // streaming job, to avoid overwriting the delta log from multiple references of the same delta // sharing table in one streaming job. private[sharing] def getFormattedTimestampWithUUID(): String = { val dateFormat = new SimpleDateFormat("yyyyMMdd_HHmmss") dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")) val formattedDateTime = dateFormat.format(System.currentTimeMillis()) val uuid = UUID.randomUUID().toString().split('-').head s"${formattedDateTime}_${uuid}" } private def removeBlockForJsonLogIfExists(blockId: BlockId): Unit = { val blockManager = SparkEnv.get.blockManager blockManager.getMatchingBlockIds(_.name == blockId.name).foreach { b => logWarning(s"Found and removing existing block for $blockId.") blockManager.removeBlock(b) } } // This is a base64 encoded string of the content of an empty delta checkpoint file. // Will be used to fake a checkpoint file in the locally constructed delta log for cdf and // streaming queries. val FAKE_CHECKPOINT_FILE_BASE64_ENCODED_STRING = """ UEFSMRUAFQ4VEhX63MfpAxwVBBUAFQYVCAAABxgDAAAAAwAAFQAVDhUSFfrcx+kDHBUEFQAVBhUIAAAHGAMAAAADAAAVABUOFRIV +tzH6QMcFQQVABUGFQgAAAcYAwAAAAMAABUAFQ4VEhX63MfpAxwVBBUAFQYVCAAABxgDAAAAAwAAFQAVGhUeFdWf39gCHBUEFQAV BhUGAAANMAIAAAADAAMAAAADAAAVABUcFSAV7J+l5AIcFQQVABUGFQYAAA40AgAAAAMABAAAAAMAAAAVABUOFRIV+tzH6QMcFQQV ABUGFQgAAAcYAwAAAAMAABUAFQ4VEhX63MfpAxwVBBUAFQYVCAAABxgDAAAAAwAAFQAVDhUSFfrcx+kDHBUEFQAVBhUIAAAHGAMA AAADAAAVABUaFR4V1Z/f2AIcFQQVABUGFQYAAA0wAgAAAAMAAwAAAAMAABUAFRwVIBXsn6XkAhwVBBUAFQYVBgAADjQCAAAAAwAE AAAAAwAAABUAFQ4VEhX63MfpAxwVBBUAFQYVCAAABxgDAAAAAwAAFQAVDhUSFfrcx+kDHBUEFQAVBhUIAAAHGAMAAAADAAAVABUO FRIV+tzH6QMcFQQVABUGFQgAAAcYAwAAAAMAABUAFQ4VEhX63MfpAxwVBBUAFQYVCAAABxgDAAAAAwAAFQAVDhUSFfrcx+kDHBUE FQAVBhUIAAAHGAMAAAADAAAVABUOFRIV+tzH6QMcFQQVABUGFQgAAAcYAwAAAAMAABUAFQ4VEhX63MfpAxwVBBUAFQYVCAAABxgD AAAAAwAAFQAVDhUSFfrcx+kDHBUEFQAVBhUIAAAHGAMAAAADAAAVABUOFRIV+tzH6QMcFQQVABUGFQgAAAcYAwAAAAMAABUAFQ4V EhX63MfpAxwVBBUAFQYVCAAABxgDAAAAAwAAFQAVDhUSFfrcx+kDHBUEFQAVBhUIAAAHGAMAAAADAAAVABUOFRIV+tzH6QMcFQQV ABUGFQgAAAcYAwAAAAMAABUAFQ4VEhX63MfpAxwVBBUAFQYVCAAABxgDAAAAAwAAFQAVDhUSFfrcx+kDHBUEFQAVBhUIAAAHGAMA AAADAAAVABUaFR4V1Z/f2AIcFQQVABUGFQYAAA0wAgAAAAMAAwAAAAMAABUAFRwVIBXsn6XkAhwVBBUAFQYVBgAADjQCAAAAAwAE AAAAAwAAABUAFQ4VEhX63MfpAxwVBBUAFQYVCAAABxgDAAAAAwAAFQAVDhUSFfrcx+kDHBUEFQAVBhUIAAAHGAMAAAADAAAVABUO FRIV+tzH6QMcFQQVABUGFQgAAAcYAwAAAAMAABUAFQ4VEhX63MfpAxwVBBUAFQYVCAAABxgDAAAAAwAAFQAVDhUSFfrcx+kDHBUE FQAVBhUIAAAHGAMAAAADAAAVABUOFRIV+tzH6QMcFQQVABUGFQgAAAcYAwAAAAMAABUAFQ4VEhX63MfpAxwVBBUAFQYVCAAABxgD AAAAAwAAFQAVDhUSFfrcx+kDHBUEFQAVBhUIAAAHGAMAAAADAAAVABUOFRIV+tzH6QMcFQQVABUGFQgAAAcYAwAAAAMAABUAFSIV JhWRlf/uBxwVBBUAFQYVCAAAEUADAAAAAwgABgAAAHRlc3RJZBUAFQ4VEhXyyKGvDxwVBBUAFQYVCAAABxgDAAAAAwQAFQAVDhUS FfLIoa8PHBUEFQAVBhUIAAAHGAMAAAADBAAVABUkFSgV3dDvmgccFQQVABUGFQgAABJEAwAAAAMMAAcAAABwYXJxdWV0FQAVHBUg FfzUikccFQQVABUGFQYAAA40AgAAAAMABAAAAAMYAAAVABUcFSAV/NSKRxwVBBUAFQYVBgAADjQCAAAAAwAEAAAAAxgAABUAFQ4V EhXyyKGvDxwVBBUAFQYVCAAABxgDAAAAAwQAFQAVHBUgFYySkKYBHBUEFQAVBhUGAAAONAIAAAADAAQAAAADEAAAFQAVGhUeFbrI 7KoEHBUEFQAVBhUGAAANMAIAAAADAAMAAAADCAAVABUcFSAVjJKQpgEcFQQVABUGFQYAAA40AgAAAAMABAAAAAMQAAAVABUOFRIV 8sihrw8cFQQVABUGFQgAAAcYAwAAAAMEABUAFRYVGhXVxIjAChwVBBUAFQYVCAAACygDAAAAAwIAAQAAABUAFRYVGhWJ+6XrCBwV BBUAFQYVCAAACygDAAAAAwIAAgAAABUAFRwVIBWCt7b4AhwVBBUAFQYVBgAADjQCAAAAAwAEAAAAAwEAABUAFRwVIBWCt7b4AhwV BBUAFQYVBgAADjQCAAAAAwAEAAAAAwEAABUAFQ4VEhX63MfpAxwVBBUAFQYVCAAABxgDAAAAAwAAFQAVDhUSFfrcx+kDHBUEFQAV BhUIAAAHGAMAAAADAAAVABUOFRIV+tzH6QMcFQQVABUGFQgAAAcYAwAAAAMAABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYE ABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYE ABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYE ABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYE ABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYE ABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYE ABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYE ABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRAhkYBnRlc3RJZBkY BnRlc3RJZBUCGRYCABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRAhkYB3BhcnF1ZXQZGAdwYXJxdWV0FQIZFgIAGREB GRgAGRgAFQIZFgQAGREBGRgAGRgAFQIZFgQAGREBGRgAGRgAFQIZFgQAGREBGRgAGRgAFQIZFgQAGREBGRgAGRgAFQIZFgQAGREB GRgAGRgAFQIZFgQAGREBGRgAGRgAFQIZFgQAGRECGRgEAQAAABkYBAEAAAAVAhkWAgAZEQIZGAQCAAAAGRgEAgAAABUCGRYCABkR ARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkRARkYABkYABUCGRYEABkc FggVQBYAAAAZHBZIFUAWAAAAGRwWiAEVQBYAAAAZHBbIARVAFgAAABkcFogCFUwWAAAAGRwW1AIVThYAAAAZHBaiAxVAFgAAABkc FuIDFUAWAAAAGRwWogQVQBYAAAAZHBbiBBVMFgAAABkcFq4FFU4WAAAAGRwW/AUVQBYAAAAZHBa8BhVAFgAAABkcFvwGFUAWAAAA GRwWvAcVQBYAAAAZHBb8BxVAFgAAABkcFrwIFUAWAAAAGRwW/AgVQBYAAAAZHBa8CRVAFgAAABkcFvwJFUAWAAAAGRwWvAoVQBYA AAAZHBb8ChVAFgAAABkcFrwLFUAWAAAAGRwW/AsVQBYAAAAZHBa8DBVAFgAAABkcFvwMFUwWAAAAGRwWyA0VThYAAAAZHBaWDhVA FgAAABkcFtYOFUAWAAAAGRwWlg8VQBYAAAAZHBbWDxVAFgAAABkcFpYQFUAWAAAAGRwW1hAVQBYAAAAZHBaWERVAFgAAABkcFtYR FUAWAAAAGRwWlhIVQBYAAAAZHBbWEhVUFgAAABkcFqoTFUAWAAAAGRwW6hMVQBYAAAAZHBaqFBVWFgAAABkcFoAVFUwWAAAAGRwW zBUVTBYAAAAZHBaYFhVAFgAAABkcFtgWFU4WAAAAGRwWphcVTBYAAAAZHBbyFxVOFgAAABkcFsAYFUAWAAAAGRwWgBkVSBYAAAAZ HBbIGRVIFgAAABkcFpAaFU4WAAAAGRwW3hoVThYAAAAZHBasGxVAFgAAABkcFuwbFUAWAAAAGRwWrBwVQBYAAAAVAhn8UUgMc3Bh cmtfc2NoZW1hFQwANQIYA3R4bhUGABUMJQIYBWFwcElkJQBMHAAAABUEJQIYB3ZlcnNpb24AFQQlAhgLbGFzdFVwZGF0ZWQANQIY A2FkZBUWABUMJQIYBHBhdGglAEwcAAAANQIYD3BhcnRpdGlvblZhbHVlcxUCFQJMLAAAADUEGAlrZXlfdmFsdWUVBAAVDCUAGANr ZXklAEwcAAAAFQwlAhgFdmFsdWUlAEwcAAAAFQQlAhgEc2l6ZQAVBCUCGBBtb2RpZmljYXRpb25UaW1lABUAJQIYCmRhdGFDaGFu Z2UANQIYBHRhZ3MVAhUCTCwAAAA1BBgJa2V5X3ZhbHVlFQQAFQwlABgDa2V5JQBMHAAAABUMJQIYBXZhbHVlJQBMHAAAADUCGA5k ZWxldGlvblZlY3RvchUMABUMJQIYC3N0b3JhZ2VUeXBlJQBMHAAAABUMJQIYDnBhdGhPcklubGluZUR2JQBMHAAAABUCJQIYBm9m ZnNldAAVAiUCGAtzaXplSW5CeXRlcwAVBCUCGAtjYXJkaW5hbGl0eQAVBCUCGAttYXhSb3dJbmRleAAVBCUCGAliYXNlUm93SWQA FQQlAhgXZGVmYXVsdFJvd0NvbW1pdFZlcnNpb24AFQwlAhgFc3RhdHMlAEwcAAAANQIYDHN0YXRzX3BhcnNlZBUCABUEJQIYCm51 bVJlY29yZHMANQIYBnJlbW92ZRUSABUMJQIYBHBhdGglAEwcAAAAFQQlAhgRZGVsZXRpb25UaW1lc3RhbXAAFQAlAhgKZGF0YUNo YW5nZQAVACUCGBRleHRlbmRlZEZpbGVNZXRhZGF0YQA1AhgPcGFydGl0aW9uVmFsdWVzFQIVAkwsAAAANQQYCWtleV92YWx1ZRUE ABUMJQAYA2tleSUATBwAAAAVDCUCGAV2YWx1ZSUATBwAAAAVBCUCGARzaXplADUCGA5kZWxldGlvblZlY3RvchUMABUMJQIYC3N0 b3JhZ2VUeXBlJQBMHAAAABUMJQIYDnBhdGhPcklubGluZUR2JQBMHAAAABUCJQIYBm9mZnNldAAVAiUCGAtzaXplSW5CeXRlcwAV BCUCGAtjYXJkaW5hbGl0eQAVBCUCGAttYXhSb3dJbmRleAAVBCUCGAliYXNlUm93SWQAFQQlAhgXZGVmYXVsdFJvd0NvbW1pdFZl cnNpb24ANQIYCG1ldGFEYXRhFRAAFQwlAhgCaWQlAEwcAAAAFQwlAhgEbmFtZSUATBwAAAAVDCUCGAtkZXNjcmlwdGlvbiUATBwA AAA1AhgGZm9ybWF0FQQAFQwlAhgIcHJvdmlkZXIlAEwcAAAANQIYB29wdGlvbnMVAhUCTCwAAAA1BBgJa2V5X3ZhbHVlFQQAFQwl ABgDa2V5JQBMHAAAABUMJQIYBXZhbHVlJQBMHAAAABUMJQIYDHNjaGVtYVN0cmluZyUATBwAAAA1AhgQcGFydGl0aW9uQ29sdW1u cxUCFQZMPAAAADUEGARsaXN0FQIAFQwlAhgHZWxlbWVudCUATBwAAAA1AhgNY29uZmlndXJhdGlvbhUCFQJMLAAAADUEGAlrZXlf dmFsdWUVBAAVDCUAGANrZXklAEwcAAAAFQwlAhgFdmFsdWUlAEwcAAAAFQQlAhgLY3JlYXRlZFRpbWUANQIYCHByb3RvY29sFQgA FQIlAhgQbWluUmVhZGVyVmVyc2lvbgAVAiUCGBBtaW5Xcml0ZXJWZXJzaW9uADUCGA5yZWFkZXJGZWF0dXJlcxUCFQZMPAAAADUE GARsaXN0FQIAFQwlAhgHZWxlbWVudCUATBwAAAA1AhgOd3JpdGVyRmVhdHVyZXMVAhUGTDwAAAA1BBgEbGlzdBUCABUMJQIYB2Vs ZW1lbnQlAEwcAAAANQIYDmRvbWFpbk1ldGFkYXRhFQYAFQwlAhgGZG9tYWluJQBMHAAAABUMJQIYDWNvbmZpZ3VyYXRpb24lAEwc AAAAFQAlAhgHcmVtb3ZlZAAWBBkcGfw2JggcFQwZNQAGCBkoA3R4bgVhcHBJZBUCFgQWPBZAJgg8NgQAGRwVABUAFQIAABaUKhUU FuwcFR4AJkgcFQQZNQAGCBkoA3R4bgd2ZXJzaW9uFQIWBBY8FkAmSDw2BAAZHBUAFQAVAgAAFqgqFRQWih0VHgAmiAEcFQQZNQAG CBkoA3R4bgtsYXN0VXBkYXRlZBUCFgQWPBZAJogBPDYEABkcFQAVABUCAAAWvCoVFhaoHRUeACbIARwVDBk1AAYIGSgDYWRkBHBh dGgVAhYEFjwWQCbIATw2BAAZHBUAFQAVAgAAFtIqFRYWxh0VHgAmiAIcFQwZJQAGGUgDYWRkD3BhcnRpdGlvblZhbHVlcwlrZXlf dmFsdWUDa2V5FQIWBBZIFkwmiAI8NgQAGRwVABUAFQIAABboKhUWFuQdFR4AJtQCHBUMGSUABhlIA2FkZA9wYXJ0aXRpb25WYWx1 ZXMJa2V5X3ZhbHVlBXZhbHVlFQIWBBZKFk4m1AI8NgQAGRwVABUAFQIAABb+KhUWFoIeFR4AJqIDHBUEGTUABggZKANhZGQEc2l6 ZRUCFgQWPBZAJqIDPDYEABkcFQAVABUCAAAWlCsVFhagHhUeACbiAxwVBBk1AAYIGSgDYWRkEG1vZGlmaWNhdGlvblRpbWUVAhYE FjwWQCbiAzw2BAAZHBUAFQAVAgAAFqorFRYWvh4VHgAmogQcFQAZNQAGCBkoA2FkZApkYXRhQ2hhbmdlFQIWBBY8FkAmogQ8NgQA GRwVABUAFQIAABbAKxUWFtweFR4AJuIEHBUMGSUABhlIA2FkZAR0YWdzCWtleV92YWx1ZQNrZXkVAhYEFkgWTCbiBDw2BAAZHBUA FQAVAgAAFtYrFRYW+h4VHgAmrgUcFQwZJQAGGUgDYWRkBHRhZ3MJa2V5X3ZhbHVlBXZhbHVlFQIWBBZKFk4mrgU8NgQAGRwVABUA FQIAABbsKxUWFpgfFR4AJvwFHBUMGTUABggZOANhZGQOZGVsZXRpb25WZWN0b3ILc3RvcmFnZVR5cGUVAhYEFjwWQCb8BTw2BAAZ HBUAFQAVAgAAFoIsFRYWth8VHgAmvAYcFQwZNQAGCBk4A2FkZA5kZWxldGlvblZlY3Rvcg5wYXRoT3JJbmxpbmVEdhUCFgQWPBZA JrwGPDYEABkcFQAVABUCAAAWmCwVFhbUHxUeACb8BhwVAhk1AAYIGTgDYWRkDmRlbGV0aW9uVmVjdG9yBm9mZnNldBUCFgQWPBZA JvwGPDYEABkcFQAVABUCAAAWriwVFhbyHxUeACa8BxwVAhk1AAYIGTgDYWRkDmRlbGV0aW9uVmVjdG9yC3NpemVJbkJ5dGVzFQIW BBY8FkAmvAc8NgQAGRwVABUAFQIAABbELBUWFpAgFR4AJvwHHBUEGTUABggZOANhZGQOZGVsZXRpb25WZWN0b3ILY2FyZGluYWxp dHkVAhYEFjwWQCb8Bzw2BAAZHBUAFQAVAgAAFtosFRYWriAVHgAmvAgcFQQZNQAGCBk4A2FkZA5kZWxldGlvblZlY3RvcgttYXhS b3dJbmRleBUCFgQWPBZAJrwIPDYEABkcFQAVABUCAAAW8CwVFhbMIBUeACb8CBwVBBk1AAYIGSgDYWRkCWJhc2VSb3dJZBUCFgQW PBZAJvwIPDYEABkcFQAVABUCAAAWhi0VFhbqIBUeACa8CRwVBBk1AAYIGSgDYWRkF2RlZmF1bHRSb3dDb21taXRWZXJzaW9uFQIW BBY8FkAmvAk8NgQAGRwVABUAFQIAABacLRUWFoghFR4AJvwJHBUMGTUABggZKANhZGQFc3RhdHMVAhYEFjwWQCb8CTw2BAAZHBUA FQAVAgAAFrItFRYWpiEVHgAmvAocFQQZNQAGCBk4A2FkZAxzdGF0c19wYXJzZWQKbnVtUmVjb3JkcxUCFgQWPBZAJrwKPDYEABkc FQAVABUCAAAWyC0VFhbEIRUeACb8ChwVDBk1AAYIGSgGcmVtb3ZlBHBhdGgVAhYEFjwWQCb8Cjw2BAAZHBUAFQAVAgAAFt4tFRYW 4iEVHgAmvAscFQQZNQAGCBkoBnJlbW92ZRFkZWxldGlvblRpbWVzdGFtcBUCFgQWPBZAJrwLPDYEABkcFQAVABUCAAAW9C0VFhaA IhUeACb8CxwVABk1AAYIGSgGcmVtb3ZlCmRhdGFDaGFuZ2UVAhYEFjwWQCb8Czw2BAAZHBUAFQAVAgAAFoouFRYWniIVHgAmvAwc FQAZNQAGCBkoBnJlbW92ZRRleHRlbmRlZEZpbGVNZXRhZGF0YRUCFgQWPBZAJrwMPDYEABkcFQAVABUCAAAWoC4VFha8IhUeACb8 DBwVDBklAAYZSAZyZW1vdmUPcGFydGl0aW9uVmFsdWVzCWtleV92YWx1ZQNrZXkVAhYEFkgWTCb8DDw2BAAZHBUAFQAVAgAAFrYu FRYW2iIVHgAmyA0cFQwZJQAGGUgGcmVtb3ZlD3BhcnRpdGlvblZhbHVlcwlrZXlfdmFsdWUFdmFsdWUVAhYEFkoWTibIDTw2BAAZ HBUAFQAVAgAAFswuFRYW+CIVHgAmlg4cFQQZNQAGCBkoBnJlbW92ZQRzaXplFQIWBBY8FkAmlg48NgQAGRwVABUAFQIAABbiLhUW FpYjFR4AJtYOHBUMGTUABggZOAZyZW1vdmUOZGVsZXRpb25WZWN0b3ILc3RvcmFnZVR5cGUVAhYEFjwWQCbWDjw2BAAZHBUAFQAV AgAAFvguFRYWtCMVHgAmlg8cFQwZNQAGCBk4BnJlbW92ZQ5kZWxldGlvblZlY3Rvcg5wYXRoT3JJbmxpbmVEdhUCFgQWPBZAJpYP PDYEABkcFQAVABUCAAAWji8VFhbSIxUeACbWDxwVAhk1AAYIGTgGcmVtb3ZlDmRlbGV0aW9uVmVjdG9yBm9mZnNldBUCFgQWPBZA JtYPPDYEABkcFQAVABUCAAAWpC8VFhbwIxUeACaWEBwVAhk1AAYIGTgGcmVtb3ZlDmRlbGV0aW9uVmVjdG9yC3NpemVJbkJ5dGVz FQIWBBY8FkAmlhA8NgQAGRwVABUAFQIAABa6LxUWFo4kFR4AJtYQHBUEGTUABggZOAZyZW1vdmUOZGVsZXRpb25WZWN0b3ILY2Fy ZGluYWxpdHkVAhYEFjwWQCbWEDw2BAAZHBUAFQAVAgAAFtAvFRYWrCQVHgAmlhEcFQQZNQAGCBk4BnJlbW92ZQ5kZWxldGlvblZl Y3RvcgttYXhSb3dJbmRleBUCFgQWPBZAJpYRPDYEABkcFQAVABUCAAAW5i8VFhbKJBUeACbWERwVBBk1AAYIGSgGcmVtb3ZlCWJh c2VSb3dJZBUCFgQWPBZAJtYRPDYEABkcFQAVABUCAAAW/C8VFhboJBUeACaWEhwVBBk1AAYIGSgGcmVtb3ZlF2RlZmF1bHRSb3dD b21taXRWZXJzaW9uFQIWBBY8FkAmlhI8NgQAGRwVABUAFQIAABaSMBUWFoYlFR4AJtYSHBUMGTUABggZKAhtZXRhRGF0YQJpZBUC FgQWUBZUJtYSPBgGdGVzdElkGAZ0ZXN0SWQWAigGdGVzdElkGAZ0ZXN0SWQAGRwVABUAFQIAABaoMBUWFqQlFTYAJqoTHBUMGTUA BggZKAhtZXRhRGF0YQRuYW1lFQIWBBY8FkAmqhM8NgQAGRwVABUAFQIAABa+MBUWFtolFR4AJuoTHBUMGTUABggZKAhtZXRhRGF0 YQtkZXNjcmlwdGlvbhUCFgQWPBZAJuoTPDYEABkcFQAVABUCAAAW1DAVFhb4JRUeACaqFBwVDBk1AAYIGTgIbWV0YURhdGEGZm9y bWF0CHByb3ZpZGVyFQIWBBZSFlYmqhQ8GAdwYXJxdWV0GAdwYXJxdWV0FgIoB3BhcnF1ZXQYB3BhcnF1ZXQAGRwVABUAFQIAABbq MBUWFpYmFToAJoAVHBUMGSUABhlYCG1ldGFEYXRhBmZvcm1hdAdvcHRpb25zCWtleV92YWx1ZQNrZXkVAhYEFkgWTCaAFTw2BAAZ HBUAFQAVAgAAFoAxFRYW0CYVHgAmzBUcFQwZJQAGGVgIbWV0YURhdGEGZm9ybWF0B29wdGlvbnMJa2V5X3ZhbHVlBXZhbHVlFQIW BBZIFkwmzBU8NgQAGRwVABUAFQIAABaWMRUWFu4mFR4AJpgWHBUMGTUABggZKAhtZXRhRGF0YQxzY2hlbWFTdHJpbmcVAhYEFjwW QCaYFjw2BAAZHBUAFQAVAgAAFqwxFRYWjCcVHgAm2BYcFQwZJQAGGUgIbWV0YURhdGEQcGFydGl0aW9uQ29sdW1ucwRsaXN0B2Vs ZW1lbnQVAhYEFkoWTibYFjw2BAAZHBUAFQAVAgAAFsIxFRYWqicVHgAmphccFQwZJQAGGUgIbWV0YURhdGENY29uZmlndXJhdGlv bglrZXlfdmFsdWUDa2V5FQIWBBZIFkwmphc8NgQAGRwVABUAFQIAABbYMRUWFsgnFR4AJvIXHBUMGSUABhlICG1ldGFEYXRhDWNv bmZpZ3VyYXRpb24Ja2V5X3ZhbHVlBXZhbHVlFQIWBBZKFk4m8hc8NgQAGRwVABUAFQIAABbuMRUWFuYnFR4AJsAYHBUEGTUABggZ KAhtZXRhRGF0YQtjcmVhdGVkVGltZRUCFgQWPBZAJsAYPDYEABkcFQAVABUCAAAWhDIVFhaEKBUeACaAGRwVAhk1AAYIGSgIcHJv dG9jb2wQbWluUmVhZGVyVmVyc2lvbhUCFgQWRBZIJoAZPBgEAQAAABgEAQAAABYCKAQBAAAAGAQBAAAAABkcFQAVABUCAAAWmjIV FhaiKBUuACbIGRwVAhk1AAYIGSgIcHJvdG9jb2wQbWluV3JpdGVyVmVyc2lvbhUCFgQWRBZIJsgZPBgEAgAAABgEAgAAABYCKAQC AAAAGAQCAAAAABkcFQAVABUCAAAWsDIVFhbQKBUuACaQGhwVDBklAAYZSAhwcm90b2NvbA5yZWFkZXJGZWF0dXJlcwRsaXN0B2Vs ZW1lbnQVAhYEFkoWTiaQGjw2BAAZHBUAFQAVAgAAFsYyFRYW/igVHgAm3hocFQwZJQAGGUgIcHJvdG9jb2wOd3JpdGVyRmVhdHVy ZXMEbGlzdAdlbGVtZW50FQIWBBZKFk4m3ho8NgQAGRwVABUAFQIAABbcMhUWFpwpFR4AJqwbHBUMGTUABggZKA5kb21haW5NZXRh ZGF0YQZkb21haW4VAhYEFjwWQCasGzw2BAAZHBUAFQAVAgAAFvIyFRYWuikVHgAm7BscFQwZNQAGCBkoDmRvbWFpbk1ldGFkYXRh DWNvbmZpZ3VyYXRpb24VAhYEFjwWQCbsGzw2BAAZHBUAFQAVAgAAFogzFRYW2CkVHgAmrBwcFQAZNQAGCBkoDmRvbWFpbk1ldGFk YXRhB3JlbW92ZWQVAhYEFjwWQCasHDw2BAAZHBUAFQAVAgAAFp4zFRYW9ikVHgAWjBsWBCYIFuQcFAAAGVwYGW9yZy5hcGFjaGUu c3BhcmsudGltZVpvbmUYE0FtZXJpY2EvTG9zX0FuZ2VsZXMAGBxvcmcuYXBhY2hlLnNwYXJrLmxlZ2FjeUlOVDk2GAAAGBhvcmcu YXBhY2hlLnNwYXJrLnZlcnNpb24YBTQuMC4wABgpb3JnLmFwYWNoZS5zcGFyay5zcWwucGFycXVldC5yb3cubWV0YWRhdGEYiyV7 InR5cGUiOiJzdHJ1Y3QiLCJmaWVsZHMiOlt7Im5hbWUiOiJ0eG4iLCJ0eXBlIjp7InR5cGUiOiJzdHJ1Y3QiLCJmaWVsZHMiOlt7 Im5hbWUiOiJhcHBJZCIsInR5cGUiOiJzdHJpbmciLCJudWxsYWJsZSI6dHJ1ZSwibWV0YWRhdGEiOnt9fSx7Im5hbWUiOiJ2ZXJz aW9uIiwidHlwZSI6ImxvbmciLCJudWxsYWJsZSI6dHJ1ZSwibWV0YWRhdGEiOnt9fSx7Im5hbWUiOiJsYXN0VXBkYXRlZCIsInR5 cGUiOiJsb25nIiwibnVsbGFibGUiOnRydWUsIm1ldGFkYXRhIjp7fX1dfSwibnVsbGFibGUiOnRydWUsIm1ldGFkYXRhIjp7fX0s eyJuYW1lIjoiYWRkIiwidHlwZSI6eyJ0eXBlIjoic3RydWN0IiwiZmllbGRzIjpbeyJuYW1lIjoicGF0aCIsInR5cGUiOiJzdHJp bmciLCJudWxsYWJsZSI6dHJ1ZSwibWV0YWRhdGEiOnt9fSx7Im5hbWUiOiJwYXJ0aXRpb25WYWx1ZXMiLCJ0eXBlIjp7InR5cGUi OiJtYXAiLCJrZXlUeXBlIjoic3RyaW5nIiwidmFsdWVUeXBlIjoic3RyaW5nIiwidmFsdWVDb250YWluc051bGwiOnRydWV9LCJu dWxsYWJsZSI6dHJ1ZSwibWV0YWRhdGEiOnt9fSx7Im5hbWUiOiJzaXplIiwidHlwZSI6ImxvbmciLCJudWxsYWJsZSI6dHJ1ZSwi bWV0YWRhdGEiOnt9fSx7Im5hbWUiOiJtb2RpZmljYXRpb25UaW1lIiwidHlwZSI6ImxvbmciLCJudWxsYWJsZSI6dHJ1ZSwibWV0 YWRhdGEiOnt9fSx7Im5hbWUiOiJkYXRhQ2hhbmdlIiwidHlwZSI6ImJvb2xlYW4iLCJudWxsYWJsZSI6dHJ1ZSwibWV0YWRhdGEi Ont9fSx7Im5hbWUiOiJ0YWdzIiwidHlwZSI6eyJ0eXBlIjoibWFwIiwia2V5VHlwZSI6InN0cmluZyIsInZhbHVlVHlwZSI6InN0 cmluZyIsInZhbHVlQ29udGFpbnNOdWxsIjp0cnVlfSwibnVsbGFibGUiOnRydWUsIm1ldGFkYXRhIjp7fX0seyJuYW1lIjoiZGVs ZXRpb25WZWN0b3IiLCJ0eXBlIjp7InR5cGUiOiJzdHJ1Y3QiLCJmaWVsZHMiOlt7Im5hbWUiOiJzdG9yYWdlVHlwZSIsInR5cGUi OiJzdHJpbmciLCJudWxsYWJsZSI6dHJ1ZSwibWV0YWRhdGEiOnt9fSx7Im5hbWUiOiJwYXRoT3JJbmxpbmVEdiIsInR5cGUiOiJz dHJpbmciLCJudWxsYWJsZSI6dHJ1ZSwibWV0YWRhdGEiOnt9fSx7Im5hbWUiOiJvZmZzZXQiLCJ0eXBlIjoiaW50ZWdlciIsIm51 bGxhYmxlIjp0cnVlLCJtZXRhZGF0YSI6e319LHsibmFtZSI6InNpemVJbkJ5dGVzIiwidHlwZSI6ImludGVnZXIiLCJudWxsYWJs ZSI6dHJ1ZSwibWV0YWRhdGEiOnt9fSx7Im5hbWUiOiJjYXJkaW5hbGl0eSIsInR5cGUiOiJsb25nIiwibnVsbGFibGUiOnRydWUs Im1ldGFkYXRhIjp7fX0seyJuYW1lIjoibWF4Um93SW5kZXgiLCJ0eXBlIjoibG9uZyIsIm51bGxhYmxlIjp0cnVlLCJtZXRhZGF0 YSI6e319XX0sIm51bGxhYmxlIjp0cnVlLCJtZXRhZGF0YSI6e319LHsibmFtZSI6ImJhc2VSb3dJZCIsInR5cGUiOiJsb25nIiwi bnVsbGFibGUiOnRydWUsIm1ldGFkYXRhIjp7fX0seyJuYW1lIjoiZGVmYXVsdFJvd0NvbW1pdFZlcnNpb24iLCJ0eXBlIjoibG9u ZyIsIm51bGxhYmxlIjp0cnVlLCJtZXRhZGF0YSI6e319LHsibmFtZSI6InN0YXRzIiwidHlwZSI6InN0cmluZyIsIm51bGxhYmxl Ijp0cnVlLCJtZXRhZGF0YSI6e319LHsibmFtZSI6InN0YXRzX3BhcnNlZCIsInR5cGUiOnsidHlwZSI6InN0cnVjdCIsImZpZWxk cyI6W3sibmFtZSI6Im51bVJlY29yZHMiLCJ0eXBlIjoibG9uZyIsIm51bGxhYmxlIjp0cnVlLCJtZXRhZGF0YSI6e319XX0sIm51 bGxhYmxlIjp0cnVlLCJtZXRhZGF0YSI6e319XX0sIm51bGxhYmxlIjp0cnVlLCJtZXRhZGF0YSI6e319LHsibmFtZSI6InJlbW92 ZSIsInR5cGUiOnsidHlwZSI6InN0cnVjdCIsImZpZWxkcyI6W3sibmFtZSI6InBhdGgiLCJ0eXBlIjoic3RyaW5nIiwibnVsbGFi bGUiOnRydWUsIm1ldGFkYXRhIjp7fX0seyJuYW1lIjoiZGVsZXRpb25UaW1lc3RhbXAiLCJ0eXBlIjoibG9uZyIsIm51bGxhYmxl Ijp0cnVlLCJtZXRhZGF0YSI6e319LHsibmFtZSI6ImRhdGFDaGFuZ2UiLCJ0eXBlIjoiYm9vbGVhbiIsIm51bGxhYmxlIjp0cnVl LCJtZXRhZGF0YSI6e319LHsibmFtZSI6ImV4dGVuZGVkRmlsZU1ldGFkYXRhIiwidHlwZSI6ImJvb2xlYW4iLCJudWxsYWJsZSI6 dHJ1ZSwibWV0YWRhdGEiOnt9fSx7Im5hbWUiOiJwYXJ0aXRpb25WYWx1ZXMiLCJ0eXBlIjp7InR5cGUiOiJtYXAiLCJrZXlUeXBl Ijoic3RyaW5nIiwidmFsdWVUeXBlIjoic3RyaW5nIiwidmFsdWVDb250YWluc051bGwiOnRydWV9LCJudWxsYWJsZSI6dHJ1ZSwi bWV0YWRhdGEiOnt9fSx7Im5hbWUiOiJzaXplIiwidHlwZSI6ImxvbmciLCJudWxsYWJsZSI6dHJ1ZSwibWV0YWRhdGEiOnt9fSx7 Im5hbWUiOiJkZWxldGlvblZlY3RvciIsInR5cGUiOnsidHlwZSI6InN0cnVjdCIsImZpZWxkcyI6W3sibmFtZSI6InN0b3JhZ2VU eXBlIiwidHlwZSI6InN0cmluZyIsIm51bGxhYmxlIjp0cnVlLCJtZXRhZGF0YSI6e319LHsibmFtZSI6InBhdGhPcklubGluZUR2 IiwidHlwZSI6InN0cmluZyIsIm51bGxhYmxlIjp0cnVlLCJtZXRhZGF0YSI6e319LHsibmFtZSI6Im9mZnNldCIsInR5cGUiOiJp bnRlZ2VyIiwibnVsbGFibGUiOnRydWUsIm1ldGFkYXRhIjp7fX0seyJuYW1lIjoic2l6ZUluQnl0ZXMiLCJ0eXBlIjoiaW50ZWdl ciIsIm51bGxhYmxlIjp0cnVlLCJtZXRhZGF0YSI6e319LHsibmFtZSI6ImNhcmRpbmFsaXR5IiwidHlwZSI6ImxvbmciLCJudWxs YWJsZSI6dHJ1ZSwibWV0YWRhdGEiOnt9fSx7Im5hbWUiOiJtYXhSb3dJbmRleCIsInR5cGUiOiJsb25nIiwibnVsbGFibGUiOnRy dWUsIm1ldGFkYXRhIjp7fX1dfSwibnVsbGFibGUiOnRydWUsIm1ldGFkYXRhIjp7fX0seyJuYW1lIjoiYmFzZVJvd0lkIiwidHlw ZSI6ImxvbmciLCJudWxsYWJsZSI6dHJ1ZSwibWV0YWRhdGEiOnt9fSx7Im5hbWUiOiJkZWZhdWx0Um93Q29tbWl0VmVyc2lvbiIs InR5cGUiOiJsb25nIiwibnVsbGFibGUiOnRydWUsIm1ldGFkYXRhIjp7fX1dfSwibnVsbGFibGUiOnRydWUsIm1ldGFkYXRhIjp7 fX0seyJuYW1lIjoibWV0YURhdGEiLCJ0eXBlIjp7InR5cGUiOiJzdHJ1Y3QiLCJmaWVsZHMiOlt7Im5hbWUiOiJpZCIsInR5cGUi OiJzdHJpbmciLCJudWxsYWJsZSI6dHJ1ZSwibWV0YWRhdGEiOnt9fSx7Im5hbWUiOiJuYW1lIiwidHlwZSI6InN0cmluZyIsIm51 bGxhYmxlIjp0cnVlLCJtZXRhZGF0YSI6e319LHsibmFtZSI6ImRlc2NyaXB0aW9uIiwidHlwZSI6InN0cmluZyIsIm51bGxhYmxl Ijp0cnVlLCJtZXRhZGF0YSI6e319LHsibmFtZSI6ImZvcm1hdCIsInR5cGUiOnsidHlwZSI6InN0cnVjdCIsImZpZWxkcyI6W3si bmFtZSI6InByb3ZpZGVyIiwidHlwZSI6InN0cmluZyIsIm51bGxhYmxlIjp0cnVlLCJtZXRhZGF0YSI6e319LHsibmFtZSI6Im9w dGlvbnMiLCJ0eXBlIjp7InR5cGUiOiJtYXAiLCJrZXlUeXBlIjoic3RyaW5nIiwidmFsdWVUeXBlIjoic3RyaW5nIiwidmFsdWVD b250YWluc051bGwiOnRydWV9LCJudWxsYWJsZSI6dHJ1ZSwibWV0YWRhdGEiOnt9fV19LCJudWxsYWJsZSI6dHJ1ZSwibWV0YWRh dGEiOnt9fSx7Im5hbWUiOiJzY2hlbWFTdHJpbmciLCJ0eXBlIjoic3RyaW5nIiwibnVsbGFibGUiOnRydWUsIm1ldGFkYXRhIjp7 fX0seyJuYW1lIjoicGFydGl0aW9uQ29sdW1ucyIsInR5cGUiOnsidHlwZSI6ImFycmF5IiwiZWxlbWVudFR5cGUiOiJzdHJpbmci LCJjb250YWluc051bGwiOnRydWV9LCJudWxsYWJsZSI6dHJ1ZSwibWV0YWRhdGEiOnt9fSx7Im5hbWUiOiJjb25maWd1cmF0aW9u IiwidHlwZSI6eyJ0eXBlIjoibWFwIiwia2V5VHlwZSI6InN0cmluZyIsInZhbHVlVHlwZSI6InN0cmluZyIsInZhbHVlQ29udGFp bnNOdWxsIjp0cnVlfSwibnVsbGFibGUiOnRydWUsIm1ldGFkYXRhIjp7fX0seyJuYW1lIjoiY3JlYXRlZFRpbWUiLCJ0eXBlIjoi bG9uZyIsIm51bGxhYmxlIjp0cnVlLCJtZXRhZGF0YSI6e319XX0sIm51bGxhYmxlIjp0cnVlLCJtZXRhZGF0YSI6e319LHsibmFt ZSI6InByb3RvY29sIiwidHlwZSI6eyJ0eXBlIjoic3RydWN0IiwiZmllbGRzIjpbeyJuYW1lIjoibWluUmVhZGVyVmVyc2lvbiIs InR5cGUiOiJpbnRlZ2VyIiwibnVsbGFibGUiOnRydWUsIm1ldGFkYXRhIjp7fX0seyJuYW1lIjoibWluV3JpdGVyVmVyc2lvbiIs InR5cGUiOiJpbnRlZ2VyIiwibnVsbGFibGUiOnRydWUsIm1ldGFkYXRhIjp7fX0seyJuYW1lIjoicmVhZGVyRmVhdHVyZXMiLCJ0 eXBlIjp7InR5cGUiOiJhcnJheSIsImVsZW1lbnRUeXBlIjoic3RyaW5nIiwiY29udGFpbnNOdWxsIjp0cnVlfSwibnVsbGFibGUi OnRydWUsIm1ldGFkYXRhIjp7fX0seyJuYW1lIjoid3JpdGVyRmVhdHVyZXMiLCJ0eXBlIjp7InR5cGUiOiJhcnJheSIsImVsZW1l bnRUeXBlIjoic3RyaW5nIiwiY29udGFpbnNOdWxsIjp0cnVlfSwibnVsbGFibGUiOnRydWUsIm1ldGFkYXRhIjp7fX1dfSwibnVs bGFibGUiOnRydWUsIm1ldGFkYXRhIjp7fX0seyJuYW1lIjoiZG9tYWluTWV0YWRhdGEiLCJ0eXBlIjp7InR5cGUiOiJzdHJ1Y3Qi LCJmaWVsZHMiOlt7Im5hbWUiOiJkb21haW4iLCJ0eXBlIjoic3RyaW5nIiwibnVsbGFibGUiOnRydWUsIm1ldGFkYXRhIjp7fX0s eyJuYW1lIjoiY29uZmlndXJhdGlvbiIsInR5cGUiOiJzdHJpbmciLCJudWxsYWJsZSI6dHJ1ZSwibWV0YWRhdGEiOnt9fSx7Im5h bWUiOiJyZW1vdmVkIiwidHlwZSI6ImJvb2xlYW4iLCJudWxsYWJsZSI6dHJ1ZSwibWV0YWRhdGEiOnt9fV19LCJudWxsYWJsZSI6 dHJ1ZSwibWV0YWRhdGEiOnt9fV19ABgfb3JnLmFwYWNoZS5zcGFyay5sZWdhY3lEYXRlVGltZRgAABhacGFycXVldC1tciB2ZXJz aW9uIDEuMTIuMy1kYXRhYnJpY2tzLTAwMDIgKGJ1aWxkIDI0ODRhOTVkYmUxNmEwMDIzZTNlYjI5YzIwMWY5OWZmOWVhNzcxZWUp Gfw2HAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAA HAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAAHAAA HAAAHAAAHAAAHAAAHAAAAJUqAABQQVIx""".stripMargin.replaceAll("\n", "") // Pre-prepare the byte array for (minVersion-1).checkpoint.parquet. val FAKE_CHECKPOINT_BYTE_ARRAY = { java.util.Base64.getDecoder.decode(FAKE_CHECKPOINT_FILE_BASE64_ENCODED_STRING) } } ================================================ FILE: sharing/src/main/scala/io/delta/sharing/spark/PrepareDeltaSharingScan.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark import org.apache.spark.sql.delta.{DeltaTableUtils => SqlDeltaTableUtils} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.{PreparedDeltaFileIndex, PrepareDeltaScan} import io.delta.sharing.client.util.ConfUtils import io.delta.sharing.spark.DeltaSharingFileIndex import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans.logical._ /** * Before query planning, we prepare any scans over delta sharing tables by pushing * any filters or limits to delta sharing server through RPC, allowing us to return only needed * files and gather more accurate statistics for CBO and metering. */ class PrepareDeltaSharingScan(override val spark: SparkSession) extends PrepareDeltaScan(spark) { /** * Prepares delta sharing scans sequentially. */ override protected def prepareDeltaScan(plan: LogicalPlan): LogicalPlan = { transformWithSubqueries(plan) { case scan @ DeltaSharingTableScan(_, filters, dsFileIndex, limit, _) => val partitionCols = dsFileIndex.partitionColumns val (partitionFilters, dataFilters) = filters.partition { e => SqlDeltaTableUtils.isPredicatePartitionColumnsOnly(e, partitionCols, spark) } logInfo(s"Classified filters: partition: $partitionFilters, data: $dataFilters, " + s"limit: $limit.") val deltaLog = dsFileIndex.fetchFilesAndConstructDeltaLog( partitionFilters, dataFilters, limit.map(_.toLong) ) val snapshot = deltaLog.snapshot val deltaScan = limit match { case Some(limit) => snapshot.filesForScan(limit, filters) case _ => snapshot.filesForScan(filters) } val preparedIndex = PreparedDeltaFileIndex( spark, deltaLog, deltaLog.dataPath, catalogTableOpt = None, preparedScan = deltaScan, versionScanned = Some(snapshot.version) ) SqlDeltaTableUtils.replaceFileIndex(scan, preparedIndex) } } // Just return the plan if statistics based skipping is off. // It will fall back to just partition pruning at planning time. // When data skipping is disabled, just convert Delta sharing scans to normal tahoe scans. // NOTE: File skipping is only disabled on the client, so we still pass filters to the server. override protected def prepareDeltaScanWithoutFileSkipping(plan: LogicalPlan): LogicalPlan = { plan.transformDown { case scan@DeltaSharingTableScan(_, filters, sharingIndex, _, _) => val partitionCols = sharingIndex.partitionColumns val (partitionFilters, dataFilters) = filters.partition { e => SqlDeltaTableUtils.isPredicatePartitionColumnsOnly(e, partitionCols, spark) } logInfo(s"Classified filters: partition: $partitionFilters, data: $dataFilters") val fileIndex = sharingIndex.asTahoeFileIndex(partitionFilters, dataFilters) SqlDeltaTableUtils.replaceFileIndex(scan, fileIndex) } } // TODO: Support metadata-only query optimization! override def optimizeQueryWithMetadata(plan: LogicalPlan): LogicalPlan = plan /** * This is an extractor object. See https://docs.scala-lang.org/tour/extractor-objects.html. */ object DeltaSharingTableScan extends DeltaTableScan[DeltaSharingFileIndex] { // Since delta library is used to read the data on constructed delta log, this should also // consider the spark config for delta limit pushdown. override def limitPushdownEnabled(plan: LogicalPlan): Boolean = ConfUtils.limitPushdownEnabled(plan.conf) && (spark.conf.get(DeltaSQLConf.DELTA_LIMIT_PUSHDOWN_ENABLED.key) == "true") override def getPartitionColumns(fileIndex: DeltaSharingFileIndex): Seq[String] = fileIndex.partitionColumns override def getPartitionFilters(fileIndex: DeltaSharingFileIndex): Seq[Expression] = Seq.empty[Expression] } } ================================================ FILE: sharing/src/main/scala/io/delta/sharing/spark/model.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark.model import java.net.URLEncoder import org.apache.spark.sql.delta.actions.{ AddCDCFile, AddFile, DeletionVectorDescriptor, FileAction, Metadata, Protocol, RemoveFile, SingleAction } import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore import org.apache.spark.sql.delta.util.JsonUtils import com.fasterxml.jackson.annotation._ import com.fasterxml.jackson.annotation.JsonInclude.Include import io.delta.sharing.client.DeltaSharingFileSystem import org.apache.spark.sql.types.{DataType, StructType} // Represents a single action in the response of a Delta Sharing rpc. sealed trait DeltaSharingAction { def wrap: DeltaSharingSingleAction def json: String = JsonUtils.toJson(wrap) } /** A serialization helper to create a common action envelope, for delta sharing actions in the * response of a rpc. */ case class DeltaSharingSingleAction( protocol: DeltaSharingProtocol = null, metaData: DeltaSharingMetadata = null, file: DeltaSharingFileAction = null) { def unwrap: DeltaSharingAction = { if (file != null) { file } else if (metaData != null) { metaData } else if (protocol != null) { protocol } else { null } } } /** * The delta sharing protocol from the response of a rpc. It only wraps a delta protocol now, but * can be extended with additional delta sharing fields if needed later. */ case class DeltaSharingProtocol(deltaProtocol: Protocol) extends DeltaSharingAction { override def wrap: DeltaSharingSingleAction = DeltaSharingSingleAction(protocol = this) } /** * The delta sharing metadata from the response of a rpc. * It wraps a delta metadata, and adds three delta sharing fields: * - version: the version of the metadata, used to generate faked delta log file on the client * side. * - size: the estimated size of the table at the version, used to estimate query size. * - numFiles: the number of files of the table at the version, used to estimate query size. */ case class DeltaSharingMetadata( version: java.lang.Long = null, size: java.lang.Long = null, numFiles: java.lang.Long = null, deltaMetadata: Metadata) extends DeltaSharingAction { /** Returns the schema as a [[StructType]] */ @JsonIgnore lazy val schema: StructType = deltaMetadata.schema /** Returns the partitionSchema as a [[StructType]] */ @JsonIgnore lazy val partitionSchema: StructType = deltaMetadata.partitionSchema override def wrap: DeltaSharingSingleAction = DeltaSharingSingleAction(metaData = this) } /** * DeltaResponseFileAction used in delta sharing protocol. It wraps a delta single action, * and adds 4 delta sharing related fields: id/version/timestamp/expirationTimestamp. * - id: used to uniquely identify a file, and in idToUrl mapping for executor to get * presigned url. * - version/timestamp: the version and timestamp of the commit, used to generate faked delta * log file on the client side. * - expirationTimestamp: indicate when the presigned url is going to expire and need a * refresh. * The server is responsible to redact sensitive fields such as "tags" before returning. */ case class DeltaSharingFileAction( id: String, version: java.lang.Long = null, timestamp: java.lang.Long = null, expirationTimestamp: java.lang.Long = null, deletionVectorFileId: String = null, deltaSingleAction: SingleAction) extends DeltaSharingAction { lazy val path: String = { deltaSingleAction.unwrap match { case file: FileAction => file.path case action => throw new IllegalStateException( s"unexpected action in delta sharing " + s"response: ${action.json}" ) } } lazy val size: Long = { deltaSingleAction.unwrap match { case add: AddFile => add.size case cdc: AddCDCFile => cdc.size case remove: RemoveFile => remove.size.getOrElse { throw new IllegalStateException( "size is missing for the remove file returned from server" + s", which is required by delta sharing client, response:${remove.json}." ) } case action => throw new IllegalStateException( s"unexpected action in delta sharing " + s"response: ${action.json}" ) } } def getDeletionVectorOpt: Option[DeletionVectorDescriptor] = { deltaSingleAction.unwrap match { case file: FileAction => Option.apply(file.deletionVector) case _ => None } } def getDeletionVectorDeltaSharingPath(tablePath: String): String = { getDeletionVectorOpt.map { deletionVector => // Adding offset to dvFileSize so it can load all needed bytes in memory, // starting from the beginning of the file instead of the `offset`. // There could be other DVs beyond this length in the file, but not needed by this DV. val dvFileSize = DeletionVectorStore.getTotalSizeOfDVFieldsInFile( deletionVector.sizeInBytes ) + deletionVector.offset.getOrElse(0) // This path is going to be put in the delta log file and processed by delta code, where // absolutePath() is applied to the path in all places, such as TahoeFileIndex and // DeletionVectorDescriptor, and in absolutePath, URI will apply a decode of the path. // Additional encoding on the tablePath and table id to allow the path still able to be // processed by DeltaSharingFileSystem after URI decodes it. DeltaSharingFileSystem .DeltaSharingPath( URLEncoder.encode(tablePath, "UTF-8"), URLEncoder.encode(deletionVectorFileId, "UTF-8"), dvFileSize ) .toPath .toString }.orNull } /** * A helper function to get the delta sharing path for this file action to put in delta log, * in the format below: * ``` * delta-sharing:///// * ``` * * This is to make a unique and unchanged path for each file action, which will be mapped to * pre-signed url by DeltaSharingFileSystem.open(). size is needed to know how much bytes to read * from the FSDataInputStream. */ def getDeltaSharingPath(tablePath: String): String = { // This path is going to be put in the delta log file and processed by delta code, where // absolutePath() is applied to the path in all places, such as TahoeFileIndex and // DeletionVectorDescriptor, and in absolutePath, URI will apply a decode of the path. // Additional encoding on the tablePath and table id to allow the path still able to be // processed by DeltaSharingFileSystem after URI decodes it. DeltaSharingFileSystem .DeltaSharingPath( URLEncoder.encode(tablePath, "UTF-8"), URLEncoder.encode(id, "UTF-8"), size ) .toPath .toString } override def wrap: DeltaSharingSingleAction = DeltaSharingSingleAction(file = this) } ================================================ FILE: sharing/src/test/scala/io/delta/sharing/spark/DeltaFormatSharingSourceSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark import java.time.LocalDateTime import org.apache.spark.sql.delta.{DeltaIllegalStateException, DeltaLog} import org.apache.spark.sql.delta.DeltaOptions.{ IGNORE_CHANGES_OPTION, IGNORE_DELETES_OPTION, SKIP_CHANGE_COMMITS_OPTION } import org.apache.spark.sql.delta.sources.{DeltaSourceOffset, DeltaSQLConf} import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import io.delta.sharing.client.DeltaSharingRestClient import io.delta.sharing.client.model.{Table => DeltaSharingTable} import org.apache.hadoop.fs.Path import org.apache.spark.SparkEnv import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.delta.sharing.DeltaSharingTestSparkUtils import io.delta.sharing.spark.test.shims.SharingStreamingTestShims.{ CheckpointFileManager, CommitMetadata, SerializedOffset, StreamingCheckpointConstants, StreamMetadata } import org.apache.spark.sql.functions.{col, lit} import org.apache.spark.sql.streaming.{StreamingQuery, StreamingQueryException, StreamTest} import org.apache.spark.sql.types.{ DateType, IntegerType, LongType, StringType, StructType, TimestampType } class DeltaFormatSharingSourceSuite extends StreamTest with DeltaSQLCommandTest with DeltaSharingTestSparkUtils with DeltaSharingDataSourceDeltaTestUtils { import testImplicits._ private def getSource(parameters: Map[String, String]): DeltaFormatSharingSource = { val options = new DeltaSharingOptions(parameters) val path = options.options.getOrElse( "path", throw DeltaSharingErrors.pathNotSpecifiedException ) val parsedPath = DeltaSharingRestClient.parsePath(path, Map.empty) val client = DeltaSharingRestClient( profileFile = parsedPath.profileFile, shareCredentialsOptions = Map.empty, forStreaming = true, responseFormat = "delta", readerFeatures = DeltaSharingUtils.STREAMING_SUPPORTED_READER_FEATURES.mkString(",") ) val dsTable = DeltaSharingTable( share = parsedPath.share, schema = parsedPath.schema, name = parsedPath.table ) DeltaFormatSharingSource( spark = spark, client = client, table = dsTable, options = options, parameters = parameters, sqlConf = sqlContext.sparkSession.sessionState.conf, metadataPath = "" ) } private def assertBlocksAreCleanedUp(): Unit = { val blockManager = SparkEnv.get.blockManager val matchingBlockIds = blockManager.getMatchingBlockIds( _.name.startsWith(DeltaSharingLogFileSystem.DELTA_SHARING_LOG_BLOCK_ID_PREFIX) ) assert(matchingBlockIds.isEmpty, "delta sharing blocks are not cleaned up.") } private def cleanUpDeltaSharingBlocks(): Unit = { val blockManager = SparkEnv.get.blockManager val matchingBlockIds = blockManager.getMatchingBlockIds( _.name.startsWith( DeltaSharingLogFileSystem.DELTA_SHARING_LOG_BLOCK_ID_PREFIX) ) matchingBlockIds.foreach(blockManager.removeBlock(_)) } test("DeltaFormatSharingSource able to get schema") { withTempDir { tempDir => val deltaTableName = "delta_table_schema" withTable(deltaTableName) { createTable(deltaTableName) val sharedTableName = "shared_table_schema" prepareMockedClientMetadata(deltaTableName, sharedTableName) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) val profileFile = prepareProfileFile(tempDir) withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val deltaSharingSource = getSource( Map("path" -> s"${profileFile.getCanonicalPath}#share1.default.$sharedTableName") ) val expectedSchema: StructType = new StructType() .add("c1", IntegerType) .add("c2", StringType) .add("c3", DateType) .add("c4", TimestampType) assert(deltaSharingSource.schema == expectedSchema) // CDF schema val cdfDeltaSharingSource = getSource( Map( "path" -> s"${profileFile.getCanonicalPath}#share1.default.$sharedTableName", "readChangeFeed" -> "true" ) ) val expectedCdfSchema: StructType = expectedSchema .copy() .add("_change_type", StringType) .add("_commit_version", LongType) .add("_commit_timestamp", TimestampType) assert(cdfDeltaSharingSource.schema == expectedCdfSchema) } } } } test("DeltaFormatSharingSource do not support cdc") { withTempDir { tempDir => val sharedTableName = "shared_streaming_table_nocdc" val profileFile = prepareProfileFile(tempDir) withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" val e = intercept[Exception] { val df = spark.readStream .format("deltaSharing") .option("responseFormat", "delta") .option("readChangeFeed", "true") .load(tablePath) testStream(df)( AssertOnQuery { q => q.processAllAvailable(); true } ) } assert(e.getMessage.contains("Delta sharing cdc streaming is not supported")) } } } test("DeltaFormatSharingSource getTableVersion error") { withTempDir { tempDir => val deltaTableName = "delta_table_version_error" withTable(deltaTableName) { sql( s""" |CREATE TABLE $deltaTableName (value STRING) |USING DELTA |""".stripMargin) val sharedTableName = "shared_streaming_table_version_error" val profileFile = prepareProfileFile(tempDir) prepareMockedClientMetadata(deltaTableName, sharedTableName) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName, Some(-1L)) withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" val e = intercept[Exception] { val df = spark.readStream .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) testStream(df)( AssertOnQuery { q => q.processAllAvailable(); true } ) } assert( e.getMessage.contains("Delta Sharing Server returning negative table version:-1,") ) } } } } // Test forceToDeltaSourceOffset directly: pass DeltaSharingSourceOffset JSON, call util. // Source construction requires getMetadata; use a real delta table and prepare mocks for // shared table "some_table". Flag on -> (DeltaSourceOffset, true); flag off -> throw. Seq(true, false).foreach { case autoResolve: Boolean => test(s"forceToDeltaSourceOffset: DeltaSharingSourceOffset JSON with flag " + s"autoResolve=$autoResolve") { withTempDir { tempDir => val deltaTableName = "delta_table_util_offset" withTable(deltaTableName) { createTable(deltaTableName) val sharedTableName = "some_table" prepareMockedClientMetadata(deltaTableName, sharedTableName) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) val profileFile = prepareProfileFile(tempDir) val tableId = "test-table-id" val autoResolveKey = DeltaSQLConf .DELTA_SHARING_STREAMING_AUTO_RESOLVE_RESPONSE_FORMAT .key withSQLConf( (getDeltaSharingClassesSQLConf ++ Seq( autoResolveKey -> autoResolve.toString )).toSeq: _* ) { val source = getSource( Map("path" -> s"${profileFile.getCanonicalPath}#share1.default.$sharedTableName") ) val tableIdField = source.getClass.getDeclaredField("tableId") tableIdField.setAccessible(true) tableIdField.set(source, tableId) val legacyJson = "{\"sourceVersion\":1," + s""""tableId":"$tableId",""" + "\"tableVersion\":1," + "\"index\":-1," + "\"isStartingVersion\":true}" val serializedOffset = SerializedOffset(legacyJson) if (autoResolve) { val (deltaOffset, fromLegacy) = source.forceToDeltaSourceOffset(serializedOffset) assert(fromLegacy, "fromLegacy should be true for DeltaSharingSourceOffset JSON") assert(deltaOffset.reservoirId === tableId) assert(deltaOffset.reservoirVersion === 1L) assert(deltaOffset.index === DeltaSourceOffset.BASE_INDEX) assert(deltaOffset.isInitialSnapshot) } else { intercept[Exception](source.forceToDeltaSourceOffset(serializedOffset)) } cleanUpDeltaSharingBlocks() } } } } } // E2E: Custom checkpoint with legacy DeltaSharingSourceOffset format; // restart with delta streaming using that checkpoint. // Flag on/off. Mocks use delta table only. Seq( (true, "flag on: restart with delta succeeds"), (false, "flag off: restart fails parsing legacy checkpoint") ).foreach { case (autoResolve, desc) => test(s"E2E: parquet streaming checkpoint then restart " + s"with delta streaming [$desc]") { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaTableName = "delta_table_e2e_parquet_then_delta" withTable(deltaTableName) { sql(s""" |CREATE TABLE $deltaTableName (value STRING) |USING DELTA |""".stripMargin) sql(s"INSERT INTO $deltaTableName VALUES ('p1'), ('p2')") val tableId = DeltaLog.forTable(spark, new TableIdentifier(deltaTableName)) .update().metadata.id val sharedTableName = "shared_streaming_table_e2e" val profileFile = prepareProfileFile(inputDir) val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" spark.sessionState.conf.setConfString( "spark.delta.sharing.streaming.queryTableVersionIntervalSeconds", "10s" ) // Build custom checkpoint with legacy DeltaSharingSourceOffset (no parquet stream run). val checkpointPath = new Path(checkpointDir.getCanonicalPath) // scalastyle:off deltahadoopconfiguration val hadoopConf = spark.sessionState.newHadoopConf() // scalastyle:on deltahadoopconfiguration val fileManager = CheckpointFileManager.create(checkpointPath, hadoopConf) val offsetsDir = StreamingCheckpointConstants.DIR_NAME_OFFSETS val commitsDir = StreamingCheckpointConstants.DIR_NAME_COMMITS val metaDir = StreamingCheckpointConstants.DIR_NAME_METADATA fileManager.mkdirs(new Path(checkpointPath, offsetsDir)) fileManager.mkdirs(new Path(checkpointPath, commitsDir)) val metadataPath = new Path(checkpointPath, metaDir) val streamId = java.util.UUID.randomUUID.toString StreamMetadata.write( StreamMetadata(streamId), metadataPath, hadoopConf) val legacyOffsetJson = "{\"sourceVersion\":1," + s""""tableId":"$tableId",""" + "\"tableVersion\":1," + "\"index\":-1," + "\"isStartingVersion\":true}" val offsetMetadataJson = """{"batchWatermarkMs":0,""" + """"batchTimestampMs":0,""" + """"conf":{},""" + """"sourceMetadataInfo":{}}""" val offsetContent = s"v1\n$offsetMetadataJson\n$legacyOffsetJson" .getBytes(java.nio.charset.StandardCharsets.UTF_8) val offsetBatchPath = new Path( new Path(checkpointPath, StreamingCheckpointConstants.DIR_NAME_OFFSETS), "0") val offsetOut = fileManager.createAtomic(offsetBatchPath, overwriteIfPossible = true) offsetOut.write(offsetContent) offsetOut.close() val commitContent = s"v1\n${CommitMetadata(0).json}" .getBytes(java.nio.charset.StandardCharsets.UTF_8) val commitBatchPath = new Path( new Path(checkpointPath, StreamingCheckpointConstants.DIR_NAME_COMMITS), "0") val commitOut = fileManager.createAtomic(commitBatchPath, overwriteIfPossible = true) commitOut.write(commitContent) commitOut.close() val autoResolveKey = DeltaSQLConf .DELTA_SHARING_STREAMING_AUTO_RESOLVE_RESPONSE_FORMAT .key withSQLConf( (getDeltaSharingClassesSQLConf ++ Seq( autoResolveKey -> autoResolve.toString )).toSeq: _* ) { prepareMockedClientMetadata(deltaTableName, sharedTableName) // Snapshot getFiles(versionAsOf=1) for initial batch when resuming from legacy offset prepareMockedClientAndFileSystemResult( deltaTableName, sharedTableName, versionAsOf = Some(1L)) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 1L, 1L) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) if (autoResolve) { val q = spark.readStream .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) .start(outputDir.toString) try { q.processAllAvailable() } finally { q.stop() } checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq("p1", "p2").toDF()) } else { var q: StreamingQuery = null val e = intercept[Exception] { q = spark.readStream .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) .start(outputDir.toString) try { q.processAllAvailable() } finally { if (q != null) q.stop() } } assert(e.getMessage != null && ( e.getMessage.contains("legacy") || e.getMessage.contains("checkpoint") || e.getCause != null && (e.getCause.getMessage.contains("legacy") || e.getCause.getMessage.contains("checkpoint"))), s"Expected legacy/checkpoint-related error, got: $e") } } } } } } // E2E: Legacy checkpoint with isStartingVersion=false (incremental // mode). The stream already processed through version 2, so on // restart it should pick up version 3 data. Seq( (true, "flag on: restart succeeds"), (false, "flag off: restart fails parsing legacy checkpoint") ).foreach { case (autoResolve, desc) => test(s"E2E: legacy checkpoint isStartingVersion=false " + s"then restart with delta streaming [$desc]") { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaTableName = "delta_table_e2e_not_starting_version" withTable(deltaTableName) { sql(s"""CREATE TABLE $deltaTableName (value STRING) |USING DELTA""".stripMargin) sql( s"INSERT INTO $deltaTableName VALUES ('p1'), ('p2')") sql( s"INSERT INTO $deltaTableName VALUES ('p3'), ('p4')") sql( s"INSERT INTO $deltaTableName VALUES ('p5'), ('p6')") val tableId = DeltaLog.forTable( spark, new TableIdentifier(deltaTableName)) .update().metadata.id val sharedTableName = "shared_streaming_table_e2e_nsv" val profileFile = prepareProfileFile(inputDir) val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" spark.sessionState.conf.setConfString( "spark.delta.sharing.streaming" + ".queryTableVersionIntervalSeconds", "10s" ) // Two committed batches so that populateStartOffsets // calls getBatch(offset_0, offset_1) with a valid // startOffset instead of None. val checkpointPath = new Path(checkpointDir.getCanonicalPath) // scalastyle:off deltahadoopconfiguration val hadoopConf = spark.sessionState.newHadoopConf() // scalastyle:on deltahadoopconfiguration val fileManager = CheckpointFileManager.create(checkpointPath, hadoopConf) val offsetsDir = StreamingCheckpointConstants.DIR_NAME_OFFSETS val commitsDir = StreamingCheckpointConstants.DIR_NAME_COMMITS val metaDir = StreamingCheckpointConstants.DIR_NAME_METADATA fileManager.mkdirs( new Path(checkpointPath, offsetsDir)) fileManager.mkdirs( new Path(checkpointPath, commitsDir)) val metadataPath = new Path(checkpointPath, metaDir) val streamId = java.util.UUID.randomUUID.toString StreamMetadata.write( StreamMetadata(streamId), metadataPath, hadoopConf) val offsetMetadataJson = """{"batchWatermarkMs":0,""" + """"batchTimestampMs":0,""" + """"conf":{},""" + """"sourceMetadataInfo":{}}""" // Batch 0: legacy offset at version 1 val legacyOffset0Json = "{\"sourceVersion\":1," + s""""tableId":"$tableId",""" + "\"tableVersion\":1," + "\"index\":-1," + "\"isStartingVersion\":false}" val offset0Content = s"v1\n$offsetMetadataJson\n$legacyOffset0Json" .getBytes(java.nio.charset.StandardCharsets.UTF_8) val offset0Path = new Path(new Path( checkpointPath, offsetsDir), "0") val offset0Out = fileManager.createAtomic( offset0Path, overwriteIfPossible = true) offset0Out.write(offset0Content) offset0Out.close() val commit0Content = s"v1\n${CommitMetadata(0).json}" .getBytes(java.nio.charset.StandardCharsets.UTF_8) val commit0Path = new Path(new Path( checkpointPath, commitsDir), "0") val commit0Out = fileManager.createAtomic( commit0Path, overwriteIfPossible = true) commit0Out.write(commit0Content) commit0Out.close() // Batch 1: legacy offset at version 2 val legacyOffset1Json = "{\"sourceVersion\":1," + s""""tableId":"$tableId",""" + "\"tableVersion\":2," + "\"index\":-1," + "\"isStartingVersion\":false}" val offset1Content = s"v1\n$offsetMetadataJson\n$legacyOffset1Json" .getBytes(java.nio.charset.StandardCharsets.UTF_8) val offset1Path = new Path(new Path( checkpointPath, offsetsDir), "1") val offset1Out = fileManager.createAtomic( offset1Path, overwriteIfPossible = true) offset1Out.write(offset1Content) offset1Out.close() val commit1Content = s"v1\n${CommitMetadata(1).json}" .getBytes(java.nio.charset.StandardCharsets.UTF_8) val commit1Path = new Path(new Path( checkpointPath, commitsDir), "1") val commit1Out = fileManager.createAtomic( commit1Path, overwriteIfPossible = true) commit1Out.write(commit1Content) commit1Out.close() val autoResolveKey = DeltaSQLConf .DELTA_SHARING_STREAMING_AUTO_RESOLVE_RESPONSE_FORMAT .key withSQLConf( (getDeltaSharingClassesSQLConf ++ Seq( autoResolveKey -> autoResolve.toString )).toSeq: _* ) { prepareMockedClientMetadata( deltaTableName, sharedTableName) // getBatch(offset_0, offset_1) uses offset_0 as // startingOffset (isInitialSnapshot=false) so the // streaming API is used from version 1 to 3. prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 1L, 3L) prepareMockedClientGetTableVersion( deltaTableName, sharedTableName) if (autoResolve) { val q = spark.readStream .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) .start(outputDir.toString) try { q.processAllAvailable() } finally { q.stop() } checkAnswer( spark.read.format("delta") .load(outputDir.getCanonicalPath), Seq("p3", "p4", "p5", "p6").toDF()) } else { var q: StreamingQuery = null val e = intercept[Exception] { q = spark.readStream .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) .start(outputDir.toString) try { q.processAllAvailable() } finally { if (q != null) q.stop() } } assert(e.getMessage != null && ( e.getMessage.contains("legacy") || e.getMessage.contains("checkpoint") || e.getCause != null && ( e.getCause.getMessage.contains("legacy") || e.getCause.getMessage .contains("checkpoint"))), s"Expected legacy/checkpoint error, got: $e") } } } } } } test("DeltaFormatSharingSource simple query works") { withTempDir { tempDir => val deltaTableName = "delta_table_simple" withTable(deltaTableName) { sql(s""" |CREATE TABLE $deltaTableName (value STRING) |USING DELTA |""".stripMargin) val sharedTableName = "shared_streaming_table_simple" prepareMockedClientMetadata(deltaTableName, sharedTableName) val profileFile = prepareProfileFile(tempDir) withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" def InsertToDeltaTable(values: String): Unit = { sql(s"INSERT INTO $deltaTableName VALUES $values") } InsertToDeltaTable("""("keep1"), ("keep2"), ("drop3")""") prepareMockedClientAndFileSystemResult(deltaTableName, sharedTableName, Some(1L)) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) val df = spark.readStream .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .filter($"value" contains "keep") spark.sessionState.conf.setConfString( "spark.delta.sharing.streaming.queryTableVersionIntervalSeconds", "9s" ) val e = intercept[Exception] { testStream(df)( AssertOnQuery { q => q.processAllAvailable(); true } ) } assert(e.getMessage.contains("must not be less than 10 seconds")) spark.sessionState.conf.setConfString( "spark.delta.sharing.streaming.queryTableVersionIntervalSeconds", "10s" ) testStream(df)( AssertOnQuery { q => q.processAllAvailable(); true }, CheckAnswer("keep1", "keep2"), StopStream ) } } } } // Mirror of batch auto-resolve test: grid over flag. When ON, getMetadata is used and we send // its format (delta or parquet); when OFF, user's responseFormat is used. Seq( (true, "shared_streaming_table_auto_resolve", "delta"), (true, "shared_parquet_table_auto_resolve", "parquet"), (false, "shared_parquet_table_streaming", "parquet"), (false, "shared_streaming_table_delta", "delta") ).foreach { case (autoResolve, sharedTableName, expectedFormat) => test(s"streaming auto-resolve [flag=$autoResolve, " + s"format=$expectedFormat]") { withTempDir { tempDir => val deltaTableName = "delta_table_auto_resolve" withTable(deltaTableName) { sql(s"DROP TABLE IF EXISTS $deltaTableName") sql( s"""CREATE TABLE $deltaTableName (value STRING) |USING DELTA""".stripMargin) val profileFile = prepareProfileFile(tempDir) val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" sql(s"INSERT INTO $deltaTableName VALUES ('a'), ('b')") spark.sessionState.conf.setConfString( "spark.delta.sharing.streaming" + ".queryTableVersionIntervalSeconds", "10s" ) if (autoResolve) { prepareMockedClientMetadata( deltaTableName, sharedTableName) if (expectedFormat == "delta") { prepareMockedClientAndFileSystemResult( deltaTableName, sharedTableName, Some(1L)) } else { prepareMockedClientAndFileSystemResultForParquet( deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForParquet( deltaTableName, sharedTableName, versionAsOf = Some(1L)) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 1L, 1L) } } else { if (expectedFormat == "parquet") { prepareMockedClientAndFileSystemResultForParquet( deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForParquet( deltaTableName, sharedTableName, versionAsOf = Some(1L)) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 1L, 1L) } else { prepareMockedClientMetadata( deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResult( deltaTableName, sharedTableName, Some(1L)) } } prepareMockedClientGetTableVersion( deltaTableName, sharedTableName) val userResponseFormat = if (autoResolve) "parquet" else expectedFormat val autoResolveKey = DeltaSQLConf .DELTA_SHARING_STREAMING_AUTO_RESOLVE_RESPONSE_FORMAT .key withSQLConf( (getDeltaSharingClassesSQLConf + (autoResolveKey -> autoResolve.toString)) .toSeq: _* ) { val df = spark.readStream .format("deltaSharing") .option("responseFormat", userResponseFormat) .load(tablePath) testStream(df)( AssertOnQuery { q => q.processAllAvailable(); true }, CheckAnswer("a", "b"), StopStream ) assertRequestedFormat( s"share1.default.$sharedTableName", Seq(expectedFormat)) } } } } } test( "restart works sharing" ) { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaTableName = "delta_table_restart" withTable(deltaTableName) { createTableForStreaming(deltaTableName) val sharedTableName = "shared_streaming_table_restart" prepareMockedClientMetadata(deltaTableName, sharedTableName) val profileFile = prepareProfileFile(inputDir) val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { def InsertToDeltaTable(values: String): Unit = { sql(s"INSERT INTO $deltaTableName VALUES $values") } // TODO: check testStream() function helper def processAllAvailableInStream(): Unit = { val q = spark.readStream .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .filter($"value" contains "keep") .writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) .start(outputDir.toString) try { q.processAllAvailable() } finally { q.stop() } } // Able to stream snapshot at version 1. InsertToDeltaTable("""("keep1"), ("keep2"), ("drop1")""") prepareMockedClientAndFileSystemResult( deltaTable = deltaTableName, sharedTable = sharedTableName, versionAsOf = Some(1L) ) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) processAllAvailableInStream() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq("keep1", "keep2").toDF() ) // No new data, so restart will not process any new data. processAllAvailableInStream() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq("keep1", "keep2").toDF() ) // Able to stream new data at version 2. InsertToDeltaTable("""("keep3"), ("keep4"), ("drop2")""") prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 2, 2 ) processAllAvailableInStream() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq("keep1", "keep2", "keep3", "keep4").toDF() ) sql(s"""OPTIMIZE $deltaTableName""") prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 2, 3 ) // Optimize doesn't produce new data, so restart will not process any new data. processAllAvailableInStream() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq("keep1", "keep2", "keep3", "keep4").toDF() ) // Able to stream new data at version 3. InsertToDeltaTable("""("keep5"), ("keep6"), ("drop3")""") prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 3, 4 ) processAllAvailableInStream() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq("keep1", "keep2", "keep3", "keep4", "keep5", "keep6").toDF() ) assertBlocksAreCleanedUp() } } } } test( "restart works sharing with special chars" ) { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaTableName = "delta_table_restart_special" withTable(deltaTableName) { // scalastyle:off nonascii sql(s"""CREATE TABLE $deltaTableName (`第一列` STRING) USING DELTA""".stripMargin) val sharedTableName = "shared_streaming_table_special" prepareMockedClientMetadata(deltaTableName, sharedTableName) val profileFile = prepareProfileFile(inputDir) val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { def InsertToDeltaTable(values: String): Unit = { sql(s"INSERT INTO $deltaTableName VALUES $values") } // TODO: check testStream() function helper def processAllAvailableInStream(): Unit = { val q = spark.readStream .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .filter($"第一列" contains "keep") .writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) .start(outputDir.toString) // scalastyle:on nonascii try { q.processAllAvailable() } finally { q.stop() } } // Able to stream snapshot at version 1. InsertToDeltaTable("""("keep1"), ("keep2"), ("drop1")""") prepareMockedClientAndFileSystemResult( deltaTable = deltaTableName, sharedTable = sharedTableName, versionAsOf = Some(1L) ) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) processAllAvailableInStream() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq("keep1", "keep2").toDF() ) // No new data, so restart will not process any new data. processAllAvailableInStream() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq("keep1", "keep2").toDF() ) // Able to stream new data at version 2. InsertToDeltaTable("""("keep3"), ("keep4"), ("drop2")""") prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 2, 2 ) processAllAvailableInStream() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq("keep1", "keep2", "keep3", "keep4").toDF() ) sql(s"""OPTIMIZE $deltaTableName""") prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 2, 3 ) // Optimize doesn't produce new data, so restart will not process any new data. processAllAvailableInStream() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq("keep1", "keep2", "keep3", "keep4").toDF() ) // Able to stream new data at version 3. InsertToDeltaTable("""("keep5"), ("keep6"), ("drop3")""") prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 3, 4 ) processAllAvailableInStream() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq("keep1", "keep2", "keep3", "keep4", "keep5", "keep6").toDF() ) assertBlocksAreCleanedUp() } } } } test("streaming works with deletes on basic table") { withTempDir { inputDir => val deltaTableName = "delta_table_deletes" withTable(deltaTableName) { createTableForStreaming(deltaTableName) val sharedTableName = "shared_streaming_table_deletes" prepareMockedClientMetadata(deltaTableName, sharedTableName) val profileFile = prepareProfileFile(inputDir) val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { def InsertToDeltaTable(values: String): Unit = { sql(s"INSERT INTO $deltaTableName VALUES $values") } def processAllAvailableInStream( sourceOptions: Map[String, String], expectations: StreamAction*): Unit = { val df = spark.readStream .format("deltaSharing") .options(sourceOptions) .load(tablePath) val base = Seq(StartStream(), ProcessAllAvailable()) testStream(df)((base ++ expectations): _*) } // Insert at version 1 and 2. InsertToDeltaTable("""("keep1")""") InsertToDeltaTable("""("keep2")""") // delete at version 3. sql(s"""DELETE FROM $deltaTableName WHERE value = "keep1" """) // update at version 4. sql(s"""UPDATE $deltaTableName SET value = "keep3" WHERE value = "keep2" """) prepareMockedClientAndFileSystemResult( deltaTable = deltaTableName, sharedTable = sharedTableName, versionAsOf = Some(4L) ) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) processAllAvailableInStream( Map("responseFormat" -> "delta"), CheckAnswer("keep3") ) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 0, 4 ) // The streaming query will fail because changes detected in version 4. // This is the original delta behavior. val e = intercept[Exception] { processAllAvailableInStream( Map("responseFormat" -> "delta", "startingVersion" -> "0") ) } for (msg <- Seq( "Detected", "not supported", "true" )) { assert(e.getMessage.contains(msg)) } // The streaming query will fail because changes detected in version 4. // This is the original delta behavior. val e2 = intercept[Exception] { processAllAvailableInStream( Map( "responseFormat" -> "delta", "startingVersion" -> "0", IGNORE_DELETES_OPTION -> "true" ) ) } for (msg <- Seq( "Detected", "not supported", "true" )) { assert(e2.getMessage.contains(msg)) } // The streaming query will succeed because ignoreChanges helps to ignore the updates, but // added updated data "keep3". processAllAvailableInStream( Map( "responseFormat" -> "delta", "startingVersion" -> "0", IGNORE_CHANGES_OPTION -> "true" ), CheckAnswer("keep1", "keep2", "keep3") ) // The streaming query will succeed because skipChangeCommits helps to ignore the whole // commit with data update, so updated data is not produced either. processAllAvailableInStream( Map( "responseFormat" -> "delta", "startingVersion" -> "0", SKIP_CHANGE_COMMITS_OPTION -> "true" ), CheckAnswer("keep1", "keep2") ) assertBlocksAreCleanedUp() } } } } test("streaming works with DV") { withTempDir { inputDir => val deltaTableName = "delta_table_dv" withTable(deltaTableName) { createSimpleTable(deltaTableName, enableCdf = false) spark.sql( s"ALTER TABLE $deltaTableName SET TBLPROPERTIES('delta.enableDeletionVectors' = true)" ) val sharedTableName = "shared_streaming_table_dv" prepareMockedClientMetadata(deltaTableName, sharedTableName) val profileFile = prepareProfileFile(inputDir) val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { def InsertToDeltaTable(values: String): Unit = { sql(s"INSERT INTO $deltaTableName VALUES $values") } def processAllAvailableInStream( sourceOptions: Map[String, String], expectations: StreamAction*): Unit = { val df = spark.readStream .format("deltaSharing") .options(sourceOptions) .load(tablePath) .filter($"c2" contains "keep") .select("c1") val base = Seq(StartStream(), ProcessAllAvailable()) testStream(df)((base ++ expectations): _*) } // Insert at version 2. InsertToDeltaTable("""(1, "keep1"),(2, "keep1"),(3, "keep1"),(1,"drop1")""") // delete at version 3. sql(s"""DELETE FROM $deltaTableName WHERE c1 >= 2 """) prepareMockedClientAndFileSystemResult( deltaTable = deltaTableName, sharedTable = sharedTableName, versionAsOf = Some(3L) ) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) processAllAvailableInStream( Map("responseFormat" -> "delta"), CheckAnswer(1) ) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, startingVersion = 0, endingVersion = 3, assertDVExists = true ) // The streaming query will fail because deletes detected in version 3. And there are no // options provided to ignore the deletion. val e = intercept[Exception] { processAllAvailableInStream( Map("responseFormat" -> "delta", "startingVersion" -> "0") ) } for (msg <- Seq( "Detected a data update", "not supported", SKIP_CHANGE_COMMITS_OPTION, "true" )) { assert(e.getMessage.contains(msg)) } // The streaming query will fail because deletes detected in version 3, and it's // recognized as updates and ignoreDeletes doesn't help. This is the original delta // behavior. val e2 = intercept[Exception] { processAllAvailableInStream( Map( "responseFormat" -> "delta", "startingVersion" -> "0", IGNORE_DELETES_OPTION -> "true" ) ) } for (msg <- Seq( "Detected a data update", "not supported", SKIP_CHANGE_COMMITS_OPTION, "true" )) { assert(e2.getMessage.contains(msg)) } // The streaming query will succeed because ignoreChanges helps to ignore the delete, but // added duplicated data 1. processAllAvailableInStream( Map( "responseFormat" -> "delta", "startingVersion" -> "0", IGNORE_CHANGES_OPTION -> "true" ), CheckAnswer(1, 2, 3, 1) ) // The streaming query will succeed because skipChangeCommits helps to ignore the whole // commit with data update, so no duplicated data is produced either. processAllAvailableInStream( Map( "responseFormat" -> "delta", "startingVersion" -> "0", SKIP_CHANGE_COMMITS_OPTION -> "true" ), CheckAnswer(1, 2, 3) ) assertBlocksAreCleanedUp() } } } } test("streaming works with timestampNTZ") { withTempDir { tempDir => val deltaTableName = "delta_table_timestampNTZ" withTable(deltaTableName) { sql(s"CREATE TABLE $deltaTableName(c1 TIMESTAMP_NTZ) USING DELTA") val sharedTableName = "shared_table_timestampNTZ" prepareMockedClientMetadata(deltaTableName, sharedTableName) val profileFile = prepareProfileFile(tempDir) val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { def InsertToDeltaTable(values: String): Unit = { sql(s"INSERT INTO $deltaTableName VALUES $values") } def processAllAvailableInStream( sourceOptions: Map[String, String], expectations: StreamAction*): Unit = { val df = spark.readStream .format("deltaSharing") .options(sourceOptions) .load(tablePath) .select("c1") val base = Seq(StartStream(), ProcessAllAvailable()) testStream(df)((base ++ expectations): _*) } // Insert at version 1. InsertToDeltaTable("""('2022-01-01 02:03:04.123456')""") // Insert at version 2. InsertToDeltaTable("""('2022-02-02 03:04:05.123456')""") prepareMockedClientAndFileSystemResult( deltaTable = deltaTableName, sharedTable = sharedTableName, versionAsOf = Some(2L) ) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) processAllAvailableInStream( Map("responseFormat" -> "delta"), CheckAnswer( LocalDateTime.parse("2022-01-01T02:03:04.123456"), LocalDateTime.parse("2022-02-02T03:04:05.123456") ) ) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, startingVersion = 2, endingVersion = 2 ) processAllAvailableInStream( Map( "responseFormat" -> "delta", "startingVersion" -> "2" ), CheckAnswer(LocalDateTime.parse("2022-02-02T03:04:05.123456")) ) assertBlocksAreCleanedUp() } } } } test( "startingVersion works" ) { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaTableName = "delta_table_startVersion" withTable(deltaTableName) { createTableForStreaming(deltaTableName) val sharedTableName = "shared_streaming_table_startVersion" prepareMockedClientMetadata(deltaTableName, sharedTableName) val profileFile = prepareProfileFile(inputDir) val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { def InsertToDeltaTable(values: String): Unit = { sql(s"INSERT INTO $deltaTableName VALUES $values") } def processAllAvailableInStream(): Unit = { val q = spark.readStream .format("deltaSharing") .option("responseFormat", "delta") .option("startingVersion", 0) .load(tablePath) .filter($"value" contains "keep") .writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) .start(outputDir.toString) try { q.processAllAvailable() } finally { q.stop() } } // Able to stream snapshot at version 1. InsertToDeltaTable("""("keep1"), ("keep2"), ("drop1")""") prepareMockedClientAndFileSystemResultForStreaming( deltaTable = deltaTableName, sharedTable = sharedTableName, startingVersion = 0L, endingVersion = 1L ) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) processAllAvailableInStream() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq("keep1", "keep2").toDF() ) // No new data, so restart will not process any new data. processAllAvailableInStream() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq("keep1", "keep2").toDF() ) // Able to stream new data at version 2. InsertToDeltaTable("""("keep3"), ("keep4"), ("drop2")""") prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 0, 2 ) processAllAvailableInStream() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq("keep1", "keep2", "keep3", "keep4").toDF() ) sql(s"""OPTIMIZE $deltaTableName""") prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 2, 3 ) // Optimize doesn't produce new data, so restart will not process any new data. processAllAvailableInStream() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq("keep1", "keep2", "keep3", "keep4").toDF() ) // No new data, so restart will not process any new data. It will ask for the // last commit so that it can figure out that there's nothing to do. prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 3, 3 ) processAllAvailableInStream() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq("keep1", "keep2", "keep3", "keep4").toDF() ) // Able to stream new data at version 3. InsertToDeltaTable("""("keep5"), ("keep6"), ("drop3")""") prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 3, 4 ) processAllAvailableInStream() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq("keep1", "keep2", "keep3", "keep4", "keep5", "keep6").toDF() ) // No new data, so restart will not process any new data. It will ask for the // last commit so that it can figure out that there's nothing to do. prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 4, 4 ) processAllAvailableInStream() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq("keep1", "keep2", "keep3", "keep4", "keep5", "keep6").toDF() ) assertBlocksAreCleanedUp() } } } } test( "files are in a stable order for streaming" ) { // This test function is to check that DeltaSharingLogFileSystem puts the files in the delta log // in a stable order for each commit, regardless of the returning order from the server, so that // the DeltaSource can produce a stable file index. // We are using maxBytesPerTrigger which causes the streaming to stop in the middle of a commit // to be able to test this behavior. withTempDirs { (inputDir, outputDir, checkpointDir) => withTempDirs { (_, outputDir2, checkpointDir2) => val deltaTableName = "delta_table_order" withTable(deltaTableName) { createSimpleTable(deltaTableName, enableCdf = false) val sharedTableName = "shared_streaming_table_order" prepareMockedClientMetadata(deltaTableName, sharedTableName) val profileFile = prepareProfileFile(inputDir) val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" def InsertToDeltaTable(values: String): Unit = { sql(s"INSERT INTO $deltaTableName VALUES $values") } // Able to stream snapshot at version 1. InsertToDeltaTable("""(1, "one"), (2, "two"), (3, "three")""") withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { def processAllAvailableInStream( outputDirStr: String, checkpointDirStr: String): Unit = { val q = spark.readStream .format("deltaSharing") .option("responseFormat", "delta") .option("maxBytesPerTrigger", "1b") .load(tablePath) .writeStream .format("delta") .option("checkpointLocation", checkpointDirStr) .start(outputDirStr) try { q.processAllAvailable() val progress = q.recentProgress.filter(_.numInputRows != 0) assert(progress.length === 3) progress.foreach { p => assert(p.numInputRows === 1) } } finally { q.stop() } } // First output, without reverseFileOrder prepareMockedClientAndFileSystemResult( deltaTable = deltaTableName, sharedTable = sharedTableName, versionAsOf = Some(1L) ) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) processAllAvailableInStream(outputDir.toString, checkpointDir.toString) checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq((1, "one"), (2, "two"), (3, "three")).toDF() ) // Second output, with reverseFileOrder = true prepareMockedClientAndFileSystemResult( deltaTable = deltaTableName, sharedTable = sharedTableName, versionAsOf = Some(1L), reverseFileOrder = true ) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) processAllAvailableInStream(outputDir2.toString, checkpointDir2.toString) checkAnswer( spark.read.format("delta").load(outputDir2.getCanonicalPath), Seq((1, "one"), (2, "two"), (3, "three")).toDF() ) // Check each version of the two output are the same, which means the files are sorted // by DeltaSharingLogFileSystem, and are processed in a deterministic order by the // DeltaSource. val deltaLog = DeltaLog.forTable(spark, new Path(outputDir.toString)) Seq(0, 1, 2).foreach { v => val version = deltaLog.snapshot.version - v val df1 = spark.read .format("delta") .option("versionAsOf", version) .load(outputDir.getCanonicalPath) val df2 = spark.read .format("delta") .option("versionAsOf", version) .load(outputDir2.getCanonicalPath) checkAnswer(df1, df2) assert(df1.count() == (3 - v)) } assertBlocksAreCleanedUp() } } } } } test( "DeltaFormatSharingSource query with two delta sharing tables works" ) { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaTableName = "delta_table_two" def InsertToDeltaTable(values: String): Unit = { sql(s"INSERT INTO $deltaTableName VALUES $values") } withTable(deltaTableName) { createSimpleTable(deltaTableName, enableCdf = false) val sharedTableName = "shared_streaming_table_two" prepareMockedClientMetadata(deltaTableName, sharedTableName) val profileFile = prepareProfileFile(inputDir) val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { InsertToDeltaTable("""(1, "one"), (2, "one")""") InsertToDeltaTable("""(1, "two"), (2, "two")""") InsertToDeltaTable("""(1, "three"), (2, "three")""") prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResult( deltaTableName, sharedTableName, Some(3L) ) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, startingVersion = 1, endingVersion = 3 ) def processAllAvailableInStream(): Unit = { val dfLatest = spark.readStream .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) val dfV1 = spark.readStream .format("deltaSharing") .option("responseFormat", "delta") .option("startingVersion", 1) .load(tablePath) .select(col("c2"), col("c1").as("v1c1")) .filter(col("v1c1") === 1) val q = dfLatest .join(dfV1, "c2") .writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) .start(outputDir.toString) try { q.processAllAvailable() } finally { q.stop() } } // c1 from dfLatest, c2 from dfLatest, c1 from dfV1 var expected = Seq( Row("one", 1, 1), Row("one", 2, 1), Row("two", 1, 1), Row("two", 2, 1), Row("three", 1, 1), Row("three", 2, 1) ) processAllAvailableInStream() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), expected ) InsertToDeltaTable("""(1, "four"), (2, "four")""") prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, startingVersion = 4, endingVersion = 4 ) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, startingVersion = 1, endingVersion = 4 ) expected = expected ++ Seq(Row("four", 1, 1), Row("four", 2, 1)) processAllAvailableInStream() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), expected ) assertBlocksAreCleanedUp() } } } } Seq( ("add a partition column", Seq("part"), Seq("is_even", "part")), ("change partition order", Seq("part", "is_even"), Seq("is_even", "part")), ("different partition column", Seq("part"), Seq("is_even")) ).foreach { case (repartitionTestCase, initPartitionCols, overwritePartitionCols) => test( "deltaSharing - repartition delta source should fail by default " + s"unless unsafe flag is set - $repartitionTestCase" ) { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaTableName = "basic_delta_table_partition_check" withTable(deltaTableName) { spark.sql( s"""CREATE TABLE $deltaTableName (id LONG, part INT, is_even BOOLEAN) |USING DELTA PARTITIONED BY (${initPartitionCols.mkString(", ")}) |""".stripMargin ) val sharedTableName = "shared_streaming_table_partition_check_" + s"${repartitionTestCase.replace(' ', '_')}" prepareMockedClientMetadata(deltaTableName, sharedTableName) val profileFile = prepareProfileFile(inputDir) val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { def processAllAvailableInStream(startingVersion: Int): Unit = { val q = spark.readStream .format("deltaSharing") .option("responseFormat", "delta") .option("skipChangeCommits", "true") .option("startingVersion", startingVersion) .load(tablePath) .writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) .start(outputDir.toString) try { q.processAllAvailable() } finally { q.stop() } } spark.range(10).withColumn("part", lit(1)) .withColumn("is_even", $"id" % 2 === 0).write .format("delta").partitionBy(initPartitionCols: _*) .mode("append") .saveAsTable(deltaTableName) spark.range(2).withColumn("part", lit(2)) .withColumn("is_even", $"id" % 2 === 0).write .format("delta").partitionBy(initPartitionCols: _*) .mode("append").saveAsTable(deltaTableName) spark.range(10).withColumn("part", lit(1)) .withColumn("is_even", $"id" % 2 === 0).write .format("delta").partitionBy(overwritePartitionCols: _*) .option("overwriteSchema", "true").mode("overwrite") .saveAsTable(deltaTableName) spark.range(2).withColumn("part", lit(2)) .withColumn("is_even", $"id" % 2 === 0).write .format("delta").partitionBy(overwritePartitionCols: _*) .mode("append").saveAsTable(deltaTableName) prepareMockedClientAndFileSystemResultForStreaming( deltaTable = deltaTableName, sharedTable = sharedTableName, startingVersion = 0L, endingVersion = 4L ) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) var e = intercept[StreamingQueryException] { processAllAvailableInStream(0) } assert(e.getCause.asInstanceOf[DeltaIllegalStateException].getErrorClass == "DELTA_SCHEMA_CHANGED_WITH_STARTING_OPTIONS") assert(e.getMessage.contains("Detected schema change in version 3")) // delta table created using sql with specified partition col // will construct their initial snapshot on the initial definition prepareMockedClientAndFileSystemResultForStreaming( deltaTable = deltaTableName, sharedTable = sharedTableName, startingVersion = 4L, endingVersion = 4L ) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) e = intercept[StreamingQueryException] { processAllAvailableInStream(4) } assert(e.getMessage.contains("Detected schema change in version 4")) // Streaming query made progress without throwing error when // unsafe flag is set to true withSQLConf( DeltaSQLConf.DELTA_STREAMING_UNSAFE_READ_ON_PARTITION_COLUMN_CHANGE.key -> "true" ) { processAllAvailableInStream(0) } } } } } } test("streaming variant query works") { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaTableName = "variant_table" withTable(deltaTableName) { sql(s"create table $deltaTableName (v VARIANT) using delta") val sharedTableName = "shared_variant_table" prepareMockedClientMetadata(deltaTableName, sharedTableName) val profileFile = prepareProfileFile(inputDir) withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" sql(s"""insert into table $deltaTableName select parse_json(format_string('{"key": %s}', id)) from range(0, 10) """) prepareMockedClientAndFileSystemResult( deltaTableName, sharedTableName, versionAsOf = Some(1L) ) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) val q = spark.readStream .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) .start(outputDir.toString) try { q.processAllAvailable() } finally { q.stop() } checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), spark.sql(s"select * from $deltaTableName") ) } } } } } ================================================ FILE: sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingCDFUtilsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark import java.io.File import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import io.delta.sharing.client.{ DeltaSharingClient, DeltaSharingProfileProvider, DeltaSharingRestClient } import io.delta.sharing.client.model.{DeltaTableFiles, DeltaTableMetadata, Table, TemporaryCredentials} import org.apache.commons.io.FileUtils import org.apache.hadoop.fs.Path import org.apache.spark.{SparkConf, SparkEnv} import org.apache.spark.delta.sharing.{PreSignedUrlCache, PreSignedUrlFetcher} import org.apache.spark.sql.{QueryTest, SparkSession} import org.apache.spark.sql.delta.sharing.DeltaSharingTestSparkUtils import org.apache.spark.sql.test.{SharedSparkSession} private object CDFTesTUtils { val paths = Seq("http://path1", "http://path2") val SparkConfForReturnExpTime = "spark.delta.sharing.fileindexsuite.returnexptime" // 10 seconds val expirationTimeMs = 10000 def getExpirationTimestampStr(returnExpTime: Boolean): String = { if (returnExpTime) { s""""expirationTimestamp":${System.currentTimeMillis() + expirationTimeMs},""" } else { "" } } // scalastyle:off line.size.limit val fileStr1Id = "11d9b72771a72f178a6f2839f7f08528" val metaDataStr = """{"metaData":{"size":809,"deltaMetadata":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"c1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["c2"],"configuration":{"delta.enableChangeDataFeed":"true"},"createdTime":1691734718560}}}""" def getAddFileStr1(path: String, returnExpTime: Boolean = false): String = { s"""{"file":{"id":"11d9b72771a72f178a6f2839f7f08528",${getExpirationTimestampStr( returnExpTime )}"deltaSingleAction":{"add":{"path":"${path}",""" + """"partitionValues":{"c2":"one"},"size":809,"modificationTime":1691734726073,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"c1\":1,\"c2\":\"one\"},\"maxValues\":{\"c1\":2,\"c2\":\"one\"},\"nullCount\":{\"c1\":0,\"c2\":0}}","tags":{"INSERTION_TIME":"1691734726073000","MIN_INSERTION_TIME":"1691734726073000","MAX_INSERTION_TIME":"1691734726073000","OPTIMIZE_TARGET_SIZE":"268435456"}}}}}""" } def getAddFileStr2(returnExpTime: Boolean = false): String = { s"""{"file":{"id":"22d9b72771a72f178a6f2839f7f08529",${getExpirationTimestampStr( returnExpTime )}""" + """"deltaSingleAction":{"add":{"path":"http://path2","partitionValues":{"c2":"two"},"size":809,"modificationTime":1691734726073,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"c1\":1,\"c2\":\"two\"},\"maxValues\":{\"c1\":2,\"c2\":\"two\"},\"nullCount\":{\"c1\":0,\"c2\":0}}","tags":{"INSERTION_TIME":"1691734726073000","MIN_INSERTION_TIME":"1691734726073000","MAX_INSERTION_TIME":"1691734726073000","OPTIMIZE_TARGET_SIZE":"268435456"}}}}}""" } // scalastyle:on line.size.limit } /** * A mocked delta sharing client for unit tests. */ class TestDeltaSharingClientForCDFUtils( profileProvider: DeltaSharingProfileProvider, timeoutInSeconds: Int = 120, numRetries: Int = 3, maxRetryDuration: Long = Long.MaxValue, retrySleepInterval: Long = 1000, sslTrustAll: Boolean = false, forStreaming: Boolean = false, responseFormat: String = DeltaSharingRestClient.RESPONSE_FORMAT_DELTA, readerFeatures: String = "", queryTablePaginationEnabled: Boolean = false, maxFilesPerReq: Int = 100000, endStreamActionEnabled: Boolean = false, enableAsyncQuery: Boolean = false, asyncQueryPollIntervalMillis: Long = 10000L, asyncQueryMaxDuration: Long = 600000L, tokenExchangeMaxRetries: Int = 5, tokenExchangeMaxRetryDurationInSeconds: Int = 60, tokenRenewalThresholdInSeconds: Int = 600, callerOrg: String = "", skipFileIdHashVerification: Boolean = false) extends DeltaSharingClient { import CDFTesTUtils._ private lazy val returnExpirationTimestamp = SparkSession.active.sessionState.conf .getConfString( SparkConfForReturnExpTime ) .toBoolean var numGetFileCalls: Int = -1 override def listAllTables(): Seq[Table] = throw new UnsupportedOperationException("not needed") override def getMetadata( table: Table, versionAsOf: Option[Long], timestampAsOf: Option[String]): DeltaTableMetadata = { throw new UnsupportedOperationException("getMetadata is not supported now.") } override def getTableVersion(table: Table, startingTimestamp: Option[String] = None): Long = { throw new UnsupportedOperationException("getTableVersion is not supported now.") } override def getFiles( table: Table, predicates: Seq[String], limit: Option[Long], versionAsOf: Option[Long], timestampAsOf: Option[String], jsonPredicateHints: Option[String], refreshToken: Option[String], fileIdHash: Option[String] ): DeltaTableFiles = { throw new UnsupportedOperationException("getFiles is not supported now.") } override def getFiles( table: Table, startingVersion: Long, endingVersion: Option[Long], fileIdHash: Option[String] ): DeltaTableFiles = { throw new UnsupportedOperationException(s"getFiles with startingVersion($startingVersion)") } override def getCDFFiles( table: Table, cdfOptions: Map[String, String], includeHistoricalMetadata: Boolean, fileIdHash: Option[String] ): DeltaTableFiles = { numGetFileCalls += 1 DeltaTableFiles( version = 0, lines = Seq[String]( """{"protocol":{"deltaProtocol":{"minReaderVersion": 1, "minWriterVersion": 1}}}""", metaDataStr, getAddFileStr1(paths(numGetFileCalls.min(1)), returnExpirationTimestamp), getAddFileStr2(returnExpirationTimestamp) ), respondedFormat = DeltaSharingRestClient.RESPONSE_FORMAT_DELTA ) } override def generateTemporaryTableCredential( table: Table, location: Option[String]): TemporaryCredentials = { throw new UnsupportedOperationException("generateTemporaryTableCredential is not implemented") } override def getForStreaming(): Boolean = forStreaming override def getProfileProvider: DeltaSharingProfileProvider = profileProvider } class DeltaSharingCDFUtilsSuite extends QueryTest with DeltaSQLCommandTest with SharedSparkSession with DeltaSharingTestSparkUtils { import CDFTesTUtils._ private val shareName = "share" private val schemaName = "default" private val sharedTableName = "table" override protected def sparkConf: SparkConf = { super.sparkConf .set("spark.delta.sharing.preSignedUrl.expirationMs", expirationTimeMs.toString) .set("spark.delta.sharing.driver.refreshCheckIntervalMs", "1000") .set("spark.delta.sharing.driver.refreshThresholdMs", "2000") .set("spark.delta.sharing.driver.accessThresholdToExpireMs", "60000") } test("refresh works") { PreSignedUrlCache.registerIfNeeded(SparkEnv.get) withTempDir { tempDir => val profileFile = new File(tempDir, "foo.share") FileUtils.writeStringToFile( profileFile, s"""{ | "shareCredentialsVersion": 1, | "endpoint": "https://localhost:12345/not-used-endpoint", | "bearerToken": "mock" |}""".stripMargin, "utf-8" ) def test(): Unit = { val profilePath = profileFile.getCanonicalPath val tablePath = new Path(s"$profilePath#$shareName.$schemaName.$sharedTableName") val client = DeltaSharingRestClient(profilePath, Map.empty, false, "delta") val dsTable = Table(share = shareName, schema = schemaName, name = sharedTableName) val options = new DeltaSharingOptions(Map("path" -> tablePath.toString)) DeltaSharingCDFUtils.prepareCDFRelation( SparkSession.active.sqlContext, options, dsTable, client ) val preSignedUrlCacheRef = PreSignedUrlCache.getEndpointRefInExecutor(SparkEnv.get) val path = options.options.getOrElse( "path", throw DeltaSharingErrors.pathNotSpecifiedException ) val fetcher = new PreSignedUrlFetcher( preSignedUrlCacheRef, DeltaSharingUtils.getTablePathWithIdSuffix( path, DeltaSharingUtils.getQueryParamsHashId(options.cdfOptions) ), fileStr1Id, 1000 ) // sleep for 25000ms to ensure that the urls are refreshed. Thread.sleep(25000) // Verify that the url is refreshed as paths(1), not paths(0) anymore. assert(fetcher.getUrl == paths(1)) } withSQLConf( "spark.delta.sharing.client.class" -> classOf[TestDeltaSharingClientForCDFUtils].getName, "fs.delta-sharing-log.impl" -> classOf[DeltaSharingLogFileSystem].getName, "spark.delta.sharing.profile.provider.class" -> "io.delta.sharing.client.DeltaSharingFileProfileProvider", SparkConfForReturnExpTime -> "true" ) { test() } withSQLConf( "spark.delta.sharing.client.class" -> classOf[TestDeltaSharingClientForCDFUtils].getName, "fs.delta-sharing-log.impl" -> classOf[DeltaSharingLogFileSystem].getName, "spark.delta.sharing.profile.provider.class" -> "io.delta.sharing.client.DeltaSharingFileProfileProvider", SparkConfForReturnExpTime -> "false" ) { test() } } } } ================================================ FILE: sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingDataSourceCMSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark import java.io.File import org.apache.spark.sql.delta.{ BatchCDFSchemaEndVersion, BatchCDFSchemaLatest, BatchCDFSchemaLegacy, DeltaUnsupportedOperationException } import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.sharing.DeltaSharingTestSparkUtils import org.apache.spark.sql.functions.col import org.apache.spark.sql.streaming.{StreamingQueryException, StreamTest, Trigger} import org.apache.spark.sql.types.{IntegerType, StringType, StructType} // Unit tests to verify that delta format sharing support column mapping (CM). class DeltaSharingDataSourceCMSuite extends StreamTest with DeltaSQLCommandTest with DeltaSharingTestSparkUtils with DeltaSharingDataSourceDeltaTestUtils { import testImplicits._ override def beforeEach(): Unit = { super.beforeEach() spark.conf.set("spark.databricks.delta.streaming.allowSourceColumnRenameAndDrop", "false") } private def testReadCMTable( deltaTableName: String, sharedTablePath: String, dropC1: Boolean = false): Unit = { val expectedSchema: StructType = if (deltaTableName == "cm_id_table") { spark.read.format("delta").table(deltaTableName).schema } else { if (dropC1) { new StructType() .add("c2rename", StringType) } else { new StructType() .add("c1", IntegerType) .add("c2rename", StringType) } } assert( expectedSchema == spark.read .format("deltaSharing") .option("responseFormat", "delta") .load(sharedTablePath) .schema ) val sharingDf = spark.read.format("deltaSharing").option("responseFormat", "delta").load(sharedTablePath) val deltaDf = spark.read.format("delta").table(deltaTableName) checkAnswer(sharingDf, deltaDf) assert(sharingDf.count() > 0) val filteredSharingDf = spark.read .format("deltaSharing") .option("responseFormat", "delta") .load(sharedTablePath) .filter(col("c2rename") === "one") val filteredDeltaDf = spark.read .format("delta") .table(deltaTableName) .filter(col("c2rename") === "one") checkAnswer(filteredSharingDf, filteredDeltaDf) assert(filteredSharingDf.count() > 0) } private def testReadCMCdf( deltaTableName: String, sharedTablePath: String, startingVersion: Int): Unit = { val schema = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("readChangeFeed", "true") .option("startingVersion", startingVersion) .load(sharedTablePath) .schema val expectedSchema = spark.read .format("delta") .option("readChangeFeed", "true") .option("startingVersion", startingVersion) .table(deltaTableName) .schema assert(expectedSchema == schema) val deltaDf = spark.read .format("delta") .option("readChangeFeed", "true") .option("startingVersion", startingVersion) .table(deltaTableName) val sharingDf = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("readChangeFeed", "true") .option("startingVersion", startingVersion) .load(sharedTablePath) if (startingVersion <= 2) { Seq(BatchCDFSchemaEndVersion, BatchCDFSchemaLatest, BatchCDFSchemaLegacy).foreach { m => withSQLConf( DeltaSQLConf.DELTA_CDF_DEFAULT_SCHEMA_MODE_FOR_COLUMN_MAPPING_TABLE.key -> m.name ) { val deltaException = intercept[DeltaUnsupportedOperationException] { deltaDf.collect() } assert( deltaException.getMessage.contains("Retrieving table changes between") && deltaException.getMessage.contains("failed because of an incompatible") ) val sharingException = intercept[DeltaUnsupportedOperationException] { sharingDf.collect() } assert( sharingException.getMessage.contains("Retrieving table changes between") && sharingException.getMessage.contains("failed because of an incompatible") ) } } } else { checkAnswer(sharingDf, deltaDf) assert(sharingDf.count() > 0) } } private def testReadingSharedCMTable( tempDir: File, deltaTableName: String, sharedTableNameBase: String): Unit = { val sharedTableNameBasic = sharedTableNameBase + "_one" prepareMockedClientAndFileSystemResult( deltaTable = deltaTableName, sharedTable = sharedTableNameBasic ) prepareMockedClientGetTableVersion(deltaTableName, sharedTableNameBasic) withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) testReadCMTable( deltaTableName = deltaTableName, sharedTablePath = s"${profileFile.getCanonicalPath}#share1.default.$sharedTableNameBasic" ) } val sharedTableNameCdf = sharedTableNameBase + "_cdf" // Test CM and CDF // Error when reading cdf with startingVersion <= 2, matches delta behavior. prepareMockedClientGetTableVersion(deltaTableName, sharedTableNameCdf) prepareMockedClientAndFileSystemResultForCdf( deltaTableName, sharedTableNameCdf, startingVersion = 0 ) prepareMockedClientAndFileSystemResultForCdf( deltaTableName, sharedTableNameCdf, startingVersion = 2 ) prepareMockedClientAndFileSystemResultForCdf( deltaTableName, sharedTableNameCdf, startingVersion = 3 ) withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) testReadCMCdf( deltaTableName, s"${profileFile.getCanonicalPath}#share1.default.$sharedTableNameCdf", 0 ) testReadCMCdf( deltaTableName, s"${profileFile.getCanonicalPath}#share1.default.$sharedTableNameCdf", 2 ) testReadCMCdf( deltaTableName, s"${profileFile.getCanonicalPath}#share1.default.$sharedTableNameCdf", 3 ) } val sharedTableNameDrop = sharedTableNameBase + "_drop" // DROP COLUMN sql(s"ALTER TABLE $deltaTableName DROP COLUMN c1") prepareMockedClientGetTableVersion(deltaTableName, sharedTableNameDrop) prepareMockedClientAndFileSystemResult( deltaTable = deltaTableName, sharedTable = sharedTableNameDrop ) prepareMockedClientGetTableVersion(deltaTableName, sharedTableNameDrop) withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) testReadCMTable( deltaTableName = deltaTableName, sharedTablePath = s"${profileFile.getCanonicalPath}#share1.default.$sharedTableNameDrop", dropC1 = true ) } } /** * column mapping tests */ test( "DeltaSharingDataSource able to read data for cm name mode" ) { withTempDir { tempDir => val deltaTableName = "delta_table_cm_name" withTable(deltaTableName) { createSimpleTable(deltaTableName, enableCdf = true) sql(s"""INSERT INTO $deltaTableName VALUES (1, "one"), (2, "one")""") spark.sql( s"""ALTER TABLE $deltaTableName SET TBLPROPERTIES('delta.minReaderVersion' = '2', |'delta.minWriterVersion' = '5', |'delta.columnMapping.mode' = 'name')""".stripMargin ) sql(s"""ALTER TABLE $deltaTableName RENAME COLUMN c2 TO c2rename""") sql(s"""INSERT INTO $deltaTableName VALUES (1, "two"), (2, "two")""") sql(s"""DELETE FROM $deltaTableName where c1=1""") sql(s"""UPDATE $deltaTableName set c1="3" where c2rename="one"""") val sharedTableName = "shared_table_cm_name" testReadingSharedCMTable(tempDir, deltaTableName, sharedTableName) } } } test("DeltaSharingDataSource able to read data for cm id mode") { withTempDir { tempDir => val deltaTableName = "delta_table_cm_id" withTable(deltaTableName) { createCMIdTableWithCdf(deltaTableName) sql(s"""INSERT INTO $deltaTableName VALUES (1, "one"), (2, "one")""") sql(s"""INSERT INTO $deltaTableName VALUES (1, "two"), (2, "two")""") sql(s"""ALTER TABLE $deltaTableName RENAME COLUMN c2 TO c2rename""") sql(s"""INSERT INTO $deltaTableName VALUES (1, "two"), (2, "two")""") sql(s"""DELETE FROM $deltaTableName where c1=1""") sql(s"""UPDATE $deltaTableName set c1="3" where c2rename="one"""") val sharedTableName = "shared_table_cm_id" testReadingSharedCMTable(tempDir, deltaTableName, sharedTableName) } } } /** * Streaming Test */ private def InsertToDeltaTable(tableName: String, values: String): Unit = { sql(s"INSERT INTO $tableName VALUES $values") } private def processAllAvailableInStream( tablePath: String, checkpointDirStr: String, outputDirStr: String): Unit = { val q = spark.readStream .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .writeStream .format("delta") .option("checkpointLocation", checkpointDirStr) .option("mergeSchema", "true") .start(outputDirStr) try { q.processAllAvailable() } finally { q.stop() } } private def processStreamWithSchemaTracking( tablePath: String, checkpointDirStr: String, outputDirStr: String, trigger: Option[Trigger] = None, maxFilesPerTrigger: Option[Int] = None): Unit = { var dataStreamReader = spark.readStream .format("deltaSharing") .option("schemaTrackingLocation", checkpointDirStr) .option("responseFormat", "delta") if (maxFilesPerTrigger.isDefined || trigger.isDefined) { // When trigger.Once is defined, maxFilesPerTrigger is ignored -- this is the // behavior of the streaming engine. And AvailableNow is converted as Once for delta sharing. dataStreamReader = dataStreamReader.option("maxFilesPerTrigger", maxFilesPerTrigger.getOrElse(1)) } var dataStreamWriter = dataStreamReader .load(tablePath) .writeStream .format("delta") .option("checkpointLocation", checkpointDirStr) .option("mergeSchema", "true") if (trigger.isDefined) { dataStreamWriter = dataStreamWriter.trigger(trigger.get) } val q = dataStreamWriter.start(outputDirStr) try { q.processAllAvailable() if (maxFilesPerTrigger.isDefined && trigger.isEmpty) { val progress = q.recentProgress.filter(_.numInputRows != 0) // 2 batches -- 2 files are processed, this is how the delta table is constructed. assert(progress.length === 2) progress.foreach { p => assert(p.numInputRows === 2) // 2 rows per batch -- 2 rows in each file. } } } finally { q.stop() } } private def prepareProcessAndCheckInitSnapshot( deltaTableName: String, sharedTableName: String, sharedTablePath: String, checkpointDirStr: String, outputDir: File, useSchemaTracking: Boolean, trigger: Option[Trigger] = None ): Unit = { InsertToDeltaTable(deltaTableName, """(1, "one"), (2, "one"), (1, "two")""") prepareMockedClientAndFileSystemResult( deltaTable = deltaTableName, sharedTable = sharedTableName, versionAsOf = Some(1L) ) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientMetadata(deltaTableName, sharedTableName) if (useSchemaTracking) { processStreamWithSchemaTracking( sharedTablePath, checkpointDirStr, outputDir.toString, trigger ) } else { processAllAvailableInStream( sharedTablePath, checkpointDirStr, outputDir.toString ) } checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq((1, "one"), (2, "one"), (1, "two")).toDF() ) } def prepareNewInsert( deltaTableName: String, sharedTableName: String, values: String, startingVersion: Long, endingVersion: Long): Unit = { InsertToDeltaTable(deltaTableName, values) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, startingVersion, endingVersion ) } private def renameColumnAndPrepareRpcResponse( deltaTableName: String, sharedTableName: String, startingVersion: Long, endingVersion: Long, insertAfterRename: Boolean): Unit = { // Rename on the original delta table. sql(s"""ALTER TABLE $deltaTableName RENAME COLUMN c2 TO c2rename""") if (insertAfterRename) { InsertToDeltaTable(deltaTableName, """(1, "three")""") InsertToDeltaTable(deltaTableName, """(2, "three")""") } // Prepare all the delta sharing rpcs. prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientMetadata(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, startingVersion, endingVersion ) } private def expectUseSchemaLogException( tablePath: String, checkpointDirStr: String, outputDirStr: String): Unit = { val error = intercept[StreamingQueryException] { processAllAvailableInStream( tablePath, checkpointDirStr, outputDirStr ) }.toString assert(error.contains("DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE_USE_SCHEMA_LOG")) assert(error.contains("Please provide a 'schemaTrackingLocation'")) } private def expectMetadataEvolutionException( tablePath: String, checkpointDirStr: String, outputDirStr: String, trigger: Option[Trigger] = None, maxFilesPerTrigger: Option[Int] = None): Unit = { val error = intercept[StreamingQueryException] { processStreamWithSchemaTracking( tablePath, checkpointDirStr, outputDirStr, trigger, maxFilesPerTrigger ) }.toString assert(error.contains("DELTA_STREAMING_METADATA_EVOLUTION")) assert(error.contains("Please restart the stream to continue")) } private def expectSqlConfException( tablePath: String, checkpointDirStr: String, outputDirStr: String, trigger: Option[Trigger] = None, maxFilesPerTrigger: Option[Int] = None): Unit = { val error = intercept[StreamingQueryException] { processStreamWithSchemaTracking( tablePath, checkpointDirStr, outputDirStr, trigger, maxFilesPerTrigger ) }.toString assert(error.contains("DELTA_STREAMING_CANNOT_CONTINUE_PROCESSING_POST_SCHEMA_EVOLUTION")) assert(error.contains("delta.streaming.allowSourceColumnRename") || error.contains("delta.streaming.allowSourceColumnDrop")) } private def processWithSqlConf( tablePath: String, checkpointDirStr: String, outputDirStr: String, trigger: Option[Trigger] = None, maxFilesPerTrigger: Option[Int] = None): Unit = { // Using allowSourceColumnRenameAndDrop instead of // allowSourceColumnRenameAndDrop.[checkpoint_hash] because the checkpointDir changes // every test. spark.conf .set("spark.databricks.delta.streaming.allowSourceColumnRenameAndDrop", "always") processStreamWithSchemaTracking( tablePath, checkpointDirStr, outputDirStr, trigger, maxFilesPerTrigger ) } private def testRestartStreamingFourTimes( tablePath: String, checkpointDir: java.io.File, outputDirStr: String): Unit = { val checkpointDirStr = checkpointDir.toString // 1. Followed the previous error message to use schemaTrackingLocation, but received // error suggesting restart. expectMetadataEvolutionException(tablePath, checkpointDirStr, outputDirStr) // 2. Followed the previous error message to restart, but need to restart again for // DeltaSource to handle offset movement, this is the SAME behavior as stream reading from // the delta table directly. expectMetadataEvolutionException(tablePath, checkpointDirStr, outputDirStr) // 3. Followed the previous error message to restart, but cannot write to the dest table. expectSqlConfException(tablePath, checkpointDirStr, outputDirStr) // 4. Restart with new sqlConf, able to process new data and writing to a new column. // Not using allowSourceColumnRenameAndDrop.[checkpoint_hash] because the checkpointDir // changes every test, using allowSourceColumnRenameAndDrop=always instead. processWithSqlConf(tablePath, checkpointDirStr, outputDirStr) } test("cm streaming works with newly added schemaTrackingLocation") { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaTableName = "delta_table_cm_streaming_basic" withTable(deltaTableName) { createCMIdTableWithCdf(deltaTableName) val sharedTableName = "shared_table_cm_streaming_basic" val profileFile = prepareProfileFile(inputDir) val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { // 1. Able to stream snapshot at version 1. // The streaming is started without schemaTrackingLocation. prepareProcessAndCheckInitSnapshot( deltaTableName = deltaTableName, sharedTableName = sharedTableName, sharedTablePath = tablePath, checkpointDirStr = checkpointDir.toString, outputDir = outputDir, useSchemaTracking = false ) // 2. Able to stream new data at version 2. // The streaming is continued without schemaTrackingLocation. prepareNewInsert( deltaTableName = deltaTableName, sharedTableName = sharedTableName, values = """(2, "two")""", startingVersion = 2, endingVersion = 2 ) processAllAvailableInStream( tablePath, checkpointDir.toString, outputDir.toString ) checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq((1, "one"), (2, "one"), (1, "two"), (2, "two")).toDF() ) // 3. column renaming at version 3, and expect exception. renameColumnAndPrepareRpcResponse( deltaTableName = deltaTableName, sharedTableName = sharedTableName, startingVersion = 2, endingVersion = 3, insertAfterRename = false ) expectUseSchemaLogException(tablePath, checkpointDir.toString, outputDir.toString) // 4. insert new data at version 4. prepareNewInsert( deltaTableName = deltaTableName, sharedTableName = sharedTableName, values = """(1, "three"), (2, "three")""", startingVersion = 2, endingVersion = 4 ) // Additional preparation for rpc because deltaSource moved the offset to (3, -20) and // (3, -19) after restart. prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 3, 4 ) // 5. with 4 restarts, able to continue the streaming // The streaming is re-started WITH schemaTrackingLocation, and it's able to capture the // schema used in previous version, based on the initial call of getBatch for the latest // offset, which pulls the metadata from the server. testRestartStreamingFourTimes(tablePath, checkpointDir, outputDir.toString) // An additional column is added to the output table. checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq( (1, "one", null), (2, "one", null), (1, "two", null), (2, "two", null), (1, null, "three"), (2, null, "three") ).toDF() ) } } } } test("cm streaming works with restart on snapshot query") { // The main difference in this test is the rename happens after processing the initial snapshot, // (instead of after making continuous progress), to test that the restart could fetch the // latest metadata and the metadata from lastest offset. withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaTableName = "delta_table_streaming_restart" withTable(deltaTableName) { createCMIdTableWithCdf(deltaTableName) val sharedTableName = "shared_table_streaming_restart" val profileFile = prepareProfileFile(inputDir) val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { // 1. Able to stream snapshot at version 1. prepareProcessAndCheckInitSnapshot( deltaTableName = deltaTableName, sharedTableName = sharedTableName, sharedTablePath = tablePath, checkpointDirStr = checkpointDir.toString, outputDir = outputDir, useSchemaTracking = false ) // 2. column renaming at version 2, and expect exception. renameColumnAndPrepareRpcResponse( deltaTableName = deltaTableName, sharedTableName = sharedTableName, startingVersion = 2, endingVersion = 2, insertAfterRename = false ) expectUseSchemaLogException(tablePath, checkpointDir.toString, outputDir.toString) // 3. insert new data at version 3. prepareNewInsert( deltaTableName = deltaTableName, sharedTableName = sharedTableName, values = """(1, "three"), (2, "three")""", startingVersion = 2, endingVersion = 3 ) // 4. with 4 restarts, able to continue the streaming testRestartStreamingFourTimes(tablePath, checkpointDir, outputDir.toString) // An additional column is added to the output table. checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq( (1, "one", null), (2, "one", null), (1, "two", null), (1, null, "three"), (2, null, "three") ).toDF() ) } } } } test("cm streaming works with schemaTracking used at start") { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaTableName = "delta_table_streaming_schematracking" withTable(deltaTableName) { createCMIdTableWithCdf(deltaTableName) val sharedTableName = "shared_table_streaming_schematracking" val profileFile = prepareProfileFile(inputDir) val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { // 1. Able to stream snapshot at version 1. prepareProcessAndCheckInitSnapshot( deltaTableName = deltaTableName, sharedTableName = sharedTableName, sharedTablePath = tablePath, checkpointDirStr = checkpointDir.toString, outputDir = outputDir, useSchemaTracking = true ) // 2. Able to stream new data at version 2. prepareNewInsert( deltaTableName = deltaTableName, sharedTableName = sharedTableName, values = """(2, "two")""", startingVersion = 2, endingVersion = 2 ) processStreamWithSchemaTracking( tablePath, checkpointDir.toString, outputDir.toString ) checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq((1, "one"), (2, "one"), (1, "two"), (2, "two")).toDF() ) // 3. column renaming at version 3, and expect exception. renameColumnAndPrepareRpcResponse( deltaTableName = deltaTableName, sharedTableName = sharedTableName, startingVersion = 2, endingVersion = 3, insertAfterRename = false ) expectMetadataEvolutionException(tablePath, checkpointDir.toString, outputDir.toString) // 4. First see exception, then with sql conf, able to stream new data at version 4. prepareNewInsert( deltaTableName = deltaTableName, sharedTableName = sharedTableName, values = """(1, "three"), (2, "three")""", startingVersion = 3, endingVersion = 4 ) expectSqlConfException(tablePath, checkpointDir.toString, outputDir.toString) processWithSqlConf(tablePath, checkpointDir.toString, outputDir.toString) checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq( (1, "one", null), (2, "one", null), (1, "two", null), (2, "two", null), (1, null, "three"), (2, null, "three") ).toDF() ) } } } } test("cm streaming works with restart with accumulated inserts after rename") { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaTableName = "delta_table_streaming_accumulate" withTable(deltaTableName) { createCMIdTableWithCdf(deltaTableName) val sharedTableName = "shared_table_streaming_accumulate" val profileFile = prepareProfileFile(inputDir) val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { // 1. Able to stream snapshot at version 1. prepareProcessAndCheckInitSnapshot( deltaTableName = deltaTableName, sharedTableName = sharedTableName, sharedTablePath = tablePath, checkpointDirStr = checkpointDir.toString, outputDir = outputDir, useSchemaTracking = false ) // 2. column renaming at version 2, and expect exception. renameColumnAndPrepareRpcResponse( deltaTableName = deltaTableName, sharedTableName = sharedTableName, startingVersion = 2, endingVersion = 4, insertAfterRename = true ) expectUseSchemaLogException(tablePath, checkpointDir.toString, outputDir.toString) // 4. with 4 restarts, able to continue the streaming testRestartStreamingFourTimes(tablePath, checkpointDir, outputDir.toString) // An additional column is added to the output table. checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq( (1, "one", null), (2, "one", null), (1, "two", null), (1, null, "three"), (2, null, "three") ).toDF() ) } } } } test("cm streaming works with column drop and add") { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaTableName = "delta_table_column_drop" withTable(deltaTableName) { createCMIdTableWithCdf(deltaTableName) val sharedTableName = "shared_table_column_drop" val profileFile = prepareProfileFile(inputDir) val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { // 1. Able to stream snapshot at version 1. prepareProcessAndCheckInitSnapshot( deltaTableName = deltaTableName, sharedTableName = sharedTableName, sharedTablePath = tablePath, checkpointDirStr = checkpointDir.toString, outputDir = outputDir, useSchemaTracking = true ) // 2. drop column c1 at version 2 sql(s"ALTER TABLE $deltaTableName DROP COLUMN c1") // 3. add column c3 at version 3 sql(s"ALTER TABLE $deltaTableName ADD COLUMN (c3 int)") prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientMetadata(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 2, 3 ) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 3, 3 ) // Needs a 3 restarts for deltaSource to catch up. expectMetadataEvolutionException(tablePath, checkpointDir.toString, outputDir.toString) expectSqlConfException(tablePath, checkpointDir.toString, outputDir.toString) spark.conf .set("spark.databricks.delta.streaming.allowSourceColumnRenameAndDrop", "always") expectMetadataEvolutionException(tablePath, checkpointDir.toString, outputDir.toString) processWithSqlConf(tablePath, checkpointDir.toString, outputDir.toString) // 4. insert at version 4 InsertToDeltaTable(deltaTableName, """("four", 4)""") // 5. insert at version 5 InsertToDeltaTable(deltaTableName, """("five", 5)""") prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 3, 5 ) processStreamWithSchemaTracking( tablePath, checkpointDir.toString, outputDir.toString ) checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq[(java.lang.Integer, String, java.lang.Integer)]( (1, "one", null), (2, "one", null), (1, "two", null), (null, "four", 4), (null, "five", 5) ).toDF() ) } } } } test("streaming works with column type widened") { // Technically not a column mapping test, but type widening and column mapping are handled in // the same way in DeltaSource, as non-additive schema changes. withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaTableName = "delta_table_column_type_widened" withTable(deltaTableName) { sql(s"""CREATE TABLE $deltaTableName (c1 BYTE, c2 STRING) USING DELTA PARTITIONED BY (c2) |TBLPROPERTIES ('delta.enableTypeWidening' = 'true') |""".stripMargin) val sharedTableName = "shared_table_column_type_widened" val profileFile = prepareProfileFile(inputDir) val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { // 1. Able to stream snapshot at version 1. prepareProcessAndCheckInitSnapshot( deltaTableName = deltaTableName, sharedTableName = sharedTableName, sharedTablePath = tablePath, checkpointDirStr = checkpointDir.toString, outputDir = outputDir, useSchemaTracking = true ) // Enable type widening on the sink to automatically change the type when writing to it // after widening the type in the source. sql(s"""ALTER TABLE delta.`$outputDir` |SET TBLPROPERTIES ('delta.enableTypeWidening' = 'true') |""".stripMargin) // 2. change column type at version 2 sql(s"ALTER TABLE $deltaTableName ALTER COLUMN c1 TYPE INT") prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientMetadata(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 2, 2 ) // Needs 3 restarts for deltaSource to catch up. expectMetadataEvolutionException(tablePath, checkpointDir.toString, outputDir.toString) val error = intercept[StreamingQueryException] { processStreamWithSchemaTracking(tablePath, checkpointDir.toString, outputDir.toString) }.toString() assert(error.contains("DELTA_STREAMING_CANNOT_CONTINUE_PROCESSING_POST_SCHEMA_EVOLUTION")) assert(error.contains("TYPE WIDENING")) assert(error.contains("delta.streaming.allowSourceColumnTypeChange")) // Unblocking allows the type change to go through spark.conf.set("spark.databricks.delta.streaming.allowSourceColumnTypeChange", "always") processStreamWithSchemaTracking(tablePath, checkpointDir.toString, outputDir.toString) // 3. insert at version 3 InsertToDeltaTable(deltaTableName, s"""(${Int.MaxValue}, "max")""") prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 2, 3 ) processStreamWithSchemaTracking( tablePath, checkpointDir.toString, outputDir.toString ) checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq[(java.lang.Integer, String)]( (1, "one"), (2, "one"), (1, "two"), (Int.MaxValue, "max") ).toDF() ) } } } } test("cm streaming works with MaxFilesPerTrigger") { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaTableName = "delta_table_maxfiles" withTable(deltaTableName) { createCMIdTableWithCdf(deltaTableName) val sharedTableName = "shared_table_maxfiles" val profileFile = prepareProfileFile(inputDir) val tablePath = profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { // 1. Able to stream snapshot at version 1. InsertToDeltaTable(deltaTableName, """(1, "one"), (2, "one"), (1, "two"), (2, "two")""") prepareMockedClientAndFileSystemResult( deltaTable = deltaTableName, sharedTable = sharedTableName, versionAsOf = Some(1L) ) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientMetadata(deltaTableName, sharedTableName) // process with maxFilesPerTrigger. processStreamWithSchemaTracking( tablePath, checkpointDir.toString, outputDir.toString, trigger = None, maxFilesPerTrigger = Some(1) ) checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq((1, "one"), (2, "one"), (1, "two"), (2, "two")).toDF() ) // 2. column renaming at version 2, no exception because of Trigger.Once. sql(s"""ALTER TABLE $deltaTableName RENAME COLUMN c2 TO c2rename""") // Prepare all the delta sharing rpcs. prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientMetadata(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, startingVersion = 1, endingVersion = 2 ) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, startingVersion = 2, endingVersion = 2 ) // maxFilesPerTrigger doesn't change whether exception is thrown or not. expectMetadataEvolutionException( tablePath, checkpointDir.toString, outputDir.toString, trigger = None, maxFilesPerTrigger = Some(1) ) // 4. First see exception, then with sql conf, able to stream new data at version 4 and 5. InsertToDeltaTable( deltaTableName, """(1, "three"), (2, "three"), (1, "four"), (2, "four")""" ) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForStreaming( deltaTableName, sharedTableName, 2, 3 ) expectSqlConfException( tablePath, checkpointDir.toString, outputDir.toString, trigger = None, maxFilesPerTrigger = Some(1) ) processWithSqlConf( tablePath, checkpointDir.toString, outputDir.toString, trigger = None, maxFilesPerTrigger = Some(1) ) checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq( (1, "one", null), (2, "one", null), (1, "two", null), (2, "two", null), (1, null, "three"), (2, null, "three"), (1, null, "four"), (2, null, "four") ).toDF() ) } } } } } ================================================ FILE: sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingDataSourceDeltaSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark // scalastyle:off import.ordering.noEmptyLine import scala.concurrent.duration._ import org.apache.spark.sql.delta.{DeltaConfigs, VariantShreddingPreviewTableFeature, VariantTypePreviewTableFeature, VariantTypeTableFeature} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.{DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.delta.sharing.DeltaSharingTestSparkUtils import org.apache.spark.sql.functions.col import org.apache.spark.sql.types.{ DateType, IntegerType, LongType, StringType, StructType, TimestampNTZType, TimestampType } trait DeltaSharingDataSourceDeltaSuiteBase extends QueryTest with DeltaSQLCommandTest with DeltaSharingTestSparkUtils with DeltaSharingDataSourceDeltaTestUtils { override def beforeEach(): Unit = { spark.sessionState.conf.setConfString( "spark.delta.sharing.jsonPredicateV2Hints.enabled", "false" ) } /** * metadata tests */ test("failed to getMetadata") { withTempDir { tempDir => val sharedTableName = "shared_table_broken_json" def test(tablePath: String, tableFullName: String): Unit = { DeltaSharingUtils.overrideIteratorBlock[String]( blockId = TestClientForDeltaFormatSharing.getBlockId(sharedTableName, "getMetadata"), values = Seq("bad protocol string", "bad metadata string").toIterator ) DeltaSharingUtils.overrideSingleBlock[Long]( blockId = TestClientForDeltaFormatSharing.getBlockId(sharedTableName, "getTableVersion"), value = 1 ) // JsonParseException on "bad protocol string" val exception = intercept[com.fasterxml.jackson.core.JsonParseException] { spark.read.format("deltaSharing").option("responseFormat", "delta").load(tablePath).schema } assert(exception.getMessage.contains("Unrecognized token 'bad'")) // table_with_broken_protocol // able to parse as a DeltaSharingSingleAction, but it's an addFile, not metadata. DeltaSharingUtils.overrideIteratorBlock[String]( blockId = TestClientForDeltaFormatSharing.getBlockId(sharedTableName, "getMetadata"), // scalastyle:off line.size.limit values = Seq( """{"add": {"path":"random","id":"random","partitionValues":{},"size":1,"motificationTime":1,"dataChange":false}}""" ).toIterator ) val exception2 = intercept[IllegalStateException] { spark.read.format("deltaSharing").option("responseFormat", "delta").load(tablePath).schema } assert( exception2.getMessage .contains(s"Failed to get Protocol for $tableFullName") ) // table_with_broken_metadata // able to parse as a DeltaSharingSingleAction, but it's an addFile, not metadata. DeltaSharingUtils.overrideIteratorBlock[String]( blockId = TestClientForDeltaFormatSharing.getBlockId(sharedTableName, "getMetadata"), values = Seq( """{"protocol":{"minReaderVersion":1}}""" ).toIterator ) val exception3 = intercept[IllegalStateException] { spark.read.format("deltaSharing").option("responseFormat", "delta").load(tablePath).schema } assert( exception3.getMessage .contains(s"Failed to get Metadata for $tableFullName") ) } withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) val tableFullName = s"share1.default.$sharedTableName" test(s"${profileFile.getCanonicalPath}#$tableFullName", tableFullName) } } } def assertLimit(tableName: String, expectedLimit: Seq[Long]): Unit = { assert(expectedLimit == TestClientForDeltaFormatSharing.limits.filter(_._1.contains(tableName)).map(_._2)) } def assertJsonPredicateHints(tableName: String, expectedHints: Seq[String]): Unit = { assert(expectedHints == TestClientForDeltaFormatSharing.jsonPredicateHints.filter(_._1.contains(tableName)).map(_._2) ) } /** * snapshot queries */ test("DeltaSharingDataSource able to read simple data") { withTempDir { tempDir => val deltaTableName = "delta_table_simple" withTable(deltaTableName) { createTable(deltaTableName) sql( s"INSERT INTO $deltaTableName" + """ VALUES (1, "one", "2023-01-01", "2023-01-01 00:00:00"), |(2, "two", "2023-02-02", "2023-02-02 00:00:00")""".stripMargin ) val sharedTableName = "shared_table_simple" prepareMockedClientAndFileSystemResult(deltaTableName, sharedTableName) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) val expectedSchema: StructType = new StructType() .add("c1", IntegerType) .add("c2", StringType) .add("c3", DateType) .add("c4", TimestampType) val expected = Seq( Row(1, "one", sqlDate("2023-01-01"), sqlTimestamp("2023-01-01 00:00:00")), Row(2, "two", sqlDate("2023-02-02"), sqlTimestamp("2023-02-02 00:00:00")) ) Seq(true, false).foreach { skippingEnabled => Seq(true, false).foreach { sharingConfig => Seq(true, false).foreach { deltaConfig => val sharedTableName = s"shared_table_simple_" + s"${skippingEnabled}_${sharingConfig}_$deltaConfig" prepareMockedClientAndFileSystemResult(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResult(deltaTableName, sharedTableName, limitHint = Some(1)) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) def test(tablePath: String, tableName: String): Unit = { assert( expectedSchema == spark.read .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .schema ) val df = spark.read.format("deltaSharing").option("responseFormat", "delta").load(tablePath) checkAnswer(df, expected) assert(df.count() > 0) assertLimit(tableName, Seq.empty[Long]) val limitDf = spark.read .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .limit(1) assert(limitDf.collect().size == 1) assertLimit(tableName, Some(1L).filter(_ => skippingEnabled && sharingConfig && deltaConfig).toSeq) } val limitPushdownConfigs = Map( "spark.delta.sharing.limitPushdown.enabled" -> sharingConfig.toString, DeltaSQLConf.DELTA_LIMIT_PUSHDOWN_ENABLED.key -> deltaConfig.toString, DeltaSQLConf.DELTA_STATS_SKIPPING.key -> skippingEnabled.toString ) withSQLConf((limitPushdownConfigs ++ getDeltaSharingClassesSQLConf).toSeq: _*) { val profileFile = prepareProfileFile(tempDir) val tableName = s"share1.default.$sharedTableName" test(s"${profileFile.getCanonicalPath}#$tableName", tableName) } } } } } } } test("DeltaSharingDataSource able to read data with changes") { withTempDir { tempDir => val deltaTableName = "delta_table_change" def test(tablePath: String, expectedCount: Int, expectedSchema: StructType): Unit = { assert( expectedSchema == spark.read .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .schema ) val deltaDf = spark.read.format("delta").table(deltaTableName) val sharingDf = spark.read.format("deltaSharing").option("responseFormat", "delta").load(tablePath) checkAnswer(deltaDf, sharingDf) assert(sharingDf.count() == expectedCount) } withTable(deltaTableName) { val sharedTableName = "shared_table_change" createTable(deltaTableName) // test 1: insert 2 rows sql( s"INSERT INTO $deltaTableName" + """ VALUES (1, "one", "2023-01-01", "2023-01-01 00:00:00"), |(2, "two", "2023-02-02", "2023-02-02 00:00:00")""".stripMargin ) prepareMockedClientAndFileSystemResult(deltaTableName, sharedTableName) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) val expectedSchema: StructType = new StructType() .add("c1", IntegerType) .add("c2", StringType) .add("c3", DateType) .add("c4", TimestampType) withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) val tableName = s"share1.default.$sharedTableName" test(s"${profileFile.getCanonicalPath}#$tableName", 2, expectedSchema) } // test 2: insert 2 more rows, and rename a column spark.sql( s"""ALTER TABLE $deltaTableName SET TBLPROPERTIES('delta.minReaderVersion' = '2', |'delta.minWriterVersion' = '5', |'delta.columnMapping.mode' = 'name', 'delta.enableDeletionVectors' = true)""".stripMargin ) sql( s"INSERT INTO $deltaTableName" + """ VALUES (3, "three", "2023-03-03", "2023-03-03 00:00:00"), |(4, "four", "2023-04-04", "2023-04-04 00:00:00")""".stripMargin ) sql(s"""ALTER TABLE $deltaTableName RENAME COLUMN c3 TO c3rename""") prepareMockedClientAndFileSystemResult(deltaTableName, sharedTableName) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) val expectedNewSchema: StructType = new StructType() .add("c1", IntegerType) .add("c2", StringType) .add("c3rename", DateType) .add("c4", TimestampType) withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) val tableName = s"share1.default.$sharedTableName" test(s"${profileFile.getCanonicalPath}#$tableName", 4, expectedNewSchema) } // test 3: delete 1 row sql(s"DELETE FROM $deltaTableName WHERE c1 = 2") prepareMockedClientAndFileSystemResult(deltaTableName, sharedTableName) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) val tableName = s"share1.default.$sharedTableName" test(s"${profileFile.getCanonicalPath}#$tableName", 3, expectedNewSchema) } } } } test("DeltaSharingDataSource able to auto resolve responseFormat") { withTempDir { tempDir => val deltaTableName = "delta_table_auto" withTable(deltaTableName) { createSimpleTable(deltaTableName, enableCdf = false) sql( s"""INSERT INTO $deltaTableName VALUES (1, "one"), (2, "one")""".stripMargin ) sql( s"""INSERT INTO $deltaTableName VALUES (1, "two"), (2, "two")""".stripMargin ) val expectedSchema: StructType = new StructType() .add("c1", IntegerType) .add("c2", StringType) def testAutoResolve(tablePath: String, tableName: String, expectedFormat: String): Unit = { assert( expectedSchema == spark.read .format("deltaSharing") .load(tablePath) .schema ) val deltaDf = spark.read.format("delta").table(deltaTableName) val sharingDf = spark.read.format("deltaSharing").load(tablePath) checkAnswer(deltaDf, sharingDf) assert(sharingDf.count() > 0) assertLimit(tableName, Seq.empty[Long]) assertRequestedFormat(tableName, Seq(expectedFormat)) val limitDf = spark.read .format("deltaSharing") .load(tablePath) .limit(1) assert(limitDf.collect().size == 1) assertLimit(tableName, Seq(1L)) val deltaDfV1 = spark.read.format("delta").option("versionAsOf", 1).table(deltaTableName) val sharingDfV1 = spark.read.format("deltaSharing").option("versionAsOf", 1).load(tablePath) checkAnswer(deltaDfV1, sharingDfV1) assert(sharingDfV1.count() > 0) assertRequestedFormat(tableName, Seq(expectedFormat)) } // Test for delta format response val sharedDeltaTable = "shared_delta_table" prepareMockedClientAndFileSystemResult(deltaTableName, sharedDeltaTable) prepareMockedClientAndFileSystemResult(deltaTableName, sharedDeltaTable, limitHint = Some(1)) prepareMockedClientAndFileSystemResult( deltaTableName, sharedDeltaTable, versionAsOf = Some(1) ) prepareMockedClientGetTableVersion(deltaTableName, sharedDeltaTable) withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) testAutoResolve( s"${profileFile.getCanonicalPath}#share1.default.$sharedDeltaTable", s"share1.default.$sharedDeltaTable", "delta" ) } // Test for parquet format response val sharedParquetTable = "shared_parquet_table" prepareMockedClientAndFileSystemResultForParquet( deltaTableName, sharedParquetTable ) prepareMockedClientAndFileSystemResultForParquet( deltaTableName, sharedParquetTable, limitHint = Some(1) ) prepareMockedClientAndFileSystemResultForParquet( deltaTableName, sharedParquetTable, versionAsOf = Some(1) ) prepareMockedClientGetTableVersion(deltaTableName, sharedParquetTable) withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) testAutoResolve( s"${profileFile.getCanonicalPath}#share1.default.$sharedParquetTable", s"share1.default.$sharedParquetTable", "parquet" ) } // Build a parquet table and query with delta format // Use a unique table name for this test as assertRequestedFormat is using a global map val sharedParquetTableForDeltaFormat = "shared_parquet_table_for_delta_format" // Use prepareMockedClientAndFileSystemResult not ForParquet because fromJson requires DeltaSharingMetadata prepareMockedClientAndFileSystemResult( deltaTableName, sharedParquetTableForDeltaFormat ) prepareMockedClientAndFileSystemResult( deltaTableName, sharedParquetTableForDeltaFormat, limitHint = Some(1) ) prepareMockedClientAndFileSystemResult( deltaTableName, sharedParquetTableForDeltaFormat, versionAsOf = Some(1) ) prepareMockedClientGetTableVersion(deltaTableName, sharedParquetTableForDeltaFormat) val overrideConfigs = Map(DeltaSQLConf.DELTA_SHARING_FORCE_DELTA_FORMAT.key -> "true") withSQLConf((overrideConfigs ++ getDeltaSharingClassesSQLConf).toSeq: _*) { val profileFile = prepareProfileFile(tempDir) testAutoResolve( s"${profileFile.getCanonicalPath}#share1.default.$sharedParquetTableForDeltaFormat", s"share1.default.$sharedParquetTableForDeltaFormat", "delta" ) } } } } test("DeltaSharingDataSource able to read data with filters and select") { withTempDir { tempDir => val deltaTableName = "delta_table_filters" withTable(deltaTableName) { createSimpleTable(deltaTableName, enableCdf = false) sql(s"""INSERT INTO $deltaTableName VALUES (1, "first"), (2, "first")""") sql(s"""INSERT INTO $deltaTableName VALUES (1, "second"), (2, "second")""") sql(s"""INSERT INTO $deltaTableName VALUES (1, "third"), (2, "third")""") Seq("c1", "c2", "c1c2").foreach { filterColumn => val sharedTableName = s"shared_table_filters_$filterColumn" prepareMockedClientAndFileSystemResult(deltaTableName, sharedTableName) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) spark.sessionState.conf.setConfString( "spark.delta.sharing.jsonPredicateV2Hints.enabled", "true" ) // The files returned from delta sharing client are the same for these queries. // This is to test the filters are passed correctly to TahoeLogFileIndex for the local delta // log. def testFiltersAndSelect(tablePath: String, tableName: String): Unit = { // select var expected = Seq(Row(1), Row(1), Row(1), Row(2), Row(2), Row(2)) var df = spark.read .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .select("c1") checkAnswer(df, expected) assertJsonPredicateHints(tableName, Seq.empty[String]) expected = Seq( Row("first"), Row("first"), Row("second"), Row("second"), Row("third"), Row("third") ) df = spark.read .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .select("c2") checkAnswer(df, expected) assertJsonPredicateHints(tableName, Seq.empty[String]) // filter var expectedJson = "" if (filterColumn == "c1c2") { expected = Seq(Row(1, "first"), Row(1, "second"), Row(1, "third"), Row(2, "second")) df = spark.read .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .filter(col("c1") === 1 || col("c2") === "second") checkAnswer(df, expected) expectedJson = """{"op":"or","children":[ | {"op":"equal","children":[ | {"op":"column","name":"c1","valueType":"int"}, | {"op":"literal","value":"1","valueType":"int"}]}, | {"op":"equal","children":[ | {"op":"column","name":"c2","valueType":"string"}, | {"op":"literal","value":"second","valueType":"string"}]} |]}""".stripMargin.replaceAll("\n", "").replaceAll(" ", "") assertJsonPredicateHints(tableName, Seq(expectedJson)) } else if (filterColumn == "c1") { expected = Seq(Row(1, "first"), Row(1, "second"), Row(1, "third")) df = spark.read .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .filter(col("c1") === 1) checkAnswer(df, expected) expectedJson = """{"op":"and","children":[ | {"op":"not","children":[ | {"op":"isNull","children":[ | {"op":"column","name":"c1","valueType":"int"}]}]}, | {"op":"equal","children":[ | {"op":"column","name":"c1","valueType":"int"}, | {"op":"literal","value":"1","valueType":"int"}]} |]}""".stripMargin.replaceAll("\n", "").replaceAll(" ", "") assertJsonPredicateHints(tableName, Seq(expectedJson)) } else { assert(filterColumn == "c2") expected = Seq(Row(1, "second"), Row(2, "second")) expectedJson = """{"op":"and","children":[ | {"op":"not","children":[ | {"op":"isNull","children":[ | {"op":"column","name":"c2","valueType":"string"}]}]}, | {"op":"equal","children":[ | {"op":"column","name":"c2","valueType":"string"}, | {"op":"literal","value":"second","valueType":"string"}]} |]}""".stripMargin.replaceAll("\n", "").replaceAll(" ", "") df = spark.read .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .filter(col("c2") === "second") checkAnswer(df, expected) assertJsonPredicateHints(tableName, Seq(expectedJson)) // filters + select as well expected = Seq(Row(1), Row(2)) df = spark.read .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .filter(col("c2") === "second") .select("c1") checkAnswer(df, expected) assertJsonPredicateHints(tableName, Seq(expectedJson)) } } withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) testFiltersAndSelect( s"${profileFile.getCanonicalPath}#share1.default.$sharedTableName", s"share1.default.$sharedTableName" ) } } } } } test("DeltaSharingDataSource able to read data with different filters") { withTempDir { tempDir => val deltaTableName = "delta_table_diff_filter" withTable(deltaTableName) { createSimpleTable(deltaTableName, enableCdf = false) sql(s"""INSERT INTO $deltaTableName VALUES (1, "first"), (2, "first")""") sql(s"""INSERT INTO $deltaTableName VALUES (1, "second"), (2, "second")""") sql(s"""INSERT INTO $deltaTableName VALUES (1, "third"), (2, "third")""") val sharedTableName = s"shared_table_filters_diff_filter" prepareMockedClientAndFileSystemResult(deltaTableName, sharedTableName, limitHint = Some(2)) prepareMockedClientAndFileSystemResult(deltaTableName, sharedTableName) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) spark.sessionState.conf.setConfString( "spark.delta.sharing.jsonPredicateV2Hints.enabled", "true" ) // The files returned from delta sharing client are the same for these queries. // This is to test the filters are passed correctly to TahoeLogFileIndex for the local delta // log. def testDiffFilter(tablePath: String, tableName: String): Unit = { val df = spark.read .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) // limit assert(df.limit(2).count() == 2) // full val expectedFull = Seq( Row(1, "first"), Row(1, "second"), Row(1, "third"), Row(2, "first"), Row(2, "second"), Row(2, "third") ) checkAnswer(df, expectedFull) } withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) testDiffFilter( s"${profileFile.getCanonicalPath}#share1.default.$sharedTableName", s"share1.default.$sharedTableName" ) } } } } test("DeltaSharingDataSource able to read data for time travel queries") { withTempDir { tempDir => val deltaTableName = "delta_table_time_travel" withTable(deltaTableName) { createTable(deltaTableName) sql( s"INSERT INTO $deltaTableName" + """ VALUES (1, "one", "2023-01-01", "2023-01-01 00:00:00")""".stripMargin ) sql( s"INSERT INTO $deltaTableName" + """ VALUES (2, "two", "2023-02-02", "2023-02-02 00:00:00")""".stripMargin ) sql( s"INSERT INTO $deltaTableName" + """ VALUES (3, "three", "2023-03-03", "2023-03-03 00:00:00")""".stripMargin ) val sharedTableNameV1 = "shared_table_v1" prepareMockedClientAndFileSystemResult( deltaTable = deltaTableName, sharedTable = sharedTableNameV1, versionAsOf = Some(1L) ) def testVersionAsOf1(tablePath: String): Unit = { val dfV1 = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("versionAsOf", 1) .load(tablePath) val expectedV1 = Seq( Row(1, "one", sqlDate("2023-01-01"), sqlTimestamp("2023-01-01 00:00:00")) ) checkAnswer(dfV1, expectedV1) } withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) testVersionAsOf1(s"${profileFile.getCanonicalPath}#share1.default.$sharedTableNameV1") } // using different table name because spark caches the content read from a file, i.e., // the delta log from 0.json. // TODO: figure out how to get a per query id and use it in getCustomTablePath to // differentiate the same table used in different queries. // TODO: Also check if it's possible to disable the file cache. val sharedTableNameV3 = "shared_table_v3" prepareMockedClientAndFileSystemResult( deltaTable = deltaTableName, sharedTable = sharedTableNameV3, versionAsOf = Some(3L) ) def testVersionAsOf3(tablePath: String): Unit = { val dfV3 = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("versionAsOf", 3) .load(tablePath) val expectedV3 = Seq( Row(1, "one", sqlDate("2023-01-01"), sqlTimestamp("2023-01-01 00:00:00")), Row(2, "two", sqlDate("2023-02-02"), sqlTimestamp("2023-02-02 00:00:00")), Row(3, "three", sqlDate("2023-03-03"), sqlTimestamp("2023-03-03 00:00:00")) ) checkAnswer(dfV3, expectedV3) } withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) testVersionAsOf3(s"${profileFile.getCanonicalPath}#share1.default.$sharedTableNameV3") } val sharedTableNameTs = "shared_table_ts" // Given the result of delta sharing rpc is mocked, the actual value of the timestampStr // can be any thing that's valid for DeltaSharingOptions, and formattedTimestamp is the // parsed result and will be sent in the delta sharing rpc. val timestampStr = "2023-01-01 00:00:00" val formattedTimestamp = "2023-01-01T08:00:00Z" prepareMockedClientGetTableVersion(deltaTableName, sharedTableNameTs) prepareMockedClientAndFileSystemResult( deltaTable = deltaTableName, sharedTable = sharedTableNameTs, versionAsOf = None, timestampAsOf = Some(formattedTimestamp) ) def testTimestampQuery(tablePath: String): Unit = { val dfTs = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("timestampAsOf", timestampStr) .load(tablePath) val expectedTs = Seq( Row(1, "one", sqlDate("2023-01-01"), sqlTimestamp("2023-01-01 00:00:00")), Row(2, "two", sqlDate("2023-02-02"), sqlTimestamp("2023-02-02 00:00:00")), Row(3, "three", sqlDate("2023-03-03"), sqlTimestamp("2023-03-03 00:00:00")) ) checkAnswer(dfTs, expectedTs) } withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) testTimestampQuery(s"${profileFile.getCanonicalPath}#share1.default.$sharedTableNameTs") } } } } test("DeltaSharingDataSource able to read data with more entries") { withTempDir { tempDir => val deltaTableName = "delta_table_more" withTable(deltaTableName) { createSimpleTable(deltaTableName, enableCdf = false) // The table operations take about 6~10 seconds. for (i <- 0 to 9) { val iteration = s"iteration $i" val valuesBuilder = Seq.newBuilder[String] for (j <- 0 to 49) { valuesBuilder += s"""(${i * 10 + j}, "$iteration")""" } sql(s"INSERT INTO $deltaTableName VALUES ${valuesBuilder.result().mkString(",")}") } val sharedTableName = "shared_table_more" prepareMockedClientAndFileSystemResult(deltaTableName, sharedTableName) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) val expectedSchema: StructType = new StructType() .add("c1", IntegerType) .add("c2", StringType) val expected = spark.read.format("delta").table(deltaTableName) def test(tablePath: String): Unit = { assert( expectedSchema == spark.read .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .schema ) val df = spark.read.format("deltaSharing").option("responseFormat", "delta").load(tablePath) checkAnswer(df, expected) } withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) test(s"${profileFile.getCanonicalPath}#share1.default.$sharedTableName") } } } } test("DeltaSharingDataSource able to read data with join on the same table") { withTempDir { tempDir => val deltaTableName = "delta_table_join" withTable(deltaTableName) { createSimpleTable(deltaTableName, enableCdf = false) sql(s"""INSERT INTO $deltaTableName VALUES (1, "first"), (2, "first")""") sql(s"""INSERT INTO $deltaTableName VALUES (1, "second"), (2, "second")""") sql(s"""INSERT INTO $deltaTableName VALUES (1, "third"), (2, "third")""") val sharedTableName = "shared_table_join" prepareMockedClientAndFileSystemResult(deltaTableName, sharedTableName) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResult( deltaTableName, sharedTableName, versionAsOf = Some(1L) ) def testJoin(tablePath: String): Unit = { // Query the same latest version val deltaDfLatest = spark.read.format("delta").table(deltaTableName) val deltaDfV1 = spark.read.format("delta").option("versionAsOf", 1).table(deltaTableName) val sharingDfLatest = spark.read.format("deltaSharing").option("responseFormat", "delta").load(tablePath) val sharingDfV1 = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("versionAsOf", 1) .load(tablePath) var deltaDfJoined = deltaDfLatest.join(deltaDfLatest, "c1") var sharingDfJoined = sharingDfLatest.join(sharingDfLatest, "c1") // CheckAnswer ensures that delta sharing produces the same result as delta. // The check on the size is used to double check that a valid dataframe is generated. checkAnswer(deltaDfJoined, sharingDfJoined) assert(sharingDfJoined.count() > 0) // Query the same versionAsOf deltaDfJoined = deltaDfV1.join(deltaDfV1, "c1") sharingDfJoined = sharingDfV1.join(sharingDfV1, "c1") checkAnswer(deltaDfJoined, sharingDfJoined) assert(sharingDfJoined.count() > 0) // Query with different versions deltaDfJoined = deltaDfLatest.join(deltaDfV1, "c1") sharingDfJoined = sharingDfLatest.join(sharingDfV1, "c1") checkAnswer(deltaDfJoined, sharingDfJoined) // Size is 6 because for each of the 6 rows in latest, there is 1 row with the same c1 // value in v1. assert(sharingDfJoined.count() > 0) } withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) testJoin(s"${profileFile.getCanonicalPath}#share1.default.$sharedTableName") } } } } test("DeltaSharingDataSource able to read empty data") { withTempDir { tempDir => val deltaTableName = "delta_table_empty" withTable(deltaTableName) { createSimpleTable(deltaTableName, enableCdf = true) sql(s"""INSERT INTO $deltaTableName VALUES (1, "first"), (2, "first")""") sql(s"""INSERT INTO $deltaTableName VALUES (1, "second"), (2, "second")""") sql(s"DELETE FROM $deltaTableName WHERE c1 <= 2") // This command is just to create an empty table version at version 4. spark.sql(s"ALTER TABLE $deltaTableName SET TBLPROPERTIES('delta.minReaderVersion' = 1)") val sharedTableName = "shared_table_empty" prepareMockedClientAndFileSystemResult(deltaTableName, sharedTableName) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) def testEmpty(tablePath: String): Unit = { val deltaDf = spark.read.format("delta").table(deltaTableName) val sharingDf = spark.read.format("deltaSharing").option("responseFormat", "delta").load(tablePath) checkAnswer(deltaDf, sharingDf) assert(sharingDf.count() == 0) val deltaCdfDf = spark.read .format("delta") .option("readChangeFeed", "true") .option("startingVersion", 4) .table(deltaTableName) val sharingCdfDf = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("readChangeFeed", "true") .option("startingVersion", 4) .load(tablePath) checkAnswer(deltaCdfDf, sharingCdfDf) assert(sharingCdfDf.count() == 0) } // There's only metadata change but not actual files in version 4. prepareMockedClientAndFileSystemResultForCdf( deltaTableName, sharedTableName, startingVersion = 4 ) withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) testEmpty(s"${profileFile.getCanonicalPath}#share1.default.$sharedTableName") } } } } /** * cdf queries */ test("DeltaSharingDataSource able to read data for simple cdf query") { withTempDir { tempDir => val deltaTableName = "delta_table_cdf" withTable(deltaTableName) { sql(s""" |CREATE TABLE $deltaTableName (c1 INT, c2 STRING) USING DELTA PARTITIONED BY (c2) |TBLPROPERTIES (delta.enableChangeDataFeed = true) |""".stripMargin) // 2 inserts in version 1, 1 with c1=2 sql(s"""INSERT INTO $deltaTableName VALUES (1, "one"), (2, "two")""") // 1 insert in version 2, 0 with c1=2 sql(s"""INSERT INTO $deltaTableName VALUES (3, "two")""") // 0 operations in version 3 sql(s"""OPTIMIZE $deltaTableName""") // 2 updates in version 4, 2 with c1=2 sql(s"""UPDATE $deltaTableName SET c2="new two" where c1=2""") // 1 delete in version 5, 1 with c1=2 sql(s"""DELETE FROM $deltaTableName WHERE c1 = 2""") val sharedTableName = "shard_table_cdf" prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) Seq(0, 1, 2, 3, 4, 5).foreach { startingVersion => val ts = getTimeStampForVersion(deltaTableName, startingVersion) val startingTimestamp = DateTimeUtils.toJavaTimestamp(ts * 1000).toInstant.toString prepareMockedClientAndFileSystemResultForCdf( deltaTableName, sharedTableName, startingVersion, Some(startingTimestamp) ) def test(tablePath: String): Unit = { val expectedSchema: StructType = new StructType() .add("c1", IntegerType) .add("c2", StringType) .add("_change_type", StringType) .add("_commit_version", LongType) .add("_commit_timestamp", TimestampType) val schema = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("readChangeFeed", "true") .option("startingVersion", startingVersion) .load(tablePath) .schema assert(expectedSchema == schema) val expected = spark.read .format("delta") .option("readChangeFeed", "true") .option("startingVersion", startingVersion) .table(deltaTableName) val df = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("readChangeFeed", "true") .option("startingVersion", startingVersion) .load(tablePath) checkAnswer(df, expected) assert(df.count() > 0) } def testFiltersAndSelect(tablePath: String): Unit = { val expectedSchema: StructType = new StructType() .add("c2", StringType) .add("_change_type", StringType) .add("_commit_version", LongType) val schema = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("readChangeFeed", "true") .option("startingVersion", startingVersion) .load(tablePath) .select("c2", "_change_type", "_commit_version") .schema assert(expectedSchema == schema) val expected = spark.read .format("delta") .option("readChangeFeed", "true") .option("startingVersion", startingVersion) .table(deltaTableName) .select("c2", "_change_type", "_commit_version") val dfVersion = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("readChangeFeed", "true") .option("startingVersion", startingVersion) .load(tablePath) .select("c2", "_change_type", "_commit_version") checkAnswer(dfVersion, expected) val dfTime = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("readChangeFeed", "true") .option("startingTimestamp", startingTimestamp) .load(tablePath) .select("c2", "_change_type", "_commit_version") checkAnswer(dfTime, expected) assert(dfTime.count() > 0) val expectedFiltered = spark.read .format("delta") .option("readChangeFeed", "true") .option("startingVersion", startingVersion) .table(deltaTableName) .select("c2", "_change_type", "_commit_version") .filter(col("c1") === 2) val dfFiltered = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("readChangeFeed", "true") .option("startingVersion", startingVersion) .load(tablePath) .select("c2", "_change_type", "_commit_version") .filter(col("c1") === 2) checkAnswer(dfFiltered, expectedFiltered) assert(dfFiltered.count() > 0) } withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) test(profileFile.getCanonicalPath + s"#share1.default.$sharedTableName") testFiltersAndSelect( profileFile.getCanonicalPath + s"#share1.default.$sharedTableName" ) } } // test join on the same table in cdf query def testJoin(tablePath: String): Unit = { val deltaV0 = spark.read .format("delta") .option("readChangeFeed", "true") .option("startingVersion", 0) .table(deltaTableName) val deltaV3 = spark.read .format("delta") .option("readChangeFeed", "true") .option("startingVersion", 3) .table(deltaTableName) val sharingV0 = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("readChangeFeed", "true") .option("startingVersion", 0) .load(tablePath) val sharingV3 = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("readChangeFeed", "true") .option("startingVersion", 3) .load(tablePath) def testJoinedDf( deltaLeft: DataFrame, deltaRight: DataFrame, sharingLeft: DataFrame, sharingRight: DataFrame, expectedSize: Int): Unit = { val deltaJoined = deltaLeft.join(deltaRight, usingColumns = Seq("c1", "c2")) val sharingJoined = sharingLeft.join(sharingRight, usingColumns = Seq("c1", "c2")) checkAnswer(deltaJoined, sharingJoined) assert(sharingJoined.count() > 0) } testJoinedDf(deltaV0, deltaV0, sharingV0, sharingV0, 10) testJoinedDf(deltaV3, deltaV3, sharingV3, sharingV3, 5) testJoinedDf(deltaV0, deltaV3, sharingV0, sharingV3, 6) } withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) testJoin(profileFile.getCanonicalPath + s"#share1.default.$sharedTableName") } } } } test("DeltaSharingDataSource able to read data for cdf query with more entries") { withTempDir { tempDir => val deltaTableName = "delta_table_cdf_more" withTable(deltaTableName) { createSimpleTable(deltaTableName, enableCdf = true) // The table operations take about 20~30 seconds. for (i <- 0 to 9) { val iteration = s"iteration $i" val valuesBuilder = Seq.newBuilder[String] for (j <- 0 to 49) { valuesBuilder += s"""(${i * 10 + j}, "$iteration")""" } sql(s"INSERT INTO $deltaTableName VALUES ${valuesBuilder.result().mkString(",")}") sql(s"""UPDATE $deltaTableName SET c1 = c1 + 100 where c2 = "${iteration}"""") sql(s"""DELETE FROM $deltaTableName where c2 = "${iteration}"""") } val sharedTableName = "shard_table_cdf_more" prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) Seq(0, 10, 20, 30).foreach { startingVersion => prepareMockedClientAndFileSystemResultForCdf( deltaTableName, sharedTableName, startingVersion ) val expected = spark.read .format("delta") .option("readChangeFeed", "true") .option("startingVersion", startingVersion) .table(deltaTableName) def test(tablePath: String): Unit = { val df = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("readChangeFeed", "true") .option("startingVersion", startingVersion) .load(tablePath) checkAnswer(df, expected) assert(df.count() > 0) } withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) test(profileFile.getCanonicalPath + s"#share1.default.$sharedTableName") } } } } } test("DeltaSharingDataSource able to read data with special chars") { withTempDir { tempDir => val deltaTableName = "delta_table_special" withTable(deltaTableName) { // scalastyle:off nonascii sql(s"""CREATE TABLE $deltaTableName (`第一列` INT, c2 STRING) |USING DELTA PARTITIONED BY (c2) |""".stripMargin) // The table operations take about 6~10 seconds. for (i <- 0 to 99) { val iteration = s"iteration $i" val valuesBuilder = Seq.newBuilder[String] for (j <- 0 to 99) { valuesBuilder += s"""(${i * 10 + j}, "$iteration")""" } sql(s"INSERT INTO $deltaTableName VALUES ${valuesBuilder.result().mkString(",")}") } val sharedTableName = "shared_table_more" prepareMockedClientAndFileSystemResult(deltaTableName, sharedTableName) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) val expectedSchema: StructType = new StructType() .add("第一列", IntegerType) .add("c2", StringType) // scalastyle:on nonascii val expected = spark.read.format("delta").table(deltaTableName) def test(tablePath: String): Unit = { assert( expectedSchema == spark.read .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .schema ) val df = spark.read.format("deltaSharing").option("responseFormat", "delta").load(tablePath) checkAnswer(df, expected) } withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) test(s"${profileFile.getCanonicalPath}#share1.default.$sharedTableName") } } } } test("DeltaSharingDataSource able to read cdf with special chars") { withTempDir { tempDir => val deltaTableName = "delta_table_cdf_special" withTable(deltaTableName) { // scalastyle:off nonascii sql(s"""CREATE TABLE $deltaTableName (`第一列` INT, c2 STRING) |USING DELTA PARTITIONED BY (c2) |TBLPROPERTIES( |delta.enableChangeDataFeed = true |)""".stripMargin) // The table operations take about 20~30 seconds. for (i <- 0 to 9) { val iteration = s"iteration $i" val valuesBuilder = Seq.newBuilder[String] for (j <- 0 to 49) { valuesBuilder += s"""(${i * 10 + j}, "$iteration")""" } sql(s"INSERT INTO $deltaTableName VALUES ${valuesBuilder.result().mkString(",")}") sql(s"""UPDATE $deltaTableName SET `第一列` = `第一列` + 100 where c2 = "${iteration}"""") // scalastyle:on nonascii sql(s"""DELETE FROM $deltaTableName where c2 = "${iteration}"""") } val sharedTableName = "shard_table_cdf_special" prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) Seq(0, 10, 20, 30).foreach { startingVersion => prepareMockedClientAndFileSystemResultForCdf( deltaTableName, sharedTableName, startingVersion ) val expected = spark.read .format("delta") .option("readChangeFeed", "true") .option("startingVersion", startingVersion) .table(deltaTableName) def test(tablePath: String): Unit = { val df = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("readChangeFeed", "true") .option("startingVersion", startingVersion) .load(tablePath) checkAnswer(df, expected) assert(df.count() > 0) } withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) test(profileFile.getCanonicalPath + s"#share1.default.$sharedTableName") } } } } } /** * deletion vector tests */ test("DeltaSharingDataSource able to read data for dv table") { withTempDir { tempDir => val deltaTableName = "delta_table_dv" withTable(deltaTableName) { spark .range(start = 0, end = 100) .withColumn("partition", col("id").divide(10).cast("int")) .write .partitionBy("partition") .format("delta") .saveAsTable(deltaTableName) spark .range(start = 100, end = 200) .withColumn("partition", col("id").mod(100).divide(10).cast("int")) .write .mode("append") .partitionBy("partition") .format("delta") .saveAsTable(deltaTableName) spark.sql( s"ALTER TABLE $deltaTableName SET TBLPROPERTIES('delta.enableDeletionVectors' = true)" ) // Delete 2 rows per partition. sql(s"""DELETE FROM $deltaTableName where mod(id, 10) < 2""") // Delete 1 more row per partition. sql(s"""DELETE FROM $deltaTableName where mod(id, 10) = 3""") // Delete 1 more row per partition. sql(s"""DELETE FROM $deltaTableName where mod(id, 10) = 6""") Seq(true, false).foreach { skippingEnabled => val sharedTableName = s"shared_table_dv_$skippingEnabled" prepareMockedClientAndFileSystemResult( deltaTable = deltaTableName, sharedTable = sharedTableName, assertMultipleDvsInOneFile = true ) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) def testReadDVTable(tablePath: String): Unit = { val expectedSchema: StructType = new StructType() .add("id", LongType) .add("partition", IntegerType) assert( expectedSchema == spark.read .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .schema ) val sharingDf = spark.read.format("deltaSharing").option("responseFormat", "delta").load(tablePath) val deltaDf = spark.read.format("delta").table(deltaTableName) val filteredSharingDf = spark.read .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .filter(col("id").mod(10) > 5) val filteredDeltaDf = spark.read .format("delta") .table(deltaTableName) .filter(col("id").mod(10) > 5) if (!skippingEnabled) { def assertError(dataFrame: DataFrame): Unit = { val ex = intercept[IllegalArgumentException] { dataFrame.collect() } assert(ex.getMessage contains "Cannot work with a non-pinned table snapshot of the TahoeFileIndex") } assertError(sharingDf) assertError(filteredDeltaDf) } else { checkAnswer(sharingDf, deltaDf) assert(sharingDf.count() > 0) checkAnswer(filteredSharingDf, filteredDeltaDf) assert(filteredSharingDf.count() > 0) } } val additionalConfigs = Map( DeltaSQLConf.DELTA_STATS_SKIPPING.key -> skippingEnabled.toString ) withSQLConf((additionalConfigs ++ getDeltaSharingClassesSQLConf).toSeq: _*) { val profileFile = prepareProfileFile(tempDir) testReadDVTable(s"${profileFile.getCanonicalPath}#share1.default.$sharedTableName") } } } } } test("DeltaSharingDataSource able to read data for dv and cdf") { withTempDir { tempDir => val deltaTableName = "delta_table_dv_cdf" withTable(deltaTableName) { createDVTableWithCdf(deltaTableName) // version 1: 20 inserts spark .range(start = 0, end = 20) .select(col("id").cast("int").as("c1")) .withColumn("partition", col("c1").divide(10).cast("int")) .write .mode("append") .format("delta") .saveAsTable(deltaTableName) // version 2: 20 inserts spark .range(start = 100, end = 120) .select(col("id").cast("int").as("c1")) .withColumn("partition", col("c1").mod(100).divide(10).cast("int")) .write .mode("append") .format("delta") .saveAsTable(deltaTableName) // version 3: 20 updates sql(s"""UPDATE $deltaTableName SET c1=c1+5 where partition=0""") // This deletes will create one DV file used by AddFile from both version 1 and version 2. // version 4: 14 deletes sql(s"""DELETE FROM $deltaTableName WHERE mod(c1, 100)<=10""") val sharedTableName = "shard_table_dv_cdf" prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) Seq(0, 1, 2, 3, 4).foreach { startingVersion => prepareMockedClientAndFileSystemResultForCdf( deltaTableName, sharedTableName, startingVersion, assertMultipleDvsInOneFile = true ) def testReadDVCdf(tablePath: String): Unit = { val schema = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("readChangeFeed", "true") .option("startingVersion", startingVersion) .load(tablePath) .schema val expectedSchema: StructType = new StructType() .add("c1", IntegerType) .add("partition", IntegerType) .add("_change_type", StringType) .add("_commit_version", LongType) .add("_commit_timestamp", TimestampType) assert(expectedSchema == schema) val deltaDf = spark.read .format("delta") .option("readChangeFeed", "true") .option("startingVersion", startingVersion) .table(deltaTableName) val sharingDf = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("readChangeFeed", "true") .option("startingVersion", startingVersion) .load(tablePath) checkAnswer(sharingDf, deltaDf) assert(sharingDf.count() > 0) } withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) testReadDVCdf(s"${profileFile.getCanonicalPath}#share1.default.$sharedTableName") } } } } } test("DeltaSharingDataSource able to read data for inline dv") { import org.apache.spark.sql.delta.deletionvectors.RoaringBitmapArrayFormat Seq(RoaringBitmapArrayFormat.Portable, RoaringBitmapArrayFormat.Native).foreach { format => withTempDir { tempDir => val deltaTableName = s"delta_table_inline_dv_$format" withTable(deltaTableName) { createDVTableWithCdf(deltaTableName) // Use divide 10 to set partition column to 0 for all values, then use repartition to // ensure the 5 values are written in one file. spark .range(start = 0, end = 5) .select(col("id").cast("int").as("c1")) .withColumn("partition", col("c1").divide(10).cast("int")) .repartition(1) .write .mode("append") .format("delta") .saveAsTable(deltaTableName) val sharedTableName = s"shared_table_inline_dv_$format" prepareMockedClientAndFileSystemResult( deltaTable = deltaTableName, sharedTable = sharedTableName, inlineDvFormat = Some(format) ) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForCdf( deltaTableName, sharedTableName, startingVersion = 1, inlineDvFormat = Some(format) ) def testReadInlineDVCdf(tablePath: String): Unit = { val deltaDf = spark.read .format("delta") .option("readChangeFeed", "true") .option("startingVersion", 1) .table(deltaTableName) .filter(col("c1") > 1) val sharingDf = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("readChangeFeed", "true") .option("startingVersion", 1) .load(tablePath) checkAnswer(sharingDf, deltaDf) assert(sharingDf.count() > 0) } def testReadInlineDV(tablePath: String): Unit = { val expectedSchema: StructType = new StructType() .add("c1", IntegerType) .add("partition", IntegerType) assert( expectedSchema == spark.read .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .schema ) val sharingDf = spark.read.format("deltaSharing").option("responseFormat", "delta").load(tablePath) val expectedDf = Seq(Row(1, 0), Row(3, 0), Row(4, 0)) checkAnswer(sharingDf, expectedDf) val filteredSharingDf = spark.read .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .filter(col("c1") < 4) val expectedFilteredDf = Seq(Row(1, 0), Row(3, 0)) checkAnswer(filteredSharingDf, expectedFilteredDf) } withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) testReadInlineDV(s"${profileFile.getCanonicalPath}#share1.default.$sharedTableName") testReadInlineDVCdf(s"${profileFile.getCanonicalPath}#share1.default.$sharedTableName") } } } } } test("DeltaSharingDataSource able to read timestampNTZ table") { withTempDir { tempDir => val deltaTableName = "delta_table_timestampNTZ" withTable(deltaTableName) { sql(s"CREATE TABLE $deltaTableName(c1 TIMESTAMP_NTZ) USING DELTA") sql(s"""INSERT INTO $deltaTableName VALUES ('2022-01-02 03:04:05.123456')""") val sharedTableName = "shared_table_timestampNTZ" prepareMockedClientAndFileSystemResult(deltaTableName, sharedTableName) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) def testReadTimestampNTZ(tablePath: String): Unit = { val expectedSchema: StructType = new StructType() .add("c1", TimestampNTZType) assert( expectedSchema == spark.read .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .schema ) val sharingDf = spark.read.format("deltaSharing").option("responseFormat", "delta").load(tablePath) val deltaDf = spark.read.format("delta").table(deltaTableName) checkAnswer(sharingDf, deltaDf) assert(sharingDf.count() > 0) } withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) testReadTimestampNTZ(s"${profileFile.getCanonicalPath}#share1.default.$sharedTableName") } } } } Seq( VariantTypePreviewTableFeature, VariantTypeTableFeature, VariantShreddingPreviewTableFeature ).foreach { feature => test(s"basic variant test - table feature: $feature") { withTempDir { tempDir => val extraConfs = feature match { case VariantShreddingPreviewTableFeature => Map( "spark.sql.variant.writeShredding.enabled" -> "true", "spark.sql.variant.allowReadingShredded" -> "true", "spark.sql.variant.forceShreddingSchemaForTest" -> "a long" ) case _ => Map.empty } withSQLConf(extraConfs.toSeq: _*) { val deltaTableName = s"variant_table_${feature.name.replaceAll("-", "_")}" withTable(deltaTableName) { if (feature == VariantShreddingPreviewTableFeature) { spark.sql(s"CREATE TABLE $deltaTableName(v variant) USING DELTA " + s"TBLPROPERTIES('${DeltaConfigs.ENABLE_VARIANT_SHREDDING.key}' = 'true')") } else { spark.sql(s"CREATE TABLE $deltaTableName(v variant) USING DELTA " + s"TBLPROPERTIES('delta.feature.${feature.name}' = 'supported')") } spark.range(0, 10000, 1, 1) .selectExpr("""parse_json(format_string('{"a": %d}', id)) v""") .write .format("delta") .mode("append") .insertInto(deltaTableName) val sharedTableName = s"shared_table_variant_${feature.name.replaceAll("-", "_")}" prepareMockedClientAndFileSystemResult(deltaTableName, sharedTableName) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) val expectedSchemaString = "StructType(StructField(v,VariantType,true))" val expected = spark.read.format("delta").table(deltaTableName) def test(tablePath: String): Unit = { val sharedDf = spark.read .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) assert(expectedSchemaString == sharedDf.schema.toString) checkAnswer(sharedDf, expected) } withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) test(s"${profileFile.getCanonicalPath}#share1.default.$sharedTableName") } } } } } } test("DeltaSharingDataSource able to read data with inline credentials") { withTempDir { tempDir => val deltaTableName = "delta_table_inline_creds" withTable(deltaTableName) { createSimpleTable(deltaTableName, enableCdf = false) sql(s"""INSERT INTO $deltaTableName VALUES (1, "one"), (2, "two")""") val sharedTableName = "shared_table_inline_creds" prepareMockedClientAndFileSystemResult(deltaTableName, sharedTableName) prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) val map = Map( "shareCredentialsVersion" -> "1", "bearerToken" -> "xxx", "endpoint" -> "https://xxx/delta-sharing/", "expirationTime" -> "2099-01-01T00:00:00.000Z" ) withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val expectedSchema: StructType = new StructType() .add("c1", IntegerType) .add("c2", StringType) val df = spark.read .format("deltaSharing") .option("responseFormat", "delta") .options(map) .load(s"share1.default.$sharedTableName") assert(expectedSchema == df.schema) val expected = spark.read.format("delta").table(deltaTableName) checkAnswer(df, expected) } } } } test("deleted file retention duration check is not applied for time-travel on delta-sharing tables") { withTempDir { tempDir => val deltaTableName = "delta_table_time_travel_retention" withTable(deltaTableName) { // file and log retention is set to 0 but still able to time-travel because of skipping enforcement. sql(s""" |CREATE TABLE $deltaTableName (c1 INT, c2 STRING) USING DELTA PARTITIONED BY (c2) |TBLPROPERTIES ('delta.deletedFileRetentionDuration' = '0 hours', |'delta.logRetentionDuration' = '0 hours') |""".stripMargin) // Insert multiple versions sql(s"""INSERT INTO $deltaTableName VALUES (1, "one")""") sql(s"""INSERT INTO $deltaTableName VALUES (2, "two")""") sql(s"""INSERT INTO $deltaTableName VALUES (3, "three")""") val sharedTableName = "shared_table_time_travel_retention" prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResult( deltaTable = deltaTableName, sharedTable = sharedTableName, versionAsOf = Some(1L) ) // Enable enforcement config - delta-sharing tables should still skip enforcement withSQLConf( DeltaSQLConf.ENFORCE_TIME_TRAVEL_WITHIN_DELETED_FILE_RETENTION_DURATION.key -> "true" ) { withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) val tablePath = s"${profileFile.getCanonicalPath}#share1.default.$sharedTableName" // This should succeed even with enforcement enabled because delta-sharing // tables use "delta-sharing-log" filesystem scheme and skip enforcement val df = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("versionAsOf", 1) .load(tablePath) val expected = Seq(Row(1, "one")) checkAnswer(df, expected) } } } } } test("deleted file retention duration check is not applied for cdf on delta-sharing tables") { withTempDir { tempDir => val deltaTableName = "delta_table_cdc_retention" withTable(deltaTableName) { // file and log retention is set to 0 but still able to time-travel because of skipping enforcement. sql(s""" |CREATE TABLE $deltaTableName (c1 INT, c2 STRING) USING DELTA PARTITIONED BY (c2) |TBLPROPERTIES (delta.enableChangeDataFeed = true, |'delta.deletedFileRetentionDuration' = '0 hours', |'delta.logRetentionDuration' = '0 hours') |""".stripMargin) // Insert multiple versions sql(s"""INSERT INTO $deltaTableName VALUES (1, "one")""") sql(s"""INSERT INTO $deltaTableName VALUES (2, "two")""") sql(s"""INSERT INTO $deltaTableName VALUES (3, "three")""") val sharedTableName = "shared_table_cdc_retention" prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) prepareMockedClientAndFileSystemResultForCdf( deltaTable = deltaTableName, sharedTable = sharedTableName, startingVersion = 0L ) // Enable enforcement config - delta-sharing tables should still skip enforcement withSQLConf( DeltaSQLConf.ENFORCE_TIME_TRAVEL_WITHIN_DELETED_FILE_RETENTION_DURATION.key -> "true" ) { withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) val tablePath = s"${profileFile.getCanonicalPath}#share1.default.$sharedTableName" val df = spark.read .format("deltaSharing") .option("responseFormat", "delta") .option("readChangeFeed", "true") .option("startingVersion", 0) .load(tablePath) .select("c1", "c2", "_change_type", "_commit_version") // CDF should return inserts for all 3 versions (1, 2, 3) // Version 0 is table creation, inserts start from version 1 val expected = Seq( Row(1, "one", "insert", 1L), Row(2, "two", "insert", 2L), Row(3, "three", "insert", 3L) ) checkAnswer(df, expected) } } } } } test("callerOrg option is passed to DeltaSharingRestClient") { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaTableName = "delta_table_caller_org" withTable(deltaTableName) { createSimpleTable(deltaTableName, enableCdf = false) sql(s"""INSERT INTO $deltaTableName VALUES (1, "one")""") val sharedTableName = "shared_table_caller_org" prepareMockedClientAndFileSystemResult( deltaTableName, sharedTableName) DeltaSharingUtils.overrideSingleBlock[Long]( blockId = TestClientForDeltaFormatSharing.getBlockId( sharedTableName, "getTableVersion"), value = 1 ) withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(inputDir) val tablePath = s"${profileFile.getCanonicalPath}#share1.default.$sharedTableName" TestClientForDeltaFormatSharing.lastCallerOrg = "" spark.read .format("deltaSharing") .option("responseFormat", "delta") .option(DeltaSharingOptions.CALLER_ORG_OPTION, "test-org") .load(tablePath) .collect() assert( TestClientForDeltaFormatSharing.lastCallerOrg == "test-org", "callerOrg should be passed through to the client" ) TestClientForDeltaFormatSharing.lastCallerOrg = "" spark.read .format("deltaSharing") .option("responseFormat", "delta") .load(tablePath) .collect() assert( TestClientForDeltaFormatSharing.lastCallerOrg == "", "callerOrg should be empty when not set" ) } } } } } class DeltaSharingDataSourceDeltaSuite extends DeltaSharingDataSourceDeltaSuiteBase {} ================================================ FILE: sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingDataSourceDeltaTestUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark import java.io.File import java.nio.charset.StandardCharsets.UTF_8 import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer import org.apache.spark.sql.delta.{DeltaLog, Snapshot} import org.apache.spark.sql.delta.actions.{ Action, AddCDCFile, AddFile, DeletionVectorDescriptor, Metadata, RemoveFile } import org.apache.spark.sql.delta.deletionvectors.{ RoaringBitmapArray, RoaringBitmapArrayFormat } import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import com.google.common.hash.Hashing import io.delta.sharing.client.model.{ AddFile => ClientAddFile, Metadata => ClientMetadata, Protocol => ClientProtocol } import io.delta.sharing.spark.model.{ DeltaSharingFileAction, DeltaSharingMetadata, DeltaSharingProtocol } import org.apache.commons.io.FileUtils import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.spark.SparkConf import org.apache.spark.paths.SparkPath import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.test.SharedSparkSession trait DeltaSharingDataSourceDeltaTestUtils extends SharedSparkSession { override def beforeAll(): Unit = { super.beforeAll() // close DeltaSharingFileSystem to avoid impact from other unit tests. FileSystem.closeAll() } override protected def sparkConf: SparkConf = { super.sparkConf .set("spark.delta.sharing.preSignedUrl.expirationMs", "30000") .set("spark.delta.sharing.driver.refreshCheckIntervalMs", "3000") .set("spark.delta.sharing.driver.refreshThresholdMs", "10000") .set("spark.delta.sharing.driver.accessThresholdToExpireMs", "300000") } private[spark] def removePartitionPrefix(filePath: String): String = { filePath.split("/").last } private def getResponseDVAndId( sharedTable: String, deletionVector: DeletionVectorDescriptor): (DeletionVectorDescriptor, String) = { if (deletionVector != null) { if (deletionVector.storageType == DeletionVectorDescriptor.INLINE_DV_MARKER) { (deletionVector, Hashing.sha256().hashString(deletionVector.uniqueId, UTF_8).toString) } else { val dvPath = deletionVector.absolutePath(new Path("not-used")) ( deletionVector.copy( pathOrInlineDv = TestDeltaSharingFileSystem.encode(sharedTable, SparkPath.fromPathString(dvPath.getName).urlEncoded), storageType = DeletionVectorDescriptor.PATH_DV_MARKER ), Hashing.sha256().hashString(deletionVector.uniqueId, UTF_8).toString ) } } else { (null, null) } } private def isDataFile(filePath: String): Boolean = { filePath.endsWith(".parquet") || filePath.endsWith(".bin") } // Convert from delta AddFile to DeltaSharingFileAction to serialize to json. private def getDeltaSharingFileActionForAddFile( addFile: AddFile, sharedTable: String, version: Long, timestamp: Long): DeltaSharingFileAction = { val parquetFile = removePartitionPrefix(addFile.path) val (responseDV, dvFileId) = getResponseDVAndId(sharedTable, addFile.deletionVector) DeltaSharingFileAction( id = Hashing.sha256().hashString(parquetFile, UTF_8).toString, version = version, timestamp = timestamp, deletionVectorFileId = dvFileId, deltaSingleAction = addFile .copy( path = TestDeltaSharingFileSystem.encode(sharedTable, parquetFile), deletionVector = responseDV ) .wrap ) } // Convert from delta RemoveFile to DeltaSharingFileAction to serialize to json. // scalastyle:off removeFile private def getDeltaSharingFileActionForRemoveFile( removeFile: RemoveFile, sharedTable: String, version: Long, timestamp: Long): DeltaSharingFileAction = { val parquetFile = removePartitionPrefix(removeFile.path) val (responseDV, dvFileId) = getResponseDVAndId(sharedTable, removeFile.deletionVector) DeltaSharingFileAction( id = Hashing.sha256().hashString(parquetFile, UTF_8).toString, version = version, timestamp = timestamp, deletionVectorFileId = dvFileId, deltaSingleAction = removeFile .copy( path = TestDeltaSharingFileSystem.encode(sharedTable, parquetFile), deletionVector = responseDV ) .wrap ) // scalastyle:on removeFile } // Reset the result for client.GetTableVersion for the sharedTable based on the latest table // version of the deltaTable, use BlockManager to store the result. private[spark] def prepareMockedClientGetTableVersion( deltaTable: String, sharedTable: String, inputVersion: Option[Long] = None): Unit = { DeltaSharingUtils.overrideSingleBlock[Long]( blockId = TestClientForDeltaFormatSharing.getBlockId(sharedTable, "getTableVersion"), value = inputVersion.getOrElse(getSnapshotToUse(deltaTable, None).version) ) } def getTimeStampForVersion(deltaTable: String, version: Long): Long = { val snapshotToUse = getSnapshotToUse(deltaTable, None) FileUtils .listFiles(new File(snapshotToUse.deltaLog.logPath.toUri()), null, true) .asScala .foreach { f => if (FileNames.isDeltaFile(new Path(f.getName))) { if (FileNames.getFileVersion(new Path(f.getName)) == version) { return f.lastModified } } } 0 } // Prepare the result(Protocol and Metadata) for client.GetMetadata for the sharedTable based on // the latest table info of the deltaTable, store them in BlockManager. private[spark] def prepareMockedClientMetadata(deltaTable: String, sharedTable: String): Unit = { val snapshotToUse = getSnapshotToUse(deltaTable, None) val dsProtocol: DeltaSharingProtocol = DeltaSharingProtocol(snapshotToUse.protocol) val dsMetadata: DeltaSharingMetadata = DeltaSharingMetadata( deltaMetadata = snapshotToUse.metadata ) // Put the metadata in blockManager for DeltaSharingClient to return for getMetadata. DeltaSharingUtils.overrideIteratorBlock[String]( blockId = TestClientForDeltaFormatSharing.getBlockId(sharedTable, "getMetadata"), values = Seq(dsProtocol.json, dsMetadata.json).toIterator ) } private def updateAddFileWithInlineDV( addFile: AddFile, inlineDvFormat: RoaringBitmapArrayFormat.Value, bitmap: RoaringBitmapArray): AddFile = { val dv = DeletionVectorDescriptor.inlineInLog( bitmap.serializeAsByteArray(inlineDvFormat), bitmap.cardinality ) addFile .removeRows( deletionVector = dv, updateStats = true ) ._1 } private def updateDvPathToCount( addFile: AddFile, pathToCount: scala.collection.mutable.Map[String, Int]): Unit = { if (addFile.deletionVector != null && addFile.deletionVector.storageType != DeletionVectorDescriptor.INLINE_DV_MARKER) { val dvPath = addFile.deletionVector.pathOrInlineDv pathToCount.put(dvPath, pathToCount.getOrElse(dvPath, 0) + 1) } } // Sort by id in decreasing order. private def deltaSharingFileActionDecreaseOrderFunc( f1: model.DeltaSharingFileAction, f2: model.DeltaSharingFileAction): Boolean = { f1.id > f2.id } // Sort by id in increasing order. private def deltaSharingFileActionIncreaseOrderFunc( f1: model.DeltaSharingFileAction, f2: model.DeltaSharingFileAction): Boolean = { f1.id < f2.id } private def getSnapshotToUse(deltaTable: String, versionAsOf: Option[Long]): Snapshot = { val deltaLog = DeltaLog.forTable(spark, new TableIdentifier(deltaTable)) if (versionAsOf.isDefined) { deltaLog.getSnapshotAt(versionAsOf.get) } else { deltaLog.update() } } // This function does 2 jobs: // 1. Prepare the result for functions of delta sharing rest client, i.e., (Protocol, Metadata) // for getMetadata, (Protocol, Metadata, and list of lines from delta actions) for getFiles, use // BlockManager to store the data to make them available across different classes. All the lines // are for responseFormat=parquet. // 2. Put the parquet file in blockManager for DeltaSharingFileSystem to load bytes out of it. private[spark] def prepareMockedClientAndFileSystemResultForParquet( deltaTable: String, sharedTable: String, versionAsOf: Option[Long] = None, limitHint: Option[Long] = None): Unit = { val lines = Seq.newBuilder[String] var totalSize = 0L val clientAddFilesArrayBuffer = ArrayBuffer[ClientAddFile]() // To prepare faked delta sharing responses with needed files for DeltaSharingClient. val snapshotToUse = getSnapshotToUse(deltaTable, versionAsOf) snapshotToUse.allFiles.collect().foreach { addFile => val parquetFile = removePartitionPrefix(addFile.path) val clientAddFile = ClientAddFile( url = TestDeltaSharingFileSystem.encode(sharedTable, parquetFile), id = Hashing.md5().hashString(parquetFile, UTF_8).toString, partitionValues = addFile.partitionValues, size = addFile.size, stats = null, version = snapshotToUse.version, timestamp = snapshotToUse.timestamp ) totalSize = totalSize + addFile.size clientAddFilesArrayBuffer += clientAddFile } // Scan through the parquet files of the local delta table, and prepare the data of parquet file // reading in DeltaSharingFileSystem. val files = FileUtils.listFiles(new File(snapshotToUse.deltaLog.dataPath.toUri()), null, true).asScala files.foreach { f => val filePath = f.getCanonicalPath val fileName = SparkPath.fromPathString(f.getName).urlEncoded if (isDataFile(filePath)) { // Put the parquet file in blockManager for DeltaSharingFileSystem to load bytes out of it. DeltaSharingUtils.overrideIteratorBlock[Byte]( blockId = TestDeltaSharingFileSystem.getBlockId(sharedTable, fileName), values = FileUtils.readFileToByteArray(f).toIterator ) } } val clientProtocol = ClientProtocol(minReaderVersion = 1) // This is specifically to set the size of the metadata. val deltaMetadata = snapshotToUse.metadata val clientMetadata = ClientMetadata( id = deltaMetadata.id, name = deltaMetadata.name, description = deltaMetadata.description, schemaString = deltaMetadata.schemaString, configuration = deltaMetadata.configuration, partitionColumns = deltaMetadata.partitionColumns, size = totalSize ) lines += JsonUtils.toJson(clientProtocol.wrap) lines += JsonUtils.toJson(clientMetadata.wrap) clientAddFilesArrayBuffer.toSeq.foreach { clientAddFile => lines += JsonUtils.toJson(clientAddFile.wrap) } // Put the metadata in blockManager for DeltaSharingClient to return metadata when being asked. DeltaSharingUtils.overrideIteratorBlock[String]( blockId = TestClientForDeltaFormatSharing.getBlockId( sharedTableName = sharedTable, queryType = "getMetadata", versionAsOf = versionAsOf ), values = Seq( JsonUtils.toJson(clientProtocol.wrap), JsonUtils.toJson(clientMetadata.wrap) ).toIterator ) // Put the delta log (list of actions) in blockManager for DeltaSharingClient to return as the // http response when getFiles is called. DeltaSharingUtils.overrideIteratorBlock[String]( blockId = TestClientForDeltaFormatSharing.getBlockId( sharedTableName = sharedTable, queryType = "getFiles", versionAsOf = versionAsOf, limit = limitHint ), values = lines.result().toIterator ) } // This function does 2 jobs: // 1. Prepare the result for functions of delta sharing rest client, i.e., (Protocol, Metadata) // for getMetadata, (Protocol, Metadata, and list of lines from delta actions) for getFiles, use // BlockManager to store the data to make them available across different classes. // 2. Put the parquet file in blockManager for DeltaSharingFileSystem to load bytes out of it. private[spark] def prepareMockedClientAndFileSystemResult( deltaTable: String, sharedTable: String, versionAsOf: Option[Long] = None, timestampAsOf: Option[String] = None, inlineDvFormat: Option[RoaringBitmapArrayFormat.Value] = None, assertMultipleDvsInOneFile: Boolean = false, reverseFileOrder: Boolean = false, limitHint: Option[Long] = None): Unit = { val lines = Seq.newBuilder[String] var totalSize = 0L // To prepare faked delta sharing responses with needed files for DeltaSharingClient. val snapshotToUse = getSnapshotToUse(deltaTable, versionAsOf) val fileActionsArrayBuffer = ArrayBuffer[model.DeltaSharingFileAction]() val dvPathToCount = scala.collection.mutable.Map[String, Int]() var numRecords = 0L snapshotToUse.allFiles.collect().foreach { addFile => if (assertMultipleDvsInOneFile) { updateDvPathToCount(addFile, dvPathToCount) } val updatedAdd = if (inlineDvFormat.isDefined) { // Remove row 0 and 2 in the AddFile. updateAddFileWithInlineDV(addFile, inlineDvFormat.get, RoaringBitmapArray(0L, 2L)) } else { addFile } if (limitHint.isEmpty || limitHint.map(_ > numRecords).getOrElse(true)) { val dsAddFile = getDeltaSharingFileActionForAddFile( updatedAdd, sharedTable, snapshotToUse.version, snapshotToUse.timestamp ) numRecords += addFile.numLogicalRecords.getOrElse(0L) totalSize = totalSize + addFile.size fileActionsArrayBuffer += dsAddFile } } val fileActionSeq = if (reverseFileOrder) { fileActionsArrayBuffer.toSeq.sortWith(deltaSharingFileActionDecreaseOrderFunc) } else { fileActionsArrayBuffer.toSeq.sortWith(deltaSharingFileActionIncreaseOrderFunc) } var previousIdOpt: Option[String] = None fileActionSeq.foreach { fileAction => if (reverseFileOrder) { assert( // Using < instead of <= because there can be a removeFile and addFile pointing to the // same parquet file which result in the same file id, since id is a hash of file path. // This is ok because eventually it can read data out of the correct parquet file. !previousIdOpt.exists(_ < fileAction.id), s"fileActions must be in decreasing order by id: ${previousIdOpt} is not smaller than" + s" ${fileAction.id}." ) previousIdOpt = Some(fileAction.id) } lines += fileAction.json } if (assertMultipleDvsInOneFile) { assert(dvPathToCount.max._2 > 1) } // Scan through the parquet files of the local delta table, and prepare the data of parquet file // reading in DeltaSharingFileSystem. val files = FileUtils.listFiles(new File(snapshotToUse.deltaLog.dataPath.toUri()), null, true).asScala files.foreach { f => val filePath = f.getCanonicalPath val fileName = SparkPath.fromPathString(f.getName).urlEncoded if (isDataFile(filePath)) { // Put the parquet file in blockManager for DeltaSharingFileSystem to load bytes out of it. DeltaSharingUtils.overrideIteratorBlock[Byte]( blockId = TestDeltaSharingFileSystem.getBlockId(sharedTable, fileName), values = FileUtils.readFileToByteArray(f).toIterator ) } } // This is specifically to set the size of the metadata. val dsMetadata = DeltaSharingMetadata( deltaMetadata = snapshotToUse.metadata, size = totalSize ) val dsProtocol = DeltaSharingProtocol(deltaProtocol = snapshotToUse.protocol) // Put the metadata in blockManager for DeltaSharingClient to return metadata when being asked. DeltaSharingUtils.overrideIteratorBlock[String]( blockId = TestClientForDeltaFormatSharing.getBlockId( sharedTableName = sharedTable, queryType = "getMetadata", versionAsOf = versionAsOf, timestampAsOf = timestampAsOf ), values = Seq(dsProtocol.json, dsMetadata.json).toIterator ) lines += dsProtocol.json lines += dsMetadata.json // Put the delta log (list of actions) in blockManager for DeltaSharingClient to return as the // http response when getFiles is called. DeltaSharingUtils.overrideIteratorBlock[String]( blockId = TestClientForDeltaFormatSharing.getBlockId( sharedTableName = sharedTable, queryType = "getFiles", versionAsOf = versionAsOf, timestampAsOf = timestampAsOf, limit = limitHint ), values = lines.result().toIterator ) } private[spark] def prepareMockedClientAndFileSystemResultForStreaming( deltaTable: String, sharedTable: String, startingVersion: Long, endingVersion: Long, assertDVExists: Boolean = false): Unit = { val actionLines = Seq.newBuilder[String] var maxVersion = -1L var totalSize = 0L val deltaLog = DeltaLog.forTable(spark, new TableIdentifier(deltaTable)) val startingSnapshot = deltaLog.getSnapshotAt(startingVersion) actionLines += DeltaSharingProtocol(deltaProtocol = startingSnapshot.protocol).json actionLines += DeltaSharingMetadata( deltaMetadata = startingSnapshot.metadata, version = startingVersion ).json val logFiles = FileUtils.listFiles(new File(deltaLog.logPath.toUri()), null, true).asScala var dvExists = false logFiles.foreach { f => if (FileNames.isDeltaFile(new Path(f.getName))) { val version = FileNames.getFileVersion(new Path(f.getName)) if (version >= startingVersion && version <= endingVersion) { // protocol/metadata are processed from startingSnapshot, only process versions greater // than startingVersion for real actions and possible metadata changes. maxVersion = maxVersion.max(version) val timestamp = f.lastModified FileUtils.readLines(f).asScala.foreach { l => val action = Action.fromJson(l) action match { case m: Metadata => actionLines += DeltaSharingMetadata( deltaMetadata = m, version = version ).json case addFile: AddFile if addFile.dataChange => // Convert from delta AddFile to DeltaSharingAddFile to serialize to json. val dsAddFile = getDeltaSharingFileActionForAddFile(addFile, sharedTable, version, timestamp) dvExists = dvExists || (dsAddFile.deletionVectorFileId != null) totalSize = totalSize + addFile.size actionLines += dsAddFile.json case removeFile: RemoveFile if removeFile.dataChange => // scalastyle:off removeFile val dsRemoveFile = getDeltaSharingFileActionForRemoveFile( removeFile, sharedTable, version, timestamp ) // scalastyle:on removeFile dvExists = dvExists || (dsRemoveFile.deletionVectorFileId != null) totalSize = totalSize + removeFile.size.getOrElse(0L) actionLines += dsRemoveFile.json case _ => // ignore all other actions such as CommitInfo. } } } } } val dataFiles = FileUtils.listFiles(new File(deltaLog.dataPath.toUri()), null, true).asScala dataFiles.foreach { f => val fileName = SparkPath.fromPathString(f.getName).urlEncoded if (isDataFile(f.getCanonicalPath)) { DeltaSharingUtils.overrideIteratorBlock[Byte]( blockId = TestDeltaSharingFileSystem.getBlockId(sharedTable, fileName), values = FileUtils.readFileToByteArray(f).toIterator ) } } if (assertDVExists) { assert(dvExists, "There should be DV in the files returned from server.") } DeltaSharingUtils.overrideIteratorBlock[String]( blockId = TestClientForDeltaFormatSharing.getBlockId( sharedTable, s"getFiles_${startingVersion}_$endingVersion" ), values = actionLines.result().toIterator ) } private[spark] def prepareMockedClientAndFileSystemResultForCdf( deltaTable: String, sharedTable: String, startingVersion: Long, startingTimestamp: Option[String] = None, inlineDvFormat: Option[RoaringBitmapArrayFormat.Value] = None, assertMultipleDvsInOneFile: Boolean = false): Unit = { val actionLines = Seq.newBuilder[String] var maxVersion = -1L var totalSize = 0L val deltaLog = DeltaLog.forTable(spark, new TableIdentifier(deltaTable)) val startingSnapshot = deltaLog.getSnapshotAt(startingVersion) actionLines += DeltaSharingProtocol(deltaProtocol = startingSnapshot.protocol).json actionLines += DeltaSharingMetadata( deltaMetadata = startingSnapshot.metadata, version = startingVersion ).json val dvPathToCount = scala.collection.mutable.Map[String, Int]() val files = FileUtils.listFiles(new File(deltaLog.logPath.toUri()), null, true).asScala files.foreach { f => if (FileNames.isDeltaFile(new Path(f.getName))) { val version = FileNames.getFileVersion(new Path(f.getName)) if (version >= startingVersion) { // protocol/metadata are processed from startingSnapshot, only process versions greater // than startingVersion for real actions and possible metadata changes. maxVersion = maxVersion.max(version) val timestamp = f.lastModified FileUtils.readLines(f).asScala.foreach { l => val action = Action.fromJson(l) action match { case m: Metadata => actionLines += DeltaSharingMetadata( deltaMetadata = m, version = version ).json case addFile: AddFile if addFile.dataChange => if (assertMultipleDvsInOneFile) { updateDvPathToCount(addFile, dvPathToCount) } val updatedAdd = if (inlineDvFormat.isDefined) { // Remove row 0 and 1 in the AddFile. updateAddFileWithInlineDV(addFile, inlineDvFormat.get, RoaringBitmapArray(0L, 1L)) } else { addFile } val dsAddFile = getDeltaSharingFileActionForAddFile(updatedAdd, sharedTable, version, timestamp) totalSize = totalSize + updatedAdd.size actionLines += dsAddFile.json case removeFile: RemoveFile if removeFile.dataChange => // scalastyle:off removeFile val dsRemoveFile = getDeltaSharingFileActionForRemoveFile( removeFile, sharedTable, version, timestamp ) // scalastyle:on removeFile totalSize = totalSize + removeFile.size.getOrElse(0L) actionLines += dsRemoveFile.json case cdcFile: AddCDCFile => val parquetFile = removePartitionPrefix(cdcFile.path) // Convert from delta AddCDCFile to DeltaSharingFileAction to serialize to json. val dsCDCFile = DeltaSharingFileAction( id = Hashing.sha256().hashString(parquetFile, UTF_8).toString, version = version, timestamp = timestamp, deltaSingleAction = cdcFile .copy( path = TestDeltaSharingFileSystem.encode(sharedTable, parquetFile) ) .wrap ) totalSize = totalSize + cdcFile.size actionLines += dsCDCFile.json case _ => // ignore other lines } } } } } val dataFiles = FileUtils.listFiles(new File(deltaLog.dataPath.toUri()), null, true).asScala dataFiles.foreach { f => val filePath = f.getCanonicalPath val fileName = SparkPath.fromPathString(f.getName).urlEncoded if (isDataFile(filePath)) { DeltaSharingUtils.overrideIteratorBlock[Byte]( blockId = TestDeltaSharingFileSystem.getBlockId(sharedTable, fileName), values = FileUtils.readFileToByteArray(f).toIterator ) } } if (assertMultipleDvsInOneFile) { assert(dvPathToCount.max._2 > 1) } DeltaSharingUtils.overrideIteratorBlock[String]( blockId = TestClientForDeltaFormatSharing.getBlockId(sharedTable, s"getCDFFiles_$startingVersion"), values = actionLines.result().toIterator ) if (startingTimestamp.isDefined) { DeltaSharingUtils.overrideIteratorBlock[String]( blockId = TestClientForDeltaFormatSharing.getBlockId( sharedTable, s"getCDFFiles_${startingTimestamp.get}" ), values = actionLines.result().toIterator ) } } protected def getDeltaSharingClassesSQLConf: Map[String, String] = { Map( "fs.delta-sharing.impl" -> classOf[TestDeltaSharingFileSystem].getName, "spark.delta.sharing.client.class" -> classOf[TestClientForDeltaFormatSharing].getName, "spark.delta.sharing.profile.provider.class" -> "io.delta.sharing.client.DeltaSharingFileProfileProvider" ) } /** Assert the response format recorded by the test client for the given table. */ protected def assertRequestedFormat(tableName: String, expectedFormat: Seq[String]): Unit = { assert( expectedFormat == TestClientForDeltaFormatSharing.requestedFormat.filter(_._1.contains(tableName)).map(_._2)) } } ================================================ FILE: sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingDataSourceTypeWideningSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark import org.apache.spark.sql.delta.DeltaConfigs import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.SparkConf import org.apache.spark.sql.{Column, DataFrame, QueryTest} import org.apache.spark.sql.catalyst.expressions.Literal import org.apache.spark.sql.delta.sharing.DeltaSharingTestSparkUtils import org.apache.spark.sql.functions.col import org.apache.spark.sql.types._ // Unit tests to verify that type widening works with delta sharing. class DeltaSharingDataSourceTypeWideningSuite extends QueryTest with DeltaSQLCommandTest with DeltaSharingTestSparkUtils with DeltaSharingDataSourceDeltaTestUtils { import testImplicits._ protected override def sparkConf: SparkConf = { super.sparkConf .set(DeltaConfigs.ENABLE_TYPE_WIDENING.defaultTablePropertyKey, true.toString) } /** Sets up delta sharing mocks to read a table and validates results. */ private def testReadingDeltaShare( tableName: String, versionAsOf: Option[Long], filter: Option[Column] = None, expectedSchema: StructType, expectedJsonPredicate: Seq[String] = Seq.empty, expectedResult: DataFrame): Unit = { withTempDir { tempDir => val sharedTableName = tableName + "shared_delta_table" prepareMockedClientMetadata(tableName, sharedTableName) prepareMockedClientGetTableVersion(tableName, sharedTableName, versionAsOf) prepareMockedClientAndFileSystemResult(tableName, sharedTableName, versionAsOf) var reader = spark.read .format("deltaSharing") .option("responseFormat", DeltaSharingOptions.RESPONSE_FORMAT_DELTA) versionAsOf.foreach { version => reader = reader.option("versionAsOf", version) } TestClientForDeltaFormatSharing.jsonPredicateHints.clear() withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { val profileFile = prepareProfileFile(tempDir) var result = reader .load(s"${profileFile.getCanonicalPath}#share1.default.$sharedTableName") filter.foreach { f => result = result.filter(f) } assert(result.schema === expectedSchema) checkAnswer(result, expectedResult) assert(getJsonPredicateHints(tableName) === expectedJsonPredicate) } } } /** Fetches JSON predicates passed to the test client when reading a table. */ private def getJsonPredicateHints(tableName: String): Seq[String] = { TestClientForDeltaFormatSharing .jsonPredicateHints .filterKeys(_.contains(tableName)) .values .toSeq } /** Creates a table and applies a type change to it. */ private def withTestTable(testBody: String => Unit): Unit = { val deltaTableName = "type_widening" withTable(deltaTableName) { sql(s"CREATE TABLE $deltaTableName (value SMALLINT) USING DELTA") sql(s"INSERT INTO $deltaTableName VALUES (1), (2)") sql(s"ALTER TABLE $deltaTableName CHANGE COLUMN value TYPE INT") sql(s"INSERT INTO $deltaTableName VALUES (3), (${Int.MaxValue})") sql(s"INSERT INTO $deltaTableName VALUES (4), (5)") testBody(deltaTableName) } } test(s"Delta sharing with type widening") { withTestTable { tableName => testReadingDeltaShare( tableName, versionAsOf = None, expectedSchema = new StructType().add("value", IntegerType), expectedResult = Seq(1, 2, 3, Int.MaxValue, 4, 5).toDF("value")) } } test("Delta sharing with type widening, time travel") { withTestTable { tableName => testReadingDeltaShare( tableName, versionAsOf = Some(3), expectedSchema = new StructType().add("value", IntegerType), expectedResult = Seq(1, 2, 3, Int.MaxValue).toDF("value")) testReadingDeltaShare( tableName, versionAsOf = Some(2), expectedSchema = new StructType().add("value", IntegerType), expectedResult = Seq(1, 2).toDF("value")) testReadingDeltaShare( tableName, versionAsOf = Some(1), expectedSchema = new StructType() .add("value", ShortType), expectedResult = Seq(1, 2).toDF("value")) } } test("jsonPredicateHints on non-partition column after type widening") { withTestTable { tableName => testReadingDeltaShare( tableName, versionAsOf = None, filter = Some(col("value") === Int.MaxValue), expectedSchema = new StructType().add("value", IntegerType), expectedResult = Seq(Int.MaxValue).toDF("value"), expectedJsonPredicate = Seq( """ |{"op":"and","children":[ | {"op":"not","children":[ | {"op":"isNull","children":[ | {"op":"column","name":"value","valueType":"int"}]}]}, | {"op":"equal","children":[ | {"op":"column","name":"value","valueType":"int"}, | {"op":"literal","value":"2147483647","valueType":"int"}]}]} """.stripMargin.replaceAll("\n", "").replaceAll(" ", "")) ) } } test("jsonPredicateHints on partition column after type widening") { val deltaTableName = "type_widening_partitioned" withTable(deltaTableName) { sql( s""" |CREATE TABLE $deltaTableName (part SMALLINT, value SMALLINT) |USING DELTA |PARTITIONED BY (part) """.stripMargin ) sql(s"INSERT INTO $deltaTableName VALUES (1, 1), (2, 2)") sql(s"ALTER TABLE $deltaTableName CHANGE COLUMN part TYPE INT") sql(s"INSERT INTO $deltaTableName VALUES (3, 3), (${Int.MaxValue}, 4)") testReadingDeltaShare( deltaTableName, versionAsOf = None, filter = Some(col("part") === Int.MaxValue), expectedSchema = new StructType() .add("part", IntegerType) .add("value", ShortType), expectedResult = Seq((Int.MaxValue, 4)).toDF("part", "value"), expectedJsonPredicate = Seq( """ |{"op":"and","children":[ | {"op":"not","children":[ | {"op":"isNull","children":[ | {"op":"column","name":"part","valueType":"int"}]}]}, | {"op":"equal","children":[ | {"op":"column","name":"part","valueType":"int"}, | {"op":"literal","value":"2147483647","valueType":"int"}]}]} """.stripMargin.replaceAll("\n", "").replaceAll(" ", "")) ) } } } ================================================ FILE: sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingFileIndexSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark import java.io.File import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import io.delta.sharing.client.{ DeltaSharingClient, DeltaSharingFileSystem, DeltaSharingProfileProvider, DeltaSharingRestClient } import io.delta.sharing.client.model.{DeltaTableFiles, DeltaTableMetadata, Table, TemporaryCredentials} import io.delta.sharing.client.util.JsonUtils import org.apache.commons.io.FileUtils import org.apache.hadoop.fs.Path import org.apache.spark.SparkEnv import org.apache.spark.delta.sharing.{PreSignedUrlCache, PreSignedUrlFetcher} import org.apache.spark.sql.{QueryTest, SparkSession} import org.apache.spark.sql.catalyst.expressions.{ AttributeReference => SqlAttributeReference, EqualTo => SqlEqualTo, Literal => SqlLiteral } import org.apache.spark.sql.delta.sharing.DeltaSharingTestSparkUtils import org.apache.spark.sql.types.{FloatType, IntegerType} private object TestUtils { val paths = Seq("http://path1", "http://path2") val refreshTokens = Seq("token1", "token2", "token3") val SparkConfForReturnExpTime = "spark.delta.sharing.fileindexsuite.returnexptime" val SparkConfForUrlExpirationMs = "spark.delta.sharing.fileindexsuite.urlExpirationMs" // 10 seconds val defaultUrlExpirationMs = 10000 def getExpirationTimestampStr(urlExpirationMs: Option[Int]): String = { if (urlExpirationMs.isDefined) { s""""expirationTimestamp":${System.currentTimeMillis() + urlExpirationMs.get},""" } else { "" } } // scalastyle:off line.size.limit val protocolStr = """{"protocol":{"deltaProtocol":{"minReaderVersion": 1, "minWriterVersion": 1}}}""" val metaDataStr = """{"metaData":{"size":809,"deltaMetadata":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"c1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["c2"],"configuration":{},"createdTime":1691734718560}}}""" val metaDataWithoutSizeStr = """{"metaData":{"deltaMetadata":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"c1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["c2"],"configuration":{},"createdTime":1691734718560}}}""" def getAddFileStr1(path: String, urlExpirationMs: Option[Int] = None): String = { s"""{"file":{"id":"11d9b72771a72f178a6f2839f7f08528",${getExpirationTimestampStr( urlExpirationMs )}"deltaSingleAction":{"add":{"path":"${path}",""" + """"partitionValues":{"c2":"one"},"size":809,"modificationTime":1691734726073,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"c1\":1,\"c2\":\"one\"},\"maxValues\":{\"c1\":2,\"c2\":\"one\"},\"nullCount\":{\"c1\":0,\"c2\":0}}","tags":{"INSERTION_TIME":"1691734726073000","MIN_INSERTION_TIME":"1691734726073000","MAX_INSERTION_TIME":"1691734726073000","OPTIMIZE_TARGET_SIZE":"268435456"}}}}}""" } def getAddFileStr2(urlExpirationMs: Option[Int] = None): String = { s"""{"file":{"id":"22d9b72771a72f178a6f2839f7f08529",${getExpirationTimestampStr( urlExpirationMs )}""" + """"deltaSingleAction":{"add":{"path":"http://path2","partitionValues":{"c2":"two"},"size":809,"modificationTime":1691734726073,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"c1\":1,\"c2\":\"two\"},\"maxValues\":{\"c1\":2,\"c2\":\"two\"},\"nullCount\":{\"c1\":0,\"c2\":0}}","tags":{"INSERTION_TIME":"1691734726073000","MIN_INSERTION_TIME":"1691734726073000","MAX_INSERTION_TIME":"1691734726073000","OPTIMIZE_TARGET_SIZE":"268435456"}}}}}""" } // scalastyle:on line.size.limit } /** * A mocked delta sharing client for unit tests. */ class TestDeltaSharingClientForFileIndex( profileProvider: DeltaSharingProfileProvider, timeoutInSeconds: Int = 120, numRetries: Int = 3, maxRetryDuration: Long = Long.MaxValue, retrySleepInterval: Long = 1000, sslTrustAll: Boolean = false, forStreaming: Boolean = false, responseFormat: String = DeltaSharingRestClient.RESPONSE_FORMAT_DELTA, readerFeatures: String = "", queryTablePaginationEnabled: Boolean = false, maxFilesPerReq: Int = 100000, endStreamActionEnabled: Boolean = false, enableAsyncQuery: Boolean = false, asyncQueryPollIntervalMillis: Long = 10000L, asyncQueryMaxDuration: Long = 600000L, tokenExchangeMaxRetries: Int = 5, tokenExchangeMaxRetryDurationInSeconds: Int = 60, tokenRenewalThresholdInSeconds: Int = 600, callerOrg: String = "", skipFileIdHashVerification: Boolean = false) extends DeltaSharingClient { import TestUtils._ private lazy val returnExpirationTimestamp = SparkSession.active.sessionState.conf .getConfString( SparkConfForReturnExpTime, "false" ) .toBoolean private lazy val urlExpirationMsOpt = if (returnExpirationTimestamp) { val urlExpirationMs = SparkSession.active.sessionState.conf .getConfString( SparkConfForUrlExpirationMs, defaultUrlExpirationMs.toString ) .toInt Some(urlExpirationMs) } else { None } var numGetFileCalls: Int = -1 var savedLimits = Seq.empty[Long] var savedJsonPredicateHints = Seq.empty[String] override def listAllTables(): Seq[Table] = throw new UnsupportedOperationException("not needed") override def getMetadata( table: Table, versionAsOf: Option[Long], timestampAsOf: Option[String]): DeltaTableMetadata = { throw new UnsupportedOperationException("getMetadata is not supported now.") } override def getTableVersion(table: Table, startingTimestamp: Option[String] = None): Long = { throw new UnsupportedOperationException("getTableVersion is not supported now.") } override def getFiles( table: Table, predicates: Seq[String], limit: Option[Long], versionAsOf: Option[Long], timestampAsOf: Option[String], jsonPredicateHints: Option[String], refreshToken: Option[String], fileIdHash: Option[String] ): DeltaTableFiles = { numGetFileCalls += 1 limit.foreach(lim => savedLimits = savedLimits :+ lim) jsonPredicateHints.foreach(p => { savedJsonPredicateHints = savedJsonPredicateHints :+ p }) if (numGetFileCalls > 0 && refreshToken.isDefined) { assert(refreshToken.get == refreshTokens(numGetFileCalls.min(2) - 1)) } DeltaTableFiles( version = 0, lines = Seq[String]( protocolStr, metaDataStr, getAddFileStr1(paths(numGetFileCalls.min(1)), urlExpirationMsOpt), getAddFileStr2(urlExpirationMsOpt) ), refreshToken = Some(refreshTokens(numGetFileCalls.min(2))), respondedFormat = DeltaSharingRestClient.RESPONSE_FORMAT_DELTA ) } override def getFiles( table: Table, startingVersion: Long, endingVersion: Option[Long], fileIdHash: Option[String] ): DeltaTableFiles = { throw new UnsupportedOperationException(s"getFiles with startingVersion($startingVersion)") } override def getCDFFiles( table: Table, cdfOptions: Map[String, String], includeHistoricalMetadata: Boolean, fileIdHash: Option[String] ): DeltaTableFiles = { throw new UnsupportedOperationException( s"getCDFFiles with cdfOptions:[$cdfOptions], " + s"includeHistoricalMetadata:$includeHistoricalMetadata" ) } override def generateTemporaryTableCredential( table: Table, location: Option[String]): TemporaryCredentials = { throw new UnsupportedOperationException("generateTemporaryTableCredential is not implemented") } override def getForStreaming(): Boolean = forStreaming override def getProfileProvider: DeltaSharingProfileProvider = profileProvider def clear() { savedLimits = Seq.empty[Long] savedJsonPredicateHints = Seq.empty[String] } } class DeltaSharingFileIndexSuite extends QueryTest with DeltaSQLCommandTest with DeltaSharingDataSourceDeltaTestUtils with DeltaSharingTestSparkUtils { import TestUtils._ private def getMockedDeltaSharingMetadata(metaData: String): model.DeltaSharingMetadata = { JsonUtils.fromJson[model.DeltaSharingSingleAction](metaData).metaData } private def getMockedDeltaSharingFileAction(id: String): model.DeltaSharingFileAction = { if (id.startsWith("11")) { JsonUtils.fromJson[model.DeltaSharingSingleAction](getAddFileStr1(paths(0))).file } else { JsonUtils.fromJson[model.DeltaSharingSingleAction](getAddFileStr2()).file } } private val shareName = "share" private val schemaName = "default" private val sharedTableName = "table" private def prepareDeltaSharingFileIndex( profilePath: String, metaData: String): (Path, DeltaSharingFileIndex, DeltaSharingClient) = { val tablePath = new Path(s"$profilePath#$shareName.$schemaName.$sharedTableName") val client = DeltaSharingRestClient(profilePath, Map.empty, false, "delta") val spark = SparkSession.active val params = new DeltaSharingFileIndexParams( tablePath, spark, DeltaSharingUtils.DeltaSharingTableMetadata( version = 0, protocol = JsonUtils.fromJson[model.DeltaSharingSingleAction](protocolStr).protocol, metadata = getMockedDeltaSharingMetadata(metaData) ), new DeltaSharingOptions(Map("path" -> tablePath.toString)) ) val dsTable = Table(share = shareName, schema = schemaName, name = sharedTableName) (tablePath, new DeltaSharingFileIndex(params, dsTable, client, None), client) } test("basic functions works") { withTempDir { tempDir => val profileFile = new File(tempDir, "foo.share") FileUtils.writeStringToFile( profileFile, s"""{ | "shareCredentialsVersion": 1, | "endpoint": "https://localhost:12345/not-used-endpoint", | "bearerToken": "mock" |}""".stripMargin, "utf-8" ) withSQLConf( "spark.delta.sharing.client.class" -> classOf[TestDeltaSharingClientForFileIndex].getName, "fs.delta-sharing-log.impl" -> classOf[DeltaSharingLogFileSystem].getName, "spark.delta.sharing.profile.provider.class" -> "io.delta.sharing.client.DeltaSharingFileProfileProvider" ) { val (tablePath, fileIndex, _) = prepareDeltaSharingFileIndex(profileFile.getCanonicalPath, metaDataStr) assert(fileIndex.sizeInBytes == 809) assert(fileIndex.partitionSchema.toDDL == "c2 STRING") assert(fileIndex.rootPaths.length == 1) assert(fileIndex.rootPaths.head == tablePath) intercept[UnsupportedOperationException] { fileIndex.inputFiles } val partitionDirectoryList = fileIndex.listFiles(Seq.empty, Seq.empty) assert(partitionDirectoryList.length == 2) partitionDirectoryList.foreach { partitionDirectory => assert(!partitionDirectory.values.anyNull) assert( partitionDirectory.values.getString(0) == "one" || partitionDirectory.values.getString(0) == "two" ) partitionDirectory.files.foreach { f => // Verify that the path can be decoded val decodedPath = DeltaSharingFileSystem.decode(f.fileStatus.getPath) val dsFileAction = getMockedDeltaSharingFileAction(decodedPath.fileId) assert(decodedPath.tablePath.startsWith(tablePath.toString)) assert(decodedPath.fileId == dsFileAction.id) assert(decodedPath.fileSize == dsFileAction.size) assert(f.fileStatus.getLen == dsFileAction.size) assert(f.fileStatus.getModificationTime == 0) assert(f.fileStatus.isDirectory == false) } } // Check exception is thrown when metadata doesn't have size val (_, fileIndex2, _) = prepareDeltaSharingFileIndex(profileFile.getCanonicalPath, metaDataWithoutSizeStr) val ex = intercept[IllegalStateException] { fileIndex2.sizeInBytes } assert(ex.toString.contains("size is null in the metadata")) } } } test("refresh works") { PreSignedUrlCache.registerIfNeeded(SparkEnv.get) withTempDir { tempDir => val profileFile = new File(tempDir, "foo.share") FileUtils.writeStringToFile( profileFile, s"""{ | "shareCredentialsVersion": 1, | "endpoint": "https://localhost:12345/not-used-endpoint", | "bearerToken": "mock" |}""".stripMargin, "utf-8" ) def test(): Unit = { val (_, fileIndex, _) = prepareDeltaSharingFileIndex(profileFile.getCanonicalPath, metaDataStr) val preSignedUrlCacheRef = PreSignedUrlCache.getEndpointRefInExecutor(SparkEnv.get) val partitionDirectoryList = fileIndex.listFiles(Seq.empty, Seq.empty) assert(partitionDirectoryList.length == 2) partitionDirectoryList.foreach { partitionDirectory => partitionDirectory.files.foreach { f => val decodedPath = DeltaSharingFileSystem.decode(f.fileStatus.getPath) if (decodedPath.fileId.startsWith("11")) { val fetcher = new PreSignedUrlFetcher( preSignedUrlCacheRef, decodedPath.tablePath, decodedPath.fileId, 1000 ) // sleep for 25000ms to ensure that the urls are refreshed. Thread.sleep(25000) // Verify that the url is refreshed as paths(1), not paths(0) anymore. assert(fetcher.getUrl == paths(1)) } } } } withSQLConf( "spark.delta.sharing.client.class" -> classOf[TestDeltaSharingClientForFileIndex].getName, "fs.delta-sharing-log.impl" -> classOf[DeltaSharingLogFileSystem].getName, "spark.delta.sharing.profile.provider.class" -> "io.delta.sharing.client.DeltaSharingFileProfileProvider", SparkConfForReturnExpTime -> "true" ) { test() } withSQLConf( "spark.delta.sharing.client.class" -> classOf[TestDeltaSharingClientForFileIndex].getName, "fs.delta-sharing-log.impl" -> classOf[DeltaSharingLogFileSystem].getName, "spark.delta.sharing.profile.provider.class" -> "io.delta.sharing.client.DeltaSharingFileProfileProvider", SparkConfForReturnExpTime -> "false" ) { test() } } } test("jsonPredicate test") { withTempDir { tempDir => val profileFile = new File(tempDir, "foo.share") FileUtils.writeStringToFile( profileFile, s"""{ | "shareCredentialsVersion": 1, | "endpoint": "https://localhost:12345/not-used-endpoint", | "bearerToken": "mock" |}""".stripMargin, "utf-8" ) withSQLConf( "spark.delta.sharing.client.class" -> classOf[TestDeltaSharingClientForFileIndex].getName, "fs.delta-sharing-log.impl" -> classOf[DeltaSharingLogFileSystem].getName, "spark.delta.sharing.profile.provider.class" -> "io.delta.sharing.client.DeltaSharingFileProfileProvider", SparkConfForReturnExpTime -> "true", SparkConfForUrlExpirationMs -> "3600000" // 1h ) { val (tablePath, fileIndex, client) = prepareDeltaSharingFileIndex(profileFile.getCanonicalPath, metaDataStr) val testClient = client.asInstanceOf[TestDeltaSharingClientForFileIndex] val spark = SparkSession.active spark.sessionState.conf .setConfString("spark.delta.sharing.jsonPredicateHints.enabled", "true") // We will send an equal op on partition filters as a SQL expression tree. val partitionSqlEq = SqlEqualTo( SqlAttributeReference("id", IntegerType)(), SqlLiteral(23, IntegerType) ) // The client should get json for jsonPredicateHints. val expectedJson = """{"op":"equal", |"children":[ | {"op":"column","name":"id","valueType":"int"}, | {"op":"literal","value":"23","valueType":"int"}] |}""".stripMargin.replaceAll("\n", "").replaceAll(" ", "") spark.sessionState.conf.setConfString( "spark.delta.sharing.jsonPredicateV2Hints.enabled", "false" ) fileIndex.listFiles(Seq(partitionSqlEq), Seq.empty) assert(testClient.savedJsonPredicateHints.size === 1) assert(expectedJson == testClient.savedJsonPredicateHints(0)) testClient.clear() // We will send another equal op as a SQL expression tree for data filters. val dataSqlEq = SqlEqualTo( SqlAttributeReference("cost", FloatType)(), SqlLiteral(23.5.toFloat, FloatType) ) // With V2 predicates disabled, the client should get json for partition filters only. fileIndex.listFiles(Seq(partitionSqlEq), Seq(dataSqlEq)) assert(testClient.savedJsonPredicateHints.size === 1) assert(expectedJson == testClient.savedJsonPredicateHints(0)) testClient.clear() // With V2 predicates enabled, the client should get json for partition and data filters // joined at the top level by an AND operation. val expectedJson2 = """{"op":"and","children":[ | {"op":"equal","children":[ | {"op":"column","name":"id","valueType":"int"}, | {"op":"literal","value":"23","valueType":"int"}]}, | {"op":"equal","children":[ | {"op":"column","name":"cost","valueType":"float"}, | {"op":"literal","value":"23.5","valueType":"float"}]} |]}""".stripMargin.replaceAll("\n", "").replaceAll(" ", "") spark.sessionState.conf.setConfString( "spark.delta.sharing.jsonPredicateV2Hints.enabled", "true" ) fileIndex.listFiles(Seq(partitionSqlEq), Seq(dataSqlEq)) assert(testClient.savedJsonPredicateHints.size === 1) assert(expectedJson2 == testClient.savedJsonPredicateHints(0)) testClient.clear() // With json predicates disabled, we should not get anything. spark.sessionState.conf .setConfString("spark.delta.sharing.jsonPredicateHints.enabled", "false") spark.sessionState.conf.setConfString( "spark.delta.sharing.jsonPredicateV2Hints.enabled", "false" ) fileIndex.listFiles(Seq(partitionSqlEq), Seq.empty) assert(testClient.savedJsonPredicateHints.size === 0) } } } } ================================================ FILE: sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingLogFileSystemSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.spark.{SharedSparkContext, SparkEnv, SparkFunSuite} import org.apache.spark.sql.delta.sharing.DeltaSharingTestSparkUtils import org.apache.spark.storage.StorageLevel class DeltaSharingLogFileSystemSuite extends SparkFunSuite with SharedSparkContext { import DeltaSharingLogFileSystem._ var hadoopConf: Configuration = new Configuration var path: Path = null var fs: FileSystem = null override def beforeAll(): Unit = { super.beforeAll() conf.set( s"spark.hadoop.fs.${DeltaSharingLogFileSystem.SCHEME}.impl", classOf[DeltaSharingLogFileSystem].getName ) hadoopConf = DeltaSharingTestSparkUtils.getHadoopConf(conf) path = encode(table1) fs = path.getFileSystem(hadoopConf) } // constants for testing. private val table1 = "table1" private val table2 = "table2" test("encode and decode") { assert(decode(encode(table1)) == table1) } test("file system should be cached") { assert(fs.isInstanceOf[DeltaSharingLogFileSystem]) assert(fs eq path.getFileSystem(hadoopConf)) assert(fs.getScheme == "delta-sharing-log") assert(fs.getWorkingDirectory == new Path("delta-sharing-log:/")) } test("unsupported functions") { intercept[UnsupportedOperationException] { fs.create(path) } intercept[UnsupportedOperationException] { fs.append(path) } intercept[UnsupportedOperationException] { fs.rename(path, new Path(path, "a")) } intercept[UnsupportedOperationException] { fs.delete(path, true) } intercept[UnsupportedOperationException] { fs.listStatusIterator(path) } intercept[UnsupportedOperationException] { fs.setWorkingDirectory(path) } intercept[UnsupportedOperationException] { fs.mkdirs(path) } } test("open works ok") { val content = "this is the content\nanother line\nthird line" SparkEnv.get.blockManager.putSingle[String]( blockId = getDeltaSharingLogBlockId(path.toString), value = content, level = StorageLevel.MEMORY_AND_DISK_SER, tellMaster = true ) assert(scala.io.Source.fromInputStream(fs.open(path)).mkString == content) } test("exists works ok") { val newPath = encode(table1) val fileAndSizeSeq = Seq[DeltaSharingLogFileStatus]( DeltaSharingLogFileStatus("filea", 10, 100) ) SparkEnv.get.blockManager.putIterator[DeltaSharingLogFileStatus]( blockId = getDeltaSharingLogBlockId(newPath.toString), values = fileAndSizeSeq.toIterator, level = StorageLevel.MEMORY_AND_DISK_SER, tellMaster = true ) assert(fs.exists(newPath)) assert(!fs.exists(new Path(newPath, "A"))) } test("listStatus works ok") { val newPath = encode(table2) val fileAndSizeSeq = Seq[DeltaSharingLogFileStatus]( DeltaSharingLogFileStatus("file_a", 10, 100), DeltaSharingLogFileStatus("file_b", 20, 200) ) SparkEnv.get.blockManager.putIterator[DeltaSharingLogFileStatus]( blockId = getDeltaSharingLogBlockId(newPath.toString), values = fileAndSizeSeq.toIterator, level = StorageLevel.MEMORY_AND_DISK_SER, tellMaster = true ) val files = fs.listStatus(newPath) assert(files.length == 2) assert(files(0).getPath == new Path("file_a")) assert(files(0).getLen == 10) assert(files(0).getModificationTime == 100) assert(files(1).getPath == new Path("file_b")) assert(files(1).getLen == 20) assert(files(1).getModificationTime == 200) intercept[java.io.FileNotFoundException] { fs.listStatus(new Path(newPath, "random")) } } } ================================================ FILE: sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingTestSparkUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.sharing import java.io.File import scala.concurrent.duration._ import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.commons.io.FileUtils import org.apache.hadoop.conf.Configuration import org.apache.spark.SparkConf import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.sql.catalyst.util.DateTimeUtils.{ getZoneId, stringToDate, stringToTimestamp, toJavaDate, toJavaTimestamp } import org.apache.spark.sql.internal.SQLConf import org.apache.spark.unsafe.types.UTF8String trait DeltaSharingTestSparkUtils extends DeltaSQLTestUtils { /** * Creates 3 temporary directories for use within a function. * * @param f function to be run with created temp directories */ protected def withTempDirs(f: (File, File, File) => Unit): Unit = { withTempDir { file1 => withTempDir { file2 => withTempDir { file3 => f(file1, file2, file3) } } } } protected def sqlDate(date: String): java.sql.Date = { toJavaDate(stringToDate(UTF8String.fromString(date)).get) } protected def sqlTimestamp(timestamp: String): java.sql.Timestamp = { toJavaTimestamp( stringToTimestamp( UTF8String.fromString(timestamp), getZoneId(SQLConf.get.sessionLocalTimeZone) ).get ) } protected def createTable(tableName: String): Unit = { sql(s"""CREATE TABLE $tableName (c1 INT, c2 STRING, c3 date, c4 timestamp) |USING DELTA PARTITIONED BY (c2) |""".stripMargin) } protected def createTableForStreaming(tableName: String, enableDV: Boolean = false): Unit = { val tablePropertiesStr = if (enableDV) { "TBLPROPERTIES (delta.enableDeletionVectors = true)" } else { "" } sql(s""" |CREATE TABLE $tableName (value STRING) |USING DELTA |$tablePropertiesStr |""".stripMargin) } protected def createSimpleTable(tableName: String, enableCdf: Boolean): Unit = { val tablePropertiesStr = if (enableCdf) { s"""TBLPROPERTIES ( |delta.minReaderVersion=1, |delta.minWriterVersion=4, |delta.enableChangeDataFeed = true)""".stripMargin } else { "" } sql(s"""CREATE TABLE $tableName (c1 INT, c2 STRING) |USING DELTA PARTITIONED BY (c2) |$tablePropertiesStr |""".stripMargin) } protected def createCMIdTableWithCdf(tableName: String): Unit = { sql(s"""CREATE TABLE $tableName (c1 INT, c2 STRING) USING DELTA PARTITIONED BY (c2) |TBLPROPERTIES ('delta.columnMapping.mode' = 'id', |delta.enableChangeDataFeed = true) |""".stripMargin) } protected def createDVTableWithCdf(tableName: String): Unit = { sql(s"""CREATE TABLE $tableName (c1 INT, partition INT) USING DELTA PARTITIONED BY (partition) |TBLPROPERTIES (delta.enableDeletionVectors = true, |delta.enableChangeDataFeed = true) |""".stripMargin) } protected def prepareProfileFile(tempDir: File): File = { val profileFile = new File(tempDir, "foo.share") FileUtils.writeStringToFile( profileFile, s"""{ | "shareCredentialsVersion": 1, | "endpoint": "https://localhost:12345/not-used-endpoint", | "bearerToken": "mock" |}""".stripMargin, "utf-8" ) profileFile } } object DeltaSharingTestSparkUtils { def getHadoopConf(sparkConf: SparkConf): Configuration = { new SparkHadoopUtil().newConfiguration(sparkConf) } } ================================================ FILE: sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingUtilsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark import scala.reflect.ClassTag import io.delta.sharing.client.{DeltaSharingClient, DeltaSharingRestClient} import io.delta.sharing.client.model.{DeltaTableFiles, DeltaTableMetadata, Table, TemporaryCredentials} import io.delta.sharing.spark.DeltaSharingUtils._ import org.apache.spark.{SharedSparkContext, SparkEnv, SparkFunSuite} import org.apache.spark.delta.sharing.TableRefreshResult import org.apache.spark.storage.BlockId class DeltaSharingUtilsSuite extends SparkFunSuite with SharedSparkContext { type RefresherFunction = Option[String] => TableRefreshResult class SimpleTestDeltaSharingClient extends DeltaSharingClient { def getStatsStr(): String = { """{ | "numRecords": 20, | "minValues": { "col-a": 0 }, | "maxValues": { "col-a": 19 }, | "nullCount": { "col-a": 0 } |}""".stripMargin .replace("\n", "") .replace(" ", "") .replace("\"", "\\\"") } def getAddFileStr(): String = { val stats = getStatsStr() s"""{ | "file": { | "id": "add_file_id1", | "expirationTimestamp": 1721350999999, | "deltaSingleAction": { | "add": { | "path": "c000.snappy.parquet", | "partitionValues": { | "col-partition": "3" | }, | "size": 1213, | "modificationTime": 1721350059000, | "dataChange": true, | "stats": "$stats", | "tags": { | "INSERTION_TIME": "1721350059000000" | } | } | } | } |}""".stripMargin } def getDeletionVectorStr(): String = { val stats = getStatsStr() s"""{ | "file": { | "id": "add_file_id2", | "expirationTimestamp": 1721350999999, | "deletionVectorFileId": "dv_file_id", | "deltaSingleAction": { | "add": { | "path": "c001.snappy.parquet", | "partitionValues": { | "col-partition": "3" | }, | "size": 1213, | "modificationTime": 1721350059000, | "dataChange": true, | "stats": "$stats", | "tags": { | "INSERTION_TIME": "1721350059000000" | }, | "deletionVector": { | "storageType": "p", | "pathOrInlineDv": "fakeurl", | "offset": 1, | "sizeInBytes": 34, | "cardinality": 1 | } | } | } | } |}""".stripMargin } def getCdcStr(): String = { s"""{"file":{ | "id":"cdc_file_id", | "expirationTimestamp":1721350999999, | "deltaSingleAction":{ | "cdc":{ | "path":"_change_data/cdc.c000.snappy.parquet", | "partitionValues":{}, | "size":1213, | "modificationTime":1721350059000, | "dataChange":false | } | } |}}""".stripMargin } override def listAllTables(): Seq[Table] = Seq.empty override def getTableVersion(table: Table, startingTimestamp: Option[String] = None): Long = 0 override def getMetadata( table: Table, versionAsOf: Option[Long] = None, timestampAsOf: Option[String] = None ): DeltaTableMetadata = throw new UnsupportedOperationException override def getFiles( table: Table, predicates: Seq[String], limit: Option[Long], versionAsOf: Option[Long], timestampAsOf: Option[String], jsonPredicateHints: Option[String], refreshToken: Option[String], fileIdHash: Option[String] ): DeltaTableFiles = { val file = getAddFileStr() val dv = getDeletionVectorStr() DeltaTableFiles( version = 0L, respondedFormat = DeltaSharingRestClient.RESPONSE_FORMAT_DELTA, lines = Seq(file, dv) ) } override def getFiles( table: Table, startingVersion: Long, endingVersion: Option[Long], fileIdHash: Option[String] ): DeltaTableFiles = { val file = getAddFileStr() val dv = getDeletionVectorStr() DeltaTableFiles( version = 0L, respondedFormat = DeltaSharingRestClient.RESPONSE_FORMAT_DELTA, lines = Seq(file, dv) ) } override def getCDFFiles( table: Table, cdfOptions: Map[String, String], includeHistoricalMetadata: Boolean, fileIdHash: Option[String]): DeltaTableFiles = { val file = getAddFileStr() val dv = getDeletionVectorStr() val cdc = getCdcStr() DeltaTableFiles( version = 0L, respondedFormat = DeltaSharingRestClient.RESPONSE_FORMAT_DELTA, lines = Seq(file, dv, cdc) ) } override def generateTemporaryTableCredential( table: Table, location: Option[String]): TemporaryCredentials = { throw new UnsupportedOperationException("generateTemporaryTableCredential is not implemented") } } test("override single block in blockmanager works") { val blockId = BlockId(s"${DeltaSharingUtils.DELTA_SHARING_BLOCK_ID_PREFIX}_1") overrideSingleBlock[Int](blockId, 1) assert(SparkEnv.get.blockManager.getSingle[Int](blockId).get == 1) SparkEnv.get.blockManager.releaseLock(blockId) overrideSingleBlock[String](blockId, "2") assert(SparkEnv.get.blockManager.getSingle[String](blockId).get == "2") SparkEnv.get.blockManager.releaseLock(blockId) } def getSeqFromBlockManager[T: ClassTag](blockId: BlockId): Seq[T] = { val iterator = SparkEnv.get.blockManager .get[T](blockId) .map( _.data.asInstanceOf[Iterator[T]] ) .get val seqBuilder = Seq.newBuilder[T] while (iterator.hasNext) { seqBuilder += iterator.next() } seqBuilder.result() } test("override iterator block in blockmanager works") { val blockId = BlockId(s"${DeltaSharingUtils.DELTA_SHARING_BLOCK_ID_PREFIX}_1") overrideIteratorBlock[Int](blockId, values = Seq(1, 2).toIterator) assert(getSeqFromBlockManager[Int](blockId) == Seq(1, 2)) overrideIteratorBlock[String](blockId, values = Seq("3", "4").toIterator) assert(getSeqFromBlockManager[String](blockId) == Seq("3", "4")) } test("getRefresherForGetFiles with deletion vector") { val client = new SimpleTestDeltaSharingClient() val table = Table(name = "table", schema = "schema", share = "share") val func: RefresherFunction = getRefresherForGetFiles( client, table, Seq.empty, None, None, None, None, useRefreshToken = true ) val idToUrls = func(None).idToUrl assert(idToUrls.size == 3) assert(idToUrls.contains("add_file_id1")) assert(idToUrls.get("add_file_id1") == Some("c000.snappy.parquet")) assert(idToUrls.contains("add_file_id2")) assert(idToUrls.get("add_file_id2") == Some("c001.snappy.parquet")) assert(idToUrls.contains("dv_file_id")) assert(idToUrls.get("dv_file_id") == Some("fakeurl")) } test("getRefresherForGetFilesWithStartingVersion with deletion vector") { val client = new SimpleTestDeltaSharingClient() val table = Table(name = "table", schema = "schema", share = "share") val func: RefresherFunction = getRefresherForGetFilesWithStartingVersion( client, table, 0L, None ) val idToUrls = func(None).idToUrl assert(idToUrls.size == 3) assert(idToUrls.contains("add_file_id1")) assert(idToUrls.get("add_file_id1") == Some("c000.snappy.parquet")) assert(idToUrls.contains("add_file_id2")) assert(idToUrls.get("add_file_id2") == Some("c001.snappy.parquet")) assert(idToUrls.contains("dv_file_id")) assert(idToUrls.get("dv_file_id") == Some("fakeurl")) } test("getRefresherForGetCDFFiles with deletion vector") { val client = new SimpleTestDeltaSharingClient() val table = Table(name = "table", schema = "schema", share = "share") val func: RefresherFunction = getRefresherForGetCDFFiles( client, table, Map[String, String]("startingVersion" -> "0") ) val idToUrls = func(None).idToUrl assert(idToUrls.size == 4) assert(idToUrls.contains("add_file_id1")) assert(idToUrls.get("add_file_id1") == Some("c000.snappy.parquet")) assert(idToUrls.contains("add_file_id2")) assert(idToUrls.get("add_file_id2") == Some("c001.snappy.parquet")) assert(idToUrls.contains("dv_file_id")) assert(idToUrls.get("dv_file_id") == Some("fakeurl")) assert(idToUrls.contains("cdc_file_id")) assert(idToUrls.get("cdc_file_id") == Some("_change_data/cdc.c000.snappy.parquet")) } test("getRefresherForGetFiles respects useRefreshToken parameter") { // Test client that tracks the refresh token parameter class RefreshTokenTrackingClient extends SimpleTestDeltaSharingClient { var lastRefreshToken: Option[String] = null override def getFiles( table: Table, predicates: Seq[String], limit: Option[Long], versionAsOf: Option[Long], timestampAsOf: Option[String], jsonPredicateHints: Option[String], refreshToken: Option[String], fileIdHash: Option[String] ): DeltaTableFiles = { lastRefreshToken = refreshToken super.getFiles(table, predicates, limit, versionAsOf, timestampAsOf, jsonPredicateHints, refreshToken, fileIdHash) } } val client = new RefreshTokenTrackingClient() val table = Table(name = "table", schema = "schema", share = "share") val testRefreshToken = Some("test-refresh-token") // Test with useRefreshToken = true - should use the provided refresh token val funcWithRefreshToken: RefresherFunction = getRefresherForGetFiles( client, table, Seq.empty, None, Some(0L), None, None, useRefreshToken = true ) funcWithRefreshToken(testRefreshToken) assert(client.lastRefreshToken == testRefreshToken, "When useRefreshToken=true, the refresh token should be passed through") // Test with useRefreshToken = false - should ignore the provided refresh token val funcWithoutRefreshToken: RefresherFunction = getRefresherForGetFiles( client, table, Seq.empty, None, Some(0L), None, None, useRefreshToken = false ) funcWithoutRefreshToken(testRefreshToken) assert(client.lastRefreshToken == None, "When useRefreshToken=false, the refresh token should be ignored and None should be used") } } ================================================ FILE: sharing/src/test/scala/io/delta/sharing/spark/TestClientForDeltaFormatSharing.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark import scala.collection.mutable.ArrayBuffer import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.util.JsonUtils import io.delta.sharing.client.{ DeltaSharingClient, DeltaSharingProfileProvider, DeltaSharingRestClient } import io.delta.sharing.client.model.{ AddFile => ClientAddFile, DeltaTableFiles, DeltaTableMetadata, SingleAction, Table, TemporaryCredentials } import org.apache.spark.SparkEnv import org.apache.spark.storage.BlockId /** * A mocked delta sharing client for DeltaFormatSharing. * The test suite need to prepare the mocked delta sharing rpc response and store them in * BlockManager. Then this client will just load the response of return upon rpc call. */ private[spark] class TestClientForDeltaFormatSharing( profileProvider: DeltaSharingProfileProvider, timeoutInSeconds: Int = 120, numRetries: Int = 3, maxRetryDuration: Long = Long.MaxValue, retrySleepInterval: Long = 1000, sslTrustAll: Boolean = false, forStreaming: Boolean = false, responseFormat: String = DeltaSharingRestClient.RESPONSE_FORMAT_DELTA, readerFeatures: String = "", queryTablePaginationEnabled: Boolean = false, maxFilesPerReq: Int = 100000, endStreamActionEnabled: Boolean = false, enableAsyncQuery: Boolean = false, asyncQueryPollIntervalMillis: Long = 10000L, asyncQueryMaxDuration: Long = 600000L, tokenExchangeMaxRetries: Int = 5, tokenExchangeMaxRetryDurationInSeconds: Int = 60, tokenRenewalThresholdInSeconds: Int = 600, callerOrg: String = "", skipFileIdHashVerification: Boolean = false) extends DeltaSharingClient { private val supportedReaderFeatures: Seq[String] = Seq( DeletionVectorsTableFeature, ColumnMappingTableFeature, TimestampNTZTableFeature, TypeWideningPreviewTableFeature, TypeWideningTableFeature, VariantTypePreviewTableFeature, VariantTypeTableFeature, VariantShreddingPreviewTableFeature ).map(_.name) assert( responseFormat == DeltaSharingRestClient.RESPONSE_FORMAT_PARQUET || supportedReaderFeatures.forall(readerFeatures.split(",").contains), s"${supportedReaderFeatures.diff(readerFeatures.split(",")).mkString(", ")} " + s"should be supported in all types of queries." ) import TestClientForDeltaFormatSharing._ TestClientForDeltaFormatSharing.lastCallerOrg = callerOrg override def listAllTables(): Seq[Table] = throw new UnsupportedOperationException("not needed") override def getMetadata( table: Table, versionAsOf: Option[Long] = None, timestampAsOf: Option[String] = None): DeltaTableMetadata = { val iterator = SparkEnv.get.blockManager .get[String](getBlockId(table.name, "getMetadata", versionAsOf, timestampAsOf)) .map(_.data.asInstanceOf[Iterator[String]]) .getOrElse { throw new IllegalStateException( s"getMetadata is missing for: ${table.name}, versionAsOf:$versionAsOf, " + s"timestampAsOf:$timestampAsOf. This shouldn't happen in the unit test." ) } // iterator.toSeq doesn't trigger CompletionIterator in BlockManager which releases the reader // lock on the underlying block. iterator hasNext does trigger it. val linesBuilder = Seq.newBuilder[String] while (iterator.hasNext) { linesBuilder += iterator.next() } if (table.name.contains("shared_parquet_table") && responseFormat.contains(DeltaSharingRestClient.RESPONSE_FORMAT_PARQUET)) { val lines = linesBuilder.result() val protocol = JsonUtils.fromJson[SingleAction](lines(0)).protocol val metadata = JsonUtils.fromJson[SingleAction](lines(1)).metaData DeltaTableMetadata( version = versionAsOf.getOrElse(getTableVersion(table)), protocol = protocol, metadata = metadata, respondedFormat = DeltaSharingRestClient.RESPONSE_FORMAT_PARQUET ) } else { DeltaTableMetadata( version = versionAsOf.getOrElse(getTableVersion(table)), lines = linesBuilder.result(), respondedFormat = DeltaSharingRestClient.RESPONSE_FORMAT_DELTA ) } } override def getTableVersion(table: Table, startingTimestamp: Option[String] = None): Long = { val versionOpt = SparkEnv.get.blockManager.getSingle[Long]( getBlockId(table.name, "getTableVersion") ) val version = versionOpt.getOrElse { throw new IllegalStateException( s"getTableVersion is missing for: ${table.name}. This shouldn't happen in the unit test." ) } SparkEnv.get.blockManager.releaseLock(getBlockId(table.name, "getTableVersion")) version } override def getFiles( table: Table, predicates: Seq[String], limit: Option[Long], versionAsOf: Option[Long], timestampAsOf: Option[String], jsonPredicateHints: Option[String], refreshToken: Option[String], fileIdHash: Option[String] ): DeltaTableFiles = { val tableFullName = s"${table.share}.${table.schema}.${table.name}" limit.foreach(lim => TestClientForDeltaFormatSharing.limits.put(tableFullName, lim)) TestClientForDeltaFormatSharing.requestedFormat.put(tableFullName, responseFormat) jsonPredicateHints.foreach(p => TestClientForDeltaFormatSharing.jsonPredicateHints.put(tableFullName, p)) val iterator = SparkEnv.get.blockManager .get[String](getBlockId( table.name, "getFiles", versionAsOf = versionAsOf, timestampAsOf = timestampAsOf, limit = limit) ) .map(_.data.asInstanceOf[Iterator[String]]) .getOrElse { throw new IllegalStateException( s"getFiles is missing for: ${table.name} versionAsOf:$versionAsOf, " + s"timestampAsOf:$timestampAsOf, limit: $limit. This shouldn't happen in the unit test." ) } // iterator.toSeq doesn't trigger CompletionIterator in BlockManager which releases the reader // lock on the underlying block. iterator hasNext does trigger it. val linesBuilder = Seq.newBuilder[String] while (iterator.hasNext) { linesBuilder += iterator.next() } if (table.name.contains("shared_parquet_table") && responseFormat.contains(DeltaSharingRestClient.RESPONSE_FORMAT_PARQUET)) { val lines = linesBuilder.result() val protocol = JsonUtils.fromJson[SingleAction](lines(0)).protocol val metadata = JsonUtils.fromJson[SingleAction](lines(1)).metaData val files = ArrayBuffer[ClientAddFile]() lines.drop(2).foreach { line => val action = JsonUtils.fromJson[SingleAction](line) if (action.file != null) { files.append(action.file) } else { throw new IllegalStateException(s"Unexpected Line:${line}") } } DeltaTableFiles( versionAsOf.getOrElse(getTableVersion(table)), protocol, metadata, files.toSeq, respondedFormat = DeltaSharingRestClient.RESPONSE_FORMAT_PARQUET ) } else { DeltaTableFiles( version = versionAsOf.getOrElse(getTableVersion(table)), lines = linesBuilder.result(), respondedFormat = DeltaSharingRestClient.RESPONSE_FORMAT_DELTA ) } } override def getFiles( table: Table, startingVersion: Long, endingVersion: Option[Long], fileIdHash: Option[String] ): DeltaTableFiles = { assert( endingVersion.isDefined, "endingVersion is not defined. This shouldn't happen in unit test." ) val tableFullName = s"${table.share}.${table.schema}.${table.name}" TestClientForDeltaFormatSharing.requestedFormat.put(tableFullName, responseFormat) val iterator = SparkEnv.get.blockManager .get[String](getBlockId(table.name, s"getFiles_${startingVersion}_${endingVersion.get}")) .map(_.data.asInstanceOf[Iterator[String]]) .getOrElse { throw new IllegalStateException( s"getFiles is missing for: ${table.name} with [${startingVersion}, " + s"${endingVersion.get}]. This shouldn't happen in the unit test." ) } // iterator.toSeq doesn't trigger CompletionIterator in BlockManager which releases the reader // lock on the underlying block. iterator hasNext does trigger it. val linesBuilder = Seq.newBuilder[String] while (iterator.hasNext) { linesBuilder += iterator.next() } DeltaTableFiles( version = getTableVersion(table), lines = linesBuilder.result(), respondedFormat = DeltaSharingRestClient.RESPONSE_FORMAT_DELTA ) } override def getCDFFiles( table: Table, cdfOptions: Map[String, String], includeHistoricalMetadata: Boolean, fileIdHash: Option[String] ): DeltaTableFiles = { val suffix = cdfOptions .get(DeltaSharingOptions.CDF_START_VERSION) .getOrElse( cdfOptions.get(DeltaSharingOptions.CDF_START_TIMESTAMP).get ) val iterator = SparkEnv.get.blockManager .get[String]( getBlockId( table.name, s"getCDFFiles_$suffix" ) ) .map( _.data.asInstanceOf[Iterator[String]] ) .getOrElse { throw new IllegalStateException( s"getCDFFiles is missing for: ${table.name}. This shouldn't happen in the unit test." ) } // iterator.toSeq doesn't trigger CompletionIterator in BlockManager which releases the reader // lock on the underlying block. iterator hasNext does trigger it. val linesBuilder = Seq.newBuilder[String] while (iterator.hasNext) { linesBuilder += iterator.next() } DeltaTableFiles( version = getTableVersion(table), lines = linesBuilder.result(), respondedFormat = DeltaSharingRestClient.RESPONSE_FORMAT_DELTA ) } override def generateTemporaryTableCredential( table: Table, location: Option[String]): TemporaryCredentials = { throw new UnsupportedOperationException("generateTemporaryTableCredential is not implemented") } override def getForStreaming(): Boolean = forStreaming override def getProfileProvider: DeltaSharingProfileProvider = profileProvider } object TestClientForDeltaFormatSharing { def getBlockId( sharedTableName: String, queryType: String, versionAsOf: Option[Long] = None, timestampAsOf: Option[String] = None, limit: Option[Long] = None): BlockId = { assert(!(versionAsOf.isDefined && timestampAsOf.isDefined)) val suffix = if (versionAsOf.isDefined) { s"_v${versionAsOf.get}" } else if (timestampAsOf.isDefined) { s"_t${timestampAsOf.get}" } else { "" } val limitSuffix = limit.map{ l => s"_l${l}"}.getOrElse("") BlockId( s"${DeltaSharingUtils.DELTA_SHARING_BLOCK_ID_PREFIX}" + s"_${sharedTableName}_$queryType$suffix$limitSuffix" ) } val limits = scala.collection.mutable.Map[String, Long]() val requestedFormat = scala.collection.mutable.Map[String, String]() val jsonPredicateHints = scala.collection.mutable.Map[String, String]() @volatile var lastCallerOrg: String = "" } ================================================ FILE: sharing/src/test/scala/io/delta/sharing/spark/TestDeltaSharingFileSystem.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark import java.io.FileNotFoundException import java.net.{URI, URLDecoder, URLEncoder} import java.util.concurrent.TimeUnit import io.delta.sharing.client.DeltaSharingFileSystem import org.apache.hadoop.fs._ import org.apache.hadoop.fs.permission.FsPermission import org.apache.hadoop.util.Progressable import org.apache.spark.SparkEnv import org.apache.spark.delta.sharing.{PreSignedUrlCache, PreSignedUrlFetcher} import org.apache.spark.storage.BlockId /** * Read-only file system for DeltaSharingDataSourceDeltaSuite. * To replace DeltaSharingFileSystem and return the content for parquet files. */ private[spark] class TestDeltaSharingFileSystem extends FileSystem { import TestDeltaSharingFileSystem._ private lazy val preSignedUrlCacheRef = PreSignedUrlCache.getEndpointRefInExecutor(SparkEnv.get) override def getScheme: String = SCHEME override def getUri(): URI = URI.create(s"$SCHEME:///") override def open(f: Path, bufferSize: Int): FSDataInputStream = { val path = DeltaSharingFileSystem.decode(f) val fetcher = new PreSignedUrlFetcher( preSignedUrlCacheRef, path.tablePath, path.fileId, TimeUnit.MINUTES.toMillis(10) ) val (tableName, parquetFilePath) = decode(fetcher.getUrl()) val arrayBuilder = Array.newBuilder[Byte] val iterator = SparkEnv.get.blockManager .get[Byte](getBlockId(tableName, parquetFilePath)) .map( _.data.asInstanceOf[Iterator[Byte]] ) .getOrElse { throw new FileNotFoundException(f.toString) } while (iterator.hasNext) { arrayBuilder += iterator.next() } new FSDataInputStream(new SeekableByteArrayInputStream(arrayBuilder.result())) } override def create( f: Path, permission: FsPermission, overwrite: Boolean, bufferSize: Int, replication: Short, blockSize: Long, progress: Progressable): FSDataOutputStream = throw new UnsupportedOperationException("create") override def append(f: Path, bufferSize: Int, progress: Progressable): FSDataOutputStream = throw new UnsupportedOperationException("append") override def rename(src: Path, dst: Path): Boolean = throw new UnsupportedOperationException("rename") override def delete(f: Path, recursive: Boolean): Boolean = throw new UnsupportedOperationException("delete") override def listStatus(f: Path): Array[FileStatus] = throw new UnsupportedOperationException("listStatus") override def setWorkingDirectory(new_dir: Path): Unit = throw new UnsupportedOperationException("setWorkingDirectory") override def getWorkingDirectory: Path = new Path(getUri) override def mkdirs(f: Path, permission: FsPermission): Boolean = throw new UnsupportedOperationException("mkdirs") override def getFileStatus(f: Path): FileStatus = { val resolved = makeQualified(f) new FileStatus(DeltaSharingFileSystem.decode(resolved).fileSize, false, 0, 1, 0, f) } override def close(): Unit = { super.close() } } private[spark] object TestDeltaSharingFileSystem { val SCHEME = "delta-sharing" def getBlockId(tableName: String, parquetFilePath: String): BlockId = { BlockId( s"${DeltaSharingUtils.DELTA_SHARING_BLOCK_ID_PREFIX}_" + s"{$tableName}_$parquetFilePath" ) } // The encoded string is purely for testing purpose to contain the table name and file path, // which will be decoded and used to find block in block manager. // In real traffic, it will be a pre-signed url. def encode(tableName: String, parquetFilePath: String): String = { val encodedTableName = URLEncoder.encode(tableName, "UTF-8") val encodedParquetFilePath = URLEncoder.encode(parquetFilePath, "UTF-8") // SCHEME:/// is needed for making this path an absolute path s"$SCHEME:///$encodedTableName/$encodedParquetFilePath" } def decode(encodedPath: String): (String, String) = { val Array(tableName, parquetFilePath) = encodedPath .stripPrefix(s"$SCHEME:///") .stripPrefix(s"$SCHEME:/") .split("/") .map( URLDecoder.decode(_, "UTF-8") ) (tableName, parquetFilePath) } } ================================================ FILE: sharing/src/test/scala-shims/spark-4.0/SharingStreamingTestShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark.test.shims import org.apache.spark.sql.execution.streaming.{ CheckpointFileManager => CheckpointFileManagerShim, CommitMetadata => CommitMetadataShim, SerializedOffset => SerializedOffsetShim, StreamMetadata => StreamMetadataShim } /** * Test shims for streaming classes that were relocated in Spark 4.1. * In Spark 4.0, these classes are in org.apache.spark.sql.execution.streaming. * StreamingCheckpointConstants does not exist in Spark 4.0, so we define * the constants directly. */ object SharingStreamingTestShims { val CheckpointFileManager: CheckpointFileManagerShim.type = CheckpointFileManagerShim val CommitMetadata: CommitMetadataShim.type = CommitMetadataShim val SerializedOffset: SerializedOffsetShim.type = SerializedOffsetShim val StreamMetadata: StreamMetadataShim.type = StreamMetadataShim object StreamingCheckpointConstants { val DIR_NAME_COMMITS = "commits" val DIR_NAME_OFFSETS = "offsets" val DIR_NAME_METADATA = "metadata" } } ================================================ FILE: sharing/src/test/scala-shims/spark-4.1/SharingStreamingTestShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark.test.shims import org.apache.spark.sql.execution.streaming.checkpointing.{ CheckpointFileManager => CheckpointFileManagerShim, CommitMetadata => CommitMetadataShim } import org.apache.spark.sql.execution.streaming.runtime.{ SerializedOffset => SerializedOffsetShim, StreamingCheckpointConstants => StreamingCheckpointConstantsShim, StreamMetadata => StreamMetadataShim } /** * Test shims for streaming classes that were relocated in Spark 4.1. * In Spark 4.1, these classes moved to checkpointing and runtime sub-packages. */ object SharingStreamingTestShims { val CheckpointFileManager: CheckpointFileManagerShim.type = CheckpointFileManagerShim val CommitMetadata: CommitMetadataShim.type = CommitMetadataShim val SerializedOffset: SerializedOffsetShim.type = SerializedOffsetShim val StreamMetadata: StreamMetadataShim.type = StreamMetadataShim val StreamingCheckpointConstants: StreamingCheckpointConstantsShim.type = StreamingCheckpointConstantsShim } ================================================ FILE: sharing/src/test/scala-shims/spark-4.2/SharingStreamingTestShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sharing.spark.test.shims import org.apache.spark.sql.execution.streaming.checkpointing.{ CheckpointFileManager => CheckpointFileManagerShim, CommitMetadata => CommitMetadataShim } import org.apache.spark.sql.execution.streaming.runtime.{ SerializedOffset => SerializedOffsetShim, StreamingCheckpointConstants => StreamingCheckpointConstantsShim, StreamMetadata => StreamMetadataShim } /** * Test shims for streaming classes that were relocated in Spark 4.1+. * In Spark 4.2, these classes remain in the same locations as Spark 4.1. */ object SharingStreamingTestShims { val CheckpointFileManager: CheckpointFileManagerShim.type = CheckpointFileManagerShim val CommitMetadata: CommitMetadataShim.type = CommitMetadataShim val SerializedOffset: SerializedOffsetShim.type = SerializedOffsetShim val StreamMetadata: StreamMetadataShim.type = StreamMetadataShim val StreamingCheckpointConstants: StreamingCheckpointConstantsShim.type = StreamingCheckpointConstantsShim } ================================================ FILE: spark/delta-suite-generator/src/main/resources/scalafmt.conf ================================================ align = none align.openParenDefnSite = false align.openParenCallSite = false align.tokens = [] indent.extendSite = 2 importSelectors = "singleLine" optIn.configStyleArguments = false danglingParentheses { defnSite = false callSite = false } docstrings { style = Asterisk wrap = no } literals.hexDigits = upper maxColumn = 100 rewrite.rules = [Imports] rewrite.imports.sort = scalastyle rewrite.imports.groups = [ ["java\\..*"], ["scala\\..*"], ["io\\.delta\\..*"], ["org\\.apache\\.spark\\.sql\\.delta.*"] ] runner.dialect = scala212 version = 3.9.6 newlines.topLevelStatementBlankLines = [ { blanks = 1 } ] ================================================ FILE: spark/delta-suite-generator/src/main/scala/io/delta/suitegenerator/ModularSuiteGenerator.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.suitegenerator import java.nio.file.{Files, Paths} import scala.meta._ import scala.util.hashing.MurmurHash3 import org.apache.commons.cli.{CommandLine, DefaultParser, HelpFormatter, Option, Options} import org.apache.commons.codec.binary.Base32 /** * The main generator for the Modular Delta Suites. Generated suite combinations can be configured * in [[SuiteGeneratorConfig]]. * * Can be run via the sbt command: `deltaSuiteGenerator / run` */ object ModularSuiteGenerator { val GENERATED_PACKAGE = s"org.apache.spark.sql.delta.generatedsuites" lazy val OUTPUT_PATH: String = "spark/src/test/scala/" + GENERATED_PACKAGE.replace('.', '/') private val DEFAULT_REPO_PATH = "~/delta" /** * Controls when to start truncating and hashing the suite names to prevent extremely long names. */ private val SUITE_NAME_CHAR_LIMIT = 255 - 148 private lazy val OPT_REPO_PATH = new Option( /* option = */ "p", /* longOption = */ "repo-path", /* hasArg = */ true, /* description = */ s"Path to the repository root. Defaults to $DEFAULT_REPO_PATH") private lazy val OPT_HELP = new Option( /* option = */ "h", /* longOption = */ "help", /* hasArg = */ false, /* description = */ "Print help") private lazy val OPTIONS = new Options().addOption(OPT_REPO_PATH).addOption(OPT_HELP) def main(args: Array[String]): Unit = { val cmd = new DefaultParser().parse(OPTIONS, args) if (cmd.hasOption(OPT_HELP)) { val formatter = new HelpFormatter() formatter.printHelp( "bazel run //sql/core/delta_suite_generator:generate -- ", OPTIONS) System.exit(0) } val suitesWriter = getWriter(cmd) // scalastyle:off println println("Generating suites...") generateSuites(suitesWriter) println("Suite generation completed successfully.") // scalastyle:on println } def generateSuites(suitesWriter: SuitesWriter): Unit = { for (testGroup <- SuiteGeneratorConfig.TEST_GROUPS) { val suites = for { testConfig <- testGroup.testConfigs baseSuite <- testConfig.baseSuites dimensions <- testConfig.dimensionCombinations } yield dimensions // Generate all combinations of dimension traits .foldLeft(List(List.empty[(String, String)])) { (acc, dimension) => (if (dimension.isOptional) acc else List.empty) ::: (for { accValue <- acc traitWithAlias <- dimension.traitsWithAliases } yield accValue :+ traitWithAlias) } .filterNot(dimTraits => SuiteGeneratorConfig.isExcluded(baseSuite, dimTraits.map(_._1))) .map(dimTraits => generateCode(baseSuite, dimTraits)) suitesWriter.writeGeneratedSuitesOfGroup(suites.flatten, testGroup) } suitesWriter.conclude() } private def getWriter(cmd: CommandLine): SuitesWriter = { var repoPath = cmd.getOptionValue(OPT_REPO_PATH, DEFAULT_REPO_PATH) // Expand `~` prefix to the user's home directory if (repoPath.startsWith("~")) { repoPath = System.getProperty("user.home") + repoPath.substring(1) } val outputPath = Paths.get(repoPath, OUTPUT_PATH) assert( Files.exists(outputPath.getParent), s"Repository could not be detected at $repoPath. Make sure to provide the " + s"repository path using the --${OPT_REPO_PATH.getLongOpt} option.") // Prevent people with multiple repository copies/worktrees accidentally generating into // the wrong one. // We assume if it's specified explicitly, it's specified correctly, and we don't need to // double-check. if (!cmd.hasOption(OPT_REPO_PATH)) { // scalastyle:off println if (System.console() == null) { // This is not an interactive shell, we can't ask for input. println( s"""Verified that a matching repository exists at target. |Generation target path is: '${outputPath}' |The path can be customised with the --${OPT_REPO_PATH.getLongOpt} option.""" .stripMargin) } else { println( s"""Verified that a matching repository exists at target. |Please double check the path: '${outputPath}' |The path can be customised with the --${OPT_REPO_PATH.getLongOpt} option. |If correct, press to generate or +c to abort.""".stripMargin) scala.io.StdIn.readLine() } // scalastyle:on println } new SuitesWriter(outputPath) } private lazy val BASE32 = new Base32() private def generateCode( baseSuite: String, mixinsAndAliases: List[(String, String)]): TestSuite = { val allMixins = SuiteGeneratorConfig .applyCustomRulesAndGetAllMixins(baseSuite, mixinsAndAliases.map(_._1)) val suiteParents = (baseSuite :: allMixins).map(_.parse[Init].get) // Generate suite name by combining the names of base suite and dimension aliases. // Remove some redundant substrings for better readability val baseSuitePrefix = baseSuite.stripSuffix("Suite").stripSuffix("Tests") val mixinSuffix = mixinsAndAliases .map(_._2.replace("Mixin", "")) .mkString("") var suiteName = baseSuitePrefix + mixinSuffix // Truncate the name and replace with a consistent hash if line becomes longer than the limit val maxSuiteNameLength = SUITE_NAME_CHAR_LIMIT - "Suite".length if (suiteName.length > maxSuiteNameLength) { // scalastyle:off println println(s"WARNING: Suite name is too long, truncating and hashing to fit within the limit. " + s"Please consider renaming the base suite or defining shorter dimension aliases. " + s"Suite: $suiteName (${suiteName.length} characters > $maxSuiteNameLength limit).") // scalastyle:on println val hashBytes = BigInt(MurmurHash3.stringHash(suiteName)).toByteArray val hashEncoded = BASE32.encodeToString(hashBytes).replace("=", "") suiteName = suiteName.substring(0, maxSuiteNameLength - hashEncoded.length) + hashEncoded } suiteName += "Suite" TestSuite( suiteName, q"""class ${Type.Name(suiteName)} extends ..$suiteParents""") } } case class TestSuite( name: String, classDefinition: Defn.Class ) ================================================ FILE: spark/delta-suite-generator/src/main/scala/io/delta/suitegenerator/SuiteGeneratorConfig.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.suitegenerator import scala.collection.mutable.ListBuffer import scala.meta._ /** * Represents a configuration trait that changes how the tests are executed. This can include Spark * configs, overrides, test excludes, and more. * @param name the name of the dimension. * @param values the possible values for this dimension, which when prepended with the name should * equal to the desired trait name that needs to be mixed in to generated suites. * @param alias an optional short alias to be used when naming suites instead of the [[name]]. */ abstract class Dimension(val name: String, val values: List[String], val alias: Option[String]) { /** * All trait names for this dimension */ lazy val traitNames: List[String] = values.map(value => name + value) lazy val traitsWithAliases: List[(String, String)] = values .map(value => ( name + value, alias.getOrElse(name) + value.replace("Enabled", "On").replace("Disabled", "Off") )) val isOptional: Boolean = false /** * same [[Dimension]] with an additional state of not being added to the suite. */ def asOptional: Dimension = new OptionalDimension(name, values) private class OptionalDimension( override val name: String, override val values: List[String] ) extends Dimension(name, values, alias) { override val isOptional: Boolean = true override def asOptional: Dimension = this } // A bit of DSL for better readability of the test configs. def and(other: Dimension): List[Dimension] = this :: other :: Nil def and(others: List[Dimension]): List[Dimension] = this :: others def alone: List[Dimension] = this :: Nil } /** * A default [[Dimension]] implementation for dimensions with multiple possible values, such as * column mapping */ case class DimensionWithMultipleValues( override val name: String, override val values: List[String], override val alias: Option[String] = None ) extends Dimension(name, values, alias) { /** * Shortcut to create a [[DimensionMixin]] with the same name and one of the values as the suffix. * @param valueSelector a functions that selects a value from this dimension's values */ def withValueAsDimension(valueSelector: List[String] => String): DimensionMixin = { DimensionMixin(name, valueSelector(values), alias) } } /** * A specialized [[Dimension]] that does not have any values, it is either present or not. */ case class DimensionMixin( override val name: String, suffix: String = "Mixin", override val alias: Option[String] = None ) extends Dimension(name, List(suffix), alias) { lazy val traitName: String = name + suffix } /** * Main configuration class for the suite generator. It allows defining a set of base suites and the * dimension combinations that should be used to generate the test configurations. Suites are * generated for each base suite and for each value combination of the dimension combinations. * @param baseSuites a list of base class or trait names that contains the actual test cases. * Ideally, these should not contain any configuration logic, and instead rely on [[Dimension]]s to * make the necessary setup. */ case class TestConfig( baseSuites: List[String], dimensionCombinations: List[List[Dimension]] = List.empty ) /** * Represents a generated Scala file with suite definitions. * @param name the name of the generated file. * @param imports a list of packages that needs to be imported in this file. * @param testConfigs a list of [[TestConfig]]s that should be generated in this file. */ case class TestGroup( name: String, imports: List[Importer], testConfigs: List[TestConfig] ) object SuiteGeneratorConfig { private object Dims { // Just to improve readability of the test configurations a bit. // `Dims.NONE` is clearer than just `Nil`. val NONE: List[Dimension] = Nil val TABLE_ACCESS = DimensionWithMultipleValues( // no alias needed, value is self-explanatory "DeltaDMLTestUtils", List("NameBased", "PathBased"), alias = Some("")) val PATH_BASED = TABLE_ACCESS.withValueAsDimension(_.last) val NAME_BASED = TABLE_ACCESS.withValueAsDimension(_.head) val MERGE_SQL = DimensionMixin("MergeIntoSQL", alias = Some("SQL")) val MERGE_SCALA = DimensionMixin("MergeIntoScala", alias = Some("Scala")) val MERGE_DVS = DimensionMixin("MergeIntoDVs", alias = Some("DVs")) val PREDPUSH = DimensionWithMultipleValues( "PredicatePushdown", List("Disabled", "Enabled"), alias = Some("PredPush")) val CDC = DimensionMixin("CDC", suffix = "Enabled") // These enables/disable DVs on new tables, but leave DML command configs untouched. val PERSISTENT_DV = DimensionWithMultipleValues( "PersistentDV", List("Disabled", "Enabled"), alias = Some("DV")) val PERSISTENT_DV_OFF = PERSISTENT_DV.withValueAsDimension(_.head) val PERSISTENT_DV_ON = PERSISTENT_DV.withValueAsDimension(_.last) val ROW_TRACKING = DimensionWithMultipleValues("RowTracking", List("Disabled", "Enabled")) val ROW_TRACKING_ON = ROW_TRACKING.withValueAsDimension(_.last) val MERGE_PERSISTENT_DV_OFF = DimensionMixin("MergePersistentDV", suffix = "Disabled") val MERGE_ROW_TRACKING_DV = DimensionMixin("RowTrackingMergeDV") val COLUMN_MAPPING = DimensionWithMultipleValues( "DeltaColumnMappingEnable", List("IdMode", "NameMode"), alias = Some("ColMap")) val UPDATE_SCALA = DimensionMixin("UpdateScala", alias = Some("Scala")) val UPDATE_SQL = DimensionMixin("UpdateSQL", alias = Some("SQL")) val UPDATE_DVS = DimensionMixin("UpdateSQLWithDeletionVectors", alias = Some("DV")) val UPDATE_ROW_TRACKING_DV = DimensionMixin("RowTrackingUpdateDV") val DELETE_SCALA = DimensionMixin("DeleteScala", alias = Some("Scala")) val DELETE_SQL = DimensionMixin("DeleteSQL", alias = Some("SQL")) val DELETE_WITH_DVS = DimensionMixin("DeleteSQLWithDeletionVectors", alias = Some("DV")) } private object Tests { val MERGE_BASE = List( "MergeIntoBasicTests", "MergeIntoTempViewsTests", "MergeIntoNestedDataTests", "MergeIntoUnlimitedMergeClausesTests", "MergeIntoAnalysisExceptionTests", "MergeIntoExtendedSyntaxTests", "MergeIntoSuiteBaseMiscTests", "MergeIntoNotMatchedBySourceSuite", "MergeIntoNotMatchedBySourceCDCPart1Tests", "MergeIntoNotMatchedBySourceCDCPart2Tests", "MergeIntoSchemaEvolutionCoreTests", "MergeIntoSchemaEvolutionBaseNewColumnTests", "MergeIntoSchemaEvolutionBaseExistingColumnTests", "MergeIntoSchemaEvoStoreAssignmentPolicyTests", "MergeIntoSchemaEvolutionNotMatchedBySourceTests", "MergeIntoNestedStructInMapEvolutionTests", "MergeIntoNestedStructEvolutionUpdateOnlyTests", "MergeIntoNestedStructEvolutionInsertTests" ) val MERGE_SQL = List( "MergeIntoSQLTests", "MergeIntoSQLNondeterministicOrderTests" ) val UPDATE_BASE = List( "UpdateBaseTempViewTests", "UpdateBaseMiscTests" ) val DELETE_BASE = List( "DeleteTempViewTests", "DeleteBaseTests" ) } implicit class DimensionListExt(val dims: List[Dimension]) { /** * @return a new list of dimension combinations where each combination has the * [[commonDims]] prepended to it. */ def prependToAll(dimensionCombinations: List[Dimension]*): List[List[Dimension]] = { dimensionCombinations.toList.map(dims ::: _) } def prependToAll(dimensionCombinations: List[List[Dimension]]): List[List[Dimension]] = { prependToAll(dimensionCombinations: _*) } // Continued DSL from the Dimension class above to work around the different // operator precedence between :: and `and`. def and(other: Dimension): List[Dimension] = dims ::: other :: Nil def and(others: List[Dimension]): List[Dimension] = dims ::: others } /** * All [[TestGroup]] definitions. The generated suites of each group will be written * to a file named after the group name. Keep in mind that [[isExcluded]] can be used to filter * out some of the test configurations, so defining a configuration here does not guarantee * generation of a suite for it. */ lazy val TEST_GROUPS: List[TestGroup] = List( // scalastyle:off line.size.limit TestGroup( name = "MergeSuites", imports = List( importer"org.apache.spark.sql.delta._", importer"org.apache.spark.sql.delta.cdc._", importer"org.apache.spark.sql.delta.rowid._" ), testConfigs = List( TestConfig( "MergeIntoScalaTests" :: Tests.MERGE_BASE, List( List(Dims.MERGE_SCALA) ) ), TestConfig( "MergeCDCTests" :: "MergeIntoDVsTests" :: Tests.MERGE_SQL ::: Tests.MERGE_BASE, List(Dims.MERGE_SQL).prependToAll( List(Dims.NAME_BASED), List(Dims.PATH_BASED, Dims.COLUMN_MAPPING.asOptional), List(Dims.PATH_BASED, Dims.MERGE_DVS, Dims.PREDPUSH), List(Dims.PATH_BASED, Dims.CDC), List(Dims.PATH_BASED, Dims.CDC, Dims.MERGE_DVS, Dims.PREDPUSH) ) ), TestConfig( List("MergeIntoMaterializeSourceTests", "MergeIntoMaterializeSourceErrorTests"), List( List(Dims.MERGE_PERSISTENT_DV_OFF) ) ), TestConfig( List("RowTrackingMergeCommonTests"), List(Dims.NAME_BASED, Dims.CDC.asOptional).prependToAll( List(Dims.MERGE_ROW_TRACKING_DV.asOptional), List(Dims.PERSISTENT_DV_OFF, Dims.MERGE_PERSISTENT_DV_OFF) ) ::: List(Dims.NAME_BASED, Dims.COLUMN_MAPPING).prependToAll( List(), List(Dims.CDC, Dims.MERGE_ROW_TRACKING_DV) ) ), TestConfig( "MergeIntoTopLevelStructEvolutionNullnessTests" :: "MergeIntoNestedStructEvolutionNullnessTests" :: "MergeIntoTopLevelArrayStructEvolutionNullnessTests" :: "MergeIntoNestedArrayStructEvolutionNullnessTests" :: "MergeIntoTopLevelMapStructEvolutionNullnessTests" :: "MergeIntoNestedMapStructEvolutionNullnessTests" :: "MergeIntoStructEvolutionNullnessMultiClauseTests" :: Nil, List( List( Dims.MERGE_SQL, Dims.NAME_BASED ) ) ) ) ), TestGroup( name = "UpdateSuites", imports = List( importer"org.apache.spark.sql.delta._", importer"org.apache.spark.sql.delta.cdc._", importer"org.apache.spark.sql.delta.rowid._", importer"org.apache.spark.sql.delta.rowtracking._" ), testConfigs = List( TestConfig( "UpdateScalaTests" :: Tests.UPDATE_BASE, List( List(Dims.UPDATE_SCALA) ) ), TestConfig( "UpdateSQLTests" :: Tests.UPDATE_BASE, List( List(Dims.UPDATE_SQL, Dims.NAME_BASED) ) ), TestConfig( "UpdateCDCWithDeletionVectorsTests" :: "UpdateCDCTests" :: "UpdateSQLWithDeletionVectorsTests" :: "UpdateSQLTests" :: Tests.UPDATE_BASE, List( List(Dims.UPDATE_SQL, Dims.PATH_BASED, Dims.CDC.asOptional, Dims.ROW_TRACKING.asOptional), List(Dims.UPDATE_SQL, Dims.PATH_BASED, Dims.CDC, Dims.UPDATE_DVS), List(Dims.UPDATE_SQL, Dims.PATH_BASED, Dims.UPDATE_DVS, Dims.PREDPUSH) ) ), TestConfig( List("RowTrackingUpdateCommonTests"), List( List(Dims.CDC.asOptional, Dims.COLUMN_MAPPING.asOptional), List(Dims.UPDATE_ROW_TRACKING_DV), List(Dims.UPDATE_ROW_TRACKING_DV, Dims.CDC, Dims.COLUMN_MAPPING.asOptional) ) ) ) ), TestGroup( name = "DeleteSuites", imports = List( importer"org.apache.spark.sql.delta._", importer"org.apache.spark.sql.delta.cdc._", importer"org.apache.spark.sql.delta.rowid._" ), testConfigs = List( TestConfig( "DeleteScalaTests" :: Tests.DELETE_BASE, List( List(Dims.DELETE_SCALA) ) ), TestConfig( "DeleteCDCTests" :: "DeleteSQLTests" :: Tests.DELETE_BASE, List( List(Dims.DELETE_SQL, Dims.NAME_BASED), List(Dims.DELETE_SQL, Dims.PATH_BASED, Dims.COLUMN_MAPPING.asOptional), List(Dims.DELETE_SQL, Dims.PATH_BASED, Dims.DELETE_WITH_DVS, Dims.PREDPUSH), List(Dims.DELETE_SQL, Dims.PATH_BASED, Dims.CDC) ) ), TestConfig( List("RowTrackingDeleteSuiteBase", "RowTrackingDeleteDvBase"), List( List(Dims.CDC.asOptional, Dims.PERSISTENT_DV), List(Dims.PERSISTENT_DV_OFF, Dims.COLUMN_MAPPING), List(Dims.CDC, Dims.PERSISTENT_DV_ON, Dims.COLUMN_MAPPING) ) ) ) ), TestGroup( name = "InsertSuites", imports = List( importer"org.apache.spark.sql.delta._" ), testConfigs = List( TestConfig( List("DeltaInsertIntoImplicitCastTests", "DeltaInsertIntoImplicitCastStreamingWriteTests"), List( List() ) ) ) ) // scalastyle:on line.size.limit ) /** * Decides if a suite with the given base test and mixins should be generated or not. This is used * to exclude certain combinations of base suites and dimensions that are known to not work * together, or it can also be used to enforce presence of some dimensions for a certain base * suite. */ def isExcluded(base: String, mixins: List[String]): Boolean = { base match { // Exclude tempViews, because DeltaTable.forName does not resolve them correctly, so no one // can use them anyway with the Scala API. case "MergeIntoTempViewsTests" => mixins.contains(Dims.MERGE_SCALA.traitName) case "UpdateBaseTempViewTests" => mixins.contains(Dims.UPDATE_SCALA.traitName) case "DeleteTempViewTests" => mixins.contains(Dims.DELETE_SCALA.traitName) // The following tests only make sense if the dimension is present case "MergeCDCTests" | "UpdateCDCTests" | "DeleteCDCTests" => !mixins.contains(Dims.CDC.traitName) case "MergeIntoDVsTests" => !mixins.contains(Dims.MERGE_DVS.traitName) case "UpdateSQLWithDeletionVectorsTests" => !mixins.contains(Dims.UPDATE_DVS.traitName) case "UpdateCDCWithDeletionVectorsTests" => !List(Dims.UPDATE_DVS, Dims.CDC).map(_.traitName).forall(mixins.contains) case "RowTrackingDeleteDvBase" => !mixins.contains(Dims.PERSISTENT_DV_ON.traitName) case _ => false } } /** * Used to add custom traits to some combinations of base suites and dimensions. * @return all traits that needs to be extended for this test combination (incl. provided mixins). */ def applyCustomRulesAndGetAllMixins(base: String, mixins: List[String]): List[String] = { var finalMixins = new ListBuffer[String] finalMixins ++= mixins if (mixins.contains(Dims.MERGE_SQL.traitName)) { if (Dims.COLUMN_MAPPING.traitNames.exists(mixins.contains)) { finalMixins += "MergeIntoSQLColumnMappingOverrides" } if (mixins.contains(Dims.CDC.traitName)) { finalMixins += "MergeCDCMixin" if (mixins.contains(Dims.MERGE_DVS.traitName)) { finalMixins += "MergeCDCWithDVsMixin" } } } if (mixins.contains(Dims.UPDATE_SQL.traitName)) { if (mixins.contains(Dims.ROW_TRACKING.traitNames.last)) { finalMixins += "UpdateWithRowTrackingOverrides" } } if (mixins.contains(Dims.DELETE_SQL.traitName)) { if (mixins.contains(Dims.CDC.traitName)) { finalMixins += "DeleteCDCMixin" } if (mixins.contains(Dims.COLUMN_MAPPING.traitNames.last)) { finalMixins += "DeleteSQLNameColumnMappingMixin" } } finalMixins.result() } } ================================================ FILE: spark/delta-suite-generator/src/main/scala/io/delta/suitegenerator/SuitesWriter.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.suitegenerator import java.nio.charset.StandardCharsets import java.nio.file.{Files, Path} import scala.collection.mutable.ListBuffer import scala.io.Source import scala.jdk.CollectionConverters._ import scala.meta._ import org.scalafmt.Scalafmt /** * Contains the constants for the SuitesWriter class */ object SuitesWriter { private val LEGAL_HEADER = """ Copyright (2021) The Delta Lake Project Authors. | | Licensed under the Apache License, Version 2.0 (the "License"); | you may not use this file except in compliance with the License. | You may obtain a copy of the License at | | http://www.apache.org/licenses/LICENSE-2.0 | | Unless required by applicable law or agreed to in writing, software | distributed under the License is distributed on an "AS IS" BASIS, | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | See the License for the specific language governing permissions and | limitations under the License.""".stripMargin private val WARNING_HEADER = """ *********************************************************************************** | * This file is automatically generated. Manual modification is not allowed. * | * There is a unit test that should prevent merging a manual change. * | * * | * To make changes to the suites, modify the generator script config at * | * SuiteGeneratorConfig.scala and run it. The generator can be run via the * | * sbt command deltaSuiteGenerator / run. * | * * | * DO NOT TOUCH ANYTHING IN THIS FILE! * | ***********************************************************************************""" .stripMargin private lazy val PACKAGE_NAME = ModularSuiteGenerator.GENERATED_PACKAGE.parse[Term].get.asInstanceOf[Term.Ref] private lazy val SRC_HEADERS = s"""/* |${LEGAL_HEADER.linesWithSeparators.map(" *" + _).mkString} | */ | |${WARNING_HEADER.linesWithSeparators.map("//" + _).mkString} | |""".stripMargin private lazy val SCALAFMT_CONFIG = { val source = Source.fromURL(getClass.getClassLoader.getResource("scalafmt.conf")) try { Scalafmt.parseHoconConfig(source.mkString).get } finally { source.close() } } } /** * Used to write the generated suites. The output is written to the given directory. This directory * should not contain any manually written files, otherwise the generator will throw an error. * @param outputDir the path to the directory where the generated suites will be written. */ class SuitesWriter(val outputDir: Path) { import SuitesWriter._ protected val allFiles: ListBuffer[Path] = ListBuffer.empty[Path] def writeGeneratedSuitesOfGroup(suites: List[TestSuite], testGroup: TestGroup): Unit = { suites // Group by parent class: first item of the extends clause (last item of class def) .groupBy(suite => suite.classDefinition.children.last.children.head.text) .foreach { case (baseSuite, suites) => val src = SRC_HEADERS + "// scalastyle:off line.size.limit\n" + source"""package $PACKAGE_NAME import ..${testGroup.imports} ..${suites.sortBy(_.name).map(_.classDefinition)}""" val srcFile = outputDir.resolve(s"${testGroup.name}$baseSuite.scala") val formattedSrc = Scalafmt.format(src, SCALAFMT_CONFIG).get writeFile(srcFile, formattedSrc) } // scalastyle:off println println(s"Wrote ${suites.size} generated suites from ${testGroup.name} group.") // scalastyle:on println } protected def writeFile(file: Path, content: String): Unit = { Files.write(file, content.getBytes(StandardCharsets.UTF_8)) allFiles += file } def conclude(): Unit = { val additionalFiles = Files.list(outputDir) .iterator() .asScala .filterNot(allFiles.contains) .map(_.getFileName) .mkString(", ") assert(additionalFiles.isEmpty, s"Unexpected files found in $outputDir: $additionalFiles. " + s"Please manually delete them.") } } ================================================ FILE: spark/delta-suite-generator/src/test/scala/io/delta/suitegenerator/ValidateGeneratedSuites.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.suitegenerator import java.nio.charset.StandardCharsets import java.nio.file.{Files, Path, Paths} import org.scalatest.funsuite.AnyFunSuite class ValidateGeneratedSuites extends AnyFunSuite { test("Generated suites are not manually modified") { // This test must be executed from the repository root for this relative path to work val outputDir = Paths.get(ModularSuiteGenerator.OUTPUT_PATH) val suitesValidator = new SuitesValidator(outputDir) ModularSuiteGenerator.generateSuites(suitesValidator) } } /** * Instead of writing to the files, validates that the files match the expected content. */ class SuitesValidator(override val outputDir: Path) extends SuitesWriter(outputDir) { override def writeFile(file: Path, content: String): Unit = { assert(Files.exists(file), s"File $file does not exist. Please run the generator to create it.") val fileContent = new String(Files.readAllBytes(file), StandardCharsets.UTF_8) assert(fileContent == content, s"File $file does not match the expected content. Please run the generator to update it.") allFiles += file } } ================================================ FILE: spark/src/main/antlr4/io/delta/sql/parser/DeltaSqlBase.g4 ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ grammar DeltaSqlBase; @members { /** * Verify whether current token is a valid decimal token (which contains dot). * Returns true if the character that follows the token is not a digit or letter or underscore. * * For example: * For char stream "2.3", "2." is not a valid decimal token, because it is followed by digit '3'. * For char stream "2.3_", "2.3" is not a valid decimal token, because it is followed by '_'. * For char stream "2.3W", "2.3" is not a valid decimal token, because it is followed by 'W'. * For char stream "12.0D 34.E2+0.12 " 12.0D is a valid decimal token because it is folllowed * by a space. 34.E2 is a valid decimal token because it is followed by symbol '+' * which is not a digit or letter or underscore. */ public boolean isValidDecimal() { int nextChar = _input.LA(1); if (nextChar >= 'A' && nextChar <= 'Z' || nextChar >= '0' && nextChar <= '9' || nextChar == '_') { return false; } else { return true; } } } tokens { DELIMITER } singleStatement : statement ';'* EOF ; // If you add keywords here that should not be reserved, add them to 'nonReserved' list. statement : VACUUM (path=stringLit | table=qualifiedName) vacuumModifiers #vacuumTable | (DESC | DESCRIBE) DETAIL (path=stringLit | table=qualifiedName) #describeDeltaDetail | GENERATE modeName=identifier FOR TABLE table=qualifiedName #generate | (DESC | DESCRIBE) HISTORY (path=stringLit | table=qualifiedName) (LIMIT limit=INTEGER_VALUE)? #describeDeltaHistory | CONVERT TO DELTA table=qualifiedName (NO STATISTICS)? (PARTITIONED BY '(' colTypeList ')')? #convert | RESTORE TABLE? table=qualifiedName TO? clause=temporalClause #restore | ALTER TABLE table=qualifiedName ADD CONSTRAINT name=identifier constraint #addTableConstraint | ALTER TABLE table=qualifiedName DROP CONSTRAINT (IF EXISTS)? name=identifier #dropTableConstraint | ALTER TABLE table=qualifiedName DROP FEATURE featureName=featureNameValue (TRUNCATE HISTORY)? #alterTableDropFeature | ALTER TABLE table=qualifiedName (clusterBySpec | CLUSTER BY NONE) #alterTableClusterBy | ALTER TABLE table=qualifiedName (ALTER | CHANGE) COLUMN? column=qualifiedName SYNC IDENTITY #alterTableSyncIdentity | OPTIMIZE (path=stringLit | table=qualifiedName) FULL? (WHERE partitionPredicate=predicateToken)? (zorderSpec)? #optimizeTable | REORG TABLE table=qualifiedName ( (WHERE partitionPredicate=predicateToken)? APPLY LEFT_PAREN PURGE RIGHT_PAREN | APPLY LEFT_PAREN UPGRADE UNIFORM LEFT_PAREN ICEBERG_COMPAT_VERSION EQ version=INTEGER_VALUE RIGHT_PAREN RIGHT_PAREN ) #reorgTable | cloneTableHeader SHALLOW CLONE source=qualifiedName clause=temporalClause? (TBLPROPERTIES tableProps=propertyList)? (LOCATION location=stringLit)? #clone | .*? clusterBySpec+ .*? #clusterBy | .*? #passThrough ; createTableHeader : CREATE TABLE (IF NOT EXISTS)? table=qualifiedName ; replaceTableHeader : (CREATE OR)? REPLACE TABLE table=qualifiedName ; cloneTableHeader : createTableHeader | replaceTableHeader ; zorderSpec : ZORDER BY LEFT_PAREN interleave+=qualifiedName (COMMA interleave+=qualifiedName)* RIGHT_PAREN | ZORDER BY interleave+=qualifiedName (COMMA interleave+=qualifiedName)* ; clusterBySpec : CLUSTER BY LEFT_PAREN interleave+=qualifiedName (COMMA interleave+=qualifiedName)* RIGHT_PAREN ; temporalClause : FOR? (SYSTEM_VERSION | VERSION) AS OF version=(INTEGER_VALUE | STRING) | FOR? (SYSTEM_TIME | TIMESTAMP) AS OF timestamp=STRING ; qualifiedName : identifier ('.' identifier)* ('.' identifier)* ; propertyList : LEFT_PAREN property (COMMA property)* RIGHT_PAREN ; property : key=propertyKey (EQ? value=propertyValue)? ; propertyKey : identifier (DOT identifier)* | stringLit ; propertyValue : INTEGER_VALUE | DECIMAL_VALUE | booleanValue | identifier LEFT_PAREN stringLit COMMA stringLit RIGHT_PAREN | value=stringLit ; featureNameValue : identifier | stringLit ; singleStringLit : STRING | DOUBLEQUOTED_STRING ; stringLit : singleStringLit+ ; booleanValue : TRUE | FALSE ; identifier : IDENTIFIER #unquotedIdentifier | quotedIdentifier #quotedIdentifierAlternative | nonReserved #unquotedIdentifier ; quotedIdentifier : BACKQUOTED_IDENTIFIER ; colTypeList : colType (',' colType)* ; colType : colName=identifier dataType (NOT NULL)? (COMMENT comment=stringLit)? ; dataType : identifier ('(' INTEGER_VALUE (',' INTEGER_VALUE)* ')')? #primitiveDataType ; vacuumModifiers : (vacuumType | inventory | retain | dryRun)* ; vacuumType : LITE|FULL ; inventory : USING INVENTORY (inventoryTable=qualifiedName | LEFT_PAREN inventoryQuery=subQuery RIGHT_PAREN) ; retain : RETAIN number HOURS ; dryRun : DRY RUN ; number : MINUS? DECIMAL_VALUE #decimalLiteral | MINUS? INTEGER_VALUE #integerLiteral | MINUS? BIGINT_LITERAL #bigIntLiteral | MINUS? SMALLINT_LITERAL #smallIntLiteral | MINUS? TINYINT_LITERAL #tinyIntLiteral | MINUS? DOUBLE_LITERAL #doubleLiteral | MINUS? BIGDECIMAL_LITERAL #bigDecimalLiteral ; constraint : CHECK '(' exprToken+ ')' #checkConstraint ; // We don't have an expression rule in our grammar here, so we just grab the tokens and defer // parsing them to later. Although this is the same as `exprToken`, we have to re-define it to // workaround an ANTLR issue (https://github.com/delta-io/delta/issues/1205) predicateToken : .+? ; // We don't have an expression rule in our grammar here, so we just grab the tokens and defer // parsing them to later. Although this is the same as `exprToken`, `predicateToken`, we have to re-define it to // workaround an ANTLR issue (https://github.com/delta-io/delta/issues/1205). Should we remove this after // https://github.com/delta-io/delta/pull/1800 subQuery : .+? ; // We don't have an expression rule in our grammar here, so we just grab the tokens and defer // parsing them to later. exprToken : .+? ; // Add keywords here so that people's queries don't break if they have a column name as one of // these tokens nonReserved : VACUUM | FULL | LITE | USING | INVENTORY | RETAIN | HOURS | DRY | RUN | CONVERT | TO | DELTA | PARTITIONED | BY | DESC | DESCRIBE | LIMIT | DETAIL | GENERATE | FOR | TABLE | CHECK | EXISTS | OPTIMIZE | FULL | IDENTITY | SYNC | COLUMN | CHANGE | REORG | APPLY | PURGE | UPGRADE | UNIFORM | ICEBERG_COMPAT_VERSION | RESTORE | AS | OF | ZORDER | LEFT_PAREN | RIGHT_PAREN | NO | STATISTICS | CLONE | SHALLOW | FEATURE | TRUNCATE | CLUSTER | NONE ; // Define how the keywords above should appear in a user's SQL statement. ADD: 'ADD'; ALTER: 'ALTER'; APPLY: 'APPLY'; AS: 'AS'; BY: 'BY'; CHANGE: 'CHANGE'; CHECK: 'CHECK'; CLONE: 'CLONE'; CLUSTER: 'CLUSTER'; COLUMN: 'COLUMN'; COMMA: ','; COMMENT: 'COMMENT'; CONSTRAINT: 'CONSTRAINT'; CONVERT: 'CONVERT'; CREATE: 'CREATE'; DELTA: 'DELTA'; DESC: 'DESC'; DESCRIBE: 'DESCRIBE'; DETAIL: 'DETAIL'; DOT: '.'; DROP: 'DROP'; DRY: 'DRY'; EXISTS: 'EXISTS'; FALSE: 'FALSE'; FEATURE: 'FEATURE'; FOR: 'FOR'; FULL: 'FULL'; GENERATE: 'GENERATE'; HISTORY: 'HISTORY'; HOURS: 'HOURS'; ICEBERG_COMPAT_VERSION: 'ICEBERG_COMPAT_VERSION'; IDENTITY: 'IDENTITY'; IF: 'IF'; INVENTORY: 'INVENTORY'; LEFT_PAREN: '('; LIMIT: 'LIMIT'; LITE: 'LITE'; LOCATION: 'LOCATION'; MINUS: '-'; NO: 'NO'; NONE: 'NONE'; NOT: 'NOT' | '!'; NULL: 'NULL'; OF: 'OF'; OR: 'OR'; OPTIMIZE: 'OPTIMIZE'; REORG: 'REORG'; PARTITIONED: 'PARTITIONED'; PURGE: 'PURGE'; REPLACE: 'REPLACE'; RESTORE: 'RESTORE'; RETAIN: 'RETAIN'; RIGHT_PAREN: ')'; RUN: 'RUN'; SHALLOW: 'SHALLOW'; SYNC: 'SYNC'; SYSTEM_TIME: 'SYSTEM_TIME'; SYSTEM_VERSION: 'SYSTEM_VERSION'; TABLE: 'TABLE'; TBLPROPERTIES: 'TBLPROPERTIES'; TIMESTAMP: 'TIMESTAMP'; TRUNCATE: 'TRUNCATE'; TO: 'TO'; TRUE: 'TRUE'; UNIFORM: 'UNIFORM'; UPGRADE: 'UPGRADE'; USING: 'USING'; VACUUM: 'VACUUM'; VERSION: 'VERSION'; WHERE: 'WHERE'; ZORDER: 'ZORDER'; STATISTICS: 'STATISTICS'; // Multi-character operator tokens need to be defined even though we don't explicitly reference // them so that they can be recognized as single tokens when parsing. If we split them up and // end up with expression text like 'a ! = b', Spark won't be able to parse '! =' back into the // != operator. EQ : '=' | '=='; NSEQ: '<=>'; NEQ : '<>'; NEQJ: '!='; LTE : '<=' | '!>'; GTE : '>=' | '!<'; CONCAT_PIPE: '||'; STRING : '\'' ( ~('\''|'\\') | ('\\' .) )* '\'' | '"' ( ~('"'|'\\') | ('\\' .) )* '"' ; DOUBLEQUOTED_STRING :'"' ( ~('"'|'\\') | ('\\' .) )* '"' ; BIGINT_LITERAL : DIGIT+ 'L' ; SMALLINT_LITERAL : DIGIT+ 'S' ; TINYINT_LITERAL : DIGIT+ 'Y' ; INTEGER_VALUE : DIGIT+ ; DECIMAL_VALUE : DIGIT+ EXPONENT | DECIMAL_DIGITS EXPONENT? {isValidDecimal()}? ; DOUBLE_LITERAL : DIGIT+ EXPONENT? 'D' | DECIMAL_DIGITS EXPONENT? 'D' {isValidDecimal()}? ; BIGDECIMAL_LITERAL : DIGIT+ EXPONENT? 'BD' | DECIMAL_DIGITS EXPONENT? 'BD' {isValidDecimal()}? ; IDENTIFIER : (LETTER | DIGIT | '_')+ ; BACKQUOTED_IDENTIFIER : '`' ( ~'`' | '``' )* '`' ; fragment DECIMAL_DIGITS : DIGIT+ '.' DIGIT* | '.' DIGIT+ ; fragment EXPONENT : 'E' [+-]? DIGIT+ ; fragment DIGIT : [0-9] ; fragment LETTER : [A-Z] ; SIMPLE_COMMENT : '--' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN) ; BRACKETED_COMMENT : '/*' .*? '*/' -> channel(HIDDEN) ; WS : [ \r\n\t]+ -> channel(HIDDEN) ; // Catch-all for anything we can't recognize. // We use this to be able to ignore and recover all the text // when splitting statements with DelimiterLexer UNRECOGNIZED : . ; ================================================ FILE: spark/src/main/java/io/delta/dynamodbcommitcoordinator/DynamoDBCommitCoordinatorClient.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.dynamodbcommitcoordinator; import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; import com.amazonaws.services.dynamodbv2.model.*; import io.delta.storage.CloseableIterator; import io.delta.storage.LogStore; import io.delta.storage.commit.*; import io.delta.storage.commit.actions.AbstractMetadata; import io.delta.storage.commit.actions.AbstractProtocol; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; import java.util.*; /** * A commit coordinator client that uses DynamoDB as the commit coordinator. The table schema is as follows: * tableId: String --- The unique identifier for the table. This is a UUID. * path: String --- The fully qualified path of the table in the file system. e.g. s3://bucket/path. * acceptingCommits: Boolean --- Whether the commit coordinator is accepting new commits. This will only * be set to false when the table is converted from coordinated commits to file system commits. * tableVersion: Number --- The version of the latest commit. * tableTimestamp: Number --- The inCommitTimestamp of the latest commit. * schemaVersion: Number --- The version of the schema used to store the data. * hasAcceptedCommits: Boolean --- Whether any actual commits have been accepted by this commit coordinator * after `registerTable`. * commits: --- The list of unbackfilled commits. * version: Number --- The version of the commit. * inCommitTimestamp: Number --- The inCommitTimestamp of the commit. * fsName: String --- The name of the unbackfilled file. * fsLength: Number --- The length of the unbackfilled file. * fsTimestamp: Number --- The modification time of the unbackfilled file. */ public class DynamoDBCommitCoordinatorClient implements CommitCoordinatorClient { private static final Logger LOG = LoggerFactory.getLogger(DynamoDBCommitCoordinatorClient.class); /** * The name of the DynamoDB table used to store unbackfilled commits. */ final String coordinatedCommitsTableName; /** * The DynamoDB client used to interact with the DynamoDB table. */ final AmazonDynamoDB client; /** * The endpoint of the DynamoDB table. */ final String endpoint; /** * The number of write capacity units to provision for the DynamoDB table if the * client ends up creating a new one. */ final long writeCapacityUnits; /** * The number of read capacity units to provision for the DynamoDB table if the * client ends up creating a new one. */ final long readCapacityUnits; /** * The number of commits to batch backfill at once. A backfill is performed * whenever commitVersion % batchSize == 0. */ public final long backfillBatchSize; /** * Whether we should skip matching the current table path against the one stored in DynamoDB * when interacting with it. */ final boolean skipPathCheck; /** * The key used to store the tableId in the coordinated commits table configuration. */ final static String TABLE_CONF_TABLE_ID_KEY = "tableId"; /** * The version of the client. This is used to ensure that the client is compatible with the * schema of the data stored in the DynamoDB table. A client should only be able to * access a table if the schema version of the table matches the client version. */ final int CLIENT_VERSION = 1; private static class GetCommitsResultInternal { final GetCommitsResponse response; final boolean hasAcceptedCommits; GetCommitsResultInternal( GetCommitsResponse response, boolean hasAcceptedCommits) { this.response = response; this.hasAcceptedCommits = hasAcceptedCommits; } } public DynamoDBCommitCoordinatorClient( String coordinatedCommitsTableName, String endpoint, AmazonDynamoDB client, long backfillBatchSize) throws IOException { this( coordinatedCommitsTableName, endpoint, client, backfillBatchSize, 5 /* readCapacityUnits */, 5 /* writeCapacityUnits */, false /* skipPathCheck */); } public DynamoDBCommitCoordinatorClient( String coordinatedCommitsTableName, String endpoint, AmazonDynamoDB client, long backfillBatchSize, long readCapacityUnits, long writeCapacityUnits, boolean skipPathCheck) throws IOException { this.coordinatedCommitsTableName = coordinatedCommitsTableName; this.endpoint = endpoint; this.client = client; this.backfillBatchSize = backfillBatchSize; this.readCapacityUnits = readCapacityUnits; this.writeCapacityUnits = writeCapacityUnits; this.skipPathCheck = skipPathCheck; tryEnsureTableExists(); } private String getTableId(Map coordinatedCommitsTableConf) { if (!coordinatedCommitsTableConf.containsKey(TABLE_CONF_TABLE_ID_KEY)) { throw new RuntimeException("tableId not found"); } return coordinatedCommitsTableConf.get(TABLE_CONF_TABLE_ID_KEY); } /** * Fetches the entry from the commit coordinator for the given table. Only the attributes defined * in attributesToGet will be fetched. */ private GetItemResult getEntryFromCommitCoordinator( Map coordinatedCommitsTableConf, String... attributesToGet) { GetItemRequest request = new GetItemRequest() .withTableName(coordinatedCommitsTableName) .addKeyEntry( DynamoDBTableEntryConstants.TABLE_ID, new AttributeValue().withS(getTableId(coordinatedCommitsTableConf))) .withAttributesToGet(attributesToGet); return client.getItem(request); } /** * Commits the given file to the commit coordinator. * A conditional write is performed to the DynamoDB table entry associated with this Delta * table. * If the conditional write goes through, the filestatus of the UUID delta file will be * appended to the list of unbackfilled commits, and other updates like setting the latest * table version to `attemptVersion` will be performed. * * For the conditional write to go through, the following conditions must be met right before * the write is performed: * 1. The latest table version in DynamoDB is equal to attemptVersion - 1. * 2. The commit coordinator is accepting new commits. * 3. The schema version of the commit coordinator matches the schema version of the client. * 4. The table path stored in DynamoDB matches the path of the table. This check is skipped * if `skipPathCheck` is set to true. * If the conditional write fails, we retrieve the current entry in DynamoDB to figure out * which condition failed. (DynamoDB does not tell us which condition failed in the rejection.) * If any of (2), (3), or (4) fail, an unretryable `CommitFailedException` will be thrown. * For (1): * If the retrieved latest table version is greater than or equal to attemptVersion, a retryable * `CommitFailedException` will be thrown. * If the retrieved latest table version is less than attemptVersion - 1, an unretryable * `CommitFailedException` will be thrown. */ protected CommitResponse commitToCoordinator( Path logPath, Map coordinatedCommitsTableConf, long attemptVersion, FileStatus commitFile, long inCommitTimestamp, boolean isCCtoFSConversion) throws CommitFailedException { // Add conditions for the conditional update. java.util.Map expectedValuesBeforeUpdate = new HashMap<>(); expectedValuesBeforeUpdate.put( DynamoDBTableEntryConstants.TABLE_LATEST_VERSION, new ExpectedAttributeValue() .withValue(new AttributeValue().withN(Long.toString(attemptVersion - 1))) ); expectedValuesBeforeUpdate.put( DynamoDBTableEntryConstants.ACCEPTING_COMMITS, new ExpectedAttributeValue() .withValue(new AttributeValue().withBOOL(true))); if (!skipPathCheck) { expectedValuesBeforeUpdate.put( DynamoDBTableEntryConstants.TABLE_PATH, new ExpectedAttributeValue() .withValue(new AttributeValue().withS(logPath.getParent().toString()))); } expectedValuesBeforeUpdate.put( DynamoDBTableEntryConstants.SCHEMA_VERSION, new ExpectedAttributeValue() .withValue(new AttributeValue().withN(Integer.toString(CLIENT_VERSION)))); java.util.Map newCommit = new HashMap<>(); newCommit.put( DynamoDBTableEntryConstants.COMMIT_VERSION, new AttributeValue().withN(Long.toString(attemptVersion))); newCommit.put( DynamoDBTableEntryConstants.COMMIT_TIMESTAMP, new AttributeValue().withN(Long.toString(inCommitTimestamp))); newCommit.put( DynamoDBTableEntryConstants.COMMIT_FILE_NAME, new AttributeValue().withS(commitFile.getPath().getName())); newCommit.put( DynamoDBTableEntryConstants.COMMIT_FILE_LENGTH, new AttributeValue().withN(Long.toString(commitFile.getLen()))); newCommit.put( DynamoDBTableEntryConstants.COMMIT_FILE_MODIFICATION_TIMESTAMP, new AttributeValue().withN(Long.toString(commitFile.getModificationTime()))); UpdateItemRequest request = new UpdateItemRequest() .withTableName(coordinatedCommitsTableName) .addKeyEntry( DynamoDBTableEntryConstants.TABLE_ID, new AttributeValue().withS(getTableId(coordinatedCommitsTableConf))) .addAttributeUpdatesEntry( DynamoDBTableEntryConstants.TABLE_LATEST_VERSION, new AttributeValueUpdate() .withValue(new AttributeValue().withN(Long.toString(attemptVersion))) .withAction(AttributeAction.PUT)) // We need to set this to true to indicate that commits have been accepted after // `registerTable`. .addAttributeUpdatesEntry( DynamoDBTableEntryConstants.HAS_ACCEPTED_COMMITS, new AttributeValueUpdate() .withValue(new AttributeValue().withBOOL(true)) .withAction(AttributeAction.PUT) ) .addAttributeUpdatesEntry( DynamoDBTableEntryConstants.TABLE_LATEST_TIMESTAMP, new AttributeValueUpdate() .withValue(new AttributeValue().withN(Long.toString(inCommitTimestamp))) .withAction(AttributeAction.PUT)) .addAttributeUpdatesEntry( DynamoDBTableEntryConstants.COMMITS, new AttributeValueUpdate() .withAction(AttributeAction.ADD) .withValue(new AttributeValue().withL( new AttributeValue().withM(newCommit) ) ) ) .withExpected(expectedValuesBeforeUpdate); if (isCCtoFSConversion) { // If this table is being converted from coordinated commits to file system commits, we need // to set acceptingCommits to false. request = request .addAttributeUpdatesEntry( DynamoDBTableEntryConstants.ACCEPTING_COMMITS, new AttributeValueUpdate() .withValue(new AttributeValue().withBOOL(false)) .withAction(AttributeAction.PUT) ); } try { client.updateItem(request); } catch (ConditionalCheckFailedException e) { // Conditional check failed. The exception will not indicate which condition failed. // We need to check the conditions ourselves by fetching the item and checking the // values. GetItemResult latestEntry = getEntryFromCommitCoordinator( coordinatedCommitsTableConf, DynamoDBTableEntryConstants.TABLE_LATEST_VERSION, DynamoDBTableEntryConstants.ACCEPTING_COMMITS, DynamoDBTableEntryConstants.TABLE_PATH, DynamoDBTableEntryConstants.SCHEMA_VERSION); int schemaVersion = Integer.parseInt( latestEntry.getItem().get(DynamoDBTableEntryConstants.SCHEMA_VERSION).getN()); if (schemaVersion != CLIENT_VERSION) { throw new CommitFailedException( false /* retryable */, false /* conflict */, "The schema version of the commit coordinator does not match the current" + "DynamoDBCommitCoordinatorClient version. The data schema version is " + " " + schemaVersion + " while the client version is " + CLIENT_VERSION + ". Make sure that the correct client is being " + "used to access this table." ); } long latestTableVersion = Long.parseLong( latestEntry.getItem().get(DynamoDBTableEntryConstants.TABLE_LATEST_VERSION).getN()); if (!skipPathCheck && !latestEntry.getItem().get("path").getS().equals(logPath.getParent().toString())) { throw new CommitFailedException( false /* retryable */, false /* conflict */, "This commit was attempted from path " + logPath.getParent() + " while the table is registered at " + latestEntry.getItem().get("path").getS() + "."); } if (!latestEntry.getItem().get(DynamoDBTableEntryConstants.ACCEPTING_COMMITS).getBOOL()) { throw new CommitFailedException( false /* retryable */, false /* conflict */, "The commit coordinator is not accepting any new commits for this table."); } if (latestTableVersion != attemptVersion - 1) { // The commit is only retryable if the conflict is due to someone else committing // a version greater than the expected version. boolean retryable = latestTableVersion > attemptVersion - 1; throw new CommitFailedException( retryable /* retryable */, retryable /* conflict */, "Commit version " + attemptVersion + " is not valid. Expected version: " + (latestTableVersion + 1) + "."); } } Commit resultantCommit = new Commit(attemptVersion, commitFile, inCommitTimestamp); return new CommitResponse(resultantCommit); } @Override public CommitResponse commit( LogStore logStore, Configuration hadoopConf, TableDescriptor tableDesc, long commitVersion, Iterator actions, UpdatedActions updatedActions) throws CommitFailedException { Path logPath = tableDesc.getLogPath(); if (commitVersion == 0) { throw new CommitFailedException( false /* retryable */, false /* conflict */, "Commit version 0 must go via filesystem."); } try { FileStatus commitFileStatus = CoordinatedCommitsUtils.writeUnbackfilledCommitFile( logStore, hadoopConf, logPath.toString(), commitVersion, actions, UUID.randomUUID().toString()); long inCommitTimestamp = updatedActions.getCommitInfo().getCommitTimestamp(); boolean isCCtoFSConversion = CoordinatedCommitsUtils.isCoordinatedCommitsToFSConversion(commitVersion, updatedActions); LOG.info("Committing version {} with UUID delta file {} to DynamoDB.", commitVersion, commitFileStatus.getPath()); CommitResponse res = commitToCoordinator( logPath, tableDesc.getTableConf(), commitVersion, commitFileStatus, inCommitTimestamp, isCCtoFSConversion); LOG.info("Commit {} was successful.", commitVersion); boolean shouldBackfillOnEveryCommit = backfillBatchSize <= 1; boolean isBatchBackfillDue = commitVersion % backfillBatchSize == 0; boolean shouldBackfill = shouldBackfillOnEveryCommit || isBatchBackfillDue || // Always attempt a backfill for coordinated commits to filesystem conversion. // Even if this fails, the next reader will attempt to backfill. isCCtoFSConversion; if (shouldBackfill) { backfillToVersion( logStore, hadoopConf, tableDesc, commitVersion, null /* lastKnownBackfilledVersion */); } return res; } catch (IOException e) { throw new CommitFailedException(false /* retryable */, false /* conflict */, e.getMessage(), e); } } private GetCommitsResultInternal getCommitsImpl( Path logPath, Map tableConf, Long startVersion, Long endVersion) throws IOException { GetItemResult latestEntry = getEntryFromCommitCoordinator( tableConf, DynamoDBTableEntryConstants.COMMITS, DynamoDBTableEntryConstants.TABLE_LATEST_VERSION, DynamoDBTableEntryConstants.HAS_ACCEPTED_COMMITS); java.util.Map item = latestEntry.getItem(); long currentVersion = Long.parseLong(item.get(DynamoDBTableEntryConstants.TABLE_LATEST_VERSION).getN()); AttributeValue allStoredCommits = item.get(DynamoDBTableEntryConstants.COMMITS); ArrayList commits = new ArrayList<>(); Path unbackfilledCommitsPath = CoordinatedCommitsUtils.commitDirPath(logPath); for(AttributeValue attr: allStoredCommits.getL()) { java.util.Map commitMap = attr.getM(); long commitVersion = Long.parseLong(commitMap.get(DynamoDBTableEntryConstants.COMMIT_VERSION).getN()); boolean commitInRange = (startVersion == null || commitVersion >= startVersion) && (endVersion == null || endVersion >= commitVersion); if (commitInRange) { Path filePath = new Path( unbackfilledCommitsPath, commitMap.get(DynamoDBTableEntryConstants.COMMIT_FILE_NAME).getS()); long length = Long.parseLong(commitMap.get(DynamoDBTableEntryConstants.COMMIT_FILE_LENGTH).getN()); long modificationTime = Long.parseLong( commitMap.get(DynamoDBTableEntryConstants.COMMIT_FILE_MODIFICATION_TIMESTAMP).getN()); FileStatus fileStatus = new FileStatus( length, false /* isDir */, 0 /* blockReplication */, 0 /* blockSize */, modificationTime, filePath); long inCommitTimestamp = Long.parseLong(commitMap.get(DynamoDBTableEntryConstants.COMMIT_TIMESTAMP).getN()); commits.add(new Commit(commitVersion, fileStatus, inCommitTimestamp)); } } GetCommitsResponse response = new GetCommitsResponse( new ArrayList(commits), currentVersion); return new GetCommitsResultInternal( response, item.get(DynamoDBTableEntryConstants.HAS_ACCEPTED_COMMITS).getBOOL()); } @Override public GetCommitsResponse getCommits( TableDescriptor tableDesc, Long startVersion, Long endVersion) { try { GetCommitsResultInternal res = getCommitsImpl(tableDesc.getLogPath(), tableDesc.getTableConf(), startVersion, endVersion); long latestTableVersionToReturn = res.response.getLatestTableVersion(); if (!res.hasAcceptedCommits) { /* * If the commit coordinator has not accepted any commits after `registerTable`, we should * return -1 as the latest table version. * ┌───────────────────────────────────┬─────────────────────────────────────────────────────┬────────────────────────────────┐ * │ Action │ Internal State │ Version returned on GetCommits │ * ├───────────────────────────────────┼─────────────────────────────────────────────────────┼────────────────────────────────┤ * │ Table is pre-registered at X │ hasAcceptedCommits = false, latestTableVersion = X │ -1 │ * │ Commit X+1 after pre-registration │ hasAcceptedCommits = true, latestTableVersion = X+1 │ X+1 │ * └───────────────────────────────────┴─────────────────────────────────────────────────────┴────────────────────────────────┘ */ latestTableVersionToReturn = -1; } return new GetCommitsResponse(res.response.getCommits(), latestTableVersionToReturn); } catch (IOException e) { throw new UncheckedIOException(e); } } /** * Writes the given actions to a file. * logStore.write(overwrite=false) will throw a FileAlreadyExistsException if the file already * exists. However, the scala LogStore interface does not declare this as part of the function * signature. This method wraps the write method and declares the exception to ensure that the * caller is aware of the exception. */ private void writeActionsToBackfilledFile( LogStore logStore, Path logPath, long version, Iterator actions, Configuration hadoopConf, boolean shouldOverwrite) throws IOException { Path targetPath = CoordinatedCommitsUtils.getBackfilledDeltaFilePath(logPath, version); logStore.write(targetPath, actions, shouldOverwrite, hadoopConf); } private void validateBackfilledFileExists( Path logPath, Configuration hadoopConf, Long lastKnownBackfilledVersion) { try { if (lastKnownBackfilledVersion == null) { return; } Path lastKnownBackfilledFile = CoordinatedCommitsUtils.getBackfilledDeltaFilePath( logPath, lastKnownBackfilledVersion); FileSystem fs = logPath.getFileSystem(hadoopConf); if (!fs.exists(lastKnownBackfilledFile)) { throw new IllegalArgumentException( "Expected backfilled file at " + lastKnownBackfilledFile + " does not exist."); } } catch (IOException e) { throw new UncheckedIOException(e); } } /** * Backfills all the unbackfilled commits returned by the commit coordinator and notifies the commit * owner of the backfills. * The version parameter is ignored in this implementation and all the unbackfilled commits * are backfilled. This method will not throw any exception if the physical backfill * succeeds but the update to the commit coordinator fails. * @throws IllegalArgumentException if the requested backfill version is greater than the latest * version for the table. */ @Override public void backfillToVersion( LogStore logStore, Configuration hadoopConf, TableDescriptor tableDesc, long version, Long lastKnownBackfilledVersion) throws IOException { LOG.info("Backfilling all unbackfilled commits."); Path logPath = tableDesc.getLogPath(); GetCommitsResponse resp; try { resp = getCommitsImpl( logPath, tableDesc.getTableConf(), lastKnownBackfilledVersion, null).response; } catch (IOException e) { throw new UncheckedIOException(e); } validateBackfilledFileExists(logPath, hadoopConf, lastKnownBackfilledVersion); if (version > resp.getLatestTableVersion()) { throw new IllegalArgumentException( "The requested backfill version " + version + " is greater than the latest " + "version " + resp.getLatestTableVersion() + " for the table."); } // If partial writes are visible in this filesystem, we should not try to overwrite existing // files. A failed overwrite can truncate the existing file. boolean shouldOverwrite = !logStore.isPartialWriteVisible( logPath, hadoopConf); for (Commit commit: resp.getCommits()) { CloseableIterator actions = logStore.read(commit.getFileStatus().getPath(), hadoopConf); try { writeActionsToBackfilledFile( logStore, logPath, commit.getVersion(), actions, hadoopConf, shouldOverwrite); } catch (java.nio.file.FileAlreadyExistsException e) { // Ignore the exception. This indicates that the file has already been backfilled. LOG.info("File {} already exists. Skipping backfill for this file.", commit.getFileStatus().getPath()); } finally { actions.close(); } } UpdateItemRequest request = new UpdateItemRequest() .withTableName(coordinatedCommitsTableName) .addKeyEntry( DynamoDBTableEntryConstants.TABLE_ID, new AttributeValue().withS(getTableId(tableDesc.getTableConf()))) .addAttributeUpdatesEntry( DynamoDBTableEntryConstants.COMMITS, new AttributeValueUpdate() .withAction(AttributeAction.PUT) .withValue(new AttributeValue().withL()) ) .withExpected(new HashMap(){ { put(DynamoDBTableEntryConstants.TABLE_LATEST_VERSION, new ExpectedAttributeValue() .withValue( new AttributeValue() .withN(Long.toString(resp.getLatestTableVersion()))) ); put(DynamoDBTableEntryConstants.TABLE_PATH, new ExpectedAttributeValue() .withValue( new AttributeValue() .withS(logPath.getParent().toString())) ); put(DynamoDBTableEntryConstants.SCHEMA_VERSION, new ExpectedAttributeValue() .withValue( new AttributeValue() .withN(Integer.toString(CLIENT_VERSION))) ); } }); try { client.updateItem(request); } catch (ConditionalCheckFailedException e) { // Ignore the exception. The backfill succeeded but the update to // the commit coordinator failed. The main purpose of a backfill operation is to ensure that // UUID commit is physically copied to a standard commit file path. A failed update to // the commit coordinator is not critical. LOG.warn("Backfill succeeded but the update to the commit coordinator failed. This is probably" + " due to a concurrent update to the commit coordinator. This is not a critical error and " + " should rectify itself."); } } @Override public Map registerTable( Path logPath, Optional tableIdentifier, long currentVersion, AbstractMetadata currentMetadata, AbstractProtocol currentProtocol) { java.util.Map item = new HashMap<>(); String tableId = java.util.UUID.randomUUID().toString(); item.put(DynamoDBTableEntryConstants.TABLE_ID, new AttributeValue().withS(tableId)); // We maintain the invariant that a commit will only succeed if the latestVersion stored // in the table is equal to attemptVersion - 1. To maintain this, even though the // filesystem-based commit after register table can fail, we still treat the attemptVersion // at registration as a valid version. Since it is expected that the commit coordinator will // return -1 as the table version if no commits have been accepted after registration, we // use another attribute (HAS_ACCEPTED_COMMITS) to track whether any commits have been // accepted. This attribute is set to true whenever any commit is accepted. // If HAS_ACCEPTED_COMMITS is false, in a getCommit request, we set the latest version to -1. long attemptVersion = currentVersion + 1; item.put( DynamoDBTableEntryConstants.TABLE_LATEST_VERSION, new AttributeValue().withN(Long.toString(attemptVersion))); // Used to indicate that no real commits have gone through the commit coordinator yet. item.put( DynamoDBTableEntryConstants.HAS_ACCEPTED_COMMITS, new AttributeValue().withBOOL(false)); item.put( DynamoDBTableEntryConstants.TABLE_PATH, new AttributeValue().withS(logPath.getParent().toString())); item.put(DynamoDBTableEntryConstants.COMMITS, new AttributeValue().withL()); item.put( DynamoDBTableEntryConstants.ACCEPTING_COMMITS, new AttributeValue().withBOOL(true)); item.put( DynamoDBTableEntryConstants.SCHEMA_VERSION, new AttributeValue().withN(Integer.toString(CLIENT_VERSION))); PutItemRequest request = new PutItemRequest() .withTableName(coordinatedCommitsTableName) .withItem(item) .withConditionExpression( String.format( "attribute_not_exists(%s)", DynamoDBTableEntryConstants.TABLE_ID)); client.putItem(request); Map tableConf = new HashMap(); tableConf.put(DynamoDBTableEntryConstants.TABLE_ID, tableId); return tableConf; } // Copied from DynamoDbLogStore. TODO: add the logging back. /** * Ensures that the table used to store commits from all Delta tables exists. If the table * does not exist, it will be created. * @throws IOException */ private void tryEnsureTableExists() throws IOException { int retries = 0; boolean created = false; while(retries < 20) { String status = "CREATING"; try { DescribeTableResult result = client.describeTable(coordinatedCommitsTableName); TableDescription descr = result.getTable(); status = descr.getTableStatus(); } catch (ResourceNotFoundException e) { LOG.info( "DynamoDB table `{}` for endpoint `{}` does not exist. " + "Creating it now with provisioned throughput of {} RCUs and {} WCUs.", coordinatedCommitsTableName, endpoint, readCapacityUnits, writeCapacityUnits); try { client.createTable( // attributeDefinitions java.util.Collections.singletonList( new AttributeDefinition( DynamoDBTableEntryConstants.TABLE_ID, ScalarAttributeType.S) ), coordinatedCommitsTableName, // keySchema java.util.Collections.singletonList( new KeySchemaElement( DynamoDBTableEntryConstants.TABLE_ID, KeyType.HASH) ), new ProvisionedThroughput(this.readCapacityUnits, this.writeCapacityUnits) ); created = true; } catch (ResourceInUseException e3) { // race condition - table just created by concurrent process } } if (status.equals("ACTIVE")) { if (created) { LOG.info("Successfully created DynamoDB table `{}`", coordinatedCommitsTableName); } else { LOG.info("Table `{}` already exists", coordinatedCommitsTableName); } break; } else if (status.equals("CREATING")) { retries += 1; LOG.info("Waiting for `{}` table creation", coordinatedCommitsTableName); try { Thread.sleep(1000); } catch(InterruptedException e) { throw new InterruptedIOException(e.getMessage()); } } else { LOG.error("table `{}` status: {}", coordinatedCommitsTableName, status); throw new RuntimeException("DynamoDBCommitCoordinatorCliet: Unable to create table with " + "name " + coordinatedCommitsTableName + " for endpoint " + endpoint + ". Ensure " + "that the credentials provided have the necessary permissions to create " + "tables in DynamoDB. If the table already exists, ensure that the table " + "is in the ACTIVE state."); } }; } @Override public boolean semanticEquals(CommitCoordinatorClient other) { if (!(other instanceof DynamoDBCommitCoordinatorClient)) { return false; } DynamoDBCommitCoordinatorClient otherStore = (DynamoDBCommitCoordinatorClient) other; return this.coordinatedCommitsTableName.equals(otherStore.coordinatedCommitsTableName) && this.endpoint.equals(otherStore.endpoint); } } ================================================ FILE: spark/src/main/java/io/delta/dynamodbcommitcoordinator/DynamoDBCommitCoordinatorClientBuilder.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.dynamodbcommitcoordinator; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; import org.apache.spark.sql.delta.coordinatedcommits.CommitCoordinatorBuilder; import org.apache.spark.sql.delta.sources.DeltaSQLConf; import io.delta.storage.commit.CommitCoordinatorClient; import org.apache.hadoop.conf.Configuration; import org.apache.spark.sql.SparkSession; import scala.collection.immutable.Map; import java.io.IOException; public class DynamoDBCommitCoordinatorClientBuilder implements CommitCoordinatorBuilder { private final long BACKFILL_BATCH_SIZE = 1L; @Override public String getName() { return "dynamodb"; } /** * Key for the name of the DynamoDB table which stores all the unbackfilled * commits for this owner. The value of this key is stored in the `conf` * which is passed to the `build` method. */ private static final String COORDINATED_COMMITS_TABLE_NAME_KEY = "dynamoDBTableName"; /** * The endpoint of the DynamoDB service. The value of this key is stored in the * `conf` which is passed to the `build` method. */ private static final String DYNAMO_DB_ENDPOINT_KEY = "dynamoDBEndpoint"; @Override public CommitCoordinatorClient build(SparkSession spark, Map conf) { String coordinatedCommitsTableName = conf.get(COORDINATED_COMMITS_TABLE_NAME_KEY).getOrElse(() -> { throw new RuntimeException(COORDINATED_COMMITS_TABLE_NAME_KEY + " not found"); }); String dynamoDBEndpoint = conf.get(DYNAMO_DB_ENDPOINT_KEY).getOrElse(() -> { throw new RuntimeException(DYNAMO_DB_ENDPOINT_KEY + " not found"); }); String awsCredentialsProviderName = spark.conf().get(DeltaSQLConf.COORDINATED_COMMITS_DDB_AWS_CREDENTIALS_PROVIDER_NAME()); int readCapacityUnits = Integer.parseInt( spark.conf().get(DeltaSQLConf.COORDINATED_COMMITS_DDB_READ_CAPACITY_UNITS().key())); int writeCapacityUnits = Integer.parseInt( spark.conf().get(DeltaSQLConf.COORDINATED_COMMITS_DDB_WRITE_CAPACITY_UNITS().key())); boolean skipPathCheck = Boolean.parseBoolean( spark.conf().get(DeltaSQLConf.COORDINATED_COMMITS_DDB_SKIP_PATH_CHECK().key())); try { AmazonDynamoDB ddbClient = createAmazonDDBClient( dynamoDBEndpoint, awsCredentialsProviderName, spark.sessionState().newHadoopConf() ); return getDynamoDBCommitCoordinatorClient( coordinatedCommitsTableName, dynamoDBEndpoint, ddbClient, BACKFILL_BATCH_SIZE, readCapacityUnits, writeCapacityUnits, skipPathCheck ); } catch (Exception e) { throw new RuntimeException("Failed to create DynamoDB client", e); } } protected DynamoDBCommitCoordinatorClient getDynamoDBCommitCoordinatorClient( String coordinatedCommitsTableName, String dynamoDBEndpoint, AmazonDynamoDB ddbClient, long backfillBatchSize, int readCapacityUnits, int writeCapacityUnits, boolean skipPathCheck ) throws IOException { return new DynamoDBCommitCoordinatorClient( coordinatedCommitsTableName, dynamoDBEndpoint, ddbClient, backfillBatchSize, readCapacityUnits, writeCapacityUnits, skipPathCheck ); } protected AmazonDynamoDB createAmazonDDBClient( String endpoint, String credentialProviderName, Configuration hadoopConf ) throws ReflectiveOperationException { AWSCredentialsProvider awsCredentialsProvider = ReflectionUtils.createAwsCredentialsProvider(credentialProviderName, hadoopConf); AmazonDynamoDBClient client = new AmazonDynamoDBClient(awsCredentialsProvider); client.setEndpoint(endpoint); return client; } } ================================================ FILE: spark/src/main/java/io/delta/dynamodbcommitcoordinator/DynamoDBTableEntryConstants.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.dynamodbcommitcoordinator; /** * Defines the field names used in the DynamoDB table entry. */ final class DynamoDBTableEntryConstants { private DynamoDBTableEntryConstants() {} /** The primary key of the DynamoDB table. */ public static final String TABLE_ID = "tableId"; /** The version of the latest commit in the corresponding Delta table. */ public static final String TABLE_LATEST_VERSION = "tableVersion"; /** The inCommitTimestamp of the latest commit in the corresponding Delta table. */ public static final String TABLE_LATEST_TIMESTAMP = "tableTimestamp"; /** Whether this commit coordinator is accepting more commits for the corresponding Delta table. */ public static final String ACCEPTING_COMMITS = "acceptingCommits"; /** The path of the corresponding Delta table. */ public static final String TABLE_PATH = "path"; /** The schema version of this DynamoDB table entry. */ public static final String SCHEMA_VERSION = "schemaVersion"; /** * Whether this commit coordinator has accepted any commits after `registerTable`. */ public static final String HAS_ACCEPTED_COMMITS = "hasAcceptedCommits"; /** The name of the field used to store unbackfilled commits. */ public static final String COMMITS = "commits"; /** The unbackfilled commit version. */ public static final String COMMIT_VERSION = "version"; /** The inCommitTimestamp of the unbackfilled commit. */ public static final String COMMIT_TIMESTAMP = "timestamp"; /** The name of the unbackfilled file. e.g. 00001.uuid.json */ public static final String COMMIT_FILE_NAME = "fsName"; /** The length of the unbackfilled file as per the file status. */ public static final String COMMIT_FILE_LENGTH = "fsLength"; /** The modification timestamp of the unbackfilled file as per the file status. */ public static final String COMMIT_FILE_MODIFICATION_TIMESTAMP = "fsTimestamp"; } ================================================ FILE: spark/src/main/java/io/delta/dynamodbcommitcoordinator/ReflectionUtils.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.dynamodbcommitcoordinator; import com.amazonaws.auth.AWSCredentialsProvider; import org.apache.hadoop.conf.Configuration; import java.util.Arrays; /** * Utility class for reflection operations. Used to create AWS credentials provider from class name. * Same as the io.delta.storage.utils.ReflectionUtils class is used in delta/storage-s3-dynamodb. */ public class ReflectionUtils { private static boolean readsCredsFromHadoopConf(Class awsCredentialsProviderClass) { return Arrays.stream(awsCredentialsProviderClass.getConstructors()) .anyMatch(constructor -> constructor.getParameterCount() == 1 && Arrays.equals(constructor.getParameterTypes(), new Class[]{Configuration.class})); } /** * Creates a AWS credentials provider from the given provider classname and {@link Configuration}. * * It first checks if AWS Credentials Provider class has a constructor with Hadoop configuration * as parameter. * If yes - create instance of class using this constructor. * If no - create instance with empty parameters constructor. * * @param credentialsProviderClassName Fully qualified name of the desired credentials provider class. * @param hadoopConf Hadoop configuration, used to create instance of AWS credentials * provider, if supported. * @return {@link AWSCredentialsProvider} object, instantiated from the class @see {credentialsProviderClassName} * @throws ReflectiveOperationException When AWS credentials provider constructor do not match. * Indicates that the class has neither a constructor with no args * nor a constructor with only Hadoop configuration as argument. */ public static AWSCredentialsProvider createAwsCredentialsProvider( String credentialsProviderClassName, Configuration hadoopConf) throws ReflectiveOperationException { Class awsCredentialsProviderClass = Class.forName(credentialsProviderClassName); if (readsCredsFromHadoopConf(awsCredentialsProviderClass)) return (AWSCredentialsProvider) awsCredentialsProviderClass .getConstructor(Configuration.class) .newInstance(hadoopConf); else return (AWSCredentialsProvider) awsCredentialsProviderClass.getConstructor().newInstance(); } } ================================================ FILE: spark/src/main/java/io/delta/dynamodbcommitcoordinator/integration_tests/dynamodb_commitcoordinator_integration_test.py ================================================ # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os import sys import threading import json from pyspark.sql import SparkSession from multiprocessing.pool import ThreadPool import time import boto3 import uuid """ Run this script in root dir of repository: # ===== Mandatory input from user ===== export RUN_ID=run001 export S3_BUCKET=delta-lake-dynamodb-test-00 export AWS_DEFAULT_REGION=us-west-2 # ===== Optional input from user ===== export DELTA_CONCURRENT_WRITERS=20 export DELTA_CONCURRENT_READERS=2 export DELTA_NUM_ROWS=200 export DELTA_DYNAMO_ENDPOINT=https://dynamodb.us-west-2.amazonaws.com # ===== Optional input from user (we calculate defaults using S3_BUCKET and RUN_ID) ===== export RELATIVE_DELTA_TABLE_PATH=___ export DELTA_DYNAMO_TABLE_NAME=___ ./run-integration-tests.py --use-local --run-dynamodb-commit-coordinator-integration-tests \ --packages org.apache.hadoop:hadoop-aws:3.4.0,com.amazonaws:aws-java-sdk-bundle:1.12.262 \ --dbb-conf io.delta.storage.credentials.provider=com.amazonaws.auth.profile.ProfileCredentialsProvider \ spark.hadoop.fs.s3a.aws.credentials.provider=com.amazonaws.auth.profile.ProfileCredentialsProvider """ # ===== Mandatory input from user ===== run_id = os.environ.get("RUN_ID") s3_bucket = os.environ.get("S3_BUCKET") # ===== Optional input from user ===== concurrent_writers = int(os.environ.get("DELTA_CONCURRENT_WRITERS", 2)) concurrent_readers = int(os.environ.get("DELTA_CONCURRENT_READERS", 2)) num_rows = int(os.environ.get("DELTA_NUM_ROWS", 16)) dynamo_endpoint = os.environ.get("DELTA_DYNAMO_ENDPOINT", "https://dynamodb.us-west-2.amazonaws.com") # ===== Optional input from user (we calculate defaults using RUN_ID) ===== relative_delta_table_path = os.environ.get("RELATIVE_DELTA_TABLE_PATH", f"tables/table_ddb_cs_{run_id}_{str(uuid.uuid4())}")\ .rstrip("/") dynamo_table_name = os.environ.get("DELTA_DYNAMO_TABLE_NAME", "test_ddb_cs_table_" + run_id) relative_delta_table1_path = relative_delta_table_path + "_tab1" relative_delta_table2_path = relative_delta_table_path + "_tab2" bucket_prefix = "s3a://" + s3_bucket + "/" delta_table1_path = bucket_prefix + relative_delta_table1_path delta_table2_path = bucket_prefix + relative_delta_table2_path if delta_table1_path is None: print(f"\nSkipping Python test {os.path.basename(__file__)} due to the missing env variable " f"`DELTA_TABLE_PATH`\n=====================") sys.exit(0) dynamodb_commit_coordinator_conf = json.dumps({ "dynamoDBTableName": dynamo_table_name, "dynamoDBEndpoint": dynamo_endpoint }) test_log = f""" ========================================== run id: {run_id} delta table1 path: {delta_table1_path} delta table2 path: {delta_table1_path} dynamo table name: {dynamo_table_name} concurrent writers: {concurrent_writers} concurrent readers: {concurrent_readers} number of rows: {num_rows} relative_delta_table_path: {relative_delta_table_path} ========================================== """ print(test_log) commit_coordinator_property_key = "coordinatedCommits.commitCoordinator" property_key_suffix = "-preview" spark = SparkSession \ .builder \ .appName("utilities") \ .master("local[*]") \ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \ .config(f"spark.databricks.delta.properties.defaults.{commit_coordinator_property_key}{property_key_suffix}", "dynamodb") \ .config(f"spark.databricks.delta.properties.defaults.coordinatedCommits.commitCoordinatorConf{property_key_suffix}", dynamodb_commit_coordinator_conf) \ .config(f"spark.databricks.delta.coordinatedCommits.commitCoordinator.dynamodb.awsCredentialsProviderName", "com.amazonaws.auth.profile.ProfileCredentialsProvider") \ .getOrCreate() print("Creating table at path ", delta_table1_path) spark.sql(f"CREATE table delta.`{delta_table1_path}` (id int, a int) USING DELTA") # commit 0 def write_tx(n): print("writing:", [n, n]) spark.sql(f"INSERT INTO delta.`{delta_table1_path}` VALUES ({n}, {n})") stop_reading = threading.Event() def read_data(): while not stop_reading.is_set(): print("Reading {:d} rows ...".format( spark.read.format("delta").load(delta_table1_path).distinct().count()) ) time.sleep(1) def start_read_thread(): thread = threading.Thread(target=read_data) thread.start() return thread print("===================== Starting reads and writes =====================") read_threads = [start_read_thread() for i in range(concurrent_readers)] pool = ThreadPool(concurrent_writers) start_t = time.time() pool.map(write_tx, range(num_rows)) stop_reading.set() for thread in read_threads: thread.join() print("===================== Evaluating number of written rows =====================") actual = spark.read.format("delta").load(delta_table1_path).distinct().count() print("Actual number of written rows:", actual) print("Expected number of written rows:", num_rows) assert actual == num_rows t = time.time() - start_t print(f"{num_rows / t:.02f} tx / sec") current_table_version = num_rows dynamodb = boto3.resource('dynamodb', endpoint_url=dynamo_endpoint) ddb_table = dynamodb.Table(dynamo_table_name) def get_dynamo_db_table_entry_id(table_path): table_properties = spark.sql(f"DESCRIBE DETAIL delta.`{table_path}`").select("properties").collect()[0][0] table_conf = table_properties.get(f"delta.coordinatedCommits.tableConf{property_key_suffix}", None) if table_conf is None: return None return json.loads(table_conf).get("tableId", None) def validate_table_version_as_per_dynamodb(table_path, expected_version): table_id = get_dynamo_db_table_entry_id(table_path) assert table_id is not None print(f"Validating table version for tableId: {table_id}") item = ddb_table.get_item( Key={ 'tableId': table_id }, AttributesToGet = ['tableVersion'] )['Item'] current_table_version = int(item['tableVersion']) assert current_table_version == expected_version delta_table_version = num_rows validate_table_version_as_per_dynamodb(delta_table1_path, delta_table_version) def perform_insert_and_validate(table_path, insert_value): spark.sql(f"INSERT INTO delta.`{table_path}` VALUES ({insert_value}, {insert_value})") res = spark.sql(f"SELECT 1 FROM delta.`{table_path}` WHERE id = {insert_value} AND a = {insert_value}").collect() assert(len(res) == 1) def check_for_delta_file_in_filesystem(delta_table_path, version, is_backfilled, should_exist): # Check for backfilled commit s3_client = boto3.client("s3") relative_table_path = delta_table_path.replace(bucket_prefix, "") relative_delta_log_path = relative_table_path + "/_delta_log/" relative_commit_folder_path = relative_delta_log_path if is_backfilled else os.path.join(relative_delta_log_path, "_staged_commits") listing_prefix = os.path.join(relative_commit_folder_path, f"{version:020}.").lstrip("/") print(f"querying {listing_prefix} from bucket {s3_bucket} for version {version}") response = s3_client.list_objects_v2(Bucket=s3_bucket, Prefix=listing_prefix) if 'Contents' not in response: assert(not should_exist, f"Listing for prefix {listing_prefix} did not return any files even though it should have.") return items = response['Contents'] commits = filter(lambda key: ".json" in key and ".tmp" not in key, map(lambda x: os.path.basename(x['Key']), items)) expected_count = 1 if should_exist else 0 matching_files = list(filter(lambda key: key.split('.')[0].endswith(f"{version:020}"), commits)) assert(len(matching_files) == expected_count) def test_downgrades_and_upgrades(delta_table_path, delta_table_version): # Downgrade to filesystem based commits should work print("===================== Evaluating downgrade to filesystem based commits =====================") spark.sql(f"ALTER TABLE delta.`{delta_table_path}` UNSET TBLPROPERTIES ('delta.{commit_coordinator_property_key}{property_key_suffix}')") delta_table_version += 1 perform_insert_and_validate(delta_table_path, 9990) delta_table_version += 1 check_for_delta_file_in_filesystem(delta_table_path, delta_table_version, is_backfilled=True, should_exist=True) # No UUID delta file should have been created for this version check_for_delta_file_in_filesystem(delta_table_path, delta_table_version, is_backfilled=False, should_exist=False) print("[SUCCESS] Downgrade to filesystem based commits worked") # Upgrade to coordinated commits should work print("===================== Evaluating upgrade to coordinated commits =====================") spark.sql(f"ALTER TABLE delta.`{delta_table_path}` SET TBLPROPERTIES ('delta.{commit_coordinator_property_key}{property_key_suffix}' = 'dynamodb')") delta_table_version += 1 check_for_delta_file_in_filesystem(delta_table_path, delta_table_version, is_backfilled=True, should_exist=True) # No UUID delta file should have been created for the enablement commit check_for_delta_file_in_filesystem(delta_table_path, delta_table_version, is_backfilled=False, should_exist=False) perform_insert_and_validate(delta_table_path, 9991) delta_table_version += 1 check_for_delta_file_in_filesystem(delta_table_path, delta_table_version, is_backfilled=True, should_exist=True) check_for_delta_file_in_filesystem(delta_table_path, delta_table_version, is_backfilled=False, should_exist=True) perform_insert_and_validate(delta_table_path, 9992) delta_table_version += 1 check_for_delta_file_in_filesystem(delta_table_path, delta_table_version, is_backfilled=True, should_exist=True) check_for_delta_file_in_filesystem(delta_table_path, delta_table_version, is_backfilled=False, should_exist=True) validate_table_version_as_per_dynamodb(delta_table_path, delta_table_version) print("[SUCCESS] Upgrade to coordinated commits worked") test_downgrades_and_upgrades(delta_table1_path, delta_table_version) print("[SUCCESS] All tests passed for Table 1") print("===================== Evaluating Table 2 =====================") # Table 2 is created with coordinated commits disabled spark.conf.unset(f"spark.databricks.delta.properties.defaults.{commit_coordinator_property_key}{property_key_suffix}") spark.sql(f"CREATE table delta.`{delta_table2_path}` (id int, a int) USING DELTA") # commit 0 table_2_version = 0 perform_insert_and_validate(delta_table2_path, 8000) table_2_version += 1 check_for_delta_file_in_filesystem(delta_table2_path, table_2_version, is_backfilled=True, should_exist=True) # No UUID delta file should have been created for this version check_for_delta_file_in_filesystem(delta_table2_path, table_2_version, is_backfilled=False, should_exist=False) print("===================== Evaluating Upgrade of Table 2 =====================") spark.sql(f"ALTER TABLE delta.`{delta_table2_path}` SET TBLPROPERTIES ('delta.{commit_coordinator_property_key}{property_key_suffix}' = 'dynamodb')") table_2_version += 1 perform_insert_and_validate(delta_table2_path, 8001) table_2_version += 1 check_for_delta_file_in_filesystem(delta_table2_path, table_2_version, is_backfilled=True, should_exist=True) # This version should have a UUID delta file check_for_delta_file_in_filesystem(delta_table2_path, table_2_version, is_backfilled=True, should_exist=True) test_downgrades_and_upgrades(delta_table2_path, table_2_version) print("[SUCCESS] All tests passed for Table 2") ================================================ FILE: spark/src/main/java/org/apache/spark/sql/delta/DeltaV2Mode.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta; import java.util.Map; import java.util.Optional; import org.apache.spark.sql.catalyst.catalog.CatalogTable; import org.apache.spark.sql.delta.sources.DeltaSQLConf$; import org.apache.spark.sql.delta.util.CatalogTableUtils; import org.apache.spark.sql.internal.SQLConf; /** * Centralized decision logic for Delta connector selection (sparkV2 vs sparkV1). * *

This class encapsulates all configuration checking for * {@code spark.databricks.delta.v2.enableMode} so that the rest of the codebase doesn't need to * directly inspect configuration values. * *

Configuration modes: *

    *
  • NONE (default): sparkV1 connector for all operations
    • *
  • AUTO: sparkV2 connector only for Unity Catalog managed tables
    • *
  • STRICT: sparkV2 connector for all tables (testing mode)
    • *
*/ public class DeltaV2Mode { private static final String STRICT = "STRICT"; private static final String AUTO = "AUTO"; private final SQLConf sqlConf; public DeltaV2Mode(SQLConf sqlConf) { this.sqlConf = sqlConf; } private String mode() { return sqlConf.getConf(DeltaSQLConf$.MODULE$.V2_ENABLE_MODE()); } /** * Determines if streaming reads should use the sparkV2 connector. * * @param catalogTable Optional catalog table metadata * @return true if sparkV2 streaming reads should be used */ public boolean isStreamingReadsEnabled(Optional catalogTable) { switch (mode()) { case STRICT: // Always use sparkV2 connector for all catalog tables return true; case AUTO: // Only use sparkV2 connector for Unity Catalog managed tables return catalogTable.map(CatalogTableUtils::isUnityCatalogManagedTable).orElse(false); default: // NONE or unknown: use sparkV1 streaming return false; } } /** * Determines if catalog should return sparkV2 (SparkTable) or sparkV1 (DeltaTableV2) tables. * * @return true if catalog should return sparkV2 tables */ public boolean shouldCatalogReturnV2Tables() { switch (mode()) { case STRICT: // STRICT mode: always return sparkV2 tables return true; default: // NONE (default) or AUTO: return sparkV1 tables // Note: AUTO mode uses sparkV2 connector only for streaming via ApplyV2Streaming rule, // not at catalog level return false; } } /** * Determines if the provided schema should be trusted without validation for streaming reads. * This is used to bypass DeltaLog schema loading for Unity Catalog tables where the catalog * already provides the correct schema. * *

If we don't bypass, we will load schema from DeltaLog and validate against the provided * schema. For UC-managed tables this extra DeltaLog access can be unnecessary and may fail when * the client doesn't have direct storage access to the managed location, even though the UC * schema is authoritative. For UC-managed tables, the DeltaLog schema should always match the * catalog schema, so re-validating provides no additional correctness guarantees. * *

This checks the parameters map for UC markers to determine if the table is UC-managed. * * @param parameters DataSource parameters map containing table storage properties * @return true if provided schema should be used without validation */ public boolean shouldBypassSchemaValidationForStreaming(Map parameters) { switch (mode()) { case STRICT: case AUTO: // In sparkV2 modes, trust the schema for Unity Catalog managed tables return CatalogTableUtils.isUnityCatalogManagedTableFromProperties(parameters); default: // NONE or unknown: always validate schema via DeltaLog return false; } } /** * Gets the current mode string (for logging/debugging). */ public String getMode() { return mode(); } } ================================================ FILE: spark/src/main/java/org/apache/spark/sql/delta/RowIndexFilter.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta; import org.apache.spark.sql.vectorized.ColumnVector; import org.apache.spark.sql.execution.vectorized.WritableColumnVector; /** * Provides filtering information for each row index within given range. * Specific filters are implemented in subclasses. */ public interface RowIndexFilter { /** * Materialize filtering information for all rows in the range [start, end) * by filling a boolean column vector batch. Assumes the indexes of the rows in the batch are * consecutive and start from 0. * * @param start Beginning index of the filtering range (inclusive). * @param end End index of the filtering range (exclusive). * @param batch The column vector for the current batch to materialize the range into. */ void materializeIntoVector(long start, long end, WritableColumnVector batch); /** * Materialize filtering information for all rows in the batch. This is achieved by probing * the roaring bitmap with the row index of every row in the batch. * * @param batchSize The size of the batch. * @param rowIndexColumn A column vector that contains the row index of each row in the batch. * @param batch The column vector for the current batch to materialize the range into. */ void materializeIntoVectorWithRowIndex( int batchSize, ColumnVector rowIndexColumn, WritableColumnVector batch); /** * Materialize filtering information for batches with a single row. * * @param rowIndex The index of the row to materialize the filtering information. * @param batch The column vector for the current batch to materialize the range into. * We assume it contains a single row. */ void materializeSingleRowWithRowIndex(long rowIndex, WritableColumnVector batch); /** * Value that must be materialised for a row to be kept after filtering. */ public static final byte KEEP_ROW_VALUE = 0; /** * Value that must be materialised for a row to be dropped during filtering. */ public static final byte DROP_ROW_VALUE = 1; } ================================================ FILE: spark/src/main/java/org/apache/spark/sql/delta/RowIndexFilterType.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta; /** Filter types corresponding to every row index filter implementations. */ public enum RowIndexFilterType { /** Corresponding to [[DropMarkedRowsFilter]]. */ IF_CONTAINED(0), /** Corresponding to [[KeepMarkedRowsFilter]]. */ IF_NOT_CONTAINED(1), /** Invalid filter type. */ UNKNOWN(-1); private final int id; RowIndexFilterType(int id) { this.id = id; } public int getId() { return this.id; } } ================================================ FILE: spark/src/main/java/org/apache/spark/sql/delta/sources/AdmittableFile.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.sources; /** * Interface for files that can be admitted by admission control in Delta streaming sources. * This abstraction allows both DSv1 and DSv2 IndexedFile implementations to be used with * the admission control logic. */ public interface AdmittableFile { /** * Returns true if this file has an associated file action (AddFile, RemoveFile, or CDCFile). * Placeholder IndexedFiles with no file action will return false. */ boolean hasFileAction(); /** * Returns the size of the file in bytes. * This method should only be called when hasFileAction() returns true. */ long getFileSize(); } ================================================ FILE: spark/src/main/java/org/apache/spark/sql/delta/util/CatalogTableUtils.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util; import static java.util.Objects.requireNonNull; import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient; import java.util.Collections; import java.util.Map; import org.apache.spark.sql.catalyst.catalog.CatalogTable; import scala.jdk.javaapi.CollectionConverters; /** * Utility helpers for inspecting Delta-related metadata persisted on Spark {@link CatalogTable} * instances by Unity Catalog. * *

Unity Catalog marks catalog-managed tables via feature flags stored in table storage * properties. This helper centralises the logic for interpreting those properties so the SparkV2 * connector can decide when to use catalog-owned (CCv2) behaviour. * *

    *
  • {@link #isCatalogManaged(CatalogTable)} checks whether either {@code * delta.feature.catalogManaged} or {@code delta.feature.catalogOwned-preview} is set to * {@code supported}, signalling that a catalog manages the table. *
  • {@link #isUnityCatalogManagedTable(CatalogTable)} additionally verifies the presence of the * Unity Catalog table identifier ({@link UCCommitCoordinatorClient#UC_TABLE_ID_KEY}) to * confirm that the table is backed by Unity Catalog. *
*/ public final class CatalogTableUtils { /** * Property key for catalog-managed feature flag. Corresponds to * delta.feature.catalogManaged and preview variant * delta.feature.catalogOwned-preview */ static final String FEATURE_CATALOG_MANAGED = "delta.feature.catalogManaged"; static final String FEATURE_CATALOG_OWNED_PREVIEW = "delta.feature.catalogOwned-preview"; private static final String SUPPORTED = "supported"; private CatalogTableUtils() {} /** * Checks whether any catalog manages this table via CCv2 semantics. * * @param table Spark {@link CatalogTable} descriptor * @return {@code true} when either catalog feature flag is set to {@code supported} */ public static boolean isCatalogManaged(CatalogTable table) { requireNonNull(table, "table is null"); Map storageProperties = getStorageProperties(table); return isCatalogManagedFeatureEnabled(storageProperties, FEATURE_CATALOG_MANAGED) || isCatalogManagedFeatureEnabled(storageProperties, FEATURE_CATALOG_OWNED_PREVIEW); } /** * Checks whether the table is Unity Catalog managed. * * @param table Spark {@link CatalogTable} descriptor * @return {@code true} when the table is catalog managed and contains the UC identifier */ public static boolean isUnityCatalogManagedTable(CatalogTable table) { requireNonNull(table, "table is null"); Map storageProperties = getStorageProperties(table); boolean isUCBacked = storageProperties.containsKey(UCCommitCoordinatorClient.UC_TABLE_ID_KEY); return isUCBacked && isCatalogManaged(table); } /** * Checks whether the table is Unity Catalog managed based on storage properties map. * *

This method checks the properties map (typically from table.storage.properties) for * UC markers, allowing UC table detection without requiring a full CatalogTable object. * This is useful when only the properties map is available, such as in DataSource APIs. * * @param properties Storage properties map (e.g., from CatalogTable.storage.properties or * DataSource parameters map) * @return {@code true} when the table is catalog managed and contains the UC identifier */ public static boolean isUnityCatalogManagedTableFromProperties(Map properties) { if (properties == null || properties.isEmpty()) { return false; } boolean isUCBacked = properties.containsKey(UCCommitCoordinatorClient.UC_TABLE_ID_KEY); boolean isCatalogManaged = isCatalogManagedFeatureEnabled(properties, FEATURE_CATALOG_MANAGED) || isCatalogManagedFeatureEnabled(properties, FEATURE_CATALOG_OWNED_PREVIEW); return isUCBacked && isCatalogManaged; } /** * Checks whether the given feature key is enabled in the table properties. * * @param tableProperties The table properties * @param featureKey The feature key * @return {@code true} when the feature key is set to {@code supported} */ private static boolean isCatalogManagedFeatureEnabled( Map tableProperties, String featureKey) { requireNonNull(tableProperties, "tableProperties is null"); requireNonNull(featureKey, "featureKey is null"); String featureValue = tableProperties.get(featureKey); if (featureValue == null) { return false; } return featureValue.equalsIgnoreCase(SUPPORTED); } /** * Returns the catalog storage properties published with a {@link CatalogTable}. * * @param table Spark {@link CatalogTable} descriptor * @return Java map view of the storage properties, never null */ private static Map getStorageProperties(CatalogTable table) { requireNonNull(table, "table is null"); if (table.storage() == null) { return Collections.emptyMap(); } scala.collection.immutable.Map scalaProps = table.storage().properties(); if (scalaProps == null || scalaProps.isEmpty()) { return Collections.emptyMap(); } return CollectionConverters.asJava(scalaProps); } } ================================================ FILE: spark/src/main/java-shims/spark-4.0/org/apache/spark/sql/delta/shims/VariantStatsShims.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.shims; import org.apache.spark.QueryContext; import org.apache.spark.SparkRuntimeException; import scala.collection.immutable.Map$; /** * Shim for variant stats functionality in Spark 4.0. * In Spark 4.0, VariantUtil.readUnsigned is a private member, so we provide our own * implementation here. */ public class VariantStatsShims { static SparkRuntimeException malformedVariant() { return new SparkRuntimeException("MALFORMED_VARIANT", Map$.MODULE$.empty(), null, new QueryContext[]{}, ""); } // Check the validity of an array index `pos`. Throw `MALFORMED_VARIANT` if it is out of bound, // meaning that the variant is malformed. private static void checkIndex(int pos, int length) { if (pos < 0 || pos >= length) throw malformedVariant(); } // Read a little-endian unsigned int value from `bytes[pos, pos + numBytes)`. The value must fit // into a non-negative int (`[0, Integer.MAX_VALUE]`). private static int readUnsigned(byte[] bytes, int pos, int numBytes) { checkIndex(pos, bytes.length); checkIndex(pos + numBytes - 1, bytes.length); int result = 0; // Similar to the `readLong` loop, but all bytes should be unsign-extended. for (int i = 0; i < numBytes; ++i) { int unsignedByteValue = bytes[pos + i] & 0xFF; result |= unsignedByteValue << (8 * i); } if (result < 0) throw malformedVariant(); return result; } // Get the length of metadata in the provided array. It is used to split metadata and value in // situations where they are serialized as a concatenated pair (e.g. Delta stats). public static int metadataSize(byte[] metadata) { checkIndex(0, metadata.length); // Similar to the logic from getMetadataKey where "id" is equal to "dictSize". int offsetSize = ((metadata[0] >> 6) & 0x3) + 1; int dictSize = readUnsigned(metadata, 1, offsetSize); int lastOffset = readUnsigned(metadata, 1 + (dictSize + 1) * offsetSize, offsetSize); int size = 1 + (dictSize + 2) * offsetSize + lastOffset; if (size > metadata.length) { throw malformedVariant(); } return size; } } ================================================ FILE: spark/src/main/java-shims/spark-4.1/org/apache/spark/sql/delta/shims/VariantStatsShims.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.shims; import org.apache.spark.QueryContext; import org.apache.spark.SparkRuntimeException; import org.apache.spark.types.variant.VariantUtil; import scala.collection.immutable.Map$; /** * Shim for variant stats functionality in Spark 4.1+. * In Spark 4.1, VariantUtil.readUnsigned is public, so we can use it directly. */ public class VariantStatsShims { static SparkRuntimeException malformedVariant() { return new SparkRuntimeException("MALFORMED_VARIANT", Map$.MODULE$.empty(), null, new QueryContext[]{}, ""); } // Check the validity of an array index `pos`. Throw `MALFORMED_VARIANT` if it is out of bound, // meaning that the variant is malformed. private static void checkIndex(int pos, int length) { if (pos < 0 || pos >= length) throw malformedVariant(); } // Get the length of metadata in the provided array. It is used to split metadata and value in // situations where they are serialized as a concatenated pair (e.g. Delta stats). public static int metadataSize(byte[] metadata) { checkIndex(0, metadata.length); // Similar to the logic from getMetadataKey where "id" is equal to "dictSize". int offsetSize = ((metadata[0] >> 6) & 0x3) + 1; int dictSize = VariantUtil.readUnsigned(metadata, 1, offsetSize); int lastOffset = VariantUtil.readUnsigned(metadata, 1 + (dictSize + 1) * offsetSize, offsetSize); int size = 1 + (dictSize + 2) * offsetSize + lastOffset; if (size > metadata.length) { throw malformedVariant(); } return size; } } ================================================ FILE: spark/src/main/java-shims/spark-4.2/org/apache/spark/sql/delta/shims/VariantStatsShims.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.shims; import org.apache.spark.QueryContext; import org.apache.spark.SparkRuntimeException; import org.apache.spark.types.variant.VariantUtil; import scala.collection.immutable.Map$; /** * Shim for variant stats functionality in Spark 4.2+. * In Spark 4.2, VariantUtil.readUnsigned is public, so we can use it directly. */ public class VariantStatsShims { static SparkRuntimeException malformedVariant() { return new SparkRuntimeException("MALFORMED_VARIANT", Map$.MODULE$.empty(), null, new QueryContext[]{}, ""); } // Check the validity of an array index `pos`. Throw `MALFORMED_VARIANT` if it is out of bound, // meaning that the variant is malformed. private static void checkIndex(int pos, int length) { if (pos < 0 || pos >= length) throw malformedVariant(); } // Get the length of metadata in the provided array. It is used to split metadata and value in // situations where they are serialized as a concatenated pair (e.g. Delta stats). public static int metadataSize(byte[] metadata) { checkIndex(0, metadata.length); // Similar to the logic from getMetadataKey where "id" is equal to "dictSize". int offsetSize = ((metadata[0] >> 6) & 0x3) + 1; int dictSize = VariantUtil.readUnsigned(metadata, 1, offsetSize); int lastOffset = VariantUtil.readUnsigned(metadata, 1 + (dictSize + 1) * offsetSize, offsetSize); int size = 1 + (dictSize + 2) * offsetSize + lastOffset; if (size > metadata.length) { throw malformedVariant(); } return size; } } ================================================ FILE: spark/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister ================================================ org.apache.spark.sql.delta.sources.DeltaDataSource ================================================ FILE: spark/src/main/resources/error/delta-error-classes.json ================================================ { "DELTA_ACTIVE_SPARK_SESSION_NOT_FOUND" : { "message" : [ "Could not find active SparkSession" ], "sqlState" : "08003" }, "DELTA_ACTIVE_TRANSACTION_ALREADY_SET" : { "message" : [ "Cannot set a new txn as active when one is already active" ], "sqlState" : "0B000" }, "DELTA_ADDING_COLUMN_WITH_INTERNAL_NAME_FAILED" : { "message" : [ "Failed to add column because the name is reserved." ], "sqlState" : "42000" }, "DELTA_ADDING_DELETION_VECTORS_DISALLOWED" : { "message" : [ "The current operation attempted to add a deletion vector to a table that does not permit the creation of new deletion vectors. Please file a bug report." ], "sqlState" : "0A000" }, "DELTA_ADDING_DELETION_VECTORS_WITH_TIGHT_BOUNDS_DISALLOWED" : { "message" : [ "All operations that add deletion vectors should set the tightBounds column in statistics to false. Please file a bug report." ], "sqlState" : "42000" }, "DELTA_ADD_COLUMN_AT_INDEX_LESS_THAN_ZERO" : { "message" : [ "Index to add column is lower than 0" ], "sqlState" : "42KD3" }, "DELTA_ADD_COLUMN_PARENT_NOT_STRUCT" : { "message" : [ "Cannot add because its parent is not a StructType. Found " ], "sqlState" : "42KD3" }, "DELTA_ADD_COLUMN_STRUCT_NOT_FOUND" : { "message" : [ "Struct not found at position " ], "sqlState" : "42KD3" }, "DELTA_ADD_CONSTRAINTS" : { "message" : [ "Please use ALTER TABLE ADD CONSTRAINT to add CHECK constraints." ], "sqlState" : "0A000" }, "DELTA_AGGREGATE_IN_CHECK_CONSTRAINT" : { "message" : [ "Found in a CHECK constraint. Aggregate expressions are not allowed in CHECK constraints." ], "sqlState" : "42621" }, "DELTA_AGGREGATE_IN_GENERATED_COLUMN" : { "message" : [ "Found . A generated column cannot use an aggregate expression" ], "sqlState" : "42621" }, "DELTA_AGGREGATION_NOT_SUPPORTED" : { "message" : [ "Aggregate functions are not supported in the ." ], "sqlState" : "42903" }, "DELTA_ALTER_TABLE_CHANGE_COL_NOT_SUPPORTED" : { "message" : [ "ALTER TABLE CHANGE COLUMN is not supported for changing column to " ], "sqlState" : "42837" }, "DELTA_ALTER_TABLE_CLUSTER_BY_NOT_ALLOWED" : { "message" : [ "ALTER TABLE CLUSTER BY is supported only for Delta table with clustering." ], "sqlState" : "42000" }, "DELTA_ALTER_TABLE_CLUSTER_BY_ON_PARTITIONED_TABLE_NOT_ALLOWED" : { "message" : [ "ALTER TABLE CLUSTER BY cannot be applied to a partitioned table." ], "sqlState" : "42000" }, "DELTA_ALTER_TABLE_SET_CLUSTERING_TABLE_FEATURE_NOT_ALLOWED" : { "message" : [ "Cannot enable table feature using ALTER TABLE SET TBLPROPERTIES. Please use CREATE OR REPLACE TABLE CLUSTER BY to create a Delta table with clustering." ], "sqlState" : "42000" }, "DELTA_AMBIGUOUS_DATA_TYPE_CHANGE" : { "message" : [ "Cannot change data type of from to . This change contains column removals and additions, therefore they are ambiguous. Please make these changes individually using ALTER TABLE [ADD | DROP | RENAME] COLUMN." ], "sqlState" : "429BQ" }, "DELTA_AMBIGUOUS_PARTITION_COLUMN" : { "message" : [ "Ambiguous partition column can be ." ], "sqlState" : "42702" }, "DELTA_AMBIGUOUS_PATHS_IN_CREATE_TABLE" : { "message" : [ "CREATE TABLE contains two different locations: and .", "You can remove the LOCATION clause from the CREATE TABLE statement, or set", " to true to skip this check.", "" ], "sqlState" : "42613" }, "DELTA_BLOCK_COLUMN_MAPPING_AND_CDC_OPERATION" : { "message" : [ "Operation \"\" is not allowed when the table has enabled change data feed (CDF) and has undergone schema changes using DROP COLUMN or RENAME COLUMN." ], "sqlState" : "42KD4" }, "DELTA_BLOOM_FILTER_DROP_ON_NON_EXISTING_COLUMNS" : { "message" : [ "Cannot drop bloom filter indices for the following non-existent column(s): " ], "sqlState" : "42703" }, "DELTA_CANNOT_CHANGE_DATA_TYPE" : { "message" : [ "Cannot change data type: " ], "sqlState" : "429BQ" }, "DELTA_CANNOT_CHANGE_LOCATION" : { "message" : [ "Cannot change the 'location' of the Delta table using SET TBLPROPERTIES. Please use ALTER TABLE SET LOCATION instead." ], "sqlState" : "42601" }, "DELTA_CANNOT_CHANGE_PROVIDER" : { "message" : [ "'provider' is a reserved table property, and cannot be altered." ], "sqlState" : "42939" }, "DELTA_CANNOT_CONVERT_TO_FILEFORMAT" : { "message" : [ "Can not convert to FileFormat." ], "sqlState" : "XXKDS" }, "DELTA_CANNOT_CREATE_BLOOM_FILTER_NON_EXISTING_COL" : { "message" : [ "Cannot create bloom filter indices for the following non-existent column(s): " ], "sqlState" : "42703" }, "DELTA_CANNOT_CREATE_LOG_PATH" : { "message" : [ "Cannot create " ], "sqlState" : "42KD5" }, "DELTA_CANNOT_DESCRIBE_VIEW_HISTORY" : { "message" : [ "Cannot describe the history of a view." ], "sqlState" : "42809" }, "DELTA_CANNOT_DROP_BLOOM_FILTER_ON_NON_INDEXED_COLUMN" : { "message" : [ "Cannot drop bloom filter index on a non indexed column: " ], "sqlState" : "42703" }, "DELTA_CANNOT_DROP_CHECK_CONSTRAINT_FEATURE" : { "message" : [ "Cannot drop the CHECK constraints table feature.", "The following constraints must be dropped first: ." ], "sqlState" : "0AKDE" }, "DELTA_CANNOT_EVALUATE_EXPRESSION" : { "message" : [ "Cannot evaluate expression: " ], "sqlState" : "0AKDC" }, "DELTA_CANNOT_FIND_VERSION" : { "message" : [ "Cannot find 'sourceVersion' in " ], "sqlState" : "XXKDS" }, "DELTA_CANNOT_GENERATE_CODE_FOR_EXPRESSION" : { "message" : [ "Cannot generate code for expression: " ], "sqlState" : "0AKDC" }, "DELTA_CANNOT_GENERATE_UPDATE_EXPRESSIONS" : { "message" : [ "Calling without generated columns should always return a update expression for each column" ], "sqlState" : "XXKDS" }, "DELTA_CANNOT_MODIFY_APPEND_ONLY" : { "message" : [ "This table is configured to only allow appends. If you would like to permit updates or deletes, use 'ALTER TABLE SET TBLPROPERTIES (=false)'." ], "sqlState" : "42809" }, "DELTA_CANNOT_MODIFY_CATALOG_MANAGED_DEPENDENCIES" : { "message" : [ "Cannot override or unset in-commit timestamp table properties because this table is catalog-managed. Remove \"delta.enableInCommitTimestamps\", \"delta.inCommitTimestampEnablementVersion\", and \"delta.inCommitTimestampEnablementTimestamp\" from the TBLPROPERTIES clause and then retry the command." ], "sqlState" : "42616" }, "DELTA_CANNOT_MODIFY_COORDINATED_COMMITS_DEPENDENCIES" : { "message" : [ " cannot override or unset in-commit timestamp table properties because coordinated commits is enabled in this table and depends on them. Please remove them (\"delta.enableInCommitTimestamps\", \"delta.inCommitTimestampEnablementVersion\", \"delta.inCommitTimestampEnablementTimestamp\") from the TBLPROPERTIES clause and then retry the command again." ], "sqlState" : "42616" }, "DELTA_CANNOT_MODIFY_TABLE_PROPERTY" : { "message" : [ "The Delta table configuration cannot be specified by the user" ], "sqlState" : "42939" }, "DELTA_CANNOT_OVERRIDE_COORDINATED_COMMITS_CONFS" : { "message" : [ " cannot override coordinated commits configurations for an existing target table. Please remove them (\"delta.coordinatedCommits.commitCoordinator-preview\", \"delta.coordinatedCommits.commitCoordinatorConf-preview\", \"delta.coordinatedCommits.tableConf-preview\") from the TBLPROPERTIES clause and then retry the command again." ], "sqlState" : "42616" }, "DELTA_CANNOT_RECONSTRUCT_PATH_FROM_URI" : { "message" : [ "A uri () which cannot be turned into a relative path was found in the transaction log." ], "sqlState" : "22KD1" }, "DELTA_CANNOT_RENAME_PATH" : { "message" : [ "Cannot rename to " ], "sqlState" : "22KD1" }, "DELTA_CANNOT_REPLACE_MISSING_TABLE" : { "message" : [ "Table cannot be replaced as it does not exist. Use CREATE OR REPLACE TABLE to create the table." ], "sqlState" : "42P01" }, "DELTA_CANNOT_RESOLVE_COLUMN" : { "message" : [ "Can't resolve column in " ], "sqlState" : "42703" }, "DELTA_CANNOT_RESOLVE_SOURCE_COLUMN" : { "message" : [ "Couldn't resolve qualified source column within the source query." ], "sqlState" : "XXKDS" }, "DELTA_CANNOT_RESTORE_TABLE_VERSION" : { "message" : [ "Cannot restore table to version . Available versions: [, ]." ], "sqlState" : "22003" }, "DELTA_CANNOT_RESTORE_TIMESTAMP_EARLIER" : { "message" : [ "Cannot restore table to timestamp () as it is before the earliest version available. Please use a timestamp after ()." ], "sqlState" : "22003" }, "DELTA_CANNOT_RESTORE_TIMESTAMP_GREATER" : { "message" : [ "Cannot restore table to timestamp () as it is after the latest version available. Please use a timestamp before ()" ], "sqlState" : "22003" }, "DELTA_CANNOT_SET_COORDINATED_COMMITS_DEPENDENCIES" : { "message" : [ " cannot set in-commit timestamp table properties together with coordinated commits, because the latter depends on the former and sets the former internally. Please remove them (\"delta.enableInCommitTimestamps\", \"delta.inCommitTimestampEnablementVersion\", \"delta.inCommitTimestampEnablementTimestamp\") from the TBLPROPERTIES clause and then retry the command again." ], "sqlState" : "42616" }, "DELTA_CANNOT_SET_LOCATION_MULTIPLE_TIMES" : { "message" : [ "Can't set location multiple times. Found " ], "sqlState" : "XXKDS" }, "DELTA_CANNOT_SET_LOCATION_ON_PATH_IDENTIFIER" : { "message" : [ "Cannot change the location of a path based table." ], "sqlState" : "42613" }, "DELTA_CANNOT_UNSET_COORDINATED_COMMITS_CONFS" : { "message" : [ "ALTER cannot unset coordinated commits configurations. To downgrade a table from coordinated commits, please try again using `ALTER TABLE [table-name] DROP FEATURE 'coordinatedCommits-preview'`." ], "sqlState" : "42616" }, "DELTA_CANNOT_UPDATE_ARRAY_FIELD" : { "message" : [ "Cannot update field type: update the element by updating `.element`." ], "sqlState" : "429BQ" }, "DELTA_CANNOT_UPDATE_MAP_FIELD" : { "message" : [ "Cannot update field type: update a map by updating `.key` or `.value`." ], "sqlState" : "429BQ" }, "DELTA_CANNOT_UPDATE_OTHER_FIELD" : { "message" : [ "Cannot update field of type " ], "sqlState" : "429BQ" }, "DELTA_CANNOT_UPDATE_STRUCT_FIELD" : { "message" : [ "Cannot update field type: update struct by adding, deleting, or updating its fields" ], "sqlState" : "429BQ" }, "DELTA_CANNOT_USE_ALL_COLUMNS_FOR_PARTITION" : { "message" : [ "Cannot use all columns for partition columns" ], "sqlState" : "428FT" }, "DELTA_CANNOT_VACUUM_LITE" : { "message" : [ "VACUUM LITE cannot delete all eligible files as some files are not referenced by the Delta log. Please run VACUUM FULL." ], "sqlState" : "55000" }, "DELTA_CANNOT_WRITE_INTO_VIEW" : { "message" : [ " is a view. Writes to a view are not supported." ], "sqlState" : "0A000" }, "DELTA_CAST_OVERFLOW_IN_TABLE_WRITE" : { "message" : [ "Failed to write a value of type into the type column due to an overflow.", "Use `try_cast` on the input value to tolerate overflow and return NULL instead.", "If necessary, set to \"LEGACY\" to bypass this error or set to true to revert to the old behaviour and follow in UPDATE and MERGE." ], "sqlState" : "22003" }, "DELTA_CDC_NON_CONSTANT_ARGUMENT" : { "message" : [ "The argument (position ) of the function requires a constant value, but got a non-constant expression: .", "" ], "sqlState" : "42K0H" }, "DELTA_CDC_NOT_ALLOWED_IN_THIS_VERSION" : { "message" : [ "Configuration delta.enableChangeDataFeed cannot be set. Change data feed from Delta is not yet available." ], "sqlState" : "0AKDC" }, "DELTA_CDC_READ_NULL_RANGE_BOUNDARY" : { "message" : [ "CDC read start/end parameters cannot be null. Please provide a valid version or timestamp." ], "sqlState" : "22004" }, "DELTA_CDC_START_VERSION_AFTER_LATEST" : { "message" : [ "Start version for change data feed exceeds the latest table version ." ], "sqlState" : "22003" }, "DELTA_CHANGE_DATA_FEED_INCOMPATIBLE_DATA_SCHEMA" : { "message" : [ "Retrieving table changes between version and failed because of an incompatible data schema.", "Your read schema is at version , but we found an incompatible data schema at version .", "If possible, please retrieve the table changes using the end version's schema by setting to `endVersion`, or contact support." ], "sqlState" : "0AKDC" }, "DELTA_CHANGE_DATA_FEED_INCOMPATIBLE_SCHEMA_CHANGE" : { "message" : [ "Retrieving table changes between version and failed because of an incompatible schema change.", "Your read schema is at version , but we found an incompatible schema change at version .", "If possible, please query table changes separately from version to - 1, and from version to ." ], "sqlState" : "0AKDC" }, "DELTA_CHANGE_TABLE_FEED_DISABLED" : { "message" : [ "Cannot write to table with delta.enableChangeDataFeed set. Change data feed from Delta is not available." ], "sqlState" : "42807" }, "DELTA_CHECKPOINT_NON_EXIST_TABLE" : { "message" : [ "Cannot checkpoint a non-existing table . Did you manually delete files in the _delta_log directory?" ], "sqlState" : "42K03" }, "DELTA_CHECKPOINT_SNAPSHOT_MISMATCH" : { "message" : [ "State of the checkpoint doesn't match that of the snapshot." ], "sqlState" : "XXKDS" }, "DELTA_CLONE_AMBIGUOUS_TARGET" : { "message" : [ "", "Two paths were provided as the CLONE target so it is ambiguous which to use. An external", "location for CLONE was provided at at the same time as the path", "." ], "sqlState" : "42613" }, "DELTA_CLONE_INCOMPATIBLE_SOURCE" : { "message" : [ "The clone source has valid format, but has unsupported feature with Delta" ], "subClass" : { "ICEBERG_MISSING_PARTITION_SPECS" : { "message" : [ "Source iceberg table has no partition specs in table" ] }, "ICEBERG_UNDERGONE_PARTITION_EVOLUTION" : { "message" : [ "Source iceberg table has undergone partition evolution." ] } }, "sqlState" : "0AKDC" }, "DELTA_CLONE_UNSUPPORTED_SOURCE" : { "message" : [ "Unsupported clone source '', whose format is .", "The supported formats are 'delta', 'iceberg' and 'parquet'." ], "sqlState" : "0AKDC" }, "DELTA_CLONE_WITH_ROW_TRACKING_WITHOUT_STATS" : { "message" : [ "Cannot shallow clone a table without statistics and with row tracking enabled.", "If you want to enable row tracking you need to first collect statistics on the source table by running:", "ANALYZE TABLE table_name COMPUTE DELTA STATISTICS", "" ], "sqlState" : "22000" }, "DELTA_CLUSTERING_COLUMNS_DATATYPE_NOT_SUPPORTED" : { "message" : [ "CLUSTER BY is not supported because the following column(s): don't support data skipping." ], "sqlState" : "0A000" }, "DELTA_CLUSTERING_COLUMNS_MISMATCH" : { "message" : [ "The provided clustering columns do not match the existing table's.", "- provided: ", "- existing: " ], "sqlState" : "42P10" }, "DELTA_CLUSTERING_COLUMN_MISSING_STATS" : { "message" : [ "Clustering requires clustering columns to have stats. Couldn't find clustering column(s) '' in stats schema:\n" ], "sqlState" : "22000" }, "DELTA_CLUSTERING_REPLACE_TABLE_WITH_PARTITIONED_TABLE" : { "message" : [ "Replacing a clustered Delta table with a partitioned table is not allowed." ], "sqlState" : "42000" }, "DELTA_CLUSTERING_WITH_PARTITION_PREDICATE" : { "message" : [ "OPTIMIZE command for Delta table with clustering doesn't support partition predicates. Please remove the predicates: ." ], "sqlState" : "0A000" }, "DELTA_CLUSTERING_WITH_ZORDER_BY" : { "message" : [ "OPTIMIZE command for Delta table with clustering cannot specify ZORDER BY. Please remove ZORDER BY ()." ], "sqlState" : "42613" }, "DELTA_CLUSTER_BY_INVALID_NUM_COLUMNS" : { "message" : [ "CLUSTER BY supports up to clustering columns, but the table has clustering columns. Please remove the extra clustering columns." ], "sqlState" : "54000" }, "DELTA_CLUSTER_BY_WITH_PARTITIONED_BY" : { "message" : [ "Clustering and partitioning cannot both be specified. Please remove partitionedBy if you want to create a Delta table with clustering." ], "sqlState" : "42613" }, "DELTA_COLUMN_DATA_SKIPPING_NOT_SUPPORTED_PARTITIONED_COLUMN" : { "message" : [ "Data skipping is not supported for partition column ''." ], "sqlState" : "0AKDC" }, "DELTA_COLUMN_DATA_SKIPPING_NOT_SUPPORTED_TYPE" : { "message" : [ "Data skipping is not supported for column '' of type ." ], "sqlState" : "0AKDC" }, "DELTA_COLUMN_MAPPING_MAX_COLUMN_ID_NOT_SET" : { "message" : [ "The max column id property () is not set on a column mapping enabled table." ], "sqlState" : "42703" }, "DELTA_COLUMN_MAPPING_MAX_COLUMN_ID_NOT_SET_CORRECTLY" : { "message" : [ "The max column id property () on a column mapping enabled table is , which cannot be smaller than the max column id for all fields ()." ], "sqlState" : "42703" }, "DELTA_COLUMN_MISSING_DATA_TYPE" : { "message" : [ "The data type of the column was not provided." ], "sqlState" : "42601" }, "DELTA_COLUMN_NOT_FOUND" : { "message" : [ "Unable to find the column `` given []" ], "sqlState" : "42703" }, "DELTA_COLUMN_NOT_FOUND_IN_MERGE" : { "message" : [ "Unable to find the column '' of the target table from the INSERT columns: . INSERT clause must specify value for all the columns of the target table." ], "sqlState" : "42703" }, "DELTA_COLUMN_NOT_FOUND_IN_SCHEMA" : { "message" : [ "Couldn't find column in:\n" ], "sqlState" : "42703" }, "DELTA_COLUMN_PATH_NOT_NESTED" : { "message" : [ "Expected to be a nested data type, but found . Was looking for the", "index of in a nested field.", "Schema:", "" ], "sqlState" : "42704" }, "DELTA_COLUMN_STRUCT_TYPE_MISMATCH" : { "message" : [ "Struct column cannot be inserted into a field in ." ], "sqlState" : "2200G" }, "DELTA_COMMAND_INVARIANT_VIOLATION" : { "message" : [ "A command internal invariant was violated in ''.", "Please retry the command.", "Exception reference: ." ], "sqlState" : "XXKDS" }, "DELTA_COMMIT_INTERMEDIATE_REDIRECT_STATE" : { "message" : [ "Cannot handle commit of table within redirect table state ''." ], "sqlState" : "42P01" }, "DELTA_COMPLEX_TYPE_COLUMN_CONTAINS_NULL_TYPE" : { "message" : [ " Found nested NullType in column which is of . Delta doesn't support writing NullType in complex types." ], "sqlState" : "22005" }, "DELTA_CONCURRENT_APPEND" : { "message" : [ "Transaction conflict detected. a concurrent added data to table committed at version ." ], "subClass" : { "WITHOUT_HINT" : { "message" : [ "The concurrent operation modified data that should have been read by this operation. Please retry the operation. Refer to for more information." ] }, "WITH_PARTITION_HINT" : { "message" : [ "The concurrent operation modified data in the partition that should have been read by this operation. Please retry the operation. Refer to for more information." ] } }, "sqlState" : "2D521" }, "DELTA_CONCURRENT_DELETE_DELETE" : { "message" : [ "Transaction conflict detected, a concurrent deleted data from table (committed at version ) that this transaction attempted to delete." ], "subClass" : { "WITHOUT_HINT" : { "message" : [ "The concurrent operation deleted data that was read by this operation. Please retry the operation. Refer to for more information." ] }, "WITH_PARTITION_HINT" : { "message" : [ "The concurrent operation deleted data in the partition that was read by this operation. Please retry the operation. Refer to for more information." ] } }, "sqlState" : "2D521" }, "DELTA_CONCURRENT_DELETE_READ" : { "message" : [ "Transaction conflict detected, a concurrent deleted data from table (committed at version ) that this transaction read." ], "subClass" : { "WITHOUT_HINT" : { "message" : [ "The concurrent operation deleted data that was read by this operation. Please retry the operation. Refer to for more information." ] }, "WITH_PARTITION_HINT" : { "message" : [ "The concurrent operation deleted data in the partition that was read by this operation. Please retry the operation. Refer to for more information." ] } }, "sqlState" : "2D521" }, "DELTA_CONCURRENT_TRANSACTION" : { "message" : [ "ConcurrentTransactionException: This error occurs when multiple streaming queries are using the same checkpoint to write into this table. Did you run multiple instances of the same streaming query at the same time?\nRefer to for more details." ], "sqlState" : "2D521" }, "DELTA_CONCURRENT_WRITE" : { "message" : [ "ConcurrentWriteException: A concurrent transaction has written new data since the current transaction read the table. Please try the operation again.\nRefer to for more details." ], "sqlState" : "2D521" }, "DELTA_CONFIGURE_SPARK_SESSION_WITH_EXTENSION_AND_CATALOG" : { "message" : [ "This Delta operation requires the SparkSession to be configured with the", "DeltaSparkSessionExtension and the DeltaCatalog. Please set the necessary", "configurations when creating the SparkSession as shown below.", "", " SparkSession.builder()", " .config(\"spark.sql.extensions\", \"\")", " .config(\"\", \"\")", " ...", " .getOrCreate()", "", "If you are using spark-shell/pyspark/spark-submit, you can add the required configurations to the command as show below:", "--conf spark.sql.extensions= --conf =", "" ], "sqlState" : "56038" }, "DELTA_CONFLICT_SET_COLUMN" : { "message" : [ "There is a conflict from these SET columns: ." ], "sqlState" : "42701" }, "DELTA_CONF_OVERRIDE_NOT_SUPPORTED_IN_COMMAND" : { "message" : [ "During , configuration \"\" cannot be set from the command. Please remove it from the TBLPROPERTIES clause and then retry the command again." ], "sqlState" : "42616" }, "DELTA_CONF_OVERRIDE_NOT_SUPPORTED_IN_SESSION" : { "message" : [ "During , configuration \"\" cannot be set from the SparkSession configurations. Please unset it by running `spark.conf.unset(\"\")` and then retry the command again." ], "sqlState" : "42616" }, "DELTA_CONSTRAINT_ALREADY_EXISTS" : { "message" : [ "Constraint '' already exists. Please delete the old constraint first.", "Old constraint:", "" ], "sqlState" : "42710" }, "DELTA_CONSTRAINT_DATA_TYPE_MISMATCH" : { "message" : [ "Column has data type and cannot be altered to data type because this column is referenced by the following check constraint(s):", "" ], "sqlState" : "42K09" }, "DELTA_CONSTRAINT_DEPENDENT_COLUMN_CHANGE" : { "message" : [ "Cannot alter column because this column is referenced by the following check constraint(s):", "" ], "sqlState" : "42K09" }, "DELTA_CONSTRAINT_DOES_NOT_EXIST" : { "message" : [ "Cannot drop nonexistent constraint from table . To avoid throwing an error, provide the parameter IF EXISTS or set the SQL session configuration to ." ], "sqlState" : "42704" }, "DELTA_CONVERSION_NO_PARTITION_FOUND" : { "message" : [ "Found no partition information in the catalog for table . Have you run \"MSCK REPAIR TABLE\" on your table to discover partitions?" ], "sqlState" : "42KD6" }, "DELTA_CONVERSION_UNSUPPORTED_COLUMN_MAPPING" : { "message" : [ "The configuration '' cannot be set to `` when using CONVERT TO DELTA." ], "sqlState" : "0AKDC" }, "DELTA_CONVERT_NON_PARQUET_TABLE" : { "message" : [ "CONVERT TO DELTA only supports parquet tables, but you are trying to convert a source: " ], "sqlState" : "0AKDC" }, "DELTA_CONVERT_TO_DELTA_ROW_TRACKING_WITHOUT_STATS" : { "message" : [ "Cannot enable row tracking without collecting statistics.", "If you want to enable row tracking, do the following:", " 1. Enable statistics collection by running the command", " SET = true", " 2. Run CONVERT TO DELTA without the NO STATISTICS option.", "", "If you do not want to collect statistics, disable row tracking:", " 1. Deactivate enabling the table feature by default by running the command:", " RESET ", " 2. Deactivate the table property by default by running:", " SET = false" ], "sqlState" : "22000" }, "DELTA_CREATE_EXTERNAL_TABLE_WITHOUT_SCHEMA" : { "message" : [ "", "You are trying to create an external table ", "from `` using Delta, but the schema is not specified when the", "input path is empty.", "", "To learn more about Delta, see " ], "sqlState" : "42601" }, "DELTA_CREATE_EXTERNAL_TABLE_WITHOUT_TXN_LOG" : { "message" : [ "You are trying to create an external table from `` using Delta, but there is no transaction log present at ``. Check the upstream job to make sure that it is writing using format(\"delta\") and that the path is the root of the table.", "To learn more about Delta, see " ], "sqlState" : "42K03" }, "DELTA_CREATE_TABLE_IDENTIFIER_LOCATION_MISMATCH" : { "message" : [ "Creating path-based Delta table with a different location isn't supported. Identifier: , Location: " ], "sqlState" : "0AKDC" }, "DELTA_CREATE_TABLE_MISSING_TABLE_NAME_OR_LOCATION" : { "message" : [ "Table name or location has to be specified." ], "sqlState" : "42601" }, "DELTA_CREATE_TABLE_SCHEME_MISMATCH" : { "message" : [ "The specified schema does not match the existing schema at .", "", "== Specified ==", "", "", "== Existing ==", "", "", "== Differences ==", "", "", "If your intention is to keep the existing schema, you can omit the", "schema from the create table command. Otherwise please ensure that", "the schema matches." ], "sqlState" : "42KD7" }, "DELTA_CREATE_TABLE_SET_CLUSTERING_TABLE_FEATURE_NOT_ALLOWED" : { "message" : [ "Cannot enable table feature using TBLPROPERTIES. Please use CREATE OR REPLACE TABLE CLUSTER BY to create a Delta table with clustering." ], "sqlState" : "42000" }, "DELTA_CREATE_TABLE_WITH_DIFFERENT_CLUSTERING" : { "message" : [ "The specified clustering columns do not match the existing clustering columns at .", "== Specified ==", "", "== Existing ==", "", "" ], "sqlState" : "42KD7" }, "DELTA_CREATE_TABLE_WITH_DIFFERENT_PARTITIONING" : { "message" : [ "The specified partitioning does not match the existing partitioning at .", "", "== Specified ==", "", "", "== Existing ==", "", "" ], "sqlState" : "42KD7" }, "DELTA_CREATE_TABLE_WITH_DIFFERENT_PROPERTY" : { "message" : [ "The specified properties do not match the existing properties at .", "", "== Specified ==", "", "", "== Existing ==", "", "" ], "sqlState" : "42KD7" }, "DELTA_CREATE_TABLE_WITH_NON_EMPTY_LOCATION" : { "message" : [ "Cannot create table (''). The associated location ('') is not empty and also not a Delta table." ], "sqlState" : "42601" }, "DELTA_DATA_CHANGE_FALSE" : { "message" : [ "Cannot change table metadata because the 'dataChange' option is set to false. Attempted operation: ''." ], "sqlState" : "0AKDE" }, "DELTA_DELETION_VECTOR_CARDINALITY_MISMATCH" : { "message" : [ "Deletion vector integrity check failed. Encountered a cardinality mismatch." ], "sqlState" : "XXKDS" }, "DELTA_DELETION_VECTOR_CHECKSUM_MISMATCH" : { "message" : [ "Could not verify deletion vector integrity, CRC checksum verification failed." ], "sqlState" : "XXKDS" }, "DELTA_DELETION_VECTOR_INVALID_ROW_INDEX" : { "message" : [ "Deletion vector integrity check failed. Encountered an invalid row index." ], "sqlState" : "XXKDS" }, "DELTA_DELETION_VECTOR_MISSING_NUM_RECORDS" : { "message" : [ "It is invalid to commit files with deletion vectors that are missing the numRecords statistic." ], "sqlState" : "2D521" }, "DELTA_DELETION_VECTOR_SIZE_MISMATCH" : { "message" : [ "Deletion vector integrity check failed. Encountered a size mismatch." ], "sqlState" : "XXKDS" }, "DELTA_DOMAIN_METADATA_NOT_SUPPORTED" : { "message" : [ "Detected DomainMetadata action(s) for domains , but DomainMetadataTableFeature is not enabled." ], "sqlState" : "0A000" }, "DELTA_DROP_COLUMN_AT_INDEX_LESS_THAN_ZERO" : { "message" : [ "Index to drop column is lower than 0" ], "sqlState" : "42KD8" }, "DELTA_DROP_COLUMN_ON_SINGLE_FIELD_SCHEMA" : { "message" : [ "Cannot drop column from a schema with a single column. Schema:", "" ], "sqlState" : "0AKDC" }, "DELTA_DUPLICATE_COLUMNS_FOUND" : { "message" : [ "Found duplicate column(s) : " ], "sqlState" : "42711" }, "DELTA_DUPLICATE_COLUMNS_ON_INSERT" : { "message" : [ "Duplicate column names in INSERT clause" ], "sqlState" : "42701" }, "DELTA_DUPLICATE_COLUMNS_ON_UPDATE_TABLE" : { "message" : [ "", "Please remove duplicate columns before you update your table." ], "sqlState" : "42701" }, "DELTA_DUPLICATE_DATA_SKIPPING_COLUMNS" : { "message" : [ "Duplicated data skipping columns found: ." ], "sqlState" : "42701" }, "DELTA_DUPLICATE_DOMAIN_METADATA_INTERNAL_ERROR" : { "message" : [ "Internal error: two DomainMetadata actions within the same transaction have the same domain " ], "sqlState" : "42601" }, "DELTA_DV_HISTOGRAM_DESERIALIZATON" : { "message" : [ "Could not deserialize the deleted record counts histogram during table integrity verification." ], "sqlState" : "22000" }, "DELTA_DYNAMIC_PARTITION_OVERWRITE_DISABLED" : { "message" : [ "Dynamic partition overwrite mode is specified by session config or write options, but it is disabled by `delta.dynamicPartitionOverwrite.enabled=false`." ], "sqlState" : "0A000" }, "DELTA_EMPTY_DATA" : { "message" : [ "Data used in creating the Delta table doesn't have any columns." ], "sqlState" : "428GU" }, "DELTA_EMPTY_DIRECTORY" : { "message" : [ "No file found in the directory: ." ], "sqlState" : "42K03" }, "DELTA_ENABLING_COLUMN_MAPPING_DISALLOWED_WHEN_COLUMN_MAPPING_METADATA_ALREADY_EXISTS" : { "message" : [ "Enabling column mapping when column mapping metadata is already present in schema is not supported.", "To use column mapping, create a new table and reload the data into it." ], "sqlState" : "XXKDS" }, "DELTA_EXCEED_CHAR_VARCHAR_LIMIT" : { "message" : [ "Value \"\" exceeds char/varchar type length limitation. Failed check: ." ], "sqlState" : "22001" }, "DELTA_EXPRESSIONS_NOT_FOUND_IN_GENERATED_COLUMN" : { "message" : [ "Cannot find the expressions in the generated column " ], "sqlState" : "XXKDS" }, "DELTA_EXTRACT_REFERENCES_FIELD_NOT_FOUND" : { "message" : [ "Field could not be found when extracting references." ], "sqlState" : "XXKDS" }, "DELTA_FAILED_CAST_PARTITION_VALUE" : { "message" : [ "Failed to cast partition value `` to " ], "sqlState" : "22018" }, "DELTA_FAILED_FIND_ATTRIBUTE_IN_OUTPUT_COLUMNS" : { "message" : [ "Could not find among the existing target output " ], "sqlState" : "42703" }, "DELTA_FAILED_FIND_PARTITION_COLUMN_IN_OUTPUT_PLAN" : { "message" : [ "Could not find in output plan." ], "sqlState" : "XXKDS" }, "DELTA_FAILED_INFER_SCHEMA" : { "message" : [ "Failed to infer schema from the given list of files." ], "sqlState" : "42KD9" }, "DELTA_FAILED_MERGE_SCHEMA_FILE" : { "message" : [ "Failed to merge schema of file :", "" ], "sqlState" : "42KDA" }, "DELTA_FAILED_READ_FILE_FOOTER" : { "message" : [ "Could not read footer for file: " ], "sqlState" : "KD001" }, "DELTA_FAILED_RECOGNIZE_PREDICATE" : { "message" : [ "Cannot recognize the predicate ''" ], "sqlState" : "42601" }, "DELTA_FAILED_SCAN_WITH_HISTORICAL_VERSION" : { "message" : [ "Expect a full scan of the latest version of the Delta source, but found a historical scan of version " ], "sqlState" : "KD002" }, "DELTA_FAILED_TO_MERGE_FIELDS" : { "message" : [ "Failed to merge fields '' and ''" ], "sqlState" : "22005" }, "DELTA_FAIL_RELATIVIZE_PATH" : { "message" : [ "Failed to relativize the path (). This can happen when absolute paths make", "it into the transaction log, which start with the scheme", "s3://, wasbs:// or adls://.", "", "If this table is NOT USED IN PRODUCTION, you can set the SQL configuration", " to true.", "Using this SQL configuration could lead to accidental data loss, therefore we do", "not recommend the use of this flag unless this is for testing purposes." ], "sqlState" : "XXKDS" }, "DELTA_FEATURES_PROTOCOL_METADATA_MISMATCH" : { "message" : [ "Unable to operate on this table because the following table features are enabled in metadata but not listed in protocol: ." ], "sqlState" : "KD004" }, "DELTA_FEATURES_REQUIRE_MANUAL_ENABLEMENT" : { "message" : [ "Your table schema requires manually enablement of the following table feature(s): .", "", "To do this, run the following command for each of features listed above:", " ALTER TABLE table_name SET TBLPROPERTIES ('delta.feature.feature_name' = 'supported')", "Replace \"table_name\" and \"feature_name\" with real values.", "", "Current supported feature(s): ." ], "sqlState" : "42000" }, "DELTA_FEATURE_CAN_ONLY_DROP_CHECKPOINT_PROTECTION_WITH_HISTORY_TRUNCATION" : { "message" : [ "Could not drop the Checkpoint Protection feature.", "This feature can only be dropped by truncating history.", "Please try again with the TRUNCATE HISTORY option:", "", " ALTER TABLE table_name DROP FEATURE checkpointProtection TRUNCATE HISTORY" ], "sqlState" : "55000" }, "DELTA_FEATURE_DROP_CHECKPOINT_FAILED" : { "message" : [ "Dropping failed due to a failure in checkpoint creation.", "Please try again later. It the issue persists, contact support." ], "sqlState" : "22KD0" }, "DELTA_FEATURE_DROP_CHECKPOINT_PROTECTION_WAIT_FOR_RETENTION_PERIOD" : { "message" : [ "The operation did not succeed because there are still traces of dropped features ", "in the table history. CheckpointProtection cannot be dropped until these historical", "versions have expired.", "", "To drop CheckpointProtection, please wait for the historical versions to", "expire, and then repeat this command. The retention period for historical versions is", "currently configured to ." ], "sqlState" : "22KD0" }, "DELTA_FEATURE_DROP_CONFLICT_REVALIDATION_FAIL" : { "message" : [ "Cannot drop feature because a concurrent transaction modified the table.", "Please try the operation again.", "" ], "sqlState" : "40000" }, "DELTA_FEATURE_DROP_DEPENDENT_FEATURE" : { "message" : [ "Cannot drop table feature `` because some other features () in this table depends on ``.", "Consider dropping them first before dropping this feature." ], "sqlState" : "55000" }, "DELTA_FEATURE_DROP_FEATURE_IS_DELTA_PROPERTY" : { "message" : [ "Cannot drop `` from this table because this is a delta table property and not a table feature." ], "sqlState" : "42000" }, "DELTA_FEATURE_DROP_FEATURE_NOT_PRESENT" : { "message" : [ "Cannot drop from this table because it is not currently present in the table's protocol." ], "sqlState" : "55000" }, "DELTA_FEATURE_DROP_HISTORICAL_VERSIONS_EXIST" : { "message" : [ "Cannot drop because the Delta log contains historical versions that use the feature.", "Please wait until the history retention period (=) ", "has passed since the feature was last active.", "", "Alternatively, please wait for the TRUNCATE HISTORY retention period to expire ()", "and then run:", " ALTER TABLE table_name DROP FEATURE feature_name TRUNCATE HISTORY" ], "sqlState" : "22KD0" }, "DELTA_FEATURE_DROP_HISTORY_TRUNCATION_NOT_ALLOWED" : { "message" : [ "The particular feature does not require history truncation." ], "sqlState" : "42000" }, "DELTA_FEATURE_DROP_NONREMOVABLE_FEATURE" : { "message" : [ "Cannot drop because dropping this feature is not supported." ], "sqlState" : "0AKDC" }, "DELTA_FEATURE_DROP_UNSUPPORTED_CLIENT_FEATURE" : { "message" : [ "Cannot drop because it is not supported by this Delta version.", "Consider using Delta with a higher version." ], "sqlState" : "0AKDC" }, "DELTA_FEATURE_DROP_WAIT_FOR_RETENTION_PERIOD" : { "message" : [ "Dropping was partially successful.", "", "The feature is now no longer used in the current version of the table. However, the feature", "is still present in historical versions of the table. The table feature cannot be dropped", "from the table protocol until these historical versions have expired.", "", "To drop the table feature from the protocol, please wait for the historical versions to", "expire, and then repeat this command. The retention period for historical versions is", "currently configured as =.", "", "Alternatively, please wait for the TRUNCATE HISTORY retention period to expire ()", "and then run:", " ALTER TABLE table_name DROP FEATURE feature_name TRUNCATE HISTORY" ], "sqlState" : "22KD0" }, "DELTA_FEATURE_REQUIRES_HIGHER_READER_VERSION" : { "message" : [ "Unable to enable table feature because it requires a higher reader protocol version (current ). Consider upgrading the table's reader protocol version to , or to a version which supports reader table features. Refer to for more information on table protocol versions." ], "sqlState" : "55000" }, "DELTA_FEATURE_REQUIRES_HIGHER_WRITER_VERSION" : { "message" : [ "Unable to enable table feature because it requires a higher writer protocol version (current ). Consider upgrading the table's writer protocol version to , or to a version which supports writer table features. Refer to for more information on table protocol versions." ], "sqlState" : "55000" }, "DELTA_FILE_ALREADY_EXISTS" : { "message" : [ "Existing file path " ], "sqlState" : "42K04" }, "DELTA_FILE_LIST_AND_PATTERN_STRING_CONFLICT" : { "message" : [ "Cannot specify both file list and pattern string." ], "sqlState" : "42613" }, "DELTA_FILE_NOT_FOUND" : { "message" : [ "File path " ], "sqlState" : "42K03" }, "DELTA_FILE_OR_DIR_NOT_FOUND" : { "message" : [ "No such file or directory: " ], "sqlState" : "42K03" }, "DELTA_FILE_TO_OVERWRITE_NOT_FOUND" : { "message" : [ "File () to be rewritten not found among candidate files:\n" ], "sqlState" : "42K03" }, "DELTA_FOUND_MAP_TYPE_COLUMN" : { "message" : [ "A MapType was found. In order to access the key or value of a MapType, specify one", "of:", " or", "", "followed by the name of the column (only if that column is a struct type).", "e.g. mymap.key.mykey", "If the column is a basic type, mymap.key or mymap.value is sufficient.", "Schema:", "" ], "sqlState" : "KD003" }, "DELTA_GENERATED_COLUMNS_DATA_TYPE_MISMATCH" : { "message" : [ "Column has data type and cannot be altered to data type because this column is referenced by the following generated column(s):", "" ], "sqlState" : "42K09" }, "DELTA_GENERATED_COLUMNS_DEPENDENT_COLUMN_CHANGE" : { "message" : [ "Cannot alter column because this column is referenced by the following generated column(s):", "" ], "sqlState" : "42K09" }, "DELTA_GENERATED_COLUMNS_EXPR_TYPE_MISMATCH" : { "message" : [ "The expression type of the generated column is , but the column type is " ], "sqlState" : "42K09" }, "DELTA_GENERATED_COLUMN_UPDATE_TYPE_MISMATCH" : { "message" : [ "Column is a generated column or a column used by a generated column. The data type is and cannot be converted to data type " ], "sqlState" : "42K09" }, "DELTA_ICEBERG_COMPAT_VIOLATION" : { "message" : [ "The validation of IcebergCompatV has failed." ], "subClass" : { "CHANGE_VERSION_NEED_REWRITE" : { "message" : [ "Changing to IcebergCompatV requires rewriting the table. Please run REORG TABLE APPLY (UPGRADE UNIFORM ('ICEBERG_COMPAT_VERSION = '));", "Note that REORG enables table feature IcebergCompatV and other Delta lake clients without that table feature support may not be able to write to the table." ] }, "COMPAT_VERSION_NOT_SUPPORTED" : { "message" : [ "IcebergCompatVersion = is not supported. Supported versions are between 1 and " ] }, "DELETION_VECTORS_NOT_PURGED" : { "message" : [ "IcebergCompatV requires Deletion Vectors to be completely purged from the table. Please run the REORG TABLE APPLY (PURGE) command." ] }, "DELETION_VECTORS_SHOULD_BE_DISABLED" : { "message" : [ "IcebergCompatV requires Deletion Vectors to be disabled on the table first. Then run REORG PURGE command to purge the Deletion Vectors on the table." ] }, "DISABLING_REQUIRED_TABLE_FEATURE" : { "message" : [ "IcebergCompatV requires feature to be supported and enabled. You cannot drop it from the table. Instead, please disable IcebergCompatV first." ] }, "FILES_NOT_ICEBERG_COMPAT" : { "message" : [ "Enabling Uniform Iceberg with IcebergCompatV requires all files to be iceberg compatible.", "There are files in table version and files are not iceberg compatible, which is usually a result of concurrent write.", "Please run the REORG TABLE table APPLY (UPGRADE UNIFORM (ICEBERG_COMPAT_VERSION=) command again." ] }, "INCOMPATIBLE_TABLE_FEATURE" : { "message" : [ "IcebergCompatV is incompatible with feature ." ] }, "MISSING_REQUIRED_TABLE_FEATURE" : { "message" : [ "IcebergCompatV requires feature to be supported and enabled." ] }, "REPLACE_TABLE_CHANGE_PARTITION_NAMES" : { "message" : [ "IcebergCompatV doesn't support replacing partitioned tables with a differently-named partition spec, because Iceberg-Spark 1.1.0 doesn't.", "Prev Partition Spec: ", "New Partition Spec: " ] }, "REWRITE_DATA_FAILED" : { "message" : [ "Rewriting data to IcebergCompatV failed.", "Please run the REORG TABLE table APPLY (UPGRADE UNIFORM (ICEBERG_COMPAT_VERSION=) command again." ] }, "UNSUPPORTED_DATA_TYPE" : { "message" : [ "IcebergCompatV does not support the data type in your schema. Your schema:", "" ] }, "UNSUPPORTED_PARTITION_DATA_TYPE" : { "message" : [ "IcebergCompatV does not support the data type for partition columns in your schema. Your partition schema:", "" ] }, "UNSUPPORTED_TYPE_WIDENING" : { "message" : [ "IcebergCompatV is incompatible with a type change applied to this table:", "Field was changed from to ." ] }, "VERSION_MUTUAL_EXCLUSIVE" : { "message" : [ "Only one IcebergCompat version can be enabled, please explicitly disable all other IcebergCompat versions that are not needed." ] }, "WRONG_REQUIRED_TABLE_PROPERTY" : { "message" : [ "IcebergCompatV requires table property '' to be set to ''. Current value: ''." ] } }, "sqlState" : "KD00E" }, "DELTA_IDENTITY_COLUMNS_ALTER_COLUMN_NOT_SUPPORTED" : { "message" : [ "ALTER TABLE ALTER COLUMN is not supported for IDENTITY columns." ], "sqlState" : "429BQ" }, "DELTA_IDENTITY_COLUMNS_ALTER_NON_DELTA_FORMAT" : { "message" : [ "ALTER TABLE ALTER COLUMN SYNC IDENTITY is only supported by Delta." ], "sqlState" : "0AKDD" }, "DELTA_IDENTITY_COLUMNS_ALTER_NON_IDENTITY_COLUMN" : { "message" : [ "ALTER TABLE ALTER COLUMN SYNC IDENTITY cannot be called on non IDENTITY columns." ], "sqlState" : "429BQ" }, "DELTA_IDENTITY_COLUMNS_EXPLICIT_INSERT_NOT_SUPPORTED" : { "message" : [ "Providing values for GENERATED ALWAYS AS IDENTITY column is not supported." ], "sqlState" : "42808" }, "DELTA_IDENTITY_COLUMNS_ILLEGAL_STEP" : { "message" : [ "IDENTITY column step cannot be 0." ], "sqlState" : "42611" }, "DELTA_IDENTITY_COLUMNS_PARTITION_NOT_SUPPORTED" : { "message" : [ "PARTITIONED BY IDENTITY column is not supported." ], "sqlState" : "42601" }, "DELTA_IDENTITY_COLUMNS_REPLACE_COLUMN_NOT_SUPPORTED" : { "message" : [ "ALTER TABLE REPLACE COLUMNS is not supported for table with IDENTITY columns." ], "sqlState" : "429BQ" }, "DELTA_IDENTITY_COLUMNS_UNSUPPORTED_DATA_TYPE" : { "message" : [ "DataType is not supported for IDENTITY columns." ], "sqlState" : "428H2" }, "DELTA_IDENTITY_COLUMNS_UPDATE_NOT_SUPPORTED" : { "message" : [ "UPDATE on IDENTITY column is not supported." ], "sqlState" : "42808" }, "DELTA_IDENTITY_COLUMNS_WITH_GENERATED_EXPRESSION" : { "message" : [ "IDENTITY column cannot be specified with a generated column expression." ], "sqlState" : "42613" }, "DELTA_ILLEGAL_FILE_FOUND" : { "message" : [ "Illegal files found in a dataChange = false transaction. Files: " ], "sqlState" : "XXKDS" }, "DELTA_ILLEGAL_OPTION" : { "message" : [ "Invalid value '' for option '', " ], "sqlState" : "42616" }, "DELTA_ILLEGAL_USAGE" : { "message" : [ "The usage of
'.", "Another stream may be reusing the same schema location, which is not allowed.", "Please provide a new unique `schemaTrackingLocation` path or `streamingSourceTrackingId` as a reader option for one of the streams from this table." ], "sqlState" : "22000" }, "DELTA_STREAMING_SCHEMA_LOCATION_NOT_UNDER_CHECKPOINT" : { "message" : [ "Schema location '' must be placed under checkpoint location ''." ], "sqlState" : "22000" }, "DELTA_STREAMING_SCHEMA_LOG_DESERIALIZE_FAILED" : { "message" : [ "Incomplete log file in the Delta streaming source schema log at ''.", "The schema log may have been corrupted. Please pick a new schema location." ], "sqlState" : "22000" }, "DELTA_STREAMING_SCHEMA_LOG_INCOMPATIBLE_DELTA_TABLE_ID" : { "message" : [ "Detected incompatible Delta table id when trying to read Delta stream.", "Persisted table id: , Table id: ", "The schema log might have been reused. Please pick a new schema location." ], "sqlState" : "22000" }, "DELTA_STREAMING_SCHEMA_LOG_INCOMPATIBLE_PARTITION_SCHEMA" : { "message" : [ "Detected incompatible partition schema when trying to read Delta stream.", "Persisted schema: , Delta partition schema: ", "Please pick a new schema location to reinitialize the schema log if you have manually changed the table's partition schema recently." ], "sqlState" : "22000" }, "DELTA_STREAMING_SCHEMA_LOG_INIT_FAILED_INCOMPATIBLE_METADATA" : { "message" : [ "We could not initialize the Delta streaming source schema log because", "we detected an incompatible schema or protocol change while serving a streaming batch from table version to ." ], "sqlState" : "22000" }, "DELTA_STREAMING_SCHEMA_LOG_PARSE_SCHEMA_FAILED" : { "message" : [ "Failed to parse the schema from the Delta streaming source schema log.", "The schema log may have been corrupted. Please pick a new schema location." ], "sqlState" : "22000" }, "DELTA_STREAMING_SCHEMA_MISMATCH_ON_RESTART" : { "message" : [ "The streaming query's schema does not match the current table schema.", "Query schema (from analysis): ", "", "Current table schema: ", "", "The table schema has changed since this streaming query was created.", "Please create a new streaming DataFrame to pick up the current schema." ], "sqlState" : "KD007" }, "DELTA_TABLE_ALREADY_CONTAINS_CDC_COLUMNS" : { "message" : [ "Unable to enable Change Data Capture on the table. The table already contains", "reserved columns that will", "be used internally as metadata for the table's Change Data Feed. To enable", "Change Data Feed on the table rename/drop these columns.", "" ], "sqlState" : "42711" }, "DELTA_TABLE_ALREADY_EXISTS" : { "message" : [ "Table already exists." ], "sqlState" : "42P07" }, "DELTA_TABLE_FOR_PATH_UNSUPPORTED_HADOOP_CONF" : { "message" : [ "Currently DeltaTable.forPath only supports hadoop configuration keys starting with but got " ], "sqlState" : "0AKDC" }, "DELTA_TABLE_FOUND_IN_EXECUTOR" : { "message" : [ "DeltaTable cannot be used in executors" ], "sqlState" : "XXKDS" }, "DELTA_TABLE_INVALID_REDIRECT_STATE_TRANSITION" : { "message" : [ "Unable to update table redirection state: Invalid state transition attempted.", "The Delta table '
' cannot change from '' to ''." ], "sqlState" : "22023" }, "DELTA_TABLE_INVALID_REMOVE_TABLE_REDIRECT" : { "message" : [ "Unable to remove table redirection for
due to its invalid state: ." ], "sqlState" : "KD007" }, "DELTA_TABLE_INVALID_SET_UNSET_REDIRECT" : { "message" : [ "Unable to SET or UNSET redirect property on
: current property '' mismatches with new property ''." ], "sqlState" : "22023" }, "DELTA_TABLE_LOCATION_MISMATCH" : { "message" : [ "The location of the existing table is . It doesn't match the specified location ." ], "sqlState" : "42613" }, "DELTA_TABLE_NOT_FOUND" : { "message" : [ "Delta table doesn't exist." ], "sqlState" : "42P01" }, "DELTA_TABLE_NOT_SUPPORTED_IN_OP" : { "message" : [ "Table is not supported in . Please use a path instead." ], "sqlState" : "42809" }, "DELTA_TABLE_ONLY_OPERATION" : { "message" : [ " is not a Delta table. is only supported for Delta tables." ], "sqlState" : "0AKDD" }, "DELTA_TABLE_UNRECOGNIZED_REDIRECT_SPEC" : { "message" : [ "The Delta log contains unrecognized table redirect spec ''." ], "sqlState" : "42704" }, "DELTA_TARGET_TABLE_FINAL_SCHEMA_EMPTY" : { "message" : [ "Target table final schema is empty." ], "sqlState" : "428GU" }, "DELTA_TIMESTAMP_EARLIER_THAN_COMMIT_RETENTION" : { "message" : [ "The provided timestamp () is before the earliest version available to", "this table (). Please use a timestamp after ." ], "sqlState" : "42816" }, "DELTA_TIMESTAMP_GREATER_THAN_COMMIT" : { "message" : [ "The provided timestamp () is after the latest version available to this", "table (). Please use a timestamp before or at ." ], "sqlState" : "42816" }, "DELTA_TIMESTAMP_INVALID" : { "message" : [ "The provided timestamp () cannot be converted to a valid timestamp." ], "sqlState" : "42816" }, "DELTA_TIME_TRAVEL_INVALID_BEGIN_VALUE" : { "message" : [ " needs to be a valid begin value." ], "sqlState" : "42604" }, "DELTA_TRUNCATED_TRANSACTION_LOG" : { "message" : [ ": Unable to reconstruct state at version as the transaction log has been truncated due to manual deletion or the log retention policy (=) and checkpoint retention policy (=)" ], "sqlState" : "42K03" }, "DELTA_TRUNCATE_TABLE_PARTITION_NOT_SUPPORTED" : { "message" : [ "Operation not allowed: TRUNCATE TABLE on Delta tables does not support partition predicates; use DELETE to delete specific partitions or rows." ], "sqlState" : "0AKDC" }, "DELTA_TXN_LOG_FAILED_INTEGRITY" : { "message" : [ "The transaction log has failed integrity checks. Failed verification at version of:", "" ], "sqlState" : "XXKDS" }, "DELTA_UDF_IN_CHECK_CONSTRAINT" : { "message" : [ "Found in a CHECK constraint. A CHECK constraint cannot use a user-defined function." ], "sqlState" : "42621" }, "DELTA_UDF_IN_GENERATED_COLUMN" : { "message" : [ "Found . A generated column cannot use a user-defined function" ], "sqlState" : "42621" }, "DELTA_UNEXPECTED_ACTION_EXPRESSION" : { "message" : [ "Unexpected action expression ." ], "sqlState" : "42601" }, "DELTA_UNEXPECTED_ALIAS" : { "message" : [ "Expected Alias but got " ], "sqlState" : "XXKDS" }, "DELTA_UNEXPECTED_ATTRIBUTE_REFERENCE" : { "message" : [ "Expected AttributeReference but got " ], "sqlState" : "XXKDS" }, "DELTA_UNEXPECTED_CHANGE_FILES_FOUND" : { "message" : [ "Change files found in a dataChange = false transaction. Files:", "" ], "sqlState" : "XXKDS" }, "DELTA_UNEXPECTED_NUM_PARTITION_COLUMNS_FROM_FILE_NAME" : { "message" : [ "Expecting partition column(s): , but found partition column(s): from parsing the file name: " ], "sqlState" : "KD009" }, "DELTA_UNEXPECTED_PARTIAL_SCAN" : { "message" : [ "Expect a full scan of Delta sources, but found a partial scan. path:" ], "sqlState" : "KD00A" }, "DELTA_UNEXPECTED_PARTITION_COLUMN_FROM_FILE_NAME" : { "message" : [ "Expecting partition column , but found partition column from parsing the file name: " ], "sqlState" : "KD009" }, "DELTA_UNEXPECTED_PARTITION_SCHEMA_FROM_USER" : { "message" : [ "CONVERT TO DELTA was called with a partition schema different from the partition schema inferred from the catalog, please avoid providing the schema so that the partition schema can be chosen from the catalog.", "", "catalog partition schema:", "", "provided partition schema:", "" ], "sqlState" : "KD009" }, "DELTA_UNEXPECTED_PROJECT" : { "message" : [ "Expected Project but got " ], "sqlState" : "XXKDS" }, "DELTA_UNIVERSAL_FORMAT_CONVERSION_FAILED" : { "message" : [ "Failed to convert the table version to the universal format . " ], "sqlState" : "KD00E" }, "DELTA_UNIVERSAL_FORMAT_VIOLATION" : { "message" : [ "The validation of Universal Format () has failed: " ], "sqlState" : "KD00E" }, "DELTA_UNKNOWN_CONFIGURATION" : { "message" : [ "Unknown configuration was specified: ", "To disable this check, set =true in the Spark session configuration." ], "sqlState" : "F0000" }, "DELTA_UNKNOWN_PRIVILEGE" : { "message" : [ "Unknown privilege: " ], "sqlState" : "42601" }, "DELTA_UNKNOWN_READ_LIMIT" : { "message" : [ "Unknown ReadLimit: " ], "sqlState" : "42601" }, "DELTA_UNRECOGNIZED_COLUMN_CHANGE" : { "message" : [ "Unrecognized column change . You may be running an out-of-date Delta Lake version." ], "sqlState" : "42601" }, "DELTA_UNRECOGNIZED_FILE_ACTION" : { "message" : [ "Unrecognized file action with type ." ], "sqlState" : "XXKDS" }, "DELTA_UNRECOGNIZED_INVARIANT" : { "message" : [ "Unrecognized invariant. Please upgrade your Spark version." ], "sqlState" : "56038" }, "DELTA_UNRECOGNIZED_LOGFILE" : { "message" : [ "Unrecognized log file " ], "sqlState" : "KD00B" }, "DELTA_UNSET_NON_EXISTENT_PROPERTY" : { "message" : [ "Attempted to unset non-existent property '' in table " ], "sqlState" : "42616" }, "DELTA_UNSUPPORTED_ABS_PATH_ADD_FILE" : { "message" : [ " does not support adding files with an absolute path" ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_ALTER_TABLE_CHANGE_COL_OP" : { "message" : [ "ALTER TABLE CHANGE COLUMN is not supported for changing column from to " ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_ALTER_TABLE_REPLACE_COL_OP" : { "message" : [ "Unsupported ALTER TABLE REPLACE COLUMNS operation. Reason:
", "", "Failed to change schema from:", "", "to:", "" ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_CREATION" : { "message" : [ "Creating a catalog-managed table using delta is unsupported." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION" : { "message" : [ " is blocked by the catalog for Catalog-Managed tables." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_CLONE_REPLACE_SAME_TABLE" : { "message" : [ "", "You tried to REPLACE an existing table () with CLONE. This operation is", "unsupported. Try a different target for CLONE or delete the table at the current target.", "" ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_COLUMN_MAPPING_MODE_CHANGE" : { "message" : [ "Changing column mapping mode from '' to '' is not supported." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_COLUMN_MAPPING_SCHEMA_CHANGE" : { "message" : [ "", "Schema change is detected:", "", "old schema:", "", "", "new schema:", "", "", "Schema changes are not allowed during the change of column mapping mode.", "", "" ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_COLUMN_MAPPING_WRITE" : { "message" : [ "Writing data with column mapping mode is not supported." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_COLUMN_TYPE_IN_BLOOM_FILTER" : { "message" : [ "Creating a bloom filter index on a column with type is unsupported: " ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_COMMENT_MAP_ARRAY" : { "message" : [ "Can't add a comment to . Adding a comment to a map key/value or array element is not supported." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_DATA_TYPES" : { "message" : [ "Found columns using unsupported data types: . You can set '' to 'false' to disable the type check. Disabling this type check may allow users to create unsupported Delta tables and should only be used when trying to read/write legacy tables." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_DATA_TYPE_IN_GENERATED_COLUMN" : { "message" : [ " cannot be the result of a generated column" ], "sqlState" : "42621" }, "DELTA_UNSUPPORTED_DEEP_CLONE" : { "message" : [ "Deep clone is not supported by this Delta version." ], "sqlState" : "0A000" }, "DELTA_UNSUPPORTED_DESCRIBE_DETAIL_VIEW" : { "message" : [ " is a view. DESCRIBE DETAIL is only supported for tables." ], "sqlState" : "42809" }, "DELTA_UNSUPPORTED_DROP_CLUSTERING_COLUMN" : { "message" : [ "Dropping clustering columns () is not allowed." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_DROP_COLUMN" : { "message" : [ "DROP COLUMN is not supported for your Delta table. " ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_DROP_NESTED_COLUMN_FROM_NON_STRUCT_TYPE" : { "message" : [ "Can only drop nested columns from StructType. Found " ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_DROP_PARTITION_COLUMN" : { "message" : [ "Dropping partition columns () is not allowed." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_EXPRESSION" : { "message" : [ "Unsupported expression type() for . The supported types are []." ], "sqlState" : "0A000" }, "DELTA_UNSUPPORTED_EXPRESSION_CHECK_CONSTRAINT" : { "message" : [ "Found in a CHECK constraint. cannot be used in a CHECK constraint." ], "sqlState" : "42621" }, "DELTA_UNSUPPORTED_EXPRESSION_GENERATED_COLUMN" : { "message" : [ " cannot be used in a generated column" ], "sqlState" : "42621" }, "DELTA_UNSUPPORTED_FEATURES_FOR_READ" : { "message" : [ "Unsupported Delta read feature: table \"\" requires reader table feature(s) that are unsupported by Delta Lake \"\": ." ], "sqlState" : "56038" }, "DELTA_UNSUPPORTED_FEATURES_FOR_WRITE" : { "message" : [ "Unsupported Delta write feature: table \"\" requires writer table feature(s) that are unsupported by Delta Lake \"\": ." ], "sqlState" : "56038" }, "DELTA_UNSUPPORTED_FEATURES_IN_CONFIG" : { "message" : [ "Table features configured in the following Spark configs or Delta table properties are not recognized by this version of Delta Lake: ." ], "sqlState" : "56038" }, "DELTA_UNSUPPORTED_FEATURE_STATUS" : { "message" : [ "Expecting the status for table feature to be \"supported\", but got \"\"." ], "sqlState" : "0AKDE" }, "DELTA_UNSUPPORTED_FIELD_UPDATE_NON_STRUCT" : { "message" : [ "Updating nested fields is only supported for StructType, but you are trying to update a field of , which is of type: ." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_GENERATE_WITH_DELETION_VECTORS" : { "message" : [ "The 'GENERATE symlink_format_manifest' command is not supported on table versions with deletion vectors.", "If you need to generate manifests, consider disabling deletion vectors on this table using 'ALTER TABLE table SET TBLPROPERTIES (delta.enableDeletionVectors = false)'." ], "sqlState" : "0A000" }, "DELTA_UNSUPPORTED_INVARIANT_NON_STRUCT" : { "message" : [ "Invariants on nested fields other than StructTypes are not supported." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_IN_SUBQUERY" : { "message" : [ "In subquery is not supported in the condition." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_MANIFEST_GENERATION_WITH_COLUMN_MAPPING" : { "message" : [ "Manifest generation is not supported for tables that leverage column mapping, as external readers cannot read these Delta tables. See Delta documentation for more details." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_MULTI_COL_IN_PREDICATE" : { "message" : [ "Multi-column In predicates are not supported in the condition." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_NESTED_COLUMN_IN_BLOOM_FILTER" : { "message" : [ "Creating a bloom filer index on a nested column is currently unsupported: " ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_NESTED_FIELD_IN_OPERATION" : { "message" : [ "Nested field is not supported in the (field = )." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_NON_EMPTY_CLONE" : { "message" : [ "The clone destination table is non-empty. Please TRUNCATE or DELETE FROM the table before running CLONE." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_OUTPUT_MODE" : { "message" : [ "Data source does not support output mode" ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_PARTITION_COLUMN_CHANGE" : { "message" : [ "Cannot change partition columns during operation (old: , new: )." ], "sqlState" : "42P10" }, "DELTA_UNSUPPORTED_PARTITION_COLUMN_IN_BLOOM_FILTER" : { "message" : [ "Creating a bloom filter index on a partitioning column is unsupported: " ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_RENAME_COLUMN" : { "message" : [ "Column rename is not supported for your Delta table. " ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_SCHEMA_DURING_READ" : { "message" : [ "Delta does not support specifying the schema at read time." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_SOURCE" : { "message" : [ " destination only supports Delta sources.\n" ], "sqlState" : "0AKDD" }, "DELTA_UNSUPPORTED_STATIC_PARTITIONS" : { "message" : [ "Specifying static partitions in the partition spec is currently not supported during inserts" ], "sqlState" : "0AKDD" }, "DELTA_UNSUPPORTED_STATS_RECOMPUTE_WITH_DELETION_VECTORS" : { "message" : [ "Statistics re-computation on a Delta table with deletion vectors is not yet supported." ], "sqlState" : "0AKDD" }, "DELTA_UNSUPPORTED_SUBQUERY" : { "message" : [ "Subqueries are not supported in the (condition = )." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_SUBQUERY_IN_PARTITION_PREDICATES" : { "message" : [ "Subquery is not supported in partition predicates." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_TIME_TRAVEL_BEYOND_DELETED_FILE_RETENTION_DURATION" : { "message" : [ "Cannot time travel beyond delta.deletedFileRetentionDuration ( HOURS) set on the table." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_TIME_TRAVEL_MULTIPLE_FORMATS" : { "message" : [ "Cannot specify time travel in multiple formats." ], "sqlState" : "42613" }, "DELTA_UNSUPPORTED_TIME_TRAVEL_VIEWS" : { "message" : [ "Cannot time travel views, subqueries, streams or change data feed queries." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_TYPE_CHANGE_IN_PREVIEW" : { "message" : [ "This table can't be read by this version of Delta because an unsupported type change was applied. Field was changed from to .", "Please upgrade to Delta 4.0 or higher to read this table, or drop the Type Widening table feature using a client that supports reading this table:", " ALTER TABLE tableName DROP FEATURE " ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_TYPE_CHANGE_IN_SCHEMA" : { "message" : [ "Unable to operate on this table because an unsupported type change was applied. Field was changed from to ." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_VACUUM_SPECIFIC_PARTITION" : { "message" : [ "Please provide the base path () when Vacuuming Delta tables. Vacuuming specific partitions is currently not supported." ], "sqlState" : "0AKDC" }, "DELTA_UNSUPPORTED_WRITES_STAGED_TABLE" : { "message" : [ "Table implementation does not support writes: " ], "sqlState" : "42807" }, "DELTA_UNSUPPORTED_WRITES_WITHOUT_COORDINATOR" : { "message" : [ "You are trying to perform writes on a table which has been registered with the commit coordinator . However, no implementation of this coordinator is available in the current environment and writes without coordinators are not allowed." ], "sqlState" : "0AKDC" }, "DELTA_UPDATE_SCHEMA_MISMATCH_EXPRESSION" : { "message" : [ "Cannot cast to . All nested columns must match." ], "sqlState" : "42846" }, "DELTA_USER_DEFINED_TYPE_COLUMN_CONTAINS_NULL_TYPE" : { "message" : [ "Found NullType in column which is of user-defined type. Delta doesn't support writing NullType in user-defined types." ], "sqlState" : "22005" }, "DELTA_VACUUM_RETENTION_PERIOD_NEGATIVE" : { "message" : [ "Retention period for Vacuum can't be less than 0 hours." ], "sqlState" : "22003" }, "DELTA_VACUUM_RETENTION_PERIOD_TOO_SHORT" : { "message" : [ "The specified VACUUM retention period is too low and may corrupt this Delta table if any writes are in progress.", "", "If no operations (insert, upsert, delete, optimize) are running, you can disable this safety check by setting:", "delta.retentionDurationCheck.enabled = false", "", "Otherwise, use a retention period of at least hours." ], "sqlState" : "22003" }, "DELTA_VERSIONS_NOT_CONTIGUOUS" : { "message" : [ "Versions () are not contiguous. ", "A gap in the delta log between versions and was detected while trying to load version ." ], "sqlState" : "KD00C" }, "DELTA_VERSION_INVALID" : { "message" : [ "The provided version () is not a valid version." ], "sqlState" : "42815" }, "DELTA_VERSION_NOT_FOUND" : { "message" : [ "Cannot time travel Delta table to version . Available versions: [, ]." ], "sqlState" : "22003" }, "DELTA_VIOLATE_CONSTRAINT_WITH_VALUES" : { "message" : [ "CHECK constraint violated by row with values:", "" ], "sqlState" : "23001" }, "DELTA_VIOLATE_TABLE_PROPERTY_VALIDATION_FAILED" : { "message" : [ "The validation of the properties of table
has been violated:" ], "subClass" : { "EXISTING_DELETION_VECTORS_WITH_INCREMENTAL_MANIFEST_GENERATION" : { "message" : [ "Symlink manifest generation is unsupported while deletion vectors are present in the table.", "In order to produce a version of the table without deletion vectors, run 'REORG TABLE
APPLY (PURGE)'." ] }, "PERSISTENT_DELETION_VECTORS_IN_NON_PARQUET_TABLE" : { "message" : [ "Persistent deletion vectors are only supported on Parquet-based Delta tables." ] }, "PERSISTENT_DELETION_VECTORS_WITH_INCREMENTAL_MANIFEST_GENERATION" : { "message" : [ "Persistent deletion vectors and incremental symlink manifest generation are mutually exclusive." ] } }, "sqlState" : "0A000" }, "DELTA_ZORDERING_COLUMN_DOES_NOT_EXIST" : { "message" : [ "Z-Ordering column does not exist in data schema." ], "sqlState" : "42703" }, "DELTA_ZORDERING_ON_COLUMN_WITHOUT_STATS" : { "message" : [ "Z-Ordering on will be ineffective, because we currently do not collect stats for these columns. You can disable this check by setting", " SET = false" ], "sqlState" : "KD00D" }, "DELTA_ZORDERING_ON_PARTITION_COLUMN" : { "message" : [ " is a partition column. Z-Ordering can only be performed on data columns" ], "sqlState" : "42P10" }, "DIFFERENT_DELTA_TABLE_READ_BY_STREAMING_SOURCE" : { "message" : [ "The streaming query was reading from an unexpected Delta table (id = ''). ", "It used to read from another Delta table (id = '') according to checkpoint. ", "This may happen when you changed the code to read from a new table or you deleted and ", "re-created a table. Please revert your change or delete your streaming query checkpoint ", "to restart from scratch." ], "sqlState" : "55019" }, "INCORRECT_NUMBER_OF_ARGUMENTS" : { "message" : [ ", requires at least arguments and at most arguments." ], "sqlState" : "42605" }, "RESERVED_CDC_COLUMNS_ON_WRITE" : { "message" : [ "", "The write contains reserved columns that are used", "internally as metadata for Change Data Feed. To write to the table either rename/drop", "these columns or disable Change Data Feed on the table by setting", " to false." ], "sqlState" : "42939" }, "WRONG_COLUMN_DEFAULTS_FOR_DELTA_ALTER_TABLE_ADD_COLUMN_NOT_SUPPORTED" : { "message" : [ "Failed to execute the command because DEFAULT values are not supported when adding new", "columns to previously existing Delta tables; please add the column without a default", "value first, then run a second ALTER TABLE ALTER COLUMN SET DEFAULT command to apply", "for future inserted rows instead." ], "sqlState" : "0AKDC" }, "WRONG_COLUMN_DEFAULTS_FOR_DELTA_FEATURE_NOT_ENABLED" : { "message" : [ "Failed to execute command because it assigned a column DEFAULT value,", "but the corresponding table feature was not enabled. Please retry the command again", "after executing ALTER TABLE tableName SET", "TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported')." ], "sqlState" : "0AKDE" }, "_LEGACY_ERROR_TEMP_DELTA_0001" : { "message" : [ "Cannot use '' as the name of a CHECK constraint." ] }, "_LEGACY_ERROR_TEMP_DELTA_0002" : { "message" : [ "Cannot create bloom filter index, invalid parameter value: ''." ] }, "_LEGACY_ERROR_TEMP_DELTA_0003" : { "message" : [ "You are trying to convert a table which already has a delta log where the table properties in the catalog don't match the configuration in the delta log.", "Table properties in catalog:", "", "Delta configuration:", "", "If you would like to merge the configurations (update existing fields and insert new ones), set the SQL configuration `` to false." ] }, "_LEGACY_ERROR_TEMP_DELTA_0006" : { "message" : [ "Inconsistent IDENTITY metadata for column detected: , , " ] }, "_LEGACY_ERROR_TEMP_DELTA_0008" : { "message" : [ "Error while searching for position of column .", "Schema:", "", "Error:", "" ] }, "_LEGACY_ERROR_TEMP_DELTA_0009" : { "message" : [ "Updating nested fields is only supported for StructType." ] }, "_LEGACY_ERROR_TEMP_DELTA_0010" : { "message" : [ "Found unsupported expression while parsing target column name parts." ] }, "_LEGACY_ERROR_TEMP_DELTA_0012" : { "message" : [ "Could not resolve expression: " ] } } ================================================ FILE: spark/src/main/resources/org/apache/spark/SparkLayout.json ================================================ { "ts": { "$resolver": "timestamp", "pattern": { "format": "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", "timeZone": "UTC", "locale": "en_US" } }, "level": { "$resolver": "level", "field": "name" }, "msg": { "$resolver": "message", "stringified": true }, "context": { "$resolver": "mdc" }, "exception": { "class": { "$resolver": "exception", "field": "className" }, "msg": { "$resolver": "exception", "field": "message", "stringified": true }, "stacktrace": { "$resolver": "exception", "field": "stackTrace", "stackTrace": { "elementTemplate": { "class": { "$resolver": "stackTraceElement", "field": "className" }, "method": { "$resolver": "stackTraceElement", "field": "methodName" }, "file": { "$resolver": "stackTraceElement", "field": "fileName" }, "line": { "$resolver": "stackTraceElement", "field": "lineNumber" } } } } }, "logger": { "$resolver": "pattern", "pattern": "%c{1}", "stackTraceEnabled": false } } ================================================ FILE: spark/src/main/scala/com/databricks/spark/util/DatabricksLogging.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.databricks.spark.util import scala.collection.mutable.ArrayBuffer /** * This file contains stub implementation for logging that exists in Databricks. */ /** Used to return a recorded usage record for testing. */ case class UsageRecord( metric: String, quantity: Double, blob: String, tags: Map[String, String] = Map.empty, opType: Option[OpType] = None, opTarget: Option[String] = None) class TagDefinition(val name: String) { def this() = this("BACKWARD COMPATIBILITY") } object TagDefinitions { object TAG_TAHOE_PATH extends TagDefinition("tahoePath") object TAG_TAHOE_ID extends TagDefinition("tahoeId") object TAG_ASYNC extends TagDefinition("async") object TAG_LOG_STORE_CLASS extends TagDefinition("logStore") object TAG_OP_TYPE extends TagDefinition("opType") } case class OpType(typeName: String, description: String) class MetricDefinition(val name: String) { def this() = this("BACKWARD COMPATIBILITY") } object MetricDefinitions { object EVENT_LOGGING_FAILURE extends MetricDefinition("loggingFailureEvent") object EVENT_TAHOE extends MetricDefinition("tahoeEvent") with CentralizableMetric val METRIC_OPERATION_DURATION = new MetricDefinition("sparkOperationDuration") with CentralizableMetric } object Log4jUsageLogger { @volatile var usageTracker: ArrayBuffer[UsageRecord] = null /** * Records and returns all usage logs that are emitted while running the given function. * Intended for testing metrics that we expect to report. Note that this class does not * support nested invocations of the tracker. */ def track(f: => Unit): Seq[UsageRecord] = { synchronized { assert(usageTracker == null, "Usage tracking does not support nested invocation.") usageTracker = new ArrayBuffer[UsageRecord]() } var records: ArrayBuffer[UsageRecord] = null try { f } finally { records = usageTracker synchronized { usageTracker = null } } records.toSeq } } trait DatabricksLogging { import MetricDefinitions._ // scalastyle:off println def logConsole(line: String): Unit = println(line) // scalastyle:on println def recordUsage( metric: MetricDefinition, quantity: Double, additionalTags: Map[TagDefinition, String] = Map.empty, blob: String = null, forceSample: Boolean = false, trimBlob: Boolean = true, silent: Boolean = false): Unit = { Log4jUsageLogger.synchronized { if (Log4jUsageLogger.usageTracker != null) { val record = UsageRecord(metric.name, quantity, blob, additionalTags.map(kv => (kv._1.name, kv._2))) Log4jUsageLogger.usageTracker.append(record) } } } def recordEvent( metric: MetricDefinition, additionalTags: Map[TagDefinition, String] = Map.empty, blob: String = null, trimBlob: Boolean = true): Unit = { recordUsage(metric, 1, additionalTags, blob, trimBlob) } def recordOperation[S]( opType: OpType, opTarget: String = null, extraTags: Map[TagDefinition, String], isSynchronous: Boolean = true, alwaysRecordStats: Boolean = false, allowAuthTags: Boolean = false, killJvmIfStuck: Boolean = false, outputMetric: MetricDefinition = METRIC_OPERATION_DURATION, silent: Boolean = true)(thunk: => S): S = { try { thunk } finally { Log4jUsageLogger.synchronized { if (Log4jUsageLogger.usageTracker != null) { val record = UsageRecord(outputMetric.name, 0, null, extraTags.map(kv => (kv._1.name, kv._2)), Some(opType), Some(opTarget)) Log4jUsageLogger.usageTracker.append(record) } } } } def recordProductUsage( metric: MetricDefinition with CentralizableMetric, quantity: Double, additionalTags: Map[TagDefinition, String] = Map.empty, blob: String = null, forceSample: Boolean = false, trimBlob: Boolean = true, silent: Boolean = false): Unit = { Log4jUsageLogger.synchronized { if (Log4jUsageLogger.usageTracker != null) { val record = UsageRecord(metric.name, quantity, blob, additionalTags.map(kv => (kv._1.name, kv._2))) Log4jUsageLogger.usageTracker.append(record) } } } def recordProductEvent( metric: MetricDefinition with CentralizableMetric, additionalTags: Map[TagDefinition, String] = Map.empty, blob: String = null, trimBlob: Boolean = true): Unit = { recordProductUsage(metric, 1, additionalTags, blob, trimBlob) } } trait CentralizableMetric ================================================ FILE: spark/src/main/scala/io/delta/exceptions/DeltaConcurrentExceptions.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.exceptions import org.apache.spark.sql.delta.{DeltaThrowable, DeltaThrowableHelper} import org.apache.spark.annotation.Evolving /** * :: Evolving :: * * The basic class for all Delta commit conflict exceptions. * * @since 1.0.0 */ @Evolving abstract class DeltaConcurrentModificationException(message: String) extends org.apache.spark.sql.delta.DeltaConcurrentModificationException(message) /** * :: Evolving :: * * Thrown when a concurrent transaction has written data after the current transaction read the * table. * * @since 1.0.0 */ @Evolving class ConcurrentWriteException(message: String) extends org.apache.spark.sql.delta.ConcurrentWriteException(message) with DeltaThrowable { def this(messageParameters: Array[String]) = { this(DeltaThrowableHelper.getMessage("DELTA_CONCURRENT_WRITE", messageParameters)) } override def getErrorClass: String = "DELTA_CONCURRENT_WRITE" override def getMessage: String = message } /** * :: Evolving :: * * Thrown when the metadata of the Delta table has changed between the time of read * and the time of commit. * * @since 1.0.0 */ @Evolving class MetadataChangedException(message: String) extends org.apache.spark.sql.delta.MetadataChangedException(message) with DeltaThrowable { def this(messageParameters: Array[String]) = { this(DeltaThrowableHelper.getMessage("DELTA_METADATA_CHANGED", messageParameters)) } override def getErrorClass: String = "DELTA_METADATA_CHANGED" override def getMessage: String = message } /** * :: Evolving :: * * Thrown when the protocol version has changed between the time of read * and the time of commit. * * @since 1.0.0 */ @Evolving class ProtocolChangedException(message: String) extends org.apache.spark.sql.delta.ProtocolChangedException(message) with DeltaThrowable { def this(messageParameters: Array[String]) = { this(DeltaThrowableHelper.getMessage("DELTA_PROTOCOL_CHANGED", messageParameters)) } override def getErrorClass: String = "DELTA_PROTOCOL_CHANGED" override def getMessage: String = message } /** * :: Evolving :: * * Thrown when files are added that would have been read by the current transaction. * * @since 1.0.0 */ @Evolving class ConcurrentAppendException private ( errorClass: String, message: String, messageParameters: Array[String] = Array.empty) extends org.apache.spark.sql.delta.ConcurrentAppendException(message) with DeltaThrowable { def this(message: String) = this(message, "DELTA_CONCURRENT_APPEND.WITHOUT_HINT", Array.empty) def this(messageParameters: Array[String]) = { this( "DELTA_CONCURRENT_APPEND.WITHOUT_HINT", DeltaThrowableHelper.getMessage("DELTA_CONCURRENT_APPEND.WITHOUT_HINT", messageParameters), messageParameters ) } override def getErrorClass: String = errorClass override def getMessage: String = message def getMessageParametersArray: Array[String] = messageParameters override def getMessageParameters: java.util.Map[String, String] = { DeltaThrowableHelper.getMessageParameters(errorClass, errorSubClass = null, messageParameters) } } object ConcurrentAppendException { def apply(subClass: String, messageParameters: Array[String]): ConcurrentAppendException = { val errorClass = s"DELTA_CONCURRENT_APPEND.$subClass" val message = DeltaThrowableHelper.getMessage(errorClass, messageParameters) new ConcurrentAppendException(errorClass, message, messageParameters) } } /** * :: Evolving :: * * Thrown when the current transaction reads data that was deleted by a concurrent transaction. * * @since 1.0.0 */ @Evolving class ConcurrentDeleteReadException private ( message: String, errorClass: String, messageParameters: Array[String] = Array.empty) extends org.apache.spark.sql.delta.ConcurrentDeleteReadException(message) with DeltaThrowable { def this(message: String) = this(message, "DELTA_CONCURRENT_DELETE_READ.WITHOUT_HINT", Array.empty) def this(messageParameters: Array[String]) = { this(DeltaThrowableHelper.getMessage( "DELTA_CONCURRENT_DELETE_READ.WITHOUT_HINT", messageParameters), "DELTA_CONCURRENT_DELETE_READ.WITHOUT_HINT", messageParameters ) } override def getErrorClass: String = errorClass override def getMessage: String = message override def getMessageParameters: java.util.Map[String, String] = { DeltaThrowableHelper.getMessageParameters(errorClass, errorSubClass = null, messageParameters) } } object ConcurrentDeleteReadException { def apply(subClass: String, messageParameters: Array[String]): ConcurrentDeleteReadException = { val errorClass = s"DELTA_CONCURRENT_DELETE_READ.$subClass" val message = DeltaThrowableHelper.getMessage(errorClass, messageParameters) new ConcurrentDeleteReadException(message, errorClass, messageParameters) } } /** * :: Evolving :: * * Thrown when the current transaction deletes data that was deleted by a concurrent transaction. * * @since 1.0.0 */ @Evolving class ConcurrentDeleteDeleteException private ( message: String, errorClass: String, messageParameters: Array[String] = Array.empty) extends org.apache.spark.sql.delta.ConcurrentDeleteDeleteException(message) with DeltaThrowable { def this(message: String) = this(message, "DELTA_CONCURRENT_DELETE_DELETE.WITHOUT_HINT", Array.empty) def this(messageParameters: Array[String]) = { this(DeltaThrowableHelper.getMessage( "DELTA_CONCURRENT_DELETE_DELETE.WITHOUT_HINT", messageParameters), "DELTA_CONCURRENT_DELETE_DELETE.WITHOUT_HINT", messageParameters ) } override def getErrorClass: String = errorClass override def getMessage: String = message override def getMessageParameters: java.util.Map[String, String] = { DeltaThrowableHelper.getMessageParameters(errorClass, errorSubClass = null, messageParameters) } } object ConcurrentDeleteDeleteException { def apply(subClass: String, messageParameters: Array[String]): ConcurrentDeleteDeleteException = { val errorClass = s"DELTA_CONCURRENT_DELETE_DELETE.$subClass" val message = DeltaThrowableHelper.getMessage(errorClass, messageParameters) new ConcurrentDeleteDeleteException(message, errorClass, messageParameters) } } /** * :: Evolving :: * * Thrown when concurrent transaction both attempt to update the same idempotent transaction. * * @since 1.0.0 */ @Evolving class ConcurrentTransactionException(message: String) extends org.apache.spark.sql.delta.ConcurrentTransactionException(message) with DeltaThrowable { def this(messageParameters: Array[String]) = { this(DeltaThrowableHelper.getMessage("DELTA_CONCURRENT_TRANSACTION", messageParameters)) } override def getErrorClass: String = "DELTA_CONCURRENT_TRANSACTION" override def getMessage: String = message } ================================================ FILE: spark/src/main/scala/io/delta/implicits/package.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta import org.apache.spark.sql.{DataFrame, DataFrameReader, DataFrameWriter} import org.apache.spark.sql.streaming.{DataStreamReader, DataStreamWriter, StreamingQuery} package object implicits { /** * Extends the DataFrameReader API by adding a delta function * Usage: * {{{ * spark.read.delta(path) * }}} */ implicit class DeltaDataFrameReader(val reader: DataFrameReader) extends AnyVal { def delta(path: String): DataFrame = { reader.format("delta").load(path) } } /** * Extends the DataStreamReader API by adding a delta function * Usage: * {{{ * spark.readStream.delta(path) * }}} */ implicit class DeltaDataStreamReader(val dataStreamReader: DataStreamReader) extends AnyVal { def delta(path: String): DataFrame = { dataStreamReader.format("delta").load(path) } } /** * Extends the DataFrameWriter API by adding a delta function * Usage: * {{{ * df.write.delta(path) * }}} */ implicit class DeltaDataFrameWriter[T](val dfWriter: DataFrameWriter[T]) extends AnyVal { def delta(output: String): Unit = { dfWriter.format("delta").save(output) } } /** * Extends the DataStreamWriter API by adding a delta function * Usage: * {{{ * ds.writeStream.delta(path) * }}} */ implicit class DeltaDataStreamWriter[T] (val dataStreamWriter: DataStreamWriter[T]) extends AnyVal { def delta(path: String): StreamingQuery = { dataStreamWriter.format("delta").start(path) } } } ================================================ FILE: spark/src/main/scala/io/delta/sql/AbstractDeltaSparkSessionExtension.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sql import scala.util.control.NonFatal import org.apache.spark.sql.delta.metric.OptimizeConditionalIncrementMetric import org.apache.spark.sql.delta.optimizer.RangePartitionIdRewrite import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.PrepareDeltaScan import io.delta.sql.parser.DeltaSqlParser import org.apache.spark.sql.SparkSessionExtensions import org.apache.spark.sql.catalyst.optimizer.ConstantFolding import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.delta.PreprocessTimeTravel import org.apache.spark.sql.internal.SQLConf /** * V1 legacy implementation. Use [[io.delta.sql.DeltaSparkSessionExtension]] instead. * See spark-unified/src/main/scala/io/delta/sql/DeltaSparkSessionExtension.scala */ class DeltaSparkSessionExtensionV1 extends AbstractDeltaSparkSessionExtension /** * Abstract base class that contains the base Delta Spark Session extension logic. * As part of evolution for Delta connector(e.g. Dsv2), new Spark session extension * will be built based on that. */ class AbstractDeltaSparkSessionExtension extends (SparkSessionExtensions => Unit) { override def apply(extensions: SparkSessionExtensions): Unit = { extensions.injectParser { (_, parser) => new DeltaSqlParser(parser) } extensions.injectResolutionRule { session => ResolveDeltaPathTable(session) } extensions.injectResolutionRule { session => PreprocessTimeTravel(session) } extensions.injectResolutionRule { session => // To ensure the parquet field id reader is turned on, these fields are required to support // id column mapping mode for Delta. // Spark has the read flag default off, so we have to turn it on manually for Delta. session.sessionState.conf.setConf(SQLConf.PARQUET_FIELD_ID_READ_ENABLED, true) session.sessionState.conf.setConf(SQLConf.PARQUET_FIELD_ID_WRITE_ENABLED, true) new DeltaAnalysis(session) } // [SPARK-45383] Spark CheckAnalysis rule misses a case for RelationTimeTravel, and so a // non-existent table throws an internal spark error instead of the expected AnalysisException. extensions.injectCheckRule { session => new CheckUnresolvedRelationTimeTravel(session) } extensions.injectCheckRule { session => DeltaUnsupportedOperationsCheck(session) } // Rule for rewriting the place holder for range_partition_id to manually construct the // `RangePartitioner` (which requires an RDD to be sampled in order to determine // range partition boundaries) extensions.injectOptimizerRule { session => new RangePartitionIdRewrite(session) } // Optimize ConditionalIncrementMetric with constant condition. extensions.injectOptimizerRule { _ => OptimizeConditionalIncrementMetric } extensions.injectPostHocResolutionRule { session => PreprocessTableUpdate(session.sessionState.conf) } extensions.injectPostHocResolutionRule { session => PreprocessTableMerge(session.sessionState.conf) } extensions.injectPostHocResolutionRule { session => PreprocessTableDelete(session.sessionState.conf) } // Resolve new UpCast expressions that might have been introduced by [[PreprocessTableUpdate]] // and [[PreprocessTableMerge]]. extensions.injectPostHocResolutionRule { session => PostHocResolveUpCast(session) } extensions.injectPlanNormalizationRule { _ => GenerateRowIDs } extensions.injectPreCBORule { session => new PrepareDeltaScan(session) } // Fold constants that may have been introduced by PrepareDeltaScan. This is only useful with // Spark 3.5 as later versions apply constant folding after pre-CBO rules. extensions.injectPreCBORule { _ => ConstantFolding } // Add skip row column and filter. extensions.injectPlannerStrategy(PreprocessTableWithDVsStrategy) // Tries to load PrepareDeltaSharingScan class with class reflection, when delta-sharing-spark // 3.1+ package is installed, this will be loaded and delta sharing batch queries with // DeltaSharingFileIndex will be handled by the rule. // When the package is not installed or upon any other issues, it should do nothing and not // affect all the existing rules. try { // scalastyle:off classforname val constructor = Class.forName("io.delta.sharing.spark.PrepareDeltaSharingScan") .getConstructor(classOf[org.apache.spark.sql.SparkSession]) // scalastyle:on classforname extensions.injectPreCBORule { session => try { // Inject the PrepareDeltaSharingScan rule if enabled, otherwise, inject the no op // rule. It can be disabled if there are any issues so all existing rules are not blocked. if ( session.conf.get(DeltaSQLConf.DELTA_SHARING_ENABLE_DELTA_FORMAT_BATCH.key) == "true" ) { constructor.newInstance(session).asInstanceOf[Rule[LogicalPlan]] } else { new NoOpRule } } catch { // Inject a no op rule which doesn't apply any changes to the logical plan. case NonFatal(_) => new NoOpRule } } } catch { case NonFatal(_) => // Do nothing } DeltaTableValueFunctions.supportedFnNames.foreach { fnName => extensions.injectTableFunction( DeltaTableValueFunctions.getTableValueFunctionInjection(fnName)) } } /** * An no op rule which doesn't apply any changes to the LogicalPlan. Used to be injected upon * exceptions. */ class NoOpRule extends Rule[LogicalPlan] { override def apply(plan: LogicalPlan): LogicalPlan = plan } } ================================================ FILE: spark/src/main/scala/io/delta/sql/parser/DeltaSqlParser.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sql.parser import java.util.Locale import scala.collection.JavaConverters._ import org.apache.spark.sql.catalyst.TimeTravel import org.apache.spark.sql.delta.skipping.clustering.temp.{AlterTableClusterBy, ClusterByParserUtils, ClusterByPlan, ClusterBySpec} import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.commands._ import io.delta.sql.parser.DeltaSqlBaseParser._ import io.delta.tables.execution.VacuumTableCommand import org.antlr.v4.runtime._ import org.antlr.v4.runtime.atn.PredictionMode import org.antlr.v4.runtime.misc.{Interval, ParseCancellationException} import org.antlr.v4.runtime.tree._ import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.spark.sql.catalyst.expressions.{Expression, Literal} import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} import org.apache.spark.sql.catalyst.analysis._ import org.apache.spark.sql.catalyst.parser.{DeltaParseException, ParseErrorListener, ParseException, ParseExceptionShims, ParserInterface} import org.apache.spark.sql.catalyst.parser.ParserUtils.{checkDuplicateClauses, string, withOrigin} import org.apache.spark.sql.catalyst.plans.logical.{AlterColumnSyncIdentity, AlterTableAddConstraint, AlterTableDropConstraint, AlterTableDropFeature, CloneTableStatement, LogicalPlan, RestoreTableStatement} import org.apache.spark.sql.catalyst.trees.Origin import org.apache.spark.sql.connector.catalog.{CatalogV2Util, TableCatalog} import org.apache.spark.sql.errors.QueryParsingErrors import org.apache.spark.sql.internal.{SQLConf, VariableSubstitution} import org.apache.spark.sql.types._ /** * A SQL parser that tries to parse Delta commands. If failing to parse the SQL text, it will * forward the call to `delegate`. */ class DeltaSqlParser(val delegate: ParserInterface) extends ParserInterface { private val builder = new DeltaSqlAstBuilder private val substitution = new VariableSubstitution override def parsePlan(sqlText: String): LogicalPlan = parse(sqlText) { parser => builder.visit(parser.singleStatement()) match { case clusterByPlan: ClusterByPlan => ClusterByParserUtils(clusterByPlan, delegate).parsePlan(sqlText) case plan: LogicalPlan => plan case _ => delegate.parsePlan(sqlText) } } /** * This API is used just for parsing the SELECT queries. Delta parser doesn't override * the Spark parser, that means this can be delegated directly to the Spark parser. */ override def parseQuery(sqlText: String): LogicalPlan = delegate.parseQuery(sqlText) // scalastyle:off line.size.limit /** * Fork from `org.apache.spark.sql.catalyst.parser.AbstractSqlParser#parse(java.lang.String, scala.Function1)`. * * @see https://github.com/apache/spark/blob/v2.4.4/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala#L81 */ // scalastyle:on protected def parse[T](command: String)(toResult: DeltaSqlBaseParser => T): T = { val lexer = new DeltaSqlBaseLexer( new UpperCaseCharStream(CharStreams.fromString(substitution.substitute(command)))) lexer.removeErrorListeners() lexer.addErrorListener(ParseErrorListener) val tokenStream = new CommonTokenStream(lexer) val parser = new DeltaSqlBaseParser(tokenStream) parser.addParseListener(PostProcessor) parser.removeErrorListeners() parser.addErrorListener(ParseErrorListener) try { try { // first, try parsing with potentially faster SLL mode parser.getInterpreter.setPredictionMode(PredictionMode.SLL) toResult(parser) } catch { case e: ParseCancellationException => // if we fail, parse with LL mode tokenStream.seek(0) // rewind input stream parser.reset() // Try Again. parser.getInterpreter.setPredictionMode(PredictionMode.LL) toResult(parser) } } catch { case e: ParseException if e.command.isDefined => throw e case e: ParseException => throw e.withCommand(command) case e: AnalysisException => val position = Origin(e.line, e.startPosition) throw ParseExceptionShims.createParseException( command = Option(command), start = position, stop = position, errorClass = "DELTA_PARSING_ANALYSIS_ERROR", messageParameters = Map("msg" -> e.message)) } } override def parseExpression(sqlText: String): Expression = delegate.parseExpression(sqlText) override def parseTableIdentifier(sqlText: String): TableIdentifier = delegate.parseTableIdentifier(sqlText) override def parseFunctionIdentifier(sqlText: String): FunctionIdentifier = delegate.parseFunctionIdentifier(sqlText) override def parseMultipartIdentifier (sqlText: String): Seq[String] = delegate.parseMultipartIdentifier(sqlText) override def parseTableSchema(sqlText: String): StructType = delegate.parseTableSchema(sqlText) override def parseDataType(sqlText: String): DataType = delegate.parseDataType(sqlText) override def parseRoutineParam(sqlText: String): StructType = delegate.parseRoutineParam(sqlText) } /** * Define how to convert an AST generated from `DeltaSqlBase.g4` to a `LogicalPlan`. The parent * class `DeltaSqlBaseBaseVisitor` defines all visitXXX methods generated from `#` instructions in * `DeltaSqlBase.g4` (such as `#vacuumTable`). */ class DeltaSqlAstBuilder extends DeltaSqlBaseBaseVisitor[AnyRef] { import org.apache.spark.sql.catalyst.parser.ParserUtils._ /** * Convert a property list into a key-value map. * This should be called through [[visitPropertyKeyValues]] or [[visitPropertyKeys]]. */ override def visitPropertyList( ctx: PropertyListContext): Map[String, String] = withOrigin(ctx) { val properties = ctx.property.asScala.map { property => val key = visitPropertyKey(property.key) val value = visitPropertyValue(property.value) key -> value } // Check for duplicate property names. checkDuplicateKeys(properties.toSeq, ctx) properties.toMap } /** * Parse a key-value map from a [[PropertyListContext]], assuming all values are specified. */ def visitPropertyKeyValues(ctx: PropertyListContext): Map[String, String] = { val props = visitPropertyList(ctx) val badKeys = props.collect { case (key, null) => key } if (badKeys.nonEmpty) { operationNotAllowed( s"Values must be specified for key(s): ${badKeys.mkString("[", ",", "]")}", ctx) } props } /** * Parse a list of keys from a [[PropertyListContext]], assuming no values are specified. */ def visitPropertyKeys(ctx: PropertyListContext): Seq[String] = { val props = visitPropertyList(ctx) val badKeys = props.filter { case (_, v) => v != null }.keys if (badKeys.nonEmpty) { operationNotAllowed( s"Values should not be specified for key(s): ${badKeys.mkString("[", ",", "]")}", ctx) } props.keys.toSeq } /** * A property key can either be String or a collection of dot separated elements. This * function extracts the property key based on whether its a string literal or a property * identifier. */ override def visitPropertyKey(key: PropertyKeyContext): String = { if (key.stringLit() != null) { visitStringLit(key.stringLit()) } else { key.getText } } /** * A property value can be String, Integer, Boolean or Decimal. This function extracts * the property value based on whether its a string, integer, boolean or decimal literal. */ override def visitPropertyValue(value: PropertyValueContext): String = { if (value == null) { null } else if (value.identifier != null) { value.identifier.getText } else if (value.value != null) { visitStringLit(value.value) } else if (value.booleanValue != null) { value.getText.toLowerCase(Locale.ROOT) } else { value.getText } } override def visitStringLit(ctx: StringLitContext): String = { if (ctx == null) return null ctx.singleStringLit().asScala.map { singleCtx => val token = if (singleCtx.STRING != null) { singleCtx.STRING.getSymbol } else { singleCtx.DOUBLEQUOTED_STRING.getSymbol } string(token) }.mkString } /** * Parse either create table header or replace table header. * @return TableIdentifier for the target table * Boolean for whether we are creating a table * Boolean for whether we are replacing a table * Boolean for whether we are creating a table if not exists */ override def visitCloneTableHeader( ctx: CloneTableHeaderContext): (TableIdentifier, Boolean, Boolean, Boolean) = withOrigin(ctx) { ctx.children.asScala.head match { case createHeader: CreateTableHeaderContext => (visitTableIdentifier(createHeader.table), true, false, createHeader.EXISTS() != null) case replaceHeader: ReplaceTableHeaderContext => (visitTableIdentifier(replaceHeader.table), replaceHeader.CREATE() != null, true, false) case _ => throw new DeltaParseException(ctx, "DELTA_PARSING_INCORRECT_CLONE_HEADER") } } /** * Creates a [[CloneTableStatement]] logical plan. Example SQL: * {{{ * CREATE [OR REPLACE] TABLE SHALLOW CLONE * [TBLPROPERTIES ('propA' = 'valueA', ...)] * [LOCATION '/path/to/cloned/table'] * }}} */ override def visitClone(ctx: CloneContext): LogicalPlan = withOrigin(ctx) { val (target, isCreate, isReplace, ifNotExists) = visitCloneTableHeader(ctx.cloneTableHeader()) if (!isCreate && ifNotExists) { throw new DeltaParseException( ctx.cloneTableHeader(), "DELTA_PARSING_MUTUALLY_EXCLUSIVE_CLAUSES", Map("clauseOne" -> "IF NOT EXISTS", "clauseTwo" -> "REPLACE") ) } // Get source for clone (and time travel source if necessary) // The source relation can be an Iceberg table in form of `catalog.db.table` so we visit // a multipart identifier instead of TableIdentifier (which does not support 3L namespace) // in Spark 3.3. In Spark 3.4 we should have TableIdentifier supporting 3L namespace so we // could revert back to that. val sourceRelation = new UnresolvedRelation(visitMultipartIdentifier(ctx.source)) val maybeTimeTravelSource = maybeTimeTravelChild(ctx.clause, sourceRelation) val targetRelation = UnresolvedRelation(target.nameParts) val tablePropertyOverrides = Option(ctx.tableProps) .map(visitPropertyKeyValues) .getOrElse(Map.empty[String, String]) CloneTableStatement( maybeTimeTravelSource, targetRelation, ifNotExists, isReplace, isCreate, tablePropertyOverrides, Option(ctx.location).map(visitStringLit)) } /** * Create a [[VacuumTableCommand]] logical plan. Example SQL: * {{{ * VACUUM ('/path/to/dir' | delta.`/path/to/dir`) * LITE|FULL * [RETAIN number HOURS] [DRY RUN]; * }}} */ override def visitVacuumTable(ctx: VacuumTableContext): AnyRef = withOrigin(ctx) { val vacuumModifiersCtx = ctx.vacuumModifiers() withOrigin(vacuumModifiersCtx) { checkDuplicateClauses(vacuumModifiersCtx.vacuumType(), "LITE/FULL", vacuumModifiersCtx) checkDuplicateClauses(vacuumModifiersCtx.inventory(), "INVENTORY", vacuumModifiersCtx) checkDuplicateClauses(vacuumModifiersCtx.retain(), "RETAIN", vacuumModifiersCtx) checkDuplicateClauses(vacuumModifiersCtx.dryRun(), "DRY RUN", vacuumModifiersCtx) if (!vacuumModifiersCtx.inventory().isEmpty && !vacuumModifiersCtx.vacuumType().isEmpty && vacuumModifiersCtx.vacuumType().asScala.head.LITE != null) { operationNotAllowed("Inventory option is not compatible with LITE", vacuumModifiersCtx) } } VacuumTableCommand( path = Option(ctx.path).map(visitStringLit), table = Option(ctx.table).map(visitTableIdentifier), inventoryTable = ctx.vacuumModifiers().inventory().asScala.headOption.collect { case i if i.inventoryTable != null => visitTableIdentifier(i.inventoryTable) }, inventoryQuery = ctx.vacuumModifiers().inventory().asScala.headOption.collect { case i if i.inventoryQuery != null => extractRawText(i.inventoryQuery) }, horizonHours = ctx.vacuumModifiers().retain().asScala.headOption.map(_.number.getText.toDouble), dryRun = ctx.vacuumModifiers().dryRun().asScala.headOption.exists(_.RUN != null), vacuumType = ctx.vacuumModifiers().vacuumType().asScala.headOption.map { t => if (t.LITE != null) "LITE" else "FULL" }, options = Map.empty ) } /** Provides a list of unresolved attributes for multi dimensional clustering. */ override def visitZorderSpec(ctx: ZorderSpecContext): Seq[UnresolvedAttribute] = { ctx.interleave.asScala .map(_.identifier.asScala.map(_.getText).toSeq) .map(new UnresolvedAttribute(_)).toSeq } /** * Create a [[OptimizeTableCommand]] logical plan. * Syntax: * {{{ * OPTIMIZE * [WHERE predicate-using-partition-columns] * [ZORDER BY [(] col1, col2 ..[)]] * }}} * Examples: * {{{ * OPTIMIZE '/path/to/delta/table'; * OPTIMIZE delta_table_name; * OPTIMIZE delta.`/path/to/delta/table`; * OPTIMIZE delta_table_name WHERE partCol = 25; * OPTIMIZE delta_table_name WHERE partCol = 25 ZORDER BY col2, col2; * }}} */ override def visitOptimizeTable(ctx: OptimizeTableContext): AnyRef = withOrigin(ctx) { if (ctx.path == null && ctx.table == null) { throw new DeltaParseException( ctx, "DELTA_PARSING_MISSING_TABLE_NAME_OR_PATH", Map("command" -> "OPTIMIZE") ) } val interleaveBy = Option(ctx.zorderSpec).map(visitZorderSpec).getOrElse(Seq.empty) OptimizeTableCommand( Option(ctx.path).map(visitStringLit), Option(ctx.table).map(visitTableIdentifier), Option(ctx.partitionPredicate).map(extractRawText(_)).toSeq, DeltaOptimizeContext(isFull = ctx.FULL != null))(interleaveBy) } /** * Creates a [[DeltaReorgTable]] logical plan. * Examples: * {{{ * -- Physically delete dropped rows and columns of target table * REORG TABLE (delta.`/path/to/table` | delta_table_name) * [WHERE partition_predicate] APPLY (PURGE) * * -- Rewrite the files in UNIFORM(ICEBERG) compliant way. * REORG TABLE table_name (delta.`/path/to/table` | catalog.db.table) * APPLY (UPGRADE UNIFORM(ICEBERG_COMPAT_VERSION=version)) * }}} */ override def visitReorgTable(ctx: ReorgTableContext): AnyRef = withOrigin(ctx) { if (ctx.table == null) { throw new DeltaParseException( ctx, "DELTA_PARSING_MISSING_TABLE_NAME_OR_PATH", Map("command" -> "REORG") ) } val targetIdentifier = visitTableIdentifier(ctx.table) val targetTable = UnresolvedTable(targetIdentifier.nameParts, "REORG") val reorgTableSpec = if (ctx.PURGE != null) { DeltaReorgTableSpec(DeltaReorgTableMode.PURGE, None) } else if (ctx.ICEBERG_COMPAT_VERSION != null) { DeltaReorgTableSpec(DeltaReorgTableMode.UNIFORM_ICEBERG, Option(ctx.version).map(_.getText.toInt)) } else { throw new ParseException( "Invalid syntax: REORG TABLE only support PURGE/UPGRADE UNIFORM.", ctx) } DeltaReorgTable(targetTable, reorgTableSpec)(Option(ctx.partitionPredicate).map(extractRawText(_)).toSeq) } override def visitDescribeDeltaDetail( ctx: DescribeDeltaDetailContext): LogicalPlan = withOrigin(ctx) { DescribeDeltaDetailCommand( Option(ctx.path).map(visitStringLit), Option(ctx.table).map(visitTableIdentifier), Map.empty) } override def visitDescribeDeltaHistory( ctx: DescribeDeltaHistoryContext): LogicalPlan = withOrigin(ctx) { DescribeDeltaHistory( Option(ctx.path).map(visitStringLit), Option(ctx.table).map(visitTableIdentifier), Option(ctx.limit).map(_.getText.toInt)) } override def visitGenerate(ctx: GenerateContext): LogicalPlan = withOrigin(ctx) { DeltaGenerateCommand( UnresolvedTable(visitTableIdentifier(ctx.table).nameParts, DeltaGenerateCommand.COMMAND_NAME), modeName = ctx.modeName.getText) } override def visitConvert(ctx: ConvertContext): LogicalPlan = withOrigin(ctx) { ConvertToDeltaCommand( visitTableIdentifier(ctx.table), Option(ctx.colTypeList).map(colTypeList => StructType(visitColTypeList(colTypeList))), ctx.STATISTICS() == null, None) } override def visitRestore(ctx: RestoreContext): LogicalPlan = withOrigin(ctx) { val tableRelation = UnresolvedRelation(visitTableIdentifier(ctx.table).nameParts) val timeTravelTableRelation = maybeTimeTravelChild(ctx.clause, tableRelation) RestoreTableStatement(timeTravelTableRelation.asInstanceOf[TimeTravel]) } /** * Captures any CLUSTER BY clause and creates a [[ClusterByPlan]] logical plan. * The plan will be used as a sentinel for DeltaSqlParser to process it further. */ override def visitClusterBy(ctx: ClusterByContext): LogicalPlan = withOrigin(ctx) { val clusterBySpecCtx = ctx.clusterBySpec.asScala.head checkDuplicateClauses(ctx.clusterBySpec, "CLUSTER BY", clusterBySpecCtx) val columnNames = clusterBySpecCtx.interleave.asScala .map(_.identifier.asScala.map(_.getText).toSeq) .map(_.asInstanceOf[Seq[String]]).toSeq // get CLUSTER BY clause positions. val startIndex = clusterBySpecCtx.getStart.getStartIndex val stopIndex = clusterBySpecCtx.getStop.getStopIndex // get CLUSTER BY parenthesis positions. val parenStartIndex = clusterBySpecCtx.LEFT_PAREN().getSymbol.getStartIndex val parenStopIndex = clusterBySpecCtx.RIGHT_PAREN().getSymbol.getStopIndex ClusterByPlan( ClusterBySpec(columnNames), startIndex, stopIndex, parenStartIndex, parenStopIndex, clusterBySpecCtx) } /** * Time travel the table to the given version or timestamp. */ private def maybeTimeTravelChild(ctx: TemporalClauseContext, child: LogicalPlan): LogicalPlan = { if (ctx == null) return child TimeTravel( child, Option(ctx.timestamp).map(token => Literal(token.getText.replaceAll("^'|'$", ""))), Option(ctx.version).map(_.getText.toLong), Some("sql")) } override def visitSingleStatement(ctx: SingleStatementContext): LogicalPlan = withOrigin(ctx) { visit(ctx.statement).asInstanceOf[LogicalPlan] } protected def visitTableIdentifier(ctx: QualifiedNameContext): TableIdentifier = withOrigin(ctx) { ctx.identifier.asScala.toSeq match { case Seq(tbl) => TableIdentifier(tbl.getText) case Seq(db, tbl) => TableIdentifier(tbl.getText, Some(db.getText)) case Seq(catalog, db, tbl) => TableIdentifier(tbl.getText, Some(db.getText), Some(catalog.getText)) case _ => throw new DeltaParseException( ctx, "DELTA_PARSING_ILLEGAL_TABLE_NAME", Map("table" -> ctx.getText)) } } protected def visitMultipartIdentifier(ctx: QualifiedNameContext): Seq[String] = withOrigin(ctx) { ctx.identifier.asScala.map(_.getText).toSeq } override def visitPassThrough(ctx: PassThroughContext): LogicalPlan = null override def visitColTypeList(ctx: ColTypeListContext): Seq[StructField] = withOrigin(ctx) { ctx.colType().asScala.map(visitColType).toSeq } override def visitColType(ctx: ColTypeContext): StructField = withOrigin(ctx) { import ctx._ val builder = new MetadataBuilder StructField( ctx.colName.getText, typedVisit[DataType](ctx.dataType), nullable = NOT == null, builder.build()) } // Build the text of the CHECK constraint expression. The user-specified whitespace is in the // HIDDEN channel where we can't get to it, so we just paste together all the tokens with a single // space. This produces some strange spacing (e.g. `structCol . arr [ 0 ]`), but right now we // think that's preferable to the additional complexity involved in trying to produce cleaner // output. private def buildCheckConstraintText(tokens: Seq[ExprTokenContext]): String = { tokens.map(_.getText).mkString(" ") } private def extractRawText(exprContext: ParserRuleContext): String = { // Extract the raw expression which will be parsed later exprContext.getStart.getInputStream.getText(new Interval( exprContext.getStart.getStartIndex, exprContext.getStop.getStopIndex)) } override def visitAddTableConstraint( ctx: AddTableConstraintContext): LogicalPlan = withOrigin(ctx) { val checkConstraint = ctx.constraint().asInstanceOf[CheckConstraintContext] AlterTableAddConstraint( UnresolvedTable(ctx.table.identifier.asScala.map(_.getText).toSeq, "ALTER TABLE ... ADD CONSTRAINT"), ctx.name.getText, buildCheckConstraintText(checkConstraint.exprToken().asScala.toSeq)) } override def visitDropTableConstraint( ctx: DropTableConstraintContext): LogicalPlan = withOrigin(ctx) { AlterTableDropConstraint( UnresolvedTable(ctx.table.identifier.asScala.map(_.getText).toSeq, "ALTER TABLE ... DROP CONSTRAINT"), ctx.name.getText, ifExists = ctx.EXISTS != null) } /** * `ALTER TABLE ... ALTER (CHANGE) COLUMN ... SYNC IDENTITY` command. */ override def visitAlterTableSyncIdentity( ctx: AlterTableSyncIdentityContext): LogicalPlan = withOrigin(ctx) { val verb = if (ctx.CHANGE != null) "CHANGE" else "ALTER" AlterColumnSyncIdentity( UnresolvedTable(ctx.table.identifier.asScala.map(_.getText).toSeq, s"ALTER TABLE ... $verb COLUMN"), UnresolvedFieldName(visitMultipartIdentifier(ctx.column)) ) } /** * A featureNameValue can either be String or an identifier. This function extracts * the featureNameValue based on whether its a string literal or an identifier. */ override def visitFeatureNameValue(featureNameValue: FeatureNameValueContext): String = { if (featureNameValue.stringLit() != null) { visitStringLit(featureNameValue.stringLit()) } else { featureNameValue.getText } } /** * Parse an ALTER TABLE DROP FEATURE command. */ override def visitAlterTableDropFeature(ctx: AlterTableDropFeatureContext): LogicalPlan = { val truncateHistory = ctx.TRUNCATE != null && ctx.HISTORY != null AlterTableDropFeature( UnresolvedTable(ctx.table.identifier.asScala.map(_.getText).toSeq, "ALTER TABLE ... DROP FEATURE"), visitFeatureNameValue(ctx.featureName), truncateHistory) } /** * Parse an ALTER TABLE CLUSTER BY command. */ override def visitAlterTableClusterBy(ctx: AlterTableClusterByContext): LogicalPlan = { val table = UnresolvedTable(ctx.table.identifier.asScala.map(_.getText).toSeq, "ALTER TABLE ... CLUSTER BY") if (ctx.NONE() != null) { AlterTableClusterBy(table, None) } else { assert(ctx.clusterBySpec() != null) val columnNames = ctx.clusterBySpec().interleave.asScala .map(_.identifier.asScala.map(_.getText).toSeq) .map(_.asInstanceOf[Seq[String]]).toSeq AlterTableClusterBy(table, Some(ClusterBySpec(columnNames))) } } protected def typedVisit[T](ctx: ParseTree): T = { ctx.accept(this).asInstanceOf[T] } override def visitPrimitiveDataType(ctx: PrimitiveDataTypeContext): DataType = withOrigin(ctx) { val dataType = ctx.identifier.getText.toLowerCase(Locale.ROOT) (dataType, ctx.INTEGER_VALUE().asScala.toList) match { case ("boolean", Nil) => BooleanType case ("tinyint" | "byte", Nil) => ByteType case ("smallint" | "short", Nil) => ShortType case ("int" | "integer", Nil) => IntegerType case ("bigint" | "long", Nil) => LongType case ("float", Nil) => FloatType case ("double", Nil) => DoubleType case ("date", Nil) => DateType case ("timestamp", Nil) => TimestampType case ("string", Nil) => StringType case ("char", length :: Nil) => CharType(length.getText.toInt) case ("varchar", length :: Nil) => VarcharType(length.getText.toInt) case ("binary", Nil) => BinaryType case ("decimal", Nil) => DecimalType.USER_DEFAULT case ("decimal", precision :: Nil) => DecimalType(precision.getText.toInt, 0) case ("decimal", precision :: scale :: Nil) => DecimalType(precision.getText.toInt, scale.getText.toInt) case ("interval", Nil) => CalendarIntervalType case (dt, params) => val dtStr = if (params.nonEmpty) s"$dt(${params.mkString(",")})" else dt throw new DeltaParseException( ctx, "DELTA_PARSING_UNSUPPORTED_DATA_TYPE", Map("dataType" -> dtStr) ) } } } // scalastyle:off line.size.limit /** * Fork from `org.apache.spark.sql.catalyst.parser.UpperCaseCharStream`. * * @see https://github.com/apache/spark/blob/v2.4.4/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala#L157 */ // scalastyle:on class UpperCaseCharStream(wrapped: CodePointCharStream) extends CharStream { override def consume(): Unit = wrapped.consume override def getSourceName(): String = wrapped.getSourceName override def index(): Int = wrapped.index override def mark(): Int = wrapped.mark override def release(marker: Int): Unit = wrapped.release(marker) override def seek(where: Int): Unit = wrapped.seek(where) override def size(): Int = wrapped.size override def getText(interval: Interval): String = { // ANTLR 4.7's CodePointCharStream implementations have bugs when // getText() is called with an empty stream, or intervals where // the start > end. See // https://github.com/antlr/antlr4/commit/ac9f7530 for one fix // that is not yet in a released ANTLR artifact. if (size() > 0 && (interval.b - interval.a >= 0)) { wrapped.getText(interval) } else { "" } } override def LA(i: Int): Int = { val la = wrapped.LA(i) if (la == 0 || la == IntStream.EOF) la else Character.toUpperCase(la) } } // scalastyle:off line.size.limit /** * Fork from `org.apache.spark.sql.catalyst.parser.PostProcessor`. * * @see https://github.com/apache/spark/blob/v2.4.4/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala#L248 */ // scalastyle:on case object PostProcessor extends DeltaSqlBaseBaseListener { /** Remove the back ticks from an Identifier. */ override def exitQuotedIdentifier(ctx: QuotedIdentifierContext): Unit = { replaceTokenByIdentifier(ctx, 1) { token => // Remove the double back ticks in the string. token.setText(token.getText.replace("``", "`")) token } } /** Treat non-reserved keywords as Identifiers. */ override def exitNonReserved(ctx: NonReservedContext): Unit = { replaceTokenByIdentifier(ctx, 0)(identity) } private def replaceTokenByIdentifier( ctx: ParserRuleContext, stripMargins: Int)( f: CommonToken => CommonToken = identity): Unit = { val parent = ctx.getParent parent.removeLastChild() val token = ctx.getChild(0).getPayload.asInstanceOf[Token] val newToken = new CommonToken( new org.antlr.v4.runtime.misc.Pair(token.getTokenSource, token.getInputStream), DeltaSqlBaseParser.IDENTIFIER, token.getChannel, token.getStartIndex + stripMargins, token.getStopIndex - stripMargins) parent.addChild(new TerminalNodeImpl(f(newToken))) } } ================================================ FILE: spark/src/main/scala/io/delta/tables/DeltaColumnBuilder.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables import org.apache.spark.sql.delta.{DeltaErrors, IdentityColumn} import org.apache.spark.sql.delta.sources.DeltaSourceUtils.{GENERATION_EXPRESSION_METADATA_KEY, IDENTITY_INFO_ALLOW_EXPLICIT_INSERT, IDENTITY_INFO_START, IDENTITY_INFO_STEP} import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.spark.annotation._ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.types.{DataType, LongType, MetadataBuilder, StructField} /** * :: Evolving :: * * Builder to specify a table column. * * See [[DeltaTableBuilder]] for examples. * @since 1.0.0 */ @Evolving class DeltaColumnBuilder private[tables]( private val spark: SparkSession, private val colName: String) { private var dataType: DataType = _ private var nullable: Boolean = true private var generationExpr: Option[String] = None private var comment: Option[String] = None private var identityStart: Option[Long] = None private var identityStep: Option[Long] = None private var identityAllowExplicitInsert: Option[Boolean] = None /** * :: Evolving :: * * Specify the column data type. * * @param dataType string column data type * @since 1.0.0 */ @Evolving def dataType(dataType: String): DeltaColumnBuilder = { this.dataType = spark.sessionState.sqlParser.parseDataType(dataType) this } /** * :: Evolving :: * * Specify the column data type. * * @param dataType DataType column data type * @since 1.0.0 */ @Evolving def dataType(dataType: DataType): DeltaColumnBuilder = { this.dataType = dataType this } /** * :: Evolving :: * * Specify whether the column can be null. * * @param nullable boolean whether the column can be null or not. * @since 1.0.0 */ @Evolving def nullable(nullable: Boolean): DeltaColumnBuilder = { this.nullable = nullable this } /** * :: Evolving :: * * Specify a expression if the column is always generated as a function of other columns. * * @param expr string the the generation expression * @since 1.0.0 */ @Evolving def generatedAlwaysAs(expr: String): DeltaColumnBuilder = { this.generationExpr = Option(expr) this } /** * :: Evolving :: * * Specify a column as an identity column with default values that is always generated * by the system (i.e. does not allow user-specified values). * * @since 3.3.0 */ @Evolving def generatedAlwaysAsIdentity(): DeltaColumnBuilder = { generatedAlwaysAsIdentity(IdentityColumn.defaultStart, IdentityColumn.defaultStep) } /** * :: Evolving :: * * Specify a column as an identity column that is always generated by the system (i.e. does not * allow user-specified values). * * @param start the start value of the identity column * @param step the increment step of the identity column * @since 3.3.0 */ @Evolving def generatedAlwaysAsIdentity(start: Long, step: Long): DeltaColumnBuilder = { this.identityStart = Some(start) this.identityStep = Some(step) this.identityAllowExplicitInsert = Some(false) this } /** * :: Evolving :: * * Specify a column as an identity column that allows user-specified values such that the * generated values use default start and step values. * * @since 3.3.0 */ @Evolving def generatedByDefaultAsIdentity(): DeltaColumnBuilder = { generatedByDefaultAsIdentity(IdentityColumn.defaultStart, IdentityColumn.defaultStep) } /** * :: Evolving :: * * Specify a column as an identity column that allows user-specified values. * * @param start the start value of the identity column * @param step the increment step of the identity column * @since 3.3.0 */ @Evolving def generatedByDefaultAsIdentity(start: Long, step: Long): DeltaColumnBuilder = { this.identityStart = Some(start) this.identityStep = Some(step) this.identityAllowExplicitInsert = Some(true) this } /** * :: Evolving :: * * Specify a column comment. * * @param comment string column description * @since 1.0.0 */ @Evolving def comment(comment: String): DeltaColumnBuilder = { this.comment = Option(comment) this } /** * :: Evolving :: * * Build the column as a structField. * * @since 1.0.0 */ @Evolving def build(): StructField = { val metadataBuilder = new MetadataBuilder() if (generationExpr.nonEmpty) { metadataBuilder.putString(GENERATION_EXPRESSION_METADATA_KEY, generationExpr.get) } identityAllowExplicitInsert.ifDefined { allowExplicitInsert => if (generationExpr.nonEmpty) { throw DeltaErrors.identityColumnWithGenerationExpression() } if (dataType != null && dataType != LongType) { throw DeltaErrors.identityColumnDataTypeNotSupported(dataType) } metadataBuilder.putBoolean( IDENTITY_INFO_ALLOW_EXPLICIT_INSERT, allowExplicitInsert) metadataBuilder.putLong(IDENTITY_INFO_START, identityStart.get) val step = identityStep.get if (step == 0L) { throw DeltaErrors.identityColumnIllegalStep() } metadataBuilder.putLong(IDENTITY_INFO_STEP, identityStep.get) } if (comment.nonEmpty) { metadataBuilder.putString("comment", comment.get) } val fieldMetadata = metadataBuilder.build() if (dataType == null) { throw DeltaErrors.columnBuilderMissingDataType(colName) } StructField( colName, dataType, nullable = nullable, metadata = fieldMetadata) } } ================================================ FILE: spark/src/main/scala/io/delta/tables/DeltaMergeBuilder.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables import scala.collection.JavaConverters._ import scala.collection.Map import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.{DeltaAnalysisException, PostHocResolveUpCast, PreprocessTableMerge, ResolveDeltaMergeInto} import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.DeltaTableUtils.withActiveSession import org.apache.spark.sql.delta.DeltaViewHelper import org.apache.spark.sql.delta.util.AnalysisHelper import org.apache.spark.annotation._ import org.apache.spark.internal.Logging import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.ExtendedAnalysisException import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions.AttributeReference import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.functions.expr import org.apache.spark.sql.internal.SQLConf /** * Builder to specify how to merge data from source DataFrame into the target Delta table. * You can specify any number of `whenMatched` and `whenNotMatched` clauses. * Here are the constraints on these clauses. * * - `whenMatched` clauses: * * - The condition in a `whenMatched` clause is optional. However, if there are multiple * `whenMatched` clauses, then only the last one may omit the condition. * * - When there are more than one `whenMatched` clauses and there are conditions (or the lack * of) such that a row satisfies multiple clauses, then the action for the first clause * satisfied is executed. In other words, the order of the `whenMatched` clauses matters. * * - If none of the `whenMatched` clauses match a source-target row pair that satisfy * the merge condition, then the target rows will not be updated or deleted. * * - If you want to update all the columns of the target Delta table with the * corresponding column of the source DataFrame, then you can use the * `whenMatched(...).updateAll()`. This is equivalent to * 
 *         whenMatched(...).updateExpr(Map(
 *           ("col1", "source.col1"),
 *           ("col2", "source.col2"),
 *           ...))
 *
* * - `whenNotMatched` clauses: * * - The condition in a `whenNotMatched` clause is optional. However, if there are * multiple `whenNotMatched` clauses, then only the last one may omit the condition. * * - When there are more than one `whenNotMatched` clauses and there are conditions (or the * lack of) such that a row satisfies multiple clauses, then the action for the first clause * satisfied is executed. In other words, the order of the `whenNotMatched` clauses matters. * * - If no `whenNotMatched` clause is present or if it is present but the non-matching source * row does not satisfy the condition, then the source row is not inserted. * * - If you want to insert all the columns of the target Delta table with the * corresponding column of the source DataFrame, then you can use * `whenNotMatched(...).insertAll()`. This is equivalent to * 
 *         whenNotMatched(...).insertExpr(Map(
 *           ("col1", "source.col1"),
 *           ("col2", "source.col2"),
 *           ...))
 *
* * - `whenNotMatchedBySource` clauses: * * - The condition in a `whenNotMatchedBySource` clause is optional. However, if there are * multiple `whenNotMatchedBySource` clauses, then only the last one may omit the condition. * * - When there are more than one `whenNotMatchedBySource` clauses and there are conditions (or * the lack of) such that a row satisfies multiple clauses, then the action for the first * clause satisfied is executed. In other words, the order of the `whenNotMatchedBySource` * clauses matters. * * - If no `whenNotMatchedBySource` clause is present or if it is present but the * non-matching target row does not satisfy any of the `whenNotMatchedBySource` clause * condition, then the target row will not be updated or deleted. * * * Scala example to update a key-value Delta table with new key-values from a source DataFrame: * {{{ * deltaTable * .as("target") * .merge( * source.as("source"), * "target.key = source.key") * .withSchemaEvolution() * .whenMatched() * .updateExpr(Map( * "value" -> "source.value")) * .whenNotMatched() * .insertExpr(Map( * "key" -> "source.key", * "value" -> "source.value")) * .whenNotMatchedBySource() * .updateExpr(Map( * "value" -> "target.value + 1")) * .execute() * }}} * * Java example to update a key-value Delta table with new key-values from a source DataFrame: * {{{ * deltaTable * .as("target") * .merge( * source.as("source"), * "target.key = source.key") * .withSchemaEvolution() * .whenMatched() * .updateExpr( * new HashMap() {{ * put("value", "source.value"); * }}) * .whenNotMatched() * .insertExpr( * new HashMap() {{ * put("key", "source.key"); * put("value", "source.value"); * }}) * .whenNotMatchedBySource() * .updateExpr( * new HashMap() {{ * put("value", "target.value + 1"); * }}) * .execute(); * }}} * * @since 0.3.0 */ class DeltaMergeBuilder private( private val targetTable: DeltaTable, private val source: DataFrame, private val onCondition: Column, private val whenClauses: Seq[DeltaMergeIntoClause], private val schemaEvolutionEnabled: Boolean) extends AnalysisHelper with Logging { def this( targetTable: DeltaTable, source: DataFrame, onCondition: Column, whenClauses: Seq[DeltaMergeIntoClause]) = this(targetTable, source, onCondition, whenClauses, schemaEvolutionEnabled = false) /** * Build the actions to perform when the merge condition was matched. This returns * [[DeltaMergeMatchedActionBuilder]] object which can be used to specify how * to update or delete the matched target table row with the source row. * @since 0.3.0 */ def whenMatched(): DeltaMergeMatchedActionBuilder = { DeltaMergeMatchedActionBuilder(this, None) } /** * Build the actions to perform when the merge condition was matched and * the given `condition` is true. This returns [[DeltaMergeMatchedActionBuilder]] object * which can be used to specify how to update or delete the matched target table row with the * source row. * * @param condition boolean expression as a SQL formatted string * @since 0.3.0 */ def whenMatched(condition: String): DeltaMergeMatchedActionBuilder = { whenMatched(expr(condition)) } /** * Build the actions to perform when the merge condition was matched and * the given `condition` is true. This returns a [[DeltaMergeMatchedActionBuilder]] object * which can be used to specify how to update or delete the matched target table row with the * source row. * * @param condition boolean expression as a Column object * @since 0.3.0 */ def whenMatched(condition: Column): DeltaMergeMatchedActionBuilder = { DeltaMergeMatchedActionBuilder(this, Some(condition)) } /** * Build the action to perform when the merge condition was not matched. This returns * [[DeltaMergeNotMatchedActionBuilder]] object which can be used to specify how * to insert the new sourced row into the target table. * @since 0.3.0 */ def whenNotMatched(): DeltaMergeNotMatchedActionBuilder = { DeltaMergeNotMatchedActionBuilder(this, None) } /** * Build the actions to perform when the merge condition was not matched and * the given `condition` is true. This returns [[DeltaMergeMatchedActionBuilder]] object * which can be used to specify how to insert the new sourced row into the target table. * * @param condition boolean expression as a SQL formatted string * @since 0.3.0 */ def whenNotMatched(condition: String): DeltaMergeNotMatchedActionBuilder = { whenNotMatched(expr(condition)) } /** * Build the actions to perform when the merge condition was not matched and * the given `condition` is true. This returns [[DeltaMergeMatchedActionBuilder]] object * which can be used to specify how to insert the new sourced row into the target table. * * @param condition boolean expression as a Column object * @since 0.3.0 */ def whenNotMatched(condition: Column): DeltaMergeNotMatchedActionBuilder = { DeltaMergeNotMatchedActionBuilder(this, Some(condition)) } /** * Build the actions to perform when the merge condition was not matched by the source. This * returns [[DeltaMergeNotMatchedBySourceActionBuilder]] object which can be used to specify how * to update or delete the target table row. * @since 2.3.0 */ def whenNotMatchedBySource(): DeltaMergeNotMatchedBySourceActionBuilder = { DeltaMergeNotMatchedBySourceActionBuilder(this, None) } /** * Build the actions to perform when the merge condition was not matched by the source and the * given `condition` is true. This returns [[DeltaMergeNotMatchedBySourceActionBuilder]] object * which can be used to specify how to update or delete the target table row. * * @param condition boolean expression as a SQL formatted string * @since 2.3.0 */ def whenNotMatchedBySource(condition: String): DeltaMergeNotMatchedBySourceActionBuilder = { whenNotMatchedBySource(expr(condition)) } /** * Build the actions to perform when the merge condition was not matched by the source and the * given `condition` is true. This returns [[DeltaMergeNotMatchedBySourceActionBuilder]] object * which can be used to specify how to update or delete the target table row . * * @param condition boolean expression as a Column object * @since 2.3.0 */ def whenNotMatchedBySource(condition: Column): DeltaMergeNotMatchedBySourceActionBuilder = { DeltaMergeNotMatchedBySourceActionBuilder(this, Some(condition)) } /** * Enable schema evolution for the merge operation. This allows the schema of the target * table/columns to be automatically updated based on the schema of the source table/columns. * * @since 3.2.0 */ def withSchemaEvolution(): DeltaMergeBuilder = { new DeltaMergeBuilder( this.targetTable, this.source, this.onCondition, this.whenClauses, schemaEvolutionEnabled = true) } /** * Execute the merge operation based on the built matched and not matched actions. * * @since 0.3.0 */ def execute(): DataFrame = improveUnsupportedOpError { val sparkSession = targetTable.toDF.sparkSession withActiveSession(sparkSession) { // Note: We are explicitly resolving DeltaMergeInto plan rather than going to through the // Analyzer using `Dataset.ofRows()` because the Analyzer incorrectly resolves all // references in the DeltaMergeInto using both source and target child plans, even before // DeltaAnalysis rule kicks in. This is because the Analyzer understands only MergeIntoTable, // and handles that separately by skipping resolution (for Delta) and letting the // DeltaAnalysis rule do the resolving correctly. This can be solved by generating // MergeIntoTable instead, which blocked by the different issue with MergeIntoTable as // explained in the function `mergePlan` and // https://issues.apache.org/jira/browse/SPARK-34962. val resolvedMergeInto = ResolveDeltaMergeInto.resolveReferencesAndSchema(mergePlan, sparkSession.sessionState.conf)( tryResolveReferencesForExpressions(sparkSession)) val strippedMergeInto = resolvedMergeInto.copy( target = DeltaViewHelper.stripTempViewForMerge(resolvedMergeInto.target, SQLConf.get) ) // Preprocess the actions and verify var mergeIntoCommand = PreprocessTableMerge(sparkSession.sessionState.conf)(strippedMergeInto) // Resolve UpCast expressions that `PreprocessTableMerge` may have introduced. mergeIntoCommand = PostHocResolveUpCast(sparkSession).apply(mergeIntoCommand) sparkSession.sessionState.analyzer.checkAnalysis(mergeIntoCommand) toDataset(sparkSession, mergeIntoCommand) } } /** * :: Unstable :: * * Private method for internal usage only. Do not call this directly. */ @Unstable private[delta] def withClause(clause: DeltaMergeIntoClause): DeltaMergeBuilder = { new DeltaMergeBuilder( this.targetTable, this.source, this.onCondition, this.whenClauses :+ clause, this.schemaEvolutionEnabled) } private def mergePlan: DeltaMergeInto = { var targetPlan = targetTable.toDF.queryExecution.analyzed var sourcePlan = source.queryExecution.analyzed var condition = onCondition.expr var clauses = whenClauses // If source and target have duplicate, pre-resolved references (can happen with self-merge), // then rewrite the references in target with new exprId to avoid ambiguity. // We rewrite the target instead of ths source because the source plan can be arbitrary and // we know that the target plan is simple combination of LogicalPlan and an // optional SubqueryAlias. val duplicateResolvedRefs = targetPlan.outputSet.intersect(sourcePlan.outputSet) if (duplicateResolvedRefs.nonEmpty) { val exprs = (condition +: clauses).map(_.transform { // If any expression contain duplicate, pre-resolved references, we can't simply // replace the references in the same way as the target because we don't know // whether the user intended to refer to the source or the target columns. Instead, // we unresolve them (only the duplicate refs) and let the analysis resolve the ambiguity // and throw the usual error messages when needed. case a: AttributeReference if duplicateResolvedRefs.contains(a) => UnresolvedAttribute(a.qualifier :+ a.name) }) // Deduplicate the attribute IDs in the target and source plans, and all the MERGE // expressions (condition and MERGE clauses), so that we can avoid duplicated attribute ID // when building the MERGE command later. val fakePlan = AnalysisHelper.FakeLogicalPlan(exprs, Seq(sourcePlan, targetPlan)) val newPlan = org.apache.spark.sql.catalyst.analysis.DeduplicateRelations(fakePlan) .asInstanceOf[AnalysisHelper.FakeLogicalPlan] sourcePlan = newPlan.children(0) targetPlan = newPlan.children(1) condition = newPlan.exprs.head clauses = newPlan.exprs.takeRight(clauses.size).asInstanceOf[Seq[DeltaMergeIntoClause]] } // Note: The Scala API cannot generate MergeIntoTable just like the SQL parser because // UpdateAction in MergeIntoTable does not have any way to differentiate between // the representations of `updateAll()` and `update(some-condition, empty-actions)`. // More specifically, UpdateAction with a list of empty Assignments implicitly represents // `updateAll()`, so there is no way to represent `update()` with zero column assignments // (possible in Scala API, but syntactically not possible in SQL). This issue is tracked // by https://issues.apache.org/jira/browse/SPARK-34962. val merge = DeltaMergeInto( targetPlan, sourcePlan, condition, clauses, withSchemaEvolution = schemaEvolutionEnabled) logDebug("Generated merged plan:\n" + merge) merge } } object DeltaMergeBuilder { /** * :: Unstable :: * * Private method for internal usage only. Do not call this directly. */ @Unstable private[delta] def apply( targetTable: DeltaTable, source: DataFrame, onCondition: Column): DeltaMergeBuilder = { new DeltaMergeBuilder(targetTable, source, onCondition, Nil) } } /** * Builder class to specify the actions to perform when a target table row has matched a * source row based on the given merge condition and optional match condition. * * See [[DeltaMergeBuilder]] for more information. * * @since 0.3.0 */ class DeltaMergeMatchedActionBuilder private( private val mergeBuilder: DeltaMergeBuilder, private val matchCondition: Option[Column]) { /** * Update the matched table rows based on the rules defined by `set`. * * @param set rules to update a row as a Scala map between target column names and * corresponding update expressions as Column objects. * @since 0.3.0 */ def update(set: Map[String, Column]): DeltaMergeBuilder = { addUpdateClause(set) } /** * Update the matched table rows based on the rules defined by `set`. * * @param set rules to update a row as a Scala map between target column names and * corresponding update expressions as SQL formatted strings. * @since 0.3.0 */ def updateExpr(set: Map[String, String]): DeltaMergeBuilder = { addUpdateClause(toStrColumnMap(set)) } /** * Update a matched table row based on the rules defined by `set`. * * @param set rules to update a row as a Java map between target column names and * corresponding expressions as Column objects. * @since 0.3.0 */ def update(set: java.util.Map[String, Column]): DeltaMergeBuilder = { addUpdateClause(set.asScala) } /** * Update a matched table row based on the rules defined by `set`. * * @param set rules to update a row as a Java map between target column names and * corresponding expressions as SQL formatted strings. * @since 0.3.0 */ def updateExpr(set: java.util.Map[String, String]): DeltaMergeBuilder = { addUpdateClause(toStrColumnMap(set.asScala)) } /** * Update all the columns of the matched table row with the values of the * corresponding columns in the source row. * @since 0.3.0 */ def updateAll(): DeltaMergeBuilder = { val updateClause = DeltaMergeIntoMatchedUpdateClause( matchCondition.map(_.expr), DeltaMergeIntoClause.toActions(Nil, Nil)) mergeBuilder.withClause(updateClause) } /** * Delete a matched row from the table. * @since 0.3.0 */ def delete(): DeltaMergeBuilder = { val deleteClause = DeltaMergeIntoMatchedDeleteClause(matchCondition.map(_.expr)) mergeBuilder.withClause(deleteClause) } private def addUpdateClause(set: Map[String, Column]): DeltaMergeBuilder = { if (set.isEmpty && matchCondition.isEmpty) { // This is a catch all clause that doesn't update anything: we can ignore it. mergeBuilder } else { val setActions = set.toSeq val updateActions = DeltaMergeIntoClause.toActions( colNames = setActions.map(x => UnresolvedAttribute.quotedString(x._1)), exprs = setActions.map(x => x._2.expr), isEmptySeqEqualToStar = false) val updateClause = DeltaMergeIntoMatchedUpdateClause( matchCondition.map(_.expr), updateActions) mergeBuilder.withClause(updateClause) } } private def toStrColumnMap(map: Map[String, String]): Map[String, Column] = map.mapValues(functions.expr(_)).toMap } object DeltaMergeMatchedActionBuilder { /** * :: Unstable :: * * Private method for internal usage only. Do not call this directly. */ @Unstable private[delta] def apply( mergeBuilder: DeltaMergeBuilder, matchCondition: Option[Column]): DeltaMergeMatchedActionBuilder = { new DeltaMergeMatchedActionBuilder(mergeBuilder, matchCondition) } } /** * Builder class to specify the actions to perform when a source row has not matched any target * Delta table row based on the merge condition, but has matched the additional condition * if specified. * * See [[DeltaMergeBuilder]] for more information. * * @since 0.3.0 */ class DeltaMergeNotMatchedActionBuilder private( private val mergeBuilder: DeltaMergeBuilder, private val notMatchCondition: Option[Column]) { /** * Insert a new row to the target table based on the rules defined by `values`. * * @param values rules to insert a row as a Scala map between target column names and * corresponding expressions as Column objects. * @since 0.3.0 */ def insert(values: Map[String, Column]): DeltaMergeBuilder = { addInsertClause(values) } /** * Insert a new row to the target table based on the rules defined by `values`. * * @param values rules to insert a row as a Scala map between target column names and * corresponding expressions as SQL formatted strings. * @since 0.3.0 */ def insertExpr(values: Map[String, String]): DeltaMergeBuilder = { addInsertClause(toStrColumnMap(values)) } /** * Insert a new row to the target table based on the rules defined by `values`. * * @param values rules to insert a row as a Java map between target column names and * corresponding expressions as Column objects. * @since 0.3.0 */ def insert(values: java.util.Map[String, Column]): DeltaMergeBuilder = { addInsertClause(values.asScala) } /** * Insert a new row to the target table based on the rules defined by `values`. * * @param values rules to insert a row as a Java map between target column names and * corresponding expressions as SQL formatted strings. * * @since 0.3.0 */ def insertExpr(values: java.util.Map[String, String]): DeltaMergeBuilder = { addInsertClause(toStrColumnMap(values.asScala)) } /** * Insert a new target Delta table row by assigning the target columns to the values of the * corresponding columns in the source row. * @since 0.3.0 */ def insertAll(): DeltaMergeBuilder = { val insertClause = DeltaMergeIntoNotMatchedInsertClause( notMatchCondition.map(_.expr), DeltaMergeIntoClause.toActions(Nil, Nil)) mergeBuilder.withClause(insertClause) } private def addInsertClause(setValues: Map[String, Column]): DeltaMergeBuilder = { val values = setValues.toSeq val insertActions = DeltaMergeIntoClause.toActions( colNames = values.map(x => UnresolvedAttribute.quotedString(x._1)), exprs = values.map(x => x._2.expr), isEmptySeqEqualToStar = false) val insertClause = DeltaMergeIntoNotMatchedInsertClause( notMatchCondition.map(_.expr), insertActions) mergeBuilder.withClause(insertClause) } private def toStrColumnMap(map: Map[String, String]): Map[String, Column] = map.mapValues(functions.expr(_)).toMap } object DeltaMergeNotMatchedActionBuilder { /** * :: Unstable :: * * Private method for internal usage only. Do not call this directly. */ @Unstable private[delta] def apply( mergeBuilder: DeltaMergeBuilder, notMatchCondition: Option[Column]): DeltaMergeNotMatchedActionBuilder = { new DeltaMergeNotMatchedActionBuilder(mergeBuilder, notMatchCondition) } } /** * Builder class to specify the actions to perform when a target table row has no match in the * source table based on the given merge condition and optional match condition. * * See [[DeltaMergeBuilder]] for more information. * * @since 2.3.0 */ class DeltaMergeNotMatchedBySourceActionBuilder private( private val mergeBuilder: DeltaMergeBuilder, private val notMatchBySourceCondition: Option[Column]) { /** * Update an unmatched target table row based on the rules defined by `set`. * * @param set rules to update a row as a Scala map between target column names and * corresponding update expressions as Column objects. * @since 2.3.0 */ def update(set: Map[String, Column]): DeltaMergeBuilder = { addUpdateClause(set) } /** * Update an unmatched target table row based on the rules defined by `set`. * * @param set rules to update a row as a Scala map between target column names and * corresponding update expressions as SQL formatted strings. * @since 2.3.0 */ def updateExpr(set: Map[String, String]): DeltaMergeBuilder = { addUpdateClause(toStrColumnMap(set)) } /** * Update an unmatched target table row based on the rules defined by `set`. * * @param set rules to update a row as a Java map between target column names and * corresponding expressions as Column objects. * @since 2.3.0 */ def update(set: java.util.Map[String, Column]): DeltaMergeBuilder = { addUpdateClause(set.asScala) } /** * Update an unmatched target table row based on the rules defined by `set`. * * @param set rules to update a row as a Java map between target column names and * corresponding expressions as SQL formatted strings. * @since 2.3.0 */ def updateExpr(set: java.util.Map[String, String]): DeltaMergeBuilder = { addUpdateClause(toStrColumnMap(set.asScala)) } /** * Delete an unmatched row from the target table. * @since 2.3.0 */ def delete(): DeltaMergeBuilder = { val deleteClause = DeltaMergeIntoNotMatchedBySourceDeleteClause(notMatchBySourceCondition.map(_.expr)) mergeBuilder.withClause(deleteClause) } private def addUpdateClause(set: Map[String, Column]): DeltaMergeBuilder = { if (set.isEmpty && notMatchBySourceCondition.isEmpty) { // This is a catch all clause that doesn't update anything: we can ignore it. mergeBuilder } else { val setActions = set.toSeq val updateActions = DeltaMergeIntoClause.toActions( colNames = setActions.map(x => UnresolvedAttribute.quotedString(x._1)), exprs = setActions.map(x => x._2.expr), isEmptySeqEqualToStar = false) val updateClause = DeltaMergeIntoNotMatchedBySourceUpdateClause( notMatchBySourceCondition.map(_.expr), updateActions) mergeBuilder.withClause(updateClause) } } private def toStrColumnMap(map: Map[String, String]): Map[String, Column] = map.mapValues(functions.expr(_)).toMap } object DeltaMergeNotMatchedBySourceActionBuilder { /** * :: Unstable :: * * Private method for internal usage only. Do not call this directly. */ @Unstable private[delta] def apply( mergeBuilder: DeltaMergeBuilder, notMatchBySourceCondition: Option[Column]): DeltaMergeNotMatchedBySourceActionBuilder = { new DeltaMergeNotMatchedBySourceActionBuilder(mergeBuilder, notMatchBySourceCondition) } } ================================================ FILE: spark/src/main/scala/io/delta/tables/DeltaOptimizeBuilder.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.DeltaTableUtils.withActiveSession import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.DeltaOptimizeContext import org.apache.spark.sql.delta.commands.OptimizeTableCommand import org.apache.spark.sql.delta.util.AnalysisHelper import org.apache.spark.annotation._ import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.DataFrame import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.{ResolvedTable, UnresolvedAttribute} import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog} /** * Builder class for constructing OPTIMIZE command and executing. * * @param sparkSession SparkSession to use for execution * @param tableIdentifier Id of the table on which to * execute the optimize * @param options Hadoop file system options for read and write. * @since 2.0.0 */ class DeltaOptimizeBuilder private(table: DeltaTableV2) extends AnalysisHelper { private var partitionFilter: Seq[String] = Seq.empty private lazy val tableIdentifier: String = table.tableIdentifier.getOrElse(s"delta.`${table.deltaLog.dataPath.toString}`") /** * Apply partition filter on this optimize command builder to limit * the operation on selected partitions. * @param partitionFilter The partition filter to apply * @return [[DeltaOptimizeBuilder]] with partition filter applied * @since 2.0.0 */ def where(partitionFilter: String): DeltaOptimizeBuilder = { this.partitionFilter = this.partitionFilter :+ partitionFilter this } /** * Compact the small files in selected partitions. * @return DataFrame containing the OPTIMIZE execution metrics * @since 2.0.0 */ def executeCompaction(): DataFrame = { execute(Seq.empty) } /** * Z-Order the data in selected partitions using the given columns. * @param columns Zero or more columns to order the data * using Z-Order curves * @return DataFrame containing the OPTIMIZE execution metrics * @since 2.0.0 */ @scala.annotation.varargs def executeZOrderBy(columns: String *): DataFrame = { val attrs = columns.map(c => UnresolvedAttribute(c)) execute(attrs) } private def execute(zOrderBy: Seq[UnresolvedAttribute]): DataFrame = { val sparkSession = table.spark withActiveSession(sparkSession) { val tableId: TableIdentifier = sparkSession .sessionState .sqlParser .parseTableIdentifier(tableIdentifier) val id = Identifier.of(tableId.database.toArray, tableId.identifier) val catalogPlugin = sparkSession.sessionState.catalogManager.currentCatalog val catalog = catalogPlugin match { case tableCatalog: TableCatalog => tableCatalog case _ => throw new IllegalArgumentException( s"Catalog ${catalogPlugin.name} does not support tables") } val resolvedTable = ResolvedTable.create(catalog, id, table) val optimize = OptimizeTableCommand( resolvedTable, partitionFilter, DeltaOptimizeContext())(zOrderBy = zOrderBy) toDataset(sparkSession, optimize) } } } private[delta] object DeltaOptimizeBuilder { /** * :: Unstable :: * * Private method for internal usage only. Do not call this directly. */ @Unstable private[delta] def apply(table: DeltaTableV2): DeltaOptimizeBuilder = new DeltaOptimizeBuilder(table) } ================================================ FILE: spark/src/main/scala/io/delta/tables/DeltaTable.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables import scala.collection.JavaConverters._ import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.DeltaTableUtils.withActiveSession import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils import org.apache.spark.sql.delta.catalog.{CatalogResolver, DeltaTableV2} import org.apache.spark.sql.delta.commands.{AlterTableDropFeatureDeltaCommand, AlterTableSetPropertiesDeltaCommand} import org.apache.spark.sql.delta.sources.DeltaSQLConf import io.delta.tables.execution._ import org.apache.hadoop.fs.Path import org.apache.spark.annotation._ import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.types.StructType /** * Main class for programmatically interacting with Delta tables. * You can create DeltaTable instances using the static methods. * {{{ * DeltaTable.forPath(sparkSession, pathToTheDeltaTable) * }}} * * @since 0.3.0 */ class DeltaTable private[tables]( @transient private val _df: Dataset[Row], @transient private val table: DeltaTableV2) extends DeltaTableOperations with Serializable { protected def deltaLog: DeltaLog = { /** Assert the codes run in the driver. */ if (table == null) { throw DeltaErrors.deltaTableFoundInExecutor() } table.deltaLog } protected def df: Dataset[Row] = { /** Assert the codes run in the driver. */ if (_df == null) { throw DeltaErrors.deltaTableFoundInExecutor() } _df } /** * Apply an alias to the DeltaTable. This is similar to `Dataset.as(alias)` or * SQL `tableName AS alias`. * * @since 0.3.0 */ def as(alias: String): DeltaTable = new DeltaTable(df.as(alias), table) /** * Apply an alias to the DeltaTable. This is similar to `Dataset.as(alias)` or * SQL `tableName AS alias`. * * @since 0.3.0 */ def alias(alias: String): DeltaTable = as(alias) /** * Get a DataFrame (that is, Dataset[Row]) representation of this Delta table. * * @since 0.3.0 */ def toDF: Dataset[Row] = df /** * Recursively delete files and directories in the table that are not needed by the table for * maintaining older versions up to the given retention threshold. This method will return an * empty DataFrame on successful completion. * * @param retentionHours The retention threshold in hours. Files required by the table for * reading versions earlier than this will be preserved and the * rest of them will be deleted. * @since 0.3.0 */ def vacuum(retentionHours: Double): DataFrame = { executeVacuum(table, Some(retentionHours)) } /** * Recursively delete files and directories in the table that are not needed by the table for * maintaining older versions up to the given retention threshold. This method will return an * empty DataFrame on successful completion. * * note: This will use the default retention period of 7 days. * * @since 0.3.0 */ def vacuum(): DataFrame = { executeVacuum(table, retentionHours = None) } /** * Get the information of the latest `limit` commits on this table as a Spark DataFrame. * The information is in reverse chronological order. * * @param limit The number of previous commands to get history for * * @since 0.3.0 */ def history(limit: Int): DataFrame = { executeHistory(deltaLog, Some(limit), table.catalogTable) } /** * Get the information available commits on this table as a Spark DataFrame. * The information is in reverse chronological order. * * @since 0.3.0 */ def history(): DataFrame = { executeHistory(deltaLog, catalogTable = table.catalogTable) } /** * :: Evolving :: * * Get the details of a Delta table such as the format, name, and size. * * @since 2.1.0 */ @Evolving def detail(): DataFrame = { executeDetails(deltaLog.dataPath.toString, table.getTableIdentifierIfExists) } /** * Generate a manifest for the given Delta Table * * @param mode Specifies the mode for the generation of the manifest. * The valid modes are as follows (not case sensitive): * - "symlink_format_manifest" : This will generate manifests in symlink format * for Presto and Athena read support. * See the online documentation for more information. * @since 0.5.0 */ def generate(mode: String): Unit = { executeGenerate(deltaLog.dataPath.toString, table.getTableIdentifierIfExists, mode) } /** * Delete data from the table that match the given `condition`. * * @param condition Boolean SQL expression * * @since 0.3.0 */ def delete(condition: String): Unit = { delete(functions.expr(condition)) } /** * Delete data from the table that match the given `condition`. * * @param condition Boolean SQL expression * * @since 0.3.0 */ def delete(condition: Column): Unit = { executeDelete(Some(condition.expr)) } /** * Delete data from the table. * * @since 0.3.0 */ def delete(): Unit = { executeDelete(None) } /** * Optimize the data layout of the table. This returns * a [[DeltaOptimizeBuilder]] object that can be used to specify * the partition filter to limit the scope of optimize and * also execute different optimization techniques such as file * compaction or order data using Z-Order curves. * * See the [[DeltaOptimizeBuilder]] for a full description * of this operation. * * Scala example to run file compaction on a subset of * partitions in the table: * {{{ * deltaTable * .optimize() * .where("date='2021-11-18'") * .executeCompaction(); * }}} * * @since 2.0.0 */ def optimize(): DeltaOptimizeBuilder = DeltaOptimizeBuilder(table) /** * Update rows in the table based on the rules defined by `set`. * * Scala example to increment the column `data`. * {{{ * import org.apache.spark.sql.functions._ * * deltaTable.update(Map("data" -> col("data") + 1)) * }}} * * @param set rules to update a row as a Scala map between target column names and * corresponding update expressions as Column objects. * @since 0.3.0 */ def update(set: Map[String, Column]): Unit = { executeUpdate(set, None) } /** * Update rows in the table based on the rules defined by `set`. * * Java example to increment the column `data`. * {{{ * import org.apache.spark.sql.Column; * import org.apache.spark.sql.functions; * * deltaTable.update( * new HashMap() {{ * put("data", functions.col("data").plus(1)); * }} * ); * }}} * * @param set rules to update a row as a Java map between target column names and * corresponding update expressions as Column objects. * @since 0.3.0 */ def update(set: java.util.Map[String, Column]): Unit = { executeUpdate(set.asScala, None) } /** * Update data from the table on the rows that match the given `condition` * based on the rules defined by `set`. * * Scala example to increment the column `data`. * {{{ * import org.apache.spark.sql.functions._ * * deltaTable.update( * col("date") > "2018-01-01", * Map("data" -> col("data") + 1)) * }}} * * @param condition boolean expression as Column object specifying which rows to update. * @param set rules to update a row as a Scala map between target column names and * corresponding update expressions as Column objects. * @since 0.3.0 */ def update(condition: Column, set: Map[String, Column]): Unit = { executeUpdate(set, Some(condition)) } /** * Update data from the table on the rows that match the given `condition` * based on the rules defined by `set`. * * Java example to increment the column `data`. * {{{ * import org.apache.spark.sql.Column; * import org.apache.spark.sql.functions; * * deltaTable.update( * functions.col("date").gt("2018-01-01"), * new HashMap() {{ * put("data", functions.col("data").plus(1)); * }} * ); * }}} * * @param condition boolean expression as Column object specifying which rows to update. * @param set rules to update a row as a Java map between target column names and * corresponding update expressions as Column objects. * @since 0.3.0 */ def update(condition: Column, set: java.util.Map[String, Column]): Unit = { executeUpdate(set.asScala, Some(condition)) } /** * Update rows in the table based on the rules defined by `set`. * * Scala example to increment the column `data`. * {{{ * deltaTable.updateExpr(Map("data" -> "data + 1"))) * }}} * * @param set rules to update a row as a Scala map between target column names and * corresponding update expressions as SQL formatted strings. * @since 0.3.0 */ def updateExpr(set: Map[String, String]): Unit = { executeUpdate(toStrColumnMap(set), None) } /** * Update rows in the table based on the rules defined by `set`. * * Java example to increment the column `data`. * {{{ * deltaTable.updateExpr( * new HashMap() {{ * put("data", "data + 1"); * }} * ); * }}} * * @param set rules to update a row as a Java map between target column names and * corresponding update expressions as SQL formatted strings. * @since 0.3.0 */ def updateExpr(set: java.util.Map[String, String]): Unit = { executeUpdate(toStrColumnMap(set.asScala), None) } /** * Update data from the table on the rows that match the given `condition`, * which performs the rules defined by `set`. * * Scala example to increment the column `data`. * {{{ * deltaTable.update( * "date > '2018-01-01'", * Map("data" -> "data + 1")) * }}} * * @param condition boolean expression as SQL formatted string object specifying * which rows to update. * @param set rules to update a row as a Scala map between target column names and * corresponding update expressions as SQL formatted strings. * @since 0.3.0 */ def updateExpr(condition: String, set: Map[String, String]): Unit = { executeUpdate(toStrColumnMap(set), Some(functions.expr(condition))) } /** * Update data from the table on the rows that match the given `condition`, * which performs the rules defined by `set`. * * Java example to increment the column `data`. * {{{ * deltaTable.update( * "date > '2018-01-01'", * new HashMap() {{ * put("data", "data + 1"); * }} * ); * }}} * * @param condition boolean expression as SQL formatted string object specifying * which rows to update. * @param set rules to update a row as a Java map between target column names and * corresponding update expressions as SQL formatted strings. * @since 0.3.0 */ def updateExpr(condition: String, set: java.util.Map[String, String]): Unit = { executeUpdate(toStrColumnMap(set.asScala), Some(functions.expr(condition))) } /** * Merge data from the `source` DataFrame based on the given merge `condition`. This returns * a [[DeltaMergeBuilder]] object that can be used to specify the update, delete, or insert * actions to be performed on rows based on whether the rows matched the condition or not. * * See the [[DeltaMergeBuilder]] for a full description of this operation and what combinations of * update, delete and insert operations are allowed. * * Scala example to update a key-value Delta table with new key-values from a source DataFrame: * {{{ * deltaTable * .as("target") * .merge( * source.as("source"), * "target.key = source.key") * .whenMatched * .updateExpr(Map( * "value" -> "source.value")) * .whenNotMatched * .insertExpr(Map( * "key" -> "source.key", * "value" -> "source.value")) * .execute() * }}} * * Java example to update a key-value Delta table with new key-values from a source DataFrame: * {{{ * deltaTable * .as("target") * .merge( * source.as("source"), * "target.key = source.key") * .whenMatched * .updateExpr( * new HashMap() {{ * put("value" -> "source.value"); * }}) * .whenNotMatched * .insertExpr( * new HashMap() {{ * put("key", "source.key"); * put("value", "source.value"); * }}) * .execute(); * }}} * * @param source source Dataframe to be merged. * @param condition boolean expression as SQL formatted string * @since 0.3.0 */ def merge(source: DataFrame, condition: String): DeltaMergeBuilder = { merge(source, functions.expr(condition)) } /** * Merge data from the `source` DataFrame based on the given merge `condition`. This returns * a [[DeltaMergeBuilder]] object that can be used to specify the update, delete, or insert * actions to be performed on rows based on whether the rows matched the condition or not. * * See the [[DeltaMergeBuilder]] for a full description of this operation and what combinations of * update, delete and insert operations are allowed. * * Scala example to update a key-value Delta table with new key-values from a source DataFrame: * {{{ * deltaTable * .as("target") * .merge( * source.as("source"), * "target.key = source.key") * .whenMatched * .updateExpr(Map( * "value" -> "source.value")) * .whenNotMatched * .insertExpr(Map( * "key" -> "source.key", * "value" -> "source.value")) * .execute() * }}} * * Java example to update a key-value Delta table with new key-values from a source DataFrame: * {{{ * deltaTable * .as("target") * .merge( * source.as("source"), * "target.key = source.key") * .whenMatched * .updateExpr( * new HashMap() {{ * put("value" -> "source.value") * }}) * .whenNotMatched * .insertExpr( * new HashMap() {{ * put("key", "source.key"); * put("value", "source.value"); * }}) * .execute() * }}} * * @param source source Dataframe to be merged. * @param condition boolean expression as a Column object * @since 0.3.0 */ def merge(source: DataFrame, condition: Column): DeltaMergeBuilder = { DeltaMergeBuilder(this, source, condition) } /** * Restore the DeltaTable to an older version of the table specified by version number. * * An example would be * {{{ io.delta.tables.DeltaTable.restoreToVersion(7) }}} * * @since 1.2.0 */ def restoreToVersion(version: Long): DataFrame = { executeRestore(table, Some(version), None) } /** * Restore the DeltaTable to an older version of the table specified by a timestamp. * * Timestamp can be of the format yyyy-MM-dd or yyyy-MM-dd HH:mm:ss * * An example would be * {{{ io.delta.tables.DeltaTable.restoreToTimestamp("2019-01-01") }}} * * @since 1.2.0 */ def restoreToTimestamp(timestamp: String): DataFrame = { executeRestore(table, None, Some(timestamp)) } /** * Updates the protocol version of the table to leverage new features. Upgrading the reader * version will prevent all clients that have an older version of Delta Lake from accessing this * table. Upgrading the writer version will prevent older versions of Delta Lake to write to this * table. The reader or writer version cannot be downgraded. * * See online documentation and Delta's protocol specification at PROTOCOL.md for more details. * * @since 0.8.0 */ def upgradeTableProtocol(readerVersion: Int, writerVersion: Int): Unit = withActiveSession(sparkSession) { val alterTableCmd = AlterTableSetPropertiesDeltaCommand( table, DeltaConfigs.validateConfigurations( Map( "delta.minReaderVersion" -> readerVersion.toString, "delta.minWriterVersion" -> writerVersion.toString))) toDataset(sparkSession, alterTableCmd) } /** * Modify the protocol to add a supported feature, and if the table does not support table * features, upgrade the protocol automatically. In such a case when the provided feature is * writer-only, the table's writer version will be upgraded to `7`, and when the provided * feature is reader-writer, both reader and writer versions will be upgraded, to `(3, 7)`. * * See online documentation and Delta's protocol specification at PROTOCOL.md for more details. * * @since 2.3.0 */ def addFeatureSupport(featureName: String): Unit = withActiveSession(sparkSession) { // Do not check for the correctness of the provided feature name. The ALTER TABLE command will // do that in a transaction. val alterTableCmd = AlterTableSetPropertiesDeltaCommand( table, Map( TableFeatureProtocolUtils.propertyKey(featureName) -> TableFeatureProtocolUtils.FEATURE_PROP_SUPPORTED)) toDataset(sparkSession, alterTableCmd) } private def executeDropFeature(featureName: String, truncateHistory: Option[Boolean]): Unit = { val alterTableCmd = AlterTableDropFeatureDeltaCommand( table = table, featureName = featureName, truncateHistory = truncateHistory.getOrElse(false)) toDataset(sparkSession, alterTableCmd) } /** * Modify the protocol to drop a supported feature. The operation always normalizes the * resulting protocol. Protocol normalization is the process of converting a table features * protocol to the weakest possible form. This primarily refers to converting a table features * protocol to a legacy protocol. A table features protocol can be represented with the legacy * representation only when the feature set of the former exactly matches a legacy protocol. * Normalization can also decrease the reader version of a table features protocol when it is * higher than necessary. For example: * * (1, 7, None, {AppendOnly, Invariants, CheckConstraints}) -> (1, 3) * (3, 7, None, {RowTracking}) -> (1, 7, RowTracking) * * The dropFeatureSupport method can be used as follows: * {{{ * io.delta.tables.DeltaTable.dropFeatureSupport("rowTracking") * }}} * * See online documentation for more details. * * @param featureName The name of the feature to drop. * @param truncateHistory Whether to truncate history before downgrading the protocol. * @return None. * @since 3.4.0 */ def dropFeatureSupport( featureName: String, truncateHistory: Boolean): Unit = withActiveSession(sparkSession) { executeDropFeature(featureName, Some(truncateHistory)) } /** * Modify the protocol to drop a supported feature. The operation always normalizes the * resulting protocol. Protocol normalization is the process of converting a table features * protocol to the weakest possible form. This primarily refers to converting a table features * protocol to a legacy protocol. A table features protocol can be represented with the legacy * representation only when the feature set of the former exactly matches a legacy protocol. * Normalization can also decrease the reader version of a table features protocol when it is * higher than necessary. For example: * * (1, 7, None, {AppendOnly, Invariants, CheckConstraints}) -> (1, 3) * (3, 7, None, {RowTracking}) -> (1, 7, RowTracking) * * The dropFeatureSupport method can be used as follows: * {{{ * io.delta.tables.DeltaTable.dropFeatureSupport("rowTracking") * }}} * * Note, this command will not truncate history. * * See online documentation for more details. * * @param featureName The name of the feature to drop. * @return None. * @since 3.4.0 */ def dropFeatureSupport(featureName: String): Unit = withActiveSession(sparkSession) { executeDropFeature(featureName, None) } /** * Clone a DeltaTable to a given destination to mirror the existing table's data and metadata. * * Specifying properties here means that the target will override any properties with the same key * in the source table with the user-defined properties. * * An example would be * {{{ * io.delta.tables.DeltaTable.clone( * "/some/path/to/table", * true, * true, * Map("foo" -> "bar")) * }}} * * @param target The path or table name to create the clone. * @param isShallow Whether to create a shallow clone or a deep clone. * @param replace Whether to replace the destination with the clone command. * @param properties The table properties to override in the clone. * * @since 3.3.0 */ def clone( target: String, isShallow: Boolean, replace: Boolean, properties: Map[String, String]): DeltaTable = { executeClone( table, target, isShallow, replace, properties, versionAsOf = None, timestampAsOf = None) } /** * clone used by Python implementation using java.util.HashMap for the properties argument. * * Specifying properties here means that the target will override any properties with the same key * in the source table with the user-defined properties. * * An example would be * {{{ * io.delta.tables.DeltaTable.clone( * "/some/path/to/table", * true, * true, * Map("foo" -> "bar")) * }}} * * @param target The path or table name to create the clone. * @param isShallow Whether to create a shallow clone or a deep clone. * @param replace Whether to replace the destination with the clone command. * @param properties The table properties to override in the clone. */ def clone( target: String, isShallow: Boolean, replace: Boolean, properties: java.util.HashMap[String, String]): DeltaTable = { val scalaProps = Option(properties).map(_.asScala.toMap).getOrElse(Map.empty[String, String]) executeClone( table, target, isShallow, replace, scalaProps, versionAsOf = None, timestampAsOf = None) } /** * Clone a DeltaTable to a given destination to mirror the existing table's data and metadata. * * An example would be * {{{ * io.delta.tables.DeltaTable.clone( * "/some/path/to/table", * true, * true) * }}} * * @param target The path or table name to create the clone. * @param isShallow Whether to create a shallow clone or a deep clone. * @param replace Whether to replace the destination with the clone command. * * @since 3.3.0 */ def clone(target: String, isShallow: Boolean, replace: Boolean): DeltaTable = { clone(target, isShallow, replace, properties = Map.empty[String, String]) } /** * Clone a DeltaTable to a given destination to mirror the existing table's data and metadata. * * An example would be * {{{ * io.delta.tables.DeltaTable.clone( * "/some/path/to/table", * true) * }}} * * @param target The path or table name to create the clone. * @param isShallow Whether to create a shallow clone or a deep clone. * * @since 3.3.0 */ def clone(target: String, isShallow: Boolean): DeltaTable = { clone(target, isShallow, replace = false) } /** * Clone a DeltaTable at a specific version to a given destination to mirror the existing * table's data and metadata at that version. * * Specifying properties here means that the target will override any properties with the same key * in the source table with the user-defined properties. * * An example would be * {{{ * io.delta.tables.DeltaTable.cloneAtVersion( * 5, * "/some/path/to/table", * true, * true, * Map("foo" -> "bar")) * }}} * * @param version The version of this table to clone from. * @param target The path or table name to create the clone. * @param isShallow Whether to create a shallow clone or a deep clone. * @param replace Whether to replace the destination with the clone command. * @param properties The table properties to override in the clone. * * @since 3.3.0 */ def cloneAtVersion( version: Long, target: String, isShallow: Boolean, replace: Boolean, properties: Map[String, String]): DeltaTable = { executeClone( table, target, isShallow, replace, properties, versionAsOf = Some(version), timestampAsOf = None) } /** * cloneAtVersion used by Python implementation using java.util.HashMap for the properties * argument. * * Specifying properties here means that the target will override any properties with the same key * in the source table with the user-defined properties. * * An example would be * {{{ * io.delta.tables.DeltaTable.cloneAtVersion( * 5, * "/some/path/to/table", * true, * true, * new java.util.HashMap[String, String](Map("foo" -> "bar").asJava)) * }}} * * @param version The version of this table to clone from. * @param target The path or table name to create the clone. * @param isShallow Whether to create a shallow clone or a deep clone. * @param replace Whether to replace the destination with the clone command. * @param properties The table properties to override in the clone. */ def cloneAtVersion( version: Long, target: String, isShallow: Boolean, replace: Boolean, properties: java.util.HashMap[String, String]): DeltaTable = { val scalaProps = Option(properties).map(_.asScala.toMap).getOrElse(Map.empty[String, String]) executeClone( table, target, isShallow, replace, scalaProps, versionAsOf = Some(version), timestampAsOf = None) } /** * Clone a DeltaTable at a specific version to a given destination to mirror the existing * table's data and metadata at that version. * * An example would be * {{{ * io.delta.tables.DeltaTable.cloneAtVersion( * 5, * "/some/path/to/table", * true, * true) * }}} * * @param version The version of this table to clone from. * @param target The path or table name to create the clone. * @param isShallow Whether to create a shallow clone or a deep clone. * @param replace Whether to replace the destination with the clone command. * * @since 3.3.0 */ def cloneAtVersion( version: Long, target: String, isShallow: Boolean, replace: Boolean): DeltaTable = { cloneAtVersion(version, target, isShallow, replace, properties = Map.empty[String, String]) } /** * Clone a DeltaTable at a specific version to a given destination to mirror the existing * table's data and metadata at that version. * * An example would be * {{{ * io.delta.tables.DeltaTable.cloneAtVersion( * 5, * "/some/path/to/table", * true) * }}} * * @param version The version of this table to clone from. * @param target The path or table name to create the clone. * @param isShallow Whether to create a shallow clone or a deep clone. * * @since 3.3.0 */ def cloneAtVersion(version: Long, target: String, isShallow: Boolean): DeltaTable = { cloneAtVersion(version, target, isShallow, replace = false) } /** * Clone a DeltaTable at a specific timestamp to a given destination to mirror the existing * table's data and metadata at that timestamp. * * Timestamp can be of the format yyyy-MM-dd or yyyy-MM-dd HH:mm:ss. * * Specifying properties here means that the target will override any properties with the same key * in the source table with the user-defined properties. * * An example would be * {{{ * io.delta.tables.DeltaTable.cloneAtTimestamp( * "2019-01-01", * "/some/path/to/table", * true, * true, * Map("foo" -> "bar")) * }}} * * @param timestamp The timestamp of this table to clone from. * @param target The path or table name to create the clone. * @param isShallow Whether to create a shallow clone or a deep clone. * @param replace Whether to replace the destination with the clone command. * @param properties The table properties to override in the clone. * * @since 3.3.0 */ def cloneAtTimestamp( timestamp: String, target: String, isShallow: Boolean, replace: Boolean, properties: Map[String, String]): DeltaTable = { executeClone( table, target, isShallow, replace, properties, versionAsOf = None, timestampAsOf = Some(timestamp) ) } /** * cloneAtTimestamp used by Python implementation using java.util.HashMap for the properties * argument. * * Clone a DeltaTable at a specific timestamp to a given destination to mirror the existing * table's data and metadata at that version. * Specifying properties here means that the target will override any properties with the same key * in the source table with the user-defined properties. * * An example would be * {{{ * io.delta.tables.DeltaTable.cloneAtVersion( * 5, * "/some/path/to/table", * true, * true, * new java.util.HashMap[String, String](Map("foo" -> "bar").asJava) * }}} * * @param timestamp The timestamp of this table to clone from. * @param target The path or table name to create the clone. * @param isShallow Whether to create a shallow clone or a deep clone. * @param replace Whether to replace the destination with the clone command. * @param properties The table properties to override in the clone. */ def cloneAtTimestamp( timestamp: String, target: String, isShallow: Boolean, replace: Boolean, properties: java.util.HashMap[String, String]): DeltaTable = { val scalaProps = Option(properties).map(_.asScala.toMap).getOrElse(Map.empty[String, String]) executeClone( table, target, isShallow, replace, scalaProps, versionAsOf = None, timestampAsOf = Some(timestamp)) } /** * Clone a DeltaTable at a specific timestamp to a given destination to mirror the existing * table's data and metadata at that timestamp. * * Timestamp can be of the format yyyy-MM-dd or yyyy-MM-dd HH:mm:ss. * * An example would be * {{{ * io.delta.tables.DeltaTable.cloneAtTimestamp( * "2019-01-01", * "/some/path/to/table", * true, * true) * }}} * * @param timestamp The timestamp of this table to clone from. * @param target The path or table name to create the clone. * @param isShallow Whether to create a shallow clone or a deep clone. * @param replace Whether to replace the destination with the clone command. * * @since 3.3.0 */ def cloneAtTimestamp( timestamp: String, target: String, isShallow: Boolean, replace: Boolean): DeltaTable = { cloneAtTimestamp(timestamp, target, isShallow, replace, properties = Map.empty[String, String]) } /** * Clone a DeltaTable at a specific timestamp to a given destination to mirror the existing * table's data and metadata at that timestamp. * * Timestamp can be of the format yyyy-MM-dd or yyyy-MM-dd HH:mm:ss. * * An example would be * {{{ * io.delta.tables.DeltaTable.cloneAtTimestamp( * "2019-01-01", * "/some/path/to/table", * true) * }}} * * @param timestamp The timestamp of this table to clone from. * @param target The path or table name to create the clone. * @param isShallow Whether to create a shallow clone or a deep clone. * * @since 3.3.0 */ def cloneAtTimestamp(timestamp: String, target: String, isShallow: Boolean): DeltaTable = { cloneAtTimestamp(timestamp, target, isShallow, replace = false) } } /** * Companion object to create DeltaTable instances. * * {{{ * DeltaTable.forPath(sparkSession, pathToTheDeltaTable) * }}} * * @since 0.3.0 */ object DeltaTable { /** * Create a DeltaTable from the given parquet table and partition schema. * Takes an existing parquet table and constructs a delta transaction log in the base path of * that table. * * Note: Any changes to the table during the conversion process may not result in a consistent * state at the end of the conversion. Users should stop any changes to the table before the * conversion is started. * * An example usage would be * {{{ * io.delta.tables.DeltaTable.convertToDelta( * spark, * "parquet.`/path`", * new StructType().add(StructField("key1", LongType)).add(StructField("key2", StringType))) * }}} * * @since 0.4.0 */ def convertToDelta( spark: SparkSession, identifier: String, partitionSchema: StructType): DeltaTable = { val tableId: TableIdentifier = spark.sessionState.sqlParser.parseTableIdentifier(identifier) DeltaConvert.executeConvert(spark, tableId, Some(partitionSchema), None) } /** * Create a DeltaTable from the given parquet table and partition schema. * Takes an existing parquet table and constructs a delta transaction log in the base path of * that table. * * Note: Any changes to the table during the conversion process may not result in a consistent * state at the end of the conversion. Users should stop any changes to the table before the * conversion is started. * * An example usage would be * {{{ * io.delta.tables.DeltaTable.convertToDelta( * spark, * "parquet.`/path`", * "key1 long, key2 string") * }}} * * @since 0.4.0 */ def convertToDelta( spark: SparkSession, identifier: String, partitionSchema: String): DeltaTable = { val tableId: TableIdentifier = spark.sessionState.sqlParser.parseTableIdentifier(identifier) DeltaConvert.executeConvert(spark, tableId, Some(StructType.fromDDL(partitionSchema)), None) } /** * Create a DeltaTable from the given parquet table. Takes an existing parquet table and * constructs a delta transaction log in the base path of the table. * * Note: Any changes to the table during the conversion process may not result in a consistent * state at the end of the conversion. Users should stop any changes to the table before the * conversion is started. * * An Example would be * {{{ * io.delta.tables.DeltaTable.convertToDelta( * spark, * "parquet.`/path`" * }}} * * @since 0.4.0 */ def convertToDelta( spark: SparkSession, identifier: String): DeltaTable = { val tableId: TableIdentifier = spark.sessionState.sqlParser.parseTableIdentifier(identifier) DeltaConvert.executeConvert(spark, tableId, None, None) } /** * Instantiate a [[DeltaTable]] object representing the data at the given path, If the given * path is invalid (i.e. either no table exists or an existing table is not a Delta table), * it throws a `not a Delta table` error. * * Note: This uses the active SparkSession in the current thread to read the table data. Hence, * this throws error if active SparkSession has not been set, that is, * `SparkSession.getActiveSession()` is empty. * * @since 0.3.0 */ def forPath(path: String): DeltaTable = { val sparkSession = SparkSession.getActiveSession.getOrElse { throw DeltaErrors.activeSparkSessionNotFound() } forPath(sparkSession, path) } /** * Instantiate a [[DeltaTable]] object representing the data at the given path, If the given * path is invalid (i.e. either no table exists or an existing table is not a Delta table), * it throws a `not a Delta table` error. * * @since 0.3.0 */ def forPath(sparkSession: SparkSession, path: String): DeltaTable = { forPath(sparkSession, path, Map.empty[String, String]) } /** * Instantiate a [[DeltaTable]] object representing the data at the given path, If the given * path is invalid (i.e. either no table exists or an existing table is not a Delta table), * it throws a `not a Delta table` error. * * @param hadoopConf Hadoop configuration starting with "fs." or "dfs." will be picked up * by `DeltaTable` to access the file system when executing queries. * Other configurations will not be allowed. * * {{{ * val hadoopConf = Map( * "fs.s3a.access.key" -> "", * "fs.s3a.secret.key" -> "" * ) * DeltaTable.forPath(spark, "/path/to/table", hadoopConf) * }}} * @since 2.2.0 */ def forPath( sparkSession: SparkSession, path: String, hadoopConf: scala.collection.Map[String, String]): DeltaTable = { // We only pass hadoopConf so that we won't pass any unsafe options to Delta. val badOptions = hadoopConf.filterKeys { k => !DeltaTableUtils.validDeltaTableHadoopPrefixes.exists(k.startsWith) }.toMap if (!badOptions.isEmpty) { throw DeltaErrors.unsupportedDeltaTableForPathHadoopConf(badOptions) } val fileSystemOptions: Map[String, String] = hadoopConf.toMap val hdpPath = new Path(path) if (DeltaTableUtils.isDeltaTable(sparkSession, hdpPath, fileSystemOptions)) { new DeltaTable(sparkSession.read.format("delta").options(fileSystemOptions).load(path), DeltaTableV2( spark = sparkSession, path = hdpPath, options = fileSystemOptions)) } else { throw DeltaErrors.notADeltaTableException(DeltaTableIdentifier(path = Some(path))) } } /** * Java friendly API to instantiate a [[DeltaTable]] object representing the data at the given * path, If the given path is invalid (i.e. either no table exists or an existing table is not a * Delta table), it throws a `not a Delta table` error. * * @param hadoopConf Hadoop configuration starting with "fs." or "dfs." will be picked up * by `DeltaTable` to access the file system when executing queries. * Other configurations will be ignored. * * {{{ * val hadoopConf = Map( * "fs.s3a.access.key" -> "", * "fs.s3a.secret.key", "" * ) * DeltaTable.forPath(spark, "/path/to/table", hadoopConf) * }}} * @since 2.2.0 */ def forPath( sparkSession: SparkSession, path: String, hadoopConf: java.util.Map[String, String]): DeltaTable = { val fsOptions = hadoopConf.asScala.toMap forPath(sparkSession, path, fsOptions) } /** * Instantiate a [[DeltaTable]] object using the given table name. If the given * tableOrViewName is invalid (i.e. either no table exists or an existing table is not a * Delta table), it throws a `not a Delta table` error. Note: Passing a view name will also * result in this error as views are not supported. * * The given tableOrViewName can also be the absolute path of a delta datasource (i.e. * delta.`path`), If so, instantiate a [[DeltaTable]] object representing the data at * the given path (consistent with the [[forPath]]). * * Note: This uses the active SparkSession in the current thread to read the table data. Hence, * this throws error if active SparkSession has not been set, that is, * `SparkSession.getActiveSession()` is empty. */ def forName(tableOrViewName: String): DeltaTable = { val sparkSession = SparkSession.getActiveSession.getOrElse { throw DeltaErrors.activeSparkSessionNotFound() } forName(sparkSession, tableOrViewName) } // Helper to resolve a table using SessionCatalog private def getDeltaTableFromSessionCatalog( spark: SparkSession, tableName: String): DeltaTable = { val tableId = spark.sessionState.sqlParser.parseTableIdentifier(tableName) if (DeltaTableUtils.isDeltaTable(spark, tableId)) { val tbl = spark.sessionState.catalog.getTableMetadata(tableId) new DeltaTable( spark.table(tableName), DeltaTableV2(spark, new Path(tbl.location), Some(tbl), Some(tableName))) } else if (DeltaTableUtils.isValidPath(tableId)) { forPath(spark, tableId.table) } else { throw DeltaErrors.notADeltaTableException(DeltaTableIdentifier(table = Some(tableId))) } } /** * Instantiate a [[DeltaTable]] object using one of the following: * 1. The given tableName using the given SparkSession and SessionCatalog. * 2. The tableName can also be the absolute path of a delta datasource (i.e. * delta.`path`), If so, instantiate a [[DeltaTable]] object representing the data at * the given path (consistent with the [[forPath]]). * 3. A fully qualified tableName is passed in the form `catalog.db.table`, If so * the table is resolved through the specified catalog instead of the default *SessionCatalog* * * If the given tableName is invalid (i.e. either no table exists or an * existing table is not a Delta table), it throws a `not a Delta table` error. Note: * Passing a view name will also result in this error as views are not supported. */ def forName(sparkSession: SparkSession, tableName: String): DeltaTable = { sparkSession.sessionState.sqlParser.parseMultipartIdentifier(tableName) match { case parts if parts.length == 3 => val (catalog, ident) = CatalogResolver.getCatalogPluginAndIdentifier(sparkSession, parts.head, parts.tail) new DeltaTable( sparkSession.table(tableName), CatalogResolver.getDeltaTableFromCatalog(sparkSession, catalog, ident) ) case _ => getDeltaTableFromSessionCatalog(sparkSession, tableName) } } /** * Check if the provided `identifier` string, in this case a file path, * is the root of a Delta table using the given SparkSession. * * An example would be * {{{ * DeltaTable.isDeltaTable(spark, "path/to/table") * }}} * * @since 0.4.0 */ def isDeltaTable(sparkSession: SparkSession, identifier: String): Boolean = { val identifierPath = new Path(identifier) if (sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_STRICT_CHECK_DELTA_TABLE)) { val rootOption = DeltaTableUtils.findDeltaTableRoot(sparkSession, identifierPath) rootOption.isDefined && DeltaLog.forTable(sparkSession, rootOption.get).tableExists } else { DeltaTableUtils.isDeltaTable(sparkSession, identifierPath) } } /** * Check if the provided `identifier` string, in this case a file path, * is the root of a Delta table. * * Note: This uses the active SparkSession in the current thread to search for the table. Hence, * this throws error if active SparkSession has not been set, that is, * `SparkSession.getActiveSession()` is empty. * * An example would be * {{{ * DeltaTable.isDeltaTable(spark, "/path/to/table") * }}} * * @since 0.4.0 */ def isDeltaTable(identifier: String): Boolean = { val sparkSession = SparkSession.getActiveSession.getOrElse { throw DeltaErrors.activeSparkSessionNotFound() } isDeltaTable(sparkSession, identifier) } /** * :: Evolving :: * * Return an instance of [[DeltaTableBuilder]] to create a Delta table, * error if the table exists (the same as SQL `CREATE TABLE`). * Refer to [[DeltaTableBuilder]] for more details. * * Note: This uses the active SparkSession in the current thread to read the table data. Hence, * this throws error if active SparkSession has not been set, that is, * `SparkSession.getActiveSession()` is empty. * * @since 1.0.0 */ @Evolving def create(): DeltaTableBuilder = { val sparkSession = SparkSession.getActiveSession.getOrElse { throw DeltaErrors.activeSparkSessionNotFound() } create(sparkSession) } /** * :: Evolving :: * * Return an instance of [[DeltaTableBuilder]] to create a Delta table, * error if the table exists (the same as SQL `CREATE TABLE`). * Refer to [[DeltaTableBuilder]] for more details. * * @param spark sparkSession sparkSession passed by the user * @since 1.0.0 */ @Evolving def create(spark: SparkSession): DeltaTableBuilder = { new DeltaTableBuilder(spark, CreateTableOptions(ifNotExists = false)) } /** * :: Evolving :: * * Return an instance of [[DeltaTableBuilder]] to create a Delta table, * if it does not exists (the same as SQL `CREATE TABLE IF NOT EXISTS`). * Refer to [[DeltaTableBuilder]] for more details. * * Note: This uses the active SparkSession in the current thread to read the table data. Hence, * this throws error if active SparkSession has not been set, that is, * `SparkSession.getActiveSession()` is empty. * * @since 1.0.0 */ @Evolving def createIfNotExists(): DeltaTableBuilder = { val sparkSession = SparkSession.getActiveSession.getOrElse { throw DeltaErrors.activeSparkSessionNotFound() } createIfNotExists(sparkSession) } /** * :: Evolving :: * * Return an instance of [[DeltaTableBuilder]] to create a Delta table, * if it does not exists (the same as SQL `CREATE TABLE IF NOT EXISTS`). * Refer to [[DeltaTableBuilder]] for more details. * * @param spark sparkSession sparkSession passed by the user * @since 1.0.0 */ @Evolving def createIfNotExists(spark: SparkSession): DeltaTableBuilder = { new DeltaTableBuilder(spark, CreateTableOptions(ifNotExists = true)) } /** * :: Evolving :: * * Return an instance of [[DeltaTableBuilder]] to replace a Delta table, * error if the table doesn't exist (the same as SQL `REPLACE TABLE`) * Refer to [[DeltaTableBuilder]] for more details. * * Note: This uses the active SparkSession in the current thread to read the table data. Hence, * this throws error if active SparkSession has not been set, that is, * `SparkSession.getActiveSession()` is empty. * * @since 1.0.0 */ @Evolving def replace(): DeltaTableBuilder = { val sparkSession = SparkSession.getActiveSession.getOrElse { throw DeltaErrors.activeSparkSessionNotFound() } replace(sparkSession) } /** * :: Evolving :: * * Return an instance of [[DeltaTableBuilder]] to replace a Delta table, * error if the table doesn't exist (the same as SQL `REPLACE TABLE`) * Refer to [[DeltaTableBuilder]] for more details. * * @param spark sparkSession sparkSession passed by the user * @since 1.0.0 */ @Evolving def replace(spark: SparkSession): DeltaTableBuilder = { new DeltaTableBuilder(spark, ReplaceTableOptions(orCreate = false)) } /** * :: Evolving :: * * Return an instance of [[DeltaTableBuilder]] to replace a Delta table * or create table if not exists (the same as SQL `CREATE OR REPLACE TABLE`) * Refer to [[DeltaTableBuilder]] for more details. * * Note: This uses the active SparkSession in the current thread to read the table data. Hence, * this throws error if active SparkSession has not been set, that is, * `SparkSession.getActiveSession()` is empty. * * @since 1.0.0 */ @Evolving def createOrReplace(): DeltaTableBuilder = { val sparkSession = SparkSession.getActiveSession.getOrElse { throw DeltaErrors.activeSparkSessionNotFound() } createOrReplace(sparkSession) } /** * :: Evolving :: * * Return an instance of [[DeltaTableBuilder]] to replace a Delta table, * or create table if not exists (the same as SQL `CREATE OR REPLACE TABLE`) * Refer to [[DeltaTableBuilder]] for more details. * * @param spark sparkSession sparkSession passed by the user. * @since 1.0.0 */ @Evolving def createOrReplace(spark: SparkSession): DeltaTableBuilder = { new DeltaTableBuilder(spark, ReplaceTableOptions(orCreate = true)) } /** * :: Evolving :: * * Return an instance of [[DeltaColumnBuilder]] to specify a column. * Refer to [[DeltaTableBuilder]] for examples and [[DeltaColumnBuilder]] detailed APIs. * * Note: This uses the active SparkSession in the current thread to read the table data. Hence, * this throws error if active SparkSession has not been set, that is, * `SparkSession.getActiveSession()` is empty. * * @param colName string the column name * @since 1.0.0 */ @Evolving def columnBuilder(colName: String): DeltaColumnBuilder = { val sparkSession = SparkSession.getActiveSession.getOrElse { throw DeltaErrors.activeSparkSessionNotFound() } columnBuilder(sparkSession, colName) } /** * :: Evolving :: * * Return an instance of [[DeltaColumnBuilder]] to specify a column. * Refer to [[DeltaTableBuilder]] for examples and [[DeltaColumnBuilder]] detailed APIs. * * @param spark sparkSession sparkSession passed by the user * @param colName string the column name * @since 1.0.0 */ @Evolving def columnBuilder(spark: SparkSession, colName: String): DeltaColumnBuilder = { new DeltaColumnBuilder(spark, colName) } } ================================================ FILE: spark/src/main/scala/io/delta/tables/DeltaTableBuilder.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables import scala.collection.mutable import org.apache.spark.sql.delta.{DeltaErrors, DeltaTableUtils} import org.apache.spark.sql.delta.DeltaTableUtils.withActiveSession import org.apache.spark.sql.delta.sources.DeltaSQLConf import io.delta.tables.execution._ import org.apache.spark.annotation._ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.plans.logical.{ColumnDefinition, CreateTable, ReplaceTable} import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.execution.SQLExecution import org.apache.spark.sql.types.{DataType, StructField, StructType} /** * :: Evolving :: * * Builder to specify how to create / replace a Delta table. * You must specify the table name or the path before executing the builder. * You can specify the table columns, the partitioning columns, the location of the data, * the table comment and the property, and how you want to create / replace the Delta table. * * After executing the builder, an instance of [[DeltaTable]] is returned. * * Scala example to create a Delta table with generated columns, using the table name: * {{{ * val table: DeltaTable = DeltaTable.create() * .tableName("testTable") * .addColumn("c1", dataType = "INT", nullable = false) * .addColumn( * DeltaTable.columnBuilder("c2") * .dataType("INT") * .generatedAlwaysAs("c1 + 10") * .build() * ) * .addColumn( * DeltaTable.columnBuilder("c3") * .dataType("INT") * .comment("comment") * .nullable(true) * .build() * ) * .partitionedBy("c1", "c2") * .execute() * }}} * * Scala example to create a delta table using the location: * {{{ * val table: DeltaTable = DeltaTable.createIfNotExists(spark) * .location("/foo/`bar`") * .addColumn("c1", dataType = "INT", nullable = false) * .addColumn( * DeltaTable.columnBuilder(spark, "c2") * .dataType("INT") * .generatedAlwaysAs("c1 + 10") * .build() * ) * .addColumn( * DeltaTable.columnBuilder(spark, "c3") * .dataType("INT") * .comment("comment") * .nullable(true) * .build() * ) * .partitionedBy("c1", "c2") * .execute() * }}} * * Java Example to replace a table: * {{{ * DeltaTable table = DeltaTable.replace() * .tableName("db.table") * .addColumn("c1", "INT", false) * .addColumn( * DeltaTable.columnBuilder("c2") * .dataType("INT") * .generatedAlwaysBy("c1 + 10") * .build() * ) * .execute(); * }}} * * @since 1.0.0 */ @Evolving class DeltaTableBuilder private[tables]( spark: SparkSession, builderOption: DeltaTableBuilderOptions) { private var identifier: String = null private var partitioningColumns: Option[Seq[String]] = None private var clusteringColumns: Option[Seq[String]] = None private var columns: mutable.Seq[StructField] = mutable.Seq.empty private var location: Option[String] = None private var tblComment: Option[String] = None private var properties = if (spark.sessionState.conf.getConf(DeltaSQLConf.TABLE_BUILDER_FORCE_TABLEPROPERTY_LOWERCASE)) { CaseInsensitiveMap(Map.empty[String, String]) } else { Map.empty[String, String] } private val FORMAT_NAME: String = "delta" /** * :: Evolving :: * * Specify the table name, optionally qualified with a database name [database_name.] table_name * * @param identifier string the table name * @since 1.0.0 */ @Evolving def tableName(identifier: String): DeltaTableBuilder = { this.identifier = identifier this } /** * :: Evolving :: * * Specify the table comment to describe the table. * * @param comment string table comment * @since 1.0.0 */ @Evolving def comment(comment: String): DeltaTableBuilder = { tblComment = Option(comment) this } /** * :: Evolving :: * * Specify the path to the directory where table data is stored, * which could be a path on distributed storage. * * @param location string the data location * @since 1.0.0 */ @Evolving def location(location: String): DeltaTableBuilder = { this.location = Option(location) this } /** * :: Evolving :: * * Specify a column. * * @param colName string the column name * @param dataType string the DDL data type * @since 1.0.0 */ @Evolving def addColumn(colName: String, dataType: String): DeltaTableBuilder = { addColumn( DeltaTable.columnBuilder(spark, colName).dataType(dataType).build() ) this } /** * :: Evolving :: * * Specify a column. * * @param colName string the column name * @param dataType dataType the DDL data type * @since 1.0.0 */ @Evolving def addColumn(colName: String, dataType: DataType): DeltaTableBuilder = { addColumn( DeltaTable.columnBuilder(spark, colName).dataType(dataType).build() ) this } /** * :: Evolving :: * * Specify a column. * * @param colName string the column name * @param dataType string the DDL data type * @param nullable boolean whether the column is nullable * @since 1.0.0 */ @Evolving def addColumn(colName: String, dataType: String, nullable: Boolean): DeltaTableBuilder = { addColumn( DeltaTable.columnBuilder(spark, colName).dataType(dataType).nullable(nullable).build() ) this } /** * :: Evolving :: * * Specify a column. * * @param colName string the column name * @param dataType dataType the DDL data type * @param nullable boolean whether the column is nullable * @since 1.0.0 */ @Evolving def addColumn(colName: String, dataType: DataType, nullable: Boolean): DeltaTableBuilder = { addColumn( DeltaTable.columnBuilder(spark, colName).dataType(dataType).nullable(nullable).build() ) this } /** * :: Evolving :: * * Specify a column. * * @param col structField the column struct * @since 1.0.0 */ @Evolving def addColumn(col: StructField): DeltaTableBuilder = { columns = columns :+ col this } /** * :: Evolving :: * * Specify columns with an existing schema. * * @param cols structType the existing schema for columns * @since 1.0.0 */ @Evolving def addColumns(cols: StructType): DeltaTableBuilder = { columns = columns ++ cols.toSeq this } /** * Validate that clusterBy is not used with partitionedBy. */ private def validatePartitioning(): Unit = { if (partitioningColumns.nonEmpty && clusteringColumns.nonEmpty) { throw DeltaErrors.clusterByWithPartitionedBy() } } /** * :: Evolving :: * * Specify the columns to partition the output on the file system. * * Note: This should only include table columns already defined in schema. * * @param colNames string* column names for partitioning * @since 1.0.0 */ @Evolving @scala.annotation.varargs def partitionedBy(colNames: String*): DeltaTableBuilder = { partitioningColumns = Option(colNames) validatePartitioning() this } /** * :: Evolving :: * * Specify the columns to cluster the output on the file system. * * Note: This should only include table columns already defined in schema. * * @param colNames string* column names for clustering * @since 3.2.0 */ @Evolving @scala.annotation.varargs def clusterBy(colNames: String*): DeltaTableBuilder = { clusteringColumns = Option(colNames) validatePartitioning() this } /** * :: Evolving :: * * Specify a key-value pair to tag the table definition. * * @param key string the table property key * @param value string the table property value * @since 1.0.0 */ @Evolving def property(key: String, value: String): DeltaTableBuilder = { this.properties = this.properties + (key -> value) this } /** * :: Evolving :: * * Execute the command to create / replace a Delta table and returns a instance of [[DeltaTable]]. * * @since 1.0.0 */ @Evolving def execute(): DeltaTable = withActiveSession(spark) { if (identifier == null && location.isEmpty) { throw DeltaErrors.createTableMissingTableNameOrLocation() } if (this.identifier == null) { identifier = s"delta.`${location.get}`" } // Return DeltaTable Object. val tableId: TableIdentifier = spark.sessionState.sqlParser.parseTableIdentifier(identifier) if (DeltaTableUtils.isValidPath(tableId) && location.nonEmpty && tableId.table != location.get) { throw DeltaErrors.createTableIdentifierLocationMismatch(identifier, location.get) } val table = spark.sessionState.sqlParser.parseMultipartIdentifier(identifier) val partitioning = partitioningColumns.map { colNames => colNames.map(name => DeltaTableUtils.parseColToTransform(name)) }.getOrElse(Seq.empty[Transform]) ++ (clusteringColumns.map { colNames => DeltaTableUtils.parseColsToClusterByTransform(colNames) }) val tableSpec = org.apache.spark.sql.catalyst.plans.logical.TableSpec( properties = properties, provider = Some(FORMAT_NAME), options = Map.empty, location = location, comment = tblComment, collation = None, serde = None, external = false) val stmt = builderOption match { case CreateTableOptions(ifNotExists) => val unresolvedTable = org.apache.spark.sql.catalyst.analysis.UnresolvedIdentifier(table) CreateTable( unresolvedTable, columns.map(ColumnDefinition.fromV1Column(_, spark.sessionState.sqlParser)).toSeq, partitioning, tableSpec, ifNotExists) case ReplaceTableOptions(orCreate) => val unresolvedTable = org.apache.spark.sql.catalyst.analysis.UnresolvedIdentifier(table) ReplaceTable( unresolvedTable, columns.map(ColumnDefinition.fromV1Column(_, spark.sessionState.sqlParser)).toSeq, partitioning, tableSpec, orCreate) } val qe = spark.sessionState.executePlan(stmt) // call `QueryExecution.toRDD` to trigger the execution of commands. SQLExecution.withNewExecutionId(qe, Some("create delta table"))(qe.toRdd) // Return DeltaTable Object. if (DeltaTableUtils.isValidPath(tableId)) { DeltaTable.forPath(spark, location.get) } else { DeltaTable.forName(spark, this.identifier) } } } ================================================ FILE: spark/src/main/scala/io/delta/tables/execution/DeltaConvert.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables.execution import org.apache.spark.sql.delta.DeltaTableUtils.withActiveSession import org.apache.spark.sql.delta.commands.ConvertToDeltaCommand import io.delta.tables.DeltaTable import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.types.StructType trait DeltaConvertBase { def executeConvert( spark: SparkSession, tableIdentifier: TableIdentifier, partitionSchema: Option[StructType], deltaPath: Option[String]): DeltaTable = withActiveSession(spark) { val cvt = ConvertToDeltaCommand(tableIdentifier, partitionSchema, collectStats = true, deltaPath) cvt.run(spark) if (cvt.isCatalogTable(spark.sessionState.analyzer, tableIdentifier)) { DeltaTable.forName(spark, tableIdentifier.toString) } else { DeltaTable.forPath(spark, tableIdentifier.table) } } } object DeltaConvert extends DeltaConvertBase {} ================================================ FILE: spark/src/main/scala/io/delta/tables/execution/DeltaTableBuilderOptions.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables.execution /** * DeltaTableBuilder option to indicate whether it's to create / replace the table. */ sealed trait DeltaTableBuilderOptions /** * Specify that the builder is to create a Delta table. * * @param ifNotExists boolean whether to ignore if the table already exists. */ case class CreateTableOptions(ifNotExists: Boolean) extends DeltaTableBuilderOptions /** * Specify that the builder is to replace a Delta table. * * @param orCreate boolean whether to create the table if the table doesn't exist. */ case class ReplaceTableOptions(orCreate: Boolean) extends DeltaTableBuilderOptions ================================================ FILE: spark/src/main/scala/io/delta/tables/execution/DeltaTableOperations.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables.execution import scala.collection.Map import org.apache.spark.sql.catalyst.TimeTravel import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.{DeltaErrors, DeltaLog} import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.DeltaTableUtils.withActiveSession import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.{DeltaGenerateCommand, DescribeDeltaDetailCommand, VacuumCommand} import org.apache.spark.sql.delta.util.AnalysisHelper import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.hadoop.fs.Path import org.apache.spark.sql.{functions, Column, DataFrame} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedRelation} import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.{Expression, Literal} import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation /** * Interface to provide the actual implementations of DeltaTable operations. */ trait DeltaTableOperations extends AnalysisHelper { self: io.delta.tables.DeltaTable => protected def executeDelete(condition: Option[Expression]): Unit = improveUnsupportedOpError { withActiveSession(sparkSession) { val delete = DeleteFromTable( self.toDF.queryExecution.analyzed, condition.getOrElse(Literal.TrueLiteral)) toDataset(sparkSession, delete) } } protected def executeHistory( deltaLog: DeltaLog, limit: Option[Int] = None, catalogTable: Option[CatalogTable] = None): DataFrame = withActiveSession(sparkSession) { val history = deltaLog.history sparkSession.createDataFrame(history.getHistory(limit, catalogTable)) } protected def executeDetails( path: String, tableIdentifier: Option[TableIdentifier]): DataFrame = withActiveSession(sparkSession) { val details = DescribeDeltaDetailCommand(Option(path), tableIdentifier, self.deltaLog.options) toDataset(sparkSession, details) } protected def executeGenerate( path: String, tableIdentifier: Option[TableIdentifier], mode: String): Unit = withActiveSession(sparkSession) { val generate = DeltaGenerateCommand(Option(path), tableIdentifier, mode, self.deltaLog.options) toDataset(sparkSession, generate) } protected def executeUpdate( set: Map[String, Column], condition: Option[Column]): Unit = improveUnsupportedOpError { withActiveSession(sparkSession) { val assignments = set.map { case (targetColName, column) => Assignment(UnresolvedAttribute.quotedString(targetColName), column.expr) }.toSeq val update = UpdateTable(self.toDF.queryExecution.analyzed, assignments, condition.map(_.expr)) toDataset(sparkSession, update) } } protected def executeVacuum( table: DeltaTableV2, retentionHours: Option[Double]): DataFrame = withActiveSession(sparkSession) { val tableId = table.getTableIdentifierIfExists val path = Option.when(tableId.isEmpty)(deltaLog.dataPath.toString) val vacuum = VacuumTableCommand( path, tableId, inventoryTable = None, inventoryQuery = None, retentionHours, dryRun = false, vacuumType = None, deltaLog.options) toDataset(sparkSession, vacuum) sparkSession.emptyDataFrame } protected def executeRestore( table: DeltaTableV2, versionAsOf: Option[Long], timestampAsOf: Option[String]): DataFrame = withActiveSession(sparkSession) { val identifier = table.getTableIdentifierIfExists.map( id => Identifier.of(id.database.toArray, id.table)) val sourceRelation = DataSourceV2Relation.create(table, None, identifier) val restore = RestoreTableStatement( TimeTravel( sourceRelation, timestampAsOf.map(Literal(_)), versionAsOf, Some("deltaTable")) ) toDataset(sparkSession, restore) } protected def executeClone( table: DeltaTableV2, target: String, isShallow: Boolean, replace: Boolean, properties: Map[String, String], versionAsOf: Option[Long] = None, timestampAsOf: Option[String] = None ): io.delta.tables.DeltaTable = withActiveSession(sparkSession) { if (!isShallow) { throw DeltaErrors.unsupportedDeepCloneException() } val sourceIdentifier = table.getTableIdentifierIfExists.map(id => Identifier.of(id.database.toArray, id.table)) val sourceRelation = DataSourceV2Relation.create(table, None, sourceIdentifier) val maybeTimeTravelSource = if (versionAsOf.isDefined || timestampAsOf.isDefined) { TimeTravel( sourceRelation, timestampAsOf.map(Literal(_)), versionAsOf, Some("deltaTable") ) } else { sourceRelation } val targetIsAbsolutePath = new Path(target).isAbsolute() val targetIdentifier = if (targetIsAbsolutePath) s"delta.`$target`" else target val targetRelation = UnresolvedRelation( sparkSession.sessionState.sqlParser.parseTableIdentifier(targetIdentifier)) val clone = CloneTableStatement( maybeTimeTravelSource, targetRelation, ifNotExists = false, replace, isCreateCommand = true, tablePropertyOverrides = properties.toMap, targetLocation = None) toDataset(sparkSession, clone) if (targetIsAbsolutePath) { io.delta.tables.DeltaTable.forPath(sparkSession, target) } else { io.delta.tables.DeltaTable.forName(sparkSession, target) } } protected def toStrColumnMap(map: Map[String, String]): Map[String, Column] = { map.toSeq.map { case (k, v) => k -> functions.expr(v) }.toMap } protected def sparkSession = self.toDF.sparkSession } ================================================ FILE: spark/src/main/scala/io/delta/tables/execution/VacuumTableCommand.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables.execution import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.UnresolvedTable import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.{DeltaErrors, DeltaLog, DeltaTableIdentifier, DeltaTableUtils, UnresolvedDeltaPathOrIdentifier} import org.apache.spark.sql.delta.commands.DeltaCommand import org.apache.spark.sql.delta.commands.VacuumCommand import org.apache.spark.sql.delta.commands.VacuumCommand.getDeltaTable import org.apache.spark.sql.execution.command.{LeafRunnableCommand, RunnableCommand} import org.apache.spark.sql.types.StringType /** * The `vacuum` command implementation for Spark SQL. Example SQL: * {{{ * VACUUM ('/path/to/dir' | delta.`/path/to/dir`) * [USING INVENTORY (delta.`/path/to/dir`| ( sub_query ))] * [RETAIN number HOURS] [DRY RUN]; * }}} */ case class VacuumTableCommand( override val child: LogicalPlan, horizonHours: Option[Double], inventoryTable: Option[LogicalPlan], inventoryQuery: Option[String], dryRun: Boolean, vacuumType: Option[String]) extends RunnableCommand with UnaryNode with DeltaCommand { override val output: Seq[Attribute] = Seq(AttributeReference("path", StringType, nullable = true)()) override protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = copy(child = newChild) override def run(sparkSession: SparkSession): Seq[Row] = { val deltaTable = getDeltaTable(child, "VACUUM") // The VACUUM command is only supported on existing delta tables. If the target table doesn't // exist or it is based on a partition directory, an exception will be thrown. if (!deltaTable.tableExists || deltaTable.hasPartitionFilters) { throw DeltaErrors.notADeltaTableException( "VACUUM", DeltaTableIdentifier(path = Some(deltaTable.path.toString))) } val inventory = inventoryTable.map(sparkSession.sessionState.analyzer.execute) .map(p => Some(getDeltaTable(p, "VACUUM").toDf(sparkSession))) .getOrElse(inventoryQuery.map(sparkSession.sql)) VacuumCommand.gc(sparkSession, deltaTable, dryRun, horizonHours, inventory, vacuumType).collect() } } object VacuumTableCommand { def apply( path: Option[String], table: Option[TableIdentifier], inventoryTable: Option[TableIdentifier], inventoryQuery: Option[String], horizonHours: Option[Double], dryRun: Boolean, vacuumType: Option[String], options: Map[String, String]): VacuumTableCommand = { val child = UnresolvedDeltaPathOrIdentifier(path, table, options, "VACUUM") val unresolvedInventoryTable = inventoryTable.map(rt => UnresolvedTable(rt.nameParts, "VACUUM")) VacuumTableCommand(child, horizonHours, unresolvedInventoryTable, inventoryQuery, dryRun, vacuumType) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/catalyst/TimeTravel.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst // scalastyle:off import.ordering.noEmptyLine import com.databricks.spark.util.DatabricksLogging import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan} /** * A logical node used to time travel the child relation to the given `timestamp` or `version`. * The `child` must support time travel, e.g. Delta, and cannot be a view, subquery or stream. * The timestamp expression cannot be a subquery. It must be a timestamp expression. * @param creationSource The API used to perform time travel, e.g. `atSyntax`, `dfReader` or SQL */ case class TimeTravel( relation: LogicalPlan, timestamp: Option[Expression], version: Option[Long], creationSource: Option[String]) extends LeafNode with DatabricksLogging { assert(version.isEmpty ^ timestamp.isEmpty, "Either the version or timestamp should be provided for time travel") override def output: Seq[Attribute] = Nil override lazy val resolved: Boolean = false } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregation/BitmapAggregator.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.expressions.aggregation import org.apache.spark.sql.delta.deletionvectors.{RoaringBitmapArray, RoaringBitmapArrayFormat} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{Expression, GenericInternalRow, ImplicitCastInputTypes} import org.apache.spark.sql.catalyst.expressions.aggregate.{ImperativeAggregate, TypedImperativeAggregate} import org.apache.spark.sql.catalyst.trees.UnaryLike import org.apache.spark.sql.types._ /** * This function returns a bitmap representing the set of values of the underlying column. * * The bitmap is simply a compressed representation of the set of all integral values that * appear in the column being aggregated over. * * @param child child expression that can produce a column value with `child.eval(inputRow)` */ case class BitmapAggregator( child: Expression, override val mutableAggBufferOffset: Int, override val inputAggBufferOffset: Int, // Take the format as string instead of [[RoaringBitmapArrayFormat.Value]], // because String is safe to serialize. serializationFormatString: String) extends TypedImperativeAggregate[RoaringBitmapArray] with ImplicitCastInputTypes with UnaryLike[Expression] { def this(child: Expression, serializationFormat: RoaringBitmapArrayFormat.Value) = this(child, 0, 0, serializationFormat.toString) override def createAggregationBuffer(): RoaringBitmapArray = new RoaringBitmapArray() override def update(buffer: RoaringBitmapArray, input: InternalRow): RoaringBitmapArray = { val value = child.eval(input) // Ignore empty rows if (value != null) { buffer.add(value.asInstanceOf[Long]) } buffer } override def merge(buffer: RoaringBitmapArray, input: RoaringBitmapArray): RoaringBitmapArray = { buffer.merge(input) buffer } /** * Return bitmap cardinality, last and serialized bitmap. */ override def eval(bitmapIntegerSet: RoaringBitmapArray): GenericInternalRow = { // reduce the serialized size via RLE optimisation bitmapIntegerSet.runOptimize() new GenericInternalRow(Array( bitmapIntegerSet.cardinality, bitmapIntegerSet.last.getOrElse(null), serialize(bitmapIntegerSet))) } override def serialize(buffer: RoaringBitmapArray): Array[Byte] = { val serializationFormat = RoaringBitmapArrayFormat.withName(serializationFormatString) buffer.serializeAsByteArray(serializationFormat) } override def deserialize(storageFormat: Array[Byte]): RoaringBitmapArray = { RoaringBitmapArray.readFrom(storageFormat) } override def withNewMutableAggBufferOffset(newMutableAggBufferOffset: Int) : ImperativeAggregate = copy(mutableAggBufferOffset = newMutableAggBufferOffset) override def withNewInputAggBufferOffset(newInputAggBufferOffset: Int) : ImperativeAggregate = copy(inputAggBufferOffset = newInputAggBufferOffset) override def nullable: Boolean = false override def dataType: StructType = StructType( Seq( StructField("cardinality", LongType), StructField("last", LongType), StructField("bitmap", BinaryType) ) ) override def inputTypes: Seq[AbstractDataType] = Seq(LongType) override protected def withNewChildInternal(newChild: Expression): BitmapAggregator = copy(child = newChild) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/CloneTableStatement.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.plans.logical import org.apache.spark.sql.catalyst.expressions.Attribute /** * CLONE TABLE statement, as parsed from SQL * * @param source source plan for table to be cloned * @param target target path or table name where clone should be instantiated * @param ifNotExists if a table exists at the target, we should not go through with the clone * @param isReplaceCommand when true, replace the target table if one exists * @param isCreateCommand when true, create the target table if none exists * @param tablePropertyOverrides user-defined table properties that should override any properties * with the same key from the source table * @param targetLocation if target is a table name then user can provide a targetLocation to * create an external table with this location */ case class CloneTableStatement( source: LogicalPlan, target: LogicalPlan, ifNotExists: Boolean, isReplaceCommand: Boolean, isCreateCommand: Boolean, tablePropertyOverrides: Map[String, String], targetLocation: Option[String]) extends BinaryNode { override def output: Seq[Attribute] = Nil override def left: LogicalPlan = source override def right: LogicalPlan = target override protected def withNewChildrenInternal( newLeft: LogicalPlan, newRight: LogicalPlan): CloneTableStatement = copy(source = newLeft, target = newRight) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DeltaDelete.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.plans.logical import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} // This only used by Delta which needs to be compatible with DBR 6 and can't use the new class // added in Spark 3.0: `DeleteFromTable`. case class DeltaDelete( child: LogicalPlan, condition: Option[Expression]) extends UnaryNode { override def output: Seq[Attribute] = Seq.empty override protected def withNewChildInternal(newChild: LogicalPlan): DeltaDelete = copy(child = newChild) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DeltaUpdateTable.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.plans.logical import org.apache.spark.sql.delta.DeltaAnalysisException import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, Expression, ExtractValue, GetStructField} /** * Perform UPDATE on a table * * @param child the logical plan representing target table * @param updateColumns: the to-be-updated target columns * @param updateExpressions: the corresponding update expression if the condition is matched * @param condition: Only rows that match the condition will be updated */ case class DeltaUpdateTable( child: LogicalPlan, updateColumns: Seq[Expression], updateExpressions: Seq[Expression], condition: Option[Expression]) extends UnaryNode { assert(updateColumns.size == updateExpressions.size) override def output: Seq[Attribute] = Seq.empty override protected def withNewChildInternal(newChild: LogicalPlan): DeltaUpdateTable = copy(child = newChild) } object DeltaUpdateTable { /** * Extracts name parts from a resolved expression referring to a nested or non-nested column * - For non-nested column, the resolved expression will be like `AttributeReference(...)`. * - For nested column, the resolved expression will be like `Alias(GetStructField(...))`. * * In the nested case, the function recursively traverses through the expression to find * the name parts. For example, a nested field of a.b.c would be resolved to an expression * * `Alias(c, GetStructField(c, GetStructField(b, AttributeReference(a)))` * * for which this method recursively extracts the name parts as follows: * * `Alias(c, GetStructField(c, GetStructField(b, AttributeReference(a)))` * -> `GetStructField(c, GetStructField(b, AttributeReference(a)))` * -> `GetStructField(b, AttributeReference(a))` ++ Seq(c) * -> `AttributeReference(a)` ++ Seq(b, c) * -> [a, b, c] */ def getTargetColNameParts( resolvedTargetCol: Expression, errMsg: => String = null): Seq[String] = { def extractRecursively(expr: Expression): Seq[String] = expr match { case attr: Attribute => Seq(attr.name) case Alias(c, _) => extractRecursively(c) case GetStructField(c, _, Some(name)) => extractRecursively(c) :+ name case _: ExtractValue => throw new DeltaAnalysisException( errorClass = "_LEGACY_ERROR_TEMP_DELTA_0009", messageParameters = Array(Option(errMsg).map(_ + " - ").getOrElse("")) ) case other => throw new DeltaAnalysisException( errorClass = "_LEGACY_ERROR_TEMP_DELTA_0010", messageParameters = Array(Option(errMsg).map(_ + " - ").getOrElse(""), other.sql) ) } extractRecursively(resolvedTargetCol) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RestoreTableStatement.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.plans.logical import org.apache.spark.sql.catalyst.TimeTravel import org.apache.spark.sql.catalyst.expressions.Attribute /** * RESTORE TABLE statement as parsed from SQL * * @param table - logical node of the table that will be restored, internally contains either * version or timestamp. */ case class RestoreTableStatement(table: TimeTravel) extends UnaryNode { override def child: LogicalPlan = table override def output: Seq[Attribute] = Nil override protected def withNewChildInternal(newChild: LogicalPlan): RestoreTableStatement = copy(table = newChild.asInstanceOf[TimeTravel]) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SyncIdentity.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.plans.logical import org.apache.spark.sql.catalyst.analysis.FieldName import org.apache.spark.sql.connector.catalog.TableChange import org.apache.spark.sql.connector.catalog.TableChange.ColumnChange /** * A `ColumnChange` to model `ALTER TABLE ... ALTER (CHANGE) COLUMN ... SYNC IDENTITY` command. * * @param fieldNames The (potentially nested) column name. */ case class SyncIdentity(fieldNames: Array[String]) extends ColumnChange { require(fieldNames.size == 1, "IDENTITY column cannot be a nested column.") } case class AlterColumnSyncIdentity(table: LogicalPlan, column: FieldName) extends AlterTableCommand { override def changes: Seq[TableChange] = { require(column.resolved, "FieldName should be resolved before it's converted to TableChange.") val colName = column.name.toArray Seq(SyncIdentity(colName)) } override protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = copy(table = newChild) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/deltaConstraints.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.plans.logical import org.apache.spark.sql.delta.constraints.{AddConstraint => AddDeltaConstraint, DropConstraint => DropDeltaConstraint} // Aliased to avoid conflicts with Spark's AddConstraint/DropConstraint import org.apache.spark.sql.connector.catalog.TableChange /** * The logical plan of the ALTER TABLE ... ADD CONSTRAINT command. */ case class AlterTableAddConstraint( table: LogicalPlan, constraintName: String, expr: String) extends AlterTableCommand { override def changes: Seq[TableChange] = Seq(AddDeltaConstraint(constraintName, expr)) protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = copy(table = newChild) } /** * The logical plan of the ALTER TABLE ... DROP CONSTRAINT command. */ case class AlterTableDropConstraint( table: LogicalPlan, constraintName: String, ifExists: Boolean) extends AlterTableCommand { override def changes: Seq[TableChange] = Seq(DropDeltaConstraint(constraintName, ifExists)) protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = copy(table = newChild) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/deltaMerge.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.plans.logical import org.apache.spark.sql.delta.{DeltaAnalysisException, DeltaIllegalArgumentException, DeltaUnsupportedOperationException} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis._ import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, UnaryExpression} import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} import org.apache.spark.sql.types.{DataType, StructType} /** * A copy of Spark SQL Unevaluable for cross-version compatibility. In 3.0, implementers of * the original Unevaluable must explicitly override foldable to false; in 3.1 onwards, this * explicit override is invalid. */ trait DeltaUnevaluable extends Expression { final override def foldable: Boolean = false final override def eval(input: InternalRow = null): Any = { throw new DeltaUnsupportedOperationException( errorClass = "DELTA_CANNOT_EVALUATE_EXPRESSION", messageParameters = Array(s"$this") ) } final override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = throw new DeltaUnsupportedOperationException( errorClass = "DELTA_CANNOT_GENERATE_CODE_FOR_EXPRESSION", messageParameters = Array(s"$this") ) } /** * Determines the nullness of target-only struct fields when generating target struct expressions * to align with the evolved target schema during MERGE operations with schema evolution enabled. * * Target-only struct fields are nested fields that exist in the target table's struct column * but not in the corresponding source struct column. For example, if the target has * `struct(a, b, c)` and the source has `struct(a, b)`, then field `c` is target-only. * * This behavior only applies when schema evolution is enabled, as target-only fields are not * allowed when schema evolution is disabled. */ object TargetOnlyStructFieldBehavior extends Enumeration { type TargetOnlyStructFieldBehavior = Value /** * Preserve target-only struct fields with their original values from the target table. * Used for: `UPDATE * [EXCEPT]` clauses with schema evolution enabled. * Example: Target row has `struct(a=1, b=2, c=3)`, source has `struct(a=10, b=20)`. * Result: `struct(a=10, b=20, c=3)` - field `c` is preserved. */ val PRESERVE = Value /** * Overwrite target-only struct fields with null. * Used for: Explicit column assignments (e.g., `UPDATE SET col = expr`), * `INSERT` clauses (no existing row to preserve values from) * Example: Target row has `struct(a=1, b=2, c=3)`, source has `struct(a=10, b=20)`. * Result: `struct(a=10, b=20, c=null)` - field `c` is set to null. */ val NULLIFY = Value /** * The expression has been fully resolved and aligned to the evolved target schema. */ val TARGET_ALIGNED = Value } /** * Represents an action in MERGE's UPDATE or INSERT clause where a target columns is assigned the * value of an expression * * @param targetColNameParts The name parts of the target column. This is a sequence to support * nested fields as targets. * @param expr Expression to generate the value of the target column. * @param targetOnlyStructFieldBehavior Determines the nullness of target-only struct fields. * Note: This parameter only takes effect when schema evolution * is enabled; otherwise it is ignored. * @param targetColNameResolved Whether the targetColNameParts have undergone resolution and checks * for validity. */ case class DeltaMergeAction( targetColNameParts: Seq[String], expr: Expression, targetOnlyStructFieldBehavior: TargetOnlyStructFieldBehavior.Value, targetColNameResolved: Boolean = false) extends UnaryExpression with DeltaUnevaluable { override def child: Expression = expr override def dataType: DataType = expr.dataType override lazy val resolved: Boolean = { childrenResolved && checkInputDataTypes().isSuccess && targetColNameResolved } override def sql: String = s"$targetColString = ${expr.sql}" override def toString: String = s"$targetColString = $expr" private lazy val targetColString: String = targetColNameParts.mkString("`", "`.`", "`") override protected def withNewChildInternal(newChild: Expression): DeltaMergeAction = copy(expr = newChild) } /** * Trait that represents a WHEN clause in MERGE. See [[DeltaMergeInto]]. It extends [[Expression]] * so that Catalyst can find all the expressions in the clause implementations. */ sealed trait DeltaMergeIntoClause extends Expression with DeltaUnevaluable { /** Optional condition of the clause */ def condition: Option[Expression] /** * Sequence of actions represented as expressions. Note that this can be only be either * UnresolvedStar, or MergeAction. */ def actions: Seq[Expression] /** * Sequence of resolved actions represented as Aliases. Actions, once resolved, must * be Aliases and not any other NamedExpressions. So it should be safe to do this casting * as long as this is called after the clause has been resolved. */ def resolvedActions: Seq[DeltaMergeAction] = { assert(actions.forall(_.resolved), "all actions have not been resolved yet") actions.map(_.asInstanceOf[DeltaMergeAction]) } /** * String representation of the clause type: Update, Delete or Insert. */ def clauseType: String override def toString: String = { val condStr = condition.map { c => s"condition: $c" } val actionStr = if (actions.isEmpty) None else { Some("actions: " + actions.mkString("[", ", ", "]")) } s"$clauseType " + Seq(condStr, actionStr).flatten.mkString("[", ", ", "]") } override def nullable: Boolean = false override def dataType: DataType = null override def children: Seq[Expression] = condition.toSeq ++ actions /** Verify whether the expressions in the actions are of the right type */ protected[logical] def verifyActions(): Unit = actions.foreach { case _: UnresolvedStar => case _: DeltaMergeAction => case a => throw new DeltaIllegalArgumentException( errorClass = "DELTA_UNEXPECTED_ACTION_EXPRESSION", messageParameters = Array(s"$a")) } } object DeltaMergeIntoClause { /** * Convert the parsed columns names and expressions into action for MergeInto. Note: * - Size of column names and expressions must be the same. * - If the sizes are zeros and `emptySeqIsStar` is true, this function assumes * that query had `*` as an action, and therefore generates a single action * with `UnresolvedStar`. This will be expanded later during analysis. * - Otherwise, this will convert the names and expressions to MergeActions. */ def toActions( colNames: Seq[UnresolvedAttribute], exprs: Seq[Expression], isEmptySeqEqualToStar: Boolean = true): Seq[Expression] = { assert(colNames.size == exprs.size) if (colNames.isEmpty && isEmptySeqEqualToStar) { Seq(UnresolvedStar(None)) } else { (colNames, exprs).zipped.map { (col, expr) => DeltaMergeAction( targetColNameParts = col.nameParts, expr = expr, // Explicit column assignments overwrite target-only struct fields with null. targetOnlyStructFieldBehavior = TargetOnlyStructFieldBehavior.NULLIFY) } } } def toActions(assignments: Seq[Assignment]): Seq[Expression] = { if (assignments.isEmpty) { Seq[Expression](UnresolvedStar(None)) } else { assignments.map { case Assignment(key: UnresolvedAttribute, expr) => DeltaMergeAction( targetColNameParts = key.nameParts, expr = expr, // Explicit column assignments overwrite target-only struct fields with null. targetOnlyStructFieldBehavior = TargetOnlyStructFieldBehavior.NULLIFY) case Assignment(key: Attribute, expr) => DeltaMergeAction( targetColNameParts = Seq(key.name), expr = expr, // Explicit column assignments overwrite target-only struct fields with null. targetOnlyStructFieldBehavior = TargetOnlyStructFieldBehavior.NULLIFY) case other => throw new DeltaAnalysisException( errorClass = "DELTA_MERGE_UNEXPECTED_ASSIGNMENT_KEY", messageParameters = Array(s"${other.getClass}", s"$other")) } } } } /** Trait that represents WHEN MATCHED clause in MERGE. See [[DeltaMergeInto]]. */ sealed trait DeltaMergeIntoMatchedClause extends DeltaMergeIntoClause /** Represents the clause WHEN MATCHED THEN UPDATE in MERGE. See [[DeltaMergeInto]]. */ case class DeltaMergeIntoMatchedUpdateClause( condition: Option[Expression], actions: Seq[Expression]) extends DeltaMergeIntoMatchedClause { def this(cond: Option[Expression], cols: Seq[UnresolvedAttribute], exprs: Seq[Expression]) = this(cond, DeltaMergeIntoClause.toActions(cols, exprs)) override def clauseType: String = "Update" override protected def withNewChildrenInternal( newChildren: IndexedSeq[Expression]): DeltaMergeIntoMatchedUpdateClause = { if (condition.isDefined) { copy(condition = Some(newChildren.head), actions = newChildren.tail) } else { copy(condition = None, actions = newChildren) } } } /** Represents the clause WHEN MATCHED THEN DELETE in MERGE. See [[DeltaMergeInto]]. */ case class DeltaMergeIntoMatchedDeleteClause(condition: Option[Expression]) extends DeltaMergeIntoMatchedClause { def this(condition: Option[Expression], actions: Seq[DeltaMergeAction]) = this(condition) override def clauseType: String = "Delete" override def actions: Seq[Expression] = Seq.empty override protected def withNewChildrenInternal( newChildren: IndexedSeq[Expression]): DeltaMergeIntoMatchedDeleteClause = copy(condition = if (condition.isDefined) Some(newChildren.head) else None) } /** Trait that represents WHEN NOT MATCHED clause in MERGE. See [[DeltaMergeInto]]. */ sealed trait DeltaMergeIntoNotMatchedClause extends DeltaMergeIntoClause /** Represents the clause WHEN NOT MATCHED THEN INSERT in MERGE. See [[DeltaMergeInto]]. */ case class DeltaMergeIntoNotMatchedInsertClause( condition: Option[Expression], actions: Seq[Expression]) extends DeltaMergeIntoNotMatchedClause { def this(cond: Option[Expression], cols: Seq[UnresolvedAttribute], exprs: Seq[Expression]) = this(cond, DeltaMergeIntoClause.toActions(cols, exprs)) override def clauseType: String = "Insert" override protected def withNewChildrenInternal( newChildren: IndexedSeq[Expression]): DeltaMergeIntoNotMatchedInsertClause = if (condition.isDefined) { copy(condition = Some(newChildren.head), actions = newChildren.tail) } else { copy(condition = None, actions = newChildren) } } /** Trait that represents WHEN NOT MATCHED BY SOURCE clause in MERGE. See [[DeltaMergeInto]]. */ sealed trait DeltaMergeIntoNotMatchedBySourceClause extends DeltaMergeIntoClause /** Represents the clause WHEN NOT MATCHED BY SOURCE THEN UPDATE in MERGE. See * [[DeltaMergeInto]]. */ case class DeltaMergeIntoNotMatchedBySourceUpdateClause( condition: Option[Expression], actions: Seq[Expression]) extends DeltaMergeIntoNotMatchedBySourceClause { def this(cond: Option[Expression], cols: Seq[UnresolvedAttribute], exprs: Seq[Expression]) = this(cond, DeltaMergeIntoClause.toActions(cols, exprs)) override def clauseType: String = "Update" override protected def withNewChildrenInternal( newChildren: IndexedSeq[Expression]): DeltaMergeIntoNotMatchedBySourceUpdateClause = { if (condition.isDefined) { copy(condition = Some(newChildren.head), actions = newChildren.tail) } else { copy(condition = None, actions = newChildren) } } } /** Represents the clause WHEN NOT MATCHED BY SOURCE THEN DELETE in MERGE. See * [[DeltaMergeInto]]. */ case class DeltaMergeIntoNotMatchedBySourceDeleteClause(condition: Option[Expression]) extends DeltaMergeIntoNotMatchedBySourceClause { def this(condition: Option[Expression], actions: Seq[DeltaMergeAction]) = this(condition) override def clauseType: String = "Delete" override def actions: Seq[Expression] = Seq.empty override protected def withNewChildrenInternal( newChildren: IndexedSeq[Expression]): DeltaMergeIntoNotMatchedBySourceDeleteClause = copy(condition = if (condition.isDefined) Some(newChildren.head) else None) } /** * Merges changes specified in the source plan into a target table, based on the given search * condition and the actions to perform when the condition is matched or not matched by the rows. * * The syntax of the MERGE statement is as follows. * {{{ * MERGE [WITH SCHEMA EVOLUTION] INTO * USING * ON * [ WHEN MATCHED [ AND ] THEN ] * [ WHEN MATCHED [ AND ] THEN ] * ... * [ WHEN NOT MATCHED [BY TARGET] [ AND ] THEN ] * [ WHEN NOT MATCHED [BY TARGET] [ AND ] THEN ] * ... * [ WHEN NOT MATCHED BY SOURCE [ AND ] THEN ] * [ WHEN NOT MATCHED BY SOURCE [ AND ] THEN ] * ... * * * where * = * DELETE | * UPDATE SET column1 = value1 [, column2 = value2 ...] | * UPDATE SET * [EXCEPT (column1, ...)] * = INSERT (column1 [, column2 ...]) VALUES (expr1 [, expr2 ...]) * = * DELETE | * UPDATE SET column1 = value1 [, column2 = value2 ...] * }}} * * - There can be any number of WHEN clauses. * - WHEN MATCHED clauses: * - Each WHEN MATCHED clause can have an optional condition. However, if there are multiple * WHEN MATCHED clauses, only the last can omit the condition. * - WHEN MATCHED clauses are dependent on their ordering; that is, the first clause that * satisfies the clause's condition has its corresponding action executed. * - WHEN NOT MATCHED clause: * - Can only have the INSERT action. If present, they must follow the last WHEN MATCHED clause. * - Each WHEN NOT MATCHED clause can have an optional condition. However, if there are multiple * clauses, only the last can omit the condition. * - WHEN NOT MATCHED clauses are dependent on their ordering; that is, the first clause that * satisfies the clause's condition has its corresponding action executed. * - WHEN NOT MATCHED BY SOURCE clauses: * - Each WHEN NOT MATCHED BY SOURCE clause can have an optional condition. However, if there are * multiple WHEN NOT MATCHED BY SOURCE clauses, only the last can omit the condition. * - WHEN NOT MATCHED BY SOURCE clauses are dependent on their ordering; that is, the first * clause that satisfies the clause's condition has its corresponding action executed. */ case class DeltaMergeInto( target: LogicalPlan, source: LogicalPlan, condition: Expression, matchedClauses: Seq[DeltaMergeIntoMatchedClause], notMatchedClauses: Seq[DeltaMergeIntoNotMatchedClause], notMatchedBySourceClauses: Seq[DeltaMergeIntoNotMatchedBySourceClause], withSchemaEvolution: Boolean, finalSchema: Option[StructType]) extends Command with SupportsSubquery { (matchedClauses ++ notMatchedClauses ++ notMatchedBySourceClauses).foreach(_.verifyActions()) // TODO: extend BinaryCommand once the new Spark version is released override def children: Seq[LogicalPlan] = Seq(target, source) override def output: Seq[Attribute] = Seq.empty override protected def withNewChildrenInternal( newChildren: IndexedSeq[LogicalPlan]): DeltaMergeInto = copy(target = newChildren(0), source = newChildren(1)) } object DeltaMergeInto { def apply( target: LogicalPlan, source: LogicalPlan, condition: Expression, whenClauses: Seq[DeltaMergeIntoClause], withSchemaEvolution: Boolean): DeltaMergeInto = { val notMatchedClauses = whenClauses.collect { case x: DeltaMergeIntoNotMatchedClause => x } val matchedClauses = whenClauses.collect { case x: DeltaMergeIntoMatchedClause => x } val notMatchedBySourceClauses = whenClauses.collect { case x: DeltaMergeIntoNotMatchedBySourceClause => x } // grammar enforcement goes here. if (whenClauses.isEmpty) { throw new DeltaAnalysisException( errorClass = "DELTA_MERGE_MISSING_WHEN", messageParameters = Array.empty ) } // Check that only the last MATCHED clause omits the condition. if (matchedClauses.length > 1 && !matchedClauses.init.forall(_.condition.nonEmpty)) { throw new DeltaAnalysisException( errorClass = "DELTA_NON_LAST_MATCHED_CLAUSE_OMIT_CONDITION", messageParameters = Array.empty) } // Check that only the last NOT MATCHED clause omits the condition. if (notMatchedClauses.length > 1 && !notMatchedClauses.init.forall(_.condition.nonEmpty)) { throw new DeltaAnalysisException( errorClass = "DELTA_NON_LAST_NOT_MATCHED_CLAUSE_OMIT_CONDITION", messageParameters = Array.empty) } // Check that only the last NOT MATCHED BY SOURCE clause omits the condition. if (notMatchedBySourceClauses.length > 1 && !notMatchedBySourceClauses.init.forall(_.condition.nonEmpty)) { throw new DeltaAnalysisException( errorClass = "DELTA_NON_LAST_NOT_MATCHED_BY_SOURCE_CLAUSE_OMIT_CONDITION", messageParameters = Array.empty) } DeltaMergeInto( target, source, condition, matchedClauses, notMatchedClauses, notMatchedBySourceClauses, withSchemaEvolution, finalSchema = Some(target.schema)) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/deltaTableFeatures.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.plans.logical import org.apache.spark.sql.delta.tablefeatures.DropFeature import org.apache.spark.sql.connector.catalog.TableChange /** * The logical plan of the ALTER TABLE ... DROP FEATURE command. */ case class AlterTableDropFeature( table: LogicalPlan, featureName: String, truncateHistory: Boolean) extends AlterTableCommand { override def changes: Seq[TableChange] = Seq(DropFeature(featureName, truncateHistory)) protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = copy(table = newChild) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/AllowedUserProvidedExpressions.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.reflect.ClassTag import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.xml._ /** This class defines the list of expressions that can be used when providing custom expressions. * e.g. in a generated column or a check constraint. * */ object AllowedUserProvidedExpressions { /** * This method has the same signature as `FunctionRegistry.expression` so that we can define the * list in the same format as `FunctionRegistry.expressions` and that's easy to diff. */ private def expression[T <: Expression : ClassTag]( name: String, setAlias: Boolean = false): Class[_] = { implicitly[ClassTag[T]].runtimeClass } // scalastyle:off /** * The white list is copied from `FunctionRegistry.expressions()` except the following types of * functions: * - explode functions. In other words, generate multiple rows from one row. * - aggerate functions. * - window functions. * - grouping sets. * - non deterministic functions. * - deterministic functions in one query but non deterministic in multiple queries, * such as, current_timestamp, rand, etc. * * To review the difference, you can run * `diff sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala sql/core/src/main/scala/com/databricks/sql/transaction/SupportedGenerationExpression.scala` */ // scalastyle:on val expressions: Set[Class[_]] = Set( // misc non-aggregate functions expression[Abs]("abs"), expression[Coalesce]("coalesce"), expression[Greatest]("greatest"), expression[If]("if"), expression[If]("iff", true), expression[IsNaN]("isnan"), expression[Nvl]("ifnull", true), expression[IsNull]("isnull"), expression[IsNotNull]("isnotnull"), expression[Least]("least"), expression[NaNvl]("nanvl"), expression[NullIf]("nullif"), expression[Nvl]("nvl"), expression[Nvl2]("nvl2"), expression[CaseWhen]("when"), // math functions expression[Acos]("acos"), expression[Acosh]("acosh"), expression[Asin]("asin"), expression[Asinh]("asinh"), expression[Atan]("atan"), expression[Atan2]("atan2"), expression[Atanh]("atanh"), expression[Bin]("bin"), expression[BRound]("bround"), expression[Cbrt]("cbrt"), expression[Ceil]("ceil"), expression[Ceil]("ceiling", true), expression[Cos]("cos"), expression[Cosh]("cosh"), expression[Conv]("conv"), expression[ToDegrees]("degrees"), expression[EulerNumber]("e"), expression[Exp]("exp"), expression[Expm1]("expm1"), expression[Floor]("floor"), expression[Factorial]("factorial"), expression[Hex]("hex"), expression[Hypot]("hypot"), expression[Logarithm]("log"), expression[Log10]("log10"), expression[Log1p]("log1p"), expression[Log2]("log2"), expression[Log]("ln"), expression[Remainder]("mod", true), expression[UnaryMinus]("negative", true), expression[Pi]("pi"), expression[Pmod]("pmod"), expression[UnaryPositive]("positive"), expression[Pow]("pow", true), expression[Pow]("power"), expression[ToRadians]("radians"), expression[Rint]("rint"), expression[Round]("round"), expression[ShiftLeft]("shiftleft"), expression[ShiftRight]("shiftright"), expression[ShiftRightUnsigned]("shiftrightunsigned"), expression[Signum]("sign", true), expression[Signum]("signum"), expression[Sin]("sin"), expression[Sinh]("sinh"), expression[StringToMap]("str_to_map"), expression[Sqrt]("sqrt"), expression[Tan]("tan"), expression[Cot]("cot"), expression[Tanh]("tanh"), expression[Add]("+"), expression[Subtract]("-"), expression[Multiply]("*"), expression[Divide]("/"), expression[IntegralDivide]("div"), expression[Remainder]("%"), // string functions expression[Ascii]("ascii"), expression[Chr]("char", true), expression[Chr]("chr"), expression[Base64]("base64"), expression[BitLength]("bit_length"), expression[Length]("char_length", true), expression[Length]("character_length", true), expression[ConcatWs]("concat_ws"), expression[Decode]("decode"), expression[Elt]("elt"), expression[Encode]("encode"), expression[FindInSet]("find_in_set"), expression[FormatNumber]("format_number"), expression[FormatString]("format_string"), expression[GetJsonObject]("get_json_object"), expression[InitCap]("initcap"), expression[StringInstr]("instr"), expression[Lower]("lcase", true), expression[Length]("length"), expression[Levenshtein]("levenshtein"), expression[Like]("like"), expression[Lower]("lower"), expression[OctetLength]("octet_length"), expression[StringLocate]("locate"), expression[StringLPad]("lpad"), expression[StringTrimLeft]("ltrim"), expression[JsonTuple]("json_tuple"), expression[ParseUrl]("parse_url"), expression[StringLocate]("position", true), expression[StringLocate]("charindex", true), expression[FormatString]("printf", true), expression[RegExpExtract]("regexp_extract"), expression[RegExpReplace]("regexp_replace"), expression[RLike]("regexp_like", true), expression[StringRepeat]("repeat"), expression[StringReplace]("replace"), expression[Overlay]("overlay"), expression[RLike]("rlike"), expression[StringRPad]("rpad"), expression[StringTrimRight]("rtrim"), expression[Sentences]("sentences"), expression[SoundEx]("soundex"), expression[StringSpace]("space"), expression[StringSplit]("split"), expression[Substring]("substr", true), expression[Substring]("substring"), expression[Left]("left"), expression[Right]("right"), expression[SubstringIndex]("substring_index"), expression[StringTranslate]("translate"), expression[StringTrim]("trim"), expression[Upper]("ucase", true), expression[UnBase64]("unbase64"), expression[Unhex]("unhex"), expression[Upper]("upper"), expression[XPathList]("xpath"), expression[XPathBoolean]("xpath_boolean"), expression[XPathDouble]("xpath_double"), expression[XPathDouble]("xpath_number", true), expression[XPathFloat]("xpath_float"), expression[XPathInt]("xpath_int"), expression[XPathLong]("xpath_long"), expression[XPathShort]("xpath_short"), expression[XPathString]("xpath_string"), // datetime functions expression[AddMonths]("add_months"), expression[DateDiff]("datediff"), expression[DateAdd]("date_add"), expression[DateFormatClass]("date_format"), expression[DateSub]("date_sub"), expression[DayOfMonth]("day", true), expression[DayOfYear]("dayofyear"), expression[DayOfMonth]("dayofmonth"), expression[FromUnixTime]("from_unixtime"), expression[FromUTCTimestamp]("from_utc_timestamp"), expression[Hour]("hour"), expression[LastDay]("last_day"), expression[Minute]("minute"), expression[Month]("month"), expression[MonthsBetween]("months_between"), expression[NextDay]("next_day"), expression[Now]("now"), expression[Quarter]("quarter"), expression[Second]("second"), expression[TimestampAdd]("timestampadd"), expression[TimestampDiff]("timestampdiff"), expression[ParseToTimestamp]("to_timestamp"), expression[ParseToDate]("to_date"), // `gettimestamp` is not a Spark built-in class but `ParseToDate` will refer to // `gettimestamp` when a format is given, so it needs to be on the allowed list expression[GetTimestamp]("gettimestamp"), expression[ToUnixTimestamp]("to_unix_timestamp"), expression[ToUTCTimestamp]("to_utc_timestamp"), expression[TruncDate]("trunc"), expression[TruncTimestamp]("date_trunc"), expression[UnixTimestamp]("unix_timestamp"), expression[DayOfWeek]("dayofweek"), expression[WeekDay]("weekday"), expression[WeekOfYear]("weekofyear"), expression[Year]("year"), expression[TimeWindow]("window"), expression[MakeDate]("make_date"), expression[MakeTimestamp]("make_timestamp"), expression[MakeInterval]("make_interval"), expression[Extract]("date_part", setAlias = true), expression[Extract]("extract"), // collection functions expression[CreateArray]("array"), expression[ArrayContains]("array_contains"), expression[ArraysOverlap]("arrays_overlap"), expression[ArrayIntersect]("array_intersect"), expression[ArrayJoin]("array_join"), expression[ArrayPosition]("array_position"), expression[ArraySort]("array_sort"), expression[ArrayExcept]("array_except"), expression[ArrayUnion]("array_union"), expression[CreateMap]("map"), expression[CreateNamedStruct]("named_struct"), expression[ElementAt]("element_at"), expression[MapFromArrays]("map_from_arrays"), expression[MapKeys]("map_keys"), expression[MapValues]("map_values"), expression[MapEntries]("map_entries"), expression[MapFromEntries]("map_from_entries"), expression[MapConcat]("map_concat"), expression[Size]("size"), expression[Slice]("slice"), expression[Size]("cardinality", true), expression[ArraysZip]("arrays_zip"), expression[SortArray]("sort_array"), expression[ArrayMin]("array_min"), expression[ArrayMax]("array_max"), expression[Reverse]("reverse"), expression[Concat]("concat"), expression[Flatten]("flatten"), expression[Sequence]("sequence"), expression[ArrayRepeat]("array_repeat"), expression[ArrayRemove]("array_remove"), expression[ArrayDistinct]("array_distinct"), expression[ArrayTransform]("transform"), expression[MapFilter]("map_filter"), expression[ArrayFilter]("filter"), expression[ArrayExists]("exists"), expression[ArrayForAll]("forall"), expression[ArrayAggregate]("aggregate"), expression[ArrayAggregate]("reduce"), expression[TransformValues]("transform_values"), expression[TransformKeys]("transform_keys"), expression[MapZipWith]("map_zip_with"), expression[ZipWith]("zip_with"), // misc functions expression[AssertTrue]("assert_true"), expression[Crc32]("crc32"), expression[Md5]("md5"), expression[Murmur3Hash]("hash"), expression[XxHash64]("xxhash64"), expression[Sha1]("sha", true), expression[Sha1]("sha1"), expression[Sha2]("sha2"), expression[TypeOf]("typeof"), // predicates expression[And]("and"), expression[In]("in"), expression[Not]("not"), expression[Or]("or"), // comparison operators expression[EqualNullSafe]("<=>"), expression[EqualTo]("="), expression[EqualTo]("=="), expression[GreaterThan](">"), expression[GreaterThanOrEqual](">="), expression[LessThan]("<"), expression[LessThanOrEqual]("<="), expression[Not]("!"), // bitwise expression[BitwiseAnd]("&"), expression[BitwiseNot]("~"), expression[BitwiseOr]("|"), expression[BitwiseXor]("^"), expression[BitwiseCount]("bit_count"), // json expression[StructsToJson]("to_json"), expression[JsonToStructs]("from_json"), expression[SchemaOfJson]("schema_of_json"), // cast expression[Cast]("cast"), // We don't need to define `castAlias` since they will use the same `Cast` expression. // csv expression[CsvToStructs]("from_csv"), expression[SchemaOfCsv]("schema_of_csv"), expression[StructsToCsv]("to_csv"), // Special expressions that are not built-in expressions. expression[AttributeReference]("col"), expression[Literal]("lit") ) val checkConstraintExpressions: Set[Class[_]] = Set( expression[Contains]("contains"), expression[StartsWith]("startswith"), expression[EndsWith]("endswith"), expression[InSet]("inset"), // Lambda Functions expression[LambdaFunction]("lambdafunction"), expression[NamedLambdaVariable]("namedlambdavariable"), // Date/Time Functions expression[CurrentDate]("current_date"), expression[CurrentTimestamp]("current_timestamp"), // Used by Extract when applied to interval types expression[ExtractANSIIntervalDays]("extractansiintervaldays"), expression[ExtractANSIIntervalHours]("extractansiintervalhours"), expression[ExtractANSIIntervalMinutes]("extractansiintervalminutes"), expression[ExtractANSIIntervalSeconds]("extractansiintervalseconds"), expression[ExtractANSIIntervalYears]("extractansiintervalyears"), expression[ExtractANSIIntervalMonths]("extractansiintervalmonths"), expression[ExtractIntervalYears]("extractintervalyears"), expression[ExtractIntervalMonths]("extractintervalmonths"), expression[ExtractIntervalDays]("extractintervaldays"), expression[ExtractIntervalHours]("extractintervalhours"), expression[ExtractIntervalMinutes]("extractintervalminutes"), expression[ExtractIntervalSeconds]("extractintervalseconds"), // Date/time arithmetic expressions expression[DatetimeSub]("datetimesub"), // Date/time arithmetic with intervals expression[TimestampAddYMInterval]("timestampaddyminterval"), expression[DateAddInterval]("dateaddinterval"), expression[DateAddYMInterval]("dateaddyminterval"), // Comparison functions expression[ILike]("ilike"), expression[LikeAny]("likeany"), expression[NotLikeAny]("notlikeany"), expression[LikeAll]("likeall"), expression[NotLikeAll]("notlikeall"), // Try arithmetic functions expression[TryAdd]("try_add"), expression[TrySubtract]("try_subtract"), expression[TryMultiply]("try_multiply"), expression[TryDivide]("try_divide"), // Try parsing/conversion functions expression[TryToBinary]("try_to_binary"), expression[TryToNumber]("try_to_number"), expression[ToNumber]("to_number"), // Collection functions expression[ArraySize]("array_size"), expression[ArrayCompact]("array_compact"), expression[ArrayAppend]("array_append"), expression[ArrayPrepend]("array_prepend"), expression[ArrayInsert]("array_insert") ) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/CheckUnresolvedRelationTimeTravel.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.analysis.RelationTimeTravel import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.trees.TreePattern.RELATION_TIME_TRAVEL /** * Custom check rule that compensates for [SPARK-45383]. It checks the (unresolved) child relation * of each [[RelationTimeTravel]] in the plan, in order to trigger a helpful table-not-found * [[AnalysisException]] instead of the internal spark error that would otherwise result. */ class CheckUnresolvedRelationTimeTravel(spark: SparkSession) extends (LogicalPlan => Unit) { override def apply(plan: LogicalPlan): Unit = { // Short circuit: We only care about (unresolved) plans containing [[RelationTimeTravel]]. if (plan.containsPattern(RELATION_TIME_TRAVEL)) { plan.foreachUp { case tt: RelationTimeTravel => spark.sessionState.analyzer.checkAnalysis0(tt.relation) case _ => () } } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/CheckpointProvider.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.collection.mutable.ArrayBuffer import scala.concurrent.duration.Duration import scala.util.control.NonFatal import org.apache.spark.sql.delta.DataFrameUtils import org.apache.spark.sql.delta.SnapshotManagement.checkpointV2ThreadPool import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.storage.LogStore import org.apache.spark.sql.delta.util.FileNames._ import org.apache.spark.sql.delta.util.threads.NonFateSharingFuture import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.sql.Dataset import org.apache.spark.sql.SparkSession import org.apache.spark.sql.types.StructType /** * Represents basic information about a checkpoint. * This is the info we always can know about a checkpoint, without doing any additional I/O. */ trait UninitializedCheckpointProvider { /** True if the checkpoint provider is empty (does not refer to a valid checkpoint) */ def isEmpty: Boolean = version < 0 /** Checkpoint version */ def version: Long /** * Top level files that represents this checkpoint. * These files could be reused again to initialize the [[CheckpointProvider]]. */ def topLevelFiles: Seq[FileStatus] /** * File index which could help derive actions stored in top level files * for the checkpoint. * This could be used to get [[Protocol]], [[Metadata]] etc from a checkpoint. * This could also be used if we want to shallow copy a checkpoint. */ def topLevelFileIndex: Option[DeltaLogFileIndex] } /** * A trait which provides information about a checkpoint to the Snapshot. */ trait CheckpointProvider extends UninitializedCheckpointProvider { /** Effective size of checkpoint across all files */ def effectiveCheckpointSizeInBytes(): Long /** * List of different file indexes which could help derive full state-reconstruction * for the checkpoint. */ def allActionsFileIndexes(): Seq[DeltaLogFileIndex] /** * The type of checkpoint (V2 vs Classic). This will be None when no checkpoint is available. * This is only intended to be used for logging and metrics. */ def checkpointPolicy: Option[CheckpointPolicy.Policy] /** * List of different file indexes and corresponding schemas which could help derive full * state-reconstruction for the checkpoint. * Different FileIndexes could have different schemas depending on `stats_parsed` / `stats` * columns in the underlying file(s). */ def allActionsFileIndexesAndSchemas( spark: SparkSession, deltaLog: DeltaLog): Seq[(DeltaLogFileIndex, StructType)] } object CheckpointProvider extends DeltaLogging { /** Helper method to convert non-empty checkpoint files to DeltaLogFileIndex */ def checkpointFileIndex(checkpointFiles: Seq[FileStatus]): DeltaLogFileIndex = { assert(checkpointFiles.nonEmpty, "checkpointFiles must not be empty") DeltaLogFileIndex(DeltaLogFileIndex.CHECKPOINT_FILE_FORMAT_PARQUET, checkpointFiles).get } /** Converts an [[UninitializedCheckpointProvider]] into a [[CheckpointProvider]] */ def apply( spark: SparkSession, snapshotDescriptor: SnapshotDescriptor, checksumOpt: Option[VersionChecksum], uninitializedCheckpointProvider: UninitializedCheckpointProvider) : CheckpointProvider = uninitializedCheckpointProvider match { // Note: snapshotDescriptor.protocol should be accessed as late as possible inside the futures // as it might need I/O. case uninitializedV2CheckpointProvider: UninitializedV2CheckpointProvider => new LazyCompleteCheckpointProvider(uninitializedV2CheckpointProvider) { override def createCheckpointProvider(): CheckpointProvider = { val (checkpointMetadataOpt, sidecarFiles) = uninitializedV2CheckpointProvider.nonFateSharingCheckpointReadFuture.get(Duration.Inf) // This must be a v2 checkpoint, so checkpointMetadataOpt must be non empty. val checkpointMetadata = checkpointMetadataOpt.getOrElse { val checkpointFile = uninitializedV2CheckpointProvider.topLevelFiles.head throw new IllegalStateException(s"V2 Checkpoint ${checkpointFile.getPath} " + s"has no CheckpointMetadata action") } require(isV2CheckpointEnabled(snapshotDescriptor.protocol)) V2CheckpointProvider( uninitializedV2CheckpointProvider, checkpointMetadata, sidecarFiles, snapshotDescriptor.deltaLog) } } case provider: UninitializedV1OrV2ParquetCheckpointProvider if isV2CheckpointEnabled(checksumOpt).contains(false) => // V2 checkpoints are specifically disabled, so it must be V1 PreloadedCheckpointProvider(provider.topLevelFiles, provider.lastCheckpointInfoOpt) case provider: UninitializedV1OrV2ParquetCheckpointProvider => // Either v2 checkpoints are explicitly enabled, or we lack a Protocol to prove otherwise. // We can't tell immediately whether it's V1 or V2, just by looking at the file name. // Start a future to start reading the v2 actions from the parquet checkpoint and return // a lazy checkpoint provider wrapping the future. we won't wait on the future unless/until // somebody calls a complete checkpoint provider method. val future = checkpointV2ThreadPool.submitNonFateSharing { spark: SparkSession => readV2ActionsFromParquetCheckpoint( spark, provider.logPath, provider.fileStatus, snapshotDescriptor.deltaLog.options) } new LazyCompleteCheckpointProvider(provider) { override def createCheckpointProvider(): CheckpointProvider = { val (checkpointMetadataOpt, sidecarFiles) = future.get(Duration.Inf) checkpointMetadataOpt match { case Some(cm) => require(isV2CheckpointEnabled(snapshotDescriptor)) V2CheckpointProvider(provider, cm, sidecarFiles, snapshotDescriptor.deltaLog) case None => PreloadedCheckpointProvider(provider.topLevelFiles, provider.lastCheckpointInfoOpt) } } } } private[delta] def isV2CheckpointEnabled(protocol: Protocol): Boolean = protocol.isFeatureSupported(V2CheckpointTableFeature) /** * Returns whether V2 Checkpoints are enabled or not. * This means an underlying checkpoint in this table could be a V2Checkpoint with sidecar files. */ def isV2CheckpointEnabled(snapshotDescriptor: SnapshotDescriptor): Boolean = isV2CheckpointEnabled(snapshotDescriptor.protocol) /** * Returns: * - Some(true) if V2 Checkpoints are enabled for the snapshot corresponding to the given * `checksumOpt`. * - Some(false) if V2 Checkpoints are disabled for the snapshot * - None if the given checksumOpt is not sufficient to identify if v2 checkpoints are enabled or * not. */ def isV2CheckpointEnabled(checksumOpt: Option[VersionChecksum]): Option[Boolean] = { checksumOpt.flatMap(checksum => Option(checksum.protocol)).map(isV2CheckpointEnabled) } private[delta] def getParquetSchema( spark: SparkSession, deltaLog: DeltaLog, parquetFile: FileStatus, schemaFromLastCheckpoint: Option[StructType]): StructType = { // Try to get the checkpoint schema from the last_checkpoint. // If it is not there then get it from filesystem by doing I/O. val fetchChkSchemaFromLastCheckpoint = spark.sessionState.conf.getConf( DeltaSQLConf.USE_CHECKPOINT_SCHEMA_FROM_CHECKPOINT_METADATA) schemaFromLastCheckpoint match { case Some(schema) if fetchChkSchemaFromLastCheckpoint => schema case _ => recordDeltaOperation(deltaLog, "snapshot.checkpointSchema.fromFileSystem") { Snapshot.getParquetFileSchemaAndRowCount(spark, deltaLog, parquetFile)._1 } } } private def sendEventForV2CheckpointRead( startTimeMs: Long, fileStatus: FileStatus, fileType: String, logPath: Path, exception: Option[Throwable]): Unit = { recordDeltaEvent( deltaLog = null, opType = "delta.checkpointV2.readV2ActionsFromCheckpoint", data = Map( "timeTakenMs" -> (System.currentTimeMillis() - startTimeMs), "v2CheckpointPath" -> fileStatus.getPath.toString, "v2CheckpointSize" -> fileStatus.getLen, "errorMessage" -> exception.map(_.toString).getOrElse(""), "fileType" -> fileType ), path = Some(logPath.getParent) ) } /** Reads and returns the [[CheckpointMetadata]] and [[SidecarFile]]s from a json v2 checkpoint */ private[delta] def readV2ActionsFromJsonCheckpoint( logStore: LogStore, logPath: Path, fileStatus: FileStatus, hadoopConf: Configuration): (CheckpointMetadata, Seq[SidecarFile]) = { val startTimeMs = System.currentTimeMillis() try { var checkpointMetadataOpt: Option[CheckpointMetadata] = None val sidecarFileActions: ArrayBuffer[SidecarFile] = ArrayBuffer.empty logStore.readAsIterator(fileStatus, hadoopConf).processAndClose { _ .map(Action.fromJson) .foreach { case cm: CheckpointMetadata if checkpointMetadataOpt.isEmpty => checkpointMetadataOpt = Some(cm) case cm: CheckpointMetadata => throw new IllegalStateException( "More than 1 CheckpointMetadata actions found in the checkpoint file") case sidecarFile: SidecarFile => sidecarFileActions.append(sidecarFile) case _ => () } } val checkpointMetadata = checkpointMetadataOpt.getOrElse { throw new IllegalStateException("Json V2 Checkpoint has no CheckpointMetadata action") } sendEventForV2CheckpointRead(startTimeMs, fileStatus, "json", logPath, exception = None) (checkpointMetadata, sidecarFileActions.toSeq) } catch { case NonFatal(e) => sendEventForV2CheckpointRead(startTimeMs, fileStatus, "json", logPath, exception = Some(e)) throw e } } /** * Reads and returns the optional [[CheckpointMetadata]], [[SidecarFile]]s from a parquet * checkpoint. * The checkpoint metadata returned might be None if the underlying parquet file is not a v2 * checkpoint. */ private[delta] def readV2ActionsFromParquetCheckpoint( spark: SparkSession, logPath: Path, fileStatus: FileStatus, deltaLogOptions: Map[String, String]): (Option[CheckpointMetadata], Seq[SidecarFile]) = { val startTimeMs = System.currentTimeMillis() try { val relation = DeltaLog.indexToRelation( spark, checkpointFileIndex(Seq(fileStatus)), deltaLogOptions, Action.logSchema) import implicits._ val rows = DataFrameUtils.ofRows(spark, relation) .select("checkpointMetadata", "sidecar") .where("checkpointMetadata.version is not null or sidecar.path is not null") .as[(CheckpointMetadata, SidecarFile)] .collect() var checkpointMetadata: Option[CheckpointMetadata] = None val checkpointSidecarFiles = ArrayBuffer.empty[SidecarFile] rows.foreach { case (cm: CheckpointMetadata, _) if checkpointMetadata.isEmpty => checkpointMetadata = Some(cm) case (cm: CheckpointMetadata, _) => throw new IllegalStateException( "More than 1 CheckpointMetadata actions found in the checkpoint file") case (_, sf: SidecarFile) => checkpointSidecarFiles.append(sf) } if (checkpointMetadata.isEmpty && checkpointSidecarFiles.nonEmpty) { throw new IllegalStateException( "sidecar files present in checkpoint even when checkpoint metadata is missing") } sendEventForV2CheckpointRead(startTimeMs, fileStatus, "parquet", logPath, exception = None) (checkpointMetadata, checkpointSidecarFiles.toSeq) } catch { case NonFatal(e) => sendEventForV2CheckpointRead(startTimeMs, fileStatus, "parquet", logPath, Some(e)) throw e } } } /** * An implementation of [[CheckpointProvider]] where the information about checkpoint files * (i.e. Seq[FileStatus]) is already known in advance. * * @param topLevelFiles - file statuses that describes the checkpoint * @param lastCheckpointInfoOpt - optional [[LastCheckpointInfo]] corresponding to this checkpoint. * This comes from _last_checkpoint file */ case class PreloadedCheckpointProvider( override val topLevelFiles: Seq[FileStatus], lastCheckpointInfoOpt: Option[LastCheckpointInfo]) extends CheckpointProvider with DeltaLogging { require(topLevelFiles.nonEmpty, "There should be atleast 1 checkpoint file") private lazy val fileIndex = CheckpointProvider.checkpointFileIndex(topLevelFiles) override def version: Long = checkpointVersion(topLevelFiles.head) override def effectiveCheckpointSizeInBytes(): Long = fileIndex.sizeInBytes override def allActionsFileIndexes(): Seq[DeltaLogFileIndex] = Seq(fileIndex) override lazy val topLevelFileIndex: Option[DeltaLogFileIndex] = Some(fileIndex) override def checkpointPolicy: Option[CheckpointPolicy.Policy] = Some(CheckpointPolicy.Classic) override def allActionsFileIndexesAndSchemas( spark: SparkSession, deltaLog: DeltaLog): Seq[(DeltaLogFileIndex, StructType)] = { Seq((fileIndex, checkpointSchema(spark, deltaLog))) } private val checkpointSchemaWithCaching = new LazyCheckpointSchemaGetter { override def fileStatus: FileStatus = topLevelFiles.head override def schemaFromLastCheckpoint: Option[StructType] = lastCheckpointInfoOpt.flatMap(_.checkpointSchema) } private def checkpointSchema(spark: SparkSession, deltaLog: DeltaLog): StructType = checkpointSchemaWithCaching.get(spark, deltaLog) } /** * An implementation for [[CheckpointProvider]] which could be used to represent a scenario when * checkpoint doesn't exist. This helps us simplify the code by making * [[LogSegment.checkpointProvider]] as non-optional. * * The [[CheckpointProvider.isEmpty]] method returns true for [[EmptyCheckpointProvider]]. Also * version is returned as -1. * For a real checkpoint, this will be returned true and version will be >= 0. */ object EmptyCheckpointProvider extends CheckpointProvider { override def version: Long = -1 override def topLevelFiles: Seq[FileStatus] = Nil override def effectiveCheckpointSizeInBytes(): Long = 0L override def allActionsFileIndexes(): Seq[DeltaLogFileIndex] = Nil override def topLevelFileIndex: Option[DeltaLogFileIndex] = None override def checkpointPolicy: Option[CheckpointPolicy.Policy] = None override def allActionsFileIndexesAndSchemas( spark: SparkSession, deltaLog: DeltaLog): Seq[(DeltaLogFileIndex, StructType)] = Nil } /** A trait representing a v2 [[UninitializedCheckpointProvider]] */ trait UninitializedV2LikeCheckpointProvider extends UninitializedCheckpointProvider { def fileStatus: FileStatus def logPath: Path def lastCheckpointInfoOpt: Option[LastCheckpointInfo] def v2CheckpointFormat: V2Checkpoint.Format override lazy val topLevelFiles: Seq[FileStatus] = Seq(fileStatus) override lazy val topLevelFileIndex: Option[DeltaLogFileIndex] = DeltaLogFileIndex(v2CheckpointFormat.fileFormat, topLevelFiles) } /** * An implementation of [[UninitializedCheckpointProvider]] to represent a parquet checkpoint * which could be either a v1 checkpoint or v2 checkpoint. * This needs to be resolved into a [[PreloadedCheckpointProvider]] or a [[V2CheckpointProvider]] * depending on whether the [[CheckpointMetadata]] action is present or not in the underlying * parquet file. */ case class UninitializedV1OrV2ParquetCheckpointProvider( override val version: Long, override val fileStatus: FileStatus, override val logPath: Path, override val lastCheckpointInfoOpt: Option[LastCheckpointInfo] ) extends UninitializedV2LikeCheckpointProvider { override val v2CheckpointFormat: V2Checkpoint.Format = V2Checkpoint.Format.PARQUET } /** * An implementation of [[UninitializedCheckpointProvider]] to for v2 checkpoints. * This needs to be resolved into a [[V2CheckpointProvider]]. * This class starts an I/O to fetch the V2 actions ([[CheckpointMetadata]], [[SidecarFile]]) as * soon as the class is initialized so that the extra overhead could be parallelized with other * operations like reading CRC. */ case class UninitializedV2CheckpointProvider( override val version: Long, override val fileStatus: FileStatus, override val logPath: Path, hadoopConf: Configuration, deltaLogOptions: Map[String, String], logStore: LogStore, override val lastCheckpointInfoOpt: Option[LastCheckpointInfo] ) extends UninitializedV2LikeCheckpointProvider { override val v2CheckpointFormat: V2Checkpoint.Format = V2Checkpoint.toFormat(fileStatus.getPath.getName) // Try to get the required actions from LastCheckpointInfo private val v2ActionsFromLastCheckpointOpt: Option[(CheckpointMetadata, Seq[SidecarFile])] = { lastCheckpointInfoOpt .flatMap(_.v2Checkpoint) .map(v2 => (v2.checkpointMetadataOpt, v2.sidecarFiles)) .collect { case (Some(checkpointMetadata), Some(sidecarFiles)) => (checkpointMetadata, sidecarFiles) } } /** Helper method to do I/O and read v2 actions from the underlying v2 checkpoint file */ private def readV2Actions(spark: SparkSession): (Option[CheckpointMetadata], Seq[SidecarFile]) = { v2CheckpointFormat match { case V2Checkpoint.Format.JSON => val (checkpointMetadata, sidecars) = CheckpointProvider.readV2ActionsFromJsonCheckpoint( logStore, logPath, fileStatus, hadoopConf) (Some(checkpointMetadata), sidecars) case V2Checkpoint.Format.PARQUET => CheckpointProvider.readV2ActionsFromParquetCheckpoint( spark, logPath, fileStatus, deltaLogOptions) } } val nonFateSharingCheckpointReadFuture : NonFateSharingFuture[(Option[CheckpointMetadata], Seq[SidecarFile])] = { checkpointV2ThreadPool.submitNonFateSharing { spark: SparkSession => v2ActionsFromLastCheckpointOpt match { case Some((cm, sidecars)) => Some(cm) -> sidecars case None => readV2Actions(spark) } } } } /** * A wrapper implementation of [[CheckpointProvider]] which wraps * `underlyingCheckpointProviderFuture` and `uninitializedCheckpointProvider` for implementing all * the [[UninitializedCheckpointProvider]] and [[CheckpointProvider]] APIs. * * @param uninitializedCheckpointProvider the underlying [[UninitializedCheckpointProvider]] */ abstract class LazyCompleteCheckpointProvider( uninitializedCheckpointProvider: UninitializedCheckpointProvider) extends CheckpointProvider { override def version: Long = uninitializedCheckpointProvider.version override def topLevelFiles: Seq[FileStatus] = uninitializedCheckpointProvider.topLevelFiles override def topLevelFileIndex: Option[DeltaLogFileIndex] = uninitializedCheckpointProvider.topLevelFileIndex protected def createCheckpointProvider(): CheckpointProvider lazy val underlyingCheckpointProvider: CheckpointProvider = createCheckpointProvider() override def effectiveCheckpointSizeInBytes(): Long = underlyingCheckpointProvider.effectiveCheckpointSizeInBytes() override def allActionsFileIndexes(): Seq[DeltaLogFileIndex] = underlyingCheckpointProvider.allActionsFileIndexes() override def checkpointPolicy: Option[CheckpointPolicy.Policy] = underlyingCheckpointProvider.checkpointPolicy override def allActionsFileIndexesAndSchemas( spark: SparkSession, deltaLog: DeltaLog): Seq[(DeltaLogFileIndex, StructType)] = { underlyingCheckpointProvider.allActionsFileIndexesAndSchemas(spark, deltaLog) } } /** * [[CheckpointProvider]] implementation for Json/Parquet V2 checkpoints. * * @param version checkpoint version for the underlying checkpoint * @param v2CheckpointFile [[FileStatus]] for the json/parquet v2 checkpoint file * @param v2CheckpointFormat format (json/parquet) for the v2 checkpoint * @param checkpointMetadata [[CheckpointMetadata]] for the v2 checkpoint * @param sidecarFiles seq of [[SidecarFile]] for the v2 checkpoint * @param lastCheckpointInfoOpt optional last checkpoint info for the v2 checkpoint * @param logPath delta log path for the underlying delta table * @param sidecarSchemaFetcher function to fetch sidecar schema. * Returns None if there are no sidecar files. */ case class V2CheckpointProvider( override val version: Long, v2CheckpointFile: FileStatus, v2CheckpointFormat: V2Checkpoint.Format, checkpointMetadata: CheckpointMetadata, sidecarFiles: Seq[SidecarFile], lastCheckpointInfoOpt: Option[LastCheckpointInfo], logPath: Path, sidecarSchemaFetcher: () => Option[StructType] ) extends CheckpointProvider with DeltaLogging { private[delta] def sidecarFileStatuses: Seq[FileStatus] = sidecarFiles.map(_.toFileStatus(logPath)) protected lazy val fileIndexesForSidecarFiles: Seq[DeltaLogFileIndex] = { // V2 checkpoints without sidecars are legal. if (sidecarFileStatuses.isEmpty) { Seq.empty } else { Seq(CheckpointProvider.checkpointFileIndex(sidecarFileStatuses)) } } protected lazy val fileIndexForV2Checkpoint: DeltaLogFileIndex = DeltaLogFileIndex(v2CheckpointFormat.fileFormat, Seq(v2CheckpointFile)).head override lazy val topLevelFiles: Seq[FileStatus] = Seq(v2CheckpointFile) override lazy val topLevelFileIndex: Option[DeltaLogFileIndex] = Some(fileIndexForV2Checkpoint) override def effectiveCheckpointSizeInBytes(): Long = sidecarFiles.map(_.sizeInBytes).sum + v2CheckpointFile.getLen override def allActionsFileIndexes(): Seq[DeltaLogFileIndex] = topLevelFileIndex ++: fileIndexesForSidecarFiles override def checkpointPolicy: Option[CheckpointPolicy.Policy] = Some(CheckpointPolicy.V2) private val v2SchemaWithCaching = new LazyCheckpointSchemaGetter { override def fileStatus: FileStatus = v2CheckpointFile override def schemaFromLastCheckpoint: Option[StructType] = lastCheckpointInfoOpt.flatMap(_.checkpointSchema) } protected def schemaForV2Checkpoint( spark: SparkSession, deltaLog: DeltaLog): StructType = { if (v2CheckpointFormat != V2Checkpoint.Format.PARQUET) { return Action.logSchema } v2SchemaWithCaching.get(spark, deltaLog) } protected def schemaForSidecarFile(spark: SparkSession, deltaLog: DeltaLog): StructType = { sidecarSchemaFetcher() .getOrElse { throw DeltaErrors.assertionFailedError("Sidecar schema asked without any sidecar files") } } override def allActionsFileIndexesAndSchemas( spark: SparkSession, deltaLog: DeltaLog): Seq[(DeltaLogFileIndex, StructType)] = { (fileIndexForV2Checkpoint, schemaForV2Checkpoint(spark, deltaLog)) +: fileIndexesForSidecarFiles.map((_, schemaForSidecarFile(spark, deltaLog))) } } object V2CheckpointProvider { /** Alternate constructor which uses [[UninitializedV2LikeCheckpointProvider]] */ def apply( uninitializedV2LikeCheckpointProvider: UninitializedV2LikeCheckpointProvider, checkpointMetadata: CheckpointMetadata, sidecarFiles: Seq[SidecarFile], deltaLog: DeltaLog): V2CheckpointProvider = { def getSidecarSchemaFetcher: () => Option[StructType] = { val nonFateSharingSidecarSchemaFuture: NonFateSharingFuture[Option[StructType]] = { checkpointV2ThreadPool.submitNonFateSharing { spark: SparkSession => sidecarFiles.headOption.map { sidecarFile => val sidecarFileStatus = sidecarFile.toFileStatus(uninitializedV2LikeCheckpointProvider.logPath) CheckpointProvider.getParquetSchema( spark, deltaLog, sidecarFileStatus, schemaFromLastCheckpoint = None) } } } () => nonFateSharingSidecarSchemaFuture.get(Duration.Inf) } V2CheckpointProvider( uninitializedV2LikeCheckpointProvider.version, uninitializedV2LikeCheckpointProvider.fileStatus, uninitializedV2LikeCheckpointProvider.v2CheckpointFormat, checkpointMetadata, sidecarFiles, uninitializedV2LikeCheckpointProvider.lastCheckpointInfoOpt, uninitializedV2LikeCheckpointProvider.logPath, getSidecarSchemaFetcher ) } } abstract class LazyCheckpointSchemaGetter { protected def fileStatus: FileStatus protected def schemaFromLastCheckpoint: Option[StructType] private var lazySchema = Option.empty[StructType] def get(spark: SparkSession, deltaLog: DeltaLog): StructType = { lazySchema.getOrElse { this.synchronized { // re-check with lock held, in case of races with other initializers if (lazySchema.isEmpty) { lazySchema = Some(CheckpointProvider.getParquetSchema( spark, deltaLog, fileStatus, schemaFromLastCheckpoint)) } lazySchema.get } } } def getIfKnown: Option[StructType] = lazySchema } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/Checkpoints.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.FileNotFoundException import java.util.UUID import scala.collection.mutable import scala.math.Ordering.Implicits._ import scala.util.Try import scala.util.control.NonFatal // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions.{Action, CheckpointMetadata, Metadata, SidecarFile, SingleAction} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.storage.LogStore import org.apache.spark.sql.delta.util.{DeltaFileOperations, DeltaLogGroupingIterator, FileNames} import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} import org.apache.spark.sql.delta.util.FileNames._ import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, FileSystem, Path} import org.apache.hadoop.mapred.{JobConf, TaskAttemptContextImpl, TaskAttemptID} import org.apache.hadoop.mapreduce.{Job, TaskType} import org.apache.spark.TaskContext import org.apache.spark.internal.MDC import org.apache.spark.paths.SparkPath import org.apache.spark.sql.{Column, DataFrame, Dataset, Row, SparkSession} import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.{Cast, ElementAt, Literal} import org.apache.spark.sql.delta.expressions.DecodeNestedZ85EncodedVariant import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.shims.VariantShreddingShims import org.apache.spark.sql.execution.SQLExecution import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.execution.datasources.OutputWriter import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.functions.{coalesce, col, struct, when} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType import org.apache.spark.util.SerializableConfiguration import org.apache.spark.util.Utils /** * A class to help with comparing checkpoints with each other, where we may have had concurrent * writers that checkpoint with different number of parts. * The `numParts` field will be present only for multipart checkpoints (represented by * Format.WITH_PARTS). * The `fileName` field is present only for V2 Checkpoints (represented by Format.V2) * These additional fields are used as a tie breaker when comparing multiple checkpoint * instance of same Format for the same `version`. */ case class CheckpointInstance( version: Long, format: CheckpointInstance.Format, fileName: Option[String] = None, numParts: Option[Int] = None) extends Ordered[CheckpointInstance] { // Assert that numParts are present when checkpoint format is Format.WITH_PARTS. // For other formats, numParts must be None. require((format == CheckpointInstance.Format.WITH_PARTS) == numParts.isDefined, s"numParts ($numParts) must be present for checkpoint format" + s" ${CheckpointInstance.Format.WITH_PARTS.name}") // Assert that filePath is present only when checkpoint format is Format.V2. // For other formats, filePath must be None. require((format == CheckpointInstance.Format.V2) == fileName.isDefined, s"fileName ($fileName) must be present for checkpoint format" + s" ${CheckpointInstance.Format.V2.name}") /** * Returns a [[CheckpointProvider]] which can tell the files corresponding to this * checkpoint. * The `lastCheckpointInfoHint` might be passed to [[CheckpointProvider]] so that underlying * [[CheckpointProvider]] provides more precise info. */ def getCheckpointProvider( deltaLog: DeltaLog, filesForCheckpointConstruction: Seq[FileStatus], lastCheckpointInfoHint: Option[LastCheckpointInfo] = None) : UninitializedCheckpointProvider = { val logPath = deltaLog.logPath val lastCheckpointInfo = lastCheckpointInfoHint.filter(cm => CheckpointInstance(cm) == this) val cpFiles = filterFiles(deltaLog, filesForCheckpointConstruction) format match { // Treat single file checkpoints also as V2 Checkpoints because we don't know if it is // actually a V2 checkpoint until we read it. case CheckpointInstance.Format.V2 | CheckpointInstance.Format.SINGLE => assert(cpFiles.size == 1) val fileStatus = cpFiles.head if (format == CheckpointInstance.Format.V2) { val hadoopConf = deltaLog.newDeltaHadoopConf() UninitializedV2CheckpointProvider( version, fileStatus, logPath, hadoopConf, deltaLog.options, deltaLog.store, lastCheckpointInfo) } else { UninitializedV1OrV2ParquetCheckpointProvider( version, fileStatus, logPath, lastCheckpointInfo) } case CheckpointInstance.Format.WITH_PARTS => PreloadedCheckpointProvider(cpFiles, lastCheckpointInfo) case CheckpointInstance.Format.SENTINEL => throw DeltaErrors.assertionFailedError( s"invalid checkpoint format ${CheckpointInstance.Format.SENTINEL}") } } def filterFiles(deltaLog: DeltaLog, filesForCheckpointConstruction: Seq[FileStatus]) : Seq[FileStatus] = { val logPath = deltaLog.logPath format match { // Treat Single File checkpoints also as V2 Checkpoints because we don't know if it is // actually a V2 checkpoint until we read it. case format if format.usesSidecars => val checkpointFileName = format match { case CheckpointInstance.Format.V2 => fileName.get case CheckpointInstance.Format.SINGLE => checkpointFileSingular(logPath, version).getName case other => throw new IllegalStateException(s"Unknown checkpoint format $other supporting sidecars") } val fileStatus = filesForCheckpointConstruction .find(_.getPath.getName == checkpointFileName) .getOrElse { throw new IllegalStateException("Failed in getting the file information for:\n" + fileName.get + "\namong\n" + filesForCheckpointConstruction.map(_.getPath.getName).mkString(" -", "\n -", "")) } Seq(fileStatus) case CheckpointInstance.Format.WITH_PARTS | CheckpointInstance.Format.SINGLE => val filePaths = if (format == CheckpointInstance.Format.WITH_PARTS) { checkpointFileWithParts(logPath, version, numParts.get).toSet } else { Set(checkpointFileSingular(logPath, version)) } val newCheckpointFileArray = filesForCheckpointConstruction.filter(f => filePaths.contains(f.getPath)) assert(newCheckpointFileArray.length == filePaths.size, "Failed in getting the file information for:\n" + filePaths.mkString(" -", "\n -", "") + "\namong\n" + filesForCheckpointConstruction.map(_.getPath).mkString(" -", "\n -", "")) newCheckpointFileArray case CheckpointInstance.Format.SENTINEL => throw DeltaErrors.assertionFailedError( s"invalid checkpoint format ${CheckpointInstance.Format.SENTINEL}") } } /** * Comparison rules: * 1. A [[CheckpointInstance]] with higher version is greater than the one with lower version. * 2. For [[CheckpointInstance]]s with same version, a Multi-part checkpoint is greater than a * Single part checkpoint. * 3. For Multi-part [[CheckpointInstance]]s corresponding to same version, the one with more * parts is greater than the one with less parts. * 4. For V2 Checkpoints corresponding to same version, we use the fileName as tie breaker. */ override def compare(other: CheckpointInstance): Int = { (version, format, numParts, fileName) compare (other.version, other.format, other.numParts, other.fileName) } } object CheckpointInstance { sealed abstract class Format(val ordinal: Int, val name: String) extends Ordered[Format] { override def compare(other: Format): Int = ordinal compare other.ordinal def usesSidecars: Boolean = this.isInstanceOf[FormatUsesSidecars] } trait FormatUsesSidecars object Format { def unapply(name: String): Option[Format] = name match { case SINGLE.name => Some(SINGLE) case WITH_PARTS.name => Some(WITH_PARTS) case V2.name => Some(V2) case _ => None } /** single-file checkpoint format */ object SINGLE extends Format(0, "SINGLE") with FormatUsesSidecars /** multi-file checkpoint format */ object WITH_PARTS extends Format(1, "WITH_PARTS") /** V2 Checkpoint format */ object V2 extends Format(2, "V2") with FormatUsesSidecars /** Sentinel, for internal use only */ object SENTINEL extends Format(Int.MaxValue, "SENTINEL") } def apply(path: Path): CheckpointInstance = { // Three formats to worry about: // * .checkpoint.parquet // * .checkpoint...parquet // * .checkpoint..parquet where u is a unique string path.getName.split("\\.") match { case Array(v, "checkpoint", uniqueStr, format) if Seq("json", "parquet").contains(format) => CheckpointInstance( version = v.toLong, format = Format.V2, numParts = None, fileName = Some(path.getName)) case Array(v, "checkpoint", "parquet") => CheckpointInstance(v.toLong, Format.SINGLE, numParts = None) case Array(v, "checkpoint", _, n, "parquet") => CheckpointInstance(v.toLong, Format.WITH_PARTS, numParts = Some(n.toInt)) case _ => throw DeltaErrors.assertionFailedError(s"Unrecognized checkpoint path format: $path") } } def apply(version: Long): CheckpointInstance = { CheckpointInstance(version, Format.SINGLE, numParts = None) } def apply(metadata: LastCheckpointInfo): CheckpointInstance = { CheckpointInstance( version = metadata.version, format = metadata.getFormatEnum(), fileName = metadata.v2Checkpoint.map(_.path), numParts = metadata.parts) } val MaxValue: CheckpointInstance = sentinelValue(versionOpt = None) def sentinelValue(versionOpt: Option[Long]): CheckpointInstance = { val version = versionOpt.getOrElse(Long.MaxValue) CheckpointInstance(version, Format.SENTINEL, numParts = None) } } trait Checkpoints extends DeltaLogging { self: DeltaLog => def logPath: Path def dataPath: Path protected def store: LogStore /** Used to clean up stale log files. */ protected def doLogCleanup( snapshotToCleanup: Snapshot, catalogTableOpt: Option[CatalogTable]): Unit /** Returns the checkpoint interval for this log. Not transactional. */ def checkpointInterval(metadata: Metadata): Int = DeltaConfigs.CHECKPOINT_INTERVAL.fromMetaData(metadata) /** The path to the file that holds metadata about the most recent checkpoint. */ val LAST_CHECKPOINT = new Path(logPath, Checkpoints.LAST_CHECKPOINT_FILE_NAME) /** * Catch non-fatal exceptions related to checkpointing, since the checkpoint is written * after the commit has completed. From the perspective of the user, the commit has * completed successfully. However, throw if this is in a testing environment - * that way any breaking changes can be caught in unit tests. */ protected def withCheckpointExceptionHandling( deltaLog: DeltaLog, opType: String)(thunk: => Unit): Unit = { try { thunk } catch { case NonFatal(e) => recordDeltaEvent( deltaLog, opType, data = Map("exception" -> e.getMessage(), "stackTrace" -> e.getStackTrace()) ) logWarning(log"Error when writing checkpoint-related files", e) val throwError = DeltaUtils.isTesting || spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_CHECKPOINT_THROW_EXCEPTION_WHEN_FAILED) if (throwError) throw e } } /** * Creates a checkpoint using the default snapshot. * * WARNING: This API is being deprecated, and will be removed in future versions. * Please use the checkpoint(Snapshot) function below to write checkpoints to the delta log. */ @deprecated("This method is deprecated and will be removed in future versions.", "12.0") def checkpoint(): Unit = checkpoint(unsafeVolatileSnapshot) /** * Creates a checkpoint using snapshotToCheckpoint. By default it uses the current log version. * Note that this function captures and logs all exceptions, since the checkpoint shouldn't fail * the overall commit operation. */ def checkpoint( snapshotToCheckpoint: Snapshot, catalogTableOpt: Option[CatalogTable] = None): Unit = recordDeltaOperation(this, "delta.checkpoint") { withCheckpointExceptionHandling(snapshotToCheckpoint.deltaLog, "delta.checkpoint.sync.error") { if (snapshotToCheckpoint.version < 0) { throw DeltaErrors.checkpointNonExistTable(dataPath) } checkpointAndCleanUpDeltaLog(snapshotToCheckpoint, catalogTableOpt) } } /** * Creates a checkpoint at given version. Does not invoke metadata cleanup as part of it. * @param version - version at which we want to create a checkpoint. */ def createCheckpointAtVersion(version: Long): Unit = recordDeltaOperation(this, "delta.createCheckpointAtVersion") { val snapshot = getSnapshotAt(version) withCheckpointExceptionHandling(this, "delta.checkpoint.sync.error") { if (snapshot.version < 0) { throw DeltaErrors.checkpointNonExistTable(dataPath) } writeCheckpointFiles(snapshot) } } def checkpointAndCleanUpDeltaLog( snapshotToCheckpoint: Snapshot, catalogTableOpt: Option[CatalogTable]): Unit = { val lastCheckpointInfo = writeCheckpointFiles(snapshotToCheckpoint, catalogTableOpt) writeLastCheckpointFile( snapshotToCheckpoint.deltaLog, lastCheckpointInfo, LastCheckpointInfo.checksumEnabled(spark)) doLogCleanup(snapshotToCheckpoint, catalogTableOpt) } protected[delta] def writeLastCheckpointFile( deltaLog: DeltaLog, lastCheckpointInfo: LastCheckpointInfo, addChecksum: Boolean): Unit = { withCheckpointExceptionHandling(deltaLog, "delta.lastCheckpoint.write.error") { val suppressOptionalFields = spark.sessionState.conf.getConf( DeltaSQLConf.SUPPRESS_OPTIONAL_LAST_CHECKPOINT_FIELDS) val lastCheckpointInfoToWrite = lastCheckpointInfo val json = LastCheckpointInfo.serializeToJson( lastCheckpointInfoToWrite, addChecksum, suppressOptionalFields) store.write(LAST_CHECKPOINT, Iterator(json), overwrite = true, newDeltaHadoopConf()) } } protected def writeCheckpointFiles( snapshotToCheckpoint: Snapshot, catalogTableOpt: Option[CatalogTable] = None): LastCheckpointInfo = { Checkpoints.writeCheckpoint(spark, this, snapshotToCheckpoint, catalogTableOpt) } /** Returns information about the most recent checkpoint. */ private[delta] def readLastCheckpointFile(): Option[LastCheckpointInfo] = { loadMetadataFromFile(0) } /** * Reads the checkpoint metadata from the `_last_checkpoint` file. This method doesn't handle any * exceptions that can be thrown, for example IOExceptions thrown when reading the data such as * FileNotFoundExceptions which is expected for a new Delta table or JSON deserialization errors. */ protected def unsafeLoadMetadataFromFile(): LastCheckpointInfo = { val lastCheckpointInfoJson = store.read(LAST_CHECKPOINT, newDeltaHadoopConf()) val validate = LastCheckpointInfo.checksumEnabled(spark) LastCheckpointInfo.deserializeFromJson(lastCheckpointInfoJson.head, validate) } /** Loads the checkpoint metadata from the _last_checkpoint file. */ protected def loadMetadataFromFile(tries: Int): Option[LastCheckpointInfo] = recordDeltaOperation(self, "delta.deltaLog.loadMetadataFromFile") { try { Some(unsafeLoadMetadataFromFile()) } catch { case _: FileNotFoundException => None case NonFatal(e) if tries < 3 => logWarning(log"Failed to parse ${MDC(DeltaLogKeys.PATH, LAST_CHECKPOINT)}. " + log"This may happen if there was an error during read operation, " + log"or a file appears to be partial. Sleeping and trying again.", e) Thread.sleep(1000) loadMetadataFromFile(tries + 1) case NonFatal(e) => recordDeltaEvent( self, "delta.lastCheckpoint.read.corruptedJson", data = Map("exception" -> Utils.exceptionString(e)) ) logWarning(log"${MDC(DeltaLogKeys.PATH, LAST_CHECKPOINT)} is corrupted. " + log"Will search the checkpoint files directly", e) // Hit a partial file. This could happen on Azure as overwriting _last_checkpoint file is // not atomic. We will try to list all files to find the latest checkpoint and restore // LastCheckpointInfo from it. val verifiedCheckpoint = findLastCompleteCheckpointBefore(checkpointInstance = None) verifiedCheckpoint.map(manuallyLoadCheckpoint) } } /** Loads the given checkpoint manually to come up with the [[LastCheckpointInfo]] */ protected def manuallyLoadCheckpoint(cv: CheckpointInstance): LastCheckpointInfo = { LastCheckpointInfo( version = cv.version, size = -1, parts = cv.numParts, sizeInBytes = None, numOfAddFiles = None, checkpointSchema = None ) } /** * Finds the first verified, complete checkpoint before the given version. * Note that the returned checkpoint will always be < `version`. * @param version The checkpoint version to compare against */ private[delta] def findLastCompleteCheckpointBefore(version: Long): Option[CheckpointInstance] = { val upperBound = CheckpointInstance(version, CheckpointInstance.Format.SINGLE, numParts = None) findLastCompleteCheckpointBefore(Some(upperBound)) } /** * Finds the first verified, complete checkpoint before the given [[CheckpointInstance]]. * If `checkpointInstance` is passed as None, then we return the last complete checkpoint in the * deltalog directory. * @param checkpointInstance The checkpoint instance to compare against */ private[delta] def findLastCompleteCheckpointBefore( checkpointInstance: Option[CheckpointInstance] = None): Option[CheckpointInstance] = { val eventData = mutable.Map[String, String]() val startTimeMs = System.currentTimeMillis() def sendUsageLog(): Unit = { eventData("totalTimeTakenMs") = (System.currentTimeMillis() - startTimeMs).toString recordDeltaEvent( self, opType = "delta.findLastCompleteCheckpointBefore", data = eventData.toMap) } try { val resultOpt = findLastCompleteCheckpointBeforeInternal(eventData, checkpointInstance) eventData("resultantCheckpointVersion") = resultOpt.map(_.version).getOrElse(-1L).toString sendUsageLog() resultOpt } catch { case e@(NonFatal(_) | _: InterruptedException | _: java.io.InterruptedIOException | _: java.nio.channels.ClosedByInterruptException) => eventData("exception") = Utils.exceptionString(e) sendUsageLog() throw e } } private def findLastCompleteCheckpointBeforeInternal( eventData: mutable.Map[String, String], checkpointInstance: Option[CheckpointInstance]): Option[CheckpointInstance] = { val upperBoundCv = checkpointInstance // If someone passes the upperBound as 0 or sentinel value, we should not do backward // listing. Instead we should list the entire directory from 0 and return the latest // available checkpoint. .filterNot(cv => cv.version < 0 || cv.version == CheckpointInstance.MaxValue.version) .getOrElse { logInfo( log"[tableId=${MDC(DeltaLogKeys.TABLE_ID, truncatedUnsafeVolatileTableId)}] Try to " + log"find Delta last complete checkpoint") eventData("listingFromZero") = true.toString return findLastCompleteCheckpoint() } eventData("efficientBackwardListingEnabled") = true.toString eventData("upperBoundVersion") = upperBoundCv.version.toString eventData("upperBoundCheckpointType") = upperBoundCv.format.name var iterations: Long = 0L var numFilesScanned: Long = 0L logInfo(log"[tableId=${MDC(DeltaLogKeys.TABLE_ID, truncatedUnsafeVolatileTableId)}] " + log"Try to find Delta last complete checkpoint before version " + log"${MDC(DeltaLogKeys.VERSION, upperBoundCv.version)}") var listingEndVersion = upperBoundCv.version // Do a backward listing from the upperBoundCv version. We list in chunks of 1000 versions. // ........................................................................................... // | // upper bound cv's version // [ iter-1 looks in this window ] // [ iter-2 window ] // [ iter-3 window ] // | // latest checkpoint while (listingEndVersion >= 0) { iterations += 1 eventData("iterations") = iterations.toString val listingStartVersion = math.max(0, listingEndVersion - 1000) val checkpoints = store .listFrom(listingPrefix(logPath, listingStartVersion), newDeltaHadoopConf()) .map { file => numFilesScanned += 1 ; file } .collect { // Also collect delta files from the listing result so that the next takeWhile helps us // terminate iterator early if no checkpoint exists upto the `listingEndVersion` // version. case DeltaFile(file, version) => (file, FileType.DELTA, version) case CheckpointFile(file, version) => (file, FileType.CHECKPOINT, version) } .takeWhile { case (_, _, currentFileVersion) => currentFileVersion <= listingEndVersion } // Checkpoint files of 0 size are invalid but Spark will ignore them silently when // reading such files, hence we drop them so that we never pick up such checkpoints. .collect { case (file, FileType.CHECKPOINT, _) if file.getLen > 0 => CheckpointInstance(file.getPath) } // We still need to filter on `upperBoundCv` to eliminate checkpoint files which are // same version as `upperBoundCv` but have higher [[CheckpointInstance.Format]]. e.g. // upperBoundCv is a V2_Checkpoint and we have a Single part checkpoint and a v2 // checkpoint at the same version. In such a scenario, we should not consider the // v2 checkpoint as it is nor lower than the upperBoundCv. .filter(_ < upperBoundCv) .toArray val lastCheckpoint = getLatestCompleteCheckpointFromList(checkpoints, Some(upperBoundCv.version)) eventData("numFilesScanned") = numFilesScanned.toString if (lastCheckpoint.isDefined) { logInfo( log"[tableId=${MDC(DeltaLogKeys.TABLE_ID, truncatedUnsafeVolatileTableId)}] Delta " + log"checkpoint is found at version " + log"${MDC(DeltaLogKeys.VERSION, lastCheckpoint.get.version)}") return lastCheckpoint } listingEndVersion = listingEndVersion - 1000 } logInfo( log"[tableId=${MDC(DeltaLogKeys.TABLE_ID, truncatedUnsafeVolatileTableId)}] No checkpoint " + log"found for Delta table before version ${MDC(DeltaLogKeys.VERSION, upperBoundCv.version)}") None } /** Returns whether a checkpoint exists at `version`. */ def checkpointExistsAtVersion(version: Long): Boolean = { val upperBoundVersion = Some(CheckpointInstance(version = version + 1)) val lastVerifiedCheckpoint = findLastCompleteCheckpointBefore(upperBoundVersion) lastVerifiedCheckpoint.exists(_.version == version) } /** Returns the last complete checkpoint in the delta log directory (if any) */ private def findLastCompleteCheckpoint(): Option[CheckpointInstance] = { val hadoopConf = newDeltaHadoopConf() val listingResult = store .listFrom(listingPrefix(logPath, 0L), hadoopConf) // Checkpoint files of 0 size are invalid but Spark will ignore them silently when // reading such files, hence we drop them so that we never pick up such checkpoints. .collect { case CheckpointFile(file, _) if file.getLen != 0 => file } new DeltaLogGroupingIterator(listingResult) .flatMap { case (_, files) => getLatestCompleteCheckpointFromList(files.map(f => CheckpointInstance(f.getPath)).toArray) }.foldLeft(Option.empty[CheckpointInstance])((_, right) => Some(right)) // ^The foldLeft here emulates the non-existing Iterator.tailOption method. } /** * Given a list of checkpoint files, pick the latest complete checkpoint instance which is not * later than `notLaterThan`. */ protected[delta] def getLatestCompleteCheckpointFromList( instances: Array[CheckpointInstance], notLaterThanVersion: Option[Long] = None): Option[CheckpointInstance] = { val sentinelCv = CheckpointInstance.sentinelValue(notLaterThanVersion) val complete = instances.filter(_ <= sentinelCv).groupBy(identity).filter { case (ci, matchingCheckpointInstances) => ci.format match { case CheckpointInstance.Format.SINGLE => matchingCheckpointInstances.length == 1 case CheckpointInstance.Format.WITH_PARTS => assert(ci.numParts.nonEmpty, "Multi-Part Checkpoint must have non empty numParts") matchingCheckpointInstances.length == ci.numParts.get case CheckpointInstance.Format.V2 => matchingCheckpointInstances.length == 1 case CheckpointInstance.Format.SENTINEL => false } } if (complete.isEmpty) None else Some(complete.keys.max) } } object Checkpoints extends DeltaLogging { /** The name of the last checkpoint file */ val LAST_CHECKPOINT_FILE_NAME = "_last_checkpoint" /** * Determines the V2 checkpoint format to use for the given snapshot, if applicable. * * This method evaluates whether V2 checkpoints should be used based on the table's * checkpoint policy and configuration settings. It performs the following checks: * * 1. Force Classic Checkpoint Check (Edge): If the Spark configuration * [[DeltaSQLConf.FORCE_CLASSIC_CHECKPOINT]] is set to true (typically due to * a file action count mismatch), this method returns None to force the use * of classic checkpoints. * * 2. V2 Checkpoint Policy Check: Examines the table's checkpoint policy from * the snapshot metadata to determine if V2 checkpoint support is required. * * 3. Format Selection: If V2 checkpoints are enabled, determines the format * for the top-level checkpoint file based on the * [[DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT]] configuration: * - JSON format (default if not specified) * - PARQUET format * * @param spark The SparkSession to retrieve configuration settings * @param snapshot The snapshot for which to determine the checkpoint format * @return Some(V2Checkpoint.Format) if V2 checkpoints should be used with the * specified format (JSON or PARQUET), or None if classic checkpoints * should be used * @throws IllegalStateException if an unknown checkpoint format is specified * in the configuration */ def getV2CheckpointFormatOpt( spark: SparkSession, snapshot: Snapshot): Option[V2Checkpoint.Format] = { val policy = DeltaConfigs.CHECKPOINT_POLICY.fromMetaData(snapshot.metadata) if (policy.needsV2CheckpointSupport) { assert(CheckpointProvider.isV2CheckpointEnabled(snapshot)) val v2Format = spark.conf.getOption(DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key) // The format of the top level file in V2 checkpoints can be configured through // the optional config [[DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT]]. // If nothing is specified, we use the json format. In the future, we may // write json/parquet dynamically based on heuristics. v2Format match { case Some(V2Checkpoint.Format.JSON.name) | None => Some(V2Checkpoint.Format.JSON) case Some(V2Checkpoint.Format.PARQUET.name) => Some(V2Checkpoint.Format.PARQUET) case _ => throw new IllegalStateException("unknown checkpoint format") } } else { None } } /** * Returns the checkpoint schema that should be written to the last checkpoint file based on * [[DeltaSQLConf.CHECKPOINT_SCHEMA_WRITE_THRESHOLD_LENGTH]] conf. */ private[delta] def checkpointSchemaToWriteInLastCheckpointFile( spark: SparkSession, schema: StructType): Option[StructType] = { val checkpointSchemaSizeThreshold = spark.sessionState.conf.getConf( DeltaSQLConf.CHECKPOINT_SCHEMA_WRITE_THRESHOLD_LENGTH) Some(schema).filter(s => JsonUtils.toJson(s).length <= checkpointSchemaSizeThreshold) } /** * Writes out the contents of a [[Snapshot]] into a checkpoint file that * can be used to short-circuit future replays of the log. * * Returns the checkpoint metadata to be committed to a file. We will use the value * in this file as the source of truth of the last valid checkpoint. */ private[delta] def writeCheckpoint( spark: SparkSession, deltaLog: DeltaLog, snapshot: Snapshot, catalogTableOpt: Option[CatalogTable]): LastCheckpointInfo = recordFrameProfile( "Delta", "Checkpoints.writeCheckpoint") { if (spark.conf.get(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED)) { snapshot.validateChecksum(Map("context" -> "writeCheckpoint")) } // Verify allFiles in checksum during checkpoint if we are not doing so already on every // commit. val allFilesInCRCEnabled = Snapshot.allFilesInCrcWritePathEnabled(spark, snapshot) val shouldVerifyAllFilesInCRCEveryCommit = Snapshot.allFilesInCrcVerificationEnabled(spark, snapshot) if (allFilesInCRCEnabled && !shouldVerifyAllFilesInCRCEveryCommit) { snapshot.checksumOpt.foreach { checksum => snapshot.validateFileListAgainstCRC( checksum, contextOpt = Some("triggeredFromCheckpoint")) } } val hadoopConf = deltaLog.newDeltaHadoopConf() // The writing of checkpoints doesn't go through log store, so we need to check with the // log store and decide whether to use rename. val useRename = deltaLog.store.isPartialWriteVisible(deltaLog.logPath, hadoopConf) val v2CheckpointFormatOpt = getV2CheckpointFormatOpt(spark, snapshot) val v2CheckpointEnabled = v2CheckpointFormatOpt.nonEmpty if (!v2CheckpointEnabled) { // Ensures that commit files are backfilled for Catalog-Managed (CC) tables when // writing Classic checkpoints. // // For CC tables with Classic checkpoint format (V2 checkpoint disabled), this method // ensures that commit files are *synchronously* backfilled from staged commits to the // _delta_log directory before writing the checkpoint. This prevents gaps in the // directory structure that could cause issues for readers not communicating with // the commit coordinator. // // Without backfilling, the directory structure might have gaps like: // {{{ // _delta_log/ // _staged_commits/ // 00017.$uuid.json // 00018.$uuid.json // 00015.json // 00016.json // 00018.checkpoint.parquet // Gap: missing 00017.json // }}} snapshot.ensureCommitFilesBackfilled(catalogTableOpt) } val checkpointRowCount = spark.sparkContext.longAccumulator("checkpointRowCount") val numOfFiles = spark.sparkContext.longAccumulator("numOfFiles") val sessionConf = spark.sessionState.conf val checkpointPartSize = sessionConf.getConf(DeltaSQLConf.DELTA_CHECKPOINT_PART_SIZE) val numParts = checkpointPartSize.map { partSize => math.ceil((snapshot.numOfFiles + snapshot.numOfRemoves).toDouble / partSize).toLong }.getOrElse(1L).toInt val legacyMultiPartCheckpoint = !v2CheckpointEnabled && numParts > 1 val base = { val repartitioned = snapshot.stateDS .repartition(numParts, coalesce(col("add.path"), col("remove.path"))) .map { action => if (action.add != null) { numOfFiles.add(1) } action } // commitInfo, cdc and remove.tags are not included in both classic and V2 checkpoints. if (v2CheckpointEnabled) { // When V2 Checkpoint is enabled, the baseCheckpoint refers to the sidecar files which will // only have AddFile and RemoveFile actions. The other non-file actions will be written // separately after sidecar files are written. repartitioned .select("add", "remove") .withColumn("remove", col("remove").dropFields("tags", "stats")) .where("add is not null or remove is not null") } else { // When V2 Checkpoint is disabled, the baseCheckpoint refers to the main classic checkpoint // which has all actions except "commitInfo", "cdc", "checkpointMetadata", "sidecar". repartitioned .drop("commitInfo", "cdc", "checkpointMetadata", "sidecar") .withColumn("remove", col("remove").dropFields("tags", "stats")) } } val chk = buildCheckpoint(base, snapshot) val schema = chk.schema.asNullable val (factory, serConf) = { val format = new ParquetFileFormat() val job = Job.getInstance(hadoopConf) // Right now, we don't shred variant stats in checkpoints. val writeOptions = VariantShreddingShims.getVariantInferShreddingSchemaOptions(false) (format.prepareWrite(spark, job, Map.empty ++ writeOptions, schema), new SerializableConfiguration(job.getConfiguration)) } // Use the SparkPath in the closure as Path is not Serializable. val logSparkPath = SparkPath.fromPath(snapshot.path) val version = snapshot.version // This is a hack to get spark to write directly to a file. val qe = chk.queryExecution def executeFinalCheckpointFiles(): Array[SerializableFileStatus] = qe .executedPlan .execute() .mapPartitions { case iter => val actualNumParts = Option(TaskContext.get()).map(_.numPartitions()) .getOrElse(numParts) val partition = TaskContext.getPartitionId() val (writtenPath, finalPath) = Checkpoints.getCheckpointWritePath( serConf.value, logSparkPath.toPath, version, actualNumParts, partition, useRename, v2CheckpointEnabled) val fs = writtenPath.getFileSystem(serConf.value) val writeAction = () => { try { val writer = factory.newInstance( writtenPath.toString, schema, new TaskAttemptContextImpl( new JobConf(serConf.value), new TaskAttemptID("", 0, TaskType.REDUCE, 0, 0))) iter.foreach { row => checkpointRowCount.add(1) writer.write(row) } // Note: `writer.close()` is not put in a `finally` clause because we don't want to // close it when an exception happens. Closing the file would flush the content to the // storage and create an incomplete file. A concurrent reader might see it and fail. // This would leak resources but we don't have a way to abort the storage request here. writer.close() } catch { case e: org.apache.hadoop.fs.FileAlreadyExistsException if !useRename => if (fs.exists(writtenPath)) { // The file has been written by a zombie task. We can just use this checkpoint file // rather than failing a Delta commit. } else { throw e } } } if (isGCSPath(serConf.value, writtenPath)) { // GCS may upload an incomplete file when the current thread is interrupted, hence we move // the write to a new thread so that the write cannot be interrupted. // TODO Remove this hack when the GCS Hadoop connector fixes the issue. DeltaFileOperations.runInNewThread("delta-gcs-checkpoint-write") { writeAction() } } else { writeAction() } if (useRename) { renameAndCleanupTempPartFile(writtenPath, finalPath, fs) } val finalPathFileStatus = try { fs.getFileStatus(finalPath) } catch { case _: FileNotFoundException if useRename => throw DeltaErrors.failOnCheckpointRename(writtenPath, finalPath) } Iterator(SerializableFileStatus.fromStatus(finalPathFileStatus)) }.collect() val finalCheckpointFiles = SQLExecution.withNewExecutionId(qe, Some("Delta checkpoint")) { executeFinalCheckpointFiles() } if (numOfFiles.value != snapshot.numOfFiles) { throw DeltaErrors.checkpointMismatchWithSnapshot } val parquetFilesSizeInBytes = finalCheckpointFiles.map(_.length).sum var overallCheckpointSizeInBytes = parquetFilesSizeInBytes var overallNumCheckpointActions: Long = checkpointRowCount.value var checkpointSchemaToWriteInLastCheckpoint: Option[StructType] = Checkpoints.checkpointSchemaToWriteInLastCheckpointFile(spark, schema) val v2Checkpoint = if (v2CheckpointEnabled) { // For CC tables, ensure commit files are backfilled right before publishing the // V2 checkpoint manifest. // At this moment, any existing async commit backfill operations almost certainly // would have completed as the full state reconstruction usually takes longer than // commit backfilling. snapshot.ensureCommitFilesBackfilled(catalogTableOpt) val (v2CheckpointFileStatus, nonFileActionsWriten, v2Checkpoint, checkpointSchema) = Checkpoints.writeTopLevelV2Checkpoint( v2CheckpointFormatOpt.get, finalCheckpointFiles, spark, schema, snapshot, deltaLog, overallNumCheckpointActions, parquetFilesSizeInBytes, hadoopConf, useRename ) overallCheckpointSizeInBytes += v2CheckpointFileStatus.getLen overallNumCheckpointActions += nonFileActionsWriten.size checkpointSchemaToWriteInLastCheckpoint = checkpointSchema Some(v2Checkpoint) } else { None } if (!v2CheckpointEnabled && checkpointRowCount.value == 0) { // In case of V2 Checkpoints, zero row count is possible. logWarning(DeltaErrors.EmptyCheckpointErrorMessage) } // If we don't parallelize, we use None for backwards compatibility val checkpointParts = if (legacyMultiPartCheckpoint) Some(numParts) else None LastCheckpointInfo( version = snapshot.version, size = overallNumCheckpointActions, parts = checkpointParts, sizeInBytes = Some(overallCheckpointSizeInBytes), numOfAddFiles = Some(snapshot.numOfFiles), v2Checkpoint = v2Checkpoint, checkpointSchema = checkpointSchemaToWriteInLastCheckpoint ) } /** * Generate a tuple of the file to write the checkpoint and where it may later need * to be copied. Should be used within a task, so that task or stage retries don't * create the same files. */ def getCheckpointWritePath( conf: Configuration, logPath: Path, version: Long, numParts: Int, part: Int, useRename: Boolean, v2CheckpointEnabled: Boolean): (Path, Path) = { def getCheckpointWritePath(path: Path): Path = { if (useRename) { val tempPath = new Path(path.getParent, s".${path.getName}.${UUID.randomUUID}.tmp") DeltaFileOperations.registerTempFileDeletionTaskFailureListener(conf, tempPath) tempPath } else { path } } val destinationName: Path = if (v2CheckpointEnabled) { newV2CheckpointSidecarFile(logPath, version, numParts, part + 1) } else { if (numParts > 1) { assert(part < numParts, s"Asked to create part: $part of max $numParts in checkpoint.") checkpointFileWithParts(logPath, version, numParts)(part) } else { checkpointFileSingular(logPath, version) } } getCheckpointWritePath(destinationName) -> destinationName } /** * Writes a top-level V2 Checkpoint file which may point to multiple * sidecar files. * * @param v2CheckpointFormat The format in which the top-level file should be * written. Currently, json and parquet are supported. * @param sidecarCheckpointFiles The list of sidecar files that have already been * written. The top-level file will store this list. * @param spark The current spark session * @param sidecarSchema The schema of the sidecar parquet files. * @param snapshot The snapshot for which the checkpoint is being written. * @param deltaLog The deltaLog instance pointing to our tables deltaLog. * @param rowsWrittenInCheckpointJob The number of rows that were written in total * to the sidecar files. * @param parquetFilesSizeInBytes The combined size of all sidecar files in bytes. * @param hadoopConf The hadoopConf to use for the filesystem operation. * @param useRename Whether we should first write to a temporary file and then * rename it to the target file name during the write. * @return A tuple containing * 1. [[FileStatus]] of the newly created top-level V2Checkpoint. * 2. The sequence of actions that were written to the top-level file. * 3. An instance of the LastCheckpointV2 containing V2-checkpoint related * metadata which can later be written to LAST_CHECKPOINT * 4. Schema of the newly written top-level file (only for parquet files) */ protected[delta] def writeTopLevelV2Checkpoint( v2CheckpointFormat: V2Checkpoint.Format, sidecarCheckpointFiles: Array[SerializableFileStatus], spark: SparkSession, sidecarSchema: StructType, snapshot: Snapshot, deltaLog: DeltaLog, rowsWrittenInCheckpointJob: Long, parquetFilesSizeInBytes: Long, hadoopConf: Configuration, useRename: Boolean) : (FileStatus, Seq[Action], LastCheckpointV2, Option[StructType]) = { // Write the main v2 checkpoint file. val sidecarFilesWritten = sidecarCheckpointFiles.map(SidecarFile(_)).toSeq // Filter out the sidecar schema if it is too large. val sidecarFileSchemaOpt = Checkpoints.checkpointSchemaToWriteInLastCheckpointFile(spark, sidecarSchema) val checkpointMetadata = CheckpointMetadata(snapshot.version) val nonFileActionsToWrite = (checkpointMetadata +: sidecarFilesWritten) ++ snapshot.nonFileActions val (v2CheckpointPath, checkpointSchemaToWriteInLastCheckpoint) = if (v2CheckpointFormat == V2Checkpoint.Format.JSON) { val v2CheckpointPath = newV2CheckpointJsonFile(deltaLog.logPath, snapshot.version) // We don't need a putIfAbsent for this write, so we set overwrite to true. // However, this can be dangerous if the cloud makes partial writes visible. val isPartialWriteVisible = deltaLog.store.isPartialWriteVisible(v2CheckpointPath, hadoopConf) deltaLog.store.write( v2CheckpointPath, nonFileActionsToWrite.map(_.json).toIterator, overwrite = !isPartialWriteVisible, hadoopConf = hadoopConf ) (v2CheckpointPath, None) } else if (v2CheckpointFormat == V2Checkpoint.Format.PARQUET) { val sparkSession = spark // scalastyle:off sparkimplicits import sparkSession.implicits._ // scalastyle:on sparkimplicits val dfToWrite = nonFileActionsToWrite.map(_.wrap).toDF() val v2CheckpointPath = newV2CheckpointParquetFile(deltaLog.logPath, snapshot.version) val schemaOfDfWritten = createCheckpointV2ParquetFile( spark, dfToWrite, v2CheckpointPath, hadoopConf, useRename) (v2CheckpointPath, Some(schemaOfDfWritten)) } else { throw DeltaErrors.assertionFailedError( s"Unrecognized checkpoint V2 format: $v2CheckpointFormat") } // Main Checkpoint V2 File written successfully. Now create the last checkpoint v2 blob so // that we can persist it in _last_checkpoint file. val v2CheckpointFileStatus = v2CheckpointPath.getFileSystem(hadoopConf).getFileStatus(v2CheckpointPath) val unfilteredV2Checkpoint = LastCheckpointV2( fileStatus = v2CheckpointFileStatus, nonFileActions = Some((snapshot.nonFileActions :+ checkpointMetadata).map(_.wrap)), sidecarFiles = Some(sidecarFilesWritten) ) ( v2CheckpointFileStatus, nonFileActionsToWrite, trimLastCheckpointV2(unfilteredV2Checkpoint, spark), checkpointSchemaToWriteInLastCheckpoint ) } /** * Helper method to create a V2 Checkpoint parquet file or the V2 Checkpoint Compat file. * V2 Checkpoint Compat files follow the same naming convention as classic checkpoints * and they are needed so that V2Checkpoint-unaware readers can read them to understand * that they don't have the capability to read table for which they were created. * This is needed in cases where commit 0 has been cleaned up and the reader needs to * read a checkpoint to read the [[Protocol]]. */ def createCheckpointV2ParquetFile( spark: SparkSession, ds: Dataset[Row], finalPath: Path, hadoopConf: Configuration, useRename: Boolean): StructType = recordFrameProfile( "Checkpoints", "createCheckpointV2ParquetFile") { val df = ds.select( "txn", "add", "remove", "metaData", "protocol", "domainMetadata", "checkpointMetadata", "sidecar") val schema = df.schema.asNullable val format = new ParquetFileFormat() val job = Job.getInstance(hadoopConf) val factory = format.prepareWrite(spark, job, Map.empty, schema) val serConf = new SerializableConfiguration(job.getConfiguration) val finalSparkPath = SparkPath.fromPath(finalPath) df.repartition(1) .queryExecution .executedPlan .execute() .mapPartitions { iter => val actualNumParts = Option(TaskContext.get()).map(_.numPartitions()).getOrElse(1) require(actualNumParts == 1, "The parquet V2 checkpoint must be written in 1 file") val partition = TaskContext.getPartitionId() val finalPath = finalSparkPath.toPath val writePath = if (useRename) { val tempPath = new Path(finalPath.getParent, s".${finalPath.getName}.${UUID.randomUUID}.tmp") DeltaFileOperations.registerTempFileDeletionTaskFailureListener(serConf.value, tempPath) tempPath } else { finalPath } val fs = writePath.getFileSystem(serConf.value) val attemptId = 0 val taskAttemptContext = new TaskAttemptContextImpl( new JobConf(serConf.value), new TaskAttemptID("", 0, TaskType.REDUCE, partition, attemptId)) var writerOpt: Option[OutputWriter] = None try { writerOpt = Some(factory.newInstance( writePath.toString, schema, taskAttemptContext)) val writer = writerOpt.get iter.foreach { row => writer.write(row) } // Note: `writer.close()` is not put in a `finally` clause because we don't want to // close it when an exception happens. Closing the file would flush the content to the // storage and create an incomplete file. A concurrent reader might see it and fail. // This would leak resources but we don't have a way to abort the storage request here. writer.close() } catch { case _: org.apache.hadoop.fs.FileAlreadyExistsException if !useRename && fs.exists(writePath) => // The file has been written by a zombie task. We can just use this checkpoint file // rather than failing a Delta commit. case t: Throwable => throw t } if (useRename) { renameAndCleanupTempPartFile(writePath, finalPath, fs) } val finalPathFileStatus = try { fs.getFileStatus(finalPath) } catch { case _: FileNotFoundException if useRename => throw DeltaErrors.failOnCheckpointRename(writePath, finalPath) } Iterator(SerializableFileStatus.fromStatus(finalPathFileStatus)) }.collect() schema } /** Bounds the size of a [[LastCheckpointV2]] by removing any oversized optional fields */ def trimLastCheckpointV2( lastCheckpointV2: LastCheckpointV2, spark: SparkSession): LastCheckpointV2 = { val nonFileActionThreshold = spark.sessionState.conf.getConf(DeltaSQLConf.LAST_CHECKPOINT_NON_FILE_ACTIONS_THRESHOLD) val sidecarThreshold = spark.sessionState.conf.getConf(DeltaSQLConf.LAST_CHECKPOINT_SIDECARS_THRESHOLD) lastCheckpointV2.copy( sidecarFiles = lastCheckpointV2.sidecarFiles.filter(_.size <= sidecarThreshold), nonFileActions = lastCheckpointV2.nonFileActions.filter(_.size <= nonFileActionThreshold)) } /** * Helper method to rename a `tempPath` checkpoint part file to `finalPath` checkpoint part file. * This also tries to handle any race conditions with Zombie tasks. */ private[delta] def renameAndCleanupTempPartFile( tempPath: Path, finalPath: Path, fs: FileSystem): Unit = { // If rename fails because the final path already exists, it's ok -- some zombie // task probably got there first. // We rely on the fact that all checkpoint writers write the same content to any given // checkpoint part file. So it shouldn't matter which writer wins the race. val renameSuccessful = try { // Note that the fs.exists check here is redundant as fs.rename should fail if destination // file already exists as per File System spec. But the LocalFS doesn't follow this and it // overrides the final path even if it already exists. So we use exists here to handle that // case. // TODO: Remove isTesting and fs.exists check after fixing LocalFS if (DeltaUtils.isTesting && fs.exists(finalPath)) { false } else { fs.rename(tempPath, finalPath) } } catch { case _: org.apache.hadoop.fs.FileAlreadyExistsException => false } if (!renameSuccessful) { try { fs.delete(tempPath, false) } catch { case NonFatal(e) => logWarning(log"Error while deleting the temporary checkpoint part file " + log"${MDC(DeltaLogKeys.PATH, tempPath)}", e) } } } // scalastyle:off line.size.limit /** * All GCS paths can only have the scheme of "gs". Note: the scheme checking is case insensitive. * See: * - https://github.com/databricks/hadoop-connectors/blob/master/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemBase.java#L493 * - https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/v2.2.3/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystem.java#L88 */ // scalastyle:on line.size.limit private[delta] def isGCSPath(hadoopConf: Configuration, path: Path): Boolean = { val scheme = path.toUri.getScheme if (scheme != null) { scheme.equalsIgnoreCase("gs") } else { // When the schema is not available in the path, we check the file system scheme resolved from // the path. path.getFileSystem(hadoopConf).getScheme.equalsIgnoreCase("gs") } } /** * Modify the contents of the add column based on the table properties */ private[delta] def buildCheckpoint(state: DataFrame, snapshot: Snapshot): DataFrame = { val additionalCols = new mutable.ArrayBuffer[Column]() val sessionConf = state.sparkSession.sessionState.conf if (Checkpoints.shouldWriteStatsAsJson(snapshot)) { additionalCols += col("add.stats").as("stats") } // We provide fine grained control using the session conf for now, until users explicitly // opt in our out of the struct conf. val includeStructColumns = shouldWriteStatsAsStruct(sessionConf, snapshot) if (includeStructColumns) { val partitionValues = Checkpoints.extractPartitionValues( snapshot.metadata.partitionSchema, "add.partitionValues") additionalCols ++= partitionValues additionalCols ++= Checkpoints.extractStats(snapshot.statsSchema, "add.stats") } state.withColumn("add", when(col("add").isNotNull, struct(Seq( col("add.path"), col("add.partitionValues"), col("add.size"), col("add.modificationTime"), col("add.dataChange"), // actually not really useful here col("add.tags"), col("add.deletionVector"), col("add.baseRowId"), col("add.defaultRowCommitVersion"), col("add.clusteringProvider")) ++ additionalCols: _* )) ) } def shouldWriteStatsAsStruct(conf: SQLConf, snapshot: Snapshot): Boolean = { DeltaConfigs.CHECKPOINT_WRITE_STATS_AS_STRUCT.fromMetaData(snapshot.metadata) && !conf.getConf(DeltaSQLConf.STATS_AS_STRUCT_IN_CHECKPOINT_FORCE_DISABLED).getOrElse(false) } def shouldWriteStatsAsJson(snapshot: Snapshot): Boolean = { DeltaConfigs.CHECKPOINT_WRITE_STATS_AS_JSON.fromMetaData(snapshot.metadata) } val STRUCT_PARTITIONS_COL_NAME = "partitionValues_parsed" val STRUCT_STATS_COL_NAME = "stats_parsed" /** * Creates a nested struct column of partition values that extract the partition values * from the original MapType. */ def extractPartitionValues(partitionSchema: StructType, partitionValuesColName: String): Option[Column] = { val partitionValues = partitionSchema.map { field => val physicalName = DeltaColumnMapping.getPhysicalName(field) val attribute = UnresolvedAttribute.quotedString(partitionValuesColName) Column(Cast( ElementAt( attribute, Literal(physicalName), failOnError = false), field.dataType, ansiEnabled = false) ).as(physicalName) } if (partitionValues.isEmpty) { None } else Some(struct(partitionValues: _*).as(STRUCT_PARTITIONS_COL_NAME)) } // This method can be overridden in tests to create a checkpoint with parsed stats. def includeStatsParsedInCheckpoint(): Boolean = true /** Parse the stats from JSON and keep as a struct field when available. */ def extractStats(statsSchema: StructType, statsColName: String): Option[Column] = { import org.apache.spark.sql.functions.from_json Option.when(includeStatsParsedInCheckpoint() && statsSchema.nonEmpty) { val parsedStats = from_json(col(statsColName), statsSchema, DeltaFileProviderUtils.jsonStatsParseOption) // If schema contains variant types, decode Z85-encoded strings to actual Variant values. // In JSON stats, variant values are stored as Z85-encoded strings. from_json creates // Variant objects containing those strings. DecodeNestedZ85EncodedVariant decodes them // to proper binary Variant representation. val decodedStats = if (SchemaUtils.checkForVariantTypeColumnsRecursively(statsSchema)) { Column(DecodeNestedZ85EncodedVariant(parsedStats.expr)) } else { parsedStats } decodedStats.as(Checkpoints.STRUCT_STATS_COL_NAME) } } } object V2Checkpoint { /** Format for V2 Checkpoints */ sealed abstract class Format(val name: String) { def fileFormat: FileFormat } def toFormat(fileName: String): Format = fileName match { case _ if fileName.endsWith(Format.JSON.name) => Format.JSON case _ if fileName.endsWith(Format.PARQUET.name) => Format.PARQUET case _ => throw new IllegalStateException(s"Unknown v2 checkpoint file format: ${fileName}") } object Format { /** json v2 checkpoint */ object JSON extends Format("json") { override def fileFormat: FileFormat = DeltaLogFileIndex.CHECKPOINT_FILE_FORMAT_JSON } /** parquet v2 checkpoint */ object PARQUET extends Format("parquet") { override def fileFormat: FileFormat = DeltaLogFileIndex.CHECKPOINT_FILE_FORMAT_PARQUET } /** All valid formats for the top level file of v2 checkpoints. */ val ALL: Set[Format] = Set(Format.JSON, Format.PARQUET) /** The string representations of all the valid formats. */ val ALL_AS_STRINGS: Set[String] = ALL.map(_.name) } } object CheckpointPolicy { sealed abstract class Policy(val name: String) { override def toString: String = name def needsV2CheckpointSupport: Boolean = true } /** * Write classic single file/multi-part checkpoints when this policy is enabled. * Note that [[V2CheckpointTableFeature]] is not required for this checkpoint policy. */ case object Classic extends Policy("classic") { override def needsV2CheckpointSupport: Boolean = false } /** * Write V2 checkpoints when this policy is enabled. * This needs [[V2CheckpointTableFeature]] to be enabled on the table. */ case object V2 extends Policy("v2") /** ALl checkpoint policies */ val ALL: Seq[Policy] = Seq(Classic, V2) /** Converts a `name` String into a [[Policy]] */ def fromName(name: String): Policy = ALL.find(_.name == name).getOrElse { throw new IllegalArgumentException(s"Invalid policy $name") } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/Checksum.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.FileNotFoundException import java.nio.charset.StandardCharsets.UTF_8 import java.util.TimeZone // scalastyle:off import.ordering.noEmptyLine import scala.collection.immutable.ListMap import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import scala.util.control.NonFatal import org.apache.spark.sql.delta.Relocated._ import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.DeletedRecordCountsHistogram import org.apache.spark.sql.delta.stats.FileSizeHistogram import org.apache.spark.sql.delta.storage.LogStore import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} import com.fasterxml.jackson.annotation.JsonAlias import com.fasterxml.jackson.databind.annotation.JsonDeserialize import org.apache.hadoop.fs.FileStatus import org.apache.hadoop.fs.Path import org.apache.spark.SparkEnv import org.apache.spark.internal.MDC import org.apache.spark.sql.SparkSession import org.apache.spark.util.{SerializableConfiguration, Utils} /** * Stats calculated within a snapshot, which we store along individual transactions for * verification. * * @param txnId Optional transaction identifier * @param tableSizeBytes The size of the table in bytes * @param numFiles Number of `AddFile` actions in the snapshot * @param numDeletedRecordsOpt The number of deleted records with Deletion Vectors. * @param numDeletionVectorsOpt The number of Deletion Vectors present in the snapshot. * @param numMetadata Number of `Metadata` actions in the snapshot * @param numProtocol Number of `Protocol` actions in the snapshot * @param histogramOpt Optional file size histogram. Note: the Delta spec field name is * `fileSizeHistogram` (used by Kernel/Java/Rust). Delta-Spark historically * wrote `histogramOpt`. The `@JsonAlias` allows reading both field names so * that CRC files written by either Kernel or Delta-Spark are compatible. * @param deletedRecordCountsHistogramOpt A histogram of the deleted records count distribution * for all the files in the snapshot. */ case class VersionChecksum( txnId: Option[String], tableSizeBytes: Long, numFiles: Long, @JsonDeserialize(contentAs = classOf[Long]) numDeletedRecordsOpt: Option[Long], @JsonDeserialize(contentAs = classOf[Long]) numDeletionVectorsOpt: Option[Long], numMetadata: Long, numProtocol: Long, @JsonDeserialize(contentAs = classOf[Long]) inCommitTimestampOpt: Option[Long], setTransactions: Option[Seq[SetTransaction]], domainMetadata: Option[Seq[DomainMetadata]], metadata: Metadata, protocol: Protocol, // Accept both "histogramOpt" (legacy Delta-Spark) and // "fileSizeHistogram" (Delta spec / Kernel). @JsonAlias(Array("fileSizeHistogram")) histogramOpt: Option[FileSizeHistogram], deletedRecordCountsHistogramOpt: Option[DeletedRecordCountsHistogram], allFiles: Option[Seq[AddFile]]) { /** * Converts to the protocol-compliant representation that serializes the histogram field as * `fileSizeHistogram` (the Delta spec field name) instead of `histogramOpt`. */ def toProtocolCompliant: VersionChecksumProtocolCompliant = VersionChecksumProtocolCompliant( txnId = txnId, tableSizeBytes = tableSizeBytes, numFiles = numFiles, numDeletedRecordsOpt = numDeletedRecordsOpt, numDeletionVectorsOpt = numDeletionVectorsOpt, numMetadata = numMetadata, numProtocol = numProtocol, inCommitTimestampOpt = inCommitTimestampOpt, setTransactions = setTransactions, domainMetadata = domainMetadata, metadata = metadata, protocol = protocol, fileSizeHistogram = histogramOpt, deletedRecordCountsHistogramOpt = deletedRecordCountsHistogramOpt, allFiles = allFiles ) } /** * Protocol-compliant version of [[VersionChecksum]] that serializes the file size histogram * using the Delta spec field name `fileSizeHistogram` instead of the legacy `histogramOpt`. * Used only for CRC file writes when the protocol-compliant flag is enabled. */ case class VersionChecksumProtocolCompliant( txnId: Option[String], tableSizeBytes: Long, numFiles: Long, @JsonDeserialize(contentAs = classOf[Long]) numDeletedRecordsOpt: Option[Long], @JsonDeserialize(contentAs = classOf[Long]) numDeletionVectorsOpt: Option[Long], numMetadata: Long, numProtocol: Long, @JsonDeserialize(contentAs = classOf[Long]) inCommitTimestampOpt: Option[Long], setTransactions: Option[Seq[SetTransaction]], domainMetadata: Option[Seq[DomainMetadata]], metadata: Metadata, protocol: Protocol, fileSizeHistogram: Option[FileSizeHistogram], deletedRecordCountsHistogramOpt: Option[DeletedRecordCountsHistogram], allFiles: Option[Seq[AddFile]]) /** * Record the state of the table as a checksum file along with a commit. */ trait RecordChecksum extends DeltaLogging { val deltaLog: DeltaLog protected def spark: SparkSession private lazy val writer = CheckpointFileManager.create(deltaLog.logPath, deltaLog.newDeltaHadoopConf()) private def getChecksum(snapshot: Snapshot): VersionChecksum = snapshot.computeChecksum protected def writeChecksumFile(txnId: String, snapshot: Snapshot): Unit = { if (!spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED)) { return } val checksumWithoutTxnId = getChecksum(snapshot) val checksum = checksumWithoutTxnId.copy(txnId = Some(txnId)) writeChecksumFile(snapshot.version, checksum) } protected def writeChecksumFile(version: Long, checksum: VersionChecksum): Unit = { val eventData = mutable.Map[String, Any]("operationSucceeded" -> false) eventData("numAddFileActions") = checksum.allFiles.map(_.size).getOrElse(-1) eventData("numSetTransactionActions") = checksum.setTransactions.map(_.size).getOrElse(-1) val startTimeMs = System.currentTimeMillis() try { val toWrite = (if (spark.conf.get( DeltaSQLConf.DELTA_CHECKSUM_HISTOGRAM_FIELD_FOLLOWS_PROTOCOL)) { JsonUtils.toJson(checksum.toProtocolCompliant) } else { JsonUtils.toJson(checksum) }) + "\n" eventData("jsonSerializationTimeTakenMs") = System.currentTimeMillis() - startTimeMs eventData("checksumLength") = toWrite.length val stream = writer.createAtomic( FileNames.checksumFile(deltaLog.logPath, version), overwriteIfPossible = false) try { stream.write(toWrite.getBytes(UTF_8)) stream.close() eventData("overallTimeTakenMs") = System.currentTimeMillis() - startTimeMs eventData("operationSucceeded") = true } catch { case NonFatal(e) => logWarning(log"Failed to write the checksum for version: " + log"${MDC(DeltaLogKeys.VERSION, version)}", e) stream.cancel() } } catch { case NonFatal(e) => logWarning(log"Failed to write the checksum for version: " + log"${MDC(DeltaLogKeys.VERSION, version)}", e) } recordDeltaEvent( deltaLog, opType = "delta.checksum.write", data = eventData) } /** * Incrementally derive checksum for the just-committed or about-to-be committed snapshot. * @param spark The SparkSession * @param deltaLog The DeltaLog * @param versionToCompute The version for which we want to compute the checksum * @param actions The actions corresponding to the version `versionToCompute` * @param metadataOpt The metadata corresponding to the version `versionToCompute` (if known) * @param protocolOpt The protocol corresponding to the version `versionToCompute` (if known) * @param operationName The operation name corresponding to the version `versionToCompute` * @param txnIdOpt The transaction identifier for the version `versionToCompute` * @param previousVersionState Contains either the versionChecksum corresponding to * `versionToCompute - 1` or a snapshot. Note that the snapshot may * belong to any version and this method will only use the snapshot if * it corresponds to `versionToCompute - 1`. * @param includeAddFilesInCrc True if the new checksum should include a [[AddFile]]s. * @return Either the new checksum or an error code string if the checksum could not be computed. */ // scalastyle:off argcount def incrementallyDeriveChecksum( spark: SparkSession, deltaLog: DeltaLog, versionToCompute: Long, actions: Seq[Action], metadataOpt: Option[Metadata], protocolOpt: Option[Protocol], operationName: String, txnIdOpt: Option[String], previousVersionState: Either[Snapshot, VersionChecksum], includeAddFilesInCrc: Boolean ): Either[String, VersionChecksum] = { // scalastyle:on argcount if (!deltaLog.incrementalCommitEnabled) { return Left("INCREMENTAL_COMMITS_DISABLED") } // Do not incrementally derive checksum for ManualUpdate operations since it may // include actions that violate delta protocol invariants. if (operationName == DeltaOperations.ManualUpdate.name) { return Left("INVALID_OPERATION_MANUAL_UPDATE") } // Try to incrementally compute a VersionChecksum for the just-committed snapshot. val expectedVersion = versionToCompute - 1 val (oldVersionChecksum, oldSnapshot) = previousVersionState match { case Right(checksum) => checksum -> None case Left(snapshot) if snapshot.version == expectedVersion => // The original snapshot is still fresh so use it directly. Note this could trigger // a state reconstruction if there is not an existing checksumOpt in the snapshot // or if the existing checksumOpt contains missing information e.g. // a null valued metadata or protocol. However, if we do not obtain a checksum here, // then we cannot incrementally derive a new checksum for the new snapshot. logInfo(log"Incremental commit: starting with snapshot version " + log"${MDC(DeltaLogKeys.VERSION, expectedVersion)}") getChecksum(snapshot).copy(numMetadata = 1, numProtocol = 1) -> Some(snapshot) case _ => previousVersionState.swap.foreach { snapshot => // Occurs when snapshot is no longer fresh due to concurrent writers. // Read CRC file and validate checksum information is complete. recordDeltaEvent(deltaLog, opType = "delta.commit.snapshotAgedOut", data = Map( "snapshotVersion" -> snapshot.version, "commitAttemptVersion" -> versionToCompute )) } val oldCrcOpt = deltaLog.readChecksum(expectedVersion) if (oldCrcOpt.isEmpty) { return Left("MISSING_OLD_CRC") } val oldCrcFiltered = oldCrcOpt .filterNot(_.metadata == null) .filterNot(_.protocol == null) val oldCrc = oldCrcFiltered.getOrElse { return Left("OLD_CRC_INCOMPLETE") } oldCrc -> None } // Incrementally compute the new version checksum, if the old one is available. val ignoreAddFilesInOperation = RecordChecksum.operationNamesWhereAddFilesIgnoredForIncrementalCrc.contains(operationName) val ignoreRemoveFilesInOperation = RecordChecksum.operationNamesWhereRemoveFilesIgnoredForIncrementalCrc.contains(operationName) // Retrieve protocol/metadata in order of precedence: // 1. Use provided protocol/metadata if available // 2. Look for a protocol/metadata action in the incremental set of actions to be applied // 3. Use protocol/metadata from previous version's checksum // 4. Return PROTOCOL_MISSING/METADATA_MISSING error if all attempts fail val protocol = protocolOpt .orElse(actions.collectFirst { case p: Protocol => p }) .orElse(Option(oldVersionChecksum.protocol)) .getOrElse { return Left("PROTOCOL_MISSING") } val metadata = metadataOpt .orElse(actions.collectFirst { case m: Metadata => m }) .orElse(Option(oldVersionChecksum.metadata)) .getOrElse { return Left("METADATA_MISSING") } val persistentDVsOnTableReadable = DeletionVectorUtils.deletionVectorsReadable(protocol, metadata) val persistentDVsOnTableWritable = DeletionVectorUtils.deletionVectorsWritable(protocol, metadata) computeNewChecksum( versionToCompute, operationName, txnIdOpt, oldVersionChecksum, oldSnapshot, actions, ignoreAddFilesInOperation, ignoreRemoveFilesInOperation, includeAddFilesInCrc, persistentDVsOnTableReadable, persistentDVsOnTableWritable ) } /** * Incrementally derive new checksum from old checksum + actions. * * @param attemptVersion commit attempt version for which we want to generate CRC. * @param operationName operation name for the attempted commit. * @param txnId transaction identifier. * @param oldVersionChecksum from previous commit (attemptVersion - 1). * @param oldSnapshot snapshot representing previous commit version (i.e. attemptVersion - 1), * None if not available. * @param actions used to incrementally compute new checksum. * @param ignoreAddFiles for transactions whose add file actions refer to already-existing files * e.g., [[DeltaOperations.ComputeStats]] transactions. * @param ignoreRemoveFiles for transactions that generate RemoveFiles for auxiliary files * e.g., [[DeltaOperations.AddDeletionVectorsTombstones]]. * @param persistentDVsOnTableReadable Indicates whether commands modifying this table are allowed * to read deletion vectors. * @param persistentDVsOnTableWritable Indicates whether commands modifying this table are allowed * to create new deletion vectors. * @return Either the new checksum or error code string if the checksum could not be computed * incrementally due to some reason. */ // scalastyle:off argcount private[delta] def computeNewChecksum( attemptVersion: Long, operationName: String, txnIdOpt: Option[String], oldVersionChecksum: VersionChecksum, oldSnapshot: Option[Snapshot], actions: Seq[Action], ignoreAddFiles: Boolean, ignoreRemoveFiles: Boolean, includeAllFilesInCRC: Boolean, persistentDVsOnTableReadable: Boolean, persistentDVsOnTableWritable: Boolean ) : Either[String, VersionChecksum] = { // scalastyle:on argcount oldSnapshot.foreach(s => require(s.version == (attemptVersion - 1))) var tableSizeBytes = oldVersionChecksum.tableSizeBytes var numFiles = oldVersionChecksum.numFiles var protocol = oldVersionChecksum.protocol var metadata = oldVersionChecksum.metadata // In incremental computation, tables initialized with DVs disabled contain None DV // statistics. DV statistics remain None even if DVs are enabled at a random point // during the lifecycle of a table. That can only change if a full snapshot recomputation // is invoked while DVs are enabled for the table. val conf = spark.sessionState.conf val isFirstVersion = oldSnapshot.forall(_.version == -1) val checksumDVMetricsEnabled = conf.getConf(DeltaSQLConf.DELTA_CHECKSUM_DV_METRICS_ENABLED) val deletedRecordCountsHistogramEnabled = conf.getConf(DeltaSQLConf.DELTA_DELETED_RECORD_COUNTS_HISTOGRAM_ENABLED) // For tables where DVs were disabled later on in the table lifecycle we want to maintain DV // statistics. val computeDVMetricsWhenDVsNotWritable = persistentDVsOnTableReadable && oldVersionChecksum.numDeletionVectorsOpt.isDefined && !isFirstVersion val computeDVMetrics = checksumDVMetricsEnabled && (persistentDVsOnTableWritable || computeDVMetricsWhenDVsNotWritable) // DV-related metrics. When the old checksum does not contain DV statistics, we attempt to // pick them up from the old snapshot. var numDeletedRecordsOpt = if (computeDVMetrics) { oldVersionChecksum.numDeletedRecordsOpt .orElse(oldSnapshot.flatMap(_.numDeletedRecordsOpt)) } else None var numDeletionVectorsOpt = if (computeDVMetrics) { oldVersionChecksum.numDeletionVectorsOpt .orElse(oldSnapshot.flatMap(_.numDeletionVectorsOpt)) } else None val deletedRecordCountsHistogramOpt = if (computeDVMetrics && deletedRecordCountsHistogramEnabled) { oldVersionChecksum.deletedRecordCountsHistogramOpt .orElse(oldSnapshot.flatMap(_.deletedRecordCountsHistogramOpt)) .map(h => DeletedRecordCountsHistogram(h.deletedRecordCounts.clone())) } else None var inCommitTimestamp : Option[Long] = None actions.foreach { case a: AddFile if !ignoreAddFiles => tableSizeBytes += a.size numFiles += 1 // Only accumulate DV statistics when base stats are not None. val (dvCount, dvCardinality) = Option(a.deletionVector).map(1L -> _.cardinality).getOrElse(0L -> 0L) numDeletedRecordsOpt = numDeletedRecordsOpt.map(_ + dvCardinality) numDeletionVectorsOpt = numDeletionVectorsOpt.map(_ + dvCount) deletedRecordCountsHistogramOpt.foreach(_.insert(dvCardinality)) case _: RemoveFile if ignoreRemoveFiles => () // extendedFileMetadata == true implies fields partitionValues, size, and tags are present case r: RemoveFile if r.extendedFileMetadata == Some(true) => val size = r.size.get tableSizeBytes -= size numFiles -= 1 // Only accumulate DV statistics when base stats are not None. val (dvCount, dvCardinality) = Option(r.deletionVector).map(1L -> _.cardinality).getOrElse(0L -> 0L) numDeletedRecordsOpt = numDeletedRecordsOpt.map(_ - dvCardinality) numDeletionVectorsOpt = numDeletionVectorsOpt.map(_ - dvCount) deletedRecordCountsHistogramOpt.foreach(_.remove(dvCardinality)) case r: RemoveFile => // Report the failure to usage logs. val msg = s"A remove action with a missing file size was detected in file ${r.path} " + "causing incremental commit to fallback to state reconstruction." recordDeltaEvent( this.deltaLog, "delta.checksum.compute", data = Map("error" -> msg)) return Left("ENCOUNTERED_REMOVE_FILE_MISSING_SIZE") case p: Protocol => protocol = p case m: Metadata => metadata = m case ci: CommitInfo => inCommitTimestamp = ci.inCommitTimestamp case _ => } val setTransactions = incrementallyComputeSetTransactions( oldSnapshot, oldVersionChecksum, attemptVersion, actions) val domainMetadata = incrementallyComputeDomainMetadatas( oldSnapshot, oldVersionChecksum, attemptVersion, actions) val computeAddFiles = if (includeAllFilesInCRC) { incrementallyComputeAddFiles( oldSnapshot = oldSnapshot, oldVersionChecksum = oldVersionChecksum, attemptVersion = attemptVersion, numFilesAfterCommit = numFiles, actionsToCommit = actions) } else if (numFiles == 0) { // If the table becomes empty after the commit, addFiles should be empty. Option(Nil) } else { None } val allFiles = computeAddFiles.filter { files => val computedNumFiles = files.size val computedTableSizeBytes = files.map(_.size).sum // Validate checksum of Incrementally computed files against the computed checksum from // incremental commits. if (computedNumFiles != numFiles || computedTableSizeBytes != tableSizeBytes) { val filePathsFromPreviousVersion = oldVersionChecksum.allFiles .orElse { recordFrameProfile("Delta", "VersionChecksum.computeNewChecksum.allFiles") { oldSnapshot.map(_.allFiles.collect().toSeq) } } .getOrElse(Seq.empty) .map(_.path) val addFilePathsInThisCommit = actions.collect { case af: AddFile => af.path } val removeFilePathsInThisCommit = actions.collect { case rf: RemoveFile => rf.path } logWarning(log"Incrementally computed files does not match the incremental checksum " + log"for commit attempt: ${MDC(DeltaLogKeys.VERSION, attemptVersion)}. " + log"addFilePathsInThisCommit: [${MDC(DeltaLogKeys.PATHS, addFilePathsInThisCommit.mkString(","))}], " + log"removeFilePathsInThisCommit: [${MDC(DeltaLogKeys.PATHS2, removeFilePathsInThisCommit.mkString(","))}], " + log"filePathsFromPreviousVersion: [${MDC(DeltaLogKeys.PATHS3, filePathsFromPreviousVersion.mkString(","))}], " + log"computedFiles: [${MDC(DeltaLogKeys.PATHS4, files.map(_.path).mkString(","))}]") val eventData = Map( "attemptVersion" -> attemptVersion, "expectedNumFiles" -> numFiles, "expectedTableSizeBytes" -> tableSizeBytes, "computedNumFiles" -> computedNumFiles, "computedTableSizeBytes" -> computedTableSizeBytes, "numAddFilePathsInThisCommit" -> addFilePathsInThisCommit.size, "numRemoveFilePathsInThisCommit" -> removeFilePathsInThisCommit.size, "numFilesInPreviousVersion" -> filePathsFromPreviousVersion.size, "operationName" -> operationName, "addFilePathsInThisCommit" -> JsonUtils.toJson(addFilePathsInThisCommit.take(10)), "removeFilePathsInThisCommit" -> JsonUtils.toJson(removeFilePathsInThisCommit.take(10)), "filePathsFromPreviousVersion" -> JsonUtils.toJson(filePathsFromPreviousVersion.take(10)), "computedFiles" -> JsonUtils.toJson(files.take(10)) ) recordDeltaEvent( deltaLog, opType = "delta.allFilesInCrc.checksumMismatch.aggregated", data = eventData) if (DeltaUtils.isTesting) { throw new IllegalStateException("Incrementally Computed State failed checksum check" + s" for commit $attemptVersion [$eventData]") } false } else { true } } Right(VersionChecksum( txnId = txnIdOpt, tableSizeBytes = tableSizeBytes, numFiles = numFiles, numDeletedRecordsOpt = numDeletedRecordsOpt, numDeletionVectorsOpt = numDeletionVectorsOpt, numMetadata = 1, numProtocol = 1, inCommitTimestampOpt = inCommitTimestamp, metadata = metadata, protocol = protocol, setTransactions = setTransactions, domainMetadata = domainMetadata, allFiles = allFiles, deletedRecordCountsHistogramOpt = deletedRecordCountsHistogramOpt, histogramOpt = None )) } /** * Incrementally compute [[Snapshot.setTransactions]] for the commit `attemptVersion`. * * @param oldSnapshot - snapshot corresponding to `attemptVersion` - 1 * @param oldVersionChecksum - [[VersionChecksum]] corresponding to `attemptVersion` - 1 * @param attemptVersion - version which we want to commit * @param actionsToCommit - actions for commit `attemptVersion` * @return Optional sequence of incrementally computed [[SetTransaction]]s for commit * `attemptVersion`. */ private def incrementallyComputeSetTransactions( oldSnapshot: Option[Snapshot], oldVersionChecksum: VersionChecksum, attemptVersion: Long, actionsToCommit: Seq[Action]): Option[Seq[SetTransaction]] = { // Check-1: check conf if (!spark.conf.get(DeltaSQLConf.DELTA_WRITE_SET_TRANSACTIONS_IN_CRC)) { return None } // Check-2: check `minSetTransactionRetentionTimestamp` is not set val newMetadataToCommit = actionsToCommit.collectFirst { case m: Metadata => m } // TODO: Add support for incrementally computing [[SetTransaction]]s even when // `minSetTransactionRetentionTimestamp` is set. // We don't incrementally compute [[SetTransaction]]s when user has configured // `minSetTransactionRetentionTimestamp` as it makes verification non-deterministic. // Check all places to figure out whether `minSetTransactionRetentionTimestamp` is set: // 1. oldSnapshot corresponding to `attemptVersion - 1` // 2. old VersionChecksum's MetaData (corresponding to `attemptVersion-1`) // 3. new VersionChecksum's MetaData (corresponding to `attemptVersion`) val setTransactionRetentionTimestampConfigured = (oldSnapshot.map(_.metadata) ++ Option(oldVersionChecksum.metadata) ++ newMetadataToCommit) .exists(DeltaLog.minSetTransactionRetentionInterval(_).nonEmpty) if (setTransactionRetentionTimestampConfigured) return None // Check-3: Check old setTransactions are available so that we can incrementally compute new. val oldSetTransactions = oldVersionChecksum.setTransactions .getOrElse { return None } // Check-4: old/new setTransactions are within the threshold. val setTransactionsToCommit = actionsToCommit.filter(_.isInstanceOf[SetTransaction]) val threshold = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_MAX_SET_TRANSACTIONS_IN_CRC) if (Math.max(setTransactionsToCommit.size, oldSetTransactions.size) > threshold) return None // We currently don't attempt incremental [[SetTransaction]] when // `minSetTransactionRetentionTimestamp` is set. So passing this as None here explicitly. // We can also ignore file retention because that only affects [[RemoveFile]] actions. val logReplay = new InMemoryLogReplay( minFileRetentionTimestamp = None, minSetTransactionRetentionTimestamp = None) logReplay.append(attemptVersion - 1, oldSetTransactions.toIterator) logReplay.append(attemptVersion, setTransactionsToCommit.toIterator) Some(logReplay.getTransactions.toSeq).filter(_.size <= threshold) } /** * Incrementally compute [[Snapshot.domainMetadata]] for the commit `attemptVersion`. * * @param oldVersionChecksum - [[VersionChecksum]] corresponding to `attemptVersion` - 1 * @param attemptVersion - version which we want to commit * @param actionsToCommit - actions for commit `attemptVersion` * @return Sequence of incrementally computed [[DomainMetadata]]s for commit * `attemptVersion`. */ private def incrementallyComputeDomainMetadatas( oldSnapshot: Option[Snapshot], oldVersionChecksum: VersionChecksum, attemptVersion: Long, actionsToCommit: Seq[Action]): Option[Seq[DomainMetadata]] = { // Check old DomainMetadatas are available so that we can incrementally compute new. val oldDomainMetadatas = oldVersionChecksum.domainMetadata .getOrElse { return None } val newDomainMetadatas = actionsToCommit.filter(_.isInstanceOf[DomainMetadata]) // We only work with DomainMetadata, so RemoveFile and SetTransaction retention don't matter. val logReplay = new InMemoryLogReplay( minFileRetentionTimestamp = None, minSetTransactionRetentionTimestamp = None) val threshold = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_MAX_DOMAIN_METADATAS_IN_CRC) logReplay.append(attemptVersion - 1, oldDomainMetadatas.iterator) logReplay.append(attemptVersion, newDomainMetadatas.iterator) // We don't truncate the set of DomainMetadata actions. Instead, we either store all of them or // none of them. The advantage of this is that you can then determine presence based on the // checksum, i.e. if the checksum contains domain metadatas but it doesn't contain the one you // are looking for, then it's not there. // // It's also worth noting that we can distinguish "no domain metadatas" versus // "domain metadatas not stored" as [[Some]] vs. [[None]]. Some(logReplay.getDomainMetadatas.toSeq).filter(_.size <= threshold) } /** * Incrementally compute [[Snapshot.allFiles]] for the commit `attemptVersion`. * * @param oldSnapshot - snapshot corresponding to `attemptVersion` - 1 * @param oldVersionChecksum - [[VersionChecksum]] corresponding to `attemptVersion` - 1 * @param attemptVersion - version which we want to commit * @param numFilesAfterCommit - number of files in the table after the attemptVersion commit. * @param actionsToCommit - actions for commit `attemptVersion` * @return Optional sequence of AddFiles which represents the incrementally computed state for * commit `attemptVersion` */ private def incrementallyComputeAddFiles( oldSnapshot: Option[Snapshot], oldVersionChecksum: VersionChecksum, attemptVersion: Long, numFilesAfterCommit: Long, actionsToCommit: Seq[Action]): Option[Seq[AddFile]] = { // We must enumerate both the pre- and post-commit file lists; give up if they are too big. val incrementalAllFilesThreshold = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_THRESHOLD_FILES) val numFilesBeforeCommit = oldVersionChecksum.numFiles if (Math.max(numFilesAfterCommit, numFilesBeforeCommit) > incrementalAllFilesThreshold) { return None } // We try to get files for `attemptVersion - 1` from the old CRC first. If the old CRC doesn't // have those files, then we will try to get that info from the oldSnapshot (corresponding to // attemptVersion - 1). Note that oldSnapshot might not be present if another concurrent commits // have happened in between. In this case we return and not store incrementally computed state // to crc. val oldAllFiles = oldVersionChecksum.allFiles .orElse { recordFrameProfile("Delta", "VersionChecksum.incrementallyComputeAddFiles") { oldSnapshot.map(_.allFiles.collect().toSeq) } } .getOrElse { return None } val canonicalPath = new DeltaLog.CanonicalPathFunction(() => deltaLog.newDeltaHadoopConf()) def normalizePath(action: Action): Action = action match { case af: AddFile => af.copy(path = canonicalPath(af.path)) case rf: RemoveFile => rf.copy(path = canonicalPath(rf.path)) case others => others } // We only work with AddFile, so RemoveFile and SetTransaction retention don't matter. val logReplay = new InMemoryLogReplay( minFileRetentionTimestamp = None, minSetTransactionRetentionTimestamp = None) logReplay.append(attemptVersion - 1, oldAllFiles.map(normalizePath).toIterator) logReplay.append(attemptVersion, actionsToCommit.map(normalizePath).toIterator) Some(logReplay.allFiles) } } object RecordChecksum { // Operations where we should ignore AddFiles in the incremental checksum computation. private[delta] val operationNamesWhereAddFilesIgnoredForIncrementalCrc = Set( // The transaction that computes stats is special -- it re-adds files that already exist, in // order to update their min/max stats. We should not count those against the totals. DeltaOperations.ComputeStats(Seq.empty).name, // Backfill/Tagging re-adds existing AddFiles without changing the underlying data files. // Incremental commits should ignore backfill commits. DeltaOperations.RowTrackingBackfill().name, // Same as Backfill. DeltaOperations.RowTrackingUnBackfill().name, // Dropping a feature may re-add existing AddFiles without changing the underlying data files. DeltaOperations.OP_DROP_FEATURE ) // Operations where we should ignore RemoveFiles in the incremental checksum computation. private[delta] val operationNamesWhereRemoveFilesIgnoredForIncrementalCrc = Set( // Deletion vector tombstones are only required to protect DVs from vacuum. They should be // ignored in checksum calculation. DeltaOperations.AddDeletionVectorsTombstones.name ) } /** * Read checksum files. */ trait ReadChecksum extends DeltaLogging { self: DeltaLog => val logPath: Path private[delta] def store: LogStore private[delta] def readChecksum( version: Long, checksumFileStatusHintOpt: Option[FileStatus] = None): Option[VersionChecksum] = { recordDeltaOperation(self, "delta.readChecksum") { val checksumFilePath = FileNames.checksumFile(logPath, version) val verifiedChecksumFileStatusOpt = checksumFileStatusHintOpt.filter(_.getPath == checksumFilePath) var exception: Option[String] = None val content = try Some( verifiedChecksumFileStatusOpt .map(store.read(_, newDeltaHadoopConf())) .getOrElse(store.read(checksumFilePath, newDeltaHadoopConf())) ) catch { case NonFatal(e) => // We expect FileNotFoundException; if it's another kind of exception, we still catch them // here but we log them in the checksum error event below. if (!e.isInstanceOf[FileNotFoundException]) { exception = Some(Utils.exceptionString(e)) } None } if (content.isEmpty) { // We may not find the checksum file in two cases: // - We just upgraded our Spark version from an old one // - Race conditions where we commit a transaction, and before we can write the checksum // this reader lists the new version, and uses it to create the snapshot. recordDeltaEvent( this, "delta.checksum.error.missing", data = Map("version" -> version) ++ exception.map("exception" -> _)) return None } val checksumData = content.get if (checksumData.isEmpty) { recordDeltaEvent( this, "delta.checksum.error.empty", data = Map("version" -> version)) return None } try { Option(JsonUtils.mapper.readValue[VersionChecksum](checksumData.head)) } catch { case NonFatal(e) => recordDeltaEvent( this, "delta.checksum.error.parsing", data = Map("exception" -> Utils.exceptionString(e))) None } } } } /** * Verify the state of the table using the checksum information. */ trait ValidateChecksum extends DeltaLogging { self: Snapshot => /** * Validate checksum (if any) by comparing it against the snapshot's state reconstruction. * @param contextInfo caller context that will be added to the logging if validation fails * @return True iff validation succeeded. * @throws IllegalStateException if validation failed and corruption is configured as fatal. */ def validateChecksum(contextInfo: Map[String, String] = Map.empty): Boolean = { val contextSuffix = contextInfo.get("context").map(c => s".context-$c").getOrElse("") val computedStateAccessor = s"ValidateChecksum.checkMismatch$contextSuffix" val computedStateToCompareAgainst = computedState val (mismatchErrorMap, detailedErrorMapForUsageLogs) = checksumOpt .map(checkMismatch(_, computedStateToCompareAgainst)) .getOrElse((Map.empty[String, String], Map.empty[String, String])) logAndThrowValidationFailure(mismatchErrorMap, detailedErrorMapForUsageLogs, contextInfo) } private def logAndThrowValidationFailure( mismatchErrorMap: Map[String, String], detailedErrorMapForUsageLogs: Map[String, String], contextInfo: Map[String, String]): Boolean = { if (mismatchErrorMap.isEmpty) return true val mismatchString = mismatchErrorMap.values.mkString("\n") // We get the active SparkSession, which may be different than the SparkSession of the // Snapshot that was created, since we cache `DeltaLog`s. val sparkOpt = SparkSession.getActiveSession // Report the failure to usage logs. recordDeltaEvent( this.deltaLog, "delta.checksum.invalid", data = Map( "error" -> mismatchString, "mismatchingFields" -> mismatchErrorMap.keys.toSeq, "detailedErrorMap" -> detailedErrorMapForUsageLogs, "v2CheckpointEnabled" -> CheckpointProvider.isV2CheckpointEnabled(this), "checkpointProviderCheckpointPolicy" -> checkpointProvider.checkpointPolicy.map(_.name).getOrElse("") ) ++ contextInfo) val spark = sparkOpt.getOrElse { throw DeltaErrors.sparkSessionNotSetException() } if (spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_CHECKSUM_MISMATCH_IS_FATAL)) { throw DeltaErrors.logFailedIntegrityCheck(version, mismatchString) } false } /** * Validate [[Snapshot.allFiles]] against given checksum.allFiles. * Returns true if validation succeeds, else return false. * In Unit Tests, this method throws [[IllegalStateException]] so that issues can be caught during * development. */ def validateFileListAgainstCRC(checksum: VersionChecksum, contextOpt: Option[String]): Boolean = { val fileSortKey = (f: AddFile) => (f.path, f.modificationTime, f.size) val filesFromCrc = checksum.allFiles.map(_.sortBy(fileSortKey)).getOrElse { return true } val filesFromStateReconstruction = recordFrameProfile("Delta", "snapshot.allFiles") { allFilesViaStateReconstruction.collect().toSeq.sortBy(fileSortKey) } if (filesFromCrc == filesFromStateReconstruction) return true val filesFromCrcWithoutStats = filesFromCrc.map(_.copy(stats = "")) val filesFromStateReconstructionWithoutStats = filesFromStateReconstruction.map(_.copy(stats = "")) val mismatchWithStatsOnly = filesFromCrcWithoutStats == filesFromStateReconstructionWithoutStats if (mismatchWithStatsOnly) { // Normalize stats in CRC as per the table schema val filesFromStateReconstructionMap = filesFromStateReconstruction.map(af => (af.path, af)).toMap val parser = DeltaFileProviderUtils.createJsonStatsParser(statsSchema) var normalizedStatsDiffer = false filesFromCrc.foreach { addFile => val statsFromSR = filesFromStateReconstructionMap(addFile.path).stats val statsFromSRParsed = parser(statsFromSR) val statsFromCrcParsed = parser(addFile.stats) if (statsFromSRParsed != statsFromCrcParsed) { normalizedStatsDiffer = true } } if (!normalizedStatsDiffer) return true } // If incremental all-files-in-crc validation fails, then there is a possibility that the // issue is not just with incremental all-files-in-crc computation but with overall incremental // commits. So run the incremental commit crc validation and find out whether that is also // failing. val contextForIncrementalCommitCheck = contextOpt.map(c => s"$c.").getOrElse("") + "delta.allFilesInCrc.checksumMismatch.validateFileListAgainstCRC" var errorForIncrementalCommitCrcValidation = "" val incrementalCommitCrcValidationPassed = try { validateChecksum(Map("context" -> contextForIncrementalCommitCheck)) } catch { case NonFatal(e) => errorForIncrementalCommitCrcValidation += e.getMessage false } val eventData = Map( "version" -> version, "mismatchWithStatsOnly" -> mismatchWithStatsOnly, "filesCountFromCrc" -> filesFromCrc.size, "filesCountFromStateReconstruction" -> filesFromStateReconstruction.size, "filesFromCrc" -> JsonUtils.toJson(filesFromCrc), "incrementalCommitCrcValidationPassed" -> incrementalCommitCrcValidationPassed, "errorForIncrementalCommitCrcValidation" -> errorForIncrementalCommitCrcValidation, "context" -> contextOpt.getOrElse("") ) val message = s"Incremental state reconstruction validation failed for version " + s"$version [${eventData.mkString(",")}]" logInfo(message) recordDeltaEvent( this.deltaLog, opType = "delta.allFilesInCrc.checksumMismatch.differentAllFiles", data = eventData) if (DeltaUtils.isTesting) throw new IllegalStateException(message) false } /** * Validates the given `checksum` against [[Snapshot.computedState]]. * Returns an tuple of Maps: * - first Map contains fieldName to user facing errorMessage mapping. * - second Map is just for usage logs purpose and contains more details for different fields. * Adding info to this map is optional. */ private def checkMismatch( checksum: VersionChecksum, computedStateToCheckAgainst: SnapshotState ): (Map[String, String], Map[String, String]) = { var errorMap = ListMap[String, String]() var detailedErrorMapForUsageLogs = ListMap[String, String]() def compare(expected: Long, found: Long, title: String, field: String): Unit = { if (expected != found) { errorMap += (field -> s"$title - Expected: $expected Computed: $found") } } def compareAction(expected: Action, found: Action, title: String, field: String): Unit = { // only compare when expected is not null for being backward compatible to the checksum // without protocol and metadata Option(expected).filterNot(_.equals(found)).foreach { expected => errorMap += (field -> s"$title - Expected: $expected Computed: $found") } } def compareSetTransactions( setTransactionsInCRC: Seq[SetTransaction], setTransactionsComputed: Seq[SetTransaction]): Unit = { val appIdsFromCrc = setTransactionsInCRC.map(_.appId) val repeatedEntriesForSameAppId = appIdsFromCrc.size != appIdsFromCrc.toSet.size val setTransactionsInCRCSet = setTransactionsInCRC.toSet val setTransactionsFromComputeStateSet = setTransactionsComputed.toSet val exactMatchFailed = setTransactionsInCRCSet != setTransactionsFromComputeStateSet if (repeatedEntriesForSameAppId || exactMatchFailed) { val repeatedAppIds = appIdsFromCrc.groupBy(identity).filter(_._2.size > 1).keySet.toSeq val matchedActions = setTransactionsInCRCSet.intersect(setTransactionsFromComputeStateSet) val unmatchedActionsInCrc = setTransactionsInCRCSet -- matchedActions val unmatchedActionsInComputed = setTransactionsFromComputeStateSet -- matchedActions val eventData = Map( "unmatchedSetTransactionsCRC" -> unmatchedActionsInCrc, "unmatchedSetTransactionsComputedState" -> unmatchedActionsInComputed, "version" -> version, "minSetTransactionRetentionTimestamp" -> minSetTransactionRetentionTimestamp, "repeatedEntriesForSameAppId" -> repeatedAppIds, "exactMatchFailed" -> exactMatchFailed) errorMap += ("setTransactions" -> s"SetTransaction mismatch") detailedErrorMapForUsageLogs += ("setTransactions" -> JsonUtils.toJson(eventData)) } } def compareDomainMetadata( domainMetadataInCRC: Seq[DomainMetadata], domainMetadataComputed: Seq[DomainMetadata]): Unit = { val domainMetadataInCRCSet = domainMetadataInCRC.toSet // Remove any tombstones from the reconstructed set before comparison. val domainMetadataInComputeStateSet = domainMetadataComputed.filterNot(_.removed).toSet val exactMatchFailed = domainMetadataInCRCSet != domainMetadataInComputeStateSet if (exactMatchFailed) { val matchedActions = domainMetadataInCRCSet.intersect(domainMetadataInComputeStateSet) val unmatchedActionsInCRC = domainMetadataInCRCSet -- matchedActions val unmatchedActionsInComputed = domainMetadataInComputeStateSet -- matchedActions val eventData = Map( "unmatchedDomainMetadataInCRC" -> unmatchedActionsInCRC, "unmatchedDomainMetadataInComputedState" -> unmatchedActionsInComputed, "version" -> version) errorMap += ("domainMetadata" -> "domainMetadata mismatch") detailedErrorMapForUsageLogs += ("domainMetadata" -> JsonUtils.toJson(eventData)) } } // Deletion vectors metrics. if (DeletionVectorUtils.deletionVectorsReadable(self)) { (checksum.numDeletedRecordsOpt zip computedState.numDeletedRecordsOpt).foreach { case (a, b) => compare(a, b, "Number of deleted records", "numDeletedRecordsOpt") } (checksum.numDeletionVectorsOpt zip computedState.numDeletionVectorsOpt).foreach { case (a, b) => compare(a, b, "Number of deleted vectors", "numDeletionVectorsOpt") } } compareAction(checksum.metadata, computedStateToCheckAgainst.metadata, "Metadata", "metadata") compareAction(checksum.protocol, computedStateToCheckAgainst.protocol, "Protocol", "protocol") compare( checksum.tableSizeBytes, computedStateToCheckAgainst.sizeInBytes, title = "Table size (bytes)", field = "tableSizeBytes") compare( checksum.numFiles, computedStateToCheckAgainst.numOfFiles, title = "Number of files", field = "numFiles") compare( checksum.numMetadata, computedStateToCheckAgainst.numOfMetadata, title = "Metadata updates", field = "numOfMetadata") compare( checksum.numProtocol, computedStateToCheckAgainst.numOfProtocol, title = "Protocol updates", field = "numOfProtocol") checksum.setTransactions.foreach { setTransactionsInCRC => compareSetTransactions(setTransactionsInCRC, computedStateToCheckAgainst.setTransactions) } checksum.domainMetadata.foreach( compareDomainMetadata(_, computedStateToCheckAgainst.domainMetadata)) (errorMap, detailedErrorMapForUsageLogs) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/ClassicColumnConversions.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.classic.ClassicConversions import org.apache.spark.sql.classic.ColumnConversions import org.apache.spark.sql.classic.ColumnNodeToExpressionConverter import org.apache.spark.sql.classic.{SparkSession => SparkSessionImpl} /** * Conversions from a [[org.apache.spark.sql.Column]] to an * [[org.apache.spark.sql.catalyst.expressions.Expression]], and vice versa. * * @note [[org.apache.spark.sql.internal.ExpressionUtils#expression]] is a cheap alternative for * [[org.apache.spark.sql.Column]] to [[org.apache.spark.sql.catalyst.expressions.Expression]] * conversions. However this can only be used when the produced expression is used in a Column * later on. */ object ClassicColumnConversions extends ClassicConversions with ColumnConversions { override def converter: ColumnNodeToExpressionConverter = ColumnNodeToExpressionConverter } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/ColumnWithDefaultExprUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import scala.collection.mutable import scala.concurrent.duration import scala.util.control.NonFatal import org.apache.spark.sql.delta.Relocated._ import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions.{Metadata, Protocol} import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.constraints.{Constraint, Constraints} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf, DeltaStreamUtils} import org.apache.spark.sql.delta.sources.DeltaSQLConf.GeneratedColumnValidateOnWriteMode import org.apache.spark.internal.MDC import org.apache.spark.sql.{Column, DataFrame} import org.apache.spark.sql.catalyst.expressions.EqualNullSafe import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns._ import org.apache.spark.sql.execution.QueryExecution import org.apache.spark.sql.types.{MetadataBuilder, StructField, StructType} /** * Provide utilities to handle columns with default expressions. * Currently we support three types of such columns: * (1) GENERATED columns. * (2) IDENTITY columns. * (3) Columns with user-specified default value expression. */ object ColumnWithDefaultExprUtils extends DeltaLogging { val USE_NULL_AS_DEFAULT_DELTA_OPTION = "__use_null_as_default" // Returns true if column `field` is defined as an IDENTITY column. def isIdentityColumn(field: StructField): Boolean = { val md = field.metadata val hasStart = md.contains(DeltaSourceUtils.IDENTITY_INFO_START) val hasStep = md.contains(DeltaSourceUtils.IDENTITY_INFO_STEP) val hasInsert = md.contains(DeltaSourceUtils.IDENTITY_INFO_ALLOW_EXPLICIT_INSERT) // Verify that we have all or none of the three fields. if (!((hasStart == hasStep) && (hasStart == hasInsert))) { throw DeltaErrors.identityColumnInconsistentMetadata(field.name, hasStart, hasStep, hasInsert) } hasStart && hasStep && hasInsert } // Return true if `schema` contains any number of IDENTITY column. def hasIdentityColumn(schema: StructType): Boolean = schema.exists(isIdentityColumn) // Return if `protocol` satisfies the requirement for IDENTITY columns. def satisfiesIdentityColumnProtocol(protocol: Protocol): Boolean = protocol.isFeatureSupported(IdentityColumnsTableFeature) || protocol.minWriterVersion == 6 || protocol.writerFeatureNames.contains("identityColumns") // Return true if the column `col` has default expressions (and can thus be omitted from the // insertion list). def columnHasDefaultExpr( protocol: Protocol, col: StructField, nullAsDefault: Boolean): Boolean = { isIdentityColumn(col) || col.metadata.contains(CURRENT_DEFAULT_COLUMN_METADATA_KEY) || (col.nullable && nullAsDefault) || GeneratedColumn.isGeneratedColumn(protocol, col) } // Return true if the table with `metadata` has default expressions. def tableHasDefaultExpr( protocol: Protocol, metadata: Metadata, nullAsDefault: Boolean): Boolean = { hasIdentityColumn(metadata.schema) || metadata.schema.exists { f => f.metadata.contains(CURRENT_DEFAULT_COLUMN_METADATA_KEY) || (f.nullable && nullAsDefault) } || GeneratedColumn.enforcesGeneratedColumns(protocol, metadata) } /** * If there are columns with default expressions in `schema`, add a new project to generate * those columns missing in the schema, and return constraints for generated columns existing in * the schema. * * @param deltaLog The table's [[DeltaLog]] used for logging. * @param queryExecution Used to check whether the original query is a streaming query or not. * @param schema Table schema. * @param data The data to be written into the table. * @param nullAsDefault If true, use null literal as the default value for missing columns. * @return The data with potentially additional default expressions projected and constraints * from generated columns if any. This includes IDENTITY column names for which we * should track the high water marks. */ def addDefaultExprsOrReturnConstraints( deltaLog: DeltaLog, protocol: Protocol, queryExecution: QueryExecution, schema: StructType, data: DataFrame, nullAsDefault: Boolean): (DataFrame, Seq[Constraint], Set[String]) = { val topLevelOutputNames = CaseInsensitiveMap(data.schema.map(f => f.name -> f).toMap) lazy val metadataOutputNames = CaseInsensitiveMap(schema.map(f => f.name -> f).toMap) val constraints = mutable.ArrayBuffer[Constraint]() // Column names for which we will track high water marks. val track = mutable.Set[String]() val generatedColumnsValidateMode = GeneratedColumnValidateOnWriteMode.fromConf(data.sparkSession.sessionState.conf) generatedColumnsValidateMode match { case GeneratedColumnValidateOnWriteMode.LOG_ONLY | GeneratedColumnValidateOnWriteMode.ASSERT => try { val startTime = System.nanoTime() GeneratedColumn.validateGeneratedColumns(data.sparkSession, schema) val durationMs = duration.NANOSECONDS.toMillis(System.nanoTime() - startTime) logInfo( log"Validated Generated Column expressions on table " + log"${MDC(DeltaLogKeys.TABLE_ID, deltaLog.unsafeVolatileTableId)} " + log"in ${MDC(DeltaLogKeys.TIME_MS, durationMs)} ms" ) } catch { case NonFatal(e) => val errorClassName = e match { case deltaException: DeltaAnalysisException => deltaException.getErrorClass case _ => e.getClass } recordDeltaEvent( deltaLog, "delta.generatedColumns.writeValidationFailure", data = Map( "errorClassName" -> errorClassName, "errorMessage" -> e.getMessage ) ) if (generatedColumnsValidateMode == GeneratedColumnValidateOnWriteMode.ASSERT) { throw e } } case GeneratedColumnValidateOnWriteMode.OFF => } var selectExprs = schema.flatMap { f => GeneratedColumn.getGenerationExpression(f) match { case Some(expr) if GeneratedColumn.satisfyGeneratedColumnProtocol(protocol) => if (topLevelOutputNames.contains(f.name)) { val column = SchemaUtils.fieldToColumn(f) // Add a constraint to make sure the value provided by the user is the same as the value // calculated by the generation expression. constraints += Constraints.Check(s"Generated Column", EqualNullSafe(column.expr, expr)) Some(column) } else { Some(Column(expr).alias(f.name)) } case _ => if (isIdentityColumn(f)) { if (topLevelOutputNames.contains(f.name)) { Some(SchemaUtils.fieldToColumn(f)) } else { // Track high water marks for generated IDENTITY values. track += f.name Some(IdentityColumn.createIdentityColumnGenerationExprAsColumn(f)) } } else { if (topLevelOutputNames.contains(f.name) || !data.sparkSession.conf.get(DeltaSQLConf.GENERATED_COLUMN_ALLOW_NULLABLE)) { Some(SchemaUtils.fieldToColumn(f)) } else { // we only want to consider columns that are in the data's schema or are generated // to allow DataFrame with null columns to be written. // The actual check for nullability on data is done in the DeltaInvariantCheckerExec getDefaultValueExprOrNullLit(f, nullAsDefault).map(Column(_)) } } } } val cdcSelectExprs = CDCReader.CDC_COLUMNS_IN_DATA.flatMap { cdcColumnName => topLevelOutputNames.get(cdcColumnName).flatMap { cdcField => if (metadataOutputNames.contains(cdcColumnName)) { // The column is in the table schema. It's not a CDC auto generated column. Skip it since // it's already in `selectExprs`. None } else { // The column is not in the table schema, // so it must be a column generated by CDC. Adding it back as it's not in `selectExprs`. Some(SchemaUtils.fieldToColumn(cdcField).alias(cdcField.name)) } } } selectExprs = selectExprs ++ cdcSelectExprs val rowIdExprs = data.queryExecution.analyzed.output .filter(RowId.RowIdMetadataAttribute.isRowIdColumn) .map(Column(_)) selectExprs = selectExprs ++ rowIdExprs val rowCommitVersionExprs = data.queryExecution.analyzed.output .filter(RowCommitVersion.MetadataAttribute.isRowCommitVersionColumn) .map(Column(_)) selectExprs = selectExprs ++ rowCommitVersionExprs val newData = queryExecution match { case incrementalExecution: IncrementalExecution => DeltaStreamUtils.selectFromStreamingDataFrame(incrementalExecution, data, selectExprs: _*) case _ => data.select(selectExprs: _*) } recordDeltaEvent(deltaLog, "delta.generatedColumns.write") (newData, constraints.toSeq, track.toSet) } // Removes the default expressions properties from the schema. If `keepGeneratedColumns` is // true, generated column expressions are kept. If `keepIdentityColumns` is true, IDENTITY column // properties are kept. def removeDefaultExpressions( schema: StructType, keepGeneratedColumns: Boolean = false, keepIdentityColumns: Boolean = false): StructType = { var updated = false val updatedSchema = schema.map { field => if (!keepGeneratedColumns && GeneratedColumn.isGeneratedColumn(field)) { updated = true val newMetadata = new MetadataBuilder() .withMetadata(field.metadata) .remove(DeltaSourceUtils.GENERATION_EXPRESSION_METADATA_KEY) .build() field.copy(metadata = newMetadata) } else if (!keepIdentityColumns && isIdentityColumn(field)) { updated = true val newMetadata = new MetadataBuilder() .withMetadata(field.metadata) .remove(DeltaSourceUtils.IDENTITY_INFO_ALLOW_EXPLICIT_INSERT) .remove(DeltaSourceUtils.IDENTITY_INFO_HIGHWATERMARK) .remove(DeltaSourceUtils.IDENTITY_INFO_START) .remove(DeltaSourceUtils.IDENTITY_INFO_STEP) .build() field.copy(metadata = newMetadata) } else { field } } if (updated) { StructType(updatedSchema) } else { schema } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/CommittedTransaction.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.collection.mutable import org.apache.spark.sql.delta.actions.{Action, AddFile, CommitInfo} import org.apache.spark.sql.delta.hooks.PostCommitHook import org.apache.spark.sql.catalyst.catalog.CatalogTable /** * Represents a successfully committed transaction. * * This class encapsulates all relevant information about a transaction that has been successfully * committed. The main usage of this class is in running the post-commit hooks. * * @param txnId the unique identifier of the committed transaction. * @param deltaLog the [[DeltaLog]] instance for the table the transaction * committed on. * @param catalogTable the catalog table at the start of the transaction for the * committed table. * @param readSnapshot the snapshot of the table at the time of the transaction's read. * @param committedVersion the version of the table after the txn committed. * @param committedActions the actions that were committed in this transaction. * @param postCommitSnapshot the snapshot of the table after the txn successfully committed. * NOTE: This may not match the committedVersion, if racing * commits were written while the snapshot was computed. * @param postCommitHooks the list of post-commit hooks to run after the commit. * @param txnExecutionTimeMs the time taken to execute the transaction. * @param needsCheckpoint whether a checkpoint is needed after the commit. * @param partitionsAddedToOpt the partitions that this txn added new files to. * @param isBlindAppend whether this transaction was a blind append. */ case class CommittedTransaction( txnId: String, deltaLog: DeltaLog, catalogTable: Option[CatalogTable], readSnapshot: Snapshot, committedVersion: Long, committedActions: Seq[Action], postCommitSnapshot: Snapshot, postCommitHooks: Seq[PostCommitHook], txnExecutionTimeMs: Long, needsCheckpoint: Boolean, partitionsAddedToOpt: Option[mutable.HashSet[Map[String, String]]], isBlindAppend: Boolean ) ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/ConcurrencyHelpers.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.concurrent.duration._ object ConcurrencyHelpers { /** * Keep checking if `check` returns `true` until it's the case or `waitTime` expires. * * Return `true` when the `check` returned `true`, and `false` if `waitTime` expired. * * Note: This function is used as a helper function for the Concurrency Testing framework, * and should not be used in production code. Production code should not use polling * and should instead use signalling to coordinate. */ def busyWaitFor( check: => Boolean, waitTime: FiniteDuration): Boolean = { val DEFAULT_SLEEP_TIME: Duration = 10.millis val deadline = waitTime.fromNow do { if (check) { return true } val sleepTimeMs = DEFAULT_SLEEP_TIME.min(deadline.timeLeft).toMillis Thread.sleep(sleepTimeMs) } while (deadline.hasTimeLeft()) false } def withOptimisticTransaction[T]( activeTransaction: Option[OptimisticTransaction])(block: => T): T = { if (activeTransaction.isDefined) { OptimisticTransaction.withActive(activeTransaction.get) { block } } else { block } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/ConflictChecker.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.util.concurrent.TimeUnit import scala.collection.mutable import org.apache.spark.sql.delta.DeltaOperations.{OP_SET_TBLPROPERTIES, ROW_TRACKING_BACKFILL_OPERATION_NAME, ROW_TRACKING_UNBACKFILL_OPERATION_NAME} import org.apache.spark.sql.delta.RowId.RowTrackingMetadataDomain import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSourceUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.DeltaSparkPlanUtils.CheckDeterministicOptions import org.apache.spark.sql.delta.util.FileNames import io.delta.storage.commit.UpdatedActions import org.apache.hadoop.fs.FileStatus import org.apache.spark.internal.{MDC, MessageWithContext} import org.apache.spark.sql.{DataFrame, SparkSession} import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionSet, Or} import org.apache.spark.sql.types.{Metadata => FieldMetadata, MetadataBuilder, StructType} /** * A class representing different attributes of current transaction needed for conflict detection. * * @param readPredicates predicates by which files have been queried by the transaction * @param readFiles files that have been seen by the transaction * @param readWholeTable whether the whole table was read during the transaction * @param readAppIds appIds that have been seen by the transaction * @param metadata table metadata for the transaction * @param actions delta log actions that the transaction wants to commit * @param readSnapshot read [[Snapshot]] used for the transaction * @param commitInfo [[CommitInfo]] for the commit */ private[delta] case class CurrentTransactionInfo( val txnId: String, val readPredicates: Vector[DeltaTableReadPredicate], val readFiles: Set[AddFile], val readWholeTable: Boolean, val readAppIds: Set[String], val metadata: Metadata, val protocol: Protocol, val actions: Seq[Action], val readSnapshot: Snapshot, val commitInfo: Option[CommitInfo], val readRowIdHighWatermark: Long, val catalogTable: Option[CatalogTable], val domainMetadata: Seq[DomainMetadata], val op: DeltaOperations.Operation) { /** * Final actions to commit - including the [[CommitInfo]] which should always come first so we can * extract it easily from a commit without having to parse an arbitrarily large file. * * TODO: We might want to cluster all non-file actions at the front, for similar reasons. */ lazy val finalActionsToCommit: Seq[Action] = commitInfo ++: actions private var newMetadata: Option[Metadata] = None actions.foreach { case m: Metadata => newMetadata = Some(m) case _ => // do nothing } def getUpdatedActions( oldMetadata: Metadata, oldProtocol: Protocol): UpdatedActions = { new UpdatedActions(commitInfo.get, metadata, protocol, oldMetadata, oldProtocol) } /** Whether this transaction wants to make any [[Metadata]] update */ lazy val metadataChanged: Boolean = newMetadata.nonEmpty /** * Partition schema corresponding to the read snapshot for this transaction. * NOTE: In conflict detection, we should be careful around whether we want to use the new schema * which this txn wants to update OR the old schema from the read snapshot. * e.g. the ConcurrentAppend check makes sure that no new files have been added concurrently * that this transaction should have read. So this should use the read snapshot partition schema * and not the new partition schema which this txn is introducing. Using the new schema can cause * issues. */ val partitionSchemaAtReadTime: StructType = readSnapshot.metadata.partitionSchema // Whether this is a row tracking backfill transaction or not. val isRowTrackingBackfillTxn = op.name == ROW_TRACKING_BACKFILL_OPERATION_NAME val isRowTrackingUnBackfillTxn = op.name == ROW_TRACKING_UNBACKFILL_OPERATION_NAME def isConflict(winningTxn: SetTransaction): Boolean = readAppIds.contains(winningTxn.appId) } /** * Summary of the Winning commit against which we want to check the conflict * @param actions - delta log actions committed by the winning commit * @param fileStatus - descriptor for the commit file * @param readTimeMs - time taken to read the commit file */ private[delta] class WinningCommitSummary( val actions: Seq[Action], val fileStatus: FileStatus, val readTimeMs: Long) { val commitVersion: Long = FileNames.deltaVersion(fileStatus) val commitFileTimestamp: Long = fileStatus.getModificationTime val metadataUpdates: Seq[Metadata] = actions.collect { case a: Metadata => a } val appLevelTransactions: Seq[SetTransaction] = actions.collect { case a: SetTransaction => a } val protocol: Option[Protocol] = actions.collectFirst { case a: Protocol => a } val commitInfo: Option[CommitInfo] = actions.collectFirst { case a: CommitInfo => a }.map( ci => ci.copy(version = Some(commitVersion))) // Whether this is a row tracking backfill transaction or not. val isRowTrackingBackfillTxn = commitInfo.exists(_.operation == ROW_TRACKING_BACKFILL_OPERATION_NAME) val isRowTrackingUnBackfillTxn = commitInfo.exists(_.operation == ROW_TRACKING_UNBACKFILL_OPERATION_NAME) val removedFiles: Seq[RemoveFile] = actions.collect { case a: RemoveFile => a } val addedFiles: Seq[AddFile] = actions.collect { case a: AddFile => a } // This is used in resolveRowTrackingBackfillConflicts. lazy val addedFilePathToActionMap: Map[String, AddFile] = addedFiles.map(af => (af.path, af)).toMap val isBlindAppendOption: Option[Boolean] = commitInfo.flatMap(_.isBlindAppend) val blindAppendAddedFiles: Seq[AddFile] = if (isBlindAppendOption.getOrElse(false)) { addedFiles } else { Seq() } val changedDataAddedFiles: Seq[AddFile] = if (isBlindAppendOption.getOrElse(false)) { Seq() } else { addedFiles } val onlyAddFiles: Boolean = actions.collect { case f: FileAction => f } .forall(_.isInstanceOf[AddFile]) // This indicates this commit contains metadata action that is solely for the purpose for // updating IDENTITY high water marks. This is used by [[ConflictChecker]] to avoid certain // conflict in [[checkNoMetadataUpdates]]. val identityOnlyMetadataUpdate = DeltaCommitTag .getTagValueFromCommitInfo(commitInfo, DeltaSourceUtils.IDENTITY_COMMITINFO_TAG) .exists(_.toBoolean) } object WinningCommitSummary { /** * Read a commit file and create the [[WinningCommitSummary]]. */ def createFromFileStatus( deltaLog: DeltaLog, fileStatus: FileStatus): WinningCommitSummary = { val startTimeNs = System.nanoTime() val actions = deltaLog.store.read( fileStatus, deltaLog.newDeltaHadoopConf() ).map(Action.fromJson) val readTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs) new WinningCommitSummary( actions = actions, fileStatus = fileStatus, readTimeMs = readTimeMs ) } } private[delta] class ConflictChecker( spark: SparkSession, initialCurrentTransactionInfo: CurrentTransactionInfo, winningCommitSummary: WinningCommitSummary, isolationLevel: IsolationLevel) extends DeltaLogging with ConflictCheckerPredicateElimination { protected val winningCommitVersion = winningCommitSummary.commitVersion protected val startTimeMs = System.currentTimeMillis() protected val timingStats = mutable.HashMap[String, Long]() protected val deltaLog = initialCurrentTransactionInfo.readSnapshot.deltaLog protected var currentTransactionInfo: CurrentTransactionInfo = initialCurrentTransactionInfo protected def recordSkippedPhase(phase: String): Unit = timingStats += phase -> 0 /** * This function checks conflict of the `initialCurrentTransactionInfo` against the * `winningCommitVersion` and returns an updated [[CurrentTransactionInfo]] that represents * the transaction as if it had started while reading the `winningCommitVersion`. */ def checkConflicts(): CurrentTransactionInfo = { // Add time to read commit in the metrics. recordTime("initialize-old-commit", winningCommitSummary.readTimeMs) // Check early the protocol and metadata compatibility that is required for subsequent // file-level checks. checkProtocolCompatibility() if (spark.conf.get(DeltaSQLConf.FEATURE_ENABLEMENT_CONFLICT_RESOLUTION_ENABLED)) { attemptToResolveMetadataConflicts() } else { checkNoMetadataUpdates() } checkIfDomainMetadataConflict() // Perform cheap check for transaction dependencies before we start checks files. checkForUpdatedApplicationTransactionIdsThatCurrentTxnDependsOn() resolveRowTrackingBackfillConflicts() resolveRowTrackingUnBackfillConflicts() // Row Tracking reconciliation. We perform this before the file checks to ensure that // no files have duplicate row IDs and avoid interacting with files that don't comply with // the protocol. reassignOverlappingRowIds() reassignRowCommitVersions() // Update the table version in newly added type widening metadata. updateTypeWideningMetadata() // Data file checks. checkForAddedFilesThatShouldHaveBeenReadByCurrentTxn() checkForDeletedFilesAgainstCurrentTxnReadFiles() checkForDeletedFilesAgainstCurrentTxnDeletedFiles() resolveTimestampOrderingConflicts() logMetrics() currentTransactionInfo } /** * Asserts that the client is up to date with the protocol and is allowed to read and write * against the protocol set by the committed transaction. */ protected def checkProtocolCompatibility(): Unit = { if (winningCommitSummary.protocol.nonEmpty) { winningCommitSummary.protocol.foreach { p => deltaLog.protocolRead(p) deltaLog.protocolWrite(p) currentTransactionInfo = currentTransactionInfo.copy(protocol = p) } if (currentTransactionInfo.actions.exists(_.isInstanceOf[Protocol])) { throw DeltaErrors.protocolChangedException(winningCommitSummary.commitInfo) } // When a protocol downgrade occurs all other interleaved txns abort. Note, that in the // opposite scenario, when the current transaction is the protocol downgrade, we resolve // the conflict and proceed with the downgrade. This is because a protocol downgrade would // be hard to succeed in concurrent workloads. On the other hand, a protocol downgrade is // a rare event and thus not that disruptive if other concurrent transactions fail. val winningProtocol = winningCommitSummary.protocol.get val readProtocol = currentTransactionInfo.readSnapshot.protocol val isWinnerDroppingFeatures = TableFeature.isProtocolRemovingFeatures( newProtocol = winningProtocol, oldProtocol = readProtocol) if (isWinnerDroppingFeatures) { throw DeltaErrors.protocolChangedException(winningCommitSummary.commitInfo) } if (spark.conf.get( DeltaSQLConf.DELTA_CONFLICT_CHECKER_ENFORCE_FEATURE_ENABLEMENT_VALIDATION)) { // Check if the winning protocol adds features that should fail concurrent transactions at // upgrade. These features are identified by the `failConcurrentTransactionsAtUpgrade` // method returning true. These features impose write-time requirements that need to be // respected by all writers beyond the protocol upgrade, and there's no custom feature // specific conflict resolution logic below to be able to have the current transaction meet // these requirements on-the-fly. val winningTxnAddedFeatures = TableFeature.getAddedFeatures(winningProtocol, readProtocol) val winningTxnUnsafeAddedFeatures = winningTxnAddedFeatures .filter(_.failConcurrentTransactionsAtUpgrade) if (winningTxnUnsafeAddedFeatures.nonEmpty) { throw DeltaErrors.protocolChangedException(winningCommitSummary.commitInfo) } } } // When the winning transaction does not change the protocol but the losing txn is // a protocol downgrade, we re-validate the invariants of the removed feature. // Furthermore, when dropping with the fast drop feature we need to adjust // requireCheckpointProtectionBeforeVersion. // TODO: only revalidate against the snapshot of the last interleaved txn. val newProtocol = currentTransactionInfo.protocol val readProtocol = currentTransactionInfo.readSnapshot.protocol if (TableFeature.isProtocolRemovingFeatures(newProtocol, readProtocol)) { // Feature specific conflict resolution logic. if (TableFeature.isFeatureDropped(newProtocol, readProtocol, RowTrackingFeature)) { currentTransactionInfo = resolveRowTrackingUnBackfillConflicts( currentTransactionInfo, winningCommitSummary) } else { val winningSnapshot = deltaLog.getSnapshotAt( winningCommitSummary.commitVersion, catalogTableOpt = currentTransactionInfo.catalogTable) val isDowngradeCommitValid = TableFeature.validateFeatureRemovalAtSnapshot( newProtocol = newProtocol, oldProtocol = readProtocol, table = DeltaTableV2( spark = spark, path = deltaLog.dataPath, catalogTable = currentTransactionInfo.catalogTable), snapshot = winningSnapshot) if (!isDowngradeCommitValid) { throw DeltaErrors.dropTableFeatureConflictRevalidationFailed( winningCommitSummary.commitInfo) } } // When the current transaction is removing a feature and CheckpointProtectionTableFeature // is enabled, the current transaction will set the requireCheckpointProtectionBeforeVersion // table property to the version of the current transaction. // So we need to update it after resolving conflicts with winning transactions. if (newProtocol.isFeatureSupported(CheckpointProtectionTableFeature) && TableFeature.isProtocolRemovingFeatureWithHistoryProtection(newProtocol, readProtocol)) { val newVersion = winningCommitVersion + 1L val newMetadata = CheckpointProtectionTableFeature.metadataWithCheckpointProtection( currentTransactionInfo.metadata, newVersion) val newActions = currentTransactionInfo.actions.collect { // Sanity check. case m: Metadata if m != currentTransactionInfo.metadata => recordDeltaEvent( deltaLog = currentTransactionInfo.readSnapshot.deltaLog, opType = "dropFeature.conflictCheck.metadataMismatch", data = Map( "transactionInfoMetadata" -> currentTransactionInfo.metadata, "actionMetadata" -> m)) CheckpointProtectionTableFeature.metadataWithCheckpointProtection(m, newVersion) case _: Metadata => newMetadata case a => a } currentTransactionInfo = currentTransactionInfo.copy( metadata = newMetadata, actions = newActions) } } } /** * RowTrackingBackfill (or backfill for short for this function) is a special operation that * materializes and recommits all existing files in table using one or several commits to ensure * that every AddFile has a base row ID and a default row commit version. When enabling * row tracking on an existing table, the following occurs: * 1. (If necessary) Protocol upgrade + Table Feature Support is added * 2. RowTrackingBackfill commit(s) * 3. Table property and metadata are updated. * RowTrackingBackfill does not do any data change. It doesn't matter whether a file is * recommitted after the table feature support from Backfill or some other concurrent transaction; * every AddFile just needs to have a base row ID and a default row commit version somehow. * However, correctness issues can arise if we don't do the checks in this method. * * Check that RowTrackingBackfill is not resurrecting files that were removed concurrently and * that an AddFile and its corresponding RemoveFile have the same base row ID and * default row commit version. To do this, we: * 1. remove AddFile's from a backfill commit if an AddFile or a RemoveFile with the same path * was added in the winning concurrent transactions. Files in a winning transaction can be * removed from backfill because they were already re-committed. * 2. copy over base row IDs and default row commit versions if the current transaction re-adds * or delete an AddFile with the same path as an Addfile from a winning backfill commit. */ private def resolveRowTrackingBackfillConflicts(): Unit = { // If row tracking is not supported, there can be no backfill commit. if (!RowTracking.isSupported(currentTransactionInfo.protocol)) { assert(!currentTransactionInfo.isRowTrackingBackfillTxn) assert(!winningCommitSummary.isRowTrackingBackfillTxn) return } val timerPhaseName = "checked-row-tracking-backfill" if (currentTransactionInfo.isRowTrackingBackfillTxn) { recordTime(timerPhaseName) { // Any winning commit seen by backfill must have row IDs and row commit versions, because // `reassignOverlappingRowIds` will add a base row ID and `reassignRowCommitVersions` // will add a default row commit versions to all files. So we don't need // Backfill to commit the same file again. val filePathsToRemoveFromBackfill = winningCommitSummary.actions.collect { case a: AddFile => a.path case r: RemoveFile => r.path }.toSet // Remove files from this Backfill commit if they were removed or re-committed by // a concurrent winning txn. if (filePathsToRemoveFromBackfill.nonEmpty) { // We keep the Row Tracking high-water mark action here but it might // be outdated since the winning commit could have increased the high-water mark. // We will reassign the current transaction's high water-mark if that is // the case, in `reassignOverlappingRowIds` which is called after // `resolveRowTrackingBackfillConflicts` in `checkConflicts`. val newActions = currentTransactionInfo.actions.filterNot { case a: AddFile => filePathsToRemoveFromBackfill.contains(a.path) case d: DomainMetadata if RowTrackingMetadataDomain.isSameDomain(d) => false case _ => throw new IllegalStateException( "RowTrackingBackfill commit has an unexpected action") } val newReadFiles = currentTransactionInfo.readFiles.filterNot( a => filePathsToRemoveFromBackfill.contains(a.path)) currentTransactionInfo = currentTransactionInfo.copy( actions = newActions, readFiles = newReadFiles) } } } if (winningCommitSummary.isRowTrackingBackfillTxn) { recordTime(timerPhaseName) { val backfillActionMap = winningCommitSummary.addedFilePathToActionMap // Copy over the base row ID and default row commit version assigned so that the AddFiles // and RemoveFiles have matching base row ID and default row commit version. // If an AddFile is re-committed, it should have the same base row ID and // default row commit version as the one assigned by Backfill. val newActions = currentTransactionInfo.actions.map { case a: AddFile if backfillActionMap.contains(a.path) => val backfillAction = backfillActionMap(a.path) a.copy(baseRowId = backfillAction.baseRowId, defaultRowCommitVersion = backfillAction.defaultRowCommitVersion) case r: RemoveFile if backfillActionMap.contains(r.path) => val backfillAction = backfillActionMap(r.path) r.copy(baseRowId = backfillAction.baseRowId, defaultRowCommitVersion = backfillAction.defaultRowCommitVersion) case a => a } currentTransactionInfo = currentTransactionInfo.copy(actions = newActions) } } } /** * Row tracking unbackfill is an operation that removes row tracking metadata from the table. * This is achieved by recommiting existing add files without base row ID and default * row commit version. The operation is invoked as part of the cleanup process when dropping * the row tracking feature from the table. * * In general, Delta writers should never generate baseRowIds while * `delta.rowTrackingSuspended` is enabled. However, the delta protocol does not enforce * the config and as a result third party writers may not respect it. The unbackfill conflict * resolver unbackfills the addFiles of the winning commits to compensate for this. */ private def resolveRowTrackingUnBackfillConflicts(): Unit = { // If row tracking is not supported, there can be no unbackfill commit. if (!RowTracking.isSupported(currentTransactionInfo.protocol)) { assert(!currentTransactionInfo.isRowTrackingUnBackfillTxn) assert(!winningCommitSummary.isRowTrackingUnBackfillTxn) return } if (!currentTransactionInfo.isRowTrackingUnBackfillTxn) { return } // Third party writers might not use the same operation name for backfill. // In that case we will proceed to conflict resolution. if (winningCommitSummary.isRowTrackingBackfillTxn) { throw DeltaErrors.rowTrackingBackfillRunningConcurrentlyWithUnbackfill() } val timerPhaseName = "checked-row-tracking-unbackfill" recordTime(timerPhaseName) { currentTransactionInfo = resolveRowTrackingUnBackfillConflicts( currentTransactionInfo, winningCommitSummary) } } /** * Resolve conflicts by cleaning up addFiles of winning commits. Furthermore, make sure * sure that removed files are not resurrected. */ private def resolveRowTrackingUnBackfillConflicts( currentTransactionInfo: CurrentTransactionInfo, winningCommitSummary: WinningCommitSummary): CurrentTransactionInfo = { // Unbackfill new AddFiles. This has the advantage that will cleanup commits // from third party writers that do not respect `delta.rowTrackingSuspended`. val (pathsToRemoveFromUnBackfill, filesToAddToUnBackfill) = winningCommitSummary.actions.collect { case a: AddFile => val fileToAdd = if (a.baseRowId.nonEmpty || a.defaultRowCommitVersion.nonEmpty) { Some(a.copy(dataChange = false, baseRowId = None, defaultRowCommitVersion = None)) } else { None } (a.path, fileToAdd) case r: RemoveFile => (r.path, None) }.unzip val pathsToRemoveFromUnBackfillSet = pathsToRemoveFromUnBackfill.toSet val filesToAddToUnBackfillSet = filesToAddToUnBackfill.flatten.toSet val newActions = currentTransactionInfo.actions.filterNot { case a: AddFile => pathsToRemoveFromUnBackfillSet.contains(a.path) case _ => false } ++ filesToAddToUnBackfillSet // We can remove pruned files from the read list. However, we should not add // the new AddFiles because that would cause a conflict, albeit, we already // resolved it. val newReadFiles = currentTransactionInfo.readFiles.filterNot( a => pathsToRemoveFromUnBackfillSet.contains(a.path)) currentTransactionInfo.copy(actions = newActions, readFiles = newReadFiles) } /** * If the winning commit only does row tracking enablement (i.e. set the table property to * true and assigns materialized row tracking column names), we can safely allow the metadata * update not to fail the current txn if we copy over the table property, materialized column * name assignments and correctly tag the current commit as not preserving row tracking data. It * is not possible to preserve row tracking data prior to the table property being set to true * since there is no guarantee of row tracking data being available on all rows. */ protected def tryResolveRowTrackingEnablementOnlyMetadataUpdateConflict(): Boolean = { if (RowTracking.canResolveMetadataUpdateConflict( currentTransactionInfo, winningCommitSummary)) { currentTransactionInfo = RowTracking.resolveRowTrackingEnablementOnlyMetadataUpdateConflict( currentTransactionInfo, winningCommitSummary) return true } false } // scalastyle:off line.size.limit /** * Check if the committed transaction has changed metadata. * * We want to deal with (and optimize for) the case where the winning commit's metadata update is * solely for updating IDENTITY high water marks. In addition, we want to allow a metadata update * that only sets the table property for row tracking enablement to true not to fail concurrent * transactions if the current transaction does not do a metadata update. * * The conflict matrix is as follows: * * | | Winning Metadata (id) | Winning Metadata Row Tracking Enablement Only | Winning Metadata (other) | Winning No Metadata | * | --------------------------------------------- | --------------------- | --------------------------------------------- | ------------------------ | ------------------- | * | Current Metadata (id) | Conflict | Conflict (3) | Conflict | No conflict | * | Current Metadata Row Tracking Enablement Only | Conflict (1) | Conflict (3) | Conflict | No conflict | * | Current Metadata (other) | Conflict (1) | Conflict (3) | Conflict | No conflict | * | Current No Metadata | No conflict (2) | No conflict (4) | Conflict | No conflict | * * The differences in cases (1), (2), (3), and (4) are: * (1) This is a case we could have done something to avoid conflict, e.g., current transaction * adds a column, while winning transaction does blind append that generates IDENTITY values. But * it's not a common case and the change to avoid conflict is non-trivial (we have to somehow * merge the metadata from winning txn and current txn). We decide to not do that and let it * conflict. * (2) This is a case that is more common (e.g., current = delete/update, winning = update high * water mark) and we will not let it conflict here. Note that it might still cause conflict in * other conflict checks. * (3) If the current txn changes the metadata too, we will fail the current txn. While it is * possible to copy over the metadata information, this scenario is unlikely to happen in practice * and properly handling this for the many edge case (e.g current txn sets the table property * to false) is risky. * (4) In a row tracking enablement only metadata update, the only difference with the previous * metadata are the row tracking table property and materialized column names. These metadata * information only affect the preservation of row tracking. If we copy over the new metadata * configurations and mark the current txn as not preserving row tracking, then the current txn * is respecting the metadata update and does not need to fail. * */ // scalastyle:on line.size.limit protected def checkNoMetadataUpdates(): Unit = { // If winning commit does not contain metadata update, no conflict. if (winningCommitSummary.metadataUpdates.isEmpty) return if (tryResolveRowTrackingEnablementOnlyMetadataUpdateConflict()) { return } // The only case in the remaining cases that we will not conflict is winning commit is // identity only metadata update and current commit has no metadata update. val tolerateIdentityOnlyMetadataUpdate = winningCommitSummary.identityOnlyMetadataUpdate && !currentTransactionInfo.metadataChanged if (!tolerateIdentityOnlyMetadataUpdate) { if (winningCommitSummary.identityOnlyMetadataUpdate) { IdentityColumn.logTransactionAbort(deltaLog) } throw DeltaErrors.metadataChangedException(winningCommitSummary.commitInfo) } } /** * Attempts to resolve metadata conflicts between the current and winning transactions. * Currently, we only support the resolution of configuration changes. This is achieved with * the use of an allow-list that defines which configuration changes are allowed. * * We primarily focus on feature enablement. Features should be considered on a case-by-case * basis whether they are eligible for white listing. The main consideration is whether * transactions that produce the output before the feature enablement are safe to commit * with the feature enabled. For some features the answer might be simply yes while some other * features might require reconciliation logic at conflict resolution. Features that require * data rewrite for reconciliation are not good candidates for white listing. */ protected def attemptToResolveMetadataConflicts(): Unit = { def throwMetadataChangedException(): Unit = throw DeltaErrors.metadataChangedException(winningCommitSummary.commitInfo) // If winning commit does not contain metadata update, no conflict. if (winningCommitSummary.metadataUpdates.isEmpty) return // Cannot resolve when both transactions have metadata updates. if (currentTransactionInfo.metadataChanged) { if (winningCommitSummary.identityOnlyMetadataUpdate) { IdentityColumn.logTransactionAbort(deltaLog) } throwMetadataChangedException() } // Add all special cases here. if (winningCommitSummary.identityOnlyMetadataUpdate) { return } val currentMetadata = currentTransactionInfo.metadata val winningCommitMetadata = winningCommitSummary.metadataUpdates.head val propertyNamesDiff = currentMetadata.diffFieldNames(winningCommitMetadata) // We only support the resolution of configuration changes at the moment and metadata // only schema changes. if (!propertyNamesDiff.subsetOf(Set("configuration", "schemaString"))) { throwMetadataChangedException() } // Clear configuration changes. var configurationChanges = ConfigurationChanges(areValid = false) if (propertyNamesDiff.contains("configuration")) { configurationChanges = checkConfigurationChangesForConflicts( currentMetadata, winningCommitMetadata) if (!configurationChanges.areValid) { throwMetadataChangedException() } } // Clear schema changes. if (propertyNamesDiff.contains("schemaString")) { if (!checkSchemaChangesForConflicts(currentMetadata, winningCommitMetadata)) { throwMetadataChangedException() } } // Metadata changes are accepted. Consolidate them. val rowTrackingEnabled = configurationChanges .addedAndChanged .getOrElse(DeltaConfigs.ROW_TRACKING_ENABLED.key, "false") .toBoolean if (rowTrackingEnabled) { currentTransactionInfo = currentTransactionInfo.copy( commitInfo = currentTransactionInfo .commitInfo .map(RowTracking.addRowTrackingNotPreservedTag)) } currentTransactionInfo = currentTransactionInfo.copy(metadata = winningCommitMetadata) } /** * Return type of [[checkConfigurationChangesForConflicts]]. It indicates whether the * configuration changes are valid and provides the details of the changes. */ private[delta] case class ConfigurationChanges( areValid: Boolean, removed: Set[String] = Set.empty, added: Map[String, String] = Map.empty, changed: Map[String, String] = Map.empty) { def addedAndChanged : Map[String, String] = added ++ changed } /** Allow list for [[checkConfigurationChangesForConflicts]]. */ private lazy val metadataConfigurationChangeAllowList: Set[String] = { val rowTrackingAllowList = Set( MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP, MaterializedRowCommitVersion.MATERIALIZED_COLUMN_NAME_PROP, DeltaConfigs.ROW_TRACKING_ENABLED.key) // We can suppress column mapping enablement conflict error since we do not need any // data rewrite to reconcile the txns. No metadata is pushed to the parquet // footers. The new schema with all the necessary column metadata is copied over // to the current transaction. val columnMappingAllowList = Set( DeltaConfigs.COLUMN_MAPPING_MODE.key, DeltaConfigs.COLUMN_MAPPING_MAX_ID.key) // Resolving a deletion vectors enablement conflict with another transaction is equivalent // of the latter transaction choosing not to generate DVs although DVs are enabled. This // is valid behavior. val dvsAllowList = Set(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key) rowTrackingAllowList ++ columnMappingAllowList ++ dvsAllowList } /** * Validates configuration changes between the current metadata and the winning metadata. * Returns a [[ConfigurationChanges]] object that indicates whether the changes are valid. */ protected[delta] def checkConfigurationChangesForConflicts( currentMetadata: Metadata, winningMetadata: Metadata, allowList: Set[String] = metadataConfigurationChangeAllowList): ConfigurationChanges = { val currentConf = currentMetadata.configuration val winningConf = winningMetadata.configuration val currentConfKeys = currentConf.keySet val winningConfKeys = winningConf.keySet val removedKeys = currentConfKeys -- winningConfKeys val addedKeys = winningConfKeys -- currentConfKeys val changedKeys = currentConfKeys.intersect(winningConfKeys).filter { key => currentConf(key) != winningConf(key) } val addedAndChangedKeys = addedKeys ++ changedKeys def configurationChanges(areValid: Boolean): ConfigurationChanges = { ConfigurationChanges( areValid = areValid, removed = removedKeys, added = addedKeys.map(key => key -> winningConf(key)).toMap, changed = changedKeys.map(key => key -> winningConf(key)).toMap) } def INVALID_CONFIGURATION_CHANGES = configurationChanges(areValid = false) def VALID_CONFIGURATION_CHANGES = configurationChanges(areValid = true) // Unsetting a configuration is not supported at the moment. if (removedKeys.nonEmpty) { return INVALID_CONFIGURATION_CHANGES } // Every added or changed configuration must be in the allow list. if (!addedAndChangedKeys.subsetOf(allowList)) { return INVALID_CONFIGURATION_CHANGES } // Schema: Key, value, isNew. val allChanges = addedKeys.map(key => (key, winningConf(key), true)) ++ changedKeys.map(key => (key, winningConf(key), false)) val validChanges = allChanges.map { case (key, value, isNew) => key match { // Row tracking related configurations. case DeltaConfigs.ROW_TRACKING_ENABLED.key => isRowTrackingConfigChangeConflictFree(value.toBoolean) case MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP => areRowTrackingPropertyChangesConflictFree(winningMetadata) case MaterializedRowCommitVersion.MATERIALIZED_COLUMN_NAME_PROP => areRowTrackingPropertyChangesConflictFree(winningMetadata) // Column mapping related configurations. case DeltaConfigs.COLUMN_MAPPING_MODE.key => areColumnMappingChangesConflictFree(currentMetadata, winningMetadata) case DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key => currentTransactionInfo.protocol.isFeatureSupported(DeletionVectorsTableFeature) && value.toBoolean case _ => true } } if (validChanges.contains(false)) { return INVALID_CONFIGURATION_CHANGES } VALID_CONFIGURATION_CHANGES } protected def isRowTrackingConfigChangeConflictFree(value: Boolean): Boolean = { if (!currentTransactionInfo.protocol.isFeatureSupported(RowTrackingFeature)) { return false } // Currently, we only allow enabling row tracking. value } protected def areRowTrackingPropertyChangesConflictFree(winningMetadata: Metadata): Boolean = { winningMetadata .configuration .getOrElse(DeltaConfigs.ROW_TRACKING_ENABLED.key, "false") .toBoolean } protected def areColumnMappingChangesConflictFree( currentMetadata: Metadata, winningMetadata: Metadata): Boolean = { // Enabling column mapping name mode is the only transition we allow. // Enabling ID mapping on an existing table is generally not allowed. // This should be already blocked by column mapping at an earlier stage. // We add an extra check here for safety. val columnMappingEnabled = currentMetadata.columnMappingMode == NoMapping && winningMetadata.columnMappingMode == NameMapping if (!columnMappingEnabled) { return false } currentTransactionInfo.protocol.isFeatureSupported(ColumnMappingTableFeature) } /** Allows key comparison between two sql.types.Metadata objects. */ class DeltaFieldMetadataComparator(metadata: FieldMetadata) extends MetadataBuilder { withMetadata(metadata) /** Returns a set of added keys by `other`. */ def addedKeys(other: DeltaFieldMetadataComparator): Set[String] = { other.getMap.keySet -- getMap.keySet } /** Returns a set of removed keys by `other`. */ def removedKeys(other: DeltaFieldMetadataComparator): Set[String] = { getMap.keySet -- other.getMap.keySet } /** Returns a set of changed keys by `other`. */ def changedKeys(other: DeltaFieldMetadataComparator): Set[String] = { getMap.keySet.intersect(other.getMap.keySet).filterNot { key => val ourValue = getMap(key) val otherValue = other.getMap(key) (ourValue, otherValue) match { case (v0: Array[Long], v1: Array[Long]) => java.util.Arrays.equals(v0, v1) case (v0: Array[Double], v1: Array[Double]) => java.util.Arrays.equals(v0, v1) case (v0: Array[Boolean], v1: Array[Boolean]) => java.util.Arrays.equals(v0, v1) case (v0: Array[AnyRef], v1: Array[AnyRef]) => java.util.Arrays.equals(v0, v1) case (v0, v1) => v0 == v1 } } } /** Returns a set of keys that were either added, removed or changed by `other`. */ def keysWithAnyChanges(other: DeltaFieldMetadataComparator): Set[String] = { removedKeys(other) .union(addedKeys(other)) .union(changedKeys(other)) } } /** Verifies whether any changes between currentMetadata and winningMetadata are valid. */ protected def checkSchemaChangesForConflicts( currentMetadata: Metadata, winningMetadata: Metadata): Boolean = { val currentSchema = currentMetadata.schema val winningSchema = winningMetadata.schema if (currentSchema.fields.length != winningSchema.fields.length) { return false } // Currently we only support column mapping metadata changes. If column mapping is not // enabled fail (assumes the method was called because schema changes were detected). val columnMappingEnabled = currentMetadata.columnMappingMode == NoMapping && winningMetadata.columnMappingMode == NameMapping if (!columnMappingEnabled) { return false } val allowedMetadataFields = DeltaColumnMapping.COLUMN_MAPPING_METADATA_KEYS currentSchema.fields.zipWithIndex.foreach { case (currentField, index) => val winningField = winningSchema.fields(index) // Currently we only allow metadata changes. if (currentField.name != winningField.name || currentField.dataType != winningField.dataType || currentField.nullable != winningField.nullable) { return false } if (currentField.metadata != winningField.metadata) { val currentFieldMetadataComparator = new DeltaFieldMetadataComparator(currentField.metadata) val winningFieldMetadataComparator = new DeltaFieldMetadataComparator(winningField.metadata) val keysWithAnyChanges = currentFieldMetadataComparator .keysWithAnyChanges(winningFieldMetadataComparator) // We allow all operations on white listed metadata fields. if (!keysWithAnyChanges.subsetOf(allowedMetadataFields)) { return false } } } true } /** * Filters the [[files]] list with the partition predicates of the current transaction * and returns the first file that is matching. */ protected def getFirstFileMatchingPartitionPredicates(files: Seq[AddFile]): Option[AddFile] = { // Blind appends do not read the table. if (currentTransactionInfo.commitInfo.flatMap(_.isBlindAppend).getOrElse(false)) { assert(currentTransactionInfo.readPredicates.isEmpty) return None } // There is no reason to filter files if the table is not partitioned. if (currentTransactionInfo.readWholeTable || currentTransactionInfo.readSnapshot.metadata.partitionColumns.isEmpty) { return files.headOption } import org.apache.spark.sql.delta.implicits._ val filesDf = files.toDF(spark) spark.conf.get(DeltaSQLConf.DELTA_CONFLICT_DETECTION_WIDEN_NONDETERMINISTIC_PREDICATES) match { case DeltaSQLConf.NonDeterministicPredicateWidening.OFF => getFirstFileMatchingPartitionPredicatesInternal( filesDf, shouldWidenNonDeterministicPredicates = false, shouldWidenAllUdf = false) case wideningMode => val fileWithWidening = getFirstFileMatchingPartitionPredicatesInternal( filesDf, shouldWidenNonDeterministicPredicates = true, shouldWidenAllUdf = true) fileWithWidening.flatMap { fileWithWidening => val fileWithoutWidening = getFirstFileMatchingPartitionPredicatesInternal( filesDf, shouldWidenNonDeterministicPredicates = false, shouldWidenAllUdf = false) if (fileWithoutWidening.isEmpty) { // Conflict due to widening of non-deterministic predicate. recordDeltaEvent(deltaLog, opType = "delta.conflictDetection.partitionLevelConcurrency." + "additionalConflictDueToWideningOfNonDeterministicPredicate", data = Map( "wideningMode" -> wideningMode, "predicate" -> currentTransactionInfo.readPredicates.map(_.partitionPredicate.toString), "deterministicUDFs" -> containsDeterministicUDF( currentTransactionInfo.readPredicates, partitionedOnly = true)) ) } if (wideningMode == DeltaSQLConf.NonDeterministicPredicateWidening.ON) { Some(fileWithWidening) } else { fileWithoutWidening } } } } private def getFirstFileMatchingPartitionPredicatesInternal( filesDf: DataFrame, shouldWidenNonDeterministicPredicates: Boolean, shouldWidenAllUdf: Boolean): Option[AddFile] = { def rewritePredicateFn( predicate: Expression, shouldRewriteFilter: Boolean): DeltaTableReadPredicate = { val rewrittenPredicate = if (shouldWidenNonDeterministicPredicates) { val checkDeterministicOptions = CheckDeterministicOptions(allowDeterministicUdf = !shouldWidenAllUdf) eliminateNonDeterministicPredicates(Seq(predicate), checkDeterministicOptions).newPredicates } else { Seq(predicate) } DeltaTableReadPredicate( partitionPredicates = rewrittenPredicate, shouldRewriteFilter = shouldRewriteFilter) } // we need to canonicalize the partition predicates per each group of rewrites vs. nonRewrites val canonicalPredicates = currentTransactionInfo.readPredicates .partition(_.shouldRewriteFilter) match { case (rewrites, nonRewrites) => val canonicalRewrites = ExpressionSet(rewrites.map(_.partitionPredicate)).map( predicate => rewritePredicateFn(predicate, shouldRewriteFilter = true)) val canonicalNonRewrites = ExpressionSet(nonRewrites.map(_.partitionPredicate)).map( predicate => rewritePredicateFn(predicate, shouldRewriteFilter = false)) canonicalRewrites ++ canonicalNonRewrites } import org.apache.spark.sql.delta.implicits._ val filesMatchingPartitionPredicates = canonicalPredicates.iterator .flatMap { readPredicate => val matchingFileOpt = DeltaLog.filterFileList( partitionSchema = currentTransactionInfo.partitionSchemaAtReadTime, files = filesDf, partitionFilters = readPredicate.partitionPredicates, shouldRewritePartitionFilters = readPredicate.shouldRewriteFilter ).as[AddFile].head(1).headOption matchingFileOpt.foreach { f => logInfo(log"Partition predicate is matching a file changed by the winning transaction: " + log"predicate=${MDC(DeltaLogKeys.DATA_FILTER, readPredicate.partitionPredicates.toVector)}, " + log"matchingFile=${MDC(DeltaLogKeys.PATH, f.path)}") } matchingFileOpt }.take(1).toArray filesMatchingPartitionPredicates.headOption } /** * RowTrackingBackfill does not do any data change. If backfill is the winning commit, the * current transaction does not need to read its AddFiles -- the exact same AddFiles have * already been read. If the current commit is backfill, it doesn't need to read the AddFiles * added by the winning transaction. Any winning transaction seen by backfill will commit base * row IDs and default row commit versions, since backfill is only done after table feature * support is added. Removing duplicate AddFiles is handled in * [[resolveRowTrackingBackfillConflicts]]. * * RowTrackingUnBackfill behaves in a similar way. It does not do any data change. When it is * the winning commit, the current transaction does not need to read its AddFiles. However, when * unbackfill it is the current transaction, it pulls the addFiles added by the winning * transaction and unbackfills them. Again, this is a metadata only change. AddFile deduplication * is handled in [[resolveRowTrackingUnBackfillConflicts]]. */ protected def skipCheckedAppendsIfExistsRowTrackingBackfillTransaction(): Boolean = { if (winningCommitSummary.isRowTrackingBackfillTxn || winningCommitSummary.isRowTrackingUnBackfillTxn || currentTransactionInfo.isRowTrackingBackfillTxn || currentTransactionInfo.isRowTrackingUnBackfillTxn) { recordSkippedPhase("checked-appends") return true } false } /** * Check if the new files added by the already committed transactions should have been read by * the current transaction. */ protected def checkForAddedFilesThatShouldHaveBeenReadByCurrentTxn(): Unit = { if (skipCheckedAppendsIfExistsRowTrackingBackfillTransaction()) { return } recordTime("checked-appends") { // Fail if new files have been added that the txn should have read. val addedFilesToCheckForConflicts = isolationLevel match { case WriteSerializable if !currentTransactionInfo.metadataChanged => winningCommitSummary.changedDataAddedFiles // don't conflict with blind appends case Serializable | WriteSerializable => winningCommitSummary.changedDataAddedFiles ++ winningCommitSummary.blindAppendAddedFiles case SnapshotIsolation => Seq.empty } val fileMatchingPartitionReadPredicates = getFirstFileMatchingPartitionPredicates(addedFilesToCheckForConflicts) if (fileMatchingPartitionReadPredicates.nonEmpty) { throw DeltaErrors.concurrentAppendException( winningCommitSummary.commitInfo, getTableNameOrPath, winningCommitVersion, getPrettyPartitionMessage(fileMatchingPartitionReadPredicates.get.partitionValues)) } } } /** * Check if [[RemoveFile]] actions added by already committed transactions conflicts with files * read by the current transaction. */ protected def checkForDeletedFilesAgainstCurrentTxnReadFiles(): Unit = { recordTime("checked-deletes") { // Fail if files have been deleted that the txn read. val readFilePaths = currentTransactionInfo.readFiles.map( f => f.path -> f.partitionValues).toMap val deleteReadOverlap = winningCommitSummary.removedFiles .find(r => readFilePaths.contains(r.path)) if (deleteReadOverlap.nonEmpty) { val partitionOpt = getPrettyPartitionMessage(readFilePaths(deleteReadOverlap.get.path)) throw DeltaErrors.concurrentDeleteReadException( winningCommitSummary.commitInfo, getTableNameOrPath, winningCommitVersion, partitionOpt) } if (winningCommitSummary.removedFiles.nonEmpty && currentTransactionInfo.readWholeTable) { throw DeltaErrors.concurrentDeleteReadException( winningCommitSummary.commitInfo, getTableNameOrPath, winningCommitVersion, partitionOpt = None) } } } /** * Check if [[RemoveFile]] actions added by already committed transactions conflicts with * [[RemoveFile]] actions this transaction is trying to add. */ protected def checkForDeletedFilesAgainstCurrentTxnDeletedFiles(): Unit = { recordTime("checked-2x-deletes") { // Fail if a file is deleted twice. val deletedFilePaths = currentTransactionInfo.actions .collect { case r: RemoveFile => r.path -> r.partitionValues } .toMap val deleteOverlap = winningCommitSummary.removedFiles .find(r => deletedFilePaths.contains(r.path)) if (deleteOverlap.nonEmpty) { val partitionOpt = getPrettyPartitionMessage(deletedFilePaths(deleteOverlap.get.path)) throw DeltaErrors.concurrentDeleteDeleteException( winningCommitSummary.commitInfo, getTableNameOrPath, winningCommitVersion, partitionOpt) } } } /** * Checks if the winning transaction corresponds to some AppId on which current transaction * also depends. */ protected def checkForUpdatedApplicationTransactionIdsThatCurrentTxnDependsOn(): Unit = { // Fail if the appIds seen by the current transaction has been updated by the winning // transaction i.e. the winning transaction have [[SetTransaction]] corresponding to // some appId on which current transaction depends on. Example - This can happen when // multiple instances of the same streaming query are running at the same time. if (winningCommitSummary.appLevelTransactions.exists(currentTransactionInfo.isConflict(_))) { throw DeltaErrors.concurrentTransactionException(winningCommitSummary.commitInfo) } } private lazy val currentTransactionIsReplaceTable: Boolean = currentTransactionInfo.op match { case _: DeltaOperations.ReplaceTable => true case _ => false } /** * Checks [[DomainMetadata]] to capture whether the current transaction conflicts with the * winning transaction at any domain. * 1. Accept the current transaction if its set of metadata domains do not overlap with the * winning transaction's set of metadata domains. * 2. Otherwise, fail the current transaction unless each conflicting domain is associated * with a table feature that defines a domain-specific way of resolving the conflict. */ private def checkIfDomainMetadataConflict(): Unit = { if (!DomainMetadataUtils.domainMetadataSupported(currentTransactionInfo.protocol)) { return } val winningDomainMetadataMap = DomainMetadataUtils.extractDomainMetadatasMap(winningCommitSummary.actions) /** * Any new well-known domains that need custom conflict resolution need to add new cases in * below case match clause. E.g. * case MonotonicCounter(value), Some(MonotonicCounter(conflictingValue)) => * MonotonicCounter(Math.max(value, conflictingValue)) */ def resolveConflict(domainMetadataFromCurrentTransaction: DomainMetadata): DomainMetadata = (domainMetadataFromCurrentTransaction, winningDomainMetadataMap.get(domainMetadataFromCurrentTransaction.domain)) match { // No-conflict case. case (domain, None) => domain case (domain, _) if RowTrackingMetadataDomain.isSameDomain(domain) => domain case (_, Some(_)) => // Any conflict not specifically handled by a previous case must fail the transaction. throw new io.delta.exceptions.ConcurrentTransactionException( s"A conflicting metadata domain ${domainMetadataFromCurrentTransaction.domain} is " + "added.") } val mergedDomainMetadata = mutable.Buffer.empty[DomainMetadata] // Resolve physical [[DomainMetadata]] conflicts (fail on logical conflict). val updatedActions: Seq[Action] = currentTransactionInfo.actions.map { case domainMetadata: DomainMetadata => val mergedAction = resolveConflict(domainMetadata) mergedDomainMetadata += mergedAction mergedAction case other => other } // For the REPLACE TABLE command, if domain metadata of a given domain is added for the first // time by the winning transaction, it may need to be marked as removed. val replaceTableRemoveNewDomainMetadataEnabled = spark.conf.get( DeltaSQLConf.DELTA_CONFLICT_DETECTION_ALLOW_REPLACE_TABLE_TO_REMOVE_NEW_DOMAIN_METADATA) val (finalUpdatedActions, finalMergedDomainMetadata) = if (replaceTableRemoveNewDomainMetadataEnabled && currentTransactionIsReplaceTable) { val (domainMetadataActions, nonDomainMetadataActions) = currentTransactionInfo.actions.partition(_.isInstanceOf[DomainMetadata]) val updatedDomainMetadataActions = DomainMetadataUtils.handleDomainMetadataForReplaceTable( winningDomainMetadataMap.values.toSeq, domainMetadataActions.map(_.asInstanceOf[DomainMetadata])) ((nonDomainMetadataActions ++ updatedDomainMetadataActions), updatedDomainMetadataActions) } else { (updatedActions, mergedDomainMetadata) } currentTransactionInfo = currentTransactionInfo.copy( domainMetadata = finalMergedDomainMetadata.toSeq, actions = finalUpdatedActions) } /** * Metadata is recorded in the table schema on type changes. This includes the table version that * the change was made in, which needs to be updated when there's a conflict. */ private def updateTypeWideningMetadata(): Unit = { if (!TypeWidening.isEnabled(currentTransactionInfo.protocol, currentTransactionInfo.metadata)) { return } val newActions = currentTransactionInfo.actions.map { case metadata: Metadata => val updatedSchema = TypeWideningMetadata.updateTypeChangeVersion( schema = metadata.schema, fromVersion = winningCommitVersion, toVersion = winningCommitVersion + 1L) metadata.copy(schemaString = updatedSchema.json) case a => a } currentTransactionInfo = currentTransactionInfo.copy(actions = newActions) } /** * Checks whether the Row IDs assigned by the current transaction overlap with the Row IDs * assigned by the winning transaction. I.e. this function checks whether both the winning and the * current transaction assigned new Row IDs. If this the case, then this check assigns new Row IDs * to the new files added by the current transaction so that they no longer overlap. */ private def reassignOverlappingRowIds(): Unit = { // The current transaction should only assign Row Ids if they are supported. val currentProtocol = currentTransactionInfo.protocol val currentMetadata = currentTransactionInfo.metadata if (!RowId.isSupported(currentProtocol)) return if (RowTracking.isSuspended(spark, currentMetadata)) return val readHighWaterMark = currentTransactionInfo.readRowIdHighWatermark // The winning transaction might have bumped the high water mark or not in case it did // not add new files to the table. val winningHighWaterMark = winningCommitSummary.actions.collectFirst { case RowTrackingMetadataDomain(domain) => domain.rowIdHighWaterMark }.getOrElse(readHighWaterMark) var highWaterMark = winningHighWaterMark val actionsWithReassignedRowIds = currentTransactionInfo.actions.flatMap { // We should only set missing row IDs and update the row IDs that were assigned by this // transaction, and not the row IDs that were assigned by an earlier transaction and merely // copied over to a new AddFile as part of this transaction. I.e., we should only update the // base row IDs that are larger than the read high watermark. case a: AddFile if !a.baseRowId.exists(_ <= readHighWaterMark) => val newBaseRowId = highWaterMark + 1L highWaterMark += a.numPhysicalRecords.getOrElse { throw DeltaErrors.rowIdAssignmentWithoutStats } Some(a.copy(baseRowId = Some(newBaseRowId))) // The row ID high water mark will be replaced if it exists. case d: DomainMetadata if RowTrackingMetadataDomain.isSameDomain(d) => None case a => Some(a) } currentTransactionInfo = currentTransactionInfo.copy( // Add row ID high water mark at the front for faster retrieval. actions = RowTrackingMetadataDomain(highWaterMark).toDomainMetadata +: actionsWithReassignedRowIds, readRowIdHighWatermark = winningHighWaterMark) } /** * Reassigns default row commit versions to correctly handle the winning transaction. * Concretely: * 1. Reassigns all default row commit versions (of AddFiles in the current transaction) equal to * the version of the winning transaction to the next commit version. * 2. Assigns all unassigned default row commit versions that do not have one assigned yet * to handle the row tracking feature being enabled by the winning transaction. */ private def reassignRowCommitVersions(): Unit = { if (!RowId.isSupported(currentTransactionInfo.protocol)) return if (RowTracking.isSuspended(spark, currentTransactionInfo.metadata)) return val newActions = currentTransactionInfo.actions.map { case a: AddFile if a.defaultRowCommitVersion.contains(winningCommitVersion) => a.copy(defaultRowCommitVersion = Some(winningCommitVersion + 1L)) case a: AddFile if a.defaultRowCommitVersion.isEmpty => // A concurrent transaction has turned on support for Row Tracking. a.copy(defaultRowCommitVersion = Some(winningCommitVersion + 1L)) case a => a } currentTransactionInfo = currentTransactionInfo.copy(actions = newActions) } /** * Adjust the current transaction's commit timestamp to account for the winning * transaction's commit timestamp. If this transaction newly enabled ICT, also update * the table properties to reflect the adjusted enablement version and timestamp. */ private def resolveTimestampOrderingConflicts(): Unit = { if (!DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(currentTransactionInfo.metadata)) { return } val winningCommitTimestamp = if (InCommitTimestampUtils.didCurrentTransactionEnableICT( currentTransactionInfo.metadata, currentTransactionInfo.readSnapshot)) { // Since the current transaction enabled inCommitTimestamps, we should use the file // timestamp from the winning transaction as its commit timestamp. winningCommitSummary.commitFileTimestamp } else { // Get the inCommitTimestamp from the winning transaction. CommitInfo.getRequiredInCommitTimestamp( winningCommitSummary.commitInfo, winningCommitVersion.toString) } val currentTransactionTimestamp = CommitInfo.getRequiredInCommitTimestamp( currentTransactionInfo.commitInfo, "NEW_COMMIT") // getRequiredInCommitTimestamp will throw an exception if commitInfo is None. val currentTransactionCommitInfo = currentTransactionInfo.commitInfo.get val updatedCommitTimestamp = Math.max(currentTransactionTimestamp, winningCommitTimestamp + 1) val updatedCommitInfo = currentTransactionCommitInfo.copy(inCommitTimestamp = Some(updatedCommitTimestamp)) currentTransactionInfo = currentTransactionInfo.copy(commitInfo = Some(updatedCommitInfo)) val nextAvailableVersion = winningCommitVersion + 1L val updatedMetadata = InCommitTimestampUtils.getUpdatedMetadataWithICTEnablementInfo( spark, updatedCommitTimestamp, currentTransactionInfo.readSnapshot, currentTransactionInfo.metadata, nextAvailableVersion) updatedMetadata.foreach { updatedMetadata => currentTransactionInfo = currentTransactionInfo.copy( metadata = updatedMetadata, actions = currentTransactionInfo.actions.map { case _: Metadata => updatedMetadata case other => other } ) } } /** A helper function for pretty printing a specific partition directory. */ protected def getPrettyPartitionMessage(partitionValues: Map[String, String]): Option[String] = { val partitionColumns = currentTransactionInfo.partitionSchemaAtReadTime if (partitionColumns.isEmpty || partitionValues == null) { None } else { Some( partitionColumns.map { field => s"${field.name}=${partitionValues(DeltaColumnMapping.getPhysicalName(field))}" }.mkString("[", ", ", "]") ) } } protected def getTableNameOrPath: String = { val tableName = currentTransactionInfo.catalogTable.map(_.qualifiedName) .getOrElse(currentTransactionInfo.metadata.name) if (tableName != null) { tableName } else { s"delta.`${currentTransactionInfo.readSnapshot.deltaLog.dataPath}`" } } protected def recordTime[T](phase: String)(f: => T): T = { val startTimeNs = System.nanoTime() val ret = f val timeTakenMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs) timingStats += phase -> timeTakenMs ret } protected def recordTime(phase: String, timeTakenMs: Long) = { timingStats += phase -> timeTakenMs } protected def logMetrics(): Unit = { val totalTimeTakenMs = System.currentTimeMillis() - startTimeMs val timingStr = timingStats.keys.toSeq.sorted.map(k => s"$k=${timingStats(k)}").mkString(",") logInfo(log"[" + logPrefix + log"] Timing stats against " + log"${MDC(DeltaLogKeys.VERSION, winningCommitVersion)} " + log"[${MDC(DeltaLogKeys.TIME_STATS, timingStr)}, totalTimeTakenMs: " + log"${MDC(DeltaLogKeys.TIME_MS, totalTimeTakenMs)}]") } protected lazy val logPrefix: MessageWithContext = { def truncate(uuid: String): String = uuid.split("-").head log"[tableId=${MDC(DeltaLogKeys.TABLE_ID, truncate(initialCurrentTransactionInfo.readSnapshot.metadata.id))}," + log"txnId=${MDC(DeltaLogKeys.TXN_ID, truncate(initialCurrentTransactionInfo.txnId))}] " } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/ConflictCheckerPredicateElimination.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.util.DeltaSparkPlanUtils import org.apache.spark.sql.delta.util.DeltaSparkPlanUtils.CheckDeterministicOptions import org.apache.spark.sql.catalyst.expressions.{And, EmptyRow, Expression, Literal, Or} import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan private[delta] trait ConflictCheckerPredicateElimination extends DeltaSparkPlanUtils { /** * This class represents the state of a expression tree transformation, whereby we try to * eliminate predicates that are non-deterministic in a way that widens the set of rows to * include any row that could be read by the original predicate. * * Example: `c1 = 5 AND c2 IN (SELECT c FROM )` would be widened to * `c1 = 5 AND True` eliminating the non-deterministic parquet table read by assuming it would * have matched all c2 values. * * `c1 = 5 OR NOT some_udf(c2)` would be widened to `c1 = 5 OR True`, eliminating the * non-deterministic `some_udf` by assuming `NOT some_udf(c2)` would have selected all rows. * * @param newPredicates * The (potentially widened) list of predicates. * @param eliminatedPredicates * The predicates that were eliminated as non-deterministic. */ protected case class PredicateElimination( newPredicates: Seq[Expression], eliminatedPredicates: Seq[String]) protected object PredicateElimination { final val EMPTY: PredicateElimination = PredicateElimination(Seq.empty, Seq.empty) def eliminate(p: Expression, eliminated: Option[String] = None): PredicateElimination = PredicateElimination( // Always eliminate with a `TrueLiteral`, implying that the eliminated expression would // have read the entire table. newPredicates = Seq(TrueLiteral), eliminatedPredicates = Seq(eliminated.getOrElse(p.prettyName))) def keep(p: Expression): PredicateElimination = PredicateElimination(newPredicates = Seq(p), eliminatedPredicates = Seq.empty) def recurse( p: Expression, recFun: Seq[Expression] => PredicateElimination): PredicateElimination = { val eliminatedChildren = recFun(p.children) if (eliminatedChildren.eliminatedPredicates.isEmpty) { // All children were ok, so keep the current expression. keep(p) } else { // Fold the new predicates after sub-expression widening. val newPredicate = p.withNewChildren(eliminatedChildren.newPredicates) match { case p if p.foldable => Literal.create(p.eval(EmptyRow), p.dataType) case Or(TrueLiteral, _) => TrueLiteral case Or(_, TrueLiteral) => TrueLiteral case And(left, TrueLiteral) => left case And(TrueLiteral, right) => right case p => p } PredicateElimination( newPredicates = Seq(newPredicate), eliminatedPredicates = eliminatedChildren.eliminatedPredicates) } } } /** * Replace non-deterministic expressions in a way that can only increase the number of selected * files when these predicates are used for file skipping. */ protected def eliminateNonDeterministicPredicates( predicates: Seq[Expression], checkDeterministicOptions: CheckDeterministicOptions): PredicateElimination = { eliminateUnsupportedPredicates(predicates) { case p @ SubqueryExpression(plan) => findFirstNonDeltaScan(plan) match { case Some(plan) => PredicateElimination.eliminate(p, eliminated = Some(plan.nodeName)) case None => findFirstNonDeterministicNode(plan, checkDeterministicOptions) match { case Some(node) => PredicateElimination.eliminate(p, eliminated = Some(planOrExpressionName(node))) case None => PredicateElimination.keep(p) } } // And and Or can safely be recursed through. Replacing any non-deterministic sub-tree // with `True` will lead us to at most select more files than necessary later. case p: And => PredicateElimination.recurse(p, p => eliminateNonDeterministicPredicates(p, checkDeterministicOptions)) case p: Or => PredicateElimination.recurse(p, p => eliminateNonDeterministicPredicates(p, checkDeterministicOptions)) // All other expressions must either be completely deterministic, // or must be replaced entirely, since replacing only their non-deterministic children // may lead to files wrongly being deselected (e.g. `NOT True`). case p => // We always look for non-deterministic child nodes, whether or not `p` is actually // deterministic. This gives us better feedback on what caused the non-determinism in // cases where `p` itself it deterministic but `p.deterministic = false` due to correctly // detected non-deterministic child nodes. findFirstNonDeterministicChildNode(p.children, checkDeterministicOptions) match { case Some(node) => PredicateElimination.eliminate(p, eliminated = Some(planOrExpressionName(node))) case None => if (p.deterministic) { PredicateElimination.keep(p) } else { PredicateElimination.eliminate(p) } } } } private def eliminateUnsupportedPredicates(predicates: Seq[Expression])( eliminatePredicates: Expression => PredicateElimination): PredicateElimination = { predicates .map(eliminatePredicates) .foldLeft(PredicateElimination.EMPTY) { case (acc, predicates) => acc.copy( newPredicates = acc.newPredicates ++ predicates.newPredicates, eliminatedPredicates = acc.eliminatedPredicates ++ predicates.eliminatedPredicates) } } private def planOrExpressionName(e: Either[LogicalPlan, Expression]): String = e match { case scala.util.Left(plan) => plan.nodeName case scala.util.Right(expression) => expression.prettyName } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DataFrameUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.{Column, DataFrame, Encoders, SparkSession} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.classic.ClassicConversions._ import org.apache.spark.sql.classic.Dataset import org.apache.spark.sql.execution.QueryExecution object DataFrameUtils { def ofRows(spark: SparkSession, plan: LogicalPlan): DataFrame = Dataset.ofRows(spark, plan) def ofRows(queryExecution: QueryExecution): DataFrame = { val ds = new Dataset(queryExecution, Encoders.row(queryExecution.analyzed.schema)) ds.asInstanceOf[DataFrame] } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DefaultRowCommitVersion.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.FileSourceConstantMetadataStructField import org.apache.spark.sql.types import org.apache.spark.sql.types.{LongType, MetadataBuilder, StructField} object DefaultRowCommitVersion { def assignIfMissing( spark: SparkSession, protocol: Protocol, snapshot: Snapshot, actions: Iterator[Action], version: Long): Iterator[Action] = { if (!RowTracking.isSupported(protocol)) { return actions } // Do not propagate defaultRowCommitVersions if generation is suspended. if (RowTracking.isSuspended(spark, snapshot.metadata)) { actions.map { case a: AddFile if a.defaultRowCommitVersion.isDefined => a.copy(defaultRowCommitVersion = None) case a => a } } else { actions.map { case a: AddFile if a.defaultRowCommitVersion.isEmpty => a.copy(defaultRowCommitVersion = Some(version)) case a => a } } } def createDefaultRowCommitVersionField( protocol: Protocol, metadata: Metadata, nullable: Boolean): Option[StructField] = { Option.when(RowTracking.isEnabled(protocol, metadata)) { MetadataStructField(nullable) } } val METADATA_STRUCT_FIELD_NAME = "default_row_commit_version" private object MetadataStructField { private val METADATA_COL_ATTR_KEY = "__default_row_version_metadata_col" def apply(nullable: Boolean): StructField = StructField( METADATA_STRUCT_FIELD_NAME, LongType, nullable, metadata = metadata) def unapply(field: StructField): Option[StructField] = Some(field).filter(isValid) private def metadata: types.Metadata = new MetadataBuilder() .withMetadata(FileSourceConstantMetadataStructField.metadata(METADATA_STRUCT_FIELD_NAME)) .putBoolean(METADATA_COL_ATTR_KEY, value = true) .build() private def isValid(s: StructField): Boolean = { FileSourceConstantMetadataStructField.isValid(s.dataType, s.metadata) && metadata.contains(METADATA_COL_ATTR_KEY) && metadata.getBoolean(METADATA_COL_ATTR_KEY) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaAnalysis.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.collection.JavaConverters._ import scala.collection.mutable import scala.util.{Failure, Success, Try} import scala.util.control.NonFatal // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.catalyst.TimeTravel import org.apache.spark.sql.delta.Relocated._ import org.apache.spark.sql.delta.DataFrameUtils import org.apache.spark.sql.delta.DeltaErrors.{TemporallyUnstableInputException, TimestampEarlierThanCommitRetentionException} import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils import org.apache.spark.sql.delta.catalog.DeltaCatalogV1 import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.catalog.IcebergTablePlaceHolder import org.apache.spark.sql.delta.commands._ import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.commands.convert.ConvertUtils import org.apache.spark.sql.delta.constraints.{AddConstraint, DropConstraint} import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CoordinatedCommitsUtils} import org.apache.spark.sql.delta.files.{TahoeFileIndex, TahoeLogFileIndex} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources._ import org.apache.spark.sql.delta.sources.DeltaSQLConf.AllowAutomaticWideningMode import org.apache.spark.sql.delta.util.AnalysisHelper import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient import org.apache.hadoop.fs.Path import org.apache.spark.sql.{AnalysisException, Dataset, SaveMode, SparkSession} import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis._ import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType, HiveTableRelation} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.parser.CatalystSqlParser import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.plans.logical.CloneTableStatement import org.apache.spark.sql.catalyst.plans.logical.RestoreTableStatement import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.streaming.WriteToStream import org.apache.spark.sql.catalyst.trees.TreeNodeTag import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttribute import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Identifier, TableCatalog} import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ import org.apache.spark.sql.connector.expressions.{FieldReference, IdentityTransform, Transform} import org.apache.spark.sql.errors.QueryCompilationErrors import org.apache.spark.sql.execution.command.CreateTableLikeCommand import org.apache.spark.sql.execution.command.RunnableCommand import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation, LogicalRelationWithTable} import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, DataSourceV2RelationShim} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.spark.sql.util.CaseInsensitiveStringMap /** * Analysis rules for Delta. Currently, these rules enable schema enforcement / evolution with * INSERT INTO. */ class DeltaAnalysis(session: SparkSession) extends Rule[LogicalPlan] with AnalysisHelper with DeltaLogging { type CastFunction = (Expression, DataType, String) => Expression override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsDown { // INSERT INTO by ordinal and df.insertInto() case a @ AppendDelta(r, d) if !a.isByName && needsSchemaAdjustmentByOrdinal(d, a.query, r.schema, a.writeOptions) => val projection = resolveQueryColumnsByOrdinal(a.query, r.output, d, a.writeOptions) if (projection != a.query) { a.copy(query = projection) } else { a } // INSERT INTO by name // AppendData.byName is also used for DataFrame append so we check for the SQL origin text // since we only want to up-cast for SQL insert into by name case a @ AppendDelta(r, d) if a.isByName && a.origin.sqlText.nonEmpty && needsSchemaAdjustmentByName(a.query, r.output, d, a.writeOptions) => val projection = resolveQueryColumnsByName( query = a.query, targetAttrs = r.output, deltaTable = d, writeOptions = a.writeOptions, allowSchemaEvolution = true) if (projection != a.query) { a.copy(query = projection) } else { a } /** * Handling create table like when a delta target (provider) * is provided explicitly or when the source table is a delta table */ case EligibleCreateTableLikeCommand(ctl, src) => val deltaTableIdentifier = DeltaTableIdentifier(session, ctl.targetTable) // Check if table is given by path val isTableByPath = DeltaTableIdentifier.isDeltaPath(session, ctl.targetTable) // Check if targetTable is given by path val targetTableIdentifier = if (isTableByPath) { TableIdentifier(deltaTableIdentifier.toString) } else { ctl.targetTable } val newStorage = if (ctl.fileFormat.inputFormat.isDefined) { ctl.fileFormat } else if (isTableByPath) { src.storage.copy(locationUri = Some(deltaTableIdentifier.get.getPath(session).toUri)) } else { src.storage.copy(locationUri = ctl.fileFormat.locationUri) } // If the location is specified or target table is given // by path, we create an external table. // Otherwise create a managed table. val tblType = if (newStorage.locationUri.isEmpty && !isTableByPath) { CatalogTableType.MANAGED } else { CatalogTableType.EXTERNAL } // Whether we are enabling Catalog-Owned via explicit property overrides. var isEnablingCatalogOwnedViaExplicitPropertyOverrides: Boolean = false val catalogTableTarget = // If source table is Delta format if (src.provider.exists(DeltaSourceUtils.isDeltaDataSourceName)) { val deltaLogSrc = DeltaTableV2(session, new Path(src.location)) // Column mapping and row tracking fields cannot be set externally. If the features are // used on the source delta table, then the corresponding fields would be set for the // sourceTable and needs to be removed from the targetTable's configuration. The fields // will then be set in the targetTable's configuration internally after. // Coordinated commits/Catalog-Owned configurations from the source delta table should // also be left out, since CREATE LIKE is similar to CLONE, and we do not copy the // commit coordinator from the source table. // If users want a commit coordinator for the target table, they can // specify the configurations in the CREATE LIKE command explicitly. val sourceMetadata = deltaLogSrc.initialSnapshot.metadata // Catalog-Owned: Specifying the table UUID in the TBLPROPERTIES clause // should be blocked. CatalogOwnedTableUtils.validateUCTableIdNotPresent(property = ctl.properties) // Check whether we are trying to enable Catalog-Owned via explicit property overrides. // The reason to check this is, if the source table is a Catalog-Owned table, and // we are also trying to enable Catalog-Owned for the target table - We do *NOT* // want to filter out [[CatalogOwnedTableFeature]] from the source table. If we do that, // the resulting target table's protocol will *NOT* have CatalogOwned table feature // present though we have explicitly specified it in the TBLPROPERTIES clause. // This only applies to cases where source table has Catalog-Owned enabled. // It works as intended if source table is a normal delta table. if (TableFeatureProtocolUtils.getSupportedFeaturesFromTableConfigs( configs = ctl.properties).contains(CatalogOwnedTableFeature)) { isEnablingCatalogOwnedViaExplicitPropertyOverrides = true } val config = sourceMetadata.configuration.-("delta.columnMapping.maxColumnId") .-(MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP) .-(MaterializedRowCommitVersion.MATERIALIZED_COLUMN_NAME_PROP) .filterKeys(!CoordinatedCommitsUtils.TABLE_PROPERTY_KEYS.contains(_)).toMap // Catalog-Owned: Do not copy table UUID from source table .filterKeys(_ != UCCommitCoordinatorClient.UC_TABLE_ID_KEY).toMap new CatalogTable( identifier = targetTableIdentifier, tableType = tblType, storage = newStorage, schema = sourceMetadata.schema, properties = config ++ ctl.properties, partitionColumnNames = sourceMetadata.partitionColumns, provider = Some("delta"), comment = Option(sourceMetadata.description) ) } else { // Source table is not delta format new CatalogTable( identifier = targetTableIdentifier, tableType = tblType, storage = newStorage, schema = src.schema, properties = src.properties ++ ctl.properties, partitionColumnNames = src.partitionColumnNames, provider = Some("delta"), comment = src.comment ) } val saveMode = if (ctl.ifNotExists) { SaveMode.Ignore } else { SaveMode.ErrorIfExists } val protocol = if (src.provider.exists(DeltaSourceUtils.isDeltaDataSourceName)) { Some(DeltaTableV2(session, new Path(src.location)).initialSnapshot.protocol) } else { None } // Catalog-Owned: Do not copy over [[CatalogOwnedTableFeature]] from source table // except the certain case. val protocolAfterFilteringCatalogOwnedFromSource = protocol match { case Some(p) if !isEnablingCatalogOwnedViaExplicitPropertyOverrides => // Only filter out [[CatalogOwnedTableFeature]] when target table is not enabling // CatalogOwned. // E.g., // - CREATE TABLE t1 LIKE t2 // - Filter CatalogOwned table feature out since target table is not enabling // CatalogOwned explicitly. // - CREATE TABLE t1 LIKE t2 TBLPROPERTIES ( // 'delta.feature.catalogManaged' = 'supported' // ) // - Do not filter CatalogOwned table feature out if target table is enabling // CatalogOwned. Some(p.removeFeature(targetFeature = CatalogOwnedTableFeature)) case _ => protocol } val newDeltaCatalog = new DeltaCatalogV1() val existingTableOpt = newDeltaCatalog.getExistingTableIfExists( catalogTableTarget.identifier, identOpt = None, operation = TableCreationModes.Create) val newTable = newDeltaCatalog .verifyTableAndSolidify( catalogTableTarget, None ) CreateDeltaTableCommand( table = newTable, existingTableOpt = existingTableOpt, mode = saveMode, query = None, output = ctl.output, protocol = protocolAfterFilteringCatalogOwnedFromSource, tableByPath = isTableByPath) // INSERT OVERWRITE by ordinal and df.insertInto() case o @ OverwriteDelta(r, d) if !o.isByName && needsSchemaAdjustmentByOrdinal(d, o.query, r.schema, o.writeOptions) => val projection = resolveQueryColumnsByOrdinal(o.query, r.output, d, o.writeOptions) if (projection != o.query) { val aliases = AttributeMap(o.query.output.zip(projection.output).collect { case (l: AttributeReference, r: AttributeReference) if !l.sameRef(r) => (l, r) }) val newDeleteExpr = o.deleteExpr.transformUp { case a: AttributeReference => aliases.getOrElse(a, a) } o.copy(deleteExpr = newDeleteExpr, query = projection) } else { o } // INSERT OVERWRITE by name // OverwriteDelta.byName is also used for DataFrame append so we check for the SQL origin text // since we only want to up-cast for SQL insert into by name case o @ OverwriteDelta(r, d) if o.isByName && o.origin.sqlText.nonEmpty && needsSchemaAdjustmentByName(o.query, r.output, d, o.writeOptions) => val projection = resolveQueryColumnsByName( query = o.query, targetAttrs = r.output, deltaTable = d, writeOptions = o.writeOptions, allowSchemaEvolution = true) if (projection != o.query) { val aliases = AttributeMap(o.query.output.zip(projection.output).collect { case (l: AttributeReference, r: AttributeReference) if !l.sameRef(r) => (l, r) }) val newDeleteExpr = o.deleteExpr.transformUp { case a: AttributeReference => aliases.getOrElse(a, a) } o.copy(deleteExpr = newDeleteExpr, query = projection) } else { o } // INSERT OVERWRITE with dynamic partition overwrite case o @ DynamicPartitionOverwriteDelta(r, d) if o.resolved => val adjustedQuery = if (!o.isByName && needsSchemaAdjustmentByOrdinal(d, o.query, r.schema, o.writeOptions)) { // INSERT OVERWRITE by ordinal and df.insertInto() resolveQueryColumnsByOrdinal(o.query, r.output, d, o.writeOptions) } else if (o.isByName && o.origin.sqlText.nonEmpty && needsSchemaAdjustmentByName(o.query, r.output, d, o.writeOptions)) { // INSERT OVERWRITE by name // OverwriteDelta.byName is also used for DataFrame append so we check for the SQL origin // text since we only want to up-cast for SQL insert into by name resolveQueryColumnsByName( query = o.query, targetAttrs = r.output, deltaTable = d, writeOptions = o.writeOptions, allowSchemaEvolution = true) } else { o.query } DeltaDynamicPartitionOverwriteCommand(r, d, adjustedQuery, o.writeOptions, o.isByName) case ResolveDeltaTableWithPartitionFilters(plan) => plan // SQL CDC table value functions "table_changes" and "table_changes_by_path" case stmt: CDCStatementBase if stmt.functionArgs.forall(_.resolved) => stmt.toTableChanges(session) case tc: TableChanges if tc.child.resolved => tc.toReadQuery // Here we take advantage of CreateDeltaTableCommand which takes a LogicalPlan for CTAS in order // to perform CLONE. We do this by passing the CloneTableCommand as the query in // CreateDeltaTableCommand and let Create handle the creation + checks of creating a table in // the metastore instead of duplicating that effort in CloneTableCommand. case cloneStatement: CloneTableStatement => // Get the info necessary to CreateDeltaTableCommand EliminateSubqueryAliases(cloneStatement.source) match { case DataSourceV2RelationShim(table: DeltaTableV2, _, _, _, _) => resolveCloneCommand(cloneStatement.target, new CloneDeltaSource(table), cloneStatement) // Pass the traveled table if a previous version is to be cloned case tt @ TimeTravel(DataSourceV2RelationShim(tbl: DeltaTableV2, _, _, _, _), _, _, _) if tt.expressions.forall(_.resolved) => val ttSpec = DeltaTimeTravelSpec(tt.timestamp, tt.version, tt.creationSource) val traveledTable = tbl.copy(timeTravelOpt = Some(ttSpec)) resolveCloneCommand( cloneStatement.target, new CloneDeltaSource(traveledTable), cloneStatement) case DataSourceV2RelationShim(table: IcebergTablePlaceHolder, _, _, _, _) => resolveCloneCommand( cloneStatement.target, CloneIcebergSource( metadataLocation = table.tableIdentifier.table, tableNameOpt = None, tablePoliciesOpt = None, deltaSnapshotOpt = None, session), cloneStatement) case DataSourceV2RelationShim(table, _, _, _, _) if table.getClass.getName.endsWith("org.apache.iceberg.spark.source.SparkTable") => val metadataLocation = ConvertUtils.getIcebergMetadataLocationFromSparkTable(table) resolveCloneCommand( cloneStatement.target, CloneIcebergSource( metadataLocation, tableNameOpt = Some(table.name()), tablePoliciesOpt = None, deltaSnapshotOpt = None, session ), cloneStatement) case u: UnresolvedRelation => u.tableNotFound(u.multipartIdentifier) case TimeTravel(u: UnresolvedRelation, _, _, _) => u.tableNotFound(u.multipartIdentifier) case LogicalRelationWithTable( HadoopFsRelation(location, _, _, _, _: ParquetFileFormat, _), catalogTable) => val tableIdent = catalogTable.map(_.identifier) .getOrElse(TableIdentifier(location.rootPaths.head.toString, Some("parquet"))) val provider = if (catalogTable.isDefined) { catalogTable.get.provider.getOrElse("Unknown") } else { "parquet" } // Only plain Parquet sources are eligible for CLONE, extensions like 'deltaSharing' are // NOT supported. if (!provider.equalsIgnoreCase("parquet")) { throw DeltaErrors.cloneFromUnsupportedSource( tableIdent.unquotedString, provider) } resolveCloneCommand( cloneStatement.target, CloneParquetSource(tableIdent, catalogTable, session), cloneStatement) case HiveTableRelation(catalogTable, _, _, _, _) => if (!ConvertToDeltaCommand.isHiveStyleParquetTable(catalogTable)) { throw DeltaErrors.cloneFromUnsupportedSource( catalogTable.identifier.unquotedString, catalogTable.storage.serde.getOrElse("Unknown")) } resolveCloneCommand( cloneStatement.target, CloneParquetSource(catalogTable.identifier, Some(catalogTable), session), cloneStatement) case v: View => throw DeltaErrors.cloneFromUnsupportedSource( v.desc.identifier.unquotedString, "View") case l: LogicalPlan => throw DeltaErrors.cloneFromUnsupportedSource( l.toString, "Unknown") } case restoreStatement @ RestoreTableStatement(target) => EliminateSubqueryAliases(target) match { // Pass the traveled table if a previous version is to be cloned case tt @ TimeTravel(DataSourceV2RelationShim(tbl: DeltaTableV2, _, _, _, _), _, _, _) if tt.expressions.forall(_.resolved) => val ttSpec = DeltaTimeTravelSpec(tt.timestamp, tt.version, tt.creationSource) val traveledTable = tbl.copy(timeTravelOpt = Some(ttSpec)) // restoring to same version as latest should be a no-op. val sourceSnapshot = try { traveledTable.initialSnapshot } catch { case v: VersionNotFoundException => throw DeltaErrors.restoreVersionNotExistException(v.userVersion, v.earliest, v.latest) case tEarlier: TimestampEarlierThanCommitRetentionException => throw DeltaErrors.restoreTimestampBeforeEarliestException( tEarlier.userTimestamp.toString, tEarlier.commitTs.toString ) case tUnstable: TemporallyUnstableInputException => throw DeltaErrors.restoreTimestampGreaterThanLatestException( tUnstable.userTs.toString, tUnstable.lastCommitTs.toString ) } // TODO: Fetch the table version from deltaLog.update().version to guarantee freshness. // This can also be used by RestoreTableCommand if (sourceSnapshot.version == traveledTable.deltaLog.unsafeVolatileSnapshot.version) { return LocalRelation(restoreStatement.output) } RestoreTableCommand(traveledTable) case u: UnresolvedRelation => u.tableNotFound(u.multipartIdentifier) case TimeTravel(u: UnresolvedRelation, _, _, _) => u.tableNotFound(u.multipartIdentifier) case _ => throw DeltaErrors.notADeltaTableException("RESTORE") } // Resolve as a resolved table if the path is for delta table. For non delta table, we keep the // path and pass it along in a ResolvedPathBasedNonDeltaTable. This is needed as DESCRIBE DETAIL // supports both delta and non delta paths. case u: UnresolvedPathBasedTable => val table = getPathBasedDeltaTable(u.path, u.options) if (Try(table.tableExists).getOrElse(false)) { // Resolve it as a path-based Delta table val catalog = session.sessionState.catalogManager.currentCatalog.asTableCatalog ResolvedTable.create( catalog, Identifier.of(Array(DeltaSourceUtils.ALT_NAME), u.path), table) } else { // Resolve it as a placeholder, to identify it as a non-Delta table. ResolvedPathBasedNonDeltaTable(u.path, u.options, u.commandName) } case u: UnresolvedPathBasedDeltaTable => val table = getPathBasedDeltaTable(u.path, u.options) if (!table.tableExists) { throw DeltaErrors.notADeltaTableException(u.commandName, u.deltaTableIdentifier) } val catalog = session.sessionState.catalogManager.currentCatalog.asTableCatalog ResolvedTable.create(catalog, u.identifier, table) case u: UnresolvedPathBasedDeltaTableRelation => val table = getPathBasedDeltaTable(u.path, u.options.asScala.toMap) if (!table.tableExists) { throw DeltaErrors.notADeltaTableException(u.deltaTableIdentifier) } DataSourceV2Relation.create(table, None, Some(u.identifier), u.options) case d: DescribeDeltaHistory if d.childrenResolved => d.toCommand case FallbackToV1DeltaRelation(v1Relation) => v1Relation case ResolvedTable(_, _, d: DeltaTableV2, _) if d.catalogTable.isEmpty && !d.tableExists => // This is DDL on a path based table that doesn't exist. CREATE will not hit this path, most // SHOW / DESC code paths will hit this throw DeltaErrors.notADeltaTableException(DeltaTableIdentifier(path = Some(d.path.toString))) // DML - TODO: Remove these Delta-specific DML logical plans and use Spark's plans directly case d @ DeleteFromTable(table, condition) if d.childrenResolved => // rewrites Delta from V2 to V1 val newTarget = stripTempViewWrapper(table).transformUp { case DeltaRelation(lr) => lr } val indices = newTarget.collect { case DeltaFullTable(_, index) => index } if (indices.isEmpty) { // Not a Delta table at all, do not transform d } else if (indices.size == 1 && indices(0).deltaLog.tableExists) { // It is a well-defined Delta table with a schema DeltaDelete(newTarget, Some(condition)) } else { // Not a well-defined Delta table throw DeltaErrors.notADeltaSourceException("DELETE", Some(d)) } case u @ UpdateTable(table, assignments, condition) if u.childrenResolved => val (cols, expressions) = assignments.map(a => a.key -> a.value).unzip // rewrites Delta from V2 to V1 val newTable = stripTempViewWrapper(table).transformUp { case DeltaRelation(lr) => lr } newTable.collectLeaves().headOption match { case Some(DeltaFullTable(_, index)) => DeltaUpdateTable(newTable, cols, expressions, condition) case o => // not a Delta table u } case merge: MergeIntoTable if merge.childrenResolved => val matchedActions = merge.matchedActions.map { case update: UpdateAction => DeltaMergeIntoMatchedUpdateClause( update.condition, DeltaMergeIntoClause.toActions(update.assignments)) case update: UpdateStarAction => DeltaMergeIntoMatchedUpdateClause(update.condition, DeltaMergeIntoClause.toActions(Nil)) case delete: DeleteAction => DeltaMergeIntoMatchedDeleteClause(delete.condition) case other => throw new IllegalArgumentException( s"${other.prettyName} clauses cannot be part of the WHEN MATCHED clause in MERGE INTO.") } val notMatchedActions = merge.notMatchedActions.map { case insert: InsertAction => DeltaMergeIntoNotMatchedInsertClause( insert.condition, DeltaMergeIntoClause.toActions(insert.assignments)) case insert: InsertStarAction => DeltaMergeIntoNotMatchedInsertClause( insert.condition, DeltaMergeIntoClause.toActions(Nil)) case other => throw new IllegalArgumentException( s"${other.prettyName} clauses cannot be part of the WHEN NOT MATCHED clause in MERGE " + "INTO.") } val notMatchedBySourceActions = merge.notMatchedBySourceActions.map { case update: UpdateAction => DeltaMergeIntoNotMatchedBySourceUpdateClause( update.condition, DeltaMergeIntoClause.toActions(update.assignments)) case delete: DeleteAction => DeltaMergeIntoNotMatchedBySourceDeleteClause(delete.condition) case other => throw new IllegalArgumentException( s"${other.prettyName} clauses cannot be part of the WHEN NOT MATCHED BY SOURCE " + "clause in MERGE INTO.") } // rewrites Delta from V2 to V1 var isDelta = false val newTarget = stripTempViewForMergeWrapper(merge.targetTable).transformUp { case DeltaRelation(lr) => isDelta = true lr } if (isDelta) { // Even if we're merging into a non-Delta target, we will catch it later and throw an // exception. val deltaMerge = DeltaMergeInto( newTarget, merge.sourceTable, merge.mergeCondition, matchedActions ++ notMatchedActions ++ notMatchedBySourceActions, // TODO: We are waiting for Spark to support the SQL "WITH SCHEMA EVOLUTION" syntax. // After that this argument will be `merge.withSchemaEvolution`. withSchemaEvolution = false ) ResolveDeltaMergeInto.resolveReferencesAndSchema(deltaMerge, conf)( tryResolveReferencesForExpressions(session)) } else { merge } case merge: MergeIntoTable if merge.targetTable.exists(_.isInstanceOf[DataSourceV2Relation]) => // When we hit here, it means the MERGE source is not resolved and we can't convert the MERGE // command to the Delta variant. We need to add a special marker to the target table, so that // this rule does not convert it to v1 relation too early, as we need to keep it as a v2 // relation to bypass the OSS MERGE resolution code in the rule `ResolveReferences`. merge.targetTable.foreach { // TreeNodeTag is not very reliable, but it's OK to use it here, as we will use it very // soon: when this rule transforms down the plan tree and hits the MERGE target table. // There is no chance in this rule that we will drop this tag. At the end, This rule will // turn MergeIntoTable into DeltaMergeInto, and convert all Delta relations inside it to // v1 relations (no need to clean up this tag). case r: DataSourceV2Relation => r.setTagValue(DeltaRelation.KEEP_AS_V2_RELATION_TAG, ()) case _ => } merge case reorg @ DeltaReorgTable(resolved @ ResolvedTable(_, _, _: DeltaTableV2, _), spec) => DeltaReorgTableCommand(resolved, spec)(reorg.predicates) case DeltaReorgTable(ResolvedTable(_, _, t, _), _) => throw DeltaErrors.notADeltaTable(t.name()) case cmd @ ShowColumns(child @ ResolvedTable(_, _, table: DeltaTableV2, _), namespace, _) => // Adapted from the rule in spark ResolveSessionCatalog.scala, which V2 tables don't trigger. // NOTE: It's probably a spark bug to check head instead of tail, for 3-part identifiers. val resolver = session.sessionState.analyzer.resolver val v1TableName = child.identifier.asTableIdentifier namespace.foreach { ns => if (v1TableName.database.exists(!resolver(_, ns.head))) { throw DeltaThrowableHelper.showColumnsWithConflictDatabasesError(ns, v1TableName) } } ShowDeltaTableColumnsCommand(child) case deltaMerge: DeltaMergeInto => val d = if (deltaMerge.childrenResolved && !deltaMerge.resolved) { ResolveDeltaMergeInto.resolveReferencesAndSchema(deltaMerge, conf)( tryResolveReferencesForExpressions(session)) } else deltaMerge d.copy(target = stripTempViewForMergeWrapper(d.target)) case origStreamWrite: WriteToStream => // The command could have Delta as source and/or sink. We need to look at both. val streamWrite = origStreamWrite match { case WriteToStream(_, _, sink @ DeltaSink(_, _, _, _, _, None), _, _, _, _, Some(ct)) => // The command has a catalog table, but the DeltaSink does not. This happens because // DeltaDataSource.createSink (Spark API) didn't have access to the catalog table when it // created the DeltaSink. Fortunately we can fix it up here. origStreamWrite.copy(sink = sink.copy(catalogTable = Some(ct))) case _ => origStreamWrite } // We also need to validate the source schema location, if the command has a Delta source. verifyDeltaSourceSchemaLocation( streamWrite.inputQuery, streamWrite.resolvedCheckpointLocation) streamWrite } /** * Creates a catalog table for CreateDeltaTableCommand. * * @param targetPath Target path containing the target path to clone to * @param byPath Whether the target is a path based table * @param tableIdent Table Identifier for the target table * @param targetLocation User specified target location for the new table * @param existingTable Existing table definition if we're going to be replacing the table * @param srcTable The source table to clone * @return catalog to CreateDeltaTableCommand with */ private def createCatalogTableForCloneCommand( targetPath: Path, byPath: Boolean, tableIdent: TableIdentifier, targetLocation: Option[String], existingTable: Option[CatalogTable], srcTable: CloneSource, propertiesOverrides: Map[String, String]): CatalogTable = { // If external location is defined then then table is an external table // If the table is a path-based table, we also say that the table is external even if no // metastore table will be created. This is done because we are still explicitly providing a // locationUri which is behavior expected only of external tables // In the case of ifNotExists being true and a table existing at the target destination, create // a managed table so we don't have to pass a fake path val (tableType, storage) = if (targetLocation.isDefined || byPath) { (CatalogTableType.EXTERNAL, CatalogStorageFormat.empty.copy(locationUri = Some(targetPath.toUri))) } else { (CatalogTableType.MANAGED, CatalogStorageFormat.empty) } var properties = srcTable.metadata.configuration val validatedOverrides = DeltaConfigs.validateConfigurations(propertiesOverrides) properties = properties.filterKeys(!validatedOverrides.keySet.contains(_)).toMap ++ validatedOverrides new CatalogTable( identifier = tableIdent, tableType = tableType, storage = storage, schema = srcTable.schema, properties = properties, provider = Some("delta"), stats = existingTable.flatMap(_.stats) ) } private def getPathBasedDeltaTable(path: String, options: Map[String, String]): DeltaTableV2 = { DeltaTableV2(session, new Path(path), options = options) } private def resolveCreateTableMode( isCreate: Boolean, isReplace: Boolean, ifNotExist: Boolean): (SaveMode, TableCreationModes.CreationMode) = { val saveMode = if (isReplace) { SaveMode.Overwrite } else if (ifNotExist) { SaveMode.Ignore } else { SaveMode.ErrorIfExists } val tableCreationMode = if (isCreate && isReplace) { TableCreationModes.CreateOrReplace } else if (isCreate) { TableCreationModes.Create } else { TableCreationModes.Replace } (saveMode, tableCreationMode) } /** * Instantiates a CreateDeltaTableCommand with CloneTableCommand as the child query. * * @param targetPlan the target of Clone as passed in a LogicalPlan * @param sourceTbl the DeltaTableV2 that was resolved as the source of the clone command * @return Resolve the clone command as the query in a CreateDeltaTableCommand. */ private def resolveCloneCommand( targetPlan: LogicalPlan, sourceTbl: CloneSource, statement: CloneTableStatement): LogicalPlan = { val isReplace = statement.isReplaceCommand val isCreate = statement.isCreateCommand val ifNotExists = statement.ifNotExists val analyzer = session.sessionState.analyzer import analyzer.{NonSessionCatalogAndIdentifier, SessionCatalogAndIdentifier} val targetLocation = statement.targetLocation val (saveMode, tableCreationMode) = resolveCreateTableMode(isCreate, isReplace, ifNotExists) // We don't use information in the catalog if the table is time travelled val sourceCatalogTable = if (sourceTbl.timeTravelOpt.isDefined) None else sourceTbl.catalogTable EliminateSubqueryAliases(targetPlan) match { // Target is a path based table case DataSourceV2RelationShim(targetTbl: DeltaTableV2, _, _, _, _) if !targetTbl.tableExists => val path = targetTbl.path val tblIdent = TableIdentifier(path.toString, Some("delta")) if (!isCreate) { throw DeltaErrors.cannotReplaceMissingTableException( Identifier.of(Array("delta"), path.toString)) } // Trying to clone something on itself should be a no-op if (sourceTbl == new CloneDeltaSource(targetTbl)) { return LocalRelation() } // If this is a path based table and an external location is also defined throw an error if (statement.targetLocation.exists(loc => new Path(loc).toString != path.toString)) { throw DeltaErrors.cloneAmbiguousTarget(statement.targetLocation.get, tblIdent) } // We're creating a table by path and there won't be a place to store catalog stats val catalog = createCatalogTableForCloneCommand(path, byPath = true, tblIdent, targetLocation, sourceCatalogTable, sourceTbl, statement.tablePropertyOverrides) CreateDeltaTableCommand( catalog, None, saveMode, Some(CloneTableCommand( sourceTbl, tblIdent, statement.tablePropertyOverrides, path)), tableByPath = true, output = CloneTableCommand.output) // Target is a metastore table case UnresolvedRelation(SessionCatalogAndIdentifier(catalog, ident), _, _) => if (!isCreate) { throw DeltaErrors.cannotReplaceMissingTableException(ident) } val tblIdent = ident .asTableIdentifier val finalTarget = new Path(statement.targetLocation.getOrElse( session.sessionState.catalog.defaultTablePath(tblIdent).toString)) val catalogTable = createCatalogTableForCloneCommand(finalTarget, byPath = false, tblIdent, targetLocation, sourceCatalogTable, sourceTbl, statement.tablePropertyOverrides) val catalogTableWithPath = if (targetLocation.isEmpty) { catalogTable.copy( storage = CatalogStorageFormat.empty.copy(locationUri = Some(finalTarget.toUri))) } else { catalogTable } CreateDeltaTableCommand( catalogTableWithPath, None, saveMode, Some(CloneTableCommand( sourceTbl, tblIdent, statement.tablePropertyOverrides, finalTarget)), operation = tableCreationMode, output = CloneTableCommand.output) case UnresolvedRelation(NonSessionCatalogAndIdentifier(catalog: TableCatalog, ident), _, _) => if (!isCreate) { throw DeltaErrors.cannotReplaceMissingTableException(ident) } val partitions: Array[Transform] = sourceTbl.metadata.partitionColumns.map { col => new IdentityTransform(new FieldReference(Seq(col))) }.toArray // HACK ALERT: since there is no DSV2 API for getting table path before creation, // here we create a table to get the path, then overwrite it with the // cloned table. val sourceConfig = sourceTbl.metadata.configuration.asJava val newTable = catalog.createTable( ident, CatalogV2Util.structTypeToV2Columns(sourceTbl.schema), partitions, sourceConfig ) try { newTable match { case targetTable: DeltaTableV2 => val path = targetTable.path val tblIdent = TableIdentifier(path.toString, Some("delta")) val catalogTable = createCatalogTableForCloneCommand(path, byPath = true, tblIdent, targetLocation, sourceCatalogTable, sourceTbl, statement.tablePropertyOverrides) CreateDeltaTableCommand( table = catalogTable, existingTableOpt = None, mode = SaveMode.Overwrite, query = Some( CloneTableCommand( sourceTable = sourceTbl, targetIdent = tblIdent, tablePropertyOverrides = statement.tablePropertyOverrides, targetPath = path)), tableByPath = true, operation = TableCreationModes.Replace, output = CloneTableCommand.output) case _ => throw DeltaErrors.notADeltaSourceException("CREATE TABLE CLONE", Some(statement)) } } catch { case NonFatal(e) => catalog.dropTable(ident) throw e } // Delta metastore table already exists at target case DataSourceV2RelationShim(deltaTableV2: DeltaTableV2, _, _, _, _) => val path = deltaTableV2.path val existingTable = deltaTableV2.catalogTable val tblIdent = existingTable match { case Some(existingCatalog) => existingCatalog.identifier case None => TableIdentifier(path.toString, Some("delta")) } val catalogTable = createCatalogTableForCloneCommand( path, byPath = existingTable.isEmpty, tblIdent, targetLocation, sourceCatalogTable, sourceTbl, statement.tablePropertyOverrides) CreateDeltaTableCommand( catalogTable, existingTable, saveMode, Some(CloneTableCommand( sourceTbl, tblIdent, statement.tablePropertyOverrides, path)), tableByPath = existingTable.isEmpty, operation = tableCreationMode, output = CloneTableCommand.output) // Non-delta metastore table already exists at target case LogicalRelationWithTable(_, existingCatalogTable @ Some(catalogTable)) => val tblIdent = catalogTable.identifier val path = new Path(catalogTable.location) val newCatalogTable = createCatalogTableForCloneCommand(path, byPath = false, tblIdent, targetLocation, sourceCatalogTable, sourceTbl, statement.tablePropertyOverrides) CreateDeltaTableCommand( newCatalogTable, existingCatalogTable, saveMode, Some(CloneTableCommand( sourceTbl, tblIdent, statement.tablePropertyOverrides, path)), operation = tableCreationMode, output = CloneTableCommand.output) case _ => throw DeltaErrors.notADeltaTableException("CLONE") } } /** * Conditionally wraps a struct expression with an IF expression to preserve NULL source values * when `DELTA_INSERT_PRESERVE_NULL_SOURCE_STRUCTS` is enabled: * IF(sourceExpr IS NULL, NULL, createStructExpr) * * This prevents null expansion where a null struct would be incorrectly expanded to a struct * with all fields set to NULL during INSERT operations. * * @param sourceExpr The source struct expression * @param createStructExpr The generated CreateStruct expression * @return The potentially wrapped expression with null preservation logic */ private def maybeWrapWithNullPreservationForInsert( sourceExpr: Expression, createStructExpr: Expression): Expression = { if (conf.getConf(DeltaSQLConf.DELTA_INSERT_PRESERVE_NULL_SOURCE_STRUCTS)) { val sourceNullCondition = IsNull(sourceExpr) val targetType = createStructExpr.dataType If( sourceNullCondition, Literal.create(null, targetType), createStructExpr ) } else { createStructExpr } } /** * Performs the schema adjustment by adding UpCasts (which are safe) and Aliases so that we * can check if the by-ordinal schema of the insert query matches our Delta table. * The schema adjustment also include string length check if it's written into a char/varchar * type column/field. */ private def resolveQueryColumnsByOrdinal( query: LogicalPlan, targetAttrs: Seq[Attribute], deltaTable: DeltaTableV2, writeOptions: Map[String, String]): LogicalPlan = { // always add a Cast. it will be removed in the optimizer if it is unnecessary. val project = query.output.zipWithIndex.map { case (attr, i) => if (i < targetAttrs.length) { val targetAttr = targetAttrs(i) addCastToColumn(attr, targetAttr, deltaTable.name(), typeWideningMode = getTypeWideningMode(deltaTable, writeOptions) ) } else { attr } } Project(project, query) } /** * Performs the schema adjustment by adding UpCasts (which are safe) so that we can insert into * the Delta table when the input data types doesn't match the table schema. Unlike * `resolveQueryColumnsByOrdinal` which ignores the names in `targetAttrs` and maps attributes * directly to query output, this method will use the names in the query output to find the * corresponding attribute to use. This method also allows users to not provide values for * generated columns. If values of any columns are not in the query output, they must be generated * columns. */ private def resolveQueryColumnsByName( query: LogicalPlan, targetAttrs: Seq[Attribute], deltaTable: DeltaTableV2, writeOptions: Map[String, String], allowSchemaEvolution: Boolean = false): LogicalPlan = { // Schema evolution is only effective when mergeSchema is enabled in write options AND // the feature is enabled via SQL conf. val effectiveSchemaEvolution = allowSchemaEvolution && new DeltaOptions(deltaTable.options ++ writeOptions, conf).canMergeSchema && session.conf.get(DeltaSQLConf.DELTA_INSERT_BY_NAME_SCHEMA_EVOLUTION_ENABLED) insertIntoByNameMissingColumn(query, targetAttrs, deltaTable, effectiveSchemaEvolution) // This is called before resolveOutputColumns in postHocResolutionRules, so we need to duplicate // the schema validation here. if (!effectiveSchemaEvolution && query.output.length > targetAttrs.length) { throw QueryCompilationErrors.cannotWriteTooManyColumnsToTableError( tableName = deltaTable.name(), expected = targetAttrs.map(_.name), queryOutput = query.output) } val project = query.output.map { attr => val targetAttr = targetAttrs.find(t => session.sessionState.conf.resolver(t.name, attr.name)) .getOrElse { if (effectiveSchemaEvolution) { attr } else { // Extra columns in the source are not allowed when schema evolution is disabled. throw DeltaErrors.missingColumn(attr, targetAttrs) } } addCastToColumn(attr, targetAttr, deltaTable.name(), typeWideningMode = getTypeWideningMode(deltaTable, writeOptions) ) } Project(project, query) } private def addCastToColumn( attr: NamedExpression, targetAttr: NamedExpression, tblName: String, typeWideningMode: TypeWideningMode): NamedExpression = { val expr = (attr.dataType, targetAttr.dataType) match { case (s, t) if s == t => attr case (s: StructType, t: StructType) if s != t => addCastsToStructs(tblName, attr, s, t, typeWideningMode) case (ArrayType(s: StructType, sNull: Boolean), ArrayType(t: StructType, tNull: Boolean)) if s != t && sNull == tNull => addCastsToArrayStructs(tblName, attr, s, t, sNull, typeWideningMode) case (s: AtomicType, t: AtomicType) if typeWideningMode.shouldWidenTo(fromType = t, toType = s) => // Keep the type from the query, the target schema will be updated to widen the existing // type to match it. attr case (s: MapType, t: MapType) if !DataType.equalsStructurally(s, t, ignoreNullability = true) => // only trigger addCastsToMaps if exists differences like extra fields, renaming or type // differences. addCastsToMaps(tblName, attr, s, t, typeWideningMode) case _ => getCastFunction(attr, targetAttr.dataType, targetAttr.name) } Alias(expr, targetAttr.name)(explicitMetadata = Option(targetAttr.metadata)) } /** * Returns the type widening mode to use for the given delta table. A type widening mode indicates * for (fromType, toType) tuples whether `fromType` is eligible to be automatically widened to * `toType` when ingesting data. If it is, the table schema is updated to `toType` before * ingestion and values are written using their original `toType` type. Otherwise, the table type * `fromType` is retained and values are downcasted on write. */ private def getTypeWideningMode( deltaTable: DeltaTableV2, writeOptions: Map[String, String]): TypeWideningMode = { val options = new DeltaOptions(deltaTable.options ++ writeOptions, conf) val snapshot = deltaTable.initialSnapshot val typeWideningEnabled = TypeWidening.isEnabled(snapshot.protocol, snapshot.metadata) val schemaEvolutionEnabled = options.canMergeSchema if (typeWideningEnabled && schemaEvolutionEnabled) { TypeWideningMode.TypeEvolution( uniformIcebergCompatibleOnly = UniversalFormat.icebergEnabled(snapshot.metadata), allowAutomaticWidening = AllowAutomaticWideningMode.fromConf(conf)) } else { TypeWideningMode.NoTypeWidening } } /** * With Delta, we ACCEPT_ANY_SCHEMA, meaning that Spark doesn't automatically adjust the schema * of INSERT INTO. This allows us to perform better schema enforcement/evolution. Since Spark * skips this step, we see if we need to perform any schema adjustment here. */ private def needsSchemaAdjustmentByOrdinal( deltaTable: DeltaTableV2, query: LogicalPlan, schema: StructType, writeOptions: Map[String, String]): Boolean = { val output = query.output if (output.length < schema.length) { throw DeltaErrors.notEnoughColumnsInInsert(deltaTable.name(), output.length, schema.length) } // Now we should try our best to match everything that already exists, and leave the rest // for schema evolution to WriteIntoDelta val existingSchemaOutput = output.take(schema.length) existingSchemaOutput.map(_.name) != schema.map(_.name) || !SchemaUtils.isReadCompatible(schema.asNullable, existingSchemaOutput.toStructType, typeWideningMode = getTypeWideningMode(deltaTable, writeOptions)) } /** * Checks for missing columns in a insert by name query and throws an exception if found. * Delta does not require users to provide values for generated columns, so any columns missing * from the query output must have a default expression. * See [[ColumnWithDefaultExprUtils.columnHasDefaultExpr]]. */ private def insertIntoByNameMissingColumn( query: LogicalPlan, targetAttrs: Seq[Attribute], deltaTable: DeltaTableV2, allowSchemaEvolution: Boolean = false): Unit = { // When allowing the source schema to contain extra columns, it can still // be missing required columns from the target schema. // // The total column count alone is not sufficient for validation. // A source may have more columns overall but still omit specific columns // that are required by the target schema. // // Example: // Target: [a, b] // Source: [a, x, y] // Since the source has 3 columns vs target's 2, but is missing column b, this should be caught. if (allowSchemaEvolution || query.output.length < targetAttrs.length) { val userSpecifiedNames = if (session.sessionState.conf.caseSensitiveAnalysis) { query.output.map(a => (a.name, a)).toMap } else { CaseInsensitiveMap(query.output.map(a => (a.name, a)).toMap) } val tableSchema = deltaTable.initialSnapshot.metadata.schema if (tableSchema.length != targetAttrs.length) { // The target attributes may contain the metadata columns by design. Throwing an exception // here in case target attributes may have the metadata columns for Delta in future. throw DeltaErrors.schemaNotConsistentWithTarget(s"$tableSchema", s"$targetAttrs") } val nullAsDefault = deltaTable.spark.sessionState.conf.useNullsForMissingDefaultColumnValues deltaTable.initialSnapshot.metadata.schema.foreach { col => if (!userSpecifiedNames.contains(col.name) && !ColumnWithDefaultExprUtils.columnHasDefaultExpr( deltaTable.initialSnapshot.protocol, col, nullAsDefault)) { throw DeltaErrors.missingColumnsInInsertInto(col.name) } } } } /** * With Delta, we ACCEPT_ANY_SCHEMA, meaning that Spark doesn't automatically adjust the schema * of INSERT INTO. Here we check if we need to perform any schema adjustment for INSERT INTO by * name queries. We also check that any columns not in the list of user-specified columns must * have a default expression. */ private def needsSchemaAdjustmentByName( query: LogicalPlan, targetAttrs: Seq[Attribute], deltaTable: DeltaTableV2, writeOptions: Map[String, String]): Boolean = { insertIntoByNameMissingColumn(query, targetAttrs, deltaTable) val userSpecifiedNames = if (session.sessionState.conf.caseSensitiveAnalysis) { query.output.map(a => (a.name, a)).toMap } else { CaseInsensitiveMap(query.output.map(a => (a.name, a)).toMap) } val specifiedTargetAttrs = targetAttrs.filter(col => userSpecifiedNames.contains(col.name)) !SchemaUtils.isReadCompatible( specifiedTargetAttrs.toStructType.asNullable, query.output.toStructType, typeWideningMode = getTypeWideningMode(deltaTable, writeOptions) ) } // Get cast operation for the level of strictness in the schema a user asked for private def getCastFunction: CastFunction = { val timeZone = conf.sessionLocalTimeZone conf.storeAssignmentPolicy match { case SQLConf.StoreAssignmentPolicy.LEGACY => (input: Expression, dt: DataType, _) => Cast(input, dt, Option(timeZone), ansiEnabled = false) case SQLConf.StoreAssignmentPolicy.ANSI => (input: Expression, dt: DataType, name: String) => { val cast = Cast(input, dt, Option(timeZone), ansiEnabled = true) cast.setTagValue(Cast.BY_TABLE_INSERTION, ()) TableOutputResolver.checkCastOverflowInTableInsert(cast, name) } case SQLConf.StoreAssignmentPolicy.STRICT => (input: Expression, dt: DataType, _) => UpCast(input, dt) } } /** * Recursively casts struct data types in case the source/target type differs. */ private def addCastsToStructs( tableName: String, parent: NamedExpression, source: StructType, target: StructType, typeWideningMode: TypeWideningMode): NamedExpression = { if (source.length < target.length) { throw DeltaErrors.notEnoughColumnsInInsert( tableName, source.length, target.length, Some(parent.qualifiedName)) } // Extracts the field at a given index in the target schema. Only matches if the index is valid. object TargetIndex { def unapply(index: Int): Option[StructField] = target.lift(index) } val fields = source.zipWithIndex.map { case (StructField(name, nested: StructType, _, metadata), i @ TargetIndex(targetField)) => targetField.dataType match { case t: StructType => val subField = Alias(GetStructField(parent, i, Option(name)), targetField.name)( explicitMetadata = Option(metadata)) addCastsToStructs(tableName, subField, nested, t, typeWideningMode) case o => val field = parent.qualifiedName + "." + name val targetName = parent.qualifiedName + "." + targetField.name throw DeltaErrors.cannotInsertIntoColumn(tableName, field, targetName, o.simpleString) } case (StructField(name, sourceType: AtomicType, _, _), i @ TargetIndex(StructField(targetName, targetType: AtomicType, _, targetMetadata))) if typeWideningMode.shouldWidenTo(fromType = targetType, toType = sourceType) => Alias( GetStructField(parent, i, Option(name)), targetName)(explicitMetadata = Option(targetMetadata)) case (sourceField, i @ TargetIndex(targetField)) => Alias( getCastFunction(GetStructField(parent, i, Option(sourceField.name)), targetField.dataType, targetField.name), targetField.name)(explicitMetadata = Option(targetField.metadata)) case (sourceField, i) => // This is a new column, so leave to schema evolution as is. Do not lose it's name so // wrap with an alias Alias( GetStructField(parent, i, Option(sourceField.name)), sourceField.name)(explicitMetadata = Option(sourceField.metadata)) } // Fix for null expansion caused by struct type cast by preserving NULL source structs. // // Problem: When inserting a struct column, if the source struct is NULL, the casting logic // will expand the NULL into a non-null struct with all fields set to NULL: // NULL -> struct(field1: null, field2: null, ..., newField: null) // // Expected: The target struct should remain NULL when the source struct is NULL: // NULL -> NULL // // Solution: Wrap the CreateStruct expression in an IF expression that preserves NULL: // IF(source_struct IS NULL, NULL, CreateStruct(...)) // // This is controlled by the DELTA_INSERT_PRESERVE_NULL_SOURCE_STRUCTS config. val createStructExpr = CreateStruct(fields) val wrappedWithNullPreservation = maybeWrapWithNullPreservationForInsert( sourceExpr = parent, createStructExpr = createStructExpr) Alias(wrappedWithNullPreservation, parent.name)( parent.exprId, parent.qualifier, Option(parent.metadata)) } private def addCastsToArrayStructs( tableName: String, parent: NamedExpression, source: StructType, target: StructType, sourceNullable: Boolean, typeWideningMode: TypeWideningMode): Expression = { val structConverter: (Expression, Expression) => Expression = (_, i) => addCastsToStructs( tableName, Alias(GetArrayItem(parent, i), i.toString)(), source, target, typeWideningMode) val transformLambdaFunc = { val elementVar = NamedLambdaVariable("elementVar", source, sourceNullable) val indexVar = NamedLambdaVariable("indexVar", IntegerType, false) LambdaFunction(structConverter(elementVar, indexVar), Seq(elementVar, indexVar)) } ArrayTransform(parent, transformLambdaFunc) } private def stripTempViewWrapper(plan: LogicalPlan): LogicalPlan = { DeltaViewHelper.stripTempView(plan, conf) } private def stripTempViewForMergeWrapper(plan: LogicalPlan): LogicalPlan = { DeltaViewHelper.stripTempViewForMerge(plan, conf) } /** * Recursively casts map data types in case the key/value type differs. */ private def addCastsToMaps( tableName: String, parent: NamedExpression, sourceMapType: MapType, targetMapType: MapType, typeWideningMode: TypeWideningMode): Expression = { val transformedKeys = if (sourceMapType.keyType != targetMapType.keyType) { // Create a transformation for the keys ArrayTransform(MapKeys(parent), { val key = NamedLambdaVariable( "key", sourceMapType.keyType, nullable = false) val keyAttr = AttributeReference( "key", targetMapType.keyType, nullable = false)() val castedKey = addCastToColumn( key, keyAttr, tableName, typeWideningMode ) LambdaFunction(castedKey, Seq(key)) }) } else { MapKeys(parent) } val transformedValues = if (sourceMapType.valueType != targetMapType.valueType) { // Create a transformation for the values ArrayTransform(MapValues(parent), { val value = NamedLambdaVariable( "value", sourceMapType.valueType, sourceMapType.valueContainsNull) val valueAttr = AttributeReference( "value", targetMapType.valueType, sourceMapType.valueContainsNull)() val castedValue = addCastToColumn( value, valueAttr, tableName, typeWideningMode ) LambdaFunction(castedValue, Seq(value)) }) } else { MapValues(parent) } // Create new map from transformed keys and values MapFromArrays(transformedKeys, transformedValues) } /** * Verify the input plan for a SINGLE streaming query with the following: * 1. Schema location must be under checkpoint location, if not lifted by flag * 2. No two duplicating delta source can share the same schema location */ private def verifyDeltaSourceSchemaLocation( inputQuery: LogicalPlan, checkpointLocation: String): Unit = { // Maps StreamingRelation to schema location, similar to how MicroBatchExecution converts // StreamingRelation to StreamingExecutionRelation. val schemaLocationMap = mutable.Map[StreamingRelation, String]() val allowSchemaLocationOutsideOfCheckpoint = session.sessionState.conf.getConf( DeltaSQLConf.DELTA_STREAMING_ALLOW_SCHEMA_LOCATION_OUTSIDE_CHECKPOINT_LOCATION) inputQuery.foreach { case streamingRelation @ StreamingRelation(dataSourceV1, sourceName, _) if DeltaSourceUtils.isDeltaDataSourceName(sourceName) => DeltaDataSource.extractSchemaTrackingLocationConfig( session, dataSourceV1.options ).foreach { rootSchemaTrackingLocation => assert(dataSourceV1.options.contains("path"), "Path for Delta table must be defined") val tableId = dataSourceV1.options("path").replace(":", "").replace("/", "_") val sourceIdOpt = dataSourceV1.options.get(DeltaOptions.STREAMING_SOURCE_TRACKING_ID) val schemaTrackingLocation = DeltaSourceMetadataTrackingLog.fullMetadataTrackingLocation( rootSchemaTrackingLocation, tableId, sourceIdOpt) // Make sure schema location is under checkpoint if (!allowSchemaLocationOutsideOfCheckpoint) { assertSchemaTrackingLocationUnderCheckpoint( checkpointLocation, schemaTrackingLocation ) } // Save schema location for this streaming relation schemaLocationMap.put(streamingRelation, schemaTrackingLocation.stripSuffix("/")) } case _ => } // Now verify all schema locations are distinct val conflictSchemaOpt = schemaLocationMap .keys .groupBy { rel => schemaLocationMap(rel) } .find(_._2.size > 1) conflictSchemaOpt.foreach { case (schemaLocation, relations) => val ds = relations.head.dataSource // Pick one source that has conflict to make it more actionable for the user val oneTableWithConflict = ds.catalogTable .map(_.identifier.toString) .getOrElse { // `path` must exist CaseInsensitiveMap(ds.options).get("path").get } throw DeltaErrors.sourcesWithConflictingSchemaTrackingLocation( schemaLocation, oneTableWithConflict) } } /** * Check and assert whether the schema tracking location is under the checkpoint location. * * Visible for testing. */ private[delta] def assertSchemaTrackingLocationUnderCheckpoint( checkpointLocation: String, schemaTrackingLocation: String): Unit = { val checkpointPath = new Path(checkpointLocation) // scalastyle:off deltahadoopconfiguration val checkpointFs = checkpointPath.getFileSystem(session.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration val qualifiedCheckpointPath = checkpointFs.makeQualified(checkpointPath) val qualifiedSchemaTrackingLocationPath = try { checkpointFs.makeQualified(new Path(schemaTrackingLocation)) } catch { case NonFatal(e) => // This can happen when the file system for the checkpoint location is completely different // from that of the schema tracking location. logWarning("Failed to make a qualified path for schema tracking location", e) throw DeltaErrors.schemaTrackingLocationNotUnderCheckpointLocation( schemaTrackingLocation, checkpointLocation) } // If we couldn't qualify the schema location or after relativization, the result is still an // absolute path, we know the schema location is not under checkpoint. if (qualifiedCheckpointPath.toUri.relativize( qualifiedSchemaTrackingLocationPath.toUri).isAbsolute) { throw DeltaErrors.schemaTrackingLocationNotUnderCheckpointLocation( schemaTrackingLocation, checkpointLocation) } } object EligibleCreateTableLikeCommand { def unapply(arg: LogicalPlan): Option[(CreateTableLikeCommand, CatalogTable)] = arg match { case c: CreateTableLikeCommand => val src = session.sessionState.catalog.getTempViewOrPermanentTableMetadata(c.sourceTable) if (src.provider.contains("delta") || c.provider.exists(DeltaSourceUtils.isDeltaDataSourceName)) { Some(c, src) } else { None } case _ => None } } } /** Matchers for dealing with a Delta table. */ object DeltaRelation extends DeltaLogging { val KEEP_AS_V2_RELATION_TAG = new TreeNodeTag[Unit]("__keep_as_v2_relation") def unapply(plan: LogicalPlan): Option[LogicalRelation] = plan match { case dsv2 @ DataSourceV2RelationShim(d: DeltaTableV2, _, _, _, options) => Some(fromV2Relation(d, dsv2.asInstanceOf[DataSourceV2Relation], options)) case lr @ DeltaTable(_) => Some(lr) case _ => None } def fromV2Relation( d: DeltaTableV2, v2Relation: DataSourceV2Relation, options: CaseInsensitiveStringMap): LogicalRelation = { recordFrameProfile("DeltaAnalysis", "fromV2Relation") { val relation = d.withOptions(options.asScala.toMap).toBaseRelation val output = if (CDCReader.isCDCRead(options)) { // Handles cdc for the spark.read.options().table() code path // Mapping needed for references to the table's columns coming from Spark Connect. val newOutput = toAttributes(relation.schema) newOutput.map { a => val existingReference = v2Relation.output .find(e => e.name == a.name && e.dataType == a.dataType && e.nullable == a.nullable) existingReference.map { e => e.copy(metadata = a.metadata)(exprId = e.exprId, qualifier = e.qualifier) }.getOrElse(a) } } else { v2Relation.output } LogicalRelation(relation, output, d.ttSafeCatalogTable, isStreaming = false, stream = None) } } } object AppendDelta { def unapply(a: AppendData): Option[(DataSourceV2Relation, DeltaTableV2)] = { if (a.query.resolved) { a.table match { case r: DataSourceV2Relation if r.table.isInstanceOf[DeltaTableV2] => Some((r, r.table.asInstanceOf[DeltaTableV2])) case _ => None } } else { None } } } object OverwriteDelta { def unapply(o: OverwriteByExpression): Option[(DataSourceV2Relation, DeltaTableV2)] = { if (o.query.resolved) { o.table match { case r: DataSourceV2Relation if r.table.isInstanceOf[DeltaTableV2] => Some((r, r.table.asInstanceOf[DeltaTableV2])) case _ => None } } else { None } } } object DynamicPartitionOverwriteDelta { def unapply(o: OverwritePartitionsDynamic): Option[(DataSourceV2Relation, DeltaTableV2)] = { if (o.query.resolved) { o.table match { case r: DataSourceV2Relation if r.table.isInstanceOf[DeltaTableV2] => Some((r, r.table.asInstanceOf[DeltaTableV2])) case _ => None } } else { None } } } /** * A `RunnableCommand` that will execute dynamic partition overwrite using [[WriteIntoDelta]]. * * This is a workaround of Spark not supporting V1 fallback for dynamic partition overwrite. * Note the following details: * - Extends `V2WriteCommmand` so that Spark can transform this plan in the same as other * commands like `AppendData`. * - Exposes the query as a child so that the Spark optimizer can optimize it. */ case class DeltaDynamicPartitionOverwriteCommand( table: NamedRelation, deltaTable: DeltaTableV2, query: LogicalPlan, writeOptions: Map[String, String], isByName: Boolean, analyzedQuery: Option[LogicalPlan] = None) extends RunnableCommand with V2WriteCommand { override def child: LogicalPlan = query override def withNewQuery(newQuery: LogicalPlan): DeltaDynamicPartitionOverwriteCommand = { copy(query = newQuery) } override def withNewTable(newTable: NamedRelation): DeltaDynamicPartitionOverwriteCommand = { copy(table = newTable) } override def storeAnalyzedQuery(): Command = copy(analyzedQuery = Some(query)) override protected def withNewChildInternal( newChild: LogicalPlan): DeltaDynamicPartitionOverwriteCommand = copy(query = newChild) override def run(sparkSession: SparkSession): Seq[Row] = { val deltaOptions = new DeltaOptions( CaseInsensitiveMap[String]( deltaTable.options ++ writeOptions ++ Seq(DeltaOptions.PARTITION_OVERWRITE_MODE_OPTION -> DeltaOptions.PARTITION_OVERWRITE_MODE_DYNAMIC)), sparkSession.sessionState.conf) // TODO: The configuration can be fetched directly from WriteIntoDelta's txn. Don't pass // in the default snapshot's metadata config here. WriteIntoDelta( deltaTable.deltaLog, SaveMode.Overwrite, deltaOptions, partitionColumns = Nil, deltaTable.deltaLog.unsafeVolatileSnapshot.metadata.configuration, DataFrameUtils.ofRows(sparkSession, query), deltaTable.catalogTable ).run(sparkSession) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaColumnMapping.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.{Locale, UUID} import scala.collection.mutable import org.apache.spark.sql.delta.RowId.RowIdMetadataStructField import org.apache.spark.sql.delta.actions.{Metadata, Protocol} import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.{SchemaMergingUtils, SchemaUtils} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.json4s.DefaultFormats import org.json4s.jackson.JsonMethods._ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, QuotingUtils} import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{ArrayType, DataType, MapType, Metadata => SparkMetadata, MetadataBuilder, StructField, StructType} /** * Information regarding a single dropped column. * @param fieldPath The logical path of the dropped column. */ private[delta] case class DroppedColumn(fieldPath: Seq[String]) /** * Information regarding a single renamed column. * @param fromFieldPath The logical path of the column before the rename. * @param toFieldPath The logical path of the column after the rename. */ private[delta] case class RenamedColumn(fromFieldPath: Seq[String], toFieldPath: Seq[String]) trait DeltaColumnMappingBase extends DeltaLogging { val PARQUET_FIELD_ID_METADATA_KEY = "parquet.field.id" val PARQUET_FIELD_NESTED_IDS_METADATA_KEY = "parquet.field.nested.ids" val COLUMN_MAPPING_METADATA_PREFIX = "delta.columnMapping." val COLUMN_MAPPING_METADATA_ID_KEY = COLUMN_MAPPING_METADATA_PREFIX + "id" val COLUMN_MAPPING_PHYSICAL_NAME_KEY = COLUMN_MAPPING_METADATA_PREFIX + "physicalName" val COLUMN_MAPPING_METADATA_NESTED_IDS_KEY = COLUMN_MAPPING_METADATA_PREFIX + "nested.ids" val PARQUET_LIST_ELEMENT_FIELD_NAME = "element" val PARQUET_MAP_KEY_FIELD_NAME = "key" val PARQUET_MAP_VALUE_FIELD_NAME = "value" /** * The list of column mapping metadata for each column in the schema. */ val COLUMN_MAPPING_METADATA_KEYS: Set[String] = Set( COLUMN_MAPPING_METADATA_ID_KEY, COLUMN_MAPPING_PHYSICAL_NAME_KEY, COLUMN_MAPPING_METADATA_NESTED_IDS_KEY, PARQUET_FIELD_ID_METADATA_KEY, PARQUET_FIELD_NESTED_IDS_METADATA_KEY ) /** * This list of internal columns (and only this list) is allowed to have missing * column mapping metadata such as field id and physical name because * they might not be present in user's table schema. * * These fields, if materialized to parquet, will always be matched by their display name in the * downstream parquet reader even under column mapping modes. * * For future developers who want to utilize additional internal columns without generating * column mapping metadata, please add them here. * * This list is case-insensitive. */ protected val DELTA_INTERNAL_COLUMNS: Set[String] = (CDCReader.CDC_COLUMNS_IN_DATA ++ Seq( CDCReader.CDC_COMMIT_VERSION, CDCReader.CDC_COMMIT_TIMESTAMP, /** * Whenever `_metadata` column is selected, Spark adds the format generated metadata * columns to `ParquetFileFormat`'s required output schema. Column `_metadata` contains * constant value subfields metadata such as `file_path` and format specific custom metadata * subfields such as `row_index` in Parquet. Spark creates the file format object with * data schema plus additional custom metadata columns required from file format to fill up * the `_metadata` column. */ ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME, DeltaParquetFileFormat.IS_ROW_DELETED_COLUMN_NAME, DeltaParquetFileFormat.ROW_INDEX_COLUMN_NAME) ).map(_.toLowerCase(Locale.ROOT)).toSet val supportedModes: Set[DeltaColumnMappingMode] = Set(IdMapping, NoMapping, NameMapping) def isInternalField(field: StructField): Boolean = DELTA_INTERNAL_COLUMNS.contains(field.name.toLowerCase(Locale.ROOT)) || RowIdMetadataStructField.isRowIdColumn(field) || RowCommitVersion.MetadataStructField.isRowCommitVersionColumn(field) /** * Allow NameMapping -> NoMapping transition behind a feature flag. * Otherwise only NoMapping -> NameMapping is allowed. */ private def allowMappingModeChange( oldMode: DeltaColumnMappingMode, newMode: DeltaColumnMappingMode): Boolean = { val removalAllowed = SparkSession.getActiveSession .exists(_.conf.get(DeltaSQLConf.ALLOW_COLUMN_MAPPING_REMOVAL)) // No change. (oldMode == newMode) || // Downgrade allowed with a flag. (removalAllowed && (oldMode != NoMapping && newMode == NoMapping)) || // Upgrade always allowed. (oldMode == NoMapping && newMode == NameMapping) } def isColumnMappingUpgrade( oldMode: DeltaColumnMappingMode, newMode: DeltaColumnMappingMode): Boolean = { oldMode == NoMapping && newMode != NoMapping } /** * If the table is already on the column mapping protocol, we block: * - changing column mapping config * otherwise, we block * - upgrading to the column mapping Protocol through configurations */ def verifyAndUpdateMetadataChange( spark: SparkSession, deltaLog: DeltaLog, oldProtocol: Protocol, oldMetadata: Metadata, newMetadata: Metadata, isCreatingNewTable: Boolean, isOverwriteSchema: Boolean): Metadata = { // field in new metadata should have been dropped val oldMappingMode = oldMetadata.columnMappingMode val newMappingMode = newMetadata.columnMappingMode if (!supportedModes.contains(newMappingMode)) { throw DeltaErrors.unsupportedColumnMappingMode(newMappingMode.name) } val isChangingModeOnExistingTable = oldMappingMode != newMappingMode && !isCreatingNewTable if (isChangingModeOnExistingTable && !allowMappingModeChange(oldMappingMode, newMappingMode)) { throw DeltaErrors.changeColumnMappingModeNotSupported( oldMappingMode.name, newMappingMode.name) } var updatedMetadata = newMetadata // If column mapping is disabled, we need to strip any column mapping metadata from the schema, // because Delta code will use them even when column mapping is not enabled. However, we cannot // strip column mapping metadata that already exist in the schema, because this would break // the table. if (newMappingMode == NoMapping && schemaHasColumnMappingMetadata(newMetadata.schema)) { val addsColumnMappingMetadata = !schemaHasColumnMappingMetadata(oldMetadata.schema) if (addsColumnMappingMetadata && spark.conf.get(DeltaSQLConf.DELTA_COLUMN_MAPPING_STRIP_METADATA)) { recordDeltaEvent(deltaLog, opType = "delta.columnMapping.stripMetadata") val strippedSchema = dropColumnMappingMetadata(newMetadata.schema) updatedMetadata = newMetadata.copy(schemaString = strippedSchema.json) } else { recordDeltaEvent( deltaLog, opType = "delta.columnMapping.updateSchema.metadataPresentButFeatureDisabled", data = Map( "addsColumnMappingMetadata" -> addsColumnMappingMetadata.toString, "isCreatingNewTable" -> isCreatingNewTable.toString, "isOverwriteSchema" -> isOverwriteSchema.toString) ) } } // If column mapping was disabled, but there was already column mapping in the schema, it is // a result of a bug in the previous version of Delta. This should no longer happen with the // stripping done above. For existing tables with this issue, we should not allow enabling // column mapping, to prevent further corruption. if (spark.conf.get(DeltaSQLConf. DELTA_COLUMN_MAPPING_DISALLOW_ENABLING_WHEN_METADATA_ALREADY_EXISTS)) { if (oldMappingMode == NoMapping && newMappingMode != NoMapping && schemaHasColumnMappingMetadata(oldMetadata.schema)) { throw DeltaErrors.enablingColumnMappingDisallowedWhenColumnMappingMetadataAlreadyExists() } } updatedMetadata = updateColumnMappingMetadata( oldMetadata, updatedMetadata, isChangingModeOnExistingTable, isOverwriteSchema) // record column mapping table creation/upgrade if (newMappingMode != NoMapping) { if (isCreatingNewTable) { recordDeltaEvent(deltaLog, "delta.columnMapping.createTable") } else if (oldMappingMode != newMappingMode) { recordDeltaEvent(deltaLog, "delta.columnMapping.upgradeTable") } } updatedMetadata } def hasColumnId(field: StructField): Boolean = field.metadata.contains(COLUMN_MAPPING_METADATA_ID_KEY) def getColumnId(field: StructField): Int = field.metadata.getLong(COLUMN_MAPPING_METADATA_ID_KEY).toInt def hasNestedColumnIds(field: StructField): Boolean = field.metadata.contains(COLUMN_MAPPING_METADATA_NESTED_IDS_KEY) def getNestedColumnIds(field: StructField): SparkMetadata = field.metadata.getMetadata(COLUMN_MAPPING_METADATA_NESTED_IDS_KEY) def getNestedColumnIdsAsLong(field: StructField): Iterable[Long] = { val nestedColumnMetadata = getNestedColumnIds(field) metadataToMap[Map[String, Long]](nestedColumnMetadata).values } private def metadataToMap[T <: Map[_, _]](metadata: SparkMetadata)(implicit m: Manifest[T]): T = { implicit val formats: DefaultFormats.type = DefaultFormats parse(metadata.json).extract[T] } def hasPhysicalName(field: StructField): Boolean = field.metadata.contains(COLUMN_MAPPING_PHYSICAL_NAME_KEY) /** * Gets the required column metadata for each column based on the column mapping mode. */ def getColumnMappingMetadata(field: StructField, mode: DeltaColumnMappingMode): SparkMetadata = { mode match { case NoMapping => // drop all column mapping related fields new MetadataBuilder() .withMetadata(field.metadata) .remove(COLUMN_MAPPING_METADATA_ID_KEY) .remove(COLUMN_MAPPING_METADATA_NESTED_IDS_KEY) .remove(PARQUET_FIELD_ID_METADATA_KEY) .remove(PARQUET_FIELD_NESTED_IDS_METADATA_KEY) .remove(COLUMN_MAPPING_PHYSICAL_NAME_KEY) .build() case IdMapping | NameMapping => if (!hasColumnId(field)) { throw DeltaErrors.missingColumnId(mode, field.name) } if (!hasPhysicalName(field)) { throw DeltaErrors.missingPhysicalName(mode, field.name) } // Delta spec requires writer to always write field_id in parquet schema for column mapping // Reader strips PARQUET_FIELD_ID_METADATA_KEY in // DeltaParquetFileFormat:prepareSchemaForRead val builder = new MetadataBuilder() .withMetadata(field.metadata) .putLong(PARQUET_FIELD_ID_METADATA_KEY, getColumnId(field)) // Nested field IDs for the 'element' and 'key'/'value' fields of Arrays // and Maps are written when Uniform with IcebergCompatV2 is enabled on a table. if (hasNestedColumnIds(field)) { builder.putMetadata(PARQUET_FIELD_NESTED_IDS_METADATA_KEY, getNestedColumnIds(field)) } builder.build() case mode => throw DeltaErrors.unsupportedColumnMappingMode(mode.name) } } /** Recursively renames columns in the given schema with their physical schema. */ def renameColumns(schema: StructType): StructType = { SchemaMergingUtils.transformColumns(schema) { (_, field, _) => field.copy(name = getPhysicalName(field)) } } def assignPhysicalName(field: StructField, physicalName: String): StructField = { field.copy(metadata = new MetadataBuilder() .withMetadata(field.metadata) .putString(COLUMN_MAPPING_PHYSICAL_NAME_KEY, physicalName) .build()) } def assignPhysicalNames(schema: StructType, reuseLogicalName: Boolean = false): StructType = { SchemaMergingUtils.transformColumns(schema) { (_, field, _) => if (hasPhysicalName(field)) field else { if (reuseLogicalName) assignPhysicalName(field, field.name) else assignPhysicalName(field, generatePhysicalName) } } } /** * Set physical name based on field path, skip if field path not found in the map. All comparisons * are case-insensitive. */ def setPhysicalNames( schema: StructType, fieldPathToPhysicalName: Map[Seq[String], String]): StructType = { if (fieldPathToPhysicalName.isEmpty) { schema } else { val lowerCasedFieldPathToPhysicalNameMap = fieldPathToPhysicalName.map { case (k, v) => k.map(_.toLowerCase(Locale.ROOT)) -> v } SchemaMergingUtils.transformColumns(schema) { (parent, field, _) => // Column comparison is case-insensitive. val path = (parent :+ field.name).map(_.toLowerCase(Locale.ROOT)) if (lowerCasedFieldPathToPhysicalNameMap.contains(path)) { assignPhysicalName(field, lowerCasedFieldPathToPhysicalNameMap(path)) } else { field } } } } def generatePhysicalName: String = "col-" + UUID.randomUUID() def getPhysicalName(field: StructField): String = { if (field.metadata.contains(COLUMN_MAPPING_PHYSICAL_NAME_KEY)) { field.metadata.getString(COLUMN_MAPPING_PHYSICAL_NAME_KEY) } else { field.name } } private def updateColumnMappingMetadata( oldMetadata: Metadata, newMetadata: Metadata, isChangingModeOnExistingTable: Boolean, isOverwritingSchema: Boolean): Metadata = { val newMappingMode = DeltaConfigs.COLUMN_MAPPING_MODE.fromMetaData(newMetadata) newMappingMode match { case IdMapping | NameMapping => assignColumnIdAndPhysicalName( newMetadata, oldMetadata, isChangingModeOnExistingTable, isOverwritingSchema) case NoMapping => newMetadata case mode => throw DeltaErrors.unsupportedColumnMappingMode(mode.name) } } def findMaxColumnId(schema: StructType): Long = { var maxId: Long = 0 SchemaMergingUtils.transformColumns(schema)((_, f, _) => { if (hasColumnId(f)) { maxId = maxId max getColumnId(f) if (hasNestedColumnIds(f)) { val nestedIds = getNestedColumnIdsAsLong(f) maxId = maxId max (if (nestedIds.nonEmpty) nestedIds.max else 0) } } f }) maxId } /** * Verify the metadata for valid column mapping metadata assignment. This is triggered for every * commit as a last defense. * * 1. Ensure column mapping metadata is set for the appropriate mode * 2. Ensure no duplicate column id/physical names set * 3. Ensure max column id is in a good state (set, and greater than all field ids available) */ def checkColumnIdAndPhysicalNameAssignments(metadata: Metadata): Unit = { val schema = metadata.schema val mode = metadata.columnMappingMode // physical name/column id -> full field path val columnIds = mutable.Set[Int]() val physicalNames = mutable.Set[String]() // use id mapping to keep all column mapping metadata // this method checks for missing physical name & column id already val physicalSchema = createPhysicalSchema(schema, schema, IdMapping, checkSupportedMode = false) // Check id / physical name duplication SchemaMergingUtils.transformColumns(physicalSchema) ((parentPhysicalPath, field, _) => { // field.name is now physical name // We also need to apply backticks to column paths with dots in them to prevent a possible // false alarm in which a column `a.b` is duplicated with `a`.`b` val curFullPhysicalPath = UnresolvedAttribute(parentPhysicalPath :+ field.name).name val columnId = getColumnId(field) if (columnIds.contains(columnId)) { throw DeltaErrors.duplicatedColumnId(mode, columnId, schema) } columnIds.add(columnId) // We should check duplication by full physical name path, because nested fields // such as `a.b.c` shouldn't conflict with `x.y.c` due to same column name. if (physicalNames.contains(curFullPhysicalPath)) { throw DeltaErrors.duplicatedPhysicalName(mode, curFullPhysicalPath, schema) } physicalNames.add(curFullPhysicalPath) field }) // Check assignment of the max id property if (SQLConf.get.getConf(DeltaSQLConf.DELTA_COLUMN_MAPPING_CHECK_MAX_COLUMN_ID)) { if (!metadata.configuration.contains(DeltaConfigs.COLUMN_MAPPING_MAX_ID.key)) { throw DeltaErrors.maxColumnIdNotSet } val fieldMaxId = DeltaColumnMapping.findMaxColumnId(schema) if (metadata.columnMappingMaxId < DeltaColumnMapping.findMaxColumnId(schema)) { throw DeltaErrors.maxColumnIdNotSetCorrectly(metadata.columnMappingMaxId, fieldMaxId) } } } /** * For each column/field in a Metadata's schema, assign id using the current maximum id * as the basis and increment from there, and assign physical name using UUID * @param newMetadata The new metadata to assign Ids and physical names * @param oldMetadata The old metadata * @param isChangingModeOnExistingTable whether this is part of a commit that changes the * mapping mode on a existing table * @return new metadata with Ids and physical names assigned */ def assignColumnIdAndPhysicalName( newMetadata: Metadata, oldMetadata: Metadata, isChangingModeOnExistingTable: Boolean, isOverwritingSchema: Boolean): Metadata = { val rawSchema = newMetadata.schema var maxId = DeltaConfigs.COLUMN_MAPPING_MAX_ID.fromMetaData(newMetadata) max DeltaConfigs.COLUMN_MAPPING_MAX_ID.fromMetaData(oldMetadata) max findMaxColumnId(rawSchema) val startId = maxId val newSchema = SchemaMergingUtils.transformColumns(rawSchema)((path, field, _) => { val builder = new MetadataBuilder().withMetadata(field.metadata) lazy val fullName = path :+ field.name lazy val existingFieldOpt = SchemaUtils.findNestedFieldIgnoreCase( oldMetadata.schema, fullName, includeCollections = true) lazy val canReuseColumnMappingMetadataDuringOverwrite = { val canReuse = isOverwritingSchema && SparkSession.getActiveSession.exists( _.conf.get(DeltaSQLConf.REUSE_COLUMN_MAPPING_METADATA_DURING_OVERWRITE)) && existingFieldOpt.exists { existingField => // Ensure data type & nullability are compatible DataType.equalsIgnoreCompatibleNullability( from = existingField.dataType, to = field.dataType ) } if (canReuse) { require(!isChangingModeOnExistingTable, "Cannot change column mapping mode while overwriting the table") assert(hasColumnId(existingFieldOpt.get) && hasPhysicalName(existingFieldOpt.get)) } canReuse } if (!hasColumnId(field)) { val columnId = if (canReuseColumnMappingMetadataDuringOverwrite) { getColumnId(existingFieldOpt.get) } else { maxId += 1 maxId } builder.putLong(COLUMN_MAPPING_METADATA_ID_KEY, columnId) } if (!hasPhysicalName(field)) { val physicalName = if (isChangingModeOnExistingTable) { if (existingFieldOpt.isEmpty) { if (oldMetadata.schema.isEmpty) { // We should relax the check for tables that have both an empty schema // and no data. Assumption: no schema => no data generatePhysicalName } else throw DeltaErrors.schemaChangeDuringMappingModeChangeNotSupported( oldMetadata.schema, newMetadata.schema) } else { // When changing from NoMapping to NameMapping mode, we directly use old display names // as physical names. This is by design: 1) We don't need to rewrite the // existing Parquet files, and 2) display names in no-mapping mode have all the // properties required for physical names: unique, stable and compliant with Parquet // column naming restrictions. existingFieldOpt.get.name } } else if (canReuseColumnMappingMetadataDuringOverwrite) { // Copy the physical name metadata over from the existing field if possible getPhysicalName(existingFieldOpt.get) } else { generatePhysicalName } builder.putString(COLUMN_MAPPING_PHYSICAL_NAME_KEY, physicalName) } field.copy(metadata = builder.build()) }) // Starting from IcebergCompatV2, we require writing field-id for List/Map nested fields val (finalSchema, newMaxId) = if (IcebergCompat.isGeqEnabled(newMetadata, 2)) { rewriteFieldIdsForIceberg(newSchema, maxId) } else { (newSchema, maxId) } newMetadata.copy( schemaString = finalSchema.json, configuration = newMetadata.configuration ++ Map(DeltaConfigs.COLUMN_MAPPING_MAX_ID.key -> newMaxId.toString) ) } def dropColumnMappingMetadata(schema: StructType): StructType = { SchemaMergingUtils.transformColumns(schema) { (_, field, _) => var strippedMetadataBuilder = new MetadataBuilder().withMetadata(field.metadata) for (key <- COLUMN_MAPPING_METADATA_KEYS) { strippedMetadataBuilder = strippedMetadataBuilder.remove(key) } val strippedMetadata = strippedMetadataBuilder.build() field.copy(metadata = strippedMetadata) } } def filterColumnMappingProperties(properties: Map[String, String]): Map[String, String] = { properties.filterKeys(_ != DeltaConfigs.COLUMN_MAPPING_MAX_ID.key).toMap } // Verify the values of internal column mapping properties are the same in two sets of config // ONLY if the config is present in both sets of properties. def verifyInternalProperties(one: Map[String, String], two: Map[String, String]): Boolean = { val key = DeltaConfigs.COLUMN_MAPPING_MAX_ID.key one.get(key).forall(value => value == two.getOrElse(key, value)) } /** * Create a physical schema for the given schema using the Delta table schema as a reference. * * @param schema the given logical schema (potentially without any metadata) * @param referenceSchema the schema from the delta log, which has all the metadata * @param columnMappingMode column mapping mode of the delta table, which determines which * metadata to fill in * @param checkSupportedMode whether we should check of the column mapping mode is supported */ def createPhysicalSchema( schema: StructType, referenceSchema: StructType, columnMappingMode: DeltaColumnMappingMode, checkSupportedMode: Boolean = true): StructType = { if (columnMappingMode == NoMapping) { return schema } // createPhysicalSchema is the narrow-waist for both read/write code path // so we could check for mode support here if (checkSupportedMode && !supportedModes.contains(columnMappingMode)) { throw DeltaErrors.unsupportedColumnMappingMode(columnMappingMode.name) } val referenceSchemaColumnMap: Map[String, StructField] = SchemaMergingUtils.explode(referenceSchema).map { case (path, field) => QuotingUtils.quoteNameParts(path).toLowerCase(Locale.ROOT) -> field }.toMap SchemaMergingUtils.transformColumns(schema) { (path, field, _) => val fullName = path :+ field.name val inSchema = referenceSchemaColumnMap.get(QuotingUtils.quoteNameParts(fullName).toLowerCase(Locale.ROOT)) inSchema.map { refField => val sparkMetadata = getColumnMappingMetadata(refField, columnMappingMode) field.copy(metadata = sparkMetadata, name = getPhysicalName(refField)) }.getOrElse { if (isInternalField(field)) { field } else { throw DeltaErrors.columnNotFound(fullName, referenceSchema) } } } } /** * Create a list of physical attributes for the given attributes using the table schema as a * reference. * * @param output the list of attributes (potentially without any metadata) * @param referenceSchema the table schema with all the metadata * @param columnMappingMode column mapping mode of the delta table, which determines which * metadata to fill in */ def createPhysicalAttributes( output: Seq[Attribute], referenceSchema: StructType, columnMappingMode: DeltaColumnMappingMode): Seq[Attribute] = { // Assign correct column mapping info to columns according to the schema val struct = createPhysicalSchema(output.toStructType, referenceSchema, columnMappingMode) output.zip(struct).map { case (attr, field) => attr.withDataType(field.dataType) // for recursive column names and metadata .withMetadata(field.metadata) .withName(field.name) } } /** * Returns a map of physicalNamePath -> field for the given `schema`, where * physicalNamePath is the [$parentPhysicalName, ..., $fieldPhysicalName] list of physical names * for every field (including nested) in the `schema`. * * Must be called after `checkColumnIdAndPhysicalNameAssignments`, so that we know the schema * is valid. */ def getPhysicalNameFieldMap(schema: StructType): Map[Seq[String], StructField] = { val physicalSchema = renameColumns(schema) val physicalSchemaFieldPaths = SchemaMergingUtils.explode(physicalSchema).map(_._1) val originalSchemaFields = SchemaMergingUtils.explode(schema).map(_._2) physicalSchemaFieldPaths.zip(originalSchemaFields).toMap } /** * Returns a map from the logical name paths to the physical name paths for the given schema. * The logical name path is the result of splitting a multi-part identifier, and the physical name * path is result of replacing all names in the logical name path with their physical names. */ def getLogicalNameToPhysicalNameMap(schema: StructType): Map[Seq[String], Seq[String]] = { val physicalSchema = renameColumns(schema) val logicalSchemaFieldPaths = SchemaMergingUtils.explode(schema).map(_._1) val physicalSchemaFieldPaths = SchemaMergingUtils.explode(physicalSchema).map(_._1) logicalSchemaFieldPaths.zip(physicalSchemaFieldPaths).toMap } /** * Returns a map from the physical name paths to the logical name paths for the given schema. * The logical name path is the result of splitting a multi-part identifier, and the physical name * path is result of replacing all names in the logical name path with their physical names. */ def getPhysicalNameToLogicalNameMap(schema: StructType): Map[Seq[String], Seq[String]] = { getLogicalNameToPhysicalNameMap(schema).map(_.swap) } /** * Returns true if Column Mapping mode is enabled and the newMetadata's schema, when compared to * the currentMetadata's schema, is indicative of a DROP COLUMN operation. * * We detect DROP COLUMNS by checking if any physical name in `currentSchema` is missing in * `newSchema`. */ def isDropColumnOperation( newSchema: StructType, currentSchema: StructType, isBothColumnMappingEnabled: Boolean): Boolean = { // We will need to compare the new schema's physical columns to the current schema's physical // columns. So, they both must have column mapping enabled. if (!isBothColumnMappingEnabled) { return false } val newPhysicalToLogicalMap = getPhysicalNameFieldMap(newSchema) val currentPhysicalToLogicalMap = getPhysicalNameFieldMap(currentSchema) // are any of the current physical names missing in the new schema? currentPhysicalToLogicalMap .keys .exists { k => !newPhysicalToLogicalMap.contains(k) } } /** * Collects the columns that were dropped between the new schema and the current schema. * @param newSchema The new schema after a potential drop. * @param currentSchema The current schema before the drop. * @return A sequence of column names that were dropped */ def collectDroppedColumns( newSchema: StructType, currentSchema: StructType): Seq[DroppedColumn] = { val newPhysicalToLogicalMap = getPhysicalNameToLogicalNameMap(newSchema) val currentPhysicalToLogicalMap = getPhysicalNameToLogicalNameMap(currentSchema) // are any of the current physical names missing in the new schema? currentPhysicalToLogicalMap .keySet .diff(newPhysicalToLogicalMap.keySet) .map { droppedPhysicalPath => DroppedColumn(currentPhysicalToLogicalMap(droppedPhysicalPath)) } .toSeq } /** * Returns true if Column Mapping mode is enabled and the newMetadata's schema, when compared to * the currentMetadata's schema, is indicative of a RENAME COLUMN operation. * * We detect RENAME COLUMNS by checking if any two columns with the same physical name have * different logical names */ def isRenameColumnOperation( newSchema: StructType, currentSchema: StructType, isBothColumnMappingEnabled: Boolean): Boolean = { // We will need to compare the new schema's physical columns to the current schema's physical // columns. So, they both must have column mapping enabled. if (!isBothColumnMappingEnabled) { return false } val newPhysicalToLogicalMap = getPhysicalNameFieldMap(newSchema) val currentPhysicalToLogicalMap = getPhysicalNameFieldMap(currentSchema) // do any two columns with the same physical name have different logical names? currentPhysicalToLogicalMap .exists { case (physicalPath, field) => newPhysicalToLogicalMap.get(physicalPath).exists(_.name != field.name) } } /** * Returns true if there is a column mapping schema change (drop/rename) or an incompatible * partition column change between the new and current schemas. */ def hasColMappingOrPartitionSchemaChange( newSchema: StructType, currentSchema: StructType, newPartitionColumns: Seq[String], oldPartitionColumns: Seq[String], isBothColumnMappingEnabled: Boolean): Boolean = { isDropColumnOperation(newSchema, currentSchema, isBothColumnMappingEnabled) || isRenameColumnOperation(newSchema, currentSchema, isBothColumnMappingEnabled) || !SchemaUtils.isPartitionCompatible(newPartitionColumns, oldPartitionColumns) } /** * Collects the column rename operations between the new schema and the current schema. * @param newSchema The new schema after a potential rename. * @param currentSchema The current schema before the rename. * @return A sequence of (oldName, newName) tuples representing the column before and after rename */ def collectRenamedColumns( newSchema: StructType, currentSchema: StructType): Seq[RenamedColumn] = { val newPhysicalToLogicalMap = getPhysicalNameToLogicalNameMap(newSchema) val currentPhysicalToLogicalMap = getPhysicalNameToLogicalNameMap(currentSchema) // do any two columns with the same physical name have different logical names? currentPhysicalToLogicalMap .flatMap { case (physicalPath, logicalPath) => newPhysicalToLogicalMap.get(physicalPath).flatMap { newLogicalPath => if (logicalPath.last != newLogicalPath.last) { Some(RenamedColumn(logicalPath, newLogicalPath)) } else None } }.toSet.toSeq } /** * Compare the old metadata's schema with new metadata's schema for column mapping schema changes. * Also check for repartition because we need to fail fast when repartition detected. * * newMetadata's snapshot version must be >= oldMetadata's snapshot version so we could reliably * detect the difference between ADD COLUMN and DROP COLUMN. * * As of now, `newMetadata` is column mapping read compatible with `oldMetadata` if * no rename column or drop column has happened in-between. */ def hasNoColumnMappingSchemaChanges(newMetadata: Metadata, oldMetadata: Metadata, allowUnsafeReadOnPartitionChanges: Boolean = false): Boolean = { def hasColMappingOrPartitionSchemaChangeByMetadata(newMetadata: Metadata, oldMetadata: Metadata): Boolean = { val isBothColumnMappingEnabled = newMetadata.columnMappingMode != NoMapping && oldMetadata.columnMappingMode != NoMapping hasColMappingOrPartitionSchemaChange( newMetadata.schema, oldMetadata.schema, // if allow unsafe row read for partition change, ignore the check if (allowUnsafeReadOnPartitionChanges) Seq.empty else newMetadata.partitionColumns, if (allowUnsafeReadOnPartitionChanges) Seq.empty else oldMetadata.partitionColumns, isBothColumnMappingEnabled) } val (oldMode, newMode) = (oldMetadata.columnMappingMode, newMetadata.columnMappingMode) if (oldMode != NoMapping && newMode != NoMapping) { require(oldMode == newMode, "changing mode is not supported") // Both changes are post column mapping enabled !hasColMappingOrPartitionSchemaChangeByMetadata(newMetadata, oldMetadata) } else if (oldMode == NoMapping && newMode != NoMapping) { // The old metadata does not have column mapping while the new metadata does, in this case // we assume an upgrade has happened in between. // So we manually construct a post-upgrade schema for the old metadata and compare that with // the new metadata, as the upgrade would use the logical name as the physical name, we could // easily capture any difference in the schema using the same is{Drop,Rename}ColumnOperation // utils. var upgradedMetadata = assignColumnIdAndPhysicalName( oldMetadata, oldMetadata, isChangingModeOnExistingTable = true, isOverwritingSchema = false ) // need to change to a column mapping mode too so the utils below can recognize upgradedMetadata = upgradedMetadata.copy( configuration = upgradedMetadata.configuration ++ Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> newMetadata.columnMappingMode.name) ) // use the same check !hasColMappingOrPartitionSchemaChangeByMetadata(newMetadata, upgradedMetadata) } else { // Prohibit reading across a downgrade. val isDowngrade = oldMode != NoMapping && newMode == NoMapping !isDowngrade } } /** * Adds the nested field IDs required by Iceberg. * * In parquet, list-type columns have a nested, implicitly defined [[element]] field and * map-type columns have implicitly defined [[key]] and [[value]] fields. By default, * Spark does not write field IDs for these fields in the parquet files. However, Iceberg * requires these *nested* field IDs to be present. This method rewrites the specified * Spark schema to add those nested field IDs. * * As list and map types are not [[StructField]]s themselves, nested field IDs are stored in * a map as part of the metadata of the *nearest* parent [[StructField]]. For example, consider * the following schema: * * col1 ARRAY(INT) * col2 MAP(INT, INT) * col3 STRUCT(a INT, b ARRAY(STRUCT(c INT, d MAP(INT, INT)))) * * col1 is a list and so requires one nested field ID for the [[element]] field in parquet. * This nested field ID will be stored in a map that is part of col1's [[StructField.metadata]]. * The same applies to the nested field IDs for col2's implicit [[key]] and [[value]] fields. * col3 itself is a Struct, consisting of an integer field and a list field named 'b'. The * nested field ID for the list of 'b' is stored in b's StructField metadata. Finally, the * list type itself is again a struct consisting of an integer field and a map field named 'd'. * The nested field IDs for the map of 'd' are stored in d's StructField metadata. * * @param schema The schema to which nested field IDs should be added * @param startId The first field ID to use for the nested field IDs */ def rewriteFieldIdsForIceberg(schema: StructType, startId: Long): (StructType, Long) = { var currFieldId = startId def initNestedIdsMetadata(field: StructField): MetadataBuilder = { if (hasNestedColumnIds(field)) { new MetadataBuilder().withMetadata(getNestedColumnIds(field)) } else { new MetadataBuilder() } } /* * Helper to add the next field ID to the specified [[MetadataBuilder]] under * the specified key. This method first checks whether this is an existing nested * field or a newly added nested field. New field IDs are only assigned to newly * added nested fields. */ def updateFieldId(metadata: MetadataBuilder, key: String): Unit = { if (!metadata.build().contains(key)) { currFieldId += 1 metadata.putLong(key, currFieldId) } } /* * Recursively adds nested field IDs for the passed data type in pre-order, * ensuring uniqueness of field IDs. * * @param dt The data type that should be transformed * @param nestedIds A MetadataBuilder that keeps track of the nested field ID * assignment. This metadata is added to the parent field. * @param path The current field path relative to the parent field */ def transform[E <: DataType](dt: E, nestedIds: MetadataBuilder, path: Seq[String]): E = { val newDt = dt match { case StructType(fields) => StructType(fields.map { field => val newNestedIds = initNestedIdsMetadata(field) val newDt = transform(field.dataType, newNestedIds, Seq(getPhysicalName(field))) val newFieldMetadata = new MetadataBuilder().withMetadata(field.metadata).putMetadata( COLUMN_MAPPING_METADATA_NESTED_IDS_KEY, newNestedIds.build()).build() field.copy(dataType = newDt, metadata = newFieldMetadata) }) case ArrayType(elementType, containsNull) => // update element type metadata and recurse into element type val elemPath = path :+ PARQUET_LIST_ELEMENT_FIELD_NAME updateFieldId(nestedIds, elemPath.mkString(".")) val elementDt = transform(elementType, nestedIds, elemPath) // return new array type with updated metadata ArrayType(elementDt, containsNull) case MapType(keyType, valType, valueContainsNull) => // update key type metadata and recurse into key type val keyPath = path :+ PARQUET_MAP_KEY_FIELD_NAME updateFieldId(nestedIds, keyPath.mkString(".")) val keyDt = transform(keyType, nestedIds, keyPath) // update value type metadata and recurse into value type val valPath = path :+ PARQUET_MAP_VALUE_FIELD_NAME updateFieldId(nestedIds, valPath.mkString(".")) val valDt = transform(valType, nestedIds, valPath) // return new map type with updated metadata MapType(keyDt, valDt, valueContainsNull) case other => other } newDt.asInstanceOf[E] } (transform(schema, new MetadataBuilder(), Seq.empty), currFieldId) } /** * Returns whether the schema contains any metadata reserved for column mapping. */ def schemaHasColumnMappingMetadata(schema: StructType): Boolean = { SchemaMergingUtils.explode(schema).exists { case (_, col) => COLUMN_MAPPING_METADATA_KEYS.exists(k => col.metadata.contains(k)) } } } object DeltaColumnMapping extends DeltaColumnMappingBase /** * A trait for Delta column mapping modes. */ sealed trait DeltaColumnMappingMode { def name: String } /** * No mapping mode uses a column's display name as its true identifier to * read and write data. * * This is the default mode and is the same mode as Delta always has been. */ case object NoMapping extends DeltaColumnMappingMode { val name = "none" } /** * Id Mapping uses column ID as the true identifier of a column. Column IDs are stored as * StructField metadata in the schema and will be used when reading and writing Parquet files. * The Parquet files in this mode will also have corresponding field Ids for each column in their * file schema. * * This mode is used for tables converted from Iceberg. */ case object IdMapping extends DeltaColumnMappingMode { val name = "id" } /** * Name Mapping uses the physical column name as the true identifier of a column. The physical name * is stored as part of StructField metadata in the schema and will be used when reading and writing * Parquet files. Even if id mapping can be used for reading the physical files, name mapping is * used for reading statistics and partition values in the DeltaLog. */ case object NameMapping extends DeltaColumnMappingMode { val name = "name" } object DeltaColumnMappingMode { def apply(columnMappingModeString: String): DeltaColumnMappingMode = { val columnMappingModeLowerCaseString = Option(columnMappingModeString) .map(_.toLowerCase(Locale.ROOT)) .getOrElse(throw DeltaErrors.unsupportedColumnMappingModeException(columnMappingModeString)) columnMappingModeLowerCaseString match { case NoMapping.name => NoMapping case IdMapping.name => IdMapping case NameMapping.name => NameMapping case mode => throw DeltaErrors.unsupportedColumnMappingMode(mode) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaCommitTag.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.actions.CommitInfo /** Marker trait for a commit tag used by delta. */ sealed trait DeltaCommitTag { /** Key to be used in the commit tags `Map[String, String]`. */ def key: String /** * Combine tags coming from multiple sub-jobs into a single tag according to the tags' * semantics. */ def merge(left: String, right: String): String } object DeltaCommitTag { trait TypedCommitTag[ValueT] extends DeltaCommitTag { /** * Combine tags coming from multiple sub-jobs into a single tag according to the tags' * semantics. */ def mergeTyped(left: ValueT, right: ValueT): ValueT override def merge(left: String, right: String): String = valueToString(mergeTyped(valueFromString(left), valueFromString(right))) /** * Combine tags coming from multiple sub-jobs into a single tag according to the tags' * semantics. * * This variant is used when adding a new typed value to a potentially existing value from a * `Map[, String]`. */ def mergeWithNewTypedValue(existingOpt: Option[String], newValue: ValueT): String = { existingOpt match { case Some(existing) => valueToString(mergeTyped(valueFromString(existing), newValue)) case None => valueToString(newValue) } } /** Deserialize a value for this tag from String. */ def valueFromString(s: String): ValueT /** Serialize a value for this tag to String. */ def valueToString(value: ValueT): String = value.toString def withValue(value: ValueT): TypedCommitTagPair[ValueT] = TypedCommitTagPair(this, value) } final case class TypedCommitTagPair[ValueT](tag: TypedCommitTag[ValueT], value: ValueT) { /** Produce a tuple for inserting into `Map[DeltaCommitTag, String]` instances. */ def stringValue: (DeltaCommitTag, String) = tag -> tag.valueToString(value) /** Produce a tuple for inserting into `Map[String, String]` instances. */ def stringPair: (String, String) = tag.key -> tag.valueToString(value) } /** Any [[DeltaCommitTag]] where `ValueT` is `Boolean`. */ trait BooleanCommitTag extends TypedCommitTag[Boolean] { override def valueFromString(value: String): Boolean = value.toBoolean } /** * Tag to indicate whether the operation preserved row tracking. If not set, it is assumed that * the operation did not preserve row tracking. */ case object PreservedRowTrackingTag extends BooleanCommitTag { override val key = "delta.rowTracking.preserved" override def mergeTyped(left: Boolean, right: Boolean): Boolean = left && right } /** * Tag to indicate whether the commit only does row tracking enablement in its metadata update. * Used to allow some concurrent txns not to fail on metadata update. */ case object RowTrackingEnablementOnlyTag extends BooleanCommitTag { override val key = "rowTrackingEnablementOnly" override def mergeTyped(left: Boolean, right: Boolean): Boolean = left && right } /** * Returns the tagKey value in the CommitInfo, if it exists. */ def getTagValueFromCommitInfo(commitInfo: Option[CommitInfo], tagKey: String): Option[String] = { commitInfo.flatMap { ci => ci.tags.flatMap(_.get(tagKey)) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.{HashMap, Locale} import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.actions.{Action, Metadata, Protocol, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.hooks.AutoCompactType import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.{DataSkippingReaderConf, StatisticsCollection} import org.apache.spark.sql.delta.util.{DeltaSqlParserUtils, JsonUtils} import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.util.{DateTimeConstants, IntervalUtils} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} import org.apache.spark.util.Utils case class DeltaConfig[T]( key: String, defaultValue: String, fromString: String => T, validationFunction: T => Boolean, helpMessage: String, editable: Boolean = true, alternateKeys: Seq[String] = Seq.empty) { /** * Recover the saved value of this configuration from `Metadata`. If undefined, fall back to * alternate keys, returning defaultValue if none matches. */ def fromMetaData(metadata: Metadata): T = { fromMap(metadata.configuration) } /** * Recover the saved value of this configuration from `Metadata`. If undefined, fall back to * alternate keys, returning `None` if none matches. */ protected[delta] def fromMetaDataOption(metadata: Metadata): Option[T] = { fromMapOption(metadata.configuration) } def fromMap(configs: Map[String, String]): T = { fromMapOption(configs).getOrElse(fromString(defaultValue)) } protected[delta] def fromMapOption(configs: Map[String, String]): Option[T] = { for (k <- key +: alternateKeys) { configs.get(k) match { case Some(value) => return Some(fromString(value)) case None => // keep looking } } None } /** Validate the setting for this configuration */ private def validate(value: String): Unit = { if (!editable) { throw DeltaErrors.cannotModifyTableProperty(key) } val onErrorMessage = s"$key $helpMessage" try { require(validationFunction(fromString(value)), onErrorMessage) } catch { case e: NumberFormatException => throw new IllegalArgumentException(onErrorMessage, e) } } /** * Validate this configuration and return the key - value pair to save into the metadata. */ def apply(value: String): (String, String) = { validate(value) key -> value } /** * SQL configuration to set for ensuring that all newly created tables have this table property. */ def defaultTablePropertyKey: String = DeltaConfigs.sqlConfPrefix + key.stripPrefix("delta.") } /** * Contains list of reservoir configs and validation checks. */ trait DeltaConfigsBase extends DeltaLogging { // Special properties stored in the Hive MetaStore that specifies which version last updated // the entry in the MetaStore with the latest schema and table property information val METASTORE_LAST_UPDATE_VERSION = "delta.lastUpdateVersion" val METASTORE_LAST_COMMIT_TIMESTAMP = "delta.lastCommitTimestamp" /** * Convert a string to [[CalendarInterval]]. This method is case-insensitive and will throw * [[IllegalArgumentException]] when the input string is not a valid interval. * * TODO Remove this method and use `CalendarInterval.fromCaseInsensitiveString` instead when * upgrading Spark. This is a fork version of `CalendarInterval.fromCaseInsensitiveString` which * will be available in the next Spark release (See SPARK-27735). * * @throws IllegalArgumentException if the string is not a valid internal. */ def parseCalendarInterval(s: String): CalendarInterval = { if (s == null || s.trim.isEmpty) { throw DeltaErrors.emptyCalendarInterval } val sInLowerCase = s.trim.toLowerCase(Locale.ROOT) val interval = if (sInLowerCase.startsWith("interval ")) sInLowerCase else "interval " + sInLowerCase val cal = IntervalUtils.safeStringToInterval(UTF8String.fromString(interval)) if (cal == null) { throw DeltaErrors.invalidInterval(s) } cal } /** * The prefix for a category of special configs for delta universal format to support the * user facing config naming convention for different table formats: * "delta.universalFormat.config.[iceberg/hudi].[config_name]" * Note that config_name can be arbitrary. */ final val DELTA_UNIVERSAL_FORMAT_CONFIG_PREFIX = "delta.universalformat.config." final val DELTA_UNIVERSAL_FORMAT_ICEBERG_CONFIG_PREFIX = s"${DELTA_UNIVERSAL_FORMAT_CONFIG_PREFIX}iceberg." /** * A global default value set as a SQLConf will overwrite the default value of a DeltaConfig. * For example, user can run: * set spark.databricks.delta.properties.defaults.randomPrefixLength = 5 * This setting will be populated to a Delta table during its creation time and overwrites * the default value of delta.randomPrefixLength. * * We accept these SQLConfs as strings and only perform validation in DeltaConfig. All the * DeltaConfigs set in SQLConf should adopt the same prefix. */ val sqlConfPrefix = "spark.databricks.delta.properties.defaults." private[delta] val entries = new HashMap[String, DeltaConfig[_]] protected def buildConfig[T]( key: String, defaultValue: String, fromString: String => T, validationFunction: T => Boolean, helpMessage: String, userConfigurable: Boolean = true, alternateConfs: Seq[DeltaConfig[T]] = Seq.empty): DeltaConfig[T] = { val deltaConfig = DeltaConfig(s"delta.$key", defaultValue, fromString, validationFunction, helpMessage, userConfigurable, alternateConfs.map(_.key)) entries.put(key.toLowerCase(Locale.ROOT), deltaConfig) deltaConfig } /** * Validates specified configurations and returns the normalized key -> value map. */ def validateConfigurations(configurations: Map[String, String]): Map[String, String] = { val allowArbitraryProperties = SparkSession.active.sessionState.conf .getConf(DeltaSQLConf.ALLOW_ARBITRARY_TABLE_PROPERTIES) configurations.map { case kv @ (key, value) => key.toLowerCase(Locale.ROOT) match { case lKey if lKey.startsWith("delta.constraints.") => // This is a CHECK constraint, we should allow it. kv case lKey if lKey.startsWith(TableFeatureProtocolUtils.FEATURE_PROP_PREFIX) => // This is a table feature, we should allow it. lKey -> value case lKey if lKey.startsWith("delta.") => Option(entries.get(lKey.stripPrefix("delta."))) match { case Some(deltaConfig) if ( lKey == DeltaConfigs.TOMBSTONE_RETENTION.key.toLowerCase(Locale.ROOT) || lKey == DeltaConfigs.LOG_RETENTION.key.toLowerCase(Locale.ROOT)) => val ret = deltaConfig(value) // validate the value validateTombstoneAndLogRetentionDurationCompatibility(configurations) ret case Some(deltaConfig) => deltaConfig(value) // validate the value case None if lKey.startsWith(DELTA_UNIVERSAL_FORMAT_CONFIG_PREFIX) => // always allow any delta universal format config with key converted to lower case lKey -> value case None if allowArbitraryProperties => logConsole( s"You are setting a property: $key that is not recognized by this " + "version of Delta") kv case None => throw DeltaErrors.unknownConfigurationKeyException(key) } case _ => if (entries.containsKey(key)) { logConsole(s""" |You are trying to set a property the key of which is the same as Delta config: $key. |If you are trying to set a Delta config, prefix it with "delta.", e.g. 'delta.$key'. """.stripMargin) } kv } } } /** * Table properties for new tables can be specified through SQL Configurations using the * [[sqlConfPrefix]] and [[TableFeatureProtocolUtils.DEFAULT_FEATURE_PROP_PREFIX]]. This method * checks to see if any of the configurations exist among the SQL configurations and merges them * with the user provided configurations. User provided configs take precedence. * * When `ignoreProtocolConfsOpt` is `true` (or `false`), this method will not (or will) copy * protocol-related configs. If `ignoreProtocolConfsOpt` is None, whether to copy * protocol-related configs will be depending on the existence of * [[DeltaConfigs.CREATE_TABLE_IGNORE_PROTOCOL_DEFAULTS]] (`delta.ignoreProtocolDefaults`) in * SQL or table configs. * * "Protocol-related configs" includes `delta.minReaderVersion`, `delta.minWriterVersion`, * `delta.ignoreProtocolDefaults`, and anything that starts with `delta.feature.` */ def mergeGlobalConfigs( sqlConfs: SQLConf, tableConf: Map[String, String], ignoreProtocolConfsOpt: Option[Boolean] = None): Map[String, String] = { val ignoreProtocolConfs = ignoreProtocolConfsOpt.getOrElse(ignoreProtocolDefaultsIsSet(sqlConfs, tableConf)) val shouldCopyFunc: (String => Boolean) = !ignoreProtocolConfs || !TableFeatureProtocolUtils.isTableProtocolProperty(_) val globalConfs = entries.asScala .filter { case (_, config) => shouldCopyFunc(config.key) } .flatMap { case (_, config) => val sqlConfKey = sqlConfPrefix + config.key.stripPrefix("delta.") Option(sqlConfs.getConfString(sqlConfKey, null)).map(config(_)) } // Table features configured in session must be merged manually because there's no // ConfigEntry registered for table features in SQL configs or Table props. val globalFeatureConfs = if (ignoreProtocolConfs) { Map.empty[String, String] } else { sqlConfs.getAllConfs .filterKeys(_.startsWith(TableFeatureProtocolUtils.DEFAULT_FEATURE_PROP_PREFIX)) .map { case (key, value) => val featureName = key.stripPrefix(TableFeatureProtocolUtils.DEFAULT_FEATURE_PROP_PREFIX) val tableKey = TableFeatureProtocolUtils.FEATURE_PROP_PREFIX + featureName tableKey -> value } } globalConfs.toMap ++ globalFeatureConfs.toMap ++ tableConf } /** * Whether [[DeltaConfigs.CREATE_TABLE_IGNORE_PROTOCOL_DEFAULTS]] is set in Spark session * configs or table properties. */ private[delta] def ignoreProtocolDefaultsIsSet( sqlConfs: SQLConf, tableConf: Map[String, String]): Boolean = { tableConf .getOrElse( DeltaConfigs.CREATE_TABLE_IGNORE_PROTOCOL_DEFAULTS.key, sqlConfs.getConfString( DeltaConfigs.CREATE_TABLE_IGNORE_PROTOCOL_DEFAULTS.defaultTablePropertyKey, "false")) .toBoolean } /** * Normalize the specified property keys if the key is for a Delta config. */ def normalizeConfigKeys(propKeys: Seq[String]): Seq[String] = { propKeys.map { key => key.toLowerCase(Locale.ROOT) match { case lKey if lKey.startsWith(TableFeatureProtocolUtils.FEATURE_PROP_PREFIX) => lKey case lKey if lKey.startsWith("delta.") => Option(entries.get(lKey.stripPrefix("delta."))).map(_.key).getOrElse(key) case _ => key } } } /** * Normalize the specified property key if the key is for a Delta config. */ def normalizeConfigKey(propKey: Option[String]): Option[String] = { propKey.map { key => key.toLowerCase(Locale.ROOT) match { case lKey if lKey.startsWith(TableFeatureProtocolUtils.FEATURE_PROP_PREFIX) => lKey case lKey if lKey.startsWith("delta.") => Option(entries.get(lKey.stripPrefix("delta."))).map(_.key).getOrElse(key) case _ => key } } } def getMilliSeconds(i: CalendarInterval): Long = { getMicroSeconds(i) / 1000L } private def getMicroSeconds(i: CalendarInterval): Long = { assert(i.months == 0) i.days * DateTimeConstants.MICROS_PER_DAY + i.microseconds } private def validateTombstoneAndLogRetentionDurationCompatibility( configs: Map[String, String]): Unit = { if (!SparkSession.active.sessionState.conf .getConf(DeltaSQLConf.ENFORCE_DELETED_FILE_AND_LOG_RETENTION_DURATION_COMPATIBILITY)) { return } val lowerCaseConfigs = configs.iterator.map { case (k, v) => k.toLowerCase(Locale.ROOT) -> v }.toMap val logRetention = DeltaConfigs.LOG_RETENTION val tombstoneRetention = DeltaConfigs.TOMBSTONE_RETENTION val logRetentionDuration: CalendarInterval = logRetention.fromString( lowerCaseConfigs.get(logRetention.key.toLowerCase(Locale.ROOT)) .getOrElse(logRetention.defaultValue)) val tombstoneRetentionDuration: CalendarInterval = tombstoneRetention.fromString( lowerCaseConfigs.get(tombstoneRetention.key.toLowerCase(Locale.ROOT)) .getOrElse(tombstoneRetention.defaultValue)) val logRetentionFound = lowerCaseConfigs.get( logRetention.key.toLowerCase(Locale.ROOT)).isDefined val errorMessage = if (logRetentionFound) { s"The table property ${DeltaConfigs.LOG_RETENTION.key}(${logRetentionDuration.toString}) " + s"needs to be greater than or equal to ${DeltaConfigs.TOMBSTONE_RETENTION.key}" + s"(${tombstoneRetentionDuration.toString})." } else { s"The table property ${DeltaConfigs.TOMBSTONE_RETENTION.key}" + s"(${tombstoneRetentionDuration.toString}) needs to be less than or equal to " + s"${DeltaConfigs.LOG_RETENTION.key}(${logRetentionDuration.toString})." } require(getMilliSeconds(logRetentionDuration) >= getMilliSeconds(tombstoneRetentionDuration), errorMessage) } /** * For configs accepting an interval, we require the user specified string must obey: * * - Doesn't use months or years, since an internal like this is not deterministic. * - The microseconds parsed from the string value must be a non-negative value. * * The method returns whether a [[CalendarInterval]] satisfies the requirements. */ def isValidIntervalConfigValue(i: CalendarInterval): Boolean = { i.months == 0 && getMicroSeconds(i) >= 0 } /** * Return all Delta configurations, including both set and unset ones. */ def getAllConfigs: Map[String, DeltaConfig[_]] = { entries.asScala.toMap } /** * The protocol reader version modelled as a table property. This property is *not* stored as * a table property in the `Metadata` action. It is stored as its own action. Having it modelled * as a table property makes it easier to upgrade, and view the version. */ val MIN_READER_VERSION = buildConfig[Int]( "minReaderVersion", Action.supportedProtocolVersion().minReaderVersion.toString, _.toInt, v => Action.supportedReaderVersionNumbers.contains(v), s"needs to be one of ${Action.supportedReaderVersionNumbers.toSeq.sorted.mkString(", ")}.") /** * The protocol reader version modelled as a table property. This property is *not* stored as * a table property in the `Metadata` action. It is stored as its own action. Having it modelled * as a table property makes it easier to upgrade, and view the version. */ val MIN_WRITER_VERSION = buildConfig[Int]( "minWriterVersion", Action.supportedProtocolVersion().minWriterVersion.toString, _.toInt, v => Action.supportedWriterVersionNumbers.contains(v), s"needs to be one of ${Action.supportedWriterVersionNumbers.toSeq.sorted.mkString(", ")}.") /** * Ignore protocol-related configs set in SQL config. * When set to true, CREATE TABLE and REPLACE TABLE commands will not consider default * protocol versions and table features in the current Spark session. */ val CREATE_TABLE_IGNORE_PROTOCOL_DEFAULTS = buildConfig[Boolean]( "ignoreProtocolDefaults", defaultValue = "false", fromString = _.toBoolean, validationFunction = _ => true, helpMessage = "needs to be a boolean.") /** * The shortest duration we have to keep delta files around before deleting them. We can only * delete delta files that are before a compaction. We may keep files beyond this duration until * the next calendar day. */ val LOG_RETENTION = buildConfig[CalendarInterval]( "logRetentionDuration", "interval 30 days", parseCalendarInterval, isValidIntervalConfigValue, "needs to be provided as a calendar interval such as '2 weeks'. Months " + "and years are not accepted. You may specify '365 days' for a year instead.") /** * The shortest duration we have to keep delta sample files around before deleting them. */ val SAMPLE_RETENTION = buildConfig[CalendarInterval]( "sampleRetentionDuration", "interval 7 days", parseCalendarInterval, isValidIntervalConfigValue, "needs to be provided as a calendar interval such as '2 weeks'. Months " + "and years are not accepted. You may specify '365 days' for a year instead.") /** * The shortest duration we have to keep checkpoint files around before deleting them. Note that * we'll never delete the most recent checkpoint. We may keep checkpoint files beyond this * duration until the next calendar day. */ val CHECKPOINT_RETENTION_DURATION = buildConfig[CalendarInterval]( "checkpointRetentionDuration", "interval 2 days", parseCalendarInterval, isValidIntervalConfigValue, "needs to be provided as a calendar interval such as '2 weeks'. Months " + "and years are not accepted. You may specify '365 days' for a year instead.") /** How often to checkpoint the delta log. */ val CHECKPOINT_INTERVAL = buildConfig[Int]( "checkpointInterval", "10", _.toInt, _ > 0, "needs to be a positive integer.") /** * This is the property that describes the table redirection detail. It is a JSON string format * of the `TableRedirectConfiguration` class, which includes following attributes: * - type(String): The type of redirection. * - state(String): The current state of the redirection: * ENABLE-REDIRECT-IN-PROGRESS, REDIRECT-READY, DROP-REDIRECT-IN-PROGRESS. * - spec(JSON String): The specification of accessing redirect destination table. This is free * form json object. Each delta service provider can customize its own * implementation. */ val REDIRECT_READER_WRITER: DeltaConfig[Option[String]] = buildConfig[Option[String]]( "redirectReaderWriter-preview", null, v => Option(v), _ => true, "A JSON representation of the TableRedirectConfiguration class, which contains all " + "information of redirect reader writer feature.") /** * This table feature is same as REDIRECT_READER_WRITER except it is a writer only table feature. */ val REDIRECT_WRITER_ONLY: DeltaConfig[Option[String]] = buildConfig[Option[String]]( "redirectWriterOnly-preview", null, v => Option(v), _ => true, "A JSON representation of the TableRedirectConfiguration class, which contains all " + "information of redirect writer only feature.") /** * Enable auto compaction for a Delta table. When enabled, we will check if files already * written to a Delta table can leverage compaction after a commit. If so, we run a post-commit * hook to compact the files. * It can be enabled by setting the property to `true` * Note that the behavior from table property can be overridden by the config: * [[org.apache.spark.sql.delta.sources.DeltaSQLConf.DELTA_AUTO_COMPACT_ENABLED]] */ val AUTO_COMPACT = buildConfig[Option[String]]( "autoOptimize.autoCompact", null, v => Option(v).map(_.toLowerCase(Locale.ROOT)), v => v.isEmpty || AutoCompactType.ALLOWED_VALUES.contains(v.get), s""""needs to be one of: ${AutoCompactType.ALLOWED_VALUES.mkString(",")}""") /** Whether to clean up expired checkpoints and delta logs. */ val ENABLE_EXPIRED_LOG_CLEANUP = buildConfig[Boolean]( "enableExpiredLogCleanup", "true", _.toBoolean, _ => true, "needs to be a boolean.") /** * If true, a delta table can be rolled back to any point within LOG_RETENTION. Leaving this on * requires converting the oldest delta file we have into a checkpoint, which we do once a day. If * doing that operation is too expensive, it can be turned off, but the table can only be rolled * back CHECKPOINT_RETENTION_DURATION ago instead of LOG_RETENTION ago. */ val ENABLE_FULL_RETENTION_ROLLBACK = buildConfig[Boolean]( "enableFullRetentionRollback", "true", _.toBoolean, _ => true, "needs to be a boolean." ) /** * The logRetention period to be used in DROP FEATURE ... TRUNCATE HISTORY command. * The value should represent the expected duration of the longest running transaction. Setting * this to a lower value than the longest running transaction may corrupt the table. */ val TABLE_FEATURE_DROP_TRUNCATE_HISTORY_LOG_RETENTION = buildConfig[CalendarInterval]( "dropFeatureTruncateHistory.retentionDuration", "interval 24 hours", parseCalendarInterval, isValidIntervalConfigValue, "needs to be provided as a calendar interval such as '2 weeks'. Months " + "and years are not accepted. You may specify '365 days' for a year instead.") /** * The shortest duration we have to keep logically deleted data files around before deleting them * physically. This is to prevent failures in stale readers after compactions or partition * overwrites. * * Note: this value should be large enough: * - It should be larger than the longest possible duration of a job if you decide to run "VACUUM" * when there are concurrent readers or writers accessing the table. * - If you are running a streaming query reading from the table, you should make sure the query * doesn't stop longer than this value. Otherwise, the query may not be able to restart as it * still needs to read old files. */ val TOMBSTONE_RETENTION = buildConfig[CalendarInterval]( "deletedFileRetentionDuration", "interval 1 week", parseCalendarInterval, isValidIntervalConfigValue, "needs to be provided as a calendar interval such as '2 weeks'. Months " + "and years are not accepted. You may specify '365 days' for a year instead.") /** * Whether to use a random prefix in a file path instead of partition information. This is * required for very high volume S3 calls to better be partitioned across S3 servers. */ val RANDOMIZE_FILE_PREFIXES = buildConfig[Boolean]( "randomizeFilePrefixes", "false", _.toBoolean, _ => true, "needs to be a boolean.") /** * Whether to use a random prefix in a file path instead of partition information. This is * required for very high volume S3 calls to better be partitioned across S3 servers. */ val RANDOM_PREFIX_LENGTH = buildConfig[Int]( "randomPrefixLength", "2", _.toInt, a => a > 0, "needs to be greater than 0.") /** * Whether this Delta table is append-only. Files can't be deleted, or values can't be updated. */ val IS_APPEND_ONLY = buildConfig[Boolean]( key = "appendOnly", defaultValue = "false", fromString = _.toBoolean, validationFunction = _ => true, helpMessage = "needs to be a boolean.") /** * Whether commands modifying this Delta table are allowed to create new deletion vectors. */ val ENABLE_DELETION_VECTORS_CREATION = buildConfig[Boolean]( key = "enableDeletionVectors", defaultValue = "false", fromString = _.toBoolean, validationFunction = _ => true, helpMessage = "needs to be a boolean.") val ENABLE_VARIANT_SHREDDING = buildConfig[Boolean]( key = "enableVariantShredding", defaultValue = "false", fromString = _.toBoolean, validationFunction = _ => true, helpMessage = "needs to be a boolean.") /** * Whether this table will automatically optimize the layout of files during writes. */ val AUTO_OPTIMIZE = buildConfig[Option[Boolean]]( "autoOptimize", null, v => Option(v).map(_.toBoolean), _ => true, "needs to be a boolean.") /** * The number of columns to collect stats on for data skipping. A value of -1 means collecting * stats for all columns. Updating this conf does not trigger stats re-collection, but redefines * the stats schema of table, i.e., it will change the behavior of future stats collection * (e.g., in append and OPTIMIZE) as well as data skipping (e.g., the column stats beyond this * number will be ignored even when they exist). */ val DATA_SKIPPING_NUM_INDEXED_COLS = buildConfig[Int]( "dataSkippingNumIndexedCols", DataSkippingReaderConf.DATA_SKIPPING_NUM_INDEXED_COLS_DEFAULT_VALUE.toString, _.toInt, a => a >= -1, "needs to be larger than or equal to -1.") /** * The names of specific columns to collect stats on for data skipping. If present, it takes * precedences over dataSkippingNumIndexedCols config, and the system will only collect stats for * columns that exactly match those specified. If a nested column is specified, the system will * collect stats for all leaf fields of that column. If a non-existent column is specified, it * will be ignored. Updating this conf does not trigger stats re-collection, but redefines the * stats schema of table, i.e., it will change the behavior of future stats collection (e.g., in * append and OPTIMIZE) as well as data skipping (e.g., the column stats not mentioned by this * config will be ignored even if they exist). */ val DATA_SKIPPING_STATS_COLUMNS = buildConfig[Option[String]]( "dataSkippingStatsColumns", null, v => Option(v), vOpt => vOpt.forall(v => DeltaSqlParserUtils.parseMultipartColumnList(v).isDefined), """ |The dataSkippingStatsColumns parameter is a comma-separated list of case-insensitive column |identifiers. Each column identifier can consist of letters, digits, and underscores. |Multiple column identifiers can be listed, separated by commas. | |If a column identifier includes special characters such as !@#$%^&*()_+-={}|[]:";'<>,.?/, |the column name should be enclosed in backticks (`) to escape the special characters. | |A column identifier can refer to one of the following: the name of a non-struct column, the |leaf field's name of a struct column, or the name of a struct column. When a struct column's |name is specified in dataSkippingStatsColumns, statistics for all its leaf fields will be |collected. |""".stripMargin) /** * For string columns, how long prefix to store in the data skipping index. * Note that the behavior from table property overrides the config: * [[DeltaSQLConf.DATA_SKIPPING_STRING_PREFIX_LENGTH]] */ val DATA_SKIPPING_STRING_PREFIX_LENGTH = buildConfig[Option[Int]]( "dataSkippingStringPrefixLength", null, v => Option(v).map(_.toInt), v => v.forall(_ >= 0), "needs to be greater or equal to zero.") val SYMLINK_FORMAT_MANIFEST_ENABLED = buildConfig[Boolean]( s"${hooks.GenerateSymlinkManifest.CONFIG_NAME_ROOT}.enabled", "false", _.toBoolean, _ => true, "needs to be a boolean.") /** * When enabled, we will write file statistics in the checkpoint in JSON format as the "stats" * column. */ val CHECKPOINT_WRITE_STATS_AS_JSON = buildConfig[Boolean]( "checkpoint.writeStatsAsJson", "true", _.toBoolean, _ => true, "needs to be a boolean.") /** * When enabled, we will write file statistics in the checkpoint in the struct format in the * "stats_parsed" column. We will also write partition values as a struct as * "partitionValues_parsed". */ val CHECKPOINT_WRITE_STATS_AS_STRUCT = buildConfig[Boolean]( "checkpoint.writeStatsAsStruct", "true", _.toBoolean, _ => true, "needs to be a boolean.") /** * Deprecated in favor of CHANGE_DATA_FEED. */ private val CHANGE_DATA_FEED_LEGACY = buildConfig[Boolean]( "enableChangeDataCapture", "false", _.toBoolean, _ => true, "needs to be a boolean.") /** * Enable change data feed output. * When enabled, DELETE, UPDATE, and MERGE INTO operations will need to do additional work to * output their change data in an efficiently readable format. */ val CHANGE_DATA_FEED = buildConfig[Boolean]( "enableChangeDataFeed", "false", _.toBoolean, _ => true, "needs to be a boolean.", alternateConfs = Seq(CHANGE_DATA_FEED_LEGACY)) val COLUMN_MAPPING_MODE = buildConfig[DeltaColumnMappingMode]( "columnMapping.mode", "none", DeltaColumnMappingMode(_), _ => true, "") /** * Maximum columnId used in the schema so far for column mapping. Internal property that cannot * be set by users. */ val COLUMN_MAPPING_MAX_ID = buildConfig[Long]( "columnMapping.maxColumnId", "0", _.toLong, _ => true, "", userConfigurable = false) /** * The shortest duration within which new [[Snapshot]]s will retain transaction identifiers (i.e. * [[SetTransaction]]s). When a new [[Snapshot]] sees a transaction identifier older than or equal * to the specified TRANSACTION_ID_RETENTION_DURATION, it considers it expired and ignores it. */ val TRANSACTION_ID_RETENTION_DURATION = buildConfig[Option[CalendarInterval]]( "setTransactionRetentionDuration", null, v => if (v == null) None else Some(parseCalendarInterval(v)), opt => opt.forall(isValidIntervalConfigValue), "needs to be provided as a calendar interval such as '2 weeks'. Months " + "and years are not accepted. You may specify '365 days' for a year instead.") /** * The isolation level of a table defines the degree to which a transaction must be isolated from * modifications made by concurrent transactions. Delta currently supports one isolation level: * Serializable. */ val ISOLATION_LEVEL = buildConfig[IsolationLevel]( "isolationLevel", Serializable.toString, IsolationLevel.fromString(_), _ == Serializable, "must be Serializable" ) /** Policy to decide what kind of checkpoint to write to a table. */ val CHECKPOINT_POLICY = buildConfig[CheckpointPolicy.Policy]( key = "checkpointPolicy", defaultValue = CheckpointPolicy.Classic.name, fromString = str => CheckpointPolicy.fromName(str), validationFunction = (v => CheckpointPolicy.ALL.exists(_.name == v.name)), helpMessage = s"can be one of the " + s"following: ${CheckpointPolicy.Classic.name}, ${CheckpointPolicy.V2.name}") /** * Indicates whether Row Tracking is enabled on the table. When this flag is turned on, all rows * are guaranteed to have Row IDs and Row Commit Versions assigned to them, and writers are * expected to preserve them by materializing them to hidden columns in the data files. */ val ROW_TRACKING_ENABLED = buildConfig[Boolean]( key = "enableRowTracking", defaultValue = false.toString, fromString = _.toBoolean, validationFunction = _ => true, helpMessage = "needs to be a boolean.") /** * Controls whether row tracking operations should be suspended. It blocks the assignment of new * baseRowIds as well as copying existing baseRowIds. It is intended to be used when dropping * row tracking. It can be enabled after setting `delta.enableRowTracking` to false. * * WARNING 1: Should never be enabled when `delta.enableRowTracking` is set to true. * WARNING 2: It should never be manually set. It is only safe to be used in the context of * DROP FEATURE. */ val ROW_TRACKING_SUSPENDED = buildConfig[Boolean]( key = "rowTrackingSuspended", defaultValue = false.toString, fromString = _.toBoolean, validationFunction = _ => true, helpMessage = "needs to be a boolean.") /** * Convert the table's metadata into other storage formats after each Delta commit. * Only Iceberg is supported for now */ val UNIVERSAL_FORMAT_ENABLED_FORMATS = buildConfig[Seq[String]]( "universalFormat.enabledFormats", "", fromString = str => if (str == null || str.isEmpty) Nil else str.split(","), validationFunction = seq => if (seq.distinct.length != seq.length) false else seq.toSet.subsetOf(UniversalFormat.SUPPORTED_FORMATS), s"Must be a comma-separated list of formats from the list: " + s"${UniversalFormat.SUPPORTED_FORMATS.mkString("{", ",", "}")}." ) val ICEBERG_COMPAT_V1_ENABLED = buildConfig[Option[Boolean]]( "enableIcebergCompatV1", null, v => Option(v).map(_.toBoolean), _ => true, "needs to be a boolean." ) val ICEBERG_COMPAT_V2_ENABLED = buildConfig[Option[Boolean]]( key = "enableIcebergCompatV2", defaultValue = null, fromString = v => Option(v).map(_.toBoolean), validationFunction = _ => true, helpMessage = "needs to be a boolean." ) val CAST_ICEBERG_TIME_TYPE = buildConfig[Boolean]( key = "castIcebergTimeType", defaultValue = "false", fromString = _.toBoolean, validationFunction = _ => true, helpMessage = "Casting Iceberg TIME type to Spark Long type enabled" ) val IGNORE_ICEBERG_BUCKET_PARTITION = buildConfig[Boolean]( key = "ignoreIcebergBucketPartition", defaultValue = "false", fromString = _.toBoolean, validationFunction = _ => true, helpMessage = "Ignore Iceberg bucket partition, which means " + "converting source iceberg table to a non-partition delta table" ) /** * Enable optimized writes into a Delta table. Optimized writes adds an adaptive shuffle before * the write to write compacted files into a Delta table during a write. */ val OPTIMIZE_WRITE = buildConfig[Option[Boolean]]( "autoOptimize.optimizeWrite", null, v => Option(v).map(_.toBoolean), _ => true, "needs to be a boolean." ) /** * Whether widening the type of an existing column or field is allowed, either manually using * ALTER TABLE CHANGE COLUMN or automatically if automatic schema evolution is enabled. */ val ENABLE_TYPE_WIDENING = buildConfig[Boolean]( key = "enableTypeWidening", defaultValue = false.toString, fromString = _.toBoolean, validationFunction = _ => true, helpMessage = "needs to be a boolean.") val COORDINATED_COMMITS_COORDINATOR_NAME = buildConfig[Option[String]]( "coordinatedCommits.commitCoordinator-preview", null, v => Option(v), _ => true, """The commit-coordinator name for this table. This is used to determine which |implementation of commit-coordinator to use when committing to this table. If this property |is not set, the table will be considered as file system table and commits will be done via |atomically publishing the commit file. |""".stripMargin) val COORDINATED_COMMITS_COORDINATOR_CONF = buildConfig[Map[String, String]]( "coordinatedCommits.commitCoordinatorConf-preview", null, v => JsonUtils.fromJson[Map[String, String]](Option(v).getOrElse("{}")), _ => true, "A string-to-string map of configuration properties for the coordinated commits-coordinator.") val COORDINATED_COMMITS_TABLE_CONF = buildConfig[Map[String, String]]( "coordinatedCommits.tableConf-preview", null, v => JsonUtils.fromJson[Map[String, String]](Option(v).getOrElse("{}")), _ => true, "A string-to-string map of configuration properties for describing the table to" + " commit-coordinator.") val IN_COMMIT_TIMESTAMPS_ENABLED = buildConfig[Boolean]( "enableInCommitTimestamps", false.toString, _.toBoolean, validationFunction = _ => true, "needs to be a boolean." ) /** * This table property is used to track the version of the table at which * inCommitTimestamps were enabled. */ val IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION = buildConfig[Option[Long]]( "inCommitTimestampEnablementVersion", null, v => Option(v).map(_.toLong), validationFunction = _ => true, "needs to be a long." ) /** * This table property is used to track the timestamp at which inCommitTimestamps * were enabled. More specifically, it is the inCommitTimestamp of the commit with * the version specified in [[IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION]]. */ val IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP = buildConfig[Option[Long]]( "inCommitTimestampEnablementTimestamp", null, v => Option(v).map(_.toLong), validationFunction = _ => true, "needs to be a long.") /** * This property is used by CheckpointProtectionTableFeature and denotes the * version up to which the checkpoints are required to be cleaned up only together with the * corresponding commits. If this is not possible, and metadata cleanup creates a new checkpoint * prior to requireCheckpointProtectionBeforeVersion, it should validate write support against * all protocols included in the commits that are being removed, or else abort. This is needed * to make sure that the writer understands how to correctly create a checkpoint for the * historic commit. * * Note, this is an internal config and should never be manually altered. */ val REQUIRE_CHECKPOINT_PROTECTION_BEFORE_VERSION = buildConfig[Long]( "requireCheckpointProtectionBeforeVersion", "0", _.toLong, _ >= 0, "needs to be greater or equal to zero.") /** * If true, enables the MaterializePartitionColumns table feature which requires partition * columns to be materialized for future parquet data files. */ val ENABLE_MATERIALIZE_PARTITION_COLUMNS_FEATURE = buildConfig[Option[Boolean]]( "enableMaterializePartitionColumnsFeature", null, v => Option(v).map(_.toBoolean), _ => true, "needs to be a boolean.") } object DeltaConfigs extends DeltaConfigsBase ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaErrors.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.io.{FileNotFoundException, IOException} import java.nio.file.FileAlreadyExistsException import java.util.{ConcurrentModificationException, UUID} import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.skipping.clustering.temp.{ClusterBySpec} import org.apache.spark.sql.delta.actions.{CommitInfo, Metadata, Protocol, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.commands.{AlterTableDropFeatureDeltaCommand, DeltaGenerateCommand} import org.apache.spark.sql.delta.constraints.Constraints import org.apache.spark.sql.delta.hooks.AutoCompactType import org.apache.spark.sql.delta.hooks.PostCommitHook import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.redirect.NoRedirectRule import org.apache.spark.sql.delta.redirect.RedirectSpec import org.apache.spark.sql.delta.redirect.RedirectState import org.apache.spark.sql.delta.schema.{DeltaInvariantViolationException, InvariantViolationException, SchemaUtils, UnsupportedDataTypeInfo} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.JsonUtils import io.delta.exceptions import org.apache.hadoop.fs.{ChecksumException, Path} import org.apache.spark.{SparkConf, SparkEnv, SparkException} import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.errors.QueryErrorsBase import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataType, StructField, StructType} trait DocsPath { /** * The URL for the base path of Delta's docs. When changing this path, ensure that the new path * works with the error messages below. */ protected def baseDocsPath(conf: SparkConf): String = "https://docs.delta.io/latest" def assertValidCallingFunction(): Unit = { val callingMethods = Thread.currentThread.getStackTrace callingMethods.foreach { method => if (errorsWithDocsLinks.contains(method.getMethodName)) { return } } assert(assertion = false, "The method throwing the error which contains a doc link must be a " + s"part of DocsPath.errorsWithDocsLinks") } /** * Get the link to the docs for the given relativePath. Validates that the error generating the * link is added to docsLinks. * Please only use this function if SparkConf is directly available. * If needed to retrieve SparkConf from SparkSession, * please use more safe function [[generateDocsLinkOption]]. * * @param relativePath the relative path after the base url to access. * @param skipValidation whether to validate that the function generating the link is * in the allowlist. * @return The entire URL of the documentation link */ def generateDocsLink( conf: SparkConf, relativePath: String, skipValidation: Boolean = false): String = { require(conf != null) if (!skipValidation) assertValidCallingFunction() baseDocsPath(conf) + relativePath } /** Safe alternative to [[generateDocsLink]] that validates sparkContext before accessing it. */ def generateDocsLinkOption( spark: SparkSession, relativePath: String, skipValidation: Boolean = false): Option[String] = Option(spark.sparkContext) .map(context => generateDocsLink(context.getConf, relativePath, skipValidation)) /** * List of error function names for all errors that have URLs. When adding your error to this list * remember to also add it to the list of errors in DeltaErrorsSuite * * @note add your error to DeltaErrorsSuiteBase after adding it to this list so that the url can * be tested */ def errorsWithDocsLinks: Seq[String] = Seq( "createExternalTableWithoutLogException", "createExternalTableWithoutSchemaException", "createManagedTableWithoutSchemaException", "multipleSourceRowMatchingTargetRowInMergeException", "ignoreStreamingUpdatesAndDeletesWarning", "concurrentAppendException", "concurrentDeleteDeleteException", "concurrentDeleteReadException", "concurrentWriteException", "concurrentTransactionException", "metadataChangedException", "protocolChangedException", "concurrentModificationExceptionMsg", "incorrectLogStoreImplementationException", "sourceNotDeterministicInMergeException", "columnMappingAdviceMessage", "icebergClassMissing", "tableFeatureReadRequiresWriteException", "tableFeatureRequiresHigherReaderProtocolVersion", "tableFeatureRequiresHigherWriterProtocolVersion", "blockStreamingReadsWithIncompatibleNonAdditiveSchemaChanges" ) } /** * A holder object for Delta errors. * * * IMPORTANT: Any time you add a test that references the docs, add to the Seq defined in * DeltaErrorsSuite so that the doc links that are generated can be verified to work in * docs.delta.io */ trait DeltaErrorsBase extends DocsPath with DeltaLogging with QueryErrorsBase { def baseDocsPath(spark: SparkSession): String = baseDocsPath(spark.sparkContext.getConf) val faqRelativePath: String = "/delta-intro.html#frequently-asked-questions" val EmptyCheckpointErrorMessage = s""" |Attempted to write an empty checkpoint without any actions. This checkpoint will not be |useful in recomputing the state of the table. However this might cause other checkpoints to |get deleted based on retention settings. """.stripMargin // scalastyle:off def assertionFailedError(msg: String): Throwable = new AssertionError(msg) // scalastyle:on def deltaSourceIgnoreDeleteError( version: Long, removedFile: String, dataPath: String): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_SOURCE_IGNORE_DELETE", messageParameters = Array(removedFile, version.toString, dataPath) ) } def initialSnapshotTooLargeForStreaming( snapshotVersion: Long, numFiles: Long, maxFiles: Int, tablePath: String): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_STREAMING_INITIAL_SNAPSHOT_TOO_LARGE", messageParameters = Array( tablePath, snapshotVersion.toString, numFiles.toString, maxFiles.toString, s"""To fix this issue, choose one of: | | 1. Increase spark.databricks.delta.streaming.initialSnapshotMaxFiles | (current: $maxFiles) | | 2. Use 'startingVersion' option to skip the initial snapshot and start | from a specific version""".stripMargin ) ) } def deltaSourceIgnoreChangesError( version: Long, removedFile: String, dataPath: String): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_SOURCE_TABLE_IGNORE_CHANGES", messageParameters = Array(removedFile, version.toString, dataPath) ) } def unknownReadLimit(limit: String): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_UNKNOWN_READ_LIMIT", messageParameters = Array(limit) ) } def unknownPrivilege(privilege: String): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_UNKNOWN_PRIVILEGE", messageParameters = Array(privilege) ) } def columnNotFound(path: Seq[String], schema: StructType): Throwable = { val name = UnresolvedAttribute(path).name cannotResolveColumn(name, schema) } def failedMergeSchemaFile(file: String, schema: String, cause: Throwable): Throwable = { new DeltaSparkException( errorClass = "DELTA_FAILED_MERGE_SCHEMA_FILE", messageParameters = Array(file, schema), cause = cause) } def missingCommitInfo(featureName: String, commitVersion: String): DeltaIllegalStateException = { new DeltaIllegalStateException( errorClass = "DELTA_MISSING_COMMIT_INFO", messageParameters = Array(featureName, commitVersion)) } def missingCommitTimestamp(commitVersion: String): DeltaIllegalStateException = { new DeltaIllegalStateException( errorClass = "DELTA_MISSING_COMMIT_TIMESTAMP", messageParameters = Array(InCommitTimestampTableFeature.name, commitVersion)) } def failOnCheckpointRename(src: Path, dest: Path): DeltaIllegalStateException = { new DeltaIllegalStateException( errorClass = "DELTA_CANNOT_RENAME_PATH", messageParameters = Array(s"${src.toString}", s"${dest.toString}")) } def checkpointMismatchWithSnapshot : Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_CHECKPOINT_SNAPSHOT_MISMATCH", messageParameters = Array.empty ) } /** * Thrown when main table data contains columns that are reserved for CDF, such as `_change_type`. */ def cdcColumnsInData(columns: Seq[String]): Throwable = { new DeltaIllegalStateException( errorClass = "RESERVED_CDC_COLUMNS_ON_WRITE", messageParameters = Array(columns.mkString("[", ",", "]"), DeltaConfigs.CHANGE_DATA_FEED.key) ) } /** * Thrown when main table data already contains columns that are reserved for CDF, such as * `_change_type`, but CDF is not yet enabled on that table. */ def tableAlreadyContainsCDCColumns(columns: Seq[String]): Throwable = { new DeltaIllegalStateException(errorClass = "DELTA_TABLE_ALREADY_CONTAINS_CDC_COLUMNS", messageParameters = Array(columns.mkString("[", ",", "]"))) } /** * Thrown when a CDC query contains conflict 'starting' or 'ending' options, e.g. when both * starting version and starting timestamp are specified. * * @param position Specifies which option was duplicated in the read. Values are "starting" or * "ending" */ def multipleCDCBoundaryException(position: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_MULTIPLE_CDC_BOUNDARY", messageParameters = Array(position, position, position) ) } def formatColumn(colName: String): String = s"`$colName`" def formatColumnList(colNames: Seq[String]): String = colNames.map(formatColumn).mkString("[", ", ", "]") def formatSchema(schema: StructType): String = schema.treeString def notNullColumnMissingException(constraint: Constraints.NotNull): Throwable = { new DeltaInvariantViolationException( errorClass = "DELTA_MISSING_NOT_NULL_COLUMN_VALUE", messageParameters = Array(s"${UnresolvedAttribute(constraint.column).name}")) } def nestedNotNullConstraint( parent: String, nested: DataType, nestType: String): AnalysisException = { new DeltaAnalysisException( errorClass = "DELTA_NESTED_NOT_NULL_CONSTRAINT", messageParameters = Array( s"$nestType", s"$parent", s"${DeltaSQLConf.ALLOW_UNENFORCED_NOT_NULL_CONSTRAINTS.key}", s"$nestType", s"${nested.prettyJson}" ) ) } def nullableParentWithNotNullNestedField : Throwable = { new DeltaAnalysisException( errorClass = "DELTA_NOT_NULL_NESTED_FIELD", messageParameters = Array.empty ) } def constraintAlreadyExists(name: String, oldExpr: String): AnalysisException = { new DeltaAnalysisException( errorClass = "DELTA_CONSTRAINT_ALREADY_EXISTS", messageParameters = Array(name, oldExpr) ) } def invalidConstraintName(name: String): AnalysisException = { new DeltaAnalysisException( errorClass = "_LEGACY_ERROR_TEMP_DELTA_0001", messageParameters = Array(name) ) } def nonexistentConstraint(constraintName: String, tableName: String): AnalysisException = { new DeltaAnalysisException( errorClass = "DELTA_CONSTRAINT_DOES_NOT_EXIST", messageParameters = Array( constraintName, tableName, DeltaSQLConf.DELTA_ASSUMES_DROP_CONSTRAINT_IF_EXISTS.key, "true")) } def checkConstraintNotBoolean(name: String, expr: String): AnalysisException = { new DeltaAnalysisException( errorClass = "DELTA_NON_BOOLEAN_CHECK_CONSTRAINT", messageParameters = Array(name, expr) ) } def newCheckConstraintViolated(num: Long, tableName: String, expr: String): AnalysisException = { new DeltaAnalysisException( errorClass = "DELTA_NEW_CHECK_CONSTRAINT_VIOLATION", messageParameters = Array(s"$num", tableName, expr) ) } def newNotNullViolated( num: Long, tableName: String, col: UnresolvedAttribute): AnalysisException = { new DeltaAnalysisException( errorClass = "DELTA_NEW_NOT_NULL_VIOLATION", messageParameters = Array(s"$num", tableName, col.name) ) } def useAddConstraints: Throwable = { new DeltaAnalysisException( errorClass = "DELTA_ADD_CONSTRAINTS", messageParameters = Array.empty) } def cannotDropCheckConstraintFeature(constraintNames: Seq[String]): AnalysisException = { new DeltaAnalysisException( errorClass = "DELTA_CANNOT_DROP_CHECK_CONSTRAINT_FEATURE", messageParameters = Array(constraintNames.map(formatColumn).mkString(", ")) ) } def checkConstraintReferToWrongColumns(colName: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_INVALID_CHECK_CONSTRAINT_REFERENCES", messageParameters = Array(colName) ) } def checkConstraintUDF(expr: Expression): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UDF_IN_CHECK_CONSTRAINT", messageParameters = Array(expr.sql)) } def checkConstraintNonDeterministicExpression(expr: Expression): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_NON_DETERMINISTIC_EXPRESSION_IN_CHECK_CONSTRAINT", messageParameters = Array(expr.sql)) } def checkConstraintAggregateExpression(expr: Expression): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_AGGREGATE_IN_CHECK_CONSTRAINT", messageParameters = Array(expr.sql)) } def checkConstraintUnsupportedExpression(expr: Expression): Throwable = { val expressionSql = expr.sql new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_EXPRESSION_CHECK_CONSTRAINT", messageParameters = Array(expressionSql, expressionSql) ) } def deltaRelationPathMismatch( relationPath: Seq[String], targetType: String, targetPath: Seq[String] ): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_RELATION_PATH_MISMATCH", messageParameters = Array( relationPath.mkString("."), targetType, targetPath.mkString(".") ) ) } def unrecognizedRedirectSpec(spec: RedirectSpec): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_TABLE_UNRECOGNIZED_REDIRECT_SPEC", messageParameters = Array(spec.toString) ) } def invalidSetUnSetRedirectCommand( table: String, newProperty: String, existingProperty: String): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_TABLE_INVALID_SET_UNSET_REDIRECT", messageParameters = Array(table, existingProperty, newProperty) ) } def invalidRedirectStateTransition( table: String, oldState: RedirectState, newState: RedirectState): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_TABLE_INVALID_REDIRECT_STATE_TRANSITION", messageParameters = Array(table, oldState.name, newState.name) ) } def invalidRemoveTableRedirect(table: String, currentState: RedirectState): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_TABLE_INVALID_REMOVE_TABLE_REDIRECT", messageParameters = Array(table, table, currentState.name) ) } def invalidCommitIntermediateRedirectState(state: RedirectState): Throwable = { new DeltaIllegalStateException ( errorClass = "DELTA_COMMIT_INTERMEDIATE_REDIRECT_STATE", messageParameters = Array(state.name) ) } def noRedirectRulesViolated( op: DeltaOperations.Operation, noRedirectRules: Set[NoRedirectRule]): Throwable = { new DeltaIllegalStateException ( errorClass = "DELTA_NO_REDIRECT_RULES_VIOLATED", messageParameters = Array(op.name, noRedirectRules.map("\"" + _ + "\"").mkString("[", ",\n", "]")) ) } def incorrectLogStoreImplementationException( sparkConf: SparkConf, cause: Throwable): Throwable = { new DeltaIOException( errorClass = "DELTA_INCORRECT_LOG_STORE_IMPLEMENTATION", messageParameters = Array(generateDocsLink(sparkConf, "/delta-storage.html")), cause = cause) } def failOnDataLossException(expectedVersion: Long, seenVersion: Long): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_MISSING_FILES_UNEXPECTED_VERSION", messageParameters = Array(s"$expectedVersion", s"$seenVersion", s"${DeltaOptions.FAIL_ON_DATA_LOSS_OPTION}") ) } def staticPartitionsNotSupportedException: Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_STATIC_PARTITIONS", messageParameters = Array.empty ) } def zOrderingOnPartitionColumnException(colName: String): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_ZORDERING_ON_PARTITION_COLUMN", messageParameters = Array(colName) ) } def zOrderingOnColumnWithNoStatsException( colNames: Seq[String], spark: SparkSession): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_ZORDERING_ON_COLUMN_WITHOUT_STATS", messageParameters = Array(colNames.mkString("[", ", ", "]"), DeltaSQLConf.DELTA_OPTIMIZE_ZORDER_COL_STAT_CHECK.key) ) } def zOrderingColumnDoesNotExistException(colName: String): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_ZORDERING_COLUMN_DOES_NOT_EXIST", messageParameters = Array(colName)) } /** * Throwable used when CDC options contain no 'start'. */ def noStartVersionForCDC(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_NO_START_FOR_CDC_READ", messageParameters = Array.empty ) } /** * Throwable used when CDC is not enabled according to table metadata. */ def changeDataNotRecordedException(version: Long, start: Long, end: Long): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_MISSING_CHANGE_DATA", messageParameters = Array(start.toString, end.toString, version.toString, DeltaConfigs.CHANGE_DATA_FEED.key)) } def deletedRecordCountsHistogramDeserializationException(): Throwable = { new DeltaChecksumException( errorClass = "DELTA_DV_HISTOGRAM_DESERIALIZATON", messageParameters = Array.empty, pos = 0) } /** Throwable used when a non-constant expression is used as a version/timestamp arg in CDC. */ def cdcNonConstantArgument( fnName: String, paramName: String, position: Int, expr: Expression): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CDC_NON_CONSTANT_ARGUMENT", messageParameters = Array(s"`$paramName`", position.toString, s"`$fnName`", expr.sql) ) } /** Throwable used when a null 'start' or 'end' is provided in CDC reads. */ def nullRangeBoundaryInCDCRead(): Throwable = { new DeltaIllegalArgumentException(errorClass = "DELTA_CDC_READ_NULL_RANGE_BOUNDARY") } /** * Throwable used for invalid CDC 'start' and 'end' options, where end < start */ def endBeforeStartVersionInCDC(start: Long, end: Long): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_INVALID_CDC_RANGE", messageParameters = Array(start.toString, end.toString) ) } /** * Throwable used for invalid CDC 'start' and 'latest' options, where latest < start */ def startVersionAfterLatestVersion(start: Long, latest: Long): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_CDC_START_VERSION_AFTER_LATEST", messageParameters = Array(start.toString, latest.toString)) } def setTransactionVersionConflict(appId: String, version1: Long, version2: Long): Throwable = { new IllegalArgumentException( s"Two SetTransaction actions within the same transaction have the same appId ${appId} but " + s"different versions ${version1} and ${version2}.") } def unexpectedChangeFilesFound(changeFiles: String): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_UNEXPECTED_CHANGE_FILES_FOUND", messageParameters = Array(changeFiles)) } def addColumnAtIndexLessThanZeroException(pos: String, col: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_ADD_COLUMN_AT_INDEX_LESS_THAN_ZERO", messageParameters = Array(pos, col)) } def dropColumnAtIndexLessThanZeroException(pos: Int): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_DROP_COLUMN_AT_INDEX_LESS_THAN_ZERO", messageParameters = Array(s"$pos") ) } def columnNameNotFoundException(colName: String, scheme: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_COLUMN_NOT_FOUND", messageParameters = Array(colName, scheme)) } def foundDuplicateColumnsException(colType: String, duplicateCols: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_DUPLICATE_COLUMNS_FOUND", messageParameters = Array(colType, duplicateCols)) } def addColumnStructNotFoundException(pos: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_ADD_COLUMN_STRUCT_NOT_FOUND", messageParameters = Array(pos)) } def addColumnParentNotStructException(column: StructField, other: DataType): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_ADD_COLUMN_PARENT_NOT_STRUCT", messageParameters = Array(s"${column.name}", s"$other")) } def operationNotSupportedException( operation: String, tableIdentifier: TableIdentifier): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_OPERATION_NOT_ALLOWED_DETAIL", messageParameters = Array(operation, tableIdentifier.toString)) } def operationNotSupportedException(operation: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_OPERATION_NOT_ALLOWED", messageParameters = Array(operation)) } def emptyDataException: Throwable = { new DeltaAnalysisException( errorClass = "DELTA_EMPTY_DATA", messageParameters = Array.empty) } def fileNotFoundException(path: String): Throwable = { new DeltaFileNotFoundException( errorClass = "DELTA_FILE_NOT_FOUND", messageParameters = Array(path)) } def fileOrDirectoryNotFoundException(path: String): Throwable = { new DeltaFileNotFoundException( errorClass = "DELTA_FILE_OR_DIR_NOT_FOUND", messageParameters = Array(path)) } def excludeRegexOptionException(regexOption: String, cause: Throwable = null): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_REGEX_OPT_SYNTAX_ERROR", messageParameters = Array(regexOption), cause = cause) } def notADeltaTableException(deltaTableIdentifier: DeltaTableIdentifier): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_MISSING_DELTA_TABLE", messageParameters = Array(s"$deltaTableIdentifier")) } def notADeltaTableException( operation: String, deltaTableIdentifier: DeltaTableIdentifier): Throwable = { notADeltaTableException(operation, deltaTableIdentifier.toString) } def notADeltaTableException(operation: String, tableName: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_TABLE_ONLY_OPERATION", messageParameters = Array(tableName, operation)) } def notADeltaTableException(operation: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_ONLY_OPERATION", messageParameters = Array(operation) ) } def notADeltaSourceException(command: String, plan: Option[LogicalPlan] = None): Throwable = { val planName = if (plan.isDefined) plan.toString else "" new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_SOURCE", messageParameters = Array(command, s"$planName") ) } def partitionColumnCastFailed( columnValue: String, dataType: String, columnName: String): Throwable = { new DeltaRuntimeException( errorClass = "DELTA_PARTITION_COLUMN_CAST_FAILED", messageParameters = Array(columnValue, dataType, columnName)) } def schemaChangedSinceAnalysis( atAnalysis: StructType, latestSchema: StructType, mentionLegacyFlag: Boolean = false): Throwable = { val schemaDiff = SchemaUtils.reportDifferences(atAnalysis, latestSchema) .map(_.replace("Specified", "Latest")) val legacyFlagMessage = if (mentionLegacyFlag) { s""" |This check can be turned off by setting the session configuration key |${DeltaSQLConf.DELTA_SCHEMA_ON_READ_CHECK_ENABLED.key} to false.""".stripMargin } else { "" } new DeltaAnalysisException( errorClass = "DELTA_SCHEMA_CHANGE_SINCE_ANALYSIS", messageParameters = Array(schemaDiff.mkString("\n"), legacyFlagMessage) ) } def cloneWithRowTrackingWithoutStats(): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_CLONE_WITH_ROW_TRACKING_WITHOUT_STATS", messageParameters = Array.empty ) } def incorrectArrayAccess(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_INCORRECT_ARRAY_ACCESS", messageParameters = Array.empty) } def invalidColumnName(name: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_INVALID_CHARACTERS_IN_COLUMN_NAME", messageParameters = Array(name)) } def invalidInventorySchema(expectedSchema: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_INVALID_INVENTORY_SCHEMA", messageParameters = Array(expectedSchema) ) } def invalidIsolationLevelException(s: String): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_INVALID_ISOLATION_LEVEL", messageParameters = Array(s)) } def invalidPartitionColumn(col: String, tbl: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_INVALID_PARTITION_COLUMN", messageParameters = Array(col, tbl)) } def invalidPartitionColumn(e: AnalysisException): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_INVALID_PARTITION_COLUMN_NAME", messageParameters = Array.empty, cause = Option(e)) } def invalidTimestampFormat( ts: String, format: String, cause: Option[Throwable] = None): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_INVALID_TIMESTAMP_FORMAT", messageParameters = Array(ts, format), cause = cause) } def missingTableIdentifierException(operationName: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_OPERATION_MISSING_PATH", messageParameters = Array(operationName) ) } def unsupportedDeepCloneException(): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_UNSUPPORTED_DEEP_CLONE", messageParameters = Array.empty ) } def viewInDescribeDetailException(view: TableIdentifier): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_DESCRIBE_DETAIL_VIEW", messageParameters = Array(s"$view") ) } def addCommentToMapArrayException(fieldPath: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_COMMENT_MAP_ARRAY", messageParameters = Array(fieldPath) ) } def alterTableChangeColumnException( fieldPath: String, oldField: StructField, newField: StructField): Throwable = { def fieldToString(field: StructField): String = field.dataType.sql + (if (!field.nullable) " NOT NULL" else "") new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_ALTER_TABLE_CHANGE_COL_OP", messageParameters = Array( fieldPath, fieldToString(oldField), fieldToString(newField)) ) } def alterTableReplaceColumnsException( oldSchema: StructType, newSchema: StructType, reason: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_ALTER_TABLE_REPLACE_COL_OP", messageParameters = Array(reason, formatSchema(oldSchema), formatSchema(newSchema)) ) } def unsupportedTypeChangeInPreview( fieldPath: Seq[String], fromType: DataType, toType: DataType, feature: TypeWideningTableFeatureBase): Throwable = new DeltaUnsupportedOperationException( errorClass = "DELTA_UNSUPPORTED_TYPE_CHANGE_IN_PREVIEW", messageParameters = Array( SchemaUtils.prettyFieldName(fieldPath), fromType.sql, toType.sql, feature.name )) def unsupportedTypeChangeInSchema( fieldPath: Seq[String], fromType: DataType, toType: DataType) : Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_UNSUPPORTED_TYPE_CHANGE_IN_SCHEMA", messageParameters = Array(SchemaUtils.prettyFieldName(fieldPath), fromType.sql, toType.sql) ) } def cannotWriteIntoView(table: TableIdentifier): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CANNOT_WRITE_INTO_VIEW", messageParameters = Array(s"$table") ) } def castingCauseOverflowErrorInTableWrite( from: DataType, to: DataType, columnName: String): ArithmeticException = { new DeltaArithmeticException( errorClass = "DELTA_CAST_OVERFLOW_IN_TABLE_WRITE", messageParameters = Array( toSQLType(from), // sourceType toSQLType(to), // targetType toSQLId(columnName), // columnName SQLConf.STORE_ASSIGNMENT_POLICY.key, // storeAssignmentPolicyFlag // updateAndMergeCastingFollowsAnsiEnabledFlag DeltaSQLConf.UPDATE_AND_MERGE_CASTING_FOLLOWS_ANSI_ENABLED_FLAG.key, SQLConf.ANSI_ENABLED.key // ansiEnabledFlag ) ) } def notADeltaTable(table: String): Throwable = { new DeltaAnalysisException(errorClass = "DELTA_NOT_A_DELTA_TABLE", messageParameters = Array(table)) } def unsupportedWriteStagedTable(tableName: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_WRITES_STAGED_TABLE", messageParameters = Array(tableName) ) } def notEnoughColumnsInInsert( table: String, query: Int, target: Int, nestedField: Option[String] = None): Throwable = { val nestedFieldStr = nestedField.map(f => s"not enough nested fields in $f") .getOrElse("not enough data columns") new DeltaAnalysisException( errorClass = "DELTA_INSERT_COLUMN_ARITY_MISMATCH", messageParameters = Array(table, nestedFieldStr, target.toString, query.toString)) } def notFoundFileToBeRewritten(absolutePath: String, candidates: Iterable[String]): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_FILE_TO_OVERWRITE_NOT_FOUND", messageParameters = Array(absolutePath, candidates.mkString("\n"))) } def cannotFindSourceVersionException(json: String): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_CANNOT_FIND_VERSION", messageParameters = Array(json)) } def cannotInsertIntoColumn( tableName: String, source: String, target: String, targetType: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_COLUMN_STRUCT_TYPE_MISMATCH", messageParameters = Array(source, targetType, target, tableName)) } def ambiguousPartitionColumnException( columnName: String, colMatches: Seq[StructField]): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_AMBIGUOUS_PARTITION_COLUMN", messageParameters = Array(formatColumn(columnName).toString, formatColumnList(colMatches.map(_.name))) ) } def tableNotSupportedException(operation: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_TABLE_NOT_SUPPORTED_IN_OP", messageParameters = Array(operation) ) } def vacuumBasePathMissingException(baseDeltaPath: Path): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_VACUUM_SPECIFIC_PARTITION", messageParameters = Array(s"$baseDeltaPath") ) } def unexpectedDataChangeException(op: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_DATA_CHANGE_FALSE", messageParameters = Array(op) ) } def unknownConfigurationKeyException(confKey: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNKNOWN_CONFIGURATION", messageParameters = Array(confKey, DeltaSQLConf.ALLOW_ARBITRARY_TABLE_PROPERTIES.key)) } def cdcNotAllowedInThisVersion(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CDC_NOT_ALLOWED_IN_THIS_VERSION", messageParameters = Array.empty ) } def cdcWriteNotAllowedInThisVersion(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CHANGE_TABLE_FEED_DISABLED", messageParameters = Array.empty ) } def pathNotSpecifiedException: Throwable = { new IllegalArgumentException("'path' is not specified") } def pathNotExistsException(path: String): Throwable = { new DeltaAnalysisException(errorClass = "DELTA_PATH_DOES_NOT_EXIST", messageParameters = Array(path)) } def directoryNotFoundException(path: String): Throwable = { new FileNotFoundException(s"$path doesn't exist") } def pathAlreadyExistsException(path: Path): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_PATH_EXISTS", messageParameters = Array(s"$path") ) } def truncatedTransactionLogException( path: Path, version: Long, metadata: Metadata): Throwable = { val logRetention = DeltaConfigs.LOG_RETENTION.fromMetaData(metadata) val checkpointRetention = DeltaConfigs.CHECKPOINT_RETENTION_DURATION.fromMetaData(metadata) new DeltaFileNotFoundException( errorClass = "DELTA_TRUNCATED_TRANSACTION_LOG", messageParameters = Array( path.toString, version.toString, DeltaConfigs.LOG_RETENTION.key, logRetention.toString, DeltaConfigs.CHECKPOINT_RETENTION_DURATION.key, checkpointRetention.toString) ) } def logFileNotFoundException( path: Path, version: Option[Long], checkpointVersion: Long): Throwable = { new DeltaFileNotFoundException( errorClass = "DELTA_LOG_FILE_NOT_FOUND", messageParameters = Array( version.map(_.toString).getOrElse("LATEST"), checkpointVersion.toString, path.toString) ) } def logFileNotFoundExceptionForStreamingSource(e: FileNotFoundException): Throwable = { new DeltaFileNotFoundException( errorClass = "DELTA_LOG_FILE_NOT_FOUND_FOR_STREAMING_SOURCE", messageParameters = Array.empty ).initCause(e) } def logFailedIntegrityCheck(version: Long, mismatchOption: String): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_TXN_LOG_FAILED_INTEGRITY", messageParameters = Array(version.toString, mismatchOption) ) } def checkpointNonExistTable(path: Path): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_CHECKPOINT_NON_EXIST_TABLE", messageParameters = Array(s"$path")) } def multipleLoadPathsException(paths: Seq[String]): Throwable = { new DeltaAnalysisException( errorClass = "MULTIPLE_LOAD_PATH", messageParameters = Array(paths.mkString("[", ",", "]"))) } def partitionColumnNotFoundException(colName: String, schema: Seq[Attribute]): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_PARTITION_COLUMN_NOT_FOUND", messageParameters = Array( s"${formatColumn(colName)}", s"${schema.map(_.name).mkString(", ")}" ) ) } def partitionPathParseException(fragment: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_INVALID_PARTITION_PATH", messageParameters = Array(fragment)) } def partitionPathInvolvesNonPartitionColumnException( badColumns: Seq[String], fragment: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_NON_PARTITION_COLUMN_SPECIFIED", messageParameters = Array(formatColumnList(badColumns), fragment) ) } def unsupportedPartitionColumnChange( operation: String, oldPartitionColumns: Seq[String], newPartitionColumns: Seq[String]): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_PARTITION_COLUMN_CHANGE", messageParameters = Array( operation, oldPartitionColumns.mkString(", "), newPartitionColumns.mkString(", ") ) ) } def nonPartitionColumnAbsentException(colsDropped: Boolean): Throwable = { val msg = if (colsDropped) { " Columns which are of NullType have been dropped." } else { "" } new DeltaAnalysisException( errorClass = "DELTA_NON_PARTITION_COLUMN_ABSENT", messageParameters = Array(msg) ) } def replaceWhereMismatchException( replaceWhere: String, invariantViolation: InvariantViolationException): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_REPLACE_WHERE_MISMATCH", messageParameters = Array(replaceWhere, invariantViolation.getMessage), cause = Some(invariantViolation)) } def replaceWhereMismatchException(replaceWhere: String, badPartitions: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_REPLACE_WHERE_MISMATCH", messageParameters = Array(replaceWhere, s"Invalid data would be written to partitions $badPartitions.")) } def illegalFilesFound(file: String): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_ILLEGAL_FILE_FOUND", messageParameters = Array(file)) } def illegalDeltaOptionException(name: String, input: String, explain: String): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_ILLEGAL_OPTION", messageParameters = Array(input, name, explain)) } def invalidIdempotentWritesOptionsException(explain: String): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_INVALID_IDEMPOTENT_WRITES_OPTIONS", messageParameters = Array(explain)) } def invalidInterval(interval: String): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_INVALID_INTERVAL", messageParameters = Array(interval) ) } def invalidTableValueFunction(function: String) : Throwable = { new DeltaAnalysisException( errorClass = "DELTA_INVALID_TABLE_VALUE_FUNCTION", messageParameters = Array(function) ) } def startingVersionAndTimestampBothSetException( versionOptKey: String, timestampOptKey: String): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_STARTING_VERSION_AND_TIMESTAMP_BOTH_SET", messageParameters = Array(versionOptKey, timestampOptKey)) } def unrecognizedLogFile(path: Path): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_UNRECOGNIZED_LOGFILE", messageParameters = Array(s"$path") ) } def modifyAppendOnlyTableException(tableName: String): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_CANNOT_MODIFY_APPEND_ONLY", // `tableName` could be null here, so convert to string first. messageParameters = Array(s"$tableName", DeltaConfigs.IS_APPEND_ONLY.key) ) } def missingPartFilesException(version: Long, ae: Exception): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_MISSING_PART_FILES", messageParameters = Array(s"$version"), cause = ae ) } def deltaVersionsNotContiguousException( spark: SparkSession, deltaVersions: Seq[Long], startVersion: Long, endVersion: Long, versionToLoad: Long): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_VERSIONS_NOT_CONTIGUOUS", messageParameters = Array( deltaVersions.mkString(", "), startVersion.toString, endVersion.toString, versionToLoad.toString ) ) } def actionNotFoundException(action: String, version: Long): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_STATE_RECOVER_ERROR", messageParameters = Array(action, version.toString)) } def schemaChangedException( readSchema: StructType, dataSchema: StructType, retryable: Boolean, version: Option[Long], includeStartingVersionOrTimestampMessage: Boolean): Throwable = { def newException(errorClass: String, messageParameters: Array[String]): Throwable = { new DeltaIllegalStateException(errorClass, messageParameters) } if (version.isEmpty) { newException("DELTA_SCHEMA_CHANGED", Array( formatSchema(readSchema), formatSchema(dataSchema) )) } else if (!includeStartingVersionOrTimestampMessage) { newException("DELTA_SCHEMA_CHANGED_WITH_VERSION", Array( version.get.toString, formatSchema(readSchema), formatSchema(dataSchema) )) } else { newException("DELTA_SCHEMA_CHANGED_WITH_STARTING_OPTIONS", Array( version.get.toString, formatSchema(readSchema), formatSchema(dataSchema), version.get.toString )) } } def streamingSchemaMismatchOnRestart( querySchema: StructType, tableSchema: StructType): RuntimeException = { new DeltaIllegalStateException( errorClass = "DELTA_STREAMING_SCHEMA_MISMATCH_ON_RESTART", messageParameters = Array(formatSchema(querySchema), formatSchema(tableSchema))) } def streamWriteNullTypeException: Throwable = { new DeltaAnalysisException( errorClass = "DELTA_NULL_SCHEMA_IN_STREAMING_WRITE", messageParameters = Array.empty ) } def schemaNotSetException: Throwable = { new DeltaAnalysisException( errorClass = "DELTA_SCHEMA_NOT_SET", messageParameters = Array.empty ) } def specifySchemaAtReadTimeException: Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_SCHEMA_DURING_READ", messageParameters = Array.empty ) } def readSourceSchemaConflictException: Throwable = { new DeltaAnalysisException( errorClass = "DELTA_READ_SOURCE_SCHEMA_CONFLICT", messageParameters = Array.empty ) } def schemaNotProvidedException: Throwable = { new DeltaAnalysisException( errorClass = "DELTA_SCHEMA_NOT_PROVIDED", messageParameters = Array.empty) } def outputModeNotSupportedException(dataSource: String, outputMode: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_OUTPUT_MODE", messageParameters = Array(dataSource, outputMode) ) } def updateSetColumnNotFoundException(col: String, colList: Seq[String]): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_MISSING_SET_COLUMN", messageParameters = Array(formatColumn(col), formatColumnList(colList))) } def updateSetConflictException(cols: Seq[String]): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CONFLICT_SET_COLUMN", messageParameters = Array(formatColumnList(cols))) } def updateNonStructTypeFieldNotSupportedException(col: String, s: DataType): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_FIELD_UPDATE_NON_STRUCT", messageParameters = Array(s"${formatColumn(col)}", s"$s") ) } def truncateTablePartitionNotSupportedException: Throwable = { new DeltaAnalysisException( errorClass = "DELTA_TRUNCATE_TABLE_PARTITION_NOT_SUPPORTED", messageParameters = Array.empty ) } def bloomFilterOnPartitionColumnNotSupportedException(name: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_PARTITION_COLUMN_IN_BLOOM_FILTER", messageParameters = Array(name)) } def bloomFilterOnNestedColumnNotSupportedException(name: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_NESTED_COLUMN_IN_BLOOM_FILTER", messageParameters = Array(name)) } def bloomFilterOnColumnTypeNotSupportedException(name: String, dataType: DataType): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_COLUMN_TYPE_IN_BLOOM_FILTER", messageParameters = Array(s"${dataType.catalogString}", name)) } def bloomFilterMultipleConfForSingleColumnException(name: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_MULTIPLE_CONF_FOR_SINGLE_COLUMN_IN_BLOOM_FILTER", messageParameters = Array(name)) } def bloomFilterCreateOnNonExistingColumnsException(unknownColumns: Seq[String]): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CANNOT_CREATE_BLOOM_FILTER_NON_EXISTING_COL", messageParameters = Array(unknownColumns.mkString(", "))) } def bloomFilterInvalidParameterValueException(message: String): Throwable = { new DeltaAnalysisException( errorClass = "_LEGACY_ERROR_TEMP_DELTA_0002", messageParameters = Array(message) ) } def bloomFilterDropOnNonIndexedColumnException(name: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CANNOT_DROP_BLOOM_FILTER_ON_NON_INDEXED_COLUMN", messageParameters = Array(name)) } def bloomFilterDropOnNonExistingColumnsException(unknownColumns: Seq[String]): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_BLOOM_FILTER_DROP_ON_NON_EXISTING_COLUMNS", messageParameters = Array(unknownColumns.mkString(", ")) ) } def cannotRenamePath(tempPath: String, path: String): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_CANNOT_RENAME_PATH", messageParameters = Array(tempPath, path)) } def cannotSpecifyBothFileListAndPatternString(): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_FILE_LIST_AND_PATTERN_STRING_CONFLICT", messageParameters = Array.empty) } def cannotUpdateArrayField(table: String, field: String): Throwable = { new DeltaAnalysisException(errorClass = "DELTA_CANNOT_UPDATE_ARRAY_FIELD", messageParameters = Array(table, field, field)) } def cannotUpdateMapField(table: String, field: String): Throwable = { new DeltaAnalysisException(errorClass = "DELTA_CANNOT_UPDATE_MAP_FIELD", messageParameters = Array(table, field, field, field)) } def cannotUpdateStructField(table: String, field: String): Throwable = { new DeltaAnalysisException(errorClass = "DELTA_CANNOT_UPDATE_STRUCT_FIELD", messageParameters = Array(table, field)) } def cannotUpdateOtherField(tableName: String, dataType: DataType): Throwable = { new DeltaAnalysisException(errorClass = "DELTA_CANNOT_UPDATE_OTHER_FIELD", messageParameters = Array(tableName, s"$dataType")) } def cannotUseDataTypeForPartitionColumnError(field: StructField): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_INVALID_PARTITION_COLUMN_TYPE", messageParameters = Array(s"${field.name}", s"${field.dataType}") ) } def unexpectedPartitionSchemaFromUserException( catalogPartitionSchema: StructType, userPartitionSchema: StructType): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNEXPECTED_PARTITION_SCHEMA_FROM_USER", messageParameters = Array( formatSchema(catalogPartitionSchema), formatSchema(userPartitionSchema)) ) } def multipleSourceRowMatchingTargetRowInMergeException(spark: SparkSession): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_MULTIPLE_SOURCE_ROW_MATCHING_TARGET_ROW_IN_MERGE", messageParameters = Array(generateDocsLinkOption(spark, "/delta-update.html#upsert-into-a-table-using-merge").getOrElse("-")) ) } def sourceMaterializationFailedRepeatedlyInMerge: Throwable = new DeltaRuntimeException(errorClass = "DELTA_MERGE_MATERIALIZE_SOURCE_FAILED_REPEATEDLY") def sourceNotDeterministicInMergeException(spark: SparkSession): Throwable = { val docRefer = generateDocsLinkOption(spark, "/delta-update.html#operation-semantics") .map(link => s" Please refer to $link for more information.") .getOrElse("") new UnsupportedOperationException( s"Cannot perform Merge because the source dataset is not deterministic.$docRefer" ) } def mergeConcurrentOperationCachedSourceException(): Throwable = new DeltaRuntimeException(errorClass = "DELTA_MERGE_SOURCE_CACHED_DURING_EXECUTION") def columnOfTargetTableNotFoundInMergeException(targetCol: String, colNames: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_COLUMN_NOT_FOUND_IN_MERGE", messageParameters = Array(targetCol, colNames) ) } def subqueryNotSupportedException(op: String, cond: Expression): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_SUBQUERY", messageParameters = Array(op, cond.sql) ) } def multiColumnInPredicateNotSupportedException(operation: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_MULTI_COL_IN_PREDICATE", messageParameters = Array(operation) ) } def nestedFieldNotSupported(operation: String, field: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_NESTED_FIELD_IN_OPERATION", messageParameters = Array(operation, field) ) } def inSubqueryNotSupportedException(operation: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_IN_SUBQUERY", messageParameters = Array(operation)) } def convertMetastoreMetadataMismatchException( tableProperties: Map[String, String], deltaConfiguration: Map[String, String]): Throwable = { def prettyMap(m: Map[String, String]): String = { m.map(e => s"${e._1}=${e._2}").mkString("[", ", ", "]") } new DeltaAnalysisException( errorClass = "_LEGACY_ERROR_TEMP_DELTA_0003", messageParameters = Array( prettyMap(tableProperties), prettyMap(deltaConfiguration), DeltaSQLConf.DELTA_CONVERT_METADATA_CHECK_ENABLED.key) ) } def createExternalTableWithoutLogException( path: Path, tableName: String, spark: SparkSession): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CREATE_EXTERNAL_TABLE_WITHOUT_TXN_LOG", messageParameters = Array( tableName, path.toString, new Path(path, "_delta_log").toString, generateDocsLinkOption(spark, "/index.html").getOrElse("-"))) } def createExternalTableWithoutSchemaException( path: Path, tableName: String, spark: SparkSession): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CREATE_EXTERNAL_TABLE_WITHOUT_SCHEMA", messageParameters = Array(tableName, path.toString, generateDocsLinkOption(spark, "/index.html").getOrElse("-"))) } def createManagedTableWithoutSchemaException( tableName: String, spark: SparkSession): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_INVALID_MANAGED_TABLE_SYNTAX_NO_SCHEMA", messageParameters = Array(tableName, generateDocsLinkOption(spark, "/index.html").getOrElse("-")) ) } def readTableWithoutSchemaException(identifier: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_READ_TABLE_WITHOUT_COLUMNS", messageParameters = Array(identifier)) } def createTableWithDifferentSchemaException( path: Path, specifiedSchema: StructType, existingSchema: StructType, diffs: Seq[String]): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CREATE_TABLE_SCHEME_MISMATCH", messageParameters = Array(path.toString, specifiedSchema.treeString, existingSchema.treeString, diffs.map("\n".r.replaceAllIn(_, "\n ")).mkString("- ", "\n- ", ""))) } def createTableWithDifferentPartitioningException( path: Path, specifiedColumns: Seq[String], existingColumns: Seq[String]): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CREATE_TABLE_WITH_DIFFERENT_PARTITIONING", messageParameters = Array( path.toString, specifiedColumns.mkString(", "), existingColumns.mkString(", ") ) ) } def createTableWithDifferentPropertiesException( path: Path, specifiedProperties: Map[String, String], existingProperties: Map[String, String]): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CREATE_TABLE_WITH_DIFFERENT_PROPERTY", messageParameters = Array(path.toString, specifiedProperties.toSeq.sorted.map { case (k, v) => s"$k=$v" }.mkString("\n"), existingProperties.toSeq.sorted.map { case (k, v) => s"$k=$v" }.mkString("\n")) ) } def aggsNotSupportedException(op: String, cond: Expression): Throwable = { val condStr = s"(condition = ${cond.sql})" new DeltaAnalysisException( errorClass = "DELTA_AGGREGATION_NOT_SUPPORTED", messageParameters = Array(op, condStr) ) } def targetTableFinalSchemaEmptyException(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_TARGET_TABLE_FINAL_SCHEMA_EMPTY", messageParameters = Array.empty) } def nonDeterministicNotSupportedException(op: String, cond: Expression): Throwable = { val condStr = s"(condition = ${cond.sql})." new DeltaAnalysisException( errorClass = "DELTA_NON_DETERMINISTIC_FUNCTION_NOT_SUPPORTED", messageParameters = Array(op, s"$condStr") ) } def noHistoryFound(logPath: Path): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_NO_COMMITS_FOUND", messageParameters = Array(logPath.toString)) } def noRecreatableHistoryFound(logPath: Path): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_NO_RECREATABLE_HISTORY_FOUND", messageParameters = Array(s"$logPath")) } def unsupportedAbsPathAddFile(str: String): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_UNSUPPORTED_ABS_PATH_ADD_FILE", messageParameters = Array(str) ) } case class TimestampEarlierThanCommitRetentionException( userTimestamp: java.sql.Timestamp, commitTs: java.sql.Timestamp, timestampString: String) extends DeltaAnalysisException( errorClass = "DELTA_TIMESTAMP_EARLIER_THAN_COMMIT_RETENTION", messageParameters = Array(userTimestamp.toString, commitTs.toString, timestampString) ) def timestampGreaterThanLatestCommit( userTs: java.sql.Timestamp, lastCommitTs: java.sql.Timestamp, maximumTsStr: String): Throwable = { TemporallyUnstableInputException(userTs, lastCommitTs, maximumTsStr) } def timestampInvalid(expr: Expression): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_TIMESTAMP_INVALID", messageParameters = Array(s"${expr.sql}") ) } def versionInvalid(version: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_VERSION_INVALID", messageParameters = Array(s"$version") ) } case class TemporallyUnstableInputException( userTs: java.sql.Timestamp, lastCommitTs: java.sql.Timestamp, maximumTsStr: String) extends DeltaAnalysisException( errorClass = "DELTA_TIMESTAMP_GREATER_THAN_COMMIT", messageParameters = Array(s"$userTs", s"$lastCommitTs", maximumTsStr)) def restoreVersionNotExistException( userVersion: Long, earliest: Long, latest: Long): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CANNOT_RESTORE_TABLE_VERSION", messageParameters = Array(userVersion.toString, earliest.toString, latest.toString)) } def restoreTimestampGreaterThanLatestException( userTimestamp: String, latestTimestamp: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CANNOT_RESTORE_TIMESTAMP_GREATER", messageParameters = Array(userTimestamp, latestTimestamp) ) } def restoreTimestampBeforeEarliestException( userTimestamp: String, earliestTimestamp: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CANNOT_RESTORE_TIMESTAMP_EARLIER", messageParameters = Array(userTimestamp, earliestTimestamp) ) } def timeTravelNotSupportedException: Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_TIME_TRAVEL_VIEWS", messageParameters = Array.empty ) } def multipleTimeTravelSyntaxUsed: Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_TIME_TRAVEL_MULTIPLE_FORMATS", messageParameters = Array.empty ) } def timeTravelBeyondDeletedFileRetentionDurationException( deletedFileRetentionDurationHours: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_TIME_TRAVEL_BEYOND_DELETED_FILE_RETENTION_DURATION", messageParameters = Array(deletedFileRetentionDurationHours) ) } def nonExistentDeltaTable(tableId: DeltaTableIdentifier): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_TABLE_NOT_FOUND", messageParameters = Array(s"$tableId")) } def differentDeltaTableReadByStreamingSource( newTableId: String, oldTableId: String): Throwable = { new DeltaIllegalStateException( errorClass = "DIFFERENT_DELTA_TABLE_READ_BY_STREAMING_SOURCE", messageParameters = Array(newTableId, oldTableId)) } def nonExistentColumnInSchema(column: String, schema: String): Throwable = { new DeltaAnalysisException("DELTA_COLUMN_NOT_FOUND_IN_SCHEMA", Array(column, schema)) } def noRelationTable(tableIdent: Identifier): Throwable = { new DeltaNoSuchTableException( errorClass = "DELTA_NO_RELATION_TABLE", errorMessageParameters = Array(s"${tableIdent.quoted}")) } def provideOneOfInTimeTravel: Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_ONEOF_IN_TIMETRAVEL", messageParameters = Array.empty) } def emptyCalendarInterval: Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_INVALID_CALENDAR_INTERVAL_EMPTY", messageParameters = Array.empty ) } def unexpectedPartialScan(path: Path): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNEXPECTED_PARTIAL_SCAN", messageParameters = Array(s"$path") ) } def deltaLogAlreadyExistsException(path: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_LOG_ALREADY_EXISTS", messageParameters = Array(path) ) } def missingProviderForConvertException(path: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_MISSING_PROVIDER_FOR_CONVERT", messageParameters = Array(path)) } def convertNonParquetTablesException(ident: TableIdentifier, sourceName: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CONVERT_NON_PARQUET_TABLE", messageParameters = Array(sourceName, ident.toString) ) } def unexpectedPartitionColumnFromFileNameException( path: String, parsedCol: String, expectedCol: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNEXPECTED_PARTITION_COLUMN_FROM_FILE_NAME", messageParameters = Array( formatColumn(expectedCol), formatColumn(parsedCol), path) ) } def unexpectedNumPartitionColumnsFromFileNameException( path: String, parsedCols: Seq[String], expectedCols: Seq[String]): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNEXPECTED_NUM_PARTITION_COLUMNS_FROM_FILE_NAME", messageParameters = Array( expectedCols.size.toString, formatColumnList(expectedCols), parsedCols.size.toString, formatColumnList(parsedCols), path) ) } def castPartitionValueException(partitionValue: String, dataType: DataType): Throwable = { new DeltaRuntimeException( errorClass = "DELTA_FAILED_CAST_PARTITION_VALUE", messageParameters = Array(partitionValue, dataType.toString)) } def emptyDirectoryException(directory: String): Throwable = { new DeltaFileNotFoundException( errorClass = "DELTA_EMPTY_DIRECTORY", messageParameters = Array(directory) ) } def alterTableSetLocationSchemaMismatchException( original: StructType, destination: StructType): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_SET_LOCATION_SCHEMA_MISMATCH", messageParameters = Array(formatSchema(original), formatSchema(destination), DeltaSQLConf.DELTA_ALTER_LOCATION_BYPASS_SCHEMA_CHECK.key)) } def sparkSessionNotSetException(): Throwable = { new DeltaIllegalStateException(errorClass = "DELTA_SPARK_SESSION_NOT_SET") } def setLocationNotSupportedOnPathIdentifiers(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CANNOT_SET_LOCATION_ON_PATH_IDENTIFIER", messageParameters = Array.empty) } def useSetLocation(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CANNOT_CHANGE_LOCATION", messageParameters = Array.empty ) } def cannotSetLocationMultipleTimes(locations : Seq[String]) : Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_CANNOT_SET_LOCATION_MULTIPLE_TIMES", messageParameters = Array(s"${locations}") ) } def cannotReplaceMissingTableException(itableIdentifier: Identifier): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CANNOT_REPLACE_MISSING_TABLE", messageParameters = Array(itableIdentifier.toString)) } def cannotCreateLogPathException(logPath: String, cause: Throwable = null): Throwable = { new DeltaIOException( errorClass = "DELTA_CANNOT_CREATE_LOG_PATH", messageParameters = Array(logPath), cause = cause) } def cannotChangeProvider(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CANNOT_CHANGE_PROVIDER", messageParameters = Array.empty ) } def describeViewHistory: Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CANNOT_DESCRIBE_VIEW_HISTORY", messageParameters = Array.empty ) } def viewNotSupported(operationName: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_OPERATION_ON_VIEW_NOT_ALLOWED", messageParameters = Array(operationName) ) } def postCommitHookFailedException( failedHook: PostCommitHook, failedOnCommitVersion: Long, extraErrorMessage: String, error: Throwable): Throwable = { var errorMessage = "" if (extraErrorMessage != null && extraErrorMessage.nonEmpty) { errorMessage += s": $extraErrorMessage" } val ex = new DeltaRuntimeException( errorClass = "DELTA_POST_COMMIT_HOOK_FAILED", messageParameters = Array(s"$failedOnCommitVersion", failedHook.name, errorMessage) ) ex.initCause(error) ex } private def unsupportedModeException(modeName: String, supportedModes: String): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_MODE_NOT_SUPPORTED", messageParameters = Array(modeName, supportedModes)) } def unsupportedColumnMappingModeException(modeName: String): Throwable = { val supportedColumnMappingModes = DeltaColumnMapping.supportedModes.map(_.name).toSeq.mkString(", ") unsupportedModeException(modeName, supportedColumnMappingModes) } def unsupportedGenerateModeException(modeName: String): Throwable = { val supportedGenerateCommandModes = DeltaGenerateCommand.modeNameToGenerationFunc.keys.toSeq.mkString(", ") unsupportedModeException(modeName, supportedGenerateCommandModes) } def illegalUsageException(option: String, operation: String): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_ILLEGAL_USAGE", messageParameters = Array(option, operation)) } def foundMapTypeColumnException(key: String, value: String, schema: DataType): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_FOUND_MAP_TYPE_COLUMN", messageParameters = Array(key, value, dataTypeToString(schema)) ) } def columnNotInSchemaException(column: String, schema: DataType): Throwable = { nonExistentColumnInSchema(column, dataTypeToString(schema)) } def metadataAbsentException(): Throwable = { new DeltaIllegalStateException(errorClass = "DELTA_METADATA_ABSENT", messageParameters = Array.empty) } def metadataAbsentForExistingCatalogTable(tableName: String, tablePath: String): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_METADATA_ABSENT_EXISTING_CATALOG_TABLE", messageParameters = Array(tableName, tablePath, tableName)) } def deltaCannotVacuumLite(): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_CANNOT_VACUUM_LITE") } def vacuumRetentionPeriodNegative(): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_VACUUM_RETENTION_PERIOD_NEGATIVE") } def vacuumRetentionPeriodTooShort(configuredRetentionHours: Long): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_VACUUM_RETENTION_PERIOD_TOO_SHORT", messageParameters = Array(configuredRetentionHours.toString)) } def updateSchemaMismatchExpression(from: StructType, to: StructType): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UPDATE_SCHEMA_MISMATCH_EXPRESSION", messageParameters = Array(from.catalogString, to.catalogString) ) } def extractReferencesFieldNotFound(field: String, exception: Throwable): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_EXTRACT_REFERENCES_FIELD_NOT_FOUND", messageParameters = Array(field), cause = exception) } def addFilePartitioningMismatchException( addFilePartitions: Seq[String], metadataPartitions: Seq[String]): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_INVALID_PARTITIONING_SCHEMA", messageParameters = Array(s"${DeltaErrors.formatColumnList(metadataPartitions)}", s"${DeltaErrors.formatColumnList(addFilePartitions)}", s"${DeltaSQLConf.DELTA_COMMIT_VALIDATION_ENABLED.key}") ) } def concurrentModificationExceptionMsg( sparkConf: SparkConf, baseMessage: String, commit: Option[CommitInfo]): String = { baseMessage + commit.map(ci => s"\nConflicting commit: ${JsonUtils.toJson(ci)}").getOrElse("") + s"\nRefer to " + s"${DeltaErrors.generateDocsLink(sparkConf, "/concurrency-control.html")} " + "for more details." } def ignoreStreamingUpdatesAndDeletesWarning(spark: SparkSession): String = { val docPage = generateDocsLinkOption(spark, "/delta-streaming.html#ignoring-updates-and-deletes") .map(link => s" Refer to $link for details.") .getOrElse("") s"""WARNING: The 'ignoreFileDeletion' option is deprecated. Switch to using one of |'ignoreDeletes' or 'ignoreChanges'.$docPage """.stripMargin } def configureSparkSessionWithExtensionAndCatalog( originalException: Option[Throwable]): Throwable = { val catalogImplConfig = SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key new DeltaAnalysisException( errorClass = "DELTA_CONFIGURE_SPARK_SESSION_WITH_EXTENSION_AND_CATALOG", messageParameters = Array("io.delta.sql.DeltaSparkSessionExtension", catalogImplConfig, "org.apache.spark.sql.delta.catalog.DeltaCatalog", "io.delta.sql.DeltaSparkSessionExtension", catalogImplConfig, "org.apache.spark.sql.delta.catalog.DeltaCatalog"), cause = originalException) } def duplicateColumnsOnUpdateTable(originalException: Throwable): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_DUPLICATE_COLUMNS_ON_UPDATE_TABLE", messageParameters = Array(originalException.getMessage), cause = Some(originalException)) } def maxCommitRetriesExceededException( attemptNumber: Int, attemptVersion: Long, initAttemptVersion: Long, numActions: Int, totalCommitAttemptTime: Long): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_MAX_COMMIT_RETRIES_EXCEEDED", messageParameters = Array(s"$attemptNumber", s"$initAttemptVersion", s"$attemptVersion", s"$numActions", s"$totalCommitAttemptTime")) } def generatedColumnsReferToWrongColumns(e: AnalysisException): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_INVALID_GENERATED_COLUMN_REFERENCES", Array.empty, cause = Some(e)) } def generatedColumnsUpdateColumnType(current: StructField, update: StructField): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_GENERATED_COLUMN_UPDATE_TYPE_MISMATCH", messageParameters = Array( s"${current.name}", s"${current.dataType.sql}", s"${update.dataType.sql}" ) ) } def generatedColumnsUDF(expr: Expression): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UDF_IN_GENERATED_COLUMN", messageParameters = Array(s"${expr.sql}")) } def generatedColumnsNonDeterministicExpression(expr: Expression): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_NON_DETERMINISTIC_EXPRESSION_IN_GENERATED_COLUMN", messageParameters = Array(s"${expr.sql}")) } def generatedColumnsAggregateExpression(expr: Expression): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_AGGREGATE_IN_GENERATED_COLUMN", messageParameters = Array(expr.sql.toString) ) } def generatedColumnsUnsupportedExpression(expr: Expression): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_EXPRESSION_GENERATED_COLUMN", messageParameters = Array(s"${expr.sql}") ) } def generatedColumnsUnsupportedType(dt: DataType): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_DATA_TYPE_IN_GENERATED_COLUMN", messageParameters = Array(s"${dt.sql}") ) } def generatedColumnsExprTypeMismatch( column: String, columnType: DataType, exprType: DataType): Throwable = { val exprTypeSql = exprType.sql val columnTypeSql = columnType.sql val (exprTypeString, columnTypeString) = if (exprTypeSql == columnTypeSql) { // We need to add some more information for the error message to be useful. (exprType.json, columnType.json) } else { (exprTypeSql, columnTypeSql) } new DeltaAnalysisException( errorClass = "DELTA_GENERATED_COLUMNS_EXPR_TYPE_MISMATCH", messageParameters = Array(column, exprTypeString, columnTypeString) ) } def generatedColumnsDataTypeMismatch( columnPath: Seq[String], columnType: DataType, dataType: DataType, generatedColumns: Map[String, String]): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_GENERATED_COLUMNS_DATA_TYPE_MISMATCH", messageParameters = Array( SchemaUtils.prettyFieldName(columnPath), columnType.sql, dataType.sql, generatedColumns.mkString("\n")) ) } def constraintDataTypeMismatch( columnPath: Seq[String], columnType: DataType, dataType: DataType, constraints: Map[String, String]): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CONSTRAINT_DATA_TYPE_MISMATCH", messageParameters = Array( SchemaUtils.prettyFieldName(columnPath), columnType.sql, dataType.sql, constraints.mkString("\n")) ) } def expressionsNotFoundInGeneratedColumn(column: String): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_EXPRESSIONS_NOT_FOUND_IN_GENERATED_COLUMN", messageParameters = Array(column) ) } def cannotChangeDataType(msg: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CANNOT_CHANGE_DATA_TYPE", messageParameters = Array(msg) ) } def ambiguousDataTypeChange(column: String, from: StructType, to: StructType): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_AMBIGUOUS_DATA_TYPE_CHANGE", messageParameters = Array(column, from.toDDL, to.toDDL) ) } def unsupportedDataTypes( unsupportedDataType: UnsupportedDataTypeInfo, moreUnsupportedDataTypes: UnsupportedDataTypeInfo*): Throwable = { val prettyMessage = (unsupportedDataType +: moreUnsupportedDataTypes) .map(dt => s"${dt.column}: ${dt.dataType}") .mkString("[", ", ", "]") new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_DATA_TYPES", messageParameters = Array(prettyMessage, DeltaSQLConf.DELTA_SCHEMA_TYPE_CHECK.key) ) } def tableAlreadyExists(table: CatalogTable): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_TABLE_ALREADY_EXISTS", messageParameters = Array(s"${table.identifier.quotedString}") ) } def tableLocationMismatch(table: CatalogTable, existingTable: CatalogTable): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_TABLE_LOCATION_MISMATCH", messageParameters = Array( s"${table.identifier.quotedString}", s"`${existingTable.location}`", s"`${table.location}`") ) } def nonSinglePartNamespaceForCatalog(ident: String): Throwable = { new DeltaNoSuchTableException( errorClass = "DELTA_NON_SINGLE_PART_NAMESPACE_FOR_CATALOG", errorMessageParameters = Array(ident)) } def indexLargerThanStruct(pos: Int, column: StructField, len: Int): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_INDEX_LARGER_THAN_STRUCT", messageParameters = Array(s"$pos", s"$column", s"$len") ) } def indexLargerOrEqualThanStruct(pos: Int, len: Int): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_INDEX_LARGER_OR_EQUAL_THAN_STRUCT", messageParameters = Array(s"$pos", s"$len") ) } def invalidV1TableCall(callVersion: String, tableVersion: String): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_INVALID_V1_TABLE_CALL", messageParameters = Array(callVersion, tableVersion) ) } def cannotGenerateUpdateExpressions(): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_CANNOT_GENERATE_UPDATE_EXPRESSIONS", messageParameters = Array.empty ) } def unrecognizedInvariant(): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_UNRECOGNIZED_INVARIANT", messageParameters = Array.empty ) } def unrecognizedColumnChange(otherClass: String) : Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_UNRECOGNIZED_COLUMN_CHANGE", messageParameters = Array(otherClass) ) } def notNullColumnNotFoundInStruct(struct: String): Throwable = { new DeltaIndexOutOfBoundsException( errorClass = "DELTA_NOT_NULL_COLUMN_NOT_FOUND_IN_STRUCT", messageParameters = Array(struct) ) } def unSupportedInvariantNonStructType: Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_UNSUPPORTED_INVARIANT_NON_STRUCT", messageParameters = Array.empty ) } def cannotResolveColumn(fieldName: String, schema: StructType): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CANNOT_RESOLVE_COLUMN", messageParameters = Array(fieldName, schema.treeString) ) } def unsupportedTruncateSampleTables: Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_TRUNCATE_SAMPLE_TABLES", messageParameters = Array.empty ) } def unrecognizedFileAction(otherAction: String, otherClass: String) : Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_UNRECOGNIZED_FILE_ACTION", messageParameters = Array(otherAction, otherClass) ) } def operationOnTempViewWithGenerateColsNotSupported(op: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_OPERATION_ON_TEMP_VIEW_WITH_GENERATED_COLS_NOT_SUPPORTED", messageParameters = Array(op, op)) } def cannotModifyTableProperty(prop: String): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_CANNOT_MODIFY_TABLE_PROPERTY", messageParameters = Array(prop)) } /** * We have plans to support more column mapping modes, but they are not implemented yet, * so we error for now to be forward compatible with tables created in the future. */ def unsupportedColumnMappingMode(mode: String): Throwable = new ColumnMappingUnsupportedException(s"The column mapping mode `$mode` is " + s"not supported for this Delta version. Please upgrade if you want to use this mode.") def missingColumnId(mode: DeltaColumnMappingMode, field: String): Throwable = { ColumnMappingException(s"Missing column ID in column mapping mode `${mode.name}`" + s" in the field: $field", mode) } def missingPhysicalName(mode: DeltaColumnMappingMode, field: String): Throwable = ColumnMappingException(s"Missing physical name in column mapping mode `${mode.name}`" + s" in the field: $field", mode) def duplicatedColumnId( mode: DeltaColumnMappingMode, id: Long, schema: StructType): Throwable = { ColumnMappingException( s"Found duplicated column id `$id` in column mapping mode `${mode.name}` \n" + s"schema: \n ${schema.prettyJson}", mode ) } def duplicatedPhysicalName( mode: DeltaColumnMappingMode, physicalName: String, schema: StructType): Throwable = { ColumnMappingException( s"Found duplicated physical name `$physicalName` in column mapping mode `${mode.name}` \n\t" + s"schema: \n ${schema.prettyJson}", mode ) } def maxColumnIdNotSet: Throwable = { new DeltaAnalysisException( errorClass = "DELTA_COLUMN_MAPPING_MAX_COLUMN_ID_NOT_SET", messageParameters = Array(DeltaConfigs.COLUMN_MAPPING_MAX_ID.key) ) } def maxColumnIdNotSetCorrectly(tableMax: Long, fieldMax: Long): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_COLUMN_MAPPING_MAX_COLUMN_ID_NOT_SET_CORRECTLY", messageParameters = Array( DeltaConfigs.COLUMN_MAPPING_MAX_ID.key, tableMax.toString, fieldMax.toString) ) } def changeColumnMappingModeNotSupported(oldMode: String, newMode: String): Throwable = { new DeltaColumnMappingUnsupportedException( errorClass = "DELTA_UNSUPPORTED_COLUMN_MAPPING_MODE_CHANGE", messageParameters = Array(oldMode, newMode)) } def enablingColumnMappingDisallowedWhenColumnMappingMetadataAlreadyExists(): Throwable = { new DeltaColumnMappingUnsupportedException( errorClass = "DELTA_ENABLING_COLUMN_MAPPING_DISALLOWED_WHEN_COLUMN_MAPPING_METADATA_ALREADY_EXISTS") } def generateManifestWithColumnMappingNotSupported: Throwable = { new DeltaColumnMappingUnsupportedException( errorClass = "DELTA_UNSUPPORTED_MANIFEST_GENERATION_WITH_COLUMN_MAPPING") } def convertToDeltaNoPartitionFound(tableName: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CONVERSION_NO_PARTITION_FOUND", messageParameters = Array(tableName) ) } def convertToDeltaWithColumnMappingNotSupported(mode: DeltaColumnMappingMode): Throwable = { new DeltaColumnMappingUnsupportedException( errorClass = "DELTA_CONVERSION_UNSUPPORTED_COLUMN_MAPPING", messageParameters = Array( DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey, mode.name)) } protected def columnMappingAdviceMessage( requiredProtocol: Protocol = ColumnMappingTableFeature.minProtocolVersion): String = { val readerVersion = requiredProtocol.minReaderVersion val writerVersion = requiredProtocol.minWriterVersion s""" |Please enable Column Mapping on your Delta table with mapping mode 'name'. |You can use one of the following commands. | |ALTER TABLE table_name SET TBLPROPERTIES ('delta.columnMapping.mode' = 'name') | |Note, if your table is not on the required protocol version it will be upgraded. |Column mapping requires at least protocol ($readerVersion, $writerVersion) |""".stripMargin } def columnRenameNotSupported: Throwable = { val adviceMsg = columnMappingAdviceMessage() new DeltaAnalysisException("DELTA_UNSUPPORTED_RENAME_COLUMN", Array(adviceMsg)) } def dropColumnNotSupported(suggestUpgrade: Boolean): Throwable = { val adviceMsg = if (suggestUpgrade) columnMappingAdviceMessage() else "" new DeltaAnalysisException("DELTA_UNSUPPORTED_DROP_COLUMN", Array(adviceMsg)) } def dropNestedColumnsFromNonStructTypeException(struct : DataType) : Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_DROP_NESTED_COLUMN_FROM_NON_STRUCT_TYPE", messageParameters = Array(s"$struct") ) } def dropPartitionColumnNotSupported(droppingPartCols: Seq[String]): Throwable = { new DeltaAnalysisException("DELTA_UNSUPPORTED_DROP_PARTITION_COLUMN", Array(droppingPartCols.mkString(","))) } def schemaChangeDuringMappingModeChangeNotSupported( oldSchema: StructType, newSchema: StructType): Throwable = new DeltaColumnMappingUnsupportedException( errorClass = "DELTA_UNSUPPORTED_COLUMN_MAPPING_SCHEMA_CHANGE", messageParameters = Array( formatSchema(oldSchema), formatSchema(newSchema))) def foundInvalidCharsInColumnNames(invalidColumnNames: Seq[String]): Throwable = new DeltaAnalysisException( errorClass = "DELTA_INVALID_CHARACTERS_IN_COLUMN_NAMES", messageParameters = Array(invalidColumnNames.mkString(", "))) def foundInvalidColumnNamesWhenRemovingColumnMapping(columnNames: Seq[String]) : Throwable = new DeltaAnalysisException( errorClass = "DELTA_INVALID_COLUMN_NAMES_WHEN_REMOVING_COLUMN_MAPPING", messageParameters = Array(columnNames.mkString(", "))) def foundViolatingConstraintsForColumnChange( columnName: String, constraints: Map[String, String]): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CONSTRAINT_DEPENDENT_COLUMN_CHANGE", messageParameters = Array(columnName, constraints.mkString("\n")) ) } def foundViolatingGeneratedColumnsForColumnChange( columnName: String, generatedColumns: Map[String, String]): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_GENERATED_COLUMNS_DEPENDENT_COLUMN_CHANGE", messageParameters = Array(columnName, generatedColumns.mkString("\n")) ) } def missingColumnsInInsertInto(column: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_INSERT_COLUMN_MISMATCH", messageParameters = Array(column)) } def schemaNotConsistentWithTarget(tableSchema: String, targetAttr: String): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_SCHEMA_NOT_CONSISTENT_WITH_TARGET", messageParameters = Array(tableSchema, targetAttr) ) } def logStoreConfConflicts(classConf: Seq[(String, String)], schemeConf: Seq[(String, String)]): Throwable = { val classConfStr = classConf.map(_._1).mkString(", ") val schemeConfStr = schemeConf.map(_._1).mkString(", ") new DeltaAnalysisException( errorClass = "DELTA_INVALID_LOGSTORE_CONF", messageParameters = Array(classConfStr, schemeConfStr) ) } def inconsistentLogStoreConfs(setKeys: Seq[(String, String)]): Throwable = { val setKeyStr = setKeys.map(_.productIterator.mkString(" = ")).mkString(", ") new DeltaIllegalArgumentException( errorClass = "DELTA_INCONSISTENT_LOGSTORE_CONFS", messageParameters = Array(setKeyStr) ) } def ambiguousPathsInCreateTableException(identifier: String, location: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_AMBIGUOUS_PATHS_IN_CREATE_TABLE", messageParameters = Array(identifier, location, DeltaSQLConf.DELTA_LEGACY_ALLOW_AMBIGUOUS_PATHS.key)) } def concurrentWriteException( conflictingCommit: Option[CommitInfo]): io.delta.exceptions.ConcurrentWriteException = { new io.delta.exceptions.ConcurrentWriteException( Array( conflictingCommit.map(ci => s"\nConflicting commit: ${JsonUtils.toJson(ci)}").getOrElse(""), DeltaErrors.generateDocsLink(SparkEnv.get.conf, "/concurrency-control.html")) ) } def metadataChangedException( conflictingCommit: Option[CommitInfo]): io.delta.exceptions.MetadataChangedException = { new io.delta.exceptions.MetadataChangedException( Array( conflictingCommit.map(ci => s"\nConflicting commit: ${JsonUtils.toJson(ci)}").getOrElse(""), DeltaErrors.generateDocsLink(SparkEnv.get.conf, "/concurrency-control.html")) ) } def protocolPropNotIntException(key: String, value: String): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_PROTOCOL_PROPERTY_NOT_INT", Array(key, value)) } def protocolChangedException( conflictingCommit: Option[CommitInfo]): io.delta.exceptions.ProtocolChangedException = { val additionalInfo = conflictingCommit.map { v => if (v.version.getOrElse(-1) == 0) { "This happens when multiple writers are writing to an empty directory. " + "Creating the table ahead of time will avoid this conflict. " } else { "" } }.getOrElse("") new io.delta.exceptions.ProtocolChangedException( Array( additionalInfo, conflictingCommit.map(ci => s"\nConflicting commit: ${JsonUtils.toJson(ci)}").getOrElse(""), DeltaErrors.generateDocsLink(SparkEnv.get.conf, "/concurrency-control.html") ) ) } def unsupportedReaderTableFeaturesInTableException( tableNameOrPath: String, unsupported: Iterable[String]): DeltaUnsupportedTableFeatureException = { new DeltaUnsupportedTableFeatureException( errorClass = "DELTA_UNSUPPORTED_FEATURES_FOR_READ", tableNameOrPath = tableNameOrPath, unsupported = unsupported) } def unsupportedWriterTableFeaturesInTableException( tableNameOrPath: String, unsupported: Iterable[String]): DeltaUnsupportedTableFeatureException = { new DeltaUnsupportedTableFeatureException( errorClass = "DELTA_UNSUPPORTED_FEATURES_FOR_WRITE", tableNameOrPath = tableNameOrPath, unsupported = unsupported) } def unsupportedTableFeatureConfigsException( configs: Iterable[String]): DeltaTableFeatureException = { new DeltaTableFeatureException( errorClass = "DELTA_UNSUPPORTED_FEATURES_IN_CONFIG", messageParameters = Array(configs.mkString(", "))) } def unsupportedTableFeatureStatusException( feature: String, status: String): DeltaTableFeatureException = { new DeltaTableFeatureException( errorClass = "DELTA_UNSUPPORTED_FEATURE_STATUS", messageParameters = Array(feature, status)) } def tableFeatureReadRequiresWriteException( requiredWriterVersion: Int): DeltaTableFeatureException = { new DeltaTableFeatureException( errorClass = "DELTA_READ_FEATURE_PROTOCOL_REQUIRES_WRITE", messageParameters = Array( requiredWriterVersion.toString, generateDocsLinkOption(SparkSession.active, "/index.html").getOrElse("-"))) } def tableFeatureRequiresHigherReaderProtocolVersion( feature: String, currentVersion: Int, requiredVersion: Int): DeltaTableFeatureException = { new DeltaTableFeatureException( errorClass = "DELTA_FEATURE_REQUIRES_HIGHER_READER_VERSION", messageParameters = Array( feature, currentVersion.toString, requiredVersion.toString, generateDocsLinkOption(SparkSession.active, "/index.html").getOrElse("-"))) } def tableFeatureRequiresHigherWriterProtocolVersion( feature: String, currentVersion: Int, requiredVersion: Int): DeltaTableFeatureException = { new DeltaTableFeatureException( errorClass = "DELTA_FEATURE_REQUIRES_HIGHER_WRITER_VERSION", messageParameters = Array( feature, currentVersion.toString, requiredVersion.toString, generateDocsLinkOption(SparkSession.active, "/index.html").getOrElse("-"))) } def tableFeatureMismatchException(features: Iterable[String]): DeltaTableFeatureException = { new DeltaTableFeatureException( errorClass = "DELTA_FEATURES_PROTOCOL_METADATA_MISMATCH", messageParameters = Array(features.mkString(", "))) } def tableFeaturesRequireManualEnablementException( unsupportedFeatures: Iterable[TableFeature], supportedFeatures: Iterable[TableFeature]): Throwable = { new DeltaTableFeatureException( errorClass = "DELTA_FEATURES_REQUIRE_MANUAL_ENABLEMENT", messageParameters = Array( unsupportedFeatures.map(_.name).toSeq.sorted.mkString(", "), supportedFeatures.map(_.name).toSeq.sorted.mkString(", "))) } case class LogRetentionConfig(key: String, value: String, truncateHistoryRetention: String) private def logRetentionConfig(metadata: Metadata): LogRetentionConfig = { val logRetention = DeltaConfigs.LOG_RETENTION val truncateHistoryRetention = DeltaConfigs.TABLE_FEATURE_DROP_TRUNCATE_HISTORY_LOG_RETENTION LogRetentionConfig( logRetention.key, logRetention.fromMetaData(metadata).toString, truncateHistoryRetention.fromMetaData(metadata).toString) } def dropTableFeatureHistoricalVersionsExist( feature: String, metadata: Metadata): DeltaTableFeatureException = { val config = logRetentionConfig(metadata) new DeltaTableFeatureException( errorClass = "DELTA_FEATURE_DROP_HISTORICAL_VERSIONS_EXIST", messageParameters = Array(feature, config.key, config.value, config.truncateHistoryRetention) ) } def dropTableFeatureWaitForRetentionPeriod( feature: String, metadata: Metadata): DeltaTableFeatureException = { val config = logRetentionConfig(metadata) new DeltaTableFeatureException( errorClass = "DELTA_FEATURE_DROP_WAIT_FOR_RETENTION_PERIOD", messageParameters = Array(feature, config.key, config.value, config.truncateHistoryRetention) ) } def dropCheckpointProtectionWaitForRetentionPeriod( metadata: Metadata): DeltaTableFeatureException = { val config = logRetentionConfig(metadata) new DeltaTableFeatureException( errorClass = "DELTA_FEATURE_DROP_CHECKPOINT_PROTECTION_WAIT_FOR_RETENTION_PERIOD", messageParameters = Array(config.truncateHistoryRetention)) } def tableFeatureDropHistoryTruncationNotAllowed(): DeltaTableFeatureException = { new DeltaTableFeatureException( errorClass = "DELTA_FEATURE_DROP_HISTORY_TRUNCATION_NOT_ALLOWED", messageParameters = Array.empty) } def dropTableFeatureNonRemovableFeature(feature: String): DeltaTableFeatureException = { new DeltaTableFeatureException( errorClass = "DELTA_FEATURE_DROP_NONREMOVABLE_FEATURE", messageParameters = Array(feature)) } def dropTableFeatureFailedBecauseOfDependentFeatures( feature: String, dependentFeatures: Seq[String]): DeltaTableFeatureException = { new DeltaTableFeatureException( errorClass = "DELTA_FEATURE_DROP_DEPENDENT_FEATURE", messageParameters = Array(feature, dependentFeatures.mkString(", "), feature)) } def dropTableFeatureConflictRevalidationFailed( conflictingCommit: Option[CommitInfo] = None): DeltaTableFeatureException = { val concurrentCommit = DeltaErrors.concurrentModificationExceptionMsg( SparkEnv.get.conf, "", conflictingCommit) new DeltaTableFeatureException( errorClass = "DELTA_FEATURE_DROP_CONFLICT_REVALIDATION_FAIL", messageParameters = Array(concurrentCommit)) } def dropTableFeatureFeatureNotSupportedByClient( feature: String): DeltaTableFeatureException = { new DeltaTableFeatureException( errorClass = "DELTA_FEATURE_DROP_UNSUPPORTED_CLIENT_FEATURE", messageParameters = Array(feature)) } def dropTableFeatureFeatureNotSupportedByProtocol( feature: String): DeltaTableFeatureException = { new DeltaTableFeatureException( errorClass = "DELTA_FEATURE_DROP_FEATURE_NOT_PRESENT", messageParameters = Array(feature)) } def dropTableFeatureFeatureIsADeltaProperty(feature: String): DeltaTableFeatureException = { new DeltaTableFeatureException( errorClass = "DELTA_FEATURE_DROP_FEATURE_IS_DELTA_PROPERTY", messageParameters = Array(feature)) } def dropTableFeatureNotDeltaTableException(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_ONLY_OPERATION", messageParameters = Array("ALTER TABLE DROP FEATURE") ) } def dropTableFeatureCheckpointFailedException(featureName: String): Throwable = { new DeltaTableFeatureException( errorClass = "DELTA_FEATURE_DROP_CHECKPOINT_FAILED", messageParameters = Array(featureName)) } def canOnlyDropCheckpointProtectionWithHistoryTruncationException: DeltaTableFeatureException = { new DeltaTableFeatureException( errorClass = "DELTA_FEATURE_CAN_ONLY_DROP_CHECKPOINT_PROTECTION_WITH_HISTORY_TRUNCATION", messageParameters = Array.empty) } def concurrentAppendException( commitInfo: Option[CommitInfo], tableName: String, version: Long, partitionOpt: Option[String]): io.delta.exceptions.ConcurrentAppendException = { val operation = commitInfo.map(_.operation).getOrElse("TRANSACTION") val docLink = DeltaErrors.generateDocsLink(SparkEnv.get.conf, "/concurrency-control.html") val subClass = if (partitionOpt.nonEmpty) { "WITH_PARTITION_HINT" } else { "WITHOUT_HINT" } val messageParameters = subClass match { case "WITH_PARTITION_HINT" => Array(operation, tableName, version.toString, partitionOpt.getOrElse(""), docLink) case _ => Array(operation, tableName, version.toString, docLink) } io.delta.exceptions.ConcurrentAppendException(subClass, messageParameters) } def concurrentDeleteReadException( commitInfo: Option[CommitInfo], tableName: String, version: Long, partitionOpt: Option[String]): io.delta.exceptions.ConcurrentDeleteReadException = { val operation = commitInfo.map(_.operation).getOrElse("TRANSACTION") val docLink = DeltaErrors.generateDocsLink(SparkEnv.get.conf, "/concurrency-control.html") val subClass = if (partitionOpt.nonEmpty) { "WITH_PARTITION_HINT" } else { "WITHOUT_HINT" } val messageParameters = subClass match { case "WITH_PARTITION_HINT" => Array(operation, tableName, version.toString, partitionOpt.getOrElse(""), docLink) case _ => Array(operation, tableName, version.toString, docLink) } io.delta.exceptions.ConcurrentDeleteReadException(subClass, messageParameters) } def concurrentDeleteDeleteException( commitInfo: Option[CommitInfo], tableName: String, version: Long, partitionOpt: Option[String]): io.delta.exceptions.ConcurrentDeleteDeleteException = { val operation = commitInfo.map(_.operation).getOrElse("TRANSACTION") val docLink = DeltaErrors.generateDocsLink(SparkEnv.get.conf, "/concurrency-control.html") val subClass = if (partitionOpt.nonEmpty) { "WITH_PARTITION_HINT" } else { "WITHOUT_HINT" } val messageParameters = subClass match { case "WITH_PARTITION_HINT" => Array(operation, tableName, version.toString, partitionOpt.getOrElse(""), docLink) case _ => Array(operation, tableName, version.toString, docLink) } io.delta.exceptions.ConcurrentDeleteDeleteException(subClass, messageParameters) } def concurrentTransactionException( conflictingCommit: Option[CommitInfo]): io.delta.exceptions.ConcurrentTransactionException = { new io.delta.exceptions.ConcurrentTransactionException( Array( conflictingCommit.map(ci => s"\nConflicting commit: ${JsonUtils.toJson(ci)}").getOrElse(""), DeltaErrors.generateDocsLink(SparkEnv.get.conf, "/concurrency-control.html")) ) } def restoreMissedDataFilesError(missedFiles: Array[String], version: Long): Throwable = new IllegalArgumentException( s"""Not all files from version $version are available in file system. | Missed files (top 100 files): ${missedFiles.mkString(",")}. | Please use more recent version or timestamp for restoring. | To disable check update option ${SQLConf.IGNORE_MISSING_FILES.key}""" .stripMargin ) def unexpectedAlias(alias : String) : Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_UNEXPECTED_ALIAS", messageParameters = Array(alias) ) } def unexpectedProject(project : String) : Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_UNEXPECTED_PROJECT", messageParameters = Array(project) ) } def unexpectedAttributeReference(ref: String): Throwable = { new DeltaIllegalStateException(errorClass = "DELTA_UNEXPECTED_ATTRIBUTE_REFERENCE", messageParameters = Array(ref)) } def unsetNonExistentProperty(key: String, table: String): Throwable = { new DeltaAnalysisException(errorClass = "DELTA_UNSET_NON_EXISTENT_PROPERTY", Array(key, table)) } def identityColumnWithGenerationExpression(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_IDENTITY_COLUMNS_WITH_GENERATED_EXPRESSION", Array.empty) } def identityColumnIllegalStep(): Throwable = { new DeltaAnalysisException(errorClass = "DELTA_IDENTITY_COLUMNS_ILLEGAL_STEP", Array.empty) } def identityColumnDataTypeNotSupported(unsupportedType: DataType): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_IDENTITY_COLUMNS_UNSUPPORTED_DATA_TYPE", messageParameters = Array(unsupportedType.typeName) ) } def identityColumnAlterNonIdentityColumnError(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_IDENTITY_COLUMNS_ALTER_NON_IDENTITY_COLUMN", messageParameters = Array.empty) } def identityColumnAlterNonDeltaFormatError(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_IDENTITY_COLUMNS_ALTER_NON_DELTA_FORMAT", messageParameters = Array.empty) } def identityColumnInconsistentMetadata( colName: String, hasStart: Boolean, hasStep: Boolean, hasInsert: Boolean): Throwable = { new DeltaAnalysisException( errorClass = "_LEGACY_ERROR_TEMP_DELTA_0006", messageParameters = Array(colName, s"$hasStart", s"$hasStep", s"$hasInsert") ) } def identityColumnExplicitInsertNotSupported(colName: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_IDENTITY_COLUMNS_EXPLICIT_INSERT_NOT_SUPPORTED", messageParameters = Array(colName)) } def identityColumnAlterColumnNotSupported(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_IDENTITY_COLUMNS_ALTER_COLUMN_NOT_SUPPORTED", messageParameters = Array.empty) } def identityColumnReplaceColumnsNotSupported(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_IDENTITY_COLUMNS_REPLACE_COLUMN_NOT_SUPPORTED", messageParameters = Array.empty) } def identityColumnPartitionNotSupported(colName: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_IDENTITY_COLUMNS_PARTITION_NOT_SUPPORTED", messageParameters = Array(colName)) } def identityColumnUpdateNotSupported(colName: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_IDENTITY_COLUMNS_UPDATE_NOT_SUPPORTED", messageParameters = Array(colName)) } def activeSparkSessionNotFound(): Throwable = { new DeltaIllegalArgumentException(errorClass = "DELTA_ACTIVE_SPARK_SESSION_NOT_FOUND") } def sparkTaskThreadNotFound: Throwable = { new DeltaIllegalStateException(errorClass = "DELTA_SPARK_THREAD_NOT_FOUND") } def iteratorAlreadyClosed(): Throwable = { new DeltaIllegalStateException(errorClass = "DELTA_ITERATOR_ALREADY_CLOSED") } def activeTransactionAlreadySet(): Throwable = { new DeltaIllegalStateException(errorClass = "DELTA_ACTIVE_TRANSACTION_ALREADY_SET") } def deltaStatsCollectionColumnNotFound(statsType: String, columnPath: String): Throwable = { new DeltaRuntimeException( errorClass = "DELTA_STATS_COLLECTION_COLUMN_NOT_FOUND", messageParameters = Array(statsType, columnPath) ) } def convertToDeltaRowTrackingEnabledWithoutStatsCollection: Throwable = { val statisticsCollectionPropertyKey = DeltaSQLConf.DELTA_COLLECT_STATS.key val rowTrackingTableFeatureDefaultKey = TableFeatureProtocolUtils.defaultPropertyKey(RowTrackingFeature) val rowTrackingDefaultPropertyKey = DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey new DeltaIllegalStateException( errorClass = "DELTA_CONVERT_TO_DELTA_ROW_TRACKING_WITHOUT_STATS", messageParameters = Array( statisticsCollectionPropertyKey, rowTrackingTableFeatureDefaultKey, rowTrackingDefaultPropertyKey)) } def rowTrackingBackfillRunningConcurrentlyWithUnbackfill(): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_ROW_TRACKING_BACKFILL_RUNNING_CONCURRENTLY_WITH_UNBACKFILL") } def rowTrackingIllegalPropertyCombination(): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_ROW_TRACKING_ILLEGAL_PROPERTY_COMBINATION", messageParameters = Array( DeltaConfigs.ROW_TRACKING_ENABLED.key, DeltaConfigs.ROW_TRACKING_SUSPENDED.key)) } /** This is a method only used for testing Py4J exception handling. */ def throwDeltaIllegalArgumentException(): Throwable = { new DeltaIllegalArgumentException(errorClass = "DELTA_UNRECOGNIZED_INVARIANT") } def invalidSourceVersion(version: String): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_INVALID_SOURCE_VERSION", messageParameters = Array(version) ) } def invalidSourceOffsetFormat(): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_INVALID_SOURCE_OFFSET_FORMAT" ) } def invalidCommittedVersion(attemptVersion: Long, currentVersion: Long): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_INVALID_COMMITTED_VERSION", messageParameters = Array(attemptVersion.toString, currentVersion.toString) ) } def nonPartitionColumnReference(colName: String, partitionColumns: Seq[String]): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_NON_PARTITION_COLUMN_REFERENCE", messageParameters = Array(colName, partitionColumns.mkString(", ")) ) } def missingColumn(attr: Attribute, targetAttrs: Seq[Attribute]): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_MISSING_COLUMN", messageParameters = Array(attr.name, targetAttrs.map(_.name).mkString(", ")) ) } def missingPartitionColumn(col: String, schemaCatalog: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_MISSING_PARTITION_COLUMN", messageParameters = Array(col, schemaCatalog) ) } def noNewAttributeId(oldAttr: AttributeReference): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_NO_NEW_ATTRIBUTE_ID", messageParameters = Array(oldAttr.qualifiedName) ) } def nonGeneratedColumnMissingUpdateExpression(columnName: String): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_NON_GENERATED_COLUMN_MISSING_UPDATE_EXPR", messageParameters = Array(columnName) ) } def failedInferSchema: Throwable = { new DeltaRuntimeException("DELTA_FAILED_INFER_SCHEMA") } def failedReadFileFooter(file: String, e: Throwable): Throwable = { new DeltaIOException( errorClass = "DELTA_FAILED_READ_FILE_FOOTER", messageParameters = Array(file), cause = e ) } def failedScanWithHistoricalVersion(historicalVersion: Long): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_FAILED_SCAN_WITH_HISTORICAL_VERSION", messageParameters = Array(historicalVersion.toString) ) } def failedRecognizePredicate(predicate: String, cause: Throwable): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_FAILED_RECOGNIZE_PREDICATE", messageParameters = Array(predicate), cause = Some(cause) ) } def failedFindAttributeInOutputColumns(newAttrName: String, targetColNames: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_FAILED_FIND_ATTRIBUTE_IN_OUTPUT_COLUMNS", messageParameters = Array(newAttrName, targetColNames) ) } def failedFindPartitionColumnInOutputPlan(partitionColumn: String): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_FAILED_FIND_PARTITION_COLUMN_IN_OUTPUT_PLAN", messageParameters = Array(partitionColumn)) } def deltaTableFoundInExecutor(): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_TABLE_FOUND_IN_EXECUTOR", messageParameters = Array.empty ) } def variantShreddingUnsupported(): Throwable = { new DeltaSparkException( errorClass = "DELTA_SHREDDING_TABLE_PROPERTY_DISABLED", messageParameters = Array.empty ) } def unsupportSubqueryInPartitionPredicates(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_SUBQUERY_IN_PARTITION_PREDICATES", messageParameters = Array.empty ) } def fileAlreadyExists(file: String): Throwable = { new DeltaFileAlreadyExistsException( errorClass = "DELTA_FILE_ALREADY_EXISTS", messageParameters = Array(file) ) } def replaceWhereUsedWithDynamicPartitionOverwrite(): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_REPLACE_WHERE_WITH_DYNAMIC_PARTITION_OVERWRITE" ) } def overwriteSchemaUsedWithDynamicPartitionOverwrite(): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_OVERWRITE_SCHEMA_WITH_DYNAMIC_PARTITION_OVERWRITE" ) } def replaceWhereUsedInOverwrite(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_REPLACE_WHERE_IN_OVERWRITE", messageParameters = Array.empty ) } def deltaDynamicPartitionOverwriteDisabled(): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_DYNAMIC_PARTITION_OVERWRITE_DISABLED" ) } def incorrectArrayAccessByName( rightName: String, wrongName: String, schema: DataType): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_INCORRECT_ARRAY_ACCESS_BY_NAME", messageParameters = Array( rightName, wrongName, dataTypeToString(schema) ) ) } def columnPathNotNested( columnPath: String, other: DataType, column: Seq[String], schema: DataType): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_COLUMN_PATH_NOT_NESTED", messageParameters = Array( s"$columnPath", s"$other", s"${SchemaUtils.prettyFieldName(column)}", dataTypeToString(schema) ) ) } def showPartitionInNotPartitionedTable(tableName: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_SHOW_PARTITION_IN_NON_PARTITIONED_TABLE", messageParameters = Array(tableName) ) } def showPartitionInNotPartitionedColumn(badColumns: Set[String]): Throwable = { val badCols = badColumns.mkString("[", ", ", "]") new DeltaAnalysisException( errorClass = "DELTA_SHOW_PARTITION_IN_NON_PARTITIONED_COLUMN", messageParameters = Array(badCols) ) } def duplicateColumnOnInsert(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_DUPLICATE_COLUMNS_ON_INSERT", messageParameters = Array.empty ) } def timeTravelInvalidBeginValue(timeTravelKey: String, cause: Throwable): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_TIME_TRAVEL_INVALID_BEGIN_VALUE", messageParameters = Array(timeTravelKey), cause = cause ) } def removeFileCDCMissingExtendedMetadata(fileName: String): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_REMOVE_FILE_CDC_MISSING_EXTENDED_METADATA", messageParameters = Array(fileName) ) } def failRelativizePath(pathName: String): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_FAIL_RELATIVIZE_PATH", messageParameters = Array( pathName, DeltaSQLConf.DELTA_VACUUM_RELATIVIZE_IGNORE_ERROR.key) ) } def invalidFormatFromSourceVersion(wrongVersion: Long, expectedVersion: Integer): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_INVALID_FORMAT_FROM_SOURCE_VERSION", messageParameters = Array(expectedVersion.toString, wrongVersion.toString) ) } def createTableWithNonEmptyLocation(tableId: String, tableLocation: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CREATE_TABLE_WITH_NON_EMPTY_LOCATION", messageParameters = Array(tableId, tableLocation) ) } def maxArraySizeExceeded(): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_MAX_ARRAY_SIZE_EXCEEDED", messageParameters = Array.empty ) } def replaceWhereWithFilterDataChangeUnset(dataFilters: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_REPLACE_WHERE_WITH_FILTER_DATA_CHANGE_UNSET", messageParameters = Array(dataFilters) ) } def blockColumnMappingAndCdcOperation(op: DeltaOperations.Operation): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_BLOCK_COLUMN_MAPPING_AND_CDC_OPERATION", messageParameters = Array(op.name) ) } def missingDeltaStorageJar(e: NoClassDefFoundError): Throwable = { // scalastyle:off line.size.limit new NoClassDefFoundError( s"""${e.getMessage} |Please ensure that the delta-storage dependency is included. | |If using Python, please ensure you call `configure_spark_with_delta_pip` or use |`--packages io.delta:delta-spark_:`. |See https://docs.delta.io/latest/quick-start.html#python. | |More information about this dependency and how to include it can be found here: |https://docs.delta.io/latest/porting.html#delta-lake-1-1-or-below-to-delta-lake-1-2-or-above. |""".stripMargin) // scalastyle:on line.size.limit } /** * If `isSchemaChange` is false, this means the `incompatVersion` actually refers to a data schema * instead of a schema change. This happens when we could not find any read-incompatible schema * changes within the querying range, but the read schema is still NOT compatible with the data * files being queried, which could happen if user falls back to `legacy` mode and read past data * using some diverged latest schema or time-travelled schema. In this uncommon case, we should * tell the user to try setting it back to endVersion, OR ask us to give them the flag to force * unblock. */ def blockBatchCdfReadWithIncompatibleSchemaChange( start: Long, end: Long, readSchema: StructType, readVersion: Long, incompatVersion: Long, isSchemaChange: Boolean = true): Throwable = { new DeltaUnsupportedOperationException( if (isSchemaChange) { "DELTA_CHANGE_DATA_FEED_INCOMPATIBLE_SCHEMA_CHANGE" } else { "DELTA_CHANGE_DATA_FEED_INCOMPATIBLE_DATA_SCHEMA" }, messageParameters = Array( start.toString, end.toString, readSchema.json, readVersion.toString, incompatVersion.toString) ++ { if (isSchemaChange) { Array(start.toString, incompatVersion.toString, incompatVersion.toString, end.toString) } else { Array(DeltaSQLConf.DELTA_CDF_DEFAULT_SCHEMA_MODE_FOR_COLUMN_MAPPING_TABLE.key) } } ) } def blockStreamingReadsWithIncompatibleNonAdditiveSchemaChanges( spark: SparkSession, readSchema: StructType, incompatibleSchema: StructType, detectedDuringStreaming: Boolean, isV2DataSource: Boolean = false): Throwable = { val docLink = "/versioning.html#column-mapping" val enableNonAdditiveSchemaEvolution = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_STREAMING_ENABLE_SCHEMA_TRACKING) new DeltaStreamingNonAdditiveSchemaIncompatibleException( readSchema, incompatibleSchema, generateDocsLinkOption(spark, docLink).getOrElse("-"), enableNonAdditiveSchemaEvolution, additionalProperties = Map( "detectedDuringStreaming" -> detectedDuringStreaming.toString, "isV2DataSource" -> isV2DataSource.toString )) } def failedToGetSnapshotDuringColumnMappingStreamingReadCheck(cause: Throwable): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_STREAMING_CHECK_COLUMN_MAPPING_NO_SNAPSHOT", messageParameters = Array(DeltaSQLConf .DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_COLUMN_MAPPING_SCHEMA_CHANGES.key), cause = Some(cause)) } def unsupportedDeltaTableForPathHadoopConf(unsupportedOptions: Map[String, String]): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_TABLE_FOR_PATH_UNSUPPORTED_HADOOP_CONF", messageParameters = Array( DeltaTableUtils.validDeltaTableHadoopPrefixes.mkString("[", ",", "]"), unsupportedOptions.mkString(",")) ) } def cloneOnRelativePath(path: String): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_INVALID_CLONE_PATH", messageParameters = Array(path)) } def cloneAmbiguousTarget(externalLocation: String, targetIdent: TableIdentifier): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_CLONE_AMBIGUOUS_TARGET", messageParameters = Array(externalLocation, s"$targetIdent") ) } def cloneFromUnsupportedSource(name: String, format: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CLONE_UNSUPPORTED_SOURCE", messageParameters = Array(name, format) ) } def cloneReplaceUnsupported(tableIdentifier: TableIdentifier): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_UNSUPPORTED_CLONE_REPLACE_SAME_TABLE", messageParameters = Array(s"$tableIdentifier") ) } def cloneReplaceNonEmptyTable: Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_UNSUPPORTED_NON_EMPTY_CLONE" ) } def cloneFromIcebergSourceWithPartitionEvolution(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CLONE_INCOMPATIBLE_SOURCE.ICEBERG_UNDERGONE_PARTITION_EVOLUTION", messageParameters = Array() ) } def cloneFromIcebergSourceWithoutSpecs(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CLONE_INCOMPATIBLE_SOURCE.ICEBERG_MISSING_PARTITION_SPECS", messageParameters = Array() ) } def partitionSchemaInIcebergTables: Throwable = { new DeltaIllegalArgumentException(errorClass = "DELTA_PARTITION_SCHEMA_IN_ICEBERG_TABLES") } def icebergClassMissing(sparkConf: SparkConf, cause: Throwable): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_MISSING_ICEBERG_CLASS", messageParameters = Array( generateDocsLink( sparkConf, "/delta-utility.html#convert-a-parquet-table-to-a-delta-table")), cause = cause) } def hudiClassMissing(sparkConf: SparkConf, cause: Throwable): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_MISSING_HUDI_CLASS", messageParameters = Array( generateDocsLink( sparkConf, "/delta-utility.html#convert-a-parquet-table-to-a-delta-table")), cause = cause) } def streamingMetadataEvolutionException( newSchema: StructType, newConfigs: Map[String, String], newProtocol: Protocol): Throwable = { new DeltaRuntimeException( errorClass = "DELTA_STREAMING_METADATA_EVOLUTION", messageParameters = Array( formatSchema(newSchema), newConfigs.map { case (k, v) => s"$k:$v" }.mkString(", "), newProtocol.simpleString )) } def streamingMetadataLogInitFailedIncompatibleMetadataException( startVersion: Long, endVersion: Long): Throwable = { new DeltaRuntimeException( errorClass = "DELTA_STREAMING_SCHEMA_LOG_INIT_FAILED_INCOMPATIBLE_METADATA", messageParameters = Array(startVersion.toString, endVersion.toString) ) } def failToDeserializeSchemaLog(location: String): Throwable = { new DeltaRuntimeException( errorClass = "DELTA_STREAMING_SCHEMA_LOG_DESERIALIZE_FAILED", messageParameters = Array(location) ) } def failToParseSchemaLog: Throwable = { new DeltaRuntimeException(errorClass = "DELTA_STREAMING_SCHEMA_LOG_PARSE_SCHEMA_FAILED") } def sourcesWithConflictingSchemaTrackingLocation( schemaTrackingLocatiob: String, tableOrPath: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_STREAMING_SCHEMA_LOCATION_CONFLICT", messageParameters = Array(schemaTrackingLocatiob, tableOrPath)) } def incompatibleSchemaLogPartitionSchema( persistedPartitionSchema: StructType, tablePartitionSchema: StructType): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_STREAMING_SCHEMA_LOG_INCOMPATIBLE_PARTITION_SCHEMA", messageParameters = Array(persistedPartitionSchema.json, tablePartitionSchema.json)) } def incompatibleSchemaLogDeltaTable( persistedTableId: String, tableId: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_STREAMING_SCHEMA_LOG_INCOMPATIBLE_DELTA_TABLE_ID", messageParameters = Array(persistedTableId, tableId)) } def schemaTrackingLocationNotUnderCheckpointLocation( schemaTrackingLocation: String, checkpointLocation: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_STREAMING_SCHEMA_LOCATION_NOT_UNDER_CHECKPOINT", messageParameters = Array(schemaTrackingLocation, checkpointLocation)) } def cannotContinueStreamingPostSchemaEvolution( nonAdditiveSchemaChangeOpType: String, previousSchemaChangeVersion: Long, currentSchemaChangeVersion: Long, checkpointHash: Int, readerOptionsUnblock: Seq[String], sqlConfsUnblock: Seq[String], prettyColumnChangeDetails: String): Throwable = { val unblockChangeOptions = readerOptionsUnblock.map { option => s""" .option("$option", "$currentSchemaChangeVersion")""" }.mkString("\n") val unblockStreamOptions = readerOptionsUnblock.map { option => s""" .option("$option", "always")""" }.mkString("\n") val unblockChangeConfs = sqlConfsUnblock.map { conf => s""" SET $conf.ckpt_$checkpointHash = $currentSchemaChangeVersion;""" }.mkString("\n") val unblockStreamConfs = sqlConfsUnblock.map { conf => s""" SET $conf.ckpt_$checkpointHash = "always";""" }.mkString("\n") val unblockAllConfs = sqlConfsUnblock.map { conf => s""" SET $conf = "always";""" }.mkString("\n") new DeltaRuntimeException( errorClass = "DELTA_STREAMING_CANNOT_CONTINUE_PROCESSING_POST_SCHEMA_EVOLUTION", messageParameters = Array( nonAdditiveSchemaChangeOpType, previousSchemaChangeVersion.toString, currentSchemaChangeVersion.toString, prettyColumnChangeDetails, currentSchemaChangeVersion.toString, unblockChangeOptions, unblockStreamOptions, unblockChangeConfs, unblockStreamConfs, unblockAllConfs ) ) } def cannotReconstructPathFromURI(uri: String): Throwable = new DeltaRuntimeException( errorClass = "DELTA_CANNOT_RECONSTRUCT_PATH_FROM_URI", messageParameters = Array(uri)) def deletionVectorCardinalityMismatch(): Throwable = { new DeltaChecksumException( errorClass = "DELTA_DELETION_VECTOR_CARDINALITY_MISMATCH", messageParameters = Array.empty, pos = 0 ) } def deletionVectorSizeMismatch(): Throwable = { new DeltaChecksumException( errorClass = "DELTA_DELETION_VECTOR_SIZE_MISMATCH", messageParameters = Array.empty, pos = 0) } def deletionVectorInvalidRowIndex(): Throwable = { new DeltaChecksumException( errorClass = "DELTA_DELETION_VECTOR_INVALID_ROW_INDEX", messageParameters = Array.empty, pos = 0) } def deletionVectorChecksumMismatch(): Throwable = { new DeltaChecksumException( errorClass = "DELTA_DELETION_VECTOR_CHECKSUM_MISMATCH", messageParameters = Array.empty, pos = 0) } def statsRecomputeNotSupportedOnDvTables(): Throwable = { new DeltaCommandUnsupportedWithDeletionVectorsException( errorClass = "DELTA_UNSUPPORTED_STATS_RECOMPUTE_WITH_DELETION_VECTORS", messageParameters = Array.empty ) } def addFileWithDVsAndTightBoundsException(): Throwable = new DeltaIllegalStateException( errorClass = "DELTA_ADDING_DELETION_VECTORS_WITH_TIGHT_BOUNDS_DISALLOWED") def addFileWithDVsMissingNumRecordsException: Throwable = new DeltaRuntimeException(errorClass = "DELTA_DELETION_VECTOR_MISSING_NUM_RECORDS") def generateNotSupportedWithDeletionVectors(): Throwable = new DeltaCommandUnsupportedWithDeletionVectorsException( errorClass = "DELTA_UNSUPPORTED_GENERATE_WITH_DELETION_VECTORS") def addingDeletionVectorsDisallowedException(): Throwable = new DeltaCommandUnsupportedWithDeletionVectorsException( errorClass = "DELTA_ADDING_DELETION_VECTORS_DISALLOWED") def unsupportedExpression( causedBy: String, expType: DataType, supportedTypes: Seq[String]): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_UNSUPPORTED_EXPRESSION", messageParameters = Array(s"$expType", causedBy, supportedTypes.mkString(",")) ) } def rowIdAssignmentWithoutStats: Throwable = { new DeltaIllegalStateException(errorClass = "DELTA_ROW_ID_ASSIGNMENT_WITHOUT_STATS") } def addingColumnWithInternalNameFailed(colName: String): Throwable = { new DeltaRuntimeException( errorClass = "DELTA_ADDING_COLUMN_WITH_INTERNAL_NAME_FAILED", messageParameters = Array(colName) ) } def materializedRowIdMetadataMissing(tableName: String): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_MATERIALIZED_ROW_TRACKING_COLUMN_NAME_MISSING", messageParameters = Array("Row ID", tableName) ) } def materializedRowCommitVersionMetadataMissing(tableName: String): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_MATERIALIZED_ROW_TRACKING_COLUMN_NAME_MISSING", messageParameters = Array("Row Commit Version", tableName) ) } def domainMetadataDuplicate(domainName: String): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_DUPLICATE_DOMAIN_METADATA_INTERNAL_ERROR", messageParameters = Array(domainName) ) } def domainMetadataTableFeatureNotSupported(domainNames: String): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_DOMAIN_METADATA_NOT_SUPPORTED", messageParameters = Array(domainNames) ) } def uniFormIcebergRequiresIcebergCompat(): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_UNIVERSAL_FORMAT_VIOLATION", messageParameters = Array( UniversalFormat.ICEBERG_FORMAT, "Requires IcebergCompat to be explicitly enabled in order for Universal Format (Iceberg) " + "to be enabled on an existing table. To enable IcebergCompatV2, set the table property " + "'delta.enableIcebergCompatV2' = 'true'." ) ) } def uniFormHudiDeleteVectorCompat(): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_UNIVERSAL_FORMAT_VIOLATION", messageParameters = Array( UniversalFormat.HUDI_FORMAT, "Requires delete vectors to be disabled." ) ) } def uniFormHudiSchemaCompat(unsupportedType: DataType): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_UNIVERSAL_FORMAT_VIOLATION", messageParameters = Array( UniversalFormat.HUDI_FORMAT, s"DataType: $unsupportedType is not currently supported." ) ) } def icebergCompatVersionMutualExclusive(version: Int): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_ICEBERG_COMPAT_VIOLATION.VERSION_MUTUAL_EXCLUSIVE", messageParameters = Array(version.toString) ) } def icebergCompatChangeVersionNeedRewrite(version: Int, newVersion: Int): Throwable = { val newVersionString = newVersion.toString new DeltaUnsupportedOperationException( errorClass = "DELTA_ICEBERG_COMPAT_VIOLATION.CHANGE_VERSION_NEED_REWRITE", messageParameters = Array(newVersionString, newVersionString, newVersionString, newVersionString) ) } def icebergCompatVersionNotSupportedException( currVersion: Int, maxVersion: Int): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_ICEBERG_COMPAT_VIOLATION.COMPAT_VERSION_NOT_SUPPORTED", messageParameters = Array( currVersion.toString, currVersion.toString, maxVersion.toString ) ) } def icebergCompatReorgAddFileTagsMissingException( tableVersion: Long, icebergCompatVersion: Int, addFilesCount: Long, addFilesWithTagsCount: Long): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_ICEBERG_COMPAT_VIOLATION.FILES_NOT_ICEBERG_COMPAT", messageParameters = Array( icebergCompatVersion.toString, icebergCompatVersion.toString, addFilesCount.toString, tableVersion.toString, (addFilesCount - addFilesWithTagsCount).toString, icebergCompatVersion.toString ) ) } def icebergCompatDataFileRewriteFailedException( icebergCompatVersion: Int, cause: Throwable): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_ICEBERG_COMPAT_VIOLATION.REWRITE_DATA_FAILED", messageParameters = Array( icebergCompatVersion.toString, icebergCompatVersion.toString, icebergCompatVersion.toString ), cause ) } def icebergCompatReplacePartitionedTableException( version: Int, prevPartitionCols: Seq[String], newPartitionCols: Seq[String]): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_ICEBERG_COMPAT_VIOLATION.REPLACE_TABLE_CHANGE_PARTITION_NAMES", messageParameters = Array( version.toString, version.toString, prevPartitionCols.mkString("(", ",", ")"), newPartitionCols.mkString("(", ",", ")") ) ) } def icebergCompatUnsupportedDataTypeException( version: Int, dataType: DataType, schema: StructType): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_ICEBERG_COMPAT_VIOLATION.UNSUPPORTED_DATA_TYPE", messageParameters = Array(version.toString, version.toString, dataType.typeName, schema.treeString) ) } def icebergCompatUnsupportedFieldException( version: Int, field: StructField, schema: StructType): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_ICEBERG_COMPAT_VIOLATION.UNSUPPORTED_DATA_TYPE", messageParameters = Array(version.toString, version.toString, s"${field.dataType.typeName}:${field.name}", schema.treeString) ) } def icebergCompatUnsupportedPartitionDataTypeException( version: Int, dataType: DataType, schema: StructType): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_ICEBERG_COMPAT_VIOLATION.UNSUPPORTED_PARTITION_DATA_TYPE", messageParameters = Array(version.toString, version.toString, dataType.typeName, schema.treeString) ) } def icebergCompatMissingRequiredTableFeatureException( version: Int, tf: TableFeature): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_ICEBERG_COMPAT_VIOLATION.MISSING_REQUIRED_TABLE_FEATURE", messageParameters = Array(version.toString, version.toString, tf.name) ) } def icebergCompatDisablingRequiredTableFeatureException( version: Int, tf: TableFeature): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_ICEBERG_COMPAT_VIOLATION.DISABLING_REQUIRED_TABLE_FEATURE", messageParameters = Array(version.toString, version.toString, tf.name, version.toString) ) } def icebergCompatIncompatibleTableFeatureException( version: Int, tf: TableFeature): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_ICEBERG_COMPAT_VIOLATION.INCOMPATIBLE_TABLE_FEATURE", messageParameters = Array(version.toString, version.toString, tf.name) ) } def icebergCompatDeletionVectorsShouldBeDisabledException(version: Int): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_ICEBERG_COMPAT_VIOLATION.DELETION_VECTORS_SHOULD_BE_DISABLED", messageParameters = Array(version.toString, version.toString) ) } def icebergCompatDeletionVectorsNotPurgedException(version: Int): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_ICEBERG_COMPAT_VIOLATION.DELETION_VECTORS_NOT_PURGED", messageParameters = Array(version.toString, version.toString) ) } def icebergCompatWrongRequiredTablePropertyException( version: Int, key: String, actualValue: String, requiredValue: String): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_ICEBERG_COMPAT_VIOLATION.WRONG_REQUIRED_TABLE_PROPERTY", messageParameters = Array(version.toString, version.toString, key, requiredValue, actualValue) ) } def icebergCompatUnsupportedTypeWideningException( version: Int, fieldPath: Seq[String], oldType: DataType, newType: DataType): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_ICEBERG_COMPAT_VIOLATION.UNSUPPORTED_TYPE_WIDENING", messageParameters = Array( version.toString, version.toString, SchemaUtils.prettyFieldName(fieldPath), toSQLType(oldType), toSQLType(newType) ) ) } def universalFormatConversionFailedException( failedOnCommitVersion: Long, format: String, errorMessage: String): Throwable = { new DeltaRuntimeException( errorClass = "DELTA_UNIVERSAL_FORMAT_CONVERSION_FAILED", messageParameters = Array(s"$failedOnCommitVersion", format, errorMessage) ) } def invalidAutoCompactType(value: String): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_INVALID_AUTO_COMPACT_TYPE", messageParameters = Array(value, AutoCompactType.ALLOWED_VALUES.mkString("(", ",", ")")) ) } def clusterByInvalidNumColumnsException( numColumnsLimit: Int, actualNumColumns: Int): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CLUSTER_BY_INVALID_NUM_COLUMNS", messageParameters = Array(numColumnsLimit.toString, actualNumColumns.toString) ) } def clusteringColumnMissingStats( clusteringColumnWithoutStats: String, statsSchema: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CLUSTERING_COLUMN_MISSING_STATS", messageParameters = Array(clusteringColumnWithoutStats, statsSchema) ) } def clusteringColumnUnsupportedDataTypes(clusteringColumnsWithDataTypes: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CLUSTERING_COLUMNS_DATATYPE_NOT_SUPPORTED", messageParameters = Array(clusteringColumnsWithDataTypes) ) } def clusteringColumnsMismatchException( providedClusteringColumns: String, existingClusteringColumns: String): Throwable = { new DeltaAnalysisException( "DELTA_CLUSTERING_COLUMNS_MISMATCH", Array(providedClusteringColumns, existingClusteringColumns) ) } def clusterByWithPartitionedBy(): Throwable = { new DeltaAnalysisException( "DELTA_CLUSTER_BY_WITH_PARTITIONED_BY", Array.empty) } def dropClusteringColumnNotSupported(droppingClusteringCols: Seq[String]): Throwable = { new DeltaAnalysisException( "DELTA_UNSUPPORTED_DROP_CLUSTERING_COLUMN", Array(droppingClusteringCols.mkString(","))) } def replacingClusteredTableWithPartitionedTableNotAllowed(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CLUSTERING_REPLACE_TABLE_WITH_PARTITIONED_TABLE", messageParameters = Array.empty) } def clusteringWithPartitionPredicatesException(predicates: Seq[String]): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_CLUSTERING_WITH_PARTITION_PREDICATE", messageParameters = Array(s"${predicates.mkString(" ")}")) } def clusteringWithZOrderByException(zOrderBy: Seq[UnresolvedAttribute]): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CLUSTERING_WITH_ZORDER_BY", messageParameters = Array(s"${zOrderBy.map(_.name).mkString(", ")}")) } def optimizeFullNotSupportedException(): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_OPTIMIZE_FULL_NOT_SUPPORTED", messageParameters = Array.empty) } def alterClusterByNotOnDeltaTableException(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_ONLY_OPERATION", messageParameters = Array("ALTER TABLE CLUSTER BY")) } def alterClusterByNotAllowedException(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_ALTER_TABLE_CLUSTER_BY_NOT_ALLOWED", messageParameters = Array.empty) } def alterTableSetClusteringTableFeatureException(tableFeature: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_ALTER_TABLE_SET_CLUSTERING_TABLE_FEATURE_NOT_ALLOWED", messageParameters = Array(tableFeature)) } def createTableSetClusteringTableFeatureException(tableFeature: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CREATE_TABLE_SET_CLUSTERING_TABLE_FEATURE_NOT_ALLOWED", messageParameters = Array(tableFeature)) } def mergeAddVoidColumn(columnName: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_MERGE_ADD_VOID_COLUMN", messageParameters = Array(toSQLId(columnName)) ) } def columnBuilderMissingDataType(colName: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_COLUMN_MISSING_DATA_TYPE", messageParameters = Array(toSQLId(colName))) } def createTableMissingTableNameOrLocation(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CREATE_TABLE_MISSING_TABLE_NAME_OR_LOCATION", messageParameters = Array.empty) } def createTableIdentifierLocationMismatch(identifier: String, location: String): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CREATE_TABLE_IDENTIFIER_LOCATION_MISMATCH", messageParameters = Array(identifier, location)) } def dropColumnOnSingleFieldSchema(schema: StructType): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_DROP_COLUMN_ON_SINGLE_FIELD_SCHEMA", messageParameters = Array(schema.treeString)) } def errorFindingColumnPosition( columnPath: Seq[String], schema: DataType, extraErrMsg: String): Throwable = { new DeltaAnalysisException( errorClass = "_LEGACY_ERROR_TEMP_DELTA_0008", messageParameters = Array( UnresolvedAttribute(columnPath).name, dataTypeToString(schema), extraErrMsg)) } def alterTableClusterByOnPartitionedTableException(): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_ALTER_TABLE_CLUSTER_BY_ON_PARTITIONED_TABLE_NOT_ALLOWED", messageParameters = Array.empty) } def createTableWithDifferentClusteringException( path: Path, specifiedClusterBySpec: Option[ClusterBySpec], existingClusterBySpec: Option[ClusterBySpec]): Throwable = { new DeltaAnalysisException( errorClass = "DELTA_CREATE_TABLE_WITH_DIFFERENT_CLUSTERING", messageParameters = Array( path.toString, specifiedClusterBySpec .map(_.columnNames.map(_.toString)) .getOrElse(Seq.empty) .mkString(", "), existingClusterBySpec .map(_.columnNames.map(_.toString)) .getOrElse(Seq.empty) .mkString(", "))) } def unsupportedWritesWithMissingCoordinators(coordinatorName: String): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_UNSUPPORTED_WRITES_WITHOUT_COORDINATOR", messageParameters = Array(coordinatorName)) } private def dataTypeToString(dt: DataType): String = dt match { case s: StructType => s.treeString case other => other.simpleString } def operationBlockedOnCatalogManagedTable(operation: String): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION", messageParameters = Array(operation)) } def deltaCannotCreateCatalogManagedTable(): Throwable = { new DeltaUnsupportedOperationException( errorClass = "DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_CREATION", messageParameters = Array.empty) } def numRecordsMismatch( operation: String, numAddedRecords: Long, numRemovedRecords: Long): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_NUM_RECORDS_MISMATCH", messageParameters = Array(operation, numAddedRecords.toString, numRemovedRecords.toString) ) } def commandInvariantViolationException( operation: String, id: UUID): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_COMMAND_INVARIANT_VIOLATION", messageParameters = Array(operation, id.toString) ) } def catalogManagedTablePathBasedAccessNotAllowed(path: Path): Throwable = { new DeltaIllegalStateException( errorClass = "DELTA_PATH_BASED_ACCESS_TO_CATALOG_MANAGED_TABLE_BLOCKED", messageParameters = Array(path.toString) ) } def cannotResolveSourceColumnException(columnPath: Seq[String]): Throwable = { new DeltaIllegalArgumentException( errorClass = "DELTA_CANNOT_RESOLVE_SOURCE_COLUMN", messageParameters = Array(s"${UnresolvedAttribute(columnPath).name}")) } } object DeltaErrors extends DeltaErrorsBase /** The basic class for all Tahoe commit conflict exceptions. */ abstract class DeltaConcurrentModificationException(message: String) extends ConcurrentModificationException(message) { /** * Type of the commit conflict. */ def conflictType: String = this.getClass.getSimpleName.stripSuffix("Exception") } /** * This class is kept for backward compatibility. * Use [[io.delta.exceptions.ConcurrentWriteException]] instead. */ class ConcurrentWriteException(message: String) extends io.delta.exceptions.DeltaConcurrentModificationException(message) { def this(conflictingCommit: Option[CommitInfo]) = this( DeltaErrors.concurrentModificationExceptionMsg( SparkEnv.get.conf, s"A concurrent transaction has written new data since the current transaction " + s"read the table. Please try the operation again.", conflictingCommit)) } /** * Thrown when time travelling to a version that does not exist in the Delta Log. * @param userVersion - the version time travelling to * @param earliest - earliest version available in the Delta Log * @param latest - The latest version available in the Delta Log */ case class VersionNotFoundException( userVersion: Long, earliest: Long, latest: Long) extends DeltaAnalysisException( errorClass = "DELTA_VERSION_NOT_FOUND", messageParameters = Array(userVersion.toString, earliest.toString, latest.toString) ) /** * This class is kept for backward compatibility. * Use [[io.delta.exceptions.MetadataChangedException]] instead. */ class MetadataChangedException(message: String) extends io.delta.exceptions.DeltaConcurrentModificationException(message) { def this(conflictingCommit: Option[CommitInfo]) = this( DeltaErrors.concurrentModificationExceptionMsg( SparkEnv.get.conf, "The metadata of the Delta table has been changed by a concurrent update. " + "Please try the operation again.", conflictingCommit)) } /** * This class is kept for backward compatibility. * Use [[io.delta.exceptions.ProtocolChangedException]] instead. */ class ProtocolChangedException(message: String) extends io.delta.exceptions.DeltaConcurrentModificationException(message) { def this(conflictingCommit: Option[CommitInfo]) = this( DeltaErrors.concurrentModificationExceptionMsg( SparkEnv.get.conf, "The protocol version of the Delta table has been changed by a concurrent update. " + "Please try the operation again.", conflictingCommit)) } /** * This class is kept for backward compatibility. * Use [[io.delta.exceptions.ConcurrentAppendException]] instead. */ class ConcurrentAppendException(message: String) extends io.delta.exceptions.DeltaConcurrentModificationException(message) { def this( conflictingCommit: Option[CommitInfo], partition: String, customRetryMsg: Option[String] = None) = this( DeltaErrors.concurrentModificationExceptionMsg( SparkEnv.get.conf, s"Files were added to $partition by a concurrent update. " + customRetryMsg.getOrElse("Please try the operation again."), conflictingCommit)) } /** * This class is kept for backward compatibility. * Use [[io.delta.exceptions.ConcurrentDeleteReadException]] instead. */ class ConcurrentDeleteReadException(message: String) extends io.delta.exceptions.DeltaConcurrentModificationException(message) { def this(conflictingCommit: Option[CommitInfo], file: String) = this( DeltaErrors.concurrentModificationExceptionMsg( SparkEnv.get.conf, "This transaction attempted to read one or more files that were deleted" + s" (for example $file) by a concurrent update. Please try the operation again.", conflictingCommit)) } /** * This class is kept for backward compatibility. * Use [[io.delta.exceptions.ConcurrentDeleteDeleteException]] instead. */ class ConcurrentDeleteDeleteException(message: String) extends io.delta.exceptions.DeltaConcurrentModificationException(message) { def this(conflictingCommit: Option[CommitInfo], file: String) = this( DeltaErrors.concurrentModificationExceptionMsg( SparkEnv.get.conf, "This transaction attempted to delete one or more files that were deleted " + s"(for example $file) by a concurrent update. Please try the operation again.", conflictingCommit)) } /** * This class is kept for backward compatibility. * Use [[io.delta.exceptions.ConcurrentTransactionException]] instead. */ class ConcurrentTransactionException(message: String) extends io.delta.exceptions.DeltaConcurrentModificationException(message) { def this(conflictingCommit: Option[CommitInfo]) = this( DeltaErrors.concurrentModificationExceptionMsg( SparkEnv.get.conf, s"This error occurs when multiple streaming queries are using the same checkpoint to write " + "into this table. Did you run multiple instances of the same streaming query" + " at the same time?", conflictingCommit)) } /** A helper class in building a helpful error message in case of metadata mismatches. */ class MetadataMismatchErrorBuilder { private var subErrors: Seq[(String, Array[String])] = Nil def addSchemaMismatch(original: StructType, data: StructType, id: String): Unit = { subErrors :+= ("SCHEMA_MISMATCH", Array( id, DeltaErrors.formatSchema(original), DeltaErrors.formatSchema(data) )) } def addPartitioningMismatch(original: Seq[String], provided: Seq[String]): Unit = { subErrors :+= ("PARTITIONING_MISMATCH", Array( DeltaErrors.formatColumnList(provided), DeltaErrors.formatColumnList(original) )) } def addOverwriteBit(): Unit = { subErrors :+= ("OVERWRITE_REQUIRED", Array.empty[String]) } def finalizeAndThrow(conf: SQLConf): Unit = { throw new DeltaAnalysisExceptionWithSubErrors( errorClass = "DELTA_METADATA_MISMATCH", messageParameters = Array.empty[String], subErrors = subErrors ) } } class DeltaColumnMappingUnsupportedException( errorClass: String, messageParameters: Array[String] = Array.empty) extends ColumnMappingUnsupportedException( DeltaThrowableHelper.getMessage(errorClass, messageParameters)) with DeltaThrowable { override def getErrorClass: String = errorClass override def getMessageParameters: java.util.Map[String, String] = DeltaThrowableHelper.getMessageParameters(errorClass, errorSubClass = null, messageParameters) } class DeltaFileNotFoundException( errorClass: String, messageParameters: Array[String] = Array.empty) extends FileNotFoundException( DeltaThrowableHelper.getMessage(errorClass, messageParameters)) with DeltaThrowable { override def getErrorClass: String = errorClass override def getMessageParameters: java.util.Map[String, String] = DeltaThrowableHelper.getMessageParameters(errorClass, errorSubClass = null, messageParameters) } class DeltaFileAlreadyExistsException( errorClass: String, messageParameters: Array[String] = Array.empty) extends FileAlreadyExistsException( DeltaThrowableHelper.getMessage(errorClass, messageParameters)) with DeltaThrowable { override def getErrorClass: String = errorClass override def getMessageParameters: java.util.Map[String, String] = DeltaThrowableHelper.getMessageParameters(errorClass, errorSubClass = null, messageParameters) } class DeltaIOException( errorClass: String, messageParameters: Array[String] = Array.empty, cause: Throwable = null) extends IOException( DeltaThrowableHelper.getMessage(errorClass, messageParameters), cause) with DeltaThrowable { override def getErrorClass: String = errorClass override def getMessageParameters: java.util.Map[String, String] = DeltaThrowableHelper.getMessageParameters(errorClass, errorSubClass = null, messageParameters) } class DeltaIllegalStateException( errorClass: String, messageParameters: Array[String] = Array.empty, cause: Throwable = null) extends IllegalStateException( DeltaThrowableHelper.getMessage(errorClass, messageParameters), cause) with DeltaThrowable { override def getErrorClass: String = errorClass override def getMessageParameters: java.util.Map[String, String] = DeltaThrowableHelper.getMessageParameters(errorClass, errorSubClass = null, messageParameters) } class DeltaIndexOutOfBoundsException( errorClass: String, messageParameters: Array[String] = Array.empty) extends IndexOutOfBoundsException( DeltaThrowableHelper.getMessage(errorClass, messageParameters)) with DeltaThrowable { override def getErrorClass: String = errorClass override def getMessageParameters: java.util.Map[String, String] = DeltaThrowableHelper.getMessageParameters(errorClass, errorSubClass = null, messageParameters) } /** Thrown when the protocol version of a table is greater than supported by this client. */ case class InvalidProtocolVersionException( tableNameOrPath: String, readerRequiredVersion: Int, writerRequiredVersion: Int, supportedReaderVersions: Seq[Int], supportedWriterVersions: Seq[Int]) extends RuntimeException(DeltaThrowableHelper.getMessage( errorClass = "DELTA_INVALID_PROTOCOL_VERSION", messageParameters = Array( tableNameOrPath, readerRequiredVersion.toString, writerRequiredVersion.toString, io.delta.VERSION, supportedReaderVersions.sorted.mkString(", "), supportedWriterVersions.sorted.mkString(", ")))) with DeltaThrowable { override def getErrorClass: String = "DELTA_INVALID_PROTOCOL_VERSION" } class ProtocolDowngradeException(oldProtocol: Protocol, newProtocol: Protocol) extends RuntimeException(DeltaThrowableHelper.getMessage( errorClass = "DELTA_INVALID_PROTOCOL_DOWNGRADE", messageParameters = Array(oldProtocol.simpleString, newProtocol.simpleString) )) with DeltaThrowable { override def getErrorClass: String = "DELTA_INVALID_PROTOCOL_DOWNGRADE" override def getMessageParameters: java.util.Map[String, String] = { DeltaThrowableHelper.getMessageParameters( "DELTA_INVALID_PROTOCOL_DOWNGRADE", errorSubClass = null, Array(oldProtocol.simpleString, newProtocol.simpleString)) } } class DeltaTableFeatureException( errorClass: String, messageParameters: Array[String] = Array.empty) extends DeltaRuntimeException(errorClass, messageParameters) case class DeltaUnsupportedTableFeatureException( errorClass: String, tableNameOrPath: String, unsupported: Iterable[String]) extends DeltaTableFeatureException( errorClass, Array(tableNameOrPath, io.delta.VERSION, unsupported.mkString(", "))) class DeltaRuntimeException( errorClass: String, val messageParameters: Array[String] = Array.empty) extends RuntimeException( DeltaThrowableHelper.getMessage(errorClass, messageParameters)) with DeltaThrowable { override def getErrorClass: String = errorClass override def getMessageParameters: java.util.Map[String, String] = { DeltaThrowableHelper.getMessageParameters(errorClass, errorSubClass = null, messageParameters) } } class DeltaSparkException( errorClass: String, messageParameters: Array[String] = Array.empty, cause: Throwable = null) extends SparkException( DeltaThrowableHelper.getMessage(errorClass, messageParameters), cause) with DeltaThrowable { override def getErrorClass: String = errorClass override def getMessageParameters: java.util.Map[String, String] = DeltaThrowableHelper.getMessageParameters(errorClass, errorSubClass = null, messageParameters) } class DeltaNoSuchTableException( errorClass: String, errorMessageParameters: Array[String] = Array.empty) extends AnalysisException( DeltaThrowableHelper.getMessage(errorClass, errorMessageParameters)) with DeltaThrowable { override def getErrorClass: String = errorClass override def getMessageParameters: java.util.Map[String, String] = { DeltaThrowableHelper .getMessageParameters(errorClass, errorSubClass = null, errorMessageParameters) } } class DeltaCommandUnsupportedWithDeletionVectorsException( errorClass: String, messageParameters: Array[String] = Array.empty) extends UnsupportedOperationException( DeltaThrowableHelper.getMessage(errorClass, messageParameters)) with DeltaThrowable { override def getErrorClass: String = errorClass } sealed trait DeltaTablePropertyValidationFailedSubClass { def tag: String /** Can be overridden in case subclasses need the table name as well. */ def messageParameters(table: String): Array[String] = Array(table) } final object DeltaTablePropertyValidationFailedSubClass { final case object PersistentDeletionVectorsWithIncrementalManifestGeneration extends DeltaTablePropertyValidationFailedSubClass { override val tag = "PERSISTENT_DELETION_VECTORS_WITH_INCREMENTAL_MANIFEST_GENERATION" } final case object ExistingDeletionVectorsWithIncrementalManifestGeneration extends DeltaTablePropertyValidationFailedSubClass { override val tag = "EXISTING_DELETION_VECTORS_WITH_INCREMENTAL_MANIFEST_GENERATION" /** This subclass needs the table parameters in two places. */ override def messageParameters(table: String): Array[String] = Array(table, table) } final case object PersistentDeletionVectorsInNonParquetTable extends DeltaTablePropertyValidationFailedSubClass { override val tag = "PERSISTENT_DELETION_VECTORS_IN_NON_PARQUET_TABLE" } } class DeltaTablePropertyValidationFailedException( table: String, subClass: DeltaTablePropertyValidationFailedSubClass) extends RuntimeException(DeltaThrowableHelper.getMessage( errorClass = "DELTA_VIOLATE_TABLE_PROPERTY_VALIDATION_FAILED" + "." + subClass.tag, messageParameters = subClass.messageParameters(table))) with DeltaThrowable { override def getMessageParameters: java.util.Map[String, String] = DeltaThrowableHelper.getMessageParameters( "DELTA_VIOLATE_TABLE_PROPERTY_VALIDATION_FAILED", subClass.tag, subClass.messageParameters(table)) override def getErrorClass: String = "DELTA_VIOLATE_TABLE_PROPERTY_VALIDATION_FAILED." + subClass.tag } /** Errors thrown around column mapping. */ class ColumnMappingUnsupportedException(msg: String) extends UnsupportedOperationException(msg) case class ColumnMappingException(msg: String, mode: DeltaColumnMappingMode) extends AnalysisException(msg) class DeltaChecksumException( errorClass: String, messageParameters: Array[String] = Array.empty, pos: Long) extends ChecksumException( DeltaThrowableHelper.getMessage(errorClass, messageParameters), pos) with DeltaThrowable { override def getErrorClass: String = errorClass } /** * Errors thrown when an operation is not supported with non-additive schema changes * (rename / drop column, type change). * * To make compatible with existing behavior for those who accidentally has already used this * operation, user should always be able to use `escapeConfigName` to fall back at own risk. */ class DeltaStreamingNonAdditiveSchemaIncompatibleException( val readSchema: StructType, val incompatibleSchema: StructType, val docLink: String, val enableNonAdditiveSchemaEvolution: Boolean = false, val additionalProperties: Map[String, String] = Map.empty) extends DeltaUnsupportedOperationException( errorClass = if (additionalProperties.getOrElse("isV2DataSource", "false") == "true") { "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE_V2" } else if (enableNonAdditiveSchemaEvolution) { "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE_USE_SCHEMA_LOG" } else { "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE" }, messageParameters = Array( docLink, readSchema.json, incompatibleSchema.json) ) ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaFileFormat.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.actions.{Metadata, Protocol} import org.apache.spark.sql.SparkSession import org.apache.spark.sql.execution.datasources.FileFormat trait DeltaFileFormat { // TODO: Add support for column mapping /** Return the current Spark session used. */ protected def spark: SparkSession /** * Build the underlying Spark `FileFormat` of the Delta table with specified metadata. * * With column mapping, some properties of the underlying file format might change during * transaction, so if possible, we should always pass in the latest transaction's metadata * instead of one from a past snapshot. */ def fileFormat(protocol: Protocol, metadata: Metadata): FileFormat = new DeltaParquetFileFormat(protocol, metadata) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaFileProviderUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.actions.Action import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.storage.ClosableIterator import org.apache.spark.sql.delta.util.FileNames.DeltaFile import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.FileStatus import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.JsonToStructs import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType import org.apache.spark.unsafe.types.UTF8String object DeltaFileProviderUtils extends DeltaLogging { protected def readThreadPool = SnapshotManagement.deltaLogAsyncUpdateThreadPool /** Put any future parsing options here. */ val jsonStatsParseOption = Map.empty[String, String] private[delta] def createJsonStatsParser(schemaToUse: StructType): String => InternalRow = { val parser = JsonToStructs( schema = schemaToUse, options = jsonStatsParseOption, child = null, timeZoneId = Some(SQLConf.get.sessionLocalTimeZone) ) (json: String) => { val utf8json = UTF8String.fromString(json) parser.nullSafeEval(utf8json).asInstanceOf[InternalRow] } } /** * Get the Delta json files present in the delta log in the range [startVersion, endVersion]. * Returns the files in sorted order, and throws if any in the range are missing. */ def getDeltaFilesInVersionRange( spark: SparkSession, deltaLog: DeltaLog, startVersion: Long, endVersion: Long, catalogTableOpt: Option[CatalogTable]): Seq[FileStatus] = { // Pass `failOnDataLoss = false` as we are doing an explicit validation on the result ourselves // to identify that there are no gaps. val result = deltaLog .getChangeLogFiles(startVersion, endVersion, catalogTableOpt, failOnDataLoss = false) .map(_._2) .collect { case DeltaFile(fs, v) => (fs, v) } .toSeq // Verify that we got the entire range requested if (result.size.toLong != endVersion - startVersion + 1) { // [[unsafeVolatileSnapshot]] maybe null, which needs to be explicitly filtered out. val snapshot = Some(deltaLog.unsafeVolatileSnapshot).filter(_ != null) recordDeltaEvent( deltaLog = deltaLog, opType = "delta.exceptions.deltaVersionsNotContiguous", data = Map( // Remove the first element of the stack trace since this represents // the [[Thread.getStackTrace]] call itself. "stackTrace" -> Thread.currentThread().getStackTrace.tail.mkString("\n\t"), "startVersion" -> startVersion, "endVersion" -> endVersion, "unsafeVolatileSnapshot.latestCheckpointVersion" -> snapshot.map(_.checkpointProvider.version).getOrElse(-1L), "unsafeVolatileSnapshot.latestSnapshotVersion" -> snapshot.map(_.version).getOrElse(-1L), "unsafeVolatileSnapshot.checksumOpt" -> snapshot.map(_.checksumOpt).orNull )) throw DeltaErrors.deltaVersionsNotContiguousException( spark = spark, deltaVersions = result.map(_._2), startVersion = startVersion, endVersion = endVersion, // Get the latest snapshot version for visibility when throwing the exception, // this is not exactly "the version to load snapshot" but // we just use the latest snapshot version here. versionToLoad = snapshot.map(_.version).getOrElse(-1L)) } result.map(_._1) } /** Helper method to read and parse the delta files parallelly into [[Action]]s. */ def parallelReadAndParseDeltaFilesAsIterator( deltaLog: DeltaLog, spark: SparkSession, files: Seq[FileStatus]): Seq[ClosableIterator[String]] = { val hadoopConf = deltaLog.newDeltaHadoopConf() parallelReadDeltaFilesBase(spark, files, hadoopConf, { file: FileStatus => deltaLog.store.readAsIterator(file, hadoopConf) }) } protected def parallelReadDeltaFilesBase[A]( spark: SparkSession, files: Seq[FileStatus], hadoopConf: Configuration, f: FileStatus => A): Seq[A] = { readThreadPool.parallelMap(spark, files) { file => f(file) }.toSeq } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaHistoryManager.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.io.FileNotFoundException import java.sql.Timestamp import java.util.concurrent.CompletableFuture import scala.collection.mutable import scala.concurrent.{ExecutionContext, ExecutionContextExecutorService, Future} import scala.concurrent.duration.Duration import org.apache.spark.sql.delta.actions.{Action, CommitInfo, CommitMarker, JobInfo, NotebookInfo} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.storage.LogStore import org.apache.spark.sql.delta.util.{DateTimeUtils, DeltaCommitFileProvider, FileNames, TimestampFormatter} import org.apache.spark.sql.delta.util.FileNames._ import org.apache.spark.sql.delta.util.threads.DeltaThreadPool import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.SparkEnv import org.apache.spark.internal.MDC import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.internal.SQLConf import org.apache.spark.util.{SerializableConfiguration, ThreadUtils} /** * This class keeps tracks of the version of commits and their timestamps for a Delta table to * help with operations like describing the history of a table. * * @param deltaLog The transaction log of this table * @param maxKeysPerList How many commits to list when performing a parallel search. Exposed for * tests. Currently set to `1000`, which is the maximum keys returned by S3 * per list call. Azure can return `5000`, therefore we choose 1000. */ class DeltaHistoryManager( deltaLog: DeltaLog, maxKeysPerList: Int = 1000) extends DeltaLogging { private def spark: SparkSession = SparkSession.active private def getSerializableHadoopConf: SerializableConfiguration = { new SerializableConfiguration(deltaLog.newDeltaHadoopConf()) } import DeltaHistoryManager._ /** * Returns the information of the latest `limit` commits made to this table in reverse * chronological order. */ def getHistory( limitOpt: Option[Int], catalogTableOpt: Option[CatalogTable]): Seq[DeltaHistory] = { val snapshot = deltaLog.update(catalogTableOpt = catalogTableOpt) val listStart = limitOpt .map { limit => math.max(snapshot.version - limit + 1, 0) } .getOrElse(getEarliestDeltaFile(deltaLog)) getHistory(listStart, end = Some(snapshot.version), catalogTableOpt) } /** * Returns the information of the latest `limit` commits made to this table in reverse * chronological order. This version does not take in a catalog table and should only be * used in testing. */ def getHistory(limitOpt: Option[Int]): Seq[DeltaHistory] = { getHistory(limitOpt, catalogTableOpt = None) } /** * Get the commit information of the Delta table from commit `[start, end]` in * reverse chronological order. An empty Seq is returned when `start > end`. * * @param useInCommitTimestamps Whether ICT should be used as the commit-timestamp for * the commits. * If `true`, all commits in the range must have ICTs and * the timestamp returned for each commit will be the ICT. * If `false`, the file modification time will be used as the * timestamp. */ private[delta] def getHistoryImpl( start: Long, end: Long, useInCommitTimestamps: Boolean, commitFileProvider: DeltaCommitFileProvider): Seq[DeltaHistory] = { import org.apache.spark.sql.delta.implicits._ val conf = getSerializableHadoopConf val logPath = deltaLog.logPath.toString // We assume that commits are contiguous, therefore we try to load all of them in order val info = spark.range(start, end + 1) .mapPartitions { versions => val logStore = LogStore(SparkEnv.get.conf, conf.value) val basePath = new Path(logPath) val fs = basePath.getFileSystem(conf.value) versions.flatMap { commit => try { val deltaFile = commitFileProvider.deltaFile(commit) val commitInfoOpt = DeltaHistoryManager .getCommitInfoOpt(logStore, deltaFile, conf.value) val timestamp = if (useInCommitTimestamps) { CommitInfo.getRequiredInCommitTimestamp(commitInfoOpt, commit.toString) } else { fs.getFileStatus(deltaFile).getModificationTime } val ci = commitInfoOpt.getOrElse(CommitInfo.empty(Some(commit))) Some(ci.withTimestamp(timestamp)) } catch { case _: FileNotFoundException => // We have a race-condition where files can be deleted while reading. It's fine to // skip those files None } }.map(DeltaHistory.fromCommitInfo) } val monotonizedCommits = if (useInCommitTimestamps) { // ICT timestamps are guaranteed to be monotonically increasing. info.collect() } else { monotonizeCommitTimestamps(info.collect()) } // Spark should return the commits in increasing order as well monotonizedCommits.reverse } /** * Get the commit information of the Delta table from commit `[start, end]` in reverse * chronological order. If `end` is `None`, we return all commits from start to now. * @param start The start of the commit range, inclusive. * @param end The end of the commit range, inclusive. * @param catalogTableOpt the catalog table associated with the Delta table. */ def getHistory( start: Long, end: Option[Long], catalogTableOpt: Option[CatalogTable] = None): Seq[DeltaHistory] = { val currentSnapshot = deltaLog.unsafeVolatileSnapshot val (snapshotNewerThanResolvedEnd, resolvedEnd) = end match { case Some(endInclusive) if currentSnapshot.version >= endInclusive => // Use the cache snapshot if it's fresh enough for the [start, endInclusive] query. (currentSnapshot, math.min(currentSnapshot.version, endInclusive)) case _ => // Either end doesn't exist or the currently cached snapshot isn't new enough to // satisfy it. val snapshot = deltaLog.update(catalogTableOpt = catalogTableOpt) val endInclusive = end.getOrElse(snapshot.version).min(snapshot.version) (snapshot, endInclusive) } val commitFileProvider = DeltaCommitFileProvider(snapshotNewerThanResolvedEnd) if (!DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData( snapshotNewerThanResolvedEnd.metadata)) { getHistoryImpl(start, resolvedEnd, useInCommitTimestamps = false, commitFileProvider) } else { val ictEnablementCommit = InCommitTimestampUtils.getValidatedICTEnablementInfo(snapshotNewerThanResolvedEnd.metadata) ictEnablementCommit match { case Some(Commit(ictEarliest, _)) => // getHistoryImpl will return an empty Seq if start > end. val nonICTCommits = getHistoryImpl( start, math.min(resolvedEnd, ictEarliest - 1), useInCommitTimestamps = false, commitFileProvider) val ictCommits = getHistoryImpl( math.max(ictEarliest, start), resolvedEnd, useInCommitTimestamps = true, commitFileProvider) // Merge the two sequences, ensuring ICT commits are listed first as they are more recent, // followed by non-ICT commits, maintaining the reverse chronological order. ictCommits ++ nonICTCommits case _ => // Enablement info not found, ICT is enabled for all available commits. getHistoryImpl(start, resolvedEnd, useInCommitTimestamps = true, commitFileProvider) } } } /** * Returns the latest commit that happened at or before `time` in the range `[start, end)`. * All the commits in the range `[start, end)` are assumed to not have inCommitTimestamps. * If no such commit exists, the earliest commit is returned. */ def getCommitFromNonICTRange(start: Long, end: Long, time: Long): Commit = { if (end - start > 2 * maxKeysPerList) { parallelSearch(time, start, end) } else { val commits = getCommitsWithNonIctTimestamps( deltaLog.store, deltaLog.logPath, start, Some(end), deltaLog.newDeltaHadoopConf()) if (commits.isEmpty) { throw DeltaErrors.noHistoryFound(deltaLog.logPath) } lastCommitBeforeTimestamp(commits, time).getOrElse(commits.head) } } /** * Returns the latest commit that happened at or before `time`. * @param timestamp The timestamp to search for * @param canReturnLastCommit Whether we can return the latest version of the table if the * provided timestamp is after the latest commit * @param mustBeRecreatable Whether the state at the given commit should be recreatable * @param canReturnEarliestCommit Whether we can return the earliest commit if no such commit * exists. */ def getActiveCommitAtTime( timestamp: Timestamp, catalogTableOpt: Option[CatalogTable], canReturnLastCommit: Boolean, mustBeRecreatable: Boolean = true, canReturnEarliestCommit: Boolean = false): Commit = { val time = timestamp.getTime val earliestVersion = if (mustBeRecreatable) { getEarliestRecreatableCommit } else { getEarliestDeltaFile(deltaLog) } val snapshot = deltaLog.update(catalogTableOpt = catalogTableOpt) val commitFileProvider = DeltaCommitFileProvider(snapshot) val latestVersion = snapshot.version // In most cases, the earliest commit should not be the result of this search. // When ICT is enabled, use -1L as the placeholder timestamp for the earliest commit // for the search and only fetch the real timestamp if the earliest commit is // the result of the search. We can potentially avoid one unnecessary IO this way. val placeholderEarliestCommit = Commit(earliestVersion, -1L) val ictEnablementCommit = if (DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(snapshot.metadata)) { InCommitTimestampUtils.getValidatedICTEnablementInfo(snapshot.metadata) // If missing, ICT is enabled for all available versions .getOrElse(placeholderEarliestCommit) } else { // Pretend ICT will be enabled after the latest version and requested timestamp. // This will force us to use the non-ICT search path below. Commit(latestVersion + 1, time + 1) } var commitOpt = if (ictEnablementCommit.timestamp <= time) { // ICT was enabled as-of the requested time if (snapshot.timestamp <= time) { // We just proved we should use the latest snapshot Some(Commit(snapshot.version, snapshot.timestamp)) } else { // start ICT search over [earliest available ICT version, latestVersion) val ictEnabledForEntireWindow = (ictEnablementCommit.version <= earliestVersion) val searchWindowLowerBoundCommit = if (ictEnabledForEntireWindow) placeholderEarliestCommit else ictEnablementCommit // Note that this search can return `placeholderEarliestCommit`. // The real timestamp of the earliest commit will be fetched later. getActiveCommitAtTimeFromICTRange( time, searchWindowLowerBoundCommit, latestVersion + 1, deltaLog.newDeltaHadoopConf(), deltaLog.logPath, deltaLog.store, numChunks = 10, spark, commitFileProvider) } } else { // ICT was NOT enabled as-of the requested time if (ictEnablementCommit.version <= earliestVersion) { // We're searching for a non-ICT time but the non-ICT commits are all missing. // If `canReturnEarliestCommit` is `false`, we need the details of the // earliest commit to populate the TimestampEarlierThanCommitRetentionException // error correctly. // Else, when `canReturnEarliestCommit` is `true`, the earliest commit // is the desired result. // The real timestamp of the earliest commit will be fetched later. Some(placeholderEarliestCommit) } else { // start non-ICT search over [earliestVersion, ictEnablementVersion) Some(getCommitFromNonICTRange(earliestVersion, end = ictEnablementCommit.version, time)) } } // We need to fetch the correct timestamp for the earliest commit if it was the result of the // search. // If commitOpt == ictEnablementCommit, we also need to validate the existence of the ICT // enablement commit. if (commitOpt.contains(placeholderEarliestCommit) || commitOpt.contains(ictEnablementCommit)) { commitOpt = getFirstCommitAndICTAfter( commitOpt.get.version, latestVersion, deltaLog.logPath, deltaLog.store, deltaLog.newDeltaHadoopConf(), commitFileProvider) } // Error handling val commit = commitOpt.getOrElse { throw DeltaErrors.noHistoryFound(deltaLog.logPath) } val commitTs = new Timestamp(commit.timestamp) val timestampFormatter = TimestampFormatter( DateTimeUtils.getTimeZone(SQLConf.get.sessionLocalTimeZone)) val tsString = DateTimeUtils.timestampToString( timestampFormatter, DateTimeUtils.fromJavaTimestamp(commitTs)) if (commit.timestamp > time && !canReturnEarliestCommit) { throw DeltaErrors.TimestampEarlierThanCommitRetentionException(timestamp, commitTs, tsString) } else if (commit.version == latestVersion && !canReturnLastCommit) { if (commit.timestamp < time) { throw DeltaErrors.timestampGreaterThanLatestCommit(timestamp, commitTs, tsString) } } commit } /** * Check whether the given version exists. * @param mustBeRecreatable whether the snapshot of this version needs to be recreated. * @param allowOutOfRange whether to allow the version is exceeding the latest snapshot version. */ def checkVersionExists( version: Long, catalogTableOpt: Option[CatalogTable], mustBeRecreatable: Boolean = true, allowOutOfRange: Boolean = false): Unit = { val earliest = if (mustBeRecreatable) { getEarliestRecreatableCommit } else { getEarliestDeltaFile(deltaLog) } val latest = deltaLog.update(catalogTableOpt = catalogTableOpt).version if (version < earliest || ((version > latest) && !allowOutOfRange)) { throw VersionNotFoundException(version, earliest, latest) } } /** * Searches for the latest commit with the timestamp, which has happened at or before `time` in * the range `[start, end)`. */ private def parallelSearch( time: Long, start: Long, end: Long): Commit = { parallelSearch0( spark, getSerializableHadoopConf, deltaLog.logPath.toString, time, start, end, maxKeysPerList) } /** * Get the earliest commit, which we can recreate. Note that this version isn't guaranteed to * exist when performing an action as a concurrent operation can delete the file during cleanup. * This value must be used as a lower bound. * * We search for the earliest checkpoint we have, or whether we have the 0th delta file, because * that way we can reconstruct the entire history of the table. This method assumes that the * commits are contiguous. */ private[delta] def getEarliestRecreatableCommit: Long = { val files = deltaLog.store.listFrom( FileNames.listingPrefix(deltaLog.logPath, 0), deltaLog.newDeltaHadoopConf()) .filter(f => FileNames.isDeltaFile(f) || FileNames.isCheckpointFile(f)) // A map of checkpoint version and number of parts, to number of parts observed val checkpointMap = new scala.collection.mutable.HashMap[(Long, Int), Int]() var smallestDeltaVersion = Long.MaxValue var lastCompleteCheckpoint: Option[Long] = None // Iterate through the log files - this will be in order starting from the lowest version. // Checkpoint files come before deltas, so when we see a checkpoint, we remember it and // return it once we detect that we've seen a smaller or equal delta version. while (files.hasNext) { val nextFilePath = files.next().getPath if (FileNames.isDeltaFile(nextFilePath)) { val version = FileNames.deltaVersion(nextFilePath) if (version == 0L) return version smallestDeltaVersion = math.min(version, smallestDeltaVersion) // Note that we also check this condition at the end of the function - we check it // here too to try and avoid more file listing when it's unnecessary. if (lastCompleteCheckpoint.exists(_ >= smallestDeltaVersion - 1)) { return lastCompleteCheckpoint.get } } else if (FileNames.isCheckpointFile(nextFilePath)) { val checkpointVersion = FileNames.checkpointVersion(nextFilePath) val parts = FileNames.numCheckpointParts(nextFilePath) if (parts.isEmpty) { lastCompleteCheckpoint = Some(checkpointVersion) } else { // if we have a multi-part checkpoint, we need to check that all parts exist val numParts = parts.getOrElse(1) val preCount = checkpointMap.getOrElse(checkpointVersion -> numParts, 0) if (numParts == preCount + 1) { lastCompleteCheckpoint = Some(checkpointVersion) } checkpointMap.put(checkpointVersion -> numParts, preCount + 1) } } } if (lastCompleteCheckpoint.exists(_ >= smallestDeltaVersion)) { return lastCompleteCheckpoint.get } else if (smallestDeltaVersion < Long.MaxValue) { throw DeltaErrors.noRecreatableHistoryFound(deltaLog.logPath) } else { // For Catalog Owned tables, there are two cases in which a DELTA_NO_COMMITS_FOUND // exception could be thrown: // // 1. If there is no checkpoint or commit 0, and there are only unbackfilled commits // in the table. // // In this case, the DELTA_NO_COMMITS_FOUND exception would be incorrect because there // are commits in the table, we just did not list them when trying to find the earliest // recreatable commit. Ideally we should throw the above NO_RECREATABLE_HISTORY_FOUND // exception but that requires an additional lookup of any unbackfilled commits at // the commit coordinator. // // It is not worth doing the extra lookup just to throw the correct exception because: // 1) We are already throwing a DELTA_NO_COMMITS_FOUND exception indicating // a potential problem. // 2) The table must be corrupted already to end up in this scenario. // // An example delta log structure for this case is shown below: // ``` // _delta_log/ // [x] 00000000000000000000.json -- Commit 0 has been backfilled but deleted. // _staged_commits/ // [√] 00000000000000000001..json -- Commit 1 and 2 have *not* been backfilled. // [√] 00000000000000000002..json // ``` // For the above table, we will throw DELTA_NO_COMMITS_FOUND exception when commit // `0.json` has been manually deleted and users are running commands like `versionAsOf 0` // on the table at the same time. // // 2. It indicates a real NO_RECREATABLE_HISTORY_FOUND exception, if all commits have been // backfilled, and we still can't find the recreatable commit. // // An example delta log structure for this case is shown below: // ``` // _delta_log/ // [x] 00000000000000000000.json -- Commit 0/1/2 have been backfilled but deleted. // [x] 00000000000000000001.json // [x] 00000000000000000002.json // _staged_commits/ // // ``` throw DeltaErrors.noHistoryFound(deltaLog.logPath) } } } /** Contains many utility methods that can also be executed on Spark executors. */ object DeltaHistoryManager extends DeltaLogging { /** * This thread pool is used by `getActiveCommitAtTime` to parallelize the search for * relevant commits when the feature inCommitTimestamps is enabled. */ private[delta] lazy val threadPool: DeltaThreadPool = DeltaThreadPool( "delta-history-manager", SparkEnv.get.conf.get(DeltaSQLConf.DELTA_HISTORY_MANAGER_THREAD_POOL_SIZE) ) /** Get the persisted commit info (if available) for the given delta file. */ def getCommitInfoOpt( logStore: LogStore, deltaFile: Path, hadoopConf: Configuration): Option[CommitInfo] = { val logs = logStore.readAsIterator(deltaFile, hadoopConf) try { logs .map(Action.fromJson) .collectFirst { case c: CommitInfo => c.copy(version = Some(deltaVersion(deltaFile))) } } finally { logs.close() } } /** * Get the earliest commit available for this table. Note that this version isn't guaranteed to * exist when performing an action as a concurrent operation can delete the file during cleanup. * This value must be used as a lower bound. */ def getEarliestDeltaFile(deltaLog: DeltaLog): Long = { deltaLog.store .listFrom( path = FileNames.listingPrefix(deltaLog.logPath, 0), hadoopConf = deltaLog.newDeltaHadoopConf()) .collectFirst { case DeltaFile(_, version) => version } .getOrElse { throw DeltaErrors.noHistoryFound(deltaLog.logPath) } } private def getCommitWithInCommitTimestamp( version: Long, commitFileStatus: FileStatus, logStore: LogStore, conf: Configuration): Option[Commit] = { val logs = logStore.readAsIterator(commitFileStatus, conf) try { val ci = logs .map(Action.fromJson) .collectFirst { case c: CommitInfo => c } Some(Commit(version, CommitInfo.getRequiredInCommitTimestamp(ci, version.toString))) } catch { case _: FileNotFoundException => None } finally { logs.close() } } /** * Returns the first available commit in the range [version, upperBoundExclusive). * The timestamp of the returned commit will be the ICT. If no ICT is found for the commit, * an exception will be thrown. * The function optimistically tries to read the commit info for `version` first. * For commits that have been backfilled as per `commitFileProvider`: if the * no commit is found at that version, it falls back to a listing. * For unbackfilled commits, an IllegalStateException is thrown if the commit is not found. */ private[delta] def getFirstCommitAndICTAfter( version: Long, upperBoundExclusive: Long, basePath: Path, logStore: LogStore, conf: Configuration, commitFileProvider: DeltaCommitFileProvider): Option[Commit] = { val deltaFile = commitFileProvider.deltaFile(version) val commitInfoOpt = try { getCommitInfoOpt(logStore, deltaFile, conf) } catch { case _: FileNotFoundException => None } if (commitInfoOpt.isDefined) { val timestamp = CommitInfo.getRequiredInCommitTimestamp(commitInfoOpt, version.toString) Some(Commit(version, timestamp)) } else if (version >= commitFileProvider.minUnbackfilledVersion) { // Unbackfilled commits should never disappear during the lifetime of time travel // query. throw new IllegalStateException( s"Could not find commit $version which was expected to be at path ${deltaFile.toString}.") } else { logStore .listFrom(FileNames.listingPrefix(basePath, version), conf) .takeWhile { fs => FileNames.getFileVersionOpt(fs.getPath).forall(_ < upperBoundExclusive) } .collectFirst { case DeltaFile(f, v) => getCommitWithInCommitTimestamp(v, f, logStore, conf) } .flatten } } /** * Returns the latest commit (with its inCommitTimestamp) that happened at or before * `searchTimestamp` in the range `[startCommit.version, end)`. * If no such commit exists, None is returned. * * The algorithm divides the range into `numChunks` chunks. It then finds the last * chunk where the ICT of its first available commit is less than or equal to `searchTimestamp`. * This chunk is then further divided into `numChunks` chunks and the process is repeated. */ private[delta] def getActiveCommitAtTimeFromICTRange( searchTimestamp: Long, startCommit: Commit, end: Long, conf: Configuration, basePath: Path, logStore: LogStore, numChunks: Long, spark: SparkSession, commitFileProvider: DeltaCommitFileProvider): Option[Commit] = { require(startCommit.version < end, "start must be less than end") var curStartCommit = startCommit var curEnd = end while (curStartCommit.version < curEnd) { val numVersionsInRange = curEnd - curStartCommit.version val chunkSize = math.max(numVersionsInRange / numChunks, 1) // min(chunkSize) = 1 and curStartCommit.version < end // therefore, getChunkEnd(chunkStart) will always be > chunkStart def getChunkEnd(chunkStart: Long): Long = math.min(chunkStart + chunkSize, curEnd) val chunkStartICTFutures = (curStartCommit.version until curEnd by chunkSize).map { chunkStart => if (chunkStart == curStartCommit.version) { CompletableFuture.completedFuture(Option(curStartCommit)) } else { threadPool.submit(spark) { getFirstCommitAndICTAfter( chunkStart, upperBoundExclusive = getChunkEnd(chunkStart), basePath, logStore, conf, commitFileProvider ) } } } val knownTightestLowerBoundCommit = chunkStartICTFutures .map(ThreadUtils.awaitResult(_, Duration.Inf)) .takeWhile(_.forall(_.timestamp <= searchTimestamp)) .flatten .lastOption .getOrElse { return None } val nextStartCommit = knownTightestLowerBoundCommit val nextEnd = getChunkEnd(nextStartCommit.version) if (nextStartCommit.version + 2 > nextEnd || knownTightestLowerBoundCommit.timestamp == searchTimestamp) { return Some(knownTightestLowerBoundCommit) } curStartCommit = nextStartCommit curEnd = nextEnd } None } /** * When calling getCommits, the initial few timestamp values may be wrong because they are not * properly monotonized. Callers should pass a start value at least * this far behind the first timestamp they care about if they need correct values. */ private[delta] val POTENTIALLY_UNMONOTONIZED_TIMESTAMPS = 100 /** * Returns the commit version and timestamps of all commits in `[start, end)`. If `end` is not * specified, will return all commits that exist after `start`. Will guarantee that the commits * returned will have both monotonically increasing versions as well as timestamps. * Note that this function will return non-ICT timestamps even for commits where * InCommitTimestamps are enabled. The caller is responsible for ensuring that the appropriate * timestamps are used. */ private[delta] def getCommitsWithNonIctTimestamps( logStore: LogStore, logPath: Path, start: Long, end: Option[Long], hadoopConf: Configuration): Array[Commit] = { val until = end.getOrElse(Long.MaxValue) val commits = logStore .listFrom(listingPrefix(logPath, start), hadoopConf) .collect { case DeltaFile(file, version) => Commit(version, file.getModificationTime) } .takeWhile(_.version < until) monotonizeCommitTimestamps(commits.toArray) } /** * Makes sure that the commit timestamps are monotonically increasing with respect to commit * versions. Requires the input commits to be sorted by the commit version. */ private def monotonizeCommitTimestamps[T <: CommitMarker](commits: Array[T]): Array[T] = { var i = 0 val length = commits.length while (i < length - 1) { val prevTimestamp = commits(i).getTimestamp assert(commits(i).getVersion < commits(i + 1).getVersion, "Unordered commits provided.") if (prevTimestamp >= commits(i + 1).getTimestamp) { logWarning(log"Found Delta commit ${MDC(DeltaLogKeys.VERSION, commits(i).getVersion)} " + log"with a timestamp ${MDC(DeltaLogKeys.TIMESTAMP, prevTimestamp)} " + log"which is greater than the next commit timestamp " + log"${MDC(DeltaLogKeys.TIMESTAMP2, commits(i + 1).getTimestamp)}.") commits(i + 1) = commits(i + 1).withTimestamp(prevTimestamp + 1).asInstanceOf[T] } i += 1 } commits } /** * Searches for the latest commit with the timestamp, which has happened at or before `time` in * the range `[start, end)`. The algorithm works as follows: * 1. We use Spark to list our commit history in parallel `maxKeysPerList` at a time. * 2. We then perform our search in each fragment of commits containing at most `maxKeysPerList` * elements. * 3. All fragments that are before `time` will return the last commit in the fragment. * 4. All fragments that are after `time` will exit early and return the first commit in the * fragment. * 5. The fragment that contains the version we are looking for will return the version we are * looking for. * 6. Once all the results are returned from Spark, we make sure that the commit timestamps are * monotonically increasing across the fragments, because we couldn't adjust for the * boundaries when working in parallel. * 7. We then return the version we are looking for in this smaller list on the Driver. * We will return the first available commit if the condition cannot be met. This method works * even for boundary commits, and can be best demonstrated through an example: * Imagine we have commits 999, 1000, 1001, 1002. t_999 < t_1000 but t_1000 > t_1001 and * t_1001 < t_1002. So at the the boundary, we will need to eventually adjust t_1001. Assume the * result needs to be t_1001 after the adjustment as t_search < t_1002 and t_search > t_1000. * What will happen is that the first fragment will return t_1000, and the second fragment will * return t_1001. On the Driver, we will adjust t_1001 = t_1000 + 1 milliseconds, and our linear * search will return t_1001. * * Placed in the static object to avoid serializability issues. * * @param spark The active SparkSession * @param conf The session specific Hadoop Configuration * @param logPath The path of the DeltaLog * @param time The timestamp to search for in milliseconds * @param start Earliest available commit version (approximate is acceptable) * @param end Latest available commit version (approximate is acceptable) * @param step The number with which to chunk each linear search across commits. Provide the * max number of keys returned by the underlying FileSystem for in a single RPC for * best results. */ private def parallelSearch0( spark: SparkSession, conf: SerializableConfiguration, logPath: String, time: Long, start: Long, end: Long, step: Long): Commit = { import org.apache.spark.sql.delta.implicits._ val possibleCommits = spark.range(start, end, step).mapPartitions { startVersions => val logStore = LogStore(SparkEnv.get.conf, conf.value) val basePath = new Path(logPath) startVersions.map { startVersion => val commits = getCommitsWithNonIctTimestamps( logStore, basePath, startVersion, Some(math.min(startVersion + step, end)), conf.value) if (commits.isEmpty) { None } else { Some(lastCommitBeforeTimestamp(commits, time).getOrElse(commits.head)) } } }.collect() // Spark should return the commits in increasing order as well val commitList = monotonizeCommitTimestamps(possibleCommits.flatten) if (commitList.isEmpty) { throw DeltaErrors.noHistoryFound(new Path(logPath)) } lastCommitBeforeTimestamp(commitList, time).getOrElse(commitList.head) } /** Returns the latest commit that happened at or before `time`. */ private def lastCommitBeforeTimestamp(commits: Seq[Commit], time: Long): Option[Commit] = { val i = commits.lastIndexWhere(_.timestamp <= time) if (i < 0) None else Some(commits(i)) } /** A helper class to represent the timestamp and version of a commit. */ case class Commit(version: Long, timestamp: Long) extends CommitMarker { override def withTimestamp(timestamp: Long): Commit = this.copy(timestamp = timestamp) override def getTimestamp: Long = timestamp override def getVersion: Long = version } /** * An iterator that helps select old log files for deletion. It takes the input iterator of log * files from the earliest file, and returns should-be-deleted files until the given maxTimestamp * or maxVersion to delete is reached. Note that this iterator may stop deleting files earlier * than maxTimestamp or maxVersion if it finds that files that need to be preserved for adjusting * the timestamps of subsequent files. Let's go through an example. Assume the following commit * history: * * +---------+-----------+--------------------+ * | Version | Timestamp | Adjusted Timestamp | * +---------+-----------+--------------------+ * | 0 | 0 | 0 | * | 1 | 5 | 5 | * | 2 | 10 | 10 | * | 3 | 7 | 11 | * | 4 | 8 | 12 | * | 5 | 14 | 14 | * +---------+-----------+--------------------+ * * As you can see from the example, we require timestamps to be monotonically increasing with * respect to the version of the commit, and each commit to have a unique timestamp. If we have * a commit which doesn't obey one of these two requirements, we adjust the timestamp of that * commit to be one millisecond greater than the previous commit. * * Given the above commit history, the behavior of this iterator will be as follows: * - For maxVersion = 1 and maxTimestamp = 9, we can delete versions 0 and 1 * - Until we receive maxVersion >= 4 and maxTimestamp >= 12, we can't delete versions 2 and 3. * This is because version 2 is used to adjust the timestamps of commits up to version 4. * - For maxVersion >= 5 and maxTimestamp >= 14 we can delete everything * The semantics of time travel guarantee that for a given timestamp, the user will ALWAYS get the * same version. Consider a user asks to get the version at timestamp 11. If all files are there, * we would return version 3 (timestamp 11) for this query. If we delete versions 0-2, the * original timestamp of version 3 (7) will not have an anchor to adjust on, and if the time * travel query is re-executed we would return version 4. This is the motivation behind this * iterator implementation. * * The implementation maintains an internal "maybeDelete" buffer of files that we are unsure of * deleting because they may be necessary to adjust time of future files. For each file we get * from the underlying iterator, we check whether it needs time adjustment or not. If it does need * time adjustment, then we cannot immediately decide whether it is safe to delete that file or * not and therefore we put it in each the buffer. Then we iteratively peek ahead at the future * files and accordingly decide whether to delete all the buffered files or retain them. * * @param underlying The iterator which gives the list of files in ascending version order * @param maxTimestamp The timestamp until which we can delete (inclusive). * @param maxVersion The version until which we can delete (inclusive). * @param versionGetter A method to get the commit version from the file path. */ class BufferingLogDeletionIterator( underlying: Iterator[FileStatus], maxTimestamp: Long, maxVersion: Long, versionGetter: Path => Long) extends Iterator[FileStatus] { /** * Our output iterator */ private val filesToDelete = new mutable.Queue[FileStatus]() /** * Our intermediate buffer which will buffer files as long as the last file requires a timestamp * adjustment. */ private val maybeDeleteFiles = new mutable.ArrayBuffer[FileStatus]() private var lastFile: FileStatus = _ private var hasNextCalled: Boolean = false // A map to keep track of multi-part checkpoints. val checkpointMap = new scala.collection.mutable.HashMap[(Long, Int), collection.mutable.Buffer[FileStatus]]() private def init(): Unit = { if (underlying.hasNext) { lastFile = underlying.next() maybeDeleteFiles.append(lastFile) } } init() /** Whether the given file can be deleted based on the version and retention timestamp input. */ private def shouldDeleteFile(file: FileStatus): Boolean = { file.getModificationTime <= maxTimestamp && versionGetter(file.getPath) <= maxVersion } /** * Files need a time adjustment if their timestamp isn't later than the lastFile. */ private def needsTimeAdjustment(file: FileStatus): Boolean = { versionGetter(lastFile.getPath) < versionGetter(file.getPath) && lastFile.getModificationTime >= file.getModificationTime } /** * Enqueue the files in the buffer if the last file is safe to delete. Clears the buffer. */ private def flushBuffer(): Unit = { if (maybeDeleteFiles.lastOption.exists(shouldDeleteFile)) { filesToDelete ++= maybeDeleteFiles } maybeDeleteFiles.clear() } /** * Peeks at the next file in the iterator. Based on the next file we can have three * possible outcomes: * - The underlying iterator returned a file, which doesn't require timestamp adjustment. If * the file in the buffer has expired, flush the buffer to our output queue. * - The underlying iterator returned a file, which requires timestamp adjustment. In this case, * we add this file to the buffer and fetch the next file * - The underlying iterator is empty. In this case, we check the last file in the buffer. If * it has expired, then flush the buffer to the output queue. * Once this method returns, the buffer is expected to have 1 file (last file of the * underlying iterator) unless the underlying iterator is fully consumed. */ private def queueFilesInBuffer(): Unit = { var continueBuffering = true while (continueBuffering && underlying.hasNext) { var currentFile = underlying.next() require(currentFile != null, "FileStatus iterator returned null") if (needsTimeAdjustment(currentFile)) { currentFile = new FileStatus( currentFile.getLen, currentFile.isDirectory, currentFile.getReplication, currentFile.getBlockSize, lastFile.getModificationTime + 1, currentFile.getPath) maybeDeleteFiles.append(currentFile) } else if (FileNames.isCheckpointFile(currentFile) && currentFile.getLen > 0) { // Only flush the buffer when we find a checkpoint. This is because we don't want to // delete the delta log files unless we have a checkpoint to ensure that non-expired // subsequent delta logs are valid. val numParts = FileNames.numCheckpointParts(currentFile.getPath) if (numParts.isEmpty) { // Single-part or V2 flushBuffer() maybeDeleteFiles.append(currentFile) continueBuffering = false } else { // Multi-part checkpoint val mpKey = versionGetter(currentFile.getPath) -> numParts.get val partBuffer = checkpointMap.getOrElse(mpKey, mutable.ArrayBuffer()) partBuffer.append(currentFile) checkpointMap.put(mpKey, partBuffer) if (numParts.get == partBuffer.size) { flushBuffer() partBuffer.foreach(f => maybeDeleteFiles.append(f)) checkpointMap.remove(mpKey) continueBuffering = false } } } else { maybeDeleteFiles.append(currentFile) } lastFile = currentFile } } override def hasNext: Boolean = { hasNextCalled = true if (filesToDelete.isEmpty) queueFilesInBuffer() filesToDelete.nonEmpty } override def next(): FileStatus = { if (!hasNextCalled) throw new NoSuchElementException() hasNextCalled = false filesToDelete.dequeue() } } } /** * class describing the output schema of * [[org.apache.spark.sql.delta.commands.DescribeDeltaHistoryCommand]] */ case class DeltaHistory( version: Option[Long], timestamp: Timestamp, userId: Option[String], userName: Option[String], operation: String, operationParameters: Map[String, String], job: Option[JobInfo], notebook: Option[NotebookInfo], clusterId: Option[String], readVersion: Option[Long], isolationLevel: Option[String], isBlindAppend: Option[Boolean], operationMetrics: Option[Map[String, String]], userMetadata: Option[String], engineInfo: Option[String]) extends CommitMarker { override def withTimestamp(timestamp: Long): DeltaHistory = { this.copy(timestamp = new Timestamp(timestamp)) } override def getTimestamp: Long = timestamp.getTime override def getVersion: Long = version.get } object DeltaHistory { /** Create an instance of [[DeltaHistory]] from [[CommitInfo]] */ def fromCommitInfo(ci: CommitInfo): DeltaHistory = { val operationParameters = CommitInfo.getLegacyPostDeserializationOperationParameters(ci.operationParameters) DeltaHistory( version = ci.version, timestamp = ci.timestamp, userId = ci.userId, userName = ci.userName, operation = ci.operation, operationParameters = operationParameters, job = ci.job, notebook = ci.notebook, clusterId = ci.clusterId, readVersion = ci.readVersion, isolationLevel = ci.isolationLevel, isBlindAppend = ci.isBlindAppend, operationMetrics = ci.operationMetrics, userMetadata = ci.userMetadata, engineInfo = ci.engineInfo) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaLog.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.io.IOException import java.lang.ref.WeakReference import java.net.URI import java.util.concurrent.TimeUnit import scala.collection.JavaConverters._ import scala.collection.mutable import scala.util.Try import scala.util.control.NonFatal import com.databricks.spark.util.TagDefinitions._ import org.apache.spark.sql.delta.DataFrameUtils import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.commands.WriteIntoDelta import org.apache.spark.sql.delta.coordinatedcommits.CoordinatedCommitsUtils import org.apache.spark.sql.delta.files.{TahoeBatchFileIndex, TahoeLogFileIndex} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.redirect.RedirectFeature import org.apache.spark.sql.delta.schema.{SchemaMergingUtils, SchemaUtils} import org.apache.spark.sql.delta.sources._ import org.apache.spark.sql.delta.storage.LogStoreProvider import org.apache.spark.sql.delta.util.{FileNames, PathWithFileSystem, Utils => DeltaUtils} import com.google.common.cache.{Cache, CacheBuilder, RemovalNotification} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, FileSystem, Path} import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.{FileSourceOptions, TableIdentifier} import org.apache.spark.sql.catalyst.analysis.{Resolver, UnresolvedAttribute} import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStatistics, CatalogTable} import org.apache.spark.sql.catalyst.expressions.{And, Attribute, Cast, Expression, Literal} import org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper import org.apache.spark.sql.catalyst.util.FailFastMode import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.expressions.UserDefinedFunction import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.sources.{BaseRelation, InsertableRelation} import org.apache.spark.sql.types.{StructField, StructType} import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.util._ /** * Used to query the current state of the log as well as modify it by adding * new atomic collections of actions. * * Internally, this class implements an optimistic concurrency control * algorithm to handle multiple readers or writers. Any single read * is guaranteed to see a consistent snapshot of the table. * * @param logPath Path of the Delta log JSONs. * @param dataPath Path of the data files. * @param options Filesystem options filtered from `allOptions`. * @param allOptions All options provided by the user, for example via `df.write.option()`. This * includes but not limited to filesystem and table properties. * @param clock Clock to be used when starting a new transaction. * @param initialCatalogTable The catalog table given when the log is initialized. */ class DeltaLog private( val logPath: Path, val dataPath: Path, val options: Map[String, String], val allOptions: Map[String, String], val clock: Clock, val initialCatalogTable: Option[CatalogTable] ) extends Checkpoints with MetadataCleanup with LogStoreProvider with SnapshotManagement with DeltaFileFormat with ProvidesUniFormConverters with ReadChecksum { import org.apache.spark.sql.delta.files.TahoeFileIndex import org.apache.spark.sql.delta.util.FileNames._ /** * Path to sidecar directory. * This is intentionally kept `lazy val` as otherwise any other constructor codepaths in DeltaLog * (e.g. SnapshotManagement etc) will see it as null as they are executed before this line is * called. */ lazy val sidecarDirPath: Path = FileNames.sidecarDirPath(logPath) protected def spark = SparkSession.active checkRequiredConfigurations() /** * Keep a reference to `SparkContext` used to create `DeltaLog`. `DeltaLog` cannot be used when * `SparkContext` is stopped. We keep the reference so that we can check whether the cache is * still valid and drop invalid `DeltaLog`` objects. */ private val sparkContext = new WeakReference(spark.sparkContext) /** * Returns the Hadoop [[Configuration]] object which can be used to access the file system. All * Delta code should use this method to create the Hadoop [[Configuration]] object, so that the * hadoop file system configurations specified in DataFrame options will come into effect. */ // scalastyle:off deltahadoopconfiguration final def newDeltaHadoopConf(): Configuration = spark.sessionState.newHadoopConfWithOptions(options) // scalastyle:on deltahadoopconfiguration /** Used to read and write physical log files and checkpoints. */ lazy val store = createLogStore(spark) /** Delta History Manager containing version and commit history. */ lazy val history = new DeltaHistoryManager( this, spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_HISTORY_PAR_SEARCH_THRESHOLD)) /** Initialize the variables in SnapshotManagement. */ createSnapshotAtInit(initialCatalogTable) /* --------------- * | Configuration | * --------------- */ /** * The max lineage length of a Snapshot before Delta forces to build a Snapshot from scratch. * Delta will build a Snapshot on top of the previous one if it doesn't see a checkpoint. * However, there is a race condition that when two writers are writing at the same time, * a writer may fail to pick up checkpoints written by another one, and the lineage will grow * and finally cause StackOverflowError. Hence we have to force to build a Snapshot from scratch * when the lineage length is too large to avoid hitting StackOverflowError. */ def maxSnapshotLineageLength: Int = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_MAX_SNAPSHOT_LINEAGE_LENGTH) private[delta] def incrementalCommitEnabled: Boolean = { spark.conf.get(DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED) } private[delta] def shouldVerifyIncrementalCommit: Boolean = { spark.conf.get(DeltaSQLConf.INCREMENTAL_COMMIT_VERIFY) || (DeltaUtils.isTesting && spark.conf.get(DeltaSQLConf.INCREMENTAL_COMMIT_FORCE_VERIFY_IN_TESTS)) } /** * The unique identifier for this table. * * WARNING: This value is volatile and can change during the lifetime of a DeltaLog instance, * e.g., when the snapshot is updated and the new snapshot has a different table id. Use with * care. */ def unsafeVolatileTableId: String = unsafeVolatileMetadata.id /** Returns the truncated table ID for logging purposes. */ private[delta] def truncatedUnsafeVolatileTableId: String = unsafeVolatileTableId.split("-").head /** * WARNING: This API is unsafe and deprecated. It will be removed in future versions. * Use the above unsafeVolatileTableId to get the most recently cached table id. */ @deprecated("This method is deprecated and will be removed in future versions. " + "Use unsafeVolatileTableId instead", "18.0") def tableId: String = unsafeVolatileTableId def getInitialCatalogTable: Option[CatalogTable] = initialCatalogTable /** * Combines the table id with the path of the table to ensure uniqueness. Normally the table id * should be globally unique, but nothing stops users from copying a Delta table directly to * a separate location, where the transaction log is copied directly, causing the table ids to * match. When users mutate the copied table, and then try to perform some checks joining the * two tables, optimizations that depend on the table id alone may not be correct. Hence we use a * composite id. */ private[delta] def compositeId: (String, Path) = unsafeVolatileTableId -> dataPath /** * Creates a [[LogicalRelation]] for a given [[DeltaLogFileIndex]], with all necessary file source * options taken from the Delta Log. All reads of Delta metadata files should use this method. */ def indexToRelation( index: DeltaLogFileIndex, schema: StructType = Action.logSchema): LogicalRelation = { DeltaLog.indexToRelation(spark, index, options, schema) } /** * Load the data using the FileIndex. This allows us to skip many checks that add overhead, e.g. * file existence checks, partitioning schema inference. */ def loadIndex( index: DeltaLogFileIndex, schema: StructType = Action.logSchema): DataFrame = { DataFrameUtils.ofRows(spark, indexToRelation(index, schema)) } /* ------------------ * | Delta Management | * ------------------ */ /** * Returns a new [[OptimisticTransaction]] that can be used to read the current state of the log * and then commit updates. The reads and updates will be checked for logical conflicts with any * concurrent writes to the log, and post-commit hooks can be used to notify the table's catalog * of schema changes, etc. * * Note that all reads in a transaction must go through the returned transaction object, and not * directly to the [[DeltaLog]] otherwise they will not be checked for conflicts. * * @param catalogTableOpt The [[CatalogTable]] for the table this transaction updates. Passing * None asserts this is a path-based table with no catalog entry. * * @param snapshotOpt THe [[Snapshot]] this transaction should use, if not latest. */ def startTransaction( catalogTableOpt: Option[CatalogTable], snapshotOpt: Option[Snapshot] = None): OptimisticTransaction = { TransactionExecutionObserver.getObserver.startingTransaction { new OptimisticTransaction(this, catalogTableOpt, snapshotOpt) } } /** Legacy/compat overload that does not require catalog table information. Avoid prod use. */ @deprecated("Please use the CatalogTable overload instead", "3.0") def startTransaction(): OptimisticTransaction = { startTransaction(catalogTableOpt = None, snapshotOpt = None) } /** * Execute a piece of code within a new [[OptimisticTransaction]]. Reads/write sets will * be recorded for this table, and all other tables will be read * at a snapshot that is pinned on the first access. * * @param catalogTableOpt The [[CatalogTable]] for the table this transaction updates. Passing * None asserts this is a path-based table with no catalog entry. * * @param snapshotOpt THe [[Snapshot]] this transaction should use, if not latest. * @note This uses thread-local variable to make the active transaction visible. So do not use * multi-threaded code in the provided thunk. */ def withNewTransaction[T]( catalogTableOpt: Option[CatalogTable], snapshotOpt: Option[Snapshot] = None)( thunk: OptimisticTransaction => T): T = { val txn = startTransaction(catalogTableOpt, snapshotOpt) OptimisticTransaction.setActive(txn) try { thunk(txn) } finally { OptimisticTransaction.clearActive() } } /** Legacy/compat overload that does not require catalog table information. Avoid prod use. */ @deprecated("Please use the CatalogTable overload instead", "3.0") def withNewTransaction[T](thunk: OptimisticTransaction => T): T = { val txn = startTransaction() OptimisticTransaction.setActive(txn) try { thunk(txn) } finally { OptimisticTransaction.clearActive() } } /** * Upgrade the table's protocol version, by default to the maximum recognized reader and writer * versions in this Delta release. This method only upgrades protocol version, and will fail if * the new protocol version is not a superset of the original one used by the snapshot. */ def upgradeProtocol( catalogTable: Option[CatalogTable], snapshot: Snapshot, newVersion: Protocol): Unit = { val currentVersion = snapshot.protocol if (newVersion == currentVersion) { logConsole(s"Table $dataPath is already at protocol version $newVersion.") return } if (!currentVersion.canUpgradeTo(newVersion)) { throw new ProtocolDowngradeException(currentVersion, newVersion) } val txn = startTransaction(catalogTable, Some(snapshot)) try { SchemaMergingUtils.checkColumnNameDuplication(txn.metadata.schema, "in the table schema") } catch { case e: AnalysisException => throw DeltaErrors.duplicateColumnsOnUpdateTable(e) } txn.commit(Seq(newVersion), DeltaOperations.UpgradeProtocol(newVersion)) logConsole(s"Upgraded table at $dataPath to $newVersion.") } /** * Get all actions starting from "startVersion" (inclusive). If `startVersion` doesn't exist, * return an empty Iterator. * Callers are encouraged to use the other override which takes the endVersion if available to * avoid I/O and improve performance of this method. */ def getChanges( startVersion: Long, catalogTableOpt: Option[CatalogTable] = None, failOnDataLoss: Boolean = false): Iterator[(Long, Seq[Action])] = { getChangeLogFiles( startVersion, catalogTableOpt, failOnDataLoss).map { case (version, status) => (version, store.read(status, newDeltaHadoopConf()).map(Action.fromJson(_))) } } private[sql] def getChanges( startVersion: Long, endVersion: Long, catalogTableOpt: Option[CatalogTable], failOnDataLoss: Boolean): Iterator[(Long, Seq[Action])] = { getChangeLogFiles( startVersion, endVersion, catalogTableOpt, failOnDataLoss).map { case (version, status) => (version, store.read(status, newDeltaHadoopConf()).map(Action.fromJson(_))) } } private[sql] def getChangeLogFiles( startVersion: Long, endVersion: Long, catalogTableOpt: Option[CatalogTable], failOnDataLoss: Boolean): Iterator[(Long, FileStatus)] = { implicit class IteratorWithStopAtHelper[T](underlying: Iterator[T]) { // This method is used to stop the iterator when the condition is met. def stopAt(stopAtFunc: (T) => Boolean): Iterator[T] = new Iterator[T] { var shouldStop = false override def hasNext: Boolean = !shouldStop && underlying.hasNext override def next(): T = { val v = underlying.next() shouldStop = stopAtFunc(v) v } } } getChangeLogFiles(startVersion, catalogTableOpt, failOnDataLoss) // takeWhile always looks at one extra item, which can trigger unnecessary work. Instead, we // stop if we've seen the item we believe should be the last interesting item, without // examining the one that follows. .stopAt { case (version, _) => version >= endVersion } // The last element in this iterator may not be <= endVersion, so we need to filter it out. .takeWhile { case (version, _) => version <= endVersion } } /** * Get access to all actions starting from "startVersion" (inclusive) via [[FileStatus]]. * If `startVersion` doesn't exist, return an empty Iterator. * Callers are encouraged to use the other override which takes the endVersion if available to * avoid I/O and improve performance of this method. */ def getChangeLogFiles( startVersion: Long, catalogTableOpt: Option[CatalogTable] = None, failOnDataLoss: Boolean = false): Iterator[(Long, FileStatus)] = { val deltasWithVersion = CoordinatedCommitsUtils.commitFilesIterator( this, catalogTableOpt, startVersion) // Subtract 1 to ensure that we have the same check for the inclusive startVersion var lastSeenVersion = startVersion - 1 deltasWithVersion.map { case (status, version) => if (failOnDataLoss && version > lastSeenVersion + 1) { throw DeltaErrors.failOnDataLossException(lastSeenVersion + 1, version) } lastSeenVersion = version (version, status) } } /* --------------------- * | Protocol validation | * --------------------- */ /** * Asserts the highest protocol supported by this client is not less than what required by the * table for performing read or write operations. This ensures the client to support a * greater-or-equal protocol versions and recognizes/supports all features enabled by the table. * * The operation type to be checked is passed as a string in `readOrWrite`. Valid values are * `read` and `write`. */ private def protocolCheck(tableProtocol: Protocol, readOrWrite: String): Unit = { val unsupportedTestFeatures = if (spark.conf.get(DeltaSQLConf.UNSUPPORTED_TESTING_FEATURES_ENABLED)) { TableFeature.testUnsupportedFeatures.toSeq } else { Seq.empty } val clientSupportedProtocol = Action.supportedProtocolVersion(featuresToExclude = unsupportedTestFeatures) // Depending on the operation, pull related protocol versions out of Protocol objects. // `getEnabledFeatures` is a pointer to pull reader/writer features out of a Protocol. val (clientSupportedVersions, tableRequiredVersion, getEnabledFeatures) = readOrWrite match { case "read" => ( Action.supportedReaderVersionNumbers, tableProtocol.minReaderVersion, (f: Protocol) => f.readerFeatureNames) case "write" => ( Action.supportedWriterVersionNumbers, tableProtocol.minWriterVersion, (f: Protocol) => f.writerFeatureNames) case _ => throw new IllegalArgumentException("Table operation must be either `read` or `write`.") } // Check is complete when both the protocol version and all referenced features are supported. val clientSupportedFeatureNames = getEnabledFeatures(clientSupportedProtocol) val tableEnabledFeatureNames = getEnabledFeatures(tableProtocol) if (tableEnabledFeatureNames.subsetOf(clientSupportedFeatureNames) && clientSupportedVersions.contains(tableRequiredVersion)) { return } // Otherwise, either the protocol version, or few features referenced by the table, is // unsupported. val clientUnsupportedFeatureNames = tableEnabledFeatureNames.diff(clientSupportedFeatureNames) // Prepare event log constants and the appropriate error message handler. val (opType, versionKey, unsupportedFeaturesException) = readOrWrite match { case "read" => ( "delta.protocol.failure.read", "minReaderVersion", DeltaErrors.unsupportedReaderTableFeaturesInTableException _) case "write" => ( "delta.protocol.failure.write", "minWriterVersion", DeltaErrors.unsupportedWriterTableFeaturesInTableException _) } recordDeltaEvent( this, opType, data = Map( "clientVersion" -> clientSupportedVersions.max, versionKey -> tableRequiredVersion, "clientFeatures" -> clientSupportedFeatureNames.mkString(","), "clientUnsupportedFeatures" -> clientUnsupportedFeatureNames.mkString(","))) if (!clientSupportedVersions.contains(tableRequiredVersion)) { throw new InvalidProtocolVersionException( dataPath.toString(), tableProtocol.minReaderVersion, tableProtocol.minWriterVersion, Action.supportedReaderVersionNumbers.toSeq, Action.supportedWriterVersionNumbers.toSeq) } else { throw unsupportedFeaturesException(dataPath.toString(), clientUnsupportedFeatureNames) } } /** * Asserts that the table's protocol enabled all features that are active in the metadata. * * A mismatch shouldn't happen when the table has gone through a proper write process because we * require all active features during writes. However, other clients may void this guarantee. */ def assertTableFeaturesMatchMetadata( targetProtocol: Protocol, targetMetadata: Metadata): Unit = { if (!targetProtocol.supportsReaderFeatures && !targetProtocol.supportsWriterFeatures) return val protocolEnabledFeatures = targetProtocol.writerFeatureNames .flatMap(TableFeature.featureNameToFeature) val activeFeatures = Protocol.extractAutomaticallyEnabledFeatures(spark, targetMetadata, targetProtocol) val activeButNotEnabled = activeFeatures.diff(protocolEnabledFeatures) if (activeButNotEnabled.nonEmpty) { throw DeltaErrors.tableFeatureMismatchException(activeButNotEnabled.map(_.name)) } } /** * Asserts that the client is up to date with the protocol and allowed to read the table that is * using the given `protocol`. */ def protocolRead(protocol: Protocol): Unit = { protocolCheck(protocol, "read") } /** * Asserts that the client is up to date with the protocol and allowed to write to the table * that is using the given `protocol`. */ def protocolWrite(protocol: Protocol): Unit = { protocolCheck(protocol, "write") } /* ---------------------------------------- * | Log Directory Management and Retention | * ---------------------------------------- */ /** * Whether a Delta table exists at this directory. * It is okay to use the cached volatile snapshot here, since the worst case is that the table * has recently started existing which hasn't been picked up here. If so, any subsequent command * that updates the table will see the right value. */ def tableExists: Boolean = unsafeVolatileSnapshot.version >= 0 def isSameLogAs(otherLog: DeltaLog): Boolean = this.compositeId == otherLog.compositeId /** Creates the log directory and commit directory if it does not exist. */ def createLogDirectoriesIfNotExists(): Unit = { val fs = PathWithFileSystem.withConf(logPath, newDeltaHadoopConf()).fs def createDirIfNotExists(path: Path): Unit = { // Optimistically attempt to create the directory first without checking its existence. // This is efficient because we're assuming it's more likely that the directory doesn't // exist and it saves an filesystem existence check in that case. val (success, mkdirsIOExceptionOpt) = try { // Return value of false should mean the directory already existed (not an error) but // we will verify below because we're paranoid about buggy FileSystem implementations. (fs.mkdirs(path), None) } catch { // A FileAlreadyExistsException is expected if a non-directory object exists but an explicit // check is needed because buggy Hadoop FileSystem.mkdir wrongly throws the exception even // on existing directories. case io: IOException => val dirExists = try { fs.getFileStatus(path).isDirectory } catch { case NonFatal(_) => false } (dirExists, Some(io)) } if (!success) { throw DeltaErrors.cannotCreateLogPathException( logPath = logPath.toString, cause = mkdirsIOExceptionOpt.orNull) } } createDirIfNotExists(FileNames.commitDirPath(logPath)) } /* ------------ * | Integration | * ------------ */ /** * Returns a [[org.apache.spark.sql.DataFrame]] containing the new files within the specified * version range. */ def createDataFrame( snapshot: SnapshotDescriptor, addFiles: Seq[AddFile], isStreaming: Boolean = false, actionTypeOpt: Option[String] = None): DataFrame = { val actionType = actionTypeOpt.getOrElse(if (isStreaming) "streaming" else "batch") // It's ok to not pass down the partitionSchema to TahoeBatchFileIndex. Schema evolution will // ensure any partitionSchema changes will be captured, and upon restart, the new snapshot will // be initialized with the correct partition schema again. val fileIndex = new TahoeBatchFileIndex(spark, actionType, addFiles, this, dataPath, snapshot) // Drop null type columns from the relation's schema if it's not a streaming query until // null type columns are fully supported. val dropNullTypeColumnsFromSchema = if (isStreaming) { // Can force the legacy behavior(dropping nullType columns) with a flag. SQLConf.get.getConf(DeltaSQLConf.DELTA_STREAMING_CREATE_DATAFRAME_DROP_NULL_COLUMNS) } else { // Allow configurable behavior for non-streaming sources. This is used for testing. SQLConf.get.getConf(DeltaSQLConf.DELTA_CREATE_DATAFRAME_DROP_NULL_COLUMNS) } val relation = buildHadoopFsRelationWithFileIndex( snapshot, fileIndex, bucketSpec = None, dropNullTypeColumnsFromSchema = dropNullTypeColumnsFromSchema) DataFrameUtils.ofRows(spark, LogicalRelation(relation, isStreaming = isStreaming)) } /** * Returns a [[BaseRelation]] that contains all of the data present * in the table. This relation will be continually updated * as files are added or removed from the table. However, new [[BaseRelation]] * must be requested in order to see changes to the schema. */ def createRelation( partitionFilters: Seq[Expression] = Nil, snapshotToUseOpt: Option[Snapshot] = None, catalogTableOpt: Option[CatalogTable] = None, isTimeTravelQuery: Boolean = false): BaseRelation = { /** Used to link the files present in the table into the query planner. */ // TODO: If snapshotToUse is unspecified, get the correct snapshot from update() val snapshotToUse = snapshotToUseOpt.getOrElse(unsafeVolatileSnapshot) if (snapshotToUse.version < 0) { // A negative version here means the dataPath is an empty directory. Read query should error // out in this case. throw DeltaErrors.pathNotExistsException(dataPath.toString) } val fileIndex = TahoeLogFileIndex( spark, this, dataPath, snapshotToUse, catalogTableOpt, partitionFilters, isTimeTravelQuery) var bucketSpec: Option[BucketSpec] = None val r = buildHadoopFsRelationWithFileIndex(snapshotToUse, fileIndex, bucketSpec = bucketSpec) new HadoopFsRelation( r.location, r.partitionSchema, r.dataSchema, r.bucketSpec, r.fileFormat, r.options )(spark) with InsertableRelation { def insert(data: DataFrame, overwrite: Boolean): Unit = { val mode = if (overwrite) SaveMode.Overwrite else SaveMode.Append WriteIntoDelta( deltaLog = DeltaLog.this, mode = mode, new DeltaOptions(Map.empty[String, String], spark.sessionState.conf), partitionColumns = Seq.empty, configuration = Map.empty, data = data, catalogTableOpt = catalogTableOpt).run(spark) } } } def buildHadoopFsRelationWithFileIndex( snapshot: SnapshotDescriptor, fileIndex: TahoeFileIndex, bucketSpec: Option[BucketSpec], dropNullTypeColumnsFromSchema: Boolean = true): HadoopFsRelation = { val dataSchema = if (dropNullTypeColumnsFromSchema) { SchemaUtils.dropNullTypeColumns(snapshot.metadata.schema) } else { snapshot.metadata.schema } HadoopFsRelation( fileIndex, partitionSchema = DeltaTableUtils.removeInternalDeltaMetadata( spark, DeltaTableUtils.removeInternalWriterMetadata(spark, snapshot.metadata.partitionSchema) ), // We pass all table columns as `dataSchema` so that Spark will preserve the partition // column locations. Otherwise, for any partition columns not in `dataSchema`, Spark would // just append them to the end of `dataSchema`. dataSchema = DeltaTableUtils.removeInternalDeltaMetadata( spark, DeltaTableUtils.removeInternalWriterMetadata(spark, dataSchema) ), bucketSpec = bucketSpec, fileFormat(snapshot.protocol, snapshot.metadata), // `metadata.format.options` is not set today. Even if we support it in future, we shouldn't // store any file system options since they may contain credentials. Hence, it will never // conflict with `DeltaLog.options`. snapshot.metadata.format.options ++ options)(spark) } /** * Verify the required Spark conf for delta * Throw `DeltaErrors.configureSparkSessionWithExtensionAndCatalog` exception if * `spark.sql.catalog.spark_catalog` config is missing. We do not check for * `spark.sql.extensions` because DeltaSparkSessionExtension can alternatively * be activated using the `.withExtension()` API. This check can be disabled * by setting DELTA_CHECK_REQUIRED_SPARK_CONF to false. */ protected def checkRequiredConfigurations(): Unit = { if (spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_REQUIRED_SPARK_CONFS_CHECK)) { if (!spark.conf.contains(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key)) { throw DeltaErrors.configureSparkSessionWithExtensionAndCatalog(None) } } } /** * Returns a proper path canonicalization function for the current Delta log. * * If `runsOnExecutors` is true, the returned method will use a broadcast Hadoop Configuration * so that the method is suitable for execution on executors. Otherwise, the returned method * will use a local Hadoop Configuration and the method can only be executed on the driver. */ private[delta] def getCanonicalPathFunction(runsOnExecutors: Boolean): String => String = { val hadoopConf = newDeltaHadoopConf() // Wrap `hadoopConf` with a method to delay the evaluation to run on executors. val getHadoopConf = if (runsOnExecutors) { val broadcastHadoopConf = spark.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) () => broadcastHadoopConf.value.value } else { () => hadoopConf } new DeltaLog.CanonicalPathFunction(getHadoopConf) } /** * Returns a proper path canonicalization UDF for the current Delta log. * * If `runsOnExecutors` is true, the returned UDF will use a broadcast Hadoop Configuration. * Otherwise, the returned UDF will use a local Hadoop Configuration and the UDF can * only be executed on the driver. */ private[delta] def getCanonicalPathUdf(runsOnExecutors: Boolean = true): UserDefinedFunction = { DeltaUDF.stringFromString(getCanonicalPathFunction(runsOnExecutors)) } } object DeltaLog extends DeltaLogging { /** * The key type of `DeltaLog` cache. It consists of * - The canonicalized table path * - File system options (options starting with "fs." or "dfs." prefix) passed into * `DataFrameReader/Writer` */ case class DeltaLogCacheKey( path: Path, fsOptions: Map[String, String] ) /** The name of the subdirectory that holds Delta metadata files */ private[delta] val LOG_DIR_NAME = "_delta_log" private[delta] def logPathFor(dataPath: String): Path = logPathFor(new Path(dataPath)) private[delta] def logPathFor(dataPath: Path): Path = DeltaTableUtils.safeConcatPaths(dataPath, LOG_DIR_NAME) /** * We create only a single [[DeltaLog]] for any given `DeltaLogCacheKey` to avoid wasted work * in reconstructing the log. */ private[delta] def getOrCreateCache(conf: SQLConf): Cache[DeltaLogCacheKey, DeltaLog] = synchronized { deltaLogCache match { case Some(c) => c case None => val builder = createCacheBuilder(conf) .removalListener( (removalNotification: RemovalNotification[DeltaLogCacheKey, DeltaLog]) => { val log = removalNotification.getValue // TODO: We should use ref-counting to uncache snapshots instead of a manual timed op try log.unsafeVolatileSnapshot.uncache() catch { case _: java.lang.NullPointerException => // Various layers will throw null pointer if the RDD is already gone. } }) deltaLogCache = Some(builder.build[DeltaLogCacheKey, DeltaLog]()) deltaLogCache.get } } private var deltaLogCache: Option[Cache[DeltaLogCacheKey, DeltaLog]] = None /** * Helper to create delta log caches */ private def createCacheBuilder(conf: SQLConf): CacheBuilder[AnyRef, AnyRef] = { val cacheRetention = conf.getConf(DeltaSQLConf.DELTA_LOG_CACHE_RETENTION_MINUTES) val cacheSize = conf .getConf(DeltaSQLConf.DELTA_LOG_CACHE_SIZE) .max(sys.props.get("delta.log.cacheSize").map(_.toLong).getOrElse(0L)) CacheBuilder .newBuilder() .expireAfterAccess(cacheRetention, TimeUnit.MINUTES) .maximumSize(cacheSize) } /** * Creates a [[LogicalRelation]] for a given [[DeltaLogFileIndex]], with all necessary file source * options taken from the Delta Log. All reads of Delta metadata files should use this method. */ def indexToRelation( spark: SparkSession, index: DeltaLogFileIndex, additionalOptions: Map[String, String], schema: StructType = Action.logSchema): LogicalRelation = { val formatSpecificOptions: Map[String, String] = index.format match { case DeltaLogFileIndex.COMMIT_FILE_FORMAT => jsonCommitParseOption case _ => Map.empty } // Delta should NEVER ignore missing or corrupt metadata files, because doing so can render the // entire table unusable. Hard-wire that into the file source options so the user can't override // it by setting spark.sql.files.ignoreCorruptFiles or spark.sql.files.ignoreMissingFiles. val allOptions = additionalOptions ++ formatSpecificOptions ++ Map( FileSourceOptions.IGNORE_CORRUPT_FILES -> "false", FileSourceOptions.IGNORE_MISSING_FILES -> "false" ) val fsRelation = HadoopFsRelation( index, index.partitionSchema, schema, None, index.format, allOptions)(spark) LogicalRelation(fsRelation) } // Don't tolerate malformed JSON when parsing Delta log actions (default is PERMISSIVE) val jsonCommitParseOption = Map("mode" -> FailFastMode.name) /** Helper for creating a log when it stored at the root of the data. */ def forTable(spark: SparkSession, dataPath: String): DeltaLog = { apply( spark, logPathFor(dataPath), options = Map.empty, initialCatalogTable = None, new SystemClock) } /** Helper for creating a log when it stored at the root of the data. */ def forTable(spark: SparkSession, dataPath: Path): DeltaLog = { apply(spark, logPathFor(dataPath), initialCatalogTable = None, new SystemClock) } /** Helper for creating a log when it stored at the root of the data. */ def forTable(spark: SparkSession, dataPath: Path, options: Map[String, String]): DeltaLog = { apply(spark, logPathFor(dataPath), options, initialCatalogTable = None, new SystemClock) } /** Helper for creating a log when it stored at the root of the data. */ def forTable(spark: SparkSession, dataPath: Path, clock: Clock): DeltaLog = { apply(spark, logPathFor(dataPath), initialCatalogTable = None, clock) } /** Helper for creating a log for the table. */ def forTable(spark: SparkSession, tableName: TableIdentifier): DeltaLog = { forTable(spark, tableName, new SystemClock) } /** Helper for creating a log for the table. */ def forTable(spark: SparkSession, table: CatalogTable): DeltaLog = { forTable(spark, table, new SystemClock) } /** Helper for creating a log for the table. */ def forTable(spark: SparkSession, tableName: TableIdentifier, clock: Clock): DeltaLog = { if (DeltaTableIdentifier.isDeltaPath(spark, tableName)) { forTable(spark, new Path(tableName.table), clock) } else { forTable(spark, spark.sessionState.catalog.getTableMetadata(tableName), clock) } } /** Helper for creating a log for the table. */ def forTable(spark: SparkSession, table: CatalogTable, options: Map[String, String]): DeltaLog = { apply( spark, logPathFor(new Path(table.location)), options, Some(table), new SystemClock) } /** Helper for creating a log for the table. */ def forTable(spark: SparkSession, table: CatalogTable, clock: Clock): DeltaLog = { apply(spark, logPathFor(new Path(table.location)), Some(table), clock) } private def apply( spark: SparkSession, rawPath: Path, initialCatalogTable: Option[CatalogTable], clock: Clock): DeltaLog = apply(spark, rawPath, options = Map.empty, initialCatalogTable, clock) /** Helper for creating a log for the table. */ private[delta] def forTable( spark: SparkSession, dataPath: Path, options: Map[String, String], catalogTable: Option[CatalogTable]): DeltaLog = apply(spark, logPathFor(dataPath), options, catalogTable, new SystemClock) /** Helper for getting a log, as well as the latest snapshot, of the table */ def forTableWithSnapshot(spark: SparkSession, dataPath: String): (DeltaLog, Snapshot) = withFreshSnapshot { clock => (forTable(spark, new Path(dataPath), clock), None) } /** Helper for getting a log, as well as the latest snapshot, of the table */ def forTableWithSnapshot(spark: SparkSession, dataPath: Path): (DeltaLog, Snapshot) = withFreshSnapshot { clock => (forTable(spark, dataPath, clock), None) } /** Helper for getting a log, as well as the latest snapshot, of the table */ def forTableWithSnapshot( spark: SparkSession, tableName: TableIdentifier): (DeltaLog, Snapshot) = { withFreshSnapshot { clock => if (DeltaTableIdentifier.isDeltaPath(spark, tableName)) { (forTable(spark, new Path(tableName.table)), None) } else { val catalogTable = spark.sessionState.catalog.getTableMetadata(tableName) (forTable(spark, catalogTable, clock), Some(catalogTable)) } } } /** Helper for getting a log, as well as the latest snapshot, of the table */ def forTableWithSnapshot( spark: SparkSession, dataPath: Path, options: Map[String, String]): (DeltaLog, Snapshot) = withFreshSnapshot { clock => val deltaLog = apply(spark, logPathFor(dataPath), options, initialCatalogTable = None, clock) (deltaLog, None) } /** Helper for getting a log, as well as the latest snapshot, of the table */ def forTableWithSnapshot( spark: SparkSession, table: CatalogTable, options: Map[String, String]): (DeltaLog, Snapshot) = withFreshSnapshot { clock => val deltaLog = apply(spark, logPathFor(new Path(table.location)), options, Some(table), clock) (deltaLog, Some(table)) } /** * Helper method for transforming a given delta log path to the consistent formal path format. */ def formalizeDeltaPath( spark: SparkSession, options: Map[String, String], rootPath: Path): Path = { val fileSystemOptions: Map[String, String] = if (spark.sessionState.conf.getConf( DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS)) { options.filterKeys { k => DeltaTableUtils.validDeltaTableHadoopPrefixes.exists(k.startsWith) }.toMap } else { Map.empty } // scalastyle:off deltahadoopconfiguration val hadoopConf = spark.sessionState.newHadoopConfWithOptions(fileSystemOptions) // scalastyle:on deltahadoopconfiguration PathWithFileSystem .withConf(rootPath, hadoopConf) .fs .makeQualified(rootPath) } /** * Helper function to be used with the forTableWithSnapshot calls. Thunk is a * partially applied DeltaLog.forTable call, which we can then wrap around with a * snapshot update. We use the system clock to avoid back-to-back updates. */ private[delta] def withFreshSnapshot( thunk: Clock => (DeltaLog, Option[CatalogTable])): (DeltaLog, Snapshot) = { val clock = new SystemClock val ts = clock.getTimeMillis() val (deltaLog, catalogTableOpt) = thunk(clock) val snapshot = deltaLog.update(checkIfUpdatedSinceTs = Some(ts), catalogTableOpt = catalogTableOpt) (deltaLog, snapshot) } private def apply( spark: SparkSession, rawPath: Path, options: Map[String, String], initialCatalogTable: Option[CatalogTable], clock: Clock ): DeltaLog = { // Construct the filesystem options based on the DataFrameReader/Writer options, and if it's // a catalog based table, we need combine both options and catalog-based table storage // properties since all cloud credential information are stored in storage properties. val catalogTableStorageProps = initialCatalogTable .map(t => t.storage.properties.filter { case (k, _) => DeltaTableUtils.validDeltaTableHadoopPrefixes.exists(k.startsWith) }) .getOrElse(Map.empty) val fileSystemOptions: Map[String, String] = if (spark.sessionState.conf.getConf( DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS)) { // We pick up only file system options so that we don't pass any parquet or json options to // the code that reads Delta transaction logs. catalogTableStorageProps ++ options.filterKeys { k => DeltaTableUtils.validDeltaTableHadoopPrefixes.exists(k.startsWith) }.toMap } else { catalogTableStorageProps } // scalastyle:off deltahadoopconfiguration val hadoopConf = spark.sessionState.newHadoopConfWithOptions(fileSystemOptions) // scalastyle:on deltahadoopconfiguration val path = PathWithFileSystem .withConf(rawPath, hadoopConf) .fs .makeQualified(rawPath) def createDeltaLog(tablePath: Path = path): DeltaLog = recordDeltaOperation( null, "delta.log.create", Map(TAG_TAHOE_PATH -> tablePath.getParent.toString)) { AnalysisHelper.allowInvokingTransformsInAnalyzer { new DeltaLog( logPath = tablePath, dataPath = tablePath.getParent, options = fileSystemOptions, allOptions = options, clock = clock, initialCatalogTable = initialCatalogTable ) } } val cacheKey = DeltaLogCacheKey( path, fileSystemOptions) def getDeltaLogFromCache: DeltaLog = { // The following cases will still create a new ActionLog even if there is a cached // ActionLog using a different format path: // - Different `scheme` // - Different `authority` (e.g., different user tokens in the path) // - Different mount point. try { getOrCreateCache(spark.sessionState.conf) .get(cacheKey, () => { createDeltaLog() } ) } catch { case e: com.google.common.util.concurrent.UncheckedExecutionException => throw e.getCause case e: java.util.concurrent.ExecutionException => throw e.getCause } } def initializeDeltaLog(): DeltaLog = { val deltaLog = getDeltaLogFromCache if (Option(deltaLog.sparkContext.get).map(_.isStopped).getOrElse(true)) { // Invalid the cached `DeltaLog` and create a new one because the `SparkContext` of the // cached `DeltaLog` has been stopped. getOrCreateCache(spark.sessionState.conf).invalidate(cacheKey) getDeltaLogFromCache } else { deltaLog } } val deltaLog = initializeDeltaLog() // The deltaLog object may be cached while other session updates table redirect property. // To avoid this potential race condition, we would add a validation inside deltaLog.update // method to ensure deltaLog points to correct place after snapshot is updated. val redirectConfigOpt = RedirectFeature.needDeltaLogRedirect( spark, deltaLog, initialCatalogTable ) redirectConfigOpt.map { redirectConfig => val (redirectLoc, catalogTableOpt) = RedirectFeature .getRedirectLocationAndTable(spark, deltaLog, redirectConfig) val formalizedPath = formalizeDeltaPath(spark, options, redirectLoc) // with redirect prefix to prevent interference between redirection and normal access. val redirectKey = new Path(RedirectFeature.DELTALOG_PREFIX, redirectLoc) val deltaLogCacheKey = DeltaLogCacheKey( redirectKey, fileSystemOptions) getOrCreateCache(spark.sessionState.conf).get( deltaLogCacheKey, () => { var redirectedDeltaLog = new DeltaLog( logPath = formalizedPath, dataPath = formalizedPath.getParent, options = fileSystemOptions, allOptions = options, clock = clock, initialCatalogTable = catalogTableOpt ) redirectedDeltaLog } ) }.getOrElse(deltaLog) } /** Invalidate the cached DeltaLog object for the given `dataPath`. */ def invalidateCache(spark: SparkSession, dataPath: Path): Unit = { try { val rawPath = logPathFor(dataPath) // scalastyle:off deltahadoopconfiguration // This method cannot be called from DataFrameReader/Writer so it's safe to assume the user // has set the correct file system configurations in the session configs. val fs = PathWithFileSystem.withConf(rawPath, spark.sessionState.newHadoopConf()).fs // scalastyle:on deltahadoopconfiguration val path = fs.makeQualified(rawPath) val deltaLogCache = getOrCreateCache(spark.sessionState.conf) if (spark.sessionState.conf.getConf( DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS)) { // We rely on the fact that accessing the key set doesn't modify the entry access time. See // `CacheBuilder.expireAfterAccess`. val keysToBeRemoved = mutable.ArrayBuffer[DeltaLogCacheKey]() val iter = deltaLogCache.asMap().keySet().iterator() while (iter.hasNext) { val key = iter.next() if (key.path == path) { keysToBeRemoved += key } } deltaLogCache.invalidateAll(keysToBeRemoved.asJava) } else { deltaLogCache.invalidate(DeltaLogCacheKey( path, fsOptions = Map.empty)) } } catch { case NonFatal(e) => logWarning(e.getMessage, e) } } def clearCache(): Unit = { deltaLogCache.foreach(_.invalidateAll()) } /** Unset the caches. Exposing for testing */ private[delta] def unsetCache(): Unit = { synchronized { deltaLogCache = None } } /** Return the number of cached `DeltaLog`s. Exposing for testing */ private[delta] def cacheSize: Long = { deltaLogCache.map(_.size()).getOrElse(0L) } /** * Filters the given [[Dataset]] by the given `partitionFilters`, returning those that match. * @param files The active files in the DeltaLog state, which contains the partition value * information * @param partitionFilters Filters on the partition columns * @param partitionColumnPrefixes The path to the `partitionValues` column, if it's nested * @param shouldRewritePartitionFilters Whether to rewrite `partitionFilters` to be over the * [[AddFile]] schema */ def filterFileList( partitionSchema: StructType, files: DataFrame, partitionFilters: Seq[Expression], partitionColumnPrefixes: Seq[String] = Nil, shouldRewritePartitionFilters: Boolean = true): DataFrame = { val rewrittenFilters = if (shouldRewritePartitionFilters) { rewritePartitionFilters( partitionSchema, files.sparkSession.sessionState.conf.resolver, partitionFilters, partitionColumnPrefixes) } else { partitionFilters } val expr = rewrittenFilters.reduceLeftOption(And).getOrElse(Literal.TrueLiteral) val columnFilter = Column(expr) files.filter(columnFilter) } /** * Rewrite the given `partitionFilters` to be used for filtering partition values. * We need to explicitly resolve the partitioning columns here because the partition columns * are stored as keys of a Map type instead of attributes in the AddFile schema (below) and thus * cannot be resolved automatically. * * @param partitionFilters Filters on the partition columns * @param partitionColumnPrefixes The path to the `partitionValues` column, if it's nested */ def rewritePartitionFilters( partitionSchema: StructType, resolver: Resolver, partitionFilters: Seq[Expression], partitionColumnPrefixes: Seq[String] = Nil): Seq[Expression] = { partitionFilters .map(_.transformUp { case a: Attribute => // If we have a special column name, e.g. `a.a`, then an UnresolvedAttribute returns // the column name as '`a.a`' instead of 'a.a', therefore we need to strip the backticks. val unquoted = a.name.stripPrefix("`").stripSuffix("`") val partitionCol = partitionSchema.find { field => resolver(field.name, unquoted) } partitionCol match { case Some(f: StructField) => val name = DeltaColumnMapping.getPhysicalName(f) Cast( UnresolvedAttribute(partitionColumnPrefixes ++ Seq("partitionValues", name)), f.dataType) case None => // This should not be able to happen, but the case was present in the original code so // we kept it to be safe. log.error(s"Partition filter referenced column ${a.name} not in the partition schema") UnresolvedAttribute(partitionColumnPrefixes ++ Seq("partitionValues", a.name)) } }) } /** * Checks whether this table only accepts appends. If so it will throw an error in operations that * can remove data such as DELETE/UPDATE/MERGE. */ def assertRemovable(snapshot: Snapshot): Unit = { val metadata = snapshot.metadata if (DeltaConfigs.IS_APPEND_ONLY.fromMetaData(metadata)) { throw DeltaErrors.modifyAppendOnlyTableException(metadata.name) } } /** How long to keep around SetTransaction actions before physically deleting them. */ def minSetTransactionRetentionInterval(metadata: Metadata): Option[Long] = { DeltaConfigs.TRANSACTION_ID_RETENTION_DURATION .fromMetaData(metadata) .map(DeltaConfigs.getMilliSeconds) } /** How long to keep around logically deleted files before physically deleting them. */ def tombstoneRetentionMillis(metadata: Metadata): Long = { DeltaConfigs.getMilliSeconds(DeltaConfigs.TOMBSTONE_RETENTION.fromMetaData(metadata)) } /** Get a function that canonicalizes a given `path`. */ private[delta] class CanonicalPathFunction(getHadoopConf: () => Configuration) extends Function[String, String] with Serializable { // Mark it `@transient lazy val` so that de-serialization happens only once on every executor. @transient private lazy val fs = { // scalastyle:off FileSystemGet FileSystem.get(getHadoopConf()) // scalastyle:on FileSystemGet } override def apply(path: String): String = { // scalastyle:off pathfromuri val hadoopPath = new Path(new URI(path)) // scalastyle:on pathfromuri if (hadoopPath.isAbsoluteAndSchemeAuthorityNull) { fs.makeQualified(hadoopPath).toUri.toString } else { // return untouched if it is a relative path or is already fully qualified hadoopPath.toUri.toString } } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaLogFileIndex.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.util.FileNames import org.apache.hadoop.fs._ import org.apache.spark.internal.{Logging, MDC} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.execution.datasources.{FileFormat, FileIndex, PartitionDirectory} import org.apache.spark.sql.execution.datasources.json.JsonFileFormat import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.types.{LongType, StructField, StructType} /** * A specialized file index for files found in the _delta_log directory. By using this file index, * we avoid any additional file listing, partitioning inference, and file existence checks when * computing the state of a Delta table. * * @param format The file format of the log files. Currently "parquet" or "json" * @param files The files to read */ case class DeltaLogFileIndex private ( format: FileFormat, files: Array[FileStatus]) extends FileIndex with Logging { import DeltaLogFileIndex._ override lazy val rootPaths: Seq[Path] = files.map(_.getPath) def listAllFiles(): Seq[PartitionDirectory] = { files .groupBy(f => FileNames.getFileVersionOpt(f.getPath).getOrElse(-1L)) .map { case (version, files) => PartitionDirectory(InternalRow(version), files) } .toSeq } override def listFiles( partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): Seq[PartitionDirectory] = { if (partitionFilters.isEmpty) { listAllFiles() } else { val predicate = partitionFilters.reduce(And) val boundPredicate = predicate.transform { case a: AttributeReference => val index = partitionSchema.indexWhere(a.name == _.name) BoundReference(index, partitionSchema(index).dataType, partitionSchema(index).nullable) } val predicateEvaluator = Predicate.create(boundPredicate, Nil) listAllFiles().filter(d => predicateEvaluator.eval(d.values)) } } override val inputFiles: Array[String] = files.map(_.getPath.toString) override def refresh(): Unit = {} override val sizeInBytes: Long = files.map(_.getLen).sum override val partitionSchema: StructType = new StructType().add(COMMIT_VERSION_COLUMN, LongType, nullable = false) override def toString: String = s"DeltaLogFileIndex($format, numFilesInSegment: ${files.size}, totalFileSize: $sizeInBytes)" logInfo(log"Created ${MDC(DeltaLogKeys.FILE_INDEX, this)}") } object DeltaLogFileIndex { val COMMIT_VERSION_COLUMN = "version" lazy val COMMIT_FILE_FORMAT = new JsonFileFormat lazy val CHECKPOINT_FILE_FORMAT_PARQUET = new ParquetFileFormat lazy val CHECKPOINT_FILE_FORMAT_JSON = new JsonFileFormat lazy val CHECKSUM_FILE_FORMAT = new JsonFileFormat def apply(format: FileFormat, fs: FileSystem, paths: Seq[Path]): DeltaLogFileIndex = { DeltaLogFileIndex(format, paths.map(fs.getFileStatus).toArray) } def apply(format: FileFormat, files: Seq[FileStatus]): Option[DeltaLogFileIndex] = { if (files.isEmpty) None else Some(DeltaLogFileIndex(format, files.toArray)) } def apply(format: FileFormat, filesOpt: Option[Seq[FileStatus]]): Option[DeltaLogFileIndex] = { filesOpt.flatMap(DeltaLogFileIndex(format, _)) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaMergeActionResolver.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.ResolveDeltaMergeInto.ResolveExpressionsFn import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.analysis._ import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.internal.SQLConf case class TargetTableResolutionResult( unresolvedAttribute: UnresolvedAttribute, expr: Expression ) /** Base trait with helpers for resolving DeltaMergeAction. */ trait DeltaMergeActionResolverBase { /** The SQL configuration for this query. */ def conf: SQLConf /** Function we want to use for resolving expressions. */ def resolveExprsFn: ResolveExpressionsFn /** The resolved target plan of the MERGE INTO statement. */ def target: LogicalPlan /** The resolved source plan of the MERGE INTO statement. */ def source: LogicalPlan /** Used for constructing error messages. */ private lazy val colsAsSQLText = target.output.map(_.sql).mkString(", ") /** Try to resolve a single target column in the Merge action. */ protected def resolveSingleTargetColumn( unresolvedAttribute: UnresolvedAttribute, mergeClauseTypeStr: String, shouldTryUnresolvedTargetExprOnSource: Boolean): Expression = { // Resolve the target column name without database/table/view qualifiers // If clause allows nested field to be target, then this will return all the // parts of the name (e.g., "a.b" -> Seq("a", "b")). Otherwise, this will // return only one string. try { ResolveDeltaMergeInto.resolveSingleExprOrFail( resolveExprsFn = resolveExprsFn, expr = unresolvedAttribute, plansToResolveExpr = Seq(target), mergeClauseTypeStr = mergeClauseTypeStr ) } catch { // Allow schema evolution for update and insert non-star when the column is not in // the target. case _: AnalysisException if shouldTryUnresolvedTargetExprOnSource => ResolveDeltaMergeInto.resolveSingleExprOrFail( resolveExprsFn = resolveExprsFn, expr = unresolvedAttribute, plansToResolveExpr = Seq(source), mergeClauseTypeStr = mergeClauseTypeStr ) } } /** * Takes the resolvedKey which refers to the target column in the relation and * the corresponding resolvedRHSExpr which describes the assignment value and return * a resolved DeltaMergeAction. */ protected def buildDeltaMergeAction( resolvedKey: Expression, resolvedRHSExpr: Expression, mergeClauseTypeStr: String): DeltaMergeAction = { lazy val sqlText = resolvedKey.sql lazy val resolutionErrorMsg = s"Cannot resolve $sqlText in target columns in $mergeClauseTypeStr given " + s"columns $colsAsSQLText" val resolvedNameParts = DeltaUpdateTable.getTargetColNameParts(resolvedKey, resolutionErrorMsg) DeltaMergeAction( targetColNameParts = resolvedNameParts, expr = resolvedRHSExpr, // Explicit column assignments overwrite target-only struct fields with null. targetOnlyStructFieldBehavior = TargetOnlyStructFieldBehavior.NULLIFY, targetColNameResolved = true) } /** * Takes a sequence of DeltaMergeActions and returns the * corresponding resolved DeltaMergeActions. */ def resolve( clauseType: String, plansToResolveAction: Seq[LogicalPlan], shouldTryUnresolvedTargetExprOnSource: Boolean, deltaMergeActions: Seq[DeltaMergeAction]): Seq[DeltaMergeAction] } class IndividualDeltaMergeActionResolver( override val target: LogicalPlan, override val source: LogicalPlan, override val conf: SQLConf, override val resolveExprsFn: ResolveExpressionsFn ) extends DeltaMergeActionResolverBase { /** Resolve DeltaMergeAction, one at a time. */ override def resolve( mergeClauseTypeStr: String, plansToResolveAction: Seq[LogicalPlan], shouldTryUnresolvedTargetExprOnSource: Boolean, deltaMergeActions: Seq[DeltaMergeAction]): Seq[DeltaMergeAction] = { deltaMergeActions.map { case d: DeltaMergeAction if !d.resolved => val unresolvedAttrib = UnresolvedAttribute(d.targetColNameParts) val resolvedKey = resolveSingleTargetColumn( unresolvedAttrib, mergeClauseTypeStr, shouldTryUnresolvedTargetExprOnSource) val resolvedExpr = resolveExprsFn(Seq(d.expr), plansToResolveAction).head ResolveDeltaMergeInto.throwIfNotResolved( resolvedExpr, plansToResolveAction, mergeClauseTypeStr) buildDeltaMergeAction(resolvedKey, resolvedExpr, mergeClauseTypeStr) // Already resolved case d => d } } } class BatchedDeltaMergeActionResolver( override val target: LogicalPlan, override val source: LogicalPlan, override val conf: SQLConf, override val resolveExprsFn: ResolveExpressionsFn ) extends DeltaMergeActionResolverBase { /** * Attempt to batch resolve the target columns reference all at once. If we are * unable to resolve against the target plan, we retry against the source plan * if schema evolution is enabled and it's appropriate for the clause type. * * @return The resolved expressions for the target columns. The sequence of * expressions is ordered the same as the unresolved attributes * sequence passed in. */ private def batchResolveTargetColumns( unresolvedAttrSeq: Seq[UnresolvedAttribute], shouldTryUnresolvedTargetExprOnSource: Boolean, mergeClauseTypeStr: String): Seq[Expression] = { val resolvedExprs = try { // Unlike [[resolveSingleTargetColumn]], this is not a [[resolveOrFail]]. // We will not throw an exception if something was not resolved, because we // want to resolve as much as possible and only retry to resolve against the // source the few columns that failed to resolve. But we must wrap this in a // try-catch to swallow exception that come from other parts of invoking the // analyzer. We need this to preserve the behaviour where we throw a different // exception in PreprocessTableMerge later on... resolveExprsFn(unresolvedAttrSeq, Seq(target)) } catch { // We don't know which attribute in the Seq lead to this exception. // We need to resolve this one by one, so we can return early here. case _: AnalysisException if shouldTryUnresolvedTargetExprOnSource => return unresolvedAttrSeq.map( resolveSingleTargetColumn(_, mergeClauseTypeStr, shouldTryUnresolvedTargetExprOnSource)) } assert(unresolvedAttrSeq.length == resolvedExprs.length, "The number of " + "resolved expressions should match the number of unresolved expressions") val targetTableResolutionResult: Seq[TargetTableResolutionResult] = unresolvedAttrSeq.zip(resolvedExprs).map { case (unresolvedAttr, expr) => TargetTableResolutionResult(unresolvedAttr, expr) } val remainingUnresolvedExprs: Seq[Expression] = targetTableResolutionResult.filterNot(_.expr.resolved).map(_.unresolvedAttribute) val orderedResolvedTargetExprs = if (remainingUnresolvedExprs.isEmpty) { // Everything was resolved, we can return the resolved expressions. resolvedExprs } else { // We were not able to resolve all the target columns against the target plan. // If we are not supposed to resolve the target column against the source and // we were not able to resolve the column, then we should throw an exception // at this point. if (!shouldTryUnresolvedTargetExprOnSource) { ResolveDeltaMergeInto.throwIfNotResolved( // Use the first of the unresolved attributes to throw the exception. targetTableResolutionResult.find(!_.expr.resolved).map(_.expr).get, Seq(target), mergeClauseTypeStr ) } // Try to resolve against the source, will throw an exception if it can't. val resolvedExprAgainstSource: Seq[Expression] = ResolveDeltaMergeInto.resolveOrFail( resolveExprsFn = resolveExprsFn, exprs = remainingUnresolvedExprs, plansToResolveExprs = Seq(source), mergeClauseTypeStr = mergeClauseTypeStr ) // Put the expressions that we resolved using the source back into the resolution result // in the correct locations. The order needs to be preserved so that we can match it with // the corresponding resolved assignment expressions. var index = -1 targetTableResolutionResult.map { case TargetTableResolutionResult(_, expr) => if (expr.resolved) { expr } else { index += 1 resolvedExprAgainstSource(index) } } } orderedResolvedTargetExprs } /** * Batch the resolution of the target column name parts against the target relation * and the resolution of assignment expression together. * * Fundamental requirement: Column/expression ordering must be preserved * by [[resolveExprsFn]]. */ override def resolve( mergeClauseTypeStr: String, plansToResolveAction: Seq[LogicalPlan], shouldTryUnresolvedTargetExprOnSource: Boolean, deltaMergeActions: Seq[DeltaMergeAction]): Seq[DeltaMergeAction] = { val (alreadyResolvedDeltaMergeActions, unresolvedDeltaMergeActions) = deltaMergeActions.partition(_.resolved) // Batch the unresolved attributes to resolve them in a single pass. val unresolvedAttrSeq = unresolvedDeltaMergeActions .map(mergeAction => UnresolvedAttribute(mergeAction.targetColNameParts)) val orderedResolvedTargetExprs = batchResolveTargetColumns( unresolvedAttrSeq, shouldTryUnresolvedTargetExprOnSource, mergeClauseTypeStr) // Now we deal with the expressions for each target column (RHS assignment). val unresolvedRHSExprSeq = unresolvedDeltaMergeActions.map(_.expr) val resolvedExprsSeq = resolveExprsFn(unresolvedRHSExprSeq, plansToResolveAction) assert(resolvedExprsSeq.length == orderedResolvedTargetExprs.length) resolvedExprsSeq.foreach( ResolveDeltaMergeInto.throwIfNotResolved(_, plansToResolveAction, mergeClauseTypeStr)) // Combine the resolved target columns and the resolved expressions to create // the final resolved DeltaMergeAction val resolvedDeltaMergeActions: Seq[DeltaMergeAction] = orderedResolvedTargetExprs.zip(resolvedExprsSeq).map { case (resolvedKey, resolvedExpr) => buildDeltaMergeAction(resolvedKey, resolvedExpr, mergeClauseTypeStr) } // The order for this Seq doesn't matter. alreadyResolvedDeltaMergeActions ++ resolvedDeltaMergeActions } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaOperations.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.DeltaOperationMetrics.MetricsTransformer import org.apache.spark.sql.delta.actions.{Metadata, Protocol} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.sql.{SaveMode, SparkSession} import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.plans.logical.DeltaMergeIntoClause import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.streaming.OutputMode import org.apache.spark.sql.types.{StructField, StructType} /** * Exhaustive list of operations that can be performed on a Delta table. These operations are * tracked as the first line in delta logs, and power `DESCRIBE HISTORY` for Delta tables. */ object DeltaOperations { /** * An operation that can be performed on a Delta table. * @param name The name of the operation. */ sealed abstract class Operation(val name: String) { def parameters: Map[String, Any] lazy val jsonEncodedValues: Map[String, String] = parameters.mapValues(JsonUtils.toJson(_)).toMap val operationMetrics: Set[String] = Set() def transformMetrics(metrics: Map[String, SQLMetric]): Map[String, String] = { metrics.filterKeys( s => operationMetrics.contains(s) ).mapValues(_.value.toString).toMap } val userMetadata: Option[String] = None /** Whether this operation changes data */ def changesData: Boolean = false /** * Manually transform the deletion vector metrics, because they are not part of * `operationMetrics` and are filtered out by the super.transformMetrics() call. */ def transformDeletionVectorMetrics( allMetrics: Map[String, SQLMetric], dvMetrics: Map[String, MetricsTransformer] = DeltaOperationMetrics.DELETION_VECTORS) : Map[String, String] = { dvMetrics.flatMap { case (metric, transformer) => transformer.transformToString(metric, allMetrics) } } /** * A transaction that commits AddFile actions with deletionVector should have column stats that * are not tight bounds. An exception to this is ComputeStats operation, which recomputes stats * on these files, and the new stats are tight bounds. Some other operations that merely take an * existing AddFile action and commit a copy of it, not changing the deletionVector or stats, * can then also recommit AddFile with deletionVector and tight bound stats that were recomputed * before. * * An operation for which this can happen, and there is no way that it could be committing * new deletion vectors, should set this to false to bypass this check. * All other operations should set this to true, so that this is validated during commit. * * This is abstract to force the implementers of all operations to think about this setting. * All operations should add a comment justifying this setting. * Any operation that sets this to false should add a test in TightBoundsSuite. */ def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean /** * Whether the transaction is updating metadata of existing files. * * The Delta protocol allows committing AddFile actions for files that already exist on the * latest version of the table, without committing corresponding RemoveFile actions. This is * used to update the metadata of existing files, e.g. to recompute statistics or add tags. * * Such operations need special handling during conflict checking, especially against * no-data-change transactions, because the read/delete conflict can be resolved with * read-file-remapping and because there is no RemoveFile action to trigger a delete/delete * conflict. In case you are adding such operation, make sure to include a test for conflicts * with business *and* no-data-change transactions, e.g. optimize. */ def isInPlaceFileMetadataUpdate: Option[Boolean] /** * Whether this operation is allowed to change the set and order of partition columns. * Operations creating tables may always change the partitioning, so it's considered supported * implicitly and checked in OptimisticTransaction. It is ignored what is returned here for * operations that create a new table. Operations can return false in that case. Operations * that replace tables or insert may return true depending on their mode and parameters. * Most other operations should return false. */ def canChangePartitionColumns: Boolean } abstract class OperationWithPredicates(name: String, val predicates: Seq[Expression]) extends Operation(name) { private val predicateString = JsonUtils.toJson(predicatesToString(predicates)) override def parameters: Map[String, Any] = Map("predicate" -> predicateString) } /** Recorded during batch inserts. Predicates can be provided for overwrites. */ val OP_WRITE = "WRITE" case class Write( mode: SaveMode, partitionBy: Option[Seq[String]] = None, predicate: Option[String] = None, override val userMetadata: Option[String] = None, isDynamicPartitionOverwrite: Option[Boolean] = None, canOverwriteSchema: Option[Boolean] = None, canMergeSchema: Option[Boolean] = None ) extends Operation(OP_WRITE) { override val parameters: Map[String, Any] = Map("mode" -> mode.name() ) ++ partitionBy.map("partitionBy" -> JsonUtils.toJson(_)) ++ // Only log these fields when explicitly set to avoid noise in DESCRIBE HISTORY when users do // not set them. This means we don't distinguish between explicitly disabled (false) and unset // (defaults to disabled), but that's fine as the distinction is not particularly interesting. predicate.map("predicate" -> _) ++ isDynamicPartitionOverwrite.map("isDynamicPartitionOverwrite" -> _) ++ canOverwriteSchema.map("canOverwriteSchema" -> _) ++ canMergeSchema.map("canMergeSchema" -> _) val replaceWhereMetricsEnabled = SparkSession.active.conf.get( DeltaSQLConf.REPLACEWHERE_METRICS_ENABLED) val insertOverwriteRemoveMetricsEnabled = SparkSession.active.conf.get( DeltaSQLConf.OVERWRITE_REMOVE_METRICS_ENABLED) override def transformMetrics(metrics: Map[String, SQLMetric]): Map[String, String] = { // Need special handling for replaceWhere as it is implemented as a Write + Delete. if (predicate.nonEmpty && replaceWhereMetricsEnabled) { var strMetrics = super.transformMetrics(metrics) // find the case where deletedRows are not captured if (strMetrics.get("numDeletedRows").exists(_ == "0") && strMetrics.get("numRemovedFiles").exists(_ != "0")) { // identify when row level metrics are unavailable. This will happen when the entire // table or partition are deleted. strMetrics -= "numDeletedRows" strMetrics -= "numCopiedRows" strMetrics -= "numAddedFiles" } // in the case when stats are not collected we need to remove all row based metrics // If the DF provided to replaceWhere is an empty DataFrame and we don't have stats // we won't return row level metrics. if (strMetrics.get("numOutputRows").exists(_ == "0") && strMetrics.get("numFiles").exists(_ != 0)) { strMetrics -= "numDeletedRows" strMetrics -= "numOutputRows" strMetrics -= "numCopiedRows" } strMetrics } else { super.transformMetrics(metrics) } } override val operationMetrics: Set[String] = if (predicate.isEmpty || !replaceWhereMetricsEnabled) { // Remove metrics are included to replaceWhere metrics // so they need to be added only when replaceWhere metrics are not presented val overwriteMetrics = if (mode == SaveMode.Overwrite && insertOverwriteRemoveMetricsEnabled) { DeltaOperationMetrics.OVERWRITE_REMOVES } else { Set.empty } DeltaOperationMetrics.WRITE ++ overwriteMetrics } else { // Need special handling for replaceWhere as rows/files are deleted as well. DeltaOperationMetrics.WRITE_REPLACE_WHERE } override def changesData: Boolean = true // This operation shouldn't be introducing AddFile actions with DVs and tight bounds stats. // DVs can be introduced by the replaceWhere operation. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = { // We don't have to return true if it is a new table, only on overwrite. mode == SaveMode.Overwrite && canOverwriteSchema.getOrElse(false) } } case class RemoveColumnMapping( override val userMetadata: Option[String] = None) extends Operation("REMOVE COLUMN MAPPING") { override def parameters: Map[String, Any] = Map() override val operationMetrics: Set[String] = DeltaOperationMetrics.REMOVE_COLUMN_MAPPING // This operation shouldn't be introducing AddFile actions at all. This check should be trivial. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } /** Recorded during streaming inserts. */ case class StreamingUpdate( outputMode: OutputMode, queryId: String, epochId: Long, override val userMetadata: Option[String] = None ) extends Operation("STREAMING UPDATE") { override val parameters: Map[String, Any] = Map("outputMode" -> outputMode.toString, "queryId" -> queryId, "epochId" -> epochId.toString ) override val operationMetrics: Set[String] = DeltaOperationMetrics.STREAMING_UPDATE override def changesData: Boolean = true // This operation shouldn't be introducing AddFile actions with DVs and tight bounds stats. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } /** Recorded while deleting certain partitions. */ val OP_DELETE = "DELETE" case class Delete(predicate: Seq[Expression]) extends OperationWithPredicates(OP_DELETE, predicate) { override val operationMetrics: Set[String] = DeltaOperationMetrics.DELETE override def transformMetrics(metrics: Map[String, SQLMetric]): Map[String, String] = { var strMetrics = super.transformMetrics(metrics) // find the case where deletedRows are not captured if (strMetrics("numDeletedRows") == "0" && strMetrics("numRemovedFiles") != "0") { // identify when row level metrics are unavailable. This will happen when the entire // table or partition are deleted. strMetrics -= "numDeletedRows" strMetrics -= "numCopiedRows" strMetrics -= "numAddedFiles" } val dvMetrics = transformDeletionVectorMetrics(metrics) strMetrics ++ dvMetrics } override def changesData: Boolean = true // This operation shouldn't be introducing AddFile actions with DVs and tight bounds stats. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } /** Recorded when truncating the table. */ case class Truncate() extends Operation("TRUNCATE") { override val parameters: Map[String, Any] = Map.empty override val operationMetrics: Set[String] = DeltaOperationMetrics.TRUNCATE override def changesData: Boolean = true // This operation shouldn't be introducing AddFile actions at all. This check should be trivial. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } /** Recorded when converting a table into a Delta table. */ case class Convert( numFiles: Long, partitionBy: Seq[String], collectStats: Boolean, catalogTable: Option[String], sourceFormat: Option[String]) extends Operation("CONVERT") { override val parameters: Map[String, Any] = Map( "numFiles" -> numFiles, "partitionedBy" -> JsonUtils.toJson(partitionBy), "collectStats" -> collectStats) ++ catalogTable.map("catalogTable" -> _) ++ sourceFormat.map("sourceFormat" -> _) override val operationMetrics: Set[String] = DeltaOperationMetrics.CONVERT override def changesData: Boolean = true // This operation shouldn't be introducing AddFile actions with DVs and non-tight bounds stats. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } /** Represents the predicates and action type (insert, update, delete) for a Merge clause */ case class MergePredicate( predicate: Option[String], actionType: String) object MergePredicate { def apply(mergeClause: DeltaMergeIntoClause): MergePredicate = { MergePredicate( predicate = mergeClause.condition.map(_.simpleString(SQLConf.get.maxToStringFields)), mergeClause.clauseType.toLowerCase()) } } /** * Recorded when a merge operation is committed to the table. * * `updatePredicate`, `deletePredicate`, and `insertPredicate` are DEPRECATED. * Only use `predicate`, `matchedPredicates`, `notMatchedPredicates` and * `notMatchedBySourcePredicates` to record the merge. */ val OP_MERGE = "MERGE" case class Merge( predicate: Option[Expression], updatePredicate: Option[String], deletePredicate: Option[String], insertPredicate: Option[String], matchedPredicates: Seq[MergePredicate], notMatchedPredicates: Seq[MergePredicate], notMatchedBySourcePredicates: Seq[MergePredicate] ) extends OperationWithPredicates(OP_MERGE, predicate.toSeq) { override val parameters: Map[String, Any] = { super.parameters ++ updatePredicate.map("updatePredicate" -> _).toMap ++ deletePredicate.map("deletePredicate" -> _).toMap ++ insertPredicate.map("insertPredicate" -> _).toMap + ("matchedPredicates" -> JsonUtils.toJson(matchedPredicates)) + ("notMatchedPredicates" -> JsonUtils.toJson(notMatchedPredicates)) + ("notMatchedBySourcePredicates" -> JsonUtils.toJson(notMatchedBySourcePredicates)) } override val operationMetrics: Set[String] = DeltaOperationMetrics.MERGE override def transformMetrics(metrics: Map[String, SQLMetric]): Map[String, String] = { var strMetrics = super.transformMetrics(metrics) strMetrics += "numSourceRows" -> metrics("operationNumSourceRows").value.toString // We have to recalculate "numOutputRows" to avoid counting CDC rows if (metrics.contains("numTargetRowsInserted") && metrics.contains("numTargetRowsUpdated") && metrics.contains("numTargetRowsCopied")) { val actualNumOutputRows = metrics("numTargetRowsInserted").value + metrics("numTargetRowsUpdated").value + metrics("numTargetRowsCopied").value strMetrics += "numOutputRows" -> actualNumOutputRows.toString } val dvMetrics = transformDeletionVectorMetrics( metrics, dvMetrics = DeltaOperationMetrics.MERGE_DELETION_VECTORS) strMetrics ++= dvMetrics strMetrics } override def changesData: Boolean = true // This operation shouldn't be introducing AddFile actions with DVs and non-tight bounds stats. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } object Merge { /** constructor to provide default values for deprecated fields */ def apply( predicate: Option[Expression], matchedPredicates: Seq[MergePredicate], notMatchedPredicates: Seq[MergePredicate], notMatchedBySourcePredicates: Seq[MergePredicate] ): Merge = Merge( predicate, updatePredicate = None, deletePredicate = None, insertPredicate = None, matchedPredicates, notMatchedPredicates, notMatchedBySourcePredicates ) } /** Recorded when an update operation is committed to the table. */ val OP_UPDATE = "UPDATE" case class Update(predicate: Option[Expression]) extends OperationWithPredicates(OP_UPDATE, predicate.toSeq) { override val operationMetrics: Set[String] = DeltaOperationMetrics.UPDATE override def changesData: Boolean = true override def transformMetrics(metrics: Map[String, SQLMetric]): Map[String, String] = { val dvMetrics = transformDeletionVectorMetrics(metrics) super.transformMetrics(metrics) ++ dvMetrics } // This operation shouldn't be introducing AddFile actions with DVs and non-tight bounds stats. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } /** Recorded when the table is created. */ case class CreateTable( metadata: Metadata, isManaged: Boolean, asSelect: Boolean = false, clusterBy: Option[Seq[String]] = None ) extends Operation("CREATE TABLE" + s"${if (asSelect) " AS SELECT" else ""}") { override val parameters: Map[String, Any] = Map( "isManaged" -> isManaged.toString, "description" -> Option(metadata.description), "partitionBy" -> JsonUtils.toJson(metadata.partitionColumns), CLUSTERING_PARAMETER_KEY -> JsonUtils.toJson(clusterBy.getOrElse(Seq.empty)), "properties" -> JsonUtils.toJson(metadata.configuration) ) override val operationMetrics: Set[String] = if (!asSelect) { Set() } else { DeltaOperationMetrics.WRITE } override def changesData: Boolean = asSelect // This operation shouldn't be introducing AddFile actions with DVs and non-tight bounds stats. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = true } /** Recorded when the table is replaced. */ case class ReplaceTable( metadata: Metadata, isManaged: Boolean, orCreate: Boolean, asSelect: Boolean = false, override val userMetadata: Option[String] = None, clusterBy: Option[Seq[String]] = None, predicate: Option[String] = None, isDynamicPartitionOverwrite: Option[Boolean] = None, canOverwriteSchema: Option[Boolean] = None, canMergeSchema: Option[Boolean] = None, isV1SaveAsTableOverwrite: Option[Boolean] = None ) extends Operation(s"${if (orCreate) "CREATE OR " else ""}REPLACE TABLE" + s"${if (asSelect) " AS SELECT" else ""}") { override val parameters: Map[String, Any] = Map( "isManaged" -> isManaged.toString, "description" -> Option(metadata.description), "partitionBy" -> JsonUtils.toJson(metadata.partitionColumns), CLUSTERING_PARAMETER_KEY -> JsonUtils.toJson(clusterBy.getOrElse(Seq.empty)), "properties" -> JsonUtils.toJson(metadata.configuration) ) ++ // Only log these fields when explicitly set to avoid noise in DESCRIBE HISTORY when users do // not set them. This means we don't distinguish between explicitly disabled (false) and unset // (defaults to disabled), but that's fine as the distinction is not particularly interesting. predicate.map("predicate" -> _) ++ isDynamicPartitionOverwrite.map("isDynamicPartitionOverwrite" -> _) ++ canOverwriteSchema.map("canOverwriteSchema" -> _) ++ canMergeSchema.map("canMergeSchema" -> _) ++ isV1SaveAsTableOverwrite.map("isV1SaveAsTableOverwrite" -> _) private val insertOverwriteRemoveMetricsEnabled = SparkSession.active.conf.get( DeltaSQLConf.OVERWRITE_REMOVE_METRICS_ENABLED) override val operationMetrics: Set[String] = if (!asSelect) { Set() } else { val overwriteMetrics = if (insertOverwriteRemoveMetricsEnabled) DeltaOperationMetrics.OVERWRITE_REMOVES else Set.empty DeltaOperationMetrics.WRITE ++ overwriteMetrics } override def changesData: Boolean = true // This operation shouldn't be introducing AddFile actions with DVs and non-tight bounds stats. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) // We allow ReplaceTable operations to change partition columns when they are // 1) creating/replacing a new table, 2) not invoked via saveAsTable or 3) invoked via // saveAsTable but with schema overwrite. override def canChangePartitionColumns: Boolean = !isV1SaveAsTableOverwrite.getOrElse(false) || (isV1SaveAsTableOverwrite.getOrElse(false) && canOverwriteSchema.getOrElse(false)) } /** Recorded when the table properties are set. */ val OP_SET_TBLPROPERTIES = "SET TBLPROPERTIES" case class SetTableProperties( properties: Map[String, String]) extends Operation(OP_SET_TBLPROPERTIES) { override val parameters: Map[String, Any] = Map("properties" -> JsonUtils.toJson(properties)) // This operation shouldn't be introducing AddFile actions at all. This check should be trivial. // Note: This operation may trigger additional actions and additional commits. For example // RowTrackingBackfill. These are separate transactions, and this check is performed separately. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } /** Recorded when the table properties are unset. */ case class UnsetTableProperties( propKeys: Seq[String], ifExists: Boolean) extends Operation("UNSET TBLPROPERTIES") { override val parameters: Map[String, Any] = Map( "properties" -> JsonUtils.toJson(propKeys), "ifExists" -> ifExists) // This operation shouldn't be introducing AddFile actions at all. This check should be trivial. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } /** Recorded when dropping a table feature. */ val OP_DROP_FEATURE = "DROP FEATURE" case class DropTableFeature( featureName: String, truncateHistory: Boolean) extends Operation(OP_DROP_FEATURE) { override val parameters: Map[String, Any] = Map( "featureName" -> featureName, "truncateHistory" -> truncateHistory) // This operation shouldn't be introducing AddFile actions at all. This check should be trivial. // Note: this operation may trigger additional actions and additional commits. These would be // separate transactions, and this check is performed separately. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(true) override def canChangePartitionColumns: Boolean = false } /** * Recorded when dropping deletion vectors. Deletion Vector tombstones directly reference * deletion vector files within the retention period. This is to protect them from deletion * against oblivious writers when vacuuming. */ object AddDeletionVectorsTombstones extends Operation("Deletion Vector Tombstones") { override val parameters: Map[String, Any] = Map.empty // This operation should only introduce RemoveFile actions. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } /** Recorded when columns are added. */ case class AddColumns( colsToAdd: Seq[QualifiedColTypeWithPositionForLog]) extends Operation("ADD COLUMNS") { override val parameters: Map[String, Any] = Map( "columns" -> JsonUtils.toJson(colsToAdd.map { case QualifiedColTypeWithPositionForLog(columnPath, column, colPosition) => Map( "column" -> structFieldToMap(columnPath, column) ) ++ colPosition.map("position" -> _.toString) })) // This operation shouldn't be introducing AddFile actions at all. This check should be trivial. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } /** Recorded when columns are dropped. */ val OP_DROP_COLUMN = "DROP COLUMNS" case class DropColumns( colsToDrop: Seq[Seq[String]]) extends Operation(OP_DROP_COLUMN) { override val parameters: Map[String, Any] = Map( "columns" -> JsonUtils.toJson(colsToDrop.map(UnresolvedAttribute(_).name))) // This operation shouldn't be introducing AddFile actions at all. This check should be trivial. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } /** Recorded when column is renamed */ val OP_RENAME_COLUMN = "RENAME COLUMN" case class RenameColumn(oldColumnPath: Seq[String], newColumnPath: Seq[String]) extends Operation(OP_RENAME_COLUMN) { override val parameters: Map[String, Any] = Map( "oldColumnPath" -> UnresolvedAttribute(oldColumnPath).name, "newColumnPath" -> UnresolvedAttribute(newColumnPath).name ) // This operation shouldn't be introducing AddFile actions at all. This check should be trivial. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = true } /** Recorded when columns are changed. */ case class ChangeColumn( columnPath: Seq[String], columnName: String, newColumn: StructField, colPosition: Option[String]) extends Operation("CHANGE COLUMN") { override val parameters: Map[String, Any] = Map( "column" -> JsonUtils.toJson(structFieldToMap(columnPath, newColumn)) ) ++ colPosition.map("position" -> _) // This operation shouldn't be introducing AddFile actions at all. This check should be trivial. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } /** Recorded when columns are changed in bulk. */ case class ChangeColumns(columns: Seq[ChangeColumn]) extends Operation("CHANGE COLUMNS") { override val parameters: Map[String, Any] = Map( "columns" -> JsonUtils.toJson( columns.map(col => structFieldToMap(col.columnPath, col.newColumn) ++ col.colPosition.map("position" -> _)) ) ) // This operation shouldn't be introducing AddFile actions at all. This check should be trivial. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } /** Recorded when columns are replaced. */ case class ReplaceColumns( columns: Seq[StructField]) extends Operation("REPLACE COLUMNS") { override val parameters: Map[String, Any] = Map( "columns" -> JsonUtils.toJson(columns.map(structFieldToMap(Seq.empty, _)))) // This operation shouldn't be introducing AddFile actions at all. This check should be trivial. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } case class UpgradeProtocol(newProtocol: Protocol) extends Operation("UPGRADE PROTOCOL") { override val parameters: Map[String, Any] = Map("newProtocol" -> JsonUtils.toJson(Map( "minReaderVersion" -> newProtocol.minReaderVersion, "minWriterVersion" -> newProtocol.minWriterVersion, "readerFeatures" -> newProtocol.readerFeatures, "writerFeatures" -> newProtocol.writerFeatures ))) // This operation shouldn't be introducing AddFile actions at all. This check should be trivial. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } object ManualUpdate extends Operation("Manual Update") { override val parameters: Map[String, Any] = Map.empty // Unsafe manual update disables checks. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = false // Manual update operations can commit arbitrary actions. In case this field is needed consider // adding a new Delta operation. For test-only code use TestOperation. override val isInPlaceFileMetadataUpdate: Option[Boolean] = None override def canChangePartitionColumns: Boolean = true } /** A commit without any actions. Could be used to force creation of new checkpoints. */ object EmptyCommit extends Operation("Empty Commit") { override val parameters: Map[String, Any] = Map.empty // This operation shouldn't be introducing AddFile actions at all. This check should be trivial. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } case class UpdateColumnMetadata( operationName: String, columns: Seq[(Seq[String], StructField)]) extends Operation(operationName) { override val parameters: Map[String, Any] = { Map("columns" -> JsonUtils.toJson(columns.map { case (path, field) => structFieldToMap(path, field) })) } // This operation shouldn't be introducing AddFile actions at all. This check should be trivial. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } case class UpdateSchema(oldSchema: StructType, newSchema: StructType) extends Operation("UPDATE SCHEMA") { override val parameters: Map[String, Any] = Map( "oldSchema" -> JsonUtils.toJson(oldSchema), "newSchema" -> JsonUtils.toJson(newSchema)) // This operation shouldn't be introducing AddFile actions at all. This check should be trivial. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } case class AddConstraint( constraintName: String, expr: String) extends Operation("ADD CONSTRAINT") { override val parameters: Map[String, Any] = Map("name" -> constraintName, "expr" -> expr) // This operation shouldn't be introducing AddFile actions at all. This check should be trivial. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } case class DropConstraint( constraintName: String, expr: Option[String]) extends Operation("DROP CONSTRAINT") { override val parameters: Map[String, Any] = { expr.map { e => Map("name" -> constraintName, "expr" -> e, "existed" -> "true") }.getOrElse { Map("name" -> constraintName, "existed" -> "false") } } // This operation shouldn't be introducing AddFile actions at all. This check should be trivial. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } /** Recorded when recomputing stats on the table. */ case class ComputeStats(predicate: Seq[Expression]) extends OperationWithPredicates("COMPUTE STATS", predicate) { // ComputeStats operation commits AddFiles with recomputed stats which are always tight bounds, // even when DVs are present. This check should be disabled. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = false // ComputeStats operation only updates statistics of existing files. override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(true) override def canChangePartitionColumns: Boolean = false } /** Recorded when restoring a Delta table to an older version. */ val OP_RESTORE = "RESTORE" case class Restore( version: Option[Long], timestamp: Option[String]) extends Operation(OP_RESTORE) { override val parameters: Map[String, Any] = Map( "version" -> version, "timestamp" -> timestamp) override def changesData: Boolean = true override val operationMetrics: Set[String] = DeltaOperationMetrics.RESTORE // Restore operation commits AddFiles with files, DVs and stats from the version it restores to. // It can happen that tight bound stats were recomputed before by ComputeStats. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = false // The restore operation could perform in-place file metadata updates. However, the difference // between the current and the restored state is computed using only the (path, DV) pairs as // identifiers, meaning that metadata differences are ignored. override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } sealed abstract class OptimizeOrReorg(override val name: String, predicates: Seq[Expression]) extends OperationWithPredicates(name, predicates) /** operation name for ROW TRACKING BACKFILL command */ val ROW_TRACKING_BACKFILL_OPERATION_NAME = "ROW TRACKING BACKFILL" val ROW_TRACKING_UNBACKFILL_OPERATION_NAME = "ROW TRACKING UNBACKFILL" /** parameter key to indicate whether it's an Auto Compaction */ val AUTO_COMPACTION_PARAMETER_KEY = "auto" /** operation name for REORG command */ val REORG_OPERATION_NAME = "REORG" /** operation name for OPTIMIZE command */ val OPTIMIZE_OPERATION_NAME = "OPTIMIZE" /** parameter key to indicate which columns to z-order by */ val ZORDER_PARAMETER_KEY = "zOrderBy" /** parameter key to indicate clustering columns */ val CLUSTERING_PARAMETER_KEY = "clusterBy" /** parameter key to indicate the operation for `OPTIMIZE tbl FULL` */ val CLUSTERING_IS_FULL_KEY = "isFull" /** Recorded when optimizing the table. */ case class Optimize( predicate: Seq[Expression], zOrderBy: Seq[String] = Seq.empty, auto: Boolean = false, clusterBy: Option[Seq[String]] = None, isFull: Boolean = false ) extends OptimizeOrReorg(OPTIMIZE_OPERATION_NAME, predicate) { override val parameters: Map[String, Any] = super.parameters ++ Map( // When clustering columns are specified, set the zOrderBy key to empty. ZORDER_PARAMETER_KEY -> JsonUtils.toJson(if (clusterBy.isEmpty) zOrderBy else Seq.empty), CLUSTERING_PARAMETER_KEY -> JsonUtils.toJson(clusterBy.getOrElse(Seq.empty)), AUTO_COMPACTION_PARAMETER_KEY -> auto ) // `isFull` is not relevant for non-clustering tables, so skip it. .++(clusterBy.filter(_.nonEmpty).map(_ => CLUSTERING_IS_FULL_KEY -> isFull)) override val operationMetrics: Set[String] = DeltaOperationMetrics.OPTIMIZE // This operation shouldn't be introducing AddFile actions with DVs and tight bounds stats. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } /** Recorded when cloning a Delta table into a new location. */ val OP_CLONE = "CLONE" case class Clone( source: String, sourceVersion: Long ) extends Operation(OP_CLONE) { override val parameters: Map[String, Any] = Map( "source" -> source, "sourceVersion" -> sourceVersion ) override def changesData: Boolean = true override val operationMetrics: Set[String] = DeltaOperationMetrics.CLONE // Clone operation commits AddFiles with files, DVs and stats copied over from the source table. // It can happen that tight bound stats were recomputed before by ComputeStats. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = false override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = true } /** * @param retentionCheckEnabled - whether retention check was enabled for this run of vacuum. * @param specifiedRetentionMillis - specified retention interval * @param defaultRetentionMillis - default retention period for the table */ case class VacuumStart( retentionCheckEnabled: Boolean, specifiedRetentionMillis: Option[Long], defaultRetentionMillis: Long) extends Operation(VacuumStart.OPERATION_NAME) { override val parameters: Map[String, Any] = Map( "retentionCheckEnabled" -> retentionCheckEnabled, "defaultRetentionMillis" -> defaultRetentionMillis ) ++ specifiedRetentionMillis.map("specifiedRetentionMillis" -> _) override val operationMetrics: Set[String] = DeltaOperationMetrics.VACUUM_START // This operation shouldn't be introducing AddFile actions at all. This check should be trivial. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } object VacuumStart { val OPERATION_NAME = "VACUUM START" } /** * @param status - whether the vacuum operation was successful; either "COMPLETED" or "FAILED" */ case class VacuumEnd(status: String) extends Operation(VacuumEnd.OPERATION_NAME) { override val parameters: Map[String, Any] = Map( "status" -> status ) override val operationMetrics: Set[String] = DeltaOperationMetrics.VACUUM_END // This operation shouldn't be introducing AddFile actions at all. This check should be trivial. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } object VacuumEnd { val OPERATION_NAME = "VACUUM END" } /** Recorded when running REORG on the table. */ case class Reorg( predicate: Seq[Expression], applyPurge: Boolean = true) extends OptimizeOrReorg(REORG_OPERATION_NAME, predicate) { override val parameters: Map[String, Any] = super.parameters ++ Map( "applyPurge" -> applyPurge ) override val operationMetrics: Set[String] = DeltaOperationMetrics.OPTIMIZE // This operation shouldn't be introducing AddFile actions with DVs and tight bounds stats. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } /** Recorded when clustering columns are changed on clustered tables. */ case class ClusterBy( oldClusteringColumns: String, newClusteringColumns: String) extends Operation("CLUSTER BY") { override val parameters: Map[String, Any] = Map( "oldClusteringColumns" -> oldClusteringColumns, "newClusteringColumns" -> newClusteringColumns) // This operation shouldn't be introducing AddFile actions at all. This check should be trivial. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } /** Recorded when we backfill a Delta table's existing AddFiles with row tracking data. */ case class RowTrackingBackfill( batchId: Int = 0) extends Operation(ROW_TRACKING_BACKFILL_OPERATION_NAME) { override val parameters: Map[String, Any] = Map( "batchId" -> JsonUtils.toJson(batchId) ) // RowTrackingBackfill operation commits AddFiles with files, DVs and stats copied over. // It can happen that tight bound stats were recomputed before by ComputeStats. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = false // RowTrackingBackfill only updates tags of existing files. override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(true) override def canChangePartitionColumns: Boolean = false } /** * Recorded when we unbackfill a Delta table's existing row tracking data from AddFiles. * This operation is used when dropping the row tracking feature. */ case class RowTrackingUnBackfill( batchId: Int = 0) extends Operation(ROW_TRACKING_UNBACKFILL_OPERATION_NAME) { override val parameters: Map[String, Any] = Map( "batchId" -> JsonUtils.toJson(batchId) ) // RowTrackingUnBackfill operation commits AddFiles with files, DVs and stats copied over. // It can happen that tight bound stats were recomputed before by ComputeStats. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = false // RowTrackingUnBackfill only updates metadata of existing files. override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(true) override def canChangePartitionColumns: Boolean = false } private def structFieldToMap(colPath: Seq[String], field: StructField): Map[String, Any] = { Map( "name" -> UnresolvedAttribute(colPath :+ field.name).name, "type" -> field.dataType.typeName, "nullable" -> field.nullable, "metadata" -> JsonUtils.mapper.readValue[Map[String, Any]](field.metadata.json) ) } /** * Recorded when cleaning up domain metadata. This process takes place when dropping * the domainMetadata feature. */ case class DomainMetadataCleanup(domainMetadataRemovedCount: Int) extends Operation("DOMAIN METADATA CLEANUP") { override val parameters: Map[String, Any] = Map( "domainMetadataRemovedCount" -> domainMetadataRemovedCount) // This operation shouldn't be introducing AddFile actions with DVs and tight bounds stats. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true // Only removes domain metadata. override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } /** * Qualified column type with position. We define a copy of the type here to avoid depending on * the parser output classes in our logging. */ case class QualifiedColTypeWithPositionForLog( columnPath: Seq[String], column: StructField, colPosition: Option[String]) /** Dummy operation only for testing with arbitrary operation names */ case class TestOperation( operationName: String = "TEST", override val isInPlaceFileMetadataUpdate: Option[Boolean] = None ) extends Operation(operationName) { override val parameters: Map[String, Any] = Map.empty // Perform the check for testing. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override def canChangePartitionColumns: Boolean = false } /** * Helper method to convert a sequence of command predicates in the form of an * [[Expression]]s to a sequence of Strings so be stored in the commit info. */ def predicatesToString(predicates: Seq[Expression]): Seq[String] = { val maxToStringFields = SQLConf.get.maxToStringFields predicates.map(_.simpleString(maxToStringFields)) } /** Recorded when the table properties are set. */ private val OP_UPGRADE_UNIFORM_BY_REORG = "REORG TABLE UPGRADE UNIFORM" /** * recorded when upgrading a table set uniform properties by REORG TABLE ... UPGRADE UNIFORM */ case class UpgradeUniformProperties(properties: Map[String, String]) extends Operation( OP_UPGRADE_UNIFORM_BY_REORG) { override val parameters: Map[String, Any] = Map("properties" -> JsonUtils.toJson(properties)) // This operation shouldn't be introducing AddFile actions with DVs and tight bounds stats. override def checkAddFileWithDeletionVectorStatsAreNotTightBounds: Boolean = true override val isInPlaceFileMetadataUpdate: Option[Boolean] = Some(false) override def canChangePartitionColumns: Boolean = false } } private[delta] object DeltaOperationMetrics { val WRITE = Set( "numFiles", // number of files written "numOutputBytes", // size in bytes of the written contents "numOutputRows" // number of rows written ) val OVERWRITE_REMOVES = Set( "numRemovedFiles", "numRemovedBytes" ) val REMOVE_COLUMN_MAPPING: Set[String] = Set( "numRewrittenFiles", "numOutputBytes", "numRemovedBytes", "numCopiedRows", "numDeletionVectorsRemoved" ) val STREAMING_UPDATE = Set( "numAddedFiles", // number of files added "numRemovedFiles", // number of files removed "numOutputRows", // number of rows written "numOutputBytes" // number of output writes ) val DELETE = Set( "numAddedFiles", // number of files added "numRemovedFiles", // number of files removed "numDeletionVectorsAdded", // number of deletion vectors added "numDeletionVectorsRemoved", // number of deletion vectors removed "numDeletionVectorsUpdated", // number of deletion vectors updated "numAddedChangeFiles", // number of CDC files "numDeletedRows", // number of rows removed "numCopiedRows", // number of rows copied in the process of deleting files "executionTimeMs", // time taken to execute the entire operation "scanTimeMs", // time taken to scan the files for matches "rewriteTimeMs", // time taken to rewrite the matched files "numRemovedBytes", // number of bytes removed "numAddedBytes" // number of bytes added ) val WRITE_REPLACE_WHERE = Set( "numFiles", // number of files written "numOutputBytes", // size in bytes of the written "numOutputRows", // number of rows written "numRemovedFiles", // number of files removed "numAddedChangeFiles", // number of CDC files "numDeletedRows", // number of rows removed "numCopiedRows", // number of rows copied in the process of deleting files "numRemovedBytes" // number of bytes removed ) val WRITE_REPLACE_WHERE_PARTITIONS = Set( "numFiles", // number of files written "numOutputBytes", // size in bytes of the written contents "numOutputRows", // number of rows written "numAddedChangeFiles", // number of CDC files "numRemovedFiles", // number of files removed // Records below only exist when DELTA_DML_METRICS_FROM_METADATA is enabled "numCopiedRows", // number of rows copied "numDeletedRows", // number of rows deleted "numRemovedBytes" // number of bytes removed ) /** * Deleting the entire table or partition will record row level metrics when * DELTA_DML_METRICS_FROM_METADATA is enabled * * DELETE_PARTITIONS is used only in test to verify specific delete cases. */ val DELETE_PARTITIONS = Set( "numRemovedFiles", // number of files removed "numAddedChangeFiles", // number of CDC files generated - generally 0 in this case "numDeletionVectorsAdded", // number of deletion vectors added "numDeletionVectorsRemoved", // number of deletion vectors removed "numDeletionVectorsUpdated", // number of deletion vectors updated "executionTimeMs", // time taken to execute the entire operation "scanTimeMs", // time taken to scan the files for matches "rewriteTimeMs", // time taken to rewrite the matched files // Records below only exist when DELTA_DML_METRICS_FROM_METADATA is enabled "numCopiedRows", // number of rows copied "numDeletedRows", // number of rows deleted "numAddedFiles", // number of files added "numRemovedBytes", // number of bytes removed "numAddedBytes" // number of bytes added ) trait MetricsTransformer { /** * Produce the output metric `metricName`, given all available metrics. * * If one or more input metrics are missing, the output metrics may be skipped by * returning `None`. */ def transform( metricName: String, allMetrics: Map[String, SQLMetric]): Option[(String, Long)] def transformToString( metricName: String, allMetrics: Map[String, SQLMetric]): Option[(String, String)] = { this.transform(metricName, allMetrics).map { case (name, metric) => name -> metric.toString } } } /** Pass metric on unaltered. */ final object PassMetric extends MetricsTransformer { override def transform( metricName: String, allMetrics: Map[String, SQLMetric]): Option[(String, Long)] = allMetrics.get(metricName).map(metric => metricName -> metric.value) } /** * Produce a new metric by summing up the values of `inputMetrics`. * * Treats missing metrics at 0. */ final case class SumMetrics(inputMetrics: String*) extends MetricsTransformer { override def transform( metricName: String, allMetrics: Map[String, SQLMetric]): Option[(String, Long)] = { var atLeastOneMetricExists = false val total = inputMetrics.map { name => val metricValueOpt = allMetrics.get(name) atLeastOneMetricExists |= metricValueOpt.isDefined metricValueOpt.map(_.value).getOrElse(0L) }.sum if (atLeastOneMetricExists) { Some(metricName -> total) } else { None } } } val DELETION_VECTORS: Map[String, MetricsTransformer] = Map( // Adding "numDeletionVectorsUpdated" here makes the values line up with how // "numFilesAdded"/"numFilesRemoved" behave. "numDeletionVectorsAdded" -> SumMetrics("numDeletionVectorsAdded", "numDeletionVectorsUpdated"), "numDeletionVectorsRemoved" -> SumMetrics("numDeletionVectorsRemoved", "numDeletionVectorsUpdated") ) // The same as [[DELETION_VECTORS]] but with the "Target" prefix that is used by MERGE. val MERGE_DELETION_VECTORS = Map( // Adding "numDeletionVectorsUpdated" here makes the values line up with how // "numFilesAdded"/"numFilesRemoved" behave. "numTargetDeletionVectorsAdded" -> SumMetrics("numTargetDeletionVectorsAdded", "numTargetDeletionVectorsUpdated"), "numTargetDeletionVectorsRemoved" -> SumMetrics("numTargetDeletionVectorsRemoved", "numTargetDeletionVectorsUpdated") ) val TRUNCATE = Set( "numRemovedFiles", // number of files removed "executionTimeMs" // time taken to execute the entire operation ) val CONVERT = Set( "numConvertedFiles" // number of parquet files that have been converted. ) val MERGE = Set( "numSourceRows", // number of rows in the source dataframe "numTargetRowsInserted", // number of rows inserted into the target table. "numTargetRowsUpdated", // number of rows updated in the target table. "numTargetRowsMatchedUpdated", // number of rows updated by a matched clause. // number of rows updated by a not matched by source clause. "numTargetRowsNotMatchedBySourceUpdated", "numTargetRowsDeleted", // number of rows deleted in the target table. "numTargetRowsMatchedDeleted", // number of rows deleted by a matched clause. // number of rows deleted by a not matched by source clause. "numTargetRowsNotMatchedBySourceDeleted", "numTargetRowsCopied", // number of target rows copied "numTargetBytesAdded", // number of target bytes added "numTargetBytesRemoved", // number of target bytes removed "numOutputRows", // total number of rows written out "numTargetFilesAdded", // num files added to the sink(target) "numTargetFilesRemoved", // number of files removed from the sink(target) "numTargetChangeFilesAdded", // number of CDC files "executionTimeMs", // time taken to execute the entire operation "materializeSourceTimeMs", // time taken to materialize source (or determine it's not needed) "scanTimeMs", // time taken to scan the files for matches "rewriteTimeMs", // time taken to rewrite the matched files "numTargetDeletionVectorsAdded", // number of deletion vectors added "numTargetDeletionVectorsRemoved", // number of deletion vectors removed "numTargetDeletionVectorsUpdated" // number of deletion vectors updated ) val UPDATE = Set( "numAddedFiles", // number of files added "numRemovedFiles", // number of files removed "numAddedChangeFiles", // number of CDC files "numDeletionVectorsAdded", // number of deletion vectors added "numDeletionVectorsRemoved", // number of deletion vectors removed "numDeletionVectorsUpdated", // number of deletion vectors updated "numUpdatedRows", // number of rows updated "numCopiedRows", // number of rows just copied over in the process of updating files. "executionTimeMs", // time taken to execute the entire operation "scanTimeMs", // time taken to scan the files for matches "rewriteTimeMs", // time taken to rewrite the matched files "numRemovedBytes", // number of bytes removed "numAddedBytes" // number of bytes added ) val OPTIMIZE = Set( "numAddedFiles", // number of data files added "numRemovedFiles", // number of data files removed "numAddedBytes", // number of data bytes added by optimize "numRemovedBytes", // number of data bytes removed by optimize "minFileSize", // the size of the smallest file "p25FileSize", // the size of the 25th percentile file "p50FileSize", // the median file size "p75FileSize", // the 75th percentile of the file sizes "maxFileSize", // the size of the largest file "numDeletionVectorsRemoved" // number of deletion vectors removed by optimize ) val RESTORE = Set( "tableSizeAfterRestore", // table size in bytes after restore "numOfFilesAfterRestore", // number of files in the table after restore "numRemovedFiles", // number of files removed by the restore operation "numRestoredFiles", // number of files that were added as a result of the restore "removedFilesSize", // size in bytes of files removed by the restore "restoredFilesSize" // size in bytes of files added by the restore ) val CLONE = Set( "sourceTableSize", // size in bytes of source table at version "sourceNumOfFiles", // number of files in source table at version "numRemovedFiles", // number of files removed from target table if delta table was replaced "numCopiedFiles", // number of files that were cloned - 0 for shallow tables "removedFilesSize", // size in bytes of files removed from an existing Delta table if one exists "copiedFilesSize" // size of files copied - 0 for shallow tables ) val VACUUM_START = Set( "numFilesToDelete", // number of files that will be deleted by vacuum "sizeOfDataToDelete" // total size in bytes of files that will be deleted by vacuum ) val VACUUM_END = Set( "numDeletedFiles", // number of files deleted by vacuum "numVacuumedDirectories" // number of directories vacuumed ) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaOptions.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.util.Locale import java.util.regex.PatternSyntaxException import scala.util.Try import scala.util.matching.Regex import org.apache.spark.sql.connector.catalog.SupportsV1OverwriteWithSaveAsTable import org.apache.spark.sql.delta.DeltaOptions.{DATA_CHANGE_OPTION, IS_DATAFRAME_WRITER_V1_SAVE_AS_TABLE_OVERWRITE, MERGE_SCHEMA_OPTION, OVERWRITE_SCHEMA_OPTION, PARTITION_OVERWRITE_MODE_OPTION} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.network.util.{ByteUnit, JavaUtils} import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.internal.SQLConf trait DeltaOptionParser { protected def sqlConf: SQLConf protected def options: CaseInsensitiveMap[String] def toBoolean(input: String, name: String): Boolean = { Try(input.toBoolean).toOption.getOrElse { throw DeltaErrors.illegalDeltaOptionException(name, input, "must be 'true' or 'false'") } } } trait DeltaWriteOptions extends DeltaWriteOptionsImpl with DeltaOptionParser { import DeltaOptions._ val replaceWhere: Option[String] = options.get(REPLACE_WHERE_OPTION) val userMetadata: Option[String] = options.get(USER_METADATA_OPTION) /** * Whether to add an adaptive shuffle before writing out the files to break skew, and coalesce * data into chunkier files. */ val optimizeWrite: Option[Boolean] = options.get(OPTIMIZE_WRITE_OPTION) .map(toBoolean(_, OPTIMIZE_WRITE_OPTION)) } trait DeltaWriteOptionsImpl extends DeltaOptionParser { import DeltaOptions._ /** * Whether the user has enabled auto schema merging in writes using either a DataFrame option * or SQL Session configuration. Automerging is off when table ACLs are enabled. * We always respect the DataFrame writer configuration over the session config. */ def canMergeSchema: Boolean = { options.get(MERGE_SCHEMA_OPTION) .map(toBoolean(_, MERGE_SCHEMA_OPTION)) .getOrElse(sqlConf.getConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE)) } /** * Whether to allow overwriting the schema of a Delta table in an overwrite mode operation. If * ACLs are enabled, we can't change the schema of an operation through a write, which requires * MODIFY permissions, when schema changes require OWN permissions. */ def canOverwriteSchema: Boolean = { options.get(OVERWRITE_SCHEMA_OPTION).exists(toBoolean(_, OVERWRITE_SCHEMA_OPTION)) } /** * Whether this write is coming from DataFrameWriter V1 saveAsTable. */ def isDataFrameWriterV1SaveAsTableOverwrite: Boolean = { options.get(IS_DATAFRAME_WRITER_V1_SAVE_AS_TABLE_OVERWRITE) .exists(toBoolean(_, IS_DATAFRAME_WRITER_V1_SAVE_AS_TABLE_OVERWRITE)) } /** * Whether to write new data to the table or just rearrange data that is already * part of the table. This option declares that the data being written by this job * does not change any data in the table and merely rearranges existing data. * This makes sure streaming queries reading from this table will not see any new changes */ def rearrangeOnly: Boolean = { options.get(DATA_CHANGE_OPTION).exists(!toBoolean(_, DATA_CHANGE_OPTION)) } val txnVersion = options.get(TXN_VERSION).map { str => Try(str.toLong).toOption.filter(_ >= 0).getOrElse { throw DeltaErrors.illegalDeltaOptionException( TXN_VERSION, str, "must be a non-negative integer") } } val txnAppId = options.get(TXN_APP_ID) private def validateIdempotentWriteOptions(): Unit = { // Either both txnVersion and txnAppId must be specified to get idempotent writes or // neither must be given. In all other cases, throw an exception. val numOptions = txnVersion.size + txnAppId.size if (numOptions != 0 && numOptions != 2) { throw DeltaErrors.invalidIdempotentWritesOptionsException("Both txnVersion and txnAppId " + "must be specified for idempotent data frame writes") } } validateIdempotentWriteOptions() /** Whether partitionOverwriteMode is provided as a DataFrameWriter option. */ val partitionOverwriteModeInOptions: Boolean = options.contains(PARTITION_OVERWRITE_MODE_OPTION) /** Whether to only overwrite partitions that have data written into it at runtime. */ def isDynamicPartitionOverwriteMode: Boolean = { val mode = options.get(PARTITION_OVERWRITE_MODE_OPTION) .getOrElse(sqlConf.getConf(SQLConf.PARTITION_OVERWRITE_MODE).toString) val modeIsDynamic = mode != null && mode.equalsIgnoreCase(PARTITION_OVERWRITE_MODE_DYNAMIC) if (!sqlConf.getConf(DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED)) { // Raise an exception when DYNAMIC_PARTITION_OVERWRITE_ENABLED=false // but users explicitly request dynamic partition overwrite. if (modeIsDynamic) { throw DeltaErrors.deltaDynamicPartitionOverwriteDisabled() } // If dynamic partition overwrite mode is disabled, fallback to the default behavior false } else { if (mode == null || !DeltaOptions.PARTITION_OVERWRITE_MODE_VALUES.exists(mode.equalsIgnoreCase(_))) { val acceptableStr = DeltaOptions.PARTITION_OVERWRITE_MODE_VALUES.map("'" + _ + "'").mkString(" or ") throw DeltaErrors.illegalDeltaOptionException( PARTITION_OVERWRITE_MODE_OPTION, mode, s"must be ${acceptableStr}" ) } modeIsDynamic } } } trait DeltaReadOptions extends DeltaOptionParser { import DeltaOptions._ val maxFilesPerTrigger = options.get(MAX_FILES_PER_TRIGGER_OPTION).map { str => Try(str.toInt).toOption.filter(_ > 0).getOrElse { throw DeltaErrors.illegalDeltaOptionException( MAX_FILES_PER_TRIGGER_OPTION, str, "must be a positive integer") } } val maxBytesPerTrigger = options.get(MAX_BYTES_PER_TRIGGER_OPTION).map { str => Try(JavaUtils.byteStringAs(str, ByteUnit.BYTE)).toOption.filter(_ > 0).getOrElse { throw DeltaErrors.illegalDeltaOptionException( MAX_BYTES_PER_TRIGGER_OPTION, str, "must be a size configuration such as '10g'") } } val ignoreFileDeletion = options.get(IGNORE_FILE_DELETION_OPTION) .exists(toBoolean(_, IGNORE_FILE_DELETION_OPTION)) val ignoreChanges = options.get(IGNORE_CHANGES_OPTION).exists(toBoolean(_, IGNORE_CHANGES_OPTION)) val ignoreDeletes = options.get(IGNORE_DELETES_OPTION).exists(toBoolean(_, IGNORE_DELETES_OPTION)) val skipChangeCommits = options.get(SKIP_CHANGE_COMMITS_OPTION) .exists(toBoolean(_, SKIP_CHANGE_COMMITS_OPTION)) val failOnDataLoss = options.get(FAIL_ON_DATA_LOSS_OPTION) .forall(toBoolean(_, FAIL_ON_DATA_LOSS_OPTION)) // thanks to forall: by default true val readChangeFeed = options.get(CDC_READ_OPTION).exists(toBoolean(_, CDC_READ_OPTION)) || options.get(CDC_READ_OPTION_LEGACY).exists(toBoolean(_, CDC_READ_OPTION_LEGACY)) val excludeRegex: Option[Regex] = try options.get(EXCLUDE_REGEX_OPTION).map(_.r) catch { case e: PatternSyntaxException => throw DeltaErrors.excludeRegexOptionException(EXCLUDE_REGEX_OPTION, e) } val startingVersion: Option[DeltaStartingVersion] = options.get(STARTING_VERSION_OPTION).map { case "latest" => StartingVersionLatest case str => Try(str.toLong).toOption.filter(_ >= 0).map(StartingVersion).getOrElse{ throw DeltaErrors.illegalDeltaOptionException( STARTING_VERSION_OPTION, str, "must be greater than or equal to zero") } } val startingTimestamp = options.get(STARTING_TIMESTAMP_OPTION) private def provideOneStartingOption(): Unit = { if (startingTimestamp.isDefined && startingVersion.isDefined) { throw DeltaErrors.startingVersionAndTimestampBothSetException( STARTING_VERSION_OPTION, STARTING_TIMESTAMP_OPTION) } } def containsStartingVersionOrTimestamp: Boolean = { options.contains(STARTING_VERSION_OPTION) || options.contains(STARTING_TIMESTAMP_OPTION) } provideOneStartingOption() val schemaTrackingLocation = options.get(SCHEMA_TRACKING_LOCATION) val sourceTrackingId = options.get(STREAMING_SOURCE_TRACKING_ID) val allowSourceColumnRename = options.get(ALLOW_SOURCE_COLUMN_RENAME) val allowSourceColumnDrop = options.get(ALLOW_SOURCE_COLUMN_DROP) val allowSourceColumnTypeChange = options.get(ALLOW_SOURCE_COLUMN_TYPE_CHANGE) } /** * Options for the Delta data source. */ class DeltaOptions( @transient protected[delta] val options: CaseInsensitiveMap[String], @transient protected val sqlConf: SQLConf) extends DeltaWriteOptions with DeltaReadOptions with Serializable { DeltaOptions.verifyOptions(options) def this(options: Map[String, String], conf: SQLConf) = this(CaseInsensitiveMap(options), conf) } object DeltaOptions extends DeltaLogging { /** Internal option to indicate write originated from DataFrameWriter V1 saveAsTable. */ val IS_DATAFRAME_WRITER_V1_SAVE_AS_TABLE_OVERWRITE = SupportsV1OverwriteWithSaveAsTable.OPTION_NAME /** An option to overwrite only the data that matches predicates over partition columns. */ val REPLACE_WHERE_OPTION = "replaceWhere" /** An option to allow automatic schema merging during a write operation. */ val MERGE_SCHEMA_OPTION = "mergeSchema" /** An option to allow overwriting schema and partitioning during an overwrite write operation. */ val OVERWRITE_SCHEMA_OPTION = "overwriteSchema" /** An option to specify user-defined metadata in commitInfo */ val USER_METADATA_OPTION = "userMetadata" val PARTITION_OVERWRITE_MODE_OPTION = "partitionOverwriteMode" val PARTITION_OVERWRITE_MODE_DYNAMIC = "DYNAMIC" val PARTITION_OVERWRITE_MODE_STATIC = "STATIC" val PARTITION_OVERWRITE_MODE_VALUES = Set(PARTITION_OVERWRITE_MODE_STATIC, PARTITION_OVERWRITE_MODE_DYNAMIC) val MAX_FILES_PER_TRIGGER_OPTION = "maxFilesPerTrigger" val MAX_FILES_PER_TRIGGER_OPTION_DEFAULT = 1000 val MAX_BYTES_PER_TRIGGER_OPTION = "maxBytesPerTrigger" val EXCLUDE_REGEX_OPTION = "excludeRegex" val IGNORE_FILE_DELETION_OPTION = "ignoreFileDeletion" val IGNORE_CHANGES_OPTION = "ignoreChanges" val IGNORE_DELETES_OPTION = "ignoreDeletes" val SKIP_CHANGE_COMMITS_OPTION = "skipChangeCommits" val FAIL_ON_DATA_LOSS_OPTION = "failOnDataLoss" val OPTIMIZE_WRITE_OPTION = "optimizeWrite" val DATA_CHANGE_OPTION = "dataChange" val STARTING_VERSION_OPTION = "startingVersion" val STARTING_TIMESTAMP_OPTION = "startingTimestamp" val CDC_START_VERSION = "startingVersion" val CDC_START_TIMESTAMP = "startingTimestamp" val CDC_END_VERSION = "endingVersion" val CDC_END_TIMESTAMP = "endingTimestamp" val CDC_READ_OPTION = "readChangeFeed" val CDC_READ_OPTION_LEGACY = "readChangeData" val VERSION_AS_OF = "versionAsOf" val TIMESTAMP_AS_OF = "timestampAsOf" val COMPRESSION = "compression" val MAX_RECORDS_PER_FILE = "maxRecordsPerFile" val TXN_APP_ID = "txnAppId" val TXN_VERSION = "txnVersion" /** * An option to allow column mapping enabled tables to conduct schema evolution during streaming */ val SCHEMA_TRACKING_LOCATION = "schemaTrackingLocation" /** * Alias for `schemaTrackingLocation`, so users familiar with AutoLoader can migrate easily. */ val SCHEMA_TRACKING_LOCATION_ALIAS = "schemaLocation" /** * An option to instruct DeltaSource to pick a customized subdirectory for schema log in case of * rare conflicts such as when a stream needs to do a self-union of two Delta sources from the * same table. * The final schema log location will be $parent/_schema_log_${tahoeId}_${sourceTrackingId}. */ val STREAMING_SOURCE_TRACKING_ID = "streamingSourceTrackingId" val ALLOW_SOURCE_COLUMN_DROP = "allowSourceColumnDrop" val ALLOW_SOURCE_COLUMN_RENAME = "allowSourceColumnRename" val ALLOW_SOURCE_COLUMN_TYPE_CHANGE = "allowSourceColumnTypeChange" /** * An option to control if delta will write partition columns to data files */ val WRITE_PARTITION_COLUMNS = "writePartitionColumns" val validOptionKeys : Set[String] = Set( IS_DATAFRAME_WRITER_V1_SAVE_AS_TABLE_OVERWRITE, REPLACE_WHERE_OPTION, MERGE_SCHEMA_OPTION, EXCLUDE_REGEX_OPTION, OVERWRITE_SCHEMA_OPTION, USER_METADATA_OPTION, PARTITION_OVERWRITE_MODE_OPTION, MAX_FILES_PER_TRIGGER_OPTION, IGNORE_FILE_DELETION_OPTION, IGNORE_CHANGES_OPTION, IGNORE_DELETES_OPTION, FAIL_ON_DATA_LOSS_OPTION, OPTIMIZE_WRITE_OPTION, DATA_CHANGE_OPTION, STARTING_TIMESTAMP_OPTION, STARTING_VERSION_OPTION, CDC_READ_OPTION, CDC_READ_OPTION_LEGACY, CDC_START_TIMESTAMP, CDC_END_TIMESTAMP, CDC_START_VERSION, CDC_END_VERSION, COMPRESSION, MAX_RECORDS_PER_FILE, TXN_APP_ID, TXN_VERSION, SCHEMA_TRACKING_LOCATION, SCHEMA_TRACKING_LOCATION_ALIAS, STREAMING_SOURCE_TRACKING_ID, "queryName", "checkpointLocation", "path", VERSION_AS_OF, TIMESTAMP_AS_OF, WRITE_PARTITION_COLUMNS ) /** Iterates over all user passed options and logs any that are not valid. */ def verifyOptions(options: CaseInsensitiveMap[String]): Unit = { val invalidUserOptions = SQLConf.get.redactOptions(options -- validOptionKeys.map(_.toLowerCase(Locale.ROOT))) if (invalidUserOptions.nonEmpty) { recordDeltaEvent(null, "delta.option.invalid", data = invalidUserOptions ) } } } /** * Definitions for the batch read schema mode for CDF */ sealed trait DeltaBatchCDFSchemaMode { def name: String } /** * `latest` batch CDF schema mode specifies that the latest schema should be used when serving * the CDF batch. */ case object BatchCDFSchemaLatest extends DeltaBatchCDFSchemaMode { val name = "latest" } /** * `endVersion` batch CDF schema mode specifies that the query range's end version's schema should * be used for serving the CDF batch. * This is the current default for column mapping enabled tables so we could read using the exact * schema at the versions being queried to reduce schema read compatibility mismatches. */ case object BatchCDFSchemaEndVersion extends DeltaBatchCDFSchemaMode { val name = "endversion" } /** * `legacy` batch CDF schema mode specifies that neither latest nor end version's schema is * strictly used for serving the CDF batch, e.g. when user uses TimeTravel with batch CDF and wants * to respect the time travelled schema. * This is the current default for non-column mapping tables. */ case object BatchCDFSchemaLegacy extends DeltaBatchCDFSchemaMode { val name = "legacy" } object DeltaBatchCDFSchemaMode { def apply(name: String): DeltaBatchCDFSchemaMode = { name.toLowerCase(Locale.ROOT) match { case BatchCDFSchemaLatest.name => BatchCDFSchemaLatest case BatchCDFSchemaEndVersion.name => BatchCDFSchemaEndVersion case BatchCDFSchemaLegacy.name => BatchCDFSchemaLegacy } } } /** * Definitions for the starting version of a Delta stream. */ sealed trait DeltaStartingVersion case object StartingVersionLatest extends DeltaStartingVersion case class StartingVersion(version: Long) extends DeltaStartingVersion ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaParquetFileFormat.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.collection.mutable.ArrayBuffer import scala.util.control.NonFatal import org.apache.spark.sql.delta.RowIndexFilterType import org.apache.spark.sql.delta.DeltaParquetFileFormat._ import org.apache.spark.sql.delta.actions.{DeletionVectorDescriptor, Metadata, Protocol} import org.apache.spark.sql.delta.commands.DeletionVectorUtils.deletionVectorsReadable import org.apache.spark.sql.delta.deletionvectors.{DropMarkedRowsFilter, KeepAllRowsFilter, KeepMarkedRowsFilter} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.schema.SchemaMergingUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.hadoop.mapreduce.Job import org.apache.parquet.hadoop.ParquetOutputFormat import org.apache.parquet.hadoop.util.ContextUtil import org.apache.spark.internal.{Logging, MDC} import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.FileSourceConstantMetadataStructField import org.apache.spark.sql.execution.datasources.OutputWriterFactory import org.apache.spark.sql.execution.datasources.PartitionedFile import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.execution.vectorized.{OffHeapColumnVector, OnHeapColumnVector, WritableColumnVector} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.sources._ import org.apache.spark.sql.types.{ByteType, LongType, MetadataBuilder, StringType, StructField, StructType} import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnarBatchRow, ColumnVector} import org.apache.spark.util.SerializableConfiguration /** * Base class for Delta Parquet file format that uses ProtocolMetadataAdapter abstraction. * A thin wrapper over the Parquet file format to support * - columns names without restrictions. * - populated a column from the deletion vector of this file (if exists) to indicate * whether the row is deleted or not according to the deletion vector. Consumers * of this scan can use the column values to filter out the deleted rows. * * @param protocolMetadataAdapter Adapter providing protocol and metadata info for the table * @param nullableRowTrackingConstantFields If true, row tracking constant fields (e.g., base row * ID, default row commit version) are nullable in schema * @param nullableRowTrackingGeneratedFields If true, row tracking generated fields are nullable * @param optimizationsEnabled Whether to enable optimizations (file splitting, predicate pushdown) * @param tablePath Table path for deletion vector support; None disables DV processing * @param isCDCRead Whether this is a CDC (Change Data Capture) read * @param useMetadataRowIndexOpt Controls row index source for DV filtering. When provided, * must match optimizationsEnabled (true enables _metadata.row_index * and file splitting; false uses internal counter, no splitting). * When None, reads from session config. */ abstract class DeltaParquetFileFormatBase( protected val protocolMetadataAdapter: ProtocolMetadataAdapter, protected val nullableRowTrackingConstantFields: Boolean = false, protected val nullableRowTrackingGeneratedFields: Boolean = false, protected val optimizationsEnabled: Boolean = true, protected val tablePath: Option[String] = None, protected val isCDCRead: Boolean = false, protected val useMetadataRowIndexOpt: Option[Boolean] = None) extends ParquetFileFormat with Logging { // Validate either we have all arguments for DV enabled read or none of them. if (hasTablePath) { useMetadataRowIndexOpt.foreach { useMetadataRowIndex => require(useMetadataRowIndex == optimizationsEnabled, "Wrong arguments for Delta table scan with deletion vectors") } } SparkSession.getActiveSession.ifDefined { session => protocolMetadataAdapter.assertTableReadable(session) } require(!nullableRowTrackingConstantFields || nullableRowTrackingGeneratedFields) val columnMappingMode: DeltaColumnMappingMode = protocolMetadataAdapter.columnMappingMode val referenceSchema: StructType = protocolMetadataAdapter.getReferenceSchema if (columnMappingMode == IdMapping) { val requiredReadConf = SQLConf.PARQUET_FIELD_ID_READ_ENABLED require(SparkSession.getActiveSession.exists(_.sessionState.conf.getConf(requiredReadConf)), s"${requiredReadConf.key} must be enabled to support Delta id column mapping mode") val requiredWriteConf = SQLConf.PARQUET_FIELD_ID_WRITE_ENABLED require(SparkSession.getActiveSession.exists(_.sessionState.conf.getConf(requiredWriteConf)), s"${requiredWriteConf.key} must be enabled to support Delta id column mapping mode") } /** * prepareSchemaForRead must only be used for parquet read. * It removes "PARQUET_FIELD_ID_METADATA_KEY" for name mapping mode which address columns by * physical name instead of id. */ def prepareSchemaForRead(inputSchema: StructType): StructType = { val schema = DeltaColumnMapping.createPhysicalSchema( inputSchema, referenceSchema, columnMappingMode) if (columnMappingMode == NameMapping) { SchemaMergingUtils.transformColumns(schema) { (_, field, _) => field.copy(metadata = new MetadataBuilder() .withMetadata(field.metadata) .remove(DeltaColumnMapping.PARQUET_FIELD_ID_METADATA_KEY) .remove(DeltaColumnMapping.PARQUET_FIELD_NESTED_IDS_METADATA_KEY) .build()) } } else schema } /** * Prepares filters so that they can be pushed down into the Parquet reader. * * If column mapping is enabled, then logical column names in the filters will be replaced with * their corresponding physical column names. This is necessary as the Parquet files will use * physical column names, and the requested schema pushed down in the Parquet reader will also use * physical column names. */ private def prepareFiltersForRead(filters: Seq[Filter]): Seq[Filter] = { if (!optimizationsEnabled) { Seq.empty } else if (columnMappingMode != NoMapping) { import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper val physicalNameMap = DeltaColumnMapping.getLogicalNameToPhysicalNameMap(referenceSchema) .map { case (logicalName, physicalName) => (logicalName.quoted, physicalName.quoted) } filters.flatMap(translateFilterForColumnMapping(_, physicalNameMap)) } else { filters } } override def isSplitable( sparkSession: SparkSession, options: Map[String, String], path: Path): Boolean = optimizationsEnabled def hasTablePath: Boolean = tablePath.isDefined override def buildReaderWithPartitionValues( sparkSession: SparkSession, dataSchema: StructType, partitionSchema: StructType, requiredSchema: StructType, filters: Seq[Filter], options: Map[String, String], hadoopConf: Configuration): PartitionedFile => Iterator[InternalRow] = { // Use explicitly provided value if available, otherwise read from config val useMetadataRowIndex = useMetadataRowIndexOpt.getOrElse( sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX)) val parquetDataReader: PartitionedFile => Iterator[InternalRow] = super.buildReaderWithPartitionValues( sparkSession, prepareSchemaForRead(dataSchema), prepareSchemaForRead(partitionSchema), prepareSchemaForRead(requiredSchema), prepareFiltersForRead(filters), options, hadoopConf) val schemaWithIndices = requiredSchema.fields.zipWithIndex def findColumn(name: String): Option[ColumnMetadata] = { val results = schemaWithIndices.filter(_._1.name == name) if (results.length > 1) { throw new IllegalArgumentException( s"There are more than one column with name=`$name` requested in the reader output") } results.headOption.map(e => ColumnMetadata(e._2, e._1)) } val isRowDeletedColumn = findColumn(IS_ROW_DELETED_COLUMN_NAME) val rowIndexColumnName = if (useMetadataRowIndex) { ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME } else { ROW_INDEX_COLUMN_NAME } val rowIndexColumn = findColumn(rowIndexColumnName) // We don't have any additional columns to generate, just return the original reader as is. if (isRowDeletedColumn.isEmpty && rowIndexColumn.isEmpty) return parquetDataReader // We are using the row_index col generated by the parquet reader and there are no more // columns to generate. if (useMetadataRowIndex && isRowDeletedColumn.isEmpty) return parquetDataReader // Verify that either predicate pushdown with metadata column is enabled or optimizations // are disabled. require(useMetadataRowIndex || !optimizationsEnabled, "Cannot generate row index related metadata with file splitting or predicate pushdown") if (hasTablePath && isRowDeletedColumn.isEmpty) { throw new IllegalArgumentException( s"Expected a column $IS_ROW_DELETED_COLUMN_NAME in the schema") } val serializableHadoopConf = new SerializableConfiguration(hadoopConf) val useOffHeapBuffers = sparkSession.sessionState.conf.offHeapColumnVectorEnabled (partitionedFile: PartitionedFile) => { val rowIteratorFromParquet = parquetDataReader(partitionedFile) try { val iterToReturn = iteratorWithAdditionalMetadataColumns( partitionedFile, rowIteratorFromParquet, isRowDeletedColumn, rowIndexColumn, useOffHeapBuffers, serializableHadoopConf, useMetadataRowIndex) iterToReturn.asInstanceOf[Iterator[InternalRow]] } catch { case NonFatal(e) => // Close the iterator if it is a closeable resource. The `ParquetFileFormat` opens // the file and returns `RecordReaderIterator` (which implements `AutoCloseable` and // `Iterator`) instance as a `Iterator`. rowIteratorFromParquet match { case resource: AutoCloseable => DeltaParquetFileFormat.closeQuietly(resource) case _ => // do nothing } throw e } } } override def supportFieldName(name: String): Boolean = { if (columnMappingMode != NoMapping) true else super.supportFieldName(name) } override def metadataSchemaFields: Seq[StructField] = { // TODO(SPARK-47731): Parquet reader in Spark has a bug where a file containing 2b+ rows // in a single rowgroup causes it to run out of the `Integer` range. // For Delta Parquet readers don't expose the row_index field as a metadata field when it is // not strictly required. We do expose it when Row Tracking or DVs are enabled. // In general, having 2b+ rows in a single rowgroup is not a common use case. When the issue is // hit an exception is thrown. if (protocolMetadataAdapter.isRowIdEnabled && !isCDCRead) { // We should not expose row tracking fields for CDC reads. val extraFields = protocolMetadataAdapter.createRowTrackingMetadataFields( nullableRowTrackingConstantFields, nullableRowTrackingGeneratedFields) super.metadataSchemaFields ++ extraFields } else if (protocolMetadataAdapter.isDeletionVectorReadable) { super.metadataSchemaFields } else { super.metadataSchemaFields.filter(_ != ParquetFileFormat.ROW_INDEX_FIELD) } } override def prepareWrite( sparkSession: SparkSession, job: Job, options: Map[String, String], dataSchema: StructType): OutputWriterFactory = { val factory = super.prepareWrite(sparkSession, job, options, dataSchema) val conf = ContextUtil.getConfiguration(job) // Always write timestamp as TIMESTAMP_MICROS for IcebergCompat based on Iceberg spec if (protocolMetadataAdapter.isIcebergCompatAnyEnabled) { conf.set(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key, SQLConf.ParquetOutputTimestampType.TIMESTAMP_MICROS.toString) } if (protocolMetadataAdapter.isIcebergCompatGeqEnabled(2)) { // Starting from IcebergCompatV2, we need to write nested field IDs for list and map // types to the parquet schema. Spark currently does not support it so we hook in our // own write support class. ParquetOutputFormat.setWriteSupportClass(job, classOf[DeltaParquetWriteSupport]) } factory } override def fileConstantMetadataExtractors: Map[String, PartitionedFile => Any] = { val extractBaseRowId: PartitionedFile => Any = { file => file.otherConstantMetadataColumnValues.getOrElse(RowId.BASE_ROW_ID, { if (nullableRowTrackingConstantFields) { null } else { throw new IllegalStateException( s"Missing ${RowId.BASE_ROW_ID} value for file '${file.filePath}'") } }) } val extractDefaultRowCommitVersion: PartitionedFile => Any = { file => file.otherConstantMetadataColumnValues .getOrElse(DefaultRowCommitVersion.METADATA_STRUCT_FIELD_NAME, { if (nullableRowTrackingConstantFields) { null } else { throw new IllegalStateException( s"Missing ${DefaultRowCommitVersion.METADATA_STRUCT_FIELD_NAME} value " + s"for file '${file.filePath}'") } }) } super.fileConstantMetadataExtractors .updated(RowId.BASE_ROW_ID, extractBaseRowId) .updated(DefaultRowCommitVersion.METADATA_STRUCT_FIELD_NAME, extractDefaultRowCommitVersion) } /** * Modifies the data read from underlying Parquet reader by populating one or both of the * following metadata columns. * - [[IS_ROW_DELETED_COLUMN_NAME]] - row deleted status from deletion vector corresponding * to this file * - [[ROW_INDEX_COLUMN_NAME]] - index of the row within the file. Note, this column is only * populated when we are not using _metadata.row_index column. */ private def iteratorWithAdditionalMetadataColumns( partitionedFile: PartitionedFile, iterator: Iterator[Object], isRowDeletedColumnOpt: Option[ColumnMetadata], rowIndexColumnOpt: Option[ColumnMetadata], useOffHeapBuffers: Boolean, serializableHadoopConf: SerializableConfiguration, useMetadataRowIndex: Boolean): Iterator[Object] = { require(!useMetadataRowIndex || rowIndexColumnOpt.isDefined, "useMetadataRowIndex is enabled but rowIndexColumn is not defined.") val rowIndexFilterOpt = isRowDeletedColumnOpt.map { col => // Fetch the DV descriptor from the broadcast map and create a row index filter val dvDescriptorOpt = partitionedFile.otherConstantMetadataColumnValues .get(FILE_ROW_INDEX_FILTER_ID_ENCODED) val filterTypeOpt = partitionedFile.otherConstantMetadataColumnValues .get(FILE_ROW_INDEX_FILTER_TYPE) if (dvDescriptorOpt.isDefined && filterTypeOpt.isDefined) { val rowIndexFilter = filterTypeOpt.get match { case RowIndexFilterType.IF_CONTAINED => DropMarkedRowsFilter case RowIndexFilterType.IF_NOT_CONTAINED => KeepMarkedRowsFilter case unexpectedFilterType => throw new IllegalStateException( s"Unexpected row index filter type: ${unexpectedFilterType}") } rowIndexFilter.createInstance( DeletionVectorDescriptor.deserializeFromBase64(dvDescriptorOpt.get.asInstanceOf[String]), serializableHadoopConf.value, tablePath.map(new Path(_))) } else if (dvDescriptorOpt.isDefined || filterTypeOpt.isDefined) { throw new IllegalStateException( s"Both ${FILE_ROW_INDEX_FILTER_ID_ENCODED} and ${FILE_ROW_INDEX_FILTER_TYPE} " + "should either both have values or no values at all.") } else { KeepAllRowsFilter } } // We only generate the row index column when predicate pushdown is not enabled. val rowIndexColumnToWriteOpt = if (useMetadataRowIndex) None else rowIndexColumnOpt val metadataColumnsToWrite = Seq(isRowDeletedColumnOpt, rowIndexColumnToWriteOpt).filter(_.nonEmpty).map(_.get) // When metadata.row_index is not used there is no way to verify the Parquet index is // starting from 0. We disable the splits, so the assumption is ParquetFileFormat respects // that. var rowIndex: Long = 0 // Used only when non-column row batches are received from the Parquet reader val tempVector = new OnHeapColumnVector(1, ByteType) iterator.map { row => row match { case batch: ColumnarBatch => // When vectorized Parquet reader is enabled. val size = batch.numRows() // Create vectors for all needed metadata columns. // We can't use the one from Parquet reader as it set the // [[WritableColumnVector.isAllNulls]] to true and it can't be reset with using any // public APIs. DeltaParquetFileFormat.trySafely( useOffHeapBuffers, size, metadataColumnsToWrite) { writableVectors => val indexVectorTuples = new ArrayBuffer[(Int, ColumnVector)] // When predicate pushdown is enabled we use _metadata.row_index. Therefore, // we only need to construct the isRowDeleted column. var index = 0 isRowDeletedColumnOpt.foreach { columnMetadata => val isRowDeletedVector = writableVectors(index) if (useMetadataRowIndex) { rowIndexFilterOpt.get.materializeIntoVectorWithRowIndex( size, batch.column(rowIndexColumnOpt.get.index), isRowDeletedVector) } else { rowIndexFilterOpt.get .materializeIntoVector(rowIndex, rowIndex + size, isRowDeletedVector) } indexVectorTuples += (columnMetadata.index -> isRowDeletedVector) index += 1 } rowIndexColumnToWriteOpt.foreach { columnMetadata => val rowIndexVector = writableVectors(index) // populate the row index column value. for (i <- 0 until size) { rowIndexVector.putLong(i, rowIndex + i) } indexVectorTuples += (columnMetadata.index -> rowIndexVector) index += 1 } val newBatch = DeltaParquetFileFormat.replaceVectors(batch, indexVectorTuples.toSeq: _*) rowIndex += size newBatch } case columnarRow: ColumnarBatchRow => // When vectorized reader is enabled but returns immutable rows instead of // columnar batches [[ColumnarBatchRow]]. So we have to copy the row as a // mutable [[InternalRow]] and set the `row_index` and `is_row_deleted` // column values. This is not efficient. It should affect only the wide // tables. https://github.com/delta-io/delta/issues/2246 val newRow = columnarRow.copy(); isRowDeletedColumnOpt.foreach { columnMetadata => val rowIndexForFiltering = if (useMetadataRowIndex) { columnarRow.getLong(rowIndexColumnOpt.get.index) } else { rowIndex } rowIndexFilterOpt.get.materializeSingleRowWithRowIndex(rowIndexForFiltering, tempVector) newRow.setByte(columnMetadata.index, tempVector.getByte(0)) } rowIndexColumnToWriteOpt .foreach(columnMetadata => newRow.setLong(columnMetadata.index, rowIndex)) rowIndex += 1 newRow case rest: InternalRow => // When vectorized Parquet reader is disabled // Temporary vector variable used to get DV values from RowIndexFilter // Currently the RowIndexFilter only supports writing into a columnar vector // and doesn't have methods to get DV value for a specific row index. // TODO: This is not efficient, but it is ok given the default reader is vectorized isRowDeletedColumnOpt.foreach { columnMetadata => val rowIndexForFiltering = if (useMetadataRowIndex) { rest.getLong(rowIndexColumnOpt.get.index) } else { rowIndex } rowIndexFilterOpt.get.materializeSingleRowWithRowIndex(rowIndexForFiltering, tempVector) rest.setByte(columnMetadata.index, tempVector.getByte(0)) } rowIndexColumnToWriteOpt .foreach(columnMetadata => rest.setLong(columnMetadata.index, rowIndex)) rowIndex += 1 rest case others => throw new RuntimeException( s"Parquet reader returned an unknown row type: ${others.getClass.getName}") } } } /** * Translates the filter to use physical column names instead of logical column names. * This is needed when the column mapping mode is set to `NameMapping` or `IdMapping` * to match the requested schema that's passed to the [[ParquetFileFormat]]. */ private def translateFilterForColumnMapping( filter: Filter, physicalNameMap: Map[String, String]): Option[Filter] = { object PhysicalAttribute { def unapply(attribute: String): Option[String] = { physicalNameMap.get(attribute) } } filter match { case EqualTo(PhysicalAttribute(physicalAttribute), value) => Some(EqualTo(physicalAttribute, value)) case EqualNullSafe(PhysicalAttribute(physicalAttribute), value) => Some(EqualNullSafe(physicalAttribute, value)) case GreaterThan(PhysicalAttribute(physicalAttribute), value) => Some(GreaterThan(physicalAttribute, value)) case GreaterThanOrEqual(PhysicalAttribute(physicalAttribute), value) => Some(GreaterThanOrEqual(physicalAttribute, value)) case LessThan(PhysicalAttribute(physicalAttribute), value) => Some(LessThan(physicalAttribute, value)) case LessThanOrEqual(PhysicalAttribute(physicalAttribute), value) => Some(LessThanOrEqual(physicalAttribute, value)) case In(PhysicalAttribute(physicalAttribute), values) => Some(In(physicalAttribute, values)) case IsNull(PhysicalAttribute(physicalAttribute)) => Some(IsNull(physicalAttribute)) case IsNotNull(PhysicalAttribute(physicalAttribute)) => Some(IsNotNull(physicalAttribute)) case And(left, right) => val newLeft = translateFilterForColumnMapping(left, physicalNameMap) val newRight = translateFilterForColumnMapping(right, physicalNameMap) (newLeft, newRight) match { case (Some(l), Some(r)) => Some(And(l, r)) case (Some(l), None) => Some(l) case (_, _) => newRight } case Or(left, right) => val newLeft = translateFilterForColumnMapping(left, physicalNameMap) val newRight = translateFilterForColumnMapping(right, physicalNameMap) (newLeft, newRight) match { case (Some(l), Some(r)) => Some(Or(l, r)) case (_, _) => None } case Not(child) => translateFilterForColumnMapping(child, physicalNameMap).map(Not) case StringStartsWith(PhysicalAttribute(physicalAttribute), value) => Some(StringStartsWith(physicalAttribute, value)) case StringEndsWith(PhysicalAttribute(physicalAttribute), value) => Some(StringEndsWith(physicalAttribute, value)) case StringContains(PhysicalAttribute(physicalAttribute), value) => Some(StringContains(physicalAttribute, value)) case AlwaysTrue() => Some(AlwaysTrue()) case AlwaysFalse() => Some(AlwaysFalse()) case _ => logError(log"Failed to translate filter ${MDC(DeltaLogKeys.FILTER, filter)}") None } } } /** * DeltaParquetFileFormat case class that uses Delta Spark's Protocol and Metadata. * Used by Delta spark v1 connector */ case class DeltaParquetFileFormat( protocol: Protocol, metadata: Metadata, override val nullableRowTrackingConstantFields: Boolean = false, override val nullableRowTrackingGeneratedFields: Boolean = false, override val optimizationsEnabled: Boolean = true, override val tablePath: Option[String] = None, override val isCDCRead: Boolean = false) extends DeltaParquetFileFormatBase( protocolMetadataAdapter = ProtocolMetadataAdapterV1(protocol, metadata), nullableRowTrackingConstantFields = nullableRowTrackingConstantFields, nullableRowTrackingGeneratedFields = nullableRowTrackingGeneratedFields, optimizationsEnabled = optimizationsEnabled, tablePath = tablePath, isCDCRead = isCDCRead, // V1: capture config at construction, used in buildReaderWithPartitionValues useMetadataRowIndexOpt = SparkSession.getActiveSession.map( _.sessionState.conf.getConf(DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX))) { /** * We sometimes need to replace FileFormat within LogicalPlans, so we have to override * `equals` to ensure file format changes are captured */ override def equals(other: Any): Boolean = { other match { case ff: DeltaParquetFileFormat => ff.columnMappingMode == columnMappingMode && ff.referenceSchema == referenceSchema && ff.nullableRowTrackingConstantFields == nullableRowTrackingConstantFields && ff.optimizationsEnabled == optimizationsEnabled case _ => false } } override def hashCode(): Int = getClass.getCanonicalName.hashCode() def copyWithDVInfo( tablePath: String, optimizationsEnabled: Boolean): DeltaParquetFileFormat = { // When predicate pushdown is enabled we allow both splits and predicate pushdown. this.copy( optimizationsEnabled = optimizationsEnabled, tablePath = Some(tablePath)) } } object DeltaParquetFileFormat { /** * Column name used to identify whether the row read from the parquet file is marked * as deleted according to the Delta table deletion vectors */ val IS_ROW_DELETED_COLUMN_NAME = "__delta_internal_is_row_deleted" val IS_ROW_DELETED_STRUCT_FIELD = StructField(IS_ROW_DELETED_COLUMN_NAME, ByteType) /** Row index for each column */ val ROW_INDEX_COLUMN_NAME = "__delta_internal_row_index" val ROW_INDEX_STRUCT_FIELD = StructField(ROW_INDEX_COLUMN_NAME, LongType) /** The key to the encoded row index filter identifier value of the * [[PartitionedFile]]'s otherConstantMetadataColumnValues map. */ val FILE_ROW_INDEX_FILTER_ID_ENCODED = "row_index_filter_id_encoded" /** The key to the row index filter type value of the * [[PartitionedFile]]'s otherConstantMetadataColumnValues map. */ val FILE_ROW_INDEX_FILTER_TYPE = "row_index_filter_type" /** Utility method to create a new writable vector */ private[delta] def newVector( useOffHeapBuffers: Boolean, size: Int, dataType: StructField): WritableColumnVector = { if (useOffHeapBuffers) { OffHeapColumnVector.allocateColumns(size, Seq(dataType).toArray)(0) } else { OnHeapColumnVector.allocateColumns(size, Seq(dataType).toArray)(0) } } /** Try the operation, if the operation fails release the created resource */ private[delta] def trySafely[R <: WritableColumnVector, T]( useOffHeapBuffers: Boolean, size: Int, columns: Seq[ColumnMetadata])(f: Seq[WritableColumnVector] => T): T = { val resources = new ArrayBuffer[WritableColumnVector](columns.size) try { columns.foreach(col => resources.append(newVector(useOffHeapBuffers, size, col.structField))) f(resources.toSeq) } catch { case NonFatal(e) => resources.foreach(closeQuietly(_)) throw e } } /** Utility method to quietly close an [[AutoCloseable]] */ private[delta] def closeQuietly(closeable: AutoCloseable): Unit = { if (closeable != null) { try { closeable.close() } catch { case NonFatal(_) => // ignore } } } /** * Helper method to replace the vectors in given [[ColumnarBatch]]. * New vectors and its index in the batch are given as tuples. */ private[delta] def replaceVectors( batch: ColumnarBatch, indexVectorTuples: (Int, ColumnVector) *): ColumnarBatch = { val vectors = ArrayBuffer[ColumnVector]() for (i <- 0 until batch.numCols()) { var replaced: Boolean = false for (indexVectorTuple <- indexVectorTuples) { val index = indexVectorTuple._1 val vector = indexVectorTuple._2 if (indexVectorTuple._1 == i) { vectors += indexVectorTuple._2 // Make sure to close the existing vector allocated in the Parquet batch.column(i).close() replaced = true } } if (!replaced) { vectors += batch.column(i) } } new ColumnarBatch(vectors.toArray, batch.numRows()) } /** Helper class to encapsulate column info */ case class ColumnMetadata(index: Int, structField: StructField) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaParquetWriteSupport.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.util.Try import org.apache.spark.sql.delta.DeltaColumnMapping._ import org.apache.hadoop.conf.Configuration import org.apache.parquet.hadoop.api.WriteSupport.WriteContext import org.apache.parquet.schema.{LogicalTypeAnnotation, Type, Types} import org.apache.parquet.schema.LogicalTypeAnnotation.{ListLogicalTypeAnnotation, MapLogicalTypeAnnotation} import org.apache.spark.SparkRuntimeException import org.apache.spark.sql.catalyst.parser.LegacyTypeStringParser import org.apache.spark.sql.catalyst.trees.Origin import org.apache.spark.sql.errors.QueryCompilationErrors import org.apache.spark.sql.execution.datasources.parquet.{ParquetSchemaConverter, ParquetWriteSupport} import org.apache.spark.sql.types.{DataType, StructField, StructType} class DeltaParquetWriteSupport extends ParquetWriteSupport { private def getNestedFieldId(field: StructField, path: Seq[String]): Int = { field.metadata .getMetadata(PARQUET_FIELD_NESTED_IDS_METADATA_KEY) .getLong(path.mkString(".")) .toInt } private def findFieldInSparkSchema(schema: StructType, path: Seq[String]): StructField = { schema.findNestedField(path, true) match { case Some((_, field)) => field case None => throw QueryCompilationErrors.invalidFieldName(Seq(path.head), path, Origin()) } } override def init(configuration: Configuration): WriteContext = { val writeContext = super.init(configuration) // Parse the Spark schema. This is the same as is done in super.init, however, the // parsed schema is stored in [[ParquetWriteSupport.schema]], which is private so // we can't access it here and need to parse it again. val schemaString = configuration.get(ParquetWriteSupport.SPARK_ROW_SCHEMA) // This code is copied from Spark StructType.fromString because it is not accessible here val parsedSchema = Try(DataType.fromJson(schemaString)).getOrElse( LegacyTypeStringParser.parseString(schemaString)) match { case t: StructType => t case _ => // This code is copied from DataTypeErrors.failedParsingStructTypeError because // it is not accessible here throw new SparkRuntimeException( errorClass = "FAILED_PARSE_STRUCT_TYPE", messageParameters = Map("raw" -> s"'$schemaString'")) } val messageType = writeContext.getSchema val newMessageTypeBuilder = Types.buildMessage() messageType.getFields.forEach { field => val parentField = findFieldInSparkSchema(parsedSchema, Seq(field.getName)) newMessageTypeBuilder.addField(convert( field, parentField, parsedSchema, Seq(field.getName), Seq(field.getName))) } val newMessageType = newMessageTypeBuilder.named( ParquetSchemaConverter.SPARK_PARQUET_SCHEMA_NAME) new WriteContext(newMessageType, writeContext.getExtraMetaData) } /** * Recursively rewrites the parquet [[Type]] by adding the nested field * IDs to list and map subtypes as defined in the schema. The * recursion needs to keep track of the absolute field path in order * to correctly identify the StructField in the spark schema for a * corresponding parquet field. As nested field IDs are referenced * by their relative path in a field's metadata, the recursion also needs * to keep track of the relative path. * * For example, consider the following column type * col1 STRUCT(a INT, b STRUCT(c INT, d ARRAY(INT))) * * The absolute path to the nested [[element]] field of the list is * col1.b.d.element whereas the relative path is d.element, i.e. relative * to the parent struct field. */ private def convert( field: Type, parentField: StructField, sparkSchema: StructType, absolutePath: Seq[String], relativePath: Seq[String]): Type = { field.getLogicalTypeAnnotation match { case _: ListLogicalTypeAnnotation => val relElemFieldPath = relativePath :+ PARQUET_LIST_ELEMENT_FIELD_NAME val id = getNestedFieldId(parentField, relElemFieldPath) val elementField = field.asGroupType().getFields.get(0).asGroupType().getFields.get(0).withId(id) val builder = Types .buildGroup(field.getRepetition).as(LogicalTypeAnnotation.listType()) .addField( Types.repeatedGroup() .addField(convert(elementField, parentField, sparkSchema, absolutePath :+ PARQUET_LIST_ELEMENT_FIELD_NAME, relElemFieldPath)) .named("list")) if (field.getId != null) { builder.id(field.getId.intValue()) } builder.named(field.getName) case _: MapLogicalTypeAnnotation => val relKeyFieldPath = relativePath :+ PARQUET_MAP_KEY_FIELD_NAME val relValFieldPath = relativePath :+ PARQUET_MAP_VALUE_FIELD_NAME val keyId = getNestedFieldId(parentField, relKeyFieldPath) val valId = getNestedFieldId(parentField, relValFieldPath) val keyField = field.asGroupType().getFields.get(0).asGroupType().getFields.get(0).withId(keyId) val valueField = field.asGroupType().getFields.get(0).asGroupType().getFields.get(1).withId(valId) val builder = Types .buildGroup(field.getRepetition).as(LogicalTypeAnnotation.mapType()) .addField( Types .repeatedGroup() .addField(convert(keyField, parentField, sparkSchema, absolutePath :+ PARQUET_MAP_KEY_FIELD_NAME, relKeyFieldPath)) .addField(convert(valueField, parentField, sparkSchema, absolutePath :+ PARQUET_MAP_VALUE_FIELD_NAME, relValFieldPath)) .named("key_value")) if (field.getId != null) { builder.id(field.getId.intValue()) } builder.named(field.getName) case _ if field.isPrimitive => field case _ => val builder = Types.buildGroup(field.getRepetition) field.asGroupType().getFields.forEach { field => val absPath = absolutePath :+ field.getName val parentField = findFieldInSparkSchema(sparkSchema, absPath) builder.addField(convert(field, parentField, sparkSchema, absPath, Seq(field.getName))) } if (field.getId != null) { builder.id(field.getId.intValue()) } builder.named(field.getName) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaSharedExceptions.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.collection.JavaConverters._ import org.antlr.v4.runtime.ParserRuleContext import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.parser.{DeltaParseException, ParseException, ParseExceptionShims, ParserUtils} import org.apache.spark.sql.catalyst.trees.Origin import org.apache.spark.QueryContext class DeltaAnalysisException( errorClass: String, messageParameters: Array[String], cause: Option[Throwable] = None, origin: Option[Origin] = None, precomputedMessage: Option[String] = None, precomputedMessageParametersMap: Option[java.util.Map[String, String]] = None) extends AnalysisException( message = precomputedMessage.getOrElse( DeltaThrowableHelper.getMessage(errorClass, messageParameters)), messageParameters = precomputedMessageParametersMap.getOrElse(DeltaThrowableHelper .getMessageParameters(errorClass, errorSubClass = null, messageParameters)).asScala.toMap, errorClass = Some(errorClass), line = origin.flatMap(_.line), startPosition = origin.flatMap(_.startPosition), context = origin.map(_.getQueryContext).getOrElse(Array.empty), cause = cause) with DeltaThrowable { def getMessageParametersArray: Array[String] = messageParameters override def getErrorClass: String = errorClass override def getMessageParameters: java.util.Map[String, String] = precomputedMessageParametersMap.getOrElse( DeltaThrowableHelper.getMessageParameters(errorClass, errorSubClass = null, messageParameters) ) override def withPosition(origin: Origin): AnalysisException = new DeltaAnalysisException(errorClass, messageParameters, cause, Some(origin), precomputedMessage, precomputedMessageParametersMap) } class DeltaAnalysisExceptionWithSubErrors( errorClass: String, messageParameters: Array[String], subErrors: Seq[(String, Array[String])]) extends DeltaAnalysisException( errorClass = errorClass, messageParameters = messageParameters, cause = None, origin = None, precomputedMessage = Some( DeltaThrowableHelper.getMessageWithSubErrors(errorClass, messageParameters, subErrors)), precomputedMessageParametersMap = Some( DeltaThrowableHelper.getMainErrorMessageParameters(errorClass, messageParameters)) ) class DeltaIllegalArgumentException( errorClass: String, messageParameters: Array[String] = Array.empty, cause: Throwable = null) extends IllegalArgumentException( DeltaThrowableHelper.getMessage(errorClass, messageParameters), cause) with DeltaThrowable { override def getErrorClass: String = errorClass def getMessageParametersArray: Array[String] = messageParameters override def getMessageParameters: java.util.Map[String, String] = { DeltaThrowableHelper.getMessageParameters(errorClass, errorSubClass = null, messageParameters) } override def getQueryContext: Array[QueryContext] = new Array(0); } class DeltaUnsupportedOperationException( errorClass: String, messageParameters: Array[String] = Array.empty) extends UnsupportedOperationException( DeltaThrowableHelper.getMessage(errorClass, messageParameters)) with DeltaThrowable { override def getErrorClass: String = errorClass def getMessageParametersArray: Array[String] = messageParameters override def getMessageParameters: java.util.Map[String, String] = { DeltaThrowableHelper.getMessageParameters(errorClass, errorSubClass = null, messageParameters) } override def getQueryContext: Array[QueryContext] = new Array(0); } // DeltaParseException is now defined in ParseExceptionShims // (see scala-spark-*/shims/ParseExceptionShims.scala) to handle the different ParseException // constructor signatures between Spark versions. // In Spark 4.1, ParseException removed the 'stop' parameter from its constructor. class DeltaArithmeticException( errorClass: String, messageParameters: Array[String]) extends ArithmeticException( DeltaThrowableHelper.getMessage(errorClass, messageParameters)) with DeltaThrowable { override def getErrorClass: String = errorClass override def getMessageParameters: java.util.Map[String, String] = { DeltaThrowableHelper.getMessageParameters(errorClass, errorSubClass = null, messageParameters) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaTable.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import scala.util.{Failure, Success, Try} import scala.util.control.NonFatal import org.apache.spark.sql.delta.files.{TahoeFileIndex, TahoeLogFileIndex} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.{DeltaLogging, LogThrottler} import org.apache.spark.sql.delta.skipping.clustering.temp.{ClusterByTransform => TempClusterByTransform} import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf} import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.spark.internal.{Logging, MDC} import org.apache.spark.sql.{Column, DataFrame, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.{NoSuchTableException, UnresolvedLeafNode, UnresolvedTable} import org.apache.spark.sql.catalyst.catalog.{CatalogTable, SessionCatalog} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke import org.apache.spark.sql.catalyst.planning.NodeWithOnlyDeterministicProjectAndFilter import org.apache.spark.sql.catalyst.plans.logical.{Filter, LeafNode, LogicalPlan, Project} import org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.connector.expressions.{FieldReference, IdentityTransform} import org.apache.spark.sql.execution.datasources.{FileFormat, FileIndex, HadoopFsRelation, LogicalRelation, LogicalRelationWithTable} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.spark.sql.util.CaseInsensitiveStringMap /** * Extractor Object for pulling out the file index of a logical relation. */ object RelationFileIndex { def unapply(a: LogicalRelation): Option[FileIndex] = a match { case LogicalRelationWithTable(hrel: HadoopFsRelation, _) => Some(hrel.location) case _ => None } } /** * Extractor Object for pulling out the table scan of a Delta table. It could be a full scan * or a partial scan. */ object DeltaTable { def unapply(a: LogicalRelation): Option[TahoeFileIndex] = a match { case RelationFileIndex(fileIndex: TahoeFileIndex) => Some(fileIndex) case _ => None } } /** * Extractor Object for pulling out the full table scan of a Delta table. */ object DeltaFullTable { def unapply(a: LogicalPlan): Option[(LogicalRelation, TahoeLogFileIndex)] = a match { // `DeltaFullTable` is not only used to match a certain query pattern, but also does // some validations to throw errors. We need to match both Project and Filter here, // so that we can check if Filter is present or not during validations. case NodeWithOnlyDeterministicProjectAndFilter(lr @ DeltaTable(index: TahoeLogFileIndex)) => if (!index.deltaLog.tableExists) return None val hasFilter = a.find(_.isInstanceOf[Filter]).isDefined if (index.partitionFilters.isEmpty && index.versionToUse.isEmpty && !hasFilter) { Some(lr -> index) } else if (index.versionToUse.nonEmpty) { throw DeltaErrors.failedScanWithHistoricalVersion(index.versionToUse.get) } else { throw DeltaErrors.unexpectedPartialScan(index.path) } // Convert V2 relations to V1 and perform the check case DeltaRelation(lr) => unapply(lr) case _ => None } } object DeltaTableUtils extends PredicateHelper with DeltaLogging { // The valid hadoop prefixes passed through `DeltaTable.forPath` or DataFrame APIs. val validDeltaTableHadoopPrefixes: List[String] = List("fs.", "dfs.") /** Check whether this table is a Delta table based on information from the Catalog. */ def isDeltaTable(table: CatalogTable): Boolean = DeltaSourceUtils.isDeltaTable(table.provider) /** * Check whether the provided table name is a Delta table based on information from the Catalog. */ def isDeltaTable(spark: SparkSession, tableName: TableIdentifier): Boolean = { val catalog = spark.sessionState.catalog val tableIsNotTemporaryTable = !catalog.isTempView(tableName) val tableExists = { (tableName.database.isEmpty || catalog.databaseExists(tableName.database.get)) && catalog.tableExists(tableName) } tableIsNotTemporaryTable && tableExists && isDeltaTable(catalog.getTableMetadata(tableName)) } /** Check if the provided path is the root or the children of a Delta table. */ def isDeltaTable( spark: SparkSession, path: Path, options: Map[String, String] = Map.empty): Boolean = { findDeltaTableRoot(spark, path, options).isDefined } /** * Checks whether TableIdentifier is a path or a table name * We assume it is a path unless the table and database both exist in the catalog * @param catalog session catalog used to check whether db/table exist * @param tableIdent the provided table or path * @return true if using table name, false if using path, error otherwise */ def isCatalogTable(catalog: SessionCatalog, tableIdent: TableIdentifier): Boolean = { val (dbExists, assumePath) = dbExistsAndAssumePath(catalog, tableIdent) // If we don't need to check that the table exists, return false since we think the tableIdent // refers to a path at this point, because the database doesn't exist if (assumePath) return false // check for dbexists otherwise catalog.tableExists may throw NoSuchDatabaseException if ((dbExists || tableIdent.database.isEmpty) && Try(catalog.tableExists(tableIdent)).getOrElse(false)) { true } else if (isValidPath(tableIdent)) { false } else { throw new NoSuchTableException(tableIdent.database.getOrElse(""), tableIdent.table) } } /** * It's possible that checking whether database exists can throw an exception. In that case, * we want to surface the exception only if the provided tableIdentifier cannot be a path. * * @param catalog session catalog used to check whether db/table exist * @param ident the provided table or path * @return tuple where first indicates whether database exists and second indicates whether there * is a need to check whether table exists */ private def dbExistsAndAssumePath( catalog: SessionCatalog, ident: TableIdentifier): (Boolean, Boolean) = { def databaseExists = { ident.database.forall(catalog.databaseExists) } Try(databaseExists) match { // DB exists, check table exists only if path is not valid case Success(true) => (true, false) // DB does not exist, check table exists only if path does not exist case Success(false) => (false, new Path(ident.table).isAbsolute) // Checking DB exists threw exception, if the path is still valid then check for table exists case Failure(_) if isValidPath(ident) => (false, true) // Checking DB exists threw exception, path is not valid so throw the initial exception case Failure(e) => throw e } } /** * @param tableIdent the provided table or path * @return whether or not the provided TableIdentifier can specify a path for parquet or delta */ def isValidPath(tableIdent: TableIdentifier): Boolean = { // If db doesnt exist or db is called delta/tahoe then check if path exists DeltaSourceUtils.isDeltaDataSourceName(tableIdent.database.getOrElse("")) && new Path(tableIdent.table).isAbsolute } /** Find the root of a Delta table from the provided path. */ def findDeltaTableRoot( spark: SparkSession, path: Path, options: Map[String, String] = Map.empty): Option[Path] = { // scalastyle:off deltahadoopconfiguration val fs = path.getFileSystem(spark.sessionState.newHadoopConfWithOptions(options)) // scalastyle:on deltahadoopconfiguration findDeltaTableRoot(fs, path) } /** Finds the root of a Delta table given a path if it exists. */ def findDeltaTableRoot(fs: FileSystem, path: Path): Option[Path] = { findDeltaTableRoot( fs, path, throwOnError = SparkSession.active.conf.get(DeltaSQLConf.DELTA_IS_DELTA_TABLE_THROW_ON_ERROR)) } /** Finds the root of a Delta table given a path if it exists. */ private[delta] def findDeltaTableRoot( fs: FileSystem, path: Path, throwOnError: Boolean): Option[Path] = { if (throwOnError) { findDeltaTableRootThrowOnError(fs, path) } else { findDeltaTableRootNoExceptions(fs, path) } } /** * Finds the root of a Delta table given a path if it exists. * * Does not throw any exceptions, but returns `None` when uncertain (old behaviour). */ private def findDeltaTableRootNoExceptions(fs: FileSystem, path: Path): Option[Path] = { var currentPath = path while (currentPath != null && currentPath.getName != "_delta_log" && currentPath.getName != "_samples") { val deltaLogPath = safeConcatPaths(currentPath, "_delta_log") if (Try(fs.exists(deltaLogPath)).getOrElse(false)) { return Option(currentPath) } currentPath = currentPath.getParent } None } /** * Finds the root of a Delta table given a path if it exists. * * If there are errors and no root could be found, throw the first error (new behaviour) */ private def findDeltaTableRootThrowOnError(fs: FileSystem, path: Path): Option[Path] = { var firstError: Option[Throwable] = None // Return `None` if `firstError` is empty, throw `firstError` otherwise. def noneOrError(): Option[Path] = { firstError match { case Some(ex) => throw ex case None => None } } var currentPath = path while (currentPath != null && currentPath.getName != "_delta_log" && currentPath.getName != "_samples") { val deltaLogPath = safeConcatPaths(currentPath, "_delta_log") try { if (fs.exists(deltaLogPath)) { return Option(currentPath) } } catch { case NonFatal(ex) if currentPath == path => // Store errors for the first path, but keep going up the hierarchy, // in case the error at this level does not matter and the delta log is found at a parent. firstError = Some(ex) case NonFatal(ex) => // If we find errors higher up the path we either treat it as a non-Delta table or // return the error we found at the original path, if any. // This gives us best-effort detection of delta logs in the hierarchy, but with more // useful error messages when access was actually missing. logThrottler.throttledWithSkippedLogMessage { skippedStr => logWarning(log"Access error while exploring path hierarchy for a delta log." + log"original path=${MDC(DeltaLogKeys.PATH, path)}, " + log"path with error=${MDC(DeltaLogKeys.PATH2, currentPath)}." + skippedStr, ex) } return noneOrError() } currentPath = currentPath.getParent } noneOrError() } private val logThrottler = new LogThrottler() /** Whether a path should be hidden for delta-related file operations, such as Vacuum and Fsck. */ def isHiddenDirectory( partitionColumnNames: Seq[String], pathName: String, shouldIcebergMetadataDirBeHidden: Boolean = true): Boolean = { // Names of the form partitionCol=[value] are partition directories, and should be // GCed even if they'd normally be hidden. The _db_index directory contains (bloom filter) // indexes and these must be GCed when the data they are tied to is GCed. // metadata name is reserved for converted iceberg metadata with delta universal format (shouldIcebergMetadataDirBeHidden && pathName.equals("metadata")) || (pathName.startsWith(".") || pathName.startsWith("_")) && !pathName.startsWith("_delta_index") && !pathName.startsWith("_change_data") && !partitionColumnNames.exists(c => pathName.startsWith(c ++ "=")) } /** * Does the predicate only contains partition columns? */ def isPredicatePartitionColumnsOnly( condition: Expression, partitionColumns: Seq[String], spark: SparkSession): Boolean = { val nameEquality = spark.sessionState.analyzer.resolver condition.references.forall { r => partitionColumns.exists(nameEquality(r.name, _)) } } /** * Partition the given condition into two sequence of conjunctive predicates: * - predicates that can be evaluated using metadata only. * - other predicates. */ def splitMetadataAndDataPredicates( condition: Expression, partitionColumns: Seq[String], spark: SparkSession): (Seq[Expression], Seq[Expression]) = { val (metadataPredicates, dataPredicates) = splitConjunctivePredicates(condition).partition( isPredicateMetadataOnly(_, partitionColumns, spark)) // Extra metadata predicates that can partially extracted from `dataPredicates`. val extraMetadataPredicates = if (dataPredicates.nonEmpty) { extractMetadataPredicates(dataPredicates.reduce(And), partitionColumns, spark) .map(splitConjunctivePredicates) .getOrElse(Seq.empty) } else { Seq.empty } (metadataPredicates ++ extraMetadataPredicates, dataPredicates) } /** * Returns a predicate that its reference is a subset of `partitionColumns` and it contains the * maximum constraints from `condition`. * When there is no such filter, `None` is returned. */ private def extractMetadataPredicates( condition: Expression, partitionColumns: Seq[String], spark: SparkSession): Option[Expression] = { condition match { case And(left, right) => val lhs = extractMetadataPredicates(left, partitionColumns, spark) val rhs = extractMetadataPredicates(right, partitionColumns, spark) (lhs.toSeq ++ rhs.toSeq).reduceOption(And) // The Or predicate is convertible when both of its children can be pushed down. // That is to say, if one/both of the children can be partially pushed down, the Or // predicate can be partially pushed down as well. // // Here is an example used to explain the reason. // Let's say we have // condition: (a1 AND a2) OR (b1 AND b2), // outputSet: AttributeSet(a1, b1) // a1 and b1 is convertible, while a2 and b2 is not. // The predicate can be converted as // (a1 OR b1) AND (a1 OR b2) AND (a2 OR b1) AND (a2 OR b2) // As per the logical in And predicate, we can push down (a1 OR b1). case Or(left, right) => for { lhs <- extractMetadataPredicates(left, partitionColumns, spark) rhs <- extractMetadataPredicates(right, partitionColumns, spark) } yield Or(lhs, rhs) // Here we assume all the `Not` operators is already below all the `And` and `Or` operators // after the optimization rule `BooleanSimplification`, so that we don't need to handle the // `Not` operators here. case other => if (isPredicatePartitionColumnsOnly(other, partitionColumns, spark)) { Some(other) } else { None } } } /** * Check if condition involves a subquery expression. */ def containsSubquery(condition: Expression): Boolean = { SubqueryExpression.hasSubquery(condition) } /** * Check if condition can be evaluated using only metadata. In Delta, this means the condition * only references partition columns and involves no subquery. */ def isPredicateMetadataOnly( condition: Expression, partitionColumns: Seq[String], spark: SparkSession): Boolean = { isPredicatePartitionColumnsOnly(condition, partitionColumns, spark) && !containsSubquery(condition) } /** * Replace the file index in a logical plan and return the updated plan. * It's a common pattern that, in Delta commands, we use data skipping to determine a subset of * files that can be affected by the command, so we replace the whole-table file index in the * original logical plan with a new index of potentially affected files, while everything else in * the original plan, e.g., resolved references, remain unchanged. * * @param target the logical plan in which we replace the file index * @param fileIndex the new file index */ def replaceFileIndex( target: LogicalPlan, fileIndex: FileIndex): LogicalPlan = { target transform { case l @ LogicalRelationWithTable(hfsr: HadoopFsRelation, _) => l.copy(relation = hfsr.copy(location = fileIndex)(hfsr.sparkSession)) } } /** * Transform the file format in a logical plan and return the updated plan. * * @param target the logical plan in which the file format is replaced. * @param rule the rule to apply to the file format. */ def transformFileFormat( target: LogicalPlan)( rule: PartialFunction[DeltaParquetFileFormat, DeltaParquetFileFormat]): LogicalPlan = { target.transform { case l@LogicalRelationWithTable(hfsr: HadoopFsRelation, _) => val newFileFormat = hfsr.fileFormat match { case format: DeltaParquetFileFormat => rule.applyOrElse(format, identity[DeltaParquetFileFormat]) } l.copy(relation = hfsr.copy(fileFormat = newFileFormat)(hfsr.sparkSession)) } } /** * Many Delta meta-queries involve nondeterminstic functions, which interfere with automatic * column pruning, so columns can be manually pruned from the scan. Note that partition columns * can never be dropped even if they're not referenced in the rest of the query. * * @param spark the spark session to use * @param target the logical plan in which drop columns * @param columnsToDrop columns to drop from the scan */ def dropColumns( spark: SparkSession, target: LogicalPlan, columnsToDrop: Seq[String]): LogicalPlan = { val resolver = spark.sessionState.analyzer.resolver // Spark does char type read-side padding via an additional Project over the scan node. // When char type read-side padding is applied, we need to apply column pruning for the // Project as well, otherwise the Project will contain missing attributes. val hasChar = target.exists { case Project(projectList, _) => def hasCharPadding(e: Expression): Boolean = e.exists { case s: StaticInvoke => s.staticObject == classOf[CharVarcharCodegenUtils] && s.functionName == "readSidePadding" case _ => false } projectList.exists { case a: Alias => hasCharPadding(a.child) && a.references.size == 1 case _ => false } case _ => false } target transformUp { case l@LogicalRelationWithTable(hfsr: HadoopFsRelation, _) => // Prune columns from the scan. val prunedOutput = l.output.filterNot { col => columnsToDrop.exists(resolver(_, col.name)) } val prunedSchema = StructType(prunedOutput.map(attr => StructField(attr.name, attr.dataType, attr.nullable, attr.metadata))) val newBaseRelation = hfsr.copy(dataSchema = prunedSchema)(hfsr.sparkSession) l.copy(relation = newBaseRelation, output = prunedOutput) case p @ Project(projectList, child) if hasChar => val newProjectList = projectList.filter { e => e.references.subsetOf(child.outputSet) } p.copy(projectList = newProjectList) } } /** Finds and returns the file source metadata column from a dataframe */ def getFileMetadataColumn(df: DataFrame): Column = df.metadataColumn(FileFormat.METADATA_NAME) /** * Update FileFormat for a plan and return the updated plan * * @param target Target plan to update * @param updatedFileFormat Updated file format * @return Updated logical plan */ def replaceFileFormat( target: LogicalPlan, updatedFileFormat: FileFormat): LogicalPlan = { target transform { case l @ LogicalRelationWithTable(hfsr: HadoopFsRelation, _) => l.copy( relation = hfsr.copy(fileFormat = updatedFileFormat)(hfsr.sparkSession)) } } /** * Check if the given path contains time travel syntax with the `@`. If the path genuinely exists, * return `None`. If the path doesn't exist, but is specifying time travel, return the * `DeltaTimeTravelSpec` as well as the real path. */ def extractIfPathContainsTimeTravel( session: SparkSession, path: String, options: Map[String, String]): (String, Option[DeltaTimeTravelSpec]) = { val conf = session.sessionState.conf if (!DeltaTimeTravelSpec.isApplicable(conf, path)) return path -> None val maybePath = new Path(path) // scalastyle:off deltahadoopconfiguration val fs = maybePath.getFileSystem(session.sessionState.newHadoopConfWithOptions(options)) // scalastyle:on deltahadoopconfiguration // If the folder really exists, quit if (fs.exists(maybePath)) return path -> None val (tt, realPath) = DeltaTimeTravelSpec.resolvePath(conf, path) realPath -> Some(tt) } /** * Given a time travel node, resolve which version it is corresponding to for the given table and * return the resolved version as well as the access type, i.e. by `version` or `timestamp`. */ def resolveTimeTravelVersion( conf: SQLConf, deltaLog: DeltaLog, catalogTableOpt: Option[CatalogTable], tt: DeltaTimeTravelSpec, canReturnLastCommit: Boolean = false): (Long, String) = { if (tt.version.isDefined) { val userVersion = tt.version.get deltaLog.history.checkVersionExists(userVersion, catalogTableOpt) userVersion -> "version" } else { val timestamp = tt.getTimestamp(conf) val commit = deltaLog.history.getActiveCommitAtTime(timestamp, catalogTableOpt, canReturnLastCommit) commit.version -> "timestamp" } } def parseColToTransform(col: String): IdentityTransform = { IdentityTransform(FieldReference(Seq(col))) } def parseColsToClusterByTransform(cols: Seq[String]): TempClusterByTransform = { TempClusterByTransform(cols.map(FieldReference(_))) } // Workaround for withActive not being visible in io/delta. def withActiveSession[T](spark: SparkSession)(body: => T): T = spark.withActive(body) /** * Uses org.apache.hadoop.fs.Path.mergePaths to concatenate a base path and a relative child path. * * This method is designed to address two specific issues in Hadoop Path: * * Issue 1: * When the base path represents a Uri with an empty path component, such as concatenating * "s3://my-bucket" and "childPath". In this case, the child path is converted to an absolute * path at the root, i.e. /childPath. This prevents a "URISyntaxException: Relative path in * absolute URI", which would be thrown by org.apache.hadoop.fs.Path(Path, String) because it * tries to convert the base path to a Uri and then resolve the child on top of it. This is * invalid for an empty base path and a relative child path according to the Uri specification, * which states that if an authority is defined, the path component needs to be either empty or * start with a '/'. * * Issue 2 (only when [[DeltaSQLConf.DELTA_WORK_AROUND_COLONS_IN_HADOOP_PATHS]] is `true`): * When the child path contains a special character ':', such as "aaaa:bbbb.csv". * This is valid in many file systems such as S3, but is actually ambiguous because it can be * parsed either as an absolute path with a scheme ("aaaa") and authority ("bbbb.csv"), or as * a relative path with a colon in the name ("aaaa:bbbb.csv"). Hadoop Path will always interpret * it as the former, which is not what we want in this case. Therefore, we prepend a '/' to the * child path to ensure that it is always interpreted as a relative path. * See [[https://issues.apache.org/jira/browse/HDFS-14762]] for more details. */ def safeConcatPaths(basePath: Path, relativeChildPath: String): Path = { val useWorkaround = SparkSession.getActiveSession.map(_.sessionState.conf) .exists(_.getConf(DeltaSQLConf.DELTA_WORK_AROUND_COLONS_IN_HADOOP_PATHS)) if (useWorkaround) { Path.mergePaths(basePath, new Path(s"/$relativeChildPath")) } else { if (basePath.toUri.getPath.isEmpty) { new Path(basePath, s"/$relativeChildPath") } else { new Path(basePath, relativeChildPath) } } } /** * A list of Spark internal metadata keys that we may save in a Delta table schema * unintentionally due to SPARK-43123. We need to remove them before handing over the schema to * Spark to avoid Spark interpreting table columns incorrectly. * * Hard-coded strings are used intentionally as we want to capture possible keys used before * SPARK-43123 regardless Spark versions. For example, if Spark changes any key string in future * after SPARK-43123, the new string won't be leaked, but we still want to clean up the old key. */ val SPARK_INTERNAL_METADATA_KEYS = Seq( "__autoGeneratedAlias", "__metadata_col", "__supports_qualified_star", // A key used by an old version. Doesn't exist in latest code "__qualified_access_only", "__file_source_metadata_col", "__file_source_constant_metadata_col", "__file_source_generated_metadata_col" ) /** * Remove leaked metadata keys from the persisted table schema. Old versions might leak metadata * intentionally. This method removes all possible metadata keys to avoid Spark interpreting * table columns incorrectly. */ def removeSparkInternalMetadata(spark: SparkSession, schema: StructType): StructType = { if (spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_SCHEMA_REMOVE_SPARK_INTERNAL_METADATA)) { var updated = false val updatedSchema = schema.map { field => if (SPARK_INTERNAL_METADATA_KEYS.exists(field.metadata.contains)) { updated = true val newMetadata = new MetadataBuilder().withMetadata(field.metadata) SPARK_INTERNAL_METADATA_KEYS.foreach(newMetadata.remove) field.copy(metadata = newMetadata.build()) } else { field } } if (updated) { StructType(updatedSchema) } else { schema } } else { schema } } /** * Removes from the given schema all the metadata keys that are not used when reading a Delta * table. This includes typically all metadata used by writer-only table features. * Note that this also removes all leaked Spark internal metadata. */ def removeInternalWriterMetadata(spark: SparkSession, schema: StructType): StructType = { ColumnWithDefaultExprUtils.removeDefaultExpressions( removeSparkInternalMetadata(spark, schema) ) } /** * Removes internal Delta metadata from the given schema. This includes tyically metadata used by * reader-writer table features that shouldn't leak outside of the table. Use * [[removeInternalWriterMetadata]] in addition / instead to remove metadata for writer-only table * features. */ def removeInternalDeltaMetadata(spark: SparkSession, schema: StructType): StructType = { val cleanedSchema = DeltaColumnMapping.dropColumnMappingMetadata(schema) val conf = spark.sessionState.conf if (conf.getConf(DeltaSQLConf.DELTA_TYPE_WIDENING_REMOVE_SCHEMA_METADATA)) { TypeWideningMetadata.removeTypeWideningMetadata(cleanedSchema)._1 } else { cleanedSchema } } } sealed abstract class UnresolvedPathBasedDeltaTableBase(path: String) extends UnresolvedLeafNode { def identifier: Identifier = Identifier.of(Array(DeltaSourceUtils.ALT_NAME), path) def deltaTableIdentifier: DeltaTableIdentifier = DeltaTableIdentifier(Some(path), None) } /** Resolves to a [[ResolvedTable]] if the DeltaTable exists */ case class UnresolvedPathBasedDeltaTable( path: String, options: Map[String, String], commandName: String) extends UnresolvedPathBasedDeltaTableBase(path) /** Resolves to a [[DataSourceV2Relation]] if the DeltaTable exists */ case class UnresolvedPathBasedDeltaTableRelation( path: String, options: CaseInsensitiveStringMap) extends UnresolvedPathBasedDeltaTableBase(path) /** * This operator represents path-based tables in general including both Delta or non-Delta tables. * It resolves to a [[ResolvedTable]] if the path is for delta table, * [[ResolvedPathBasedNonDeltaTable]] if the path is for a non-Delta table. */ case class UnresolvedPathBasedTable( path: String, options: Map[String, String], commandName: String) extends LeafNode { override lazy val resolved: Boolean = false override val output: Seq[Attribute] = Nil } /** * This operator is a placeholder that identifies a non-Delta path-based table. Given the fact * that some Delta commands (e.g. DescribeDeltaDetail) support non-Delta table, we introduced * ResolvedPathBasedNonDeltaTable as the resolved placeholder after analysis on a non delta path * from UnresolvedPathBasedTable. */ case class ResolvedPathBasedNonDeltaTable( path: String, options: Map[String, String], commandName: String) extends LeafNode { override val output: Seq[Attribute] = Nil } /** * A helper object with an apply method to transform a path or table identifier to a LogicalPlan. * If the path is set, it will be resolved to an [[UnresolvedPathBasedDeltaTable]] whereas if the * tableIdentifier is set, the LogicalPlan will be an [[UnresolvedTable]]. If neither of the two * options or both of them are set, [[apply]] will throw an exception. */ object UnresolvedDeltaPathOrIdentifier { def apply( path: Option[String], tableIdentifier: Option[TableIdentifier], options: Map[String, String], cmd: String): LogicalPlan = { (path, tableIdentifier) match { case (Some(p), None) => UnresolvedPathBasedDeltaTable(p, options, cmd) case (None, Some(t)) => UnresolvedTable(t.nameParts, cmd) case _ => throw new IllegalArgumentException( s"Exactly one of path or tableIdentifier must be provided to $cmd") } } def apply( path: Option[String], tableIdentifier: Option[TableIdentifier], cmd: String): LogicalPlan = this(path, tableIdentifier, Map.empty, cmd) } /** * A helper object with an apply method to transform a path or table identifier to a LogicalPlan. * This is required by Delta commands that can also run against non-Delta tables, e.g. DESC DETAIL, * VACUUM command. If the tableIdentifier is set, the LogicalPlan will be an [[UnresolvedTable]]. * If the tableIdentifier is not set but the path is set, it will be resolved to an * [[UnresolvedPathBasedTable]] since we can not tell if the path is for delta table or non delta * table at this stage. If neither of the two are set, throws an exception. */ object UnresolvedPathOrIdentifier { def apply( path: Option[String], tableIdentifier: Option[TableIdentifier], options: Map[String, String], cmd: String): LogicalPlan = { (path, tableIdentifier) match { case (_, Some(t)) => UnresolvedTable(t.nameParts, cmd) case (Some(p), None) => UnresolvedPathBasedTable(p, options, cmd) case _ => throw new IllegalArgumentException( s"At least one of path or tableIdentifier must be provided to $cmd") } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaTableIdentifier.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSourceUtils import org.apache.hadoop.fs.Path import org.apache.spark.internal.MDC import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier // scalastyle:on import.ordering.noEmptyLine /** * An identifier for a Delta table containing one of the path or the table identifier. */ case class DeltaTableIdentifier( path: Option[String] = None, table: Option[TableIdentifier] = None) { assert(path.isDefined ^ table.isDefined, "Please provide one of the path or the table identifier") val identifier: String = path.getOrElse(table.get.identifier) def database: Option[String] = table.flatMap(_.database) def getPath(spark: SparkSession): Path = { path.map(new Path(_)).getOrElse { val metadata = spark.sessionState.catalog.getTableMetadata(table.get) new Path(metadata.location) } } /** * Escapes back-ticks within the identifier name with double-back-ticks. */ private def quoteIdentifier(name: String): String = name.replace("`", "``") def quotedString: String = { val replacedId = quoteIdentifier(identifier) val replacedDb = database.map(quoteIdentifier) if (replacedDb.isDefined) s"`${replacedDb.get}`.`$replacedId`" else s"`$replacedId`" } def unquotedString: String = { if (database.isDefined) s"${database.get}.$identifier" else identifier } override def toString: String = quotedString } /** * Utilities for DeltaTableIdentifier. * TODO(burak): Get rid of these utilities. DeltaCatalog should be the skinny-waist for figuring * these things out. */ object DeltaTableIdentifier extends DeltaLogging { /** * Check the specified table identifier represents a Delta path. */ def isDeltaPath(spark: SparkSession, identifier: TableIdentifier): Boolean = { val catalog = spark.sessionState.catalog def tableIsTemporaryTable = catalog.isTempView(identifier) def tableExists: Boolean = { try { catalog.databaseExists(identifier.database.get) && catalog.tableExists(identifier) } catch { case e: AnalysisException if gluePermissionError(e) => logWarning(log"Received an access denied error from Glue. Will check to see if this " + log"identifier (${MDC(DeltaLogKeys.TABLE_NAME, identifier)}) is path based.", e) false } } spark.sessionState.conf.runSQLonFile && new Path(identifier.table).isAbsolute && DeltaSourceUtils.isDeltaTable(identifier.database) && !tableIsTemporaryTable && !tableExists } /** * Creates a [[DeltaTableIdentifier]] if the specified table identifier represents a Delta table, * otherwise returns [[None]]. */ def apply(spark: SparkSession, identifier: TableIdentifier) : Option[DeltaTableIdentifier] = recordFrameProfile( "DeltaAnalysis", "DeltaTableIdentifier.resolve") { if (isDeltaPath(spark, identifier)) { Some(DeltaTableIdentifier(path = Option(identifier.table))) } else if (DeltaTableUtils.isDeltaTable(spark, identifier)) { Some(DeltaTableIdentifier(table = Option(identifier))) } else { None } } /** * When users try to access Delta tables by path, e.g. delta.`/some/path`, we need to first check * if such a table exists in the MetaStore (due to Spark semantics :/). The Glue MetaStore may * return Access Denied errors during this check. This method matches on this failure mode. */ def gluePermissionError(e: AnalysisException): Boolean = e.getCause match { case h: Exception if h.getClass.getName == "org.apache.hadoop.hive.ql.metadata.HiveException" => Seq("AWSGlue", "AccessDeniedException").forall { kw => h.getMessage.contains(kw) } case _ => false } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaTableValueFunctions.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.text.SimpleDateFormat import java.util.{Date, Locale} import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.sources.DeltaDataSource import org.apache.spark.SparkException import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.analysis.{FunctionRegistryBase, NamedRelation, TableFunctionRegistry, UnresolvedLeafNode, UnresolvedRelation} import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, ExpressionInfo, Literal, StringLiteral} import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, UnaryNode} import org.apache.spark.sql.connector.catalog.V1Table import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, DataSourceV2RelationShim} import org.apache.spark.sql.types.{IntegerType, LongType, StringType, TimestampType} import org.apache.spark.sql.util.CaseInsensitiveStringMap /** * Resolve Delta specific table-value functions. */ object DeltaTableValueFunctions { val CDC_NAME_BASED = "table_changes" val CDC_PATH_BASED = "table_changes_by_path" val supportedFnNames = Seq(CDC_NAME_BASED, CDC_PATH_BASED) // For use with SparkSessionExtensions type TableFunctionDescription = (FunctionIdentifier, ExpressionInfo, TableFunctionRegistry.TableFunctionBuilder) /** * For a supported Delta table value function name, get the TableFunctionDescription to be * injected in DeltaSparkSessionExtension */ def getTableValueFunctionInjection(fnName: String): TableFunctionDescription = { val (info, builder) = fnName match { case CDC_NAME_BASED => FunctionRegistryBase.build[CDCNameBased](fnName, since = None) case CDC_PATH_BASED => FunctionRegistryBase.build[CDCPathBased](fnName, since = None) case _ => throw DeltaErrors.invalidTableValueFunction(fnName) } val ident = FunctionIdentifier(fnName) (ident, info, builder) } } /////////////////////////////////////////////////////////////////////////// // Logical plans for Delta TVFs // /////////////////////////////////////////////////////////////////////////// /** * Represents an unresolved Delta Table Value Function */ trait DeltaTableValueFunction extends UnresolvedLeafNode { def fnName: String val functionArgs: Seq[Expression] } /** * Base trait for analyzing `table_changes` and `table_changes_for_path`. The resolution works as * follows: * 1. The TVF logical plan is resolved using the TableFunctionRegistry in the Analyzer. This uses * reflection to create one of `CDCNameBased` or `CDCPathBased` by passing all the arguments. * 2. DeltaAnalysis turns the plans to a `TableChanges` node to resolve the DeltaTable. This can * be resolved by the DeltaCatalog for tables or DeltaAnalysis for the path based use. * 3. TableChanges then turns into a LogicalRelation that returns the CDC relation. */ trait CDCStatementBase extends DeltaTableValueFunction { /** Get the table that the function is being called on as an unresolved relation */ protected def getTable(spark: SparkSession, name: Expression): LogicalPlan if (functionArgs.size < 2) { throw new DeltaAnalysisException( errorClass = "INCORRECT_NUMBER_OF_ARGUMENTS", messageParameters = Array( "not enough args", // failure fnName, "2", // minArgs "3")) // maxArgs } if (functionArgs.size > 3) { throw new DeltaAnalysisException( errorClass = "INCORRECT_NUMBER_OF_ARGUMENTS", messageParameters = Array( "too many args", // failure fnName, "2", // minArgs "3")) // maxArgs } protected def getOptions: CaseInsensitiveStringMap = { def toDeltaOption(keyPrefix: String, value: Expression, position: Int): (String, String) = { val evaluated = try { val fakePlan = util.AnalysisHelper.FakeLogicalPlan(Seq(value), Nil) val timestampExpression = org.apache.spark.sql.catalyst.optimizer.ComputeCurrentTime(fakePlan).expressions.head timestampExpression.eval().toString } catch { case _: NullPointerException => throw DeltaErrors.nullRangeBoundaryInCDCRead() case e: SparkException if e.getErrorClass == "INTERNAL_ERROR" => throw DeltaErrors.cdcNonConstantArgument(fnName, keyPrefix, position, value) } value.dataType match { // We dont need to explicitly handle ShortType as it is parsed as IntegerType. case _: IntegerType | LongType => (keyPrefix + "Version") -> evaluated case _: StringType => (keyPrefix + "Timestamp") -> evaluated case _: TimestampType => (keyPrefix + "Timestamp") -> { val fmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS") // when evaluated the time is represented with microseconds, which needs to be trimmed. fmt.format(new Date(evaluated.toLong / 1000)) } case _ => throw DeltaErrors.unsupportedExpression(s"${keyPrefix} option", value.dataType, Seq("IntegerType", "LongType", "StringType", "TimestampType")) } } val startingOption = toDeltaOption("starting", functionArgs(1), 2) val endingOption = functionArgs.drop(2).headOption.map(toDeltaOption("ending", _, 3)) val options = Map(DeltaDataSource.CDC_ENABLED_KEY -> "true", startingOption) ++ endingOption new CaseInsensitiveStringMap(options.asJava) } protected def getStringLiteral(e: Expression, whatFor: String): String = e match { case StringLiteral(value) => value case o => throw DeltaErrors.unsupportedExpression(whatFor, o.dataType, Seq("StringType literal")) } def toTableChanges(spark: SparkSession): TableChanges = TableChanges(getTable(spark, functionArgs.head), fnName) } /** * Plan for the "table_changes" function */ case class CDCNameBased(override val functionArgs: Seq[Expression]) extends CDCStatementBase { override def fnName: String = DeltaTableValueFunctions.CDC_NAME_BASED // Provide a constructor to get a better error message, when no expressions are provided def this() = this(Nil) override protected def getTable(spark: SparkSession, name: Expression): LogicalPlan = { val stringId = getStringLiteral(name, "table name") val identifier = spark.sessionState.sqlParser.parseMultipartIdentifier(stringId) UnresolvedRelation(identifier, getOptions, isStreaming = false) } } /** * Plan for the "table_changes_by_path" function */ case class CDCPathBased(override val functionArgs: Seq[Expression]) extends CDCStatementBase { override def fnName: String = DeltaTableValueFunctions.CDC_PATH_BASED // Provide a constructor to get a better error message, when no expressions are provided def this() = this(Nil) override protected def getTable(spark: SparkSession, name: Expression): LogicalPlan = { UnresolvedPathBasedDeltaTableRelation(getStringLiteral(name, "table path"), getOptions) } } case class TableChanges( child: LogicalPlan, fnName: String, cdcAttr: Seq[Attribute] = CDCReader.cdcAttributes) extends UnaryNode { override lazy val resolved: Boolean = false override def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = this.copy(child = newChild) override def output: Seq[Attribute] = Nil /** Converts the table changes plan to a query over a Delta table */ def toReadQuery: LogicalPlan = child.transformUp { case DataSourceV2RelationShim(d: DeltaTableV2, _, _, _, options) => // withOptions empties the catalog table stats d.withOptions(options.asScala.toMap).toLogicalRelation case r: NamedRelation => throw DeltaErrors.notADeltaTableException(fnName, r.name) case l: LogicalRelation => val relationName = l.catalogTable.map(_.identifier.toString).getOrElse("relation") throw DeltaErrors.notADeltaTableException(fnName, relationName) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaThrowable.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.SparkThrowable /** * The trait for all exceptions of Delta code path. */ trait DeltaThrowable extends SparkThrowable { override def getCondition(): String = getErrorClass() // Portable error identifier across SQL engines // If null, error class or SQLSTATE is not set override def getSqlState: String = DeltaThrowableHelper.getSqlState(this.getErrorClass.split('.').head) // True if this error is an internal error. override def isInternalError: Boolean = DeltaThrowableHelper.isInternalError(this.getErrorClass) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaThrowableHelper.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.io.FileNotFoundException import java.net.URL import scala.collection.JavaConverters._ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.errors.QueryCompilationErrors import org.apache.spark.{SparkException, SparkThrowable} import org.apache.spark.ErrorClassesJsonReader import org.apache.spark.util.Utils /** * The helper object for Delta code base to pick error class template and compile * the exception message. */ object DeltaThrowableHelper { /** * Handles a breaking change (SPARK-46810) between Spark 3.5 and Spark Master (4.0) where * `error-classes.json` was renamed to `error-conditions.json`. */ val SPARK_ERROR_CLASS_SOURCE_FILE = "error/error-conditions.json" def showColumnsWithConflictDatabasesError( db: Seq[String], v1TableName: TableIdentifier): Throwable = { QueryCompilationErrors.showColumnsWithConflictNamespacesError( namespaceA = db, namespaceB = v1TableName.database.get :: Nil) } /** * Try to find the error class source file and throw exception if it is no found. */ private def safeGetErrorClassesSource(sourceFile: String): URL = { val classLoader = Utils.getContextOrSparkClassLoader Option(classLoader.getResource(sourceFile)).getOrElse { throw new FileNotFoundException( s"""Cannot find the error class definition file on path $sourceFile" through the """ + s"class loader ${classLoader.toString}") } } lazy val sparkErrorClassSource: URL = { safeGetErrorClassesSource(SPARK_ERROR_CLASS_SOURCE_FILE) } def deltaErrorClassSource: URL = { safeGetErrorClassesSource("error/delta-error-classes.json") } private val errorClassReader = new ErrorClassesJsonReader( Seq(deltaErrorClassSource, sparkErrorClassSource)) /** * @return The formated error message. The format for standalone error classes is: * [ERROR_CLASS] Main error message * The format for errors with sub-error classes: * [MAIN_CLASS.SUB_CLASS] Main error message Sub-error message */ def getMessage(errorClass: String, messageParameters: Array[String]): String = { validateParameterValues(errorClass, errorSubClass = null, messageParameters) val template = errorClassReader.getMessageTemplate(errorClass) val message = formatMessage(errorClass, messageParameters, template) s"[$errorClass] $message" } private def formatMessage( errorClass: String, messageParameters: Array[String], template: String) = { String.format(template.replaceAll("<[a-zA-Z0-9_-]+>", "%s"), messageParameters: _*) } /** * Returns a combined error message for an error class with multiple sub-error classes. * Use [[getMessage]] to load a single sub-error class message prefixed with * the main class message. * @return The formatted error message including main and sub-error messages. The format is: * [ERROR_CLASS] Main error message * - Sub-error message 1 * - Sub-error message 2 * ... */ def getMessageWithSubErrors( mainErrorClass: String, mainMessageParameters: Array[String], subErrorInformationSeq: Seq[(String, Array[String])]): String = { require(subErrorInformationSeq.nonEmpty) // Get main message val mainMessage = { val template = getMainMessageTemplate(mainErrorClass) formatMessage(mainErrorClass, mainMessageParameters, template) } // Get sub-error messages val subMessageSeq = subErrorInformationSeq.map { case (subErrorClass, subMessageParameters) => val fullErrorClass = s"$mainErrorClass.$subErrorClass" val template = getSubMessageTemplate(fullErrorClass) formatMessage(fullErrorClass, subMessageParameters, template) } // Combine main and sub errors s"[$mainErrorClass] $mainMessage\n${subMessageSeq.map("- " + _ + "\n") .mkString.stripSuffix("\n")}" } /** * Get the message template for a main error class. * @param errorClass The main error class. It can only be MAIN_CLASS (not MAIN_CLASS.SUB_CLASS). * @return The message template. */ def getMainMessageTemplate(errorClass: String): String = { val errorClasses = errorClass.split("\\.") assert(errorClasses.length == 1) val mainErrorClass = errorClasses.head val errorInfo = errorClassReader.errorInfoMap.getOrElse( mainErrorClass, throw SparkException.internalError(s"Cannot find main error class '$errorClass'")) errorInfo.messageTemplate } /** * Get the message template for a sub error class without prefixing with the main error template. * @param errorClass The sub error class. It can only be MAIN_CLASS.SUB_CLASS (not MAIN_CLASS). * @return The message template. */ def getSubMessageTemplate(errorClass: String): String = { val errorClasses = errorClass.split("\\.") assert(errorClasses.length == 2) val mainErrorClass = errorClasses.head val subErrorClass = errorClasses.last val errorInfo = errorClassReader.errorInfoMap.getOrElse( mainErrorClass, throw SparkException.internalError(s"Cannot find main error class '$errorClass'")) assert(errorInfo.subClass.isDefined, errorClass) val errorSubInfo = errorInfo.subClass.get.getOrElse( subErrorClass, throw SparkException.internalError(s"Cannot find sub error class '$errorClass'")) errorSubInfo.messageTemplate } def getSqlState(errorClass: String): String = errorClassReader.getSqlState(errorClass) def isInternalError(errorClass: String): Boolean = errorClass == "INTERNAL_ERROR" def getParameterNames(errorClass: String, errorSubClass: String): Array[String] = { val wholeErrorClass = if (errorSubClass == null) { errorClass } else { errorClass + "." + errorSubClass } val parameterizedMessage = errorClassReader.getMessageTemplate(wholeErrorClass) parsePrameterNamesFromParameterizedMessage(parameterizedMessage) } private def parsePrameterNamesFromParameterizedMessage(parameterizedMessage: String) = { val pattern = "<[a-zA-Z0-9_-]+>".r val matches = pattern.findAllIn(parameterizedMessage) val parameterSeq = matches.toArray val parameterNames = parameterSeq.map(p => p.stripPrefix("<").stripSuffix(">")) parameterNames } def getMessageParameters( errorClass: String, errorSubClass: String, parameterValues: Array[String]): java.util.Map[String, String] = { validateParameterValues(errorClass, errorSubClass, parameterValues) getParameterNames(errorClass, errorSubClass).zip(parameterValues).toMap.asJava } def getMainErrorMessageParameters( errorClass: String, parameterValues: Array[String]): java.util.Map[String, String] = { val parameterizedMessage = getMainMessageTemplate(errorClass) parsePrameterNamesFromParameterizedMessage(parameterizedMessage) .zip(parameterValues).toMap.asJava } /** * Verify that the provided parameter values match the parameter names in the error message * template. The number of parameters must match, and the parameters with the same name must * have the same value. */ private def validateParameterValues( errorClass: String, errorSubClass: String, parameterValues: Array[String]): Unit = if (Utils.isTesting) { val parameterNames = getParameterNames(errorClass, errorSubClass) assert(parameterNames.size == parameterValues.size, "The number of parameter values provided " + s"to error class $errorClass ${Option(errorSubClass).getOrElse("")} does not match the " + s"number of parameters in the error message template.\n" + s"Parameters in the template: ${parameterNames.mkString(", ")}\n" + s"Parameter values provided: ${parameterValues.mkString(", ")}") val parameterPairs = parameterNames.zip(parameterValues) val parameterMap = parameterPairs.toMap parameterPairs.foreach { case (name, value) => assert(parameterMap(name) == value, s"Parameter <$name> in the error message for error " + s"class $errorClass ${Option(errorSubClass).getOrElse("")} was assigned two different " + s"values: ${parameterMap(name)} and $value") } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaTimeTravelSpec.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.sql.Timestamp import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.commons.lang3.time.FastDateFormat import org.apache.spark.sql.catalyst.expressions.{Cast, Expression, Literal, PreciseTimestampConversion, RuntimeReplaceable, Unevaluable} import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{LongType, TimestampType} /** * The specification to time travel a Delta Table to the given `timestamp` or `version`. * @param timestamp An expression that can be evaluated into a timestamp. The expression cannot * be a subquery. * @param version The version of the table to time travel to. Must be >= 0. * @param creationSource The API used to perform time travel, e.g. `atSyntax`, `dfReader` or SQL * @param enforceRetention Whether to enforce file delete retention and block access to expired * snapshot regardless of the VACUUM status. */ case class DeltaTimeTravelSpec( timestamp: Option[Expression], version: Option[Long], creationSource: Option[String], enforceRetention: Boolean = true) extends DeltaLogging { assert(version.isEmpty ^ timestamp.isEmpty, "Either the version or timestamp should be provided for time travel") /** * Compute the timestamp to use for time travelling the relation from the given expression for * the given time zone. */ def getTimestamp(conf: SQLConf): Timestamp = { // note @brkyvz (2020-04-13): not great that we need to handle RuntimeReplaceable expressions... val timeZone = conf.sessionLocalTimeZone val evaluable = timestamp match { case Some(e) => e.transform { case rr: RuntimeReplaceable => rr.children.head case e: Unevaluable => recordDeltaEvent(null, "delta.timeTravel.unexpected", data = e.sql) throw new IllegalStateException(s"Unsupported expression (${e.sql}) for time travel.") } case None => // scalastyle:off throwerror throw new AssertionError( "Should not ask to get Timestamp for time travel when the timestamp was not available") // scalastyle:on throwerror } val strict = conf.getConf(DeltaSQLConf.DELTA_TIME_TRAVEL_STRICT_TIMESTAMP_PARSING) val castResult = Cast(evaluable, TimestampType, Option(timeZone), ansiEnabled = false).eval() if (strict && castResult == null) { throw DeltaErrors.timestampInvalid(evaluable) } DateTimeUtils.toJavaTimestamp(castResult.asInstanceOf[java.lang.Long]) } /** * Compute the timestamp to use for time travelling the relation from the given expression for * the given time zone if this spec has a timestamp defined. */ def getTimestampOpt(conf: SQLConf): Option[Timestamp] = { timestamp.map(_ => getTimestamp(conf)) } } object DeltaTimeTravelSpec { /** A regex which looks for the pattern ...@v(some numbers) for extracting the version number */ private val VERSION_URI_FOR_TIME_TRAVEL = ".*@[vV](\\d+)$".r /** The timestamp format which we accept after the `@` character. */ private val TIMESTAMP_FORMAT = "yyyyMMddHHmmssSSS" /** Length of yyyyMMddHHmmssSSS */ private val TIMESTAMP_FORMAT_LENGTH = TIMESTAMP_FORMAT.length /** A regex which looks for the pattern ...@(yyyyMMddHHmmssSSS) for extracting timestamps. */ private val TIMESTAMP_URI_FOR_TIME_TRAVEL = s".*@(\\d{$TIMESTAMP_FORMAT_LENGTH})$$".r /** Returns whether the given table identifier may contain time travel syntax. */ def isApplicable(conf: SQLConf, identifier: String): Boolean = { conf.getConf(DeltaSQLConf.RESOLVE_TIME_TRAVEL_ON_IDENTIFIER) && identifierContainsTimeTravel(identifier) } /** Checks if the table identifier contains patterns that resemble time travel syntax. */ private def identifierContainsTimeTravel(identifier: String): Boolean = identifier match { case TIMESTAMP_URI_FOR_TIME_TRAVEL(ts) => true case VERSION_URI_FOR_TIME_TRAVEL(v) => true case _ => false } /** Adds a time travel node based on the special syntax in the table identifier. */ def resolvePath(conf: SQLConf, identifier: String): (DeltaTimeTravelSpec, String) = { identifier match { case TIMESTAMP_URI_FOR_TIME_TRAVEL(ts) => val timestamp = parseTimestamp(ts, conf.sessionLocalTimeZone) // Drop the 18 characters in the right, which is the timestamp format and the @ character. val realIdentifier = identifier.dropRight(TIMESTAMP_FORMAT_LENGTH + 1) DeltaTimeTravelSpec(Some(timestamp), None, Some("atSyntax.path")) -> realIdentifier case VERSION_URI_FOR_TIME_TRAVEL(v) => // Drop the version, and `@v` characters from the identifier val realIdentifier = identifier.dropRight(v.length + 2) DeltaTimeTravelSpec(None, Some(v.toLong), Some("atSyntax.path")) -> realIdentifier } } /** * Parse the given timestamp string into a proper Catalyst TimestampType. We support millisecond * level precision, therefore don't use standard SQL timestamp functions, which only support * second level precision. * * @throws `AnalysisException` when the timestamp format doesn't match our criteria */ private def parseTimestamp(ts: String, timeZone: String): Expression = { val format = FastDateFormat.getInstance(TIMESTAMP_FORMAT, DateTimeUtils.getTimeZone(timeZone)) try { val sqlTs = DateTimeUtils.fromJavaTimestamp(new java.sql.Timestamp(format.parse(ts).getTime)) PreciseTimestampConversion(Literal(sqlTs), LongType, TimestampType) } catch { case e: java.text.ParseException => throw DeltaErrors.invalidTimestampFormat(ts, TIMESTAMP_FORMAT, Some(e)) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaUDF.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.actions.{DeletionVectorDescriptor, Protocol} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.DeletedRecordCountsHistogram import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.encoders.encoderFor import org.apache.spark.sql.expressions.{SparkUserDefinedFunction, UserDefinedFunction} import org.apache.spark.sql.functions.udf /** * Define a few templates for udfs used by Delta. Use these templates to create * `SparkUserDefinedFunction` to avoid creating new Encoders. This would save us from touching * `ScalaReflection` to reduce the lock contention in concurrent queries. */ object DeltaUDF { def stringFromString(f: String => String): UserDefinedFunction = createUdfFromTemplateUnsafe(stringFromStringTemplate, f, udf(f)) def intFromString(f: String => Int): UserDefinedFunction = createUdfFromTemplateUnsafe(intFromStringTemplate, f, udf(f)) def intFromStringBoolean(f: (String, Boolean) => Int): UserDefinedFunction = createUdfFromTemplateUnsafe(intFromStringBooleanTemplate, f, udf(f)) def boolean(f: () => Boolean): UserDefinedFunction = createUdfFromTemplateUnsafe(booleanTemplate, f, udf(f)) def stringFromMap(f: Map[String, String] => String): UserDefinedFunction = createUdfFromTemplateUnsafe(stringFromMapTemplate, f, udf(f)) def deletedRecordCountsHistogramFromArrayLong( f: Array[Long] => DeletedRecordCountsHistogram): UserDefinedFunction = createUdfFromTemplateUnsafe(deletedRecordCountsHistogramFromArrayLongTemplate, f, udf(f)) def stringFromDeletionVectorDescriptor( f: DeletionVectorDescriptor => String): UserDefinedFunction = createUdfFromTemplateUnsafe(stringFromDeletionVectorDescriptorTemplate, f, udf(f)) def stringOptionFromDeletionVectorDescriptor( f: DeletionVectorDescriptor => Option[String]): UserDefinedFunction = createUdfFromTemplateUnsafe(stringOptionFromDeletionVectorDescriptorTemplate, f, udf(f)) def booleanFromDeletionVectorDescriptor( f: DeletionVectorDescriptor => Boolean): UserDefinedFunction = createUdfFromTemplateUnsafe(booleanFromDeletionVectorDescriptorTemplate, f, udf(f)) def booleanFromString(s: String => Boolean): UserDefinedFunction = createUdfFromTemplateUnsafe(booleanFromStringTemplate, s, udf(s)) def booleanFromProtocol(f: Protocol => Boolean): UserDefinedFunction = createUdfFromTemplateUnsafe(booleanFromProtocol, f, udf(f)) def booleanFromMap(f: Map[String, String] => Boolean): UserDefinedFunction = createUdfFromTemplateUnsafe(booleanFromMapTemplate, f, udf(f)) def booleanFromByte(x: Byte => Boolean): UserDefinedFunction = createUdfFromTemplateUnsafe(booleanFromByteTemplate, x, udf(x)) private lazy val stringFromStringTemplate = udf[String, String](identity).asInstanceOf[SparkUserDefinedFunction] private lazy val booleanTemplate = udf(() => true).asInstanceOf[SparkUserDefinedFunction] private lazy val intFromStringTemplate = udf((_: String) => 1).asInstanceOf[SparkUserDefinedFunction] private lazy val intFromStringBooleanTemplate = udf((_: String, _: Boolean) => 1).asInstanceOf[SparkUserDefinedFunction] private lazy val stringFromMapTemplate = udf((_: Map[String, String]) => "").asInstanceOf[SparkUserDefinedFunction] private lazy val deletedRecordCountsHistogramFromArrayLongTemplate = udf((_: Array[Long]) => DeletedRecordCountsHistogram(Array.empty)) .asInstanceOf[SparkUserDefinedFunction] private lazy val stringFromDeletionVectorDescriptorTemplate = udf((_: DeletionVectorDescriptor) => "").asInstanceOf[SparkUserDefinedFunction] private lazy val stringOptionFromDeletionVectorDescriptorTemplate = udf((_: DeletionVectorDescriptor) => Some("")).asInstanceOf[SparkUserDefinedFunction] private lazy val booleanFromDeletionVectorDescriptorTemplate = udf((_: DeletionVectorDescriptor) => false).asInstanceOf[SparkUserDefinedFunction] private lazy val booleanFromStringTemplate = udf((_: String) => false).asInstanceOf[SparkUserDefinedFunction] private lazy val booleanFromProtocol = udf((_: Protocol) => true).asInstanceOf[SparkUserDefinedFunction] private lazy val booleanFromMapTemplate = udf((_: Map[String, String]) => true).asInstanceOf[SparkUserDefinedFunction] private lazy val booleanFromByteTemplate = udf((_: Byte) => true).asInstanceOf[SparkUserDefinedFunction] /** * Return a `UserDefinedFunction` for the given `f` from `template` if * `INTERNAL_UDF_OPTIMIZATION_ENABLED` is enabled. Otherwise, `orElse` will be called to create a * new `UserDefinedFunction`. */ private def createUdfFromTemplateUnsafe( template: SparkUserDefinedFunction, f: AnyRef, orElse: => UserDefinedFunction): UserDefinedFunction = { if (SparkSession.active.sessionState.conf .getConf(DeltaSQLConf.INTERNAL_UDF_OPTIMIZATION_ENABLED)) { val inputEncoders = template.inputEncoders.map(_.map(e => encoderFor(e))) val outputEncoder = template.outputEncoder.map(e => encoderFor(e)) template.copy(f = f, inputEncoders = inputEncoders, outputEncoder = outputEncoder) } else { orElse } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaUnsupportedOperationsCheck.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.util.control.NonFatal // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSourceUtils import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.ResolvedTable import org.apache.spark.sql.catalyst.catalog.CatalogTableType import org.apache.spark.sql.catalyst.plans.logical.{AppendData, DropTable, LogicalPlan, OverwriteByExpression, ShowCreateTable, V2WriteCommand} import org.apache.spark.sql.execution.command._ import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, DataSourceV2RelationShim} /** * A rule to add helpful error messages when Delta is being used with unsupported Hive operations * or if an unsupported operation is being made, e.g. a DML operation like * INSERT/UPDATE/DELETE/MERGE when a table doesn't exist. */ case class DeltaUnsupportedOperationsCheck(spark: SparkSession) extends (LogicalPlan => Unit) with DeltaLogging { private def fail(operation: String, tableIdent: TableIdentifier): Unit = { val metadata = try Some(spark.sessionState.catalog.getTableMetadata(tableIdent)) catch { case NonFatal(_) => None } if (metadata.exists(DeltaTableUtils.isDeltaTable)) { throw DeltaErrors.operationNotSupportedException(operation, tableIdent) } } private def fail(operation: String, provider: String): Unit = { if (DeltaSourceUtils.isDeltaDataSourceName(provider)) { throw DeltaErrors.operationNotSupportedException(operation) } } def apply(plan: LogicalPlan): Unit = plan.foreach { // Unsupported Hive commands case a: AnalyzePartitionCommand => recordDeltaEvent(null, "delta.unsupported.analyzePartition") fail(operation = "ANALYZE TABLE PARTITION", a.tableIdent) case a: AlterTableAddPartitionCommand => recordDeltaEvent(null, "delta.unsupported.addPartition") fail(operation = "ALTER TABLE ADD PARTITION", a.tableName) case a: AlterTableDropPartitionCommand => recordDeltaEvent(null, "delta.unsupported.dropPartition") fail(operation = "ALTER TABLE DROP PARTITION", a.tableName) case a: RepairTableCommand => recordDeltaEvent(null, "delta.unsupported.recoverPartitions") fail(operation = "ALTER TABLE RECOVER PARTITIONS", a.tableName) case a: AlterTableSerDePropertiesCommand => recordDeltaEvent(null, "delta.unsupported.alterSerDe") fail(operation = "ALTER TABLE SET SERDEPROPERTIES", a.tableName) case l: LoadDataCommand => recordDeltaEvent(null, "delta.unsupported.loadData") fail(operation = "LOAD DATA", l.table) case i: InsertIntoDataSourceDirCommand => recordDeltaEvent(null, "delta.unsupported.insertDirectory") fail(operation = "INSERT OVERWRITE DIRECTORY", i.provider) case ShowCreateTable(t: ResolvedTable, _, _) if t.table.isInstanceOf[DeltaTableV2] => recordDeltaEvent(null, "delta.unsupported.showCreateTable") fail(operation = "SHOW CREATE TABLE", "DELTA") // Delta table checks case append: AppendData => val op = if (append.isByName) "APPEND" else "INSERT" checkDeltaTableExists(append, op) case overwrite: OverwriteByExpression => checkDeltaTableExists(overwrite, "OVERWRITE") case _: DropTable => // For Delta tables being dropped, we do not need the underlying Delta log to exist so this is // OK return case DataSourceV2RelationShim(tbl: DeltaTableV2, _, _, _, _) if !tbl.tableExists => throw DeltaErrors.pathNotExistsException(tbl.deltaLog.dataPath.toString) case r: ResolvedTable if r.table.isInstanceOf[DeltaTableV2] && !r.table.asInstanceOf[DeltaTableV2].tableExists => throw DeltaErrors.pathNotExistsException( r.table.asInstanceOf[DeltaTableV2].deltaLog.dataPath.toString) case _ => // OK } /** * Check that the given operation is being made on a full Delta table that exists. */ private def checkDeltaTableExists(command: V2WriteCommand, operation: String): Unit = { command.table match { case DeltaRelation(lr) => // the extractor performs the check that we want if this is indeed being called on a Delta // table. It should leave others unchanged if (DeltaFullTable.unapply(lr).isEmpty) { throw DeltaErrors.notADeltaTableException(operation) } case _ => } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DeltaViewHelper.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, Cast, NamedExpression} import org.apache.spark.sql.catalyst.optimizer.CollapseProject import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, SubqueryAlias, View, ViewShims} import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.internal.SQLConf object DeltaViewHelper { def stripTempViewForMerge(plan: LogicalPlan, conf: SQLConf): LogicalPlan = { // Check that the two expression lists have the same names and types in the same order, and // are either attributes or direct casts of attributes. def attributesMatch(left: Seq[NamedExpression], right: Seq[NamedExpression]): Boolean = { if (left.length != right.length) return false val allowedExprs = (left ++ right).forall { case _: Attribute => true case Alias(Cast(attr: Attribute, _, _, _), name) => conf.resolver(attr.name, name) case _ => false } val exprsMatch = left.zip(right).forall { case (a, b) => a.dataType == b.dataType && conf.resolver(a.name, b.name) } allowedExprs && exprsMatch } // We have to do a pretty complicated transformation here to support using two specific things // which are not a Delta table as the target of Delta DML commands: // A view defined as `SELECT * FROM underlying_tbl` // A view defined as `SELECT * FROM underlying_tbl as alias` // This requires stripping their intermediate nodes and pulling out just the scan, because // some of our internal attribute fiddling requires the target plan to have the same attribute // IDs as the underlying scan. object ViewPlan { def unapply( plan: LogicalPlan): Option[(CatalogTable, Seq[NamedExpression], LogicalRelation)] = { // A `SELECT * from underlying_table` view will have: // * A View node marking it as a view. // * An outer Project explicitly casting the scanned types to the types defined in the // metastore for the view. We don't need this cast for Delta DML commands and it will // end up being eliminated. // * An arbitrary number of inner no-op project and subquery aliases. // * The actual scan of the Delta table. // We check for this by removing all subquery aliases and collapsing all Projects into one // and ensuring that the project list exactly match the output of the scan. CollapseProject(EliminateSubqueryAliases(plan)) match { case ViewShims.TempViewWithChild(desc, Project(outerList, scan: LogicalRelation)) if attributesMatch(outerList, scan.output) => Some(desc, outerList, scan) case _ => None } } } plan.transformUp { case ViewPlan(desc, outerList, scan) => // Produce a scan with the outer list's attribute IDs aliased to the view's name. val newOutput = scan.output.map { oldAttr => val newId = outerList.collectFirst { case newAttr if conf.resolver(oldAttr.name, newAttr.name) => newAttr.exprId }.getOrElse { throw DeltaErrors.noNewAttributeId(oldAttr) } oldAttr.withExprId(newId) } SubqueryAlias(desc.qualifiedName, scan.copy(output = newOutput)) case v: View if v.isTempView => v.child } } def stripTempView(plan: LogicalPlan, conf: SQLConf): LogicalPlan = { plan.transformUp { case v: View if v.isTempView => v.child } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/DomainMetadataUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.skipping.clustering.ClusteredTableUtils import org.apache.spark.sql.delta.actions.{Action, DomainMetadata, Protocol} import org.apache.spark.sql.delta.clustering.ClusteringMetadataDomain import org.apache.spark.sql.delta.metering.DeltaLogging /** * Domain metadata utility functions. */ trait DomainMetadataUtilsBase extends DeltaLogging { // List of metadata domains that will be removed for the REPLACE TABLE operation. protected val METADATA_DOMAINS_TO_REMOVE_FOR_REPLACE_TABLE: Set[String] = Set( ClusteringMetadataDomain.domainName) // List of metadata domains that will be copied from the table we are restoring to. // Note that ClusteringMetadataDomain are recreated in handleDomainMetadataForRestoreTable // instead of being blindly copied over. protected val METADATA_DOMAIN_TO_COPY_FOR_RESTORE_TABLE: Set[String] = Set.empty // List of metadata domains that will be copied from the table on a CLONE operation. protected val METADATA_DOMAIN_TO_COPY_FOR_CLONE_TABLE: Set[String] = Set( ClusteringMetadataDomain.domainName) /** * Returns whether the protocol version supports the [[DomainMetadata]] action. */ def domainMetadataSupported(protocol: Protocol): Boolean = protocol.isFeatureSupported(DomainMetadataTableFeature) /** * Given a list of [[Action]]s, build a domain name to [[DomainMetadata]] map. * Note duplicated domain name is not expected otherwise an internal error is thrown. */ def extractDomainMetadatasMap(actions: Seq[Action]): Map[String, DomainMetadata] = { actions .collect { case action: DomainMetadata => action } .groupBy(_.domain) .map { case (name, domains) => if (domains.length != 1) { throw DeltaErrors.domainMetadataDuplicate(domains.head.domain) } name -> domains.head } } /** * Validate there are no two [[DomainMetadata]] actions with the same domain name. An internal * exception is thrown if any duplicated domains are detected. * * @param actions: Actions the current transaction wants to commit. */ def validateDomainMetadataSupportedAndNoDuplicate( actions: Seq[Action], protocol: Protocol): Seq[DomainMetadata] = { val domainMetadatas = extractDomainMetadatasMap(actions) if (domainMetadatas.nonEmpty && !domainMetadataSupported(protocol)) { throw DeltaErrors.domainMetadataTableFeatureNotSupported( domainMetadatas.map(_._2.domain).mkString("[", ",", "]")) } domainMetadatas.values.toSeq } /** * Generates a new sequence of DomainMetadata to commits for REPLACE TABLE. * - By default, existing metadata domains survive as long as they don't appear in the * new metadata domains, in which case new metadata domains overwrite the existing ones. * - Existing domains will be removed only if they appear in the pre-defined * "removal" list (e.g., table features require some specific domains to be removed). */ def handleDomainMetadataForReplaceTable( existingDomainMetadatas: Seq[DomainMetadata], newDomainMetadatas: Seq[DomainMetadata]): Seq[DomainMetadata] = { val newDomainNames = newDomainMetadatas.map(_.domain).toSet existingDomainMetadatas // Filter out metadata domains unless they are in the list to be removed // and they don't appear in the new metadata domains. .filter(m => !newDomainNames.contains(m.domain) && METADATA_DOMAINS_TO_REMOVE_FOR_REPLACE_TABLE.contains(m.domain)) .map(_.copy(removed = true)) ++ newDomainMetadatas } /** * Generates a new sequence of DomainMetadata to commits for RESTORE TABLE. * - Domains in the toSnapshot will be copied if they appear in the pre-defined * "copy" list (e.g., table features require some specific domains to be copied). * - All other domains not in the list are dropped from the "toSnapshot". * * For clustering metadata domain, it overwrites the existing domain metadata in the * fromSnapshot with the following clustering columns. * 1. If toSnapshot is not a clustered table or missing domain metadata, use empty clustering * columns. * 2. If toSnapshot is a clustered table, use the clustering columns from toSnapshot. * * @param toSnapshot The snapshot being restored to, which is referred as "source" table. * @param fromSnapshot The snapshot being restored from, which is the current state. */ def handleDomainMetadataForRestoreTable( toSnapshot: Snapshot, fromSnapshot: Snapshot): Seq[DomainMetadata] = { val filteredDomainMetadata = toSnapshot.domainMetadata.filter { m => METADATA_DOMAIN_TO_COPY_FOR_RESTORE_TABLE.contains(m.domain) } val clusteringColumnsToRestore = ClusteredTableUtils.getClusteringColumnsOptional(toSnapshot) val isRestoringToClusteredTable = ClusteredTableUtils.isSupported(toSnapshot.protocol) && clusteringColumnsToRestore.nonEmpty val clusteringColumns = if (isRestoringToClusteredTable) { // We overwrite the clustering columns in the fromSnapshot with the clustering columns // in the toSnapshot. clusteringColumnsToRestore.get } else { // toSnapshot is not a clustered table or missing domain metadata, so we write domain // metadata with empty clustering columns. Seq.empty } val matchingMetadataDomain = ClusteredTableUtils.getMatchingMetadataDomain( clusteringColumns, fromSnapshot.domainMetadata) // RESTORE table is effectively replacing the current table state (`fromSnapshot`) with a // previous snapshot (`toSnapshot`). Like for REPLACE table, this means any DomainMetadata in // the previous `fromSnapshot` without an equivalent domain in the `fromSnapshot` must be marked // as removed. handleDomainMetadataForReplaceTable( toSnapshot.domainMetadata, filteredDomainMetadata ++ matchingMetadataDomain.clusteringDomainOpt) } /** * Generates sequence of DomainMetadata to commit for CLONE TABLE command. */ def handleDomainMetadataForCloneTable( sourceSnapshot: Snapshot, targetSnapshot: Snapshot): Seq[DomainMetadata] = { val newDomainMetadata = sourceSnapshot.domainMetadata.filter { m => METADATA_DOMAIN_TO_COPY_FOR_CLONE_TABLE.contains(m.domain) } // A CLONE operation may overwrite an existing snapshot (effectively a REPLACE operation). // Handle the removed DomainMetadata accordingly. handleDomainMetadataForReplaceTable(targetSnapshot.domainMetadata, newDomainMetadata) } } object DomainMetadataUtils extends DomainMetadataUtilsBase ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/FallbackToV1Relations.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation /** * Fall back to V1 nodes, since we don't have a V2 reader for Delta right now */ object FallbackToV1DeltaRelation { def unapply(dsv2: DataSourceV2Relation): Option[LogicalRelation] = dsv2.table match { case d: DeltaTableV2 if dsv2.getTagValue(DeltaRelation.KEEP_AS_V2_RELATION_TAG).isEmpty => Some(DeltaRelation.fromV2Relation(d, dsv2, dsv2.options)) case _ => None } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/FileMetadataMaterializationTracker.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.concurrent.Semaphore import java.util.concurrent.atomic.AtomicInteger import org.apache.spark.sql.delta.FileMetadataMaterializationTracker.TaskLevelPermitAllocator import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.internal.{Logging, MDC} import org.apache.spark.sql.SparkSession /** * An instance of this class tracks and controls the materialization usage of a single command * query (e.g. Backfill) with respect to the driver limits. Each query must use one instance of the * FileMaterializationTracker. * * tasks - tasks are the basic unit of computation. * For example, in Backfill, each task bins multiple files into batches to be executed. * * A task has to be materialized in its entirety, so in the case where we are unable to acquire * permits to materialize a task we acquire an over allocation lock that will allow tasks to * complete materializing. Over allocation is only allowed for one thread at once in the driver. * This allows us to restrict the amount of file metadata being materialized at once on the driver. * * Accessed by the thread materializing files and by the thread releasing resources after execution. * */ class FileMetadataMaterializationTracker extends Logging { /** The number of permits allocated from the global file materialization semaphore */ @volatile private var numPermitsFromSemaphore: Int = 0 /** The number of permits over allocated by holding the overAllocationLock */ @volatile private var numOverAllocatedPermits: Int = 0 private val materializationMetrics = new FileMetadataMaterializationMetrics() /** * @return The collected materialization metrics for this query. */ def getMetrics(): FileMetadataMaterializationMetrics = { materializationMetrics } /** * Signals to execute the batch early in the event that we overallocated to * materialize a task. */ def executeBatchEarly(): Boolean = { numOverAllocatedPermits > 0 } /** * A per task permit allocator which allows materializing a new task. * @return - TaskLevelPermitAllocator to be used to materialize a task */ def createTaskLevelPermitAllocator(): TaskLevelPermitAllocator = { new TaskLevelPermitAllocator(this) } /** * Acquire a permit from the materialization semaphore, if there is no permit available the thread * acquires the overAllocationLock which allows it to freely acquire permits in the future. * Only one thread can over allocate at once. * * @param isNewTask - indicates whether the permit is being acquired for a new task, this will * allow us to prevent overallocation to spill over to new tasks. */ private def acquirePermit(isNewTask: Boolean = false): Unit = { var hasAcquiredPermit = false if (isNewTask) { FileMetadataMaterializationTracker.materializationSemaphore.acquire(1) hasAcquiredPermit = true } else if (numOverAllocatedPermits > 0) { materializationMetrics.overAllocFilesMaterializedCount += 1 } else if (!FileMetadataMaterializationTracker.materializationSemaphore.tryAcquire(1)) { // we acquire the overAllocationLock for this thread logInfo(log"Acquiring over allocation lock for this query.") val startTime = System.currentTimeMillis() FileMetadataMaterializationTracker.overAllocationLock.acquire(1) val waitTime = System.currentTimeMillis() - startTime logInfo(log"Acquired over allocation lock for this query in " + log"${MDC(DeltaLogKeys.TIME_MS, waitTime)} ms") materializationMetrics.overAllocWaitTimeMs += waitTime materializationMetrics.overAllocWaitCount += 1 materializationMetrics.overAllocFilesMaterializedCount += 1 } else { // tryAcquire was successful hasAcquiredPermit = true } if (hasAcquiredPermit) { this.synchronized { numPermitsFromSemaphore += 1 } } else { this.synchronized { numOverAllocatedPermits += 1 } } materializeOneFile() } /** Increment the number of materialized file in materializationMetrics. */ def materializeOneFile(): Unit = materializationMetrics.filesMaterializedCount += 1 /** * Release `numPermits` file permits and release overAllocationLock lock if held by the thread * and the number of over allocated files is 0. */ def releasePermits(numPermits: Int): Unit = { var permitsToRelease = numPermits this.synchronized { if (numOverAllocatedPermits > 0) { val overAllocatedPermitsToRelease = Math.min(numOverAllocatedPermits, numPermits) numOverAllocatedPermits -= overAllocatedPermitsToRelease permitsToRelease -= overAllocatedPermitsToRelease if (numOverAllocatedPermits == 0) { FileMetadataMaterializationTracker.overAllocationLock.release(1) logInfo(log"Released over allocation lock.") } } numPermitsFromSemaphore -= permitsToRelease } FileMetadataMaterializationTracker.materializationSemaphore.release(permitsToRelease) } /** * This will release all acquired file permits by the tracker. */ def releaseAllPermits(): Unit = { this.synchronized { if (numOverAllocatedPermits > 0) { FileMetadataMaterializationTracker.overAllocationLock.release(1) } if (numPermitsFromSemaphore > 0) { FileMetadataMaterializationTracker.materializationSemaphore.release(numPermitsFromSemaphore) } numPermitsFromSemaphore = 0 numOverAllocatedPermits = 0 } } } object FileMetadataMaterializationTracker extends DeltaLogging { // Global limit for number of files that can be materialized at once on the driver private val globalFileMaterializationLimit: AtomicInteger = new AtomicInteger(-1) // Semaphore to control file materialization private var materializationSemaphore: Semaphore = _ /** * Global lock that is held by a thread and allows it to materialize files without * acquiring permits the materializationSemaphore. * * This lock is released when the thread completes executing the command's job that * acquired it, or when all permits are released during bin packing. */ private val overAllocationLock = new Semaphore(1) /** * Initialize the global materialization semaphore using an existing semaphore. This is used * for unit tests. */ private[sql] def initializeSemaphoreForTests(semaphore: Semaphore): Unit = { globalFileMaterializationLimit.set(semaphore.availablePermits()) materializationSemaphore = semaphore } /** * Initialize materialization semaphore if this is the first query running on the cluster that * uses the file materialization tracker. */ private def initializeMaterializationSemaphore(spark: SparkSession): Unit = { if (globalFileMaterializationLimit.compareAndSet(-1, spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_COMMAND_FILE_MATERIALIZATION_LIMIT))) { if (globalFileMaterializationLimit.get() > 0) { materializationSemaphore = new Semaphore(globalFileMaterializationLimit.get) } } } def withTracker( origTxn: OptimisticTransaction, spark: SparkSession, metricsOpType: String)(f: FileMetadataMaterializationTracker => Unit): Unit = { initializeMaterializationSemaphore(spark) val shouldTrack = spark.conf.get( DeltaSQLConf.DELTA_COMMAND_FILE_MATERIALIZATION_TRACKING_ENABLED) val tracker = if (shouldTrack) { new FileMetadataMaterializationTracker() } else { logInfo(log"File metadata materialization tracking is disabled for this query." + log" Please set ${MDC(DeltaLogKeys.CONFIG_KEY, DeltaSQLConf.DELTA_COMMAND_FILE_MATERIALIZATION_TRACKING_ENABLED.key)} " + log"to true to enable it.") noopTracker } try { f(tracker) val trackerMetrics = tracker.getMetrics() logInfo(log"File metadata materialization metrics for the completed query: " + log"${MDC(DeltaLogKeys.METRICS, trackerMetrics)}") recordDeltaEvent( deltaLog = origTxn.deltaLog, opType = metricsOpType, data = trackerMetrics) } finally { tracker.releaseAllPermits() } } /** * @return - return a version of the FileMetadataMaterializationTracker where every operation * is a noop */ val noopTracker: FileMetadataMaterializationTracker = new FileMetadataMaterializationTracker() { override def releasePermits(numPermits: Int): Unit = { } override def createTaskLevelPermitAllocator() = new TaskLevelPermitAllocator(this) { override def acquirePermit(): Unit = { } } override def executeBatchEarly(): Boolean = false override def releaseAllPermits(): Unit = { } override def getMetrics(): FileMetadataMaterializationMetrics = { new FileMetadataMaterializationMetrics() } } /** * A per task level allocator that controls permit allocation and releasing for the task */ class TaskLevelPermitAllocator(tracker: FileMetadataMaterializationTracker) { /** Indicates whether the file materialization is for a new task */ var isNewTask = true /** * Acquire a single file materialization permit. */ def acquirePermit(): Unit = { if (isNewTask) { logInfo(log"Acquiring file materialization permits for a new task") } tracker.acquirePermit(isNewTask = isNewTask) isNewTask = false } } } /** * Instance of this class is used for recording metrics of the FileMetadataMaterializationTracker */ case class FileMetadataMaterializationMetrics( /** Total number of files materialized */ var filesMaterializedCount: Long = 0L, /** Number of times we wait to acquire the over allocation lock */ var overAllocWaitCount: Long = 0L, /** Total time waited to acquire the over allocation lock in ms */ var overAllocWaitTimeMs: Long = 0L, /** Number of files materialized by using over allocation lock */ var overAllocFilesMaterializedCount: Long = 0L) { override def toString(): String = { s"Number of files materialized: $filesMaterializedCount, " + s"Number of times over-allocated: $overAllocWaitCount, " + s"Total time spent waiting to acquire over-allocation lock: $overAllocWaitTimeMs, " + s"Files materialized by over allocation: $overAllocFilesMaterializedCount" } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/GenerateIdentityValues.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import com.databricks.spark.util.MetricDefinitions import com.databricks.spark.util.TagDefinitions.TAG_OP_TYPE import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.{SparkException, TaskContext} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{Expression, LeafExpression, Nondeterministic} import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGenerator, ExprCode, FalseLiteral} import org.apache.spark.sql.catalyst.expressions.codegen.Block._ import org.apache.spark.sql.types.{DataType, LongType} /** * Returns the next generated IDENTITY column value based on the underlying * [[PartitionIdentityValueGenerator]]. */ case class GenerateIdentityValues(generator: PartitionIdentityValueGenerator) extends LeafExpression with Nondeterministic { override protected def initializeInternal(partitionIndex: Int): Unit = { generator.initialize(partitionIndex) } override protected def evalInternal(input: InternalRow): Long = generator.next() override def nullable: Boolean = false /** * Returns Java source code that can be compiled to evaluate this expression. * The default behavior is to call the eval method of the expression. Concrete expression * implementations should override this to do actual code generation. * * @param ctx a [[CodegenContext]] * @param ev an [[ExprCode]] with unique terms. * @return an [[ExprCode]] containing the Java source code to generate the given expression */ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { val generatorTerm = ctx.addReferenceObj("generator", generator, classOf[PartitionIdentityValueGenerator].getName) ctx.addPartitionInitializationStatement(s"$generatorTerm.initialize(partitionIndex);") ev.copy(code = code""" final ${CodeGenerator.javaType(dataType)} ${ev.value} = $generatorTerm.next(); """, isNull = FalseLiteral) } /** * Returns the [[DataType]] of the result of evaluating this expression. It is * invalid to query the dataType of an unresolved expression (i.e., when `resolved` == false). */ override def dataType: DataType = LongType } object GenerateIdentityValues { def apply(start: Long, step: Long, highWaterMarkOpt: Option[Long]): GenerateIdentityValues = { new GenerateIdentityValues(PartitionIdentityValueGenerator(start, step, highWaterMarkOpt)) } } /** * Generator of IDENTITY value for one partition. * * @param start The configured start value for the identity column. * @param highWaterMarkOpt The optional high watermark for the identity value generation. If this is * None, that means that no identity values has been generated in the past and * we should start the identity value generation from the `start`. * @param step IDENTITY value increment. */ case class PartitionIdentityValueGenerator( start: Long, step: Long, highWaterMarkOpt: Option[Long]) { require(step != 0) // The value generation logic requires high water mark to follow the start and step configuration. highWaterMarkOpt.foreach(highWaterMark => require((highWaterMark - start) % step == 0)) private lazy val base = highWaterMarkOpt.map(Math.addExact(_, step)).getOrElse(start) private var partitionIndex: Int = -1 private var nextValue: Long = -1L private var increment: Long = -1L def initialize(partitionIndex: Int): Unit = { if (this.partitionIndex < 0) { this.partitionIndex = partitionIndex this.nextValue = try { Math.addExact(base, Math.multiplyExact(partitionIndex, step)) } catch { case e: ArithmeticException => IdentityOverflowLogger.logOverflow() throw e } // Each value is incremented by numPartitions * step from the previous value. this.increment = try { // Total number of partitions. In local execution case where TaskContext is not set, the // task is executed as a single partition. val numPartitions = Option(TaskContext.get()).map(_.numPartitions()).getOrElse(1) Math.multiplyExact(numPartitions, step) } catch { case e: ArithmeticException => IdentityOverflowLogger.logOverflow() throw e } } else if (this.partitionIndex != partitionIndex) { throw SparkException.internalError("Same PartitionIdentityValueGenerator object " + s"initialized with two different partitionIndex [oldValue: ${this.partitionIndex}, " + s"newValue: $partitionIndex]") } } private def assertInitialized(): Unit = if (partitionIndex == -1) { throw SparkException.internalError("PartitionIdentityValueGenerator is not initialized.") } // Generate the next IDENTITY value. def next(): Long = { try { assertInitialized() val ret = nextValue nextValue = Math.addExact(nextValue, increment) ret } catch { case e: ArithmeticException => IdentityOverflowLogger.logOverflow() throw e } } } object IdentityOverflowLogger extends DeltaLogging { def logOverflow(): Unit = { recordEvent( MetricDefinitions.EVENT_TAHOE, Map(TAG_OP_TYPE -> "delta.identityColumn.overflow") ) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/GenerateRowIDs.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.collection.mutable import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project} import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.trees.TreePattern.PLAN_EXPRESSION import org.apache.spark.sql.execution.datasources.{FileFormat, HadoopFsRelation, LogicalRelation, LogicalRelationWithTable} import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.types.StructType /** * This rule adds a Project on top of Delta tables that support the Row tracking table feature to * provide a default generated Row ID and row commit version for rows that don't have them * materialized in the data file. */ object GenerateRowIDs extends Rule[LogicalPlan] { /** * Matcher for a scan on a Delta table that has Row tracking enabled. */ private object DeltaScanWithRowTrackingEnabled { def unapply(plan: LogicalPlan): Option[LogicalRelation] = plan match { case scan @ LogicalRelationWithTable(relation: HadoopFsRelation, _) => relation.fileFormat match { case format: DeltaParquetFileFormat if RowTracking.isEnabled(format.protocol, format.metadata) => Some(scan) case _ => None } case _ => None } } override def apply(plan: LogicalPlan): LogicalPlan = plan.transformUpWithNewOutput { case DeltaScanWithRowTrackingEnabled( scan @ LogicalRelationWithTable(baseRelation: HadoopFsRelation, _)) => // While Row IDs and commit versions are non-nullable, we'll use the Row ID & commit // version attributes to read the materialized values from now on, which can be null. We make // the materialized Row ID & commit version attributes nullable in the scan here. // Update nullability in the scan `metadataOutput` by updating the delta file format. val newFileFormat = baseRelation.fileFormat match { case format: DeltaParquetFileFormat => format.copy(nullableRowTrackingGeneratedFields = true) } val newBaseRelation = baseRelation.copy(fileFormat = newFileFormat)(baseRelation.sparkSession) // Update the output metadata column's data type (now with nullable row tracking fields). val newOutput = scan.output.map { case MetadataAttributeWithLogicalName(metadata, FileFormat.METADATA_NAME) => metadata.withDataType(newFileFormat.createFileMetadataCol().dataType) case other => other } val newScan = scan.copy(relation = newBaseRelation, output = newOutput) newScan.copyTagsFrom(scan) // Add projection with row tracking column expressions. val updatedAttributes = mutable.Buffer.empty[(Attribute, Attribute)] val projectList = newOutput.map { case MetadataAttributeWithLogicalName(metadata, FileFormat.METADATA_NAME) => val updatedMetadata = metadataWithRowTrackingColumnsProjection(metadata) updatedAttributes += metadata -> updatedMetadata.toAttribute updatedMetadata case other => other } Project(projectList = projectList, child = newScan) -> updatedAttributes.toSeq case o => val newPlan = o.transformExpressionsWithPruning(_.containsPattern(PLAN_EXPRESSION)) { // Recurse into subquery plans. Similar to how [[transformUpWithSubqueries]] works except // that it allows us to still use [[transformUpWithNewOutput]] on subquery plans to // correctly update references to the metadata attribute when going up the plan. // Get around type erasure by explicitly checking the plan type and removing warning. case planExpression: PlanExpression[LogicalPlan @unchecked] if planExpression.plan.isInstanceOf[LogicalPlan] => planExpression.withNewPlan(apply(planExpression.plan)) } newPlan -> Nil } /** * Expression that reads the Row IDs from the materialized Row ID column if the value is * present and returns the default generated Row ID using the file's base Row ID and current row * index if not: * coalesce(_metadata.row_id, _metadata.base_row_id + _metadata.row_index). */ private def rowIdExpr(metadata: AttributeReference): Expression = { Coalesce(Seq( getField(metadata, RowId.ROW_ID), Add( getField(metadata, RowId.BASE_ROW_ID), getField(metadata, ParquetFileFormat.ROW_INDEX)))) } /** * Expression that reads the Row commit versions from the materialized Row commit version column * if the value is present and returns the default Row commit version from the file if not: * coalesce(_metadata.row_commit_Version, _metadata.default_row_commit_version). */ private def rowCommitVersionExpr(metadata: AttributeReference): Expression = { Coalesce(Seq( getField(metadata, RowCommitVersion.METADATA_STRUCT_FIELD_NAME), getField(metadata, DefaultRowCommitVersion.METADATA_STRUCT_FIELD_NAME))) } /** * Extract a field from the metadata column. */ private def getField(metadata: AttributeReference, name: String): GetStructField = { ExtractValue(metadata, Literal(name), conf.resolver) match { case field: GetStructField => field case _ => throw new IllegalStateException(s"The metadata column '${metadata.name}' is not a struct.") } } /** * Create a new metadata struct where the Row ID and row commit version values are populated using * the materialized values if present, or the default Row ID / row commit version values if not. */ private def metadataWithRowTrackingColumnsProjection( metadata: AttributeReference): NamedExpression = { val metadataFields = metadata.dataType.asInstanceOf[StructType].map { case field if field.name == RowId.ROW_ID => field -> rowIdExpr(metadata) case field if field.name == RowCommitVersion.METADATA_STRUCT_FIELD_NAME => field -> rowCommitVersionExpr(metadata) case field => field -> getField(metadata, field.name) }.flatMap { case (oldField, newExpr) => // Propagate the type metadata from the old fields to the new fields. val newField = Alias(newExpr, oldField.name)(explicitMetadata = Some(oldField.metadata)) Seq(Literal(oldField.name), newField) } Alias(CreateNamedStruct(metadataFields), metadata.name)() } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/GeneratedColumn.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.util.Locale import org.apache.spark.sql.delta.DataFrameUtils import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions.{Metadata, Protocol} import org.apache.spark.sql.delta.files.{TahoeBatchFileIndex, TahoeFileIndex} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaUtils.quoteIdentifier import org.apache.spark.sql.delta.sources.DeltaSourceUtils.GENERATION_EXPRESSION_METADATA_KEY import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.AnalysisHelper import org.apache.spark.sql.{AnalysisException, Column, Dataset, SparkSession} import org.apache.spark.sql.catalyst.analysis.Analyzer import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke import org.apache.spark.sql.catalyst.optimizer.CollapseProject import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, Project} import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes import org.apache.spark.sql.catalyst.util.{quoteIfNeeded, CaseInsensitiveMap, CharVarcharCodegenUtils} import org.apache.spark.sql.execution.SQLExecution import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.spark.sql.types.{Metadata => FieldMetadata} /** * Provide utility methods to implement Generated Columns for Delta. Users can use the following * SQL syntax to create a table with generated columns. * * ``` * CREATE TABLE table_identifier( * column_name column_type, * column_name column_type GENERATED ALWAYS AS ( generation_expr ), * ... * ) * USING delta * [ PARTITIONED BY (partition_column_name, ...) ] * ``` * * This is an example: * ``` * CREATE TABLE foo( * id bigint, * type string, * subType string GENERATED ALWAYS AS ( SUBSTRING(type FROM 0 FOR 4) ), * data string, * eventTime timestamp, * day date GENERATED ALWAYS AS ( days(eventTime) ) * USING delta * PARTITIONED BY (type, day) * ``` * * When writing to a table, for these generated columns: * - If the output is missing a generated column, we will add an expression to generate it. * - If a generated column exists in the output, in other words, we will add a constraint to ensure * the given value doesn't violate the generation expression. */ object GeneratedColumn extends DeltaLogging with AnalysisHelper { def satisfyGeneratedColumnProtocol(protocol: Protocol): Boolean = protocol.isFeatureSupported(GeneratedColumnsTableFeature) /** * Whether the field contains the generation expression. Note: this doesn't mean the column is a * generated column. A column is a generated column only if the table's * `minWriterVersion` >= `GeneratedColumn.MIN_WRITER_VERSION` and the column metadata contains * generation expressions. Use the other `isGeneratedColumn` to check whether it's a generated * column instead. */ private[delta] def isGeneratedColumn(field: StructField): Boolean = { field.metadata.contains(GENERATION_EXPRESSION_METADATA_KEY) } /** Whether a column is a generated column. */ def isGeneratedColumn(protocol: Protocol, field: StructField): Boolean = { satisfyGeneratedColumnProtocol(protocol) && isGeneratedColumn(field) } /** * Whether any generation expressions exist in the schema. Note: this doesn't mean the table * contains generated columns. A table has generated columns only if its protocol satisfies * Generated Column (listed in Table Features or supported implicitly) and some of columns in * the table schema contain generation expressions. Use `enforcesGeneratedColumns` to check * generated column tables instead. */ def hasGeneratedColumns(schema: StructType): Boolean = { schema.exists(isGeneratedColumn) } /** * Returns the generated columns of a table. A column is a generated column requires: * - The table writer protocol >= GeneratedColumn.MIN_WRITER_VERSION; * - It has a generation expression in the column metadata. */ def getGeneratedColumns(snapshot: SnapshotDescriptor): Seq[StructField] = { if (satisfyGeneratedColumnProtocol(snapshot.protocol)) { snapshot.metadata.schema.partition(isGeneratedColumn)._1 } else { Nil } } /** * Whether the table has generated columns. A table has generated columns only if its * protocol satisfies Generated Column (listed in Table Features or supported implicitly) and * some of columns in the table schema contain generation expressions. * * As Spark will propagate column metadata storing the generation expression through * the entire plan, old versions that don't support generated columns may create tables whose * schema contain generation expressions. However, since these old versions has a lower writer * version, we can use the table's `minWriterVersion` to identify such tables and treat them as * normal tables. * * @param protocol the table protocol. * @param metadata the table metadata. */ def enforcesGeneratedColumns(protocol: Protocol, metadata: Metadata): Boolean = { satisfyGeneratedColumnProtocol(protocol) && metadata.schema.exists(isGeneratedColumn) } /** Return the generation expression from a field metadata if any. */ def getGenerationExpressionStr(metadata: FieldMetadata): Option[String] = { if (metadata.contains(GENERATION_EXPRESSION_METADATA_KEY)) { Some(metadata.getString(GENERATION_EXPRESSION_METADATA_KEY)) } else { None } } /** * Return the generation expression from a field if any. This method doesn't check the protocl. * The caller should make sure the table writer protocol meets `satisfyGeneratedColumnProtocol` * before calling method. */ def getGenerationExpression(field: StructField): Option[Expression] = { getGenerationExpressionStr(field.metadata).map { exprStr => parseGenerationExpression(SparkSession.active, exprStr) } } /** Return the generation expression from a field if any. */ private def getGenerationExpressionStr(field: StructField): Option[String] = { getGenerationExpressionStr(field.metadata) } /** Parse a generation expression string and convert it to an [[Expression]] object. */ private def parseGenerationExpression(spark: SparkSession, exprString: String): Expression = { spark.sessionState.sqlParser.parseExpression(exprString) } /** * SPARK-27561 added support for lateral column alias. This means generation expressions that * reference other generated columns no longer fail analysis in `validateGeneratedColumns`. * * This method checks for and throws an error if: * - A generated column references itself * - A generated column references another generated column */ def validateColumnReferences( spark: SparkSession, fieldName: String, expression: Expression, schema: StructType): Unit = { val allowedBaseColumns = schema .filterNot(_.name == fieldName) // Can't reference itself .filterNot(isGeneratedColumn) // Can't reference other generated columns val relation = new LocalRelation(toAttributes(StructType(allowedBaseColumns))) try { val analyzer: Analyzer = spark.sessionState.analyzer val analyzed = analyzer.execute(Project(Seq(Alias(expression, fieldName)()), relation)) analyzer.checkAnalysis(analyzed) } catch { case ex: AnalysisException => // Improve error message if possible if (ex.getErrorClass == "UNRESOLVED_COLUMN.WITH_SUGGESTION") { throw DeltaErrors.generatedColumnsReferToWrongColumns(ex) } throw ex } } /** * If the schema contains generated columns, check the following unsupported cases: * - Refer to a non-existent column or another generated column. * - Use an unsupported expression. * - The expression type is not the same as the column type. */ def validateGeneratedColumns(spark: SparkSession, schema: StructType): Unit = { val (generatedColumns, normalColumns) = schema.partition(isGeneratedColumn) generatedColumns.foreach { c => // Generated columns cannot be variant types because the writer must be able to enforce that // the <=> . Variants are currently not comprable so // this condition is impossible to enforce. if (c.dataType.isInstanceOf[VariantType]) { throw DeltaErrors.generatedColumnsUnsupportedType(c.dataType) } } // Create a fake relation using the normal columns and add a project with generation expressions // on top of it to ask Spark to analyze the plan. This will help us find out the following // errors: // - Refer to a non existent column in a generation expression. // - Refer to a generated column in another one. val relation = new LocalRelation(toAttributes(StructType(normalColumns))) val selectExprs = generatedColumns.map { f => getGenerationExpressionStr(f) match { case Some(exprString) => val expr = parseGenerationExpression(spark, exprString) validateColumnReferences(spark, f.name, expr, schema) Column(expr).alias(f.name) case None => // Should not happen throw DeltaErrors.expressionsNotFoundInGeneratedColumn(f.name) } } val dfWithExprs = try { val plan = Project(selectExprs.map(_.expr.asInstanceOf[NamedExpression]), relation) DataFrameUtils.ofRows(spark, plan) } catch { case e: AnalysisException if e.getMessage != null => val regexCandidates = Seq( ("A column, variable, or function parameter with name .*?cannot be resolved. " + "Did you mean one of the following?.*?").r, "cannot resolve.*?given input columns:.*?".r, "Column.*?does not exist.".r ) if (regexCandidates.exists(_.findFirstMatchIn(e.getMessage).isDefined)) { throw DeltaErrors.generatedColumnsReferToWrongColumns(e) } else { throw e } } // Check whether the generation expressions are valid dfWithExprs.queryExecution.analyzed.transformAllExpressions { case expr: Alias => // Alias will be non deterministic if it points to a non deterministic expression. // Skip `Alias` to provide a better error for a non deterministic expression. expr case expr @ (_: GetStructField | _: GetArrayItem) => // The complex type extractors don't have a function name, so we need to check them // separately. `GetMapValue` and `GetArrayStructFields` are not supported because Delta // Invariant Check doesn't support them. expr case expr: UserDefinedExpression => throw DeltaErrors.generatedColumnsUDF(expr) case expr if !expr.deterministic => throw DeltaErrors.generatedColumnsNonDeterministicExpression(expr) case expr if expr.isInstanceOf[AggregateExpression] => throw DeltaErrors.generatedColumnsAggregateExpression(expr) case expr if !AllowedUserProvidedExpressions.expressions.contains(expr.getClass) => throw DeltaErrors.generatedColumnsUnsupportedExpression(expr) } // Compare the columns types defined in the schema and the expression types. generatedColumns.zip(dfWithExprs.schema).foreach { case (column, expr) => if (!DataType.equalsIgnoreNullability(column.dataType, expr.dataType)) { throw DeltaErrors.generatedColumnsExprTypeMismatch( column.name, column.dataType, expr.dataType) } } } def getGeneratedColumnsAndColumnsUsedByGeneratedColumns(schema: StructType): Set[String] = { val generationExprs = schema.flatMap { col => getGenerationExpressionStr(col).map { exprStr => val expr = parseGenerationExpression(SparkSession.active, exprStr) Column(expr).alias(col.name) } } if (generationExprs.isEmpty) { return Set.empty } val df = DataFrameUtils.ofRows(SparkSession.active, new LocalRelation(toAttributes(schema))) val generatedColumnsAndColumnsUsedByGeneratedColumns = df.select(generationExprs: _*).queryExecution.analyzed match { case Project(exprs, _) => exprs.flatMap { case Alias(expr, column) => expr.references.map { case a: AttributeReference => a.name case other => // Should not happen since the columns should be resolved throw DeltaErrors.unexpectedAttributeReference(s"$other") }.toSeq :+ column case other => // Should not happen since we use `Alias` expressions. throw DeltaErrors.unexpectedAlias(s"$other") } case other => // Should not happen since `select` should use `Project`. throw DeltaErrors.unexpectedProject(other.toString()) } // Converting columns to lower case is fine since Delta's schema is always case insensitive. generatedColumnsAndColumnsUsedByGeneratedColumns.map(_.toLowerCase(Locale.ROOT)).toSet } private def createFieldPath(nameParts: Seq[String]): String = { nameParts.map(quoteIfNeeded _).mkString(".") } /** * Try to get `OptimizablePartitionExpression`s of a data column when a partition column is * defined as a generated column and refers to this data column. * * @param schema the table schema * @param partitionSchema the partition schema. If a partition column is defined as a generated * column, its column metadata should contain the generation expression. */ def getOptimizablePartitionExpressions( schema: StructType, partitionSchema: StructType): Map[String, Seq[OptimizablePartitionExpression]] = { val partitionGenerationExprs = partitionSchema.flatMap { col => getGenerationExpressionStr(col).map { exprStr => val expr = parseGenerationExpression(SparkSession.active, exprStr) Column(expr).alias(col.name) } } if (partitionGenerationExprs.isEmpty) { return Map.empty } val spark = SparkSession.active val resolver = spark.sessionState.analyzer.resolver // `a.name` comes from the generation expressions which users may use different cases. We // need to normalize it to the same case so that we can group expressions for the same // column name together. val nameNormalizer: String => String = if (spark.sessionState.conf.caseSensitiveAnalysis) x => x else _.toLowerCase(Locale.ROOT) /** * Returns a normalized column name with its `OptimizablePartitionExpression` */ def createExpr(nameParts: Seq[String])(func: => OptimizablePartitionExpression): Option[(String, OptimizablePartitionExpression)] = { if (schema.findNestedField(nameParts, resolver = resolver).isDefined) { Some(nameNormalizer(createFieldPath(nameParts)) -> func) } else { None } } val df = DataFrameUtils.ofRows(SparkSession.active, new LocalRelation(toAttributes(schema))) val extractedPartitionExprs = df.select(partitionGenerationExprs: _*).queryExecution.analyzed match { case Project(exprs, _) => exprs.flatMap { case Alias(expr, partColName) => expr match { case Cast(ExtractBaseColumn(name, TimestampType), DateType, _, _) => createExpr(name)(DatePartitionExpr(partColName)) case Cast(ExtractBaseColumn(name, DateType), DateType, _, _) => createExpr(name)(DatePartitionExpr(partColName)) case Year(ExtractBaseColumn(name, DateType)) => createExpr(name)(YearPartitionExpr(partColName)) case Year(Cast(ExtractBaseColumn(name, TimestampType), DateType, _, _)) => createExpr(name)(YearPartitionExpr(partColName)) case Year(Cast(ExtractBaseColumn(name, DateType), DateType, _, _)) => createExpr(name)(YearPartitionExpr(partColName)) case Month(Cast(ExtractBaseColumn(name, TimestampType), DateType, _, _)) => createExpr(name)(MonthPartitionExpr(partColName)) case DateFormatClass( Cast(ExtractBaseColumn(name, DateType), TimestampType, _, _), StringLiteral(format), _) => format match { case DATE_FORMAT_YEAR_MONTH => createExpr(name)( DateFormatPartitionExpr(partColName, DATE_FORMAT_YEAR_MONTH)) case _ => None } case DateFormatClass(ExtractBaseColumn(name, TimestampType), StringLiteral(format), _) => format match { case DATE_FORMAT_YEAR_MONTH => createExpr(name)( DateFormatPartitionExpr(partColName, DATE_FORMAT_YEAR_MONTH)) case DATE_FORMAT_YEAR_MONTH_DAY => createExpr(name)( DateFormatPartitionExpr(partColName, DATE_FORMAT_YEAR_MONTH_DAY)) case DATE_FORMAT_YEAR_MONTH_DAY_HOUR => createExpr(name)( DateFormatPartitionExpr(partColName, DATE_FORMAT_YEAR_MONTH_DAY_HOUR)) case _ => None } case DayOfMonth(Cast(ExtractBaseColumn(name, TimestampType), DateType, _, _)) => createExpr(name)(DayPartitionExpr(partColName)) case Hour(ExtractBaseColumn(name, TimestampType), _) => createExpr(name)(HourPartitionExpr(partColName)) case Substring(ExtractBaseColumn(name, StringType), IntegerLiteral(pos), IntegerLiteral(len)) => createExpr(name)(SubstringPartitionExpr(partColName, pos, len)) case TruncTimestamp( StringLiteral(format), ExtractBaseColumn(name, TimestampType), _) => createExpr(name)(TimestampTruncPartitionExpr(format, partColName)) case TruncTimestamp( StringLiteral(format), Cast(ExtractBaseColumn(name, DateType), TimestampType, _, _), _) => createExpr(name)(TimestampTruncPartitionExpr(format, partColName)) case ExtractBaseColumn(name, _) => createExpr(name)(IdentityPartitionExpr(partColName)) case TruncDate(ExtractBaseColumn(name, DateType), StringLiteral(format)) => createExpr(name)(TruncDatePartitionExpr(partColName, format)) case TruncDate(Cast( ExtractBaseColumn(name, TimestampType | StringType), DateType, _, _), StringLiteral(format)) => createExpr(name)(TruncDatePartitionExpr(partColName, format)) case _ => None } case other => // Should not happen since we use `Alias` expressions. throw DeltaErrors.unexpectedAlias(s"$other") } case other => // Should not happen since `select` should use `Project`. throw DeltaErrors.unexpectedProject(other.toString()) } extractedPartitionExprs.groupBy(_._1).map { case (name, group) => val groupedExprs = group.map(_._2) val mergedExprs = mergePartitionExpressionsIfPossible(groupedExprs) if (log.isDebugEnabled) { logDebug(s"Optimizable partition expressions for column $name:") mergedExprs.foreach(expr => logDebug(expr.toString)) } name -> mergedExprs } } /** * Merge multiple partition expressions into one if possible. For example, users may define * three partitions columns, `year`, `month` and `day`, rather than defining a single `date` * partition column. Hence, we need to take the multiple partition columns into a single * part to consider when optimizing queries. */ private def mergePartitionExpressionsIfPossible( exprs: Seq[OptimizablePartitionExpression]): Seq[OptimizablePartitionExpression] = { def isRedundantPartitionExpr(f: OptimizablePartitionExpression): Boolean = { f.isInstanceOf[YearPartitionExpr] || f.isInstanceOf[MonthPartitionExpr] || f.isInstanceOf[DayPartitionExpr] || f.isInstanceOf[HourPartitionExpr] } // Take the first option because it's safe to drop other duplicate partition expressions val year = exprs.collect { case y: YearPartitionExpr => y }.headOption val month = exprs.collect { case m: MonthPartitionExpr => m }.headOption val day = exprs.collect { case d: DayPartitionExpr => d }.headOption val hour = exprs.collect { case h: HourPartitionExpr => h }.headOption (year ++ month ++ day ++ hour) match { case Seq( year: YearPartitionExpr, month: MonthPartitionExpr, day: DayPartitionExpr, hour: HourPartitionExpr) => exprs.filterNot(isRedundantPartitionExpr) :+ YearMonthDayHourPartitionExpr(year.yearPart, month.monthPart, day.dayPart, hour.hourPart) case Seq(year: YearPartitionExpr, month: MonthPartitionExpr, day: DayPartitionExpr) => exprs.filterNot(isRedundantPartitionExpr) :+ YearMonthDayPartitionExpr(year.yearPart, month.monthPart, day.dayPart) case Seq(year: YearPartitionExpr, month: MonthPartitionExpr) => exprs.filterNot(isRedundantPartitionExpr) :+ YearMonthPartitionExpr(year.yearPart, month.monthPart) case _ => exprs } } def partitionFilterOptimizationEnabled(spark: SparkSession): Boolean = { spark.sessionState.conf .getConf(DeltaSQLConf.GENERATED_COLUMN_PARTITION_FILTER_OPTIMIZATION_ENABLED) } /** * Try to generate partition filters from data filters if possible. * * @param delta the logical plan that outputs the same attributes as the table schema. This will * be used to resolve auto generated expressions. */ def generatePartitionFilters( spark: SparkSession, snapshot: SnapshotDescriptor, dataFilters: Seq[Expression], delta: LogicalPlan): Seq[Expression] = { if (!satisfyGeneratedColumnProtocol(snapshot.protocol)) { return Nil } if (snapshot.metadata.optimizablePartitionExpressions.isEmpty) { return Nil } val optimizablePartitionExpressions = if (spark.sessionState.conf.caseSensitiveAnalysis) { snapshot.metadata.optimizablePartitionExpressions } else { CaseInsensitiveMap(snapshot.metadata.optimizablePartitionExpressions) } /** * Preprocess the data filter such as reordering to ensure the column name appears on the left * and the literal appears on the right. */ def preprocess(filter: Expression): Expression = filter match { case LessThan(lit: Literal, e: Expression) => GreaterThan(e, lit) case LessThanOrEqual(lit: Literal, e: Expression) => GreaterThanOrEqual(e, lit) case EqualTo(lit: Literal, e: Expression) => EqualTo(e, lit) case GreaterThan(lit: Literal, e: Expression) => LessThan(e, lit) case GreaterThanOrEqual(lit: Literal, e: Expression) => LessThanOrEqual(e, lit) case e => e } /** * Find the `OptimizablePartitionExpression`s of column `a` and apply them to get the partition * filters. */ def toPartitionFilter( nameParts: Seq[String], func: (OptimizablePartitionExpression) => Option[Expression]): Seq[Expression] = { optimizablePartitionExpressions.get(createFieldPath(nameParts)).toSeq.flatMap { exprs => exprs.flatMap(expr => func(expr)) } } val partitionFilters = dataFilters.flatMap { filter => preprocess(filter) match { case LessThan(ExtractBaseColumn(nameParts, _), lit: Literal) => toPartitionFilter(nameParts, _.lessThan(lit)) case LessThanOrEqual(ExtractBaseColumn(nameParts, _), lit: Literal) => toPartitionFilter(nameParts, _.lessThanOrEqual(lit)) case EqualTo(ExtractBaseColumn(nameParts, _), lit: Literal) => toPartitionFilter(nameParts, _.equalTo(lit)) case GreaterThan(ExtractBaseColumn(nameParts, _), lit: Literal) => toPartitionFilter(nameParts, _.greaterThan(lit)) case GreaterThanOrEqual(ExtractBaseColumn(nameParts, _), lit: Literal) => toPartitionFilter(nameParts, _.greaterThanOrEqual(lit)) case IsNull(ExtractBaseColumn(nameParts, _)) => toPartitionFilter(nameParts, _.isNull()) case _ => Nil } } val resolvedPartitionFilters = resolveReferencesForExpressions(spark, partitionFilters, delta) if (log.isDebugEnabled) { logDebug("User provided data filters:") dataFilters.foreach(f => logDebug(f.sql)) logDebug("Auto generated partition filters:") partitionFilters.foreach(f => logDebug(f.sql)) logDebug("Resolved generated partition filters:") resolvedPartitionFilters.foreach(f => logDebug(f.sql)) } val executionId = Option(spark.sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY)) .getOrElse("unknown") recordDeltaEvent( snapshot.deltaLog, "delta.generatedColumns.optimize", data = Map( "executionId" -> executionId, "triggered" -> resolvedPartitionFilters.nonEmpty )) resolvedPartitionFilters } /** * Check whether executing DML (Merge or Update) with this plan as the target plan and * generated column is allowed. * It is already checked by the caller that the table is a Delta table, and that it has * generated columns, so it is not checked again here. * * In general it is allowed to Merge or Update into a temporary view over a Delta table, but * this is not allowed if the table contains a generated column. This is because the generated * column definition is a SQL expression text, and it would not handle any transformation done by * the view (e.g. if the view was `SELECT a as b, b as a FROM table`, the generated column would * not handle the aliasing). * * This function checks if the target plan is a bare reference to the Delta table, or if the * transformations are the result of internal processing introduced not by the user, but * internally during analysis, which need to be taken into account and allowed. * * @param deltaLogicalPlan Target plan of the DML (Merge or Update) * @param conf SQLConf object. * @return true if allowed, * false if DeltaErrors.operationOnTempViewWithGenerateColsNotSupported should be thrown. */ def allowDMLTargetPlan(deltaLogicalPlan: LogicalPlan, conf: SQLConf): Boolean = { // Simple quick path: pure scan. // It is already checked by PreprocessTable{Merge|Update} that this is a Delta scan. deltaLogicalPlan.isInstanceOf[LogicalRelation] || ( CollapseProject(deltaLogicalPlan) match { case Project(projectList, r: LogicalRelation) if conf.readSideCharPadding => // Check if s is a char padding applied to a. def isCharPadding(s: StaticInvoke, a: Attribute): Boolean = { s.staticObject == classOf[CharVarcharCodegenUtils] && s.functionName == "readSidePadding" && s.arguments.size == 2 && (s.arguments(0) match { case arg: Attribute => arg.exprId == a.exprId case _ => false }) } projectList.length == r.output.length && projectList.zip(r.output).forall { // Attribute forwarding. case (p: Attribute, a: Attribute) if p.exprId == a.exprId => true // See Spark's ApplyCharTypePaddingHelper.readSidePadding which applies this projection. // p alias must have the same name as input attribute a, // and be char padding applied to it. case (p: Alias, a: Attribute) if conf.resolver(p.name, a.name) => p.child match { case s: StaticInvoke if isCharPadding(s, a) => true case _ => false } case _ => false } // Pure scan. // It is already checked by PreprocessTable{Merge|Update} that this is a Delta scan. case _: LogicalRelation => true case _ => false } ) } private val DATE_FORMAT_YEAR_MONTH = "yyyy-MM" private val DATE_FORMAT_YEAR_MONTH_DAY = "yyyy-MM-dd" private val DATE_FORMAT_YEAR_MONTH_DAY_HOUR = "yyyy-MM-dd-HH" } /** * Finds the full dot-separated path to a field and the data type of the field. This unifies * handling of nested and non-nested fields, and allows pattern matching on the data type. */ object ExtractBaseColumn { def unapply(e: Expression): Option[(Seq[String], DataType)] = e match { case AttributeReference(name, dataType, _, _) => Some(Seq(name), dataType) case g: GetStructField => g.child match { case ExtractBaseColumn(nameParts, _) => Some(nameParts :+ g.extractFieldName, g.dataType) case _ => None } case _ => None } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/IcebergCompat.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.DeltaConfigs._ import org.apache.spark.sql.delta.actions.{Action, AddFile, Metadata, Protocol} import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.internal.MDC import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.types._ /** * Utils to validate the IcebergCompatV1 table feature, which is responsible for keeping Delta * tables in valid states (see the Delta spec for full invariants, dependencies, and requirements) * so that they are capable of having Delta to Iceberg metadata conversion applied to them. The * IcebergCompatV1 table feature does not implement, specify, or control the actual metadata * conversion; that is handled by the Delta UniForm feature. * * Note that UniForm (Iceberg) depends on IcebergCompatV1, but IcebergCompatV1 does not depend on or * require UniForm (Iceberg). It is perfectly valid for a Delta table to have IcebergCompatV1 * enabled but UniForm (Iceberg) not enabled. */ object IcebergCompatV1 extends IcebergCompatBase( version = 1, icebergFormatVersion = 2, config = DeltaConfigs.ICEBERG_COMPAT_V1_ENABLED, tableFeature = IcebergCompatV1TableFeature, requiredTableProperties = Seq(RequireColumnMapping), incompatibleTableFeatures = Set(DeletionVectorsTableFeature), checks = Seq( CheckOnlySingleVersionEnabled, CheckAddFileHasStats, CheckNoPartitionEvolution, CheckNoListMapNullType, CheckDeletionVectorDisabled, CheckTypeWideningSupported ) ) object IcebergCompatV2 extends IcebergCompatBase( version = 2, icebergFormatVersion = 2, config = DeltaConfigs.ICEBERG_COMPAT_V2_ENABLED, tableFeature = IcebergCompatV2TableFeature, requiredTableProperties = Seq(RequireColumnMapping), incompatibleTableFeatures = Set(DeletionVectorsTableFeature), checks = Seq( CheckOnlySingleVersionEnabled, CheckAddFileHasStats, CheckTypeInV2AllowList, CheckPartitionDataTypeInV2AllowList, CheckNoPartitionEvolution, CheckDeletionVectorDisabled, CheckTypeWideningSupported ) ) /** * All IcebergCompatVx should extend from this base class * * @param version the compat version number * @param icebergFormatVersion iceberg format version written by this compat * @param config the DeltaConfig for this IcebergCompat version * @param requiredTableFeatures a list of table features it relies on * @param requiredTableProperties a list of table properties it relies on. * See [[RequiredDeltaTableProperty]] * @param incompatibleTableFeatures a set of table features it is incompatible * with. Used by [[IcebergCompat.isAnyIncompatibleEnabled]] * @param checks a list of checks this IcebergCompatVx will perform. * @see [[RequiredDeltaTableProperty]] */ case class IcebergCompatBase( version: Int, icebergFormatVersion: Int, config: DeltaConfig[Option[Boolean]], tableFeature: TableFeature, requiredTableProperties: Seq[RequiredDeltaTableProperty[_<:Any]], incompatibleTableFeatures: Set[TableFeature] = Set.empty, checks: Seq[IcebergCompatCheck]) extends DeltaLogging { def isEnabled(metadata: Metadata): Boolean = config.fromMetaData(metadata).getOrElse(false) /** * @return true if the feature should be auto enabled on the table created / updated with * the schema */ def shouldAutoEnable(schema: StructType, properties: Map[String, String]): Boolean = false /** * Expected to be called after the newest metadata and protocol have been ~ finalized. * * Furthermore, this should be called *after* * [[UniversalFormat.enforceIcebergInvariantsAndDependencies]]. * * If you are enabling IcebergCompatV1 and are creating a new table, this method will * automatically upgrade the table protocol to support ColumnMapping and set it to 'name' mode, * too. * * If you are disabling IcebergCompatV1, this method will also disable Universal Format (Iceberg), * if it is enabled. * * @param actions The actions to be committed in the txn. We will only look at the [[AddFile]]s. * * @return tuple of options of (updatedProtocol, updatedMetadata). For either action, if no * updates need to be applied, will return None. */ def enforceInvariantsAndDependencies( spark: SparkSession, catalogTable: Option[CatalogTable], prevSnapshot: Snapshot, newestProtocol: Protocol, newestMetadata: Metadata, operation: Option[DeltaOperations.Operation], actions: Seq[Action]): (Option[Protocol], Option[Metadata]) = { val prevProtocol = prevSnapshot.protocol val prevMetadata = prevSnapshot.metadata val wasEnabled = this.isEnabled(prevMetadata) val isEnabled = this.isEnabled(newestMetadata) val tableId = newestMetadata.id val isCreatingOrReorgTable = UniversalFormat.isCreatingOrReorgTable(operation) (wasEnabled, isEnabled) match { case (_, false) => (None, None) // not enable or disabling, Ignore case (_, true) => // Enabling now or already-enabled val tblFeatureUpdates = scala.collection.mutable.Set.empty[TableFeature] val tblPropertyUpdates = scala.collection.mutable.Map.empty[String, String] // Check we have all required table features tableFeature.requiredFeatures.foreach { f => (prevProtocol.isFeatureSupported(f), newestProtocol.isFeatureSupported(f)) match { case (_, true) => // all good case (false, false) => // txn has not supported it! auto-add the table feature tblFeatureUpdates += f case (true, false) => // txn is removing/un-supporting it! handleDisablingRequiredTableFeature(f) } } // Check we have all required delta table properties requiredTableProperties.foreach { case RequiredDeltaTableProperty( deltaConfig, validator, autoSetValue, autoEnableOnExistingTable) => val newestValue = deltaConfig.fromMetaData(newestMetadata) val newestValueOkay = validator(newestValue) val newestValueExplicitlySet = newestMetadata.configuration.contains(deltaConfig.key) if (!newestValueOkay) { if (!newestValueExplicitlySet && (isCreatingOrReorgTable || autoEnableOnExistingTable)) { // This case covers both CREATE and REPLACE TABLE commands that // did not explicitly specify the required deltaConfig. In these // cases, we set the property automatically. // If autoEnableOnExistingTable = true, it auto sets in all cases tblPropertyUpdates += deltaConfig.key -> autoSetValue } else { // In all other cases, if the property value is not compatible // with the IcebergV1 requirements, we fail handleMissingRequiredTableProperties( deltaConfig.key, newestValue.toString, autoSetValue) } } } // Update Protocol and Metadata if necessary val protocolResult = if (tblFeatureUpdates.nonEmpty) { logInfo(log"[tableId=${MDC(DeltaLogKeys.TABLE_ID, tableId)}] " + log"IcebergCompatV1 auto-supporting table features: " + log"${MDC(DeltaLogKeys.TABLE_FEATURES, tblFeatureUpdates.map(_.name))}") Some(newestProtocol.merge(tblFeatureUpdates.map(Protocol.forTableFeature).toSeq: _*)) } else None val metadataResult = if (tblPropertyUpdates.nonEmpty) { logInfo(log"[tableId=${MDC(DeltaLogKeys.TABLE_ID, tableId)}] " + log"IcebergCompatV1 auto-setting table properties: " + log"${MDC(DeltaLogKeys.TBL_PROPERTIES, tblPropertyUpdates)}") val newConfiguration = newestMetadata.configuration ++ tblPropertyUpdates.toMap var tmpNewMetadata = newestMetadata.copy(configuration = newConfiguration) requiredTableProperties.foreach { tp => tmpNewMetadata = tp.postProcess(prevMetadata, tmpNewMetadata, isCreatingOrReorgTable) } Some(tmpNewMetadata) } else None // Apply additional checks val context = IcebergCompatContext( spark, catalogTable, prevSnapshot, protocolResult.getOrElse(newestProtocol), metadataResult.getOrElse(newestMetadata), operation, actions, tableId, version ) checks.foreach(_.apply(context)) (protocolResult, metadataResult) } } protected def handleMissingTableFeature(feature: TableFeature): Unit = throw DeltaErrors.icebergCompatMissingRequiredTableFeatureException(version, feature) protected def handleDisablingRequiredTableFeature(feature: TableFeature): Unit = throw DeltaErrors.icebergCompatDisablingRequiredTableFeatureException(version, feature) protected def handleMissingRequiredTableProperties( confKey: String, actualVal: String, requiredVal: String): Unit = throw DeltaErrors.icebergCompatWrongRequiredTablePropertyException( version, confKey, actualVal, requiredVal) } /** * Util methods to manage between IcebergCompat versions */ case class IcebergCompatVersionBase(knownVersions: Set[IcebergCompatBase]) { /** * Fetch from Metadata the current enabled IcebergCompat version. * @return a number indicate the version. E.g., 1 for CompatV1. * None if no version enabled. */ def getEnabledVersion(metadata: Metadata): Option[Int] = knownVersions .find{ _.config.fromMetaData(metadata).getOrElse(false) } .map{ _.version } /** * Get the IcebergCompat by version. If version is not valid, * throw an exception. * @return the IcebergCompatVx object */ def getForVersion(version: Int): IcebergCompatBase = knownVersions .find(_.version == version) .getOrElse( throw DeltaErrors.icebergCompatVersionNotSupportedException( version, knownVersions.size ) ) /** * @return any enabled IcebergCompat in the conf */ def anyEnabled(conf: Map[String, String]): Option[IcebergCompatBase] = knownVersions.find { compat => conf.getOrElse[String](compat.config.key, "false").toBoolean } def anyEnabled(metadata: Metadata): Option[IcebergCompatBase] = knownVersions.find { _.config.fromMetaData(metadata).getOrElse(false) } /** * @return true if any version of IcebergCompat is enabled */ def isAnyEnabled(conf: Map[String, String]): Boolean = anyEnabled(conf).nonEmpty def isAnyEnabled(metadata: Metadata): Boolean = knownVersions.exists { _.config.fromMetaData(metadata).getOrElse(false) } /** * @return true if a CompatVx greater or eq to the required version is enabled */ def isGeqEnabled(metadata: Metadata, requiredVersion: Int): Boolean = anyEnabled(metadata).exists(_.version >= requiredVersion) /** * @return true if any version of IcebergCompat is enabled, and is incompatible * with the given table feature */ def isAnyIncompatibleEnabled( configuration: Map[String, String], feature: TableFeature): Boolean = knownVersions.exists { compat => configuration.getOrElse[String](compat.config.key, "false").toBoolean && compat.incompatibleTableFeatures.contains(feature) } } object IcebergCompat extends IcebergCompatVersionBase( Set(IcebergCompatV1, IcebergCompatV2) ) with DeltaLogging /** * Wrapper class for table property validation * * @param deltaConfig [[DeltaConfig]] we are checking * @param validator A generic method to validate the given value * @param autoSetValue The value to set if we can auto-set this value * @param autoEnableOnExistingTable this can be true only when the feature * can be confidently enabled on existing table */ case class RequiredDeltaTableProperty[T]( deltaConfig: DeltaConfig[T], validator: T => Boolean, autoSetValue: String, autoEnableOnExistingTable: Boolean = false) { /** * A callback after all required properties are added to the new metadata. * @return Updated metadata. None if no change */ def postProcess( prevMetadata: Metadata, newMetadata: Metadata, isCreatingNewTable: Boolean) : Metadata = newMetadata } class RequireColumnMapping(allowedModes: Seq[DeltaColumnMappingMode]) extends RequiredDeltaTableProperty( deltaConfig = DeltaConfigs.COLUMN_MAPPING_MODE, validator = (mode: DeltaColumnMappingMode) => allowedModes.contains(mode), autoSetValue = if (allowedModes.contains(NameMapping)) NameMapping.name else IdMapping.name) { override def postProcess( prevMetadata: Metadata, newMetadata: Metadata, isCreatingNewTable: Boolean): Metadata = { if (!prevMetadata.configuration.contains(DeltaConfigs.COLUMN_MAPPING_MODE.key) && newMetadata.configuration.contains(DeltaConfigs.COLUMN_MAPPING_MODE.key)) { val tmpNewMetadata = DeltaColumnMapping.assignColumnIdAndPhysicalName( newMetadata = newMetadata, oldMetadata = prevMetadata, isChangingModeOnExistingTable = false, isOverwritingSchema = false ) DeltaColumnMapping.checkColumnIdAndPhysicalNameAssignments(tmpNewMetadata) tmpNewMetadata } else { newMetadata } } } object RequireColumnMapping extends RequireColumnMapping(Seq(NameMapping, IdMapping)) case class IcebergCompatContext( spark: SparkSession, catalogTable: Option[CatalogTable], prevSnapshot: Snapshot, newestProtocol: Protocol, newestMetadata: Metadata, operation: Option[DeltaOperations.Operation], actions: Seq[Action], tableId: String, version: Integer) { def prevMetadata: Metadata = prevSnapshot.metadata def prevProtocol: Protocol = prevSnapshot.protocol } trait IcebergCompatCheck extends (IcebergCompatContext => Unit) /** * Checks that ensures no more than one IcebergCompatVx is enabled. */ object CheckOnlySingleVersionEnabled extends IcebergCompatCheck { override def apply(context: IcebergCompatContext): Unit = { val numEnabled = IcebergCompat.knownVersions.toSeq .map { compat => if (compat.isEnabled(context.newestMetadata)) 1 else 0 }.sum if (numEnabled > 1) { throw DeltaErrors.icebergCompatVersionMutualExclusive(context.version) } } } object CheckAddFileHasStats extends IcebergCompatCheck { override def apply(context: IcebergCompatContext): Unit = { // If this field is empty, then the AddFile is missing the `numRecords` statistic. context.actions.collect { case a: AddFile if a.numLogicalRecords.isEmpty => throw new UnsupportedOperationException(s"[tableId=${context.tableId}] " + s"IcebergCompatV${context.version} requires all AddFiles to contain " + s"the numRecords statistic. AddFile ${a.path} is missing this statistic. " + s"Stats: ${a.stats}") } } } object CheckNoPartitionEvolution extends IcebergCompatCheck { override def apply(context: IcebergCompatContext): Unit = { // Note: Delta doesn't support partition evolution, but you can change the partitionColumns // by doing a REPLACE or DataFrame overwrite. // // Iceberg-Spark itself *doesn't* support the following cases // - CREATE TABLE partitioned by colA; REPLACE TABLE partitioned by colB // - CREATE TABLE partitioned by colA; REPLACE TABLE not partitioned // // While Iceberg-Spark *does* support // - CREATE TABLE not partitioned; REPLACE TABLE not partitioned // - CREATE TABLE not partitioned; REPLACE TABLE partitioned by colA // - CREATE TABLE partitioned by colA dataType1; REPLACE TABLE partitioned by colA dataType2 if (context.prevMetadata.partitionColumns.nonEmpty && context.prevMetadata.partitionColumns != context.newestMetadata.partitionColumns) { throw DeltaErrors.icebergCompatReplacePartitionedTableException( context.version, context.prevMetadata.partitionColumns, context.newestMetadata.partitionColumns) } } } object CheckNoListMapNullType extends IcebergCompatCheck { override def apply(context: IcebergCompatContext): Unit = { SchemaUtils.findAnyTypeRecursively(context.newestMetadata.schema) { f => f.isInstanceOf[MapType] || f.isInstanceOf[ArrayType] || f.isInstanceOf[NullType] } match { case Some(unsupportedType) => throw DeltaErrors.icebergCompatUnsupportedDataTypeException( context.version, unsupportedType, context.newestMetadata.schema) case _ => } } } class CheckTypeInAllowList extends IcebergCompatCheck { def allowTypes: Set[Class[_]] = Set() override def apply(context: IcebergCompatContext): Unit = { SchemaUtils .findAnyTypeRecursively(context.newestMetadata.schema)(t => !allowTypes.contains(t.getClass)) match { case Some(unsupportedType) => throw DeltaErrors.icebergCompatUnsupportedDataTypeException( context.version, unsupportedType, context.newestMetadata.schema) case _ => } } } object CheckTypeInV2AllowList extends CheckTypeInAllowList { override val allowTypes: Set[Class[_]] = Set[Class[_]] ( ByteType.getClass, ShortType.getClass, IntegerType.getClass, LongType.getClass, FloatType.getClass, DoubleType.getClass, classOf[DecimalType], StringType.getClass, BinaryType.getClass, BooleanType.getClass, TimestampType.getClass, TimestampNTZType.getClass, DateType.getClass, classOf[ArrayType], classOf[MapType], classOf[StructType]) } object CheckPartitionDataTypeInV2AllowList extends IcebergCompatCheck { private val allowedTypes = Set[Class[_]] ( ByteType.getClass, ShortType.getClass, IntegerType.getClass, LongType.getClass, FloatType.getClass, DoubleType.getClass, DecimalType.getClass, StringType.getClass, BinaryType.getClass, BooleanType.getClass, TimestampType.getClass, TimestampNTZType.getClass, DateType.getClass ) override def apply(context: IcebergCompatContext): Unit = { val partitionSchema = context.newestMetadata.partitionSchema partitionSchema.fields.find(field => !allowedTypes.contains(field.dataType.getClass)) match { case Some(field) => throw DeltaErrors.icebergCompatUnsupportedPartitionDataTypeException( context.version, field.dataType, partitionSchema) case _ => } } } /** * Check if the deletion vector has been disabled by previous snapshot * or newest metadata and protocol depending on whether the operation * is REORG UPGRADE UNIFORM or not. */ object CheckDeletionVectorDisabled extends IcebergCompatCheck { override def apply(context: IcebergCompatContext): Unit = { if (context.newestProtocol.isFeatureSupported(DeletionVectorsTableFeature)) { // note: user will need to *separately* disable deletion vectors if this check fails, // i.e., ALTER TABLE SET TBLPROPERTIES ('delta.enableDeletionVectors' = 'false'); val isReorgUpgradeUniform = UniversalFormat.isReorgUpgradeUniform(context.operation) // for REORG UPGRADE UNIFORM, we only need to check whether DV // is enabled in the newest metadata and protocol, this conforms with // the semantics of REORG UPGRADE UNIFORM, which will automatically disable // DV and rewrite all the parquet files with DV removed as for now. if (isReorgUpgradeUniform) { if (DeletionVectorUtils.deletionVectorsWritable( protocol = context.newestProtocol, metadata = context.newestMetadata )) { throw DeltaErrors.icebergCompatDeletionVectorsShouldBeDisabledException(context.version) } } else { // for other commands, we need to check whether DV is disabled from the // previous snapshot, in case there are concurrent writers. // plus, we also need to check from the newest metadata and protocol, // in case we are creating a new uniform table with DV enabled. if (DeletionVectorUtils.deletionVectorsWritable(context.prevSnapshot) || DeletionVectorUtils.deletionVectorsWritable( protocol = context.newestProtocol, metadata = context.newestMetadata )) { throw DeltaErrors.icebergCompatDeletionVectorsShouldBeDisabledException(context.version) } } } } } /** * Checks that the table didn't go through any type changes that Iceberg doesn't support. See * `TypeWidening.isTypeChangeSupportedByIceberg()` for supported type changes. * Note that this check covers both: * - When the table had an unsupported type change applied in the past and Uniform is being enabled. * - When Uniform is enabled and a new, unsupported type change is being applied. */ object CheckTypeWideningSupported extends IcebergCompatCheck { override def apply(context: IcebergCompatContext): Unit = { val skipCheck = context.spark.sessionState.conf .getConf(DeltaSQLConf.DELTA_TYPE_WIDENING_ALLOW_UNSUPPORTED_ICEBERG_TYPE_CHANGES) if (skipCheck || !TypeWidening.isSupported(context.newestProtocol)) return TypeWideningMetadata.getAllTypeChanges(context.newestMetadata.schema).foreach { case (fieldPath, TypeChange(_, fromType: AtomicType, toType: AtomicType, _)) // We ignore type changes that are not generally supported with type widening to reduce the // risk of this check misfiring. These are handled by `TypeWidening.assertTableReadable()`. // The error here only captures type changes that are supported in Delta but not Iceberg. if TypeWidening.isTypeChangeSupported(fromType, toType) && !TypeWidening.isTypeChangeSupportedByIceberg(fromType, toType) => throw DeltaErrors.icebergCompatUnsupportedTypeWideningException( context.version, fieldPath, fromType, toType) case _ => () // ignore } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/IdentityColumn.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.collection.mutable import org.apache.spark.sql.delta.DataFrameUtils import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSourceUtils._ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.{DeltaFileStatistics, DeltaJobStatisticsTracker} import org.apache.spark.sql.delta.util.JsonUtils import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.sql.{Column, DataFrame, Dataset, SparkSession} import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan} import org.apache.spark.sql.execution.datasources.WriteTaskStats import org.apache.spark.sql.functions.{array, max, min, to_json} import org.apache.spark.sql.types.{MetadataBuilder, StructField, StructType} /** * This object holds String constants used the field `debugInfo` for * logging [[IdentityColumn.opTypeHighWaterMarkUpdate]]. * Each string represents an unexpected or notable event while calculating the high water mark. */ object IdentityColumnHighWaterMarkUpdateInfo { val EXISTING_WATER_MARK_BEFORE_START = "existing_water_mark_before_start" val CANDIDATE_HIGH_WATER_MARK_ROUNDED = "candidate_high_watermark_rounded" val CANDIDATE_HIGH_WATER_MARK_BEFORE_START = "candidate_high_water_mark_before_start" } /** * Provide utility methods related to IDENTITY column support for Delta. */ object IdentityColumn extends DeltaLogging { case class IdentityInfo(start: Long, step: Long, highWaterMark: Option[Long]) // Default start and step configuration if not specified by user. val defaultStart = 1 val defaultStep = 1 // Operation types in usage logs. // When IDENTITY columns are defined. val opTypeDefinition = "delta.identityColumn.definition" // When table with IDENTITY columns are written into. val opTypeWrite = "delta.identityColumn.write" // When IDENTITY column update causes transaction to abort. val opTypeAbort = "delta.identityColumn.abort" // When we update the high watermark of an IDENTITY column. val opTypeHighWaterMarkUpdate = "delta.identityColumn.highWaterMarkUpdate" // Return true if `field` is an identity column that allows explicit insert. Caller must ensure // `isIdentityColumn(field)` is true. def allowExplicitInsert(field: StructField): Boolean = { field.metadata.getBoolean(IDENTITY_INFO_ALLOW_EXPLICIT_INSERT) } // Return all the IDENTITY columns from `schema`. def getIdentityColumns(schema: StructType): Seq[StructField] = { schema.filter(ColumnWithDefaultExprUtils.isIdentityColumn) } // Return the number of IDENTITY columns in `schema`. private def getNumberOfIdentityColumns(schema: StructType): Int = { getIdentityColumns(schema).size } // Create expression to generate IDENTITY values for the column `field`. def createIdentityColumnGenerationExpr(field: StructField): Expression = { val info = IdentityColumn.getIdentityInfo(field) GenerateIdentityValues(info.start, info.step, info.highWaterMark) } // Create a column to generate IDENTITY values for the column `field`. def createIdentityColumnGenerationExprAsColumn(field: StructField): Column = { Column(createIdentityColumnGenerationExpr(field)).alias(field.name) } /** * Create a stats tracker to collect IDENTITY column high water marks if its values are system * generated. * * @param spark The SparkSession associated with this query. * @param hadoopConf The Hadoop configuration object to use on an executor. * @param path Root Reservoir path * @param schema The schema of the table to be written into. * @param statsDataSchema The schema of the output data (this does not include partition columns). * @param trackHighWaterMarks Column names for which we should track high water marks. * @return The stats tracker. */ def createIdentityColumnStatsTracker( spark: SparkSession, hadoopConf: Configuration, path: Path, schema: StructType, statsDataSchema: Seq[Attribute], trackHighWaterMarks: Set[String] ) : Option[DeltaIdentityColumnStatsTracker] = { if (trackHighWaterMarks.isEmpty) return None val identityColumnInfo = schema .filter(f => trackHighWaterMarks.contains(f.name)) .map(f => DeltaColumnMapping.getPhysicalName(f) -> // Get identity column physical names (f.metadata.getLong(IDENTITY_INFO_STEP) > 0L)) // We should have found all IDENTITY columns to track high water marks. assert(identityColumnInfo.size == trackHighWaterMarks.size, s"expect: $trackHighWaterMarks, found (physical names): ${identityColumnInfo.map(_._1)}") // Build the expression to collect high water marks of all IDENTITY columns as a single // expression. It is essentially a json array containing one max or min aggregate expression // for each IDENTITY column. // // Example: for the following table // // CREATE TABLE t1 ( // id1 BIGINT GENERATED BY DEFAULT AS IDENTITY (START WITH 1 INCREMENT BY 1), // id2 BIGINT GENERATED BY DEFAULT AS IDENTITY (START WITH 1 INCREMENT BY -1), // value STRING // ) USING delta; // // The expression will be: to_json(array(max(id1), min(id2))) val aggregates = identityColumnInfo.map { case (name, positiveStep) => val col = Column(UnresolvedAttribute.quoted(name)) if (positiveStep) max(col) else min(col) } val unresolvedExpr = to_json(array(aggregates: _*)) // Resolve the collection expression by constructing a query to select the expression from a // table with the statsSchema and get the analyzed expression. val resolvedPlan = DataFrameUtils.ofRows(spark, LocalRelation(statsDataSchema)) .select(unresolvedExpr).queryExecution.analyzed // We have to use the new attributes with regenerated attribute IDs, because the Analyzer // doesn't guarantee that attributes IDs will stay the same val newStatsDataSchema = resolvedPlan.children.head.output Some(new DeltaIdentityColumnStatsTracker( hadoopConf, path, newStatsDataSchema, resolvedPlan.expressions.head, identityColumnInfo )) } /** Round `value` to the next value that follows start and step configuration. */ protected[delta] def roundToNext(start: Long, step: Long, value: Long): Long = { val valueOffset = Math.subtractExact(value, start) if (valueOffset % step == 0) { value } else { // An identity value follows the formula start + step * n. So n = (value - start) / step. // Where n is a non-negative integer if the value respects the start. // Since the value doesn't follow this formula, we need to ceil n. // corrected value = start + step * ceil(n). // However, we can't cast to Double for division because it's only accurate up to 54 bits. // Instead, we will do a floored division and add 1. // start + step * ((value - start) / step + 1) val quotient = valueOffset / step // `valueOffset` will have the same sign as `step` if `value` respects the start. val stepMultiple = if (Math.signum(valueOffset) == Math.signum(step)) { Math.addExact(quotient, 1L) } else { // Don't add one. Otherwise, we end up rounding 2 values up, which may skip the start. quotient } Math.addExact( start, Math.multiplyExact(step, stepMultiple) ) } } /** * Update the high water mark of the IDENTITY column based on `candidateHighWaterMark`. * * We validate against the identity column definition (start, step) and may insert a high * watermark that's different from `candidateHighWaterMark` if it's not valid. This method * may also not update the high watermark if the candidate doesn't respect the start, is * below the current watermark or is a NOOP. * * @param field The IDENTITY column to update. * @param candidateHighWaterMark The candidate high water mark to update to. * @param allowLoweringHighWaterMarkForSyncIdentity Whether to allow lowering the high water mark. * Lowering the high water mark is NOT SAFE in * general, but may be a valid operation in SYNC * IDENTITY (e.g. repair a high water mark after * a bad sync). * @return A new `StructField` with the high water mark updated to `candidateHighWaterMark` and * a Seq[String] that contains debug information for logging. */ protected[delta] def updateToValidHighWaterMark( field: StructField, candidateHighWaterMark: Long, allowLoweringHighWaterMarkForSyncIdentity: Boolean ): (StructField, Seq[String]) = { require(ColumnWithDefaultExprUtils.isIdentityColumn(field)) val info = getIdentityInfo(field) val positiveStep = info.step > 0 val orderInStepDirection = if (positiveStep) Ordering.Long else Ordering.Long.reverse val loggingBuffer = new mutable.ArrayBuffer[String] // We check `candidateHighWaterMark` and not `newHighWaterMark` because // newHighWaterMark may not be part of the column. E.g. a generated by default column // has candidateHighWaterMark = 9, start = 10, step = 3, and previous highWaterMark = None. // We don't want to bump the high water mark to 10 because the next value generated will // be 13, and we'll miss the specified start entirely. val isBeforeStart = orderInStepDirection.lt(candidateHighWaterMark, info.start) if (isBeforeStart) { loggingBuffer.append( IdentityColumnHighWaterMarkUpdateInfo.CANDIDATE_HIGH_WATER_MARK_BEFORE_START) } // We must round on the generated by default case because the candidate may be a user inserted // value and may not follow the identity column definition. We're not skipping this check // for the generated always case. It's effectively a NOOP since generated always values should // theoretically always respect the identity column definition. If the high watermark was // wrong (for some reason), this is our chance to fix it. val roundedCandidateHighWaterMark = roundToNext(info.start, info.step, candidateHighWaterMark) if (roundedCandidateHighWaterMark != candidateHighWaterMark) { loggingBuffer.append(IdentityColumnHighWaterMarkUpdateInfo.CANDIDATE_HIGH_WATER_MARK_ROUNDED) } // If allowLoweringHighWaterMarkForSyncIdentity is true, we can ignore the existing high water // mark. val newHighWaterMark = info.highWaterMark match { case Some(oldWaterMark) if !allowLoweringHighWaterMarkForSyncIdentity => orderInStepDirection.max(oldWaterMark, roundedCandidateHighWaterMark) case _ => roundedCandidateHighWaterMark } val tableHasBadHighWaterMark = info.highWaterMark.exists(oldWaterMark => orderInStepDirection.lt(oldWaterMark, info.start)) if (tableHasBadHighWaterMark) { loggingBuffer.append( IdentityColumnHighWaterMarkUpdateInfo.EXISTING_WATER_MARK_BEFORE_START) } val isChanged = !info.highWaterMark.contains(newHighWaterMark) // If a table already has a bad high water mark, we shouldn't prevent them from updating the // high water mark. Always try to update to newHighWaterMark, which is guaranteed to be a better // choice than the existing one since we do a max(). // Note that means if a table has bad water mark, we can set the high water to the start due to // the rounding logic. // Don't update if it's before start or the high watermark is the same. if (tableHasBadHighWaterMark || (!isBeforeStart && isChanged)) { val newMetadata = new MetadataBuilder().withMetadata(field.metadata) .putLong(IDENTITY_INFO_HIGHWATERMARK, newHighWaterMark) .build() (field.copy(metadata = newMetadata), loggingBuffer.toIndexedSeq) } else { // If we don't update the high watermark, we don't need to log the update. (field, Nil) } } /** * Return a new schema with IDENTITY high water marks updated in the schema. * The new high watermarks are decided based on the `updatedIdentityHighWaterMarks` and old high * watermark values present in the passed `schema`. */ def updateSchema( deltaLog: DeltaLog, schema: StructType, updatedIdentityHighWaterMarks: Seq[(String, Long)] ): StructType = { val updatedIdentityHighWaterMarksGrouped = updatedIdentityHighWaterMarks.groupBy(_._1).mapValues(v => v.map(_._2)) StructType(schema.map { f => updatedIdentityHighWaterMarksGrouped.get(DeltaColumnMapping.getPhysicalName(f)) match { case Some(newWatermarks) if ColumnWithDefaultExprUtils.isIdentityColumn(f) => val oldIdentityInfo = getIdentityInfo(f) val positiveStep = oldIdentityInfo.step > 0 val candidateHighWaterMark = if (positiveStep) { newWatermarks.max } else { newWatermarks.min } val (newField, loggingSeq) = updateToValidHighWaterMark( f, candidateHighWaterMark, allowLoweringHighWaterMarkForSyncIdentity = false) if (loggingSeq.nonEmpty) { recordDeltaEvent( deltaLog = deltaLog, opType = opTypeHighWaterMarkUpdate, data = Map( "columnName" -> f.name, "debugInfo" -> loggingSeq.mkString(", "), "oldHighWaterMark" -> oldIdentityInfo.highWaterMark, "candidateHighWaterMark" -> candidateHighWaterMark, "updatedFrom" -> "updateSchema" ) ) } newField case _ => f } }) } // Block explicitly provided IDENTITY values if column definition does not allow so. def blockExplicitIdentityColumnInsert( schema: StructType, query: LogicalPlan): Unit = { val nonInsertableIdentityColumns = schema.filter { f => ColumnWithDefaultExprUtils.isIdentityColumn(f) && !IdentityColumn.allowExplicitInsert(f) }.map(_.name) blockIdentityColumn( nonInsertableIdentityColumns, query.output.map(attr => Seq(attr.name)), isUpdate = false ) } // Block explicitly provided IDENTITY values if column definition does not allow so. def blockExplicitIdentityColumnInsert( identityColumns: Seq[StructField], insertedColNameParts: Seq[Seq[String]]): Unit = { val nonInsertableIdentityColumns = identityColumns .filter(!allowExplicitInsert(_)) .map(_.name) blockIdentityColumn( nonInsertableIdentityColumns, insertedColNameParts, isUpdate = false) } // Block updating IDENTITY columns. def blockIdentityColumnUpdate( schema: StructType, updatedColNameParts: Seq[Seq[String]]): Unit = { blockIdentityColumnUpdate(getIdentityColumns(schema), updatedColNameParts) } // Block updating IDENTITY columns. def blockIdentityColumnUpdate( identityColumns: Seq[StructField], updatedColNameParts: Seq[Seq[String]]): Unit = { blockIdentityColumn( identityColumns.map(_.name), updatedColNameParts, isUpdate = true) } def logTableCreation(deltaLog: DeltaLog, schema: StructType): Unit = { val numIdentityColumns = getNumberOfIdentityColumns(schema) if (numIdentityColumns != 0) { recordDeltaEvent( deltaLog, opTypeDefinition, data = Map( "numIdentityColumns" -> numIdentityColumns ) ) } } def logTableWrite( snapshot: Snapshot, generatedIdentityColumns: Set[String], numInsertedRowsOpt: Option[Long]): Unit = { val identityColumns = getIdentityColumns(snapshot.schema) if (identityColumns.nonEmpty) { val explicitIdentityColumns = identityColumns.filter { f => !generatedIdentityColumns.contains(f.name) }.map(_.name) recordDeltaEvent( snapshot.deltaLog, opTypeWrite, data = Map( "numInsertedRows" -> numInsertedRowsOpt, "generatedIdentityColumnNames" -> generatedIdentityColumns.mkString(","), "generatedIdentityColumnCount" -> generatedIdentityColumns.size, "explicitIdentityColumnNames" -> explicitIdentityColumns.mkString(","), "explicitIdentityColumnCount" -> explicitIdentityColumns.size ) ) } } def logTransactionAbort(deltaLog: DeltaLog): Unit = { recordDeltaEvent(deltaLog, opTypeAbort) } // Calculate the sync'ed IDENTITY high water mark based on actual data and returns a // potentially updated `StructField`. def syncIdentity( deltaLog: DeltaLog, field: StructField, df: DataFrame, allowLoweringHighWaterMarkForSyncIdentity: Boolean ): StructField = { assert(ColumnWithDefaultExprUtils.isIdentityColumn(field)) // Run a query to get the actual high water mark (max or min value of the IDENTITY column) from // the actual data. val info = getIdentityInfo(field) val positiveStep = info.step > 0 val expr = if (positiveStep) max(field.name) else min(field.name) val resultRow = df.select(expr).collect().head if (!resultRow.isNullAt(0)) { val candidateHighWaterMark = resultRow.getLong(0) val (newField, loggingSeq) = updateToValidHighWaterMark( field, candidateHighWaterMark, allowLoweringHighWaterMarkForSyncIdentity) if (loggingSeq.nonEmpty) { recordDeltaEvent( deltaLog = deltaLog, opType = opTypeHighWaterMarkUpdate, data = Map( "columnName" -> field.name, "debugInfo" -> loggingSeq.mkString(", "), "oldHighWaterMark" -> info.highWaterMark, "candidateHighWaterMark" -> candidateHighWaterMark, "updatedFrom" -> "syncIdentity" ) ) } newField } else { field } } /** * Returns a copy of `schemaToCopy` in which the high water marks of the identity columns have * been merged with the corresponding high water marks of `schemaWithHighWaterMarksToMerge`. */ def copySchemaWithMergedHighWaterMarks( deltaLog: DeltaLog, schemaToCopy: StructType, schemaWithHighWaterMarksToMerge: StructType ): StructType = { val newHighWatermarks = getIdentityColumns(schemaWithHighWaterMarksToMerge).flatMap { f => val info = getIdentityInfo(f) info.highWaterMark.map(waterMark => DeltaColumnMapping.getPhysicalName(f) -> waterMark) } updateSchema( deltaLog, schemaToCopy, newHighWatermarks ) } // Check `colNameParts` does not contain any column from `columnNamesToBlock`. private def blockIdentityColumn( columnNamesToBlock: Seq[String], colNameParts: Seq[Seq[String]], isUpdate: Boolean): Unit = { if (columnNamesToBlock.nonEmpty) { val resolver = SparkSession.active.sessionState.analyzer.resolver for (namePart <- colNameParts) { // IDENTITY column cannot be nested columns, so we only need to check top level columns. if (namePart.size == 1) { val colName = namePart.head if (columnNamesToBlock.exists(resolver(_, colName))) { if (isUpdate) { throw DeltaErrors.identityColumnUpdateNotSupported(colName) } else { throw DeltaErrors.identityColumnExplicitInsertNotSupported(colName) } } } } } } // Return IDENTITY information of column `field`. Caller must ensure `isIdentityColumn(field)` // is true. def getIdentityInfo(field: StructField): IdentityInfo = { val md = field.metadata val start = md.getLong(IDENTITY_INFO_START) val step = md.getLong(IDENTITY_INFO_STEP) // If system hasn't generated IDENTITY values for this column (either it hasn't been // inserted into, or every inserts provided values for this IDENTITY column), high water mark // field will not present in column metadata. In this case, high water mark will be set to // (start - step) so that the first value generated is start (high water mark + step). val highWaterMark = if (md.contains(IDENTITY_INFO_HIGHWATERMARK)) { Some(md.getLong(IDENTITY_INFO_HIGHWATERMARK)) } else { None } IdentityInfo(start, step, highWaterMark) } } /** * Stats tracker for IDENTITY column high water marks. The only difference between this class and * `DeltaJobStatisticsTracker` is how the stats are aggregated on the driver. * * @param hadoopConf The Hadoop configuration object to use on an executor. * @param path Root Reservoir path * @param dataCols Resolved data (i.e. non-partitionBy) columns of the dataframe to be written. * @param statsColExpr The expression to collect high water marks. * @param identityColumnInfo Information of IDENTITY columns. It contains a pair of column name * and whether it has a positive step for each IDENTITY column. */ class DeltaIdentityColumnStatsTracker( @transient private val hadoopConf: Configuration, @transient path: Path, dataCols: Seq[Attribute], statsColExpr: Expression, val identityColumnInfo: Seq[(String, Boolean)] ) extends DeltaJobStatisticsTracker( hadoopConf, path, dataCols, statsColExpr ) { // Map of column name to its corresponding collected high water mark. var highWaterMarks = scala.collection.mutable.Map[String, Long]() // Process the stats on the driver. In `stats` we have a sequence of `DeltaFileStatistics`, // whose stats is a map of file path to its corresponding array of high water marks in json. override def processStats(stats: Seq[WriteTaskStats], jobCommitTime: Long): Unit = { stats.map(_.asInstanceOf[DeltaFileStatistics]).flatMap(_.stats).map { case (_, statsString) => val fileHighWaterMarks = JsonUtils.fromJson[Array[Long]](statsString) // We must have high water marks collected for all IDENTITY columns and we have guaranteed // that their orders in the array follow the orders in `identityInfo` by aligning the // order of expression and `identityColumnInfo` in `createIdentityColumnStatsTracker`. require(fileHighWaterMarks.size == identityColumnInfo.size) identityColumnInfo.zip(fileHighWaterMarks).map { case ((name, positiveStep), value) => val updated = highWaterMarks.get(name).map { v => if (positiveStep) v.max(value) else v.min(value) }.getOrElse(value) highWaterMarks.update(name, updated) } } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/JsonMetadataDomain.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.actions.DomainMetadata import org.apache.spark.sql.delta.util.JsonUtils /** * A trait for capturing metadata domain of type T. */ trait JsonMetadataDomain[T] { val domainName: String /** * Creates [[DomainMetadata]] with configuration set as a JSON-serialized value of * the metadata domain of type T. */ def toDomainMetadata[T: Manifest]: DomainMetadata = DomainMetadata(domainName, JsonUtils.toJson(this.asInstanceOf[T]), removed = false) } abstract class JsonMetadataDomainUtils[T: Manifest] { protected val domainName: String /** * Returns the metadata domain's configuration as type T for domain metadata that * matches "domainName" in the given snapshot. Returns None if there is no matching * domain metadata. */ def fromSnapshot(snapshot: Snapshot): Option[T] = { snapshot.domainMetadata .find(_.domain == domainName) .map(m => fromJsonConfiguration(m)) } protected def fromJsonConfiguration(domain: DomainMetadata): T = JsonUtils.fromJson[T](domain.configuration) def isSameDomain(d: DomainMetadata): Boolean = d.domain == domainName } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/LastCheckpointInfo.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.collection.mutable.ArrayBuffer import org.apache.spark.sql.delta.actions.{CheckpointMetadata, SidecarFile, SingleAction} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.FileNames.{checkpointVersion, numCheckpointParts} import org.apache.spark.sql.delta.util.JsonUtils import com.fasterxml.jackson.annotation.{JsonIgnore, JsonIgnoreProperties, JsonPropertyOrder} import com.fasterxml.jackson.databind.{DeserializationFeature, JsonNode} import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.fasterxml.jackson.databind.node.ObjectNode import org.apache.commons.codec.digest.DigestUtils import org.apache.hadoop.fs.FileStatus import org.apache.spark.sql.SparkSession import org.apache.spark.sql.types.StructType /** * Information about the V2 Checkpoint in the LAST_CHECKPOINT file * @param path file name corresponding to the uuid-named v2 checkpoint * @param sizeInBytes size in bytes for the uuid-named v2 checkpoint * @param modificationTime modification time for the uuid-named v2 checkpoint * @param nonFileActions all non file actions for the v2 checkpoint. This info may or may not be * available. A None value means that info is missing. * If it is not None, then it should have all the non-FileAction * corresponding to the checkpoint. * @param sidecarFiles sidecar files corresponding to the v2 checkpoint. This info may or may * not be available. A None value means that this info is missing. * An empty list denotes that the v2 checkpoint has no sidecars. */ case class LastCheckpointV2( path: String, sizeInBytes: Long, modificationTime: Long, nonFileActions: Option[Seq[SingleAction]], sidecarFiles: Option[Seq[SidecarFile]]) { @JsonIgnore lazy val checkpointMetadataOpt: Option[CheckpointMetadata] = nonFileActions.flatMap(_.map(_.unwrap).collectFirst { case cm: CheckpointMetadata => cm }) } object LastCheckpointV2 { def apply( fileStatus: FileStatus, nonFileActions: Option[Seq[SingleAction]] = None, sidecarFiles: Option[Seq[SidecarFile]] = None): LastCheckpointV2 = { LastCheckpointV2( path = fileStatus.getPath.getName, sizeInBytes = fileStatus.getLen, modificationTime = fileStatus.getModificationTime, nonFileActions = nonFileActions, sidecarFiles = sidecarFiles) } } /** * Records information about a checkpoint. * * This class provides the checksum validation logic, needed to ensure that content of * LAST_CHECKPOINT file points to a valid json. The readers might read some part from old file and * some part from the new file (if the file is read across multiple requests). In some rare * scenarios, the split read might produce a valid json and readers will be able to parse it and * convert it into a [[LastCheckpointInfo]] object that contains invalid data. In order to prevent * using it, we do a checksum match on the read json to validate that it is consistent. * * For old Delta versions, which do not have checksum logic, we want to make sure that the old * fields (i.e. version, size, parts) are together in the beginning of last_checkpoint json. All * these fields together are less than 50 bytes, so even in split read scenario, we want to make * sure that old delta readers which do not do have checksum validation logic, gets all 3 fields * from one read request. For this reason, we use `JsonPropertyOrder` to force them in the beginning * together. * * @param version the version of this checkpoint * @param size the number of actions in the checkpoint, -1 if the information is unavailable. * @param parts the number of parts when the checkpoint has multiple parts. None if this is a * singular checkpoint * @param sizeInBytes the number of bytes of the checkpoint * @param numOfAddFiles the number of AddFile actions in the checkpoint * @param checkpointSchema the schema of the underlying checkpoint files * @param checksum the checksum of the [[LastCheckpointInfo]]. */ @JsonPropertyOrder(Array("version", "size", "parts")) case class LastCheckpointInfo( version: Long, size: Long, parts: Option[Int], @JsonDeserialize(contentAs = classOf[java.lang.Long]) sizeInBytes: Option[Long], @JsonDeserialize(contentAs = classOf[java.lang.Long]) numOfAddFiles: Option[Long], checkpointSchema: Option[StructType], v2Checkpoint: Option[LastCheckpointV2] = None, checksum: Option[String] = None) { @JsonIgnore def getFormatEnum(): CheckpointInstance.Format = parts match { case _ if v2Checkpoint.nonEmpty => CheckpointInstance.Format.V2 case Some(_) => CheckpointInstance.Format.WITH_PARTS case None => CheckpointInstance.Format.SINGLE } /** Whether two [[LastCheckpointInfo]] represents the same checkpoint */ def semanticEquals(other: LastCheckpointInfo): Boolean = { CheckpointInstance(this) == CheckpointInstance(other) } } object LastCheckpointInfo { val STORED_CHECKSUM_KEY = "checksum" /** Whether to store checksum OR do checksum validations around [[LastCheckpointInfo]] */ def checksumEnabled(spark: SparkSession): Boolean = spark.sessionState.conf.getConf(DeltaSQLConf.LAST_CHECKPOINT_CHECKSUM_ENABLED) /** * Returns the json representation of this [[LastCheckpointInfo]] object. * Also adds the checksum to the returned json if `addChecksum` is set. The checksum can be * used by readers to validate consistency of the [[LastCheckpointInfo]]. * It is calculated using rules mentioned in "JSON checksum" section in PROTOCOL.md. */ def serializeToJson( lastCheckpointInfo: LastCheckpointInfo, addChecksum: Boolean, suppressOptionalFields: Boolean = false): String = { if (suppressOptionalFields) { return JsonUtils.toJson( LastCheckpointInfo( lastCheckpointInfo.version, lastCheckpointInfo.size, lastCheckpointInfo.parts, sizeInBytes = None, numOfAddFiles = None, v2Checkpoint = None, checkpointSchema = None)) } val jsonStr: String = JsonUtils.toJson(lastCheckpointInfo.copy(checksum = None)) if (!addChecksum) return jsonStr val rootNode = JsonUtils.mapper.readValue(jsonStr, classOf[ObjectNode]) val checksum = treeNodeToChecksum(rootNode) rootNode.put(STORED_CHECKSUM_KEY, checksum).toString } /** * Converts the given `jsonStr` into a [[LastCheckpointInfo]] object. * if `validate` is set, then it also validates the consistency of the json: * - calculating the checksum and comparing it with the `storedChecksum`. * - json should not have any duplicates. */ def deserializeFromJson(jsonStr: String, validate: Boolean): LastCheckpointInfo = { if (validate) { val (storedChecksumOpt, actualChecksum) = LastCheckpointInfo.getChecksums(jsonStr) storedChecksumOpt.filter(_ != actualChecksum).foreach { storedChecksum => throw new IllegalStateException(s"Checksum validation failed for json: $jsonStr,\n" + s"storedChecksum:$storedChecksum, actualChecksum:$actualChecksum") } } // This means: // 1) EITHER: Checksum validation is config-disabled // 2) OR: The json lacked a checksum (e.g. written by old client). Nothing to validate. // 3) OR: The Stored checksum matches the calculated one. Validation succeeded. JsonUtils.fromJson[LastCheckpointInfo](jsonStr) } /** * Analyzes the json representation of [[LastCheckpointInfo]] and returns checksum tuple where * - first element refers to the stored checksum in the json representation of * [[LastCheckpointInfo]], None if the checksum is not present. * - second element refers to the checksum computed from the canonicalized json representation of * the [[LastCheckpointInfo]]. */ def getChecksums(jsonStr: String): (Option[String], String) = { val reader = JsonUtils.mapper.reader().withFeatures(DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY) val rootNode = reader.readTree(jsonStr) val storedChecksum = if (rootNode.has(STORED_CHECKSUM_KEY)) { Some(rootNode.get(STORED_CHECKSUM_KEY).asText()) } else { None } val actualChecksum = treeNodeToChecksum(rootNode) storedChecksum -> actualChecksum } /** * Canonicalizes the given `treeNode` json and returns its md5 checksum. * Refer to "JSON checksum" section in PROTOCOL.md for canonicalization steps. */ def treeNodeToChecksum(treeNode: JsonNode): String = { val jsonEntriesBuffer = ArrayBuffer.empty[(String, String)] import scala.collection.JavaConverters._ def traverseJsonNode(currentNode: JsonNode, prefix: ArrayBuffer[String]): Unit = { if (currentNode.isObject) { currentNode.fields().asScala.foreach { entry => prefix.append(encodeString(entry.getKey)) traverseJsonNode(entry.getValue, prefix) prefix.trimEnd(1) } } else if (currentNode.isArray) { currentNode.asScala.zipWithIndex.foreach { case (jsonNode, index) => prefix.append(index.toString) traverseJsonNode(jsonNode, prefix) prefix.trimEnd(1) } } else { var nodeValue = currentNode.asText() if (currentNode.isTextual) nodeValue = encodeString(nodeValue) jsonEntriesBuffer.append(prefix.mkString("+") -> nodeValue) } } traverseJsonNode(treeNode, prefix = ArrayBuffer.empty) import Ordering.Implicits._ val normalizedJsonKeyValues = jsonEntriesBuffer .filter { case (k, _) => k != s""""$STORED_CHECKSUM_KEY"""" } .map { case (k, v) => s"$k=$v" } .sortBy(_.toSeq: Seq[Char]) .mkString(",") DigestUtils.md5Hex(normalizedJsonKeyValues) } private val isUnreservedOctet = (Set.empty ++ ('a' to 'z') ++ ('A' to 'Z') ++ ('0' to '9') ++ "-._~").map(_.toByte) /** * URL encodes a String based on the following rules: * 1. Use uppercase hexadecimals for all percent encodings * 2. percent-encode everything other than unreserved characters * 3. unreserved characters are = a-z / A-Z / 0-9 / "-" / "." / "_" / "~" */ private def encodeString(str: String): String = { val result = str.getBytes(java.nio.charset.StandardCharsets.UTF_8).map { case b if isUnreservedOctet(b) => b.toChar.toString case b => // convert to char equivalent of unsigned byte val c = (b & 0xff) f"%%$c%02X" }.mkString s""""$result"""" } def fromFiles(files: Seq[FileStatus]): LastCheckpointInfo = { assert(files.nonEmpty, "files should be non empty to construct LastCheckpointInfo") LastCheckpointInfo( version = checkpointVersion(files.head), size = -1L, parts = numCheckpointParts(files.head.getPath), sizeInBytes = Some(files.map(_.getLen).sum), numOfAddFiles = None, checkpointSchema = None ) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/MaterializedRowTrackingColumn.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.UUID import org.apache.spark.sql.delta.actions.{Metadata, Protocol} import org.apache.spark.sql.DataFrame import org.apache.spark.sql.catalyst.expressions.Attribute /** * Represents a materialized row tracking column. Concrete implementations are [[MaterializedRowId]] * and [[MaterializedRowCommitVersion]]. */ abstract class MaterializedRowTrackingColumn { /** * Table metadata configuration property name storing the name of this materialized row tracking * column. */ val MATERIALIZED_COLUMN_NAME_PROP: String /** Prefix to use for the name of this materialized row tracking column */ val MATERIALIZED_COLUMN_NAME_PREFIX: String /** * Returns the exception to throw when the materialized column name is not set in the table * metadata. The table name is passed as argument. */ def missingMetadataException: String => Throwable /** * Generate a random name for a materialized row tracking column. The generated name contains a * unique UUID, we assume it shall not conflict with existing column. */ private def generateMaterializedColumnName: String = MATERIALIZED_COLUMN_NAME_PREFIX + UUID.randomUUID().toString /** * Update this materialized row tracking column name in the metadata. * - If row tracking is not allowed or not supported, this operation is a noop. * - If row tracking is supported on the table and no name is assigned to the old metadata, we * assign a name. If a name was already assigned, we copy over this name. * Throws in case the assignment of a new name fails due to a conflict. */ private[delta] def updateMaterializedColumnName( protocol: Protocol, oldMetadata: Metadata, newMetadata: Metadata): Metadata = { if (!RowTracking.isSupported(protocol)) { // During a CLONE we might not enable row tracking, but still receive the materialized column // name from the source. In this case, we need to remove the column name to not have the same // column name in two different tables. return newMetadata.copy( configuration = newMetadata.configuration - MATERIALIZED_COLUMN_NAME_PROP) } // Take the materialized column name from the old metadata, as this is the materialized column // name of the current table. We overwrite the materialized column name of the new metadata as // it could contain a materialized column name from another table, e.g. the source table during // a CLONE. val materializedColumnName = oldMetadata.configuration .getOrElse(MATERIALIZED_COLUMN_NAME_PROP, generateMaterializedColumnName) newMetadata.copy(configuration = newMetadata.configuration + (MATERIALIZED_COLUMN_NAME_PROP -> materializedColumnName)) } /** * Throws an exception if row tracking is allowed and the materialized column name conflicts with * another column name. */ private[delta] def throwIfMaterializedColumnNameConflictsWithSchema(metadata: Metadata): Unit = { val logicalColumnNames = metadata.schema.fields.map(_.name) val physicalColumnNames = metadata.schema.fields .map(field => DeltaColumnMapping.getPhysicalName(field)) metadata.configuration.get(MATERIALIZED_COLUMN_NAME_PROP).foreach { columnName => if (logicalColumnNames.contains(columnName) || physicalColumnNames.contains(columnName)) { throw DeltaErrors.addingColumnWithInternalNameFailed(columnName) } } } /** Extract the materialized column name from the [[Metadata]] of a [[DeltaLog]]. */ def getMaterializedColumnName(protocol: Protocol, metadata: Metadata): Option[String] = { if (RowTracking.isEnabled(protocol, metadata)) { metadata.configuration.get(MATERIALIZED_COLUMN_NAME_PROP) } else { None } } /** Convenience method that throws if the materialized column name cannot be extracted. */ def getMaterializedColumnNameOrThrow( protocol: Protocol, metadata: Metadata, tableId: String): String = { getMaterializedColumnName(protocol, metadata).getOrElse { throw missingMetadataException(tableId) } } /** * If Row tracking is enabled, return an Expression referencing this Row tracking column Attribute * in 'dataFrame' if one is available. Otherwise returns None. */ private[delta] def getAttribute( snapshot: Snapshot, dataFrame: DataFrame): Option[Attribute] = { if (!RowTracking.isEnabled(snapshot.protocol, snapshot.metadata)) { return None } val materializedColumnName = getMaterializedColumnNameOrThrow( snapshot.protocol, snapshot.metadata, snapshot.deltaLog.unsafeVolatileTableId) val analyzedPlan = dataFrame.queryExecution.analyzed analyzedPlan.outputSet.view.find(attr => materializedColumnName == attr.name) } } object MaterializedRowId extends MaterializedRowTrackingColumn { /** * Table metadata configuration property name storing the name of the column in which the * Row IDs are materialized. */ val MATERIALIZED_COLUMN_NAME_PROP = "delta.rowTracking.materializedRowIdColumnName" /** Prefix to use for the name of the materialized Row ID column */ val MATERIALIZED_COLUMN_NAME_PREFIX = "_row-id-col-" def missingMetadataException: String => Throwable = DeltaErrors.materializedRowIdMetadataMissing } object MaterializedRowCommitVersion extends MaterializedRowTrackingColumn { /** * Table metadata configuration property name storing the name of the column in which the * Row commit versions are materialized. */ val MATERIALIZED_COLUMN_NAME_PROP = "delta.rowTracking.materializedRowCommitVersionColumnName" /** Prefix to use for the name of the materialized Row commit version column */ val MATERIALIZED_COLUMN_NAME_PREFIX = "_row-commit-version-col-" def missingMetadataException: String => Throwable = DeltaErrors.materializedRowCommitVersionMetadataMissing } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/MetadataCleanup.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.{Calendar, TimeZone} import scala.collection.immutable.NumericRange import scala.collection.mutable.ArrayBuffer import org.apache.spark.sql.delta.DeltaHistoryManager.BufferingLogDeletionIterator import org.apache.spark.sql.delta.TruncationGranularity.{DAY, HOUR, MINUTE, TruncationGranularity} import org.apache.spark.sql.delta.actions.{Action, Metadata} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.FileNames import org.apache.spark.sql.delta.util.FileNames._ import org.apache.commons.lang3.time.DateUtils import org.apache.hadoop.fs.{FileStatus, FileSystem, Path} import org.apache.spark.internal.MDC import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.functions.{col, isnull, lit, not, when} private[delta] object TruncationGranularity extends Enumeration { type TruncationGranularity = Value val DAY, HOUR, MINUTE = Value } /** Cleans up expired Delta table metadata. */ trait MetadataCleanup extends DeltaLogging { self: DeltaLog => protected type VersionRange = NumericRange.Inclusive[Long] protected def versionRange(start: Long, end: Long): VersionRange = NumericRange.inclusive[Long](start = start, end = end, step = 1) /** Whether to clean up expired log files and checkpoints. */ def enableExpiredLogCleanup(metadata: Metadata): Boolean = DeltaConfigs.ENABLE_EXPIRED_LOG_CLEANUP.fromMetaData(metadata) /** * Returns the duration in millis for how long to keep around obsolete logs. We may keep logs * beyond this duration until the next calendar day to avoid constantly creating checkpoints. */ def deltaRetentionMillis(metadata: Metadata): Long = { val interval = DeltaConfigs.LOG_RETENTION.fromMetaData(metadata) DeltaConfigs.getMilliSeconds(interval) } override def doLogCleanup( snapshotToCleanup: Snapshot, catalogTableOpt: Option[CatalogTable]): Unit = { if (enableExpiredLogCleanup(unsafeVolatileSnapshot.metadata)) { cleanUpExpiredLogs(snapshotToCleanup, catalogTableOpt) } } /** Clean up expired delta and checkpoint logs. Exposed for testing. */ private[delta] def cleanUpExpiredLogs( snapshotToCleanup: Snapshot, catalogTableOpt: Option[CatalogTable] = None, deltaRetentionMillisOpt: Option[Long] = None, cutoffTruncationGranularity: TruncationGranularity = DAY): Unit = { recordDeltaOperation(this, "delta.log.cleanup") { val retentionMillis = deltaRetentionMillisOpt.getOrElse(deltaRetentionMillis(unsafeVolatileSnapshot.metadata)) val fileCutOffTime = truncateDate(clock.getTimeMillis() - retentionMillis, cutoffTruncationGranularity).getTime val formattedDate = fileCutOffTime.toGMTString logInfo( log"[tableId=${MDC(DeltaLogKeys.TABLE_ID, truncatedUnsafeVolatileTableId)}] " + log"Starting the deletion of log files older than ${MDC(DeltaLogKeys.DATE, formattedDate)}") if (!metadataCleanupAllowed(snapshotToCleanup, fileCutOffTime.getTime)) { logInfo("Metadata cleanup was skipped due to not satisfying the requirements " + "of CheckpointProtectionTableFeature.") return } val fs = logPath.getFileSystem(newDeltaHadoopConf()) var numDeleted = 0 val expiredDeltaLogs = listExpiredDeltaLogs(fileCutOffTime.getTime) if (expiredDeltaLogs.hasNext) { // Trigger compatibility checkpoint creation logic only when this round of metadata cleanup // is going to delete any deltas/checkpoint files. // We need to create compat checkpoint before deleting delta/checkpoint files so that we // don't have a window in b/w where the old checkpoint is deleted and there is no // compat-checkpoint available. val v2CompatCheckpointMetrics = new V2CompatCheckpointMetrics createSinglePartCheckpointForBackwardCompat(snapshotToCleanup, v2CompatCheckpointMetrics) logInfo(log"Compatibility checkpoint creation metrics: " + log"${MDC(DeltaLogKeys.METRICS, v2CompatCheckpointMetrics)}") } var wasCheckpointDeleted = false var maxBackfilledVersionDeleted = -1L expiredDeltaLogs.map(_.getPath).foreach { path => // recursive = false if (fs.delete(path, false)) { numDeleted += 1 if (FileNames.isCheckpointFile(path)) { wasCheckpointDeleted = true } if (FileNames.isDeltaFile(path)) { maxBackfilledVersionDeleted = Math.max(maxBackfilledVersionDeleted, FileNames.deltaVersion(path)) } } } val commitDirPath = FileNames.commitDirPath(logPath) // Commit Directory might not exist on tables created in older versions and // never updated since. val expiredUnbackfilledDeltaLogs: Iterator[FileStatus] = if (fs.exists(commitDirPath)) { store .listFrom(listingPrefix(commitDirPath, 0), newDeltaHadoopConf()) .takeWhile { case UnbackfilledDeltaFile(_, fileVersion, _) => fileVersion <= maxBackfilledVersionDeleted } } else { Iterator.empty } val numDeletedUnbackfilled = expiredUnbackfilledDeltaLogs.count( log => fs.delete(log.getPath, false)) if (wasCheckpointDeleted) { // Trigger sidecar deletion only when some checkpoints have been deleted as part of this // round of Metadata cleanup. val sidecarDeletionMetrics = new SidecarDeletionMetrics identifyAndDeleteUnreferencedSidecarFiles( snapshotToCleanup, fileCutOffTime.getTime, sidecarDeletionMetrics) logInfo(log"Sidecar deletion metrics: ${MDC(DeltaLogKeys.METRICS, sidecarDeletionMetrics)}") } logInfo(log"[tableId=${MDC(DeltaLogKeys.TABLE_ID, truncatedUnsafeVolatileTableId)}] " + log"Deleted ${MDC(DeltaLogKeys.NUM_FILES, numDeleted.toLong)} log files and " + log"${MDC(DeltaLogKeys.NUM_FILES2, numDeletedUnbackfilled.toLong)} unbackfilled commit " + log"files older than ${MDC(DeltaLogKeys.DATE, formattedDate)}") } } /** Helper function for getting the version of a checkpoint or a commit. */ def getDeltaFileChecksumOrCheckpointVersion(filePath: Path): Long = { require(isCheckpointFile(filePath) || isDeltaFile(filePath) || isChecksumFile(filePath)) getFileVersion(filePath) } /** * Returns an iterator of expired delta logs that can be cleaned up. For a delta log to be * considered as expired, it must: * - have a checkpoint file after it * - be older than `fileCutOffTime` */ private def listExpiredDeltaLogs(fileCutOffTime: Long): Iterator[FileStatus] = { val latestCheckpoint = readLastCheckpointFile() if (latestCheckpoint.isEmpty) return Iterator.empty val threshold = latestCheckpoint.get.version - 1L val files = store.listFrom(listingPrefix(logPath, 0), newDeltaHadoopConf()) .filter(f => isCheckpointFile(f) || isDeltaFile(f) || isChecksumFile(f)) new BufferingLogDeletionIterator( files, fileCutOffTime, threshold, getDeltaFileChecksumOrCheckpointVersion) } protected def checkpointExistsAtCleanupBoundary(deltaLog: DeltaLog, version: Long): Boolean = { if (spark.conf.get(DeltaSQLConf.ALLOW_METADATA_CLEANUP_CHECKPOINT_EXISTENCE_CHECK_DISABLED)) { return false } val upperBoundVersion = Some(CheckpointInstance(version = version + 1)) deltaLog .findLastCompleteCheckpointBefore(upperBoundVersion) .exists(_.version == version) } /** * Validates whether the metadata cleanup adheres to the CheckpointProtectionTableFeature * requirements. Metadata cleanup is only allowed if we can cleanup everything before * requireCheckpointProtectionBeforeVersion. If this is not possible, we can still cleanup * if there is already a checkpoint at the cleanup boundary version. * * If none of the invariants above are satisfied, we validate whether we support all * protocols in the commit range we are planning to delete. If we encounter an unsupported * protocol we skip the cleanup. */ private def metadataCleanupAllowed( snapshot: Snapshot, fileCutOffTime: Long): Boolean = { def expandVersionRange(currentRange: VersionRange, versionToCover: Long): VersionRange = versionRange(currentRange.start.min(versionToCover), currentRange.end.max(versionToCover)) val checkpointProtectionVersion = CheckpointProtectionTableFeature.getCheckpointProtectionVersion(snapshot) if (checkpointProtectionVersion <= 0) return true val expiredDeltaLogs = listExpiredDeltaLogs(fileCutOffTime) if (expiredDeltaLogs.isEmpty) return true val deltaLog = snapshot.deltaLog val toCleanVersionRange = expiredDeltaLogs .filter(isDeltaFile) .collect { case DeltaFile(_, version) => version } // Stop early if we cannot cleanup beyond the checkpointProtectionVersion. // We include equality for the CheckpointProtection invariant check below. // Assumes commit versions are continuous. .takeWhile { _ <= checkpointProtectionVersion - 1 } .foldLeft(versionRange(Long.MaxValue, 0L))(expandVersionRange) // CheckpointProtectionTableFeature main invariant. if (toCleanVersionRange.end >= checkpointProtectionVersion - 1) return true // If we cannot delete until the checkpoint protection version. Check if a checkpoint already // exists at the cleanup boundary. If it does, it is safe to clean up to the boundary. if (checkpointExistsAtCleanupBoundary(deltaLog, toCleanVersionRange.end + 1L)) return true // If the CheckpointProtectionTableFeature invariants do not hold, we must support all // protocols for commits that we are cleaning up. Also, we have to support the first // commit that we retain, because we will be creating a new checkpoint for that commit. allProtocolsSupported( deltaLog, versionRange(toCleanVersionRange.start, toCleanVersionRange.end + 1L)) } /** * Validates whether the client supports read for all the protocols in the provided checksums * as well as write for `versionThatRequiresWriteSupport`. * * @param deltaLog The log of the delta table. * @param checksumsToValidate An iterator with the checksum files we need to validate. The client * needs read support for all the encountered protocols. * @param versionThatRequiresWriteSupport The version the client needs write support. This * is the version we are creating a new checkpoint. * @param expectedChecksumFileCount The expected number of checksum files. If the iterator * contains less files, the function returns false. * @return Returns false if there is a non-supported or null protocol in the provided checksums. * Returns true otherwise. */ protected[delta] def allProtocolsSupported( deltaLog: DeltaLog, checksumsToValidate: Iterator[FileStatus], versionThatRequiresWriteSupport: Long, expectedChecksumFileCount: Long): Boolean = { if (!spark.conf.get(DeltaSQLConf.ALLOW_METADATA_CLEANUP_WHEN_ALL_PROTOCOLS_SUPPORTED)) { return false } val schemaToUse = Action.logSchema(Set("protocol")) val supportedForRead = DeltaUDF.booleanFromProtocol(_.supportedForRead())(col("protocol")) val supportedForWrite = DeltaUDF.booleanFromProtocol(_.supportedForWrite())(col("protocol")) val supportedForReadAndWrite = supportedForRead && supportedForWrite val supported = when(col("version") === lit(versionThatRequiresWriteSupport), supportedForReadAndWrite) .otherwise(supportedForRead) val fileIndexOpt = DeltaLogFileIndex(DeltaLogFileIndex.CHECKSUM_FILE_FORMAT, checksumsToValidate.toSeq) val fileIndexSupportedOpt = fileIndexOpt.map { index => if (index.inputFiles.length != expectedChecksumFileCount) return false deltaLog .loadIndex(index, schemaToUse) // If we find any CRC with no protocol definition we need to abort. .filter(isnull(col("protocol")) || not(supported)) .take(1) .isEmpty } fileIndexSupportedOpt.getOrElse(true) } protected[delta] def allProtocolsSupported( deltaLog: DeltaLog, versionRange: VersionRange): Boolean = { // We only expect back filled commits in the range. val checksumsToValidate = deltaLog .listFrom(versionRange.start) .collect { case ChecksumFile(fileStatus, version) => (fileStatus, version) } .takeWhile { case (_, version) => version <= versionRange.end } .map { case (fileStatus, _) => fileStatus } allProtocolsSupported( deltaLog, checksumsToValidate, versionThatRequiresWriteSupport = versionRange.end, expectedChecksumFileCount = versionRange.end - versionRange.start + 1) } /** * Truncates a timestamp down to a given unit. The unit can be either DAY, HOUR or MINUTE. * - DAY: The timestamp it truncated to the previous midnight. * - HOUR: The timestamp it truncated to the last hour. * - MINUTE: The timestamp it truncated to the last minute. */ private[delta] def truncateDate(timeMillis: Long, unit: TruncationGranularity): Calendar = { val date = Calendar.getInstance(TimeZone.getTimeZone("UTC")) date.setTimeInMillis(timeMillis) val calendarUnit = unit match { case DAY => Calendar.DAY_OF_MONTH case HOUR => Calendar.HOUR_OF_DAY case MINUTE => Calendar.MINUTE } DateUtils.truncate(date, calendarUnit) } /** Truncates a timestamp down to the previous midnight and returns the time. */ private[delta] def truncateDay(timeMillis: Long): Calendar = { truncateDate(timeMillis, TruncationGranularity.DAY) } /** * Helper method to create a compatibility classic single file checkpoint file for this table. * This is needed so that any legacy reader which do not understand [[V2CheckpointTableFeature]] * could read the legacy classic checkpoint file and fail gracefully with Protocol requirement * failure. */ protected[delta] def createSinglePartCheckpointForBackwardCompat( snapshotToCleanup: Snapshot, metrics: V2CompatCheckpointMetrics): Unit = { // Do nothing if this table does not use V2 Checkpoints, or has no checkpoints at all. if (!CheckpointProvider.isV2CheckpointEnabled(snapshotToCleanup)) return if (snapshotToCleanup.checkpointProvider.isEmpty) return val startTimeMs = System.currentTimeMillis() val hadoopConf = newDeltaHadoopConf() val checkpointInstance = CheckpointInstance(snapshotToCleanup.checkpointProvider.topLevelFiles.head.getPath) // The current checkpoint provider is already using a checkpoint with the naming // scheme of classic checkpoints. There is no need to create a compatibility checkpoint // in this case. if (checkpointInstance.format != CheckpointInstance.Format.V2) return val checkpointVersion = snapshotToCleanup.checkpointProvider.version val checkpoints = listFrom(checkpointVersion) .takeWhile(file => FileNames.getFileVersionOpt(file.getPath).exists(_ <= checkpointVersion)) .collect { case file if FileNames.isCheckpointFile(file) => CheckpointInstance(file.getPath) } .filter(_.format != CheckpointInstance.Format.V2) .toArray val availableNonV2Checkpoints = getLatestCompleteCheckpointFromList(checkpoints, Some(checkpointVersion)) if (availableNonV2Checkpoints.nonEmpty) { metrics.v2CheckpointCompatLogicTimeTakenMs = System.currentTimeMillis() - startTimeMs return } // topLevelFileIndex must be non-empty when topLevelFiles are present val shallowCopyDf = loadIndex(snapshotToCleanup.checkpointProvider.topLevelFileIndex.get, Action.logSchema) val finalPath = FileNames.checkpointFileSingular(snapshotToCleanup.deltaLog.logPath, checkpointVersion) Checkpoints.createCheckpointV2ParquetFile( spark, shallowCopyDf, finalPath, hadoopConf, useRename = false) metrics.v2CheckpointCompatLogicTimeTakenMs = System.currentTimeMillis() - startTimeMs metrics.checkpointVersion = checkpointVersion } /** Deletes any unreferenced files from the sidecar directory `_delta_log/_sidecar` */ protected def identifyAndDeleteUnreferencedSidecarFiles( snapshotToCleanup: Snapshot, checkpointRetention: Long, metrics: SidecarDeletionMetrics): Unit = { val startTimeMs = System.currentTimeMillis() // If v2 checkpoints are not enabled on the table, we don't need to attempt the sidecar cleanup. if (!CheckpointProvider.isV2CheckpointEnabled(snapshotToCleanup)) return val hadoopConf = newDeltaHadoopConf() val fs = sidecarDirPath.getFileSystem(hadoopConf) // This can happen when the V2 Checkpoint feature is present in the Protocol but // only Classic checkpoints have been created for the table. if (!fs.exists(sidecarDirPath)) return val (parquetCheckpointFiles, otherFiles) = store .listFrom(listingPrefix(logPath, 0), hadoopConf) .collect { case CheckpointFile(status, _) => (status, CheckpointInstance(status.getPath)) } .collect { case (fileStatus, ci) if ci.format.usesSidecars => fileStatus } .toSeq .partition(_.getPath.getName.endsWith("parquet")) val (jsonCheckpointFiles, unknownFormatCheckpointFiles) = otherFiles.partition(_.getPath.getName.endsWith("json")) if (unknownFormatCheckpointFiles.nonEmpty) { logWarning( log"Found checkpoint files other than parquet and json: " + log"${MDC(DeltaLogKeys.PATHS, unknownFormatCheckpointFiles.map(_.getPath.toString).mkString(","))}") } metrics.numActiveParquetCheckpointFiles = parquetCheckpointFiles.size metrics.numActiveJsonCheckpointFiles = jsonCheckpointFiles.size val parquetCheckpointsFileIndex = DeltaLogFileIndex(DeltaLogFileIndex.CHECKPOINT_FILE_FORMAT_PARQUET, parquetCheckpointFiles) val jsonCheckpointsFileIndex = DeltaLogFileIndex(DeltaLogFileIndex.CHECKPOINT_FILE_FORMAT_JSON, jsonCheckpointFiles) val identifyActiveSidecarsStartTimeMs = System.currentTimeMillis() metrics.activeCheckpointsListingTimeTakenMs = identifyActiveSidecarsStartTimeMs - startTimeMs import org.apache.spark.sql.delta.implicits._ val df = (parquetCheckpointsFileIndex ++ jsonCheckpointsFileIndex) .map(loadIndex(_, Action.logSchema(Set("sidecar")))) .reduceOption(_ union _) .getOrElse { return } val activeSidecarFiles = df .select("sidecar.path") .where("path is not null") .as[String] .collect() .map(p => new Path(p).getName) // Get bare file names .toSet val identifyAndDeleteSidecarsStartTimeMs = System.currentTimeMillis() metrics.identifyActiveSidecarsTimeTakenMs = identifyAndDeleteSidecarsStartTimeMs - identifyActiveSidecarsStartTimeMs // Retain all files created in the checkpoint retention window - irrespective of whether they // are referenced in a checkpoint or not. This is to make sure that we don't end up deleting an // in-progress checkpoint. val retentionTimestamp: Long = checkpointRetention val sidecarFilesIterator = new Iterator[FileStatus] { // Hadoop's RemoteIterator is neither java nor scala Iterator, so have to wrap it val remoteIterator = fs.listStatusIterator(sidecarDirPath) override def hasNext: Boolean = remoteIterator.hasNext() override def next(): FileStatus = remoteIterator.next() } val sidecarFilesToDelete = sidecarFilesIterator .collect { case file if file.getModificationTime < retentionTimestamp => file.getPath } .filterNot(path => activeSidecarFiles.contains(path.getName)) val sidecarDeletionStartTimeMs = System.currentTimeMillis() logInfo( log"[tableId=${MDC(DeltaLogKeys.TABLE_ID, truncatedUnsafeVolatileTableId)}] " + log"Starting the deletion of unreferenced sidecar files") val count = deleteMultiple(fs, sidecarFilesToDelete) logInfo(log"[tableId=${MDC(DeltaLogKeys.TABLE_ID, truncatedUnsafeVolatileTableId)}] Deleted " + log"${MDC(DeltaLogKeys.COUNT, count)} sidecar files") metrics.numSidecarFilesDeleted = count val endTimeMs = System.currentTimeMillis() metrics.identifyAndDeleteSidecarsTimeTakenMs = sidecarDeletionStartTimeMs - identifyAndDeleteSidecarsStartTimeMs metrics.overallSidecarProcessingTimeTakenMs = endTimeMs - startTimeMs } private def deleteMultiple(fs: FileSystem, paths: Iterator[Path]): Long = { paths.map { path => if (fs.delete(path, false)) 1L else 0L }.sum } /** Class to track metrics related to V2 Checkpoint Sidecars deletion. */ protected class SidecarDeletionMetrics { // number of sidecar files deleted var numSidecarFilesDeleted: Long = -1 // number of active parquet checkpoint files present in delta log directory var numActiveParquetCheckpointFiles: Long = -1 // number of active json checkpoint files present in delta log directory var numActiveJsonCheckpointFiles: Long = -1 // time taken (in ms) to list and identify active checkpoints var activeCheckpointsListingTimeTakenMs: Long = -1 // time taken (in ms) to list the sidecar directory to get all sidecars and delete those which // aren't referenced by any checkpoint anymore var identifyAndDeleteSidecarsTimeTakenMs: Long = -1 // time taken (in ms) to read the active checkpoint json / parquet files and identify active // sidecar files var identifyActiveSidecarsTimeTakenMs: Long = -1 // time taken (in ms) for everything related to sidecar processing var overallSidecarProcessingTimeTakenMs: Long = -1 } /** Class to track metrics related to V2 Compatibility checkpoint creation. */ protected[delta] class V2CompatCheckpointMetrics { // time taken (in ms) to run the v2 checkpoint compat logic var v2CheckpointCompatLogicTimeTakenMs: Long = -1 // the version at which we have created a v2 compat checkpoint, -1 if no compat checkpoint was // created. var checkpointVersion: Long = -1 } /** * Finds a checkpoint such that we are able to construct table snapshot for all versions at or * greater than the checkpoint version returned. */ def findEarliestReliableCheckpoint: Option[Long] = { val hadoopConf = newDeltaHadoopConf() var earliestCheckpointVersionOpt: Option[Long] = None // This is used to collect the checkpoint files from the current version that we are listing. // When we list a file that is not part of the checkpoint, then we must have seen the entire // checkpoint. We then verify if the checkpoint was complete, and if it is not, we clear the // collection and wait for the next checkpoint to appear in the file listing. // Whenever we see a complete checkpoint for the first time, we remember it as the earliest // checkpoint. val currentCheckpointFiles = ArrayBuffer.empty[Path] var prevCommitVersion = 0L def currentCheckpointVersionOpt: Option[Long] = currentCheckpointFiles.headOption.map(checkpointVersion(_)) def isCurrentCheckpointComplete: Boolean = { val instances = currentCheckpointFiles.map(CheckpointInstance(_)).toArray getLatestCompleteCheckpointFromList(instances).isDefined } // Iterate logs files in ascending order to find the earliest reliable checkpoint, for the same // version, checkpoint is always processed before commit so that we can identify the candidate // checkpoint first and then verify commits since the candidate's version (inclusive) store.listFrom(listingPrefix(logPath, 0L), hadoopConf) .map(_.getPath) .foreach { case CheckpointFile(f, checkpointVersion) // Invalidate the candidate if we observe missing commits before the current checkpoint. // the incoming commit will invalidate the candidate as well, but then we miss the current // checkpoint, which is also a valid candidate. if earliestCheckpointVersionOpt.isEmpty || checkpointVersion > prevCommitVersion + 1 => earliestCheckpointVersionOpt = None if (!currentCheckpointVersionOpt.contains(checkpointVersion)) { // If it's a different checkpoint, clear the existing one. currentCheckpointFiles.clear() } currentCheckpointFiles += f case DeltaFile(_, deltaVersion) => if (earliestCheckpointVersionOpt.isEmpty && isCurrentCheckpointComplete) { // We have found a complete checkpoint, but we should not stop here. If a future // commit version is missing, then this checkpoint will be discarded and we will need // to restart the search from that point. // Ensure that the commit json is there at the checkpoint version. If it's not there, // we don't consider such a checkpoint as a reliable checkpoint. if (currentCheckpointVersionOpt.contains(deltaVersion)) { earliestCheckpointVersionOpt = currentCheckpointVersionOpt prevCommitVersion = deltaVersion } } // Need to clear it so that if there is a gap in commit versions, we are forced to // look for a new complete checkpoint. currentCheckpointFiles.clear() if (deltaVersion > prevCommitVersion + 1) { // Missing commit versions. Restart the search. earliestCheckpointVersionOpt = None } prevCommitVersion = deltaVersion case _ => } earliestCheckpointVersionOpt } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/NumRecordsStats.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.actions.{Action, AddFile, RemoveFile} import org.apache.spark.sql.util.ScalaExtensions._ /** * Container class for statistics related to number of records in a Delta commit. */ case class NumRecordsStats ( // Number of logical records in AddFile actions with numRecords. numLogicalRecordsAddedPartial: Long, // Number of logical records in RemoveFile actions with numRecords. numLogicalRecordsRemovedPartial: Long, numDeletionVectorRecordsAdded: Long, numDeletionVectorRecordsRemoved: Long, numFilesAddedWithoutNumRecords: Long, numFilesRemovedWithoutNumRecords: Long, numLogicalRecordsAddedInFilesWithDeletionVectorsPartial: Long) { def allFilesHaveNumRecords: Boolean = numFilesAddedWithoutNumRecords == 0 && numFilesRemovedWithoutNumRecords == 0 /** * The number of logical records in all AddFile actions or None if any file does not contain * statistics. */ def numLogicalRecordsAdded: Option[Long] = Option.when(numFilesAddedWithoutNumRecords == 0)( numLogicalRecordsAddedPartial) /** * The number of logical records in all RemoveFile actions or None if any file does not contain * statistics. */ def numLogicalRecordsRemoved: Option[Long] = Option.when(numFilesRemovedWithoutNumRecords == 0)( numLogicalRecordsRemovedPartial) /** * The number of logical records in all AddFile actions that have a deletion vector or None * if any file does not contain statistics. */ def numLogicalRecordsAddedInFilesWithDeletionVectors: Option[Long] = Option.when(numFilesAddedWithoutNumRecords == 0)( numLogicalRecordsAddedInFilesWithDeletionVectorsPartial) } object NumRecordsStats { def fromActions(actions: Seq[Action]): NumRecordsStats = { var numFilesAdded = 0L var numFilesRemoved = 0L var numFilesAddedWithoutNumRecords = 0L var numFilesRemovedWithoutNumRecords = 0L var numLogicalRecordsAddedPartial: Long = 0L var numLogicalRecordsRemovedPartial: Long = 0L var numDeletionVectorRecordsAdded = 0L var numDeletionVectorRecordsRemoved = 0L var numLogicalRecordsAddedInFilesWithDeletionVectorsPartial = 0L actions.foreach { case a: AddFile => numFilesAdded += 1 numLogicalRecordsAddedPartial += a.numLogicalRecords.getOrElse { numFilesAddedWithoutNumRecords += 1 0L } numDeletionVectorRecordsAdded += a.numDeletedRecords if (a.deletionVector != null) { numLogicalRecordsAddedInFilesWithDeletionVectorsPartial += a.numLogicalRecords.getOrElse(0L) } case r: RemoveFile => numFilesRemoved += 1 numLogicalRecordsRemovedPartial += r.numLogicalRecords.getOrElse { numFilesRemovedWithoutNumRecords += 1 0L } numDeletionVectorRecordsRemoved += r.numDeletedRecords case _ => // Do nothing } NumRecordsStats( numLogicalRecordsAddedPartial = numLogicalRecordsAddedPartial, numLogicalRecordsRemovedPartial = numLogicalRecordsRemovedPartial, numDeletionVectorRecordsAdded = numDeletionVectorRecordsAdded, numDeletionVectorRecordsRemoved = numDeletionVectorRecordsRemoved, numFilesAddedWithoutNumRecords = numFilesAddedWithoutNumRecords, numFilesRemovedWithoutNumRecords = numFilesRemovedWithoutNumRecords, numLogicalRecordsAddedInFilesWithDeletionVectorsPartial = numLogicalRecordsAddedInFilesWithDeletionVectorsPartial ) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.nio.file.FileAlreadyExistsException import java.util.{ConcurrentModificationException, Optional, UUID} import java.util.concurrent.TimeUnit.{MINUTES, NANOSECONDS} import scala.collection.JavaConverters._ import scala.collection.mutable import scala.collection.mutable.{ArrayBuffer, HashSet} import scala.util.control.NonFatal import com.databricks.spark.util.TagDefinitions.TAG_LOG_STORE_CLASS import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.DeltaOperations.{ChangeColumn, ChangeColumns, CreateTable, Operation, ReplaceColumns, ReplaceTable, UpdateSchema} import org.apache.spark.sql.delta.RowId.RowTrackingMetadataDomain import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.constraints.{Constraints, Invariants} import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CoordinatedCommitsUtils, TableCommitCoordinatorClient, UCCommitCoordinatorBuilder} import org.apache.spark.sql.delta.files._ import org.apache.spark.sql.delta.hooks.{CheckpointHook, ChecksumHook, GenerateSymlinkManifest, HudiConverterHook, IcebergConverterHook, PostCommitHook, UpdateCatalogFactory} import org.apache.spark.sql.delta.implicits.addFileEncoder import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.redirect.{RedirectFeature, TableRedirectConfiguration} import org.apache.spark.sql.delta.schema.{SchemaMergingUtils, SchemaUtils} import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf} import org.apache.spark.sql.delta.stats._ import org.apache.spark.sql.delta.util.{DeltaCommitFileProvider, JsonUtils, TransactionHelper} import org.apache.spark.sql.util.ScalaExtensions._ import io.delta.storage.commit._ import io.delta.storage.commit.actions.{AbstractMetadata, AbstractProtocol} import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient import org.apache.commons.lang3.NotImplementedException import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.SparkException import org.apache.spark.internal.{MDC, MessageWithContext} import org.apache.spark.sql.{AnalysisException, Column, DataFrame, SaveMode, SparkSession} import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans.logical.UnsetTableProperties import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes import org.apache.spark.sql.catalyst.util.{CharVarcharUtils, ResolveDefaultColumns} import org.apache.spark.sql.delta.clustering.ClusteringMetadataDomain import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{StructField, StructType} import org.apache.spark.util.{Clock, Utils} object CoordinatedCommitType extends Enumeration { type CoordinatedCommitType = Value val FS_COMMIT, CC_COMMIT, CO_COMMIT, FS_TO_CC_UPGRADE_COMMIT, FS_TO_CO_UPGRADE_COMMIT, CC_TO_FS_DOWNGRADE_COMMIT = Value } case class CoordinatedCommitsStats( coordinatedCommitsType: String, commitCoordinatorName: String, commitCoordinatorConf: Map[String, String]) /** Record metrics about a successful commit. */ case class CommitStats( /** The version read by the txn when it starts. */ startVersion: Long, /** The version committed by the txn. */ commitVersion: Long, /** The version read by the txn right after it commits. It usually equals to commitVersion, * but can be larger than commitVersion when there are concurrent commits. */ readVersion: Long, txnDurationMs: Long, commitDurationMs: Long, fsWriteDurationMs: Long, stateReconstructionDurationMs: Long, numAdd: Int, numRemove: Int, /** The number of [[SetTransaction]] actions in the committed actions. */ numSetTransaction: Int, bytesNew: Long, /** The number of files in the table as of version `readVersion`. */ numFilesTotal: Long, /** The table size in bytes as of version `readVersion`. */ sizeInBytesTotal: Long, /** The number and size of CDC files added in this operation. */ numCdcFiles: Long, cdcBytesNew: Long, /** The protocol as of version `readVersion`. */ protocol: Protocol, /** The size of the newly committed (usually json) file */ commitSizeBytes: Long, /** The size of the checkpoint committed, if present */ checkpointSizeBytes: Long, totalCommitsSizeSinceLastCheckpoint: Long, /** Will we attempt a checkpoint after this commit is completed */ checkpointAttempt: Boolean, info: CommitInfo, newMetadata: Option[Metadata], numAbsolutePathsInAdd: Int, numDistinctPartitionsInAdd: Int, numPartitionColumnsInTable: Int, isolationLevel: String, coordinatedCommitsInfo: CoordinatedCommitsStats, fileSizeHistogram: Option[FileSizeHistogram] = None, addFilesHistogram: Option[FileSizeHistogram] = None, removeFilesHistogram: Option[FileSizeHistogram] = None, numOfDomainMetadatas: Long = 0, txnId: Option[String] = None ) /** * Represents the partition and data predicates of a query on a Delta table. * * Partition predicates can either reference the table's logical partition columns, or the * physical [[AddFile]]'s schema. When a predicate refers to the logical partition columns it needs * to be rewritten to be over the [[AddFile]]'s schema before filtering files. This is indicated * with shouldRewriteFilter=true. * * Currently the only path for a predicate with shouldRewriteFilter=false is through DPO * (dynamic partition overwrite) since we filter directly on [[AddFile.partitionValues]]. * * For example, consider a table with the schema below and partition column "a" * |-- a: integer {physicalName = "XX"} * |-- b: integer {physicalName = "YY"} * * An example of a predicate that needs to be written is: (a = 0) * Before filtering the [[AddFile]]s, this predicate needs to be rewritten to: * (partitionValues.XX = 0) * * An example of a predicate that does not need to be rewritten is: * (partitionValues = Map(XX -> 0)) */ private[delta] case class DeltaTableReadPredicate( partitionPredicates: Seq[Expression] = Seq.empty, dataPredicates: Seq[Expression] = Seq.empty, shouldRewriteFilter: Boolean = true) { val partitionPredicate: Expression = partitionPredicates.reduceLeftOption(And).getOrElse(Literal.TrueLiteral) } /** * Used to perform a set of reads in a transaction and then commit a set of updates to the * state of the log. All reads from the [[DeltaLog]], MUST go through this instance rather * than directly to the [[DeltaLog]] otherwise they will not be check for logical conflicts * with concurrent updates. * * This class is not thread-safe. * * @param deltaLog The Delta Log for the table this transaction is modifying. * @param snapshot The snapshot that this transaction is reading at. */ class OptimisticTransaction( override val deltaLog: DeltaLog, override val catalogTable: Option[CatalogTable], override val snapshot: Snapshot) extends OptimisticTransactionImpl with DeltaLogging { def this( deltaLog: DeltaLog, catalogTable: Option[CatalogTable], snapshotOpt: Option[Snapshot] = None) = this( deltaLog, catalogTable, snapshotOpt.getOrElse(deltaLog.update(catalogTableOpt = catalogTable))) } object CommitConflictFailure { def unapply(e: Exception): Option[Exception] = e match { case _: FileAlreadyExistsException => Some(e) case e: CommitFailedException if e.getConflict => Some(e) case _ => None } } object OptimisticTransaction { private val active = new ThreadLocal[OptimisticTransaction] /** Get the active transaction */ def getActive(): Option[OptimisticTransaction] = Option(active.get()) /** * Runs the passed block of code with the given active transaction. This fails if a transaction is * already active unless `overrideExistingTransaction` is set. */ def withActive[T]( activeTransaction: OptimisticTransaction, overrideExistingTransaction: Boolean = false)(block: => T): T = { val original = getActive() if (overrideExistingTransaction) { clearActive() } setActive(activeTransaction) try { block } finally { clearActive() if (original.isDefined) { setActive(original.get) } } } /** * Sets a transaction as the active transaction. * * @note This is not meant for being called directly, only from * `OptimisticTransaction.withNewTransaction`. Use that to create and set active txns. */ private[delta] def setActive(txn: OptimisticTransaction): Unit = { getActive() match { case Some(activeTxn) => if (!(activeTxn eq txn)) { throw DeltaErrors.activeTransactionAlreadySet() } case _ => active.set(txn) } } /** * Clears the active transaction as the active transaction. * * @note This is not meant for being called directly, `OptimisticTransaction.withNewTransaction`. */ private[delta] def clearActive(): Unit = { active.set(null) } } /** * Used to perform a set of reads in a transaction and then commit a set of updates to the * state of the log. All reads from the [[DeltaLog]], MUST go through this instance rather * than directly to the [[DeltaLog]] otherwise they will not be check for logical conflicts * with concurrent updates. * * This trait is not thread-safe. */ trait OptimisticTransactionImpl extends TransactionHelper with TransactionalWrite with SQLMetricsReporting with DeltaScanGenerator with RecordChecksum with DeltaLogging { import org.apache.spark.sql.delta.util.FileNames._ // Intentionally cache the values of these configs to ensure stable commit code path // and avoid race conditions between committing and dynamic config changes. protected val incrementalCommitEnabled = deltaLog.incrementalCommitEnabled protected val shouldVerifyIncrementalCommit = deltaLog.shouldVerifyIncrementalCommit protected val forcedChecksumValidationInterval = spark.conf.get(DeltaSQLConf.FORCED_CHECKSUM_VALIDATION_INTERVAL) protected val forcedChecksumValidationMinTimeIntervalMinutes = spark.conf.get(DeltaSQLConf.FORCED_CHECKSUM_VALIDATION_MIN_TIME_INTERVAL_MINUTES) def clock: Clock = deltaLog.clock // This would be a quick operation if we already validated the checksum // Otherwise, we should at least perform the validation here. // NOTE: When incremental commits are enabled, skip validation unless it was specifically // requested. This allows us to maintain test converage internally, while avoiding the extreme // overhead of those checks in prod or benchmark settings. if (!incrementalCommitEnabled || shouldVerifyIncrementalCommit) { snapshot.validateChecksum(Map("context" -> "transactionInitialization")) } else if ( forcedChecksumValidationInterval >= 0 && snapshot.version - snapshot.checkpointProvider.version >= forcedChecksumValidationInterval) { val fileToUseForFreshnessCheck = snapshot.checkpointProvider.topLevelFiles .headOption .getOrElse(snapshot.logSegment.deltas.head) // If the table is very fast moving, the checkpoint could much longer than // forcedChecksumValidationInterval to land. To avoid slowing down // such tables, we skip validation if the checkpoint is fresh as per // the modification time. val skipValidationForFastMovingTable = { val checkpointModificationTime = fileToUseForFreshnessCheck.getModificationTime val currentTime = System.currentTimeMillis() val timeGapMillis = Math.max(currentTime - checkpointModificationTime, 0L) // Only force validation if checkpoint is older than the minimum time gap timeGapMillis < MINUTES.toMillis(forcedChecksumValidationMinTimeIntervalMinutes) } if ( !skipValidationForFastMovingTable ) { snapshot.validateChecksum( Map( "context" -> "forceValidateChecksumDueToStaleCheckpoint::transactionInitialization", "currentVersion" -> snapshot.version.toString, "checkpointVersion" -> snapshot.checkpointProvider.version.toString, "forcedValidationInterval" -> forcedChecksumValidationInterval.toString, "forcedValidationMinTimeGap" -> forcedChecksumValidationMinTimeIntervalMinutes.toString ) ) } } /** Tracks the appIds that have been seen by this transaction. */ protected val readTxn = new ArrayBuffer[String] /** * Tracks the data that could have been seen by recording the partition * predicates by which files have been queried by this transaction. */ protected val readPredicates = new java.util.concurrent.ConcurrentLinkedQueue[DeltaTableReadPredicate] /** Tracks specific files that have been seen by this transaction. */ protected val readFiles = new HashSet[AddFile] /** Whether the whole table was read during the transaction. */ protected var readTheWholeTable = false /** Tracks if this transaction has already committed. */ protected var committed: Option[CommittedTransaction] = None def getCommitted: Option[CommittedTransaction] = committed /** Contains the execution instrumentation set via thread-local. No-op by default. */ protected[delta] var executionObserver: TransactionExecutionObserver = TransactionExecutionObserver.getObserver /** * Stores the updated metadata (if any) that will result from this txn. * * This is just one way to change metadata. * New metadata can also be added during commit from actions. * But metadata should *not* be updated via both paths. */ protected var newMetadata: Option[Metadata] = None /** Stores the updated protocol (if any) that will result from this txn. */ protected var newProtocol: Option[Protocol] = None /** The transaction start time. */ private val txnStartNano = System.nanoTime() override val snapshotToScan: Snapshot = snapshot /** * Tracks the first-access snapshots of other Delta logs read by this transaction. * The snapshots are keyed by the log's unique id. */ protected var readSnapshots = new java.util.concurrent.ConcurrentHashMap[(String, Path), Snapshot] /** The transaction commit start time. */ protected var commitStartNano = -1L /** The transaction commit end time. */ protected var commitEndNano = -1L; protected var commitInfo: CommitInfo = _ def getCommitInfo: CommitInfo = commitInfo /** Whether the txn should trigger a checkpoint after the commit */ private[delta] var needsCheckpoint = false // Whether this transaction is creating a new table. private var isCreatingNewTable: Boolean = false // Whether this transaction is overwriting the existing schema (i.e. overwriteSchema = true). // When overwriting schema (and data) of a table, `isCreatingNewTable` should not be true, // except for config REUSE_COLUMN_METADATA_DURING_REPLACE_TABLE is set to true. private var isOverwritingSchema: Boolean = false // Whether this is a transaction that can select any new protocol, potentially downgrading // the existing protocol of the table during REPLACE table operations. private def canAssignAnyNewProtocol: Boolean = readVersion == -1 || (isCreatingNewTable && spark.conf.get(DeltaSQLConf.REPLACE_TABLE_PROTOCOL_DOWNGRADE_ALLOWED)) /** * Tracks the start time since we started trying to write a particular commit. * Used for logging duration of retried transactions. */ protected var commitAttemptStartTimeMillis: Long = _ /** * Tracks actions within the transaction, will commit along with the passed-in actions in the * commit function. */ protected val actions = new ArrayBuffer[Action] /** * Record a SetTransaction action that will be committed as part of this transaction. */ def updateSetTransaction(appId: String, version: Long, lastUpdate: Option[Long]): Unit = { actions += SetTransaction(appId, version, lastUpdate) } /** The version that this transaction is reading from. */ def readVersion: Long = snapshot.version /** Creates new metadata with global Delta configuration defaults. */ private def withGlobalConfigDefaults(metadata: Metadata): Metadata = { val isActiveReplaceCommand = isCreatingNewTable && readVersion != -1 val shouldUnsetCatalogOwnedConf = isActiveReplaceCommand && CatalogOwnedTableUtils.defaultCatalogOwnedEnabled(spark) val conf = if (shouldUnsetCatalogOwnedConf) { // Unset default CatalogOwned enablement iff: // 0. `isCreatingNewTable` indicates that this either is a REPLACE or CREATE command. // 1. `readVersion != 1` indicates the table already exists. // - 0) and 1) suggest that this is an active REPLACE command. // 2. Default CC enablement is set in the spark conf. // This prevents any unintended modifications to the `newProtocol`. // E.g., [[CatalogOwnedTableFeature]] and its dependent features // [[InCommitTimestampTableFeature]] & [[VacuumProtocolCheckTableFeature]]. // // Note that this does *not* affect global spark conf state as we are modifying // the copy of `spark.sessionState.conf`. Thus, `defaultCatalogOwnedFeatureEnabledKey` // will remain unchanged for any concurrent operations that use the same SparkSession. val defaultCatalogOwnedFeatureEnabledKey = TableFeatureProtocolUtils.defaultPropertyKey(CatalogOwnedTableFeature) // Isolate the spark conf to be used in the subsequent [[DeltaConfigs.mergeGlobalConfigs]] // by cloning the existing configuration. // Note: [[SQLConf.clone]] is already atomic so no extra synchronization is needed. val clonedConf = spark.sessionState.conf.clone() // Unset default CC conf on the cloned spark conf. clonedConf.unsetConf(defaultCatalogOwnedFeatureEnabledKey) clonedConf } else { spark.sessionState.conf } metadata.copy(configuration = DeltaConfigs.mergeGlobalConfigs( conf, metadata.configuration)) } protected val postCommitHooks = new ArrayBuffer[PostCommitHook]() registerPostCommitHook(ChecksumHook) catalogTable.foreach { ct => registerPostCommitHook(UpdateCatalogFactory.getUpdateCatalogHook(ct, spark)) } // The CheckpointHook will only checkpoint if necessary, so always register it to run. registerPostCommitHook(CheckpointHook) registerPostCommitHook(HudiConverterHook) /** The protocol of the snapshot that this transaction is reading at. */ def protocol: Protocol = newProtocol.getOrElse(snapshot.protocol) /** Start time of txn in nanoseconds */ def txnStartTimeNs: Long = txnStartNano /** Unique identifier for the transaction */ val txnId: String = UUID.randomUUID().toString /** Whether to check unsupported data type when updating the table schema */ protected var checkUnsupportedDataType: Boolean = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_SCHEMA_TYPE_CHECK) // An array of tuples where each tuple represents a pair (colName, newHighWatermark). // This is collected after a write into Delta table with IDENTITY columns. If it's not // empty, we will update the high water marks during transaction commit. Note that the same // column can have multiple entries here if A single transaction involves multiple write // operations. E.g. Overwrite+ReplaceWhere operation involves two phases: Phase-1 to write just // new data and Phase-2 to delete old data. So both phases can generate tuples for a given column // here. protected val updatedIdentityHighWaterMarks = ArrayBuffer.empty[(String, Long)] // The names of columns for which we will track the IDENTITY high water marks at transaction // writes. protected var trackHighWaterMarks: Option[Set[String]] = None // Set to true if this transaction is ALTER TABLE ALTER COLUMN SYNC IDENTITY. protected var syncIdentity: Boolean = false def setTrackHighWaterMarks(track: Set[String]): Unit = { assert(trackHighWaterMarks.isEmpty, "The tracking set shouldn't have been set") trackHighWaterMarks = Some(track) } def setSyncIdentity(): Unit = { syncIdentity = true } /** * Records an update to the metadata that should be committed with this transaction. As this is * called after write, it skips checking `!hasWritten`. We do not have a full protocol of what * `updating metadata after write` should behave, as currently this is only used to update * IDENTITY columns high water marks. As a result, it goes through all the steps needed to update * schema BEFORE writes, except skipping the check mentioned above. Note that schema evolution * and IDENTITY update can happen inside a single transaction so this function does not check * we have only one metadata update in a transaction. * * IMPORTANT: It is the responsibility of the caller to ensure that files currently present in * the table and written by this transaction are valid under the new metadata. */ private def updateMetadataAfterWrite(updatedMetadata: Metadata): Unit = { updateMetadataInternal(updatedMetadata, ignoreDefaultProperties = false) } // Returns whether this transaction updates metadata solely for IDENTITY high water marks (this // can be either a write that generates IDENTITY values or an ALTER TABLE ALTER COLUMN SYNC // IDENTITY command). This must be called before precommitUpdateSchemaWithIdentityHighWaterMarks // as it might update `newMetadata`. def isIdentityOnlyMetadataUpdate(): Boolean = { syncIdentity || (updatedIdentityHighWaterMarks.nonEmpty && newMetadata.isEmpty) } // Called before commit to update table schema with collected IDENTITY column high water marks // so that the change can be committed to delta log. def precommitUpdateSchemaWithIdentityHighWaterMarks(): Unit = { if (updatedIdentityHighWaterMarks.nonEmpty) { val newSchema = IdentityColumn.updateSchema( deltaLog, metadata.schema, updatedIdentityHighWaterMarks.toSeq ) val updatedMetadata = metadata.copy(schemaString = newSchema.json) updateMetadataAfterWrite(updatedMetadata) } } /** The set of distinct partitions that contain added files by current transaction. */ protected[delta] var partitionsAddedToOpt: Option[mutable.HashSet[Map[String, String]]] = None /** True if this transaction is a blind append. This is only valid after commit. */ protected[delta] var isBlindAppend: Boolean = false /** * The logSegment of the snapshot prior to the commit. * Will be updated only when retrying due to a conflict. */ private[delta] var preCommitLogSegment: LogSegment = snapshot.logSegment.copy(checkpointProvider = snapshot.checkpointProvider) /** The end to end execution time of this transaction. */ def txnExecutionTimeMs: Option[Long] = if (commitEndNano == -1) { None } else { Some(NANOSECONDS.toMillis(commitEndNano - txnStartNano)) } /** Gets the stats collector for the table at the snapshot this transaction has. */ def statsCollector: Column = snapshot.statsCollector /** * Returns the metadata for this transaction. The metadata refers to the metadata of the snapshot * at the transaction's read version unless updated during the transaction. */ def metadata: Metadata = newMetadata.getOrElse(snapshot.metadata) /** * Records an update to the metadata that should be committed with this transaction. * Note that this must be done before writing out any files so that file writing * and checks happen with the final metadata for the table. * * IMPORTANT: It is the responsibility of the caller to ensure that files currently * present in the table are still valid under the new metadata. */ def updateMetadata( proposedNewMetadata: Metadata, ignoreDefaultProperties: Boolean = false): Unit = { assert(!hasWritten, "Cannot update the metadata in a transaction that has already written data.") assert(newMetadata.isEmpty, "Cannot change the metadata more than once in a transaction.") updateMetadataInternal(proposedNewMetadata, ignoreDefaultProperties) // Temporary: block metadata changes on UC-managed CatalogOwned tables until Delta supports // propagating metadata updates to UC. UC is identified by catalog implementation class (handles // "spark_catalog" registration). New table creation is naturally excluded because // isCatalogOwned is false until the first commit. REPLACE TABLE is currently also blocked // here and will need to be explicitly allowed once UC supports metadata propagation. // Intentionally conservative: configuration is compared as a whole map, which also // catches Delta-internal additions (e.g. table-feature flags). This is acceptable for // a temporary kill switch - once Delta supports propagating metadata updates to UC, // this check will be removed entirely. if (!isCreatingNewTable) { throwIfUCManagedMetadataChanged(snapshot.metadata, context = "updateMetadata") } } /** * Returns true if the proposed metadata differs from the existing metadata for a UC-managed * table. */ private def hasUCManagedMetadataChange( existingMetadata: Metadata, proposedMetadata: Metadata): Boolean = { proposedMetadata.schemaString != existingMetadata.schemaString || proposedMetadata.partitionColumns != existingMetadata.partitionColumns || proposedMetadata.description != existingMetadata.description || proposedMetadata.configuration != existingMetadata.configuration } private def throwIfUCManagedMetadataChanged( existingMetadata: Metadata, context: String): Unit = { val proposedMetadata = newMetadata.getOrElse(existingMetadata) if (isUCManagedTable && hasUCManagedMetadataChange(existingMetadata, proposedMetadata)) { logWarning(log"Blocking UC-managed metadata update during " + log"${MDC(DeltaLogKeys.OPERATION, context)} because metadata changed: " + log"${MDC(DeltaLogKeys.METADATA_OLD, existingMetadata)} => " + log"${MDC(DeltaLogKeys.METADATA_NEW, proposedMetadata)}") throw DeltaErrors.operationNotSupportedException( "Metadata changes on Unity Catalog managed tables") } } /** * True if this transaction targets a UC-managed CatalogOwned table. * * Computed once as a lazy val because catalogTable and SparkSession are immutable for * the lifetime of a transaction. Visibility is protected[delta] (not private) to allow * test subclasses to override without requiring UCSingleCatalog. */ protected[delta] lazy val isUCManagedTable: Boolean = { snapshot.isCatalogOwned && catalogTable.exists { ct => ct.tableType == CatalogTableType.MANAGED && CatalogOwnedTableUtils.getCatalogName(spark, ct.identifier) .contains(UCCommitCoordinatorBuilder.COORDINATOR_NAME) } } /** * Returns true if committing [[dm]] would change the clustering columns on a UC-managed * CatalogOwned table and should therefore be blocked. * * Both a missing entry and a removed=true tombstone mean "no clustering", so the effective * configuration is normalised to Option[String] before comparison. */ private def isClusteringChangedOnUCManagedTable(dm: DomainMetadata): Boolean = { if (dm.domain != ClusteringMetadataDomain.domainName) return false if (!isUCManagedTable) return false val existingConfig = snapshot.domainMetadata .find(_.domain == ClusteringMetadataDomain.domainName) .filterNot(_.removed) .map(_.configuration) val incomingConfig = if (dm.removed) None else Some(dm.configuration) incomingConfig != existingConfig } /** * Can this transaction still update the metadata? * This is allowed only once per transaction. */ def canUpdateMetadata: Boolean = { !hasWritten && newMetadata.isEmpty } /** * This updates the protocol for the table with a given protocol. * Note that the protocol set by this method can be overwritten by other methods, * such as [[updateMetadata]]. */ def updateProtocol(protocol: Protocol): Unit = { newProtocol = Some(protocol) } /** * Do the actual checks and works to update the metadata and save it into the `newMetadata` * field, which will be added to the actions to commit in [[prepareCommit]]. */ protected def updateMetadataInternal( proposedNewMetadata: Metadata, ignoreDefaultProperties: Boolean): Unit = { var newMetadataTmp = proposedNewMetadata // Validate all indexed columns are inside table's schema. StatisticsCollection.validateDeltaStatsColumns(newMetadataTmp) if (readVersion == -1 || isCreatingNewTable) { // We need to ignore the default properties when trying to create an exact copy of a table // (as in CLONE and SHALLOW CLONE). if (!ignoreDefaultProperties) { newMetadataTmp = withGlobalConfigDefaults(newMetadataTmp) } isCreatingNewTable = true } val identityColumnAllowed = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_IDENTITY_COLUMN_ENABLED) if (!identityColumnAllowed && ColumnWithDefaultExprUtils.hasIdentityColumn(newMetadataTmp.schema)) { throw DeltaErrors.unsupportedWriterTableFeaturesInTableException( deltaLog.dataPath.toString, Seq(IdentityColumnsTableFeature.name)) } val protocolBeforeUpdate = protocol // `readVersion == -1` indicates the current transaction is reading from a snapshot // where the table has not existed yet. // `isCreatingNewTable` will be true for commands like REPLACE and CREATE, // this is just a double check since we only want to auto-enable QoL features // when creating a new CatalogOwned table through CREATE. if (CatalogOwnedTableUtils.shouldEnableCatalogOwned( spark, propertyOverrides = newMetadataTmp.configuration) && isCreatingNewTable && this.readVersion == -1) { // For CatalogOwned table, we add "quality of life" table features as a part of CCv2 table // creation. Look for [[CatalogOwnedTableUtils.updateMetadataForQoLFeatures]] to see // what features are in the list. // Note that we need to add features here because features like `ColumnMapping` or // `RowTracking` have their own validation/update logic below. newMetadataTmp = CatalogOwnedTableUtils.updateMetadataForQoLFeatures( spark, metadata = newMetadataTmp ) } // The `.schema` cannot be generated correctly unless the column mapping metadata is correctly // filled for all the fields. Therefore, the column mapping changes need to happen first. newMetadataTmp = DeltaColumnMapping.verifyAndUpdateMetadataChange( spark, deltaLog, protocolBeforeUpdate, snapshot.metadata, newMetadataTmp, isCreatingNewTable, isOverwritingSchema) if (newMetadataTmp.schemaString != null) { // Replace CHAR and VARCHAR with StringType val schema = CharVarcharUtils.replaceCharVarcharWithStringInSchema( newMetadataTmp.schema) newMetadataTmp = newMetadataTmp.copy(schemaString = schema.json) } newMetadataTmp = if (snapshot.metadata.schemaString == newMetadataTmp.schemaString) { // Shortcut when the schema hasn't changed to avoid generating spurious schema change logs. // It's fine if two different but semantically equivalent schema strings skip this special // case - that indicates that something upstream attempted to do a no-op schema change, and // we'll just end up doing a bit of redundant work in the else block. newMetadataTmp } else { val fixedSchema = SchemaUtils.removeUnenforceableNotNullConstraints( newMetadataTmp.schema, spark.sessionState.conf).json newMetadataTmp.copy(schemaString = fixedSchema) } if (canAssignAnyNewProtocol) { // Check for the new protocol version after the removal of the unenforceable not null // constraints newProtocol = Some(Protocol.forNewTable(spark, Some(newMetadataTmp))) } else if (newMetadataTmp.configuration.contains(Protocol.MIN_READER_VERSION_PROP) || newMetadataTmp.configuration.contains(Protocol.MIN_WRITER_VERSION_PROP)) { // Table features Part 1: bump protocol version numbers // // Collect new reader and writer versions from table properties, which could be provided by // the user in `ALTER TABLE TBLPROPERTIES` or copied over from session defaults. val readerVersionAsTableProp = Protocol.getReaderVersionFromTableConf(newMetadataTmp.configuration) .getOrElse(protocolBeforeUpdate.minReaderVersion) val writerVersionAsTableProp = Protocol.getWriterVersionFromTableConf(newMetadataTmp.configuration) .getOrElse(protocolBeforeUpdate.minWriterVersion) val newProtocolForLatestMetadata = Protocol(readerVersionAsTableProp, writerVersionAsTableProp) // The user-supplied protocol version numbers are treated as a group of features // that must all be enabled. This ensures that the feature-enabling behavior is the // same on Table Features-enabled protocols as on legacy protocols, i.e., exactly // the same set of features are enabled. // // This is useful for supporting protocol downgrades to legacy protocol versions. // When the protocol versions are explicitly set on table features protocol we may // normalize to legacy protocol versions. Legacy protocol versions can only be // used if a table supports *exactly* the set of features in that legacy protocol // version, with no "gaps". By merging in the protocol features from a particular // protocol version, we may end up with such a "gap-free" protocol. E.g. if a table // has only table feature "checkConstraints" (added by writer protocol version 3) // but not "invariants" and "appendOnly", then setting the minWriterVersion to // 2 or 3 will add "invariants" and "appendOnly", filling in the gaps for writer // protocol version 3, and then we can downgrade to version 3. val proposedNewProtocol = protocolBeforeUpdate.merge(newProtocolForLatestMetadata) if (proposedNewProtocol != protocolBeforeUpdate) { // The merged protocol has higher versions and/or supports more features. // It's a valid upgrade. newProtocol = Some(proposedNewProtocol) } else { // The merged protocol is identical to the original one. Two possibilities: // (1) the provided versions are lower than the original one, and all features supported by // the provided versions are already supported. This is a no-op. if (readerVersionAsTableProp < protocolBeforeUpdate.minReaderVersion || writerVersionAsTableProp < protocolBeforeUpdate.minWriterVersion) { recordProtocolChanges( "delta.protocol.downgradeIgnored", fromProtocol = protocolBeforeUpdate, toProtocol = newProtocolForLatestMetadata, isCreatingNewTable = false) } else { // (2) the new protocol versions is identical to the existing versions. Also a no-op. } } } newMetadataTmp = if (isCreatingNewTable) { // Creating a new table will drop all existing data, so we don't need to fix the old // metadata. newMetadataTmp } else { // This is not a new table. The new schema may be merged from the existing schema. We // decide whether we should keep the Generated or IDENTITY columns by checking whether the // protocol satisfies the requirements. val keepGeneratedColumns = GeneratedColumn.satisfyGeneratedColumnProtocol(protocolBeforeUpdate) val keepIdentityColumns = ColumnWithDefaultExprUtils.satisfiesIdentityColumnProtocol(protocolBeforeUpdate) if (keepGeneratedColumns && keepIdentityColumns) { // If a protocol satisfies both requirements, we do nothing here. newMetadataTmp } else { // As the protocol doesn't match, this table is created by an old version that doesn't // support generated columns or identity columns. We should remove the generation // expressions to fix the schema to avoid bumping the writer version incorrectly. val newSchema = ColumnWithDefaultExprUtils.removeDefaultExpressions( newMetadataTmp.schema, keepGeneratedColumns = keepGeneratedColumns, keepIdentityColumns = keepIdentityColumns) if (newSchema ne newMetadataTmp.schema) { newMetadataTmp.copy(schemaString = newSchema.json) } else { newMetadataTmp } } } // Table features Part 2: add manually-supported features specified in table properties, aka // those start with [[FEATURE_PROP_PREFIX]]. // // This transaction's new metadata might contain some table properties to support some // features (props start with [[FEATURE_PROP_PREFIX]]). We silently add them to the `protocol` // action, and bump the protocol version to (3, 7) or (_, 7), depending on the existence of // any reader-writer feature. val newProtocolBeforeAddingFeatures = newProtocol.getOrElse(protocolBeforeUpdate) val newFeaturesFromTableConf = TableFeatureProtocolUtils.getSupportedFeaturesFromTableConfigs(newMetadataTmp.configuration) val readerVersionForNewProtocol = { // All features including those required features are considered to decide reader version. if (Protocol() .withFeatures(newFeaturesFromTableConf) .readerAndWriterFeatureNames .flatMap(TableFeature.featureNameToFeature) .exists(_.isReaderWriterFeature)) { TableFeatureProtocolUtils.TABLE_FEATURES_MIN_READER_VERSION } else { newProtocolBeforeAddingFeatures.minReaderVersion } } val existingFeatureNames = newProtocolBeforeAddingFeatures.readerAndWriterFeatureNames if (!newFeaturesFromTableConf.map(_.name).subsetOf(existingFeatureNames)) { newProtocol = Some( Protocol( readerVersionForNewProtocol, TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(newFeaturesFromTableConf) .merge(newProtocolBeforeAddingFeatures)) } // For CatalogOwned table feature, we don't support the upgrade yet. newProtocol.foreach { p => if (!isCreatingNewTable && p.readerAndWriterFeatureNames.contains(CatalogOwnedTableFeature.name) && !existingFeatureNames.contains(CatalogOwnedTableFeature.name)) { throw new NotImplementedException("Upgrading to CatalogOwned table is not yet " + s"supported. Please create a new table with the CatalogOwned table feature.") } } // We are done with protocol versions and features, time to remove related table properties. val configsWithoutProtocolProps = Protocol.filterProtocolPropsFromTableProps(newMetadataTmp.configuration) // Table features Part 3: add automatically-enabled features by looking at the new table // metadata. // // This code path is for existing tables and during `REPLACE` if the downgrade flag is not set. // The new table case has been handled by [[Protocol.forNewTable]] earlier in this method. if (!canAssignAnyNewProtocol) { setNewProtocolWithFeaturesEnabledByMetadata(newMetadataTmp) } if (isCreatingNewTable) { IdentityColumn.logTableCreation(deltaLog, newMetadataTmp.schema) } newMetadataTmp = newMetadataTmp.copy(configuration = configsWithoutProtocolProps) Protocol.assertMetadataContainsNoProtocolProps(newMetadataTmp) newMetadataTmp = MaterializedRowId.updateMaterializedColumnName( protocol, oldMetadata = snapshot.metadata, newMetadataTmp) newMetadataTmp = MaterializedRowCommitVersion.updateMaterializedColumnName( protocol, oldMetadata = snapshot.metadata, newMetadataTmp) assertMetadata(newMetadataTmp) logInfo(log"Updated metadata from " + log"${MDC(DeltaLogKeys.METADATA_OLD, newMetadata.getOrElse("-"))} to " + log"${MDC(DeltaLogKeys.METADATA_NEW, newMetadataTmp)}") newMetadata = Some(newMetadataTmp) // Check that the metadata change is valid for CDC enabled tables. performCdcMetadataCheck() } /** * Records an update to the metadata that should be committed with this transaction and when * this transaction is logically creating a new table, e.g. replacing a previous table with new * metadata. Note that this must be done before writing out any files so that file writing * and checks happen with the final metadata for the table. * IMPORTANT: It is the responsibility of the caller to ensure that files currently * present in the table are still valid under the new metadata. */ def updateMetadataForNewTable(metadata: Metadata): Unit = { isCreatingNewTable = true updateMetadata(metadata) } /** * Updates the metadata of the target table in an effective REPLACE command. Note that replacing * a table is similar to dropping a table and then recreating it. However, the backing catalog * object does not change. For now, for Coordinated Commit tables, this function retains the * coordinator details (and other associated Coordinated Commits properties) from the original * table during a REPLACE. And if the table had a coordinator, existing ICT properties are also * retained; otherwise, default ICT properties are included. * TODO (YumingxuanGuo): Remove this once the exact semantic on default Coordinated Commits * configurations is finalized. */ def updateMetadataForNewTableInReplace(metadata: Metadata): Unit = { assert(CoordinatedCommitsUtils.getExplicitCCConfigurations(metadata.configuration).isEmpty, "Command-specified Coordinated Commits configurations should have been blocked earlier.") assert(!metadata.configuration.contains(UCCommitCoordinatorClient.UC_TABLE_ID_KEY), "Command-specified Catalog-Owned table UUID (ucTableId) should have been blocked earlier.") // Extract any existing ucTableId from the snapshot metadata. val existingUCTableIdConf: Map[String, String] = snapshot.metadata.configuration.filter { case (k, v) => k == UCCommitCoordinatorClient.UC_TABLE_ID_KEY } // Extract the existing Coordinated Commits configurations and ICT dependency configurations // from the existing table metadata. val existingCCConfs = CoordinatedCommitsUtils.getExplicitCCConfigurations(snapshot.metadata.configuration) val existingICTConfs = CoordinatedCommitsUtils.getExplicitICTConfigurations(snapshot.metadata.configuration) val existingQoLConfs = CoordinatedCommitsUtils.getExplicitQoLConfigurations(snapshot.metadata.configuration) val oldMappingMode = snapshot.metadata.columnMappingMode val newMappingMode = metadata.columnMappingMode val shouldReuseColumnMetadataForReplaceTable = spark.conf.get(DeltaSQLConf.REUSE_COLUMN_METADATA_DURING_REPLACE_TABLE) if (oldMappingMode == newMappingMode && shouldReuseColumnMetadataForReplaceTable) { isOverwritingSchema = true } // Update the metadata. updateMetadataForNewTable(metadata) // Now the `txn.metadata` contains all the command-specified properties and all the default // properties. The latter might still contain Coordinated Commits configurations, so we need // to remove them and retain the Coordinated Commits configurations from the existing table. val newConfsWithoutCC = newMetadata.get.configuration -- CoordinatedCommitsUtils.TABLE_PROPERTY_KEYS val existingQoLConfsToRetain = existingQoLConfs.filterNot { case (key, _) => newConfsWithoutCC.contains(key) } var newConfs: Map[String, String] = newConfsWithoutCC ++ existingCCConfs ++ existingQoLConfsToRetain ++ existingUCTableIdConf // We also need to retain the existing ICT dependency configurations, but only when the // existing table does have Coordinated Commits configurations or Catalog-Owned enabled. // Otherwise, we treat the ICT configurations the same as any other configurations, // by merging them from the default. val isCatalogOwnedEnabledBeforeReplace = snapshot.protocol .readerAndWriterFeatureNames.contains(CatalogOwnedTableFeature.name) if (existingCCConfs.nonEmpty || isCatalogOwnedEnabledBeforeReplace) { val newConfsWithoutICT = newConfs -- CoordinatedCommitsUtils.ICT_TABLE_PROPERTY_KEYS newConfs = newConfsWithoutICT ++ existingICTConfs } newMetadata = Some(newMetadata.get.copy(configuration = newConfs)) throwIfUCManagedMetadataChanged( snapshot.metadata, context = "updateMetadataForNewTableInReplace") } /** * Records an update to the metadata that should be committed with this transaction and when * this transaction is attempt to overwrite the data and schema using .mode('overwrite') and * .option('overwriteSchema', true). * REPLACE the table is not considered in this category, because that is logically equivalent * to DROP and RECREATE the table. */ def updateMetadataForTableOverwrite(proposedNewMetadata: Metadata): Unit = { isOverwritingSchema = true updateMetadata(proposedNewMetadata) } /** * Remove the 'EXISTS_DEFAULT' metadata key from the schema. This is used for new tables that are * not re-using data files of existing tables (i.e. CREATE TABLE, REPLACE TABLE, CTAS). It is not * used on code paths of commands that create new tables but re-use data files (i.e. CONVERT TO * DELTA, CLONE) because we cannot assure that 'EXISTS_DEFAULT' values is actually not required * without reading the data. */ def removeExistsDefaultFromSchema(): Unit = { if (spark.sessionState.conf.getConf(DeltaSQLConf.REMOVE_EXISTS_DEFAULT_FROM_SCHEMA)) { if (newMetadata.isDefined) { val schemaWithRemovedExistsDefaults = SchemaUtils.removeExistsDefaultMetadata(newMetadata.get.schema) if (schemaWithRemovedExistsDefaults != newMetadata.get.schema) { newMetadata = newMetadata.map(_.copy(schemaString = schemaWithRemovedExistsDefaults.json)) } } } } protected def assertMetadata(metadata: Metadata): Unit = { assert(!CharVarcharUtils.hasCharVarchar(metadata.schema), "The schema in Delta log should not contain char/varchar type.") SchemaMergingUtils.checkColumnNameDuplication(metadata.schema, "in the metadata update") if (metadata.columnMappingMode == NoMapping) { SchemaUtils.checkSchemaFieldNames(metadata.dataSchema, metadata.columnMappingMode) val partitionColCheckIsFatal = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_PARTITION_COLUMN_CHECK_ENABLED) try { SchemaUtils.checkFieldNames(metadata.partitionColumns) } catch { case e: AnalysisException => recordDeltaEvent( deltaLog, "delta.schema.invalidPartitionColumn", data = Map( "checkEnabled" -> partitionColCheckIsFatal, "columns" -> metadata.partitionColumns ) ) if (partitionColCheckIsFatal) throw DeltaErrors.invalidPartitionColumn(e) } } else { DeltaColumnMapping.checkColumnIdAndPhysicalNameAssignments(metadata) } if (GeneratedColumn.hasGeneratedColumns(metadata.schema)) { recordDeltaOperation(deltaLog, "delta.generatedColumns.check") { GeneratedColumn.validateGeneratedColumns(spark, metadata.schema) } recordDeltaEvent(deltaLog, "delta.generatedColumns.definition") } if (checkUnsupportedDataType) { val unsupportedTypes = SchemaUtils.findUnsupportedDataTypes(metadata.schema) if (unsupportedTypes.nonEmpty) { throw DeltaErrors.unsupportedDataTypes(unsupportedTypes.head, unsupportedTypes.tail: _*) } } if (spark.conf.get(DeltaSQLConf.DELTA_TABLE_PROPERTY_CONSTRAINTS_CHECK_ENABLED)) { Protocol.assertTablePropertyConstraintsSatisfied(spark, metadata, snapshot) } MaterializedRowId.throwIfMaterializedColumnNameConflictsWithSchema(metadata) MaterializedRowCommitVersion.throwIfMaterializedColumnNameConflictsWithSchema(metadata) } /** * Some features require their pre-requisite features to not only be present * in the protocol but also be enabled. This method sets the flags required * to enable these pre-requisite features. */ private def getMetadataWithDependentFeaturesEnabled( metadata: Metadata, protocols: Seq[Protocol]): Metadata = { if (DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.fromMetaData(metadata).isDefined || protocols.exists(_.readerAndWriterFeatureNames.contains(CatalogOwnedTableFeature.name))) { // coordinated-commits/catalog-owned require ICT to be enabled as per the spec. // If ICT is just in Protocol and not in Metadata, // then it is in a 'supported' state but not enabled. // In order to enable ICT, we have to set the table property in Metadata. val ictEnablementConfigOpt = Option.when(!DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(metadata))( (DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key -> "true")) val configWithICT = metadata.configuration ++ ictEnablementConfigOpt metadata.copy(configuration = configWithICT) } else { metadata } } private def setNewProtocolWithFeaturesEnabledByMetadata(metadata: Metadata): Unit = { val requiredProtocolOpt = Protocol.upgradeProtocolFromMetadataForExistingTable(spark, metadata, protocol) if (requiredProtocolOpt.isDefined) { newProtocol = requiredProtocolOpt } } // Make sure shredded writes are only performed if the shredding table property was set. private def assertShreddingStateConsistent() = { if (!DeltaConfigs.ENABLE_VARIANT_SHREDDING.fromMetaData(metadata)) { val isVariantShreddingSchemaForced = spark.sessionState.conf .getConfString("spark.sql.variant.forceShreddingSchemaForTest", "").nonEmpty if (isVariantShreddingSchemaForced) { throw DeltaErrors.variantShreddingUnsupported() } } } /** * Must make sure that deletion vectors are never added to a table where that isn't allowed. * Note, statistics recomputation is still allowed even though DVs might be currently disabled. * * This method returns a function that can be used to validate a single Action. */ protected def getAssertDeletionVectorWellFormedFunc( spark: SparkSession, op: DeltaOperations.Operation): (Action => Unit) = { val commitCheckEnabled = spark.conf.get(DeltaSQLConf.DELETION_VECTORS_COMMIT_CHECK_ENABLED) if (!commitCheckEnabled) { return _ => {} } // Whether DVs are supported, i.e. the table is allowed to contain any DVs. val deletionVectorsSupported = DeletionVectorUtils.deletionVectorsReadable(snapshot, newProtocol, newMetadata) // Whether DVs are enabled, i.e. operations are allowed to create new DVs. val deletionVectorsEnabled = DeletionVectorUtils.deletionVectorsWritable(snapshot, newProtocol, newMetadata) // If the operation does not define whether it performs in-place metadata updates, we are // conservative and assume that it is not, which makes the check stricter. val isInPlaceFileMetadataUpdate = op.isInPlaceFileMetadataUpdate.getOrElse(false) val deletionVectorAllowedForAddFiles = deletionVectorsSupported && (deletionVectorsEnabled || isInPlaceFileMetadataUpdate) val addFileMustHaveWideBounds = op.checkAddFileWithDeletionVectorStatsAreNotTightBounds action => action match { case a: AddFile if a.deletionVector != null => if (!deletionVectorAllowedForAddFiles) { throw DeltaErrors.addingDeletionVectorsDisallowedException() } // Protocol requirement checks: // 1. All files with DVs must have `stats` with `numRecords`. if (a.stats == null || a.numPhysicalRecords.isEmpty) { throw DeltaErrors.addFileWithDVsMissingNumRecordsException } // 2. All operations that add new DVs should always turn bounds to wide. // Operations that only update files with existing DVs may opt-out from this rule // via `checkAddFileWithDeletionVectorStatsAreNotTightBounds`. // See that field comment in DeltaOperation for more details. // Note, the absence of the tightBounds column when DVs exist is also an illegal state. if (addFileMustHaveWideBounds && // Extra inversion to also catch absent `tightBounds`. !a.tightBounds.contains(false)) { throw DeltaErrors.addFileWithDVsAndTightBoundsException() } case _ => // Not an AddFile, nothing to do. } } /** * Returns the [[DeltaScanGenerator]] for the given log, which will be used to generate * [[DeltaScan]]s. Every time this method is called on a log, the returned generator * generator will read a snapshot that is pinned on the first access for that log. * * Internally, if the given log is the same as the log associated with this * transaction, then it returns this transaction, otherwise it will return a snapshot of * given log */ def getDeltaScanGenerator(index: TahoeLogFileIndex): DeltaScanGenerator = { if (index.deltaLog.isSameLogAs(deltaLog)) return this val compositeId = index.deltaLog.compositeId // Will be called only when the log is accessed the first time readSnapshots.computeIfAbsent(compositeId, _ => index.getSnapshot) } /** Returns a[[DeltaScan]] based on the given filters. */ override def filesForScan( filters: Seq[Expression], keepNumRecords: Boolean = false ): DeltaScan = { val scan = snapshot.filesForScan(filters, keepNumRecords) trackReadPredicates(filters) trackFilesRead(scan.files) scan } /** Returns a[[DeltaScan]] based on the given partition filters, projections and limits. */ override def filesForScan( limit: Long, partitionFilters: Seq[Expression]): DeltaScan = { partitionFilters.foreach { f => assert( DeltaTableUtils.isPredicatePartitionColumnsOnly(f, metadata.partitionColumns, spark), s"Only filters on partition columns [${metadata.partitionColumns.mkString(", ")}]" + s" expected, found $f") } val scan = snapshot.filesForScan(limit, partitionFilters) trackReadPredicates(partitionFilters, partitionOnly = true) trackFilesRead(scan.files) scan } override def filesWithStatsForScan(partitionFilters: Seq[Expression]): DataFrame = { val metadata = snapshot.filesWithStatsForScan(partitionFilters) trackReadPredicates(partitionFilters, partitionOnly = true) trackFilesRead(filterFiles(partitionFilters)) metadata } /** Returns files matching the given predicates. */ def filterFiles(): Seq[AddFile] = filterFiles(Seq(Literal.TrueLiteral)) /** Returns files matching the given predicates. */ def filterFiles(filters: Seq[Expression], keepNumRecords: Boolean = false): Seq[AddFile] = { val scan = snapshot.filesForScan(filters, keepNumRecords) trackReadPredicates(filters) trackFilesRead(scan.files) scan.files } /** * Returns files within the given partitions. * * `partitions` is a set of the `partitionValues` stored in [[AddFile]]s. This means they refer to * the physical column names, and values are stored as strings. * */ def filterFiles(partitions: Set[Map[String, String]]): Seq[AddFile] = { import org.apache.spark.sql.functions.col val df = snapshot.allFiles.toDF() val isFileInTouchedPartitions = DeltaUDF.booleanFromMap(partitions.contains)(col("partitionValues")) val filteredFiles = df .filter(isFileInTouchedPartitions) .withColumn("stats", DataSkippingReader.nullStringLiteral) .as[AddFile] .collect() trackReadPredicates( Seq(isFileInTouchedPartitions.expr), partitionOnly = true, shouldRewriteFilter = false) filteredFiles } /** Mark the entire table as tainted by this transaction. */ def readWholeTable(): Unit = { trackReadPredicates(Seq.empty) readTheWholeTable = true } /** Mark the given files as read within this transaction. */ def trackFilesRead(files: Seq[AddFile]): Unit = { readFiles ++= files } /** Mark the predicates that have been queried by this transaction. */ def trackReadPredicates( filters: Seq[Expression], partitionOnly: Boolean = false, shouldRewriteFilter: Boolean = true): Unit = { val (partitionFilters, dataFilters) = if (partitionOnly) { (filters, Seq.empty[Expression]) } else { filters.partition { f => DeltaTableUtils.isPredicatePartitionColumnsOnly(f, metadata.partitionColumns, spark) } } readPredicates.add(DeltaTableReadPredicate( partitionPredicates = partitionFilters, dataPredicates = dataFilters, shouldRewriteFilter = shouldRewriteFilter) ) } /** * Returns the latest version that has committed for the idempotent transaction with given `id`. */ def txnVersion(id: String): Long = { readTxn += id snapshot.transactions.getOrElse(id, -1L) } /** * Return the operation metrics for the operation if it is enabled */ def getOperationMetrics(op: Operation): Option[Map[String, String]] = { if (spark.conf.get(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED)) { Some(getMetricsForOperation(op)) } else { None } } def reportAutoCompactStatsError(e: Throwable): Unit = { recordDeltaEvent(deltaLog, "delta.collectStats", data = Map("message" -> e.getMessage)) logError(e.getMessage) } /** * Collects auto optimize stats from the given actions. * This method computes the stats as a side effect of iterating through the actions. * The computed stats are only available after the returned iterator is fully consumed. * `finalizeStats` must be called on the returned collector to finalize the stats. * @param actions An iterator of actions that are being committed in this transaction. * @return A tuple containing: * 1. An iterator of actions with the auto optimize stats computed as a side effect. * 2. An instance of [[AutoCompactPartitionStatsCollector]] that contains the * computed stats. */ private def collectAutoOptimizeStats( actions: Iterator[Action]): (Iterator[Action], AutoCompactPartitionStatsCollector) = { val collector = createAutoCompactStatsCollector() if (collector.isInstanceOf[DisabledAutoCompactPartitionStatsCollector]) { return (actions, collector) } val actionsIter = AutoCompactPartitionStats.instance(spark) .collectPartitionStats(collector, actions) (actionsIter, collector) } /** * Collects auto optimize stats from the given actions and finalizes the stats. * This method consumes the actions iterator to compute the stats and then finalizes them. * @param actions A sequence of actions that are being committed in this transaction. * @param tableId The ID of the table for which the stats are being collected. */ def collectAutoOptimizeStatsAndFinalize( actions: Seq[Action], tableId: String): Unit = { val (actionsIter, acStatsCollector) = collectAutoOptimizeStats(actions.toIterator) // Consume the iterator to hydrate the stats collector. actionsIter.foreach(_ => ()) acStatsCollector.finalizeStats(tableId) } /** * A subclass of AutoCompactPartitionStatsCollector that's to be used if the config to collect * auto compaction stats is turned off. This subclass intentionally does nothing. */ class DisabledAutoCompactPartitionStatsCollector extends AutoCompactPartitionStatsCollector { override def collectPartitionStatsForAdd(file: AddFile): Unit = {} override def collectPartitionStatsForRemove(file: RemoveFile): Unit = {} override def finalizeStats(tableId: String): Unit = {} } def createAutoCompactStatsCollector(): AutoCompactPartitionStatsCollector = { try { if (spark.conf.get(DeltaSQLConf.DELTA_AUTO_COMPACT_RECORD_PARTITION_STATS_ENABLED)) { val minFileSize = spark.conf .get(DeltaSQLConf.DELTA_AUTO_COMPACT_MIN_FILE_SIZE) .getOrElse(Long.MaxValue) return AutoCompactPartitionStats.instance(spark) .createStatsCollector(minFileSize, reportAutoCompactStatsError) } } catch { case NonFatal(e) => reportAutoCompactStatsError(e) } // If config-disabled, or error caught, fall though and use a no-op stats collector. new DisabledAutoCompactPartitionStatsCollector } /** * Checks if the new schema contains any CDC columns (which is invalid) and throws the appropriate * error */ protected def performCdcMetadataCheck(): Unit = { if (newMetadata.nonEmpty) { CDCReader.checkMetadataChange( spark, newMetadata = newMetadata.get, oldMetadata = snapshot.metadata) } } /** * Validates that an AddFile does not reference a zero-byte parquet file. * Logs a delta event regardless of the flag; throws only when * [[DeltaSQLConf.DELTA_EMPTY_FILE_CHECK_THROW_ENABLED]] is set. */ protected def validateAddFileNotEmpty(addFile: AddFile): Unit = { if (addFile.size == 0) { recordDeltaEvent( deltaLog, "delta.sanityCheck.emptyParquetFile", data = Map( "addFile" -> addFile.json, "stackTrace" -> Thread.currentThread().getStackTrace.take(20).mkString("\n") )) if (spark.conf.get(DeltaSQLConf.DELTA_EMPTY_FILE_CHECK_THROW_ENABLED)) { throw new IllegalStateException( s"AddFile ${addFile.path} references a zero-byte (empty) parquet file") } } } /** * Validates that partition columns that have NOT NULL * constraints are not null in the AddFile action. * * @param addFile The AddFile action to validate * @param notNullPartitionCols Partition columns with NOT NULL constraints */ protected def validateAddFileForNullPartitions( addFile: AddFile, notNullPartitionCols: Set[String]): Unit = { notNullPartitionCols.foreach { col => addFile.partitionValues.get(col) match { case None | Some(null) => recordDeltaEvent( deltaLog, "delta.constraints.nullPartitionViolation", data = Map( "addFile" -> addFile.json, "notNullPartitionCols" -> notNullPartitionCols.toSeq.mkString(","), "stackTrace" -> Thread.currentThread().getStackTrace.take(20).mkString("\n") )) if (spark.conf.get(DeltaSQLConf.DELTA_NULL_PARTITION_CHECK_THROW_ENABLED)) { throw new IllegalStateException( s"AddFile ${addFile.path} has null partition value for NOT NULL column '$col'") } case Some(_) => // Valid non-null partition value } } } /** * Returns the physical names of partition columns that have NOT NULL constraints. * Physical names are used because AddFile.partitionValues keys use physical column names * when column mapping is enabled. */ protected def getNotNullPartitionCols(metadata: Metadata): Set[String] = { val notNullColumns = Invariants.getFromSchema(metadata.schema, spark) .collect { case Constraints.NotNull(cols) => cols.mkString(".") } .toSet metadata.partitionSchema .filter(f => notNullColumns.contains(f.name)) .map(DeltaColumnMapping.getPhysicalName) .toSet } /** * Runs all AddFile sanity checks: empty-file detection and null-partition validation. */ protected def validateAddFileInvariants( addFile: AddFile, notNullPartitionCols: Set[String]): Unit = { validateAddFileNotEmpty(addFile) validateAddFileForNullPartitions(addFile, notNullPartitionCols) } /** * Iterates over all actions and validates AddFile invariants. */ protected def validateActionsAddFileInvariants( actions: Seq[Action], metadata: Metadata): Unit = { val notNullPartitionCols = getNotNullPartitionCols(metadata) actions.foreach { case a: AddFile => validateAddFileInvariants(a, notNullPartitionCols) case _ => } } /** * Checks if the passed-in actions have internal SetTransaction conflicts, will throw exceptions * in case of conflicts. This function will also remove duplicated [[SetTransaction]]s. */ protected def checkForSetTransactionConflictAndDedup(actions: Seq[Action]): Seq[Action] = { val finalActions = new ArrayBuffer[Action] val txnIdToVersionMap = new mutable.HashMap[String, Long].empty for (action <- actions) { action match { case st: SetTransaction => txnIdToVersionMap.get(st.appId).map { version => if (version != st.version) { throw DeltaErrors.setTransactionVersionConflict(st.appId, version, st.version) } } getOrElse { txnIdToVersionMap += (st.appId -> st.version) finalActions += action } case _ => finalActions += action } } finalActions.toSeq } /** * We want to future-proof and explicitly block any occurrences of * - table has CDC enabled and there are FileActions to write, AND * - table has column mapping enabled and there is a column mapping related metadata action * * This is because the semantics for this combination of features and file changes is undefined. */ private def performCdcColumnMappingCheck( actions: Seq[Action], op: DeltaOperations.Operation): Unit = { if (newMetadata.nonEmpty) { val _newMetadata = newMetadata.get val _currentMetadata = snapshot.metadata val cdcEnabled = CDCReader.isCDCEnabledOnTable(_newMetadata, spark) val columnMappingEnabled = _newMetadata.columnMappingMode != NoMapping val isColumnMappingUpgrade = DeltaColumnMapping.isColumnMappingUpgrade( oldMode = _currentMetadata.columnMappingMode, newMode = _newMetadata.columnMappingMode ) val isBothColumnMappingEnabled = _newMetadata.columnMappingMode != NoMapping && _currentMetadata.columnMappingMode != NoMapping def dropColumnOp: Boolean = DeltaColumnMapping.isDropColumnOperation( _newMetadata.schema, _currentMetadata.schema, isBothColumnMappingEnabled) def renameColumnOp: Boolean = DeltaColumnMapping.isRenameColumnOperation( _newMetadata.schema, _currentMetadata.schema, isBothColumnMappingEnabled) def columnMappingChange: Boolean = isColumnMappingUpgrade || dropColumnOp || renameColumnOp def existsFileActions: Boolean = actions.exists { _.isInstanceOf[FileAction] } if (cdcEnabled && columnMappingEnabled && columnMappingChange && existsFileActions) { throw DeltaErrors.blockColumnMappingAndCdcOperation(op) } } } /** * Validates that partition column changes are only performed by operations that explicitly * allow them. This check helps prevent accidental partition schema changes that could lead * to data inconsistencies. * * The validation can be configured via [[DeltaSQLConf.DELTA_PARTITION_COLUMN_CHANGE_CHECK]]. * * @param op The operation being performed * @param newMetadata The new metadata (if any) being set in this transaction */ private def validatePartitionColumnChanges( op: DeltaOperations.Operation, newMetadata: Option[Metadata]): Unit = { val checkMode = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_PARTITION_COLUMN_CHANGE_CHECK) if (checkMode != DeltaSQLConf.BooleanStringOrLogOnly.FALSE) { val isNewTable = snapshot.version == -1L // Validate that partition column changes are only performed by allowed operations. newMetadata.foreach { newMeta => val oldCols = snapshot.metadata.partitionColumns val newCols = newMeta.partitionColumns val partitionColsChanged = oldCols != newCols val illegalColChange = !isNewTable && partitionColsChanged && !op.canChangePartitionColumns if (illegalColChange) { recordDeltaEvent( deltaLog = deltaLog, opType = "delta.metadataCheck.illegalPartitionColumnChange", data = Map( "operation" -> op.name, "operationParameters" -> op.jsonEncodedValues, "oldPartitionColumns" -> oldCols, "newPartitionColumns" -> newCols ) ) if (checkMode == DeltaSQLConf.BooleanStringOrLogOnly.TRUE) { throw DeltaErrors.unsupportedPartitionColumnChange( operation = op.name, oldPartitionColumns = oldCols, newPartitionColumns = newCols ) } } } } } /** * Modifies the state of the log by adding a new commit that is based on a read at * [[readVersion]]. In the case of a conflict with a concurrent writer this * method will throw an exception. * * Also skips creating the commit if the configured [[IsolationLevel]] doesn't need us to record * the commit from correctness perspective. * * Returns the new version the transaction committed or None if the commit was skipped. */ def commitIfNeeded( actions: Seq[Action], op: DeltaOperations.Operation, tags: Map[String, String] = Map.empty): Option[Long] = { commitImpl(actions, op, canSkipEmptyCommits = true, tags = tags) } /** * Modifies the state of the log by adding a new commit that is based on a read at * [[readVersion]]. In the case of a conflict with a concurrent writer this * method will throw an exception. * * @param actions Set of actions to commit * @param op Details of operation that is performing this transactional commit */ def commit( actions: Seq[Action], op: DeltaOperations.Operation): Long = { commitImpl(actions, op, canSkipEmptyCommits = false, tags = Map.empty).getOrElse { throw new SparkException(s"Unknown error while trying to commit for operation $op") } } /** * Modifies the state of the log by adding a new commit that is based on a read at * [[readVersion]]. In the case of a conflict with a concurrent writer this * method will throw an exception. * * @param actions Set of actions to commit * @param op Details of operation that is performing this transactional commit * @param tags Extra tags to set to the CommitInfo action */ def commit( actions: Seq[Action], op: DeltaOperations.Operation, tags: Map[String, String]): Long = { commitImpl(actions, op, canSkipEmptyCommits = false, tags = tags).getOrElse { throw new SparkException(s"Unknown error while trying to commit for operation $op") } } /** * This method goes through all no-redirect-rules inside redirect feature to determine * whether the current operation is valid to run on this table. */ private def performNoRedirectRulesCheck( op: DeltaOperations.Operation, redirectConfig: TableRedirectConfiguration ): Unit = { // If this transaction commits to the redirect destination location, then there is no // need to validate the subsequent no-redirect rules. val configuration = deltaLog.newDeltaHadoopConf() val dataPath = snapshot.deltaLog.dataPath.toUri.getPath val catalog = spark.sessionState.catalog val isRedirectDest = redirectConfig.spec.isRedirectDest(catalog, configuration, dataPath) if (isRedirectDest) return // Find all rules that match with the current application name. // If appName is not present, its no-redirect-rule are included. // If appName is present, includes its no-redirect-rule only when appName // matches with "spark.app.name". val rulesOfMatchedApps = redirectConfig.noRedirectRules.filter { rule => rule.appName.forall(_.equalsIgnoreCase(spark.conf.get("spark.app.name"))) } // Determine whether any rule is satisfied the given operation. val noRuleSatisfied = !rulesOfMatchedApps.exists(_.allowedOperations.contains(op.name)) // If there is no rule satisfied, block the given operation. if (noRuleSatisfied) { throw DeltaErrors.noRedirectRulesViolated(op, redirectConfig.noRedirectRules) } } /** * This method determines whether `op` is valid when the table redirect feature is * set on current table. * 1. If redirect table feature is in progress state, no DML/DDL is allowed to execute. * 2. If user tries to access redirect source table, only the allowed operations listed * inside no-redirect-rules are valid. */ protected def performRedirectCheck(op: DeltaOperations.Operation): Unit = { // If redirect conflict check is not enable, skips all remaining validations. if (spark.conf.get(DeltaSQLConf.SKIP_REDIRECT_FEATURE)) return // If redirect feature is not set, then skips validation. if (!RedirectFeature.isFeatureSupported(snapshot)) return // If this transaction tried to unset redirect feature, then skips validation. if (RedirectFeature.isUpdateProperty(snapshot, op)) return // If this transaction tried to drop redirect feature, then skips validation. if (RedirectFeature.isDropFeature(op)) return // Get the redirect configuration from current snapshot. val redirectConfigOpt = RedirectFeature.getRedirectConfiguration(snapshot) redirectConfigOpt.foreach { redirectConfig => // If the redirect state is in EnableRedirectInProgress or DropRedirectInProgress, // all DML and DDL operation should be aborted. if (redirectConfig.isInProgressState) { throw DeltaErrors.invalidCommitIntermediateRedirectState(redirectConfig.redirectState) } // Validates the no redirect rules on the transactions that access redirect source table. performNoRedirectRulesCheck(op, redirectConfig) } } /** * Records a delta event for a commit conflict exception, including the operation type * of the winning/conflicting transaction for observability purposes. */ protected def recordConflictEvent(e: DeltaConcurrentModificationException): Unit = { // Extract the operation of the winning/conflicting transaction from the exception message. // This is for visibility/observability purpose to track which type of transaction // (e.g., OPTIMIZE/VACUUM) is causing the conflict. // Handle two message formats: // 1. New structured errors: "A concurrent added/modified/deleted data..." // 2. Old JSON format in conflicting commit: "operation":"" val newFormatPattern = """[Aa] concurrent (.+?) (?:added|modified|deleted)""".r val oldFormatPattern = """"operation"\s*:\s*"([^"]+)"""".r val winningTxnOperation = newFormatPattern .findFirstMatchIn(e.getMessage).map(_.group(1)) .orElse(oldFormatPattern.findFirstMatchIn(e.getMessage).map(_.group(1))) .getOrElse("Unknown Operation") recordDeltaEvent( deltaLog, opType = "delta.commit.conflict." + e.conflictType, data = Map("winningTxnOperation" -> winningTxnOperation)) } @throws(classOf[ConcurrentModificationException]) protected def commitImpl( actions: Seq[Action], op: DeltaOperations.Operation, canSkipEmptyCommits: Boolean, tags: Map[String, String]): Option[Long] = recordDeltaOperation(deltaLog, "delta.commit") { commitStartNano = System.nanoTime() val version = try { // Check for satisfaction of no redirect rules performRedirectCheck(op) // Check for CDC metadata columns performCdcMetadataCheck() // Check for internal SetTransaction conflicts and dedup. val finalActions = checkForSetTransactionConflictAndDedup(actions ++ this.actions.toSeq) val identityOnlyMetadataUpdate = isIdentityOnlyMetadataUpdate() // Update schema for IDENTITY column writes if necessary. This has to be called before // `prepareCommit` because it might change metadata and `prepareCommit` is responsible for // converting updated metadata into a `Metadata` action. precommitUpdateSchemaWithIdentityHighWaterMarks() // Try to commit at the next version. var preparedActions = executionObserver.preparingCommit { prepareCommit(finalActions, op) } validateActionsAddFileInvariants(preparedActions, metadata) // Find the isolation level to use for this commit val isolationLevelToUse = getIsolationLevelToUse(preparedActions, op) // Check for duplicated [[MetadataAction]] with the same domain names and validate the table // feature is enabled if [[MetadataAction]] is submitted. val domainMetadata = DomainMetadataUtils.validateDomainMetadataSupportedAndNoDuplicate(finalActions, protocol) isBlindAppend = { val dependsOnFiles = !readPredicates.isEmpty || readFiles.nonEmpty val onlyAddFiles = preparedActions.collect { case f: FileAction => f }.forall(_.isInstanceOf[AddFile]) onlyAddFiles && !dependsOnFiles } val readRowIdHighWatermark = RowId.extractHighWatermark(snapshot).getOrElse(RowId.MISSING_HIGH_WATER_MARK) val autoTags = mutable.HashMap.empty[String, String] if (identityOnlyMetadataUpdate) { autoTags += (DeltaSourceUtils.IDENTITY_COMMITINFO_TAG -> "true") } val allTags = tags ++ autoTags commitAttemptStartTimeMillis = clock.getTimeMillis() commitInfo = CommitInfo( time = commitAttemptStartTimeMillis, operation = op.name, inCommitTimestamp = generateInCommitTimestampForFirstCommitAttempt(commitAttemptStartTimeMillis), operationParameters = op.jsonEncodedValues, commandContext = Map.empty, readVersion = Some(readVersion).filter(_ >= 0), isolationLevel = Option(isolationLevelToUse.toString), isBlindAppend = Some(isBlindAppend), operationMetrics = getOperationMetrics(op), userMetadata = getUserMetadata(op), tags = if (allTags.nonEmpty) Some(allTags) else None, txnId = Some(txnId)) val firstAttemptVersion = getFirstAttemptVersion val metadataUpdatedWithCoordinatedCommitsInfo = updateMetadataWithCoordinatedCommitsConfs() val metadataUpdatedWithIctInfo = updateMetadataWithInCommitTimestamp(commitInfo) if (metadataUpdatedWithIctInfo || metadataUpdatedWithCoordinatedCommitsInfo) { preparedActions = preparedActions.map { case _: Metadata => metadata case other => other } } val currentTransactionInfo = CurrentTransactionInfo( txnId = txnId, readPredicates = readPredicates.asScala.toVector, readFiles = readFiles.toSet, readWholeTable = readTheWholeTable, readAppIds = readTxn.toSet, metadata = metadata, protocol = protocol, actions = preparedActions, readSnapshot = snapshot, commitInfo = Some(commitInfo), readRowIdHighWatermark = readRowIdHighWatermark, catalogTable = catalogTable, domainMetadata = domainMetadata, op = op) // Register post-commit hooks if any lazy val hasFileActions = preparedActions.exists { case _: FileAction => true case _ => false } if (DeltaConfigs.SYMLINK_FORMAT_MANIFEST_ENABLED.fromMetaData(metadata) && hasFileActions) { registerPostCommitHook(GenerateSymlinkManifest) } if (preparedActions.isEmpty && canSkipEmptyCommits && skipRecordingEmptyCommitAllowed(isolationLevelToUse)) { return None } // Try to commit at the next version. executionObserver.beginDoCommit() val (commitVersion, postCommitSnapshot, updatedCurrentTransactionInfo) = doCommitRetryIteratively(firstAttemptVersion, currentTransactionInfo, isolationLevelToUse) setCommitted(commitVersion, postCommitSnapshot, updatedCurrentTransactionInfo.actions) logInfo(log"Committed delta #${MDC(DeltaLogKeys.VERSION, commitVersion)} to " + log"${MDC(DeltaLogKeys.PATH, deltaLog.logPath)}") commitVersion } catch { case e: DeltaConcurrentModificationException => recordConflictEvent(e) executionObserver.transactionAborted() throw e case NonFatal(e) => recordDeltaEvent( deltaLog, "delta.commit.failure", data = Map("exception" -> Utils.exceptionString(e))) executionObserver.transactionAborted() throw e } runPostCommitHooks(committed.get) executionObserver.transactionCommitted() Some(version) } /** * This method makes the necessary changes to Metadata based on ICT: If ICT is getting enabled as * part of this commit, then it updates the Metadata with the ICT enablement information. * * @param commitInfo commitInfo for the commit * @return true if changes were made to Metadata else false. */ protected def updateMetadataWithInCommitTimestamp(commitInfo: CommitInfo): Boolean = { val firstAttemptVersion = getFirstAttemptVersion val metadataWithIctInfo = commitInfo.inCommitTimestamp .flatMap { inCommitTimestamp => InCommitTimestampUtils.getUpdatedMetadataWithICTEnablementInfo( spark, inCommitTimestamp, snapshot, metadata, firstAttemptVersion) }.getOrElse { return false } newMetadata = Some(metadataWithIctInfo) true } /** * This method makes the necessary changes to Metadata based on coordinated-commits: If the table * is being converted from file-system to coordinated commits, then it registers the table with * the commit-coordinator and updates the Metadata with the necessary configuration information * from the commit-coordinator. * * @return A boolean which represents whether we have updated the table Metadata with * coordinated-commits information. If no changed were made, returns false. */ protected def updateMetadataWithCoordinatedCommitsConfs(): Boolean = { validateCoordinatedCommitsConfInMetadata(newMetadata) val newCoordinatedCommitsTableConfOpt = registerTableForCoordinatedCommitsIfNeeded(metadata, protocol) val newCoordinatedCommitsTableConf = newCoordinatedCommitsTableConfOpt.getOrElse { return false } // FS to CC conversion val finalMetadata = metadata val coordinatedCommitsTableConfJson = JsonUtils.toJson(newCoordinatedCommitsTableConf) val extraKVConf = DeltaConfigs.COORDINATED_COMMITS_TABLE_CONF.key -> coordinatedCommitsTableConfJson newMetadata = Some(finalMetadata.copy( configuration = finalMetadata.configuration + extraKVConf)) true } protected def validateCoordinatedCommitsConfInMetadata(newMetadataOpt: Option[Metadata]): Unit = { // Validate that the [[DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF]] is json parse-able. // Also do this validation if this table property has changed. newMetadataOpt .filter { newMetadata => val newCoordinatedCommitsConf = newMetadata.configuration.get(DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF.key) val oldCoordinatedCommitsConf = snapshot.metadata.configuration.get(DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF.key) newCoordinatedCommitsConf != oldCoordinatedCommitsConf }.foreach(DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF.fromMetaData) } /** Whether to skip recording the commit in DeltaLog */ protected def skipRecordingEmptyCommitAllowed(isolationLevelToUse: IsolationLevel): Boolean = { if (!spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_SKIP_RECORDING_EMPTY_COMMITS)) { return false } // Recording of empty commits in deltalog can be skipped only for SnapshotIsolation and // Serializable mode. Seq(SnapshotIsolation, Serializable).contains(isolationLevelToUse) } /** * Create a large commit on the Delta log by directly writing an iterator of FileActions to the * LogStore. This function only commits the next possible version and will not check whether the * commit is retry-able. If the next version has already been committed, then this function * will fail. * This bypasses all optimistic concurrency checks. We assume that transaction conflicts should be * rare because this method is typically used to create new tables (e.g. CONVERT TO DELTA) or * apply some commands which rarely receive other transactions (e.g. CLONE/RESTORE). * In addition, the expectation is that the list of actions performed by the transaction * remains an iterator and is never materialized, given the nature of a large commit potentially * touching many files. * The `nonProtocolMetadataActions` parameter should only contain non-{protocol, metadata} * actions only. If the protocol of table needs to be updated, it should be passed in the * `newProtocolOpt` parameter. */ def commitLarge( spark: SparkSession, nonProtocolMetadataActions: Iterator[Action], newProtocolOpt: Option[Protocol], op: DeltaOperations.Operation, context: Map[String, String], metrics: Map[String, String] ): (Long, Snapshot) = recordDeltaOperation(deltaLog, "delta.commit.large") { assert(committed.isEmpty, "Transaction already committed.") commitStartNano = System.nanoTime() val attemptVersion = getFirstAttemptVersion executionObserver.preparingCommit() // From this point onwards, newProtocolOpt should not be used. // `newProtocol` or `protocol` should be used instead. // The updateMetadataAndProtocolWithRequiredFeatures method will // directly update the global `newProtocol` if needed. newProtocol = newProtocolOpt // If a feature requires another feature to be enabled, we enable the required // feature in the metadata (if needed) and add it to the protocol. // e.g. Coordinated Commits requires ICT and VacuumProtocolCheck to be enabled. updateMetadataAndProtocolWithRequiredFeatures(newMetadata, newProtocol.toSeq) def recordCommitLargeFailure(ex: Throwable, op: DeltaOperations.Operation): Unit = { val coordinatedCommitsExceptionOpt = ex match { case e: CommitFailedException => Some(e) case _ => None } val data = Map( "exception" -> Utils.exceptionString(ex), "operation" -> op.name, "fromCoordinatedCommits" -> coordinatedCommitsExceptionOpt.isDefined, "fromCoordinatedCommitsConflict" -> coordinatedCommitsExceptionOpt.map(_.getConflict).getOrElse(""), "fromCoordinatedCommitsRetryable" -> coordinatedCommitsExceptionOpt.map(_.getRetryable).getOrElse("")) recordDeltaEvent(deltaLog, "delta.commitLarge.failure", data = data) } try { val tags = Map.empty[String, String] val commitTimestampMs = clock.getTimeMillis() val commitInfo = CommitInfo( commitTimestampMs, operation = op.name, generateInCommitTimestampForFirstCommitAttempt(commitTimestampMs), operationParameters = op.jsonEncodedValues, context, readVersion = Some(readVersion), isolationLevel = Some(Serializable.toString), isBlindAppend = Some(false), Some(metrics), userMetadata = getUserMetadata(op), tags = if (tags.nonEmpty) Some(tags) else None, txnId = Some(txnId)) val assertDeletionVectorWellFormed = getAssertDeletionVectorWellFormedFunc(spark, op) updateMetadataWithCoordinatedCommitsConfs() updateMetadataWithInCommitTimestamp(commitInfo) // Precompute NOT NULL partition columns for validation during action processing val notNullPartitionCols = getNotNullPartitionCols(metadata) var allActions = Iterator(commitInfo, metadata) ++ nonProtocolMetadataActions ++ newProtocol.toIterator allActions = allActions.map { action => action match { case dm: DomainMetadata if isClusteringChangedOnUCManagedTable(dm) => // Temporary: block clustering changes on UC-managed tables (commitLarge() path). // commitLarge() bypasses prepareCommit(), so this guard is needed separately. // The check is intentionally inside the lazy map: commitLarge streams actions to // avoid materialising large sets, so an eager pre-scan is not practical. The // exception is thrown before any data is written to the commit coordinator because // the iterator is consumed first during serialisation. throw DeltaErrors.operationNotSupportedException( "Clustering column changes on Unity Catalog managed tables") case a: AddFile => assertDeletionVectorWellFormed(a) validateAddFileInvariants(a, notNullPartitionCols) case p: Protocol => recordProtocolChanges( "delta.protocol.change", fromProtocol = snapshot.protocol, toProtocol = p, isCreatingNewTable) DeltaTableV2.withEnrichedUnsupportedTableException(catalogTable) { deltaLog.protocolWrite(p) } case _ => } action } val (allActions2, acStatsCollector) = collectAutoOptimizeStats(allActions) allActions = allActions2 // Validate protocol support, specifically writer features. DeltaTableV2.withEnrichedUnsupportedTableException(catalogTable) { deltaLog.protocolWrite(snapshot.protocol) } allActions = RowId.assignFreshRowIds(spark, protocol, snapshot, allActions, op) allActions = DefaultRowCommitVersion.assignIfMissing( spark, protocol, snapshot, allActions, getFirstAttemptVersion) val commitStatsComputer = new CommitStatsComputer() allActions = commitStatsComputer.addToCommitStats(allActions) executionObserver.beginDoCommit() if (readVersion < 0) { deltaLog.createLogDirectoriesIfNotExists() } val fsWriteStartNano = System.nanoTime() val jsonActions = allActions.map(_.json) var commitSizeBytes = 0L jsonActions.map { action => commitSizeBytes += action.size } val effectiveTableCommitCoordinatorClient = readSnapshotTableCommitCoordinatorClientOpt.getOrElse { TableCommitCoordinatorClient( commitCoordinatorClient = new FileSystemBasedCommitCoordinatorClient(deltaLog), deltaLog = deltaLog, coordinatedCommitsTableConf = snapshot.metadata.coordinatedCommitsTableConf) } val updatedActions = new UpdatedActions( commitInfo, metadata, protocol, snapshot.metadata, snapshot.protocol) val commitResponse = TransactionExecutionObserver.withObserver(executionObserver) { effectiveTableCommitCoordinatorClient.commit( attemptVersion, jsonActions, updatedActions, catalogTable.map(_.identifier)) } // TODO(coordinated-commits): Use the right timestamp method on top of CommitInfo once ICT is // merged. partitionsAddedToOpt = Some(commitStatsComputer.getPartitionsAddedByTransaction) // If the metadata didn't change, `newMetadata` is empty, and we can re-use the old id. acStatsCollector.finalizeStats(newMetadata.map(_.id).getOrElse(snapshot.metadata.id)) spark.sessionState.conf.setConf( DeltaSQLConf.DELTA_LAST_COMMIT_VERSION_IN_SESSION, Some(attemptVersion)) commitEndNano = System.nanoTime() executionObserver.beginPostCommit() // NOTE: commitLarge cannot run postCommitHooks (such as the CheckpointHook). // Instead, manually run any necessary actions in updateAndCheckpoint. val postCommitSnapshot = updateAndCheckpoint( spark, deltaLog, commitStatsComputer.getNumActions, attemptVersion, commitResponse.getCommit, txnId) setCommitted(attemptVersion, postCommitSnapshot, committedActions = Seq.empty) val postCommitReconstructionTime = System.nanoTime() commitStatsComputer.finalizeAndEmitCommitStats( spark, attemptVersion, snapshot.version, commitDurationMs = NANOSECONDS.toMillis(commitEndNano - commitStartNano), fsWriteDurationMs = NANOSECONDS.toMillis(commitEndNano - fsWriteStartNano), txnExecutionTimeMs = NANOSECONDS.toMillis(commitEndNano - txnStartTimeNs), stateReconstructionDurationMs = NANOSECONDS.toMillis(postCommitReconstructionTime - commitEndNano), postCommitSnapshot, // We manually triggered a checkpoint in `updateAndCheckpoint` above. computedNeedsCheckpoint = true, isolationLevel = Serializable, commitInfoOpt = Some(commitInfo), commitSizeBytes = commitSizeBytes ) executionObserver.transactionCommitted() (attemptVersion, postCommitSnapshot) } catch { case e: Throwable => e match { case CommitConflictFailure(e) => recordCommitLargeFailure(e, op) // Actions of a commit which went in before ours. // Requires updating deltaLog to retrieve these actions, as another writer may have used // CommitCoordinatorClient for writing. val fileProvider = DeltaCommitFileProvider( deltaLog.update(catalogTableOpt = catalogTable)) val logs = deltaLog.store.readAsIterator( fileProvider.deltaFile(attemptVersion), deltaLog.newDeltaHadoopConf()) try { val winningCommitActions = logs.map(Action.fromJson) val commitInfo = winningCommitActions.collectFirst { case a: CommitInfo => a } .map(ci => ci.copy(version = Some(attemptVersion))) throw DeltaErrors.concurrentWriteException(commitInfo) } finally { logs.close() executionObserver.transactionAborted() } case NonFatal(_) => recordCommitLargeFailure(e, op) executionObserver.transactionAborted() throw e case _ => throw e } } } /** * Splits a transaction into smaller child transactions that operate on disjoint sets of the files * read by the parent transaction. This function is typically used when you want to break a large * operation into one that can be committed separately / incrementally. * * @param readFilesSubset The subset of files read by the current transaction that will be handled * by the new transaction. */ def split(readFilesSubset: Seq[AddFile]): OptimisticTransaction = { assert(newMetadata.isEmpty) assert(OptimisticTransaction.getActive().isEmpty, "Splitting a transaction is not supported when there is an active transaction.") val t = new OptimisticTransaction(deltaLog, catalogTable, snapshot) t.executionObserver = executionObserver.createChild() t.readPredicates.addAll(readPredicates) t.readFiles ++= readFilesSubset t.readTxn ++= readTxn t } /** * This method registers the table with the commit-coordinator via the [[CommitCoordinatorClient]] * if the table is transitioning from file-system based table to coordinated-commits table. * @param finalMetadata the effective [[Metadata]] of the table. Note that this refers to the * new metadata if this commit is updating the table Metadata. * @param finalProtocol the effective [[Protocol]] of the table. Note that this refers to the * new protocol if this commit is updating the table Protocol. * @return The new coordinated-commits table metadata if the table is transitioning from * file-system based table to coordinated-commits table. Otherwise, None. * This metadata should be added to the [[Metadata.configuration]] before doing the * commit. */ protected def registerTableForCoordinatedCommitsIfNeeded( finalMetadata: Metadata, finalProtocol: Protocol): Option[Map[String, String]] = { val (oldOwnerName, oldOwnerConf) = CoordinatedCommitsUtils.getCoordinatedCommitsConfs(snapshot.metadata) var newCoordinatedCommitsTableConf: Option[Map[String, String]] = None if (finalMetadata.configuration != snapshot.metadata.configuration || snapshot.version == -1L) { val newCommitCoordinatorClientOpt = CoordinatedCommitsUtils.getCommitCoordinatorClient( spark, deltaLog, finalMetadata, finalProtocol, failIfImplUnavailable = true) (newCommitCoordinatorClientOpt, readSnapshotTableCommitCoordinatorClientOpt) match { case (Some(newCommitCoordinatorClient), None) => // FS -> CC conversion val (commitCoordinatorName, commitCoordinatorConf) = CoordinatedCommitsUtils.getCoordinatedCommitsConfs(finalMetadata) logInfo(log"Table ${MDC(DeltaLogKeys.PATH, deltaLog.logPath)} transitioning from " + log"file-system based table to coordinated-commits table: " + log"[commit-coordinator: ${MDC(DeltaLogKeys.COORDINATOR_NAME, commitCoordinatorName)}" + log", conf: ${MDC(DeltaLogKeys.COORDINATOR_CONF, commitCoordinatorConf)}]") val tableIdentifierOpt = CoordinatedCommitsUtils.toCCTableIdentifier(catalogTable.map(_.identifier)) newCoordinatedCommitsTableConf = Some(newCommitCoordinatorClient.registerTable( deltaLog.logPath, tableIdentifierOpt, readVersion, finalMetadata, protocol).asScala.toMap) case (None, Some(readCommitCoordinatorClient)) => // CC -> FS conversion val (newOwnerName, newOwnerConf) = CoordinatedCommitsUtils.getCoordinatedCommitsConfs(snapshot.metadata) logInfo(log"Table ${MDC(DeltaLogKeys.PATH, deltaLog.logPath)} transitioning from " + log"coordinated-commits table to file-system table: " + log"[commit-coordinator: ${MDC(DeltaLogKeys.COORDINATOR_NAME, newOwnerName)}, " + log"conf: ${MDC(DeltaLogKeys.COORDINATOR_CONF, newOwnerConf)}]") case (Some(newCommitCoordinatorClient), Some(readCommitCoordinatorClient)) if !readCommitCoordinatorClient.semanticsEquals(newCommitCoordinatorClient) => // CC1 -> CC2 conversion is not allowed. // In order to transfer the table from one commit-coordinator to another, transfer the // table from current commit-coordinator to filesystem first and then filesystem to the // commit-coordinator. val (newOwnerName, newOwnerConf) = CoordinatedCommitsUtils.getCoordinatedCommitsConfs(finalMetadata) val message = s"Transition of table ${deltaLog.logPath} from one commit-coordinator to" + s" another commit-coordinator is not allowed: [old commit-coordinator: $oldOwnerName," + s" new commit-coordinator: $newOwnerName, old commit-coordinator conf: $oldOwnerConf," + s" new commit-coordinator conf: $newOwnerConf]." throw new IllegalStateException(message) case _ => // no owner change () } } newCoordinatedCommitsTableConf } /** Update the table now that the commit has been made, and write a checkpoint. */ protected def updateAndCheckpoint( spark: SparkSession, deltaLog: DeltaLog, commitSize: Int, attemptVersion: Long, commit: Commit, txnId: String): Snapshot = { val currentSnapshot = deltaLog.updateAfterCommit( attemptVersion, commit, newChecksumOpt = None, preCommitLogSegment = preCommitLogSegment, catalogTable) if (currentSnapshot.version != attemptVersion) { throw DeltaErrors.invalidCommittedVersion(attemptVersion, currentSnapshot.version) } logInfo(log"Committed delta #${MDC(DeltaLogKeys.VERSION, attemptVersion)} to " + log"${MDC(DeltaLogKeys.PATH, deltaLog.logPath)}. Wrote " + log"${MDC(DeltaLogKeys.NUM_ACTIONS, commitSize.toLong)} actions.") deltaLog.checkpoint(currentSnapshot, catalogTable) currentSnapshot } /** * A metadata update can enable a feature that requires a protocol upgrade. * Furthermore, a feature can have dependencies on other features. This method * enables the dependent features in the metadata. * It then updates the protocol with the features enabled by the metadata. * The global `newMetadata` and `newProtocol` are updated with the new * metadata and protocol if needed. * @param metadataOpt The new metadata that is being set. * @param protocols The new protocols that are being set. */ protected def updateMetadataAndProtocolWithRequiredFeatures( metadataOpt: Option[Metadata], protocols: Seq[Protocol]): Unit = { metadataOpt.foreach { m => assertMetadata(m) val metadataWithRequiredFeatureEnablementFlags = getMetadataWithDependentFeaturesEnabled(m, protocols) setNewProtocolWithFeaturesEnabledByMetadata(metadataWithRequiredFeatureEnablementFlags) // Also update `newMetadata` so that the behaviour later is consistent irrespective of whether // metadata was set via `updateMetadata` or `actions`. newMetadata = Some(metadataWithRequiredFeatureEnablementFlags) } } /** * Prepare for a commit by doing all necessary pre-commit checks and modifications to the actions. * @return The finalized set of actions. */ protected def prepareCommit( actions: Seq[Action], op: DeltaOperations.Operation): Seq[Action] = { assert(committed.isEmpty, "Transaction already committed.") val (metadatasAndProtocols, otherActions) = actions .partition(a => a.isInstanceOf[Metadata] || a.isInstanceOf[Protocol]) // New metadata can come either from `newMetadata` or from the `actions` there. val metadataChanges = newMetadata.toSeq ++ metadatasAndProtocols.collect { case m: Metadata => m } if (metadataChanges.length > 1) { recordDeltaEvent(deltaLog, "delta.metadataCheck.multipleMetadataActions", data = Map( "metadataChanges" -> metadataChanges )) assert( metadataChanges.length <= 1, "Cannot change the metadata more than once in a transaction.") } // There be at most one metadata entry at this point. // Update the global `newMetadata` and `newProtocol` with any extra metadata and protocol // changes needed for pre-requisite features. val protocolActions = metadatasAndProtocols.collect { case p: Protocol => p } val protocolChangesBeforeUpdate = newProtocol.toSeq ++ protocolActions updateMetadataAndProtocolWithRequiredFeatures( metadataChanges.headOption, protocolChangesBeforeUpdate) // A protocol change can be *explicit*, i.e. specified as a Protocol action as part of the // commit actions, or *implicit*. Implicit protocol changes are mostly caused by setting // new table properties that enable features that require a protocol upgrade. These implicit // changes are usually captured in newProtocol. In case there is more than one protocol action, // it is likely that it is due to a mix of explicit and implicit changes. val protocolChanges = newProtocol.toSeq ++ protocolActions if (protocolChanges.length > 1) { recordDeltaEvent(deltaLog, "delta.protocolCheck.multipleProtocolActions", data = Map( "protocolChanges" -> protocolChanges )) assert(protocolChanges.length <= 1, "Cannot change the protocol more than once in a " + "transaction. More than one protocol change in a transaction is likely due to an " + "explicitly specified Protocol action and an implicit protocol upgrade triggered by " + "a table property.") } // Update newProtocol so that the behaviour later is consistent irrespective of whether // the protocol was set via update/verifyMetadata or actions. // NOTE: There is at most one protocol change at this point. protocolChanges.foreach { p => newProtocol = Some(p) recordProtocolChanges( "delta.protocol.change", snapshot.protocol, p, isCreatingNewTable, operationNameOpt = Some(op.name)) DeltaTableV2.withEnrichedUnsupportedTableException(catalogTable) { deltaLog.protocolWrite(p) } } // Now, we know that there is at most 1 Metadata change (stored in newMetadata) and at most 1 // Protocol change (stored in newProtocol) val (protocolUpdate1, metadataUpdate1) = UniversalFormat.enforceInvariantsAndDependencies( spark, catalogTable, // Note: if this txn has no protocol or metadata updates, then `prev` will equal `newest`. snapshot, newestProtocol = protocol, // Note: this will try to use `newProtocol` newestMetadata = metadata, // Note: this will try to use `newMetadata` Some(op), otherActions ) newProtocol = protocolUpdate1.orElse(newProtocol) newMetadata = metadataUpdate1.orElse(newMetadata) var finalActions = newMetadata.toSeq ++ newProtocol.toSeq ++ otherActions // Block future cases of CDF + Column Mapping changes + file changes // This check requires having called // DeltaColumnMapping.checkColumnIdAndPhysicalNameAssignments which is done in the // `assertMetadata` call above. performCdcColumnMappingCheck(finalActions, op) // Ensure Commit Directory exists when coordinated commits is enabled on an existing table. lazy val isFsToCcConversion = snapshot.metadata.coordinatedCommitsCoordinatorName.isEmpty && newMetadata.flatMap(_.coordinatedCommitsCoordinatorName).nonEmpty val shouldCreateLogDirs = snapshot.version == -1 || isFsToCcConversion if (shouldCreateLogDirs) { deltaLog.createLogDirectoriesIfNotExists() } if (snapshot.version == -1) { // If this is the first commit and no protocol is specified, initialize the protocol version. if (!finalActions.exists(_.isInstanceOf[Protocol])) { finalActions = protocol +: finalActions } // If this is the first commit and no metadata is specified, throw an exception if (!finalActions.exists(_.isInstanceOf[Metadata])) { recordDeltaEvent( deltaLog, opType = "delta.metadataCheck.noMetadataInInitialCommit", data = Map("stacktrace" -> Thread.currentThread.getStackTrace.toSeq.take(20).mkString("\n\t")) ) if (spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_COMMIT_VALIDATION_ENABLED)) { throw DeltaErrors.metadataAbsentException() } logWarning( log"Detected no metadata in initial commit but commit validation was turned off.") } } // Validate that partition column changes are only performed by allowed operations. validatePartitionColumnChanges(op, newMetadata) val partitionColumns = metadata.physicalPartitionSchema.fieldNames.toSet finalActions = finalActions.map { case newVersion: Protocol => require(newVersion.minReaderVersion > 0, "The reader version needs to be greater than 0") require(newVersion.minWriterVersion > 0, "The writer version needs to be greater than 0") if (!canAssignAnyNewProtocol) { val currentVersion = snapshot.protocol if (!currentVersion.canTransitionTo(newVersion, op)) { throw new ProtocolDowngradeException(currentVersion, newVersion) } } newVersion case a: AddFile if partitionColumns != a.partitionValues.keySet => // If the partitioning in metadata does not match the partitioning in the AddFile recordDeltaEvent(deltaLog, "delta.metadataCheck.partitionMismatch", data = Map( "tablePartitionColumns" -> metadata.partitionColumns, "filePartitionValues" -> a.partitionValues )) if (spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_COMMIT_VALIDATION_ENABLED)) { throw DeltaErrors.addFilePartitioningMismatchException( a.partitionValues.keySet.toSeq, partitionColumns.toSeq) } logWarning( log""" |Detected mismatch in partition values between AddFile and table metadata but |commit validation was turned off. |To turn it back on set |${MDC(DeltaLogKeys.CONFIG_KEY, DeltaSQLConf.DELTA_COMMIT_VALIDATION_ENABLED.key)} |to "true" """.stripMargin) a case other => other } DeltaTableV2.withEnrichedUnsupportedTableException(catalogTable) { newProtocol.foreach(deltaLog.protocolWrite) deltaLog.protocolWrite(snapshot.protocol) } finalActions = RowId.assignFreshRowIds( spark, protocol, snapshot, finalActions.toIterator, op).toList finalActions = DefaultRowCommitVersion.assignIfMissing( spark, protocol, snapshot, finalActions.toIterator, getFirstAttemptVersion).toList // We make sure that this isn't an appendOnly table as we check if we need to delete // files. val removes = actions.collect { case r: RemoveFile => r } if (removes.exists(_.dataChange)) DeltaLog.assertRemovable(snapshot) val assertDeletionVectorWellFormed = getAssertDeletionVectorWellFormedFunc(spark, op) actions.foreach(assertDeletionVectorWellFormed) // Make sure shredded writes are only performed if the shredding table property was set assertShreddingStateConsistent() // Make sure this operation does not include default column values if the corresponding table // feature is not enabled. if (!protocol.isFeatureSupported(AllowColumnDefaultsTableFeature)) { checkNoColumnDefaults(op) } finalActions } // Returns the isolation level to use for committing the transaction protected def getIsolationLevelToUse( preparedActions: Seq[Action], op: DeltaOperations.Operation): IsolationLevel = { val isolationLevelToUse = if (canDowngradeToSnapshotIsolation(preparedActions, op.changesData)) { SnapshotIsolation } else { getDefaultIsolationLevel() } isolationLevelToUse } /** Log protocol change events. */ private def recordProtocolChanges( opType: String, fromProtocol: Protocol, toProtocol: Protocol, isCreatingNewTable: Boolean, operationNameOpt: Option[String] = None): Unit = { val payload: Map[String, Any] = if (isCreatingNewTable) { Map("toProtocol" -> toProtocol.fieldsForLogging, "operationName" -> "CREATE TABLE") } else { Map( "fromProtocol" -> fromProtocol.fieldsForLogging, "toProtocol" -> toProtocol.fieldsForLogging, "operationName" -> operationNameOpt.orNull) } recordDeltaEvent(deltaLog, opType, data = payload) } private[delta] def isCommitLockEnabled: Boolean = { spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_COMMIT_LOCK_ENABLED).getOrElse( deltaLog.store.isPartialWriteVisible(deltaLog.logPath, deltaLog.newDeltaHadoopConf())) } private def lockCommitIfEnabled[T](body: => T): T = { if (isCommitLockEnabled) { // We are borrowing the `snapshotLock` even for commits. Ideally we should be // using a separate lock for this purpose, because multiple threads fighting over // a commit shouldn't interfere with normal snapshot updates by readers. deltaLog.withSnapshotLockInterruptibly(body) } else { body } } /** * Commit the txn represented by `currentTransactionInfo` using `attemptVersion` version number. * If there are any conflicts that are found, we will retry a fixed number of times. * * @return the real version that was committed, the postCommitSnapshot, and the txn info * NOTE: The postCommitSnapshot may not be the same as the version committed if racing * commits were written while we updated the snapshot. */ protected def doCommitRetryIteratively( attemptVersion: Long, currentTransactionInfo: CurrentTransactionInfo, isolationLevel: IsolationLevel) : (Long, Snapshot, CurrentTransactionInfo) = recordDeltaOperation( deltaLog, "delta.commit.allAttempts") { lockCommitIfEnabled { var commitVersion = attemptVersion var updatedCurrentTransactionInfo = currentTransactionInfo val isFsToCcCommit = snapshot.metadata.coordinatedCommitsCoordinatorName.isEmpty && metadata.coordinatedCommitsCoordinatorName.nonEmpty val maxRetryAttempts = spark.conf.get(DeltaSQLConf.DELTA_MAX_RETRY_COMMIT_ATTEMPTS) val maxNonConflictRetryAttempts = spark.conf.get(DeltaSQLConf.DELTA_MAX_NON_CONFLICT_RETRY_COMMIT_ATTEMPTS) var nonConflictAttemptNumber = 0 var shouldCheckForConflicts = false for (attemptNumber <- 0 to maxRetryAttempts) { try { val postCommitSnapshot = if (!shouldCheckForConflicts) { doCommit(commitVersion, updatedCurrentTransactionInfo, attemptNumber, isolationLevel) } else recordDeltaOperation(deltaLog, "delta.commit.retry") { val (newCommitVersion, newCurrentTransactionInfo) = checkForConflicts( commitVersion, updatedCurrentTransactionInfo, attemptNumber, isolationLevel) commitVersion = newCommitVersion updatedCurrentTransactionInfo = newCurrentTransactionInfo doCommit(commitVersion, updatedCurrentTransactionInfo, attemptNumber, isolationLevel) } return (commitVersion, postCommitSnapshot, updatedCurrentTransactionInfo) } catch { case _: FileAlreadyExistsException if isFsToCcCommit => // Don't retry if this commit tries to upgrade the table from filesystem to managed // commits and the first attempt failed due to a conflict. throw DeltaErrors.concurrentWriteException(conflictingCommit = None) case _: FileAlreadyExistsException if readSnapshotTableCommitCoordinatorClientOpt.isEmpty => // For filesystem based tables, we use LogStore to do the commit. On a conflict, // LogStore returns FileAlreadyExistsException necessitating conflict resolution. // For commit-coordinators, FileAlreadyExistsException isn't expected under normal // operations and thus retries are not performed if this exception is thrown by // CommitCoordinatorClient. shouldCheckForConflicts = true // Do nothing, retry with next available attemptVersion case ex: CommitFailedException if ex.getRetryable && ex.getConflict => shouldCheckForConflicts = true // Reset nonConflictAttemptNumber if a conflict is detected. nonConflictAttemptNumber = 0 // For coordinated-commits, only retry with next available attemptVersion when // retryable is set and it was a case of conflict. case ex: CommitFailedException if ex.getRetryable && !ex.getConflict => if (nonConflictAttemptNumber < maxNonConflictRetryAttempts) { nonConflictAttemptNumber += 1 } else { // Rethrow the exception if max retries for non-conflict case have been reached throw ex } } } // retries all failed val totalCommitAttemptTime = clock.getTimeMillis() - commitAttemptStartTimeMillis throw DeltaErrors.maxCommitRetriesExceededException( maxRetryAttempts + 1, commitVersion, attemptVersion, updatedCurrentTransactionInfo.finalActionsToCommit.length, totalCommitAttemptTime) } } /** * Commit `actions` using `attemptVersion` version number. Throws a FileAlreadyExistsException * if any conflicts are detected. * * @return the post-commit snapshot of the deltaLog */ protected def doCommit( attemptVersion: Long, currentTransactionInfo: CurrentTransactionInfo, attemptNumber: Int, isolationLevel: IsolationLevel): Snapshot = { val actions = currentTransactionInfo.finalActionsToCommit logInfo( log"Attempting to commit version ${MDC(DeltaLogKeys.VERSION, attemptVersion)} with " + log"${MDC(DeltaLogKeys.NUM_ACTIONS, actions.size.toLong)} actions with " + log"${MDC(DeltaLogKeys.ISOLATION_LEVEL, isolationLevel)} isolation level") if (readVersion > -1 && metadata.id != snapshot.metadata.id) { val msg = s"Change in the table id detected in txn. Table id for txn on table at " + s"${deltaLog.dataPath} was ${snapshot.metadata.id} when the txn was created and " + s"is now changed to ${metadata.id}." logWarning(msg) recordDeltaEvent(deltaLog, "delta.metadataCheck.commit", data = Map( "readSnapshotVersion" -> snapshot.version, "readSnapshotMetadata" -> snapshot.metadata, "txnMetadata" -> metadata, "commitAttemptVersion" -> attemptVersion, "commitAttemptNumber" -> attemptNumber)) } val fsWriteStartNano = System.nanoTime() val jsonActions = actions.map(_.json) val (newChecksumOpt, commit) = writeCommitFile(attemptVersion, jsonActions.toIterator, currentTransactionInfo) spark.sessionState.conf.setConf( DeltaSQLConf.DELTA_LAST_COMMIT_VERSION_IN_SESSION, Some(attemptVersion)) commitEndNano = System.nanoTime() executionObserver.beginPostCommit() val postCommitSnapshot = deltaLog.updateAfterCommit( attemptVersion, commit, newChecksumOpt, preCommitLogSegment, catalogTable) val postCommitReconstructionTime = System.nanoTime() needsCheckpoint = isCheckpointNeeded(attemptVersion, postCommitSnapshot) val commitStatsComputer = new CommitStatsComputer() // Add to commit stats and consume the returned iterator. commitStatsComputer.addToCommitStats(actions.toIterator).foreach(_ => ()) partitionsAddedToOpt = Some(commitStatsComputer.getPartitionsAddedByTransaction) collectAutoOptimizeStatsAndFinalize(actions, deltaLog.unsafeVolatileTableId) val commitSizeBytes: Long = jsonActions.map(_.length.toLong).sum commitStatsComputer.finalizeAndEmitCommitStats( spark, attemptVersion, startVersion = snapshot.version, commitDurationMs = NANOSECONDS.toMillis(commitEndNano - commitStartNano), fsWriteDurationMs = NANOSECONDS.toMillis(commitEndNano - fsWriteStartNano), txnExecutionTimeMs = NANOSECONDS.toMillis(commitEndNano - txnStartNano), stateReconstructionDurationMs = NANOSECONDS.toMillis(postCommitReconstructionTime - commitEndNano), postCommitSnapshot = postCommitSnapshot, computedNeedsCheckpoint = needsCheckpoint, isolationLevel = isolationLevel, commitInfoOpt = currentTransactionInfo.commitInfo, commitSizeBytes = commitSizeBytes ) postCommitSnapshot } class FileSystemBasedCommitCoordinatorClient(val deltaLog: DeltaLog) extends CommitCoordinatorClient { override def commit( logStore: io.delta.storage.LogStore, hadoopConf: Configuration, tableDesc: TableDescriptor, commitVersion: Long, actions: java.util.Iterator[String], updatedActions: UpdatedActions): CommitResponse = { val logPath = tableDesc.getLogPath // Get thread local observer for Fuzz testing purpose. val executionObserver = TransactionExecutionObserver.getObserver val commitFile = util.FileNames.unsafeDeltaFile(logPath, commitVersion) val commitFileStatus = doCommit(logStore, hadoopConf, logPath, commitFile, commitVersion, actions) executionObserver.beginBackfill() val ictEnabled = updatedActions.getNewMetadata.getConfiguration.asScala.getOrElse( DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key, "false") == "true" val commitTimestamp = if (ictEnabled) { // CommitInfo.getCommitTimestamp will return the inCommitTimestamp. updatedActions.getCommitInfo.getCommitTimestamp } else { commitFileStatus.getModificationTime } new CommitResponse(new Commit( commitVersion, commitFileStatus, commitTimestamp )) } protected def doCommit( logStore: io.delta.storage.LogStore, hadoopConf: Configuration, logPath: Path, commitFile: Path, commitVersion: Long, actions: java.util.Iterator[String]): FileStatus = { logStore.write(commitFile, actions, false, hadoopConf) logPath.getFileSystem(hadoopConf).getFileStatus(commitFile) } override def getCommits( tableDesc: TableDescriptor, startVersion: java.lang.Long, endVersion: java.lang.Long): GetCommitsResponse = new GetCommitsResponse(Seq.empty.asJava, -1) override def backfillToVersion( logStore: io.delta.storage.LogStore, hadoopConf: Configuration, tableDesc: TableDescriptor, version: Long, lastKnownBackfilledVersion: java.lang.Long): Unit = {} /** * [[FileSystemBasedCommitCoordinatorClient]] is supposed to be treated as a singleton object * for a Delta Log and is equal to all other instances of * [[FileSystemBasedCommitCoordinatorClient]] for the same Delta Log. */ override def semanticEquals(other: CommitCoordinatorClient): Boolean = { other match { case fsCommitCoordinatorClient: FileSystemBasedCommitCoordinatorClient => fsCommitCoordinatorClient.deltaLog == deltaLog case _ => false } } override def registerTable( logPath: Path, tableIdentifier: Optional[TableIdentifier], currentVersion: Long, currentMetadata: AbstractMetadata, currentProtocol: AbstractProtocol): java.util.Map[String, String] = Map.empty[String, String].asJava } /** * Writes the json actions provided to the commit file corresponding to attemptVersion. * If coordinated-commits are enabled, this method must return a non-empty [[Commit]] * since we can't guess it from the FileSystem. */ protected def writeCommitFile( attemptVersion: Long, jsonActions: Iterator[String], currentTransactionInfo: CurrentTransactionInfo) : (Option[VersionChecksum], Commit) = { val commitCoordinatorClient = readSnapshotTableCommitCoordinatorClientOpt.getOrElse { TableCommitCoordinatorClient( new FileSystemBasedCommitCoordinatorClient(deltaLog), deltaLog, snapshot.metadata.coordinatedCommitsTableConf) } val commitFile = writeCommitFileImpl( attemptVersion, jsonActions, commitCoordinatorClient, currentTransactionInfo) val newChecksumOpt = incrementallyDeriveChecksum(attemptVersion, currentTransactionInfo) (newChecksumOpt, commitFile) } protected def writeCommitFileImpl( attemptVersion: Long, jsonActions: Iterator[String], tableCommitCoordinatorClient: TableCommitCoordinatorClient, currentTransactionInfo: CurrentTransactionInfo ): Commit = { val updatedActions = currentTransactionInfo.getUpdatedActions(snapshot.metadata, snapshot.protocol) val commitResponse = TransactionExecutionObserver.withObserver(executionObserver) { tableCommitCoordinatorClient.commit( attemptVersion, jsonActions, updatedActions, catalogTable.map(_.identifier)) } if (attemptVersion == 0L) { val expectedPathForCommitZero = unsafeDeltaFile(deltaLog.logPath, version = 0L).toUri val actualCommitPath = commitResponse.getCommit.getFileStatus.getPath.toUri if (actualCommitPath != expectedPathForCommitZero) { throw new IllegalStateException("Expected 0th commit to be written to " + s"$expectedPathForCommitZero but was written to $actualCommitPath") } } commitResponse.getCommit } /** * Given an attemptVersion, obtain checksum for previous snapshot version * (i.e., attemptVersion - 1) and incrementally derives a new checksum from * the actions of the current transaction. * * @param attemptVersion that the current transaction is committing * @param currentTransactionInfo containing actions of the current transaction * @return */ protected def incrementallyDeriveChecksum( attemptVersion: Long, currentTransactionInfo: CurrentTransactionInfo): Option[VersionChecksum] = { incrementallyDeriveChecksum( spark, deltaLog, attemptVersion, actions = currentTransactionInfo.finalActionsToCommit, metadataOpt = Some(currentTransactionInfo.metadata), protocolOpt = Some(currentTransactionInfo.protocol), operationName = currentTransactionInfo.op.name, txnIdOpt = Some(currentTransactionInfo.txnId), previousVersionState = scala.Left(snapshot), includeAddFilesInCrc = Snapshot.shouldIncludeAddFilesInCrc(spark, snapshot, metadata) ).toOption } /** * Looks at actions that have happened since the txn started and checks for logical * conflicts with the read/writes. Resolve conflicts and returns a tuple representing * the commit version to attempt next and the commit summary which we need to commit. */ protected def checkForConflicts( checkVersion: Long, currentTransactionInfo: CurrentTransactionInfo, attemptNumber: Int, commitIsolationLevel: IsolationLevel) : (Long, CurrentTransactionInfo) = recordDeltaOperation( deltaLog, "delta.commit.retry.conflictCheck", tags = Map(TAG_LOG_STORE_CLASS -> deltaLog.store.getClass.getName)) { DeltaTableV2.withEnrichedUnsupportedTableException(catalogTable) { val fileStatuses = getConflictingVersions(checkVersion) val nextAttemptVersion = checkVersion + fileStatuses.size // validate that information about conflicting winning commit files is continuous and in the // right order. val expected = (checkVersion until nextAttemptVersion) val found = fileStatuses.map(deltaVersion) val mismatch = expected.zip(found).dropWhile{ case (v1, v2) => v1 == v2 }.take(10) assert(mismatch.isEmpty, s"Expected ${mismatch.map(_._1).mkString(",")} but got ${mismatch.map(_._2).mkString(",")}") val logPrefix = log"[attempt ${MDC(DeltaLogKeys.NUM_ATTEMPT, attemptNumber)}] " val txnDetailsLog = { var adds = 0L var removes = 0L currentTransactionInfo.actions.foreach { case _: AddFile => adds += 1 case _: RemoveFile => removes += 1 case _ => } log"${MDC(DeltaLogKeys.NUM_ACTIONS, adds)} adds, " + log"${MDC(DeltaLogKeys.NUM_ACTIONS2, removes)} removes, " + log"${MDC(DeltaLogKeys.NUM_PREDICATES, readPredicates.size)} read predicates, " + log"${MDC(DeltaLogKeys.NUM_FILES, readFiles.size.toLong)} read files" } logInfo(logPrefix + log"Checking for conflicts with versions " + log"[${MDC(DeltaLogKeys.VERSION, checkVersion)}, " + log"${MDC(DeltaLogKeys.VERSION2, nextAttemptVersion)}) " + log"with current txn having " + txnDetailsLog) val updatedCurrentTransactionInfo = { if (expected.isEmpty) { currentTransactionInfo } else { resolveConflicts( currentTransactionInfo = currentTransactionInfo, firstWinningVersion = expected.head, lastWinningVersion = expected.last, conflictingCommitFiles = fileStatuses, commitIsolationLevel = commitIsolationLevel) } } logInfo(logPrefix + log"No conflicts with versions " + log"[${MDC(DeltaLogKeys.VERSION, checkVersion)}, " + log"${MDC(DeltaLogKeys.VERSION2, nextAttemptVersion)}) " + log"with current txn having " + txnDetailsLog + log"${MDC(DeltaLogKeys.TIME_MS, clock.getTimeMillis() - commitAttemptStartTimeMillis)} " + log"ms since start") (nextAttemptVersion, updatedCurrentTransactionInfo) } } /** * Loads the summaries of the conflicting commits and uses [[ConflictChecker]] to * resolve conflicts. * * @param currentTransactionInfo The current transaction information to check for conflicts * @param firstWinningVersion The first version number for conflict checking (inclusive) * @param lastWinningVersion The last version number for conflict checking (inclusive) * @param conflictingCommitFiles The sequence of file statuses representing conflicting commits * @param commitIsolationLevel The isolation level to use for conflict checking * @return Updated transaction information after resolving all conflicts */ protected def resolveConflicts( currentTransactionInfo: CurrentTransactionInfo, firstWinningVersion: Long, lastWinningVersion: Long, conflictingCommitFiles: Seq[FileStatus], commitIsolationLevel: IsolationLevel) : CurrentTransactionInfo = { var updatedCurrentTransactionInfo = currentTransactionInfo (firstWinningVersion to lastWinningVersion) .zip(conflictingCommitFiles) .foreach { case (otherCommitVersion, otherCommitFileStatus) => val winningCommitSummary = WinningCommitSummary.createFromFileStatus( deltaLog, otherCommitFileStatus) val conflictChecker = new ConflictChecker( spark, updatedCurrentTransactionInfo, winningCommitSummary, commitIsolationLevel) updatedCurrentTransactionInfo = conflictChecker.checkConflicts() logInfo(logPrefix + log"No conflicts in version ${MDC(DeltaLogKeys.VERSION, otherCommitVersion)}, " + log"${MDC(DeltaLogKeys.DURATION, clock.getTimeMillis() - commitAttemptStartTimeMillis)} ms since start") } updatedCurrentTransactionInfo } /** Returns the version that the first attempt will try to commit at. */ private[delta] def getFirstAttemptVersion: Long = readVersion + 1L /** Returns the conflicting commit information */ protected def getConflictingVersions(previousAttemptVersion: Long): Seq[FileStatus] = { assert(previousAttemptVersion == preCommitLogSegment.version + 1) val (newPreCommitLogSegment, newCommitFileStatuses) = deltaLog.getUpdatedLogSegment( preCommitLogSegment, readSnapshotTableCommitCoordinatorClientOpt, catalogTable) assert(preCommitLogSegment.version + newCommitFileStatuses.size == newPreCommitLogSegment.version) preCommitLogSegment = newPreCommitLogSegment newCommitFileStatuses } protected def setCommitted( committedVersion: Long, postCommitSnapshot: Snapshot, committedActions: Seq[Action]): Unit = committed = Some(CommittedTransaction( txnId = txnId, deltaLog = deltaLog, catalogTable = catalogTable, readSnapshot = snapshot, committedVersion = committedVersion, committedActions = committedActions, postCommitSnapshot = postCommitSnapshot, postCommitHooks = postCommitHooks.toSeq, txnExecutionTimeMs = txnExecutionTimeMs.get, needsCheckpoint = needsCheckpoint, partitionsAddedToOpt = partitionsAddedToOpt, isBlindAppend = isBlindAppend )) /** Register a hook that will be executed once a commit is successful. */ def registerPostCommitHook(hook: PostCommitHook): Unit = { if (!postCommitHooks.contains(hook)) { postCommitHooks.append(hook) } } def containsPostCommitHook(hook: PostCommitHook): Boolean = postCommitHooks.contains(hook) /** Executes the registered post commit hooks. */ protected def runPostCommitHooks(committedTransaction: CommittedTransaction): Unit = { assert(committed.isDefined, "Can't call post commit hooks before committing") val postCommitHooksToRun = committedTransaction.postCommitHooks // Keep track of the active txn because hooks may create more txns and overwrite the active one. val activeCommit = OptimisticTransaction.getActive() OptimisticTransaction.clearActive() try { postCommitHooksToRun.foreach(runPostCommitHook(_, committedTransaction)) } finally { activeCommit.foreach(OptimisticTransaction.setActive) } } private[delta] def unregisterPostCommitHooksWhere(predicate: PostCommitHook => Boolean): Unit = postCommitHooks --= postCommitHooks.filter(predicate) protected lazy val logPrefix: MessageWithContext = { def truncate(uuid: String): String = uuid.split("-").head log"[tableId=${MDC(DeltaLogKeys.METADATA_ID, truncate(snapshot.metadata.id))}," + log"txnId=${MDC(DeltaLogKeys.TXN_ID, truncate(txnId))}] " } def logInfo(msg: MessageWithContext): Unit = { super.logInfo(logPrefix + msg) } def logWarning(msg: MessageWithContext): Unit = { super.logWarning(logPrefix + msg) } def logWarning(msg: MessageWithContext, throwable: Throwable): Unit = { super.logWarning(logPrefix + msg, throwable) } def logError(msg: MessageWithContext): Unit = { super.logError(logPrefix + msg) } def logError(msg: MessageWithContext, throwable: Throwable): Unit = { super.logError(logPrefix + msg, throwable) } /** * If the operation assigns or modifies column default values, this method checks that the * corresponding table feature is enabled and throws an error if not. */ protected def checkNoColumnDefaults(op: DeltaOperations.Operation): Unit = { def usesDefaults(column: StructField): Boolean = { column.metadata.contains(ResolveDefaultColumns.CURRENT_DEFAULT_COLUMN_METADATA_KEY) || column.metadata.contains(ResolveDefaultColumns.EXISTS_DEFAULT_COLUMN_METADATA_KEY) } def throwError(errorClass: String, parameters: Array[String]): Unit = { throw new DeltaAnalysisException( errorClass = errorClass, messageParameters = parameters) } op match { case change: ChangeColumn if usesDefaults(change.newColumn) => throwError("WRONG_COLUMN_DEFAULTS_FOR_DELTA_FEATURE_NOT_ENABLED", Array("ALTER TABLE")) case changes: ChangeColumns if changes.columns.exists(c => usesDefaults(c.newColumn)) => throwError("WRONG_COLUMN_DEFAULTS_FOR_DELTA_FEATURE_NOT_ENABLED", Array("ALTER TABLE")) case create: CreateTable if create.metadata.schema.fields.exists(usesDefaults) => throwError("WRONG_COLUMN_DEFAULTS_FOR_DELTA_FEATURE_NOT_ENABLED", Array("CREATE TABLE")) case replace: ReplaceColumns if replace.columns.exists(usesDefaults) => throwError("WRONG_COLUMN_DEFAULTS_FOR_DELTA_FEATURE_NOT_ENABLED", Array("CREATE TABLE")) case replace: ReplaceTable if replace.metadata.schema.fields.exists(usesDefaults) => throwError("WRONG_COLUMN_DEFAULTS_FOR_DELTA_FEATURE_NOT_ENABLED", Array("CREATE TABLE")) case update: UpdateSchema if update.newSchema.fields.exists(usesDefaults) => throwError("WRONG_COLUMN_DEFAULTS_FOR_DELTA_FEATURE_NOT_ENABLED", Array("ALTER TABLE")) case _ => } } // Backfill any unbackfilled commits if coordinated commits are disabled -- in the Optimistic // Transaction constructor. CoordinatedCommitsUtils.backfillWhenCoordinatedCommitsDisabled(snapshot) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/PostHocResolveUpCast.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.rules.{Rule, RuleExecutor} import org.apache.spark.sql.internal.SQLConf /** * Post-hoc resolution rules [[PreprocessTableMerge]] and [[PreprocessTableUpdate]] may introduce * new unresolved UpCast expressions that won't be resolved by [[ResolveUpCast]] that ran in the * previous resolution phase. This rule ensures these UpCast expressions get resolved in the * Post-hoc resolution phase. * * Note: we can't inject [[ResolveUpCast]] directly because we need an initialized analyzer instance * for that which is not available at the time Delta rules are injected. [[PostHocResolveUpCast]] is * delaying the access to the analyzer until after it's initialized. */ case class PostHocResolveUpCast(spark: SparkSession) extends Rule[LogicalPlan] { override def apply(plan: LogicalPlan): LogicalPlan = if (!plan.resolved) PostHocUpCastResolver.execute(plan) else plan /** * A rule executor that runs [[ResolveUpCast]] until all UpCast expressions have been resolved. */ object PostHocUpCastResolver extends RuleExecutor[LogicalPlan] { final override protected def batches: Seq[Batch] = Seq( Batch( "Post-hoc UpCast Resolution", FixedPoint( conf.analyzerMaxIterations, errorOnExceed = true, maxIterationsSetting = SQLConf.ANALYZER_MAX_ITERATIONS.key), spark.sessionState.analyzer.ResolveUpCast) ) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/PreDowngradeTableFeatureCommand.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.concurrent.TimeUnit import scala.util.control.NonFatal import org.apache.spark.sql.delta.actions.{DeletionVectorDescriptor, RemoveFile} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.{AlterTableSetPropertiesDeltaCommand, AlterTableUnsetPropertiesDeltaCommand, DeltaReorgTableCommand, DeltaReorgTableMode, DeltaReorgTableSpec} import org.apache.spark.sql.delta.commands.backfill.RowTrackingUnBackfillCommand import org.apache.spark.sql.delta.commands.columnmapping.RemoveColumnMappingCommand import org.apache.spark.sql.delta.commands.optimize.OptimizeMetrics import org.apache.spark.sql.delta.constraints.Constraints import org.apache.spark.sql.delta.coordinatedcommits.CoordinatedCommitsUtils import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.hadoop.fs.Path import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.analysis.ResolvedTable import org.apache.spark.sql.functions.{approx_count_distinct, col, not} /** * Used as the return type of `removeFeatureTracesIfNeeded`. The contents are the following: * * 1) performedChanges. True when the preDowngrade command performed a cleaning action. * False otherwise. * 2) lastCommitVersionOpt. Optionally, it returns the version of the last commit. This is used as * a starting version for the protocol downgrade commit. Defining the last commit allows * to conflict resolve all the commits that occurred between the last pre-downgrade commit * and the protocol downgrade commit. */ sealed case class PreDowngradeStatus( performedChanges: Boolean, lastCommitVersionOpt: Option[Long] = None) object PreDowngradeStatus { val DID_NOT_PERFORM_CHANGES = PreDowngradeStatus(performedChanges = false) val PERFORMED_CHANGES = PreDowngradeStatus(performedChanges = true) } /** * A base class for implementing a preparation command for removing table features. * Must implement a run method. Note, the run method must be implemented in a way that when * it finishes, the table does not use the feature that is being removed, and nobody is * allowed to start using it again implicitly. One way to achieve this is by * disabling the feature on the table before proceeding to the actual removal. * See [[RemovableFeature.preDowngradeCommand]]. */ sealed abstract class PreDowngradeTableFeatureCommand { def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus } case class TestWriterFeaturePreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand with DeltaLogging { // To remove the feature we only need to remove the table property. override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = { // Make sure feature data/metadata exist before proceeding. if (TestRemovableWriterFeature.validateDropInvariants(table, table.initialSnapshot)) { return PreDowngradeStatus.DID_NOT_PERFORM_CHANGES } if (DeltaUtils.isTesting) { recordDeltaEvent(table.deltaLog, "delta.test.TestWriterFeaturePreDowngradeCommand") } val properties = Seq(TestRemovableWriterFeature.TABLE_PROP_KEY) AlterTableUnsetPropertiesDeltaCommand( table, properties, ifExists = true, fromDropFeatureCommand = true).run(spark) PreDowngradeStatus.PERFORMED_CHANGES } } case class TestUnsupportedReaderWriterFeaturePreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand { override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = PreDowngradeStatus.PERFORMED_CHANGES } case class TestUnsupportedWriterFeaturePreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand { override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = PreDowngradeStatus.PERFORMED_CHANGES } case class TestWriterWithHistoryValidationFeaturePreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand with DeltaLogging { // To remove the feature we only need to remove the table property. override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = { // Make sure feature data/metadata exist before proceeding. val snapshot = table.initialSnapshot if (TestRemovableWriterWithHistoryTruncationFeature.validateDropInvariants(table, snapshot)) { return PreDowngradeStatus.DID_NOT_PERFORM_CHANGES } val properties = Seq(TestRemovableWriterWithHistoryTruncationFeature.TABLE_PROP_KEY) AlterTableUnsetPropertiesDeltaCommand( table, properties, ifExists = true, fromDropFeatureCommand = true).run(spark) PreDowngradeStatus.PERFORMED_CHANGES } } case class TestReaderWriterFeaturePreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand with DeltaLogging { // To remove the feature we only need to remove the table property. override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = { // Make sure feature data/metadata exist before proceeding. if (TestRemovableReaderWriterFeature.validateDropInvariants(table, table.initialSnapshot)) { return PreDowngradeStatus.DID_NOT_PERFORM_CHANGES } if (DeltaUtils.isTesting) { recordDeltaEvent(table.deltaLog, "delta.test.TestReaderWriterFeaturePreDowngradeCommand") } val properties = Seq(TestRemovableReaderWriterFeature.TABLE_PROP_KEY) AlterTableUnsetPropertiesDeltaCommand( table, properties, ifExists = true, fromDropFeatureCommand = true).run(spark) PreDowngradeStatus.PERFORMED_CHANGES } } case class TestLegacyWriterFeaturePreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand { /** Return true if we removed the property, false if no action was needed. */ override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = { if (TestRemovableLegacyWriterFeature.validateDropInvariants(table, table.initialSnapshot)) { return PreDowngradeStatus.DID_NOT_PERFORM_CHANGES } val properties = Seq(TestRemovableLegacyWriterFeature.TABLE_PROP_KEY) AlterTableUnsetPropertiesDeltaCommand( table, properties, ifExists = true, fromDropFeatureCommand = true).run(spark) PreDowngradeStatus.PERFORMED_CHANGES } } case class TestLegacyReaderWriterFeaturePreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand { /** Return true if we removed the property, false if no action was needed. */ override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = { val snapshot = table.initialSnapshot if (TestRemovableLegacyReaderWriterFeature.validateDropInvariants(table, snapshot)) { return PreDowngradeStatus.DID_NOT_PERFORM_CHANGES } val properties = Seq(TestRemovableLegacyReaderWriterFeature.TABLE_PROP_KEY) AlterTableUnsetPropertiesDeltaCommand( table, properties, ifExists = true, fromDropFeatureCommand = true).run(spark) PreDowngradeStatus.PERFORMED_CHANGES } } private[delta] class DeletionVectorsRemovalMetrics( val numDeletionVectorsToRemove: Long, val numDeletionVectorRowsToRemove: Long, var dvTombstonesWithinRetentionPeriod: Long = 0L, var addDVTombstonesTime: Long = 0L, var downgradeTimeMs: Long = 0L) case class DeletionVectorsPreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand with DeltaLogging { /** * Create RemoveFiles (tombstones) that directly reference deletion vector within the retention * period. These protect the latter from accidental removal from clients that do not support * deletion vectors. * * Note, we always create the DV tombstones even for the drop feature with history * truncation implementation. This is to protect against a corner case where the user run * drop feature with fastDropFeature.enabled = false and then run again with * fastDropFeature.enabled = true. * * @param checkIfSnapshotUpdatedSinceTs The timestamp to use for updating the snapshot. * @param metrics The deletion vectors removal metrics. This function only updates the DV * tombstone related metrics. */ private def generateDVTombstones( spark: SparkSession, checkIfSnapshotUpdatedSinceTs: Long, metrics: DeletionVectorsRemovalMetrics): Unit = { import scala.jdk.CollectionConverters._ import org.apache.spark.sql.delta.implicits._ if (!spark.conf.get(DeltaSQLConf.FAST_DROP_FEATURE_GENERATE_DV_TOMBSTONES)) return val startTimeNs = System.nanoTime() val snapshotToUse = table.update(checkIfUpdatedSinceTs = Some(checkIfSnapshotUpdatedSinceTs)) val deletionVectorPath = DeletionVectorDescriptor.urlEncodedRelativePathIfExists( deletionVectorCol = col("deletionVector"), tablePath = table.deltaLog.dataPath) val isInlineDeletionVector = DeletionVectorDescriptor.isInline(col("deletionVector")) // SnapshotToUse.tombstones returns only the tombstones within the retention period. The // default tombstone retention period is 7 days. Note, that if a RemoveFile contains // DeletionVectorDescriptor, it is guaranteed it is not a DV Tombstone. Furthermore, we // use distinct to deduplicate the DV references. This is because we merge DVs, and as a // result, several AddFiles may point to the same DV file. val removeFilesWithDVs = snapshotToUse.tombstones .filter(col("deletionVector").isNotNull) .filter(not(isInlineDeletionVector)) .select(deletionVectorPath.as("path")) .filter(col("path").isNotNull) .distinct() // This is a union of the DV tombstones and the regular data file tombstones without DVs (we // cannot tell the difference). We use it to identify which DV tombstones are already created. val filesWithoutDVs = snapshotToUse.tombstones .filter(col("deletionVector").isNull) .select("path") val dvTombstonePathsToAdd = removeFilesWithDVs .join(filesWithoutDVs, "path", "left_anti") .as[String] val actionsToCommit = dvTombstonePathsToAdd.toLocalIterator().asScala.map { dvPath => // Disable scala style rules to ignore warning that RemoveFile files should never be // instantiated directly. // scalastyle:off RemoveFile( path = dvPath, deletionTimestamp = Some(table.deltaLog.clock.getTimeMillis()), dataChange = false) // scalastyle:on } // We pay some overhead here to estimate the memory required to hold the results. // Above some threshold we use commitLarge. This allows to use an iterator instead of // materializing results in memory. However, it comes with some disadvantages: if there is a // conflict the commit is not retried. // A cheaper alternative would be to use snapshot.numDeletionVectorsOpt // (right before the reorg in drop feature) but this does not capture deduplication as well as // any reorgs that occurred before dropping DVs. // We assume 1024 bytes are required per RemoveFile. val tombstonesToAddCount = dvTombstonePathsToAdd.select(approx_count_distinct("path")).as[Long].first val tombstoneCountThreshold = spark.conf.get(DeltaSQLConf.FAST_DROP_FEATURE_DV_TOMBSTONE_COUNT_THRESHOLD) if (tombstonesToAddCount > tombstoneCountThreshold) { table.startTransaction(Some(snapshotToUse)).commitLarge( spark, nonProtocolMetadataActions = actionsToCommit, op = DeltaOperations.AddDeletionVectorsTombstones, newProtocolOpt = None, context = Map.empty, metrics = Map("dvTombstonesWithinRetentionPeriod" -> tombstonesToAddCount.toString)) } else { table.startTransaction(Some(snapshotToUse)) .commit(actionsToCommit.toList, DeltaOperations.AddDeletionVectorsTombstones) } metrics.addDVTombstonesTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs) metrics.dvTombstonesWithinRetentionPeriod = tombstonesToAddCount } private def reorgTable(spark: SparkSession) = { // Wrap `table` in a ResolvedTable that can be passed to DeltaReorgTableCommand. The catalog & // table ID won't be used by DeltaReorgTableCommand. import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ val catalog = table.spark.sessionState.catalogManager.currentCatalog.asTableCatalog val tableId = Seq(table.name()).asIdentifier DeltaReorgTableCommand(target = ResolvedTable.create(catalog, tableId, table))(Nil) .run(table.spark) } /** * We first remove the table feature property to prevent any transactions from committing * new DVs. This will cause any concurrent transactions tox fail. Then, we run PURGE * to remove existing DVs from the latest snapshot. * Note, during the protocol downgrade phase we validate whether all invariants still hold. * This should detect if any concurrent txns enabled the feature and/or added DVs again. * * @return Returns true if it removed DV metadata property and/or DVs. False otherwise. */ override def removeFeatureTracesIfNeeded( spark: SparkSession): PreDowngradeStatus = { val startTimeNs = table.deltaLog.clock.nanoTime() // Latest snapshot looks clean. No action is required. We may proceed // to the protocol downgrade phase. val snapshot = table.update() val tracesFound = !DeletionVectorsTableFeature.validateDropInvariants(table, snapshot) if (tracesFound) { val properties = Seq(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key) AlterTableUnsetPropertiesDeltaCommand( table, properties, ifExists = true, fromDropFeatureCommand = true).run(spark) reorgTable(spark) } val metrics = new DeletionVectorsRemovalMetrics( numDeletionVectorsToRemove = snapshot.numDeletionVectorsOpt.getOrElse(0L), numDeletionVectorRowsToRemove = snapshot.numDeletedRecordsOpt.getOrElse(0L)) reorgTable(spark) // Even if there no DV traces in the table we check if there are missing DV tombstones. // This is to protect against an edge case where all DV traces are cleaned before invoking // the drop feature command. generateDVTombstones(spark, startTimeNs, metrics) metrics.downgradeTimeMs = TimeUnit.NANOSECONDS.toMillis(table.deltaLog.clock.nanoTime() - startTimeNs) recordDeltaEvent( table.deltaLog, opType = "delta.deletionVectorsFeatureRemovalMetrics", data = metrics) PreDowngradeStatus(performedChanges = tracesFound) } } case class V2CheckpointPreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand with DeltaLogging { /** * We set the checkpoint policy to classic to prevent any transactions from creating * v2 checkpoints. * * @return True if it changed checkpoint policy metadata property to classic. * False otherwise. */ override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = { if (V2CheckpointTableFeature.validateDropInvariants(table, table.initialSnapshot)) { return PreDowngradeStatus.DID_NOT_PERFORM_CHANGES } val startTimeNs = System.nanoTime() val properties = Map(DeltaConfigs.CHECKPOINT_POLICY.key -> CheckpointPolicy.Classic.name) AlterTableSetPropertiesDeltaCommand(table, properties).run(spark) recordDeltaEvent( table.deltaLog, opType = "delta.v2CheckpointFeatureRemovalMetrics", data = Map(("downgradeTimeMs", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs))) ) PreDowngradeStatus.PERFORMED_CHANGES } } case class InCommitTimestampsPreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand with DeltaLogging { /** * We disable the feature by: * - Removing the table properties: * 1. DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP * 2. DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION * - Setting the table property DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED to false. * Technically, only setting IN_COMMIT_TIMESTAMPS_ENABLED to false is enough to disable the * feature. However, we can use this opportunity to clean up the metadata. * * @return true if any change to the metadata (the three properties listed above) was made. * False otherwise. */ override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = { val startTimeNs = System.nanoTime() val currentMetadata = table.initialSnapshot.metadata val currentTableProperties = currentMetadata.configuration val enablementProperty = DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED val ictEnabledInMetadata = enablementProperty.fromMetaData(currentMetadata) val provenanceProperties = Seq( DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.key, DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.key) val propertiesToRemove = provenanceProperties.filter(currentTableProperties.contains) val traceRemovalNeeded = propertiesToRemove.nonEmpty || ictEnabledInMetadata if (traceRemovalNeeded) { val propertiesToDisable = Option.when(ictEnabledInMetadata)(enablementProperty.key -> "false") val desiredTableProperties = currentTableProperties .filterNot{ case (k, _) => propertiesToRemove.contains(k) } ++ propertiesToDisable val deltaOperation = DeltaOperations.UnsetTableProperties( (propertiesToRemove ++ propertiesToDisable.map(_._1)).toSeq, ifExists = true) table.startTransaction().commit( Seq(currentMetadata.copy(configuration = desiredTableProperties.toMap)), deltaOperation) } val provenancePropertiesPresenceLogs = provenanceProperties.map { prop => prop -> currentTableProperties.contains(prop).toString } recordDeltaEvent( table.deltaLog, opType = "delta.inCommitTimestampFeatureRemovalMetrics", data = Map( "downgradeTimeMs" -> TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs), "traceRemovalNeeded" -> traceRemovalNeeded.toString, enablementProperty.key -> ictEnabledInMetadata ) ++ provenancePropertiesPresenceLogs ) PreDowngradeStatus(performedChanges = traceRemovalNeeded) } } case class VacuumProtocolCheckPreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand with DeltaLogging { /** * Returns true when it performs a cleaning action. When no action was required * it returns false. * For downgrading the [[VacuumProtocolCheckTableFeature]], we don't need remove any traces, we * just need to remove the feature from the [[Protocol]]. */ override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = PreDowngradeStatus.DID_NOT_PERFORM_CHANGES } case class CoordinatedCommitsPreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand with DeltaLogging { /** * We disable the feature by removing the following table properties: * 1. DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.key * 2. DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF.key * 3. DeltaConfigs.COORDINATED_COMMITS_TABLE_CONF.key * If these properties have been removed but unbackfilled commits are still present, we * backfill them. * * @return true if any change to the metadata (the three properties listed above) was made OR * if there were any unbackfilled commits that were backfilled. * false otherwise. */ override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = { val startTimeNs = System.nanoTime() var traceRemovalNeeded = false var exceptionOpt = Option.empty[Throwable] val propertyPresenceLogs = CoordinatedCommitsUtils.TABLE_PROPERTY_KEYS.map( key => key -> table.initialSnapshot.metadata.configuration.contains(key).toString ) if (CoordinatedCommitsUtils.tablePropertiesPresent(table.initialSnapshot.metadata)) { traceRemovalNeeded = true try { AlterTableUnsetPropertiesDeltaCommand( table, CoordinatedCommitsUtils.TABLE_PROPERTY_KEYS, ifExists = true, fromDropFeatureCommand = true ).run(spark) } catch { case NonFatal(e) => exceptionOpt = Some(e) } } var postDisablementUnbackfilledCommitsPresent = false if (exceptionOpt.isEmpty) { val snapshotAfterDisabling = table.update() assert(snapshotAfterDisabling.getTableCommitCoordinatorForWrites.isEmpty) postDisablementUnbackfilledCommitsPresent = CoordinatedCommitsUtils.unbackfilledCommitsPresent(snapshotAfterDisabling) if (postDisablementUnbackfilledCommitsPresent) { traceRemovalNeeded = true // Coordinated commits have already been disabled but there are unbackfilled commits. CoordinatedCommitsUtils.backfillWhenCoordinatedCommitsDisabled(snapshotAfterDisabling) } } recordDeltaEvent( table.deltaLog, opType = "delta.coordinatedCommitsFeatureRemovalMetrics", data = Map( "downgradeTimeMs" -> TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs), "traceRemovalNeeded" -> traceRemovalNeeded.toString, "traceRemovalSuccess" -> exceptionOpt.isEmpty.toString, "traceRemovalException" -> exceptionOpt.map(_.getMessage).getOrElse(""), "postDisablementUnbackfilledCommitsPresent" -> postDisablementUnbackfilledCommitsPresent.toString ) ++ propertyPresenceLogs ) exceptionOpt.foreach(throw _) PreDowngradeStatus(performedChanges = traceRemovalNeeded) } } case class TypeWideningPreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand with DeltaLogging { /** * Unset the type widening table property to prevent new type changes to be applied to the table, * then removes traces of the feature: * - Rewrite files that have columns or fields with a different type than in the current table * schema. These are all files not added or modified after the last type change. * - Remove the type widening metadata attached to fields in the current table schema. * * @return Return true if files were rewritten or metadata was removed. False otherwise. */ override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = { if (TypeWideningTableFeature.validateDropInvariants(table, table.initialSnapshot)) { return PreDowngradeStatus.DID_NOT_PERFORM_CHANGES } val startTimeNs = System.nanoTime() val properties = Seq(DeltaConfigs.ENABLE_TYPE_WIDENING.key) AlterTableUnsetPropertiesDeltaCommand( table, properties, ifExists = true, fromDropFeatureCommand = true).run(spark) val numFilesRewritten = rewriteFilesIfNeeded(spark) val metadataRemoved = removeMetadataIfNeeded() recordDeltaEvent( table.deltaLog, opType = "delta.typeWidening.featureRemoval", data = Map( "downgradeTimeMs" -> TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs), "numFilesRewritten" -> numFilesRewritten, "metadataRemoved" -> metadataRemoved ) ) PreDowngradeStatus(performedChanges = numFilesRewritten > 0 || metadataRemoved) } /** * Rewrite files that have columns or fields with a different type than in the current table * schema. These are all files not added or modified after the last type change. * @return Return the number of files rewritten. */ private def rewriteFilesIfNeeded(spark: SparkSession): Long = { if (!TypeWideningMetadata.containsTypeWideningMetadata(table.initialSnapshot.schema)) { return 0L } // Wrap `table` in a ResolvedTable that can be passed to DeltaReorgTableCommand. The catalog & // table ID won't be used by DeltaReorgTableCommand. import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ val catalog = spark.sessionState.catalogManager.currentCatalog.asTableCatalog val tableId = Seq(table.name()).asIdentifier val reorg = DeltaReorgTableCommand( target = ResolvedTable.create(catalog, tableId, table), reorgTableSpec = DeltaReorgTableSpec(DeltaReorgTableMode.REWRITE_TYPE_WIDENING, None) )(Nil) val rows = reorg.run(spark) val metrics = rows.head.getAs[OptimizeMetrics](1) metrics.numFilesRemoved } /** * Remove the type widening metadata attached to fields in the current table schema. * @return Return true if any metadata was removed. False otherwise. */ private def removeMetadataIfNeeded(): Boolean = { if (!TypeWideningMetadata.containsTypeWideningMetadata(table.initialSnapshot.schema)) { return false } val txn = table.startTransaction() val metadata = txn.metadata val (cleanedSchema, changes) = TypeWideningMetadata.removeTypeWideningMetadata(metadata.schema) txn.commit( metadata.copy(schemaString = cleanedSchema.json) :: Nil, DeltaOperations.UpdateColumnMetadata("DROP FEATURE", changes)) true } } case class ColumnMappingPreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand with DeltaLogging { /** * We first remove the table feature property to prevent any transactions from writing data * files with the physical names. This will cause any concurrent transactions to fail. * Then, we run RemoveColumnMappingCommand to rewrite the files rename columns. * Note, during the protocol downgrade phase we validate whether all invariants still hold. * This should detect if any concurrent txns enabled the table property again. * * @return Returns true if it removed table property and/or has rewritten the data. * False otherwise. */ override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = { // Latest snapshot looks clean. No action is required. We may proceed // to the protocol downgrade phase. if (ColumnMappingTableFeature.validateDropInvariants(table, table.initialSnapshot)) { return PreDowngradeStatus.DID_NOT_PERFORM_CHANGES } recordDeltaOperation( table.deltaLog, opType = "delta.columnMappingFeatureRemoval") { RemoveColumnMappingCommand(table.deltaLog, table.catalogTable) .run(spark, removeColumnMappingTableProperty = true) } PreDowngradeStatus.PERFORMED_CHANGES } } case class CheckConstraintsPreDowngradeTableFeatureCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand { /** * Throws an exception if the table has CHECK constraints, and returns false otherwise (as no * action was required). * * We intentionally error out instead of removing the CHECK constraints here, as dropping a * table feature should not never alter the logical representation of a table (only its physical * representation). Instead, we ask the user to explicitly drop the constraints before the table * feature can be dropped. */ override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = { val checkConstraintNames = Constraints.getCheckConstraintNames(table.initialSnapshot.metadata) if (checkConstraintNames.isEmpty) return PreDowngradeStatus.DID_NOT_PERFORM_CHANGES throw DeltaErrors.cannotDropCheckConstraintFeature(checkConstraintNames) } } case class CheckpointProtectionPreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand { import org.apache.spark.sql.delta.actions.DropTableFeatureUtils._ import org.apache.spark.sql.delta.CheckpointProtectionTableFeature._ /** * To remove the feature we need to truncate all history prior to the atomic cleanup version. * For this cleanup operation we use a shorter log retention period of 24 hours as defined in * (delta.dropFeatureTruncateHistory.retentionDuration). The history truncation here needs to * adhere to all the invariants established by the CheckpointProtectionTableFeature, similarly * to any other metadata cleanup invocations (see doc in CheckpointProtectionTableFeature and * REQUIRE_CHECKPOINT_PROTECTION_BEFORE_VERSION). * * The pre-downgrade process here mimics the downgrade process of the legacy drop feature * implementation for features with requiresHistoryProtection=true. * * Note, this feature can only be dropped with the TRUNCATE HISTORY option. Therefore, the * removal of CheckpointProtection does not require the addition of CheckpointProtection to * protect history. * * Always returns false since we do not perform any modifications that require history * expiration. This allows the drop process to proceed immediately after we cleanup the history * prior to requireCheckpointProtectionBeforeVersion. */ override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = { val snapshot = table.initialSnapshot if (!historyPriorToCheckpointProtectionVersionIsTruncated(snapshot, table.catalogTable)) { // Add a checkpoint here to make sure we can cleanup up everything before this commit. // This is because metadata cleanup operations, can only clean up to the latest checkpoint. createEmptyCommitAndCheckpoint(table, table.deltaLog.clock.nanoTime()) table.deltaLog.cleanUpExpiredLogs( snapshot, table.catalogTable, deltaRetentionMillisOpt = Some(truncateHistoryLogRetentionMillis(snapshot.metadata)), cutoffTruncationGranularity = TruncationGranularity.MINUTE) if (!historyPriorToCheckpointProtectionVersionIsTruncated(snapshot, table.catalogTable)) { throw DeltaErrors.dropCheckpointProtectionWaitForRetentionPeriod( table.initialSnapshot.metadata) } } // If history is truncated we do not need the property anymore. val property = DeltaConfigs.REQUIRE_CHECKPOINT_PROTECTION_BEFORE_VERSION.key AlterTableUnsetPropertiesDeltaCommand( table, Seq(property), ifExists = true, fromDropFeatureCommand = true).run(spark) // We did not do any changes that require history expiration. It is ok if the removed property // exists in history. PreDowngradeStatus.DID_NOT_PERFORM_CHANGES } } case class RedirectWriterOnlyPreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand with DeltaLogging { /** * We disable the feature by removing [[DeltaConfigs.REDIRECT_WRITER_ONLY]]. * * @return True if the property is removed. False otherwise. */ override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = { // Make sure feature data/metadata exist before proceeding. if (RedirectWriterOnlyFeature.validateDropInvariants(table, table.initialSnapshot)) { return PreDowngradeStatus.DID_NOT_PERFORM_CHANGES } val properties = Seq(DeltaConfigs.REDIRECT_WRITER_ONLY.key) AlterTableUnsetPropertiesDeltaCommand( table, properties, ifExists = false, fromDropFeatureCommand = true).run(spark) PreDowngradeStatus.PERFORMED_CHANGES } } case class RedirectReaderWriterPreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand with DeltaLogging { /** * We disable the feature by removing [[DeltaConfigs.REDIRECT_READER_WRITER]]. * * @return True if the property is removed. False otherwise. */ override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = { // Make sure feature data/metadata exist before proceeding. if (RedirectReaderWriterFeature.validateDropInvariants(table, table.initialSnapshot)) { return PreDowngradeStatus.DID_NOT_PERFORM_CHANGES } val properties = Seq(DeltaConfigs.REDIRECT_READER_WRITER.key) AlterTableUnsetPropertiesDeltaCommand( table, properties, ifExists = false, fromDropFeatureCommand = true).run(spark) PreDowngradeStatus.PERFORMED_CHANGES } } case class RowTrackingPreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand with DeltaLogging { /** * Disabling the feature involves the following steps: * * 1) Set `delta.enableRowTracking` to false so clients do not expect anymore all files * to have row IDs. * 2) Set `delta.rowTrackingSuspended` to true to suspend row ID generation. * 3) Unbackfill all existing row IDs. * * Note, the remaining relevant properties/metadataDomains are removed at the downgrade protocol * commit. * * @return True if the feature traces are removed. False otherwise. */ override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = { if (RowTrackingFeature.validateDropInvariants(table, table.update())) { return PreDowngradeStatus.DID_NOT_PERFORM_CHANGES } val propertiesToSet = Map( DeltaConfigs.ROW_TRACKING_ENABLED.key -> "false", DeltaConfigs.ROW_TRACKING_SUSPENDED.key -> "true") AlterTableSetPropertiesDeltaCommand(table, propertiesToSet).run(spark) val commitSeq = RowTrackingUnBackfillCommand( table.deltaLog, nameOfTriggeringOperation = DeltaOperations.OP_DROP_FEATURE, table.catalogTable).run(spark) PreDowngradeStatus( performedChanges = true, commitSeq.lastOption.map(_.getLong(0))) } } case class DomainMetadataPreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand { /** * Removes domain metadata from the table. In general, each feature should be responsible * for removing its own domain metadata when dropped. The cleanup process here is to make sure * the domainMetadata feature can be dropped in cases where domain metadata is leaked. * * Note, the domainMetadata feature can only be dropped when no dependent features are present * in the table. This ensures that any domain metadata found on the table are leaked metadata. * * @return True if the feature traces are removed. False otherwise. */ override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = { val snapshot = table.update() if (DomainMetadataTableFeature.validateDropInvariants(table, snapshot)) { return PreDowngradeStatus.DID_NOT_PERFORM_CHANGES } val actionsToCommit = snapshot .domainMetadata .map(_.copy(removed = true)) table .startTransaction() .commit(actionsToCommit, DeltaOperations.DomainMetadataCleanup(actionsToCommit.length)) PreDowngradeStatus.PERFORMED_CHANGES } } /** * PreDowngrade command for MaterializePartitionColumns feature. * This feature doesn't require any special cleanup actions when being dropped. */ case class MaterializePartitionColumnsPreDowngradeCommand(table: DeltaTableV2) extends PreDowngradeTableFeatureCommand { /** * No cleanup actions are needed. The table property is automatically removed by the DROP FEATURE * via tablePropertiesToRemoveAtDowngradeCommit. */ override def removeFeatureTracesIfNeeded(spark: SparkSession): PreDowngradeStatus = { PreDowngradeStatus.DID_NOT_PERFORM_CHANGES } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/PreprocessTableDelete.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.commands.DeleteCommand import org.apache.spark.sql.catalyst.expressions.SubqueryExpression import org.apache.spark.sql.catalyst.plans.logical.{DeltaDelete, LogicalPlan} import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.internal.SQLConf /** * Preprocess the [[DeltaDelete]] plan to convert to [[DeleteCommand]]. */ case class PreprocessTableDelete(sqlConf: SQLConf) extends Rule[LogicalPlan] { override def apply(plan: LogicalPlan): LogicalPlan = { plan.resolveOperators { case d: DeltaDelete if d.resolved => d.condition.foreach { cond => if (SubqueryExpression.hasSubquery(cond)) { throw DeltaErrors.subqueryNotSupportedException("DELETE", cond) } } DeleteCommand(d) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/PreprocessTableMerge.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.time.{Instant, LocalDateTime} import java.util.Locale import scala.collection.mutable import scala.reflect.ClassTag import org.apache.spark.sql.delta.commands.MergeIntoCommand import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression import org.apache.spark.sql.catalyst.optimizer.ComputeCurrentTime import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.trees.TreePattern.CURRENT_LIKE import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.catalyst.util.DateTimeUtils.{instantToMicros, localDateTimeToMicros} import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataType, DateType, StringType, StructField, StructType, TimestampNTZType, TimestampType} case class PreprocessTableMerge(override val conf: SQLConf) extends Rule[LogicalPlan] with UpdateExpressionsSupport { override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperators { case m: DeltaMergeInto if m.resolved => apply(m, true) } def apply(mergeInto: DeltaMergeInto, transformToCommand: Boolean): LogicalPlan = { val DeltaMergeInto( target, source, condition, matched, notMatched, notMatchedBySource, withSchemaEvolution, finalSchemaOpt) = mergeInto if (finalSchemaOpt.isEmpty) { throw DeltaErrors.targetTableFinalSchemaEmptyException() } val postEvolutionTargetSchema = finalSchemaOpt.get def checkCondition(cond: Expression, conditionName: String): Unit = { if (!cond.deterministic) { throw DeltaErrors.nonDeterministicNotSupportedException( s"$conditionName condition of MERGE operation", cond) } if (cond.find(_.isInstanceOf[AggregateExpression]).isDefined) { throw DeltaErrors.aggsNotSupportedException( s"$conditionName condition of MERGE operation", cond) } if (SubqueryExpression.hasSubquery(cond)) { throw DeltaErrors.subqueryNotSupportedException( s"$conditionName condition of MERGE operation", cond) } } checkCondition(condition, "search") (matched ++ notMatched ++ notMatchedBySource).filter(_.condition.nonEmpty).foreach { clause => checkCondition(clause.condition.get, clause.clauseType.toUpperCase(Locale.ROOT)) } val deltaLogicalPlan = EliminateSubqueryAliases(target) val tahoeFileIndex = deltaLogicalPlan match { case DeltaFullTable(_, index) => index case o => throw DeltaErrors.notADeltaSourceException("MERGE", Some(o)) } val generatedColumns = GeneratedColumn.getGeneratedColumns( tahoeFileIndex.snapshotAtAnalysis) if (generatedColumns.nonEmpty && !GeneratedColumn.allowDMLTargetPlan(deltaLogicalPlan, conf)) { throw DeltaErrors.operationOnTempViewWithGenerateColsNotSupported("MERGE INTO") } val identityColumns = IdentityColumn.getIdentityColumns( tahoeFileIndex.snapshotAtAnalysis.metadata.schema) // A mapping from the identity column struct field to the GenerateIdentityColumnValues // expression for the target table in the MERGE clause. val identityColumnExpressionMap = mutable.Map[StructField, Expression]() // Column names for which we need to track IDENTITY high water marks. var trackHighWaterMarks = Set[String]() val processedMatched = matched.map { case m: DeltaMergeIntoMatchedUpdateClause => val alignedActions = alignUpdateActions( target, m.resolvedActions, whenClauses = matched ++ notMatched ++ notMatchedBySource, identityColumns = identityColumns, generatedColumns = generatedColumns, allowSchemaEvolution = withSchemaEvolution, postEvolutionTargetSchema = postEvolutionTargetSchema) m.copy(m.condition, alignedActions) case m: DeltaMergeIntoMatchedDeleteClause => m // Delete does not need reordering } val processedNotMatchedBySource = notMatchedBySource.map { case m: DeltaMergeIntoNotMatchedBySourceUpdateClause => val alignedActions = alignUpdateActions( target, m.resolvedActions, whenClauses = matched ++ notMatched ++ notMatchedBySource, identityColumns = identityColumns, generatedColumns = generatedColumns, allowSchemaEvolution = withSchemaEvolution, postEvolutionTargetSchema = postEvolutionTargetSchema) m.copy(m.condition, alignedActions) case m: DeltaMergeIntoNotMatchedBySourceDeleteClause => m // Delete does not need reordering } val processedNotMatched = notMatched.map { case m: DeltaMergeIntoNotMatchedInsertClause => // Check if columns are distinct. All actions should have targetColNameParts.size = 1. m.resolvedActions.foreach { a => if (a.targetColNameParts.size > 1) { throw DeltaErrors.nestedFieldNotSupported( "INSERT clause of MERGE operation", a.targetColNameParts.mkString("`", "`.`", "`") ) } } IdentityColumn.blockExplicitIdentityColumnInsert( identityColumns, m.resolvedActions.map(_.targetColNameParts)) val targetColNames = m.resolvedActions.map(_.targetColNameParts.head) if (targetColNames.distinct.size < targetColNames.size) { throw DeltaErrors.duplicateColumnOnInsert() } // Generate actions for columns that are not explicitly inserted. They might come from // the original schema of target table or the schema evolved columns. In either case they are // covered by `finalSchema`. val implicitActions = postEvolutionTargetSchema.filterNot { col => m.resolvedActions.exists { insertAct => conf.resolver(insertAct.targetColNameParts.head, col.name) } }.map { col => import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns.getDefaultValueExprOrNullLit val defaultValue: Expression = getDefaultValueExprOrNullLit(col, conf.useNullsForMissingDefaultColumnValues) .getOrElse(Literal(null, col.dataType)) DeltaMergeAction( targetColNameParts = Seq(col.name), expr = defaultValue, // INSERT * operations set target-only struct fields to null, since there is no existing // target row to preserve values from. targetOnlyStructFieldBehavior = TargetOnlyStructFieldBehavior.NULLIFY, targetColNameResolved = true) } val actions = m.resolvedActions ++ implicitActions val (actionsWithGeneratedColumns, trackFromInsert) = resolveImplicitColumns( m.resolvedActions, actions, source, generatedColumns.map(f => (f, true)) ++ identityColumns.map(f => (f, false)), postEvolutionTargetSchema, identityColumnExpressionMap) trackHighWaterMarks ++= trackFromInsert val alignedActions: Seq[DeltaMergeAction] = postEvolutionTargetSchema.map { targetAttrib => actionsWithGeneratedColumns.find { a => conf.resolver(targetAttrib.name, a.targetColNameParts.head) }.map { a => DeltaMergeAction( targetColNameParts = Seq(targetAttrib.name), expr = castIfNeeded( a.expr, targetAttrib.dataType, castingBehavior = MergeOrUpdateCastingBehavior(withSchemaEvolution), targetAttrib.name), // The action has been aligned/cast to the target schema. targetOnlyStructFieldBehavior = TargetOnlyStructFieldBehavior.TARGET_ALIGNED, targetColNameResolved = true) }.getOrElse { // If a target table column was not found in the INSERT columns and expressions, // then throw exception as there must be an expression to set every target column. throw DeltaErrors.columnOfTargetTableNotFoundInMergeException( targetAttrib.name, targetColNames.mkString(", ")) } } m.copy(m.condition, alignedActions) } if (transformToCommand) { val (relation, tahoeFileIndex) = EliminateSubqueryAliases(target) match { case DeltaFullTable(rel, index) => rel -> index case o => throw DeltaErrors.notADeltaSourceException("MERGE", Some(o)) } /** * Because source and target are not children of MergeIntoCommand they are not processed when * invoking the [[ComputeCurrentTime]] rule. This is why they need special handling. */ val now = Instant.now() // Transform timestamps for the MergeIntoCommand, source, and target using the same instant. // Called explicitly because source and target are not children of MergeIntoCommand. transformTimestamps( MergeIntoCommand( transformTimestamps(source, now), transformTimestamps(target, now), relation.catalogTable, tahoeFileIndex, condition, processedMatched, processedNotMatched, processedNotMatchedBySource, migratedSchema = finalSchemaOpt, trackHighWaterMarks = trackHighWaterMarks, schemaEvolutionEnabled = withSchemaEvolution), now) } else { DeltaMergeInto( source, target, condition, processedMatched, processedNotMatched, processedNotMatchedBySource, withSchemaEvolution, finalSchemaOpt) } } private def transformTimestamps(plan: LogicalPlan, instant: Instant): LogicalPlan = { import org.apache.spark.sql.delta.implicits._ val currentTimestampMicros = instantToMicros(instant) val currentTime = Literal.create(currentTimestampMicros, TimestampType) val timezone = Literal.create(conf.sessionLocalTimeZone, StringType) plan.transformUpWithSubqueries { case subQuery => subQuery.transformAllExpressionsUpWithPruning(_.containsPattern(CURRENT_LIKE)) { case cd: CurrentDate => Literal.create(DateTimeUtils.microsToDays(currentTimestampMicros, cd.zoneId), DateType) case CurrentTimestamp() | Now() => currentTime case CurrentTimeZone() => timezone case localTimestamp: LocalTimestamp => val asDateTime = LocalDateTime.ofInstant(instant, localTimestamp.zoneId) Literal.create(localDateTimeToMicros(asDateTime), TimestampNTZType) } } } /** * Generates update expressions for columns that are not present in the target table and are * introduced by one of the update or insert merge clauses. The generated update expressions and * the update expressions for the existing columns are aligned to match the order in the * target output schema. * * @param target Logical plan node of the target table of merge. * @param resolvedActions Merge actions of the update clause being processed. * @param whenClauses All merge clauses of the merge operation. * @param identityColumns Additional identity columns present in the table. * @param generatedColumns List of the generated columns in the table. See * [[UpdateExpressionsSupport]]. * @param allowSchemaEvolution Whether to allow schema to evolve. See * [[UpdateExpressionsSupport]]. * @param postEvolutionTargetSchema The schema of the target table after the merge operation. * @return Update actions aligned on the target output schema `postEvolutionTargetSchema`. */ private def alignUpdateActions( target: LogicalPlan, resolvedActions: Seq[DeltaMergeAction], whenClauses: Seq[DeltaMergeIntoClause], identityColumns: Seq[StructField], generatedColumns: Seq[StructField], allowSchemaEvolution: Boolean, postEvolutionTargetSchema: StructType) : Seq[DeltaMergeAction] = { IdentityColumn.blockIdentityColumnUpdate( identityColumns, resolvedActions.map(_.targetColNameParts)) // Get the operations for columns that already exist... val existingUpdateOps = resolvedActions.map { a => UpdateOperation( targetColNameParts = a.targetColNameParts, updateExpr = a.expr, targetOnlyStructFieldBehavior = a.targetOnlyStructFieldBehavior) } val newUpdateOps = if (UpdateExpressionsSupport.isWholeStructAssignmentPreserveNullSourceStructsEnabled(conf)) { // We don't want to call `generateUpdateOpsForNewTargetFields` here because: // 1. `generateUpdateExpressions` (below) already handles any fields in the evolved schema // that don't have corresponding actions by assigning them default expressions (NULL for // new fields or the existing target value for existing target fields). // 2. `generateUpdateOpsForNewTargetFields` generates leaf-level operations for new target // fields. If these fields are null structs in the source, they will be expanded to // non-null structs with null fields in the target. Seq.empty } else { // And construct operations for columns that the insert/update clauses will add. generateUpdateOpsForNewTargetFields(target, postEvolutionTargetSchema, resolvedActions) } // Use the helper methods in UpdateExpressionsSupport to generate expressions such that nested // fields can be updated (only for existing columns). val alignedExprs = generateUpdateExpressions( targetSchema = postEvolutionTargetSchema, updateOps = existingUpdateOps ++ newUpdateOps, defaultExprs = target.output, resolver = conf.resolver, allowSchemaEvolution = allowSchemaEvolution, generatedColumns = generatedColumns) val alignedExprsWithGenerationExprs = if (alignedExprs.forall(_.nonEmpty)) { alignedExprs.map(_.get) } else { generateUpdateExprsForGeneratedColumns(target, generatedColumns, alignedExprs, Some(postEvolutionTargetSchema)) } alignedExprsWithGenerationExprs .zip(postEvolutionTargetSchema) .map { case (expr, field) => DeltaMergeAction( targetColNameParts = Seq(field.name), expr = expr, // The action has been aligned to target schema. targetOnlyStructFieldBehavior = TargetOnlyStructFieldBehavior.TARGET_ALIGNED, targetColNameResolved = true) } } /** * Generate expressions to set to null the new (potentially nested) fields that are added to the * target table by schema evolution and are not already set by any of the `resolvedActions` from * the merge clause. * * @param target Logical plan node of the target table of merge. * @param postEvolutionTargetSchema The schema of the target table after the merge operation. * @param resolvedActions Merge actions of the update clause being processed. * @return List of update operations */ private def generateUpdateOpsForNewTargetFields( target: LogicalPlan, postEvolutionTargetSchema: StructType, resolvedActions: Seq[DeltaMergeAction]) : Seq[UpdateOperation] = { // Collect all fields in the final schema that were added by schema evolution. // `SchemaPruning.pruneSchema` only prunes nested fields, we then filter out top-level fields // ourself. val targetSchemaBeforeEvolution = target.schema.map(SchemaPruning.RootField(_, derivedFromAtt = false)) val newTargetFields = StructType(SchemaPruning.pruneSchema(postEvolutionTargetSchema, targetSchemaBeforeEvolution) .filterNot { topLevelField => target.schema.exists(_.name == topLevelField.name) }) /** * Remove the field corresponding to `pathFilter` (if any) from `schema`. */ def filterSchema(schema: StructType, pathFilter: Seq[String]) : Seq[StructField] = schema.flatMap { case StructField(name, struct: StructType, _, _) if name == pathFilter.head && pathFilter.length > 1 => Some(StructField(name, StructType(filterSchema(struct, pathFilter.drop(1))))) case f: StructField if f.name == pathFilter.head => None case f => Some(f) } // Then filter out fields that are set by one of the merge actions. val newTargetFieldsWithoutAssignment = resolvedActions .map(_.targetColNameParts) .foldRight(newTargetFields) { (pathFilter, schema) => StructType(filterSchema(schema, pathFilter)) } /** * Generate the list of all leaf fields and their corresponding data type from `schema`. */ def leafFields(schema: StructType, prefix: Seq[String] = Seq.empty) : Seq[(Seq[String], DataType)] = schema.flatMap { field => val name = prefix :+ field.name.toLowerCase(Locale.ROOT) field.dataType match { case struct: StructType => leafFields(struct, name) case dataType => Seq((name, dataType)) } } // Finally, generate an update operation for each remaining field to set it to null. leafFields(newTargetFieldsWithoutAssignment).map { case (name, dataType) => UpdateOperation( targetColNameParts = name, updateExpr = Literal(null, dataType), // Leaf-level operations are aligned with the target schema naturally. targetOnlyStructFieldBehavior = TargetOnlyStructFieldBehavior.TARGET_ALIGNED) } } /** * Resolves any non explicitly inserted generated columns in `allActions` to its * corresponding generated expression. * * For each action, if it's a generated column that is not explicitly inserted, we will * use its generated expression to calculate its value by resolving to a fake project of all the * inserted values. Note that this fake project is created after we set all non explicitly * inserted columns to nulls. This guarantees that all columns referenced by the generated * column, regardless of whether they are explicitly inserted or not, will have a * corresponding expression in the fake project and hence the generated expression can * always be resolved. * * @param explicitActions Actions explicitly specified by users. * @param allActions Actions with non explicitly specified columns added with nulls. * @param sourcePlan Logical plan node of the source table of merge. * @param columnWithDefaultExpr All the generated columns in the target table. * @param identityColumnExpressionMap A mapping from identity column struct fields to expressions * @return `allActions` with expression for non explicitly inserted generated columns expression * resolved, and columns names for which we will track high water marks. */ private def resolveImplicitColumns( explicitActions: Seq[DeltaMergeAction], allActions: Seq[DeltaMergeAction], sourcePlan: LogicalPlan, columnWithDefaultExpr: Seq[(StructField, Boolean)], postEvolutionTargetSchema: StructType, identityColumnExpressionMap: mutable.Map[StructField, Expression]) : (Seq[DeltaMergeAction], Set[String]) = { val implicitColumns = columnWithDefaultExpr.filter { case (field, _) => !explicitActions.exists { insertAct => conf.resolver(insertAct.targetColNameParts.head, field.name) } } if (implicitColumns.isEmpty) { return (allActions, Set[String]()) } assert(postEvolutionTargetSchema.size == allActions.size, "Invalid number of columns in INSERT clause with generated columns. Expected schema: " + s"$postEvolutionTargetSchema, INSERT actions: $allActions") val track = mutable.Set[String]() // Fake projection used to resolve generated column expressions. val fakeProjectMap = allActions.map { action => { val exprForProject = Alias(action.expr, action.targetColNameParts.head)() exprForProject.exprId -> exprForProject } }.toMap val fakeProject = Project(fakeProjectMap.values.toArray[Alias], sourcePlan) val resolvedActions = allActions.map { action => val colName = action.targetColNameParts.head implicitColumns.find { case (field, _) => conf.resolver(field.name, colName) } match { case Some((field, true)) => val expr = GeneratedColumn.getGenerationExpression(field).get val resolvedExpr = resolveReferencesForExpressions(SparkSession.active, expr :: Nil, fakeProject).head // Replace references to fakeProject with original expression. val transformedExpr = resolvedExpr.transform { case a: AttributeReference if fakeProjectMap.contains(a.exprId) => fakeProjectMap(a.exprId).child } action.copy(expr = transformedExpr) case Some((field, false)) => // This is the IDENTITY column case. Track the high water marks collection and produce // IDENTITY value generation function. track += field.name // Reuse the existing identityExp which we might have already generated. This is to make // sure that we use the same identity column generation expression across different // WHEN NOT MATCHED branches for a given identity column - so that we can generate // identity values from the same generator and prevent duplicate identity values. val identityExp = identityColumnExpressionMap.getOrElseUpdate( field, IdentityColumn.createIdentityColumnGenerationExpr(field)) action.copy(expr = identityExp) case _ => action } } (resolvedActions, track.toSet) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/PreprocessTableUpdate.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.commands.UpdateCommand import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases import org.apache.spark.sql.catalyst.expressions.SubqueryExpression import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.internal.SQLConf /** * Preprocesses the [[DeltaUpdateTable]] logical plan before converting it to [[UpdateCommand]]. * - Adjusts the column order, which could be out of order, based on the destination table * - Generates expressions to compute the value of all target columns in Delta table, while taking * into account that the specified SET clause may only update some columns or nested fields of * columns. */ case class PreprocessTableUpdate(sqlConf: SQLConf) extends Rule[LogicalPlan] with UpdateExpressionsSupport { override def conf: SQLConf = sqlConf override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperators { case u: DeltaUpdateTable if u.resolved => u.condition.foreach { cond => if (SubqueryExpression.hasSubquery(cond)) { throw DeltaErrors.subqueryNotSupportedException("UPDATE", cond) } } toCommand(u) } def toCommand(update: DeltaUpdateTable): UpdateCommand = { val deltaLogicalNode = EliminateSubqueryAliases(update.child) val (relation, index) = deltaLogicalNode match { case DeltaFullTable(rel, tahoeFileIndex) => rel -> tahoeFileIndex case o => throw DeltaErrors.notADeltaSourceException("UPDATE", Some(o)) } val generatedColumns = GeneratedColumn.getGeneratedColumns(index) if (generatedColumns.nonEmpty && !GeneratedColumn.allowDMLTargetPlan(deltaLogicalNode, conf)) { // Disallow temp views referring to a Delta table that contains generated columns. When the // user doesn't provide expressions for generated columns, we need to create update // expressions for them automatically. Currently, we assume `update.child.output` is the same // as the table schema when checking whether a column in `update.child.output` is a generated // column in the table. throw DeltaErrors.operationOnTempViewWithGenerateColsNotSupported("UPDATE") } val targetColNameParts = update.updateColumns.map(DeltaUpdateTable.getTargetColNameParts(_)) IdentityColumn.blockIdentityColumnUpdate(index.snapshotAtAnalysis.schema, targetColNameParts) val alignedUpdateExprs = generateUpdateExpressions( targetSchema = update.child.schema, defaultExprs = update.child.output, nameParts = targetColNameParts, updateExprs = update.updateExpressions, resolver = conf.resolver, generatedColumns = generatedColumns ) val alignedUpdateExprsAfterAddingGenerationExprs = if (alignedUpdateExprs.forall(_.nonEmpty)) { alignedUpdateExprs.map(_.get) } else { // Some expressions for generated columns are not specified by the user, so we need to // create them based on the generation expressions. generateUpdateExprsForGeneratedColumns(update.child, generatedColumns, alignedUpdateExprs) } UpdateCommand( index, relation.catalogTable, update.child, alignedUpdateExprsAfterAddingGenerationExprs, update.condition) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/PreprocessTableWithDVs.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.DeltaParquetFileFormat._ import org.apache.spark.sql.delta.commands.DeletionVectorUtils.deletionVectorsReadable import org.apache.spark.sql.delta.files.{TahoeFileIndex, TahoeLogFileIndex} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Literal} import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project} import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation, LogicalRelationWithTable} import org.apache.spark.sql.execution.datasources.FileFormat.METADATA_NAME import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.types.StructType /** * Plan transformer to inject a filter that removes the rows marked as deleted according to * deletion vectors. For tables with no deletion vectors, this transformation has no effect. * * It modifies for plan for tables with deletion vectors as follows: * Before rule: -> Delta Scan (key, value). * - Here we are reading `key`, `value`` columns from the Delta table * After rule: * -> * Project(key, value) -> * Filter (__skip_row == 0) -> * Delta Scan (key, value, __skip_row) * - Here we insert a new column `__skip_row` in Delta scan. This value is populated by the * Parquet reader using the DV corresponding to the Parquet file read * (See [[DeltaParquetFileFormat]]) and it contains 0 if we want to keep the row. * - Filter created filters out rows with __skip_row equals to 0 * - And at the end we have a Project to keep the plan node output same as before the rule is * applied. */ trait PreprocessTableWithDVs extends SubqueryTransformerHelper { def preprocessTablesWithDVs(plan: LogicalPlan): LogicalPlan = { transformWithSubqueries(plan) { case ScanWithDeletionVectors(dvScan) => dvScan } } } object ScanWithDeletionVectors { def unapply(a: LogicalRelation): Option[LogicalPlan] = a match { case scan @ LogicalRelationWithTable( relation @ HadoopFsRelation( index: TahoeFileIndex, _, _, _, format: DeltaParquetFileFormat, _), _) => dvEnabledScanFor(scan, relation, format, index) case _ => None } def dvEnabledScanFor( scan: LogicalRelation, hadoopRelation: HadoopFsRelation, fileFormat: DeltaParquetFileFormat, index: TahoeFileIndex): Option[LogicalPlan] = { // If the table has no DVs enabled, no change needed if (!deletionVectorsReadable(index.protocol, index.metadata)) return None // See if the relation is already modified to include DV reads as part of // a previous invocation of this rule on this table if (fileFormat.hasTablePath) return None // See if any files actually have a DV. // IMPORTANT: Check this BEFORE requiring pinned snapshot: // 1. Tables can have DV feature enabled without any DV files (e.g., no DELETEs performed yet) // 2. Reading such tables doesn't require DV processing -> doesn't need pinned snapshots // 3. Unnecessary pinned snapshot requirements break legitimate use cases like transaction // read tracking with TahoeLogFileIndex (e.g., OptimisticTransaction.withNewTransaction) // 4. Performance: Avoids forcing expensive pinned snapshots when not needed val filesWithDVs = index .matchingFiles(partitionFilters = Seq(TrueLiteral), dataFilters = Seq(TrueLiteral)) .filter(_.deletionVector != null) if (filesWithDVs.isEmpty) return None // At this point, we know there are actual files with DVs, so we need a pinned snapshot. // TahoeLogFileIndex (non-pinned) cannot be used with deletion vectors as it may not // have a consistent view of the table state required for correct DV filtering. require(!index.isInstanceOf[TahoeLogFileIndex], "Cannot work with a non-pinned table snapshot of the TahoeFileIndex") // Get the list of columns in the output of the `LogicalRelation` we are // trying to modify. At the end of the plan, we need to return a // `LogicalRelation` that has the same output as this `LogicalRelation` val planOutput = scan.output val spark = SparkSession.getActiveSession.get val newScan = createScanWithSkipRowColumn(spark, scan, fileFormat, index, hadoopRelation) // On top of the scan add a filter that filters out the rows which have // skip row column value non-zero val rowIndexFilter = createRowIndexFilterNode(newScan) // Now add a project on top of the row index filter node to // remove the skip row column Some(Project(planOutput, rowIndexFilter)) } /** * Helper function that adds row_index column to _metadata if missing. */ private def addRowIndexIfMissing(attribute: AttributeReference): AttributeReference = { require(attribute.name == METADATA_NAME) val dataType = attribute.dataType.asInstanceOf[StructType] if (dataType.fieldNames.contains(ParquetFileFormat.ROW_INDEX)) return attribute val newDatatype = dataType.add(ParquetFileFormat.ROW_INDEX_FIELD) attribute.copy( dataType = newDatatype)(exprId = attribute.exprId, qualifier = attribute.qualifier) } /** * Helper method that creates a new `LogicalRelation` for existing scan that outputs * an extra column which indicates whether the row needs to be skipped or not. */ private def createScanWithSkipRowColumn( spark: SparkSession, inputScan: LogicalRelation, fileFormat: DeltaParquetFileFormat, tahoeFileIndex: TahoeFileIndex, hadoopFsRelation: HadoopFsRelation): LogicalRelation = { val useMetadataRowIndex = spark.sessionState.conf.getConf(DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX) // Create a new `LogicalRelation` that has modified `DeltaFileFormat` and output with an extra // column to indicate whether to skip the row or not // Add a column for SKIP_ROW to the base output. Value of 0 means the row needs be kept, any // other values mean the row needs be skipped. val skipRowField = IS_ROW_DELETED_STRUCT_FIELD val scanOutputWithMetadata = if (useMetadataRowIndex) { // When predicate pushdown is enabled, make sure the output contains metadata.row_index. if (inputScan.output.map(_.name).contains(METADATA_NAME)) { // If the scan already contains a metadata column without a row_index, add it. inputScan.output.collect { case a: AttributeReference if a.name == METADATA_NAME => addRowIndexIfMissing(a) case o => o } } else { inputScan.output :+ fileFormat.createFileMetadataCol() } } else { inputScan.output } val newScanOutput = scanOutputWithMetadata :+ AttributeReference(skipRowField.name, skipRowField.dataType)() // Data schema and scan schema could be different. The scan schema may contain additional // columns such as `_metadata.file_path` (metadata columns) which are populated in Spark scan // operator after the data is read from the underlying file reader. val newDataSchema = hadoopFsRelation.dataSchema.add(skipRowField) val newFileFormat = fileFormat.copyWithDVInfo( tablePath = tahoeFileIndex.path.toString, optimizationsEnabled = useMetadataRowIndex) val newRelation = hadoopFsRelation.copy( fileFormat = newFileFormat, dataSchema = newDataSchema)(hadoopFsRelation.sparkSession) // Create a new scan LogicalRelation inputScan.copy(relation = newRelation, output = newScanOutput) } private def createRowIndexFilterNode(newScan: LogicalRelation): Filter = { val skipRowColumnRefs = newScan.output.filter(_.name == IS_ROW_DELETED_COLUMN_NAME) require(skipRowColumnRefs.size == 1, s"Expected only one column with name=$IS_ROW_DELETED_COLUMN_NAME") val skipRowColumnRef = skipRowColumnRefs.head Filter(EqualTo(skipRowColumnRef, Literal(RowIndexFilter.KEEP_ROW_VALUE)), newScan) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/PreprocessTableWithDVsStrategy.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.planning.ScanOperation import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.execution.{SparkPlan, SparkStrategy} import org.apache.spark.sql.execution.datasources.{FileSourceStrategy, HadoopFsRelation, LogicalRelationWithTable} /** * Strategy to process tables with DVs and add the skip row column and filters. * * This strategy will apply all transformations needed to tables with DVs and delegate to * [[FileSourceStrategy]] to create the final plan. The DV filter will be the bottom-most filter in * the plan and so it will be pushed down to the FileSourceScanExec at the beginning of the filter * list. */ case class PreprocessTableWithDVsStrategy(session: SparkSession) extends SparkStrategy with PreprocessTableWithDVs { override def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match { case ScanOperation(_, _, _, _ @ LogicalRelationWithTable(_: HadoopFsRelation, _)) => val updatedPlan = preprocessTablesWithDVs(plan) FileSourceStrategy(updatedPlan) case _ => Nil } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/PreprocessTimeTravel.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.catalyst.TimeTravel import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.hadoop.fs.Path import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, ResolvedTable, UnresolvedRelation} import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation import org.apache.spark.sql.internal.SQLConf /** * Resolves the [[UnresolvedRelation]] in command 's child [[TimeTravel]]. * Currently Delta depends on Spark 3.2 which does not resolve the [[UnresolvedRelation]] * in [[TimeTravel]]. Once Delta upgrades to Spark 3.3, this code can be removed. * * TODO: refactoring this analysis using Spark's native [[TimeTravelRelation]] logical plan */ case class PreprocessTimeTravel(sparkSession: SparkSession) extends Rule[LogicalPlan] { override def conf: SQLConf = sparkSession.sessionState.conf override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperators { case _ @ RestoreTableStatement(tt @ TimeTravel(ur @ UnresolvedRelation(_, _, _), _, _, _)) => val sourceRelation = resolveTimeTravelTable(sparkSession, ur, "RESTORE") return RestoreTableStatement( TimeTravel( sourceRelation, tt.timestamp, tt.version, tt.creationSource)) case ct @ CloneTableStatement( tt @ TimeTravel(ur: UnresolvedRelation, _, _, _), _, _, _, _, _, _) => val sourceRelation = resolveTimeTravelTable(sparkSession, ur, "CLONE TABLE") ct.copy(source = TimeTravel( sourceRelation, tt.timestamp, tt.version, tt.creationSource)) } /** * Helper to resolve a [[TimeTravel]] logical plan to Delta DSv2 relation. */ private def resolveTimeTravelTable( sparkSession: SparkSession, ur: UnresolvedRelation, commandName: String): LogicalPlan = { // Since TimeTravel is a leaf node, the table relation within TimeTravel won't be resolved // automatically by the Apache Spark analyzer rule `ResolveRelations`. // Thus, we need to explicitly use the rule `ResolveRelations` to table resolution here. EliminateSubqueryAliases(sparkSession.sessionState.analyzer.ResolveRelations(ur)) match { case _: View => // If the identifier is a view, throw not supported error throw DeltaErrors.notADeltaTableException(commandName) case tableRelation if tableRelation.resolved => tableRelation case _ => // If the identifier doesn't exist as a table, try resolving it as a path table. ResolveDeltaPathTable.resolveAsPathTableRelation(sparkSession, ur).getOrElse { ur.tableNotFound(ur.multipartIdentifier) } } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/ProtocolMetadataAdapter.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.SparkSession import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{StructField, StructType} /** * Abstraction layer over Delta Protocol and Metadata hide implementation details of * Protocol and Metadata classes, enabling DeltaParquetFileFormat to be reused without depending on * specific action class implementations. * This helps delta kernel based connector reusing DeltaParquetFileFormat. */ trait ProtocolMetadataAdapter { /** * Returns the column mapping mode for this table. */ def columnMappingMode: DeltaColumnMappingMode /** * Returns the logical schema of the table. */ def getReferenceSchema: StructType /** * Returns whether Row IDs(Row tracking) are enabled on this table. */ def isRowIdEnabled: Boolean /** * Returns whether Deletion Vectors are readable on this table. */ def isDeletionVectorReadable: Boolean /** * Returns whether any version of IcebergCompat is enabled on this table. */ def isIcebergCompatAnyEnabled: Boolean /** * Returns whether IcebergCompat is enabled at or above the specified version. * @param version The IcebergCompat version to check (e.g., 2, 3) */ def isIcebergCompatGeqEnabled(version: Int): Boolean /** * Asserts that the table is readable given the current configuration. * Throws an exception if the table cannot be read. * * @param sparkSession The current Spark session */ def assertTableReadable(sparkSession: SparkSession): Unit /** * Creates the metadata struct fields for row tracking. * * @param nullableRowTrackingConstantFields whether constant fields should be nullable * @param nullableRowTrackingGeneratedFields whether generated fields should be nullable * @return metadata fields for row tracking (_metadata.row_id, _metadata.base_row_id, etc.) */ def createRowTrackingMetadataFields( nullableRowTrackingConstantFields: Boolean, nullableRowTrackingGeneratedFields: Boolean): Iterable[StructField] } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/ProtocolMetadataAdapterV1.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.actions.{Metadata, Protocol} import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.SparkSession import org.apache.spark.sql.types.{StructField, StructType} /** * Implementation of ProtocolMetadataAdapter for delta-spark v1 Protocol and Metadata. * * This class adapts the existing delta-spark Protocol and Metadata actions to the * ProtocolMetadataAdapter interface, enabling code reuse in DeltaParquetFileFormat. */ case class ProtocolMetadataAdapterV1( protocol: Protocol, metadata: Metadata) extends ProtocolMetadataAdapter { override def columnMappingMode: DeltaColumnMappingMode = metadata.columnMappingMode override def getReferenceSchema: StructType = metadata.schema override def isRowIdEnabled: Boolean = RowId.isEnabled(protocol, metadata) override def isDeletionVectorReadable: Boolean = DeletionVectorUtils.deletionVectorsReadable(protocol, metadata) override def isIcebergCompatAnyEnabled: Boolean = IcebergCompat.isAnyEnabled(metadata) override def isIcebergCompatGeqEnabled(version: Int): Boolean = IcebergCompat.isGeqEnabled(metadata, version) override def assertTableReadable(sparkSession: SparkSession): Unit = { TypeWidening.assertTableReadable(sparkSession.sessionState.conf, protocol, metadata) } override def createRowTrackingMetadataFields( nullableRowTrackingConstantFields: Boolean, nullableRowTrackingGeneratedFields: Boolean): Iterable[StructField] = { RowTracking.createMetadataStructFields( protocol, metadata, nullableConstantFields = nullableRowTrackingConstantFields, nullableGeneratedFields = nullableRowTrackingGeneratedFields) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/ProvidesUniFormConverters.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.lang.reflect.InvocationTargetException import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.spark.util.Utils trait ProvidesUniFormConverters { self: DeltaLog => /** * Helper trait to instantiate the icebergConverter member variable of the [[DeltaLog]]. We do * this through reflection so that delta-spark doesn't have a compile-time dependency on the * shaded iceberg module. */ protected lazy val _icebergConverter: UniversalFormatConverter = try { val clazz = Utils.classForName("org.apache.spark.sql.delta.icebergShaded.IcebergConverter") clazz.getConstructor().newInstance() } catch { case e: ClassNotFoundException => logError(log"Failed to find Iceberg converter class", e) throw DeltaErrors.icebergClassMissing(spark.sparkContext.getConf, e) case e: InvocationTargetException => logError(log"Got error when creating an Iceberg converter", e) // The better error is within the cause throw ExceptionUtils.getRootCause(e) } protected lazy val _hudiConverter: UniversalFormatConverter = try { val clazz = Utils.classForName("org.apache.spark.sql.delta.hudi.HudiConverter") clazz.getConstructor().newInstance() } catch { case e: ClassNotFoundException => logError(log"Failed to find Hudi converter class", e) throw DeltaErrors.hudiClassMissing(spark.sparkContext.getConf, e) case e: InvocationTargetException => logError(log"Got error when creating an Hudi converter", e) // The better error is within the cause throw ExceptionUtils.getRootCause(e) } /** Visible for tests (to be able to mock). */ private[delta] var testIcebergConverter: Option[UniversalFormatConverter] = None private[delta] var testHudiConverter: Option[UniversalFormatConverter] = None def icebergConverter: UniversalFormatConverter = testIcebergConverter.getOrElse(_icebergConverter) def hudiConverter: UniversalFormatConverter = testHudiConverter.getOrElse(_hudiConverter) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/ResolveDeltaMergeInto.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.Locale import scala.collection.mutable import org.apache.spark.sql.delta.schema.SchemaMergingUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.sources.DeltaSQLConf.AllowAutomaticWideningMode import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.analysis._ import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{AtomicType, StructField, StructType} /** * Implements logic to resolve conditions and actions in MERGE clauses and handles schema evolution. */ object ResolveDeltaMergeInto { type ResolveExpressionsFn = (Seq[Expression], Seq[LogicalPlan]) => Seq[Expression] def throwIfNotResolved( expr: Expression, plans: Seq[LogicalPlan], mergeClauseTypeStr: String): Unit = { for (a <- expr.flatMap(_.references).filterNot(_.resolved)) { // Note: This will throw error only on unresolved attribute issues, // not other resolution errors like mismatched data types. val cols = plans.flatMap(_.output).map(_.sql).mkString(", ") throw new DeltaAnalysisException( errorClass = "DELTA_MERGE_UNRESOLVED_EXPRESSION", messageParameters = Array(a.sql, mergeClauseTypeStr, cols), origin = Some(a.origin)) } } /** * Resolves expressions against given plans or fail using given message. It makes a best-effort * attempt to throw specific error messages on which part of the query has a problem. */ def resolveOrFail( resolveExprsFn: ResolveExpressionsFn, exprs: Seq[Expression], plansToResolveExprs: Seq[LogicalPlan], mergeClauseTypeStr: String): Seq[Expression] = { val resolvedExprs = resolveExprsFn(exprs, plansToResolveExprs) resolvedExprs.foreach(throwIfNotResolved(_, plansToResolveExprs, mergeClauseTypeStr)) resolvedExprs } /** * Convenience wrapper around `resolveOrFail()` when resolving a single expression. */ def resolveSingleExprOrFail( resolveExprsFn: ResolveExpressionsFn, expr: Expression, plansToResolveExpr: Seq[LogicalPlan], mergeClauseTypeStr: String): Expression = { resolveOrFail(resolveExprsFn, Seq(expr), plansToResolveExpr, mergeClauseTypeStr).head } /** * Computes the target schema after applying schema evolution. * * When schema evolution is enabled, this method adds new columns or nested fields from the source * that are assigned to in merge actions. It filters the source schema to retain only referenced * fields, then merges this with the target schema. * * @param canEvolveSchema whether schema evolution is enabled * @param resolvedMatchedClauses resolved MATCHED clauses * @param resolvedNotMatchedClauses resolved NOT MATCHED clauses * @param target the target table plan * @param source the source data plan * @param conf SQL configuration * @return the evolved target schema (or original target schema if evolution is disabled) */ private def computePostEvolutionTargetSchema( canEvolveSchema: Boolean, resolvedMatchedClauses: Seq[DeltaMergeIntoClause], resolvedNotMatchedClauses: Seq[DeltaMergeIntoClause], target: LogicalPlan, source: LogicalPlan, conf: SQLConf): StructType = { if (canEvolveSchema) { // When schema evolution is enabled, add to the target table new columns or nested fields that // are assigned to in merge actions and not already part of the target schema. This is done by // collecting all assignments from merge actions and using them to filter out the source // schema before merging it with the target schema. We don't consider NOT MATCHED BY SOURCE // clauses since these can't by definition reference source columns and thus can't introduce // new columns in the target schema. val actions = (resolvedMatchedClauses ++ resolvedNotMatchedClauses).flatMap(_.actions) val assignments = actions.collect { case a: DeltaMergeAction => a.targetColNameParts } val containsStarAction = actions.exists { case _: UnresolvedStar => true case _ => false } // Filter the source schema to retain only fields that are referenced by at least one merge // clause, then merge this schema with the target to give the final schema. def filterSchema(sourceSchema: StructType, basePath: Seq[String]): StructType = StructType(sourceSchema.flatMap { field => val fieldPath = basePath :+ field.name // Helper method to check if a given field path is a prefix of another path. Delegates // equality to conf.resolver to correctly handle case sensitivity. def isPrefix(prefix: Seq[String], path: Seq[String]): Boolean = prefix.length <= path.length && prefix.zip(path).forall { case (prefixNamePart, pathNamePart) => conf.resolver(prefixNamePart, pathNamePart) } // Helper method to check if a given field path is equal to another path. def isEqual(path1: Seq[String], path2: Seq[String]): Boolean = path1.length == path2.length && isPrefix(path1, path2) field.dataType match { // Specifically assigned to in one clause: always keep, including all nested attributes case _ if assignments.exists(isEqual(_, fieldPath)) => Some(field) // If this is a struct and one of the children is being assigned to in a merge clause, // keep it and continue filtering children. case struct: StructType if assignments.exists(isPrefix(fieldPath, _)) => Some(field.copy(dataType = filterSchema(struct, fieldPath))) // The field isn't assigned to directly or indirectly (i.e. its children) in any non-* // clause. Check if it should be kept with any * action. case struct: StructType if containsStarAction => Some(field.copy(dataType = filterSchema(struct, fieldPath))) case _ if containsStarAction => Some(field) // The field and its children are not assigned to in any * or non-* action, drop it. case _ => None } }) val migrationSchema = filterSchema(source.schema, Seq.empty) val typeWideningMode = target.collectFirst { case DeltaTable(index) if TypeWidening.isEnabled(index.protocol, index.metadata) => TypeWideningMode.TypeEvolution( uniformIcebergCompatibleOnly = UniversalFormat.icebergEnabled(index.metadata), allowAutomaticWidening = AllowAutomaticWideningMode.fromConf(conf)) }.getOrElse(TypeWideningMode.NoTypeWidening) // The implicit conversions flag allows any type to be merged from source to target if Spark // SQL considers the source type implicitly castable to the target. Normally, mergeSchemas // enforces Parquet-level write compatibility, which would mean an INT source can't be merged // into a LONG target. SchemaMergingUtils.mergeSchemas( target.schema, migrationSchema, allowImplicitConversions = true, typeWideningMode = typeWideningMode ) } else { target.schema } } /** * Resolves a merge clause by resolving its actions and condition. * * Actions are split into two groups: * (1) DeltaMergeActions (like `UPDATE SET x = a, y = b`): resolved with DeltaMergeActionResolver * (2) Star expressions (like `UPDATE SET *` or `INSERT *`): resolved with resolveStar(Except) * * @param clause the merge clause to resolve (MATCHED UPDATE, NOT MATCHED INSERT, etc.) * @param plansToResolveAction the logical plans to use for resolving action expressions * @param target the target table plan * @param source the source data plan * @param canEvolveSchema whether schema evolution is enabled * @param mergeActionResolver resolver for DeltaMergeAction expressions * @param resolveExprsFn function to resolve expressions * @param conf SQL configuration * @return the resolved clause */ private def resolveClause[T <: DeltaMergeIntoClause]( clause: T, plansToResolveAction: Seq[LogicalPlan], target: LogicalPlan, source: LogicalPlan, canEvolveSchema: Boolean, mergeActionResolver: DeltaMergeActionResolverBase, resolveExprsFn: ResolveExpressionsFn, conf: SQLConf): T = { val clauseType = clause.clauseType.toUpperCase(Locale.ROOT) val mergeClauseTypeStr = s"$clauseType clause" // We split the actions of a clause (expressions) into two mutually exclusive groups: // 1) DeltaMergeActions and 2) everything else (UnresolvedStar). // The DeltaMergeActions can be resolved already or unresolved at this point. // Unresolved DeltaMergeActions correspond to actions like `UPDATE SET x = a, y = b` or // `INSERT (x, y) VALUES (a, b)`. // By the end of this function, every action needs to be transformed into a resolved // DeltaMergeAction. We handle the DeltaMergeActions separately in [[DeltaMergeActionResolver]] // as we have different strategies to enable better analysis performance. val (deltaMergeActions, allOtherExpressions) = clause.actions.partition { case _: DeltaMergeAction => true case _ => false } assert( deltaMergeActions.isEmpty || allOtherExpressions.isEmpty, s"Cannot have DeltaMergeActions combined with other expressions in a $mergeClauseTypeStr") val shouldTryUnresolvedTargetExprOnSource = clause match { case _: DeltaMergeIntoMatchedUpdateClause | _: DeltaMergeIntoNotMatchedClause => canEvolveSchema case _ => false } val resolvedDeltaMergeActions: Seq[DeltaMergeAction] = mergeActionResolver.resolve( mergeClauseTypeStr, plansToResolveAction, shouldTryUnresolvedTargetExprOnSource, deltaMergeActions.map(_.asInstanceOf[DeltaMergeAction]) ) val resolvedOtherExpressions: Seq[DeltaMergeAction] = allOtherExpressions.flatMap { action => action match { // For actions like `UPDATE SET *` or `INSERT *` case _: UnresolvedStar => resolveStar( clause, target, source, canEvolveSchema, resolveExprsFn, mergeClauseTypeStr, conf) case _ => action.failAnalysis("INTERNAL_ERROR", Map("message" -> s"Unexpected action expression '$action' in clause $clause")) } } val resolvedCondition = clause.condition.map { condExpr => resolveSingleExprOrFail( resolveExprsFn, condExpr, plansToResolveAction, mergeClauseTypeStr = s"$clauseType condition") } clause.makeCopy(Array(resolvedCondition, resolvedDeltaMergeActions ++ resolvedOtherExpressions )).asInstanceOf[T] } /** * Resolves UnresolvedStar (`*`) for `UPDATE SET *` or `INSERT *` actions. * * When schema evolution is disabled: expands `*` for all target columns * * When schema evolution is enabled: * - For INSERT clauses: expands `*` for all source columns * - For UPDATE clauses: expands `*` for all source leaf fields * * @param clause the merge clause being resolved (INSERT or UPDATE) * @param target the target table plan * @param source the source data plan * @param canEvolveSchema whether schema evolution is enabled * @param resolveExprsFn function to resolve expressions * @param mergeClauseTypeStr string description of the clause type for error messages * @param conf SQL configuration * @return sequence of resolved DeltaMergeActions */ private def resolveStar( clause: DeltaMergeIntoClause, target: LogicalPlan, source: LogicalPlan, canEvolveSchema: Boolean, resolveExprsFn: ResolveExpressionsFn, mergeClauseTypeStr: String, conf: SQLConf): Seq[DeltaMergeAction] = { if (!canEvolveSchema) { // Expand `*` into seq of [ `columnName = sourceColumnBySameName` ] for every target // column name. The target columns do not need resolution. The right hand side // expression (i.e. sourceColumnBySameName) needs to be resolved only by the source plan. val unresolvedExprs = target.output.map { attr => UnresolvedAttribute.quotedString(s"`${attr.name}`") } val resolvedExprs = resolveOrFail( resolveExprsFn = resolveExprsFn, exprs = unresolvedExprs, plansToResolveExprs = Seq(source), mergeClauseTypeStr = mergeClauseTypeStr) (resolvedExprs, target.output.map(_.name)) .zipped .map { (resolvedExpr, targetColName) => DeltaMergeAction( targetColNameParts = Seq(targetColName), expr = resolvedExpr, // Schema evolution is disabled, so the action expression should already be aligned // with the target schema. targetOnlyStructFieldBehavior = TargetOnlyStructFieldBehavior.TARGET_ALIGNED, targetColNameResolved = true) } } else { clause match { case _: DeltaMergeIntoNotMatchedInsertClause => // Expand `*` into seq of [ `columnName = sourceColumnBySameName` ] for every source // column name. Target columns not present in the source will be filled in // with null later. source.output.map { attr => DeltaMergeAction( targetColNameParts = Seq(attr.name), expr = attr, // INSERT sets target-only struct fields to null since there is no existing target // row to preserve values from. targetOnlyStructFieldBehavior = TargetOnlyStructFieldBehavior.NULLIFY, targetColNameResolved = true) } case _: DeltaMergeIntoMatchedUpdateClause => // Expand `*` into seq of [ `columnName = sourceColumnBySameName` ] for every source // column name. Target columns not present in the source will be filled in with // no-op actions later. if (UpdateExpressionsSupport.isUpdateStarPreserveNullSourceStructsEnabled(conf)) { // Expand `*` into column-level actions to fix null expansion in UPDATE *, i.e. a null // source struct is expanded into a non-null struct with all fields set to null. source.output.map { attr => DeltaMergeAction( targetColNameParts = Seq(attr.name), expr = attr, // Preserve the original value of target-only struct fields to be consistent with // the behavior without the null expansion fix. This avoids the breaking change that // causes data loss. targetOnlyStructFieldBehavior = TargetOnlyStructFieldBehavior.PRESERVE, targetColNameResolved = true) } } else { // Nested columns are unfolded to accommodate the case where a source struct has a // subset of the nested columns in the target. If a source struct (a, b) is writing // into a target (a, b, c), the final struct after filling in the no-op actions will // be (s.a, s.b, t.c). getLeafActionsForSchema(source.schema, Seq.empty, source, conf) } } } } /** * Returns the sequence of [[DeltaMergeActions]] corresponding to * [ `columnName = sourceColumnBySameName` ] for every column name in the schema. Nested * columns are unfolded to create an assignment for each leaf. * * @param currSchema: schema to generate DeltaMergeAction for every 'leaf' * @param qualifier: used to recurse to leaves; represents the qualifier of the current schema * @param source: source plan to resolve expressions against * @param conf: SQL configuration * @return seq of DeltaMergeActions corresponding to columnName = sourceColumnName updates */ private def getLeafActionsForSchema( currSchema: StructType, qualifier: Seq[String], source: LogicalPlan, conf: SQLConf): Seq[DeltaMergeAction] = { currSchema.flatMap { case StructField(name, struct: StructType, _, _) => getLeafActionsForSchema(struct, qualifier :+ name, source, conf) case StructField(name, _, _, _) => val nameParts = qualifier :+ name val sourceExpr = source.resolve(nameParts, conf.resolver).getOrElse { // if we use getActions to expand target columns, this will fail on target columns not // present in the source throw DeltaErrors.cannotResolveSourceColumnException(nameParts) } Seq( DeltaMergeAction( targetColNameParts = nameParts, expr = sourceExpr, // Leaf-level operations are aligned with the target schema naturally. targetOnlyStructFieldBehavior = TargetOnlyStructFieldBehavior.TARGET_ALIGNED, targetColNameResolved = true)) } } def resolveReferencesAndSchema( merge: DeltaMergeInto, conf: SQLConf)(resolveExprsFn: ResolveExpressionsFn): DeltaMergeInto = { val DeltaMergeInto( target, source, condition, matchedClauses, notMatchedClauses, notMatchedBySourceClauses, withSchemaEvolution, _) = merge val canEvolveSchema = withSchemaEvolution || conf.getConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE) val mergeActionResolver = if (conf.getConf(DeltaSQLConf.DELTA_MERGE_ANALYSIS_BATCH_RESOLUTION)) { new BatchedDeltaMergeActionResolver(target, source, conf, resolveExprsFn) } else { new IndividualDeltaMergeActionResolver(target, source, conf, resolveExprsFn) } // We must do manual resolution as the expressions in different clauses of the MERGE have // visibility of the source, the target or both. val resolvedCond = resolveSingleExprOrFail( resolveExprsFn, expr = condition, plansToResolveExpr = Seq(target, source), mergeClauseTypeStr = "search condition") val resolvedMatchedClauses = matchedClauses.map { resolveClause( _, Seq(target, source), target, source, canEvolveSchema, mergeActionResolver, resolveExprsFn, conf) } val resolvedNotMatchedClauses = notMatchedClauses.map { resolveClause( _, Seq(source), target, source, canEvolveSchema, mergeActionResolver, resolveExprsFn, conf) } val resolvedNotMatchedBySourceClauses = notMatchedBySourceClauses.map { resolveClause( _, Seq(target), target, source, canEvolveSchema, mergeActionResolver, resolveExprsFn, conf) } val postEvolutionTargetSchema = computePostEvolutionTargetSchema( canEvolveSchema, resolvedMatchedClauses, resolvedNotMatchedClauses, target, source, conf) val resolvedMerge = DeltaMergeInto( target, source, resolvedCond, resolvedMatchedClauses, resolvedNotMatchedClauses, resolvedNotMatchedBySourceClauses, withSchemaEvolution = canEvolveSchema, finalSchema = Some(postEvolutionTargetSchema)) // Its possible that pre-resolved expressions (e.g. `sourceDF("key") = targetDF("key")`) have // attribute references that are not present in the output attributes of the children (i.e., // incorrect DataFrame was used in the `df("col")` form). if (resolvedMerge.missingInput.nonEmpty) { val missingAttributes = resolvedMerge.missingInput.mkString(",") val input = resolvedMerge.inputSet.mkString(",") throw new DeltaAnalysisException( errorClass = "DELTA_MERGE_RESOLVED_ATTRIBUTE_MISSING_FROM_INPUT", messageParameters = Array(missingAttributes, input, resolvedMerge.simpleString(SQLConf.get.maxToStringFields)), origin = Some(resolvedMerge.origin) ) } resolvedMerge } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/ResolveDeltaPathTable.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.hadoop.fs.Path import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.analysis.{ResolvedTable, UnresolvedRelation, UnresolvedTable} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.{CatalogHelper, MultipartIdentifierHelper} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation /** * Replaces [[UnresolvedTable]]s if the plan is for direct query on files. */ case class ResolveDeltaPathTable(sparkSession: SparkSession) extends Rule[LogicalPlan] { override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperators { case u: UnresolvedTable => ResolveDeltaPathTable .resolveAsPathTable(sparkSession, u.multipartIdentifier) .getOrElse(u) } } object ResolveDeltaPathTable { /** * Try resolving the input table as a Path table. * If the path table exists, return a [[DataSourceV2Relation]] instance. Otherwise, return None. */ def resolveAsPathTableRelation( sparkSession: SparkSession, u: UnresolvedRelation) : Option[DataSourceV2Relation] = { resolveAsPathTable(sparkSession, u.multipartIdentifier) .map { resolvedTable => DataSourceV2Relation.create( resolvedTable.table, Some(resolvedTable.catalog), Some(resolvedTable.identifier)) } } /** * Try resolving the input table as a Path table. * If the path table exists, return a [[ResolvedTable]] instance. Otherwise, return None. */ private def resolveAsPathTable( sparkSession: SparkSession, multipartIdentifier: Seq[String], options: Map[String, String] = Map.empty): Option[ResolvedTable] = { val sessionState = sparkSession.sessionState if (!sessionState.conf.runSQLonFile || multipartIdentifier.size != 2) { return None } val tableId = multipartIdentifier.asTableIdentifier if (!DeltaTableUtils.isValidPath(tableId)) { return None } val deltaTableV2 = DeltaTableV2(sparkSession, new Path(tableId.table), options = options) val sessionCatalog = sessionState.catalogManager.v2SessionCatalog.asTableCatalog Some(ResolvedTable.create(sessionCatalog, multipartIdentifier.asIdentifier, deltaTableV2)) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/ResolveDeltaTableWithPartitionFilters.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.files.TahoeLogFileIndex import org.apache.spark.sql.catalyst.expressions.And import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan} /** * Pull out the partition filter that may be part of the FileIndex. This can happen when someone * queries a Delta table such as spark.read.format("delta").load("/some/table/partition=2") */ object ResolveDeltaTableWithPartitionFilters { def unapply(plan: LogicalPlan): Option[LogicalPlan] = plan match { case relation @ DeltaTable(index: TahoeLogFileIndex) if index.partitionFilters.nonEmpty => val result = Filter( index.partitionFilters.reduce(And), DeltaTableUtils.replaceFileIndex(relation, index.copy(partitionFilters = Nil)) ) Some(result) case _ => None } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/RowCommitVersion.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.DeltaColumnMapping.PARQUET_FIELD_ID_METADATA_KEY import org.apache.spark.sql.delta.actions.{Metadata, Protocol} import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.spark.sql.{types, Column, DataFrame} import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, FileSourceGeneratedMetadataStructField} import org.apache.spark.sql.catalyst.types.DataTypeUtils import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.functions.lit import org.apache.spark.sql.types.{DataType, LongType, MetadataBuilder, StructField} object RowCommitVersion { val METADATA_STRUCT_FIELD_NAME = "row_commit_version" val QUALIFIED_COLUMN_NAME = s"${FileFormat.METADATA_NAME}.$METADATA_STRUCT_FIELD_NAME" def createMetadataStructField( protocol: Protocol, metadata: Metadata, nullable: Boolean = false, shouldSetIcebergReservedFieldId: Boolean): Option[StructField] = MaterializedRowCommitVersion.getMaterializedColumnName(protocol, metadata) .map(MetadataStructField(_, nullable, shouldSetIcebergReservedFieldId)) /** * Add a new column to `dataFrame` that has the name of the materialized Row Commit Version column * and holds Row Commit Versions. The column also is tagged with the appropriate metadata such * that it can be used to write materialized Row Commit Versions. */ private[delta] def preserveRowCommitVersions( dataFrame: DataFrame, snapshot: SnapshotDescriptor): DataFrame = { if (!RowTracking.isEnabled(snapshot.protocol, snapshot.metadata)) { return dataFrame } val materializedColumnName = MaterializedRowCommitVersion.getMaterializedColumnNameOrThrow( snapshot.protocol, snapshot.metadata, snapshot.deltaLog.unsafeVolatileTableId) val rowCommitVersionColumn = DeltaTableUtils.getFileMetadataColumn(dataFrame).getField(METADATA_STRUCT_FIELD_NAME) val shouldSetIcebergReservedFieldId = IcebergCompat.isGeqEnabled(snapshot.metadata, 3) preserveRowCommitVersionsUnsafe( dataFrame, materializedColumnName, rowCommitVersionColumn, shouldSetIcebergReservedFieldId ) } private[delta] def preserveRowCommitVersionsUnsafe( dataFrame: DataFrame, materializedColumnName: String, rowCommitVersionColumn: Column, shouldSetIcebergReservedFieldId: Boolean): DataFrame = { dataFrame .withColumn(materializedColumnName, rowCommitVersionColumn) .withMetadata( materializedColumnName, MetadataStructField.metadata(materializedColumnName, shouldSetIcebergReservedFieldId)) } object MetadataStructField { private val METADATA_COL_ATTR_KEY = "__row_commit_version_metadata_col" def apply( materializedColumnName: String, nullable: Boolean = false, shouldSetIcebergReservedFieldId: Boolean): StructField = StructField( METADATA_STRUCT_FIELD_NAME, LongType, // The Row commit version field is used to read the materialized Row commit version value // which is nullable. The actual Row commit version expression is created using a projection // injected before the optimizer pass by the [[GenerateRowIDs] rule at which point the Row // commit version field is non-nullable. nullable, metadata = metadata(materializedColumnName, shouldSetIcebergReservedFieldId)) def unapply(field: StructField): Option[StructField] = Option.when(isValid(field.dataType, field.metadata))(field) def metadata( materializedColumnName: String, shouldSetIcebergReservedFieldId: Boolean): types.Metadata = { val metadataBuilder = new MetadataBuilder() .withMetadata( FileSourceGeneratedMetadataStructField.metadata( METADATA_STRUCT_FIELD_NAME, materializedColumnName)) .putBoolean(METADATA_COL_ATTR_KEY, value = true) // If IcebergCompatV3 or higher is enabled, assign the field ID of Delta // Row commit version column to match the reserved `_last_updated_sequence_number` // field defined in the Iceberg spec. // This ensures that Iceberg can recognize and track the same column for row lineage purposes. if (shouldSetIcebergReservedFieldId) { metadataBuilder.putLong( PARQUET_FIELD_ID_METADATA_KEY, IcebergConstants.ICEBERG_ROW_TRACKING_LAST_UPDATED_SEQUENCE_NUMBER_FIELD_ID ) } metadataBuilder.build() } /** Return true if the column is a Row Commit Version column. */ def isRowCommitVersionColumn(structField: StructField): Boolean = isValid(structField.dataType, structField.metadata) private[delta] def isValid(dataType: DataType, metadata: types.Metadata): Boolean = { FileSourceGeneratedMetadataStructField.isValid(dataType, metadata) && metadata.contains(METADATA_COL_ATTR_KEY) && metadata.getBoolean(METADATA_COL_ATTR_KEY) } } def columnMetadata( materializedColumnName: String, shouldSetIcebergReservedFieldId: Boolean): types.Metadata = MetadataStructField.metadata(materializedColumnName, shouldSetIcebergReservedFieldId) object MetadataAttribute { def apply( materializedColumnName: String, shouldSetIcebergReservedFieldId: Boolean): AttributeReference = DataTypeUtils .toAttribute( MetadataStructField( materializedColumnName, shouldSetIcebergReservedFieldId = shouldSetIcebergReservedFieldId )) .withName(materializedColumnName) def unapply(attr: Attribute): Option[Attribute] = if (isRowCommitVersionColumn(attr)) Some(attr) else None /** Return true if the column is a Row Commit Version column. */ def isRowCommitVersionColumn(attr: Attribute): Boolean = MetadataStructField.isValid(attr.dataType, attr.metadata) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/RowId.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.DeltaColumnMapping.PARQUET_FIELD_ID_METADATA_KEY import org.apache.spark.sql.delta.actions.{Action, AddFile, DomainMetadata, Metadata, Protocol} import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils.propertyKey import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.spark.sql.{Column, DataFrame, SparkSession} import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, FileSourceConstantMetadataStructField, FileSourceGeneratedMetadataStructField} import org.apache.spark.sql.catalyst.types.DataTypeUtils import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.types import org.apache.spark.sql.types.{DataType, LongType, MetadataBuilder, StructField} /** * Collection of helpers to handle Row IDs. * * This file includes the following Row ID features: * - Enabling Row IDs using table feature and table property. * - Assigning fresh Row IDs. * - Reading back Row IDs. * - Preserving stable Row IDs. */ object RowId { /** * Metadata domain for the high water mark stored using a [[DomainMetadata]] action. */ case class RowTrackingMetadataDomain(rowIdHighWaterMark: Long) extends JsonMetadataDomain[RowTrackingMetadataDomain] { override val domainName: String = RowTrackingMetadataDomain.domainName } object RowTrackingMetadataDomain extends JsonMetadataDomainUtils[RowTrackingMetadataDomain] { override protected val domainName = "delta.rowTracking" def unapply(action: Action): Option[RowTrackingMetadataDomain] = action match { case d: DomainMetadata if d.domain == domainName => Some(fromJsonConfiguration(d)) case _ => None } } val MISSING_HIGH_WATER_MARK: Long = -1L /** * Returns whether the protocol version supports the Row ID table feature. Whenever Row IDs are * supported, fresh Row IDs must be assigned to all newly committed files, even when Row IDs are * disabled in the current table version. */ def isSupported(protocol: Protocol): Boolean = RowTracking.isSupported(protocol) /** * Returns whether Row IDs are enabled on this table version. Checks that Row IDs are supported, * which is a pre-requisite for enabling Row IDs, throws an error if not. */ def isEnabled(protocol: Protocol, metadata: Metadata): Boolean = { val isEnabled = DeltaConfigs.ROW_TRACKING_ENABLED.fromMetaData(metadata) if (isEnabled && !isSupported(protocol)) { throw new IllegalStateException( s"Table property '${DeltaConfigs.ROW_TRACKING_ENABLED.key}' is " + s"set on the table but this table version doesn't support table feature " + s"'${propertyKey(RowTrackingFeature)}'.") } isEnabled } /** * Assigns fresh row IDs to all AddFiles inside `actions` that do not have row IDs yet and emits * a [[RowIdHighWaterMark]] action with the new high-water mark. */ private[delta] def assignFreshRowIds( spark: SparkSession, protocol: Protocol, snapshot: Snapshot, actions: Iterator[Action], operation: DeltaOperations.Operation): Iterator[Action] = { if (!isSupported(protocol)) return actions def metadataDomainSetException = new IllegalStateException( "Manually setting the Row ID high water mark is not allowed") // Do not propagate row IDs if generation is suspended. if (RowTracking.isSuspended(spark, snapshot.metadata)) { return actions.map { case a: AddFile if a.baseRowId.isDefined => a.copy(baseRowId = None) case d: DomainMetadata if RowTrackingMetadataDomain.isSameDomain(d) => throw metadataDomainSetException case o => o } } val oldHighWatermark = extractHighWatermark(snapshot).getOrElse(MISSING_HIGH_WATER_MARK) var newHighWatermark = oldHighWatermark val actionsWithFreshRowIds = actions.map { case a: AddFile if a.baseRowId.isEmpty => val baseRowId = newHighWatermark + 1L newHighWatermark += a.numPhysicalRecords.getOrElse { operation match { case op: DeltaOperations.Clone => throw DeltaErrors.cloneWithRowTrackingWithoutStats() case _ => throw DeltaErrors.rowIdAssignmentWithoutStats } } a.copy(baseRowId = Some(baseRowId)) case d: DomainMetadata if RowTrackingMetadataDomain.isSameDomain(d) => throw metadataDomainSetException case other => other } val newHighWatermarkAction: Iterator[Action] = new Iterator[Action] { // Iterators are lazy, so the first call to `hasNext` won't happen until after we // exhaust the remapped actions iterator. At that point, the watermark (changed or not) // decides whether the iterator is empty or infinite; take(1) below to bound it. override def hasNext: Boolean = newHighWatermark != oldHighWatermark override def next(): Action = RowTrackingMetadataDomain(newHighWatermark).toDomainMetadata } actionsWithFreshRowIds ++ newHighWatermarkAction.take(1) } /** * Extracts the high watermark of row IDs from a snapshot. */ private[delta] def extractHighWatermark(snapshot: Snapshot): Option[Long] = if (isSupported(snapshot.protocol)) { RowTrackingMetadataDomain.fromSnapshot(snapshot).map(_.rowIdHighWaterMark) } else { None } /** Base Row ID column name */ val BASE_ROW_ID = "base_row_id" /* * A specialization of [[FileSourceConstantMetadataStructField]] used to represent base RowId * columns. */ object BaseRowIdMetadataStructField { private val BASE_ROW_ID_METADATA_COL_ATTR_KEY = s"__base_row_id_metadata_col" def metadata: types.Metadata = new MetadataBuilder() .withMetadata(FileSourceConstantMetadataStructField.metadata(BASE_ROW_ID)) .putBoolean(BASE_ROW_ID_METADATA_COL_ATTR_KEY, value = true) .build() def apply(nullable: Boolean): StructField = StructField( BASE_ROW_ID, LongType, nullable, metadata = metadata) def unapply(field: StructField): Option[StructField] = Some(field).filter(isBaseRowIdColumn) /** Return true if the column is a base Row ID column. */ def isBaseRowIdColumn(structField: StructField): Boolean = isValid(structField.dataType, structField.metadata) def isValid(dataType: DataType, metadata: types.Metadata): Boolean = { FileSourceConstantMetadataStructField.isValid(dataType, metadata) && metadata.contains(BASE_ROW_ID_METADATA_COL_ATTR_KEY) && metadata.getBoolean(BASE_ROW_ID_METADATA_COL_ATTR_KEY) } } /** * The field readers can use to access the base row id column. */ def createBaseRowIdField( protocol: Protocol, metadata: Metadata, nullable: Boolean): Option[StructField] = Option.when(RowId.isEnabled(protocol, metadata)) { BaseRowIdMetadataStructField(nullable) } /** Row ID column name */ val ROW_ID = "row_id" val QUALIFIED_COLUMN_NAME = s"${FileFormat.METADATA_NAME}.${ROW_ID}" /** Column metadata to be used in conjunction [[QUALIFIED_COLUMN_NAME]] to mark row id columns */ def columnMetadata( materializedColumnName: String, shouldSetIcebergReservedFieldId: Boolean): types.Metadata = RowIdMetadataStructField.metadata(materializedColumnName, shouldSetIcebergReservedFieldId) /** * The field readers can use to access the generated row id column. The scanner's internal column * name is obtained from the table's metadata. */ def createRowIdField( protocol: Protocol, metadata: Metadata, nullable: Boolean, shouldSetIcebergReservedFieldId: Boolean): Option[StructField] = MaterializedRowId.getMaterializedColumnName(protocol, metadata) .map(RowIdMetadataStructField(_, nullable, shouldSetIcebergReservedFieldId)) /* * A specialization of [[FileSourceGeneratedMetadataStructField]] used to represent RowId columns. * * - Row ID columns can be read by adding '_metadata.row_id' to the read schema * - To write to the materialized Row ID column * - use the materialized Row ID column name which can be obtained using * [[getMaterializedColumnName]] * - add [[COLUMN_METADATA]] which is part of [[RowId]] as metadata to the column * - nulls are replaced with fresh Row IDs */ object RowIdMetadataStructField { val ROW_ID_METADATA_COL_ATTR_KEY = "__row_id_metadata_col" def metadata( materializedColumnName: String, shouldSetIcebergReservedFieldId: Boolean): types.Metadata = { val metadataBuilder = new MetadataBuilder() .withMetadata( FileSourceGeneratedMetadataStructField.metadata(RowId.ROW_ID, materializedColumnName)) .putBoolean(ROW_ID_METADATA_COL_ATTR_KEY, value = true) // If IcebergCompatV3 or higher is enabled, assign the field ID of Delta row id column // to match the reserved `_row_id` field defined in the Iceberg spec. // This ensures that Iceberg can recognize and track the same column for row lineage purposes. if (shouldSetIcebergReservedFieldId) { metadataBuilder.putLong( PARQUET_FIELD_ID_METADATA_KEY, IcebergConstants.ICEBERG_ROW_TRACKING_ROW_ID_FIELD_ID ) } metadataBuilder.build() } def apply( materializedColumnName: String, nullable: Boolean = false, shouldSetIcebergReservedFieldId: Boolean): StructField = StructField( RowId.ROW_ID, LongType, // The Row ID field is used to read the materialized Row ID value which is nullable. The // actual Row ID expression is created using a projection injected before the optimizer pass // by the [[GenerateRowIDs] rule at which point the Row ID field is non-nullable. nullable, metadata = metadata(materializedColumnName, shouldSetIcebergReservedFieldId)) def unapply(field: StructField): Option[StructField] = if (isRowIdColumn(field)) Some(field) else None /** Return true if the column is a Row Id column. */ def isRowIdColumn(structField: StructField): Boolean = isValid(structField.dataType, structField.metadata) def isValid(dataType: DataType, metadata: types.Metadata): Boolean = { FileSourceGeneratedMetadataStructField.isValid(dataType, metadata) && metadata.contains(ROW_ID_METADATA_COL_ATTR_KEY) && metadata.getBoolean(ROW_ID_METADATA_COL_ATTR_KEY) } } object RowIdMetadataAttribute { /** Creates an attribute for writing out the materialized column name */ def apply( materializedColumnName: String, shouldSetIcebergReservedFieldId: Boolean): AttributeReference = DataTypeUtils .toAttribute( RowIdMetadataStructField( materializedColumnName, shouldSetIcebergReservedFieldId = shouldSetIcebergReservedFieldId)) .withName(materializedColumnName) def unapply(attr: Attribute): Option[Attribute] = if (isRowIdColumn(attr)) Some(attr) else None /** Return true if the column is a Row Id column. */ def isRowIdColumn(attr: Attribute): Boolean = RowIdMetadataStructField.isValid(attr.dataType, attr.metadata) } /** * Throw if row tracking is supported and columns in the write schema tagged as materialized row * IDs do not reference the materialized row id column name. */ private[delta] def throwIfMaterializedRowIdColumnNameIsInvalid( data: DataFrame, metadata: Metadata, protocol: Protocol, tableId: String): Unit = { if (!RowTracking.isEnabled(protocol, metadata)) { return } val materializedColumnName = metadata.configuration.get(MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP) if (materializedColumnName.isEmpty) { // If row tracking is enabled, a missing materialized column name is a bug and we need to // throw an error. If row tracking is only supported, we should just return, as it's fine // for the materialized column to not be assigned. if (RowTracking.isEnabled(protocol, metadata)) { throw DeltaErrors.materializedRowIdMetadataMissing(tableId) } return } toAttributes(data.schema).foreach { case RowIdMetadataAttribute(attribute) => if (attribute.name != materializedColumnName.get) { throw new UnsupportedOperationException("Materialized Row IDs column name " + s"${attribute.name} is invalid. Must be ${materializedColumnName.get}.") } case _ => } } /** * Add a new column to 'dataFrame' that has the name of the materialized Row ID column and holds * Row IDs. The column also is tagged with the appropriate metadata such that it can be used to * write materialized Row IDs. */ private[delta] def preserveRowIds( dataFrame: DataFrame, snapshot: SnapshotDescriptor): DataFrame = { if (!isEnabled(snapshot.protocol, snapshot.metadata)) { return dataFrame } val materializedColumnName = MaterializedRowId.getMaterializedColumnNameOrThrow( snapshot.protocol, snapshot.metadata, snapshot.deltaLog.unsafeVolatileTableId) val rowIdColumn = DeltaTableUtils.getFileMetadataColumn(dataFrame).getField(ROW_ID) val shouldSetIcebergReservedFieldId = IcebergCompat.isGeqEnabled(snapshot.metadata, 3) preserveRowIdsUnsafe( dataFrame, materializedColumnName, rowIdColumn, shouldSetIcebergReservedFieldId) } /** * Add a new column to 'dataFrame' that has 'materializedColumnName' and holds Row IDs. The column * is also tagged with the appropriate metadata so it can be used to write materialized Row IDs. * * Internal method, exposed only for testing. */ private[delta] def preserveRowIdsUnsafe( dataFrame: DataFrame, materializedColumnName: String, rowIdColumn: Column, shouldSetIcebergReservedFieldId: Boolean): DataFrame = { dataFrame .withColumn(materializedColumnName, rowIdColumn) .withMetadata( materializedColumnName, columnMetadata(materializedColumnName, shouldSetIcebergReservedFieldId)) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/RowTracking.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.DeltaCommitTag.PreservedRowTrackingTag import org.apache.spark.sql.delta.actions.{Metadata, Protocol, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.actions.CommitInfo import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} import org.apache.spark.sql.{DataFrame, SparkSession} import org.apache.spark.sql.functions.col import org.apache.spark.sql.types.StructField /** * Utility functions for Row Tracking that are shared between Row IDs and Row Commit Versions. */ object RowTracking { /** * Returns whether the protocol version supports the Row Tracking table feature. Whenever Row * Tracking is support, fresh Row IDs and Row Commit Versions must be assigned to all newly * committed files, even when Row IDs are disabled in the current table version. */ def isSupported(protocol: Protocol): Boolean = protocol.isFeatureSupported(RowTrackingFeature) /** * Returns whether Row Tracking is enabled on this table version. Checks that Row Tracking is * supported, which is a pre-requisite for enabling Row Tracking, throws an error if not. */ def isEnabled(protocol: Protocol, metadata: Metadata): Boolean = { val isEnabled = DeltaConfigs.ROW_TRACKING_ENABLED.fromMetaData(metadata) if (isEnabled && !isSupported(protocol)) { throw new IllegalStateException( s"Table property '${DeltaConfigs.ROW_TRACKING_ENABLED.key}' is " + s"set on the table but this table version doesn't support table feature " + s"'${TableFeatureProtocolUtils.propertyKey(RowTrackingFeature)}'.") } isEnabled } def isSuspended(spark: SparkSession, metadata: Metadata): Boolean = { val ignoreIsSuspended = spark.conf.get(DeltaSQLConf.DELTA_ROW_TRACKING_IGNORE_SUSPENSION) if (DeltaUtils.isTesting && ignoreIsSuspended) return false val isEnabled = DeltaConfigs.ROW_TRACKING_ENABLED.fromMetaData(metadata) val isSuspended = DeltaConfigs.ROW_TRACKING_SUSPENDED.fromMetaData(metadata) // Make sure a third party client did not miss ROW_TRACKING_SUSPENDED table property. if (isEnabled && isSuspended) { throw DeltaErrors.rowTrackingIllegalPropertyCombination() } isSuspended } /** * Checks whether CONVERT TO DELTA collects statistics if row tracking is supported. If it does * not collect statistics, we cannot assign fresh row IDs, hence we throw an error to either rerun * the command without enabling the row tracking table feature, or to enable the necessary * flags to collect statistics. */ private[delta] def checkStatsCollectedIfRowTrackingSupported( protocol: Protocol, convertToDeltaShouldCollectStats: Boolean, statsCollectionEnabled: Boolean): Unit = { if (!isSupported(protocol)) return if (!convertToDeltaShouldCollectStats || !statsCollectionEnabled) { throw DeltaErrors.convertToDeltaRowTrackingEnabledWithoutStatsCollection } } /** * @return the Row Tracking metadata fields for the file's _metadata * when Row Tracking is enabled. */ def createMetadataStructFields( protocol: Protocol, metadata: Metadata, nullableConstantFields: Boolean, nullableGeneratedFields: Boolean): Iterable[StructField] = { val shouldSetIcebergReservedFieldId = false RowId.createRowIdField( protocol, metadata, nullableGeneratedFields, shouldSetIcebergReservedFieldId) ++ RowId.createBaseRowIdField(protocol, metadata, nullableConstantFields) ++ DefaultRowCommitVersion.createDefaultRowCommitVersionField( protocol, metadata, nullableConstantFields) ++ RowCommitVersion.createMetadataStructField( protocol, metadata, nullableGeneratedFields, shouldSetIcebergReservedFieldId) } /** * @param preserved The value of [[DeltaCommitTag.PreservedRowTrackingTag.key]] tag * @return A copy of ``tagsMap`` with the [[DeltaCommitTag.PreservedRowTrackingTag.key]] tag added * or replaced with the new value. */ private def addPreservedRowTrackingTag( tagsMap: Map[String, String], preserved: Boolean = true): Map[String, String] = { tagsMap + (DeltaCommitTag.PreservedRowTrackingTag.key -> preserved.toString) } /** * Sets the [[DeltaCommitTag.PreservedRowTrackingTag.key]] tag to true if not set. We add the tag * to every operation because we assume all operations preserve row tracking by default. The * absence of the tag means that row tracking is not preserved. * Operations can set the tag to mark row tracking as preserved/not preserved. */ private[delta] def addPreservedRowTrackingTagIfNotSet( snapshot: SnapshotDescriptor, tagsMap: Map[String, String] = Map.empty): Map[String, String] = { if (!isEnabled(snapshot.protocol, snapshot.metadata) || tagsMap.contains(PreservedRowTrackingTag.key)) { return tagsMap } addPreservedRowTrackingTag(tagsMap) } /** * Returns a copy of the CommitInfo passed in with the PreservedRowTrackingTag tag set to false. */ private[delta] def addRowTrackingNotPreservedTag(commitInfo: CommitInfo): CommitInfo = { val tagsMap = commitInfo.tags.getOrElse(Map.empty[String, String]) val newCommitInfoTags = addPreservedRowTrackingTag(tagsMap, preserved = false) commitInfo.copy(tags = Some(newCommitInfoTags)) } /** * Checks whether the CommitInfo has the RowTrackingEnablementOnly tag set to true. * If omitted, we assume it is false. */ private def isRowTrackingEnablementOnlyCommit(commitInfo: Option[CommitInfo]): Boolean = { DeltaCommitTag .getTagValueFromCommitInfo(commitInfo, DeltaCommitTag.RowTrackingEnablementOnlyTag.key) .exists(_.toBoolean) } /** * Returns a Boolean indicating whether it is safe the resolve the metadata update conflict * between the current and winning transaction conflict, from the perspective of row tracking * enablement. */ def canResolveMetadataUpdateConflict( currentTransactionInfo: CurrentTransactionInfo, winningCommitSummary: WinningCommitSummary): Boolean = { if (!isSupported(currentTransactionInfo.protocol)) return false RowTracking.isRowTrackingEnablementOnlyCommit(winningCommitSummary.commitInfo) && !currentTransactionInfo.metadataChanged } /** * Update the currentTransactionInfo properly to resolve a metadata update conflict when the * winning commit is tagged as RowTrackingEnablementOnly. It is only safe to call this function if * [[RowTracking.canResolveMetadataUpdateConflict]] returns true. * * See [[ConflictCheckerEdge.checkNoMetadataUpdates()]] for more details. */ def resolveRowTrackingEnablementOnlyMetadataUpdateConflict( currentTransactionInfo: CurrentTransactionInfo, winningCommitSummary: WinningCommitSummary): CurrentTransactionInfo = { require(canResolveMetadataUpdateConflict(currentTransactionInfo, winningCommitSummary)) // If the CommitInfo is None, do nothing because the absence of the // [[DeltaCommitTag.PreservedRowTrackingTag]] means it is false. val newCommitInfo = currentTransactionInfo.commitInfo.map(RowTracking.addRowTrackingNotPreservedTag) val newMetadata = winningCommitSummary.metadataUpdates.head // OptimisticTransactions sets the metadata seen by the current txn // (currentTransactionInfo.metadata) to the updated metadata. To be consistent, let's do this // even if the current txn does not update metadata. currentTransactionInfo.copy( metadata = newMetadata, commitInfo = newCommitInfo ) } def preserveRowTrackingColumns( dfWithoutRowTrackingColumns: DataFrame, snapshot: SnapshotDescriptor): DataFrame = { val dfWithRowIds = RowId.preserveRowIds(dfWithoutRowTrackingColumns, snapshot) RowCommitVersion.preserveRowCommitVersions(dfWithRowIds, snapshot) } /** * Verifies that the [[RowTrackingFeature]] is enabled and all files have base row IDs in the * given snapshot. These invariants need to hold to enable the RowTracking table property. */ def verifyInvariantsForTablePropertyEnablement(snapshot: Snapshot): Unit = { if (!snapshot.protocol.isFeatureSupported(RowTrackingFeature)) { throw new ProtocolChangedException(None) } val filesRequiringBackfill = snapshot.allFiles.where(col("baseRowId").isNull) if (!filesRequiringBackfill.isEmpty) { throw new ProtocolChangedException(None) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/Snapshot.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.util.{Locale, TimeZone} import scala.collection.JavaConverters._ import scala.collection.mutable import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.actions.Action.logSchema import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CommitCoordinatorClient, CommitCoordinatorProvider, CoordinatedCommitsUsageLogs, CoordinatedCommitsUtils, TableCommitCoordinatorClient} import org.apache.spark.sql.delta.expressions.EncodeNestedVariantAsZ85String import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.DataSkippingReader import org.apache.spark.sql.delta.stats.DataSkippingReaderConf import org.apache.spark.sql.delta.stats.DeltaStatsColumnSpec import org.apache.spark.sql.delta.stats.StatisticsCollection import org.apache.spark.sql.delta.util.DeltaCommitFileProvider import org.apache.spark.sql.delta.util.FileNames import org.apache.spark.sql.delta.util.StateCache import org.apache.spark.sql.util.ScalaExtensions._ import io.delta.storage.commit.CommitCoordinatorClient import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.parquet.format.converter.ParquetMetadataConverter.NO_FILTER import org.apache.parquet.hadoop.Footer import org.apache.parquet.hadoop.ParquetFileReader import org.apache.spark.internal.{MDC, MessageWithContext} import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat, ParquetToSparkSchemaConverter} import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.StructType import org.apache.spark.util.Utils /** * A description of a Delta [[Snapshot]], including basic information such its [[DeltaLog]] * metadata, protocol, and version. */ trait SnapshotDescriptor { def deltaLog: DeltaLog def version: Long def metadata: Metadata def protocol: Protocol def schema: StructType = metadata.schema protected[delta] def numOfFilesIfKnown: Option[Long] protected[delta] def sizeInBytesIfKnown: Option[Long] /** Whether the table has [[CatalogOwnedTableFeature]] enabled */ def isCatalogOwned: Boolean = { version >= 0 && protocol.readerAndWriterFeatureNames.contains(CatalogOwnedTableFeature.name) } } /** * An immutable snapshot of the state of the log at some delta version. Internally * this class manages the replay of actions stored in checkpoint or delta files. * * After resolving any new actions, it caches the result and collects the * following basic information to the driver: * - Protocol Version * - Metadata * - Transaction state * * @param inCommitTimestampOpt The in-commit-timestamp of the latest commit in milliseconds. Can * be set to None if * 1. The timestamp has not been read yet - generally the case for cold tables. * 2. Or the table has not been initialized, i.e. `version = -1`. * 3. Or the table does not have [[InCommitTimestampTableFeature]] enabled. * */ class Snapshot( val path: Path, override val version: Long, val logSegment: LogSegment, override val deltaLog: DeltaLog, val checksumOpt: Option[VersionChecksum] ) extends SnapshotDescriptor with SnapshotStateManager with StateCache with StatisticsCollection with DataSkippingReader with ValidateChecksum with DeltaLogging { import Snapshot._ import DeltaLogFileIndex.COMMIT_VERSION_COLUMN // For implicits which re-use Encoder: import org.apache.spark.sql.delta.implicits._ protected def spark = SparkSession.active /** Snapshot to scan by the DeltaScanGenerator for metadata query optimizations */ override val snapshotToScan: Snapshot = this override def columnMappingMode: DeltaColumnMappingMode = metadata.columnMappingMode /** * Returns the timestamp of the latest commit of this snapshot. * For an uninitialized snapshot, this returns -1. * * When InCommitTimestampTableFeature is enabled, the timestamp * is retrieved from the CommitInfo of the latest commit which * can result in an IO operation. */ def timestamp: Long = getInCommitTimestampOpt.getOrElse(logSegment.lastCommitFileModificationTimestamp) /** * Returns the inCommitTimestamp if ICT is enabled, otherwise returns None. * This potentially triggers an IO operation to read the inCommitTimestamp. * This is a lazy val, so repeated calls will not trigger multiple IO operations. */ protected lazy val getInCommitTimestampOpt: Option[Long] = Option.when(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(metadata)) { _reconstructedProtocolMetadataAndICT.inCommitTimestamp .getOrElse { val startTime = System.currentTimeMillis() var exception = Option.empty[Throwable] try { val commitInfoOpt = DeltaHistoryManager.getCommitInfoOpt( deltaLog.store, DeltaCommitFileProvider(this).deltaFile(version), deltaLog.newDeltaHadoopConf()) CommitInfo.getRequiredInCommitTimestamp(commitInfoOpt, version.toString) } catch { case e: Throwable => exception = Some(e) throw e } finally { recordDeltaEvent( deltaLog, "delta.inCommitTimestamp.read", data = Map( "version" -> version, "callSite" -> "Snapshot.getInCommitTimestampOpt", "checkpointVersion" -> logSegment.checkpointProvider.version, "durationMs" -> (System.currentTimeMillis() - startTime), "exceptionMessage" -> exception.map(_.getMessage).getOrElse(""), "exceptionStackTrace" -> exception.map(_.getStackTrace.mkString("\n")).getOrElse(""), "isCRCPresent" -> checksumOpt.isDefined ) ) } } } private[delta] lazy val nonFileActions: Seq[Action] = { Seq(protocol, metadata) ++ setTransactions ++ domainMetadata } @volatile private[delta] var stateReconstructionTriggered = false /** * The last known backfilled version of this snapshot. This can be larger than the last * backfilled file in the snapshot's LogSegment so is separately tracked in this mutable * variable. The reason why this is needed is as follows: * * In general, we update a snapshot's LogSegment after a commit by appending the latest * commit file. This can be an unbackfilled commit. The next time we call update(), we * check, if we can reuse the post commit snapshot or if we need to create a new snapshot. * The update performs a listing and creates a new LogSegment and the criteria for * keeping or replacing the old snapshot is whether the old snapshot's LogSegment is equal * to the LogSegment created by the update() call (see getSnapshotForLogSegment). * * If an unbackfilled commit has been backfilled before update() is called, the new LogSegment * would contain the backfilled version of this commit and so the old and new LogSegments are * determined to be different and the snapshot is swapped. However, the snapshots are in fact * identical and so swapping the snapshot is not necessary and wold only lead to a loss of the * cached state of the old snapshot. * * To prevent this, we don't swap the snapshot in this case (see * LogSegment.lastMatchingBackfilledCommitIsEqual). This means that we'll continue to use * the old LogSegment, which contains the unbackfilled commit(s). To correctly keep track of * the fact that all commits in the LogSegment have indeed been backfilled, we keep the * last known backfilled version of the snapshot in this variable and update it each time * during LogSegment comparison. This allows callers to figure out whether this snapshot * indeed contains any unbackfilled commits or the LogSegment is just based on an older * version. */ @volatile private var lastKnownBackfilledVersion: Long = logSegment.lastBackfilledVersionInSegment def getLastKnownBackfilledVersion: Long = lastKnownBackfilledVersion def updateLastKnownBackfilledVersion(newVersion: Long): Unit = { if (newVersion > this.version) { throw new IllegalStateException("Can't update the last known backfilled version " + "to a version greater than the snapshot's version.") } lastKnownBackfilledVersion = math.max(lastKnownBackfilledVersion, newVersion) } /** * Helper method to determine, whether this snapshot contains "actual" unbackfilled * commits. See [[Snapshot.lastKnownBackfilledVersion]] for more details on why a * LogSegment may contain unbackfilled commits, even though these files have already * been backfilled. */ private[delta] def allCommitsBackfilled: Boolean = { lastKnownBackfilledVersion >= FileNames.getFileVersion(logSegment.deltas.last) && // This should always be true because we synchronously backfill during checkpoint // creation and always create a new snapshot after that, which will force the // latest LogSegment to be used. lastKnownBackfilledVersion >= logSegment.checkpointProvider.version } /** * Use [[stateReconstruction]] to create a representation of the actions in this table. * Cache the resultant output. */ private lazy val cachedState = recordFrameProfile("Delta", "snapshot.cachedState") { stateReconstructionTriggered = true cacheDS(stateReconstruction, s"Delta Table State #$version - $redactedPath") } /** * Given the list of files from `LogSegment`, create respective file indices to help create * a DataFrame and short-circuit the many file existence and partition schema inference checks * that exist in DataSource.resolveRelation(). */ protected[delta] lazy val deltaFileIndexOpt: Option[DeltaLogFileIndex] = { assertLogFilesBelongToTable(path, logSegment.deltas) DeltaLogFileIndex(DeltaLogFileIndex.COMMIT_FILE_FORMAT, logSegment.deltas) } protected lazy val fileIndices: Seq[DeltaLogFileIndex] = { val checkpointFileIndexes = checkpointProvider.allActionsFileIndexes() checkpointFileIndexes ++ deltaFileIndexOpt.toSeq } /** * Protocol, Metadata, and In-Commit Timestamp retrieved through * `protocolMetadataAndICTReconstruction` which skips a full state reconstruction. */ case class ReconstructedProtocolMetadataAndICT( protocol: Protocol, metadata: Metadata, inCommitTimestamp: Option[Long]) /** * Generate the protocol and metadata for this snapshot. This is usually cheaper than a * full state reconstruction, but still only compute it when necessary. */ private lazy val _reconstructedProtocolMetadataAndICT: ReconstructedProtocolMetadataAndICT = { // Should be small. At most 'checkpointInterval' rows, unless new commits are coming // in before a checkpoint can be written var protocol: Protocol = null var metadata: Metadata = null var inCommitTimestamp: Option[Long] = None protocolMetadataAndICTReconstruction().foreach { case ReconstructedProtocolMetadataAndICT(p: Protocol, _, _) => protocol = p case ReconstructedProtocolMetadataAndICT(_, m: Metadata, _) => metadata = m case ReconstructedProtocolMetadataAndICT(_, _, ict: Option[Long]) => inCommitTimestamp = ict } if (protocol == null) { recordDeltaEvent( deltaLog, opType = "delta.assertions.missingAction", data = Map( "version" -> version.toString, "action" -> "Protocol", "source" -> "Snapshot")) throw DeltaErrors.actionNotFoundException("protocol", version) } if (metadata == null) { recordDeltaEvent( deltaLog, opType = "delta.assertions.missingAction", data = Map( "version" -> version.toString, "action" -> "Metadata", "source" -> "Snapshot")) throw DeltaErrors.actionNotFoundException("metadata", version) } ReconstructedProtocolMetadataAndICT(protocol, metadata, inCommitTimestamp) } /** * [[CommitCoordinatorClient]] for the given delta table as of this snapshot. * - This should not be None when a coordinator has been configured for this table. However, if * the configured coordinator implementation has not been registered, this will be None. In such * cases, the user will see potentially stale reads for the table. For strict enforcement of * coordinated commits, the user can set the configuration * [[DeltaSQLConf.COORDINATED_COMMITS_IGNORE_MISSING_COORDINATOR_IMPLEMENTATION]] to false. * - This must be None when coordinated commits is disabled. */ val tableCommitCoordinatorClientOpt: Option[TableCommitCoordinatorClient] = { val failIfImplUnavailable = !spark.conf.get(DeltaSQLConf.COORDINATED_COMMITS_IGNORE_MISSING_COORDINATOR_IMPLEMENTATION) CoordinatedCommitsUtils.getTableCommitCoordinator( spark, deltaLog, this, failIfImplUnavailable ) } /** * Returns the [[TableCommitCoordinatorClient]] that should be used for any type of mutation * operation on the table. This includes, data writes, backfills etc. * This method will throw an error if the configured coordinator could not be instantiated. * @return [[TableCommitCoordinatorClient]] if the table is configured for coordinated commits, * None if the table is not configured for coordinated commits. */ def getTableCommitCoordinatorForWrites: Option[TableCommitCoordinatorClient] = { val coordinatorOpt = tableCommitCoordinatorClientOpt val coordinatorName = DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.fromMetaData(metadata) if (coordinatorName.isDefined && coordinatorOpt.isEmpty) { recordDeltaEvent( deltaLog, CoordinatedCommitsUsageLogs.COMMIT_COORDINATOR_MISSING_IMPLEMENTATION_WRITE, data = Map( "commitCoordinatorName" -> coordinatorName.get, "registeredCommitCoordinators" -> CommitCoordinatorProvider.getRegisteredCoordinatorNames.mkString(", "), "readVersion" -> version.toString ) ) throw DeltaErrors.unsupportedWritesWithMissingCoordinators(coordinatorName.get) } coordinatorOpt } /** Number of columns to collect stats on for data skipping */ override lazy val statsColumnSpec: DeltaStatsColumnSpec = StatisticsCollection.configuredDeltaStatsColumnSpec(metadata) /** Performs validations during initialization */ protected def init(): Unit = { deltaLog.protocolRead(protocol) deltaLog.assertTableFeaturesMatchMetadata(protocol, metadata) SchemaUtils.recordUndefinedTypes(deltaLog, metadata.schema) } /** The current set of actions in this [[Snapshot]] as plain Rows */ def stateDF: DataFrame = recordFrameProfile("Delta", "stateDF") { cachedState.getDF } /** The current set of actions in this [[Snapshot]] as a typed Dataset. */ def stateDS: Dataset[SingleAction] = recordFrameProfile("Delta", "stateDS") { cachedState.getDS } private[delta] def allFilesViaStateReconstruction: Dataset[AddFile] = { stateDS.where("add IS NOT NULL").select(col("add").as[AddFile]) } // Here we need to bypass the ACL checks for SELECT anonymous function permissions. /** All of the files present in this [[Snapshot]]. */ def allFiles: Dataset[AddFile] = allFilesViaStateReconstruction /** All unexpired tombstones. */ def tombstones: Dataset[RemoveFile] = { // Temporary workarround for SPARK-51356. stateDS.where("remove IS NOT NULL").map(_.remove) } def deltaFileSizeInBytes(): Long = deltaFileIndexOpt.map(_.sizeInBytes).getOrElse(0L) def checkpointSizeInBytes(): Long = checkpointProvider.effectiveCheckpointSizeInBytes() override def metadata: Metadata = _reconstructedProtocolMetadataAndICT.metadata override def protocol: Protocol = _reconstructedProtocolMetadataAndICT.protocol /** * Tries to retrieve the protocol, metadata, and in-commit-timestamp (if needed) from the * checksum file. If the checksum file is not present or if the protocol or metadata is missing * this will return None. */ protected def getProtocolMetadataAndIctFromCrc(checksumOpt: Option[VersionChecksum]): Option[Array[ReconstructedProtocolMetadataAndICT]] = { if (!spark.sessionState.conf.getConf( DeltaSQLConf.USE_PROTOCOL_AND_METADATA_FROM_CHECKSUM_ENABLED)) { return None } checksumOpt.map(c => (c.protocol, c.metadata, c.inCommitTimestampOpt)).flatMap { case (p: Protocol, m: Metadata, ict: Option[Long]) => Some(Array((p, null, None), (null, m, None), (null, null, ict)) .map(ReconstructedProtocolMetadataAndICT.tupled)) case (p, m, _) if p != null || m != null => // One was missing from the .crc file... warn and fall back to an optimized query val protocolStr = Option(p).map(_.toString).getOrElse("null") val metadataStr = Option(m).map(_.toString).getOrElse("null") recordDeltaEvent( deltaLog, opType = "delta.assertions.missingEitherProtocolOrMetadataFromChecksum", data = Map( "version" -> version.toString, "protocol" -> protocolStr, "source" -> metadataStr)) logWarning(log"Either protocol or metadata is null from checksum; " + log"version:${MDC(DeltaLogKeys.VERSION, version)} " + log"protocol:${MDC(DeltaLogKeys.PROTOCOL, protocolStr)} " + log"metadata:${MDC(DeltaLogKeys.DELTA_METADATA, metadataStr)}") None case _ => None // both missing... fall back to an optimized query } } /** * Pulls the protocol and metadata of the table from the files that are used to compute the * Snapshot directly--without triggering a full state reconstruction. This is important, because * state reconstruction depends on protocol and metadata for correctness. * If the current table version does not have a checkpoint, this function will also return the * in-commit-timestamp of the latest commit if available. * * Also this method should only access methods defined in [[UninitializedCheckpointProvider]] * which are not present in [[CheckpointProvider]]. This is because initialization of * [[Snapshot.checkpointProvider]] depends on [[Snapshot.protocolMetadataAndICTReconstruction()]] * and so if [[Snapshot.protocolMetadataAndICTReconstruction()]] starts depending on * [[Snapshot.checkpointProvider]] then there will be cyclic dependency. */ protected def protocolMetadataAndICTReconstruction(): Array[ReconstructedProtocolMetadataAndICT] = { import implicits._ getProtocolMetadataAndIctFromCrc(checksumOpt).foreach { protocolMetadataAndIctFromCrc => return protocolMetadataAndIctFromCrc } val schemaToUse = Action.logSchema(Set("protocol", "metaData", "commitInfo")) val checkpointOpt = checkpointProvider.topLevelFileIndex.map { index => deltaLog.loadIndex(index, schemaToUse) .withColumn(COMMIT_VERSION_COLUMN, lit(checkpointProvider.version)) } (checkpointOpt ++ deltaFileIndexOpt.map(deltaLog.loadIndex(_, schemaToUse)).toSeq) .reduceOption(_.union(_)).getOrElse(emptyDF) .select("protocol", "metaData", "commitInfo.inCommitTimestamp", COMMIT_VERSION_COLUMN) .where("protocol.minReaderVersion is not null or metaData.id is not null " + s"or (commitInfo.inCommitTimestamp is not null and version = $version)") .as[(Protocol, Metadata, Option[Long], Long)] .collect() .sortBy(_._4) .map { case (p, m, ict, _) => ReconstructedProtocolMetadataAndICT(p, m, ict) } } // Reconstruct the state by applying deltas in order to the checkpoint. // We partition by path as it is likely the bulk of the data is add/remove. // Non-path based actions will be collocated to a single partition. protected def stateReconstruction: Dataset[SingleAction] = { recordFrameProfile("Delta", "snapshot.stateReconstruction") { // for serializability val localMinFileRetentionTimestamp = minFileRetentionTimestamp val localMinSetTransactionRetentionTimestamp = minSetTransactionRetentionTimestamp val canonicalPath = deltaLog.getCanonicalPathUdf() // Canonicalize the paths so we can repartition the actions correctly, but only rewrite the // add/remove actions themselves after partitioning and sorting are complete. Otherwise, the // optimizer can generate a really bad plan that re-evaluates _EVERY_ field of the rewritten // struct(...) projection every time we touch _ANY_ field of the rewritten struct. // // NOTE: We sort by [[COMMIT_VERSION_COLUMN]] (provided by [[loadActions]]), to ensure that // actions are presented to InMemoryLogReplay in the ascending version order it expects. val ADD_PATH_CANONICAL_COL_NAME = "add_path_canonical" val REMOVE_PATH_CANONICAL_COL_NAME = "remove_path_canonical" loadActions .withColumn(ADD_PATH_CANONICAL_COL_NAME, when( col("add.path").isNotNull, canonicalPath(col("add.path")))) .withColumn(REMOVE_PATH_CANONICAL_COL_NAME, when( col("remove.path").isNotNull, canonicalPath(col("remove.path")))) .repartition( getNumPartitions, coalesce(col(ADD_PATH_CANONICAL_COL_NAME), col(REMOVE_PATH_CANONICAL_COL_NAME))) .sortWithinPartitions(COMMIT_VERSION_COLUMN) .withColumn("add", when( col("add.path").isNotNull, struct( col(ADD_PATH_CANONICAL_COL_NAME).as("path"), col("add.partitionValues"), col("add.size"), col("add.modificationTime"), col("add.dataChange"), col(ADD_STATS_TO_USE_COL_NAME).as("stats"), col("add.tags"), col("add.deletionVector"), col("add.baseRowId"), col("add.defaultRowCommitVersion"), col("add.clusteringProvider") ))) .withColumn("remove", when( col("remove.path").isNotNull, col("remove").withField("path", col(REMOVE_PATH_CANONICAL_COL_NAME)))) .as[SingleAction] .mapPartitions { iter => val state: LogReplay = new InMemoryLogReplay( Some(localMinFileRetentionTimestamp), localMinSetTransactionRetentionTimestamp) state.append(0, iter.map(_.unwrap)) state.checkpoint.map(_.wrap) } } } /** * Loads the file indices into a DataFrame that can be used for LogReplay. * * In addition to the usual nested columns provided by the SingleAction schema, it should provide * two additional columns to simplify the log replay process: [[COMMIT_VERSION_COLUMN]] (which, * when sorted in ascending order, will order older actions before newer ones, as required by * [[InMemoryLogReplay]]); and [[ADD_STATS_TO_USE_COL_NAME]] (to handle certain combinations of * config settings for delta.checkpoint.writeStatsAsJson and delta.checkpoint.writeStatsAsStruct). * When we see a V2 checkpoint without the old stats column, but the stats_parsed column, we * json encode the stats_parsed column back as "stats" again. This is a temporary correctness * hack. */ protected def loadActions: DataFrame = { if (fileIndices.isEmpty) return emptyDF // Augment the schema with a NullType add.stats_parsed column, as a place-holder for // compatibility with the checkpoint parquet. Both deltas and checkpoints generally use this // schema. HOWEVER, IF (and only if) a checkpoint actually exists, AND it provides an // add.stats_parsed column AND it lacks an add.stats column, THEN (and only then) the checkpoint // DF includes the actual add.stats_parsed column -- not a NullType placeholder -- from which we // generate the add_stats_to_use column (add.stats is unused in that case). Meanwhile, JSON // deltas always map add.stats to add_stats_to_use, and always use the placeholder. val logSchemaToUse = Action.logSchema val jsonStatsCol = col("add.stats") val deltas = deltaFileIndexOpt.map(deltaLog.loadIndex(_, logSchemaToUse)) .map(_.withColumn(ADD_STATS_TO_USE_COL_NAME, jsonStatsCol)) val checkpointDataframes = checkpointProvider .allActionsFileIndexesAndSchemas(spark, deltaLog) .map { case (index, schema) => val addSchema = schema("add").dataType.asInstanceOf[StructType] val (checkpointSchemaToUse, checkpointStatsColToUse) = if (addSchema.exists(_.name == "stats_parsed") && !addSchema.exists(_.name == "stats")) { val statsParsedSchema = addSchema("stats_parsed").dataType.asInstanceOf[StructType] val checkpointSchemaToUse = Action.logSchemaWithAddStatsParsed(addSchema("stats_parsed")) val statsCol = col("add.stats_parsed") // Only use EncodeNestedVariantAsZ85String if the schema contains VariantType. // This avoids performance overhead for tables without variant columns. val encodedStatsCol = if (SchemaUtils.checkForVariantTypeColumnsRecursively(statsParsedSchema)) { Column(EncodeNestedVariantAsZ85String(statsCol.expr)) } else { statsCol } ( checkpointSchemaToUse, to_json(encodedStatsCol) ) } else { // Normal (JSON-like) schema suffices (logSchemaToUse, jsonStatsCol) } // For schema compat, make sure to discard add.stats_parsed (if present) deltaLog.loadIndex(index, checkpointSchemaToUse) .withColumn(COMMIT_VERSION_COLUMN, lit(checkpointProvider.version)) .withColumn(ADD_STATS_TO_USE_COL_NAME, checkpointStatsColToUse) .withColumn("add", col("add").dropFields("stats_parsed")) } (checkpointDataframes ++ deltas).reduce(_.union(_)) } /** * Tombstones before the [[minFileRetentionTimestamp]] timestamp will be dropped from the * checkpoint. */ private[delta] def minFileRetentionTimestamp: Long = { deltaLog.clock.getTimeMillis() - DeltaLog.tombstoneRetentionMillis(metadata) } /** * [[SetTransaction]]s before [[minSetTransactionRetentionTimestamp]] will be considered expired * and dropped from the snapshot. */ private[delta] def minSetTransactionRetentionTimestamp: Option[Long] = { DeltaLog.minSetTransactionRetentionInterval(metadata).map(deltaLog.clock.getTimeMillis() - _) } private[delta] def getNumPartitions: Int = { spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_SNAPSHOT_PARTITIONS) .getOrElse(Snapshot.defaultNumSnapshotPartitions) } /** * Computes all the information that is needed by the checksum for the current snapshot. * May kick off state reconstruction if needed by any of the underlying fields. * Note that it's safe to set txnId to none, since the snapshot doesn't always have a txn * attached. E.g. if a snapshot is created by reading a checkpoint, then no txnId is present. */ def computeChecksum: VersionChecksum = VersionChecksum( txnId = None, inCommitTimestampOpt = getInCommitTimestampOpt, metadata = metadata, protocol = protocol, allFiles = checksumOpt.flatMap(_.allFiles), tableSizeBytes = checksumOpt.map(_.tableSizeBytes).getOrElse(sizeInBytes), numFiles = checksumOpt.map(_.numFiles).getOrElse(numOfFiles), numMetadata = checksumOpt.map(_.numMetadata).getOrElse(numOfMetadata), numProtocol = checksumOpt.map(_.numProtocol).getOrElse(numOfProtocol), // Only return setTransactions and domainMetadata if they are either already present // in the checksum or if they have already been computed in the current snapshot. setTransactions = checksumOpt.flatMap(_.setTransactions) .orElse { Option.when(_computedStateTriggered && // Only extract it from the current snapshot if set transaction // writes are enabled. spark.conf.get(DeltaSQLConf.DELTA_WRITE_SET_TRANSACTIONS_IN_CRC)) { setTransactions } }, domainMetadata = checksumOpt.flatMap(_.domainMetadata) .orElse(Option.when(_computedStateTriggered)(domainMetadata)), numDeletedRecordsOpt = checksumOpt.flatMap(_.numDeletedRecordsOpt) .orElse(Option.when(_computedStateTriggered)(numDeletedRecordsOpt).flatten) .filter(_ => deletionVectorsReadableAndMetricsEnabled), numDeletionVectorsOpt = checksumOpt.flatMap(_.numDeletionVectorsOpt) .orElse(Option.when(_computedStateTriggered)(numDeletionVectorsOpt).flatten) .filter(_ => deletionVectorsReadableAndMetricsEnabled), deletedRecordCountsHistogramOpt = checksumOpt.flatMap(_.deletedRecordCountsHistogramOpt) .orElse(Option.when(_computedStateTriggered)(deletedRecordCountsHistogramOpt).flatten) .filter(_ => deletionVectorsReadableAndHistogramEnabled), histogramOpt = checksumOpt.flatMap(_.histogramOpt) ) /** Returns the data schema of the table, used for reading stats */ def tableSchema: StructType = metadata.dataSchema def outputTableStatsSchema: StructType = metadata.dataSchema def outputAttributeSchema: StructType = metadata.dataSchema /** Returns the schema of the columns written out to file (overridden in write path) */ def dataSchema: StructType = metadata.dataSchema /** Return the set of properties of the table. */ def getProperties: mutable.Map[String, String] = { Snapshot.getProperties(metadata, protocol) } /** The [[CheckpointProvider]] for the underlying checkpoint */ lazy val checkpointProvider: CheckpointProvider = logSegment.checkpointProvider match { case cp: CheckpointProvider => cp case uninitializedProvider: UninitializedCheckpointProvider => CheckpointProvider(spark, this, checksumOpt, uninitializedProvider) case o => throw new IllegalStateException(s"Unknown checkpoint provider: ${o.getClass.getName}") } def redactedPath: String = Utils.redact(spark.sessionState.conf.stringRedactionPattern, path.toUri.toString) /** * Ensures that commit files are backfilled up to the current version in the snapshot. * * This method checks if there are any un-backfilled versions up to the current version and * triggers the backfilling process using the commit-coordinator. It verifies that the delta file * for the current version exists after the backfilling process. * * @throws IllegalStateException * if the delta file for the current version is not found after backfilling. */ def ensureCommitFilesBackfilled(catalogTableOpt: Option[CatalogTable]): Unit = { val tableCommitCoordinatorClientOpt = if (isCatalogOwned) { CatalogOwnedTableUtils.populateTableCommitCoordinatorFromCatalog(spark, catalogTableOpt, this) } else { getTableCommitCoordinatorForWrites } val tableCommitCoordinatorClient = tableCommitCoordinatorClientOpt.getOrElse { return } val minUnbackfilledVersion = DeltaCommitFileProvider(this).minUnbackfilledVersion if (minUnbackfilledVersion <= version) { val hadoopConf = deltaLog.newDeltaHadoopConf() tableCommitCoordinatorClient.backfillToVersion( catalogTableOpt.map(_.identifier), version, lastKnownBackfilledVersion = Some(minUnbackfilledVersion - 1)) val fs = deltaLog.logPath.getFileSystem(hadoopConf) val expectedBackfilledDeltaFile = FileNames.unsafeDeltaFile(deltaLog.logPath, version) if (!fs.exists(expectedBackfilledDeltaFile)) { throw new IllegalStateException("Backfilling of commit files failed. " + s"Expected delta file $expectedBackfilledDeltaFile not found.") } } } protected def emptyDF: DataFrame = spark.createDataFrame(spark.sparkContext.emptyRDD[Row], logSchema) def logInfo(msg: MessageWithContext): Unit = { val tableId = deltaLog.unsafeVolatileTableId super.logInfo(log"[tableId=${MDC(DeltaLogKeys.TABLE_ID, tableId)}] " + msg) } def logWarning(msg: MessageWithContext): Unit = { val tableId = deltaLog.unsafeVolatileTableId super.logWarning(log"[tableId=${MDC(DeltaLogKeys.TABLE_ID, tableId)}] " + msg) } def logWarning(msg: MessageWithContext, throwable: Throwable): Unit = { val tableId = deltaLog.unsafeVolatileTableId super.logWarning(log"[tableId=${MDC(DeltaLogKeys.TABLE_ID, tableId)}] " + msg, throwable) } def logError(msg: MessageWithContext): Unit = { val tableId = deltaLog.unsafeVolatileTableId super.logError(log"[tableId=${MDC(DeltaLogKeys.TABLE_ID, tableId)}] " + msg) } def logError(msg: MessageWithContext, throwable: Throwable): Unit = { val tableId = deltaLog.unsafeVolatileTableId super.logError(log"[tableId=${MDC(DeltaLogKeys.TABLE_ID, tableId)}] " + msg, throwable) } override def toString: String = s"${getClass.getSimpleName}(path=$path, version=$version, metadata=$metadata, " + s"logSegment=$logSegment, checksumOpt=$checksumOpt)" logInfo(log"Created snapshot ${MDC(DeltaLogKeys.SNAPSHOT, this)}") init() } object Snapshot extends DeltaLogging { // Used by [[loadActions]] and [[stateReconstruction]] val ADD_STATS_TO_USE_COL_NAME = "add_stats_to_use" private val defaultNumSnapshotPartitions: Int = 50 /** Verifies that a set of delta or checkpoint files to be read actually belongs to this table. */ private def assertLogFilesBelongToTable(logBasePath: Path, files: Seq[FileStatus]): Unit = { val logPath = new Path(logBasePath.toUri) val commitDirPath = FileNames.commitDirPath(logPath) files.map(_.getPath).foreach { filePath => val commitParent = new Path(filePath.toUri).getParent if (commitParent != logPath && commitParent != commitDirPath) { // scalastyle:off throwerror throw new AssertionError(s"File ($filePath) doesn't belong in the " + s"transaction log at $logBasePath.") // scalastyle:on throwerror } } } /** Whether to write allFiles in [[VersionChecksum.allFiles]] */ private[delta] def allFilesInCrcWritePathEnabled( spark: SparkSession, snapshot: Snapshot): Boolean = { // disable if config is off. if (!spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_ENABLED)) return false // Also disable if all stats (structs/json) are disabled in checkpoints. // When checkpoint stats are disabled (both in terms of structs/json), then the // snapshot.allFiles from state reconstruction may/may not have stats (files coming from // checkpoint won't have stats and files coming from deltas will have stats). // But CRC.allFiles will have stats as VersionChecksum.allFiles is created // incrementally using each commit. To prevent this inconsistency, we disable the feature when // both json/struct stats are disabled for checkpoint. if (!Checkpoints.shouldWriteStatsAsJson(snapshot) && !Checkpoints.shouldWriteStatsAsStruct(spark.sessionState.conf, snapshot)) { return false } // Disable if table is configured to collect stats on more than the default number of columns // to avoid bloating the .crc file. val numIndexedColsThreshold = spark.sessionState.conf .getConf(DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_THRESHOLD_INDEXED_COLS) .getOrElse(DataSkippingReaderConf.DATA_SKIPPING_NUM_INDEXED_COLS_DEFAULT_VALUE) val configuredNumIndexCols = DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.fromMetaData(snapshot.metadata) if (configuredNumIndexCols > numIndexedColsThreshold) return false true } /** * If true, force a verification of [[VersionChecksum.allFiles]] irrespective of the value of * DELTA_ALL_FILES_IN_CRC_VERIFICATION_MODE_ENABLED flag (if they're written). */ private[delta] def allFilesInCrcVerificationForceEnabled( spark: SparkSession): Boolean = { val forceVerificationForNonUTCEnabled = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_FORCE_VERIFICATION_MODE_FOR_NON_UTC_ENABLED) if (!forceVerificationForNonUTCEnabled) return false // This is necessary because timestamps for older dates (pre-1883) are not correctly serialized // in non-UTC timezones due to unusual historical offsets (e.g. -07:52:58 for LA). // These serialization discrepancies can lead to spurious CRC verification failures. // By forcing verification of all files in non-UTC environments, we can continue to detect and // work towards fixing this issues. // Note: Display Name for UTC is Etc/UTC, so we check for UTC substring in the timezone. val sparkSessionTimeZone = spark.sessionState.conf.sessionLocalTimeZone val defaultJVMTimeZone = TimeZone.getDefault.getID val systemTimeZone = System.getProperty("user.timezone", "Etc/UTC") val isNonUtcTimeZone = List(sparkSessionTimeZone, defaultJVMTimeZone, systemTimeZone) .exists(!_.toLowerCase(Locale.ROOT).contains("utc")) isNonUtcTimeZone } /** * If true, do verification of [[VersionChecksum.allFiles]] computed by incremental commit CRC * by doing state-reconstruction. */ private[delta] def allFilesInCrcVerificationEnabled( spark: SparkSession, snapshot: Snapshot): Boolean = { val verificationConfEnabled = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_VERIFICATION_MODE_ENABLED) val shouldVerify = verificationConfEnabled || allFilesInCrcVerificationForceEnabled(spark) allFilesInCrcWritePathEnabled(spark, snapshot) && shouldVerify } /** * Don't include [[AddFile]]s in CRC if this commit is modifying the schema of table in some * way. This is to make sure we don't carry any DROPPED column from previous CRC to this CRC * forever and can start fresh from next commit. * If the oldSnapshot itself is missing, we don't incrementally compute the checksum. */ private[delta] def shouldIncludeAddFilesInCrc( spark: SparkSession, snapshot: Snapshot, metadata: Metadata): Boolean = { allFilesInCrcWritePathEnabled(spark, snapshot) && (snapshot.version == -1 || snapshot.metadata.schema == metadata.schema) } /** * Return the set of properties for a given metadata and protocol. */ def getProperties(metadata: Metadata, protocol: Protocol): mutable.Map[String, String] = { val base = new mutable.LinkedHashMap[String, String]() metadata.configuration.foreach { case (k, v) => if (k != "path") { base.put(k, v) } } base.put(Protocol.MIN_READER_VERSION_PROP, protocol.minReaderVersion.toString) base.put(Protocol.MIN_WRITER_VERSION_PROP, protocol.minWriterVersion.toString) if (protocol.supportsReaderFeatures || protocol.supportsWriterFeatures) { val features = protocol.readerAndWriterFeatureNames.map(name => s"${TableFeatureProtocolUtils.FEATURE_PROP_PREFIX}$name" -> TableFeatureProtocolUtils.FEATURE_PROP_SUPPORTED) base ++ features.toSeq.sorted } else { base } } /** * Gets the schema of a single parquet file by reading its footer. Code here is copied from * ParquetFileFormat. */ private[delta] def getParquetFileSchemaAndRowCount( spark: SparkSession, deltaLog: DeltaLog, file: FileStatus): (StructType, Long) = { // Converter used to convert Parquet `MessageType` to Spark SQL `StructType` val converter = new ParquetToSparkSchemaConverter( assumeBinaryIsString = spark.sessionState.conf.isParquetBinaryAsString, assumeInt96IsTimestamp = spark.sessionState.conf.isParquetINT96AsTimestamp) val conf = deltaLog.newDeltaHadoopConf() val parquetMetadata = { ParquetFileReader.readFooter(deltaLog.newDeltaHadoopConf(), file.getPath) } val rowCount = parquetMetadata.getBlocks.asScala.map(_.getRowCount).sum val footer = new Footer(file.getPath(), parquetMetadata) (ParquetFileFormat.readSchemaFromFooter(footer, converter), rowCount) } } /** * A dummy snapshot with only metadata and protocol specified. It is used for a targeted table * version that does not exist yet before commiting a change. This can be used to create a * DataFrame, or to derive the stats schema from an existing Parquet table when converting it to * Delta or cloning it to a Delta table prior to the actual snapshot being available after a commit. * * Note that the snapshot state reconstruction contains only the protocol and metadata - it does not * include add/remove actions, appids, or metadata domains, even if the actual table currently has * or will have them in the future. * * @param logPath the path to transaction log * @param deltaLog the delta log object * @param metadata the metadata of the table * @param protocolOpt the protocol version of the table (optional). If not specified, a default * protocol will be computed based on the metadata. This must be explicitly * specified when replacing an existing Delta table, otherwise using the metadata * to compute the protocol might result in a protocol downgrade for the table. */ class DummySnapshot( val logPath: Path, override val deltaLog: DeltaLog, override val metadata: Metadata, protocolOpt: Option[Protocol] = None) extends Snapshot( path = logPath, version = -1, logSegment = LogSegment.empty(logPath), deltaLog = deltaLog, checksumOpt = None ) { def this(logPath: Path, deltaLog: DeltaLog) = this( logPath, deltaLog, Metadata( configuration = DeltaConfigs.mergeGlobalConfigs( sqlConfs = SparkSession.active.sessionState.conf, tableConf = Map.empty, ignoreProtocolConfsOpt = Some( DeltaConfigs.ignoreProtocolDefaultsIsSet( sqlConfs = SparkSession.active.sessionState.conf, tableConf = deltaLog.allOptions))), createdTime = Some(System.currentTimeMillis()))) override def stateDS: Dataset[SingleAction] = emptyDF.as[SingleAction] override def stateDF: DataFrame = emptyDF override def protocol: Protocol = protocolOpt.getOrElse(Protocol.forNewTable(spark, Some(metadata))) override protected lazy val computedState: SnapshotState = initialState(metadata, protocol) override protected lazy val getInCommitTimestampOpt: Option[Long] = None _computedStateTriggered = true // The [[InitialSnapshot]] is not backed by any external commit-coordinator. override val tableCommitCoordinatorClientOpt: Option[TableCommitCoordinatorClient] = None // Commit 0 cannot be performed through a commit coordinator. override def getTableCommitCoordinatorForWrites: Option[TableCommitCoordinatorClient] = None override def timestamp: Long = -1L } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/SnapshotManagement.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.FileNotFoundException import java.util.Objects import java.util.concurrent.{CompletableFuture, Future} import java.util.concurrent.locks.ReentrantLock import scala.collection.JavaConverters._ import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import scala.concurrent.duration._ import scala.util.control.NonFatal // scalastyle:off import.ordering.noEmptyLine import com.databricks.spark.util.TagDefinitions.TAG_ASYNC import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CoordinatedCommitsUsageLogs, CoordinatedCommitsUtils, TableCommitCoordinatorClient} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} import org.apache.spark.sql.delta.util.FileNames._ import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.sql.delta.util.threads.DeltaThreadPool import com.fasterxml.jackson.annotation.JsonIgnore import io.delta.storage.commit.{Commit, GetCommitsResponse} import org.apache.hadoop.fs.{BlockLocation, FileStatus, LocatedFileStatus, Path} import org.apache.spark.{SparkContext, SparkException} import org.apache.spark.internal.MDC import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.util.ThreadUtils /** * Wraps the most recently updated snapshot along with the timestamp the update was started. * Defined outside the class since it's used in tests. */ case class CapturedSnapshot(snapshot: Snapshot, updateTimestamp: Long) /** * Manages the creation, computation, and access of Snapshot's for Delta tables. Responsibilities * include: * - Figuring out the set of files that are required to compute a specific version of a table * - Updating and exposing the latest snapshot of the Delta table in a thread-safe manner */ trait SnapshotManagement { self: DeltaLog => import SnapshotManagement.verifyDeltaVersions @volatile private[delta] var asyncUpdateTask: Future[Unit] = _ /** Use ReentrantLock to allow us to call `lockInterruptibly` */ protected val snapshotLock = new ReentrantLock() /** * Cached fileStatus for the latest CRC file seen in the deltaLog. */ @volatile protected var lastSeenChecksumFileStatusOpt: Option[FileStatus] = None /** * Cached latest snapshot. This is initialized in `createSnapshotAtInit` */ @volatile protected var currentSnapshot: CapturedSnapshot = _ /** * Run `body` inside `snapshotLock` lock using `lockInterruptibly` so that the thread * can be interrupted when waiting for the lock. */ def withSnapshotLockInterruptibly[T](body: => T): T = { snapshotLock.lockInterruptibly() try { body } finally { snapshotLock.unlock() } } /** Get an iterator of files in the _delta_log directory starting with the startVersion. */ private[delta] def listFrom(startVersion: Long): Iterator[FileStatus] = { store.listFrom(listingPrefix(logPath, startVersion), newDeltaHadoopConf()) } /** Returns true if the path is delta log files. Delta log files can be delta commit file * (e.g., 000000000.json), or checkpoint file. (e.g., 000000001.checkpoint.00001.00003.parquet) * @param path Path of a file * @return Boolean Whether the file is delta log files */ protected def isDeltaCommitOrCheckpointFile(path: Path): Boolean = { isCheckpointFile(path) || isDeltaFile(path) } /** * @return A tuple where the first element is an array of log files (possibly empty, if no * usable log files are found), and the second element is the latest checksum file found * which has a version less than or equal to `versionToLoad`. */ private[delta] def listFromFileSystemInternal( startVersion: Long, versionToLoad: Option[Long], includeMinorCompactions: Boolean ): (Option[Array[(FileStatus, FileType.Value, Long)]], Option[FileStatus]) = { var latestAvailableChecksumFileStatus = Option.empty[FileStatus] // LIST the directory, starting from the provided lower bound (treat missing dir as empty). // NOTE: "empty/missing" is _NOT_ equivalent to "contains no useful commit files." val filesOpt = try { Some(listFrom(startVersion)).filterNot(_.isEmpty) } catch { case _: FileNotFoundException => None } val files = filesOpt.map { _.flatMap { case DeltaFile(f, fileVersion) => Some((f, FileType.DELTA, fileVersion)) case CompactedDeltaFile(f, startVersion, endVersion) if includeMinorCompactions && versionToLoad.forall(endVersion <= _) => Some((f, FileType.COMPACTED_DELTA, startVersion)) case CheckpointFile(f, fileVersion) if f.getLen > 0 => Some((f, FileType.CHECKPOINT, fileVersion)) case ChecksumFile(f, version) if versionToLoad.forall(version <= _) => latestAvailableChecksumFileStatus = Some(f) None case _ => None } // take files up to the version we want to load .takeWhile { case (_, _, fileVersion) => versionToLoad.forall(fileVersion <= _) } .toArray } (files, latestAvailableChecksumFileStatus) } /** * This method is designed to efficiently and reliably list delta, compacted delta, and * checkpoint files associated with a Delta Lake table. It makes parallel calls to both the * file-system and a commit-coordinator (if available), reconciles the results to account for * asynchronous backfill operations, and ensures a comprehensive list of file statuses without * missing any concurrently backfilled files. * *Note*: If table is a coordinated-commits table, the commit coordinator MUST be passed to * correctly list the commits. * The function also collects the latest checksum file found in the listings and returns it. * * @param startVersion the version to start. Inclusive. * @param tableCommitCoordinatorClientOpt the optional commit coordinator to use for fetching * un-backfilled commits. * @param catalogTableOpt the optional catalog table to pass to the commit coordinator client. * @param versionToLoad the optional parameter to set the max version we should return. Inclusive. * @param includeMinorCompactions Whether to include minor compaction files in the result * @return A tuple where the first element is an array of log files (possibly empty, if no * usable log files are found), and the second element is the latest checksum file found * which has a version less than or equal to `versionToLoad`. */ protected def listDeltaCompactedDeltaCheckpointFilesAndLatestChecksumFile( startVersion: Long, tableCommitCoordinatorClientOpt: Option[TableCommitCoordinatorClient], catalogTableOpt: Option[CatalogTable], versionToLoad: Option[Long], includeMinorCompactions: Boolean): (Option[Array[FileStatus]], Option[FileStatus]) = { val tableCommitCoordinatorClient = tableCommitCoordinatorClientOpt.getOrElse { val (filesOpt, checksumOpt) = listFromFileSystemInternal( startVersion, versionToLoad, includeMinorCompactions ) return (filesOpt.map(_.map(_._1)), checksumOpt) } // Submit a potential async call to get commits from commit coordinator if available val threadPool = SnapshotManagement.commitCoordinatorGetCommitsThreadPool def getCommitsTask(isAsyncRequest: Boolean): GetCommitsResponse = { CoordinatedCommitsUtils.getCommitsFromCommitCoordinatorWithUsageLogs( this, tableCommitCoordinatorClient, catalogTableOpt, startVersion, versionToLoad, isAsyncRequest) } def getGetCommitsResponseFuture(): Future[GetCommitsResponse] = { if (threadPool.getActiveCount < threadPool.getMaximumPoolSize) { threadPool.submit[GetCommitsResponse](spark) { getCommitsTask(isAsyncRequest = true) } } else { // If the thread pool is full, we should not submit more tasks to it. Instead, we should // run the task in the current thread. logInfo(log"Getting un-backfilled commits from commit coordinator in the same " + log"thread for table ${MDC(DeltaLogKeys.PATH, dataPath)}") recordDeltaEvent( this, CoordinatedCommitsUsageLogs.COMMIT_COORDINATOR_LISTING_THREADPOOL_FULL) CompletableFuture.completedFuture(getCommitsTask(isAsyncRequest = false)) } } val unbackfilledCommitsResponseFuture = getGetCommitsResponseFuture var maxDeltaVersionSeen = startVersion - 1 val (initialLogTuplesFromFsListingOpt, initialChecksumOpt) = listFromFileSystemInternal( startVersion, versionToLoad, includeMinorCompactions ) // Ideally listFromFileSystemInternal should return lexicographically sorted files and so // maxDeltaVersionSeen should be equal to the last delta version. But we are being // defensive here and taking max of all the delta fileVersions seen. initialLogTuplesFromFsListingOpt.foreach { logTuples => logTuples.filter(_._2 == FileType.DELTA).map(_._3).foreach { deltaVersion => maxDeltaVersionSeen = Math.max(maxDeltaVersionSeen, deltaVersion) } } val unbackfilledCommitsResponse = try { unbackfilledCommitsResponseFuture.get() } catch { case e: java.util.concurrent.ExecutionException => throw new CommitCoordinatorGetCommitsFailedException(e.getCause) } def requiresAdditionalListing(): Boolean = { // A gap in delta versions may occur if some delta files are backfilled "after" the // file-system listing but before the commit-coordinator listing. To handle this scenario, we // perform an additional listing from the file system because those missing files would be // backfilled by now and show up in the file-system. // Note: We only care about missing delta files with version <= versionToLoad val areDeltaFilesMissing = unbackfilledCommitsResponse.getCommits.asScala.headOption match { case Some(commit) => // Missing Delta files: [maxDeltaVersionSeen + 1, commit.head.version - 1] maxDeltaVersionSeen + 1 < commit.getVersion case None => // Missing Delta files: [maxDeltaVersionSeen + 1, latestTableVersion] // When there are no commits, we should consider the latestTableVersion from the commit // store to detect if ALL trailing commits were concurrently backfilled. unbackfilledCommitsResponse.getLatestTableVersion >= 0 && maxDeltaVersionSeen < unbackfilledCommitsResponse.getLatestTableVersion } versionToLoad.forall(maxDeltaVersionSeen < _) && areDeltaFilesMissing } val initialMaxDeltaVersionSeen = maxDeltaVersionSeen val (additionalLogTuplesFromFsListingOpt, additionalChecksumOpt) = if (requiresAdditionalListing()) { recordDeltaEvent( this, CoordinatedCommitsUsageLogs.COMMIT_COORDINATOR_ADDITIONAL_LISTING_REQUIRED) listFromFileSystemInternal( startVersion = initialMaxDeltaVersionSeen + 1, versionToLoad, includeMinorCompactions ) } else { (None, initialChecksumOpt) } additionalLogTuplesFromFsListingOpt.foreach { logTuples => logTuples.filter(_._2 == FileType.DELTA).map(_._3).foreach { deltaVersion => maxDeltaVersionSeen = Math.max(maxDeltaVersionSeen, deltaVersion) } } if (requiresAdditionalListing()) { // We should not have any gaps in File-System versions and CommitCoordinator versions after // the additional listing. val eventData = Map( "initialCommitVersionsFromFsListingOpt" -> initialLogTuplesFromFsListingOpt.map(_.map(_._3).toSeq), "initialMaxDeltaVersionSeen" -> initialMaxDeltaVersionSeen, "additionalCommitVersionsFromFsListingOpt" -> additionalLogTuplesFromFsListingOpt.map(_.map(_._3).toSeq), "maxDeltaVersionSeen" -> maxDeltaVersionSeen, "unbackfilledCommitVersions" -> unbackfilledCommitsResponse.getCommits.asScala.map(commit => commit.getVersion), "latestCommitVersion" -> unbackfilledCommitsResponse.getLatestTableVersion) recordDeltaEvent( deltaLog = this, opType = CoordinatedCommitsUsageLogs.FS_COMMIT_COORDINATOR_LISTING_UNEXPECTED_GAPS, data = eventData) if (DeltaUtils.isTesting) { throw new IllegalStateException( s"Delta table at $dataPath unexpectedly still requires additional file-system listing " + s"after an additional file-system listing was already performed to reconcile the gap " + s"between concurrent file-system and commit-owner calls. Details: $eventData" ) } } val finalLogTuplesFromFsListingOpt: Option[Array[(FileStatus, FileType.Value, Long)]] = (initialLogTuplesFromFsListingOpt, additionalLogTuplesFromFsListingOpt) match { case (Some(initial), Some(additional)) => // Filter initial list to exclude files with versions beyond // `initialListingMaxDeltaVersionSeen` to prevent duplicating non-delta files with // higher versions in the combined list. Ideally we shouldn't need this, but we are // being defensive here if the log has missing files. // E.g. initial = [0.json, 1.json, 2.checkpoint], initialListingMaxDeltaVersionSeen = 1, // additional = [2.checkpoint], final = [0.json, 1.json, 2.checkpoint] Some(initial.takeWhile(_._3 <= initialMaxDeltaVersionSeen) ++ additional) case (Some(initial), None) => Some(initial) case (None, Some(additional)) => Some(additional) case _ => None } val unbackfilledCommitsFiltered = unbackfilledCommitsResponse.getCommits.asScala .dropWhile(_.getVersion <= maxDeltaVersionSeen) .takeWhile(commit => versionToLoad.forall(commit.getVersion <= _)) .map(_.getFileStatus) // If result from fs listing is None and result from commit-coordinator is empty, return none. // This is used by caller to distinguish whether table doesn't exist. val logTuplesToReturn = finalLogTuplesFromFsListingOpt.map { logTuplesFromFsListing => logTuplesFromFsListing.map(_._1) ++ unbackfilledCommitsFiltered } val latestChecksumOpt = additionalChecksumOpt.orElse(initialChecksumOpt) (logTuplesToReturn, latestChecksumOpt) } /** * This method is designed to efficiently and reliably list delta, compacted delta, and * checkpoint files associated with a Delta Lake table. It makes parallel calls to both the * file-system and a commit-coordinator (if available), reconciles the results to account for * asynchronous backfill operations, and ensures a comprehensive list of file statuses without * missing any concurrently backfilled files. * *Note*: If table is a coordinated-commits table, the commit-coordinator client MUST be passed * to correctly list the commits. * * @param startVersion the version to start. Inclusive. * @param tableCommitCoordinatorClientOpt the optional commit-coordinator client to use for * fetching un-backfilled commits. * @param catalogTableOpt the optional catalog table to pass to the commit coordinator client. * @param versionToLoad the optional parameter to set the max version we should return. Inclusive. * @param includeMinorCompactions Whether to include minor compaction files in the result * @return Some array of files found (possibly empty, if no usable commit files are present), or * None if the listing returned no files at all. */ protected final def listDeltaCompactedDeltaAndCheckpointFiles( startVersion: Long, tableCommitCoordinatorClientOpt: Option[TableCommitCoordinatorClient], catalogTableOpt: Option[CatalogTable], versionToLoad: Option[Long], includeMinorCompactions: Boolean): Option[Array[FileStatus]] = { recordDeltaOperation(self, "delta.deltaLog.listDeltaAndCheckpointFiles") { val (logTuplesOpt, latestChecksumOpt) = listDeltaCompactedDeltaCheckpointFilesAndLatestChecksumFile( startVersion, tableCommitCoordinatorClientOpt, catalogTableOpt, versionToLoad, includeMinorCompactions) lastSeenChecksumFileStatusOpt = latestChecksumOpt logTuplesOpt } } /** * Get a list of files that can be used to compute a Snapshot at version `versionToLoad`, If * `versionToLoad` is not provided, will generate the list of files that are needed to load the * latest version of the Delta table. This method also performs checks to ensure that the delta * files are contiguous. * * @param versionToLoad A specific version to load. Typically used with time travel and the * Delta streaming source. If not provided, we will try to load the latest * version of the table. * @param oldCheckpointProviderOpt The [[CheckpointProvider]] from the previous snapshot. This is * used as a start version for the listing when `startCheckpoint` is * unavailable. This is also used to initialize the [[LogSegment]]. * @param tableCommitCoordinatorClientOpt the optional commit-coordinator client to use for * fetching un-backfilled commits. * @param catalogTableOpt the optional catalog table to pass to the commit coordinator client. * @param lastCheckpointInfo [[LastCheckpointInfo]] from the _last_checkpoint. This could be * used to initialize the Snapshot's [[LogSegment]]. * @return Some LogSegment to build a Snapshot if files do exist after the given * startCheckpoint. None, if the directory was missing or empty. */ protected def createLogSegment( versionToLoad: Option[Long] = None, oldCheckpointProviderOpt: Option[UninitializedCheckpointProvider] = None, tableCommitCoordinatorClientOpt: Option[TableCommitCoordinatorClient] = None, catalogTableOpt: Option[CatalogTable] = None, lastCheckpointInfo: Option[LastCheckpointInfo] = None): Option[LogSegment] = { // List based on the last known checkpoint version. // if that is -1, list from version 0L val lastCheckpointVersion = getCheckpointVersion(lastCheckpointInfo, oldCheckpointProviderOpt) val listingStartVersion = Math.max(0L, lastCheckpointVersion) val includeMinorCompactions = spark.conf.get(DeltaSQLConf.DELTALOG_MINOR_COMPACTION_USE_FOR_READS) val newFiles = listDeltaCompactedDeltaAndCheckpointFiles( listingStartVersion, tableCommitCoordinatorClientOpt, catalogTableOpt, versionToLoad, includeMinorCompactions) getLogSegmentForVersion( versionToLoad, newFiles, validateLogSegmentWithoutCompactedDeltas = true, oldCheckpointProviderOpt = oldCheckpointProviderOpt, tableCommitCoordinatorClientOpt = tableCommitCoordinatorClientOpt, catalogTableOpt = catalogTableOpt, lastCheckpointInfo = lastCheckpointInfo ) } private def createLogSegment( previousSnapshot: Snapshot, catalogTableOpt: Option[CatalogTable], commitCoordinatorOpt: Option[TableCommitCoordinatorClient]): Option[LogSegment] = { createLogSegment( oldCheckpointProviderOpt = Some(previousSnapshot.checkpointProvider), tableCommitCoordinatorClientOpt = commitCoordinatorOpt, catalogTableOpt = catalogTableOpt) } /** * Returns the last known checkpoint version based on [[LastCheckpointInfo]] or * [[CheckpointProvider]]. * Returns -1 if both the info is not available. */ protected def getCheckpointVersion( lastCheckpointInfoOpt: Option[LastCheckpointInfo], oldCheckpointProviderOpt: Option[UninitializedCheckpointProvider]): Long = { lastCheckpointInfoOpt.map(_.version) .orElse(oldCheckpointProviderOpt.map(_.version)) .getOrElse(-1) } /** * Helper method to validate that selected deltas are contiguous from checkpoint version till * the required `versionToLoad`. * @param selectedDeltas - deltas selected for snapshot creation. * @param checkpointVersion - checkpoint version selected for snapshot creation. Should be `-1` if * no checkpoint is selected. * @param versionToLoad - version for which we want to create the Snapshot. */ private def validateDeltaVersions( selectedDeltas: Array[FileStatus], checkpointVersion: Long, versionToLoad: Option[Long]): Unit = { // checkpointVersion should be passed as -1 if no checkpoint is needed for the LogSegment. // We may just be getting a checkpoint file. selectedDeltas.headOption.foreach { headDelta => val headDeltaVersion = deltaVersion(headDelta) val lastDeltaVersion = selectedDeltas.last match { case CompactedDeltaFile(_, _, endV) => endV case DeltaFile(_, v) => v } if (headDeltaVersion != checkpointVersion + 1) { throw DeltaErrors.truncatedTransactionLogException( unsafeDeltaFile(logPath, checkpointVersion + 1), lastDeltaVersion, unsafeVolatileMetadata) // metadata is best-effort only } val deltaVersions = selectedDeltas.flatMap { case CompactedDeltaFile(_, startV, endV) => (startV to endV) case DeltaFile(_, v) => Seq(v) } verifyDeltaVersions( spark = spark, versions = deltaVersions, expectedStartVersion = Some(checkpointVersion + 1), expectedEndVersion = versionToLoad, cachedSnapshot = Some(unsafeVolatileSnapshot)) } } /** * Helper function for the getLogSegmentForVersion above. Called with a provided files list, * and will then try to construct a new LogSegment using that. * *Note*: If table is a coordinated-commits table, the commit-coordinator MUST be passed to * correctly list the commits. */ protected def getLogSegmentForVersion( versionToLoad: Option[Long], files: Option[Array[FileStatus]], validateLogSegmentWithoutCompactedDeltas: Boolean, tableCommitCoordinatorClientOpt: Option[TableCommitCoordinatorClient], catalogTableOpt: Option[CatalogTable], oldCheckpointProviderOpt: Option[UninitializedCheckpointProvider], lastCheckpointInfo: Option[LastCheckpointInfo]): Option[LogSegment] = { recordFrameProfile("Delta", "SnapshotManagement.getLogSegmentForVersion") { val lastCheckpointVersion = getCheckpointVersion(lastCheckpointInfo, oldCheckpointProviderOpt) val newFiles = files.filterNot(_.isEmpty) .getOrElse { // No files found even when listing from 0 => empty directory => table does not exist yet. if (lastCheckpointVersion < 0) return None // We always write the commit and checkpoint files before updating _last_checkpoint. // If the listing came up empty, then we either encountered a list-after-put // inconsistency in the underlying log store, or somebody corrupted the table by // deleting files. Either way, we can't safely continue. // // For now, we preserve existing behavior by returning Array.empty, which will trigger a // recursive call to [[createLogSegment]] below. Array.empty[FileStatus] } if (newFiles.isEmpty && lastCheckpointVersion < 0) { // We can't construct a snapshot because the directory contained no usable commit // files... but we can't return None either, because it was not truly empty. throw DeltaErrors.logFileNotFoundException(logPath, versionToLoad, lastCheckpointVersion) } else if (newFiles.isEmpty) { // The directory may be deleted and recreated and we may have stale state in our DeltaLog // singleton, so try listing from the first version return createLogSegment( versionToLoad = versionToLoad, catalogTableOpt = catalogTableOpt) } val (checkpoints, deltasAndCompactedDeltas) = newFiles.partition(isCheckpointFile) val (deltas, compactedDeltas) = deltasAndCompactedDeltas.partition(isDeltaFile) // Find the latest checkpoint in the listing that is not older than the versionToLoad val checkpointFiles = checkpoints.map(f => CheckpointInstance(f.getPath)) val newCheckpoint = getLatestCompleteCheckpointFromList(checkpointFiles, versionToLoad) val newCheckpointVersion = newCheckpoint.map(_.version).getOrElse { // If we do not have any checkpoint, pass new checkpoint version as -1 so that first // delta version can be 0. if (lastCheckpointVersion >= 0) { // `startCheckpoint` was given but no checkpoint found on delta log. This means that the // last checkpoint we thought should exist (the `_last_checkpoint` file) no longer exists. // Try to look up another valid checkpoint and create `LogSegment` from it. // This case can arise if the user deleted the table (all commits and checkpoints) but // left the _last_checkpoint intact. recordDeltaEvent(this, "delta.checkpoint.error.partial") val snapshotVersion = versionToLoad.getOrElse(deltaVersion(deltas.last)) getLogSegmentWithMaxExclusiveCheckpointVersion( snapshotVersion, lastCheckpointVersion, tableCommitCoordinatorClientOpt, catalogTableOpt ).foreach { alternativeLogSegment => return Some(alternativeLogSegment) } // No alternative found, but the directory contains files so we cannot return None. throw DeltaErrors.missingPartFilesException( lastCheckpointVersion, new FileNotFoundException( s"Checkpoint file to load version: $lastCheckpointVersion is missing.")) } -1L } // If there is a new checkpoint, start new lineage there. If `newCheckpointVersion` is -1, // it will list all existing delta files. val deltasAfterCheckpoint = deltas.filter { file => deltaVersion(file) > newCheckpointVersion } // Here we validate that we are able to create a valid LogSegment by just using commit deltas // and without considering minor-compacted deltas. We want to fail early if log is messed up // i.e. some commit deltas are missing (although compacted-deltas are present). // We should not do this validation when we want to update the logSegment after a conflict // via the [[SnapshotManagement.getUpdatedLogSegment]] method. In that specific flow, we just // list from the committed version and reuse existing pre-commit logsegment together with // listing result to create the new pre-commit logsegment. Because of this, we don't have info // about all the delta files (e.g. when minor compactions are used in existing preCommit log // segment) and hence the validation if attempted will fail. So we need to set // `validateLogSegmentWithoutCompactedDeltas` to false in that case. if (validateLogSegmentWithoutCompactedDeltas) { validateDeltaVersions( selectedDeltas = deltasAfterCheckpoint, checkpointVersion = newCheckpointVersion, versionToLoad = versionToLoad) } val newVersion = deltasAfterCheckpoint.lastOption.map(deltaVersion).getOrElse(newCheckpoint.get.version) // reuse the oldCheckpointProvider if it is same as what we are looking for. val checkpointProviderOpt = newCheckpoint.map { ci => oldCheckpointProviderOpt .collect { case cp if cp.version == ci.version => cp } .getOrElse(ci.getCheckpointProvider(this, checkpoints, lastCheckpointInfo)) } // In the case where `deltasAfterCheckpoint` is empty, `deltas` should still not be empty, // they may just be before the checkpoint version unless we have a bug in log cleanup. if (deltas.isEmpty) { throw new IllegalStateException(s"Could not find any delta files for version $newVersion") } if (versionToLoad.exists(_ != newVersion)) { throwNonExistentVersionError(versionToLoad.get) } val lastCommitTimestamp = deltas.last.getModificationTime val deltasAndCompactedDeltasForLogSegment = useCompactedDeltasForLogSegment( deltasAndCompactedDeltas, deltasAfterCheckpoint, latestCommitVersion = newVersion, checkpointVersionToUse = newCheckpointVersion) validateDeltaVersions( selectedDeltas = deltasAndCompactedDeltasForLogSegment, checkpointVersion = newCheckpointVersion, versionToLoad = versionToLoad) Some(LogSegment( logPath, newVersion, deltasAndCompactedDeltasForLogSegment, checkpointProviderOpt, lastCommitTimestamp)) } } /** * @param deltasAndCompactedDeltas - all deltas or compacted deltas which could be used * @param deltasAfterCheckpoint - deltas after the last checkpoint file * @param latestCommitVersion - commit version for which we are trying to create Snapshot for * @param checkpointVersionToUse - underlying checkpoint version to use in Snapshot, -1 if no * checkpoint is used. * @return Returns a list of deltas/compacted-deltas which can be used to construct the * [[LogSegment]] instead of `deltasAfterCheckpoint`. */ protected def useCompactedDeltasForLogSegment( deltasAndCompactedDeltas: Seq[FileStatus], deltasAfterCheckpoint: Array[FileStatus], latestCommitVersion: Long, checkpointVersionToUse: Long): Array[FileStatus] = { val selectedDeltas = mutable.ArrayBuffer.empty[FileStatus] var highestVersionSeen = checkpointVersionToUse val commitRangeCovered = mutable.ArrayBuffer.empty[Long] // track if there is at least 1 compacted delta in `deltasAndCompactedDeltas` var hasCompactedDeltas = false for (file <- deltasAndCompactedDeltas) { val (startVersion, endVersion) = file match { case CompactedDeltaFile(_, startVersion, endVersion) => hasCompactedDeltas = true (startVersion, endVersion) case DeltaFile(_, version) => (version, version) } // select the compacted delta if the startVersion doesn't straddle `highestVersionSeen` and // the endVersion doesn't cross the latestCommitVersion. if (highestVersionSeen < startVersion && endVersion <= latestCommitVersion) { commitRangeCovered.appendAll(startVersion to endVersion) selectedDeltas += file highestVersionSeen = endVersion } } // If there are no compacted deltas in the `deltasAndCompactedDeltas` list, return from this // method. if (!hasCompactedDeltas) return deltasAfterCheckpoint // Validation-1: Commits represented by `compactedDeltasToUse` should be unique and there must // not be any duplicates. val coveredCommits = commitRangeCovered.toSet val hasDuplicates = (commitRangeCovered.size != coveredCommits.size) // Validation-2: All commits from (CheckpointVersion + 1) to latestCommitVersion should be // either represented by compacted delta or by the delta. val requiredCommits = (checkpointVersionToUse + 1) to latestCommitVersion val missingCommits = requiredCommits.toSet -- coveredCommits if (!hasDuplicates && missingCommits.isEmpty) return selectedDeltas.toArray // If the above check failed, that means the compacted delta validation failed. // Just record that event and return just the deltas (deltasAfterCheckpoint). val eventData = Map( "deltasAndCompactedDeltas" -> deltasAndCompactedDeltas.map(_.getPath.getName), "deltasAfterCheckpoint" -> deltasAfterCheckpoint.map(_.getPath.getName), "latestCommitVersion" -> latestCommitVersion, "checkpointVersionToUse" -> checkpointVersionToUse, "hasDuplicates" -> hasDuplicates, "missingCommits" -> missingCommits ) recordDeltaEvent( deltaLog = this, opType = "delta.getLogSegmentForVersion.compactedDeltaValidationFailed", data = eventData) if (DeltaUtils.isTesting) { assert(false, s"Validation around Compacted deltas failed while creating Snapshot. " + s"[${JsonUtils.toJson(eventData)}]") } deltasAfterCheckpoint } def throwNonExistentVersionError(versionToLoad: Long): Unit = { throw new IllegalStateException( s"Trying to load a non-existent version $versionToLoad") } /** * Load the Snapshot for this Delta table at initialization. This method uses the `lastCheckpoint` * file as a hint on where to start listing the transaction log directory. If the _delta_log * directory doesn't exist, this method will return an `InitialSnapshot`. */ protected def createSnapshotAtInit(initialCatalogTable: Option[CatalogTable]): Unit = withSnapshotLockInterruptibly { recordFrameProfile("Delta", "SnapshotManagement.createSnapshotAtInit") { val snapshotInitWallclockTime = clock.getTimeMillis() val lastCheckpointOpt = readLastCheckpointFile() val initialSegmentForNewSnapshot = createLogSegment( versionToLoad = None, catalogTableOpt = initialCatalogTable, lastCheckpointInfo = lastCheckpointOpt) val snapshot = getUpdatedSnapshot( oldSnapshotOpt = None, initialSegmentForNewSnapshot = initialSegmentForNewSnapshot, initialTableCommitCoordinatorClient = None, catalogTableOpt = initialCatalogTable, isAsync = false) currentSnapshot = CapturedSnapshot(snapshot, snapshotInitWallclockTime) } } /** * Returns the current snapshot. This does not automatically `update()`. * * WARNING: This is not guaranteed to give you the latest snapshot of the log, nor stay * consistent across multiple accesses. If you need the latest snapshot, it is recommended * to fetch it using `deltaLog.update()`; and save the returned snapshot so it does not * unexpectedly change from under you. See how [[OptimisticTransaction]] and [[DeltaScan]] * use the snapshot as examples for write/read paths respectively. * This API should only be used in scenarios where any recent snapshot will suffice and an * update is undesired, or by internal code that holds the DeltaLog lock to prevent races. */ def unsafeVolatileSnapshot: Snapshot = Option(currentSnapshot).map(_.snapshot).orNull /** * WARNING: This API is unsafe and deprecated. It will be removed in future versions. * Use the above unsafeVolatileSnapshot to get the most recently cached snapshot on * the cluster. */ @deprecated("This method is deprecated and will be removed in future versions. " + "Use unsafeVolatileSnapshot instead", "12.0") def snapshot: Snapshot = unsafeVolatileSnapshot /** * Unsafe due to thread races that can change it at any time without notice, even between two * calls in the same method. Like [[unsafeVolatileSnapshot]] it depends on, this method should be * used only with extreme care in production code (or by unit tests where no races are possible). */ private[delta] def unsafeVolatileMetadata = Option(unsafeVolatileSnapshot).map(_.metadata).getOrElse(Metadata()) protected def createSnapshot( initSegment: LogSegment, tableCommitCoordinatorClientOpt: Option[TableCommitCoordinatorClient], catalogTableOpt: Option[CatalogTable], checksumOpt: Option[VersionChecksum]): Snapshot = { val startingFrom = if (!initSegment.checkpointProvider.isEmpty) { log" starting from checkpoint version " + log"${MDC(DeltaLogKeys.START_VERSION, initSegment.checkpointProvider.version)}." } else log"." logInfo(log"[tableId=${MDC(DeltaLogKeys.TABLE_ID, truncatedUnsafeVolatileTableId)}] " + log"Loading version ${MDC(DeltaLogKeys.VERSION, initSegment.version)}" + startingFrom) createSnapshotFromGivenOrEquivalentLogSegment( initSegment, tableCommitCoordinatorClientOpt, catalogTableOpt) { segment => new Snapshot( path = logPath, version = segment.version, logSegment = segment, deltaLog = this, checksumOpt = checksumOpt.orElse( readChecksum(segment.version, lastSeenChecksumFileStatusOpt)) ) } } /** * Returns a [[LogSegment]] for reading `snapshotVersion` such that the segment's checkpoint * version (if checkpoint present) is LESS THAN `maxExclusiveCheckpointVersion`. * This is useful when trying to skip a bad checkpoint. Returns `None` when we are not able to * construct such [[LogSegment]], for example, no checkpoint can be used but we don't have the * entire history from version 0 to version `snapshotVersion`. * *Note*: If table is a coordinated-commits table, the commit-coordinator MUST be passed to * correctly list the commits. */ private def getLogSegmentWithMaxExclusiveCheckpointVersion( snapshotVersion: Long, maxExclusiveCheckpointVersion: Long, tableCommitCoordinatorClientOpt: Option[TableCommitCoordinatorClient], catalogTableOpt: Option[CatalogTable]): Option[LogSegment] = { assert( snapshotVersion >= maxExclusiveCheckpointVersion, s"snapshotVersion($snapshotVersion) is less than " + s"maxExclusiveCheckpointVersion($maxExclusiveCheckpointVersion)") val upperBoundVersion = math.min(snapshotVersion + 1, maxExclusiveCheckpointVersion) val previousCp = if (upperBoundVersion > 0) findLastCompleteCheckpointBefore(upperBoundVersion) else None previousCp match { case Some(cp) => val filesSinceCheckpointVersion = listDeltaCompactedDeltaAndCheckpointFiles( startVersion = cp.version, tableCommitCoordinatorClientOpt = tableCommitCoordinatorClientOpt, catalogTableOpt = catalogTableOpt, versionToLoad = Some(snapshotVersion), includeMinorCompactions = false ).getOrElse(Array.empty) val (checkpoints, deltas) = filesSinceCheckpointVersion.partition(isCheckpointFile) if (deltas.isEmpty) { // We cannot find any delta files. Returns None as we cannot construct a `LogSegment` only // from checkpoint files. This is because in order to create a `LogSegment`, we need to // set `LogSegment.lastCommitTimestamp`, and it must be read from the file modification // time of the delta file for `snapshotVersion`. It cannot be the file modification time // of a checkpoint file because it should be deterministic regardless how we construct the // Snapshot, and only delta json log files can ensure that. return None } // `checkpoints` may contain multiple checkpoints for different part sizes, we need to // search `FileStatus`s of the checkpoint files for `cp`. val checkpointProvider = cp.getCheckpointProvider(this, checkpoints, lastCheckpointInfoHint = None) // Create the list of `FileStatus`s for delta files after `cp.version`. val deltasAfterCheckpoint = deltas.filter { file => deltaVersion(file) > cp.version } val deltaVersions = deltasAfterCheckpoint.map(deltaVersion) // `deltaVersions` should not be empty and `verifyDeltaVersions` will verify it try { verifyDeltaVersions( spark = spark, versions = deltaVersions, expectedStartVersion = Some(cp.version + 1), expectedEndVersion = Some(snapshotVersion), cachedSnapshot = Some(unsafeVolatileSnapshot)) } catch { case NonFatal(e) => logWarning(log"Failed to find a valid LogSegment for " + log"${MDC(DeltaLogKeys.VERSION, snapshotVersion)}", e) return None } Some(LogSegment( logPath, snapshotVersion, deltas, Some(checkpointProvider), deltas.last.getModificationTime)) case None => val listFromResult = listDeltaCompactedDeltaAndCheckpointFiles( startVersion = 0, tableCommitCoordinatorClientOpt = tableCommitCoordinatorClientOpt, catalogTableOpt = catalogTableOpt, versionToLoad = Some(snapshotVersion), includeMinorCompactions = false) val (deltas, deltaVersions) = listFromResult .getOrElse(Array.empty) .flatMap(DeltaFile.unapply(_)) .unzip try { verifyDeltaVersions( spark = spark, versions = deltaVersions, expectedStartVersion = Some(0), expectedEndVersion = Some(snapshotVersion), cachedSnapshot = Some(unsafeVolatileSnapshot)) } catch { case NonFatal(e) => logWarning(log"Failed to find a valid LogSegment for " + log"${MDC(DeltaLogKeys.VERSION, snapshotVersion)}", e) return None } Some(LogSegment( logPath = logPath, version = snapshotVersion, deltas = deltas, checkpointProviderOpt = None, lastCommitTimestamp = deltas.last.getModificationTime)) } } /** * Used to compute the LogSegment after a commit, by adding the delta file with the specified * version to the preCommitLogSegment (which must match the immediately preceding version). */ protected[delta] def getLogSegmentAfterCommit( committedVersion: Long, newChecksumOpt: Option[VersionChecksum], preCommitLogSegment: LogSegment, commit: Commit, tableCommitCoordinatorClientOpt: Option[TableCommitCoordinatorClient], catalogTableOpt: Option[CatalogTable], oldCheckpointProvider: CheckpointProvider): LogSegment = recordFrameProfile( "Delta", "SnapshotManagement.getLogSegmentAfterCommit") { // If the table doesn't have any competing updates, then go ahead and use the optimized // incremental logSegment computation to fetch the LogSegment for the committedVersion. // See the comment in the getLogSegmentAfterCommit overload for why we can't always safely // return the committedVersion's snapshot when there is contention. val useFastSnapshotConstruction = !snapshotLock.hasQueuedThreads if (useFastSnapshotConstruction) { SnapshotManagement.appendCommitToLogSegment( preCommitLogSegment, commit.getFileStatus, committedVersion) } else { val latestCheckpointProvider = Seq(preCommitLogSegment.checkpointProvider, oldCheckpointProvider).maxBy(_.version) getLogSegmentAfterCommit( tableCommitCoordinatorClientOpt, catalogTableOpt, latestCheckpointProvider) } } protected[delta] def getLogSegmentAfterCommit( tableCommitCoordinatorClientOpt: Option[TableCommitCoordinatorClient], catalogTableOpt: Option[CatalogTable], oldCheckpointProvider: UninitializedCheckpointProvider): LogSegment = { /** * We can't specify `versionToLoad = committedVersion` for the call below. * If there are a lot of concurrent commits to the table on the same cluster, each * would generate a different snapshot, and thus each would trigger a new state * reconstruction. The last commit would get stuck waiting for each of the previous * jobs to finish to grab the update lock. * Instead, just do a general update to the latest available version. The racing commits * can then use the version check short-circuit to avoid constructing a new snapshot. */ createLogSegment( oldCheckpointProviderOpt = Some(oldCheckpointProvider), tableCommitCoordinatorClientOpt = tableCommitCoordinatorClientOpt, catalogTableOpt = catalogTableOpt ).getOrElse { // This shouldn't be possible right after a commit logError(log"No delta log found for the Delta table at ${MDC(DeltaLogKeys.PATH, logPath)}") throw DeltaErrors.logFileNotFoundException( logPath, version = None, checkpointVersion = getCheckpointVersion(None, Some(oldCheckpointProvider))) } } /** * Create a [[Snapshot]] from the given [[LogSegment]]. If failing to create the snapshot, we will * search an equivalent [[LogSegment]] using a different checkpoint and retry up to * [[DeltaSQLConf.DELTA_SNAPSHOT_LOADING_MAX_RETRIES]] times. */ protected def createSnapshotFromGivenOrEquivalentLogSegment( initSegment: LogSegment, tableCommitCoordinatorClientOpt: Option[TableCommitCoordinatorClient], catalogTableOpt: Option[CatalogTable]) (snapshotCreator: LogSegment => Snapshot): Snapshot = { val numRetries = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_SNAPSHOT_LOADING_MAX_RETRIES) var attempt = 0 var segment = initSegment // Remember the first error we hit. If all retries fail, we will throw the first error to // provide the root cause. We catch `SparkException` because corrupt checkpoint files are // detected in the executor side when a task is trying to read them. var firstError: SparkException = null while (true) { try { return snapshotCreator(segment) } catch { case e: SparkException if attempt < numRetries && !segment.checkpointProvider.isEmpty => if (firstError == null) { firstError = e } logWarning(log"Failed to create a snapshot from log segment " + log"${MDC(DeltaLogKeys.LOG_SEGMENT, segment)}. Trying a different checkpoint.", e) segment = getLogSegmentWithMaxExclusiveCheckpointVersion( segment.version, segment.checkpointProvider.version, tableCommitCoordinatorClientOpt, catalogTableOpt).getOrElse { // Throw the first error if we cannot find an equivalent `LogSegment`. throw firstError } attempt += 1 case e: SparkException if firstError != null => logWarning(log"Failed to create a snapshot from log segment " + log"${MDC(DeltaLogKeys.LOG_SEGMENT, segment)}", e) throw firstError } } throw new IllegalStateException("should not happen") } /** Checks if the given timestamp is outside the current staleness window */ protected def isCurrentlyStale: Long => Boolean = { val limit = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT) val cutoffOpt = if (limit > 0) Some(math.max(0, clock.getTimeMillis() - limit)) else None timestamp => cutoffOpt.forall(timestamp < _) } /** * Get the newest logSegment, using the previous logSegment as a hint. This is faster than * doing a full update, but it won't work if the table's log directory was replaced. */ def getUpdatedLogSegment( oldLogSegment: LogSegment, tableCommitCoordinatorClientOpt: Option[TableCommitCoordinatorClient], catalogTableOpt: Option[CatalogTable] ): (LogSegment, Seq[FileStatus]) = { val includeCompactions = spark.conf.get(DeltaSQLConf.DELTALOG_MINOR_COMPACTION_USE_FOR_READS) val newFilesOpt = listDeltaCompactedDeltaAndCheckpointFiles( startVersion = oldLogSegment.version + 1, tableCommitCoordinatorClientOpt = tableCommitCoordinatorClientOpt, catalogTableOpt = catalogTableOpt, versionToLoad = None, includeMinorCompactions = includeCompactions) // The file listing may return the following results: // 1. None => implies the log directory does not exist or is completely empty. // 2. Some(empty) => implies the log directory exists but there are no usable files. // 3. Some(non-empty) => implies the log directory exists and there are some usable files. // Therefore, we need to handle both cases (1) and (2) here. val newFiles = newFilesOpt.filter(_.nonEmpty).getOrElse { // An empty listing likely implies a list-after-write inconsistency or that somebody clobbered // the Delta log. return (oldLogSegment, Nil) } val allFiles = ( oldLogSegment.checkpointProvider.topLevelFiles ++ oldLogSegment.deltas ++ newFiles ).toArray val lastCheckpointInfo = Option.empty[LastCheckpointInfo] val newLogSegment = getLogSegmentForVersion( versionToLoad = None, files = Some(allFiles), validateLogSegmentWithoutCompactedDeltas = false, tableCommitCoordinatorClientOpt = tableCommitCoordinatorClientOpt, catalogTableOpt = catalogTableOpt, lastCheckpointInfo = lastCheckpointInfo, oldCheckpointProviderOpt = Some(oldLogSegment.checkpointProvider) ).getOrElse(oldLogSegment) val fileStatusesOfConflictingCommits = newFiles.collect { case DeltaFile(f, v) if v <= newLogSegment.version => f } (newLogSegment, fileStatusesOfConflictingCommits) } /** * Returns the snapshot, if it has been updated since the specified timestamp. * * Note that this should be used differently from isSnapshotStale. Staleness is * used to allow async updates if the table has been updated within the staleness * window, which allows for better perf in exchange for possibly using a slightly older * view of the table. For eg, if a table is queried multiple times in quick succession. * * On the other hand, getSnapshotIfFresh is used to identify duplicate updates within a * single transaction. For eg, if a table isn't cached and the snapshot was fetched from the * logstore, then updating the snapshot again in the same transaction is superfluous. We can * use this function to detect and skip such an update. */ private def getSnapshotIfFresh( capturedSnapshot: CapturedSnapshot, checkIfUpdatedSinceTs: Option[Long]): Option[Snapshot] = { checkIfUpdatedSinceTs.collect { case ts if ts <= capturedSnapshot.updateTimestamp => capturedSnapshot.snapshot } } /** * Update ActionLog by applying the new delta files if any. * * @param stalenessAcceptable Whether we can accept working with a stale version of the table. If * the table has surpassed our staleness tolerance, we will update to * the latest state of the table synchronously. If staleness is * acceptable, and the table hasn't passed the staleness tolerance, we * will kick off a job in the background to update the table state, * and can return a stale snapshot in the meantime. * @param checkIfUpdatedSinceTs Skip the update if we've already updated the snapshot since the * specified timestamp. * @param catalogTableOpt The catalog table of the current table. */ def update( stalenessAcceptable: Boolean = false, checkIfUpdatedSinceTs: Option[Long] = None, catalogTableOpt: Option[CatalogTable] = None): Snapshot = { val startTimeMs = System.currentTimeMillis() // currentSnapshot is volatile. Make a local copy of it at the start of the update call, so // that there's no chance of a race condition changing the snapshot partway through the update. val capturedSnapshot = currentSnapshot val oldVersion = capturedSnapshot.snapshot.version def sendEvent( newSnapshot: Snapshot, snapshotAlreadyUpdatedAfterRequiredTimestamp: Boolean = false ): Unit = { recordDeltaEvent( this, opType = "deltaLog.update", data = Map( "snapshotAlreadyUpdatedAfterRequiredTimestamp" -> snapshotAlreadyUpdatedAfterRequiredTimestamp, "newVersion" -> newSnapshot.version, "oldVersion" -> oldVersion, "timeTakenMs" -> (System.currentTimeMillis() - startTimeMs) ) ) } // Eagerly exit if the snapshot is already new enough to satisfy the caller getSnapshotIfFresh(capturedSnapshot, checkIfUpdatedSinceTs).foreach { snapshot => sendEvent(snapshot, snapshotAlreadyUpdatedAfterRequiredTimestamp = true) return snapshot } val doAsync = stalenessAcceptable && !isCurrentlyStale(capturedSnapshot.updateTimestamp) if (!doAsync) { recordFrameProfile("Delta", "SnapshotManagement.update") { val snapshot = withSnapshotLockInterruptibly { val newSnapshot = updateInternal( isAsync = false, catalogTableOpt) sendEvent(newSnapshot = newSnapshot) newSnapshot } logCurrentSnapshot() snapshot } } else { // Kick off an async update, if one is not already obviously running. Intentionally racy. if (Option(asyncUpdateTask).forall(_.isDone)) { try { val jobGroup = spark.sparkContext.getLocalProperty(SparkContext.SPARK_JOB_GROUP_ID) asyncUpdateTask = SnapshotManagement.deltaLogAsyncUpdateThreadPool.submit(spark) { spark.sparkContext.setLocalProperty("spark.scheduler.pool", "deltaStateUpdatePool") spark.sparkContext.setJobGroup( jobGroup, s"Updating state of Delta table at ${capturedSnapshot.snapshot.path}", interruptOnCancel = true) tryUpdate( isAsync = true, catalogTableOpt) } } catch { case NonFatal(e) if !DeltaUtils.isTesting => // Failed to schedule the future -- fail in testing, but just log it in prod. recordDeltaEvent(this, "delta.snapshot.asyncUpdateFailed", data = Map("exception" -> e)) } } logCurrentSnapshot() currentSnapshot.snapshot } } /** * Try to update ActionLog. If another thread is updating ActionLog, then this method returns * at once and return the current snapshot. The return snapshot may be stale. */ private def tryUpdate( isAsync: Boolean, catalogTableOpt: Option[CatalogTable]): Snapshot = recordDeltaOperation(this, "delta.log.update", Map(TAG_ASYNC -> isAsync.toString)) { if (snapshotLock.tryLock()) { try { updateInternal( isAsync, catalogTableOpt) } finally { snapshotLock.unlock() } } else { currentSnapshot.snapshot } } /** * Queries the store for new delta files and applies them to the current state. * Note: the caller should hold `snapshotLock` before calling this method. */ protected def updateInternal( isAsync: Boolean, catalogTableOpt: Option[CatalogTable]): Snapshot = { val updateStartTimeMs = clock.getTimeMillis() val previousSnapshot = currentSnapshot.snapshot val commitCoordinatorOpt = populateCommitCoordinator(spark, catalogTableOpt, previousSnapshot) val segmentOpt = createLogSegment( previousSnapshot, catalogTableOpt, commitCoordinatorOpt) val newSnapshot = getUpdatedSnapshot( oldSnapshotOpt = Some(previousSnapshot), initialSegmentForNewSnapshot = segmentOpt, initialTableCommitCoordinatorClient = commitCoordinatorOpt, catalogTableOpt = catalogTableOpt, isAsync = isAsync) installSnapshot(newSnapshot, updateStartTimeMs) } /** * Updates and installs a new snapshot in the `currentSnapshot`. * This method takes care of recursively creating new snapshots if the commit-coordinator has * changed. * @param oldSnapshotOpt The previous snapshot, if any. * @param initialSegmentForNewSnapshot the log segment constructed for the new snapshot * @param initialTableCommitCoordinatorClient the commit-coordinator used for constructing the * `initialSegmentForNewSnapshot` * @param catalogTableOpt the optional catalog table to pass to the commit coordinator client. * @param isAsync Whether the update is async. * @return The new snapshot. */ protected def getUpdatedSnapshot( oldSnapshotOpt: Option[Snapshot], initialSegmentForNewSnapshot: Option[LogSegment], initialTableCommitCoordinatorClient: Option[TableCommitCoordinatorClient], catalogTableOpt: Option[CatalogTable], isAsync: Boolean): Snapshot = { var newSnapshot = getSnapshotForLogSegment( oldSnapshotOpt, initialSegmentForNewSnapshot, initialTableCommitCoordinatorClient, catalogTableOpt, isAsync ) // Identify whether the snapshot was created using a "stale" commit-coordinator. If yes, we need // to again invoke [[updateSnapshot]] so that we could get the latest commits from the updated // commit-coordinator client. We need to do it only once as the delta spec mandates the commit // which changes the commit-coordinator to be backfilled. val usedStaleCommitCoordinator = newSnapshot.tableCommitCoordinatorClientOpt.exists { newStore => initialTableCommitCoordinatorClient.forall(!_.semanticsEquals(newStore)) } // If the snapshot is catalog owned and if this call site is invoked from initial snapshot // creation, we can only identify that this is CatalogOwned table after reading it from // filesystem. In this case, now we know that this is CatalogOwned table, do another read to get // the commits from the catalog's commit-coordinator. val needToReadSnapshotUsingCatalogCommitCoordinator = newSnapshot.isCatalogOwned && initialTableCommitCoordinatorClient.isEmpty if (usedStaleCommitCoordinator || needToReadSnapshotUsingCatalogCommitCoordinator) { val commitCoordinatorOpt = populateCommitCoordinator(spark, catalogTableOpt, newSnapshot) val segmentOpt = createLogSegment( newSnapshot, catalogTableOpt, commitCoordinatorOpt) newSnapshot = getSnapshotForLogSegment( Some(newSnapshot), segmentOpt, commitCoordinatorOpt, catalogTableOpt, isAsync) } newSnapshot } /** * Creates a Snapshot for the given `segmentOpt` and handles log segment equality. */ protected def getSnapshotForLogSegment( previousSnapshotOpt: Option[Snapshot], segmentOpt: Option[LogSegment], tableCommitCoordinatorClientOpt: Option[TableCommitCoordinatorClient], catalogTableOpt: Option[CatalogTable], isAsync: Boolean): Snapshot = { val logSegmentsEqual = previousSnapshotOpt.nonEmpty && segmentOpt.nonEmpty && previousSnapshotOpt.get.logSegment == segmentOpt.get getSnapshotForLogSegmentInternal( previousSnapshotOpt, segmentOpt, tableCommitCoordinatorClientOpt, catalogTableOpt, previousSnapshotLogSegmentEquals = logSegmentsEqual, isAsync) } /** * Creates a Snapshot for the given `segmentOpt` * * @param previousSnapshotOpt The previous snapshot, if any. * @param segmentOpt The log segment to create a snapshot for. * @param tableCommitCoordinatorClientOpt The commit coordinator client to use. * @param catalogTableOpt The catalog table to use. * @param prefetchedCheckpoint The prefetched checkpoint and metadata. * @param previousSnapshotLogSegmentEquals Whether the previous snapshot log segment equals the * given segment. If `previousSnapshotOpt` or `segmentOpt` * is empty, this should be false. * @param isAsync Whether the update is async - if so, the checksum will also be computed. * @return The new snapshot. */ protected def getSnapshotForLogSegmentInternal( previousSnapshotOpt: Option[Snapshot], segmentOpt: Option[LogSegment], tableCommitCoordinatorClientOpt: Option[TableCommitCoordinatorClient], catalogTableOpt: Option[CatalogTable], previousSnapshotLogSegmentEquals: Boolean, isAsync: Boolean): Snapshot = { segmentOpt.map { segment => if (previousSnapshotLogSegmentEquals) { // If the previous snapshot log segment equals the given segment, the previous snapshot must // be defined. val previousSnapshot = previousSnapshotOpt.get previousSnapshot.updateLastKnownBackfilledVersion(segment.lastBackfilledVersionInSegment) previousSnapshot } else { val newSnapshot = createSnapshot( initSegment = segment, tableCommitCoordinatorClientOpt = tableCommitCoordinatorClientOpt, catalogTableOpt = catalogTableOpt, checksumOpt = None) previousSnapshotOpt.foreach(logMetadataTableIdChange(_, newSnapshot)) newSnapshot } }.getOrElse { logInfo(log"Creating initial snapshot without metadata, because the directory is empty") new DummySnapshot(logPath, this) } } /** Installs the given `newSnapshot` as the `currentSnapshot` */ protected def installSnapshot(newSnapshot: Snapshot, updateTimestamp: Long): Snapshot = { if (!snapshotLock.isHeldByCurrentThread) { if (DeltaUtils.isTesting) { throw new RuntimeException("DeltaLog snapshot replaced without taking lock") } recordDeltaEvent(this, "delta.update.unsafeReplace") } if (currentSnapshot == null) { // cold snapshot initialization currentSnapshot = CapturedSnapshot(newSnapshot, updateTimestamp) return newSnapshot } val CapturedSnapshot(oldSnapshot, oldTimestamp) = currentSnapshot if (oldSnapshot eq newSnapshot) { // Same snapshot as before, so just refresh the timestamp val timestampToUse = math.max(updateTimestamp, oldTimestamp) currentSnapshot = CapturedSnapshot(newSnapshot, timestampToUse) } else { // Install the new snapshot and uncache the old one currentSnapshot = CapturedSnapshot(newSnapshot, updateTimestamp) oldSnapshot.uncache() } newSnapshot } /** Log a change in the metadata's table id whenever we install a newer version of a snapshot */ private def logMetadataTableIdChange(previousSnapshot: Snapshot, newSnapshot: Snapshot): Unit = { if (previousSnapshot.version > -1 && previousSnapshot.metadata.id != newSnapshot.metadata.id) { val msg = s"Change in the table id detected while updating snapshot. " + s"\nPrevious snapshot = $previousSnapshot\nNew snapshot = $newSnapshot." logWarning(msg) recordDeltaEvent(self, "delta.metadataCheck.update", data = Map( "prevSnapshotVersion" -> previousSnapshot.version, "prevSnapshotMetadata" -> previousSnapshot.metadata, "nextSnapshotVersion" -> newSnapshot.version, "nextSnapshotMetadata" -> newSnapshot.metadata)) } } private def logCurrentSnapshot(): Unit = { val CapturedSnapshot(snapshot, updatedTimestamp) = currentSnapshot var logLine = log"Updated snapshot to ${MDC(DeltaLogKeys.SNAPSHOT, snapshot)}. " logLine += log"Updated at: ${MDC(DeltaLogKeys.TIMESTAMP, updatedTimestamp)}. " // Only check table size when checksum is available to avoid triggering state reconstruction. snapshot.checksumOpt.foreach { checksum => logLine += log"Number of files: ${MDC(DeltaLogKeys.NUM_FILES, checksum.numFiles)} files. " logLine += log"Table size: ${MDC(DeltaLogKeys.NUM_BYTES, checksum.tableSizeBytes)} bytes. " val threshold = spark.conf.get(DeltaSQLConf.DELTA_SNAPSHOT_LOGGING_MAX_FILES_THRESHOLD) if (threshold > 0 && checksum.numFiles > threshold) { logWarning( log"Snapshot at ${MDC(DeltaLogKeys.PATH, logPath)}, " + log"version ${MDC(DeltaLogKeys.VERSION, snapshot.version)} has too many files " + log"(files: ${MDC(DeltaLogKeys.NUM_FILES, checksum.numFiles)}, " + log"threshold: ${MDC(DeltaLogKeys.NUM_FILES, threshold)}). This generally happens " + log"when the table is over-partitioned or have lots of small files. Consider fixing " + log"the partitioning scheme or running OPTIMIZE on the table.") } } logInfo(logLine) } /** * Creates a snapshot for a new delta commit. */ protected def createSnapshotAfterCommit( initSegment: LogSegment, newChecksumOpt: Option[VersionChecksum], tableCommitCoordinatorClientOpt: Option[TableCommitCoordinatorClient], catalogTableOpt: Option[CatalogTable], committedVersion: Long): Snapshot = { logInfo( log"Creating a new snapshot v${MDC(DeltaLogKeys.VERSION, initSegment.version)} " + log"for commit version ${MDC(DeltaLogKeys.VERSION2, committedVersion)}") // Guard against race condition when a txn commits after this txn but before // reaching createLogSegment(...) above. var checksumContext = "incrementalCommit" val passedChecksumIsUsable = newChecksumOpt.isDefined && committedVersion == initSegment.version val snapChecksumOpt = newChecksumOpt .filter(_ => passedChecksumIsUsable) .orElse { checksumContext = "fallbackToReadChecksumFile" readChecksum(initSegment.version) } def createSnapshotWithCrc(checksumOpt: Option[VersionChecksum]): Snapshot = { createSnapshot( initSegment, tableCommitCoordinatorClientOpt, catalogTableOpt, checksumOpt) } var newSnapshot = createSnapshotWithCrc(snapChecksumOpt) // Skip validation in 0th commit when number of files in underlying snapshot is 0 in order to // avoid state reconstruction - since there is nothing to verify from allFilesInCrc perspective. val skipValidationForZerothCommit = committedVersion == 0 && newChecksumOpt.forall { crc => crc.numFiles == 0 && crc.allFiles.forall(_.isEmpty) } if (passedChecksumIsUsable && !skipValidationForZerothCommit && Snapshot.allFilesInCrcVerificationEnabled(spark, unsafeVolatileSnapshot)) { snapChecksumOpt.collect { case crc if !newSnapshot.validateFileListAgainstCRC(crc, contextOpt = Some("triggeredFromCommit")) => // If the verification for [[VersionChecksum.allFiles]] failed, then strip off `allFiles` // and create the snapshot again with new CRC (without addFiles in it). newSnapshot = createSnapshotWithCrc(snapChecksumOpt.map(_.copy(allFiles = None))) } } // Verify when enabled or when tests run to help future proof IC if (shouldVerifyIncrementalCommit) { val crcIsValid = try { // NOTE: Validation is a no-op with incremental commit disabled. newSnapshot.validateChecksum(Map("context" -> checksumContext)) } catch { case e: IllegalStateException if !DeltaUtils.isTesting => logWarning(log"Incremental checksum validation failed: " + log"${MDC(DeltaLogKeys.ERROR, e.getMessage)}") false } if (!crcIsValid) { // Create snapshot without incremental checksum. This will fallback to creating // a checksum based on state reconstruction. Disable incremental commit to avoid // further error triggers in this session. logWarning(log"Disabling incremental commit for this session due to checksum " + log"validation failure at version " + log"${MDC(DeltaLogKeys.VERSION, newSnapshot.version)}") spark.sessionState.conf.setConf(DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED, false) spark.sessionState.conf.setConf(DeltaSQLConf.DELTA_WRITE_SET_TRANSACTIONS_IN_CRC, false) return createSnapshotWithCrc(checksumOpt = None) } } newSnapshot } /** * Called after committing a transaction and updating the state of the table. * * @param committedVersion the version that was committed * @param commit information about the commit file. * @param newChecksumOpt the checksum for the new commit, if available. * Usually None, since the commit would have just finished. * @param preCommitLogSegment the log segment of the table prior to commit * @param catalogTableOpt the current catalog table */ def updateAfterCommit( committedVersion: Long, commit: Commit, newChecksumOpt: Option[VersionChecksum], preCommitLogSegment: LogSegment, catalogTableOpt: Option[CatalogTable]): Snapshot = { var previousSnapshot: Snapshot = null recordDeltaOperation(this, "delta.log.updateAfterCommit") { val updatedSnapshot = withSnapshotLockInterruptibly { val updateTimestamp = clock.getTimeMillis() previousSnapshot = currentSnapshot.snapshot // Somebody else could have already updated the snapshot while we waited for the lock if (committedVersion <= previousSnapshot.version) return previousSnapshot val commitCoordinatorOpt = populateCommitCoordinator( spark, catalogTableOpt, previousSnapshot ) val segment = getLogSegmentAfterCommit( committedVersion, newChecksumOpt, preCommitLogSegment, commit, commitCoordinatorOpt, catalogTableOpt, previousSnapshot.checkpointProvider) // This likely implies a list-after-write inconsistency if (segment.version < committedVersion) { recordDeltaEvent(this, "delta.commit.inconsistentList", data = Map( "committedVersion" -> committedVersion, "currentVersion" -> segment.version )) throw DeltaErrors.invalidCommittedVersion(committedVersion, segment.version) } val newSnapshot = createSnapshotAfterCommit( segment, newChecksumOpt, commitCoordinatorOpt, catalogTableOpt, committedVersion) installSnapshot(newSnapshot, updateTimestamp) } logMetadataTableIdChange(previousSnapshot, updatedSnapshot) logCurrentSnapshot() updatedSnapshot } } /** Get the snapshot at `version`. */ def getSnapshotAt( version: Long, lastCheckpointHint: Option[CheckpointInstance] = None, catalogTableOpt: Option[CatalogTable] = None, enforceTimeTravelWithinDeletedFileRetention: Boolean = false): Snapshot = { getSnapshotAt( version, lastCheckpointHint, lastCheckpointProvider = None, catalogTableOpt, enforceTimeTravelWithinDeletedFileRetention) } /** * Get the snapshot at `version` using the given `lastCheckpointProvider` or `lastCheckpointHint` * as the listing hint. */ private[delta] def getSnapshotAt( version: Long, lastCheckpointHint: Option[CheckpointInstance], lastCheckpointProvider: Option[CheckpointProvider], catalogTableOpt: Option[CatalogTable], enforceTimeTravelWithinDeletedFileRetention: Boolean): Snapshot = { // See if the version currently cached on the cluster satisfies the requirement val currentSnapshot = unsafeVolatileSnapshot val upperBoundSnapshot = if (currentSnapshot.version >= version) { // current snapshot is already newer than what we are looking for. so it could be used as // upper bound. currentSnapshot } else { val latestSnapshot = update(catalogTableOpt = catalogTableOpt) if (latestSnapshot.version < version) { throwNonExistentVersionError(version) } latestSnapshot } if (upperBoundSnapshot.version == version) { return upperBoundSnapshot } val commitCoordinatorOpt = populateCommitCoordinator(spark, catalogTableOpt, upperBoundSnapshot) val (lastCheckpointInfoOpt, lastCheckpointProviderOpt) = lastCheckpointProvider match { // NOTE: We must ignore any hint whose version is higher than the requested version. case Some(checkpointProvider) if checkpointProvider.version <= version => // Prefer the last checkpoint provider hint, because it doesn't require any I/O to use. None -> Some(checkpointProvider) case _ => val lastCheckpointInfoForListing = lastCheckpointHint .filter(_.version <= version) .orElse(findLastCompleteCheckpointBefore(version)) .map(manuallyLoadCheckpoint) lastCheckpointInfoForListing -> None } val logSegmentOpt = createLogSegment( versionToLoad = Some(version), oldCheckpointProviderOpt = lastCheckpointProviderOpt, tableCommitCoordinatorClientOpt = commitCoordinatorOpt, catalogTableOpt = catalogTableOpt, lastCheckpointInfo = lastCheckpointInfoOpt) val logSegment = logSegmentOpt.getOrElse { // We can't return InitialSnapshot because our caller asked for a specific snapshot version. throw DeltaErrors.logFileNotFoundException( logPath, Some(version), getCheckpointVersion(lastCheckpointInfoOpt, lastCheckpointProviderOpt)) } val ret = createSnapshot( initSegment = logSegment, tableCommitCoordinatorClientOpt = commitCoordinatorOpt, catalogTableOpt = catalogTableOpt, checksumOpt = None) if (enforceTimeTravelWithinDeletedFileRetention) { enforceTimeTravelWithinDeletedFileRetentionDuration(ret, currentSnapshot) } ret } private def enforceTimeTravelWithinDeletedFileRetentionDuration( targetSnapshot: Snapshot, latestSnapshot: Snapshot): Unit = { if (!SparkSession.active.sessionState.conf.getConf( DeltaSQLConf.ENFORCE_TIME_TRAVEL_WITHIN_DELETED_FILE_RETENTION_DURATION)) { return } // Skip enforcement for delta-sharing tables since they create a faked delta-log // where the version timestamp may be set to 0. val deltasharingLogFileSystemSchema = // Cannot import io.delta.sharing.spark.DeltaSharingLogFileSystemConstants.SCHEME // in the current (spark) module since sharing depends on spark; falling back to // string comparison. "delta-sharing-log" if (logPath.toUri.getScheme == deltasharingLogFileSystemSchema) { return } // Time travel to the latest version is always allowed if (targetSnapshot.version == latestSnapshot.version) return val deletedFileRetentionDuration = DeltaLog.tombstoneRetentionMillis(latestSnapshot.metadata) val currentTime = clock.getTimeMillis() if (targetSnapshot.timestamp < (currentTime - deletedFileRetentionDuration)) { recordDeltaEvent(this, s"delta.timeTravel.fail", data = Map( // Log the cached version of the table on the cluster "latestVersion" -> latestSnapshot.version, "queriedVersion" -> targetSnapshot.version, "currentTimestamp" -> currentTime, "targetSnapshotTimestamp" -> targetSnapshot.timestamp, "deletedFileRetentionDuration" -> deletedFileRetentionDuration )) throw DeltaErrors.timeTravelBeyondDeletedFileRetentionDurationException( deletedFileRetentionDuration.millis.toHours.toString) } } // Populate commit coordinator using catalogOpt if the snapshot is catalog owned. protected def populateCommitCoordinator( spark: SparkSession, catalogTableOpt: Option[CatalogTable], snapshot: Snapshot) : Option[TableCommitCoordinatorClient] = { if (snapshot.isCatalogOwned) { CatalogOwnedTableUtils.populateTableCommitCoordinatorFromCatalog( spark, catalogTableOpt, snapshot ) } else { snapshot.tableCommitCoordinatorClientOpt } } // Visible for testing private[delta] def getCapturedSnapshot(): CapturedSnapshot = currentSnapshot } object SnapshotManagement extends DeltaLogging { // A thread pool for reading checkpoint files and collecting checkpoint v2 actions like // checkpointMetadata, sidecarFiles. private[delta] lazy val checkpointV2ThreadPool = { val numThreads = SparkSession.active.sessionState.conf.getConf( DeltaSQLConf.CHECKPOINT_V2_DRIVER_THREADPOOL_PARALLELISM) DeltaThreadPool("checkpointV2-threadpool", numThreads) } protected[delta] lazy val deltaLogAsyncUpdateThreadPool = { val tpe = ThreadUtils.newDaemonCachedThreadPool("delta-state-update", 8) new DeltaThreadPool(tpe) } private lazy val commitCoordinatorGetCommitsThreadPool = { val numThreads = SparkSession.active.sessionState.conf .getConf(DeltaSQLConf.COORDINATED_COMMITS_GET_COMMITS_THREAD_POOL_SIZE) val tpe = ThreadUtils.newDaemonCachedThreadPool("commit-coordinator-get-commits", numThreads) new DeltaThreadPool(tpe) } /** * - Verify the versions are contiguous. * - Verify the versions start with `expectedStartVersion` if it's specified. * - Verify the versions end with `expectedEndVersion` if it's specified. */ def verifyDeltaVersions( spark: SparkSession, versions: Array[Long], expectedStartVersion: Option[Long], expectedEndVersion: Option[Long], cachedSnapshot: Option[Snapshot]): Unit = { if (versions.nonEmpty) { // Turn this to a vector so that we can compare it with a range. val deltaVersions = versions.toVector if ((deltaVersions.head to deltaVersions.last) != deltaVersions) { // [[cachedSnapshot]] maybe null (e.g., uninitialized snapshot being passed in) // in some cases, which needs to be explicitly filtered out. val snapshot = cachedSnapshot.filter(_ != null) recordDeltaEvent( deltaLog = null, opType = "delta.exceptions.deltaVersionsNotContiguous", data = Map( // Remove the first element of the stack trace since this represents // the [[Thread.getStackTrace]] call itself. "stackTrace" -> Thread.currentThread().getStackTrace.tail.mkString("\n\t"), "startVersion" -> deltaVersions.head, "endVersion" -> deltaVersions.last, "versionToLoad" -> expectedEndVersion.getOrElse(-1L), "unsafeVolatileSnapshot.latestCheckpointVersion" -> snapshot.map(_.checkpointProvider.version).getOrElse(-1L), "unsafeVolatileSnapshot.latestSnapshotVersion" -> snapshot.map(_.version).getOrElse(-1L), "unsafeVolatileSnapshot.checksumOpt" -> snapshot.map(_.checksumOpt).orNull )) throw DeltaErrors.deltaVersionsNotContiguousException( spark = spark, deltaVersions = deltaVersions, startVersion = deltaVersions.head, endVersion = deltaVersions.last, // `expectedEndVersion` is the version we'd like to construct/load the [[Snapshot]], // pass -1L if it's not available/specified. versionToLoad = expectedEndVersion.getOrElse(-1L)) } } expectedStartVersion.foreach { v => require(versions.nonEmpty && versions.head == v, "Did not get the first delta " + s"file version: $v to compute Snapshot") } expectedEndVersion.foreach { v => require(versions.nonEmpty && versions.last == v, "Did not get the last delta " + s"file version: $v to compute Snapshot") } } def appendCommitToLogSegment( oldLogSegment: LogSegment, commitFileStatus: FileStatus, committedVersion: Long): LogSegment = { require(oldLogSegment.version + 1 == committedVersion) oldLogSegment.copy( version = committedVersion, deltas = oldLogSegment.deltas :+ commitFileStatus, lastCommitFileModificationTimestamp = commitFileStatus.getModificationTime) } } /** A serializable variant of HDFS's FileStatus. */ case class SerializableFileStatus( path: String, length: Long, isDir: Boolean, modificationTime: Long) { // Important note! This is very expensive to compute, but we don't want to cache it // as a `val` because Paths internally contain URIs and therefore consume lots of memory. @JsonIgnore def getHadoopPath: Path = new Path(path) def toFileStatus: FileStatus = { new FileStatus(length, isDir, 0, 0, modificationTime, new Path(path)) } override def equals(obj: Any): Boolean = obj match { // We only compare the paths to stay consistent with FileStatus.equals. case other: SerializableFileStatus => Objects.equals(path, other.path) case _ => false } // We only use the path to stay consistent with FileStatus.hashCode. override def hashCode(): Int = Objects.hashCode(path) } object SerializableFileStatus { def fromStatus(status: FileStatus): SerializableFileStatus = { SerializableFileStatus( Option(status.getPath).map(_.toString).orNull, status.getLen, status.isDirectory, status.getModificationTime) } val EMPTY: SerializableFileStatus = fromStatus(new FileStatus()) } /** * Provides information around which files in the transaction log need to be read to create * the given version of the log. * * @param logPath The path to the _delta_log directory * @param version The Snapshot version to generate * @param deltas The delta commit files (.json) to read * @param checkpointProvider provider to give information about Checkpoint files. * @param lastCommitFileModificationTimestamp The "unadjusted" file modification timestamp of the * last commit within this segment. By unadjusted, we mean that the commit timestamps may * not necessarily be monotonically increasing for the commits within this segment. */ case class LogSegment( logPath: Path, version: Long, deltas: Seq[FileStatus], checkpointProvider: UninitializedCheckpointProvider, lastCommitFileModificationTimestamp: Long) { override def hashCode(): Int = logPath.hashCode() * 31 + (lastCommitFileModificationTimestamp % 10000).toInt /** * An efficient way to check if a cached Snapshot's contents actually correspond to a new * segment returned through file listing. */ override def equals(obj: Any): Boolean = { obj match { case other: LogSegment => version == other.version && logPath == other.logPath && checkpointProvider.version == other.checkpointProvider.version && lastMatchingBackfilledCommitIsEqual(other) case _ => false } } private def lastMatchingBackfilledCommitIsEqual(other: LogSegment): Boolean = { def fileStatusEquals(fileStatus1: FileStatus, fileStatus2: FileStatus): Boolean = { fileStatus1.getPath == fileStatus2.getPath && fileStatus1.getLen == fileStatus2.getLen && fileStatus1.getModificationTime == fileStatus2.getModificationTime } val backfilledPrefixThis = deltas.takeWhile(isBackfilledDeltaFile) val backfilledPrefixOther = other.deltas.takeWhile(isBackfilledDeltaFile) val sizeToAnalyze = math.min(backfilledPrefixThis.size, backfilledPrefixOther.size) val backfilledPrefixThisStripped = backfilledPrefixThis.take(sizeToAnalyze) val backfilledPrefixOtherStripped = backfilledPrefixOther.take(sizeToAnalyze) backfilledPrefixThisStripped.zip(backfilledPrefixOtherStripped) .forall { case (delta1, delta2) => fileStatusEquals(delta1, delta2) } && checkpointProvider.topLevelFiles.size == other.checkpointProvider.topLevelFiles.size && checkpointProvider.topLevelFiles.zip(other.checkpointProvider.topLevelFiles).forall { case (cp1, cp2) => fileStatusEquals(cp1, cp2) } } private[delta] lazy val lastBackfilledVersionInSegment = // This works if the last backfilled file is a minor-compaction, because // FileNames.getFileVersion returns the minor-compaction end version, // which correctly initializes the lastBackfilledVersionInSegment. CoordinatedCommitsUtils.getLastBackfilledFile(deltas).map(getFileVersion) .getOrElse(checkpointProvider.version) } /** Exception thrown When [[TableCommitCoordinatorClient.getCommits]] fails due to any reason. */ class CommitCoordinatorGetCommitsFailedException(cause: Throwable) extends Exception(cause) object LogSegment { def apply( logPath: Path, version: Long, deltas: Seq[FileStatus], checkpointProviderOpt: Option[UninitializedCheckpointProvider], lastCommitTimestamp: Long): LogSegment = { val checkpointProvider = checkpointProviderOpt.getOrElse(EmptyCheckpointProvider) LogSegment(logPath, version, deltas, checkpointProvider, lastCommitTimestamp) } /** The LogSegment for an empty transaction log directory. */ def empty(path: Path): LogSegment = LogSegment( logPath = path, version = -1L, deltas = Nil, checkpointProviderOpt = None, lastCommitTimestamp = -1L) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/SnapshotState.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.actions.{Metadata, Protocol, SetTransaction} import org.apache.spark.sql.delta.actions.DomainMetadata import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.DeletedRecordCountsHistogram import org.apache.spark.sql.delta.stats.DeletedRecordCountsHistogramUtils import org.apache.spark.sql.delta.stats.FileSizeHistogram import org.apache.spark.sql.{Column, DataFrame} import org.apache.spark.sql.functions.{coalesce, col, collect_set, count, last, lit, sum, when} import org.apache.spark.util.Utils /** * Metrics and metadata computed around the Delta table. * * @param sizeInBytes The total size of the table (of active files, not including tombstones). * @param numOfSetTransactions Number of streams writing to this table. * @param numOfFiles The number of files in this table. * @param numOfRemoves The number of tombstones in the state. * @param numDeletedRecordsOpt The total number of records deleted with Deletion Vectors. * @param numDeletionVectorsOpt The number of Deletion Vectors present in the table. * @param numOfMetadata The number of metadata actions in the state. Should be 1. * @param numOfProtocol The number of protocol actions in the state. Should be 1. * @param setTransactions The streaming queries writing to this table. * @param metadata The metadata of the table. * @param protocol The protocol version of the Delta table. * @param fileSizeHistogram A Histogram class tracking the file counts and total bytes * in different size ranges. * @param deletedRecordCountsHistogramOpt A histogram of deletion records counts distribution * for all files. */ case class SnapshotState( sizeInBytes: Long, numOfSetTransactions: Long, numOfFiles: Long, numOfRemoves: Long, numDeletedRecordsOpt: Option[Long], numDeletionVectorsOpt: Option[Long], numOfMetadata: Long, numOfProtocol: Long, setTransactions: Seq[SetTransaction], domainMetadata: Seq[DomainMetadata], metadata: Metadata, protocol: Protocol, fileSizeHistogram: Option[FileSizeHistogram] = None, deletedRecordCountsHistogramOpt: Option[DeletedRecordCountsHistogram] = None ) /** * A helper class that manages the SnapshotState for a given snapshot. Will generate it only * when necessary. */ trait SnapshotStateManager extends DeltaLogging { self: Snapshot => // For implicits which re-use Encoder: import implicits._ /** Whether computedState is already computed or not */ @volatile protected var _computedStateTriggered: Boolean = false /** A map to look up transaction version by appId. */ lazy val transactions: Map[String, Long] = setTransactions.map(t => t.appId -> t.version).toMap /** * Compute the SnapshotState of a table. Uses the stateDF from the Snapshot to extract * the necessary stats. */ protected lazy val computedState: SnapshotState = { withStatusCode("DELTA", s"Compute snapshot for version: $version") { recordFrameProfile("Delta", "snapshot.computedState") { val startTime = System.nanoTime() val _computedState = extractComputedState(stateDF) if (_computedState.protocol == null) { recordDeltaEvent( deltaLog, opType = "delta.assertions.missingAction", data = Map( "version" -> version.toString, "action" -> "Protocol", "source" -> "Snapshot")) throw DeltaErrors.actionNotFoundException("protocol", version) } else if (_computedState.protocol != protocol) { recordDeltaEvent( deltaLog, opType = "delta.assertions.mismatchedAction", data = Map( "version" -> version.toString, "action" -> "Protocol", "source" -> "Snapshot", "computedState.protocol" -> _computedState.protocol, "extracted.protocol" -> protocol)) throw DeltaErrors.actionNotFoundException("protocol", version) } if (_computedState.metadata == null) { recordDeltaEvent( deltaLog, opType = "delta.assertions.missingAction", data = Map( "version" -> version.toString, "action" -> "Metadata", "source" -> "Metadata")) throw DeltaErrors.actionNotFoundException("metadata", version) } else if (_computedState.metadata != metadata) { recordDeltaEvent( deltaLog, opType = "delta.assertions.mismatchedAction", data = Map( "version" -> version.toString, "action" -> "Metadata", "source" -> "Snapshot", "computedState.metadata" -> _computedState.metadata, "extracted.metadata" -> metadata)) throw DeltaErrors.actionNotFoundException("metadata", version) } _computedStateTriggered = true _computedState } } } /** * Extract the SnapshotState from the provided dataframe of actions. Requires that the dataframe * has already been deduplicated (either through logReplay or some other method). */ protected def extractComputedState(stateDF: DataFrame): SnapshotState = { recordFrameProfile("Delta", "snapshot.computedState.aggregations") { val aggregations = aggregationsToComputeState.map { case (alias, agg) => agg.as(alias) }.toSeq stateDF.select(aggregations: _*).as[SnapshotState].first() } } /** * A Map of alias to aggregations which needs to be done to calculate the `computedState` */ protected def aggregationsToComputeState: Map[String, Column] = { val checksumDVMetricsEnabled = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_CHECKSUM_DV_METRICS_ENABLED) val deletedRecordCountsHistogramEnabled = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_DELETED_RECORD_COUNTS_HISTOGRAM_ENABLED) lazy val persistentDVsOnTableSupported = DeletionVectorUtils.deletionVectorsWritable(this) val computeChecksumDVMetrics = checksumDVMetricsEnabled && persistentDVsOnTableSupported val persistentDVsAggs = if (computeChecksumDVMetrics) { Map( "numDeletedRecordsOpt" -> sum(coalesce(col("add.deletionVector.cardinality"), lit(0L))), "numDeletionVectorsOpt" -> count(col("add.deletionVector"))) } else { Map("numDeletedRecordsOpt" -> lit(null), "numDeletionVectorsOpt" -> lit(null)) } val histogramDVsAggExpr = if (computeChecksumDVMetrics && deletedRecordCountsHistogramEnabled) { DeletedRecordCountsHistogramUtils.histogramAggregate( when(col("add").isNotNull, coalesce(col("add.deletionVector.cardinality"), lit(0L)))) } else { lit(null).cast(DeletedRecordCountsHistogram.schema) } val histogramDVsAgg = Seq("deletedRecordCountsHistogramOpt" -> histogramDVsAggExpr) Map( // sum may return null for empty data set. "sizeInBytes" -> coalesce(sum(col("add.size")), lit(0L)), "numOfSetTransactions" -> count(col("txn")), "numOfFiles" -> count(col("add")), "numOfRemoves" -> count(col("remove")), "numOfMetadata" -> count(col("metaData")), "numOfProtocol" -> count(col("protocol")), "setTransactions" -> collect_set(col("txn")), "domainMetadata" -> collect_set(col("domainMetadata")), "metadata" -> last(col("metaData"), ignoreNulls = true), "protocol" -> last(col("protocol"), ignoreNulls = true), "fileSizeHistogram" -> lit(null).cast(FileSizeHistogram.schema) ) ++ persistentDVsAggs ++ histogramDVsAgg } /** * The following is a list of convenience methods for accessing the computedState. */ def sizeInBytes: Long = computedState.sizeInBytes def numOfSetTransactions: Long = computedState.numOfSetTransactions def numOfFiles: Long = computedState.numOfFiles def numOfRemoves: Long = computedState.numOfRemoves def numOfMetadata: Long = computedState.numOfMetadata def numOfProtocol: Long = computedState.numOfProtocol def setTransactions: Seq[SetTransaction] = computedState.setTransactions def fileSizeHistogram: Option[FileSizeHistogram] = computedState.fileSizeHistogram def domainMetadata: Seq[DomainMetadata] = computedState.domainMetadata protected[delta] def sizeInBytesIfKnown: Option[Long] = Some(sizeInBytes) protected[delta] def setTransactionsIfKnown: Option[Seq[SetTransaction]] = Some(setTransactions) protected[delta] def numOfFilesIfKnown: Option[Long] = Some(numOfFiles) protected[delta] def domainMetadatasIfKnown: Option[Seq[DomainMetadata]] = Some(domainMetadata) def numDeletedRecordsOpt: Option[Long] = computedState.numDeletedRecordsOpt def numDeletionVectorsOpt: Option[Long] = computedState.numDeletionVectorsOpt def deletedRecordCountsHistogramOpt: Option[DeletedRecordCountsHistogram] = computedState.deletedRecordCountsHistogramOpt protected def deletionVectorsReadableAndMetricsEnabled: Boolean = { val checksumDVMetricsEnabled = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_CHECKSUM_DV_METRICS_ENABLED) val dvsReadable = DeletionVectorUtils.deletionVectorsReadable(snapshotToScan) checksumDVMetricsEnabled && dvsReadable } protected def deletionVectorsReadableAndHistogramEnabled: Boolean = { val deletedRecordCountsHistogramEnabled = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_DELETED_RECORD_COUNTS_HISTOGRAM_ENABLED) deletionVectorsReadableAndMetricsEnabled && deletedRecordCountsHistogramEnabled } /** Generate a default SnapshotState of a new table given the table metadata and the protocol. */ protected def initialState(metadata: Metadata, protocol: Protocol): SnapshotState = { val deletedRecordCountsHistogramOpt = if (spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_DELETED_RECORD_COUNTS_HISTOGRAM_ENABLED)) { Some(DeletedRecordCountsHistogramUtils.emptyHistogram) } else None SnapshotState( sizeInBytes = 0L, numOfSetTransactions = 0L, numOfFiles = 0L, numOfRemoves = 0L, // DV metrics are initialized to Some(0) to allow incremental computation. For tables where // DVs are disabled, there are turned to None by the incremental computation. numDeletedRecordsOpt = Some(0), numDeletionVectorsOpt = Some(0), numOfMetadata = 1L, numOfProtocol = 1L, setTransactions = Nil, domainMetadata = Nil, metadata = metadata, protocol = protocol, deletedRecordCountsHistogramOpt = deletedRecordCountsHistogramOpt ) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/SubqueryTransformerHelper.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.catalyst.expressions.SubqueryExpression import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Subquery, SupportsSubquery} /** * Trait to allow processing a special transformation of [[SubqueryExpression]] * instances in a query plan. */ trait SubqueryTransformerHelper { /** * Transform all nodes matched by the rule in the query plan rooted at given `plan`. * It traverses the tree starting from the leaves, whenever a [[SubqueryExpression]] * expression is encountered, given [[rule]] is applied to the subquery plan `plan` * in [[SubqueryExpression]] starting from the `plan` root until leaves. * * This is slightly different behavior compared to [[QueryPlan.transformUpWithSubqueries]] * or [[QueryPlan.transformDownWithSubqueries]] * * It requires that the given plan already gone through [[OptimizeSubqueries]] and the * root node denoting a subquery is removed and optimized appropriately. */ def transformWithSubqueries(plan: LogicalPlan) (rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan = { require(!isSubqueryRoot(plan)) transformSubqueries(plan, rule) transform (rule) } /** Is the give plan a subquery root. */ def isSubqueryRoot(plan: LogicalPlan): Boolean = { plan.isInstanceOf[Subquery] || plan.isInstanceOf[SupportsSubquery] } private def transformSubqueries( plan: LogicalPlan, rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan = { import org.apache.spark.sql.delta.implicits._ plan transformAllExpressionsUp { case subquery: SubqueryExpression => subquery.withNewPlan(transformWithSubqueries(subquery.plan)(rule)) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/TableFeature.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.Locale import org.apache.spark.sql.delta.RowId.RowTrackingMetadataDomain import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillCommand import org.apache.spark.sql.delta.constraints.{Constraints, Invariants} import org.apache.spark.sql.delta.coordinatedcommits.CoordinatedCommitsUtils import org.apache.spark.sql.delta.redirect.{RedirectReaderWriter, RedirectWriterOnly} import org.apache.spark.sql.delta.schema.SchemaMergingUtils import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} import org.apache.spark.sql.delta.util.FileNames import org.apache.spark.sql.{Dataset, SparkSession} import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.functions.col import org.apache.spark.sql.types.TimestampNTZType /* --------------------------------------- * | Table features base class definitions | * --------------------------------------- */ /** * A base class for all table features. * * A feature can be explicitly supported by a table's protocol when the protocol contains a * feature's `name`. Writers (for writer-only features) or readers and writers (for reader-writer * features) must recognize supported features and must handle them appropriately. * * A table feature that released before Delta Table Features (reader version 3 and writer version * 7) is considered as a legacy feature. Legacy features are implicitly supported * when (a) the protocol does not support table features, i.e., has reader version less than 3 or * writer version less than 7 and (b) the feature's minimum reader/writer version is less than or * equal to the current protocol's reader/writer version. * * Separately, a feature can be automatically supported by a table's metadata when certain * feature-specific table properties are set. For example, `changeDataFeed` is automatically * supported when there's a table property `delta.enableChangeDataFeed=true`. This is independent * of the table's enabled features. When a feature is supported (explicitly or implicitly) by the * table protocol but its metadata requirements are not satisfied, then clients still have to * understand the feature (at least to the extent that they can read and preserve the existing * data in the table that uses the feature). See the documentation of * [[FeatureAutomaticallyEnabledByMetadata]] for more information. * * @param name * a globally-unique string indicator to represent the feature. All characters must be letters * (a-z, A-Z), digits (0-9), '-', or '_'. Words must be in camelCase. * @param minReaderVersion * the minimum reader version this feature requires. For a feature that can only be explicitly * supported, this is either `0` or `3` (the reader protocol version that supports table * features), depending on the feature is writer-only or reader-writer. For a legacy feature * that can be implicitly supported, this is the first protocol version which the feature is * introduced. * @param minWriterVersion * the minimum writer version this feature requires. For a feature that can only be explicitly * supported, this is the writer protocol `7` that supports table features. For a legacy feature * that can be implicitly supported, this is the first protocol version which the feature is * introduced. */ // @TODO: distinguish Delta and 3rd-party features and give appropriate error messages sealed abstract class TableFeature( val name: String, val minReaderVersion: Int, val minWriterVersion: Int) extends java.io.Serializable { require(name.forall(c => c.isLetterOrDigit || c == '-' || c == '_')) /** * Get a [[Protocol]] object stating the minimum reader and writer versions this feature * requires. For a feature that can only be explicitly supported, this method returns a protocol * version that supports table features, either `(0,7)` or `(3,7)` depending on the feature is * writer-only or reader-writer. For a legacy feature that can be implicitly supported, this * method returns the first protocol version which introduced the said feature. * * For all features, if the table's protocol version does not support table features, then the * minimum protocol version is enough. However, if the protocol version supports table features * for the feature type (writer-only or reader-writer), then the minimum protocol version is not * enough to support a feature. In this case the feature must also be explicitly listed in the * appropriate feature sets in the [[Protocol]]. */ def minProtocolVersion: Protocol = Protocol(minReaderVersion, minWriterVersion) /** Determine if this feature applies to both readers and writers. */ def isReaderWriterFeature: Boolean = this.isInstanceOf[ReaderWriterFeatureType] /** * Determine if this feature is a legacy feature. See the documentation of [[TableFeature]] for * more information. */ def isLegacyFeature: Boolean = this.isInstanceOf[LegacyFeatureType] /** * True if this feature can be removed. */ def isRemovable: Boolean = this.isInstanceOf[RemovableFeature] /** * True if the addition of this feature in the protocol is expected to fail concurrent * transactions. This is desirable for features that are implicitly enabled by being present * in the protocol, and also impose write-time requirements that need to be respected by all * writers beyond the protocol upgrade. Note that features that do reconciliation at conflict * checking time (e.g. RowTrackingFeature) should return false. */ def failConcurrentTransactionsAtUpgrade: Boolean = true /** * Set of table features that this table feature depends on. I.e. the set of features that need * to be enabled if this table feature is enabled. */ def requiredFeatures: Set[TableFeature] = Set.empty } /** A trait to indicate a feature applies to readers and writers. */ sealed trait ReaderWriterFeatureType /** A trait to indicate a feature is legacy, i.e., released before Table Features. */ sealed trait LegacyFeatureType /** * A trait indicating this feature can be automatically enabled via a change in a table's * metadata, e.g., through setting particular values of certain feature-specific table properties. * * When the feature's metadata requirements are satisfied for new tables, or for * existing tables when [[automaticallyUpdateProtocolOfExistingTables]] set to `true`, the * client will silently add the feature to the protocol's `readerFeatures` and/or * `writerFeatures`. Otherwise, a proper protocol version bump must be present in the same * transaction. */ sealed trait FeatureAutomaticallyEnabledByMetadata { this: TableFeature => /** * Whether the feature can automatically update the protocol of an existing table when the * metadata requirements are satisfied. As a rule of thumb, a table feature that requires * explicit operations (e.g., turning on a table property) should set this flag to `true`, while * features that are used implicitly (e.g., when using a new data type) should set this flag to * `false`. */ def automaticallyUpdateProtocolOfExistingTables: Boolean = this.isLegacyFeature /** * Determine whether the feature must be supported and enabled because its metadata requirements * are satisfied. */ def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean require( !this.isLegacyFeature || automaticallyUpdateProtocolOfExistingTables, "Legacy feature must be auto-update capable.") } /** * A trait indicating a feature can be removed. Classes that extend the trait need to * implement the following four functions: * * a) preDowngradeCommand. This is where all required actions for removing the feature are * implemented. For example, to remove the DVs feature we need to remove metadata config * and purge all DVs from table. This action takes place before the protocol downgrade in * separate commit(s). Note, the command needs to be implemented in a way concurrent * transactions do not nullify the effect. For example, disabling DVs on a table before * purging will stop concurrent transactions from adding DVs. During protocol downgrade * we perform a validation in [[validateDropInvariants]] to make sure all invariants still hold. * * b) validateDropInvariants. Add any feature-specific checks before proceeding to the protocol * downgrade. This function is guaranteed to be called at the latest version before the * protocol downgrade is committed to the table. When the protocol downgrade txn conflicts, * the validation is repeated against the winning txn snapshot. As soon as the protocol * downgrade succeeds, all subsequent interleaved txns are aborted. * The implementation should return true if there are no feature traces in the latest * version. False otherwise. * * c) requiresHistoryProtection. It indicates whether the feature leaves traces in the table * history that may result in incorrect behaviour if the table is read/written by a client * that does not support the feature. This is by default true for all reader+writer features * and false for writer features. * WARNING: Disabling [[requiresHistoryProtection]] for relevant features could result in * incorrect snapshot reconstruction. * * d) actionUsesFeature. For features that require history truncation we verify whether past * versions contain any traces of the removed feature. This is achieved by calling * [[actionUsesFeature]] for every action of every reachable commit version in the log. * Note, a feature may leave traces in both data and metadata. Depending on the feature, we * need to check several types of actions such as Metadata, AddFile, RemoveFile etc. * * WARNING: actionUsesFeature should not check Protocol actions for the feature being removed, * because at the time actionUsesFeature is invoked the protocol downgrade did not happen yet. * Thus, the feature-to-remove is still active. As a result, any unrelated operations that * produce a protocol action (while we are waiting for the retention period to expire) will * "carry" the feature-to-remove. Checking protocol for that feature would result in an * unnecessary failure during the history validation of the next DROP FEATURE call. Note, * while the feature-to-remove is supported in the protocol we cannot generate a legit protocol * action that adds support for that feature since it is already supported. * * Furthermore, methods `tablePropertiesToRemoveAtDowngradeCommit` and * `actionsToIncludeAtDowngradeCommit` can be optionally implemented. They can be used for * defining properties/actions that need to be removed/included at the protocol downgrade * commit. */ sealed trait RemovableFeature { self: TableFeature => def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean def requiresHistoryProtection: Boolean = isReaderWriterFeature def actionUsesFeature(action: Action): Boolean def tablePropertiesToRemoveAtDowngradeCommit: Seq[String] = Seq.empty def actionsToIncludeAtDowngradeCommit(snapshot: Snapshot): Seq[Action] = Seq.empty /** * Examines all historical commits for traces of the removableFeature. * This is achieved as follows: * * 1) We find the earliest valid checkpoint, recreate a snapshot at that version and we check * whether there any traces of the feature-to-remove. * 2) We check all commits that exist between version 0 and the current version. * This includes the versions we validated the snapshots. This is because a commit * might include information that is not available in the snapshot. Examples include * CommitInfo, CDCInfo etc. Note, there can still be valid log commit files with * versions prior the earliest checkpoint version. * 3) We do not need to recreate a snapshot at the current version because this is already being * handled by validateDropInvariants. * * Note, this is a slow process. * * @param spark The SparkSession. * @param downgradeTxnReadSnapshot The read snapshot of the protocol downgrade transaction. * @return True if the history contains any trace of the feature. */ def historyContainsFeature( spark: SparkSession, table: DeltaTableV2, downgradeTxnReadSnapshot: Snapshot): Boolean = { require(requiresHistoryProtection) val deltaLog = downgradeTxnReadSnapshot.deltaLog val earliestCheckpointVersion = deltaLog.findEarliestReliableCheckpoint.getOrElse(0L) val toVersion = downgradeTxnReadSnapshot.version // Use the snapshot at earliestCheckpointVersion to validate the checkpoint identified by // findEarliestReliableCheckpoint. val earliestSnapshot = table.getSnapshotAt(earliestCheckpointVersion) // Tombstones may contain traces of the removed feature. The earliest snapshot will include // all tombstones within the tombstoneRetentionPeriod. This may disallow protocol downgrade // because the log retention period is not aligned with the tombstoneRetentionPeriod. // To resolve this issue, we filter out all tombstones from the earliest checkpoint. // Tombstones at the earliest checkpoint should be irrelevant and should not be an // issue for readers that do not support the feature. if (containsFeatureTraces(earliestSnapshot.stateDS.filter("remove is null"))) { return true } // Check if commits between 0 version and toVersion contain any traces of the feature. val allHistoricalDeltaFiles = deltaLog .getChangeLogFiles(startVersion = 0, catalogTableOpt = table.catalogTable) .takeWhile { case (version, _) => version <= toVersion } .map { case (_, file) => file } .filter(FileNames.isDeltaFile) .toSeq DeltaLogFileIndex(DeltaLogFileIndex.COMMIT_FILE_FORMAT, allHistoricalDeltaFiles) .exists(i => containsFeatureTraces(deltaLog.loadIndex(i, Action.logSchema).as[SingleAction])) } /** Returns whether a dataset of actions contains any trace of this feature. */ private def containsFeatureTraces(ds: Dataset[SingleAction]): Boolean = { import org.apache.spark.sql.delta.implicits._ ds.mapPartitions { actions => actions .map(_.unwrap) .collectFirst { case a if actionUsesFeature(a) => true } .toIterator }.take(1).nonEmpty } } /** * A base class for all writer-only table features that can only be explicitly supported. * * @param name * a globally-unique string indicator to represent the feature. All characters must be letters * (a-z, A-Z), digits (0-9), '-', or '_'. Words must be in camelCase. */ sealed abstract class WriterFeature(name: String) extends TableFeature( name, minReaderVersion = 0, minWriterVersion = TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION) /** * A base class for all reader-writer table features that can only be explicitly supported. * * @param name * a globally-unique string indicator to represent the feature. All characters must be letters * (a-z, A-Z), digits (0-9), '-', or '_'. Words must be in camelCase. */ sealed abstract class ReaderWriterFeature(name: String) extends WriterFeature(name) with ReaderWriterFeatureType { override val minReaderVersion: Int = TableFeatureProtocolUtils.TABLE_FEATURES_MIN_READER_VERSION } /** * A base class for all table legacy writer-only features. * * @param name * a globally-unique string indicator to represent the feature. Allowed characters are letters * (a-z, A-Z), digits (0-9), '-', and '_'. Words must be in camelCase. * @param minWriterVersion * the minimum writer protocol version that supports this feature. */ sealed abstract class LegacyWriterFeature(name: String, minWriterVersion: Int) extends TableFeature(name, minReaderVersion = 0, minWriterVersion = minWriterVersion) with LegacyFeatureType /** * A base class for all legacy writer-only table features. * * @param name * a globally-unique string indicator to represent the feature. Allowed characters are letters * (a-z, A-Z), digits (0-9), '-', and '_'. Words must be in camelCase. * @param minReaderVersion * the minimum reader protocol version that supports this feature. * @param minWriterVersion * the minimum writer protocol version that supports this feature. */ sealed abstract class LegacyReaderWriterFeature( name: String, override val minReaderVersion: Int, minWriterVersion: Int) extends LegacyWriterFeature(name, minWriterVersion) with ReaderWriterFeatureType object TableFeature { val isTesting = DeltaUtils.isTesting /** * All table features recognized by this client. Update this set when you added a new Table * Feature. * * Warning: Do not call `get` on this Map to get a specific feature because keys in this map are * in lower cases. Use [[featureNameToFeature]] instead. */ def allSupportedFeaturesMap: Map[String, TableFeature] = { val testingFeaturesEnabled = try { SparkSession .getActiveSession .map(_.conf.get(DeltaSQLConf.TABLE_FEATURES_TEST_FEATURES_ENABLED)) .getOrElse(true) } catch { case _ => true } var features: Set[TableFeature] = Set( AllowColumnDefaultsTableFeature, AppendOnlyTableFeature, ChangeDataFeedTableFeature, CheckConstraintsTableFeature, ClusteringTableFeature, DomainMetadataTableFeature, GeneratedColumnsTableFeature, IdentityColumnsTableFeature, InvariantsTableFeature, ColumnMappingTableFeature, MaterializePartitionColumnsTableFeature, TimestampNTZTableFeature, TypeWideningPreviewTableFeature, TypeWideningTableFeature, IcebergCompatV1TableFeature, IcebergCompatV2TableFeature, DeletionVectorsTableFeature, VacuumProtocolCheckTableFeature, V2CheckpointTableFeature, RowTrackingFeature, InCommitTimestampTableFeature, VariantTypePreviewTableFeature, VariantTypeTableFeature, VariantShreddingPreviewTableFeature, VariantShreddingTableFeature, CatalogOwnedTableFeature, CoordinatedCommitsTableFeature, CheckpointProtectionTableFeature) if (isTesting && testingFeaturesEnabled) { features ++= Set( RedirectReaderWriterFeature, RedirectWriterOnlyFeature, TestLegacyWriterFeature, TestLegacyReaderWriterFeature, TestWriterFeature, TestUnsupportedWriterFeature, TestWriterMetadataNoAutoUpdateFeature, TestReaderWriterFeature, TestUnsupportedReaderWriterFeature, TestUnsupportedNoHistoryProtectionReaderWriterFeature, TestReaderWriterMetadataAutoUpdateFeature, TestReaderWriterMetadataNoAutoUpdateFeature, TestRemovableWriterFeature, TestRemovableWriterFeatureWithDependency, TestRemovableWriterWithHistoryTruncationFeature, TestRemovableLegacyWriterFeature, TestRemovableReaderWriterFeature, TestRemovableLegacyReaderWriterFeature, TestFeatureWithDependency, TestFeatureWithTransitiveDependency, TestWriterFeatureWithTransitiveDependency) } val featureMap = features.map(f => f.name.toLowerCase(Locale.ROOT) -> f).toMap require(features.size == featureMap.size, "Lowercase feature names must not duplicate.") featureMap } /** Test only features that appear unsupported in order to test protocol validations. */ def testUnsupportedFeatures: Set[TableFeature] = { if (!isTesting) return Set.empty Set(TestUnsupportedReaderWriterFeature, TestUnsupportedNoHistoryProtectionReaderWriterFeature, TestUnsupportedWriterFeature) } private val allDependentFeaturesMap: Map[TableFeature, Set[TableFeature]] = { val dependentFeatureTuples = allSupportedFeaturesMap.values.toSeq.flatMap(f => f.requiredFeatures.map(_ -> f)) dependentFeatureTuples .groupBy(_._1) .mapValues(_.map(_._2).toSet) .toMap } /** Get a [[TableFeature]] object by its name. */ def featureNameToFeature(featureName: String): Option[TableFeature] = allSupportedFeaturesMap.get(featureName.toLowerCase(Locale.ROOT)) /** Returns a set of [[TableFeature]]s that require the given feature to be enabled. */ def getDependentFeatures(feature: TableFeature): Set[TableFeature] = allDependentFeaturesMap.getOrElse(feature, Set.empty) /** * Extracts the removed features by comparing new and old protocols. * Returns None if there are no removed features. */ protected def getDroppedFeatures( newProtocol: Protocol, oldProtocol: Protocol): Set[TableFeature] = { val newFeatures = newProtocol.implicitlyAndExplicitlySupportedFeatures val oldFeatures = oldProtocol.implicitlyAndExplicitlySupportedFeatures oldFeatures -- newFeatures } /** * Extracts the added features by comparing new and old protocols. * Returns None if there are no added features. */ def getAddedFeatures( newProtocol: Protocol, oldProtocol: Protocol): Set[TableFeature] = { val newFeatures = newProtocol.implicitlyAndExplicitlySupportedFeatures val oldFeatures = oldProtocol.implicitlyAndExplicitlySupportedFeatures newFeatures -- oldFeatures } /** Identifies whether there was any feature removal between two protocols. */ def isProtocolRemovingFeatures(newProtocol: Protocol, oldProtocol: Protocol): Boolean = { getDroppedFeatures(newProtocol = newProtocol, oldProtocol = oldProtocol).nonEmpty } /** Returns true when `newProtocol` drops `feature`. */ def isFeatureDropped( newProtocol: Protocol, oldProtocol: Protocol, feature: TableFeature): Boolean = { getDroppedFeatures(newProtocol = newProtocol, oldProtocol = oldProtocol).contains(feature) } /** * Identifies whether there were any features with requiresHistoryProtection removed * between the two protocols. */ def isProtocolRemovingFeatureWithHistoryProtection( newProtocol: Protocol, oldProtocol: Protocol): Boolean = { getDroppedFeatures(newProtocol = newProtocol, oldProtocol = oldProtocol).exists { case r: RemovableFeature if r.requiresHistoryProtection => true case _ => false } } /** * Validates whether all requirements of a removed feature hold against the provided snapshot. */ def validateFeatureRemovalAtSnapshot( newProtocol: Protocol, oldProtocol: Protocol, table: DeltaTableV2, snapshot: Snapshot): Boolean = { val droppedFeatures = TableFeature.getDroppedFeatures( newProtocol = newProtocol, oldProtocol = oldProtocol) val droppedFeature = droppedFeatures match { case f if f.size == 1 => f.head // We do not support dropping more than one feature at a time so we have to reject // the validation. case f if f.size > 1 => return false case _ => return true } droppedFeature match { case feature: RemovableFeature => feature.validateDropInvariants(table, snapshot) case _ => throw DeltaErrors.dropTableFeatureFeatureNotSupportedByClient(droppedFeature.name) } } } /* ---------------------------------------- * | All table features known to the client | * ---------------------------------------- */ object AppendOnlyTableFeature extends LegacyWriterFeature(name = "appendOnly", minWriterVersion = 2) with FeatureAutomaticallyEnabledByMetadata { override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { DeltaConfigs.IS_APPEND_ONLY.fromMetaData(metadata) } override def failConcurrentTransactionsAtUpgrade: Boolean = false } object InvariantsTableFeature extends LegacyWriterFeature(name = "invariants", minWriterVersion = 2) with FeatureAutomaticallyEnabledByMetadata { override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { Invariants.getFromSchema(metadata.schema, spark).nonEmpty } } object CheckConstraintsTableFeature extends LegacyWriterFeature(name = "checkConstraints", minWriterVersion = 3) with FeatureAutomaticallyEnabledByMetadata with RemovableFeature { override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { Constraints.getCheckConstraints(metadata, spark).nonEmpty } override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = CheckConstraintsPreDowngradeTableFeatureCommand(table) override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = Constraints.getCheckConstraintNames(snapshot.metadata).isEmpty override def actionUsesFeature(action: Action): Boolean = { // This method is never called, as it is only used for ReaderWriterFeatures. throw new UnsupportedOperationException() } } object ChangeDataFeedTableFeature extends LegacyWriterFeature(name = "changeDataFeed", minWriterVersion = 4) with FeatureAutomaticallyEnabledByMetadata { override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { DeltaConfigs.CHANGE_DATA_FEED.fromMetaData(metadata) } override def failConcurrentTransactionsAtUpgrade: Boolean = false } object GeneratedColumnsTableFeature extends LegacyWriterFeature(name = "generatedColumns", minWriterVersion = 4) with FeatureAutomaticallyEnabledByMetadata { override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { GeneratedColumn.hasGeneratedColumns(metadata.schema) } } object ColumnMappingTableFeature extends LegacyReaderWriterFeature( name = "columnMapping", minReaderVersion = 2, minWriterVersion = 5) with RemovableFeature with FeatureAutomaticallyEnabledByMetadata { override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { metadata.columnMappingMode match { case NoMapping => false case _ => true } } override def failConcurrentTransactionsAtUpgrade: Boolean = false override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = { val schemaHasNoColumnMappingMetadata = !DeltaColumnMapping.schemaHasColumnMappingMetadata(snapshot.schema) val metadataHasNoMappingMode = snapshot.metadata.columnMappingMode match { case NoMapping => true case _ => false } schemaHasNoColumnMappingMetadata && metadataHasNoMappingMode } override def actionUsesFeature(action: Action): Boolean = action match { case m: Metadata => DeltaConfigs.COLUMN_MAPPING_MODE.fromMetaData(m) != NoMapping case _ => false } override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = ColumnMappingPreDowngradeCommand(table) } object IdentityColumnsTableFeature extends LegacyWriterFeature(name = "identityColumns", minWriterVersion = 6) with FeatureAutomaticallyEnabledByMetadata { override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { ColumnWithDefaultExprUtils.hasIdentityColumn(metadata.schema) } } object TimestampNTZTableFeature extends ReaderWriterFeature(name = "timestampNtz") with FeatureAutomaticallyEnabledByMetadata { override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { SchemaUtils.checkForTimestampNTZColumnsRecursively(metadata.schema) } override def failConcurrentTransactionsAtUpgrade: Boolean = false } object RedirectReaderWriterFeature extends ReaderWriterFeature(name = "redirectReaderWriter-preview") with FeatureAutomaticallyEnabledByMetadata with RemovableFeature { override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession ): Boolean = RedirectReaderWriter.isFeatureSet(metadata) override def automaticallyUpdateProtocolOfExistingTables: Boolean = true override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = RedirectReaderWriterPreDowngradeCommand(table) /** * [[RedirectReaderWriterPreDowngradeCommand]] will try to remove * [[DeltaConfigs.REDIRECT_READER_WRITER]], * we check that here to make sure there is no concurrent txn that re-enables redirection. */ override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = !RedirectReaderWriter.isFeatureSet(snapshot.metadata) // There is no action that is associated with this feature. override def actionUsesFeature(action: Action): Boolean = false // There is no action associated with this feature, so we don't need to truncate history to remove // the traces of it. Note that the table properties for this feature will be left in the history // but legacy clients who don't understand this feature will simply ignore them. override def requiresHistoryProtection: Boolean = false } object RedirectWriterOnlyFeature extends WriterFeature(name = "redirectWriterOnly-preview") with FeatureAutomaticallyEnabledByMetadata with RemovableFeature { override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession ): Boolean = RedirectWriterOnly.isFeatureSet(metadata) override def automaticallyUpdateProtocolOfExistingTables: Boolean = true override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = RedirectWriterOnlyPreDowngradeCommand(table) /** * [[RedirectWriterOnlyPreDowngradeCommand]] will try to remove * [[DeltaConfigs.REDIRECT_WRITER_ONLY]], * we check that here to make sure there is no concurrent txn that re-enables redirection. */ override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = !RedirectWriterOnly.isFeatureSet(snapshot.metadata) // Writer features should directly return false, as it is only used for reader+writer features. override def actionUsesFeature(action: Action): Boolean = false } trait BinaryVariantTableFeature { def forcePreviewTableFeature: Boolean = SparkSession .getActiveSession .map(_.conf.get(DeltaSQLConf.FORCE_USE_PREVIEW_VARIANT_FEATURE)) .getOrElse(false) } /** * Preview feature for variant. The preview feature isn't enabled automatically anymore when * variants are present in the table schema and the GA feature is used instead. * * Note: Users can manually add both the preview and stable features to a table using ADD FEATURE, * although that's undocumented. The feature spec did not change between preview and GA so the two * feature specifications are compatible and supported. */ object VariantTypePreviewTableFeature extends ReaderWriterFeature(name = "variantType-preview") with FeatureAutomaticallyEnabledByMetadata with BinaryVariantTableFeature { override def automaticallyUpdateProtocolOfExistingTables: Boolean = true override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { if (forcePreviewTableFeature) { SchemaUtils.checkForVariantTypeColumnsRecursively(metadata.schema) && // Do not require this table feature to be enabled when the 'variantType' table feature is // enabled so existing tables with variant columns with only 'variantType' and not // 'variantType-preview' can be operated on when the 'FORCE_USE_PREVIEW_VARIANT_FEATURE' // config is enabled. !protocol.isFeatureSupported(VariantTypeTableFeature) } else { false } } } object VariantTypeTableFeature extends ReaderWriterFeature(name = "variantType") with FeatureAutomaticallyEnabledByMetadata with BinaryVariantTableFeature { override def automaticallyUpdateProtocolOfExistingTables: Boolean = true override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { if (forcePreviewTableFeature) { false } else { SchemaUtils.checkForVariantTypeColumnsRecursively(metadata.schema) && // Do not require this table feature to be enabled when the 'variantType-preview' table // feature is enabled so old tables with only the preview table feature can be read. !protocol.isFeatureSupported(VariantTypePreviewTableFeature) } } } trait VariantShreddingTableFeatureBase { def forcePreviewTableFeature: Boolean = SparkSession .getActiveSession .map(_.conf.get(DeltaSQLConf.FORCE_USE_PREVIEW_SHREDDING_FEATURE)) .getOrElse(false) } object VariantShreddingPreviewTableFeature extends ReaderWriterFeature(name = "variantShredding-preview") with FeatureAutomaticallyEnabledByMetadata with VariantShreddingTableFeatureBase { override def automaticallyUpdateProtocolOfExistingTables: Boolean = true override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { forcePreviewTableFeature && DeltaConfigs.ENABLE_VARIANT_SHREDDING.fromMetaData(metadata) && // Do not require this table feature to be enabled when the 'variantShredding' table feature // is enabled so existing tables with shredding with only 'variantShredding' and not // 'variantShredding-preview' can be operated on when the // 'FORCE_USE_PREVIEW_SHREDDING_FEATURE' config is enabled. !protocol.isFeatureSupported(VariantShreddingTableFeature) } } object VariantShreddingTableFeature extends ReaderWriterFeature(name = "variantShredding") with FeatureAutomaticallyEnabledByMetadata with VariantShreddingTableFeatureBase { override def automaticallyUpdateProtocolOfExistingTables: Boolean = VariantShreddingPreviewTableFeature.automaticallyUpdateProtocolOfExistingTables override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { !forcePreviewTableFeature && DeltaConfigs.ENABLE_VARIANT_SHREDDING.fromMetaData(metadata) && // Do not require this table feature to be enabled when the 'variantShredding-preview' table // feature is enabled so old tables with only the preview table feature can be read. !protocol.isFeatureSupported(VariantShreddingPreviewTableFeature) } } object DeletionVectorsTableFeature extends ReaderWriterFeature(name = "deletionVectors") with RemovableFeature with FeatureAutomaticallyEnabledByMetadata { override def automaticallyUpdateProtocolOfExistingTables: Boolean = true override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.fromMetaData(metadata) } override def failConcurrentTransactionsAtUpgrade: Boolean = false /** * Validate whether all deletion vector traces are removed from the snapshot. * * Note, we do not need to validate whether DV tombstones exist. These are added in the * pre-downgrade stage and always cover all DVs within the retention period. This invariant can * never change unless we enable again DVs. If DVs are enabled before the protocol downgrade * we will abort the operation. */ override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = { val dvsWritable = DeletionVectorUtils.deletionVectorsWritable(snapshot) val dvsExist = snapshot.numDeletionVectorsOpt.getOrElse(0L) > 0 !(dvsWritable || dvsExist) } override def actionUsesFeature(action: Action): Boolean = { action match { case m: Metadata => DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.fromMetaData(m) case a: AddFile => a.deletionVector != null case r: RemoveFile => r.deletionVector != null // In general, CDC actions do not contain DVs. We added this for safety. case cdc: AddCDCFile => cdc.deletionVector != null case _ => false } } override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = DeletionVectorsPreDowngradeCommand(table) } object RowTrackingFeature extends WriterFeature(name = "rowTracking") with RemovableFeature with FeatureAutomaticallyEnabledByMetadata { override def automaticallyUpdateProtocolOfExistingTables: Boolean = true override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = DeltaConfigs.ROW_TRACKING_ENABLED.fromMetaData(metadata) override def requiredFeatures: Set[TableFeature] = Set(DomainMetadataTableFeature) override def failConcurrentTransactionsAtUpgrade: Boolean = false /** * When dropping row tracking we remove all relevant properties at downgrade commit. * This is because concurrent transactions may still use them while the feature exists in the * protocol. */ override def tablePropertiesToRemoveAtDowngradeCommit: Seq[String] = { Seq( DeltaConfigs.ROW_TRACKING_ENABLED.key, DeltaConfigs.ROW_TRACKING_SUSPENDED.key, MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP, MaterializedRowCommitVersion.MATERIALIZED_COLUMN_NAME_PROP) } /** Remove rowTracking domain metadata at downgrade commit. */ override def actionsToIncludeAtDowngradeCommit(snapshot: Snapshot): Seq[Action] = { val domainOpt = RowTrackingMetadataDomain.fromSnapshot(snapshot) Seq.empty ++ domainOpt.map(_.toDomainMetadata.copy(removed = true)) } override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = { RowTrackingPreDowngradeCommand(table) } private[delta] def validateConfigurations(configurations: Map[String, String]): Unit = { val enabled = configurations.getOrElse( DeltaConfigs.ROW_TRACKING_ENABLED.key, "false").toBoolean val suspended = configurations.getOrElse( DeltaConfigs.ROW_TRACKING_SUSPENDED.key, "false").toBoolean if (enabled && suspended) { throw DeltaErrors.rowTrackingIllegalPropertyCombination() } } private[delta] def validateAndBackfill( spark: SparkSession, table: DeltaTableV2, newConfiguration: Map[String, String]): Unit = { // If there is no relevant configuration change, we do not need to do anything. if (!newConfiguration.contains(DeltaConfigs.ROW_TRACKING_ENABLED.key) && !newConfiguration.contains(DeltaConfigs.ROW_TRACKING_SUSPENDED.key)) { return } val snapshot = table.deltaLog.update(catalogTableOpt = table.catalogTable) // For overlapping configs, we keep the values of new configuration. validateConfigurations(snapshot.metadata.configuration ++ newConfiguration) val justEnabled = newConfiguration.getOrElse( DeltaConfigs.ROW_TRACKING_ENABLED.key, "false").toBoolean // If we're enabling row tracking on an existing table, we need to complete a backfill process // prior to updating the table metadata. if (justEnabled) { RowTrackingBackfillCommand( table.deltaLog, nameOfTriggeringOperation = DeltaOperations.OP_SET_TBLPROPERTIES, table.catalogTable).run(spark) } } /** * Returns true if no relevant row tracking metadata exist on the table. This excludes * properties/domain metadata that are only removed at the downgrade commit. * * Returns false otherwise. */ override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = { val rowTrackingEnabled = DeltaConfigs.ROW_TRACKING_ENABLED.fromMetaData(snapshot.metadata) val rowTrackingSuspended = DeltaConfigs.ROW_TRACKING_SUSPENDED.fromMetaData(snapshot.metadata) if (rowTrackingEnabled || !rowTrackingSuspended) return false // In most cases, we should only reach this expensive check only at the protocol downgrade // commit validation. snapshot .allFiles .filter(col("baseRowId").isNotNull || col("defaultRowCommitVersion").isNotNull) .isEmpty } /** * Even though Row tracking is a writer-only feature it could benefit from history protection. * Without history protection, oblivious writers could replace past checkpoints that contain * Row Tracking metadata. That could break time travel, i.e. row tracking might appear enabled * in a past version but metadata might be missing. * * On the other hand, history protection dictates the addition of the checkpointProtection * feature when dropping row tracking. For this reason, we choose not to protect history. There * should be no (or very limited) uses cases where row tracking is expected to work for past * versions. */ override def requiresHistoryProtection: Boolean = false override def actionUsesFeature(action: Action): Boolean = false } object DomainMetadataTableFeature extends WriterFeature(name = "domainMetadata") with RemovableFeature { /** * Returns true if no domain metadata exist on the table. * Returns false otherwise. */ override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = { snapshot.domainMetadata.isEmpty } override def failConcurrentTransactionsAtUpgrade: Boolean = false override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = { DomainMetadataPreDowngradeCommand(table) } override def requiresHistoryProtection: Boolean = false override def actionUsesFeature(action: Action): Boolean = false } object IcebergCompatV1TableFeature extends WriterFeature(name = "icebergCompatV1") with FeatureAutomaticallyEnabledByMetadata { override def automaticallyUpdateProtocolOfExistingTables: Boolean = true override def failConcurrentTransactionsAtUpgrade: Boolean = false override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = IcebergCompatV1.isEnabled(metadata) override def requiredFeatures: Set[TableFeature] = Set(ColumnMappingTableFeature) } object IcebergCompatV2TableFeature extends WriterFeature(name = "icebergCompatV2") with FeatureAutomaticallyEnabledByMetadata { override def automaticallyUpdateProtocolOfExistingTables: Boolean = true override def failConcurrentTransactionsAtUpgrade: Boolean = false override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = IcebergCompatV2.isEnabled(metadata) override def requiredFeatures: Set[TableFeature] = Set(ColumnMappingTableFeature) } /** * Clustering table feature is enabled when a table is created with CLUSTER BY clause. */ object ClusteringTableFeature extends WriterFeature("clustering") { override val requiredFeatures: Set[TableFeature] = Set(DomainMetadataTableFeature) } /** * This table feature represents support for column DEFAULT values for Delta Lake. With this * feature, it is possible to assign default values to columns either at table creation time or * later by using commands of the form: ALTER TABLE t ALTER COLUMN c SET DEFAULT v. Thereafter, * queries from the table will return the specified default value instead of NULL when the * corresponding field is not present in storage. * * We create this as a writer-only feature rather than a reader/writer feature in order to simplify * the query execution implementation for scanning Delta tables. This means that commands of the * following form are not allowed: ALTER TABLE t ADD COLUMN c DEFAULT v. The reason is that when * commands of that form execute (such as for other data sources like CSV or JSON), then the data * source scan implementation must take responsibility to return the supplied default value for all * rows, including those previously present in the table before the command executed. We choose to * avoid this complexity for Delta table scans, so we make this a writer-only feature instead. * Therefore, the analyzer can take care of the entire job when processing commands that introduce * new rows into the table by injecting the column default value (if present) into the corresponding * query plan. This comes at the expense of preventing ourselves from easily adding a default value * to an existing non-empty table, because all data files would need to be rewritten to include the * new column value in an expensive backfill. */ object AllowColumnDefaultsTableFeature extends WriterFeature(name = "allowColumnDefaults") /** * This table feature requires materialization of partition columns in data files. * * This is a writer-only feature because: * - Writers need to understand when to materialize partition columns into data files * - Readers can read the data regardless of whether partition columns are materialized or not, as * they read the partition values from the AddFile. * * The feature is automatically enabled when the table property * `delta.enableMaterializePartitionColumnsFeature` is set to true. * * This makes data files more flexible with external readers that require the presence of * partition columns in parquet, or for future data layout changes. This is a removable feature * that can be dropped when partition column materialization is no longer needed. */ object MaterializePartitionColumnsTableFeature extends WriterFeature(name = "materializePartitionColumns") with FeatureAutomaticallyEnabledByMetadata with RemovableFeature { override def automaticallyUpdateProtocolOfExistingTables: Boolean = true /** * MaterializePartitionColumnsTableFeature is always enabled when present in the protocol. * The Delta protocol does not require any metadata or domain metadata configs for this * feature to be effective. */ override def failConcurrentTransactionsAtUpgrade: Boolean = true override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { DeltaConfigs.ENABLE_MATERIALIZE_PARTITION_COLUMNS_FEATURE .fromMetaData(metadata) .getOrElse(false) } /** dropping this feature is always allowed without any action */ override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = true override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = { MaterializePartitionColumnsPreDowngradeCommand(table) } override def requiresHistoryProtection: Boolean = false override def tablePropertiesToRemoveAtDowngradeCommit: Seq[String] = { Seq(DeltaConfigs.ENABLE_MATERIALIZE_PARTITION_COLUMNS_FEATURE.key) } override def actionUsesFeature(action: Action): Boolean = false } /** * V2 Checkpoint table feature is for checkpoints with sidecars and the new format and * file naming scheme. */ object V2CheckpointTableFeature extends ReaderWriterFeature(name = "v2Checkpoint") with RemovableFeature with FeatureAutomaticallyEnabledByMetadata { override def automaticallyUpdateProtocolOfExistingTables: Boolean = true private def isV2CheckpointSupportNeededByMetadata(metadata: Metadata): Boolean = DeltaConfigs.CHECKPOINT_POLICY.fromMetaData(metadata).needsV2CheckpointSupport override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = isV2CheckpointSupportNeededByMetadata(metadata) override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = { // Fail validation if v2 checkpoints are still enabled in the current snapshot if (isV2CheckpointSupportNeededByMetadata(snapshot.metadata)) return false // Validation also fails if the current snapshot might depend on a v2 checkpoint. // NOTE: Empty and preloaded checkpoint providers never reference v2 checkpoints. snapshot.checkpointProvider match { case p if p.isEmpty => true case _: PreloadedCheckpointProvider => true case lazyProvider: LazyCompleteCheckpointProvider => lazyProvider.underlyingCheckpointProvider.isInstanceOf[PreloadedCheckpointProvider] case _ => false } } override def actionUsesFeature(action: Action): Boolean = action match { case m: Metadata => isV2CheckpointSupportNeededByMetadata(m) case _: CheckpointMetadata => true case _: SidecarFile => true case _ => false } override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = V2CheckpointPreDowngradeCommand(table) } /** Table feature to represent tables whose commits are managed by separate commit-coordinator */ object CoordinatedCommitsTableFeature extends WriterFeature(name = "coordinatedCommits-preview") with FeatureAutomaticallyEnabledByMetadata with RemovableFeature { override def automaticallyUpdateProtocolOfExistingTables: Boolean = true override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.fromMetaData(metadata).nonEmpty } override def requiredFeatures: Set[TableFeature] = Set(InCommitTimestampTableFeature, VacuumProtocolCheckTableFeature) override def preDowngradeCommand(table: DeltaTableV2) : PreDowngradeTableFeatureCommand = CoordinatedCommitsPreDowngradeCommand(table) override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = { !CoordinatedCommitsUtils.tablePropertiesPresent(snapshot.metadata) && !CoordinatedCommitsUtils.unbackfilledCommitsPresent(snapshot) } // This is a writer feature, so it should directly return false. override def actionUsesFeature(action: Action): Boolean = false } /** Table feature to represent tables that commits are managed by catalog */ object CatalogOwnedTableFeature extends ReaderWriterFeature(name = "catalogManaged") with RemovableFeature { override def requiredFeatures: Set[TableFeature] = Set(InCommitTimestampTableFeature, VacuumProtocolCheckTableFeature) override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = { // Note: We don't support downgrade for this feature yet. throw DeltaErrors.dropTableFeatureFeatureNotSupportedByClient(name) } override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = { !CoordinatedCommitsUtils.unbackfilledCommitsPresent(snapshot) } // Before downgrade, we require to backfill all unbackfilled commits, hence time-travel is safe. override def actionUsesFeature(action: Action): Boolean = false } /** Common base shared by the preview and stable type widening table features. */ abstract class TypeWideningTableFeatureBase(name: String) extends ReaderWriterFeature(name) with RemovableFeature { protected def isTypeWideningSupportNeededByMetadata(metadata: Metadata): Boolean = DeltaConfigs.ENABLE_TYPE_WIDENING.fromMetaData(metadata) override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = !isTypeWideningSupportNeededByMetadata(snapshot.metadata) && !TypeWideningMetadata.containsTypeWideningMetadata(snapshot.metadata.schema) override def actionUsesFeature(action: Action): Boolean = action match { case m: Metadata => TypeWideningMetadata.containsTypeWideningMetadata(m.schema) case _ => false } override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = TypeWideningPreDowngradeCommand(table) override def failConcurrentTransactionsAtUpgrade: Boolean = false } /** * Feature used for the preview phase of type widening. Tables that enabled this feature during the * preview are still supported after the preview. * * Note: Users can manually add both the preview and stable features to a table using ADD FEATURE, * although that's undocumented for type widening. This is allowed: the two feature specifications * are compatible and supported. */ object TypeWideningPreviewTableFeature extends TypeWideningTableFeatureBase(name = "typeWidening-preview") /** * Stable feature for type widening. */ object TypeWideningTableFeature extends TypeWideningTableFeatureBase(name = "typeWidening") with FeatureAutomaticallyEnabledByMetadata { override def automaticallyUpdateProtocolOfExistingTables: Boolean = true override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = isTypeWideningSupportNeededByMetadata(metadata) && // Don't automatically enable the stable feature if the preview feature is already supported, to // avoid possibly breaking old clients that only support the preview feature. !protocol.isFeatureSupported(TypeWideningPreviewTableFeature) } /** * inCommitTimestamp table feature is a writer feature that makes * every writer write a monotonically increasing timestamp inside the commit file. */ object InCommitTimestampTableFeature extends WriterFeature(name = "inCommitTimestamp") with FeatureAutomaticallyEnabledByMetadata with RemovableFeature { override def automaticallyUpdateProtocolOfExistingTables: Boolean = true override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(metadata) } override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = InCommitTimestampsPreDowngradeCommand(table) override def failConcurrentTransactionsAtUpgrade: Boolean = false /** * As per the spec, we can disable ICT by just setting * [[DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED]] to `false`. There is no need to remove the * provenance properties. However, [[InCommitTimestampsPreDowngradeCommand]] will try to remove * these properties because they can be removed as part of the same metadata update that sets * [[DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED]] to `false`. We check all three properties here * as well for consistency. */ override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = { val provenancePropertiesAbsent = Seq( DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.key, DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.key) .forall(!snapshot.metadata.configuration.contains(_)) val ictEnabledInMetadata = DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(snapshot.metadata) provenancePropertiesAbsent && !ictEnabledInMetadata } // Writer features should directly return false, as it is only used for reader+writer features. override def actionUsesFeature(action: Action): Boolean = false } /** * A ReaderWriter table feature for VACUUM. If this feature is enabled: * A writer should follow one of the following: * 1. Non-Support for Vacuum: Writers can explicitly state that they do not support VACUUM for * any table, regardless of whether the Vacuum Protocol Check Table feature exists. * 2. Implement Writer Protocol Check: Ensure that the VACUUM implementation includes a writer * protocol check before any file deletions occur. * Readers don't need to understand or change anything new; they just need to acknowledge the * feature exists */ object VacuumProtocolCheckTableFeature extends ReaderWriterFeature(name = "vacuumProtocolCheck") with RemovableFeature { override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = { VacuumProtocolCheckPreDowngradeCommand(table) } // The delta snapshot doesn't have any trace of the [[VacuumProtocolCheckTableFeature]] feature. // Other than it being present in PROTOCOL, which will be handled by the table feature downgrade // command once this method returns true. override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = true // None of the actions uses [[VacuumProtocolCheckTableFeature]] override def actionUsesFeature(action: Action): Boolean = false } /** * Writer feature that enforces writers to cleanup metadata iff metadata can be cleaned up to * requireCheckpointProtectionBeforeVersion in one go. This means that a single cleanup * operation should truncate up to requireCheckpointProtectionBeforeVersion as opposed to * several cleanup operations truncating in chunks. * * The are two exceptions to this rule. If any of the two holds, the rule * above can be ignored: * * a) The writer verifies it supports all protocols between * [start, min(requireCheckpointProtectionBeforeVersion, targetCleanupVersion)] versions * it intends to truncate. * b) The writer does not create any checkpoints during history cleanup and does not erase any * checkpoints after the truncation version. * * The CheckpointProtectionTableFeature can only be removed if history is truncated up to * at least requireCheckpointProtectionBeforeVersion. */ object CheckpointProtectionTableFeature extends WriterFeature(name = "checkpointProtection") with RemovableFeature { /** * Gets the version requiring checkpoint protection from `metadata`. If the table property is * not set, return `None`. */ def getCheckpointProtectionVersionOption(protocol: Protocol, metadata: Metadata): Option[Long] = { if (!protocol.isFeatureSupported(CheckpointProtectionTableFeature)) return None DeltaConfigs.REQUIRE_CHECKPOINT_PROTECTION_BEFORE_VERSION.fromMetaDataOption(metadata) } /** * Gets the version requiring checkpoint protection from `snapshot`. If the table property is * not set, return the default value 0. */ def getCheckpointProtectionVersion(snapshot: Snapshot): Long = { getCheckpointProtectionVersionOption(snapshot.protocol, snapshot.metadata).getOrElse(0) } def metadataWithCheckpointProtection(metadata: Metadata, version: Long): Metadata = { val versionPropKey = DeltaConfigs.REQUIRE_CHECKPOINT_PROTECTION_BEFORE_VERSION.key val versionConf = versionPropKey -> version.toString metadata.copy(configuration = metadata.configuration + versionConf) } /** Verify whether any deltas exist between version 0 to toVersion (inclusive). */ private def deltasUpToVersionAreTruncated( deltaLog: DeltaLog, catalogTableOpt: Option[CatalogTable], toVersion: Long): Boolean = { deltaLog .getChangeLogFiles( startVersion = 0, endVersion = toVersion, catalogTableOpt = catalogTableOpt, failOnDataLoss = false) .map { case (_, file) => file } .filter(FileNames.isDeltaFile) .take(1).isEmpty } def historyPriorToCheckpointProtectionVersionIsTruncated( snapshot: Snapshot, catalogTableOpt: Option[CatalogTable]): Boolean = { val checkpointProtectionVersion = getCheckpointProtectionVersion(snapshot) if (checkpointProtectionVersion <= 0) return true val deltaLog = snapshot.deltaLog // In most cases, the earliest checkpoint matches the version of the earliest commit. This is // not true for new tables that were never cleaned up. Furthermore, if there is no checkpoint it // means history is not truncated. deltaLog.findEarliestReliableCheckpoint.exists(_ >= checkpointProtectionVersion) && deltasUpToVersionAreTruncated(deltaLog, catalogTableOpt, checkpointProtectionVersion - 1) } /** * This is a special feature in the sense that it requires history truncation but implements it * as part of its downgrade process. This is implemented like this for 2 reasons: * * 1. It allows us to remove the feature table property after the clean up in the preDowngrade * command is successful. * 2. It does not require to scan the history for features traces as long as all history * before requireCheckpointProtectionBeforeVersion is truncated. */ override def requiresHistoryProtection: Boolean = false override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = { CheckpointProtectionPreDowngradeCommand(table) } /** Returns true if table property is absent. */ override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = { val property = DeltaConfigs.REQUIRE_CHECKPOINT_PROTECTION_BEFORE_VERSION.key !snapshot.metadata.configuration.contains(property) } /** * The feature uses the `requireCheckpointProtectionBeforeVersion` property. This is removed when * dropping the feature but we allow it to exist in the history. This is to allow history * truncation at the boundary of requireCheckpointProtectionBeforeVersion rather than the last * 24 hours. Otherwise, dropping the feature would always require 24 hour waiting time. */ override def actionUsesFeature(action: Action): Boolean = false } /** * Features below are for testing only, and are being registered to the system only in the testing * environment. See [[TableFeature.allSupportedFeaturesMap]] for the registration. */ object TestLegacyWriterFeature extends LegacyWriterFeature(name = "testLegacyWriter", minWriterVersion = 5) object TestWriterFeature extends WriterFeature(name = "testWriter") object TestWriterMetadataNoAutoUpdateFeature extends WriterFeature(name = "testWriterMetadataNoAutoUpdate") with FeatureAutomaticallyEnabledByMetadata { val TABLE_PROP_KEY = "_123testWriterMetadataNoAutoUpdate321_" override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { metadata.configuration.get(TABLE_PROP_KEY).exists(_.toBoolean) } } object TestLegacyReaderWriterFeature extends LegacyReaderWriterFeature( name = "testLegacyReaderWriter", minReaderVersion = 2, minWriterVersion = 5) object TestReaderWriterFeature extends ReaderWriterFeature(name = "testReaderWriter") object TestReaderWriterMetadataNoAutoUpdateFeature extends ReaderWriterFeature(name = "testReaderWriterMetadataNoAutoUpdate") with FeatureAutomaticallyEnabledByMetadata { val TABLE_PROP_KEY = "_123testReaderWriterMetadataNoAutoUpdate321_" override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { metadata.configuration.get(TABLE_PROP_KEY).exists(_.toBoolean) } } object TestReaderWriterMetadataAutoUpdateFeature extends ReaderWriterFeature(name = "testReaderWriterMetadataAutoUpdate") with FeatureAutomaticallyEnabledByMetadata { val TABLE_PROP_KEY = "_123testReaderWriterMetadataAutoUpdate321_" override def automaticallyUpdateProtocolOfExistingTables: Boolean = true override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { metadata.configuration.get(TABLE_PROP_KEY).exists(_.toBoolean) } } object TestRemovableWriterFeature extends WriterFeature(name = "testRemovableWriter") with FeatureAutomaticallyEnabledByMetadata with RemovableFeature { val TABLE_PROP_KEY = "_123TestRemovableWriter321_" override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { metadata.configuration.get(TABLE_PROP_KEY).exists(_.toBoolean) } /** Make sure the property is not enabled on the table. */ override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = !snapshot.metadata.configuration.get(TABLE_PROP_KEY).exists(_.toBoolean) override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = TestWriterFeaturePreDowngradeCommand(table) override def actionUsesFeature(action: Action): Boolean = false } /** Test feature that appears unsupported and it is used for testing protocol checks. */ object TestUnsupportedReaderWriterFeature extends ReaderWriterFeature(name = "testUnsupportedReaderWriter") with RemovableFeature { override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = true override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = TestUnsupportedReaderWriterFeaturePreDowngradeCommand(table) override def actionUsesFeature(action: Action): Boolean = false } /** * Test feature that appears unsupported and can be dropped without checkpoint protection. * it is used only for testing purposes. */ object TestUnsupportedNoHistoryProtectionReaderWriterFeature extends ReaderWriterFeature(name = "testUnsupportedNoHistoryProtectionReaderWriter") with RemovableFeature { override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = true override def requiresHistoryProtection: Boolean = false override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = TestUnsupportedReaderWriterFeaturePreDowngradeCommand(table) override def actionUsesFeature(action: Action): Boolean = false } object TestUnsupportedWriterFeature extends WriterFeature(name = "testUnsupportedWriter") with RemovableFeature { /** Make sure the property is not enabled on the table. */ override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = true override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = TestUnsupportedWriterFeaturePreDowngradeCommand(table) override def actionUsesFeature(action: Action): Boolean = false } private[sql] object TestRemovableWriterFeatureWithDependency extends WriterFeature(name = "testRemovableWriterFeatureWithDependency") with FeatureAutomaticallyEnabledByMetadata with RemovableFeature { val TABLE_PROP_KEY = "_123TestRemovableWriterFeatureWithDependency321_" override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { metadata.configuration.get(TABLE_PROP_KEY).exists(_.toBoolean) } /** Make sure the property is not enabled on the table. */ override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = !snapshot.metadata.configuration.get(TABLE_PROP_KEY).exists(_.toBoolean) override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = TestWriterFeaturePreDowngradeCommand(table) override def actionUsesFeature(action: Action): Boolean = false override def requiredFeatures: Set[TableFeature] = Set(TestRemovableReaderWriterFeature, TestRemovableWriterFeature) } object TestRemovableReaderWriterFeature extends ReaderWriterFeature(name = "testRemovableReaderWriter") with FeatureAutomaticallyEnabledByMetadata with RemovableFeature { val TABLE_PROP_KEY = "_123TestRemovableReaderWriter321_" override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { metadata.configuration.get(TABLE_PROP_KEY).exists(_.toBoolean) } /** Make sure the property is not enabled on the table. */ override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = !snapshot.metadata.configuration.get(TABLE_PROP_KEY).exists(_.toBoolean) override def actionUsesFeature(action: Action): Boolean = action match { case m: Metadata => m.configuration.get(TABLE_PROP_KEY).exists(_.toBoolean) case _ => false } override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = TestReaderWriterFeaturePreDowngradeCommand(table) } object TestRemovableLegacyWriterFeature extends LegacyWriterFeature(name = "testRemovableLegacyWriter", minWriterVersion = 5) with FeatureAutomaticallyEnabledByMetadata with RemovableFeature { val TABLE_PROP_KEY = "_123TestRemovableLegacyWriter321_" override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { metadata.configuration.get(TABLE_PROP_KEY).exists(_.toBoolean) } override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = { !snapshot.metadata.configuration.contains(TABLE_PROP_KEY) } override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = TestLegacyWriterFeaturePreDowngradeCommand(table) override def actionUsesFeature(action: Action): Boolean = false } object TestRemovableLegacyReaderWriterFeature extends LegacyReaderWriterFeature( name = "testRemovableLegacyReaderWriter", minReaderVersion = 2, minWriterVersion = 5) with FeatureAutomaticallyEnabledByMetadata with RemovableFeature { val TABLE_PROP_KEY = "_123TestRemovableLegacyReaderWriter321_" override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { metadata.configuration.get(TABLE_PROP_KEY).exists(_.toBoolean) } override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = { !snapshot.metadata.configuration.contains(TABLE_PROP_KEY) } override def actionUsesFeature(action: Action): Boolean = { action match { case m: Metadata => m.configuration.contains(TABLE_PROP_KEY) case _ => false } } override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = TestLegacyReaderWriterFeaturePreDowngradeCommand(table) } object TestFeatureWithDependency extends ReaderWriterFeature(name = "testFeatureWithDependency") with FeatureAutomaticallyEnabledByMetadata { val TABLE_PROP_KEY = "_123testFeatureWithDependency321_" override def automaticallyUpdateProtocolOfExistingTables: Boolean = true override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { metadata.configuration.get(TABLE_PROP_KEY).exists(_.toBoolean) } override def requiredFeatures: Set[TableFeature] = Set(TestReaderWriterFeature) } object TestFeatureWithTransitiveDependency extends ReaderWriterFeature(name = "testFeatureWithTransitiveDependency") { override def requiredFeatures: Set[TableFeature] = Set(TestFeatureWithDependency) } object TestWriterFeatureWithTransitiveDependency extends WriterFeature(name = "testWriterFeatureWithTransitiveDependency") { override def requiredFeatures: Set[TableFeature] = Set(TestFeatureWithDependency) } object TestRemovableWriterWithHistoryTruncationFeature extends WriterFeature(name = "TestRemovableWriterWithHistoryTruncationFeature") with FeatureAutomaticallyEnabledByMetadata with RemovableFeature { val TABLE_PROP_KEY = "_123TestRemovableWriterWithHistoryTruncationFeature321_" override def metadataRequiresFeatureToBeEnabled( protocol: Protocol, metadata: Metadata, spark: SparkSession): Boolean = { metadata.configuration.get(TABLE_PROP_KEY).exists(_.toBoolean) } /** Make sure the property is not enabled on the table. */ override def validateDropInvariants(table: DeltaTableV2, snapshot: Snapshot): Boolean = !snapshot.metadata.configuration.get(TABLE_PROP_KEY).exists(_.toBoolean) override def preDowngradeCommand(table: DeltaTableV2): PreDowngradeTableFeatureCommand = TestWriterWithHistoryValidationFeaturePreDowngradeCommand(table) override def actionUsesFeature(action: Action): Boolean = action match { case m: Metadata => m.configuration.get(TABLE_PROP_KEY).exists(_.toBoolean) case _ => false } override def requiresHistoryProtection: Boolean = true } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/ThreadStorageExecutionObserver.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta trait ThreadStorageExecutionObserver[T <: ChainableExecutionObserver[T]] { /** Thread-local observer instance loaded by [[T]] */ protected val threadObserver: ThreadLocal[T] = ThreadLocal.withInitial(() => initialValue) protected def initialValue: T def getObserver: T = threadObserver.get() def setObserver(observer: T): Unit = threadObserver.set(observer) /** Instrument all executions created and completed within `thunk` with `newObserver`. */ def withObserver[S](newObserver: T)(thunk: => S): S = { val oldObserver = threadObserver.get() threadObserver.set(newObserver) try { thunk } finally { // reset threadObserver.set(oldObserver) } } /** Update the current thread observer with its next one. */ def advanceToNextObserver(): Unit = threadObserver.get.advanceToNextThreadObserver() } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/TransactionExecutionObserver.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta trait ChainableExecutionObserver[O] { /** * The next txn observer for this thread. * The next observer is used to test threads that perform multiple transactions, i.e. * commands that perform multiple commits. */ @volatile protected var nextObserver: Option[O] = None /** Set the next observer for this thread. */ def setNextObserver(nextTxnObserver: O): Unit = { nextObserver = Some(nextTxnObserver) } /** Update the observer of this thread with the next observer. */ def advanceToNextThreadObserver(): Unit } /** * Track different stages of the execution of a transaction. * * This is mostly meant for test instrumentation. * * The default is a no-op implementation. */ trait TransactionExecutionObserver extends ChainableExecutionObserver[TransactionExecutionObserver] { /** * Create a child instance of this observer for use in [[OptimisticTransactionImpl.split()]]. * * It's up to each observer type what state new child needs to hold. */ def createChild(): TransactionExecutionObserver /* * This is called outside the transaction object, * since it wraps its creation. */ /** Wraps transaction creation. */ def startingTransaction(f: => OptimisticTransaction): OptimisticTransaction /* * These are called from within the transaction object. */ /** Wraps `prepareCommit`. */ def preparingCommit[T](f: => T): T /* * The next three methods before/after-style instead of wrapping like above, * because the commit code is large and in a try-catch block, * making wrapping impractical. */ /** Called before the first `doCommit` attempt. */ def beginDoCommit(): Unit /** Called after publishing the commit file but before the `backfill` attempt. */ def beginBackfill(): Unit /** Called after backfill but before the `postCommit` attempt. */ def beginPostCommit(): Unit /** Called once a commit succeeded. */ def transactionCommitted(): Unit /** * Called once the transaction failed. * * *Note:* It can happen that [[transactionAborted()]] is called * without [[beginDoCommit()]] being called first. * This occurs when there is an Exception thrown during the transaction's body. */ def transactionAborted(): Unit override def advanceToNextThreadObserver(): Unit = { TransactionExecutionObserver.setObserver( nextObserver.getOrElse(NoOpTransactionExecutionObserver)) } } object TransactionExecutionObserver extends ThreadStorageExecutionObserver[TransactionExecutionObserver] { override protected val initialValue: TransactionExecutionObserver = NoOpTransactionExecutionObserver } /** Default observer does nothing. */ object NoOpTransactionExecutionObserver extends TransactionExecutionObserver { override def startingTransaction(f: => OptimisticTransaction): OptimisticTransaction = f override def preparingCommit[T](f: => T): T = f override def beginDoCommit(): Unit = () override def beginBackfill(): Unit = () override def beginPostCommit(): Unit = () override def transactionCommitted(): Unit = () override def transactionAborted(): Unit = () override def createChild(): TransactionExecutionObserver = { // This mimics the original behaviour of this code. TransactionExecutionObserver.getObserver } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/TypeWidening.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.actions.{AddFile, Metadata, Protocol, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.catalyst.expressions.Cast import org.apache.spark.sql.functions.{col, lit} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ object TypeWidening { /** * Returns whether the protocol version supports the Type Widening table feature. */ def isSupported(protocol: Protocol): Boolean = Seq(TypeWideningPreviewTableFeature, TypeWideningTableFeature) .exists(protocol.isFeatureSupported) /** * Returns whether Type Widening is enabled on this table version. Checks that Type Widening is * supported, which is a pre-requisite for enabling Type Widening, throws an error if * not. When Type Widening is enabled, the type of existing columns or fields can be widened * using ALTER TABLE CHANGE COLUMN. */ def isEnabled(protocol: Protocol, metadata: Metadata): Boolean = { val isEnabled = DeltaConfigs.ENABLE_TYPE_WIDENING.fromMetaData(metadata) if (isEnabled && !isSupported(protocol)) { throw new IllegalStateException( s"Table property '${DeltaConfigs.ENABLE_TYPE_WIDENING.key}' is " + s"set on the table but this table version doesn't support table feature " + s"'${TableFeatureProtocolUtils.propertyKey(TypeWideningTableFeature)}'.") } isEnabled } /** * Checks that the type widening table property wasn't disabled or enabled between the two given * states, throws an errors if it was. */ def ensureFeatureConsistentlyEnabled( protocol: Protocol, metadata: Metadata, otherProtocol: Protocol, otherMetadata: Metadata): Unit = { if (isEnabled(protocol, metadata) != isEnabled(otherProtocol, otherMetadata)) { throw DeltaErrors.metadataChangedException(None) } } /** * Returns whether the given type change is eligible for widening. This only checks atomic types. * It is the responsibility of the caller to recurse into structs, maps and arrays. * * Type widening supports: * - byte -> short -> int -> long. * - float -> double. * - date -> timestamp_ntz. * - {byte, short, int} -> double. * - decimal -> wider decimal. * - {byte, short, int} -> decimal(10, 0) and wider. * - long -> decimal(20, 0) and wider. */ def isTypeChangeSupported(fromType: AtomicType, toType: AtomicType): Boolean = (fromType, toType) match { case (from, to) if from == to => true // All supported type changes below are supposed to be widening, but to be safe, reject any // non-widening change upfront. case (from, to) if !Cast.canUpCast(from, to) => false case (from: IntegralType, to: IntegralType) => from.defaultSize <= to.defaultSize case (FloatType, DoubleType) => true case (DateType, TimestampNTZType) => true case (ByteType | ShortType | IntegerType, DoubleType) => true case (from: DecimalType, to: DecimalType) => to.isWiderThan(from) // Byte, Short, Integer are all stored as INT32 in parquet. The parquet readers support // converting INT32 to Decimal(10, 0) and wider. case (ByteType | ShortType | IntegerType, d: DecimalType) => d.isWiderThan(IntegerType) // The parquet readers support converting INT64 to Decimal(20, 0) and wider. case (LongType, d: DecimalType) => d.isWiderThan(LongType) case _ => false } def isTypeChangeSupported( fromType: AtomicType, toType: AtomicType, uniformIcebergCompatibleOnly: Boolean): Boolean = isTypeChangeSupported(fromType, toType) && (!uniformIcebergCompatibleOnly || isTypeChangeSupportedByIceberg(fromType = fromType, toType = toType)) /** * Returns whether the given type change can be applied during schema evolution. Only a * subset of supported type changes are considered for schema evolution. */ def isTypeChangeSupportedForSchemaEvolution( fromType: AtomicType, toType: AtomicType, uniformIcebergCompatibleOnly: Boolean): Boolean = { val supportedForSchemaEvolution = (fromType, toType) match { case (from, to) if from == to => true case (from, to) if !isTypeChangeSupported(from, to) => false case (from: IntegralType, to: IntegralType) => from.defaultSize <= to.defaultSize case (FloatType, DoubleType) => true case (from: DecimalType, to: DecimalType) => to.isWiderThan(from) case (DateType, TimestampNTZType) => true case _ => false } supportedForSchemaEvolution && ( !uniformIcebergCompatibleOnly || isTypeChangeSupportedByIceberg(fromType = fromType, toType = toType) ) } /** * Returns whether the given type change is supported by Iceberg, and by extension can be read * using Uniform. See https://iceberg.apache.org/spec/#schema-evolution. * Note that these are type promotions supported by Iceberg V1 & V2 (both support the same type * promotions). Iceberg V3 will add support for date -> timestamp_ntz and void -> any but Uniform * doesn't currently support Iceberg V3. */ def isTypeChangeSupportedByIceberg(fromType: AtomicType, toType: AtomicType): Boolean = (fromType, toType) match { case (from, to) if from == to => true case (from, to) if !isTypeChangeSupported(from, to) => false case (from: IntegralType, to: IntegralType) => from.defaultSize <= to.defaultSize case (FloatType, DoubleType) => true case (from: DecimalType, to: DecimalType) if from.scale == to.scale && from.precision <= to.precision => true case _ => false } /** * Asserts that the given table doesn't contain any unsupported type changes. This should never * happen unless a non-compliant writer applied a type change that is not part of the feature * specification. */ def assertTableReadable(conf: SQLConf, protocol: Protocol, metadata: Metadata): Unit = { if (conf.getConf(DeltaSQLConf.DELTA_TYPE_WIDENING_BYPASS_UNSUPPORTED_TYPE_CHANGE_CHECK) || !isSupported(protocol) || !TypeWideningMetadata.containsTypeWideningMetadata(metadata.schema)) { return } TypeWideningMetadata.getAllTypeChanges(metadata.schema).foreach { case (_, TypeChange(_, from: AtomicType, to: AtomicType, _)) if isTypeChangeSupported(from, to) => // Char/Varchar/String type changes are allowed and independent from type widening. // Implementations shouldn't record these type changes in the table metadata per the Delta // spec, but in case that happen we really shouldn't block reading the table. case (_, TypeChange(_, _: StringType | CharType(_) | VarcharType(_), _: StringType | CharType(_) | VarcharType(_), _)) => case (fieldPath, TypeChange(_, from: AtomicType, to: AtomicType, _)) if stableFeatureCanReadTypeChange(from, to) => val featureName = if (protocol.isFeatureSupported(TypeWideningPreviewTableFeature)) { TypeWideningPreviewTableFeature } else { TypeWideningTableFeature } throw DeltaErrors.unsupportedTypeChangeInPreview(fieldPath, from, to, featureName) case (fieldPath, invalidChange) => throw DeltaErrors.unsupportedTypeChangeInSchema( fieldPath ++ invalidChange.fieldPath, invalidChange.fromType, invalidChange.toType ) } } /** * Whether the given type change is supported in the stable version of the feature. Used to * provide a helpful error message during the preview phase if upgrading to Delta 4.0 would allow * reading the table. */ private def stableFeatureCanReadTypeChange(fromType: AtomicType, toType: AtomicType): Boolean = (fromType, toType) match { case (from, to) if from == to => true case (from: IntegralType, to: IntegralType) => from.defaultSize <= to.defaultSize case (FloatType, DoubleType) => true case (DateType, TimestampNTZType) => true case (ByteType | ShortType | IntegerType, DoubleType) => true case (from: DecimalType, to: DecimalType) => to.isWiderThan(from) // Byte, Short, Integer are all stored as INT32 in parquet. The parquet readers support // converting INT32 to Decimal(10, 0) and wider. case (ByteType | ShortType | IntegerType, d: DecimalType) => d.isWiderThan(IntegerType) // The parquet readers support converting INT64 to Decimal(20, 0) and wider. case (LongType, d: DecimalType) => d.isWiderThan(LongType) case _ => false } /** * Compares `from` and `to` and returns whether the type was widened, or, for nested types, * whether one of the nested fields was widened. */ def containsWideningTypeChanges(from: DataType, to: DataType): Boolean = (from, to) match { case (from: StructType, to: StructType) => TypeWideningMetadata.collectTypeChanges(from, to).nonEmpty case (from: MapType, to: MapType) => containsWideningTypeChanges(from.keyType, to.keyType) || containsWideningTypeChanges(from.valueType, to.valueType) case (from: ArrayType, to: ArrayType) => containsWideningTypeChanges(from.elementType, to.elementType) case (from: AtomicType, to: AtomicType) => isTypeChangeSupported(from, to) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/TypeWideningMetadata.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.collection.mutable import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.{SchemaMergingUtils, SchemaUtils} import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.spark.sql.types._ /** * Information corresponding to a single type change. * @param version (Deprecated) The version of the table where the type change was made. This is * only populated by clients using the preview of type widening. * @param fromType The original type before the type change. * @param toType The new type after the type change. * @param fieldPath The path inside nested maps and arrays to the field where the type change was * made. Each path element is either `key`/`value` for maps or `element` for * arrays. The path is empty if the type change was applied inside a map or array. */ private[delta] case class TypeChange( version: Option[Long], fromType: DataType, toType: DataType, fieldPath: Seq[String]) { import TypeChange._ /** Serialize this type change to a [[Metadata]] object. */ def toMetadata: Metadata = { val builder = new MetadataBuilder() version.foreach(builder.putLong(TABLE_VERSION_METADATA_KEY, _)) builder .putString(FROM_TYPE_METADATA_KEY, fromType.typeName) .putString(TO_TYPE_METADATA_KEY, toType.typeName) if (fieldPath.nonEmpty) { builder.putString(FIELD_PATH_METADATA_KEY, fieldPath.mkString(".")) } builder.build() } } private[delta] object TypeChange { // tableVersion was a field present during the preview and removed afterwards. We preserve it if // it's already present in the type change metadata of the table to avoid breaking older clients // that use it to decide which files must be rewritten when dropping the feature. val TABLE_VERSION_METADATA_KEY: String = "tableVersion" val FROM_TYPE_METADATA_KEY: String = "fromType" val TO_TYPE_METADATA_KEY: String = "toType" val FIELD_PATH_METADATA_KEY: String = "fieldPath" /** Deserialize this type change from a [[Metadata]] object. */ def fromMetadata(metadata: Metadata): TypeChange = { val fieldPath = if (metadata.contains(FIELD_PATH_METADATA_KEY)) { metadata.getString(FIELD_PATH_METADATA_KEY).split("\\.").toSeq } else { Seq.empty } val version = if (metadata.contains(TABLE_VERSION_METADATA_KEY)) { Some(metadata.getLong(TABLE_VERSION_METADATA_KEY)) } else { None } TypeChange( version, fromType = DataType.fromDDL(metadata.getString(FROM_TYPE_METADATA_KEY)), toType = DataType.fromDDL(metadata.getString(TO_TYPE_METADATA_KEY)), fieldPath ) } } /** * Represents all type change information for a single struct field * @param typeChanges The type changes that have been applied to the field. */ private[delta] case class TypeWideningMetadata(typeChanges: Seq[TypeChange]) { import TypeWideningMetadata._ /** * Add the type changes to the metadata of the given field, preserving any pre-existing type * widening metadata. */ def appendToField(field: StructField): StructField = { if (typeChanges.isEmpty) return field val existingTypeChanges = fromField(field).map(_.typeChanges).getOrElse(Seq.empty) val allTypeChanges = existingTypeChanges ++ typeChanges val newMetadata = new MetadataBuilder().withMetadata(field.metadata) .putMetadataArray(TYPE_CHANGES_METADATA_KEY, allTypeChanges.map(_.toMetadata).toArray) .build() field.copy(metadata = newMetadata) } } private[delta] object TypeWideningMetadata extends DeltaLogging { val TYPE_CHANGES_METADATA_KEY: String = "delta.typeChanges" /** Read the type widening metadata from the given field. */ def fromField(field: StructField): Option[TypeWideningMetadata] = { Option.when(field.metadata.contains(TYPE_CHANGES_METADATA_KEY)) { val typeChanges = field.metadata.getMetadataArray(TYPE_CHANGES_METADATA_KEY) .map { changeMetadata => TypeChange.fromMetadata(changeMetadata) }.toSeq TypeWideningMetadata(typeChanges) } } /** * Computes the type changes from `oldSchema` to `schema` and adds corresponding type change * metadata to `schema`. */ def addTypeWideningMetadata( txn: OptimisticTransaction, schema: StructType, oldSchema: StructType): StructType = { if (!TypeWidening.isEnabled(txn.protocol, txn.metadata)) return schema if (DataType.equalsIgnoreNullability(schema, oldSchema)) return schema val changesToRecord = mutable.Buffer.empty[TypeChange] val schemaWithMetadata = SchemaMergingUtils.transformColumns(schema, oldSchema) { case (_, newField, Some(oldField), _) => var typeChanges = collectTypeChangesInStructField( oldField.dataType, newField.dataType, logNonWideningChanges = true ) // The version field isn't used anymore but we need to populate it in case the table uses // the preview feature, as preview clients may then rely on the field being present. if (txn.protocol.isFeatureSupported(TypeWideningPreviewTableFeature)) { typeChanges = typeChanges.map { change => change.copy(version = Some(txn.getFirstAttemptVersion)) } } changesToRecord ++= typeChanges TypeWideningMetadata(typeChanges).appendToField(newField) case (_, newField, None, _) => // The field was just added, no need to process. newField } if (changesToRecord.nonEmpty) { recordDeltaEvent( deltaLog = txn.snapshot.deltaLog, opType = "delta.typeWidening.typeChanges", data = Map( "changes" -> changesToRecord.map { change => Map( "fromType" -> change.fromType.sql, "toType" -> change.toType.sql) } )) } schemaWithMetadata } /** * Recursively compare `from` and `to` to collect all primitive widening type changes, including * in nested structs, maps and arrays. */ def collectTypeChanges(from: StructType, to: StructType): Seq[TypeChange] = { val changes = mutable.Buffer.empty[TypeChange] SchemaMergingUtils.transformColumns(schema = to, other = from) { case (path, newField, Some(oldField), _) => changes ++= collectTypeChangesInStructField( oldField.dataType, newField.dataType, logNonWideningChanges = false ).map { change => change.copy(fieldPath = path ++ Seq(newField.name) ++ change.fieldPath) } newField case (_, field, _, _) => field } changes.toSeq } /** * Collects all primitive widening type changes inside a single struct field. This includes type * changes inside nested maps and arrays but not inside other nested structs. * @param fromType The previous type of the struct field. * @param toType The new type of the struct field. * @param logNonWideningChanges Whether to log / fail in tests if a non-widening change is found. */ private def collectTypeChangesInStructField( fromType: DataType, toType: DataType, logNonWideningChanges: Boolean): Seq[TypeChange] = (fromType, toType) match { case (from: MapType, to: MapType) => collectTypeChangesInStructField(from.keyType, to.keyType, logNonWideningChanges) .map { typeChange => typeChange.copy(fieldPath = "key" +: typeChange.fieldPath) } ++ collectTypeChangesInStructField(from.valueType, to.valueType, logNonWideningChanges) .map { typeChange => typeChange.copy(fieldPath = "value" +: typeChange.fieldPath) } case (from: ArrayType, to: ArrayType) => collectTypeChangesInStructField(from.elementType, to.elementType, logNonWideningChanges) .map { typeChange => typeChange.copy(fieldPath = "element" +: typeChange.fieldPath) } case (fromType: AtomicType, toType: AtomicType) if fromType != toType && TypeWidening.isTypeChangeSupported(fromType, toType) => Seq(TypeChange( version = None, fromType, toType, fieldPath = Seq.empty )) // Char/Varchar/String and collation type changes are expected and unrelated to type widening. // We don't record them in the table schema metadata and don't log them as unexpected type // changes either. case (fromType: AtomicType, toType: AtomicType) if isStringTypeChange(fromType, toType) => Seq.empty // Don't recurse inside structs, `collectTypeChanges` should be called directly on each struct // fields instead to only collect type changes inside these fields. case (_: StructType, _: StructType) => Seq.empty case _ => deltaAssert(!logNonWideningChanges || fromType == toType, name = "typeWidening.unexpectedTypeChange", msg = s"Trying to apply an unsupported type change: $fromType to $toType", data = Map( "fromType" -> fromType.sql, "toType" -> toType.sql ) ) Seq.empty } /** Returns whether the given type change is Char/Varchar/String or collation type change. */ private def isStringTypeChange(from: AtomicType, to: AtomicType): Boolean = (from, to) match { case ( _: StringType | CharType(_) | VarcharType(_), _: StringType | CharType(_) | VarcharType(_)) => true case _ => false } /** * Change the `tableVersion` value in the type change metadata present in `schema`. Used during * conflict resolution to update the version associated with the transaction is incremented. * * Note: The `tableVersion` field is only populated for tables that use the preview of type * widening, we could remove this if/when there are no more tables using the preview of the * feature. */ def updateTypeChangeVersion(schema: StructType, fromVersion: Long, toVersion: Long): StructType = SchemaMergingUtils.transformColumns(schema) { case (_, field, _) => fromField(field) match { case Some(typeWideningMetadata) => val updatedTypeChanges = typeWideningMetadata.typeChanges.map { case typeChange if typeChange.version.contains(fromVersion) => typeChange.copy(version = Some(toVersion)) case olderTypeChange => olderTypeChange } val newMetadata = new MetadataBuilder().withMetadata(field.metadata) .putMetadataArray( TYPE_CHANGES_METADATA_KEY, updatedTypeChanges.map(_.toMetadata).toArray) .build() field.copy(metadata = newMetadata) case None => field } } /** * Remove the type widening metadata from all the fields in the given schema. * Return the cleaned schema and a list of fields with their path that had type widening metadata. */ def removeTypeWideningMetadata(schema: StructType) : (StructType, Seq[(Seq[String], StructField)]) = { if (!containsTypeWideningMetadata(schema)) return (schema, Seq.empty) val changes = mutable.Buffer.empty[(Seq[String], StructField)] val newSchema = SchemaMergingUtils.transformColumns(schema) { case (fieldPath: Seq[String], field: StructField, _) if field.metadata.contains(TYPE_CHANGES_METADATA_KEY) => changes.append((fieldPath, field)) val cleanMetadata = new MetadataBuilder() .withMetadata(field.metadata) .remove(TYPE_CHANGES_METADATA_KEY) .build() field.copy(metadata = cleanMetadata) case (_, field: StructField, _) => field } newSchema -> changes.toSeq } /** Recursively checks whether any struct field in the schema contains type widening metadata. */ def containsTypeWideningMetadata(schema: StructType): Boolean = schema.existsRecursively { case s: StructType => s.exists(_.metadata.contains(TYPE_CHANGES_METADATA_KEY)) case _ => false } /** * Return all type changes recorded in the table schema. * @return A list of tuples (field path, type change). */ def getAllTypeChanges(schema: StructType): Seq[(Seq[String], TypeChange)] = { if (!containsTypeWideningMetadata(schema)) return Seq.empty val allStructFields = SchemaUtils.filterRecursively(schema, checkComplexTypes = true) { _ => true } def getTypeChanges(field: StructField): Seq[TypeChange] = fromField(field) .map(_.typeChanges) .getOrElse(Seq.empty) allStructFields.flatMap { case (fieldPath, field) => getTypeChanges(field).map((fieldPath :+ field.name, _)) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/TypeWideningMode.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.catalyst.analysis.DecimalPrecisionTypeCoercion import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.sources.DeltaSQLConf.AllowAutomaticWideningMode import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{AtomicType, ByteType, DecimalType, IntegerType, IntegralType, LongType, ShortType} /** * A type widening mode captures a specific set of type changes that are allowed to be applied. * Currently: * - NoTypeWidening: No type change is allowed. * - AllTypeWidening: Allows widening to the target type using any supported type change. * - TypeEvolution: Only allows widening to the target type if the type change is eligible to be * applied automatically during schema evolution. * - AllTypeWideningToCommonWiderType: Allows widening to a common (possibly different) wider type * using any supported type change. * - TypeEvolutionToCommonWiderType: Allows widening to a common (possibly different) wider type * using only type changes that are eligible to be applied automatically during schema * evolution. * * TypeEvolution modes can be restricted to only type changes supported by Iceberg by passing * `uniformIcebergCompatibleOnly = truet`, to ensure that we don't automatically apply a type change * that would break Iceberg compatibility. */ sealed trait TypeWideningMode extends DeltaLogging { def getWidenedType(fromType: AtomicType, toType: AtomicType): Option[AtomicType] def shouldWidenTo(fromType: AtomicType, toType: AtomicType): Boolean = getWidenedType(fromType, toType).contains(toType) } object TypeWideningMode { /** * No type change allowed. Typically because type widening and/or schema evolution isn't enabled. */ case object NoTypeWidening extends TypeWideningMode { override def getWidenedType(fromType: AtomicType, toType: AtomicType): Option[AtomicType] = None } /** All supported type widening changes are allowed. */ case object AllTypeWidening extends TypeWideningMode { override def getWidenedType(fromType: AtomicType, toType: AtomicType): Option[AtomicType] = Option.when(TypeWidening.isTypeChangeSupported(fromType = fromType, toType = toType))(toType) } /** * Type changes that are eligible to be applied automatically during schema evolution are allowed. * * uniformIcebergCompatibleOnly: Restricts widenings to those supported by Iceberg. * allowAutomaticWidening: Controls widening behavior. Options: * - 'always': enables all supported widenings, * - 'same_family_type': uses default behavior, * - 'never': disables all widenings. */ case class TypeEvolution( uniformIcebergCompatibleOnly: Boolean, allowAutomaticWidening: AllowAutomaticWideningMode.Value) extends TypeWideningMode { override def getWidenedType(fromType: AtomicType, toType: AtomicType): Option[AtomicType] = { Option.when(canWiden(fromType, toType))(toType).orElse { logMissedWidening(fromType = fromType, toType = toType) None } } private def logMissedWidening(fromType: AtomicType, toType: AtomicType): Unit = { // Check if widening is possible under the least restricting conditions. val allowAllTypeEvolution = TypeEvolution( uniformIcebergCompatibleOnly = false, allowAutomaticWidening = AllowAutomaticWideningMode.ALWAYS) if (allowAllTypeEvolution.canWiden(fromType, toType)) { recordDeltaEvent(null, opType = "delta.typeWidening.missedAutomaticWidening", data = Map( "fromType" -> fromType.sql, "toType" -> toType.sql, "uniformIcebergCompatibleOnly" -> uniformIcebergCompatibleOnly, "allowAutomaticWidening" -> allowAutomaticWidening )) } } private def canWiden(fromType: AtomicType, toType: AtomicType): Boolean = { if (allowAutomaticWidening == AllowAutomaticWideningMode.ALWAYS) { TypeWidening.isTypeChangeSupported( fromType = fromType, toType = toType, uniformIcebergCompatibleOnly) } else if (allowAutomaticWidening == AllowAutomaticWideningMode.SAME_FAMILY_TYPE) { TypeWidening.isTypeChangeSupportedForSchemaEvolution( fromType = fromType, toType = toType, uniformIcebergCompatibleOnly) } else { false } } } /** * All supported type widening changes are allowed. Unlike [[AllTypeWidening]], this also allows * widening `to` to `from`, and for decimals, widening to a different decimal type that is wider * than both input types. Use for example when merging two unrelated schemas and we want just want * to find a wider schema to use. */ case object AllTypeWideningToCommonWiderType extends TypeWideningMode { private def getDecimalType(t: IntegralType): DecimalType = { t match { case _: ByteType => DecimalType(3, 0) case _: ShortType => DecimalType(5, 0) case _: IntegerType => DecimalType(10, 0) case _: LongType => DecimalType(20, 0) } } override def getWidenedType(left: AtomicType, right: AtomicType): Option[AtomicType] = { val allowIntegralDecimalCoercion: Boolean = SQLConf.get.getConf(DeltaSQLConf.DELTA_TYPE_WIDENING_ALLOW_INTEGRAL_DECIMAL_COERCION) (left, right) match { case (l, r) if TypeWidening.isTypeChangeSupported(l, r) => Some(r) case (l, r) if TypeWidening.isTypeChangeSupported(r, l) => Some(l) case (l: IntegralType, r: DecimalType) if allowIntegralDecimalCoercion => getWidenedType(getDecimalType(l), r) case (l: DecimalType, r: IntegralType) if allowIntegralDecimalCoercion => getWidenedType(getDecimalType(r), l) case (l: DecimalType, r: DecimalType) => val wider = DecimalPrecisionTypeCoercion.widerDecimalType(l, r) Option.when( TypeWidening.isTypeChangeSupported(l, wider) && TypeWidening.isTypeChangeSupported(r, wider))(wider) case _ => None } } } /** * Type changes that are eligible to be applied automatically during schema evolution are allowed. * Can be restricted to only type changes supported by Iceberg. Unlike [[TypeEvolution]], this * also allows widening `to` to `from`, and for decimals, widening to a different decimal type * that is wider han both input types. Use for example when merging two unrelated schemas and we * want just want to find a wider schema to use. */ case class TypeEvolutionToCommonWiderType(uniformIcebergCompatibleOnly: Boolean) extends TypeWideningMode { override def getWidenedType(left: AtomicType, right: AtomicType): Option[AtomicType] = { def typeChangeSupported: (AtomicType, AtomicType) => Boolean = TypeWidening.isTypeChangeSupportedForSchemaEvolution(_, _, uniformIcebergCompatibleOnly) (left, right) match { case (l, r) if typeChangeSupported(l, r) => Some(r) case (l, r) if typeChangeSupported(r, l) => Some(l) case (l: DecimalType, r: DecimalType) => val wider = DecimalPrecisionTypeCoercion.widerDecimalType(l, r) Option.when(typeChangeSupported(l, wider) && typeChangeSupported(r, wider))(wider) case _ => None } } } /** * Same as TypeEvolution with AllowAutomaticWideningMode.ALWAYS, but * additionally gets the wider decimal type given two types that are * DecimalType-compatible. */ case object AllTypeWideningWithDecimalCoercion extends TypeWideningMode { private def getDecimalType(t: IntegralType): DecimalType = { t match { case _: ByteType => DecimalType(3, 0) case _: ShortType => DecimalType(5, 0) case _: IntegerType => DecimalType(10, 0) case _: LongType => DecimalType(20, 0) } } private def getWiderDecimalTypeWithInteger( integralType: IntegralType, decimalType: DecimalType): Option[DecimalType] = { val wider = DecimalPrecisionTypeCoercion.widerDecimalType( getDecimalType(integralType), decimalType) Option.when( TypeWidening.isTypeChangeSupported(getDecimalType(integralType), wider) && TypeWidening.isTypeChangeSupported(decimalType, wider))(wider) } override def getWidenedType(fromType: AtomicType, toType: AtomicType): Option[AtomicType] = (fromType, toType) match { case (from, to) if TypeWidening.isTypeChangeSupported(from, to) => Some(to) case (l: IntegralType, r: DecimalType) => getWiderDecimalTypeWithInteger(l, r) case (l: DecimalType, r: IntegralType) => getWiderDecimalTypeWithInteger(r, l) case (l: DecimalType, r: DecimalType) => val wider = DecimalPrecisionTypeCoercion.widerDecimalType(l, r) Option.when( TypeWidening.isTypeChangeSupported(l, wider) && TypeWidening.isTypeChangeSupported(r, wider))(wider) case _ => None } } /** * Same as TypeEvolution with AllowAutomaticWideningMode.SAME_FAMILY_TYPE, * but additionally gets the wider decimal type given two types that are * DecimalType-compatible. */ case object TypeEvolutionWithDecimalCoercion extends TypeWideningMode { override def getWidenedType(fromType: AtomicType, toType: AtomicType): Option[AtomicType] = { def typeChangeSupported: (AtomicType, AtomicType) => Boolean = TypeWidening.isTypeChangeSupportedForSchemaEvolution(_, _, uniformIcebergCompatibleOnly = false) (fromType, toType) match { case (from, to) if typeChangeSupported(from, to) => Some(to) case (l: DecimalType, r: DecimalType) => val wider = DecimalPrecisionTypeCoercion.widerDecimalType(l, r) Option.when(typeChangeSupported(l, wider) && typeChangeSupported(r, wider))(wider) case _ => None } } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/UniversalFormat.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.actions.{Action, Metadata, Protocol} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.internal.MDC import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.types.{ByteType, CalendarIntervalType, NullType, ShortType, TimestampNTZType} /** * Utils to validate the Universal Format (UniForm) Delta feature (NOT a table feature). * * The UniForm Delta feature governs and implements the actual conversion of Delta metadata into * other formats. * * UniForm supports both Iceberg and Hudi. When `delta.universalFormat.enabledFormats` contains * "iceberg", we say that Universal Format (Iceberg) is enabled. When it contains "hudi", we say * that Universal Format (Hudi) is enabled. * * [[enforceInvariantsAndDependencies]] ensures that all of UniForm's requirements for the * specified format are met (e.g. for 'iceberg' that IcebergCompatV1 or V2 is enabled). * It doesn't verify that its nested requirements are met (e.g. IcebergCompat's requirements, * like Column Mapping). That is the responsibility of format-specific validations such as * [[IcebergCompatV1.enforceInvariantsAndDependencies]] * and [[IcebergCompatV2.enforceInvariantsAndDependencies]]. * * * Note that UniForm (Iceberg) depends on IcebergCompat, but IcebergCompat does not * depend on or require UniForm (Iceberg). It is perfectly valid for a Delta table to have * IcebergCompatV1 or V2 enabled but UniForm (Iceberg) not enabled. */ object UniversalFormat extends DeltaLogging { val ICEBERG_FORMAT = "iceberg" val HUDI_FORMAT = "hudi" val SUPPORTED_FORMATS = Set(HUDI_FORMAT, ICEBERG_FORMAT) /** * Check if the operation is CREATE/REPLACE TABLE or REORG UPGRADE UNIFORM commands. * * @param op the delta operation to be checked. * @return whether the operation is create or reorg. */ def isCreatingOrReorgTable(op: Option[DeltaOperations.Operation]): Boolean = op match { case Some(_: DeltaOperations.CreateTable) | Some(_: DeltaOperations.UpgradeUniformProperties) | // REPLACE TABLE is also considered creating table to preserve the // the semantics for `isCreatingNewTable` in `OptimisticTransaction`. Some(_: DeltaOperations.ReplaceTable) => true // this is to conform with the semantics in `enforceDependenciesInConfiguration` case None => true case _ => false } /** * Check if the operation is REORG UPGRADE UNIFORM command. * * @param op the delta operation to be checked. * @return whether the operation is REORG UPGRADE UNIFORM. */ def isReorgUpgradeUniform(op: Option[DeltaOperations.Operation]): Boolean = op match { case Some(_: DeltaOperations.UpgradeUniformProperties) => true case _ => false } def icebergEnabled(metadata: Metadata): Boolean = { DeltaConfigs.UNIVERSAL_FORMAT_ENABLED_FORMATS.fromMetaData(metadata).contains(ICEBERG_FORMAT) } def hudiEnabled(metadata: Metadata): Boolean = { DeltaConfigs.UNIVERSAL_FORMAT_ENABLED_FORMATS.fromMetaData(metadata).contains(HUDI_FORMAT) } def hudiEnabled(properties: Map[String, String]): Boolean = { properties.get(DeltaConfigs.UNIVERSAL_FORMAT_ENABLED_FORMATS.key) .exists(value => value.contains(HUDI_FORMAT)) } def icebergEnabled(properties: Map[String, String]): Boolean = { properties.get(DeltaConfigs.UNIVERSAL_FORMAT_ENABLED_FORMATS.key) .exists(value => value.contains(ICEBERG_FORMAT)) } /** * Expected to be called after the newest metadata and protocol have been ~ finalized. * * @return tuple of options of (updatedProtocol, updatedMetadata). For either action, if no * updates need to be applied, will return None. */ def enforceInvariantsAndDependencies( spark: SparkSession, catalogTable: Option[CatalogTable], snapshot: Snapshot, newestProtocol: Protocol, newestMetadata: Metadata, operation: Option[DeltaOperations.Operation], actions: Seq[Action]): (Option[Protocol], Option[Metadata]) = { enforceHudiDependencies(newestMetadata, snapshot) enforceIcebergInvariantsAndDependencies( spark, catalogTable, snapshot, newestProtocol, newestMetadata, operation, actions) } /** * If you are enabling Hudi, this method ensures that Deletion Vectors are not enabled. New * conditions may be added here in the future to make sure the source is compatible with Hudi. * @param newestMetadata the newest metadata * @param snapshot current snapshot * @return N/A, throws exception if condition is not met */ def enforceHudiDependencies(newestMetadata: Metadata, snapshot: Snapshot): Any = { if (hudiEnabled(newestMetadata)) { if (DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.fromMetaData(newestMetadata)) { throw DeltaErrors.uniFormHudiDeleteVectorCompat() } SchemaUtils.findAnyTypeRecursively(newestMetadata.schema) { f => f.isInstanceOf[NullType] | f.isInstanceOf[ByteType] | f.isInstanceOf[ShortType] | f.isInstanceOf[TimestampNTZType] } match { case Some(unsupportedType) => throw DeltaErrors.uniFormHudiSchemaCompat(unsupportedType) case _ => } } } /** * If you are enabling Universal Format (Iceberg), this method ensures that at least one of * IcebergCompat is enabled. If you are disabling Universal Format (Iceberg), this method * will leave the current IcebergCompat version untouched. * * @return tuple of options of (updatedProtocol, updatedMetadata). For either action, if no * updates need to be applied, will return None. */ def enforceIcebergInvariantsAndDependencies( spark: SparkSession, catalogTable: Option[CatalogTable], snapshot: Snapshot, newestProtocol: Protocol, newestMetadata: Metadata, operation: Option[DeltaOperations.Operation], actions: Seq[Action]): (Option[Protocol], Option[Metadata]) = { val prevMetadata = snapshot.metadata val uniformIcebergWasEnabled = UniversalFormat.icebergEnabled(prevMetadata) val uniformIcebergIsEnabled = UniversalFormat.icebergEnabled(newestMetadata) val tableId = newestMetadata.id var changed = false val (uniformProtocol, uniformMetadata) = (uniformIcebergWasEnabled, uniformIcebergIsEnabled) match { case (_, false) => (None, None) // Ignore case (_, true) => // Enabling now or already-enabled val icebergCompatWasEnabled = IcebergCompat.isAnyEnabled(prevMetadata) val icebergCompatIsEnabled = IcebergCompat.isAnyEnabled(newestMetadata) if (icebergCompatIsEnabled) { (None, None) } else if (icebergCompatWasEnabled) { // IcebergCompat is being disabled. We need to also disable Universal Format (Iceberg) val remainingSupportedFormats = DeltaConfigs.UNIVERSAL_FORMAT_ENABLED_FORMATS .fromMetaData(newestMetadata) .filterNot(_ == UniversalFormat.ICEBERG_FORMAT) val newConfiguration = if (remainingSupportedFormats.isEmpty) { newestMetadata.configuration - DeltaConfigs.UNIVERSAL_FORMAT_ENABLED_FORMATS.key } else { newestMetadata.configuration ++ Map(DeltaConfigs.UNIVERSAL_FORMAT_ENABLED_FORMATS.key -> remainingSupportedFormats.mkString(",")) } logInfo(log"[${MDC(DeltaLogKeys.TABLE_ID, tableId)}] " + log"IcebergCompat is being disabled. Auto-disabling Universal Format (Iceberg), too.") (None, Some(newestMetadata.copy(configuration = newConfiguration))) } else { throw DeltaErrors.uniFormIcebergRequiresIcebergCompat() } } var protocolToCheck = uniformProtocol.getOrElse(newestProtocol) var metadataToCheck = uniformMetadata.getOrElse(newestMetadata) changed = uniformProtocol.nonEmpty || uniformMetadata.nonEmpty var protocolUpdate: Option[Protocol] = None var metadataUpdate: Option[Metadata] = None val compatChecks: Seq[ (SparkSession, Option[CatalogTable], Snapshot, Protocol, Metadata, Option[DeltaOperations.Operation], Seq[Action]) => (Option[Protocol], Option[Metadata])] = Seq( IcebergCompatV1.enforceInvariantsAndDependencies, IcebergCompatV2.enforceInvariantsAndDependencies ) compatChecks.foreach { compatCheck => val updates = compatCheck( spark, catalogTable, snapshot, protocolToCheck, metadataToCheck, operation, actions ) protocolUpdate = updates._1 metadataUpdate = updates._2 protocolToCheck = protocolUpdate.getOrElse(protocolToCheck) metadataToCheck = metadataUpdate.getOrElse(metadataToCheck) changed ||= protocolUpdate.nonEmpty || metadataUpdate.nonEmpty } if (changed) { ( protocolUpdate.orElse(Some(protocolToCheck)), metadataUpdate.orElse(Some(metadataToCheck)) ) } else { (None, None) } } /** * This method is used to build UniForm metadata dependencies closure. * It checks configuration conflicts and adds missing properties. * It will call [[enforceIcebergInvariantsAndDependencies]] to perform the actual check. * @param configuration the original metadata configuration. * @return updated configuration if any changes are required, * otherwise the original configuration. */ def enforceDependenciesInConfiguration( spark: SparkSession, catalogTable: CatalogTable, configuration: Map[String, String], snapshot: Snapshot): Map[String, String] = { var metadata = snapshot.metadata.copy(configuration = configuration) // Check UniversalFormat related property dependencies val (_, universalMetadata) = UniversalFormat.enforceInvariantsAndDependencies( spark, catalogTable = Some(catalogTable), snapshot, newestProtocol = snapshot.protocol, newestMetadata = metadata, operation = None, actions = Seq() ) universalMetadata match { case Some(valid) => valid.configuration case _ => configuration } } val ICEBERG_TABLE_TYPE_KEY = "table_type" /** * HiveTableOperations ensures table_type is 'ICEBERG' when uniform is enabled * This enforceSupportInCatalog ensure table_type is not 'ICEBERG' when uniform is not enabled * * @param table catalogTable before change * @param metadata snapshot metadata * @return the converted catalog, or None if no change is made */ def enforceSupportInCatalog(table: CatalogTable, metadata: Metadata): Option[CatalogTable] = { val icebergInCatalog = table.properties.get(ICEBERG_TABLE_TYPE_KEY) match { case Some(value) => value.equalsIgnoreCase(ICEBERG_FORMAT) case _ => false } (icebergEnabled(metadata), icebergInCatalog) match { case (false, true) => Some(table.copy(properties = table.properties - ICEBERG_TABLE_TYPE_KEY)) case _ => None } } } /** Class to facilitate the conversion of Delta into other table formats. */ abstract class UniversalFormatConverter { /** The current Spark session. */ def spark: SparkSession = SparkSession.active /** * Perform an asynchronous conversion. * * This will start an async job to run the conversion, unless there already is an async conversion * running for this table. In that case, it will queue up the provided snapshot to be run after * the existing job completes. */ def enqueueSnapshotForConversion( snapshotToConvert: Snapshot, txn: CommittedTransaction): Unit /** * Perform a blocking conversion when performing an OptimisticTransaction * on a delta table. * * @param snapshotToConvert the snapshot that needs to be converted to Iceberg * @param txn the transaction that triggers the conversion. Used as a hint to * avoid recomputing old metadata. It must contain the catalogTable * this conversion targets. * @return Converted Delta version and commit timestamp */ def convertSnapshot( snapshotToConvert: Snapshot, txn: CommittedTransaction): Option[(Long, Long)] /** * Perform a blocking conversion for the given catalogTable * * @param snapshotToConvert the snapshot that needs to be converted to Iceberg * @param catalogTable the catalogTable this conversion targets. * @return Converted Delta version and commit timestamp */ def convertSnapshot( snapshotToConvert: Snapshot, catalogTable: CatalogTable): Option[(Long, Long)] /** * Fetch the delta version corresponding to the latest conversion. * @param snapshot the snapshot to be converted * @param table the catalogTable with info of previous conversions * @return None if no previous conversion found */ def loadLastDeltaVersionConverted(snapshot: Snapshot, table: CatalogTable): Option[Long] } object IcebergConstants { val ICEBERG_TBLPROP_METADATA_LOCATION = "metadata_location" val ICEBERG_PROVIDER = "iceberg" val ICEBERG_NAME_MAPPING_PROPERTY = "schema.name-mapping.default" // Reserved field ID for the `_row_id` column // Iceberg spec: https://iceberg.apache.org/spec/?h=row#reserved-field-ids val ICEBERG_ROW_TRACKING_ROW_ID_FIELD_ID = 2147483540L // Reserved field ID for the `_last_updated_sequence_number` column val ICEBERG_ROW_TRACKING_LAST_UPDATED_SEQUENCE_NUMBER_FIELD_ID = 2147483539L } object HudiConstants { val HUDI_PROVIDER = "hudi" } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/UpdateExpressionsSupport.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.AnalysisHelper import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.{InternalRow, SQLConfHelper} import org.apache.spark.sql.catalyst.analysis.Resolver import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGenerator, ExprCode} import org.apache.spark.sql.catalyst.expressions.codegen.Block._ import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project} import org.apache.spark.sql.catalyst.plans.logical.TargetOnlyStructFieldBehavior import org.apache.spark.sql.catalyst.types.DataTypeUtils import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ /** * Trait with helper functions to generate expressions to update target columns, even if they are * nested fields. */ trait UpdateExpressionsSupport extends SQLConfHelper with AnalysisHelper with DeltaLogging { /** * Specifies an operation that updates a target column with the given expression. * The target column may or may not be a nested field and it is specified as a full quoted name * or as a sequence of split into parts. * * @param targetColNameParts The name parts of the target column * @param updateExpr The expression to update the column with * @param targetOnlyStructFieldBehavior Determines the nullness of target-only struct fields. * Note: This parameter only takes effect when schema * evolution is enabled; otherwise it is ignored. */ case class UpdateOperation( targetColNameParts: Seq[String], updateExpr: Expression, targetOnlyStructFieldBehavior: TargetOnlyStructFieldBehavior.Value) /** * The following trait and classes define casting behaviors to use in `castIfNeeded()`. * @param resolveStructsByName Whether struct fields should be resolved by name or by position * during struct cast. * @param allowMissingStructField Whether missing struct fields are allowed in the data to cast. * Only relevant when struct fields are resolved by name. * When true, missing struct fields in the input are set to null. * When false, an error is thrown. * Note: this should be set to true for schema evolution to work as * the target schema may typically contain new struct fields not * present in the input. */ sealed trait CastingBehavior { val resolveStructsByName: Boolean val allowMissingStructField: Boolean } case class CastByPosition() extends CastingBehavior { val resolveStructsByName: Boolean = false val allowMissingStructField: Boolean = false } case class CastByName(allowMissingStructField: Boolean) extends CastingBehavior { val resolveStructsByName: Boolean = true } /* * MERGE and UPDATE casting behavior is configurable using internal configs to allow reverting to * legacy behavior. In particular: * - 'resolveMergeUpdateStructsByName.enabled': defaults to resolution by name for struct fields, * can be disabled to revert to resolution by position. * - 'updateAndMergeCastingFollowsAnsiEnabledFlag': defaults to following * 'spark.sql.storeAssignmentPolicy' for the type of cast to use, can be enabled to revert to * following 'spark.sql.ansi.enabled'. See `cast()` below. */ trait MergeOrUpdateCastingBehavior object MergeOrUpdateCastingBehavior { def apply(schemaEvolutionEnabled: Boolean): CastingBehavior = if (conf.getConf(DeltaSQLConf.DELTA_RESOLVE_MERGE_UPDATE_STRUCTS_BY_NAME)) { new CastByName(allowMissingStructField = schemaEvolutionEnabled) with MergeOrUpdateCastingBehavior } else { new CastByPosition() with MergeOrUpdateCastingBehavior } } /** * Add a cast to the child expression if it differs from the specified data type. Note that * structs here are cast by name, rather than the Spark SQL default of casting by position. * * @param fromExpression the expression to cast * @param dataType The data type to cast to. * @param castingBehavior Configures the casting behavior to use, see [[CastingBehavior]]. * @param columnName The name of the column written to. It is used for the error message. * @param originalTargetExprOpt Optional expression representing the original target column before * the update. It is only relevant in MERGE ... UPDATE * with schema * evolution to preserve the original values of target-only struct * fields. In other cases, it is None and the target-only fields will * be overwritten with null. */ protected def castIfNeeded( fromExpression: Expression, dataType: DataType, castingBehavior: CastingBehavior, columnName: String, originalTargetExprOpt: Option[Expression] = None): Expression = { fromExpression match { // Need to deal with NullType here, as some types cannot be casted from NullType, e.g., // StructType. case Literal(nul, NullType) => Literal(nul, dataType) case otherExpr => (fromExpression.dataType, dataType) match { case (ArrayType(fromEt: StructType, fromNullable), to @ ArrayType(toEt: StructType, toNullable)) if !(DataTypeUtils.sameType(fromEt, toEt) && fromNullable == toNullable) => fromExpression match { // If fromExpression is an array function returning an array, cast the // underlying array first and then perform the function on the transformed array. case ArrayUnion(leftExpression, rightExpression) => val castedLeft = castIfNeeded(leftExpression, dataType, castingBehavior, columnName) val castedRight = castIfNeeded(rightExpression, dataType, castingBehavior, columnName) ArrayUnion(castedLeft, castedRight) case ArrayIntersect(leftExpression, rightExpression) => val castedLeft = castIfNeeded(leftExpression, dataType, castingBehavior, columnName) val castedRight = castIfNeeded(rightExpression, dataType, castingBehavior, columnName) ArrayIntersect(castedLeft, castedRight) case ArrayExcept(leftExpression, rightExpression) => val castedLeft = castIfNeeded(leftExpression, dataType, castingBehavior, columnName) val castedRight = castIfNeeded(rightExpression, dataType, castingBehavior, columnName) ArrayExcept(castedLeft, castedRight) case ArrayRemove(leftExpression, rightExpression) => val castedLeft = castIfNeeded(leftExpression, dataType, castingBehavior, columnName) // ArrayRemove removes all elements that equal to element from the given array. // In this case, the element to be removed also needs to be casted into the target // array's element type. val castedRight = castIfNeeded(rightExpression, toEt, castingBehavior, columnName) ArrayRemove(castedLeft, castedRight) case ArrayDistinct(expression) => val castedExpr = castIfNeeded(expression, dataType, castingBehavior, columnName) ArrayDistinct(castedExpr) case _ => // generate a lambda function to cast each array item into to element struct type. val structConverter: (Expression, Expression) => Expression = (_, i) => castIfNeeded( GetArrayItem(fromExpression, i), toEt, castingBehavior, columnName) val transformLambdaFunc = { val elementVar = NamedLambdaVariable("elementVar", toEt, toNullable) val indexVar = NamedLambdaVariable("indexVar", IntegerType, false) LambdaFunction(structConverter(elementVar, indexVar), Seq(elementVar, indexVar)) } // Transforms every element in the array using the lambda function. // Because castIfNeeded is called recursively for array elements, which // generates nullable expression, ArrayTransform will generate an ArrayType with // containsNull as true. Thus, the ArrayType to be casted to need to have // containsNull as true to avoid casting failures. cast( ArrayTransform(fromExpression, transformLambdaFunc), to.asNullable, castingBehavior, columnName ) } case (from: MapType, to: MapType) if !Cast.canCast(from, to) || ( // Structs can be nested into the MapType, so if we need to do by-name casts, // we need to recurse into the children here. castingBehavior.resolveStructsByName && containsNestedStruct(from) && containsNestedStruct(to) && !DataTypeUtils.equalsIgnoreCaseAndNullability(from, to)) => // Manually convert map keys and values if the types are not compatible to allow schema // evolution. This is slower than direct cast so we only do it when required. def createMapConverter(convert: (Expression, Expression) => Expression): Expression = { val keyVar = NamedLambdaVariable("keyVar", from.keyType, nullable = false) val valueVar = NamedLambdaVariable("valueVar", from.valueType, from.valueContainsNull) LambdaFunction(convert(keyVar, valueVar), Seq(keyVar, valueVar)) } var transformedKeysAndValues = fromExpression if (from.keyType != to.keyType) { transformedKeysAndValues = TransformKeys(transformedKeysAndValues, createMapConverter { (key, _) => castIfNeeded(key, to.keyType, castingBehavior, columnName) }) } if (from.valueType != to.valueType) { transformedKeysAndValues = TransformValues(transformedKeysAndValues, createMapConverter { (_, value) => castIfNeeded(value, to.valueType, castingBehavior, columnName) }) } cast(transformedKeysAndValues, to.asNullable, castingBehavior, columnName) case (from: StructType, to: StructType) if !DataTypeUtils.equalsIgnoreCaseAndNullability(from, to) && castingBehavior.resolveStructsByName => // All from fields must be present in the final schema, or we'll silently lose data. if (from.exists { f => !to.exists(_.name.equalsIgnoreCase(f.name))}) { throw DeltaErrors.updateSchemaMismatchExpression(from, to) } // In addition, if we don't allow missing struct fields, then the number of fields must // necessarily match. if (from.length != to.length && !castingBehavior.allowMissingStructField) { throw DeltaErrors.updateSchemaMismatchExpression(from, to) } val originalTargetChildExprsOpt: Option[Map[String, Expression]] = if (UpdateExpressionsSupport.isUpdateStarPreserveNullSourceStructsEnabled(conf)) { extractOriginalTargetChildExprs(originalTargetExprOpt, to) } else { None } val nameMappedStruct = CreateNamedStruct(to.flatMap { field => val fieldNameLit = Literal(field.name) // flatMap returns None if (1) originalTargetChildExprsOpt is None, or // (2) originalTargetChildExprsOpt.get.get(field.name) is None, which happens when // the field doesn't exist in the original target (newly added via schema evolution). val targetFieldExprOpt: Option[Expression] = originalTargetChildExprsOpt.flatMap(_.get(field.name)) val extractedField = from .find { f => SchemaUtils.DELTA_COL_RESOLVER(f.name, field.name) } .map { _ => ExtractValue(fromExpression, fieldNameLit, SchemaUtils.DELTA_COL_RESOLVER) }.getOrElse { // This shouldn't be possible - if all columns aren't present when missing struct // fields aren't allowed, we should have thrown an error earlier. if (!castingBehavior.allowMissingStructField) { throw DeltaErrors.extractReferencesFieldNotFound(s"$field", DeltaErrors.updateSchemaMismatchExpression(from, to)) } // If the expression of the original target column is not provided or there is no // such field in the target column, fill the field with null. targetFieldExprOpt.getOrElse(Literal(null)) } Seq(fieldNameLit, castIfNeeded( extractedField, field.dataType, castingBehavior, field.name, targetFieldExprOpt)) }) // Fix for null expansion caused by struct type cast by preserving NULL source structs. // // Problem: When assigning a struct column, e.g., MERGE ... WHEN MATCHED THEN UPDATE SET // t.col = s.col, if the source struct is NULL, the casting logic will expand the NULL // into a non-null struct with all fields set to NULL: // NULL -> struct(field1: null, field2: null, ..., newField: null) // // Expected: The target struct should remain NULL when the source struct is NULL: // NULL -> NULL // // Solution: Wrap the named_struct expression in an IF expression that preserves NULL: // IF(source_struct IS NULL, NULL, named_struct(...)) // // Additional behavior when originalTargetExpr is provided (MERGE ... UPDATE * with // schema evolution): // For target-only fields (fields in target but not in source), we need to preserve // their original values from the target. The expression becomes: // IF(source_struct IS NULL AND target_struct IS NULL, // NULL, // named_struct( // source_fields..., // target_only_field1: original_target_value1, // target_only_field2: original_target_value2 // )) // This is to match the behavior of UPDATE * that target-only fields retain their // values. val wrappedWithNullPreservation = maybeWrapWithNullPreservation( sourceExpr = fromExpression, sourceType = from, targetType = to.asNullable, targetNamedStructExpr = nameMappedStruct, originalTargetExprOpt = originalTargetExprOpt) cast(wrappedWithNullPreservation, to.asNullable, castingBehavior, columnName) case (from, to) if from != to => cast(fromExpression, dataType, castingBehavior, columnName) case _ => fromExpression } } } /** * Extracts child expressions from the original target struct for fields that exist in both * the original target and the evolved target schemas. * * This is used during MERGE UPDATE * with schema evolution to preserve target-only field values. * * @param originalTargetExprOpt The original target column expression (before schema evolution) * @param evolvedTargetStruct The evolved target struct type (after schema evolution) * @return A map from evolved field names that exist in the original target schema to extraction * expressions from the original target, or None if no original target expression is * provided. */ private def extractOriginalTargetChildExprs( originalTargetExprOpt: Option[Expression], evolvedTargetStruct: StructType): Option[Map[String, Expression]] = { originalTargetExprOpt.map { e => require(e.dataType.isInstanceOf[StructType], s"originalTargetExprOpt dataType must be StructType but got ${e.dataType}") val originalTargetStruct = e.dataType.asInstanceOf[StructType] evolvedTargetStruct.flatMap { field => originalTargetStruct.find(f => SchemaUtils.DELTA_COL_RESOLVER(f.name, field.name)) .map { matchedField => // `field` is present in the target struct before schema evolution. // Use matchedField.name to extract from the original target expression. field.name -> ExtractValue( e, Literal(matchedField.name), SchemaUtils.DELTA_COL_RESOLVER) } }.toMap } } /** * Conditionally wraps an expression with an IF expression to preserve NULL source values, when * `DELTA_MERGE_PRESERVE_NULL_SOURCE_STRUCTS` is enabled: * IF(sourceExpr IS NULL, NULL, targetNamedStructExpr) * * When originalTargetExprOpt is defined (MERGE ... UPDATE * with schema evolution), the * null condition is extended to check whether both the source and target struct are null: * IF(sourceExpr IS NULL AND targetExpr IS NULL, NULL, targetNamedStructExpr) * This prevents data loss when the source is null but the target has non-null values in * target-only fields. * This is to match the behavior of UPDATE * that target-only fields retain their values. * * @param sourceExpr The expression of the source field * @param sourceType The source struct type * @param targetType The target struct type * @param targetNamedStructExpr The generated target named struct expression * @param originalTargetExprOpt The expression of the original target column. None when the * fix is disabled or no original target expression is provided. */ private def maybeWrapWithNullPreservation( sourceExpr: Expression, sourceType: StructType, targetType: StructType, targetNamedStructExpr: Expression, originalTargetExprOpt: Option[Expression]): Expression = { if (UpdateExpressionsSupport.isWholeStructAssignmentPreserveNullSourceStructsEnabled(conf)) { val sourceNullCondition = IsNull(sourceExpr) val targetHasExtraFieldsToPreserveValue = UpdateExpressionsSupport.hasExtraStructFieldsToPreserveValue(sourceType, targetType) val fullNullCondition = originalTargetExprOpt match { case Some(originalTargetExpr) if targetHasExtraFieldsToPreserveValue => // When there are target-only fields to preserve, we need to check whether both the source // and the original target are null. And(sourceNullCondition, IsNull(originalTargetExpr)) case Some(_) if !targetHasExtraFieldsToPreserveValue => // No target-only fields to preserve values for. sourceNullCondition case None => // No original target expression provided, which means we overwrite the target with the // source. sourceNullCondition } If( fullNullCondition, Literal.create(null, targetType), targetNamedStructExpr ) } else { targetNamedStructExpr } } /** * Given a target schema and a set of update operations, generate a list of update expressions, * which are aligned with the given schema. * * For update operations to nested struct fields, this method recursively walks down schema tree * and apply the update expressions along the way. * For example, assume table `target` has the following schema: * s1 struct, s2 struct, z int * * Given an update command: * * - UPDATE target SET s1.a = 1, s1.b = 2, z = 3 * * this method works as follows: * * generateUpdateExpressions( * targetSchema=[s1,s2,z], defaultExprs=[s1,s2, z], updateOps=[(s1.a, 1), (s1.b, 2), (z, 3)]) * -> generates expression for s1 - build recursively from child assignments * generateUpdateExpressions( * targetSchema=[a,b,c], defaultExprs=[a, b, c], updateOps=[(a, 1),(b, 2)], pathPrefix=["s1"]) * end-of-recursion * -> returns (1, 2, a.c) * -> generates expression for s2 - no child assignment and no update expression: use * default expression `s2` * -> generates expression for z - use available update expression `3` * -> returns ((1, 2, a.c), s2, 3) * * @param targetSchema schema to follow to generate update expressions. Due to schema evolution, * it may contain additional columns or fields not present in the original * table schema. * @param updateOps a set of update operations. * @param defaultExprs the expressions to use when no update operation is provided for a column * or field. This is typically the output from the base table. * @param pathPrefix the path from root to the current (nested) column. Only used for printing out * full column path in error messages. * @param allowSchemaEvolution Whether to allow generating expressions for new columns or fields * added by schema evolution. * @param generatedColumns the list of the generated columns in the table. When a column is a * generated column and the user doesn't provide a update expression, its * update expression in the return result will be None. * If `generatedColumns` is empty, any of the options in the return result * must be non-empty. * @return a sequence of expression options. The elements in the sequence are options because * when a column is a generated column but the user doesn't provide an update expression * for this column, we need to generate the update expression according to the generated * column definition. But this method doesn't have enough context to do that. Hence, we * return a `None` for this case so that the caller knows it should generate the update * expression for such column. For other cases, we will always return Some(expr). */ protected def generateUpdateExpressions( targetSchema: StructType, updateOps: Seq[UpdateOperation], defaultExprs: Seq[NamedExpression], resolver: Resolver, pathPrefix: Seq[String] = Nil, allowSchemaEvolution: Boolean = false, generatedColumns: Seq[StructField] = Nil): Seq[Option[Expression]] = { // Check that the head of nameParts in each update operation can match a target col. This avoids // silently ignoring invalid column names specified in update operations. updateOps.foreach { u => if (!targetSchema.exists(f => resolver(f.name, u.targetColNameParts.head))) { throw DeltaErrors.updateSetColumnNotFoundException( (pathPrefix :+ u.targetColNameParts.head).mkString("."), targetSchema.map(f => (pathPrefix :+ f.name).mkString("."))) } } // Transform each targetCol to a possibly updated expression targetSchema.map { targetCol => // The prefix of a update path matches the current targetCol path. val prefixMatchedOps = updateOps.filter(u => resolver(u.targetColNameParts.head, targetCol.name)) val defaultExpr = defaultExprs.find(f => resolver(f.name, targetCol.name)) // No prefix matches this target column, return its original expression. if (prefixMatchedOps.isEmpty) { // Check whether it's a generated column or not. If so, we will return `None` so that the // caller will generate an expression for this column. We cannot generate an expression at // this moment because a generated column may use other columns which we don't know their // update expressions yet. if (generatedColumns.find(f => resolver(f.name, targetCol.name)).nonEmpty) { None } else if (defaultExpr.nonEmpty) { if (conf.getConf(DeltaSQLConf.DELTA_MERGE_SCHEMA_EVOLUTION_FIX_NESTED_STRUCT_ALIGNMENT)) { Some(castIfNeeded( defaultExpr.get, targetCol.dataType, castingBehavior = MergeOrUpdateCastingBehavior(allowSchemaEvolution), targetCol.name)) } else { defaultExpr } } else { // This is a new column or field added by schema evolution that doesn't have an assignment // in this MERGE clause. Set it to null. // Log an assertion for now (and fail in test) if schema evolution is disabled. We should // turn this into an error in the future. deltaAssert(allowSchemaEvolution, name = "generateUpdateExpressions.allowSchemaEvolution", msg = "Generating an expression for a new column or field but schema evolution is " + "disabled." ) Some(Literal(null)) } } else { // The update operation whose path exactly matches the current targetCol path. val fullyMatchedOp = prefixMatchedOps.find(_.targetColNameParts.size == 1) if (fullyMatchedOp.isDefined) { // If a full match is found, then it should be the ONLY prefix match. Any other match // would be a conflict, whether it is a full match or prefix-only. For example, // when users are updating a nested column a.b, they can't simultaneously update a // descendant of a.b, such as a.b.c. if (prefixMatchedOps.size > 1) { throw DeltaErrors.updateSetConflictException( prefixMatchedOps.map(op => (pathPrefix ++ op.targetColNameParts).mkString("."))) } val preserveTargetOnlyFields = UpdateExpressionsSupport.isUpdateStarPreserveNullSourceStructsEnabled(conf) && allowSchemaEvolution && fullyMatchedOp.get.targetOnlyStructFieldBehavior == TargetOnlyStructFieldBehavior.PRESERVE val originalTargetExprOpt = if (preserveTargetOnlyFields) { // Expression corresponding to the original target column before the update. defaultExpr } else { None } // For an exact match, return the updateExpr from the update operation. Some(castIfNeeded( fullyMatchedOp.get.updateExpr, targetCol.dataType, castingBehavior = MergeOrUpdateCastingBehavior(allowSchemaEvolution), targetCol.name, originalTargetExprOpt)) } else { // So there are prefix-matched update operations, but none of them is a full match. Then // that means targetCol is a complex data type, so we recursively pass along the update // operations to its children. targetCol.dataType match { case childSchema: StructType => val defaultChildExprs = defaultExpr match { case Some(expr @ NamedExpression(_, StructType(fields))) => fields.zipWithIndex.map { case (field, ordinal) => Alias(GetStructField(expr, ordinal, Some(field.name)), field.name)() } case _ => Array.empty[NamedExpression] } // Recursively apply update operations to the children val childTargetExprs = generateUpdateExpressions( childSchema, prefixMatchedOps.map(u => u.copy(targetColNameParts = u.targetColNameParts.tail)), defaultChildExprs, resolver, pathPrefix :+ targetCol.name, allowSchemaEvolution, // Set `generatedColumns` to Nil because they are only valid in the top level. generatedColumns = Nil) .map(_.getOrElse { // Should not happen throw DeltaErrors.cannotGenerateUpdateExpressions() }) // Reconstruct the expression for targetCol using its possibly updated children val namedStructExprs = childSchema .zip(childTargetExprs) .flatMap { case (field, expr) => Seq(Literal(field.name), expr) } Some(CreateNamedStruct(namedStructExprs)) case otherType => throw DeltaErrors.updateNonStructTypeFieldNotSupportedException( (pathPrefix :+ targetCol.name).mkString("."), otherType) } } } } } /** See docs on overloaded method. */ protected def generateUpdateExpressions( targetSchema: StructType, defaultExprs: Seq[NamedExpression], nameParts: Seq[Seq[String]], updateExprs: Seq[Expression], resolver: Resolver, generatedColumns: Seq[StructField]): Seq[Option[Expression]] = { assert(nameParts.size == updateExprs.size) val updateOps = nameParts.zip(updateExprs).map { case (nameParts, expr) => UpdateOperation( targetColNameParts = nameParts, updateExpr = expr, // This method is called from regular UPDATE statements, where schema // evolution is not allowed. targetOnlyStructFieldBehavior = TargetOnlyStructFieldBehavior.TARGET_ALIGNED) } generateUpdateExpressions( targetSchema = targetSchema, updateOps = updateOps, defaultExprs = defaultExprs, resolver = resolver, generatedColumns = generatedColumns ) } /** * Generate update expressions for generated columns that the user doesn't provide a update * expression. For each item in `updateExprs` that's None, we will find its generation expression * from `generatedColumns`. In order to resolve this generation expression, we will create a * fake Project which contains all update expressions and resolve the generation expression with * this project. Source columns of a generation expression will also be replaced with their * corresponding update expressions. * * For example, given a table that has a generated column `g` defined as `c1 + 10`. For the * following update command: * * UPDATE target SET c1 = c2 + 100, c2 = 1000 * * We will generate the update expression `(c2 + 100) + 10`` for column `g`. Note: in this update * expression, we should use the old `c2` attribute rather than its new value 1000. * * @param updateTarget The logical plan of the table to be updated. * @param generatedColumns A list of generated columns. * @param updateExprs The aligned (with `postEvolutionTargetSchema` if not None, or * `updateTarget.output` otherwise) update actions. * @param postEvolutionTargetSchema In case of UPDATE in MERGE when schema evolution happened, * this is the final schema of the target table. This might not * be the same as the output of `updateTarget`. * @return a sequence of update expressions for all of columns in the table. */ protected def generateUpdateExprsForGeneratedColumns( updateTarget: LogicalPlan, generatedColumns: Seq[StructField], updateExprs: Seq[Option[Expression]], postEvolutionTargetSchema: Option[StructType] = None): Seq[Expression] = { val targetSchema = postEvolutionTargetSchema.getOrElse(updateTarget.schema) assert( targetSchema.size == updateExprs.length, s"'generateUpdateExpressions' should return expressions that are aligned with the column " + s"list. Expected size: ${targetSchema.size}, actual size: ${updateExprs.length}") val schemaWithExprs = targetSchema.zip(updateExprs) val exprsForProject = schemaWithExprs.flatMap { case (field, Some(expr)) => // Create a named expression so that we can use it in Project val exprForProject = Alias(expr, field.name)() Some(exprForProject.exprId -> exprForProject) case (_, None) => None }.toMap // Create a fake Project to resolve the generation expressions val fakePlan = Project(exprsForProject.values.toArray[NamedExpression], updateTarget) schemaWithExprs.map { case (_, Some(expr)) => expr case (targetCol, None) => // `targetCol` is a generated column and the user doesn't provide a update expression. val resolvedExpr = generatedColumns.find(f => conf.resolver(f.name, targetCol.name)) match { case Some(field) => val expr = GeneratedColumn.getGenerationExpression(field).get resolveReferencesForExpressions(SparkSession.active, expr :: Nil, fakePlan).head case None => // Should not happen throw DeltaErrors.nonGeneratedColumnMissingUpdateExpression(targetCol.name) } // As `resolvedExpr` will refer to attributes in `fakePlan`, we need to manually replace // these attributes with their update expressions. resolvedExpr.transform { case a: AttributeReference if exprsForProject.contains(a.exprId) => exprsForProject(a.exprId).child } } } /** * Replaces 'CastSupport.cast'. Selects a cast based on 'spark.sql.storeAssignmentPolicy'. * Legacy behavior for UPDATE and MERGE followed 'spark.sql.ansi.enabled' instead, this legacy * behavior can be re-enabled by setting * 'spark.databricks.delta.updateAndMergeCastingFollowsAnsiEnabledFlag' to true. */ private def cast( child: Expression, dataType: DataType, castingBehavior: CastingBehavior, columnName: String): Expression = { if (castingBehavior.isInstanceOf[MergeOrUpdateCastingBehavior] && conf.getConf(DeltaSQLConf.UPDATE_AND_MERGE_CASTING_FOLLOWS_ANSI_ENABLED_FLAG)) { return Cast(child, dataType, Option(conf.sessionLocalTimeZone)) } conf.storeAssignmentPolicy match { case SQLConf.StoreAssignmentPolicy.LEGACY => Cast(child, dataType, Some(conf.sessionLocalTimeZone), ansiEnabled = false) case SQLConf.StoreAssignmentPolicy.ANSI => val cast = Cast(child, dataType, Some(conf.sessionLocalTimeZone), ansiEnabled = true) if (canCauseCastOverflow(cast)) { castingBehavior match { case _: MergeOrUpdateCastingBehavior => CheckOverflowInTableWrite(cast, columnName) case _ => cast.setTagValue(Cast.BY_TABLE_INSERTION, ()) CheckOverflowInTableInsert(cast, columnName) } } else { cast } case SQLConf.StoreAssignmentPolicy.STRICT => UpCast(child, dataType) } } private def containsNestedStruct(dt: DataType): Boolean = dt match { case _: StructType => true case _: AtomicType => false case a: ArrayType => containsNestedStruct(a.elementType) case m: MapType => containsNestedStruct(m.keyType) || containsNestedStruct(m.valueType) // Let's defensively pretend it might have a nested struct if we don't recognise something. case _ => true } private def containsIntegralOrDecimalType(dt: DataType): Boolean = dt match { case _: IntegralType | _: DecimalType => true case a: ArrayType => containsIntegralOrDecimalType(a.elementType) case m: MapType => containsIntegralOrDecimalType(m.keyType) || containsIntegralOrDecimalType(m.valueType) case s: StructType => s.fields.exists(sf => containsIntegralOrDecimalType(sf.dataType)) case _ => false } private def canCauseCastOverflow(cast: Cast): Boolean = { containsIntegralOrDecimalType(cast.dataType) && !Cast.canUpCast(cast.child.dataType, cast.dataType) } } case class CheckOverflowInTableWrite(child: Expression, columnName: String) extends UnaryExpression { override protected def withNewChildInternal(newChild: Expression): Expression = { copy(child = newChild) } private def getCast: Option[Cast] = child match { case c: Cast => Some(c) case ExpressionProxy(c: Cast, _, _) => Some(c) case _ => None } override def eval(input: InternalRow): Any = try { child.eval(input) } catch { case e: ArithmeticException => getCast match { case Some(cast) => throw DeltaErrors.castingCauseOverflowErrorInTableWrite( cast.child.dataType, cast.dataType, columnName) case None => throw e } } override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { getCast match { case Some(child) => doGenCodeWithBetterErrorMsg(ctx, ev, child) case None => child.genCode(ctx) } } def doGenCodeWithBetterErrorMsg(ctx: CodegenContext, ev: ExprCode, child: Cast): ExprCode = { val childGen = child.genCode(ctx) val exceptionClass = classOf[ArithmeticException].getCanonicalName assert(child.isInstanceOf[Cast]) val cast = child.asInstanceOf[Cast] val fromDt = ctx.addReferenceObj("from", cast.child.dataType, cast.child.dataType.getClass.getName) val toDt = ctx.addReferenceObj("to", child.dataType, child.dataType.getClass.getName) val col = ctx.addReferenceObj("colName", columnName, "java.lang.String") // scalastyle:off line.size.limit ev.copy(code = code""" boolean ${ev.isNull} = true; ${CodeGenerator.javaType(dataType)} ${ev.value} = ${CodeGenerator.defaultValue(dataType)}; try { ${childGen.code} ${ev.isNull} = ${childGen.isNull}; ${ev.value} = ${childGen.value}; } catch ($exceptionClass e) { throw org.apache.spark.sql.delta.DeltaErrors .castingCauseOverflowErrorInTableWrite($fromDt, $toDt, $col); }""" ) // scalastyle:on line.size.limit } override def dataType: DataType = child.dataType override def sql: String = child.sql override def toString: String = child.toString } object UpdateExpressionsSupport { /** * Returns true if preserving null source structs for whole-struct assignments is enabled. * This fix addresses the issue where a null source struct is incorrectly expanded into * a non-null struct with all fields set to null in whole-struct assignments, e.g. * UPDATE SET col = s.col. */ def isWholeStructAssignmentPreserveNullSourceStructsEnabled(conf: SQLConf): Boolean = { conf.getConf(DeltaSQLConf.DELTA_MERGE_PRESERVE_NULL_SOURCE_STRUCTS) } /** * Returns true if preserving null source structs for UPDATE * is enabled. * This fix addresses the issue where a null source struct is incorrectly expanded into * a non-null struct with all fields set to null in UPDATE * operations. */ def isUpdateStarPreserveNullSourceStructsEnabled(conf: SQLConf): Boolean = { isWholeStructAssignmentPreserveNullSourceStructsEnabled(conf) && conf.getConf(DeltaSQLConf.DELTA_MERGE_PRESERVE_NULL_SOURCE_STRUCTS_UPDATE_STAR) } /** * Returns true if `targetType` contains extra struct fields compared to `sourceType` that we * want to preserve values for. * * We recursively check target-only fields of structs nested within structs, but we do not * check structs nested within arrays or maps because we don't preserve original target values * for arrays or maps. * * Field name comparison is case-insensitive, aligning with * `DataTypeUtils.equalsIgnoreCaseAndNullability`, which is used in * `UpdateExpressionSupport.castIfNeeded` to decide whether struct type cast is needed. * * @param sourceStruct the source struct to compare against * @param targetStruct the target struct to check for extra fields to preserve values for * @return true if `targetStruct` has more struct fields than `sourceStruct` at any nesting level * that we want to preserve values for. */ private def hasExtraStructFieldsToPreserveValue( sourceStruct: StructType, targetStruct: StructType): Boolean = { // Fast check: if target has more fields, it definitely has extra fields than source. if (targetStruct.length > sourceStruct.length) { return true } // Partition target fields into target-only and common fields. val (commonFields, targetOnlyFields) = targetStruct.partition { targetField => sourceStruct.exists(_.name.equalsIgnoreCase(targetField.name)) } // If there are any target-only fields, we have extra fields to preserve. if (targetOnlyFields.nonEmpty) { return true } // No extra fields at this level - recursively check common fields that are `StructType`. commonFields.exists { targetField => sourceStruct.find(_.name.equalsIgnoreCase(targetField.name)).exists { sourceField => (sourceField.dataType, targetField.dataType) match { case (sourceStruct: StructType, targetStruct: StructType) => hasExtraStructFieldsToPreserveValue(sourceStruct, targetStruct) case _ => // Don't recurse into arrays or maps, as we don't preserve values for arrays or maps. false } } } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptor.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.actions import java.io.{ByteArrayInputStream, ByteArrayOutputStream, DataInputStream, DataOutputStream} import java.net.URI import java.util.{Base64, UUID} import org.apache.spark.sql.delta.DeltaErrors import org.apache.spark.sql.delta.DeltaUDF import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.{Codec, DeltaEncoder, JsonUtils} import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.databind.annotation.JsonDeserialize import org.apache.hadoop.fs.Path import org.apache.spark.paths.SparkPath import org.apache.spark.sql.{Column, Encoder} import org.apache.spark.sql.functions.{concat, lit, when} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ /** Information about a deletion vector attached to a file action. */ case class DeletionVectorDescriptor( /** * Indicates how the DV is stored. * Should be a single letter (see [[pathOrInlineDv]] below.) */ storageType: String, /** * Contains the actual data that allows accessing the DV. * * Three options are currently supported: * - `storageType="u"` format: `` * The deletion vector is stored in a file with a path relative to * the data directory of this Delta Table, and the file name can be * reconstructed from the UUID. * The encoded UUID is always exactly 20 characters, so the random * prefix length can be determined any characters exceeding 20. * - `storageType="i"` format: `` * The deletion vector is stored inline in the log. * - `storageType="p"` format: `` * The DV is stored in a file with an absolute path given by this * url. Special characters in this path must be escaped. */ pathOrInlineDv: String, /** * Start of the data for this DV in number of bytes from the beginning of the file it is stored * in. * * Always None when storageType = "i". */ @JsonDeserialize(contentAs = classOf[java.lang.Integer]) offset: Option[Int] = None, /** Size of the serialized DV in bytes (raw data size, i.e. before base85 encoding). */ sizeInBytes: Int, /** Number of rows the DV logically removes from the file. */ cardinality: Long, /** * Transient property that is used to validate DV correctness. * It is not stored in the log. */ @JsonDeserialize(contentAs = classOf[java.lang.Long]) maxRowIndex: Option[Long] = None) { import DeletionVectorDescriptor._ @JsonIgnore @transient lazy val uniqueId: String = { offset match { case Some(offset) => s"$uniqueFileId@$offset" case None => uniqueFileId } } @JsonIgnore @transient lazy val uniqueFileId: String = s"$storageType$pathOrInlineDv" @JsonIgnore protected[delta] def isOnDisk: Boolean = !isInline @JsonIgnore protected[delta] def isInline: Boolean = storageType == INLINE_DV_MARKER @JsonIgnore protected[delta] def isRelative: Boolean = storageType == UUID_DV_MARKER @JsonIgnore protected[delta] def isAbsolute: Boolean = storageType == PATH_DV_MARKER @JsonIgnore protected[delta] def isEmpty: Boolean = cardinality == 0 def absolutePath(tableLocation: Path): Path = { require(isOnDisk, "Can't get a path for an inline deletion vector") storageType match { case UUID_DV_MARKER => val (randomPrefix, uuid) = getRandomPrefixAndUuid.get assembleDeletionVectorPath(tableLocation, uuid, randomPrefix) case PATH_DV_MARKER => // Since there is no need for legacy support for relative paths for DVs, // relative DVs should *always* use the UUID variant. val parsedUri = new URI(pathOrInlineDv) assert(parsedUri.isAbsolute, "Relative URIs are not supported for DVs") new Path(parsedUri) case _ => throw DeltaErrors.cannotReconstructPathFromURI(pathOrInlineDv) } } /** Returns the url encoded absolute path of the deletion vector. */ def urlEncodedPath(tablePath: Path): String = SparkPath.fromPath(absolutePath(tablePath)).urlEncoded /** * Returns the url encoded relative path of the deletion vector if possible. * If the DV path is outside the table directory, returns None. */ def urlEncodedRelativePathIfExists(tablePath: Path): Option[String] = { if (isRelative) { return Some(SparkPath.fromPath(absolutePath(new Path("."))).urlEncoded) } // DV path is not relative. Attempt to relativize it. val basePathUri = tablePath.toUri val absolutePathUri = absolutePath(tablePath).toUri val relativePath = basePathUri.relativize(absolutePathUri) if (!relativePath.isAbsolute) { Some(SparkPath.fromUri(relativePath).urlEncoded) } else { None } } /** * Parse the prefix and UUID of a relative DV. Returns None if the DV is not relative. */ @JsonIgnore def getRandomPrefixAndUuid: Option[(String, UUID)] = storageType match { case UUID_DV_MARKER => // If the file was written with a random prefix, we have to extract that, // before decoding the UUID. val randomPrefixLength = pathOrInlineDv.length - Codec.Base85Codec.ENCODED_UUID_LENGTH val (randomPrefix, encodedUuid) = pathOrInlineDv.splitAt(randomPrefixLength) Some((randomPrefix, Codec.Base85Codec.decodeUUID(encodedUuid))) case _ => None } /** * Produce a copy of this DV, but using an absolute path. * * If the DV already has an absolute path or is inline, then this is just a normal copy. */ def copyWithAbsolutePath(tableLocation: Path): DeletionVectorDescriptor = { storageType match { case UUID_DV_MARKER => this.copy( storageType = PATH_DV_MARKER, pathOrInlineDv = urlEncodedPath(tableLocation)) case PATH_DV_MARKER | INLINE_DV_MARKER => this.copy() } } /** * Produce a copy of this DV, with `pathOrInlineDv` replaced by a relative path based on `id` * and `randomPrefix`. * * If the DV already has a relative path or is inline, then this is just a normal copy. */ def copyWithNewRelativePath(id: UUID, randomPrefix: String): DeletionVectorDescriptor = { storageType match { case PATH_DV_MARKER => this.copy(storageType = UUID_DV_MARKER, pathOrInlineDv = encodeUUID(id, randomPrefix)) case UUID_DV_MARKER | INLINE_DV_MARKER => this.copy() } } @JsonIgnore def inlineData: Array[Byte] = { require(isInline, "Can't get data for an on-disk DV from the log.") // The sizeInBytes is used to remove any padding that might have been added during encoding. Codec.Base85Codec.decodeBytes(pathOrInlineDv, sizeInBytes) } /** Returns the estimated number of bytes required to serialize this object. */ @JsonIgnore protected[delta] lazy val estimatedSerializedSize: Int = { // (cardinality(8) + sizeInBytes(4)) + storageType + pathOrInlineDv + option[offset(4)] 12 + storageType.length + pathOrInlineDv.length + (if (offset.isDefined) 4 else 0) } /* * Serialize the DV descriptor to a base64 encoded string. */ def serializeToBase64(): String = { val bs = new ByteArrayOutputStream() val ds = new DataOutputStream(bs) try { ds.writeLong(cardinality) ds.writeInt(sizeInBytes) val storageTypeBytes = storageType.getBytes() assert(storageTypeBytes.length == 1, s"Storage type must be 1byte value: $storageType") ds.writeByte(storageTypeBytes.head) if (storageType != INLINE_DV_MARKER) { assert(offset.isDefined) ds.writeInt(offset.get) } else { assert(offset.isEmpty) } ds.writeUTF(pathOrInlineDv) Base64.getEncoder.encodeToString(bs.toByteArray) } finally { ds.close() } } } object DeletionVectorDescriptor { /** Prefix that is used in all file names generated by deletion vector store. */ val DELETION_VECTOR_FILE_NAME_PREFIX = SQLConf.get.getConf(DeltaSQLConf.TEST_DV_NAME_PREFIX) /** String that is used in all file names generated by deletion vector store */ val DELETION_VECTOR_FILE_NAME_CORE = DELETION_VECTOR_FILE_NAME_PREFIX + "deletion_vector" // Markers to separate different kinds of DV storage. final val PATH_DV_MARKER: String = "p" final val INLINE_DV_MARKER: String = "i" final val UUID_DV_MARKER: String = "u" private final val deletionVectorFileNameRegex = raw"${new Path(DELETION_VECTOR_FILE_NAME_CORE).toUri}_([^.]+)\.bin".r private final val deletionVectorFileNamePattern = deletionVectorFileNameRegex.pattern final lazy val STRUCT_TYPE: StructType = Action.addFileSchema("deletionVector").dataType.asInstanceOf[StructType] private lazy val _encoder = new DeltaEncoder[DeletionVectorDescriptor] implicit def encoder: Encoder[DeletionVectorDescriptor] = _encoder.get /** Utility method to create an on-disk [[DeletionVectorDescriptor]] */ def onDiskWithRelativePath( id: UUID, randomPrefix: String = "", sizeInBytes: Int, cardinality: Long, offset: Option[Int] = None, maxRowIndex: Option[Long] = None): DeletionVectorDescriptor = DeletionVectorDescriptor( storageType = UUID_DV_MARKER, pathOrInlineDv = encodeUUID(id, randomPrefix), offset = offset, sizeInBytes = sizeInBytes, cardinality = cardinality, maxRowIndex = maxRowIndex) /** Utility method to create an on-disk [[DeletionVectorDescriptor]] */ def onDiskWithAbsolutePath( path: String, sizeInBytes: Int, cardinality: Long, offset: Option[Int] = None, maxRowIndex: Option[Long] = None): DeletionVectorDescriptor = DeletionVectorDescriptor( storageType = PATH_DV_MARKER, pathOrInlineDv = path, offset = offset, sizeInBytes = sizeInBytes, cardinality = cardinality, maxRowIndex = maxRowIndex) /** Utility method to create an inline [[DeletionVectorDescriptor]] */ def inlineInLog( data: Array[Byte], cardinality: Long): DeletionVectorDescriptor = DeletionVectorDescriptor( storageType = INLINE_DV_MARKER, pathOrInlineDv = encodeData(data), sizeInBytes = data.length, cardinality = cardinality) /** * Returns whether the path points to a deletion vector file. * Note, external writers are no enforced to create DV files with the same naming convertions. * This function is intended for testing. */ private[delta] def isDeletionVectorPath(path: Path): Boolean = deletionVectorFileNamePattern.matcher(path.getName).matches() /** Only for testing. */ private[delta] def isDeletionVectorPath(path: String): Boolean = isDeletionVectorPath(new Path(path)) /** Same as above but as a column expression. Only for testing. */ private[delta] def isDeletionVectorPath(pathCol: Column): Column = DeltaUDF.booleanFromString(isDeletionVectorPath)(pathCol) /** Returns a boolean column that corresponds to whether each deletion vector is inline. */ def isInline(dv: Column): Column = DeltaUDF.booleanFromDeletionVectorDescriptor(_.isInline)(dv) /** * Returns a column with the url encoded deletion vector paths. * * WARNING: It throws an exception if it encounters any inline DVs. The caller is responsible * for handling these separately. */ def urlEncodedPath(deletionVectorCol: Column, tablePath: Path): Column = DeltaUDF.stringFromDeletionVectorDescriptor(_.urlEncodedPath(tablePath))(deletionVectorCol) /** * Returns a column with the url encoded deletion vector relative paths. For paths that cannot * be relativized, it returns None. * * WARNING: It throws an exception if it encounters any inline DVs. The caller is responsible * for handling these separately. */ def urlEncodedRelativePathIfExists(deletionVectorCol: Column, tablePath: Path): Column = DeltaUDF.stringOptionFromDeletionVectorDescriptor( _.urlEncodedRelativePathIfExists(tablePath))(deletionVectorCol) /** * This produces the same output as [[DeletionVectorDescriptor.uniqueId]] but as a column * expression, so it can be used directly in a Spark query. */ def uniqueIdExpression(deletionVectorCol: Column): Column = { when(deletionVectorCol("offset").isNotNull, concat( deletionVectorCol("storageType"), deletionVectorCol("pathOrInlineDv"), lit('@'), deletionVectorCol("offset"))) .otherwise(concat( deletionVectorCol("storageType"), deletionVectorCol("pathOrInlineDv"))) } /** * Return the unique path under `parentPath` that is based on `id`. * * Optionally, prepend a `prefix` to the name. */ def assembleDeletionVectorPath(targetParentPath: Path, id: UUID, prefix: String = ""): Path = { val fileName = assembleDeletionVectorFileName(id) if (prefix.nonEmpty) { new Path(new Path(targetParentPath, prefix), fileName) } else { new Path(targetParentPath, fileName) } } /** * Return the unique file name for a deletion vector based on `id`. */ def assembleDeletionVectorFileName(id: UUID): String = s"${DELETION_VECTOR_FILE_NAME_CORE}_${id}.bin" /** Descriptor for an empty stored bitmap. */ val EMPTY: DeletionVectorDescriptor = DeletionVectorDescriptor( storageType = INLINE_DV_MARKER, pathOrInlineDv = "", sizeInBytes = 0, cardinality = 0) private[delta] def encodeUUID(id: UUID, randomPrefix: String): String = { val uuidData = Codec.Base85Codec.encodeUUID(id) // This should always be true and we are relying on it for separating out the // prefix again later without having to spend an extra character as a separator. assert(uuidData.length == 20) s"$randomPrefix$uuidData" } def encodeData(bytes: Array[Byte]): String = Codec.Base85Codec.encodeBytes(bytes) /* * Deserialize the base64 encoded string to a DV descriptor. * * The format must be in sync with [[DeletionVectorDescriptor.serializeToBase64]]. */ def deserializeFromBase64(encoded: String): DeletionVectorDescriptor = { val buffer = Base64.getDecoder.decode(encoded) val ds = new DataInputStream(new ByteArrayInputStream(buffer)) try { val cardinality = ds.readLong() val sizeInBytes = ds.readInt() val storageType = ds.readByte().toChar.toString val offset = if (storageType != INLINE_DV_MARKER) Some(ds.readInt()) else None val pathOrInlineDv = ds.readUTF() DeletionVectorDescriptor(storageType, pathOrInlineDv, offset, sizeInBytes, cardinality) } finally { ds.close() } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/actions/InMemoryLogReplay.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.actions import java.net.URI /** * Replays a history of actions, resolving them to produce the current state * of the table. The protocol for resolution is as follows: * - The most recent [[AddFile]] and accompanying metadata for any `(path, dv id)` tuple wins. * - [[RemoveFile]] deletes a corresponding [[AddFile]] and is retained as a * tombstone until `minFileRetentionTimestamp` has passed. If `minFileRetentionTimestamp` is * None, all [[RemoveFile]] actions are retained. * A [[RemoveFile]] "corresponds" to the [[AddFile]] that matches both the parquet file URI * *and* the deletion vector's URI (if any). * - The most recent version for any `appId` in a [[SetTransaction]] wins. * - The most recent [[Metadata]] wins. * - The most recent [[Protocol]] version wins. * - For each `(path, dv id)` tuple, this class should always output only one [[FileAction]] * (either [[AddFile]] or [[RemoveFile]]) * * This class is not thread safe. */ class InMemoryLogReplay( minFileRetentionTimestamp: Option[Long], minSetTransactionRetentionTimestamp: Option[Long]) extends LogReplay { import InMemoryLogReplay._ private var currentProtocolVersion: Protocol = null private var currentVersion: Long = -1 private var currentMetaData: Metadata = null private val transactions = new scala.collection.mutable.HashMap[String, SetTransaction]() private val domainMetadatas = collection.mutable.Map.empty[String, DomainMetadata] private val activeFiles = new scala.collection.mutable.HashMap[UniqueFileActionTuple, AddFile]() // RemoveFiles that had cancelled AddFile during replay private val cancelledRemoveFiles = new scala.collection.mutable.HashMap[UniqueFileActionTuple, RemoveFile]() // RemoveFiles that had NOT cancelled any AddFile during replay private val activeRemoveFiles = new scala.collection.mutable.HashMap[UniqueFileActionTuple, RemoveFile]() override def append(version: Long, actions: Iterator[Action]): Unit = { assert(currentVersion == -1 || version == currentVersion + 1, s"Attempted to replay version $version, but state is at $currentVersion") currentVersion = version actions.foreach { case a: SetTransaction => transactions(a.appId) = a case a: DomainMetadata if a.removed => domainMetadatas.remove(a.domain) case a: DomainMetadata if !a.removed => domainMetadatas(a.domain) = a case _: CheckpointOnlyAction => // Ignore this while doing LogReplay case a: Metadata => currentMetaData = a case a: Protocol => currentProtocolVersion = a case add: AddFile => val uniquePath = UniqueFileActionTuple(add.pathAsUri, add.getDeletionVectorUniqueId) activeFiles(uniquePath) = add.copy(dataChange = false) // Remove the tombstone to make sure we only output one `FileAction`. cancelledRemoveFiles.remove(uniquePath) // Remove from activeRemoveFiles to handle commits that add a previously-removed file activeRemoveFiles.remove(uniquePath) case remove: RemoveFile => val uniquePath = UniqueFileActionTuple(remove.pathAsUri, remove.getDeletionVectorUniqueId) activeFiles.remove(uniquePath) match { case Some(_) => cancelledRemoveFiles(uniquePath) = remove case None => activeRemoveFiles(uniquePath) = remove } case _: CommitInfo => // do nothing case _: AddCDCFile => // do nothing case null => // Some crazy future feature. Ignore } } private def getTombstones: Iterable[FileAction] = { val allRemovedFiles = cancelledRemoveFiles.values ++ activeRemoveFiles.values val filteredRemovedFiles = minFileRetentionTimestamp match { case None => allRemovedFiles case Some(timestamp) => allRemovedFiles.filter(_.delTimestamp > timestamp) } filteredRemovedFiles.map(_.copy(dataChange = false)) } private[delta] def getTransactions: Iterable[SetTransaction] = { minSetTransactionRetentionTimestamp match { case None => transactions.values case Some(timestamp) => transactions.values.filter { txn => txn.lastUpdated.exists(_ > timestamp) } } } private[delta] def getDomainMetadatas: Iterable[DomainMetadata] = domainMetadatas.values /** Returns the current state of the Table as an iterator of actions. */ override def checkpoint: Iterator[Action] = { val fileActions = (activeFiles.values ++ getTombstones).toSeq.sortBy(_.path) Option(currentProtocolVersion).toIterator ++ Option(currentMetaData).toIterator ++ getDomainMetadatas ++ getTransactions ++ fileActions.toIterator } /** Returns all [[AddFile]] actions after the Log Replay */ private[delta] def allFiles: Seq[AddFile] = activeFiles.values.toSeq } object InMemoryLogReplay{ /** The unit of path uniqueness in delta log actions is the tuple `(parquet file, dv)`. */ final case class UniqueFileActionTuple(fileURI: URI, deletionVectorURI: Option[String]) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/actions/LogReplay.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.actions /** * Replays a history of actions, resolving them to produce the current state * of the table. */ trait LogReplay { /** Append these `actions` to the state. Must only be called in ascending order of `version`. */ def append(version: Long, actions: Iterator[Action]): Unit /** Returns the current state of the Table as an iterator of actions. */ def checkpoint: Iterator[Action] } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/actions/TableFeatureSupport.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.actions import java.util.Locale import scala.collection.mutable import scala.util.control.NonFatal import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.DeltaOperations.Operation import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import com.fasterxml.jackson.annotation.JsonIgnore /** * Trait to be mixed into the [[Protocol]] case class to enable Table Features. * * Protocol reader version 3 and writer version 7 start to support reader and writer table * features. Reader version 3 supports only reader-writer features in an explicit way, * by adding its name to `readerFeatures`. Similarly, writer version 7 supports only writer-only * or reader-writer features in an explicit way, by adding its name to `writerFeatures`. * When reading or writing a table, clients MUST respect all supported features. * * See also the document of [[TableFeature]] for feature-specific terminologies. */ trait TableFeatureSupport { this: Protocol => /** Check if this protocol is capable of adding features into its `readerFeatures` field. */ def supportsReaderFeatures: Boolean = TableFeatureProtocolUtils.supportsReaderFeatures(minReaderVersion) /** Check if this protocol is capable of adding features into its `writerFeatures` field. */ def supportsWriterFeatures: Boolean = TableFeatureProtocolUtils.supportsWriterFeatures(minWriterVersion) /** * Get a new Protocol object that has `feature` supported. Writer-only features will be added to * `writerFeatures` field, and reader-writer features will be added to `readerFeatures` and * `writerFeatures` fields. * * If `feature` is already implicitly supported in the current protocol's legacy reader or * writer protocol version, the new protocol will not modify the original protocol version, * i.e., the feature will not be explicitly added to the protocol's `readerFeatures` or * `writerFeatures`. This is to avoid unnecessary protocol upgrade for feature that it already * supports. */ def withFeature(feature: TableFeature): Protocol = { def shouldAddRead: Boolean = { if (supportsReaderFeatures) return true if (feature.minReaderVersion <= minReaderVersion) return false throw DeltaErrors.tableFeatureRequiresHigherReaderProtocolVersion( feature.name, minReaderVersion, feature.minReaderVersion) } def shouldAddWrite: Boolean = { if (supportsWriterFeatures) return true if (feature.minWriterVersion <= minWriterVersion) return false throw DeltaErrors.tableFeatureRequiresHigherWriterProtocolVersion( feature.name, minWriterVersion, feature.minWriterVersion) } var shouldAddToReaderFeatures = feature.isReaderWriterFeature var shouldAddToWriterFeatures = true if (feature.isLegacyFeature) { if (feature.isReaderWriterFeature) { shouldAddToReaderFeatures = shouldAddRead } shouldAddToWriterFeatures = shouldAddWrite } val protocolWithDependencies = withFeatures(feature.requiredFeatures) protocolWithDependencies.withFeature( feature.name, addToReaderFeatures = shouldAddToReaderFeatures, addToWriterFeatures = shouldAddToWriterFeatures) } /** * Get a new Protocol object with multiple features supported. * * See the documentation of [[withFeature]] for more information. */ def withFeatures(features: Iterable[TableFeature]): Protocol = { features.foldLeft(this)(_.withFeature(_)) } /** * Get a new Protocol object with an additional feature descriptor. If `addToReaderFeatures` is * set to `true`, the descriptor will be added to the protocol's `readerFeatures` field. If * `addToWriterFeatures` is set to `true`, the descriptor will be added to the protocol's * `writerFeatures` field. * * The method does not require the feature to be recognized by the client, therefore will not * try keeping the protocol's `readerFeatures` and `writerFeatures` in sync. Use with caution. */ private[actions] def withFeature( name: String, addToReaderFeatures: Boolean, addToWriterFeatures: Boolean): Protocol = { if (addToReaderFeatures && !supportsReaderFeatures) { throw DeltaErrors.tableFeatureRequiresHigherReaderProtocolVersion( name, currentVersion = minReaderVersion, requiredVersion = TableFeatureProtocolUtils.TABLE_FEATURES_MIN_READER_VERSION) } if (addToWriterFeatures && !supportsWriterFeatures) { throw DeltaErrors.tableFeatureRequiresHigherWriterProtocolVersion( name, currentVersion = minWriterVersion, requiredVersion = TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION) } val addedReaderFeatureOpt = if (addToReaderFeatures) Some(name) else None val addedWriterFeatureOpt = if (addToWriterFeatures) Some(name) else None copy( readerFeatures = this.readerFeatures.map(_ ++ addedReaderFeatureOpt), writerFeatures = this.writerFeatures.map(_ ++ addedWriterFeatureOpt)) } /** * Get a new Protocol object with additional feature descriptors added to the protocol's * `readerFeatures` field. * * The method does not require the features to be recognized by the client, therefore will not * try keeping the protocol's `readerFeatures` and `writerFeatures` in sync. Use with caution. */ private[delta] def withReaderFeatures(names: Iterable[String]): Protocol = { names.foldLeft(this)( _.withFeature(_, addToReaderFeatures = true, addToWriterFeatures = false)) } /** * Get a new Protocol object with additional feature descriptors added to the protocol's * `writerFeatures` field. * * The method does not require the features to be recognized by the client, therefore will not * try keeping the protocol's `readerFeatures` and `writerFeatures` in sync. Use with caution. */ private[delta] def withWriterFeatures(names: Iterable[String]): Protocol = { names.foldLeft(this)( _.withFeature(_, addToReaderFeatures = false, addToWriterFeatures = true)) } /** * Get all feature names in this protocol's `readerFeatures` field. Returns an empty set when * this protocol does not support reader features. */ def readerFeatureNames: Set[String] = this.readerFeatures.getOrElse(Set()) /** * Get a set of all feature names in this protocol's `writerFeatures` field. Returns an empty * set when this protocol does not support writer features. */ def writerFeatureNames: Set[String] = this.writerFeatures.getOrElse(Set()) /** * Get a set of all feature names in this protocol's `readerFeatures` and `writerFeatures` * field. Returns an empty set when this protocol supports none of reader and writer features. */ @JsonIgnore lazy val readerAndWriterFeatureNames: Set[String] = readerFeatureNames ++ writerFeatureNames /** * Same as above but returns a sequence of [[TableFeature]] instead of a set of feature names. */ @JsonIgnore lazy val readerAndWriterFeatures: Seq[TableFeature] = readerAndWriterFeatureNames.toSeq.flatMap(TableFeature.featureNameToFeature) /** * A sequence of native [[TableFeature]]s. This is derived by filtering out all explicitly * supported legacy features. */ @JsonIgnore lazy val nativeReaderAndWriterFeatures: Seq[TableFeature] = readerAndWriterFeatures.filterNot(_.isLegacyFeature) /** * Get all features that are implicitly supported by this protocol, for example, `Protocol(1,2)` * implicitly supports `appendOnly` and `invariants`. When this protocol is capable of requiring * writer features, no feature can be implicitly supported. */ @JsonIgnore lazy val implicitlySupportedFeatures: Set[TableFeature] = { if (supportsReaderFeatures && supportsWriterFeatures) { // this protocol uses both reader and writer features, no feature can be implicitly supported Set() } else { TableFeature.allSupportedFeaturesMap.values .filter(_.isLegacyFeature) .filterNot(supportsReaderFeatures || this.minReaderVersion < _.minReaderVersion) .filterNot(supportsWriterFeatures || this.minWriterVersion < _.minWriterVersion) .toSet } } /** * Get all features that are supported by this protocol, implicitly and explicitly. When the * protocol supports table features, this method returns the same set of features as * [[readerAndWriterFeatureNames]]; when the protocol does not support table features, this * method becomes equivalent to [[implicitlySupportedFeatures]]. */ @JsonIgnore lazy val implicitlyAndExplicitlySupportedFeatures: Set[TableFeature] = { readerAndWriterFeatureNames.flatMap(TableFeature.featureNameToFeature) ++ implicitlySupportedFeatures } /** * Determine whether this protocol can be safely upgraded to a new protocol `to`. This means: * - all features supported by this protocol are supported by `to`. * * Examples regarding feature status: * - from `[appendOnly]` to `[appendOnly]` => allowed. * - from `[appendOnly, changeDataFeed]` to `[appendOnly]` => not allowed. * - from `[appendOnly]` to `[appendOnly, changeDataFeed]` => allowed. */ def canUpgradeTo(to: Protocol): Boolean = // All features supported by `this` are supported by `to`. implicitlyAndExplicitlySupportedFeatures.subsetOf(to.implicitlyAndExplicitlySupportedFeatures) /** * Determine whether this protocol can be safely downgraded to a new protocol `to`. * All the implicit and explicit features between the two protocols need to match, * excluding the dropped feature. We also need to take into account that in some cases * the downgrade process may add the CheckpointProtectionTableFeature. * * Note, the conditions above also account for cases where we downgrade from table features * to legacy protocol versions. */ def canDowngradeTo(to: Protocol, droppedFeatureName: String): Boolean = { val thisFeatures = this.implicitlyAndExplicitlySupportedFeatures val toFeatures = to.implicitlyAndExplicitlySupportedFeatures val allowedNewFeatures: Set[TableFeature] = Set(CheckpointProtectionTableFeature) val droppedFeature = Seq(droppedFeatureName).flatMap(TableFeature.featureNameToFeature) val newFeatures = toFeatures -- thisFeatures newFeatures.subsetOf(allowedNewFeatures) && (thisFeatures -- droppedFeature == toFeatures -- newFeatures) } /** * True if this protocol can be upgraded or downgraded to the 'to' protocol. */ def canTransitionTo(to: Protocol, op: Operation): Boolean = { op match { case drop: DeltaOperations.DropTableFeature => canDowngradeTo(to, drop.featureName) case _ => canUpgradeTo(to) } } /** * Merge this protocol with multiple `protocols` to have the highest reader and writer versions * plus all explicitly and implicitly supported features. */ def merge(others: Protocol*): Protocol = { val protocols = this +: others val mergedReaderVersion = protocols.map(_.minReaderVersion).max val mergedWriterVersion = protocols.map(_.minWriterVersion).max val mergedReaderFeatures = protocols.flatMap(_.readerFeatureNames) val mergedWriterFeatures = protocols.flatMap(_.writerFeatureNames) val mergedImplicitFeatures = protocols.flatMap(_.implicitlySupportedFeatures) val mergedProtocol = Protocol(mergedReaderVersion, mergedWriterVersion) .withReaderFeatures(mergedReaderFeatures) .withWriterFeatures(mergedWriterFeatures) .withFeatures(mergedImplicitFeatures) // The merged protocol is always normalized in order to represent the protocol // with the weakest possible form. This enables backward compatibility. // This is preceded by a denormalization step. This allows to fix invalid legacy Protocols. // For example, (2, 3) is normalized to (1, 3). This is because there is no legacy feature // in the set with reader version 2 unless the writer version is at least 5. mergedProtocol.denormalizedNormalized } /** * Remove writer feature from protocol. To remove a writer feature we only need to * remove it from the writerFeatures set. */ private[delta] def removeWriterFeature(targetWriterFeature: TableFeature): Protocol = { require(targetWriterFeature.isRemovable) require(!targetWriterFeature.isReaderWriterFeature) copy(writerFeatures = writerFeatures.map(_ - targetWriterFeature.name)) } /** * Remove reader+writer feature from protocol. To remove a reader+writer feature we need to * remove it from the readerFeatures set and the writerFeatures set. */ private[delta] def removeReaderWriterFeature( targetReaderWriterFeature: TableFeature): Protocol = { require(targetReaderWriterFeature.isRemovable) require(targetReaderWriterFeature.isReaderWriterFeature) val newReaderFeatures = readerFeatures.map(_ - targetReaderWriterFeature.name) val newWriterFeatures = writerFeatures.map(_ - targetReaderWriterFeature.name) copy(readerFeatures = newReaderFeatures, writerFeatures = newWriterFeatures) } /** * Remove feature wrapper for removing either Reader/Writer or Writer features. We assume * the feature exists in the protocol. There is a relevant validation at * [[AlterTableDropFeatureDeltaCommand]]. We also require targetFeature is removable. * * After removing the feature we normalize the protocol. */ def removeFeature(targetFeature: TableFeature): Protocol = { require(targetFeature.isRemovable) val currentProtocol = this.denormalized val newProtocol = targetFeature match { case f@(_: ReaderWriterFeature | _: LegacyReaderWriterFeature) => currentProtocol.removeReaderWriterFeature(f) case f@(_: WriterFeature | _: LegacyWriterFeature) => currentProtocol.removeWriterFeature(f) case f => throw DeltaErrors.dropTableFeatureNonRemovableFeature(f.name) } newProtocol.normalized } /** * Protocol normalization is the process of converting a table features protocol to the weakest * possible form. This primarily refers to converting a table features protocol to a legacy * protocol. A Table Features protocol can be represented with the legacy representation only * when the features set of the former exactly matches a legacy protocol. * * Normalization can also decrease the reader version of a table features protocol when it is * higher than necessary. * * For example: * (1, 7, AppendOnly, Invariants, CheckConstraints) -> (1, 3) * (3, 7, RowTracking) -> (1, 7, RowTracking) */ def normalized: Protocol = { // Normalization can only be applied to table feature protocols. if (!supportsWriterFeatures) return this val (minReaderVersion, minWriterVersion) = TableFeatureProtocolUtils.minimumRequiredVersions(readerAndWriterFeatures) val newProtocol = Protocol(minReaderVersion, minWriterVersion) if (this.implicitlyAndExplicitlySupportedFeatures == newProtocol.implicitlyAndExplicitlySupportedFeatures) { newProtocol } else { Protocol(minReaderVersion, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(readerAndWriterFeatures) } } /** * Protocol denormalization is the process of converting a legacy protocol to the * the equivalent table features protocol. This is the inverse of protocol normalization. * It can be used to allow operations on legacy protocols that yield result which * cannot be represented anymore by a legacy protocol. */ def denormalized: Protocol = { // Denormalization can only be applied to legacy protocols. if (supportsWriterFeatures) return this val (minReaderVersion, _) = TableFeatureProtocolUtils.minimumRequiredVersions(implicitlySupportedFeatures.toSeq) Protocol(minReaderVersion, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(implicitlySupportedFeatures) } /** * Helper method that applies both denormalization and normalization. This can be used to * normalize invalid legacy protocols such as (2, 3), (1, 5). A legacy protocol is invalid * when the version numbers are higher than required to support the implied feature set. */ def denormalizedNormalized: Protocol = denormalized.normalized /** * Check if a `feature` is supported by this protocol. This means either (a) the protocol does * not support table features and implicitly supports the feature, or (b) the protocol supports * table features and references the feature. */ def isFeatureSupported(feature: TableFeature): Boolean = { // legacy feature + legacy protocol (feature.isLegacyFeature && this.implicitlySupportedFeatures.contains(feature)) || // new protocol readerAndWriterFeatureNames.contains(feature.name) } /** Returns whether this client supports writing in a table with this protocol. */ def supportedForWrite(): Boolean = { val supportedWriterVersions = Action.supportedWriterVersionNumbers val supportedWriterFeatures = Action.supportedProtocolVersion().writerFeatureNames val testUnsupportedFeatures: Set[String] = TableFeature.testUnsupportedFeatures .filterNot(_.isReaderWriterFeature) .map(_.name) supportedWriterVersions.contains(this.minWriterVersion) && this.writerFeatureNames.subsetOf(supportedWriterFeatures -- testUnsupportedFeatures) } /** Returns whether this client supports reading a table with this protocol. */ def supportedForRead(): Boolean = { val supportedReaderVersions = Action.supportedReaderVersionNumbers val supportedReaderFeatures = Action.supportedProtocolVersion().readerFeatureNames val testUnsupportedFeatures: Set[String] = TableFeature.testUnsupportedFeatures .filter(_.isReaderWriterFeature).map(_.name) supportedReaderVersions.contains(this.minReaderVersion) && this.readerFeatureNames.subsetOf(supportedReaderFeatures -- testUnsupportedFeatures) } } object TableFeatureProtocolUtils { /** Prop prefix in table properties. */ val FEATURE_PROP_PREFIX = "delta.feature." /** Prop prefix in Spark sessions configs. */ val DEFAULT_FEATURE_PROP_PREFIX = "spark.databricks.delta.properties.defaults.feature." /** * The string constant "enabled" for uses in table properties. * @deprecated * This value is deprecated to avoid confusion with features that are actually enabled by * table metadata. Use [[FEATURE_PROP_SUPPORTED]] instead. */ val FEATURE_PROP_ENABLED = "enabled" /** The string constant "supported" for uses in table properties. */ val FEATURE_PROP_SUPPORTED = "supported" /** Min reader version that supports reader features. */ val TABLE_FEATURES_MIN_READER_VERSION = 3 /** Min reader version that supports writer features. */ val TABLE_FEATURES_MIN_WRITER_VERSION = 7 /** Get the table property config key for the `feature`. */ def propertyKey(feature: TableFeature): String = propertyKey(feature.name) /** Get the table property config key for the `featureName`. */ def propertyKey(featureName: String): String = s"$FEATURE_PROP_PREFIX$featureName" /** Get the session default config key for the `feature`. */ def defaultPropertyKey(feature: TableFeature): String = defaultPropertyKey(feature.name) /** Get the session default config key for the `featureName`. */ def defaultPropertyKey(featureName: String): String = s"$DEFAULT_FEATURE_PROP_PREFIX$featureName" /** * Determine whether a [[Protocol]] with the given reader protocol version is capable of adding * features into its `readerFeatures` field. */ def supportsReaderFeatures(readerVersion: Int): Boolean = { readerVersion >= TABLE_FEATURES_MIN_READER_VERSION } /** * Determine whether a [[Protocol]] with the given writer protocol version is capable of adding * features into its `writerFeatures` field. */ def supportsWriterFeatures(writerVersion: Int): Boolean = { writerVersion >= TABLE_FEATURES_MIN_WRITER_VERSION } /** * Get a set of [[TableFeature]]s representing supported features set in a table properties map. */ def getSupportedFeaturesFromTableConfigs(configs: Map[String, String]): Set[TableFeature] = { val featureConfigs = configs.filterKeys(_.startsWith(FEATURE_PROP_PREFIX)) val unsupportedFeatureConfigs = mutable.Set.empty[String] val collectedFeatures = featureConfigs.flatMap { case (key, value) => // Feature name is lower cased in table properties but not in Spark session configs. // Feature status is not lower cased in any case. val name = key.stripPrefix(FEATURE_PROP_PREFIX).toLowerCase(Locale.ROOT) val status = value.toLowerCase(Locale.ROOT) if (status != FEATURE_PROP_SUPPORTED && status != FEATURE_PROP_ENABLED) { throw DeltaErrors.unsupportedTableFeatureStatusException(name, status) } val featureOpt = TableFeature.featureNameToFeature(name) if (!featureOpt.isDefined) { unsupportedFeatureConfigs += key } featureOpt }.toSet if (unsupportedFeatureConfigs.nonEmpty) { throw DeltaErrors.unsupportedTableFeatureConfigsException(unsupportedFeatureConfigs) } collectedFeatures } /** * Checks if the the given table property key is a Table Protocol property, i.e., * `delta.minReaderVersion`, `delta.minWriterVersion`, ``delta.ignoreProtocolDefaults``, or * anything that starts with `delta.feature.` */ def isTableProtocolProperty(key: String): Boolean = { key == Protocol.MIN_READER_VERSION_PROP || key == Protocol.MIN_WRITER_VERSION_PROP || key == DeltaConfigs.CREATE_TABLE_IGNORE_PROTOCOL_DEFAULTS.key || key.startsWith(TableFeatureProtocolUtils.FEATURE_PROP_PREFIX) } /** * Returns the minimum reader/writer versions required to support all provided features. */ def minimumRequiredVersions(features: Seq[TableFeature]): (Int, Int) = ((features.map(_.minReaderVersion) :+ 1).max, (features.map(_.minWriterVersion) :+ 1).max) } object DropTableFeatureUtils extends DeltaLogging { private val MAX_CHECKPOINT_RETRIES = 3 // The number of barrier checkpoints to create before the version requiring checkpoint protection. val NUMBER_OF_BARRIER_CHECKPOINTS = 3 /** * Helper function for creating checkpoints. If checkpoint creation fails we retry up * to [[MAX_CHECKPOINT_RETRIES]] times. * * @param snapshotRefreshStartTimeTs The timestamp to use as a starting point for refreshing * the snapshot. This value is used to improve the performance * of the snapshot refresh operation. */ def createCheckpointWithRetries( table: DeltaTableV2, snapshotRefreshStartTimeTs: Long): Boolean = { val log = table.deltaLog val snapshot = table.update(checkIfUpdatedSinceTs = Some(snapshotRefreshStartTimeTs)) def checkpointAndVerify(snapshot: Snapshot): Boolean = { try { table.checkpoint(snapshot) log.checkpointExistsAtVersion(snapshot.version) } catch { case NonFatal(e) => recordDeltaEvent( deltaLog = log, opType = "dropFeature.checkpointAndVerify.error", data = Map( "message" -> e.getMessage, "stackTrace" -> e.getStackTrace().mkString("\n"))) false } } (1 to MAX_CHECKPOINT_RETRIES).collectFirst { case _ if checkpointAndVerify(snapshot) => true }.getOrElse(false) } def createEmptyCommitAndCheckpoint( table: DeltaTableV2, snapshotRefreshStartTs: Long, retryOnFailure: Boolean = false): Boolean = { val log = table.deltaLog val snapshot = table.update(checkIfUpdatedSinceTs = Some(snapshotRefreshStartTs)) val emptyCommitTS = table.deltaLog.clock.nanoTime() log.startTransaction(table.catalogTable, Some(snapshot)) .commit(Nil, DeltaOperations.EmptyCommit) // retryOnFailure is temporary to avoid affecting the behavior of the legacy Drop Feature // command behavior. if (retryOnFailure) { createCheckpointWithRetries(table, emptyCommitTS) } else { table.checkpoint(table.update(checkIfUpdatedSinceTs = Some(emptyCommitTS))) true } } def truncateHistoryLogRetentionMillis(metadata: Metadata): Long = { val truncateHistoryLogRetention = DeltaConfigs .TABLE_FEATURE_DROP_TRUNCATE_HISTORY_LOG_RETENTION .fromMetaData(metadata) DeltaConfigs.getMilliSeconds(truncateHistoryLogRetention) } /** * Returns new metadata without `tablePropertiesToRemoveAtDowngradeCommit` table properties. */ def getDowngradedProtocolMetadata( feature: RemovableFeature, metadata: Metadata): Metadata = { val propKeys = feature.tablePropertiesToRemoveAtDowngradeCommit val normalizedKeys = DeltaConfigs.normalizeConfigKeys(propKeys) val newConfiguration = metadata.configuration.filterNot { case (key, _) => normalizedKeys.contains(key) } metadata.copy(configuration = newConfiguration) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.actions // scalastyle:off import.ordering.noEmptyLine import java.net.URI import java.sql.Timestamp import java.util.Locale import java.util.concurrent.TimeUnit import scala.annotation.tailrec import scala.collection.JavaConverters._ import scala.collection.mutable import scala.util.control.NonFatal import com.databricks.spark.util.TagDefinition import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.{DeltaFileOperations, JsonUtils, Utils => DeltaUtils} import org.apache.spark.sql.delta.util.FileNames import org.apache.spark.sql.delta.util.PartitionUtils import com.fasterxml.jackson.annotation._ import com.fasterxml.jackson.annotation.JsonInclude.Include import com.fasterxml.jackson.core.{JsonGenerator, JsonParser} import com.fasterxml.jackson.databind._ import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize} import com.fasterxml.jackson.databind.node.ObjectNode import io.delta.storage.commit.actions.{ AbstractCommitInfo => StorageAbstractCommitInfo, AbstractMetadata => StorageAbstractMetadata, AbstractProtocol => StorageAbstractProtocol } import org.apache.spark.sql.delta.v2.interop.{ AbstractCommitInfo => SparkAbstractCommitInfo, AbstractMetadata => SparkAbstractMetadata, AbstractProtocol => SparkAbstractProtocol } import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.internal.Logging import org.apache.spark.paths.SparkPath import org.apache.spark.sql.{Column, Encoder, SparkSession} import org.apache.spark.sql.catalyst.ScalaReflection import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.catalyst.expressions.Literal import org.apache.spark.sql.types.{DataType, StructField, StructType} import org.apache.spark.util.Utils object Action extends DeltaLogging { /** * The maximum version of the protocol that this version of Delta understands by default. * * Use [[supportedProtocolVersion()]] instead, except to define new feature-gated versions. */ private[actions] val readerVersion = TableFeatureProtocolUtils.TABLE_FEATURES_MIN_READER_VERSION private[actions] val writerVersion = TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION private[actions] val protocolVersion: Protocol = Protocol(readerVersion, writerVersion) // We can't extend the [[Action]] class itself since it affects serialization. // Instead, we add a wrapper here as a helper method for logging. override def recordDeltaEvent( deltaLog: DeltaLog, opType: String, tags: Map[TagDefinition, String] = Map.empty, data: AnyRef = null, path: Option[Path] = None): Unit = { super.recordDeltaEvent(deltaLog, opType, tags, data, path) } /** * The maximum protocol version we are currently allowed to use, with or without all recognized * features. Optionally, some features can be excluded using `featuresToExclude`. */ private[delta] def supportedProtocolVersion( withAllFeatures: Boolean = true, featuresToExclude: Seq[TableFeature] = Seq.empty): Protocol = { if (withAllFeatures) { val featuresToAdd = TableFeature.allSupportedFeaturesMap.values.toSet -- featuresToExclude protocolVersion.withFeatures(featuresToAdd) } else { protocolVersion } } /** All reader protocol version numbers supported by the system. */ private[delta] lazy val supportedReaderVersionNumbers: Set[Int] = { val allVersions = supportedProtocolVersion().implicitlyAndExplicitlySupportedFeatures.map(_.minReaderVersion) + 1 // Version 1 does not introduce new feature, it's always supported. if (DeltaUtils.isTesting) { allVersions + 0 // Allow Version 0 in tests } else { allVersions - 0 // Delete 0 produced by writer-only features } } /** All writer protocol version numbers supported by the system. */ private[delta] lazy val supportedWriterVersionNumbers: Set[Int] = { val allVersions = supportedProtocolVersion().implicitlyAndExplicitlySupportedFeatures.map(_.minWriterVersion) + 1 // Version 1 does not introduce new feature, it's always supported. if (DeltaUtils.isTesting) { allVersions + 0 // Allow Version 0 in tests } else { allVersions - 0 // Delete 0 produced by reader-only features - we don't have any - for safety } } def fromJson(json: String): Action = { JsonUtils.mapper.readValue[SingleAction](json).unwrap } lazy val logSchema = ExpressionEncoder[SingleAction]().schema lazy val addFileSchema = logSchema("add").dataType.asInstanceOf[StructType] /** * Same as [[logSchema]], but with a user-specified add.stats_parsed column. This is useful for * reading parquet checkpoint files that provide add.stats_parsed instead of add.stats. */ def logSchemaWithAddStatsParsed(statsParsed: StructField): StructType = { val logAddSchema = logSchema("add").dataType.asInstanceOf[StructType] val fields = logSchema.map { f => if (f.name == "add") f.copy(dataType = logAddSchema.add(statsParsed)) else f } StructType(fields) } } /** * Represents a single change to the state of a Delta table. An order sequence * of actions can be replayed using [[InMemoryLogReplay]] to derive the state * of the table at a given point in time. */ sealed trait Action { def wrap: SingleAction def json: String = JsonUtils.toJson(wrap) } /** * Used to block older clients from reading or writing the log when backwards incompatible changes * are made to the protocol. Readers and writers are responsible for checking that they meet the * minimum versions before performing any other operations. * * This action allows us to explicitly block older clients in the case of a breaking change to the * protocol. Absent a protocol change, Clients MUST silently ignore messages and fields that they * do not understand. * * Note: Please initialize this class using the companion object's `apply` method, which will * assign correct values (`Set()` vs `None`) to [[readerFeatures]] and [[writerFeatures]]. */ case class Protocol private ( minReaderVersion: Int, minWriterVersion: Int, @JsonInclude(Include.NON_ABSENT) // write to JSON only when the field is not `None` readerFeatures: Option[Set[String]], @JsonInclude(Include.NON_ABSENT) writerFeatures: Option[Set[String]]) extends Action with SparkAbstractProtocol with StorageAbstractProtocol with TableFeatureSupport { // Correctness check // Reader and writer versions must match the status of reader and writer features require( supportsReaderFeatures == readerFeatures.isDefined, "Mismatched minReaderVersion and readerFeatures.") require( supportsWriterFeatures == writerFeatures.isDefined, "Mismatched minWriterVersion and writerFeatures.") // When reader is on table features, writer must be on table features too if (supportsReaderFeatures && !supportsWriterFeatures) { throw DeltaErrors.tableFeatureReadRequiresWriteException( TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION) } override def wrap: SingleAction = SingleAction(protocol = this) /** * Return a reader-friendly string representation of this Protocol. * * Returns the protocol versions and referenced features when the protocol does support table * features, such as `3,7,{},{appendOnly}` and `2,7,None,{appendOnly}`. Otherwise returns only * the protocol version such as `2,6`. */ @JsonIgnore lazy val simpleString: String = { if (!supportsReaderFeatures && !supportsWriterFeatures) { s"$minReaderVersion,$minWriterVersion" } else { val readerFeaturesStr = readerFeatures .map(_.toSeq.sorted.mkString("[", ",", "]")) .getOrElse("None") val writerFeaturesStr = writerFeatures .map(_.toSeq.sorted.mkString("[", ",", "]")) .getOrElse("None") s"$minReaderVersion,$minWriterVersion,$readerFeaturesStr,$writerFeaturesStr" } } /** * Return a map that contains the protocol versions and supported features of this Protocol. */ @JsonIgnore private[delta] lazy val fieldsForLogging: Map[String, Any] = { Map( "minReaderVersion" -> minReaderVersion, // Number "minWriterVersion" -> minWriterVersion, // Number "supportedFeatures" -> implicitlyAndExplicitlySupportedFeatures.map(_.name).toSeq.sorted // Array[String] ) } override def toString: String = s"Protocol($simpleString)" override def getMinReaderVersion: Int = minReaderVersion override def getMinWriterVersion: Int = minWriterVersion override def getReaderFeatures: java.util.Set[String] = readerFeatures.map(_.asJava).orNull override def getWriterFeatures: java.util.Set[String] = writerFeatures.map(_.asJava).orNull } object Protocol { import TableFeatureProtocolUtils._ val MIN_READER_VERSION_PROP = "delta.minReaderVersion" val MIN_WRITER_VERSION_PROP = "delta.minWriterVersion" /** * Construct a [[Protocol]] case class of the given reader and writer versions. This method will * initialize table features fields when reader and writer versions are capable. */ def apply( minReaderVersion: Int = Action.readerVersion, minWriterVersion: Int = Action.writerVersion): Protocol = { new Protocol( minReaderVersion = minReaderVersion, minWriterVersion = minWriterVersion, readerFeatures = if (supportsReaderFeatures(minReaderVersion)) Some(Set()) else None, writerFeatures = if (supportsWriterFeatures(minWriterVersion)) Some(Set()) else None) } /** Returns the required protocol for a given feature. Takes into account dependent features. */ def forTableFeature(tf: TableFeature): Protocol = { // Every table feature is a writer feature. val writerFeatures = tf.requiredFeatures + tf val readerFeatures = writerFeatures.filter(f => f.isReaderWriterFeature && !f.isLegacyFeature) val writerFeaturesNames = writerFeatures.map(_.name) val readerFeaturesNames = readerFeatures.map(_.name) val minWriterVersion = TABLE_FEATURES_MIN_WRITER_VERSION val minReaderVersion = (readerFeatures.map(_.minReaderVersion) + 1).max new Protocol( minReaderVersion, minWriterVersion, readerFeatures = Option(readerFeaturesNames).filter(_.nonEmpty), writerFeatures = Some(writerFeaturesNames)) } /** * Picks the protocol version for a new table given the Delta table metadata. The result * satisfies all active features in the metadata and protocol-related configs in table * properties, i.e., configs with keys [[MIN_READER_VERSION_PROP]], [[MIN_WRITER_VERSION_PROP]], * and [[FEATURE_PROP_PREFIX]]. This method will also consider protocol-related configs: default * reader version, default writer version, and features enabled by * [[DEFAULT_FEATURE_PROP_PREFIX]]. */ def forNewTable(spark: SparkSession, metadataOpt: Option[Metadata]): Protocol = { // `minProtocolComponentsFromMetadata` does not consider sessions defaults, // so we must copy sessions defaults to table metadata. val conf = spark.sessionState.conf val ignoreProtocolDefaults = DeltaConfigs.ignoreProtocolDefaultsIsSet( sqlConfs = conf, tableConf = metadataOpt.map(_.configuration).getOrElse(Map.empty)) val defaultGlobalConf = if (ignoreProtocolDefaults) { Map(MIN_READER_VERSION_PROP -> 1.toString, MIN_WRITER_VERSION_PROP -> 1.toString) } else { Map( MIN_READER_VERSION_PROP -> conf.getConf(DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION).toString, MIN_WRITER_VERSION_PROP -> conf.getConf(DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION).toString) } val overrideGlobalConf = DeltaConfigs .mergeGlobalConfigs( sqlConfs = spark.sessionState.conf, tableConf = Map.empty, ignoreProtocolConfsOpt = Some(ignoreProtocolDefaults)) // We care only about protocol related stuff .filter { case (k, _) => TableFeatureProtocolUtils.isTableProtocolProperty(k) } var metadata = metadataOpt.getOrElse(Metadata()) // Priority: user-provided > override of session defaults > session defaults metadata = metadata.copy(configuration = defaultGlobalConf ++ overrideGlobalConf ++ metadata.configuration) val (readerVersion, writerVersion, enabledFeatures) = minProtocolComponentsFromMetadata(spark, metadata) // New table protocols should always be denormalized and then normalized to convert the // protocol to the weakest possible form. This means either converting a table features // protocol to a legacy protocol or reducing the versions of a table features protocol. // For example: // 1) (3, 7, RowTracking) is normalized to (1, 7, RowTracking). // 2) (3, 7, AppendOnly, Invariants) is normalized to (1, 2). // 3) (2, 3) is normalized to (1, 3). Protocol(readerVersion, writerVersion) .withFeatures(enabledFeatures) .denormalizedNormalized } /** * Returns the smallest set of table features that contains `features` and that also contains * all dependencies of all features in the returned set. */ @tailrec private def getDependencyClosure(features: Set[TableFeature]): Set[TableFeature] = { val requiredFeatures = features ++ features.flatMap(_.requiredFeatures) if (features == requiredFeatures) { features } else { getDependencyClosure(requiredFeatures) } } /** * Extracts all table features that are enabled by the given metadata and the optional protocol. * This includes all already enabled features (if a protocol is provided), the features enabled * directly by metadata, and all of their (transitive) dependencies. */ def extractAutomaticallyEnabledFeatures( spark: SparkSession, metadata: Metadata, protocol: Protocol): Set[TableFeature] = { val protocolEnabledFeatures = protocol .writerFeatureNames .flatMap(TableFeature.featureNameToFeature) val metadataEnabledFeatures = TableFeature .allSupportedFeaturesMap.values .collect { case f: TableFeature with FeatureAutomaticallyEnabledByMetadata if f.metadataRequiresFeatureToBeEnabled(protocol, metadata, spark) => f } .toSet getDependencyClosure(protocolEnabledFeatures ++ metadataEnabledFeatures) } /** * Given the Delta table metadata, returns the minimum required reader and writer version that * satisfies all enabled features in the metadata and protocol-related configs in table * properties, i.e., configs with keys [[MIN_READER_VERSION_PROP]], [[MIN_WRITER_VERSION_PROP]], * and [[FEATURE_PROP_PREFIX]]. * * This function returns the protocol versions and features individually instead of a * [[Protocol]], so the caller can identify the features that caused the protocol version. For * example, if the return values are (2, 5, columnMapping + preceding features), the caller * can safely ignore all other features required by the protocol with a reader and writer * version of 2 and 5. * * Note that this method does not consider features configured in session defaults. * To make them effective, copy them to `metadata` using [[DeltaConfigs.mergeGlobalConfigs]]. */ def minProtocolComponentsFromMetadata( spark: SparkSession, metadata: Metadata): (Int, Int, Set[TableFeature]) = { val tableConf = metadata.configuration // There might be features enabled by the table properties aka // `CREATE TABLE ... TBLPROPERTIES ...`. val tablePropEnabledFeatures = getSupportedFeaturesFromTableConfigs(tableConf) // To enable features that are being dependent by `tablePropEnabledFeatures`, we pass it here to // let [[getDependencyClosure]] collect them. val metaEnabledFeatures = extractAutomaticallyEnabledFeatures( spark, metadata, Protocol().withFeatures(tablePropEnabledFeatures)) val allEnabledFeatures = tablePropEnabledFeatures ++ metaEnabledFeatures // Protocol version provided in table properties can upgrade the protocol, but only when they // are higher than which required by the enabled features. val (readerVersionFromTableConfOpt, writerVersionFromTableConfOpt) = getProtocolVersionsFromTableConf(tableConf) // If the user explicitly sets the table versions, we need to take into account the // relevant implicit features. val implicitFeaturesFromTableConf = (readerVersionFromTableConfOpt, writerVersionFromTableConfOpt) match { case (Some(readerVersion), Some(writerVersion)) => // We cannot have a table features reader version if the protocol does not // support writer features. val sanitizedReaderVersion = if (supportsWriterFeatures(writerVersion)) { readerVersion } else { Math.min(2, readerVersion) } Protocol(sanitizedReaderVersion, writerVersion).implicitlySupportedFeatures case _ => Set.empty } // Construct the minimum required protocol for the enabled features. val minProtocol = Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(allEnabledFeatures ++ implicitFeaturesFromTableConf) .normalized // Return the minimum protocol components. (minProtocol.minReaderVersion, minProtocol.minWriterVersion, minProtocol.implicitlyAndExplicitlySupportedFeatures) } /** * Given the Delta table metadata, returns the minimum required reader and writer version * that satisfies all enabled table features in the metadata plus all enabled features as a set. * * This function returns the protocol versions and features individually instead of a * [[Protocol]], so the caller can identify the features that caused the protocol version. For * example, if the return values are (2, 5, columnMapping), the caller can safely ignore all * other features required by the protocol with a reader and writer version of 2 and 5. * * This method does not process protocol-related configs in table properties or session * defaults, i.e., configs with keys [[MIN_READER_VERSION_PROP]], [[MIN_WRITER_VERSION_PROP]], * and [[FEATURE_PROP_PREFIX]]. */ def minProtocolComponentsFromAutomaticallyEnabledFeatures( spark: SparkSession, metadata: Metadata, current: Protocol): (Int, Int, Set[TableFeature]) = { val enabledFeatures = extractAutomaticallyEnabledFeatures(spark, metadata, current) var (readerVersion, writerVersion) = (0, 0) enabledFeatures.foreach { feature => readerVersion = math.max(readerVersion, feature.minReaderVersion) writerVersion = math.max(writerVersion, feature.minWriterVersion) } (readerVersion, writerVersion, enabledFeatures) } /** Cast the table property for the protocol version to an integer. */ private def tryCastProtocolVersionToInt(key: String, value: String): Int = { try value.toInt catch { case _: NumberFormatException => throw DeltaErrors.protocolPropNotIntException(key, value) } } def getReaderVersionFromTableConf(conf: Map[String, String]): Option[Int] = { conf.get(MIN_READER_VERSION_PROP).map(tryCastProtocolVersionToInt(MIN_READER_VERSION_PROP, _)) } def getWriterVersionFromTableConf(conf: Map[String, String]): Option[Int] = { conf.get(MIN_WRITER_VERSION_PROP).map(tryCastProtocolVersionToInt(MIN_WRITER_VERSION_PROP, _)) } def getProtocolVersionsFromTableConf(conf: Map[String, String]): (Option[Int], Option[Int]) = { (getReaderVersionFromTableConf(conf), getWriterVersionFromTableConf(conf)) } def filterProtocolPropsFromTableProps(properties: Map[String, String]): Map[String, String] = properties.filterNot { case (k, _) => TableFeatureProtocolUtils.isTableProtocolProperty(k) } /** Assert a table metadata contains no protocol-related table properties. */ def assertMetadataContainsNoProtocolProps(metadata: Metadata): Unit = { assert( !metadata.configuration.contains(MIN_READER_VERSION_PROP), "Should not have the " + s"protocol version ($MIN_READER_VERSION_PROP) as part of table properties") assert( !metadata.configuration.contains(MIN_WRITER_VERSION_PROP), "Should not have the " + s"protocol version ($MIN_WRITER_VERSION_PROP) as part of table properties") assert( !metadata.configuration.keys.exists(_.startsWith(FEATURE_PROP_PREFIX)), "Should not have " + s"table features (starts with '$FEATURE_PROP_PREFIX') as part of table properties") assert( !metadata.configuration.contains(DeltaConfigs.CREATE_TABLE_IGNORE_PROTOCOL_DEFAULTS.key), "Should not have the table property " + s"${DeltaConfigs.CREATE_TABLE_IGNORE_PROTOCOL_DEFAULTS.key} stored in table metadata") } /** * Upgrade the current protocol to satisfy all auto-update capable features required by the table * metadata. An Delta error will be thrown if a non-auto-update capable feature is required by * the metadata and not in the resulting protocol, in such a case the user must run `ALTER TABLE` * to add support for this feature beforehand using the `delta.feature.featureName` table * property. * * Refer to [[FeatureAutomaticallyEnabledByMetadata.automaticallyUpdateProtocolOfExistingTables]] * to know more about "auto-update capable" features. * * Note: this method only considers metadata-enabled features. To avoid confusion, the caller * must apply and remove protocol-related table properties from the metadata before calling this * method. */ def upgradeProtocolFromMetadataForExistingTable( spark: SparkSession, metadata: Metadata, current: Protocol): Option[Protocol] = { val required = Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(extractAutomaticallyEnabledFeatures(spark, metadata, current)) .normalized if (!required.canUpgradeTo(current)) { // When the current protocol does not satisfy metadata requirement, some additional features // must be supported by the protocol. We assert those features can actually perform the // auto-update. assertMetadataTableFeaturesAutomaticallySupported( current.implicitlyAndExplicitlySupportedFeatures, required.implicitlyAndExplicitlySupportedFeatures) Some(required.merge(current)) } else { None } } /** * Ensure all features listed in `currentFeatures` are also listed in `requiredFeatures`, or, if * one is not listed, it must be capable to auto-update a protocol. * * Refer to [[FeatureAutomaticallyEnabledByMetadata.automaticallyUpdateProtocolOfExistingTables]] * to know more about "auto-update capable" features. * * Note: Caller must make sure `requiredFeatures` is obtained from a min protocol that satisfies * a table metadata. */ private def assertMetadataTableFeaturesAutomaticallySupported( currentFeatures: Set[TableFeature], requiredFeatures: Set[TableFeature]): Unit = { val (autoUpdateCapableFeatures, nonAutoUpdateCapableFeatures) = requiredFeatures.diff(currentFeatures) .collect { case f: FeatureAutomaticallyEnabledByMetadata => f } .partition(_.automaticallyUpdateProtocolOfExistingTables) if (nonAutoUpdateCapableFeatures.nonEmpty) { // The "current features" we give to the user are from the original protocol, plus // features newly supported by table properties in the current transaction, plus // metadata-enabled features that are auto-update capable. The first two are provided by // `currentFeatures`. throw DeltaErrors.tableFeaturesRequireManualEnablementException( nonAutoUpdateCapableFeatures, currentFeatures ++ autoUpdateCapableFeatures) } } /** * Verify that the table properties satisfy legality constraints. Throw an exception if not. */ def assertTablePropertyConstraintsSatisfied( spark: SparkSession, metadata: Metadata, snapshot: Snapshot): Unit = { import DeltaTablePropertyValidationFailedSubClass._ val tableName = if (metadata.name != null) metadata.name else metadata.id val configs = metadata.configuration.map { case (k, v) => k.toLowerCase(Locale.ROOT) -> v } val dvsEnabled = { val lowerCaseKey = DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key.toLowerCase(Locale.ROOT) configs.get(lowerCaseKey).exists(_.toBoolean) } if (dvsEnabled && metadata.format.provider != "parquet") { // DVs only work with parquet-based delta tables. throw new DeltaTablePropertyValidationFailedException( table = tableName, subClass = PersistentDeletionVectorsInNonParquetTable) } val manifestGenerationEnabled = { val lowerCaseKey = DeltaConfigs.SYMLINK_FORMAT_MANIFEST_ENABLED.key.toLowerCase(Locale.ROOT) configs.get(lowerCaseKey).exists(_.toBoolean) } if (dvsEnabled && manifestGenerationEnabled) { throw new DeltaTablePropertyValidationFailedException( table = tableName, subClass = PersistentDeletionVectorsWithIncrementalManifestGeneration) } if (manifestGenerationEnabled) { // Only allow enabling this, if there are no DVs present. if (!DeletionVectorUtils.isTableDVFree(snapshot)) { throw new DeltaTablePropertyValidationFailedException( table = tableName, subClass = ExistingDeletionVectorsWithIncrementalManifestGeneration) } } } } /** * Sets the committed version for a given application. Used to make operations * like streaming append idempotent. */ case class SetTransaction( appId: String, version: Long, @JsonDeserialize(contentAs = classOf[java.lang.Long]) lastUpdated: Option[Long]) extends Action { override def wrap: SingleAction = SingleAction(txn = this) } /** * The domain metadata action contains a configuration (string-string map) for a named metadata * domain. Two overlapping transactions conflict if they both contain a domain metadata action for * the same metadata domain. * * [[domain]]: A string used to identify a specific feature. * [[configuration]]: A string containing configuration options for the conflict domain. * [[removed]]: If it is true it serves as a tombstone to logically delete a [[DomainMetadata]] * action. */ case class DomainMetadata( domain: String, configuration: String, removed: Boolean) extends Action { override def wrap: SingleAction = SingleAction(domainMetadata = this) } /** Actions pertaining to the addition and removal of files. */ sealed trait FileAction extends Action { val path: String val dataChange: Boolean @JsonIgnore val tags: Map[String, String] @JsonIgnore lazy val pathAsUri: URI = { // Paths like http:example.com are opaque URIs that have schema and scheme-specific parts, but // path is not defined. We do not support such paths, so we throw an exception. val uri = new URI(path) if (uri.getPath == null) { throw DeltaErrors.cannotReconstructPathFromURI(path) } uri } @JsonIgnore def numLogicalRecords: Option[Long] @JsonIgnore val partitionValues: Map[String, String] @JsonIgnore def getFileSize: Long def stats: String def deletionVector: DeletionVectorDescriptor /** Returns the approx size of the remaining records after excluding the deleted ones. */ @JsonIgnore def estLogicalFileSize: Option[Long] /** Returns [[tags]] or an empty Map if null */ @JsonIgnore def tagsOrEmpty: Map[String, String] = Option(tags).getOrElse(Map.empty[String, String]) /** * Return tag value if tags is not null and the tag present. */ @JsonIgnore def getTag(tagName: String): Option[String] = Option(tags).flatMap(_.get(tagName)) /** Returns the [[SparkPath]] for this file action. */ def sparkPath: SparkPath = SparkPath.fromUrlString(path) /** Returns the [[Path]] for this file action (not URL-encoded). */ def toPath: Path = sparkPath.toPath /** Returns the absolute [[Path]] for this file action (not URL-encoded). */ def absolutePath(deltaLog: DeltaLog): Path = { // dataPath is not URL-encoded. val dataPath: Path = deltaLog.dataPath // this.path is a URL-encoded String, that is either the relative or absolute path. DeltaFileOperations.absolutePath(dataPath.toString, path) } } case class ParsedStatsFields( numLogicalRecords: Option[Long], tightBounds: Option[Boolean]) /** * Common trait for AddFile and RemoveFile actions providing methods for the computation of * logical, physical and deleted number of records based on the statistics and the Deletion Vector * of the file. */ trait HasNumRecords { this: FileAction => @JsonIgnore @transient protected lazy val parsedStatsFields: Option[ParsedStatsFields] = Option(stats).collect { case stats if stats.nonEmpty => val node = new ObjectMapper().readTree(stats) val numLogicalRecords = if (node.has("numRecords")) { Some(node.get("numRecords")).filterNot(_.isNull).map(_.asLong()) .map(_ - numDeletedRecords) } else None val tightBounds = if (node.has("tightBounds")) { Some(node.get("tightBounds")).filterNot(_.isNull).map(_.asBoolean()) } else None ParsedStatsFields(numLogicalRecords, tightBounds) } /** Returns the number of logical records, which do not include those marked as deleted. */ @JsonIgnore @transient override lazy val numLogicalRecords: Option[Long] = parsedStatsFields.flatMap(_.numLogicalRecords) /** Returns the number of records marked as deleted. */ @JsonIgnore def numDeletedRecords: Long = deletionVector match { case dv: DeletionVectorDescriptor => dv.cardinality case _ => 0L } /** Returns the total number of records, including those marked as deleted. */ @JsonIgnore def numPhysicalRecords: Option[Long] = numLogicalRecords.map(_ + numDeletedRecords) /** Returns the estimated size of the logical records in the file. */ @JsonIgnore override def estLogicalFileSize: Option[Long] = logicalToPhysicalRecordsRatio.map(n => (n * getFileSize).toLong) /** Returns the ratio of the logical number of records to the total number of records. */ @JsonIgnore def logicalToPhysicalRecordsRatio: Option[Double] = numLogicalRecords.map { numLogicalRecords => numLogicalRecords.toDouble / (numLogicalRecords + numDeletedRecords) } /** Returns the ratio of number of deleted records to the total number of records. */ @JsonIgnore def deletedToPhysicalRecordsRatio: Option[Double] = logicalToPhysicalRecordsRatio.map(1.0d - _) /** Returns whether the statistics are tight or wide. */ @JsonIgnore @transient lazy val tightBounds: Option[Boolean] = parsedStatsFields.flatMap(_.tightBounds) } /** * Adds a new file to the table. When multiple [[AddFile]] file actions * are seen with the same `path` only the metadata from the last one is * kept. * * [[path]] is URL-encoded. */ case class AddFile( override val path: String, @JsonInclude(JsonInclude.Include.ALWAYS) /** * [[partitionValues]] can be a raw and not normalized string. This is critical for a certain * data types such as timestamps. Use [[normalizedPartitionValues]] instead if you want a * normalized value. */ partitionValues: Map[String, String], size: Long, modificationTime: Long, override val dataChange: Boolean, override val stats: String = null, override val tags: Map[String, String] = null, override val deletionVector: DeletionVectorDescriptor = null, @JsonDeserialize(contentAs = classOf[java.lang.Long]) baseRowId: Option[Long] = None, @JsonDeserialize(contentAs = classOf[java.lang.Long]) defaultRowCommitVersion: Option[Long] = None, clusteringProvider: Option[String] = None ) extends FileAction with HasNumRecords { require(path.nonEmpty) override def wrap: SingleAction = SingleAction(add = this) def remove: RemoveFile = removeWithTimestamp() def removeWithTimestamp( timestamp: Long = System.currentTimeMillis(), dataChange: Boolean = true ): RemoveFile = { var newTags = tags // scalastyle:off RemoveFile( path, Some(timestamp), dataChange, extendedFileMetadata = Some(true), partitionValues, Some(size), newTags, deletionVector = deletionVector, baseRowId = baseRowId, defaultRowCommitVersion = defaultRowCommitVersion, stats = stats ) // scalastyle:on } /** * Logically remove rows by associating a `deletionVector` with the file. * @param deletionVector: The descriptor of the DV that marks rows as deleted. * @param dataChange: When false, the actions are marked as no-data-change actions. */ def removeRows( deletionVector: DeletionVectorDescriptor, updateStats: Boolean, dataChange: Boolean = true): (AddFile, RemoveFile) = { // Verify DV does not contain any invalid row indexes. Note, maxRowIndex is optional // and not all commands may set it when updating DVs. (numPhysicalRecords, deletionVector.maxRowIndex) match { case (Some(numPhysicalRecords), Some(maxRowIndex)) if (maxRowIndex + 1 > numPhysicalRecords) => throw DeltaErrors.deletionVectorInvalidRowIndex() case _ => // Nothing to check. } // We make sure maxRowIndex is not stored in the log. val dvDescriptorWithoutMaxRowIndex = deletionVector.maxRowIndex match { case Some(_) => deletionVector.copy(maxRowIndex = None) case _ => deletionVector } var addFileWithNewDv = this.copy(deletionVector = dvDescriptorWithoutMaxRowIndex, dataChange = dataChange) if (updateStats) { addFileWithNewDv = addFileWithNewDv.withoutTightBoundStats } val removeFileWithOldDv = this.removeWithTimestamp(dataChange = dataChange) // Sanity check for incremental DV updates. if (addFileWithNewDv.numDeletedRecords < removeFileWithOldDv.numDeletedRecords) { throw DeltaErrors.deletionVectorSizeMismatch() } (addFileWithNewDv, removeFileWithOldDv) } /** * Return the unique id of the deletion vector, if present, or `None` if there's no DV. * * The unique id differentiates DVs, even if there are multiple in the same file * or the DV is stored inline. */ @JsonIgnore def getDeletionVectorUniqueId: Option[String] = Option(deletionVector).map(_.uniqueId) /** Update stats to have tightBounds = false, if file has any stats. */ def withoutTightBoundStats: AddFile = { if (stats == null || stats.isEmpty) { this } else { val node = JsonUtils.mapper.readTree(stats).asInstanceOf[ObjectNode] if (node.has("tightBounds") && !node.get("tightBounds").asBoolean(true)) { this } else { node.put("tightBounds", false) val newStatsString = JsonUtils.mapper.writer.writeValueAsString(node) this.copy(stats = newStatsString) } } } /** * Return partition values as literals with the correct data type according to the partition * schema. Typed literals are safe for comparison purposes as the value and not the string * format is compared. * @return Map of partition column names to literals with the correct data type. */ def normalizedPartitionValues( spark: SparkSession, deltaTxn: OptimisticTransaction): Map[String, Literal] = { normalizedPartitionValues(spark, deltaTxn.metadata.physicalPartitionSchema, Some(deltaTxn)) } /** * Return partition values as literals with the correct data type according to the partition * schema. Typed literals are safe for comparison purposes as the value and not the string * format is compared. * @return Map of partition column names to literals with the correct data type. */ def normalizedPartitionValues( spark: SparkSession, partitionSchema: StructType, deltaTxn: Option[OptimisticTransaction] = None): Map[String, Literal] = { def partitionValuesAsStringLiterals: Map[String, Literal] = { // Convert all partition values to string literals partitionValues.map { case (k, v) => (k, Literal(v)) } } val normalizePartitionValuesOnRead = spark.conf.get(DeltaSQLConf.DELTA_NORMALIZE_PARTITION_VALUES_ON_READ) if (normalizePartitionValuesOnRead) { val timeZone = spark.sessionState.conf.sessionLocalTimeZone try { val typedPartitionValueLiterals = PartitionUtils.parsePartitionValues( partitionValues, partitionSchema, java.util.TimeZone.getDefault.getID, validatePartitionColumns = true) val stringNormalizedPartitionValues = typedPartitionValueLiterals.map { case (k, v) => (k, PartitionUtils.literalToNormalizedString( v, Some(timeZone), useUtcNormalizedTimestamp = true)) } if (stringNormalizedPartitionValues != partitionValues) { Action.recordDeltaEvent( deltaTxn.map(_.deltaLog).orNull, opType = "delta.normalizedPartitionValues.unnormalizedValuesExist", data = Map( "readSnapshotMetadata" -> deltaTxn.map(_.snapshot.metadata).orNull, "txnMetadata" -> deltaTxn.map(_.metadata).orNull, "commitInfo" -> deltaTxn.map(_.getCommitInfo).orNull ) ) } typedPartitionValueLiterals } catch { case NonFatal(e) => val opTypeSuffix = PartitionUtils.classifyPartitionValueParsingError(e) Action.recordDeltaEvent( deltaTxn.map(_.deltaLog).orNull, opType = "delta.normalizedPartitionValues.partitionValueParsingError" + opTypeSuffix, data = Map( "exceptionMessage" -> e.getMessage, "readSnapshotMetadata" -> deltaTxn.map(_.snapshot.metadata).orNull, "txnMetadata" -> deltaTxn.map(_.metadata).orNull, "commitInfo" -> deltaTxn.map(_.getCommitInfo).orNull, "readSnapshotVersion" -> deltaTxn.map(_.snapshot.version).getOrElse(-1L), "timeZone" -> timeZone ) ) partitionValuesAsStringLiterals } } else { partitionValuesAsStringLiterals } } // Don't use lazy val because we want to save memory. @JsonIgnore def insertionTime: Long = longTag(AddFile.Tags.INSERTION_TIME) // From modification time in milliseconds to microseconds. .getOrElse(TimeUnit.MICROSECONDS.convert(modificationTime, TimeUnit.MILLISECONDS)) def copyWithTags(newTags: Map[String, String]): AddFile = copy(tags = tagsOrEmpty ++ newTags) def tag(tag: AddFile.Tags.KeyType): Option[String] = getTag(tag.name) def longTag(tagKey: AddFile.Tags.KeyType): Option[Long] = tag(tagKey).map(_.toLong) def copyWithTag(tag: AddFile.Tags.KeyType, value: String): AddFile = copy(tags = tagsOrEmpty + (tag.name -> value)) def copyWithoutTag(tag: AddFile.Tags.KeyType): AddFile = { if (tags == null) { this } else { copy(tags = tags - tag.name) } } @JsonIgnore override def getFileSize: Long = size /** * Before serializing make sure deletionVector.maxRowIndex is not defined. * This is only a transient property and it is not intended to be stored in the log. */ override def json: String = { if (deletionVector != null) assert(!deletionVector.maxRowIndex.isDefined) super.json } } object AddFile { /** * Misc file-level metadata. * * The convention is that clients may safely ignore any/all of these tags and this should never * have an impact on correctness. * * Otherwise, the information should go as a field of the AddFile action itself and the Delta * protocol version should be bumped. */ object Tags { sealed abstract class KeyType(val name: String) /** [[ZCUBE_ID]]: identifier of the OPTIMIZE ZORDER BY job that this file was produced by */ object ZCUBE_ID extends AddFile.Tags.KeyType("ZCUBE_ID") /** [[ZCUBE_ZORDER_BY]]: ZOrdering of the corresponding ZCube */ object ZCUBE_ZORDER_BY extends AddFile.Tags.KeyType("ZCUBE_ZORDER_BY") /** [[ZCUBE_ZORDER_CURVE]]: Clustering strategy of the corresponding ZCube */ object ZCUBE_ZORDER_CURVE extends AddFile.Tags.KeyType("ZCUBE_ZORDER_CURVE") /** * [[INSERTION_TIME]]: the latest timestamp in micro seconds when the data in the file * was inserted */ object INSERTION_TIME extends AddFile.Tags.KeyType("INSERTION_TIME") /** [[PARTITION_ID]]: rdd partition id that has written the file, will not be stored in the physical log, only used for communication */ object PARTITION_ID extends AddFile.Tags.KeyType("PARTITION_ID") /** [[OPTIMIZE_TARGET_SIZE]]: target file size the file was optimized to. */ object OPTIMIZE_TARGET_SIZE extends AddFile.Tags.KeyType("OPTIMIZE_TARGET_SIZE") /** [[ICEBERG_COMPAT_VERSION]]: IcebergCompat version */ object ICEBERG_COMPAT_VERSION extends AddFile.Tags.KeyType("ICEBERG_COMPAT_VERSION") } /** Convert a [[Tags.KeyType]] to a string to be used in the AddMap.tags Map[String, String]. */ def tag(tagKey: Tags.KeyType): String = tagKey.name } /** * Logical removal of a given file from the reservoir. Acts as a tombstone before a file is * deleted permanently. * * Note that for protocol compatibility reasons, the fields `partitionValues`, `size`, and `tags` * are only present when the extendedFileMetadata flag is true. New writers should generally be * setting this flag, but old writers (and FSCK) won't, so readers must check this flag before * attempting to consume those values. * * Since old tables would not have `extendedFileMetadata` and `size` field, we should make them * nullable by setting their type Option. * * [[path]] is URL-encoded. */ // scalastyle:off case class RemoveFile( override val path: String, @JsonDeserialize(contentAs = classOf[java.lang.Long]) deletionTimestamp: Option[Long], override val dataChange: Boolean = true, extendedFileMetadata: Option[Boolean] = None, partitionValues: Map[String, String] = null, @JsonDeserialize(contentAs = classOf[java.lang.Long]) size: Option[Long] = None, override val tags: Map[String, String] = null, override val deletionVector: DeletionVectorDescriptor = null, @JsonDeserialize(contentAs = classOf[java.lang.Long]) baseRowId: Option[Long] = None, @JsonDeserialize(contentAs = classOf[java.lang.Long]) defaultRowCommitVersion: Option[Long] = None, override val stats: String = null ) extends FileAction with HasNumRecords { override def wrap: SingleAction = SingleAction(remove = this) @JsonIgnore val delTimestamp: Long = deletionTimestamp.getOrElse(0L) /** * Return the unique id of the deletion vector, if present, or `None` if there's no DV. * * The unique id differentiates DVs, even if there are multiple in the same file * or the DV is stored inline. */ @JsonIgnore def getDeletionVectorUniqueId: Option[String] = Option(deletionVector).map(_.uniqueId) /** * Create a copy with the new tag. `extendedFileMetadata` is copied unchanged. */ def copyWithTag(tag: String, value: String): RemoveFile = copy( tags = tagsOrEmpty + (tag -> value)) /** * Create a copy without the tag. */ def copyWithoutTag(tag: String): RemoveFile = copy(tags = tagsOrEmpty - tag) @JsonIgnore override def getFileSize: Long = size.getOrElse(0L) /** Only for testing. */ @JsonIgnore private [delta] def isDVTombstone: Boolean = DeletionVectorDescriptor.isDeletionVectorPath(new Path(path)) } // scalastyle:on /** * A change file containing CDC data for the Delta version it's within. Non-CDC readers should * ignore this, CDC readers should scan all ChangeFiles in a version rather than computing * changes from AddFile and RemoveFile actions. * * [[path]] is URL-encoded. */ @JsonIgnoreProperties(Array("stats")) case class AddCDCFile( override val path: String, @JsonInclude(JsonInclude.Include.ALWAYS) partitionValues: Map[String, String], size: Long, override val tags: Map[String, String] = null, override val stats: String = null) extends FileAction with HasNumRecords { override val dataChange = false @JsonIgnore override val deletionVector: DeletionVectorDescriptor = null override def wrap: SingleAction = SingleAction(cdc = this) @JsonIgnore override def getFileSize: Long = size @JsonIgnore override def estLogicalFileSize: Option[Long] = None } case class Format( provider: String = "parquet", // If we support `options` in future, we should not store any file system options since they may // contain credentials. options: Map[String, String] = Map.empty) /** * Updates the metadata of the table. Only the last update to the [[Metadata]] * of a table is kept. It is the responsibility of the writer to ensure that * any data already present in the table is still valid after any change. */ case class Metadata( id: String = java.util.UUID.randomUUID().toString, name: String = null, description: String = null, format: Format = Format(), schemaString: String = null, partitionColumns: Seq[String] = Nil, configuration: Map[String, String] = Map.empty, @JsonDeserialize(contentAs = classOf[java.lang.Long]) createdTime: Option[Long] = None) extends Action with SparkAbstractMetadata with StorageAbstractMetadata { // The `schema` and `partitionSchema` methods should be vals or lazy vals, NOT // defs, because parsing StructTypes from JSON is extremely expensive and has // caused perf. problems here in the past: /** * Compare this metadata with other. * Returns a set of field names that differ between the two metadata objects. * Returns an empty set when there are no differences. */ def diffFieldNames(other: Metadata): Set[String] = { import scala.reflect.runtime.universe._ // In scala 2.13, we can directly use productElementName(n: Int) along with productArity. val fieldNames = typeOf[Metadata].members.sorted.collect { case m: MethodSymbol if m.isCaseAccessor => m.name.toString } // It relies on the fact that members.sorted outputs fields in declaration order. fieldNames.zipWithIndex.collect { case (name, i) if this.productElement(i) != other.productElement(i) => name }.toSet } /** * Column mapping mode for this table */ @JsonIgnore lazy val columnMappingMode: DeltaColumnMappingMode = DeltaConfigs.COLUMN_MAPPING_MODE.fromMetaData(this) /** * Column mapping max id for this table */ @JsonIgnore lazy val columnMappingMaxId: Long = DeltaConfigs.COLUMN_MAPPING_MAX_ID.fromMetaData(this) /** Returns the schema as a [[StructType]] */ @JsonIgnore lazy val schema: StructType = Option(schemaString) .map(DataType.fromJson(_).asInstanceOf[StructType]) .getOrElse(StructType.apply(Nil)) /** Returns the partitionSchema as a [[StructType]] */ @JsonIgnore lazy val partitionSchema: StructType = new StructType(partitionColumns.map(c => schema(c)).toArray) /** Partition value keys in the AddFile map. */ @JsonIgnore lazy val physicalPartitionSchema: StructType = DeltaColumnMapping.renameColumns(partitionSchema) /** Columns written out to files. */ @JsonIgnore lazy val dataSchema: StructType = { val partitions = partitionColumns.toSet StructType(schema.filterNot(f => partitions.contains(f.name))) } /** Partition value written out to files */ @JsonIgnore lazy val physicalPartitionColumns: Seq[String] = physicalPartitionSchema.fieldNames.toSeq /** * Store non-partition columns and their corresponding [[OptimizablePartitionExpression]] which * can be used to create partition filters from data filters of these non-partition columns. */ @JsonIgnore lazy val optimizablePartitionExpressions: Map[String, Seq[OptimizablePartitionExpression]] = GeneratedColumn.getOptimizablePartitionExpressions(schema, partitionSchema) /** * The name of commit-coordinator which arbitrates the commits to the table. This must be * available if this is a coordinated-commits table. */ @JsonIgnore lazy val coordinatedCommitsCoordinatorName: Option[String] = DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.fromMetaData(this) /** The configuration to uniquely identify the commit-coordinator for coordinated-commits. */ @JsonIgnore lazy val coordinatedCommitsCoordinatorConf: Map[String, String] = DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF.fromMetaData(this) /** The table specific configuration for coordinated-commits. */ @JsonIgnore lazy val coordinatedCommitsTableConf: Map[String, String] = DeltaConfigs.COORDINATED_COMMITS_TABLE_CONF.fromMetaData(this) override def wrap: SingleAction = SingleAction(metaData = this) override def getId: String = id override def getName: String = name override def getDescription: String = description @JsonIgnore override def getProvider: String = format.provider @JsonIgnore override def getFormatOptions: java.util.Map[String, String] = format.options.asJava override def getSchemaString: String = schemaString override def getPartitionColumns: java.util.List[String] = partitionColumns.asJava override def getConfiguration: java.util.Map[String, String] = configuration.asJava override def getCreatedTime: java.lang.Long = createdTime.map(Long.box).orNull } /** * Interface for objects that represents the information for a commit. Commits can be referred to * using a version and timestamp. The timestamp of a commit comes from the remote storage * `lastModifiedTime`, and can be adjusted for clock skew. Hence we have the method `withTimestamp`. */ trait CommitMarker { /** Get the timestamp of the commit as millis after the epoch. */ def getTimestamp: Long /** Return a copy object of this object with the given timestamp. */ def withTimestamp(timestamp: Long): CommitMarker /** Get the version of the commit. */ def getVersion: Long } /** * Holds provenance information about changes to the table. This [[Action]] * is not stored in the checkpoint and has reduced compatibility guarantees. * Information stored in it is best effort (i.e. can be falsified by the writer). * * @param inCommitTimestamp A monotonically increasing timestamp that represents the time since * epoch in milliseconds when the commit write was started. This should * only be set when the feature inCommitTimestamps is enabled. * @param isBlindAppend Whether this commit has blindly appended without caring about existing files * @param engineInfo The information for the engine that makes the commit. * If a commit is made by Delta Lake 1.1.0 or above, it will be * `Apache-Spark/x.y.z Delta-Lake/x.y.z`. */ case class CommitInfo( // The commit version should be left unfilled during commit(). When reading a delta file, we can // infer the commit version from the file name and fill in this field then. @JsonDeserialize(contentAs = classOf[java.lang.Long]) version: Option[Long], @JsonDeserialize(contentAs = classOf[java.lang.Long]) inCommitTimestamp: Option[Long], timestamp: Timestamp, userId: Option[String], userName: Option[String], operation: String, @JsonSerialize(using = classOf[JsonMapSerializer]) @JsonDeserialize(using = classOf[JsonMapDeserializer]) operationParameters: Map[String, String], job: Option[JobInfo], notebook: Option[NotebookInfo], clusterId: Option[String], @JsonDeserialize(contentAs = classOf[java.lang.Long]) readVersion: Option[Long], isolationLevel: Option[String], isBlindAppend: Option[Boolean], operationMetrics: Option[Map[String, String]], userMetadata: Option[String], tags: Option[Map[String, String]], engineInfo: Option[String], txnId: Option[String]) extends Action with CommitMarker with SparkAbstractCommitInfo with StorageAbstractCommitInfo { override def wrap: SingleAction = SingleAction(commitInfo = this) override def withTimestamp(timestamp: Long): CommitInfo = { this.copy(timestamp = new Timestamp(timestamp)) } // We need to explicitly ignore this field during serialization as Jackson // by default calls all public getters of an object, which would lead to // either an exception or the inCommitTimestamp being serialized twice. @JsonIgnore override def getCommitTimestamp: Long = { inCommitTimestamp.getOrElse { throw DeltaErrors.missingCommitTimestamp(version.map(_.toString).getOrElse("unknown")) } } override def getTimestamp: Long = timestamp.getTime @JsonIgnore override def getVersion: Long = version.get } case class JobInfo( jobId: String, jobName: String, jobRunId: String, runId: String, jobOwnerId: String, triggerType: String) object JobInfo { def fromContext(context: Map[String, String]): Option[JobInfo] = { context.get("jobId").map { jobId => JobInfo( jobId, context.get("jobName").orNull, context.get("multitaskParentRunId").orNull, context.get("runId").orNull, context.get("jobOwnerId").orNull, context.get("jobTriggerType").orNull) } } } case class NotebookInfo(notebookId: String) object NotebookInfo { def fromContext(context: Map[String, String]): Option[NotebookInfo] = { context.get("notebookId").orElse(context.get("notebook_id")).map { nbId => NotebookInfo(nbId) } } } object CommitInfo { def empty(version: Option[Long] = None): CommitInfo = { CommitInfo(version, None, null, None, None, null, null, None, None, None, None, None, None, None, None, None, None, None) } // scalastyle:off argcount def apply( time: Long, operation: String, inCommitTimestamp: Option[Long] = None, operationParameters: Map[String, String], commandContext: Map[String, String], readVersion: Option[Long], isolationLevel: Option[String], isBlindAppend: Option[Boolean], operationMetrics: Option[Map[String, String]], userMetadata: Option[String], tags: Option[Map[String, String]], txnId: Option[String]): CommitInfo = { apply(None, time, operation, inCommitTimestamp, operationParameters, commandContext, readVersion, isolationLevel, isBlindAppend, operationMetrics, userMetadata, tags, txnId) } def apply( version: Option[Long], time: Long, operation: String, inCommitTimestamp: Option[Long], operationParameters: Map[String, String], commandContext: Map[String, String], readVersion: Option[Long], isolationLevel: Option[String], isBlindAppend: Option[Boolean], operationMetrics: Option[Map[String, String]], userMetadata: Option[String], tags: Option[Map[String, String]], txnId: Option[String]): CommitInfo = { val getUserName = commandContext.get("user").flatMap { case "unknown" => None case other => Option(other) } CommitInfo( version, inCommitTimestamp, new Timestamp(time), commandContext.get("userId"), getUserName, operation, operationParameters, JobInfo.fromContext(commandContext), NotebookInfo.fromContext(commandContext), commandContext.get("clusterId"), readVersion, isolationLevel, isBlindAppend, operationMetrics, userMetadata, tags, getEngineInfo, txnId) } // scalastyle:on argcount private def getEngineInfo: Option[String] = { Some(s"Apache-Spark/${org.apache.spark.SPARK_VERSION} Delta-Lake/${io.delta.VERSION}") } /** * Returns the `inCommitTimestamp` of the given `commitInfoOpt` if it is defined. * Throws an exception if `commitInfoOpt` is empty or contains an empty `inCommitTimestamp`. */ def getRequiredInCommitTimestamp(commitInfoOpt: Option[CommitInfo], version: String): Long = { val commitInfo = commitInfoOpt.getOrElse { throw DeltaErrors.missingCommitInfo(InCommitTimestampTableFeature.name, version) } commitInfo.inCommitTimestamp.getOrElse { throw DeltaErrors.missingCommitTimestamp(version) } } /** * Returns the legacy value of operation parameters after deserialization. * See [[CommitInfoOperationParametersOnly]] and [[JsonMapDeserializer]] for more * details about how operation parameter deserialization was broken before. These * legacy values are the same as the original broken version. */ def getLegacyPostDeserializationOperationParameters( operationParameters: Map[String, String]): Map[String, String] = { // Use JsonMapSerializer to serialize the operation parameters val serializedOperationParameters = JsonUtils.toJson(CommitInfoOperationParametersOnly(operationParameters)) // Instead of using JsonMapDeserializer, // we can use JsonUtils.fromJson to deserialize the operation parameters JsonUtils.fromJson[CommitInfoOperationParametersOnly](serializedOperationParameters) .operationParameters } } /** A trait to represent actions which can only be part of Checkpoint */ sealed trait CheckpointOnlyAction extends Action /** * An [[Action]] containing the information about a sidecar file. * * @param path - sidecar path relative to `_delta_log/_sidecar` directory * @param sizeInBytes - size in bytes for the sidecar file * @param modificationTime - modification time of the sidecar file * @param tags - attributes of the sidecar file, defaults to null (which is semantically same as an * empty Map). This is kept null to ensure that the field is not present in the * generated json. */ case class SidecarFile( path: String, sizeInBytes: Long, modificationTime: Long, tags: Map[String, String] = null) extends CheckpointOnlyAction { override def wrap: SingleAction = SingleAction(sidecar = this) def toFileStatus(logPath: Path): FileStatus = { val partFilePath = new Path(FileNames.sidecarDirPath(logPath), path) new FileStatus(sizeInBytes, false, 0, 0, modificationTime, partFilePath) } } object SidecarFile { def apply(fileStatus: SerializableFileStatus): SidecarFile = { SidecarFile(fileStatus.getHadoopPath.getName, fileStatus.length, fileStatus.modificationTime) } def apply(fileStatus: FileStatus): SidecarFile = { SidecarFile(fileStatus.getPath.getName, fileStatus.getLen, fileStatus.getModificationTime) } } /** * Holds information about the Delta Checkpoint. This action will only be part of checkpoints. * * @param version version of the checkpoint * @param tags attributes of the checkpoint, defaults to null (which is semantically same as an * empty Map). This is kept null to ensure that the field is not present in the * generated json. */ case class CheckpointMetadata( version: Long, tags: Map[String, String] = null) extends CheckpointOnlyAction { override def wrap: SingleAction = SingleAction(checkpointMetadata = this) } /** A serialization helper to create a common action envelope. */ case class SingleAction( txn: SetTransaction = null, add: AddFile = null, remove: RemoveFile = null, metaData: Metadata = null, protocol: Protocol = null, cdc: AddCDCFile = null, checkpointMetadata: CheckpointMetadata = null, sidecar: SidecarFile = null, domainMetadata: DomainMetadata = null, commitInfo: CommitInfo = null) { def unwrap: Action = { if (add != null) { add } else if (remove != null) { remove } else if (metaData != null) { metaData } else if (txn != null) { txn } else if (protocol != null) { protocol } else if (cdc != null) { cdc } else if (sidecar != null) { sidecar } else if (checkpointMetadata != null) { checkpointMetadata } else if (domainMetadata != null) { domainMetadata } else if (commitInfo != null) { commitInfo } else { null } } } object SingleAction extends Logging { implicit def encoder: Encoder[SingleAction] = org.apache.spark.sql.delta.implicits.singleActionEncoder implicit def addFileEncoder: Encoder[AddFile] = org.apache.spark.sql.delta.implicits.addFileEncoder lazy val nullLitForRemoveFile: Column = Column(Literal(null, ScalaReflection.schemaFor[RemoveFile].dataType)) lazy val nullLitForAddCDCFile: Column = Column(Literal(null, ScalaReflection.schemaFor[AddCDCFile].dataType)) lazy val nullLitForMetadataAction: Column = Column(Literal(null, ScalaReflection.schemaFor[Metadata].dataType)) } /** Serializes Maps containing JSON strings without extra escaping. */ class JsonMapSerializer extends JsonSerializer[Map[String, String]] { def serialize( parameters: Map[String, String], jgen: JsonGenerator, provider: SerializerProvider): Unit = { jgen.writeStartObject() parameters.foreach { case (key, value) => if (value == null) { jgen.writeNullField(key) } else { jgen.writeFieldName(key) // Write value as raw data, since it's already JSON text jgen.writeRawValue(value) } } jgen.writeEndObject() } } /** * This is effectively performs an inverse of [[JsonMapSerializer]]. * * The in-memory representation of operation params of any Delta Operation can be * a combination of json encoded strings, simple strings, or primitives single-encoded * as strings. i.e. the values can be any of "abc", "123", "true", "1.0", "\"true\"", * "\"1.0\"" or more complex json encoded strings. * Due to how [[JsonMapSerializer]] strips one level of encoding for these values * during serialization, these can end up being written out in this form: * "123" -> 123 * "true" -> true * "1.0" -> 1.0 * "\"true\"" -> "true" * "\"1.0\"" -> "1.0" * Since operationParameters is a Map[String, String], during the deserialization phase, the * deserializer intelligently converts primitive types from above to simple strings. * i.e. * "123" -> 123 -> "123" * "true" -> true -> "true" * "1.0" -> 1.0 -> "1.0" * "\"true\"" -> "true" -> "true" * "\"1.0\"" -> "1.0" -> "1.0" * Since we stripped one level of encoding during serialization, we need to add it back to * get closer to the original in-memory representation. * i.e. * "123" -> 123 -> "123" -> "\"123\"" * "true" -> true -> "true" -> "\"true\"" * "1.0" -> 1.0 -> "1.0" -> "\"1.0\"" * "\"true\"" -> "true" -> "true" -> "\"true\"" * "\"1.0\"" -> "1.0" -> "1.0" -> "\"1.0\"" * Note how values that were single-encoded as strings originally are now double-encoded as * strings. * (i.e. "true" -> true -> "true" -> "\"true\""). This is because the deserializer converted * the primitive values to strings as well as retained simple strings as strings. In this process, * we lost some information about the original values. To fix this, we first deserialize the * values as a java.util.HashMap[String, Any] i.e.: * "123" -> 123 -> 123 (type: Integer) * "true" -> true -> true (type: Boolean) * "1.0" -> 1.0 -> 1.0 (type: Double) * "\"true\"" -> "true" -> "true" (type: String) * "\"1.0\"" -> "1.0" -> "1.0" (type: String) * and then JsonEncode them once to get the original in-memory representation. i.e. * "123" -> 123 -> 123 -> "123" * "true" -> true -> true -> "true" * "1.0" -> 1.0 -> 1.0 -> "1.0" * "\"true\"" -> "true" -> "true" -> "\"true\"" * "\"1.0\"" -> "1.0" -> "1.0" -> "\"1.0\"" */ class JsonMapDeserializer extends JsonDeserializer[Map[String, String]] { def deserialize(jp: JsonParser, ctxt: DeserializationContext): Map[String, String] = { // First read the map as a Map[String, Any]. Then use JsonUtils.toJson to convert // the values to JSON strings. val map = ctxt.readValue(jp, classOf[Map[String, Any]]) map.mapValues(JsonUtils.toJson(_)).toMap } } /** * This class is only used by [[CommitInfo.getLegacyPostDeserializationOperationParameters]] * to regenerate legacy operation parameters. * The legacy deserialization of operationParameters goes as follows. * - The in-memory representation of a Delta operation's parameters was a (String -> Any) map. * - When setting the operation parameters in a [[CommitInfo]], the map is transformed into * (String -> JsonEncodedString(Any)). * - A [[CommitInfo]] is serialized with a `JsonMapSerializer`, which wrote the values as raw * strings. * - However, the deserialization process used a default deserializer that was not an inverse of * JsonMapSerializer. The default deserializer automatically decoded JSON strings into their * inferred types, and then cast the values to strings. This meant the output of the * deserialization process was a (String -> String(Any)). * - Note that String() and JsonEncodedString() are not the same. For example, if the original * value was a string "abc", then String("abc") is still "abc", but JsonEncodedString("abc") is * "\"abc\"". In the new deserializer, we have changed it so that the deserialization process to * recover the input of the serialization process, i.e. (String -> JsonEncodedString(Any)). */ case class CommitInfoOperationParametersOnly( @JsonSerialize(using = classOf[JsonMapSerializer]) operationParameters: Map[String, String] ) ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/catalog/AbstractDeltaCatalog.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.catalog // scalastyle:off import.ordering.noEmptyLine import java.sql.Timestamp import java.util import java.util.Locale import scala.collection.JavaConverters._ import scala.collection.immutable.ListMap import scala.collection.mutable import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient.UC_TABLE_ID_KEY import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient.UC_TABLE_ID_KEY_OLD import org.apache.spark.sql.delta.skipping.clustering.ClusteredTableUtils import org.apache.spark.sql.delta.skipping.clustering.temp.{ClusterBy, ClusterBySpec} import org.apache.spark.sql.delta.skipping.clustering.temp.{ClusterByTransform => TempClusterByTransform} import org.apache.spark.sql.delta.{ColumnWithDefaultExprUtils, DeltaConfigs, DeltaErrors, DeltaTableUtils} import org.apache.spark.sql.delta.{DeltaOptions, IdentityColumn} import org.apache.spark.sql.delta.DeltaTableIdentifier.gluePermissionError import org.apache.spark.sql.delta.commands._ import org.apache.spark.sql.delta.constraints.{AddConstraint, DropConstraint} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.redirect.RedirectFeature import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.serverSidePlanning.ServerSidePlannedTable import org.apache.spark.sql.delta.sources.{DeltaDataSource, DeltaSourceUtils, DeltaSQLConf} import org.apache.spark.sql.delta.stats.StatisticsCollection import org.apache.spark.sql.delta.tablefeatures.DropFeature import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} import org.apache.spark.sql.delta.util.PartitionUtils import org.apache.hadoop.fs.Path import org.apache.spark.SparkException import org.apache.spark.internal.MDC import org.apache.spark.sql.{AnalysisException, DataFrame, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchNamespaceException, NoSuchTableException, UnresolvedAttribute, UnresolvedFieldName, UnresolvedFieldPosition} import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable, CatalogTableType, CatalogUtils, SessionCatalog} import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, QualifiedColType, QualifiedColTypeShims, SyncIdentity} import org.apache.spark.sql.connector.catalog.{DelegatingCatalogExtension, Identifier, StagedTable, StagingTableCatalog, SupportsWrite, Table, TableCapability, TableCatalog, TableChange, V1Table} import org.apache.spark.sql.connector.catalog.TableCapability._ import org.apache.spark.sql.connector.catalog.TableChange._ import org.apache.spark.sql.connector.expressions.{FieldReference, IdentityTransform, Literal, NamedReference, Transform} import org.apache.spark.sql.connector.write.{LogicalWriteInfo, SupportsTruncate, V1Write, WriteBuilder} import org.apache.spark.sql.execution.datasources.{DataSource, PartitioningUtils} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.sources.InsertableRelation import org.apache.spark.sql.types.{IntegerType, StructField, StructType} /** * V1 legacy implementation. Use [[org.apache.spark.sql.delta.catalog.DeltaCatalog]] instead. * See spark-unified/src/main/java/org/apache/spark/sql/delta/catalog/DeltaCatalog.java */ class DeltaCatalogV1 extends AbstractDeltaCatalog /** * Base class for Dsv2 catalog implementation, it contains all dsv1 based connector logic. * Introduced for compatibility purpose in the implementation of dsv2 based connector */ class AbstractDeltaCatalog extends DelegatingCatalogExtension with StagingTableCatalog with SupportsPathIdentifier with DeltaLogging { val spark = SparkSession.active private lazy val isUnityCatalog: Boolean = { val delegateField = classOf[DelegatingCatalogExtension].getDeclaredField("delegate") delegateField.setAccessible(true) delegateField.get(this).getClass.getCanonicalName.startsWith("io.unitycatalog.") } /** * Creates a Delta table * * @param ident The identifier of the table * @param schema The schema of the table * @param partitions The partition transforms for the table * @param allTableProperties The table properties that configure the behavior of the table or * provide information about the table * @param writeOptions Options specific to the write during table creation or replacement * @param sourceQuery A query if this CREATE request came from a CTAS or RTAS * @param operation The specific table creation mode, whether this is a Create/Replace/Create or * Replace */ private def createDeltaTable( ident: Identifier, schema: StructType, partitions: Array[Transform], allTableProperties: util.Map[String, String], writeOptions: Map[String, String], sourceQuery: Option[DataFrame], operation: TableCreationModes.CreationMode ): Table = recordFrameProfile( "DeltaCatalog", "createDeltaTable") { // These two keys are tableProperties in data source v2 but not in v1, so we have to filter // them out. Otherwise property consistency checks will fail. val tableProperties = allTableProperties.asScala.filterKeys { case TableCatalog.PROP_LOCATION => false case TableCatalog.PROP_PROVIDER => false case TableCatalog.PROP_COMMENT => false case TableCatalog.PROP_OWNER => false case TableCatalog.PROP_EXTERNAL => false case "path" => false case "option.path" => false case _ => true }.toMap val (partitionColumns, maybeBucketSpec, maybeClusterBySpec) = convertTransforms(partitions) validateClusterBySpec(maybeClusterBySpec, schema) // Check partition columns are not IDENTITY columns. partitionColumns.foreach { colName => if (ColumnWithDefaultExprUtils.isIdentityColumn(schema(colName))) { throw DeltaErrors.identityColumnPartitionNotSupported(colName) } } var newSchema = schema var newPartitionColumns = partitionColumns var newBucketSpec = maybeBucketSpec val conf = spark.sessionState.conf allTableProperties.asScala .get(DeltaConfigs.DATA_SKIPPING_STATS_COLUMNS.key) .foreach(StatisticsCollection.validateDeltaStatsColumns(schema, partitionColumns, _)) val isByPath = isPathIdentifier(ident) if (isByPath && !conf.getConf(DeltaSQLConf.DELTA_LEGACY_ALLOW_AMBIGUOUS_PATHS) && allTableProperties.containsKey("location") // The location property can be qualified and different from the path in the identifier, so // we check `endsWith` here. && Option(allTableProperties.get("location")).exists(!_.endsWith(ident.name())) ) { throw DeltaErrors.ambiguousPathsInCreateTableException( ident.name(), allTableProperties.get("location")) } val location = if (isByPath) { Option(ident.name()) } else { Option(allTableProperties.get("location")) } val id = { // Preserve the catalog name in the V1 identifier for Unity Catalog because this `id` // becomes `tableDesc.identifier` for the rest of the create/replace flow. Downstream // catalog-update logic reads `table.identifier.catalog`; without it, the table looks like // an unqualified V1 table and catalog updates route through the current/session catalog // instead of the delegated Unity Catalog entry. val base = TableIdentifier(ident.name(), ident.namespace().lastOption) if (isUnityCatalog) { base.copy(catalog = Some(name())) // `name()` here is the catalog name. } else { base } } var locUriOpt = location.map(CatalogUtils.stringToURI) val existingTableOpt = getExistingTableIfExists(id, Some(ident), operation) // PROP_IS_MANAGED_LOCATION indicates that the table location is not user-specified but // system-generated. The table should be created as managed table in this case. val isManagedLocation = Option(allTableProperties.get(TableCatalog.PROP_IS_MANAGED_LOCATION)) .exists(_.equalsIgnoreCase("true")) // Note: Spark generates the table location for managed tables in // `DeltaCatalog#delegate#createTable`, so `isManagedLocation` should never be true if // Unity Catalog is not involved. For safety we also check `isUnityCatalog` here. val respectManagedLoc = isUnityCatalog || DeltaUtils.isTesting val tableType = if (location.isEmpty || (isManagedLocation && respectManagedLoc)) { CatalogTableType.MANAGED } else { CatalogTableType.EXTERNAL } val loc = locUriOpt .orElse(existingTableOpt.flatMap(_.storage.locationUri)) .getOrElse(spark.sessionState.catalog.defaultTablePath(id)) val storage = DataSource.buildStorageFormatFromOptions(writeOptions) .copy(locationUri = Option(loc)) val commentOpt = Option(allTableProperties.get("comment")) var tableDesc = new CatalogTable( identifier = id, tableType = tableType, storage = storage, schema = newSchema, provider = Some(DeltaSourceUtils.ALT_NAME), partitionColumnNames = newPartitionColumns, bucketSpec = newBucketSpec, properties = tableProperties, comment = commentOpt ) val withDb = verifyTableAndSolidify( tableDesc, None, maybeClusterBySpec ) val writer = sourceQuery.map { df => // For safety, only extract the file system options here, to create deltaLog. val fileSystemOptions = writeOptions.filter { case (k, _) => DeltaTableUtils.validDeltaTableHadoopPrefixes.exists(k.startsWith) } WriteIntoDelta( DeltaUtils.getDeltaLogFromTableOrPath(spark, existingTableOpt, new Path(loc), fileSystemOptions), operation.mode, new DeltaOptions(withDb.storage.properties, spark.sessionState.conf), withDb.partitionColumnNames, withDb.properties ++ commentOpt.map("comment" -> _), df, Some(tableDesc), schemaInCatalog = if (newSchema != schema) Some(newSchema) else None) } CreateDeltaTableCommand( withDb, existingTableOpt, operation.mode, writer, operation, tableByPath = isByPath, allowCatalogManaged = isUnityCatalog && tableType == CatalogTableType.MANAGED, // We should invoke the Spark catalog plugin API to create the table, to // respect third party catalogs. // TODO: Spark `V2SessionCatalog` mistakenly treat tables with location as EXTERNAL table. // Before this bug is fixed, we should only call the catalog plugin API to create tables // if UC is enabled to replace `V2SessionCatalog`. createTableFunc = Option.when(isUnityCatalog) { v1Table => { val t = V1Table(v1Table) super.createTable(ident, t.columns(), t.partitioning, t.properties) } }).run(spark) loadTable(ident) } override def loadTable(ident: Identifier): Table = recordFrameProfile( "DeltaCatalog", "loadTable") { try { val table = super.loadTable(ident) ServerSidePlannedTable.tryCreate(spark, ident, table, isUnityCatalog).foreach { sspt => return sspt } table match { case v1: V1Table if DeltaTableUtils.isDeltaTable(v1.catalogTable) => loadCatalogTable(ident, v1.catalogTable) case o => o } } catch { case e @ ( _: NoSuchDatabaseException | _: NoSuchNamespaceException | _: NoSuchTableException) => if (isPathIdentifier(ident)) { loadPathTable(ident) } else if (isIcebergPathIdentifier(ident)) { newIcebergPathTable(ident) } else { throw e } case e: AnalysisException if gluePermissionError(e) && isPathIdentifier(ident) => logWarning(log"Received an access denied error from Glue. Assuming this " + log"identifier (${MDC(DeltaLogKeys.TABLE_NAME, ident)}) is path based.", e) loadPathTable(ident) } } override def loadTable(ident: Identifier, timestamp: Long): Table = { loadTableWithTimeTravel(ident, version = None, Some(timestamp)) } override def loadTable(ident: Identifier, version: String): Table = { loadTableWithTimeTravel(ident, Some(version), timestamp = None) } /** * Helper method which loads a Delta table with given time travel parameters. * Exactly one of the timetravel parameters (version or timestamp) must be present. * * @param version The table version to load * @param timestamp The timestamp for the table to load, in microseconds */ private def loadTableWithTimeTravel( ident: Identifier, version: Option[String], timestamp: Option[Long]): Table = { assert(version.isEmpty ^ timestamp.isEmpty, "Either the version or timestamp should be provided for time travel") val table = loadTable(ident) table match { case deltaTable: DeltaTableV2 => val ttOpts = Map(DeltaDataSource.TIME_TRAVEL_SOURCE_KEY -> "SQL") ++ (if (version.isDefined) { Map(DeltaDataSource.TIME_TRAVEL_VERSION_KEY -> version.get) } else { val timestampMs = timestamp.get / 1000 Map(DeltaDataSource.TIME_TRAVEL_TIMESTAMP_KEY -> new Timestamp(timestampMs).toString) }) deltaTable.withOptions(ttOpts) // punt this problem up to the parent case _ if version.isDefined => super.loadTable(ident, version.get) case _ if timestamp.isDefined => super.loadTable(ident, timestamp.get) } } // Perform checks on ClusterBySpec. def validateClusterBySpec( maybeClusterBySpec: Option[ClusterBySpec], schema: StructType): Unit = { maybeClusterBySpec.foreach { clusterBy => // Check if the specified cluster by columns exists in the table. val resolver = spark.sessionState.conf.resolver clusterBy.columnNames.foreach { column => // This is the same check as in rules.scala, to keep the behaviour consistent. SchemaUtils.findColumnPosition(column.fieldNames(), schema, resolver) } // Check that columns are not duplicated in the cluster by statement. PartitionUtils.checkColumnNameDuplication( clusterBy.columnNames.map(_.toString), "in CLUSTER BY", resolver) // Check number of clustering columns is within allowed range. ClusteredTableUtils.validateNumClusteringColumns( clusterBy.columnNames.map(_.fieldNames.toSeq)) } } /** * Loads a Delta table that is registered in the catalog. * * @param ident The identifier of the table in the catalog. * @param catalogTable The catalog table metadata containing table properties and location. * @return A DeltaTableV2 instance with catalog metadata attached. */ protected def loadCatalogTable(ident: Identifier, catalogTable: CatalogTable): Table = { DeltaTableV2( spark, new Path(catalogTable.location), catalogTable = Some(catalogTable), tableIdentifier = Some(ident.toString)) } /** * Loads a Delta table directly from a path. * This is used for path-based table access where the identifier name is the table path. * * @param ident The identifier whose name contains the path to the Delta table. * @return A DeltaTableV2 instance loaded from the specified path. */ protected def loadPathTable(ident: Identifier): Table = { DeltaTableV2(spark, new Path(ident.name())) } private def getProvider(properties: util.Map[String, String]): String = { Option(properties.get("provider")) .getOrElse(spark.sessionState.conf.getConf(SQLConf.DEFAULT_DATA_SOURCE_NAME)) } private def createCatalogTable( ident: Identifier, schema: StructType, partitions: Array[Transform], properties: util.Map[String, String] ): Table = { super.createTable(ident, schema, partitions, properties) } override def createTable( ident: Identifier, columns: Array[org.apache.spark.sql.connector.catalog.Column], partitions: Array[Transform], properties: util.Map[String, String]): Table = { createTable( ident, org.apache.spark.sql.connector.catalog.CatalogV2Util.v2ColumnsToStructType(columns), partitions, properties) } override def createTable( ident: Identifier, schema: StructType, partitions: Array[Transform], properties: util.Map[String, String]) : Table = recordFrameProfile("DeltaCatalog", "createTable") { if (DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties))) { // TODO: we should extract write options from table properties for all the cases. We // can remove the UC check when we have confidence. val isUC = isUnityCatalog || properties.containsKey("test.simulateUC") val (props, writeOptions) = if (isUC) { val (props, writeOptions) = getTablePropsAndWriteOptions(properties) expandTableProps(props, writeOptions, spark.sessionState.conf) props.remove("test.simulateUC") translateUCTableIdProperty(props) (props, writeOptions) } else { (properties, Map.empty[String, String]) } createDeltaTable( ident, schema, partitions, props, writeOptions, sourceQuery = None, TableCreationModes.Create ) } else { createCatalogTable(ident, schema, partitions, properties ) } } override def stageReplace( ident: Identifier, schema: StructType, partitions: Array[Transform], properties: util.Map[String, String]): StagedTable = recordFrameProfile("DeltaCatalog", "stageReplace") { if (DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties))) { new StagedDeltaTableV2( ident, schema, partitions, properties, TableCreationModes.Replace ) } else { super.dropTable(ident) val table = createCatalogTable(ident, schema, partitions, properties ) BestEffortStagedTable(ident, table, this) } } override def stageCreateOrReplace( ident: Identifier, schema: StructType, partitions: Array[Transform], properties: util.Map[String, String]): StagedTable = recordFrameProfile("DeltaCatalog", "stageCreateOrReplace") { if (DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties))) { new StagedDeltaTableV2( ident, schema, partitions, properties, TableCreationModes.CreateOrReplace ) } else { try super.dropTable(ident) catch { case _: NoSuchDatabaseException => // this is fine case _: NoSuchTableException => // this is fine } val table = createCatalogTable(ident, schema, partitions, properties ) BestEffortStagedTable(ident, table, this) } } override def stageCreate( ident: Identifier, schema: StructType, partitions: Array[Transform], properties: util.Map[String, String]): StagedTable = recordFrameProfile("DeltaCatalog", "stageCreate") { if (DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties))) { new StagedDeltaTableV2( ident, schema, partitions, properties, TableCreationModes.Create ) } else { val table = createCatalogTable(ident, schema, partitions, properties ) BestEffortStagedTable(ident, table, this) } } // Copy of V2SessionCatalog.convertTransforms, which is private. private def convertTransforms( partitions: Seq[Transform]): (Seq[String], Option[BucketSpec], Option[ClusterBySpec]) = { val identityCols = new mutable.ArrayBuffer[String] var bucketSpec = Option.empty[BucketSpec] var clusterBySpec = Option.empty[ClusterBySpec] partitions.map { case IdentityTransform(FieldReference(Seq(col))) => identityCols += col case BucketTransform(numBuckets, bucketCols, sortCols) => bucketSpec = Some(BucketSpec( numBuckets, bucketCols.map(_.fieldNames.head), sortCols.map(_.fieldNames.head))) case TempClusterByTransform(columnNames) => if (clusterBySpec.nonEmpty) { // Parser guarantees that it only passes down one TempClusterByTransform. throw SparkException.internalError("Cannot have multiple cluster by transforms.") } clusterBySpec = Some(ClusterBySpec(columnNames)) case transform => throw DeltaErrors.operationNotSupportedException(s"Partitioning by expressions") } // Parser guarantees that partition and cluster by can't both exist. assert(!(identityCols.toSeq.nonEmpty && clusterBySpec.nonEmpty)) // Parser guarantees that bucketing and cluster by can't both exist. assert(!(bucketSpec.nonEmpty && clusterBySpec.nonEmpty)) (identityCols.toSeq, bucketSpec, clusterBySpec) } /** Performs checks on the parameters provided for table creation for a Delta table. */ def verifyTableAndSolidify( tableDesc: CatalogTable, query: Option[LogicalPlan], maybeClusterBySpec: Option[ClusterBySpec] = None): CatalogTable = { if (tableDesc.bucketSpec.isDefined) { throw DeltaErrors.operationNotSupportedException("Bucketing", tableDesc.identifier) } val schema = query.map { plan => assert(tableDesc.schema.isEmpty, "Can't specify table schema in CTAS.") plan.schema.asNullable }.getOrElse(tableDesc.schema) PartitioningUtils.validatePartitionColumn( schema, tableDesc.partitionColumnNames, caseSensitive = false) // Delta is case insensitive var validatedConfigurations = DeltaConfigs.validateConfigurations(tableDesc.properties) ClusteredTableUtils.validateExistingTableFeatureProperties(validatedConfigurations) // Add needed configs for Clustered table. Note that [[PROP_CLUSTERING_COLUMNS]] can only // be added after [[DeltaConfigs.validateConfigurations]] to avoid non-user configurable check // failure. if (maybeClusterBySpec.nonEmpty) { validatedConfigurations = validatedConfigurations ++ ClusteredTableUtils.getClusteringColumnsAsProperty(maybeClusterBySpec) ++ ClusteredTableUtils.getTableFeatureProperties(validatedConfigurations) } val db = tableDesc.identifier.database.getOrElse(catalog.getCurrentDatabase) val tableIdentWithDB = tableDesc.identifier.copy(database = Some(db)) tableDesc.copy( identifier = tableIdentWithDB, schema = schema, properties = validatedConfigurations) } /** * Checks if a Delta table already exists for the provided identifier. * * This first goes through the legacy V1 SessionCatalog/HMS lookup using [[TableIdentifier]]. * For operations that target an existing Unity Catalog table, it then falls back to the * delegated V2 catalog plugin lookup (for example Unity Catalog) when the V1 lookup does not * surface the table entry. */ def getExistingTableIfExists( table: TableIdentifier, identOpt: Option[Identifier], operation: TableCreationModes.CreationMode) : Option[CatalogTable] = { // If this is a path identifier, we cannot return an existing CatalogTable. The Create command // will check the file system itself if (isPathIdentifier(table)) return None val tableExists = catalog.tableExists(table) if (tableExists) { val oldTable = catalog.getTableMetadata(table) if (oldTable.tableType == CatalogTableType.VIEW) { throw DeltaErrors.cannotWriteIntoView(table) } if (!DeltaSourceUtils.isDeltaTable(oldTable.provider)) { throw DeltaErrors.notADeltaTable(table.table) } Some(oldTable) } else if (operation != TableCreationModes.Create) { identOpt match { case Some(ident) => getExistingTableFromDelegatedCatalog(ident) case None => logDebug(log"Delegated catalog lookup skipped because no V2 identifier was provided " + log"for ${MDC(DeltaLogKeys.TABLE_NAME, table)} during ${MDC(DeltaLogKeys.OPERATION, operation.toString)}.") None } } else { None } } /** * Returns an existing Delta table by querying the delegated catalog for Unity Catalog paths * where the V1 SessionCatalog lookup does not surface the existing table entry. * * [[getExistingTableIfExists]] first checks the V1 catalog using a [[TableIdentifier]]. For * Unity Catalog, some staged create/replace paths need a delegated V2 catalog lookup on the * original [[Identifier]] to recover the existing table metadata. * * Even though this goes through the delegated V2 catalog plugin, Spark can still surface the * result as a [[V1Table]] wrapper when the delegated catalog is exposing V1-backed table * metadata. In that case we unwrap the embedded [[CatalogTable]] and continue on the V1 Delta * path. */ private def getExistingTableFromDelegatedCatalog(ident: Identifier): Option[CatalogTable] = { if (isUnityCatalog) { try { super.loadTable(ident) match { case v1: V1Table if DeltaTableUtils.isDeltaTable(v1.catalogTable) => Some(v1.catalogTable) case _ => logDebug(log"Delegated catalog lookup for ${MDC(DeltaLogKeys.TABLE_NAME, ident)} " + log"did not return a Delta table.") None } } catch { case _: NoSuchTableException => logDebug(log"Delegated catalog lookup did not find an existing table for " + log"${MDC(DeltaLogKeys.TABLE_NAME, ident)}.") None } } else { None } } private def getTablePropsAndWriteOptions(properties: util.Map[String, String]) : (util.Map[String, String], Map[String, String]) = { val props = new util.HashMap[String, String]() // Options passed in through the SQL API will show up both with an "option." prefix and // without in Spark 3.1, so we need to remove those from the properties val optionsThroughProperties = properties.asScala.collect { case (k, _) if k.startsWith(TableCatalog.OPTION_PREFIX) => k.stripPrefix(TableCatalog.OPTION_PREFIX) }.toSet val writeOptions = new util.HashMap[String, String]() properties.asScala.foreach { case (k, v) => if (!k.startsWith(TableCatalog.OPTION_PREFIX) && !optionsThroughProperties.contains(k)) { // Add to properties props.put(k, v) } else if (optionsThroughProperties.contains(k)) { writeOptions.put(k, v) } } (props, writeOptions.asScala.toMap) } private def expandTableProps( props: util.Map[String, String], options: Map[String, String], conf: SQLConf): Unit = { if (conf.getConf(DeltaSQLConf.DELTA_LEGACY_STORE_WRITER_OPTIONS_AS_PROPS)) { // Legacy behavior options.foreach { case (k, v) => props.put(k, v) } } else { options.foreach { case (k, v) => // Continue putting in Delta prefixed options to avoid breaking workloads if (k.toLowerCase(Locale.ROOT).startsWith("delta.")) { props.put(k, v) } } } } /** * Normalizes the deprecated UC table ID property key to the canonical key. * * This is temporary compatibility for callers that still send the old key during the rename * transition. If both keys are present, we drop the old one and keep the canonical key only. * TODO(issue #6296): remove once all callers stop sending the deprecated key. */ private def translateUCTableIdProperty(props: util.Map[String, String]): Unit = { val oldTableIdProperty = Option(props.remove(UC_TABLE_ID_KEY_OLD)) oldTableIdProperty.foreach(props.putIfAbsent(UC_TABLE_ID_KEY, _)) } /** * A staged delta table, which creates a HiveMetaStore entry and appends data if this was a * CTAS/RTAS command. We have a ugly way of using this API right now, but it's the best way to * maintain old behavior compatibility between Databricks Runtime and OSS Delta Lake. */ private class StagedDeltaTableV2( ident: Identifier, override val schema: StructType, val partitions: Array[Transform], override val properties: util.Map[String, String], operation: TableCreationModes.CreationMode ) extends StagedTable with SupportsWrite { private var asSelectQuery: Option[DataFrame] = None private var writeOptions: Map[String, String] = Map.empty override def partitioning(): Array[Transform] = partitions override def commitStagedChanges(): Unit = recordFrameProfile( "DeltaCatalog", "commitStagedChanges") { val conf = spark.sessionState.conf val (props, sqlWriteOptions) = getTablePropsAndWriteOptions(properties) if (writeOptions.isEmpty && sqlWriteOptions.nonEmpty) { writeOptions = sqlWriteOptions } expandTableProps(props, writeOptions, conf) if (isUnityCatalog) { // Unity Catalog callers may still send the deprecated `ucTableId` property key. // Normalize it here to the canonical `io.unitycatalog.tableId` key before create/replace. translateUCTableIdProperty(props) } createDeltaTable( ident, schema, partitions, props, writeOptions, asSelectQuery, operation ) } override def name(): String = ident.name() override def abortStagedChanges(): Unit = {} override def capabilities(): util.Set[TableCapability] = { Set(V1_BATCH_WRITE, TRUNCATE).asJava } override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = { writeOptions = info.options.asCaseSensitiveMap().asScala.toMap new DeltaV1WriteBuilder } /* * WriteBuilder for creating a Delta table. */ private class DeltaV1WriteBuilder extends WriteBuilder with SupportsTruncate { override def truncate(): this.type = this override def build(): V1Write = new V1Write { override def toInsertableRelation(): InsertableRelation = { new InsertableRelation { override def insert(data: DataFrame, overwrite: Boolean): Unit = { asSelectQuery = Option(data) } } } } } } override def alterTable(ident: Identifier, changes: TableChange*): Table = recordFrameProfile( "DeltaCatalog", "alterTable") { // We group the table changes by their type, since Delta applies each in a separate action. // We also must define an artificial type for SetLocation, since data source V2 considers // location just another property but it's special in catalog tables. class SetLocation {} val grouped = ListMap(changes.groupBy { case s: SetProperty if s.property() == "location" => classOf[SetLocation] case c => c.getClass }.toSeq.sortBy { // force SetProperty first to handle if other TableChange requires table feature enabled case (cls, _) if cls == classOf[SetProperty] => 0 case _ => 1 }: _*) // Determines whether this DDL SET or UNSET the table redirect property. If it is, the table // redirect feature should be disabled to ensure the DDL can be applied onto the source or // destination table properly. val isUpdateTableRedirectDDL = grouped.map { case (t, s: Seq[RemoveProperty]) if t == classOf[RemoveProperty] => s.map { prop => prop.property() }.exists(RedirectFeature.isRedirectProperty) case (t, s: Seq[SetProperty]) if t == classOf[SetProperty] => RedirectFeature.hasRedirectConfig(s.map(prop => prop.property() -> prop.value()).toMap) case (_, _) => false }.toSeq.exists(a => a) RedirectFeature.withUpdateTableRedirectDDL(isUpdateTableRedirectDDL) { val table = loadTable(ident) match { case deltaTable: DeltaTableV2 => deltaTable case _ if changes.exists(_.isInstanceOf[ClusterBy]) => throw DeltaErrors.alterClusterByNotOnDeltaTableException() case _ if changes.exists(_.isInstanceOf[SyncIdentity]) => throw DeltaErrors.identityColumnAlterNonDeltaFormatError() case _ => return super.alterTable(ident, changes: _*) } val columnUpdates = new mutable.HashMap[Seq[String], DeltaChangeColumnSpec]() val isReplaceColumnsCommand = grouped.get(classOf[DeleteColumn]) match { case Some(deletes) if grouped.contains(classOf[AddColumn]) => // Convert to Seq so that contains method works val deleteSet = deletes.asInstanceOf[Seq[DeleteColumn]].map(_.fieldNames().toSeq).toSet // Ensure that all the table top level columns are being deleted table.schema().fieldNames.forall(f => deleteSet.contains(Seq(f))) case _ => false } if (isReplaceColumnsCommand && spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_REPLACE_COLUMNS_SAFE)) { // The new schema is essentially the AddColumn operators val tableToUpdate = table val colsToAdd = grouped(classOf[AddColumn]).asInstanceOf[Seq[AddColumn]] val structFields = colsToAdd.map { col => assert( col.fieldNames().length == 1, "We don't expect replace to provide nested column adds") var field = StructField(col.fieldNames().head, col.dataType, col.isNullable) Option(col.comment()).foreach { comment => field = field.withComment(comment) } Option(col.defaultValue()).foreach { defValue => field = field.withCurrentDefaultValue(defValue.getSql) } field } AlterTableReplaceColumnsDeltaCommand(tableToUpdate, structFields).run(spark) return loadTable(ident) } grouped.foreach { case (t, newColumns) if t == classOf[AddColumn] => val tableToUpdate = table AlterTableAddColumnsDeltaCommand( tableToUpdate, newColumns.asInstanceOf[Seq[AddColumn]].map { col => // Convert V2 `AddColumn` to V1 `QualifiedColType` as `AlterTableAddColumnsDeltaCommand` // is a V1 command. val name = col.fieldNames() val path = if (name.length > 1) Some(UnresolvedFieldName(name.init)) else None QualifiedColType( path, name.last, col.dataType(), col.isNullable, Option(col.comment()), Option(col.position()).map(UnresolvedFieldPosition), QualifiedColTypeShims.getDefaultValueArgFromAddColumn(col) ) }).run(spark) case (t, deleteColumns) if t == classOf[DeleteColumn] => AlterTableDropColumnsDeltaCommand( table, deleteColumns.asInstanceOf[Seq[DeleteColumn]].map(_.fieldNames().toSeq)).run(spark) case (t, newProperties) if t == classOf[SetProperty] => AlterTableSetPropertiesDeltaCommand( table, DeltaConfigs.validateConfigurations( newProperties.asInstanceOf[Seq[SetProperty]].map { prop => prop.property() -> prop.value() }.toMap) ).run(spark) case (t, oldProperties) if t == classOf[RemoveProperty] => AlterTableUnsetPropertiesDeltaCommand( table, oldProperties.asInstanceOf[Seq[RemoveProperty]].map(_.property()), // Data source V2 REMOVE PROPERTY is always IF EXISTS. ifExists = true).run(spark) case (t, columnChanges) if classOf[ColumnChange].isAssignableFrom(t) => // TODO: Theoretically we should be able to fetch the snapshot from a txn. val schema = table.initialSnapshot.schema def getColumn(fieldNames: Seq[String]) : DeltaChangeColumnSpec = { columnUpdates.getOrElseUpdate(fieldNames, { val colName = UnresolvedAttribute(fieldNames).name val fieldOpt = schema.findNestedField(fieldNames, includeCollections = true, spark.sessionState.conf.resolver) .map(_._2) val field = fieldOpt.getOrElse { throw DeltaErrors.nonExistentColumnInSchema(colName, schema.treeString) } DeltaChangeColumnSpec( fieldNames.init, fieldNames.last, field, colPosition = None, syncIdentity = false) }) } // Any ColumnChange not explicitly on the allowlist is blocked from making changes on // Identity Columns val disallowedColumnChangesOnIdentityColumns = columnChanges.filterNot { case _: UpdateColumnComment | _: UpdateColumnPosition | _: RenameColumn | _: SyncIdentity => true case _ => false } disallowedColumnChangesOnIdentityColumns.foreach { case change: ColumnChange => val field = change.fieldNames() val spec = getColumn(field) if (ColumnWithDefaultExprUtils.isIdentityColumn(spec.newColumn)) { throw DeltaErrors.identityColumnAlterColumnNotSupported() } } columnChanges.foreach { case comment: UpdateColumnComment => val field = comment.fieldNames() val spec = getColumn(field) columnUpdates(field) = spec.copy( newColumn = spec.newColumn.withComment(comment.newComment())) case dataType: UpdateColumnType => val field = dataType.fieldNames() val spec = getColumn(field) val newField = SchemaUtils.setFieldDataTypeCharVarcharSafe( spec.newColumn, dataType.newDataType()) columnUpdates(field) = spec.copy(newColumn = newField) case position: UpdateColumnPosition => val field = position.fieldNames() val spec = getColumn(field) columnUpdates(field) = spec.copy(colPosition = Option(position.position())) case nullability: UpdateColumnNullability => val field = nullability.fieldNames() val spec = getColumn(field) columnUpdates(field) = spec.copy( newColumn = spec.newColumn.copy(nullable = nullability.nullable())) case rename: RenameColumn => val field = rename.fieldNames() val spec = getColumn(field) columnUpdates(field) = spec.copy( newColumn = spec.newColumn.copy(name = rename.newName())) case sync: SyncIdentity => val field = sync.fieldNames val spec = getColumn(field).copy(syncIdentity = true) columnUpdates(field) = spec if (!ColumnWithDefaultExprUtils.isIdentityColumn(spec.newColumn)) { throw DeltaErrors.identityColumnAlterNonIdentityColumnError() } // If the IDENTITY column does not allow explicit insert, high water mark should // always be sync'ed and this is a no-op. // TODO: This is redundant at the moment because columnUpdates is always set above. // The original intention was to avoid running sync identity when the column does not // allow explicit insert, so columnUpdates should only be set here, but doing so would // fail a test related to DELTA_IDENTITY_ALLOW_SYNC_IDENTITY_TO_LOWER_HIGH_WATER_MARK. // This should be fixed in the future. if (IdentityColumn.allowExplicitInsert(spec.newColumn)) { columnUpdates(field) = spec } case updateDefault: UpdateColumnDefaultValue => val field = updateDefault.fieldNames() val spec = getColumn(field) val updatedField = updateDefault.newDefaultValue() match { case "" => spec.newColumn.clearCurrentDefaultValue() case newDefault => spec.newColumn.withCurrentDefaultValue(newDefault) } columnUpdates(field) = spec.copy(newColumn = updatedField) case other => throw DeltaErrors.unrecognizedColumnChange(s"${other.getClass}") } case (t, locations) if t == classOf[SetLocation] => if (locations.size != 1) { throw DeltaErrors.cannotSetLocationMultipleTimes( locations.asInstanceOf[Seq[SetProperty]].map(_.value())) } if (table.tableIdentifier.isEmpty) { throw DeltaErrors.setLocationNotSupportedOnPathIdentifiers() } AlterTableSetLocationDeltaCommand( table, locations.head.asInstanceOf[SetProperty].value()).run(spark) case (t, constraints) if t == classOf[AddConstraint] => constraints.foreach { constraint => val c = constraint.asInstanceOf[AddConstraint] AlterTableAddConstraintDeltaCommand(table, c.constraintName, c.expr).run(spark) } case (t, constraints) if t == classOf[DropConstraint] => constraints.foreach { constraint => val c = constraint.asInstanceOf[DropConstraint] AlterTableDropConstraintDeltaCommand(table, c.constraintName, c.ifExists).run(spark) } case (t, dropFeature) if t == classOf[DropFeature] => // Only single feature removal is supported. val dropFeatureTableChange = dropFeature.head.asInstanceOf[DropFeature] val featureName = dropFeatureTableChange.featureName val truncateHistory = dropFeatureTableChange.truncateHistory AlterTableDropFeatureDeltaCommand( table, featureName, truncateHistory = truncateHistory).run(spark) case (t, clusterBy) if t == classOf[ClusterBy] => clusterBy.asInstanceOf[Seq[ClusterBy]].foreach { c => if (c.clusteringColumns.nonEmpty) { val clusterBySpec = ClusterBySpec(c.clusteringColumns.toSeq) validateClusterBySpec(Some(clusterBySpec), table.schema()) } if (table.initialSnapshot.metadata.partitionColumns.nonEmpty) { throw DeltaErrors.alterTableClusterByOnPartitionedTableException() } AlterTableClusterByDeltaCommand( table, c.clusteringColumns.map(_.fieldNames().toSeq).toSeq).run(spark) } } if (columnUpdates.nonEmpty) { AlterTableChangeColumnDeltaCommand(table, columnUpdates.values.toSeq).run(spark) } loadTable(ident) } } // We want our catalog to handle Delta, therefore for other data sources that want to be // created, we just have this wrapper StagedTable to only drop the table if the commit fails. private case class BestEffortStagedTable( ident: Identifier, table: Table, catalog: TableCatalog) extends StagedTable with SupportsWrite { override def abortStagedChanges(): Unit = catalog.dropTable(ident) override def commitStagedChanges(): Unit = {} // Pass through override def name(): String = table.name() override def schema(): StructType = table.schema() override def partitioning(): Array[Transform] = table.partitioning() override def capabilities(): util.Set[TableCapability] = table.capabilities() override def properties(): util.Map[String, String] = table.properties() override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = table match { case supportsWrite: SupportsWrite => supportsWrite.newWriteBuilder(info) case _ => throw DeltaErrors.unsupportedWriteStagedTable(name) } } } /** * A trait for handling table access through delta.`/some/path`. This is a stop-gap solution * until PathIdentifiers are implemented in Apache Spark. */ trait SupportsPathIdentifier extends TableCatalog { self: AbstractDeltaCatalog => private def supportSQLOnFile: Boolean = spark.sessionState.conf.runSQLonFile protected lazy val catalog: SessionCatalog = spark.sessionState.catalog private def hasDeltaNamespace(ident: Identifier): Boolean = { ident.namespace().length == 1 && DeltaSourceUtils.isDeltaDataSourceName(ident.namespace().head) } private def hasIcebergNamespace(ident: Identifier): Boolean = { ident.namespace().length == 1 && ident.namespace().head.equalsIgnoreCase("iceberg") } protected def isIcebergPathIdentifier(ident: Identifier): Boolean = { hasIcebergNamespace(ident) && new Path(ident.name()).isAbsolute } protected def newIcebergPathTable(ident: Identifier): IcebergTablePlaceHolder = { IcebergTablePlaceHolder(TableIdentifier(ident.name(), Some("iceberg"))) } protected def isPathIdentifier(ident: Identifier): Boolean = { // Should be a simple check of a special PathIdentifier class in the future try { supportSQLOnFile && hasDeltaNamespace(ident) && new Path(ident.name()).isAbsolute } catch { case _: IllegalArgumentException => false } } protected def isPathIdentifier(table: CatalogTable): Boolean = { isPathIdentifier(table.identifier) } protected def isPathIdentifier(tableIdentifier: TableIdentifier) : Boolean = { isPathIdentifier(Identifier.of(tableIdentifier.database.toArray, tableIdentifier.table)) } override def tableExists(ident: Identifier): Boolean = recordFrameProfile( "DeltaCatalog", "tableExists") { if (isPathIdentifier(ident)) { val path = new Path(ident.name()) // scalastyle:off deltahadoopconfiguration val fs = path.getFileSystem(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration fs.exists(path) && fs.listStatus(path).nonEmpty } else { super.tableExists(ident) } } } object BucketTransform { def unapply(transform: Transform): Option[(Int, Seq[NamedReference], Seq[NamedReference])] = { val arguments = transform.arguments() if (transform.name() == "sorted_bucket") { var posOfLit: Int = -1 var numOfBucket: Int = -1 arguments.zipWithIndex.foreach { case (literal: Literal[_], i) if literal.dataType() == IntegerType => numOfBucket = literal.value().asInstanceOf[Integer] posOfLit = i case _ => } Some(numOfBucket, arguments.take(posOfLit).map(_.asInstanceOf[NamedReference]), arguments.drop(posOfLit + 1).map(_.asInstanceOf[NamedReference])) } else if (transform.name() == "bucket") { val numOfBucket = arguments(0) match { case literal: Literal[_] if literal.dataType() == IntegerType => literal.value().asInstanceOf[Integer] case _ => throw new IllegalStateException("invalid bucket transform") } Some(numOfBucket, arguments.drop(1).map(_.asInstanceOf[NamedReference]), Seq.empty[FieldReference]) } else { None } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/catalog/CatalogResolver.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.catalog import org.apache.spark.sql.delta.{DeltaErrors, DeltaTableIdentifier, DeltaTableUtils} import org.apache.hadoop.fs.Path import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.connector.catalog.{CatalogNotFoundException, CatalogPlugin, Identifier, Table, V1Table} import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.CatalogHelper import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper /** * Helper object for resolving Delta tables using a *non-session* catalog. */ object CatalogResolver { def getDeltaTableFromCatalog( spark: SparkSession, catalog: CatalogPlugin, ident: Identifier): DeltaTableV2 = { catalog.asTableCatalog.loadTable(ident) match { case v2: DeltaTableV2 => v2 case v1: V1Table if DeltaTableUtils.isDeltaTable(v1.v1Table) => DeltaTableV2( spark, path = new Path(v1.v1Table.location), catalogTable = Some(v1.v1Table), tableIdentifier = Some(ident.toString) ) case table => throw DeltaErrors.notADeltaTableException( DeltaTableIdentifier(table = Some(TableIdentifier(table.name())))) } } /** Returns a [[(CatalogPlugin, Identifier)]] if a catalog exists with the input name, otherwise throws a [[CatalogNotFoundException]] */ def getCatalogPluginAndIdentifier( spark: SparkSession, catalog: String, ident: Seq[String]): (CatalogPlugin, Identifier) = { (spark.sessionState.catalogManager.catalog(catalog), ident.asIdentifier) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/catalog/DeltaTableV2.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.catalog import java.{util => ju} // scalastyle:off import.ordering.noEmptyLine import scala.collection.JavaConverters._ import scala.collection.mutable import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.DataFrameUtils import org.apache.spark.sql.delta.skipping.clustering.{ClusteredTableUtils, ClusteringColumnInfo} import org.apache.spark.sql.delta.skipping.clustering.temp.ClusterBySpec import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.commands.WriteIntoDelta import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.{DeltaDataSource, DeltaSourceUtils} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.sources.DeltaSQLConf.ENABLE_TABLE_REDIRECT_FEATURE import org.apache.hadoop.fs.Path import org.apache.spark.sql.{DataFrame, Dataset, SaveMode, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.{ResolvedTable, UnresolvedTable} import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType, CatalogUtils} import org.apache.spark.sql.catalyst.plans.logical.{AnalysisHelper, LogicalPlan, SubqueryAlias} import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes import org.apache.spark.sql.connector.catalog.{SupportsWrite, Table, TableCapability, TableCatalog, V2TableWithV1Fallback} import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ import org.apache.spark.sql.connector.catalog.TableCapability._ import org.apache.spark.sql.connector.catalog.V1Table import org.apache.spark.sql.connector.expressions._ import org.apache.spark.sql.connector.write.{LogicalWriteInfo, SupportsDynamicOverwrite, SupportsOverwrite, SupportsTruncate, V1Write, WriteBuilder} import org.apache.spark.sql.errors.QueryCompilationErrors import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.sources.{BaseRelation, Filter, InsertableRelation} import org.apache.spark.sql.types.StructType import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.util.{Clock, SystemClock} /** * The data source V2 representation of a Delta table that exists. * * @param path The path to the table * @param tableIdentifier The table identifier for this table */ class DeltaTableV2 private( val spark: SparkSession, val path: Path, val catalogTable: Option[CatalogTable], val tableIdentifier: Option[String], val timeTravelOpt: Option[DeltaTimeTravelSpec], val options: Map[String, String]) extends Table with SupportsWrite with V2TableWithV1Fallback with DeltaLogging { case class PathInfo( rootPath: Path, private[delta] var partitionFilters: Seq[(String, String)], private[delta] var timeTravelByPath: Option[DeltaTimeTravelSpec] ) private lazy val pathInfo: PathInfo = { if (catalogTable.isDefined) { // Fast path for reducing path munging overhead PathInfo(new Path(catalogTable.get.location), Seq.empty, None) } else { val (rootPath, filters, timeTravel) = DeltaDataSource.parsePathIdentifier(spark, path.toString, options) PathInfo(rootPath, filters, timeTravel) } } private def rootPath = pathInfo.rootPath private def partitionFilters = pathInfo.partitionFilters private def timeTravelByPath = pathInfo.timeTravelByPath def hasPartitionFilters: Boolean = partitionFilters.nonEmpty // This MUST be initialized before the deltaLog object is created, in order to accurately // bound the creation time of the table. private val creationTimeMs = { System.currentTimeMillis() } // The loading of the DeltaLog is lazy in order to reduce the amount of FileSystem calls, // in cases where we will fallback to the V1 behavior. lazy val deltaLog: DeltaLog = { DeltaTableV2.withEnrichedUnsupportedTableException(catalogTable, tableIdentifier) { // Ideally the table storage properties should always be the same as the options load from // the Delta log, as Delta CREATE TABLE command guarantees it. However, custom catalogs such // as Unity Catalog may add more table storage properties on the fly. We should respect it // and merge the table storage properties and Delta options. val dataSourceOptions = if (catalogTable.isDefined) { // To be safe, here we only extract file system options from table storage properties and // the original `options` has higher priority than the table storage properties. val fileSystemOptions = catalogTable.get.storage.properties.filter { case (k, _) => DeltaTableUtils.validDeltaTableHadoopPrefixes.exists(k.startsWith) } fileSystemOptions ++ options } else { options } DeltaLog.forTable( spark, rootPath, dataSourceOptions, catalogTable) } } /** * Updates the delta log for this table and returns a new snapshot */ def update(): Snapshot = deltaLog.update(catalogTableOpt = catalogTable) def update(checkIfUpdatedSinceTs: Option[Long]): Snapshot = deltaLog.update(checkIfUpdatedSinceTs = checkIfUpdatedSinceTs, catalogTableOpt = catalogTable) /** * Gets the snapshot at the given version of this table */ def getSnapshotAt(version: Long): Snapshot = deltaLog.getSnapshotAt(version, catalogTableOpt = catalogTable) def getTableIdentifierIfExists: Option[TableIdentifier] = tableIdentifier.map { tableName => spark.sessionState.sqlParser.parseMultipartIdentifier(tableName).asTableIdentifier } override def name(): String = catalogTable.map(_.identifier.unquotedString) .orElse(tableIdentifier) .getOrElse(s"delta.`${deltaLog.dataPath}`") private lazy val timeTravelSpec: Option[DeltaTimeTravelSpec] = { if (timeTravelOpt.isDefined && timeTravelByPath.isDefined) { throw DeltaErrors.multipleTimeTravelSyntaxUsed } timeTravelOpt.orElse(timeTravelByPath) } private lazy val caseInsensitiveOptions = new CaseInsensitiveStringMap(options.asJava) /** * The snapshot initially associated with this table. It is captured on first access, usually (but * not always) shortly after the table was first created, and is immutable once captured. * * WARNING: This snapshot could be arbitrarily stale for long-lived [[DeltaTableV2]] instances, * such as the ones [[DeltaTable]] uses internally. Callers who cannot tolerate this potential * staleness should use [[getFreshSnapshot]] instead. * * WARNING: Because the snapshot is captured lazily, callers should explicitly access the snapshot * if they want to be certain it has been captured. */ lazy val initialSnapshot: Snapshot = DeltaTableV2.withEnrichedUnsupportedTableException( catalogTable, tableIdentifier) { timeTravelSpec.map { spec => // By default, block using CDF + time-travel if (CDCReader.isCDCRead(caseInsensitiveOptions) && !spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_CDF_ALLOW_TIME_TRAVEL_OPTIONS)) { throw DeltaErrors.timeTravelNotSupportedException } val (version, accessType) = DeltaTableUtils.resolveTimeTravelVersion( spark.sessionState.conf, deltaLog, catalogTable, spec) val source = spec.creationSource.getOrElse("unknown") recordDeltaEvent(deltaLog, s"delta.timeTravel.$source", data = Map( // Log the cached version of the table on the cluster "tableVersion" -> deltaLog.unsafeVolatileSnapshot.version, "queriedVersion" -> version, "accessType" -> accessType )) deltaLog.getSnapshotAt( version, catalogTableOpt = catalogTable, enforceTimeTravelWithinDeletedFileRetention = spec.enforceRetention) }.getOrElse( deltaLog.update( stalenessAcceptable = true, checkIfUpdatedSinceTs = Some(creationTimeMs), catalogTableOpt = catalogTable ) ) } // We get the cdcRelation ahead of time if this is a CDC read to be able to return the correct // schema. The schema for CDC reads are currently convoluted due to column mapping behavior private lazy val cdcRelation: Option[BaseRelation] = { if (CDCReader.isCDCRead(caseInsensitiveOptions)) { recordDeltaEvent(deltaLog, "delta.cdf.read", data = caseInsensitiveOptions.asCaseSensitiveMap()) Some(CDCReader.getCDCRelation( spark, initialSnapshot, catalogTable, timeTravelSpec.nonEmpty, spark.sessionState.conf, caseInsensitiveOptions)) } else { None } } private lazy val tableSchema: StructType = { val baseSchema = cdcRelation.map(_.schema).getOrElse(initialSnapshot.schema) DeltaTableUtils.removeInternalDeltaMetadata( spark, DeltaTableUtils.removeInternalWriterMetadata(spark, baseSchema) ) } override def schema(): StructType = tableSchema override def partitioning(): Array[Transform] = { initialSnapshot.metadata.partitionColumns.map { col => new IdentityTransform(new FieldReference(Seq(col))) }.toArray } override def properties(): ju.Map[String, String] = { val base = initialSnapshot.getProperties base.put(TableCatalog.PROP_PROVIDER, "delta") base.put(TableCatalog.PROP_LOCATION, CatalogUtils.URIToString(path.toUri)) catalogTable.foreach { table => if (table.owner != null && table.owner.nonEmpty) { base.put(TableCatalog.PROP_OWNER, table.owner) } v1Table.storage.properties.foreach { case (key, value) => if (!DeltaTableV2.HIDDEN_STORAGE_PROPERTY_PREFIXES.exists(key.startsWith)) { base.put(TableCatalog.OPTION_PREFIX + key, value) } } if (v1Table.tableType == CatalogTableType.EXTERNAL) { base.put(TableCatalog.PROP_EXTERNAL, "true") } } // Don't use [[PROP_CLUSTERING_COLUMNS]] from CatalogTable because it may be stale. // Since ALTER TABLE updates it using an async post-commit hook. clusterBySpec.foreach { clusterBy => ClusterBySpec.toProperties(clusterBy).foreach { case (key, value) => base.put(key, value) } } Option(initialSnapshot.metadata.description).foreach(base.put(TableCatalog.PROP_COMMENT, _)) base.asJava } override def capabilities(): ju.Set[TableCapability] = Set( ACCEPT_ANY_SCHEMA, BATCH_READ, V1_BATCH_WRITE, OVERWRITE_BY_FILTER, TRUNCATE, OVERWRITE_DYNAMIC ).asJava def tableExists: Boolean = deltaLog.tableExists override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = { new WriteIntoDeltaBuilder( this, info.options, spark.sessionState.conf.useNullsForMissingDefaultColumnValues) } /** * Starts a transaction for this table, using the snapshot captured during table resolution. * * WARNING: Caller is responsible to ensure that table resolution was recent (e.g. if working with * [[DataFrame]] or [[DeltaTable]] API, where the table could have been resolved long ago). */ def startTransactionWithInitialSnapshot(): OptimisticTransaction = startTransaction(Some(initialSnapshot)) /** * Starts a transaction for this table, using Some provided snapshot, or a fresh snapshot if None * was provided. */ def startTransaction(snapshotOpt: Option[Snapshot] = None): OptimisticTransaction = { deltaLog.startTransaction(catalogTable, snapshotOpt) } /** * Creates a checkpoint for this table. * @param snapshotToCheckpoint The snapshot to checkpoint. */ def checkpoint(snapshotToCheckpoint: Snapshot): Unit = { deltaLog.checkpoint(snapshotToCheckpoint, catalogTable) } /** * Creates a V1 BaseRelation from this Table to allow read APIs to go through V1 DataSource code * paths. */ lazy val toBaseRelation: BaseRelation = { // force update() if necessary in DataFrameReader.load code initialSnapshot if (!tableExists) { // special error handling for path based tables if (catalogTable.isEmpty && !rootPath.getFileSystem(deltaLog.newDeltaHadoopConf()).exists(rootPath)) { throw QueryCompilationErrors.dataPathNotExistError(rootPath.toString) } val id = catalogTable.map(ct => DeltaTableIdentifier(table = Some(ct.identifier))) .getOrElse(DeltaTableIdentifier(path = Some(path.toString))) throw DeltaErrors.nonExistentDeltaTable(id) } val partitionPredicates = DeltaDataSource.verifyAndCreatePartitionFilters( path.toString, initialSnapshot, partitionFilters) cdcRelation.getOrElse { deltaLog.createRelation( partitionPredicates, Some(initialSnapshot), catalogTable, timeTravelSpec.isDefined) } } /** Creates a [[LogicalRelation]] that represents this table */ lazy val toLogicalRelation: LogicalRelation = { val relation = this.toBaseRelation LogicalRelation( relation, toAttributes(relation.schema), ttSafeCatalogTable, isStreaming = false, stream = None) } /** Creates a [[DataFrame]] that uses the requested spark session to read from this table */ def toDf(sparkSession: SparkSession): DataFrame = { val plan = catalogTable.foldLeft[LogicalPlan](toLogicalRelation) { (child, ct) => // Catalog based tables need a SubqueryAlias that carries their fully-qualified name SubqueryAlias(ct.identifier.nameParts, child) } DataFrameUtils.ofRows(sparkSession, plan) } /** Creates a [[DataFrame]] that reads from this table */ lazy val toDf: DataFrame = toDf(spark) /** * Check the passed in options and existing timeTravelOpt, set new time travel by options. */ def withOptions(newOptions: Map[String, String]): DeltaTableV2 = { val ttSpec = DeltaDataSource.getTimeTravelVersion(newOptions) // Spark 4.0 and 3.5 handle time travel options differently. // Validate that only one time travel spec is being used (timeTravelOpt, ttSpec) match { case (Some(currSpec), Some(newSpec)) if currSpec.version != newSpec.version || currSpec.getTimestampOpt(spark.sessionState.conf).map(_.getTime) != newSpec.getTimestampOpt(spark.sessionState.conf).map(_.getTime) => throw DeltaErrors.multipleTimeTravelSyntaxUsed case _ => } val caseInsensitiveNewOptions = new CaseInsensitiveStringMap(newOptions.asJava) if (timeTravelOpt.isEmpty && ttSpec.nonEmpty) { copy(timeTravelOpt = ttSpec) } else if (CDCReader.isCDCRead(caseInsensitiveNewOptions)) { checkCDCOptionsValidity(caseInsensitiveNewOptions) // Do not use statistics during CDF reads this.copy(catalogTable = catalogTable.map(_.copy(stats = None)), options = newOptions) } else { this } } private def checkCDCOptionsValidity(options: CaseInsensitiveStringMap): Unit = { // check if we have both version and timestamp parameters if (options.containsKey(DeltaDataSource.CDC_START_TIMESTAMP_KEY) && options.containsKey(DeltaDataSource.CDC_START_VERSION_KEY)) { throw DeltaErrors.multipleCDCBoundaryException("starting") } if (options.containsKey(DeltaDataSource.CDC_END_VERSION_KEY) && options.containsKey(DeltaDataSource.CDC_END_TIMESTAMP_KEY)) { throw DeltaErrors.multipleCDCBoundaryException("ending") } if (!options.containsKey(DeltaDataSource.CDC_START_VERSION_KEY) && !options.containsKey(DeltaDataSource.CDC_START_TIMESTAMP_KEY)) { throw DeltaErrors.noStartVersionForCDC() } } /** A "clean" version of the catalog table, safe for use with or without time travel. */ lazy val ttSafeCatalogTable: Option[CatalogTable] = catalogTable match { case Some(ct) if timeTravelSpec.isDefined => Some(ct.copy(stats = None)) case other => other } override def v1Table: CatalogTable = ttSafeCatalogTable.getOrElse { throw DeltaErrors.invalidV1TableCall("v1Table", "DeltaTableV2") } lazy val clusterBySpec: Option[ClusterBySpec] = { // Always get the clustering columns from metadata domain in delta log. if (ClusteredTableUtils.isSupported(initialSnapshot.protocol)) { val clusteringColumns = ClusteringColumnInfo.extractLogicalNames( initialSnapshot) Some(ClusterBySpec.fromColumnNames(clusteringColumns)) } else { None } } def copy( spark: SparkSession = this.spark, path: Path = this.path, catalogTable: Option[CatalogTable] = this.catalogTable, tableIdentifier: Option[String] = this.tableIdentifier, timeTravelOpt: Option[DeltaTimeTravelSpec] = this.timeTravelOpt, options: Map[String, String] = this.options ): DeltaTableV2 = { val deltaTableV2 = new DeltaTableV2(spark, path, catalogTable, tableIdentifier, timeTravelOpt, options) deltaTableV2.pathInfo.timeTravelByPath = timeTravelByPath deltaTableV2.pathInfo.partitionFilters = partitionFilters deltaTableV2 } override def toString: String = s"DeltaTableV2($spark,$path,$catalogTable,$tableIdentifier,$timeTravelOpt,$options)" } object DeltaTableV2 { /** * Storage property key prefixes that should be excluded from the user-visible V2 table * properties returned by [[DeltaTableV2.properties()]]. These are Hadoop filesystem * configuration options that may contain sensitive credentials (access keys, session * tokens, etc.) injected by catalogs at table-load time. */ private[delta] val HIDDEN_STORAGE_PROPERTY_PREFIXES: Seq[String] = Seq("fs.") def unapply(deltaTable: DeltaTableV2): Option[( SparkSession, Path, Option[CatalogTable], Option[String], Option[DeltaTimeTravelSpec], Map[String, String]) ] = { Some(( deltaTable.spark, deltaTable.path, deltaTable.catalogTable, deltaTable.tableIdentifier, deltaTable.timeTravelOpt, deltaTable.options) ) } def apply( spark: SparkSession, path: Path, catalogTable: Option[CatalogTable] = None, tableIdentifier: Option[String] = None, options: Map[String, String] = Map.empty[String, String], timeTravelOpt: Option[DeltaTimeTravelSpec] = None ): DeltaTableV2 = { val deltaTable = new DeltaTableV2( spark, path, catalogTable = catalogTable, tableIdentifier = tableIdentifier, timeTravelOpt = timeTravelOpt, options = options ) if (spark == null || spark.sessionState == null || !spark.sessionState.conf.getConf(ENABLE_TABLE_REDIRECT_FEATURE)) { return deltaTable } // This following code ensure passing the path and catalogTable of the redirected table object. // Note: the DeltaTableV2 can only be created using this method. AnalysisHelper.allowInvokingTransformsInAnalyzer { val deltaLog = deltaTable.deltaLog val rootDeltaLogPath = DeltaLog.logPathFor(deltaTable.rootPath.toString) val finalDeltaLogPath = DeltaLog.formalizeDeltaPath(spark, options, rootDeltaLogPath) if (finalDeltaLogPath == deltaLog.logPath) { // If there is no redirection, use existing delta table. deltaTable } else { // If there is redirection, use the catalogTable of deltaLog. val catalogTable = deltaLog.getInitialCatalogTable val newPath = new Path(deltaLog.dataPath.toUri) new DeltaTableV2( spark, path = newPath, catalogTable = catalogTable, tableIdentifier = catalogTable.map(_.identifier.identifier), timeTravelOpt = timeTravelOpt, options = options ) } } } /** Resolves a path into a DeltaTableV2, leveraging standard v2 table resolution. */ def apply(spark: SparkSession, tablePath: Path, options: Map[String, String], cmd: String) : DeltaTableV2 = { val unresolved = UnresolvedPathBasedDeltaTable(tablePath.toString, options, cmd) extractFrom((new DeltaAnalysis(spark))(unresolved), cmd) } /** Resolves a table identifier into a DeltaTableV2, leveraging standard v2 table resolution. */ def apply(spark: SparkSession, tableId: TableIdentifier, cmd: String): DeltaTableV2 = { val unresolved = UnresolvedTable(tableId.nameParts, cmd) extractFrom(spark.sessionState.analyzer.ResolveRelations(unresolved), cmd) } /** * Extracts the DeltaTableV2 from a resolved Delta table plan node, throwing "table not found" if * the node does not actually represent a resolved Delta table. */ def extractFrom(plan: LogicalPlan, cmd: String): DeltaTableV2 = maybeExtractFrom(plan).getOrElse(throw DeltaErrors.notADeltaTableException(cmd)) /** * Extracts the DeltaTableV2 from a resolved Delta table plan node, returning None if the node * does not actually represent a resolved Delta table. */ def maybeExtractFrom(plan: LogicalPlan): Option[DeltaTableV2] = plan match { case ResolvedTable(_, _, d: DeltaTableV2, _) => Some(d) case ResolvedTable(_, _, t: V1Table, _) if DeltaTableUtils.isDeltaTable(t.catalogTable) => Some(DeltaTableV2(SparkSession.active, new Path(t.v1Table.location), Some(t.v1Table))) case _ => None } /** * Creates a DeltaTableV2 instance with a custom DeltaLog object for testing purposes. This is * useful because the DeltaTableV2 constructor is private and cannot be called from * DeltaTestImplicit. */ def testOnlyApplyWithCustomDeltaLog( spark: SparkSession, path: Path, clock: Clock): DeltaTableV2 = { new DeltaTableV2( spark, path, catalogTable = None, tableIdentifier = None, timeTravelOpt = None, options = Map.empty ) { override lazy val deltaLog: DeltaLog = DeltaLog.forTable(spark, path, clock) } } /** * When Delta Log throws InvalidProtocolVersionException it doesn't know the table name and uses * the data path in the message, this wrapper throw a new InvalidProtocolVersionException with * table name and sets its Cause to the original InvalidProtocolVersionException. */ def withEnrichedUnsupportedTableException[T]( catalogTable: Option[CatalogTable], tableName: Option[String] = None)(thunk: => T): T = { lazy val tableNameToUse = catalogTable match { case Some(ct) => Some(ct.identifier.copy(catalog = None).unquotedString) case None => tableName } try thunk catch { case e: InvalidProtocolVersionException if tableNameToUse.exists(_ != e.tableNameOrPath) => throw e.copy(tableNameOrPath = tableNameToUse.get).initCause(e) case e: DeltaUnsupportedTableFeatureException if tableNameToUse.exists(_ != e.tableNameOrPath) => throw e.copy(tableNameOrPath = tableNameToUse.get).initCause(e) } } } private class WriteIntoDeltaBuilder( table: DeltaTableV2, writeOptions: CaseInsensitiveStringMap, nullAsDefault: Boolean) extends WriteBuilder with SupportsOverwrite with SupportsTruncate with SupportsDynamicOverwrite { private var forceOverwrite = false private val options = mutable.HashMap[String, String](writeOptions.asCaseSensitiveMap().asScala.toSeq: _*) override def truncate(): WriteIntoDeltaBuilder = { forceOverwrite = true this } override def overwrite(filters: Array[Filter]): WriteBuilder = { if (writeOptions.containsKey("replaceWhere")) { throw DeltaErrors.replaceWhereUsedInOverwrite() } options.put("replaceWhere", DeltaSourceUtils.translateFilters(filters).sql) forceOverwrite = true this } override def overwriteDynamicPartitions(): WriteBuilder = { options.put( DeltaOptions.PARTITION_OVERWRITE_MODE_OPTION, DeltaOptions.PARTITION_OVERWRITE_MODE_DYNAMIC) forceOverwrite = true this } override def build(): V1Write = new V1Write { override def toInsertableRelation(): InsertableRelation = { new InsertableRelation { override def insert(data: DataFrame, overwrite: Boolean): Unit = { val session = data.sparkSession // Normal table insertion should be the only place that can use null as the default // column value. We put a special option here so that `TransactionalWrite#writeFiles` // will recognize it and apply null-as-default. if (nullAsDefault) { options.put( ColumnWithDefaultExprUtils.USE_NULL_AS_DEFAULT_DELTA_OPTION, "true" ) } // TODO: Get the config from WriteIntoDelta's txn. WriteIntoDelta( table.deltaLog, if (forceOverwrite) SaveMode.Overwrite else SaveMode.Append, new DeltaOptions(options.toMap, session.sessionState.conf), Nil, table.deltaLog.unsafeVolatileSnapshot.metadata.configuration, data, table.catalogTable).run(session) // TODO: Push this to Apache Spark // Re-cache all cached plans(including this relation itself, if it's cached) that refer // to this data source relation. This is the behavior for InsertInto session.sharedState.cacheManager.recacheByPlan(session, table.toLogicalRelation) } } } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/catalog/IcebergTablePlaceHolder.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.catalog import scala.collection.JavaConverters._ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.connector.catalog.{Table, TableCapability} import org.apache.spark.sql.types.StructType /** A place holder used to resolve Iceberg table as a relation during analysis */ case class IcebergTablePlaceHolder(tableIdentifier: TableIdentifier) extends Table { override def name(): String = tableIdentifier.unquotedString override def schema(): StructType = new StructType() override def capabilities(): java.util.Set[TableCapability] = Set.empty[TableCapability].asJava } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/clustering/ClusteringMetadataDomain.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.clustering import org.apache.spark.sql.delta.skipping.clustering.ClusteringColumn import org.apache.spark.sql.delta.{JsonMetadataDomain, JsonMetadataDomainUtils} import org.apache.spark.sql.delta.actions.{Action, DomainMetadata} /** * Metadata domain for Clustered table which tracks clustering columns. */ case class ClusteringMetadataDomain(clusteringColumns: Seq[Seq[String]]) extends JsonMetadataDomain[ClusteringMetadataDomain] { override val domainName: String = ClusteringMetadataDomain.domainName } object ClusteringMetadataDomain extends JsonMetadataDomainUtils[ClusteringMetadataDomain] { override val domainName = "delta.clustering" // Extracts the ClusteringMetadataDomain and the removed field. def unapply(action: Action): Option[(ClusteringMetadataDomain, Boolean)] = action match { case d: DomainMetadata if d.domain == domainName => Some((fromJsonConfiguration(d), d.removed)) case _ => None } def fromClusteringColumns(clusteringColumns: Seq[ClusteringColumn]): ClusteringMetadataDomain = { ClusteringMetadataDomain(clusteringColumns.map(_.physicalName)) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/CloneTableBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands // scalastyle:off import.ordering.noEmptyLine import java.io.Closeable import java.util.UUID import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.skipping.clustering.ClusteredTableUtils import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CoordinatedCommitsUtils} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util._ import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.spark.internal.Logging import org.apache.spark.sql.{Dataset, Row, SparkSession} import org.apache.spark.sql.catalyst.SQLConfHelper import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.plans.logical.LeafCommand import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.types.StructType import org.apache.spark.util.{Clock, SerializableConfiguration} // scalastyle:on import.ordering.noEmptyLine /** * An interface of the source table to be cloned from. */ trait CloneSource extends Closeable { /** The format of the source table */ def format: String /** The source table's protocol */ def protocol: Protocol /** A system clock */ def clock: Clock /** The source table name */ def name: String /** The path of the source table */ def dataPath: Path /** The source table schema */ def schema: StructType /** The catalog table of the source table, if exists */ def catalogTable: Option[CatalogTable] /** The time travel spec of the source table, if exists */ def timeTravelOpt: Option[DeltaTimeTravelSpec] /** A snapshot of the source table, if exists */ def snapshot: Option[Snapshot] /** The metadata of the source table */ def metadata: Metadata /** All of the files present in the source table */ def allFiles: Dataset[AddFile] /** Total size of data files in bytes */ def sizeInBytes: Long /** Total number of data files */ def numOfFiles: Long /** Describe this clone source */ def description: String } // Clone source table formats object CloneSourceFormat { val DELTA = "Delta" val ICEBERG = "Iceberg" val PARQUET = "Parquet" val UNKNOWN = "Unknown" } trait CloneTableBaseUtils extends DeltaLogging { import CloneTableCommand._ /** Make a map of operation metrics for the executed command for DeltaLog commits */ protected def getOperationMetricsForDeltaLog( opMetrics: SnapshotOverwriteOperationMetrics): Map[String, Long] = { Map( SOURCE_TABLE_SIZE -> opMetrics.sourceSnapshotSizeInBytes, SOURCE_NUM_OF_FILES -> opMetrics.sourceSnapshotFileCount, NUM_REMOVED_FILES -> 0L, NUM_COPIED_FILES -> 0L, REMOVED_FILES_SIZE -> 0L, COPIED_FILES_SIZE -> 0L ) } /** * Make a map of operation metrics for the executed command for recording events. * Any command can extend to overwrite or add new metrics */ protected def getOperationMetricsForEventRecord( opMetrics: SnapshotOverwriteOperationMetrics): Map[String, Long] = getOperationMetricsForDeltaLog(opMetrics) /** Make a output Seq[Row] of metrics for the executed command */ protected def getOutputSeq(operationMetrics: Map[String, Long]): Seq[Row] protected def checkColumnMappingMode(beforeMetadata: Metadata, afterMetadata: Metadata): Unit = { val beforeColumnMappingMode = beforeMetadata.columnMappingMode val afterColumnMappingMode = afterMetadata.columnMappingMode // can't switch column mapping mode if (beforeColumnMappingMode != afterColumnMappingMode) { throw DeltaErrors.changeColumnMappingModeNotSupported( beforeColumnMappingMode.name, afterColumnMappingMode.name) } } // Return a copy of the AddFiles with path being absolutized, indicating a SHALLOW CLONE protected def handleNewDataFiles( opName: String, datasetOfNewFilesToAdd: Dataset[AddFile], qualifiedSourceTableBasePath: String, destTable: DeltaLog ): Dataset[AddFile] = { recordDeltaOperation(destTable, s"delta.${opName.toLowerCase()}.makeAbsolute") { val absolutePaths = DeltaFileOperations.makePathsAbsolute( qualifiedSourceTableBasePath, datasetOfNewFilesToAdd) absolutePaths } } } abstract class CloneTableBase( sourceTable: CloneSource, tablePropertyOverrides: Map[String, String], targetPath: Path) extends LeafCommand with CloneTableBaseUtils with SQLConfHelper { import CloneTableBase._ def dataChangeInFileAction: Boolean = true /** Returns whether the table exists at the given snapshot version. */ def tableExists(snapshot: SnapshotDescriptor): Boolean = snapshot.version >= 0 /** * Handles the transaction logic for the CLONE command. * * @param spark [[SparkSession]] to use * @param txn [[OptimisticTransaction]] to use for the commit to the target table. * @param destinationTable [[DeltaLog]] of the destination table. * @param deltaOperation [[DeltaOperations.Operation]] to use when commit changes to DeltaLog * @return */ protected def handleClone( spark: SparkSession, txn: OptimisticTransaction, destinationTable: DeltaLog, hdpConf: Configuration, deltaOperation: DeltaOperations.Operation, commandMetrics: Option[Map[String, SQLMetric]]): Seq[Row] = { val targetFs = targetPath.getFileSystem(hdpConf) val qualifiedTarget = targetFs.makeQualified(targetPath).toString val qualifiedSource = { val sourcePath = sourceTable.dataPath val sourceFs = sourcePath.getFileSystem(hdpConf) sourceFs.makeQualified(sourcePath).toString } if (txn.readVersion < 0) { destinationTable.createLogDirectoriesIfNotExists() } val metadataToUpdate = determineTargetMetadata(spark, txn.snapshot, deltaOperation.name) // Don't merge in the default properties when cloning, or we'll end up with different sets of // properties between source and target. txn.updateMetadata(metadataToUpdate, ignoreDefaultProperties = true) val ( addedFileList ) = { // Make sure target table is empty before running clone if (txn.snapshot.allFiles.count() > 0) { throw DeltaErrors.cloneReplaceNonEmptyTable } val toAdd = sourceTable.allFiles // absolutize file paths handleNewDataFiles( deltaOperation.name, toAdd, qualifiedSource, destinationTable).collectAsList() } val (addedFileCount, addedFilesSize) = (addedFileList.size.toLong, totalDataSize(addedFileList.iterator)) val newProtocol = determineTargetProtocol(spark, txn, deltaOperation.name) val addFileIter = addedFileList.iterator.asScala try { var actions: Iterator[Action] = addFileIter.map { fileToCopy => val copiedFile = fileToCopy.copy(dataChange = dataChangeInFileAction) // CLONE does not preserve Row IDs and Commit Versions copiedFile.copy(baseRowId = None, defaultRowCommitVersion = None) } sourceTable.snapshot.foreach { sourceSnapshot => // Handle DomainMetadata for cloning a table. if (deltaOperation.name == DeltaOperations.OP_CLONE) { actions ++= DomainMetadataUtils.handleDomainMetadataForCloneTable(sourceSnapshot, txn.snapshot) } } val sourceName = sourceTable.name // Override source table metadata with user-defined table properties val context = Map[String, String]() val isReplaceDelta = txn.readVersion >= 0 val opMetrics = SnapshotOverwriteOperationMetrics( sourceTable.sizeInBytes, sourceTable.numOfFiles, addedFileCount, addedFilesSize) val commitOpMetrics = getOperationMetricsForDeltaLog(opMetrics) // Propagate the metrics back to the caller. commandMetrics.foreach { commandMetrics => commitOpMetrics.foreach { kv => commandMetrics(kv._1).set(kv._2) } } recordDeltaOperation( destinationTable, s"delta.${deltaOperation.name.toLowerCase()}.commit") { txn.commitLarge( spark, actions, Some(newProtocol), deltaOperation, context, commitOpMetrics.mapValues(_.toString()).toMap) } val cloneLogData = getOperationMetricsForEventRecord(opMetrics) ++ Map( SOURCE -> sourceName, SOURCE_FORMAT -> sourceTable.format, SOURCE_PATH -> qualifiedSource, TARGET -> qualifiedTarget, PARTITION_BY -> sourceTable.metadata.partitionColumns, IS_REPLACE_DELTA -> isReplaceDelta) ++ sourceTable.snapshot.map(s => SOURCE_VERSION -> s.version) recordDeltaEvent( destinationTable, s"delta.${deltaOperation.name.toLowerCase()}", data = cloneLogData) getOutputSeq(commitOpMetrics) } finally { sourceTable.close() } } /** * Prepares the source metadata by making it compatible with the existing target metadata. */ private def prepareSourceMetadata( targetSnapshot: SnapshotDescriptor, opName: String): Metadata = { var clonedMetadata = sourceTable.metadata.copy( id = UUID.randomUUID().toString, name = targetSnapshot.metadata.name, description = targetSnapshot.metadata.description) // Existing target table if (tableExists(targetSnapshot)) { // Set the ID equal to the target ID clonedMetadata = clonedMetadata.copy(id = targetSnapshot.metadata.id) } val filteredConfiguration = clonedMetadata.configuration // Coordinated Commit configurations are never copied over to the target table. .filterKeys(!CoordinatedCommitsUtils.TABLE_PROPERTY_KEYS.contains(_)) // Catalog-Owned enabled table's UC table ID property is never copied over to the // target table. .filterKeys(_ != UCCommitCoordinatorClient.UC_TABLE_ID_KEY) .toMap val clonedSchema = IdentityColumn.copySchemaWithMergedHighWaterMarks( deltaLog = targetSnapshot.deltaLog, schemaToCopy = clonedMetadata.schema, schemaWithHighWaterMarksToMerge = targetSnapshot.metadata.schema ) clonedMetadata.copy(configuration = filteredConfiguration, schemaString = clonedSchema.json) } /** * Verifies metadata invariants. */ private def verifyMetadataInvariants( targetSnapshot: SnapshotDescriptor, updatedMetadataWithOverrides: Metadata): Unit = { // TODO: we have not decided on how to implement switching column mapping modes // so we block this feature for now // 1. Validate configuration overrides // this checks if columnMapping.maxId is unexpected set in the properties DeltaConfigs.validateConfigurations(tablePropertyOverrides) // 2. Check for column mapping mode conflict with the source metadata w/ tablePropertyOverrides checkColumnMappingMode(sourceTable.metadata, updatedMetadataWithOverrides) // 3. Checks for column mapping mode conflicts with existing metadata if there's any if (tableExists(targetSnapshot)) { checkColumnMappingMode(targetSnapshot.metadata, updatedMetadataWithOverrides) } } /** * Priority of Coordinated Commits configurations: * - When CLONE into a new table, explicit command specification takes precedence over default * SparkSession configurations. * - When CLONE into an existing table, use the existing table's configurations. */ private def determineCoordinatedCommitsConfigurations( spark: SparkSession, targetSnapshot: SnapshotDescriptor, validatedOverrides: Map[String, String]): Map[String, String] = { if (tableExists(targetSnapshot)) { assert(validatedOverrides.isEmpty, "Explicit overrides on Coordinated Commits configurations for existing tables" + " are not supported, and should have been caught earlier.") CoordinatedCommitsUtils.getExplicitCCConfigurations(targetSnapshot.metadata.configuration) } else { if (validatedOverrides.nonEmpty) { validatedOverrides } else { CoordinatedCommitsUtils.getDefaultCCConfigurations(spark) } } } /** * Helper function to determine [[UCCommitCoordinatorClient.UC_TABLE_ID_KEY]] * for the target table. */ private def determineCatalogOwnedUCTableId( targetSnapshot: SnapshotDescriptor): Map[String, String] = { // For REPLACE TABLE command, extract the UC table ID from the target table // if it exists. if (tableExists(targetSnapshot)) { targetSnapshot.metadata.configuration.filter { case (k, _) => k == UCCommitCoordinatorClient.UC_TABLE_ID_KEY } } else { Map.empty } } /** * Determines the expected metadata of the target. */ private def determineTargetMetadata( spark: SparkSession, targetSnapshot: SnapshotDescriptor, opName: String) : Metadata = { var metadata = prepareSourceMetadata(targetSnapshot, opName) val validatedConfigurations = DeltaConfigs.validateConfigurations(tablePropertyOverrides) // Finalize Coordinated Commits configurations for the target val coordinatedCommitsConfigurationOverrides = CoordinatedCommitsUtils.getExplicitCCConfigurations(validatedConfigurations) val validatedConfigurationsWithoutCoordinatedCommits = validatedConfigurations -- coordinatedCommitsConfigurationOverrides.keys val finalCoordinatedCommitsConfigurations = determineCoordinatedCommitsConfigurations( spark, targetSnapshot, coordinatedCommitsConfigurationOverrides) val finalCatalogOwnedMetadata = finalCoordinatedCommitsConfigurations ++ determineCatalogOwnedUCTableId(targetSnapshot) // Merge source configuration, table property overrides and coordinated-commits configurations. metadata = metadata.copy(configuration = metadata.configuration ++ validatedConfigurationsWithoutCoordinatedCommits ++ finalCatalogOwnedMetadata) verifyMetadataInvariants(targetSnapshot, metadata) metadata } /** * Determines the final protocol of the target. The metadata of the `txn` must be updated before * determining the protocol. */ private def determineTargetProtocol( spark: SparkSession, txn: OptimisticTransaction, opName: String): Protocol = { // Catalog-Owned: Do not copy over [[CatalogOwnedTableFeature]] from source table. val sourceProtocol = sourceTable.protocol.removeFeature(targetFeature = CatalogOwnedTableFeature) // Pre-transaction version of the target table. val targetProtocol = txn.snapshot.protocol // Overriding properties during the CLONE can change the minimum required protocol for target. // We need to look at the metadata of the transaction to see the entire set of table properties // for the post-transaction state and decide a version based on that. We also need to re-add // the table property overrides as table features set by it won't be in the transaction // metadata anymore. val validatedConfigurations = DeltaConfigs.validateConfigurations(tablePropertyOverrides) // For CREATE CLONE, check the default spark configuration for Catalog-Owned. val catalogOwnedEnabledByDefaultConf = if (CatalogOwnedTableUtils.defaultCatalogOwnedEnabled(spark) && !tableExists(txn.snapshot)) { // Append [[CatalogOwnedTableFeature]] to the `configWithOverrides` below if table // does not exist, to ensure the final target protocol contains CatalogOwned // if enabled by default. // Note: We need this because CatalogOwned is enabled through single protocol // without auxiliary metadata. Map(s"delta.feature.${CatalogOwnedTableFeature.name}" -> "supported") } else { Map.empty } val configWithOverrides = txn.metadata.configuration ++ validatedConfigurations ++ catalogOwnedEnabledByDefaultConf val metadataWithOverrides = txn.metadata.copy(configuration = configWithOverrides) var (minReaderVersion, minWriterVersion, enabledFeatures) = Protocol.minProtocolComponentsFromMetadata(spark, metadataWithOverrides) // Only upgrade the protocol, never downgrade (unless allowed by flag), since that may break // time travel. val protocolDowngradeAllowed = conf.getConf(DeltaSQLConf.RESTORE_TABLE_PROTOCOL_DOWNGRADE_ALLOWED) || // It's not a real downgrade if the table doesn't exist before the CLONE. !tableExists(txn.snapshot) if (protocolDowngradeAllowed) { minReaderVersion = minReaderVersion.max(sourceProtocol.minReaderVersion) minWriterVersion = minWriterVersion.max(sourceProtocol.minWriterVersion) val minProtocol = Protocol(minReaderVersion, minWriterVersion).withFeatures(enabledFeatures) sourceProtocol.merge(minProtocol) } else { // Take the maximum of all protocol versions being merged to ensure that table features // from table property overrides are correctly added to the table feature list or are only // implicitly enabled minReaderVersion = Seq(targetProtocol.minReaderVersion, sourceProtocol.minReaderVersion, minReaderVersion).max minWriterVersion = Seq( targetProtocol.minWriterVersion, sourceProtocol.minWriterVersion, minWriterVersion).max val minProtocol = Protocol(minReaderVersion, minWriterVersion).withFeatures(enabledFeatures) targetProtocol.merge(sourceProtocol, minProtocol) } } } object CloneTableBase extends Logging { val SOURCE = "source" val SOURCE_FORMAT = "sourceFormat" val SOURCE_PATH = "sourcePath" val SOURCE_VERSION = "sourceVersion" val TARGET = "target" val IS_REPLACE_DELTA = "isReplaceDelta" val PARTITION_BY = "partitionBy" /** Utility method returns the total size of all files in the given iterator */ private def totalDataSize(fileList: java.util.Iterator[AddFile]): Long = { var totalSize = 0L fileList.asScala.foreach { f => totalSize += f.size } totalSize } } /** * Metrics of snapshot overwrite operation. * @param sourceSnapshotSizeInBytes Total size of the data in the source snapshot. * @param sourceSnapshotFileCount Number of data files in the source snapshot. * @param destSnapshotAddedFileCount Number of new data files added to the destination * snapshot as part of the execution. * @param destSnapshotAddedFilesSizeInBytes Total size (in bytes) of the data files that were * added to the destination snapshot. */ case class SnapshotOverwriteOperationMetrics( sourceSnapshotSizeInBytes: Long, sourceSnapshotFileCount: Long, destSnapshotAddedFileCount: Long, destSnapshotAddedFilesSizeInBytes: Long) ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/CloneTableCommand.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands // scalastyle:off import.ordering.noEmptyLine import java.io.FileNotFoundException import org.apache.spark.sql.delta.{DeltaErrors, DeltaLog, DeltaTimeTravelSpec, OptimisticTransaction, Snapshot} import org.apache.spark.sql.delta.DeltaOperations.Clone import org.apache.spark.sql.delta.actions.{AddFile, Metadata, Protocol} import org.apache.spark.sql.delta.actions.Protocol.extractAutomaticallyEnabledFeatures import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.convert.{ConvertTargetTable, ConvertUtils} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.hadoop.fs.Path import org.apache.spark.internal.MDC import org.apache.spark.sql.{Column, Dataset, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} import org.apache.spark.sql.connector.catalog.Table import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{LongType, StructType} import org.apache.spark.util.{Clock, SerializableConfiguration, SystemClock} // scalastyle:on import.ordering.noEmptyLine /** * Clones a Delta table to a new location with a new table id. * The clone can be performed as a shallow clone (i.e. shallow = true), * where we do not copy the files, but just point to them. * If a table exists at the given targetPath, that table will be replaced. * * @param sourceTable is the table to be cloned * @param targetIdent destination table identifier to clone to * @param tablePropertyOverrides user-defined table properties that should override any properties * with the same key from the source table * @param targetPath the actual destination */ case class CloneTableCommand( sourceTable: CloneSource, targetIdent: TableIdentifier, tablePropertyOverrides: Map[String, String], targetPath: Path ) extends CloneTableBase(sourceTable, tablePropertyOverrides, targetPath) { import CloneTableCommand._ /** Return the CLONE command output from the execution metrics */ override protected def getOutputSeq(operationMetrics: Map[String, Long]): Seq[Row] = { Seq(Row( operationMetrics.get(SOURCE_TABLE_SIZE), operationMetrics.get(SOURCE_NUM_OF_FILES), operationMetrics.get(NUM_REMOVED_FILES), operationMetrics.get(NUM_COPIED_FILES), operationMetrics.get(REMOVED_FILES_SIZE), operationMetrics.get(COPIED_FILES_SIZE) )) } /** * Handles the transaction logic for the CLONE command. * @param txn [[OptimisticTransaction]] to use for the commit to the target table. * @param targetDeltaLog [[DeltaLog]] of the target table. * @return */ def handleClone( sparkSession: SparkSession, txn: OptimisticTransaction, targetDeltaLog: DeltaLog, commandMetrics: Option[Map[String, SQLMetric]] = None): Seq[Row] = { if (!targetPath.isAbsolute) { throw DeltaErrors.cloneOnRelativePath(targetIdent.toString) } /** Log clone command information */ logInfo(log"Cloning ${MDC(DeltaLogKeys.CLONE_SOURCE_DESC, sourceTable.description)} to " + log"${MDC(DeltaLogKeys.PATH, targetPath)}") // scalastyle:off deltahadoopconfiguration val hdpConf = sparkSession.sessionState.newHadoopConf() // scalastyle:on deltahadoopconfiguration if (!sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_CLONE_REPLACE_ENABLED)) { val targetFs = targetPath.getFileSystem(hdpConf) try { val subFiles = targetFs.listStatus(targetPath) if (subFiles.nonEmpty) { throw DeltaErrors.cloneReplaceUnsupported(targetIdent) } } catch { case _: FileNotFoundException => // we want the path to not exist targetFs.mkdirs(targetPath) } } handleClone( sparkSession, txn, targetDeltaLog, hdpConf = hdpConf, deltaOperation = Clone( sourceTable.name, sourceTable.snapshot.map(_.version).getOrElse(-1) ), commandMetrics = commandMetrics) } } object CloneTableCommand { // Names of the metrics - added to the Delta commit log as part of Clone transaction val SOURCE_TABLE_SIZE = "sourceTableSize" val SOURCE_NUM_OF_FILES = "sourceNumOfFiles" val NUM_REMOVED_FILES = "numRemovedFiles" val NUM_COPIED_FILES = "numCopiedFiles" val REMOVED_FILES_SIZE = "removedFilesSize" val COPIED_FILES_SIZE = "copiedFilesSize" // SQL way column names for metrics in command execution output private val COLUMN_SOURCE_TABLE_SIZE = "source_table_size" private val COLUMN_SOURCE_NUM_OF_FILES = "source_num_of_files" private val COLUMN_NUM_REMOVED_FILES = "num_removed_files" private val COLUMN_NUM_COPIED_FILES = "num_copied_files" private val COLUMN_REMOVED_FILES_SIZE = "removed_files_size" private val COLUMN_COPIED_FILES_SIZE = "copied_files_size" val output: Seq[Attribute] = Seq( AttributeReference(COLUMN_SOURCE_TABLE_SIZE, LongType)(), AttributeReference(COLUMN_SOURCE_NUM_OF_FILES, LongType)(), AttributeReference(COLUMN_NUM_REMOVED_FILES, LongType)(), AttributeReference(COLUMN_NUM_COPIED_FILES, LongType)(), AttributeReference(COLUMN_REMOVED_FILES_SIZE, LongType)(), AttributeReference(COLUMN_COPIED_FILES_SIZE, LongType)() ) } /** A delta table source to be cloned from */ class CloneDeltaSource( sourceTable: DeltaTableV2) extends CloneSource { private val deltaLog = sourceTable.deltaLog private val sourceSnapshot = sourceTable.initialSnapshot def format: String = CloneSourceFormat.DELTA def protocol: Protocol = sourceSnapshot.protocol def clock: Clock = deltaLog.clock def name: String = sourceTable.name() def dataPath: Path = deltaLog.dataPath def schema: StructType = sourceTable.schema() def catalogTable: Option[CatalogTable] = sourceTable.catalogTable def timeTravelOpt: Option[DeltaTimeTravelSpec] = sourceTable.timeTravelOpt def snapshot: Option[Snapshot] = Some(sourceSnapshot) def metadata: Metadata = sourceSnapshot.metadata def allFiles: Dataset[AddFile] = sourceSnapshot.allFiles def sizeInBytes: Long = sourceSnapshot.sizeInBytes def numOfFiles: Long = sourceSnapshot.numOfFiles def description: String = s"${format} table ${name} at version ${sourceSnapshot.version}" override def close(): Unit = {} } /** A convertible non-delta table source to be cloned from */ abstract class CloneConvertedSource(spark: SparkSession) extends CloneSource { // The converter which produces delta metadata from non-delta table, child class must implement // this converter. protected def convertTargetTable: ConvertTargetTable def format: String = CloneSourceFormat.UNKNOWN def protocol: Protocol = { // This is quirky but necessary to add table features such as column mapping if the default // protocol version supports table features. Protocol().withFeatures(extractAutomaticallyEnabledFeatures(spark, metadata, Protocol())) } override val clock: Clock = new SystemClock() def dataPath: Path = new Path(convertTargetTable.fileManifest.basePath) def schema: StructType = convertTargetTable.tableSchema def timeTravelOpt: Option[DeltaTimeTravelSpec] = None def snapshot: Option[Snapshot] = None override lazy val metadata: Metadata = { val conf = catalogTable // Hive adds some transient table properties which should be ignored .map(_.properties.filterKeys(_ != "transient_lastDdlTime").toMap) .foldRight(convertTargetTable.properties.toMap)(_ ++ _) { Metadata( schemaString = convertTargetTable.tableSchema.json, partitionColumns = convertTargetTable.partitionSchema.fieldNames, configuration = conf, createdTime = Some(System.currentTimeMillis())) } } override lazy val allFiles: Dataset[AddFile] = { import org.apache.spark.sql.delta.implicits._ // scalastyle:off deltahadoopconfiguration val serializableConf = new SerializableConfiguration(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration val baseDir = dataPath.toString val conf = spark.sparkContext.broadcast(serializableConf) val partitionSchema = convertTargetTable.partitionSchema val enforceRelativePathCheck = enforceRelativePath { convertTargetTable.fileManifest.allFiles.mapPartitions { targetFile => val basePath = new Path(baseDir) val fs = basePath.getFileSystem(conf.value.value) targetFile.map(ConvertUtils.createAddFile( _, basePath, fs, SQLConf.get, Some(partitionSchema), !enforceRelativePathCheck)) } } } /** * All data file paths are required to be relative to the table path when true */ def enforceRelativePath: Boolean = true def sizeInBytes: Long = convertTargetTable.sizeInBytes def numOfFiles: Long = convertTargetTable.numFiles def description: String = s"${format} table ${name}" override def close(): Unit = convertTargetTable.fileManifest.close() } /** * A parquet table source to be cloned from */ case class CloneParquetSource( tableIdentifier: TableIdentifier, override val catalogTable: Option[CatalogTable], spark: SparkSession) extends CloneConvertedSource(spark) { override lazy val convertTargetTable: ConvertTargetTable = { val baseDir = catalogTable.map(_.location.toString).getOrElse(tableIdentifier.table) ConvertUtils.getParquetTable(spark, baseDir, catalogTable, None) } override def format: String = CloneSourceFormat.PARQUET override def name: String = catalogTable.map(_.identifier.unquotedString) .getOrElse(s"parquet.`${tableIdentifier.table}`") } case class TablePolicies( hasRowPolicies: Boolean, hasColumnPolicies: Boolean ) /** * A iceberg table source to be cloned from */ case class CloneIcebergSource( metadataLocation: String, tableNameOpt: Option[String], tablePoliciesOpt: Option[TablePolicies], deltaSnapshotOpt: Option[Snapshot], spark: SparkSession) extends CloneConvertedSource(spark) { override lazy val convertTargetTable: ConvertTargetTable = ConvertUtils.getIcebergTable(spark, metadataLocation, deltaSnapshotOpt) override def format: String = CloneSourceFormat.ICEBERG override def name: String = tableNameOpt.getOrElse(s"iceberg.`$metadataLocation`") override def catalogTable: Option[CatalogTable] = None } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/ConvertToDeltaCommand.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands // scalastyle:off import.ordering.noEmptyLine import java.io.Closeable import java.lang.reflect.InvocationTargetException import java.util.Locale import scala.collection.JavaConverters._ import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.{AddFile, Metadata} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.VacuumCommand.{generateCandidateFileMap, getTouchedFile} import org.apache.spark.sql.delta.commands.convert.{ConvertTargetFileManifest, ConvertTargetTable, ConvertUtils} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf} import org.apache.spark.sql.delta.util._ import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.spark.SparkContext import org.apache.spark.internal.MDC import org.apache.spark.sql.{AnalysisException, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.{Analyzer, NoSuchTableException} import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType, SessionCatalog} import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog, V1Table} import org.apache.spark.sql.execution.command.LeafRunnableCommand import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} import org.apache.spark.sql.types.StructType /** * Convert an existing parquet table to a delta table by creating delta logs based on * existing files. Here are the main components: * * - File Listing: Launch a spark job to list files from a given directory in parallel. * * - Schema Inference: Given an iterator on the file list result, we group the iterator into * sequential batches and launch a spark job to infer schema for each batch, * and finally merge schemas from all batches. * * - Stats collection: Again, we group the iterator on file list results into sequential batches * and launch a spark job to collect stats for each batch. * * - Commit the files: We take the iterator of files with stats and write out a delta * log file as the first commit. This bypasses the transaction protocol, but * it's ok as this would be the very first commit. * * @param tableIdentifier the target parquet table. * @param partitionSchema the partition schema of the table, required when table is partitioned. * @param collectStats Should collect column stats per file on convert. * @param deltaPath if provided, the delta log will be written to this location. */ abstract class ConvertToDeltaCommandBase( tableIdentifier: TableIdentifier, partitionSchema: Option[StructType], collectStats: Boolean, deltaPath: Option[String]) extends LeafRunnableCommand with DeltaCommand { protected lazy val statsEnabled: Boolean = conf.getConf(DeltaSQLConf.DELTA_COLLECT_STATS) protected lazy val icebergEnabled: Boolean = conf.getConf(DeltaSQLConf.DELTA_CONVERT_ICEBERG_ENABLED) override lazy val metrics: Map[String, SQLMetric] = Map ( "numConvertedFiles" -> SQLMetrics.createMetric(SparkContext.getOrCreate(), "number of files converted") ) protected def isParquetPathProvider(provider: String): Boolean = provider.equalsIgnoreCase("parquet") protected def isIcebergPathProvider(provider: String): Boolean = icebergEnabled && provider.equalsIgnoreCase("iceberg") protected def isSupportedPathTableProvider(provider: String): Boolean = { isParquetPathProvider(provider) || isIcebergPathProvider(provider) } override def run(spark: SparkSession): Seq[Row] = { val convertProperties = resolveConvertTarget(spark, tableIdentifier) match { case Some(props) if !DeltaSourceUtils.isDeltaTable(props.provider) => props case _ => // Make convert to delta idempotent logConsole("The table you are trying to convert is already a delta table") return Seq.empty[Row] } val targetTable = getTargetTable(spark, convertProperties) val deltaPathToUse = new Path(deltaPath.getOrElse(convertProperties.targetDir)) val deltaLog = DeltaLog.forTable(spark, deltaPathToUse) val txn = deltaLog.startTransaction(convertProperties.catalogTable) if (txn.readVersion > -1) { handleExistingTransactionLog(spark, txn, convertProperties, targetTable.format) return Seq.empty[Row] } performConvert(spark, txn, convertProperties, targetTable) } /** Given the table identifier, figure out what our conversion target is. */ private def resolveConvertTarget( spark: SparkSession, tableIdentifier: TableIdentifier): Option[ConvertTarget] = { val v2SessionCatalog = spark.sessionState.catalogManager.v2SessionCatalog.asInstanceOf[TableCatalog] // TODO: Leverage the analyzer for all this work if (isCatalogTable(spark.sessionState.analyzer, tableIdentifier)) { val namespace = tableIdentifier.database.map(Array(_)) .getOrElse(spark.sessionState.catalogManager.currentNamespace) val ident = Identifier.of(namespace, tableIdentifier.table) v2SessionCatalog.loadTable(ident) match { case v1: V1Table if v1.catalogTable.tableType == CatalogTableType.VIEW => throw DeltaErrors.operationNotSupportedException( "Converting a view to a Delta table", tableIdentifier) case v1: V1Table => val table = v1.catalogTable // Hive adds some transient table properties which should be ignored val props = table.properties.filterKeys(_ != "transient_lastDdlTime").toMap Some(ConvertTarget(Some(table), table.provider, new Path(table.location).toString, props)) case _: DeltaTableV2 => // Already a Delta table None case other => throw DeltaErrors.operationNotSupportedException( s"Converting an unsupported table type $other to a Delta table", tableIdentifier) } } else { Some(ConvertTarget( None, tableIdentifier.database, tableIdentifier.table, Map.empty[String, String])) } } /** * When converting a table to delta using table name, we should also change the metadata in the * catalog table because the delta log should be the source of truth for the metadata rather than * the metastore. * * @param catalogTable metadata of the table to be converted * @param sessionCatalog session catalog of the metastore used to update the metadata */ private def convertMetadata( catalogTable: CatalogTable, sessionCatalog: SessionCatalog): Unit = { var newCatalog = catalogTable.copy( provider = Some("delta"), // TODO: Schema changes unfortunately doesn't get reflected in the HiveMetaStore. Should be // fixed in Apache Spark schema = new StructType(), partitionColumnNames = Seq.empty, properties = Map.empty, // TODO: Serde information also doesn't get removed storage = catalogTable.storage.copy( inputFormat = None, outputFormat = None, serde = None) ) sessionCatalog.alterTable(newCatalog) logInfo(log"Convert to Delta converted metadata") } /** * Calls DeltaCommand.isCatalogTable. With Convert, we may get a format check error in cases where * the metastore and the underlying table don't align, e.g. external table where the underlying * files are converted to delta but the metadata has not been converted yet. In these cases, * catch the error and return based on whether the provided Table Identifier could reasonably be * a path * * @param analyzer The session state analyzer to call * @param tableIdent Table Identifier to determine whether is path based or not * @return Boolean where true means that the table is a table in a metastore and false means the * table is a path based table */ override def isCatalogTable(analyzer: Analyzer, tableIdent: TableIdentifier): Boolean = { try { super.isCatalogTable(analyzer, tableIdentifier) } catch { case e: AnalysisException if e.getMessage.contains("Incompatible format detected") => !isPathIdentifier(tableIdentifier) case e: AssertionError if e.getMessage.contains("Conflicting directory structures") => !isPathIdentifier(tableIdentifier) case _: NoSuchTableException if tableIdent.database.isEmpty && new Path(tableIdent.table).isAbsolute => throw DeltaErrors.missingProviderForConvertException(tableIdent.table) } } /** * Override this method since parquet paths are valid for Convert * * @param tableIdent the provided table or path * @return Whether or not the ident provided can refer to a table by path */ override def isPathIdentifier(tableIdent: TableIdentifier): Boolean = { val provider = tableIdent.database.getOrElse("") // If db doesnt exist or db is called delta/tahoe then check if path exists (DeltaSourceUtils.isDeltaDataSourceName(provider) || isSupportedPathTableProvider(provider)) && new Path(tableIdent.table).isAbsolute } /** * If there is already a transaction log we should handle what happens when convert to delta is * run once again. It may be the case that the table is entirely converted i.e. the underlying * files AND the catalog (if one exists) are updated. Or it may be the case that the table is * partially converted i.e. underlying files are converted but catalog (if one exists) * has not been updated. * * @param spark spark session to get session catalog * @param txn existing transaction log * @param target properties that contains: the provider and the catalogTable when * converting using table name */ private def handleExistingTransactionLog( spark: SparkSession, txn: OptimisticTransaction, target: ConvertTarget, sourceFormat: String): Unit = { // In the case that the table is a delta table but the provider has not been updated we should // update table metadata to reflect that the table is a delta table and table properties should // also be updated if (isParquetCatalogTable(target)) { val catalogTable = target.catalogTable val tableProps = target.properties val deltaLogConfig = txn.metadata.configuration val mergedConfig = deltaLogConfig ++ tableProps if (mergedConfig != deltaLogConfig) { if (deltaLogConfig.nonEmpty && conf.getConf(DeltaSQLConf.DELTA_CONVERT_METADATA_CHECK_ENABLED)) { throw DeltaErrors.convertMetastoreMetadataMismatchException(tableProps, deltaLogConfig) } val newMetadata = txn.metadata.copy( configuration = mergedConfig ) txn.commit( newMetadata :: Nil, DeltaOperations.Convert( numFiles = 0L, partitionSchema.map(_.fieldNames.toSeq).getOrElse(Nil), collectStats = false, catalogTable = catalogTable.map(t => t.identifier.toString), sourceFormat = Some(sourceFormat) )) } convertMetadata( catalogTable.get, spark.sessionState.catalog ) } else { logConsole("The table you are trying to convert is already a delta table") } } /** Is the target table a parquet table defined in an external catalog. */ private def isParquetCatalogTable(target: ConvertTarget): Boolean = { target.catalogTable match { case Some(ct) => ConvertToDeltaCommand.isHiveStyleParquetTable(ct) || target.provider.get.toLowerCase(Locale.ROOT) == "parquet" case None => false } } protected def performStatsCollection( spark: SparkSession, txn: OptimisticTransaction, addFiles: Seq[AddFile]): Iterator[AddFile] = { val dummySnapshot = new DummySnapshot(txn.deltaLog.logPath, txn.deltaLog, txn.metadata) ConvertToDeltaCommand.computeStats(txn.deltaLog, dummySnapshot, addFiles) } /** * Given the file manifest, create corresponding AddFile actions for the entire list of files. */ protected def createDeltaActions( spark: SparkSession, manifest: ConvertTargetFileManifest, partitionSchema: StructType, txn: OptimisticTransaction, fs: FileSystem): Iterator[AddFile] = { val shouldCollectStats = collectStats && statsEnabled val statsBatchSize = conf.getConf(DeltaSQLConf.DELTA_IMPORT_BATCH_SIZE_STATS_COLLECTION) var numFiles = 0L manifest.getFiles.grouped(statsBatchSize).flatMap { batch => val adds = batch.map( ConvertUtils.createAddFile( _, txn.deltaLog.dataPath, fs, conf, Some(partitionSchema), deltaPath.isDefined)) if (shouldCollectStats) { logInfo( log"Collecting stats for a batch of " + log"${MDC(DeltaLogKeys.NUM_FILES, batch.size.toLong)} files; " + log"finished ${MDC(DeltaLogKeys.NUM_FILES2, numFiles)} so far" ) numFiles += statsBatchSize performStatsCollection(spark, txn, adds) } else if (collectStats) { logWarning(log"collectStats is set to true but ${MDC(DeltaLogKeys.CONFIG_KEY, DeltaSQLConf.DELTA_COLLECT_STATS.key)}" + log" is false. Skip statistics collection") adds.toIterator } else { adds.toIterator } } } /** Get the instance of the convert target table, which provides file manifest and schema */ protected def getTargetTable(spark: SparkSession, target: ConvertTarget): ConvertTargetTable = { target.provider match { case Some(providerName) => providerName.toLowerCase(Locale.ROOT) match { case checkProvider if target.catalogTable.exists(ConvertToDeltaCommand.isHiveStyleParquetTable) || isParquetPathProvider(checkProvider) => ConvertUtils.getParquetTable( spark, target.targetDir, target.catalogTable, partitionSchema) case checkProvider if isIcebergPathProvider(checkProvider) => if (partitionSchema.isDefined) { throw DeltaErrors.partitionSchemaInIcebergTables } ConvertUtils.getIcebergTable( spark, target.targetDir, deltaSnapshotOpt = None, collectStats ) case other => throw DeltaErrors.convertNonParquetTablesException(tableIdentifier, other) } case None => throw DeltaErrors.missingProviderForConvertException(target.targetDir) } } /** * Converts the given table to a Delta table. First gets the file manifest for the table. Then * in the first pass, it infers the schema of the table. Then in the second pass, it generates * the relevant Actions for Delta's transaction log, namely the AddFile actions for each file * in the manifest. Once a commit is made, updates an external catalog, e.g. Hive MetaStore, * if this table was referenced through a table in a catalog. */ private def performConvert( spark: SparkSession, txn: OptimisticTransaction, convertProperties: ConvertTarget, targetTable: ConvertTargetTable): Seq[Row] = recordDeltaOperation(txn.deltaLog, "delta.convert") { txn.deltaLog.createLogDirectoriesIfNotExists() val targetPath = new Path(convertProperties.targetDir) // scalastyle:off deltahadoopconfiguration val sessionHadoopConf = spark.sessionState.newHadoopConf() // scalastyle:on deltahadoopconfiguration val fs = targetPath.getFileSystem(sessionHadoopConf) val manifest = targetTable.fileManifest try { if (!manifest.getFiles.hasNext) { throw DeltaErrors.emptyDirectoryException(convertProperties.targetDir) } val partitionFields = targetTable.partitionSchema val schema = targetTable.tableSchema val metadata = Metadata( schemaString = schema.json, partitionColumns = partitionFields.fieldNames, configuration = convertProperties.properties ++ targetTable.properties, createdTime = Some(System.currentTimeMillis())) txn.updateMetadataForNewTable(metadata) checkConversionIsAllowed(txn, targetTable) val numFiles = targetTable.numFiles val addFilesIter = createDeltaActions(spark, manifest, partitionFields, txn, fs) val transactionMetrics = Map[String, String]( "numConvertedFiles" -> numFiles.toString ) metrics("numConvertedFiles") += numFiles sendDriverMetrics(spark, metrics) txn.commitLarge( spark, addFilesIter, Some(txn.protocol), getOperation(numFiles, convertProperties, targetTable.format), getContext, transactionMetrics) } finally { manifest.close() } // If there is a catalog table, convert metadata if (convertProperties.catalogTable.isDefined) { convertMetadata( convertProperties.catalogTable.get, spark.sessionState.catalog ) } Seq.empty[Row] } protected def getContext: Map[String, String] = { Map.empty } /** Get the operation to store in the commit message. */ protected def getOperation( numFilesConverted: Long, convertProperties: ConvertTarget, sourceFormat: String): DeltaOperations.Operation = { DeltaOperations.Convert( numFilesConverted, partitionSchema.map(_.fieldNames.toSeq).getOrElse(Nil), collectStats = collectStats && statsEnabled, convertProperties.catalogTable.map(t => t.identifier.toString), sourceFormat = Some(sourceFormat)) } protected case class ConvertTarget( catalogTable: Option[CatalogTable], provider: Option[String], targetDir: String, properties: Map[String, String]) private def checkColumnMapping( txnMetadata: Metadata, convertTargetTable: ConvertTargetTable): Unit = { if (convertTargetTable.requiredColumnMappingMode != txnMetadata.columnMappingMode) { throw DeltaErrors.convertToDeltaWithColumnMappingNotSupported(txnMetadata.columnMappingMode) } } /** Check if the conversion is allowed. */ private def checkConversionIsAllowed( txn: OptimisticTransaction, targetTable: ConvertTargetTable): Unit = { // TODO: we have not decided on how to implement CONVERT TO DELTA under column mapping modes // for some convert targets so we block this feature for them here checkColumnMapping(txn.metadata, targetTable) RowTracking.checkStatsCollectedIfRowTrackingSupported( txn.protocol, collectStats, statsEnabled) } } case class ConvertToDeltaCommand( tableIdentifier: TableIdentifier, partitionSchema: Option[StructType], collectStats: Boolean, deltaPath: Option[String]) extends ConvertToDeltaCommandBase(tableIdentifier, partitionSchema, collectStats, deltaPath) object ConvertToDeltaCommand extends DeltaLogging { def isHiveStyleParquetTable(catalogTable: CatalogTable): Boolean = { catalogTable.provider.contains("hive") && catalogTable.storage.serde.contains( "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe") } def computeStats( deltaLog: DeltaLog, snapshot: Snapshot, addFiles: Seq[AddFile]): Iterator[AddFile] = { import org.apache.spark.sql.functions._ val filesWithStats = deltaLog.createDataFrame(snapshot, addFiles) .groupBy(input_file_name()).agg(to_json(snapshot.statsCollector)) val pathToAddFileMap = generateCandidateFileMap(deltaLog.dataPath, addFiles) filesWithStats.collect().iterator.map { row => val addFile = getTouchedFile(deltaLog.dataPath, row.getString(0), pathToAddFileMap) addFile.copy(stats = row.getString(1)) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableCommand.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands // scalastyle:off import.ordering.noEmptyLine import java.util.concurrent.TimeUnit import scala.util.Try import org.apache.spark.sql.delta.skipping.clustering.ClusteredTableUtils import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.constraints.Constraints import org.apache.spark.sql.delta.DeltaColumnMapping.filterColumnMappingProperties import org.apache.spark.sql.delta.actions.{Action, Metadata, Protocol, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.actions.DomainMetadata import org.apache.spark.sql.delta.commands.DMLUtils.TaggedCommitData import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CoordinatedCommitsUtils} import org.apache.spark.sql.delta.hooks.{HudiConverterHook, IcebergConverterHook} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.spark.SparkContext import org.apache.spark.internal.MDC import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType} import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.execution.command.{LeafRunnableCommand, RunnableCommand} import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType import org.apache.spark.util.Utils /** * Single entry point for all write or declaration operations for Delta tables accessed through * the table name. * * @param table `CatalogTable` object representing the table to create * @param existingTableOpt The existing table for the same identifier if exists * @param mode The save mode when writing data. Relevant when the query is empty or set to Ignore * with `CREATE TABLE IF NOT EXISTS`. * @param query The query to commit into the Delta table if it exist. This can come from * - CTAS * - saveAsTable * @param operation The table creation mode * @param tableByPath Whether the table is identified by path * @param output SQL output of the command * @param protocol This is used to create a table with specific protocol version * @param allowCatalogManaged This is used to create UC managed table with catalogManaged feature * @param createTableFunc If specified, call this function to create the table, instead of * Spark `SessionCatalog#createTable` which is backed by Hive Metastore. */ case class CreateDeltaTableCommand( override val table: CatalogTable, override val existingTableOpt: Option[CatalogTable], override val mode: SaveMode, query: Option[LogicalPlan], override val operation: TableCreationModes.CreationMode = TableCreationModes.Create, override val tableByPath: Boolean = false, override val output: Seq[Attribute] = Nil, protocol: Option[Protocol] = None, override val allowCatalogManaged: Boolean = false, createTableFunc: Option[CatalogTable => Unit] = None) extends LeafRunnableCommand with DeltaCommand with DeltaLogging with CreateDeltaTableLike { @transient private lazy val sc: SparkContext = SparkContext.getOrCreate() override lazy val metrics = Map[String, SQLMetric]( "numCopiedFiles" -> createMetric(sc, "number of files copied"), "copiedFilesSize" -> createMetric(sc, "size of files copied"), "executionTimeMs" -> createMetric(sc, "time taken to execute the entire operation"), "numRemovedBytes" -> createMetric(sc, "number of bytes removed"), "removedFilesSize" -> createMetric(sc, "size of files removed"), "sourceTableSize" -> createMetric(sc, "size of source table"), "numOutputRows" -> createMetric(sc, "number of output rows"), "numParts" -> createMetric(sc, "number of partitions"), "numFiles" -> createMetric(sc, "number of written files"), "sourceNumOfFiles" -> createMetric(sc, "number of files in source table"), "numRemovedFiles" -> createMetric(sc, "number of files removed."), "numOutputBytes" -> createMetric(sc, "number of output bytes"), "taskCommitTime" -> createMetric(sc, "task commit time"), "jobCommitTime" -> createMetric(sc, "job commit time"), "numOfSyncedTransactions" -> createMetric(sc, "number of synced transactions") ) override def run(sparkSession: SparkSession): Seq[Row] = { assert(table.tableType != CatalogTableType.VIEW) assert(table.identifier.database.isDefined, "Database should've been fixed at analysis") // There is a subtle race condition here, where the table can be created by someone else // while this command is running. Nothing we can do about that though :( val tableExistsInCatalog = existingTableOpt.isDefined if (mode == SaveMode.Ignore && tableExistsInCatalog) { // Early exit on ignore return Nil } else if (mode == SaveMode.ErrorIfExists && tableExistsInCatalog) { throw DeltaErrors.tableAlreadyExists(table) } // This check should be relaxed once the UC client supports creating tables, // It gets bypassed in UTs to allow tests that use InMemoryCommitCoordinator to create tables val tableFeatures = TableFeatureProtocolUtils. getSupportedFeaturesFromTableConfigs(table.properties) if (!Utils.isTesting && !allowCatalogManaged && (tableFeatures.contains(CatalogOwnedTableFeature) || CatalogOwnedTableUtils.defaultCatalogOwnedEnabled(spark = sparkSession))) { throw DeltaErrors.deltaCannotCreateCatalogManagedTable() } val tableWithLocation = getCatalogTableWithLocation(sparkSession) val tableLocation = getDeltaTablePath(tableWithLocation) // To be safe, here we only extract file system options from table storage properties, to create // the DeltaLog. val fileSystemOptions = table.storage.properties.filter { case (k, _) => DeltaTableUtils.validDeltaTableHadoopPrefixes.exists(k.startsWith) } val deltaLog = DeltaUtils.getDeltaLogFromTableOrPath( sparkSession, existingTableOpt, tableLocation, fileSystemOptions) CoordinatedCommitsUtils.validateConfigurationsForCreateDeltaTableCommand( sparkSession, deltaLog.tableExists, query, tableWithLocation.properties) CatalogOwnedTableUtils.validatePropertiesForCreateDeltaTableCommand( spark = sparkSession, tableExists = deltaLog.tableExists, query = query, catalogTableProperties = tableWithLocation.properties, existingTableSnapshotOpt = if (deltaLog.tableExists) Some(deltaLog.unsafeVolatileSnapshot) else None) recordDeltaOperation(deltaLog, "delta.ddl.createTable") { val result = handleCommit(sparkSession, deltaLog, tableWithLocation) sendDriverMetrics(sparkSession, metrics) result } } /** * Handles the transaction logic for the command. Returns the operation metrics in case of CLONE. */ private def handleCommit( sparkSession: SparkSession, deltaLog: DeltaLog, tableWithLocation: CatalogTable): Seq[Row] = { val tableExistsInCatalog = existingTableOpt.isDefined val hadoopConf = deltaLog.newDeltaHadoopConf() val tableLocation = getDeltaTablePath(tableWithLocation) val fs = tableLocation.getFileSystem(hadoopConf) def checkPathEmpty(txn: OptimisticTransaction): Unit = { // Verify the table does not exist. if (mode == SaveMode.Ignore || mode == SaveMode.ErrorIfExists) { // We should have returned earlier in Ignore and ErrorIfExists mode if the table // is already registered in the catalog. assert(!tableExistsInCatalog) // Verify that the data path does not contain any data. // We may have failed a previous write. The retry should still succeed even if we have // garbage data if (txn.readVersion > -1 || !fs.exists(deltaLog.logPath)) { assertPathEmpty(hadoopConf, tableWithLocation) } } } var txn = startTxnForTableCreation(sparkSession, deltaLog, tableWithLocation) OptimisticTransaction.withActive(txn) { val result = query match { // CLONE handled separately from other CREATE TABLE syntax case Some(cmd: CloneTableCommand) => checkPathEmpty(txn) cmd.handleClone( sparkSession, txn, targetDeltaLog = deltaLog, commandMetrics = Some(metrics)) case Some(deltaWriter: WriteIntoDeltaLike) => checkPathEmpty(txn) txn = handleCreateTableAsSelect( sparkSession, txn, deltaLog, deltaWriter, tableWithLocation) Nil case Some(query) => checkPathEmpty(txn) require(!query.isInstanceOf[RunnableCommand]) // When using V1 APIs, the `query` plan is not yet optimized, therefore, it is safe // to once again go through analysis val data = DataFrameUtils.ofRows(sparkSession, query) val options = new DeltaOptions(table.storage.properties, sparkSession.sessionState.conf) val deltaWriter = WriteIntoDelta( deltaLog = deltaLog, mode = mode, options, partitionColumns = table.partitionColumnNames, configuration = tableWithLocation.properties + ("comment" -> table.comment.orNull), data = data, Some(tableWithLocation)) txn = handleCreateTableAsSelect( sparkSession, txn, deltaLog, deltaWriter, tableWithLocation) Nil case _ => handleCreateTable(sparkSession, txn, tableWithLocation, fs, hadoopConf) Nil } runPostCommitUpdates(sparkSession, txn, deltaLog, tableWithLocation) result } } /** * Runs updates post table creation commit, such as updating the catalog * with relevant information. */ private def runPostCommitUpdates( sparkSession: SparkSession, txnUsedForCommit: OptimisticTransaction, deltaLog: DeltaLog, tableWithLocation: CatalogTable): Unit = { // Note that someone may have dropped and recreated the table in a separate location in the // meantime... Unfortunately we can't do anything there at the moment, because Hive sucks. logInfo(log"Table is path-based table: ${MDC(DeltaLogKeys.IS_PATH_TABLE, tableByPath)}. " + log"Update catalog with mode: ${MDC(DeltaLogKeys.OPERATION, operation)}") val opStartTs = TimeUnit.NANOSECONDS.toMillis(txnUsedForCommit.txnStartTimeNs) val postCommitSnapshot = deltaLog.update( checkIfUpdatedSinceTs = Some(opStartTs), catalogTableOpt = Some(tableWithLocation)) val didNotChangeMetadata = txnUsedForCommit.metadata == txnUsedForCommit.snapshot.metadata updateCatalog( sparkSession, tableWithLocation, postCommitSnapshot, query, didNotChangeMetadata, createTableFunc) if (UniversalFormat.hudiEnabled(postCommitSnapshot.metadata) && !txnUsedForCommit.containsPostCommitHook(HudiConverterHook)) { deltaLog.hudiConverter.convertSnapshot(postCommitSnapshot, tableWithLocation) } } /** * Handles the transaction logic for CTAS-like statements, i.e.: * CREATE TABLE AS SELECT * CREATE OR REPLACE TABLE AS SELECT * .saveAsTable in DataframeWriter API * * @return the txn used to make Delta commit */ private def handleCreateTableAsSelect( sparkSession: SparkSession, txn: OptimisticTransaction, deltaLog: DeltaLog, deltaWriter: WriteIntoDeltaLike, tableWithLocation: CatalogTable): OptimisticTransaction = { val isManagedTable = tableWithLocation.tableType == CatalogTableType.MANAGED val options = new DeltaOptions(table.storage.properties, sparkSession.sessionState.conf) // Execute write command for `deltaWriter` by // - replacing the metadata new target table for DataFrameWriterV2 writer if it is a // REPLACE or CREATE_OR_REPLACE command, // - running the write procedure of DataFrameWriter command and returning the // new created actions, // - returning the Delta Operation type of this DataFrameWriter def doDeltaWrite( deltaWriter: WriteIntoDeltaLike, schema: StructType): (TaggedCommitData[Action], DeltaOperations.Operation) = { // In the V2 Writer, methods like "replace" and "createOrReplace" implicitly mean that // the metadata should be changed. This wasn't the behavior for DataFrameWriterV1. if (!isV1WriterSaveAsTableOverwrite) { replaceMetadataIfNecessary( txn, tableWithLocation, options, sparkSession, schema) } var taggedCommitData = deltaWriter.writeAndReturnCommitData( txn, sparkSession, ClusteredTableUtils.getClusterBySpecOptional(table), // Pass this option to the writer so that it can differentiate between an INSERT and a // REPLACE command. This is needed because the writer is shared between the two commands. // But some options, such as dynamic partition overwrite, are only valid for INSERT. // Only allow createOrReplace command which is not a V1 writer. // saveAsTable() command uses this same code path and is marked as a V1 writer. // We do not want saveAsTable() to be treated as a REPLACE command wrt dynamic partition // overwrite. isTableReplace = isReplace && !isV1WriterSaveAsTableOverwrite ) // The 'deltaWriter' initialized the schema. Remove 'EXISTS_DEFAULT' metadata keys because // they are not required on tables created by CTAS. txn.removeExistsDefaultFromSchema() // Metadata updates for creating table (with any writer) and replacing table // (only with V1 writer) will be handled inside WriteIntoDelta. // For createOrReplace operation, metadata updates are handled here if the table already // exists (replacing table), otherwise it is handled inside WriteIntoDelta (creating table). if (!isV1WriterSaveAsTableOverwrite && isReplace && txn.readVersion > -1L) { val newDomainMetadata = Seq.empty[DomainMetadata] ++ ClusteredTableUtils.getDomainMetadataFromTransaction( ClusteredTableUtils.getClusterBySpecOptional(table), txn) // Ensure to remove any domain metadata for REPLACE TABLE. val newActions = taggedCommitData.actions ++ DomainMetadataUtils.handleDomainMetadataForReplaceTable( txn.snapshot.domainMetadata, newDomainMetadata) taggedCommitData = taggedCommitData.copy(actions = newActions) } val op = getOperation(txn.metadata, isManagedTable, Some(options), clusterBy = ClusteredTableUtils.getLogicalClusteringColumnNames( txn, taggedCommitData.actions), // Only recording "true" to reduce noise in DESCRIBE HISTORY when it doesn't apply. isV1SaveAsTableOverwrite = if (isV1WriterSaveAsTableOverwrite) Some(true) else None ) (taggedCommitData, op) } val updatedConfiguration = UniversalFormat.enforceDependenciesInConfiguration( sparkSession, tableWithLocation, deltaWriter.configuration, txn.snapshot ) val updatedWriter = deltaWriter.withNewWriterConfiguration(updatedConfiguration) var txnToReturn = txn // We are either appending/overwriting with saveAsTable or creating a new table with CTAS if (!hasBeenExecuted(txn, sparkSession, Some(options))) { val (taggedCommitData, op) = doDeltaWrite(updatedWriter, updatedWriter.data.schema.asNullable) txn.commit(taggedCommitData.actions, op, tags = taggedCommitData.stringTags) } txnToReturn } /** * Handles the transaction logic for CREATE OR REPLACE TABLE statement * without the AS [CLONE, SELECT] clause. */ private def handleCreateTable( sparkSession: SparkSession, txn: OptimisticTransaction, tableWithLocation: CatalogTable, fs: FileSystem, hadoopConf: Configuration): Unit = { val isManagedTable = tableWithLocation.tableType == CatalogTableType.MANAGED val tableLocation = getDeltaTablePath(tableWithLocation) val tableExistsInCatalog = existingTableOpt.isDefined val options = new DeltaOptions(table.storage.properties, sparkSession.sessionState.conf) def createActionsForNewTableOrVerify(): Seq[Action] = { if (isManagedTable) { // When creating a managed table, the table path should not exist or is empty, or // users would be surprised to see the data, or see the data directory being dropped // after the table is dropped. assertPathEmpty(hadoopConf, tableWithLocation) } // However, if we allow creating an empty schema table and indeed the table is new, we // would need to make sure txn.readVersion <= 0 so we are either: // 1) Creating a new empty schema table (version = -1) or // 2) Restoring an existing empty schema table at version 0. An empty schema table should // not have versions > 0 because it must be written with schema changes after initial // creation. val emptySchemaTableFlag = sparkSession.sessionState.conf .getConf(DeltaSQLConf.DELTA_ALLOW_CREATE_EMPTY_SCHEMA_TABLE) val allowRestoringExistingEmptySchemaTable = emptySchemaTableFlag && txn.metadata.schema.isEmpty && txn.readVersion == 0 val allowCreatingNewEmptySchemaTable = emptySchemaTableFlag && tableWithLocation.schema.isEmpty && txn.readVersion == -1 // This is either a new table, or, we never defined the schema of the table. While it is // unexpected that `txn.metadata.schema` to be empty when txn.readVersion >= 0, we still // guard against it, in case of checkpoint corruption bugs. val noExistingMetadata = txn.readVersion == -1 || txn.metadata.schema.isEmpty if (noExistingMetadata && !allowRestoringExistingEmptySchemaTable) { assertTableSchemaDefined( fs, tableLocation, tableWithLocation, sparkSession, allowCreatingNewEmptySchemaTable ) assertPathEmpty(hadoopConf, tableWithLocation) // This is a user provided schema. // Doesn't come from a query, Follow nullability invariants. var newMetadata = getProvidedMetadata(tableWithLocation, table.schema.json) newMetadata = newMetadata.copy(configuration = UniversalFormat.enforceDependenciesInConfiguration( sparkSession, tableWithLocation, newMetadata.configuration, txn.snapshot )) txn.updateMetadataForNewTable(newMetadata) // Remove 'EXISTS_DEFAULT' because it is not required for tables created with CREATE TABLE. txn.removeExistsDefaultFromSchema() protocol.foreach { protocol => // For commands like CREATE LIKE, the `protocol` here may contain table features // from source table. It will override the `newProtocol` being created in the above // `txn.updateMetadataForNewTable`. // In order to enable [[CatalogOwnedTableFeature]] for target table w/ default // spark configuration of CatalogOwned enabled, we need to manually append // [[CatalogOwnedTableFeature]] here to the existing source table protocol. val finalizedProtocol = if (CatalogOwnedTableUtils.defaultCatalogOwnedEnabled( spark = sparkSession)) { val minCatalogOwnedProtocol = Protocol( CatalogOwnedTableFeature.minReaderVersion, CatalogOwnedTableFeature.minWriterVersion).withFeature(CatalogOwnedTableFeature) protocol.merge(minCatalogOwnedProtocol) } else { protocol } txn.updateProtocol(finalizedProtocol) } ClusteredTableUtils.getDomainMetadataFromTransaction( ClusteredTableUtils.getClusterBySpecOptional(table), txn).toSeq } else { verifyTableMetadata(sparkSession, txn, tableWithLocation) Nil } } // We are defining a table using the Create or Replace Table statements. val actionsToCommit = operation match { case TableCreationModes.Create => require(!tableExistsInCatalog, "Can't recreate a table when it exists") createActionsForNewTableOrVerify() case TableCreationModes.CreateOrReplace if !tableExistsInCatalog => // If the table doesn't exist, CREATE OR REPLACE must provide a schema if (tableWithLocation.schema.isEmpty) { throw DeltaErrors.schemaNotProvidedException } createActionsForNewTableOrVerify() case _ => // When the operation is a REPLACE or CREATE OR REPLACE, then the schema shouldn't be // empty, since we'll use the entry to replace the schema if (tableWithLocation.schema.isEmpty) { throw DeltaErrors.schemaNotProvidedException } // This can happen if someone deleted files from the filesystem but // the table still exists in the catalog. if (txn.readVersion == -1 && tableExistsInCatalog) { throw DeltaErrors.metadataAbsentForExistingCatalogTable( tableWithLocation.identifier.toString, txn.deltaLog.logPath.toString) } // We need to replace replaceMetadataIfNecessary( txn, tableWithLocation, options, sparkSession, tableWithLocation.schema) // Remove 'EXISTS_DEFAULT' because it is not required for tables created with REPLACE TABLE. txn.removeExistsDefaultFromSchema() // Truncate the table val operationTimestamp = System.currentTimeMillis() var actionsToCommit = Seq.empty[Action] val removes = txn.filterFiles().map(_.removeWithTimestamp(operationTimestamp)) actionsToCommit = removes ++ DomainMetadataUtils.handleDomainMetadataForReplaceTable( txn.snapshot.domainMetadata, ClusteredTableUtils.getDomainMetadataFromTransaction( ClusteredTableUtils.getClusterBySpecOptional(table), txn).toSeq) actionsToCommit } // Validate check constraints for CREATE/REPLACE TABLE val checkConstraints = Constraints.getAll(txn.metadata, sparkSession) Constraints.validateCheckConstraints( sparkSession, checkConstraints, txn.deltaLog, txn.metadata.schema ) val changedMetadata = txn.metadata != txn.snapshot.metadata val changedProtocol = txn.protocol != txn.snapshot.protocol if (actionsToCommit.nonEmpty || changedMetadata || changedProtocol) { val op = getOperation(txn.metadata, isManagedTable, None, clusterBy = ClusteredTableUtils.getLogicalClusteringColumnNames( txn, actionsToCommit) ) txn.commit(actionsToCommit, op) } } private def getProvidedMetadata(table: CatalogTable, schemaString: String): Metadata = { Metadata( description = table.comment.orNull, schemaString = schemaString, partitionColumns = table.partitionColumnNames, // Filter out ephemeral clustering columns config because we don't want to persist // it in delta log. This will be persisted in CatalogTable's table properties instead. configuration = ClusteredTableUtils.removeClusteringColumnsProperty(table.properties), createdTime = Some(System.currentTimeMillis())) } private def assertPathEmpty( hadoopConf: Configuration, tableWithLocation: CatalogTable): Unit = { val path = getDeltaTablePath(tableWithLocation) val fs = path.getFileSystem(hadoopConf) // Verify that the table location associated with CREATE TABLE doesn't have any data. Note that // we intentionally diverge from this behavior w.r.t regular datasource tables (that silently // overwrite any previous data) if (fs.exists(path) && fs.listStatus(path).nonEmpty) { throw DeltaErrors.createTableWithNonEmptyLocation( tableWithLocation.identifier.toString, path.toString) } } private def assertTableSchemaDefined( fs: FileSystem, path: Path, table: CatalogTable, sparkSession: SparkSession, allowEmptyTableSchema: Boolean): Unit = { // Users did not specify the schema. We expect the schema exists in Delta. if (table.schema.isEmpty) { if (table.tableType == CatalogTableType.EXTERNAL) { if (fs.exists(path) && fs.listStatus(path).nonEmpty) { throw DeltaErrors.createExternalTableWithoutLogException( path, table.identifier.quotedString, sparkSession) } else { if (allowEmptyTableSchema) return throw DeltaErrors.createExternalTableWithoutSchemaException( path, table.identifier.quotedString, sparkSession) } } else { if (allowEmptyTableSchema) return throw DeltaErrors.createManagedTableWithoutSchemaException( table.identifier.quotedString, sparkSession) } } } /** * When creating an external table in a location where some table already existed, we make sure * that the specified table properties match the existing table properties. Since Coordinated * Commits is not designed to be overridden, we should not error out if the command omits these * properties. If the existing table has Coordinated Commits enabled, we also do not error out if * the command omits the ICT properties, which are the dependencies for Coordinated Commits. */ private def filterCoordinatedCommitsProperties( existingProperties: Map[String, String], tableProperties: Map[String, String]): Map[String, String] = { var filteredExistingProperties = existingProperties val overridingCCConfs = CoordinatedCommitsUtils.getExplicitCCConfigurations(tableProperties) val existingCCConfs = CoordinatedCommitsUtils.getExplicitCCConfigurations(existingProperties) if (existingCCConfs.nonEmpty && overridingCCConfs.isEmpty) { filteredExistingProperties --= CoordinatedCommitsUtils.TABLE_PROPERTY_KEYS val overridingICTConfs = CoordinatedCommitsUtils.getExplicitICTConfigurations(tableProperties) val existingICTConfs = CoordinatedCommitsUtils.getExplicitICTConfigurations( existingProperties) if (existingICTConfs.nonEmpty && overridingICTConfs.isEmpty) { filteredExistingProperties --= CoordinatedCommitsUtils.ICT_TABLE_PROPERTY_KEYS } } filteredExistingProperties } /** * Verify against our transaction metadata that the user specified the right metadata for the * table. */ private def verifyTableMetadata( sparkSession: SparkSession, txn: OptimisticTransaction, tableDesc: CatalogTable): Unit = { val existingMetadata = txn.metadata val path = getDeltaTablePath(tableDesc) // The delta log already exists. If they give any configuration, we'll make sure it all matches. // Otherwise we'll just go with the metadata already present in the log. // The schema compatibility checks will be made in `WriteIntoDelta` for CreateTable // with a query if (txn.readVersion > -1) { if (tableDesc.schema.nonEmpty) { // We check exact alignment on create table if everything is provided // However, if in column mapping mode, we can safely ignore the related metadata fields in // existing metadata because new table desc will not have related metadata assigned yet val differences = SchemaUtils.reportDifferences( DeltaTableUtils.removeInternalDeltaMetadata(sparkSession, existingMetadata.schema), tableDesc.schema) if (differences.nonEmpty) { throw DeltaErrors.createTableWithDifferentSchemaException( path, tableDesc.schema, existingMetadata.schema, differences) } // If schema is specified, we must make sure the partitioning matches, even the partitioning // is not specified. if (tableDesc.partitionColumnNames != existingMetadata.partitionColumns) { throw DeltaErrors.createTableWithDifferentPartitioningException( path, tableDesc.partitionColumnNames, existingMetadata.partitionColumns) } // If schema is specified, we must make sure the clustering column matches (includes when // clustering is not specified). val specifiedClusterBySpec = ClusteredTableUtils.getClusterBySpecOptional(tableDesc) val existingClusterBySpec = ClusteredTableUtils.getClusterBySpecOptional(txn.snapshot) if (specifiedClusterBySpec != existingClusterBySpec) { throw DeltaErrors.createTableWithDifferentClusteringException( path, specifiedClusterBySpec, existingClusterBySpec) } } if (tableDesc.properties.nonEmpty) { // When comparing properties of the existing table and the new table, remove some // internal column mapping properties for the sake of comparison. var filteredTableProperties = filterColumnMappingProperties( tableDesc.properties) // We also need to remove any protocol-related properties as we're filtering these // from the metadata so they won't be present in the table properties. filteredTableProperties = Protocol.filterProtocolPropsFromTableProps(filteredTableProperties) var filteredExistingProperties = filterColumnMappingProperties( existingMetadata.configuration) // Clustered table has internal table properties in Metadata configurations and they are // never configured by the user so remove them before validation. if (ClusteredTableUtils.isSupported(txn.protocol)) { filteredExistingProperties = ClusteredTableUtils.removeInternalTableProperties(filteredExistingProperties) ++ // Validate clustering columns in CatalogTable.PROP_CLUSTERING_COLUMNS // are matched. ClusteredTableUtils.getClusteringColumnsAsProperty(txn.snapshot) // Note that clustering columns are already stored in the key // CatalogTable.PROP_CLUSTERING_COLUMNS. filteredTableProperties = ClusteredTableUtils.removeInternalTableProperties(filteredTableProperties) } filteredExistingProperties = filterCoordinatedCommitsProperties(filteredExistingProperties, filteredTableProperties) if (filteredTableProperties != filteredExistingProperties) { throw DeltaErrors.createTableWithDifferentPropertiesException( path, filteredTableProperties, filteredExistingProperties) } // If column mapping properties are present in both configs, verify they're the same value. if (!DeltaColumnMapping.verifyInternalProperties( tableDesc.properties, existingMetadata.configuration)) { throw DeltaErrors.createTableWithDifferentPropertiesException( path, tableDesc.properties, existingMetadata.configuration) } } } } /** * Based on the table creation operation, and parameters, we can resolve to different operations. * A lot of this is needed for legacy reasons in Databricks Runtime. * @param metadata The table metadata, which we are creating or replacing * @param isManagedTable Whether we are creating or replacing a managed table * @param options Write options, if this was a CTAS/RTAS */ private def getOperation( metadata: Metadata, isManagedTable: Boolean, options: Option[DeltaOptions], clusterBy: Option[Seq[String]], isV1SaveAsTableOverwrite: Option[Boolean] = None ): DeltaOperations.Operation = operation match { // This is legacy saveAsTable behavior in Databricks Runtime case TableCreationModes.Create if existingTableOpt.isDefined && query.isDefined && options.nonEmpty => DeltaOperations.Write( mode = mode, partitionBy = Option(table.partitionColumnNames), predicate = options.get.replaceWhere, userMetadata = options.flatMap(_.userMetadata), isDynamicPartitionOverwrite = options.flatMap( o => if (Try(o.isDynamicPartitionOverwriteMode).getOrElse(false)) Some(true) else None), canOverwriteSchema = options.flatMap(o => if (o.canOverwriteSchema) Some(true) else None), canMergeSchema = options.flatMap(o => if (o.canMergeSchema) Some(true) else None) ) // DataSourceV2 table creation // CREATE TABLE (non-DataFrameWriter API) doesn't have options syntax // (userMetadata uses SQLConf in this case) case TableCreationModes.Create => DeltaOperations.CreateTable( metadata, isManagedTable, query.isDefined, clusterBy = clusterBy ) // DataSourceV2 table replace // REPLACE TABLE (non-DataFrameWriter API) doesn't have options syntax // (userMetadata uses SQLConf in this case) case TableCreationModes.Replace => DeltaOperations.ReplaceTable( metadata = metadata, isManaged = isManagedTable, orCreate = false, asSelect = query.isDefined, clusterBy = clusterBy, predicate = options.flatMap(_.replaceWhere), isDynamicPartitionOverwrite = options.flatMap( o => if (Try(o.isDynamicPartitionOverwriteMode).getOrElse(false)) Some(true) else None), canOverwriteSchema = options.flatMap(o => if (o.canOverwriteSchema) Some(true) else None), canMergeSchema = options.flatMap(o => if (o.canMergeSchema) Some(true) else None), isV1SaveAsTableOverwrite = isV1SaveAsTableOverwrite ) // Legacy saveAsTable with Overwrite mode case TableCreationModes.CreateOrReplace if options.exists(_.replaceWhere.isDefined) => DeltaOperations.Write( mode = mode, partitionBy = Option(table.partitionColumnNames), predicate = options.get.replaceWhere, userMetadata = options.flatMap(_.userMetadata), isDynamicPartitionOverwrite = options.flatMap( o => if (Try(o.isDynamicPartitionOverwriteMode).getOrElse(false)) Some(true) else None), canOverwriteSchema = options.flatMap(o => if (o.canOverwriteSchema) Some(true) else None), canMergeSchema = options.flatMap(o => if (o.canMergeSchema) Some(true) else None) ) // New DataSourceV2 saveAsTable with overwrite mode behavior case TableCreationModes.CreateOrReplace => DeltaOperations.ReplaceTable( metadata = metadata, isManaged = isManagedTable, orCreate = true, asSelect = query.isDefined, userMetadata = options.flatMap(_.userMetadata), clusterBy = clusterBy, predicate = options.flatMap(_.replaceWhere), isDynamicPartitionOverwrite = options.flatMap( o => if (Try(o.isDynamicPartitionOverwriteMode).getOrElse(false)) Some(true) else None), canOverwriteSchema = options.flatMap(o => if (o.canOverwriteSchema) Some(true) else None), canMergeSchema = options.flatMap(o => if (o.canMergeSchema) Some(true) else None), isV1SaveAsTableOverwrite = isV1SaveAsTableOverwrite ) } private def getDeltaTablePath(table: CatalogTable): Path = { new Path(table.location) } /** * With DataFrameWriterV2, methods like `replace()` or `createOrReplace()` mean that the * metadata of the table should be replaced. If overwriteSchema=false is provided with these * methods, then we will verify that the metadata match exactly. */ private def replaceMetadataIfNecessary( txn: OptimisticTransaction, tableDesc: CatalogTable, options: DeltaOptions, sparkSession: SparkSession, schema: StructType): Unit = { // If a user explicitly specifies not to overwrite the schema, during a replace, we should // tell them that it's not supported val dontOverwriteSchema = options.options.contains(DeltaOptions.OVERWRITE_SCHEMA_OPTION) && !options.canOverwriteSchema if (isReplace && dontOverwriteSchema) { throw DeltaErrors.illegalUsageException(DeltaOptions.OVERWRITE_SCHEMA_OPTION, "replacing") } if (txn.readVersion > -1L && isReplace && !dontOverwriteSchema) { // When a table already exists, and we're using the DataFrameWriterV2 API to replace // or createOrReplace a table, we blindly overwrite the metadata. var newMetadata = getProvidedMetadata(table, schema.json) val updatedConfig = UniversalFormat.enforceDependenciesInConfiguration( sparkSession, tableDesc, newMetadata.configuration, txn.snapshot) newMetadata = newMetadata.copy(configuration = updatedConfig) if (allowCatalogManaged && txn.snapshot.isCatalogOwned) { // Preserve the existing Delta metadata id across REPLACE. This is distinct from the // Unity Catalog table id stored in `io.unitycatalog.tableId`. newMetadata = newMetadata.copy(id = txn.snapshot.metadata.id) } txn.updateMetadataForNewTableInReplace(newMetadata) } } /** Returns true if the current operation could be replacing a table. */ private def isReplace: Boolean = { operation == TableCreationModes.CreateOrReplace || operation == TableCreationModes.Replace } /** Returns the transaction that should be used for the CREATE/REPLACE commit. */ private def startTxnForTableCreation( sparkSession: SparkSession, deltaLog: DeltaLog, tableWithLocation: CatalogTable, snapshotOpt: Option[Snapshot] = None): OptimisticTransaction = { val txn = deltaLog.startTransaction(existingTableOpt, snapshotOpt) validatePrerequisitesForClusteredTable(txn.snapshot.protocol, txn.deltaLog) // During CREATE (not REPLACE/overwrites), we synchronously run conversion // (if Uniform is enabled) so we always remove the post commit hook here. if (!isReplace) { txn.unregisterPostCommitHooksWhere(hook => hook.name == IcebergConverterHook.name) txn.unregisterPostCommitHooksWhere(hook => hook.name == HudiConverterHook.name) } txn } /** * Validate pre-requisites for clustered tables for CREATE/REPLACE operations. * @param protocol Protocol used for validations. This protocol should * be used during the CREATE/REPLACE commit. * @param deltaLog Delta log used for logging purposes. */ private def validatePrerequisitesForClusteredTable( protocol: Protocol, deltaLog: DeltaLog): Unit = { // Validate a clustered table is not replaced by a partitioned table. if (table.partitionColumnNames.nonEmpty && ClusteredTableUtils.isSupported(protocol)) { throw DeltaErrors.replacingClusteredTableWithPartitionedTableNotAllowed() } } } // isCreate is true for Create and CreateOrReplace modes. It is false for Replace mode. object TableCreationModes { sealed trait CreationMode { def mode: SaveMode def isCreate: Boolean = true } case object Create extends CreationMode { override def mode: SaveMode = SaveMode.ErrorIfExists } case object CreateOrReplace extends CreationMode { override def mode: SaveMode = SaveMode.Overwrite } case object Replace extends CreationMode { override def mode: SaveMode = SaveMode.Overwrite override def isCreate: Boolean = false } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableLike.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands import org.apache.spark.sql.delta.{DeltaErrors, DeltaOptions, Snapshot} import org.apache.spark.sql.delta.hooks.{UpdateCatalog, UpdateCatalogFactory} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.{SaveMode, SparkSession} import org.apache.spark.sql.catalyst.SQLConfHelper import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.util.CharVarcharUtils import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.types.StructType /** * A common trait implementing utility functions (e.g. catalog operations) for all commands that * create a Delta table. */ trait CreateDeltaTableLike extends SQLConfHelper { // The table to create. val table: CatalogTable // The existing table for the same identifier if exists. val existingTableOpt: Option[CatalogTable] // The table creation mode. val operation: TableCreationModes.CreationMode // Whether the table is accessed by path. val tableByPath: Boolean = false // The save mode when writing data. Relevant when the query is empty or set to Ignore with `CREATE // TABLE IF NOT EXISTS`. val mode: SaveMode // Whether the table is UC managed table with catalogManaged feature. val allowCatalogManaged: Boolean /** * Generates a `CatalogTable` with its `locationUri` set appropriately, depending on whether the * table already exists or is newly created. */ protected def getCatalogTableWithLocation(sparkSession: SparkSession): CatalogTable = { val tableExistsInCatalog = existingTableOpt.isDefined if (tableExistsInCatalog) { val existingTable = existingTableOpt.get table.storage.locationUri match { case Some(location) if location.getPath != existingTable.location.getPath => throw DeltaErrors.tableLocationMismatch(table, existingTable) case _ => } table.copy( storage = existingTable.storage, tableType = existingTable.tableType) } else if (table.storage.locationUri.isEmpty) { // We are defining a new managed table assert(table.tableType == CatalogTableType.MANAGED) val loc = sparkSession.sessionState.catalog.defaultTablePath(table.identifier) table.copy(storage = table.storage.copy(locationUri = Some(loc))) } else { // 1. We are defining a new external table // 2. It's a managed table which already has the location populated. This can happen in DSV2 // CTAS flow. table } } /** * Here we disambiguate the catalog alterations we need to do based on the table operation, and * whether we have reached here through legacy code or DataSourceV2 code paths. */ protected def updateCatalog( spark: SparkSession, table: CatalogTable, snapshot: Snapshot, query: Option[LogicalPlan], didNotChangeMetadata: Boolean, createTableFunc: Option[CatalogTable => Unit] = None ): Unit = { val cleaned = cleanupTableDefinition(spark, table, snapshot) operation match { case _ if tableByPath => // do nothing with the metastore if this is by path case TableCreationModes.Create => if (createTableFunc.isDefined) { createTableFunc.get.apply(cleaned) } else { spark.sessionState.catalog.createTable( cleaned, ignoreIfExists = existingTableOpt.isDefined || mode == SaveMode.Ignore, validateLocation = false) } case TableCreationModes.Replace | TableCreationModes.CreateOrReplace if existingTableOpt.isDefined => // Catalog-managed / CC tables are owned by the delegated V2 catalog plugin (for example // Unity Catalog), so SessionCatalog's post-commit UpdateCatalogHook must not run. if (!allowCatalogManaged) { UpdateCatalogFactory.getUpdateCatalogHook(table, spark).updateSchema(spark, snapshot) } case TableCreationModes.Replace => val ident = Identifier.of(table.identifier.database.toArray, table.identifier.table) throw DeltaErrors.cannotReplaceMissingTableException(ident) case TableCreationModes.CreateOrReplace => createTableFunc match { case Some(createFunc) => // This is the new missing-table path where creation is delegated through the V2 // catalog plugin (for example Unity Catalog) instead of SessionCatalog.createTable(). createFunc(cleaned) case None => spark.sessionState.catalog.createTable( cleaned, ignoreIfExists = false, validateLocation = false) } } if (conf.getConf(DeltaSQLConf.HMS_FORCE_ALTER_TABLE_DATA_SCHEMA)) { spark.sessionState.catalog.alterTableDataSchema(cleaned.identifier, cleaned.schema) } } /** Clean up the information we pass on to store in the catalog. */ private def cleanupTableDefinition(spark: SparkSession, table: CatalogTable, snapshot: Snapshot) : CatalogTable = { // These actually have no effect on the usability of Delta, but feature flagging legacy // behavior for now val storageProps = if (conf.getConf(DeltaSQLConf.DELTA_LEGACY_STORE_WRITER_OPTIONS_AS_PROPS)) { // Legacy behavior table.storage } else { table.storage.copy(properties = Map.empty) } // If we have to update the catalog, use the correct schema and table properties, otherwise // empty out the schema and property information if (conf.getConf(DeltaSQLConf.DELTA_UPDATE_CATALOG_ENABLED)) { val truncationThreshold = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_UPDATE_CATALOG_LONG_FIELD_TRUNCATION_THRESHOLD) val (truncatedSchema, additionalProperties) = UpdateCatalog.truncateSchemaIfNecessary( snapshot.schema, truncationThreshold) table.copy( schema = truncatedSchema, // Hive does not allow for the removal of partition columns once stored. // To avoid returning the incorrect schema when the partition columns change, // we store the partition columns as regular data columns. partitionColumnNames = Nil, properties = UpdateCatalog.updatedProperties(snapshot) ++ additionalProperties, storage = storageProps, tracksPartitionsInCatalog = true) } else if (allowCatalogManaged) { // Setting table properties is required for creating catalogManaged tables. table.copy( // Here we use snapshot.schema instead of table.schema because it reflects the actual // committed state of the table. // Delta does not have a distinct storage type for Char/Varchar; in snapshots, they are // represented in String type with extra type metadata. We convert them to back to the // original Char/Varchar types when storing them in the catalog. schema = CharVarcharUtils.getRawSchema(snapshot.schema), partitionColumnNames = snapshot.metadata.partitionColumns, properties = UpdateCatalog.updatedProperties(snapshot), storage = storageProps, tracksPartitionsInCatalog = true) } else { table.copy( schema = new StructType(), properties = Map.empty, partitionColumnNames = Nil, // Remove write specific options when updating the catalog storage = storageProps, tracksPartitionsInCatalog = true) } } /** * Differentiate between DataFrameWriterV1 and V2 so that we can decide * what to do with table metadata. In DataFrameWriterV1, mode("overwrite").saveAsTable, * behaves as a CreateOrReplace table, but we have asked for "overwriteSchema" as an * explicit option to overwrite partitioning or schema information. With DataFrameWriterV2, * the behavior asked for by the user is clearer: .createOrReplace(), which means that we * should overwrite schema and/or partitioning. Therefore we have this hack. */ protected def isV1WriterSaveAsTableOverwrite: Boolean = { val options = new DeltaOptions(table.storage.properties, conf) CreateDeltaTableLikeShims.isV1WriterSaveAsTableOverwrite(options, mode) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/DMLUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands import org.apache.spark.sql.delta.DeltaCommitTag import org.apache.spark.sql.delta.actions.{Action, FileAction} object DMLUtils { /** Holder for some of the parameters for a `OptimisticTransaction.commit` */ case class TaggedCommitData[A <: Action]( actions: Seq[A], tags: Map[DeltaCommitTag, String] = Map.empty) { def withTag[T](key: DeltaCommitTag.TypedCommitTag[T], value: T): TaggedCommitData[A] = { val mergedValue = key.mergeWithNewTypedValue(tags.get(key), value) this.copy(tags = this.tags + (key -> mergedValue)) } def stringTags: Map[String, String] = tags.map { case (k, v) => k.key -> v } } object TaggedCommitData { def empty[A <: Action]: TaggedCommitData[A] = TaggedCommitData(Seq.empty[A]) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/DMLWithDeletionVectorsHelper.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands import java.util.UUID import scala.collection.generic.Sizing import org.apache.spark.sql.catalyst.expressions.aggregation.BitmapAggregator import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.{DataFrameUtils, DeltaLog, DeltaParquetFileFormat, OptimisticTransaction, Snapshot} import org.apache.spark.sql.delta.DeltaParquetFileFormat._ import org.apache.spark.sql.delta.actions.{AddFile, DeletionVectorDescriptor, FileAction} import org.apache.spark.sql.delta.deletionvectors.{RoaringBitmapArray, RoaringBitmapArrayFormat, StoredBitmap} import org.apache.spark.sql.delta.files.{TahoeBatchFileIndex, TahoeFileIndex} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.StatsCollectionUtils import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore import org.apache.spark.sql.delta.util.{BinPackingIterator, DeltaEncoder, PathWithFileSystem, Utils => DeltaUtils} import org.apache.spark.sql.delta.util.DeltaFileOperations.absolutePath import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.paths.SparkPath import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression} import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project} import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelationWithTable} import org.apache.spark.sql.execution.datasources.FileFormat.{FILE_PATH, METADATA_NAME} import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.functions.{col, lit} import org.apache.spark.sql.types.StructType import org.apache.spark.util.{SerializableConfiguration, Utils => SparkUtils} /** * Contains utility classes and method for performing DML operations with Deletion Vectors. */ object DMLWithDeletionVectorsHelper extends DeltaCommand { val SUPPORTED_DML_COMMANDS: Seq[String] = Seq("DELETE", "UPDATE") /** * Creates a DataFrame that can be used to scan for rows matching the condition in the given * files. Generally the given file list is a pruned file list using the stats based pruning. */ def createTargetDfForScanningForMatches( spark: SparkSession, target: LogicalPlan, fileIndex: TahoeFileIndex): DataFrame = { DataFrameUtils.ofRows(spark, replaceFileIndex(spark, target, fileIndex)) } /** * Replace the file index in a logical plan and return the updated plan. * It's a common pattern that, in Delta commands, we use data skipping to determine a subset of * files that can be affected by the command, so we replace the whole-table file index in the * original logical plan with a new index of potentially affected files, while everything else in * the original plan, e.g., resolved references, remain unchanged. * * In addition we also request a metadata column and a row index column from the Scan to help * generate the Deletion Vectors. When predicate pushdown is enabled, we only request the * metadata column. This is because we can utilize _metadata.row_index instead of generating a * custom one. * * @param spark the active spark session * @param target the logical plan in which we replace the file index * @param fileIndex the new file index */ private def replaceFileIndex( spark: SparkSession, target: LogicalPlan, fileIndex: TahoeFileIndex): LogicalPlan = { val useMetadataRowIndex = spark.sessionState.conf.getConf(DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX) // This is only used when predicate pushdown is disabled. val rowIndexCol = AttributeReference(ROW_INDEX_COLUMN_NAME, ROW_INDEX_STRUCT_FIELD.dataType)() var fileMetadataCol: AttributeReference = null val newTarget = target.transformUp { case l @ LogicalRelationWithTable( hfsr @ HadoopFsRelation(_, _, _, _, format: DeltaParquetFileFormat, _), _) => fileMetadataCol = format.createFileMetadataCol() // Take the existing schema and add additional metadata columns if (useMetadataRowIndex) { l.copy( relation = hfsr.copy(location = fileIndex)(hfsr.sparkSession), output = l.output :+ fileMetadataCol) } else { val newDataSchema = StructType(hfsr.dataSchema).add(ROW_INDEX_STRUCT_FIELD) val finalOutput = l.output ++ Seq(rowIndexCol, fileMetadataCol) // Disable splitting and filter pushdown in order to generate the row-indexes. val newFormat = format.copy(optimizationsEnabled = false) val newBaseRelation = hfsr.copy( location = fileIndex, dataSchema = newDataSchema, fileFormat = newFormat)(hfsr.sparkSession) l.copy(relation = newBaseRelation, output = finalOutput) } case p @ Project(projectList, _) => if (fileMetadataCol == null) { throw new IllegalStateException("File metadata column is not yet created.") } val rowIndexColOpt = if (useMetadataRowIndex) None else Some(rowIndexCol) val additionalColumns = Seq(fileMetadataCol) ++ rowIndexColOpt p.copy(projectList = projectList ++ additionalColumns) } newTarget } /** * Find the target table files that contain rows that satisfy the condition and a DV attached * to each file that indicates a the rows marked as deleted from the file */ def findTouchedFiles( sparkSession: SparkSession, txn: OptimisticTransaction, hasDVsEnabled: Boolean, deltaLog: DeltaLog, targetDf: DataFrame, fileIndex: TahoeFileIndex, condition: Expression, opName: String): Seq[TouchedFileWithDV] = { require( SUPPORTED_DML_COMMANDS.contains(opName), s"Expecting opName to be one of ${SUPPORTED_DML_COMMANDS.mkString(", ")}, " + s"but got '$opName'.") recordDeltaOperation(deltaLog, opType = s"$opName.findTouchedFiles") { val candidateFiles = fileIndex match { case f: TahoeBatchFileIndex => f.addFiles case _ => throw new IllegalArgumentException("Unexpected file index found!") } val matchedRowIndexSets = DeletionVectorBitmapGenerator.buildRowIndexSetsForFilesMatchingCondition( sparkSession, txn, hasDVsEnabled, targetDf, candidateFiles, condition) val nameToAddFileMap = generateCandidateFileMap(txn.deltaLog.dataPath, candidateFiles) findFilesWithMatchingRows(txn, nameToAddFileMap, matchedRowIndexSets) } } /** * Finds the files in nameToAddFileMap in which rows were deleted by checking the row index set. */ def findFilesWithMatchingRows( txn: OptimisticTransaction, nameToAddFileMap: Map[String, AddFile], matchedFileRowIndexSets: Seq[DeletionVectorResult]): Seq[TouchedFileWithDV] = { // Get the AddFiles using the touched file names and group them together with other // information we need for later phases. val dataPath = txn.deltaLog.dataPath val touchedFilesWithMatchedRowIndices = matchedFileRowIndexSets.map { fileRowIndex => val filePath = fileRowIndex.filePath val addFile = getTouchedFile(dataPath, filePath, nameToAddFileMap) TouchedFileWithDV( filePath, addFile, fileRowIndex.deletionVector, fileRowIndex.matchedRowCount) } logTrace("findTouchedFiles: matched files:\n\t" + s"${touchedFilesWithMatchedRowIndices.map(_.inputFilePath).mkString("\n\t")}") touchedFilesWithMatchedRowIndices.filterNot(_.isUnchanged) } def processUnmodifiedData( spark: SparkSession, touchedFiles: Seq[TouchedFileWithDV], snapshot: Snapshot, stringTruncateLength: Int): (Seq[FileAction], Map[String, Long]) = { val numModifiedRows = touchedFiles.map(_.numberOfModifiedRows).sum.toLong val numRemovedFiles = touchedFiles.count(_.isFullyReplaced()).toLong val (fullyRemovedFiles, notFullyRemovedFiles) = touchedFiles.partition(_.isFullyReplaced()) val timestamp = System.currentTimeMillis() val fullyRemoved = fullyRemovedFiles.map(_.fileLogEntry.removeWithTimestamp(timestamp)) val dvUpdates = notFullyRemovedFiles.map { fileWithDVInfo => fileWithDVInfo.fileLogEntry.removeRows( deletionVector = fileWithDVInfo.newDeletionVector, updateStats = false )} val (dvAddFiles, dvRemoveFiles) = dvUpdates.unzip val dvAddFilesWithStats = getActionsWithStats(spark, dvAddFiles, snapshot, stringTruncateLength) var (numDeletionVectorsAdded, numDeletionVectorsRemoved, numDeletionVectorsUpdated) = dvUpdates.foldLeft((0L, 0L, 0L)) { case ((added, removed, updated), (addFile, removeFile)) => (Option(addFile.deletionVector), Option(removeFile.deletionVector)) match { case (Some(_), Some(_)) => (added, removed, updated + 1) case (None, Some(_)) => (added, removed + 1, updated) case (Some(_), None) => (added + 1, removed, updated) case _ => (added, removed, updated) } } numDeletionVectorsRemoved += fullyRemoved.count(_.deletionVector != null) val metricMap = Map( "numModifiedRows" -> numModifiedRows, "numRemovedFiles" -> numRemovedFiles, "numDeletionVectorsAdded" -> numDeletionVectorsAdded, "numDeletionVectorsRemoved" -> numDeletionVectorsRemoved, "numDeletionVectorsUpdated" -> numDeletionVectorsUpdated) (fullyRemoved ++ dvAddFilesWithStats ++ dvRemoveFiles, metricMap) } /** Fetch stats for `addFiles`. */ private def getActionsWithStats( spark: SparkSession, addFilesWithNewDvs: Seq[AddFile], snapshot: Snapshot, stringTruncateLength: Int): Seq[AddFile] = { import org.apache.spark.sql.delta.implicits._ if (addFilesWithNewDvs.isEmpty) return Seq.empty val selectionPathAndStatsCols = Seq(col("path"), col("stats")) val addFilesWithNewDvsDf = addFilesWithNewDvs.toDF(spark) // These files originate from snapshot.filesForScan which resets column statistics. // Since these object don't carry stats and tags, if we were to use them as result actions of // the operation directly, we'd effectively be removing all stats and tags. To resolve this // we join the list of files with DVs with the log (allFiles) to retrieve statistics. This is // expected to have better performance than supporting full stats retrieval // in snapshot.filesForScan because it only affects a subset of the scanned files. // Find the current metadata with stats for all files with new DV val addFileWithStatsDf = snapshot.withStats .join(addFilesWithNewDvsDf.select("path"), "path") // Update the existing stats to set the tightBounds to false and also set the appropriate // null count. We want to set the bounds before the AddFile has DV descriptor attached. // Attaching the DV descriptor here, causes wrong logical records computation in // `updateStatsToWideBounds`. val statsColName = snapshot.getBaseStatsColumnName val addFilesWithWideBoundsDf = snapshot .updateStatsToWideBounds(addFileWithStatsDf, statsColName) val (filesWithNoStats, filesWithExistingStats) = { // numRecords is the only stat we really have to guarantee. // If the others are missing, we do not need to fetch them. addFilesWithWideBoundsDf.as[AddFile].collect().toSeq .partition(_.numPhysicalRecords.isEmpty) } // If we encounter files with no stats we fetch the stats from the parquet footer. // Files with persistent DVs *must* have (at least numRecords) stats according to the // Delta spec. val filesWithFetchedStats = if (filesWithNoStats.nonEmpty) { StatsCollectionUtils.computeStats(spark, conf = snapshot.deltaLog.newDeltaHadoopConf(), deltaLog = snapshot.deltaLog, snapshot = snapshot, addFiles = filesWithNoStats.toDS(spark), numFilesOpt = Some(filesWithNoStats.size), stringTruncateLength = stringTruncateLength, setBoundsToWide = true) .collect() .toSeq } else { Seq.empty } val allAddFilesWithUpdatedStats = (filesWithExistingStats ++ filesWithFetchedStats).toSeq.toDF(spark) // Now join the allAddFilesWithUpdatedStats with addFilesWithNewDvs // so that the updated stats are joined with the new DV info addFilesWithNewDvsDf.drop("stats") .join( allAddFilesWithUpdatedStats.select(selectionPathAndStatsCols: _*), "path") .as[AddFile] .collect() .toSeq } } object DeletionVectorBitmapGenerator { final val FILE_NAME_COL = "filePath" final val FILE_DV_ID_COL = "deletionVectorId" final val ROW_INDEX_COL = "rowIndexCol" final val DELETED_ROW_INDEX_BITMAP = "deletedRowIndexSet" final val DELETED_ROW_INDEX_COUNT = "deletedRowIndexCount" final val MAX_ROW_INDEX_COL = "maxRowIndexCol" private class DeletionVectorSet( spark: SparkSession, target: DataFrame, targetDeltaLog: DeltaLog, prefixLen: Int) { case object CardinalityAndBitmapStruct { val name: String = "CardinalityAndBitmapStruct" def cardinality: String = s"$name.cardinality" def bitmap: String = s"$name.bitmap" } def computeResult(): Seq[DeletionVectorResult] = { val aggregated = target .groupBy(col(FILE_NAME_COL), col(FILE_DV_ID_COL)) .agg(aggColumns.head, aggColumns.tail: _*) .select(outputColumns: _*) import DeletionVectorResult.encoder val rowIndexData = aggregated.as[DeletionVectorData] val storedResults = rowIndexData.mapPartitions(bitmapStorageMapper()) storedResults.as[DeletionVectorResult].collect() } protected def aggColumns: Seq[Column] = { Seq(createBitmapSetAggregator(col(ROW_INDEX_COL)).as(CardinalityAndBitmapStruct.name)) } /** Create a bitmap set aggregator over the given column */ private def createBitmapSetAggregator(indexColumn: Column): Column = { val func = new BitmapAggregator(indexColumn.expr, RoaringBitmapArrayFormat.Portable) Column(func.toAggregateExpression(isDistinct = false)) } protected def outputColumns: Seq[Column] = Seq( col(FILE_NAME_COL), col(FILE_DV_ID_COL), col(CardinalityAndBitmapStruct.bitmap).as(DELETED_ROW_INDEX_BITMAP), col(CardinalityAndBitmapStruct.cardinality).as(DELETED_ROW_INDEX_COUNT) ) protected def bitmapStorageMapper() : Iterator[DeletionVectorData] => Iterator[DeletionVectorResult] = { DeletionVectorWriter.createMapperToStoreDeletionVectors( spark, targetDeltaLog.newDeltaHadoopConf(), targetDeltaLog.dataPath, prefixLen) } } /** * Build bitmap compressed sets of row indices for each file in [[target]] using * [[ROW_INDEX_COL]]. * Write those sets out to temporary files and collect the file names, * together with some encoded metadata about the contents. * * @param target DataFrame with expected schema [[FILE_NAME_COL]], [[ROW_INDEX_COL]], */ def buildDeletionVectors( spark: SparkSession, target: DataFrame, targetDeltaLog: DeltaLog, prefixLen: Int): Seq[DeletionVectorResult] = { val rowIndexSet = new DeletionVectorSet(spark, target, targetDeltaLog, prefixLen) rowIndexSet.computeResult() } /** * Build a dataframe to find filtered rows with metadata (e.g., file name, row index and * existing DV) from candidate files for DV update. * * @param sparkSession the active spark session * @param tablePath table path of candidate files, used to absolutize path * @param tableHasDVs whether table has DV enabled * @param targetDf a scan of candidate files, whose attribute reference matches 'condition' * @param candidateFiles candidate files to be scanned, used to get existing DVs. * @param condition filter to be applied, whose attribute reference matches 'targetDf' * @param fileNameColumnOpt optional overwrite of file name column name * @param rowIndexColumnOpt optional overwrite of row index column name * @return a dataframe containing filtered rows with corresponding metadata for DV update. */ def buildRowIndexSetsForFilesMatchingCondition( sparkSession: SparkSession, tablePath: String, tableHasDVs: Boolean, targetDf: DataFrame, candidateFiles: Seq[AddFile], condition: Expression, fileNameColumnOpt: Option[Column], rowIndexColumnOpt: Option[Column]): DataFrame = { val useMetadataRowIndexConf = DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX val useMetadataRowIndex = sparkSession.sessionState.conf.getConf(useMetadataRowIndexConf) val fileNameColumn = fileNameColumnOpt.getOrElse(col(s"${METADATA_NAME}.${FILE_PATH}")) val rowIndexColumn = if (useMetadataRowIndex) { rowIndexColumnOpt.getOrElse(col(s"${METADATA_NAME}.${ParquetFileFormat.ROW_INDEX}")) } else { rowIndexColumnOpt.getOrElse(col(ROW_INDEX_COLUMN_NAME)) } val matchedRowsDf = targetDf .withColumn(FILE_NAME_COL, fileNameColumn) // Filter after getting input file name as the filter might introduce a join and we // cannot get input file name on join's output. .filter(Column(condition)) .withColumn(ROW_INDEX_COL, rowIndexColumn) if (tableHasDVs) { // When the table already has DVs, join the `matchedRowDf` above to attach for each matched // file its existing DeletionVectorDescriptor val filePathToDV = candidateFiles.map { add => val serializedDV = Option(add.deletionVector).map(_.serializeToBase64()) // Paths in the metadata column are canonicalized. Thus we must canonicalize the DV path. FileToDvDescriptor( SparkPath.fromPath(absolutePath(tablePath, add.path)).urlEncoded, serializedDV) } val filePathToDVDf = sparkSession.createDataset(filePathToDV) val joinExpr = filePathToDVDf("path") === matchedRowsDf(FILE_NAME_COL) // Perform leftOuter join to make sure we do not eliminate any rows because of path // encoding issues. If there is such an issue we will detect it during the aggregation // of the bitmaps. val joinedDf = matchedRowsDf.join(filePathToDVDf, joinExpr, "leftOuter") .drop(FILE_NAME_COL) .withColumnRenamed("path", FILE_NAME_COL) joinedDf } else { // When the table has no DVs, just add a column to indicate that the existing dv is null matchedRowsDf.withColumn(FILE_DV_ID_COL, lit(null)) } } /** The same as above, except it also updates DVs for the table using the dataframe. */ def buildRowIndexSetsForFilesMatchingCondition( sparkSession: SparkSession, txn: OptimisticTransaction, tableHasDVs: Boolean, targetDf: DataFrame, candidateFiles: Seq[AddFile], condition: Expression, fileNameColumnOpt: Option[Column] = None, rowIndexColumnOpt: Option[Column] = None): Seq[DeletionVectorResult] = { val df = buildRowIndexSetsForFilesMatchingCondition( sparkSession, txn.deltaLog.dataPath.toString, tableHasDVs, targetDf, candidateFiles, condition, fileNameColumnOpt, rowIndexColumnOpt ) DeletionVectorBitmapGenerator.buildDeletionVectors( sparkSession, df, txn.deltaLog, DeltaUtils.getRandomPrefixLength(txn.metadata)) } } /** * Holds a mapping from a file path (url-encoded) to an (optional) serialized Deletion Vector * descriptor. */ case class FileToDvDescriptor(path: String, deletionVectorId: Option[String]) object FileToDvDescriptor { private lazy val _encoder = new DeltaEncoder[FileToDvDescriptor] implicit def encoder: Encoder[FileToDvDescriptor] = _encoder.get } /** * Row containing the file path and its new deletion vector bitmap in memory * * @param filePath Absolute path of the data file this DV result is generated for. * @param deletionVectorId Existing [[DeletionVectorDescriptor]] serialized in JSON format. * This info is used to load the existing DV with the new DV. * @param deletedRowIndexSet In-memory Deletion vector bitmap generated containing the newly * deleted row indexes from data file. * @param deletedRowIndexCount Count of rows marked as deleted using the [[deletedRowIndexSet]]. */ case class DeletionVectorData( filePath: String, deletionVectorId: Option[String], deletedRowIndexSet: Array[Byte], deletedRowIndexCount: Long) extends Sizing { @transient lazy val deletionVectorDescriptor: Option[DeletionVectorDescriptor] = deletionVectorId.map { id => DeletionVectorDescriptor.deserializeFromBase64(id) } /** The size of the bitmaps to use in [[BinPackingIterator]]. */ override def size: Int = { val sizeWithoutExistingDV: Int = deletedRowIndexSet.length // Add the size of the existing DV that we will eventually merge with, so that // [[BinPackingIterator]] can get a better estimate. It's an estimate since the // row indices are not merged and serialized. We add the size of the checksum // and the size of the data size, which are fixed sizes added for every DV. val sizeWithDV = sizeWithoutExistingDV + deletionVectorDescriptor.map(_.sizeInBytes).getOrElse(0) + DeletionVectorStore.CHECKSUM_LEN + DeletionVectorStore.DATA_SIZE_LEN // If we have an int overflow, we can end up with a negative size. In that case, // let's return the maximum value of Int and fail later when writing the DV writer. if (sizeWithDV < 0) { Int.MaxValue } else { sizeWithDV } } } object DeletionVectorData { private lazy val _encoder = new DeltaEncoder[DeletionVectorData] implicit def encoder: Encoder[DeletionVectorData] = _encoder.get def apply(filePath: String, rowIndexSet: Array[Byte], rowIndexCount: Long): DeletionVectorData = { DeletionVectorData( filePath = filePath, deletionVectorId = None, deletedRowIndexSet = rowIndexSet, deletedRowIndexCount = rowIndexCount) } } /** Final output for each file containing the file path, DeletionVectorDescriptor and how many * rows are marked as deleted in this file as part of the this operation (doesn't include rows that * are already marked as deleted). * * @param filePath Absolute path of the data file this DV result is generated for. * @param deletionVector Deletion vector generated containing the newly deleted row indices from * data file. * @param matchedRowCount Number of rows marked as deleted using the [[deletionVector]]. */ case class DeletionVectorResult( filePath: String, deletionVector: DeletionVectorDescriptor, matchedRowCount: Long) { } object DeletionVectorResult { private lazy val _encoder = new DeltaEncoder[DeletionVectorResult] implicit def encoder: Encoder[DeletionVectorResult] = _encoder.get def fromDeletionVectorData( data: DeletionVectorData, deletionVector: DeletionVectorDescriptor): DeletionVectorResult = { DeletionVectorResult( filePath = data.filePath, deletionVector = deletionVector, matchedRowCount = data.deletedRowIndexCount) } } case class TouchedFileWithDV( inputFilePath: String, fileLogEntry: AddFile, newDeletionVector: DeletionVectorDescriptor, deletedRows: Long) { /** * Checks the *sufficient* condition for a file being fully replaced by the current operation. * (That is, all rows are either being updated or deleted.) */ def isFullyReplaced(): Boolean = { fileLogEntry.numLogicalRecords match { case Some(numRecords) => numRecords == numberOfModifiedRows case None => false // must make defensive assumption if no statistics are available } } /** * Checks if the file is unchanged by the current operation. * (That is no row has been updated or deleted.) */ def isUnchanged: Boolean = { // If the bitmap is empty then no row would be removed during the rewrite, // thus the file is unchanged. numberOfModifiedRows == 0 } /** * The number of rows that are modified in this file. */ def numberOfModifiedRows: Long = newDeletionVector.cardinality - fileLogEntry.numDeletedRecords } /** * Utility methods to write the deletion vector to storage. If a particular file already * has an existing DV, it will be merged with the new deletion vector and written to storage. */ object DeletionVectorWriter extends DeltaLogging { /** * The context for [[createDeletionVectorMapper]] callback functions. Contains the DV writer that * is used by callback functions to write the new DVs. */ case class DeletionVectorMapperContext( dvStore: DeletionVectorStore, writer: DeletionVectorStore.Writer, tablePath: Path, fileId: UUID, prefix: String) /** * Prepare a mapper function for storing deletion vectors. * * For each DeletionVector the writer will create a [[DeletionVectorMapperContext]] that contains * a DV writer that is used by to write the DV into a file. * * The result can be used with [[org.apache.spark.sql.Dataset.mapPartitions()]] and must thus be * serialized. */ def createDeletionVectorMapper[InputT <: Sizing, OutputT]( sparkSession: SparkSession, hadoopConf: Configuration, table: Path, prefixLength: Int) (callbackFn: (DeletionVectorMapperContext, InputT) => OutputT) : Iterator[InputT] => Iterator[OutputT] = { val broadcastHadoopConf = sparkSession.sparkContext.broadcast( new SerializableConfiguration(hadoopConf)) // hadoop.fs.Path is not Serializable, so close over the String representation instead val tablePathString = DeletionVectorStore.pathToEscapedString(table) val packingTargetSize = sparkSession.conf.get(DeltaSQLConf.DELETION_VECTOR_PACKING_TARGET_SIZE) // This is the (partition) mapper function we are returning (rowIterator: Iterator[InputT]) => { val dvStore = DeletionVectorStore.createInstance(broadcastHadoopConf.value.value) val tablePath = DeletionVectorStore.escapedStringToPath(tablePathString) val tablePathWithFS = dvStore.pathWithFileSystem(tablePath) val perBinFunction: Seq[InputT] => Seq[OutputT] = (rows: Seq[InputT]) => { val prefix = DeltaUtils.getRandomPrefix(prefixLength) val (writer, fileId) = createWriter(dvStore, tablePathWithFS, prefix) val ctx = DeletionVectorMapperContext( dvStore, writer, tablePath, fileId, prefix) val result = SparkUtils.tryWithResource(writer) { writer => rows.map(r => callbackFn(ctx, r)) } result } val binPackedRowIterator = new BinPackingIterator(rowIterator, packingTargetSize) binPackedRowIterator.flatMap(perBinFunction) } } /** * Creates a writer for writing multiple DVs in the same file. * * Returns the writer and the UUID of the new file. */ def createWriter( dvStore: DeletionVectorStore, tablePath: PathWithFileSystem, prefix: String = ""): (DeletionVectorStore.Writer, UUID) = { val fileId = UUID.randomUUID() val writer = dvStore.createWriter(dvStore.generateFileNameInTable(tablePath, fileId, prefix)) (writer, fileId) } /** Store the `bitmapData` on cloud storage. */ def storeSerializedBitmap( ctx: DeletionVectorMapperContext, bitmapData: Array[Byte], cardinality: Long): DeletionVectorDescriptor = { if (cardinality == 0L) { DeletionVectorDescriptor.EMPTY } else { val dvRange = ctx.writer.write(bitmapData) DeletionVectorDescriptor.onDiskWithRelativePath( id = ctx.fileId, randomPrefix = ctx.prefix, sizeInBytes = bitmapData.length, cardinality = cardinality, offset = Some(dvRange.offset)) } } /** * Prepares a mapper function that can be used by DML commands to store the Deletion Vectors * that are in described in [[DeletionVectorData]] and return their descriptors * [[DeletionVectorResult]]. */ def createMapperToStoreDeletionVectors( sparkSession: SparkSession, hadoopConf: Configuration, table: Path, prefixLength: Int): Iterator[DeletionVectorData] => Iterator[DeletionVectorResult] = createDeletionVectorMapper(sparkSession, hadoopConf, table, prefixLength) { (ctx, row) => storeBitmapAndGenerateResult(ctx, row) } /** * Helper to generate and store the deletion vector bitmap. The deletion vector is merged with * the file's already existing deletion vector before being stored. */ def storeBitmapAndGenerateResult(ctx: DeletionVectorMapperContext, row: DeletionVectorData) : DeletionVectorResult = { // If a group with null path exists it means there was an issue while joining with the log to // fetch the DeletionVectorDescriptors. assert(row.filePath != null, s""" |Encountered a non matched file path. |It is likely that _metadata.file_path is not encoded by Spark as expected. |""".stripMargin) val fileDvDescriptor = row.deletionVectorDescriptor val finalDvDescriptor = fileDvDescriptor match { case Some(existingDvDescriptor) if row.deletedRowIndexCount > 0 => // Load the existing bit map val existingBitmap = StoredBitmap.create(existingDvDescriptor, ctx.tablePath).load(ctx.dvStore) val newBitmap = DeletionVectorUtils.deserialize(row.deletedRowIndexSet, Some(ctx.tablePath)) // Merge both the existing and new bitmaps into one, and finally persist on disk existingBitmap.merge(newBitmap) val serializedBitmap = DeletionVectorUtils.serialize( existingBitmap, RoaringBitmapArrayFormat.Portable, Some(ctx.tablePath), debugInfo = Map("existingDvDescriptor" -> existingDvDescriptor)) storeSerializedBitmap( ctx, serializedBitmap, existingBitmap.cardinality) case Some(existingDvDescriptor) => existingDvDescriptor // This is already stored. case None => // Persist the new bitmap storeSerializedBitmap(ctx, row.deletedRowIndexSet, row.deletedRowIndexCount) } DeletionVectorResult.fromDeletionVectorData(row, deletionVector = finalDvDescriptor) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands import java.util.concurrent.TimeUnit import scala.util.control.NonFatal import org.apache.spark.sql.delta.metric.IncrementMetric import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions.{Action, AddCDCFile, AddFile, FileAction} import org.apache.spark.sql.delta.commands.DeleteCommand.{rewritingFilesMsg, FINDING_TOUCHED_FILES_MSG} import org.apache.spark.sql.delta.commands.MergeIntoCommandBase.totalBytesAndDistinctPartitionValues import org.apache.spark.sql.delta.files.TahoeBatchFileIndex import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.StatsCollectionUtils import com.fasterxml.jackson.databind.annotation.JsonDeserialize import org.apache.spark.SparkContext import org.apache.spark.sql.{Column, DataFrame, Dataset, Row, SparkSession} import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, EqualNullSafe, Expression, If, Literal, Not} import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral import org.apache.spark.sql.catalyst.plans.QueryPlan import org.apache.spark.sql.catalyst.plans.logical.{DeltaDelete, LogicalPlan} import org.apache.spark.sql.delta.DeltaOperations.Operation import org.apache.spark.sql.execution.command.LeafRunnableCommand import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.execution.metric.SQLMetrics.{createMetric, createTimingMetric} import org.apache.spark.sql.functions.input_file_name import org.apache.spark.sql.types.LongType trait DeleteCommandMetrics { self: LeafRunnableCommand => @transient private lazy val sc: SparkContext = SparkContext.getOrCreate() def createMetrics: Map[String, SQLMetric] = Map[String, SQLMetric]( "numRemovedFiles" -> createMetric(sc, "number of files removed."), "numAddedFiles" -> createMetric(sc, "number of files added."), "numDeletedRows" -> createMetric(sc, "number of rows deleted."), "numFilesBeforeSkipping" -> createMetric(sc, "number of files before skipping"), "numBytesBeforeSkipping" -> createMetric(sc, "number of bytes before skipping"), "numFilesAfterSkipping" -> createMetric(sc, "number of files after skipping"), "numBytesAfterSkipping" -> createMetric(sc, "number of bytes after skipping"), "numPartitionsAfterSkipping" -> createMetric(sc, "number of partitions after skipping"), "numPartitionsAddedTo" -> createMetric(sc, "number of partitions added"), "numPartitionsRemovedFrom" -> createMetric(sc, "number of partitions removed"), "numCopiedRows" -> createMetric(sc, "number of rows copied"), "numAddedBytes" -> createMetric(sc, "number of bytes added"), "numRemovedBytes" -> createMetric(sc, "number of bytes removed"), "executionTimeMs" -> createTimingMetric(sc, "time taken to execute the entire operation"), "scanTimeMs" -> createTimingMetric(sc, "time taken to scan the files for matches"), "rewriteTimeMs" -> createTimingMetric(sc, "time taken to rewrite the matched files"), "numAddedChangeFiles" -> createMetric(sc, "number of change data capture files generated"), "changeFileBytes" -> createMetric(sc, "total size of change data capture files generated"), "numTouchedRows" -> createMetric(sc, "number of rows touched"), "numDeletionVectorsAdded" -> createMetric(sc, "number of deletion vectors added"), "numDeletionVectorsRemoved" -> createMetric(sc, "number of deletion vectors removed"), "numDeletionVectorsUpdated" -> createMetric(sc, "number of deletion vectors updated") ) def getDeletedRowsFromAddFilesAndUpdateMetrics(files: Seq[AddFile]) : Option[Long] = { if (!conf.getConf(DeltaSQLConf.DELTA_DML_METRICS_FROM_METADATA)) { return None; } // No file to get metadata, return none to be consistent with metadata stats disabled if (files.isEmpty) { return None } // Return None if any file does not contain numLogicalRecords status var count: Long = 0 for (file <- files) { if (file.numLogicalRecords.isEmpty) { return None } count += file.numLogicalRecords.get } metrics("numDeletedRows").set(count) return Some(count) } } /** * Performs a Delete based on the search condition * * Algorithm: * 1) Scan all the files and determine which files have * the rows that need to be deleted. * 2) Traverse the affected files and rebuild the touched files. * 3) Use the Delta protocol to atomically write the remaining rows to new files and remove * the affected files that are identified in step 1. */ case class DeleteCommand( deltaLog: DeltaLog, catalogTable: Option[CatalogTable], target: LogicalPlan, condition: Option[Expression]) extends LeafRunnableCommand with DeltaCommand with DeleteCommandMetrics { override def innerChildren: Seq[QueryPlan[_]] = Seq(target) override val output: Seq[Attribute] = Seq(AttributeReference("num_affected_rows", LongType)()) override lazy val metrics = createMetrics final override def run(sparkSession: SparkSession): Seq[Row] = { recordDeltaOperation(deltaLog, "delta.dml.delete") { deltaLog.withNewTransaction(catalogTable) { txn => DeltaLog.assertRemovable(txn.snapshot) if (hasBeenExecuted(txn, sparkSession)) { sendDriverMetrics(sparkSession, metrics) return Seq.empty } val (deleteActions, deleteMetrics) = performDelete(sparkSession, deltaLog, txn) val numRecordsStats = NumRecordsStats.fromActions(deleteActions) val operation = DeltaOperations.Delete(condition.toSeq) validateNumRecords(deleteActions, numRecordsStats, operation) val commitVersion = txn.commitIfNeeded( actions = deleteActions, op = operation, tags = RowTracking.addPreservedRowTrackingTagIfNotSet(txn.snapshot)) recordDeltaEvent( deltaLog, "delta.dml.delete.stats", data = deleteMetrics.copy(commitVersion = commitVersion)) } // Re-cache all cached plans(including this relation itself, if it's cached) that refer to // this data source relation. sparkSession.sharedState.cacheManager.recacheByPlan(sparkSession, target) } // Adjust for deletes at partition boundaries. Deletes at partition boundaries is a metadata // operation, therefore we don't actually have any information around how many rows were deleted // While this info may exist in the file statistics, it's not guaranteed that we have these // statistics. To avoid any performance regressions, we currently just return a -1 in such cases if (metrics("numRemovedFiles").value > 0 && metrics("numDeletedRows").value == 0) { Seq(Row(-1L)) } else { Seq(Row(metrics("numDeletedRows").value)) } } def performDelete( sparkSession: SparkSession, deltaLog: DeltaLog, txn: OptimisticTransaction): (Seq[Action], DeleteMetric) = { import org.apache.spark.sql.delta.implicits._ var numRemovedFiles: Long = 0 var numAddedFiles: Long = 0 var numAddedChangeFiles: Long = 0 var scanTimeMs: Long = 0 var rewriteTimeMs: Long = 0 var numAddedBytes: Long = 0 var changeFileBytes: Long = 0 var numRemovedBytes: Long = 0 var numFilesBeforeSkipping: Long = 0 var numBytesBeforeSkipping: Long = 0 var numFilesAfterSkipping: Long = 0 var numBytesAfterSkipping: Long = 0 var numPartitionsAfterSkipping: Option[Long] = None var numPartitionsRemovedFrom: Option[Long] = None var numPartitionsAddedTo: Option[Long] = None var numDeletedRows: Option[Long] = None var numCopiedRows: Option[Long] = None var numDeletionVectorsAdded: Long = 0 var numDeletionVectorsRemoved: Long = 0 var numDeletionVectorsUpdated: Long = 0 val startTime = System.nanoTime() val numFilesTotal = txn.snapshot.numOfFiles val deleteActions: Seq[Action] = condition match { case None => // Case 1: Delete the whole table if the condition is true val reportRowLevelMetrics = conf.getConf(DeltaSQLConf.DELTA_DML_METRICS_FROM_METADATA) val allFiles = txn.filterFiles(Nil, keepNumRecords = reportRowLevelMetrics) numRemovedFiles = allFiles.size numDeletionVectorsRemoved = allFiles.count(_.deletionVector != null) scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 val (numBytes, numPartitions) = totalBytesAndDistinctPartitionValues(allFiles) numRemovedBytes = numBytes numFilesBeforeSkipping = numRemovedFiles numBytesBeforeSkipping = numBytes numFilesAfterSkipping = numRemovedFiles numBytesAfterSkipping = numBytes numDeletedRows = getDeletedRowsFromAddFilesAndUpdateMetrics(allFiles) if (txn.metadata.partitionColumns.nonEmpty) { numPartitionsAfterSkipping = Some(numPartitions) numPartitionsRemovedFrom = Some(numPartitions) numPartitionsAddedTo = Some(0) } val operationTimestamp = System.currentTimeMillis() allFiles.map(_.removeWithTimestamp(operationTimestamp)) case Some(cond) => val (metadataPredicates, otherPredicates) = DeltaTableUtils.splitMetadataAndDataPredicates( cond, txn.metadata.partitionColumns, sparkSession) numFilesBeforeSkipping = txn.snapshot.numOfFiles numBytesBeforeSkipping = txn.snapshot.sizeInBytes if (otherPredicates.isEmpty) { // Case 2: The condition can be evaluated using metadata only. // Delete a set of files without the need of scanning any data files. val operationTimestamp = System.currentTimeMillis() val reportRowLevelMetrics = conf.getConf(DeltaSQLConf.DELTA_DML_METRICS_FROM_METADATA) val candidateFiles = txn.filterFiles(metadataPredicates, keepNumRecords = reportRowLevelMetrics) scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 numRemovedFiles = candidateFiles.size numRemovedBytes = candidateFiles.map(_.size).sum numFilesAfterSkipping = candidateFiles.size numDeletionVectorsRemoved = candidateFiles.count(_.deletionVector != null) val (numCandidateBytes, numCandidatePartitions) = totalBytesAndDistinctPartitionValues(candidateFiles) numBytesAfterSkipping = numCandidateBytes numDeletedRows = getDeletedRowsFromAddFilesAndUpdateMetrics(candidateFiles) if (txn.metadata.partitionColumns.nonEmpty) { numPartitionsAfterSkipping = Some(numCandidatePartitions) numPartitionsRemovedFrom = Some(numCandidatePartitions) numPartitionsAddedTo = Some(0) } candidateFiles.map(_.removeWithTimestamp(operationTimestamp)) } else { // Case 3: Delete the rows based on the condition. // Should we write the DVs to represent the deleted rows? val shouldWriteDVs = shouldWritePersistentDeletionVectors(sparkSession, txn) val candidateFiles = txn.filterFiles( metadataPredicates ++ otherPredicates, keepNumRecords = shouldWriteDVs) // `candidateFiles` contains the files filtered using statistics and delete condition // They may or may not contains any rows that need to be deleted. numFilesAfterSkipping = candidateFiles.size val (numCandidateBytes, numCandidatePartitions) = totalBytesAndDistinctPartitionValues(candidateFiles) numBytesAfterSkipping = numCandidateBytes if (txn.metadata.partitionColumns.nonEmpty) { numPartitionsAfterSkipping = Some(numCandidatePartitions) } val nameToAddFileMap = generateCandidateFileMap(deltaLog.dataPath, candidateFiles) val fileIndex = new TahoeBatchFileIndex( sparkSession, "delete", candidateFiles, deltaLog, deltaLog.dataPath, txn.snapshot) if (shouldWriteDVs) { val targetDf = DMLWithDeletionVectorsHelper.createTargetDfForScanningForMatches( sparkSession, target, fileIndex) // Does the target table already has DVs enabled? If so, we need to read the table // with deletion vectors. val mustReadDeletionVectors = DeletionVectorUtils.deletionVectorsReadable(txn.snapshot) val touchedFiles = DMLWithDeletionVectorsHelper.findTouchedFiles( sparkSession, txn, mustReadDeletionVectors, deltaLog, targetDf, fileIndex, cond, opName = "DELETE") if (touchedFiles.nonEmpty) { val stringTruncateLength = StatsCollectionUtils.getDataSkippingStringPrefixLength( sparkSession, txn.metadata) val (actions, metricMap) = DMLWithDeletionVectorsHelper.processUnmodifiedData( sparkSession, touchedFiles, txn.snapshot, stringTruncateLength) metrics("numDeletedRows").set(metricMap("numModifiedRows")) numDeletionVectorsAdded = metricMap("numDeletionVectorsAdded") numDeletionVectorsRemoved = metricMap("numDeletionVectorsRemoved") numDeletionVectorsUpdated = metricMap("numDeletionVectorsUpdated") numRemovedFiles = metricMap("numRemovedFiles") actions } else { Nil // Nothing to update } } else { // Keep everything from the resolved target except a new TahoeFileIndex // that only involves the affected files instead of all files. val newTarget = DeltaTableUtils.replaceFileIndex(target, fileIndex) val data = DataFrameUtils.ofRows(sparkSession, newTarget) val incrDeletedCountExpr = IncrementMetric(TrueLiteral, metrics("numDeletedRows")) val filesToRewrite = withStatusCode("DELTA", FINDING_TOUCHED_FILES_MSG) { if (candidateFiles.isEmpty) { Array.empty[String] } else { data.filter(Column(cond)) .select(input_file_name()) .filter(Column(incrDeletedCountExpr)) .distinct() .as[String] .collect() } } numRemovedFiles = filesToRewrite.length scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 if (filesToRewrite.isEmpty) { // Case 3.1: no row matches and no delete will be triggered if (txn.metadata.partitionColumns.nonEmpty) { numPartitionsRemovedFrom = Some(0) numPartitionsAddedTo = Some(0) } Nil } else { // Case 3.2: some files need an update to remove the deleted files // Do the second pass and just read the affected files val baseRelation = buildBaseRelation( sparkSession, txn, "delete", deltaLog.dataPath, filesToRewrite, nameToAddFileMap) // Keep everything from the resolved target except a new TahoeFileIndex // that only involves the affected files instead of all files. val newTarget = DeltaTableUtils.replaceFileIndex(target, baseRelation.location) val targetDF = RowTracking.preserveRowTrackingColumns( dfWithoutRowTrackingColumns = DataFrameUtils.ofRows(sparkSession, newTarget), snapshot = txn.snapshot) val filterCond = Not(EqualNullSafe(cond, Literal.TrueLiteral)) val rewrittenActions = rewriteFiles(txn, targetDF, filterCond, filesToRewrite.length) val (changeFiles, rewrittenFiles) = rewrittenActions .partition(_.isInstanceOf[AddCDCFile]) numAddedFiles = rewrittenFiles.size val removedFiles = filesToRewrite.map(f => getTouchedFile(deltaLog.dataPath, f, nameToAddFileMap)) val (removedBytes, removedPartitions) = totalBytesAndDistinctPartitionValues(removedFiles) numRemovedBytes = removedBytes val (rewrittenBytes, rewrittenPartitions) = totalBytesAndDistinctPartitionValues(rewrittenFiles) numAddedBytes = rewrittenBytes if (txn.metadata.partitionColumns.nonEmpty) { numPartitionsRemovedFrom = Some(removedPartitions) numPartitionsAddedTo = Some(rewrittenPartitions) } numAddedChangeFiles = changeFiles.size changeFileBytes = changeFiles.collect { case f: AddCDCFile => f.size }.sum rewriteTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime) - scanTimeMs numDeletedRows = Some(metrics("numDeletedRows").value) numCopiedRows = Some(metrics("numTouchedRows").value - metrics("numDeletedRows").value) numDeletionVectorsRemoved = removedFiles.count(_.deletionVector != null) val operationTimestamp = System.currentTimeMillis() removeFilesFromPaths( deltaLog, nameToAddFileMap, filesToRewrite, operationTimestamp) ++ rewrittenActions } } } } metrics("numRemovedFiles").set(numRemovedFiles) metrics("numAddedFiles").set(numAddedFiles) val executionTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 metrics("executionTimeMs").set(executionTimeMs) metrics("scanTimeMs").set(scanTimeMs) metrics("rewriteTimeMs").set(rewriteTimeMs) metrics("numAddedChangeFiles").set(numAddedChangeFiles) metrics("changeFileBytes").set(changeFileBytes) metrics("numAddedBytes").set(numAddedBytes) metrics("numRemovedBytes").set(numRemovedBytes) metrics("numFilesBeforeSkipping").set(numFilesBeforeSkipping) metrics("numBytesBeforeSkipping").set(numBytesBeforeSkipping) metrics("numFilesAfterSkipping").set(numFilesAfterSkipping) metrics("numBytesAfterSkipping").set(numBytesAfterSkipping) metrics("numDeletionVectorsAdded").set(numDeletionVectorsAdded) metrics("numDeletionVectorsRemoved").set(numDeletionVectorsRemoved) metrics("numDeletionVectorsUpdated").set(numDeletionVectorsUpdated) numPartitionsAfterSkipping.foreach(metrics("numPartitionsAfterSkipping").set) numPartitionsAddedTo.foreach(metrics("numPartitionsAddedTo").set) numPartitionsRemovedFrom.foreach(metrics("numPartitionsRemovedFrom").set) numCopiedRows.foreach(metrics("numCopiedRows").set) txn.registerSQLMetrics(sparkSession, metrics) sendDriverMetrics(sparkSession, metrics) val numRecordsStats = NumRecordsStats.fromActions(deleteActions) val deleteMetric = DeleteMetric( condition = condition.map(_.sql).getOrElse("true"), numFilesTotal, numFilesAfterSkipping, numAddedFiles, numRemovedFiles, numAddedFiles, numAddedChangeFiles = numAddedChangeFiles, numFilesBeforeSkipping, numBytesBeforeSkipping, numFilesAfterSkipping, numBytesAfterSkipping, numPartitionsAfterSkipping, numPartitionsAddedTo, numPartitionsRemovedFrom, numCopiedRows, numDeletedRows, numAddedBytes, numRemovedBytes, changeFileBytes = changeFileBytes, scanTimeMs, rewriteTimeMs, numDeletionVectorsAdded, numDeletionVectorsRemoved, numDeletionVectorsUpdated, numLogicalRecordsAdded = numRecordsStats.numLogicalRecordsAdded, numLogicalRecordsRemoved = numRecordsStats.numLogicalRecordsRemoved) val actionsToCommit = if (deleteActions.nonEmpty) { createSetTransaction(sparkSession, deltaLog).toSeq ++ deleteActions } else { Seq.empty } (actionsToCommit, deleteMetric) } /** * Returns the list of [[AddFile]]s and [[AddCDCFile]]s that have been re-written. */ private def rewriteFiles( txn: OptimisticTransaction, baseData: DataFrame, filterCondition: Expression, numFilesToRewrite: Long): Seq[FileAction] = { val shouldWriteCdc = DeltaConfigs.CHANGE_DATA_FEED.fromMetaData(txn.metadata) // number of total rows that we have seen / are either copying or deleting (sum of both). val incrTouchedCountExpr = IncrementMetric(TrueLiteral, metrics("numTouchedRows")) withStatusCode( "DELTA", rewritingFilesMsg(numFilesToRewrite)) { val dfToWrite = if (shouldWriteCdc) { import org.apache.spark.sql.delta.commands.cdc.CDCReader._ // The logic here ends up being surprisingly elegant, with all source rows ending up in // the output. Recall that we flipped the user-provided delete condition earlier, before the // call to `rewriteFiles`. All rows which match this latest `filterCondition` are retained // as table data, while all rows which don't match are removed from the rewritten table data // but do get included in the output as CDC events. baseData .filter(Column(incrTouchedCountExpr)) .withColumn( CDC_TYPE_COLUMN_NAME, Column(If(filterCondition, CDC_TYPE_NOT_CDC, CDC_TYPE_DELETE)) ) } else { baseData .filter(Column(incrTouchedCountExpr)) .filter(Column(filterCondition)) } txn.writeFiles(dfToWrite) } } def shouldWritePersistentDeletionVectors( spark: SparkSession, txn: OptimisticTransaction): Boolean = { spark.conf.get(DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS) && DeletionVectorUtils.deletionVectorsWritable(txn.snapshot) } /** * Validates that the number of records does not increase. * * Note: ideally we would also compare the number of added/removed rows in the statistics with the * number of deleted/copied rows in the SQL metrics, but unfortunately this is not possible, as * sql metrics are not reliable when there are task or stage retries. */ private def validateNumRecords( actions: Seq[Action], numRecordsStats: NumRecordsStats, op: Operation): Unit = { (numRecordsStats.numLogicalRecordsAdded, numRecordsStats.numLogicalRecordsRemoved, numRecordsStats.numLogicalRecordsAddedInFilesWithDeletionVectors) match { case ( Some(numAddedRecords), Some(numRemovedRecords), Some(numRecordsNotCopied)) => if (numAddedRecords > numRemovedRecords) { logNumRecordsMismatch(deltaLog, actions, numRecordsStats, op) if (conf.getConf(DeltaSQLConf.NUM_RECORDS_VALIDATION_ENABLED)) { throw DeltaErrors.numRecordsMismatch( operation = "DELETE", numAddedRecords, numRemovedRecords ) } } if (conf.getConf(DeltaSQLConf.COMMAND_INVARIANT_CHECKS_USE_UNRELIABLE)) { // and also using regular (unreliable) metrics for baseline validateMetricBasedCommandInvariants( numAddedRecords, numRemovedRecords, numRecordsNotCopied, op, deltaLog) } case _ => recordDeltaEvent(deltaLog, opType = "delta.assertions.statsNotPresentForNumRecordsCheck") logWarning(log"Could not validate number of records due to missing statistics.") } } private def validateMetricBasedCommandInvariants( numAddedRecords: Long, numRemovedRecords: Long, numRecordsNotCopied: Long, op: Operation, deltaLog: DeltaLog): Unit = try { val numRowsDeleted = CommandInvariantMetricValueFromSingle(metrics("numDeletedRows")) val numRowsCopied = CommandInvariantMetricValueFromSingle(metrics("numCopiedRows")) val recordMetricsFromMetadata = conf.getConf(DeltaSQLConf.DELTA_DML_METRICS_FROM_METADATA) if (numRowsDeleted.getOrDummy == 0 && !recordMetricsFromMetadata) { // If we don't record metrics we can't use them to perform invariant checks. return } checkCommandInvariant( invariant = () => numRowsDeleted.getOrThrow + numRowsCopied.getOrThrow + numRecordsNotCopied == numRemovedRecords, label = "numRowsDeleted + numRowsCopied + numRecordsNotCopied + " + "numRowsRemovedByMetadataOnlyDelete == numRemovedRecords", op = op, deltaLog = deltaLog, parameters = Map( "numRowsDeleted" -> numRowsDeleted.getOrDummy, "numRowsCopied" -> numRowsCopied.getOrDummy, "numRemovedRecords" -> numRemovedRecords, "numRecordsNotCopied" -> numRecordsNotCopied ), additionalInfo = Map( DeltaSQLConf.DELTA_DML_METRICS_FROM_METADATA.key -> recordMetricsFromMetadata.toString ) ) checkCommandInvariant( invariant = () => numRowsCopied.getOrThrow + numRecordsNotCopied == numAddedRecords, label = "numRowsCopied + numRecordsNotCopied == numAddedRecords", op = op, deltaLog = deltaLog, parameters = Map( "numRowsCopied" -> numRowsCopied.getOrDummy, "numAddedRecords" -> numAddedRecords, "numRecordsNotCopied" -> numRecordsNotCopied ), additionalInfo = Map( DeltaSQLConf.DELTA_DML_METRICS_FROM_METADATA.key -> recordMetricsFromMetadata.toString ) ) } catch { // Immediately re-throw actual command invariant violations, so we don't re-wrap them below. case e: DeltaIllegalStateException if e.getErrorClass == "DELTA_COMMAND_INVARIANT_VIOLATION" => throw e case NonFatal(e) => logWarning(log"Unexpected error in validateMetricBasedCommandInvariants", e) checkCommandInvariant( invariant = () => false, label = "Unexpected error in validateMetricBasedCommandInvariants", op = op, deltaLog = deltaLog, parameters = Map.empty ) } } object DeleteCommand { def apply(delete: DeltaDelete): DeleteCommand = { EliminateSubqueryAliases(delete.child) match { case DeltaFullTable(relation, fileIndex) => DeleteCommand(fileIndex.deltaLog, relation.catalogTable, delete.child, delete.condition) case o => throw DeltaErrors.notADeltaSourceException("DELETE", Some(o)) } } val FILE_NAME_COLUMN: String = "_input_file_name_" val FINDING_TOUCHED_FILES_MSG: String = "Finding files to rewrite for DELETE operation" def rewritingFilesMsg(numFilesToRewrite: Long): String = s"Rewriting $numFilesToRewrite files for DELETE operation" } /** * Used to report details about delete. * * @param condition: what was the delete condition * @param numFilesTotal: how big is the table * @param numTouchedFiles: how many files did we touch. Alias for `numFilesAfterSkipping` * @param numRewrittenFiles: how many files had to be rewritten. Alias for `numAddedFiles` * @param numRemovedFiles: how many files we removed. Alias for `numTouchedFiles` * @param numAddedFiles: how many files we added. Alias for `numRewrittenFiles` * @param numAddedChangeFiles: how many change files were generated * @param numFilesBeforeSkipping: how many candidate files before skipping * @param numBytesBeforeSkipping: how many candidate bytes before skipping * @param numFilesAfterSkipping: how many candidate files after skipping * @param numBytesAfterSkipping: how many candidate bytes after skipping * @param numPartitionsAfterSkipping: how many candidate partitions after skipping * @param numPartitionsAddedTo: how many new partitions were added * @param numPartitionsRemovedFrom: how many partitions were removed * @param numCopiedRows: how many rows were copied * @param numDeletedRows: how many rows were deleted * @param numBytesAdded: how many bytes were added * @param numBytesRemoved: how many bytes were removed * @param changeFileBytes: total size of change files generated * @param scanTimeMs: how long did finding take * @param rewriteTimeMs: how long did rewriting take * @param numDeletionVectorsAdded: how many deletion vectors were added * @param numDeletionVectorsRemoved: how many deletion vectors were removed * @param numDeletionVectorsUpdated: how many deletion vectors were updated * * @note All the time units are milliseconds. */ case class DeleteMetric( condition: String, numFilesTotal: Long, numTouchedFiles: Long, numRewrittenFiles: Long, numRemovedFiles: Long, numAddedFiles: Long, numAddedChangeFiles: Long, numFilesBeforeSkipping: Long, numBytesBeforeSkipping: Long, numFilesAfterSkipping: Long, numBytesAfterSkipping: Long, numPartitionsAfterSkipping: Option[Long], numPartitionsAddedTo: Option[Long], numPartitionsRemovedFrom: Option[Long], @JsonDeserialize(contentAs = classOf[java.lang.Long]) numCopiedRows: Option[Long], @JsonDeserialize(contentAs = classOf[java.lang.Long]) numDeletedRows: Option[Long], numBytesAdded: Long, numBytesRemoved: Long, changeFileBytes: Long, scanTimeMs: Long, rewriteTimeMs: Long, numDeletionVectorsAdded: Long, numDeletionVectorsRemoved: Long, numDeletionVectorsUpdated: Long, @JsonDeserialize(contentAs = classOf[java.lang.Long]) commitVersion: Option[Long] = None, isWriteCommand: Boolean = false, @JsonDeserialize(contentAs = classOf[java.lang.Long]) numLogicalRecordsAdded: Option[Long] = None, @JsonDeserialize(contentAs = classOf[java.lang.Long]) numLogicalRecordsRemoved: Option[Long] = None ) ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/DeletionVectorUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands import org.apache.spark.sql.delta.{DeletionVectorsTableFeature, DeltaConfigs, Snapshot, SnapshotDescriptor} import org.apache.spark.sql.delta.actions.{Metadata, Protocol} import org.apache.spark.sql.delta.deletionvectors.{RoaringBitmapArray, RoaringBitmapArrayFormat} import org.apache.spark.sql.delta.files.SupportsRowIndexFilters import org.apache.spark.sql.delta.files.TahoeFileIndex import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.hadoop.fs.Path import org.apache.spark.sql.SparkSession import org.apache.spark.sql.execution.datasources.FileIndex import org.apache.spark.sql.functions.col import org.apache.spark.util.Utils trait DeletionVectorUtils extends DeltaLogging { /** * Run a query on the delta log to determine if the given snapshot contains no deletion vectors. * Return `false` if it does contain deletion vectors. */ def isTableDVFree(snapshot: Snapshot): Boolean = { val dvsReadable = deletionVectorsReadable(snapshot) if (dvsReadable) { val dvCount = snapshot.allFiles .filter(col("deletionVector").isNotNull) .limit(1) .count() dvCount == 0L } else { true } } /** * Returns true if persistent deletion vectors are enabled and * readable with the current reader version. */ def fileIndexSupportsReadingDVs(fileIndex: FileIndex): Boolean = fileIndex match { case index: TahoeFileIndex => deletionVectorsReadable(index) case _: SupportsRowIndexFilters => true case _ => false } def deletionVectorsWritable( snapshot: SnapshotDescriptor, newProtocol: Option[Protocol] = None, newMetadata: Option[Metadata] = None): Boolean = deletionVectorsWritable( protocol = newProtocol.getOrElse(snapshot.protocol), metadata = newMetadata.getOrElse(snapshot.metadata)) def deletionVectorsWritable(protocol: Protocol, metadata: Metadata): Boolean = protocol.isFeatureSupported(DeletionVectorsTableFeature) && DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.fromMetaData(metadata) def deletionVectorsReadable( snapshot: SnapshotDescriptor, newProtocol: Option[Protocol] = None, newMetadata: Option[Metadata] = None): Boolean = { deletionVectorsReadable( newProtocol.getOrElse(snapshot.protocol), newMetadata.getOrElse(snapshot.metadata)) } def deletionVectorsReadable( protocol: Protocol, metadata: Metadata): Boolean = { protocol.isFeatureSupported(DeletionVectorsTableFeature) && metadata.format.provider == "parquet" // DVs are only supported on parquet tables. } /** * Serializes `bitmap` into a byte array using `serializationFormat`. If it fails, it records a * delta event and re-throws the exception. */ def serialize( bitmap: RoaringBitmapArray, serializationFormat: RoaringBitmapArrayFormat.Value, tablePath: Option[Path] = None, debugInfo: Map[String, Any] = Map.empty): Array[Byte] = { try { bitmap.serializeAsByteArray(serializationFormat) } catch { case e: Exception => recordDeltaEvent( deltaLog = null, opType = "delta.assertions.deletionVectorSerializationError", data = debugInfo ++ Map( "serializationFormat" -> serializationFormat, "cardinality" -> bitmap.cardinality, "errorMsg" -> e.getMessage, "errorStackTrace" -> e.getStackTrace), path = tablePath) throw e } } /** * Deserializes a RoaringBitmapArray from `bytes`. If it fails, it records a delta event and * re-throws the exception. */ def deserialize( bytes: Array[Byte], tablePath: Option[Path] = None, debugInfo: Map[String, Any] = Map.empty): RoaringBitmapArray = { try { RoaringBitmapArray.readFrom(bytes) } catch { case e: Exception => recordDeltaEvent( deltaLog = null, "delta.assertions.deletionVectorDeserializationError", data = debugInfo ++ Map( "errorMsg" -> e.getMessage, "errorStackTrace" -> e.getStackTrace), path = tablePath) throw e } } } // To access utilities from places where mixing in a trait is inconvenient. object DeletionVectorUtils extends DeletionVectorUtils ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/DeltaCommand.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands // scalastyle:off import.ordering.noEmptyLine import java.util.concurrent.TimeUnit.NANOSECONDS import scala.util.control.NonFatal import org.apache.spark.sql.delta.{DeltaAnalysisException, DeltaErrors, DeltaLog, DeltaOptions, DeltaTableIdentifier, DeltaTableUtils, NumRecordsStats, OptimisticTransaction, ResolvedPathBasedNonDeltaTable} import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.catalog.{DeltaTableV2, IcebergTablePlaceHolder} import org.apache.spark.sql.delta.files.TahoeBatchFileIndex import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf} import org.apache.spark.sql.delta.util.DeltaFileOperations import org.apache.hadoop.fs.Path import org.apache.spark.internal.MDC import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.{Analyzer, EliminateSubqueryAliases, NoSuchTableException, ResolvedTable, UnresolvedAttribute, UnresolvedRelation} import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType} import org.apache.spark.sql.catalyst.expressions.{Expression, SubqueryExpression} import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.connector.catalog.V1Table import org.apache.spark.sql.delta.DeltaOperations.Operation import org.apache.spark.sql.delta.sources.DeltaSQLConf.DELTA_COLLECT_STATS import org.apache.spark.sql.execution.SQLExecution import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelationWithTable} import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, DataSourceV2RelationShim} import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} /** * Helper trait for all delta commands. */ trait DeltaCommand extends DeltaLogging with DeltaCommandInvariants { /** * Converts string predicates into [[Expression]]s relative to a transaction. * * @throws AnalysisException if a non-partition column is referenced. */ protected def parsePredicates( spark: SparkSession, predicate: String): Seq[Expression] = { try { spark.sessionState.sqlParser.parseExpression(predicate) :: Nil } catch { case e: ParseException => throw DeltaErrors.failedRecognizePredicate(predicate, e) case e: NullPointerException if predicate == null => throw DeltaErrors.failedRecognizePredicate("NULL", e) } } def verifyPartitionPredicates( spark: SparkSession, partitionColumns: Seq[String], predicates: Seq[Expression]): Unit = { predicates.foreach { pred => if (SubqueryExpression.hasSubquery(pred)) { throw DeltaErrors.unsupportSubqueryInPartitionPredicates() } pred.references.foreach { col => val colName = col match { case u: UnresolvedAttribute => // Note: `UnresolvedAttribute(Seq("a.b"))` and `UnresolvedAttribute(Seq("a", "b"))` will // return the same name. We accidentally treated the latter as the same as the former. // Because some users may already rely on it, we keep supporting both. u.nameParts.mkString(".") case _ => col.name } val nameEquality = spark.sessionState.conf.resolver partitionColumns.find(f => nameEquality(f, colName)).getOrElse { throw DeltaErrors.nonPartitionColumnReference(colName, partitionColumns) } } } } /** * Generates a map of file names to add file entries for operations where we will need to * rewrite files such as delete, merge, update. We expect file names to be unique, because * each file contains a UUID. */ def generateCandidateFileMap( basePath: Path, candidateFiles: Seq[AddFile]): Map[String, AddFile] = { val nameToAddFileMap = candidateFiles.map(add => DeltaFileOperations.absolutePath(basePath.toString, add.path).toString -> add).toMap assert(nameToAddFileMap.size == candidateFiles.length, s"File name collisions found among:\n${candidateFiles.map(_.path).mkString("\n")}") nameToAddFileMap } /** * This method provides the RemoveFile actions that are necessary for files that are touched and * need to be rewritten in methods like Delete, Update, and Merge. * * @param deltaLog The DeltaLog of the table that is being operated on * @param nameToAddFileMap A map generated using `generateCandidateFileMap`. * @param filesToRewrite Absolute paths of the files that were touched. We will search for these * in `candidateFiles`. Obtained as the output of the `input_file_name` * function. * @param operationTimestamp The timestamp of the operation */ protected def removeFilesFromPaths( deltaLog: DeltaLog, nameToAddFileMap: Map[String, AddFile], filesToRewrite: Seq[String], operationTimestamp: Long): Seq[RemoveFile] = { filesToRewrite.map { absolutePath => val addFile = getTouchedFile(deltaLog.dataPath, absolutePath, nameToAddFileMap) addFile.removeWithTimestamp(operationTimestamp) } } /** * Build a base relation of files that need to be rewritten as part of an update/delete/merge * operation. */ protected def buildBaseRelation( spark: SparkSession, txn: OptimisticTransaction, actionType: String, rootPath: Path, inputLeafFiles: Seq[String], nameToAddFileMap: Map[String, AddFile]): HadoopFsRelation = { val deltaLog = txn.deltaLog val scannedFiles = inputLeafFiles.map(f => getTouchedFile(rootPath, f, nameToAddFileMap)) val fileIndex = new TahoeBatchFileIndex( spark, actionType, scannedFiles, deltaLog, rootPath, txn.snapshot) HadoopFsRelation( fileIndex, partitionSchema = txn.metadata.partitionSchema, dataSchema = txn.metadata.schema, bucketSpec = None, deltaLog.fileFormat(txn.protocol, txn.metadata), txn.metadata.format.options)(spark) } /** * Find the AddFile record corresponding to the file that was read as part of a * delete/update/merge operation. * * @param basePath The path of the table. Must not be escaped. * @param escapedFilePath The path to a file that can be either absolute or relative. All special * chars in this path must be already escaped by URI standards. * @param nameToAddFileMap Map generated through `generateCandidateFileMap()`. */ def getTouchedFile( basePath: Path, escapedFilePath: String, nameToAddFileMap: Map[String, AddFile]): AddFile = { val absolutePath = DeltaFileOperations.absolutePath(basePath.toString, escapedFilePath).toString nameToAddFileMap.getOrElse(absolutePath, { throw DeltaErrors.notFoundFileToBeRewritten(absolutePath, nameToAddFileMap.keys) }) } /** * Use the analyzer to resolve the identifier provided * @param analyzer The session state analyzer to call * @param identifier Table Identifier to determine whether is path based or not * @return */ protected def resolveIdentifier(analyzer: Analyzer, identifier: TableIdentifier): LogicalPlan = { EliminateSubqueryAliases(analyzer.execute(UnresolvedRelation(identifier))) } /** * Use the analyzer to see whether the provided TableIdentifier is for a path based table or not * @param analyzer The session state analyzer to call * @param tableIdent Table Identifier to determine whether is path based or not * @return Boolean where true means that the table is a table in a metastore and false means the * table is a path based table */ def isCatalogTable(analyzer: Analyzer, tableIdent: TableIdentifier): Boolean = { try { resolveIdentifier(analyzer, tableIdent) match { // is path case LogicalRelationWithTable(HadoopFsRelation(_, _, _, _, _, _), None) => false // is table case LogicalRelationWithTable(HadoopFsRelation(_, _, _, _, _, _), Some(_)) => true // is iceberg table case DataSourceV2RelationShim(_: IcebergTablePlaceHolder, _, _, _, _) => false // could not resolve table/db case _: UnresolvedRelation => throw new NoSuchTableException(tableIdent.database.getOrElse(""), tableIdent.table) // other e.g. view case _ => true } } catch { // Checking for table exists/database exists may throw an error in some cases in which case, // see if the table is a path-based table, otherwise throw the original error case _: AnalysisException if isPathIdentifier(tableIdent) => false } } /** * Checks if the given identifier can be for a delta table's path * @param tableIdent Table Identifier for which to check */ protected def isPathIdentifier(tableIdent: TableIdentifier): Boolean = { val provider = tableIdent.database.getOrElse("") // If db doesnt exist or db is called delta/tahoe then check if path exists DeltaSourceUtils.isDeltaDataSourceName(provider) && new Path(tableIdent.table).isAbsolute } /** * Utility method to return the [[DeltaLog]] of an existing Delta table referred * by either the given [[path]] or [[tableIdentifier]]. * * @param spark [[SparkSession]] reference to use. * @param path Table location. Expects a non-empty [[tableIdentifier]] or [[path]]. * @param tableIdentifier Table identifier. Expects a non-empty [[tableIdentifier]] or [[path]]. * @param operationName Operation that is getting the DeltaLog, used in error messages. * @param hadoopConf Hadoop file system options used to build DeltaLog. * @return DeltaLog of the table * @throws AnalysisException If either no Delta table exists at the given path/identifier or * there is neither [[path]] nor [[tableIdentifier]] is provided. */ protected def getDeltaLog( spark: SparkSession, path: Option[String], tableIdentifier: Option[TableIdentifier], operationName: String, hadoopConf: Map[String, String] = Map.empty): DeltaLog = { val (deltaLog, catalogTable) = if (path.nonEmpty) { (DeltaLog.forTable(spark, new Path(path.get), hadoopConf), None) } else if (tableIdentifier.nonEmpty) { val sessionCatalog = spark.sessionState.catalog lazy val metadata = sessionCatalog.getTableMetadata(tableIdentifier.get) DeltaTableIdentifier(spark, tableIdentifier.get) match { case Some(id) if id.path.nonEmpty => (DeltaLog.forTable(spark, new Path(id.path.get), hadoopConf), None) case Some(id) if id.table.nonEmpty => (DeltaLog.forTable(spark, metadata, hadoopConf), Some(metadata)) case _ => if (metadata.tableType == CatalogTableType.VIEW) { throw DeltaErrors.viewNotSupported(operationName) } throw DeltaErrors.notADeltaTableException(operationName) } } else { throw DeltaErrors.missingTableIdentifierException(operationName) } val startTime = Some(System.currentTimeMillis) if (deltaLog .update(checkIfUpdatedSinceTs = startTime, catalogTableOpt = catalogTable) .version < 0) { throw DeltaErrors.notADeltaTableException( operationName, DeltaTableIdentifier(path, tableIdentifier)) } deltaLog } /** * Send the driver-side metrics. * * This is needed to make the SQL metrics visible in the Spark UI. * All metrics are default initialized with 0 so that's what we're * reporting in case we skip an already executed action. */ protected def sendDriverMetrics(spark: SparkSession, metrics: Map[String, SQLMetric]): Unit = { val executionId = spark.sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) SQLMetrics.postDriverMetricUpdates(spark.sparkContext, executionId, metrics.values.toSeq) } /** * Extracts the [[DeltaTableV2]] from a LogicalPlan iff the LogicalPlan is a [[ResolvedTable]] * with either a [[DeltaTableV2]] or a [[V1Table]] that is referencing a Delta table. In all * other cases this method will throw a "Table not found" exception. */ def getDeltaTable(target: LogicalPlan, cmd: String): DeltaTableV2 = { // TODO: Remove this wrapper and let former callers invoke DeltaTableV2.extractFrom directly. DeltaTableV2.extractFrom(target, cmd) } /** * Extracts [[CatalogTable]] metadata from a LogicalPlan if the plan is a [[ResolvedTable]]. The * table can be a non delta table. */ def getTableCatalogTable(target: LogicalPlan, cmd: String): Option[CatalogTable] = { target match { case ResolvedTable(_, _, d: DeltaTableV2, _) => d.catalogTable case ResolvedTable(_, _, t: V1Table, _) => Some(t.catalogTable) case _ => None } } /** * Helper method to extract the table id or path from a LogicalPlan representing * a Delta table. This uses [[DeltaCommand.getDeltaTable]] to convert the LogicalPlan * to a [[DeltaTableV2]] and then extracts either the path or identifier from it. If * the [[DeltaTableV2]] has a [[CatalogTable]], the table identifier will be returned. * Otherwise, the table's path will be returned. Throws an exception if the LogicalPlan * does not represent a Delta table. */ def getDeltaTablePathOrIdentifier( target: LogicalPlan, cmd: String): (Option[TableIdentifier], Option[String]) = { val table = getDeltaTable(target, cmd) table.catalogTable match { case Some(catalogTable) => (Some(catalogTable.identifier), None) case _ => (None, Some(table.path.toString)) } } /** * Helper method to extract the table id or path from a LogicalPlan representing a resolved table * or path. This calls getDeltaTablePathOrIdentifier if the resolved table is a delta table. For * non delta table with identifier, we extract its identifier. For non delta table with path, it * expects the path to be wrapped in an ResolvedPathBasedNonDeltaTable and extracts it from there. */ def getTablePathOrIdentifier( target: LogicalPlan, cmd: String): (Option[TableIdentifier], Option[String]) = { target match { case ResolvedTable(_, _, t: DeltaTableV2, _) => getDeltaTablePathOrIdentifier(target, cmd) case ResolvedTable(_, _, t: V1Table, _) if DeltaTableUtils.isDeltaTable(t.catalogTable) => getDeltaTablePathOrIdentifier(target, cmd) case ResolvedTable(_, _, t: V1Table, _) => (Some(t.catalogTable.identifier), None) case p: ResolvedPathBasedNonDeltaTable => (None, Some(p.path)) case _ => (None, None) } } /** * Returns true if there is information in the spark session that indicates that this write * has already been successfully written. */ protected def hasBeenExecuted(txn: OptimisticTransaction, sparkSession: SparkSession, options: Option[DeltaOptions] = None): Boolean = { val (txnVersionOpt, txnAppIdOpt, isFromSessionConf) = getTxnVersionAndAppId( sparkSession, options) // only enter if both txnVersion and txnAppId are set for (version <- txnVersionOpt; appId <- txnAppIdOpt) { val currentVersion = txn.txnVersion(appId) if (currentVersion >= version) { logInfo(log"Already completed batch ${MDC(DeltaLogKeys.VERSION, version)} in application " + log"${MDC(DeltaLogKeys.APP_ID, appId)}. This will be skipped.") if (isFromSessionConf && sparkSession.sessionState.conf.getConf( DeltaSQLConf.DELTA_IDEMPOTENT_DML_AUTO_RESET_ENABLED)) { // if we got txnAppId and txnVersion from the session config, we reset the // version here, after skipping the current transaction, as a safety measure to // prevent data loss if the user forgets to manually reset txnVersion sparkSession.sessionState.conf.unsetConf(DeltaSQLConf.DELTA_IDEMPOTENT_DML_TXN_VERSION) } return true } } false } /** * Returns SetTransaction if a valid app ID and version are present. Otherwise returns * an empty list. */ protected def createSetTransaction( sparkSession: SparkSession, deltaLog: DeltaLog, options: Option[DeltaOptions] = None): Option[SetTransaction] = { val (txnVersionOpt, txnAppIdOpt, isFromSessionConf) = getTxnVersionAndAppId( sparkSession, options) // only enter if both txnVersion and txnAppId are set for (version <- txnVersionOpt; appId <- txnAppIdOpt) { if (isFromSessionConf && sparkSession.sessionState.conf.getConf( DeltaSQLConf.DELTA_IDEMPOTENT_DML_AUTO_RESET_ENABLED)) { // if we got txnAppID and txnVersion from the session config, we reset the // version here as a safety measure to prevent data loss if the user forgets // to manually reset txnVersion sparkSession.sessionState.conf.unsetConf(DeltaSQLConf.DELTA_IDEMPOTENT_DML_TXN_VERSION) } return Some(SetTransaction(appId, version, Some(deltaLog.clock.getTimeMillis()))) } None } /** * Helper method to retrieve the current txn version and app ID. These are either * retrieved from user-provided write options or from session configurations. */ private def getTxnVersionAndAppId( sparkSession: SparkSession, options: Option[DeltaOptions]): (Option[Long], Option[String], Boolean) = { var txnVersion: Option[Long] = None var txnAppId: Option[String] = None for (o <- options) { txnVersion = o.txnVersion txnAppId = o.txnAppId } var numOptions = txnVersion.size + txnAppId.size // numOptions can only be 0 or 2, as enforced by // DeltaWriteOptionsImpl.validateIdempotentWriteOptions so this // assert should never be triggered assert(numOptions == 0 || numOptions == 2, s"Only one of txnVersion and txnAppId " + s"has been set via dataframe writer options: txnVersion = $txnVersion txnAppId = $txnAppId") var fromSessionConf = false if (numOptions == 0) { txnVersion = sparkSession.sessionState.conf.getConf( DeltaSQLConf.DELTA_IDEMPOTENT_DML_TXN_VERSION) // don't need to check for valid conversion to Long here as that // is already enforced at set time txnAppId = sparkSession.sessionState.conf.getConf( DeltaSQLConf.DELTA_IDEMPOTENT_DML_TXN_APP_ID) // check that both session configs are set numOptions = txnVersion.size + txnAppId.size if (numOptions != 0 && numOptions != 2) { throw DeltaErrors.invalidIdempotentWritesOptionsException( "Both spark.databricks.delta.write.txnAppId and " + "spark.databricks.delta.write.txnVersion must be specified for " + "idempotent Delta writes") } fromSessionConf = true } (txnVersion, txnAppId, fromSessionConf) } protected def logNumRecordsMismatch( deltaLog: DeltaLog, actions: Seq[Action], stats: NumRecordsStats, op: Operation): Unit = { var numRemove = 0 var numAdd = 0 actions.foreach { case _: AddFile => numAdd += 1 case _: RemoveFile => numRemove += 1 case _ => } val info = NumRecordsCheckInfo( operation = op.name, numAdd = numAdd, numRemove = numRemove, numRecordsRemoved = stats.numLogicalRecordsRemovedPartial, numRecordsAdded = stats.numLogicalRecordsAddedPartial, numDeletionVectorRecordsRemoved = stats.numDeletionVectorRecordsRemoved, numDeletionVectorRecords = stats.numDeletionVectorRecordsAdded, operationParameters = op.jsonEncodedValues, statsCollectionEnabled = SparkSession.getActiveSession.get.conf.get(DELTA_COLLECT_STATS) ) recordDeltaEvent(deltaLog, opType = "delta.assertions.numRecordsChanged", data = info) logWarning(log"Number of records validation failed. Number of added records" + log" (${MDC(DeltaLogKeys.NUM_RECORDS, stats.numLogicalRecordsAddedPartial)})" + log" does not match removed records" + log" (${MDC(DeltaLogKeys.NUM_RECORDS2, stats.numLogicalRecordsRemovedPartial)})") } } // Recorded when number of records check for unchanged data fails. case class NumRecordsCheckInfo( operation: String, numAdd: Int, numRemove: Int, numRecordsAdded: Long, numRecordsRemoved: Long, numDeletionVectorRecordsRemoved: Long = 0, // number of DV records removed by the RemoveFiles numDeletionVectorRecords: Long = 0, // number of DV records present in all AddFiles operationParameters: Map[String, String], statsCollectionEnabled: Boolean ) ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/DeltaCommandInvariants.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands import java.util.UUID import scala.util.Try import scala.util.control.NonFatal import org.apache.spark.sql.delta.{DeltaErrors, DeltaLog} import org.apache.spark.sql.delta.DeltaOperations.Operation import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.internal.{LogEntry, Logging, MDC} import org.apache.spark.sql.catalyst.SQLConfHelper import org.apache.spark.sql.execution.metric.SQLMetric trait DeltaCommandInvariants extends DeltaLogging with SQLConfHelper with Logging { /** * Evaluate that `invariant` "holds" (evaluates to `true`) or log the violation. * @param invariant The invariant to evaluate. * @param label It's suggested to make `label` the same text as `invariant` so it's easy to * see in the logs what was evaluated. * @param op Operation name. * @param deltaLog Delta log of the table this invariant is evaluated on. * @param parameters Parameters that were used to evaluate the invariant. * @param additionalInfo Additional info to be included in usage logs. */ protected def checkCommandInvariant( invariant: () => Boolean, label: String, op: Operation, deltaLog: DeltaLog, parameters: => Map[String, Long], additionalInfo: => Map[String, String] = Map.empty): Unit = { val id = UUID.randomUUID() val invariantResult = try { invariant() } catch { case NonFatal(e) => logWarning(log"Exception thrown while evaluating command invariant." + log" Reference: ${MDC(DeltaLogKeys.ERROR_ID, id.toString)}.", e) false } if (!invariantResult) { val shouldFail = conf.getConf(DeltaSQLConf.COMMAND_INVARIANT_CHECKS_THROW) try { val opType = "delta.assertions.unreliable.commandInvariantViolated" val info = CommandInvariantCheckInfo( exceptionThrown = shouldFail, id = id, invariantExpression = label, invariantParameters = parameters, operation = op.name, operationParameters = op.jsonEncodedValues, additionalInfo = additionalInfo ) recordDeltaEvent( deltaLog, opType = opType, data = info) // Log this to Spark logs as well, so someone looking through there knows to look for the // details in usage logs. // FIXME: Needs inline type annotations because otherwise compiler gets confused on // implicit conversions. val logEntry: LogEntry = log"Delta Command Invariant violated." + log" Reference: ${MDC(DeltaLogKeys.ERROR_ID, id.toString)}." + log" Info: ${MDC(DeltaLogKeys.INVARIANT_CHECK_INFO, info)}." logWarning(logEntry) } catch { case NonFatal(e) => logWarning(log"Unexpected error while logging command invariant violation." + log" Reference: ${MDC(DeltaLogKeys.ERROR_ID, id.toString)}.", e) } if (shouldFail) { throw DeltaErrors.commandInvariantViolationException(operation = op.name, id = id) } } } } // Recorded when a command invariant is violated. case class CommandInvariantCheckInfo( exceptionThrown: Boolean, id: UUID, invariantExpression: String, invariantParameters: Map[String, Long], operation: String, operationParameters: Map[String, String], additionalInfo: Map[String, String]) abstract class CommandInvariantMetricValue extends Logging { protected def value: Try[Long] def getOrThrow: Long = value.get def getOrDummy: Long = value.getOrElse(-1L) } case class CommandInvariantMetricValueFromSingle( metric: SQLMetric) extends CommandInvariantMetricValue { override protected val value: Try[Long] = Try { metric.value } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/DeltaGenerateCommand.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.{DeltaErrors, DeltaLog, UnresolvedDeltaPathOrIdentifier} import org.apache.spark.sql.delta.hooks.GenerateSymlinkManifest import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode} import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.execution.command.RunnableCommand case class DeltaGenerateCommand(override val child: LogicalPlan, modeName: String) extends RunnableCommand with UnaryNode with DeltaCommand { import DeltaGenerateCommand._ override def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = copy(child = newChild) override def run(sparkSession: SparkSession): Seq[Row] = { if (!modeNameToGenerationFunc.contains(modeName)) { throw DeltaErrors.unsupportedGenerateModeException(modeName) } val generationFunc = modeNameToGenerationFunc(modeName) val table = getDeltaTable(child, COMMAND_NAME) generationFunc(sparkSession, table.deltaLog, table.catalogTable) Seq.empty } } object DeltaGenerateCommand { val modeNameToGenerationFunc : CaseInsensitiveMap[(SparkSession, DeltaLog, Option[CatalogTable]) => Unit] = CaseInsensitiveMap( Map[String, (SparkSession, DeltaLog, Option[CatalogTable]) => Unit]( "symlink_format_manifest" -> GenerateSymlinkManifest.generateFullManifest ) ) val COMMAND_NAME = "GENERATE" def apply( path: Option[String], tableIdentifier: Option[TableIdentifier], modeName: String, options: Map[String, String] ): DeltaGenerateCommand = { // Exactly one of path or tableIdentifier should be specified val plan = UnresolvedDeltaPathOrIdentifier( path.filter(_ => tableIdentifier.isEmpty), tableIdentifier, options, COMMAND_NAME) DeltaGenerateCommand(plan, modeName) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/DeltaReorgTableCommand.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands import org.apache.spark.sql.delta.{DeltaColumnMapping, DeltaErrors, Snapshot} import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.catalyst.plans.logical.{IgnoreCachedData, LeafCommand, LogicalPlan, UnaryCommand} object DeltaReorgTableMode extends Enumeration { val PURGE, UNIFORM_ICEBERG, REWRITE_TYPE_WIDENING = Value } case class DeltaReorgTableSpec( reorgTableMode: DeltaReorgTableMode.Value, icebergCompatVersionOpt: Option[Int] ) case class DeltaReorgTable( target: LogicalPlan, reorgTableSpec: DeltaReorgTableSpec = DeltaReorgTableSpec(DeltaReorgTableMode.PURGE, None))( val predicates: Seq[String]) extends UnaryCommand { def child: LogicalPlan = target protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = copy(target = newChild)(predicates) override val otherCopyArgs: Seq[AnyRef] = predicates :: Nil } /** * The REORG TABLE command. */ case class DeltaReorgTableCommand( target: LogicalPlan, reorgTableSpec: DeltaReorgTableSpec = DeltaReorgTableSpec(DeltaReorgTableMode.PURGE, None))( val predicates: Seq[String]) extends OptimizeTableCommandBase with ReorgTableForUpgradeUniformHelper with LeafCommand with IgnoreCachedData { override val otherCopyArgs: Seq[AnyRef] = predicates :: Nil override def optimizeByReorg(sparkSession: SparkSession): Seq[Row] = { val command = OptimizeTableCommand( target, predicates, optimizeContext = DeltaOptimizeContext( reorg = Some(reorgOperation), minFileSize = Some(0L), maxDeletedRowsRatio = Some(0d)) )(zOrderBy = Nil) command.run(sparkSession) } override def run(sparkSession: SparkSession): Seq[Row] = reorgTableSpec match { case DeltaReorgTableSpec( DeltaReorgTableMode.PURGE | DeltaReorgTableMode.REWRITE_TYPE_WIDENING, None) => optimizeByReorg(sparkSession) case DeltaReorgTableSpec(DeltaReorgTableMode.UNIFORM_ICEBERG, Some(icebergCompatVersion)) => val table = getDeltaTable(target, "REORG") if (table.update().isCatalogOwned) { throw DeltaErrors.operationBlockedOnCatalogManagedTable("REORG") } upgradeUniformIcebergCompatVersion(table, sparkSession, icebergCompatVersion) } protected def reorgOperation: DeltaReorgOperation = reorgTableSpec match { case DeltaReorgTableSpec(DeltaReorgTableMode.PURGE, None) => new DeltaPurgeOperation() case DeltaReorgTableSpec(DeltaReorgTableMode.UNIFORM_ICEBERG, Some(icebergCompatVersion)) => new DeltaUpgradeUniformOperation(icebergCompatVersion) case DeltaReorgTableSpec(DeltaReorgTableMode.REWRITE_TYPE_WIDENING, None) => new DeltaRewriteTypeWideningOperation() } } /** * Defines a Reorg operation to be applied during optimize. */ sealed trait DeltaReorgOperation { /** * Collects files that need to be processed by the reorg operation from the list of candidate * files. */ def filterFilesToReorg(spark: SparkSession, snapshot: Snapshot, files: Seq[AddFile]): Seq[AddFile] } /** * Reorg operation to purge files with soft deleted rows. * This operation will also try finding and removing the dropped columns from parquet files, * if ever exists such column that does not present in the current table schema. */ class DeltaPurgeOperation extends DeltaReorgOperation with ReorgTableHelper { override def filterFilesToReorg(spark: SparkSession, snapshot: Snapshot, files: Seq[AddFile]) : Seq[AddFile] = { val physicalSchema = DeltaColumnMapping.renameColumns(snapshot.schema) val protocol = snapshot.protocol val metadata = snapshot.metadata val filesWithDroppedColumns: Seq[AddFile] = filterParquetFilesOnExecutors(spark, files, snapshot, ignoreCorruptFiles = false) { schema => fileHasExtraColumns(schema, physicalSchema, protocol, metadata) } val filesWithDV: Seq[AddFile] = files.filter { file => (file.deletionVector != null && file.numPhysicalRecords.isEmpty) || file.numDeletedRecords > 0L } (filesWithDroppedColumns ++ filesWithDV).distinct } } /** * Reorg operation to upgrade the iceberg compatibility version of a table. */ class DeltaUpgradeUniformOperation(icebergCompatVersion: Int) extends DeltaReorgOperation { override def filterFilesToReorg(spark: SparkSession, snapshot: Snapshot, files: Seq[AddFile]) : Seq[AddFile] = { def shouldRewriteToBeIcebergCompatible(file: AddFile): Boolean = { if (file.tags == null) return true val fileIcebergCompatVersion = file.tags.getOrElse(AddFile.Tags.ICEBERG_COMPAT_VERSION.name, "0") fileIcebergCompatVersion != icebergCompatVersion.toString } files.filter(shouldRewriteToBeIcebergCompatible) } } /** * Internal reorg operation to rewrite files to conform to the current table schema when dropping * the type widening table feature. */ class DeltaRewriteTypeWideningOperation extends DeltaReorgOperation with ReorgTableHelper { override def filterFilesToReorg(spark: SparkSession, snapshot: Snapshot, files: Seq[AddFile]) : Seq[AddFile] = { val physicalSchema = DeltaColumnMapping.renameColumns(snapshot.schema) filterParquetFilesOnExecutors(spark, files, snapshot, ignoreCorruptFiles = false) { schema => fileHasDifferentTypes(schema, physicalSchema) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/DescribeDeltaDetailsCommand.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands // scalastyle:off import.ordering.noEmptyLine import java.sql.Timestamp import org.apache.spark.sql.delta.skipping.clustering.{ClusteredTableUtils, ClusteringColumnInfo} import org.apache.spark.sql.delta.{DeltaErrors, DeltaLog, Snapshot, UnresolvedPathOrIdentifier} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.util.DeltaCommitFileProvider import org.apache.hadoop.fs.Path import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.catalyst.{CatalystTypeConverters, ScalaReflection, TableIdentifier} import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType} import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode} import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes import org.apache.spark.sql.execution.command.RunnableCommand import org.apache.spark.sql.types.StructType /** The result returned by the `describe detail` command. */ case class TableDetail( format: String, id: String, name: String, description: String, location: String, createdAt: Timestamp, lastModified: Timestamp, partitionColumns: Seq[String], clusteringColumns: Seq[String], numFiles: java.lang.Long, sizeInBytes: java.lang.Long, properties: Map[String, String], minReaderVersion: java.lang.Integer, minWriterVersion: java.lang.Integer, tableFeatures: Seq[String] ) object TableDetail { val schema = ScalaReflection.schemaFor[TableDetail].dataType.asInstanceOf[StructType] private lazy val converter: TableDetail => Row = { val toInternalRow = CatalystTypeConverters.createToCatalystConverter(schema) val toExternalRow = CatalystTypeConverters.createToScalaConverter(schema) toInternalRow.andThen(toExternalRow).asInstanceOf[TableDetail => Row] } def toRow(table: TableDetail): Row = converter(table) } /** * A command for describing the details of a table such as the format, name, and size. */ case class DescribeDeltaDetailCommand( override val child: LogicalPlan, hadoopConf: Map[String, String]) extends RunnableCommand with UnaryNode with DeltaLogging with DeltaCommand { override val output: Seq[Attribute] = toAttributes(TableDetail.schema) override protected def withNewChildInternal(newChild: LogicalPlan): DescribeDeltaDetailCommand = copy(child = newChild) override def run(sparkSession: SparkSession): Seq[Row] = { val tableMetadata = getTableCatalogTable(child, DescribeDeltaDetailCommand.CMD_NAME) val (_, path) = getTablePathOrIdentifier(child, DescribeDeltaDetailCommand.CMD_NAME) val deltaLog = (tableMetadata, path) match { case (Some(metadata), _) => DeltaLog.forTable(sparkSession, metadata, hadoopConf) case (_, Some(path)) => DeltaLog.forTable(sparkSession, new Path(path), hadoopConf) case _ => throw DeltaErrors.missingTableIdentifierException(DescribeDeltaDetailCommand.CMD_NAME) } recordDeltaOperation(deltaLog, "delta.ddl.describeDetails") { val snapshot = deltaLog.update(catalogTableOpt = tableMetadata) if (snapshot.version == -1) { if (path.nonEmpty) { val fs = new Path(path.get).getFileSystem(deltaLog.newDeltaHadoopConf()) // Throw FileNotFoundException when the path doesn't exist since there may be a typo if (!fs.exists(new Path(path.get))) { throw DeltaErrors.fileNotFoundException(path.get) } describeNonDeltaPath(path.get) } else { describeNonDeltaTable(tableMetadata.get) } } else { describeDeltaTable(sparkSession, deltaLog, snapshot, tableMetadata) } } } private def toRows(detail: TableDetail): Seq[Row] = TableDetail.toRow(detail) :: Nil private def describeNonDeltaTable(table: CatalogTable): Seq[Row] = { toRows( TableDetail( format = table.provider.orNull, id = null, name = table.qualifiedName, description = table.comment.getOrElse(""), location = table.storage.locationUri.map(new Path(_).toString).orNull, createdAt = new Timestamp(table.createTime), lastModified = null, partitionColumns = table.partitionColumnNames, clusteringColumns = null, numFiles = null, sizeInBytes = null, properties = table.properties, minReaderVersion = null, minWriterVersion = null, tableFeatures = null )) } private def describeNonDeltaPath(path: String): Seq[Row] = { toRows( TableDetail( format = null, id = null, name = null, description = null, location = path, createdAt = null, lastModified = null, partitionColumns = null, clusteringColumns = null, numFiles = null, sizeInBytes = null, properties = Map.empty, minReaderVersion = null, minWriterVersion = null, tableFeatures = null)) } private def describeDeltaTable( sparkSession: SparkSession, deltaLog: DeltaLog, snapshot: Snapshot, tableMetadata: Option[CatalogTable]): Seq[Row] = { val currentVersionPath = DeltaCommitFileProvider(snapshot).deltaFile(snapshot.version) val fs = currentVersionPath.getFileSystem(deltaLog.newDeltaHadoopConf()) val tableName = tableMetadata.map(_.qualifiedName).getOrElse(snapshot.metadata.name) val featureNames = ( snapshot.protocol.implicitlySupportedFeatures.map(_.name) ++ snapshot.protocol.readerAndWriterFeatureNames).toSeq.sorted val clusteringColumns = if (ClusteredTableUtils.isSupported(snapshot.protocol)) { ClusteringColumnInfo.extractLogicalNames(snapshot) } else { Nil } toRows( TableDetail( format = "delta", id = snapshot.metadata.id, name = tableName, description = snapshot.metadata.description, location = deltaLog.dataPath.toString, createdAt = snapshot.metadata.createdTime.map(new Timestamp(_)).orNull, lastModified = new Timestamp(fs.getFileStatus(currentVersionPath).getModificationTime), partitionColumns = snapshot.metadata.partitionColumns, clusteringColumns = clusteringColumns, numFiles = snapshot.numOfFiles, sizeInBytes = snapshot.sizeInBytes, properties = snapshot.metadata.configuration, minReaderVersion = snapshot.protocol.minReaderVersion, minWriterVersion = snapshot.protocol.minWriterVersion, tableFeatures = featureNames )) } } object DescribeDeltaDetailCommand { val CMD_NAME = "DESCRIBE DETAIL" def apply( path: Option[String], tableIdentifier: Option[TableIdentifier], hadoopConf: Map[String, String] ): DescribeDeltaDetailCommand = { val plan = UnresolvedPathOrIdentifier( path, tableIdentifier, hadoopConf, CMD_NAME ) DescribeDeltaDetailCommand(plan, hadoopConf) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/DescribeDeltaHistoryCommand.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.{DeltaErrors, DeltaHistory, DeltaTableIdentifier, UnresolvedDeltaPathOrIdentifier, UnresolvedPathBasedDeltaTable} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.{MultiInstanceRelation, UnresolvedTable} import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet} import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, UnaryNode} import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes import org.apache.spark.sql.execution.command.LeafRunnableCommand object DescribeDeltaHistory { /** * Alternate constructor that converts a provided path or table identifier into the * correct child LogicalPlan node. If both path and tableIdentifier are specified (or * if both are None), this method will throw an exception. If a table identifier is * specified, the child LogicalPlan will be an [[UnresolvedTable]] whereas if a path * is specified, it will be an [[UnresolvedPathBasedDeltaTable]]. * * Note that the returned command will have an *unresolved* child table and hence, the command * needs to be analyzed before it can be executed. */ def apply( path: Option[String], tableIdentifier: Option[TableIdentifier], limit: Option[Int]): DescribeDeltaHistory = { val plan = UnresolvedDeltaPathOrIdentifier(path, tableIdentifier, COMMAND_NAME) DescribeDeltaHistory(plan, limit) } val COMMAND_NAME = "DESCRIBE HISTORY" } /** * A logical placeholder for describing a Delta table's history, so that the history can be * leveraged in subqueries. Replaced with `DescribeDeltaHistoryCommand` during planning. * * @param options: Hadoop file system options used for read and write. */ case class DescribeDeltaHistory( override val child: LogicalPlan, limit: Option[Int], override val output: Seq[Attribute] = toAttributes(ExpressionEncoder[DeltaHistory]().schema)) extends UnaryNode with MultiInstanceRelation with DeltaCommand { override def newInstance(): LogicalPlan = copy(output = output.map(_.newInstance())) override def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = copy(child = newChild) /** * Define this operator as having no attributes provided by children in order to prevent column * pruning from trying to insert projections above the source relation. */ override lazy val references: AttributeSet = AttributeSet.empty override def inputSet: AttributeSet = AttributeSet.empty assert(!child.isInstanceOf[Project], s"The child operator of DescribeDeltaHistory must not contain any projection: $child") /** Converts this operator into an executable command. */ def toCommand: DescribeDeltaHistoryCommand = { // Max array size if (limit.exists(_ > Int.MaxValue - 8)) { throw DeltaErrors.maxArraySizeExceeded() } val deltaTableV2: DeltaTableV2 = getDeltaTable(child, DescribeDeltaHistory.COMMAND_NAME) DescribeDeltaHistoryCommand(table = deltaTableV2, limit = limit, output = output) } } /** * A command for describing the history of a Delta table. */ case class DescribeDeltaHistoryCommand( @transient table: DeltaTableV2, limit: Option[Int], override val output: Seq[Attribute] = toAttributes(ExpressionEncoder[DeltaHistory]().schema)) extends LeafRunnableCommand with MultiInstanceRelation with DeltaLogging { override def newInstance(): LogicalPlan = copy(output = output.map(_.newInstance())) override def run(sparkSession: SparkSession): Seq[Row] = { val deltaLog = table.deltaLog recordDeltaOperation(deltaLog, "delta.ddl.describeHistory") { if (!deltaLog.tableExists) { throw DeltaErrors.notADeltaTableException( DescribeDeltaHistory.COMMAND_NAME, DeltaTableIdentifier(path = Some(table.path.toString)) ) } import org.apache.spark.sql.delta.implicits._ val commits = deltaLog.history.getHistory(limit, table.catalogTable) sparkSession.implicits.localSeqToDatasetHolder(commits).toDF().collect().toSeq } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands import java.util.concurrent.TimeUnit import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions.FileAction import org.apache.spark.sql.delta.commands.merge.{ClassicMergeExecutor, InsertOnlyMergeExecutor, MergeIntoMaterializeSourceReason} import org.apache.spark.sql.delta.files._ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.types.{LongType, StructType} /** * Performs a merge of a source query/table into a Delta table. * * Issues an error message when the ON search_condition of the MERGE statement can match * a single row from the target table with multiple rows of the source table-reference. * * Algorithm: * * Phase 1: Find the input files in target that are touched by the rows that satisfy * the condition and verify that no two source rows match with the same target row. * This is implemented as an inner-join using the given condition. See [[ClassicMergeExecutor]] * for more details. * * Phase 2: Read the touched files again and write new files with updated and/or inserted rows. * * Phase 3: Use the Delta protocol to atomically remove the touched files and add the new files. * * @param source Source data to merge from * @param target Target table to merge into * @param targetFileIndex TahoeFileIndex of the target table * @param condition Condition for a source row to match with a target row * @param matchedClauses All info related to matched clauses. * @param notMatchedClauses All info related to not matched clauses. * @param notMatchedBySourceClauses All info related to not matched by source clauses. * @param migratedSchema The final schema of the target - may be changed by schema * evolution. * @param trackHighWaterMarks The column names for which we will track IDENTITY high water * marks. */ case class MergeIntoCommand( @transient source: LogicalPlan, @transient target: LogicalPlan, @transient catalogTable: Option[CatalogTable], @transient targetFileIndex: TahoeFileIndex, condition: Expression, matchedClauses: Seq[DeltaMergeIntoMatchedClause], notMatchedClauses: Seq[DeltaMergeIntoNotMatchedClause], notMatchedBySourceClauses: Seq[DeltaMergeIntoNotMatchedBySourceClause], migratedSchema: Option[StructType], trackHighWaterMarks: Set[String] = Set.empty, schemaEvolutionEnabled: Boolean = false) extends MergeIntoCommandBase with InsertOnlyMergeExecutor with ClassicMergeExecutor { override val output: Seq[Attribute] = Seq( AttributeReference("num_affected_rows", LongType)(), AttributeReference("num_updated_rows", LongType)(), AttributeReference("num_deleted_rows", LongType)(), AttributeReference("num_inserted_rows", LongType)()) protected def runMerge(spark: SparkSession): Seq[Row] = { recordDeltaOperation(targetDeltaLog, "delta.dml.merge") { val startTime = System.nanoTime() targetDeltaLog.withNewTransaction(catalogTable) { deltaTxn => if (hasBeenExecuted(deltaTxn, spark)) { sendDriverMetrics(spark, metrics) return Seq.empty } if (target.schema.size != deltaTxn.metadata.schema.size) { throw DeltaErrors.schemaChangedSinceAnalysis( atAnalysis = target.schema, latestSchema = deltaTxn.metadata.schema) } // Check that type widening wasn't enabled/disabled between analysis and the start of the // transaction. TypeWidening.ensureFeatureConsistentlyEnabled( protocol = targetFileIndex.protocol, metadata = targetFileIndex.metadata, otherProtocol = deltaTxn.protocol, otherMetadata = deltaTxn.metadata ) if (canMergeSchema) { updateMetadata( spark, deltaTxn, migratedSchema.getOrElse(target.schema), deltaTxn.metadata.partitionColumns, deltaTxn.metadata.configuration, isOverwriteMode = false, rearrangeOnly = false) } checkIdentityColumnHighWaterMarks(deltaTxn) deltaTxn.setTrackHighWaterMarks(trackHighWaterMarks) // Materialize the source if needed. prepareMergeSource( spark, source, condition, matchedClauses, notMatchedClauses, isInsertOnly) val mergeActions = { if (isInsertOnly && spark.conf.get(DeltaSQLConf.MERGE_INSERT_ONLY_ENABLED)) { // This is a single-job execution so there is no WriteChanges. performedSecondSourceScan = false writeOnlyInserts( spark, deltaTxn, filterMatchedRows = true, numSourceRowsMetric = "numSourceRows") } else { val (filesToRewrite, deduplicateCDFDeletes) = findTouchedFiles(spark, deltaTxn) if (filesToRewrite.nonEmpty) { val shouldWriteDeletionVectors = shouldWritePersistentDeletionVectors(spark, deltaTxn) if (shouldWriteDeletionVectors) { val newWrittenFiles = withStatusCode("DELTA", "Writing modified data") { writeAllChanges( spark, deltaTxn, filesToRewrite, deduplicateCDFDeletes, writeUnmodifiedRows = false) } val dvActions = withStatusCode( "DELTA", "Writing Deletion Vectors for modified data") { writeDVs(spark, deltaTxn, filesToRewrite) } newWrittenFiles ++ dvActions } else { val newWrittenFiles = withStatusCode("DELTA", "Writing modified data") { writeAllChanges( spark, deltaTxn, filesToRewrite, deduplicateCDFDeletes, writeUnmodifiedRows = true) } newWrittenFiles ++ filesToRewrite.map(_.remove) } } else { // Run an insert-only job instead of WriteChanges writeOnlyInserts( spark, deltaTxn, filterMatchedRows = false, numSourceRowsMetric = "numSourceRowsInSecondScan") } } } commitAndRecordStats( spark, deltaTxn, mergeActions, startTime, getMergeSource.materializeReason) } spark.sharedState.cacheManager.recacheByPlan(spark, target) } sendDriverMetrics(spark, metrics) val num_affected_rows = metrics("numTargetRowsUpdated").value + metrics("numTargetRowsDeleted").value + metrics("numTargetRowsInserted").value Seq(Row( num_affected_rows, metrics("numTargetRowsUpdated").value, metrics("numTargetRowsDeleted").value, metrics("numTargetRowsInserted").value)) } /** * Finalizes the merge operation before committing it to the delta log and records merge metrics: * - Checks that the source table didn't change during the merge operation. * - Register SQL metrics to be updated during commit. * - Commit the operations. * - Collects final merge stats and record them with a Delta event. */ private def commitAndRecordStats( spark: SparkSession, deltaTxn: OptimisticTransaction, mergeActions: Seq[FileAction], startTime: Long, materializeSourceReason: MergeIntoMaterializeSourceReason.MergeIntoMaterializeSourceReason ): Unit = { checkNonDeterministicSource(spark) // Metrics should be recorded before commit (where they are written to delta logs). setOperationNumSourceRowsMetric() metrics("executionTimeMs").set(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime)) deltaTxn.registerSQLMetrics(spark, metrics) val finalActions = createSetTransaction(spark, targetDeltaLog).toSeq ++ mergeActions val numRecordsStats = NumRecordsStats.fromActions(finalActions) val operation = DeltaOperations.Merge( predicate = Option(condition), matchedPredicates = matchedClauses.map(DeltaOperations.MergePredicate(_)), notMatchedPredicates = notMatchedClauses.map(DeltaOperations.MergePredicate(_)), notMatchedBySourcePredicates = notMatchedBySourceClauses.map(DeltaOperations.MergePredicate(_)) ) validateNumRecords(finalActions, numRecordsStats, operation, deltaTxn.deltaLog) val commitVersion = deltaTxn.commitIfNeeded( actions = finalActions, op = operation, tags = RowTracking.addPreservedRowTrackingTagIfNotSet(deltaTxn.snapshot)) val stats = collectMergeStats(deltaTxn, materializeSourceReason, commitVersion, numRecordsStats) recordDeltaEvent(targetDeltaLog, "delta.dml.merge.stats", data = stats) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommandBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands import java.util.concurrent.TimeUnit import scala.collection.mutable import scala.util.control.NonFatal import org.apache.spark.sql.delta.metric.IncrementMetric import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.{Action, AddFile, FileAction} import org.apache.spark.sql.delta.commands.merge.{MergeIntoMaterializeSource, MergeIntoMaterializeSourceReason, MergeStats} import org.apache.spark.sql.delta.files.{TahoeBatchFileIndex, TahoeFileIndex, TransactionalWrite} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.{ImplicitMetadataOperation, SchemaUtils} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.SparkContext import org.apache.spark.sql.{AnalysisException, DataFrame, Row, SparkSession} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.execution.command.LeafRunnableCommand import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} import org.apache.spark.sql.functions.col import org.apache.spark.sql.types.StructType trait MergeIntoCommandBase extends LeafRunnableCommand with DeltaCommand with DeltaLogging with PredicateHelper with ImplicitMetadataOperation with MergeIntoMaterializeSource with UpdateExpressionsSupport with SupportsNonDeterministicExpression { @transient val source: LogicalPlan @transient val target: LogicalPlan @transient val targetFileIndex: TahoeFileIndex val condition: Expression val matchedClauses: Seq[DeltaMergeIntoMatchedClause] val notMatchedClauses: Seq[DeltaMergeIntoNotMatchedClause] val notMatchedBySourceClauses: Seq[DeltaMergeIntoNotMatchedBySourceClause] val migratedSchema: Option[StructType] val schemaEvolutionEnabled: Boolean protected def shouldWritePersistentDeletionVectors( spark: SparkSession, txn: OptimisticTransaction): Boolean = { spark.conf.get(DeltaSQLConf.MERGE_USE_PERSISTENT_DELETION_VECTORS) && DeletionVectorUtils.deletionVectorsWritable(txn.snapshot) } override val (canMergeSchema, canOverwriteSchema) = { // Delta options can't be passed to MERGE INTO currently, so they'll always be empty. // The methods in options check if user has instructed to turn on schema evolution for this // statement, or the auto migration DeltaSQLConf is on, in which case schema evolution // will be allowed. val options = new DeltaOptions(Map.empty[String, String], conf) (schemaEvolutionEnabled || options.canMergeSchema, options.canOverwriteSchema) } @transient protected lazy val sc: SparkContext = SparkContext.getOrCreate() @transient protected lazy val targetDeltaLog: DeltaLog = targetFileIndex.deltaLog /** * Map to get target read attributes by name. The case sensitivity of the map is set accordingly * to Spark configuration. */ @transient private lazy val preEvolutionTargetAttributesMap: Map[String, Attribute] = { val attrMap: Map[String, Attribute] = target .outputSet.view .map(attr => attr.name -> attr).toMap if (conf.caseSensitiveAnalysis) { attrMap } else { CaseInsensitiveMap(attrMap) } } /** * Expressions to convert from a pre-evolution target row to the post-evolution target row. These * expressions are used for columns that are not modified in updated rows or to copy rows that are * not modified. * There are two kinds of expressions here: * * References to existing columns in the target dataframe. Note that these references may have * a different data type than they originally did due to schema evolution so we add a cast that * supports schema evolution. The references will be marked as nullable if `makeNullable` is * set to true, which allows the attributes to reference the output of an outer join. * * Literal nulls, for new columns which are being added to the target table as part of * this transaction, since new columns will have a value of null for all existing rows. */ protected def postEvolutionTargetExpressions(makeNullable: Boolean = false) : Seq[NamedExpression] = { val schema = if (makeNullable) { migratedSchema.getOrElse(target.schema).asNullable } else { migratedSchema.getOrElse(target.schema) } schema.map { col => preEvolutionTargetAttributesMap .get(col.name) .map { attr => Alias( castIfNeeded( attr.withNullability(attr.nullable || makeNullable), col.dataType, castingBehavior = MergeOrUpdateCastingBehavior(canMergeSchema), col.name), col.name )() } .getOrElse(Alias(Literal(null), col.name)()) } } /** Whether this merge statement only has MATCHED clauses. */ protected def isMatchedOnly: Boolean = notMatchedClauses.isEmpty && matchedClauses.nonEmpty && notMatchedBySourceClauses.isEmpty /** Whether this merge statement only has insert (NOT MATCHED) clauses. */ protected def isInsertOnly: Boolean = matchedClauses.isEmpty && notMatchedClauses.nonEmpty && notMatchedBySourceClauses.isEmpty /** Whether this merge statement only has delete clauses. */ protected lazy val isDeleteOnly: Boolean = matchedClauses.forall(_.isInstanceOf[DeltaMergeIntoMatchedDeleteClause]) && notMatchedClauses.isEmpty && notMatchedBySourceClauses.forall(_.isInstanceOf[DeltaMergeIntoNotMatchedBySourceDeleteClause]) /** Whether this merge statement includes inserts statements. */ protected def includesInserts: Boolean = notMatchedClauses.nonEmpty /** Whether this merge statement includes delete statements. */ protected def includesDeletes: Boolean = { matchedClauses.exists(_.isInstanceOf[DeltaMergeIntoMatchedDeleteClause]) || notMatchedBySourceClauses.exists(_.isInstanceOf[DeltaMergeIntoNotMatchedBySourceDeleteClause]) } protected def isCdcEnabled(deltaTxn: OptimisticTransaction): Boolean = DeltaConfigs.CHANGE_DATA_FEED.fromMetaData(deltaTxn.metadata) protected def runMerge(spark: SparkSession): Seq[Row] override def run(spark: SparkSession): Seq[Row] = { metrics("executionTimeMs").set(0) metrics("scanTimeMs").set(0) metrics("rewriteTimeMs").set(0) if (migratedSchema.isDefined) { // Block writes of void columns in the Delta log. Currently void columns are not properly // supported and are dropped on read, but this is not enough for merge command that is also // reading the schema from the Delta log. Until proper support we prefer to fail merge // queries that add void columns. val newNullColumn = SchemaUtils.findNullTypeColumn(migratedSchema.get) if (newNullColumn.isDefined) { throw DeltaErrors.mergeAddVoidColumn(newNullColumn.get) } } val (materializeSource, _) = shouldMaterializeSource(spark, source, isInsertOnly) if (!materializeSource) { runMerge(spark) } else { // If it is determined that source should be materialized, wrap the execution with retries, // in case the data of the materialized source is lost. runWithMaterializedSourceLostRetries( spark, targetFileIndex.deltaLog, metrics, runMerge) } } import SQLMetrics._ override lazy val metrics: Map[String, SQLMetric] = baseMetrics lazy val baseMetrics: Map[String, SQLMetric] = Map( "numSourceRows" -> createMetric(sc, "number of source rows"), "numSourceRowsInSecondScan" -> createMetric(sc, "number of source rows (during repeated scan)"), "operationNumSourceRows" -> createMetric(sc, "number of source rows (reported)"), "numTargetRowsCopied" -> createMetric(sc, "number of target rows rewritten unmodified"), "numTargetRowsInserted" -> createMetric(sc, "number of inserted rows"), "numTargetRowsUpdated" -> createMetric(sc, "number of updated rows"), "numTargetRowsMatchedUpdated" -> createMetric(sc, "number of rows updated by a matched clause"), "numTargetRowsNotMatchedBySourceUpdated" -> createMetric(sc, "number of rows updated by a not matched by source clause"), "numTargetRowsDeleted" -> createMetric(sc, "number of deleted rows"), "numTargetRowsMatchedDeleted" -> createMetric(sc, "number of rows deleted by a matched clause"), "numTargetRowsNotMatchedBySourceDeleted" -> createMetric(sc, "number of rows deleted by a not matched by source clause"), "numTargetFilesBeforeSkipping" -> createMetric(sc, "number of target files before skipping"), "numTargetFilesAfterSkipping" -> createMetric(sc, "number of target files after skipping"), "numTargetFilesRemoved" -> createMetric(sc, "number of files removed to target"), "numTargetFilesAdded" -> createMetric(sc, "number of files added to target"), "numTargetChangeFilesAdded" -> createMetric(sc, "number of change data capture files generated"), "numTargetChangeFileBytes" -> createMetric(sc, "total size of change data capture files generated"), "numTargetBytesBeforeSkipping" -> createMetric(sc, "number of target bytes before skipping"), "numTargetBytesAfterSkipping" -> createMetric(sc, "number of target bytes after skipping"), "numTargetBytesRemoved" -> createMetric(sc, "number of target bytes removed"), "numTargetBytesAdded" -> createMetric(sc, "number of target bytes added"), "numTargetPartitionsAfterSkipping" -> createMetric(sc, "number of target partitions after skipping"), "numTargetPartitionsRemovedFrom" -> createMetric(sc, "number of target partitions from which files were removed"), "numTargetPartitionsAddedTo" -> createMetric(sc, "number of target partitions to which files were added"), "executionTimeMs" -> createTimingMetric(sc, "time taken to execute the entire operation"), "materializeSourceTimeMs" -> createTimingMetric(sc, "time taken to materialize source (or determine it's not needed)"), "scanTimeMs" -> createTimingMetric(sc, "time taken to scan the files for matches"), "rewriteTimeMs" -> createTimingMetric(sc, "time taken to rewrite the matched files"), "numTargetDeletionVectorsAdded" -> createMetric(sc, "number of deletion vectors added"), "numTargetDeletionVectorsRemoved" -> createMetric(sc, "number of deletion vectors removed"), "numTargetDeletionVectorsUpdated" -> createMetric(sc, "number of deletion vectors updated") ) /** * Collects the merge operation stats and metrics into a [[MergeStats]] object that can be * recorded with `recordDeltaEvent`. Merge stats should be collected after committing all new * actions as metrics may still be updated during commit. */ protected def collectMergeStats( deltaTxn: OptimisticTransaction, materializeSourceReason: MergeIntoMaterializeSourceReason.MergeIntoMaterializeSourceReason, commitVersion: Option[Long], numRecordsStats: NumRecordsStats): MergeStats = { val stats = MergeStats.fromMergeSQLMetrics( metrics, condition, matchedClauses, notMatchedClauses, notMatchedBySourceClauses, isPartitioned = deltaTxn.metadata.partitionColumns.nonEmpty, performedSecondSourceScan = performedSecondSourceScan, commitVersion = commitVersion, numRecordsStats = numRecordsStats ) stats.copy( materializeSourceReason = Some(materializeSourceReason.toString), materializeSourceAttempts = Some(attempt)) } protected def shouldOptimizeMatchedOnlyMerge(spark: SparkSession): Boolean = { isMatchedOnly && spark.conf.get(DeltaSQLConf.MERGE_MATCHED_ONLY_ENABLED) } // There is only one when matched clause and it's a Delete and it does not have a condition. protected val isOnlyOneUnconditionalDelete: Boolean = matchedClauses == Seq(DeltaMergeIntoMatchedDeleteClause(None)) // We over-count numTargetRowsDeleted when there are multiple matches; // this is the amount of the overcount, so we can subtract it to get a correct final metric. protected var multipleMatchDeleteOnlyOvercount: Option[Long] = None // Throw error if multiple matches are ambiguous or cannot be computed correctly. protected def throwErrorOnMultipleMatches( hasMultipleMatches: Boolean, spark: SparkSession): Unit = { // Multiple matches are not ambiguous when there is only one unconditional delete as // all the matched row pairs in the 2nd join in `writeAllChanges` will get deleted. if (hasMultipleMatches && !isOnlyOneUnconditionalDelete) { throw DeltaErrors.multipleSourceRowMatchingTargetRowInMergeException(spark) } } /** * Write the output data to files, repartitioning the output DataFrame by the partition columns * if table is partitioned and `merge.repartitionBeforeWrite.enabled` is set to true. */ protected def writeFiles( spark: SparkSession, txn: OptimisticTransaction, outputDF: DataFrame): Seq[FileAction] = { val partitionColumns = txn.metadata.partitionColumns // If the write will be an optimized write, which shuffles the data anyway, then don't // repartition. Optimized writes can handle both splitting very large tasks and coalescing // very small ones. if (partitionColumns.nonEmpty && spark.conf.get(DeltaSQLConf.MERGE_REPARTITION_BEFORE_WRITE) && !TransactionalWrite.shouldOptimizeWrite(txn.metadata, spark.sessionState.conf)) { txn.writeFiles(outputDF.repartition(partitionColumns.map(col): _*)) } else { txn.writeFiles(outputDF) } } /** * Builds a new logical plan to read the given `files` instead of the whole target table. * The plan returned has the same output columns (exprIds) as the `target` logical plan, so that * existing update/insert expressions can be applied on this new plan. Unneeded non-partition * columns may be dropped. */ protected def buildTargetPlanWithFiles( spark: SparkSession, deltaTxn: OptimisticTransaction, files: Seq[AddFile], columnsToDrop: Seq[String]): LogicalPlan = { // Action type "batch" is a historical artifact; the original implementation used it. val fileIndex = new TahoeBatchFileIndex( spark, actionType = "batch", files, deltaTxn.deltaLog, targetFileIndex.path, deltaTxn.snapshot) buildTargetPlanWithIndex( spark, fileIndex, columnsToDrop ) } /** * Builds a new logical plan to read the target table using the given `fileIndex`. * The plan returned has the same output columns (exprIds) as the `target` logical plan, so that * existing update/insert expressions can be applied on this new plan. * * @param columnsToDrop unneeded non-partition columns to be dropped */ protected def buildTargetPlanWithIndex( spark: SparkSession, fileIndex: TahoeFileIndex, columnsToDrop: Seq[String]): LogicalPlan = { var newTarget = DeltaTableUtils.replaceFileIndex(target, fileIndex) newTarget = DeltaTableUtils.dropColumns(spark, newTarget, columnsToDrop) newTarget } /** @return An `Expression` to increment a SQL metric */ protected def incrementMetricAndReturnBool( name: String, valueToReturn: Boolean): Expression = { IncrementMetric(Literal(valueToReturn), metrics(name)) } /** @return An `Expression` to increment SQL metrics */ protected def incrementMetricsAndReturnBool( names: Seq[String], valueToReturn: Boolean): Expression = { val incExpr = incrementMetricAndReturnBool(names.head, valueToReturn) names.tail.foldLeft(incExpr) { case (expr, name) => IncrementMetric(expr, metrics(name)) } } protected def getTargetOnlyPredicates(spark: SparkSession): Seq[Expression] = { val targetOnlyPredicatesOnCondition = splitConjunctivePredicates(condition).filter(_.references.subsetOf(target.outputSet)) if (!isMatchedOnly) { targetOnlyPredicatesOnCondition } else { val targetOnlyMatchedPredicate = matchedClauses .map(_.condition.getOrElse(Literal.TrueLiteral)) .map { condition => splitConjunctivePredicates(condition) .filter(_.references.subsetOf(target.outputSet)) .reduceOption(And) .getOrElse(Literal.TrueLiteral) } .reduceOption(Or) targetOnlyPredicatesOnCondition ++ targetOnlyMatchedPredicate } } protected def seqToString(exprs: Seq[Expression]): String = exprs.map(_.sql).mkString("\n\t") /** * Execute the given `thunk` and return its result while recording the time taken to do it * and setting additional local properties for better UI visibility. * * @param extraOpType extra operation name recorded in the logs * @param status human readable status string describing what the thunk is doing * @param sqlMetricName name of SQL metric to update with the time taken by the thunk * @param thunk the code to execute */ protected def recordMergeOperation[A]( extraOpType: String = "", status: String = null, sqlMetricName: String = null)( thunk: => A): A = { val changedOpType = if (extraOpType == "") { "delta.dml.merge" } else { s"delta.dml.merge.$extraOpType" } val prevDesc = sc.getLocalProperty(SparkContext.SPARK_JOB_DESCRIPTION) val newDesc = Option(status).map { s => // Append the status to existing description if any val prefix = Option(prevDesc).filter(_.nonEmpty).map(_ + " - ").getOrElse("") prefix + s } def executeThunk(): A = { try { val startTimeNs = System.nanoTime() newDesc.foreach { d => sc.setJobDescription(d) } val r = thunk val timeTakenMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs) if (sqlMetricName != null) { if (timeTakenMs > 0) { metrics(sqlMetricName) += timeTakenMs } else if (metrics(sqlMetricName).isZero) { // Make it always at least 1ms if it ran, to distinguish whether it ran or not. metrics(sqlMetricName) += 1 } } r } finally { if (newDesc.isDefined) { sc.setJobDescription(prevDesc) } } } recordDeltaOperation(targetDeltaLog, changedOpType) { if (status == null) { executeThunk() } else { withStatusCode("DELTA", status) { executeThunk() } } } } // Whether the source was scanned the second time, and it was guaranteed to be a scan without // pruning. protected var secondSourceScanWasFullScan: Boolean = false /** * Sets operationNumSourceRows as numSourceRowsInSecondScan if the source was scanned without * possibility of pruning in the 2nd scan. Uses numSourceRows otherwise. */ protected def setOperationNumSourceRowsMetric(): Unit = { val operationNumSourceRows = if (secondSourceScanWasFullScan) { metrics("numSourceRowsInSecondScan").value } else { metrics("numSourceRows").value } metrics("operationNumSourceRows").set(operationNumSourceRows) } // Whether we actually scanned the source twice or the value in numSourceRowsInSecondScan is // uninitialised. protected var performedSecondSourceScan: Boolean = true /** * Throws an exception if merge metrics indicate that the source table changed between the first * and the second source table scans. */ protected def checkNonDeterministicSource(spark: SparkSession): Unit = { // We only detect changes in the number of source rows. This is a best-effort detection; a // more comprehensive solution would be to checksum the values for the columns that we read // in both jobs. // If numSourceRowsInSecondScan is < 0 then it hasn't run, e.g. for insert-only merges. // In that case we have only read the source table once. if (performedSecondSourceScan && metrics("numSourceRows").value != metrics("numSourceRowsInSecondScan").value) { log.warn(s"Merge source has ${metrics("numSourceRows")} rows in initial scan but " + s"${metrics("numSourceRowsInSecondScan")} rows in second scan") if (conf.getConf(DeltaSQLConf.MERGE_FAIL_IF_SOURCE_CHANGED)) { throw DeltaErrors.sourceNotDeterministicInMergeException(spark) } } } /** * Check whether (part of) the give source dataframe is cached and logs an assertion or fails if * it is. Query caching doesn't pin versions of delta tables and can lead to incorrect results so * cached source plans must be materialized. */ def checkSourcePlanIsNotCached(spark: SparkSession, source: LogicalPlan): Unit = { val sourceIsCached = planContainsCachedRelation(DataFrameUtils.ofRows(spark, source)) if (sourceIsCached && spark.conf.get(DeltaSQLConf.MERGE_FAIL_SOURCE_CACHED_AFTER_MATERIALIZATION)) { throw DeltaErrors.mergeConcurrentOperationCachedSourceException() } deltaAssert( !sourceIsCached, name = "merge.sourceCachedAfterMaterializationStep", msg = "Cached source plans must be materialized in MERGE but the source only got cached " + "after the decision to materialize was taken.", deltaLog = targetDeltaLog ) } override protected def prepareMergeSource( spark: SparkSession, source: LogicalPlan, condition: Expression, matchedClauses: Seq[DeltaMergeIntoMatchedClause], notMatchedClauses: Seq[DeltaMergeIntoNotMatchedClause], isInsertOnly: Boolean ): Unit = recordMergeOperation( extraOpType = "materializeSource", status = "MERGE operation - materialize source", sqlMetricName = "materializeSourceTimeMs") { super.prepareMergeSource( spark, source, condition, matchedClauses, notMatchedClauses, isInsertOnly) } /** * Verify that the high water marks used by the identity column generators still match the * the high water marks in the version of the table read by the current transaction. * These high water marks were determined during analysis in [[PreprocessTableMerge]], * which runs outside of the current transaction, so they may no longer be valid. */ protected def checkIdentityColumnHighWaterMarks(deltaTxn: OptimisticTransaction): Unit = { notMatchedClauses.foreach { clause => if (deltaTxn.metadata.schema.length != clause.resolvedActions.length) { throw new IllegalStateException } deltaTxn.metadata.schema.zip(clause.resolvedActions.map(_.expr)).foreach { case (f, GenerateIdentityValues(gen)) => val info = IdentityColumn.getIdentityInfo(f) if (info.highWaterMark != gen.highWaterMarkOpt) { IdentityColumn.logTransactionAbort(deltaTxn.deltaLog) throw DeltaErrors.metadataChangedException(conflictingCommit = None) } case (f, _) if ColumnWithDefaultExprUtils.isIdentityColumn(f) && !IdentityColumn.allowExplicitInsert(f) => throw new IllegalStateException case _ => () } } } /** Returns whether it allows non-deterministic expressions. */ override def allowNonDeterministicExpression: Boolean = { def isConditionDeterministic(mergeClause: DeltaMergeIntoClause): Boolean = { mergeClause.condition match { case Some(c) => c.deterministic case None => true } } // Allow actions to be non-deterministic while all the conditions // must be deterministic. condition.deterministic && matchedClauses.forall(isConditionDeterministic) && notMatchedClauses.forall(isConditionDeterministic) && notMatchedBySourceClauses.forall(isConditionDeterministic) } /** * Validates that the number of records does not increase if there are no insert clauses, and does * not decrease if there are no delete clauses. * * Note: ideally we would also compare the number of added/removed rows in the statistics with the * number of inserted/updated/deleted/copied rows in the SQL metrics, but unfortunately this is * not possible, as sql metrics are not reliable when there are task or stage retries. */ protected def validateNumRecords( actions: Seq[Action], numRecordsStats: NumRecordsStats, op: DeltaOperations.Merge, deltaLog: DeltaLog): Unit = { (numRecordsStats.numLogicalRecordsAdded, numRecordsStats.numLogicalRecordsRemoved, numRecordsStats.numLogicalRecordsAddedInFilesWithDeletionVectors) match { case ( Some(numAddedRecords), Some(numRemovedRecords), Some(numRecordsNotCopied)) => if (!includesInserts && numAddedRecords > numRemovedRecords) { logNumRecordsMismatch(targetDeltaLog, actions, numRecordsStats, op) if (conf.getConf(DeltaSQLConf.NUM_RECORDS_VALIDATION_ENABLED)) { throw DeltaErrors.numRecordsMismatch( operation = "MERGE without INSERT clauses", numAddedRecords, numRemovedRecords ) } } if (!includesDeletes && numAddedRecords < numRemovedRecords) { logNumRecordsMismatch(targetDeltaLog, actions, numRecordsStats, op) if (conf.getConf(DeltaSQLConf.NUM_RECORDS_VALIDATION_ENABLED)) { throw DeltaErrors.numRecordsMismatch( operation = "MERGE without DELETE clauses", numAddedRecords, numRemovedRecords ) } } if (conf.getConf(DeltaSQLConf.COMMAND_INVARIANT_CHECKS_USE_UNRELIABLE)) { // and also using regular (unreliable) metrics for baseline validateMetricBasedCommandInvariants( numAddedRecords, numRemovedRecords, numRecordsNotCopied, op, deltaLog) } case _ => recordDeltaEvent( targetDeltaLog, opType = "delta.assertions.statsNotPresentForNumRecordsCheck") logWarning(log"Could not validate number of records due to missing statistics.") } } private def validateMetricBasedCommandInvariants( numAddedRecords: Long, numRemovedRecords: Long, numRecordsNotCopied: Long, op: DeltaOperations.Merge, deltaLog: DeltaLog): Unit = try { val numRowsInserted = CommandInvariantMetricValueFromSingle(metrics("numTargetRowsInserted")) val numRowsUpdated = CommandInvariantMetricValueFromSingle(metrics("numTargetRowsUpdated")) val numRowsDeleted = CommandInvariantMetricValueFromSingle(metrics("numTargetRowsDeleted")) val numRowsCopied = CommandInvariantMetricValueFromSingle(metrics("numTargetRowsCopied")) checkCommandInvariant( invariant = () => includesInserts || numRowsInserted.getOrThrow == 0, label = "includesInserts || numRowsInserted == 0", op = op, deltaLog = deltaLog, parameters = Map( "numRowsInserted" -> numRowsInserted.getOrDummy ) ) checkCommandInvariant( invariant = () => includesDeletes || numRowsDeleted.getOrThrow == 0, label = "includesDeletes || numRowsDeleted == 0", op = op, deltaLog = deltaLog, parameters = Map( "numRowsDeleted" -> numRowsDeleted.getOrDummy ) ) checkCommandInvariant( invariant = () => !isDeleteOnly || numRowsUpdated.getOrThrow + numRowsInserted.getOrThrow == 0, label = "!isDeleteOnlyMerge || numRowsUpdated + numRowsInserted == 0", op = op, deltaLog = deltaLog, parameters = Map( "numRowsUpdated" -> numRowsUpdated.getOrDummy, "numRowsInserted" -> numRowsInserted.getOrDummy ) ) if (conf.getConf(DeltaSQLConf.MERGE_INSERT_ONLY_ENABLED)) { checkCommandInvariant( invariant = () => !isInsertOnly || numRowsUpdated.getOrThrow + numRowsDeleted.getOrThrow + numRowsCopied.getOrThrow + numRecordsNotCopied == 0, label = "!isInsertOnly" + " || numRowsUpdated + numRowsDeleted + numRowsCopied + numRecordsNotCopied == 0", op = op, deltaLog = deltaLog, parameters = Map( "numRowsUpdated" -> numRowsUpdated.getOrDummy, "numRowsDeleted" -> numRowsDeleted.getOrDummy, "numRowsCopied" -> numRowsCopied.getOrDummy, "numRecordsNotCopied" -> numRecordsNotCopied ) ) } else { // When the special insert-only codepath is disabled, we may end up copying some rows. checkCommandInvariant( invariant = () => !isInsertOnly || numRowsUpdated.getOrThrow + numRowsDeleted.getOrThrow == 0, label = "!isInsertOnly" + " || numRowsUpdated + numRowsDeleted == 0", op = op, deltaLog = deltaLog, parameters = Map( "numRowsUpdated" -> numRowsUpdated.getOrDummy, "numRowsDeleted" -> numRowsDeleted.getOrDummy ) ) } checkCommandInvariant( invariant = () => numRowsUpdated.getOrThrow + numRowsDeleted.getOrThrow + numRowsCopied.getOrThrow + numRecordsNotCopied == numRemovedRecords, label = "numRowsUpdated + numRowsDeleted + numRowsCopied + numRecordsNotCopied ==" + " numRemovedRecords", op = op, deltaLog = deltaLog, parameters = Map( "numRowsUpdated" -> numRowsUpdated.getOrDummy, "numRowsDeleted" -> numRowsDeleted.getOrDummy, "numRowsCopied" -> numRowsCopied.getOrDummy, "numRemovedRecords" -> numRemovedRecords, "numRecordsNotCopied" -> numRecordsNotCopied ) ) checkCommandInvariant( invariant = () => numRowsInserted.getOrThrow + numRowsUpdated.getOrThrow + numRowsCopied.getOrThrow + numRecordsNotCopied == numAddedRecords, label = "numRowsInserted + numRowsUpdated + numRowsCopied + numRecordsNotCopied ==" + " numAddedRecords", op = op, deltaLog = deltaLog, parameters = Map( "numRowsInserted" -> numRowsInserted.getOrDummy, "numRowsUpdated" -> numRowsUpdated.getOrDummy, "numRowsCopied" -> numRowsCopied.getOrDummy, "numAddedRecords" -> numAddedRecords, "numRecordsNotCopied" -> numRecordsNotCopied ) ) } catch { // Immediately re-throw actual command invariant violations, so we don't re-wrap them below. case e: DeltaIllegalStateException if e.getErrorClass == "DELTA_COMMAND_INVARIANT_VIOLATION" => throw e case NonFatal(e) => logWarning(log"Unexpected error in validateMetricBasedCommandInvariants", e) checkCommandInvariant( invariant = () => false, label = "Unexpected error in validateMetricBasedCommandInvariants", op = op, deltaLog = deltaLog, parameters = Map.empty ) } } object MergeIntoCommandBase { val ROW_ID_COL = "_row_id_" val FILE_NAME_COL = "_file_name_" val SOURCE_ROW_PRESENT_COL = "_source_row_present_" val TARGET_ROW_PRESENT_COL = "_target_row_present_" val ROW_DROPPED_COL = "_row_dropped_" val PRECOMPUTED_CONDITION_COL = "_condition_" /** * Spark UI will track all normal accumulators along with Spark tasks to show them on Web UI. * However, the accumulator used by `MergeIntoCommand` can store a very large value since it * tracks all files that need to be rewritten. We should ask Spark UI to not remember it, * otherwise, the UI data may consume lots of memory. Hence, we use the prefix `internal.metrics.` * to make this accumulator become an internal accumulator, so that it will not be tracked by * Spark UI. */ val TOUCHED_FILES_ACCUM_NAME = "internal.metrics.MergeIntoDelta.touchedFiles" /** Count the number of distinct partition values among the AddFiles in the given set. */ def totalBytesAndDistinctPartitionValues(files: Seq[FileAction]): (Long, Int) = { val distinctValues = new mutable.HashSet[Map[String, String]]() var bytes = 0L files.collect { case file: AddFile => distinctValues += file.partitionValues bytes += file.size }.toList // If the only distinct value map is an empty map, then it must be an unpartitioned table. // Return 0 in that case. val numDistinctValues = if (distinctValues.size == 1 && distinctValues.head.isEmpty) 0 else distinctValues.size (bytes, numDistinctValues) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands import java.util.ConcurrentModificationException import scala.collection.mutable.ArrayBuffer import org.apache.spark.sql.delta.skipping.MultiDimClustering import org.apache.spark.sql.delta.skipping.clustering.{ClusteredTableUtils, ClusteringColumnInfo} import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.DeltaOperations.Operation import org.apache.spark.sql.delta.actions.{Action, AddFile, DeletionVectorDescriptor, FileAction, RemoveFile} import org.apache.spark.sql.delta.commands.optimize._ import org.apache.spark.sql.delta.files.SQLMetricsReporting import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.BinPackingUtils import org.apache.spark.SparkContext import org.apache.spark.SparkContext.SPARK_JOB_GROUP_ID import org.apache.spark.internal.MDC import org.apache.spark.sql.{AnalysisException, Encoders, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedTable} import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression} import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode} import org.apache.spark.sql.execution.command.RunnableCommand import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric import org.apache.spark.sql.types._ import org.apache.spark.util.{SystemClock, ThreadUtils} import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.delta.actions.InMemoryLogReplay.UniqueFileActionTuple /** Base class defining abstract optimize command */ abstract class OptimizeTableCommandBase extends RunnableCommand with DeltaCommand { override val output: Seq[Attribute] = Seq( AttributeReference("path", StringType)(), AttributeReference("metrics", Encoders.product[OptimizeMetrics].schema)()) /** * Validates ZOrderBy columns * - validates that partitions columns are not used in `unresolvedZOrderByCols` * - validates that we already collect stats for all the columns used in `unresolvedZOrderByCols` * * @param spark [[SparkSession]] to use * @param snapshot the [[Snapshot]] being used to optimize from * @param unresolvedZOrderByCols Seq of [[UnresolvedAttribute]] corresponding to zOrderBy columns */ def validateZorderByColumns( spark: SparkSession, snapshot: Snapshot, unresolvedZOrderByCols: Seq[UnresolvedAttribute]): Unit = { if (unresolvedZOrderByCols.isEmpty) return val metadata = snapshot.metadata val partitionColumns = metadata.partitionColumns.toSet val dataSchema = StructType(metadata.schema.filterNot(c => partitionColumns.contains(c.name))) val df = spark.createDataFrame(new java.util.ArrayList[Row](), dataSchema) val checkColStat = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_OPTIMIZE_ZORDER_COL_STAT_CHECK) val statCollectionSchema = snapshot.statCollectionLogicalSchema val colsWithoutStats = ArrayBuffer[String]() unresolvedZOrderByCols.foreach { colAttribute => val colName = colAttribute.name if (checkColStat) { try { SchemaUtils.findColumnPosition(colAttribute.nameParts, statCollectionSchema) } catch { case e: AnalysisException if e.getMessage.contains("Couldn't find column") => colsWithoutStats.append(colName) } } val isNameEqual = spark.sessionState.conf.resolver if (partitionColumns.find(isNameEqual(_, colName)).nonEmpty) { throw DeltaErrors.zOrderingOnPartitionColumnException(colName) } if (df.queryExecution.analyzed.resolve(colAttribute.nameParts, isNameEqual).isEmpty) { throw DeltaErrors.zOrderingColumnDoesNotExistException(colName) } } if (checkColStat && colsWithoutStats.nonEmpty) { throw DeltaErrors.zOrderingOnColumnWithNoStatsException( colsWithoutStats.toSeq, spark) } } } object OptimizeTableCommand { /** * Alternate constructor that converts a provided path or table identifier into the * correct child LogicalPlan node. If both path and tableIdentifier are specified (or * if both are None), this method will throw an exception. If a table identifier is * specified, the child LogicalPlan will be an [[UnresolvedTable]] whereas if a path * is specified, it will be an [[UnresolvedPathBasedDeltaTable]]. * * Note that the returned OptimizeTableCommand will have an *unresolved* child table * and hence, the command needs to be analyzed before it can be executed. */ def apply( path: Option[String], tableIdentifier: Option[TableIdentifier], userPartitionPredicates: Seq[String], optimizeContext: DeltaOptimizeContext = DeltaOptimizeContext())( zOrderBy: Seq[UnresolvedAttribute]): OptimizeTableCommand = { val plan = UnresolvedDeltaPathOrIdentifier(path, tableIdentifier, "OPTIMIZE") OptimizeTableCommand(plan, userPartitionPredicates, optimizeContext)(zOrderBy) } } /** * The `optimize` command implementation for Spark SQL. Example SQL: * {{{ * OPTIMIZE ('/path/to/dir' | delta.table) [WHERE part = 25] [FULL]; * }}} * * Note FULL and WHERE clauses are set exclusively. */ case class OptimizeTableCommand( override val child: LogicalPlan, userPartitionPredicates: Seq[String], optimizeContext: DeltaOptimizeContext)( val zOrderBy: Seq[UnresolvedAttribute]) extends OptimizeTableCommandBase with UnaryNode { override val otherCopyArgs: Seq[AnyRef] = zOrderBy :: Nil override protected def withNewChildInternal(newChild: LogicalPlan): OptimizeTableCommand = copy(child = newChild)(zOrderBy) override def run(sparkSession: SparkSession): Seq[Row] = { val table = getDeltaTable(child, "OPTIMIZE") val snapshot = table.update() if (snapshot.version == -1) { throw DeltaErrors.notADeltaTableException(table.deltaLog.dataPath.toString) } if (snapshot.isCatalogOwned) { throw DeltaErrors.operationBlockedOnCatalogManagedTable("OPTIMIZE") } val isClusteredTable = ClusteredTableUtils.isSupported(snapshot.protocol) if (isClusteredTable) { if (userPartitionPredicates.nonEmpty) { throw DeltaErrors.clusteringWithPartitionPredicatesException(userPartitionPredicates) } if (zOrderBy.nonEmpty) { throw DeltaErrors.clusteringWithZOrderByException(zOrderBy) } } lazy val clusteringColumns = ClusteringColumnInfo.extractLogicalNames(snapshot) if (optimizeContext.isFull && (!isClusteredTable || clusteringColumns.isEmpty)) { throw DeltaErrors.optimizeFullNotSupportedException() } val partitionColumns = snapshot.metadata.partitionColumns // Parse the predicate expression into Catalyst expression and verify only simple filters // on partition columns are present val partitionPredicates = userPartitionPredicates.flatMap { predicate => val predicates = parsePredicates(sparkSession, predicate) verifyPartitionPredicates( sparkSession, partitionColumns, predicates) predicates } validateZorderByColumns(sparkSession, snapshot, zOrderBy) val zOrderByColumns = zOrderBy.map(_.name).toSeq new OptimizeExecutor( sparkSession, snapshot, table.catalogTable, partitionPredicates, zOrderByColumns, isAutoCompact = false, optimizeContext ).optimize() } } /** * Stored all runtime context information that can control the execution of optimize. * * @param reorg The REORG operation that triggered the rewriting task, if any. * @param minFileSize Files which are smaller than this threshold will be selected for compaction. * If not specified, [[DeltaSQLConf.DELTA_OPTIMIZE_MIN_FILE_SIZE]] will be used. * This parameter must be set to `0` when [[reorg]] is set. * @param maxDeletedRowsRatio Files with a ratio of soft-deleted rows to the total rows larger than * this threshold will be rewritten by the OPTIMIZE command. If not * specified, [[DeltaSQLConf.DELTA_OPTIMIZE_MAX_DELETED_ROWS_RATIO]] * will be used. This parameter must be set to `0` when [[reorg]] is set. * @param isFull whether OPTIMIZE FULL is run. This is only for clustered tables. */ case class DeltaOptimizeContext( reorg: Option[DeltaReorgOperation] = None, minFileSize: Option[Long] = None, maxFileSize: Option[Long] = None, maxDeletedRowsRatio: Option[Double] = None, isFull: Boolean = false) { if (reorg.nonEmpty) { require( minFileSize.contains(0L) && maxDeletedRowsRatio.contains(0d), "minFileSize and maxDeletedRowsRatio must be 0 when running REORG TABLE.") } } /** * A bin represents a single set of files that are being re-written in a single Spark job. * For compaction, this represents a single file being written. For clustering, this is * an entire partition for Z-ordering, or an entire ZCube for liquid clustering. * * @param partitionValues The partition this set of files is in * @param files The list of files being re-written */ case class Bin(partitionValues: Map[String, String], files: Seq[AddFile]) /** * A batch represents all the bins that will be processed and commited in a single transaction. * * @param bins The set of bins to process in this transaction */ case class Batch(bins: Seq[Bin]) /** * Optimize job which compacts small files into larger files to reduce * the number of files and potentially allow more efficient reads. * * @param sparkSession Spark environment reference. * @param snapshot The snapshot of the table to optimize * @param partitionPredicate List of partition predicates to select subset of files to optimize. */ class OptimizeExecutor( sparkSession: SparkSession, snapshot: Snapshot, catalogTable: Option[CatalogTable], partitionPredicate: Seq[Expression], zOrderByColumns: Seq[String], isAutoCompact: Boolean, optimizeContext: DeltaOptimizeContext) extends DeltaCommand with SQLMetricsReporting with Serializable { /** * In which mode the Optimize command is running. There are three valid modes: * 1. Compaction * 2. ZOrder * 3. Clustering */ private val optimizeStrategy = OptimizeTableStrategy(sparkSession, snapshot, optimizeContext, zOrderByColumns) /** Timestamp to use in [[FileAction]] */ private val operationTimestamp = new SystemClock().getTimeMillis() private val isClusteredTable = ClusteredTableUtils.isSupported(snapshot.protocol) private val isMultiDimClustering = optimizeStrategy.isInstanceOf[ClusteringStrategy] || optimizeStrategy.isInstanceOf[ZOrderStrategy] private val clusteringColumns: Seq[String] = { if (zOrderByColumns.nonEmpty) { zOrderByColumns } else if (isClusteredTable) { ClusteringColumnInfo.extractLogicalNames(snapshot) } else { Nil } } private val partitionSchema = snapshot.metadata.partitionSchema def optimize(): Seq[Row] = { recordDeltaOperation(snapshot.deltaLog, "delta.optimize") { val minFileSize = optimizeContext.minFileSize.getOrElse( sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_MIN_FILE_SIZE)) val maxFileSize = optimizeContext.maxFileSize.getOrElse( sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_MAX_FILE_SIZE)) val maxDeletedRowsRatio = optimizeContext.maxDeletedRowsRatio.getOrElse( sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_MAX_DELETED_ROWS_RATIO)) val batchSize = sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_BATCH_SIZE) // Get all the files from the snapshot, we will register them with the individual // transactions later val candidateFiles = snapshot.filesForScan(partitionPredicate, keepNumRecords = true).files val filesToProcess = optimizeContext.reorg match { case Some(reorgOperation) => reorgOperation.filterFilesToReorg(sparkSession, snapshot, candidateFiles) case None => filterCandidateFileList(minFileSize, maxDeletedRowsRatio, candidateFiles) } val partitionsToCompact = filesToProcess.groupBy(_.partitionValues).toSeq val jobs = groupFilesIntoBins(partitionsToCompact) val batchResults = batchSize match { case Some(size) => val batches = BinPackingUtils.binPackBySize[Bin, Bin]( jobs, bin => bin.files.map(_.size).sum, bin => bin, size) batches.map(batch => runOptimizeBatch(Batch(batch), maxFileSize)) case None => Seq(runOptimizeBatch(Batch(jobs), maxFileSize)) } val addedFiles = batchResults.map(_._1).flatten val removedFiles = batchResults.map(_._2).flatten val removedDVs = batchResults.map(_._3).flatten val optimizeStats = OptimizeStats() optimizeStats.addedFilesSizeStats.merge(addedFiles) optimizeStats.removedFilesSizeStats.merge(removedFiles) optimizeStats.numPartitionsOptimized = jobs.map(j => j.partitionValues).distinct.size optimizeStats.numBins = jobs.size optimizeStats.numBatches = batchResults.size optimizeStats.totalConsideredFiles = candidateFiles.size optimizeStats.totalFilesSkipped = optimizeStats.totalConsideredFiles - removedFiles.size optimizeStats.totalClusterParallelism = sparkSession.sparkContext.defaultParallelism val numTableColumns = snapshot.metadata.schema.size optimizeStats.numTableColumns = numTableColumns optimizeStats.numTableColumnsWithStats = DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.fromMetaData(snapshot.metadata) .min(numTableColumns) if (removedDVs.size > 0) { optimizeStats.deletionVectorStats = Some(DeletionVectorStats( numDeletionVectorsRemoved = removedDVs.size, numDeletionVectorRowsRemoved = removedDVs.map(_.cardinality).sum)) } optimizeStrategy.updateOptimizeStats(optimizeStats, removedFiles, jobs) return Seq(Row(snapshot.deltaLog.dataPath.toString, optimizeStats.toOptimizeMetrics)) } } /** * Helper method to prune the list of selected files based on fileSize and ratio of * deleted rows according to the deletion vector in [[AddFile]]. */ private def filterCandidateFileList( minFileSize: Long, maxDeletedRowsRatio: Double, files: Seq[AddFile]): Seq[AddFile] = { // Select all files in case of multi-dimensional clustering if (isMultiDimClustering) return files def shouldCompactBecauseOfDeletedRows(file: AddFile): Boolean = { // Always compact files with DVs but without numRecords stats. // This may be overly aggressive, but it fixes the problem in the long-term, // as the compacted files will have stats. (file.deletionVector != null && file.numPhysicalRecords.isEmpty) || file.deletedToPhysicalRecordsRatio.getOrElse(0d) > maxDeletedRowsRatio } // Select files that are small or have too many deleted rows files.filter( addFile => addFile.size < minFileSize || shouldCompactBecauseOfDeletedRows(addFile)) } /** * Utility methods to group files into bins for optimize. * * @param partitionsToCompact List of files to compact group by partition. * Partition is defined by the partition values (partCol -> partValue) * @return Sequence of bins. Each bin contains one or more files from the same * partition and targeted for one output file. */ private def groupFilesIntoBins( partitionsToCompact: Seq[(Map[String, String], Seq[AddFile])]) : Seq[Bin] = { val maxBinSize = optimizeStrategy.maxBinSize partitionsToCompact.flatMap { case (partition, files) => val bins = new ArrayBuffer[Seq[AddFile]]() val currentBin = new ArrayBuffer[AddFile]() var currentBinSize = 0L val preparedFiles = optimizeStrategy.prepareFilesPerPartition(files) preparedFiles.foreach { file => // Generally, a bin is a group of existing files, whose total size does not exceed the // desired maxBinSize. The output file size depends on the mode: // 1. Compaction: Files in a bin will be coalesced into a single output file. // 2. ZOrder: all files in a partition will be read by the // same job, the data will be range-partitioned and // numFiles = totalFileSize / maxFileSize will be produced. // 3. Clustering: Files in a bin belongs to one ZCUBE, the data will be // range-partitioned and numFiles = totalFileSize / maxFileSize. if (file.size + currentBinSize > maxBinSize) { bins += currentBin.toVector currentBin.clear() currentBin += file currentBinSize = file.size } else { currentBin += file currentBinSize += file.size } } if (currentBin.nonEmpty) { bins += currentBin.toVector } bins.filter { bin => bin.size > 1 || // bin has more than one file or bin.size == 1 && optimizeContext.reorg.nonEmpty || // always rewrite files during reorg isMultiDimClustering // multi-clustering }.map(b => Bin(partition, b)) } } private def runOptimizeBatch( batch: Batch, maxFileSize: Long ): (Seq[AddFile], Seq[RemoveFile], Seq[DeletionVectorDescriptor]) = { val txn = snapshot.deltaLog.startTransaction(catalogTable, Some(snapshot)) val filesToProcess = batch.bins.flatMap(_.files) txn.trackFilesRead(filesToProcess) txn.trackReadPredicates(partitionPredicate) val maxThreads = sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_MAX_THREADS) val updates = ThreadUtils.parmap(batch.bins, "OptimizeJob", maxThreads) { partitionBinGroup => runOptimizeBinJob(txn, partitionBinGroup.partitionValues, partitionBinGroup.files, maxFileSize) }.flatten val addedFiles = updates.collect { case a: AddFile => a } val removedFiles = updates.collect { case r: RemoveFile => r } val removedDVs = filesToProcess.filter(_.deletionVector != null).map(_.deletionVector).toSeq if (addedFiles.size > 0) { val metrics = createMetrics(sparkSession.sparkContext, addedFiles, removedFiles, removedDVs) commitAndRetry(txn, getOperation(), updates, metrics) { newTxn => val newPartitionSchema = newTxn.metadata.partitionSchema // Note: When checking if the candidate set is the same, we need to consider (Path, DV) // as the key. val candidateSetOld = filesToProcess. map(f => UniqueFileActionTuple(f.pathAsUri, f.getDeletionVectorUniqueId)).toSet // We specifically don't list the files through the transaction since we are potentially // only processing a subset of them below. If the transaction is still valid, we will // register the files and predicate below val candidateSetNew = newTxn.snapshot.filesForScan(partitionPredicate).files .map(f => UniqueFileActionTuple(f.pathAsUri, f.getDeletionVectorUniqueId)).toSet // As long as all of the files that we compacted are still part of the table, // and the partitioning has not changed it is valid to continue to try // and commit this checkpoint. if (candidateSetOld.subsetOf(candidateSetNew) && partitionSchema == newPartitionSchema) { // Make sure the files we are processing are registered with the transaction newTxn.trackFilesRead(filesToProcess) newTxn.trackReadPredicates(partitionPredicate) true } else { val deleted = candidateSetOld -- candidateSetNew logWarning(log"The following compacted files were deleted " + log"during checkpoint ${MDC(DeltaLogKeys.PATHS, deleted.mkString(","))}. " + log"Aborting the compaction.") false } } } (addedFiles, removedFiles, removedDVs) } /** * Utility method to run a Spark job to compact the files in given bin * * @param txn [[OptimisticTransaction]] instance in use to commit the changes to DeltaLog. * @param partition Partition values of the partition that files in [[bin]] belongs to. * @param bin List of files to compact into one large file. * @param maxFileSize Targeted output file size in bytes */ private def runOptimizeBinJob( txn: OptimisticTransaction, partition: Map[String, String], bin: Seq[AddFile], maxFileSize: Long): Seq[FileAction] = { val baseTablePath = txn.deltaLog.dataPath var input = txn.deltaLog.createDataFrame(txn.snapshot, bin, actionTypeOpt = Some("Optimize")) input = RowTracking.preserveRowTrackingColumns(input, txn.snapshot) val repartitionDF = if (isMultiDimClustering) { val totalSize = bin.map(_.size).sum val approxNumFiles = Math.max(1, totalSize / maxFileSize).toInt MultiDimClustering.cluster( input, approxNumFiles, clusteringColumns, optimizeStrategy.curve) } else { val useRepartition = sparkSession.sessionState.conf.getConf( DeltaSQLConf.DELTA_OPTIMIZE_REPARTITION_ENABLED) if (useRepartition) { input.repartition(numPartitions = 1) } else { input.coalesce(numPartitions = 1) } } val partitionDesc = partition.toSeq.map(entry => entry._1 + "=" + entry._2).mkString(",") val partitionName = if (partition.isEmpty) "" else s" in partition ($partitionDesc)" val description = s"$baseTablePath
Optimizing ${bin.size} files" + partitionName sparkSession.sparkContext.setJobGroup( sparkSession.sparkContext.getLocalProperty(SPARK_JOB_GROUP_ID), description) val binInfo = optimizeStrategy.initNewBin val addFiles = txn.writeFiles(repartitionDF, None, isOptimize = true, Nil).collect { case a: AddFile => optimizeStrategy.tagAddFile(a, binInfo) case other => throw new IllegalStateException( s"Unexpected action $other with type ${other.getClass}. File compaction job output" + s"should only have AddFiles") } val removeFiles = bin.map(f => f.removeWithTimestamp(operationTimestamp, dataChange = false)) val updates = addFiles ++ removeFiles updates } /** * Attempts to commit the given actions to the log. In the case of a concurrent update, * the given function will be invoked with a new transaction to allow custom conflict * detection logic to indicate it is safe to try again, by returning `true`. * * This function will continue to try to commit to the log as long as `f` returns `true`, * otherwise throws a subclass of [[ConcurrentModificationException]]. */ private def commitAndRetry( txn: OptimisticTransaction, optimizeOperation: Operation, actions: Seq[Action], metrics: Map[String, SQLMetric])(f: OptimisticTransaction => Boolean): Unit = { try { txn.registerSQLMetrics(sparkSession, metrics) txn.commit(actions, optimizeOperation, RowTracking.addPreservedRowTrackingTagIfNotSet(txn.snapshot)) } catch { case e: ConcurrentModificationException => val newTxn = txn.deltaLog.startTransaction(txn.catalogTable) if (f(newTxn)) { logInfo( log"Retrying commit after checking for semantic conflicts with concurrent updates.") commitAndRetry(newTxn, optimizeOperation, actions, metrics)(f) } else { logWarning(log"Semantic conflicts detected. Aborting operation.") throw e } } } /** Create the appropriate [[Operation]] object for txn commit history */ private def getOperation(): Operation = { if (optimizeContext.reorg.nonEmpty) { DeltaOperations.Reorg(partitionPredicate) } else { DeltaOperations.Optimize( predicate = partitionPredicate, zOrderBy = zOrderByColumns, auto = isAutoCompact, clusterBy = if (isClusteredTable) Option(clusteringColumns).filter(_.nonEmpty) else None, isFull = optimizeContext.isFull) } } /** Create a map of SQL metrics for adding to the commit history. */ private def createMetrics( sparkContext: SparkContext, addedFiles: Seq[AddFile], removedFiles: Seq[RemoveFile], removedDVs: Seq[DeletionVectorDescriptor]): Map[String, SQLMetric] = { def setAndReturnMetric(description: String, value: Long) = { val metric = createMetric(sparkContext, description) metric.set(value) metric } def totalSize(actions: Seq[FileAction]): Long = { var totalSize = 0L actions.foreach { file => val fileSize = file match { case addFile: AddFile => addFile.size case removeFile: RemoveFile => removeFile.size.getOrElse(0L) case default => throw new IllegalArgumentException(s"Unknown FileAction type: ${default.getClass}") } totalSize += fileSize } totalSize } val (deletionVectorRowsRemoved, deletionVectorBytesRemoved) = removedDVs.map(dv => (dv.cardinality, dv.sizeInBytes.toLong)) .reduceLeftOption((dv1, dv2) => (dv1._1 + dv2._1, dv1._2 + dv2._2)) .getOrElse((0L, 0L)) val dvMetrics: Map[String, SQLMetric] = Map( "numDeletionVectorsRemoved" -> setAndReturnMetric( "total number of deletion vectors removed", removedDVs.size), "numDeletionVectorRowsRemoved" -> setAndReturnMetric( "total number of deletion vector rows removed", deletionVectorRowsRemoved), "numDeletionVectorBytesRemoved" -> setAndReturnMetric( "total number of bytes of removed deletion vectors", deletionVectorBytesRemoved)) val sizeStats = FileSizeStatsWithHistogram.create(addedFiles.map(_.size).sorted) Map[String, SQLMetric]( "minFileSize" -> setAndReturnMetric("minimum file size", sizeStats.get.min), "p25FileSize" -> setAndReturnMetric("25th percentile file size", sizeStats.get.p25), "p50FileSize" -> setAndReturnMetric("50th percentile file size", sizeStats.get.p50), "p75FileSize" -> setAndReturnMetric("75th percentile file size", sizeStats.get.p75), "maxFileSize" -> setAndReturnMetric("maximum file size", sizeStats.get.max), "numAddedFiles" -> setAndReturnMetric("total number of files added.", addedFiles.size), "numRemovedFiles" -> setAndReturnMetric("total number of files removed.", removedFiles.size), "numAddedBytes" -> setAndReturnMetric("total number of bytes added", totalSize(addedFiles)), "numRemovedBytes" -> setAndReturnMetric("total number of bytes removed", totalSize(removedFiles)) ) ++ dvMetrics } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableStrategy.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands import org.apache.spark.sql.delta.skipping.clustering.{ClusteredTableUtils, ClusteringColumnInfo, ClusteringStatsCollector} import org.apache.spark.sql.delta.skipping.clustering.ZCube import org.apache.spark.sql.delta.{DeltaConfigs, DeltaErrors, OptimisticTransaction, Snapshot} import org.apache.spark.sql.delta.actions.{AddFile, DeletionVectorDescriptor, FileAction, RemoveFile} import org.apache.spark.sql.delta.commands.OptimizeTableStrategy.DummyBinInfo import org.apache.spark.sql.delta.commands.optimize.{AddFileWithNumRecords, DeletionVectorStats, OptimizeStats, ZOrderFileStats, ZOrderStats} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.sources.DeltaSQLConf.{DELTA_OPTIMIZE_CLUSTERING_MIN_CUBE_SIZE, DELTA_OPTIMIZE_CLUSTERING_TARGET_CUBE_SIZE} import org.apache.spark.sql.delta.zorder.ZCubeInfo import org.apache.spark.sql.SparkSession object OptimizeTableMode extends Enumeration { type OptimizeTableMode = Value val COMPACTION, ZORDER, CLUSTERING = Value } /** * Defines set of utilities used in OptimizeTableCommand. The behavior of these utilities will * change based on the [[OptimizeTableMode]]: COMPACTION, ZORDER and CLUSTERING. */ trait OptimizeTableStrategy { def sparkSession: SparkSession /** * Utility method to get max bin size in bytes to group files into. */ def maxBinSize: Long /** * Utility method to prepare files in a partition for optimization. * * By default it sorts files on the size for the binpack. * * @return Prepared files for the subsequent optimization. */ def prepareFilesPerPartition(inputFiles: Seq[AddFile]): Seq[AddFile] = inputFiles.sortBy(_.size) /** The optimize mode the strategy instance is created for. */ def optimizeTableMode: OptimizeTableMode.Value /** * The clustering algorithm to be used by either by ZORDER or Liquid CLUSTERING. * An error is thrown for COMPACTION. */ def curve: String /** * Prepare a new Bin and returns its initialized [[BinInfo]]. * * This function is expected to be called once for each bin * before [[tagAddFile]] is called. */ def initNewBin: OptimizeTableStrategy.BinInfo = DummyBinInfo() /** * Incorporate essential tags for optimized files based on the [[OptimizeTableMode]]. */ def tagAddFile(file: AddFile, binInfo: OptimizeTableStrategy.BinInfo): AddFile = file.copy(dataChange = false) /** * Utility to update additional metrics after optimization. * * @param optimizeStats The input stats to update on. * @param removedFiles Removed files. * @param bins Sequence of bin-packed file groups, * where each group consists of a partition value * and its associated files. */ def updateOptimizeStats( optimizeStats: OptimizeStats, removedFiles: Seq[RemoveFile], bins: Seq[Bin]): Unit } object OptimizeTableStrategy { // A trait representing the context for a Bin. sealed trait BinInfo /** Default [[BinInfo]] implementation. */ case class DummyBinInfo() extends BinInfo /** [[ClusteringStrategy]]'s [[BinInfo]]. */ case class ZCubeBinInfo(zCubeInfo: ZCubeInfo) extends BinInfo def apply( sparkSession: SparkSession, snapshot: Snapshot, optimizeContext: DeltaOptimizeContext, zOrderBy: Seq[String]): OptimizeTableStrategy = getMode(snapshot, zOrderBy) match { case OptimizeTableMode.CLUSTERING => ClusteringStrategy( sparkSession, ClusteringColumnInfo.extractLogicalNames(snapshot), optimizeContext) case OptimizeTableMode.ZORDER => ZOrderStrategy(sparkSession, zOrderBy) case OptimizeTableMode.COMPACTION => CompactionStrategy(sparkSession, optimizeContext) case other => throw new UnsupportedOperationException(s"Unsupported mode $other") } private def getMode(snapshot: Snapshot, zOrderBy: Seq[String]): OptimizeTableMode.Value = { val isClusteredTable = ClusteredTableUtils.isSupported(snapshot.protocol) val hasClusteringColumns = ClusteringColumnInfo.extractLogicalNames(snapshot).nonEmpty val isZOrderBy = zOrderBy.nonEmpty if (isClusteredTable && hasClusteringColumns) { assert(!isZOrderBy) OptimizeTableMode.CLUSTERING } else if (isZOrderBy) { OptimizeTableMode.ZORDER } else { OptimizeTableMode.COMPACTION } } } /** Implements compaction strategy */ case class CompactionStrategy( override val sparkSession: SparkSession, optimizeContext: DeltaOptimizeContext) extends OptimizeTableStrategy { override val optimizeTableMode: OptimizeTableMode.Value = OptimizeTableMode.COMPACTION // In COMPACTION, all files within a bin are written into single larger file. override val maxBinSize: Long = { optimizeContext.maxFileSize.getOrElse( sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_MAX_FILE_SIZE)) } override def curve: String = { throw new UnsupportedOperationException("Compaction doesn't support clustering.") } override def updateOptimizeStats( optimizeStats: OptimizeStats, removedFiles: Seq[RemoveFile], bins: Seq[Bin]): Unit = {} } /** Implements ZOrder strategy */ case class ZOrderStrategy( override val sparkSession: SparkSession, zOrderColumns: Seq[String]) extends OptimizeTableStrategy { assert(zOrderColumns.nonEmpty) override val optimizeTableMode: OptimizeTableMode.Value = OptimizeTableMode.ZORDER override val curve: String = "zorder" // For ZORDER, set maxBinSize the maximal LONG value to have single BIN for each partition. override val maxBinSize: Long = Long.MaxValue override def updateOptimizeStats( optimizeStats: OptimizeStats, removedFiles: Seq[RemoveFile], bins: Seq[Bin]): Unit = { val inputFileStats = ZOrderFileStats(removedFiles.size, removedFiles.map(_.size.getOrElse(0L)).sum) optimizeStats.zOrderStats = Some(ZOrderStats( strategyName = "all", // means process all files in a partition inputCubeFiles = ZOrderFileStats(0, 0), inputOtherFiles = inputFileStats, inputNumCubes = 0, mergedFiles = inputFileStats, // There will one z-cube for each partition numOutputCubes = optimizeStats.numPartitionsOptimized)) } } /** Implements clustering strategy for clustered tables */ case class ClusteringStrategy( override val sparkSession: SparkSession, clusteringColumns: Seq[String], optimizeContext: DeltaOptimizeContext) extends OptimizeTableStrategy { override val optimizeTableMode: OptimizeTableMode.Value = OptimizeTableMode.CLUSTERING override val curve: String = "hilbert" /** * In clustering, the bin size corresponds to a ZCube size that can be adjusted through * configurations. */ override val maxBinSize: Long = { Math.max( sparkSession.sessionState.conf.getConf(DELTA_OPTIMIZE_CLUSTERING_TARGET_CUBE_SIZE), sparkSession.sessionState.conf.getConf(DELTA_OPTIMIZE_CLUSTERING_MIN_CUBE_SIZE)) } // Prepare files for binpack. override def prepareFilesPerPartition(inputFiles: Seq[AddFile]): Seq[AddFile] = { // Un-clustered files don't have a ZCUBE_ID, and are sorted before clustered files (None is // considered the smallest element). We also don't consider partitionValues because // clustered tables should always be unpartitioned. applyMinZCube(inputFiles.sortBy(_.tag(AddFile.Tags.ZCUBE_ID))) } // Upon a new ZCube, allocate a [[ZCubeInfo]] with a new ZCUBE ID. override def initNewBin: OptimizeTableStrategy.BinInfo = { OptimizeTableStrategy.ZCubeBinInfo(ZCubeInfo(clusteringColumns)) } override def tagAddFile(file: AddFile, binInfo: OptimizeTableStrategy.BinInfo): AddFile = { val taggedFile = super.tagAddFile(file, binInfo) val zCubeInfo = binInfo.asInstanceOf[OptimizeTableStrategy.ZCubeBinInfo].zCubeInfo ZCubeInfo.setForFile( taggedFile.copy(clusteringProvider = Some(ClusteredTableUtils.clusteringProvider)), zCubeInfo) } override def updateOptimizeStats( optimizeStats: OptimizeStats, removedFiles: Seq[RemoveFile], bins: Seq[Bin]): Unit = { clusteringStatsCollector.numOutputZCubes = bins.size optimizeStats.clusteringStats = Option(clusteringStatsCollector.getClusteringStats) } /** * Given a sequence of files sorted by ZCubeId, return candidate files for * clustering. The requirements to pick candidate files are: * * 1. Candidate files are either un-clustered (missing clusteringProvider) or the * clusteringProvider is "liquid" when isFull is unset. * 2. Clustered files with different clustering columns are handled differently based * on isFull setting: If isFull is unset, existing clustered files with different columns are * skipped. If isFull is set, all files are considered. * 3. Files that belong to the partial ZCubes are picked. A ZCube is considered as a partial * ZCube if its size is smaller than [[DELTA_OPTIMIZE_CLUSTERING_MIN_CUBE_SIZE]]. * 4. If there is only single ZCUBE with all files are clustered and if all clustered files * belong to that ZCube, all files are filtered out. */ private def applyMinZCube(files: Seq[AddFile]): Seq[AddFile] = { val targetSize = sparkSession.sessionState.conf.getConf(DELTA_OPTIMIZE_CLUSTERING_MIN_CUBE_SIZE) // Keep all files if isFull is set, otherwise skip files with different clusteringProviders // or files clustered by a different set of clustering columns. val (candidateFiles, skippedClusteredFiles) = files.iterator.map { f => // Note that updateStats is moved out of Iterator.partition lambda since // scala2.13 doesn't call the lambda in the order of files which violates // the updateStats' requirement which requires files are ordered in the // ZCUBE id (files have been ordered before calling applyMinZCube). clusteringStatsCollector.inputStats.updateStats(f) f }.partition { file => val sameOrMissingClusteringProvider = file.clusteringProvider.forall(_ == ClusteredTableUtils.clusteringProvider) // If clustered before, remove those with different clustering columns. val zCubeInfo = ZCubeInfo.getForFile(file) val unmatchedClusteringColumns = zCubeInfo.exists(_.zOrderBy != clusteringColumns) sameOrMissingClusteringProvider && !unmatchedClusteringColumns } // Skip files that belong to a ZCUBE that is larger than target ZCUBE size. // Note that ZCube.filterOutLargeZCubes requires clustered files have // the same clustering columns, so skippedClusteredFiles are not included. val smallZCubeFiles = ZCube.filterOutLargeZCubes( candidateFiles.map(AddFileWithNumRecords.createFromFile), targetSize) if (optimizeContext.isFull && skippedClusteredFiles.nonEmpty) { // Clustered files with different clustering columns have to be re-clustered. val finalFiles = (smallZCubeFiles.map(_.addFile) ++ skippedClusteredFiles).toSeq finalFiles.map { f => clusteringStatsCollector.outputStats.updateStats(f) f } } else { // Skip smallZCubeFiles if they all belong to a single ZCUBE. ZCube.filterOutSingleZCubes(smallZCubeFiles).map { file => clusteringStatsCollector.outputStats.updateStats(file.addFile) file.addFile }.toSeq } } /** Metrics for clustering when [[isClusteredTable]] is true. */ private val clusteringStatsCollector: ClusteringStatsCollector = ClusteringStatsCollector(clusteringColumns, optimizeContext) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/ReorgTableForUpgradeUniformHelper.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands // scalastyle:off import.ordering.noEmptyLine import scala.util.control.NonFatal import org.apache.spark.sql.delta.{DeletionVectorsTableFeature, DeltaConfigs, DeltaErrors, DeltaOperations, IcebergCompatBase, Snapshot} import org.apache.spark.sql.delta.IcebergCompat.{getEnabledVersion, getForVersion} import org.apache.spark.sql.delta.UniversalFormat.{icebergEnabled, ICEBERG_FORMAT} import org.apache.spark.sql.delta.actions.{AddFile, Protocol} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.util.Utils.try_element_at import org.apache.spark.internal.MDC import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.functions.col /** * Helper trait for ReorgTableCommand to rewrite the table to be Iceberg compatible. */ trait ReorgTableForUpgradeUniformHelper extends DeltaLogging { private val rewriteCheckTable: Map[Int, Set[Int]] = Map(0 -> Set(2, 3), 1 -> Set(2, 3), 2 -> Set(2, 3), 3 -> Set(2, 3)) /** * Check if the given pair of (old_version, new_version) should trigger a rewrite check. * NOTE: Actual rewrite only happens when not all addFiles has tags with newVersion. */ private def shallCheckRewrite(oldVersion: Int, newVersion: Int): Boolean = { rewriteCheckTable.getOrElse(oldVersion, Set.empty[Int]).contains(newVersion) } /** * Helper function to rewrite the table. Implemented by Reorg Table Command. */ def optimizeByReorg(sparkSession: SparkSession): Seq[Row] /** * Enable the new IcebergCompat on the table by updating table conf. */ private def enableIcebergCompat( table: DeltaTableV2, currentCompatVersion: Option[Int], compatToEnable: IcebergCompatBase): Unit = { var newConf: Map[String, String] = Map( compatToEnable.config.key -> "true", DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name") ++ currentCompatVersion.map(getForVersion(_).config.key -> "false") // Disable old IcebergCompat if (compatToEnable.incompatibleTableFeatures.contains(DeletionVectorsTableFeature)) { newConf += DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key -> "false" } val alterConfTxn = table.startTransaction() if (alterConfTxn.protocol.minWriterVersion < 7) { newConf += Protocol.MIN_WRITER_VERSION_PROP -> "7" } if (alterConfTxn.protocol.minReaderVersion < 3) { newConf += Protocol.MIN_READER_VERSION_PROP -> "3" } val metadata = alterConfTxn.metadata val newMetadata = metadata.copy( description = metadata.description, configuration = metadata.configuration ++ newConf) alterConfTxn.updateMetadata(newMetadata) alterConfTxn.commit( Nil, DeltaOperations.UpgradeUniformProperties(newConf) ) } /** * Helper function to get the num of addFiles as well as * num of addFiles with ICEBERG_COMPAT_VERSION tag. * @param icebergCompatVersion target iceberg compat version * @param snapshot current snapshot * @return (NumOfAddFiles, NumOfAddFilesWithIcebergCompatTag) */ private def getNumOfAddFiles( icebergCompatVersion: Int, table: DeltaTableV2, snapshot: Snapshot): (Long, Long) = { val numOfAddFilesWithTag = snapshot.allFiles .select("tags") .where(try_element_at(col("tags"), AddFile.Tags.ICEBERG_COMPAT_VERSION.name) === icebergCompatVersion.toString) .count() val numOfAddFiles = snapshot.numOfFiles logInfo(log"For table ${MDC(DeltaLogKeys.TABLE_NAME, table.tableIdentifier)} " + log"at version ${MDC(DeltaLogKeys.VERSION, snapshot.version)}, there are " + log"${MDC(DeltaLogKeys.NUM_FILES, numOfAddFiles)} addFiles, and " + log"${MDC(DeltaLogKeys.NUM_FILES2, numOfAddFilesWithTag)} addFiles with " + log"ICEBERG_COMPAT_VERSION=${MDC(DeltaLogKeys.VERSION2, icebergCompatVersion.toLong)} tag.") (numOfAddFiles, numOfAddFilesWithTag) } /** * Helper function to rewrite the table data files in Iceberg compatible way. * This method would do following things: * 1. Update the table properties to enable the target iceberg compat version and disable the * existing iceberg compat version. * 2. If target iceberg compat version require rewriting and not all addFiles has * ICEBERG_COMPAT_VERSION=version tag, rewrite the table data files to be iceberg compatible * and adding tag to all addFiles. * 3. If universal format not enabled, alter the table properties to enable * universalFormat = Iceberg. * * * There are six possible write combinations: * | CurrentIcebergCompatVersion | TargetIcebergCompatVersion | Required steps| * | --------------------------- | -------------------------- | ------------- | * | None | 1 | 1, 3 | * | None | 2+ | 1, 2, 3 | * | 1 | 1 | 3 | * | 1 | 2+ | 1, 2, 3 | * | 2+ | 1 | 1, 3 | * | 2+ | 2+ | 2, 3 | */ private def doRewrite( target: DeltaTableV2, sparkSession: SparkSession, targetIcebergCompatVersion: Int): Seq[Row] = { val snapshot = target.update() val currIcebergCompatVersionOpt = getEnabledVersion(snapshot.metadata) if (targetIcebergCompatVersion >= 3) { throw DeltaErrors.icebergCompatVersionNotSupportedException(targetIcebergCompatVersion, 2) } val targetIcebergCompatObject = getForVersion(targetIcebergCompatVersion) val mayNeedRewrite = shallCheckRewrite( currIcebergCompatVersionOpt.getOrElse(0), targetIcebergCompatVersion) // Step 1: Update the table properties to enable the target iceberg compat version val didUpdateIcebergCompatVersion = if (!currIcebergCompatVersionOpt.contains(targetIcebergCompatVersion)) { enableIcebergCompat(target, currIcebergCompatVersionOpt, targetIcebergCompatObject) logInfo(log"Update table ${MDC(DeltaLogKeys.TABLE_NAME, target.tableIdentifier)} " + log"to iceberg compat version = " + log"${MDC(DeltaLogKeys.VERSION, targetIcebergCompatVersion)} successfully.") true } else { false } // Step 2: Rewrite the table data files to be Iceberg compatible. val (numOfAddFilesBefore, numOfAddFilesWithTagBefore) = getNumOfAddFiles( targetIcebergCompatVersion, target, snapshot) val allAddFilesHaveTag = numOfAddFilesWithTagBefore == numOfAddFilesBefore // The table needs to be rewritten if: // 1. The target iceberg compat version requires rewrite. // 2. Not all addFile have ICEBERG_COMPAT_VERSION=targetVersion tag val (metricsOpt, didRewrite) = if (mayNeedRewrite && !allAddFilesHaveTag) { logInfo(log"Reorg Table ${MDC(DeltaLogKeys.TABLE_NAME, target.tableIdentifier)} to " + log"iceberg compat version = ${MDC(DeltaLogKeys.VERSION, targetIcebergCompatVersion)} " + log"need rewrite data files.") val metrics = try { optimizeByReorg(sparkSession) } catch { case NonFatal(e) => throw DeltaErrors.icebergCompatDataFileRewriteFailedException( targetIcebergCompatVersion, e) } logInfo(log"Rewrite table ${MDC(DeltaLogKeys.TABLE_NAME, target.tableIdentifier)} " + log"to iceberg compat version = ${MDC(DeltaLogKeys.VERSION, targetIcebergCompatVersion)} successfully.") (Some(metrics), true) } else { (None, false) } val updatedSnapshot = target.update() val (numOfAddFiles, numOfAddFilesWithIcebergCompatTag) = getNumOfAddFiles( targetIcebergCompatVersion, target, updatedSnapshot) if (mayNeedRewrite && numOfAddFilesWithIcebergCompatTag != numOfAddFiles) { throw DeltaErrors.icebergCompatReorgAddFileTagsMissingException( updatedSnapshot.version, targetIcebergCompatVersion, numOfAddFiles, numOfAddFilesWithIcebergCompatTag ) } // Step 3: Update the table properties to enable the universalFormat = Iceberg. if (!icebergEnabled(updatedSnapshot.metadata)) { val enableUniformConf = Map( DeltaConfigs.UNIVERSAL_FORMAT_ENABLED_FORMATS.key -> ICEBERG_FORMAT) AlterTableSetPropertiesDeltaCommand(target, enableUniformConf).run(sparkSession) logInfo(log"Enabling universal format with iceberg compat version = " + log"${MDC(DeltaLogKeys.VERSION, targetIcebergCompatVersion)} for table " + log"${MDC(DeltaLogKeys.TABLE_NAME, target.tableIdentifier)} succeeded.") } recordDeltaEvent(updatedSnapshot.deltaLog, "delta.upgradeUniform.success", data = Map( "currIcebergCompatVersion" -> currIcebergCompatVersionOpt.toString, "targetIcebergCompatVersion" -> targetIcebergCompatVersion.toString, "metrics" -> metricsOpt.toString, "didUpdateIcebergCompatVersion" -> didUpdateIcebergCompatVersion.toString, "needRewrite" -> mayNeedRewrite.toString, "didRewrite" -> didRewrite.toString, "numOfAddFilesBefore" -> numOfAddFilesBefore.toString, "numOfAddFilesWithIcebergCompatTagBefore" -> numOfAddFilesWithTagBefore.toString, "numOfAddFilesAfter" -> numOfAddFiles.toString, "numOfAddFilesWithIcebergCompatTagAfter" -> numOfAddFilesWithIcebergCompatTag.toString, "universalFormatIcebergEnabled" -> icebergEnabled(target.update().metadata).toString )) metricsOpt.getOrElse(Seq.empty[Row]) } /** * Helper function to upgrade the table to uniform iceberg compat version. */ protected def upgradeUniformIcebergCompatVersion( target: DeltaTableV2, sparkSession: SparkSession, targetIcebergCompatVersion: Int): Seq[Row] = { try { doRewrite(target, sparkSession, targetIcebergCompatVersion) } catch { case NonFatal(e) => recordDeltaEvent(target.deltaLog, "delta.upgradeUniform.exception", data = Map( "targetIcebergCompatVersion" -> targetIcebergCompatVersion.toString, "exception" -> e.toString )) throw e } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/ReorgTableHelper.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands import org.apache.spark.sql.delta.{MaterializedRowCommitVersion, MaterializedRowId, Snapshot} import org.apache.spark.sql.delta.actions.{AddFile, Metadata, Protocol} import org.apache.spark.sql.delta.commands.VacuumCommand.generateCandidateFileMap import org.apache.spark.sql.delta.schema.{SchemaMergingUtils, SchemaUtils} import org.apache.spark.sql.delta.util.DeltaFileOperations import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.sql.SparkSession import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat, ParquetToSparkSchemaConverter} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{AtomicType, StructField, StructType} import org.apache.spark.util.SerializableConfiguration trait ReorgTableHelper extends Serializable { /** * Determine whether `fileSchema` has any column that has a type that differs from * `tablePhysicalSchema`. * * @param fileSchema the current parquet schema to be checked. * @param tablePhysicalSchema the current table schema. * @return whether the file has any column that has a different type from table column. */ protected def fileHasDifferentTypes( fileSchema: StructType, tablePhysicalSchema: StructType): Boolean = { SchemaMergingUtils.transformColumns(fileSchema, tablePhysicalSchema) { case (_, StructField(_, fileType: AtomicType, _, _), Some(StructField(_, tableType: AtomicType, _, _)), _) if fileType != tableType => return true case (_, field, _, _) => field } false } /** * Determine whether `fileSchema` has any column that does not exist in the * `tablePhysicalSchema`, this is possible by running ALTER TABLE commands, * e.g., ALTER TABLE DROP COLUMN. * * @param fileSchema the current parquet schema to be checked. * @param tablePhysicalSchema the current table schema. * @param protocol the protocol used to check `row_id` and `row_commit_version`. * @param metadata the metadata used to check `row_id` and `row_commit_version`. * @return whether the file has any dropped column. */ protected def fileHasExtraColumns( fileSchema: StructType, tablePhysicalSchema: StructType, protocol: Protocol, metadata: Metadata): Boolean = { // 0. get the materialized names for `row_id` and `row_commit_version`. val materializedRowIdColumnNameOpt = MaterializedRowId.getMaterializedColumnName(protocol, metadata) val materializedRowCommitVersionColumnNameOpt = MaterializedRowCommitVersion.getMaterializedColumnName(protocol, metadata) SchemaMergingUtils.transformColumns(fileSchema) { (path, field, _) => // 1. check whether the field exists in the `tablePhysicalSchema`. val fullName = path :+ field.name val inTableFieldOpt = SchemaUtils.findNestedFieldIgnoreCase( tablePhysicalSchema, fullName, includeCollections = true) // 2. check whether the current `field` is `row_id` or `row_commit_version` // column; if so, we need to explicitly keep these columns since they are // not part of the table schema but exist in the parquet file. val isRowIdOrRowCommitVersion = materializedRowIdColumnNameOpt.contains(field.name) || materializedRowCommitVersionColumnNameOpt.contains(field.name) if (inTableFieldOpt.isEmpty && !isRowIdOrRowCommitVersion) { return true } field } false } /** * Apply a filter on the list of AddFile to only keep the files that have physical parquet schema * that satisfies the given filter function. * * Note: Filtering happens on the executors: **any variable captured by `filterFileFn` must be * Serializable** */ protected def filterParquetFilesOnExecutors( spark: SparkSession, files: Seq[AddFile], snapshot: Snapshot, ignoreCorruptFiles: Boolean)( filterFileFn: StructType => Boolean): Seq[AddFile] = { val serializedConf = new SerializableConfiguration(snapshot.deltaLog.newDeltaHadoopConf()) val dataPath = new Path(snapshot.deltaLog.dataPath.toString) import org.apache.spark.sql.delta.implicits._ files.toDF(spark).as[AddFile].mapPartitions { iter => val sqlConf = SparkSession.active.sessionState.conf filterParquetFiles( sqlConf, iter.toList, dataPath, serializedConf.value, ignoreCorruptFiles)(filterFileFn).toIterator }.collect() } protected def filterParquetFiles( sqlConf: SQLConf, files: Seq[AddFile], dataPath: Path, configuration: Configuration, ignoreCorruptFiles: Boolean)( filterFileFn: StructType => Boolean): Seq[AddFile] = { val nameToAddFileMap = generateCandidateFileMap(dataPath, files) val fileStatuses = nameToAddFileMap.map { case (absPath, addFile) => new FileStatus( /* length */ addFile.size, /* isDir */ false, /* blockReplication */ 0, /* blockSize */ 1, /* modificationTime */ addFile.modificationTime, new Path(absPath) ) } val footers = DeltaFileOperations.readParquetFootersInParallel( configuration, fileStatuses.toList, ignoreCorruptFiles) // Spark 4.0.1 changed the primary ctor signature (added a param), which breaks binary // compatibility for code compiled against Spark 4.0.0. Use the stable SQLConf-based ctor // that takes the current SparkSession's SQLConf instead. val converter = new ParquetToSparkSchemaConverter(sqlConf) val filesNeedToRewrite = footers.filter { footer => val fileSchema = ParquetFileFormat.readSchemaFromFooter(footer, converter) filterFileFn(fileSchema) }.map(_.getFile.toString) filesNeedToRewrite.map(absPath => nameToAddFileMap(absPath)) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/RestoreTableCommand.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands import java.sql.Timestamp import scala.collection.JavaConverters._ import scala.util.{Success, Try} import org.apache.spark.sql.delta.{DeltaErrors, DeltaLog, DeltaOperations, DomainMetadataUtils, IdentityColumn, Snapshot} import org.apache.spark.sql.delta.actions.{AddFile, DeletionVectorDescriptor, RemoveFile} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.DeltaFileOperations.absolutePath import org.apache.hadoop.fs.Path import org.apache.spark.sql.{Column, DataFrame, Dataset, Row, SparkSession} import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Literal} import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.execution.command.LeafRunnableCommand import org.apache.spark.sql.functions.{column, lit} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.IGNORE_MISSING_FILES import org.apache.spark.sql.types.LongType import org.apache.spark.unsafe.types.UTF8String import org.apache.spark.util.SerializableConfiguration /** Base trait class for RESTORE. Defines command output schema and metrics. */ trait RestoreTableCommandBase { // RESTORE operation metrics val TABLE_SIZE_AFTER_RESTORE = "tableSizeAfterRestore" val NUM_OF_FILES_AFTER_RESTORE = "numOfFilesAfterRestore" val NUM_REMOVED_FILES = "numRemovedFiles" val NUM_RESTORED_FILES = "numRestoredFiles" val REMOVED_FILES_SIZE = "removedFilesSize" val RESTORED_FILES_SIZE = "restoredFilesSize" // SQL way column names for RESTORE command output private val COLUMN_TABLE_SIZE_AFTER_RESTORE = "table_size_after_restore" private val COLUMN_NUM_OF_FILES_AFTER_RESTORE = "num_of_files_after_restore" private val COLUMN_NUM_REMOVED_FILES = "num_removed_files" private val COLUMN_NUM_RESTORED_FILES = "num_restored_files" private val COLUMN_REMOVED_FILES_SIZE = "removed_files_size" private val COLUMN_RESTORED_FILES_SIZE = "restored_files_size" val outputSchema: Seq[Attribute] = Seq( AttributeReference(COLUMN_TABLE_SIZE_AFTER_RESTORE, LongType)(), AttributeReference(COLUMN_NUM_OF_FILES_AFTER_RESTORE, LongType)(), AttributeReference(COLUMN_NUM_REMOVED_FILES, LongType)(), AttributeReference(COLUMN_NUM_RESTORED_FILES, LongType)(), AttributeReference(COLUMN_REMOVED_FILES_SIZE, LongType)(), AttributeReference(COLUMN_RESTORED_FILES_SIZE, LongType)() ) } /** * Perform restore of delta table to a specified version or timestamp * * Algorithm: * 1) Read the latest snapshot of the table. * 2) Read snapshot for version or timestamp to restore * 3) Compute files available in snapshot for restoring (files were removed by some commit) * but missed in the latest. Add these files into commit as AddFile action. * 4) Compute files available in the latest snapshot (files were added after version to restore) * but missed in the snapshot to restore. Add these files into commit as RemoveFile action. * 5) If SQLConf.IGNORE_MISSING_FILES option is false (default value) check availability of AddFile * in file system. * 6) Commit metadata, Protocol, all RemoveFile and AddFile actions * into delta log using `commitLarge` (commit will be failed in case of parallel transaction) * 7) If table was modified in parallel then ignore restore and raise exception. * */ case class RestoreTableCommand(sourceTable: DeltaTableV2) extends LeafRunnableCommand with DeltaCommand with RestoreTableCommandBase { override val output: Seq[Attribute] = outputSchema override def run(spark: SparkSession): Seq[Row] = { val deltaLog = sourceTable.deltaLog val catalogTableOpt = sourceTable.catalogTable val version = sourceTable.timeTravelOpt.get.version val timestamp = getTimestamp() recordDeltaOperation(deltaLog, "delta.restore") { require(version.isEmpty ^ timestamp.isEmpty, "Either the version or timestamp should be provided for restore") val versionToRestore = version.getOrElse { deltaLog .history .getActiveCommitAtTime( parseStringToTs(timestamp), catalogTableOpt, canReturnLastCommit = true) .version } val latestVersion = deltaLog .update(catalogTableOpt = catalogTableOpt) .version require(versionToRestore < latestVersion, s"Version to restore ($versionToRestore)" + s"should be less then last available version ($latestVersion)") deltaLog.withNewTransaction(catalogTableOpt) { txn => val latestSnapshot = txn.snapshot val snapshotToRestore = deltaLog.getSnapshotAt( versionToRestore, catalogTableOpt = txn.catalogTable, enforceTimeTravelWithinDeletedFileRetention = true) val latestSnapshotFiles = latestSnapshot.allFiles val snapshotToRestoreFiles = snapshotToRestore.allFiles import org.apache.spark.sql.delta.implicits._ // If either source version or destination version contains DVs, // we have to take them into account during deduplication. val targetMayHaveDVs = DeletionVectorUtils.deletionVectorsReadable(latestSnapshot) val sourceMayHaveDVs = DeletionVectorUtils.deletionVectorsReadable(snapshotToRestore) val normalizedSourceWithoutDVs = snapshotToRestoreFiles.mapPartitions { files => files.map(file => (file, file.path)) }.toDF("srcAddFile", "srcPath") val normalizedTargetWithoutDVs = latestSnapshotFiles.mapPartitions { files => files.map(file => (file, file.path)) }.toDF("tgtAddFile", "tgtPath") def addDVsToNormalizedDF( mayHaveDVs: Boolean, dvIdColumnName: String, dvAccessColumn: Column, normalizedDf: DataFrame): DataFrame = { if (mayHaveDVs) { normalizedDf.withColumn( dvIdColumnName, DeletionVectorDescriptor.uniqueIdExpression(dvAccessColumn)) } else { normalizedDf.withColumn(dvIdColumnName, lit(null)) } } val normalizedSource = addDVsToNormalizedDF( mayHaveDVs = sourceMayHaveDVs, dvIdColumnName = "srcDeletionVectorId", dvAccessColumn = column("srcAddFile.deletionVector"), normalizedDf = normalizedSourceWithoutDVs) val normalizedTarget = addDVsToNormalizedDF( mayHaveDVs = targetMayHaveDVs, dvIdColumnName = "tgtDeletionVectorId", dvAccessColumn = column("tgtAddFile.deletionVector"), normalizedDf = normalizedTargetWithoutDVs) val joinExprs = column("srcPath") === column("tgtPath") and // Use comparison operator where NULL == NULL column("srcDeletionVectorId") <=> column("tgtDeletionVectorId") val filesToAdd = normalizedSource .join(normalizedTarget, joinExprs, "left_anti") .select(column("srcAddFile").as[AddFile]) .map(_.copy(dataChange = true)) val filesToRemove = normalizedTarget .join(normalizedSource, joinExprs, "left_anti") .select(column("tgtAddFile").as[AddFile]) .map(_.removeWithTimestamp()) val ignoreMissingFiles = spark .sessionState .conf .getConf(IGNORE_MISSING_FILES) if (!ignoreMissingFiles) { checkSnapshotFilesAvailability(deltaLog, filesToAdd, versionToRestore) } // Commit files, metrics, protocol and metadata to delta log val metrics = withDescription("metrics") { computeMetrics(filesToAdd, filesToRemove, snapshotToRestore) } val addActions = withDescription("add actions") { filesToAdd.toLocalIterator().asScala } val removeActions = withDescription("remove actions") { filesToRemove.toLocalIterator().asScala } // We need to merge the schema of the latest snapshot with the schema of the snapshot // we're restoring to ensure that the high water mark is correct. val mergedSchema = IdentityColumn.copySchemaWithMergedHighWaterMarks( deltaLog = deltaLog, schemaToCopy = snapshotToRestore.metadata.schema, schemaWithHighWaterMarksToMerge = latestSnapshot.metadata.schema ) txn.updateMetadata(snapshotToRestore.metadata.copy(schemaString = mergedSchema.json)) val sourceProtocol = snapshotToRestore.protocol val targetProtocol = latestSnapshot.protocol // Only upgrade the protocol, never downgrade (unless allowed by flag), since that may break // time travel. val protocolDowngradeAllowed = conf.getConf(DeltaSQLConf.RESTORE_TABLE_PROTOCOL_DOWNGRADE_ALLOWED) val newProtocol = if (protocolDowngradeAllowed) { sourceProtocol } else { sourceProtocol.merge(targetProtocol) } val actions = addActions ++ removeActions ++ DomainMetadataUtils.handleDomainMetadataForRestoreTable(snapshotToRestore, latestSnapshot) txn.commitLarge( spark, actions, Some(newProtocol), DeltaOperations.Restore(version, timestamp), Map.empty, metrics.mapValues(_.toString).toMap) Seq(Row( metrics.get(TABLE_SIZE_AFTER_RESTORE), metrics.get(NUM_OF_FILES_AFTER_RESTORE), metrics.get(NUM_REMOVED_FILES), metrics.get(NUM_RESTORED_FILES), metrics.get(REMOVED_FILES_SIZE), metrics.get(RESTORED_FILES_SIZE))) } } } private def withDescription[T](action: String)(f: => T): T = withStatusCode("DELTA", s"RestoreTableCommand: compute $action (table path ${sourceTable.deltaLog.dataPath})") { f } private def parseStringToTs(timestamp: Option[String]): Timestamp = { Try { timestamp.flatMap { tsStr => val tz = DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone) val utfStr = UTF8String.fromString(tsStr) DateTimeUtils.stringToTimestamp(utfStr, tz) } } match { case Success(Some(tsMicroseconds)) => new Timestamp(tsMicroseconds / 1000) case _ => throw DeltaErrors.timestampInvalid(Literal(timestamp.get)) } } private def computeMetrics( toAdd: Dataset[AddFile], toRemove: Dataset[RemoveFile], snapshot: Snapshot ): Map[String, Long] = { // scalastyle:off sparkimplicits import toAdd.sparkSession.implicits._ // scalastyle:on sparkimplicits val (numRestoredFiles, restoredFilesSize) = toAdd .agg("size" -> "count", "size" -> "sum").as[(Long, Option[Long])].head() val (numRemovedFiles, removedFilesSize) = toRemove .agg("size" -> "count", "size" -> "sum").as[(Long, Option[Long])].head() Map( NUM_RESTORED_FILES -> numRestoredFiles, RESTORED_FILES_SIZE -> restoredFilesSize.getOrElse(0), NUM_REMOVED_FILES -> numRemovedFiles, REMOVED_FILES_SIZE -> removedFilesSize.getOrElse(0), NUM_OF_FILES_AFTER_RESTORE -> snapshot.numOfFiles, TABLE_SIZE_AFTER_RESTORE -> snapshot.sizeInBytes ) } /* Prevent users from running restore to table version with missed * data files (manually deleted or vacuumed). Restoring to this version partially * is still possible if spark.sql.files.ignoreMissingFiles is set to true */ private def checkSnapshotFilesAvailability( deltaLog: DeltaLog, files: Dataset[AddFile], version: Long): Unit = withDescription("missing files validation") { val spark: SparkSession = files.sparkSession val pathString = deltaLog.dataPath.toString val hadoopConf = spark.sparkContext.broadcast( new SerializableConfiguration(deltaLog.newDeltaHadoopConf())) import org.apache.spark.sql.delta.implicits._ val missedFiles = files .mapPartitions { files => val path = new Path(pathString) val fs = path.getFileSystem(hadoopConf.value.value) files.filterNot(f => fs.exists(absolutePath(pathString, f.path))) } .map(_.path) .head(100) if (missedFiles.nonEmpty) { throw DeltaErrors.restoreMissedDataFilesError(missedFiles, version) } } /** If available get the timestamp referring to a snapshot in the source table timeline */ private def getTimestamp(): Option[String] = { if (sourceTable.timeTravelOpt.get.timestamp.isDefined) { Some(sourceTable.timeTravelOpt.get.getTimestamp(conf).toString) } else { None } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/ShowDeltaTableColumnsCommand.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode} import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes import org.apache.spark.sql.execution.command.RunnableCommand /** * The column format of the result returned by the `SHOW COLUMNS` command. */ case class TableColumns(col_name: String) /** * A command for listing all column names of a Delta table. * * @param child The resolved Delta table */ case class ShowDeltaTableColumnsCommand(child: LogicalPlan) extends RunnableCommand with UnaryNode with DeltaCommand { override val output: Seq[Attribute] = toAttributes(ExpressionEncoder[TableColumns]().schema) override protected def withNewChildInternal(newChild: LogicalPlan): ShowDeltaTableColumnsCommand = copy(child = newChild) override def run(sparkSession: SparkSession): Seq[Row] = { // Return the schema from snapshot if it is an Delta table. Or raise // `DeltaErrors.notADeltaTableException` if it is a non-Delta table. val deltaTable = getDeltaTable(child, "SHOW COLUMNS") recordDeltaOperation(deltaTable.deltaLog, "delta.ddl.showColumns") { deltaTable.update().schema.fieldNames.map { x => Row(x) }.toSeq } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands // scalastyle:off import.ordering.noEmptyLine import java.util.concurrent.TimeUnit import scala.util.control.NonFatal import org.apache.spark.sql.delta.metric.IncrementMetric import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions.{Action, AddCDCFile, AddFile, FileAction} import org.apache.spark.sql.delta.commands.cdc.CDCReader.{CDC_TYPE_COLUMN_NAME, CDC_TYPE_NOT_CDC, CDC_TYPE_UPDATE_POSTIMAGE, CDC_TYPE_UPDATE_PREIMAGE} import org.apache.spark.sql.delta.files.{TahoeBatchFileIndex, TahoeFileIndex} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.StatsCollectionUtils import com.fasterxml.jackson.databind.annotation.JsonDeserialize import org.apache.hadoop.fs.Path import org.apache.spark.SparkContext import org.apache.spark.sql.{Column, DataFrame, Dataset, Row, SparkSession} import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression, If, Literal} import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral import org.apache.spark.sql.catalyst.plans.QueryPlan import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.delta.DeltaOperations.Operation import org.apache.spark.sql.execution.command.LeafRunnableCommand import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.execution.metric.SQLMetrics.{createMetric, createTimingMetric} import org.apache.spark.sql.functions.{array, col, explode, input_file_name, lit, struct} import org.apache.spark.sql.types.LongType /** * Performs an Update using `updateExpression` on the rows that match `condition` * * Algorithm: * 1) Identify the affected files, i.e., the files that may have the rows to be updated. * 2) Scan affected files, apply the updates, and generate a new DF with updated rows. * 3) Use the Delta protocol to atomically write the new DF as new files and remove * the affected files that are identified in step 1. */ case class UpdateCommand( tahoeFileIndex: TahoeFileIndex, catalogTable: Option[CatalogTable], target: LogicalPlan, updateExpressions: Seq[Expression], condition: Option[Expression]) extends LeafRunnableCommand with DeltaCommand { override val output: Seq[Attribute] = { Seq(AttributeReference("num_affected_rows", LongType)()) } override def innerChildren: Seq[QueryPlan[_]] = Seq(target) @transient private lazy val sc: SparkContext = SparkContext.getOrCreate() override lazy val metrics = Map[String, SQLMetric]( "numAddedFiles" -> createMetric(sc, "number of files added."), "numAddedBytes" -> createMetric(sc, "number of bytes added"), "numRemovedFiles" -> createMetric(sc, "number of files removed."), "numRemovedBytes" -> createMetric(sc, "number of bytes removed"), "numUpdatedRows" -> createMetric(sc, "number of rows updated."), "numCopiedRows" -> createMetric(sc, "number of rows copied."), "executionTimeMs" -> createTimingMetric(sc, "time taken to execute the entire operation"), "scanTimeMs" -> createTimingMetric(sc, "time taken to scan the files for matches"), "rewriteTimeMs" -> createTimingMetric(sc, "time taken to rewrite the matched files"), "numAddedChangeFiles" -> createMetric(sc, "number of change data capture files generated"), "changeFileBytes" -> createMetric(sc, "total size of change data capture files generated"), "numTouchedRows" -> createMetric(sc, "number of rows touched (copied + updated)"), "numDeletionVectorsAdded" -> createMetric(sc, "number of deletion vectors added"), "numDeletionVectorsRemoved" -> createMetric(sc, "number of deletion vectors removed"), "numDeletionVectorsUpdated" -> createMetric(sc, "number of deletion vectors updated") ) final override def run(sparkSession: SparkSession): Seq[Row] = { recordDeltaOperation(tahoeFileIndex.deltaLog, "delta.dml.update") { val deltaLog = tahoeFileIndex.deltaLog deltaLog.withNewTransaction(catalogTable) { txn => DeltaLog.assertRemovable(txn.snapshot) if (hasBeenExecuted(txn, sparkSession)) { sendDriverMetrics(sparkSession, metrics) return Seq.empty } performUpdate(sparkSession, deltaLog, txn) } // Re-cache all cached plans(including this relation itself, if it's cached) that refer to // this data source relation. sparkSession.sharedState.cacheManager.recacheByPlan(sparkSession, target) } Seq(Row(metrics("numUpdatedRows").value)) } private def performUpdate( sparkSession: SparkSession, deltaLog: DeltaLog, txn: OptimisticTransaction): Unit = { import org.apache.spark.sql.delta.implicits._ var numTouchedFiles: Long = 0 var numRewrittenFiles: Long = 0 var numAddedBytes: Long = 0 var numRemovedBytes: Long = 0 var numAddedChangeFiles: Long = 0 var changeFileBytes: Long = 0 var scanTimeMs: Long = 0 var rewriteTimeMs: Long = 0 var numDeletionVectorsAdded: Long = 0 var numDeletionVectorsRemoved: Long = 0 var numDeletionVectorsUpdated: Long = 0 val startTime = System.nanoTime() val numFilesTotal = txn.snapshot.numOfFiles val updateCondition = condition.getOrElse(Literal.TrueLiteral) val (metadataPredicates, dataPredicates) = DeltaTableUtils.splitMetadataAndDataPredicates( updateCondition, txn.metadata.partitionColumns, sparkSession) // Should we write the DVs to represent updated rows? val shouldWriteDeletionVectors = shouldWritePersistentDeletionVectors(sparkSession, txn) val candidateFiles = txn.filterFiles( metadataPredicates ++ dataPredicates, keepNumRecords = shouldWriteDeletionVectors) val nameToAddFile = generateCandidateFileMap(deltaLog.dataPath, candidateFiles) scanTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime) val filesToRewrite: Seq[TouchedFileWithDV] = if (candidateFiles.isEmpty) { // Case 1: Do nothing if no row qualifies the partition predicates // that are part of Update condition Nil } else if (dataPredicates.isEmpty) { // Case 2: Update all the rows from the files that are in the specified partitions // when the data filter is empty candidateFiles .map(f => TouchedFileWithDV(f.path, f, newDeletionVector = null, deletedRows = 0L)) } else { // Case 3: Find all the affected files using the user-specified condition val fileIndex = new TahoeBatchFileIndex( sparkSession, "update", candidateFiles, deltaLog, tahoeFileIndex.path, txn.snapshot) val touchedFilesWithDV = if (shouldWriteDeletionVectors) { // Case 3.1: Find all the affected files via DV path val targetDf = DMLWithDeletionVectorsHelper.createTargetDfForScanningForMatches( sparkSession, target, fileIndex) // Does the target table already has DVs enabled? If so, we need to read the table // with deletion vectors. val mustReadDeletionVectors = DeletionVectorUtils.deletionVectorsReadable(txn.snapshot) DMLWithDeletionVectorsHelper.findTouchedFiles( sparkSession, txn, mustReadDeletionVectors, deltaLog, targetDf, fileIndex, updateCondition, opName = "UPDATE") } else { // Case 3.2: Find all the affected files using the non-DV path // Keep everything from the resolved target except a new TahoeFileIndex // that only involves the affected files instead of all files. val newTarget = DeltaTableUtils.replaceFileIndex(target, fileIndex) val data = DataFrameUtils.ofRows(sparkSession, newTarget) val incrUpdatedCountExpr = IncrementMetric(TrueLiteral, metrics("numUpdatedRows")) val pathsToRewrite = withStatusCode("DELTA", UpdateCommand.FINDING_TOUCHED_FILES_MSG) { data.filter(Column(updateCondition)) .select(input_file_name()) .filter(Column(incrUpdatedCountExpr)) .distinct() .as[String] .collect() } // Wrap AddFile into TouchedFileWithDV that has empty DV. pathsToRewrite .map(getTouchedFile(deltaLog.dataPath, _, nameToAddFile)) .map(f => TouchedFileWithDV(f.path, f, newDeletionVector = null, deletedRows = 0L)) .toSeq } // Refresh scan time for Case 3, since we performed scan here. scanTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime) touchedFilesWithDV } val totalActions = { // When DV is on, we first mask removed rows with DVs and generate (remove, add) pairs. val actionsForExistingFiles = if (shouldWriteDeletionVectors) { // When there's no data predicate, all matched files are removed. if (dataPredicates.isEmpty) { val operationTimestamp = System.currentTimeMillis() filesToRewrite.map(_.fileLogEntry.removeWithTimestamp(operationTimestamp)) } else { // When there is data predicate, we generate (remove, add) pairs. val filesToRewriteWithDV = filesToRewrite.filter(_.newDeletionVector != null) val stringTruncateLength = StatsCollectionUtils.getDataSkippingStringPrefixLength( sparkSession, txn.metadata) val (dvActions, metricMap) = DMLWithDeletionVectorsHelper.processUnmodifiedData( sparkSession, filesToRewriteWithDV, txn.snapshot, stringTruncateLength) metrics("numUpdatedRows").set(metricMap("numModifiedRows")) numDeletionVectorsAdded = metricMap("numDeletionVectorsAdded") numDeletionVectorsRemoved = metricMap("numDeletionVectorsRemoved") numDeletionVectorsUpdated = metricMap("numDeletionVectorsUpdated") numTouchedFiles = metricMap("numRemovedFiles") dvActions } } else { // Without DV we'll leave the job to `rewriteFiles`. Nil } // When DV is on, we write out updated rows only. The return value will be only `add` actions. // When DV is off, we write out updated rows plus unmodified rows from the same file, then // return `add` and `remove` actions. val rewriteStartNs = System.nanoTime() val actionsForNewFiles = withStatusCode("DELTA", UpdateCommand.rewritingFilesMsg(filesToRewrite.size)) { if (filesToRewrite.nonEmpty) { rewriteFiles( sparkSession, txn, rootPath = tahoeFileIndex.path, inputLeafFiles = filesToRewrite.map(_.fileLogEntry), nameToAddFileMap = nameToAddFile, condition = updateCondition, generateRemoveFileActions = !shouldWriteDeletionVectors, copyUnmodifiedRows = !shouldWriteDeletionVectors) } else { Nil } } rewriteTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - rewriteStartNs) numTouchedFiles = filesToRewrite.length val (addActions, removeActions) = actionsForNewFiles.partition(_.isInstanceOf[AddFile]) numRewrittenFiles = addActions.size numAddedBytes = addActions.map(_.getFileSize).sum numRemovedBytes = removeActions.map(_.getFileSize).sum actionsForExistingFiles ++ actionsForNewFiles } val changeActions = totalActions.collect { case f: AddCDCFile => f } numAddedChangeFiles = changeActions.size changeFileBytes = changeActions.map(_.size).sum metrics("numAddedFiles").set(numRewrittenFiles) metrics("numAddedBytes").set(numAddedBytes) metrics("numAddedChangeFiles").set(numAddedChangeFiles) metrics("changeFileBytes").set(changeFileBytes) metrics("numRemovedFiles").set(numTouchedFiles) metrics("numRemovedBytes").set(numRemovedBytes) metrics("executionTimeMs").set(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime)) metrics("scanTimeMs").set(scanTimeMs) metrics("rewriteTimeMs").set(rewriteTimeMs) // In the case where the numUpdatedRows is not captured, we can siphon out the metrics from // the BasicWriteStatsTracker. This is for case 2 where the update condition contains only // metadata predicates and so the entire partition is re-written. val outputRows = txn.getMetric("numOutputRows").map(_.value).getOrElse(-1L) if (metrics("numUpdatedRows").value == 0 && outputRows != 0 && metrics("numCopiedRows").value == 0) { // We know that numTouchedRows = numCopiedRows + numUpdatedRows. // Since an entire partition was re-written, no rows were copied. // So numTouchedRows == numUpdateRows metrics("numUpdatedRows").set(metrics("numTouchedRows").value) } else { // This is for case 3 where the update condition contains both metadata and data predicates // so relevant files will have some rows updated and some rows copied. We don't need to // consider case 1 here, where no files match the update condition, as we know that // `totalActions` is empty. metrics("numCopiedRows").set( metrics("numTouchedRows").value - metrics("numUpdatedRows").value) metrics("numDeletionVectorsAdded").set(numDeletionVectorsAdded) metrics("numDeletionVectorsRemoved").set(numDeletionVectorsRemoved) metrics("numDeletionVectorsUpdated").set(numDeletionVectorsUpdated) } txn.registerSQLMetrics(sparkSession, metrics) val finalActions = createSetTransaction(sparkSession, deltaLog).toSeq ++ totalActions val numRecordsStats = NumRecordsStats.fromActions(finalActions) val operation = DeltaOperations.Update(condition) validateNumRecords(finalActions, numRecordsStats, operation) val commitVersion = txn.commitIfNeeded( actions = finalActions, op = operation, tags = RowTracking.addPreservedRowTrackingTagIfNotSet(txn.snapshot)) sendDriverMetrics(sparkSession, metrics) recordDeltaEvent( deltaLog, "delta.dml.update.stats", data = UpdateMetric( condition = condition.map(_.sql).getOrElse("true"), numFilesTotal, numTouchedFiles, numRewrittenFiles, numAddedChangeFiles, changeFileBytes, scanTimeMs, rewriteTimeMs, numDeletionVectorsAdded, numDeletionVectorsRemoved, numDeletionVectorsUpdated, commitVersion = commitVersion, numLogicalRecordsAdded = numRecordsStats.numLogicalRecordsAdded, numLogicalRecordsRemoved = numRecordsStats.numLogicalRecordsRemoved) ) } /** * Scan all the affected files and write out the updated files. * * When CDF is enabled, includes the generation of CDC preimage and postimage columns for * changed rows. * * @return a list of [[FileAction]]s, consisting of newly-written data and CDC files and old * files that have been removed. */ private def rewriteFiles( spark: SparkSession, txn: OptimisticTransaction, rootPath: Path, inputLeafFiles: Seq[AddFile], nameToAddFileMap: Map[String, AddFile], condition: Expression, generateRemoveFileActions: Boolean, copyUnmodifiedRows: Boolean): Seq[FileAction] = { // Number of total rows that we have seen, i.e. are either copying or updating (sum of both). // This will be used later, along with numUpdatedRows, to determine numCopiedRows. val incrTouchedCountExpr = IncrementMetric(TrueLiteral, metrics("numTouchedRows")) // Containing the map from the relative file path to AddFile val baseRelation = buildBaseRelation( spark, txn, "update", rootPath, inputLeafFiles.map(_.path), nameToAddFileMap) val newTarget = DeltaTableUtils.replaceFileIndex(target, baseRelation.location) val (targetDf, finalOutput, finalUpdateExpressions) = UpdateCommand.preserveRowTrackingColumns( targetDfWithoutRowTrackingColumns = DataFrameUtils.ofRows(spark, newTarget), snapshot = txn.snapshot, targetOutput = target.output, updateExpressions) val targetDfWithEvaluatedCondition = { val evalDf = targetDf.withColumn(UpdateCommand.CONDITION_COLUMN_NAME, Column(condition)) val copyAndUpdateRowsDf = if (copyUnmodifiedRows) { evalDf } else { evalDf.filter(Column(UpdateCommand.CONDITION_COLUMN_NAME)) } copyAndUpdateRowsDf.filter(Column(incrTouchedCountExpr)) } val updatedDataFrame = UpdateCommand.withUpdatedColumns( finalOutput, finalUpdateExpressions, condition, targetDfWithEvaluatedCondition, UpdateCommand.shouldOutputCdc(txn)) val addFiles = txn.writeFiles(updatedDataFrame) val removeFiles = if (generateRemoveFileActions) { val operationTimestamp = System.currentTimeMillis() inputLeafFiles.map(_.removeWithTimestamp(operationTimestamp)) } else { Nil } addFiles ++ removeFiles } def shouldWritePersistentDeletionVectors( spark: SparkSession, txn: OptimisticTransaction): Boolean = { spark.conf.get(DeltaSQLConf.UPDATE_USE_PERSISTENT_DELETION_VECTORS) && DeletionVectorUtils.deletionVectorsWritable(txn.snapshot) } /** * Validates that the number of records does not change. */ private def validateNumRecords( actions: Seq[Action], numRecordsStats: NumRecordsStats, op: Operation): Unit = { val deltaLog = tahoeFileIndex.deltaLog (numRecordsStats.numLogicalRecordsAdded, numRecordsStats.numLogicalRecordsRemoved, numRecordsStats.numLogicalRecordsAddedInFilesWithDeletionVectors) match { case ( Some(numAddedRecords), Some(numRemovedRecords), Some(numRecordsNotCopied)) => if (numAddedRecords != numRemovedRecords) { logNumRecordsMismatch(deltaLog, actions, numRecordsStats, op) if (conf.getConf(DeltaSQLConf.NUM_RECORDS_VALIDATION_ENABLED)) { throw DeltaErrors.numRecordsMismatch( operation = "UPDATE", numAddedRecords, numRemovedRecords ) } } if (conf.getConf(DeltaSQLConf.COMMAND_INVARIANT_CHECKS_USE_UNRELIABLE)) { // and also using regular (unreliable) metrics for baseline validateMetricBasedCommandInvariants( numAddedRecords, numRemovedRecords, numRecordsNotCopied, op, deltaLog) } case _ => recordDeltaEvent(deltaLog, opType = "delta.assertions.statsNotPresentForNumRecordsCheck") logWarning(log"Could not validate number of records due to missing statistics.") } } private def validateMetricBasedCommandInvariants( numAddedRecords: Long, numRemovedRecords: Long, numRecordsNotCopied: Long, op: Operation, deltaLog: DeltaLog): Unit = try { // Note: These are redundant w.r.t. validateNumRecords, but they ensure correct metrics. val numRowsUpdated = CommandInvariantMetricValueFromSingle(metrics("numUpdatedRows")) val numRowsCopied = CommandInvariantMetricValueFromSingle(metrics("numCopiedRows")) // There's a bug where Spark eliminates the entire plan and just rewrites the input files 1:1 // for no-op updates and in this case we don't record any metrics. if (numRowsUpdated.getOrDummy == 0 && numRowsCopied.getOrDummy == 0) { return } checkCommandInvariant( invariant = () => numRowsUpdated.getOrThrow + numRowsCopied.getOrThrow + numRecordsNotCopied == numRemovedRecords, label = "numRowsUpdated + numRowsCopied + numRecordsNotCopied == numRemovedRecords", op = op, deltaLog = deltaLog, parameters = Map( "numRowsUpdated" -> numRowsUpdated.getOrDummy, "numRowsCopied" -> numRowsCopied.getOrDummy, "numRemovedRecords" -> numRemovedRecords, "numRecordsNotCopied" -> numRecordsNotCopied ) ) checkCommandInvariant( invariant = () => numRowsUpdated.getOrThrow + numRowsCopied.getOrDummy + numRecordsNotCopied == numAddedRecords, label = "numRowsUpdated + numRowsCopied + numRecordsNotCopied == numAddedRecords", op = op, deltaLog = deltaLog, parameters = Map( "numRowsUpdated" -> numRowsUpdated.getOrDummy, "numRowsCopied" -> numRowsCopied.getOrDummy, "numAddedRecords" -> numAddedRecords, "numRecordsNotCopied" -> numRecordsNotCopied ) ) } catch { // Immediately re-throw actual command invariant violations, so we don't re-wrap them below. case e: DeltaIllegalStateException if e.getErrorClass == "DELTA_COMMAND_INVARIANT_VIOLATION" => throw e case NonFatal(e) => logWarning(log"Unexpected error in validateMetricBasedCommandInvariants", e) checkCommandInvariant( invariant = () => false, label = "Unexpected error in validateMetricBasedCommandInvariants", op = op, deltaLog = deltaLog, parameters = Map.empty ) } } object UpdateCommand { val FILE_NAME_COLUMN = "_input_file_name_" val CONDITION_COLUMN_NAME = "__condition__" val FINDING_TOUCHED_FILES_MSG: String = "Finding files to rewrite for UPDATE operation" def rewritingFilesMsg(numFilesToRewrite: Long): String = s"Rewriting $numFilesToRewrite files for UPDATE operation" /** * Whether or not CDC is enabled on this table and, thus, if we should output CDC data during this * UPDATE operation. */ def shouldOutputCdc(txn: OptimisticTransaction): Boolean = { DeltaConfigs.CHANGE_DATA_FEED.fromMetaData(txn.metadata) } /** * Build the new columns. If the condition matches, generate the new value using * the corresponding UPDATE EXPRESSION; otherwise, keep the original column value. * * When CDC is enabled, includes the generation of CDC pre-image and post-image columns for * changed rows. * * @param originalExpressions the original column values * @param updateExpressions the update transformation to perform on the input DataFrame * @param dfWithEvaluatedCondition source DataFrame on which we will apply the update expressions * with an additional column CONDITION_COLUMN_NAME which is the * true/false value of if the update condition is satisfied * @param condition update condition * @param shouldOutputCdc if we should output CDC data during this UPDATE operation. * @return the updated DataFrame, with extra CDC columns if CDC is enabled */ def withUpdatedColumns( originalExpressions: Seq[Attribute], updateExpressions: Seq[Expression], condition: Expression, dfWithEvaluatedCondition: DataFrame, shouldOutputCdc: Boolean): DataFrame = { val resultDf = if (shouldOutputCdc) { val namedUpdateCols = updateExpressions.zip(originalExpressions).map { case (expr, targetCol) => Column(expr).as(targetCol.name, targetCol.metadata) } // Build an array of output rows to be unpacked later. If the condition is matched, we // generate CDC pre and postimages in addition to the final output row; if the condition // isn't matched, we just generate a rewritten no-op row without any CDC events. val preimageCols = originalExpressions.map(Column(_)) :+ lit(CDC_TYPE_UPDATE_PREIMAGE).as(CDC_TYPE_COLUMN_NAME) val postimageCols = namedUpdateCols :+ lit(CDC_TYPE_UPDATE_POSTIMAGE).as(CDC_TYPE_COLUMN_NAME) val notCdcCol = Column(CDC_TYPE_NOT_CDC).as(CDC_TYPE_COLUMN_NAME) val updatedDataCols = namedUpdateCols :+ notCdcCol val noopRewriteCols = originalExpressions.map(Column(_)) :+ notCdcCol val packedUpdates = array( struct(preimageCols: _*), struct(postimageCols: _*), struct(updatedDataCols: _*) ).expr val packedData = if (condition == Literal.TrueLiteral) { packedUpdates } else { If( UnresolvedAttribute(CONDITION_COLUMN_NAME), packedUpdates, // if it should be updated, then use `packagedUpdates` array(struct(noopRewriteCols: _*)).expr) // else, this is a noop rewrite } // Explode the packed array, and project back out the final data columns. val finalColumns = (originalExpressions :+ UnresolvedAttribute(CDC_TYPE_COLUMN_NAME)).map { a => col(s"packedData.`${a.name}`").as(a.name, a.metadata) } dfWithEvaluatedCondition .select(explode(Column(packedData)).as("packedData")) .select(finalColumns: _*) } else { val finalCols = updateExpressions.zip(originalExpressions).map { case (update, original) => val updated = if (condition == Literal.TrueLiteral) { update } else { If(UnresolvedAttribute(CONDITION_COLUMN_NAME), update, original) } Column(updated).as(original.name, original.metadata) } dfWithEvaluatedCondition.select(finalCols: _*) } resultDf.drop(CONDITION_COLUMN_NAME) } /** * Preserve the row tracking columns when performing an UPDATE. * * @param targetDfWithoutRowTrackingColumns The target DataFrame on which the UPDATE * operation is to be performed. * @param snapshot Snapshot of the Delta table at the start of * the transaction. * @param targetOutput The output schema of the target DataFrame. * @param updateExpressions The update transformation to perform on the * target DataFrame. * @return * 1. targetDf: The target DataFrame that includes the preserved row tracking columns. * 2. finalOutput: The final output schema, including the preserved row tracking columns. * 3. finalUpdateExpressions: The final update expressions, including transformations * for the preserved row tracking columns. */ def preserveRowTrackingColumns( targetDfWithoutRowTrackingColumns: DataFrame, snapshot: Snapshot, targetOutput: Seq[Attribute] = Seq.empty, updateExpressions: Seq[Expression] = Seq.empty): (DataFrame, Seq[Attribute], Seq[Expression]) = { val targetDf = RowTracking.preserveRowTrackingColumns( targetDfWithoutRowTrackingColumns, snapshot) val rowIdAttributeOpt = MaterializedRowId.getAttribute(snapshot, targetDf) val rowCommitVersionAttributeOpt = MaterializedRowCommitVersion.getAttribute(snapshot, targetDf) val finalOutput = targetOutput ++ rowIdAttributeOpt ++ rowCommitVersionAttributeOpt val finalUpdateExpressions = updateExpressions ++ rowIdAttributeOpt ++ rowCommitVersionAttributeOpt.map(_ => Literal(null, LongType)) (targetDf, finalOutput, finalUpdateExpressions) } } /** * Used to report details about update. * * @param condition: what was the update condition * @param numFilesTotal: how big is the table * @param numTouchedFiles: how many files did we touch * @param numRewrittenFiles: how many files had to be rewritten * @param numAddedChangeFiles: how many change files were generated * @param changeFileBytes: total size of change files generated * @param scanTimeMs: how long did finding take * @param rewriteTimeMs: how long did rewriting take * * @note All the time units are milliseconds. */ case class UpdateMetric( condition: String, numFilesTotal: Long, numTouchedFiles: Long, numRewrittenFiles: Long, numAddedChangeFiles: Long, changeFileBytes: Long, scanTimeMs: Long, rewriteTimeMs: Long, numDeletionVectorsAdded: Long, numDeletionVectorsRemoved: Long, numDeletionVectorsUpdated: Long, @JsonDeserialize(contentAs = classOf[java.lang.Long]) commitVersion: Option[Long] = None, @JsonDeserialize(contentAs = classOf[java.lang.Long]) numLogicalRecordsAdded: Option[Long] = None, @JsonDeserialize(contentAs = classOf[java.lang.Long]) numLogicalRecordsRemoved: Option[Long] = None ) ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands // scalastyle:off import.ordering.noEmptyLine import java.io.File import java.io.FileNotFoundException import java.net.URI import java.sql.Timestamp import java.util.Date import java.util.concurrent.TimeUnit import scala.collection.JavaConverters._ import scala.math.min import scala.util.control.NonFatal import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.{AddCDCFile, AddFile, FileAction, RemoveFile, SingleAction} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.{DeltaCommitFileProvider, DeltaFileOperations, FileNames, JsonUtils, Utils => DeltaUtils} import org.apache.spark.sql.delta.util.DeltaFileOperations.tryDeleteNonRecursive import org.apache.spark.sql.delta.util.FileNames._ import com.fasterxml.jackson.databind.annotation.JsonDeserialize import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.spark.broadcast.Broadcast import org.apache.spark.internal.MDC import org.apache.spark.paths.SparkPath import org.apache.spark.sql.{Column, DataFrame, Dataset, Encoder, SparkSession} import org.apache.spark.sql.catalyst.catalog.CatalogTableType import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric import org.apache.spark.sql.functions.{col, count, lit, replace, startswith, substr, sum} import org.apache.spark.sql.types.{BooleanType, LongType, StringType, StructField, StructType} import org.apache.spark.util.{Clock, SerializableConfiguration, SystemClock, Utils} /** * Vacuums the table by clearing all untracked files and folders within this table. * First lists all the files and directories in the table, and gets the relative paths with * respect to the base of the table. Then it gets the list of all tracked files for this table, * which may or may not be within the table base path, and gets the relative paths of * all the tracked files with respect to the base of the table. Files outside of the table path * will be ignored. Then we take a diff of the files and delete directories that were already empty, * and all files that are within the table that are no longer tracked. */ object VacuumCommand extends VacuumCommandImpl with Serializable { case class FileNameAndSize(path: String, length: Long) /** * path : fully qualified uri * length: size in bytes * isDir: boolean indicating if it is a directory * modificationTime: file update time in milliseconds */ val INVENTORY_SCHEMA = StructType( Seq( StructField("path", StringType), StructField("length", LongType), StructField("isDir", BooleanType), StructField("modificationTime", LongType) )) object VacuumType extends Enumeration { type VacuumType = Value val LITE = Value("LITE") val FULL = Value("FULL") } def getFilesFromInventory( basePath: String, partitionColumns: Seq[String], inventory: DataFrame, shouldIcebergMetadataDirBeHidden: Boolean): Dataset[SerializableFileStatus] = { implicit val fileNameAndSizeEncoder: Encoder[SerializableFileStatus] = org.apache.spark.sql.Encoders.product[SerializableFileStatus] // filter out required fields from provided inventory DF val inventorySchema = StructType( inventory.schema.fields.filter(f => INVENTORY_SCHEMA.fields.map(_.name).contains(f.name)) ) if (inventorySchema != INVENTORY_SCHEMA) { throw DeltaErrors.invalidInventorySchema(INVENTORY_SCHEMA.treeString) } inventory .filter(startswith(col("path"), lit(s"$basePath/"))) .select( substr(col("path"), lit(basePath.length + 2)).as("path"), col("length"), col("isDir"), col("modificationTime") ) .flatMap { row => val path = row.getString(0) if(!DeltaTableUtils.isHiddenDirectory( partitionColumns, path, shouldIcebergMetadataDirBeHidden) ) { Seq(SerializableFileStatus(path, row.getLong(1), row.getBoolean(2), row.getLong(3))) } else { None } } .map { f => // Below logic will make paths url-encoded SerializableFileStatus(pathStringtoUrlEncodedString(f.path), f.length, f.isDir, f.modificationTime) } } /** * Clears all untracked files and folders within this table. If the inventory is not provided * then the command first lists all the files and directories in the table, if inventory is * provided then it will be used for identifying files and directories within the table and * gets the relative paths with respect to the base of the table. Then the command gets the * list of all tracked files for this table, which may or may not be within the table base path, * and gets the relative paths of all the tracked files with respect to the base of the table. * Files outside of the table path will be ignored. Then we take a diff of the files and delete * directories that were already empty, and all files that are within the table that are no longer * tracked. * * @param dryRun If set to true, no files will be deleted. Instead, we will list all files and * directories that will be cleared. * @param retentionHours An optional parameter to override the default Delta tombstone retention * period * @param inventory An optional dataframe of files and directories within the table generated * from sources like blob store inventory report * @return A Dataset containing the paths of the files/folders to delete in dryRun mode. Otherwise * returns the base path of the table. */ // scalastyle:off argcount def gc( spark: SparkSession, table: DeltaTableV2, dryRun: Boolean = true, retentionHours: Option[Double] = None, inventory: Option[DataFrame] = None, vacuumTypeOpt: Option[String] = None, commandMetrics: Map[String, SQLMetric] = Map.empty, clock: Clock = new SystemClock): DataFrame = { // scalastyle:on argcount val deltaLog = table.deltaLog recordDeltaOperation(deltaLog, "delta.gc") { val vacuumStartTime = System.currentTimeMillis() val path = deltaLog.dataPath val deltaHadoopConf = deltaLog.newDeltaHadoopConf() val fs = path.getFileSystem(deltaHadoopConf) import org.apache.spark.sql.delta.implicits._ val snapshot = table.update() deltaLog.protocolWrite(snapshot.protocol) if (snapshot.isCatalogOwned) { table.catalogTable.foreach { catalogTable => assert( catalogTable.tableType == CatalogTableType.MANAGED, s"All Catalog Owned tables should be MANAGED tables, " + s"but found ${catalogTable.tableType} for table ${catalogTable.identifier}." ) } throw DeltaErrors.operationBlockedOnCatalogManagedTable("VACUUM") } // By default, we will do full vacuum unless LITE vacuum conf is set val isLiteVacuumEnabled = spark.sessionState.conf.getConf(DeltaSQLConf.LITE_VACUUM_ENABLED) val defaultType = if (isLiteVacuumEnabled) VacuumType.LITE else VacuumType.FULL val vacuumType = vacuumTypeOpt.map(VacuumType.withName).getOrElse(defaultType) val snapshotTombstoneRetentionMillis = DeltaLog.tombstoneRetentionMillis(snapshot.metadata) val retentionMillis = retentionHours.flatMap { h => val retentionArgument = TimeUnit.HOURS.toMillis(math.round(h)) // We ignore retention window argument unless the specified value is 0 hours. if (spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_VACUUM_RETENTION_WINDOW_IGNORE_ENABLED) && retentionArgument != 0L) { logWarning(s"Vacuum with retention threshold other than 0 hours is ignored." + s" Please set ${DeltaConfigs.TOMBSTONE_RETENTION.key} table property to configure" + s" the retention period.") None } else { Some(retentionArgument) } } val deleteBeforeTimestamp = retentionMillis match { case Some(millis) => clock.getTimeMillis() - millis case _ => snapshot.minFileRetentionTimestamp } logInfo(log"Starting garbage collection (dryRun = " + log"${MDC(DeltaLogKeys.IS_DRY_RUN, dryRun)}) of untracked " + log"files older than ${MDC(DeltaLogKeys.DATE, new Date(deleteBeforeTimestamp).toGMTString)} in " + log"${MDC(DeltaLogKeys.PATH, path)}") val hadoopConf = spark.sparkContext.broadcast( new SerializableConfiguration(deltaHadoopConf)) val basePath = fs.makeQualified(path).toString val parallelDeleteEnabled = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_PARALLEL_DELETE_ENABLED) val parallelDeletePartitions = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_PARALLEL_DELETE_PARALLELISM) .getOrElse(spark.sessionState.conf.numShufflePartitions) val startTimeToIdentifyEligibleFiles = System.currentTimeMillis() val validFilesResult = getValidFilesFromSnapshot( spark, basePath, snapshot, retentionMillis, hadoopConf, clock, ValidFilesConfig( checkAbsolutePathOnly = false, performRetentionSafetyCheck = true, relativizeIgnoreError = None, dvDiscoveryDisabled = None ) ) val validFiles = validFilesResult.validFiles // Extract DataFrame val partitionColumns = snapshot.metadata.partitionSchema.fieldNames val parallelism = spark.sessionState.conf.parallelPartitionDiscoveryParallelism val shouldIcebergMetadataDirBeHidden = UniversalFormat.icebergEnabled(snapshot.metadata) val latestCommitVersionOutsideOfRetentionWindowOpt: Option[Long] = if (vacuumType == VacuumType.LITE) { try { val timestamp = new Timestamp(deleteBeforeTimestamp) val commit = new DeltaHistoryManager(deltaLog).getActiveCommitAtTime( timestamp, table.catalogTable, canReturnLastCommit = true, mustBeRecreatable = false) Some(commit.version) } catch { case ex: DeltaErrors.TimestampEarlierThanCommitRetentionException => None } } else { None } // eligibleStartCommitVersionOpt and eligibleEndCommitVersionOpt are valid in case of // lite vacuum. They represent the range of commit versions(inclusive) which give us the // eligible files to be deleted. val (allFilesAndDirsWithDuplicates, eligibleStartCommitVersionOpt, eligibleEndCommitVersionOpt) = inventory match { case Some(inventoryDF) => val files = getFilesFromInventory( basePath, partitionColumns, inventoryDF, shouldIcebergMetadataDirBeHidden) (files, None, None) case _ if vacuumType == VacuumType.LITE => getFilesFromDeltaLog(spark, snapshot, basePath, hadoopConf, latestCommitVersionOutsideOfRetentionWindowOpt) case _ => val files = getFilesFromFilesystem( spark, deltaLog, snapshot, hadoopConf, shouldIcebergMetadataDirBeHidden, applyHiddenFilters = true, parallelism = Option(parallelism) ) (files, None, None) } val allFilesAndDirs = allFilesAndDirsWithDuplicates.groupByKey(_.path) .mapGroups { (k, v) => val duplicates = v.toSeq // of all the duplicates we can return the newest file. duplicates.maxBy(_.modificationTime) } recordFrameProfile("Delta", "VacuumCommand.gc") { try { allFilesAndDirs.cache() implicit val fileNameAndSizeEncoder = org.apache.spark.sql.Encoders.product[FileNameAndSize] val dirCounts = allFilesAndDirs.where(col("isDir")).count() + 1 // +1 for the base path val filesAndDirsPresentBeforeDelete = allFilesAndDirs.count() // The logic below is as follows: // 1. We take all the files and directories listed in our reservoir // 2. We filter all files older than our tombstone retention period and directories // 3. We get the subdirectories of all files so that we can find non-empty directories // 4. We groupBy each path, and count to get how many files are in each sub-directory // 5. We subtract all the valid files and tombstones in our state // 6. We filter all paths with a count of 1, which will correspond to files not in the // state, and empty directories. We can safely delete all of these val diff = includeRespectiveDirectoriesWithFilesAndSafetyCheck( allFilesAndDirs, basePath, Some(deleteBeforeTimestamp), hadoopConf) .groupBy(col("path")).agg(count(new Column("*")).as("count"), sum("length").as("length")) .join(validFiles, Seq("path"), "leftanti") .where(col("count") === 1) val sizeOfDataToDeleteRow = diff.agg(sum("length").cast("long")).first() val sizeOfDataToDelete = if (sizeOfDataToDeleteRow.isNullAt(0)) { 0L } else { sizeOfDataToDeleteRow.getLong(0) } val diffFiles = diff .select(col("path")) .as[String] .map { relativePath => assert(!urlEncodedStringToPath(relativePath).isAbsolute, "Shouldn't have any absolute paths for deletion here.") pathToUrlEncodedString(DeltaFileOperations.absolutePath(basePath, relativePath)) } val timeTakenToIdentifyEligibleFiles = System.currentTimeMillis() - startTimeToIdentifyEligibleFiles val numFiles = diffFiles.count() if (dryRun) { val stats = DeltaVacuumStats( isDryRun = true, specifiedRetentionMillis = retentionMillis, defaultRetentionMillis = snapshotTombstoneRetentionMillis, minRetainedTimestamp = deleteBeforeTimestamp, dirsPresentBeforeDelete = dirCounts, filesAndDirsPresentBeforeDelete = filesAndDirsPresentBeforeDelete, objectsDeleted = numFiles, sizeOfDataToDelete = sizeOfDataToDelete, timeTakenToIdentifyEligibleFiles = timeTakenToIdentifyEligibleFiles, timeTakenForDelete = 0L, vacuumStartTime = vacuumStartTime, vacuumEndTime = System.currentTimeMillis, numPartitionColumns = partitionColumns.size, latestCommitVersion = snapshot.version, eligibleStartCommitVersion = eligibleStartCommitVersionOpt, eligibleEndCommitVersion = eligibleEndCommitVersionOpt, typeOfVacuum = vacuumType.toString ) recordDeltaEvent(deltaLog, "delta.gc.stats", data = stats) logInfo(log"Found ${MDC(DeltaLogKeys.NUM_FILES, numFiles.toLong)} files " + log"(${MDC(DeltaLogKeys.NUM_BYTES, sizeOfDataToDelete)} bytes) and directories in " + log"a total of ${MDC(DeltaLogKeys.NUM_DIRS, dirCounts)} directories " + log"that are safe to delete. Vacuum stats: ${MDC(DeltaLogKeys.VACUUM_STATS, stats)}") return diffFiles.map(f => urlEncodedStringToPath(f).toString).toDF("path") } logVacuumStart( spark, table, diffFiles, sizeOfDataToDelete, retentionMillis, snapshotTombstoneRetentionMillis) val deleteStartTime = System.currentTimeMillis() val filesDeleted = try { delete(diffFiles, spark, basePath, hadoopConf, parallelDeleteEnabled, parallelDeletePartitions) } catch { case t: Throwable => logVacuumEnd(spark, table, commandMetrics = commandMetrics) throw t } val timeTakenForDelete = System.currentTimeMillis() - deleteStartTime val stats = DeltaVacuumStats( isDryRun = false, specifiedRetentionMillis = retentionMillis, defaultRetentionMillis = snapshotTombstoneRetentionMillis, minRetainedTimestamp = deleteBeforeTimestamp, dirsPresentBeforeDelete = dirCounts, filesAndDirsPresentBeforeDelete = filesAndDirsPresentBeforeDelete, objectsDeleted = filesDeleted, sizeOfDataToDelete = sizeOfDataToDelete, timeTakenToIdentifyEligibleFiles = timeTakenToIdentifyEligibleFiles, timeTakenForDelete = timeTakenForDelete, vacuumStartTime = vacuumStartTime, vacuumEndTime = System.currentTimeMillis, numPartitionColumns = partitionColumns.size, latestCommitVersion = snapshot.version, eligibleStartCommitVersion = eligibleStartCommitVersionOpt, eligibleEndCommitVersion = eligibleEndCommitVersionOpt, typeOfVacuum = vacuumType.toString) recordDeltaEvent(deltaLog, "delta.gc.stats", data = stats) logVacuumEnd( spark, table, commandMetrics = commandMetrics, Some(filesDeleted), Some(dirCounts)) LastVacuumInfo.persistLastVacuumInfo( LastVacuumInfo(latestCommitVersionOutsideOfRetentionWindowOpt), deltaLog) logInfo(log"Deleted ${MDC(DeltaLogKeys.NUM_FILES, filesDeleted)} files " + log"(${MDC(DeltaLogKeys.NUM_BYTES, sizeOfDataToDelete)} bytes) and directories in " + log"a total of ${MDC(DeltaLogKeys.NUM_DIRS, dirCounts)} directories. " + log"Vacuum stats: ${MDC(DeltaLogKeys.VACUUM_STATS, stats)}") spark.createDataset(Seq(basePath)).toDF("path") } finally { allFilesAndDirs.unpersist() } } } } /** * Returns eligible files to be deleted by looking at the delta log. Additionally, it returns * the start and the end commit versions(inclusive) which give us the eligible files to be * deleted. */ protected def getFilesFromDeltaLog( spark: SparkSession, snapshot: Snapshot, basePath: String, hadoopConf: Broadcast[SerializableConfiguration], latestCommitVersionOutsideOfRetentionWindowOpt: Option[Long]) : (Dataset[SerializableFileStatus], Option[Long], Option[Long]) = { import org.apache.spark.sql.delta.implicits._ val deltaLog = snapshot.deltaLog val earliestCommitVersion = DeltaHistoryManager.getEarliestDeltaFile(deltaLog) val latestCommitVersionOutsideOfRetentionWindowAsOfLastVacuumOpt = LastVacuumInfo.getLastVacuumInfo(deltaLog) .flatMap(_.latestCommitVersionOutsideOfRetentionWindow) // If there are no commit versions outside of the retention window, // then there is nothing to Vacuum. val latestCommitVersionOutsideOfRetentionWindow = latestCommitVersionOutsideOfRetentionWindowOpt.getOrElse { return (spark.emptyDataset[SerializableFileStatus], None, None) } // In the following two conditions, we return error saying lite vacuum is not possible: // 1. We are not able to locate the last vacuum info and we don't have commit files starting // from 0 // 2. Last vacuum info is there but metadata cleanup has cleaned up commit files since // the last Vacuum's latest commit version outside of the retention window. if (earliestCommitVersion != 0 && latestCommitVersionOutsideOfRetentionWindowAsOfLastVacuumOpt .forall(_ < earliestCommitVersion)) { throw DeltaErrors.deltaCannotVacuumLite() } // The start and the end commit versions give the range of commit files we want to look into // to get the list of eligible files for deletion. val eligibleStartCommitVersion = math.min( snapshot.version, latestCommitVersionOutsideOfRetentionWindowAsOfLastVacuumOpt .map(_ + 1).getOrElse(earliestCommitVersion)) val eligibleEndCommitVersion = latestCommitVersionOutsideOfRetentionWindow // If there are no additional commit files to look into, then // there is nothing to vacuum. if (eligibleStartCommitVersion > latestCommitVersionOutsideOfRetentionWindow) { return (spark.emptyDataset[SerializableFileStatus], None, None) } (getFilesFromDeltaLog(spark, deltaLog, basePath, hadoopConf, eligibleStartCommitVersion, eligibleEndCommitVersion, relativizeIgnoreError = None), Some(eligibleStartCommitVersion), Some(eligibleEndCommitVersion) ) } case class ValidFilesResult( validFiles: DataFrame ) } trait VacuumCommandImpl extends DeltaCommand { private val supportedFsForLogging = Seq( "wasbs", "wasbss", "abfs", "abfss", "adl", "gs", "file", "hdfs" ) /** * Returns whether we should record vacuum metrics in the delta log. */ private def shouldLogVacuum( spark: SparkSession, table: DeltaTableV2, hadoopConf: Configuration, path: Path): Boolean = { val logVacuumConf = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_LOGGING_ENABLED) if (logVacuumConf.nonEmpty) { return logVacuumConf.get } val deltaLog = table.deltaLog val logStore = deltaLog.store try { val rawResolvedUri: URI = logStore.resolvePathOnPhysicalStorage(path, hadoopConf).toUri val scheme = rawResolvedUri.getScheme supportedFsForLogging.contains(scheme) } catch { case _: UnsupportedOperationException => logWarning(log"Vacuum event logging" + " not enabled on this file system because we cannot detect your cloud storage type.") false } } /** * Record Vacuum specific metrics in the commit log at the START of vacuum. * * @param spark - spark session * @param table - the delta table * @param diff - the list of paths (files, directories) that are safe to delete * @param sizeOfDataToDelete - the amount of data (bytes) to be deleted * @param specifiedRetentionMillis - the optional override retention period (millis) to keep * logically removed files before deleting them * @param defaultRetentionMillis - the default retention period (millis) */ protected def logVacuumStart( spark: SparkSession, table: DeltaTableV2, diff: Dataset[String], sizeOfDataToDelete: Long, specifiedRetentionMillis: Option[Long], defaultRetentionMillis: Long): Unit = { val deltaLog = table.deltaLog logInfo( log"Deleting untracked files and empty directories in " + log"${MDC(DeltaLogKeys.PATH, deltaLog.dataPath)}. The amount " + log"of data to be deleted is ${MDC(DeltaLogKeys.NUM_BYTES, sizeOfDataToDelete)} (in bytes)" ) // We perform an empty commit in order to record information about the Vacuum if (shouldLogVacuum(spark, table, deltaLog.newDeltaHadoopConf(), deltaLog.dataPath)) { val checkEnabled = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED) val txn = table.startTransaction() val metrics = Map[String, SQLMetric]( "numFilesToDelete" -> createMetric(spark.sparkContext, "number of files to deleted"), "sizeOfDataToDelete" -> createMetric(spark.sparkContext, "The total amount of data to be deleted in bytes") ) metrics("numFilesToDelete").set(diff.count()) metrics("sizeOfDataToDelete").set(sizeOfDataToDelete) txn.registerSQLMetrics(spark, metrics) val version = txn.commit(actions = Seq(), DeltaOperations.VacuumStart( checkEnabled, specifiedRetentionMillis, defaultRetentionMillis )) setCommitClock(deltaLog, version) } } /** * Record Vacuum specific metrics in the commit log at the END of vacuum. * * @param spark - spark session * @param table - the delta table * @param filesDeleted - if the vacuum completed this will contain the number of files deleted. * if the vacuum failed, this will be None. * @param dirCounts - if the vacuum completed this will contain the number of directories * vacuumed. if the vacuum failed, this will be None. */ protected def logVacuumEnd( spark: SparkSession, table: DeltaTableV2, commandMetrics: Map[String, SQLMetric], filesDeleted: Option[Long] = None, dirCounts: Option[Long] = None): Unit = { val deltaLog = table.deltaLog if (shouldLogVacuum(spark, table, deltaLog.newDeltaHadoopConf(), deltaLog.dataPath)) { val txn = table.startTransaction() val status = if (filesDeleted.isEmpty && dirCounts.isEmpty) { "FAILED" } else { "COMPLETED" } if (filesDeleted.nonEmpty && dirCounts.nonEmpty) { // Populate top level metrics. commandMetrics.get("numDeletedFiles").foreach(_.set(filesDeleted.get)) commandMetrics.get("numVacuumedDirectories").foreach(_.set(dirCounts.get)) // Additionally, create a separate metrics map in case the commandMetrics is empty. val metrics = Map[String, SQLMetric]( "numDeletedFiles" -> createMetric(spark.sparkContext, "number of files deleted."), "numVacuumedDirectories" -> createMetric(spark.sparkContext, "num of directories vacuumed."), "status" -> createMetric(spark.sparkContext, "status of vacuum") ) metrics("numDeletedFiles").set(filesDeleted.get) metrics("numVacuumedDirectories").set(dirCounts.get) txn.registerSQLMetrics(spark, metrics) } val version = txn.commit(actions = Seq(), DeltaOperations.VacuumEnd( status )) setCommitClock(deltaLog, version) } if (filesDeleted.nonEmpty) { logConsole(s"Deleted ${filesDeleted.get} files and directories in a total " + s"of ${dirCounts.get} directories.") } } protected def setCommitClock(deltaLog: DeltaLog, version: Long) = { // This is done to make sure that the commit timestamp reflects the one provided by the clock // object. if (DeltaUtils.isTesting) { val fs = deltaLog.logPath.getFileSystem(deltaLog.newDeltaHadoopConf()) val filePath = DeltaCommitFileProvider(deltaLog.update()).deltaFile(version) if (fs.exists(filePath)) { fs.setTimes(filePath, deltaLog.clock.getTimeMillis(), deltaLog.clock.getTimeMillis()) } } } /** * Attempts to relativize the `path` with respect to the `reservoirBase` and converts the path to * a string. */ protected def relativize( path: Path, fs: FileSystem, reservoirBase: Path, isDir: Boolean): String = { pathToUrlEncodedString(DeltaFileOperations.tryRelativizePath(fs, reservoirBase, path)) } /** * Wrapper function for DeltaFileOperations.getAllSubDirectories * returns all subdirectories that `file` has with respect to `base`. */ protected def getAllSubdirs(base: String, file: String, fs: FileSystem): Iterator[String] = { DeltaFileOperations.getAllSubDirectories(base, file)._1 } /** * Attempts to delete the list of candidate files. Returns the number of files deleted. */ protected def delete( diff: Dataset[String], spark: SparkSession, basePath: String, hadoopConf: Broadcast[SerializableConfiguration], parallel: Boolean, parallelPartitions: Int): Long = { import org.apache.spark.sql.delta.implicits._ if (parallel) { diff.repartition(parallelPartitions).mapPartitions { files => val fs = new Path(basePath).getFileSystem(hadoopConf.value.value) val filesDeletedPerPartition = files.map(p => urlEncodedStringToPath(p)).count(f => tryDeleteNonRecursive(fs, f)) Iterator(filesDeletedPerPartition) }.collect().sum } else { val fs = new Path(basePath).getFileSystem(hadoopConf.value.value) val fileResultSet = diff.toLocalIterator().asScala fileResultSet.map(p => urlEncodedStringToPath(p)).count(f => tryDeleteNonRecursive(fs, f)) } } protected def urlEncodedStringToPath(path: String): Path = SparkPath.fromUrlString(path).toPath protected def pathToUrlEncodedString(path: Path): String = SparkPath.fromPath(path).toString protected def pathStringtoUrlEncodedString(path: String) = SparkPath.fromPathString(path).toString protected def getActionRelativePath( action: FileAction, fs: FileSystem, basePath: Path, relativizeIgnoreError: Boolean): Option[String] = { getRelativePath(action.path, fs, basePath, relativizeIgnoreError) } /** Returns the relative path of a file or None if the file lives outside of the table. */ protected def getRelativePath( path: String, fs: FileSystem, basePath: Path, relativizeIgnoreError: Boolean): Option[String] = { val filePath = urlEncodedStringToPath(path) if (filePath.isAbsolute) { val maybeRelative = DeltaFileOperations.tryRelativizePath(fs, basePath, filePath, relativizeIgnoreError) if (maybeRelative.isAbsolute) { // This file lives outside the directory of the table. None } else { Some(pathToUrlEncodedString(maybeRelative)) } } else { Some(pathToUrlEncodedString(filePath)) } } /** * Returns the relative paths of all files and subdirectories for this action that must be * retained during GC. */ protected def getValidRelativePathsAndSubdirs( action: FileAction, fs: FileSystem, basePath: Path, relativizeIgnoreError: Boolean, dvDiscoveryDisabled: Boolean ): Seq[String] = { val paths = getActionRelativePath(action, fs, basePath, relativizeIgnoreError) .map { relativePath => Seq(relativePath) ++ getAllSubdirs("/", relativePath, fs) }.getOrElse(Seq.empty) val deletionVectorPath = if (dvDiscoveryDisabled) None else getDeletionVectorRelativePathAndSize(action).map(_._1) paths ++ deletionVectorPath.toSeq } /** * Returns the path of the on-disk deletion vector if it is stored relative to the * `basePath` and it's size otherwise `None`. */ protected def getDeletionVectorRelativePathAndSize(action: FileAction): Option[(String, Long)] = { val dv = action match { case a: AddFile if a.deletionVector != null => Some(a.deletionVector) case r: RemoveFile if r.deletionVector != null => Some(r.deletionVector) case _ => None } dv match { case Some(dv) if dv.isOnDisk => if (dv.isRelative) { // We actually want a relative path here. Some((pathToUrlEncodedString(dv.absolutePath(new Path("."))), dv.sizeInBytes)) } else { assert(dv.isAbsolute) // This is never going to be a path relative to `basePath` for DVs. None } case None => None } } // Utility methods for use by VacuumCommand and other commands /** * Additional check on retention duration to prevent people from shooting themselves in the foot. */ protected def checkRetentionPeriodSafety( spark: SparkSession, retentionMs: Option[Long], configuredRetention: Long): Unit = { if (retentionMs.exists(_ < 0)) { throw DeltaErrors.vacuumRetentionPeriodNegative() } val checkEnabled = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED) val retentionSafe = retentionMs.forall(_ >= configuredRetention) var configuredRetentionHours = TimeUnit.MILLISECONDS.toHours(configuredRetention) if (TimeUnit.HOURS.toMillis(configuredRetentionHours) < configuredRetention) { configuredRetentionHours += 1 } if (checkEnabled && !retentionSafe) { throw DeltaErrors.vacuumRetentionPeriodTooShort(configuredRetentionHours) } } /** * Configuration for getValidFilesFromSnapshot behavior. * * @param checkAbsolutePathOnly If true, filters out files not in table path (for clones) * @param performRetentionSafetyCheck If true, validates retention period is safe * @param relativizeIgnoreError If None, reads from config; if Some(value), uses that value * @param dvDiscoveryDisabled If None, reads from config+test; if Some(value), uses that value */ case class ValidFilesConfig( checkAbsolutePathOnly: Boolean, performRetentionSafetyCheck: Boolean, relativizeIgnoreError: Option[Boolean], dvDiscoveryDisabled: Option[Boolean] ) /** * Returns eligible files (RemoveFile + CDC) from a range of commit versions. * * @param relativizeIgnoreError If None, reads from config; if Some(value), uses that value */ protected def getFilesFromDeltaLog( spark: SparkSession, deltaLog: DeltaLog, basePath: String, hadoopConf: Broadcast[SerializableConfiguration], eligibleStartCommitVersion: Long, eligibleEndCommitVersion: Long, relativizeIgnoreError: Option[Boolean]): Dataset[SerializableFileStatus] = { import org.apache.spark.sql.delta.implicits._ // When coordinated commits are enabled, commit files could be found in _delta_log directory // as well as in commit directory. We get the delta log files outside of the retention window // from both the places. val prefix = listingPrefix(deltaLog.logPath, eligibleStartCommitVersion) val eligibleDeltaLogFilesFromDeltaLogDirectory = deltaLog.store.listFrom(prefix, deltaLog.newDeltaHadoopConf) .collect { case DeltaFile(f, deltaFileVersion) => (f, deltaFileVersion) } .takeWhile(_._2 <= eligibleEndCommitVersion) .toSeq val fs = deltaLog.logPath.getFileSystem(deltaLog.newDeltaHadoopConf()) val commitDirPath = FileNames.commitDirPath(deltaLog.logPath) val updatedStartCommitVersion = eligibleDeltaLogFilesFromDeltaLogDirectory.lastOption.map(_._2) .getOrElse(eligibleStartCommitVersion) val eligibleDeltaLogFilesFromCommitDirectory = if (fs.exists(commitDirPath)) { deltaLog.store .listFrom(listingPrefix(commitDirPath, updatedStartCommitVersion), deltaLog.newDeltaHadoopConf) .collect { case UnbackfilledDeltaFile(f, deltaFileVersion, _) => (f, deltaFileVersion) } .takeWhile(_._2 <= eligibleEndCommitVersion) .toSeq } else { Seq.empty } val allDeltaLogFilesOutsideTheRetentionWindow = eligibleDeltaLogFilesFromDeltaLogDirectory ++ eligibleDeltaLogFilesFromCommitDirectory val deltaLogFileIndex = DeltaLogFileIndex(DeltaLogFileIndex.COMMIT_FILE_FORMAT, allDeltaLogFilesOutsideTheRetentionWindow.map(_._1)).get val allActions = deltaLog.loadIndex(deltaLogFileIndex).as[SingleAction] val nonCDFFiles = allActions .where("remove IS NOT NULL") .select(col("remove") .as[RemoveFile]) .mapPartitions { iter => iter.flatMap { r => val modificationTime = r.deletionTimestamp.getOrElse(0L) val dv = getDeletionVectorRelativePathAndSize(r).map { case (path, length) => SerializableFileStatus(path, length, isDir = false, modificationTime) } dv.iterator ++ Iterator.single(SerializableFileStatus( r.path, r.size.getOrElse(0L), isDir = false, modificationTime)) } } .as[SerializableFileStatus] val cdfFiles = allActions .where("cdc IS NOT NULL") .select(col("cdc") .as[AddCDCFile]) .map(cdc => SerializableFileStatus(cdc.path, cdc.size, isDir = false, modificationTime = 0L)) val relativizeIgnoreErrorValue = relativizeIgnoreError.getOrElse( spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_RELATIVIZE_IGNORE_ERROR) ) nonCDFFiles.union(cdfFiles).mapPartitions { iter => val tableBasePath = new Path(basePath) val fs = tableBasePath.getFileSystem(hadoopConf.value.value) iter.flatMap { f => // if file path is outside of the table base path, those files are not considered as they // are not part of this table. Shallow clone is one example where this happens. getRelativePath(f.path, fs, tableBasePath, relativizeIgnoreErrorValue) .map(SerializableFileStatus(_, f.length, f.isDir, f.modificationTime)) } } } /** * Returns files from filesystem via recursive directory listing. * * @param spark SparkSession * @param deltaLog DeltaLog for the table * @param snapshot Current snapshot * @param hadoopConf Hadoop configuration * @param shouldIcebergMetadataDirBeHidden Whether to hide Iceberg metadata directories * @param applyHiddenFilters If true, excludes hidden directories (_delta_log, etc.). * @param parallelism Optional parallelism for file listing * @return Dataset of SerializableFileStatus with url-encoded relative paths */ protected def getFilesFromFilesystem( spark: SparkSession, deltaLog: DeltaLog, snapshot: Snapshot, hadoopConf: Broadcast[SerializableConfiguration], shouldIcebergMetadataDirBeHidden: Boolean, applyHiddenFilters: Boolean, parallelism: Option[Int]): Dataset[SerializableFileStatus] = { import org.apache.spark.sql.delta.implicits._ val basePath = deltaLog.dataPath.toString val partitionColumns = snapshot.metadata.partitionColumns val (hiddenDirFilter, hiddenFileFilter) = if (applyHiddenFilters) { // Apply filters to exclude _delta_log and other hidden directories val filter = DeltaTableUtils.isHiddenDirectory( partitionColumns, _: String, shouldIcebergMetadataDirBeHidden) (filter, filter) } else { // Include all directories and files ((_: String) => false, (_: String) => false) } // Use DeltaFileOperations.recursiveListDirs val files = DeltaFileOperations.recursiveListDirs( spark, Seq(basePath), hadoopConf, hiddenDirNameFilter = hiddenDirFilter, hiddenFileNameFilter = hiddenFileFilter, fileListingParallelism = parallelism ) .map { f => // Make paths url-encoded (same pattern as VacuumCommand) val path = pathStringtoUrlEncodedString(f.path) SerializableFileStatus(path, f.length, f.isDir, f.modificationTime) } files } /** * Helper to compute all valid files based on basePath and Snapshot provided. * Returns a DataFrame with a single column "path" containing all files that should be * protected from vacuum (active files, tombstones in retention, DVs, subdirs, etc.) * * @param config Configuration for behavior customization */ protected def getValidFilesFromSnapshot( spark: SparkSession, basePath: String, snapshot: Snapshot, retentionMillis: Option[Long], hadoopConf: Broadcast[SerializableConfiguration], clock: Clock, config: ValidFilesConfig): VacuumCommand.ValidFilesResult = { import org.apache.spark.sql.delta.implicits._ require(snapshot.version >= 0, "No state defined for this table. Is this really " + "a Delta table? Refusing to garbage collect.") val snapshotTombstoneRetentionMillis = DeltaLog.tombstoneRetentionMillis(snapshot.metadata) // Safety check only for vacuum (not for read-only metrics) if (config.performRetentionSafetyCheck) { checkRetentionPeriodSafety(spark, retentionMillis, snapshotTombstoneRetentionMillis) } val deleteBeforeTimestamp = retentionMillis match { case Some(millis) => clock.getTimeMillis() - millis case _ => snapshot.minFileRetentionTimestamp } // Use provided values or read from config val relativizeIgnoreErrorValue = config.relativizeIgnoreError.getOrElse( spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_RELATIVIZE_IGNORE_ERROR) ) val dvDiscoveryDisabledValue = config.dvDiscoveryDisabled.getOrElse( DeltaUtils.isTesting && spark.sessionState.conf.getConf( DeltaSQLConf.FAST_DROP_FEATURE_DV_DISCOVERY_IN_VACUUM_DISABLED) ) val canonicalizedBasePath = SparkPath.fromPathString(basePath).urlEncoded val files = snapshot.stateDS.mapPartitions { actions => val reservoirBase = new Path(basePath) val fs = reservoirBase.getFileSystem(hadoopConf.value.value) actions.flatMap { _.unwrap match { // Existing tables may not store canonicalized paths, so we check both the canonicalized // and non-canonicalized paths to ensure we don't accidentally delete wrong files. case fa: FileAction if config.checkAbsolutePathOnly && !fa.path.contains(basePath) && !fa.path.contains(canonicalizedBasePath) => Nil case tombstone: RemoveFile if tombstone.delTimestamp < deleteBeforeTimestamp => Nil case fa: FileAction => getValidRelativePathsAndSubdirs( fa, fs, reservoirBase, relativizeIgnoreErrorValue, dvDiscoveryDisabledValue ) case _ => Nil } } } val validFiles = files .toDF("path") VacuumCommand.ValidFilesResult( validFiles ) } /** * Expands files into their parent directories. * Used by both VacuumCommand (for deletion). * * For each file, this creates entries for all parent directories. * The caller must then aggregate and perform safety checks: * 1. GroupBy path and aggregate (count, sum length, max modificationTime) * 2. Join with validFiles (leftanti) to remove protected files * 3. Filter where count === 1 (safety check - only unique paths are safe to delete) * * @param files Dataset of files to process * @param basePath Base path of the table * @param deleteBeforeTimestamp If Some(timestamp), filters files by modificationTime * < timestamp (for VacuumCommand). If None, includes all files * regardless of time. * @param hadoopConf Hadoop configuration * * @return DataFrame with schema (path: String, length: Long, * isDir: Boolean, modificationTime: Long) * The caller should groupBy, aggregate, join with validFiles, then filter count === 1 */ protected def includeRespectiveDirectoriesWithFilesAndSafetyCheck( files: Dataset[SerializableFileStatus], basePath: String, deleteBeforeTimestamp: Option[Long], hadoopConf: Broadcast[SerializableConfiguration] ): DataFrame = { import org.apache.spark.sql.functions.col implicit val serializableFileStatusEncoder = org.apache.spark.sql.Encoders.product[SerializableFileStatus] val canonicalizedBasePath = SparkPath.fromPathString(basePath).urlEncoded val filteredFiles = deleteBeforeTimestamp match { case Some(timestamp) => files.where(col("modificationTime") < timestamp || col("isDir")) case None => files } filteredFiles .mapPartitions { fileStatusIterator => val reservoirBase = new Path(basePath) val fs = reservoirBase.getFileSystem(hadoopConf.value.value) fileStatusIterator.flatMap { fileStatus => if (fileStatus.isDir) { Iterator.single(SerializableFileStatus( relativize(urlEncodedStringToPath(fileStatus.path), fs, reservoirBase, isDir = true), 0L, isDir = true, fileStatus.modificationTime)) } else { val dirs = getAllSubdirs(canonicalizedBasePath, fileStatus.path, fs) val dirsWithSlash = dirs.map { p => val relativizedPath = relativize(urlEncodedStringToPath(p), fs, reservoirBase, isDir = true) SerializableFileStatus(relativizedPath, 0L, isDir = true, fileStatus.modificationTime) } dirsWithSlash ++ Iterator( SerializableFileStatus(relativize( urlEncodedStringToPath(fileStatus.path), fs, reservoirBase, isDir = false), fileStatus.length, isDir = false, fileStatus.modificationTime)) } } }.toDF() } } case class DeltaVacuumStats( isDryRun: Boolean, @JsonDeserialize(contentAs = classOf[java.lang.Long]) specifiedRetentionMillis: Option[Long], defaultRetentionMillis: Long, minRetainedTimestamp: Long, dirsPresentBeforeDelete: Long, filesAndDirsPresentBeforeDelete: Long, objectsDeleted: Long, sizeOfDataToDelete: Long, timeTakenToIdentifyEligibleFiles: Long, timeTakenForDelete: Long, vacuumStartTime: Long, vacuumEndTime: Long, numPartitionColumns: Long, latestCommitVersion: Long, @JsonDeserialize(contentAs = classOf[java.lang.Long]) eligibleStartCommitVersion: Option[Long], @JsonDeserialize(contentAs = classOf[java.lang.Long]) eligibleEndCommitVersion: Option[Long], typeOfVacuum: String ) case class LastVacuumInfo( @JsonDeserialize(contentAs = classOf[java.lang.Long]) latestCommitVersionOutsideOfRetentionWindow: Option[Long] = None ) object LastVacuumInfo extends DeltaCommand { private val LAST_VACUUM_INFO_FILE_NAME = "_last_vacuum_info" /** The path to the file that holds metadata about the most recent Vacuum. */ private def getLastVacuumInfoPath(logPath: Path): Path = new Path(logPath, LAST_VACUUM_INFO_FILE_NAME) def getLastVacuumInfo(deltaLog: DeltaLog): Option[LastVacuumInfo] = { try { val path = getLastVacuumInfoPath(deltaLog.logPath) val json = deltaLog.store.read(path, deltaLog.newDeltaHadoopConf()).head Some(JsonUtils.mapper.readValue[LastVacuumInfo](json)) } catch { case _: FileNotFoundException => None case NonFatal(e) => recordDeltaEvent( deltaLog, "delta.lastVacuumInfo.read.corruptedJson", data = Map("exception" -> Utils.exceptionString(e)) ) None } } def persistLastVacuumInfo(lastVacuumInfo: LastVacuumInfo, deltaLog: DeltaLog): Unit = { try { val path = getLastVacuumInfoPath(deltaLog.logPath) val json = Iterator.single(JsonUtils.toJson(lastVacuumInfo)) deltaLog.store.write(path, json, overwrite = true, deltaLog.newDeltaHadoopConf()) } catch { case NonFatal(e) => recordDeltaEvent( deltaLog, "delta.lastVacuumInfo.write.failure", data = Map("exception" -> Utils.exceptionString(e)) ) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/WriteIntoDelta.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands import scala.collection.mutable import scala.util.Try // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.commands.DMLUtils.TaggedCommitData import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.constraints.Constraint import org.apache.spark.sql.delta.constraints.Constraints.Check import org.apache.spark.sql.delta.constraints.Invariants.ArbitraryExpression import org.apache.spark.sql.delta.schema.{ImplicitMetadataOperation, InvariantViolationException, SchemaUtils} import org.apache.spark.sql.delta.skipping.clustering.ClusteredTableUtils import org.apache.spark.sql.delta.skipping.clustering.temp.ClusterBySpec import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.{And, Attribute, Expression, Literal} import org.apache.spark.sql.catalyst.plans.logical.DeleteFromTable import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, CharVarcharUtils} import org.apache.spark.sql.execution.command.LeafRunnableCommand import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.functions.{array, col, explode, lit, struct} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{StringType, StructType} /** * Used to write a [[DataFrame]] into a delta table. * * New Table Semantics * - The schema of the [[DataFrame]] is used to initialize the table. * - The partition columns will be used to partition the table. * * Existing Table Semantics * - The save mode will control how existing data is handled (i.e. overwrite, append, etc) * - The schema of the DataFrame will be checked and if there are new columns present * they will be added to the tables schema. Conflicting columns (i.e. a INT, and a STRING) * will result in an exception * - The partition columns, if present are validated against the existing metadata. If not * present, then the partitioning of the table is respected. * * In combination with `Overwrite`, a `replaceWhere` option can be used to transactionally * replace data that matches a predicate. * * In combination with `Overwrite` dynamic partition overwrite mode (option `partitionOverwriteMode` * set to `dynamic`, or in spark conf `spark.sql.sources.partitionOverwriteMode` set to `dynamic`) * is also supported. * * Dynamic partition overwrite mode conflicts with `replaceWhere`: * - If a `replaceWhere` option is provided, and dynamic partition overwrite mode is enabled in * the DataFrameWriter options, an error will be thrown. * - If a `replaceWhere` option is provided, and dynamic partition overwrite mode is enabled in * the spark conf, data will be overwritten according to the `replaceWhere` expression * * @param catalogTableOpt Should explicitly be set when table is accessed from catalog * @param schemaInCatalog The schema created in Catalog. We will use this schema to update metadata * when it is set (in CTAS code path), and otherwise use schema from `data`. */ case class WriteIntoDelta( override val deltaLog: DeltaLog, mode: SaveMode, options: DeltaOptions, partitionColumns: Seq[String], override val configuration: Map[String, String], override val data: DataFrame, val catalogTableOpt: Option[CatalogTable] = None, schemaInCatalog: Option[StructType] = None ) extends LeafRunnableCommand with ImplicitMetadataOperation with DeltaCommand with WriteIntoDeltaLike { override protected val canMergeSchema: Boolean = options.canMergeSchema private def isOverwriteOperation: Boolean = mode == SaveMode.Overwrite override protected val canOverwriteSchema: Boolean = options.canOverwriteSchema && isOverwriteOperation && options.replaceWhere.isEmpty override def run(sparkSession: SparkSession): Seq[Row] = { deltaLog.withNewTransaction(catalogTableOpt) { txn => if (hasBeenExecuted(txn, sparkSession, Some(options))) { return Seq.empty } val taggedCommitData = writeAndReturnCommitData( txn, sparkSession ) val operation = DeltaOperations.Write( mode = mode, partitionBy = Option(partitionColumns), predicate = options.replaceWhere, userMetadata = options.userMetadata, isDynamicPartitionOverwrite = if (Try(options.isDynamicPartitionOverwriteMode).getOrElse(false)) Some(true) else None, canOverwriteSchema = if (options.canOverwriteSchema) Some(true) else None, canMergeSchema = if (options.canMergeSchema) Some(true) else None ) txn.commitIfNeeded(taggedCommitData.actions, operation, tags = taggedCommitData.stringTags) } Seq.empty } override def writeAndReturnCommitData( txn: OptimisticTransaction, sparkSession: SparkSession, clusterBySpecOpt: Option[ClusterBySpec] = None, isTableReplace: Boolean = false): TaggedCommitData[Action] = { import org.apache.spark.sql.delta.implicits._ if (txn.readVersion > -1) { // This table already exists, check if the insert is valid. if (mode == SaveMode.ErrorIfExists) { throw DeltaErrors.pathAlreadyExistsException(deltaLog.dataPath) } else if (mode == SaveMode.Ignore) { return TaggedCommitData.empty } else if (mode == SaveMode.Overwrite) { DeltaLog.assertRemovable(txn.snapshot) } } val isReplaceWhere = mode == SaveMode.Overwrite && options.replaceWhere.nonEmpty val finalClusterBySpecOpt = if (mode == SaveMode.Append || isReplaceWhere) { clusterBySpecOpt.foreach { clusterBySpec => ClusteredTableUtils.validateClusteringColumnsInSnapshot(txn.snapshot, clusterBySpec) } // Append mode and replaceWhere cannot update the clustering columns. None } else { clusterBySpecOpt } val rearrangeOnly = options.rearrangeOnly val charPadding = sparkSession.conf.get(SQLConf.READ_SIDE_CHAR_PADDING) val charAsVarchar = sparkSession.conf.get(SQLConf.CHAR_AS_VARCHAR) val dataSchema = if (!charAsVarchar && charPadding) { data.schema } else { // If READ_SIDE_CHAR_PADDING is not enabled, CHAR type is the same as VARCHAR. The change // below makes DESC TABLE to show VARCHAR instead of CHAR. CharVarcharUtils.replaceCharVarcharWithStringInSchema( CharVarcharUtils.replaceCharWithVarchar(CharVarcharUtils.getRawSchema(data.schema)) .asInstanceOf[StructType]) } val finalSchema = schemaInCatalog.getOrElse(dataSchema) if (txn.metadata.schemaString != null) { // In cases other than CTAS (INSERT INTO, DataFrame write), block if values are provided for // GENERATED ALWAYS AS IDENTITY columns. IdentityColumn.blockExplicitIdentityColumnInsert( txn.metadata.schema, data.queryExecution.analyzed) } // We need to cache this canUpdateMetadata before calling updateMetadata, as that will update // it to true. This is unavoidable as getNewDomainMetadata uses information generated by // updateMetadata, so it needs to be run after that. val canUpdateMetadata = txn.canUpdateMetadata updateMetadata(data.sparkSession, txn, finalSchema, partitionColumns, configuration, isOverwriteOperation, rearrangeOnly ) val newDomainMetadata = getNewDomainMetadata( txn, canUpdateMetadata, isReplacingTable = isOverwriteOperation && options.replaceWhere.isEmpty, finalClusterBySpecOpt ) val replaceWhereOnDataColsEnabled = sparkSession.conf.get(DeltaSQLConf.REPLACEWHERE_DATACOLUMNS_ENABLED) val useDynamicPartitionOverwriteMode = { if (txn.metadata.partitionColumns.isEmpty) { // We ignore dynamic partition overwrite mode for non-partitioned tables false } else if (isTableReplace) { // A replace table command should always replace the table, not just some partitions. false } else if (options.replaceWhere.nonEmpty) { if (options.partitionOverwriteModeInOptions && options.isDynamicPartitionOverwriteMode) { // replaceWhere and dynamic partition overwrite conflict because they both specify which // data to overwrite. We throw an error when: // 1. replaceWhere is provided in a DataFrameWriter option // 2. partitionOverwriteMode is set to "dynamic" in a DataFrameWriter option throw DeltaErrors.replaceWhereUsedWithDynamicPartitionOverwrite() } else { // If replaceWhere is provided, we do not use dynamic partition overwrite, even if it's // enabled in the spark session configuration, since generally query-specific configs take // precedence over session configs false } } else { options.isDynamicPartitionOverwriteMode } } if (useDynamicPartitionOverwriteMode && canOverwriteSchema) { throw DeltaErrors.overwriteSchemaUsedWithDynamicPartitionOverwrite() } // Validate partition predicates var containsDataFilters = false val replaceWhere = options.replaceWhere.flatMap { replace => val parsed = parsePredicates(sparkSession, replace) if (replaceWhereOnDataColsEnabled) { // Helps split the predicate into separate expressions val (metadataPredicates, dataFilters) = DeltaTableUtils.splitMetadataAndDataPredicates( parsed.head, txn.metadata.partitionColumns, sparkSession) if (rearrangeOnly && dataFilters.nonEmpty) { throw DeltaErrors.replaceWhereWithFilterDataChangeUnset(dataFilters.mkString(",")) } containsDataFilters = dataFilters.nonEmpty Some(metadataPredicates ++ dataFilters) } else if (mode == SaveMode.Overwrite) { verifyPartitionPredicates(sparkSession, txn.metadata.partitionColumns, parsed) Some(parsed) } else { None } } if (txn.readVersion < 0) { // Initialize the log path deltaLog.createLogDirectoriesIfNotExists() } val (newFiles, addFiles, deletedFiles) = (mode, replaceWhere) match { case (SaveMode.Overwrite, Some(predicates)) if !replaceWhereOnDataColsEnabled => // fall back to match on partition cols only when replaceArbitrary is disabled. val newFiles = txn.writeFiles(data, Some(options)) val addFiles = newFiles.collect { case a: AddFile => a } // Check to make sure the files we wrote out were actually valid. val matchingFiles = DeltaLog.filterFileList( txn.metadata.partitionSchema, addFiles.toDF(sparkSession), predicates).as[AddFile] .collect() val invalidFiles = addFiles.toSet -- matchingFiles if (invalidFiles.nonEmpty) { val badPartitions = invalidFiles .map(_.partitionValues) .map { _.map { case (k, v) => s"$k=$v" }.mkString("/") } .mkString(", ") throw DeltaErrors.replaceWhereMismatchException(options.replaceWhere.get, badPartitions) } (newFiles, addFiles, txn.filterFiles(predicates).map(_.remove)) case (SaveMode.Overwrite, Some(conditions)) if txn.snapshot.version >= 0 => val constraints = extractConstraints(sparkSession, conditions) val removedFileActions = removeFiles(sparkSession, txn, conditions) val cdcExistsInRemoveOp = removedFileActions.exists(_.isInstanceOf[AddCDCFile]) // The above REMOVE will not produce explicit CDF data when persistent DV is enabled. // Therefore here we need to decide whether to produce explicit CDF for INSERTs, because // the CDF protocol requires either (i) all CDF data are generated explicitly as AddCDCFile, // or (ii) all CDF data can be deduced from [[AddFile]] and [[RemoveFile]]. val dataToWrite = if (containsDataFilters && CDCReader.isCDCEnabledOnTable(txn.metadata, sparkSession) && sparkSession.conf.get(DeltaSQLConf.REPLACEWHERE_DATACOLUMNS_WITH_CDF_ENABLED) && cdcExistsInRemoveOp) { var dataWithDefaultExprs = data // Add identity columns if they are not in `data`. // Column names for which we will track identity column high water marks. val trackHighWaterMarks = mutable.Set.empty[String] val topLevelOutputNames = CaseInsensitiveMap(data.schema.map(f => f.name -> f).toMap) val selectExprs = txn.metadata.schema.map { f => if (ColumnWithDefaultExprUtils.isIdentityColumn(f) && !topLevelOutputNames.contains(f.name)) { // Track high water marks for generated IDENTITY values. trackHighWaterMarks += f.name IdentityColumn.createIdentityColumnGenerationExprAsColumn(f) } else { SchemaUtils.fieldToColumn(f).alias(f.name) } } if (trackHighWaterMarks.nonEmpty) { txn.setTrackHighWaterMarks(trackHighWaterMarks.toSet) dataWithDefaultExprs = data.select(selectExprs: _*) } // pack new data and cdc data into an array of structs and unpack them into rows // to share values in outputCols on both branches, avoiding re-evaluating // non-deterministic expression twice. val outputCols = dataWithDefaultExprs.schema.map(SchemaUtils.fieldToColumn(_)) val insertCols = outputCols :+ lit(CDCReader.CDC_TYPE_INSERT).as(CDCReader.CDC_TYPE_COLUMN_NAME) val insertDataCols = outputCols :+ Column(CDCReader.CDC_TYPE_NOT_CDC) .as(CDCReader.CDC_TYPE_COLUMN_NAME) val packedInserts = array( struct(insertCols: _*), struct(insertDataCols: _*) ).expr dataWithDefaultExprs .select(explode(Column(packedInserts)).as("packedData")) .select( (dataWithDefaultExprs.schema.map(_.name) :+ CDCReader.CDC_TYPE_COLUMN_NAME) .map { n => col(s"packedData.`$n`").as(n) }: _*) } else { data } val newFiles = try txn.writeFiles(dataToWrite, Some(options), constraints) catch { case e: InvariantViolationException => throw DeltaErrors.replaceWhereMismatchException( options.replaceWhere.get, e) } (newFiles, newFiles.collect { case a: AddFile => a }, removedFileActions) case (SaveMode.Overwrite, None) => val newFiles = writeFiles( txn, data, options ) val addFiles = newFiles.collect { case a: AddFile => a } val deletedFiles = if (useDynamicPartitionOverwriteMode) { // with dynamic partition overwrite for any partition that is being written to all // existing data in that partition will be deleted. // the selection what to delete is on the next two lines val updatePartitions = addFiles.map(_.partitionValues).toSet txn.filterFiles(updatePartitions).map(_.remove) } else { txn.filterFiles().map(_.remove) } (newFiles, addFiles, deletedFiles) case _ => val newFiles = writeFiles( txn, data, options ) (newFiles, newFiles.collect { case a: AddFile => a }, Nil) } // Need to handle replace where metrics separately. if (replaceWhere.nonEmpty && replaceWhereOnDataColsEnabled && sparkSession.conf.get(DeltaSQLConf.REPLACEWHERE_METRICS_ENABLED)) { registerReplaceWhereMetrics(sparkSession, txn, newFiles, deletedFiles) } else if (mode == SaveMode.Overwrite && sparkSession.conf.get(DeltaSQLConf.OVERWRITE_REMOVE_METRICS_ENABLED)) { registerOverwriteRemoveMetrics(sparkSession, txn, deletedFiles) } val fileActions = if (rearrangeOnly) { val changeFiles = newFiles.collect { case c: AddCDCFile => c } if (changeFiles.nonEmpty) { throw DeltaErrors.unexpectedChangeFilesFound(changeFiles.mkString("\n")) } addFiles.map(_.copy(dataChange = !rearrangeOnly)) ++ deletedFiles.map { case add: AddFile => add.copy(dataChange = !rearrangeOnly) case remove: RemoveFile => remove.copy(dataChange = !rearrangeOnly) case other => throw DeltaErrors.illegalFilesFound(other.toString) } } else { newFiles ++ deletedFiles } val allActions = newDomainMetadata ++ createSetTransaction(sparkSession, deltaLog, Some(options)).toSeq ++ fileActions TaggedCommitData(allActions) } private def writeFiles( txn: OptimisticTransaction, data: DataFrame, options: DeltaOptions ): Seq[FileAction] = { txn.writeFiles(data, Some(options)) } private def removeFiles( spark: SparkSession, txn: OptimisticTransaction, conditions: Seq[Expression]): Seq[Action] = { val relation = LogicalRelation( txn.deltaLog.createRelation(snapshotToUseOpt = Some(txn.snapshot), catalogTableOpt = txn.catalogTable)) val processedCondition = conditions.reduceOption(And) val command = spark.sessionState.analyzer.execute( DeleteFromTable(relation, processedCondition.getOrElse(Literal.TrueLiteral))) spark.sessionState.analyzer.checkAnalysis(command) val (deleteActions, deleteMetrics) = command.asInstanceOf[DeleteCommand].performDelete(spark, txn.deltaLog, txn) recordDeltaEvent( deltaLog, "delta.dml.write.removeFiles.stats", data = deleteMetrics.copy(isWriteCommand = true) ) deleteActions } override def withNewWriterConfiguration(updatedConfiguration: Map[String, String]) : WriteIntoDeltaLike = this.copy(configuration = updatedConfiguration) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/WriteIntoDeltaLike.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.skipping.clustering.temp.ClusterBySpec import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.OptimisticTransaction import org.apache.spark.sql.delta.actions.Action import org.apache.spark.sql.delta.actions.AddCDCFile import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.actions.RemoveFile import org.apache.spark.sql.delta.commands.DMLUtils.TaggedCommitData import org.apache.spark.sql.delta.constraints.Constraint import org.apache.spark.sql.delta.constraints.Constraints.Check import org.apache.spark.sql.delta.constraints.Invariants.ArbitraryExpression import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.DataFrame import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} import org.apache.spark.sql.types.StructType /** * An interface for writing [[data]] into Delta tables. */ trait WriteIntoDeltaLike { /** * A helper method to create a new instances of [[WriteIntoDeltaLike]] with * updated [[configuration]]. */ def withNewWriterConfiguration(updatedConfiguration: Map[String, String]): WriteIntoDeltaLike /** * The configuration to be used for writing [[data]] into Delta table. */ val configuration: Map[String, String] /** * Data to be written into Delta table. */ val data: DataFrame /** * Write [[data]] into Delta table as part of [[txn]] and @return the actions to be committed. */ def writeAndReturnCommitData( txn: OptimisticTransaction, sparkSession: SparkSession, clusterBySpecOpt: Option[ClusterBySpec] = None, isTableReplace: Boolean = false): TaggedCommitData[Action] def write( txn: OptimisticTransaction, sparkSession: SparkSession, clusterBySpecOpt: Option[ClusterBySpec] = None, isTableReplace: Boolean = false): Seq[Action] = writeAndReturnCommitData( txn, sparkSession, clusterBySpecOpt, isTableReplace).actions val deltaLog: DeltaLog // Helper for creating a SQLMetric and setting its value, since it isn't valid to create a // SQLMetric with a positive `initValue`. private def createSumMetricWithValue( spark: SparkSession, name: String, value: Long): SQLMetric = { val metric = new SQLMetric("sum") metric.register(spark.sparkContext, Some(name)) metric.set(value) metric } /** * Overwrite mode produces extra delete metrics that are registered here. * @param deleteActions - RemoveFiles added by Delete job */ protected def registerOverwriteRemoveMetrics( spark: SparkSession, txn: OptimisticTransaction, deleteActions: Seq[Action]): Unit = { var numRemovedFiles = 0 var numRemovedBytes = 0L deleteActions.foreach { case action: RemoveFile => numRemovedFiles += 1 numRemovedBytes += action.getFileSize case _ => () // do nothing } val sqlMetrics = Map( "numRemovedFiles" -> createSumMetricWithValue( spark, "number of files removed", numRemovedFiles), "numRemovedBytes" -> createSumMetricWithValue( spark, "number of bytes removed", numRemovedBytes) ) txn.registerSQLMetrics(spark, sqlMetrics) } /** * Replace where operationMetrics need to be recorded separately. * @param newFiles - AddFile and AddCDCFile added by write job * @param deleteActions - AddFile, RemoveFile, AddCDCFile added by Delete job */ protected def registerReplaceWhereMetrics( spark: SparkSession, txn: OptimisticTransaction, newFiles: Seq[Action], deleteActions: Seq[Action]): Unit = { var numFiles = 0L var numCopiedRows = 0L var numOutputBytes = 0L var numNewRows = 0L var numAddedChangedFiles = 0L var hasRowLevelMetrics = true newFiles.foreach { case a: AddFile => numFiles += 1 numOutputBytes += a.size if (a.numLogicalRecords.isEmpty) { hasRowLevelMetrics = false } else { numNewRows += a.numLogicalRecords.get } case cdc: AddCDCFile => numAddedChangedFiles += 1 case _ => } deleteActions.foreach { case a: AddFile => numFiles += 1 numOutputBytes += a.size if (a.numLogicalRecords.isEmpty) { hasRowLevelMetrics = false } else { numCopiedRows += a.numLogicalRecords.get } case _: AddCDCFile => numAddedChangedFiles += 1 // Remove metrics will be handled by the delete command. case _ => } var sqlMetrics = Map( "numFiles" -> createSumMetricWithValue(spark, "number of files written", numFiles), "numOutputBytes" -> createSumMetricWithValue(spark, "number of output bytes", numOutputBytes), "numAddedChangeFiles" -> createSumMetricWithValue( spark, "number of change files added", numAddedChangedFiles) ) if (hasRowLevelMetrics) { sqlMetrics ++= Map( "numOutputRows" -> createSumMetricWithValue( spark, "number of rows added", numNewRows + numCopiedRows), "numCopiedRows" -> createSumMetricWithValue(spark, "number of copied rows", numCopiedRows) ) } else { // this will get filtered out in DeltaOperations.WRITE transformMetrics sqlMetrics ++= Map( "numOutputRows" -> createSumMetricWithValue(spark, "number of rows added", 0L), "numCopiedRows" -> createSumMetricWithValue(spark, "number of copied rows", 0L) ) } txn.registerSQLMetrics(spark, sqlMetrics) } protected def extractConstraints( sparkSession: SparkSession, exprs: Seq[Expression]): Seq[Constraint] = { if (!sparkSession.conf.get(DeltaSQLConf.REPLACEWHERE_CONSTRAINT_CHECK_ENABLED)) { Seq.empty } else { exprs.flatMap { e => // While writing out the new data, we only want to enforce constraint on expressions // with UnresolvedAttribute, that is, containing column name. Because we parse a // predicate string without analyzing it, if there's a column name, it has to be // unresolved. e.collectFirst { case _: UnresolvedAttribute => val arbitraryExpression = ArbitraryExpression(e) Check(arbitraryExpression.name, arbitraryExpression.expression) } } } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/alterDeltaTableCommands.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands // scalastyle:off import.ordering.noEmptyLine import java.util.Locale import java.util.concurrent.TimeUnit import scala.util.control.NonFatal import org.apache.spark.sql.delta.skipping.clustering.ClusteredTableUtils import org.apache.spark.sql.delta.skipping.clustering.ClusteringColumnInfo import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions.{DropTableFeatureUtils, Protocol, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillCommand import org.apache.spark.sql.delta.commands.columnmapping.RemoveColumnMappingCommand import org.apache.spark.sql.delta.constraints.{CharVarcharConstraint, Constraints} import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CoordinatedCommitsUtils} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.redirect.RedirectFeature import org.apache.spark.sql.delta.schema.{SchemaMergingUtils, SchemaUtils} import org.apache.spark.sql.delta.schema.SchemaUtils.transformSchema import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.StatisticsCollection import org.apache.hadoop.fs.Path import org.apache.spark.internal.MDC import org.apache.spark.internal.config.ConfigEntry import org.apache.spark.sql.{AnalysisException, Column, Row, SparkSession} import org.apache.spark.sql.catalyst.analysis.{Resolver, UnresolvedAttribute} import org.apache.spark.sql.catalyst.catalog.CatalogUtils import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans.logical.{Filter, IgnoreCachedData, QualifiedColType, QualifiedColTypeShims} import org.apache.spark.sql.catalyst.util.{CharVarcharUtils, SparkCharVarcharUtils} import org.apache.spark.sql.connector.catalog.TableCatalog import org.apache.spark.sql.connector.catalog.TableChange.{After, ColumnPosition, First} import org.apache.spark.sql.connector.expressions.FieldReference import org.apache.spark.sql.execution.command.LeafRunnableCommand import org.apache.spark.sql.types._ /** * A super trait for alter table commands that modify Delta tables. */ trait AlterDeltaTableCommand extends DeltaCommand { def table: DeltaTableV2 protected def startTransaction(): OptimisticTransaction = { // WARNING: It's not safe to use startTransactionWithInitialSnapshot here. Some commands call // this method more than once, and some commands can be created with a stale table. val txn = table.startTransaction() if (txn.readVersion == -1) { throw DeltaErrors.notADeltaTableException(table.name()) } txn } /** * Check if the column to change has any dependent expressions: * - generated column expressions * - check constraints */ protected def checkDependentExpressions( sparkSession: SparkSession, columnParts: Seq[String], oldMetadata: actions.Metadata, protocol: Protocol): Unit = { if (!sparkSession.sessionState.conf.getConf( DeltaSQLConf.DELTA_ALTER_TABLE_CHANGE_COLUMN_CHECK_EXPRESSIONS)) { return } // check if the column to change is referenced by check constraints val dependentConstraints = Constraints.findDependentConstraints(sparkSession, columnParts, oldMetadata) if (dependentConstraints.nonEmpty) { throw DeltaErrors.foundViolatingConstraintsForColumnChange( UnresolvedAttribute(columnParts).name, dependentConstraints) } // check if the column to change is referenced by any generated columns val dependentGenCols = SchemaUtils.findDependentGeneratedColumns( sparkSession, columnParts, protocol, oldMetadata.schema) if (dependentGenCols.nonEmpty) { throw DeltaErrors.foundViolatingGeneratedColumnsForColumnChange( UnresolvedAttribute(columnParts).name, dependentGenCols) } } } /** * A command that sets Delta table configuration. * * The syntax of this command is: * {{{ * ALTER TABLE table1 SET TBLPROPERTIES ('key1' = 'val1', 'key2' = 'val2', ...); * }}} */ case class AlterTableSetPropertiesDeltaCommand( table: DeltaTableV2, configuration: Map[String, String]) extends LeafRunnableCommand with AlterDeltaTableCommand with IgnoreCachedData { override def run(sparkSession: SparkSession): Seq[Row] = { val deltaLog = table.deltaLog RowTrackingFeature.validateAndBackfill(sparkSession, table, configuration) val columnMappingPropertyKey = DeltaConfigs.COLUMN_MAPPING_MODE.key val disableColumnMapping = configuration.get(columnMappingPropertyKey).contains("none") val columnMappingRemovalAllowed = sparkSession.sessionState.conf.getConf( DeltaSQLConf.ALLOW_COLUMN_MAPPING_REMOVAL) if (disableColumnMapping && columnMappingRemovalAllowed) { RemoveColumnMappingCommand(deltaLog, table.catalogTable) .run(sparkSession, removeColumnMappingTableProperty = false) // Not changing anything else, so we can return early. if (configuration.size == 1) { return Seq.empty[Row] } } recordDeltaOperation(deltaLog, "delta.ddl.alter.setProperties") { val txn = startTransaction() val metadata = txn.metadata val filteredConfs = configuration.filterKeys { case k if k.toLowerCase(Locale.ROOT).startsWith("delta.constraints.") => throw DeltaErrors.useAddConstraints case k if k == TableCatalog.PROP_LOCATION => throw DeltaErrors.useSetLocation() case k if k == TableCatalog.PROP_COMMENT => false case k if k == TableCatalog.PROP_PROVIDER => throw DeltaErrors.cannotChangeProvider() case k if k == TableFeatureProtocolUtils.propertyKey(ClusteringTableFeature) => throw DeltaErrors.alterTableSetClusteringTableFeatureException( ClusteringTableFeature.name) case k if k == ClusteredTableUtils.PROP_CLUSTERING_COLUMNS => throw DeltaErrors.cannotModifyTableProperty(k) case _ => true }.toMap // For Coordinated Commits table validation CoordinatedCommitsUtils.validateConfigurationsForAlterTableSetPropertiesDeltaCommand( existingConfs = metadata.configuration, propertyOverrides = filteredConfs) // For Catalog Owned table validation CatalogOwnedTableUtils.validatePropertiesForAlterTableSetPropertiesDeltaCommand( txn.snapshot, propertyOverrides = filteredConfs) // If table redirect feature is updated, validates its property. RedirectFeature.validateTableRedirect(txn.snapshot, table.catalogTable, configuration) val newMetadata = metadata.copy( description = configuration.getOrElse(TableCatalog.PROP_COMMENT, metadata.description), configuration = metadata.configuration ++ filteredConfs) txn.updateMetadata(newMetadata) // Tag if the metadata update is _only_ for enabling row tracking. This allows for // an optimization where we can safely not fail concurrent txns from the metadata update. var tags = Map.empty[String, String] val enableRowTracking = configuration .getOrElse(DeltaConfigs.ROW_TRACKING_ENABLED.key, "false") .toBoolean if (enableRowTracking) { RowTrackingFeature.validateConfigurations(txn.metadata.configuration ++ configuration) // In general, we should be able to detect any relevant state changes during backfill. // Nevertheless, before enabling row tracking make sure the main invariants are satisfied. // Note, Delta makes sure conflicting txns backfill their own files. However, this does // not cover third party writers. RowTracking.verifyInvariantsForTablePropertyEnablement(txn.snapshot) if (configuration.size == 1) { tags += (DeltaCommitTag.RowTrackingEnablementOnlyTag.key -> "true") } } txn.commit(Nil, DeltaOperations.SetTableProperties(configuration), tags) Seq.empty[Row] } } } /** * A command that unsets Delta table configuration. * If ifExists is false, each individual key will be checked if it exists or not, it's a * one-by-one operation, not an all or nothing check. Otherwise, non-existent keys will be ignored. * * The syntax of this command is: * {{{ * ALTER TABLE table1 UNSET TBLPROPERTIES [IF EXISTS] ('key1', 'key2', ...); * }}} */ case class AlterTableUnsetPropertiesDeltaCommand( table: DeltaTableV2, propKeys: Seq[String], ifExists: Boolean, fromDropFeatureCommand: Boolean = false) extends LeafRunnableCommand with AlterDeltaTableCommand with IgnoreCachedData { override def run(sparkSession: SparkSession): Seq[Row] = { val deltaLog = table.deltaLog val columnMappingPropertyKey = DeltaConfigs.COLUMN_MAPPING_MODE.key val disableColumnMapping = propKeys.contains(columnMappingPropertyKey) val columnMappingRemovalAllowed = sparkSession.sessionState.conf.getConf( DeltaSQLConf.ALLOW_COLUMN_MAPPING_REMOVAL) if (disableColumnMapping && columnMappingRemovalAllowed) { RemoveColumnMappingCommand(deltaLog, table.catalogTable) .run(sparkSession, removeColumnMappingTableProperty = true) if (propKeys.size == 1) { // Not unsetting anything else, so we can return early. return Seq.empty[Row] } } recordDeltaOperation(deltaLog, "delta.ddl.alter.unsetProperties") { val txn = startTransaction() val metadata = txn.metadata val normalizedKeys = DeltaConfigs.normalizeConfigKeys(propKeys) if (!ifExists) { normalizedKeys.foreach { k => if (!metadata.configuration.contains(k)) { throw DeltaErrors.unsetNonExistentProperty(k, table.name()) } } } if (!fromDropFeatureCommand) { CoordinatedCommitsUtils.validateConfigurationsForAlterTableUnsetPropertiesDeltaCommand( existingConfs = metadata.configuration, propKeysToUnset = normalizedKeys) CatalogOwnedTableUtils.validatePropertiesForAlterTableUnsetPropertiesDeltaCommand( txn.snapshot, propKeysToUnset = normalizedKeys) } val newConfiguration = metadata.configuration.filterNot { case (key, _) => normalizedKeys.contains(key) } val description = if (normalizedKeys.contains(TableCatalog.PROP_COMMENT)) null else { metadata.description } val newMetadata = metadata.copy( description = description, configuration = newConfiguration) txn.updateMetadata(newMetadata) txn.commit(Nil, DeltaOperations.UnsetTableProperties(normalizedKeys, ifExists)) Seq.empty[Row] } } } /** * A command that removes an existing feature from the table. The feature needs to implement the * [[RemovableFeature]] trait. * * The syntax of the command is: * {{{ * ALTER TABLE t DROP FEATURE f [TRUNCATE HISTORY] * }}} * * When dropping a feature, remove the feature traces from the latest version. However, the table * history still contains feature traces. This creates two problems: * * 1) Reconstructing the state of the latest version may require replaying log records prior to * feature removal. Log replay is based on checkpoints which is used by clients as a starting * point for replaying history. Any actions before the checkpoint do not need to be replayed. * However, checkpoints may be deleted at any time, which can then expose readers to older * log records. * 2) Clients could create checkpoints in past versions. These could lead to incorrect behavior * if the client that created the checkpoint did not support all features. * * To address these issues, we currently provide two implementations: * * 1) [[executeDropFeatureWithHistoryTruncation]]. We truncate history at the boundary of version * of the dropped feature (when required). Requires two executions of the drop feature command * with a waiting time in between the two executions. * 2) [[executeDropFeatureWithCheckpointProtection]], i.e. fast drop feature. We create barrier * checkpoints to protect against log replay and checkpoint creation. The behavior is enforced * with the aid of CheckpointProtectionTableFeature. * * Config tableFeatures.fastDropFeature.enabled can be used to control which implementation * is used. Furthermore, please note the option [TRUNCATE HISTORY] in the SQL syntax is only * relevant for [[executeDropFeatureWithHistoryTruncation]]. When used, we always fallback to that * implementation. * * At a high level, dropping a feature consists of two stages (see [[RemovableFeature]]): * * 1) preDowngradeCommand. This command is responsible for removing any data and metadata * related to the feature. * 2) Protocol downgrade. Removes the feature from the current version's protocol. * During this stage we also validate whether all traces of the feature-to-be-removed are gone. * * For removing features with requiresHistoryProtection=false the two steps above are sufficient. * For features that require history protection, we follow a different approach for each of the * implementations listed above. Please see the corresponding functions for more details. * * Note, legacy features can be removed as well. When removing a legacy feature from a legacy * protocol, if the result cannot be represented with a legacy representation we use the * table features representation. For example, removing Invariants from (1, 3) results to * (1, 7, None, [AppendOnly, CheckConstraints]). Adding back Invariants to the protocol is * normalized back to (1, 3). This allows to consistently transition back and forth between legacy * protocols and table feature protocols. */ case class AlterTableDropFeatureDeltaCommand( table: DeltaTableV2, featureName: String, truncateHistory: Boolean = false) extends LeafRunnableCommand with AlterDeltaTableCommand with IgnoreCachedData { import org.apache.spark.sql.delta.actions.DropTableFeatureUtils._ override def run(sparkSession: SparkSession): Seq[Row] = { // Check whether the protocol contains the feature in either the writer features list or // the reader+writer features list. Note, protocol needs to denormalized to allow dropping // features from legacy protocols. val protocol = table.update().protocol val protocolContainsFeatureName = protocol.implicitlyAndExplicitlySupportedFeatures.map(_.name).contains(featureName) val featureInLowerCase = featureName.toLowerCase(Locale.ROOT) val removableFeature = TableFeature.featureNameToFeature(featureName) match { // Check if a property was passed instead of a feature, featureName has to // start with "delta." if that is the case. case _ if !protocolContainsFeatureName && featureInLowerCase.startsWith("delta.") && DeltaConfigs.getAllConfigs.contains(featureInLowerCase.stripPrefix("delta.")) => throw DeltaErrors.dropTableFeatureFeatureIsADeltaProperty(featureName) case Some(_) if !protocolContainsFeatureName => throw DeltaErrors.dropTableFeatureFeatureNotSupportedByProtocol(featureName) case Some(feature: RemovableFeature) => feature case Some(_) => throw DeltaErrors.dropTableFeatureNonRemovableFeature(featureName) case None => throw DeltaErrors.dropTableFeatureFeatureNotSupportedByClient(featureName) } val historyTruncationEligibleFeature = removableFeature.requiresHistoryProtection || removableFeature == CheckpointProtectionTableFeature if (truncateHistory && !historyTruncationEligibleFeature) { throw DeltaErrors.tableFeatureDropHistoryTruncationNotAllowed() } if (removableFeature == CheckpointProtectionTableFeature && !truncateHistory) { throw DeltaErrors.canOnlyDropCheckpointProtectionWithHistoryTruncationException } // Validate that the `removableFeature` is not a dependency of any other feature that is // enabled on the table. dependentFeatureCheck(removableFeature, protocol) // If the user uses the truncate history option we always fallback to the old implementation. if (!truncateHistory && conf.getConf(DeltaSQLConf.FAST_DROP_FEATURE_ENABLED)) { executeDropFeatureWithCheckpointProtection(sparkSession, removableFeature) } else { executeDropFeatureWithHistoryTruncation(sparkSession, removableFeature) } } /** * Drop features with history truncation. When dropping a feature that * requiresHistoryProtection, we need to truncate history prior to feature removal to ensure * the history does not contain any traces of the removed feature. The user journey is the * following: * * 1) The user runs the remove feature command which removes any traces of the feature from * the latest version. The removal command throws a message that there was partial success * and the retention period must pass before a protocol downgrade is possible. * 2) The user runs again the command after the retention period is over. The command checks the * current state again and the history. If everything is clean, it proceeds with the protocol * downgrade. The TRUNCATE HISTORY option may be used here to automatically set * the log retention period to a minimum of 24 hours before clearing the logs. The minimum * value is based on the expected duration of the longest running transaction. This is the * lowest retention period we can set without endangering concurrent transactions. * If transactions do run for longer than this period while this command is run, then this * can lead to data corruption. */ private def executeDropFeatureWithHistoryTruncation( sparkSession: SparkSession, removableFeature: TableFeature with RemovableFeature): Seq[Row] = { val deltaLog = table.deltaLog recordDeltaOperation(deltaLog, "delta.ddl.alter.dropFeature") { // The removableFeature.preDowngradeCommand needs to adhere to the following requirements: // // a) Bring the table to a state the validation passes. // b) To not allow concurrent commands to alter the table in a way the validation does not // pass. This can be done by first disabling the relevant metadata property. // c) Undoing (b) should cause the preDowngrade command to fail. // // Note, for features that cannot be disabled we solely rely for correctness on // isSnapshotClean. val requiresHistoryValidation = removableFeature.requiresHistoryProtection val startTimeNs = table.deltaLog.clock.nanoTime() val status = removableFeature .preDowngradeCommand(table) .removeFeatureTracesIfNeeded(sparkSession) val preDowngradeMadeChanges = status.performedChanges if (requiresHistoryValidation) { // Generate a checkpoint after the cleanup that is based on commits that do not use // the feature. This intends to help slow-moving tables to qualify for history truncation // asap. The checkpoint is based on a new commit to avoid creating a checkpoint // on a commit that still contains traces of the removed feature. // Note, the checkpoint is created in both executions of DROP FEATURE command. createEmptyCommitAndCheckpoint(table, startTimeNs) // If the pre-downgrade command made changes, then the table's historical versions // certainly still contain traces of the feature. We don't have to run an expensive // explicit check, but instead we fail straight away. if (preDowngradeMadeChanges) { throw DeltaErrors.dropTableFeatureWaitForRetentionPeriod( featureName, table.initialSnapshot.metadata) } } val startSnapshotOpt = status.lastCommitVersionOpt.map(table.getSnapshotAt(_)) val txn = table.startTransaction(snapshotOpt = startSnapshotOpt) val snapshot = txn.snapshot // Verify whether all requirements hold before performing the protocol downgrade. // If any concurrent transactions interfere with the protocol downgrade txn we // revalidate the requirements against the snapshot of the winning txn. if (!removableFeature.validateDropInvariants(table, snapshot)) { TransactionExecutionObserver.getObserver.transactionAborted() throw DeltaErrors.dropTableFeatureConflictRevalidationFailed() } // For reader+writer features, before downgrading the protocol we need to ensure there are no // traces of the feature in past versions. If traces are found, the user is advised to wait // until the retention period is over. This is a slow operation. // Note, if this txn conflicts, we check all winning commits for traces of the feature. // Therefore, we do not need to check again for historical versions during conflict // resolution. if (requiresHistoryValidation) { // Clean up expired logs before checking history. This also makes sure there is no // concurrent metadataCleanup during findEarliestReliableCheckpoint. Note, this // cleanUpExpiredLogs call truncates the cutoff at a minute granularity. deltaLog.cleanUpExpiredLogs( snapshotToCleanup = snapshot, catalogTableOpt = table.catalogTable, deltaRetentionMillisOpt = if (truncateHistory) Some(truncateHistoryLogRetentionMillis(txn.metadata)) else None, cutoffTruncationGranularity = if (truncateHistory) TruncationGranularity.MINUTE else TruncationGranularity.DAY) val historyContainsFeature = removableFeature.historyContainsFeature( spark = sparkSession, table = table, downgradeTxnReadSnapshot = snapshot) if (historyContainsFeature) { throw DeltaErrors.dropTableFeatureHistoricalVersionsExist(featureName, snapshot.metadata) } } val op = DeltaOperations.DropTableFeature(featureName, truncateHistory) txn.updateProtocol(txn.protocol.removeFeature(removableFeature)) val metadataWithNewConfiguration = DropTableFeatureUtils .getDowngradedProtocolMetadata(removableFeature, txn.metadata) val commitActions = removableFeature.actionsToIncludeAtDowngradeCommit(txn.snapshot) txn.updateMetadata(metadataWithNewConfiguration) txn.commit(commitActions, op) recordDeltaEvent( deltaLog = deltaLog, opType = "dropFeatureCompleted.withHistoryTruncation", data = Map("droppedFeature" -> removableFeature.name)) Nil } } /** * Drop [[removableFeature]] and enforce correctness with protected checkpoints. When dropping a * feature that requiresHistoryProtection we need to make sure: * * 1) Clients will not process historical commits that contain traces of the dropped feature. * 2) Clients will not create checkpoints in historical versions when they do not support the * required features. * * This can be achieved as follows: * 1) Create DELTA_SNAPSHOT_LOADING_MAX_RETRIES + 1 checkpoints (3 checkpoints), * after the execution of the pre-downgrade command but before the protocol downgrade commit. * Clients should never attempt to read prior to these checkpoints. We create multiple * checkpoints because the Delta Spark client may try multiple earlier checkpoints when it * encounters a checkpoint that it cannot read. By adding multiple checkpoints, we make * sure it gives up before it proceeds to earlier checkpoints. * 2) Protect checkpoints 1-3 from metadata cleanup operations. This is achieved with the aid of * the CheckpointProtectionTableFeature. Using a table property we store the version of * the last dropped feature, V. With CheckpointProtectionTableFeature we enforce clients * can only delete checkpoints before V if they clean up the corresponding commit history at * the same time and do not create any checkpoints prior to V. To create a checkpoint prior * to V, they must support all features for the versions they truncate. * 3) In a single commit, drop the feature from the protocol, set V = the version the current * feature is dropped and add the CheckpointProtectionTableFeature. * 4) Create a checkpoint after the protocol downgrade. This is optional and it is used to allow * log replay from checkpoint 3 to the protocol downgrade commit. This is for clients that do * not support the dropped feature and choose to validate against the protocols of all commits * used in the replay of a snapshot instead of only validating against the final resulting * protocol. * */ private def executeDropFeatureWithCheckpointProtection( sparkSession: SparkSession, removableFeature: TableFeature with RemovableFeature): Seq[Row] = { val deltaLog = table.deltaLog recordDeltaOperation(deltaLog, "delta.ddl.alter.dropFeatureWithCheckpointProtection") { var startTimeNs = System.nanoTime() val status = removableFeature .preDowngradeCommand(table) .removeFeatureTracesIfNeeded(sparkSession) // Create and validate the barrier checkpoints. The checkpoint are created on top of // empty commits. However, this is not guaranteed. Other txns might interleave the empty // commit. As a result the checkpoint could be created on top of an unrelated non-empty // commit. This is not a problem. We only care about the checkpoint being created after the // pre-downgrade process and before the protocol downgrade commit. // Furthermore, each checkpoint validation requires a roundtrip to the object store. // Could be optimized, if required, by performing a single list operation and checking // whether NUMBER_OF_BARRIER_CHECKPOINTS exist after the pre-downgrade commit. val historyBarrierIsRequired = removableFeature.requiresHistoryProtection if (historyBarrierIsRequired) { (1 to DropTableFeatureUtils.NUMBER_OF_BARRIER_CHECKPOINTS).foreach { _ => // This call also cleans up the logs. In most of the cases we should be able to truncate // the history of a previous drop feature operation. if (!createEmptyCommitAndCheckpoint(table, startTimeNs, retryOnFailure = true)) { throw DeltaErrors.dropTableFeatureCheckpointFailedException(removableFeature.name) } startTimeNs = System.nanoTime() } } val startSnapshotOpt = status.lastCommitVersionOpt.map(table.getSnapshotAt(_)) val txn = table.startTransaction(snapshotOpt = startSnapshotOpt) val snapshot = txn.snapshot // Verify whether all requirements hold before performing the protocol downgrade. // If any concurrent transactions interfere with the protocol downgrade txn we // revalidate the requirements against the snapshot of the winning txn. if (!removableFeature.validateDropInvariants(table, snapshot)) { TransactionExecutionObserver.getObserver.transactionAborted() throw DeltaErrors.dropTableFeatureConflictRevalidationFailed() } val metadataWithNewConfiguration = DropTableFeatureUtils .getDowngradedProtocolMetadata(removableFeature, txn.metadata) if (historyBarrierIsRequired) { val newProtocol = txn.protocol .denormalized .withFeature(CheckpointProtectionTableFeature) .removeFeature(removableFeature) txn.updateProtocol(newProtocol) val newMetadata = CheckpointProtectionTableFeature.metadataWithCheckpointProtection( metadataWithNewConfiguration, txn.readVersion + 1L) txn.updateMetadata(newMetadata) } else { txn.updateProtocol(txn.protocol.removeFeature(removableFeature)) txn.updateMetadata(metadataWithNewConfiguration) } val commitActions = removableFeature.actionsToIncludeAtDowngradeCommit(txn.snapshot) txn.commit(commitActions, DeltaOperations.DropTableFeature(featureName, false)) // This is a protected checkpoint. if (historyBarrierIsRequired) createCheckpointWithRetries(table, System.nanoTime()) recordDeltaEvent( deltaLog = deltaLog, opType = "dropFeatureCompleted.withCheckpointProtection", data = Map("droppedFeature" -> removableFeature.name)) Nil } } /** * Checks if the `removableFeature` is a requirement for some other feature that is enabled on the * table. In such a scenario, we need to fail the drop feature command. The dependent features * needs to be dropped first before this `removableFeature` can be removed. */ private def dependentFeatureCheck( removableFeature: TableFeature, protocol: Protocol): Unit = { val dependentFeatures = TableFeature.getDependentFeatures(removableFeature) if (dependentFeatures.nonEmpty) { val dependentFeaturesInProtocol = dependentFeatures.filter(protocol.isFeatureSupported) if (dependentFeaturesInProtocol.nonEmpty) { val dependentFeatureNames = dependentFeaturesInProtocol.map(_.name) throw DeltaErrors.dropTableFeatureFailedBecauseOfDependentFeatures( removableFeature.name, dependentFeatureNames.toSeq) } } } } /** * A command that add columns to a Delta table. * The syntax of using this command in SQL is: * {{{ * ALTER TABLE table_identifier * ADD COLUMNS (col_name data_type [COMMENT col_comment], ...); * }}} */ case class AlterTableAddColumnsDeltaCommand( table: DeltaTableV2, colsToAddWithPosition: Seq[QualifiedColType]) extends LeafRunnableCommand with AlterDeltaTableCommand with IgnoreCachedData { override def run(sparkSession: SparkSession): Seq[Row] = { val deltaLog = table.deltaLog recordDeltaOperation(deltaLog, "delta.ddl.alter.addColumns") { val txn = startTransaction() if (SchemaUtils.filterRecursively( StructType(colsToAddWithPosition.map { case QualifiedColTypeWithPosition(_, column, _) => column }), true)(!_.nullable).nonEmpty) { throw DeltaErrors.operationNotSupportedException("NOT NULL in ALTER TABLE ADD COLUMNS") } // TODO: remove this after auto cache refresh is merged. table.tableIdentifier.foreach { identifier => try sparkSession.catalog.uncacheTable(identifier) catch { case NonFatal(e) => log.warn(s"Exception when attempting to uncache table $identifier", e) } } val metadata = txn.metadata val oldSchema = metadata.schema val resolver = sparkSession.sessionState.conf.resolver val newSchema = colsToAddWithPosition.foldLeft(oldSchema) { case (schema, QualifiedColTypeWithPosition(columnPath, column, None)) => val parentPosition = SchemaUtils.findColumnPosition(columnPath, schema, resolver) val insertPosition = SchemaUtils.getNestedTypeFromPosition(schema, parentPosition) match { case s: StructType => s.size case other => throw DeltaErrors.addColumnParentNotStructException(column, other) } SchemaUtils.addColumn(schema, column, parentPosition :+ insertPosition) case (schema, QualifiedColTypeWithPosition(columnPath, column, Some(_: First))) => val parentPosition = SchemaUtils.findColumnPosition(columnPath, schema, resolver) SchemaUtils.addColumn(schema, column, parentPosition :+ 0) case (schema, QualifiedColTypeWithPosition(columnPath, column, Some(after: After))) => val prevPosition = SchemaUtils.findColumnPosition(columnPath :+ after.column, schema, resolver) val position = prevPosition.init :+ (prevPosition.last + 1) SchemaUtils.addColumn(schema, column, position) } SchemaMergingUtils.checkColumnNameDuplication(newSchema, "in adding columns") SchemaUtils.checkSchemaFieldNames(newSchema, metadata.columnMappingMode) val newMetadata = metadata.copy(schemaString = newSchema.json) txn.updateMetadata(newMetadata) txn.commit(Nil, DeltaOperations.AddColumns( colsToAddWithPosition.map { case QualifiedColTypeWithPosition(path, col, colPosition) => DeltaOperations.QualifiedColTypeWithPositionForLog( path, col, colPosition.map(_.toString)) })) Seq.empty[Row] } } object QualifiedColTypeWithPosition { private def toV2Position(input: Any): ColumnPosition = { input.asInstanceOf[org.apache.spark.sql.catalyst.analysis.FieldPosition].position } def unapply( col: QualifiedColType): Option[(Seq[String], StructField, Option[ColumnPosition])] = { val builder = new MetadataBuilder col.comment.foreach(builder.putString("comment", _)) val field = StructField(col.name.last, col.dataType, col.nullable, builder.build()) QualifiedColTypeShims.getDefaultValueStr(col).map { defaultStr => Some((col.name.init, field.withCurrentDefaultValue(defaultStr), col.position.map(toV2Position))) }.getOrElse { Some((col.name.init, field, col.position.map(toV2Position))) } } } } /** * A command that drop columns from a Delta table. * The syntax of using this command in SQL is: * {{{ * ALTER TABLE table_identifier * DROP COLUMN(S) (col_name_1, col_name_2, ...); * }}} */ case class AlterTableDropColumnsDeltaCommand( table: DeltaTableV2, columnsToDrop: Seq[Seq[String]]) extends LeafRunnableCommand with AlterDeltaTableCommand with IgnoreCachedData { override def run(sparkSession: SparkSession): Seq[Row] = { if (!sparkSession.sessionState.conf.getConf( DeltaSQLConf.DELTA_ALTER_TABLE_DROP_COLUMN_ENABLED)) { // this featue is still behind the flag and not ready for release. throw DeltaErrors.dropColumnNotSupported(suggestUpgrade = false) } val deltaLog = table.deltaLog recordDeltaOperation(deltaLog, "delta.ddl.alter.dropColumns") { val txn = startTransaction() val metadata = txn.metadata if (txn.metadata.columnMappingMode == NoMapping) { throw DeltaErrors.dropColumnNotSupported(suggestUpgrade = true) } val newSchema = columnsToDrop.foldLeft(metadata.schema) { case (schema, columnPath) => val parentPosition = SchemaUtils.findColumnPosition( columnPath, schema, sparkSession.sessionState.conf.resolver) SchemaUtils.dropColumn(schema, parentPosition)._1 } // in case any of the dropped column is partition columns val droppedColumnSet = columnsToDrop.map(UnresolvedAttribute(_).name).toSet val droppingPartitionCols = metadata.partitionColumns.filter(droppedColumnSet.contains(_)) if (droppingPartitionCols.nonEmpty) { throw DeltaErrors.dropPartitionColumnNotSupported(droppingPartitionCols) } // Disallow dropping clustering columns. val clusteringCols = ClusteringColumnInfo.extractLogicalNames(txn.snapshot) val droppingClusteringCols = clusteringCols.filter(droppedColumnSet.contains(_)) if (droppingClusteringCols.nonEmpty) { throw DeltaErrors.dropClusteringColumnNotSupported(droppingClusteringCols) } // Updates the delta statistics column list by removing the dropped columns from it. val newConfiguration = metadata.configuration ++ StatisticsCollection.dropDeltaStatsColumns(metadata, columnsToDrop) val newMetadata = metadata.copy( schemaString = newSchema.json, configuration = newConfiguration ) columnsToDrop.foreach { columnParts => checkDependentExpressions(sparkSession, columnParts, metadata, txn.protocol) } txn.updateMetadata(newMetadata) txn.commit(Nil, DeltaOperations.DropColumns(columnsToDrop)) Seq.empty[Row] } } } case class DeltaChangeColumnSpec( columnPath: Seq[String], columnName: String, newColumn: StructField, colPosition: Option[ColumnPosition], syncIdentity: Boolean) /** * A command to change the column for a Delta table, support changing the comment of a column and * reordering columns. * * The syntax of using this command in SQL is: * {{{ * ALTER TABLE table_identifier * CHANGE [COLUMN] column_old_name column_new_name column_dataType [COMMENT column_comment] * [FIRST | AFTER column_name]; * }}} */ case class AlterTableChangeColumnDeltaCommand( table: DeltaTableV2, columnChanges: Seq[DeltaChangeColumnSpec]) extends LeafRunnableCommand with AlterDeltaTableCommand with IgnoreCachedData { override def run(sparkSession: SparkSession): Seq[Row] = { val deltaLog = table.deltaLog recordDeltaOperation(deltaLog, "delta.ddl.alter.changeColumns") { val txn = startTransaction() val metadata = txn.metadata val bypassCharVarcharToStringFix = sparkSession.conf.get(DeltaSQLConf.DELTA_BYPASS_CHARVARCHAR_TO_STRING_FIX) val oldSchema = metadata.schema val resolver = sparkSession.sessionState.conf.resolver columnChanges.foreach(change => { val columnName = change.columnName val columnPath = change.columnPath val newColumn = change.newColumn if (newColumn.name != columnName) { // need to validate the changes if the column is renamed checkDependentExpressions( sparkSession, columnPath :+ columnName, metadata, txn.protocol) } // Verify that the columnName provided actually exists in the schema SchemaUtils.findColumnPosition(columnPath :+ columnName, oldSchema, resolver) }) def transformSchemaOnce(prevSchema: StructType, change: DeltaChangeColumnSpec) = { val columnPath = change.columnPath val columnName = change.columnName val newColumn = change.newColumn transformSchema(prevSchema, Some(columnName)) { case (`columnPath`, struct @ StructType(fields), _) => val oldColumn = struct(columnName) verifyColumnChange(change, sparkSession, oldColumn, resolver, txn) val newField = { if (change.syncIdentity) { assert(oldColumn == newColumn) val df = txn.snapshot.deltaLog.createDataFrame(txn.snapshot, txn.filterFiles()) val allowLoweringHighWaterMarkForSyncIdentity = sparkSession.conf .get(DeltaSQLConf.DELTA_IDENTITY_ALLOW_SYNC_IDENTITY_TO_LOWER_HIGH_WATER_MARK) val field = IdentityColumn.syncIdentity( deltaLog, newColumn, df, allowLoweringHighWaterMarkForSyncIdentity ) txn.setSyncIdentity() txn.readWholeTable() field } else { // Take the name, comment, nullability and data type from newField // It's crucial to keep the old column's metadata, which may contain column mapping // metadata. var result = newColumn.getComment().map(oldColumn.withComment).getOrElse(oldColumn) // Apply the current default value as well, if any. result = newColumn.getCurrentDefaultValue() match { case Some(newDefaultValue) => result.withCurrentDefaultValue(newDefaultValue) case None => result.clearCurrentDefaultValue() } result = SchemaUtils.changeFieldDataTypeCharVarcharSafe(result, newColumn, resolver) result.copy( name = newColumn.name, nullable = newColumn.nullable) } } // Replace existing field with new field val newFieldList = fields.map { field => if (DeltaColumnMapping.getPhysicalName(field) == DeltaColumnMapping.getPhysicalName(newField)) { newField } else field } // Reorder new field to correct position if necessary StructType(change.colPosition.map { position => reorderFieldList(columnName, struct, newFieldList, newField, position, resolver) }.getOrElse(newFieldList.toSeq)) case (`columnPath`, m: MapType, _) if columnName == "key" => val originalField = StructField(columnName, m.keyType, nullable = false) verifyMapArrayChange(change, sparkSession, originalField, resolver, txn) val fieldWithNewDataType = SchemaUtils.changeFieldDataTypeCharVarcharSafe( originalField, newColumn, resolver) m.copy(keyType = fieldWithNewDataType.dataType) case (`columnPath`, m: MapType, _) if columnName == "value" => val originalField = StructField(columnName, m.valueType, nullable = m.valueContainsNull) verifyMapArrayChange(change, sparkSession, originalField, resolver, txn) val fieldWithNewDataType = SchemaUtils.changeFieldDataTypeCharVarcharSafe( originalField, newColumn, resolver) m.copy(valueType = fieldWithNewDataType.dataType) case (`columnPath`, a: ArrayType, _) if columnName == "element" => val originalField = StructField(columnName, a.elementType, nullable = a.containsNull) verifyMapArrayChange(change, sparkSession, originalField, resolver, txn) val fieldWithNewDataType = SchemaUtils.changeFieldDataTypeCharVarcharSafe( originalField, newColumn, resolver) a.copy(elementType = fieldWithNewDataType.dataType) case (_, other @ (_: StructType | _: ArrayType | _: MapType), _) => other } } val transformedSchema = columnChanges.foldLeft(oldSchema)(transformSchemaOnce) // Validate clustering columns remain in stats schema after column reordering validateClusteringColumnsAfterReordering(sparkSession, txn, columnChanges) val newSchemaWithTypeWideningMetadata = TypeWideningMetadata.addTypeWideningMetadata( txn, schema = transformedSchema, oldSchema = metadata.schema) val metadataWithNewSchema = metadata.copy( schemaString = newSchemaWithTypeWideningMetadata.json) def updateMetadataOnce(prevMetadata: actions.Metadata, change: DeltaChangeColumnSpec) = { val columnPath = change.columnPath val columnName = change.columnName val newColumn = change.newColumn // update `partitionColumns` if the changed column is a partition column val newPartitionColumns = if (columnPath.isEmpty) { metadata.partitionColumns.map { partCol => if (partCol == columnName) newColumn.name else partCol } } else metadata.partitionColumns val oldColumnPath = columnPath :+ columnName val newColumnPath = columnPath :+ newColumn.name // Rename the column in the delta statistics columns configuration, if present. val newConfiguration = metadata.configuration ++ StatisticsCollection.renameDeltaStatsColumn(metadata, oldColumnPath, newColumnPath) val updatedMetadata = prevMetadata.copy( partitionColumns = newPartitionColumns, configuration = newConfiguration ) updatedMetadata } val newMetadata = columnChanges.foldLeft(metadataWithNewSchema)(updateMetadataOnce) txn.updateMetadata(newMetadata) def getDeltaChangeColumnOperation(change: DeltaChangeColumnSpec) = DeltaOperations.ChangeColumn( change.columnPath, change.columnName, change.newColumn, change.colPosition.map(_.toString)) val operation = if (columnChanges.size == 1) { val change = columnChanges.head val columnName = change.columnName val newColumn = change.newColumn if (newColumn.name != columnName) { val columnPath = change.columnPath val oldColumnPath = columnPath :+ columnName val newColumnPath = columnPath :+ newColumn.name // record column rename separately DeltaOperations.RenameColumn(oldColumnPath, newColumnPath) } else { getDeltaChangeColumnOperation(change) } } else { val changes = columnChanges.map(getDeltaChangeColumnOperation) DeltaOperations.ChangeColumns(changes) } txn.commit(Nil, operation) Seq.empty[Row] } } /** * Reorder the given fieldList to place `field` at the given `position` in `fieldList` * * @param columnName Name of the column being reordered * @param struct The initial StructType with the original field at its original position * @param fieldList List of fields with the changed field in the original position * @param field The field that is to be added * @param position Position where the field is to be placed * @return Returns a new list of fields with the changed field in the new position */ private def reorderFieldList( columnName: String, struct: StructType, fieldList: Array[StructField], field: StructField, position: ColumnPosition, resolver: Resolver): Seq[StructField] = { val startIndex = struct.fieldIndex(columnName) val filtered = fieldList.filterNot(_.name == columnName) val newFieldList = position match { case _: First => field +: filtered case after: After if after.column() == columnName => filtered.slice(0, startIndex)++ Seq(field) ++ filtered.slice(startIndex, filtered.length) case after: After => val endIndex = filtered.indexWhere(i => resolver(i.name, after.column())) if (endIndex < 0) { throw DeltaErrors.columnNotInSchemaException(after.column(), struct) } filtered.slice(0, endIndex + 1) ++ Seq(field) ++ filtered.slice(endIndex + 1, filtered.length) } newFieldList.toSeq } /** * Validates that clustering columns remain in the stats schema after column reordering. * * This validation ensures that when a user executes `ALTER TABLE ALTER COLUMN col1 AFTER col2`, * all clustering columns that were in the stats schema before the reordering remain in the * stats schema after the operation. When DELTA_LIQUID_ALTER_COLUMN_AFTER_STATS_SCHEMA_CHECK * is enabled, the validation runs and throws an error if any clustering column would lose * stats collection due to position-based indexing. When disabled (default), no validation * is performed and stats collection may follow position-based indexing rules. * * @param spark The SparkSession * @param txn The transaction * @param columnChanges The column changes being applied */ private def validateClusteringColumnsAfterReordering( spark: SparkSession, txn: OptimisticTransaction, columnChanges: Seq[DeltaChangeColumnSpec]): Unit = { if (!spark.conf.get( DeltaSQLConf.DELTA_LIQUID_ALTER_COLUMN_AFTER_STATS_SCHEMA_CHECK)) { return } // Only validate if table supports clustering and check is enabled if (ClusteredTableUtils.isSupported(txn.snapshot.protocol) && columnChanges.exists(_.colPosition.isDefined)) { val clusteringColumns = ClusteringColumnInfo.extractLogicalNames(txn.snapshot) if (clusteringColumns.nonEmpty) { // Validate that prior stats schema is preserved (clustering columns remain in stats) ClusteredTableUtils.validateClusteringColumnsInStatsSchema( txn.snapshot, clusteringColumns) } } } /** * Given two columns, verify whether replacing the original column with the new column is a valid * operation. * * Note that this requires a full table scan in the case of SET NOT NULL to verify that all * existing values are valid. * * @param change Information about the column change * @param originalField The existing column */ private def verifyColumnChange( change: DeltaChangeColumnSpec, spark: SparkSession, originalField: StructField, resolver: Resolver, txn: OptimisticTransaction): Unit = { val columnPath = change.columnPath val columnName = change.columnName val newColumn = change.newColumn originalField.dataType match { case same if same == newColumn.dataType => // just changing comment or position so this is fine case s: StructType if s != newColumn.dataType => val fieldName = UnresolvedAttribute(columnPath :+ columnName).name throw DeltaErrors.cannotUpdateStructField(table.name(), fieldName) case m: MapType if m != newColumn.dataType => val fieldName = UnresolvedAttribute(columnPath :+ columnName).name throw DeltaErrors.cannotUpdateMapField(table.name(), fieldName) case a: ArrayType if a != newColumn.dataType => val fieldName = UnresolvedAttribute(columnPath :+ columnName).name throw DeltaErrors.cannotUpdateArrayField(table.name(), fieldName) case _: AtomicType => // update is okay case o => throw DeltaErrors.cannotUpdateOtherField(table.name(), o) } // Analyzer already validates the char/varchar type change of ALTER COLUMN in // `CheckAnalysis.checkAlterTableCommand`. We should normalize char/varchar type to string type // first (original data type is already normalized as we store char/varchar as string type with // special metadata in the Delta log), then apply Delta-specific checks. val newType = CharVarcharUtils.replaceCharVarcharWithString(newColumn.dataType) if (SchemaUtils.canChangeDataType( originalField.dataType, newType, resolver, txn.metadata.columnMappingMode, columnPath :+ originalField.name, allowTypeWidening = TypeWidening.isEnabled(txn.protocol, txn.metadata) ).nonEmpty) { throw DeltaErrors.alterTableChangeColumnException( fieldPath = UnresolvedAttribute(columnPath :+ originalField.name).name, oldField = originalField, newField = newColumn ) } if (columnName != newColumn.name) { if (txn.metadata.columnMappingMode == NoMapping) { throw DeltaErrors.columnRenameNotSupported } } if (originalField.dataType != newType) { checkDependentExpressions( spark, columnPath :+ columnName, txn.metadata, txn.protocol) } if (originalField.nullable && !newColumn.nullable) { throw DeltaErrors.alterTableChangeColumnException( fieldPath = UnresolvedAttribute(columnPath :+ originalField.name).name, oldField = originalField, newField = newColumn ) } } /** * Verify whether replacing the original map key/value or array element with a new data type is a * valid operation. * * @param change Information about the column change * @param originalField the original map key/value or array element to update. */ private def verifyMapArrayChange( change: DeltaChangeColumnSpec, spark: SparkSession, originalField: StructField, resolver: Resolver, txn: OptimisticTransaction): Unit = { val columnPath = change.columnPath val columnName = change.columnName val newColumn = change.newColumn // Map key/value and array element can't have comments. if (newColumn.getComment().nonEmpty) { throw DeltaErrors.addCommentToMapArrayException( fieldPath = UnresolvedAttribute(columnPath :+ columnName).name ) } // Changing the nullability of map key/value or array element isn't supported. if (originalField.nullable != newColumn.nullable) { throw DeltaErrors.alterTableChangeColumnException( fieldPath = UnresolvedAttribute(columnPath :+ originalField.name).name, oldField = originalField, newField = newColumn ) } verifyColumnChange(change, spark, originalField, resolver, txn) } } /** * A command to replace columns for a Delta table, support changing the comment of a column, * reordering columns, and loosening nullabilities. * * The syntax of using this command in SQL is: * {{{ * ALTER TABLE table_identifier REPLACE COLUMNS (col_spec[, col_spec ...]); * }}} */ case class AlterTableReplaceColumnsDeltaCommand( table: DeltaTableV2, columns: Seq[StructField]) extends LeafRunnableCommand with AlterDeltaTableCommand with IgnoreCachedData { override def run(sparkSession: SparkSession): Seq[Row] = { recordDeltaOperation(table.deltaLog, "delta.ddl.alter.replaceColumns") { val txn = startTransaction() val metadata = txn.metadata val existingSchema = metadata.schema if (ColumnWithDefaultExprUtils.hasIdentityColumn(table.initialSnapshot.schema)) { throw DeltaErrors.identityColumnReplaceColumnsNotSupported() } val resolver = sparkSession.sessionState.conf.resolver val changingSchema = StructType(columns) SchemaUtils.canChangeDataType( existingSchema, changingSchema, resolver, txn.metadata.columnMappingMode, allowTypeWidening = TypeWidening.isEnabled(txn.protocol, txn.metadata), failOnAmbiguousChanges = true ).foreach { operation => throw DeltaErrors.alterTableReplaceColumnsException( existingSchema, changingSchema, operation) } val newSchema = SchemaUtils.changeDataType(existingSchema, changingSchema, resolver) .asInstanceOf[StructType] SchemaMergingUtils.checkColumnNameDuplication(newSchema, "in replacing columns") SchemaUtils.checkSchemaFieldNames(newSchema, metadata.columnMappingMode) val newSchemaWithTypeWideningMetadata = TypeWideningMetadata.addTypeWideningMetadata( txn, schema = newSchema, oldSchema = existingSchema ) val newMetadata = metadata.copy(schemaString = newSchemaWithTypeWideningMetadata.json) txn.updateMetadata(newMetadata) txn.commit(Nil, DeltaOperations.ReplaceColumns(columns)) Nil } } } /** * A command to change the location of a Delta table. Effectively, this only changes the symlink * in the Hive MetaStore from one Delta table to another. * * This command errors out if the new location is not a Delta table. By default, the new Delta * table must have the same schema as the old table, but we have a SQL conf that allows users * to bypass this schema check. * * The syntax of using this command in SQL is: * {{{ * ALTER TABLE table_identifier SET LOCATION 'path/to/new/delta/table'; * }}} */ case class AlterTableSetLocationDeltaCommand( table: DeltaTableV2, location: String) extends LeafRunnableCommand with AlterDeltaTableCommand with IgnoreCachedData { override def run(sparkSession: SparkSession): Seq[Row] = { val catalog = sparkSession.sessionState.catalog if (table.catalogTable.isEmpty) { throw DeltaErrors.setLocationNotSupportedOnPathIdentifiers() } val catalogTable = table.catalogTable.get val locUri = CatalogUtils.stringToURI(location) val oldTable = table.update() if (oldTable.version == -1) { throw DeltaErrors.notADeltaTableException(table.name()) } val oldMetadata = oldTable.metadata var updatedTable = catalogTable.withNewStorage(locationUri = Some(locUri)) val (_, newTable) = DeltaLog.forTableWithSnapshot(sparkSession, updatedTable, options = Map.empty[String, String]) if (newTable.version == -1) { throw DeltaErrors.notADeltaTableException(DeltaTableIdentifier(path = Some(location))) } val newMetadata = newTable.metadata val bypassSchemaCheck = sparkSession.sessionState.conf.getConf( DeltaSQLConf.DELTA_ALTER_LOCATION_BYPASS_SCHEMA_CHECK) if (!bypassSchemaCheck && !schemasEqual(sparkSession, oldMetadata, newMetadata)) { throw DeltaErrors.alterTableSetLocationSchemaMismatchException( oldMetadata.schema, newMetadata.schema) } catalog.alterTable(updatedTable) Seq.empty[Row] } private def schemasEqual( sparkSession: SparkSession, oldMetadata: actions.Metadata, newMetadata: actions.Metadata): Boolean = { DeltaTableUtils.removeInternalDeltaMetadata(sparkSession, oldMetadata.schema) == DeltaTableUtils.removeInternalDeltaMetadata(sparkSession, newMetadata.schema) && DeltaTableUtils.removeInternalDeltaMetadata(sparkSession, oldMetadata.partitionSchema) == DeltaTableUtils.removeInternalDeltaMetadata(sparkSession, newMetadata.partitionSchema) } } trait AlterTableConstraintDeltaCommand extends LeafRunnableCommand with AlterDeltaTableCommand with IgnoreCachedData { def getConstraintWithName( table: DeltaTableV2, name: String, metadata: actions.Metadata, sparkSession: SparkSession): Option[String] = { val expr = Constraints.getExprTextByName(name, metadata, sparkSession) if (expr.nonEmpty) { return expr } None } } /** * Command to add a constraint to a Delta table. Currently only CHECK constraints are supported. * * Adding a constraint will scan all data in the table to verify the constraint currently holds. * * @param table The table to which the constraint should be added. * @param name The name of the new constraint. * @param exprText The contents of the new CHECK constraint, to be parsed and evaluated. */ case class AlterTableAddConstraintDeltaCommand( table: DeltaTableV2, name: String, exprText: String) extends AlterTableConstraintDeltaCommand { override def run(sparkSession: SparkSession): Seq[Row] = { val deltaLog = table.deltaLog if (name == CharVarcharConstraint.INVARIANT_NAME) { throw DeltaErrors.invalidConstraintName(name) } recordDeltaOperation(deltaLog, "delta.ddl.alter.addConstraint") { val txn = startTransaction() getConstraintWithName(table, name, txn.metadata, sparkSession).foreach { oldExpr => throw DeltaErrors.constraintAlreadyExists(name, oldExpr) } val newMetadata = txn.metadata.copy( configuration = txn.metadata.configuration + (Constraints.checkConstraintPropertyName(name) -> exprText) ) val df = txn.snapshot.deltaLog.createDataFrame(txn.snapshot, txn.filterFiles()) val unresolvedExpr = sparkSession.sessionState.sqlParser.parseExpression(exprText) try { df.where(Column(unresolvedExpr)).queryExecution.analyzed } catch { case a: AnalysisException if a.errorClass.contains("DATATYPE_MISMATCH.FILTER_NOT_BOOLEAN") => throw DeltaErrors.checkConstraintNotBoolean(name, exprText) case a: AnalysisException => // Strip out the context of the DataFrame that was used to analyze the expression. throw a.copy(context = Array.empty) } Constraints.validateCheckConstraints( sparkSession, Seq(Constraints.Check(name, unresolvedExpr)), deltaLog, txn.metadata.schema ) logInfo(log"Checking that ${MDC(DeltaLogKeys.EXPR, exprText)} " + log"is satisfied for existing data. This will require a full table scan.") recordDeltaOperation( txn.snapshot.deltaLog, "delta.ddl.alter.addConstraint.checkExisting") { val n = df.where(Column(Or(Not(unresolvedExpr), IsUnknown(unresolvedExpr)))).count() if (n > 0) { throw DeltaErrors.newCheckConstraintViolated(n, table.name(), exprText) } } txn.commit(newMetadata :: Nil, DeltaOperations.AddConstraint(name, exprText)) } Seq() } } /** * Command to drop a constraint from a Delta table. No-op if a constraint with the given name * doesn't exist. * * Currently only CHECK constraints are supported. * * @param table The table from which the constraint should be dropped * @param name The name of the constraint to drop */ case class AlterTableDropConstraintDeltaCommand( table: DeltaTableV2, name: String, ifExists: Boolean) extends AlterTableConstraintDeltaCommand { override def run(sparkSession: SparkSession): Seq[Row] = { val deltaLog = table.deltaLog recordDeltaOperation(deltaLog, "delta.ddl.alter.dropConstraint") { val txn = startTransaction() val oldExprText = Constraints.getExprTextByName(name, txn.metadata, sparkSession) if (oldExprText.isEmpty && !ifExists && !sparkSession.sessionState.conf.getConf( DeltaSQLConf.DELTA_ASSUMES_DROP_CONSTRAINT_IF_EXISTS)) { val quotedTableName = table.getTableIdentifierIfExists.map(_.quotedString) .orElse(table.catalogTable.map(_.identifier.quotedString)) .getOrElse(table.name()) throw DeltaErrors.nonexistentConstraint(name, quotedTableName) } val newMetadata = txn.metadata.copy( configuration = txn.metadata.configuration - Constraints.checkConstraintPropertyName(name)) txn.commit(newMetadata :: Nil, DeltaOperations.DropConstraint(name, oldExprText)) } Seq() } } /** * Command for altering clustering columns for clustered tables. * - ALTER TABLE .. CLUSTER BY (col1, col2, ...) * - ALTER TABLE .. CLUSTER BY NONE * * Note that the given `clusteringColumns` are empty when CLUSTER BY NONE is specified. * Also, `clusteringColumns` are validated (e.g., duplication / existence check) in * DeltaCatalog.alterTable(). */ case class AlterTableClusterByDeltaCommand( table: DeltaTableV2, clusteringColumns: Seq[Seq[String]]) extends LeafRunnableCommand with AlterDeltaTableCommand with IgnoreCachedData { override def run(sparkSession: SparkSession): Seq[Row] = { val deltaLog = table.deltaLog ClusteredTableUtils.validateNumClusteringColumns(clusteringColumns, Some(deltaLog)) // If the target table is not a clustered table and there are no clustering columns being added // (CLUSTER BY NONE), do not convert the table into a clustered table. val snapshot = table.update() if (clusteringColumns.isEmpty && !ClusteredTableUtils.isSupported(snapshot.protocol)) { logInfo(log"Skipping ALTER TABLE CLUSTER BY NONE on a non-clustered table: " + log"${MDC(DeltaLogKeys.TABLE_NAME, table.name())}.") recordDeltaEvent( deltaLog, "delta.ddl.alter.clusterBy", data = Map( "isClusterByNoneSkipped" -> true, "isNewClusteredTable" -> false, "oldColumnsCount" -> 0, "newColumnsCount" -> 0)) return Seq.empty } recordDeltaOperation(deltaLog, "delta.ddl.alter.clusterBy") { val txn = startTransaction() val clusteringColsLogicalNames = ClusteringColumnInfo.extractLogicalNames(txn.snapshot) val oldLogicalClusteringColumnsString = clusteringColsLogicalNames.mkString(",") val oldColumnsCount = clusteringColsLogicalNames.size val newLogicalClusteringColumns = clusteringColumns.map(FieldReference(_).toString) ClusteredTableUtils.validateClusteringColumnsInStatsSchema( txn.snapshot, newLogicalClusteringColumns) val newDomainMetadata = ClusteredTableUtils .getClusteringDomainMetadataForAlterTableClusterBy(newLogicalClusteringColumns, txn) recordDeltaEvent( deltaLog, "delta.ddl.alter.clusterBy", data = Map( "isClusterByNoneSkipped" -> false, "isNewClusteredTable" -> !ClusteredTableUtils.isSupported(txn.protocol), "oldColumnsCount" -> oldColumnsCount, "newColumnsCount" -> clusteringColumns.size)) // Add clustered table properties if the current table is not clustered. // [[DeltaCatalog.alterTable]] already ensures that the table is not partitioned. if (!ClusteredTableUtils.isSupported(txn.protocol)) { txn.updateMetadata( txn.metadata.copy( configuration = txn.metadata.configuration ++ ClusteredTableUtils.getTableFeatureProperties(txn.metadata.configuration) )) } txn.commit( newDomainMetadata, DeltaOperations.ClusterBy( oldLogicalClusteringColumnsString, newLogicalClusteringColumns.mkString(","))) } Seq.empty[Row] } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/backfill/BackfillBatch.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.backfill import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger import org.apache.spark.sql.delta.OptimisticTransaction import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.internal.MDC import org.apache.spark.sql.SparkSession trait BackfillBatch extends DeltaLogging { /** The files in this batch. */ def filesInBatch: Seq[AddFile] def backfillBatchStatsOpType: String protected def prepareFilesAndCommit( spark: SparkSession, txn: OptimisticTransaction, batchId: Int): Long /** * The main method of this trait. This method commits the backfill batch, records metrics and * updates the two atomic counters passed in. * * @param spark The Spark session. * @param backfillTxnId the transaction id associated with the parent command. * @param batchId an integer identifier of the batch within a parent [[BackfillCommand]]. * @param txn transaction used to construct the current batch. * @param numSuccessfulBatch an AtomicInteger which serves as a counter for the total number of * batches that were successful. * @param numFailedBatch an AtomicInteger which serves as a counter for the total number of * batches that failed. */ def execute( spark: SparkSession, backfillTxnId: String, batchId: Int, txn: OptimisticTransaction, numSuccessfulBatch: AtomicInteger, numFailedBatch: AtomicInteger): Long = { val startTimeNs = System.nanoTime() def recordBackfillBatchStats(txnId: String, wasSuccessful: Boolean): Unit = { if (wasSuccessful) { numSuccessfulBatch.incrementAndGet() } else { numFailedBatch.incrementAndGet() } val totalExecutionTimeInMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs) val batchStats = BackfillBatchStats( backfillTxnId, txnId, batchId, filesInBatch.size, totalExecutionTimeInMs, wasSuccessful) recordDeltaEvent( txn.deltaLog, opType = backfillBatchStatsOpType, data = batchStats ) } logInfo(log"Batch ${MDC(DeltaLogKeys.BATCH_ID, batchId.toLong)} starting, committing " + log"${MDC(DeltaLogKeys.NUM_FILES, filesInBatch.size.toLong)} candidate files") val txnId = txn.txnId try { val commitVersion = prepareFilesAndCommit(spark, txn, batchId) recordBackfillBatchStats(txnId, wasSuccessful = true) commitVersion } catch { case t: Throwable => recordBackfillBatchStats(txnId, wasSuccessful = false) throw t } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/backfill/BackfillBatchStats.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.backfill /** * Metrics for each BackfillBatch. * * @param parentTransactionId The transaction id associated with the parent command. * @param transactionId The transaction id used in this batch. * @param batchId An integer identifier of the batch within a parent BackfillCommand. * @param initialNumFiles The number of files in BackfillBatch prior to conflict * resolution. * @param totalExecutionTimeInMs The total execution time in milliseconds. * @param wasSuccessful Boolean indicating whether the batch was successfully committed. */ case class BackfillBatchStats( parentTransactionId: String, transactionId: String, batchId: Int, initialNumFiles: Long, totalExecutionTimeInMs: Long, wasSuccessful: Boolean ) ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/backfill/BackfillCommand.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.backfill import java.util.UUID import java.util.concurrent.TimeUnit import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.commands.DeltaCommand import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.execution.command.LeafRunnableCommand /** * This command will lazily materialize AllFiles and split them into multiple backfill commits * if the number of files exceeds the threshold set by * [[DeltaSQLConf.DELTA_BACKFILL_MAX_NUM_FILES_PER_COMMIT]]. */ trait BackfillCommand extends LeafRunnableCommand with DeltaCommand { def deltaLog: DeltaLog def nameOfTriggeringOperation: String def catalogTableOpt: Option[CatalogTable] def getBackfillExecutor( spark: SparkSession, deltaLog: DeltaLog, catalogTableOpt: Option[CatalogTable], backfillId: String, backfillStats: BackfillCommandStats): BackfillExecutor def opType: String override def run(spark: SparkSession): Seq[Row] = { recordDeltaOperation(deltaLog, opType) { val backfillId = UUID.randomUUID().toString val maxNumFilesPerCommit = spark.conf.get(DeltaSQLConf.DELTA_BACKFILL_MAX_NUM_FILES_PER_COMMIT) val startTimeNs = System.nanoTime() val backfillStats = BackfillCommandStats( transactionId = backfillId, nameOfTriggeringOperation ) try { val backfillExecutor = getBackfillExecutor( spark, deltaLog, catalogTableOpt, backfillId, backfillStats) val lastCommitOpt = backfillExecutor.run(maxNumFilesPerCommit) backfillStats.wasSuccessful = true Array.empty[Row] ++ lastCommitOpt.map(Row(_)) } finally { val totalExecutionTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs) backfillStats.totalExecutionTimeMs = totalExecutionTimeMs recordDeltaEvent( deltaLog, opType = opType + ".stats", data = backfillStats ) } } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/backfill/BackfillCommandStats.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.backfill /** * Metrics for the BackfillCommand. * * @param transactionId: The transaction id associated with this BackfillCommand that * is the parent transaction for BackfillBatch commits. * @param nameOfTriggeringOperation: The name of the operation that triggered backfill. For now, * this can be ALTER TABLE SET TBLPROPERTIES * @param totalExecutionTimeMs The total execution time in milliseconds. * @param numSuccessfulBatches The number of BackfillBatch's that was successfully committed. * @param numFailedBatches The number of BackfillBatch's that failed. * @param wasSuccessful Boolean indicating whether this BackfillCommand didn't have any error. */ case class BackfillCommandStats( transactionId: String, nameOfTriggeringOperation: String, var totalExecutionTimeMs: Long = 0, var numSuccessfulBatches: Int = 0, var numFailedBatches: Int = 0, var wasSuccessful: Boolean = false ) ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/backfill/BackfillExecutionObserver.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.backfill import org.apache.spark.sql.delta.{ChainableExecutionObserver, NoOpTransactionExecutionObserver, ThreadStorageExecutionObserver, TransactionExecutionObserver} trait BackfillExecutionObserver extends ChainableExecutionObserver[BackfillExecutionObserver] { def executeBatch[T](f: => T): T override def advanceToNextThreadObserver(): Unit = { BackfillExecutionObserver.setObserver(nextObserver.getOrElse(NoOpBackfillExecutionObserver)) } } object BackfillExecutionObserver extends ThreadStorageExecutionObserver[BackfillExecutionObserver] { override protected val threadObserver: ThreadLocal[BackfillExecutionObserver] = new InheritableThreadLocal[BackfillExecutionObserver] { override def initialValue(): BackfillExecutionObserver = NoOpBackfillExecutionObserver } override protected def initialValue: BackfillExecutionObserver = NoOpBackfillExecutionObserver } object NoOpBackfillExecutionObserver extends BackfillExecutionObserver { def executeBatch[T](f: => T): T = f } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/backfill/BackfillExecutor.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.backfill import java.util.concurrent.atomic.AtomicInteger import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.{Dataset, SparkSession} import org.apache.spark.sql.catalyst.catalog.CatalogTable trait BackfillExecutor extends DeltaLogging { def spark: SparkSession def deltaLog: DeltaLog def catalogTableOpt: Option[CatalogTable] def backfillTxnId: String def backfillStats: BackfillCommandStats def backFillBatchOpType: String def filesToBackfill(snapshot: Snapshot): Dataset[AddFile] def constructBatch(files: Seq[AddFile]): BackfillBatch /** * Execute the command by consuming a sequence of [[BackfillBatch]]. * Returns an option with the last commit version when available. Otherwise, it returns None. */ def run(maxNumFilesPerCommit: Int): Option[Long] = { executeBackfillBatches(maxNumFilesPerCommit) } /** * Execute all available [[BackfillBatch]]. * Returns an option with the last commit version when available. Otherwise, it returns None. * * Note, in the case of competing concurrent transactions, this method will exit after processing * a maximum of `backfill.maxNumFilesFactor` times the total number of files in the table. */ private def executeBackfillBatches(maxNumFilesPerCommit: Int): Option[Long] = { val observer = BackfillExecutionObserver.getObserver val numSuccessfulBatch = new AtomicInteger(0) val numFailedBatch = new AtomicInteger(0) val totalFileCount = deltaLog.update(catalogTableOpt = catalogTableOpt).numOfFiles val factor = spark.conf.get(DeltaSQLConf.DELTA_BACKFILL_MAX_NUM_FILES_FACTOR) val maxFilesToProcess = totalFileCount * factor var batchId = 0 var totalFilesProcessed = 0L var filesInBatch = Seq.empty[AddFile] var lastCommitOpt: Option[Long] = None // If the last batch contained fewer files than the maxNumFilesPerCommit we exit. // This protects against live-locking with fast concurrent txns that only commit // a few files. // Having excluded this option the backfill can only live-lock with an equally fast // concurrent txn, i.e a competing un-backfill that only commits logs files. // To protect against this we set a maximum backfill limit equal to a factor of the // total table file count. def moreFilesToProcess(): Boolean = { filesInBatch.length == maxNumFilesPerCommit && totalFilesProcessed < maxFilesToProcess } try { do { val snapshot = deltaLog.update(catalogTableOpt = catalogTableOpt) filesInBatch = filesToBackfill(snapshot).limit(maxNumFilesPerCommit).collect() if (filesInBatch.isEmpty) { return lastCommitOpt } val batch = constructBatch(filesInBatch) observer.executeBatch { val txn = deltaLog.startTransaction(catalogTableOpt, Some(snapshot)) txn.trackFilesRead(filesInBatch) recordDeltaOperation(deltaLog, backFillBatchOpType) { lastCommitOpt = Some(batch.execute( spark, backfillTxnId, batchId, txn, numSuccessfulBatch, numFailedBatch)) } batchId += 1 totalFilesProcessed += filesInBatch.length } } while (moreFilesToProcess()) if (totalFilesProcessed >= maxFilesToProcess) { recordDeltaEvent( deltaLog, opType = "delta.backfillExceededMaxFilesToProcess", data = Map("maxFilesProcessed" -> maxFilesToProcess)) } lastCommitOpt } finally { backfillStats.numSuccessfulBatches = numSuccessfulBatch.get() backfillStats.numFailedBatches = numFailedBatch.get() } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/backfill/RowTrackingBackfillBatch.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.backfill import org.apache.spark.sql.delta.{DeltaConfigs, DeltaOperations, OptimisticTransaction, RowTrackingFeature} import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} import org.apache.spark.sql.SparkSession case class RowTrackingBackfillBatch(filesInBatch: Seq[AddFile]) extends BackfillBatch { override val backfillBatchStatsOpType = "delta.rowTracking.backfill.batch.stats" /** Mark all files as dataChange = false and commit. */ override protected def prepareFilesAndCommit( spark: SparkSession, txn: OptimisticTransaction, batchId: Int): Long = { val protocol = txn.snapshot.protocol val metadata = txn.snapshot.metadata val isRowTrackingSupported = protocol.isFeatureSupported(RowTrackingFeature) val ignoreProperty = DeltaSQLConf.DELTA_ROW_TRACKING_IGNORE_SUSPENSION.key val ignoreSuspension = DeltaUtils.isTesting && spark.conf.get(ignoreProperty).toBoolean val suspendRowTracking = DeltaConfigs.ROW_TRACKING_SUSPENDED.fromMetaData(metadata) && !ignoreSuspension if (!isRowTrackingSupported || suspendRowTracking) { throw new IllegalStateException( """ |Cannot run backfill command if row tracking is not supported or |row ID generation is suspended.""".stripMargin) } val filesToCommit = filesInBatch.map(_.copy(dataChange = false)) // Base Row IDs are added as part of the OptimisticTransaction.prepareCommit(), so we don't // need to do anything here other than recommit the files. // Note: A backfill commit can be empty ,i.e. have no file actions, at commit time due to // files being removed by concurrent conflict resolution. txn.commit(filesToCommit, DeltaOperations.RowTrackingBackfill(batchId)) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/backfill/RowTrackingBackfillCommand.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.backfill import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.{AddFile, Protocol, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} import org.apache.spark.sql.{Dataset, Row, SparkSession} import org.apache.spark.sql.catalyst.catalog.CatalogTable /** * This command re-commits all AddFiles in the current snapshot that do not have a base row IDs. * After the backfill command finishes, the snapshot has row IDs for all files. All commits * afterwards must include row IDs. * * First, we will add the table feature support, if necessary. * Then, the command will lazily materialize AllFiles and split them into multiple backfill commits * if the number of files exceeds the threshold set by * [[DeltaSQLConf.DELTA_BACKFILL_MAX_NUM_FILES_PER_COMMIT]]. * * Note: We expect Backfill to be called before the table property is set. Furthermore, we do not * set the table property [[DeltaConfigs.ROW_TRACKING_ENABLED]] as part of backfill. The metadata * update needs to be handled by the caller. */ case class RowTrackingBackfillCommand( override val deltaLog: DeltaLog, override val nameOfTriggeringOperation: String, override val catalogTableOpt: Option[CatalogTable]) extends BackfillCommand { override def getBackfillExecutor( spark: SparkSession, deltaLog: DeltaLog, catalogTableOpt: Option[CatalogTable], backfillId: String, backfillStats: BackfillCommandStats): BackfillExecutor = new RowTrackingBackfillExecutor(spark, deltaLog, catalogTableOpt, backfillId, backfillStats) override def opType: String = "delta.rowTracking.backfill" /** * Add Row tracking table feature support. This will also upgrade the minWriterVersion if * the current protocol cannot support write table feature. */ private def upgradeProtocolIfRequired(): Unit = { val snapshot = deltaLog.update(catalogTableOpt = catalogTableOpt) if (!snapshot.protocol.isFeatureSupported(RowTrackingFeature)) { val minProtocolAllowingWriteTableFeature = Protocol( snapshot.protocol.minReaderVersion, TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION) val newProtocol = snapshot.protocol .merge(minProtocolAllowingWriteTableFeature.withFeature(RowTrackingFeature)) deltaLog.upgradeProtocol(catalogTableOpt, snapshot, newProtocol) } } override def run(spark: SparkSession): Seq[Row] = { if (!spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_ROW_TRACKING_BACKFILL_ENABLED)) { throw new UnsupportedOperationException("Cannot enable Row IDs on an existing table.") } // Upgrade the protocol to support the table feature if it isn't already supported. // This steps bounds the number of files the command must commit, since all actions after // we support the table feature must have base row IDs. upgradeProtocolIfRequired() super.run(spark) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/backfill/RowTrackingBackfillExecutor.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.backfill import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.{Dataset, SparkSession} import org.apache.spark.sql.catalyst.catalog.CatalogTable class RowTrackingBackfillExecutor( override val spark: SparkSession, override val deltaLog: DeltaLog, override val catalogTableOpt: Option[CatalogTable], override val backfillTxnId: String, override val backfillStats: BackfillCommandStats) extends BackfillExecutor { override val backFillBatchOpType = "delta.rowTracking.backfill.batch" override def filesToBackfill(snapshot: Snapshot): Dataset[AddFile] = RowTrackingBackfillExecutor.getCandidateFilesToBackfill(snapshot) override def constructBatch(files: Seq[AddFile]): BackfillBatch = RowTrackingBackfillBatch(files) } private[delta] object RowTrackingBackfillExecutor extends DeltaLogging { /** * Returns the dataset with the list of candidate files to backfill. */ def getCandidateFilesToBackfill(snapshot: Snapshot): Dataset[AddFile] = { // Note: We can't use txn.filterFiles() because it drops the file statistics. snapshot .allFiles .filter(_.baseRowId.isEmpty) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/backfill/RowTrackingUnBackfillBatch.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.backfill import org.apache.spark.sql.delta.{DeltaConfigs, DeltaOperations, OptimisticTransaction} import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.SparkSession case class RowTrackingUnBackfillBatch(filesInBatch: Seq[AddFile]) extends BackfillBatch { override val backfillBatchStatsOpType = "delta.rowTracking.unbackfill.batch.stats" /** Remove relevant metadata from addFiles. */ override protected def prepareFilesAndCommit( spark: SparkSession, txn: OptimisticTransaction, batchId: Int): Long = { val metadata = txn.snapshot.metadata val isEnabled = DeltaConfigs.ROW_TRACKING_ENABLED.fromMetaData(metadata) val suspendIdGeneration = DeltaConfigs.ROW_TRACKING_SUSPENDED.fromMetaData(metadata) if (isEnabled || !suspendIdGeneration) { throw new IllegalStateException( "Cannot run unbackfill when row tracking is enabled or not suspended.") } val filesToCommit = filesInBatch.map(_.copy( baseRowId = None, defaultRowCommitVersion = None, dataChange = false)) txn.commit(filesToCommit, DeltaOperations.RowTrackingUnBackfill(batchId)) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/backfill/RowTrackingUnBackfillCommand.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.backfill import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.{AddFile, Protocol, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.{Dataset, Row, SparkSession} import org.apache.spark.sql.catalyst.catalog.CatalogTable /** * This command cleans up tracking metadata from a delta table. In particular, it removes * `baseRowId` and `defaultRowCommitVersion`. This is achieved by re-commiting addFiles with * `dataChance = false`. This requires to commit the AddFiles of the entire table. * * Similarly to all backfilling operations, the relevant files are commited in multiple batches. * Each batch contains [[DeltaSQLConf.DELTA_BACKFILL_MAX_NUM_FILES_PER_COMMIT]]. This is to avoid * generating large commits in big tables. */ case class RowTrackingUnBackfillCommand( override val deltaLog: DeltaLog, override val nameOfTriggeringOperation: String, override val catalogTableOpt: Option[CatalogTable]) extends BackfillCommand { override def getBackfillExecutor( spark: SparkSession, deltaLog: DeltaLog, catalogTableOpt: Option[CatalogTable], backfillId: String, backfillStats: BackfillCommandStats): BackfillExecutor = { new RowTrackingUnBackfillExecutor(spark, deltaLog, catalogTableOpt, backfillId, backfillStats) } override def opType: String = "delta.rowTracking.unbackfill" } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/backfill/RowTrackingUnBackfillExecutor.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.backfill import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.{Dataset, SparkSession} import org.apache.spark.sql.catalyst.catalog.CatalogTable class RowTrackingUnBackfillExecutor( override val spark: SparkSession, override val deltaLog: DeltaLog, override val catalogTableOpt: Option[CatalogTable], override val backfillTxnId: String, override val backfillStats: BackfillCommandStats) extends BackfillExecutor { override val backFillBatchOpType = "delta.rowTracking.unbackfill.batch" override def filesToBackfill(snapshot: Snapshot): Dataset[AddFile] = { RowTrackingUnBackfillExecutor.getCandidateFilesToUnBackfill(snapshot) } override def constructBatch(files: Seq[AddFile]): BackfillBatch = RowTrackingUnBackfillBatch(files) } private[delta] object RowTrackingUnBackfillExecutor extends DeltaLogging { /** Returns the dataset with the list of candidate files to unbackfill. */ def getCandidateFilesToUnBackfill( snapshot: Snapshot): Dataset[AddFile] = { // Note: We can't use txn.filterFiles() because it drops the file statistics. snapshot .allFiles .filter(a => a.baseRowId.nonEmpty || a.defaultRowCommitVersion.nonEmpty) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReader.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.cdc import java.sql.Timestamp import scala.collection.mutable.{ListBuffer, Map => MutableMap} import scala.util.Try import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.deletionvectors.{RoaringBitmapArray, RoaringBitmapArrayFormat} import org.apache.spark.sql.delta.files.{CdcAddFileIndex, TahoeChangeFileIndex, TahoeFileIndexWithSnapshotDescriptor, TahoeRemoveFileIndex} import org.apache.spark.sql.delta.sources.DeltaDataSource import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore import org.apache.spark.sql.util.ScalaExtensions.OptionExt import org.apache.spark.rdd.RDD import org.apache.spark.sql.{Column, DataFrame, Row, SparkSession, SQLContext} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression, Literal} import org.apache.spark.sql.catalyst.plans.logical.Statistics import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes import org.apache.spark.sql.execution.LogicalRDD import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} import org.apache.spark.sql.sources.BaseRelation import org.apache.spark.sql.types.{LongType, StringType, StructType, TimestampType} import org.apache.spark.sql.util.CaseInsensitiveStringMap /** * The API that allows reading Change data between two versions of a table. * * The basic abstraction here is the CDC type column defined by [[CDCReader.CDC_TYPE_COLUMN_NAME]]. * When CDC is enabled, our writer will treat this column as a special partition column even though * it's not part of the table. Writers should generate a query that has two types of rows in it: * the main data in partition CDC_TYPE_NOT_CDC and the CDC data with the appropriate CDC type value. * * [[org.apache.spark.sql.delta.files.DelayedCommitProtocol]] * does special handling for this column, dispatching the main data to its normal location while the * CDC data is sent to [[AddCDCFile]] entries. */ object CDCReader extends CDCReaderImpl { // Definitions for the CDC type column. Delta writers will write data with a non-null value for // this column into [[AddCDCFile]] actions separate from the main table, and the CDC reader will // read this column to determine what type of change it was. val CDC_TYPE_COLUMN_NAME = "_change_type" // emitted from data val CDC_COMMIT_VERSION = "_commit_version" // inferred by reader val CDC_COMMIT_TIMESTAMP = "_commit_timestamp" // inferred by reader val CDC_TYPE_DELETE_STRING = "delete" val CDC_TYPE_DELETE = Literal(CDC_TYPE_DELETE_STRING) val CDC_TYPE_INSERT = "insert" val CDC_TYPE_UPDATE_PREIMAGE = "update_preimage" val CDC_TYPE_UPDATE_POSTIMAGE = "update_postimage" /** * Append CDC metadata columns to the provided schema. */ def cdcAttributes: Seq[Attribute] = Seq( AttributeReference(CDC_TYPE_COLUMN_NAME, StringType)(), AttributeReference(CDC_COMMIT_VERSION, LongType)(), AttributeReference(CDC_COMMIT_TIMESTAMP, TimestampType)()) // A special sentinel value indicating rows which are part of the main table rather than change // data. Delta writers will partition rows with this value away from the CDC data and // write them as normal to the main table. // Note that we specifically avoid using `null` here, because partition values of `null` are in // some scenarios mapped to a special string for Hive compatibility. val CDC_TYPE_NOT_CDC: Literal = Literal(null, StringType) // The virtual column name used for dividing CDC data from main table data. Delta writers should // permit this column through even though it's not part of the main table, and the // [[DelayedCommitProtocol]] will apply some special handling, ensuring there's only a // subfolder with __is_cdc = true and writing data with __is_cdc = false to the base location // as it would with CDC output off. // This is a bit redundant with CDC_TYPE_COL, but partitioning directly on the type would mean // that CDC of each type is partitioned away separately, exacerbating small file problems. val CDC_PARTITION_COL = "__is_cdc" // emitted by data // The top-level folder within the Delta table containing change data. This folder may contain // partitions within itself. val CDC_LOCATION = "_change_data" // CDC specific columns in data written by operations val CDC_COLUMNS_IN_DATA = Seq(CDC_PARTITION_COL, CDC_TYPE_COLUMN_NAME) /** * A special BaseRelation wrapper for CDF reads. */ case class DeltaCDFRelation( snapshotWithSchemaMode: SnapshotWithSchemaMode, sqlContext: SQLContext, catalogTableOpt: Option[CatalogTable], startingVersion: Option[Long], endingVersion: Option[Long]) extends DeltaCDFRelationBase( snapshotWithSchemaMode, sqlContext, catalogTableOpt, startingVersion, endingVersion) { override def buildScan(requiredColumns: Seq[Attribute], filters: Seq[Expression]): RDD[Row] = { val df = changesToBatchDF( deltaLog, startingVersion.get, // The actual ending version we should scan until during execution, as it might have changed endingVersion.getOrElse { deltaLog.update(catalogTableOpt = catalogTableOpt).version }, sqlContext.sparkSession, catalogTableOpt, readSchemaSnapshot = Some(snapshotForBatchSchema)) constructRDD(df, requiredColumns, filters) } } case class CDCDataSpec[T <: FileAction]( version: Long, timestamp: Timestamp, actions: Seq[T], commitInfo: Option[CommitInfo]) { def this( tableVersion: TableVersion, actions: Seq[T], commitInfo: Option[CommitInfo]) = { this( tableVersion.version, tableVersion.timestamp, actions, commitInfo) } } /** A version number of a Delta table, with the version's timestamp. */ case class TableVersion(version: Long, timestamp: Timestamp) { def this(wp: FilePathWithTableVersion) = this(wp.version, wp.timestamp) } /** Path of a file of a Delta table, together with it's origin table version & timestamp. */ case class FilePathWithTableVersion( path: String, commitInfo: Option[CommitInfo], version: Long, timestamp: Timestamp) } trait CDCReaderImpl extends CDCReaderBase { import org.apache.spark.sql.delta.commands.cdc.CDCReader._ /** * Function to check if file actions should be skipped for no-op merges based on * CommitInfo metrics. * MERGE will sometimes rewrite files in a way which *could* have changed data * (so dataChange = true) but did not actually do so (so no CDC will be produced). * In this case the correct CDC output is empty - we shouldn't serve it from * those files. This should be handled within the command, but as a hotfix-safe fix, we check * the metrics. If the command reported 0 rows inserted, updated, or deleted, then CDC * shouldn't be produced. */ def shouldSkipFileActionsInCommit(commitInfo: CommitInfo): Boolean = { val isMerge = commitInfo.operation == DeltaOperations.OP_MERGE val knownToHaveNoChangedRows = { val metrics = commitInfo.operationMetrics.getOrElse(Map.empty) // Note that if any metrics are missing, this condition will be false and we won't skip. // Unfortunately there are no predefined constants for these metric values. Seq("numTargetRowsInserted", "numTargetRowsUpdated", "numTargetRowsDeleted").forall { metrics.get(_).contains("0") } } isMerge && knownToHaveNoChangedRows } /** * For a sequence of changes(AddFile, RemoveFile, AddCDCFile) create a DataFrame that represents * that captured change data between start and end inclusive. * * Builds the DataFrame using the following logic: Per each change of type (Long, Seq[Action]) in * `changes`, iterates over the actions and handles two cases. * - If there are any CDC actions, then we ignore the AddFile and RemoveFile actions in that * version and create an AddCDCFile instead. * - If there are no CDC actions, then we must infer the CDC data from the AddFile and RemoveFile * actions, taking only those with `dataChange = true`. * * These buffers of AddFile, RemoveFile, and AddCDCFile actions are then used to create * corresponding FileIndexes (e.g. [[TahoeChangeFileIndex]]), where each is suited to use the * given action type to read CDC data. These FileIndexes are then unioned to produce the final * DataFrame. * * @param readSchemaSnapshot - Snapshot for the table for which we are creating a CDF * Dataframe, the schema of the snapshot is expected to be * the change DF's schema. We have already adjusted this * snapshot with the schema mode if there's any. We don't use * its data actually. * @param start - startingVersion of the changes * @param end - endingVersion of the changes * @param changes - changes is an iterator of all FileActions for a particular commit version. * Note that for log files where InCommitTimestamps are enabled, the iterator * must also contain the [[CommitInfo]] action. * @param spark - SparkSession * @param catalogTableOpt - The catalog table for the Delta table * @param isStreaming - indicates whether the DataFrame returned is a streaming DataFrame * @param useCoarseGrainedCDC - ignores checks related to CDC being disabled in any of the * versions and computes CDC entirely from AddFiles/RemoveFiles (ignoring * AddCDCFile actions) * @param startVersionSnapshot - The snapshot of the starting version. * @return CDCInfo which contains the DataFrame of the changes as well as the statistics * related to the changes */ // scalastyle:off argcount def changesToDF( readSchemaSnapshot: SnapshotDescriptor, start: Long, end: Long, changes: Iterator[(Long, Seq[Action])], spark: SparkSession, catalogTableOpt: Option[CatalogTable], isStreaming: Boolean = false, useCoarseGrainedCDC: Boolean = false, startVersionSnapshot: Option[SnapshotDescriptor] = None): CDCVersionDiffInfo = { // scalastyle:on argcount val deltaLog = readSchemaSnapshot.deltaLog if (end < start) { throw DeltaErrors.endBeforeStartVersionInCDC(start, end) } // A map from change version to associated file modification timestamps. // We only need these for non-InCommitTimestamp commits because for InCommitTimestamp commits, // the timestamps are already stored in the commit info. val nonICTTimestampsByVersion: Map[Long, Timestamp] = getNonICTTimestampsByVersion(deltaLog, start, end) val changeFiles = ListBuffer[CDCDataSpec[AddCDCFile]]() val addFiles = ListBuffer[CDCDataSpec[AddFile]]() val removeFiles = ListBuffer[CDCDataSpec[RemoveFile]]() val startVersionMetadata = startVersionSnapshot.map(_.metadata).getOrElse { deltaLog.getSnapshotAt(start, catalogTableOpt = catalogTableOpt).metadata } if (!useCoarseGrainedCDC && !isCDCEnabledOnTable(startVersionMetadata, spark)) { throw DeltaErrors.changeDataNotRecordedException(start, start, end) } val checkSchemaToBlockRead = shouldCheckSchemaToBlockBatchRead( spark, deltaLog, isStreaming ) var totalBytes = 0L var numAddFiles, numRemoveFiles, numAddCRCFiles = 0L changes.foreach { case (v, actions) => // Check whether CDC was newly disabled in this version. (We should have already checked // that it's enabled for the starting version, so checking this for each version // incrementally is sufficient to ensure that it's enabled for the entire range.) val cdcDisabled = actions.exists { case m: Metadata => !isCDCEnabledOnTable(m, spark) case _ => false } if (cdcDisabled && !useCoarseGrainedCDC) { throw DeltaErrors.changeDataNotRecordedException(v, start, end) } // Check all intermediary metadata schema changes, this guarantees that there will be no // read-incompatible schema changes across the querying range. // Note that we don't have to check the schema change if it's at the start version, because: // 1. If it's an initialization, e.g. CREATE AS SELECT, we don't have to consider this // as a schema change and report weird error messages. // 2. If it's indeed a schema change, as we won't be reading any data prior to it that // falls back to the previous (possibly incorrect) schema, we will be safe. Also if there // are any data file residing in the same commit, it will follow the new schema as well. if (v > start) { actions.collect { case a: Metadata => a }.foreach { metadata => // Verify with start snapshot to check for any read-incompatible changes // This also detects the corner case in that there's only one schema change between // start and end, which looks exactly like the end schema. checkBatchCdfReadSchemaIncompatibility( readSchemaSnapshot, start, end, checkSchemaToBlockRead, metadata, v, isSchemaChange = true) } } // Set up buffers for all action types to avoid multiple passes. val cdcActions = ListBuffer[AddCDCFile]() // Note that the CommitInfo is *not* guaranteed to be generated in 100% of cases. // We are using it only for a hotfix-safe mitigation/defense-in-depth - the value // extracted here cannot be relied on for correctness. var commitInfo: Option[CommitInfo] = None actions.foreach { case c: AddCDCFile => cdcActions.append(c) numAddCRCFiles += 1L totalBytes += c.size case a: AddFile => numAddFiles += 1L totalBytes += a.size case r: RemoveFile => numRemoveFiles += 1L totalBytes += r.size.getOrElse(0L) case i: CommitInfo => commitInfo = Some(i) case _ => // do nothing } // If the commit has an In-Commit Timestamp, we should use that as the commit timestamp. // Note that it is technically possible for a commit range to begin with ICT commits // followed by non-ICT commits, and end with ICT commits again. Ideally, for these commits // we should use the file modification time for the first two ranges. However, this // scenario is an edge case not worth optimizing for. val ts = commitInfo .flatMap(_.inCommitTimestamp) .map(ict => new Timestamp(ict)) .getOrElse(nonICTTimestampsByVersion.get(v).orNull) // When `isStreaming` = `true` the [CommitInfo] action is only used for passing the // in-commit timestamp to this method. We should filter them out. commitInfo = if (isStreaming) None else commitInfo // If there are CDC actions, we read them exclusively if we should not use the // Add and RemoveFiles. if (cdcActions.nonEmpty && !useCoarseGrainedCDC) { changeFiles.append(CDCDataSpec(v, ts, cdcActions.toSeq, commitInfo)) } else { val shouldSkipIndexedFile = commitInfo.exists(CDCReader.shouldSkipFileActionsInCommit) if (shouldSkipIndexedFile) { // This was introduced for a hotfix, so we're mirroring the existing logic as closely // as possible - it'd likely be safe to just return an empty dataframe here. addFiles.append(CDCDataSpec(v, ts, Nil, commitInfo)) removeFiles.append(CDCDataSpec(v, ts, Nil, commitInfo)) } else { // Otherwise, we take the AddFile and RemoveFile actions with dataChange = true and // infer CDC from them. val addActions = actions.collect { case a: AddFile if a.dataChange => a } val removeActions = actions.collect { case r: RemoveFile if r.dataChange => r } addFiles.append( CDCDataSpec( version = v, timestamp = ts, actions = addActions, commitInfo = commitInfo) ) removeFiles.append( CDCDataSpec( version = v, timestamp = ts, actions = removeActions, commitInfo = commitInfo) ) } } } // Verify the final read schema with the start snapshot version once again // This is needed to: // 1. Handle the case in that there are no read-incompatible schema change with the range, BUT // the latest schema may still be incompatible as it COULD be arbitrary. // 2. Similarly, handle the corner case when there are no read-incompatible schema change with // the range, BUT time-travel is used so the read schema could also be arbitrary. // It is sufficient to just verify with the start version schema because we have already // verified that all data being queried is read-compatible with start schema. checkBatchCdfReadSchemaIncompatibility( readSchemaSnapshot, start, end, checkSchemaToBlockRead, startVersionMetadata, start, isSchemaChange = false) val dfs = ListBuffer[DataFrame]() if (changeFiles.nonEmpty) { dfs.append(scanIndex( spark, new TahoeChangeFileIndex( spark, changeFiles.toSeq, deltaLog, deltaLog.dataPath, readSchemaSnapshot), isStreaming)) } val deletedAndAddedRows = getDeletedAndAddedRows( addFiles.toSeq, removeFiles.toSeq, deltaLog, readSchemaSnapshot, isStreaming, spark) dfs.append(deletedAndAddedRows: _*) val readSchema = cdcReadSchema(readSchemaSnapshot.metadata.schema) // build an empty DS. This DS retains the table schema and the isStreaming property // NOTE: We need to manually set the stats to 0 otherwise we will use default stats of INT_MAX, // which causes lots of optimizations to be applied wrong. val emptyRdd = LogicalRDD( toAttributes(readSchema), spark.sparkContext.emptyRDD[InternalRow], isStreaming = isStreaming )(spark.sqlContext.sparkSession, Some(Statistics(0, Some(0)))) val emptyDf = DataFrameUtils.ofRows(spark.sqlContext.sparkSession, emptyRdd) recordDeltaEvent( deltaLog, "delta.changeDataFeed.changesToDF", data = Map( "startVersion" -> start, "endVersion" -> end, "useCoarseGrainedCDC" -> useCoarseGrainedCDC, "numAddFiles" -> numAddFiles, "numRemoveFiles" -> numRemoveFiles, "numAddCRCFiles" -> numAddCRCFiles, "totalBytes" -> totalBytes, "isStreaming" -> isStreaming ) ) val totalFiles = numAddFiles + numRemoveFiles + numAddCRCFiles CDCVersionDiffInfo( (emptyDf +: dfs).reduce((df1, df2) => df1.union( df2 )), totalFiles, totalBytes) } /** * Generate CDC rows by looking at added and removed files, together with Deletion Vectors they * may have. * * When DV is used, the same file can be removed then added in the same version, and the only * difference is the assigned DVs. The base method does not consider DVs in this case, thus will * produce CDC that *all* rows in file being removed then *some* re-added. The correct answer, * however, is to compare two DVs and apply the diff to the file to get removed and re-added rows. * * Currently it is always the case that in the log "remove" comes first, followed by "add" -- * which means that the file stays alive with a new DV. There's another possibility, though not * make many senses, that a file is "added" to log then "removed" in the same version. If this * becomes possible in future, we have to reconstruct the timeline considering the order of * actions rather than simply matching files by path. */ protected def getDeletedAndAddedRows( addFileSpecs: Seq[CDCDataSpec[AddFile]], removeFileSpecs: Seq[CDCDataSpec[RemoveFile]], deltaLog: DeltaLog, snapshot: SnapshotDescriptor, isStreaming: Boolean, spark: SparkSession): Seq[DataFrame] = { // Transform inputs to maps indexed by version and path and map each version to a CommitInfo // object. val versionToCommitInfo = MutableMap.empty[Long, CommitInfo] val addFilesMap = addFileSpecs.flatMap { spec => spec.commitInfo.ifDefined { ci => versionToCommitInfo(spec.version) = ci } spec.actions.map { action => val key = FilePathWithTableVersion(action.path, spec.commitInfo, spec.version, spec.timestamp) key -> action } }.toMap val removeFilesMap = removeFileSpecs.flatMap { spec => spec.commitInfo.ifDefined { ci => versionToCommitInfo(spec.version) = ci } spec.actions.map { action => val key = FilePathWithTableVersion(action.path, spec.commitInfo, spec.version, spec.timestamp) key -> action } }.toMap val finalAddFiles = MutableMap[TableVersion, ListBuffer[AddFile]]() val finalRemoveFiles = MutableMap[TableVersion, ListBuffer[RemoveFile]]() // If a path is only being added, then scan it normally as inserted rows (addFilesMap.keySet -- removeFilesMap.keySet).foreach { addKey => finalAddFiles .getOrElseUpdate(new TableVersion(addKey), ListBuffer()) .append(addFilesMap(addKey)) } // If a path is only being removed, then scan it normally as removed rows (removeFilesMap.keySet -- addFilesMap.keySet).foreach { removeKey => finalRemoveFiles .getOrElseUpdate(new TableVersion(removeKey), ListBuffer()) .append(removeFilesMap(removeKey)) } // Convert maps back into Seq[CDCDataSpec] and feed it into a single scan. This will greatly // reduce the number of tasks. val finalAddFilesSpecs = buildCDCDataSpecSeq(finalAddFiles, versionToCommitInfo) val finalRemoveFilesSpecs = buildCDCDataSpecSeq(finalRemoveFiles, versionToCommitInfo) val dfAddsAndRemoves = ListBuffer[DataFrame]() if (finalAddFilesSpecs.nonEmpty) { dfAddsAndRemoves.append( scanIndex( spark, new CdcAddFileIndex(spark, finalAddFilesSpecs, deltaLog, deltaLog.dataPath, snapshot), isStreaming)) } if (finalRemoveFilesSpecs.nonEmpty) { dfAddsAndRemoves.append( scanIndex( spark, new TahoeRemoveFileIndex( spark, finalRemoveFilesSpecs, deltaLog, deltaLog.dataPath, snapshot), isStreaming)) } val dfGeneratedDvScanActions = processDeletionVectorActions( addFilesMap, removeFilesMap, versionToCommitInfo.toMap, deltaLog, snapshot, isStreaming, spark) dfAddsAndRemoves.toSeq ++ dfGeneratedDvScanActions } def processDeletionVectorActions( addFilesMap: Map[FilePathWithTableVersion, AddFile], removeFilesMap: Map[FilePathWithTableVersion, RemoveFile], versionToCommitInfo: Map[Long, CommitInfo], deltaLog: DeltaLog, snapshot: SnapshotDescriptor, isStreaming: Boolean, spark: SparkSession): Seq[DataFrame] = { val finalReplaceAddFiles = MutableMap[TableVersion, ListBuffer[AddFile]]() val finalReplaceRemoveFiles = MutableMap[TableVersion, ListBuffer[RemoveFile]]() val dvStore = DeletionVectorStore.createInstance(deltaLog.newDeltaHadoopConf()) (addFilesMap.keySet intersect removeFilesMap.keySet).foreach { key => val add = addFilesMap(key) val remove = removeFilesMap(key) val generatedActions = generateFileActionsWithInlineDv(add, remove, dvStore, deltaLog) generatedActions.foreach { case action: AddFile => finalReplaceAddFiles .getOrElseUpdate(new TableVersion(key), ListBuffer()) .append(action) case action: RemoveFile => finalReplaceRemoveFiles .getOrElseUpdate(new TableVersion(key), ListBuffer()) .append(action) case _ => throw new Exception("Expecting AddFile or RemoveFile.") } } // We have to build one scan for each version because DVs attached to actions will be // broadcasted in [[ScanWithDeletionVectors.createBroadcastDVMap]] which is not version-aware. // Here, one file can have different row index filters in different versions. val dfs = ListBuffer[DataFrame]() // Scan for masked rows as change_type = "insert", // see explanation in [[generateFileActionsWithInlineDv]]. finalReplaceAddFiles.foreach { case (tableVersion, addFiles) => val commitInfo = versionToCommitInfo.get(tableVersion.version) dfs.append( scanIndex( spark, new CdcAddFileIndex( spark, Seq(new CDCDataSpec(tableVersion, addFiles.toSeq, commitInfo)), deltaLog, deltaLog.dataPath, snapshot, rowIndexFilters = Some(fileActionsToIfNotContainedRowIndexFilters(addFiles.toSeq))), isStreaming)) } // Scan for masked rows as change_type = "delete", // see explanation in [[generateFileActionsWithInlineDv]]. finalReplaceRemoveFiles.foreach { case (tableVersion, removeFiles) => val commitInfo = versionToCommitInfo.get(tableVersion.version) dfs.append( scanIndex( spark, new TahoeRemoveFileIndex( spark, Seq(new CDCDataSpec(tableVersion, removeFiles.toSeq, commitInfo)), deltaLog, deltaLog.dataPath, snapshot, rowIndexFilters = Some(fileActionsToIfNotContainedRowIndexFilters(removeFiles.toSeq))), isStreaming)) } dfs.toSeq } /** * Get the block of change data from start to end Delta log versions (both sides inclusive). * The returned DataFrame has isStreaming set to false. * * @param readSchemaSnapshot The snapshot with the desired schema that will be used to * serve this CDF batch. It is usually passed upstream from * e.g. DeltaTableV2 as an effort to stablize the schema used for the * batch DF. We don't actually use its data. * If not set, it will fallback to the legacy behavior of using * whatever deltaLog.unsafeVolatileSnapshot is. This should be * avoided in production. */ def changesToBatchDF( deltaLog: DeltaLog, start: Long, end: Long, spark: SparkSession, catalogTableOpt: Option[CatalogTable] = None, readSchemaSnapshot: Option[Snapshot] = None, useCoarseGrainedCDC: Boolean = false, startVersionSnapshot: Option[SnapshotDescriptor] = None): DataFrame = { val changesWithinRange = deltaLog.getChanges( start, end, catalogTableOpt, failOnDataLoss = false) changesToDF( readSchemaSnapshot.getOrElse(deltaLog.unsafeVolatileSnapshot), start, end, changesWithinRange, spark, catalogTableOpt, isStreaming = false, useCoarseGrainedCDC = useCoarseGrainedCDC, startVersionSnapshot = startVersionSnapshot) .fileChangeDf } /** * Build a dataframe from the specified file index. We can't use a DataFrame scan directly on the * file names because that scan wouldn't include partition columns. * * It can optionally take a customReadSchema for the dataframe generated. */ protected def scanIndex( spark: SparkSession, index: TahoeFileIndexWithSnapshotDescriptor, isStreaming: Boolean = false): DataFrame = { val relation = HadoopFsRelation( location = index, partitionSchema = index.partitionSchema, dataSchema = cdcReadSchema(index.schema), bucketSpec = None, new DeltaParquetFileFormat(index.protocol, index.metadata, isCDCRead = true), options = index.deltaLog.options)(spark) val plan = LogicalRelation(relation, isStreaming = isStreaming) DataFrameUtils.ofRows(spark, plan) } /** * Based on the read options passed it indicates whether the read was a cdc read or not. */ def isCDCRead(options: CaseInsensitiveStringMap): Boolean = { // Consistent with DeltaOptions.readChangeFeed, // but CDCReader use CaseInsensitiveStringMap vs. CaseInsensitiveMap used by DataFrameReader. def toBoolean(input: String, name: String): Boolean = { Try(input.toBoolean).toOption.getOrElse { throw DeltaErrors.illegalDeltaOptionException(name, input, "must be 'true' or 'false'") } } val cdcEnabled = options.containsKey(DeltaDataSource.CDC_ENABLED_KEY) && toBoolean(options.get(DeltaDataSource.CDC_ENABLED_KEY), DeltaDataSource.CDC_ENABLED_KEY) val cdcLegacyConfEnabled = options.containsKey(DeltaDataSource.CDC_ENABLED_KEY_LEGACY) && toBoolean( options.get(DeltaDataSource.CDC_ENABLED_KEY_LEGACY), DeltaDataSource.CDC_ENABLED_KEY_LEGACY) cdcEnabled || cdcLegacyConfEnabled } /** * Determine if the metadata provided has cdc enabled or not. */ def isCDCEnabledOnTable(metadata: Metadata, spark: SparkSession): Boolean = { ChangeDataFeedTableFeature.metadataRequiresFeatureToBeEnabled( protocol = Protocol(), metadata, spark) } /** * Check metadata changes in CDC enabled tables. * * - Check that CDC is not enabled in a table that contains columns reserved by CDC. * - Check that columns reserved by CDC are not added to a table with CDC enabled. */ def checkMetadataChange( spark: SparkSession, newMetadata: Metadata, oldMetadata: Metadata): Unit = { if (!isCDCEnabledOnTable(newMetadata, spark)) { return } val newSchema = newMetadata.schema.fieldNames val reservedColumnsUsed = CDCReader.cdcReadSchema(new StructType()).fieldNames.intersect(newSchema) if (reservedColumnsUsed.length > 0) { if (!isCDCEnabledOnTable(oldMetadata, spark)) { // cdc was not enabled previously but reserved columns are present in the new schema. throw DeltaErrors.tableAlreadyContainsCDCColumns(reservedColumnsUsed) } else { // cdc was enabled but reserved columns are present in the new metadata. throw DeltaErrors.cdcColumnsInData(reservedColumnsUsed) } } } /** * Given `add` and `remove` actions of the same file, manipulate DVs to get rows that are deleted * and re-added from `add` to `remove`. * * @return One or more [[AddFile]] and [[RemoveFile]], corresponding to CDC change_type "insert" * and "delete". Rows masked by inline DVs are changed rows. */ private def generateFileActionsWithInlineDv( add: AddFile, remove: RemoveFile, dvStore: DeletionVectorStore, deltaLog: DeltaLog): Seq[FileAction] = { val removeDvOpt = Option(remove.deletionVector) val addDvOpt = Option(add.deletionVector) val newActions = ListBuffer[FileAction]() // Four cases: // 1) Remove without DV, add without DV: // Not possible. This case has been handled before. // 2) Remove without DV, add with DV1: // Rows masked by DV1 are deleted. // 3) Remove with DV1, add without DV: // Rows masked by DV1 are added. May happen when restoring a table. // 4) Remove with DV1, add with DV2: // a) Rows masked by DV2 but not DV1 are deleted. // b) Rows masked by DV1 but not DV2 are re-added. May happen when restoring a table. (removeDvOpt, addDvOpt) match { case (None, None) => throw new Exception("Expecting one or both of add and remove contain DV.") case (None, Some(addDv)) => newActions += remove.copy(deletionVector = addDv) case (Some(removeDv), None) => newActions += add.copy(deletionVector = removeDv) case (Some(removeDv), Some(addDv)) => val removeBitmap = dvStore.read(removeDv, deltaLog.dataPath) val addBitmap = dvStore.read(addDv, deltaLog.dataPath) // Case 4a val finalRemovedRowsBitmap = getDeletionVectorsDiff(addBitmap, removeBitmap) // Case 4b val finalReAddedRowsBitmap = getDeletionVectorsDiff(removeBitmap, addBitmap) val finalRemovedRowsDv = DeletionVectorDescriptor.inlineInLog( DeletionVectorUtils.serialize( finalRemovedRowsBitmap, RoaringBitmapArrayFormat.Portable, Some(deltaLog.dataPath)), finalRemovedRowsBitmap.cardinality) val finalReAddedRowsDv = DeletionVectorDescriptor.inlineInLog( DeletionVectorUtils.serialize( finalReAddedRowsBitmap, RoaringBitmapArrayFormat.Portable, Some(deltaLog.dataPath)), finalReAddedRowsBitmap.cardinality) newActions += remove.copy(deletionVector = finalRemovedRowsDv) newActions += add.copy(deletionVector = finalReAddedRowsDv) } newActions.toSeq } /** * Return a map of file paths to IfNotContained row index filters, to keep only the marked rows. */ private def fileActionsToIfNotContainedRowIndexFilters( actions: Seq[FileAction]): Map[String, RowIndexFilterType] = { actions.map(f => f.path -> RowIndexFilterType.IF_NOT_CONTAINED).toMap } /** * Get a new [[RoaringBitmapArray]] copy storing values that are in `left` but not in `right`. */ private def getDeletionVectorsDiff( left: RoaringBitmapArray, right: RoaringBitmapArray): RoaringBitmapArray = { val leftCopy = left.copy() leftCopy.diff(right) leftCopy } private def buildCDCDataSpecSeq[T <: FileAction]( actionsByVersion: MutableMap[TableVersion, ListBuffer[T]], versionToCommitInfo: MutableMap[Long, CommitInfo] ): Seq[CDCDataSpec[T]] = actionsByVersion.map { case (fileVersion, addFiles) => val commitInfo = versionToCommitInfo.get(fileVersion.version) new CDCDataSpec(fileVersion, addFiles.toSeq, commitInfo) }.toSeq override def getConstructedCDCRelation( snapshotWithSchema: SnapshotWithSchemaMode, sqlContext: SQLContext, catalogTableOpt: Option[CatalogTable], startingVersion: Option[Long], endingVersion: Option[Long]): BaseRelation = { DeltaCDFRelation( snapshotWithSchema, sqlContext, catalogTableOpt, startingVersion, endingVersion ) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/cdc/CDCReaderBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.cdc import java.sql.Timestamp import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.commands.cdc.CDCReader.{cdcReadSchema, DeltaCDFRelation} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.{DeltaDataSource, DeltaSource, DeltaSQLConf} import org.apache.spark.internal.MDC import org.apache.spark.rdd.RDD import org.apache.spark.sql.{Column, DataFrame, Row, SparkSession, SQLContext} import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.{And, Attribute, Expression, Literal} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.sources.{BaseRelation, CatalystScan, Filter} import org.apache.spark.sql.types.{LongType, StringType, StructType, TimestampType} import org.apache.spark.sql.util.CaseInsensitiveStringMap /** * Represents a Delta log version, and how the version is determined. * @param version the determined version. * @param timestamp the commit timestamp of the determined version. Will be filled when the * version is determined by timestamp. */ case class ResolvedCDFVersion(version: Long, timestamp: Option[Timestamp]) { /** Whether this version is resolved by timestamp. */ def resolvedByTimestamp: Boolean = timestamp.isDefined } // A snapshot coupled with a schema mode that user specified case class SnapshotWithSchemaMode(snapshot: Snapshot, schemaMode: DeltaBatchCDFSchemaMode) /** * A special BaseRelation wrapper for CDF reads. */ abstract class DeltaCDFRelationBase( snapshotWithSchemaMode: SnapshotWithSchemaMode, sqlContext: SQLContext, catalogTableOpt: Option[CatalogTable], startingVersion: Option[Long], endingVersion: Option[Long]) extends BaseRelation with CatalystScan { protected val deltaLog: DeltaLog = snapshotWithSchemaMode.snapshot.deltaLog protected lazy val latestVersionOfTableDuringAnalysis: Long = deltaLog.update(catalogTableOpt = catalogTableOpt).version /** * There may be a slight divergence here in terms of what schema is in the latest data vs what * schema we have captured during analysis, but this is an inherent limitation of Spark. * * However, if there are schema changes between analysis and execution, since we froze this * schema, our schema incompatibility checks will kick in during the scan so we will always * be safe - Although it is a notable caveat that user should be aware of because the CDC query * may break. */ protected lazy val endingVersionForBatchSchema: Long = endingVersion.map { v => // As defined in the method doc, if ending version is greater than the latest version, we will // just use the latest version to find the schema. latestVersionOfTableDuringAnalysis min v }.getOrElse { // Or if endingVersion is not specified, we just use the latest schema. latestVersionOfTableDuringAnalysis } // The final snapshot whose schema is going to be used as this CDF relation's schema protected val snapshotForBatchSchema: Snapshot = snapshotWithSchemaMode.schemaMode match { case BatchCDFSchemaEndVersion => // Fetch the ending version and its schema deltaLog.getSnapshotAt(endingVersionForBatchSchema, catalogTableOpt = catalogTableOpt) case _ => // Apply the default, either latest generated by DeltaTableV2 or specified by Time-travel // options. snapshotWithSchemaMode.snapshot } override val schema: StructType = { cdcReadSchema( DeltaTableUtils.removeInternalDeltaMetadata( sqlContext.sparkSession, DeltaTableUtils.removeInternalWriterMetadata( sqlContext.sparkSession, snapshotForBatchSchema.metadata.schema ) ) ) } override def unhandledFilters(filters: Array[Filter]): Array[Filter] = Array.empty protected def constructRDD( df: DataFrame, requiredColumns: Seq[Attribute], filters: Seq[Expression]): RDD[Row] = { // Rewrite the attributes in the required columns and // pushed down filters to match the output of the internal DataFrame. val outputMap = df.queryExecution.analyzed.output.map(a => a.name -> a).toMap val projections = requiredColumns.map(a => Column(outputMap(a.name))) val filter = Column( filters .map(_.transform { case a: Attribute => outputMap(a.name) }) .reduceOption(And) .getOrElse(Literal.TrueLiteral) ) df.filter(filter).select(projections: _*).rdd } } /** * Base trait for CDC readers that contains common functionality * shared across different CDC reader implementations. */ trait CDCReaderBase extends DeltaLogging { /** * Given timestamp or version, this method returns the corresponding version for that timestamp * or the version itself, as well as how the return version is obtained: by `version` or * `timestamp`. */ private def getVersionForCDC( spark: SparkSession, deltaLog: DeltaLog, catalogTableOpt: Option[CatalogTable], conf: SQLConf, options: CaseInsensitiveStringMap, versionKey: String, timestampKey: String): Option[ResolvedCDFVersion] = { if (options.containsKey(versionKey)) { val version = options.get(versionKey) try { Some(ResolvedCDFVersion(version.toLong, timestamp = None)) } catch { case _: NumberFormatException => throw DeltaErrors.versionInvalid(version) } } else if (options.containsKey(timestampKey)) { val ts = options.get(timestampKey) val spec = DeltaTimeTravelSpec(Some(Literal(ts)), None, Some("cdcReader")) val timestamp = spec.getTimestamp(spark.sessionState.conf) val allowOutOfRange = conf.getConf(DeltaSQLConf.DELTA_CDF_ALLOW_OUT_OF_RANGE_TIMESTAMP) val resolvedVersion = if (timestampKey == DeltaDataSource.CDC_START_TIMESTAMP_KEY) { // For the starting timestamp we need to find a version after the provided timestamp // we can use the same semantics as streaming. DeltaSource.getStartingVersionFromTimestamp( spark, deltaLog, catalogTableOpt, timestamp, allowOutOfRange) } else { // For ending timestamp the version should be before the provided timestamp. DeltaTableUtils.resolveTimeTravelVersion( conf, deltaLog, catalogTableOpt, spec, allowOutOfRange)._1 } Some(ResolvedCDFVersion(resolvedVersion, Some(timestamp))) } else { None } } /** * Get the batch cdf schema mode for a table, considering whether it has column mapping enabled * or not. */ def getBatchSchemaModeForTable( spark: SparkSession, columnMappingEnabled: Boolean): DeltaBatchCDFSchemaMode = { if (columnMappingEnabled) { // Tables with column-mapping enabled can specify which schema version to use with this // config. DeltaBatchCDFSchemaMode(spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_CDF_DEFAULT_SCHEMA_MODE_FOR_COLUMN_MAPPING_TABLE)) } else { // Non column-mapping table uses the current default, which is typically `legacy` - usually // the latest schema is used, but it can depend on time-travel arguments as well. BatchCDFSchemaLegacy } } /** * Get a Relation that represents change data between two snapshots of the table. * * @param spark Spark session * @param snapshotToUse Snapshot to use to provide read schema and version * @param isTimeTravelQuery Whether this CDC scan is used in conjunction with time-travel args * @param conf SQL conf * @param options CDC specific options */ def getCDCRelation( spark: SparkSession, snapshotToUse: Snapshot, catalogTableOpt: Option[CatalogTable], isTimeTravelQuery: Boolean, conf: SQLConf, options: CaseInsensitiveStringMap): BaseRelation = { val startingVersion = getVersionForCDC( spark, snapshotToUse.deltaLog, catalogTableOpt, conf, options, DeltaDataSource.CDC_START_VERSION_KEY, DeltaDataSource.CDC_START_TIMESTAMP_KEY).getOrElse { throw DeltaErrors.noStartVersionForCDC() } val endingVersionOpt = getVersionForCDC( spark, snapshotToUse.deltaLog, catalogTableOpt, conf, options, DeltaDataSource.CDC_END_VERSION_KEY, DeltaDataSource.CDC_END_TIMESTAMP_KEY ) verifyStartingVersion(spark, snapshotToUse, catalogTableOpt, conf, startingVersion) match { case Some(toReturn) => return toReturn case None => } verifyEndingVersion( spark, snapshotToUse, catalogTableOpt, startingVersion, endingVersionOpt) match { case Some(toReturn) => return toReturn case None => } logInfo( log"startingVersion: ${MDC(DeltaLogKeys.START_VERSION, startingVersion.version)}, " + log"endingVersion: ${MDC(DeltaLogKeys.END_VERSION, endingVersionOpt.map(_.version))}") val startingSnapshot = snapshotToUse.deltaLog.getSnapshotAt( startingVersion.version, catalogTableOpt = catalogTableOpt, enforceTimeTravelWithinDeletedFileRetention = true) val columnMappingEnabledAtStartingVersion = startingSnapshot.metadata.columnMappingMode != NoMapping val columnMappingEnabledAtEndVersion = endingVersionOpt.exists { endingVersion => // End version could be after the snapshot to use version, in which case it might not exist. if (endingVersion.version > snapshotToUse.version) { false } else { val endingSnapshot = snapshotToUse.deltaLog.getSnapshotAt(endingVersion.version, catalogTableOpt = catalogTableOpt) endingSnapshot.metadata.columnMappingMode != NoMapping && endingVersion.version <= snapshotToUse.version } } val columnMappingEnabledAtSnapshotToUseVersion = snapshotToUse.metadata.columnMappingMode != NoMapping // Special handling for tables with column mapping mode enabled in any of the versions. val columnMappingEnabled = columnMappingEnabledAtSnapshotToUseVersion || columnMappingEnabledAtEndVersion || columnMappingEnabledAtStartingVersion val schemaMode = getBatchSchemaModeForTable(spark, columnMappingEnabled = columnMappingEnabled) // Non-legacy schema mode options cannot be used with time-travel because the schema to use // will be confusing. if (isTimeTravelQuery && schemaMode != BatchCDFSchemaLegacy) { throw DeltaErrors.illegalDeltaOptionException( DeltaSQLConf.DELTA_CDF_DEFAULT_SCHEMA_MODE_FOR_COLUMN_MAPPING_TABLE.key, schemaMode.name, s"${DeltaSQLConf.DELTA_CDF_DEFAULT_SCHEMA_MODE_FOR_COLUMN_MAPPING_TABLE.key} " + s"cannot be used with time travel options.") } getConstructedCDCRelation( SnapshotWithSchemaMode(snapshotToUse, schemaMode), spark.sqlContext, catalogTableOpt, Some(startingVersion.version), endingVersionOpt.map(_.version) ) } private def verifyStartingVersion( spark: SparkSession, snapshotToUse: Snapshot, catalogTableOpt: Option[CatalogTable], conf: SQLConf, startingVersion: ResolvedCDFVersion): Option[BaseRelation] = { // add a version check here that is cheap instead of after trying to list a large version // that doesn't exist if (startingVersion.version > snapshotToUse.version) { val allowOutOfRange = conf.getConf(DeltaSQLConf.DELTA_CDF_ALLOW_OUT_OF_RANGE_TIMESTAMP) if (allowOutOfRange) { return Some(emptyCDFRelation(spark, snapshotToUse, catalogTableOpt, BatchCDFSchemaLegacy)) } throw DeltaErrors.startVersionAfterLatestVersion( startingVersion.version, snapshotToUse.version) } None } private def verifyEndingVersion( spark: SparkSession, snapshotToUse: Snapshot, catalogTableOpt: Option[CatalogTable], startingVersion: ResolvedCDFVersion, endingVersionOpt: Option[ResolvedCDFVersion]): Option[BaseRelation] = { // Given two timestamps, there is a case when both of them lay closely between two versions: // version: 4 5 // ---------|-------------------------------------------------|-------- // ^ start timestamp ^ end timestamp // In this case the starting version will be 5 and ending version will be 4. We must not // throw `endBeforeStartVersionInCDC` but return empty result. endingVersionOpt.foreach { endingVersion => if (startingVersion.resolvedByTimestamp && endingVersion.resolvedByTimestamp) { // The next `if` is true when end is less than start but no commit is in between. // We need to capture such a case and throw early. if (startingVersion.timestamp.get.after(endingVersion.timestamp.get)) { throw DeltaErrors.endBeforeStartVersionInCDC( startingVersion.version, endingVersion.version) } if (endingVersion.version == startingVersion.version - 1) { return Some(emptyCDFRelation(spark, snapshotToUse, catalogTableOpt, BatchCDFSchemaLegacy)) } } if (endingVersionOpt.exists(_.version < startingVersion.version)) { throw DeltaErrors.endBeforeStartVersionInCDC( startingVersion.version, endingVersionOpt.get.version) } } None } private def emptyCDFRelation( spark: SparkSession, snapshot: Snapshot, catalogTableOpt: Option[CatalogTable], schemaMode: DeltaBatchCDFSchemaMode) = { new DeltaCDFRelation( SnapshotWithSchemaMode(snapshot, schemaMode), spark.sqlContext, catalogTableOpt, startingVersion = None, endingVersion = None) { override def buildScan(requiredColumns: Seq[Attribute], filters: Seq[Expression]): RDD[Row] = sqlContext.sparkSession.sparkContext.emptyRDD[Row] } } /** * Append CDC metadata columns to the provided schema. */ def cdcReadSchema(deltaSchema: StructType): StructType = { deltaSchema .add(CDCReader.CDC_TYPE_COLUMN_NAME, StringType) .add(CDCReader.CDC_COMMIT_VERSION, LongType) .add(CDCReader.CDC_COMMIT_TIMESTAMP, TimestampType) } /** * Check metadata (which may contain schema change)'s read compatibility with read schema. */ protected def checkBatchCdfReadSchemaIncompatibility( readSchemaSnapshot: SnapshotDescriptor, start: Long, end: Long, shouldCheckSchemaToBlockBatchRead: Boolean, metadata: Metadata, metadataVer: Long, isSchemaChange: Boolean): Unit = { // We do not check for any incompatibility if the global "I don't care" flag is turned on if (shouldCheckSchemaToBlockBatchRead) { // Column mapping incompatibilities val compatible = { // For column mapping schema change, the order matters because we don't want to treat // an ADD COLUMN as an inverse DROP COLUMN. if (metadataVer <= readSchemaSnapshot.version) { DeltaColumnMapping.hasNoColumnMappingSchemaChanges( newMetadata = readSchemaSnapshot.metadata, oldMetadata = metadata) } else { DeltaColumnMapping.hasNoColumnMappingSchemaChanges( newMetadata = metadata, oldMetadata = readSchemaSnapshot.metadata) } } && { // Other standard read incompatibilities if (metadataVer <= readSchemaSnapshot.version) { // If the metadata is before the read schema version, make sure: // a) metadata schema is a part of the read schema, i.e. only ADD COLUMN can evolve // metadata schema into read schema // b) data type for common fields remain the same // c) metadata schema should not contain field that is nullable=true but the read schema // is nullable=false. SchemaUtils.isReadCompatible( existingSchema = metadata.schema, readSchema = readSchemaSnapshot.schema, forbidTightenNullability = true) } else { // If the metadata is POST the read schema version, which can happen due to time-travel // or simply a divergence between analyzed version and the actual latest // version during scan, we will make sure the other way around: // a) the metadata must be a super set of the read schema, i.e. only ADD COLUMN can // evolve read schema into metadata schema // b) data type for common fields remain the same // c) read schema should not contain field that is nullable=false but the metadata // schema has nullable=true. SchemaUtils.isReadCompatible( existingSchema = readSchemaSnapshot.schema, readSchema = metadata.schema, forbidTightenNullability = false) } } if (!compatible) { throw DeltaErrors.blockBatchCdfReadWithIncompatibleSchemaChange( start, end, // The consistent read schema readSchemaSnapshot.metadata.schema, readSchemaSnapshot.version, // The conflicting schema or schema change version metadataVer, isSchemaChange ) } } } def shouldCheckSchemaToBlockBatchRead( spark: SparkSession, deltaLog: DeltaLog, isStreaming: Boolean): Boolean = { // Check schema read-compatibility val allowUnsafeBatchReadOnIncompatibleSchemaChanges = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_CDF_UNSAFE_BATCH_READ_ON_INCOMPATIBLE_SCHEMA_CHANGES) if (allowUnsafeBatchReadOnIncompatibleSchemaChanges) { recordDeltaEvent(deltaLog, "delta.unsafe.cdf.readOnColumnMappingSchemaChanges") } !isStreaming && !allowUnsafeBatchReadOnIncompatibleSchemaChanges } def getConstructedCDCRelation( snapshotWithSchemaMode: SnapshotWithSchemaMode, sqlContext: SQLContext, catalogTableOpt: Option[CatalogTable], startingVersion: Option[Long], endingVersion: Option[Long]): BaseRelation /** * Builds a map from commit versions to associated commit timestamps where the timestamp * is the modification time of the commit file. Note that this function will not return * InCommitTimestamps, it is up to the consumer of this function to decide whether the * file modification time is the correct commit timestamp or whether they need to read the ICT. * * @param start start commit version * @param end end commit version (inclusive) */ def getNonICTTimestampsByVersion( deltaLog: DeltaLog, start: Long, end: Long): Map[Long, Timestamp] = { // Correct timestamp values are only available through DeltaHistoryManager.getCommits(). Commit // info timestamps are wrong, and file modification times are wrong because they need to be // monotonized first. This just performs a list (we don't read the contents of the files in // getCommits()) so the performance overhead is minimal. val monotonizationStart = math.max(start - DeltaHistoryManager.POTENTIALLY_UNMONOTONIZED_TIMESTAMPS, 0) val commits = DeltaHistoryManager.getCommitsWithNonIctTimestamps( deltaLog.store, deltaLog.logPath, monotonizationStart, Some(end + 1), deltaLog.newDeltaHadoopConf()) // Note that the timestamps come from filesystem modification timestamps, so they're // milliseconds since epoch and we don't need to deal with timezones. commits.map(f => (f.version -> new Timestamp(f.timestamp))).toMap } /** * Represents the changes between some start and end version of a Delta table * @param fileChangeDf contains all of the file changes (AddFile, RemoveFile, AddCDCFile) * @param numFiles the number of AddFile + RemoveFile + AddCDCFiles that are in the df * @param numBytes the total size of the AddFile + RemoveFile + AddCDCFiles that are in the df */ case class CDCVersionDiffInfo(fileChangeDf: DataFrame, numFiles: Long, numBytes: Long) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/columnmapping/RemoveColumnMappingCommand.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.columnmapping import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.schema.{ImplicitMetadataOperation, SchemaUtils} import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.types.StructType /** * A command to remove the column mapping from a table. */ class RemoveColumnMappingCommand( val deltaLog: DeltaLog, val catalogOpt: Option[CatalogTable]) extends ImplicitMetadataOperation { override protected val canMergeSchema: Boolean = false override protected val canOverwriteSchema: Boolean = true /** * Remove the column mapping from the table. * @param removeColumnMappingTableProperty - whether to remove the column mapping property from * the table instead of setting it to 'none' */ def run(spark: SparkSession, removeColumnMappingTableProperty: Boolean): Unit = { deltaLog.withNewTransaction(catalogOpt) { txn => val originalFiles = txn.filterFiles() val originalData = buildDataFrame(txn, originalFiles) val originalSchema = txn.snapshot.schema val newSchema = DeltaColumnMapping.dropColumnMappingMetadata(originalSchema) verifySchemaFieldNames(newSchema) updateMetadata(removeColumnMappingTableProperty, txn, newSchema) val deltaOptions = getDeltaOptionsForWrite(spark) val addedFiles = writeData(txn, originalData, deltaOptions) val removeFileActions = originalFiles.map(_.removeWithTimestamp(dataChange = false)) txn.commit(actions = removeFileActions ++ addedFiles, op = DeltaOperations.RemoveColumnMapping(), tags = RowTracking.addPreservedRowTrackingTagIfNotSet(txn.snapshot) ) } } /** * Verify none of the schema fields contain invalid column names. */ def verifySchemaFieldNames(schema: StructType): Unit = { val invalidColumnNames = SchemaUtils.findInvalidColumnNamesInSchema(schema) if (invalidColumnNames.nonEmpty) { throw DeltaErrors .foundInvalidColumnNamesWhenRemovingColumnMapping(invalidColumnNames) } } /** * Update the metadata to remove the column mapping table properties and * update the schema to remove the column mapping metadata. */ def updateMetadata( removeColumnMappingTableProperty: Boolean, txn: OptimisticTransaction, newSchema: StructType): Unit = { val newConfiguration = getConfigurationWithoutColumnMapping(txn, removeColumnMappingTableProperty) val newMetadata = txn.metadata.copy( schemaString = newSchema.json, configuration = newConfiguration) txn.updateMetadata(newMetadata) } def getConfigurationWithoutColumnMapping( txn: OptimisticTransaction, removeColumnMappingTableProperty: Boolean): Map[String, String] = { // Scanned schema does not include the column mapping metadata and can be reused as is. val columnMappingPropertyKey = DeltaConfigs.COLUMN_MAPPING_MODE.key val columnMappingMaxIdPropertyKey = DeltaConfigs.COLUMN_MAPPING_MAX_ID.key // Unset or overwrite the column mapping mode to none and remove max id property // while keeping other properties. (if (removeColumnMappingTableProperty) { txn.metadata.configuration - columnMappingPropertyKey } else { txn.metadata.configuration + (columnMappingPropertyKey -> "none") }) - columnMappingMaxIdPropertyKey } def getDeltaOptionsForWrite(spark: SparkSession): DeltaOptions = { new DeltaOptions( // Prevent files from being split by writers. Map(DeltaOptions.MAX_RECORDS_PER_FILE -> "0"), spark.sessionState.conf) } def buildDataFrame( txn: OptimisticTransaction, originalFiles: Seq[AddFile]): DataFrame = recordDeltaOperation(txn.deltaLog, "delta.removeColumnMapping.setupDataFrame") { txn.deltaLog.createDataFrame(txn.snapshot, originalFiles) } def writeData( txn: OptimisticTransaction, data: DataFrame, deltaOptions: DeltaOptions): Seq[AddFile] = { txn.writeFiles( inputData = RowTracking.preserveRowTrackingColumns(data, txn.snapshot), writeOptions = Some(deltaOptions), isOptimize = true, additionalConstraints = Seq.empty) .asInstanceOf[Seq[AddFile]] // Mark as no data change to not generate CDC data. We are only removing column mapping. .map(_.copy(dataChange = false)) } } object RemoveColumnMappingCommand { def apply( deltaLog: DeltaLog, catalogOpt: Option[CatalogTable]): RemoveColumnMappingCommand = { new RemoveColumnMappingCommand(deltaLog, catalogOpt) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/convert/ConvertUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.convert import java.lang.reflect.InvocationTargetException import org.apache.spark.sql.delta.{DeltaColumnMapping, DeltaErrors, SerializableFileStatus, Snapshot} import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaMergingUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.{DateFormatter, DeltaFileOperations, PartitionUtils, TimestampFormatter} import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.spark.sql.{AnalysisException, Dataset, SparkSession} import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.Cast import org.apache.spark.sql.connector.catalog.Table import org.apache.spark.sql.execution.datasources.PartitioningUtils import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{StringType, StructType} import org.apache.spark.util.{SerializableConfiguration, Utils} object ConvertUtils extends ConvertUtilsBase trait ConvertUtilsBase extends DeltaLogging { val timestampPartitionPattern = "yyyy-MM-dd HH:mm:ss[.S]" var icebergSparkTableClassPath = "org.apache.spark.sql.delta.commands.convert.IcebergTable" var icebergLibTableClassPath = "shadedForDelta.org.apache.iceberg.Table" /** * Creates a source Parquet table for conversion. * * @param spark: the spark session to use. * @param targetDir: the target directory of the Parquet table. * @param catalogTable: the optional catalog table of the Parquet table. * @param partitionSchema: the user provided partition schema (if exists) of the Parquet table. * @return a target Parquet table. */ def getParquetTable( spark: SparkSession, targetDir: String, catalogTable: Option[CatalogTable], partitionSchema: Option[StructType]): ConvertTargetTable = { val qualifiedDir = getQualifiedPath(spark, new Path(targetDir)).toString new ParquetTable(spark, qualifiedDir, catalogTable, partitionSchema) } /** * Creates a source Iceberg table for conversion. * * @param spark: the spark session to use. * @param targetDir: the target directory of the Iceberg table. * @param sparkTable: the optional V2 table interface of the Iceberg table. * @param deltaTable: the existing converted Delta table (if exists) of the Iceberg table. * @param collectStats: collect column stats on convert * @return a target Iceberg table. */ def getIcebergTable( spark: SparkSession, targetDir: String, deltaSnapshotOpt: Option[Snapshot], collectStats: Boolean = true): ConvertTargetTable = { try { val convertIcebergStats = collectStats && spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_CONVERT_ICEBERG_STATS) val clazz = Utils.classForName(icebergSparkTableClassPath) val baseDir = getQualifiedPath(spark, new Path(targetDir)).toString val constFromPath = clazz.getConstructor( classOf[SparkSession], classOf[String], classOf[Option[Snapshot]], java.lang.Boolean.TYPE) constFromPath.newInstance(spark, baseDir, deltaSnapshotOpt, java.lang.Boolean.valueOf(convertIcebergStats)) } catch { case e: ClassNotFoundException => logError(log"Failed to find Iceberg class", e) throw DeltaErrors.icebergClassMissing(spark.sparkContext.getConf, e) case e: InvocationTargetException => logError(log"Got error when creating an Iceberg Converter", e) // The better error is within the cause throw ExceptionUtils.getRootCause(e) } } /** * Get Iceberg metadata location from spark catalog resolved Iceberg table, * which means it is a SparkTable * Needs to use reflection because shaded Iceberg classes are not accessible here * It is equivalent to call * table.asInstanceOf[SparkTable].table().operations().current().metadataFileLocation() * @param table the iceberg table resolved spark catalog * @return metadata location corresponding to table's latest snapshot */ def getIcebergMetadataLocationFromSparkTable(table: Table): String = { val tableMethod = table.getClass.getMethod("table") val icebergTable = tableMethod.invoke(table) val operationsMethod = icebergTable.getClass.getMethod("operations") val operations = operationsMethod.invoke(icebergTable) val currentMethod = operations.getClass.getMethod("current") val currentMetadata = currentMethod.invoke(operations) val metadataFileLocationMethod = currentMetadata.getClass.getMethod("metadataFileLocation") metadataFileLocationMethod.invoke(currentMetadata).asInstanceOf[String] } /** * Generates a qualified Hadoop path from a given path. * * @param spark: the spark session to use * @param path: the raw path used to generate the qualified path. * @return the qualified path of the provided raw path. */ def getQualifiedPath(spark: SparkSession, path: Path): Path = { // scalastyle:off deltahadoopconfiguration val sessionHadoopConf = spark.sessionState.newHadoopConf() // scalastyle:on deltahadoopconfiguration val fs = path.getFileSystem(sessionHadoopConf) val qualifiedPath = fs.makeQualified(path) if (!fs.exists(qualifiedPath)) { throw DeltaErrors.directoryNotFoundException(qualifiedPath.toString) } qualifiedPath } /** * Generates AddFile from ConvertTargetFile for conversion. * * @param targetFile: the target file to convert. * @param basePath: the table directory of the target file. * @param fs: the file system to access the target file. * @param conf: the SQL configures use to convert. * @param partitionSchema: the partition schema of the target file if exists. * @param useAbsolutePath: whether to use absolute path instead of relative path in the AddFile. * @return an AddFile corresponding to the provided ConvertTargetFile. */ def createAddFile( targetFile: ConvertTargetFile, basePath: Path, fs: FileSystem, conf: SQLConf, partitionSchema: Option[StructType], useAbsolutePath: Boolean = false): AddFile = { val partitionFields = partitionSchema.map(_.fields.toSeq).getOrElse(Nil) val partitionColNames = partitionSchema.map(_.fieldNames.toSeq).getOrElse(Nil) val physicalPartitionColNames = partitionSchema.map(_.map { f => DeltaColumnMapping.getPhysicalName(f) }).getOrElse(Nil) val file = targetFile.fileStatus val path = file.getHadoopPath val partition = targetFile.partitionValues.getOrElse { // partition values are not provided by the source table format, so infer from the file path val pathStr = file.getHadoopPath.toUri.toString val dateFormatter = DateFormatter() val timestampFormatter = TimestampFormatter(timestampPartitionPattern, java.util.TimeZone.getDefault) val resolver = conf.resolver val dir = if (file.isDir) file.getHadoopPath else file.getHadoopPath.getParent val (partitionOpt, _) = PartitionUtils.parsePartition( dir, typeInference = false, basePaths = Set(basePath), userSpecifiedDataTypes = Map.empty, validatePartitionColumns = false, java.util.TimeZone.getDefault, dateFormatter, timestampFormatter) partitionOpt.map { partValues => if (partitionColNames.size != partValues.columnNames.size) { throw DeltaErrors.unexpectedNumPartitionColumnsFromFileNameException( pathStr, partValues.columnNames, partitionColNames) } val tz = Option(conf.sessionLocalTimeZone) // Check if the partition value can be casted to the provided type if (!conf.getConf(DeltaSQLConf.DELTA_CONVERT_PARTITION_VALUES_IGNORE_CAST_FAILURE)) { partValues.literals.zip(partitionFields).foreach { case (literal, field) => if (literal.eval() != null && Cast(literal, field.dataType, tz, ansiEnabled = false).eval() == null) { val partitionValue = Cast(literal, StringType, tz, ansiEnabled = false).eval() val partitionValueStr = Option(partitionValue).map(_.toString).orNull throw DeltaErrors.castPartitionValueException(partitionValueStr, field.dataType) } } } val values = partValues .literals .map(PartitionUtils.literalToNormalizedString(_, tz)) partitionColNames.zip(partValues.columnNames).foreach { case (expected, parsed) => if (!resolver(expected, parsed)) { throw DeltaErrors.unexpectedPartitionColumnFromFileNameException( pathStr, parsed, expected) } } physicalPartitionColNames.zip(values).toMap }.getOrElse { if (partitionColNames.nonEmpty) { throw DeltaErrors.unexpectedNumPartitionColumnsFromFileNameException( pathStr, Seq.empty, partitionColNames) } Map[String, String]() } } val pathStrForAddFile = if (!useAbsolutePath) { val relativePath = DeltaFileOperations.tryRelativizePath(fs, basePath, path) assert(!relativePath.isAbsolute, s"Fail to relativize path $path against base path $basePath.") relativePath.toUri.toString } else { fs.makeQualified(path).toUri.toString } AddFile( pathStrForAddFile, partition, file.length, file.modificationTime, dataChange = true, stats = targetFile.stats.orNull ) } /** * A helper function to check whether a directory should be skipped during conversion. * * @param dirName: the directory name to check. * @return true if directory should be skipped for conversion, otherwise false. */ def dirNameFilter(dirName: String): Boolean = { // Allow partition column name starting with underscore and dot DeltaFileOperations.defaultHiddenFileFilter(dirName) && !dirName.contains("=") } /** * Lists directories non-recursively in the distributed manner. * * @param spark: the spark session to use. * @param rootDir: the root directory of all directories to list * @param dirs: the list of directories to list. * @param serializableConf: the hadoop configure to use. * @return a dataset of files from the listing. */ def listDirsInParallel( spark: SparkSession, rootDir: String, dirs: Seq[String], serializableConf: SerializableConfiguration): Dataset[SerializableFileStatus] = { import org.apache.spark.sql.delta.implicits._ val conf = spark.sparkContext.broadcast(serializableConf) val parallelism = spark.sessionState.conf.parallelPartitionDiscoveryParallelism val rdd = spark.sparkContext.parallelize(dirs, math.min(parallelism, dirs.length)) .mapPartitions { batch => batch.flatMap { dir => DeltaFileOperations .localListDirs(conf.value.value, Seq(dir), recursive = false) .filter(!_.isDir) } } spark.createDataset(rdd) } /** * Merges the schemas of the ConvertTargetFiles. * * @param spark: the SparkSession used for schema merging. * @param partitionSchema: the partition schema to be merged with the data schema. * @param convertTargetFiles: the Dataset of ConvertTargetFiles to be merged. * @return the merged StructType representing the combined schema of the Parquet files. * @throws DeltaErrors.failedInferSchema If no schemas are found for merging. */ def mergeSchemasInParallel( spark: SparkSession, partitionSchema: StructType, convertTargetFiles: Dataset[ConvertTargetFile]): StructType = { import org.apache.spark.sql.delta.implicits._ val partiallyMergedSchemas = recordFrameProfile("Delta", "ConvertUtils.mergeSchemasInParallel") { convertTargetFiles.mapPartitions { iterator => var dataSchema: StructType = StructType(Seq()) iterator.foreach { file => try { dataSchema = SchemaMergingUtils.mergeSchemas(dataSchema, StructType.fromDDL(file.parquetSchemaDDL.get).asNullable) } catch { case cause: AnalysisException => throw DeltaErrors.failedMergeSchemaFile( file.fileStatus.path, StructType.fromDDL(file.parquetSchemaDDL.get).treeString, cause) } } Iterator.single(dataSchema.toDDL) }.collect().filter(_.nonEmpty) } if (partiallyMergedSchemas.isEmpty) { throw DeltaErrors.failedInferSchema } var mergedSchema: StructType = StructType(Seq()) partiallyMergedSchemas.foreach { schema => mergedSchema = SchemaMergingUtils.mergeSchemas(mergedSchema, StructType.fromDDL(schema)) } PartitioningUtils.mergeDataAndPartitionSchema( mergedSchema, StructType(partitionSchema.fields.toSeq), spark.sessionState.conf.caseSensitiveAnalysis)._1 } } /** * Configuration for fetching Parquet schema. * * @param assumeBinaryIsString: whether unannotated BINARY fields should be assumed to be Spark * SQL [[StringType]] fields. * @param assumeInt96IsTimestamp: whether unannotated INT96 fields should be assumed to be Spark * SQL [[TimestampType]] fields. * @param ignoreCorruptFiles: a boolean indicating whether corrupt files should be ignored during * schema retrieval. */ case class ParquetSchemaFetchConfig( assumeBinaryIsString: Boolean, assumeInt96IsTimestamp: Boolean, ignoreCorruptFiles: Boolean) ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/convert/ParquetFileManifest.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.convert import org.apache.spark.sql.delta.Relocated._ import org.apache.spark.sql.delta.{DeltaErrors, SerializableFileStatus} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.{DeltaFileOperations, PartitionUtils} import org.apache.hadoop.fs.Path import org.apache.spark.sql.{Dataset, SparkSession} import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat, ParquetToSparkSchemaConverter} import org.apache.spark.sql.types.StructType import org.apache.spark.util.SerializableConfiguration /** A file manifest generated through recursively listing a base path. */ class ManualListingFileManifest( spark: SparkSession, override val basePath: String, partitionSchema: StructType, parquetSchemaFetchConfig: ParquetSchemaFetchConfig, serializableConf: SerializableConfiguration) extends ConvertTargetFileManifest with DeltaLogging { protected def doList(): Dataset[SerializableFileStatus] = { val conf = spark.sparkContext.broadcast(serializableConf) DeltaFileOperations .recursiveListDirs(spark, Seq(basePath), conf, ConvertUtils.dirNameFilter) .where("!isDir") } override lazy val allFiles: Dataset[ConvertTargetFile] = { import org.apache.spark.sql.delta.implicits._ val conf = spark.sparkContext.broadcast(serializableConf) val fetchConfig = parquetSchemaFetchConfig val files = doList().mapPartitions { iter => val fileStatuses = iter.toSeq val pathToStatusMapping = fileStatuses.map { fileStatus => fileStatus.path -> fileStatus }.toMap val footerSeq = DeltaFileOperations.readParquetFootersInParallel( conf.value.value, fileStatuses.map(_.toFileStatus), fetchConfig.ignoreCorruptFiles) val schemaConverter = new ParquetToSparkSchemaConverter( assumeBinaryIsString = fetchConfig.assumeBinaryIsString, assumeInt96IsTimestamp = fetchConfig.assumeInt96IsTimestamp ) footerSeq.map { footer => val fileStatus = pathToStatusMapping(footer.getFile.toString) val schema = ParquetFileFormat.readSchemaFromFooter(footer, schemaConverter) ConvertTargetFile(fileStatus, None, Some(schema.toDDL)) }.toIterator } files.cache() files } override lazy val parquetSchema: Option[StructType] = { recordDeltaOperationForTablePath(basePath, "delta.convert.schemaInference") { Some(ConvertUtils.mergeSchemasInParallel(spark, partitionSchema, allFiles)) } } override def close(): Unit = allFiles.unpersist() } /** A file manifest generated through listing partition paths from Metastore catalog. */ class CatalogFileManifest( spark: SparkSession, override val basePath: String, catalogTable: CatalogTable, partitionSchema: StructType, parquetSchemaFetchConfig: ParquetSchemaFetchConfig, serializableConf: SerializableConfiguration) extends ConvertTargetFileManifest with DeltaLogging { private val useCatalogSchema = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_CONVERT_USE_CATALOG_SCHEMA) // List of partition directories and corresponding partition values. private lazy val partitionList = { if (catalogTable.partitionSchema.isEmpty) { // Not a partitioned table. Seq(basePath -> Map.empty[String, String]) } else { val partitions = spark.sessionState.catalog.listPartitions(catalogTable.identifier) partitions.map { partition => // Convert URI into Path first to decode special characters. val partitionDir = partition.storage.locationUri.map(new Path(_).toString()) .getOrElse { val partitionDir = PartitionUtils.getPathFragment(partition.spec, catalogTable.partitionSchema) basePath.stripSuffix("/") + "/" + partitionDir } partitionDir -> partition.spec } } } protected def doList(): Dataset[SerializableFileStatus] = { if (partitionList.isEmpty) { throw DeltaErrors.convertToDeltaNoPartitionFound(catalogTable.identifier.unquotedString) } ConvertUtils.listDirsInParallel(spark, basePath, partitionList.map(_._1), serializableConf) } override lazy val allFiles: Dataset[ConvertTargetFile] = { import org.apache.spark.sql.delta.implicits._ // Avoid the serialization of this CatalogFileManifest during distributed execution. val conf = spark.sparkContext.broadcast(serializableConf) val useParquetSchema = !useCatalogSchema val dirToPartitionSpec = partitionList.toMap val fetchConfig = parquetSchemaFetchConfig val files = doList().mapPartitions { iter => val fileStatuses = iter.toSeq if (useParquetSchema) { val pathToFile = fileStatuses.map { fileStatus => fileStatus.path -> fileStatus }.toMap val footerSeq = DeltaFileOperations.readParquetFootersInParallel( conf.value.value, fileStatuses.map(_.toFileStatus), fetchConfig.ignoreCorruptFiles) val schemaConverter = new ParquetToSparkSchemaConverter( assumeBinaryIsString = fetchConfig.assumeBinaryIsString, assumeInt96IsTimestamp = fetchConfig.assumeInt96IsTimestamp ) footerSeq.map { footer => val schema = ParquetFileFormat.readSchemaFromFooter(footer, schemaConverter) val fileStatus = pathToFile(footer.getFile.toString) ConvertTargetFile( fileStatus, dirToPartitionSpec.get(footer.getFile.getParent.toString), Some(schema.toDDL)) }.toIterator } else { // TODO: Currently "spark.sql.files.ignoreCorruptFiles" is not respected for // CatalogFileManifest when catalog schema is used to avoid performance regression. fileStatuses.map { fileStatus => ConvertTargetFile( fileStatus, dirToPartitionSpec.get(fileStatus.getHadoopPath.getParent.toString), None) }.toIterator } } files.cache() files } override lazy val parquetSchema: Option[StructType] = { if (useCatalogSchema) { Some(catalogTable.schema) } else { recordDeltaOperationForTablePath(basePath, "delta.convert.schemaInference") { Some(ConvertUtils.mergeSchemasInParallel(spark, partitionSchema, allFiles)) } } } override def close(): Unit = allFiles.unpersist() } /** A file manifest generated from pre-existing parquet MetadataLog. */ class MetadataLogFileManifest( spark: SparkSession, override val basePath: String, partitionSchema: StructType, parquetSchemaFetchConfig: ParquetSchemaFetchConfig, serializableConf: SerializableConfiguration) extends ConvertTargetFileManifest with DeltaLogging { val index = createMetadataLogFileIndex(spark, new Path(basePath), Map.empty, None) protected def doList(): Dataset[SerializableFileStatus] = { import org.apache.spark.sql.delta.implicits._ val rdd = spark.sparkContext.parallelize(index.allFiles()).mapPartitions { _ .map(SerializableFileStatus.fromStatus) } spark.createDataset(rdd) } override lazy val allFiles: Dataset[ConvertTargetFile] = { import org.apache.spark.sql.delta.implicits._ val conf = spark.sparkContext.broadcast(serializableConf) val fetchConfig = parquetSchemaFetchConfig val files = doList().mapPartitions { iter => val fileStatuses = iter.toSeq val pathToStatusMapping = fileStatuses.map { fileStatus => fileStatus.path -> fileStatus }.toMap val footerSeq = DeltaFileOperations.readParquetFootersInParallel( conf.value.value, fileStatuses.map(_.toFileStatus), fetchConfig.ignoreCorruptFiles) val schemaConverter = new ParquetToSparkSchemaConverter( assumeBinaryIsString = fetchConfig.assumeBinaryIsString, assumeInt96IsTimestamp = fetchConfig.assumeInt96IsTimestamp ) footerSeq.map { footer => val fileStatus = pathToStatusMapping(footer.getFile.toString) val schema = ParquetFileFormat.readSchemaFromFooter(footer, schemaConverter) ConvertTargetFile(fileStatus, None, Some(schema.toDDL)) }.toIterator } files.cache() files } override lazy val parquetSchema: Option[StructType] = { recordDeltaOperationForTablePath(basePath, "delta.convert.schemaInference") { Some(ConvertUtils.mergeSchemasInParallel(spark, partitionSchema, allFiles)) } } override def close(): Unit = allFiles.unpersist() } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/convert/ParquetTable.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.convert import org.apache.spark.sql.delta.Relocated._ import org.apache.spark.sql.delta.{DeltaErrors, SerializableFileStatus} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaMergingUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.DeltaFileOperations import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.execution.datasources.PartitioningUtils import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat, ParquetToSparkSchemaConverter} import org.apache.spark.sql.types.StructType import org.apache.spark.util.SerializableConfiguration /** * A target Parquet table for conversion to a Delta table. * * @param spark: spark session to use. * @param basePath: the root directory of the Parquet table. * @param catalogTable: optional catalog table (if exists) of the Parquet table. * @param userPartitionSchema: user provided partition schema of the Parquet table. */ class ParquetTable( val spark: SparkSession, val basePath: String, val catalogTable: Option[CatalogTable], val userPartitionSchema: Option[StructType]) extends ConvertTargetTable with DeltaLogging { // Validate user provided partition schema if catalogTable is available. if (catalogTable.isDefined && userPartitionSchema.isDefined && !catalogTable.get.partitionSchema.equals(userPartitionSchema.get)) { throw DeltaErrors.unexpectedPartitionSchemaFromUserException( catalogTable.get.partitionSchema, userPartitionSchema.get) } protected lazy val serializableConf: SerializableConfiguration = { // scalastyle:off deltahadoopconfiguration new SerializableConfiguration(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration } override val partitionSchema: StructType = { userPartitionSchema.orElse(catalogTable.map(_.partitionSchema)).getOrElse(new StructType()) } override lazy val numFiles: Long = fileManifest.numFiles override lazy val sizeInBytes: Long = fileManifest.sizeInBytes def tableSchema: StructType = fileManifest.parquetSchema.get override val format: String = "parquet" val fileManifest: ConvertTargetFileManifest = { val fetchConfig = ParquetSchemaFetchConfig( spark.sessionState.conf.isParquetBinaryAsString, spark.sessionState.conf.isParquetINT96AsTimestamp, spark.sessionState.conf.ignoreCorruptFiles ) if (spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_CONVERT_USE_METADATA_LOG) && FileStreamSink.hasMetadata(Seq(basePath), serializableConf.value, spark.sessionState.conf)) { new MetadataLogFileManifest(spark, basePath, partitionSchema, fetchConfig, serializableConf) } else if (spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_CONVERT_USE_CATALOG_PARTITIONS) && catalogTable.isDefined) { new CatalogFileManifest( spark, basePath, catalogTable.get, partitionSchema, fetchConfig, serializableConf) } else { new ManualListingFileManifest( spark, basePath, partitionSchema, fetchConfig, serializableConf) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/convert/interfaces.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.convert import java.io.Closeable import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.{DeltaColumnMappingMode, DeltaLog, NoMapping, SerializableFileStatus} import org.apache.spark.sql.Dataset import org.apache.spark.sql.functions.sum import org.apache.spark.sql.types.StructType /** * An interface for the table to be converted to Delta. */ trait ConvertTargetTable { /** The table schema of the target table */ def tableSchema: StructType /** The table properties of the target table */ def properties: Map[String, String] = Map.empty /** The partition schema of the target table */ def partitionSchema: StructType /** The file manifest of the target table */ def fileManifest: ConvertTargetFileManifest /** The number of files from the target table */ def numFiles: Long /** The number of bytes from the target table */ def sizeInBytes: Long /** Whether this table requires column mapping to be converted */ def requiredColumnMappingMode: DeltaColumnMappingMode = NoMapping /* The format of the table */ def format: String } /** An interface for providing an iterator of files for a table. */ trait ConvertTargetFileManifest extends Closeable { /** The base path of a table. Should be a qualified, normalized path. */ val basePath: String /** Return all files as a Dataset for parallelized processing. */ def allFiles: Dataset[ConvertTargetFile] /** Return the active files for a table in sequence */ def getFiles: Iterator[ConvertTargetFile] = allFiles.toLocalIterator().asScala /** Return the number of files for the table */ def numFiles: Long = allFiles.count() /** Return the number of bytes for the table */ def sizeInBytes: Long = { allFiles.select("fileStatus.*").select(sum("length")).collect().head.getLong(0) } /** Return the parquet schema for the table. * Defined only when the schema cannot be inferred from CatalogTable. */ def parquetSchema: Option[StructType] = None } /** * An interface for the file to be included during conversion. * * @param fileStatus the file info * @param partitionValues partition values of this file that may be available from the source * table format. If none, the converter will infer partition values from the * file path, assuming the Hive directory format. * @param parquetSchemaDDL the Parquet schema DDL associated with the file. * @param stats Stats information extracted from the source file. */ case class ConvertTargetFile( fileStatus: SerializableFileStatus, partitionValues: Option[Map[String, String]] = None, parquetSchemaDDL: Option[String] = None, stats: Option[String] = None ) extends Serializable ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/merge/ClassicMergeExecutor.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.merge import scala.collection.JavaConverters._ import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions.{AddCDCFile, AddFile, FileAction} import org.apache.spark.sql.delta.commands.{DeletionVectorBitmapGenerator, DMLWithDeletionVectorsHelper, MergeIntoCommandBase} import org.apache.spark.sql.delta.commands.cdc.CDCReader.{CDC_TYPE_COLUMN_NAME, CDC_TYPE_NOT_CDC} import org.apache.spark.sql.delta.commands.merge.MergeOutputGeneration.{SOURCE_ROW_INDEX_COL, TARGET_ROW_INDEX_COL} import org.apache.spark.sql.delta.files.TahoeBatchFileIndex import org.apache.spark.sql.delta.stats.StatsCollectionUtils import org.apache.spark.sql.delta.util.SetAccumulator import org.apache.spark.sql.{Column, Dataset, SparkSession} import org.apache.spark.sql.catalyst.expressions.{And, Expression, Literal, Or} import org.apache.spark.sql.catalyst.plans.logical.DeltaMergeIntoClause import org.apache.spark.sql.functions.{coalesce, col, count, input_file_name, lit, monotonically_increasing_id, sum} /** * Trait with merge execution in two phases: * * Phase 1: Find the input files in target that are touched by the rows that satisfy * the condition and verify that no two source rows match with the same target row. * This is implemented as an inner-join using the given condition (see [[findTouchedFiles]]). * In the special case that there is no update clause we write all the non-matching * source data as new files and skip phase 2. * Issues an error message when the ON search_condition of the MERGE statement can match * a single row from the target table with multiple rows of the source table-reference. * * Phase 2: Read the touched files again and write new files with updated and/or inserted rows. * If there are updates, then use an outer join using the given condition to write the * updates and inserts (see [[writeAllChanges()]]). If there are no matches for updates, * only inserts, then write them directly (see [[writeInsertsOnlyWhenNoMatches()]]). * * Note, when deletion vectors are enabled, phase 2 is split into two parts: * 2.a. Read the touched files again and only write modified and new * rows (see [[writeAllChanges()]]). * 2.b. Read the touched files and generate deletion vectors for the modified * rows (see [[writeDVs()]]). * * If there are no matches for updates, only inserts, then write them directly * (see [[writeInsertsOnlyWhenNoMatches()]]). This remains the same when DVs are enabled since there * are no modified rows. Furthermore, eee [[InsertOnlyMergeExecutor]] for the optimized executor * used in case there are only inserts. */ trait ClassicMergeExecutor extends MergeOutputGeneration { self: MergeIntoCommandBase => import MergeIntoCommandBase._ /** * Find the target table files that contain the rows that satisfy the merge condition. This is * implemented as an inner-join between the source query/table and the target table using * the merge condition. */ protected def findTouchedFiles( spark: SparkSession, deltaTxn: OptimisticTransaction ): (Seq[AddFile], DeduplicateCDFDeletes) = recordMergeOperation( extraOpType = "findTouchedFiles", status = "MERGE operation - scanning files for matches", sqlMetricName = "scanTimeMs") { val columnComparator = spark.sessionState.analyzer.resolver // Accumulator to collect all the distinct touched files val touchedFilesAccum = new SetAccumulator[String]() spark.sparkContext.register(touchedFilesAccum, TOUCHED_FILES_ACCUM_NAME) // Prune non-matching files if we don't need to collect them for NOT MATCHED BY SOURCE clauses. val dataSkippedFiles = if (notMatchedBySourceClauses.isEmpty) { deltaTxn.filterFiles(getTargetOnlyPredicates(spark), keepNumRecords = true) } else { deltaTxn.filterFiles(filters = Seq(Literal.TrueLiteral), keepNumRecords = true) } // Join the source and target table using the merge condition to find touched files. An inner // join collects all candidate files for MATCHED clauses, a right outer join also includes // candidates for NOT MATCHED BY SOURCE clauses. // In addition, we attach two columns // - a monotonically increasing row id for target rows to later identify whether the same // target row is modified by multiple user or not // - the target file name the row is from to later identify the files touched by matched rows val joinType = if (notMatchedBySourceClauses.isEmpty) "inner" else "right_outer" // When they are only MATCHED clauses, after the join we prune files that have no rows that // satisfy any of the clause conditions. val matchedPredicate = if (isMatchedOnly) { matchedClauses // An undefined condition (None) is implicitly true .map(_.condition.getOrElse(Literal.TrueLiteral)) .reduce((a, b) => Or(a, b)) } else Literal.TrueLiteral // Compute the columns needed for the inner join. val targetColsNeeded = { condition.references.map(_.name) ++ deltaTxn.snapshot.metadata.partitionColumns ++ matchedPredicate.references.map(_.name) } val columnsToDrop = deltaTxn.snapshot.metadata.schema.map(_.name) .filterNot { field => targetColsNeeded.exists { name => columnComparator(name, field) } } val incrSourceRowCountExpr = incrementMetricAndReturnBool("numSourceRows", valueToReturn = true) // We can't use filter() directly on the expression because that will prevent // column pruning. We don't need the SOURCE_ROW_PRESENT_COL so we immediately drop it. val sourceDF = getMergeSource.df .withColumn(SOURCE_ROW_PRESENT_COL, Column(incrSourceRowCountExpr)) .filter(SOURCE_ROW_PRESENT_COL) .drop(SOURCE_ROW_PRESENT_COL) val targetPlan = buildTargetPlanWithFiles( spark, deltaTxn, dataSkippedFiles, columnsToDrop) val targetDF = DataFrameUtils.ofRows(spark, targetPlan) .withColumn(ROW_ID_COL, monotonically_increasing_id()) .withColumn(FILE_NAME_COL, input_file_name()) val joinToFindTouchedFiles = sourceDF.join(targetDF, Column(condition), joinType) // UDFs to records touched files names and add them to the accumulator val recordTouchedFileName = DeltaUDF.intFromStringBoolean { (fileName, shouldRecord) => if (shouldRecord) { touchedFilesAccum.add(fileName) } 1 }.asNondeterministic() // Process the matches from the inner join to record touched files and find multiple matches val collectTouchedFiles = joinToFindTouchedFiles .select(col(ROW_ID_COL), recordTouchedFileName(col(FILE_NAME_COL), Column(matchedPredicate)).as("one")) // Calculate frequency of matches per source row val matchedRowCounts = collectTouchedFiles.groupBy(ROW_ID_COL).agg(sum("one").as("count")) // Get multiple matches and simultaneously collect (using touchedFilesAccum) the file names import org.apache.spark.sql.delta.implicits._ val (multipleMatchCount, multipleMatchSum) = matchedRowCounts .filter("count > 1") .select(coalesce(count(Column("*")), lit(0)), coalesce(sum("count"), lit(0))) .as[(Long, Long)] .collect() .head checkSourcePlanIsNotCached(spark, getMergeSource.df.queryExecution.logical) val hasMultipleMatches = multipleMatchCount > 0 throwErrorOnMultipleMatches(hasMultipleMatches, spark) if (hasMultipleMatches) { // This is only allowed for delete-only queries. // This query will count the duplicates for numTargetRowsDeleted in Job 2, // because we count matches after the join and not just the target rows. // We have to compensate for this by subtracting the duplicates later, // so we need to record them here. val duplicateCount = multipleMatchSum - multipleMatchCount multipleMatchDeleteOnlyOvercount = Some(duplicateCount) } // Get the AddFiles using the touched file names. val touchedFileNames = touchedFilesAccum.value.iterator().asScala.toSeq logTrace(s"findTouchedFiles: matched files:\n\t${touchedFileNames.mkString("\n\t")}") val nameToAddFileMap = generateCandidateFileMap(targetDeltaLog.dataPath, dataSkippedFiles) val touchedAddFiles = touchedFileNames.map( getTouchedFile(targetDeltaLog.dataPath, _, nameToAddFileMap)) if (metrics("numSourceRows").value == 0 && (dataSkippedFiles.isEmpty || dataSkippedFiles.forall(_.numLogicalRecords.getOrElse(0) == 0))) { // The target table is empty, and the optimizer optimized away the join entirely OR the // source table is truly empty. In that case, scanning the source table once is the only // way to get the correct metric. val numSourceRows = sourceDF.count() metrics("numSourceRows").set(numSourceRows) } metrics("numTargetFilesBeforeSkipping") += deltaTxn.snapshot.numOfFiles metrics("numTargetBytesBeforeSkipping") += deltaTxn.snapshot.sizeInBytes val (afterSkippingBytes, afterSkippingPartitions) = totalBytesAndDistinctPartitionValues(dataSkippedFiles) metrics("numTargetFilesAfterSkipping") += dataSkippedFiles.size metrics("numTargetBytesAfterSkipping") += afterSkippingBytes metrics("numTargetPartitionsAfterSkipping") += afterSkippingPartitions val (removedBytes, removedPartitions) = totalBytesAndDistinctPartitionValues(touchedAddFiles) metrics("numTargetFilesRemoved") += touchedAddFiles.size metrics("numTargetBytesRemoved") += removedBytes metrics("numTargetPartitionsRemovedFrom") += removedPartitions val dedupe = DeduplicateCDFDeletes( hasMultipleMatches && isCdcEnabled(deltaTxn), includesInserts) (touchedAddFiles, dedupe) } /** * Helper function that produces an expression by combining a sequence of clauses with OR. * Requires the sequence to be non-empty. */ protected def clauseDisjunction(clauses: Seq[DeltaMergeIntoClause]): Expression = { require(clauses.nonEmpty) clauses .map(_.condition.getOrElse(Literal.TrueLiteral)) .reduceLeft(Or) } /** * Returns the expression that can be used for selecting the modified rows generated * by the merge operation. The expression is to designed to work irrespectively * of the join type used between the source and target tables. * * The expression consists of two parts, one for each of the action clause types that produce * row modifications: MATCHED, NOT MATCHED BY SOURCE. All actions of the same clause type form * a disjunctive clause. The result is then conjucted to an expression that filters the rows * of the particular action clause type. For example: * * MERGE INTO t * USING s * ON s.id = t.id * WHEN MATCHED AND id < 5 THEN ... * WHEN MATCHED AND id > 10 THEN ... * WHEN NOT MATCHED BY SOURCE AND id > 20 THEN ... * * Produces the following expression: * * ((as.id = t.id) AND (id < 5 OR id > 10)) * OR * ((SOURCE TABLE IS NULL) AND (id > 20)) */ protected def generateFilterForModifiedRows(): Expression = { val matchedExpression = if (matchedClauses.nonEmpty) { And(condition, clauseDisjunction(matchedClauses)) } else { Literal.FalseLiteral } val notMatchedBySourceExpression = if (notMatchedBySourceClauses.nonEmpty) { val combinedClauses = clauseDisjunction(notMatchedBySourceClauses) And(col(SOURCE_ROW_PRESENT_COL).isNull.expr, combinedClauses) } else { Literal.FalseLiteral } Or(matchedExpression, notMatchedBySourceExpression) } /** * Returns the expression that can be used for selecting the new rows generated * by the merge operation. */ protected def generateFilterForNewRows(): Expression = { if (notMatchedClauses.nonEmpty) { val combinedClauses = clauseDisjunction(notMatchedClauses) And(col(TARGET_ROW_PRESENT_COL).isNull.expr, combinedClauses) } else { Literal.FalseLiteral } } /** * Write new files by reading the touched files and updating/inserting data using the source * query/table. This is implemented using a full-outer-join using the merge condition. * * Note that unlike the insert-only code paths with just one control column ROW_DROPPED_COL, this * method has a second control column CDC_TYPE_COL_NAME used for handling CDC when enabled. */ protected def writeAllChanges( spark: SparkSession, deltaTxn: OptimisticTransaction, filesToRewrite: Seq[AddFile], deduplicateCDFDeletes: DeduplicateCDFDeletes, writeUnmodifiedRows: Boolean): Seq[FileAction] = recordMergeOperation( extraOpType = if (!writeUnmodifiedRows) { "writeModifiedRowsOnly" } else if (shouldOptimizeMatchedOnlyMerge(spark)) { "writeAllUpdatesAndDeletes" } else { "writeAllChanges" }, status = s"MERGE operation - Rewriting ${filesToRewrite.size} files", sqlMetricName = "rewriteTimeMs") { val cdcEnabled = isCdcEnabled(deltaTxn) require( !deduplicateCDFDeletes.enabled || cdcEnabled, "CDF delete duplication is enabled but overall the CDF generation is disabled") // Generate a new target dataframe that has same output attributes exprIds as the target plan. // This allows us to apply the existing resolved update/insert expressions. val targetPlan = buildTargetPlanWithFiles( spark, deltaTxn, filesToRewrite, columnsToDrop = Nil) val baseTargetDF = RowTracking.preserveRowTrackingColumns( dfWithoutRowTrackingColumns = DataFrameUtils.ofRows(spark, targetPlan), snapshot = deltaTxn.snapshot) val joinType = if (writeUnmodifiedRows) { if (shouldOptimizeMatchedOnlyMerge(spark)) { "rightOuter" } else { "fullOuter" } } else { // Since we do not need to write unmodified rows, we can perform stricter joins. if (isMatchedOnly) { "inner" } else if (notMatchedBySourceClauses.isEmpty) { "leftOuter" } else if (notMatchedClauses.isEmpty) { "rightOuter" } else { "fullOuter" } } if (joinType == "fullOuter" || joinType == "leftOuter") { secondSourceScanWasFullScan = true } logDebug(s"""writeAllChanges using $joinType join: | source.output: ${source.outputSet} | target.output: ${target.outputSet} | condition: $condition | newTarget.output: ${baseTargetDF.queryExecution.logical.outputSet} """.stripMargin) // Expressions to update metrics val incrSourceRowCountExpr = incrementMetricAndReturnBool( "numSourceRowsInSecondScan", valueToReturn = true) val incrNoopCountExpr = incrementMetricAndReturnBool( "numTargetRowsCopied", valueToReturn = false) // Apply an outer join to find both, matches and non-matches. We are adding two boolean fields // with value `true`, one to each side of the join. Whether this field is null or not after // the outer join, will allow us to identify whether the joined row was a // matched inner result or an unmatched result with null on one side. val joinedBaseDF = { var sourceDF = getMergeSource.df if (deduplicateCDFDeletes.enabled && deduplicateCDFDeletes.includesInserts) { // Add row index for the source rows to identify inserted rows during the cdf deleted rows // deduplication. See [[deduplicateCDFDeletes()]] sourceDF = sourceDF.withColumn(SOURCE_ROW_INDEX_COL, monotonically_increasing_id()) } val left = sourceDF .withColumn(SOURCE_ROW_PRESENT_COL, Column(incrSourceRowCountExpr)) // In some cases, the optimizer (incorrectly) decides to omit the metrics column. // This causes issues in the source determinism validation. We work around the issue by // adding a redundant dummy filter to make sure the column is not pruned. .filter(SOURCE_ROW_PRESENT_COL) val targetDF = baseTargetDF .withColumn(TARGET_ROW_PRESENT_COL, lit(true)) val right = if (deduplicateCDFDeletes.enabled) { targetDF.withColumn(TARGET_ROW_INDEX_COL, monotonically_increasing_id()) } else { targetDF } left.join(right, Column(condition), joinType) } val joinedDF = if (writeUnmodifiedRows) { joinedBaseDF } else { val filter = Or(generateFilterForModifiedRows(), generateFilterForNewRows()) joinedBaseDF.filter(Column(filter)) } // Precompute conditions in matched and not matched clauses and generate // the joinedDF with precomputed columns and clauses with rewritten conditions. val (joinedAndPrecomputedConditionsDF, clausesWithPrecompConditions) = generatePrecomputedConditionsAndDF( joinedDF, clauses = matchedClauses ++ notMatchedClauses ++ notMatchedBySourceClauses) // In case Row IDs are preserved, get the attribute expression of the Row ID column. val rowIdColumnExpressionOpt = MaterializedRowId.getAttribute(deltaTxn.snapshot, joinedAndPrecomputedConditionsDF) val rowCommitVersionColumnExpressionOpt = MaterializedRowCommitVersion.getAttribute(deltaTxn.snapshot, joinedAndPrecomputedConditionsDF) // The target output columns need to be marked as nullable here, as they are going to be used // to reference the output of an outer join. val targetWriteCols = postEvolutionTargetExpressions(makeNullable = true) // If there are N columns in the target table, the full outer join output will have: // - N columns for target table // - Two optional Row ID / Row commit version preservation columns with their physical name. // - ROW_DROPPED_COL to define whether the generated row should be dropped or written // - if CDC is enabled, also CDC_TYPE_COLUMN_NAME with the type of change being performed // in a particular row // (N+1 or N+2 columns depending on CDC disabled / enabled) val outputColNames = targetWriteCols.map(_.name) ++ rowIdColumnExpressionOpt.map(_.name) ++ rowCommitVersionColumnExpressionOpt.map(_.name) ++ Seq(ROW_DROPPED_COL) ++ (if (cdcEnabled) Some(CDC_TYPE_COLUMN_NAME) else None) // Copy expressions to copy the existing target row and not drop it (ROW_DROPPED_COL=false), // and in case CDC is enabled, set it to CDC_TYPE_NOT_CDC. // (N+1 or N+2 or N+3 columns depending on CDC disabled / enabled and if Row IDs are preserved) val noopCopyExprs = targetWriteCols ++ rowIdColumnExpressionOpt ++ rowCommitVersionColumnExpressionOpt ++ Seq(incrNoopCountExpr) ++ (if (cdcEnabled) Seq(CDC_TYPE_NOT_CDC) else Seq()) // Generate output columns. val needSetRowTrackingFieldIdForUniform = IcebergCompat.isGeqEnabled(deltaTxn.metadata, 3) val outputCols = generateWriteAllChangesOutputCols( targetWriteCols, rowIdColumnExpressionOpt, rowCommitVersionColumnExpressionOpt, outputColNames, noopCopyExprs, writeUnmodifiedRows, clausesWithPrecompConditions, cdcEnabled, needSetRowTrackingFieldIdForUniform ) val preOutputDF = if (cdcEnabled) { generateCdcAndOutputRows( joinedAndPrecomputedConditionsDF, outputCols, outputColNames, noopCopyExprs, rowIdColumnExpressionOpt.map(_.name), rowCommitVersionColumnExpressionOpt.map(_.name), deduplicateCDFDeletes, needSetRowTrackingFieldIdForUniform = needSetRowTrackingFieldIdForUniform ) } else { // change data capture is off, just output the normal data joinedAndPrecomputedConditionsDF .select(outputCols: _*) } // The filter ensures we only consider rows that are not dropped. // The drop ensures that the dropped flag does not leak out to the output. val outputDF = preOutputDF .filter(s"$ROW_DROPPED_COL = false") .drop(ROW_DROPPED_COL) logDebug("writeAllChanges: join output plan:\n" + outputDF.queryExecution) // Write to Delta val newFiles = writeFiles(spark, deltaTxn, outputDF) checkSourcePlanIsNotCached(spark, getMergeSource.df.queryExecution.logical) // Update metrics val (addedBytes, addedPartitions) = totalBytesAndDistinctPartitionValues(newFiles) metrics("numTargetFilesAdded") += newFiles.count(_.isInstanceOf[AddFile]) metrics("numTargetChangeFilesAdded") += newFiles.count(_.isInstanceOf[AddCDCFile]) metrics("numTargetChangeFileBytes") += newFiles.collect{ case f: AddCDCFile => f.size }.sum metrics("numTargetBytesAdded") += addedBytes metrics("numTargetPartitionsAddedTo") += addedPartitions if (multipleMatchDeleteOnlyOvercount.isDefined) { // Compensate for counting duplicates during the query. val actualRowsDeleted = metrics("numTargetRowsDeleted").value - multipleMatchDeleteOnlyOvercount.get assert(actualRowsDeleted >= 0) metrics("numTargetRowsDeleted").set(actualRowsDeleted) val actualRowsMatchedDeleted = metrics("numTargetRowsMatchedDeleted").value - multipleMatchDeleteOnlyOvercount.get assert(actualRowsMatchedDeleted >= 0) metrics("numTargetRowsMatchedDeleted").set(actualRowsMatchedDeleted) } newFiles } /** * Writes Deletion Vectors for rows modified by the merge operation. */ protected def writeDVs( spark: SparkSession, deltaTxn: OptimisticTransaction, filesToRewrite: Seq[AddFile]): Seq[FileAction] = recordMergeOperation( extraOpType = "writeDeletionVectors", status = s"MERGE operation - Rewriting Deletion Vectors to ${filesToRewrite.size} files", sqlMetricName = "rewriteTimeMs") { val fileIndex = new TahoeBatchFileIndex( spark, actionType = "merge", addFiles = filesToRewrite, deltaLog = deltaTxn.deltaLog, path = deltaTxn.deltaLog.dataPath, snapshot = deltaTxn.snapshot) val targetDF = DMLWithDeletionVectorsHelper.createTargetDfForScanningForMatches( spark, target, fileIndex) // For writing DVs we are only interested in the target table. When there are no // notMatchedBySource clauses an inner join is sufficient. Otherwise, we need an rightOuter // join to include target rows that are not matched. val joinType = if (notMatchedBySourceClauses.isEmpty) { "inner" } else { "rightOuter" } val joinedDF = getMergeSource.df .withColumn(SOURCE_ROW_PRESENT_COL, lit(true)) .join(targetDF, Column(condition), joinType) val modifiedRowsFilter = generateFilterForModifiedRows() val matchedDVResult = DeletionVectorBitmapGenerator.buildRowIndexSetsForFilesMatchingCondition( spark, deltaTxn, tableHasDVs = true, targetDf = joinedDF, candidateFiles = filesToRewrite, condition = modifiedRowsFilter ) val nameToAddFileMap = generateCandidateFileMap(targetDeltaLog.dataPath, filesToRewrite) val touchedFilesWithDVs = DMLWithDeletionVectorsHelper .findFilesWithMatchingRows(deltaTxn, nameToAddFileMap, matchedDVResult) val (dvActions, metricsMap) = DMLWithDeletionVectorsHelper.processUnmodifiedData( spark, touchedFilesWithDVs, deltaTxn.snapshot, StatsCollectionUtils.getDataSkippingStringPrefixLength(spark, deltaTxn.metadata)) metrics("numTargetDeletionVectorsAdded") .set(metricsMap.getOrElse("numDeletionVectorsAdded", 0L)) metrics("numTargetDeletionVectorsRemoved") .set(metricsMap.getOrElse("numDeletionVectorsRemoved", 0L)) metrics("numTargetDeletionVectorsUpdated") .set(metricsMap.getOrElse("numDeletionVectorsUpdated", 0L)) // When DVs are enabled we override metrics related to removed files. metrics("numTargetFilesRemoved").set(metricsMap.getOrElse("numRemovedFiles", 0L)) val fullyRemovedFiles = touchedFilesWithDVs.filter(_.isFullyReplaced()).map(_.fileLogEntry) val (removedBytes, removedPartitions) = totalBytesAndDistinctPartitionValues(fullyRemovedFiles) metrics("numTargetBytesRemoved").set(removedBytes) metrics("numTargetPartitionsRemovedFrom").set(removedPartitions) dvActions } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/merge/InsertOnlyMergeExecutor.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.merge import org.apache.spark.sql.delta.metric.IncrementMetric import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions.{AddFile, FileAction} import org.apache.spark.sql.delta.commands.MergeIntoCommandBase import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.expressions.{Alias, CaseWhen, Expression, Literal} import org.apache.spark.sql.catalyst.plans.logical._ /** * Trait with optimized execution for merges that only inserts new data. * There are two cases for inserts only: when there are no matched clauses for the merge command * and when there is nothing matched for the merge command even if there are matched clauses. */ trait InsertOnlyMergeExecutor extends MergeOutputGeneration { self: MergeIntoCommandBase => import MergeIntoCommandBase._ /** * Optimization to write new files by inserting only new data. * * When there are no matched clauses for the merge command, data is skipped * based on the merge condition and left anti join is performed on the source * data to find the rows to be inserted. * * When there is nothing matched for the merge command even if there are matched clauses, * the source table is used to perform inserting. * * @param spark The spark session. * @param deltaTxn The existing transaction. * @param filterMatchedRows Whether to filter away matched data or not. * @param numSourceRowsMetric The name of the metric in which to record the number of source rows */ protected def writeOnlyInserts( spark: SparkSession, deltaTxn: OptimisticTransaction, filterMatchedRows: Boolean, numSourceRowsMetric: String): Seq[FileAction] = { val extraOpType = if (filterMatchedRows) { "writeInsertsOnlyWhenNoMatchedClauses" } else "writeInsertsOnlyWhenNoMatches" recordMergeOperation( extraOpType = extraOpType, status = "MERGE operation - writing new files for only inserts", sqlMetricName = "rewriteTimeMs") { // If nothing to do when not matched, then nothing to insert, that is, no new files to write if (!includesInserts && !filterMatchedRows) { performedSecondSourceScan = false return Seq.empty } // source DataFrame val mergeSource = getMergeSource // Expression to update metrics. val incrSourceRowCountExpr = incrementMetricAndReturnBool(numSourceRowsMetric, true) val sourceDF = filterSource(mergeSource.df.filter(Column(incrSourceRowCountExpr))) var dataSkippedFiles: Option[Seq[AddFile]] = None val preparedSourceDF = if (filterMatchedRows) { // This is an optimization of the case when there is no update clause for the merge. // We perform an left anti join on the source data to find the rows to be inserted. // Skip data based on the merge condition val conjunctivePredicates = splitConjunctivePredicates(condition) val targetOnlyPredicates = conjunctivePredicates.filter(_.references.subsetOf(target.outputSet)) dataSkippedFiles = Some(deltaTxn.filterFiles(targetOnlyPredicates)) val targetPlan = buildTargetPlanWithFiles( spark, deltaTxn, dataSkippedFiles.get, columnsToDrop = Nil) val targetDF = DataFrameUtils.ofRows(spark, targetPlan) sourceDF.join(targetDF, Column(condition), "leftanti") } else { sourceDF } val outputDF = generateInsertsOnlyOutputDF(spark, preparedSourceDF, deltaTxn) logDebug(s"$extraOpType: output plan:\n" + outputDF.queryExecution) val newFiles = writeFiles(spark, deltaTxn, outputDF) // Update metrics if (filterMatchedRows) { metrics("numTargetFilesBeforeSkipping") += deltaTxn.snapshot.numOfFiles metrics("numTargetBytesBeforeSkipping") += deltaTxn.snapshot.sizeInBytes if (dataSkippedFiles.nonEmpty) { val (afterSkippingBytes, afterSkippingPartitions) = totalBytesAndDistinctPartitionValues(dataSkippedFiles.get) metrics("numTargetFilesAfterSkipping") += dataSkippedFiles.get.size metrics("numTargetBytesAfterSkipping") += afterSkippingBytes metrics("numTargetPartitionsAfterSkipping") += afterSkippingPartitions } metrics("numTargetFilesRemoved") += 0 metrics("numTargetBytesRemoved") += 0 metrics("numTargetPartitionsRemovedFrom") += 0 } metrics("numTargetFilesAdded") += newFiles.count(_.isInstanceOf[AddFile]) val (addedBytes, addedPartitions) = totalBytesAndDistinctPartitionValues(newFiles) metrics("numTargetBytesAdded") += addedBytes metrics("numTargetPartitionsAddedTo") += addedPartitions newFiles } } private def filterSource(source: DataFrame): DataFrame = { // If there is only one insert clause, then filter out the source rows that do not // satisfy the clause condition because those rows will not be written out. if (notMatchedClauses.size == 1 && notMatchedClauses.head.condition.isDefined) { source.filter(Column(notMatchedClauses.head.condition.get)) } else { source } } /** * Generate the DataFrame to write out for merges that contains only inserts - either, insert-only * clauses or inserts when no matches were found. * * Specifically, it handles insert clauses in two cases: when there is only one insert clause, * and when there are multiple insert clauses. */ private def generateInsertsOnlyOutputDF( spark: SparkSession, preparedSourceDF: DataFrame, deltaTxn: OptimisticTransaction): DataFrame = { val targetWriteColNames = deltaTxn.metadata.schema.map(_.name) // When there is only one insert clause, there is no need for ROW_DROPPED_COL and // output df can be generated without CaseWhen. if (notMatchedClauses.size == 1) { val outputCols = generateOneInsertOutputCols(targetWriteColNames) return preparedSourceDF .filter(Column(incrementMetricAndReturnBool("numTargetRowsInserted", valueToReturn = true))) .select(outputCols: _*) } // Precompute conditions in insert clauses and generate source data frame with precomputed // boolean columns and insert clauses with rewritten conditions. val (sourceWithPrecompConditions, insertClausesWithPrecompConditions) = generatePrecomputedConditionsAndDF(preparedSourceDF, notMatchedClauses) // Generate output cols. val outputCols = generateInsertsOnlyOutputCols( targetWriteColNames, insertClausesWithPrecompConditions .collect { case c: DeltaMergeIntoNotMatchedInsertClause => c }) sourceWithPrecompConditions .select(outputCols: _*) .filter(s"$ROW_DROPPED_COL = false") .drop(ROW_DROPPED_COL) } /** * Generate output columns when there is only one insert clause. * * It assumes that the caller has already filtered out the source rows (`preparedSourceDF`) * that do not satisfy the insert clause condition (if any). * Then it simply applies the insertion action expression to generate * the output target table rows. */ private def generateOneInsertOutputCols( targetWriteColNames: Seq[String] ): Seq[Column] = { val outputExprs = notMatchedClauses.head.resolvedActions.map(_.expr) assert(outputExprs.nonEmpty) // generate the outputDF without `CaseWhen` expressions. outputExprs.zip(targetWriteColNames).map { case (expr, name) => Column(Alias(expr, name)()) } } /** * Generate the output columns for inserts only when there are multiple insert clauses. * * It combines all the conditions and corresponding actions expressions * into complicated CaseWhen expressions - one CaseWhen expression for * each column in the target row. If a source row does not match any of the clause conditions, * then the row will be dropped. These CaseWhen expressions basically look like this. * * For the i-th output column, * CASE * WHEN [insert condition 1] THEN [execute i-th expression of insert action 1] * WHEN [insert condition 2] THEN [execute i-th expression of insert action 2] * ELSE [mark the source row to be dropped] */ private def generateInsertsOnlyOutputCols( targetWriteColNames: Seq[String], insertClausesWithPrecompConditions: Seq[DeltaMergeIntoNotMatchedClause] ): Seq[Column] = { // ==== Generate the expressions to generate the target rows from the source rows ==== // If there are N columns in the target table, there will be N + 1 columns generated // - N columns for target table // - ROW_DROPPED_COL to define whether the generated row should be dropped or written out // To generate these N + 1 columns, we will generate N + 1 expressions val outputColNames = targetWriteColNames :+ ROW_DROPPED_COL val numOutputCols = outputColNames.size // Generate the sequence of N + 1 expressions from the sequence of INSERT clauses val allInsertExprs: Seq[Seq[Expression]] = insertClausesWithPrecompConditions.map { clause => clause.resolvedActions.map(_.expr) :+ incrementMetricAndReturnBool( "numTargetRowsInserted", false) } // Expressions to drop the source row when it does not match any of the insert clause // conditions. Note that it sets the N+1-th column ROW_DROPPED_COL to true. val dropSourceRowExprs = targetWriteColNames.map { _ => Literal(null)} :+ Literal.TrueLiteral // Generate the final N + 1 expressions to generate the final target output rows. // There are multiple not match clauses. Use `CaseWhen` to conditionally evaluate the right // action expressions to output columns. val outputExprs: Seq[Expression] = { val allInsertConditions = insertClausesWithPrecompConditions.map(_.condition.getOrElse(Literal.TrueLiteral)) (0 until numOutputCols).map { i => // For the i-th output column, generate // CASE // WHEN THEN // WHEN THEN // ... // val conditionalBranches = allInsertConditions.zip(allInsertExprs).map { case (notMatchCond, notMatchActionExprs) => notMatchCond -> notMatchActionExprs(i) } CaseWhen(conditionalBranches, dropSourceRowExprs(i)) } } assert(outputExprs.size == numOutputCols, s"incorrect # not matched expressions:\n\t" + seqToString(outputExprs)) logDebug("prepareInsertsOnlyOutputDF: not matched expressions\n\t" + seqToString(outputExprs)) outputExprs.zip(outputColNames).map { case (expr, name) => Column(Alias(expr, name)()) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/merge/MergeIntoMaterializeSource.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.merge import scala.annotation.tailrec import scala.util.control.NonFatal import org.apache.spark.sql.delta.{DataFrameUtils, DeltaErrors, DeltaLog} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.DeltaSparkPlanUtils import org.apache.spark.SparkException import org.apache.spark.internal.MDC import org.apache.spark.rdd.RDD import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession} import org.apache.spark.sql.catalyst.FileSourceOptions import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{AttributeSet, Expression} import org.apache.spark.sql.catalyst.optimizer.EliminateResolvedHint import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.execution.LogicalRDD import org.apache.spark.sql.execution.datasources.HadoopFsRelation import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.internal.SQLConf._ import org.apache.spark.sql.sources.BaseRelation import org.apache.spark.storage.StorageLevel /** * Trait with logic and utilities used for materializing a snapshot of MERGE source * in case we can't guarantee deterministic repeated reads from it. * * We materialize source if it is not safe to assume that it's deterministic * (override with MERGE_SOURCE_MATERIALIZATION). * Otherwise, if source changes between the phases of the MERGE, it can produce wrong results. * We use local checkpointing for the materialization, which saves the source as a * materialized RDD[InternalRow] on the executor local disks. * * 1st concern is that if an executor is lost, this data can be lost. * When Spark executor decommissioning API is used, it should attempt to move this * materialized data safely out before removing the executor. * * 2nd concern is that if an executor is lost for another reason (e.g. spot kill), we will * still lose that data. To mitigate that, we implement a retry loop. * The whole Merge operation needs to be restarted from the beginning in this case. * When we retry, we increase the replication level of the materialized data from 1 to 2. * (override with MERGE_SOURCE_MATERIALIZATION_RDD_STORAGE_LEVEL_RETRY). * If it still fails after the maximum number of attempts (MERGE_MATERIALIZE_SOURCE_MAX_ATTEMPTS), * we record the failure for tracking purposes. * * 3rd concern is that executors run out of disk space with the extra materialization. * We record such failures for tracking purposes. */ trait MergeIntoMaterializeSource extends DeltaLogging with DeltaSparkPlanUtils { import MergeIntoMaterializeSource._ protected def operation: String = "MERGE" protected def enableColumnPruningBeforeMaterialize: Boolean = true protected def materializeSourceErrorOpType: String = MergeIntoMaterializeSourceError.OP_TYPE protected def getMaterializeSourceMode(spark: SparkSession): String = spark.conf.get(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE) /** * Prepared Dataframe with source data. * If needed, it is materialized, @see prepareMergeSource */ private var mergeSource: Option[MergeSource] = None /** * If the source was materialized, reference to the checkpointed RDD. */ protected var materializedSourceRDD: Option[RDD[InternalRow]] = None /** * True if source materialization is used. * It is set when materializedSourceRDD may not yet be initialized. */ private var materializeSource = false /** * StorageLevel used for source materialization. * It is set when materializedSourceRDD may not yet be initialized. */ private var materializeSourceStorageLevel = StorageLevel.NONE /** * Track which attempt or retry it is in runWithMaterializedSourceAndRetries */ protected var attempt: Int = 0 /** * Run the Merge with retries in case it detects an RDD block lost error of the * materialized source RDD. * It will also record out of disk error, if such happens - possibly because of increased disk * pressure from the materialized source RDD. */ protected def runWithMaterializedSourceLostRetries( spark: SparkSession, deltaLog: DeltaLog, metrics: Map[String, SQLMetric], runOperationFunc: SparkSession => Seq[Row]): Seq[Row] = { var doRetry = false var runResult: Seq[Row] = null attempt = 1 do { doRetry = false metrics.values.foreach(_.reset()) try { runResult = runOperationFunc(spark) } catch { case NonFatal(ex) => val isLastAttempt = attempt == spark.conf.get(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE_MAX_ATTEMPTS) handleExceptionDuringAttempt(ex, isLastAttempt, deltaLog) match { case RetryHandling.Retry => logInfo(log"Retrying ${MDC(DeltaLogKeys.OPERATION, operation)} " + log"with materialized source." + log"Attempt ${MDC(DeltaLogKeys.NUM_ATTEMPT, attempt)} failed.") doRetry = true attempt += 1 case RetryHandling.ExhaustedRetries => logError(log"Exhausted retries after ${MDC(DeltaLogKeys.NUM_ATTEMPT, attempt)}" + log" attempts in ${MDC(DeltaLogKeys.OPERATION, operation)} " + log"with materialized source. Logging latest exception.", ex) throw DeltaErrors.sourceMaterializationFailedRepeatedlyInMerge case RetryHandling.RethrowException => logError(log"Fatal error in ${MDC(DeltaLogKeys.OPERATION, operation)} " + log"with materialized source in " + log"attempt ${MDC(DeltaLogKeys.NUM_ATTEMPT, attempt)}", ex) throw ex } } finally { // Remove source from RDD cache (noop if wasn't cached) materializedSourceRDD.foreach { rdd => rdd.unpersist() } materializedSourceRDD = None mergeSource = None materializeSource = false materializeSourceStorageLevel = StorageLevel.NONE } } while (doRetry) runResult } object RetryHandling extends Enumeration { type Result = Value val Retry, RethrowException, ExhaustedRetries = Value } /** * Handle exception that was thrown from runMerge(). * Search for errors to log, or that can be handled by retry. * It may need to descend into ex.getCause() to find the errors, as Spark may have wrapped them. * @param isLastAttempt indicates that it's the last allowed attempt and there shall be no retry. * @return true if the exception is handled and merge should retry * false if the caller should rethrow the error */ @tailrec private def handleExceptionDuringAttempt( ex: Throwable, isLastAttempt: Boolean, deltaLog: DeltaLog): RetryHandling.Result = ex match { // If Merge failed because the materialized source lost blocks from the // locally checkpointed RDD, we want to retry the whole operation. // If a checkpointed RDD block is lost, it throws // SparkCoreErrors.checkpointRDDBlockIdNotFoundError from LocalCheckpointRDD.compute. case s: SparkException if materializedSourceRDD.nonEmpty && s.getErrorClass == "CHECKPOINT_RDD_BLOCK_ID_NOT_FOUND" && s.getMessageParameters.get("rddBlockId").contains(s"rdd_${materializedSourceRDD.get.id}") => logWarning(log"Materialized ${MDC(DeltaLogKeys.OPERATION, operation)} source RDD block " + log"lost. ${MDC(DeltaLogKeys.OPERATION, operation)} needs to be restarted. " + log"This was attempt number ${MDC(DeltaLogKeys.ATTEMPT, attempt)}.") if (!isLastAttempt) { RetryHandling.Retry } else { // Record situations where we lost RDD materialized source blocks, despite retries. recordDeltaEvent( deltaLog, materializeSourceErrorOpType, data = MergeIntoMaterializeSourceError( errorType = MergeIntoMaterializeSourceErrorType.RDD_BLOCK_LOST.toString, attempt = attempt, materializedSourceRDDStorageLevel = materializedSourceRDD.get.getStorageLevel.toString ) ) RetryHandling.ExhaustedRetries } // Record if we ran out of executor disk space when we materialized the source. case s: SparkException if materializeSource && s.getMessage.contains("java.io.IOException: No space left on device") => // Record situations where we ran out of disk space, possibly because of the space took // by the materialized RDD. recordDeltaEvent( deltaLog, materializeSourceErrorOpType, data = MergeIntoMaterializeSourceError( errorType = MergeIntoMaterializeSourceErrorType.OUT_OF_DISK.toString, attempt = attempt, materializedSourceRDDStorageLevel = materializeSourceStorageLevel.toString ) ) RetryHandling.RethrowException // Descend into ex.getCause. // The errors that we are looking for above might have been wrapped inside another exception. case NonFatal(ex) if ex.getCause() != null => handleExceptionDuringAttempt(ex.getCause(), isLastAttempt, deltaLog) // Descended to the bottom of the causes without finding a retryable error case _ => RetryHandling.RethrowException } private def planContainsIgnoreUnreadableFilesReadOptions(plan: LogicalPlan): Boolean = { def relationContainsOptions(relation: BaseRelation): Boolean = { relation match { case hdpRelation: HadoopFsRelation => hdpRelation.options.get(FileSourceOptions.IGNORE_CORRUPT_FILES).contains("true") || hdpRelation.options.get(FileSourceOptions.IGNORE_MISSING_FILES).contains("true") case _ => false } } val res = plan.collectFirst { case lr: LogicalRelation if relationContainsOptions(lr.relation) => lr } res.nonEmpty } private def ignoreUnreadableFilesConfigsAreSet(plan: LogicalPlan, spark: SparkSession) : Boolean = { spark.conf.get(IGNORE_MISSING_FILES) || spark.conf.get(IGNORE_CORRUPT_FILES) || planContainsIgnoreUnreadableFilesReadOptions(plan) } /** * @return pair of boolean whether source should be materialized * and the source materialization reason */ protected def shouldMaterializeSource( spark: SparkSession, source: LogicalPlan, isInsertOnly: Boolean ): (Boolean, MergeIntoMaterializeSourceReason.MergeIntoMaterializeSourceReason) = { val materializeType = getMaterializeSourceMode(spark) val forceMaterializationWithUnreadableFiles = spark.conf.get(DeltaSQLConf.MERGE_FORCE_SOURCE_MATERIALIZATION_WITH_UNREADABLE_FILES) import DeltaSQLConf.MergeMaterializeSource._ val checkDeterministicOptions = DeltaSparkPlanUtils.CheckDeterministicOptions(allowDeterministicUdf = true) val materializeCachedSource = spark.conf.get(DeltaSQLConf.MERGE_MATERIALIZE_CACHED_SOURCE) materializeType match { case ALL => (true, MergeIntoMaterializeSourceReason.MATERIALIZE_ALL) case NONE => (false, MergeIntoMaterializeSourceReason.NOT_MATERIALIZED_NONE) case AUTO => if (isInsertOnly && spark.conf.get(DeltaSQLConf.MERGE_INSERT_ONLY_ENABLED)) { (false, MergeIntoMaterializeSourceReason.NOT_MATERIALIZED_AUTO_INSERT_ONLY) } else if (!planContainsOnlyDeltaScans(source)) { (true, MergeIntoMaterializeSourceReason.NON_DETERMINISTIC_SOURCE_NON_DELTA) } else if (!planIsDeterministic(source, checkDeterministicOptions)) { (true, MergeIntoMaterializeSourceReason.NON_DETERMINISTIC_SOURCE_OPERATORS) // Force source materialization if Spark configs IGNORE_CORRUPT_FILES, // IGNORE_MISSING_FILES or file source read options FileSourceOptions.IGNORE_CORRUPT_FILES // FileSourceOptions.IGNORE_MISSING_FILES are enabled on the source. // This is done so to prevent irrecoverable data loss or unexpected results. } else if (forceMaterializationWithUnreadableFiles && ignoreUnreadableFilesConfigsAreSet(source, spark)) { (true, MergeIntoMaterializeSourceReason.IGNORE_UNREADABLE_FILES_CONFIGS_ARE_SET) } else if (planContainsUdf(source)) { // Force source materialization if the source contains a User Defined Function, even if // the user defined function is marked as deterministic, as it is often incorrectly marked // as such. (true, MergeIntoMaterializeSourceReason.NON_DETERMINISTIC_SOURCE_WITH_DETERMINISTIC_UDF) } else if (materializeCachedSource && planContainsCachedRelation(DataFrameUtils.ofRows(spark, source))) { // The query cache doesn't pin the version of cached Delta tables, cache can get // concurrently updated in the middle of MERGE execution. We materialize the source in // that case to avoid this issue. (true, MergeIntoMaterializeSourceReason.SOURCE_CACHED) } else { (false, MergeIntoMaterializeSourceReason.NOT_MATERIALIZED_AUTO) } case _ => // If the config is invalidly set, also materialize. (true, MergeIntoMaterializeSourceReason.INVALID_CONFIG) } } /** * If source needs to be materialized, prepare the materialized dataframe in sourceDF * Otherwise, prepare regular dataframe. * @return the source materialization reason */ protected def prepareMergeSource( spark: SparkSession, source: LogicalPlan, condition: Expression, matchedClauses: Seq[DeltaMergeIntoMatchedClause], notMatchedClauses: Seq[DeltaMergeIntoNotMatchedClause], isInsertOnly: Boolean): Unit = { val (materialize, materializeReason) = shouldMaterializeSource(spark, source, isInsertOnly) materializeSource = materialize if (!materialize) { // Does not materialize, simply return the dataframe from source plan mergeSource = Some( MergeSource( df = DataFrameUtils.ofRows(spark, source), isMaterialized = false, materializeReason = materializeReason ) ) return } val referencedSourceColumns = if (enableColumnPruningBeforeMaterialize) { getReferencedSourceColumns(source, condition, matchedClauses, notMatchedClauses) } else { assert(matchedClauses.isEmpty && notMatchedClauses.isEmpty, "If column pruning is disabled, then there should be no MERGE clauses.") assert(operation != "MERGE", "Column pruning before materialization must be done for MERGE.") source.output } val baseSourcePlanDF = if (enableColumnPruningBeforeMaterialize) { // When we materialize the source, we want to make sure that columns got pruned // before caching. val sourceWithSelectedColumns = Project(referencedSourceColumns, source) DataFrameUtils.ofRows(spark, sourceWithSelectedColumns) } else { DataFrameUtils.ofRows(spark, source) } // Select appropriate StorageLevel materializeSourceStorageLevel = StorageLevel.fromString( if (attempt == 1) { spark.conf.get(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE_RDD_STORAGE_LEVEL) } else if (attempt == 2) { // If it failed the first time, potentially use a different storage level on retry. The // first retry has its own conf to allow gradually increasing the replication level. spark.conf.get(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE_RDD_STORAGE_LEVEL_FIRST_RETRY) } else { spark.conf.get(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE_RDD_STORAGE_LEVEL_RETRY) } ) // Caches the source in RDD cache using localCheckpoint, which cuts away the RDD lineage, // which shall ensure that the source cannot be recomputed and thus become inconsistent. // // WARNING: if eager == false, the source used during the first Spark Job that uses this may // still be inconsistent with source materialized afterwards. // This is because doCheckpoint that finalizes the lazy checkpoint is called after the Job // that triggered the lazy checkpointing finished. // If blocks were lost during that job, they may still get recomputed and changed compared // to how they were used during the execution of the job. val checkpointedSourcePlanDF = baseSourcePlanDF.localCheckpoint( eager = true, storageLevel = materializeSourceStorageLevel) // We have to reach through the crust and into the plan of the checkpointed DF // to get the RDD that was actually checkpointed, to be able to unpersist it later... var checkpointedPlan = checkpointedSourcePlanDF.queryExecution.analyzed val rdd = checkpointedPlan.asInstanceOf[LogicalRDD].rdd assert(rdd.isCheckpointed) materializedSourceRDD = Some(rdd) rdd.setName("mergeMaterializedSource") // We should still keep the hints from the input plan. checkpointedPlan = addHintsToPlan(source, checkpointedPlan) mergeSource = Some( MergeSource( df = DataFrameUtils.ofRows(spark, checkpointedPlan), isMaterialized = true, materializeReason = materializeReason ) ) logDebug(s"Materializing $operation with pruned columns $referencedSourceColumns.") logDebug(s"Materialized $operation source plan:\n${getMergeSource.df.queryExecution}") } /** Returns the prepared merge source. */ protected def getMergeSource: MergeSource = mergeSource match { case Some(source) => source case None => throw new IllegalStateException( "mergeSource was not initialized! Call prepareMergeSource before.") } private def addHintsToPlan(sourcePlan: LogicalPlan, plan: LogicalPlan): LogicalPlan = { val hints = EliminateResolvedHint.extractHintsFromPlan(sourcePlan)._2 // This follows similar code in CacheManager from https://github.com/apache/spark/pull/24580 if (hints.nonEmpty) { // The returned hint list is in top-down order, we should create the hint nodes from // right to left. val planWithHints = hints.foldRight[LogicalPlan](plan) { case (hint, p) => ResolvedHint(p, hint) } planWithHints } else { plan } } } object MergeIntoMaterializeSource { case class MergeSource( df: DataFrame, isMaterialized: Boolean, materializeReason: MergeIntoMaterializeSourceReason.MergeIntoMaterializeSourceReason) { assert(!isMaterialized || MergeIntoMaterializeSourceReason.MATERIALIZED_REASONS.contains(materializeReason)) } /** * @return The columns of the source plan that are used in this MERGE */ private def getReferencedSourceColumns( source: LogicalPlan, condition: Expression, matchedClauses: Seq[DeltaMergeIntoMatchedClause], notMatchedClauses: Seq[DeltaMergeIntoNotMatchedClause]) = { val conditionCols = condition.references val matchedCondCols = matchedClauses.flatMap(_.condition).flatMap(_.references) val notMatchedCondCols = notMatchedClauses.flatMap(_.condition).flatMap(_.references) val matchedActionsCols = matchedClauses .flatMap(_.resolvedActions) .flatMap(_.expr.references) val notMatchedActionsCols = notMatchedClauses .flatMap(_.resolvedActions) .flatMap(_.expr.references) val allCols = AttributeSet( conditionCols ++ matchedCondCols ++ notMatchedCondCols ++ matchedActionsCols ++ notMatchedActionsCols) source.output.filter(allCols.contains) } } /** * Enumeration with possible reasons that source may be materialized in a MERGE command. */ object MergeIntoMaterializeSourceReason extends Enumeration { type MergeIntoMaterializeSourceReason = Value // It was determined to not materialize on auto config. val NOT_MATERIALIZED_AUTO = Value("notMaterializedAuto") // Config was set to never materialize source. val NOT_MATERIALIZED_NONE = Value("notMaterializedNone") // Insert only merge is single pass, no need for materialization val NOT_MATERIALIZED_AUTO_INSERT_ONLY = Value("notMaterializedAutoInsertOnly") // Config was set to always materialize source. val MATERIALIZE_ALL = Value("materializeAll") // The source query is considered non-deterministic, because it contains a non-delta scan. val NON_DETERMINISTIC_SOURCE_NON_DELTA = Value("materializeNonDeterministicSourceNonDelta") // The source query is considered non-deterministic, because it contains non-deterministic // operators. val NON_DETERMINISTIC_SOURCE_OPERATORS = Value("materializeNonDeterministicSourceOperators") // Either spark configs to ignore unreadable files are set or the source plan contains relations // with ignore unreadable files options. val IGNORE_UNREADABLE_FILES_CONFIGS_ARE_SET = Value("materializeIgnoreUnreadableFilesConfigsAreSet") // The source query is considered non-determistic because it contains a User Defined Function. val NON_DETERMINISTIC_SOURCE_WITH_DETERMINISTIC_UDF = Value("materializeNonDeterministicSourceWithDeterministicUdf") // Materialize when the configuration is invalid val INVALID_CONFIG = Value("invalidConfigurationFailsafe") // Materialize when the source is cached. val SOURCE_CACHED = Value("materializeCachedSource") // Catch-all case. val UNKNOWN = Value("unknown") // Set of reasons that result in source materialization. final val MATERIALIZED_REASONS: Set[MergeIntoMaterializeSourceReason] = Set( MATERIALIZE_ALL, NON_DETERMINISTIC_SOURCE_NON_DELTA, NON_DETERMINISTIC_SOURCE_OPERATORS, IGNORE_UNREADABLE_FILES_CONFIGS_ARE_SET, NON_DETERMINISTIC_SOURCE_WITH_DETERMINISTIC_UDF, INVALID_CONFIG, SOURCE_CACHED ) } /** * Structure with data for "delta.dml.merge.materializeSourceError" event. * Note: We log only errors that we want to track (out of disk or lost RDD blocks). */ case class MergeIntoMaterializeSourceError( errorType: String, attempt: Int, materializedSourceRDDStorageLevel: String ) object MergeIntoMaterializeSourceError { val OP_TYPE = "delta.dml.merge.materializeSourceError" } object MergeIntoMaterializeSourceErrorType extends Enumeration { type MergeIntoMaterializeSourceError = Value val RDD_BLOCK_LOST = Value("materializeSourceRDDBlockLostRetriesFailure") val OUT_OF_DISK = Value("materializeSourceOutOfDiskFailure") } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/merge/MergeOutputGeneration.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.merge import scala.collection.mutable import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.{RowCommitVersion, RowId} import org.apache.spark.sql.delta.commands.MergeIntoCommandBase import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.functions._ /** * Contains logic to transform the merge clauses into expressions that can be evaluated to obtain * the output of the merge operation. */ trait MergeOutputGeneration { self: MergeIntoCommandBase => import CDCReader._ import MergeIntoCommandBase._ import MergeOutputGeneration._ /** * Precompute conditions in MATCHED and NOT MATCHED clauses and generate the source * data frame with precomputed boolean columns. * @param sourceDF the source DataFrame. * @param clauses the merge clauses to precompute. * @return Generated sourceDF with precomputed boolean columns, matched clauses with * possible rewritten clause conditions, insert clauses with possible rewritten * clause conditions */ protected def generatePrecomputedConditionsAndDF( sourceDF: DataFrame, clauses: Seq[DeltaMergeIntoClause]): (DataFrame, Seq[DeltaMergeIntoClause]) = { // // ==== Precompute conditions in MATCHED and NOT MATCHED clauses ==== // If there are conditions in the clauses, each condition will be computed once for every // column (and obviously for every row) within the per-column CaseWhen expressions. Since, the // conditions can be arbitrarily expensive, it is likely to be more efficient to // precompute them into boolean columns and use these new columns in the CaseWhen exprs. // Then each condition will be computed only once per row, and the resultant boolean reused // for all the columns in the row. // val preComputedClauseConditions = new mutable.ArrayBuffer[(String, Expression)]() // Rewrite clause condition into a simple lookup of precomputed column def rewriteCondition[T <: DeltaMergeIntoClause](clause: T): T = { clause.condition match { case Some(conditionExpr) => val colName = s"""_${clause.clauseType}${PRECOMPUTED_CONDITION_COL} |${preComputedClauseConditions.length}_ |""".stripMargin.replaceAll("\n", "") // ex: _update_condition_0_ preComputedClauseConditions += ((colName, conditionExpr)) clause.makeCopy(Array(Some(UnresolvedAttribute(colName)), clause.actions)).asInstanceOf[T] case None => clause } } // Get the clauses with possibly rewritten clause conditions. // This will automatically populate the `preComputedClauseConditions` // (as part of `rewriteCondition`) val clausesWithPrecompConditions = clauses.map(rewriteCondition) // Add the columns to the given `sourceDF` to precompute clause conditions val sourceWithPrecompConditions = { val newCols = preComputedClauseConditions.map { case (colName, conditionExpr) => Column(conditionExpr).as(colName) }.toSeq sourceDF.select(col("*") +: newCols: _*) } (sourceWithPrecompConditions, clausesWithPrecompConditions) } /** * Generate the expressions to process full-outer join output and generate target rows. * * To generate these N + 2 columns, we generate N + 2 expressions and apply them * on the joinedDF. The CDC column will be either used for CDC generation or dropped before * performing the final write, and the other column will always be dropped after executing the * increment metric expression and filtering on ROW_DROPPED_COL. */ protected def generateWriteAllChangesOutputCols( targetWriteCols: Seq[Expression], rowIdColumnExpressionOpt: Option[NamedExpression], rowCommitVersionColumnExpressionOpt: Option[NamedExpression], targetWriteColNames: Seq[String], noopCopyExprs: Seq[Expression], writeUnmodifiedRows: Boolean, clausesWithPrecompConditions: Seq[DeltaMergeIntoClause], cdcEnabled: Boolean, needSetRowTrackingFieldIdForUniform: Boolean, shouldCountDeletedRows: Boolean = true): IndexedSeq[Column] = { val numOutputCols = targetWriteColNames.size // ==== Generate N + 2 (N + 4 preserving Row Tracking) expressions for MATCHED clauses ==== val processedMatchClauses: Seq[ProcessedClause] = generateAllActionExprs( targetWriteCols, rowIdColumnExpressionOpt, rowCommitVersionColumnExpressionOpt, clausesWithPrecompConditions.collect { case c: DeltaMergeIntoMatchedClause => c }, cdcEnabled, shouldCountDeletedRows) val matchedExprs: Seq[Expression] = generateClauseOutputExprs( numOutputCols, processedMatchClauses, noopCopyExprs, writeUnmodifiedRows) // N + 1 (or N + 2 with CDC, N + 4 preserving Row Tracking and CDC) expressions to delete the // unmatched source row when it should not be inserted. `target.output` will produce NULLs // which will get deleted eventually. val deletedColsForUnmatchedTarget = if (cdcEnabled) targetWriteCols else targetWriteCols.map(e => Cast(Literal(null), e.dataType)) val deleteSourceRowExprs = (deletedColsForUnmatchedTarget ++ rowIdColumnExpressionOpt.map(_ => Literal(null)) ++ rowCommitVersionColumnExpressionOpt.map(_ => Literal(null)) ++ Seq(Literal(true))) ++ (if (cdcEnabled) Seq(CDC_TYPE_NOT_CDC) else Seq()) // ==== Generate N + 2 (N + 4 preserving Row Tracking) expressions for NOT MATCHED clause ==== val processedNotMatchClauses: Seq[ProcessedClause] = generateAllActionExprs( targetWriteCols, rowIdColumnExpressionOpt, rowCommitVersionColumnExpressionOpt, clausesWithPrecompConditions.collect { case c: DeltaMergeIntoNotMatchedClause => c }, cdcEnabled, shouldCountDeletedRows) val notMatchedExprs: Seq[Expression] = generateClauseOutputExprs( numOutputCols, processedNotMatchClauses, deleteSourceRowExprs, writeUnmodifiedRows) // === Generate N + 2 (N + 4 with Row Tracking) expressions for NOT MATCHED BY SOURCE clause === val processedNotMatchBySourceClauses: Seq[ProcessedClause] = generateAllActionExprs( targetWriteCols, rowIdColumnExpressionOpt, rowCommitVersionColumnExpressionOpt, clausesWithPrecompConditions.collect { case c: DeltaMergeIntoNotMatchedBySourceClause => c }, cdcEnabled, shouldCountDeletedRows) val notMatchedBySourceExprs: Seq[Expression] = generateClauseOutputExprs( numOutputCols, processedNotMatchBySourceClauses, noopCopyExprs, writeUnmodifiedRows) // ==== Generate N + 2 (N + 4 preserving Row Tracking) expressions that invokes the MATCHED, // NOT MATCHED and NOT MATCHED BY SOURCE expressions ==== // That is, conditionally invokes them based on whether there was a match in the outer join. // Predicates to check whether there was a match in the full outer join. val ifSourceRowNull = expression(col(SOURCE_ROW_PRESENT_COL).isNull) val ifTargetRowNull = expression(col(TARGET_ROW_PRESENT_COL).isNull) val outputCols = targetWriteColNames.zipWithIndex.map { case (name, i) => // Coupled with the clause conditions, the resultant possibly-nested CaseWhens can // be the following for every i-th column. (In the case with single matched/not-matched // clauses, instead of nested CaseWhens, there will be If/Else.) // // CASE WHEN (source row is null) // CASE WHEN // THEN // WHEN // THEN // ... // ELSE // // WHEN (target row is null) // THEN // CASE WHEN // THEN // WHEN // THEN // ... // ELSE // // ELSE (both source and target row are not null) // CASE WHEN // THEN // WHEN // THEN // ... // ELSE // val caseWhen = CaseWhen(Seq( ifSourceRowNull -> notMatchedBySourceExprs(i), ifTargetRowNull -> notMatchedExprs(i)), /* otherwise */ matchedExprs(i)) if (rowIdColumnExpressionOpt.exists(_.name == name)) { // Add Row ID metadata to allow writing the column. Column(Alias(caseWhen, name)( explicitMetadata = Some(RowId.columnMetadata(name, needSetRowTrackingFieldIdForUniform)))) } else if (rowCommitVersionColumnExpressionOpt.exists(_.name == name)) { // Add Row Commit Versions metadata to allow writing the column. Column(Alias(caseWhen, name)( explicitMetadata = Some( RowCommitVersion.columnMetadata(name, needSetRowTrackingFieldIdForUniform)))) } else { Column(Alias(caseWhen, name)()) } } logDebug("writeAllChanges: join output expressions\n\t" + seqToString( outputCols.map(expression))) outputCols }.toIndexedSeq /** * Represents a merge clause after its condition and action expressions have been processed before * generating the final output expression. * @param condition Optional precomputed condition. * @param actions List of output expressions generated from every action of the clause. */ protected case class ProcessedClause(condition: Option[Expression], actions: Seq[Expression]) /** * Generate expressions for every output column and every merge clause based on the corresponding * UPDATE, DELETE and/or INSERT action(s). * @param targetWriteCols List of output column expressions from the target table. Used to * generate CDC data for DELETE. * @param rowIdColumnExpressionOpt The optional Row ID preservation column with the physical * Row ID name, it stores stable Row IDs of the table. * @param rowCommitVersionColumnExpressionOpt The optional Row Commit Version preservation * column with the physical Row Commit Version name, it stores * stable Row Commit Versions. * @param clausesWithPrecompConditions List of merge clauses with precomputed conditions. Action * expressions are generated for each of these clauses. * @param cdcEnabled Whether the generated expressions should include CDC information. * @param shouldCountDeletedRows Whether metrics for number of deleted rows should be incremented * here. * @return For each merge clause, a list of [[ProcessedClause]] each with a precomputed * condition and N+2 action expressions (N output columns + [[ROW_DROPPED_COL]] + * [[CDC_TYPE_COLUMN_NAME]]) to apply on a row when that clause matches. */ protected def generateAllActionExprs( targetWriteCols: Seq[Expression], rowIdColumnExpressionOpt: Option[NamedExpression], rowCommitVersionColumnExpressionOpt: Option[NamedExpression], clausesWithPrecompConditions: Seq[DeltaMergeIntoClause], cdcEnabled: Boolean, shouldCountDeletedRows: Boolean): Seq[ProcessedClause] = { clausesWithPrecompConditions.map { clause => val actions = clause match { // Seq of up to N+3 expressions to generate output rows based on the UPDATE, DELETE and/or // INSERT action(s) case u: DeltaMergeIntoMatchedUpdateClause => val incrCountExpr = incrementMetricsAndReturnBool( names = Seq("numTargetRowsUpdated", "numTargetRowsMatchedUpdated"), valueToReturn = false) // Generate update expressions and set ROW_DROPPED_COL = false u.resolvedActions.map(_.expr) ++ rowIdColumnExpressionOpt ++ rowCommitVersionColumnExpressionOpt.map(_ => Literal(null)) ++ Seq(incrCountExpr) ++ (if (cdcEnabled) Some(Literal(CDC_TYPE_UPDATE_POSTIMAGE)) else None) case u: DeltaMergeIntoNotMatchedBySourceUpdateClause => val incrCountExpr = incrementMetricsAndReturnBool( names = Seq("numTargetRowsUpdated", "numTargetRowsNotMatchedBySourceUpdated"), valueToReturn = false) // Generate update expressions and set ROW_DROPPED_COL = false u.resolvedActions.map(_.expr) ++ rowIdColumnExpressionOpt ++ rowCommitVersionColumnExpressionOpt.map(_ => Literal(null)) ++ Seq(incrCountExpr) ++ (if (cdcEnabled) Some(Literal(CDC_TYPE_UPDATE_POSTIMAGE)) else None) case _: DeltaMergeIntoMatchedDeleteClause => val incrCountExpr = { if (shouldCountDeletedRows) { incrementMetricsAndReturnBool( names = Seq("numTargetRowsDeleted", "numTargetRowsMatchedDeleted"), valueToReturn = true) } else { Literal.TrueLiteral } } // Generate expressions to set the ROW_DROPPED_COL = true and mark as a DELETE // Only read full target columns if CDC is enabled val deletedDataExprs = if (cdcEnabled) { targetWriteCols } else { targetWriteCols.map(e => Cast(Literal(null), e.dataType)) } deletedDataExprs ++ rowIdColumnExpressionOpt ++ rowCommitVersionColumnExpressionOpt ++ Seq(incrCountExpr) ++ (if (cdcEnabled) Some(CDC_TYPE_DELETE) else None) case _: DeltaMergeIntoNotMatchedBySourceDeleteClause => val incrCountExpr = { if (shouldCountDeletedRows) { incrementMetricsAndReturnBool( names = Seq("numTargetRowsDeleted", "numTargetRowsNotMatchedBySourceDeleted"), valueToReturn = true) } else { Literal.TrueLiteral } } // Generate expressions to set the ROW_DROPPED_COL = true and mark as a DELETE // Only read full target columns if CDC is enabled val deletedColsForUnmatchedTarget = if (cdcEnabled) targetWriteCols else targetWriteCols.map(e => Cast(Literal(null), e.dataType)) deletedColsForUnmatchedTarget ++ rowIdColumnExpressionOpt ++ rowCommitVersionColumnExpressionOpt ++ Seq(incrCountExpr) ++ (if (cdcEnabled) Some(CDC_TYPE_DELETE) else None) case i: DeltaMergeIntoNotMatchedInsertClause => val incrInsertedCountExpr = incrementMetricsAndReturnBool( names = Seq("numTargetRowsInserted"), valueToReturn = false) i.resolvedActions.map(_.expr) ++ rowIdColumnExpressionOpt.map(_ => Literal(null)) ++ rowCommitVersionColumnExpressionOpt.map(_ => Literal(null)) ++ Seq(incrInsertedCountExpr) ++ (if (cdcEnabled) Some(Literal(CDC_TYPE_INSERT)) else None) } ProcessedClause(clause.condition, actions) } } /** * Generate the output expression for each output column to apply the correct action for a type of * merge clause. For each output column, the resulting expression dispatches the correct action * based on all clause conditions. * @param numOutputCols Number of output columns. * @param clauses List of preprocessed merge clauses to bind together. * @param noopCopyExprs The expressions to use for unmatched rows if writeUnmodifiedRows is true. * @param writeUnmodifiedRows Whether to fallback to noopCopyExprs for unmatched rows. * @return A list of one expression per output column to apply for a type of merge clause. */ protected def generateClauseOutputExprs( numOutputCols: Int, clauses: Seq[ProcessedClause], noopCopyExprs: Seq[Expression], writeUnmodifiedRows: Boolean ): Seq[Expression] = { val clauseExprs = { if (clauses.isEmpty) { if (writeUnmodifiedRows) { noopCopyExprs } else { // In this case, merge-on-read is enabled *and* there is no action defined for // the MATCHED, NOT MATCHED or NOT MATCHED BY SOURCE cases. // In this case, these expressions will never be evaluated, because // we filtered to rows that match at least one merge clause. // Returning RaiseError here is a sanity check to ensure that the code is correct. val errExpr = RaiseError( Literal("Unexpected row: did not match any merge clause") ) Seq.fill(numOutputCols)(errExpr) } } else if (clauses.head.condition.isEmpty) { // Only one clause without any condition, so the corresponding action expressions // can be evaluated directly to generate the output columns. clauses.head.actions } else if (clauses.length == 1) { // Only one clause _with_ a condition, so generate IF/THEN instead of CASE WHEN. // For the i-th output column, generate // IF THEN // ELSE fallback (noopCopyExprs or RaiseError) val condition = clauses.head.condition.get clauses.head.actions.zipWithIndex.map { case (a, i) => if (writeUnmodifiedRows) { If(condition, a, noopCopyExprs(i)) } else { // Since writeUnmodifiedRows is false, we know we will never reach the else branch // because we filtered to rows that match at least one merge clause. If(condition, a, RaiseError(Literal("Unexpected row: did not match any merge clause"))) } } } else { // There are multiple clauses. Use `CaseWhen` to conditionally evaluate the right // action expressions to output columns Seq.range(0, numOutputCols).map { i => // For the i-th output column, generate // CASE // WHEN THEN // WHEN THEN // ... // ELSE fallback (noopCopyExprs or RaiseError) // // val conditionalBranches = clauses.map { precomp => precomp.condition.getOrElse(Literal.TrueLiteral) -> precomp.actions(i) } val elseBranch = if (writeUnmodifiedRows) Some(noopCopyExprs(i)) // Since writeUnmodifiedRows is false, we know we will never reach the else branch // because we filtered to rows that match at least one merge clause. else Some( RaiseError( Literal("Unexpected row: did not match any merge clause") ) ) CaseWhen(conditionalBranches, elseBranch) } } } // If there are clauses, we should have the correct number of expressions. assert(clauseExprs.size == numOutputCols, s"incorrect # expressions:\n\t" + seqToString(clauseExprs)) logDebug(s"writeAllChanges: expressions\n\t" + seqToString(clauseExprs)) clauseExprs } /** * Build the full output as an array of packed rows, then explode into the final result. Based * on the CDC type as originally marked, we produce both rows for the CDC_TYPE_NOT_CDC partition * to be written to the main table and rows for the CDC partitions to be written as CDC files. * * See [[CDCReader]] for general details on how partitioning on the CDC type column works. */ protected def generateCdcAndOutputRows( sourceDf: DataFrame, outputCols: Seq[Column], outputColNames: Seq[String], noopCopyExprs: Seq[Expression], rowIdColumnNameOpt: Option[String], rowCommitVersionColumnNameOpt: Option[String], deduplicateDeletes: DeduplicateCDFDeletes, needSetRowTrackingFieldIdForUniform: Boolean): DataFrame = { import org.apache.spark.sql.delta.commands.cdc.CDCReader._ // The main partition just needs to swap in the CDC_TYPE_NOT_CDC value. val mainDataOutput = outputCols.dropRight(1) :+ Column(CDC_TYPE_NOT_CDC).as(CDC_TYPE_COLUMN_NAME) // Deleted rows are sent to the CDC partition instead of the main partition. These rows are // marked as dropped, we need to retain them while incrementing the original metric column // ourselves. val keepRowAndIncrDeletedCountExpr = !outputCols(outputCols.length - 2) val deleteCdcOutput = outputCols .updated(outputCols.length - 2, keepRowAndIncrDeletedCountExpr.as(ROW_DROPPED_COL)) // Update preimages need special handling. This is conceptually the same as the // transformation for cdcOutputCols, but we have to transform the noop exprs to columns // ourselves because it hasn't already been done. val cdcNoopExprs = noopCopyExprs.dropRight(2) :+ Literal.FalseLiteral :+ Literal(CDC_TYPE_UPDATE_PREIMAGE) val updatePreimageCdcOutput = cdcNoopExprs.zipWithIndex.map { case (e, i) => Column(Alias(e, outputColNames(i))()) } // To avoid duplicate evaluation of nondeterministic column values such as // [[GenerateIdentityValues]], we EXPLODE CDC rows first, from which we EXPLODE again, // and for each of "insert" and "update_postimage" rows, generate main data rows. // The first EXPLODE will force evaluation all nondeterministic expressions, // and the second EXPLODE will just copy the generated value from CDC rows // to main data. By doing so we ensure nondeterministic column values in CDC and // main data rows stay the same. val cdcTypeCol = outputCols.last val cdcArray = Column(CaseWhen(Seq( EqualNullSafe(expression(cdcTypeCol), Literal(CDC_TYPE_INSERT)) -> expression(array( struct(outputCols: _*))), EqualNullSafe(expression(cdcTypeCol), Literal(CDC_TYPE_UPDATE_POSTIMAGE)) -> expression(array( struct(updatePreimageCdcOutput: _*), struct(outputCols: _*))), EqualNullSafe(expression(cdcTypeCol), CDC_TYPE_DELETE) -> expression(array( struct(deleteCdcOutput: _*)))), // If none of the CDC cases apply (true for purely rewritten target rows, dropped source // rows, etc.) just stick to the normal output. expression(array(struct(mainDataOutput: _*))) )) val cdcToMainDataArray = Column(If( Or( EqualNullSafe(expression(col(s"packedCdc.$CDC_TYPE_COLUMN_NAME")), Literal(CDC_TYPE_INSERT)), EqualNullSafe(expression(col(s"packedCdc.$CDC_TYPE_COLUMN_NAME")), Literal(CDC_TYPE_UPDATE_POSTIMAGE))), expression(array( col("packedCdc"), struct( outputColNames .dropRight(1) .map { n => col(s"packedCdc.`$n`") } :+ Column(CDC_TYPE_NOT_CDC).as(CDC_TYPE_COLUMN_NAME): _*) )), expression(array(col("packedCdc"))) )) if (deduplicateDeletes.enabled) { deduplicateCDFDeletes( deduplicateDeletes, sourceDf, cdcArray, cdcToMainDataArray, rowIdColumnNameOpt, rowCommitVersionColumnNameOpt, outputColNames, needSetRowTrackingFieldIdForUniform ) } else { packAndExplodeCDCOutput( sourceDf, cdcArray, cdcToMainDataArray, rowIdColumnNameOpt, rowCommitVersionColumnNameOpt, outputColNames, dedupColumns = Nil, needSetRowTrackingFieldIdForUniform) } } /** * Applies the transformations to generate the CDC output: * 1. Transform each input row into its corresponding array of CDC rows, e.g. an updated row * yields: array(update_preimage, update_postimage). * 2. Add the main data output for inserted/updated rows to the previously packed CDC data. * 3. Explode the result to flatten the packed arrays. * * @param sourceDf The dataframe generated after processing the merge output. * @param cdcArray Transforms the merge output into the corresponding packed CDC data that will be * written to the CDC partition. * @param cdcToMainDataArray Transforms the packed CDC data to add the main data output, i.e. rows * that are inserted or updated and will be written to the main * partition. * @param rowIdColumnNameOpt The optional Row ID preservation column with the physical Row ID * name, it stores stable Row IDs. * @param rowCommitVersionColumnNameOpt The optional Row Commit Version preservation column * with the physical Row Commit Version name, it stores * stable Row Commit Versions. * @param outputColNames All the main and CDC columns to use in the output. * @param dedupColumns Additional columns to add to enable deduplication. */ private def packAndExplodeCDCOutput( sourceDf: DataFrame, cdcArray: Column, cdcToMainDataArray: Column, rowIdColumnNameOpt: Option[String], rowCommitVersionColumnNameOpt: Option[String], outputColNames: Seq[String], dedupColumns: Seq[Column], needSetRowTrackingFieldIdForUniform: Boolean): DataFrame = { val unpackedCols = outputColNames.map { name => if (rowIdColumnNameOpt.contains(name)) { // Add metadata to allow writing the column although it is not part of the schema. col(s"packedData.`$name`").as(name, RowId.columnMetadata(name, needSetRowTrackingFieldIdForUniform)) } else if (rowCommitVersionColumnNameOpt.contains(name)) { col(s"packedData.`$name`").as(name, RowCommitVersion.columnMetadata(name, needSetRowTrackingFieldIdForUniform)) } else { col(s"packedData.`$name`").as(name) } } sourceDf // `explode()` creates a [[Generator]] which can't handle non-deterministic expressions that // we use to increment metric counters. We first project the CDC array so that the expressions // are evaluated before we explode the array. .select(cdcArray.as("projectedCDC") +: dedupColumns: _*) .select(explode(col("projectedCDC")).as("packedCdc") +: dedupColumns: _*) .select(explode(cdcToMainDataArray).as("packedData") +: dedupColumns: _*) .select(unpackedCols ++ dedupColumns: _*) } /** * This method deduplicates CDF deletes where a target row has potentially multiple matches. It * assumes that the input dataframe contains the [[TARGET_ROW_INDEX_COL]] and * to detect inserts the [[SOURCE_ROW_INDEX_COL]] column to track the origin of the row. * * All duplicates of deleted rows have the same [[TARGET_ROW_INDEX_COL]] and * [[CDC_TYPE_COLUMN_NAME]] therefore we use both columns as compound deduplication key. * In case the input data frame contains additional insert rows we leave them untouched by using * the [[SOURCE_ROW_INDEX_COL]] to fill the null values of the [[TARGET_ROW_INDEX_COL]]. This * may lead to duplicates as part of the final row index but this is not a problem since if * an insert and a delete have the same [[TARGET_ROW_INDEX_COL]] they definitely have a * different [[CDC_TYPE_COLUMN_NAME]]. */ private def deduplicateCDFDeletes( deduplicateDeletes: DeduplicateCDFDeletes, df: DataFrame, cdcArray: Column, cdcToMainDataArray: Column, rowIdColumnNameOpt: Option[String], rowCommitVersionColumnNameOpt: Option[String], outputColNames: Seq[String], needSetRowTrackingFieldIdForUniform: Boolean): DataFrame = { val dedupColumns = if (deduplicateDeletes.includesInserts) { Seq(col(TARGET_ROW_INDEX_COL), col(SOURCE_ROW_INDEX_COL)) } else { Seq(col(TARGET_ROW_INDEX_COL)) } val cdcDf = packAndExplodeCDCOutput( df, cdcArray, cdcToMainDataArray, rowIdColumnNameOpt, rowCommitVersionColumnNameOpt, outputColNames, dedupColumns, needSetRowTrackingFieldIdForUniform ) val cdcDfWithIncreasingIds = if (deduplicateDeletes.includesInserts) { cdcDf.withColumn( TARGET_ROW_INDEX_COL, coalesce(col(TARGET_ROW_INDEX_COL), col(SOURCE_ROW_INDEX_COL))) } else { cdcDf } cdcDfWithIncreasingIds .dropDuplicates(TARGET_ROW_INDEX_COL, CDC_TYPE_COLUMN_NAME) .drop(TARGET_ROW_INDEX_COL, SOURCE_ROW_INDEX_COL) } } /** * This class enables and configures the deduplication of CDF deletes in case the merge statement * contains an unconditional delete statement that matches multiple target rows. * * @param enabled CDF generation should be enabled and duplicate target matches are detected * @param includesInserts in addition to the unconditional deletes the merge also inserts rows */ case class DeduplicateCDFDeletes( enabled: Boolean, includesInserts: Boolean) object MergeOutputGeneration { final val TARGET_ROW_INDEX_COL = "_target_row_index_" final val SOURCE_ROW_INDEX_COL = "_source_row_index" } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/merge/MergeStats.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.merge import scala.collection.mutable.ArrayBuffer import org.apache.spark.sql.delta.NumRecordsStats import org.apache.spark.sql.util.ScalaExtensions._ import com.fasterxml.jackson.databind.annotation.JsonDeserialize import org.apache.commons.lang3.StringUtils import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.plans.logical.{DeltaMergeIntoClause, DeltaMergeIntoMatchedClause, DeltaMergeIntoNotMatchedBySourceClause, DeltaMergeIntoNotMatchedClause} import org.apache.spark.sql.execution.metric.SQLMetric case class MergeDataSizes( @JsonDeserialize(contentAs = classOf[java.lang.Long]) rows: Option[Long] = None, @JsonDeserialize(contentAs = classOf[java.lang.Long]) files: Option[Long] = None, @JsonDeserialize(contentAs = classOf[java.lang.Long]) bytes: Option[Long] = None, @JsonDeserialize(contentAs = classOf[java.lang.Long]) partitions: Option[Long] = None) /** * Represents the state of a single merge clause: * - merge clause's (optional) predicate * - action type (insert, update, delete) * - action's expressions */ case class MergeClauseStats( condition: Option[String], actionType: String, actionExpr: Seq[String]) object MergeClauseStats { def apply(mergeClause: DeltaMergeIntoClause): MergeClauseStats = { MergeClauseStats( condition = mergeClause.condition.map(c => StringUtils.abbreviate(c.sql, 256)), mergeClause.clauseType.toLowerCase(), actionExpr = truncateSeq( mergeClause.actions.map(a => StringUtils.abbreviate(a.sql, 256)), maxLength = 512) ) } /** * Truncate a list of items to be serialized to around 'maxLength' characters. * Always include at least on item. */ private def truncateSeq(seq: Seq[String], maxLength: Long): Seq[String] = { val buffer = ArrayBuffer.empty[String] var length = 0L for (x <- seq if length + x.length <= maxLength || buffer.isEmpty) { length += x.length + 3 // quotes and comma buffer.append(x) } val numTruncatedItems = seq.length - buffer.length if (numTruncatedItems > 0) { buffer.append("... " + numTruncatedItems + " more fields") } buffer.toSeq } } /** State for a merge operation */ case class MergeStats( // Merge condition expression conditionExpr: String, // Expressions used in old MERGE stats, now always Null updateConditionExpr: String, updateExprs: Seq[String], insertConditionExpr: String, insertExprs: Seq[String], deleteConditionExpr: String, // Newer expressions used in MERGE with any number of MATCHED/NOT MATCHED/NOT MATCHED BY SOURCE matchedStats: Seq[MergeClauseStats], notMatchedStats: Seq[MergeClauseStats], notMatchedBySourceStats: Seq[MergeClauseStats], // Timings executionTimeMs: Long, materializeSourceTimeMs: Long, scanTimeMs: Long, rewriteTimeMs: Long, // Data sizes of source and target at different stages of processing source: MergeDataSizes, targetBeforeSkipping: MergeDataSizes, targetAfterSkipping: MergeDataSizes, @JsonDeserialize(contentAs = classOf[java.lang.Long]) sourceRowsInSecondScan: Option[Long], // Data change sizes targetFilesRemoved: Long, targetFilesAdded: Long, @JsonDeserialize(contentAs = classOf[java.lang.Long]) targetChangeFilesAdded: Option[Long], @JsonDeserialize(contentAs = classOf[java.lang.Long]) targetChangeFileBytes: Option[Long], @JsonDeserialize(contentAs = classOf[java.lang.Long]) targetBytesRemoved: Option[Long], @JsonDeserialize(contentAs = classOf[java.lang.Long]) targetBytesAdded: Option[Long], @JsonDeserialize(contentAs = classOf[java.lang.Long]) targetPartitionsRemovedFrom: Option[Long], @JsonDeserialize(contentAs = classOf[java.lang.Long]) targetPartitionsAddedTo: Option[Long], targetRowsCopied: Long, targetRowsUpdated: Long, targetRowsMatchedUpdated: Long, targetRowsNotMatchedBySourceUpdated: Long, targetRowsInserted: Long, targetRowsDeleted: Long, targetRowsMatchedDeleted: Long, targetRowsNotMatchedBySourceDeleted: Long, numTargetDeletionVectorsAdded: Long, numTargetDeletionVectorsRemoved: Long, numTargetDeletionVectorsUpdated: Long, // MergeMaterializeSource stats materializeSourceReason: Option[String] = None, @JsonDeserialize(contentAs = classOf[java.lang.Long]) materializeSourceAttempts: Option[Long] = None, @JsonDeserialize(contentAs = classOf[java.lang.Long]) numLogicalRecordsAdded: Option[Long], @JsonDeserialize(contentAs = classOf[java.lang.Long]) numLogicalRecordsRemoved: Option[Long], @JsonDeserialize(contentAs = classOf[java.lang.Long]) commitVersion: Option[Long] = None ) object MergeStats { def fromMergeSQLMetrics( metrics: Map[String, SQLMetric], condition: Expression, matchedClauses: Seq[DeltaMergeIntoMatchedClause], notMatchedClauses: Seq[DeltaMergeIntoNotMatchedClause], notMatchedBySourceClauses: Seq[DeltaMergeIntoNotMatchedBySourceClause], isPartitioned: Boolean, performedSecondSourceScan: Boolean, commitVersion: Option[Long], numRecordsStats: NumRecordsStats ): MergeStats = { def metricValueIfPartitioned(metricName: String): Option[Long] = { if (isPartitioned) Some(metrics(metricName).value) else None } MergeStats( // Merge condition expression conditionExpr = StringUtils.abbreviate(condition.sql, 2048), // Newer expressions used in MERGE with any number of MATCHED/NOT MATCHED/ // NOT MATCHED BY SOURCE matchedStats = matchedClauses.map(MergeClauseStats(_)), notMatchedStats = notMatchedClauses.map(MergeClauseStats(_)), notMatchedBySourceStats = notMatchedBySourceClauses.map(MergeClauseStats(_)), // Timings executionTimeMs = metrics("executionTimeMs").value, materializeSourceTimeMs = metrics("materializeSourceTimeMs").value, scanTimeMs = metrics("scanTimeMs").value, rewriteTimeMs = metrics("rewriteTimeMs").value, // Data sizes of source and target at different stages of processing source = MergeDataSizes(rows = Some(metrics("numSourceRows").value)), targetBeforeSkipping = MergeDataSizes( files = Some(metrics("numTargetFilesBeforeSkipping").value), bytes = Some(metrics("numTargetBytesBeforeSkipping").value)), targetAfterSkipping = MergeDataSizes( files = Some(metrics("numTargetFilesAfterSkipping").value), bytes = Some(metrics("numTargetBytesAfterSkipping").value), partitions = metricValueIfPartitioned("numTargetPartitionsAfterSkipping")), sourceRowsInSecondScan = Option.when(performedSecondSourceScan)(metrics("numSourceRowsInSecondScan").value), // Data change sizes targetFilesAdded = metrics("numTargetFilesAdded").value, targetChangeFilesAdded = metrics.get("numTargetChangeFilesAdded").map(_.value), targetChangeFileBytes = metrics.get("numTargetChangeFileBytes").map(_.value), targetFilesRemoved = metrics("numTargetFilesRemoved").value, targetBytesAdded = Some(metrics("numTargetBytesAdded").value), targetBytesRemoved = Some(metrics("numTargetBytesRemoved").value), targetPartitionsRemovedFrom = metricValueIfPartitioned("numTargetPartitionsRemovedFrom"), targetPartitionsAddedTo = metricValueIfPartitioned("numTargetPartitionsAddedTo"), targetRowsCopied = metrics("numTargetRowsCopied").value, targetRowsUpdated = metrics("numTargetRowsUpdated").value, targetRowsMatchedUpdated = metrics("numTargetRowsMatchedUpdated").value, targetRowsNotMatchedBySourceUpdated = metrics("numTargetRowsNotMatchedBySourceUpdated").value, targetRowsInserted = metrics("numTargetRowsInserted").value, targetRowsDeleted = metrics("numTargetRowsDeleted").value, targetRowsMatchedDeleted = metrics("numTargetRowsMatchedDeleted").value, targetRowsNotMatchedBySourceDeleted = metrics("numTargetRowsNotMatchedBySourceDeleted").value, // Deletion Vector metrics. numTargetDeletionVectorsAdded = metrics("numTargetDeletionVectorsAdded").value, numTargetDeletionVectorsRemoved = metrics("numTargetDeletionVectorsRemoved").value, numTargetDeletionVectorsUpdated = metrics("numTargetDeletionVectorsUpdated").value, commitVersion = commitVersion, numLogicalRecordsAdded = numRecordsStats.numLogicalRecordsAdded, numLogicalRecordsRemoved = numRecordsStats.numLogicalRecordsRemoved, // Deprecated fields updateConditionExpr = null, updateExprs = null, insertConditionExpr = null, insertExprs = null, deleteConditionExpr = null) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/optimize/AddFileWithNumRecords.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.optimize import org.apache.spark.sql.delta.actions.AddFile /** * Wrapper over an [AddFile] and its stats: * @param numPhysicalRecords The number of records physically present in the file. * Equivalent to `addFile.numTotalRecords`. * @param numLogicalRecords The physical number of records minus the Deletion Vector cardinality. * Equivalent to `addFile.numRecords`. */ case class AddFileWithNumRecords( addFile: AddFile, numPhysicalRecords: java.lang.Long, numLogicalRecords: java.lang.Long) { /** Returns the approx size of the remaining records after excluding the deleted ones. */ def estLogicalFileSize: Long = { (addFile.size * logicalToPhysicalRecordsRatio).toLong } /** Returns the ratio of the logical number of records to the total number of records. */ def logicalToPhysicalRecordsRatio: Double = { if (numLogicalRecords == null || numPhysicalRecords == null || numPhysicalRecords == 0) { 1.0d } else { numLogicalRecords.toDouble / numPhysicalRecords.toDouble } } /** Returns the ratio of the deleted number of records to the total number of records */ def deletedToPhysicalRecordsRatio: Double = { 1.0d - logicalToPhysicalRecordsRatio } } object AddFileWithNumRecords { def createFromFile(file: AddFile): AddFileWithNumRecords = { val numPhysicalRecords = file.numPhysicalRecords.getOrElse(0L) val numLogicalRecords = file.numLogicalRecords.getOrElse(0L) AddFileWithNumRecords(file, numPhysicalRecords, numLogicalRecords) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/optimize/OptimizeStats.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.optimize import org.apache.spark.sql.delta.skipping.clustering.ClusteringStats import org.apache.spark.sql.delta.actions.{AddFile, FileAction, RemoveFile} // scalastyle:off import.ordering.noEmptyLine /** * Stats for an OPTIMIZE operation accumulated across all batches. */ case class OptimizeStats( var addedFilesSizeStats: FileSizeStats = FileSizeStats(), var removedFilesSizeStats: FileSizeStats = FileSizeStats(), var numPartitionsOptimized: Long = 0, var zOrderStats: Option[ZOrderStats] = None, var clusteringStats: Option[ClusteringStats] = None, var numBins: Long = 0, var numBatches: Long = 0, var totalConsideredFiles: Long = 0, var totalFilesSkipped: Long = 0, var preserveInsertionOrder: Boolean = false, var numFilesSkippedToReduceWriteAmplification: Long = 0, var numBytesSkippedToReduceWriteAmplification: Long = 0, startTimeMs: Long = System.currentTimeMillis(), var endTimeMs: Long = 0, var totalClusterParallelism: Long = 0, var totalScheduledTasks: Long = 0, var deletionVectorStats: Option[DeletionVectorStats] = None, var numTableColumns: Long = 0, var numTableColumnsWithStats: Long = 0, var autoCompactParallelismStats: AutoCompactParallelismStats = AutoCompactParallelismStats()) { def toOptimizeMetrics: OptimizeMetrics = { OptimizeMetrics( numFilesAdded = addedFilesSizeStats.totalFiles, numFilesRemoved = removedFilesSizeStats.totalFiles, filesAdded = addedFilesSizeStats.toFileSizeMetrics, filesRemoved = removedFilesSizeStats.toFileSizeMetrics, partitionsOptimized = numPartitionsOptimized, zOrderStats = zOrderStats, clusteringStats = clusteringStats, numBins = numBins, numBatches = numBatches, totalConsideredFiles = totalConsideredFiles, totalFilesSkipped = totalFilesSkipped, preserveInsertionOrder = preserveInsertionOrder, numFilesSkippedToReduceWriteAmplification = numFilesSkippedToReduceWriteAmplification, numBytesSkippedToReduceWriteAmplification = numBytesSkippedToReduceWriteAmplification, startTimeMs = startTimeMs, endTimeMs = endTimeMs, totalClusterParallelism = totalClusterParallelism, totalScheduledTasks = totalScheduledTasks, deletionVectorStats = deletionVectorStats, numTableColumns = numTableColumns, numTableColumnsWithStats = numTableColumnsWithStats, autoCompactParallelismStats = autoCompactParallelismStats.toMetrics) } } /** * This statistics class keeps tracking the parallelism usage of Auto Compaction. * It collects following metrics: * -- the min/max parallelism among the whole cluster are used for Auto Compact, * -- the min/max parallelism occupied by current Auto Compact session, */ case class AutoCompactParallelismStats( var maxClusterUsedParallelism: Long = 0, var minClusterUsedParallelism: Long = 0, var maxSessionUsedParallelism: Long = 0, var minSessionUsedParallelism: Long = 0) { def toMetrics: Option[ParallelismMetrics] = { if (maxSessionUsedParallelism == 0) { return None } Some(ParallelismMetrics( Some(maxClusterUsedParallelism), Some(minClusterUsedParallelism), Some(maxSessionUsedParallelism), Some(minSessionUsedParallelism))) } /** Update the statistics of parallelism of current Auto Compact command. */ def update(clusterUsedParallelism: Long, sessionUsedParallelism: Long): Unit = { maxClusterUsedParallelism = Math.max(maxClusterUsedParallelism, clusterUsedParallelism) minClusterUsedParallelism = if (minClusterUsedParallelism == 0) { clusterUsedParallelism } else { Math.min(minClusterUsedParallelism, clusterUsedParallelism) } maxSessionUsedParallelism = Math.max(maxSessionUsedParallelism, sessionUsedParallelism) minSessionUsedParallelism = if (minSessionUsedParallelism == 0) { sessionUsedParallelism } else { Math.min(minSessionUsedParallelism, sessionUsedParallelism) } } } case class FileSizeStats( var minFileSize: Long = 0, var maxFileSize: Long = 0, var totalFiles: Long = 0, var totalSize: Long = 0) { def avgFileSize: Double = if (totalFiles > 0) { totalSize * 1.0 / totalFiles } else { 0.0 } def merge(candidateFiles: Seq[FileAction]): Unit = { if (totalFiles == 0 && candidateFiles.nonEmpty) { minFileSize = Long.MaxValue maxFileSize = Long.MinValue } candidateFiles.foreach { file => val fileSize = file match { case addFile: AddFile => addFile.size case removeFile: RemoveFile => removeFile.size.getOrElse(0L) case default => throw new IllegalArgumentException(s"Unknown FileAction type: ${default.getClass}") } minFileSize = math.min(fileSize, minFileSize) maxFileSize = math.max(fileSize, maxFileSize) totalSize += fileSize } totalFiles += candidateFiles.length } def toFileSizeMetrics: FileSizeMetrics = { if (totalFiles == 0) { return FileSizeMetrics(min = None, max = None, avg = 0, totalFiles = 0, totalSize = 0) } FileSizeMetrics( min = Some(minFileSize), max = Some(maxFileSize), avg = avgFileSize, totalFiles = totalFiles, totalSize = totalSize) } } /** * Percentiles on the file sizes in this batch. * @param min Size of the smallest file * @param p25 Size of the 25th percentile file * @param p50 Size of the 50th percentile file * @param p75 Size of the 75th percentile file * @param max Size of the largest file */ case class FileSizeStatsWithHistogram( min: Long, p25: Long, p50: Long, p75: Long, max: Long) object FileSizeStatsWithHistogram { /** * Creates a [[FileSizeStatsWithHistogram]] based on the passed sorted file sizes * @return Some(fileSizeStatsWithHistogram) if sizes are non-empty, else returns None */ def create(sizes: Seq[Long]): Option[FileSizeStatsWithHistogram] = { if (sizes.isEmpty) { return None } val count = sizes.length Some(FileSizeStatsWithHistogram( min = sizes.head, // we do not need to ceil the computed index as arrays start at 0 p25 = sizes(count / 4), p50 = sizes(count / 2), p75 = sizes(count * 3 / 4), max = sizes.last)) } } /** * Metrics returned by the optimize command. * * @param numFilesAdded number of files added by optimize * @param numFilesRemoved number of files removed by optimize * @param filesAdded Stats for the files added * @param filesRemoved Stats for the files removed * @param partitionsOptimized Number of partitions optimized * @param zOrderStats Z-Order stats * @param clusteringStats Clustering stats * @param numBins Number of bins * @param numBatches Number of batches * @param totalConsideredFiles Number of files considered for the Optimize operation. * @param totalFilesSkipped Number of files that are skipped from being Optimized. * @param preserveInsertionOrder If optimize was run with insertion preservation enabled. * @param numFilesSkippedToReduceWriteAmplification Number of files skipped for reducing write * amplification. * @param numBytesSkippedToReduceWriteAmplification Number of bytes skipped for reducing write * amplification. * @param startTimeMs The start time of Optimize command. * @param endTimeMs The end time of Optimize command. * @param totalClusterParallelism The total number of parallelism of this cluster. * @param totalScheduledTasks The total number of optimize task scheduled. * @param autoCompactParallelismStats The metrics of cluster and session parallelism. * @param deletionVectorStats Statistics related with Deletion Vectors. * @param numTableColumns Number of columns in the table. * @param numTableColumnsWithStats Number of table columns to collect data skipping stats. */ case class OptimizeMetrics( numFilesAdded: Long, numFilesRemoved: Long, filesAdded: FileSizeMetrics = FileSizeMetrics(min = None, max = None, avg = 0, totalFiles = 0, totalSize = 0), filesRemoved: FileSizeMetrics = FileSizeMetrics(min = None, max = None, avg = 0, totalFiles = 0, totalSize = 0), partitionsOptimized: Long = 0, zOrderStats: Option[ZOrderStats] = None, clusteringStats: Option[ClusteringStats] = None, numBins: Long, numBatches: Long, totalConsideredFiles: Long, totalFilesSkipped: Long = 0, preserveInsertionOrder: Boolean = false, numFilesSkippedToReduceWriteAmplification: Long = 0, numBytesSkippedToReduceWriteAmplification: Long = 0, startTimeMs: Long = 0, endTimeMs: Long = 0, totalClusterParallelism: Long = 0, totalScheduledTasks: Long = 0, autoCompactParallelismStats: Option[ParallelismMetrics] = None, deletionVectorStats: Option[DeletionVectorStats] = None, numTableColumns: Long = 0, numTableColumnsWithStats: Long = 0 ) /** * Basic Stats on file sizes. * * @param min Minimum file size * @param max Maximum file size * @param avg Average of the file size * @param totalFiles Total number of files * @param totalSize Total size of the files */ case class FileSizeMetrics( min: Option[Long], max: Option[Long], avg: Double, totalFiles: Long, totalSize: Long) /** * This statistics contains following metrics: * -- the min/max parallelism among the whole cluster are used, * -- the min/max parallelism occupied by current session, */ case class ParallelismMetrics( maxClusterActiveParallelism: Option[Long] = None, minClusterActiveParallelism: Option[Long] = None, maxSessionActiveParallelism: Option[Long] = None, minSessionActiveParallelism: Option[Long] = None) /** * Accumulator for statistics related with Deletion Vectors. * Note that this case class contains mutable variables and cannot be used in places where immutable * case classes can be used (e.g. map/set keys). */ case class DeletionVectorStats( var numDeletionVectorsRemoved: Long = 0, var numDeletionVectorRowsRemoved: Long = 0) ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/optimize/ZCubeFileStatsCollector.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.optimize import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.zorder.ZCubeInfo import org.apache.spark.sql.delta.zorder.ZCubeInfo.ZCubeID /** * ZCube file statistics collector. An object of this class can be used to collect ZCube statistics. * The file statistics collection can be started by initializing an object of this class and * calling updateStats on every new file seen. * The number of ZCubes, number of files from matching cubes and number of unoptimized files are * captured here. * * @param zOrderBy zOrder or clustering columns. * @param isFull whether OPTIMIZE FULL is run. This is only for clustered tables. */ class ZCubeFileStatsCollector(zOrderBy: Seq[String], isFull: Boolean) { /** map that holds the file statistics Map("element" -> (number of files, total file size)) */ private var processedZCube: ZCubeID = _ /** number of distinct zCubes seen so far */ var numZCubes = 0 private var matchingCubeCnt = 0 private var matchingCubeSize = 0L private var otherFilesCnt = 0 private var otherFilesSize = 0L def fileStats: Map[String, (Int, Long)] = Map( "matchingCube" -> ((matchingCubeCnt, matchingCubeSize)), "otherFiles" -> ((otherFilesCnt, otherFilesSize)) ) /** method to update the zCubeFileStats incrementally by file */ def updateStats(file: AddFile): AddFile = { val zCubeInfo = ZCubeInfo.getForFile(file) // Note that clustered files with different clustering columns are considered candidate // files when isFull is set. if (zCubeInfo.isDefined && (isFull || zCubeInfo.get.zOrderBy == zOrderBy)) { if (processedZCube != zCubeInfo.get.zCubeID) { processedZCube = zCubeInfo.get.zCubeID numZCubes += 1 } matchingCubeCnt += 1 matchingCubeSize += file.size } else { otherFilesCnt += 1 otherFilesSize += file.size } file } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/commands/optimize/ZOrderMetrics.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.optimize /** * A class to create blob structure for zorder metrics and events. */ class ZOrderMetrics(zOrderBy: Seq[String]) { var strategyName: String = _ val inputStats = new ZCubeFileStatsCollector(zOrderBy, isFull = false) val outputStats = new ZCubeFileStatsCollector(zOrderBy, isFull = false) var numOutputCubes = 0 def getZOrderStats(): ZOrderStats = { ZOrderStats( strategyName, inputNumCubes = inputStats.numZCubes, inputCubeFiles = ZOrderFileStats(inputStats.fileStats.get("matchingCube")), inputOtherFiles = ZOrderFileStats(inputStats.fileStats.get("otherFiles")), mergedFiles = ZOrderFileStats(outputStats.fileStats.values), numOutputCubes = numOutputCubes ) } } /** * Aggregated file stats for a category of ZCube files. * @param num Total number of files. * @param size Total size of files in bytes. */ case class ZOrderFileStats(num: Long, size: Long) object ZOrderFileStats { def apply(v: Iterable[(Int, Long)]): ZOrderFileStats = { v.foldLeft(ZOrderFileStats(0, 0)) { (a, b) => ZOrderFileStats(a.num + b._1, a.size + b._2) } } } /** * Aggregated stats for OPTIMIZE ZORDERBY command. * This is a public facing API, consider any change carefully. * * @param strategyName ZCubeMergeStrategy used. * @param inputCubeFiles Files in the ZCube matching the current OPTIMIZE operation. * @param inputOtherFiles Files not in any ZCube or in other ZCube orderings. * @param inputNumCubes Number of different cubes among input files. * @param mergedFiles Subset of input files merged by the current operation * @param numOutputCubes Number of output ZCubes written out * @param mergedNumCubes Number of different cubes among merged files. */ case class ZOrderStats( strategyName: String, inputCubeFiles: ZOrderFileStats, inputOtherFiles: ZOrderFileStats, inputNumCubes: Long, mergedFiles: ZOrderFileStats, numOutputCubes: Long, mergedNumCubes: Option[Long] = None ) ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/constraints/CharVarcharConstraint.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.constraints import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.util.CharVarcharUtils import org.apache.spark.sql.types._ // Delta implements char/varchar length check with CONSTRAINTS, and needs to generate predicate // expression which is different from the OSS version. object CharVarcharConstraint { final val INVARIANT_NAME = "__CHAR_VARCHAR_STRING_LENGTH_CHECK__" def stringConstraints(schema: StructType): Seq[Constraint] = { schema.flatMap { f => val targetType = CharVarcharUtils.getRawType(f.metadata).getOrElse(f.dataType) val col = UnresolvedAttribute(Seq(f.name)) checkStringLength(col, targetType).map { lengthCheckExpr => Constraints.Check(INVARIANT_NAME, lengthCheckExpr) } } } private def checkStringLength(expr: Expression, dt: DataType): Option[Expression] = dt match { case v: VarcharType => Some(Or(IsNull(expr), LessThanOrEqual(Length(expr), Literal(v.length)))) case c: CharType => checkStringLength(expr, VarcharType(c.length)) case StructType(fields) => fields.zipWithIndex.flatMap { case (f, i) => checkStringLength(GetStructField(expr, i, Some(f.name)), f.dataType) }.reduceOption(And(_, _)) case ArrayType(et, containsNull) => checkStringLengthInArray(expr, et, containsNull) case MapType(kt, vt, valueContainsNull) => (checkStringLengthInArray(MapKeys(expr), kt, false) ++ checkStringLengthInArray(MapValues(expr), vt, valueContainsNull)) .reduceOption(And(_, _)) case _ => None } private def checkStringLengthInArray( arr: Expression, et: DataType, containsNull: Boolean): Option[Expression] = { val cleanedType = CharVarcharUtils.replaceCharVarcharWithString(et) val param = NamedLambdaVariable("x", cleanedType, containsNull) checkStringLength(param, et).map { checkExpr => Or(IsNull(arr), ArrayForAll(arr, LambdaFunction(checkExpr, Seq(param)))) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/constraints/CheckDeltaInvariant.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.constraints import org.apache.spark.sql.delta.constraints.Constraints.{Check, NotNull} import org.apache.spark.sql.delta.schema.DeltaInvariantViolationException import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{Expression, NonSQLExpression} import org.apache.spark.sql.catalyst.expressions.codegen._ import org.apache.spark.sql.catalyst.expressions.codegen.Block._ import org.apache.spark.sql.types.{DataType, NullType} /** * An expression that validates a specific invariant on a column, before writing into Delta. * * @param child The fully resolved expression to be evaluated to check the constraint. * @param columnExtractors Extractors for each referenced column. Used to generate readable errors. * @param constraint The original constraint definition. */ case class CheckDeltaInvariant( child: Expression, columnExtractors: Seq[(String, Expression)], constraint: Constraint) extends Expression with NonSQLExpression with CodegenFallback { override def children: Seq[Expression] = child +: columnExtractors.map(_._2) override def dataType: DataType = NullType override def foldable: Boolean = false override def nullable: Boolean = true private def assertRule(input: InternalRow): Unit = constraint match { case n: NotNull => if (child.eval(input) == null) { throw DeltaInvariantViolationException(n) } case c: Check => val result = child.eval(input) if (result == null || result == false) { throw DeltaInvariantViolationException( c, columnExtractors.map { case (column, extractor) => column -> extractor.eval(input) }.toMap ) } } override def eval(input: InternalRow): Any = { assertRule(input) null } private def generateNotNullCode(ctx: CodegenContext): Block = { val childGen = child.genCode(ctx) val invariantField = ctx.addReferenceObj("errMsg", constraint) code"""${childGen.code} | |if (${childGen.isNull}) { | throw org.apache.spark.sql.delta.schema.DeltaInvariantViolationException.apply( | $invariantField); |} """.stripMargin } /** * Generate the code to extract values for the columns referenced in a violated CHECK constraint. * We build parallel lists of full column names and their extracted values in the row which * violates the constraint, to be passed to the [[InvariantViolationException]] constructor * in [[generateExpressionValidationCode()]]. * * Note that this code is a bit expensive, so it shouldn't be run until we already * know the constraint has been violated. */ private def generateColumnValuesCode( colList: String, valList: String, ctx: CodegenContext): Block = { val start = code""" |java.util.List $colList = new java.util.ArrayList(); |java.util.List $valList = new java.util.ArrayList(); |""".stripMargin columnExtractors.map { case (name, extractor) => val colValue = extractor.genCode(ctx) code""" |$colList.add("$name"); |${colValue.code} |if (${colValue.isNull}) { | $valList.add(null); |} else { | $valList.add(${colValue.value}); |} |""".stripMargin }.fold(start)(_ + _) } private def generateExpressionValidationCode(ctx: CodegenContext): Block = { val elementValue = child.genCode(ctx) val invariantField = ctx.addReferenceObj("errMsg", constraint) val colListName = ctx.freshName("colList") val valListName = ctx.freshName("valList") code"""${elementValue.code} | |if (${elementValue.isNull} || ${elementValue.value} == false) { | ${generateColumnValuesCode(colListName, valListName, ctx)} | throw org.apache.spark.sql.delta.schema.DeltaInvariantViolationException.apply( | $invariantField, $colListName, $valListName); |} """.stripMargin } override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { val code = constraint match { case _: NotNull => generateNotNullCode(ctx) case _: Check => generateExpressionValidationCode(ctx) } ev.copy(code = code, isNull = TrueLiteral, value = JavaCode.literal("null", NullType)) } override protected def withNewChildrenInternal( newChildren: IndexedSeq[Expression]): Expression = { copy( child = newChildren.head, columnExtractors = columnExtractors.map(_._1).zip(newChildren.tail) ) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/constraints/Constraints.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.constraints import java.util.Locale import scala.concurrent.duration import scala.util.control.NonFatal import org.apache.spark.sql.delta.{AllowedUserProvidedExpressions, DeltaErrors, DeltaLog} import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf.ValidateCheckConstraintsMode import org.apache.spark.SparkThrowable import org.apache.spark.internal.MDC import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions.{Alias, Expression, GetArrayItem, GetMapValue, GetStructField, IsNotNull, UserDefinedExpression} import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, Project} import org.apache.spark.sql.catalyst.types.DataTypeUtils import org.apache.spark.sql.catalyst.util.CharVarcharUtils import org.apache.spark.sql.types.{BooleanType, StructType} /** * A constraint defined on a Delta table, which writers must verify before writing. */ sealed trait Constraint { val name: String } /** * Utilities for handling constraints. Right now this includes: * - Column-level invariants delegated to [[Invariants]], including both NOT NULL constraints and * an old style of CHECK constraint specified in the column metadata * - Table-level CHECK constraints */ object Constraints extends DeltaLogging { /** * A constraint that the specified column must not be NULL. Note that when the column is nested, * this implies its parents must also not be NULL. */ case class NotNull(column: Seq[String]) extends Constraint { override val name: String = "NOT NULL" } /** A SQL expression to check for when writing out data. */ case class Check(name: String, expression: Expression) extends Constraint def getCheckConstraintNames(metadata: Metadata): Seq[String] = { metadata.configuration.keys.collect { case key if key.toLowerCase(Locale.ROOT).startsWith("delta.constraints.") => key.stripPrefix("delta.constraints.") }.toSeq } /** * Extract CHECK constraints from the table properties. Note that some CHECK constraints may also * come from schema metadata; these constraints were never released in a public API but are * maintained for protocol compatibility. */ def getCheckConstraints(metadata: Metadata, spark: SparkSession): Seq[Constraint] = { metadata.configuration.collect { case (key, constraintText) if key.toLowerCase(Locale.ROOT).startsWith("delta.constraints.") => val name = key.stripPrefix("delta.constraints.") val expression = spark.sessionState.sqlParser.parseExpression(constraintText) Check(name, expression) }.toSeq } /** Extract all constraints from the given Delta table metadata. */ def getAll(metadata: Metadata, spark: SparkSession): Seq[Constraint] = { val checkConstraints = getCheckConstraints(metadata, spark) val constraintsFromSchema = Invariants.getFromSchema(metadata.schema, spark) val charVarcharLengthChecks = if (spark.sessionState.conf.charVarcharAsString) { Nil } else { CharVarcharConstraint.stringConstraints(metadata.schema) } (checkConstraints ++ constraintsFromSchema ++ charVarcharLengthChecks).toSeq } /** Get the expression text for a constraint with the given name, if present. */ def getExprTextByName( name: String, metadata: Metadata, spark: SparkSession): Option[String] = { metadata.configuration.get(checkConstraintPropertyName(name)) } def checkConstraintPropertyName(constraintName: String): String = { "delta.constraints." + constraintName.toLowerCase(Locale.ROOT) } /** * Find all the check constraints that reference the given column name. Returns a map of * constraint names to their corresponding expression. */ def findDependentConstraints( sparkSession: SparkSession, columnName: Seq[String], metadata: Metadata): Map[String, String] = { metadata.configuration.filter { case (key, constraint) if key.toLowerCase(Locale.ROOT).startsWith("delta.constraints.") => SchemaUtils.containsDependentExpression( sparkSession, columnName, constraint, metadata.schema, sparkSession.sessionState.conf.resolver) case _ => false } } /** * Validates check constraints with rollout logic for safe deployment. * This wrapper handles feature flag checks, error logging, and mode-based error handling. */ def validateCheckConstraints( spark: SparkSession, constraints: Seq[Constraint], deltaLog: DeltaLog, schema: StructType): Unit = { val validateCheckConstraints = ValidateCheckConstraintsMode.fromConf(spark.sessionState.conf) if (validateCheckConstraints == ValidateCheckConstraintsMode.OFF) return if (constraints.isEmpty) return try { val startTime = System.nanoTime() validateCheckConstraintsInternal(spark, constraints, schema) val durationMs = duration.NANOSECONDS.toMillis(System.nanoTime() - startTime) logInfo( log"Validated CHECK constraints on table " + log"${MDC(DeltaLogKeys.TABLE_ID, deltaLog.unsafeVolatileTableId)} " + log"in ${MDC(DeltaLogKeys.TIME_MS, durationMs)} ms and processed " + log"${MDC(DeltaLogKeys.NUM_PREDICATES, constraints.size)} constraints" ) } catch { case NonFatal(e) => val errorClassName = e match { case sparkEx: SparkThrowable => sparkEx.getErrorClass case _ => e.getClass } recordDeltaEvent( deltaLog, "delta.checkConstraints.validationFailure", data = Map( "errorClassName" -> errorClassName, "errorMessage" -> e.getMessage ) ) if (validateCheckConstraints == ValidateCheckConstraintsMode.ASSERT) { throw e } } } /** * Internal validation logic for check constraints. */ private def validateCheckConstraintsInternal( spark: SparkSession, constraints: Seq[Constraint], schema: StructType): Unit = { // Create NamedExpressions for analysis (type checking will happen after resolution) // We unresolve expressions to validate against the `LocalRelation` we later create val selectExprs = constraints.map { case Check(name, expression) => Alias(expression, name)() case NotNull(columnPath) => // Create an IsNotNull expression to validate the column exists val columnRef = UnresolvedAttribute(columnPath) val isNotNullExpr = IsNotNull(columnRef) Alias(isNotNullExpr, s"NOT NULL ${columnPath.mkString(".")}")() } // Analyze all constraint expressions to ensure they can be properly resolved // Use LocalRelation with the table schema to ensure column references can be validated val analyzed = try { val analyzer = spark.sessionState.analyzer val relation = LocalRelation( DataTypeUtils.toAttributes(CharVarcharUtils.replaceCharVarcharWithStringInSchema(schema)) ) val plan = analyzer.execute(Project(selectExprs, relation)) analyzer.checkAnalysis(plan) plan } catch { case e: SparkThrowable if e.getErrorClass != null && e.getErrorClass.startsWith("UNRESOLVED_COLUMN") => // Check if this is an unresolved column/field error by examining the error class throw DeltaErrors.checkConstraintReferToWrongColumns( e.getMessageParameters.getOrDefault("objectName", "") ) case e => throw e } // Check that all Check constraints return boolean type analyzed match { case Project(projectList, _) => projectList.foreach { case a: Alias => if (a.dataType != BooleanType) { throw DeltaErrors.checkConstraintNotBoolean(a.name, a.child.sql) } case _ => // We should only the Aliases we previously created. } case _ => // We should only have a single projection } // Validate the analyzed expressions analyzed.transformAllExpressions { case expr: Alias => // Alias will be non deterministic if it points to a non deterministic expression. // Skip `Alias` to provide a better error for a non deterministic expression. expr case expr@(_: GetStructField | _: GetArrayItem | _: GetMapValue) => // The complex type extractors don't have a function name, so we need to check them // separately. Unlike generated columns we do allow `GetMapValue`. expr case expr: UserDefinedExpression => throw DeltaErrors.checkConstraintUDF(expr) case expr if !expr.deterministic => throw DeltaErrors.checkConstraintNonDeterministicExpression(expr) case expr if expr.isInstanceOf[AggregateExpression] => throw DeltaErrors.checkConstraintAggregateExpression(expr) case expr if !AllowedUserProvidedExpressions.expressions.contains(expr.getClass) && !AllowedUserProvidedExpressions .checkConstraintExpressions.contains(expr.getClass) => throw DeltaErrors.checkConstraintUnsupportedExpression(expr) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/constraints/DeltaInvariantCheckerExec.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.constraints import scala.collection.mutable import org.apache.spark.sql.delta.{DeltaErrors, DeltaIllegalStateException} import org.apache.spark.sql.delta.constraints.Constraints.{Check, NotNull} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.util.AnalysisHelper import org.apache.spark.rdd.RDD import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis._ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.optimizer import org.apache.spark.sql.catalyst.optimizer.ReplaceExpressions import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode} import org.apache.spark.sql.catalyst.plans.physical.Partitioning import org.apache.spark.sql.catalyst.rules.RuleExecutor import org.apache.spark.sql.execution.{SparkPlan, SparkStrategy, UnaryExecNode} import org.apache.spark.sql.types.StructType /** * Operator that validates that records satisfy provided constraints before they are written into * Delta. Each row is left unchanged after validations. */ case class DeltaInvariantChecker( child: LogicalPlan, deltaConstraints: Seq[CheckDeltaInvariant]) extends UnaryNode { assert(deltaConstraints.nonEmpty) override def output: Seq[Attribute] = child.output override protected def withNewChildInternal(newChild: LogicalPlan): DeltaInvariantChecker = copy(child = newChild) } object DeltaInvariantChecker { def apply( spark: SparkSession, child: LogicalPlan, constraints: Seq[Constraint]): DeltaInvariantChecker = { val invariantChecks = DeltaInvariantCheckerExec.buildInvariantChecks(child.output, constraints, spark) DeltaInvariantChecker(child, invariantChecks) } } object DeltaInvariantCheckerStrategy extends SparkStrategy { override def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match { case DeltaInvariantChecker(child, constraints) => DeltaInvariantCheckerExec(planLater(child), constraints) :: Nil case _ => Nil } } /** * A physical operator that validates records, before they are written into Delta. Each row * is left unchanged after validations. */ case class DeltaInvariantCheckerExec( child: SparkPlan, constraints: Seq[CheckDeltaInvariant]) extends UnaryExecNode { override def output: Seq[Attribute] = child.output override protected def doExecute(): RDD[InternalRow] = { if (constraints.isEmpty) return child.execute() // Resolve current_date()/current_time() expressions. // We resolve currentTime for all invariants together to make sure we use the same timestamp. val invariantsFakePlan = AnalysisHelper.FakeLogicalPlan(constraints, Nil) val newInvariantsPlan = optimizer.ComputeCurrentTime(invariantsFakePlan) val constraintsWithFixedTime = newInvariantsPlan.expressions.toArray val childOutput = child.output child.execute().mapPartitionsInternal { rows => val assertions = UnsafeProjection.create(constraintsWithFixedTime, childOutput) rows.map { row => assertions(row) row } } } override def outputOrdering: Seq[SortOrder] = child.outputOrdering override def outputPartitioning: Partitioning = child.outputPartitioning override protected def withNewChildInternal(newChild: SparkPlan): DeltaInvariantCheckerExec = copy(child = newChild) } object DeltaInvariantCheckerExec extends DeltaLogging { def apply( spark: SparkSession, child: SparkPlan, constraints: Seq[Constraint]): DeltaInvariantCheckerExec = { val invariantChecks = DeltaInvariantCheckerExec.buildInvariantChecks(child.output, constraints, spark) DeltaInvariantCheckerExec(child, invariantChecks) } // Specialized optimizer to run necessary rules so that the check expressions can be evaluated. object DeltaInvariantCheckerOptimizer extends RuleExecutor[LogicalPlan] { import org.apache.spark.sql.catalyst.optimizer.{ReplaceExpressions, RewriteWithExpression} final override protected def batches = Seq( Batch("Finish Analysis", Once, ReplaceExpressions), Batch("Rewrite With expression", Once, RewriteWithExpression) ) } /** Build the extractor for a particular column. */ private def buildExtractor(output: Seq[Attribute], column: Seq[String]): Option[Expression] = { assert(column.nonEmpty) val topLevelColumn = column.head val topLevelRefOpt = output.collectFirst { case a: AttributeReference if SchemaUtils.DELTA_COL_RESOLVER(a.name, topLevelColumn) => a } if (column.length == 1) { topLevelRefOpt } else { topLevelRefOpt.flatMap { topLevelRef => try { val nested = column.tail.foldLeft[Expression](topLevelRef) { case (e, fieldName) => e.dataType match { case StructType(fields) => val ordinal = fields.indexWhere(f => SchemaUtils.DELTA_COL_RESOLVER(f.name, fieldName)) if (ordinal == -1) { throw DeltaErrors.notNullColumnNotFoundInStruct( s"${fields.map(_.name).mkString("[", ",", "]")}") } GetStructField(e, ordinal, Some(fieldName)) case _ => // NOTE: We should also update `GeneratedColumn.validateGeneratedColumns` to enable // `GetMapValue` and `GetArrayStructFields` expressions when this is supported. throw DeltaErrors.unSupportedInvariantNonStructType } } Some(nested) } catch { case _: IndexOutOfBoundsException => None } } } } def buildInvariantChecks( output: Seq[Attribute], constraints: Seq[Constraint], spark: SparkSession): Seq[CheckDeltaInvariant] = { constraints.map { constraint => val columnExtractors = mutable.Map[String, Expression]() val executableExpr = constraint match { case n @ NotNull(column) => buildExtractor(output, column).getOrElse { throw DeltaErrors.notNullColumnMissingException(n) } case Check(name, expr) => // We need to do two stages of resolution here: // * Build the extractors to evaluate attribute references against input InternalRows. // * Do logical analysis to handle nested field extractions, functions, etc. val attributesExtracted = expr.transformUp { case a: UnresolvedAttribute => val ex = buildExtractor(output, a.nameParts).getOrElse(Literal(null)) columnExtractors(a.name) = ex ex } val wrappedPlan: LogicalPlan = ExpressionLogicalPlanWrapper(attributesExtracted) val analyzedLogicalPlan = spark.sessionState.analyzer.execute(wrappedPlan) val optimizedLogicalPlan = DeltaInvariantCheckerOptimizer.execute(analyzedLogicalPlan) val resolvedExpr = optimizedLogicalPlan match { case ExpressionLogicalPlanWrapper(e) => e // This should never happen. case plan => throw new DeltaIllegalStateException( errorClass = "INTERNAL_ERROR", messageParameters = Array( "Applying type casting resulted in a bad plan rather than a simple expression.\n" + s"Plan:${plan.prettyJson}\n")) } // Cap the maximum length when logging an unresolved expression to avoid issues. This is a // CHECK constraint expression and should be relatively simple. val MAX_OUTPUT_LENGTH = 10 * 1024 if (!resolvedExpr.resolved) { // If the plan is not resolved, check the plan so that a user-facing exception is // thrown. spark.sessionState.analyzer.checkAnalysis(wrappedPlan) deltaAssert( check = false, name = "invariant.unresolvedExpression", msg = s"CHECK constraint child expression was not properly resolved", data = Map( "name" -> name, "checkExpr" -> expr.treeString.take(MAX_OUTPUT_LENGTH), "attributesExtracted" -> attributesExtracted.treeString.take(MAX_OUTPUT_LENGTH), "analyzedLogicalPlan" -> analyzedLogicalPlan.treeString.take(MAX_OUTPUT_LENGTH), "optimizedLogicalPlan" -> optimizedLogicalPlan.treeString.take(MAX_OUTPUT_LENGTH), "resolvedExpr" -> resolvedExpr.treeString.take(MAX_OUTPUT_LENGTH) ) ) } resolvedExpr } CheckDeltaInvariant(executableExpr, columnExtractors.toSeq, constraint) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/constraints/ExpressionLogicalPlanWrapper.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.constraints import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} import org.apache.spark.sql.catalyst.plans.logical.LeafNode /** * A dummy wrapper for expressions so we can pass them to the [[Analyzer]]. */ private[constraints] case class ExpressionLogicalPlanWrapper(e: Expression) extends LeafNode { override def output: Seq[Attribute] = Seq.empty } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/constraints/Invariants.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.constraints import org.apache.spark.sql.delta.DeltaErrors import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.types.StructType /** * List of invariants that can be defined on a Delta table that will allow us to perform * validation checks during changes to the table. */ object Invariants { sealed trait Rule { val name: String } /** Used for columns that should never be null. */ case object NotNull extends Rule { override val name: String = "NOT NULL" } sealed trait RulePersistedInMetadata { def wrap: PersistedRule def json: String = JsonUtils.toJson(wrap) } /** Rules that are persisted in the metadata field of a schema. */ case class PersistedRule(expression: PersistedExpression = null) { def unwrap: RulePersistedInMetadata = { if (expression != null) { expression } else { null } } } /** A SQL expression to check for when writing out data. */ case class ArbitraryExpression(expression: Expression) extends Rule { override val name: String = s"EXPRESSION($expression)" } object ArbitraryExpression { def apply(sparkSession: SparkSession, exprString: String): ArbitraryExpression = { val expr = sparkSession.sessionState.sqlParser.parseExpression(exprString) ArbitraryExpression(expr) } } /** Persisted companion of the ArbitraryExpression rule. */ case class PersistedExpression(expression: String) extends RulePersistedInMetadata { override def wrap: PersistedRule = PersistedRule(expression = this) } /** Extract invariants from the given schema */ def getFromSchema(schema: StructType, spark: SparkSession): Seq[Constraint] = { val columns = SchemaUtils.filterRecursively(schema, checkComplexTypes = false) { field => !field.nullable || field.metadata.contains(INVARIANTS_FIELD) } columns.map { case (parents, field) if !field.nullable => Constraints.NotNull(parents :+ field.name) case (parents, field) => val rule = field.metadata.getString(INVARIANTS_FIELD) val invariant = Option(JsonUtils.mapper.readValue[PersistedRule](rule).unwrap) match { case Some(PersistedExpression(exprString)) => ArbitraryExpression(spark, exprString) case _ => throw DeltaErrors.unrecognizedInvariant() } Constraints.Check(invariant.name, invariant.expression) } } val INVARIANTS_FIELD = "delta.invariants" } /** A rule applied on a column to ensure data hygiene. */ case class Invariant(column: Seq[String], rule: Invariants.Rule) ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/constraints/tableChanges.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.constraints import org.apache.spark.sql.connector.catalog.TableChange /** * Change to add a CHECK constraint to a table. * * @param constraintName The name of the new constraint. Note that constraint names are * case insensitive. * @param expr The expression to add, as a SQL parseable string. */ case class AddConstraint(constraintName: String, expr: String) extends TableChange {} /** * Change to drop a constraint from a table. Note that this is always idempotent - no error * will be thrown if the constraint doesn't exist. * * @param constraintName the name of the constraint to drop - case insensitive * @param ifExists if false, throws an error if the constraint to be dropped does not exist */ case class DropConstraint(constraintName: String, ifExists: Boolean) extends TableChange {} ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/AbstractBatchBackfillingCommitCoordinatorClient.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import java.nio.file.FileAlreadyExistsException import java.util.{Optional, UUID} import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.TransactionExecutionObserver import org.apache.spark.sql.delta.actions.CommitInfo import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.util.FileNames import io.delta.storage.LogStore import io.delta.storage.commit.{CommitCoordinatorClient, CommitFailedException => JCommitFailedException, CommitResponse, CoordinatedCommitsUtils => JCoordinatedCommitsUtils, TableDescriptor, TableIdentifier, UpdatedActions} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.internal.{Logging, MDC} /** * An abstract [[CommitCoordinatorClient]] which triggers backfills every n commits. * - every commit version which satisfies `commitVersion % batchSize == 0` will trigger a backfill. */ trait AbstractBatchBackfillingCommitCoordinatorClient extends CommitCoordinatorClient with Logging { /** * Size of batch that should be backfilled. So every commit version which satisfies * `commitVersion % batchSize == 0` will trigger a backfill. */ val batchSize: Long /** * Commit a given `commitFile` to the table represented by given `logPath` at the * given `commitVersion` */ private[delta] def commitImpl( logStore: LogStore, hadoopConf: Configuration, logPath: Path, coordinatedCommitsTableConf: Map[String, String], commitVersion: Long, commitFile: FileStatus, commitTimestamp: Long): CommitResponse override def commit( logStore: LogStore, hadoopConf: Configuration, tableDesc: TableDescriptor, commitVersion: Long, actions: java.util.Iterator[String], updatedActions: UpdatedActions): CommitResponse = { val logPath = tableDesc.getLogPath val executionObserver = TransactionExecutionObserver.getObserver val tablePath = JCoordinatedCommitsUtils.getTablePath(logPath) if (commitVersion == 0) { throw new JCommitFailedException(false, false, "Commit version 0 must go via filesystem.") } logInfo(log"Attempting to commit version " + log"${MDC(DeltaLogKeys.VERSION, commitVersion)} on table " + log"${MDC(DeltaLogKeys.PATH, tablePath)}") val fs = logPath.getFileSystem(hadoopConf) if (batchSize <= 1) { // Backfill until `commitVersion - 1` logInfo(log"Making sure commits are backfilled until " + log"${MDC(DeltaLogKeys.VERSION, commitVersion - 1)} version for" + log" table ${MDC(DeltaLogKeys.PATH, tablePath.toString)}") backfillToVersion( logStore, hadoopConf, tableDesc, commitVersion - 1, null) } // Write new commit file in `_staged_commits` directory val fileStatus = JCoordinatedCommitsUtils.writeUnbackfilledCommitFile( logStore, hadoopConf, logPath.toString, commitVersion, actions, generateUUID()) // Do the actual commit val commitTimestamp = updatedActions.getCommitInfo.getCommitTimestamp var commitResponse = commitImpl( logStore, hadoopConf, logPath, tableDesc.getTableConf.asScala.toMap, commitVersion, fileStatus, commitTimestamp) val mcToFsConversion = isCoordinatedCommitsToFSConversion(commitVersion, updatedActions) // Backfill if needed executionObserver.beginBackfill() if (batchSize <= 1) { // Always backfill when batch size is configured as 1 backfill(logStore, hadoopConf, logPath, commitVersion, fileStatus) val targetFile = FileNames.unsafeDeltaFile(logPath, commitVersion) val targetFileStatus = fs.getFileStatus(targetFile) val newCommit = commitResponse.getCommit.withFileStatus(targetFileStatus) commitResponse = new CommitResponse(newCommit) } else if (commitVersion % batchSize == 0 || mcToFsConversion) { logInfo(log"Making sure commits are backfilled till " + log"${MDC(DeltaLogKeys.VERSION, commitVersion)} " + log"version for table ${MDC(DeltaLogKeys.PATH, tablePath.toString)}") backfillToVersion(logStore, hadoopConf, tableDesc, commitVersion, null) } logInfo(log"Commit ${MDC(DeltaLogKeys.VERSION, commitVersion)} done successfully on table " + log"${MDC(DeltaLogKeys.PATH, tablePath)}") commitResponse } private def isCoordinatedCommitsToFSConversion( commitVersion: Long, updatedActions: UpdatedActions): Boolean = { val oldMetadataHasCoordinatedCommits = JCoordinatedCommitsUtils.getCoordinatorName(updatedActions.getOldMetadata).isPresent val newMetadataHasCoordinatedCommits = JCoordinatedCommitsUtils.getCoordinatorName(updatedActions.getNewMetadata).isPresent oldMetadataHasCoordinatedCommits && !newMetadataHasCoordinatedCommits && commitVersion > 0 } protected def generateUUID(): String = UUID.randomUUID().toString override def backfillToVersion( logStore: LogStore, hadoopConf: Configuration, tableDesc: TableDescriptor, version: Long, lastKnownBackfilledVersionOpt: java.lang.Long): Unit = { val logPath = tableDesc.getLogPath // Confirm the last backfilled version by checking the backfilled delta file's existence. val validLastKnownBackfilledVersionOpt = Option(lastKnownBackfilledVersionOpt) .filter { version => val fs = logPath.getFileSystem(hadoopConf) fs.exists(FileNames.unsafeDeltaFile(logPath, version)) } val startVersionOpt: Long = validLastKnownBackfilledVersionOpt.map(_ + 1).map(Long.box).orNull getCommits(tableDesc, startVersionOpt, version) .getCommits.asScala .foreach { commit => backfill(logStore, hadoopConf, logPath, commit.getVersion, commit.getFileStatus) } } /** Backfills a given `fileStatus` to `version`.json */ protected def backfill( logStore: LogStore, hadoopConf: Configuration, logPath: Path, version: Long, fileStatus: FileStatus): Unit = { val targetFile = FileNames.unsafeDeltaFile(logPath, version) logInfo(log"Backfilling commit ${MDC(DeltaLogKeys.PATH, fileStatus.getPath)} to " + log"${MDC(DeltaLogKeys.PATH2, targetFile.toString)}") val commitContentIterator = logStore.read(fileStatus.getPath, hadoopConf) try { logStore.write( targetFile, commitContentIterator, false, hadoopConf) registerBackfill(logPath, version) } catch { case _: FileAlreadyExistsException => logInfo(log"The backfilled file ${MDC(DeltaLogKeys.FILE_NAME, targetFile)} already exists.") } finally { commitContentIterator.close() } } /** * Callback to tell the CommitCoordinator that all commits <= `backfilledVersion` are backfilled. */ protected[delta] def registerBackfill( logPath: Path, backfilledVersion: Long): Unit } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CommitCoordinatorClient.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import scala.collection.mutable import io.delta.dynamodbcommitcoordinator.DynamoDBCommitCoordinatorClientBuilder import io.delta.storage.commit.CommitCoordinatorClient import org.apache.spark.sql.SparkSession object CommitCoordinatorClient { def semanticEquals( commitCoordinatorClientOpt1: Option[CommitCoordinatorClient], commitCoordinatorClientOpt2: Option[CommitCoordinatorClient]): Boolean = { (commitCoordinatorClientOpt1, commitCoordinatorClientOpt2) match { case (Some(commitCoordinatorClient1), Some(commitCoordinatorClient2)) => commitCoordinatorClient1.semanticEquals(commitCoordinatorClient2) case (None, None) => true case _ => false } } } /** A builder interface for [[CommitCoordinatorClient]] */ trait CommitCoordinatorBuilder { /** Name of the commit-coordinator */ def getName: String /** Returns a commit-coordinator client based on the given conf */ def build(spark: SparkSession, conf: Map[String, String]): CommitCoordinatorClient } /** An extended builder interface for [[CommitCoordinatorClient]] with CatalogOwned table feature */ trait CatalogOwnedCommitCoordinatorBuilder extends CommitCoordinatorBuilder { /** Returns a catalog-owned commit-coordinator client based for the given catalog. */ def buildForCatalog(spark: SparkSession, catalogName: String): CommitCoordinatorClient } /** Factory to get the correct [[CommitCoordinatorClient]] for a table */ object CommitCoordinatorProvider { // mapping from different commit-coordinator names to the corresponding // [[CommitCoordinatorBuilder]]s. private val nameToBuilderMapping = mutable.Map.empty[String, CommitCoordinatorBuilder] /** Registers a new [[CommitCoordinatorBuilder]] with the [[CommitCoordinatorProvider]] */ def registerBuilder(commitCoordinatorBuilder: CommitCoordinatorBuilder): Unit = synchronized { nameToBuilderMapping.get(commitCoordinatorBuilder.getName) match { case Some(existingBuilder: CommitCoordinatorBuilder) => throw new IllegalArgumentException( s"commit-coordinator: ${existingBuilder.getName} already" + s" registered with builder ${existingBuilder.getClass.getName}") case None => nameToBuilderMapping.put(commitCoordinatorBuilder.getName, commitCoordinatorBuilder) } } /** * Returns a [[CommitCoordinatorClient]] for the given `name`, `conf`, and `spark`. * If the commit-coordinator with the given name is not registered, an exception is thrown. */ def getCommitCoordinatorClient( name: String, conf: Map[String, String], spark: SparkSession): CommitCoordinatorClient = synchronized { getCommitCoordinatorClientOpt(name, conf, spark).getOrElse { throw new IllegalArgumentException(s"Unknown commit-coordinator: $name") } } /** * Returns a [[CommitCoordinatorClient]] for the given `name`, `conf`, and `spark`. * Returns None if the commit-coordinator with the given name is not registered. */ def getCommitCoordinatorClientOpt( name: String, conf: Map[String, String], spark: SparkSession): Option[CommitCoordinatorClient] = synchronized { nameToBuilderMapping.get(name).map(_.build(spark, conf)) } def getRegisteredCoordinatorNames: Seq[String] = synchronized { nameToBuilderMapping.keys.toSeq } // Visible only for UTs private[delta] def clearNonDefaultBuilders(): Unit = synchronized { val initialCommitCoordinatorNames = initialCommitCoordinatorBuilders.map(_.getName).toSet nameToBuilderMapping.retain((k, _) => initialCommitCoordinatorNames.contains(k)) } private[delta] def clearAllBuilders(): Unit = synchronized { nameToBuilderMapping.clear() } private val initialCommitCoordinatorBuilders = Seq[CommitCoordinatorBuilder]( UCCommitCoordinatorBuilder, new DynamoDBCommitCoordinatorClientBuilder() ) initialCommitCoordinatorBuilders.foreach(registerBuilder) } /** Factory to get the correct [[CatalogOwnedCommitCoordinatorBuilder]] for a catalog-owned table */ object CatalogOwnedCommitCoordinatorProvider { // mapping from catalog names to the corresponding [[CatalogOwnedCommitCoordinatorBuilder]]s. private val catalogNameToBuilderMapping = mutable.Map.empty[String, CatalogOwnedCommitCoordinatorBuilder] // Visible only for UTs private[delta] def clearBuilders(): Unit = synchronized { catalogNameToBuilderMapping.clear() } /** Registers a new [[CommitCoordinatorBuilder]] with the [[CommitCoordinatorProvider]] */ def registerBuilder( catalogName: String, commitCoordinatorBuilder: CatalogOwnedCommitCoordinatorBuilder): Unit = synchronized { catalogNameToBuilderMapping.get(catalogName) match { case Some(existingBuilder: CommitCoordinatorBuilder) => throw new IllegalArgumentException( s"commit-coordinator for catalog: $catalogName already" + s" registered with builder ${existingBuilder.getClass.getName}") case None => catalogNameToBuilderMapping.put(catalogName, commitCoordinatorBuilder) } } def getBuilder(catalogName: String): Option[CatalogOwnedCommitCoordinatorBuilder] = catalogNameToBuilderMapping.get(catalogName) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUsageLogs.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits /** Class containing usage logs emitted by Catalog Owned. */ object CatalogOwnedUsageLogs { /** Common prefix for all catalog-owned usage logs. */ val PREFIX = "delta.catalogOwned" /** * Usage log emitted when we populate the commit coordinator via invalid path-based access. * I.e., No catalog table is present when trying to populate commit coordinator, which is * almost a bug case for CC tables. */ val COMMIT_COORDINATOR_POPULATION_INVALID_PATH_BASED_ACCESS = s"$PREFIX.commitCoordinatorPopulation.invalidPathBasedAccess" } /** Class containing usage logs emitted by Coordinated Commits. */ object CoordinatedCommitsUsageLogs { // Common prefix for all coordinated-commits usage logs. private val PREFIX = "delta.coordinatedCommits" // Usage log emitted as part of [[CommitCoordinatorClient.getCommits]] call. val COMMIT_COORDINATOR_CLIENT_GET_COMMITS = s"$PREFIX.commitCoordinatorClient.getCommits" // Usage log emitted when listing files in CommitCoordinatorClient (i.e. getCommits) can't be done // in separate thread because the thread pool is full. val COMMIT_COORDINATOR_LISTING_THREADPOOL_FULL = s"$PREFIX.listDeltaAndCheckpointFiles.GetCommitsThreadpoolFull" // Usage log emitted when we need a 2nd roundtrip to list files in FileSystem. // This happens when: // 1. FileSystem returns File 1/2/3 // 2. CommitCoordinatorClient returns File 5/6 -- 4 got backfilled by the time our request reached // CommitCoordinatorClient // 3. We need to list again in FileSystem to get File 4. val COMMIT_COORDINATOR_ADDITIONAL_LISTING_REQUIRED = s"$PREFIX.listDeltaAndCheckpointFiles.requiresAdditionalFsListing" // Usage log emitted when listing files via FileSystem and CommitCoordinatorClient // (i.e. getCommits) shows an unexpected gap. val FS_COMMIT_COORDINATOR_LISTING_UNEXPECTED_GAPS = s"$PREFIX.listDeltaAndCheckpointFiles.unexpectedGapsInResults" // Usage log emitted when a requested Commit Coordinator implementation is missing. val COMMIT_COORDINATOR_MISSING_IMPLEMENTATION = s"$PREFIX.commitCoordinator.missingImplementation" // Usage log emitted when a client attempts to write to a CC table even though the // commit coordinator implementation is missing. val COMMIT_COORDINATOR_MISSING_IMPLEMENTATION_WRITE = s"$PREFIX.commitCoordinator.missingImplementationWrite" } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import java.util.Optional import scala.collection.JavaConverters._ import scala.util.control.NonFatal import org.apache.spark.sql.delta.{CatalogOwnedTableFeature, CheckpointPolicy, CoordinatedCommitsTableFeature, DeletionVectorsTableFeature, DeltaConfig, DeltaConfigs, DeltaErrors, DeltaIllegalArgumentException, DeltaLog, NameMapping, OptimisticTransaction, RowTrackingFeature, Snapshot, SnapshotDescriptor, TableFeature, V2CheckpointTableFeature} import org.apache.spark.sql.delta.actions.{Metadata, Protocol, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.commands.CloneTableCommand import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import org.apache.spark.sql.delta.util.FileNames.{BackfilledDeltaFile, CompactedDeltaFile, DeltaFile, UnbackfilledDeltaFile} import io.delta.storage.LogStore import io.delta.storage.commit.{CommitCoordinatorClient, GetCommitsResponse => JGetCommitsResponse, TableIdentifier} import io.delta.storage.commit.actions.AbstractMetadata import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.internal.MDC import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.{TableIdentifier => CatalystTableIdentifier} import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.connector.catalog.CatalogPlugin import org.apache.spark.util.Utils object CatalogOwnedTableUtils extends DeltaLogging { /** The default catalog name only used for testing. */ val DEFAULT_CATALOG_NAME_FOR_TESTING: String = "spark_catalog" /** * Sets or removes CatalogOwnedTableFeature from a transaction's protocol. * * This helper method is used to control whether a transaction should use CC. * * @param txn The OptimisticTransaction to modify protocol and metadata on * @param withCatalogOwnedTableFeature If true, adds CatalogOwnedTableFeature to the protocol * and ensures ICT enablement. If false, removes it. * * @note When adding CatalogOwnedTableFeature, this method also forces a metadata update * to ensure ICT (In-Commit Timestamp) is properly enabled. See the below comment * for details. */ def setTxnProtocol(txn: OptimisticTransaction, withCatalogOwnedTableFeature: Boolean): Unit = { if (withCatalogOwnedTableFeature) { val p = txn.protocol.withFeature(feature = CatalogOwnedTableFeature) txn.updateProtocol(protocol = p) // Force a metadata update to trigger ICT (In-Commit Timestamp) enablement. // CatalogOwnedTableFeature requires ICT to be enabled in the metadata, but // updateMetadataAndProtocolWithRequiredFeatures in prepareCommit only runs // when there's an explicit metadata change (metadataChanges.headOption is non-empty). // Since we're only updating the protocol here without changing metadata content, // we need to explicitly call updateMetadata to ensure metadataChanges is non-empty // during prepareCommit, which will then enable ICT in the metadata. // Without this, generateInCommitTimestampForFirstCommitAttempt would return None, // causing UCCommitCoordinatorClient to fail with DELTA_MISSING_COMMIT_TIMESTAMP. txn.updateMetadata(proposedNewMetadata = txn.metadata) } else { val p = txn.protocol.removeFeature(targetFeature = CatalogOwnedTableFeature) txn.updateProtocol(protocol = p) } } // Populate table commit coordinator using table identifier inside CatalogTable. def populateTableCommitCoordinatorFromCatalog( spark: SparkSession, catalogTableOpt: Option[CatalogTable], snapshot: Snapshot): Option[TableCommitCoordinatorClient] = { if (!snapshot.isCatalogOwned) { return None } catalogTableOpt.map { catalogTable => // Resolve commit coordinator name by contacting catalog. val cc = getCommitCoordinator( spark, catalogTable.identifier).getOrElse { throw new IllegalStateException( "Couldn't locate commit coordinator for: " + catalogTable.identifier) } val tableConf = snapshot.metadata.configuration TableCommitCoordinatorClient( commitCoordinatorClient = cc, logPath = snapshot.deltaLog.logPath, tableConf = tableConf, hadoopConf = snapshot.deltaLog.newDeltaHadoopConf(), logStore = snapshot.deltaLog.store ) } .orElse { if (Utils.isTesting) { // In unit test with a path based access, it is possible to enable CatalogOwned with // in-memory commit coordinator. In this case, we return table commit coordinator // registered in the provider so that it can still test CatalogOwned table feature // capability. CatalogOwnedCommitCoordinatorProvider.getBuilder(DEFAULT_CATALOG_NAME_FOR_TESTING) .flatMap{ builder => Some(builder.buildForCatalog(spark, DEFAULT_CATALOG_NAME_FOR_TESTING)) } .map { cc => return Some(TableCommitCoordinatorClient( cc, logPath = snapshot.deltaLog.logPath, tableConf = snapshot.metadata.configuration, hadoopConf = snapshot.deltaLog.newDeltaHadoopConf(), logStore = snapshot.deltaLog.store) ) } } // This table is catalog owned table but catalogTableOpt is not defined. This means // that the caller is accessing this table by path-based or the calling code path is missing // the CatalogTable. // TODO: Better error message with proper error code. logAndThrowPathBasedAccessNotAllowed(snapshot) } } // Directly returns the commit coordinator client for the given catalog table. def getCommitCoordinator( spark: SparkSession, identifier: CatalystTableIdentifier): Option[CommitCoordinatorClient] = { identifier.nameParts match { case spark.sessionState.analyzer.CatalogAndIdentifier(catalog, _) => CatalogOwnedCommitCoordinatorProvider.getBuilder(catalog.name) .map(_.buildForCatalog(spark, catalog.name)).orElse { if (catalog.getClass.getName == UCCommitCoordinatorBuilder.UNITY_CATALOG_CONNECTOR_CLASS) { Some(UCCommitCoordinatorBuilder.buildForCatalog(spark, catalog.name)) } else { None } } case _ => throw new IllegalStateException( "Failed to resolve the catalog: " + identifier) } } /** * Returns the catalog name from the given catalog table identifier. * If the catalog table is not present, returns None. */ def getCatalogName( spark: SparkSession, identifier: CatalystTableIdentifier): Option[String] = { identifier.nameParts match { case spark.sessionState.analyzer.CatalogAndIdentifier(catalog, _) => if (catalog.getClass.getName == UCCommitCoordinatorBuilder.UNITY_CATALOG_CONNECTOR_CLASS) { // UC is the current commit coordinator. Some(UCCommitCoordinatorBuilder.COORDINATOR_NAME) } else { // Other catalog (e.g., `spark_catalog`) is the commit coordinator. Some(catalog.name) } case _ => None } } /** * The "Quality of Life" table features that will be enabled automatically * when creating CatalogOwned tables. * Note that we also include the properties (i.e., DeltaConfig and target value) * used to determine whether the table features and the corresponding * properties/metadata have been enabled or not. */ val QOL_TABLE_FEATURES_AND_PROPERTIES: Seq[(TableFeature, DeltaConfig[_], String)] = qolTableFeatureAndProperties def qolTableFeatureAndProperties: Seq[(TableFeature, DeltaConfig[_], String)] = Seq( (DeletionVectorsTableFeature, DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION, "true"), (V2CheckpointTableFeature, DeltaConfigs.CHECKPOINT_POLICY, CheckpointPolicy.V2.name), (RowTrackingFeature, DeltaConfigs.ROW_TRACKING_ENABLED, "true") ) /** * Return true if we should enable CatalogOwned either via default spark * session configuration during creating a new table, * or via the explicit table property overrides. */ def shouldEnableCatalogOwned( spark: SparkSession, propertyOverrides: Map[String, String], isCreatingNew: Boolean = true): Boolean = { // Check explicit property overrides when creating a new or upgrading an existing table. val isExplicitlyEnablingCO = TableFeatureProtocolUtils.getSupportedFeaturesFromTableConfigs( configs = propertyOverrides).contains(CatalogOwnedTableFeature) // Check default spark session configuration only when creating a new table. val isEnablingCOByDefault = isCreatingNew && CatalogOwnedTableUtils.defaultCatalogOwnedEnabled(spark) isExplicitlyEnablingCO || isEnablingCOByDefault } /** * Checks if a configuration is already set in metadata or Spark defaults. * Ensures we don't override user preferences. */ private def isAlreadyConfigured( config: DeltaConfig[_], configuration: Map[String, String], spark: SparkSession): Boolean = { configuration.contains(config.key) || spark.sessionState.conf.contains(config.defaultTablePropertyKey) } /** * Updates table metadata with appropriate QoL features for CatalogManaged tables. * * Main entry point for QoL feature enablement during table creation. * See [[getQoLConfigsToAdd]] for the logic that determines which features are added. * * @param spark SparkSession for configuration * @param metadata Table metadata to update * @return Updated metadata with QoL features */ def updateMetadataForQoLFeatures( spark: SparkSession, metadata: Metadata): Metadata = { val qoLConfigsToAdd = QOL_TABLE_FEATURES_AND_PROPERTIES.collect { case (feature, config, targetValue) if !isAlreadyConfigured(config, metadata.configuration, spark) => config.key -> targetValue }.toMap metadata.copy(configuration = metadata.configuration ++ qoLConfigsToAdd) } val ICT_TABLE_PROPERTY_CONFS = Seq( DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED, DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION, DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP) /** * The main ICT table properties used as dependencies for Catalog-Owned enabled table. */ val ICT_TABLE_PROPERTY_KEYS: Seq[String] = ICT_TABLE_PROPERTY_CONFS.map(_.key) /** * Verifies that the property keys do not contain any ICT dependencies for Catalog-Owned. */ private[delta] def verifyNotContainsICTConfigurations(propKeys: Seq[String]): Unit = { ICT_TABLE_PROPERTY_KEYS.foreach { key => if (propKeys.contains(key)) { throw new DeltaIllegalArgumentException( "DELTA_CANNOT_MODIFY_CATALOG_MANAGED_DEPENDENCIES", messageParameters = Array.empty) } } } /** * Validates the Catalog-Owned configurations in explicit command overrides for * `AlterTableSetPropertiesDeltaCommand`. * * If [[CatalogOwnedTableFeature]] presents, we do NOT allow users to * modify any ICT properties that Catalog-Owned depends on. */ def validatePropertiesForAlterTableSetPropertiesDeltaCommand( snapshot: Snapshot, propertyOverrides: Map[String, String]): Unit = { if (snapshot.isCatalogOwned) { // For Catalog-Owned enabled tables, check the dependent ICT properties. // Note: Upgrade/Downgrade have been blocked earlier, which do not need to be // checked here. verifyNotContainsICTConfigurations(propKeys = propertyOverrides.keys.toSeq) } } /** * Validates the configurations to unset for `AlterTableUnsetPropertiesDeltaCommand`. * * If the table already has [[CatalogOwnedTableFeature]] present, * we do not allow users to unset any of the ICT properties that Catalog-Owned depends on. */ def validatePropertiesForAlterTableUnsetPropertiesDeltaCommand( snapshot: Snapshot, propKeysToUnset: Seq[String]): Unit = { if (snapshot.isCatalogOwned) { verifyNotContainsICTConfigurations(propKeys = propKeysToUnset) } } /** * Validates the CatalogManaged properties in explicit command overrides and default * SparkSession properties for `CreateDeltaTableCommand`. * * @param spark The SparkSession. * @param tableExists Whether the table already exists. * @param query The query to be executed (e.g., CloneTableCommand). * @param catalogTableProperties The table properties from the catalog table. * @param existingTableSnapshotOpt The snapshot of the existing table, if it exists. */ def validatePropertiesForCreateDeltaTableCommand( spark: SparkSession, tableExists: Boolean, query: Option[LogicalPlan], catalogTableProperties: Map[String, String], existingTableSnapshotOpt: Option[Snapshot] = None): Unit = { val (command, propertyOverrides) = query match { // For CLONE, we cannot use the properties from the catalog table, because they are already // the result of merging the source table properties with the overrides, but we do not // consider the source table properties for CatalogManaged tables. case Some(cmd: CloneTableCommand) => (if (tableExists) "REPLACE with CLONE" else "CREATE with CLONE", cmd.tablePropertyOverrides) case _ => (if (tableExists) "REPLACE" else "CREATE", catalogTableProperties) } // We do not allow users to modify [[UCCommitCoordinatorClient.UC_TABLE_ID_KEY]] and // [[CatalogOwnedTableFeature.name]] in any explicit overrides for REPLACE command. if (tableExists) { // Must be "REPLACE" or "REPLACE with CLONE" if the table already exists. assert(command == "REPLACE with CLONE" || command == "REPLACE", s"Unexpected command: $command") validateUCTableIdNotPresent(property = propertyOverrides) val isSpecifyingCatalogManaged = TableFeatureProtocolUtils .getSupportedFeaturesFromTableConfigs(propertyOverrides) .contains(CatalogOwnedTableFeature) val existingTableIsCatalogManaged = existingTableSnapshotOpt.exists(_.isCatalogOwned) // Allow specifying CatalogManaged in REPLACE TABLE if the existing table is already // CatalogManaged. In this case, the commit coordinator properties are treated as a no-op. // Block if trying to enable CatalogManaged on a non-CatalogManaged table via REPLACE. // Users should either upgrade the existing table or create a fresh CatalogManaged table. // // Note: We intentionally use `&& !` instead of `!=` here. Using `!=` would also block the // case where a CatalogManaged table is replaced without explicitly specifying CatalogManaged // properties, which would hurt customer experience by forcing them to always specify CC // properties on every REPLACE command. Since the existing Delta behavior already preserves // the CatalogManaged status during REPLACE (the table type won't change), there's no need // to block that case. if (isSpecifyingCatalogManaged && !existingTableIsCatalogManaged) { throw new IllegalStateException( "Specifying CatalogManaged in REPLACE TABLE command is not supported " + "for tables that are not already CatalogManaged. " + "Please either upgrade the existing table or create a fresh CatalogManaged table.") } } } /** * Validates that the UC table ID is not present in the provided property (overrides). * Errors out if it is present. * * @param property The property to validate. */ def validateUCTableIdNotPresent(property: Map[String, String]): Unit = { if (property.contains(UCCommitCoordinatorClient.UC_TABLE_ID_KEY)) { throw DeltaErrors.cannotModifyTableProperty( prop = UCCommitCoordinatorClient.UC_TABLE_ID_KEY) } } /** * Whether Catalog-Owned is enabled via default SparkSession configuration. * * @param spark The SparkSession to check. * @return True if Catalog-Owned is enabled by default, false otherwise. */ def defaultCatalogOwnedEnabled(spark: SparkSession): Boolean = { spark.conf .getOption(TableFeatureProtocolUtils.defaultPropertyKey(CatalogOwnedTableFeature)) .contains("supported") } /** * Helper function to log invalid path-based access and throw the appropriate error. * * @param snapshot The snapshot being processed */ private def logAndThrowPathBasedAccessNotAllowed(snapshot: Snapshot): Nothing = { recordCommitCoordinatorPopulationUsageLog( snapshot.deltaLog, opType = CatalogOwnedUsageLogs.COMMIT_COORDINATOR_POPULATION_INVALID_PATH_BASED_ACCESS, snapshot, catalogTableOpt = None, commitCoordinatorOpt = None, includeStackTrace = true, includeAdditionalDiagnostics = true ) throw DeltaErrors.catalogManagedTablePathBasedAccessNotAllowed(snapshot.path) } /** * Records usage logs for commit coordinator population with common fields. * * @param deltaLog The delta log instance * @param opType The operation type for the usage log * @param snapshot The table snapshot * @param catalogTableOpt Optional catalog table information * @param commitCoordinatorOpt Optional commit coordinator instance * @param includeStackTrace Whether to include stack trace in the log * @param includeAdditionalDiagnostics Whether to include additional diagnostic information */ private def recordCommitCoordinatorPopulationUsageLog( deltaLog: DeltaLog, opType: String, snapshot: Snapshot, catalogTableOpt: Option[CatalogTable], commitCoordinatorOpt: Option[CommitCoordinatorClient] = None, includeStackTrace: Boolean = false, includeAdditionalDiagnostics: Boolean = false): Unit = { // Base data that's common to *all* usage logs for tccc population. val baseData = Map( "version" -> snapshot.version.toString, "path" -> snapshot.path.toString ) val catalogData = catalogTableOpt.map { catalogTable => Map( "catalogTable.identifier" -> catalogTable.identifier.toString, "catalogTable.tableType" -> catalogTable.tableType.toString ) }.getOrElse(Map.empty[String, String]) val coordinatorData = commitCoordinatorOpt.map { cc => Map("commitCoordinator.getClass" -> cc.getClass.getName) }.getOrElse(Map.empty[String, String]) val stackTraceData = if (includeStackTrace) { Map("stackTrace" -> Thread.currentThread().getStackTrace.tail.mkString("\n\t")) } else { Map.empty[String, String] } val diagnosticData = if (includeAdditionalDiagnostics) { Map( "latestCheckpointVersion" -> snapshot.checkpointProvider.version, "checksumOpt" -> snapshot.checksumOpt, "properties" -> snapshot.getProperties, "logStore" -> snapshot.deltaLog.store.getClass.getName ) } else { Map.empty[String, Any] } val allData = baseData ++ catalogData ++ coordinatorData ++ stackTraceData ++ diagnosticData recordDeltaEvent( deltaLog = deltaLog, opType = opType, data = allData ) } } object CoordinatedCommitsUtils extends DeltaLogging { /** * Returns the [[CommitCoordinatorClient.getCommits]] response for the given startVersion and * versionToLoad. */ def getCommitsFromCommitCoordinatorWithUsageLogs( deltaLog: DeltaLog, tableCommitCoordinatorClient: TableCommitCoordinatorClient, catalogTableOpt: Option[CatalogTable], startVersion: Long, versionToLoad: Option[Long], isAsyncRequest: Boolean): JGetCommitsResponse = { recordFrameProfile("DeltaLog", s"CommitCoordinatorClient.getCommits.async=$isAsyncRequest") { val startTimeMs = System.currentTimeMillis() def recordEvent(additionalData: Map[String, Any]): Unit = { recordDeltaEvent( deltaLog, opType = CoordinatedCommitsUsageLogs.COMMIT_COORDINATOR_CLIENT_GET_COMMITS, data = Map( "startVersion" -> startVersion, "versionToLoad" -> versionToLoad.getOrElse(-1L), "async" -> isAsyncRequest.toString, "durationMs" -> (System.currentTimeMillis() - startTimeMs).toString ) ++ additionalData ) } try { val response = tableCommitCoordinatorClient.getCommits( catalogTableOpt.map(_.identifier), Some(startVersion), endVersion = versionToLoad ) val additionalEventData = Map( "responseCommitsSize" -> response.getCommits.size, "responseLatestTableVersion" -> response.getLatestTableVersion) recordEvent(additionalEventData) response } catch { case NonFatal(e) => recordEvent( Map( "exceptionClass" -> e.getClass.getName, "exceptionString" -> Utils.exceptionString(e) ) ) throw e } } } /** * Returns an iterator of commit files starting from startVersion. * If the iterator is consumed beyond what the file system listing shows, this method do a * deltaLog.update() to find the latest version and returns listing results upto that version. * * @return an iterator of (file status, version) pair corresponding to commit files */ def commitFilesIterator( deltaLog: DeltaLog, catalogTableOpt: Option[CatalogTable], startVersion: Long): Iterator[(FileStatus, Long)] = { def listDeltas(startVersion: Long, endVersion: Option[Long]): Iterator[(FileStatus, Long)] = { deltaLog .listFrom(startVersion) .collect { case DeltaFile(fileStatus, version) => (fileStatus, version) } .takeWhile { case (_, version) => endVersion.forall(version <= _) } } var maxVersionSeen = startVersion - 1 val listedDeltas = listDeltas(startVersion, endVersion = None).filter { case (_, version) => maxVersionSeen = math.max(maxVersionSeen, version) true } def tailFromSnapshot(): Iterator[(FileStatus, Long)] = { val currentSnapshotInDeltaLog = deltaLog.unsafeVolatileSnapshot if (currentSnapshotInDeltaLog.version == maxVersionSeen && (currentSnapshotInDeltaLog.tableCommitCoordinatorClientOpt.isEmpty && !currentSnapshotInDeltaLog.isCatalogOwned)) { // If the last version in listing is same as the `unsafeVolatileSnapshot` in deltaLog and // if that snapshot doesn't have a commit-coordinator => this table was not a // coordinated-commits table at the time of listing. This is because the commit which // converts the file-system table to a coordinated-commits table must be a file-system // commit as per the spec. return Iterator.empty } val endSnapshot = deltaLog.update(catalogTableOpt = catalogTableOpt) // No need to worry if we already reached the end if (maxVersionSeen >= endSnapshot.version) { return Iterator.empty } val unbackfilledDeltas = endSnapshot.logSegment.deltas.collect { case UnbackfilledDeltaFile(fileStatus, version, _) if version > maxVersionSeen => (fileStatus, version) } // Check for a gap between listing and commit files in the logsegment val gapListing = unbackfilledDeltas.headOption match { case Some((_, version)) if maxVersionSeen + 1 < version => listDeltas(maxVersionSeen + 1, Some(version)) // no gap before case _ => Iterator.empty } gapListing ++ unbackfilledDeltas } // We want to avoid invoking `tailFromSnapshot()` as it internally calls deltaLog.update() // So we append the two iterators and the second iterator will be created only if the first one // is exhausted. Iterator(1, 2).flatMap { case 1 => listedDeltas case 2 => tailFromSnapshot() } } def getCommitCoordinatorClient( spark: SparkSession, deltaLog: DeltaLog, // Used for logging metadata: Metadata, protocol: Protocol, failIfImplUnavailable: Boolean): Option[CommitCoordinatorClient] = { metadata.coordinatedCommitsCoordinatorName.flatMap { commitCoordinatorStr => assert(protocol.isFeatureSupported(CoordinatedCommitsTableFeature), "coordinated commits table feature is not supported") val coordinatorConf = metadata.coordinatedCommitsCoordinatorConf val coordinatorOpt = CommitCoordinatorProvider.getCommitCoordinatorClientOpt( commitCoordinatorStr, coordinatorConf, spark) if (coordinatorOpt.isEmpty) { recordDeltaEvent( deltaLog, CoordinatedCommitsUsageLogs.COMMIT_COORDINATOR_MISSING_IMPLEMENTATION, data = Map( "commitCoordinatorName" -> commitCoordinatorStr, "registeredCommitCoordinators" -> CommitCoordinatorProvider.getRegisteredCoordinatorNames.mkString(", "), "commitCoordinatorConf" -> coordinatorConf, "failIfImplUnavailable" -> failIfImplUnavailable.toString ) ) if (failIfImplUnavailable) { throw new IllegalArgumentException( s"Unknown commit-coordinator: $commitCoordinatorStr") } } coordinatorOpt } } /** * Get the table commit coordinator client from the provided snapshot descriptor. * Returns None if either this is not a coordinated-commits table. Also returns None when * `failIfImplUnavailable` is false and the commit-coordinator implementation is not available. */ def getTableCommitCoordinator( spark: SparkSession, deltaLog: DeltaLog, // Used for logging snapshotDescriptor: SnapshotDescriptor, failIfImplUnavailable: Boolean): Option[TableCommitCoordinatorClient] = { getCommitCoordinatorClient( spark, deltaLog, snapshotDescriptor.metadata, snapshotDescriptor.protocol, failIfImplUnavailable).map { commitCoordinator => TableCommitCoordinatorClient( commitCoordinator, snapshotDescriptor.deltaLog.logPath, snapshotDescriptor.metadata.coordinatedCommitsTableConf, snapshotDescriptor.deltaLog.newDeltaHadoopConf(), snapshotDescriptor.deltaLog.store ) } } def getCoordinatedCommitsConfs(metadata: Metadata): (Option[String], Map[String, String]) = { metadata.coordinatedCommitsCoordinatorName match { case Some(name) => (Some(name), metadata.coordinatedCommitsCoordinatorConf) case None => (None, Map.empty) } } val TABLE_PROPERTY_CONFS = Seq( DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME, DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF, DeltaConfigs.COORDINATED_COMMITS_TABLE_CONF) val ICT_TABLE_PROPERTY_CONFS = Seq( DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED, DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION, DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP) /** * The main table properties used to instantiate a TableCommitCoordinatorClient. */ val TABLE_PROPERTY_KEYS: Seq[String] = TABLE_PROPERTY_CONFS.map(_.key) /** * The main ICT table properties used as dependencies for Coordinated Commits. */ val ICT_TABLE_PROPERTY_KEYS: Seq[String] = ICT_TABLE_PROPERTY_CONFS.map(_.key) /** * Returns true if any CoordinatedCommits-related table properties is present in the metadata. */ def tablePropertiesPresent(metadata: Metadata): Boolean = { TABLE_PROPERTY_KEYS.exists(metadata.configuration.contains) } /** * Returns true if the snapshot is backed by unbackfilled commits. */ def unbackfilledCommitsPresent(snapshot: Snapshot): Boolean = { snapshot.logSegment.deltas.exists { case FileNames.UnbackfilledDeltaFile(_, _, _) => true case _ => false } && !snapshot.allCommitsBackfilled } /** * This method takes care of backfilling any unbackfilled delta files when coordinated commits is * not enabled on the table (i.e. commit-coordinator is not present) but there are still * unbackfilled delta files in the table. This can happen if an error occurred during the CC -> FS * commit where the commit-coordinator was able to register the downgrade commit but it failed to * backfill it. This method must be invoked before doing the next commit as otherwise there will * be a gap in the backfilled commit sequence. */ def backfillWhenCoordinatedCommitsDisabled(snapshot: Snapshot): Unit = { if (snapshot.getTableCommitCoordinatorForWrites.nonEmpty || snapshot.isCatalogOwned) { // Coordinated commits or Catalog-owned is enabled on the table. Don't backfill // as backfills are managed by commit-coordinators. return } val unbackfilledFilesAndVersions = snapshot.logSegment.deltas.collect { case UnbackfilledDeltaFile(unbackfilledDeltaFile, version, _) => (unbackfilledDeltaFile, version) } if (unbackfilledFilesAndVersions.isEmpty) return // Coordinated commits are disabled on the table but the table still has un-backfilled files. val deltaLog = snapshot.deltaLog val hadoopConf = deltaLog.newDeltaHadoopConf() val fs = deltaLog.logPath.getFileSystem(hadoopConf) val overwrite = !deltaLog.store.isPartialWriteVisible(deltaLog.logPath, hadoopConf) var numAlreadyBackfilledFiles = 0L unbackfilledFilesAndVersions.foreach { case (unbackfilledDeltaFile, version) => val backfilledFilePath = FileNames.unsafeDeltaFile(deltaLog.logPath, version) if (!fs.exists(backfilledFilePath)) { val actionsIter = deltaLog.store.readAsIterator(unbackfilledDeltaFile.getPath, hadoopConf) deltaLog.store.write( backfilledFilePath, actionsIter, overwrite, hadoopConf) logInfo(log"Delta file ${MDC(DeltaLogKeys.PATH, unbackfilledDeltaFile.getPath.toString)} " + log"backfilled to path ${MDC(DeltaLogKeys.PATH2, backfilledFilePath.toString)}.") } else { numAlreadyBackfilledFiles += 1 logInfo(log"Delta file ${MDC(DeltaLogKeys.PATH, unbackfilledDeltaFile.getPath.toString)} " + log"already backfilled.") } } recordDeltaEvent( deltaLog, opType = "delta.coordinatedCommits.backfillWhenCoordinatedCommitsSupportedAndDisabled", data = Map( "numUnbackfilledFiles" -> unbackfilledFilesAndVersions.size, "unbackfilledFiles" -> unbackfilledFilesAndVersions.map(_._1.getPath.toString), "numAlreadyBackfilledFiles" -> numAlreadyBackfilledFiles ) ) } /** * Returns the last backfilled file in the given list of `deltas` if it exists. This could be * 1. A backfilled delta * 2. A minor compaction */ def getLastBackfilledFile(deltas: Seq[FileStatus]): Option[FileStatus] = { var maxFile: Option[FileStatus] = None deltas.foreach { case BackfilledDeltaFile(f, _) => maxFile = Some(f) case CompactedDeltaFile(f, _, _) => maxFile = Some(f) case _ => // do nothing } maxFile } /** * Extracts the Coordinated Commits configurations from the provided properties. */ def getExplicitCCConfigurations( properties: Map[String, String]): Map[String, String] = { properties.filter { case (k, _) => TABLE_PROPERTY_KEYS.contains(k) } } /** * Extracts the ICT configurations from the provided properties. */ def getExplicitICTConfigurations(properties: Map[String, String]): Map[String, String] = { properties.filter { case (k, _) => ICT_TABLE_PROPERTY_KEYS.contains(k) } } /** * Extracts the explicit QoL configurations from the provided properties. * * These are preserved across catalog-managed REPLACE when the existing table already has the * QoL defaults materialized in metadata, so a no-op REPLACE does not accidentally drop them * while rebuilding the configuration map. */ def getExplicitQoLConfigurations(properties: Map[String, String]): Map[String, String] = { val qolKeys = CatalogOwnedTableUtils.QOL_TABLE_FEATURES_AND_PROPERTIES.map(_._2.key).toSet properties.filter { case (k, _) => qolKeys.contains(k) } } /** * Fetches the SparkSession default configurations for ICT. The `withDefaultKey` * flag controls whether the keys in the returned map should have the default prefix or not. */ def getDefaultICTConfigurations( spark: SparkSession, withDefaultKey: Boolean = false): Map[String, String] = { ICT_TABLE_PROPERTY_CONFS.flatMap { conf => spark.conf.getOption(conf.defaultTablePropertyKey).map { value => val finalKey = if (withDefaultKey) conf.defaultTablePropertyKey else conf.key finalKey -> value } }.toMap } /** * Fetches the SparkSession default configurations for Coordinated Commits. The `withDefaultKey` * flag controls whether the keys in the returned map should have the default prefix or not. * For example, if property 'coordinatedCommits.commitCoordinator-preview' is set to 'dynamodb' * in SparkSession default, then * * - fetchDefaultCoordinatedCommitsConfigurations(spark) => * Map("delta.coordinatedCommits.commitCoordinator-preview" -> "dynamodb") * * - fetchDefaultCoordinatedCommitsConfigurations(spark, withDefaultKey = true) => * Map("spark.databricks.delta.properties.defaults * .coordinatedCommits.commitCoordinator-preview" -> "dynamodb") */ def getDefaultCCConfigurations( spark: SparkSession, withDefaultKey: Boolean = false): Map[String, String] = { TABLE_PROPERTY_CONFS.flatMap { conf => spark.conf.getOption(conf.defaultTablePropertyKey).map { value => val finalKey = if (withDefaultKey) conf.defaultTablePropertyKey else conf.key finalKey -> value } }.toMap } /** * Verifies that the properties contain exactly the Coordinator Name and Coordinator Conf. * If `fromDefault` is true, then the properties have keys with the default prefix. */ private def verifyContainsOnlyCoordinatorNameAndConf( properties: Map[String, String], command: String, fromDefault: Boolean): Unit = { Seq(DeltaConfigs.COORDINATED_COMMITS_TABLE_CONF).foreach { conf => if (fromDefault) { if (properties.contains(conf.defaultTablePropertyKey)) { throw new DeltaIllegalArgumentException( errorClass = "DELTA_CONF_OVERRIDE_NOT_SUPPORTED_IN_SESSION", messageParameters = Array( command, conf.defaultTablePropertyKey, conf.defaultTablePropertyKey)) } } else { if (properties.contains(conf.key)) { throw new DeltaIllegalArgumentException( errorClass = "DELTA_CONF_OVERRIDE_NOT_SUPPORTED_IN_COMMAND", messageParameters = Array(command, conf.key)) } } } Seq( DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME, DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF).foreach { conf => if (fromDefault) { if (!properties.contains(conf.defaultTablePropertyKey)) { throw new DeltaIllegalArgumentException( errorClass = "DELTA_MUST_SET_ALL_COORDINATED_COMMITS_CONFS_IN_SESSION", messageParameters = Array(command, conf.defaultTablePropertyKey)) } } else { if (!properties.contains(conf.key)) { throw new DeltaIllegalArgumentException( errorClass = "DELTA_MUST_SET_ALL_COORDINATED_COMMITS_CONFS_IN_COMMAND", messageParameters = Array(command, conf.key)) } } } } /** * Verifies that the property keys do not contain any ICT dependencies for Coordinated Commits. */ private def verifyNotContainsICTConfigurations( propKeys: Seq[String], command: String, errorClass: String): Unit = { ICT_TABLE_PROPERTY_KEYS.foreach { key => if (propKeys.contains(key)) { throw new DeltaIllegalArgumentException( errorClass, messageParameters = Array(command)) } } } /** * Validates the Coordinated Commits configurations in explicit command overrides for * `AlterTableSetPropertiesDeltaCommand`. * * If the table already has Coordinated Commits configurations present, then we do not allow * users to override them via `ALTER TABLE t SET TBLPROPERTIES ...`. Users must downgrade the * table and then upgrade it with the new Coordinated Commits configurations. * If the table is a Coordinated Commits table or will be one via this ALTER command, then we * do not allow users to disable any ICT properties that Coordinated Commits depends on. */ def validateConfigurationsForAlterTableSetPropertiesDeltaCommand( existingConfs: Map[String, String], propertyOverrides: Map[String, String]): Unit = { val existingCoordinatedCommitsConfs = getExplicitCCConfigurations(existingConfs) val coordinatedCommitsOverrides = getExplicitCCConfigurations(propertyOverrides) if (coordinatedCommitsOverrides.nonEmpty) { if (existingCoordinatedCommitsConfs.nonEmpty) { throw new DeltaIllegalArgumentException( "DELTA_CANNOT_OVERRIDE_COORDINATED_COMMITS_CONFS", Array("ALTER")) } verifyNotContainsICTConfigurations(propertyOverrides.keys.toSeq, command = "ALTER", errorClass = "DELTA_CANNOT_SET_COORDINATED_COMMITS_DEPENDENCIES") verifyContainsOnlyCoordinatorNameAndConf( coordinatedCommitsOverrides, command = "ALTER", fromDefault = false) } if (existingCoordinatedCommitsConfs.nonEmpty) { verifyNotContainsICTConfigurations(propertyOverrides.keys.toSeq, command = "ALTER", errorClass = "DELTA_CANNOT_MODIFY_COORDINATED_COMMITS_DEPENDENCIES") } } /** * Validates the configurations to unset for `AlterTableUnsetPropertiesDeltaCommand`. * * If the table already has Coordinated Commits configurations present, then we do not allow users * to unset them via `ALTER TABLE t UNSET TBLPROPERTIES ...`. Users could only downgrade the table * via `ALTER TABLE t DROP FEATURE ...`. We also do not allow users to unset any ICT properties * that Coordinated Commits depends on. */ def validateConfigurationsForAlterTableUnsetPropertiesDeltaCommand( existingConfs: Map[String, String], propKeysToUnset: Seq[String]): Unit = { // If the table does not have any Coordinated Commits configurations, then we do not check the // properties to unset. This is because unsetting non-existent entries would either be caught // earlier (without `IF EXISTS`) or simply be a no-op (with `IF EXISTS`). Thus, we ignore them // instead of throwing an exception. if (getExplicitCCConfigurations(existingConfs).nonEmpty) { if (propKeysToUnset.exists(TABLE_PROPERTY_KEYS.contains)) { throw new DeltaIllegalArgumentException( "DELTA_CANNOT_UNSET_COORDINATED_COMMITS_CONFS", Array.empty) } verifyNotContainsICTConfigurations(propKeysToUnset, command = "ALTER", errorClass = "DELTA_CANNOT_MODIFY_COORDINATED_COMMITS_DEPENDENCIES") } } /** * Validates the Coordinated Commits configurations in explicit command overrides and default * SparkSession properties for `CreateDeltaTableCommand`. * See `validateConfigurationsForCreateDeltaTableCommandImpl` for details. */ def validateConfigurationsForCreateDeltaTableCommand( spark: SparkSession, tableExists: Boolean, query: Option[LogicalPlan], catalogTableProperties: Map[String, String]): Unit = { val (command, propertyOverrides) = query match { // For CLONE, we cannot use the properties from the catalog table, because they are already // the result of merging the source table properties with the overrides, but we do not // consider the source table properties for Coordinated Commits. case Some(cmd: CloneTableCommand) => (if (tableExists) "REPLACE with CLONE" else "CREATE with CLONE", cmd.tablePropertyOverrides) case _ => (if (tableExists) "REPLACE" else "CREATE", catalogTableProperties) } validateConfigurationsForCreateDeltaTableCommandImpl( spark, propertyOverrides, tableExists, command) } /** * Validates the Coordinated Commits configurations for the table. * - If the table already exists, the explicit command property overrides must not contain any * Coordinated Commits configurations. * - If the table does not exist, the explicit command property overrides must contain exactly * the Coordinator Name and Coordinator Conf, and no Table Conf. Default configurations are * checked similarly if none of the three properties is present in explicit overrides. */ private[delta] def validateConfigurationsForCreateDeltaTableCommandImpl( spark: SparkSession, propertyOverrides: Map[String, String], tableExists: Boolean, command: String): Unit = { val coordinatedCommitsConfs = getExplicitCCConfigurations(propertyOverrides) if (tableExists) { if (coordinatedCommitsConfs.nonEmpty) { throw new DeltaIllegalArgumentException( "DELTA_CANNOT_OVERRIDE_COORDINATED_COMMITS_CONFS", Array(command)) } } else { if (coordinatedCommitsConfs.nonEmpty) { verifyContainsOnlyCoordinatorNameAndConf( coordinatedCommitsConfs, command, fromDefault = false) } else { val defaultCoordinatedCommitsConfs = getDefaultCCConfigurations( spark, withDefaultKey = true) if (defaultCoordinatedCommitsConfs.nonEmpty) { verifyContainsOnlyCoordinatorNameAndConf( defaultCoordinatedCommitsConfs, command, fromDefault = true) } } } } /** * Converts a given Spark [[CatalystTableIdentifier]] to Coordinated Commits [[TableIdentifier]] */ def toCCTableIdentifier( catalystTableIdentifierOpt: Option[CatalystTableIdentifier]): Optional[TableIdentifier] = { catalystTableIdentifierOpt.map { catalystTableIdentifier => val namespace = catalystTableIdentifier.catalog.toSeq ++ catalystTableIdentifier.database.toSeq new TableIdentifier(namespace.toArray, catalystTableIdentifier.table) }.map(Optional.of[TableIdentifier]).getOrElse(Optional.empty[TableIdentifier]) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/InMemoryCommitCoordinator.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import java.util.{Map => JMap, Optional} import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.locks.ReentrantReadWriteLock import scala.collection.JavaConverters._ import scala.collection.mutable import org.apache.spark.sql.delta.logging.DeltaLogKeys import io.delta.storage.LogStore import io.delta.storage.commit.{ Commit => JCommit, CommitCoordinatorClient, CommitFailedException => JCommitFailedException, CommitResponse, GetCommitsResponse => JGetCommitsResponse, TableDescriptor, TableIdentifier } import io.delta.storage.commit.actions.{AbstractMetadata, AbstractProtocol} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.internal.MDC import org.apache.spark.sql.SparkSession class InMemoryCommitCoordinator(val batchSize: Long) extends AbstractBatchBackfillingCommitCoordinatorClient { /** * @param maxCommitVersion represents the max commit version known for the table. This is * initialized at the time of pre-registration and updated whenever a * commit is successfully added to the commit-coordinator. * @param active represents whether this commit-coordinator has ratified any commit or not. * |----------------------------|------------------|---------------------------| * | State | maxCommitVersion | active | * |----------------------------|------------------|---------------------------| * | Table is pre-registered | currentVersion+1 | false | * |----------------------------|------------------|---------------------------| * | Table is pre-registered | X | true | * | and more commits are done | | | * |----------------------------|------------------|---------------------------| */ private[coordinatedcommits] class PerTableData( var maxCommitVersion: Long = -1, var active: Boolean = false ) { def updateLastRatifiedCommit(commitVersion: Long): Unit = { active = true maxCommitVersion = commitVersion } /** * Returns the last ratified commit version for the table. If no commits have been done from * commit-coordinator yet, returns -1. */ def lastRatifiedCommitVersion: Long = if (!active) -1 else maxCommitVersion // Map from version to Commit data val commitsMap: mutable.SortedMap[Long, JCommit] = mutable.SortedMap.empty // We maintain maxCommitVersion explicitly since commitsMap might be empty // if all commits for a table have been backfilled. val lock: ReentrantReadWriteLock = new ReentrantReadWriteLock() } private[coordinatedcommits] val perTableMap = new ConcurrentHashMap[Path, PerTableData]() private[coordinatedcommits] def withWriteLock[T](logPath: Path)(operation: => T): T = { val tableData = perTableMap.computeIfAbsent(logPath, _ => new PerTableData()) val lock = tableData.lock.writeLock() lock.lock() try { operation } finally { lock.unlock() } } private[coordinatedcommits] def withReadLock[T](logPath: Path)(operation: => T): T = { val tableData = perTableMap.computeIfAbsent(logPath, _ => new PerTableData()) val lock = tableData.lock.readLock() lock.lock() try { operation } finally { lock.unlock() } } /** * This method acquires a write lock, validates the commit version is next in line, * updates commit maps, and releases the lock. * * @throws CommitFailedException if the commit version is not the expected next version, * indicating a version conflict. */ private[delta] def commitImpl( logStore: LogStore, hadoopConf: Configuration, logPath: Path, coordinatedCommitsTableConf: Map[String, String], commitVersion: Long, commitFile: FileStatus, commitTimestamp: Long): CommitResponse = { addToMap(logPath, commitVersion, commitFile, commitTimestamp) } private[sql] def addToMap( logPath: Path, commitVersion: Long, commitFile: FileStatus, commitTimestamp: Long): CommitResponse = { withWriteLock[CommitResponse](logPath) { val tableData = perTableMap.get(logPath) val expectedVersion = tableData.maxCommitVersion + 1 if (commitVersion != expectedVersion && tableData.maxCommitVersion != -1) { throw new JCommitFailedException( commitVersion < expectedVersion, commitVersion < expectedVersion, s"Commit version $commitVersion is not valid. Expected version: $expectedVersion.") } val commit = new JCommit(commitVersion, commitFile, commitTimestamp) tableData.commitsMap(commitVersion) = commit tableData.updateLastRatifiedCommit(commitVersion) logInfo(log"Added commit file ${MDC(DeltaLogKeys.PATH, commitFile.getPath)} " + log"to commit-coordinator.") new CommitResponse(commit) } } override def getCommits( tableDesc: TableDescriptor, startVersion: java.lang.Long, endVersion: java.lang.Long): JGetCommitsResponse = { withReadLock[JGetCommitsResponse](tableDesc.getLogPath) { val startVersionOpt: Option[Long] = Option(startVersion).map(_.toLong) val endVersionOpt: Option[Long] = Option(endVersion).map(_.toLong) val tableData = perTableMap.get(tableDesc.getLogPath) val effectiveStartVersion = startVersionOpt.getOrElse(0L) // Calculate the end version for the range, or use the last key if endVersion is not provided val effectiveEndVersion = endVersionOpt.getOrElse( tableData.commitsMap.lastOption.map(_._1).getOrElse(effectiveStartVersion)) val commitsInRange = tableData.commitsMap.range( effectiveStartVersion, effectiveEndVersion + 1) new JGetCommitsResponse( commitsInRange.values.toSeq.asJava, tableData.lastRatifiedCommitVersion) } } override protected[sql] def registerBackfill( logPath: Path, backfilledVersion: Long): Unit = { withWriteLock(logPath) { val tableData = perTableMap.get(logPath) if (backfilledVersion > tableData.lastRatifiedCommitVersion) { throw new IllegalArgumentException( s"Unexpected backfill version: $backfilledVersion. " + s"Max backfill version: ${tableData.maxCommitVersion}") } // Remove keys with versions less than or equal to 'untilVersion' val versionsToRemove = tableData.commitsMap.keys.takeWhile(_ <= backfilledVersion).toList versionsToRemove.foreach(tableData.commitsMap.remove) } } override def registerTable( logPath: Path, tableIdentifier: Optional[TableIdentifier], currentVersion: Long, currentMetadata: AbstractMetadata, currentProtocol: AbstractProtocol): JMap[String, String] = { val newPerTableData = new PerTableData(currentVersion + 1) perTableMap.compute(logPath, (_, existingData) => { if (existingData != null) { if (existingData.lastRatifiedCommitVersion != -1) { throw new IllegalStateException( s"Table $logPath already exists in the commit-coordinator.") } // If lastRatifiedCommitVersion is -1 i.e. the commit-coordinator has never attempted any // commit for this table => this table was just pre-registered. If there is another // pre-registration request for an older version, we reject it and table can't go backward. if (currentVersion < existingData.maxCommitVersion) { throw new IllegalStateException( s"Table $logPath already registered with commit-coordinator") } } newPerTableData }) Map.empty[String, String].asJava } def dropTable(logPath: Path): Unit = { withWriteLock(logPath) { perTableMap.remove(logPath) } } override def semanticEquals(other: CommitCoordinatorClient): Boolean = this == other private[delta] def removeCommitTestOnly( logPath: Path, commitVersion: Long ): Unit = { val tableData = perTableMap.get(logPath) tableData.commitsMap.remove(commitVersion) if (commitVersion == tableData.maxCommitVersion) { tableData.maxCommitVersion -= 1 } } } /** * The [[InMemoryCommitCoordinatorBuilder]] class is responsible for creating singleton instances of * [[InMemoryCommitCoordinator]] with the specified batchSize. */ case class InMemoryCommitCoordinatorBuilder(batchSize: Long) extends CatalogOwnedCommitCoordinatorBuilder { private lazy val inMemoryStore = new InMemoryCommitCoordinator(batchSize) /** Name of the commit-coordinator */ def getName: String = "in-memory" /** Returns a commit-coordinator based on the given conf */ def build(spark: SparkSession, conf: Map[String, String]): CommitCoordinatorClient = { inMemoryStore } /** Returns a commit-coordinator based on the given catalog name */ def buildForCatalog(spark: SparkSession, catalogName: String): CommitCoordinatorClient = { inMemoryStore } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/InMemoryUCClient.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import java.lang.{Long => JLong} import java.net.URI import java.util.Optional import org.apache.spark.sql.delta.actions.{Metadata, Protocol} import io.delta.storage.commit.{Commit => JCommit, GetCommitsResponse => JGetCommitsResponse} import io.delta.storage.commit.actions.{AbstractMetadata, AbstractProtocol} import io.delta.storage.commit.uccommitcoordinator.UCClient import io.delta.storage.commit.uniform.UniformMetadata /** * An in-memory implementation of [[UCClient]] for testing purposes. * This implementation simulates Unity Catalog operations without actually connecting to a remote * service. It maintains all state in memory in [[InMemoryUCCommitCoordinator]] * * This class provides a lightweight way to test Delta table operations that would * normally require interaction with the Unity Catalog. * * Example usage: * {{{ * val metastoreId = "test-metastore" * val ucCommitCoordinator = new InMemoryUCCommitCoordinator() * val client = new InMemoryUCClient(metastoreId, ucCommitCoordinator) * * // Use the client for testing * val getCommitsResponse = client.getCommits( * "tableId", * new URI("tableUri"), * Optional.empty(), * Optional.empty()) * }}} * * @param metastoreId The identifier for the simulated metastore * @param ucCommitCoordinator The in-memory coordinator to handle commit operations */ class InMemoryUCClient( metastoreId: String, ucCommitCoordinator: InMemoryUCCommitCoordinator) extends UCClient { override def getMetastoreId: String = metastoreId override def commit( tableId: String, tableUri: URI, commit: Optional[JCommit], lastKnownBackfilledVersion: Optional[JLong], disown: Boolean, newMetadata: Optional[AbstractMetadata], newProtocol: Optional[AbstractProtocol], uniform: Optional[UniformMetadata] = Optional.empty()): Unit = { ucCommitCoordinator.commitToCoordinator( tableId, tableUri, Option(commit.orElse(null)).map(_.getFileStatus.getPath.getName), Option(commit.orElse(null)).map(_.getVersion), Option(commit.orElse(null)).map(_.getFileStatus.getLen), Option(commit.orElse(null)).map(_.getFileStatus.getModificationTime), Option(commit.orElse(null)).map(_.getCommitTimestamp), Option(lastKnownBackfilledVersion.orElse(null)).map(_.toLong), disown, Option(newProtocol.orElse(null)).map(_.asInstanceOf[Protocol]), Option(newMetadata.orElse(null)).map(_.asInstanceOf[Metadata])) } override def getCommits( tableId: String, tableUri: URI, startVersion: Optional[JLong], endVersion: Optional[JLong]): JGetCommitsResponse = { ucCommitCoordinator.getCommitsFromCoordinator( tableId, tableUri, Option(startVersion.orElse(null)).map(_.toLong), Option(endVersion.orElse(null)).map(_.toLong)) } override def close(): Unit = {} } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/InMemoryUCCommitCoordinator.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import java.io.IOException import java.net.URI import java.util.UUID import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.locks.ReentrantReadWriteLock import scala.collection.JavaConverters._ import scala.collection.mutable import org.apache.spark.sql.delta.DeltaTableUtils import org.apache.spark.sql.delta.util.FileNames import io.delta.storage.commit.{ Commit => JCommit, CommitFailedException => JCommitFailedException, GetCommitsResponse => JGetCommitsResponse } import io.delta.storage.commit.actions.{AbstractMetadata, AbstractProtocol} import io.delta.storage.commit.uccommitcoordinator.{CommitLimitReachedException => JCommitLimitReachedException, InvalidTargetTableException => JInvalidTargetTableException} import org.apache.hadoop.fs.{FileStatus, Path} final object UCCoordinatedCommitsRequestType extends Enumeration { type UCCoordinatedCommitsRequestType = Value val COMMIT = Value val GET_COMMITS = Value } /** * A mock UC commit coordinator for testing purposes. */ class InMemoryUCCommitCoordinator { /** * Represents the data associated with a table. * `ucCommits` mimics the underlying list for the commit files. */ private class PerTableData(val path: URI) { /** * Represents a UC commit record. * @param commit represents the commit itself. * @param isDisownCommit represents whether the commit is a disown commit or not. * @param isBackfilled represents whether the commit is backfilled or not. */ private case class UCCommit( commit: JCommit, isDisownCommit: Boolean, isBackfilled: Boolean = false) { /** Version of the underlying commit file */ val version: Long = commit.getVersion } /** Underlying storage of UC commit records */ private val ucCommits: mutable.ArrayBuffer[UCCommit] = mutable.ArrayBuffer.empty /** RWLock to protect the commitsMap */ val lock: ReentrantReadWriteLock = new ReentrantReadWriteLock() /** * Returns the last ratified commit version for the table. * If no commits have been done from commit-coordinator yet, returns -1. */ def lastRatifiedCommitVersion: Long = ucCommits.lastOption.map(_.version).getOrElse(-1L) /** * Returns true if: * - the table has ratified any commit, and * - the last one is not a disown commit. */ def isActive: Boolean = ucCommits.lastOption.exists(!_.isDisownCommit) /** * Returns true if: * - the table has ratified any commit, and * - the last one is a disown commit. */ def isDisowned: Boolean = ucCommits.lastOption.exists(_.isDisownCommit) /** Appends a commit to the table's commit history */ def appendCommit(commit: JCommit, isDisownCommit: Boolean): Unit = { ucCommits += UCCommit(commit, isDisownCommit) } /** Removes all commits until the given version (inclusive) */ def removeCommitsUntilVersion(version: Long): Unit = { val toRemove = ucCommits.takeWhile(_.version <= version) ucCommits --= toRemove } /** Marks the last commit as backfilled */ def markLastCommitBackfilled(): Unit = { ucCommits.lastOption.foreach { lastUCCommit => ucCommits.update(ucCommits.size - 1, lastUCCommit.copy(isBackfilled = true)) } } /** * Returns the unbackfilled commits in the given range. * If `startVersion` is not provided, the first commit is used. * If `endVersion` is not provided, the last commit is used. */ def getCommits(startVersion: Option[Long], endVersion: Option[Long]): Seq[JCommit] = { val effectiveStartVersion = startVersion.getOrElse(0L) val effectiveEndVersion = endVersion.getOrElse( ucCommits.lastOption.map(_.version).getOrElse(return Seq.empty)) // Collect unbackfilled `Commit`s from the `UCCommit`s in the range. ucCommits.filter(c => effectiveStartVersion <= c.version && c.version <= effectiveEndVersion && !c.isBackfilled ).map(_.commit).toSeq } } /** * Variable to allow to control the behavior of the InMemoryUCCommitCoordinator * externally. If set to true, the coordinator will throw an IOException after * a successful commit. This will be reset to false once the exception has been * thrown. */ var throwIOExceptionAfterCommit: Boolean = false /** * Variable to allow to control the behavior of the InMemoryUCCommitCoordinator * externally. If set to true, the coordinator will throw an IOException before * persisting a commit to the in memory map. This will be reset to false once the * exception has been thrown. */ var throwIOExceptionBeforeCommit: Boolean = false /** The maximum number of unbackfilled commits this commit coordinator can store at a time */ private val MAX_NUM_COMMITS = 10 /** * Map from table UUID to the data associated with the table. * Mimics the underlying storage for the commit files of different tables. */ private val perTableMap = new ConcurrentHashMap[UUID, PerTableData]() /** Performs the given operation with lock acquired on the table entry */ private def withLock[T](tableUUID: UUID, writeLock: Boolean = false)(operation: => T): T = { val tableData = Option(perTableMap.get(tableUUID)).getOrElse { throw new IllegalArgumentException(s"Unknown table $tableUUID.") } val lock = if (writeLock) tableData.lock.writeLock() else tableData.lock.readLock() lock.lock() try { operation } finally { lock.unlock() } } private def validateTableURI( srcTable: URI, targetTable: URI, request: UCCoordinatedCommitsRequestType.UCCoordinatedCommitsRequestType): Unit = { if (srcTable != targetTable) { val errorMsg = s"Source table $srcTable and targetTable $targetTable do not match for " + s"$request" throw new JInvalidTargetTableException(errorMsg) } } // scalastyle:off argcount /** * Validates the commit and backfill parameters. * - Ensures that all the fields are provided. * - Makes sure that `lastKnownBackfilledVersion` is not more than the latest table version. * - Ensures that the commit version is the next expected version. * - Blocks committing to the table if the number of unbackfilled commits exceeds the limit. * This function does not mutate any state. */ private def getValidatedCommit( tableId: String, tableUri: URI, commitFileName: Option[String] = None, commitVersion: Option[Long] = None, commitFileSize: Option[Long] = None, commitFileModTime: Option[Long] = None, commitTimestamp: Option[Long] = None, lastKnownBackfilledVersion: Option[Long] = None, isDisownCommit: Boolean = false): Option[JCommit] = { val tableUUID = UUID.fromString(tableId) val path = tableUri val tableData = perTableMap.get(tableUUID) lastKnownBackfilledVersion.foreach { backfilledUntil => val maxBackfillVersion = commitVersion.getOrElse(0L).max(tableData.lastRatifiedCommitVersion) if (backfilledUntil > maxBackfillVersion) { throw new IllegalArgumentException( s"Unexpected backfill version: $backfilledUntil. " + s"Max backfill version: ${maxBackfillVersion}") } } commitFileName.map { fileName => // ensure that all other necessary parameters are provided require(commitVersion.nonEmpty) require(commitFileSize.nonEmpty) require(commitFileModTime.nonEmpty) require(commitTimestamp.nonEmpty) validateTableURI(path, tableUri, UCCoordinatedCommitsRequestType.COMMIT) // Check that there is still space in the commit coordinator. val tableIdStr = tableUUID.toString val currentNumCommits = getCommitsFromCoordinator( tableIdStr, tableUri, startVersion = None, endVersion = None).getCommits.size if (currentNumCommits == MAX_NUM_COMMITS) { val errorMsg = s"Too many unbackfilled commits for $tableIdStr. Cannot " + s"store more than $MAX_NUM_COMMITS commits" throw new JCommitLimitReachedException(errorMsg) } if (throwIOExceptionBeforeCommit) { throwIOExceptionBeforeCommit = false throw new IOException("Problem before comitting") } // Store the commit. For the InMemoryUCCommit coordinator, we concatenate the full commit path // here already so that we don't have to do it during getCommits. val basePath = FileNames.commitDirPath( DeltaTableUtils.safeConcatPaths(new Path(tableUri), "_delta_log")) val commitFilePath = new Path(basePath, fileName) val fileStatus = new FileStatus( commitFileSize.get, false, 0, 0, commitFileModTime.get, commitFilePath) // We only check the expected version matches the commit version if the table is active. // If the table is disowned, the check was already done above. // If the table was just registered, the check is not necessary. if (tableData.isActive) { val expectedVersion = tableData.lastRatifiedCommitVersion + 1 if (commitVersion.get != expectedVersion) { throw new JCommitFailedException( commitVersion.get < expectedVersion, commitVersion.get < expectedVersion, s"Commit version ${commitVersion.get} is not valid. " + s"Expected version: $expectedVersion.") } } new JCommit(commitVersion.get, fileStatus, commitTimestamp.get) } } private def backfillAfterCommitToCoordinatorInternal( tableId: String, lastKnownBackfilledVersion: Option[Long] = None): Unit = { val tableUUID = UUID.fromString(tableId) // Register any backfills. lastKnownBackfilledVersion.foreach { backfilledUntil => val tableData = perTableMap.get(tableUUID) val maxVersionToRemove = if (backfilledUntil == tableData.lastRatifiedCommitVersion) { // If the backfill version is the last ratified commit version, we remove all but the // last commit, and mark the last commit as backfilled. This is to ensure that every // active table keeps track of at least one commit record. tableData.markLastCommitBackfilled() backfilledUntil - 1 } else { // We have already validated that the backfill version is not more than the last // ratified version in getValidatedCommit. backfilledUntil } tableData.removeCommitsUntilVersion(maxVersionToRemove) } } def commitToCoordinator( tableId: String, tableUri: URI, commitFileName: Option[String] = None, commitVersion: Option[Long] = None, commitFileSize: Option[Long] = None, commitFileModTime: Option[Long] = None, commitTimestamp: Option[Long] = None, lastKnownBackfilledVersion: Option[Long] = None, isDisownCommit: Boolean = false, protocolOpt: Option[AbstractProtocol] = None, metadataOpt: Option[AbstractMetadata] = None): Unit = { // either commitFileName or backfilledUntil (or both) need to be set require(commitFileName.nonEmpty || lastKnownBackfilledVersion.nonEmpty) // Onboard the table if it is not already present in the perTableMap. if (commitVersion.nonEmpty) { val tableUuid = UUID.fromString(tableId) perTableMap.putIfAbsent(tableUuid, new PerTableData(tableUri)) } withLock( UUID.fromString(tableId), writeLock = true ) { val commitToAppendOpt = getValidatedCommit( tableId, tableUri, commitFileName, commitVersion, commitFileSize, commitFileModTime, commitTimestamp, lastKnownBackfilledVersion, isDisownCommit ) commitToAppendOpt.foreach { commitToAppend => val tableData = perTableMap.get(UUID.fromString(tableId)) tableData.appendCommit(commitToAppend, isDisownCommit) } if (throwIOExceptionAfterCommit) { throwIOExceptionAfterCommit = false throw new IOException("Problem after comitting") } backfillAfterCommitToCoordinatorInternal( tableId, lastKnownBackfilledVersion ) } } def getCommitsFromCoordinator( tableId: String, tableUri: URI, startVersion: Option[Long], endVersion: Option[Long]): JGetCommitsResponse = { val tableUUID = UUID.fromString(tableId) val path = Option(perTableMap.get(tableUUID)).map(_.path).getOrElse { return new JGetCommitsResponse(Seq.empty.asJava, -1) } validateTableURI(path, tableUri, UCCoordinatedCommitsRequestType.GET_COMMITS) withLock[JGetCommitsResponse](tableUUID) { val tableData = perTableMap.get(tableUUID) val commits = tableData.getCommits(startVersion, endVersion) new JGetCommitsResponse(commits.asJava, tableData.lastRatifiedCommitVersion) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/TableCommitCoordinatorClient.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.storage.{LogStore, LogStoreInverseAdaptor} import io.delta.storage.commit.{ CommitCoordinatorClient => JCommitCoordinatorClient, CommitResponse, GetCommitsResponse => JGetCommitsResponse, TableDescriptor, UpdatedActions } import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.sql.catalyst.{TableIdentifier => CatalystTableIdentifier} /** * A wrapper around [[CommitCoordinatorClient]] that provides a more user-friendly API for * committing/ accessing commits to a specific table. This class takes care of passing the * table specific configuration to the underlying [[CommitCoordinatorClient]] e.g. logPath / * logStore / coordinatedCommitsTableConf / hadoopConf. * * @param commitCoordinatorClient the underlying [[CommitCoordinatorClient]] * @param logPath the path to the log directory * @param tableConf the table specific coordinated-commits configuration * @param hadoopConf hadoop configuration * @param logStore the log store */ case class TableCommitCoordinatorClient( commitCoordinatorClient: JCommitCoordinatorClient, logPath: Path, tableConf: Map[String, String], hadoopConf: Configuration, logStore: LogStore) { private def makeTableDesc( tableIdentifierOpt: Option[CatalystTableIdentifier]): TableDescriptor = { val ccTableIdentifier = CoordinatedCommitsUtils.toCCTableIdentifier(tableIdentifierOpt) new TableDescriptor(logPath, ccTableIdentifier, tableConf.asJava) } def commit( commitVersion: Long, actions: Iterator[String], updatedActions: UpdatedActions, tableIdentifierOpt: Option[CatalystTableIdentifier]): CommitResponse = { commitCoordinatorClient.commit( LogStoreInverseAdaptor(logStore, hadoopConf), hadoopConf, makeTableDesc(tableIdentifierOpt), commitVersion, actions.asJava, updatedActions) } def getCommits( tableIdentifierOpt: Option[CatalystTableIdentifier], startVersion: Option[Long] = None, endVersion: Option[Long] = None): JGetCommitsResponse = { commitCoordinatorClient.getCommits( makeTableDesc(tableIdentifierOpt), startVersion.map(Long.box).orNull, endVersion.map(Long.box).orNull) } def backfillToVersion( tableIdentifierOpt: Option[CatalystTableIdentifier], version: Long, lastKnownBackfilledVersion: Option[Long] = None): Unit = { commitCoordinatorClient.backfillToVersion( LogStoreInverseAdaptor(logStore, hadoopConf), hadoopConf, makeTableDesc(tableIdentifierOpt), version, lastKnownBackfilledVersion.map(Long.box).orNull) } /** * Checks whether the signature of the underlying backing [[CommitCoordinatorClient]] is the same * as the given `otherCommitCoordinatorClient` */ def semanticsEquals(otherCommitCoordinatorClient: JCommitCoordinatorClient): Boolean = { CommitCoordinatorClient.semanticEquals( Some(commitCoordinatorClient), Some(otherCommitCoordinatorClient)) } /** * Checks whether the signature of the underlying backing [[CommitCoordinatorClient]] is the same * as the given `otherCommitCoordinatorClient` */ def semanticsEquals(otherCommitCoordinatorClient: TableCommitCoordinatorClient): Boolean = { semanticsEquals(otherCommitCoordinatorClient.commitCoordinatorClient) } } object TableCommitCoordinatorClient { def apply( commitCoordinatorClient: JCommitCoordinatorClient, deltaLog: DeltaLog, coordinatedCommitsTableConf: Map[String, String]): TableCommitCoordinatorClient = { val hadoopConf = deltaLog.newDeltaHadoopConf() new TableCommitCoordinatorClient( commitCoordinatorClient, deltaLog.logPath, coordinatedCommitsTableConf, hadoopConf, deltaLog.store) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/UCCommitCoordinatorBuilder.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import java.net.{URI, URISyntaxException} import java.util.concurrent.ConcurrentHashMap import scala.collection.JavaConverters._ import scala.util.control.NonFatal import io.delta.storage.commit.CommitCoordinatorClient import io.delta.storage.commit.uccommitcoordinator.{UCClient, UCCommitCoordinatorClient, UCTokenBasedRestClient} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import io.unitycatalog.client.auth.TokenProvider import org.apache.spark.internal.MDC import org.apache.spark.sql.SparkSession /** * Builder for Unity Catalog Commit Coordinator Clients. * * This builder is responsible for creating and caching UCCommitCoordinatorClient instances * based on the provided metastore IDs and catalog configurations. * * It caches the UCCommitCoordinatorClient instance for a given metastore ID upon its first access. */ object UCCommitCoordinatorBuilder extends CatalogOwnedCommitCoordinatorBuilder with DeltaLogging { /** The coordinator name used in table metadata to identify UC-managed tables. */ final val COORDINATOR_NAME: String = "unity-catalog" /** Prefix for Spark SQL catalog configurations. */ final private val SPARK_SQL_CATALOG_PREFIX = "spark.sql.catalog." /** Connector class name for filtering relevant Unity Catalog catalogs. */ final private[delta] val UNITY_CATALOG_CONNECTOR_CLASS: String = "io.unitycatalog.spark.UCSingleCatalog" /** Suffix for the URI configuration of a catalog. */ final private val URI_SUFFIX = "uri" /** Cache for UCCommitCoordinatorClient instances. */ private val commitCoordinatorClientCache = new ConcurrentHashMap[String, UCCommitCoordinatorClient]() // Helper cache for (uri, authConfig) to metastoreId to avoid redundant calls to getMetastoreId private val uriAuthConfigToMetastoreIdCache = new ConcurrentHashMap[(String, Map[String, String]), String]() // Use a var instead of val for ease of testing by injecting different UCClientFactory. private[delta] var ucClientFactory: UCClientFactory = UCTokenBasedRestClientFactory override def getName: String = COORDINATOR_NAME override def build(spark: SparkSession, conf: Map[String, String]): CommitCoordinatorClient = { val metastoreId = conf.getOrElse( UCCommitCoordinatorClient.UC_METASTORE_ID_KEY, throw new IllegalArgumentException( s"UC metastore ID not found in the provided coordinator conf: $conf")) commitCoordinatorClientCache.computeIfAbsent( metastoreId, _ => new UCCommitCoordinatorClient(conf.asJava, getMatchingUCClient(spark, metastoreId))) } override def buildForCatalog( spark: SparkSession, catalogName: String): CommitCoordinatorClient = { val client = getCatalogConfigs(spark).find(_._1 == catalogName) match { case Some((_, uri, authConfig)) => ucClientFactory.createUCClient(uri, authConfig) case None => throw new IllegalArgumentException( s"Catalog $catalogName not found in the provided SparkSession configurations.") } val conf = Map.empty[String, String] new UCCommitCoordinatorClient(conf.asJava, client) } /** * Finds and returns a UCClient that matches the given metastore ID. * * This method iterates through all configured catalogs in SparkSession, creates UCClients for * each, gets their metastore ID and returns the one that matches the provided metastore ID. * If no matching catalog is found or if multiple matching catalogs are found, it throws an * appropriate exception. */ private def getMatchingUCClient(spark: SparkSession, metastoreId: String): UCClient = { val matchingClients: List[(String, Map[String, String])] = getCatalogConfigs(spark) .map { case (name, uri, authConfig) => (uri, authConfig) } .distinct // Remove duplicates since multiple catalogs can have the same uri and config .filter { case (uri, authConfig) => getMetastoreId(uri, authConfig).contains(metastoreId) } matchingClients match { case Nil => throw noMatchingCatalogException(metastoreId) case (uri, authConfig) :: Nil => ucClientFactory.createUCClient(uri, authConfig) case multiple => throw multipleMatchingCatalogs(metastoreId, multiple.map(_._1)) } } /** * Retrieves the metastore ID for a given URI and auth configuration map. * * This method creates a UCClient using the provided URI and auth configuration map, then * retrieves its metastore ID. The result is cached to avoid unnecessary getMetastoreId requests * in future calls. If there's an error, it returns None and logs a warning. */ private def getMetastoreId(uri: String, authConfig: Map[String, String]): Option[String] = { try { val metastoreId = uriAuthConfigToMetastoreIdCache.computeIfAbsent( (uri, authConfig), _ => { val ucClient = ucClientFactory.createUCClient(uri, authConfig) try { ucClient.getMetastoreId } finally { safeClose(ucClient, uri) } }) Some(metastoreId) } catch { case NonFatal(e) => logWarning(log"Failed to getMetastoreSummary with ${MDC(DeltaLogKeys.URI, uri)}", e) None } } private def noMatchingCatalogException(metastoreId: String) = { new IllegalStateException( s"No matching catalog found for UC metastore ID $metastoreId. " + "Please ensure the catalog is configured correctly by setting " + "`spark.sql.catalog.`, `spark.sql.catalog..uri` and " + "any required Unity Catalog authentication configurations. " + "Note that the matching process involves retrieving the metastoreId using the " + "provided configuration in Spark Session configs.") } private def multipleMatchingCatalogs(metastoreId: String, uris: List[String]) = { new IllegalStateException( s"Found multiple catalogs for UC metastore ID $metastoreId at $uris. " + "Please ensure the catalog is configured correctly by setting " + "`spark.sql.catalog.`, `spark.sql.catalog..uri` and " + "any required Unity Catalog authentication configurations. " + "Note that the matching process involves retrieving the metastoreId using the " + "provided configuration in Spark Session configs.") } /** * Retrieves the catalog configurations from the SparkSession. * * This method supports both the new auth.* format and the legacy token format for backward * compatibility: * * New format: * spark.sql.catalog.catalog1.uri = "https://dbc-123abc.databricks.com" * spark.sql.catalog.catalog1.auth.type = "static" * spark.sql.catalog.catalog1.auth.token = "dapi1234567890" * * Legacy format (for backward compatibility): * spark.sql.catalog.catalog1.uri = "https://dbc-123abc.databricks.com" * spark.sql.catalog.catalog1.token = "dapi1234567890" * * When the legacy format is detected (token without auth. prefix), it is automatically * converted to the new format (type=static, token=value) for TokenProvider. * * @return * A list of tuples containing (catalogName, uri, authConfigMap) for each properly configured * catalog. The authConfigMap contains authentication configurations ready to be passed to * TokenProvider.create(). */ private[delta] def getCatalogConfigs( spark: SparkSession): List[(String, String, Map[String, String])] = { val catalogConfigs = spark.conf.getAll.filterKeys(_.startsWith(SPARK_SQL_CATALOG_PREFIX)) // First, identify all Unity Catalog catalogs val ucCatalogNames = catalogConfigs .keys .map(_.split("\\.")) .filter(_.length == 4) .map(_(3)) .filter { catalogName: String => val connector = catalogConfigs.get(s"$SPARK_SQL_CATALOG_PREFIX$catalogName") connector.contains(UNITY_CATALOG_CONNECTOR_CLASS) } // For each UC catalog, extract its URI and auth configurations ucCatalogNames .flatMap { catalogName: String => val catalogPrefix = s"$SPARK_SQL_CATALOG_PREFIX$catalogName." val authPrefix = s"${catalogPrefix}auth." val uriOpt = catalogConfigs.get(s"$catalogPrefix$URI_SUFFIX") uriOpt match { case Some(uri) => try { new URI(uri) // Validate the URI // Extract all auth.* configuration keys for this catalog // and strip the "spark.sql.catalog..auth." prefix var authConfigMap = catalogConfigs .filterKeys(_.startsWith(authPrefix)) .map { case (fullKey, value) => // Remove the auth prefix to get just the auth config key // e.g., "spark.sql.catalog.catalog1.auth.type" -> "type" // e.g., "spark.sql.catalog.catalog1.auth.oauth.uri" -> "oauth.uri" val authKey = fullKey.stripPrefix(authPrefix) (authKey, value) } .toMap // Support legacy format: if no auth.* configs but token exists, // convert to new format (type=static, token=value) if (authConfigMap.isEmpty) { val legacyTokenOpt = catalogConfigs.get(s"${catalogPrefix}token") legacyTokenOpt match { case Some(token) => authConfigMap = Map("type" -> "static", "token" -> token) case None => // No auth configs found } } if (authConfigMap.isEmpty) { logWarning( log"Skipping catalog ${MDC(DeltaLogKeys.CATALOG, catalogName)} as it " + "does not have any authentication configurations in Spark Session.") None } else { Some((catalogName, uri, authConfigMap)) } } catch { case _: URISyntaxException => logWarning( log"Skipping catalog ${MDC(DeltaLogKeys.CATALOG, catalogName)} as it " + log"does not have a valid URI ${MDC(DeltaLogKeys.URI, uri)}.") None } case None => logWarning( log"Skipping catalog ${MDC(DeltaLogKeys.CATALOG, catalogName)} as it does " + "not have uri configured in Spark Session.") None } } .toList } /** * Returns catalog configurations as a Map for O(1) lookup by catalog name. * Wraps [[getCatalogConfigs]] results in [[UCCatalogConfig]] for better readability. */ private[delta] def getCatalogConfigMap(spark: SparkSession): Map[String, UCCatalogConfig] = { getCatalogConfigs(spark).map { case (name, uri, authConfig) => name -> UCCatalogConfig(name, uri, authConfig) }.toMap } private def safeClose(ucClient: UCClient, uri: String): Unit = { try { ucClient.close() } catch { case NonFatal(e) => logWarning(log"Failed to close UCClient for uri ${MDC(DeltaLogKeys.URI, uri)}", e) } } def clearCache(): Unit = { commitCoordinatorClientCache.clear() uriAuthConfigToMetastoreIdCache.clear() } } trait UCClientFactory { def createUCClient(uri: String, authConfig: Map[String, String]): UCClient } object UCTokenBasedRestClientFactory extends UCClientFactory { override def createUCClient(uri: String, authConfig: Map[String, String]): UCClient = { createUCClientWithVersions(uri, authConfig, defaultAppVersions) } /** * Creates a UC client with the given application versions for telemetry. * The provided `appVersions` map is used as-is; callers are responsible for * including all desired version entries. */ def createUCClientWithVersions( uri: String, authConfig: Map[String, String], appVersions: Map[String, String]): UCClient = { // Create TokenProvider from the authentication configuration map // We pass the configuration through without interpreting any specific keys, // as those are managed by the Unity Catalog client library val tokenProvider = TokenProvider.create(authConfig.asJava) new UCTokenBasedRestClient(uri, tokenProvider, appVersions.asJava) } private[coordinatedcommits] def defaultAppVersions: Map[String, String] = { Map( "Delta" -> io.delta.VERSION, "Spark" -> org.apache.spark.SPARK_VERSION, "Scala" -> scala.util.Properties.versionNumberString, "Java" -> System.getProperty("java.version") ) } /** Returns the default app versions as a mutable Java map for easy extension. */ def defaultAppVersionsAsJava: java.util.Map[String, String] = { new java.util.HashMap(defaultAppVersions.asJava) } /** Java-friendly overload that accepts a java.util.Map */ def createUCClient(uri: String, authConfig: java.util.Map[String, String]): UCClient = { createUCClient(uri, authConfig.asScala.toMap) } /** Java-friendly overload that accepts application versions for telemetry. */ def createUCClientWithVersions( uri: String, authConfig: java.util.Map[String, String], appVersions: java.util.Map[String, String]): UCClient = { createUCClientWithVersions(uri, authConfig.asScala.toMap, appVersions.asScala.toMap) } } /** * Holder for Unity Catalog configuration extracted from Spark configs. * Used by [[UCCommitCoordinatorBuilder.getCatalogConfigMap]]. */ case class UCCatalogConfig(catalogName: String, uri: String, authConfig: Map[String, String]) ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/deletionvectors/RoaringBitmapArray.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.deletionvectors import java.io.IOException import java.nio.{ByteBuffer, ByteOrder} import scala.collection.immutable.NumericRange import com.google.common.primitives.{Ints, UnsignedInts} import org.roaringbitmap.{RelativeRangeConsumer, RoaringBitmap} /** * A 64-bit extension of [[RoaringBitmap]] that is optimized for cases that usually fit within * a 32-bit bitmap, but may run over by a few bits on occasion. * * This focus makes it different from [[org.roaringbitmap.longlong.Roaring64NavigableMap]] and * [[org.roaringbitmap.longlong.Roaring64Bitmap]] which focus on sparse bitmaps over the whole * 64-bit range. * * Structurally, this implementation simply uses the most-significant 4 bytes to index into * an array of 32-bit [[RoaringBitmap]] instances. * The array is grown as necessary to accommodate the largest value in the bitmap. * * *Note:* As opposed to the other two 64-bit bitmap implementations mentioned above, * this implementation cannot accommodate `Long` values where the most significant * bit is non-zero (i.e., negative `Long` values). * It cannot even accommodate values where the 4 high-order bytes are `Int.MaxValue`, * because then the length of the `bitmaps` array would be a negative number * (`Int.MaxValue + 1`). */ final class RoaringBitmapArray extends Equals { import RoaringBitmapArray._ private var bitmaps: Array[RoaringBitmap] = Array.empty /** * Add the value to the container (set the value to `true`), * whether it already appears or not. */ def add(value: Long): Unit = { require(value >= 0 && value <= MAX_REPRESENTABLE_VALUE) val (high, low) = decomposeHighLowBytes(value) if (high >= bitmaps.length) { extendBitmaps(newLength = high + 1) } val highBitmap = bitmaps(high) highBitmap.add(low) } /** Add all `values` to the container. For testing purposes only. */ protected[delta] def addAll(values: Long*): Unit = values.foreach(add) /** Add all values in `range` to the container. */ protected[delta] def addRange(range: Range): Unit = { require(0 <= range.start && range.start <= range.end) if (range.isEmpty) return // Nothing to do. if (range.step != 1) { // Can't optimize in this case. range.foreach(i => add(UnsignedInts.toLong(i))) return } // This is an Int range, so it must fit completely into the first bitmap. if (bitmaps.isEmpty) { extendBitmaps(newLength = 1) } val end = if (range.isInclusive) range.end + 1 else range.end bitmaps.head.add(range.start, end) } /** Add all values in `range` to the container. */ protected[delta] def addRange(range: NumericRange[Long]): Unit = { require(0L <= range.start && range.start <= range.end && range.end <= MAX_REPRESENTABLE_VALUE) if (range.isEmpty) return // Nothing to do. if (range.step != 1L) { // Can't optimize in this case. range.foreach(add) return } // Decompose into sub-ranges that target a single bitmap, // to use the range adds within a bitmap for efficiency. val (startHigh, startLow) = decomposeHighLowBytes(range.start) val (endHigh, endLow) = decomposeHighLowBytes(range.end) val lastHigh = if (endLow == 0 && !range.isInclusive) endHigh - 1 else endHigh if (lastHigh >= bitmaps.length) { extendBitmaps(newLength = lastHigh + 1) } var currentHigh = startHigh while (currentHigh <= lastHigh) { val start = if (currentHigh == startHigh) UnsignedInts.toLong(startLow) else 0L // RoaringBitmap.add is exclusive the end boundary. val end = if (currentHigh == endHigh) { if (range.isInclusive) UnsignedInts.toLong(endLow) + 1L else UnsignedInts.toLong(endLow) } else { 0xFFFFFFFFL + 1L } bitmaps(currentHigh).add(start, end) currentHigh += 1 } } /** * If present, remove the `value` (effectively, sets its bit value to false). * * @param value The index in a bitmap. */ protected[deletionvectors] def remove(value: Long): Unit = { require(value >= 0 && value <= MAX_REPRESENTABLE_VALUE) val (high, low) = decomposeHighLowBytes(value) if (high < bitmaps.length) { val highBitmap = bitmaps(high) highBitmap.remove(low) if (highBitmap.isEmpty) { // Clean up all bitmaps that are now empty (from the end). var latestNonEmpty = bitmaps.length - 1 var done = false while (!done && latestNonEmpty >= 0) { if (bitmaps(latestNonEmpty).isEmpty) { latestNonEmpty -= 1 } else { done = true } } shrinkBitmaps(latestNonEmpty + 1) } } } /** Remove all values from the bitmap. */ def clear(): Unit = { bitmaps = Array.empty } /** * Checks whether the value is included, * which is equivalent to checking if the corresponding bit is set. */ def contains(value: Long): Boolean = { require(value >= 0 && value <= MAX_REPRESENTABLE_VALUE) val high = highBytes(value) if (high >= bitmaps.length) { false } else { val highBitmap = bitmaps(high) val low = lowBytes(value) highBitmap.contains(low) } } /** * Return the set values as an array, if the cardinality is smaller than 2147483648. * * The integer values are in sorted order. */ def toArray: Array[Long] = { val cardinality = this.cardinality require(cardinality <= Int.MaxValue) val values = Array.ofDim[Long](cardinality.toInt) var valuesIndex = 0 for ((bitmap, bitmapIndex) <- bitmaps.zipWithIndex) { bitmap.forEach((value: Int) => { values(valuesIndex) = composeFromHighLowBytes(bitmapIndex, value) valuesIndex += 1 }) } values } /** Materialise the whole set into an array */ def values: Array[Long] = toArray /** Returns the number of distinct integers added to the bitmap (e.g., number of bits set). */ def cardinality: Long = bitmaps.foldLeft(0L)((sum, bitmap) => sum + bitmap.getLongCardinality) /** Tests whether the bitmap is empty. */ def isEmpty: Boolean = bitmaps.forall(_.isEmpty) /** * Use a run-length encoding where it is more space efficient. * * @return `true` if a change was applied */ def runOptimize(): Boolean = { var changeApplied = false for (bitmap <- bitmaps) { changeApplied |= bitmap.runOptimize() } changeApplied } /** * Remove run-length encoding even when it is more space efficient. * * @return `true` if a change was applied */ def removeRunCompression(): Boolean = { var changeApplied = false for (bitmap <- bitmaps) { changeApplied |= bitmap.removeRunCompression() } changeApplied } /** * In-place bitwise OR (union) operation. * * The current bitmap is modified. */ def or(that: RoaringBitmapArray): Unit = { if (this.bitmaps.length < that.bitmaps.length) { extendBitmaps(newLength = that.bitmaps.length) } for (index <- that.bitmaps.indices) { val thisBitmap = this.bitmaps(index) val thatBitmap = that.bitmaps(index) thisBitmap.or(thatBitmap) } } /** Merges the `other` set into this one. */ def merge(other: RoaringBitmapArray): Unit = this.or(other) /** Get values in `this` but not `that`. */ def diff(other: RoaringBitmapArray): Unit = this.andNot(other) /** Copy `this` along with underlying bitmaps to a new instance. */ def copy(): RoaringBitmapArray = { val newBitmap = new RoaringBitmapArray() newBitmap.merge(this) newBitmap } /** * In-place bitwise AND (this & that) operation. * * The current bitmap is modified. */ def and(that: RoaringBitmapArray): Unit = { for (index <- 0 until this.bitmaps.length) { val thisBitmap = this.bitmaps(index) if (index < that.bitmaps.length) { val thatBitmap = that.bitmaps(index) thisBitmap.and(thatBitmap) } else { thisBitmap.clear() } } } /** * In-place bitwise AND-NOT (this & ~that) operation. * * The current bitmap is modified. */ def andNot(that: RoaringBitmapArray): Unit = { val validLength = math.min(this.bitmaps.length, that.bitmaps.length) for (index <- 0 until validLength) { val thisBitmap = this.bitmaps(index) val thatBitmap = that.bitmaps(index) thisBitmap.andNot(thatBitmap) } } /** * Report the number of bytes required to serialize this bitmap. * * This is the number of bytes written out when using the [[serialize]] method. */ def serializedSizeInBytes(format: RoaringBitmapArrayFormat.Value): Long = { val magicNumberSize = 4 val serializedBitmapsSize = format.formatImpl.serializedSizeInBytes(bitmaps) magicNumberSize + serializedBitmapsSize } /** * Serialize this [[RoaringBitmapArray]] into the `buffer`. * * == Format == * - A Magic Number indicating the format used (4 bytes) * - The actual data as specified by the format. * */ def serialize(buffer: ByteBuffer, format: RoaringBitmapArrayFormat.Value): Unit = { require(ByteOrder.LITTLE_ENDIAN == buffer.order(), "RoaringBitmapArray has to be serialized using a little endian buffer") // Magic number to make sure we don't try to deserialize a simple RoaringBitmap or the wrong // format later. buffer.putInt(format.formatImpl.MAGIC_NUMBER) format.formatImpl.serialize(bitmaps, buffer) } /** Serializes this [[RoaringBitmapArray]] and returns the serialized form as a byte array. */ def serializeAsByteArray(format: RoaringBitmapArrayFormat.Value): Array[Byte] = { val size = serializedSizeInBytes(format) if (!size.isValidInt) { throw new IOException( s"A bitmap was too big to be serialized into an array ($size bytes)") } val buffer = ByteBuffer.allocate(size.toInt) buffer.order(ByteOrder.LITTLE_ENDIAN) // This is faster than Java serialization. // See: https://richardstartin.github.io/posts/roaringbitmap-performance-tricks#serialisation serialize(buffer, format) buffer.array() } /** * Deserialize the contents of `buffer` into this [[RoaringBitmapArray]]. * * All existing content will be discarded! * * See [[serialize]] for the expected serialization format. */ def deserialize(buffer: ByteBuffer): Unit = { require(ByteOrder.LITTLE_ENDIAN == buffer.order(), "RoaringBitmapArray has to be deserialized using a little endian buffer") val magicNumber = buffer.getInt val serializationFormat = magicNumber match { case NativeRoaringBitmapArraySerializationFormat.MAGIC_NUMBER => NativeRoaringBitmapArraySerializationFormat case PortableRoaringBitmapArraySerializationFormat.MAGIC_NUMBER => PortableRoaringBitmapArraySerializationFormat case _ => throw new IOException(s"Unexpected RoaringBitmapArray magic number $magicNumber") } bitmaps = serializationFormat.deserialize(buffer) } /** * Consume presence information for all values in the range `[start, start + length)`. * * @param start Lower bound of values to consume. * @param length Maximum number of values to consume. * @param rrc Code to be executed for each present or absent value. */ def forAllInRange(start: Long, length: Int, consumer: RelativeRangeConsumer): Unit = { // This one is complicated and deserves its own PR, // when we actually want to enable it. throw new UnsupportedOperationException } /** Execute the `consume` function for every value in the set represented by this bitmap. */ def forEach(consume: Long => Unit): Unit = { for ((bitmap, high) <- bitmaps.zipWithIndex) { bitmap.forEach { low: Int => val value = composeFromHighLowBytes(high, low) consume(value) } } } override def canEqual(that: Any): Boolean = that.isInstanceOf[RoaringBitmapArray] override def equals(other: Any): Boolean = { other match { case that: RoaringBitmapArray => (this eq that) || // don't need to check canEqual because class is final java.util.Arrays.deepEquals( this.bitmaps.asInstanceOf[Array[AnyRef]], that.bitmaps.asInstanceOf[Array[AnyRef]]) case _ => false } } override def hashCode: Int = 131 * java.util.Arrays.deepHashCode( bitmaps.asInstanceOf[Array[AnyRef]]) def mkString(start: String = "", sep: String = "", end: String = ""): String = toArray.mkString(start, sep, end) def first: Option[Long] = { for ((bitmap, high) <- bitmaps.zipWithIndex) { if (!bitmap.isEmpty) { val low = bitmap.first() return Some(composeFromHighLowBytes(high, low)) } } None } def last: Option[Long] = { for ((bitmap, high) <- bitmaps.zipWithIndex.reverse) { if (!bitmap.isEmpty) { val low = bitmap.last() return Some(composeFromHighLowBytes(high, low)) } } None } /** * Utility method to extend the array of [[RoaringBitmap]] to given length, keeping * the existing elements in place. */ private def extendBitmaps(newLength: Int): Unit = { // Optimization for the most common case if (bitmaps.isEmpty && newLength == 1) { bitmaps = Array(new RoaringBitmap()) return } val newBitmaps = Array.ofDim[RoaringBitmap](newLength) System.arraycopy( bitmaps, // source 0, // source start pos newBitmaps, // dest 0, // dest start pos bitmaps.length) // number of entries to copy for (i <- bitmaps.length until newLength) { newBitmaps(i) = new RoaringBitmap() } bitmaps = newBitmaps } /** Utility method to shrink the array of [[RoaringBitmap]] to given length. */ private def shrinkBitmaps(newLength: Int): Unit = { if (newLength == 0) { bitmaps = Array.empty } else { val newBitmaps = Array.ofDim[RoaringBitmap](newLength) System.arraycopy( bitmaps, // source 0, // source start pos newBitmaps, // dest 0, // dest start pos newLength) // number of entries to copy bitmaps = newBitmaps } } // For testing purposes protected[delta] def toBitmap32Bit(): RoaringBitmap = { val bitmap32 = new RoaringBitmap() forEach { value => val value32 = Ints.checkedCast(value) bitmap32.add(value32) } bitmap32.runOptimize() bitmap32 } } object RoaringBitmapArray { /** The largest value a [[RoaringBitmapArray]] can possibly represent. */ final val MAX_REPRESENTABLE_VALUE: Long = composeFromHighLowBytes(Int.MaxValue - 1, Int.MinValue) final val MAX_BITMAP_CARDINALITY: Long = 1L << 32 /** Create a new [[RoaringBitmapArray]] with the given `values`. */ def apply(values: Long*): RoaringBitmapArray = { val bitmap = new RoaringBitmapArray bitmap.addAll(values: _*) bitmap } /** * * @param value Any `Long`; positive or negative. * @return An `Int` holding the 4 high-order bytes of information of the input `value`. */ def highBytes(value: Long): Int = (value >> 32).toInt /** * * @param value Any `Long`; positive or negative. * @return An `Int` holding the 4 low-order bytes of information of the input `value`. */ def lowBytes(value: Long): Int = value.toInt /** Separate high and low 4 bytes into a pair of `Int`s (high, low). */ def decomposeHighLowBytes(value: Long): (Int, Int) = (highBytes(value), lowBytes(value)) /** * Combine high and low 4 bytes of a pair of `Int`s into a `Long`. * * This is essentially the inverse of [[decomposeHighLowBytes()]]. * * @param high An `Int` representing the 4 high-order bytes of the output `Long` * @param low An `Int` representing the 4 low-order bytes of the output `Long` * @return A `Long` composing the `high` and `low` bytes. */ def composeFromHighLowBytes(high: Int, low: Int): Long = (high.toLong << 32) | (low.toLong & 0xFFFFFFFFL) // Must bitmask to avoid sign extension. /** Deserialize the right instance from the given bytes */ def readFrom(bytes: Array[Byte]): RoaringBitmapArray = { val buffer = ByteBuffer.wrap(bytes) buffer.order(ByteOrder.LITTLE_ENDIAN) val bitmap = new RoaringBitmapArray() bitmap.deserialize(buffer) bitmap } } /** * Abstracts out how to (de-)serialize the array. * * All formats are indicated by a magic number in the first 4-bytes, * which must be add/stripped by the *caller*. */ private[deletionvectors] sealed trait RoaringBitmapArraySerializationFormat { /** Magic number prefix for serialization with this format. */ val MAGIC_NUMBER: Int /** The number of bytes written out when using the [[serialize]] method. */ def serializedSizeInBytes(bitmaps: Array[RoaringBitmap]): Long /** Serialize `bitmaps` into `buffer`. */ def serialize(bitmaps: Array[RoaringBitmap], buffer: ByteBuffer): Unit /** Deserialize all bitmaps from the `buffer` into a fresh array. */ def deserialize(buffer: ByteBuffer): Array[RoaringBitmap] } /** Legal values for the serialization format for [[RoaringBitmapArray]]. */ object RoaringBitmapArrayFormat extends Enumeration { protected case class Format(formatImpl: RoaringBitmapArraySerializationFormat) extends super.Val import scala.language.implicitConversions implicit def valueToFormat(x: Value): Format = x.asInstanceOf[Format] val Native = Format(NativeRoaringBitmapArraySerializationFormat) val Portable = Format(PortableRoaringBitmapArraySerializationFormat) } private[deletionvectors] object NativeRoaringBitmapArraySerializationFormat extends RoaringBitmapArraySerializationFormat { override val MAGIC_NUMBER: Int = 1681511376 override def serializedSizeInBytes(bitmaps: Array[RoaringBitmap]): Long = { val roaringBitmapsCountSize = 4 val roaringBitmapLengthSize = 4 val roaringBitmapsSize = bitmaps.foldLeft(0L) { (sum, bitmap) => sum + bitmap.serializedSizeInBytes() + roaringBitmapLengthSize } roaringBitmapsCountSize + roaringBitmapsSize } /** * Serialize `bitmaps` into the `buffer`. * * == Format == * - Number of bitmaps (4 bytes) * - For each individual bitmap: * - Length of the serialized bitmap * - Serialized bitmap data using the standard format * (see https://github.com/RoaringBitmap/RoaringFormatSpec) */ override def serialize(bitmaps: Array[RoaringBitmap], buffer: ByteBuffer): Unit = { buffer.putInt(bitmaps.length) for (bitmap <- bitmaps) { val placeholderPos = buffer.position() buffer.putInt(-1) // Placeholder for the serialized size val startPos = placeholderPos + 4 bitmap.serialize(buffer) val endPos = buffer.position() val writtenBytes = endPos - startPos buffer.putInt(placeholderPos, writtenBytes) } } override def deserialize(buffer: ByteBuffer): Array[RoaringBitmap] = { val numberOfBitmaps = buffer.getInt if (numberOfBitmaps < 0) { throw new IOException(s"Invalid RoaringBitmapArray length" + s" ($numberOfBitmaps < 0)") } val bitmaps = Array.fill(numberOfBitmaps)(new RoaringBitmap()) for (index <- 0 until numberOfBitmaps) { val bitmapSize = buffer.getInt bitmaps(index).deserialize(buffer) // RoaringBitmap.deserialize doesn't move the buffer's pointer buffer.position(buffer.position() + bitmapSize) } bitmaps } } /** * This is the "official" portable format defined in the spec. * * See [[https://github.com/RoaringBitmap/RoaringFormatSpec#extention-for-64-bit-implementations]] */ private[sql] object PortableRoaringBitmapArraySerializationFormat extends RoaringBitmapArraySerializationFormat { override val MAGIC_NUMBER: Int = 1681511377 override def serializedSizeInBytes(bitmaps: Array[RoaringBitmap]): Long = { val bitmapCountSize = 8 val individualBitmapKeySize = 4 val bitmapSizes = bitmaps.foldLeft(0L) { (sum, bitmap) => sum + bitmap.serializedSizeInBytes() + individualBitmapKeySize } bitmapCountSize + bitmapSizes } /** * Serialize `bitmaps` into the `buffer`. * * ==Format== * - Number of bitmaps (8 bytes, upper 4 are basically padding) * - For each individual bitmap, in increasing key order (unsigned, technically, but * RoaringBitmapArray doesn't support negative keys anyway.): * - key of the bitmap (upper 32 bit) * - Serialized bitmap data using the standard format (see * https://github.com/RoaringBitmap/RoaringFormatSpec) */ override def serialize(bitmaps: Array[RoaringBitmap], buffer: ByteBuffer): Unit = { buffer.putLong(bitmaps.length.toLong) // Iterate in index-order, so that the keys are ascending as required by spec. for ((bitmap, index) <- bitmaps.zipWithIndex) { // In our array-based implementation the index is the key. buffer.putInt(index) bitmap.serialize(buffer) } } override def deserialize(buffer: ByteBuffer): Array[RoaringBitmap] = { val numberOfBitmaps = buffer.getLong // These cases are allowed by the format, but out implementation doesn't support them. if (numberOfBitmaps < 0L) { throw new IOException(s"Invalid RoaringBitmapArray length ($numberOfBitmaps < 0)") } if (numberOfBitmaps > Int.MaxValue) { throw new IOException( s"Invalid RoaringBitmapArray length ($numberOfBitmaps > ${Int.MaxValue})") } // This format is designed for sparse bitmaps, so numberOfBitmaps is only a lower bound for the // actual size of the array. val minimumArraySize = numberOfBitmaps.toInt val bitmaps = Array.newBuilder[RoaringBitmap] bitmaps.sizeHint(minimumArraySize) var lastIndex = 0 for (_ <- 0L until numberOfBitmaps) { val key = buffer.getInt if (key < 0L) { throw new IOException(s"Invalid unsigned entry in RoaringBitmapArray ($key)") } assert(key >= lastIndex, "Keys are required to be sorted in ascending order.") // Fill gaps in sparse data. while (lastIndex < key) { bitmaps += new RoaringBitmap() lastIndex += 1 } val bitmap = new RoaringBitmap() bitmap.deserialize(buffer) bitmaps += bitmap lastIndex += 1 // RoaringBitmap.deserialize doesn't move the buffer's pointer buffer.position(buffer.position() + bitmap.serializedSizeInBytes()) } bitmaps.result() } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/deletionvectors/RowIndexMarkingFilters.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.deletionvectors import org.apache.spark.sql.delta.RowIndexFilter import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.sql.execution.vectorized.WritableColumnVector import org.apache.spark.sql.vectorized.ColumnVector /** * Base class for row index filters. * @param bitmap Represents the deletion vector. */ abstract sealed class RowIndexMarkingFilters(bitmap: RoaringBitmapArray) extends RowIndexFilter { val valueWhenContained: Byte val valueWhenNotContained: Byte private def isContainedInBitmap(rowIndex: Long): Byte = { val isContained = bitmap.contains(rowIndex) if (isContained) { valueWhenContained } else { valueWhenNotContained } } override def materializeIntoVector(start: Long, end: Long, batch: WritableColumnVector): Unit = { val batchSize = (end - start).toInt var rowId = 0 while (rowId < batchSize) { val isContained = isContainedInBitmap(start + rowId.toLong) batch.putByte(rowId, isContained) rowId += 1 } } override def materializeIntoVectorWithRowIndex( batchSize: Int, rowIndexColumn: ColumnVector, batch: WritableColumnVector): Unit = { for (rowNumber <- 0 until batchSize) { val rowIndex = rowIndexColumn.getLong(rowNumber) val isContained = isContainedInBitmap(rowIndex) batch.putByte(rowNumber, isContained) } } override def materializeSingleRowWithRowIndex( rowIndex: Long, batch: WritableColumnVector): Unit = { val isContained = isContainedInBitmap(rowIndex) // Assumes the batch has only one element. batch.putByte(0, isContained) } } sealed trait RowIndexMarkingFiltersBuilder { def getFilterForEmptyDeletionVector(): RowIndexFilter def getFilterForNonEmptyDeletionVector(bitmap: RoaringBitmapArray): RowIndexFilter def createInstance( deletionVector: DeletionVectorDescriptor, hadoopConf: Configuration, tablePath: Option[Path]): RowIndexFilter = { if (deletionVector.cardinality == 0) { getFilterForEmptyDeletionVector() } else { require(tablePath.nonEmpty, "Table path is required for non-empty deletion vectors") val dvStore = DeletionVectorStore.createInstance(hadoopConf) val storedBitmap = StoredBitmap.create(deletionVector, tablePath.get) val bitmap = storedBitmap.load(dvStore) getFilterForNonEmptyDeletionVector(bitmap) } } } /** * Implementation of [[RowIndexFilter]] which checks, for a given row index and deletion vector, * whether the row index is present in the deletion vector. If present, the row is marked for * skipping. * @param bitmap Represents the deletion vector */ final class DropMarkedRowsFilter(bitmap: RoaringBitmapArray) extends RowIndexMarkingFilters(bitmap) { override val valueWhenContained: Byte = RowIndexFilter.DROP_ROW_VALUE override val valueWhenNotContained: Byte = RowIndexFilter.KEEP_ROW_VALUE } /** * Utility methods that creates [[DropMarkedRowsFilter]] to filter out row indices that are present * in the given deletion vector. */ object DropMarkedRowsFilter extends RowIndexMarkingFiltersBuilder { override def getFilterForEmptyDeletionVector(): RowIndexFilter = KeepAllRowsFilter override def getFilterForNonEmptyDeletionVector(bitmap: RoaringBitmapArray): RowIndexFilter = new DropMarkedRowsFilter(bitmap) } /** * Implementation of [[RowIndexFilter]] which checks, for a given row index and deletion vector, * whether the row index is present in the deletion vector. If not present, the row is marked for * skipping. * @param bitmap Represents the deletion vector */ final class KeepMarkedRowsFilter(bitmap: RoaringBitmapArray) extends RowIndexMarkingFilters(bitmap) { override val valueWhenContained: Byte = RowIndexFilter.KEEP_ROW_VALUE override val valueWhenNotContained: Byte = RowIndexFilter.DROP_ROW_VALUE } /** * Utility methods that creates [[KeepMarkedRowsFilter]] to filter out row indices that are present * in the given deletion vector. */ object KeepMarkedRowsFilter extends RowIndexMarkingFiltersBuilder { override def getFilterForEmptyDeletionVector(): RowIndexFilter = DropAllRowsFilter override def getFilterForNonEmptyDeletionVector(bitmap: RoaringBitmapArray): RowIndexFilter = new KeepMarkedRowsFilter(bitmap) } case object DropAllRowsFilter extends RowIndexFilter { override def materializeIntoVector(start: Long, end: Long, batch: WritableColumnVector): Unit = { val batchSize = (end - start).toInt var rowId = 0 while (rowId < batchSize) { batch.putByte(rowId, RowIndexFilter.DROP_ROW_VALUE) rowId += 1 } } override def materializeIntoVectorWithRowIndex( batchSize: Int, rowIndexColumn: ColumnVector, batch: WritableColumnVector): Unit = { for (rowId <- 0 until batchSize) { batch.putByte(rowId, RowIndexFilter.DROP_ROW_VALUE) } } override def materializeSingleRowWithRowIndex( rowIndex: Long, batch: WritableColumnVector): Unit = // Assumes the batch has only one element. batch.putByte(0, RowIndexFilter.DROP_ROW_VALUE) } case object KeepAllRowsFilter extends RowIndexFilter { override def materializeIntoVector(start: Long, end: Long, batch: WritableColumnVector): Unit = { val batchSize = (end - start).toInt var rowId = 0 while (rowId < batchSize) { batch.putByte(rowId, RowIndexFilter.KEEP_ROW_VALUE) rowId += 1 } } override def materializeIntoVectorWithRowIndex( batchSize: Int, rowIndexColumn: ColumnVector, batch: WritableColumnVector): Unit = { for (rowId <- 0 until batchSize) { batch.putByte(rowId, RowIndexFilter.KEEP_ROW_VALUE) } } override def materializeSingleRowWithRowIndex( rowIndex: Long, batch: WritableColumnVector): Unit = // Assumes the batch has only one element. batch.putByte(0, RowIndexFilter.KEEP_ROW_VALUE) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/deletionvectors/StoredBitmap.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.deletionvectors import java.io.{IOException, ObjectInputStream} import org.apache.spark.sql.delta.DeltaErrors import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore import org.apache.hadoop.fs.Path import org.apache.spark.util.Utils /** * Interface for bitmaps that are stored as Deletion Vectors. */ trait StoredBitmap { /** * Read the bitmap into memory. * Use `dvStore` if this variant is in cloud storage, otherwise just deserialize. */ def load(dvStore: DeletionVectorStore): RoaringBitmapArray /** * The serialized size of the stored bitmap in bytes. * Can be used for planning memory management without a round-trip to cloud storage. */ def size: Int /** * The number of entries in the bitmap. */ def cardinality: Long /** * Returns a unique identifier for this bitmap (Deletion Vector serialized as a JSON object). */ def getUniqueId: String } /** * Bitmap for a Deletion Vector, implemented as a thin wrapper around a Deletion Vector * Descriptor. The bitmap can be empty, inline or on-disk. In case of on-disk deletion * vectors, `tableDataPath` must be set to the data path of the Delta table, which is where * deletion vectors are stored. */ case class DeletionVectorStoredBitmap( dvDescriptor: DeletionVectorDescriptor, tableDataPath: Option[Path] = None ) extends StoredBitmap with DeltaLogging { require(tableDataPath.isDefined || !dvDescriptor.isOnDisk, "Table path is required for on-disk deletion vectors") override def load(dvStore: DeletionVectorStore): RoaringBitmapArray = { val bitmap = if (isEmpty) { new RoaringBitmapArray() } else if (isInline) { DeletionVectorUtils.deserialize( dvDescriptor.inlineData, tableDataPath, debugInfo = Map("dvDescriptor" -> dvDescriptor)) } else { assert(isOnDisk) dvStore.read(onDiskPath.get, dvDescriptor.offset.getOrElse(0), dvDescriptor.sizeInBytes) } // Verify that the cardinality in the bitmap matches the DV descriptor. if (bitmap.cardinality != dvDescriptor.cardinality) { recordDeltaEvent( deltaLog = null, opType = "delta.assertions.deletionVectorReadCardinalityMismatch", data = Map( "deletionVectorPath" -> onDiskPath, "deletionVectorCardinality" -> bitmap.cardinality, "deletionVectorDescriptor" -> dvDescriptor), path = tableDataPath) throw DeltaErrors.deletionVectorCardinalityMismatch() } bitmap } override def size: Int = dvDescriptor.sizeInBytes override def cardinality: Long = dvDescriptor.cardinality override lazy val getUniqueId: String = dvDescriptor.serializeToBase64() private def isEmpty: Boolean = dvDescriptor.isEmpty private def isInline: Boolean = dvDescriptor.isInline private def isOnDisk: Boolean = dvDescriptor.isOnDisk /** The absolute path for on-disk deletion vectors. */ private lazy val onDiskPath: Option[Path] = tableDataPath.map(dvDescriptor.absolutePath) } object StoredBitmap { /** The stored bitmap of an empty deletion vector. */ final val EMPTY = DeletionVectorStoredBitmap(DeletionVectorDescriptor.EMPTY, None) /** Factory for inline deletion vectors. */ def inline(dvDescriptor: DeletionVectorDescriptor): StoredBitmap = { require(dvDescriptor.isInline) DeletionVectorStoredBitmap(dvDescriptor, None) } /** Factory for deletion vectors. */ def create(dvDescriptor: DeletionVectorDescriptor, tablePath: Path): StoredBitmap = { if (dvDescriptor.isOnDisk) { DeletionVectorStoredBitmap(dvDescriptor, Some(tablePath)) } else { DeletionVectorStoredBitmap(dvDescriptor, None) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/expressions/DecodeNestedZ85EncodedVariant.scala ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.expressions import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.TypeCheckResult import org.apache.spark.sql.catalyst.expressions.{Expression, UnaryExpression} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.delta.util.DeltaStatsJsonUtils import org.apache.spark.sql.types._ import org.apache.spark.types.variant.{Variant, VariantUtil} import org.apache.spark.unsafe.types.VariantVal /** * An expression that replaces Z85-encoded variant strings with decoded VariantVals. * * When parsing JSON stats with variant fields, the variants are initially encoded as Z85 strings. * The standard from_json treats these as regular strings and creates VariantVal objects that * contain the Z85 string representation. This expression walks through the result and decodes * any Z85-encoded variants to their proper binary representation. * * @param child The expression producing the row with Z85-encoded variants. */ case class DecodeNestedZ85EncodedVariant(child: Expression) extends UnaryExpression with CodegenFallback { override def dataType: DataType = child.dataType override def nullable: Boolean = child.nullable override def checkInputDataTypes(): TypeCheckResult = { if (!child.dataType.isInstanceOf[StructType]) { TypeCheckResult.TypeCheckFailure(s"The top-level data type for the input to " + s"DecodeNestedZ85EncodedVariant must be StructType but this is not true " + s"in: ${child.dataType}.") } else if (!isValidType(child.dataType)) { TypeCheckResult.TypeCheckFailure( s"DecodeNestedZ85EncodedVariant does not support arrays or maps in schema. " + s"Found: ${child.dataType}") } else { TypeCheckResult.TypeCheckSuccess } } // The data type cannot contain arrays or maps since stats structs do not have arrays or maps yet. private def isValidType(dataType: DataType): Boolean = { dataType match { case _: ArrayType => false case _: MapType => false case st: StructType => st.fields.forall(field => isValidType(field.dataType)) case _ => true } } override protected def nullSafeEval(input: Any): Any = { transformValue(input, child.dataType) } private def transformValue(value: Any, dataType: DataType): Any = { if (value == null) { return null } dataType match { case VariantType => val variantVal = value.asInstanceOf[VariantVal] val variant = new Variant(variantVal.getValue, variantVal.getMetadata) if (VariantUtil.getType(variant.getValue, 0) == VariantUtil.Type.STRING) { val z85String = variant.getString() DeltaStatsJsonUtils.decodeVariantFromZ85(z85String) } else { throw new IllegalStateException( s"Expected Z85-encoded variant string but got type " + s"${VariantUtil.getType(variant.getValue, 0)}") } case st: StructType => val row = value.asInstanceOf[InternalRow] val newValues = st.fields.zipWithIndex.map { case (field, i) => val fieldValue = row.get(i, field.dataType) transformValue(fieldValue, field.dataType) } InternalRow.fromSeq(newValues) case _ => value } } override def prettyName: String = "replace_variant_z85_with_variant_val" override protected def withNewChildInternal(newChild: Expression) : DecodeNestedZ85EncodedVariant = copy(child = newChild) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/expressions/EncodeNestedVariantAsZ85String.scala ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.expressions import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.TypeCheckResult import org.apache.spark.sql.catalyst.expressions.{Expression, UnaryExpression} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.delta.util.DeltaStatsJsonUtils import org.apache.spark.sql.types._ import org.apache.spark.types.variant.Variant import org.apache.spark.unsafe.types.{UTF8String, VariantVal} /** * An expression that encodes VariantVal fields in a struct as Z85 strings. * * When converting stats structs to JSON for state reconstruction, variants need to be * encoded as Z85 strings to preserve their binary representation. This expression walks * through the struct and replaces any VariantVal fields with their Z85 string encoding. * * The output schema has VariantType fields replaced with StringType. * * @param child The expression producing the row with VariantVal fields. */ case class EncodeNestedVariantAsZ85String(child: Expression) extends UnaryExpression with CodegenFallback { override def dataType: DataType = transformDataType(child.dataType) override def nullable: Boolean = child.nullable override def checkInputDataTypes(): TypeCheckResult = { if (!child.dataType.isInstanceOf[StructType]) { TypeCheckResult.TypeCheckFailure(s"The top-level data type for the input to " + s"EncodeNestedVariantAsZ85String must be StructType but this is not true " + s"in: ${child.dataType}.") } else if (!isValidType(child.dataType)) { TypeCheckResult.TypeCheckFailure( s"EncodeNestedVariantAsZ85String does not support arrays or maps in schema. " + s"Found: ${child.dataType}") } else { TypeCheckResult.TypeCheckSuccess } } // The data type cannot contain arrays or maps since stats structs do not have arrays or maps yet. private def isValidType(dataType: DataType): Boolean = { dataType match { case _: ArrayType => false case _: MapType => false case st: StructType => st.fields.forall(field => isValidType(field.dataType)) case _ => true } } /** * Transform the data type by replacing VariantType with StringType. */ private def transformDataType(dataType: DataType): DataType = { dataType match { case VariantType => StringType case st: StructType => StructType(st.fields.map { field => field.copy(dataType = transformDataType(field.dataType)) }) case other => other } } override protected def nullSafeEval(input: Any): Any = { transformValue(input, child.dataType) } private def transformValue(value: Any, dataType: DataType): Any = { if (value == null) { return null } dataType match { case VariantType => val variantVal = value.asInstanceOf[VariantVal] val variant = new Variant(variantVal.getValue, variantVal.getMetadata) val z85String = DeltaStatsJsonUtils.encodeVariantAsZ85(variant) UTF8String.fromString(z85String) case st: StructType => val row = value.asInstanceOf[InternalRow] val newValues = st.fields.zipWithIndex.map { case (field, i) => val fieldValue = row.get(i, field.dataType) transformValue(fieldValue, field.dataType) } InternalRow.fromSeq(newValues) case _ => value } } override def prettyName: String = "encode_variant_as_z85_string" override protected def withNewChildInternal(newChild: Expression) : EncodeNestedVariantAsZ85String = copy(child = newChild) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/expressions/HilbertIndex.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.expressions import java.util import scala.collection.mutable import org.apache.spark.sql.delta.expressions.HilbertUtils._ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.types.{AbstractDataType, DataType, DataTypes} /** * Represents a hilbert index built from the provided columns. * The columns are expected to all be Ints and to have at most numBits individually. * The points along the hilbert curve are represented by Longs. */ private[sql] case class HilbertLongIndex(numBits: Int, children: Seq[Expression]) extends Expression with ExpectsInputTypes with CodegenFallback { private val n: Int = children.size private val nullValue: Int = 0 override def nullable: Boolean = false // pre-initialize working set array private val ints = new Array[Int](n) override def eval(input: InternalRow): Any = { var i = 0 while (i < n) { ints(i) = children(i).eval(input) match { case null => nullValue case int: Integer => int case any => throw new IllegalArgumentException( s"${this.getClass.getSimpleName} expects only inputs of type Int, but got: " + s"$any of type${any.getClass.getSimpleName}") } i += 1 } HilbertStates.getStateList(n).translateNPointToDKey(ints, numBits) } override def dataType: DataType = DataTypes.LongType override def inputTypes: Seq[AbstractDataType] = Seq.fill(n)(DataTypes.IntegerType) override protected def withNewChildrenInternal( newChildren: IndexedSeq[Expression]): HilbertLongIndex = copy(children = newChildren) } /** * Represents a hilbert index built from the provided columns. * The columns are expected to all be Ints and to have at most numBits. * The points along the hilbert curve are represented by Byte arrays. */ private[sql] case class HilbertByteArrayIndex(numBits: Int, children: Seq[Expression]) extends Expression with ExpectsInputTypes with CodegenFallback { private val n: Int = children.size private val nullValue: Int = 0 override def nullable: Boolean = false // pre-initialize working set array private val ints = new Array[Int](n) override def eval(input: InternalRow): Any = { var i = 0 while (i < n) { ints(i) = children(i).eval(input) match { case null => nullValue case int: Integer => int case any => throw new IllegalArgumentException( s"${this.getClass.getSimpleName} expects only inputs of type Int, but got: " + s"$any of type${any.getClass.getSimpleName}") } i += 1 } HilbertStates.getStateList(n).translateNPointToDKeyArray(ints, numBits) } override def dataType: DataType = DataTypes.BinaryType override def inputTypes: Seq[AbstractDataType] = Seq.fill(n)(DataTypes.IntegerType) override protected def withNewChildrenInternal( newChildren: IndexedSeq[Expression]): HilbertByteArrayIndex = copy(children = newChildren) } // scalastyle:off line.size.limit /** * The following code is based on this paper: * https://citeseerx.ist.psu.edu/document?repid=rep1&type=pdf&doi=bfd6d94c98627756989b0147a68b7ab1f881a0d6 * with optimizations around matrix manipulation taken from this one: * https://pdfs.semanticscholar.org/4043/1c5c43a2121e1bc071fc035e90b8f4bb7164.pdf * * At a high level you construct a GeneratorTable with the getStateGenerator method. * That represents the information necessary to construct a state list for a given number * of dimension, N. * Once you have the generator table for your dimension you can construct a state list. * You can then turn those state lists into compact state lists that store all the information * in one large array of longs. */ // scalastyle:on line.size.limit object HilbertIndex { private type CompactStateList = HilbertCompactStateList val SIZE_OF_INT = 32 /** * Construct the generator table for a space of dimension n. * This table consists of 2^n rows, each row containing Y, X1, and TY. * Y The index in the array representing the table. (0 to (2^n - 1)) * X1 A coordinate representing points on the curve expressed as an n-point. * These are arranged such that if two rows differ by 1 in Y then the binary * representation of their X1 values differ by exactly one bit. * These are the "Gray-codes" of their Y value. * TY A transformation matrix that transforms X2(1) to the X1 value where Y is zero and * transforms X2(2) to the X1 value where Y is (2^n - 1) */ def getStateGenerator(n: Int): GeneratorTable = { val x2s = getX2GrayCodes(n) val len = 1 << n val rows = (0 until len).map { i => // A pair of n-points corresponding to the first and last points on the first order curve to // which X1 transforms in the construction of a second order curve. val x21 = x2s(i << 1) val x22 = x2s((i << 1) + 1) // Represents the magnitude of difference between X2 values in this row. val dy = x21 ^ x22 Row( y = i, x1 = i ^ (i >>> 1), m = HilbertMatrix(n, x21, getSetColumn(n, dy)) ) } new GeneratorTable(n, rows) } // scalastyle:off line.size.limit /** * This will construct an x2-gray-codes sequence of order n as described in * https://citeseerx.ist.psu.edu/document?repid=rep1&type=pdf&doi=bfd6d94c98627756989b0147a68b7ab1f881a0d6 * * Each pair of values corresponds to the first and last coordinates of points on a first * order curve to which a point taken from column X1 transforms to at the second order. */ // scalastyle:on line.size.limit private[this] def getX2GrayCodes(n: Int) : Array[Int] = { if (n == 1) { // hard code the base case return Array(0, 1, 0, 1) } val mask = 1 << (n - 1) val base = getX2GrayCodes(n - 1) base(base.length - 1) = base(base.length - 2) + mask val result = Array.fill(base.length * 2)(0) base.indices.foreach { i => result(i) = base(i) result(result.length - 1 - i) = base(i) ^ mask } result } private[this] case class Row(y: Int, x1: Int, m: HilbertMatrix) private[this] case class PointState(y: Int, var x1: Int = 0, var state: Int = 0) private[this] case class State(id: Int, matrix: HilbertMatrix, var pointStates: Seq[PointState]) private[sql] class StateList(n: Int, states: Map[Int, State]) { def getNPointToDKeyStateMap: CompactStateList = { val numNPoints = 1 << n val array = new Array[Long](numNPoints * states.size) states.foreach { case (stateIdx, state) => val stateStartIdx = stateIdx * numNPoints state.pointStates.foreach { ps => val psLong = (ps.y.toLong << SIZE_OF_INT) | ps.state.toLong array(stateStartIdx + ps.x1) = psLong } } new CompactStateList(n, array) } def getDKeyToNPointStateMap: CompactStateList = { val numNPoints = 1 << n val array = new Array[Long](numNPoints * states.size) states.foreach { case (stateIdx, state) => val stateStartIdx = stateIdx * numNPoints state.pointStates.foreach { ps => val psLong = (ps.x1.toLong << SIZE_OF_INT) | ps.state.toLong array(stateStartIdx + ps.y) = psLong } } new CompactStateList(n, array) } } private[sql] class GeneratorTable(n: Int, rows: Seq[Row]) { def generateStateList(): StateList = { val result = mutable.Map[Int, State]() val list = new util.LinkedList[State]() var nextStateNum = 1 val initialState = State(0, HilbertMatrix.identity(n), rows.map(r => PointState(r.y, r.x1))) result.put(0, initialState) rows.foreach { row => val matrix = row.m result.find { case (_, s) => s.matrix == matrix } match { case Some((_, s)) => initialState.pointStates(row.y).state = s.id case _ => initialState.pointStates(row.y).state = nextStateNum val newState = State(nextStateNum, matrix, Seq()) result.put(nextStateNum, newState) list.addLast(newState) nextStateNum += 1 } } while (!list.isEmpty) { val currentState = list.removeFirst() currentState.pointStates = rows.indices.map(r => PointState(r)) rows.indices.foreach { i => val j = currentState.matrix.transform(i) val p = initialState.pointStates.find(_.x1 == j).get val currentPointState = currentState.pointStates(p.y) currentPointState.x1 = i val tm = result(p.state).matrix.multiply(currentState.matrix) result.find { case (_, s) => s.matrix == tm } match { case Some((_, s)) => currentPointState.state = s.id case _ => currentPointState.state = nextStateNum val newState = State(nextStateNum, tm, Seq()) result.put(nextStateNum, newState) list.addLast(newState) nextStateNum += 1 } } } new StateList(n, result.toMap) } } } /** * Represents a compact state map. This is used in the mapping between n-points and d-keys. * [[array]] is treated as a Map(Int -> Map(Int -> (Int, Int))) * * Each values in the array will be a combination of two things, a point and the index of the * next state, in the most- and least- significant bits, respectively. * state -> coord -> [point + nextState] */ private[sql] class HilbertCompactStateList(n: Int, array: Array[Long]) { private val maxNumN = 1 << n private val mask = maxNumN - 1 private val intMask = (1L << HilbertIndex.SIZE_OF_INT) - 1 // point and nextState @inline def transform(nPoint: Int, state: Int): (Int, Int) = { val value = array(state * maxNumN + nPoint) ( (value >>> HilbertIndex.SIZE_OF_INT).toInt, (value & intMask).toInt ) } // These while loops are to minimize overhead. // This method exists only for testing private[expressions] def translateDKeyToNPoint(key: Long, k: Int): Array[Int] = { val result = new Array[Int](n) var currentState = 0 var i = 0 while (i < k) { val h = (key >> ((k - 1 - i) * n)) & mask val (z, nextState) = transform(h.toInt, currentState) var j = 0 while (j < n) { val v = (z >> (n - 1 - j)) & 1 result(j) = (result(j) << 1) | v j += 1 } currentState = nextState i += 1 } result } // These while loops are to minimize overhead. // This method exists only for testing private[expressions] def translateDKeyArrayToNPoint(key: Array[Byte], k: Int): Array[Int] = { val result = new Array[Int](n) val initialOffset = (key.length * 8) - (k * n) var currentState = 0 var i = 0 while (i < k) { val offset = initialOffset + (i * n) val h = getBits(key, offset, n) val (z, nextState) = transform(h, currentState) var j = 0 while (j < n) { val v = (z >> (n - 1 - j)) & 1 result(j) = (result(j) << 1) | v j += 1 } currentState = nextState i += 1 } result } /** * Translate an n-dimensional point into it's corresponding position on the n-dimensional * hilbert curve. * @param point An n-dimensional point. (assumed to have n elements) * @param k The number of meaningful bits in each value of the point. */ def translateNPointToDKey(point: Array[Int], k: Int): Long = { var result = 0L var currentState = 0 var i = 0 while (i < k) { var z = 0 var j = 0 while (j < n) { z = (z << 1) | ((point(j) >> (k - 1 - i)) & 1) j += 1 } val (h, nextState) = transform(z, currentState) result = (result << n) | h currentState = nextState i += 1 } result } /** * Translate an n-dimensional point into it's corresponding position on the n-dimensional * hilbert curve. Returns the resulting integer as an array of bytes. * @param point An n-dimensional point. (assumed to have n elements) * @param k The number of meaningful bits in each value of the point. */ def translateNPointToDKeyArray(point: Array[Int], k: Int): Array[Byte] = { val numBits = k * n val numBytes = (numBits + 7) / 8 val result = new Array[Byte](numBytes) val initialOffset = (numBytes * 8) - numBits var currentState = 0 var i = 0 while (i < k) { var z = 0 var j = 0 while (j < n) { z = (z << 1) | ((point(j) >> (k - 1 - i)) & 1) j += 1 } val (h, nextState) = transform(z, currentState) setBits(result, initialOffset + (i * n), h, n) currentState = nextState i += 1 } result } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/expressions/HilbertStates.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.expressions; import org.apache.spark.SparkException; public class HilbertStates { /** * Constructs a hilbert state for the given arity, [[n]]. * This state list can be used to map n-points to their corresponding d-key value. * * @param n The number of bits in this space (we assert 2 <= n <= 9 for simplicity) * @return The CompactStateList for mapping from n-point to hilbert distance key. */ private static HilbertCompactStateList constructHilbertState(int n) { HilbertIndex.GeneratorTable generator = HilbertIndex.getStateGenerator(n); return generator.generateStateList().getNPointToDKeyStateMap(); } private HilbertStates() { } private static class HilbertIndex2 { static final HilbertCompactStateList STATE_LIST = constructHilbertState(2); } private static class HilbertIndex3 { static final HilbertCompactStateList STATE_LIST = constructHilbertState(3); } private static class HilbertIndex4 { static final HilbertCompactStateList STATE_LIST = constructHilbertState(4); } private static class HilbertIndex5 { static final HilbertCompactStateList STATE_LIST = constructHilbertState(5); } private static class HilbertIndex6 { static final HilbertCompactStateList STATE_LIST = constructHilbertState(6); } private static class HilbertIndex7 { static final HilbertCompactStateList STATE_LIST = constructHilbertState(7); } private static class HilbertIndex8 { static final HilbertCompactStateList STATE_LIST = constructHilbertState(8); } private static class HilbertIndex9 { static final HilbertCompactStateList STATE_LIST = constructHilbertState(9); } public static HilbertCompactStateList getStateList(int n) throws SparkException { switch (n) { case 2: return HilbertIndex2.STATE_LIST; case 3: return HilbertIndex3.STATE_LIST; case 4: return HilbertIndex4.STATE_LIST; case 5: return HilbertIndex5.STATE_LIST; case 6: return HilbertIndex6.STATE_LIST; case 7: return HilbertIndex7.STATE_LIST; case 8: return HilbertIndex8.STATE_LIST; case 9: return HilbertIndex9.STATE_LIST; default: throw new SparkException(String.format("Cannot perform hilbert clustering on " + "fewer than 2 or more than 9 dimensions; got %d dimensions", n)); } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/expressions/HilbertUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.expressions object HilbertUtils { /** * Returns the column number that is set. We assume that a bit is set. */ @inline def getSetColumn(n: Int, i: Int): Int = { n - 1 - Integer.numberOfTrailingZeros(i) } @inline def circularLeftShift(n: Int, i: Int, shift: Int): Int = { ((i << shift) | (i >>> (n - shift))) & ((1 << n) - 1) } @inline def circularRightShift(n: Int, i: Int, shift: Int): Int = { ((i >>> shift) | (i << (n - shift))) & ((1 << n) - 1) } @inline private[expressions] def getBits(key: Array[Byte], offset: Int, n: Int): Int = { // [ ][ ][ ][ ][ ] // <---offset---> [ n-bits ] <- this is the result var result = 0 var remainingBits = n var keyIndex = offset / 8 // initial key offset var keyOffset = offset - (keyIndex * 8) while (remainingBits > 0) { val bitsFromIdx = math.min(remainingBits, 8 - keyOffset) val newInt = if (remainingBits >= 8) { java.lang.Byte.toUnsignedInt(key(keyIndex)) } else { java.lang.Byte.toUnsignedInt(key(keyIndex)) >>> (8 - keyOffset - bitsFromIdx) } result = (result << bitsFromIdx) | (newInt & ((1 << bitsFromIdx) - 1)) remainingBits -= (8 - keyOffset) keyOffset = 0 keyIndex += 1 } result } @inline private[expressions] def setBits( key: Array[Byte], offset: Int, newBits: Int, n: Int): Array[Byte] = { // bits: [ meaningless bits ][ n meaningful bits ] // // [ ][ ][ ][ ][ ] // <---offset---> [ n-bits ] // move meaningful bits to the far left var bits = newBits << (32 - n) var remainingBits = n // initial key index var keyIndex = offset / 8 // initial key offset var keyOffset = offset - (keyIndex * 8) while (remainingBits > 0) { key(keyIndex) = (key(keyIndex) | (bits >>> (24 + keyOffset))).toByte remainingBits -= (8 - keyOffset) bits = bits << (8 - keyOffset) keyOffset = 0 keyIndex += 1 } key } /** * treats `key` as an Integer and adds 1 */ @inline def addOne(key: Array[Byte]): Array[Byte] = { var idx = key.length - 1 var overflow = true while (overflow && idx >= 0) { key(idx) = (key(idx) + 1.toByte).toByte overflow = key(idx) == 0 idx -= 1 } key } def manhattanDist(p1: Array[Int], p2: Array[Int]): Int = { assert(p1.length == p2.length) p1.zip(p2).map { case (a, b) => math.abs(a - b) }.sum } /** * This is not really a matrix, but a representation of one. Due to the constraints of this * system the necessary matrices can be defined by two values: dY and X2. DY is the amount * of right shifting of the identity matrix, and X2 is a bitmask for which column values are * negative. The [[toString]] method is overridden to construct and print the matrix to aid * in debugging. * Instead of constructing the matrix directly we store and manipulate these values. */ case class HilbertMatrix(n: Int, x2: Int, dy: Int) { override def toString(): String = { val sb = new StringBuilder() val base = 1 << (n - 1 - dy) (0 until n).foreach { i => sb.append('\n') val row = circularRightShift(n, base, i) (0 until n).foreach { j => if (isColumnSet(row, j)) { if (isColumnSet(x2, j)) { sb.append('-') } else { sb.append(' ') } sb.append('1') } else { sb.append(" 0") } } } sb.append('\n') sb.toString } // columns count from the left: 0, 1, 2 ... , n @inline def isColumnSet(i: Int, column: Int): Boolean = { val mask = 1 << (n - 1 - column) (i & mask) > 0 } def transform(e: Int): Int = { circularLeftShift(n, e ^ x2, dy) } def multiply(other: HilbertMatrix): HilbertMatrix = { HilbertMatrix(n, circularRightShift(n, x2, other.dy) ^ other.x2, (dy + other.dy) % n) } } object HilbertMatrix { def identity(n: Int): HilbertMatrix = { HilbertMatrix(n, 0, 0) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/expressions/InterleaveBits.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.expressions import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.catalyst.{InternalRow, SQLConfHelper} import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.types.{BinaryType, DataType, IntegerType} /** * Interleaves the bits of its input data in a round-robin fashion. * * If the input data is seen as a series of multidimensional points, this function computes the * corresponding Z-values, in a way that's preserving data locality: input points that are close * in the multidimensional space will be mapped to points that are close on the Z-order curve. * * The returned value is a byte array where the size of the array is 4 * num of input columns. * * @see https://en.wikipedia.org/wiki/Z-order_curve * * @note Only supports input expressions of type Int for now. */ case class InterleaveBits(children: Seq[Expression]) extends Expression with ExpectsInputTypes with SQLConfHelper with CodegenFallback /* TODO: implement doGenCode() */ { private val n: Int = children.size override def inputTypes: Seq[DataType] = Seq.fill(n)(IntegerType) override def dataType: DataType = BinaryType override def nullable: Boolean = false /** Nulls in the input will be treated like this value */ val nullValue: Int = 0 private val childrenArray: Array[Expression] = children.toArray private val fastInterleaveBitsEnabled = conf.getConf(DeltaSQLConf.FAST_INTERLEAVE_BITS_ENABLED) private val ints = new Array[Int](n) override def eval(input: InternalRow): Any = { var i = 0 while (i < n) { val int = childrenArray(i).eval(input) match { case null => nullValue case int: Int => int case any => throw new IllegalArgumentException( s"${this.getClass.getSimpleName} expects only inputs of type Int, but got: " + s"$any of type${any.getClass.getSimpleName}") } ints.update(i, int) i += 1 } InterleaveBits.interleaveBits(ints, fastInterleaveBitsEnabled) } override protected def withNewChildrenInternal( newChildren: IndexedSeq[Expression]): InterleaveBits = copy(children = newChildren) } object InterleaveBits { private[expressions] def interleaveBits( inputs: Array[Int], fastInterleaveBitsEnabled: Boolean): Array[Byte] = { if (fastInterleaveBitsEnabled) { inputs.length match { // The default algorithm has the complexity O(32 * n) (n is the number of input columns) // The new algorithm has O(4 * 8) complexity when the number of Z-Order by columns is // less than 9. It uses the algorithm described here // http://graphics.stanford.edu/~seander/bithacks.html#InterleaveTableObvious case 0 => Array.empty case 1 => intToByte(inputs(0)) case 2 => interleave2Ints(inputs(1), inputs(0)) case 3 => interleave3Ints(inputs(2), inputs(1), inputs(0)) case 4 => interleave4Ints(inputs(3), inputs(2), inputs(1), inputs(0)) case 5 => interleave5Ints(inputs(4), inputs(3), inputs(2), inputs(1), inputs(0)) case 6 => interleave6Ints(inputs(5), inputs(4), inputs(3), inputs(2), inputs(1), inputs(0)) case 7 => interleave7Ints(inputs(6), inputs(5), inputs(4), inputs(3), inputs(2), inputs(1), inputs(0)) case 8 => interleave8Ints(inputs(7), inputs(6), inputs(5), inputs(4), inputs(3), inputs(2), inputs(1), inputs(0)) case _ => defaultInterleaveBits(inputs, inputs.length) } } else { defaultInterleaveBits(inputs, inputs.length) } } private def defaultInterleaveBits(inputs: Array[Int], numCols: Int): Array[Byte] = { val ret = new Array[Byte](numCols * 4) var ret_idx: Int = 0 var ret_bit: Int = 7 var ret_byte: Byte = 0 var bit = 31 /* going from most to least significant bit */ while (bit >= 0) { var idx = 0 while (idx < numCols) { ret_byte = (ret_byte | (((inputs(idx) >> bit) & 1) << ret_bit)).toByte ret_bit -= 1 if (ret_bit == -1) { // finished processing a byte ret.update(ret_idx, ret_byte) ret_byte = 0 ret_idx += 1 ret_bit = 7 } idx += 1 } bit -= 1 } assert(ret_idx == numCols * 4) assert(ret_bit == 7) ret } private def interleave2Ints(i1: Int, i2: Int): Array[Byte] = { val result = new Array[Byte](8) var i = 0 while (i < 4) { val tmp1 = ((i1 >> (i * 8)) & 0xFF).toByte val tmp2 = ((i2 >> (i * 8)) & 0xFF).toByte var z = 0 var j = 0 while (j < 8) { val x_masked = tmp1 & (1 << j) val y_masked = tmp2 & (1 << j) z |= (x_masked << j) z |= (y_masked << (j + 1)) j = j + 1 } result((3 - i) * 2 + 1) = (z & 0xFF).toByte result((3 - i) * 2) = ((z >> 8) & 0xFF).toByte i = i + 1 } result } private def intToByte(input: Int): Array[Byte] = { val result = new Array[Byte](4) var i = 0 while (i <= 3) { val offset = i * 8 result(3 - i) = ((input >> offset) & 0xFF).toByte i += 1 } result } private def interleave3Ints(i1: Int, i2: Int, i3: Int): Array[Byte] = { val result = new Array[Byte](12) var i = 0 while (i < 4) { val tmp1 = ((i1 >> (i * 8)) & 0xFF).toByte val tmp2 = ((i2 >> (i * 8)) & 0xFF).toByte val tmp3 = ((i3 >> (i * 8)) & 0xFF).toByte var z = 0 var j = 0 while (j < 8) { val r1_mask = tmp1 & (1 << j) val r2_mask = tmp2 & (1 << j) val r3_mask = tmp3 & (1 << j) z |= (r1_mask << (2 * j)) | (r2_mask << (2 * j + 1)) | (r3_mask << (2 * j + 2)) j = j + 1 } result((3 - i) * 3 + 2) = (z & 0xFF).toByte result((3 - i) * 3 + 1) = ((z >> 8) & 0xFF).toByte result((3 - i) * 3) = ((z >> 16) & 0xFF).toByte i = i + 1 } result } private def interleave4Ints(i1: Int, i2: Int, i3: Int, i4: Int): Array[Byte] = { val result = new Array[Byte](16) var i = 0 while (i < 4) { val tmp1 = ((i1 >> (i * 8)) & 0xFF).toByte val tmp2 = ((i2 >> (i * 8)) & 0xFF).toByte val tmp3 = ((i3 >> (i * 8)) & 0xFF).toByte val tmp4 = ((i4 >> (i * 8)) & 0xFF).toByte var z = 0 var j = 0 while (j < 8) { val r1_mask = tmp1 & (1 << j) val r2_mask = tmp2 & (1 << j) val r3_mask = tmp3 & (1 << j) val r4_mask = tmp4 & (1 << j) z |= (r1_mask << (3 * j)) | (r2_mask << (3 * j + 1)) | (r3_mask << (3 * j + 2)) | (r4_mask << (3 * j + 3)) j = j + 1 } result((3 - i) * 4 + 3) = (z & 0xFF).toByte result((3 - i) * 4 + 2) = ((z >> 8) & 0xFF).toByte result((3 - i) * 4 + 1) = ((z >> 16) & 0xFF).toByte result((3 - i) * 4) = ((z >> 24) & 0xFF).toByte i = i + 1 } result } private def interleave5Ints( i1: Int, i2: Int, i3: Int, i4: Int, i5: Int): Array[Byte] = { val result = new Array[Byte](20) var i = 0 while (i < 4) { val tmp1 = ((i1 >> (i * 8)) & 0xFF).toByte val tmp2 = ((i2 >> (i * 8)) & 0xFF).toByte val tmp3 = ((i3 >> (i * 8)) & 0xFF).toByte val tmp4 = ((i4 >> (i * 8)) & 0xFF).toByte val tmp5 = ((i5 >> (i * 8)) & 0xFF).toByte var z = 0L var j = 0 while (j < 8) { val r1_mask = tmp1 & (1 << j).toLong val r2_mask = tmp2 & (1 << j).toLong val r3_mask = tmp3 & (1 << j).toLong val r4_mask = tmp4 & (1 << j).toLong val r5_mask = tmp5 & (1 << j).toLong z |= (r1_mask << (4 * j)) | (r2_mask << (4 * j + 1)) | (r3_mask << (4 * j + 2)) | (r4_mask << (4 * j + 3)) | (r5_mask << (4 * j + 4)) j = j + 1 } result((3 - i) * 5 + 4) = (z & 0xFF).toByte result((3 - i) * 5 + 3) = ((z >> 8) & 0xFF).toByte result((3 - i) * 5 + 2) = ((z >> 16) & 0xFF).toByte result((3 - i) * 5 + 1) = ((z >> 24) & 0xFF).toByte result((3 - i) * 5) = ((z >> 32) & 0xFF).toByte i = i + 1 } result } private def interleave6Ints( i1: Int, i2: Int, i3: Int, i4: Int, i5: Int, i6: Int): Array[Byte] = { val result = new Array[Byte](24) var i = 0 while (i < 4) { val tmp1 = ((i1 >> (i * 8)) & 0xFF).toByte val tmp2 = ((i2 >> (i * 8)) & 0xFF).toByte val tmp3 = ((i3 >> (i * 8)) & 0xFF).toByte val tmp4 = ((i4 >> (i * 8)) & 0xFF).toByte val tmp5 = ((i5 >> (i * 8)) & 0xFF).toByte val tmp6 = ((i6 >> (i * 8)) & 0xFF).toByte var z = 0L var j = 0 while (j < 8) { val r1_mask = tmp1 & (1 << j).toLong val r2_mask = tmp2 & (1 << j).toLong val r3_mask = tmp3 & (1 << j).toLong val r4_mask = tmp4 & (1 << j).toLong val r5_mask = tmp5 & (1 << j).toLong val r6_mask = tmp6 & (1 << j).toLong z |= (r1_mask << (5 * j)) | (r2_mask << (5 * j + 1)) | (r3_mask << (5 * j + 2)) | (r4_mask << (5 * j + 3)) | (r5_mask << (5 * j + 4)) | (r6_mask << (5 * j + 5)) j = j + 1 } result((3 - i) * 6 + 5) = (z & 0xFF).toByte result((3 - i) * 6 + 4) = ((z >> 8) & 0xFF).toByte result((3 - i) * 6 + 3) = ((z >> 16) & 0xFF).toByte result((3 - i) * 6 + 2) = ((z >> 24) & 0xFF).toByte result((3 - i) * 6 + 1) = ((z >> 32) & 0xFF).toByte result((3 - i) * 6) = ((z >> 40) & 0xFF).toByte i = i + 1 } result } private def interleave7Ints( i1: Int, i2: Int, i3: Int, i4: Int, i5: Int, i6: Int, i7: Int): Array[Byte] = { val result = new Array[Byte](28) var i = 0 while (i < 4) { val tmp1 = ((i1 >> (i * 8)) & 0xFF).toByte val tmp2 = ((i2 >> (i * 8)) & 0xFF).toByte val tmp3 = ((i3 >> (i * 8)) & 0xFF).toByte val tmp4 = ((i4 >> (i * 8)) & 0xFF).toByte val tmp5 = ((i5 >> (i * 8)) & 0xFF).toByte val tmp6 = ((i6 >> (i * 8)) & 0xFF).toByte val tmp7 = ((i7 >> (i * 8)) & 0xFF).toByte var z = 0L var j = 0 while (j < 8) { val r1_mask = tmp1 & (1 << j).toLong val r2_mask = tmp2 & (1 << j).toLong val r3_mask = tmp3 & (1 << j).toLong val r4_mask = tmp4 & (1 << j).toLong val r5_mask = tmp5 & (1 << j).toLong val r6_mask = tmp6 & (1 << j).toLong val r7_mask = tmp7 & (1 << j).toLong z |= (r1_mask << (6 * j)) | (r2_mask << (6 * j + 1)) | (r3_mask << (6 * j + 2)) | (r4_mask << (6 * j + 3)) | (r5_mask << (6 * j + 4)) | (r6_mask << (6 * j + 5)) | (r7_mask << (6 * j + 6)) j = j + 1 } result((3 - i) * 7 + 6) = (z & 0xFF).toByte result((3 - i) * 7 + 5) = ((z >> 8) & 0xFF).toByte result((3 - i) * 7 + 4) = ((z >> 16) & 0xFF).toByte result((3 - i) * 7 + 3) = ((z >> 24) & 0xFF).toByte result((3 - i) * 7 + 2) = ((z >> 32) & 0xFF).toByte result((3 - i) * 7 + 1) = ((z >> 40) & 0xFF).toByte result((3 - i) * 7) = ((z >> 48) & 0xFF).toByte i = i + 1 } result } private def interleave8Ints( i1: Int, i2: Int, i3: Int, i4: Int, i5: Int, i6: Int, i7: Int, i8: Int): Array[Byte] = { val result = new Array[Byte](32) var i = 0 while (i < 4) { val tmp1 = ((i1 >> (i * 8)) & 0xFF).toByte val tmp2 = ((i2 >> (i * 8)) & 0xFF).toByte val tmp3 = ((i3 >> (i * 8)) & 0xFF).toByte val tmp4 = ((i4 >> (i * 8)) & 0xFF).toByte val tmp5 = ((i5 >> (i * 8)) & 0xFF).toByte val tmp6 = ((i6 >> (i * 8)) & 0xFF).toByte val tmp7 = ((i7 >> (i * 8)) & 0xFF).toByte val tmp8 = ((i8 >> (i * 8)) & 0xFF).toByte var z = 0L var j = 0 while (j < 8) { val r1_mask = tmp1 & (1 << j).toLong val r2_mask = tmp2 & (1 << j).toLong val r3_mask = tmp3 & (1 << j).toLong val r4_mask = tmp4 & (1 << j).toLong val r5_mask = tmp5 & (1 << j).toLong val r6_mask = tmp6 & (1 << j).toLong val r7_mask = tmp7 & (1 << j).toLong val r8_mask = tmp8 & (1 << j).toLong z |= (r1_mask << (7 * j)) | (r2_mask << (7 * j + 1)) | (r3_mask << (7 * j + 2)) | (r4_mask << (7 * j + 3)) | (r5_mask << (7 * j + 4)) | (r6_mask << (7 * j + 5)) | (r7_mask << (7 * j + 6)) | (r8_mask << (7 * j + 7)) j = j + 1 } result((3 - i) * 8 + 7) = (z & 0xFF).toByte result((3 - i) * 8 + 6) = ((z >> 8) & 0xFF).toByte result((3 - i) * 8 + 5) = ((z >> 16) & 0xFF).toByte result((3 - i) * 8 + 4) = ((z >> 24) & 0xFF).toByte result((3 - i) * 8 + 3) = ((z >> 32) & 0xFF).toByte result((3 - i) * 8 + 2) = ((z >> 40) & 0xFF).toByte result((3 - i) * 8 + 1) = ((z >> 48) & 0xFF).toByte result((3 - i) * 8) = ((z >> 56) & 0xFF).toByte i = i + 1 } result } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/expressions/JoinedProjection.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.expressions // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap, BoundReference, Expression, GetStructField} import org.apache.spark.sql.catalyst.types.DataTypeUtils import org.apache.spark.sql.types.StructType /** * Helper class for generating a joined projection. * * * This class is used to instantiate a "Joined Row" - a wrapper that makes two rows appear to be a * single concatenated row, by using nested access. It is primarily used during statistics * collection to update a buffer of per-column aggregates (i.e. the left-hand side row) with stats * from the latest row processed (i.e. the right-hand side row). * * Implementation Note: If we instead stored `leftRow` and `rightRow` we would have to perform size * checks on `leftRow` during every access, which is slow. */ object JoinedProjection { /** * Bind attributes for a joined projection. This resulting project list expects an input row * that has two nested struct fields, the struct at position 0 must be the left hand row of the * join, and the struct at position 1 must be the right hand row of the join. * * The following shows example shows how this can be used for updating an aggregation buffer: * {{{ * val buffer = new GenericInternalRow() * * val update = GenerateMutableProjection.generate( * expressions = JoinedProjection( * leftAttributes = bufferAttrs, * rightAttributes = dataCols, * projectList = aggregates.flatMap(_.updateExpressions)), * inputSchema = Nil, * useSubexprElimination = true * ).target(buffer) * * val joinedRow = new GenericInternalRow(2) * joinedRow.update(0, input) * * def updateBuffer(input: InternalRow): Unit = { * joinedRow.update(1, input) * update(joinedRow) * } * }}} */ def bind( leftAttributes: Seq[Attribute], rightAttributes: Seq[Attribute], projectList: Seq[Expression], leftCanBeNull: Boolean = false, rightCanBeNull: Boolean = false): Seq[Expression] = { val mapping = AttributeMap( createMapping(0, leftCanBeNull, leftAttributes) ++ createMapping(1, rightCanBeNull, rightAttributes)) projectList.map { expr => expr.transformUp { case a: Attribute => mapping(a) } } } /** * Helper method to create a nested struct field with efficient value extraction. */ private def createMapping( index: Int, nullable: Boolean, attributes: Seq[Attribute]): Seq[(Attribute, Expression)] = { val ref = BoundReference( index, DataTypeUtils.fromAttributes(attributes), nullable) attributes.zipWithIndex.map { case (a, ordinal) => a -> GetStructField(ref, ordinal, Option(a.name)) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/expressions/RangePartitionId.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.expressions import org.apache.spark.Partitioner import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.TypeCheckResult import org.apache.spark.sql.catalyst.expressions.{Expression, GenericInternalRow, RowOrdering, UnaryExpression, Unevaluable} import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} import org.apache.spark.sql.types._ /** * Unevaluable placeholder expression to be rewritten by the optimizer into [[PartitionerExpr]] * * This is just a convenient way to introduce the former, without the need to manually construct the * [[RangePartitioner]] beforehand, which requires an RDD to be sampled in order to determine range * partition boundaries. The optimizer rule will take care of all that. * * @see [[org.apache.spark.sql.delta.optimizer.RangeRepartitionIdRewrite]] */ case class RangePartitionId(child: Expression, numPartitions: Int) extends UnaryExpression with Unevaluable { require(numPartitions > 0, "expected the number partitions to be greater than zero") override def checkInputDataTypes(): TypeCheckResult = { if (RowOrdering.isOrderable(child.dataType)) { TypeCheckResult.TypeCheckSuccess } else { TypeCheckResult.TypeCheckFailure(s"cannot sort data type ${child.dataType.simpleString}") } } override def dataType: DataType = IntegerType override def nullable: Boolean = false override protected def withNewChildInternal(newChild: Expression): RangePartitionId = copy(child = newChild) } /** * Thin wrapper around [[Partitioner]] instances that are used in Shuffle operations. * TODO: If needed elsewhere, consider moving it into its own file. */ case class PartitionerExpr(child: Expression, partitioner: Partitioner) extends UnaryExpression { override def dataType: DataType = IntegerType override def nullable: Boolean = false private lazy val row = new GenericInternalRow(Array[Any](null)) override def eval(input: InternalRow): Any = { val value: Any = child.eval(input) row.update(0, value) partitioner.getPartition(row) } override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { val partitionerReference = ctx.addReferenceObj("partitioner", partitioner) val rowReference = ctx.addReferenceObj("row", row) nullSafeCodeGen(ctx, ev, input => s"""$rowReference.update(0, $input); |${ev.value} = $partitionerReference.getPartition($rowReference); """.stripMargin) } override protected def withNewChildInternal(newChild: Expression): PartitionerExpr = copy(child = newChild) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/files/CdcAddFileIndex.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.files import java.text.SimpleDateFormat import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.commands.cdc.CDCReader._ import org.apache.spark.sql.delta.implicits._ import org.apache.hadoop.fs.Path import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.types.StructType /** * A [[TahoeFileIndex]] for scanning a sequence of added files as CDC. Similar to * [[TahoeBatchFileIndex]], with a bit of special handling to attach the log version * and CDC type on a per-file basis. * @param spark The Spark session. * @param filesByVersion Grouped FileActions, one per table version. * @param deltaLog The delta log instance. * @param path The table's data path. * @param snapshot The snapshot where we read CDC from. * @param rowIndexFilters Map from URI-encoded file path to a row index filter type. * * Note: Please also consider other CDC-related file indexes like [[TahoeChangeFileIndex]] * and [[TahoeRemoveFileIndex]] when modifying this file index. */ class CdcAddFileIndex( spark: SparkSession, filesByVersion: Seq[CDCDataSpec[AddFile]], deltaLog: DeltaLog, path: Path, snapshot: SnapshotDescriptor, override val rowIndexFilters: Option[Map[String, RowIndexFilterType]] = None ) extends TahoeBatchFileIndex( spark, "cdcRead", filesByVersion.flatMap(_.actions), deltaLog, path, snapshot) { override def matchingFiles( partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): Seq[AddFile] = { val addFiles = filesByVersion.flatMap { case CDCDataSpec(version, ts, files, ci) => files.map { f => // We add the metadata as faked partition columns in order to attach it on a per-file // basis. val tsOpt = Option(ts) .map(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS Z").format(_)).orNull val newPartitionVals = f.partitionValues + (CDC_COMMIT_VERSION -> version.toString) + (CDC_COMMIT_TIMESTAMP -> tsOpt) + (CDC_TYPE_COLUMN_NAME -> CDC_TYPE_INSERT) f.copy(partitionValues = newPartitionVals) } } DeltaLog.filterFileList(partitionSchema, addFiles.toDF(spark), partitionFilters) .as[AddFile] .collect() } override def inputFiles: Array[String] = { filesByVersion.flatMap(_.actions).map(f => absolutePath(f.path).toString).toArray } override val partitionSchema: StructType = CDCReader.cdcReadSchema(snapshot.metadata.partitionSchema) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/files/DelayedCommitProtocol.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.files // scalastyle:off import.ordering.noEmptyLine import java.net.URI import java.util.UUID import scala.collection.mutable.ArrayBuffer import org.apache.spark.sql.delta.DeltaErrors import org.apache.spark.sql.delta.actions.{AddCDCFile, AddFile, FileAction} import org.apache.spark.sql.delta.commands.cdc.CDCReader.{CDC_LOCATION, CDC_PARTITION_COL} import org.apache.spark.sql.delta.util.{DateFormatter, PartitionUtils, TimestampFormatter, Utils => DeltaUtils} import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.hadoop.mapreduce.{JobContext, TaskAttemptContext} import org.apache.spark.internal.Logging import org.apache.spark.internal.io.FileCommitProtocol import org.apache.spark.internal.io.FileCommitProtocol.TaskCommitMessage import org.apache.spark.sql.catalyst.expressions.Cast import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.delta.files.DeltaFileFormatWriter.PartitionedTaskAttemptContextImpl import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataType, StringType, TimestampType} import org.apache.spark.util.Utils /** * Writes out the files to `path` and returns a list of them in `addedStatuses`. Includes * special handling for partitioning on [[CDC_PARTITION_COL]] for * compatibility between enabled and disabled CDC; partitions with a value of false in this * column produce no corresponding partitioning directory. * @param path The base path files will be written * @param randomPrefixLength The length of random subdir name under 'path' that files been written * @param subdir The immediate subdir under path; If randomPrefixLength and subdir both exist, file * path will be path/subdir/[rand str of randomPrefixLength]/file */ class DelayedCommitProtocol( jobId: String, path: String, randomPrefixLength: Option[Int], subdir: Option[String]) extends FileCommitProtocol with Serializable with Logging { // Track the list of files added by a task, only used on the executors. @transient protected var addedFiles: ArrayBuffer[(Map[String, String], String)] = _ // Track the change files added, only used on the driver. Files are sorted between this buffer // and addedStatuses based on the value of the [[CDC_TYPE_COLUMN_NAME]] partition column - a // file goes to addedStatuses if the value is CDC_TYPE_NOT_CDC and changeFiles otherwise. @transient val changeFiles = new ArrayBuffer[AddCDCFile] // Track the overall files added, only used on the driver. // // In rare cases, some of these AddFiles can be empty (i.e. contain no logical records). // If the caller wishes to have only non-empty AddFiles, they must collect stats and perform // the filter themselves. See TransactionalWrite::writeFiles. This filter will be best-effort, // since there's no guarantee the stats will exist. @transient val addedStatuses = new ArrayBuffer[AddFile] // Constants for CDC partition manipulation. Used only in newTaskTempFile(), but we define them // here to avoid building a new redundant regex for every file. protected val cdcPartitionFalse = s"${CDC_PARTITION_COL}=false" protected val cdcPartitionTrue = s"${CDC_PARTITION_COL}=true" protected val cdcPartitionTrueRegex = cdcPartitionTrue.r override def setupJob(jobContext: JobContext): Unit = { } /** * Commits a job after the writes succeed. Must be called on the driver. Partitions the written * files into [[AddFile]]s and [[AddCDCFile]]s as these metadata actions are treated differently * by [[TransactionalWrite]] (i.e. AddFile's may have additional statistics injected) */ override def commitJob(jobContext: JobContext, taskCommits: Seq[TaskCommitMessage]): Unit = { val (addFiles, changeFiles) = taskCommits.flatMap(_.obj.asInstanceOf[Seq[_]]) .partition { case _: AddFile => true case _: AddCDCFile => false case other => throw DeltaErrors.unrecognizedFileAction(s"$other", s"${other.getClass}") } // we cannot add type information above because of type erasure addedStatuses ++= addFiles.map(_.asInstanceOf[AddFile]) this.changeFiles ++= changeFiles.map(_.asInstanceOf[AddCDCFile]).toArray[AddCDCFile] } override def abortJob(jobContext: JobContext): Unit = { // TODO: Best effort cleanup } override def setupTask(taskContext: TaskAttemptContext): Unit = { addedFiles = new ArrayBuffer[(Map[String, String], String)] } /** Prefix added in testing mode to all filenames to test special chars that need URL-encoding. */ val FILE_NAME_PREFIX = SQLConf.get.getConf(DeltaSQLConf.TEST_FILE_NAME_PREFIX) protected def getFileName( taskContext: TaskAttemptContext, ext: String, partitionValues: Map[String, String]): String = { // The file name looks like part-r-00000-2dd664f9-d2c4-4ffe-878f-c6c70c1fb0cb_00003.gz.parquet // Note that %05d does not truncate the split number, so if we have more than 100000 tasks, // the file name is fine and won't overflow. val split = taskContext.getTaskAttemptID.getTaskID.getId val uuid = UUID.randomUUID.toString // CDC files (CDC_PARTITION_COL = true) are named with "cdc-..." instead of "part-...". val typePrefix = if (partitionValues.get(CDC_PARTITION_COL).contains("true")) "cdc-" else "part-" f"${FILE_NAME_PREFIX}${typePrefix}${split}%05d-${uuid}${ext}" } protected def parsePartitions( dir: String, taskContext: TaskAttemptContext): Map[String, String] = { // TODO: enable validatePartitionColumns? val useUtcNormalizedTimestamps = taskContext match { case _: PartitionedTaskAttemptContextImpl => taskContext.getConfiguration.getBoolean( DeltaSQLConf.UTC_TIMESTAMP_PARTITION_VALUES.key, true) case _ => false } val partitionColumnToDataType: Map[String, DataType] = taskContext.asInstanceOf[PartitionedTaskAttemptContextImpl] .partitionColToDataType .filter(partitionCol => partitionCol._2 == TimestampType) val dateFormatter = DateFormatter() // if adjusting to UTC make sure to interpret timezones using Spark // config, otherwise fallback to JVM timezone val timezone = { if (useUtcNormalizedTimestamps) { DateTimeUtils.getTimeZone(SQLConf.get.sessionLocalTimeZone) } else { java.util.TimeZone.getDefault } } val timestampFormatter = TimestampFormatter(PartitionUtils.timestampPartitionPattern, timezone) /** * ToDo: Remove the use of this PartitionUtils API with type inference logic * since the types are already known from the Delta metadata! * * Currently types are passed to the PartitionUtils.parsePartition API to facilitate * timestamp conversion to UTC. In all other cases, the type is just inferred as a String. * Note: the passed in timestampFormatter and timezone detail * is used for parsing from the string timestamp. * If utc normalization is enabled the parsed partition value will be adjusted to UTC * and output in iso8601 format. */ val parsedPartition = PartitionUtils .parsePartition( new Path(dir), typeInference = false, Set.empty, userSpecifiedDataTypes = partitionColumnToDataType, validatePartitionColumns = false, java.util.TimeZone.getDefault, dateFormatter, timestampFormatter, useUtcNormalizedTimestamps) ._1 .get parsedPartition .columnNames .zip( parsedPartition .literals .map(PartitionUtils.literalToNormalizedString( _, Some(timezone.getID), useUtcNormalizedTimestamps))) .toMap } /** * Notifies the commit protocol to add a new file, and gets back the full path that should be * used. * * Includes special logic for CDC files and paths. Specifically, if the directory `dir` contains * the CDC partition `__is_cdc=true` then * - the file name begins with `cdc-` instead of `part-` * - the directory has the `__is_cdc=true` partition removed and is placed in the `_changed_data` * folder */ override def newTaskTempFile( taskContext: TaskAttemptContext, dir: Option[String], ext: String): String = { val partitionValues = dir.map(dir => parsePartitions(dir, taskContext)) .getOrElse(Map.empty[String, String]) val filename = getFileName(taskContext, ext, partitionValues) val relativePath = randomPrefixLength.map { prefixLength => DeltaUtils.getRandomPrefix(prefixLength) // Generate a random prefix as a first choice }.orElse { dir // or else write into the partition directory if it is partitioned }.map { subDir => // Do some surgery on the paths we write out to eliminate the CDC_PARTITION_COL. Non-CDC // data is written to the base location, while CDC data is written to a special folder // _change_data. // The code here gets a bit complicated to accommodate two corner cases: an empty subdir // can't be passed to new Path() at all, and a single-level subdir won't have a trailing // slash. if (subDir == cdcPartitionFalse) { new Path(filename) } else if (subDir.startsWith(cdcPartitionTrue)) { val cleanedSubDir = cdcPartitionTrueRegex.replaceFirstIn(subDir, CDC_LOCATION) new Path(cleanedSubDir, filename) } else if (subDir.startsWith(cdcPartitionFalse)) { // We need to remove the trailing slash in addition to the directory - otherwise // it'll be interpreted as an absolute path and fail. val cleanedSubDir = subDir.stripPrefix(cdcPartitionFalse + "/") new Path(cleanedSubDir, filename) } else { new Path(subDir, filename) } }.getOrElse(new Path(filename)) // or directly write out to the output path val relativePathWithSubdir = subdir.map(new Path(_, relativePath)).getOrElse(relativePath) addedFiles.append((partitionValues, relativePathWithSubdir.toUri.toString)) new Path(path, relativePathWithSubdir).toString } override def newTaskTempFileAbsPath( taskContext: TaskAttemptContext, absoluteDir: String, ext: String): String = { throw DeltaErrors.unsupportedAbsPathAddFile(s"$this") } protected def buildActionFromAddedFile( f: (Map[String, String], String), stat: FileStatus, taskContext: TaskAttemptContext): FileAction = { // The partitioning in the Delta log action will be read back as part of the data, so our // virtual CDC_PARTITION_COL needs to be stripped out. val partitioning = f._1.filter { case (k, v) => k != CDC_PARTITION_COL } f._1.get(CDC_PARTITION_COL) match { case Some("true") => val partitioning = f._1.filter { case (k, v) => k != CDC_PARTITION_COL } AddCDCFile(f._2, partitioning, stat.getLen) case _ => val addFile = AddFile(f._2, partitioning, stat.getLen, stat.getModificationTime, true) addFile } } override def commitTask(taskContext: TaskAttemptContext): TaskCommitMessage = { if (addedFiles.nonEmpty) { val fs = new Path(path, addedFiles.head._2).getFileSystem(taskContext.getConfiguration) val statuses: Seq[FileAction] = addedFiles.map { f => // scalastyle:off pathfromuri val filePath = new Path(path, new Path(new URI(f._2))) // scalastyle:on pathfromuri val stat = fs.getFileStatus(filePath) buildActionFromAddedFile(f, stat, taskContext) }.toSeq new TaskCommitMessage(statuses) } else { new TaskCommitMessage(Nil) } } override def abortTask(taskContext: TaskAttemptContext): Unit = { // TODO: we can also try delete the addedFiles as a best-effort cleanup. } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/files/DeltaFileFormatWriter.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.files import java.util.{Date, UUID} import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.DeltaOptions import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileAlreadyExistsException, Path} import org.apache.hadoop.mapreduce._ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl import org.apache.spark._ import org.apache.spark.internal.{Logging, MDC} import org.apache.spark.internal.io.{FileCommitProtocol, SparkHadoopWriterUtils} import org.apache.spark.shuffle.FetchFailedException import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.catalog.BucketSpec import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReferences import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils} import org.apache.spark.sql.connector.write.WriterCommitMessage import org.apache.spark.sql.errors.QueryExecutionErrors import org.apache.spark.sql.execution.{ProjectExec, SortExec, SparkPlan, SQLExecution, UnsafeExternalRowSorter} import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.execution.datasources.FileFormatWriter._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.DataType import org.apache.spark.util.{SerializableConfiguration, Utils} /** * A helper object for writing FileFormat data out to a location. * Logic is copied from FileFormatWriter from Spark 3.5 with added functionality to write partition * values to data files. Specifically L123-126, L132, and L140 where it adds option * WRITE_PARTITION_COLUMNS */ object DeltaFileFormatWriter extends Logging { /** * A variable used in tests to check whether the output ordering of the query matches the * required ordering of the write command. */ private var outputOrderingMatched: Boolean = false /** * A variable used in tests to check the final executed plan. */ private var executedPlan: Option[SparkPlan] = None // scalastyle:off argcount /** * Basic work flow of this command is: * 1. Driver side setup, including output committer initialization and data source specific * preparation work for the write job to be issued. * 2. Issues a write job consists of one or more executor side tasks, each of which writes all * rows within an RDD partition. * 3. If no exception is thrown in a task, commits that task, otherwise aborts that task; If any * exception is thrown during task commitment, also aborts that task. * 4. If all tasks are committed, commit the job, otherwise aborts the job; If any exception is * thrown during job commitment, also aborts the job. * 5. If the job is successfully committed, perform post-commit operations such as * processing statistics. * @return The set of all partition paths that were updated during this write job. */ def write( sparkSession: SparkSession, plan: SparkPlan, fileFormat: FileFormat, committer: FileCommitProtocol, outputSpec: OutputSpec, hadoopConf: Configuration, partitionColumns: Seq[Attribute], bucketSpec: Option[BucketSpec], statsTrackers: Seq[WriteJobStatsTracker], options: Map[String, String], numStaticPartitionCols: Int = 0): Set[String] = { require(partitionColumns.size >= numStaticPartitionCols) val job = Job.getInstance(hadoopConf) job.setOutputKeyClass(classOf[Void]) job.setOutputValueClass(classOf[InternalRow]) FileOutputFormat.setOutputPath(job, new Path(outputSpec.outputPath)) val partitionSet = AttributeSet(partitionColumns) // cleanup the internal metadata information of // the file source metadata attribute if any before write out val finalOutputSpec = outputSpec.copy( outputColumns = outputSpec.outputColumns .map(FileSourceMetadataAttribute.cleanupFileSourceMetadataInformation) ) val dataColumns = finalOutputSpec.outputColumns.filterNot(partitionSet.contains) val writerBucketSpec = V1WritesUtils.getWriterBucketSpec(bucketSpec, dataColumns, options) val sortColumns = V1WritesUtils.getBucketSortColumns(bucketSpec, dataColumns) val caseInsensitiveOptions = CaseInsensitiveMap(options) val dataSchema = dataColumns.toStructType DataSourceUtils.verifySchema(fileFormat, dataSchema) DataSourceUtils.checkFieldNames(fileFormat, dataSchema) // Note: prepareWrite has side effect. It sets "job". val outputDataColumns = if (caseInsensitiveOptions.get(DeltaOptions.WRITE_PARTITION_COLUMNS).contains("true")) { dataColumns ++ partitionColumns } else dataColumns val outputWriterFactory = fileFormat.prepareWrite( sparkSession, job, caseInsensitiveOptions, outputDataColumns.toStructType ) val description = new WriteJobDescription( uuid = UUID.randomUUID.toString, serializableHadoopConf = new SerializableConfiguration(job.getConfiguration), outputWriterFactory = outputWriterFactory, allColumns = finalOutputSpec.outputColumns, dataColumns = outputDataColumns, partitionColumns = partitionColumns, bucketSpec = writerBucketSpec, path = finalOutputSpec.outputPath, customPartitionLocations = finalOutputSpec.customPartitionLocations, maxRecordsPerFile = caseInsensitiveOptions .get("maxRecordsPerFile") .map(_.toLong) .getOrElse(sparkSession.sessionState.conf.maxRecordsPerFile), timeZoneId = caseInsensitiveOptions .get(DateTimeUtils.TIMEZONE_OPTION) .getOrElse(sparkSession.sessionState.conf.sessionLocalTimeZone), statsTrackers = statsTrackers ) // We should first sort by dynamic partition columns, then bucket id, and finally sorting // columns. val requiredOrdering = partitionColumns.drop(numStaticPartitionCols) ++ writerBucketSpec.map(_.bucketIdExpression) ++ sortColumns val writeFilesOpt = V1WritesUtils.getWriteFilesOpt(plan) // SPARK-40588: when planned writing is disabled and AQE is enabled, // plan contains an AdaptiveSparkPlanExec, which does not know // its final plan's ordering, so we have to materialize that plan first // it is fine to use plan further down as the final plan is cached in that plan def materializeAdaptiveSparkPlan(plan: SparkPlan): SparkPlan = plan match { case a: AdaptiveSparkPlanExec => a.finalPhysicalPlan case p: SparkPlan => p.withNewChildren(p.children.map(materializeAdaptiveSparkPlan)) } // the sort order doesn't matter val actualOrdering = writeFilesOpt .map(_.child) .getOrElse(materializeAdaptiveSparkPlan(plan)) .outputOrdering val orderingMatched = V1WritesUtils.isOrderingMatched(requiredOrdering, actualOrdering) SQLExecution.checkSQLExecutionId(sparkSession) // propagate the description UUID into the jobs, so that committers // get an ID guaranteed to be unique. job.getConfiguration.set("spark.sql.sources.writeJobUUID", description.uuid) // When `PLANNED_WRITE_ENABLED` is true, the optimizer rule V1Writes will add logical sort // operator based on the required ordering of the V1 write command. So the output // ordering of the physical plan should always match the required ordering. Here // we set the variable to verify this behavior in tests. // There are two cases where FileFormatWriter still needs to add physical sort: // 1) When the planned write config is disabled. // 2) When the concurrent writers are enabled (in this case the required ordering of a // V1 write command will be empty). if (DeltaUtils.isTesting) outputOrderingMatched = orderingMatched if (writeFilesOpt.isDefined) { // build `WriteFilesSpec` for `WriteFiles` val concurrentOutputWriterSpecFunc = (plan: SparkPlan) => { val sortPlan = createSortPlan(plan, requiredOrdering, outputSpec) createConcurrentOutputWriterSpec(sparkSession, sortPlan, sortColumns) } val writeSpec = WriteFilesSpec( description = description, committer = committer, concurrentOutputWriterSpecFunc = concurrentOutputWriterSpecFunc ) executeWrite(sparkSession, plan, writeSpec, job) } else { executeWrite( sparkSession, plan, job, description, committer, outputSpec, requiredOrdering, partitionColumns, sortColumns, orderingMatched ) } } // scalastyle:on argcount private def executeWrite( sparkSession: SparkSession, plan: SparkPlan, job: Job, description: WriteJobDescription, committer: FileCommitProtocol, outputSpec: OutputSpec, requiredOrdering: Seq[Expression], partitionColumns: Seq[Attribute], sortColumns: Seq[Attribute], orderingMatched: Boolean): Set[String] = { val projectList = V1WritesUtils.convertEmptyToNull(plan.output, partitionColumns) val empty2NullPlan = if (projectList.nonEmpty) ProjectExec(projectList, plan) else plan writeAndCommit(job, description, committer) { val (planToExecute, concurrentOutputWriterSpec) = if (orderingMatched) { (empty2NullPlan, None) } else { val sortPlan = createSortPlan(empty2NullPlan, requiredOrdering, outputSpec) val concurrentOutputWriterSpec = createConcurrentOutputWriterSpec(sparkSession, sortPlan, sortColumns) if (concurrentOutputWriterSpec.isDefined) { (empty2NullPlan, concurrentOutputWriterSpec) } else { (sortPlan, concurrentOutputWriterSpec) } } // In testing, this is the only way to get hold of the actually executed plan written to file if (DeltaUtils.isTesting) executedPlan = Some(planToExecute) val rdd = planToExecute.execute() // SPARK-23271 If we are attempting to write a zero partition rdd, create a dummy single // partition rdd to make sure we at least set up one write task to write the metadata. val rddWithNonEmptyPartitions = if (rdd.partitions.length == 0) { sparkSession.sparkContext.parallelize(Array.empty[InternalRow], 1) } else { rdd } val jobTrackerID = SparkHadoopWriterUtils.createJobTrackerID(new Date()) val ret = new Array[WriteTaskResult](rddWithNonEmptyPartitions.partitions.length) val partitionColumnToDataType = description.partitionColumns .map(attr => (attr.name, attr.dataType)).toMap sparkSession.sparkContext.runJob( rddWithNonEmptyPartitions, (taskContext: TaskContext, iter: Iterator[InternalRow]) => { executeTask( description = description, jobTrackerID = jobTrackerID, sparkStageId = taskContext.stageId(), sparkPartitionId = taskContext.partitionId(), sparkAttemptNumber = taskContext.taskAttemptId().toInt & Integer.MAX_VALUE, committer, iterator = iter, concurrentOutputWriterSpec = concurrentOutputWriterSpec, partitionColumnToDataType ) }, rddWithNonEmptyPartitions.partitions.indices, (index, res: WriteTaskResult) => { committer.onTaskCommit(res.commitMsg) ret(index) = res } ) ret } } private def writeAndCommit( job: Job, description: WriteJobDescription, committer: FileCommitProtocol)(f: => Array[WriteTaskResult]): Set[String] = { // This call shouldn't be put into the `try` block below because it only initializes and // prepares the job, any exception thrown from here shouldn't cause abortJob() to be called. committer.setupJob(job) try { val ret = f val commitMsgs = ret.map(_.commitMsg) logInfo(log"Start to commit write Job ${MDC(DeltaLogKeys.JOB_ID, description.uuid)}.") val (_, duration) = Utils.timeTakenMs { committer.commitJob(job, commitMsgs) } logInfo(log"Write Job ${MDC(DeltaLogKeys.JOB_ID, description.uuid)} committed. " + log"Elapsed time: ${MDC(DeltaLogKeys.DURATION, duration)} ms.") processStats(description.statsTrackers, ret.map(_.summary.stats), duration) logInfo(log"Finished processing stats for write job " + log"${MDC(DeltaLogKeys.JOB_ID, description.uuid)}.") // return a set of all the partition paths that were updated during this job ret.map(_.summary.updatedPartitions).reduceOption(_ ++ _).getOrElse(Set.empty) } catch { case cause: Throwable => logError(log"Aborting job ${MDC(DeltaLogKeys.JOB_ID, description.uuid)}", cause) committer.abortJob(job) throw cause } } /** * Write files using [[SparkPlan.executeWrite]] */ private def executeWrite( session: SparkSession, planForWrites: SparkPlan, writeFilesSpec: WriteFilesSpec, job: Job): Set[String] = { val committer = writeFilesSpec.committer val description = writeFilesSpec.description // In testing, this is the only way to get hold of the actually executed plan written to file if (DeltaUtils.isTesting) executedPlan = Some(planForWrites) writeAndCommit(job, description, committer) { val rdd = planForWrites.executeWrite(writeFilesSpec) val ret = new Array[WriteTaskResult](rdd.partitions.length) session.sparkContext.runJob( rdd, (context: TaskContext, iter: Iterator[WriterCommitMessage]) => { assert(iter.hasNext) val commitMessage = iter.next() assert(!iter.hasNext) commitMessage }, rdd.partitions.indices, (index, res: WriterCommitMessage) => { assert(res.isInstanceOf[WriteTaskResult]) val writeTaskResult = res.asInstanceOf[WriteTaskResult] committer.onTaskCommit(writeTaskResult.commitMsg) ret(index) = writeTaskResult } ) ret } } private def createSortPlan( plan: SparkPlan, requiredOrdering: Seq[Expression], outputSpec: OutputSpec): SortExec = { // SPARK-21165: the `requiredOrdering` is based on the attributes from analyzed plan, and // the physical plan may have different attribute ids due to optimizer removing some // aliases. Here we bind the expression ahead to avoid potential attribute ids mismatch. val orderingExpr = bindReferences(requiredOrdering.map(SortOrder(_, Ascending)), outputSpec.outputColumns) SortExec(orderingExpr, global = false, child = plan) } private def createConcurrentOutputWriterSpec( sparkSession: SparkSession, sortPlan: SortExec, sortColumns: Seq[Attribute]): Option[ConcurrentOutputWriterSpec] = { val maxWriters = sparkSession.sessionState.conf.maxConcurrentOutputFileWriters val concurrentWritersEnabled = maxWriters > 0 && sortColumns.isEmpty if (concurrentWritersEnabled) { Some(ConcurrentOutputWriterSpec(maxWriters, () => sortPlan.createSorter())) } else { None } } class PartitionedTaskAttemptContextImpl( conf: Configuration, taskId: TaskAttemptID, partitionColumnToDataType: Map[String, DataType]) extends TaskAttemptContextImpl(conf, taskId) { val partitionColToDataType: Map[String, DataType] = partitionColumnToDataType } /** Writes data out in a single Spark task. */ private def executeTask( description: WriteJobDescription, jobTrackerID: String, sparkStageId: Int, sparkPartitionId: Int, sparkAttemptNumber: Int, committer: FileCommitProtocol, iterator: Iterator[InternalRow], concurrentOutputWriterSpec: Option[ConcurrentOutputWriterSpec], partitionColumnToDataType: Map[String, DataType]): WriteTaskResult = { val jobId = SparkHadoopWriterUtils.createJobID(jobTrackerID, sparkStageId) val taskId = new TaskID(jobId, TaskType.MAP, sparkPartitionId) val taskAttemptId = new TaskAttemptID(taskId, sparkAttemptNumber) // Set up the attempt context required to use in the output committer. val taskAttemptContext: TaskAttemptContext = { // Set up the configuration object val hadoopConf = description.serializableHadoopConf.value hadoopConf.set("mapreduce.job.id", jobId.toString) hadoopConf.set("mapreduce.task.id", taskAttemptId.getTaskID.toString) hadoopConf.set("mapreduce.task.attempt.id", taskAttemptId.toString) hadoopConf.setBoolean("mapreduce.task.ismap", true) hadoopConf.setInt("mapreduce.task.partition", 0) if (partitionColumnToDataType.isEmpty) { new TaskAttemptContextImpl(hadoopConf, taskAttemptId) } else { new PartitionedTaskAttemptContextImpl(hadoopConf, taskAttemptId, partitionColumnToDataType) } } committer.setupTask(taskAttemptContext) val dataWriter = if (sparkPartitionId != 0 && !iterator.hasNext) { // In case of empty job, leave first partition to save meta for file format like parquet. new EmptyDirectoryDataWriter(description, taskAttemptContext, committer) } else if (description.partitionColumns.isEmpty && description.bucketSpec.isEmpty) { new SingleDirectoryDataWriter(description, taskAttemptContext, committer) } else { concurrentOutputWriterSpec match { case Some(spec) => new DynamicPartitionDataConcurrentWriter( description, taskAttemptContext, committer, spec ) case _ => new DynamicPartitionDataSingleWriter(description, taskAttemptContext, committer) } } try { Utils.tryWithSafeFinallyAndFailureCallbacks(block = { // Execute the task to write rows out and commit the task. dataWriter.writeWithIterator(iterator) dataWriter.commit() })(catchBlock = { // If there is an error, abort the task dataWriter.abort() logError(log"Job ${MDC(DeltaLogKeys.JOB_ID, jobId)} aborted.") }, finallyBlock = { dataWriter.close() }) } catch { case e: FetchFailedException => throw e case f: FileAlreadyExistsException if SQLConf.get.fastFailFileFormatOutput => // If any output file to write already exists, it does not make sense to re-run this task. // We throw the exception and let Executor throw ExceptionFailure to abort the job. throw new TaskOutputFileAlreadyExistException(f) case t: Throwable => throw QueryExecutionErrors.taskFailedWhileWritingRowsError(description.path, t) } } /** * For every registered [[WriteJobStatsTracker]], call `processStats()` on it, passing it * the corresponding [[WriteTaskStats]] from all executors. */ private def processStats( statsTrackers: Seq[WriteJobStatsTracker], statsPerTask: Seq[Seq[WriteTaskStats]], jobCommitDuration: Long): Unit = { val numStatsTrackers = statsTrackers.length assert( statsPerTask.forall(_.length == numStatsTrackers), s"""Every WriteTask should have produced one `WriteTaskStats` object for every tracker. |There are $numStatsTrackers statsTrackers, but some task returned |${statsPerTask.find(_.length != numStatsTrackers).get.length} results instead. """.stripMargin ) val statsPerTracker = if (statsPerTask.nonEmpty) { statsPerTask.transpose } else { statsTrackers.map(_ => Seq.empty) } statsTrackers.zip(statsPerTracker).foreach { case (statsTracker, stats) => statsTracker.processStats(stats, jobCommitDuration) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/files/DeltaSourceSnapshot.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.files import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.{DeltaLog, DeltaTableUtils, Snapshot} import org.apache.spark.sql.delta.actions.SingleAction import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.sources.IndexedFile import org.apache.spark.sql.delta.stats.DataSkippingReader import org.apache.spark.sql.delta.util.StateCache import org.apache.spark.internal.MDC import org.apache.spark.sql.{Dataset, SparkSession} import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.functions._ /** * Converts a `Snapshot` into the initial set of files read when starting a new streaming query. * The list of files that represent the table at the time the query starts are selected by: * - Adding `version` and `index` to each file to enable splitting of the initial state into * multiple batches. * - Filtering files that don't match partition predicates, while preserving the aforementioned * indexing. */ class DeltaSourceSnapshot( val spark: SparkSession, val snapshot: Snapshot, val filters: Seq[Expression]) extends StateCache { protected val version = snapshot.version protected val path = snapshot.path protected lazy val (partitionFilters, dataFilters) = { val partitionCols = snapshot.metadata.partitionColumns val (part, data) = filters.partition { e => DeltaTableUtils.isPredicatePartitionColumnsOnly(e, partitionCols, spark) } logInfo(log"Classified filters: partition: ${MDC(DeltaLogKeys.PARTITION_FILTER, part)}, " + log"data: ${MDC(DeltaLogKeys.DATA_FILTER, data)}") (part, data) } private[delta] def filteredFiles: Dataset[IndexedFile] = { import spark.implicits.rddToDatasetHolder import org.apache.spark.sql.delta.implicits._ val initialFiles = snapshot.allFiles // This allows us to control the number of partitions created from the sort instead of // using the shufflePartitions setting .repartitionByRange(snapshot.getNumPartitions, col("modificationTime"), col("path")) .sort("modificationTime", "path") .rdd.zipWithIndex() .toDF("add", "index") // Stats aren't used for streaming reads right now, so decrease // the size of the files by nulling out the stats if they exist .withColumn("add", col("add").withField("stats", DataSkippingReader.nullStringLiteral)) .withColumn("remove", SingleAction.nullLitForRemoveFile) .withColumn("cdc", SingleAction.nullLitForAddCDCFile) .withColumn("version", lit(version)) .withColumn("isLast", lit(false)) .withColumn("shouldSkip", lit(false)) DeltaLog.filterFileList( snapshot.metadata.partitionSchema, initialFiles, partitionFilters, Seq("add")).as[IndexedFile] } private lazy val cachedState = { cacheDS(filteredFiles, s"Delta Source Snapshot #$version - ${snapshot.redactedPath}") } def iterator(): Iterator[IndexedFile] = { cachedState.getDS.toLocalIterator().asScala } def close(unpersistSnapshot: Boolean): Unit = { uncache() if (unpersistSnapshot) { snapshot.uncache() } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/files/SQLMetricsReporting.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.files import org.apache.spark.sql.delta.DeltaOperations.Operation import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.SparkSession import org.apache.spark.sql.execution.metric.SQLMetric /** * This trait is used to register SQL metrics for a Delta Operation. * Registering will allow the metrics to be instrumented via the CommitInfo and is accessible via * DescribeHistory */ trait SQLMetricsReporting { // Map of SQL Metrics private var operationSQLMetrics = Map[String, SQLMetric]() /** * Register SQL metrics for an operation by appending the supplied metrics map to the * operationSQLMetrics map. */ def registerSQLMetrics(spark: SparkSession, metrics: Map[String, SQLMetric]): Unit = { if (spark.conf.get(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED)) { operationSQLMetrics = operationSQLMetrics ++ metrics } } /** * Get the metrics for an operation based on collected SQL Metrics and filtering out * the ones based on the metric parameters for that operation. */ def getMetricsForOperation(operation: Operation): Map[String, String] = { operation.transformMetrics(operationSQLMetrics) } /** Returns the metric with `name` registered for the given transaction if it exists. */ def getMetric(name: String): Option[SQLMetric] = { operationSQLMetrics.get(name) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/files/TahoeChangeFileIndex.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.files import java.text.SimpleDateFormat import org.apache.spark.sql.delta.{DeltaLog, Snapshot, SnapshotDescriptor} import org.apache.spark.sql.delta.actions.{AddCDCFile, AddFile} import org.apache.spark.sql.delta.commands.cdc.CDCReader.{CDC_COMMIT_TIMESTAMP, CDC_COMMIT_VERSION, CDCDataSpec} import org.apache.spark.sql.delta.implicits._ import org.apache.hadoop.fs.Path import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.types.{LongType, StructType, TimestampType} /** * A [[TahoeFileIndex]] for scanning a sequence of CDC files. Similar to [[TahoeBatchFileIndex]], * the equivalent for reading [[AddFile]] actions. * * Note: Please also consider other CDC-related file indexes like [[CdcAddFileIndex]] * and [[TahoeRemoveFileIndex]] when modifying this file index. */ class TahoeChangeFileIndex( spark: SparkSession, val filesByVersion: Seq[CDCDataSpec[AddCDCFile]], deltaLog: DeltaLog, path: Path, snapshot: SnapshotDescriptor) extends TahoeFileIndexWithSnapshotDescriptor(spark, deltaLog, path, snapshot) { override def matchingFiles( partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): Seq[AddFile] = { // Make some fake AddFiles to satisfy the interface. val addFiles = filesByVersion.flatMap { case CDCDataSpec(version, ts, files, ci) => files.map { f => // We add the metadata as faked partition columns in order to attach it on a per-file // basis. val tsOpt = Option(ts) .map(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS Z").format(_)).orNull val newPartitionVals = f.partitionValues + (CDC_COMMIT_VERSION -> version.toString) + (CDC_COMMIT_TIMESTAMP -> tsOpt) AddFile(f.path, newPartitionVals, f.size, 0, dataChange = false, tags = f.tags) } } DeltaLog.filterFileList(partitionSchema, addFiles.toDF(spark), partitionFilters) .as[AddFile] .collect() } override def inputFiles: Array[String] = { filesByVersion.flatMap(_.actions).map(f => absolutePath(f.path).toString).toArray } override val partitionSchema: StructType = super.partitionSchema .add(CDC_COMMIT_VERSION, LongType) .add(CDC_COMMIT_TIMESTAMP, TimestampType) override def refresh(): Unit = {} override val sizeInBytes: Long = filesByVersion.flatMap(_.actions).map(_.size).sum } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/files/TahoeFileIndex.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.files // scalastyle:off import.ordering.noEmptyLine import java.net.URI import java.util.Objects import scala.collection.mutable import org.apache.spark.sql.delta.RowIndexFilterType import org.apache.spark.sql.delta.{DeltaColumnMapping, DeltaErrors, DeltaLog, DeltaParquetFileFormat, Snapshot, SnapshotDescriptor} import org.apache.spark.sql.delta.DefaultRowCommitVersion import org.apache.spark.sql.delta.RowId import org.apache.spark.sql.delta.actions.{AddFile, Metadata, Protocol} import org.apache.spark.sql.delta.implicits._ import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.hadoop.fs.FileStatus import org.apache.hadoop.fs.Path import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.{Cast, Expression, GenericInternalRow, Literal} import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.types.StructType /** * Similar to [[FileListingResult]], but maintains the partitions as [[AddFile]]. */ case class DeltaFileListingResult( partitions: Seq[(InternalRow, Seq[AddFile])], addFiles: Seq[AddFile], sortTime: Long = 0L) /** * A [[FileIndex]] that generates the list of files managed by the Tahoe protocol. */ abstract class TahoeFileIndex( val spark: SparkSession, override val deltaLog: DeltaLog, val path: Path) extends FileIndex with SupportsRowIndexFilters with SnapshotDescriptor { override def rootPaths: Seq[Path] = path :: Nil /** * Returns all matching/valid files by the given `partitionFilters` and `dataFilters`. * Implementations may avoid evaluating data filters when doing so would be expensive, but * *must* evaluate the partition filters; wrong results will be produced if AddFile entries * which don't match the partition filters are returned. */ def matchingFiles( partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): Seq[AddFile] /** * Utility method to convert a sequence of partition values (represented as a Map) and AddFiles * to a sequence of (partitionValuesRow, files) tuples. The partitionValuesRow is a * [[InternalRow]] representing the partition values. The files are represented as a * sequence of [[AddFile]]. */ private def convertPartitionsToInternalRow( partitions: Seq[(Map[String, String], Seq[AddFile])]): Seq[(InternalRow, Seq[AddFile])] = { partitions.map { case (partitionValues, addFiles) => (getPartitionValuesRow(partitionValues), addFiles) } } /** * Returns (i) tuples of partition directories to their respective AddFile actions and * (ii) a collection of matched AddFiles. The matched AddFiles are those that meet the criteria * set by the partition and data filters. Essentially, this is a collection of all the files * associated with the identified partitions. */ def listPartitionsAsAddFiles( partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): (Seq[(InternalRow, Seq[AddFile])], Seq[AddFile]) = { val matchedFiles = matchingFiles(partitionFilters, dataFilters) val partitionValuesToFiles = matchedFiles.groupBy(_.partitionValues) (convertPartitionsToInternalRow(partitionValuesToFiles.toSeq), matchedFiles) } override def listFiles( partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): Seq[PartitionDirectory] = { val partitionValuesToFiles = listAddFiles(partitionFilters, dataFilters) makePartitionDirectories(convertPartitionsToInternalRow(partitionValuesToFiles.toSeq)) } private def listAddFiles( partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): Map[Map[String, String], Seq[AddFile]] = { matchingFiles(partitionFilters, dataFilters).groupBy(_.partitionValues) } /** * Generates a FileStatusWithMetadata using data extracted from a given AddFile. */ def fileStatusWithMetadataFromAddFile(addFile: AddFile): FileStatusWithMetadata = { val fs = new FileStatus( /* length */ addFile.size, /* isDir */ false, /* blockReplication */ 0, /* blockSize */ 1, /* modificationTime */ addFile.modificationTime, /* path */ absolutePath(addFile.path)) val metadata = mutable.Map.empty[String, Any] addFile.baseRowId.foreach(baseRowId => metadata.put(RowId.BASE_ROW_ID, baseRowId)) addFile.defaultRowCommitVersion.foreach(defaultRowCommitVersion => metadata.put(DefaultRowCommitVersion.METADATA_STRUCT_FIELD_NAME, defaultRowCommitVersion)) if (addFile.deletionVector != null) { metadata.put(DeltaParquetFileFormat.FILE_ROW_INDEX_FILTER_ID_ENCODED, addFile.deletionVector.serializeToBase64()) // Set the filter type to IF_CONTAINED by default to let [[DeltaParquetFileFormat]] filter // out rows unless a filter type was explicitly provided in rowIndexFilters. This can happen // e.g. when reading CDC data to keep deleted rows instead of filtering them out. val filterType = rowIndexFilters.getOrElse(Map.empty) .getOrElse(addFile.path, RowIndexFilterType.IF_CONTAINED) metadata.put(DeltaParquetFileFormat.FILE_ROW_INDEX_FILTER_TYPE, filterType) } FileStatusWithMetadata(fs, metadata.toMap) } def makePartitionDirectories( partitionValuesToFiles: Seq[(InternalRow, Seq[AddFile])]): Seq[PartitionDirectory] = { val timeZone = spark.sessionState.conf.sessionLocalTimeZone partitionValuesToFiles.map { case (partitionValues, files) => val fileStatuses = files.map(f => fileStatusWithMetadataFromAddFile(f)).toArray PartitionDirectory(partitionValues, fileStatuses) } } protected def getPartitionValuesRow(partitionValues: Map[String, String]): GenericInternalRow = { val timeZone = spark.sessionState.conf.sessionLocalTimeZone val partitionRowValues = partitionSchema.map { p => val colName = DeltaColumnMapping.getPhysicalName(p) val partValue = Literal(partitionValues.get(colName).orNull) Cast(partValue, p.dataType, Option(timeZone), ansiEnabled = false).eval() }.toArray new GenericInternalRow(partitionRowValues) } override def partitionSchema: StructType = metadata.partitionSchema def absolutePath(child: String): Path = { // scalastyle:off pathfromuri val p = new Path(new URI(child)) // scalastyle:on pathfromuri if (p.isAbsolute) { p } else { new Path(path, p) } } override def toString: String = { // the rightmost 100 characters of the path val truncatedPath = truncateRight(path.toString, len = 100) s"Delta[version=$version, $truncatedPath]" } /** * Gets the rightmost {@code len} characters of a String. * * @return the trimmed and formatted string. */ private def truncateRight(input: String, len: Int): String = { if (input.length > len) { "... " + input.takeRight(len) } else { input } } /** * Returns the path of the base directory of the given file path (i.e. its parent directory with * all the partition directories stripped off). */ def getBasePath(filePath: Path): Option[Path] = Some(path) } /** A [[TahoeFileIndex]] that works with a specific [[SnapshotDescriptor]]. */ abstract class TahoeFileIndexWithSnapshotDescriptor( spark: SparkSession, deltaLog: DeltaLog, path: Path, snapshot: SnapshotDescriptor) extends TahoeFileIndex(spark, deltaLog, path) { override def version: Long = snapshot.version override def metadata: Metadata = snapshot.metadata override def protocol: Protocol = snapshot.protocol protected[delta] def numOfFilesIfKnown: Option[Long] = snapshot.numOfFilesIfKnown protected[delta] def sizeInBytesIfKnown: Option[Long] = snapshot.sizeInBytesIfKnown } /** * A lightweight [[SnapshotDescriptor]] implementation that points to an actual [[Snapshot]]. * * @param snapshot the [[Snapshot]] this pointer points to */ class ShallowSnapshotDescriptor( snapshot: Snapshot, catalogTableOpt: Option[CatalogTable]) extends SnapshotDescriptor { override val deltaLog: DeltaLog = snapshot.deltaLog override val version: Long = snapshot.version override val metadata: Metadata = snapshot.metadata override val protocol: Protocol = snapshot.protocol // Avoid eager state reconstruction override protected[delta] def numOfFilesIfKnown: Option[Long] = deltaLog.getSnapshotAt(version, catalogTableOpt = catalogTableOpt).numOfFilesIfKnown override protected[delta] def sizeInBytesIfKnown: Option[Long] = deltaLog.getSnapshotAt(version, catalogTableOpt = catalogTableOpt).sizeInBytesIfKnown } /** * A [[TahoeFileIndex]] that generates the list of files from DeltaLog with given partition filters. * * NOTE: This is NOT a [[TahoeFileIndexWithSnapshotDescriptor]] because we only use * [[snapshotAtAnalysis]] for actual data skipping if this is a time travel query. */ case class TahoeLogFileIndex( override val spark: SparkSession, override val deltaLog: DeltaLog, override val path: Path, snapshotAtAnalysis: SnapshotDescriptor, catalogTableOpt: Option[CatalogTable], partitionFilters: Seq[Expression], isTimeTravelQuery: Boolean) extends TahoeFileIndex(spark, deltaLog, path) { def this( spark: SparkSession, deltaLog: DeltaLog, path: Path, snapshotAtAnalysis: Snapshot, catalogTableOpt: Option[CatalogTable], partitionFilters: Seq[Expression] = Nil, isTimeTravelQuery: Boolean = false ) = this ( spark, deltaLog, path, if (isTimeTravelQuery) snapshotAtAnalysis else new ShallowSnapshotDescriptor(snapshotAtAnalysis, catalogTableOpt), catalogTableOpt, partitionFilters, isTimeTravelQuery) require(!isTimeTravelQuery || snapshotAtAnalysis.isInstanceOf[Snapshot]) // WARNING: Stability of this method is _NOT_ guaranteed! override def version: Long = { if (isTimeTravelQuery) snapshotAtAnalysis.version else deltaLog.unsafeVolatileSnapshot.version } // WARNING: These methods are intentionally pinned to the analysis-time snapshot, which may differ // from the one returned by [[getSnapshot]] that we will eventually scan. override def metadata: Metadata = snapshotAtAnalysis.metadata override def protocol: Protocol = snapshotAtAnalysis.protocol private def checkSchemaOnRead: Boolean = { spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_SCHEMA_ON_READ_CHECK_ENABLED) } private def includeTableIdInComparisons: Boolean = spark.conf.get(DeltaSQLConf.DELTA_INCLUDE_TABLE_ID_IN_FILE_INDEX_COMPARISON) protected def getSnapshotToScan: Snapshot = { if (isTimeTravelQuery) { snapshotAtAnalysis.asInstanceOf[Snapshot] } else { deltaLog.update(stalenessAcceptable = true, catalogTableOpt = catalogTableOpt) } } /** Provides the version that's being used as part of the scan if this is a time travel query. */ def versionToUse: Option[Long] = if (isTimeTravelQuery) Some(snapshotAtAnalysis.version) else None def getSnapshot: Snapshot = { val snapshotToScan = getSnapshotToScan // Always check read compatibility with column mapping tables if (checkSchemaOnRead) { // Ensure that the schema hasn't changed in an incompatible manner since analysis time: // 1. Check logical schema incompatibility // 2. Check column mapping read compatibility. The above check is not sufficient // when the schema's logical names are not changing but the underlying physical name has // changed. In this case, the data files cannot be read using the old schema any more. val snapshotSchema = snapshotToScan.metadata.schema if (!SchemaUtils.isReadCompatible(snapshotAtAnalysis.schema, snapshotSchema) || !DeltaColumnMapping.hasNoColumnMappingSchemaChanges( snapshotToScan.metadata, snapshotAtAnalysis.metadata)) { throw DeltaErrors.schemaChangedSinceAnalysis(snapshotAtAnalysis.schema, snapshotSchema) } } // disallow reading table with empty schema, which we support creating now if (snapshotToScan.schema.isEmpty) { // print the catalog identifier or delta.`/path/to/table` var message = TableIdentifier(deltaLog.dataPath.toString, Some("delta")).quotedString throw DeltaErrors.readTableWithoutSchemaException(message) } snapshotToScan } override def matchingFiles( partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): Seq[AddFile] = { getSnapshot.filesForScan(this.partitionFilters ++ partitionFilters ++ dataFilters).files } override def inputFiles: Array[String] = { getSnapshot .filesForScan(partitionFilters).files .map(f => absolutePath(f.path).toString) .toArray } override def refresh(): Unit = {} override def sizeInBytes: Long = deltaLog.unsafeVolatileSnapshot.sizeInBytes override def equals(that: Any): Boolean = that match { case t: TahoeLogFileIndex => t.path == path && (if (includeTableIdInComparisons) { t.deltaLog.isSameLogAs(deltaLog) } else { t.deltaLog.dataPath == deltaLog.dataPath }) && t.versionToUse == versionToUse && t.partitionFilters == partitionFilters case _ => false } override def hashCode: scala.Int = { if (includeTableIdInComparisons) { Objects.hashCode(path, deltaLog.compositeId, versionToUse, partitionFilters) } else { Objects.hashCode(path, deltaLog.dataPath, versionToUse, partitionFilters) } } protected[delta] def numOfFilesIfKnown: Option[Long] = deltaLog.unsafeVolatileSnapshot.numOfFilesIfKnown protected[delta] def sizeInBytesIfKnown: Option[Long] = deltaLog.unsafeVolatileSnapshot.sizeInBytesIfKnown } object TahoeLogFileIndex { def apply( spark: SparkSession, deltaLog: DeltaLog, catalogTableOpt: Option[CatalogTable]): TahoeLogFileIndex = new TahoeLogFileIndex( spark, deltaLog, deltaLog.dataPath, deltaLog.unsafeVolatileSnapshot, catalogTableOpt) def apply( spark: SparkSession, deltaLog: DeltaLog, path: Path, snapshotAtAnalysis: Snapshot, catalogTableOpt: Option[CatalogTable], partitionFilters: Seq[Expression] = Nil, isTimeTravelQuery: Boolean = false): TahoeLogFileIndex = new TahoeLogFileIndex( spark, deltaLog, path, snapshotAtAnalysis, catalogTableOpt, partitionFilters, isTimeTravelQuery) } /** * A [[TahoeFileIndex]] that generates the list of files from a given list of files * that are within a version range of DeltaLog. */ class TahoeBatchFileIndex( spark: SparkSession, val actionType: String, val addFiles: Seq[AddFile], deltaLog: DeltaLog, path: Path, val snapshot: SnapshotDescriptor, val partitionFiltersGenerated: Boolean = false) extends TahoeFileIndexWithSnapshotDescriptor(spark, deltaLog, path, snapshot) { override def matchingFiles( partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): Seq[AddFile] = { DeltaLog.filterFileList(partitionSchema, addFiles.toDF(spark), partitionFilters) .as[AddFile] .collect() } override def inputFiles: Array[String] = { addFiles.map(a => absolutePath(a.path).toString).toArray } override def refresh(): Unit = {} override lazy val sizeInBytes: Long = addFiles.map(_.size).sum } trait SupportsRowIndexFilters { /** * If we know a-priori which exact rows we want to read (e.g., from a previous scan) * find the per-file filter here, which must be passed down to the appropriate reader. * * @return a mapping from file names to the row index filter for that file. */ def rowIndexFilters: Option[Map[String, RowIndexFilterType]] = None } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/files/TahoeRemoveFileIndex.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.files import java.text.SimpleDateFormat import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.{AddFile, RemoveFile} import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.commands.cdc.CDCReader._ import org.apache.spark.sql.delta.implicits._ import org.apache.hadoop.fs.Path import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.types.StructType /** * A [[TahoeFileIndex]] for scanning a sequence of removed files as CDC. Similar to * [[TahoeBatchFileIndex]], the equivalent for reading [[AddFile]] actions. * @param spark The Spark session. * @param filesByVersion Grouped FileActions, one per table version. * @param deltaLog The delta log instance. * @param path The table's data path. * @param snapshot The snapshot where we read CDC from. * @param rowIndexFilters Map from URI-encoded file path to a row index filter type. * * Note: Please also consider other CDC-related file indexes like [[CdcAddFileIndex]] * and [[TahoeChangeFileIndex]] when modifying this file index. */ class TahoeRemoveFileIndex( spark: SparkSession, val filesByVersion: Seq[CDCDataSpec[RemoveFile]], deltaLog: DeltaLog, path: Path, snapshot: SnapshotDescriptor, override val rowIndexFilters: Option[Map[String, RowIndexFilterType]] = None ) extends TahoeFileIndexWithSnapshotDescriptor(spark, deltaLog, path, snapshot) { override def matchingFiles( partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): Seq[AddFile] = { // Make some fake AddFiles to satisfy the interface. val addFiles = filesByVersion.flatMap { case CDCDataSpec(version, ts, files, ci) => files.map { r => if (!r.extendedFileMetadata.getOrElse(false)) { // This shouldn't happen in user queries - the CDC flag was added at the same time as // extended metadata, so all removes in a table with CDC enabled should have it. (The // only exception is FSCK removes, which we screen out separately because they have // dataChange set to false.) throw DeltaErrors.removeFileCDCMissingExtendedMetadata(r.toString) } // We add the metadata as faked partition columns in order to attach it on a per-file // basis. val tsOpt = Option(ts) .map(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS Z").format(_)).orNull val newPartitionVals = r.partitionValues + (CDC_COMMIT_VERSION -> version.toString) + (CDC_COMMIT_TIMESTAMP -> tsOpt) + (CDC_TYPE_COLUMN_NAME -> CDC_TYPE_DELETE_STRING) AddFile( path = r.path, partitionValues = newPartitionVals, size = r.size.getOrElse(0L), modificationTime = 0, dataChange = r.dataChange, tags = r.tags, deletionVector = r.deletionVector, baseRowId = r.baseRowId ) } } DeltaLog.filterFileList(partitionSchema, addFiles.toDF(spark), partitionFilters) .as[AddFile] .collect() } override def inputFiles: Array[String] = { filesByVersion.flatMap(_.actions).map(f => absolutePath(f.path).toString).toArray } override def partitionSchema: StructType = CDCReader.cdcReadSchema(super.partitionSchema) override def refresh(): Unit = {} override val sizeInBytes: Long = filesByVersion.flatMap(_.actions).map(_.size.getOrElse(0L)).sum } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/files/TransactionalWrite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.files import scala.collection.mutable.ListBuffer import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.constraints.{Constraint, Constraints, DeltaInvariantCheckerExec} import org.apache.spark.sql.delta.hooks.AutoCompact import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.perf.DeltaOptimizedWriterExec import org.apache.spark.sql.delta.schema._ import org.apache.spark.sql.delta.shims.VariantShreddingShims import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.sources.DeltaSQLConf.DELTA_COLLECT_STATS_USING_TABLE_SCHEMA import org.apache.spark.sql.delta.stats.{ DeltaJobStatisticsTracker, StatisticsCollection, StatsCollectionUtils } import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.hadoop.fs.Path import org.apache.spark.sql.{DataFrame, Dataset, SparkSession} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans.logical.LocalRelation import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes import org.apache.spark.sql.connector.catalog._ import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.datasources.{BasicWriteJobStatsTracker, FileFormatWriter, WriteJobStatsTracker} import org.apache.spark.sql.functions.{col, to_json} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{StringType, StructField, StructType} import org.apache.spark.util.SerializableConfiguration /** * Adds the ability to write files out as part of a transaction. Checks * are performed to ensure that the data being written matches either the * current metadata or the new metadata being set by this transaction. */ trait TransactionalWrite extends DeltaLogging { self: OptimisticTransactionImpl => protected var hasWritten = false private[delta] val deltaDataSubdir = if (spark.sessionState.conf.getConf(DeltaSQLConf.WRITE_DATA_FILES_TO_SUBDIR)) { Some("data") } else None // It's okay to make this a lazy val. Once this is read, the metadata will be marked as read // and can't be changed again within the transaction, otherwise it will throw an exception. private lazy val randomizeFilePrefixes = DeltaConfigs.RANDOMIZE_FILE_PREFIXES.fromMetaData(metadata) private lazy val randomPrefixLength = DeltaConfigs.RANDOM_PREFIX_LENGTH.fromMetaData(metadata) protected def getCommitter(outputPath: Path): DelayedCommitProtocol = { // We force the use of random prefixes in column mapping modes. // Note that here we need to use the txn metadata instead of the snapshot's metadata val prefixLengthOpt = if (randomizeFilePrefixes || metadata.columnMappingMode != NoMapping) { Some(randomPrefixLength) } else None new DelayedCommitProtocol("delta", outputPath.toString, prefixLengthOpt, deltaDataSubdir) } /** Makes the output attributes nullable, so that we don't write unreadable parquet files. */ protected def makeOutputNullable(output: Seq[Attribute]): Seq[Attribute] = { output.map { case ref: AttributeReference => val nullableDataType = SchemaUtils.typeAsNullable(ref.dataType) ref.copy(dataType = nullableDataType, nullable = true)(ref.exprId, ref.qualifier) case attr => attr.withNullability(true) } } /** Replace the output attributes with the physical mapping information. */ protected def mapColumnAttributes( output: Seq[Attribute], mappingMode: DeltaColumnMappingMode): Seq[Attribute] = { DeltaColumnMapping.createPhysicalAttributes(output, metadata.schema, mappingMode) } /** * Used to perform all required normalizations before writing out the data. * Returns the QueryExecution to execute. */ protected def normalizeData( deltaLog: DeltaLog, options: Option[DeltaOptions], data: DataFrame): (QueryExecution, Seq[Attribute], Seq[Constraint], Set[String]) = { val (normalizedSchema, output, constraints, trackHighWaterMarks) = normalizeSchema( deltaLog, options, data) (normalizedSchema.queryExecution, output, constraints, trackHighWaterMarks) } /** * Normalize the schema of the query, and returns the updated DataFrame. If the table has * generated columns and users provide these columns in the output, we will also return * constraints that should be respected. If any constraints are returned, the caller should apply * these constraints when writing data. * * Note: The schema of the DataFrame may not match the attributes we return as the * output schema. This is because streaming queries create `IncrementalExecution`, which cannot be * further modified. We can however have the Parquet writer use the physical plan from * `IncrementalExecution` and the output schema provided through the attributes. */ protected def normalizeSchema( deltaLog: DeltaLog, options: Option[DeltaOptions], data: DataFrame): (DataFrame, Seq[Attribute], Seq[Constraint], Set[String]) = { val normalizedData = SchemaUtils.normalizeColumnNames( deltaLog, metadata.schema, data ) // Validate that write columns for Row IDs have the correct name. RowId.throwIfMaterializedRowIdColumnNameIsInvalid( normalizedData, metadata, protocol, deltaLog.unsafeVolatileTableId) val nullAsDefault = options.isDefined && options.get.options.contains(ColumnWithDefaultExprUtils.USE_NULL_AS_DEFAULT_DELTA_OPTION) val enforcesDefaultExprs = ColumnWithDefaultExprUtils.tableHasDefaultExpr( protocol, metadata, nullAsDefault) val (dataWithDefaultExprs, generatedColumnConstraints, trackHighWaterMarks) = if (enforcesDefaultExprs) { ColumnWithDefaultExprUtils.addDefaultExprsOrReturnConstraints( deltaLog, protocol, // We need the original query execution if this is a streaming query, because // `normalizedData` may add a new projection and change its type. data.queryExecution, metadata.schema, normalizedData, nullAsDefault) } else { (normalizedData, Nil, Set[String]()) } val cleanedData = SchemaUtils.dropNullTypeColumns(dataWithDefaultExprs) val finalData = if (cleanedData.schema != dataWithDefaultExprs.schema) { // This must be batch execution as DeltaSink doesn't accept NullType in micro batch DataFrame. // For batch executions, we need to use the latest DataFrame query execution cleanedData } else if (enforcesDefaultExprs) { dataWithDefaultExprs } else { assert( normalizedData == dataWithDefaultExprs, "should not change data when there is no generate column") normalizedData } val nullableOutput = makeOutputNullable(cleanedData.queryExecution.analyzed.output) val columnMapping = metadata.columnMappingMode // Check partition column errors checkPartitionColumns( metadata.partitionSchema, nullableOutput, nullableOutput.length < data.schema.size ) // Rewrite column physical names if using a mapping mode val mappedOutput = if (columnMapping == NoMapping) nullableOutput else { mapColumnAttributes(nullableOutput, columnMapping) } (finalData, mappedOutput, generatedColumnConstraints, trackHighWaterMarks) } protected def checkPartitionColumns( partitionSchema: StructType, output: Seq[Attribute], colsDropped: Boolean): Unit = { val partitionColumns: Seq[Attribute] = partitionSchema.map { col => // schema is already normalized, therefore we can do an equality check output.find(f => f.name == col.name).getOrElse( throw DeltaErrors.partitionColumnNotFoundException(col.name, output) ) } if (partitionColumns.nonEmpty && partitionColumns.length == output.length) { throw DeltaErrors.nonPartitionColumnAbsentException(colsDropped) } } protected def getPartitioningColumns( partitionSchema: StructType, output: Seq[Attribute]): Seq[Attribute] = { val partitionColumns: Seq[Attribute] = partitionSchema.map { col => // schema is already normalized, therefore we can do an equality check // we have already checked for missing columns, so the fields must exist output.find(f => f.name == col.name).get } partitionColumns } /** * If there is any string partition column and there are constraints defined, add a projection to * convert empty string to null for that column. The empty strings will be converted to null * eventually even without this convert, but we want to do this earlier before check constraints * so that empty strings are correctly rejected. Note that this should not cause the downstream * logic in `FileFormatWriter` to add duplicate conversions because the logic there checks the * partition column using the original plan's output. When the plan is modified with additional * projections, the partition column check won't match and will not add more conversion. * * @param plan The original SparkPlan. * @param partCols The partition columns. * @param constraints The defined constraints. * @return A SparkPlan potentially modified with an additional projection on top of `plan` */ protected def convertEmptyToNullIfNeeded( plan: SparkPlan, partCols: Seq[Attribute], constraints: Seq[Constraint]): SparkPlan = { if (!spark.conf.get(DeltaSQLConf.CONVERT_EMPTY_TO_NULL_FOR_STRING_PARTITION_COL)) { return plan } // No need to convert if there are no constraints. The empty strings will be converted later by // FileFormatWriter and FileFormatDataWriter. Note that we might still do unnecessary convert // here as the constraints might not be related to the string partition columns. A precise // check will need to walk the constraints to see if such columns are really involved. It // doesn't seem to worth the effort. if (constraints.isEmpty) return plan val partSet = AttributeSet(partCols) var needConvert = false val projectList: Seq[NamedExpression] = plan.output.map { case p if partSet.contains(p) && p.dataType == StringType => needConvert = true Alias(org.apache.spark.sql.catalyst.expressions.Empty2Null(p), p.name)() case attr => attr } if (needConvert) ProjectExec(projectList, plan) else plan } def writeFiles( data: Dataset[_], additionalConstraints: Seq[Constraint]): Seq[FileAction] = { writeFiles(data, None, additionalConstraints) } def writeFiles( data: Dataset[_], writeOptions: Option[DeltaOptions]): Seq[FileAction] = { writeFiles(data, writeOptions, Nil) } def writeFiles(data: Dataset[_]): Seq[FileAction] = { writeFiles(data, Nil) } def writeFiles( data: Dataset[_], deltaOptions: Option[DeltaOptions], additionalConstraints: Seq[Constraint]): Seq[FileAction] = { writeFiles(data, deltaOptions, isOptimize = false, additionalConstraints) } /** * Returns a tuple of (data, partition schema). For CDC writes, a `__is_cdc` column is added to * the data and `__is_cdc=true/false` is added to the front of the partition schema. */ protected def performCDCPartition(inputData: Dataset[_]): (DataFrame, StructType) = { // If this is a CDC write, we need to generate the CDC_PARTITION_COL in order to properly // dispatch rows between the main table and CDC event records. This is a virtual partition // and will be stripped out later in [[DelayedCommitProtocolEdge]]. // Note that the ordering of the partition schema is relevant - CDC_PARTITION_COL must // come first in order to ensure CDC data lands in the right place. if (CDCReader.isCDCEnabledOnTable(metadata, spark) && inputData.schema.fieldNames.contains(CDCReader.CDC_TYPE_COLUMN_NAME)) { val augmentedData = inputData.withColumn( CDCReader.CDC_PARTITION_COL, col(CDCReader.CDC_TYPE_COLUMN_NAME).isNotNull) val partitionSchema = StructType( StructField(CDCReader.CDC_PARTITION_COL, StringType) +: metadata.physicalPartitionSchema) (augmentedData, partitionSchema) } else { (inputData.toDF(), metadata.physicalPartitionSchema) } } /** * Return a tuple of (outputStatsCollectionSchema, statsCollectionSchema). * outputStatsCollectionSchema is the data source schema from DataFrame used for stats collection. * It contains the columns in the DataFrame output, excluding the partition columns. * tableStatsCollectionSchema is the schema to collect stats for. It contains the columns in the * table schema, excluding the partition columns. * Note: We only collect NULL_COUNT stats (as the number of rows) for the columns in * statsCollectionSchema but missing in outputStatsCollectionSchema */ protected def getStatsSchema( dataFrameOutput: Seq[Attribute], partitionSchema: StructType): (Seq[Attribute], Seq[Attribute]) = { val partitionColNames = partitionSchema.map(_.name).toSet // The outputStatsCollectionSchema comes from DataFrame output // schema should be normalized, therefore we can do an equality check val outputStatsCollectionSchema = dataFrameOutput .filterNot(c => partitionColNames.contains(c.name)) // The tableStatsCollectionSchema comes from table schema val statsTableSchema = toAttributes(metadata.schema) val mappedStatsTableSchema = if (metadata.columnMappingMode == NoMapping) { statsTableSchema } else { mapColumnAttributes(statsTableSchema, metadata.columnMappingMode) } // It's important to first do the column mapping and then drop the partition columns val tableStatsCollectionSchema = mappedStatsTableSchema .filterNot(c => partitionColNames.contains(c.name)) (outputStatsCollectionSchema, tableStatsCollectionSchema) } /** * Returns a resolved `statsCollection.statsCollector` expression with `statsDataSchema` * attributes re-resolved to be used for writing Delta file stats. */ protected def getStatsColExpr( statsDataSchema: Seq[Attribute], statsCollection: StatisticsCollection): (Expression, Seq[Attribute]) = { val resolvedPlan = DataFrameUtils.ofRows(spark, LocalRelation(statsDataSchema)) .select(to_json(statsCollection.statsCollector)) .queryExecution.analyzed // We have to use the new attributes with regenerated attribute IDs, because the Analyzer // doesn't guarantee that attributes IDs will stay the same val newStatsDataSchema = resolvedPlan.children.head.output resolvedPlan.expressions.head -> newStatsDataSchema } /** Return the pair of optional stats tracker and stats collection class */ protected def getOptionalStatsTrackerAndStatsCollection( output: Seq[Attribute], outputPath: Path, partitionSchema: StructType, data: DataFrame): ( Option[DeltaJobStatisticsTracker], Option[StatisticsCollection]) = { // check whether we should collect Delta stats val collectStats = (spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_COLLECT_STATS) ) if (collectStats) { val (outputStatsCollectionSchema, tableStatsCollectionSchema) = getStatsSchema(output, partitionSchema) val statsCollection = new StatisticsCollection { override val columnMappingMode: DeltaColumnMappingMode = metadata.columnMappingMode override def tableSchema: StructType = metadata.schema override def outputTableStatsSchema: StructType = { // If collecting stats uses the table schema, then we pass in tableStatsCollectionSchema; // otherwise, pass in outputStatsCollectionSchema to collect stats using the DataFrame // schema. if (spark.sessionState.conf.getConf(DELTA_COLLECT_STATS_USING_TABLE_SCHEMA)) { tableStatsCollectionSchema.toStructType } else { outputStatsCollectionSchema.toStructType } } override def outputAttributeSchema: StructType = outputStatsCollectionSchema.toStructType override val spark: SparkSession = data.sparkSession override val statsColumnSpec = StatisticsCollection.configuredDeltaStatsColumnSpec(metadata) override val protocol: Protocol = newProtocol.getOrElse(snapshot.protocol) override def getDataSkippingStringPrefixLength: Int = StatsCollectionUtils.getDataSkippingStringPrefixLength(spark, metadata) } val (statsColExpr, newOutputStatsCollectionSchema) = getStatsColExpr(outputStatsCollectionSchema, statsCollection) (Some(new DeltaJobStatisticsTracker(deltaLog.newDeltaHadoopConf(), outputPath, newOutputStatsCollectionSchema, statsColExpr )), Some(statsCollection)) } else { (None, None) } } /** * Writes out the dataframe after performing schema validation. Returns a list of * actions to append these files to the reservoir. * * @param inputData Data to write out. * @param writeOptions Options to decide how to write out the data. * @param isOptimize Whether the operation writing this is Optimize or not. * @param additionalConstraints Additional constraints on the write. */ def writeFiles( inputData: Dataset[_], writeOptions: Option[DeltaOptions], isOptimize: Boolean, additionalConstraints: Seq[Constraint]): Seq[FileAction] = { hasWritten = true val spark = inputData.sparkSession val (data, partitionSchema) = performCDCPartition(inputData) val outputPath = deltaLog.dataPath val (queryExecution, output, generatedColumnConstraints, trackFromData) = normalizeData(deltaLog, writeOptions, data) // Use the track set from the transaction if set, // otherwise use the track set from `normalizeData()`. val trackIdentityHighWaterMarks = trackHighWaterMarks.getOrElse(trackFromData) val partitioningColumns = getPartitioningColumns(partitionSchema, output) val committer = getCommitter(outputPath) val (statsDataSchema, _) = getStatsSchema(output, partitionSchema) // If Statistics Collection is enabled, then create a stats tracker that will be injected during // the FileFormatWriter.write call below and will collect per-file stats using // StatisticsCollection val (optionalStatsTracker, _) = getOptionalStatsTrackerAndStatsCollection(output, outputPath, partitionSchema, data) val constraints = Constraints.getAll(metadata, spark) ++ generatedColumnConstraints ++ additionalConstraints Constraints.validateCheckConstraints(spark, constraints, deltaLog, metadata.schema) val identityTrackerOpt = IdentityColumn.createIdentityColumnStatsTracker( spark, deltaLog.newDeltaHadoopConf(), outputPath, metadata.schema, statsDataSchema, trackIdentityHighWaterMarks ) SQLExecution.withNewExecutionId(queryExecution, Option("deltaTransactionalWrite")) { val outputSpec = FileFormatWriter.OutputSpec( outputPath.toString, Map.empty, output) val empty2NullPlan = convertEmptyToNullIfNeeded(queryExecution.executedPlan, partitioningColumns, constraints) val checkInvariants = DeltaInvariantCheckerExec(spark, empty2NullPlan, constraints) // No need to plan optimized write if the write command is OPTIMIZE, which aims to produce // evenly-balanced data files already. val physicalPlan = if (!isOptimize && shouldOptimizeWrite(writeOptions, spark.sessionState.conf)) { DeltaOptimizedWriterExec(checkInvariants, metadata.partitionColumns, deltaLog) } else { checkInvariants } val statsTrackers: ListBuffer[WriteJobStatsTracker] = ListBuffer() if (spark.conf.get(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED)) { val basicWriteJobStatsTracker = new BasicWriteJobStatsTracker( new SerializableConfiguration(deltaLog.newDeltaHadoopConf()), BasicWriteJobStatsTracker.metrics) registerSQLMetrics(spark, basicWriteJobStatsTracker.driverSideMetrics) statsTrackers.append(basicWriteJobStatsTracker) } // Iceberg spec requires partition columns in data files val writePartitionColumns = IcebergCompat.isAnyEnabled(metadata) || protocol.isFeatureSupported(MaterializePartitionColumnsTableFeature) // Retain only a minimal selection of Spark writer options to avoid any potential // compatibility issues val options = (writeOptions match { case None => Map.empty[String, String] case Some(writeOptions) => writeOptions.options.filterKeys { key => key.equalsIgnoreCase(DeltaOptions.MAX_RECORDS_PER_FILE) || key.equalsIgnoreCase(DeltaOptions.COMPRESSION) }.toMap }) + (DeltaOptions.WRITE_PARTITION_COLUMNS -> writePartitionColumns.toString) ++ VariantShreddingShims.getVariantInferShreddingSchemaOptions( DeltaConfigs.ENABLE_VARIANT_SHREDDING.fromMetaData(metadata)) try { DeltaFileFormatWriter.write( sparkSession = spark, plan = physicalPlan, fileFormat = deltaLog.fileFormat(protocol, metadata), // TODO support changing formats. committer = committer, outputSpec = outputSpec, // scalastyle:off deltahadoopconfiguration hadoopConf = spark.sessionState.newHadoopConfWithOptions(metadata.configuration ++ deltaLog.options), // scalastyle:on deltahadoopconfiguration partitionColumns = partitioningColumns, bucketSpec = None, statsTrackers = optionalStatsTracker.toSeq ++ statsTrackers ++ identityTrackerOpt.toSeq, options = options) } catch { case InnerInvariantViolationException(violationException) => // Pull an InvariantViolationException up to the top level if it was the root cause. throw violationException } statsTrackers.foreach { case tracker: BasicWriteJobStatsTracker => val numOutputRowsOpt = tracker.driverSideMetrics.get("numOutputRows").map(_.value) IdentityColumn.logTableWrite(snapshot, trackIdentityHighWaterMarks, numOutputRowsOpt) case _ => () } } var resultFiles = (if (optionalStatsTracker.isDefined) { committer.addedStatuses.map { a => a.copy(stats = optionalStatsTracker.map( _.recordedStats(a.toPath.getName)).getOrElse(a.stats)) } } else { committer.addedStatuses }) .filter { // In some cases, we can write out an empty `inputData`. Some examples of this (though, they // may be fixed in the future) are the MERGE command when you delete with empty source, or // empty target, or on disjoint tables. This is hard to catch before the write without // collecting the DF ahead of time. Instead, we can return only the AddFiles that // a) actually add rows, or // b) don't have any stats so we don't know the number of rows at all case a: AddFile => a.numLogicalRecords.forall(_ > 0) case _ => true } // add [[AddFile.Tags.ICEBERG_COMPAT_VERSION.name]] tags to addFiles // starting from IcebergCompatV2 val enabledCompat = IcebergCompat.anyEnabled(metadata) if (enabledCompat.exists(_.version >= 2)) { resultFiles = resultFiles.map { addFile => addFile.copy(tags = Option(addFile.tags).getOrElse(Map.empty[String, String]) + (AddFile.Tags.ICEBERG_COMPAT_VERSION.name -> enabledCompat.get.version.toString) ) } } if (resultFiles.nonEmpty && !isOptimize) registerPostCommitHook(AutoCompact) // Record the updated high water marks to be used during transaction commit. identityTrackerOpt.ifDefined { tracker => updatedIdentityHighWaterMarks.appendAll(tracker.highWaterMarks.toSeq) } resultFiles.toSeq ++ committer.changeFiles } /** * Optimized writes can be enabled/disabled through the following order: * - Through DataFrameWriter options * - Through SQL configuration * - Through the table parameter */ private def shouldOptimizeWrite( writeOptions: Option[DeltaOptions], sessionConf: SQLConf): Boolean = { writeOptions.flatMap(_.optimizeWrite) .getOrElse(TransactionalWrite.shouldOptimizeWrite(metadata, sessionConf)) } } object TransactionalWrite { def shouldOptimizeWrite(metadata: Metadata, sessionConf: SQLConf): Boolean = { sessionConf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_WRITE_ENABLED) .orElse(DeltaConfigs.OPTIMIZE_WRITE.fromMetaData(metadata)) .getOrElse(false) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/fuzzer/AtomicBarrier.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.fuzzer import java.util.concurrent.atomic.AtomicInteger /** * An atomic barrier is similar to a countdown latch, * except that the content is a state transition system with semantic meaning * instead of a simple counter. * * It is designed with a single writer ("unblocker") thread and a single reader ("waiter") thread * in mind. It is concurrency safe with more writers and readers, but using more is likely to cause * race conditions for legal transitions. That is to say, trying to perform an otherwise * legal transition twice is illegal and may occur if there is more than one unblocker or * waiter thread. * Having additional passive state observers that only call [[load()]] is never an issue. * * Legal transitions are: * - BLOCKED -> UNBLOCKED * - BLOCKED -> REQUESTED * - REQUESTED -> UNBLOCKED * - UNBLOCKED -> PASSED */ class AtomicBarrier { import AtomicBarrier._ private final val state: AtomicInteger = new AtomicInteger(State.Blocked.ordinal) /** Get the current state. */ def load(): State = { val ordinal = state.get() // We should never be putting illegal state ordinals into `state`, // so this should always succeed. stateIndex(ordinal) } /** Transition to the Unblocked state. */ def unblock(): Unit = { // Just hot-retry this, since it never needs to wait to make progress. var successful = false while(!successful) { val currentValue = state.get() if (currentValue == State.Blocked.ordinal || currentValue == State.Requested.ordinal) { this.synchronized { successful = state.compareAndSet(currentValue, State.Unblocked.ordinal) if (successful) { this.notifyAll() } } } else { // if it's in any other state we will never make progress throw new IllegalStateTransitionException(stateIndex(currentValue), State.Unblocked) } } } /** Wait until this barrier can be passed and then mark it as Passed. */ def waitToPass(): Unit = { while (true) { val currentState = load() currentState match { case State.Unblocked => val updated = state.compareAndSet(currentState.ordinal, State.Passed.ordinal) if (updated) { return } case State.Passed => throw new IllegalStateTransitionException(State.Passed, State.Passed) case State.Requested => this.synchronized { if (load().ordinal == State.Requested.ordinal) { this.wait() } } case State.Blocked => this.synchronized { val updated = state.compareAndSet(currentState.ordinal, State.Requested.ordinal) if (updated) { this.wait() } } // else (if we didn't succeed) just hot-retry until we do // (or more likely pass, since unblocking is the only legal concurrent // update with a single concurrent "waiter") } } } override def toString: String = s"AtomicBarrier(state=${load()})" } object AtomicBarrier { sealed trait State { def ordinal: Int } object State { case object Blocked extends State { override final val ordinal = 0 } case object Unblocked extends State { override final val ordinal = 1 } case object Requested extends State { override final val ordinal = 2 } case object Passed extends State { override final val ordinal = 3 } } final val stateIndex: Map[Int, State] = List(State.Blocked, State.Unblocked, State.Requested, State.Passed) .map(state => state.ordinal -> state) .toMap } class IllegalStateTransitionException(fromState: AtomicBarrier.State, toState: AtomicBarrier.State) extends RuntimeException(s"State transition from $fromState to $toState is illegal.") ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/fuzzer/ExecutionPhaseLock.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.fuzzer /** * An ExecutionPhaseLock is an abstraction to keep multiple transactions moving in * a pre-selected lock-step sequence. * * In order to pass a phase, we first wait on the `entryBarrier`. Once we are allowed to pass there, * we can execute the code that belongs to this phase, and then we unblock the `exitBarrier`. * * @param name human readable name for debugging */ case class ExecutionPhaseLock( name: String, entryBarrier: AtomicBarrier = new AtomicBarrier(), exitBarrier: AtomicBarrier = new AtomicBarrier()) { def hasEntered: Boolean = entryBarrier.load() == AtomicBarrier.State.Passed def hasLeft: Boolean = { val current = exitBarrier.load() current == AtomicBarrier.State.Unblocked || current == AtomicBarrier.State.Passed } /** Blocks at this point until the phase has been entered. */ def waitToEnter(): Unit = entryBarrier.waitToPass() /** Unblock the next dependent phase. */ def leave(): Unit = exitBarrier.unblock() /** * Wait to enter this phase, then execute `f`, and leave before returning the result of `f`. * * @return the result of evaluating `f` */ def execute[T](f: => T): T = { waitToEnter() try { f } finally { leave() } } /** * If there is nothing that needs to be done in this phase, * we can leave immediately after entering. */ def passThrough(): Unit = { waitToEnter() leave() } def hasReached: Boolean = { val current = entryBarrier.load() current == AtomicBarrier.State.Requested || current == AtomicBarrier.State.Passed } /** Blocks at this point until the phase has been left. */ def waitToLeave(): Unit = exitBarrier.waitToPass() } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/fuzzer/OptimisticTransactionPhases.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.fuzzer case class OptimisticTransactionPhases( initialPhase: ExecutionPhaseLock, preparePhase: ExecutionPhaseLock, commitPhase: ExecutionPhaseLock, backfillPhase: ExecutionPhaseLock, postCommitPhase: ExecutionPhaseLock) object OptimisticTransactionPhases { private final val PREFIX = "TXN_" final val INITIAL_PHASE_LABEL = PREFIX + "INIT" final val PREPARE_PHASE_LABEL = PREFIX + "PREPARE" final val COMMIT_PHASE_LABEL = PREFIX + "COMMIT" final val BACKFILL_PHASE_LABEL = PREFIX + "BACKFILL" final val POST_COMMIT_PHASE_LABEL = PREFIX + "POST_COMMIT" def forName(txnName: String): OptimisticTransactionPhases = { def toTxnPhaseLabel(phaseLabel: String): String = txnName + "-" + phaseLabel OptimisticTransactionPhases( initialPhase = ExecutionPhaseLock(toTxnPhaseLabel(INITIAL_PHASE_LABEL)), preparePhase = ExecutionPhaseLock(toTxnPhaseLabel(PREPARE_PHASE_LABEL)), commitPhase = ExecutionPhaseLock(toTxnPhaseLabel(COMMIT_PHASE_LABEL)), backfillPhase = ExecutionPhaseLock(toTxnPhaseLabel(BACKFILL_PHASE_LABEL)), postCommitPhase = ExecutionPhaseLock(toTxnPhaseLabel(POST_COMMIT_PHASE_LABEL))) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/fuzzer/PhaseLockingExecutionObserver.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.fuzzer /** * Trait representing execution observers that rely on phases with entry and exit barriers to * control the order of execution of the observed code paths. See [[ExecutionPhaseLock]]. */ trait PhaseLockingExecutionObserver { val phaseLocks: Seq[ExecutionPhaseLock] /** Return `true` if we have left all phases, `false` otherwise. */ def allPhasesHavePassed: Boolean = phaseLocks.forall(_.hasLeft) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/fuzzer/PhaseLockingTransactionExecutionObserver.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.fuzzer import org.apache.spark.sql.delta.{OptimisticTransaction, TransactionExecutionObserver} private[delta] class PhaseLockingTransactionExecutionObserver( val phases: OptimisticTransactionPhases) extends TransactionExecutionObserver with PhaseLockingExecutionObserver { override val phaseLocks: Seq[ExecutionPhaseLock] = Seq( phases.initialPhase, phases.preparePhase, phases.commitPhase, phases.backfillPhase, phases.postCommitPhase) override def createChild(): TransactionExecutionObserver = { // Just return the current thread observer. // This is equivalent to the behaviour of the use-site before introduction of // `createChild`. TransactionExecutionObserver.getObserver } /** * When set to true this observer will automatically update the thread's current observer to * the next one. Also, it will not unblock the exit barrier of the commit phase automatically. * Instead, the caller will have to automatically unblock it. This allows writing tests that * can capture errors caused by code written between the end of the last txn and the start of * the next txn. */ @volatile protected var autoAdvanceNextObserver: Boolean = false override def startingTransaction(f: => OptimisticTransaction): OptimisticTransaction = phases.initialPhase.execute(f) override def preparingCommit[T](f: => T): T = phases.preparePhase.execute(f) override def beginDoCommit(): Unit = { phases.commitPhase.waitToEnter() } override def beginBackfill(): Unit = { phases.commitPhase.leave() phases.backfillPhase.waitToEnter() } override def beginPostCommit(): Unit = { phases.backfillPhase.leave() phases.postCommitPhase.waitToEnter() } override def transactionCommitted(): Unit = { if (nextObserver.nonEmpty && autoAdvanceNextObserver) { waitForLastPhaseAndAdvanceToNextObserver() } else { phases.postCommitPhase.leave() } } override def transactionAborted(): Unit = { if (!phases.commitPhase.hasLeft) { if (!phases.commitPhase.hasEntered) { phases.commitPhase.waitToEnter() } phases.commitPhase.leave() } if (!phases.backfillPhase.hasLeft) { if (!phases.backfillPhase.hasEntered) { phases.backfillPhase.waitToEnter() } phases.backfillPhase.leave() } if (!phases.postCommitPhase.hasEntered) { phases.postCommitPhase.waitToEnter() } if (nextObserver.nonEmpty && autoAdvanceNextObserver) { waitForLastPhaseAndAdvanceToNextObserver() } else { phases.postCommitPhase.leave() } } /* * Wait for the last phase to pass but do not unblock it so that callers can write tests * that capture errors caused by code between the end of the last txn and the start of the * new txn. After the commit phase is passed, update the thread observer of the thread to * the next observer. */ def waitForLastPhaseAndAdvanceToNextObserver(): Unit = { require(nextObserver.nonEmpty) phases.postCommitPhase.waitToLeave() advanceToNextThreadObserver() } /** * Set the next observer, which will replace the txn observer on the thread after a successful * commit. This method only works as expected if we haven't entered the commit phase yet. * * Note that when a next observer is set, the caller needs to manually unblock the exit barrier * of the commit phase. * * For example, see [[waitForLastPhaseAndAdvanceToNextObserver]]. */ def setNextObserver( nextTxnObserver: TransactionExecutionObserver, autoAdvance: Boolean): Unit = { setNextObserver(nextTxnObserver) autoAdvanceNextObserver = autoAdvance } override def advanceToNextThreadObserver(): Unit = super.advanceToNextThreadObserver() } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/hooks/AutoCompact.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.hooks import org.apache.spark.sql.delta.skipping.clustering.ClusteredTableUtils import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.commands.{DeltaOptimizeContext, OptimizeExecutor} import org.apache.spark.sql.delta.commands.optimize._ import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.AutoCompactPartitionStats import org.apache.spark.internal.MDC import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.internal.SQLConf /** * A trait for post commit hook which compacts files in a Delta table. This hook acts as a cheaper * version of the OPTIMIZE command, by attempting to compact small files together into fewer bigger * files. * * Auto Compact chooses files to compact greedily by looking at partition directories which * have the largest number of files that are under a certain size threshold and launches a bounded * number of optimize tasks based on the capacity of the cluster. */ trait AutoCompactBase extends PostCommitHook with DeltaLogging { override val name: String = "Auto Compact" private[delta] val OP_TYPE = "delta.commit.hooks.autoOptimize" /** * This method returns the type of Auto Compaction to use on a delta table or returns None * if Auto Compaction is disabled. * Prioritization: * 1. The highest priority is given to [[DeltaSQLConf.DELTA_AUTO_COMPACT_ENABLED]] config. * 2. Then we check if the deprecated property `DeltaConfigs.AUTO_OPTIMIZE` is set. If yes, then * we return [[AutoCompactType.Enabled]] type. * 3. Then we check the table property [[DeltaConfigs.AUTO_COMPACT]]. * 4. If none of 1/2/3 are set explicitly, then we return None */ def getAutoCompactType(conf: SQLConf, metadata: Metadata): Option[AutoCompactType] = { // If user-facing conf is set to something, use that value. val autoCompactTypeFromConf = conf.getConf(DeltaSQLConf.DELTA_AUTO_COMPACT_ENABLED).map(AutoCompactType(_)) if (autoCompactTypeFromConf.nonEmpty) return autoCompactTypeFromConf.get // If user-facing conf is not set, use what table property says. val deprecatedFlag = DeltaConfigs.AUTO_OPTIMIZE.fromMetaData(metadata) val autoCompactTypeFromPropertyOrDefaultValue = deprecatedFlag match { case Some(true) => Some(AutoCompactType.Enabled) case _ => // If the legacy property `DeltaConfigs.AUTO_OPTIMIZE` is false or not set, then check // the new table property `DeltaConfigs.AUTO_COMPACT`. val confValueFromTableProperty = DeltaConfigs.AUTO_COMPACT.fromMetaData(metadata) confValueFromTableProperty match { case Some(v) => // Table property is set to something explicitly by user. AutoCompactType(v) case None => AutoCompactType(AutoCompactType.DISABLED) // Default to disabled } } autoCompactTypeFromPropertyOrDefaultValue } private[hooks] def shouldSkipAutoCompact( autoCompactTypeOpt: Option[AutoCompactType], spark: SparkSession, txn: CommittedTransaction): Boolean = { // If auto compact type is empty, then skip compaction if (autoCompactTypeOpt.isEmpty) return true // Skip Auto Compaction, if one of the following conditions is satisfied: // -- Auto Compaction is not enabled. // -- Transaction execution time is empty, which means the parent transaction is not committed. !AutoCompactUtils.isQualifiedForAutoCompact(spark, txn) } override def run(spark: SparkSession, txn: CommittedTransaction): Unit = { val conf = spark.sessionState.conf val autoCompactTypeOpt = getAutoCompactType(conf, txn.postCommitSnapshot.metadata) // Skip Auto Compact if current transaction is not qualified or the table is not qualified // based on the value of autoCompactTypeOpt. if (shouldSkipAutoCompact(autoCompactTypeOpt, spark, txn)) return compactIfNecessary( spark, txn, OP_TYPE, maxDeletedRowsRatio = None) } /** * Compact the target table of write transaction `txn` only when there are sufficient amount of * small size files. */ private[delta] def compactIfNecessary( spark: SparkSession, txn: CommittedTransaction, opType: String, maxDeletedRowsRatio: Option[Double] ): Seq[OptimizeMetrics] = { val tableId = txn.deltaLog.unsafeVolatileTableId val autoCompactRequest = AutoCompactUtils.prepareAutoCompactRequest( spark, txn, opType, maxDeletedRowsRatio) if (autoCompactRequest.shouldCompact) { try { val metrics = AutoCompact .compact( spark, txn.deltaLog, txn.catalogTable, autoCompactRequest.targetPartitionsPredicate, opType, maxDeletedRowsRatio ) val partitionsStats = AutoCompactPartitionStats.instance(spark) // Mark partitions as compacted before releasing them. // Otherwise an already compacted partition might get picked up by a concurrent thread. // But only marks it as compacted, if no exception was thrown by auto compaction so that the // partitions stay eligible for subsequent auto compactions. partitionsStats.markPartitionsAsCompacted( tableId, autoCompactRequest.allowedPartitions ) metrics } catch { case e: Throwable => logError(log"Auto Compaction failed with: ${MDC(DeltaLogKeys.ERROR, e.getMessage)}") recordDeltaEvent( txn.deltaLog, opType = "delta.autoCompaction.error", data = getErrorData(e)) throw e } finally { if (AutoCompactUtils.reservePartitionEnabled(spark)) { AutoCompactPartitionReserve.releasePartitions( tableId, autoCompactRequest.allowedPartitions ) } } } else { Seq.empty[OptimizeMetrics] } } /** * Launch Auto Compaction jobs if there is sufficient capacity. * @param spark The spark session of the parent transaction that triggers this Auto Compaction. * @param deltaLog The delta log of the parent transaction. * @return the optimize metrics of this compaction job. */ private[delta] def compact( spark: SparkSession, deltaLog: DeltaLog, catalogTable: Option[CatalogTable], partitionPredicates: Seq[Expression] = Nil, opType: String = OP_TYPE, maxDeletedRowsRatio: Option[Double] = None) : Seq[OptimizeMetrics] = recordDeltaOperation(deltaLog, opType) { val maxFileSize = spark.conf.get(DeltaSQLConf.DELTA_AUTO_COMPACT_MAX_FILE_SIZE) val minFileSizeOpt = Some(spark.conf.get(DeltaSQLConf.DELTA_AUTO_COMPACT_MIN_FILE_SIZE) .getOrElse(maxFileSize / 2)) val maxFileSizeOpt = Some(maxFileSize) recordDeltaOperation(deltaLog, s"$opType.execute") { val optimizeContext = DeltaOptimizeContext( reorg = None, minFileSizeOpt, maxFileSizeOpt, maxDeletedRowsRatio = maxDeletedRowsRatio ) val rows = new OptimizeExecutor( spark, deltaLog.update(catalogTableOpt = catalogTable), catalogTable, partitionPredicates, zOrderByColumns = Seq(), isAutoCompact = true, optimizeContext ) .optimize() val metrics = rows.map(_.getAs[OptimizeMetrics](1)) recordDeltaEvent(deltaLog, s"$opType.execute.metrics", data = metrics.head) metrics } } } /** * Post commit hook for Auto Compaction. */ case object AutoCompact extends AutoCompactBase /** * A trait describing the type of Auto Compaction. */ sealed trait AutoCompactType { val configValueStrings: Seq[String] } object AutoCompactType { private[hooks] val DISABLED = "false" /** * Enable auto compact. * 1. MAX_FILE_SIZE is configurable and defaults to 128 MB unless overridden. * 2. MIN_FILE_SIZE is configurable and defaults to MAX_FILE_SIZE / 2 unless overridden. * Note: User can use DELTA_AUTO_COMPACT_MAX_FILE_SIZE to override this value. */ case object Enabled extends AutoCompactType { override val configValueStrings = Seq( "true" ) } /** * Converts the config value String (coming from [[DeltaSQLConf.DELTA_AUTO_COMPACT_ENABLED]] conf * or [[DeltaConfigs.AUTO_COMPACT]] table property) and translates into the [[AutoCompactType]]. */ def apply(value: String): Option[AutoCompactType] = { if (Enabled.configValueStrings.contains(value)) return Some(Enabled) if (value == DISABLED) return None throw DeltaErrors.invalidAutoCompactType(value) } // All allowed values for [[DeltaSQLConf.DELTA_AUTO_COMPACT_ENABLED]] and // [[DeltaConfigs.AUTO_COMPACT]]. val ALLOWED_VALUES = Enabled.configValueStrings ++ Seq(DISABLED) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/hooks/AutoCompactUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.hooks import scala.collection.mutable // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.{DeltaLog, Snapshot} import org.apache.spark.sql.delta.CommittedTransaction import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.sources.DeltaSQLConf._ import org.apache.spark.sql.delta.stats.AutoCompactPartitionStats import org.apache.spark.internal.config.ConfigEntry import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions.{And, Cast, EqualNullSafe, Expression, Literal, Or} import org.apache.spark.sql.functions.collect_list /** * The request class that contains all information needed for Auto Compaction. * @param shouldCompact True if Auto Compact should start. * @param optimizeContext The context that control execution of optimize command. * @param targetPartitionsPredicate The predicate of the target partitions of this Auto Compact * request. */ case class AutoCompactRequest( shouldCompact: Boolean, allowedPartitions: AutoCompactUtils.PartitionKeySet, targetPartitionsPredicate: Seq[Expression] = Nil) { } object AutoCompactRequest { /** Return a default AutoCompactRequest object that doesn't trigger Auto Compact. */ def noopRequest: AutoCompactRequest = AutoCompactRequest( shouldCompact = false, allowedPartitions = Set.empty ) } object AutoCompactUtils extends DeltaLogging { type PartitionKey = Map[String, String] type PartitionKeySet = Set[PartitionKey] val STATUS_NAME = { "status" } /** Create partition predicate from a partition key. */ private def createPartitionPredicate( postCommitSnapshot: Snapshot, partitions: PartitionKeySet): Seq[Expression] = { val schema = postCommitSnapshot.metadata.physicalPartitionSchema val partitionBranches = partitions.filterNot(_.isEmpty).map { partition => partition .toSeq .map { case (key, value) => val field = schema(key) EqualNullSafe(UnresolvedAttribute.quoted(key), Cast(Literal(value), field.dataType)) } .reduceLeft[Expression](And.apply) } if (partitionBranches.size > 1) { Seq(partitionBranches.reduceLeft[Expression](Or.apply)) } else if (partitionBranches.size == 1) { partitionBranches.toList } else { Seq.empty } } /** True if Auto Compaction only runs on modified partitions. */ def isModifiedPartitionsOnlyAutoCompactEnabled(spark: SparkSession): Boolean = spark.sessionState.conf.getConf(DELTA_AUTO_COMPACT_MODIFIED_PARTITIONS_ONLY_ENABLED) def isNonBlindAppendAutoCompactEnabled(spark: SparkSession): Boolean = spark.sessionState.conf.getConf(DELTA_AUTO_COMPACT_NON_BLIND_APPEND_ENABLED) def reservePartitionEnabled(spark: SparkSession): Boolean = spark.sessionState.conf.getConf(DELTA_AUTO_COMPACT_RESERVE_PARTITIONS_ENABLED) /** * Get the minimum number of files to trigger Auto Compact. */ def minNumFilesForAutoCompact(spark: SparkSession): Int = { spark.sessionState.conf.getConf(DELTA_AUTO_COMPACT_MIN_NUM_FILES) } /** * Try to reserve partitions inside `partitionsAddedToOpt` for Auto Compaction. * @return (shouldCompact, finalPartitions) The value of needCompaction is True if Auto * Compaction needs to run. `finalPartitions` is the set of target partitions that were * reserved for compaction. If finalPartitions is empty, then all partitions need to be * considered. */ private def reserveTablePartitions( spark: SparkSession, deltaLog: DeltaLog, postCommitSnapshot: Snapshot, partitionsAddedToOpt: Option[PartitionKeySet], opType: String, maxDeletedRowsRatio: Option[Double]): (Boolean, PartitionKeySet) = { import AutoCompactPartitionReserve._ if (partitionsAddedToOpt.isEmpty) { recordDeltaEvent(deltaLog, opType, data = Map(STATUS_NAME -> "skipEmptyIngestion")) // If partitionsAddedToOpt is empty, then just skip compact since it means there is no file // added in parent transaction and we do not want to hook AC on empty commits. return (false, Set.empty[PartitionKey]) } // Reserve partitions as following: // 1) First check if any partitions are free, i.e. no concurrent auto-compact thread is running. // 2) From free partitions check if any are eligible based on the number of small files. // 3) From free partitions check if any are eligible based on the deletion vectors. // 4) Try and reserve the union of the two lists. // All concurrent accesses to partitions reservation and partition stats are managed by the // [[AutoCompactPartitionReserve]] and [[AutoCompactPartitionStats]] singletons. val shouldReservePartitions = isModifiedPartitionsOnlyAutoCompactEnabled(spark) && reservePartitionEnabled(spark) val freePartitions = if (shouldReservePartitions) { filterFreePartitions(deltaLog.unsafeVolatileTableId, partitionsAddedToOpt.get) } else { partitionsAddedToOpt.get } // Early abort if all partitions are reserved. if (freePartitions.isEmpty) { recordDeltaEvent(deltaLog, opType, data = Map(STATUS_NAME -> "skipAllPartitionsAlreadyReserved")) return (false, Set.empty[PartitionKey]) } // Check min number of files criteria. val ChosenPartitionsResult(shouldCompactBasedOnNumFiles, chosenPartitionsBasedOnNumFiles, minNumFilesLogMsg) = choosePartitionsBasedOnMinNumSmallFiles( spark, deltaLog, postCommitSnapshot, freePartitions ) if (shouldCompactBasedOnNumFiles && chosenPartitionsBasedOnNumFiles.isEmpty) { // Run on all partitions, no need to check other criteria. // Note: this outcome of [choosePartitionsBasedOnMinNumSmallFiles] // is also only possible if partitions reservation is turned off, // so we do not need to reserve partitions. recordDeltaEvent(deltaLog, opType, data = Map(STATUS_NAME -> "runOnAllPartitions")) return (shouldCompactBasedOnNumFiles, chosenPartitionsBasedOnNumFiles) } // Check files with DVs criteria. val (shouldCompactBasedOnDVs, chosenPartitionsBasedOnDVs) = choosePartitionsBasedOnDVs(freePartitions, postCommitSnapshot, maxDeletedRowsRatio) var finalPartitions = chosenPartitionsBasedOnNumFiles ++ chosenPartitionsBasedOnDVs if (isModifiedPartitionsOnlyAutoCompactEnabled(spark)) { val maxNumPartitions = spark.conf.get(DELTA_AUTO_COMPACT_MAX_NUM_MODIFIED_PARTITIONS) finalPartitions = if (finalPartitions.size > maxNumPartitions) { // Choose maxNumPartitions at random. scala.util.Random.shuffle(finalPartitions.toIndexedSeq).take(maxNumPartitions).toSet } else { finalPartitions } } val numChosenPartitions = finalPartitions.size if (shouldReservePartitions) { finalPartitions = tryReservePartitions( deltaLog.unsafeVolatileTableId, finalPartitions) } // Abort if all chosen partitions were reserved by a concurrent thread. if (numChosenPartitions > 0 && finalPartitions.isEmpty) { recordDeltaEvent(deltaLog, opType, data = Map(STATUS_NAME -> "skipAllPartitionsAlreadyReserved")) return (false, Set.empty[PartitionKey]) } val shouldCompact = shouldCompactBasedOnNumFiles || shouldCompactBasedOnDVs val statusLogMessage = if (!shouldCompact) { "skip" + minNumFilesLogMsg } else if (shouldCompactBasedOnNumFiles && !shouldCompactBasedOnDVs) { "run" + minNumFilesLogMsg } else if (shouldCompactBasedOnNumFiles && shouldCompactBasedOnDVs) { "run" + minNumFilesLogMsg + "AndPartitionsWithDVs" } else if (!shouldCompactBasedOnNumFiles && shouldCompactBasedOnDVs) { "runOnPartitionsWithDVs" } val logData = scala.collection.mutable.Map(STATUS_NAME -> statusLogMessage) if (finalPartitions.nonEmpty) { logData += ("partitions" -> finalPartitions.size.toString) } recordDeltaEvent(deltaLog, opType, data = logData) (shouldCompactBasedOnNumFiles || shouldCompactBasedOnDVs, finalPartitions) } private case class ChosenPartitionsResult( shouldRunAC: Boolean, chosenPartitions: PartitionKeySet, logMessage: String) private def choosePartitionsBasedOnMinNumSmallFiles( spark: SparkSession, deltaLog: DeltaLog, postCommitSnapshot: Snapshot, freePartitionsAddedTo: PartitionKeySet): ChosenPartitionsResult = { def getConf[T](entry: ConfigEntry[T]): T = spark.sessionState.conf.getConf(entry) val minNumFiles = minNumFilesForAutoCompact(spark) val partitionEarlySkippingEnabled = getConf(DELTA_AUTO_COMPACT_EARLY_SKIP_PARTITION_TABLE_ENABLED) val tablePartitionStats = AutoCompactPartitionStats.instance(spark) if (isModifiedPartitionsOnlyAutoCompactEnabled(spark)) { // If modified partition only Auto Compact is enabled, pick the partitions that have more // number of files than minNumFiles. // If table partition early skipping feature is enabled, use the current minimum number of // files threshold; otherwise, use 0 to indicate that any partition is qualified. val minNumFilesPerPartition = if (partitionEarlySkippingEnabled) minNumFiles else 0L val pickedPartitions = tablePartitionStats.filterPartitionsWithSmallFiles( deltaLog.unsafeVolatileTableId, freePartitionsAddedTo, minNumFilesPerPartition) if (pickedPartitions.isEmpty) { ChosenPartitionsResult(shouldRunAC = false, chosenPartitions = pickedPartitions, logMessage = "InsufficientFilesInModifiedPartitions") } else { ChosenPartitionsResult(shouldRunAC = true, chosenPartitions = pickedPartitions, logMessage = "OnModifiedPartitions") } } else if (partitionEarlySkippingEnabled) { // If only early skipping is enabled, then check whether there is any partition with more // files than minNumFiles. val maxNumFiles = tablePartitionStats.maxNumFilesInTable(deltaLog.unsafeVolatileTableId) val shouldCompact = maxNumFiles >= minNumFiles if (shouldCompact) { ChosenPartitionsResult(shouldRunAC = true, chosenPartitions = Set.empty[PartitionKey], logMessage = "OnAllPartitions") } else { ChosenPartitionsResult(shouldRunAC = false, chosenPartitions = Set.empty[PartitionKey], logMessage = "InsufficientInAllPartitions") } } else { // If both are disabled, then Auto Compaction should search all partitions of the target // table. ChosenPartitionsResult( shouldRunAC = true, chosenPartitions = Set.empty[PartitionKey], logMessage = "OnAllPartitions") } } private def choosePartitionsBasedOnDVs( freePartitionsAddedTo: PartitionKeySet, postCommitSnapshot: Snapshot, maxDeletedRowsRatio: Option[Double]) = { var partitionsWithDVs = if (maxDeletedRowsRatio.nonEmpty) { postCommitSnapshot .allFiles .where("deletionVector IS NOT NULL") .where( s""" |(deletionVector.cardinality / stats:`numRecords`) > ${maxDeletedRowsRatio.get} |""".stripMargin) // Cast map to string so we can group by it. // The string representation might not be deterministic. // Still, there is only a limited number of representations we could get for a given map, // Which should sufficiently reduce the data collected on the driver. // We then make sure the partitions are distinct on the driver. .selectExpr("CAST(partitionValues AS STRING) as partitionValuesStr", "partitionValues") .groupBy("partitionValuesStr") .agg(collect_list("partitionValues").as("partitionValues")) .selectExpr("partitionValues[0] as partitionValues") .collect() .map(_.getAs[Map[String, String]]("partitionValues")).toSet } else { Set.empty[PartitionKey] } partitionsWithDVs = partitionsWithDVs.intersect(freePartitionsAddedTo) (partitionsWithDVs.nonEmpty, partitionsWithDVs) } /** * Prepare an [[AutoCompactRequest]] object based on the statistics of partitions inside * `partitionsAddedToOpt` in `txn`. * * @param maxDeletedRowsRatio If set, signals to Auto Compaction to rewrite files with * DVs with maxDeletedRowsRatio above this threshold. */ def prepareAutoCompactRequest( spark: SparkSession, txn: CommittedTransaction, opType: String, maxDeletedRowsRatio: Option[Double]): AutoCompactRequest = { val partitionsAddedToOpt = txn.partitionsAddedToOpt.map(_.toSet) val (needAutoCompact, reservedPartitions) = reserveTablePartitions( spark, txn.deltaLog, txn.postCommitSnapshot, partitionsAddedToOpt, opType, maxDeletedRowsRatio) AutoCompactRequest( needAutoCompact, reservedPartitions, createPartitionPredicate(txn.postCommitSnapshot, reservedPartitions)) } /** * True if this transaction is qualified for Auto Compaction. * - When current transaction is not blind append, it is safe to enable Auto Compaction when * DELTA_AUTO_COMPACT_MODIFIED_PARTITIONS_ONLY_ENABLED is true, or it's an un-partitioned table, * because then we cannot introduce _additional_ conflicts with concurrent write transactions. */ def isQualifiedForAutoCompact( spark: SparkSession, txn: CommittedTransaction): Boolean = { // If modified partitions only mode is not enabled, return true to avoid subsequent checking. if (!isModifiedPartitionsOnlyAutoCompactEnabled(spark)) return true !(isNonBlindAppendAutoCompactEnabled(spark) && txn.isBlindAppend) } } /** * Thread-safe singleton to keep track of partitions reserved for auto-compaction. */ object AutoCompactPartitionReserve { import org.apache.spark.sql.delta.hooks.AutoCompactUtils.PartitionKey // Key is table id and the value the set of currently reserved partition hashes. private val reservedTablesPartitions = new mutable.LinkedHashMap[String, Set[Int]] /** * @return Partitions from targetPartitions that are not reserved. */ def filterFreePartitions(tableId: String, targetPartitions: Set[PartitionKey]) : Set[PartitionKey] = synchronized { val reservedPartitionKeys = reservedTablesPartitions.getOrElse(tableId, Set.empty) targetPartitions.filter(partition => !reservedPartitionKeys.contains(partition.##)) } /** * Try to reserve partitions from [[targetPartitions]] which are not yet reserved. * @return partitions from targetPartitions which were not previously reserved. */ def tryReservePartitions(tableId: String, targetPartitions: Set[PartitionKey]) : Set[PartitionKey] = synchronized { val allReservedPartitions = reservedTablesPartitions.getOrElse(tableId, Set.empty) val unReservedPartitionsFromTarget = targetPartitions .filter(targetPartition => !allReservedPartitions.contains(targetPartition.##)) val newAllReservedPartitions = allReservedPartitions ++ unReservedPartitionsFromTarget.map(_.##) reservedTablesPartitions.update(tableId, newAllReservedPartitions) unReservedPartitionsFromTarget } /** * Releases the reserved table partitions to allow other threads to reserve them. * @param tableId The identity of the target table of Auto Compaction. * @param reservedPartitions The set of partitions, which were reserved and which need releasing. */ def releasePartitions( tableId: String, reservedPartitions: Set[PartitionKey]): Unit = synchronized { val allReservedPartitions = reservedTablesPartitions.getOrElse(tableId, Set.empty) val newPartitions = allReservedPartitions -- reservedPartitions.map(_.##) reservedTablesPartitions.update(tableId, newPartitions) } /** This is test only code to reset the state of table partition reservations. */ private[delta] def resetTestOnly(): Unit = synchronized { reservedTablesPartitions.clear() } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/hooks/CheckpointHook.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.hooks import org.apache.spark.sql.delta.CommittedTransaction import org.apache.spark.sql.SparkSession /** Write a new checkpoint at the version committed by the txn if required. */ object CheckpointHook extends PostCommitHook { override val name: String = "Post commit checkpoint trigger" override def run(spark: SparkSession, txn: CommittedTransaction): Unit = { if (!txn.needsCheckpoint) return // Since the postCommitSnapshot isn't guaranteed to match committedVersion, we have to // explicitly checkpoint the snapshot at the committedVersion. val cp = txn.postCommitSnapshot.checkpointProvider val snapshotToCheckpoint = txn.deltaLog.getSnapshotAt( txn.committedVersion, lastCheckpointHint = None, lastCheckpointProvider = Some(cp), catalogTableOpt = txn.catalogTable, enforceTimeTravelWithinDeletedFileRetention = false) txn.deltaLog.checkpoint(snapshotToCheckpoint, txn.catalogTable) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/hooks/ChecksumHook.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.hooks // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.{CommittedTransaction, DeltaLog, RecordChecksum, Snapshot} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.internal.MDC import org.apache.spark.sql.SparkSession /** Write a new checksum at the version committed by the txn if possible. */ object ChecksumHook extends PostCommitHook with DeltaLogging { // Helper that creates a RecordChecksum and uses it to write a checksum file case class WriteChecksum( override val spark: SparkSession, override val deltaLog: DeltaLog, txnId: String, snapshot: Snapshot) extends RecordChecksum { writeChecksumFile(txnId, snapshot) } override val name: String = "Post commit checksum trigger" override def run(spark: SparkSession, txn: CommittedTransaction): Unit = { // Only write the checksum if the postCommitSnapshot matches the version that was committed. if (txn.postCommitSnapshot.version != txn.committedVersion) return logInfo( log"Writing checksum file for table path ${MDC(DeltaLogKeys.PATH, txn.deltaLog.logPath)} " + log"version ${MDC(DeltaLogKeys.VERSION, txn.committedVersion)}") writeChecksum(spark, txn) } private def writeChecksum(spark: SparkSession, txn: CommittedTransaction): Unit = { WriteChecksum(spark, txn.deltaLog, txn.txnId, txn.postCommitSnapshot) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/hooks/GenerateSymlinkManifest.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.hooks // scalastyle:off import.ordering.noEmptyLine import java.net.URI import scala.collection.mutable import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.commands.DeletionVectorUtils.isTableDVFree import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.storage.LogStore import org.apache.spark.sql.delta.util.DeltaFileOperations import org.apache.hadoop.fs.Path import org.apache.spark.SparkEnv import org.apache.spark.internal.MDC import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.catalog.{CatalogTable, ExternalCatalogUtils} import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, Concat, Expression, Literal, ScalaUDF} import org.apache.spark.sql.execution.datasources.InMemoryFileIndex import org.apache.spark.sql.functions.col import org.apache.spark.sql.types.StringType import org.apache.spark.util.SerializableConfiguration /** * Post commit hook to generate hive-style manifests for Delta table. This is useful for * compatibility with Presto / Athena. */ object GenerateSymlinkManifest extends GenerateSymlinkManifestImpl // A separate singleton to avoid creating encoders from scratch every time object GenerateSymlinkManifestUtils extends DeltaLogging { private[hooks] lazy val mapEncoder = try { ExpressionEncoder[Map[String, String]]() } catch { case e: Throwable => logError(e.getMessage, e) throw e } } trait GenerateSymlinkManifestImpl extends PostCommitHook with DeltaLogging with Serializable { val CONFIG_NAME_ROOT = "compatibility.symlinkFormatManifest" val MANIFEST_LOCATION = "_symlink_format_manifest" val OP_TYPE_ROOT = "delta.compatibility.symlinkFormatManifest" val FULL_MANIFEST_OP_TYPE = s"$OP_TYPE_ROOT.full" val INCREMENTAL_MANIFEST_OP_TYPE = s"$OP_TYPE_ROOT.incremental" override val name: String = "Generate Symlink Format Manifest" override def run(spark: SparkSession, txn: CommittedTransaction): Unit = { generateIncrementalManifest(spark, txn, txn.postCommitSnapshot) } override def handleError(spark: SparkSession, error: Throwable, version: Long): Unit = { error match { case e: ColumnMappingUnsupportedException => throw e case e: DeltaCommandUnsupportedWithDeletionVectorsException => throw e case _ => throw DeltaErrors.postCommitHookFailedException(this, version, name, error) } } /** * Generate manifest files incrementally, that is, only for the table partitions touched by the * given actions. */ protected def generateIncrementalManifest( spark: SparkSession, txn: CommittedTransaction, currentSnapshot: Snapshot): Unit = recordManifestGeneration(txn.deltaLog, full = false) { import org.apache.spark.sql.delta.implicits._ checkColumnMappingMode(currentSnapshot.metadata) val deltaLog = txn.deltaLog val partitionCols = currentSnapshot.metadata.partitionColumns val manifestRootDirPath = new Path(deltaLog.dataPath, MANIFEST_LOCATION) val hadoopConf = new SerializableConfiguration(deltaLog.newDeltaHadoopConf()) val fs = deltaLog.dataPath.getFileSystem(hadoopConf.value) if (!fs.exists(manifestRootDirPath)) { generateFullManifest(spark, deltaLog, txn.catalogTable) return } // Find all the manifest partitions that need to updated or deleted val (allFilesInUpdatedPartitions, nowEmptyPartitions) = if (partitionCols.nonEmpty) { val actions = txn.committedActions val (addFiles, otherActions) = actions.partition(_.isInstanceOf[AddFile]) val (removeFiles, _) = otherActions.partition(_.isInstanceOf[RemoveFile]) // Get the partitions where files were added val partitionsOfAddedFiles = addFiles.collect { case a: AddFile => a.partitionValues }.toSet // Get the partitions where files were deleted val removedFileNames = spark.createDataset(removeFiles.collect { case r: RemoveFile => r.path }.toSeq).toDF("path") val partitionValuesOfRemovedFiles = txn.readSnapshot.allFiles.join(removedFileNames, "path").select("partitionValues").persist() try { val partitionsOfRemovedFiles = partitionValuesOfRemovedFiles .as[Map[String, String]](GenerateSymlinkManifestUtils.mapEncoder).collect().toSet // Get the files present in the updated partitions val partitionsUpdated: Set[Map[String, String]] = partitionsOfAddedFiles ++ partitionsOfRemovedFiles val filesInUpdatedPartitions = currentSnapshot.allFiles.filter { a => partitionsUpdated.contains(a.partitionValues) } // Find the current partitions val currentPartitionRelativeDirs = withRelativePartitionDir(spark, partitionCols, currentSnapshot.allFiles) .select("relativePartitionDir").distinct() // Find the partitions that became empty and delete their manifests val partitionRelativeDirsOfRemovedFiles = withRelativePartitionDir(spark, partitionCols, partitionValuesOfRemovedFiles) .select("relativePartitionDir").distinct() val partitionsThatBecameEmpty = partitionRelativeDirsOfRemovedFiles.join( currentPartitionRelativeDirs, Seq("relativePartitionDir"), "leftanti") .as[String].collect() (filesInUpdatedPartitions, partitionsThatBecameEmpty) } finally { partitionValuesOfRemovedFiles.unpersist() } } else { (currentSnapshot.allFiles, Array.empty[String]) } val manifestFilePartitionsWritten = writeManifestFiles( deltaLog.dataPath, manifestRootDirPath.toString, allFilesInUpdatedPartitions, partitionCols, hadoopConf) if (nowEmptyPartitions.nonEmpty) { deleteManifestFiles(manifestRootDirPath.toString, nowEmptyPartitions, hadoopConf) } // Post stats val stats = SymlinkManifestStats( filesWritten = manifestFilePartitionsWritten.size, filesDeleted = nowEmptyPartitions.length, partitioned = partitionCols.nonEmpty) recordDeltaEvent(deltaLog, s"$INCREMENTAL_MANIFEST_OP_TYPE.stats", data = stats) } /** * Generate manifest files for all the partitions in the table. Note, this will ensure that * that stale and unnecessary files will be vacuumed. */ def generateFullManifest( spark: SparkSession, deltaLog: DeltaLog, catalogTableOpt: Option[CatalogTable]): Unit = { val snapshot = deltaLog.update(catalogTableOpt = catalogTableOpt) assertTableIsDVFree(spark, snapshot) generateFullManifestWithSnapshot(spark, deltaLog, snapshot) } // Separated out to allow overriding with a specific snapshot. protected def generateFullManifestWithSnapshot( spark: SparkSession, deltaLog: DeltaLog, snapshot: Snapshot): Unit = recordManifestGeneration(deltaLog, full = true) { val partitionCols = snapshot.metadata.partitionColumns val manifestRootDirPath = new Path(deltaLog.dataPath, MANIFEST_LOCATION).toString val hadoopConf = new SerializableConfiguration(deltaLog.newDeltaHadoopConf()) checkColumnMappingMode(snapshot.metadata) // Update manifest files of the current partitions val newManifestPartitionRelativePaths = writeManifestFiles( deltaLog.dataPath, manifestRootDirPath, snapshot.allFiles, partitionCols, hadoopConf) // Get the existing manifest files as relative partition paths, that is, // [ "col1=0/col2=0", "col1=1/col2=1", "col1=2/col2=2" ] val fs = deltaLog.dataPath.getFileSystem(hadoopConf.value) val existingManifestPartitionRelativePaths = { val manifestRootDirAbsPath = fs.makeQualified(new Path(manifestRootDirPath)) if (fs.exists(manifestRootDirAbsPath)) { val index = new InMemoryFileIndex( spark, Seq(manifestRootDirAbsPath), deltaLog.options, None) val prefixToStrip = manifestRootDirAbsPath.toUri.getPath index.inputFiles.map { p => // Remove root directory "rootDir" path from the manifest file paths like // "rootDir/col1=0/col2=0/manifest" to get the relative partition dir "col1=0/col2=0". // Note: It important to compare only the "path" in the URI and not the user info in it. // In s3a://access-key:secret-key@host/path, the access-key and secret-key may change // unknowingly to `\` and `%` encoding between the root dir and file names generated // by listing. val relativeManifestFilePath = new URI(p).getPath.stripPrefix(prefixToStrip).stripPrefix(Path.SEPARATOR) new Path(relativeManifestFilePath).getParent.toString // returns "col1=0/col2=0" }.filterNot(_.trim.isEmpty).toSet } else Set.empty[String] } // paths returned from inputFiles are URI encoded so we need to convert them back to string. // So that they can compared with newManifestPartitionRelativePaths in the next step. // Delete manifest files for partitions that are not in current and so weren't overwritten val manifestFilePartitionsToDelete = existingManifestPartitionRelativePaths.diff(newManifestPartitionRelativePaths) deleteManifestFiles(manifestRootDirPath, manifestFilePartitionsToDelete, hadoopConf) // Post stats val stats = SymlinkManifestStats( filesWritten = newManifestPartitionRelativePaths.size, filesDeleted = manifestFilePartitionsToDelete.size, partitioned = partitionCols.nonEmpty) recordDeltaEvent(deltaLog, s"$FULL_MANIFEST_OP_TYPE.stats", data = stats) } protected def assertTableIsDVFree(spark: SparkSession, snapshot: Snapshot): Unit = { if (!isTableDVFree(snapshot)) { throw DeltaErrors.generateNotSupportedWithDeletionVectors() } } /** * Write the manifest files and return the partition relative paths of the manifests written. * * @param deltaLogDataPath path of the table data (e.g., tablePath which has _delta_log in it) * @param manifestRootDirPath root directory of the manifest files (e.g., tablePath/_manifest/) * @param fileNamesForManifest relative paths or file names of data files for being written into * the manifest (e.g., partition=1/xyz.parquet) * @param partitionCols Table partition columns * @param hadoopConf Hadoop configuration to use * @return Set of partition relative paths of the written manifest files (e.g., part1=1/part2=2) */ private def writeManifestFiles( deltaLogDataPath: Path, manifestRootDirPath: String, fileNamesForManifest: Dataset[AddFile], partitionCols: Seq[String], hadoopConf: SerializableConfiguration): Set[String] = { val spark = fileNamesForManifest.sparkSession import org.apache.spark.sql.delta.implicits._ val tableAbsPathForManifest = LogStore(spark) .resolvePathOnPhysicalStorage(deltaLogDataPath, hadoopConf.value).toString /** Write the data file relative paths to manifestDirAbsPath/manifest as absolute paths */ def writeSingleManifestFile( manifestDirAbsPath: String, dataFileRelativePaths: Iterator[String]): Unit = { val manifestFilePath = new Path(manifestDirAbsPath, "manifest") val fs = manifestFilePath.getFileSystem(hadoopConf.value) fs.mkdirs(manifestFilePath.getParent()) val manifestContent = dataFileRelativePaths.map { relativePath => DeltaFileOperations.absolutePath(tableAbsPathForManifest, relativePath).toString } val logStore = LogStore(SparkEnv.get.conf, hadoopConf.value) logStore.write(manifestFilePath, manifestContent, overwrite = true, hadoopConf.value) } val newManifestPartitionRelativePaths = if (fileNamesForManifest.isEmpty && partitionCols.isEmpty) { writeSingleManifestFile(manifestRootDirPath, Iterator()) Set.empty[String] } else { withRelativePartitionDir(spark, partitionCols, fileNamesForManifest) .select("relativePartitionDir", "path").as[(String, String)] .groupByKey(_._1).mapGroups { (relativePartitionDir: String, relativeDataFilePath: Iterator[(String, String)]) => val manifestPartitionDirAbsPath = { if (relativePartitionDir == null || relativePartitionDir.isEmpty) manifestRootDirPath else new Path(manifestRootDirPath, relativePartitionDir).toString } writeSingleManifestFile(manifestPartitionDirAbsPath, relativeDataFilePath.map(_._2)) relativePartitionDir }.collect().toSet } logInfo(log"Generated manifest partitions for ${MDC(DeltaLogKeys.PATH, deltaLogDataPath)} " + log"[${MDC(DeltaLogKeys.NUM_PARTITIONS, newManifestPartitionRelativePaths.size)}]:\n\t" + log"${MDC(DeltaLogKeys.PATHS, newManifestPartitionRelativePaths.mkString("\n\t"))}") newManifestPartitionRelativePaths } /** * Delete manifest files in the given paths. * * @param manifestRootDirPath root directory of the manifest files (e.g., tablePath/_manifest/) * @param partitionRelativePathsToDelete partitions to delete manifest files from * (e.g., part1=1/part2=2/) * @param hadoopConf Hadoop configuration to use */ private def deleteManifestFiles( manifestRootDirPath: String, partitionRelativePathsToDelete: Iterable[String], hadoopConf: SerializableConfiguration): Unit = { val fs = new Path(manifestRootDirPath).getFileSystem(hadoopConf.value) partitionRelativePathsToDelete.foreach { path => val absPathToDelete = new Path(manifestRootDirPath, path) fs.delete(absPathToDelete, true) } logInfo(log"Deleted manifest partitions [" + log"${MDC(DeltaLogKeys.NUM_FILES, partitionRelativePathsToDelete.size.toLong)}]:\n\t" + log"${MDC(DeltaLogKeys.PATHS, partitionRelativePathsToDelete.mkString("\n\t"))}") } /** * Append a column `relativePartitionDir` to the given Dataset which has `partitionValues` as * one of the columns. `partitionValues` is a map-type column that contains values of the * given `partitionCols`. */ private def withRelativePartitionDir( spark: SparkSession, partitionCols: Seq[String], datasetWithPartitionValues: Dataset[_]) = { require(datasetWithPartitionValues.schema.fieldNames.contains("partitionValues")) val colNamePrefix = "_col_" // Flatten out nested partition value columns while renaming them, so that the new columns do // not conflict with existing columns in DF `pathsWithPartitionValues. val colToRenamedCols = partitionCols.map { column => column -> s"$colNamePrefix$column" } val df = colToRenamedCols.foldLeft(datasetWithPartitionValues.toDF()) { case(currentDs, (column, renamedColumn)) => currentDs.withColumn(renamedColumn, col(s"partitionValues.`$column`")) } // Mapping between original column names to use for generating partition path and // attributes referring to corresponding columns added to DF `pathsWithPartitionValues`. val colNameToAttribs = colToRenamedCols.map { case (col, renamed) => col -> UnresolvedAttribute.quoted(renamed) } // Build an expression that can generate the path fragment col1=value/col2=value/ from the // partition columns. Note: The session time zone maybe different from the time zone that was // used to write the partition structure of the actual data files. This may lead to // inconsistencies between the partition structure of metadata files and data files. val relativePartitionDirExpression = generatePartitionPathExpression( colNameToAttribs, spark.sessionState.conf.sessionLocalTimeZone) df.withColumn("relativePartitionDir", Column(relativePartitionDirExpression)) .drop(colToRenamedCols.map(_._2): _*) } /** Expression that given partition columns builds a path string like: col1=val/col2=val/... */ protected def generatePartitionPathExpression( partitionColNameToAttrib: Seq[(String, Attribute)], timeZoneId: String): Expression = Concat( partitionColNameToAttrib.zipWithIndex.flatMap { case ((colName, col), i) => val partitionName = ScalaUDF( ExternalCatalogUtils.getPartitionPathString _, StringType, Seq(Literal(colName), Cast(col, StringType, Option(timeZoneId)))) if (i == 0) Seq(partitionName) else Seq(Literal(Path.SEPARATOR), partitionName) } ) private def recordManifestGeneration(deltaLog: DeltaLog, full: Boolean)(thunk: => Unit): Unit = { val (opType, manifestType) = if (full) FULL_MANIFEST_OP_TYPE -> "full" else INCREMENTAL_MANIFEST_OP_TYPE -> "incremental" recordDeltaOperation(deltaLog, opType) { withStatusCode("DELTA", s"Updating $manifestType Hive manifest for the Delta table") { thunk } } } /** * Generating manifests, when column mapping used is not supported, * because external systems will not be able to read Delta tables that leverage * column mapping correctly. */ private def checkColumnMappingMode(metadata: Metadata): Unit = { if (metadata.columnMappingMode != NoMapping) { throw DeltaErrors.generateManifestWithColumnMappingNotSupported } } case class SymlinkManifestStats( filesWritten: Int, filesDeleted: Int, partitioned: Boolean) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/hooks/HudiConverterHook.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.hooks import org.apache.spark.sql.delta.{CommittedTransaction, UniversalFormat} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf.DELTA_UNIFORM_HUDI_SYNC_CONVERT_ENABLED import org.apache.spark.sql.SparkSession /** Write a new Hudi commit for the version committed by the txn, if required. */ object HudiConverterHook extends PostCommitHook with DeltaLogging { override val name: String = "Post-commit Hudi metadata conversion" val ASYNC_HUDI_CONVERTER_THREAD_NAME = "async-hudi-converter" override def run(spark: SparkSession, txn: CommittedTransaction): Unit = { val postCommitSnapshot = txn.postCommitSnapshot // Only convert to Hudi if the snapshot matches the version committed. // This is to skip converting the same actions multiple times - they'll be written out // by another commit anyways. if (txn.committedVersion != postCommitSnapshot.version || !UniversalFormat.hudiEnabled(postCommitSnapshot.metadata)) { return } val converter = postCommitSnapshot.deltaLog.hudiConverter if (spark.sessionState.conf.getConf(DELTA_UNIFORM_HUDI_SYNC_CONVERT_ENABLED)) { converter.convertSnapshot(postCommitSnapshot, txn) } else { converter.enqueueSnapshotForConversion(postCommitSnapshot, txn) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/hooks/IcebergConverterHook.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.hooks import org.apache.spark.sql.delta.{CommittedTransaction, DeltaErrors, UniversalFormat, UniversalFormatConverter} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf.DELTA_UNIFORM_ICEBERG_SYNC_CONVERT_ENABLED import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.spark.sql.SparkSession /** Write a new Iceberg metadata file at the version committed by the txn, if required. */ trait IcebergConverterHook extends PostCommitHook with DeltaLogging { override val name: String = "Post-commit Iceberg metadata conversion" val ASYNC_ICEBERG_CONVERTER_THREAD_NAME = "async-iceberg-converter" override def run(spark: SparkSession, txn: CommittedTransaction): Unit = { val postCommitSnapshot = txn.postCommitSnapshot // Only convert to Iceberg if the snapshot matches the version committed. // This is to skip converting the same actions multiple times - they'll be written out // by another commit anyways. if (txn.committedVersion != postCommitSnapshot.version || !UniversalFormat.icebergEnabled(postCommitSnapshot.metadata)) { return } val converter = postCommitSnapshot.deltaLog.icebergConverter triggerIcebergConversion(converter, spark, txn) } // Always throw when sync Iceberg conversion fails. Async conversion exception // is handled in the async thread. override def handleError(spark: SparkSession, error: Throwable, version: Long): Unit = { logError(error.getMessage, error) throw DeltaErrors.universalFormatConversionFailedException( version, "iceberg", ExceptionUtils.getMessage(error)) } def triggerIcebergConversion( converter: UniversalFormatConverter, spark: SparkSession, txn: CommittedTransaction): Unit = { val postCommitSnapshot = txn.postCommitSnapshot if (spark.sessionState.conf.getConf(DELTA_UNIFORM_ICEBERG_SYNC_CONVERT_ENABLED) || !UniversalFormat.icebergEnabled(txn.readSnapshot.metadata)) { // UniForm was not enabled converter.convertSnapshot(postCommitSnapshot, txn) } else { converter.enqueueSnapshotForConversion(postCommitSnapshot, txn) } } } object IcebergConverterHook extends IcebergConverterHook object IcebergSyncConverterHook extends IcebergConverterHook { override def triggerIcebergConversion( converter: UniversalFormatConverter, spark: SparkSession, txn: CommittedTransaction): Unit = { converter.convertSnapshot(txn.postCommitSnapshot, txn) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/hooks/PostCommitHook.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.hooks import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.SparkSession /** * A hook which can be executed after a transaction. These hooks are registered to a * [[OptimisticTransaction]], and are executed after a *successful* commit takes place. */ trait PostCommitHook { /** A user-friendly name for the hook for error reporting purposes. */ val name: String /** * Executes the hook. * @param txn The txn that made the commit, after which this PostCommitHook was run */ def run(spark: SparkSession, txn: CommittedTransaction): Unit /** * Handle any error caused while running the hook. By default, all errors are ignored as * default policy should be to not let post-commit hooks to cause failures in the operation. */ def handleError(spark: SparkSession, error: Throwable, version: Long): Unit = { if (spark.conf.get(DeltaSQLConf.DELTA_POST_COMMIT_HOOK_THROW_ON_ERROR)) { throw DeltaErrors.postCommitHookFailedException(this, version, name, error) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/hooks/UpdateCatalog.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.hooks import java.nio.charset.Charset import java.util.concurrent.atomic.AtomicInteger import scala.collection.JavaConverters._ import scala.concurrent.{ExecutionContext, Future, TimeoutException} import scala.util.Try import scala.util.control.NonFatal import org.apache.spark.sql.delta.skipping.clustering.{ClusteredTableUtils, ClusteringColumnInfo} import org.apache.spark.sql.delta.skipping.clustering.temp.ClusterBySpec import org.apache.spark.sql.delta.{CommittedTransaction, DeltaConfigs, DeltaTableIdentifier, Snapshot} import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.threads.DeltaThreadPool import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.spark.internal.MDC import org.apache.spark.internal.config.ConfigEntry import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType import org.apache.spark.util.ThreadUtils /** * Factory object to create an UpdateCatalog post commit hook. This should always be used * instead of directly creating a specific hook. */ object UpdateCatalogFactory { def getUpdateCatalogHook(table: CatalogTable, spark: SparkSession): UpdateCatalogBase = { UpdateCatalog(table) } } /** * Base trait for post commit hooks that want to update the catalog with the * latest table schema and properties. */ trait UpdateCatalogBase extends PostCommitHook with DeltaLogging { protected val table: CatalogTable override def run(spark: SparkSession, txn: CommittedTransaction): Unit = { // There's a potential race condition here, where a newer commit has already triggered // this to run. That's fine. executeOnWrite(spark, txn.postCommitSnapshot) } /** * Used to manually execute an UpdateCatalog hook during a write. */ def executeOnWrite( spark: SparkSession, snapshot: Snapshot ): Unit /** * Update the schema in the catalog based on the provided snapshot. */ def updateSchema(spark: SparkSession, snapshot: Snapshot): Unit /** * Update the properties in the catalog based on the provided snapshot. */ protected def updateProperties(spark: SparkSession, snapshot: Snapshot): Unit /** * Checks if the table schema has changed in the Snapshot with respect to what's stored in * the catalog. */ protected def schemaHasChanged(snapshot: Snapshot, spark: SparkSession): Boolean /** * Checks if the table properties have changed in the Snapshot with respect to what's stored in * the catalog. * * Visible for testing. */ protected[sql] def propertiesHaveChanged( properties: Map[String, String], metadata: Metadata, spark: SparkSession): Boolean protected def shouldRun( spark: SparkSession, snapshot: Snapshot ): Boolean = { if (!spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_UPDATE_CATALOG_ENABLED)) { return false } // Do not execute for path based tables, because they don't exist in the MetaStore if (isPathBasedDeltaTable(table, spark)) return false // Only execute if this is a Delta table if (snapshot.version < 0) return false true } private def isPathBasedDeltaTable(table: CatalogTable, spark: SparkSession): Boolean = { return DeltaTableIdentifier.isDeltaPath(spark, table.identifier) } /** Check if the clustering columns from snapshot doesn't match what's in the table properties. */ protected def clusteringColumnsChanged(snapshot: Snapshot): Boolean = { if (!ClusteredTableUtils.isSupported(snapshot.protocol)) { return false } val currentLogicalClusteringNames = ClusteringColumnInfo.extractLogicalNames(snapshot).mkString(",") val clusterBySpecOpt = ClusterBySpec.fromProperties(table.properties) // Since we don't remove the clustering columns table property, this can't happen. assert(!(currentLogicalClusteringNames.nonEmpty && clusterBySpecOpt.isEmpty)) clusterBySpecOpt.exists(_.columnNames.map(_.toString).mkString(",") != currentLogicalClusteringNames) } /** Update the entry in the Catalog to reflect the latest schema and table properties. */ protected def execute( spark: SparkSession, snapshot: Snapshot): Unit = { recordDeltaOperation(snapshot.deltaLog, "delta.catalog.update") { val properties = snapshot.getProperties.toMap val v = table.properties.get(DeltaConfigs.METASTORE_LAST_UPDATE_VERSION) .flatMap(v => Try(v.toLong).toOption) .getOrElse(-1L) val lastCommitTimestamp = table.properties.get(DeltaConfigs.METASTORE_LAST_COMMIT_TIMESTAMP) .flatMap(v => Try(v.toLong).toOption) .getOrElse(-1L) // If the metastore entry is at an older version and not the timestamp of that version, e.g. // a table can be rm -rf'd and get the same version number with a different timestamp if (v <= snapshot.version || lastCommitTimestamp < snapshot.timestamp) { try { val loggingData = Map( "identifier" -> table.identifier, "snapshotVersion" -> snapshot.version, "snapshotTimestamp" -> snapshot.timestamp, "catalogVersion" -> v, "catalogTimestamp" -> lastCommitTimestamp ) if (schemaHasChanged(snapshot, spark)) { updateSchema(spark, snapshot) recordDeltaEvent( snapshot.deltaLog, "delta.catalog.update.schema", data = loggingData ) } else if (propertiesHaveChanged(properties, snapshot.metadata, spark)) { updateProperties(spark, snapshot) recordDeltaEvent( snapshot.deltaLog, "delta.catalog.update.properties", data = loggingData ) } else if (clusteringColumnsChanged(snapshot)) { // If the clustering columns changed, we'll update the catalog with the new // table properties. updateProperties(spark, snapshot) recordDeltaEvent( snapshot.deltaLog, "delta.catalog.update.clusteringColumns", data = loggingData ) } } catch { case NonFatal(e) => recordDeltaEvent( snapshot.deltaLog, "delta.catalog.update.error", data = Map( "exceptionMsg" -> ExceptionUtils.getMessage(e), "stackTrace" -> ExceptionUtils.getStackTrace(e)) ) logWarning(log"Failed to update the catalog for " + log"${MDC(DeltaLogKeys.TABLE_NAME, table.identifier)} with the latest " + log"table information.", e) } } } } } /** * A post-commit hook that allows us to cache the most recent schema and table properties of a Delta * table in an External Catalog. In addition to the schema and table properties, we also store the * last commit timestamp and version for which we updated the catalog. This prevents us from * updating the MetaStore with potentially stale information. */ case class UpdateCatalog(table: CatalogTable) extends UpdateCatalogBase { override val name: String = "Update Catalog" override def executeOnWrite( spark: SparkSession, snapshot: Snapshot ): Unit = { executeAsync(spark, snapshot) } override protected def schemaHasChanged(snapshot: Snapshot, spark: SparkSession): Boolean = { // We need to check whether the schema in the catalog matches the current schema. // Depending on the schema validation policy, the schema might need to be truncated. // Therefore, we should use what we want to store in the catalog for comparison. val truncationThreshold = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_UPDATE_CATALOG_LONG_FIELD_TRUNCATION_THRESHOLD) val schemaChanged = table.schema != UpdateCatalog.truncateSchemaIfNecessary( snapshot.schema, truncationThreshold)._1 // The table may have been dropped as we're just about to update the information. There is // unfortunately no great way to avoid a race condition, but we do one last check here as // updates may have been queued for some time. schemaChanged && spark.sessionState.catalog.tableExists(table.identifier) } /** * Checks if the table properties have changed in the Snapshot with respect to what's stored in * the catalog. We check to see if our table properties are a subset of what is in the MetaStore * to avoid flip-flopping the information between older and newer versions of Delta. The * assumption here is that newer Delta releases will only add newer table properties and not * remove them. */ override protected[sql] def propertiesHaveChanged( properties: Map[String, String], metadata: Metadata, spark: SparkSession): Boolean = { val propertiesChanged = !properties.forall { case (k, v) => table.properties.get(k) == Some(v) } // The table may have been dropped as we're just about to update the information. There is // unfortunately no great way to avoid a race condition, but we do one last check here as // updates may have been queued for some time. propertiesChanged && spark.sessionState.catalog.tableExists(table.identifier) } override def updateSchema(spark: SparkSession, snapshot: Snapshot): Unit = { UpdateCatalog.replaceTable(spark, snapshot, table) } override protected def updateProperties(spark: SparkSession, snapshot: Snapshot): Unit = { spark.sessionState.catalog.alterTable( table.copy(properties = UpdateCatalog.updatedProperties(snapshot))) } /** * Update the entry in the Catalog to reflect the latest schema and table properties * asynchronously. */ private def executeAsync( spark: SparkSession, snapshot: Snapshot): Unit = { if (!shouldRun(spark, snapshot)) return UpdateCatalog.activeAsyncRequests.incrementAndGet() Future[Unit] { execute(spark, snapshot) }(UpdateCatalog.getOrCreateExecutionContext(spark.sessionState.conf)).onComplete { _ => UpdateCatalog.activeAsyncRequests.decrementAndGet() }(UpdateCatalog.getOrCreateExecutionContext(spark.sessionState.conf)) } } object UpdateCatalog { // Exposed for testing. private[delta] var tp: ExecutionContext = _ // This is the encoding of the database for the Hive MetaStore private val latin1 = Charset.forName("ISO-8859-1") val ERROR_KEY = "delta.catalogUpdateError" val LONG_SCHEMA_ERROR: String = "The schema contains a very long nested field and cannot be " + "stored in the catalog." val NON_LATIN_CHARS_ERROR: String = "The schema contains non-latin encoding characters and " + "cannot be stored in the catalog." val HIVE_METASTORE_NAME = "hive_metastore" private def getOrCreateExecutionContext(conf: SQLConf): ExecutionContext = synchronized { if (tp == null) { tp = ExecutionContext.fromExecutorService(DeltaThreadPool.newDaemonCachedThreadPool( "delta-catalog-update", conf.getConf(DeltaSQLConf.DELTA_UPDATE_CATALOG_THREAD_POOL_SIZE) ) ) } tp } /** Keeps track of active or queued async requests. */ private val activeAsyncRequests = new AtomicInteger(0) /** * Waits for all active and queued updates to finish until the given timeout. Will return true * if all async threads have completed execution. Will return false if not. Exposed for tests. */ def awaitCompletion(timeoutMillis: Long): Boolean = { try { ThreadUtils.runInNewThread("UpdateCatalog-awaitCompletion") { val startTime = System.currentTimeMillis() while (activeAsyncRequests.get() > 0) { Thread.sleep(100) val currentTime = System.currentTimeMillis() if (currentTime - startTime > timeoutMillis) { throw new TimeoutException( s"Timed out waiting for catalog updates to complete after $currentTime ms") } } } true } catch { case _: TimeoutException => false } } /** Replace the table definition in the MetaStore. */ private def replaceTable(spark: SparkSession, snapshot: Snapshot, table: CatalogTable): Unit = { val catalog = spark.sessionState.catalog val qualifiedIdentifier = catalog.qualifyIdentifier(TableIdentifier(table.identifier.table, Some(table.database))) val db = qualifiedIdentifier.database.get val tblName = qualifiedIdentifier.table val truncationThreshold = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_UPDATE_CATALOG_LONG_FIELD_TRUNCATION_THRESHOLD) val (schema, additionalProperties) = truncateSchemaIfNecessary( snapshot.schema, truncationThreshold) // We call the lower level API so that we can actually drop columns. We also assume that // all columns are data columns so that we don't have to deal with partition columns // having to be at the end of the schema, which Hive follows. val catalogName = table.identifier.catalog.getOrElse( spark.sessionState.catalogManager.currentCatalog.name()) if ( (catalogName == UpdateCatalog.HIVE_METASTORE_NAME || catalogName == SESSION_CATALOG_NAME) && catalog.externalCatalog.tableExists(db, tblName)) { catalog.externalCatalog.alterTableDataSchema(db, tblName, schema) } // We have to update the properties anyway with the latest version/timestamp information catalog.alterTable(table.copy(properties = updatedProperties(snapshot) ++ additionalProperties)) } /** Updates our properties map with the version and timestamp information of the snapshot. */ def updatedProperties(snapshot: Snapshot): Map[String, String] = { var newProperties = snapshot.getProperties.toMap ++ Map( DeltaConfigs.METASTORE_LAST_UPDATE_VERSION -> snapshot.version.toString, DeltaConfigs.METASTORE_LAST_COMMIT_TIMESTAMP -> snapshot.timestamp.toString) if (ClusteredTableUtils.isSupported(snapshot.protocol)) { val clusteringColumns = ClusteringColumnInfo.extractLogicalNames(snapshot) val properties = ClusterBySpec.toProperties( ClusterBySpec.fromColumnNames(clusteringColumns)) properties.foreach { case (key, value) => newProperties += (key -> value) } } newProperties } /** * If the schema contains non-latin encoding characters, the schema can become garbled. * We need to truncate the schema in that case. * Also, if any of the fields is longer than `truncationThreshold`, then the schema will be * truncated to an empty schema to avoid corruption. * * @return a tuple of the truncated schema and a map of error messages if any. * The error message is only set if the schema is truncated. Truncation * can happen if the schema is too long or if it contains non-latin characters. */ def truncateSchemaIfNecessary( schema: StructType, truncationThreshold: Long): (StructType, Map[String, String]) = { // Encoders are not threadsafe val encoder = latin1.newEncoder() schema.foreach { f => if (f.dataType.catalogString.length > truncationThreshold) { return (new StructType(), Map(UpdateCatalog.ERROR_KEY -> LONG_SCHEMA_ERROR)) } if (!encoder.canEncode(f.name) || !encoder.canEncode(f.dataType.catalogString)) { return (new StructType(), Map(UpdateCatalog.ERROR_KEY -> NON_LATIN_CHARS_ERROR)) } } (schema, Map.empty) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/implicits/RichSparkClasses.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.implicits import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.plans.QueryPlan import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.rules.{RuleId, UnknownRuleId} import org.apache.spark.sql.catalyst.trees.{AlwaysProcess, TreePatternBits} import org.apache.spark.sql.delta.util.DeltaEncoders import org.apache.spark.sql.types.{ArrayType, MapType, StructField, StructType} trait RichSparkClasses { /** * This implicit class is used to provide helpful methods used throughout the code that are not * provided by Spark-Catalyst's StructType. */ implicit class RichStructType(structType: StructType) { /** * Returns a field in this struct and its child structs, case insensitively. * * If includeCollections is true, this will return fields that are nested in maps and arrays. * * @param fieldNames The path to the field, in order from the root. For example, the column * nested.a.b.c would be Seq("nested", "a", "b", "c"). */ def findNestedFieldIgnoreCase( fieldNames: Seq[String], includeCollections: Boolean = false): Option[StructField] = { val fieldOption = fieldNames.headOption.flatMap { fieldName => structType.find(_.name.equalsIgnoreCase(fieldName)) } fieldOption match { case Some(field) => (fieldNames.tail, field.dataType, includeCollections) match { case (Seq(), _, _) => Some(field) case (names, struct: StructType, _) => struct.findNestedFieldIgnoreCase(names, includeCollections) case (_, _, false) => None // types nested in maps and arrays are not used case (Seq("key"), MapType(keyType, _, _), true) => // return the key type as a struct field to include nullability Some(StructField("key", keyType, nullable = false)) case (Seq("key", names @ _*), MapType(struct: StructType, _, _), true) => struct.findNestedFieldIgnoreCase(names, includeCollections) case (Seq("value"), MapType(_, valueType, isNullable), true) => // return the value type as a struct field to include nullability Some(StructField("value", valueType, nullable = isNullable)) case (Seq("value", names @ _*), MapType(_, struct: StructType, _), true) => struct.findNestedFieldIgnoreCase(names, includeCollections) case (Seq("element"), ArrayType(elementType, isNullable), true) => // return the element type as a struct field to include nullability Some(StructField("element", elementType, nullable = isNullable)) case (Seq("element", names @ _*), ArrayType(struct: StructType, _), true) => struct.findNestedFieldIgnoreCase(names, includeCollections) case _ => None } case _ => None } } } /** * This implicit class is used to provide helpful methods used throughout the code that are not * provided by Spark-Catalyst's LogicalPlan. */ implicit class RichLogicalPlan(plan: LogicalPlan) { /** * Returns the result of running QueryPlan.transformExpressionsUpWithPruning on this node * and all its children. */ def transformAllExpressionsUpWithPruning( cond: TreePatternBits => Boolean, ruleId: RuleId = UnknownRuleId)( rule: PartialFunction[Expression, Expression] ): LogicalPlan = { plan.transformUpWithPruning(cond, ruleId) { case q: QueryPlan[_] => q.transformExpressionsUpWithPruning(cond, ruleId)(rule) } } /** * Returns the result of running QueryPlan.transformExpressionsUp on this node * and all its children. */ def transformAllExpressionsUp( rule: PartialFunction[Expression, Expression]): LogicalPlan = { transformAllExpressionsUpWithPruning(AlwaysProcess.fn, UnknownRuleId)(rule) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/implicits/package.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.util.DeltaEncoders import org.apache.spark.sql.{DataFrame, Dataset, SparkSession} package object implicits extends DeltaEncoders with RichSparkClasses { // Define a few implicit classes to provide the `toDF` method. These classes are not using generic // types to avoid touching Scala reflection. implicit class RichAddFileSeq(files: Seq[AddFile]) { def toDF(spark: SparkSession): DataFrame = spark.implicits.localSeqToDatasetHolder(files).toDF() def toDS(spark: SparkSession): Dataset[AddFile] = spark.implicits.localSeqToDatasetHolder(files).toDS() } implicit class RichStringSeq(strings: Seq[String]) { def toDF(spark: SparkSession): DataFrame = spark.implicits.localSeqToDatasetHolder(strings).toDF() def toDF(spark: SparkSession, colNames: String*): DataFrame = spark.implicits.localSeqToDatasetHolder(strings).toDF(colNames: _*) } implicit class RichIntSeq(ints: Seq[Int]) { def toDF(spark: SparkSession): DataFrame = spark.implicits.localSeqToDatasetHolder(ints).toDF() def toDF(spark: SparkSession, colNames: String*): DataFrame = spark.implicits.localSeqToDatasetHolder(ints).toDF(colNames: _*) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/isolationLevels.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta /** * Trait that defines the level consistency guarantee is going to be provided by * `OptimisticTransaction.commit()`. [[Serializable]] is the most * strict level and [[SnapshotIsolation]] is the least strict one. * * @see [[IsolationLevel.allLevelsInDescOrder]] for all the levels in the descending order * of strictness and [[IsolationLevel.DEFAULT]] for the default table isolation level. */ sealed trait IsolationLevel { override def toString: String = this.getClass.getSimpleName.stripSuffix("$") } /** * This isolation level will ensure serializability between all read and write operations. * Specifically, for write operations, this mode will ensure that the result of * the table will be perfectly consistent with the visible history of operations, that is, * as if all the operations were executed sequentially one by one. */ case object Serializable extends IsolationLevel /** * This isolation level will ensure snapshot isolation consistency guarantee between write * operations only. In other words, if only the write operations are considered, then * there exists a serializable sequence between them that would produce the same result * as seen in the table. However, if both read and write operations are considered, then * there may not exist a serializable sequence that would explain all the observed reads. * * This provides a lower consistency guarantee than [[Serializable]] but a higher * availability than that. For example, unlike [[Serializable]], this level allows an UPDATE * operation to be committed even if there was a concurrent INSERT operation that has already * added data that should have been read by the UPDATE. It will be as if the UPDATE was executed * before the INSERT even if the former was committed after the latter. As a side effect, * the visible history of operations may not be consistent with the * result expected if these operations were executed sequentially one by one. */ case object WriteSerializable extends IsolationLevel /** * This isolation level will ensure that all reads will see a consistent * snapshot of the table and any transactional write will successfully commit only * if the values updated by the transaction have not been changed externally since * the snapshot was read by the transaction. * * This provides a lower consistency guarantee than [[WriteSerializable]] but a higher * availability than that. For example, unlike [[WriteSerializable]], this level allows two * concurrent UPDATE operations reading the same data to be committed successfully as long as * they don't modify the same data. * * Note that for operations that do not modify data in the table, Snapshot isolation is same * as Serializablity. Hence such operations can be safely committed with Snapshot isolation level. */ case object SnapshotIsolation extends IsolationLevel object IsolationLevel { val DEFAULT = WriteSerializable /** All possible isolation levels in descending order of guarantees provided */ val allLevelsInDescOrder: Seq[IsolationLevel] = Seq( Serializable, WriteSerializable, SnapshotIsolation) /** All the valid isolation levels that can be specified as the table isolation level */ val validTableIsolationLevels = Set[IsolationLevel](Serializable, WriteSerializable) def fromString(s: String): IsolationLevel = { allLevelsInDescOrder.find(_.toString.equalsIgnoreCase(s)).getOrElse { throw DeltaErrors.invalidIsolationLevelException(s) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/logging/DeltaLogKeys.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.logging // DeltaLogKey is provided by LogKeyShims (see scala-shims//LogKeyShims.scala) // to handle the difference between Spark versions: // - Spark 4.0: LogKey is a Scala trait with a default `name` implementation // - Spark 4.1: LogKey is a Java interface requiring explicit `name()` implementation /** * Various keys used for mapped diagnostic contexts(MDC) in logging. All structured logging keys * should be defined here for standardization. */ trait DeltaLogKeysBase { case object APP_ID extends DeltaLogKey case object ATTEMPT extends DeltaLogKey case object BATCH_ID extends DeltaLogKey case object BATCH_SIZE extends DeltaLogKey case object CATALOG extends DeltaLogKey case object CLONE_SOURCE_DESC extends DeltaLogKey case object CONFIG extends DeltaLogKey case object CONFIG_KEY extends DeltaLogKey case object COORDINATOR_CONF extends DeltaLogKey case object COORDINATOR_NAME extends DeltaLogKey case object COUNT extends DeltaLogKey case object DATA_FILTER extends DeltaLogKey case object DATE extends DeltaLogKey case object DELTA_COMMIT_INFO extends DeltaLogKey case object DELTA_METADATA extends DeltaLogKey case object DIR extends DeltaLogKey case object DURATION extends DeltaLogKey case object ERROR_ID extends DeltaLogKey case object END_INDEX extends DeltaLogKey case object END_OFFSET extends DeltaLogKey case object END_VERSION extends DeltaLogKey case object ERROR extends DeltaLogKey case object EXCEPTION extends DeltaLogKey case object EXECUTOR_ID extends DeltaLogKey case object EXPR extends DeltaLogKey case object FILE_INDEX extends DeltaLogKey case object FILE_NAME extends DeltaLogKey case object FILE_STATUS extends DeltaLogKey case object FILE_SYSTEM_SCHEME extends DeltaLogKey case object FILTER extends DeltaLogKey case object FILTER2 extends DeltaLogKey case object HOOK_NAME extends DeltaLogKey case object INVARIANT_CHECK_INFO extends DeltaLogKey case object ISOLATION_LEVEL extends DeltaLogKey case object IS_DRY_RUN extends DeltaLogKey case object IS_INIT_SNAPSHOT extends DeltaLogKey case object IS_PATH_TABLE extends DeltaLogKey case object JOB_ID extends DeltaLogKey case object LOG_SEGMENT extends DeltaLogKey case object MAX_SIZE extends DeltaLogKey case object METADATA_ID extends DeltaLogKey case object METADATA_NEW extends DeltaLogKey case object METADATA_OLD extends DeltaLogKey case object METRICS extends DeltaLogKey case object METRIC_NAME extends DeltaLogKey case object MIN_SIZE extends DeltaLogKey case object NUM_ACTIONS extends DeltaLogKey case object NUM_ACTIONS2 extends DeltaLogKey case object NUM_ATTEMPT extends DeltaLogKey case object NUM_BYTES extends DeltaLogKey case object NUM_DIRS extends DeltaLogKey case object NUM_FILES extends DeltaLogKey case object NUM_FILES2 extends DeltaLogKey case object NUM_PARTITIONS extends DeltaLogKey case object NUM_PREDICATES extends DeltaLogKey case object NUM_RECORDS extends DeltaLogKey case object NUM_RECORDS2 extends DeltaLogKey case object NUM_SKIPPED extends DeltaLogKey case object OFFSET extends DeltaLogKey case object OPERATION extends DeltaLogKey case object OP_NAME extends DeltaLogKey case object PARTITION_FILTER extends DeltaLogKey case object PATH extends DeltaLogKey case object PATH2 extends DeltaLogKey case object PATHS extends DeltaLogKey case object PATHS2 extends DeltaLogKey case object PATHS3 extends DeltaLogKey case object PATHS4 extends DeltaLogKey case object PROTOCOL extends DeltaLogKey case object QUERY_ID extends DeltaLogKey case object SCHEMA extends DeltaLogKey case object SCHEMA_DIFF extends DeltaLogKey case object SNAPSHOT extends DeltaLogKey case object START_INDEX extends DeltaLogKey case object START_VERSION extends DeltaLogKey case object STATS extends DeltaLogKey case object STATUS extends DeltaLogKey case object STATUS_MESSAGE extends DeltaLogKey case object SYSTEM_CLASS_NAME extends DeltaLogKey case object TABLE_FEATURES extends DeltaLogKey case object TABLE_ID extends DeltaLogKey case object TABLE_NAME extends DeltaLogKey case object TBL_PROPERTIES extends DeltaLogKey case object THREAD_NAME extends DeltaLogKey case object TIMESTAMP extends DeltaLogKey case object TIMESTAMP2 extends DeltaLogKey case object TIME_MS extends DeltaLogKey case object TIME_STATS extends DeltaLogKey case object TXN_ID extends DeltaLogKey case object URI extends DeltaLogKey case object VACUUM_STATS extends DeltaLogKey case object VERSION extends DeltaLogKey case object VERSION2 extends DeltaLogKey } object DeltaLogKeys extends DeltaLogKeysBase ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/metering/DeltaLogging.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.metering import scala.concurrent.duration._ import scala.util.Try import scala.util.control.NonFatal // scalastyle:off import.ordering.noEmptyLine import com.databricks.spark.util.{DatabricksLogging, OpType, TagDefinition} import com.databricks.spark.util.MetricDefinitions.{EVENT_LOGGING_FAILURE, EVENT_TAHOE} import com.databricks.spark.util.TagDefinitions.{ TAG_OP_TYPE, TAG_TAHOE_ID, TAG_TAHOE_PATH } import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} import org.apache.spark.sql.delta.util.DeltaProgressReporter import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.hadoop.fs.Path import org.apache.spark.SparkThrowable import org.apache.spark.internal.{Logging, MDC, MessageWithContext} /** * Convenience wrappers for logging that include delta specific options and * avoids the need to predeclare all operations. Metrics in Delta should respect the following * conventions: * - Tags should identify the context of the event (which shard, user, table, machine, etc). * - All actions initiated by a user should be wrapped in a recordOperation so we can track usage * latency and failures. If there is a significant (more than a few seconds) subaction like * identifying candidate files, consider nested recordOperation. * - Events should be used to return detailed statistics about usage. Generally these should be * defined with a case class to ease analysis later. * - Events can also be used to record that a particular codepath was hit (i.e. a checkpoint * failure, a conflict, or a specific optimization). * - Both events and operations should be named hierarchically to allow for analysis at different * levels. For example, to look at the latency of all DDL operations we could scan for operations * that match "delta.ddl.%". * * Underneath these functions use the standard usage log reporting defined in * [[com.databricks.spark.util.DatabricksLogging]]. */ trait DeltaLogging extends DeltaProgressReporter with DatabricksLogging { /** * Used to record the occurrence of a single event or report detailed, operation specific * statistics. * * @param path Used to log the path of the delta table when `deltaLog` is null. */ protected def recordDeltaEvent( deltaLog: DeltaLog, opType: String, tags: Map[TagDefinition, String] = Map.empty, data: AnyRef = null, path: Option[Path] = None): Unit = recordFrameProfile("Delta", "recordDeltaEvent") { try { val json = if (data != null) JsonUtils.toJson(data) else "" val tableTags = if (deltaLog != null) { getCommonTags(deltaLog, Try(deltaLog.unsafeVolatileSnapshot.metadata.id).getOrElse(null)) } else if (path.isDefined) { Map(TAG_TAHOE_PATH -> path.get.toString) } else { Map.empty[TagDefinition, String] } recordProductEvent( EVENT_TAHOE, Map((TAG_OP_TYPE: TagDefinition) -> opType) ++ tableTags ++ tags, blob = json) } catch { case NonFatal(e) => recordEvent( EVENT_LOGGING_FAILURE, blob = JsonUtils.toJson( Map("exception" -> e.getMessage, "opType" -> opType, "method" -> "recordDeltaEvent")) ) } } /** * Used to report the duration as well as the success or failure of an operation on a `tahoePath`. */ protected def recordDeltaOperationForTablePath[A]( tablePath: String, opType: String, tags: Map[TagDefinition, String] = Map.empty)( thunk: => A): A = { recordDeltaOperationInternal(Map(TAG_TAHOE_PATH -> tablePath), opType, tags)(thunk) } /** * Used to report the duration as well as the success or failure of an operation on a `deltaLog`. */ protected def recordDeltaOperation[A]( deltaLog: DeltaLog, opType: String, tags: Map[TagDefinition, String] = Map.empty)( thunk: => A): A = { val tableTags: Map[TagDefinition, String] = if (deltaLog != null) { getCommonTags(deltaLog, Try(deltaLog.unsafeVolatileSnapshot.metadata.id).getOrElse(null)) } else { Map.empty } recordDeltaOperationInternal(tableTags, opType, tags)(thunk) } private def recordDeltaOperationInternal[A]( tableTags: Map[TagDefinition, String], opType: String, tags: Map[TagDefinition, String])(thunk: => A): A = { recordOperation( new OpType(opType, ""), extraTags = tableTags ++ tags) { recordFrameProfile("Delta", opType) { thunk } } } /** * Helper method to check invariants in Delta code. Fails when running in tests, records a delta * assertion event and logs a warning otherwise. */ protected def deltaAssert( check: => Boolean, name: String, msg: String, deltaLog: DeltaLog = null, data: AnyRef = null, path: Option[Path] = None) : Unit = { if (DeltaUtils.isTesting) { assert(check, msg) } else if (!check) { recordDeltaEvent( deltaLog = deltaLog, opType = s"delta.assertions.$name", data = data, path = path ) logWarning(msg) } } protected def recordFrameProfile[T](group: String, name: String)(thunk: => T): T = { // future work to capture runtime information ... thunk } private def withDmqTag[T](thunk: => T): T = { thunk } // Extract common tags from the delta log and snapshot. def getCommonTags(deltaLog: DeltaLog, tahoeId: String): Map[TagDefinition, String] = { ( Map( TAG_TAHOE_ID -> tahoeId, TAG_TAHOE_PATH -> Try(deltaLog.dataPath.toString).getOrElse(null) ) ) } /* * Returns error data suitable for logging. * * It will recursively look for the error class and sql state in the cause of the exception. */ def getErrorData(e: Throwable): Map[String, Any] = { var data = Map[String, Any]("exceptionMessage" -> e.getMessage) e condDo { case sparkEx: SparkThrowable if sparkEx.getErrorClass != null && sparkEx.getErrorClass.nonEmpty => data ++= Map( "errorClass" -> sparkEx.getErrorClass, "sqlState" -> sparkEx.getSqlState ) case NonFatal(e) if e.getCause != null => data = getErrorData(e.getCause) } data } } object DeltaLogging { // The opType for delta commit stats. final val DELTA_COMMIT_STATS_OPTYPE = "delta.commit.stats" } /** * A thread-safe token bucket-based throttler implementation with nanosecond accuracy. * * Each instance must be shared across all scopes it should throttle. * For global throttling that means either by extending this class in an `object` or * by creating the instance as a field of an `object`. * * @param bucketSize This corresponds to the largest possible burst without throttling, * in number of executions. * @param tokenRecoveryInterval Time between two tokens being added back to the bucket. * This is reciprocal of the long-term average unthrottled rate. * * Example: With a bucket size of 100 and a recovery interval of 1s, we could log up to 100 events * in under a second without throttling, but at that point the bucket is exhausted and we only * regain the ability to log more events at 1 event per second. If we log less than 1 event/s * the bucket will slowly refill until it's back at 100. * Either way, we can always log at least 1 event/s. */ class LogThrottler( val bucketSize: Int = 100, val tokenRecoveryInterval: FiniteDuration = 1.second, val timeSource: NanoTimeTimeSource = SystemNanoTimeSource) extends Logging { private var remainingTokens = bucketSize private var nextRecovery: DeadlineWithTimeSource = DeadlineWithTimeSource.now(timeSource) + tokenRecoveryInterval private var numSkipped: Long = 0 /** * Run `thunk` as long as there are tokens remaining in the bucket, * otherwise skip and remember number of skips. * * The argument to `thunk` is how many previous invocations have been skipped since the last time * an invocation actually ran. * * Note: This method is `synchronized`, so it is concurrency safe. * However, that also means no heavy-lifting should be done as part of this * if the throttler is shared between concurrent threads. * This also means that the synchronized block of the `thunk` that *does* execute will still * hold up concurrent `thunk`s that will actually get rejected once they hold the lock. * This is fine at low concurrency/low recovery rates. But if we need this to be more efficient at * some point, we will need to decouple the check from the `thunk` execution. */ def throttled(thunk: Long => Unit): Unit = this.synchronized { tryRecoverTokens() if (remainingTokens > 0) { thunk(numSkipped) numSkipped = 0 remainingTokens -= 1 } else { numSkipped += 1L } } /** * Same as [[throttled]] but turns the number of skipped invocations into a logging message * that can be appended to item being logged in `thunk`. */ def throttledWithSkippedLogMessage(thunk: MessageWithContext => Unit): Unit = { this.throttled { numSkipped => val skippedStr = if (numSkipped != 0L) { log" [${MDC(DeltaLogKeys.NUM_SKIPPED, numSkipped)} similar messages were skipped.]" } else { log"" } thunk(skippedStr) } } /** * Try to recover tokens, if the rate allows. * * Only call from within a `this.synchronized` block! */ private def tryRecoverTokens(): Unit = { try { // Doing it one-by-one is a bit inefficient for long periods, but it's easy to avoid jumps // and rounding errors this way. The inefficiency shouldn't matter as long as the bucketSize // isn't huge. while (remainingTokens < bucketSize && nextRecovery.isOverdue()) { remainingTokens += 1 nextRecovery += tokenRecoveryInterval } if (remainingTokens == bucketSize && (DeadlineWithTimeSource.now(timeSource) - nextRecovery) > tokenRecoveryInterval) { // Reset the recovery time, so we don't accumulate infinite recovery while nothing is // going on. nextRecovery = DeadlineWithTimeSource.now(timeSource) + tokenRecoveryInterval } } catch { case _: IllegalArgumentException => // Adding FiniteDuration throws IllegalArgumentException instead of wrapping on overflow. // Given that this happens every ~300 years, we can afford some non-linearity here, // rather than taking the effort to properly work around that. nextRecovery = DeadlineWithTimeSource(Duration(-Long.MaxValue, NANOSECONDS), timeSource) } } } /** * This is essentially the same as Scala's [[Deadline]], * just with a custom source of nanoTime so it can actually be tested properly. */ case class DeadlineWithTimeSource( time: FiniteDuration, timeSource: NanoTimeTimeSource = SystemNanoTimeSource) { // Only implemented the methods LogThrottler actually needs for now. /** * Return a deadline advanced (i.e., moved into the future) by the given duration. */ def +(other: FiniteDuration): DeadlineWithTimeSource = copy(time = time + other) /** * Calculate time difference between this and the other deadline, where the result is directed * (i.e., may be negative). */ def -(other: DeadlineWithTimeSource): FiniteDuration = time - other.time /** * Determine whether the deadline lies in the past at the point where this method is called. */ def isOverdue(): Boolean = (time.toNanos - timeSource.nanoTime()) <= 0 } object DeadlineWithTimeSource { /** * Construct a deadline due exactly at the point where this method is called. Useful for then * advancing it to obtain a future deadline, or for sampling the current time exactly once and * then comparing it to multiple deadlines (using subtraction). */ def now(timeSource: NanoTimeTimeSource = SystemNanoTimeSource): DeadlineWithTimeSource = DeadlineWithTimeSource(Duration(timeSource.nanoTime(), NANOSECONDS), timeSource) } /** Generalisation of [[System.nanoTime()]]. */ private[delta] trait NanoTimeTimeSource { def nanoTime(): Long } private[delta] object SystemNanoTimeSource extends NanoTimeTimeSource { override def nanoTime(): Long = System.nanoTime() } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/metering/ScanReport.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.metering import org.apache.spark.sql.delta.stats.DataSize import com.fasterxml.jackson.databind.annotation.JsonDeserialize case class ScanReport( tableId: String, path: String, scanType: String, deltaDataSkippingType: String, partitionFilters: Seq[String], dataFilters: Seq[String], partitionLikeDataFilters: Seq[String], rewrittenPartitionLikeDataFilters: Seq[String], unusedFilters: Seq[String], size: Map[String, DataSize], @JsonDeserialize(contentAs = classOf[java.lang.Long]) metrics: Map[String, Long], @JsonDeserialize(contentAs = classOf[java.lang.Long]) versionScanned: Option[Long], annotations: Map[String, Long], usedPartitionColumns: Seq[String], numUsedPartitionColumns: Long, allPartitionColumns: Seq[String], numAllPartitionColumns: Long, // Number of output rows from parent filter node if it is available and has the same // predicates as dataFilters. @JsonDeserialize(contentAs = classOf[java.lang.Long]) parentFilterOutputRows: Option[Long]) object ScanReport { // Several of the ScanReport fields are only relevant for certain types of delta scans. // Provide an alternative constructor for callers that don't need to set those fields. // scalastyle:off argcount def apply( tableId: String, path: String, scanType: String, partitionFilters: Seq[String], partitionLikeDataFilters: Seq[String], rewrittenPartitionLikeDataFilters: Seq[String], dataFilters: Seq[String], unusedFilters: Seq[String], size: Map[String, DataSize], metrics: Map[String, Long], versionScanned: Option[Long], annotations: Map[String, Long], parentFilterOutputRows: Option[Long] ): ScanReport = { // scalastyle:on ScanReport( tableId = tableId, path = path, scanType = scanType, deltaDataSkippingType = "", partitionFilters = partitionFilters, dataFilters = dataFilters, partitionLikeDataFilters = partitionLikeDataFilters, rewrittenPartitionLikeDataFilters = rewrittenPartitionLikeDataFilters, unusedFilters = unusedFilters, size = size, metrics = metrics, versionScanned = versionScanned, annotations = annotations, usedPartitionColumns = Nil, numUsedPartitionColumns = 0L, allPartitionColumns = Nil, numAllPartitionColumns = 0L, parentFilterOutputRows = parentFilterOutputRows) } // Similar as above, but without parentFilterOutputRows def apply( tableId: String, path: String, scanType: String, partitionFilters: Seq[String], dataFilters: Seq[String], partitionLikeDataFilters: Seq[String], rewrittenPartitionLikeDataFilters: Seq[String], unusedFilters: Seq[String], size: Map[String, DataSize], metrics: Map[String, Long], versionScanned: Option[Long], annotations: Map[String, Long]): ScanReport = { ScanReport( tableId = tableId, path = path, scanType = scanType, partitionFilters = partitionFilters, dataFilters = dataFilters, partitionLikeDataFilters = partitionLikeDataFilters, rewrittenPartitionLikeDataFilters = rewrittenPartitionLikeDataFilters, unusedFilters = unusedFilters, size = size, metrics = metrics, versionScanned = versionScanned, annotations = annotations, parentFilterOutputRows = None) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/metric/IncrementMetric.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.metric import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.TypeCheckResult import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription, Literal, Nondeterministic, UnaryExpression} import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} import org.apache.spark.sql.catalyst.expressions.codegen.Block._ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.util.TypeUtils import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{BooleanType, DataType} /** * IncrementMetric is used to count the number of rows passing through it. It can be used to * wrap a child expression to count the number of rows. Its currently only accessible via the Scala * DSL. * * For example, consider the following expression returning a string literal: * If(SomeCondition, * IncrementMetric(Literal("ValueIfTrue"), countTrueMetric), * IncrementMetric(Literal("ValueIfFalse"), countFalseMetric)) * * The SQLMetric `countTrueMetric` would be incremented whenever the condition `SomeCondition` is * true, and conversely `countFalseMetric` would be incremented whenever the condition is false. * * The expression does not really compute anything, and merely forwards the value computed by the * child expression. * * It is marked as non deterministic to ensure that it retains strong affinity with the `child` * expression, so as to accurately update the `metric`. * * It takes the following parameters: * @param child is the actual expression to call. * @param metric is the SQLMetric to increment. */ @ExpressionDescription( usage = "_FUNC_(expr, metric) - Returns `expr` as is, while incrementing metric.") case class IncrementMetric(child: Expression, metric: SQLMetric) extends UnaryExpression with Nondeterministic { override def nullable: Boolean = child.nullable override def dataType: DataType = child.dataType override protected def initializeInternal(partitionIndex: Int): Unit = {} override def toString: String = child.toString override def prettyName: String = "increment_metric" override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { // codegen for children expressions val eval = child.genCode(ctx) val metricRef = ctx.addReferenceObj(metric.name.getOrElse("metric"), metric) eval.copy(code = code"""$metricRef.add(1L);""" + eval.code) } override def evalInternal(input: InternalRow): Any = { metric.add(1L) child.eval(input) } override protected def withNewChildInternal(newChild: Expression): IncrementMetric = copy(child = newChild) } /** * ConditionalIncrementMetric is used to count the number of rows passing through it based on * a condition. It can be used to wrap a child expression to count the number of rows only when * the condition is true. Its currently only accessible via the Scala DSL. * * For example, consider the following expression: * ConditionalIncrementMetric(Literal("SomeValue"), GreaterThan(col("count"), Literal(10)), * countMetric) * * The SQLMetric `countMetric` would be incremented whenever the condition is true. * Note: Be careful about nullability! The metric will not be incremented if the condition * evaluates to NULL. * When trying to invert a condition, use `condition = false` instead of `not condition`, * if the metrics is supposed to be incremented if condition was NULL. * * The Expression returns the value computed by the child expression. * * It is marked as non deterministic to ensure that it retains strong affinity with the `child` * expression, and is not optimized out, so as to accurately update the `metric`. * * It takes the following parameters: * @param child is the actual expression to call. * @param condition is the Boolean expression that determines whether to increment the metric. * @param metric is the SQLMetric to increment. */ @ExpressionDescription( usage = "_FUNC_(expr, condition, metric) " + "- Returns `expr` as is, while incrementing metric when condition is true.") case class ConditionalIncrementMetric(child: Expression, condition: Expression, metric: SQLMetric) extends Expression with Nondeterministic { override def checkInputDataTypes(): TypeCheckResult = { if (condition.dataType != BooleanType) { TypeCheckResult.DataTypeMismatch( errorSubClass = "UNEXPECTED_INPUT_TYPE", messageParameters = Map( "paramIndex" -> "second", "requiredType" -> TypeUtils.toSQLType(BooleanType), "inputSql" -> TypeUtils.toSQLExpr(condition), "inputType" -> TypeUtils.toSQLType(condition.dataType) ) ) } else { TypeCheckResult.TypeCheckSuccess } } override def nullable: Boolean = child.nullable override def dataType: DataType = child.dataType override protected def initializeInternal(partitionIndex: Int): Unit = {} override def toString: String = s"conditional_increment_metric($child, $condition)" override def prettyName: String = "conditional_increment_metric" override def children: Seq[Expression] = Seq(child, condition) override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { val childEval = child.genCode(ctx) val conditionEval = condition.genCode(ctx) val metricRef = ctx.addReferenceObj(metric.name.getOrElse("metric"), metric) val incrementCode = code""" |${conditionEval.code} |if (!${conditionEval.isNull} && ${conditionEval.value}) { | $metricRef.add(1L); |} |""".stripMargin childEval.copy(code = incrementCode + childEval.code) } override def evalInternal(input: InternalRow): Any = { val conditionResult = condition.eval(input) if (conditionResult != null && conditionResult.asInstanceOf[Boolean]) { metric.add(1L) } child.eval(input) } override protected def withNewChildrenInternal( newChildren: IndexedSeq[Expression]): ConditionalIncrementMetric = { require(newChildren.length == 2, "ConditionalIncrementMetric requires exactly 2 children") copy(child = newChildren(0), condition = newChildren(1)) } } /** * Optimization rule that simplifies ConditionalIncrementMetric expressions with constant * conditions. */ object OptimizeConditionalIncrementMetric extends Rule[LogicalPlan] { private def isEnabled: Boolean = SQLConf.get.getConf(DeltaSQLConf.DELTA_OPTIMIZE_CONDITIONAL_INCREMENT_METRIC_ENABLED) override def apply(plan: LogicalPlan): LogicalPlan = if (isEnabled) { plan.transformAllExpressionsWithSubqueries { case ConditionalIncrementMetric(child, Literal(true, BooleanType), metric) => // Always true condition: convert to regular IncrementMetric IncrementMetric(child, metric) case ConditionalIncrementMetric(child, Literal(false, BooleanType), metric) => // Always false condition: remove metric logic, keep only child child case ConditionalIncrementMetric(child, Literal(null, BooleanType), metric) => // Null condition: remove metric logic, keep only child child } } else { plan } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/optimizablePartitionExpressions.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.OptimizablePartitionExpression._ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.expressions.{Cast, DateFormatClass, DayOfMonth, Expression, Hour, IsNull, Literal, Month, Or, Substring, TruncDate, TruncTimestamp, UnixTimestamp, Year} import org.apache.spark.sql.catalyst.util.quoteIfNeeded import org.apache.spark.sql.types.{DateType, StringType, TimestampType} /** * Defines rules to convert a data filter to a partition filter for a special generation expression * of a partition column. * * Note: * - This may be shared cross multiple `SparkSession`s, implementations should not store any * state (such as expressions) referring to a specific `SparkSession`. * - Partition columns may have different behaviors than data columns. For example, writing an empty * string to a partition column would become `null` (SPARK-24438). We need to pay attention to * these slight behavior differences and make sure applying the auto generated partition filters * would still return the same result as if they were not applied. */ sealed trait OptimizablePartitionExpression { /** * Assume we have a partition column `part`, and a data column `col`. Return a partition filter * based on `part` for a data filter `col < lit`. */ def lessThan(lit: Literal): Option[Expression] = None /** * Assume we have a partition column `part`, and a data column `col`. Return a partition filter * based on `part` for a data filter `col <= lit`. */ def lessThanOrEqual(lit: Literal): Option[Expression] = None /** * Assume we have a partition column `part`, and a data column `col`. Return a partition filter * based on `part` for a data filter `col = lit`. */ def equalTo(lit: Literal): Option[Expression] = None /** * Assume we have a partition column `part`, and a data column `col`. Return a partition filter * based on `part` for a data filter `col > lit`. */ def greaterThan(lit: Literal): Option[Expression] = None /** * Assume we have a partition column `part`, and a data column `col`. Return a partition filter * based on `part` for a data filter `col >= lit`. */ def greaterThanOrEqual(lit: Literal): Option[Expression] = None /** * Assume we have a partition column `part`, and a data column `col`. Return a partition filter * based on `part` for a data filter `col IS NULL`. */ def isNull(): Option[Expression] = None } object OptimizablePartitionExpression { /** Provide a convenient method to convert a string to a column expression */ implicit class ColumnExpression(val colName: String) extends AnyVal { // This will always be a top level column so quote it if necessary def toPartCol: Expression = Column(quoteIfNeeded(colName)).expr } } /** The rules for the generation expression `CAST(col AS DATE)`. */ case class DatePartitionExpr(partitionColumn: String) extends OptimizablePartitionExpression { override def lessThan(lit: Literal): Option[Expression] = { // As the partition column has truncated information, we need to turn "<" to "<=". lessThanOrEqual(lit) } override def lessThanOrEqual(lit: Literal): Option[Expression] = { val expr = lit.dataType match { case TimestampType => Some(partitionColumn.toPartCol <= Cast(lit, DateType)) case DateType => Some(partitionColumn.toPartCol <= lit) case _ => None } // to avoid any expression which yields null expr.map(e => Or(e, IsNull(e))) } override def equalTo(lit: Literal): Option[Expression] = { val expr = lit.dataType match { case TimestampType => Some(partitionColumn.toPartCol === Cast(lit, DateType)) case DateType => Some(partitionColumn.toPartCol === lit) case _ => None } // to avoid any expression which yields null expr.map(e => Or(e, IsNull(e))) } override def greaterThan(lit: Literal): Option[Expression] = { // As the partition column has truncated information, we need to turn ">" to ">=". greaterThanOrEqual(lit) } override def greaterThanOrEqual(lit: Literal): Option[Expression] = { val expr = lit.dataType match { case TimestampType => Some(partitionColumn.toPartCol >= Cast(lit, DateType)) case DateType => Some(partitionColumn.toPartCol >= lit) case _ => None } // to avoid any expression which yields null expr.map(e => Or(e, IsNull(e))) } override def isNull(): Option[Expression] = Some(partitionColumn.toPartCol.isNull) } /** * The rules for the generation expression `YEAR(col)`. * * @param yearPart the year partition column name. */ case class YearPartitionExpr(yearPart: String) extends OptimizablePartitionExpression { override def lessThan(lit: Literal): Option[Expression] = { // As the partition column has truncated information, we need to turn "<" to "<=". lessThanOrEqual(lit) } override def lessThanOrEqual(lit: Literal): Option[Expression] = { val expr = lit.dataType match { case TimestampType | DateType => Some(yearPart.toPartCol <= Year(lit)) case _ => None } // to avoid any expression which yields null expr.map(e => Or(e, IsNull(e))) } override def equalTo(lit: Literal): Option[Expression] = { val expr = lit.dataType match { case TimestampType | DateType => Some(yearPart.toPartCol.expr === Year(lit)) case _ => None } // to avoid any expression which yields null expr.map(e => Or(e, IsNull(e))) } override def greaterThan(lit: Literal): Option[Expression] = { // As the partition column has truncated information, we need to turn ">" to ">=". greaterThanOrEqual(lit) } override def greaterThanOrEqual(lit: Literal): Option[Expression] = { val expr = lit.dataType match { case TimestampType | DateType => Some(yearPart.toPartCol >= Year(lit)) case _ => None } // to avoid any expression which yields null expr.map(e => Or(e, IsNull(e))) } override def isNull(): Option[Expression] = Some(yearPart.toPartCol.isNull) } /** * This is a placeholder to catch `month(col)` so that we can merge [[YearPartitionExpr]] and * [[MonthPartitionExpr]]to [[YearMonthDayPartitionExpr]]. * * @param monthPart the month partition column name. */ case class MonthPartitionExpr(monthPart: String) extends OptimizablePartitionExpression /** * This is a placeholder to catch `day(col)` so that we can merge [[YearPartitionExpr]], * [[MonthPartitionExpr]] and [[DayPartitionExpr]] to [[YearMonthDayPartitionExpr]]. * * @param dayPart the day partition column name. */ case class DayPartitionExpr(dayPart: String) extends OptimizablePartitionExpression /** * This is a placeholder to catch `hour(col)` so that we can merge [[YearPartitionExpr]], * [[MonthPartitionExpr]], [[DayPartitionExpr]] and [[HourPartitionExpr]] to * [[YearMonthDayHourPartitionExpr]]. */ case class HourPartitionExpr(hourPart: String) extends OptimizablePartitionExpression /** * Optimize the case that two partition columns uses YEAR and MONTH using the same column, such * as `YEAR(eventTime)` and `MONTH(eventTime)`. * * @param yearPart the year partition column name * @param monthPart the month partition column name */ case class YearMonthPartitionExpr( yearPart: String, monthPart: String) extends OptimizablePartitionExpression { override def lessThan(lit: Literal): Option[Expression] = { // As the partition column has truncated information, we need to turn "<" to "<=". lessThanOrEqual(lit) } override def lessThanOrEqual(lit: Literal): Option[Expression] = { lit.dataType match { case TimestampType => Some( (yearPart.toPartCol < Year(lit)) || (yearPart.toPartCol === Year(lit) && monthPart.toPartCol <= Month(lit)) ) case _ => None } } override def equalTo(lit: Literal): Option[Expression] = { lit.dataType match { case TimestampType => Some( yearPart.toPartCol === Year(lit) && monthPart.toPartCol === Month(lit) ) case _ => None } } override def greaterThan(lit: Literal): Option[Expression] = { // As the partition column has truncated information, we need to turn ">" to ">=". greaterThanOrEqual(lit) } override def greaterThanOrEqual(lit: Literal): Option[Expression] = { lit.dataType match { case TimestampType => Some( (yearPart.toPartCol > Year(lit)) || (yearPart.toPartCol === Year(lit) && monthPart.toPartCol >= Month(lit)) ) case _ => None } } override def isNull(): Option[Expression] = { // `yearPart` and `monthPart` are derived columns, so they must be `null` when the input column // is `null`. Some(yearPart.toPartCol.isNull && monthPart.toPartCol.isNull) } } /** * Optimize the case that three partition columns uses YEAR, MONTH and DAY using the same column, * such as `YEAR(eventTime)`, `MONTH(eventTime)` and `DAY(eventTime)`. * * @param yearPart the year partition column name * @param monthPart the month partition column name * @param dayPart the day partition column name */ case class YearMonthDayPartitionExpr( yearPart: String, monthPart: String, dayPart: String) extends OptimizablePartitionExpression { override def lessThan(lit: Literal): Option[Expression] = { // As the partition column has truncated information, we need to turn "<" to "<=". lessThanOrEqual(lit) } override def lessThanOrEqual(lit: Literal): Option[Expression] = { lit.dataType match { case TimestampType => Some( (yearPart.toPartCol < Year(lit)) || (yearPart.toPartCol === Year(lit) && monthPart.toPartCol < Month(lit)) || ( yearPart.toPartCol === Year(lit) && monthPart.toPartCol === Month(lit) && dayPart.toPartCol <= DayOfMonth(lit) ) ) case _ => None } } override def equalTo(lit: Literal): Option[Expression] = { lit.dataType match { case TimestampType => Some( yearPart.toPartCol === Year(lit) && monthPart.toPartCol === Month(lit) && dayPart.toPartCol === DayOfMonth(lit)) case _ => None } } override def greaterThan(lit: Literal): Option[Expression] = { // As the partition column has truncated information, we need to turn ">" to ">=". greaterThanOrEqual(lit) } override def greaterThanOrEqual(lit: Literal): Option[Expression] = { lit.dataType match { case TimestampType => Some( (yearPart.toPartCol > Year(lit)) || (yearPart.toPartCol === Year(lit) && monthPart.toPartCol > Month(lit)) || ( yearPart.toPartCol === Year(lit) && monthPart.toPartCol === Month(lit) && dayPart.toPartCol >= DayOfMonth(lit) ) ) case _ => None } } override def isNull(): Option[Expression] = { // `yearPart`, `monthPart` and `dayPart` are derived columns, so they must be `null` when the // input column is `null`. Some(yearPart.toPartCol.isNull && monthPart.toPartCol.isNull && dayPart.toPartCol.isNull) } } /** * Optimize the case that four partition columns uses YEAR, MONTH, DAY and HOUR using the same * column, such as `YEAR(eventTime)`, `MONTH(eventTime)`, `DAY(eventTime)`, `HOUR(eventTime)`. * * @param yearPart the year partition column name * @param monthPart the month partition column name * @param dayPart the day partition column name * @param hourPart the hour partition column name */ case class YearMonthDayHourPartitionExpr( yearPart: String, monthPart: String, dayPart: String, hourPart: String) extends OptimizablePartitionExpression { override def lessThan(lit: Literal): Option[Expression] = { // As the partition column has truncated information, we need to turn "<" to "<=". lessThanOrEqual(lit) } override def lessThanOrEqual(lit: Literal): Option[Expression] = { lit.dataType match { case TimestampType => Some( (yearPart.toPartCol < Year(lit)) || (yearPart.toPartCol === Year(lit) && monthPart.toPartCol < Month(lit)) || ( yearPart.toPartCol === Year(lit) && monthPart.toPartCol === Month(lit) && dayPart.toPartCol < DayOfMonth(lit) ) || ( yearPart.toPartCol === Year(lit) && monthPart.toPartCol === Month(lit) && dayPart.toPartCol === DayOfMonth(lit) && hourPart.toPartCol <= Hour(lit) ) ) case _ => None } } override def equalTo(lit: Literal): Option[Expression] = { lit.dataType match { case TimestampType => Some( yearPart.toPartCol === Year(lit) && monthPart.toPartCol === Month(lit) && dayPart.toPartCol === DayOfMonth(lit) && hourPart.toPartCol === Hour(lit)) case _ => None } } override def greaterThan(lit: Literal): Option[Expression] = { // As the partition column has truncated information, we need to turn ">" to ">=". greaterThanOrEqual(lit) } override def greaterThanOrEqual(lit: Literal): Option[Expression] = { lit.dataType match { case TimestampType => Some( (yearPart.toPartCol > Year(lit)) || (yearPart.toPartCol === Year(lit) && monthPart.toPartCol > Month(lit)) || ( yearPart.toPartCol === Year(lit) && monthPart.toPartCol === Month(lit) && dayPart.toPartCol > DayOfMonth(lit) ) || ( yearPart.toPartCol === Year(lit) && monthPart.toPartCol === Month(lit) && dayPart.toPartCol === DayOfMonth(lit) && hourPart.toPartCol >= Hour(lit) ) ) case _ => None } } override def isNull(): Option[Expression] = { // `yearPart`, `monthPart`, `dayPart` and `hourPart` are derived columns, so they must be `null` // when the input column is `null`. Some(yearPart.toPartCol.isNull && monthPart.toPartCol.isNull && dayPart.toPartCol.isNull && hourPart.toPartCol.isNull) } } /** * The rules for the generation expression `SUBSTRING(col, pos, len)`. Note: * - Writing an empty string to a partition column would become `null` (SPARK-24438) so generated * partition filters always pick up the `null` partition for safety. * - When `pos` is 0, we also support optimizations for comparison operators. When `pos` is not 0, * we only support optimizations for EqualTo. * * @param partitionColumn the partition column name using SUBSTRING in its generation expression. * @param substringPos the `pos` parameter of SUBSTRING in the generation expression. * @param substringLen the `len` parameter of SUBSTRING in the generation expression. */ case class SubstringPartitionExpr( partitionColumn: String, substringPos: Int, substringLen: Int) extends OptimizablePartitionExpression { override def lessThan(lit: Literal): Option[Expression] = { // As the partition column has truncated information, we need to turn "<" to "<=". lessThanOrEqual(lit) } override def lessThanOrEqual(lit: Literal): Option[Expression] = { // Both `pos == 0` and `pos == 1` start from the first char. See UTF8String.substringSQL. if (substringPos == 0 || substringPos == 1) { lit.dataType match { case StringType => Some( partitionColumn.toPartCol.isNull || partitionColumn.toPartCol <= Substring(lit, substringPos, substringLen)) case _ => None } } else { None } } override def equalTo(lit: Literal): Option[Expression] = { lit.dataType match { case StringType => Some( partitionColumn.toPartCol.isNull || partitionColumn.toPartCol === Substring(lit, substringPos, substringLen)) case _ => None } } override def greaterThan(lit: Literal): Option[Expression] = { // As the partition column has truncated information, we need to turn ">" to ">=". greaterThanOrEqual(lit) } override def greaterThanOrEqual(lit: Literal): Option[Expression] = { // Both `pos == 0` and `pos == 1` start from the first char. See UTF8String.substringSQL. if (substringPos == 0 || substringPos == 1) { lit.dataType match { case StringType => Some( partitionColumn.toPartCol.isNull || partitionColumn.toPartCol >= Substring(lit, substringPos, substringLen)) case _ => None } } else { None } } override def isNull(): Option[Expression] = Some(partitionColumn.toPartCol.isNull) } /** * The rules for the generation expression `DATE_FORMAT(col, format)`, such as: * DATE_FORMAT(timestamp, 'yyyy-MM'), DATE_FORMAT(timestamp, 'yyyy-MM-dd-HH') * * @param partitionColumn the partition column name using DATE_FORMAT in its generation expression. * @param format the `format` parameter of DATE_FORMAT in the generation expression. * * unix_timestamp('12345-12', 'yyyy-MM') | unix_timestamp('+12345-12', 'yyyy-MM') * EXCEPTION fail | 327432240000 * CORRECTED null | 327432240000 * LEGACY 327432240000 | null */ case class DateFormatPartitionExpr( partitionColumn: String, format: String) extends OptimizablePartitionExpression { private val partitionColumnUnixTimestamp = UnixTimestamp(partitionColumn.toPartCol, format) private def litUnixTimestamp(lit: Literal): UnixTimestamp = UnixTimestamp(DateFormatClass(lit, format), format) override def lessThan(lit: Literal): Option[Expression] = { // As the partition column has truncated information, we need to turn "<" to "<=". // timestamp + date are truncated to yyyy-MM // timestamp are truncated to yyyy-MM-dd-HH lessThanOrEqual(lit) } override def lessThanOrEqual(lit: Literal): Option[Expression] = { val expr = lit.dataType match { case TimestampType | DateType => Some(partitionColumnUnixTimestamp <= litUnixTimestamp(lit)) case _ => None } // when write and read timeParserPolicy-s are different, UnixTimestamp will yield null // thus e would be null if either of two operands is null, we should not drop the data expr.map(e => Or(e, IsNull(e))) } override def equalTo(lit: Literal): Option[Expression] = { val expr = lit.dataType match { case TimestampType | DateType => Some(partitionColumnUnixTimestamp === litUnixTimestamp(lit)) case _ => None } // when write and read timeParserPolicy-s are different, UnixTimestamp will yield null // thus e would be null if either of two operands is null, we should not drop the data expr.map(e => Or(e, IsNull(e))) } override def greaterThan(lit: Literal): Option[Expression] = { // As the partition column has truncated information, we need to turn ">" to ">=". // timestamp + date are truncated to yyyy-MM // timestamp are truncated to yyyy-MM-dd-HH greaterThanOrEqual(lit) } override def greaterThanOrEqual(lit: Literal): Option[Expression] = { val expr = lit.dataType match { case TimestampType | DateType => Some(partitionColumnUnixTimestamp >= litUnixTimestamp(lit)) case _ => None } // when write and read timeParserPolicy-s are different, UnixTimestamp will yield null // thus e would be null if either of two operands is null, we should not drop the data expr.map(e => Or(e, IsNull(e))) } override def isNull(): Option[Expression] = { Some(partitionColumn.toPartCol.isNull) } } /** The rules for the generation expression `date_trunc(field, col)`. */ case class TimestampTruncPartitionExpr(format: String, partitionColumn: String) extends OptimizablePartitionExpression { override def lessThan(lit: Literal): Option[Expression] = { // As the partition column has truncated information, we need to turn "<" to "<=". lessThanOrEqual(lit) } override def lessThanOrEqual(lit: Literal): Option[Expression] = { val expr = lit.dataType match { case TimestampType => Some(partitionColumn.toPartCol <= TruncTimestamp(format, lit)) case DateType => Some( partitionColumn.toPartCol <= TruncTimestamp(format, Cast(lit, TimestampType))) case _ => None } // to avoid any expression which yields null expr.map(e => Or(e, IsNull(e))) } override def equalTo(lit: Literal): Option[Expression] = { val expr = lit.dataType match { case TimestampType => Some(partitionColumn.toPartCol === TruncTimestamp(format, lit)) case DateType => Some( partitionColumn.toPartCol === TruncTimestamp(format, Cast(lit, TimestampType))) case _ => None } // to avoid any expression which yields null expr.map(e => Or(e, IsNull(e))) } override def greaterThan(lit: Literal): Option[Expression] = { // As the partition column has truncated information, we need to turn ">" to ">=". greaterThanOrEqual(lit) } override def greaterThanOrEqual(lit: Literal): Option[Expression] = { val expr = lit.dataType match { case TimestampType => Some(partitionColumn.toPartCol >= TruncTimestamp(format, lit)) case DateType => Some( partitionColumn.toPartCol >= TruncTimestamp(format, Cast(lit, TimestampType))) case _ => None } // to avoid any expression which yields null expr.map(e => Or(e, IsNull(e))) } override def isNull(): Option[Expression] = Some(partitionColumn.toPartCol.isNull) } /** * The rules for the generation of identity expressions, used for partitioning on a nested column. * Note: * - Writing an empty string to a partition column would become `null` (SPARK-24438) so generated * partition filters always pick up the `null` partition for safety. * * @param partitionColumn the partition column name used in the generation expression. */ case class IdentityPartitionExpr(partitionColumn: String) extends OptimizablePartitionExpression { override def lessThan(lit: Literal): Option[Expression] = { Some(partitionColumn.toPartCol.isNull || partitionColumn.toPartCol < lit) } override def lessThanOrEqual(lit: Literal): Option[Expression] = { Some(partitionColumn.toPartCol.isNull || partitionColumn.toPartCol <= lit) } override def equalTo(lit: Literal): Option[Expression] = { Some(partitionColumn.toPartCol.isNull || partitionColumn.toPartCol === lit) } override def greaterThan(lit: Literal): Option[Expression] = { Some(partitionColumn.toPartCol.isNull || partitionColumn.toPartCol > lit) } override def greaterThanOrEqual(lit: Literal): Option[Expression] = { Some(partitionColumn.toPartCol.isNull || partitionColumn.toPartCol >= lit) } override def isNull(): Option[Expression] = Some(partitionColumn.toPartCol.isNull) } /** * The rules for generation expression that use the function trunc(col, format) such as * trunc(timestamp, 'year'), trunc(date, 'week') and trunc(timestampStr, 'hour') * @param partitionColumn partition column using trunc function in the generation expression * @param format the format that specifies the unit of truncation applied to the partitionColumn */ case class TruncDatePartitionExpr(partitionColumn: String, format: String) extends OptimizablePartitionExpression { override def lessThan(lit: Literal): Option[Expression] = { lessThanOrEqual(lit) } override def lessThanOrEqual(lit: Literal): Option[Expression] = { val expr = lit.dataType match { case TimestampType | DateType | StringType => Some(partitionColumn.toPartCol <= TruncDate(lit, Literal(format))) case _ => None } expr.map(e => Or(e, IsNull(e))) } override def equalTo(lit: Literal): Option[Expression] = { val expr = lit.dataType match { case TimestampType | DateType | StringType => Some(partitionColumn.toPartCol === TruncDate(lit, Literal(format))) case _ => None } expr.map(e => Or(e, IsNull(e))) } override def greaterThan(lit: Literal): Option[Expression] = { greaterThanOrEqual(lit) } override def greaterThanOrEqual(lit: Literal): Option[Expression] = { val expr = lit.dataType match { case TimestampType | DateType | StringType => Some(partitionColumn.toPartCol >= TruncDate(lit, Literal(format))) case _ => None } expr.map(e => Or(e, IsNull(e))) } override def isNull(): Option[Expression] = { Some(partitionColumn.toPartCol.isNull) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/optimizer/RangePartitionIdRewrite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.optimizer import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.expressions.{PartitionerExpr, RangePartitionId} import org.apache.spark.{RangePartitioner, SparkContext} import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{Alias, Ascending, IsNotNull, SortOrder} import org.apache.spark.sql.catalyst.expressions.codegen.LazilyGeneratedOrdering import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project, UnaryNode} import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution.{QueryExecution, SQLExecution} import org.apache.spark.util.MutablePair /** * Rewrites all [[RangePartitionId]] into [[PartitionerExpr]] by running sampling jobs * on the child RDD in order to determine the range boundaries. */ case class RangePartitionIdRewrite(session: SparkSession) extends Rule[LogicalPlan] { import RangePartitionIdRewrite._ private def sampleSizeHint: Int = conf.rangeExchangeSampleSizePerPartition def apply(plan: LogicalPlan): LogicalPlan = plan transformUp { case node: UnaryNode => node.transformExpressionsUp { case RangePartitionId(expr, n) => val aliasedExpr = Alias(expr, "__RPI_child_col__")() val exprAttr = aliasedExpr.toAttribute val planForSampling = Filter(IsNotNull(exprAttr), Project(Seq(aliasedExpr), node.child)) val qeForSampling = new QueryExecution(session, planForSampling) val desc = s"RangePartitionId($expr, $n) sampling" val jobGroupId = session.sparkContext.getLocalProperty(SparkContext.SPARK_JOB_GROUP_ID) withCallSite(session.sparkContext, desc) { SQLExecution.withNewExecutionId(qeForSampling) { withJobGroup(session.sparkContext, jobGroupId, desc) { // The code below is inspired from ShuffleExchangeExec.prepareShuffleDependency() // Internally, RangePartitioner runs a job on the RDD that samples keys to compute // partition bounds. To get accurate samples, we need to copy the mutable keys. val rddForSampling = qeForSampling.toRdd.mapPartitionsInternal { iter => val mutablePair = new MutablePair[InternalRow, Null]() iter.map(row => mutablePair.update(row.copy(), null)) } val sortOrder = SortOrder(exprAttr, Ascending) implicit val ordering = new LazilyGeneratedOrdering(Seq(sortOrder), Seq(exprAttr)) val partitioner = new RangePartitioner(n, rddForSampling, true, sampleSizeHint) PartitionerExpr(expr, partitioner) } } } } } } object RangePartitionIdRewrite { /** * Executes the equivalent [[SparkContext.setJobGroup()]] call, runs the given `body`, * then restores the original jobGroup. */ private def withJobGroup[T]( sparkContext: SparkContext, groupId: String, description: String) (body: => T): T = { val oldJobDesc = sparkContext.getLocalProperty("spark.job.description") val oldGroupId = sparkContext.getLocalProperty("spark.jobGroup.id") val oldJobInterrupt = sparkContext.getLocalProperty("spark.job.interruptOnCancel") sparkContext.setJobGroup(groupId, description, interruptOnCancel = true) try body finally { sparkContext.setJobGroup( oldGroupId, oldJobDesc, Option(oldJobInterrupt).map(_.toBoolean).getOrElse(false)) } } /** * Executes the equivalent setCallSite() call, runs the given `body`, * then restores the original call site. */ private def withCallSite[T](sparkContext: SparkContext, shortCallSite: String)(body: => T): T = { val oldCallSiteShortForm = sparkContext.getLocalProperty("callSite.short") val oldCallSiteLongForm = sparkContext.getLocalProperty("callSite.long") sparkContext.setCallSite(shortCallSite) try body finally { sparkContext.setLocalProperty("callSite.short", oldCallSiteShortForm) sparkContext.setLocalProperty("callSite.long", oldCallSiteLongForm) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/perf/DeltaOptimizedWriterExec.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.perf import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import scala.concurrent.duration.Duration import org.apache.spark.sql.delta.{DeltaErrors, DeltaLog} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.BinPackingUtils // scalastyle:off import.ordering.noEmptyLine import org.apache.spark._ import org.apache.spark.internal.config import org.apache.spark.internal.config.ConfigEntry import org.apache.spark.network.util.ByteUnit import org.apache.spark.rdd.RDD import org.apache.spark.shuffle._ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning import org.apache.spark.sql.execution.{ShuffledRowRDD, SparkPlan, UnaryExecNode} import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics, SQLShuffleReadMetricsReporter, SQLShuffleWriteMetricsReporter} import org.apache.spark.storage._ import org.apache.spark.util.ThreadUtils /** * An execution node which shuffles data to a target output of `DELTA_OPTIMIZE_WRITE_SHUFFLE_BLOCKS` * blocks, hash partitioned on the table partition columns. We group all blocks by their * reducer_id's and bin-pack into `DELTA_OPTIMIZE_WRITE_BIN_SIZE` bins. Then we launch a Spark task * per bin to write out a single file for each bin. * * @param child The execution plan * @param partitionColumns The partition columns of the table. Used for hash partitioning the write * @param deltaLog The DeltaLog for the table. Used for logging only */ case class DeltaOptimizedWriterExec( child: SparkPlan, partitionColumns: Seq[String], @transient deltaLog: DeltaLog ) extends UnaryExecNode with DeltaLogging { override def output: Seq[Attribute] = child.output private lazy val writeMetrics = SQLShuffleWriteMetricsReporter.createShuffleWriteMetrics(sparkContext) private lazy val readMetrics = SQLShuffleReadMetricsReporter.createShuffleReadMetrics(sparkContext) override lazy val metrics: Map[String, SQLMetric] = Map( "dataSize" -> SQLMetrics.createSizeMetric(sparkContext, "data size") ) ++ readMetrics ++ writeMetrics private lazy val childNumPartitions = child.execute().getNumPartitions private lazy val numPartitions: Int = { val targetShuffleBlocks = getConf(DeltaSQLConf.DELTA_OPTIMIZE_WRITE_SHUFFLE_BLOCKS) math.min( math.max(targetShuffleBlocks / childNumPartitions, 1), getConf(DeltaSQLConf.DELTA_OPTIMIZE_WRITE_MAX_SHUFFLE_PARTITIONS)) } @transient private var cachedShuffleRDD: ShuffledRowRDD = _ @transient private lazy val mapTracker = SparkEnv.get.mapOutputTracker /** Creates a ShuffledRowRDD for facilitating the shuffle in the map side. */ private def getShuffleRDD: ShuffledRowRDD = { if (cachedShuffleRDD == null) { val resolver = org.apache.spark.sql.catalyst.analysis.caseInsensitiveResolution val saltedPartitioning = HashPartitioning( partitionColumns.map(p => output.find(o => resolver(p, o.name)).getOrElse( throw DeltaErrors.failedFindPartitionColumnInOutputPlan(p))), numPartitions) val shuffledRDD = ShuffleExchangeExec(saltedPartitioning, child).execute().asInstanceOf[ShuffledRowRDD] cachedShuffleRDD = shuffledRDD } cachedShuffleRDD } private def computeBins(): Array[List[(BlockManagerId, ArrayBuffer[(BlockId, Long, Int)])]] = { // Get all shuffle information val shuffleStats = getShuffleStats() // Group by blockId instead of block manager val blockInfo = shuffleStats.flatMap { case (bmId, blocks) => blocks.map { case (blockId, size, index) => (blockId, (bmId, size, index)) } }.toMap val maxBinSize = ByteUnit.BYTE.convertFrom(getConf(DeltaSQLConf.DELTA_OPTIMIZE_WRITE_BIN_SIZE), ByteUnit.MiB) val bins = shuffleStats.toSeq.flatMap(_._2).groupBy(_._1.asInstanceOf[ShuffleBlockId].reduceId) .flatMap { case (_, blocks) => BinPackingUtils.binPackBySize[(BlockId, Long, Int), BlockId]( blocks, _._2, // size _._1, // blockId maxBinSize) } bins .map { bin => var binSize = 0L val blockLocations = new mutable.HashMap[BlockManagerId, ArrayBuffer[(BlockId, Long, Int)]]() for (blockId <- bin) { val (bmId, size, index) = blockInfo(blockId) binSize += size val blocksAtBM = blockLocations.getOrElseUpdate( bmId, new ArrayBuffer[(BlockId, Long, Int)]()) blocksAtBM.append((blockId, size, index)) } (binSize, blockLocations.toList) } .toArray .sortBy(_._1)(Ordering[Long].reverse) // submit largest blocks first .map(_._2) } /** Performs the shuffle before the write, so that we can bin-pack output data. */ private def getShuffleStats(): Array[(BlockManagerId, collection.Seq[(BlockId, Long, Int)])] = { val dep = getShuffleRDD.dependency // Gets the shuffle output stats def getStats() = mapTracker.getMapSizesByExecutorId( dep.shuffleId, 0, Int.MaxValue, 0, numPartitions).toArray // Executes the shuffle map stage in case we are missing output stats def awaitShuffleMapStage(): Unit = { assert(dep != null, "Shuffle dependency should not be null") // hack to materialize the shuffle files in a fault tolerant way ThreadUtils.awaitResult(sparkContext.submitMapStage(dep), Duration.Inf) } try { val res = getStats() if (res.isEmpty) awaitShuffleMapStage() getStats() } catch { case e: FetchFailedException => logWarning(log"Failed to fetch shuffle blocks for the optimized writer. Retrying", e) awaitShuffleMapStage() getStats() } } override def doExecute(): RDD[InternalRow] = { // Single partitioned tasks can simply be written if (childNumPartitions <= 1) return child.execute() val shuffledRDD = getShuffleRDD val partitions = computeBins() recordDeltaEvent(deltaLog, "delta.optimizeWrite.planned", data = Map( "originalPartitions" -> childNumPartitions, "outputPartitions" -> partitions.length, "shufflePartitions" -> numPartitions, "numShuffleBlocks" -> getConf(DeltaSQLConf.DELTA_OPTIMIZE_WRITE_SHUFFLE_BLOCKS), "binSize" -> getConf(DeltaSQLConf.DELTA_OPTIMIZE_WRITE_BIN_SIZE), "maxShufflePartitions" -> getConf(DeltaSQLConf.DELTA_OPTIMIZE_WRITE_MAX_SHUFFLE_PARTITIONS) ) ) new DeltaOptimizedWriterRDD( sparkContext, shuffledRDD.dependency, readMetrics, new OptimizedWriterBlocks(partitions)) } private def getConf[T](entry: ConfigEntry[T]): T = { conf.getConf(entry) } override protected def withNewChildInternal(newChild: SparkPlan): DeltaOptimizedWriterExec = copy(child = newChild) } /** * A wrapper class to make the blocks non-serializable. If we serialize the blocks and send them to * the executors, it may cause memory problems. * NOTE!!!: By wrapping the Array in a non-serializable class we enforce that the field needs to * be transient, and gives us extra security against a developer making a mistake. */ class OptimizedWriterBlocks( val bins: Array[List[(BlockManagerId, ArrayBuffer[(BlockId, Long, Int)])]]) /** * A specialized implementation similar to `ShuffledRowRDD`, where a partition reads a prepared * set of shuffle blocks. */ private class DeltaOptimizedWriterRDD( @transient sparkContext: SparkContext, var dep: ShuffleDependency[Int, _, InternalRow], metrics: Map[String, SQLMetric], @transient blocks: OptimizedWriterBlocks) extends RDD[InternalRow](sparkContext, Seq(dep)) with DeltaLogging { override def getPartitions: Array[Partition] = Array.tabulate(blocks.bins.length) { i => ShuffleBlockRDDPartition(i, blocks.bins(i)) } override def compute(split: Partition, context: TaskContext): Iterator[InternalRow] = { val tempMetrics = context.taskMetrics().createTempShuffleReadMetrics() val sqlMetricsReporter = new SQLShuffleReadMetricsReporter(tempMetrics, metrics) val blocks = if (context.stageAttemptNumber() > 0) { // We lost shuffle blocks, so we need to now get new manager addresses val executorTracker = SparkEnv.get.mapOutputTracker val oldBlockLocations = split.asInstanceOf[ShuffleBlockRDDPartition].blocks // assumes we bin-pack by reducerId val reducerId = oldBlockLocations.head._2.head._1.asInstanceOf[ShuffleBlockId].reduceId // Get block addresses val newLocations = executorTracker.getMapSizesByExecutorId(dep.shuffleId, reducerId) .flatMap { case (bmId, newBlocks) => newBlocks.map { blockInfo => (blockInfo._3, (bmId, blockInfo)) } }.toMap val blockLocations = new mutable.HashMap[BlockManagerId, ArrayBuffer[(BlockId, Long, Int)]]() oldBlockLocations.foreach { case (_, oldBlocks) => oldBlocks.foreach { oldBlock => val (bmId, blockInfo) = newLocations(oldBlock._3) val blocksAtBM = blockLocations.getOrElseUpdate(bmId, new ArrayBuffer[(BlockId, Long, Int)]()) blocksAtBM.append(blockInfo) } } blockLocations.iterator } else { split.asInstanceOf[ShuffleBlockRDDPartition].blocks.iterator } val reader = new OptimizedWriterShuffleReader( dep, context, blocks, sqlMetricsReporter) reader.read().map(_._2) } override def clearDependencies(): Unit = { super.clearDependencies() dep = null } } /** The list of blocks that need to be read by a partition of the ShuffleBlockRDD. */ private case class ShuffleBlockRDDPartition( index: Int, blocks: List[(BlockManagerId, ArrayBuffer[(BlockId, Long, Int)])]) extends Partition /** A simplified implementation of the `BlockStoreShuffleReader` for reading shuffle blocks. */ private class OptimizedWriterShuffleReader( dep: ShuffleDependency[Int, _, InternalRow], context: TaskContext, blocks: Iterator[(BlockManagerId, ArrayBuffer[(BlockId, Long, Int)])], readMetrics: ShuffleReadMetricsReporter) extends ShuffleReader[Int, InternalRow] { /** Read the combined key-values for this reduce task */ override def read(): Iterator[Product2[Int, InternalRow]] = { val wrappedStreams = new ShuffleBlockFetcherIterator( context, SparkEnv.get.blockManager.blockStoreClient, SparkEnv.get.blockManager, SparkEnv.get.mapOutputTracker, blocks, SparkEnv.get.serializerManager.wrapStream, // Note: we use getSizeAsMb when no suffix is provided for backwards compatibility SparkEnv.get.conf.getSizeAsMb("spark.reducer.maxSizeInFlight", "48m") * 1024 * 1024, SparkEnv.get.conf.getInt("spark.reducer.maxReqsInFlight", Int.MaxValue), SparkEnv.get.conf.get(config.REDUCER_MAX_BLOCKS_IN_FLIGHT_PER_ADDRESS), SparkEnv.get.conf.get(config.MAX_REMOTE_BLOCK_SIZE_FETCH_TO_MEM), SparkEnv.get.conf.get(config.SHUFFLE_MAX_ATTEMPTS_ON_NETTY_OOM), SparkEnv.get.conf.getBoolean("spark.shuffle.detectCorrupt", true), SparkEnv.get.conf.getBoolean("spark.shuffle.detectCorrupt.useExtraMemory", false), SparkEnv.get.conf.getBoolean("spark.shuffle.checksum.enabled", true), SparkEnv.get.conf.get("spark.shuffle.checksum.algorithm", "ADLER32"), readMetrics, false) val serializerInstance = dep.serializer.newInstance() // Create a key/value iterator for each stream val recordIter = wrappedStreams.flatMap { case (_, wrappedStream) => // Note: the asKeyValueIterator below wraps a key/value iterator inside of a // NextIterator. The NextIterator makes sure that close() is called on the // underlying InputStream when all records have been read. serializerInstance.deserializeStream(wrappedStream).asKeyValueIterator }.asInstanceOf[Iterator[Product2[Int, InternalRow]]] new InterruptibleIterator[Product2[Int, InternalRow]](context, recordIter) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/perf/OptimizeMetadataOnlyDeltaQuery.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.perf import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate._ import org.apache.spark.sql.catalyst.planning.PhysicalOperation import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils} import org.apache.spark.sql.delta.{DeltaColumnMapping, DeltaTable, Snapshot} import org.apache.spark.sql.delta.commands.DeletionVectorUtils.isTableDVFree import org.apache.spark.sql.delta.files.TahoeLogFileIndex import org.apache.spark.sql.delta.stats.DeltaScanGenerator import org.apache.spark.sql.functions._ import org.apache.spark.sql.types._ import java.sql.Date import java.util.Locale /** Optimize COUNT, MIN and MAX expressions on Delta tables. * This optimization is only applied when the following conditions are met: * - The MIN/MAX columns are not nested and data type is supported by the optimization (ByteType, * ShortType, IntegerType, LongType, FloatType, DoubleType, DateType). * - All AddFiles in the Delta Log must have stats on columns used in MIN/MAX expressions, * or the columns must be partitioned, in the latter case it uses partitionValues, a required field. * - Table has no deletion vectors, or query has no MIN/MAX expressions. * - COUNT has no DISTINCT. * - Query has no filters. * - Query has no GROUP BY. * Example of valid query: SELECT COUNT(*), MIN(id), MAX(partition_col) FROM MyDeltaTable */ trait OptimizeMetadataOnlyDeltaQuery extends Logging { def optimizeQueryWithMetadata(plan: LogicalPlan): LogicalPlan = { plan.transformUpWithSubqueries { case agg@MetadataOptimizableAggregate(tahoeLogFileIndex) => createLocalRelationPlan(agg, tahoeLogFileIndex) } } protected def getDeltaScanGenerator(index: TahoeLogFileIndex): DeltaScanGenerator private def createLocalRelationPlan( plan: Aggregate, tahoeLogFileIndex: TahoeLogFileIndex): LogicalPlan = { val aggColumnsNames = Set(extractMinMaxFieldNames(plan).map(_.toLowerCase(Locale.ROOT)) : _*) val (rowCount, columnStats) = extractCountMinMaxFromDeltaLog(tahoeLogFileIndex, aggColumnsNames) def checkStatsExists(attrRef: AttributeReference): Boolean = { columnStats.contains(attrRef.name) && // Avoid StructType, it is not supported by this optimization. // Sanity check only. If reference is nested column it would be GetStructType // instead of AttributeReference. attrRef.references.size == 1 && attrRef.references.head.dataType != StructType } def convertValueIfRequired(attrRef: AttributeReference, value: Any): Any = { if (attrRef.dataType == DateType && value != null) { DateTimeUtils.fromJavaDate(value.asInstanceOf[Date]) } else { value } } val rewrittenAggregationValues = plan.aggregateExpressions.collect { case Alias(AggregateExpression( Count(Seq(Literal(1, _))), Complete, false, None, _), _) if rowCount.isDefined => rowCount.get case Alias(tps@ToPrettyString(AggregateExpression( Count(Seq(Literal(1, _))), Complete, false, None, _), _), _) if rowCount.isDefined => tps.copy(child = Literal(rowCount.get)).eval() case Alias(AggregateExpression( Min(minReference: AttributeReference), Complete, false, None, _), _) if checkStatsExists(minReference) => convertValueIfRequired(minReference, columnStats(minReference.name).min) case Alias(tps@ToPrettyString(AggregateExpression( Min(minReference: AttributeReference), Complete, false, None, _), _), _) if checkStatsExists(minReference) => val v = columnStats(minReference.name).min tps.copy(child = Literal(v)).eval() case Alias(AggregateExpression( Max(maxReference: AttributeReference), Complete, false, None, _), _) if checkStatsExists(maxReference) => convertValueIfRequired(maxReference, columnStats(maxReference.name).max) case Alias(tps@ToPrettyString(AggregateExpression( Max(maxReference: AttributeReference), Complete, false, None, _), _), _) if checkStatsExists(maxReference) => val v = columnStats(maxReference.name).max tps.copy(child = Literal(v)).eval() } if (plan.aggregateExpressions.size == rewrittenAggregationValues.size) { val r = LocalRelation( plan.output, Seq(InternalRow.fromSeq(rewrittenAggregationValues))) r } else { logInfo(log"Query can't be optimized using metadata because stats are missing") plan } } private def extractMinMaxFieldNames(plan: Aggregate): Seq[String] = { plan.aggregateExpressions.collect { case Alias(AggregateExpression( Min(minReference: AttributeReference), _, _, _, _), _) => minReference.name case Alias(AggregateExpression( Max(maxReference: AttributeReference), _, _, _, _), _) => maxReference.name case Alias(ToPrettyString(AggregateExpression( Min(minReference: AttributeReference), _, _, _, _), _), _) => minReference.name case Alias(ToPrettyString(AggregateExpression( Max(maxReference: AttributeReference), _, _, _, _), _), _) => maxReference.name } } /** * Min and max values from Delta Log stats or partitionValues. */ case class DeltaColumnStat(min: Any, max: Any) private def extractCountMinMaxFromStats( deltaScanGenerator: DeltaScanGenerator, lowerCaseColumnNames: Set[String]): (Option[Long], Map[String, DeltaColumnStat]) = { val snapshot = deltaScanGenerator.snapshotToScan // Count - account for deleted rows according to deletion vectors val dvCardinality = coalesce(col("deletionVector.cardinality"), lit(0)) val numLogicalRecords = (col("stats.numRecords") - dvCardinality).as("numLogicalRecords") val filesWithStatsForScan = deltaScanGenerator.filesWithStatsForScan(Nil) // Validate all the files has stats val filesStatsCount = filesWithStatsForScan.select( sum(numLogicalRecords).as("numLogicalRecords"), count(when(col("stats.numRecords").isNull, 1)).as("missingNumRecords"), count(when(col("stats.numRecords") > 0, 1)).as("countNonEmptyFiles")).head // If any numRecords is null, we have incomplete stats; val allRecordsHasStats = filesStatsCount.getAs[Long]("missingNumRecords") == 0 if (!allRecordsHasStats) { return (None, Map.empty) } // the sum agg is either null (for an empty table) or gives an accurate record count. val numRecords = if (filesStatsCount.isNullAt(0)) 0 else filesStatsCount.getLong(0) lazy val numFiles: Long = filesStatsCount.getAs[Long]("countNonEmptyFiles") val dataColumns = snapshot.statCollectionPhysicalSchema.filter(col => lowerCaseColumnNames.contains(col.name.toLowerCase(Locale.ROOT))) // DELETE operations creates AddFile records with 0 rows, and no column stats. // We can safely ignore it since there is no data. lazy val files = filesWithStatsForScan.filter(col("stats.numRecords") > 0) lazy val statsMinMaxNullColumns = files.select(col("stats.*")) val minColName = "minValues" val maxColName = "maxValues" val nullColName = "nullCount" if (dataColumns.isEmpty || dataColumns.size != lowerCaseColumnNames.size || !isTableDVFree(snapshot) // When DV enabled we can't rely on stats values easily || numFiles == 0 || !statsMinMaxNullColumns.columns.contains(minColName) || !statsMinMaxNullColumns.columns.contains(maxColName) || !statsMinMaxNullColumns.columns.contains(nullColName)) { return (Some(numRecords), Map.empty) } // dataColumns can contain columns without stats if dataSkippingNumIndexedCols // has been increased val columnsWithStats = files.select( col(s"stats.$minColName.*"), col(s"stats.$maxColName.*"), col(s"stats.$nullColName.*")) .columns.groupBy(identity).mapValues(_.size) .filter(x => x._2 == 3) // 3: minValues, maxValues, nullCount .map(x => x._1).toSet // Creates a tuple with physical name to avoid recalculating it multiple times val dataColumnsWithStats = dataColumns.map(x => (x, DeltaColumnMapping.getPhysicalName(x))) .filter(x => columnsWithStats.contains(x._2)) val columnsToQuery = dataColumnsWithStats.flatMap { columnAndPhysicalName => val dataType = columnAndPhysicalName._1.dataType val physicalName = columnAndPhysicalName._2 Seq(col(s"stats.$minColName.`$physicalName`").cast(dataType).as(s"min.$physicalName"), col(s"stats.$maxColName.`$physicalName`").cast(dataType).as(s"max.$physicalName"), col(s"stats.$nullColName.`$physicalName`").as(s"null_count.$physicalName")) } ++ Seq(col(s"stats.numRecords").as(s"numRecords")) val minMaxExpr = dataColumnsWithStats.flatMap { columnAndPhysicalName => val physicalName = columnAndPhysicalName._2 // To validate if the column has stats we do two validation: // 1-) COUNT(null_count.columnName) should be equals to numFiles, // since null_count is always non-null. // 2-) The number of files with non-null min/max: // a. count(min.columnName)|count(max.columnName) + // the number of files where all rows are NULL: // b. count of (ISNULL(min.columnName) and null_count.columnName == numRecords) // should be equals to numFiles Seq( s"""case when $numFiles = count(`null_count.$physicalName`) | AND $numFiles = (count(`min.$physicalName`) + sum(case when | ISNULL(`min.$physicalName`) and `null_count.$physicalName` = numRecords | then 1 else 0 end)) | AND $numFiles = (count(`max.$physicalName`) + sum(case when | ISNULL(`max.$physicalName`) AND `null_count.$physicalName` = numRecords | then 1 else 0 end)) | then TRUE else FALSE end as `complete_$physicalName`""".stripMargin, s"min(`min.$physicalName`) as `min_$physicalName`", s"max(`max.$physicalName`) as `max_$physicalName`") } val statsResults = files.select(columnsToQuery: _*).selectExpr(minMaxExpr: _*).head (Some(numRecords), dataColumnsWithStats .filter(x => statsResults.getAs[Boolean](s"complete_${x._2}")) .map { columnAndPhysicalName => val column = columnAndPhysicalName._1 val physicalName = columnAndPhysicalName._2 column.name -> DeltaColumnStat( statsResults.getAs(s"min_$physicalName"), statsResults.getAs(s"max_$physicalName")) }.toMap) } private def extractMinMaxFromPartitionValue( snapshot: Snapshot, lowerCaseColumnNames: Set[String]): Map[String, DeltaColumnStat] = { val partitionedColumns = snapshot.metadata.partitionSchema .filter(col => lowerCaseColumnNames.contains(col.name.toLowerCase(Locale.ROOT))) .map(col => (col, DeltaColumnMapping.getPhysicalName(col))) if (partitionedColumns.isEmpty) { Map.empty } else { val partitionedColumnsValues = partitionedColumns.map { partitionedColumn => val physicalName = partitionedColumn._2 col(s"partitionValues.`$physicalName`") .cast(partitionedColumn._1.dataType).as(physicalName) } val partitionedColumnsAgg = partitionedColumns.flatMap { partitionedColumn => val physicalName = partitionedColumn._2 Seq(min(s"`$physicalName`").as(s"min_$physicalName"), max(s"`$physicalName`").as(s"max_$physicalName")) } val partitionedColumnsQuery = snapshot.allFiles .select(partitionedColumnsValues: _*) .agg(partitionedColumnsAgg.head, partitionedColumnsAgg.tail: _*) .head() partitionedColumns.map { partitionedColumn => val physicalName = partitionedColumn._2 partitionedColumn._1.name -> DeltaColumnStat( partitionedColumnsQuery.getAs(s"min_$physicalName"), partitionedColumnsQuery.getAs(s"max_$physicalName")) }.toMap } } /** * Extract the Count, Min and Max values from Delta Log stats and partitionValues. * The first field is the rows count in the table or `None` if we cannot calculate it from stats * If the column is not partitioned, the values are extracted from stats when it exists. * If the column is partitioned, the values are extracted from partitionValues. */ private def extractCountMinMaxFromDeltaLog( tahoeLogFileIndex: TahoeLogFileIndex, lowerCaseColumnNames: Set[String]): (Option[Long], CaseInsensitiveMap[DeltaColumnStat]) = { val deltaScanGen = getDeltaScanGenerator(tahoeLogFileIndex) val partitionedValues = extractMinMaxFromPartitionValue( deltaScanGen.snapshotToScan, lowerCaseColumnNames) val partitionedColNames = partitionedValues.keySet.map(_.toLowerCase(Locale.ROOT)) val dataColumnNames = lowerCaseColumnNames -- partitionedColNames val (rowCount, columnStats) = extractCountMinMaxFromStats(deltaScanGen, dataColumnNames) (rowCount, CaseInsensitiveMap(columnStats ++ partitionedValues)) } object MetadataOptimizableAggregate { /** Only data type that are stored in stats without any loss of precision are supported. */ def isSupportedDataType(dataType: DataType): Boolean = { // DecimalType is not supported because not all the values are correctly stored // For example -99999999999999999999999999999999999999 in stats is -1e38 (dataType.isInstanceOf[NumericType] && !dataType.isInstanceOf[DecimalType]) || dataType.isInstanceOf[DateType] } private def getAggFunctionOptimizable( aggExpr: AggregateExpression): Option[DeclarativeAggregate] = { aggExpr match { case AggregateExpression( c@Count(Seq(Literal(1, _))), Complete, false, None, _) => Some(c) case AggregateExpression( min@Min(minExpr), Complete, false, None, _) if isSupportedDataType(minExpr.dataType) => Some(min) case AggregateExpression( max@Max(maxExpr), Complete, false, None, _) if isSupportedDataType(maxExpr.dataType) => Some(max) case _ => None } } private def isStatsOptimizable(aggExprs: Seq[Expression]): Boolean = aggExprs.forall { case Alias(aggExpr: AggregateExpression, _) => getAggFunctionOptimizable(aggExpr).isDefined case Alias(ToPrettyString(aggExpr: AggregateExpression, _), _) => getAggFunctionOptimizable(aggExpr).isDefined case _ => false } private def fieldsAreAttributeReference(fields: Seq[NamedExpression]): Boolean = fields.forall { // Fields should be AttributeReference to avoid getting the incorrect column name // from stats when we create the Local Relation, example // SELECT MAX(Column2) FROM (SELECT Column1 AS Column2 FROM TableName) // the AggregateExpression contains a reference to Column2, instead of Column1 case _: AttributeReference => true case _ => false } def unapply(plan: Aggregate): Option[TahoeLogFileIndex] = { // GROUP BY is not supports. All AggregateExpression must be stats optimizable. if (plan.groupingExpressions.nonEmpty || plan.aggregateExpressions.isEmpty || !isStatsOptimizable(plan.aggregateExpressions)) { return None } plan.child match { case PhysicalOperation(fields, Nil, DeltaTable(fileIndex: TahoeLogFileIndex)) if fileIndex.partitionFilters.isEmpty && fieldsAreAttributeReference(fields) => Some(fileIndex) case DeltaTable(fileIndex: TahoeLogFileIndex) if fileIndex.partitionFilters.isEmpty => // When all columns are selected, there are no Project/PhysicalOperation Some(fileIndex) case _ => None } } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/redirect/TableRedirect.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.redirect import java.util.{Locale, UUID} import scala.reflect.ClassTag import scala.util.DynamicVariable // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.{ DeltaConfig, DeltaConfigs, DeltaErrors, DeltaLog, DeltaOperations, RedirectReaderWriterFeature, RedirectWriterOnlyFeature, Snapshot } import org.apache.spark.sql.delta.DeltaLog.logPathFor import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.sources.DeltaSQLConf.ENABLE_TABLE_REDIRECT_FEATURE import org.apache.spark.sql.delta.util.JsonUtils import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.scala.DefaultScalaModule import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.catalog.SessionCatalog import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ /** * The table redirection feature includes specific states that manage the behavior of Delta clients * during various stages of redirection. These states ensure query result consistency and prevent * data loss. There are four states: * 0. NO-REDIRECT: Indicates that table redirection is not enabled. * 1. ENABLE-REDIRECT-IN-PROGRESS: Table redirection is being enabled. Only read-only queries are * allowed on the source table, while all write and metadata * transactions are aborted. * 2. REDIRECT-READY: The redirection setup is complete, and all queries on the source table are * routed to the destination table. * 3. DROP-REDIRECT-IN-PROGRESS: Table redirection is being disabled. Only read-only queries are * allowed on the destination table, with all write and metadata * transactions aborted. * The valid procedures of state transition are: * 0. NO-REDIRECT -> ENABLE-REDIRECT-IN-PROGRESS: Begins the table redirection process by * transitioning the table to * 'ENABLE-REDIRECT-IN-PROGRESS.' During this setup * phase, all concurrent DML and DDL operations are * temporarily blocked.. * 1. ENABLE-REDIRECT-IN-PROGRESS -> REDIRECT-READY: Completes the setup for the table redirection * feature. The table starts redirecting all * queries to the destination location. * 2. REDIRECT-READY -> DROP-REDIRECT-IN-PROGRESS: Initiates the process of removing table * redirection by setting the table to * 'DROP-REDIRECT-IN-PROGRESS.' This ensures that * concurrent DML/DDL operations do not interfere * with the cancellation process. * 3. DROP-REDIRECT-IN-PROGRESS -> NO-REDIRECT: Completes the removal of table redirection. As a * result, all DML, DDL, and read-only queries are no * longer redirected to the previous destination. * 4. ENABLE-REDIRECT-IN-PROGRESS -> NO-REDIRECT: This transition involves canceling table * redirection while it is still in the process of * being enabled. */ sealed trait RedirectState { val name: String } /** This state indicates that redirect is not enabled on the table. */ case object NoRedirect extends RedirectState { override val name = "NO-REDIRECT" } /** This state indicates that the redirect process is still going on. */ case object EnableRedirectInProgress extends RedirectState { override val name = "ENABLE-REDIRECT-IN-PROGRESS" } /** * This state indicates that the redirect process is completed. All types of queries would be * redirected to the table specified inside RedirectSpec object. */ case object RedirectReady extends RedirectState { override val name = "REDIRECT-READY" } /** * The table redirection is under withdrawal and the redirection property is going to be removed * from the delta table. In this state, the delta client stops redirecting new queries to redirect * destination tables, and only accepts read-only queries to access the redirect source table. * The on-going redirected write or metadata transactions, which are visiting redirect * destinations, can not commit. */ case object DropRedirectInProgress extends RedirectState { override val name = "DROP-REDIRECT-IN-PROGRESS" } /** * This is the abstract class of the redirect specification, which stores the information * of accessing the redirect destination table. */ abstract class RedirectSpec() { /** Determine whether `dataPath` is the redirect destination location. */ def isRedirectDest(catalog: SessionCatalog, config: Configuration, dataPath: String): Boolean /** Determine whether `dataPath` is the redirect source location. */ def isRedirectSource(dataPath: String): Boolean } /** * The default redirect spec that is used for OSS delta. * This is the specification about how to access the redirect destination table. * One example of its JSON presentation is: * { * ...... * "spec": { * "redirectSrc": "s3:///tables/" * "redirectDest": "s3:///tables/" * } * } * * @param sourcePath this is the path where stores the redirect source table's location. * @param destPath: this is the path where stores the redirect destination table's location. */ class PathBasedRedirectSpec( val sourcePath: String, val destPath: String ) extends RedirectSpec { def isRedirectDest(catalog: SessionCatalog, config: Configuration, dataPath: String): Boolean = { destPath == dataPath } def isRedirectSource(dataPath: String): Boolean = sourcePath == dataPath } object PathBasedRedirectSpec { /** * This is the path based redirection. Delta client uses the `tablePath` of PathBasedRedirectSpec * to access the delta log files on the redirect destination location. */ final val REDIRECT_TYPE = "PathBasedRedirect" } /** * The customized JSON deserializer that parses the redirect specification's content into * RedirectSpec object. This class is passed to the JSON execution time object mapper. */ class RedirectSpecDeserializer[T <: RedirectSpec : ClassTag] { def deserialize(specValue: String): T = { val mapper = new ObjectMapper() mapper.registerModule(DefaultScalaModule) val clazz = implicitly[ClassTag[T]].runtimeClass.asInstanceOf[Class[T]] mapper.readValue(specValue, clazz) } } object RedirectSpec { def getDeserializeModule(redirectType: String): RedirectSpecDeserializer[_ <: RedirectSpec] = { new RedirectSpecDeserializer[PathBasedRedirectSpec]() } } /** * This class defines the rule of allowing transaction to access redirect source table. * @param appName The application name that is allowed to commit transaction defined inside * the `allowedOperations` set. If a rules' appName is empty, then all application * should fulfill its `allowedOperations`. * @param allowedOperations The set of operation names that are allowed to commit on the * redirect source table. * The example of usage of NoRedirectRule. * { * "type": "PathBasedRedirect", * "state": "REDIRECT-READY", * "spec": { * "tablePath": "s3:///tables/" * }, * "noRedirectRules": [ * {"allowedOperations": ["Write", "Delete", "Refresh"] }, * {"appName": "maintenance-job", "allowedOperations": ["Refresh"] } * ] * } */ case class NoRedirectRule( @JsonProperty("appName") appName: Option[String], @JsonProperty("allowedOperations") allowedOperations: Set[String] ) /** * This class stores all values defined inside table redirection property. * @param type: The type of redirection. * @param state: The current state of the redirection: * ENABLE-REDIRECT-IN-PROGRESS, REDIRECT-READY, DROP-REDIRECT-IN-PROGRESS. * @param specValue: The specification of accessing redirect destination table. * @param noRedirectRules: The set of rules that applications should fulfill to access * redirect source table. * This class would be serialized into a JSON string during commit. One example of its JSON * presentation is: * PathBasedRedirect: * { * "type": "PathBasedRedirect", * "state": "DROP-REDIRECT-IN-PROGRESS", * "spec": { * "tablePath": "s3:///tables/" * }, * "noRedirectRules": [ * {"allowedOperations": ["Write", "Refresh"] }, * {"appName": "maintenance-job", "allowedOperations": ["Refresh"] } * ] * } */ case class TableRedirectConfiguration( `type`: String, state: String, @JsonProperty("spec") specValue: String, @JsonProperty("noRedirectRules") noRedirectRules: Set[NoRedirectRule] = Set.empty) { @JsonIgnore val spec: RedirectSpec = RedirectSpec.getDeserializeModule(`type`).deserialize(specValue) @JsonIgnore val redirectState: RedirectState = state match { case EnableRedirectInProgress.name => EnableRedirectInProgress case RedirectReady.name => RedirectReady case DropRedirectInProgress.name => DropRedirectInProgress case _ => throw new IllegalArgumentException(s"Unrecognizable Table Redirect State: $state") } @JsonIgnore val isInProgressState: Boolean = { redirectState == EnableRedirectInProgress || redirectState == DropRedirectInProgress } /** Determines whether the current application fulfills the no-redirect rules. */ private def isNoRedirectApp(spark: SparkSession): Boolean = { noRedirectRules.exists { rule => // If rule.appName is empty, then it applied to "spark.app.name" rule.appName.forall(_.equalsIgnoreCase(spark.conf.get("spark.app.name"))) } } /** Determines whether the current session needs to access the redirect dest location. */ def needRedirect(spark: SparkSession, logPath: Path): Boolean = { !isNoRedirectApp(spark) && redirectState == RedirectReady && spec.isRedirectSource(logPath.toUri.getPath) } /** * Get the redirect destination location from `deltaLog` object. */ def getRedirectLocation(deltaLog: DeltaLog, spark: SparkSession): Path = { spec match { case spec: PathBasedRedirectSpec => val location = new Path(spec.destPath) val fs = location.getFileSystem(deltaLog.newDeltaHadoopConf()) fs.makeQualified(logPathFor(location)) case other => throw DeltaErrors.unrecognizedRedirectSpec(other) } } } /** * This is the main class of the table redirect that interacts with other components. */ class TableRedirect(val config: DeltaConfig[Option[String]]) { /** * Determine whether the property of table redirect feature is set. */ def isFeatureSet(metadata: Metadata): Boolean = config.fromMetaData(metadata).nonEmpty /** * Parse the property of table redirect feature to be an in-memory object of * TableRedirectConfiguration. */ def getRedirectConfiguration(deltaLogMetadata: Metadata): Option[TableRedirectConfiguration] = { config.fromMetaData(deltaLogMetadata).map { propertyValue => RedirectFeature.parseRedirectConfiguration(propertyValue) } } /** * Generate the key-value pair of the table redirect property. Its key is the table redirect * property name and its name is the JSON string of TableRedirectConfiguration. */ def generateRedirectMetadata( redirectType: String, state: RedirectState, redirectSpec: RedirectSpec, noRedirectRules: Set[NoRedirectRule] ): Map[String, String] = { val redirectConfiguration = TableRedirectConfiguration( redirectType, state.name, JsonUtils.toJson(redirectSpec), noRedirectRules ) val redirectJson = JsonUtils.toJson(redirectConfiguration) Map(config.key -> redirectJson) } /** * Issues a commit to update the table redirect property on the `catalogTableOpt`. * For the commits update the `state`, a validation is applied to ensure the state * transition is valid. * @param deltaLog The deltaLog object of the table to be redirected. * @param catalogTableOpt The CatalogTable object of the table to be redirected. * @param state The new state of redirection. * @param spec The specification of redirection contains all necessary detail of looking up the * redirect destination table. */ def update( deltaLog: DeltaLog, catalogTableOpt: Option[CatalogTable], state: RedirectState, spec: RedirectSpec, noRedirectRules: Set[NoRedirectRule] = Set.empty[NoRedirectRule] ): Unit = { val txn = deltaLog.startTransaction(catalogTableOpt) val deltaMetadata = txn.snapshot.metadata val currentConfigOpt = getRedirectConfiguration(deltaMetadata) val tableIdent = catalogTableOpt.map(_.identifier.quotedString).getOrElse { s"delta.`${deltaLog.dataPath.toString}`" } // There should be an existing table redirect configuration. if (currentConfigOpt.isEmpty) { throw DeltaErrors.invalidRedirectStateTransition(tableIdent, NoRedirect, state) } val currentConfig = currentConfigOpt.get val redirectState = currentConfig.redirectState RedirectFeature.validateStateTransition(tableIdent, redirectState, state) val properties = generateRedirectMetadata(currentConfig.`type`, state, spec, noRedirectRules) val newConfigs = txn.metadata.configuration ++ properties val newMetadata = txn.metadata.copy(configuration = newConfigs) txn.updateMetadata(newMetadata) txn.commit(Nil, DeltaOperations.SetTableProperties(properties)) } /** * Issues a commit to add the redirect property with state `EnableRedirectInProgress` * to the `catalogTableOpt`. * @param deltaLog The deltaLog object of the table to be redirected. * @param catalogTableOpt The CatalogTable object of the table to be redirected. * @param redirectType The type of redirection is used as an identifier to deserialize the content * of `spec`. * @param spec The specification of redirection contains all necessary detail of looking up the * redirect destination table. */ def add( deltaLog: DeltaLog, catalogTableOpt: Option[CatalogTable], redirectType: String, spec: RedirectSpec, noRedirectRules: Set[NoRedirectRule] = Set.empty[NoRedirectRule] ): Unit = { val txn = deltaLog.startTransaction(catalogTableOpt) val snapshot = txn.snapshot getRedirectConfiguration(snapshot.metadata).foreach { currentConfig => throw DeltaErrors.invalidRedirectStateTransition( catalogTableOpt.map(_.identifier.quotedString).getOrElse { s"delta.`${deltaLog.dataPath.toString}`" }, currentConfig.redirectState, EnableRedirectInProgress ) } val properties = generateRedirectMetadata( redirectType, EnableRedirectInProgress, spec, noRedirectRules ) val newConfigs = txn.metadata.configuration ++ properties val newMetadata = txn.metadata.copy(configuration = newConfigs) txn.updateMetadata(newMetadata) txn.commit(Nil, DeltaOperations.SetTableProperties(properties)) } /** Issues a commit to remove the redirect property from the `catalogTableOpt`. */ def remove(deltaLog: DeltaLog, catalogTableOpt: Option[CatalogTable]): Unit = { val txn = deltaLog.startTransaction(catalogTableOpt) val currentConfigOpt = getRedirectConfiguration(txn.snapshot.metadata) val tableIdent = catalogTableOpt.map(_.identifier.quotedString).getOrElse { s"delta.`${deltaLog.dataPath.toString}`" } if (currentConfigOpt.isEmpty) { DeltaErrors.invalidRemoveTableRedirect(tableIdent, NoRedirect) } val redirectState = currentConfigOpt.get.redirectState if (redirectState != DropRedirectInProgress && redirectState != EnableRedirectInProgress) { DeltaErrors.invalidRemoveTableRedirect(tableIdent, redirectState) } val newConfigs = txn.metadata.configuration.filterNot { case (key, _) => key == config.key } txn.updateMetadata(txn.metadata.copy(configuration = newConfigs)) txn.commit(Nil, DeltaOperations.UnsetTableProperties(Seq(config.key), ifExists = true)) } } object RedirectReaderWriter extends TableRedirect(config = DeltaConfigs.REDIRECT_READER_WRITER) { /** True if `snapshot` enables redirect-reader-writer feature. */ def isFeatureSupported(snapshot: Snapshot): Boolean = { snapshot.protocol.isFeatureSupported(RedirectReaderWriterFeature) } /** True if the update property command tries to set/unset redirect-reader-writer feature. */ def isUpdateProperty(snapshot: Snapshot, propKeys: Seq[String]): Boolean = { propKeys.contains(DeltaConfigs.REDIRECT_READER_WRITER.key) && isFeatureSupported(snapshot) } } object RedirectWriterOnly extends TableRedirect(config = DeltaConfigs.REDIRECT_WRITER_ONLY) { /** True if `snapshot` enables redirect-writer-only feature. */ def isFeatureSupported(snapshot: Snapshot): Boolean = { snapshot.protocol.isFeatureSupported(RedirectWriterOnlyFeature) } /** True if the update property command tries to set/unset redirect-writer-only feature. */ def isUpdateProperty(snapshot: Snapshot, propKeys: Seq[String]): Boolean = { propKeys.contains(DeltaConfigs.REDIRECT_WRITER_ONLY.key) && isFeatureSupported(snapshot) } } object RedirectFeature { /** * Determine whether the redirect-reader-writer or the redirect-writer-only feature is supported. */ def isFeatureSupported(snapshot: Snapshot): Boolean = { RedirectReaderWriter.isFeatureSupported(snapshot) || RedirectWriterOnly.isFeatureSupported(snapshot) } private def getRedirectConfigurationFromDeltaLog( spark: SparkSession, deltaLog: DeltaLog, initialCatalogTable: Option[CatalogTable] ): Option[TableRedirectConfiguration] = { val snapshot = deltaLog.update( catalogTableOpt = initialCatalogTable ) getRedirectConfiguration(snapshot.getProperties.toMap) } /** * This is the main method that redirect `deltaLog` to the destination location. */ def getRedirectLocationAndTable( spark: SparkSession, deltaLog: DeltaLog, redirectConfig: TableRedirectConfiguration ): (Path, Option[CatalogTable]) = { // Try to get the catalogTable object for the redirect destination table. val catalogTableOpt = redirectConfig.spec match { case pathRedirect: PathBasedRedirectSpec => withUpdateTableRedirectDDL(updateTableRedirectDDL = true) { val analyzer = spark.sessionState.analyzer import analyzer.CatalogAndIdentifier val CatalogAndIdentifier(catalog, ident) = Seq("delta", pathRedirect.destPath) catalog.asTableCatalog.loadTable(ident).asInstanceOf[DeltaTableV2].catalogTable } } // Get the redirect destination location. val redirectLocation = redirectConfig.getRedirectLocation(deltaLog, spark) (redirectLocation, catalogTableOpt) } def parseRedirectConfiguration(configString: String): TableRedirectConfiguration = { val mapper = new ObjectMapper() mapper.registerModule(DefaultScalaModule) mapper.readValue(configString, classOf[TableRedirectConfiguration]) } /** * Get the current `TableRedirectConfiguration` object from the table properties. * Note that the redirect-reader-writer takes precedence over redirect-writer-only. */ def getRedirectConfiguration( properties: Map[String, String]): Option[TableRedirectConfiguration] = { properties.get(DeltaConfigs.REDIRECT_READER_WRITER.key) .orElse(properties.get(DeltaConfigs.REDIRECT_WRITER_ONLY.key)) .map(parseRedirectConfiguration) } /** * Determine whether the operation `op` updates the existing redirect-reader-writer or * redirect-writer-only table property of a table with `snapshot`. */ def isUpdateProperty(snapshot: Snapshot, op: DeltaOperations.Operation): Boolean = { op match { case _ @ DeltaOperations.SetTableProperties(properties) => val propertyKeys = properties.keySet.toSeq RedirectReaderWriter.isUpdateProperty(snapshot, propertyKeys) || RedirectWriterOnly.isUpdateProperty(snapshot, propertyKeys) case _ @ DeltaOperations.UnsetTableProperties(propertyKeys, _) => RedirectReaderWriter.isUpdateProperty(snapshot, propertyKeys) || RedirectWriterOnly.isUpdateProperty(snapshot, propertyKeys) case _ => false } } /** * Determine whether the operation `op` is dropping either the redirect-reader-writer or * redirect-writer-only table feature. */ def isDropFeature(op: DeltaOperations.Operation): Boolean = op match { case DeltaOperations.DropTableFeature(featureName, _) => isRedirectFeature(featureName) case _ => false } def isRedirectFeature(name: String): Boolean = { name.toLowerCase(Locale.ROOT) == RedirectReaderWriterFeature.name.toLowerCase(Locale.ROOT) || name.toLowerCase(Locale.ROOT) == RedirectWriterOnlyFeature.name.toLowerCase(Locale.ROOT) } /** * Get the current `TableRedirectConfiguration` object from the snapshot. * Note that the redirect-reader-writer takes precedence over redirect-writer-only. */ def getRedirectConfiguration(snapshot: Snapshot): Option[TableRedirectConfiguration] = { getRedirectConfiguration(snapshot.metadata.configuration) } /** Determines whether `configs` contains redirect configuration. */ def hasRedirectConfig(configs: Map[String, String]): Boolean = getRedirectConfiguration(configs).isDefined /** Determines whether the property `name` is redirect property. */ def isRedirectProperty(name: String): Boolean = { name == DeltaConfigs.REDIRECT_READER_WRITER.key || name == DeltaConfigs.REDIRECT_WRITER_ONLY.key } // Helper method to validate state transitions def validateStateTransition( identifier: String, currentState: RedirectState, newState: RedirectState ): Unit = { (currentState, newState) match { case (state, RedirectReady) => if (state == DropRedirectInProgress) { throw DeltaErrors.invalidRedirectStateTransition(identifier, state, newState) } case (state, DropRedirectInProgress) => if (state != RedirectReady) { throw DeltaErrors.invalidRedirectStateTransition(identifier, state, newState) } case (state, _) => throw DeltaErrors.invalidRedirectStateTransition(identifier, state, newState) } } /** Determine whether the current `deltaLog` needs to skip redirect feature. */ def needDeltaLogRedirect( spark: SparkSession, deltaLog: DeltaLog, initialCatalogTable: Option[CatalogTable] ): Option[TableRedirectConfiguration] = { // It can skip redirect, if the table fulfills any of the following conditions: // - redirect feature is not enable, // - current command is an DDL that updates table redirect property, or // - deltaLog doesn't have valid table. val canSkipTableRedirect = !spark.conf.get(ENABLE_TABLE_REDIRECT_FEATURE) || isUpdateTableRedirectDDL.value || !deltaLog.tableExists if (canSkipTableRedirect) return None val redirectConfigOpt = getRedirectConfigurationFromDeltaLog( spark, deltaLog, initialCatalogTable ) val needRedirectToDest = redirectConfigOpt.exists { redirectConfig => // If the current deltaLog already points to destination, early returns since // no need to redirect deltaLog. redirectConfig.needRedirect(spark, deltaLog.dataPath) } if (needRedirectToDest) redirectConfigOpt else None } def validateTableRedirect( snapshot: Snapshot, catalogTable: Option[CatalogTable], configs: Map[String, String] ): Unit = { val identifier = catalogTable .map(_.identifier.quotedString) .getOrElse(s"delta.`${snapshot.deltaLog.logPath.toString}`") if (configs.contains(DeltaConfigs.REDIRECT_READER_WRITER.key)) { if (RedirectWriterOnly.isFeatureSet(snapshot.metadata)) { throw DeltaErrors.invalidSetUnSetRedirectCommand( identifier, DeltaConfigs.REDIRECT_READER_WRITER.key, DeltaConfigs.REDIRECT_WRITER_ONLY.key ) } } else if (configs.contains(DeltaConfigs.REDIRECT_WRITER_ONLY.key)) { if (RedirectReaderWriter.isFeatureSet(snapshot.metadata)) { throw DeltaErrors.invalidSetUnSetRedirectCommand( identifier, DeltaConfigs.REDIRECT_WRITER_ONLY.key, DeltaConfigs.REDIRECT_READER_WRITER.key ) } } else { return } val currentRedirectConfigOpt = getRedirectConfiguration(snapshot) val newRedirectConfigOpt = getRedirectConfiguration(configs) newRedirectConfigOpt.foreach { newRedirectConfig => val newState = newRedirectConfig.redirectState // Validate state transitions based on current and new states currentRedirectConfigOpt match { case Some(currentRedirectConfig) => validateStateTransition(identifier, currentRedirectConfig.redirectState, newState) case None if newState == DropRedirectInProgress => throw DeltaErrors.invalidRedirectStateTransition( identifier, newState, DropRedirectInProgress ) case _ => // No action required for valid transitions } } } val DELTALOG_PREFIX = "redirect-delta-log://" /** * The thread local variable for indicating whether the current session is an * DDL that updates redirect table property. */ @SuppressWarnings( Array( "BadMethodCall-DynamicVariable", """ Reason: The redirect feature implementation requires a thread-local variable to control enable/disable states during SET and UNSET operations. This approach is necessary because: - Parameter Passing Limitation: The call stack cannot propagate this state via method parameters, as the feature is triggered through an external open-source API interface that does not expose this configurability. - Concurrency Constraints: A global variable (without thread-local isolation) would allow unintended cross-thread interference, risking undefined behavior in concurrent transactions. We can not use lock because the lock would introduce big critical session and create performance issue. By using thread-local storage, the feature ensures transaction-specific state isolation while maintaining compatibility with the third-party API's design.""" ) ) private val isUpdateTableRedirectDDL = new DynamicVariable[Boolean](false) /** * Execute `thunk` while `isUpdateTableRedirectDDL` is set to `updateTableRedirectDDL`. */ def withUpdateTableRedirectDDL[T](updateTableRedirectDDL: Boolean)(thunk: => T): T = { isUpdateTableRedirectDDL.withValue(updateTableRedirectDDL) { thunk } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/schema/ImplicitMetadataOperation.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.schema import org.apache.spark.sql.delta.skipping.clustering.ClusteredTableUtils import org.apache.spark.sql.delta.skipping.clustering.temp.ClusterBySpec import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.{DomainMetadata, Metadata, Protocol} import org.apache.spark.sql.delta.constraints.Constraints import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf.AllowAutomaticWideningMode import org.apache.spark.sql.delta.util.PartitionUtils import org.apache.spark.internal.MDC import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.FileSourceGeneratedMetadataStructField import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes import org.apache.spark.sql.types.{ArrayType, AtomicType, DataType, MapType, StructType} /** * A trait that writers into Delta can extend to update the schema and/or partitioning of the table. */ trait ImplicitMetadataOperation extends DeltaLogging { import ImplicitMetadataOperation._ protected val canMergeSchema: Boolean protected val canOverwriteSchema: Boolean private def normalizePartitionColumns( spark: SparkSession, partitionCols: Seq[String], schema: StructType): Seq[String] = { partitionCols.map { columnName => val colMatches = schema.filter(s => SchemaUtils.DELTA_COL_RESOLVER(s.name, columnName)) if (colMatches.length > 1) { throw DeltaErrors.ambiguousPartitionColumnException(columnName, colMatches) } else if (colMatches.isEmpty) { throw DeltaErrors.partitionColumnNotFoundException(columnName, toAttributes(schema)) } colMatches.head.name } } /** Remove all file source generated metadata columns from the schema. */ private def dropGeneratedMetadataColumns(structType: StructType): StructType = { val fields = structType.filter { case FileSourceGeneratedMetadataStructField(_, _) => false case _ => true } StructType(fields) } protected final def updateMetadata( spark: SparkSession, txn: OptimisticTransaction, schema: StructType, partitionColumns: Seq[String], configuration: Map[String, String], isOverwriteMode: Boolean, rearrangeOnly: Boolean ): Unit = { // To support the new column mapping mode, we drop existing metadata on data schema // so that all the column mapping related properties can be reinitialized in // OptimisticTransaction.updateMetadata var dataSchema = DeltaColumnMapping.dropColumnMappingMetadata(schema.asNullable) // File Source generated columns are not added to the stored schema. dataSchema = dropGeneratedMetadataColumns(dataSchema) val mergedSchema = mergeSchema(spark, txn, dataSchema, isOverwriteMode, canOverwriteSchema) val normalizedPartitionCols = normalizePartitionColumns(spark, partitionColumns, dataSchema) // Merged schema will contain additional columns at the end def isNewSchema: Boolean = txn.metadata.schema != mergedSchema // We need to make sure that the partitioning order and naming is consistent // if provided. Otherwise we follow existing partitioning def isNewPartitioning: Boolean = normalizedPartitionCols.nonEmpty && txn.metadata.partitionColumns != normalizedPartitionCols def isPartitioningChanged: Boolean = txn.metadata.partitionColumns != normalizedPartitionCols PartitionUtils.validatePartitionColumn( mergedSchema, normalizedPartitionCols, // Delta is case insensitive regarding internal column naming caseSensitive = false) if (!txn.deltaLog.tableExists) { if (dataSchema.isEmpty) { throw DeltaErrors.emptyDataException } recordDeltaEvent(txn.deltaLog, "delta.ddl.initializeSchema") // If this is the first write, configure the metadata of the table. if (rearrangeOnly) { throw DeltaErrors.unexpectedDataChangeException("Create a Delta table") } val description = configuration.get("comment").orNull // Filter out the property for clustering columns from Metadata action. val cleanedConfs = ClusteredTableUtils.removeClusteringColumnsProperty( configuration.filterKeys(_ != "comment").toMap) txn.updateMetadata( Metadata( description = description, schemaString = dataSchema.json, partitionColumns = normalizedPartitionCols, configuration = cleanedConfs , createdTime = Some(System.currentTimeMillis()))) } else if (isOverwriteMode && canOverwriteSchema && (isNewSchema || isPartitioningChanged )) { // Can define new partitioning in overwrite mode val newMetadata = txn.metadata.copy( schemaString = dataSchema.json, partitionColumns = normalizedPartitionCols ) recordDeltaEvent(txn.deltaLog, "delta.ddl.overwriteSchema") if (rearrangeOnly) { throw DeltaErrors.unexpectedDataChangeException("Overwrite the Delta table schema or " + "change the partition schema") } txn.updateMetadataForTableOverwrite(newMetadata) } else if (isNewSchema && canMergeSchema && !isNewPartitioning ) { logInfo(log"New merged schema: ${MDC(DeltaLogKeys.SCHEMA, mergedSchema.treeString)}") recordDeltaEvent(txn.deltaLog, "delta.ddl.mergeSchema") if (rearrangeOnly) { throw DeltaErrors.unexpectedDataChangeException("Change the Delta table schema") } val schemaWithTypeWideningMetadata = TypeWideningMetadata.addTypeWideningMetadata( txn, schema = mergedSchema, oldSchema = txn.metadata.schema ) txn.updateMetadata(txn.metadata.copy(schemaString = schemaWithTypeWideningMetadata.json )) } else if (isNewSchema || isNewPartitioning ) { recordDeltaEvent(txn.deltaLog, "delta.schemaValidation.failure") val errorBuilder = new MetadataMismatchErrorBuilder if (isNewSchema) { errorBuilder.addSchemaMismatch(txn.metadata.schema, dataSchema, txn.metadata.id) } if (isNewPartitioning) { errorBuilder.addPartitioningMismatch(txn.metadata.partitionColumns, normalizedPartitionCols) } if (isOverwriteMode) { errorBuilder.addOverwriteBit() } errorBuilder.finalizeAndThrow(spark.sessionState.conf) } } /** * Returns a sequence of new DomainMetadata if canUpdateMetadata is true and the operation is * either create table or replace the whole table (not replaceWhere operation). This is because * we only update Domain Metadata when creating or replacing table, and replace table for DDL * and DataFrameWriterV2 are already handled in CreateDeltaTableCommand. In that case, * canUpdateMetadata is false, so we don't update again. * * @param txn [[OptimisticTransaction]] being used to create or replace table. * @param canUpdateMetadata true if the metadata is not updated yet. * @param isReplacingTable true if the operation is replace table without replaceWhere option. * @param clusterBySpecOpt optional ClusterBySpec containing user-specified clustering columns. */ protected final def getNewDomainMetadata( txn: OptimisticTransaction, canUpdateMetadata: Boolean, isReplacingTable: Boolean, clusterBySpecOpt: Option[ClusterBySpec] = None): Seq[DomainMetadata] = { if (canUpdateMetadata && (!txn.deltaLog.tableExists || isReplacingTable)) { val newDomainMetadata = Seq.empty[DomainMetadata] ++ ClusteredTableUtils.getDomainMetadataFromTransaction(clusterBySpecOpt, txn) if (!txn.deltaLog.tableExists) { newDomainMetadata } else { // Handle domain metadata for replacing a table. DomainMetadataUtils.handleDomainMetadataForReplaceTable( txn.snapshot.domainMetadata, newDomainMetadata) } } else { Seq.empty } } } object ImplicitMetadataOperation { /** * Merge schemas based on transaction state and delta options * @param txn Target transaction * @param dataSchema New data schema * @param isOverwriteMode Whether we are overwriting * @param canOverwriteSchema Whether we can overwrite * @return Merged schema */ private[delta] def mergeSchema( spark: SparkSession, txn: OptimisticTransaction, dataSchema: StructType, isOverwriteMode: Boolean, canOverwriteSchema: Boolean): StructType = { if (isOverwriteMode && canOverwriteSchema) { dataSchema } else { checkDependentExpressions(spark, txn.protocol, txn.metadata, dataSchema) val typeWideningMode = if (TypeWidening.isEnabled(txn.protocol, txn.metadata)) { TypeWideningMode.TypeEvolution( uniformIcebergCompatibleOnly = UniversalFormat.icebergEnabled(txn.metadata), allowAutomaticWidening = AllowAutomaticWideningMode.fromConf(spark.sessionState.conf)) } else { TypeWideningMode.NoTypeWidening } SchemaMergingUtils.mergeSchemas( txn.metadata.schema, dataSchema, typeWideningMode = typeWideningMode) } } /** * Check whether there are dependant (CHECK) constraints for * the provided `currentDt`; if so, throw an error indicating * the constraint data type mismatch. * * @param spark the spark session used. * @param path the full column path for the current field. * @param metadata the metadata used for checking dependant (CHECK) constraints. * @param currentDt the current data type. * @param updateDt the updated data type. */ private def checkDependentConstraints( spark: SparkSession, path: Seq[String], metadata: Metadata, currentDt: DataType, updateDt: DataType): Unit = { val dependentConstraints = Constraints.findDependentConstraints(spark, path, metadata) if (dependentConstraints.nonEmpty) { throw DeltaErrors.constraintDataTypeMismatch( path, currentDt, updateDt, dependentConstraints ) } } /** * Check whether there are dependant generated columns for * the provided `currentDt`; if so, throw an error indicating * the generated columns data type mismatch. * * @param spark the spark session used. * @param path the full column path for the current field. * @param protocol the protocol used. * @param metadata the metadata used for checking dependant generated columns. * @param currentDt the current data type. * @param updateDt the updated data type. */ private def checkDependentGeneratedColumns( spark: SparkSession, path: Seq[String], protocol: Protocol, metadata: Metadata, currentDt: DataType, updateDt: DataType): Unit = { val dependentGeneratedColumns = SchemaUtils.findDependentGeneratedColumns( spark, path, protocol, metadata.schema) if (dependentGeneratedColumns.nonEmpty) { throw DeltaErrors.generatedColumnsDataTypeMismatch( path, currentDt, updateDt, dependentGeneratedColumns ) } } /** * Check whether the provided field is currently being referenced * by CHECK constraints or generated columns. * Note that we explicitly ignore the check for `StructType` in this * function by only inspecting its inner fields to relax the check; * plus, any `StructType` will be traversed in [[checkDependentExpressions]]. * * @param spark the spark session used. * @param path the full column path for the current field. * @param protocol the protocol used. * @param metadata the metadata used for checking constraints and generated columns. * @param currentDt the current data type. * @param updateDt the updated data type. */ private def checkConstraintsOrGeneratedColumnsOnStructField( spark: SparkSession, path: Seq[String], protocol: Protocol, metadata: Metadata, currentDt: DataType, updateDt: DataType): Unit = (currentDt, updateDt) match { // we explicitly ignore the check for `StructType` here. case (_: StructType, _: StructType) => case (current: ArrayType, update: ArrayType) => checkConstraintsOrGeneratedColumnsOnStructField( spark, path :+ "element", protocol, metadata, current.elementType, update.elementType) case (current: MapType, update: MapType) => checkConstraintsOrGeneratedColumnsOnStructField( spark, path :+ "key", protocol, metadata, current.keyType, update.keyType) checkConstraintsOrGeneratedColumnsOnStructField( spark, path :+ "value", protocol, metadata, current.valueType, update.valueType) case (_, _) => if (currentDt != updateDt) { checkDependentConstraints(spark, path, metadata, currentDt, updateDt) checkDependentGeneratedColumns(spark, path, protocol, metadata, currentDt, updateDt) } } /** * Finds all fields that change between the current schema and the new data schema and fail if any * of them are referenced by check constraints or generated columns. */ private def checkDependentExpressions( sparkSession: SparkSession, protocol: Protocol, metadata: actions.Metadata, dataSchema: StructType): Unit = SchemaMergingUtils.transformColumns(metadata.schema, dataSchema) { case (fieldPath, currentField, Some(updateField), _) if !SchemaMergingUtils.equalsIgnoreCaseAndCompatibleNullability( currentField.dataType, updateField.dataType ) => checkConstraintsOrGeneratedColumnsOnStructField( spark = sparkSession, path = fieldPath :+ currentField.name, protocol = protocol, metadata = metadata, currentDt = currentField.dataType, updateDt = updateField.dataType ) // We don't transform the schema but just perform checks, // the returned field won't be used anyway. updateField case (_, field, _, _) => field } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/schema/InvariantViolationException.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.schema // scalastyle:off import.ordering.noEmptyLine import java.util import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.{DeltaThrowable, DeltaThrowableHelper} import org.apache.spark.sql.delta.constraints.{CharVarcharConstraint, Constraints} import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.spark.SparkException import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute /** Thrown when the given data doesn't match the rules defined on the table. */ case class InvariantViolationException(message: String) extends RuntimeException(message) /** * Match a [[SparkException]] and return the root cause Exception if it is a * InvariantViolationException. */ object InnerInvariantViolationException { def unapply(t: Throwable): Option[InvariantViolationException] = t match { case s: SparkException => Option(ExceptionUtils.getRootCause(s)) match { case Some(i: InvariantViolationException) => Some(i) case _ => None } case _ => None } } object DeltaInvariantViolationException { def getNotNullInvariantViolationException(colName: String): DeltaInvariantViolationException = { new DeltaInvariantViolationException( errorClass = "DELTA_NOT_NULL_CONSTRAINT_VIOLATED", messageParameters = Array(colName) ) } def apply(constraint: Constraints.NotNull): DeltaInvariantViolationException = { getNotNullInvariantViolationException(UnresolvedAttribute(constraint.column).name) } def getCharVarcharLengthInvariantViolationException( exprStr: String, valueStr: String ): DeltaInvariantViolationException = { new DeltaInvariantViolationException( errorClass = "DELTA_EXCEED_CHAR_VARCHAR_LIMIT", messageParameters = Array(valueStr, exprStr) ) } def getConstraintViolationWithValuesException( constraintName: String, sqlStr: String, valueLines: String ): DeltaInvariantViolationException = { new DeltaInvariantViolationException( errorClass = "DELTA_VIOLATE_CONSTRAINT_WITH_VALUES", messageParameters = Array(constraintName, sqlStr, valueLines) ) } /** * Build an exception to report the current row failed a CHECK constraint. * * @param constraint the constraint definition * @param values a map of full column names to their evaluated values in the failed row */ def apply( constraint: Constraints.Check, values: Map[String, Any]): DeltaInvariantViolationException = { if (constraint.name == CharVarcharConstraint.INVARIANT_NAME) { return getCharVarcharLengthInvariantViolationException( exprStr = constraint.expression.sql, valueStr = values.head._2.toString) } // Sort by the column name to generate consistent error messages in Scala 2.12 and 2.13. val valueLines = values.toSeq.sortBy(_._1).map { case (column, value) => s" - $column : $value" }.mkString("\n") getConstraintViolationWithValuesException( constraint.name, constraint.expression.sql, valueLines ) } /** * Columns and values in parallel lists as a shim for Java codegen compatibility. */ def apply( constraint: Constraints.Check, columns: java.util.List[String], values: java.util.List[Any]): DeltaInvariantViolationException = { apply(constraint, columns.asScala.zip(values.asScala).toMap) } } class DeltaInvariantViolationException( errorClass: String, messageParameters: Array[String]) extends InvariantViolationException( DeltaThrowableHelper.getMessage(errorClass, messageParameters)) with DeltaThrowable { override def getErrorClass: String = errorClass override def getMessageParameters: util.Map[String, String] = { DeltaThrowableHelper.getMessageParameters(errorClass, errorSubClass = null, messageParameters) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaMergingUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.schema import scala.util.control.NonFatal import org.apache.spark.sql.delta.{DeltaAnalysisException, TypeWideningMode} import org.apache.spark.sql.catalyst.analysis.{Resolver, TypeCoercion, UnresolvedAttribute} import org.apache.spark.sql.catalyst.expressions.Literal import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.types._ /** * Utils to merge table schema with data schema. * This is split from SchemaUtils, because finalSchema is introduced into DeltaMergeInto, * and resolving the final schema is now part of * [[ResolveDeltaMergeInto.resolveReferencesAndSchema]]. */ object SchemaMergingUtils { val DELTA_COL_RESOLVER: (String, String) => Boolean = org.apache.spark.sql.catalyst.analysis.caseInsensitiveResolution /** * Returns pairs of (full column name path, field) in this schema as a list. For example, a schema * like: * | - a * | | - 1 * | | - 2 * | - b * | - c * | | - `foo.bar` * | | - 3 * will return [ * ([a], ), ([a, 1], ), ([a, 2], ), ([b], ), * ([c], ), ([c, foo.bar], ), ([c, foo.bar, 3], ) * ] */ def explode(schema: StructType): Seq[(Seq[String], StructField)] = { def recurseIntoComplexTypes(complexType: DataType): Seq[(Seq[String], StructField)] = { complexType match { case s: StructType => explode(s) case a: ArrayType => recurseIntoComplexTypes(a.elementType) .map { case (path, field) => (Seq("element") ++ path, field) } case m: MapType => recurseIntoComplexTypes(m.keyType) .map { case (path, field) => (Seq("key") ++ path, field) } ++ recurseIntoComplexTypes(m.valueType) .map { case (path, field) => (Seq("value") ++ path, field) } case _ => Nil } } schema.flatMap { case f @ StructField(name, s: StructType, _, _) => Seq((Seq(name), f)) ++ explode(s).map { case (path, field) => (Seq(name) ++ path, field) } case f @ StructField(name, a: ArrayType, _, _) => Seq((Seq(name), f)) ++ recurseIntoComplexTypes(a).map { case (path, field) => (Seq(name) ++ path, field) } case f @ StructField(name, m: MapType, _, _) => Seq((Seq(name), f)) ++ recurseIntoComplexTypes(m).map { case (path, field) => (Seq(name) ++ path, field) } case f => (Seq(f.name), f) :: Nil } } /** * Returns all column names in this schema as a flat list. For example, a schema like: * | - a * | | - 1 * | | - 2 * | - b * | - c * | | - nest * | | - 3 * will get flattened to: "a", "a.1", "a.2", "b", "c", "c.nest", "c.nest.3" */ def explodeNestedFieldNames(schema: StructType): Seq[String] = { explode(schema).map { case (path, _) => path }.map(UnresolvedAttribute.apply(_).name) } /** * Checks if input column names have duplicate identifiers. This throws an exception if * the duplication exists. * * @param schema the schema to check for duplicates * @param colType column type name, used in an exception message * @param caseSensitive Whether we should exception if two columns have casing conflicts. This * should default to false for Delta. */ def checkColumnNameDuplication( schema: StructType, colType: String, caseSensitive: Boolean = false): Unit = { val columnNames = explodeNestedFieldNames(schema) // scalastyle:off caselocale val names = if (caseSensitive) { columnNames } else { columnNames.map(_.toLowerCase) } // scalastyle:on caselocale if (names.distinct.length != names.length) { val duplicateColumns = names.groupBy(identity).collect { case (x, ys) if ys.length > 1 => s"$x" } throw new DeltaAnalysisException( errorClass = "DELTA_DUPLICATE_COLUMNS_FOUND", messageParameters = Array(colType, duplicateColumns.mkString(", "))) } } /** * A variant of [[mergeDataTypes]] with common default values and enforce struct type * as inputs for Delta table operation. * * Check whether we can write to the Delta table, which has `tableSchema`, using a query that has * `dataSchema`. Our rules are that: * - `dataSchema` may be missing columns or have additional columns * - We don't trust the nullability in `dataSchema`. Assume fields are nullable. * - We only allow nested StructType expansions. For all other complex types, we check for * strict equality * - `dataSchema` can't have duplicate column names. Columns that only differ by case are also * not allowed. * The following merging strategy is * applied: * - The name of the current field is used. * - The data types are merged by calling this function. * - We respect the current field's nullability. * - The metadata is current field's metadata. * * Schema merging occurs in a case insensitive manner. Hence, column names that only differ * by case are not accepted in the `dataSchema`. */ def mergeSchemas( tableSchema: StructType, dataSchema: StructType, allowImplicitConversions: Boolean = false, keepExistingType: Boolean = false, typeWideningMode: TypeWideningMode = TypeWideningMode.NoTypeWidening, caseSensitive: Boolean = false): StructType = { checkColumnNameDuplication(dataSchema, "in the data to save", caseSensitive) mergeDataTypes( tableSchema, dataSchema, allowImplicitConversions, keepExistingType, typeWideningMode, caseSensitive, allowOverride = false, overrideMetadata = false ).asInstanceOf[StructType] } /** * @param current The current data type. * @param update The data type of the new data being written. * @param allowImplicitConversions Whether to allow Spark SQL implicit conversions. By default, * we merge according to Parquet write compatibility - for * example, an integer type data field will throw when merged to a * string type table field, because int and string aren't stored * the same way in Parquet files. With this flag enabled, the * merge will succeed, because once we get to write time Spark SQL * will support implicitly converting the int to a string. * @param keepExistingType Whether to keep existing types instead of trying to merge types. * @param typeWideningMode Identifies the (current, update) type tuples where `current` can be * widened to `update`, in which case `update` is used. See * [[TypeWideningMode]]. * @param caseSensitive Whether we should keep field mapping case-sensitively. * This should default to false for Delta, which is case insensitive. * @param allowOverride Whether to let incoming type override the existing type if unmatched. * @param overrideMetadata Whether to let metadata of new fields override the existing * metadata of matching fields */ def mergeDataTypes( current: DataType, update: DataType, allowImplicitConversions: Boolean, keepExistingType: Boolean, typeWideningMode: TypeWideningMode, caseSensitive: Boolean, allowOverride: Boolean, overrideMetadata: Boolean): DataType = { def merge(current: DataType, update: DataType): DataType = { (current, update) match { case (StructType(currentFields), StructType(updateFields)) => // Merge existing fields. val updateFieldMap = toFieldMap(updateFields, caseSensitive) val updatedCurrentFields = currentFields.map { currentField => updateFieldMap.get(currentField.name) match { case Some(updateField) => try { val updatedCurrentFieldMetadata = if (overrideMetadata) updateField.metadata else currentField.metadata StructField( currentField.name, merge(currentField.dataType, updateField.dataType), currentField.nullable, updatedCurrentFieldMetadata) } catch { case NonFatal(e) => throw new DeltaAnalysisException( errorClass = "DELTA_FAILED_TO_MERGE_FIELDS", messageParameters = Array(currentField.name, updateField.name), cause = Some(e) ) } case None => // Retain the old field. currentField } } // Identify the newly added fields. val nameToFieldMap = toFieldMap(currentFields, caseSensitive) val newFields = updateFields.filterNot(f => nameToFieldMap.contains(f.name)) // Create the merged struct, the new fields are appended at the end of the struct. StructType(updatedCurrentFields ++ newFields) case (ArrayType(currentElementType, currentContainsNull), ArrayType(updateElementType, _)) => ArrayType( merge(currentElementType, updateElementType), currentContainsNull) case (MapType(currentKeyType, currentElementType, currentContainsNull), MapType(updateKeyType, updateElementType, _)) => MapType( merge(currentKeyType, updateKeyType), merge(currentElementType, updateElementType), currentContainsNull) // If type widening is enabled and the type can be widened, it takes precedence over // keepExistingType. case (current: AtomicType, update: AtomicType) if typeWideningMode.getWidenedType(fromType = current, toType = update).isDefined => typeWideningMode.getWidenedType(fromType = current, toType = update).get // Simply keeps the existing type for primitive types case (current, _) if keepExistingType => current case (_, update) if allowOverride => update // If implicit conversions are allowed, that means we can use any valid implicit cast to // perform the merge. case (current, update) if allowImplicitConversions && typeForImplicitCast(update, current).isDefined => typeForImplicitCast(update, current).get case (DecimalType.Fixed(leftPrecision, leftScale), DecimalType.Fixed(rightPrecision, rightScale)) => if ((leftPrecision == rightPrecision) && (leftScale == rightScale)) { current } else if ((leftPrecision != rightPrecision) && (leftScale != rightScale)) { throw new DeltaAnalysisException( errorClass = "DELTA_MERGE_INCOMPATIBLE_DECIMAL_TYPE", messageParameters = Array( s"precision $leftPrecision and $rightPrecision & scale $leftScale and $rightScale")) } else if (leftPrecision != rightPrecision) { throw new DeltaAnalysisException( errorClass = "DELTA_MERGE_INCOMPATIBLE_DECIMAL_TYPE", messageParameters = Array(s"precision $leftPrecision and $rightPrecision")) } else { throw new DeltaAnalysisException( errorClass = "DELTA_MERGE_INCOMPATIBLE_DECIMAL_TYPE", messageParameters = Array(s"scale $leftScale and $rightScale")) } case _ if current == update => current // Parquet physically stores ByteType, ShortType and IntType as IntType, so when a parquet // column is of one of these three types, you can read this column as any of these three // types. Since Parquet doesn't complain, we should also allow upcasting among these // three types when merging schemas. case (ByteType, ShortType) => ShortType case (ByteType, IntegerType) => IntegerType case (ShortType, ByteType) => ShortType case (ShortType, IntegerType) => IntegerType case (IntegerType, ShortType) => IntegerType case (IntegerType, ByteType) => IntegerType case (NullType, _) => update case (_, NullType) => current case _ => throw new DeltaAnalysisException(errorClass = "DELTA_MERGE_INCOMPATIBLE_DATATYPE", messageParameters = Array(current.toString, update.toString)) } } merge(current, update) } /** * Try to cast the source data type to the target type, returning the final type or None if * there's no valid cast. */ private def typeForImplicitCast(sourceType: DataType, targetType: DataType): Option[DataType] = { TypeCoercion.implicitCast(Literal.default(sourceType), targetType).map(_.dataType) } def toFieldMap( fields: Seq[StructField], caseSensitive: Boolean = false): Map[String, StructField] = { val fieldMap = fields.map(field => field.name -> field).toMap if (caseSensitive) { fieldMap } else { CaseInsensitiveMap(fieldMap) } } /** * Transform (nested) columns in a schema. * * @param schema to transform. * @param tf function to apply. * @return the transformed schema. */ def transformColumns[T <: DataType]( schema: T)( tf: (Seq[String], StructField, Resolver) => StructField): T = { def transform[E <: DataType](path: Seq[String], dt: E): E = { val newDt = dt match { case s: StructType if org.apache.spark.sql.execution.datasources.VariantMetadata.isVariantStruct(s) => // A variant struct is logically still a variant, so we should not recurse into its // fields like a normal struct. s case StructType(fields) => StructType(fields.map { field => val newField = tf(path, field, DELTA_COL_RESOLVER) // maintain the old name as we recurse into the subfields newField.copy(dataType = transform(path :+ field.name, newField.dataType)) }) case ArrayType(elementType, containsNull) => ArrayType(transform(path :+ "element", elementType), containsNull) case MapType(keyType, valueType, valueContainsNull) => MapType( transform(path :+ "key", keyType), transform(path :+ "value", valueType), valueContainsNull) case other => other } newDt.asInstanceOf[E] } transform(Seq.empty, schema) } /** * Prune all nested empty structs from the schema. Return None if top level struct is also empty. * @param dataType the data type to prune. */ def pruneEmptyStructs(dataType: DataType): Option[DataType] = { dataType match { case StructType(fields) => val newFields = fields.flatMap { f => pruneEmptyStructs(f.dataType).map { newType => StructField(f.name, newType, f.nullable, f.metadata) } } // when there is no fields, i.e., the struct is empty, we will return None to indicate // we don't want to include that field. if (newFields.isEmpty) { None } else { Option(StructType(newFields)) } case ArrayType(currentElementType, containsNull) => // if the array element type is from from_json, we will exclude the array. pruneEmptyStructs(currentElementType).map { newType => ArrayType(newType, containsNull) } case MapType(keyType, elementType, containsNull) => // if the map key/element type is from from_json, we will exclude the map. val filtertedKeyType = pruneEmptyStructs(keyType) val filtertedValueType = pruneEmptyStructs(elementType) if (filtertedKeyType.isEmpty || filtertedValueType.isEmpty) { None } else { Option(MapType(filtertedKeyType.get, filtertedValueType.get, containsNull)) } case _ => Option(dataType) } } /** * Transform (nested) columns in `schema` by walking down `schema` and `other` simultaneously. * This allows comparing the two schemas and transforming `schema` based on the comparison. * Columns or fields present only in `other` are ignored while `None` is passed to the transform * function for columns or fields missing in `other`. * @param schema Schema to transform. * @param other Schema to compare with. * @param tf Function to apply. The function arguments are the full path of the current field to * transform, the current field in `schema` and, if present, the corresponding field in * `other`. */ def transformColumns( schema: StructType, other: StructType)( tf: (Seq[String], StructField, Option[StructField], Resolver) => StructField): StructType = { def transform[E <: DataType](path: Seq[String], dt: E, otherDt: E): E = { val newDt = (dt, otherDt) match { case (struct: StructType, otherStruct: StructType) => val otherFields = SchemaMergingUtils.toFieldMap(otherStruct.fields, caseSensitive = true) StructType(struct.map { field => val otherField = otherFields.get(field.name) val newField = tf(path, field, otherField, DELTA_COL_RESOLVER) otherField match { case Some(other) => newField.copy( dataType = transform(path :+ field.name, field.dataType, other.dataType) ) case None => newField } }) case (map: MapType, otherMap: MapType) => map.copy( keyType = transform(path :+ "key", map.keyType, otherMap.keyType), valueType = transform(path :+ "value", map.valueType, otherMap.valueType) ) case (array: ArrayType, otherArray: ArrayType) => array.copy( elementType = transform(path :+ "element", array.elementType, otherArray.elementType) ) case _ => dt } newDt.asInstanceOf[E] } transform(Seq.empty, schema, other) } /** * * Taken from DataType * * Compares two types, ignoring compatible nullability of ArrayType, MapType, StructType, and * ignoring case sensitivity of field names in StructType. * * Compatible nullability is defined as follows: * - If `from` and `to` are ArrayTypes, `from` has a compatible nullability with `to` * if and only if `to.containsNull` is true, or both of `from.containsNull` and * `to.containsNull` are false. * - If `from` and `to` are MapTypes, `from` has a compatible nullability with `to` * if and only if `to.valueContainsNull` is true, or both of `from.valueContainsNull` and * `to.valueContainsNull` are false. * - If `from` and `to` are StructTypes, `from` has a compatible nullability with `to` * if and only if for all every pair of fields, `to.nullable` is true, or both * of `fromField.nullable` and `toField.nullable` are false. */ def equalsIgnoreCaseAndCompatibleNullability(from: DataType, to: DataType): Boolean = { (from, to) match { case (ArrayType(fromElement, fn), ArrayType(toElement, tn)) => (tn || !fn) && equalsIgnoreCaseAndCompatibleNullability(fromElement, toElement) case (MapType(fromKey, fromValue, fn), MapType(toKey, toValue, tn)) => (tn || !fn) && equalsIgnoreCaseAndCompatibleNullability(fromKey, toKey) && equalsIgnoreCaseAndCompatibleNullability(fromValue, toValue) case (StructType(fromFields), StructType(toFields)) => fromFields.length == toFields.length && fromFields.zip(toFields).forall { case (fromField, toField) => fromField.name.equalsIgnoreCase(toField.name) && (toField.nullable || !fromField.nullable) && equalsIgnoreCaseAndCompatibleNullability(fromField.dataType, toField.dataType) } case (fromDataType, toDataType) => fromDataType == toDataType } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.schema import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import scala.util.control.NonFatal import org.apache.spark.sql.delta.Relocated._ import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.{DeltaAnalysisException, DeltaColumnMappingMode, DeltaErrors, DeltaLog, GeneratedColumn, NoMapping, TypeWidening, TypeWideningMode} import org.apache.spark.sql.delta.{RowCommitVersion, RowId} import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions.Protocol import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaMergingUtils._ import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} import org.apache.spark.sql.delta.sources.DeltaSourceUtils.GENERATION_EXPRESSION_METADATA_KEY import org.apache.spark.sql.delta.sources.{DeltaSQLConf, DeltaStreamUtils} import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.spark.internal.MDC import org.apache.spark.sql._ import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.analysis.{Resolver, UnresolvedAttribute} import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference, Expression, GetArrayItem, GetArrayStructFields, GetMapValue, GetStructField} import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, Project} import org.apache.spark.sql.catalyst.util.{CharVarcharUtils, ResolveDefaultColumnsUtils} import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ object SchemaUtils extends DeltaLogging { // We use case insensitive resolution while writing into Delta val DELTA_COL_RESOLVER: (String, String) => Boolean = org.apache.spark.sql.catalyst.analysis.caseInsensitiveResolution private val ARRAY_ELEMENT_INDEX = 0 private val MAP_KEY_INDEX = 0 private val MAP_VALUE_INDEX = 1 /** * Finds `StructField`s that match a given check `f`. Returns the path to the column, and the * field. * * @param checkComplexTypes While `StructType` is also a complex type, since we're returning * StructFields, we definitely recurse into StructTypes. This flag * defines whether we should recurse into ArrayType and MapType. */ def filterRecursively( schema: DataType, checkComplexTypes: Boolean)(f: StructField => Boolean): Seq[(Seq[String], StructField)] = { def recurseIntoComplexTypes( complexType: DataType, columnStack: Seq[String]): Seq[(Seq[String], StructField)] = complexType match { case s: StructType => s.fields.flatMap { sf => val includeLevel = if (f(sf)) Seq((columnStack, sf)) else Nil includeLevel ++ recurseIntoComplexTypes(sf.dataType, columnStack :+ sf.name) } case a: ArrayType if checkComplexTypes => recurseIntoComplexTypes(a.elementType, columnStack :+ "element") case m: MapType if checkComplexTypes => recurseIntoComplexTypes(m.keyType, columnStack :+ "key") ++ recurseIntoComplexTypes(m.valueType, columnStack :+ "value") case _ => Nil } recurseIntoComplexTypes(schema, Nil) } /** Copied over from DataType for visibility reasons. */ def typeExistsRecursively(dt: DataType)(f: DataType => Boolean): Boolean = dt match { case s: StructType => f(s) || s.fields.exists(field => typeExistsRecursively(field.dataType)(f)) case a: ArrayType => f(a) || typeExistsRecursively(a.elementType)(f) case m: MapType => f(m) || typeExistsRecursively(m.keyType)(f) || typeExistsRecursively(m.valueType)(f) case other => f(other) } def findAnyTypeRecursively(dt: DataType)(f: DataType => Boolean): Option[DataType] = dt match { case s: StructType => Some(s).filter(f).orElse(s.fields .flatMap(field => findAnyTypeRecursively(field.dataType)(f)).find(_ => true)) case a: ArrayType => Some(a).filter(f).orElse(findAnyTypeRecursively(a.elementType)(f)) case m: MapType => Some(m).filter(f).orElse(findAnyTypeRecursively(m.keyType)(f)) .orElse(findAnyTypeRecursively(m.valueType)(f)) case other => Some(other).filter(f) } /** * Checks if a given data type contains a NullType, including inside UDTs. * `typeExistsRecursively` does not recurse into UDT sqlTypes, so this method * explicitly handles that case. */ def nullTypeExistsRecursively( t: DataType ): Boolean = { typeExistsRecursively(t) { case _: NullType => true case udt: UserDefinedType[_] => nullTypeExistsRecursively(udt.sqlType) case _ => false } } /** Turns the data types to nullable in a recursive manner for nested columns. */ def typeAsNullable(dt: DataType): DataType = dt match { case s: StructType => s.asNullable case a @ ArrayType(s: StructType, _) => a.copy(s.asNullable, containsNull = true) case a: ArrayType => a.copy(containsNull = true) case m @ MapType(s1: StructType, s2: StructType, _) => m.copy(s1.asNullable, s2.asNullable, valueContainsNull = true) case m @ MapType(s1: StructType, _, _) => m.copy(keyType = s1.asNullable, valueContainsNull = true) case m @ MapType(_, s2: StructType, _) => m.copy(valueType = s2.asNullable, valueContainsNull = true) case other => other } /** * Drops null types from the DataFrame if they exist. We don't have easy ways of generating types * such as MapType and ArrayType, therefore if these types contain NullType in their elements, * we will throw an AnalysisException. */ def dropNullTypeColumns(df: DataFrame): DataFrame = { val schema = df.schema if (!nullTypeExistsRecursively(schema)) return df def generateSelectExpr(sf: StructField, nameStack: Seq[String]): Column = sf.dataType match { case st: StructType => val nested = st.fields.flatMap { f => if (f.dataType.isInstanceOf[NullType]) { None } else { Some(generateSelectExpr(f, nameStack :+ sf.name)) } } val colName = UnresolvedAttribute.apply(nameStack :+ sf.name).sql when(col(colName).isNull, null) .otherwise(struct(nested: _*)) .alias(sf.name) case a: ArrayType if nullTypeExistsRecursively(a) => val colName = UnresolvedAttribute.apply(nameStack :+ sf.name).sql throw new DeltaAnalysisException( errorClass = "DELTA_COMPLEX_TYPE_COLUMN_CONTAINS_NULL_TYPE", messageParameters = Array(colName, "ArrayType")) case m: MapType if nullTypeExistsRecursively(m) => val colName = UnresolvedAttribute.apply(nameStack :+ sf.name).sql throw new DeltaAnalysisException( errorClass = "DELTA_COMPLEX_TYPE_COLUMN_CONTAINS_NULL_TYPE", messageParameters = Array(colName, "MapType")) case udt: UserDefinedType[_] if nullTypeExistsRecursively(udt.sqlType) => val colName = UnresolvedAttribute.apply(nameStack :+ sf.name).sql throw new DeltaAnalysisException( errorClass = "DELTA_USER_DEFINED_TYPE_COLUMN_CONTAINS_NULL_TYPE", messageParameters = Array(colName, udt.userClass.getName)) case _ => val colName = UnresolvedAttribute.apply(nameStack :+ sf.name).sql col(colName).alias(sf.name) } val selectExprs = schema.flatMap { f => if (f.dataType.isInstanceOf[NullType]) None else Some(generateSelectExpr(f, Nil)) } df.select(selectExprs: _*) } /** * A char(x)/varchar(x) related types are internally stored as string type with the constraint * information stored in the metadata. For example: * + char(10) is (string, char_varchar_metadata = "char(10)") * + array[varchar(10)] is (array[string], char_varchar_metadata = "array[varchar(10)]") * This method converts the string + metadata representation to the actual type. * + (string, char_varchar_metadata = "char(10)") -> (char(10), char_varchar_metadata = "") * + (array[string], char_varchar_metadata = "array[varchar(10)]") * -> (array[varchar(10)], char_varchar_metadata = "") */ private def getRawFieldWithoutCharVarcharMetadata(field: StructField): StructField = { val rawField = CharVarcharUtils.getRawType(field.metadata) .map(dt => field.copy(dataType = dt)) .getOrElse(field) val throwAwayAttrRef = AttributeReference( rawField.name, rawField.dataType, nullable = rawField.nullable, rawField.metadata)() val cleanedMetadata = CharVarcharUtils.cleanAttrMetadata(throwAwayAttrRef).metadata rawField.copy(metadata = cleanedMetadata) } /** * Sets a data type to a field in a char/varchar-safe manner. A char(x)/varchar(x) related types * consists of two parts: a string-based type and the constraint information stored in the * metadata. Simply changing the data type will lead to unexpected results. * * For example, an array[varchar(10)] type is internally represented as * (array[string], char_varchar_metadata = "array[varchar(10)]"). If we convert it into an * array[string] simply by setting the data type part, the metadata part will still be there, and * the type will still stay as array[varchar(10)]. * * This method first converts the field into its raw type without the metadata part, then sets the * data type part to the new data type, and finally converts the whole thing back to the original * representation. * * In the above example, this methods will convert the array[varchar(10)] representation into * (array[varchar(10)], char_varchar_metadata = ""), set the data type to array[string] * (array[string], char_varchar_metadata = ""), and finally convert it back to * (array[string], char_varchar_metadata = ""), which happens to be the same. */ def setFieldDataTypeCharVarcharSafe(field: StructField, newDataType: DataType): StructField = { val byPassCharVarcharToStringFix = SparkSession.active.conf.get(DeltaSQLConf.DELTA_BYPASS_CHARVARCHAR_TO_STRING_FIX) // Convert the field into its raw type without the metadata part val rawField = if (byPassCharVarcharToStringFix) { field } else { getRawFieldWithoutCharVarcharMetadata(field) } // Set the new data type val rawFieldWithNewDataType = rawField.copy(dataType = newDataType) // Convert it back to the original representation if (byPassCharVarcharToStringFix) { rawFieldWithNewDataType } else { val throwAwayStructType = StructType(Seq(rawFieldWithNewDataType)) CharVarcharUtils.replaceCharVarcharWithStringInSchema(throwAwayStructType) .head } } /** * Drops null types from the schema if they exist. We do not recurse into Array and Map types, * because we do not expect null types to exist in those columns, as Delta doesn't allow it during * writes. */ def dropNullTypeColumns(schema: StructType): StructType = { def recurseAndRemove(struct: StructType): Seq[StructField] = { struct.flatMap { case sf @ StructField(_, s: StructType, _, _) => Some(sf.copy(dataType = StructType(recurseAndRemove(s)))) case StructField(_, n: NullType, _, _) => None case other => Some(other) } } StructType(recurseAndRemove(schema)) } /** * Returns the name of the first column/field that has null type (void). */ def findNullTypeColumn(schema: StructType): Option[String] = { // Helper method to recursively check nested structs. def findNullTypeColumnRec(s: StructType, nameStack: Seq[String]): Option[String] = { val nullFields = s.flatMap { case StructField(name, n: NullType, _, _) => Some((nameStack :+ name).mkString(".")) case StructField(name, s: StructType, _, _) => findNullTypeColumnRec(s, nameStack :+ name) // Note that we don't recursively check Array and Map types because NullTypes are already // not allowed (see 'dropNullTypeColumns'). case _ => None } return nullFields.headOption } if (nullTypeExistsRecursively(schema)) { findNullTypeColumnRec(schema, Seq.empty) } else { None } } /** * Recursively rewrite the query field names according to the table schema within nested * data types. * * The same assumptions as in [[normalizeColumnNames]] are made. * * @param sourceDataType The data type that needs normalizing. * @param tableDataType The normalization template from the table's schema. * @param sourceParentFields The path (starting from the top level) to the nested field * with `sourceDataType`. * @param tableSchema The entire schema of the table. * * @return A normalized version of `sourceDataType`. */ def normalizeColumnNamesInDataType( deltaLog: DeltaLog, sourceDataType: DataType, tableDataType: DataType, sourceParentFields: Seq[String], tableSchema: StructType ): DataType = { def getMatchingTableField( sourceField: StructField, tableFields: Map[String, StructField]): StructField = { tableFields.get(sourceField.name) match { case Some(tableField) => tableField case None => val columnPath = (sourceParentFields ++ Seq(sourceField.name)).mkString(".") throw DeltaErrors.cannotResolveColumn(columnPath, tableSchema) } } (sourceDataType, tableDataType) match { case (sourceStruct: StructType, tableStruct: StructType) => val tableFields = toFieldMap(tableStruct.fields, caseSensitive = false) val normalizedFields = sourceStruct.fields.map { sourceField => val tableField = getMatchingTableField(sourceField, tableFields) val normalizedDataType = normalizeColumnNamesInDataType(deltaLog, sourceField.dataType, tableField.dataType, sourceParentFields :+ sourceField.name, tableSchema) val normalizedName = tableField.name sourceField.copy( name = normalizedName, dataType = normalizedDataType ) } sourceStruct.copy(fields = normalizedFields) case (sourceArray: ArrayType, tableArray: ArrayType) => val normalizedElementType = normalizeColumnNamesInDataType(deltaLog, sourceArray.elementType, tableArray.elementType, sourceParentFields, tableSchema) sourceArray.copy(elementType = normalizedElementType) case (sourceMap: MapType, tableMap: MapType) => val normalizedKeyType = normalizeColumnNamesInDataType(deltaLog, sourceMap.keyType, tableMap.keyType, sourceParentFields, tableSchema) val normalizedValueType = normalizeColumnNamesInDataType(deltaLog, sourceMap.valueType, tableMap.valueType, sourceParentFields, tableSchema) sourceMap.copy( keyType = normalizedKeyType, valueType = normalizedValueType ) case (_: NullType, _) => // When schema evolution adds a new column during MERGE, it can be represented with // a NullType in the schema of the data written by the MERGE. sourceDataType case (_: AtomicType, _: AtomicType) => // Some atomic types (e.g. integral types) can be cast to each other later on. For now, // it's enough to know that there are no nested fields inside the atomic types that might // require normalization. sourceDataType case _ => if (DeltaUtils.isTesting) { assert(sourceDataType == tableDataType, s"Types without nesting should match but $sourceDataType != $tableDataType") } else if (sourceDataType != tableDataType) { recordDeltaEvent( deltaLog = deltaLog, opType = "delta.assertions.schemaNormalization.nonNestedTypeMismatch", tags = Map.empty, data = Map( "sourceDataType" -> sourceDataType.json, "tableDataType" -> tableDataType.json ), path = None) } // The data types are compatible. sourceDataType } } /** * Rewrite the query field names according to the table schema. This method assumes that all * schema validation checks have been made and this is the last operation before writing into * Delta. */ def normalizeColumnNames( deltaLog: DeltaLog, baseSchema: StructType, data: Dataset[_] ): DataFrame = { val dataSchema = data.schema val dataFields = explodeNestedFieldNames(dataSchema).toSet val tableFields = explodeNestedFieldNames(baseSchema).toSet if (dataFields.subsetOf(tableFields)) { data.toDF() } else { // Allow the same shortcut logic (as the above `if` stmt) if the only extra fields are CDC // metadata fields. val nonCdcFields = dataFields.filterNot { f => f == CDCReader.CDC_PARTITION_COL || f == CDCReader.CDC_TYPE_COLUMN_NAME } if (nonCdcFields.subsetOf(tableFields)) { return data.toDF() } val baseFields = toFieldMap(baseSchema, caseSensitive = false) val aliasExpressions = dataSchema.map { field => val (originalCase, castDataType): (String, Option[DataType]) = baseFields.get(field.name) match { case Some(original) => val normalizedDataType = normalizeColumnNamesInDataType(deltaLog, field.dataType, original.dataType, Seq(field.name), baseSchema) (original.name, Option.when(field.dataType != normalizedDataType)(normalizedDataType)) // This is a virtual partition column used for doing CDC writes. It's not actually // in the table schema. case None if field.name == CDCReader.CDC_TYPE_COLUMN_NAME || field.name == CDCReader.CDC_PARTITION_COL => (field.name, None) // Consider Row Id columns internal if Row Ids are enabled. case None if RowId.RowIdMetadataStructField.isRowIdColumn(field) => (field.name, None) case None if RowCommitVersion.MetadataStructField.isRowCommitVersionColumn(field) => (field.name, None) case None => throw DeltaErrors.cannotResolveColumn(field.name, baseSchema) } var expression = fieldToColumn(field) castDataType.foreach { castType => expression = expression.cast(castType) } if (originalCase != field.name) { expression = expression.as(originalCase) } expression } data.queryExecution match { case incrementalExecution: IncrementalExecution => DeltaStreamUtils.selectFromStreamingDataFrame( incrementalExecution, data.toDF(), aliasExpressions: _*) case _ => data.select(aliasExpressions: _*) } } } /** * A helper function to check if partition columns are the same. * This function only checks for partition column names. * Please use with other schema check functions for detecting type change etc. */ def isPartitionCompatible( newPartitionColumns: Seq[String] = Seq.empty, oldPartitionColumns: Seq[String] = Seq.empty): Boolean = { newPartitionColumns == oldPartitionColumns } /** * As the Delta snapshots update, the schema may change as well. This method defines whether the * new schema of a Delta table can be used with a previously analyzed LogicalPlan. Our * rules are to return false if: * - Dropping any column or struct field that was present in the existing schema, if not * allowMissingColumns * - Any change of datatype, unless eligible for widening. The caller specifies eligible type * changes via `typeWideningMode`. * - Change of partition columns. Although analyzed LogicalPlan is not changed, * physical structure of data is changed and thus is considered not read compatible. * - If `forbidTightenNullability` = true: * - Forbids tightening the nullability (existing nullable=true -> read nullable=false) * - Typically Used when the existing schema refers to the schema of written data, such as * when a Delta streaming source reads a schema change (existingSchema) which * has nullable=true, using the latest schema which has nullable=false, so we should not * project nulls from the data into the non-nullable read schema. * - Otherwise: * - Forbids relaxing the nullability (existing nullable=false -> read nullable=true) * - Typically Used when the read schema refers to the schema of written data, such as during * Delta scan, the latest schema during execution (readSchema) has nullable=true but during * analysis phase the schema (existingSchema) was nullable=false, so we should not project * nulls from the later data onto a non-nullable schema analyzed in the past. */ def isReadCompatible( existingSchema: StructType, readSchema: StructType, forbidTightenNullability: Boolean = false, allowMissingColumns: Boolean = false, typeWideningMode: TypeWideningMode = TypeWideningMode.NoTypeWidening, newPartitionColumns: Seq[String] = Seq.empty, oldPartitionColumns: Seq[String] = Seq.empty): Boolean = { def isNullabilityCompatible(existingNullable: Boolean, readNullable: Boolean): Boolean = { if (forbidTightenNullability) { readNullable || !existingNullable } else { existingNullable || !readNullable } } def isDatatypeReadCompatible(existing: DataType, newtype: DataType): Boolean = { (existing, newtype) match { case (e: StructType, n: StructType) => isReadCompatible(e, n, forbidTightenNullability, typeWideningMode = typeWideningMode, allowMissingColumns = allowMissingColumns ) case (e: ArrayType, n: ArrayType) => // if existing elements are non-nullable, so should be the new element isNullabilityCompatible(e.containsNull, n.containsNull) && isDatatypeReadCompatible(e.elementType, n.elementType) case (e: MapType, n: MapType) => // if existing value is non-nullable, so should be the new value isNullabilityCompatible(e.valueContainsNull, n.valueContainsNull) && isDatatypeReadCompatible(e.keyType, n.keyType) && isDatatypeReadCompatible(e.valueType, n.valueType) case (e: AtomicType, n: AtomicType) if typeWideningMode.shouldWidenTo(fromType = e, toType = n) => true case (a, b) => a == b } } def isStructReadCompatible(existing: StructType, newtype: StructType): Boolean = { val existingFields = toFieldMap(existing) // scalastyle:off caselocale val existingFieldNames = existing.fieldNames.map(_.toLowerCase).toSet assert(existingFieldNames.size == existing.length, "Delta tables don't allow field names that only differ by case") val newFields = newtype.fieldNames.map(_.toLowerCase).toSet assert(newFields.size == newtype.length, "Delta tables don't allow field names that only differ by case") // scalastyle:on caselocale if (!allowMissingColumns && !(existingFieldNames.subsetOf(newFields) && isPartitionCompatible(newPartitionColumns, oldPartitionColumns))) { // Dropped a column that was present in the DataFrame schema return false } newtype.forall { newField => // new fields are fine, they just won't be returned existingFields.get(newField.name).forall { existingField => // we know the name matches modulo case - now verify exact match (existingField.name == newField.name // if existing value is non-nullable, so should be the new value && isNullabilityCompatible(existingField.nullable, newField.nullable) // and the type of the field must be compatible, too && isDatatypeReadCompatible(existingField.dataType, newField.dataType)) } } } isStructReadCompatible(existingSchema, readSchema) } /** * Compare an existing schema to a specified new schema and * return a message describing the first difference found, if any: * - different field name or datatype * - different metadata */ def reportDifferences(existingSchema: StructType, specifiedSchema: StructType): Seq[String] = { def canOrNot(can: Boolean) = if (can) "can" else "can not" def isOrNon(b: Boolean) = if (b) "" else "non-" def missingFieldsMessage(fields: Set[String]) : String = { s"Specified schema is missing field(s): ${fields.mkString(", ")}" } def additionalFieldsMessage(fields: Set[String]) : String = { s"Specified schema has additional field(s): ${fields.mkString(", ")}" } def fieldNullabilityMessage(field: String, specified: Boolean, existing: Boolean) : String = { s"Field $field is ${isOrNon(specified)}nullable in specified " + s"schema but ${isOrNon(existing)}nullable in existing schema." } def arrayNullabilityMessage(field: String, specified: Boolean, existing: Boolean) : String = { s"Array field $field ${canOrNot(specified)} contain null in specified schema " + s"but ${canOrNot(existing)} in existing schema" } def valueNullabilityMessage(field: String, specified: Boolean, existing: Boolean) : String = { s"Map field $field ${canOrNot(specified)} contain null values in specified schema " + s"but ${canOrNot(existing)} in existing schema" } def removeGenerationExpressionMetadata(metadata: Metadata): Metadata = { new MetadataBuilder() .withMetadata(metadata) .remove(GENERATION_EXPRESSION_METADATA_KEY) .build() } def metadataDifferentMessage(field: String, specified: Metadata, existing: Metadata) : String = { val specifiedGenerationExpr = GeneratedColumn.getGenerationExpressionStr(specified) val existingGenerationExpr = GeneratedColumn.getGenerationExpressionStr(existing) var metadataDiffMessage = "" if (specifiedGenerationExpr != existingGenerationExpr) { metadataDiffMessage += s"""Specified generation expression for field $field is different from existing schema: |Specified: ${specifiedGenerationExpr.getOrElse("")} |Existing: ${existingGenerationExpr.getOrElse("")}""".stripMargin } val specifiedMetadataWithoutGenerationExpr = removeGenerationExpressionMetadata(specified) val existingMetadataWithoutGenerationExpr = removeGenerationExpressionMetadata(existing) if (specifiedMetadataWithoutGenerationExpr != existingMetadataWithoutGenerationExpr) { if (metadataDiffMessage.nonEmpty) metadataDiffMessage += "\n" metadataDiffMessage += s"""Specified metadata for field $field is different from existing schema: |Specified: $specifiedMetadataWithoutGenerationExpr |Existing: $existingMetadataWithoutGenerationExpr""".stripMargin } metadataDiffMessage } def typeDifferenceMessage(field: String, specified: DataType, existing: DataType) : String = { s"""Specified type for $field is different from existing schema: |Specified: ${specified.typeName} |Existing: ${existing.typeName}""".stripMargin } // prefix represents the nested field(s) containing this schema def structDifference(existing: StructType, specified: StructType, prefix: String) : Seq[String] = { // 1. ensure set of fields is the same val existingFieldNames = existing.fieldNames.toSet val specifiedFieldNames = specified.fieldNames.toSet val missingFields = existingFieldNames diff specifiedFieldNames val missingFieldsDiffs = if (missingFields.isEmpty) Nil else Seq(missingFieldsMessage(missingFields.map(prefix + _))) val extraFields = specifiedFieldNames diff existingFieldNames val extraFieldsDiffs = if (extraFields.isEmpty) Nil else Seq(additionalFieldsMessage(extraFields.map(prefix + _))) // 2. for each common field, ensure it has the same type and metadata val existingFields = toFieldMap(existing) val specifiedFields = toFieldMap(specified) val fieldsDiffs = (existingFieldNames intersect specifiedFieldNames).flatMap( (name: String) => fieldDifference(existingFields(name), specifiedFields(name), prefix)) missingFieldsDiffs ++ extraFieldsDiffs ++ fieldsDiffs } def fieldDifference(existing: StructField, specified: StructField, prefix: String) : Seq[String] = { val name = s"$prefix${existing.name}" val nullabilityDiffs = if (existing.nullable == specified.nullable) Nil else Seq(fieldNullabilityMessage(s"$name", specified.nullable, existing.nullable)) val metadataDiffs = if (existing.metadata == specified.metadata) Nil else Seq(metadataDifferentMessage(s"$name", specified.metadata, existing.metadata)) val typeDiffs = typeDifference(existing.dataType, specified.dataType, name) nullabilityDiffs ++ metadataDiffs ++ typeDiffs } def typeDifference(existing: DataType, specified: DataType, field: String) : Seq[String] = { (existing, specified) match { case (e: StructType, s: StructType) => structDifference(e, s, s"$field.") case (e: ArrayType, s: ArrayType) => arrayDifference(e, s, s"$field[]") case (e: MapType, s: MapType) => mapDifference(e, s, s"$field") case (e, s) if e != s => Seq(typeDifferenceMessage(field, s, e)) case _ => Nil } } def arrayDifference(existing: ArrayType, specified: ArrayType, field: String): Seq[String] = { val elementDiffs = typeDifference(existing.elementType, specified.elementType, field) val nullabilityDiffs = if (existing.containsNull == specified.containsNull) Nil else Seq(arrayNullabilityMessage(field, specified.containsNull, existing.containsNull)) elementDiffs ++ nullabilityDiffs } def mapDifference(existing: MapType, specified: MapType, field: String) : Seq[String] = { val keyDiffs = typeDifference(existing.keyType, specified.keyType, s"$field[key]") val valueDiffs = typeDifference(existing.valueType, specified.valueType, s"$field[value]") val nullabilityDiffs = if (existing.valueContainsNull == specified.valueContainsNull) Nil else Seq( valueNullabilityMessage(field, specified.valueContainsNull, existing.valueContainsNull)) keyDiffs ++ valueDiffs ++ nullabilityDiffs } structDifference( existingSchema, CharVarcharUtils.replaceCharVarcharWithStringInSchema(specifiedSchema), "" ) } /** * Copied verbatim from Apache Spark. * * Returns a field in this struct and its child structs, case insensitively. This is slightly less * performant than the case sensitive version. * * If includeCollections is true, this will return fields that are nested in maps and arrays. * * @param fieldNames The path to the field, in order from the root. For example, the column * nested.a.b.c would be Seq("nested", "a", "b", "c"). */ def findNestedFieldIgnoreCase( schema: StructType, fieldNames: Seq[String], includeCollections: Boolean = false): Option[StructField] = { @scala.annotation.tailrec def findRecursively( dataType: DataType, fieldNames: Seq[String], includeCollections: Boolean): Option[StructField] = { (fieldNames, dataType, includeCollections) match { case (Seq(fieldName, names @ _*), struct: StructType, _) => val field = struct.find(_.name.equalsIgnoreCase(fieldName)) if (names.isEmpty || field.isEmpty) { field } else { findRecursively(field.get.dataType, names, includeCollections) } case (_, _, false) => None // types nested in maps and arrays are not used case (Seq("key"), MapType(keyType, _, _), true) => // return the key type as a struct field to include nullability Some(StructField("key", keyType, nullable = false)) case (Seq("key", names @ _*), MapType(keyType, _, _), true) => findRecursively(keyType, names, includeCollections) case (Seq("value"), MapType(_, valueType, isNullable), true) => // return the value type as a struct field to include nullability Some(StructField("value", valueType, nullable = isNullable)) case (Seq("value", names @ _*), MapType(_, valueType, _), true) => findRecursively(valueType, names, includeCollections) case (Seq("element"), ArrayType(elementType, isNullable), true) => // return the element type as a struct field to include nullability Some(StructField("element", elementType, nullable = isNullable)) case (Seq("element", names @ _*), ArrayType(elementType, _), true) => findRecursively(elementType, names, includeCollections) case _ => None } } findRecursively(schema, fieldNames, includeCollections) } /** * Returns the path of the given column in `schema` as a list of ordinals (0-based), each value * representing the position at the current nesting level starting from the root. * * For ArrayType: accessing the array's element adds a position 0 to the position list. * e.g. accessing a.element.y would have the result -> Seq(..., positionOfA, 0, positionOfY) * * For MapType: accessing the map's key adds a position 0 to the position list. * e.g. accessing m.key.y would have the result -> Seq(..., positionOfM, 0, positionOfY) * * For MapType: accessing the map's value adds a position 1 to the position list. * e.g. accessing m.key.y would have the result -> Seq(..., positionOfM, 1, positionOfY) * * @param column The column to search for in the given struct. If the length of `column` is * greater than 1, we expect to enter a nested field. * @param schema The current struct we are looking at. * @param resolver The resolver to find the column. */ def findColumnPosition( column: Seq[String], schema: DataType, resolver: Resolver = DELTA_COL_RESOLVER): Seq[Int] = { def findRecursively( searchPath: Seq[String], currentType: DataType, currentPath: Seq[String] = Nil): Seq[Int] = { if (searchPath.isEmpty) return Nil val currentFieldName = searchPath.head val currentPathWithNestedField = currentPath :+ currentFieldName (currentType, currentFieldName) match { case (struct: StructType, _) => lazy val columnPath = UnresolvedAttribute(currentPathWithNestedField).name val pos = struct.indexWhere(f => resolver(f.name, currentFieldName)) if (pos == -1) { throw DeltaErrors.columnNotInSchemaException(columnPath, schema) } val childPosition = findRecursively( searchPath = searchPath.tail, currentType = struct(pos).dataType, currentPath = currentPathWithNestedField) pos +: childPosition case (map: MapType, "key") => val childPosition = findRecursively( searchPath = searchPath.tail, currentType = map.keyType, currentPath = currentPathWithNestedField) MAP_KEY_INDEX +: childPosition case (map: MapType, "value") => val childPosition = findRecursively( searchPath = searchPath.tail, currentType = map.valueType, currentPath = currentPathWithNestedField) MAP_VALUE_INDEX +: childPosition case (_: MapType, _) => throw DeltaErrors.foundMapTypeColumnException( prettyFieldName(currentPath :+ "key"), prettyFieldName(currentPath :+ "value"), schema) case (array: ArrayType, "element") => val childPosition = findRecursively( searchPath = searchPath.tail, currentType = array.elementType, currentPath = currentPathWithNestedField) ARRAY_ELEMENT_INDEX +: childPosition case (_: ArrayType, _) => throw DeltaErrors.incorrectArrayAccessByName( prettyFieldName(currentPath :+ "element"), prettyFieldName(currentPath), schema) case _ => throw DeltaErrors.columnPathNotNested(currentFieldName, currentType, currentPath, schema) } } try { findRecursively(column, schema) } catch { case e: DeltaAnalysisException => throw e case e: AnalysisException => throw DeltaErrors.errorFindingColumnPosition(column, schema, e.getMessage) } } /** * Returns the nested field at the given position in `parent`. See [[findColumnPosition]] for the * representation used for `position`. * @param parent The field used for the lookup. * @param position A list of ordinals (0-based) representing the path to the nested field in * `parent`. */ def getNestedFieldFromPosition(parent: StructField, position: Seq[Int]): StructField = { if (position.isEmpty) return parent val fieldPos = position.head parent.dataType match { case struct: StructType if fieldPos >= 0 && fieldPos < struct.size => getNestedFieldFromPosition(struct(fieldPos), position.tail) case map: MapType if fieldPos == MAP_KEY_INDEX => getNestedFieldFromPosition(StructField("key", map.keyType), position.tail) case map: MapType if fieldPos == MAP_VALUE_INDEX => getNestedFieldFromPosition(StructField("value", map.valueType), position.tail) case array: ArrayType if fieldPos == ARRAY_ELEMENT_INDEX => getNestedFieldFromPosition(StructField("element", array.elementType), position.tail) case _: StructType | _: ArrayType | _: MapType => throw new IllegalArgumentException( s"Invalid child position $fieldPos in ${parent.dataType}") case other => throw new IllegalArgumentException(s"Invalid indexing into non-nested type $other") } } /** * Returns the nested type at the given position in `schema`. See [[findColumnPosition]] for the * representation used for `position`. * @param parent The root schema used for the lookup. * @param position A list of ordinals (0-based) representing the path to the nested field in * `parent`. */ def getNestedTypeFromPosition(schema: DataType, position: Seq[Int]): DataType = getNestedFieldFromPosition(StructField("schema", schema), position).dataType /** * Pretty print the column path passed in. */ def prettyFieldName(columnPath: Seq[String]): String = { UnresolvedAttribute(columnPath).name } /** * Add a column to its child. * @param parent The parent data type. * @param column The column to add. * @param position The position to add the column. */ def addColumn[T <: DataType](parent: T, column: StructField, position: Seq[Int]): T = { if (position.isEmpty) { throw DeltaErrors.addColumnParentNotStructException(column, parent) } parent match { case struct: StructType => addColumnToStruct(struct, column, position).asInstanceOf[T] case map: MapType if position.head == MAP_KEY_INDEX => map.copy(keyType = addColumn(map.keyType, column, position.tail)).asInstanceOf[T] case map: MapType if position.head == MAP_VALUE_INDEX => map.copy(valueType = addColumn(map.valueType, column, position.tail)).asInstanceOf[T] case array: ArrayType if position.head == ARRAY_ELEMENT_INDEX => array.copy(elementType = addColumn(array.elementType, column, position.tail)).asInstanceOf[T] case _: ArrayType => throw DeltaErrors.incorrectArrayAccess() case other => throw DeltaErrors.addColumnParentNotStructException(column, other) } } /** * Add `column` to the specified `position` in a struct `schema`. * @param position A Seq of ordinals on where this column should go. It is a Seq to denote * positions in nested columns (0-based). For example: * * tableSchema: , b,c:STRUCT> * column: c2 * position: Seq(2, 1) * will return * result: , b,c:STRUCT> */ private def addColumnToStruct( schema: StructType, column: StructField, position: Seq[Int]): StructType = { // If the proposed new column includes a default value, return a specific "not supported" error. // The rationale is that such operations require the data source scan operator to implement // support for filling in the specified default value when the corresponding field is not // present in storage. That is not implemented yet for Delta, so we return this error instead. // The error message is descriptive and provides an easy workaround for the user. if (column.metadata.contains("CURRENT_DEFAULT")) { throw new DeltaAnalysisException( errorClass = "WRONG_COLUMN_DEFAULTS_FOR_DELTA_ALTER_TABLE_ADD_COLUMN_NOT_SUPPORTED", messageParameters = Array.empty) } require(position.nonEmpty, s"Don't know where to add the column $column") val slicePosition = position.head if (slicePosition < 0) { throw DeltaErrors.addColumnAtIndexLessThanZeroException( slicePosition.toString, column.toString) } val length = schema.length if (slicePosition > length) { throw DeltaErrors.indexLargerThanStruct(slicePosition, column, length) } if (slicePosition == length) { if (position.length > 1) { throw DeltaErrors.addColumnStructNotFoundException(slicePosition.toString) } return StructType(schema :+ column) } val (pre, post) = schema.splitAt(slicePosition) if (position.length > 1) { val field = post.head if (!column.nullable && field.nullable) { throw DeltaErrors.nullableParentWithNotNullNestedField } val mid = field.copy(dataType = addColumn(field.dataType, column, position.tail)) StructType(pre ++ Seq(mid) ++ post.tail) } else { StructType(pre ++ Seq(column) ++ post) } } /** * Drop a column from its child. * @param parent The parent data type. * @param position The position to drop the column. */ def dropColumn[T <: DataType](parent: T, position: Seq[Int]): (T, StructField) = { if (position.isEmpty) { throw DeltaErrors.dropNestedColumnsFromNonStructTypeException(parent) } parent match { case struct: StructType => val (t, s) = dropColumnInStruct(struct, position) (t.asInstanceOf[T], s) case map: MapType if position.head == MAP_KEY_INDEX => val (newKeyType, droppedColumn) = dropColumn(map.keyType, position.tail) map.copy(keyType = newKeyType).asInstanceOf[T] -> droppedColumn case map: MapType if position.head == MAP_VALUE_INDEX => val (newValueType, droppedColumn) = dropColumn(map.valueType, position.tail) map.copy(valueType = newValueType).asInstanceOf[T] -> droppedColumn case array: ArrayType if position.head == ARRAY_ELEMENT_INDEX => val (newElementType, droppedColumn) = dropColumn(array.elementType, position.tail) array.copy(elementType = newElementType).asInstanceOf[T] -> droppedColumn case _: ArrayType => throw DeltaErrors.incorrectArrayAccess() case other => throw DeltaErrors.dropNestedColumnsFromNonStructTypeException(other) } } /** * Drop from the specified `position` in `schema` and return with the original column. * @param position A Seq of ordinals on where this column should go. It is a Seq to denote * positions in nested columns (0-based). For example: * * tableSchema: , b,c:STRUCT> * position: Seq(2, 1) * will return * result: , b,c:STRUCT> */ private def dropColumnInStruct( schema: StructType, position: Seq[Int]): (StructType, StructField) = { require(position.nonEmpty, "Don't know where to drop the column") val slicePosition = position.head if (slicePosition < 0) { throw DeltaErrors.dropColumnAtIndexLessThanZeroException(slicePosition) } val length = schema.length if (slicePosition >= length) { throw DeltaErrors.indexLargerOrEqualThanStruct(slicePosition, length) } val (pre, post) = schema.splitAt(slicePosition) val field = post.head if (position.length > 1) { val (newType, droppedColumn) = dropColumn(field.dataType, position.tail) val mid = field.copy(dataType = newType) StructType(pre ++ Seq(mid) ++ post.tail) -> droppedColumn } else { if (length == 1) { throw DeltaErrors.dropColumnOnSingleFieldSchema(schema) } StructType(pre ++ post.tail) -> field } } /** * Check if the two data types can be changed. * * @param failOnAmbiguousChanges Throw an error if a StructField both has columns dropped and new * columns added. These are ambiguous changes, because we don't * know if a column needs to be renamed, dropped, or added. * @param allowTypeWidening Whether widening type changes as defined in [[TypeWidening]] * can be applied. * @return None if the data types can be changed, otherwise Some(err) containing the reason. */ def canChangeDataType( from: DataType, to: DataType, resolver: Resolver, columnMappingMode: DeltaColumnMappingMode, columnPath: Seq[String] = Nil, failOnAmbiguousChanges: Boolean = false, allowTypeWidening: Boolean = false): Option[String] = { def verify(cond: Boolean, err: => String): Unit = { if (!cond) { throw DeltaErrors.cannotChangeDataType(err) } } def verifyNullability(fn: Boolean, tn: Boolean, columnPath: Seq[String]): Unit = { verify(tn || !fn, s"tightening nullability of ${UnresolvedAttribute(columnPath).name}") } def check(fromDt: DataType, toDt: DataType, columnPath: Seq[String]): Unit = { (fromDt, toDt) match { case (ArrayType(fromElement, fn), ArrayType(toElement, tn)) => verifyNullability(fn, tn, columnPath) check(fromElement, toElement, columnPath :+ "element") case (MapType(fromKey, fromValue, fn), MapType(toKey, toValue, tn)) => verifyNullability(fn, tn, columnPath) check(fromKey, toKey, columnPath :+ "key") check(fromValue, toValue, columnPath :+ "value") case (f @ StructType(fromFields), t @ StructType(toFields)) => val remainingFields = mutable.Set[StructField]() remainingFields ++= fromFields var addingColumns = false toFields.foreach { toField => fromFields.find(field => resolver(field.name, toField.name)) match { case Some(fromField) => remainingFields -= fromField val newPath = columnPath :+ fromField.name verifyNullability(fromField.nullable, toField.nullable, newPath) check(fromField.dataType, toField.dataType, newPath) case None => addingColumns = true verify(toField.nullable, "adding non-nullable column " + UnresolvedAttribute(columnPath :+ toField.name).name) } } val columnName = UnresolvedAttribute(columnPath).name if (failOnAmbiguousChanges && remainingFields.nonEmpty && addingColumns) { throw DeltaErrors.ambiguousDataTypeChange(columnName, f, t) } if (columnMappingMode == NoMapping) { verify(remainingFields.isEmpty, s"dropping column(s) [${remainingFields.map(_.name).mkString(", ")}]" + (if (columnPath.nonEmpty) s" from $columnName" else "")) } case (fromDataType: AtomicType, toDataType: AtomicType) if allowTypeWidening => verify(TypeWidening.isTypeChangeSupported(fromDataType, toDataType), s"changing data type of ${UnresolvedAttribute(columnPath).name} " + s"from $fromDataType to $toDataType") case (fromDataType, toDataType) => verify(fromDataType == toDataType, s"changing data type of ${UnresolvedAttribute(columnPath).name} " + s"from $fromDataType to $toDataType") } } try { check(from, to, columnPath) None } catch { case e: AnalysisException => Some(e.message) } } /** * Copy the nested data type between two data types in a char/varchar safe manner. * See documentation of [[getRawFieldWithoutCharVarcharMetadata]] and * [[setFieldDataTypeCharVarcharSafe]] for more context. * * This method uses [[getRawFieldWithoutCharVarcharMetadata]] on both the source and * target fields to ensure that the metadata information is included in the data type * before changing the data type. For example, to convert from a varchar(1) to varchar(10), * we first change their representation: * * Source: (string, char_varchar_metadata = "varchar(1)") * -> (varchar(1), char_varchar_metadata = "") * Target: (string, char_varchar_metadata = "varchar(10)") * -> (varchar(10), char_varchar_metadata = "") * * Then, we change the data type of the target to that of the source: * (varchar(1), char_varchar_metadata = "") -> (varchar(10), char_varchar_metadata = "") * * Finally, we set the metadata back to the target: * (varchar(10), char_varchar_metadata = "") -> (string, char_varchar_metadata = "varchar(10)") */ def changeFieldDataTypeCharVarcharSafe( fromField: StructField, toField: StructField, resolver: Resolver): StructField = { val (safeFromField, safeToField) = if (SparkSession.active.conf.get(DeltaSQLConf.DELTA_BYPASS_CHARVARCHAR_TO_STRING_FIX)) { (fromField, toField) } else { (getRawFieldWithoutCharVarcharMetadata(fromField), getRawFieldWithoutCharVarcharMetadata(toField)) } val newDataType = SchemaUtils.changeDataType( safeFromField.dataType, safeToField.dataType, resolver) setFieldDataTypeCharVarcharSafe(fromField, newDataType) } /** * Copy the nested data type between two data types. */ def changeDataType(from: DataType, to: DataType, resolver: Resolver): DataType = { (from, to) match { case (ArrayType(fromElement, fn), ArrayType(toElement, _)) => ArrayType(changeDataType(fromElement, toElement, resolver), fn) case (MapType(fromKey, fromValue, fn), MapType(toKey, toValue, _)) => MapType( changeDataType(fromKey, toKey, resolver), changeDataType(fromValue, toValue, resolver), fn) case (StructType(fromFields), StructType(toFields)) => StructType( toFields.map { toField => fromFields.find(field => resolver(field.name, toField.name)).map { fromField => toField.getComment().map(fromField.withComment).getOrElse(fromField) .copy( dataType = changeDataType(fromField.dataType, toField.dataType, resolver), nullable = toField.nullable) }.getOrElse(toField) } ) case (_, toDataType) => toDataType } } /** * Runs the transform function `tf` on all nested StructTypes, MapTypes and ArrayTypes in the * schema. * If `colName` is defined, the transform function is only applied to all the fields with the * given name. There may be multiple matches if nested fields with the same name exist in the * schema, it is the responsibility of the caller to check the full field path before transforming * a field. * @param schema to transform. * @param colName Optional name to match for * @param tf function to apply on the StructType. * @return the transformed schema. */ def transformSchema( schema: StructType, colName: Option[String] = None)( tf: (Seq[String], DataType, Resolver) => DataType): StructType = { def transform[E <: DataType](path: Seq[String], dt: E): E = { val newDt = dt match { case struct @ StructType(fields) => val newStruct = if (colName.isEmpty || fields.exists(f => colName.contains(f.name))) { tf(path, struct, DELTA_COL_RESOLVER).asInstanceOf[StructType] } else { struct } StructType(newStruct.fields.map { field => field.copy(dataType = transform(path :+ field.name, field.dataType)) }) case array: ArrayType => val newArray = if (colName.isEmpty || colName.contains("element")) { tf(path, array, DELTA_COL_RESOLVER).asInstanceOf[ArrayType] } else { array } newArray.copy(elementType = transform(path :+ "element", newArray.elementType)) case map: MapType => val newMap = if (colName.isEmpty || colName.contains("key") || colName.contains("value")) { tf(path, map, DELTA_COL_RESOLVER).asInstanceOf[MapType] } else { map } newMap.copy( keyType = transform(path :+ "key", newMap.keyType), valueType = transform(path :+ "value", newMap.valueType)) case other => other } newDt.asInstanceOf[E] } transform(Seq.empty, schema) } /** * Transform (nested) columns in a schema using the given path and parameter pairs. The transform * function is only invoked when a field's path matches one of the input paths. * * @param schema to transform * @param input paths and parameter pairs. The paths point to fields we want to transform. The * parameters will be passed to the transform function for a matching field. * @param tf function to apply per matched field. This function takes the field path, the field * itself and the input names and payload pairs that matched the field name. It should * return a new field. * @tparam E the type of the payload used for transforming fields. * @return the transformed schema. */ def transformColumns[E]( schema: StructType, input: Seq[(Seq[String], E)])( tf: (Seq[String], StructField, Seq[(Seq[String], E)]) => StructField): StructType = { // scalastyle:off caselocale val inputLookup = input.groupBy(_._1.map(_.toLowerCase)) SchemaMergingUtils.transformColumns(schema) { (path, field, resolver) => // Find the parameters that match this field name. val fullPath = path :+ field.name val normalizedFullPath = fullPath.map(_.toLowerCase) val matches = inputLookup.get(normalizedFullPath).toSeq.flatMap { // Keep only the input name(s) that actually match the field name(s). Note // that the Map guarantees that the zipped sequences have the same size. _.filter(_._1.zip(fullPath).forall(resolver.tupled)) } if (matches.nonEmpty) { tf(path, field, matches) } else { field } } // scalastyle:on caselocale } /** * Check if the schema contains invalid char in the column names depending on the mode. */ def checkSchemaFieldNames(schema: StructType, columnMappingMode: DeltaColumnMappingMode): Unit = { if (columnMappingMode != NoMapping) { return } val invalidColumnNames = findInvalidColumnNames(SchemaMergingUtils.explodeNestedFieldNames(schema)) if (invalidColumnNames.nonEmpty) { throw DeltaErrors.foundInvalidCharsInColumnNames(invalidColumnNames) } } /** * Verifies that the column names are acceptable by Parquet and henceforth Delta. Parquet doesn't * accept the characters ' ,;{}()\n\t='. We ensure that neither the data columns nor the partition * columns have these characters. */ def checkFieldNames(names: Seq[String]): Unit = { val invalidColumnNames = findInvalidColumnNames(names) if (invalidColumnNames.nonEmpty) { throw DeltaErrors.invalidColumnName(invalidColumnNames.head) } } /** * Finds columns with invalid names, i.e. names containing any of the ' ,;{}()\n\t=' characters. */ def findInvalidColumnNamesInSchema(schema: StructType): Seq[String] = { findInvalidColumnNames(SchemaMergingUtils.explodeNestedFieldNames(schema)) } private def findInvalidColumnNames(columnNames: Seq[String]): Seq[String] = { val badChars = Seq(' ', ',', ';', '{', '}', '(', ')', '\n', '\t', '=') columnNames.filter(colName => badChars.map(_.toString).exists(colName.contains)) } /** * Go through the schema to look for unenforceable NOT NULL constraints. By default we'll throw * when they're encountered, but if this is suppressed through SQLConf they'll just be silently * removed. * * Note that this should only be applied to schemas created from explicit user DDL - in other * scenarios, the nullability information may be inaccurate and Delta should always coerce the * nullability flag to true. */ def removeUnenforceableNotNullConstraints(schema: StructType, conf: SQLConf): StructType = { val allowUnenforceableNotNulls = conf.getConf(DeltaSQLConf.ALLOW_UNENFORCED_NOT_NULL_CONSTRAINTS) def checkField(path: Seq[String], f: StructField, r: Resolver): StructField = f match { case StructField(name, ArrayType(elementType, containsNull), nullable, metadata) => val nullableElementType = SchemaUtils.typeAsNullable(elementType) if (elementType != nullableElementType && !allowUnenforceableNotNulls) { throw DeltaErrors.nestedNotNullConstraint( prettyFieldName(path :+ f.name), elementType, nestType = "element") } StructField( name, ArrayType(nullableElementType, containsNull), nullable, metadata) case f @ StructField( name, MapType(keyType, valueType, containsNull), nullable, metadata) => val nullableKeyType = SchemaUtils.typeAsNullable(keyType) val nullableValueType = SchemaUtils.typeAsNullable(valueType) if (keyType != nullableKeyType && !allowUnenforceableNotNulls) { throw DeltaErrors.nestedNotNullConstraint( prettyFieldName(path :+ f.name), keyType, nestType = "key") } if (valueType != nullableValueType && !allowUnenforceableNotNulls) { throw DeltaErrors.nestedNotNullConstraint( prettyFieldName(path :+ f.name), valueType, nestType = "value") } StructField( name, MapType(nullableKeyType, nullableValueType, containsNull), nullable, metadata) case s: StructField => s } SchemaMergingUtils.transformColumns(schema)(checkField) } def fieldToColumn(field: StructField): Column = { Column(UnresolvedAttribute.quoted(field.name)) } /** converting field name to column type with quoted back-ticks */ def fieldNameToColumn(field: String): Column = { col(quoteIdentifier(field)) } // Escapes back-ticks within the identifier name with double-back-ticks, and then quote the // identifier with back-ticks. def quoteIdentifier(part: String): String = s"`${part.replace("`", "``")}`" private def analyzeExpression( spark: SparkSession, expr: Expression, schema: StructType): Expression = { // Workaround for `exp` analyze val relation = LocalRelation(schema) val relationWithExp = Project(Seq(Alias(expr, "validate_column")()), relation) val analyzedPlan = spark.sessionState.analyzer.execute(relationWithExp) analyzedPlan.collectFirst { case Project(Seq(a: Alias), _: LocalRelation) => a.child }.get } /** * Collects all attribute references in the given expression tree as a list of paths. * In particular, generates paths for nested fields accessed using extraction expressions. * For example: * - GetStructField(AttributeReference("struct"), "a") -> ["struct.a"] * - Size(AttributeReference("array")) -> ["array"] */ private def collectUsedColumns(expression: Expression): Seq[Seq[String]] = { val result = new collection.mutable.ArrayBuffer[Seq[String]]() // Firstly, try to get referenced column for a child's expression. // If it exists then we try to extend it by current expression. // In case if we cannot extend one, we save the received column path (it's as long as possible). def traverseAllPaths(exp: Expression): Option[Seq[String]] = exp match { case GetStructField(child, _, Some(name)) => traverseAllPaths(child).map(_ :+ name) case GetMapValue(child, key) => traverseAllPaths(key).foreach(result += _) traverseAllPaths(child).map { childPath => result += childPath :+ "key" childPath :+ "value" } case arrayExtract: GetArrayItem => traverseAllPaths(arrayExtract.child).map(_ :+ "element") case arrayExtract: GetArrayStructFields => traverseAllPaths(arrayExtract.child).map(_ :+ "element" :+ arrayExtract.field.name) case refCol: AttributeReference => Some(Seq(refCol.name)) case _ => exp.children.foreach(child => traverseAllPaths(child).foreach(result += _)) None } traverseAllPaths(expression).foreach(result += _) result.toSeq } private def fallbackContainsDependentExpression( expression: Expression, columnToChange: Seq[String], resolver: Resolver): Boolean = { expression.foreach { case refCol: UnresolvedAttribute => // columnToChange is the referenced column or its prefix val prefixMatched = columnToChange.size <= refCol.nameParts.size && refCol.nameParts.zip(columnToChange).forall(pair => resolver(pair._1, pair._2)) if (prefixMatched) return true case _ => } false } /** * Will a column change, e.g., rename, need to be populated to the expression. This is true when * the column to change itself or any of its descendent column is referenced by expression. * For example: * - a, length(a) -> true * - b, (b.c + 1) -> true, because renaming b1 will need to change the expr to (b1.c + 1). * - b.c, (cast b as string) -> true, because change b.c to b.c1 affects (b as string) result. */ def containsDependentExpression( spark: SparkSession, columnToChange: Seq[String], exprString: String, schema: StructType, resolver: Resolver): Boolean = { val expression = spark.sessionState.sqlParser.parseExpression(exprString) if (spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_CHANGE_COLUMN_CHECK_DEPENDENT_EXPRESSIONS_USE_V2)) { try { val analyzedExpr = analyzeExpression(spark, expression, schema) val exprColumns = collectUsedColumns(analyzedExpr) exprColumns.exists { exprColumn => // Changed column violates expression's column only when: // 1) the changed column is a prefix of the referenced column, // for example changing type of `col` affects `hash(col[0]) == 0`; // 2) or the referenced column is a prefix of the changed column, // for example changing type of `col.element` affects `concat_ws('', col) == 'abc'`; // 3) or they are equal. exprColumn.zip(columnToChange).forall { case (exprFieldName, changedFieldName) => resolver(exprFieldName, changedFieldName) } } } catch { case NonFatal(e) => deltaAssert( check = false, name = "containsDependentExpression.checkV2Error", msg = "Exception during dependent expression V2 checking: " + e.getMessage ) fallbackContainsDependentExpression(expression, columnToChange, resolver) } } else { fallbackContainsDependentExpression(expression, columnToChange, resolver) } } /** * Find the unsupported data type in a table schema. Return all columns that are using unsupported * data types. For example, * `findUnsupportedDataType(struct<a: struct<b: unsupported_type>>)` will return * `Some(unsupported_type, Some("a.b"))`. */ def findUnsupportedDataTypes(schema: StructType): Seq[UnsupportedDataTypeInfo] = { val unsupportedDataTypes = mutable.ArrayBuffer[UnsupportedDataTypeInfo]() findUnsupportedDataTypesRecursively(unsupportedDataTypes, schema) unsupportedDataTypes.toSeq } /** * Find TimestampNTZ columns in the table schema. */ def checkForTimestampNTZColumnsRecursively(schema: StructType): Boolean = { SchemaUtils.typeExistsRecursively(schema)(_.isInstanceOf[TimestampNTZType]) } /** * Returns 'true' if any VariantType exists in the table schema. */ def checkForVariantTypeColumnsRecursively(schema: StructType): Boolean = { SchemaUtils.typeExistsRecursively(schema)(_.isInstanceOf[VariantType]) } /** * Find the unsupported data types in a `DataType` recursively. Add the unsupported data types to * the provided `unsupportedDataTypes` buffer. * * @param unsupportedDataTypes the buffer to store the found unsupport data types and the column * paths. * @param dataType the data type to search. * @param columnPath the column path to access the given data type. The callder should make sure * `columnPath` is not empty when `dataType` is not `StructType`. */ private def findUnsupportedDataTypesRecursively( unsupportedDataTypes: mutable.ArrayBuffer[UnsupportedDataTypeInfo], dataType: DataType, columnPath: Seq[String] = Nil): Unit = dataType match { case NullType => case BooleanType => case ByteType => case ShortType => case IntegerType => case dt: YearMonthIntervalType => assert(columnPath.nonEmpty, "'columnPath' must not be empty") unsupportedDataTypes += UnsupportedDataTypeInfo(prettyFieldName(columnPath), dt) case LongType => case dt: DayTimeIntervalType => assert(columnPath.nonEmpty, "'columnPath' must not be empty") unsupportedDataTypes += UnsupportedDataTypeInfo(prettyFieldName(columnPath), dt) case FloatType => case DoubleType => case StringType => case DateType => case TimestampType => case TimestampNTZType => case dt if dt.isInstanceOf[VariantType] => case BinaryType => case _: DecimalType => case a: ArrayType => assert(columnPath.nonEmpty, "'columnPath' must not be empty") findUnsupportedDataTypesRecursively( unsupportedDataTypes, a.elementType, columnPath.dropRight(1) :+ columnPath.last + "[]") case m: MapType => assert(columnPath.nonEmpty, "'columnPath' must not be empty") findUnsupportedDataTypesRecursively( unsupportedDataTypes, m.keyType, columnPath.dropRight(1) :+ columnPath.last + "[key]") findUnsupportedDataTypesRecursively( unsupportedDataTypes, m.valueType, columnPath.dropRight(1) :+ columnPath.last + "[value]") case s: StructType => s.fields.foreach { f => findUnsupportedDataTypesRecursively( unsupportedDataTypes, f.dataType, columnPath :+ f.name) } case udt: UserDefinedType[_] => findUnsupportedDataTypesRecursively(unsupportedDataTypes, udt.sqlType, columnPath) case dt: DataType => assert(columnPath.nonEmpty, "'columnPath' must not be empty") unsupportedDataTypes += UnsupportedDataTypeInfo(prettyFieldName(columnPath), dt) } /** * Find all the generated columns that depend on the given target column. Returns a map of * generated names to their corresponding expression. */ def findDependentGeneratedColumns( sparkSession: SparkSession, targetColumn: Seq[String], protocol: Protocol, schema: StructType): Map[String, String] = { if (GeneratedColumn.satisfyGeneratedColumnProtocol(protocol) && GeneratedColumn.hasGeneratedColumns(schema)) { val dependentGenCols = mutable.Map[String, String]() SchemaMergingUtils.transformColumns(schema) { (_, field, _) => GeneratedColumn.getGenerationExpressionStr(field.metadata).foreach { exprStr => val needsToChangeExpr = SchemaUtils.containsDependentExpression( sparkSession, targetColumn, exprStr, schema, sparkSession.sessionState.conf.resolver) if (needsToChangeExpr) dependentGenCols += field.name -> exprStr } field } dependentGenCols.toMap } else { Map.empty } } /** Recursively find all types not defined in Delta protocol but used in `dt` */ def findUndefinedTypes(dt: DataType): Seq[DataType] = dt match { // Types defined in Delta protocol case NullType => Nil case BooleanType => Nil case ByteType | ShortType | IntegerType | LongType => Nil case FloatType | DoubleType | _: DecimalType => Nil case StringType | BinaryType => Nil case DateType | TimestampType => Nil // Recursively search complex data types case s: StructType => s.fields.flatMap(f => findUndefinedTypes(f.dataType)) case a: ArrayType => findUndefinedTypes(a.elementType) case m: MapType => findUndefinedTypes(m.keyType) ++ findUndefinedTypes(m.valueType) // Other types are not defined in Delta protocol case undefinedType => Seq(undefinedType) } /** Record all types not defined in Delta protocol but used in the `schema`. */ def recordUndefinedTypes(deltaLog: DeltaLog, schema: StructType): Unit = { try { findUndefinedTypes(schema).map(_.getClass.getName).toSet.foreach { className: String => recordDeltaEvent(deltaLog, "delta.undefined.type", data = Map("className" -> className)) } } catch { case NonFatal(e) => logWarning(log"Failed to log undefined types for table " + log"${MDC(DeltaLogKeys.PATH, deltaLog.logPath)}", e) } } // Helper method to validate that two logical column names are equal using the Delta column // resolver (case insensitive comparison). def areLogicalNamesEqual(col1: Seq[String], col2: Seq[String]): Boolean = { col1.length == col2.length && col1.zip(col2).forall(DELTA_COL_RESOLVER.tupled) } def removeExistsDefaultMetadata(schema: StructType): StructType = { // 'EXISTS_DEFAULT' is not used in Delta because it is not allowed to add a column with a // default value. Spark does though still add the metadata key when a column with a default // value is added at table creation. // We remove the metadata field here because it is not part of the Delta protocol and // having it in the schema prohibits CTAS from a table with a dropped default value. // @TODO: Clarify if active default values should be propagated to the target table in CTAS or // not and if not also remove 'CURRENT_DEFAULT' in CTAS. SchemaUtils.transformSchema(schema) { case (_, StructType(fields), _) if fields.exists(_.metadata.contains( ResolveDefaultColumnsUtils.EXISTS_DEFAULT_COLUMN_METADATA_KEY)) => val newFields = fields.map { field => val builder = new MetadataBuilder() .withMetadata(field.metadata) .remove(ResolveDefaultColumnsUtils.EXISTS_DEFAULT_COLUMN_METADATA_KEY) field.copy(metadata = builder.build()) } StructType(newFields) case (_, other, _) => other } } /** * Renames a column in the metadata, given the old column path, new column path, and an optional * list of column names. If the column names are provided, they will be updated to reflect the * new path. * * @param oldColumnPath The original physical name path of the column to be renamed. * @param newColumnPath The new physical name path for the column. * @param columnNameOpt An optional sequence of unresolved attributes representing the column * logical name. * @param deltaConfig The configuration key for columns that need to be renamed from metadata. * @return A map containing the updated Delta configuration with new column paths. */ def renameColumnForConfig( oldColumnPath: Seq[String], newColumnPath: Seq[String], columnNameOpt: Option[Seq[UnresolvedAttribute]], deltaConfig: String): Map[String, String] = { columnNameOpt.map { deltaColumnsNames => val deltaColumnsPath = deltaColumnsNames .map(_.nameParts) .map { attributeNameParts => val commonPrefix = oldColumnPath.zip(attributeNameParts) .takeWhile { case (left, right) => left == right } .size if (commonPrefix == oldColumnPath.size) { newColumnPath ++ attributeNameParts.takeRight(attributeNameParts.size - commonPrefix) } else { attributeNameParts } } .map(columnParts => if (SparkSession.active.conf.get(DeltaSQLConf.DELTA_RENAME_COLUMN_ESCAPE_NAME)) { UnresolvedAttribute(columnParts).sql } else { UnresolvedAttribute(columnParts).name } ) Map(deltaConfig -> deltaColumnsPath.mkString(",")) }.getOrElse(Map.empty[String, String]) } } /** * The information of unsupported data type returned by [[SchemaUtils.findUnsupportedDataTypes]]. * * @param column the column path to access the column using an unsupported data type, such as `a.b`. * @param dataType the unsupported data type. */ case class UnsupportedDataTypeInfo(column: String, dataType: DataType) ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTable.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.serverSidePlanning import java.util import java.util.Locale import scala.collection.JavaConverters._ import org.apache.spark.paths.SparkPath import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} import org.apache.spark.sql.connector.catalog.{SupportsRead, Table, TableCapability} import org.apache.spark.sql.connector.read._ import org.apache.spark.sql.execution.datasources.{FileFormat, PartitionedFile} import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.sources.{And, Filter} import org.apache.spark.sql.types.StructType import org.apache.spark.sql.util.{CaseInsensitiveStringMap, SchemaUtils} /** * Companion object for ServerSidePlannedTable with factory methods. */ object ServerSidePlannedTable extends DeltaLogging { /** * Determine if server-side planning should be used based on catalog type, * credential availability, and configuration. * * Decision logic: * - Requires enableServerSidePlanning flag to be enabled (prevents accidental enablement) * - In production: Also requires Unity Catalog table that lacks credentials * - In test mode: Only requires the enable flag (allows testing without UC setup) * - Otherwise use normal table loading path * * The logic is: ((isUnityCatalog && !hasCredentials) || skipUCRequirementForTests) && enableFlag * * @param isUnityCatalog Whether this is a Unity Catalog instance * @param hasCredentials Whether the table has credentials available * @param enableServerSidePlanning Whether to enable server-side planning (config flag) * @param skipUCRequirementForTests Whether to skip Unity Catalog requirement for testing * with non-UC tables * @return true if server-side planning should be used */ private[serverSidePlanning] def shouldUseServerSidePlanning( isUnityCatalog: Boolean, hasCredentials: Boolean, enableServerSidePlanning: Boolean, skipUCRequirementForTests: Boolean): Boolean = { ((isUnityCatalog && !hasCredentials) || skipUCRequirementForTests) && enableServerSidePlanning } /** * Try to create a ServerSidePlannedTable if server-side planning is needed. * Returns None if not needed or if the planning client factory is not available. * * This method encapsulates all the logic to decide whether to use server-side planning: * - Checks if Unity Catalog table lacks credentials * - Checks if server-side planning is enabled via config (required for all cases) * - In test mode, Unity Catalog check is bypassed to allow testing * - Extracts catalog name and table identifiers * - Attempts to create the planning client * * Test coverage: ServerSidePlanningSuite tests verify the decision logic through * shouldUseServerSidePlanning() method with different input combinations. * * @param spark The SparkSession * @param ident The table identifier * @param table The loaded table from the delegate catalog * @param isUnityCatalog Whether this is a Unity Catalog instance * @return Some(ServerSidePlannedTable) if server-side planning should be used, None otherwise */ def tryCreate( spark: SparkSession, ident: Identifier, table: Table, isUnityCatalog: Boolean): Option[ServerSidePlannedTable] = { // Check if we should enable server-side planning (for testing) val enableServerSidePlanning = spark.conf.get(DeltaSQLConf.ENABLE_SERVER_SIDE_PLANNING.key, "false").toBoolean val hasTableCredentials = hasCredentials(table) // Check if we should use server-side planning if (shouldUseServerSidePlanning( isUnityCatalog, hasTableCredentials, enableServerSidePlanning, skipUCRequirementForTests = DeltaUtils.isTesting)) { val namespace = ident.namespace().mkString(".") val tableName = ident.name() // Create metadata from table val metadata = ServerSidePlanningMetadata.fromTable(table, spark, ident, isUnityCatalog) // Try to create ServerSidePlannedTable with server-side planning val plannedTable = tryCreate(spark, namespace, tableName, table.schema(), metadata) if (plannedTable.isEmpty) { logWarning( s"Server-side planning not available for catalog ${metadata.catalogName}. " + "Falling back to normal table loading.") } plannedTable } else { None } } /** * Try to create a ServerSidePlannedTable with server-side planning. * Returns None if the planning client factory is not available. * * @param spark The SparkSession * @param databaseName The database name (may include catalog prefix) * @param tableName The table name * @param tableSchema The table schema * @param metadata Metadata extracted from loadTable response * @return Some(ServerSidePlannedTable) if successful, None if factory not registered */ private def tryCreate( spark: SparkSession, databaseName: String, tableName: String, tableSchema: StructType, metadata: ServerSidePlanningMetadata): Option[ServerSidePlannedTable] = { try { val client = ServerSidePlanningClientFactory.buildClient(spark, metadata) Some(new ServerSidePlannedTable(spark, databaseName, tableName, tableSchema, client)) } catch { case _: IllegalStateException => // Factory not registered - this shouldn't happen in production but could during testing None } } /** * Check if a table has credentials available. * UC injects credentials as table properties with "option.fs.*" prefix for filesystem configs. * See: CredPropsUtil in UCSingleCatalog. */ private def hasCredentials(table: Table): Boolean = { val properties = table.properties() val keys = properties.keySet() val iter = keys.iterator() while (iter.hasNext) { if (iter.next().startsWith("option.fs.")) { return true } } false } } /** * A Spark Table implementation that uses server-side scan planning * to get the list of files to read. Used as a fallback when Unity Catalog * doesn't provide credentials. * * Similar to DeltaTableV2, we accept SparkSession as a constructor parameter * since Tables are created on the driver and are not serialized to executors. */ class ServerSidePlannedTable( spark: SparkSession, databaseName: String, tableName: String, tableSchema: StructType, planningClient: ServerSidePlanningClient) extends Table with SupportsRead with AutoCloseable with DeltaLogging { // Returns fully qualified name (e.g., "catalog.database.table"). // The databaseName parameter receives ident.namespace().mkString(".") from DeltaCatalog, // which includes the catalog name when present, similar to DeltaTableV2's name() method. override def name(): String = s"$databaseName.$tableName" override def schema(): StructType = tableSchema override def capabilities(): util.Set[TableCapability] = { Set(TableCapability.BATCH_READ).asJava } override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = { new ServerSidePlannedScanBuilder(spark, databaseName, tableName, tableSchema, planningClient) } override def close(): Unit = { planningClient.close() } } /** * ScanBuilder that uses ServerSidePlanningClient to plan the scan. * Implements SupportsPushDownFilters to enable WHERE clause pushdown to the server. * Implements SupportsPushDownRequiredColumns to enable column pruning pushdown to the server. * Implements SupportsPushDownLimit to enable LIMIT pushdown to the server. */ class ServerSidePlannedScanBuilder( spark: SparkSession, databaseName: String, tableName: String, tableSchema: StructType, planningClient: ServerSidePlanningClient) extends ScanBuilder with SupportsPushDownFilters with SupportsPushDownRequiredColumns with SupportsPushDownLimit with DeltaLogging { // Filters that have been pushed down and will be sent to the server private var _pushedFilters: Array[Filter] = Array.empty // Required schema (columns to read). Defaults to full table schema. private var _requiredSchema: StructType = tableSchema // Limit that has been pushed down. None means no limit. private var _limit: Option[Int] = None /** * Push filters to the server-side planning client. * * Strategy: * - If ALL filters convert to server's native format: Returns empty array (no residuals) * This enables Spark to push down LIMIT in addition to filters * - If ANY filter fails conversion: Returns all filters as residuals * This falls back to safety mode where Spark re-applies all filters locally * * The server receives converted filters in both cases, but residuals provide a safety net * for correctness if the server silently ignores unsupported filters. */ override def pushFilters(filters: Array[Filter]): Array[Filter] = { // Store filters to send to IRC server _pushedFilters = filters // Strategy: Check if all filters can be converted upfront // Case 1: ALL convert -> return empty residuals -> enables filter+limit pushdown // Case 2: ANY fails -> return all residuals -> only filter pushdown (safety mode) if (filters.isEmpty) { // No filters to push return Array.empty } // Check if all filters are convertible val allConvertible = planningClient.canConvertFilters(filters) if (allConvertible) { // All filters successfully converted to server's native format // Trust that the server can handle them - return no residuals // This enables Spark to call pushLimit() for combined filter+limit pushdown logInfo(s"All ${filters.length} filters convertible, " + "returning empty residuals to enable limit pushdown") Array.empty } else { // At least one filter failed to convert // Return all filters as residuals for safety (Spark will re-apply) // Note: Server will still receive converted filters, but Spark provides safety net logWarning(s"Some filters failed to convert, " + "returning all as residuals (limit pushdown disabled)") filters } } override def pushedFilters(): Array[Filter] = _pushedFilters override def pruneColumns(requiredSchema: StructType): Unit = { _requiredSchema = requiredSchema } override def pushLimit(limit: Int): Boolean = { _limit = Some(limit) true // Return true to indicate the limit is fully pushed down to the server } override def isPartiallyPushed(): Boolean = { // Return true if we have a limit - indicates partial pushdown so Spark applies it too _limit.isDefined } override def build(): Scan = { new ServerSidePlannedScan( spark, databaseName, tableName, tableSchema, planningClient, _pushedFilters, _requiredSchema, _limit) } } /** * Scan implementation that calls the server-side planning API to get file list. */ class ServerSidePlannedScan( spark: SparkSession, databaseName: String, tableName: String, tableSchema: StructType, planningClient: ServerSidePlanningClient, pushedFilters: Array[Filter], requiredSchema: StructType, limit: Option[Int]) extends Scan with Batch { override def readSchema(): StructType = requiredSchema override def toBatch: Batch = this // Convert pushed filters to a single Spark Filter for the API call. // If no filters, pass None. If filters exist, combine them into a single filter. private val combinedFilter: Option[Filter] = { if (pushedFilters.isEmpty) { None } else if (pushedFilters.length == 1) { Some(pushedFilters.head) } else { // Combine multiple filters with And Some(pushedFilters.reduce((left, right) => And(left, right))) } } // Only pass projection if columns are actually pruned (not SELECT *) // Extract field names for planning client (server only needs names, not types) // Use Spark's SchemaUtils.explodeNestedFieldNames to flatten and escape field names, // then filter out parent structs by keeping only fields that have no children. // For example, for schema STRUCT>: // - explodeNestedFieldNames returns: ["a", "a.`b.c`"] // - We filter to leaf fields only: ["a.`b.c`"] // This ensures projections only include actual data columns, not parent containers. private val projectionColumnNames: Option[Seq[String]] = { if (requiredSchema.fieldNames.toSet == tableSchema.fieldNames.toSet) { None } else { val allFields = SchemaUtils.explodeNestedFieldNames(requiredSchema) Some(allFields.filter { field => !allFields.exists(other => other.startsWith(field + ".")) }) } } // Call the server-side planning API to get the scan plan with files AND credentials. // Close the client after planning - the scan plan contains all data needed for partition // creation and reading, so the client (and its HTTP connection) is no longer needed. private lazy val scanPlan: ScanPlan = { val plan = planningClient.planScan( databaseName, tableName, combinedFilter, projectionColumnNames, limit) planningClient.close() plan } // Explicitly signal that columnar is unsupported to prevent early enumeration of the partitions override def columnarSupportMode(): Scan.ColumnarSupportMode = Scan.ColumnarSupportMode.UNSUPPORTED override def planInputPartitions(): Array[InputPartition] = { // Convert each file to an InputPartition scanPlan.files.map { file => ServerSidePlannedFileInputPartition(file.filePath, file.fileSizeInBytes, file.fileFormat) }.toArray } override def createReaderFactory(): PartitionReaderFactory = { new ServerSidePlannedFilePartitionReaderFactory( spark, tableSchema, requiredSchema, scanPlan.credentials) } } /** * InputPartition representing a single file from the server-side scan plan. */ case class ServerSidePlannedFileInputPartition( filePath: String, fileSizeInBytes: Long, fileFormat: String) extends InputPartition /** * Factory for creating PartitionReaders that read server-side planned files. * Builds reader functions on the driver for Parquet files. * * @param tableSchema The full table schema (all columns in the file) * @param requiredSchema The required schema (columns to read after projection pushdown) * @param credentials Optional storage credentials from server-side planning response */ class ServerSidePlannedFilePartitionReaderFactory( spark: SparkSession, tableSchema: StructType, requiredSchema: StructType, credentials: Option[ScanPlanStorageCredentials]) extends PartitionReaderFactory { import org.apache.spark.util.SerializableConfiguration // scalastyle:off deltahadoopconfiguration // We use sessionState.newHadoopConf() here instead of deltaLog.newDeltaHadoopConf(). // This means DataFrame options (like custom S3 credentials) passed by users will NOT be // included in the Hadoop configuration. This is intentional: // - Server-side planning uses server-provided credentials, not user-specified credentials // - ServerSidePlannedTable is NOT a Delta table, so we don't want Delta-specific options // from deltaLog.newDeltaHadoopConf() // - General Spark options from spark.hadoop.* are included and work for all tables private val hadoopConf = { val conf = spark.sessionState.newHadoopConf() // Inject temporary credentials from IRC server response. // Disable FileSystem cache for S3, Azure, and GCS so each scan uses fresh credentials // (avoids AccessDenied when temp creds expire and a cached FS is reused). // Aligns with CredPropsUtil in the Unity Catalog connector. credentials.foreach(_.configure(conf)) new SerializableConfiguration(conf) } // scalastyle:on deltahadoopconfiguration // Pre-build reader function for Parquet on the driver // This function will be serialized and sent to executors // tableSchema: All columns in the file (full table schema) // requiredSchema: Columns to actually read (after projection pushdown) private val parquetReaderBuilder = new ParquetFileFormat().buildReaderWithPartitionValues( sparkSession = spark, dataSchema = tableSchema, partitionSchema = StructType(Nil), requiredSchema = requiredSchema, filters = Seq.empty, options = Map( FileFormat.OPTION_RETURNING_BATCH -> "false" ), hadoopConf = hadoopConf.value ) override def createReader(partition: InputPartition): PartitionReader[InternalRow] = { val filePartition = partition.asInstanceOf[ServerSidePlannedFileInputPartition] // Verify file format is Parquet // Scalastyle suppression needed: the caselocale regex incorrectly flags even correct usage // of toLowerCase(Locale.ROOT). Similar to PartitionUtils.scala and SchemaUtils.scala. // scalastyle:off caselocale if (filePartition.fileFormat.toLowerCase(Locale.ROOT) != "parquet") { // scalastyle:on caselocale throw new UnsupportedOperationException( s"File format '${filePartition.fileFormat}' is not supported. Only Parquet is supported.") } new ServerSidePlannedFilePartitionReader(filePartition, parquetReaderBuilder) } } /** * PartitionReader that reads a single file using a pre-built reader function. * The reader function was created on the driver and is executed on the executor. */ class ServerSidePlannedFilePartitionReader( partition: ServerSidePlannedFileInputPartition, readerBuilder: PartitionedFile => Iterator[InternalRow]) extends PartitionReader[InternalRow] { // Create PartitionedFile for this file private val partitionedFile = PartitionedFile( partitionValues = InternalRow.empty, filePath = SparkPath.fromPathString(partition.filePath), start = 0, length = partition.fileSizeInBytes ) // Track the iterator so we can close it properly // Using Option to avoid initializing the iterator if close() is called before next() private var readerIterator: Option[Iterator[InternalRow]] = None // Get or create the reader iterator private def getIterator: Iterator[InternalRow] = { readerIterator.getOrElse { val iter = readerBuilder(partitionedFile) readerIterator = Some(iter) iter } } override def next(): Boolean = { getIterator.hasNext } override def get(): InternalRow = { getIterator.next() } override def close(): Unit = { // Close the iterator if it implements AutoCloseable (which Parquet iterators do) readerIterator.foreach { case closeable: AutoCloseable => closeable.close() case _ => // No cleanup needed } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningClient.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.serverSidePlanning import org.apache.spark.sql.SparkSession import org.apache.spark.sql.sources.Filter /** * Simple data class representing a file to scan. * No dependencies on Iceberg types. */ case class ScanFile( filePath: String, fileSizeInBytes: Long, fileFormat: String // "parquet", "orc", etc. ) /** * Interface for planning table scans via server-side planning. * This interface uses Spark's standard `org.apache.spark.sql.sources.Filter` as the universal * representation for filter pushdown. This keeps the interface catalog-agnostic while allowing * each server-side planning catalog implementation to convert filters to their own native format. */ trait ServerSidePlanningClient extends AutoCloseable { /** * Plan a table scan and return the list of files to read. * * @param databaseName The database or schema name * @param table The table name * @param filterOption Optional filter expression to push down to server (Spark Filter format) * @param projectionOption Optional projection (column names) to push down to server * @param limitOption Optional limit to push down to server * @return ScanPlan containing files to read */ def planScan( databaseName: String, table: String, filterOption: Option[Filter] = None, projectionOption: Option[Seq[String]] = None, limitOption: Option[Int] = None): ScanPlan /** * Check if all given filters can be converted to the server's native filter format. * This is used during filter pushdown to determine whether to return residuals to Spark. * * @param filters Array of Spark filters to check * @return true if ALL filters can be converted, false if ANY filter cannot be converted */ def canConvertFilters(filters: Array[Filter]): Boolean /** * Close any resources held by this client. * Default implementation is a no-op for clients that don't hold resources. */ override def close(): Unit = {} } /** * Factory for creating ServerSidePlanningClient instances. * This allows for configurable implementations (REST, mock, Spark-based, etc.) */ private[serverSidePlanning] trait ServerSidePlanningClientFactory { /** * Create a client using metadata necessary for server-side planning. * * @param spark The SparkSession * @param metadata Metadata necessary for server-side planning * @return A ServerSidePlanningClient configured with the metadata */ def buildClient( spark: SparkSession, metadata: ServerSidePlanningMetadata): ServerSidePlanningClient } /** * Registry for client factories. Automatically discovers and registers implementations * using reflection-based auto-discovery on first access to the factory. Manual registration * using setFactory() is only needed for testing or to override the auto-discovered factory. */ private[serverSidePlanning] object ServerSidePlanningClientFactory { // Fully qualified class name for auto-registration via reflection private val ICEBERG_FACTORY_CLASS_NAME = "org.apache.spark.sql.delta.serverSidePlanning.IcebergRESTCatalogPlanningClientFactory" @volatile private var registeredFactory: Option[ServerSidePlanningClientFactory] = None @volatile private var autoRegistrationAttempted: Boolean = false // Lazy initialization - only runs when getFactory() is called and no factory is set. // Uses reflection to load the hardcoded IcebergRESTCatalogPlanningClientFactory class. private def tryAutoRegisterFactory(): Unit = { // Double-checked locking pattern to ensure initialization happens only once if (!autoRegistrationAttempted) { synchronized { if (!autoRegistrationAttempted) { autoRegistrationAttempted = true try { // Use reflection to load the Iceberg factory class // scalastyle:off classforname val clazz = Class.forName(ICEBERG_FACTORY_CLASS_NAME) // scalastyle:on classforname val factory = clazz.getConstructor().newInstance() .asInstanceOf[ServerSidePlanningClientFactory] registeredFactory = Some(factory) } catch { case e: Exception => throw new IllegalStateException( "No ServerSidePlanningClientFactory has been registered. " + "Ensure delta-iceberg JAR is on the classpath for auto-registration, " + "or call ServerSidePlanningClientFactory.setFactory() to register manually.", e) } } } } } /** * Set a factory, overriding any auto-registered factory. * Synchronized to prevent race conditions with auto-registration. */ private[serverSidePlanning] def setFactory(factory: ServerSidePlanningClientFactory): Unit = { synchronized { registeredFactory = Some(factory) } } /** * Clear the registered factory. * Synchronized to ensure atomic reset of both flags. */ private[serverSidePlanning] def clearFactory(): Unit = { synchronized { registeredFactory = None autoRegistrationAttempted = false } } /** * Get the currently registered factory. * Throws IllegalStateException if no factory has been registered (either via reflection-based * auto-discovery or explicit setFactory() call). */ def getFactory(): ServerSidePlanningClientFactory = { // Try auto-registration if not already attempted and no factory is manually set if (registeredFactory.isEmpty) { tryAutoRegisterFactory() } registeredFactory.getOrElse { throw new IllegalStateException( "No ServerSidePlanningClientFactory has been registered. " + "Ensure delta-iceberg JAR is on the classpath for auto-registration, " + "or call ServerSidePlanningClientFactory.setFactory() to register manually.") } } /** * Convenience method to create a client from metadata using the registered factory. */ def buildClient( spark: SparkSession, metadata: ServerSidePlanningMetadata): ServerSidePlanningClient = { getFactory().buildClient(spark, metadata) } } /** * Functional interface for applying storage credentials to a Hadoop configuration. * Implementations are responsible for setting the appropriate Hadoop config keys * for their respective cloud provider. */ trait ScanPlanStorageCredentials { def configure(conf: org.apache.hadoop.conf.Configuration): Unit } /** * Result of a table scan plan operation. */ case class ScanPlan( files: Seq[ScanFile], credentials: Option[ScanPlanStorageCredentials] = None) ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningMetadata.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.serverSidePlanning import org.apache.spark.sql.SparkSession import org.apache.spark.sql.connector.catalog.{Identifier, Table} /** * Metadata required for creating a server-side planning client. * * This interface captures all information from the catalog's loadTable response * that is needed to create and configure a ServerSidePlanningClient. */ private[serverSidePlanning] trait ServerSidePlanningMetadata { /** * The base URI for the planning endpoint. */ def planningEndpointUri: String /** * Authentication token for the planning endpoint. */ def authToken: Option[String] /** * Catalog name for configuration lookups. */ def catalogName: String /** * Additional table properties that may be needed. * For example, table UUID, credential hints, etc. */ def tableProperties: Map[String, String] } /** * Default metadata for non-UC catalogs. * Used when server-side planning is force-enabled for testing/development. */ private[serverSidePlanning] case class DefaultMetadata( catalogName: String, tableProps: Map[String, String] = Map.empty) extends ServerSidePlanningMetadata { override def planningEndpointUri: String = "" override def authToken: Option[String] = None override def tableProperties: Map[String, String] = tableProps } object ServerSidePlanningMetadata { /** * Create metadata from a loaded table. * * Returns UnityCatalogMetadata for Unity Catalog tables, or DefaultMetadata otherwise. */ def fromTable( table: Table, spark: SparkSession, ident: Identifier, isUnityCatalog: Boolean): ServerSidePlanningMetadata = { if (isUnityCatalog) { UnityCatalogMetadata.fromTable(table, spark, ident) } else { val catalogName = extractCatalogName(ident) DefaultMetadata(catalogName, Map.empty) } } private def extractCatalogName(ident: Identifier): String = { if (ident.namespace().length > 1) { ident.namespace().head } else { "spark_catalog" } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/UnityCatalogMetadata.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.serverSidePlanning import org.apache.spark.sql.SparkSession import org.apache.spark.sql.connector.catalog.{Identifier, Table} /** * Metadata for Unity Catalog tables. * Provides base Iceberg REST endpoint for server-side planning. */ case class UnityCatalogMetadata( catalogName: String, ucUri: String, ucToken: String, tableProps: Map[String, String]) extends ServerSidePlanningMetadata { override def planningEndpointUri: String = { // Return base Iceberg REST path up to /v1/ // The IcebergRESTCatalogPlanningClient will call /v1/config to get the prefix // and construct the full URL according to the Iceberg REST catalog spec val base = if (ucUri.endsWith("/")) ucUri.dropRight(1) else ucUri s"$base/api/2.1/unity-catalog/iceberg-rest/v1" } override def authToken: Option[String] = Some(ucToken) override def tableProperties: Map[String, String] = tableProps } object UnityCatalogMetadata { def fromTable( table: Table, spark: SparkSession, ident: Identifier): UnityCatalogMetadata = { val catalogName = if (ident.namespace().length > 1) { ident.namespace().head } else { // Use current catalog from session // This allows queries with 2-part names (schema.table) to work with Unity Catalog spark.sessionState.catalogManager.currentCatalog.name() } // Read UC configuration from Spark conf val ucUri = spark.conf.get(s"spark.sql.catalog.$catalogName.uri", "") val ucToken = spark.conf.get(s"spark.sql.catalog.$catalogName.token", "") // Table properties currently unused, may be needed in future val tableProps = Map.empty[String, String] UnityCatalogMetadata(catalogName, ucUri, ucToken, tableProps) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/skipping/MultiDimClustering.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.skipping import java.util.UUID import org.apache.spark.sql.delta.skipping.MultiDimClusteringFunctions._ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.SparkException import org.apache.spark.internal.Logging import org.apache.spark.sql._ import org.apache.spark.sql.functions._ import org.apache.spark.sql.types._ /** Trait for changing the data layout using a multi-dimensional clustering algorithm */ trait MultiDimClustering extends Logging { /** Repartition the given `df` into `approxNumPartitions` based on the provided `colNames`. */ def cluster( df: DataFrame, colNames: Seq[String], approxNumPartitions: Int, randomizationExpressionOpt: Option[Column] ): DataFrame } object MultiDimClustering { /** * Repartition the given dataframe `df` based on the given `curve` type into * `approxNumPartitions` on the given `colNames`. */ def cluster( df: DataFrame, approxNumPartitions: Int, colNames: Seq[String], curve: String): DataFrame = { assert(colNames.nonEmpty, "Cannot cluster by zero columns!") val clusteringImpl = curve match { case "hilbert" if colNames.size == 1 => ZOrderClustering case "hilbert" => HilbertClustering case "zorder" => ZOrderClustering case unknownCurve => throw new SparkException(s"Unknown curve ($unknownCurve), unable to perform multi " + "dimensional clustering.") } clusteringImpl.cluster(df, colNames, approxNumPartitions, randomizationExpressionOpt = None) } } /** Base class for space filling curve based clustering e.g. ZOrder */ trait SpaceFillingCurveClustering extends MultiDimClustering { protected def getClusteringExpression(cols: Seq[Column], numRanges: Int): Column override def cluster( df: DataFrame, colNames: Seq[String], approxNumPartitions: Int, randomizationExpressionOpt: Option[Column]): DataFrame = { val conf = df.sparkSession.sessionState.conf val numRanges = conf.getConf(DeltaSQLConf.MDC_NUM_RANGE_IDS) val addNoise = conf.getConf(DeltaSQLConf.MDC_ADD_NOISE) val sortWithinFiles = conf.getConf(DeltaSQLConf.MDC_SORT_WITHIN_FILES) val cols = colNames.map(df(_)) val mdcCol = getClusteringExpression(cols, numRanges) val repartitionKeyColName = s"${UUID.randomUUID().toString}-rpKey1" var repartitionedDf = if (addNoise) { val randByteColName = s"${UUID.randomUUID().toString}-rpKey2" val randByteCol = randomizationExpressionOpt.getOrElse((rand() * 255 - 128).cast(ByteType)) df.withColumn(repartitionKeyColName, mdcCol).withColumn(randByteColName, randByteCol) .repartitionByRange(approxNumPartitions, col(repartitionKeyColName), col(randByteColName)) .drop(randByteColName) } else { df.withColumn(repartitionKeyColName, mdcCol) .repartitionByRange(approxNumPartitions, col(repartitionKeyColName)) } if (sortWithinFiles) { repartitionedDf = repartitionedDf.sortWithinPartitions(repartitionKeyColName) } repartitionedDf.drop(repartitionKeyColName) } } /** Implement Z-Order clustering */ object ZOrderClustering extends SpaceFillingCurveClustering { override protected[skipping] def getClusteringExpression( cols: Seq[Column], numRanges: Int): Column = { assert(cols.size >= 1, "Cannot do Z-Order clustering by zero columns!") val rangeIdCols = cols.map(range_partition_id(_, numRanges)) interleave_bits(rangeIdCols: _*).cast(StringType) } } object HilbertClustering extends SpaceFillingCurveClustering with Logging { override protected def getClusteringExpression(cols: Seq[Column], numRanges: Int): Column = { assert(cols.size > 1, "Cannot do Hilbert clustering by zero or one column!") val rangeIdCols = cols.map(range_partition_id(_, numRanges)) val numBits = Integer.numberOfTrailingZeros(Integer.highestOneBit(numRanges)) + 1 hilbert_index(numBits, rangeIdCols: _*) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/skipping/MultiDimClusteringFunctions.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.skipping // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.expressions.{HilbertByteArrayIndex, HilbertLongIndex, InterleaveBits, RangePartitionId} import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.SparkException import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.{Cast, Expression} import org.apache.spark.sql.types.StringType /** Functions for multi-dimensional clustering of the data */ object MultiDimClusteringFunctions { private def withExpr(expr: Expression): Column = Column(expr) /** * Conceptually range-partitions the domain of values of the given column into `numPartitions` * partitions and computes the partition number that every value of that column corresponds to. * One can think of this as an approximate rank() function. * * Ex. For a column with values (0, 1, 3, 15, 36, 99) and numPartitions = 3 returns * partition range ids as (0, 0, 1, 1, 2, 2). */ def range_partition_id(col: Column, numPartitions: Int): Column = withExpr { RangePartitionId(expression(col), numPartitions) } /** * Interleaves the bits of its input data in a round-robin fashion. * * If the input data is seen as a series of multidimensional points, this function computes the * corresponding Z-values, in a way that's preserving data locality: input points that are close * in the multidimensional space will be mapped to points that are close on the Z-order curve. * * The returned value is a byte array where the size of the array is 4 * num of input columns. * * @see https://en.wikipedia.org/wiki/Z-order_curve * * @note Only supports input expressions of type Int for now. */ def interleave_bits(cols: Column*): Column = withExpr { InterleaveBits(cols.map(expression)) } // scalastyle:off line.size.limit /** * Transforms the provided integer columns into their corresponding position in the hilbert * curve for the given dimension. * @see https://citeseerx.ist.psu.edu/document?repid=rep1&type=pdf&doi=bfd6d94c98627756989b0147a68b7ab1f881a0d6 * @see https://en.wikipedia.org/wiki/Hilbert_curve * @param numBits The number of bits to consider in each column. * @param cols The integer columns to map to the curve. */ // scalastyle:on line.size.limit def hilbert_index(numBits: Int, cols: Column*): Column = withExpr { if (cols.size > 9) { throw new SparkException("Hilbert indexing can only be used on 9 or fewer columns.") } val hilbertBits = cols.length * numBits if (hilbertBits < 64) { HilbertLongIndex(numBits, cols.map(expression)) } else { Cast(HilbertByteArrayIndex(numBits, cols.map(expression)), StringType) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/skipping/clustering/ClusteredTableUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.skipping.clustering import org.apache.spark.sql.delta.skipping.clustering.temp.ClusterBySpec import org.apache.spark.sql.delta.{ClusteringTableFeature, DeltaColumnMappingMode, DeltaErrors, DeltaLog, OptimisticTransaction, Snapshot} import org.apache.spark.sql.delta.actions.{Action, DomainMetadata, Metadata, Protocol, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.clustering.ClusteringMetadataDomain import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.{DeltaStatistics, SkippingEligibleDataType, StatisticsCollection, StatsCollectionUtils} import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{StructField, StructType} case class MatchingMetadataDomain( clusteringDomainOpt: Option[DomainMetadata] ) /** * Clustered table utility functions. */ trait ClusteredTableUtilsBase extends DeltaLogging { // Clustering columns property key. The column names are logical and separated by comma. // This will be removed when we integrate with OSS Spark and use // [[CatalogTable.PROP_CLUSTERING_COLUMNS]] directly. val PROP_CLUSTERING_COLUMNS: String = "clusteringColumns" /** * Returns whether the protocol version supports the Liquid table feature. */ def isSupported(protocol: Protocol): Boolean = protocol.isFeatureSupported(ClusteringTableFeature) /** The clustering implementation name for [[AddFile.clusteringProvider]] */ def clusteringProvider: String = "liquid" /** * Returns an optional [[ClusterBySpec]] from the given CatalogTable. */ def getClusterBySpecOptional(table: CatalogTable): Option[ClusterBySpec] = { table.properties.get(PROP_CLUSTERING_COLUMNS).map(ClusterBySpec.fromProperty) } /** * Returns an optional [[ClusterBySpec]] from the given Snapshot. */ def getClusterBySpecOptional(snapshot: Snapshot): Option[ClusterBySpec] = { if (isSupported(snapshot.protocol)) { val clusteringColumns = ClusteringColumnInfo.extractLogicalNames(snapshot) Some(ClusterBySpec.fromColumnNames(clusteringColumns)) } else { None } } /** * Extract clustering columns from ClusterBySpec. * * @param maybeClusterBySpec optional ClusterBySpec. If it's empty, will return the * original properties. * @return an optional pair with clustering columns. */ def getClusteringColumnsAsProperty( maybeClusterBySpec: Option[ClusterBySpec]): Option[(String, String)] = { maybeClusterBySpec.map(ClusterBySpec.toProperty) } /** * Extract clustering columns from a given snapshot. */ def getClusteringColumnsAsProperty(snapshot: Snapshot): Option[(String, String)] = { val clusterBySpec = getClusterBySpecOptional(snapshot) getClusteringColumnsAsProperty(clusterBySpec) } /** * Returns table feature properties that's required to create a clustered table. * * @param existingProperties Table properties set by the user when creating a clustered table. */ def getTableFeatureProperties(existingProperties: Map[String, String]): Map[String, String] = { val properties = collection.mutable.Map.empty[String, String] properties += TableFeatureProtocolUtils.propertyKey(ClusteringTableFeature) -> TableFeatureProtocolUtils.FEATURE_PROP_SUPPORTED properties.toMap } /** * Verify user didn't set clustering table feature in table properties. * * @param existingProperties Table properties set by the user when creating a clustered table. */ def validateExistingTableFeatureProperties(existingProperties: Map[String, String]): Unit = { if (existingProperties.contains( TableFeatureProtocolUtils.propertyKey(ClusteringTableFeature))) { throw DeltaErrors.createTableSetClusteringTableFeatureException(ClusteringTableFeature.name) } } /** * Validate the number of clustering columns doesn't exceed the limit. * * @param clusteringColumns clustering columns for the table. * @param deltaLogOpt optional delta log. If present, will be used to record a delta event. */ def validateNumClusteringColumns( clusteringColumns: Seq[Seq[String]], deltaLogOpt: Option[DeltaLog] = None): Unit = { val numColumnsLimit = SQLConf.get.getConf(DeltaSQLConf.DELTA_NUM_CLUSTERING_COLUMNS_LIMIT) val actualNumColumns = clusteringColumns.size if (actualNumColumns > numColumnsLimit) { deltaLogOpt.foreach { deltaLog => recordDeltaEvent( deltaLog, opType = "delta.clusteredTable.invalidNumClusteringColumns", data = Map( "numCols" -> clusteringColumns.size, "numColsLimit" -> numColumnsLimit)) } throw DeltaErrors.clusterByInvalidNumColumnsException(numColumnsLimit, actualNumColumns) } } /** * Remove clustered table internal table properties. These properties are never stored into * [[Metadata.configuration]] such as table features. */ def removeInternalTableProperties( props: scala.collection.Map[String, String]): Map[String, String] = { props.toMap -- // Clustering table feature and dependent table features Seq(ClusteringTableFeature).flatMap { feature => (feature +: feature.requiredFeatures.toSeq).map(TableFeatureProtocolUtils.propertyKey) } } /** * Remove PROP_CLUSTERING_COLUMNS from metadata action. * Clustering columns should only exist in: * 1. CatalogTable.properties(PROP_CLUSTERING_COLUMNS) * 2. Clustering metadata domain. * @param configuration original configuration. * @return new configuration without clustering columns property */ def removeClusteringColumnsProperty(configuration: Map[String, String]): Map[String, String] = { configuration - PROP_CLUSTERING_COLUMNS } /** * Returns [[DomainMetadata]] action to store clustering columns. * If clusterBySpecOpt is not empty (clustering columns are specified by CLUSTER BY), it creates * the domain metadata based on the clustering columns. * Otherwise (CLUSTER BY is not specified for REPLACE TABLE), it creates the domain metadata * with empty clustering columns if a clustering domain exists. * * This is used for CREATE TABLE and REPLACE TABLE. */ def getDomainMetadataFromTransaction( clusterBySpecOpt: Option[ClusterBySpec], txn: OptimisticTransaction): Seq[DomainMetadata] = { clusterBySpecOpt.map { clusterBy => ClusteredTableUtils.validateClusteringColumnsInStatsSchema( txn.protocol, txn.metadata, clusterBy) val clusteringColumns = clusterBy.columnNames.map(_.toString).map(ClusteringColumn(txn.metadata.schema, _)) Seq(createDomainMetadata(clusteringColumns)) }.getOrElse { getMatchingMetadataDomain( clusteringColumns = Seq.empty, txn.snapshot.domainMetadata).clusteringDomainOpt.toSeq } } /** * Returns a sequence of [[DomainMetadata]] actions to update the existing domain metadata with * the given clustering columns. * * This is mainly used for REPLACE TABLE and RESTORE TABLE. */ def getMatchingMetadataDomain( clusteringColumns: Seq[ClusteringColumn], existingDomainMetadata: Seq[DomainMetadata]): MatchingMetadataDomain = { val clusteringMetadataDomainOpt = if (existingDomainMetadata.exists(_.domain == ClusteringMetadataDomain.domainName)) { Some(ClusteringMetadataDomain.fromClusteringColumns(clusteringColumns).toDomainMetadata) } else { None } MatchingMetadataDomain( clusteringMetadataDomainOpt ) } /** * Create a [[DomainMetadata]] action to store clustering columns. */ def createDomainMetadata(clusteringColumns: Seq[ClusteringColumn]): DomainMetadata = { ClusteringMetadataDomain.fromClusteringColumns(clusteringColumns).toDomainMetadata } /** * Extract [[ClusteringColumn]]s from a given snapshot. Return None if the clustering domain * metadata is missing. */ def getClusteringColumnsOptional(snapshot: Snapshot): Option[Seq[ClusteringColumn]] = { ClusteringMetadataDomain .fromSnapshot(snapshot) .map(_.clusteringColumns.map(ClusteringColumn.apply)) } /** * Extract [[DomainMetadata]] for storing clustering columns from a given snapshot. * It returns clustering domain metadata if exists. * Return empty if the clustering domain metadata is missing. */ def getClusteringDomainMetadata(snapshot: Snapshot): Seq[DomainMetadata] = { ClusteringMetadataDomain.fromSnapshot(snapshot).map(_.toDomainMetadata).toSeq } /** * Create new clustering [[DomainMetadata]] actions given updated column names for * 'ALTER TABLE ... CLUSTER BY'. */ def getClusteringDomainMetadataForAlterTableClusterBy( newLogicalClusteringColumns: Seq[String], txn: OptimisticTransaction): Seq[DomainMetadata] = { val newClusteringColumns = newLogicalClusteringColumns.map(ClusteringColumn(txn.metadata.schema, _)) val clusteringMetadataDomainOpt = Some(ClusteringMetadataDomain.fromClusteringColumns(newClusteringColumns).toDomainMetadata) clusteringMetadataDomainOpt.toSeq } /** * Extract the logical clustering column names from the to-be committed domain metadata action. * * @param txn the transaction being used to commit the actions. * @param actionsToCommit the actions to be committed. * @return optional logical clustering column names. */ def getLogicalClusteringColumnNames( txn: OptimisticTransaction, actionsToCommit: Seq[Action]): Option[Seq[String]] = { def getLogicalColumnNames(clusteringColumns: Seq[ClusteringColumn]): Seq[String] = { clusteringColumns.map(ClusteringColumnInfo(txn.metadata.schema, _).logicalName) } actionsToCommit.collectFirst { // Only consider clustering domain metadata actions that are getting added // (removed = false). case ClusteringMetadataDomain(domain, removed) if !removed => getLogicalColumnNames(domain.clusteringColumns.map(ClusteringColumn.apply)) } } /** * Validate stats will be collected for all clustering columns. */ def validateClusteringColumnsInStatsSchema( snapshot: Snapshot, logicalClusteringColumns: Seq[String]): Unit = { validateClusteringColumnsInStatsSchema( snapshot, logicalClusteringColumns.map { name => ClusteringColumnInfo(snapshot.schema, ClusteringColumn(snapshot.schema, name)) }) } /** * Returns true if stats will be collected for all clustering columns. */ def areClusteringColumnsInStatsSchema( snapshot: Snapshot, logicalClusteringColumns: Seq[String]): Boolean = { getClusteringColumnsNotInStatsSchema( snapshot, logicalClusteringColumns.map { name => ClusteringColumnInfo(snapshot.schema, ClusteringColumn(snapshot.schema, name)) }).isEmpty } /** * Validate stats will be collected for all clustering columns. * * This version is used when [[Snapshot]] doesn't have latest stats column information such as * `CREATE TABLE...` where the initial snapshot doesn't have updated metadata / protocol yet. */ def validateClusteringColumnsInStatsSchema( protocol: Protocol, metadata: Metadata, clusterBy: ClusterBySpec): Unit = { validateClusteringColumnsInStatsSchema( statisticsCollectionFromMetadata(protocol, metadata), clusterBy.columnNames.map { column => ClusteringColumnInfo(metadata.schema, ClusteringColumn(metadata.schema, column.toString)) }) } /** * Build a [[StatisticsCollection]] with minimal requirements that can be used to find stats * columns. * * We can not use [[Snapshot]] as in a normal case during table creation such as `CREATE TABLE` * because the initial snapshot doesn't have the updated metadata / protocol to find latest stats * columns. */ private def statisticsCollectionFromMetadata( p: Protocol, metadata: Metadata): StatisticsCollection = { new StatisticsCollection { override val tableSchema: StructType = metadata.schema override val outputAttributeSchema: StructType = tableSchema // [[outputTableStatsSchema]] is the candidate schema to find statistics columns. override val outputTableStatsSchema: StructType = tableSchema override val statsColumnSpec = StatisticsCollection.configuredDeltaStatsColumnSpec(metadata) override val columnMappingMode: DeltaColumnMappingMode = metadata.columnMappingMode override val protocol: Protocol = p override def getDataSkippingStringPrefixLength: Int = StatsCollectionUtils.getDataSkippingStringPrefixLength(spark, metadata) override def spark: SparkSession = { throw new Exception("Method not used in statisticsCollectionFromMetadata") } } } /** * Validate physical clustering columns can be found in the latest stats columns. * * @param statsCollection Provides latest stats columns. * @param clusteringColumnInfos Clustering columns in physical names. * * A [[AnalysisException]] is thrown if the clustering column can not be found in the latest * stats columns. The error message contains logical names only for better user experience. */ private def validateClusteringColumnsInStatsSchema( statsCollection: StatisticsCollection, clusteringColumnInfos: Seq[ClusteringColumnInfo]): Unit = { val missingColumns = getClusteringColumnsNotInStatsSchema(statsCollection, clusteringColumnInfos) if (missingColumns.nonEmpty) { // Check DataType eligibility. val missingColumnInfos = clusteringColumnInfos.filter( info => missingColumns.contains(info.logicalName)) // This assertion must hold since missingColumns are subset of clusteringColumnInfos. assert(missingColumnInfos.length == missingColumns.length) val nonSkippingEligibleMissingColumnInfos = missingColumnInfos.filter(info => !SkippingEligibleDataType(info.dataType)) if (nonSkippingEligibleMissingColumnInfos.nonEmpty) { val columnNameWithDataTypes = nonSkippingEligibleMissingColumnInfos .map(info => s"${info.logicalName} : ${info.dataType.sql}") .mkString(", ") throw DeltaErrors.clusteringColumnUnsupportedDataTypes(columnNameWithDataTypes) } throw DeltaErrors.clusteringColumnMissingStats( missingColumns.mkString(", "), statsCollection.statCollectionLogicalSchema.treeString) } } /** * Validate that the given clusterBySpec matches the existing table's in the given snapshot. * This is used for append mode and replaceWhere. */ def validateClusteringColumnsInSnapshot( snapshot: Snapshot, clusterBySpec: ClusterBySpec): Unit = { // This uses physical column names to compare. val providedClusteringColumns = Some(clusterBySpec.columnNames.map(col => ClusteringColumn(snapshot.schema, col.toString))) val existingClusteringColumns = ClusteredTableUtils.getClusteringColumnsOptional(snapshot) if (providedClusteringColumns != existingClusteringColumns) { throw DeltaErrors.clusteringColumnsMismatchException( clusterBySpec.columnNames.map(_.toString).mkString(","), existingClusteringColumns.map(_.map( ClusteringColumnInfo(snapshot.schema, _).logicalName).mkString(",")).getOrElse("") ) } } /** * Returns empty if all physical clustering columns can be found in the latest stats columns. * Otherwise, returns the logical names of the all clustering columns that are not found. * * [[StatisticsCollection.statsSchema]] has converted field's name to physical name and also it * filters out any columns that are NOT qualified as a stats data type * through [[SkippingEligibleDataType]]. * * @param statsCollection Provides latest stats columns. * @param clusteringColumnInfos Clustering columns in physical names. */ private def getClusteringColumnsNotInStatsSchema( statsCollection: StatisticsCollection, clusteringColumnInfos: Seq[ClusteringColumnInfo]): Seq[String] = { clusteringColumnInfos.flatMap { info => val path = DeltaStatistics.MIN +: info.physicalName SchemaUtils.findNestedFieldIgnoreCase(statsCollection.statsSchema, path) match { // Validate that the column exists in the stats schema and is not a struct // in the stats schema (to catch CLUSTER BY an entire struct). case None | Some(StructField(_, _: StructType, _, _)) => Some(info.logicalName) case _ => None } } } } object ClusteredTableUtils extends ClusteredTableUtilsBase ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/skipping/clustering/ClusteringColumn.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.skipping.clustering import org.apache.spark.sql.delta.{DeltaColumnMapping, DeltaErrors, Snapshot} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.catalyst.expressions.variant.VariantExpressionEvalUtils import org.apache.spark.sql.connector.expressions.FieldReference import org.apache.spark.sql.types.{DataType, StructType} /** * A wrapper class that stores a clustering column's physical name parts. */ case class ClusteringColumn(physicalName: Seq[String]) object ClusteringColumn { /** * Note: `logicalName` must be validated to exist in the given `schema`. */ def apply(schema: StructType, logicalName: String): ClusteringColumn = { val resolver = SchemaUtils.DELTA_COL_RESOLVER // Note that we use AttributeNameParser instead of CatalystSqlParser to account for the case // where the column name is a backquoted string with spaces. val logicalNameParts = FieldReference(logicalName).fieldNames val physicalNameParts = logicalNameParts.foldLeft[(DataType, Seq[String])]((schema, Nil)) { (partial, namePart) => val (currStructType, currPhysicalNameSeq) = partial val field = currStructType match { case fieldType: StructType => fieldType.find(field => resolver(field.name, namePart)) match { case Some(f) => f case None => throw DeltaErrors.columnNotInSchemaException(logicalName, schema) } case _ => throw DeltaErrors.columnNotInSchemaException(logicalName, schema) } // Variant columns cannot be used as clustering columns because they are not orderable. if (VariantExpressionEvalUtils.typeContainsVariant(field.dataType)) { throw DeltaErrors.clusteringColumnUnsupportedDataTypes( s"$logicalName : ${field.dataType.sql}") } (field.dataType, currPhysicalNameSeq :+ DeltaColumnMapping.getPhysicalName(field)) }._2 ClusteringColumn(physicalNameParts) } } /** * A wrapper class that stores a clustering column's physical name parts and data type. */ case class ClusteringColumnInfo( physicalName: Seq[String], dataType: DataType, schema: StructType) { lazy val logicalName: String = { val reversePhysicalNameParts = physicalName.reverse val resolver = SchemaUtils.DELTA_COL_RESOLVER val logicalNameParts = reversePhysicalNameParts .foldRight[(Seq[String], DataType)]((Nil, schema)) { (namePart, state) => val (logicalNameParts, parentRawDataType) = state val parentDataType = parentRawDataType.asInstanceOf[StructType] val nextField = parentDataType .find(field => resolver(DeltaColumnMapping.getPhysicalName(field), namePart)) .get (nextField.name +: logicalNameParts, nextField.dataType) }._1.reverse FieldReference(logicalNameParts).toString } } object ClusteringColumnInfo extends DeltaLogging { def apply(schema: StructType, clusteringColumn: ClusteringColumn): ClusteringColumnInfo = apply(schema, clusteringColumn.physicalName) def apply(schema: StructType, physicalName: Seq[String]): ClusteringColumnInfo = { val resolver = SchemaUtils.DELTA_COL_RESOLVER val dataType = physicalName.foldLeft[DataType](schema) { (currStructType, namePart) => currStructType.asInstanceOf[StructType].find { field => resolver(DeltaColumnMapping.getPhysicalName(field), namePart) }.get.dataType } ClusteringColumnInfo(physicalName, dataType, schema) } def extractLogicalNames(snapshot: Snapshot): Seq[String] = { ClusteredTableUtils.getClusteringColumnsOptional(snapshot).map { clusteringColumns => clusteringColumns.map(ClusteringColumnInfo(snapshot.schema, _).logicalName) }.getOrElse(Seq.empty) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/skipping/clustering/ClusteringStats.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.skipping.clustering import org.apache.spark.sql.delta.commands.DeltaOptimizeContext import org.apache.spark.sql.delta.commands.optimize.ZCubeFileStatsCollector /** * Aggregated file stats for a category of ZCube files. * * @param numFiles Total number of files. * @param size Total physical size of files in bytes. */ case class ClusteringFileStats(numFiles: Long, size: Long) object ClusteringFileStats { def apply(v: Iterable[(Int, Long)]): ClusteringFileStats = { v.foldLeft(ClusteringFileStats(0, 0)) { (a, b) => ClusteringFileStats(a.numFiles + b._1, a.size + b._2) } } } /** * Aggregated stats for OPTIMIZE command on clustered tables. * * @param inputZCubeFiles Files in the ZCubes matching the current OPTIMIZE operation. * @param inputOtherFiles Files not in any ZCubes or in other ZCubes with different * clustering columns. * @param inputNumZCubes Number of different cubes among input files. * @param mergedFiles Subset of input files merged by the current operation * @param numOutputZCubes Number of output ZCubes written out */ case class ClusteringStats( inputZCubeFiles: ClusteringFileStats, inputOtherFiles: ClusteringFileStats, inputNumZCubes: Long, mergedFiles: ClusteringFileStats, numOutputZCubes: Long) /** * A class help collecting ClusteringStats. */ case class ClusteringStatsCollector(zOrderBy: Seq[String], optimizeContext: DeltaOptimizeContext) { val inputStats = new ZCubeFileStatsCollector(zOrderBy, optimizeContext.isFull) val outputStats = new ZCubeFileStatsCollector(zOrderBy, optimizeContext.isFull) var numOutputZCubes = 0 def getClusteringStats: ClusteringStats = { ClusteringStats( inputNumZCubes = inputStats.numZCubes, inputZCubeFiles = ClusteringFileStats(inputStats.fileStats.get("matchingCube")), inputOtherFiles = ClusteringFileStats(inputStats.fileStats.get("otherFiles")), mergedFiles = ClusteringFileStats(outputStats.fileStats.values), numOutputZCubes = numOutputZCubes) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/skipping/clustering/ZCube.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.skipping.clustering import scala.collection.mutable.ArrayBuffer import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.commands.optimize.AddFileWithNumRecords import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} import org.apache.spark.sql.delta.zorder.ZCubeInfo import org.apache.spark.sql.delta.zorder.ZCubeInfo.{getForFile => getZCubeInfo} /** * Collection of files that were produced by the same job in a run of the clustering command. */ case class ZCube(files: Seq[AddFile]) { require(files.nonEmpty) if (DeltaUtils.isTesting) { assert(files.forall(getZCubeInfo(_) == Some(zCubeInfo))) } lazy val zCubeInfo: ZCubeInfo = getZCubeInfo(files.head).get lazy val totalFileSize: Long = files.foldLeft(0L)(_ + _.size) } object ZCube { /** * Given an iterator of files sorted by ZCubeId, returns a filtered iterator of files, * where files belonging to large ZCubes (ZCube size >= target ZCube size ) are filtered * out. * * @param files - Files sorted by ZCubeId, unoptimized files first. */ def filterOutLargeZCubes( files: Iterator[AddFileWithNumRecords], targetCubeSize: Long): Iterator[AddFileWithNumRecords] = { val currentZCube = new ArrayBuffer[AddFileWithNumRecords]() var currentZCubeSize = 0L var currentZCubeId: String = null def appendZCube(file: AddFileWithNumRecords): Unit = { currentZCube.append(file) currentZCubeSize += file.addFile.estLogicalFileSize.getOrElse(file.addFile.size) } def resetZCube(): Unit = { currentZCube.clear() currentZCubeSize = 0 } def returnAndResetCurrentZCube(): Seq[AddFileWithNumRecords] = { val res = if (currentZCubeSize >= targetCubeSize) { // Drop the current ZCube. Seq.empty } else { // Return a copy of current. currentZCube.toVector } resetZCube() res } files.flatMap { addFileWithNumRecords => val file = addFileWithNumRecords.addFile val res = ZCubeInfo.getForFile(file) match { case Some(ZCubeInfo(zCubeID, _)) => // Note: check for ZCubes' ids to group files from the same ZCube. if (zCubeID == currentZCubeId) { // Add to the same ZCube. appendZCube(addFileWithNumRecords) // Skip to next file. Nil } else { // New ZCube. val currentZCubeResult = returnAndResetCurrentZCube() // Start a new ZCube. appendZCube(addFileWithNumRecords) currentZCubeId = zCubeID currentZCubeResult } case None => // Return current ZCube and this file. returnAndResetCurrentZCube() :+ addFileWithNumRecords } if (!files.hasNext) { // Last file, return the current ZCube and the result. returnAndResetCurrentZCube() ++ res } else { res } } } /** * Filter out files belonging to single ZCube. * * @param files - Files sorted by ZCubeId, unoptimized files first. */ def filterOutSingleZCubes( files: Iterator[AddFileWithNumRecords]): Iterator[AddFileWithNumRecords] = { val currentZCube = new ArrayBuffer[AddFileWithNumRecords]() var singleZCube = true var currentZCubeId: String = null def appendZCube(file: AddFileWithNumRecords): Unit = { currentZCube.append(file) } def returnAndResetCurrentZCube(): Seq[AddFileWithNumRecords] = { val res = if (singleZCube) { // Drop the current ZCube. Seq.empty } else { // Return a copy of current. currentZCube.toVector } resetZCube() res } def resetZCube(): Unit = { currentZCube.clear() } files.flatMap { addFileWithNumRecords => val file = addFileWithNumRecords.addFile val res = ZCubeInfo.getForFile(file) match { case Some(ZCubeInfo(zCubeID, _)) => if (zCubeID == currentZCubeId || currentZCubeId == null) { if (currentZCubeId == null) { currentZCubeId = zCubeID } // Same ZCube. appendZCube(addFileWithNumRecords) Nil } else { // New ZCube. currentZCubeId = zCubeID // Return the current ZCube and start a new ZCube. singleZCube = false val currentZCubeResult = returnAndResetCurrentZCube() appendZCube(addFileWithNumRecords) currentZCubeResult } case None => val resZCube = returnAndResetCurrentZCube() // Unoptimized file means the following ZCube is not alone. singleZCube = false resZCube :+ addFileWithNumRecords } if (!files.hasNext) { // Last file, return the current ZCube. returnAndResetCurrentZCube() ++ res } else { res } } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/skipping/clustering/temp/AlterTableClusterBy.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.skipping.clustering.temp import org.apache.spark.sql.catalyst.plans.logical.{AlterTableCommand, LogicalPlan} import org.apache.spark.sql.connector.catalog.TableChange import org.apache.spark.sql.connector.expressions.NamedReference /** * The logical plan of the following commands: * - ALTER TABLE ... CLUSTER BY (col1, col2, ...) * - ALTER TABLE ... CLUSTER BY NONE */ case class AlterTableClusterBy( table: LogicalPlan, clusterBySpec: Option[ClusterBySpec]) extends AlterTableCommand { override def changes: Seq[TableChange] = Seq(ClusterBy(clusterBySpec .map(_.columnNames) // CLUSTER BY (col1, col2, ...) .getOrElse(Seq.empty))) // CLUSTER BY NONE protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = copy(table = newChild) } /** A TableChange to alter clustering columns for a table. */ case class ClusterBy(clusteringColumns: Seq[NamedReference]) extends TableChange {} ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/skipping/clustering/temp/ClusterBySpec.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.skipping.clustering.temp import scala.reflect.ClassTag import org.apache.spark.sql.delta.skipping.clustering.ClusteredTableUtils import com.fasterxml.jackson.annotation.JsonInclude.Include import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} import com.fasterxml.jackson.module.scala.{ClassTagExtensions, DefaultScalaModule} import org.antlr.v4.runtime.ParserRuleContext import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.catalyst.parser.{ParseException, ParserInterface, ParserUtils} import org.apache.spark.sql.catalyst.plans.logical.{CreateTable, CreateTableAsSelect, LeafNode, LogicalPlan, ReplaceTable, ReplaceTableAsSelect} import org.apache.spark.sql.connector.expressions.{BucketTransform, FieldReference, NamedReference, Transform} /** * A container for clustering information. Copied from OSS Spark. * * This class will be removed when we integrate with OSS Spark's CLUSTER BY implementation. * @see https://github.com/apache/spark/pull/42577 * * @param columnNames the names of the columns used for clustering. */ case class ClusterBySpec(columnNames: Seq[NamedReference]) { override def toString: String = toJson def toJson: String = ClusterBySpec.mapper.writeValueAsString(columnNames.map(_.fieldNames)) } object ClusterBySpec { private val mapper = { val ret = new ObjectMapper() with ClassTagExtensions ret.setSerializationInclusion(Include.NON_ABSENT) ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) ret.registerModule(DefaultScalaModule) ret } // ClassTag is added to avoid the "same type after erasure" issue with the case class. def apply[_: ClassTag](columnNames: Seq[Seq[String]]): ClusterBySpec = { ClusterBySpec(columnNames.map(FieldReference(_))) } // Convert from table property back to ClusterBySpec. def fromProperty(columns: String): ClusterBySpec = { ClusterBySpec(mapper.readValue[Seq[Seq[String]]](columns).map(FieldReference(_))) } def toProperty(clusterBySpec: ClusterBySpec): (String, String) = { ClusteredTableUtils.PROP_CLUSTERING_COLUMNS -> clusterBySpec.toJson } def fromProperties(properties: Map[String, String]): Option[ClusterBySpec] = { properties.get(ClusteredTableUtils.PROP_CLUSTERING_COLUMNS).map { clusteringColumns => fromProperty(clusteringColumns) } } def toProperties(clusterBySpec: ClusterBySpec): Map[String, String] = { val columnValue = mapper.writeValueAsString(clusterBySpec.columnNames.map(_.fieldNames)) Map(ClusteredTableUtils.PROP_CLUSTERING_COLUMNS -> columnValue) } def fromColumnNames(names: Seq[String]): ClusterBySpec = { ClusterBySpec(names.map(FieldReference(_))) } } /** * A [[LogicalPlan]] representing a CLUSTER BY clause. * * This class will be removed when we integrate with OSS Spark's CLUSTER BY implementation. * @see https://github.com/apache/spark/pull/42577 * * @param clusterBySpec: clusterBySpec which contains the clustering columns. * @param startIndex: start index of CLUSTER BY clause. * @param stopIndex: stop index of CLUSTER BY clause. * @param parenStartIndex: start index of the left parenthesis in CLUSTER BY clause. * @param parenStopIndex: stop index of the right parenthesis in CLUSTER BY clause. * @param ctx: parser rule context of the CLUSTER BY clause. */ case class ClusterByPlan( clusterBySpec: ClusterBySpec, startIndex: Int, stopIndex: Int, parenStartIndex: Int, parenStopIndex: Int, ctx: ParserRuleContext) extends LeafNode { override def withNewChildrenInternal(newChildren: IndexedSeq[LogicalPlan]): LogicalPlan = this override def output: Seq[Attribute] = Seq.empty } /** * Parser utils for parsing a [[ClusterByPlan]] and converts it to table properties. * * This class will be removed when we integrate with OSS Spark's CLUSTER BY implementation. * @see https://github.com/apache/spark/pull/42577 * * @param clusterByPlan: the ClusterByPlan to parse. * @param delegate: delegate parser. */ case class ClusterByParserUtils(clusterByPlan: ClusterByPlan, delegate: ParserInterface) { // Update partitioning to include clustering columns as transforms. private def updatePartitioning(partitioning: Seq[Transform]): Seq[Transform] = { // Validate no bucketing is specified. if (partitioning.exists(t => t.isInstanceOf[BucketTransform])) { ParserUtils.operationNotAllowed( "Clustering and bucketing cannot both be specified. " + "Please remove CLUSTERED BY INTO BUCKETS if you " + "want to create a Delta table with clustering", clusterByPlan.ctx) } Seq(ClusterByTransform(clusterByPlan.clusterBySpec.columnNames)) } /** * Parse the [[ClusterByPlan]] by replacing CLUSTER BY with PARTITIONED BY and * leverage Spark SQL parser to perform the validation. After parsing, store the * clustering columns in the logical plan's partitioning transforms. * * @param sqlText: original SQL text. * @return the logical plan after parsing. */ def parsePlan(sqlText: String): LogicalPlan = { val colText = sqlText.substring(clusterByPlan.parenStartIndex, clusterByPlan.parenStopIndex + 1) // Replace CLUSTER BY with PARTITIONED BY to let SparkSqlParser do the validation for us. // This serves as a short-term workaround until Spark incorporates CREATE TABLE ... CLUSTER BY // syntax. val partitionedByText = "PARTITIONED BY " + colText val newSqlText = sqlText.substring(0, clusterByPlan.startIndex) + partitionedByText + sqlText.substring(clusterByPlan.stopIndex + 1) try { delegate.parsePlan(newSqlText) match { case create: CreateTable => create.copy(partitioning = updatePartitioning(create.partitioning)) case ctas: CreateTableAsSelect => ctas.copy(partitioning = updatePartitioning(ctas.partitioning)) case replace: ReplaceTable => replace.copy(partitioning = updatePartitioning(replace.partitioning)) case rtas: ReplaceTableAsSelect => rtas.copy(partitioning = updatePartitioning(rtas.partitioning)) case plan => plan } } catch { case e: ParseException if (e.errorClass.contains("DUPLICATE_CLAUSES")) => // Since we replace CLUSTER BY with PARTITIONED BY, duplicated clauses means we // encountered CLUSTER BY with PARTITIONED BY. ParserUtils.operationNotAllowed( "Clustering and partitioning cannot both be specified. " + "Please remove PARTITIONED BY if you want to create a Delta table with clustering", clusterByPlan.ctx) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/skipping/clustering/temp/ClusterByTransform.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.skipping.clustering.temp import org.apache.spark.sql.connector.expressions.{Expression, NamedReference, Transform} /** * Minimal version of Spark's ClusterByTransform. We'll remove this when we integrate with OSS * Spark's CLUSTER BY implementation. * * This class represents a transform for `ClusterBySpec`. This is used to bundle * ClusterBySpec in CreateTable's partitioning transforms to pass it down to analyzer/delta. */ final case class ClusterByTransform( columnNames: Seq[NamedReference]) extends Transform { override val name: String = "temp_cluster_by" override def arguments: Array[Expression] = columnNames.toArray override def toString: String = s"$name(${arguments.map(_.describe).mkString(", ")})" } /** * Convenience extractor for ClusterByTransform. */ object ClusterByTransform { def unapply(transform: Transform): Option[Seq[NamedReference]] = transform match { case NamedTransform("temp_cluster_by", arguments) => Some(arguments.map(_.asInstanceOf[NamedReference])) case _ => None } } /** * Copied from OSS Spark. We'll remove this when we integrate with OSS Spark's CLUSTER BY. * Convenience extractor for any Transform. */ private object NamedTransform { def unapply(transform: Transform): Some[(String, Seq[Expression])] = { Some((transform.name, transform.arguments)) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaDataSource.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.sources import scala.collection.JavaConverters._ import scala.collection.mutable import scala.util.{Failure, Success, Try} // scalastyle:off import.ordering.noEmptyLine import com.databricks.spark.util.DatabricksLogging import org.apache.spark.internal.MDC import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.WriteIntoDelta import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.util.{PartitionUtils, Utils} import org.apache.hadoop.fs.Path import org.json4s.{Formats, NoTypeHints} import org.json4s.jackson.Serialization import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.{EqualTo, Expression, Literal} import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.connector.catalog.{SupportsV1OverwriteWithSaveAsTable, Table, TableProvider} import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.execution.streaming.{Sink, Source} import org.apache.spark.sql.sources._ import org.apache.spark.sql.streaming.OutputMode import org.apache.spark.sql.types.{DataType, StructType, VariantType} import org.apache.spark.sql.util.CaseInsensitiveStringMap /** A DataSource V1 for integrating Delta into Spark SQL batch and Streaming APIs. */ class DeltaDataSource extends RelationProvider with StreamSourceProvider with StreamSinkProvider with CreatableRelationProvider with DataSourceRegister with TableProvider with SupportsV1OverwriteWithSaveAsTable with DeltaLogging { /** * WARNING: This field has complex initialization timing. * * This field is not initialized in the constructor because the DataSource V1 API does not allow * for passing a catalog table. As a work around, we set this field immediately after the * `DeltaDataSource` is constructed in `DataSource::providingInstance()`. */ private var catalogTableOpt: Option[CatalogTable] = None /** * Internal method used only by `DataSource.providingInstance()` right after `DeltaDataSource` * construction to plumb the catalog table. This is intended to be set once per instance; * subsequent sets are ignored by a guard. */ def setCatalogTableOpt(newCatalogTableOpt: Option[CatalogTable]): Unit = { if (catalogTableOpt.isEmpty) { catalogTableOpt = newCatalogTableOpt } } /** * Construct a snapshot from either the catalog table or a path. * * If catalogTableOpt is defined, use it to construct the snapshot; otherwise, fall back to use * path-based snapshot construction. */ private def getSnapshotFromTableOrPath( sparkSession: SparkSession, path: Path, options: Map[String, String]): Snapshot = { catalogTableOpt .map(catalogTable => DeltaLog.forTableWithSnapshot( sparkSession, catalogTable, options)) .getOrElse(DeltaLog.forTableWithSnapshot(sparkSession, path, options))._2 } def inferSchema: StructType = new StructType() // empty override def inferSchema(options: CaseInsensitiveStringMap): StructType = inferSchema override def getTable( schema: StructType, partitioning: Array[Transform], properties: java.util.Map[String, String]): Table = { val options = new CaseInsensitiveStringMap(properties) val path = options.get("path") if (path == null) throw DeltaErrors.pathNotSpecifiedException DeltaTableV2(SparkSession.active, new Path(path), options = options.asScala.toMap) } override def sourceSchema( sqlContext: SQLContext, schema: Option[StructType], providerName: String, parameters: Map[String, String]): (String, StructType) = { val options = new CaseInsensitiveStringMap(parameters.asJava) // Check if we should bypass DeltaLog schema loading for UC-managed tables. // DeltaV2Mode checks the parameters map for UC markers and returns true for // AUTO/STRICT modes with UC tables. val deltaV2Mode = new DeltaV2Mode(sqlContext.sparkSession.sessionState.conf) if (schema.isDefined && deltaV2Mode.shouldBypassSchemaValidationForStreaming(parameters.asJava)) { require(!CDCReader.isCDCRead(options), "CDC read is not supported for schema bypass.") return (shortName(), schema.get) } val path = parameters.getOrElse("path", { throw DeltaErrors.pathNotSpecifiedException }) val (_, maybeTimeTravel) = DeltaTableUtils.extractIfPathContainsTimeTravel( sqlContext.sparkSession, path, Map.empty) if (maybeTimeTravel.isDefined) throw DeltaErrors.timeTravelNotSupportedException if (DeltaDataSource.getTimeTravelVersion(parameters).isDefined) { throw DeltaErrors.timeTravelNotSupportedException } val snapshot = getSnapshotFromTableOrPath(sqlContext.sparkSession, new Path(path), parameters) // This is the analyzed schema for Delta streaming val readSchema = { // Check if we would like to merge consecutive schema changes, this would allow customers // to write queries based on their latest changes instead of an arbitrary schema in the past. val shouldMergeConsecutiveSchemas = sqlContext.sparkSession.sessionState.conf.getConf( DeltaSQLConf.DELTA_STREAMING_ENABLE_SCHEMA_TRACKING_MERGE_CONSECUTIVE_CHANGES ) // This method is invoked during the analysis phase and would determine the schema for the // streaming dataframe. We only need to merge consecutive schema changes here because the // process would create a new entry in the schema log such that when the schema log is // looked up again in the execution phase, we would use the correct schema. DeltaDataSource.getMetadataTrackingLogForDeltaSource( sqlContext.sparkSession, snapshot, catalogTableOpt, parameters, mergeConsecutiveSchemaChanges = shouldMergeConsecutiveSchemas) .flatMap(_.getCurrentTrackedMetadata.map(_.dataSchema)) .getOrElse(snapshot.schema) } DeltaDataSource.verifyReadSchemaMatchesTheTableSchema(schema, readSchema) val schemaToUse = DeltaTableUtils.removeInternalDeltaMetadata( sqlContext.sparkSession, DeltaTableUtils.removeInternalWriterMetadata(sqlContext.sparkSession, readSchema) ) if (schemaToUse.isEmpty) { throw DeltaErrors.schemaNotSetException } if (CDCReader.isCDCRead(options)) { (shortName(), CDCReader.cdcReadSchema(schemaToUse)) } else { (shortName(), schemaToUse) } } override def createSource( sqlContext: SQLContext, metadataPath: String, schema: Option[StructType], providerName: String, parameters: Map[String, String]): Source = { val path = parameters.getOrElse("path", { throw DeltaErrors.pathNotSpecifiedException }) val options = new DeltaOptions(parameters, sqlContext.sparkSession.sessionState.conf) val snapshot = getSnapshotFromTableOrPath(sqlContext.sparkSession, new Path(path), parameters) val schemaTrackingLogOpt = DeltaDataSource.getMetadataTrackingLogForDeltaSource( sqlContext.sparkSession, snapshot, catalogTableOpt, parameters, // Pass in the metadata path opt so we can use it for validation sourceMetadataPathOpt = Some(metadataPath)) val readSchema = schemaTrackingLogOpt.flatMap(_.getCurrentTrackedMetadata).map { metadata => logInfo(log"Delta source schema fetched from tracking log version " + log"${MDC(DeltaLogKeys.VERSION2, schemaTrackingLogOpt.get.getCurrentTrackedSeqNum)}" + log" with Delta commit version " + log"${MDC(DeltaLogKeys.VERSION2, metadata.deltaCommitVersion)}") metadata.dataSchema }.getOrElse { logInfo(log"Delta source schema fetched from Delta snapshot version " + log"${MDC(DeltaLogKeys.VERSION2, snapshot.version)}") snapshot.schema } DeltaDataSource.verifyReadSchemaMatchesTheTableSchema(schema, readSchema) if (readSchema.isEmpty) { throw DeltaErrors.schemaNotSetException } DeltaSource( sqlContext.sparkSession, snapshot.deltaLog, catalogTableOpt, options, snapshot, metadataPath, schemaTrackingLogOpt ) } override def createSink( sqlContext: SQLContext, parameters: Map[String, String], partitionColumns: Seq[String], outputMode: OutputMode): Sink = { val path = parameters.getOrElse("path", { throw DeltaErrors.pathNotSpecifiedException }) if (outputMode != OutputMode.Append && outputMode != OutputMode.Complete) { throw DeltaErrors.outputModeNotSupportedException(getClass.getName, outputMode.toString) } val deltaOptions = new DeltaOptions(parameters, sqlContext.sparkSession.sessionState.conf) // NOTE: Spark API doesn't give access to the CatalogTable here, but DeltaAnalysis will pick // that info out of the containing WriteToStream (if present), and update the sink there. new DeltaSink(sqlContext, new Path(path), partitionColumns, outputMode, deltaOptions) } override def createRelation( sqlContext: SQLContext, mode: SaveMode, parameters: Map[String, String], data: DataFrame): BaseRelation = { val path = parameters.getOrElse("path", { throw DeltaErrors.pathNotSpecifiedException }) val partitionColumns = parameters.get(DeltaSourceUtils.PARTITIONING_COLUMNS_KEY) .map(DeltaDataSource.decodePartitioningColumns) .getOrElse(Nil) val deltaLog = Utils.getDeltaLogFromTableOrPath( sqlContext.sparkSession, catalogTableOpt, new Path(path), parameters) WriteIntoDelta( deltaLog = deltaLog, mode = mode, new DeltaOptions(parameters, sqlContext.sparkSession.sessionState.conf), partitionColumns = partitionColumns, configuration = DeltaConfigs.validateConfigurations( parameters.filterKeys(_.startsWith("delta.")).toMap), data = data, // empty catalogTable is acceptable as the code path is only for path based writes // (df.write.save("path")) which does not need to use/update catalog catalogTableOpt = None ).run(sqlContext.sparkSession) deltaLog.createRelation(catalogTableOpt = catalogTableOpt) } override def createRelation( sqlContext: SQLContext, parameters: Map[String, String]): BaseRelation = { recordFrameProfile("Delta", "DeltaDataSource.createRelation") { val maybePath = parameters.getOrElse("path", { throw DeltaErrors.pathNotSpecifiedException }) // Log any invalid options that are being passed in DeltaOptions.verifyOptions(CaseInsensitiveMap(parameters)) val timeTravelByParams = DeltaDataSource.getTimeTravelVersion(parameters) var cdcOptions: mutable.Map[String, String] = mutable.Map.empty val caseInsensitiveParams = new CaseInsensitiveStringMap(parameters.asJava) if (CDCReader.isCDCRead(caseInsensitiveParams)) { cdcOptions = mutable.Map[String, String](DeltaDataSource.CDC_ENABLED_KEY -> "true") if (caseInsensitiveParams.containsKey(DeltaDataSource.CDC_START_VERSION_KEY)) { cdcOptions(DeltaDataSource.CDC_START_VERSION_KEY) = caseInsensitiveParams.get( DeltaDataSource.CDC_START_VERSION_KEY) } if (caseInsensitiveParams.containsKey(DeltaDataSource.CDC_START_TIMESTAMP_KEY)) { cdcOptions(DeltaDataSource.CDC_START_TIMESTAMP_KEY) = caseInsensitiveParams.get( DeltaDataSource.CDC_START_TIMESTAMP_KEY) } if (caseInsensitiveParams.containsKey(DeltaDataSource.CDC_END_VERSION_KEY)) { cdcOptions(DeltaDataSource.CDC_END_VERSION_KEY) = caseInsensitiveParams.get( DeltaDataSource.CDC_END_VERSION_KEY) } if (caseInsensitiveParams.containsKey(DeltaDataSource.CDC_END_TIMESTAMP_KEY)) { cdcOptions(DeltaDataSource.CDC_END_TIMESTAMP_KEY) = caseInsensitiveParams.get( DeltaDataSource.CDC_END_TIMESTAMP_KEY) } } val dfOptions: Map[String, String] = if (sqlContext.sparkSession.sessionState.conf.getConf( DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS)) { parameters ++ cdcOptions } else { cdcOptions.toMap } DeltaTableV2( sqlContext.sparkSession, new Path(maybePath), timeTravelOpt = timeTravelByParams, options = dfOptions ).toBaseRelation } } /** * Extend the default `supportsDataType` to allow VariantType. */ override def supportsDataType(dt: DataType): Boolean = { dt.isInstanceOf[VariantType] || super.supportsDataType(dt) } override def shortName(): String = { DeltaSourceUtils.ALT_NAME } } object DeltaDataSource extends DatabricksLogging { private implicit val formats: Formats = Serialization.formats(NoTypeHints) final val TIME_TRAVEL_SOURCE_KEY = "__time_travel_source__" /** * The option key for time traveling using a timestamp. The timestamp should be a valid * timestamp string which can be cast to a timestamp type. */ final val TIME_TRAVEL_TIMESTAMP_KEY = "timestampAsOf" /** * The option key for time traveling using a version of a table. This value should be * castable to a long. */ final val TIME_TRAVEL_VERSION_KEY = "versionAsOf" final val CDC_START_VERSION_KEY = "startingVersion" final val CDC_START_TIMESTAMP_KEY = "startingTimestamp" final val CDC_END_VERSION_KEY = "endingVersion" final val CDC_END_TIMESTAMP_KEY = "endingTimestamp" final val CDC_ENABLED_KEY = "readChangeFeed" final val CDC_ENABLED_KEY_LEGACY = "readChangeData" def encodePartitioningColumns(columns: Seq[String]): String = { Serialization.write(columns) } def decodePartitioningColumns(str: String): Seq[String] = { Serialization.read[Seq[String]](str) } /** * For Delta, we allow certain magic to be performed through the paths that are provided by users. * Normally, a user specified path should point to the root of a Delta table. However, some users * are used to providing specific partition values through the path, because of how expensive it * was to perform partition discovery before. We treat these partition values as logical partition * filters, if a table does not exist at the provided path. * * In addition, we allow users to provide time travel specifications through the path. This is * provided after an `@` symbol after a path followed by a time specification in * `yyyyMMddHHmmssSSS` format, or a version number preceded by a `v`. * * This method parses these specifications and returns these modifiers only if a path does not * really exist at the provided path. We first parse out the time travel specification, and then * the partition filters. For example, a path specified as: * /some/path/partition=1@v1234 * will be parsed into `/some/path` with filters `partition=1` and a time travel spec of version * 1234. * * @return A tuple of the root path of the Delta table, partition filters, and time travel options */ def parsePathIdentifier( spark: SparkSession, userPath: String, options: Map[String, String]): (Path, Seq[(String, String)], Option[DeltaTimeTravelSpec]) = { // Handle time travel val (path, timeTravelByPath) = DeltaTableUtils.extractIfPathContainsTimeTravel(spark, userPath, options) val hadoopPath = new Path(path) val rootPath = DeltaTableUtils.findDeltaTableRoot(spark, hadoopPath, options).getOrElse(hadoopPath) val partitionFilters = if (rootPath != hadoopPath) { logConsole( """ |WARNING: loading partitions directly with delta is not recommended. |If you are trying to read a specific partition, use a where predicate. | |CORRECT: spark.read.format("delta").load("/data").where("part=1") |INCORRECT: spark.read.format("delta").load("/data/part=1") """.stripMargin) val fragment = hadoopPath.toString.substring(rootPath.toString.length() + 1) try { PartitionUtils.parsePathFragmentAsSeq(fragment) } catch { case _: ArrayIndexOutOfBoundsException => throw DeltaErrors.partitionPathParseException(fragment) } } else { Nil } (rootPath, partitionFilters, timeTravelByPath) } /** * Verifies that the provided partition filters are valid and returns the corresponding * expressions. */ def verifyAndCreatePartitionFilters( userPath: String, snapshot: Snapshot, partitionFilters: Seq[(String, String)]): Seq[Expression] = { if (partitionFilters.nonEmpty) { val metadata = snapshot.metadata val badColumns = partitionFilters.map(_._1).filterNot(metadata.partitionColumns.contains) if (badColumns.nonEmpty) { val fragment = partitionFilters.map(f => s"${f._1}=${f._2}").mkString("/") throw DeltaErrors.partitionPathInvolvesNonPartitionColumnException(badColumns, fragment) } val filters = partitionFilters.map { case (key, value) => // Nested fields cannot be partitions, so we pass the key as a identifier EqualTo(UnresolvedAttribute(Seq(key)), Literal(value)) } val files = DeltaLog.filterFileList( metadata.partitionSchema, snapshot.allFiles.toDF(), filters) if (files.count() == 0) { throw DeltaErrors.pathNotExistsException(userPath) } filters } else { Nil } } /** Extracts whether users provided the option to time travel a relation. */ def getTimeTravelVersion(parameters: Map[String, String]): Option[DeltaTimeTravelSpec] = { val caseInsensitive = CaseInsensitiveMap[String](parameters) val tsOpt = caseInsensitive.get(DeltaDataSource.TIME_TRAVEL_TIMESTAMP_KEY) val versionOpt = caseInsensitive.get(DeltaDataSource.TIME_TRAVEL_VERSION_KEY) val sourceOpt = caseInsensitive.get(DeltaDataSource.TIME_TRAVEL_SOURCE_KEY) if (tsOpt.isDefined && versionOpt.isDefined) { throw DeltaErrors.provideOneOfInTimeTravel } else if (tsOpt.isDefined) { Some(DeltaTimeTravelSpec(Some(Literal(tsOpt.get)), None, sourceOpt.orElse(Some("dfReader")))) } else if (versionOpt.isDefined) { val version = Try(versionOpt.get.toLong) match { case Success(v) => v case Failure(t) => throw DeltaErrors.timeTravelInvalidBeginValue(DeltaDataSource.TIME_TRAVEL_VERSION_KEY, t) } Some(DeltaTimeTravelSpec(None, Some(version), sourceOpt.orElse(Some("dfReader")))) } else { None } } /** * Extract the schema tracking location from options. */ def extractSchemaTrackingLocationConfig( spark: SparkSession, parameters: Map[String, String]): Option[String] = { val options = new CaseInsensitiveStringMap(parameters.asJava) Option(options.get(DeltaOptions.SCHEMA_TRACKING_LOCATION)) .orElse(Option(options.get(DeltaOptions.SCHEMA_TRACKING_LOCATION_ALIAS))) } /** * Create a schema log for Delta streaming source if possible */ def getMetadataTrackingLogForDeltaSource( spark: SparkSession, sourceSnapshot: SnapshotDescriptor, catalogTableOpt: Option[CatalogTable], parameters: Map[String, String], sourceMetadataPathOpt: Option[String] = None, mergeConsecutiveSchemaChanges: Boolean = false): Option[DeltaSourceMetadataTrackingLog] = { DeltaDataSource.extractSchemaTrackingLocationConfig(spark, parameters) .map { schemaTrackingLocation => if (!spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_STREAMING_ENABLE_SCHEMA_TRACKING)) { throw new UnsupportedOperationException( "Schema tracking location is not supported for Delta streaming source") } DeltaSourceMetadataTrackingLog.create( spark, schemaTrackingLocation, sourceSnapshot, catalogTableOpt, parameters, sourceMetadataPathOpt, mergeConsecutiveSchemaChanges ) } } private def verifyReadSchemaMatchesTheTableSchema( schema: Option[StructType], readSchema: StructType): Unit = { if (schema.nonEmpty && schema.get.nonEmpty && !DataType.equalsIgnoreCompatibleNullability(readSchema, schema.get)) { throw DeltaErrors.readSourceSchemaConflictException } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.sources import java.util.Locale import java.util.concurrent.TimeUnit import org.apache.spark.internal.config.ConfigBuilder import org.apache.spark.internal.config.ConfigEntry import org.apache.spark.network.util.ByteUnit import org.apache.spark.sql.catalyst.FileSourceOptions import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.storage.StorageLevel /** * Utility trait providing common configuration building methods for Delta SQL configs. * * This trait contains only utility methods and constants, no actual config entries. * It is designed to be extended by multiple configuration objects without causing * duplicate config registration. */ trait DeltaSQLConfUtils { val SQL_CONF_PREFIX = "spark.databricks.delta" def buildConf(key: String): ConfigBuilder = SQLConf.buildConf(s"$SQL_CONF_PREFIX.$key") def buildStaticConf(key: String): ConfigBuilder = SQLConf.buildStaticConf(s"spark.databricks.delta.$key") } /** * [[SQLConf]] entries for Delta features. */ trait DeltaSQLConfBase extends DeltaSQLConfUtils { // Values for flags that also support log-only mode. final object BooleanStringOrLogOnly { final val FALSE = "false" final val TRUE = "true" final val LOG_ONLY = "log-only" final val VALUES = Set(FALSE, TRUE, LOG_ONLY) } object DeltaBreakingChangeEnum { val OFF = "OFF" val LOG_ONLY = "LOG_ONLY" val ASSERT = "ASSERT" val validValues: Set[String] = Set(OFF, LOG_ONLY, ASSERT) } abstract class DeltaBreakingChangeEnum(configEntry: ConfigEntry[String]) extends Enumeration { val OFF = Value("OFF") val LOG_ONLY = Value("LOG_ONLY") val ASSERT = Value("ASSERT") def fromConf(conf: SQLConf): Value = withName(conf.getConf(configEntry)) def default: Value = withName(configEntry.defaultValueString) def confName: String = configEntry.key } val RESOLVE_TIME_TRAVEL_ON_IDENTIFIER = buildConf("timeTravel.resolveOnIdentifier.enabled") .internal() .doc("When true, we will try to resolve patterns as `@v123` in identifiers as time " + "travel nodes.") .booleanConf .createWithDefault(true) val DELTA_COMMIT_LOCK_ENABLED = buildConf("commitLock.enabled") .internal() .doc("Whether to lock a Delta table when doing a commit.") .booleanConf .createOptional val DELTA_COLLECT_STATS = buildConf("stats.collect") .internal() .doc("When true, statistics are collected while writing files into a Delta table.") .booleanConf .createWithDefault(true) val DELTA_DML_METRICS_FROM_METADATA = buildConf("dmlMetricsFromMetadata.enabled") .internal() .doc( """ When enabled, metadata only Delete, ReplaceWhere and Truncate operations will report row | level operation metrics by reading the file statistics for number of rows. | """.stripMargin) .booleanConf .createWithDefault(true) val DELTA_COLLECT_STATS_USING_TABLE_SCHEMA = buildConf("stats.collect.using.tableSchema") .internal() .doc("When collecting stats while writing files into Delta table" + s" (${DELTA_COLLECT_STATS.key} needs to be true), whether to use the table schema (true)" + " or the DataFrame schema (false) as the stats collection schema.") .booleanConf .createWithDefault(true) val DELTA_USER_METADATA = buildConf("commitInfo.userMetadata") .doc("Arbitrary user-defined metadata to include in CommitInfo.") .stringConf .createOptional val DELTA_FORCE_ALL_COMMIT_STATS = buildConf("commitStats.force") .internal() .doc( """When true, forces commit statistics to be collected for logging purposes. | Enabling this feature requires the Snapshot State to be computed, which is | potentially expensive. """.stripMargin) .booleanConf .createWithDefault(false) val DELTA_CONVERT_USE_METADATA_LOG = buildConf("convert.useMetadataLog") .doc( """ When converting to a Parquet table that was created by Structured Streaming, whether | to use the transaction log under `_spark_metadata` as the source of truth for files | contained in the table. """.stripMargin) .booleanConf .createWithDefault(true) val DELTA_CONVERT_USE_CATALOG_PARTITIONS = buildConf("convert.useCatalogPartitions") .internal() .doc( """ When converting a catalog Parquet table, whether to use the partition information from | the Metastore catalog and only commit files under the directories of active partitions. |""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_CONVERT_USE_CATALOG_SCHEMA = buildConf("convert.useCatalogSchema") .doc( """ When converting to a catalog Parquet table, whether to use the catalog schema as the | source of truth. |""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_CONVERT_PARTITION_VALUES_IGNORE_CAST_FAILURE = buildConf("convert.partitionValues.ignoreCastFailure") .doc( """ When converting to Delta, ignore the failure when casting a partition value to | the specified data type, in which case the partition column will be filled with null. """.stripMargin) .booleanConf .createWithDefault(false) val DELTA_CONVERT_ICEBERG_USE_NATIVE_PARTITION_VALUES = buildConf("convert.iceberg.useNativePartitionValues") .doc( """ When enabled, obtain the partition values from Iceberg table's metadata, instead | of inferring from file paths. |""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_SNAPSHOT_PARTITIONS = buildConf("snapshotPartitions") .internal() .doc("Number of partitions to use when building a Delta Lake snapshot.") .intConf .checkValue(n => n > 0, "Delta snapshot partition number must be positive.") .createOptional val DELTA_SNAPSHOT_LOADING_MAX_RETRIES = buildConf("snapshotLoading.maxRetries") .internal() .doc("How many times to retry when failing to load a snapshot. Each retry will try to use " + "a different checkpoint in order to skip potential corrupt checkpoints.") .intConf .checkValue(n => n >= 0, "must not be negative.") .createWithDefault(2) val DELTA_SNAPSHOT_CACHE_STORAGE_LEVEL = buildConf("snapshotCache.storageLevel") .internal() .doc("StorageLevel to use for caching the DeltaLog Snapshot. In general, this should not " + "be used unless you are pretty sure that caching has a negative impact.") .stringConf .createWithDefault("MEMORY_AND_DISK_SER") val DELTA_SNAPSHOT_LOGGING_MAX_FILES_THRESHOLD = buildConf("snapshot.logging.maxFilesThreshold") .internal() .doc("Threshold for number of files in a snapshot. When exceeded, emits a warning with " + "remediation hints. Set to 0 to disable snapshot logging completely.") .longConf .checkValue(_ >= 0, "must be non-negative") .createWithDefault(500000L) val DELTA_PARTITION_COLUMN_CHECK_ENABLED = buildConf("partitionColumnValidity.enabled") .internal() .doc("Whether to check whether the partition column names have valid names, just like " + "the data columns.") .booleanConf .createWithDefault(true) val DELTA_PARTITION_COLUMN_CHANGE_CHECK = buildConf("partitionColumnChangeCheck") .internal() .doc("""Controls the validation behavior when changes to partition columns are detected. |Possible values: | - "false": Disables validation for partition column changes. | - "true": Enables validation and throws an error if an illegal change is detected. | - "log-only": Logs the detected illegal change but does not block the operation. |""".stripMargin) .stringConf .transform(_.toLowerCase(Locale.ROOT)) .checkValues(BooleanStringOrLogOnly.VALUES) .createWithDefault( if (DeltaUtils.isTesting) { BooleanStringOrLogOnly.TRUE } else { BooleanStringOrLogOnly.LOG_ONLY } ) val DELTA_COMMIT_VALIDATION_ENABLED = buildConf("commitValidation.enabled") .internal() .doc("Whether to perform validation checks before commit or not.") .booleanConf .createWithDefault(true) val DELTA_EMPTY_FILE_CHECK_THROW_ENABLED = buildConf("emptyFileCheck.throwEnabled") .internal() .doc("When true, throws IllegalStateException if a commit contains AddFile actions " + "referencing parquet files with size 0 bytes. " + "When false, only logs. Logging always occurs regardless of this setting.") .booleanConf .createWithDefault(false) val DELTA_NULL_PARTITION_CHECK_THROW_ENABLED = buildConf("nullPartitionCheck.throwEnabled") .internal() .doc("When true, throws IllegalStateException if a commit contains AddFile actions with " + "null partition values for columns that have NOT NULL constraints. " + "When false, only logs. Logging always occurs regardless of this setting.") .booleanConf .createWithDefault(false) val DELTA_SCHEMA_ON_READ_CHECK_ENABLED = buildConf("checkLatestSchemaOnRead") .doc("In Delta, we always try to give users the latest version of their data without " + "having to call REFRESH TABLE or redefine their DataFrames when used in the context of " + "streaming. There is a possibility that the schema of the latest version of the table " + "may be incompatible with the schema at the time of DataFrame creation. This flag " + "enables a check that ensures that users won't read corrupt data if the source schema " + "changes in an incompatible way.") .booleanConf .createWithDefault(true) val DELTA_INCLUDE_TABLE_ID_IN_FILE_INDEX_COMPARISON = buildConf("includeTableIdInFileIndexComparison") .internal() .doc( """ |Include the deltaLog.unsafeVolatileTableId field in equals and hashCode for |TahoeLogFileIndex. The field is unstable, so including it can lead semantic violations |for equals and hashCode.""".stripMargin) .booleanConf // TODO: Phase this out towards `false` eventually remove the flag altogether again. .createWithDefault(true) val DELTA_ALLOW_CREATE_EMPTY_SCHEMA_TABLE = buildConf("createEmptySchemaTable.enabled") .internal() .doc( s"""If enabled, creating a Delta table with an empty schema will be allowed through SQL API |`CREATE TABLE table () USING delta ...`, or Delta table APIs. |Creating a Delta table with empty schema table using dataframe operations or |`CREATE OR REPLACE` syntax are not supported. |The result Delta table can be updated using schema evolution operations such as |`df.save()` with `mergeSchema = true`. |Reading the empty schema table using DataframeReader or `SELECT` is not allowed. |""".stripMargin) .booleanConf .createWithDefault(true) val AUTO_COMPACT_ALLOWED_VALUES = Seq( "false", "true" ) val DELTA_AUTO_COMPACT_ENABLED = buildConf("autoCompact.enabled") .doc(s"""Whether to compact files after writes made into Delta tables from this session. This | conf can be set to "true" to enable Auto Compaction, OR "false" to disable Auto Compaction | on all writes across all delta tables in this session. | """.stripMargin) .stringConf .transform(_.toLowerCase(Locale.ROOT)) .checkValue(AUTO_COMPACT_ALLOWED_VALUES.contains(_), """"spark.databricks.delta.autoCompact.enabled" must be one of: """ + s"""${AUTO_COMPACT_ALLOWED_VALUES.mkString("(", ",", ")")}""") .createOptional val DELTA_AUTO_COMPACT_RECORD_PARTITION_STATS_ENABLED = buildConf("autoCompact.recordPartitionStats.enabled") .internal() .doc(s"""When enabled, each committed write delta transaction records the number of qualified |files of each partition of the target table for Auto Compact in driver's |memory.""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_AUTO_COMPACT_EARLY_SKIP_PARTITION_TABLE_ENABLED = buildConf("autoCompact.earlySkipPartitionTable.enabled") .internal() .doc(s"""Auto Compaction will be skipped if there is no partition with |sufficient number of small files.""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_AUTO_COMPACT_MAX_TABLE_PARTITION_STATS = buildConf("autoCompact.maxTablePartitionStats") .internal() .doc( s"""The maximum number of Auto Compaction partition statistics of each table. This controls |the maximum number of partitions statistics each delta table can have. Increasing |this value reduces the hash conflict and makes partitions statistics more accurate with |the cost of more memory consumption. |""".stripMargin) .intConf .checkValue(_ > 0, "The value of maxTablePartitionStats should be positive.") .createWithDefault(16 * 1024) val DELTA_AUTO_COMPACT_PARTITION_STATS_SIZE = buildConf("autoCompact.partitionStatsSize") .internal() .doc( s"""The total number of partitions statistics entries can be kept in memory for all |tables in each driver. If this threshold is reached, the partitions statistics of |least recently accessed tables will be evicted out.""".stripMargin) .intConf .checkValue(_ > 0, "The value of partitionStatsSize should be positive.") .createWithDefault(64 * 1024) val DELTA_AUTO_COMPACT_MAX_FILE_SIZE = buildConf("autoCompact.maxFileSize") .internal() .doc(s"Target file size produced by auto compaction. The default value of this config" + " is 128 MB.") .longConf .checkValue(_ >= 0, "maxFileSize has to be positive") .createWithDefault(128 * 1024 * 1024) val DELTA_AUTO_COMPACT_MIN_NUM_FILES = buildConf("autoCompact.minNumFiles") .internal() .doc("Number of small files that need to be in a directory before it can be optimized.") .intConf .checkValue(_ >= 0, "minNumFiles has to be positive") .createWithDefault(50) val DELTA_AUTO_COMPACT_MIN_FILE_SIZE = buildConf("autoCompact.minFileSize") .internal() .doc("Files which are smaller than this threshold (in bytes) will be grouped together and " + "rewritten as larger files by the Auto Compaction. The default value of this config " + s"is set to half of the config ${DELTA_AUTO_COMPACT_MAX_FILE_SIZE.key}") .longConf .checkValue(_ >= 0, "minFileSize has to be positive") .createOptional val DELTA_AUTO_COMPACT_MODIFIED_PARTITIONS_ONLY_ENABLED = buildConf("autoCompact.modifiedPartitionsOnly.enabled") .internal() .doc( s"""When enabled, Auto Compaction only works on the modified partitions of the delta |transaction that triggers compaction.""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_AUTO_COMPACT_NON_BLIND_APPEND_ENABLED = buildConf("autoCompact.nonBlindAppend.enabled") .internal() .doc( s"""When enabled, Auto Compaction is only triggered by non-blind-append write |transaction.""".stripMargin) .booleanConf .createWithDefault(false) val DELTA_AUTO_COMPACT_MAX_NUM_MODIFIED_PARTITIONS = buildConf("autoCompact.maxNumModifiedPartitions") .internal() .doc( s"""The maximum number of partition can be selected for Auto Compaction when | Auto Compaction runs on modified partition is enabled.""".stripMargin) .intConf .checkValue(_ > 0, "The value of maxNumModifiedPartitions should be positive.") .createWithDefault(128) val DELTA_AUTO_COMPACT_RESERVE_PARTITIONS_ENABLED = buildConf("autoCompact.reservePartitions.enabled") .internal() .doc( s"""When enabled, each Auto Compact thread reserves its target partitions and skips the |partitions that are under Auto Compaction by another thread |concurrently.""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_IMPORT_BATCH_SIZE_STATS_COLLECTION = buildConf("import.batchSize.statsCollection") .internal() .doc("The number of files per batch for stats collection during import.") .intConf .createWithDefault(50000) val DELTA_IMPORT_BATCH_SIZE_SCHEMA_INFERENCE = buildConf("import.batchSize.schemaInference") .internal() .doc("The number of files per batch for schema inference during import.") .intConf .createWithDefault(1000000) val DELTA_SAMPLE_ESTIMATOR_ENABLED = buildConf("sampling.enabled") .internal() .doc("Enable sample based estimation.") .booleanConf .createWithDefault(false) val DELTA_CONVERT_METADATA_CHECK_ENABLED = buildConf("convert.metadataCheck.enabled") .doc( """ |If enabled, during convert to delta, if there is a difference between the catalog table's |properties and the Delta table's configuration, we should error. If disabled, merge |the two configurations with the same semantics as update and merge. """.stripMargin) .booleanConf .createWithDefault(true) val DELTA_STATS_SKIPPING = buildConf("stats.skipping") .internal() .doc("When true, statistics are used for skipping") .booleanConf .createWithDefault(true) val DELTA_ALWAYS_COLLECT_STATS = buildConf("alwaysCollectStats.enabled") .internal() .doc("When true, row counts are collected from file statistics even when there are no " + "data filters. This is useful for ensuring PreparedDeltaFileIndex always has row count " + "information available. Note: this may have a small performance overhead as it requires " + "summing numRecords from all files.") .booleanConf .createWithDefault(false) val DELTA_LIMIT_PUSHDOWN_ENABLED = buildConf("stats.limitPushdown.enabled") .internal() .doc("If true, use the limit clause and file statistics to prune files before " + "they are collected to the driver. ") .booleanConf .createWithDefault(true) val DELTA_MAX_RETRY_COMMIT_ATTEMPTS = buildConf("maxCommitAttempts") .internal() .doc("The maximum number of commit attempts we will try for a single commit before failing") .intConf .checkValue(_ >= 0, "maxCommitAttempts has to be positive") .createWithDefault(10000000) val DELTA_MAX_NON_CONFLICT_RETRY_COMMIT_ATTEMPTS = buildConf("maxNonConflictCommitAttempts") .internal() .doc("The maximum number of non-conflict commit attempts we will try for a single commit " + "before failing") .intConf .checkValue(_ >= 0, "maxNonConflictCommitAttempts has to be positive") .createWithDefault(10) val DELTA_CONFLICT_CHECKER_ENFORCE_FEATURE_ENABLEMENT_VALIDATION = buildConf("conflictChecker.enforceConcurrentFeatureEnablement.enabled") .internal() .doc("When enabled, the conflict checker will enforce that features that are marked " + "as failing concurrent transactions at upgrade, will fail any conflicting commits with " + "their enablement protocol changes.") .booleanConf .createWithDefault(false) val FEATURE_ENABLEMENT_CONFLICT_RESOLUTION_ENABLED = buildConf("featureEnablement.conflictResolution.enabled") .internal() .doc( """Controls whether we attempt to resolve feature enablement with allowlist. |This is only intended to be used as a kill switch.""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_PROTOCOL_DEFAULT_WRITER_VERSION = buildConf("properties.defaults.minWriterVersion") .doc("The default writer protocol version to create new tables with, unless a feature " + "that requires a higher version for correctness is enabled.") .intConf .checkValues(Set(1, 2, 3, 4, 5, 6, 7)) .createWithDefault(2) val DELTA_PROTOCOL_DEFAULT_READER_VERSION = buildConf("properties.defaults.minReaderVersion") .doc("The default reader protocol version to create new tables with, unless a feature " + "that requires a higher version for correctness is enabled.") .intConf .checkValues(Set(1, 2, 3)) .createWithDefault(1) val TABLE_FEATURES_TEST_FEATURES_ENABLED = buildConf("tableFeatures.testFeatures.enabled") .internal() .doc("Controls whether test features are enabled in testing mode. " + "This config is only used for testing purposes. ") .booleanConf .createWithDefault(true) val UNSUPPORTED_TESTING_FEATURES_ENABLED = buildConf("tableFeatures.dev.unsupportedTableFeatures.enabled") .internal() .doc( """When turned on, it emulates the existence of unsupported features by the client. |This config is only used for testing purposes.""".stripMargin) .booleanConf .createWithDefault(false) val ALLOW_METADATA_CLEANUP_WHEN_ALL_PROTOCOLS_SUPPORTED = buildConf("tableFeatures.allowMetadataCleanupWhenAllProtocolsSupported") .internal() .doc( """Whether to perform protocol validation when the client is unable to clean |up to 'delta.requireCheckpointProtectionBeforeVersion'.""".stripMargin) .booleanConf .createWithDefault(true) val ALLOW_METADATA_CLEANUP_CHECKPOINT_EXISTENCE_CHECK_DISABLED = buildConf("tableFeatures.dev.allowMetadataCleanupCheckpointExistenceCheck.disabled") .internal() .doc( """Whether to disable the checkpoint check at the cleanup boundary when performing |the CheckpointProtectionTableFeature validations. |This is only used for testing purposes.'.""".stripMargin) .booleanConf .createWithDefault(false) val FAST_DROP_FEATURE_ENABLED = buildConf("tableFeatures.fastDropFeature.enabled") .internal() .doc( """Whether to allow dropping features with the fast drop feature feature |functionality.""".stripMargin) .booleanConf .createWithDefault(true) val FAST_DROP_FEATURE_DV_DISCOVERY_IN_VACUUM_DISABLED = buildConf("tableFeatures.dev.fastDropFeature.DVDiscoveryInVacuum.disabled") .internal() .doc( """Whether to allow DV discovery in Vacuum. |This is config is only intended for testing purposes.""".stripMargin) .booleanConf .createWithDefault(false) val FAST_DROP_FEATURE_GENERATE_DV_TOMBSTONES = buildConf("tableFeatures.fastDropFeature.generateDVTombstones.enabled") .internal() .doc( """Whether to generate DV tombstones when dropping deletion vectors. |These make sure deletion vector files won't accidentally be vacuumed by clients |that do not support DVs.""".stripMargin) .booleanConf .createWithDefaultFunction(() => SQLConf.get.getConf(DeltaSQLConf.FAST_DROP_FEATURE_ENABLED)) val FAST_DROP_FEATURE_DV_TOMBSTONE_COUNT_THRESHOLD = buildConf("tableFeatures.fastDropFeature.dvTombstoneCountThreshold") .doc( """The maximum number of DV tombstones we are allowed store to memory when dropping |deletion vectors. When the resulting number of DV tombstones is higher, we use |a special commit for large outputs. This does not materialize results to memory |but does not retry in case of a conflict.""".stripMargin) .intConf .checkValue(_ >= 0, "DVTombstoneCountThreshold must not be negative.") .createWithDefault(10000) val FAST_DROP_FEATURE_STREAMING_ALWAYS_VALIDATE_PROTOCOL = buildConf("tableFeatures.fastDropFeature.alwaysValidateProtocolInStreaming.enabled") .internal() .doc( """Whether to validate the protocol when starting a stream from arbitrary |versions.""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_MAX_SNAPSHOT_LINEAGE_LENGTH = buildConf("maxSnapshotLineageLength") .internal() .doc("The max lineage length of a Snapshot before Delta forces to build a Snapshot from " + "scratch.") .intConf .checkValue(_ > 0, "maxSnapshotLineageLength must be positive.") .createWithDefault(50) val DELTA_REPLACE_COLUMNS_SAFE = buildConf("alter.replaceColumns.safe.enabled") .internal() .doc("Prevents an ALTER TABLE REPLACE COLUMNS method from dropping all columns, which " + "leads to losing all data. It will only allow safe, unambiguous column changes.") .booleanConf .createWithDefault(true) val DELTA_HISTORY_PAR_SEARCH_THRESHOLD = buildConf("history.maxKeysPerList") .internal() .doc("How many commits to list when performing a parallel search. Currently set to 1000, " + "which is the maximum keys returned by S3 per list call. Azure can return 5000, " + "therefore we choose 1000.") .intConf .createWithDefault(1000) val DELTA_HISTORY_METRICS_ENABLED = buildConf("history.metricsEnabled") .doc("Enables Metrics reporting in Describe History. CommitInfo will now record the " + "Operation Metrics.") .booleanConf .createWithDefault(true) val DELTA_VACUUM_RETENTION_WINDOW_IGNORE_ENABLED = buildConf("vacuum.retentionWindowIgnore.enabled") .internal() .doc("When set, retention window as part of Vacuum will be ignored unless the value is 0") .booleanConf .createWithDefault(true) val DELTA_HISTORY_MANAGER_THREAD_POOL_SIZE = buildConf("history.threadPoolSize") .internal() .doc("The size of the thread pool used for search during DeltaHistory operations. " + "This configuration is only used when the feature inCommitTimestamps is enabled.") .intConf .checkValue(_ > 0, "history.threadPoolSize must be positive") .createWithDefault(10) val ENFORCE_TIME_TRAVEL_WITHIN_DELETED_FILE_RETENTION_DURATION = buildConf("vacuum.enforceTimeTravelWithinDeletedFileRetentionDuration") .internal() .doc("Enforces time travel within delta.deletedFileRetentionDuration.") .booleanConf .createWithDefault(true) val DELTA_VACUUM_LOGGING_ENABLED = buildConf("vacuum.logging.enabled") .doc("Whether to log vacuum information into the Delta transaction log." + " Users should only set this config to 'true' when the underlying file system safely" + " supports concurrent writes.") .booleanConf .createOptional val LITE_VACUUM_ENABLED = buildConf("vacuum.lite.enabled") .doc("Allows Vacuum to be run in Lite mode") .booleanConf .createWithDefault(false) val DELTA_VACUUM_RETENTION_CHECK_ENABLED = buildConf("retentionDurationCheck.enabled") .doc("Adds a check preventing users from running vacuum with a very short retention " + "period, which may end up corrupting the Delta Log.") .booleanConf .createWithDefault(true) val DELTA_VACUUM_PARALLEL_DELETE_ENABLED = buildConf("vacuum.parallelDelete.enabled") .doc("Enables parallelizing the deletion of files during a vacuum command. Enabling " + "may result hitting rate limits on some storage backends. When enabled, parallelization " + "is controlled 'spark.databricks.delta.vacuum.parallelDelete.parallelism'.") .booleanConf .createWithDefault(false) val DELTA_VACUUM_PARALLEL_DELETE_PARALLELISM = buildConf("vacuum.parallelDelete.parallelism") .doc("Sets the number of partitions to use for parallel deletes. If not set, defaults to " + "spark.sql.shuffle.partitions.") .intConf .checkValue(_ > 0, "parallelDelete.parallelism must be positive") .createOptional val ENFORCE_DELETED_FILE_AND_LOG_RETENTION_DURATION_COMPATIBILITY = buildConf("vacuum.enforceDeletedFileAndLogRetentionDurationCompatibility") .internal() .doc("Throws an error if log retention duration is less than deletedFileRetentionDuration") .booleanConf .createWithDefault(true) val DELTA_SCHEMA_AUTO_MIGRATE = buildConf("schema.autoMerge.enabled") .doc("If true, enables schema merging on appends and on overwrites.") .booleanConf .createWithDefault(false) val DELTA_MERGE_SCHEMA_EVOLUTION_FIX_NESTED_STRUCT_ALIGNMENT = buildConf("schemaEvolution.merge.fixNestedStructAlignment") .internal() .doc("Internal flag covering a fix for a regression in schema evolution inside nested " + "structs in MERGE. Disabling this fix may cause MERGE operations to fail when a new " + "field is added to a struct that is omitted in at least one MATCHED clause.") .booleanConf .createWithDefault(true) val DELTA_MERGE_PRESERVE_NULL_SOURCE_STRUCTS = buildConf("merge.preserveNullSourceStructs") .internal() .doc( """Fixes the null expansion issue by preserving NULL structs in MERGE operations. When set |to true, a NULL struct in the source will be preserved as NULL in the target after MERGE, |rather than being incorrectly expanded to a struct with NULL fields. When set to false, |NULL structs are expanded. This fix addresses null expansion caused by (1) struct type |cast, and (2) expanding UPDATE SET * to leaf-level actions in schema evolution (when |`spark.databricks.delta.merge.preserveNullSourceStructs.updateStar` is also enabled). |Note: The fix for struct type cast also fixes the null expansion issue in UPDATE queries |and streaming inserts with struct type cast. |""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_MERGE_PRESERVE_NULL_SOURCE_STRUCTS_UPDATE_STAR = buildConf("merge.preserveNullSourceStructs.updateStar") .internal() .doc("""Fixes the null expansion issue in MERGE with UPDATE SET * actions in schema evolution. |When set to true, and `spark.databricks.delta.merge.preserveNullSourceStructs` is also |true, a NULL struct in the source will be preserved as NULL in the target after MERGE, |rather than being incorrectly expanded to a struct with NULL fields. Otherwise, NULL |structs are expanded.""".stripMargin) .fallbackConf(DELTA_MERGE_PRESERVE_NULL_SOURCE_STRUCTS) val DELTA_INSERT_PRESERVE_NULL_SOURCE_STRUCTS = buildConf("insert.preserveNullSourceStructs") .internal() .doc( """Fixes the null expansion issue by preserving NULL structs in INSERT operations. When set |to true, a NULL struct in the source will be preserved as NULL in the target after |INSERT, rather than being incorrectly expanded to a struct with NULL fields. When set to |false, NULL structs are expanded. This fix addresses null expansion caused by struct |type cast during INSERT operations. |""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_INSERT_BY_NAME_SCHEMA_EVOLUTION_ENABLED = buildConf("insert.byName.schemaEvolution.enabled") .internal() .doc( """When enabled, SQL INSERT INTO BY NAME operations allow schema evolution: extra columns in |the source that are not in the target table schema are added to the target schema when |schema evolution (mergeSchema) is also enabled. Disable this flag to revert to the old |behavior where extra columns always cause an error, regardless of schema evolution |settings.""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_SCHEMA_TYPE_CHECK = buildConf("schema.typeCheck.enabled") .doc( """Enable the data type check when updating the table schema. Disabling this flag may | allow users to create unsupported Delta tables and should only be used when trying to | read/write legacy tables.""".stripMargin) .internal() .booleanConf .createWithDefault(true) val DELTA_SCHEMA_REMOVE_SPARK_INTERNAL_METADATA = buildConf("schema.removeSparkInternalMetadata") .doc( """Whether to remove leaked Spark's internal metadata from the table schema before returning |to Spark. These internal metadata might be stored unintentionally in tables created by |old Spark versions""".stripMargin) .internal() .booleanConf .createWithDefault(true) val DELTA_UPDATE_CATALOG_ENABLED = buildConf("catalog.update.enabled") .internal() .doc("When enabled, we will cache the schema of the Delta table and the table properties " + "in the external catalog, e.g. the Hive MetaStore.") .booleanConf .createWithDefault(false) val DELTA_UPDATE_CATALOG_THREAD_POOL_SIZE = buildStaticConf("catalog.update.threadPoolSize") .internal() .doc("The size of the thread pool for updating the external catalog.") .intConf .checkValue(_ > 0, "threadPoolSize must be positive") .createWithDefault(20) val COORDINATED_COMMITS_GET_COMMITS_THREAD_POOL_SIZE = buildStaticConf("coordinatedCommits.getCommits.threadPoolSize") .internal() .doc("The size of the thread pool for listing files from the commit-coordinator.") .intConf .checkValue(_ > 0, "threadPoolSize must be positive") .createWithDefault(5) val COORDINATED_COMMITS_IGNORE_MISSING_COORDINATOR_IMPLEMENTATION = buildConf("coordinatedCommits.ignoreMissingCoordinatorImplementation") .internal() .doc("When enabled, reads will not fail if the commit coordinator implementation " + "is missing. Writes will still fail and reads will just rely on backfilled commits. " + "This also means that reads can be stale.") .booleanConf .createWithDefault(true) val REMOVE_EXISTS_DEFAULT_FROM_SCHEMA = buildConf("schema.removeExistsDefault") .internal() .doc("When enabled, do not store the 'EXISTS_DEFAULT' metadata key when a table with a " + "default value is created and this table does not re-use existing data files." + "'EXISTS_DEFAULT' holds values that are used in Spark for existing rows when a new column" + "with a default value is added to a table. Since we do not support adding columns with a" + "default value in Delta, this metadata key can be omitted, except in cases like when" + "we convert a table to Delta that does actually require 'EXISTS_DEFAULT'.") .booleanConf .createWithDefault(true) val HMS_FORCE_ALTER_TABLE_DATA_SCHEMA = buildConf("hms.schema.forceAlterTableDataSchema") .internal() .doc( """ | This conf fixes the schema in tableCatalog object and force an alter table | schema command after upload the schema. As in spark project the schema is removed | because delta is not a valid serDe configuration. This is a problem known only to HMS. |""".stripMargin) .booleanConf .createWithDefault(false) ////////////////////////////////////////////// // DynamoDB Commit Coordinator-specific configs ///////////////////////////////////////////// val COORDINATED_COMMITS_DDB_AWS_CREDENTIALS_PROVIDER_NAME = buildConf("coordinatedCommits.commitCoordinator.dynamodb.awsCredentialsProviderName") .internal() .doc("The fully qualified class name of the AWS credentials provider to use for " + "interacting with DynamoDB in the DynamoDB Commit Coordinator Client. e.g. " + "com.amazonaws.auth.DefaultAWSCredentialsProviderChain.") .stringConf .createWithDefault("com.amazonaws.auth.DefaultAWSCredentialsProviderChain") val COORDINATED_COMMITS_DDB_SKIP_PATH_CHECK = buildConf("coordinatedCommits.commitCoordinator.dynamodb.skipPathCheckEnabled") .internal() .doc("When enabled, the DynamoDB Commit Coordinator will not enforce that the table path " + "of the current Delta table matches the stored in the corresponding DynamoDB table. This " + "should only be used when the observed table path for the same physical table varies " + "depending on how it is accessed (e.g. abfs://path1 vs abfss://path1). Leaving this " + "enabled can be dangerous as every physical copy of a Delta table with try to write to" + " the same DynamoDB table.") .booleanConf .createWithDefault(false) val COORDINATED_COMMITS_DDB_READ_CAPACITY_UNITS = buildConf("coordinatedCommits.commitCoordinator.dynamodb.readCapacityUnits") .internal() .doc("Controls the provisioned read capacity units for the DynamoDB table backing the " + "DynamoDB Commit Coordinator. This configuration is only used when the DynamoDB table " + "is first provisioned and cannot be used configure an existing table.") .intConf .createWithDefault(5) val COORDINATED_COMMITS_DDB_WRITE_CAPACITY_UNITS = buildConf("coordinatedCommits.commitCoordinator.dynamodb.writeCapacityUnits") .internal() .doc("Controls the provisioned write capacity units for the DynamoDB table backing the " + "DynamoDB Commit Coordinator. This configuration is only used when the DynamoDB table " + "is first provisioned and cannot be used configure an existing table.") .intConf .createWithDefault(5) ////////////////////////////////////////////// // DynamoDB Commit Coordinator-specific configs end ///////////////////////////////////////////// val IN_COMMIT_TIMESTAMP_RETAIN_ENABLEMENT_INFO_FIX_ENABLED = buildConf("inCommitTimestamp.retainEnablementInfoFix.enabled") .internal() .doc("When disabled, Delta can end up dropping " + s"inCommitTimestampEnablementVersion and inCommitTimestampEnablementTimestamp " + s"during a REPLACE or CLONE command. This accidental removal of these " + s"properties can result in failures on time travel queries.") .booleanConf .createWithDefault(true) val DELTA_UPDATE_CATALOG_LONG_FIELD_TRUNCATION_THRESHOLD = buildConf("catalog.update.longFieldTruncationThreshold") .internal() .doc( "When syncing table schema to the catalog, Delta will truncate the whole schema " + "if any field is longer than this threshold.") .longConf .createWithDefault(4000) val DELTA_ASSUMES_DROP_CONSTRAINT_IF_EXISTS = buildConf("constraints.assumesDropIfExists.enabled") .doc("""If true, DROP CONSTRAINT quietly drops nonexistent constraints even without |IF EXISTS. """) .booleanConf .createWithDefault(false) val DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT = buildConf("stalenessLimit") .doc( """Setting a non-zero time limit will allow you to query the last loaded state of the Delta |table without blocking on a table update. You can use this configuration to reduce the |latency on queries when up-to-date results are not a requirement. Table updates will be |scheduled on a separate scheduler pool in a FIFO queue, and will share cluster resources |fairly with your query. If a table hasn't updated past this time limit, we will block |on a synchronous state update before running the query. """.stripMargin) .timeConf(TimeUnit.MILLISECONDS) .checkValue(_ >= 0, "Staleness limit cannot be negative") .createWithDefault(0L) // Don't let tables go stale val DELTA_ALTER_LOCATION_BYPASS_SCHEMA_CHECK = buildConf("alterLocation.bypassSchemaCheck") .doc("If true, Alter Table Set Location on Delta will go through even if the Delta table " + "in the new location has a different schema from the original Delta table.") .booleanConf .createWithDefault(false) val DUMMY_FILE_MANAGER_NUM_OF_FILES = buildConf("dummyFileManager.numOfFiles") .internal() .doc("How many dummy files to write in DummyFileManager") .intConf .checkValue(_ >= 0, "numOfFiles can not be negative.") .createWithDefault(3) val DUMMY_FILE_MANAGER_PREFIX = buildConf("dummyFileManager.prefix") .internal() .doc("The file prefix to use in DummyFileManager") .stringConf .createWithDefault(".s3-optimization-") val DELTA_MERGE_ANALYSIS_BATCH_RESOLUTION = buildConf("merge.analysis.batchActionResolution.enabled") .internal() .doc( """ | Whether to batch the analysis of all DeltaMergeActions within a clause | during merge's analysis resolution. |""".stripMargin) .booleanConf .createWithDefault(true) val MERGE_INSERT_ONLY_ENABLED = buildConf("merge.optimizeInsertOnlyMerge.enabled") .internal() .doc( """ |If enabled, merge without any matched clause (i.e., insert-only merge) will be optimized |by avoiding rewriting old files and just inserting new files. """.stripMargin) .booleanConf .createWithDefault(true) val MERGE_REPARTITION_BEFORE_WRITE = buildConf("merge.repartitionBeforeWrite.enabled") .internal() .doc( """ |When enabled, merge will repartition the output by the table's partition columns before |writing the files. """.stripMargin) .booleanConf .createWithDefault(true) val MERGE_MATCHED_ONLY_ENABLED = buildConf("merge.optimizeMatchedOnlyMerge.enabled") .internal() .doc( """If enabled, merge without 'when not matched' clause will be optimized to use a |right outer join instead of a full outer join. """.stripMargin) .booleanConf .createWithDefault(true) val MERGE_SKIP_OSS_RESOLUTION_WITH_STAR = buildConf("merge.skipOssResolutionWithStar") .internal() .doc( """ |If enabled, then any MERGE operation having UPDATE * / INSERT * will skip Apache |Spark's resolution logic and use Delta's specific resolution logic. This is to avoid |bug with star and temp views. See SC-72276 for details. """.stripMargin) .booleanConf .createWithDefault(true) val MERGE_FAIL_IF_SOURCE_CHANGED = buildConf("merge.failIfSourceChanged") .internal() .doc( """ |When enabled, MERGE will fail if it detects that the source dataframe was changed. |This can be triggered as a result of modified input data or the use of nondeterministic |query plans. The detection is best-effort. """.stripMargin) .booleanConf .createWithDefault(false) final object MergeMaterializeSource { // See value explanations in the doc below. final val NONE = "none" final val ALL = "all" final val AUTO = "auto" final val list = Set(NONE, ALL, AUTO) } val MERGE_MATERIALIZE_SOURCE = buildConf("merge.materializeSource") .internal() .doc("When to materialize the source plan during MERGE execution. " + "The value 'none' means source will never be materialized. " + "The value 'all' means source will always be materialized. " + "The value 'auto' means sources will not be materialized when they are certain to be " + "deterministic." ) .stringConf .transform(_.toLowerCase(Locale.ROOT)) .checkValues(MergeMaterializeSource.list) .createWithDefault(MergeMaterializeSource.AUTO) val MERGE_FORCE_SOURCE_MATERIALIZATION_WITH_UNREADABLE_FILES = buildConf("merge.forceSourceMaterializationWithUnreadableFilesConfig") .internal() .doc( s""" |When set to true, merge command will force source materialization if Spark configs |${SQLConf.IGNORE_CORRUPT_FILES.key}, ${SQLConf.IGNORE_MISSING_FILES.key} or |file source read options ${FileSourceOptions.IGNORE_CORRUPT_FILES} |${FileSourceOptions.IGNORE_MISSING_FILES} are enabled on the source. |This is done so to prevent irrecoverable data loss or unexpected results. |""".stripMargin) .booleanConf .createWithDefault(true) val MERGE_MATERIALIZE_CACHED_SOURCE = buildConf("merge.materializeCachedSource") .internal() .doc( """ |When enabled, materialize the source in MERGE if it is cached (e.g. via df.cache()). This |prevents incorrect results due to query caching not pinning the version of cached Delta |tables. |""".stripMargin) .booleanConf .createWithDefault(true) val MERGE_FAIL_SOURCE_CACHED_AFTER_MATERIALIZATION = buildConf("merge.failSourceCachedAfterMaterialization") .internal() .doc( """ |Enables a check that fails the MERGE operation if the source was cached (using |df.cache()) after the source materialization phase. Query caching doesn't pin the version |of Delta tables and we should materialize cached source plans. In rare cases, the source |might get cached after the decision to materialize, which could lead to incorrect results |if we let the operation succeed. |""".stripMargin) .booleanConf .createWithDefault(true) val MERGE_MATERIALIZE_SOURCE_RDD_STORAGE_LEVEL = buildConf("merge.materializeSource.rddStorageLevel") .internal() .doc("What StorageLevel to use to persist the source RDD. Note: will always use disk.") .stringConf .transform(_.toUpperCase(Locale.ROOT)) .checkValue( v => try { StorageLevel.fromString(v).isInstanceOf[StorageLevel] } catch { case _: IllegalArgumentException => true }, """"spark.databricks.delta.merge.materializeSource.rddStorageLevel" """ + "must be a valid StorageLevel") .createWithDefault("DISK_ONLY") val MERGE_MATERIALIZE_SOURCE_RDD_STORAGE_LEVEL_FIRST_RETRY = buildConf("merge.materializeSource.rddStorageLevelFirstRetry") .internal() .doc("What StorageLevel to use to persist the source RDD when MERGE is retried the first" + "time. Note: will always use disk.") .stringConf .transform(_.toUpperCase(Locale.ROOT)) .checkValue( v => try { StorageLevel.fromString(v).isInstanceOf[StorageLevel] } catch { case _: IllegalArgumentException => true }, """"spark.databricks.delta.merge.materializeSource.rddStorageLevelFirstRetry" """ + "must be a valid StorageLevel") .createWithDefault("DISK_ONLY_2") val MERGE_MATERIALIZE_SOURCE_RDD_STORAGE_LEVEL_RETRY = buildConf("merge.materializeSource.rddStorageLevelRetry") .internal() .doc("What StorageLevel to use to persist the source RDD when MERGE is retried after the " + "first retry. The storage level to use for the first retry can be configured using" + """"spark.databricks.delta.merge.materializeSource.rddStorageLevelFirstRetry" """ + "Note: will always use disk.") .stringConf .transform(_.toUpperCase(Locale.ROOT)) .checkValue( v => try { StorageLevel.fromString(v).isInstanceOf[StorageLevel] } catch { case _: IllegalArgumentException => true }, """"spark.databricks.delta.merge.materializeSource.rddStorageLevelRetry" """ + "must be a valid StorageLevel") .createWithDefault("DISK_ONLY_3") val MERGE_MATERIALIZE_SOURCE_MAX_ATTEMPTS = buildStaticConf("merge.materializeSource.maxAttempts") .doc("How many times to try MERGE in case of lost RDD materialized source data") .intConf .createWithDefault(4) val DELTA_LAST_COMMIT_VERSION_IN_SESSION = buildConf("lastCommitVersionInSession") .doc("The version of the last commit made in the SparkSession for any table.") .longConf .checkValue(_ >= 0, "the version must be >= 0") .createOptional val ALLOW_UNENFORCED_NOT_NULL_CONSTRAINTS = buildConf("constraints.allowUnenforcedNotNull.enabled") .internal() .doc("If enabled, NOT NULL constraints within array and map types will be permitted in " + "Delta table creation, even though Delta can't enforce them.") .booleanConf .createWithDefault(false) val CHECKPOINT_SCHEMA_WRITE_THRESHOLD_LENGTH = buildConf("checkpointSchema.writeThresholdLength") .internal() .doc("Checkpoint schema larger than this threshold won't be written to the last checkpoint" + " file") .intConf .createWithDefault(20000) val LAST_CHECKPOINT_CHECKSUM_ENABLED = buildConf("lastCheckpoint.checksum.enabled") .internal() .doc("Controls whether to write the checksum while writing the LAST_CHECKPOINT file and" + " whether to validate it while reading the LAST_CHECKPOINT file") .booleanConf .createWithDefault(true) val SUPPRESS_OPTIONAL_LAST_CHECKPOINT_FIELDS = buildConf("lastCheckpoint.suppressOptionalFields") .internal() .doc("If set, the LAST_CHECKPOINT file will contain only version, size, and parts fields. " + "For compatibility with broken third-party connectors that choke on unrecognized fields.") .booleanConf .createWithDefault(false) val DELTA_CHECKPOINT_PART_SIZE = buildConf("checkpoint.partSize") .internal() .doc("The limit at which we will start parallelizing the checkpoint. We will attempt to " + "write a maximum of this many actions per checkpoint file.") .longConf .checkValue(_ > 0, "partSize has to be positive") .createOptional ///////////////////////////////// // File Materialization Tracker ///////////////////////////////// val DELTA_COMMAND_FILE_MATERIALIZATION_TRACKING_ENABLED = buildConf("command.fileMaterializationLimit.enabled") .internal() .doc( """ |When enabled, tracks the file metadata materialized on the driver and restricts the |number of files materialized on the driver to be within the global file |materialization limit. """.stripMargin ) .booleanConf .createWithDefault(true) val DELTA_COMMAND_FILE_MATERIALIZATION_LIMIT = buildStaticConf("command.fileMaterializationLimit.softMax") .internal() .doc( s""" |The soft limit for the total number of file metadata that can be materialized at once on |the driver. This config will take effect only when |${DELTA_COMMAND_FILE_MATERIALIZATION_TRACKING_ENABLED.key} is enabled. """.stripMargin ) .intConf .checkValue(_ >= 0, "'command.fileMaterializationLimit.softMax' must be positive") .createWithDefault(10000000) //////////////////////// // BACKFILL //////////////////////// val DELTA_ROW_TRACKING_BACKFILL_ENABLED = buildConf("rowTracking.backfill.enabled") .internal() .doc("Whether Row Tracking backfill can be performed.") .booleanConf .createWithDefault(true) val DELTA_ROW_TRACKING_BACKFILL_MAX_NUM_FILES_PER_COMMIT = buildConf("rowTracking.backfill.maxNumFiles") .internal() .doc("The maximum number of files to include in a single commit when running " + "RowTrackingBackfillCommand. The default maximum aims to keep every " + "delta log entry below 100mb.") .intConf .checkValue(_ > 0, "'backfill.maxNumFiles' must be positive.") .createWithDefault(22000) val DELTA_BACKFILL_MAX_NUM_FILES_PER_COMMIT = buildConf("backfill.maxNumFiles") .internal() .doc("The maximum number of files to include in a single commit when running " + "BackfillCommand. The default maximum aims to keep every " + "delta log entry below 100mb.") .fallbackConf(DELTA_ROW_TRACKING_BACKFILL_MAX_NUM_FILES_PER_COMMIT) val DELTA_BACKFILL_MAX_NUM_FILES_FACTOR = buildConf("backfill.maxNumFilesFactor") .internal() .doc( """The factor used to compute the maximum number of files to backfill. |The maximum number of files to compute in backfill is computed as |number of files in table * factor.""".stripMargin) .doubleConf .checkValue(_ > 0, "'backfill.maxNumFilesFactor' must be greater than zero.") .createWithDefault(3) val DELTA_ROW_TRACKING_IGNORE_SUSPENSION = buildConf("rowTracking.ignoreSuspension") .internal() .doc( """Controls whether to ignore `delta.rowTrackingSuspended` property. |This is a testing only config.""".stripMargin) .booleanConf .createWithDefault(false) //////////////////////////////////// // Checkpoint V2 Specific Configs //////////////////////////////////// val CHECKPOINT_V2_DRIVER_THREADPOOL_PARALLELISM = buildStaticConf("checkpointV2.threadpool.size") .doc("The size of the threadpool for fetching CheckpointMetadata and SidecarFiles from a" + " checkpoint.") .internal() .intConf .createWithDefault(32) val CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT = buildConf("checkpointV2.topLevelFileFormat") .internal() .doc( """ |The file format to use for the top level checkpoint file in V2 Checkpoints. | This can be set to either json or parquet. The appropriate format will be | picked automatically if this config is not specified. |""".stripMargin) .stringConf .checkValues(Set("json", "parquet")) .createOptional // This is temporary conf to make sure v2 checkpoints are not used by anyone other than devs as // the feature is not fully ready. val EXPOSE_CHECKPOINT_V2_TABLE_FEATURE_FOR_TESTING = buildConf("checkpointV2.exposeTableFeatureForTesting") .internal() .doc( """ |This conf controls whether v2 checkpoints table feature is exposed or not. Note that | v2 checkpoints are in development and this should config should be used only for | testing/benchmarking. |""".stripMargin) .booleanConf .createWithDefault(false) val LAST_CHECKPOINT_NON_FILE_ACTIONS_THRESHOLD = buildConf("lastCheckpoint.nonFileActions.threshold") .internal() .doc(""" |Threshold for total number of non file-actions to store in the last_checkpoint | corresponding to the checkpoint v2. |""".stripMargin) .intConf .createWithDefault(30) val STATS_AS_STRUCT_IN_CHECKPOINT_FORCE_DISABLED = buildConf("statsAsStructInCheckpoint.forcedDisabled") .internal() .doc(""" |Force disables storing statistics as struct in the checkpoint. |Note that should only be used as a kill switch. |This functionality should normally be controlled using the delta config |'checkpoint.writeStatsAsStruct'. |""".stripMargin) .booleanConf .createOptional val LAST_CHECKPOINT_SIDECARS_THRESHOLD = buildConf("lastCheckpoint.sidecars.threshold") .internal() .doc(""" |Threshold for total number of sidecar files to store in the last_checkpoint | corresponding to the checkpoint v2. |""".stripMargin) .intConf .createWithDefault(30) val USE_CHECKPOINT_SCHEMA_FROM_CHECKPOINT_METADATA = buildConf("checkpointSchema.useFromCheckpointMetadata") .internal() .doc("If enabled, use checkpoint schema from checkpoint metadata file instead of reading it" + " from the checkpoint file") .booleanConf .createWithDefault(true) val DELTA_WRITE_CHECKSUM_ENABLED = buildConf("writeChecksumFile.enabled") .doc("Whether the checksum file can be written.") .booleanConf .createWithDefault(true) val DELTA_CHECKSUM_HISTOGRAM_FIELD_FOLLOWS_PROTOCOL = buildConf("writeChecksumFile.histogramFollowsProtocol") .internal() .doc("""When true, writes the file size histogram to CRC files using the Delta spec field |name "fileSizeHistogram". When false, uses the legacy Delta-Spark field name |"histogramOpt".""".stripMargin) .booleanConf .createWithDefault(true) private val FORCED_CHECKSUM_VALIDATION_INTERVAL_DEFAULT = 400 val FORCED_CHECKSUM_VALIDATION_INTERVAL = buildConf("versionChecksum.forcedValidationInterval") .internal() .doc("The number of commits since the last checkpoint at which we " + "should force validation of the version checksum. This is done before " + "a commit to block further writes in case of checksum mismatch." + "Set to -1 to disable, set to 0 to validate on every commit. " + "The validation will be skipped if the checkpoint was created " + "within the time gap specified by versionChecksum.forcedValidationMinTimeIntevalMinutes.") .intConf .createWithDefault(FORCED_CHECKSUM_VALIDATION_INTERVAL_DEFAULT) val FORCED_CHECKSUM_VALIDATION_MIN_TIME_INTERVAL_MINUTES = buildConf("versionChecksum.forcedValidationMinTimeIntevalMinutes") .internal() .doc("The minimum time gap in minutes between the checkpoint creation time and " + "current time for forced checksum validation. If the checkpoint was created " + "within this time gap, forced validation is skipped even if the number of " + "commits since the checkpoint exceeds the forcedValidationInterval threshold. " + "For fast moving tables, the checkpoint can lag much behind " + "versionChecksum.forcedValidationInterval. This helps us avoid slowing " + "them down. Set to 0 to disable this optimization.") .intConf .checkValue(_ >= 0, "'versionChecksum.forcedValidationMinTimeIntevalMinutes' must be non-negative.") .createWithDefault(12*60) // 12 hours val INCREMENTAL_COMMIT_ENABLED = buildConf("incremental.commit.enabled") .internal() .doc("If true, Delta will incrementally compute the content of the commit checksum " + "file, which avoids the full state reconstruction that would otherwise be required.") .booleanConf .createWithDefault(true) val DELTA_CHECKSUM_MISMATCH_IS_FATAL = buildConf("checksum.mismatch.fatal") .internal() .doc( """If true, throws a fatal error when the recreated Delta State doesn't |match committed checksum file. """) .booleanConf .createWithDefault(true) val INCREMENTAL_COMMIT_VERIFY = buildConf("incremental.commit.verify") .internal() .doc("If true, Delta commit will validate the commit checksum file content before and " + "after each incremental commit. Note that this requires two full state reconstructions.") .booleanConf .createWithDefault(false) // This config is effective only in unit tests. val INCREMENTAL_COMMIT_FORCE_VERIFY_IN_TESTS = buildConf("incremental.commit.forceVerifyInTests") .internal() .doc("If true, Delta commit will validate the commit checksum file content before and " + "after each incremental commit as part of Unit Tests. Note that this overrides any " + s"behaviour from ${INCREMENTAL_COMMIT_VERIFY.key} config.") .booleanConf .createWithDefault(true) val DELTA_WRITE_SET_TRANSACTIONS_IN_CRC = buildConf("setTransactionsInCrc.writeOnCommit") .internal() .doc("When enabled, each commit will incrementally compute and cache all SetTransaction" + " actions in the .crc file. Note that this only happens when incremental commits" + s" are enabled (${INCREMENTAL_COMMIT_ENABLED.key})") .booleanConf .createWithDefault(true) val DELTA_MAX_SET_TRANSACTIONS_IN_CRC = buildConf("setTransactionsInCrc.maxAllowed") .internal() .doc("Threshold of the number of SetTransaction actions below which this optimization" + " should be enabled") .longConf .createWithDefault(100) val DELTA_MAX_DOMAIN_METADATAS_IN_CRC = buildConf("domainMetadatasInCrc.maxAllowed") .internal() .doc("Threshold of the number of DomainMetadata actions below which this optimization" + " should be enabled") .longConf .createWithDefault(10) val DELTA_ALL_FILES_IN_CRC_THRESHOLD_FILES = buildConf("allFilesInCrc.thresholdNumFiles") .internal() .doc("Threshold of the number of AddFiles below which AddFiles will be added to CRC.") .intConf .createWithDefault(50) val DELTA_ALL_FILES_IN_CRC_ENABLED = buildConf("allFilesInCrc.enabled") .internal() .doc("When enabled, [[Snapshot.allFiles]] will be stored in the .crc file when the " + "length is less than the threshold specified by " + s"${DELTA_ALL_FILES_IN_CRC_THRESHOLD_FILES.key}. " + "Note that this config only takes effect when incremental commits are enabled " + s"(${INCREMENTAL_COMMIT_ENABLED.key})." ) .booleanConf .createWithDefault(true) val DELTA_ALL_FILES_IN_CRC_VERIFICATION_MODE_ENABLED = buildConf("allFilesInCrc.verificationMode.enabled") .internal() .doc(s"This will be effective only if ${DELTA_ALL_FILES_IN_CRC_ENABLED.key} is set. When" + " enabled, We will have additional verification of the incrementally computed state by" + " doing an actual state reconstruction on every commit.") .booleanConf .createWithDefault(false) val DELTA_ALL_FILES_IN_CRC_FORCE_VERIFICATION_MODE_FOR_NON_UTC_ENABLED = buildConf("allFilesInCrc.verificationMode.forceOnNonUTC.enabled") .internal() .doc(s"This will be effective only if " + s"${DELTA_ALL_FILES_IN_CRC_VERIFICATION_MODE_ENABLED.key} is not set. When enabled, we " + s"will force verification of the incrementally computed state by doing an actual state " + s"reconstruction on every commit for tables that are not using UTC timezone.") .booleanConf .createWithDefault(true) val DELTA_ALL_FILES_IN_CRC_THRESHOLD_INDEXED_COLS = buildConf("allFilesInCrc.thresholdIndexedCols") .internal() .doc("If the delta table is configured to collect stats on more columns than this" + " threshold, then disable storage of `[[Snapshot.allFiles]]` in the .crc file.") .intConf .createOptional val USE_PROTOCOL_AND_METADATA_FROM_CHECKSUM_ENABLED = buildConf("readProtocolAndMetadataFromChecksum.enabled") .internal() .doc("If enabled, delta log snapshot will read the protocol, metadata, and ICT " + "(if applicable) from the checksum file and use those to avoid a spark job over the " + "checkpoint for the two rows of protocol and metadata") .booleanConf .createWithDefault(true) val DELTA_CHECKSUM_DV_METRICS_ENABLED = buildConf("checksumDVMetrics.enabled") .internal() .doc(s"""When enabled, each delta transaction includes vector metrics in the checksum. |Only applies to tables that use Deletion Vectors.""" .stripMargin) .booleanConf .createWithDefault(true) val DELTA_DELETED_RECORD_COUNTS_HISTOGRAM_ENABLED = buildConf("checksumDeletedRecordCountsHistogramMetrics.enabled") .internal() .doc(s"""When enabled, each delta transaction includes in the checksum the deleted |record count distribution histogram for all the files. To enable this feature |${DELTA_CHECKSUM_DV_METRICS_ENABLED.key} needs to be enabled as well. Only |applies to tables that use Deletion Vectors.""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_CHECKPOINT_THROW_EXCEPTION_WHEN_FAILED = buildConf("checkpoint.exceptionThrowing.enabled") .internal() .doc("Throw an error if checkpoint is failed. This flag is intentionally used for " + "testing purpose to catch the checkpoint issues proactively. In production, we " + "should not set this flag to be true because successful commit should return " + "success to client regardless of the checkpoint result without throwing.") .booleanConf .createWithDefault(false) val DELTA_RESOLVE_MERGE_UPDATE_STRUCTS_BY_NAME = buildConf("resolveMergeUpdateStructsByName.enabled") .internal() .doc("Whether to resolve structs by name in UPDATE operations of UPDATE and MERGE INTO " + "commands. If disabled, Delta will revert to the legacy behavior of resolving by position.") .booleanConf .createWithDefault(true) val DELTA_TIME_TRAVEL_STRICT_TIMESTAMP_PARSING = buildConf("timeTravel.parsing.strict") .internal() .doc("Whether to require time travel timestamps to parse to a valid timestamp. If " + "disabled, Delta will revert to the legacy behavior of treating invalid timestamps as " + "equivalent to unix time 0 (1970-01-01 00:00:00).") .booleanConf .createWithDefault(true) val DELTA_STRICT_CHECK_DELTA_TABLE = buildConf("isDeltaTable.strictCheck") .internal() .doc(""" | When enabled, io.delta.tables.DeltaTable.isDeltaTable | should return false when the _delta_log directory doesn't | contain any transaction logs. |""".stripMargin) .booleanConf .createWithDefault(true) /** * Internal config to bypass the check that ensures a table doesn't contain any unsupported type * change when reading it. Meant as a mitigation in case the check incorrectly flags valid cases. */ val DELTA_TYPE_WIDENING_BYPASS_UNSUPPORTED_TYPE_CHANGE_CHECK = buildConf("typeWidening.bypassUnsupportedTypeChangeCheck") .internal() .doc(""" | Disables check that ensures a table doesn't contain any unsupported type change when | reading it. |""".stripMargin) .booleanConf .createWithDefault(false) val DELTA_ALLOW_TYPE_WIDENING_STREAMING_SOURCE = buildConf("typeWidening.allowTypeChangeStreamingDeltaSource") .doc("Accept incoming widening type changes when streaming from a Delta source.") .internal() .booleanConf .createWithDefault(true) object AllowAutomaticWideningMode extends Enumeration { val NEVER, SAME_FAMILY_TYPE, ALWAYS = Value def fromConf(conf: SQLConf): Value = withName(conf.getConf(DELTA_ALLOW_AUTOMATIC_WIDENING)) def default: Value = withName(DELTA_ALLOW_AUTOMATIC_WIDENING.defaultValueString) } val DELTA_ALLOW_AUTOMATIC_WIDENING = buildConf("typeWidening.allowAutomaticWidening") .doc("Controls the scope of enabled widening conversions in automatic schema widening " + "during schema evolution. This flag is guarded by the flag 'delta.enableTypeWidening'" + "All supported widenings are enabled with 'always' selected, which allows some " + "conversions between integer types and floating numbers. The value 'same_family_type' " + "was the historical behavior. 'never' allows no widenings.") .internal() .stringConf .transform(_.toUpperCase(Locale.ROOT)) .checkValues(AllowAutomaticWideningMode.values.map(_.toString)) .createWithDefault(AllowAutomaticWideningMode.ALWAYS.toString) val DELTA_TYPE_WIDENING_ENABLE_STREAMING_SCHEMA_TRACKING = buildConf("typeWidening.enableStreamingSchemaTracking") .doc("Whether to enable schema tracking when streaming from a Delta source that had a " + "widening type change applied. This allows blocking the stream on restart until the user " + "acknowledges the type change. When disabled, we will not initialize a schema tracking " + "log when first detecting a type change and will automatically accept the type change " + "instead.") .internal() .booleanConf .createWithDefault(false) val DELTA_TYPE_WIDENING_BYPASS_STREAMING_TYPE_CHANGE_CHECK = buildConf("typeWidening.bypassStreamingTypeChangeCheck") .doc("Controls the check performed when a type change is detected when streaming from a " + "Delta source. This check fails the streaming query in case a type change may impact the " + "semantics of the query and requests user intervention.") .internal() .booleanConf .createWithDefault(false) /** * Internal config to bypass check that prevents applying type changes that are not supported by * Iceberg when Uniform is enabled with Iceberg compatibility. */ val DELTA_TYPE_WIDENING_ALLOW_UNSUPPORTED_ICEBERG_TYPE_CHANGES = buildConf("typeWidening.allowUnsupportedIcebergTypeChanges") .internal() .doc( """ |By default, type changes that aren't supported by Iceberg are rejected when Uniform is |enabled with Iceberg compatibility. This config allows bypassing this restriction, but |reading the affected column with Iceberg clients will likely fail or behave erratically. |""".stripMargin) .booleanConf .createWithDefault(false) val DELTA_TYPE_WIDENING_REMOVE_SCHEMA_METADATA = buildConf("typeWidening.removeSchemaMetadata") .doc("When true, type widening metadata is removed from schemas that are surfaced outside " + "of Delta or used for schema comparisons") .internal() .booleanConf .createWithDefault(true) val DELTA_TYPE_WIDENING_ALLOW_INTEGRAL_DECIMAL_COERCION = buildConf("typeWidening.allowIntegralDecimalCoercion") .doc("When true, the type widening mode `AllTypeWideningToCommonWiderType` " + "should allow converting integral types to DecimalType and use decimal " + "coercion to find a common wider type with another DecimalType") .internal() .booleanConf .createWithDefault(true) val DELTA_IS_DELTA_TABLE_THROW_ON_ERROR = buildConf("isDeltaTable.throwOnError") .internal() .doc(""" | If checking the path provided to isDeltaTable (or findDeltaTableRoot) throws an exception, | then propagate this exception unless a _delta_log directory is found in an | accessible parent. | When disabled, such any exception leads to a result indicating that this is not a | Delta table. |""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_LEGACY_STORE_WRITER_OPTIONS_AS_PROPS = buildConf("legacy.storeOptionsAsProperties") .internal() .doc(""" |Delta was unintentionally storing options provided by the DataFrameWriter in the |saveAsTable method as table properties in the transaction log. This was unsupported |behavior (it was a bug), and it has security implications (accidental storage of |credentials). This flag prevents the storage of arbitrary options as table properties. |Set this flag to true to continue setting non-delta prefixed table properties through |table options. |""".stripMargin) .booleanConf .createWithDefault(false) val DELTA_VACUUM_RELATIVIZE_IGNORE_ERROR = buildConf("vacuum.relativize.ignoreError") .internal() .doc(""" |When enabled, the error when trying to relativize an absolute path when |vacuuming a delta table will be ignored. This usually happens when a table is |shallow cloned across FileSystems, such as across buckets or across cloud storage |systems. We do not recommend enabling this configuration in production or using it |with production datasets. |""".stripMargin) .booleanConf .createWithDefault(false) val DELTA_LEGACY_ALLOW_AMBIGUOUS_PATHS = buildConf("legacy.allowAmbiguousPathsInCreateTable") .internal() .doc(""" |Delta was unintentionally allowing CREATE TABLE queries with both 'delta.`path`' |and 'LOCATION path' clauses. In the new version, we will raise an error |for this case. This flag is added to allow users to skip the check. When it's set to |true and there are two paths in CREATE TABLE, the LOCATION path clause will be |ignored like what the old version does.""".stripMargin) .booleanConf .createWithDefault(false) val DELTA_WORK_AROUND_COLONS_IN_HADOOP_PATHS = buildConf("workAroundColonsInHadoopPaths.enabled") .internal() .doc(""" |When enabled, Delta will work around to allow colons in file paths. Normally Hadoop |does not support colons in file paths due to ambiguity, but some file systems like |S3 allow them. |""".stripMargin) .booleanConf .createWithDefault(true) val REPLACEWHERE_DATACOLUMNS_ENABLED = buildConf("replaceWhere.dataColumns.enabled") .doc( """ |When enabled, replaceWhere on arbitrary expression and arbitrary columns is enabled. |If disabled, it falls back to the old behavior |to replace on partition columns only.""".stripMargin) .booleanConf .createWithDefault(true) val REPLACEWHERE_METRICS_ENABLED = buildConf("replaceWhere.dataColumns.metrics.enabled") .internal() .doc( """ |When enabled, replaceWhere operations metrics on arbitrary expression and |arbitrary columns is enabled. This will not report row level metrics for partitioned |tables and tables with no stats.""".stripMargin) .booleanConf .createWithDefault(true) val REPLACEWHERE_CONSTRAINT_CHECK_ENABLED = buildConf("replaceWhere.constraintCheck.enabled") .doc( """ |When enabled, replaceWhere on arbitrary expression and arbitrary columns will |enforce the constraint check to replace the target table only when all the |rows in the source dataframe match that constraint. |If disabled, it will skip the constraint check and replace with all the rows |from the new dataframe.""".stripMargin) .booleanConf .createWithDefault(true) val REPLACEWHERE_DATACOLUMNS_WITH_CDF_ENABLED = buildConf("replaceWhere.dataColumnsWithCDF.enabled") .internal() .doc( """ |When enabled, replaceWhere on arbitrary expression and arbitrary columns will produce |results for CDF. If disabled, it will fall back to the old behavior.""".stripMargin) .booleanConf .createWithDefault(true) val OVERWRITE_REMOVE_METRICS_ENABLED = buildConf("insertOverwrite.removeMetrics.enabled") .internal() .doc( """ |When enabled, insert operations in overwrite mode will add metrics describing |removed data to table's history""".stripMargin) .booleanConf .createWithDefault(true) val LOG_SIZE_IN_MEMORY_THRESHOLD = buildConf("streaming.logSizeInMemoryThreshold") .internal() .doc( """ |The threshold of transaction log file size to read into the memory. When a file is larger |than this, we will read the log file in multiple passes rather than loading it into |the memory entirely.""".stripMargin) .longConf .createWithDefault(128L * 1024 * 1024) // 128MB val STREAMING_OFFSET_VALIDATION = buildConf("streaming.offsetValidation.enabled") .internal() .doc("Whether to validate whether delta streaming source generates a smaller offset and " + "moves backward.") .booleanConf .createWithDefault(true) val LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS = buildConf("loadFileSystemConfigsFromDataFrameOptions") .internal() .doc( """Whether to load file systems configs provided in DataFrameReader/Writer options when |calling `DataFrameReader.load/DataFrameWriter.save` using a Delta table path. |`DataFrameReader.table/DataFrameWriter.saveAsTable` doesn't support this.""".stripMargin) .booleanConf .createWithDefault(true) val CONVERT_EMPTY_TO_NULL_FOR_STRING_PARTITION_COL = buildConf("convertEmptyToNullForStringPartitionCol") .internal() .doc( """ |If true, always convert empty string to null for string partition columns before |constraint checks. |""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_DATASKIPPING_PARTITION_LIKE_FILTERS_ENABLED = buildConf("skipping.partitionLikeFilters.enabled") .doc( """ |If true, during data skipping, apply arbitrary data filters to "partition-like" |files (files with the same min-max values and no nulls on all referenced attributes). |""".stripMargin) .internal() .booleanConf .createWithDefault(false) val DELTA_DATASKIPPING_PARTITION_LIKE_FILTERS_THRESHOLD = buildConf("skipping.partitionLikeDataSkippingFilesThreshold") .internal() .doc("Partition-like data skipping on files with the same min-max values will only be" + "attempted when a Delta table has a number of files larger than this threshold.") .intConf .createWithDefault(100) val DELTA_DATASKIPPING_PARTITION_LIKE_FILTERS_CLUSTERING_COLUMNS_ONLY = buildConf("skipping.partitionLikeDataSkipping.limitToClusteringColumns") .internal() .doc("Limits partition-like data skipping to filters referencing only clustering columns" + "In general, clustering columns will be most likely to produce files with the same" + "min-max values, though this restriction might exclude filters on columns highly " + "correlated with the clustering columns.") .booleanConf .createWithDefault(true) val DELTA_DATASKIPPING_PARTITION_LIKE_FILTERS_ADDITIONAL_SUPPORTED_EXPRESSIONS = buildConf("skipping.partitionLikeDataSkipping.additionalSupportedExpressions") .internal() .doc("Comma-separated list of the canonical class names of additional expressions for which" + "partition-like data skipping can be safely applied.") .stringConf .createOptional val DELTA_DATASKIPPING_ISNULL_PUSHDOWN_EXPRS_ENABLED = buildConf("skipping.enhancedIsNullPushdownExprs.enabled") .doc("If true, support pushing down IsNull on additional null-intolerant expressions for " + "data skipping.") .internal() .booleanConf .createWithDefault(true) val DELTA_DATASKIPPING_ISNULL_PUSHDOWN_EXPRS_MAX_DEPTH = buildConf("skipping.enhancedIsNullPushdownExprs.maxDepth") .doc("The maximum number of times a complex expression like Or or And would have an IsNull " + "pushed down in it for data skipping.") .internal() .intConf .createWithDefault(8) /** * The below confs have a special prefix `spark.databricks.io` because this is the conf value * already used by Databricks' data skipping implementation. There's no benefit to making OSS * users, some of whom are Databricks customers, have to keep track of two different conf * values for the same data skipping parameter. */ val DATA_SKIPPING_STRING_PREFIX_LENGTH = SQLConf.buildConf("spark.databricks.io.skipping.stringPrefixLength") .internal() .doc("For string columns, how long prefix to store in the data skipping index.") .intConf .createWithDefault(32) val MDC_NUM_RANGE_IDS = SQLConf.buildConf("spark.databricks.io.skipping.mdc.rangeId.max") .internal() .doc("This controls the domain of rangeId values to be interleaved. The bigger, the better " + "granularity, but at the expense of performance (more data gets sampled).") .intConf .checkValue(_ > 1, "'spark.databricks.io.skipping.mdc.rangeId.max' must be greater than 1") .createWithDefault(1000) val MDC_ADD_NOISE = SQLConf.buildConf("spark.databricks.io.skipping.mdc.addNoise") .internal() .doc("Whether or not a random byte should be added as a suffix to the interleaved bits " + "when computing the Z-order values for MDC. This can help deal with skew, but may " + "have a negative impact on overall min/max skipping effectiveness.") .booleanConf .createWithDefault(true) val MDC_SORT_WITHIN_FILES = SQLConf.buildConf("spark.databricks.io.skipping.mdc.sortWithinFiles") .internal() .doc("If enabled, sort within files by the specified MDC curve. " + "This might improve row-group skipping and data compression, at " + "the cost of additional overhead for sorting.") .booleanConf .createWithDefault(false) val DELTA_OPTIMIZE_ZORDER_COL_STAT_CHECK = buildConf("optimize.zorder.checkStatsCollection.enabled") .internal() .doc(s"When enabled, we will check if the column we're actually collecting stats " + "on the columns we are z-ordering on.") .booleanConf .createWithDefault(true) val FAST_INTERLEAVE_BITS_ENABLED = buildConf("optimize.zorder.fastInterleaveBits.enabled") .internal() .doc("When true, a faster version of the bit interleaving algorithm is used.") .booleanConf .createWithDefault(false) val INTERNAL_UDF_OPTIMIZATION_ENABLED = buildConf("internalUdfOptimization.enabled") .internal() .doc( """If true, create udfs used by Delta internally from templates to reduce lock contention |caused by Scala Reflection. |""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_OPTIMIZE_CONDITIONAL_INCREMENT_METRIC_ENABLED = buildConf("optimize.conditionalIncrementMetric.enabled") .internal() .doc("Whether to enable optimization of ConditionalIncrementMetric expressions with " + "constant conditions.") .booleanConf .createWithDefault(true) val GENERATED_COLUMN_PARTITION_FILTER_OPTIMIZATION_ENABLED = buildConf("generatedColumn.partitionFilterOptimization.enabled") .internal() .doc( "Whether to extract partition filters automatically from data filters for a partition" + " generated column if possible") .booleanConf .createWithDefault(true) val GENERATED_COLUMN_ALLOW_NULLABLE = buildConf("generatedColumn.allowNullableIngest.enabled") .internal() .doc("When enabled this will allow tables with generated columns enabled to be able " + "to write data without providing values for a nullable column via DataFrame.write") .booleanConf .createWithDefault(true) object GeneratedColumnValidateOnWriteMode extends Enumeration { val OFF, LOG_ONLY, ASSERT = Value def fromConf(conf: SQLConf): Value = withName(conf.getConf(GENERATED_COLUMN_VALIDATE_ON_WRITE)) def default: Value = withName(GENERATED_COLUMN_VALIDATE_ON_WRITE.defaultValueString) } val GENERATED_COLUMN_VALIDATE_ON_WRITE = buildConf("generatedColumn.validateOnWrite.enabled") .internal() .doc("When enabled, validates generated column expressions during write operations to " + "protect against disallowed expressions.") .stringConf .transform(_.toUpperCase(Locale.ROOT)) .checkValues(GeneratedColumnValidateOnWriteMode.values.map(_.toString)) .createWithDefault(GeneratedColumnValidateOnWriteMode.LOG_ONLY.toString) object ValidateCheckConstraintsMode extends Enumeration { val OFF, LOG_ONLY, ASSERT = Value def fromConf(conf: SQLConf): Value = withName(conf.getConf(VALIDATE_CHECK_CONSTRAINTS)) def default: Value = withName(VALIDATE_CHECK_CONSTRAINTS.defaultValueString) } val VALIDATE_CHECK_CONSTRAINTS = buildConf("checkConstraints.validation.enabled") .internal() .doc("When enabled, validates check constraints expressions during both creation and write" + " paths to protect against disallowed expressions.") .stringConf .transform(_.toUpperCase(Locale.ROOT)) .checkValues(ValidateCheckConstraintsMode.values.map(_.toString)) .createWithDefault(ValidateCheckConstraintsMode.LOG_ONLY.toString) val DELTA_CONVERT_ICEBERG_ENABLED = buildConf("convert.iceberg.enabled") .internal() .doc("If enabled, Iceberg tables can be converted into a Delta table.") .booleanConf .createWithDefault(true) val DELTA_CONVERT_ICEBERG_PARTITION_EVOLUTION_ENABLED = buildConf("convert.iceberg.partitionEvolution.enabled") .doc("If enabled, support conversion of iceberg tables experienced partition evolution.") .booleanConf .createWithDefault(false) val DELTA_CONVERT_ICEBERG_BUCKET_PARTITION_ENABLED = buildConf("convert.iceberg.bucketPartition.enabled") .doc("If enabled, convert iceberg table with bucket partition to unpartitioned delta table.") .internal() .booleanConf .createWithDefault(true) val DELTA_CONVERT_ICEBERG_CAST_TIME_TYPE = { buildConf("convert.iceberg.castTimeType") .internal() .doc("Cast Iceberg TIME type to Spark Long when converting to Delta") .booleanConf .createWithDefault(false) } final object NonDeterministicPredicateWidening { final val OFF = "off" final val LOGGING = "logging" final val ON = "on" final val list = Set(OFF, LOGGING, ON) } val DELTA_CONFLICT_DETECTION_WIDEN_NONDETERMINISTIC_PREDICATES = buildConf("conflictDetection.partitionLevelConcurrency.widenNonDeterministicPredicates") .doc("Whether to widen non-deterministic predicates during partition-level concurrency. " + "Widening can lead to additional conflicts." + "When the value is 'off', non-deterministic predicates are not widened during conflict " + "resolution." + "The value 'logging' will log whether the widening of non-deterministic predicates lead " + "to additional conflicts. The conflict resolution is still done without widening. " + "When the value is 'on', non-deterministic predicates are widened during conflict " + "resolution.") .internal() .stringConf .transform(_.toLowerCase(Locale.ROOT)) .checkValues(NonDeterministicPredicateWidening.list) .createWithDefault(NonDeterministicPredicateWidening.ON) val DELTA_CONFLICT_DETECTION_ALLOW_REPLACE_TABLE_TO_REMOVE_NEW_DOMAIN_METADATA = buildConf("conflictDetection.allowReplaceTableToRemoveNewDomainMetadata") .doc("Whether to allow removing new domain metadatas from concurrent transactions during " + "conflict resolution for a REPLACE TABLE operation. Note that this flag applies only " + "to metadata domains where the table snapshot read by the REPLACE TABLE command did " + "not contain a domain metadata of the same domain.") .internal() .booleanConf .createWithDefault(true) val DELTA_UNIFORM_ICEBERG_SYNC_CONVERT_ENABLED = buildConf("uniform.iceberg.sync.convert.enabled") .doc("If enabled, iceberg conversion will be done synchronously. " + "This can cause slow down in Delta commits and should only be used " + "for debugging or in test suites.") .internal() .booleanConf .createWithDefault(false) val DELTA_UNIFORM_HUDI_SYNC_CONVERT_ENABLED = buildConf("uniform.hudi.sync.convert.enabled") .doc("If enabled, Hudi conversion will be done synchronously.") .internal() .booleanConf .createWithDefault(false) val DELTA_UNIFORM_ICEBERG_RETRY_TIMES = buildConf("uniform.iceberg.retry.times") .doc("The number of retries iceberg conversions should have in case " + "of failures") .internal() .intConf .createWithDefault(3) val DELTA_UNIFORM_ICEBERG_INCLUDE_BASE_CONVERTED_VERSION = buildConf("uniform.iceberg.include.base.converted.version") .doc("If true, include the base converted delta version as a tbl property in Iceberg " + "metadata to indicate the delta version that the conversion started from") .internal() .booleanConf .createWithDefault(true) val DELTA_OPTIMIZE_MIN_FILE_SIZE = buildConf("optimize.minFileSize") .internal() .doc( """Files which are smaller than this threshold (in bytes) will be grouped together | and rewritten as larger files by the OPTIMIZE command. |""".stripMargin) .longConf .checkValue(_ >= 0, "minFileSize has to be positive") .createWithDefault(1024 * 1024 * 1024) val DELTA_OPTIMIZE_MAX_FILE_SIZE = buildConf("optimize.maxFileSize") .internal() .doc("Target file size produced by the OPTIMIZE command.") .longConf .checkValue(_ >= 0, "maxFileSize has to be positive") .createWithDefault(1024 * 1024 * 1024) val DELTA_OPTIMIZE_MAX_THREADS = buildConf("optimize.maxThreads") .internal() .doc( """ |Maximum number of parallel jobs allowed in OPTIMIZE command. Increasing the maximum | parallel jobs allows the OPTIMIZE command to run faster, but increases the job | management on the Spark driver side. |""".stripMargin) .intConf .checkValue(_ > 0, "'optimize.maxThreads' must be positive.") .createWithDefault(15) val DELTA_OPTIMIZE_BATCH_SIZE = buildConf("optimize.batchSize") .internal() .doc( """ |The size of a batch within an OPTIMIZE JOB. After a batch is complete, its | progress will be committed to the transaction log, allowing for incremental | progress. |""".stripMargin) .bytesConf(ByteUnit.BYTE) .checkValue(_ > 0, "batchSize has to be positive") .createOptional val DELTA_OPTIMIZE_REPARTITION_ENABLED = buildConf("optimize.repartition.enabled") .internal() .doc("Use repartition(1) instead of coalesce(1) to merge small files. " + "coalesce(1) is executed with only one task, if there are many tiny files " + "within a bin (e.g. 1000 files of 50MB), it cannot be optimized with more executors. " + "repartition(1) incurs a shuffle stage, but the job can be distributed." ) .booleanConf .createWithDefault(false) val DELTA_ALTER_TABLE_CHANGE_COLUMN_CHECK_EXPRESSIONS = buildConf("alterTable.changeColumn.checkExpressions") .internal() .doc( """ |Given an ALTER TABLE command that changes columns, check if there are expressions used | in Check Constraints and Generated Columns that reference this column and thus will | be affected by this change. | |This is a safety switch - we should only turn this off when there is an issue with |expression checking logic that prevents a valid column change from going through. |""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_LIQUID_ALTER_COLUMN_AFTER_STATS_SCHEMA_CHECK = buildConf("liquid.alterColumnAfter.statsSchemaCheck") .internal() .doc( """ |When enabled, validates that clustering columns remain in the stats schema after | a user executes `ALTER TABLE ALTER COLUMN col1 AFTER col2`. The validation checks | that all clustering columns that were in the stats schema before the column reordering | remain in the stats schema after the operation. This ensures that clustering columns | continue to have statistics collected even if their position in the table schema | changes. When disabled, no validation is performed and stats collection may follow | position-based indexing rules (e.g., `dataSkippingNumIndexedCols`), potentially | causing clustering columns to lose stats collection if they move outside the indexed | range. """.stripMargin) .booleanConf .createWithDefault(true) val DELTA_CHANGE_COLUMN_CHECK_DEPENDENT_EXPRESSIONS_USE_V2 = buildConf("changeColumn.checkDependentExpressionsUseV2") .internal() .doc( """ |More accurate implementation of checker for altering/renaming/dropping columns |that might be referenced by constraints or generation rules. |It respects nested arrays and maps, unlike the V1 checker. | |This is a safety switch - we should only turn this off when there is an issue with |expression checking logic that prevents a valid column change from going through. |""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_RENAME_COLUMN_ESCAPE_NAME = buildConf("changeColumn.renameColumnEscapeName") .internal() .doc( """ |Properly escape column names when renaming a column in the metadata. | |This is a safety switch - we should only set this to false if the fix introduces some |regression. |""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_ALTER_TABLE_DROP_COLUMN_ENABLED = buildConf("alterTable.dropColumn.enabled") .internal() .doc( """Whether to enable the drop column feature for Delta. |This is a safety switch - we should only turn this off when there is an issue. |""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_BYPASS_CHARVARCHAR_TO_STRING_FIX = buildConf("alterTable.bypassCharVarcharToStringFix") .internal() .doc( """Whether to bypass the fix for CHAR/VARCHAR to STRING type conversion in ALTER TABLE. |This is a safety switch - we should only set this to true if the fix introduces some |regression. |The fix in question strips CHAR/VARCHAR metadata from columns and converts |StringType to CHAR/VARCHAR Type temporarily during alter table column commands. |""".stripMargin) .booleanConf .createWithDefault(false) val DELTA_CDF_ALLOW_OUT_OF_RANGE_TIMESTAMP = { buildConf("changeDataFeed.timestampOutOfRange.enabled") .doc( """When enabled, Change Data Feed queries with starting and ending timestamps | exceeding the newest delta commit timestamp will not error out. For starting timestamp | out of range we will return an empty DataFrame, for ending timestamps out of range we | will consider the latest Delta version as the ending version.""".stripMargin) .booleanConf .createWithDefault(false) } val DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_COLUMN_MAPPING_SCHEMA_CHANGES = buildConf("streaming.unsafeReadOnIncompatibleColumnMappingSchemaChanges.enabled") .doc( "Streaming read on Delta table with column mapping schema operations " + "(e.g. rename or drop column) is currently blocked due to potential data loss and " + "schema confusion. However, existing users may use this flag to force unblock " + "if they'd like to take the risk.") .internal() .booleanConf .createWithDefault(false) val DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_SCHEMA_CHANGES_DURING_STREAM_START = buildConf("streaming.unsafeReadOnIncompatibleSchemaChangesDuringStreamStart.enabled") .doc( """A legacy config to disable schema read-compatibility check on the start version schema |when starting a streaming query. The config is added to allow legacy problematic queries |disabling the check to keep running if users accept the potential risks of incompatible |schema reading.""".stripMargin) .internal() .booleanConf .createWithDefault(false) val DELTA_STREAMING_UNSAFE_READ_ON_PARTITION_COLUMN_CHANGE = buildConf("streaming.unsafeReadOnPartitionColumnChanges.enabled") .doc( "Streaming read on Delta table with partition column overwrite " + "(e.g. changing partition column) is currently blocked due to potential data loss. " + "However, existing users may use this flag to force unblock " + "if they'd like to take the risk.") .internal() .booleanConf .createWithDefault(false) val DELTA_STREAMING_IGNORE_INTERNAL_METADATA_FOR_SCHEMA_CHANGE = buildConf("streaming.ignoreInternalMetadataForSchemaChange.enabled") .doc( "Whether to ignore internal metadata attached to struct fields when detecting schema " + "changes in Delta sources, e.g. identity columns internal high-water mark tracking.") .internal() .booleanConf .createWithDefault(true) val DELTA_STREAMING_ENABLE_SCHEMA_TRACKING = buildConf("streaming.schemaTracking.enabled") .doc( """If enabled, Delta streaming source can support non-additive schema evolution for |operations such as rename or drop column on column mapping enabled tables. |""".stripMargin) .internal() .booleanConf .createWithDefault(true) val DELTA_STREAMING_ENABLE_SCHEMA_TRACKING_MERGE_CONSECUTIVE_CHANGES = buildConf("streaming.schemaTracking.mergeConsecutiveSchemaChanges.enabled") .doc( "When enabled, schema tracking in Delta streaming would consider multiple consecutive " + "schema changes as one.") .internal() .booleanConf .createWithDefault(true) val DELTA_STREAMING_ALLOW_SCHEMA_LOCATION_OUTSIDE_CHECKPOINT_LOCATION = buildConf("streaming.allowSchemaLocationOutsideCheckpointLocation") .doc( "When enabled, Delta streaming can set a schema location outside of the " + "query's checkpoint location. This is not recommended.") .internal() .booleanConf .createWithDefault(false) val DELTA_STREAMING_SCHEMA_TRACKING_METADATA_PATH_CHECK_ENABLED = buildConf("streaming.schemaTracking.metadataPathCheck.enabled") .doc( "When enabled, Delta streaming with schema tracking will ensure the schema log entry " + "must match the source's unique checkpoint metadata location.") .internal() .booleanConf .createWithDefault(true) val DELTA_STREAM_UNSAFE_READ_ON_NULLABILITY_CHANGE = buildConf("streaming.unsafeReadOnNullabilityChange.enabled") .doc( """A legacy config to disable unsafe nullability check. The config is added to allow legacy |problematic queries disabling the check to keep running if users accept the potential |risks of incompatible schema reading.""".stripMargin) .internal() .booleanConf .createWithDefault(false) val DELTA_STREAMING_CREATE_DATAFRAME_DROP_NULL_COLUMNS = buildConf("streaming.createDataFrame.dropNullColumns") .internal() .doc("Whether to drop columns with NullType in DeltaLog.createDataFrame.") .booleanConf .createWithDefault(false) val DELTA_CREATE_DATAFRAME_DROP_NULL_COLUMNS = buildConf("createDataFrame.dropNullColumns") .internal() .doc("Whether to drop columns with NullType in DeltaLog.createDataFrame.") .booleanConf .createWithDefault(true) val DELTA_STREAMING_SINK_ALLOW_IMPLICIT_CASTS = buildConf("streaming.sink.allowImplicitCasts") .internal() .doc( """Whether to accept writing data to a Delta streaming sink when the data type doesn't |match the type in the underlying Delta table. When true, data is cast to the expected |type before the write. When false, the write fails. |The casting behavior is governed by 'spark.sql.storeAssignmentPolicy'. |""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_STREAMING_SINK_IMPLICIT_CAST_FOR_TYPE_MISMATCH_ONLY = buildConf("streaming.sink.implicitCastForTypeMismatchOnly") .internal() .doc( """Controls when an implicit cast is added when writing data to a Delta table using |streaming. |When true, a cast is added only when there is a type mismatch between a column or |nested field in the data and table schema. |When false, missing, extra or reordered columns or nested fields also trigger adding an |implicit cast. |Only takes effect when implicit casting is enabled in streaming writes to a Delta table |via `spark.databricks.delta.streaming.sink.allowImplicitCasts`. |""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_STREAMING_SINK_IMPLICIT_CAST_ESCAPE_COLUMN_NAMES = buildConf("streaming.sink.implicitCastEscapeColumnNames") .internal() .doc( """ |When true, the code paths handling implicit casting in streaming will escape column names |to properly handle e.g. dots in column names. |This is a kill-switch and shouldn't be disabled unless necessary to mitigate an issue. |Only takes effect when implicit casting is enabled in streaming writes to a Delta table |via `spark.databricks.delta.streaming.sink.allowImplicitCasts`. |""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_CDF_UNSAFE_BATCH_READ_ON_INCOMPATIBLE_SCHEMA_CHANGES = buildConf("changeDataFeed.unsafeBatchReadOnIncompatibleSchemaChanges.enabled") .doc( "Reading change data in batch (e.g. using `table_changes()`) on Delta table with " + "column mapping schema operations is currently blocked due to potential data loss and " + "schema confusion. However, existing users may use this flag to force unblock " + "if they'd like to take the risk.") .internal() .booleanConf .createWithDefault(false) val DELTA_CDF_DEFAULT_SCHEMA_MODE_FOR_COLUMN_MAPPING_TABLE = buildConf("changeDataFeed.defaultSchemaModeForColumnMappingTable") .doc( """Reading batch CDF on column mapping enabled table requires schema mode to be set to |`endVersion` so the ending version's schema will be used. |Set this to `latest` to use the schema of the latest available table version, |or to `legacy` to fallback to the non column-mapping default behavior, in which |the time travel option can be used to select the version of the schema.""".stripMargin) .internal() .stringConf .createWithDefault("endVersion") val DELTA_CDF_ALLOW_TIME_TRAVEL_OPTIONS = buildConf("changeDataFeed.allowTimeTravelOptionsForSchema") .doc( s"""If allowed, user can specify time-travel reader options such as |'versionAsOf' or 'timestampAsOf' to specify the read schema while |reading change data feed.""".stripMargin) .internal() .booleanConf .createWithDefault(false) val DELTA_COLUMN_MAPPING_CHECK_MAX_COLUMN_ID = buildConf("columnMapping.checkMaxColumnId") .doc( s"""If enabled, check if delta.columnMapping.maxColumnId is correctly assigned at each |Delta transaction commit. |""".stripMargin) .internal() .booleanConf .createWithDefault(true) val DELTA_COLUMN_MAPPING_STRIP_METADATA = buildConf("columnMapping.stripMetadata") .doc( """ |Transactions might try to update the schema of a table with columns that contain |column mapping metadata, even when column mapping is not enabled. For example, this |can happen when transactions copy the schema from another table. When this setting is |enabled, we will strip the column mapping metadata from the schema before applying it. |Note that this config applies only when the existing schema of the table does not |contain any column mapping metadata. |""".stripMargin) .internal() .booleanConf .createWithDefault(true) val DELTA_COLUMN_MAPPING_DISALLOW_ENABLING_WHEN_METADATA_ALREADY_EXISTS = buildConf("columnMapping.disallowEnablingWhenColumnMappingMetadataAlreadyExists") .doc( """ |If Delta table already has column mapping metadata before the feature is enabled, it is |as a result of a corruption or a bug. Enabling column mapping in such a case can lead to |further corruption of the table and should be disallowed. |""".stripMargin) .internal() .booleanConf .createWithDefault(true) val DYNAMIC_PARTITION_OVERWRITE_ENABLED = buildConf("dynamicPartitionOverwrite.enabled") .doc("Whether to overwrite partitions dynamically when 'partitionOverwriteMode' is set to " + "'dynamic' in either the SQL conf, or a DataFrameWriter option. When this is disabled " + "'partitionOverwriteMode' will be ignored.") .internal() .booleanConf .createWithDefault(true) val ALLOW_ARBITRARY_TABLE_PROPERTIES = buildConf("allowArbitraryProperties.enabled") .doc( """Whether we allow arbitrary Delta table properties. When this is enabled, table properties |with the prefix 'delta.' are not checked for validity. Table property validity is based |on the current Delta version being used and feature support in that version. Arbitrary |properties without the 'delta.' prefix are always allowed regardless of this config. | |Please use with caution. When enabled, there will be no warning when unsupported table |properties for the Delta version being used are set, or when properties are set |incorrectly (for example, misspelled).""".stripMargin ) .internal() .booleanConf .createWithDefault(false) val TABLE_BUILDER_FORCE_TABLEPROPERTY_LOWERCASE = buildConf("deltaTableBuilder.forceTablePropertyLowerCase.enabled") .internal() .doc( """Whether the keys of table properties should be set to lower case. | Turn on this flag if you want keys of table properties not starting with delta | to be backward compatible when the table is created via DeltaTableBuilder | Please note that if you set this to true, the lower case of the | key will be used for non delta prefix table properties. |""".stripMargin) .booleanConf .createWithDefault(false) val DELTA_REQUIRED_SPARK_CONFS_CHECK = buildConf("requiredSparkConfsCheck.enabled") .doc("Whether to verify SparkSession is initialized with required configurations.") .internal() .booleanConf .createWithDefault(true) val RESTORE_TABLE_PROTOCOL_DOWNGRADE_ALLOWED = buildConf("restore.protocolDowngradeAllowed") .doc(""" | Whether a table RESTORE or CLONE operation may downgrade the protocol of the table. | Note that depending on the protocol and the enabled table features, downgrading the | protocol may break snapshot reconstruction and make the table unreadable. Protocol | downgrades may also make the history unreadable.""".stripMargin) .booleanConf .createWithDefault(false) val DELTA_CLONE_REPLACE_ENABLED = buildConf("clone.replaceEnabled") .internal() .doc("If enabled, the table will be replaced when cloning over an existing Delta table.") .booleanConf .createWithDefault(true) val DELTA_OPTIMIZE_METADATA_QUERY_ENABLED = buildConf("optimizeMetadataQuery.enabled") .internal() .doc("Whether we can use the metadata in the DeltaLog to" + " optimize queries that can be run purely on metadata.") .booleanConf .createWithDefault(true) val DELTA_SKIP_RECORDING_EMPTY_COMMITS = buildConf("skipRecordingEmptyCommits") .internal() .doc( """ | Whether to skip recording an empty commit in the Delta Log. This only works when table | is using SnapshotIsolation or Serializable Isolation Mode. |""".stripMargin) .booleanConf .createWithDefault(true) val REPLACE_TABLE_PROTOCOL_DOWNGRADE_ALLOWED = buildConf("replace.protocolDowngradeAllowed") .internal() .doc(""" | Whether a REPLACE operation may downgrade the protocol of the table. | Note that depending on the protocol and the enabled table features, downgrading the | protocol may break snapshot reconstruction and make the table unreadable. Protocol | downgrades may also make the history unreadable.""".stripMargin) .booleanConf .createWithDefault(false) ////////////////// // Idempotent DML ////////////////// val DELTA_IDEMPOTENT_DML_TXN_APP_ID = buildConf("write.txnAppId") .internal() .doc(""" |The application ID under which this write will be committed. | If specified, spark.databricks.delta.write.txnVersion also needs to | be set. |""".stripMargin) .stringConf .createOptional val DELTA_IDEMPOTENT_DML_TXN_VERSION = buildConf("write.txnVersion") .internal() .doc(""" |The user-defined version under which this write will be committed. | If specified, spark.databricks.delta.write.txnAppId also needs to | be set. To ensure idempotency, txnVersions across different writes | need to be monotonically increasing. |""".stripMargin) .longConf .createOptional val DELTA_IDEMPOTENT_DML_AUTO_RESET_ENABLED = buildConf("write.txnVersion.autoReset.enabled") .internal() .doc(""" |If true, will automatically reset spark.databricks.delta.write.txnVersion |after every write. This is false by default. |""".stripMargin) .booleanConf .createWithDefault(false) val DELTA_OPTIMIZE_MAX_DELETED_ROWS_RATIO = buildConf("optimize.maxDeletedRowsRatio") .internal() .doc("Files with a ratio of deleted rows to the total rows larger than this threshold " + "will be rewritten by the OPTIMIZE command.") .doubleConf .checkValue(_ >= 0, "maxDeletedRowsRatio must be in range [0.0, 1.0]") .checkValue(_ <= 1, "maxDeletedRowsRatio must be in range [0.0, 1.0]") .createWithDefault(0.05d) val DELTA_TABLE_PROPERTY_CONSTRAINTS_CHECK_ENABLED = buildConf("tablePropertyConstraintsCheck.enabled") .internal() .doc( """Check that all table-properties satisfy validity constraints. |Only change this for testing!""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_DUPLICATE_ACTION_CHECK_ENABLED = buildConf("duplicateActionCheck.enabled") .internal() .doc(""" |Verify only one action is specified for each file path in one commit. |""".stripMargin) .booleanConf .createWithDefault(true) val DELETE_USE_PERSISTENT_DELETION_VECTORS = buildConf("delete.deletionVectors.persistent") .internal() .doc("Enable persistent Deletion Vectors in the Delete command.") .booleanConf .createWithDefault(true) val MERGE_USE_PERSISTENT_DELETION_VECTORS = buildConf("merge.deletionVectors.persistent") .internal() .doc("Enable persistent Deletion Vectors in Merge command.") .booleanConf .createWithDefault(true) val UPDATE_USE_PERSISTENT_DELETION_VECTORS = buildConf("update.deletionVectors.persistent") .internal() .doc("Enable persistent Deletion Vectors in the Update command.") .booleanConf .createWithDefault(true) val DELETION_VECTOR_PACKING_TARGET_SIZE = buildConf("deletionVectors.packing.targetSize") .internal() .doc("Controls the target file deletion vector file size when packing multiple" + "deletion vectors in a single file.") .bytesConf(ByteUnit.BYTE) /** * A [[DeletionVectorDescriptor]] stores an offset as a 32-bit integer into the file where the * deletion vector is stored. There is a hard limit of ~2.1GB for this file before the offset * integer overflows. Since we do bin packing with estimates, we set a lower internal * limit to be safe. */ .checkValue(_ >= 0, "deletionVectors.packing.targetSize must be non-negative") .checkValue(_ < 3L * 1024L * 1024L * 1024L / 2L, "deletionVectors.packing.targetSize must be less than 1.5GB") .createWithDefault(2L * 1024L * 1024L) val TIGHT_BOUND_COLUMN_ON_FILE_INIT_DISABLED = buildConf("deletionVectors.disableTightBoundOnFileCreationForDevOnly") .internal() .doc("""Controls whether we generate a tightBounds column in statistics on file creation. |The tightBounds column annotates whether the statistics of the file are tight or wide. |This flag is only used for testing purposes. """.stripMargin) .booleanConf .createWithDefault(false) val DELETION_VECTORS_USE_METADATA_ROW_INDEX = buildConf("deletionVectors.useMetadataRowIndex") .internal() .doc( """Controls whether we use the Parquet reader generated row_index column for | filtering deleted rows with deletion vectors. When enabled, it allows | predicate pushdown and file splitting in scans.""".stripMargin) .booleanConf .createWithDefault(true) val WRITE_DATA_FILES_TO_SUBDIR = buildConf("write.dataFilesToSubdir") .internal() .doc("Delta will write all data files to subdir 'data/' under table dir if enabled") .booleanConf .createWithDefault(false) val DELETION_VECTORS_COMMIT_CHECK_ENABLED = buildConf("deletionVectors.skipCommitCheck") .internal() .doc( """Check the table-property and verify that deletion vectors may be added |to this table. |Only change this for testing!""".stripMargin) .booleanConf .createWithDefault(true) val REUSE_COLUMN_MAPPING_METADATA_DURING_OVERWRITE = buildConf("columnMapping.reuseColumnMetadataDuringOverwrite") .internal() .doc( """ |If enabled, when a column mapping table is overwritten, the new schema will reuse as many |old schema's column mapping metadata (field id and physical name) as possible. |This allows the analyzed schema from prior to the overwrite to be still read-compatible |with the data post the overwrite, enabling better user experience when, for example, |the column mapping table is being continuously scanned in a streaming query, the analyzed |table schema will still be readable after the table is overwritten. |""".stripMargin) .booleanConf .createWithDefault(true) val REUSE_COLUMN_METADATA_DURING_REPLACE_TABLE = buildConf("columnMapping.reuseColumnMetadataDuringReplace") .internal() .doc( """ |If enabled, when a column mapping table is replaced, the new schema will reuse as many |old schema's column mapping metadata (field id and physical name) as possible. |""".stripMargin) .booleanConf .createWithDefault(true) val ALLOW_COLUMN_MAPPING_REMOVAL = buildConf("columnMapping.allowRemoval") .internal() .doc( """ |If enabled, allow the column mapping to be removed from a table. |""".stripMargin) .booleanConf .createWithDefault(true) val DELTALOG_MINOR_COMPACTION_USE_FOR_READS = buildConf("deltaLog.minorCompaction.useForReads") .doc("If true, minor compacted delta log files will be used for creating Snapshots") .internal() .booleanConf .createWithDefault(true) val ICEBERG_MAX_COMMITS_TO_CONVERT = buildConf("iceberg.maxPendingCommits") .doc(""" |The maximum number of pending Delta commits to convert to Iceberg incrementally. |If the table hasn't been converted to Iceberg in longer than this number of commits, |we start from scratch, replacing the previously converted Iceberg table contents. |""".stripMargin) .intConf .createWithDefault(100) val HUDI_MAX_COMMITS_TO_CONVERT = buildConf("hudi.maxPendingCommits") .doc(""" |The maximum number of pending Delta commits to convert to Hudi incrementally. |If the table hasn't been converted to Hudi in longer than this number of commits, |we start from scratch, replacing the previously converted Hudi table contents. |""".stripMargin) .intConf .createWithDefault(100) val ICEBERG_MAX_ACTIONS_TO_CONVERT = buildConf("iceberg.maxPendingActions") .doc(""" |[Deprecated] |The maximum number of pending Delta actions to convert to Iceberg incrementally. |If there are more than this number of outstanding actions, chunk them into separate |Iceberg commits. |""".stripMargin) .intConf .createWithDefault(100 * 1000) val UPDATE_AND_MERGE_CASTING_FOLLOWS_ANSI_ENABLED_FLAG = buildConf("updateAndMergeCastingFollowsAnsiEnabledFlag") .internal() .doc("""If false, casting behaviour in implicit casts in UPDATE and MERGE follows |'spark.sql.storeAssignmentPolicy'. If true, these casts follow 'ansi.enabled'. |""".stripMargin) .booleanConf .createWithDefault(false) val DELTA_USE_MULTI_THREADED_STATS_COLLECTION = buildConf("collectStats.useMultiThreadedStatsCollection") .internal() .doc("Whether to use multi-threaded statistics collection. If false, statistics will be " + "collected sequentially within each partition.") .booleanConf .createWithDefault(true) val DELTA_STATS_COLLECTION_NUM_FILES_PARTITION = buildConf("collectStats.numFilesPerPartition") .internal() .doc("Controls the number of files that should be within a RDD partition " + "during multi-threaded optimized statistics collection. A larger number will lead to " + "less parallelism, but can reduce scheduling overhead.") .intConf .checkValue(v => v >= 1, "Must be at least 1.") .createWithDefault(100) val DELTA_STATS_COLLECTION_FALLBACK_TO_INTERPRETED_PROJECTION = buildConf("collectStats.fallbackToInterpretedProjection") .internal() .doc("When enabled, the updateStats expression will use the standard code path" + " that falls back to an interpreted expression if codegen fails. This should" + " always be true. The config only exists to force the old behavior, which was" + " to always use codegen.") .booleanConf .createWithDefault(true) val DELTA_CONVERT_ICEBERG_STATS = buildConf("collectStats.convertIceberg") .internal() .doc("When enabled, attempts to convert Iceberg stats to Delta stats when cloning from " + "an Iceberg source.") .booleanConf .createWithDefault(true) val DELTA_CONVERT_ICEBERG_DECIMAL_STATS = buildConf("collectStats.convertIceberg.decimal") .internal() .doc("When enabled, attempts to convert Iceberg stats for DECIMAL to Delta stats" + "when cloning from an Iceberg source.") .booleanConf .createWithDefault(true) val DELTA_CONVERT_ICEBERG_DATE_STATS = buildConf("collectStats.convertIceberg.date") .internal() .doc("When enabled, attempts to convert Iceberg stats for DATE to Delta stats" + "when cloning from an Iceberg source.") .booleanConf .createWithDefault(true) val DELTA_CONVERT_ICEBERG_TIMESTAMP_STATS = buildConf("collectStats.convertIceberg.timestamp") .internal() .doc("When enabled, attempts to convert Iceberg stats for TIMESTAMP to Delta stats" + "when cloning from an Iceberg source.") .booleanConf .createWithDefault(true) /** * For iceberg clone, * When stats conversion from iceberg off, fallback to slow stats conversion enabled * When stats conversion from iceberg on, * fallback to slow stats conversion will not happen if partial stats conversion enabled * fallback only happens if partial stats conversion disabled and iceberg has partial stats * - either minValues or maxValues is missing */ val DELTA_CLONE_ICEBERG_ALLOW_PARTIAL_STATS = buildConf("clone.iceberg.allowPartialStats") .internal() .doc("If true, allow converting partial stats from iceberg stats " + "to delta stats during clone." ) .booleanConf .createWithDefault(true) ///////////////////// // Optimized Write ///////////////////// val DELTA_OPTIMIZE_WRITE_ENABLED = buildConf("optimizeWrite.enabled") .doc("Whether to optimize writes made into Delta tables from this session.") .booleanConf .createOptional val DELTA_OPTIMIZE_WRITE_SHUFFLE_BLOCKS = buildConf("optimizeWrite.numShuffleBlocks") .internal() .doc("Maximum number of shuffle blocks to target for the adaptive shuffle " + "in optimized writes.") .intConf .createWithDefault(50000000) val SKIP_REDIRECT_FEATURE = buildConf("skipRedirectFeature") .doc("True if skipping the redirect feature.") .internal() .booleanConf .createWithDefault(false) val ENABLE_TABLE_REDIRECT_FEATURE = buildConf("enableTableRedirectFeature") .doc("True if enabling the table redirect feature.") .internal() .booleanConf .createWithDefault(false) val DELTA_OPTIMIZE_WRITE_MAX_SHUFFLE_PARTITIONS = buildConf("optimizeWrite.maxShufflePartitions") .internal() .doc("Max number of output buckets (reducers) that can be used by optimized writes. This " + "can be thought of as: 'how many target partitions are we going to write to in our " + "table in one write'. This should not be larger than " + "spark.shuffle.minNumPartitionsToHighlyCompress. Otherwise, partition coalescing and " + "skew split may not work due to incomplete stats from HighlyCompressedMapStatus") .intConf .createWithDefault(2000) val DELTA_OPTIMIZE_WRITE_BIN_SIZE = buildConf("optimizeWrite.binSize") .internal() .doc("Bin size for the adaptive shuffle in optimized writes in megabytes.") .bytesConf(ByteUnit.MiB) .createWithDefault(512) val DELTA_OPTIMIZE_CLUSTERING_MIN_CUBE_SIZE = buildConf("optimize.clustering.mergeStrategy.minCubeSize.threshold") .internal() .doc( "Z-cube size at which new data will no longer be merged with it during incremental " + "OPTIMIZE." ) .longConf .checkValue(_ >= 0, "the threshold must be >= 0") .createWithDefault(100 * DELTA_OPTIMIZE_MAX_FILE_SIZE.defaultValue.get) val DELTA_OPTIMIZE_CLUSTERING_TARGET_CUBE_SIZE = buildConf("optimize.clustering.mergeStrategy.minCubeSize.targetCubeSize") .internal() .doc( "Target size of the Z-cubes we will create. This is not a hard max; we will continue " + "adding files to a Z-cube until their combined size exceeds this value. This value " + s"must be greater than or equal to ${DELTA_OPTIMIZE_CLUSTERING_MIN_CUBE_SIZE.key}. " ) .longConf .checkValue(_ >= 0, "the target must be >= 0") .createWithDefault((DELTA_OPTIMIZE_CLUSTERING_MIN_CUBE_SIZE.defaultValue.get * 1.5).toLong) ////////////////// // Clustered Table ////////////////// val DELTA_NUM_CLUSTERING_COLUMNS_LIMIT = buildStaticConf("clusteredTable.numClusteringColumnsLimit") .internal() .doc("""The maximum number of clustering columns allowed for a clustered table. """.stripMargin) .intConf .checkValue( _ > 0, "'clusteredTable.numClusteringColumnsLimit' must be positive." ) .createWithDefault(4) val DELTA_LOG_CACHE_SIZE = buildConf("delta.log.cacheSize") .internal() .doc("The maximum number of DeltaLog instances to cache in memory.") .longConf .createWithDefault(10000) val DELTA_LOG_CACHE_RETENTION_MINUTES = buildConf("delta.log.cacheRetentionMinutes") .internal() .doc("The rentention duration of DeltaLog instances in the cache") .timeConf(TimeUnit.MINUTES) .createWithDefault(60) ////////////////// // Delta Sharing ////////////////// val DELTA_SHARING_ENABLE_DELTA_FORMAT_BATCH = buildConf("spark.sql.delta.sharing.enableDeltaFormatBatch") .doc("Enable delta format sharing in case of issues.") .internal() .booleanConf .createWithDefault(true) val DELTA_SHARING_FORCE_DELTA_FORMAT = buildConf("spark.sql.delta.sharing.forceDeltaFormat") .doc("Force queries to use delta format when no responseFormat is specified.") .internal() .booleanConf .createWithDefault(false) val DELTA_SHARING_STREAMING_AUTO_RESOLVE_RESPONSE_FORMAT = buildConf("spark.sql.delta.sharing.streamingAutoResolveResponseFormat") .doc("When true, auto-resolve Delta Sharing streaming source format by calling getMetadata " + "on the table and using the server's responded format (parquet or delta). When false, " + "use the responseFormat option from the user.") .internal() .booleanConf .createWithDefault(false) /////////////////// // IDENTITY COLUMN /////////////////// val DELTA_IDENTITY_COLUMN_ENABLED = buildConf("identityColumn.enabled") .internal() .doc( """ | The umbrella config to turn on/off the IDENTITY column support. | If true, enable Delta IDENTITY column write support. If a table has an IDENTITY column, | it is not writable but still readable if this config is set to false. |""".stripMargin) .booleanConf .createWithDefault(true) val DELTA_IDENTITY_ALLOW_SYNC_IDENTITY_TO_LOWER_HIGH_WATER_MARK = buildConf("identityColumn.allowSyncIdentityToLowerHighWaterMark.enabled") .internal() .doc( """ | If true, the SYNC IDENTITY command can reduce the high water mark in a Delta IDENTITY | column. If false, the high water mark will only be updated if it | respects the column's specified start, step, and existing high watermark value. |""".stripMargin) .booleanConf .createWithDefault(false) /////////// // VARIANT /////////////////// val FORCE_USE_PREVIEW_VARIANT_FEATURE = buildConf("variant.forceUsePreviewTableFeature") .internal() .doc( """ | If true, creating new tables with variant columns only attaches the 'variantType-preview' | table feature. Attempting to operate on existing tables created with the stable feature | does not require that the preview table feature be present. |""".stripMargin) .booleanConf .createWithDefault(false) val FORCE_USE_PREVIEW_SHREDDING_FEATURE = buildConf("variantShredding.forceUsePreviewTableFeature") .internal() .doc( """ | If true, attach the 'variantShredding-preview' table feature when enabling shredding | on a table. When false, the 'variantShredding' feature is used instead.""".stripMargin) .booleanConf .createWithDefault(true) val COLLECT_VARIANT_DATA_SKIPPING_STATS = buildConf("variantShredding.collectVariantDataSkippingStats") .internal() .doc( """ | If enabled, Spark writes to Delta could collect data skipping stats for Variant | columns. Currently, this config is used to ensure that new checkpoints preserve previous | Variant stats.""" .stripMargin) .booleanConf .createWithDefault(true) /////////// // TESTING /////////// val DELTA_POST_COMMIT_HOOK_THROW_ON_ERROR = buildConf("postCommitHook.throwOnError") .internal() .doc("If true, post-commit hooks will by default throw an exception when they fail.") .booleanConf .createWithDefault(DeltaUtils.isTesting) val TEST_FILE_NAME_PREFIX = buildStaticConf("testOnly.dataFileNamePrefix") .internal() .doc("[TEST_ONLY]: The prefix to use for the names of all Parquet data files.") .stringConf .createWithDefault(if (DeltaUtils.isTesting) "test%file%prefix-" else "") val TEST_DV_NAME_PREFIX = buildStaticConf("testOnly.dvFileNamePrefix") .internal() .doc("[TEST_ONLY]: The prefix to use for the names of all Deletion Vector files.") .stringConf .createWithDefault(if (DeltaUtils.isTesting) "test%dv%prefix-" else "") /////////// // UTC TIMESTAMP PARTITION VALUES /////////////////// val UTC_TIMESTAMP_PARTITION_VALUES = buildConf("write.utcTimestampPartitionValues") .internal() .doc( """ | If true, write UTC normalized timestamp partition values to Delta Log. |""".stripMargin) .booleanConf .createWithDefault(true) ///////////////////////////////////// // NORMALIZE PARTITION VALUES ON READ //////////////////////////////////// val DELTA_NORMALIZE_PARTITION_VALUES_ON_READ = buildConf("normalizePartitionValuesOnRead") .internal() .doc( "When true, we will normalize partition values on read by parsing them " + "to their actual types for comparison instead of using raw strings. This helps prevent " + "issues with inconsistently formatted partition values. " + "UTC_TIMESTAMP_PARTITION_VALUES normalized timestamp partition values on write. However, " + "data written before this flag existed may not be normalized and needs to be normalized " + "on read." ) .booleanConf .createWithDefault(true) ////////////////// // CORRECTNESS ////////////////// val NUM_RECORDS_VALIDATION_ENABLED = buildConf("numRecordsValidation.enabled") .internal() .doc( """ |When enabled, adds a check to MERGE, UPDATE and DELETE that validates the number of |records that were added and removed. | |- For MERGE without INSERT statements it checks that the number of records does not | increase. |- For MERGE without DELETE statements it checks that the number of records does not | decrease. |- For UPDATE statements it checks that the number of records does not change. |- For DELETE statements it checks that the number of records does not increase. | |When disabled, we only log a warning. |""".stripMargin ) .booleanConf .createWithDefault(true) val COMMAND_INVARIANT_CHECKS_USE_UNRELIABLE = buildConf("commandInvariantChecksUseUnreliable") .internal() .doc("When enabled all DML commands will check and log invariants using unreliable metrics.") .booleanConf .createWithDefault(true) val COMMAND_INVARIANT_CHECKS_THROW = buildConf("commandInvariantChecksThrow") .internal() .doc( """When disabled all DML commands using reliable metrics just log a warning on command |invariant violation and proceed to commit. |When enabled, it's decided by a per-command flag.""".stripMargin) .booleanConf .createWithDefault(false) val ENABLE_SERVER_SIDE_PLANNING = buildConf("catalog.enableServerSidePlanning") .internal() .doc( """When enabled, DeltaCatalog will use server-side scan planning path |instead of normal table loading.""".stripMargin) .booleanConf .createWithDefault(false) /** * Controls which connector implementation to use for Delta table operations. * * Valid values: * - NONE: sparkV2 connector is disabled, always use sparkV1 connector (DeltaTableV2) - default * - AUTO: Automatically use sparkV2 connector (SparkTable) for Unity Catalog managed tables * in streaming queries and sparkV1 connector (DeltaTableV2) for all other tables * - STRICT: sparkV2 connector is strictly enforced, always use sparkV2 connector (SparkTable). * Intended for testing sparkV2 connector capabilities * * sparkV1 vs sparkV2 Connectors: * - sparkV1 Connector (DeltaTableV2): Legacy Delta connector with full read/write support, * uses DeltaLog for metadata management * - sparkV2 Connector (SparkTable): New kernel-based connector with read-only support, * uses Kernel's Table API for metadata management * * See [[org.apache.spark.sql.delta.DeltaV2Mode]] for the centralized logic that interprets * this configuration. */ val V2_ENABLE_MODE = buildConf("v2.enableMode") .doc( "Controls the Delta connector enable mode. " + "NONE (use v1 connector for all cases), AUTO (use v2 only for v2 " + "supported operations, default), STRICT (should ONLY be enabled for testing).") .stringConf .checkValues(Set("AUTO", "NONE", "STRICT")) .createWithDefault("AUTO") val DELTA_STREAMING_INITIAL_SNAPSHOT_MAX_FILES = buildConf("streaming.initialSnapshotMaxFiles") .internal() .doc("Maximum number of files allowed in initial snapshot for V2 streaming.") .intConf .createWithDefault(100000) } object DeltaSQLConf extends DeltaSQLConfBase ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSink.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.sources import java.util.concurrent.ConcurrentHashMap import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.Relocated._ import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.DeltaOperations.StreamingUpdate import org.apache.spark.sql.delta.actions.{FileAction, Metadata, Protocol, SetTransaction} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.{ImplicitMetadataOperation, SchemaMergingUtils, SchemaUtils} import org.apache.spark.sql.delta.sources.DeltaSQLConf.AllowAutomaticWideningMode import org.apache.spark.sql.delta.util.{Utils => DeltaUtils} import org.apache.hadoop.fs.Path // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.internal.MDC import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.{Alias, Expression} import org.apache.spark.sql.catalyst.types.DataTypeUtils import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, QuotingUtils} import org.apache.spark.sql.execution.{QueryExecution, SQLExecution} import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric import org.apache.spark.sql.execution.streaming.Sink import org.apache.spark.sql.streaming.OutputMode import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType} import org.apache.spark.util.Utils /** * A streaming sink that writes data into a Delta Table. */ case class DeltaSink( sqlContext: SQLContext, path: Path, partitionColumns: Seq[String], outputMode: OutputMode, options: DeltaOptions, catalogTable: Option[CatalogTable] = None) extends Sink with ImplicitMetadataOperation with UpdateExpressionsSupport with DeltaLogging { private lazy val deltaLog = DeltaUtils.getDeltaLogFromTableOrPath( sqlContext.sparkSession, catalogTable, path) private val sqlConf = sqlContext.sparkSession.sessionState.conf // This have to be lazy because queryId is a thread local property that is not available // when the Sink object is created. lazy val queryId = sqlContext.sparkContext.getLocalProperty(StreamExecution.QUERY_ID_KEY) override protected val canOverwriteSchema: Boolean = outputMode == OutputMode.Complete() && options.canOverwriteSchema override protected val canMergeSchema: Boolean = options.canMergeSchema case class PendingTxn(batchId: Long, optimisticTransaction: OptimisticTransaction, streamingUpdate: StreamingUpdate, newFiles: Seq[FileAction], deletedFiles: Seq[FileAction]) { def commit(): Unit = { val sc = sqlContext.sparkContext val metrics = Map[String, SQLMetric]( "numAddedFiles" -> createMetric(sc, "number of files added"), "numRemovedFiles" -> createMetric(sc, "number of files removed") ) metrics("numRemovedFiles").set(deletedFiles.size) metrics("numAddedFiles").set(newFiles.size) optimisticTransaction.registerSQLMetrics(sqlContext.sparkSession, metrics) val setTxn = SetTransaction(appId = queryId, version = batchId, lastUpdated = Some(deltaLog.clock.getTimeMillis())) :: Nil val (_, durationMs) = Utils.timeTakenMs { optimisticTransaction .commit(actions = setTxn ++ newFiles ++ deletedFiles , op = streamingUpdate) } logInfo( log"Committed transaction, batchId=${MDC(DeltaLogKeys.BATCH_ID, batchId)}, " + log"duration=${MDC(DeltaLogKeys.DURATION, durationMs)} ms, " + log"added ${MDC(DeltaLogKeys.NUM_FILES, newFiles.size.toLong)} files, " + log"removed ${MDC(DeltaLogKeys.NUM_FILES2, deletedFiles.size.toLong)} files.") val executionId = sc.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) SQLMetrics.postDriverMetricUpdates(sc, executionId, metrics.values.toSeq) } } override def addBatch(batchId: Long, data: DataFrame): Unit = { addBatchWithStatusImpl(batchId, data) } private def addBatchWithStatusImpl(batchId: Long, data: DataFrame): Boolean = { val txn = deltaLog.startTransaction(catalogTable) assert(queryId != null) if (SchemaUtils.nullTypeExistsRecursively(data.schema)) { throw DeltaErrors.streamWriteNullTypeException } IdentityColumn.blockExplicitIdentityColumnInsert( txn.snapshot.schema, data.queryExecution.analyzed) // If the batch reads the same Delta table as this sink is going to write to, then this // write has dependencies. Then make sure that this commit set hasDependencies to true // by injecting a read on the whole table. This needs to be done explicitly because // MicroBatchExecution has already enforced all the data skipping (by forcing the generation // of the executed plan) even before the transaction was started. val selfScan = data.queryExecution.analyzed.collectFirst { case DeltaTable(index) if index.deltaLog.isSameLogAs(txn.deltaLog) => true }.nonEmpty if (selfScan) { txn.readWholeTable() } val writeSchema = getWriteSchema(txn.protocol, txn.metadata, data.schema) // Streaming sinks can't blindly overwrite schema. See Schema Management design doc for details updateMetadata(data.sparkSession, txn, writeSchema, partitionColumns, Map.empty, outputMode == OutputMode.Complete(), rearrangeOnly = false) val currentVersion = txn.txnVersion(queryId) if (currentVersion >= batchId) { logInfo(log"Skipping already complete epoch ${MDC(DeltaLogKeys.BATCH_ID, batchId)}, " + log"in query ${MDC(DeltaLogKeys.QUERY_ID, queryId)}") return false } val deletedFiles = outputMode match { case o if o == OutputMode.Complete() => DeltaLog.assertRemovable(txn.snapshot) txn.filterFiles().map(_.remove) case _ => Nil } val (newFiles, writeFilesTimeMs) = Utils.timeTakenMs{ txn.writeFiles(castDataIfNeeded(data, writeSchema), Some(options)) } val totalSize = newFiles.map(_.getFileSize).sum val totalLogicalRecords = newFiles.map(_.numLogicalRecords.getOrElse(0L)).sum logInfo( log"Wrote ${MDC(DeltaLogKeys.NUM_FILES, newFiles.size.toLong)} files, " + log"with total size ${MDC(DeltaLogKeys.NUM_BYTES, totalSize)}, " + log"${MDC(DeltaLogKeys.NUM_RECORDS, totalLogicalRecords)} logical records, " + log"duration=${MDC(DeltaLogKeys.DURATION, writeFilesTimeMs)} ms.") val info = DeltaOperations.StreamingUpdate(outputMode, queryId, batchId, options.userMetadata ) val pendingTxn = PendingTxn(batchId, txn, info, newFiles, deletedFiles) pendingTxn.commit() return true } /** * Returns the schema to use to write data to this delta table. The write schema includes new * columns to add with schema evolution and reconciles types to match the table types when * possible or apply type widening if enabled. */ private def getWriteSchema( protocol: Protocol, metadata: Metadata, dataSchema: StructType): StructType = { if (!sqlConf.getConf(DeltaSQLConf.DELTA_STREAMING_SINK_ALLOW_IMPLICIT_CASTS)) return dataSchema if (canOverwriteSchema) return dataSchema val typeWideningMode = if (canMergeSchema && TypeWidening.isEnabled(protocol, metadata)) { TypeWideningMode.TypeEvolution( uniformIcebergCompatibleOnly = UniversalFormat.icebergEnabled(metadata), allowAutomaticWidening = AllowAutomaticWideningMode.fromConf(sqlConf)) } else { TypeWideningMode.NoTypeWidening } SchemaMergingUtils.mergeSchemas( metadata.schema, dataSchema, allowImplicitConversions = true, typeWideningMode = typeWideningMode ) } /** Casts columns in the given dataframe to match the target schema. */ private def castDataIfNeeded(data: DataFrame, targetSchema: StructType): DataFrame = { if (!sqlConf.getConf(DeltaSQLConf.DELTA_STREAMING_SINK_ALLOW_IMPLICIT_CASTS)) return data // We should respect 'spark.sql.caseSensitive' here but writing to a Delta sink is currently // case insensitive so we align with that. val targetTypes = CaseInsensitiveMap[DataType](targetSchema.map(field => field.name -> field.dataType).toMap) val needCast = if (sqlConf.getConf(DeltaSQLConf.DELTA_STREAMING_SINK_IMPLICIT_CAST_FOR_TYPE_MISMATCH_ONLY)) { def hasTypeMismatch(from: DataType, to: DataType): Boolean = (from, to) match { case (from: StructType, to: StructType) => val otherFields = SchemaMergingUtils.toFieldMap(to.fields, caseSensitive = false) from.exists { field => otherFields.get(field.name) match { case Some(other) => hasTypeMismatch(field.dataType, other.dataType) // Ignore extra fields. case None => false } } case (from: MapType, to: MapType) => hasTypeMismatch(from.keyType, to.keyType) || hasTypeMismatch(from.valueType, to.valueType) case (from: ArrayType, to: ArrayType) => hasTypeMismatch(from.elementType, to.elementType) case (from, to) => from != to } hasTypeMismatch(data.schema, targetSchema) } else { // This will also return true if there are missing/extra nested fields or if nested fields // are in a different order than in the table schema. We don't actually need implicit // casting in these cases since Parquet will automatically fill missing fields with nulls // and resolves fields by name. data.schema.exists { field => !DataTypeUtils.equalsIgnoreCaseAndNullability(field.dataType, targetTypes(field.name)) } } if (!needCast) return data def exprForColumn(df: DataFrame, columnName: String): Expression = if (sqlConf.getConf(DeltaSQLConf.DELTA_STREAMING_SINK_IMPLICIT_CAST_ESCAPE_COLUMN_NAMES)) { df.col(QuotingUtils.quoteIdentifier(columnName)).expr } else { df.col(columnName).expr } val castColumns = data.columns.map { columnName => val castExpr = castIfNeeded( fromExpression = exprForColumn(data, columnName), dataType = targetTypes(columnName), castingBehavior = CastByName(allowMissingStructField = true), columnName = columnName ) Column(Alias(castExpr, columnName)()) } data.queryExecution match { case i: IncrementalExecution => DeltaStreamUtils.selectFromStreamingDataFrame(i, data, castColumns: _*) case _: QueryExecution => data.select(castColumns: _*) } } override def toString(): String = s"DeltaSink[$path]" } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSource.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.sources // scalastyle:off import.ordering.noEmptyLine import java.io.FileNotFoundException import java.sql.Timestamp import scala.util.{Failure, Success, Try} import scala.util.control.NonFatal import scala.util.matching.Regex import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.files.DeltaSourceSnapshot import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.storage.{ClosableIterator, SupportsRewinding} import org.apache.spark.sql.delta.storage.ClosableIterator._ import org.apache.spark.sql.delta.util.{DateTimeUtils, TimestampFormatter} import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.hadoop.fs.FileStatus import org.apache.spark.internal.MDC import org.apache.spark.sql.{DataFrame, SparkSession} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.{Expression, Literal} import org.apache.spark.sql.catalyst.plans.logical.LocalRelation import org.apache.spark.sql.connector.read.streaming import org.apache.spark.sql.connector.read.streaming.{ReadAllAvailable, ReadLimit, ReadMaxFiles, SupportsAdmissionControl, SupportsTriggerAvailableNow} import org.apache.spark.sql.execution.streaming._ import org.apache.spark.sql.types.StructType import org.apache.spark.util.Utils /** * A case class to help with `Dataset` operations regarding Offset indexing, representing AddFile * actions in a Delta log. For proper offset tracking (SC-19523), there are also special sentinel * values with negative index = [[DeltaSourceOffset.BASE_INDEX]] and add = null. * * This class is not designed to be persisted in offset logs or such. * * @param version The version of the Delta log containing this AddFile. * @param index The index of this AddFile in the Delta log. * @param add The AddFile. * @param remove The RemoveFile if any. * @param cdc the CDC File if any. * @param isLast A flag to indicate whether this is the last AddFile in the version. This is used * to resolve an off-by-one issue in the streaming offset interface; once we've read * to the end of a log version file, we check this flag to advance immediately to the * next one in the persisted offset. Without this special case we would re-read the * already completed log file. * @param shouldSkip A flag to indicate whether this IndexedFile should be skipped. Currently, we * skip processing an IndexedFile on no-op merges to avoid producing redundant * records. */ private[delta] case class IndexedFile( version: Long, index: Long, add: AddFile, remove: RemoveFile = null, cdc: AddCDCFile = null, shouldSkip: Boolean = false) extends AdmittableFile { require(Option(add).size + Option(remove).size + Option(cdc).size <= 1, "IndexedFile must have at most one of add, remove, or cdc") def getFileAction: FileAction = { if (add != null) { add } else if (remove != null) { remove } else { cdc } } override def hasFileAction(): Boolean = { getFileAction != null } override def getFileSize(): Long = { if (add != null) { add.size } else if (remove != null) { remove.size.getOrElse(0) } else { cdc.size } } } /** * Base trait for the Delta Source, that contains methods that deal with * getting changes from the delta log. */ trait DeltaSourceBase extends Source with SupportsAdmissionControl with SupportsTriggerAvailableNow with DeltaLogging { self: DeltaSource => /** * Configuration options for handling schema changes behavior. Controls unsafe operations like * column mapping changes, partition column changes, nullability changes, and type widening. */ protected lazy val schemaReadOptions: DeltaStreamUtils.SchemaReadOptions = { val schemaReadOptions = DeltaStreamUtils.SchemaReadOptions.fromSparkSession( spark = spark, isStreamingFromColumnMappingTable = snapshotAtSourceInit.metadata.columnMappingMode != NoMapping, isTypeWideningSupportedInProtocol = TypeWidening.isSupported(snapshotAtSourceInit.protocol)) if (schemaReadOptions.allowUnsafeStreamingReadOnColumnMappingSchemaChanges) recordDeltaEvent( deltaLog, "delta.unsafe.streaming.readOnColumnMappingSchemaChanges" ) schemaReadOptions } /** * The persisted schema from the schema log that must be used to read data files in this Delta * streaming source. */ protected val persistedMetadataAtSourceInit: Option[PersistedMetadata] = metadataTrackingLog.flatMap(_.getCurrentTrackedMetadata) /** * The read schema for this source during initialization, taking in account of SchemaLog. */ protected val readSchemaAtSourceInit: StructType = readSnapshotDescriptor.metadata.schema protected val readPartitionSchemaAtSourceInit: StructType = readSnapshotDescriptor.metadata.partitionSchema protected val readProtocolAtSourceInit: Protocol = readSnapshotDescriptor.protocol protected val readConfigurationsAtSourceInit: Map[String, String] = readSnapshotDescriptor.metadata.configuration /** * Create a snapshot descriptor, customizing its metadata using metadata tracking if necessary */ protected lazy val readSnapshotDescriptor: SnapshotDescriptor = persistedMetadataAtSourceInit.map { customMetadata => // Construct a snapshot descriptor with custom schema inline new SnapshotDescriptor { val deltaLog: DeltaLog = snapshotAtSourceInit.deltaLog val metadata: Metadata = snapshotAtSourceInit.metadata.copy( schemaString = customMetadata.dataSchemaJson, partitionColumns = customMetadata.partitionSchema.fieldNames, // Copy the configurations so the correct file format can be constructed configuration = customMetadata.tableConfigurations // Fallback for backward compat only, this should technically not be triggered .getOrElse { val config = snapshotAtSourceInit.metadata.configuration logWarning(log"Using snapshot's table configuration: " + log"${MDC(DeltaLogKeys.CONFIG, config)}") config } ) val protocol: Protocol = customMetadata.protocol.getOrElse { val protocol = snapshotAtSourceInit.protocol logWarning(log"Using snapshot's protocol: ${MDC(DeltaLogKeys.PROTOCOL, protocol)}") protocol } // The following are not important in stream reading val version: Long = customMetadata.deltaCommitVersion val numOfFilesIfKnown = snapshotAtSourceInit.numOfFilesIfKnown val sizeInBytesIfKnown = snapshotAtSourceInit.sizeInBytesIfKnown } }.getOrElse(snapshotAtSourceInit) /** * A global flag to mark whether we have done a per-stream start check for column mapping * schema changes (rename / drop). */ @volatile protected var hasCheckedReadIncompatibleSchemaChangesOnStreamStart: Boolean = false override val schema: StructType = { val readSchemaWithCdc = if (options.readChangeFeed) { CDCReader.cdcReadSchema(readSchemaAtSourceInit) } else { readSchemaAtSourceInit } DeltaTableUtils.removeInternalDeltaMetadata( spark, DeltaTableUtils.removeInternalWriterMetadata(spark, readSchemaWithCdc)) } // A dummy empty dataframe that can be returned at various point during streaming protected val emptyDataFrame: DataFrame = DataFrameUtils.ofRows(spark, LocalRelation(schema).copy(isStreaming = true)) /** * When `AvailableNow` is used, this offset will be the upper bound where this run of the query * will process up. We may run multiple micro batches, but the query will stop itself when it * reaches this offset. */ protected var lastOffsetForTriggerAvailableNow: Option[DeltaSourceOffset] = None private var isLastOffsetForTriggerAvailableNowInitialized = false private var isTriggerAvailableNow = false override def prepareForTriggerAvailableNow(): Unit = { logInfo(log"The streaming query reports to use Trigger.AvailableNow.") isTriggerAvailableNow = true } /** * initialize the internal states for AvailableNow if this method is called first time after * `prepareForTriggerAvailableNow`. */ protected def initForTriggerAvailableNowIfNeeded( startOffsetOpt: Option[DeltaSourceOffset]): Unit = { if (isTriggerAvailableNow && !isLastOffsetForTriggerAvailableNowInitialized) { isLastOffsetForTriggerAvailableNowInitialized = true initLastOffsetForTriggerAvailableNow(startOffsetOpt) } } protected def initLastOffsetForTriggerAvailableNow( startOffsetOpt: Option[DeltaSourceOffset]): Unit = { val offset = latestOffsetInternal(startOffsetOpt, ReadLimit.allAvailable()) lastOffsetForTriggerAvailableNow = offset lastOffsetForTriggerAvailableNow.foreach { lastOffset => logInfo(log"lastOffset for Trigger.AvailableNow has set to " + log"${MDC(DeltaLogKeys.OFFSET, lastOffset.json)}") } } /** An internal `latestOffsetInternal` to get the latest offset. */ protected def latestOffsetInternal( startOffset: Option[DeltaSourceOffset], limit: ReadLimit): Option[DeltaSourceOffset] protected def getFileChangesWithRateLimit( fromVersion: Long, fromIndex: Long, isInitialSnapshot: Boolean, limits: Option[DeltaSource.AdmissionLimits] = Some(DeltaSource.AdmissionLimits(options))) : ClosableIterator[IndexedFile] = { val iter = if (options.readChangeFeed) { // In this CDC use case, we need to consider RemoveFile and AddCDCFiles when getting the // offset. // This method is only used to get the offset so we need to return an iterator of IndexedFile. getFileChangesForCDC(fromVersion, fromIndex, isInitialSnapshot, limits, None).flatMap(_._2) .toClosable } else { val changes = getFileChanges(fromVersion, fromIndex, isInitialSnapshot) // Take each change until we've seen the configured number of addFiles. Some changes don't // represent file additions; we retain them for offset tracking, but they don't count towards // the maxFilesPerTrigger conf. if (limits.isEmpty) { changes } else { val admissionControl = limits.get changes.withClose { it => it.takeWhile { admissionControl.admit(_) } } } } // Stop before any schema change barrier if detected. stopIndexedFileIteratorAtSchemaChangeBarrier(iter) } /** * get the changes from startVersion, startIndex to the end * @param startVersion - calculated starting version * @param startIndex - calculated starting index * @param isInitialSnapshot - whether the stream has to return the initial snapshot or not * @param endOffset - Offset that signifies the end of the stream. * @return */ protected def getFileChangesAndCreateDataFrame( startVersion: Long, startIndex: Long, isInitialSnapshot: Boolean, endOffset: DeltaSourceOffset): DataFrame = { if (options.readChangeFeed) { getCDCFileChangesAndCreateDataFrame(startVersion, startIndex, isInitialSnapshot, endOffset) } else { val fileActionsIter = getFileChanges( startVersion, startIndex, isInitialSnapshot, endOffset = Some(endOffset) ) try { val filteredIndexedFiles = fileActionsIter.filter { indexedFile => indexedFile.getFileAction != null && excludeRegex.forall(_.findFirstIn(indexedFile.getFileAction.path).isEmpty) } val (result, duration) = Utils.timeTakenMs { createDataFrame(filteredIndexedFiles) } logInfo(log"Getting dataFrame for delta_log_path=" + log"${MDC(DeltaLogKeys.PATH, deltaLog.logPath)} with " + log"startVersion=${MDC(DeltaLogKeys.START_VERSION, startVersion)}, " + log"startIndex=${MDC(DeltaLogKeys.START_INDEX, startIndex)}, " + log"isInitialSnapshot=${MDC(DeltaLogKeys.IS_INIT_SNAPSHOT, isInitialSnapshot)}, " + log"endOffset=${MDC(DeltaLogKeys.END_INDEX, endOffset)} took timeMs=" + log"${MDC(DeltaLogKeys.DURATION, duration)} ms") result } finally { fileActionsIter.close() } } } /** * Given an iterator of file actions, create a DataFrame representing the files added to a table * Only AddFile actions will be used to create the DataFrame. * @param indexedFiles actions iterator from which to generate the DataFrame. */ protected def createDataFrame(indexedFiles: Iterator[IndexedFile]): DataFrame = { val addFiles = indexedFiles .filter(_.getFileAction.isInstanceOf[AddFile]) .toSeq val hasDeletionVectors = addFiles.exists(_.getFileAction.asInstanceOf[AddFile].deletionVector != null) if (hasDeletionVectors) { // Read AddFiles from different versions in different scans. // This avoids an issue where we might read the same file with different deletion vectors in // the same scan, which we cannot support as long we broadcast a map of DVs for lookup. // This code can be removed once we can pass the DVs into the scan directly together with the // AddFile/PartitionedFile entry. addFiles .groupBy(_.version) .values .map { addFilesList => deltaLog.createDataFrame( readSnapshotDescriptor, addFilesList.map(_.getFileAction.asInstanceOf[AddFile]), isStreaming = true) } .reduceOption(_ union _) .getOrElse { // If we filtered out all the values before the groupBy, just return an empty DataFrame. deltaLog.createDataFrame( readSnapshotDescriptor, Seq.empty[AddFile], isStreaming = true) } } else { deltaLog.createDataFrame( readSnapshotDescriptor, addFiles.map(_.getFileAction.asInstanceOf[AddFile]), isStreaming = true) } } /** * Returns the offset that starts from a specific delta table version. This function is * called when starting a new stream query. * * @param fromVersion The version of the delta table to calculate the offset from. * @param isInitialSnapshot Whether the delta version is for the initial snapshot or not. * @param limits Indicates how much data can be processed by a micro batch. */ protected def getStartingOffsetFromSpecificDeltaVersion( fromVersion: Long, isInitialSnapshot: Boolean, limits: Option[DeltaSource.AdmissionLimits]): Option[DeltaSourceOffset] = { // Initialize schema tracking log if possible, no-op if already initialized // This is one of the two places can initialize schema tracking. // This case specifically handles when we have a fresh stream. if (readyToInitializeMetadataTrackingEagerly) { initializeMetadataTrackingAndExitStream(fromVersion) } val changes = getFileChangesWithRateLimit( fromVersion, fromIndex = DeltaSourceOffset.BASE_INDEX, isInitialSnapshot = isInitialSnapshot, limits) val lastFileChange = DeltaSource.iteratorLast(changes) if (lastFileChange.isEmpty) { None } else { // Block latestOffset() from generating an invalid offset by proactively verifying // incompatible schema changes under column mapping. See more details in the method doc. checkReadIncompatibleSchemaChangeOnStreamStartOnce(fromVersion) Some(DeltaSource.buildOffsetFromIndexedFile( tableId, lastFileChange.get.version, lastFileChange.get.index, fromVersion, isInitialSnapshot)) } } /** * Return the next offset when previous offset exists. */ protected def getNextOffsetFromPreviousOffset( previousOffset: DeltaSourceOffset, limits: Option[DeltaSource.AdmissionLimits]): Option[DeltaSourceOffset] = { if (trackingMetadataChange) { getNextOffsetFromPreviousOffsetIfPendingSchemaChange(previousOffset) match { case None => case updatedPreviousOffsetOpt => // Stop generating new offset if there were pending schema changes return updatedPreviousOffsetOpt } } val changes = getFileChangesWithRateLimit( previousOffset.reservoirVersion, previousOffset.index, previousOffset.isInitialSnapshot, limits) val lastFileChange = DeltaSource.iteratorLast(changes) if (lastFileChange.isEmpty) { Some(previousOffset) } else { // Similarly, block latestOffset() from generating an invalid offset by proactively // verifying incompatible schema changes under column mapping. See more details in the // method scala doc. checkReadIncompatibleSchemaChangeOnStreamStartOnce(previousOffset.reservoirVersion) Some(DeltaSource.buildOffsetFromIndexedFile( tableId, lastFileChange.get.version, lastFileChange.get.index, previousOffset.reservoirVersion, previousOffset.isInitialSnapshot)) } } /** * Return the DataFrame between start and end offset. */ protected def createDataFrameBetweenOffsets( startVersion: Long, startIndex: Long, isInitialSnapshot: Boolean, startOffsetOption: Option[DeltaSourceOffset], endOffset: DeltaSourceOffset): DataFrame = { getFileChangesAndCreateDataFrame(startVersion, startIndex, isInitialSnapshot, endOffset) } protected def cleanUpSnapshotResources(): Unit = { if (initialState != null) { initialState.close(unpersistSnapshot = initialStateVersion < snapshotAtSourceInit.version) initialState = null } } /** * Check read-incompatible schema changes during stream (re)start so we could fail fast. * * This only needs to be called ONCE in the life cycle of a stream, either at the very first * latestOffset, or the very first getBatch to make sure we have detected an incompatible * schema change. * Typically, the verifyStreamHygiene that was called maybe good enough to detect these * schema changes, there may be cases that wouldn't work, e.g. consider this sequence: * 1. User starts a new stream @ startingVersion 1 * 2. latestOffset is called before getBatch() because there was no previous commits so * getBatch won't be called as a recovery mechanism. * Suppose there's a single rename/drop/nullability change S during computing next offset, S * would look exactly the same as the latest schema so verifyStreamHygiene would not work. * 3. latestOffset would return this new offset cross the schema boundary. * * If a schema log is already initialized, we don't have to run the initialization nor schema * checks any more. * * @param batchStartVersion Start version we want to verify read compatibility against * @param batchEndVersionOpt Optionally, if we are checking against an existing constructed batch * during streaming initialization, we would also like to verify all * schema changes in between as well before we can lazily initialize the * schema log if needed. */ protected def checkReadIncompatibleSchemaChangeOnStreamStartOnce( batchStartVersion: Long, batchEndVersionOpt: Option[Long] = None): Unit = { if (trackingMetadataChange) return if (hasCheckedReadIncompatibleSchemaChangesOnStreamStart) return lazy val (startVersionSnapshotOpt, errOpt) = Try(deltaLog.getSnapshotAt(batchStartVersion, catalogTableOpt = catalogTableOpt)) match { case Success(snapshot) => (Some(snapshot), None) case Failure(exception) => (None, Some(exception)) } // Cannot perfectly verify column mapping schema changes if we cannot compute a start snapshot. if (!schemaReadOptions.allowUnsafeStreamingReadOnColumnMappingSchemaChanges && schemaReadOptions.isStreamingFromColumnMappingTable && errOpt.isDefined) { throw DeltaErrors.failedToGetSnapshotDuringColumnMappingStreamingReadCheck(errOpt.get) } // Perform schema check if we need to, considering all escape flags. if (!schemaReadOptions.allowUnsafeStreamingReadOnColumnMappingSchemaChanges || schemaReadOptions.typeWideningEnabled || !schemaReadOptions. forceEnableStreamingReadOnReadIncompatibleSchemaChangesDuringStreamStart) { startVersionSnapshotOpt.foreach { snapshot => checkReadIncompatibleSchemaChanges( snapshot.metadata, snapshot.version, batchStartVersion, batchEndVersionOpt, validatedDuringStreamStart = true ) // If end version is defined (i.e. we have a pending batch), let's also eagerly check all // intermediate schema changes against the stream read schema to capture corners cases such // as rename and rename back. for { endVersion <- batchEndVersionOpt (version, metadata) <- collectMetadataActions(batchStartVersion, endVersion) } { checkReadIncompatibleSchemaChanges( metadata, version, batchStartVersion, Some(endVersion), validatedDuringStreamStart = true) } } } // Mark as checked hasCheckedReadIncompatibleSchemaChangesOnStreamStart = true } /** * Narrow waist to verify a metadata action for read-incompatible schema changes, specifically: * 1. Any column mapping related schema changes (rename / drop) columns * 2. Standard read-compatibility changes including: * a) No missing columns * b) No data type changes * c) No read-incompatible nullability changes * If the check fails, we throw an exception to exit the stream. * If lazy log initialization is required, we also run a one time scan to safely initialize the * metadata tracking log upon any non-additive schema change failures. * @param metadata Metadata that contains a potential schema change * @param version Version for the metadata action * @param validatedDuringStreamStart Whether this check is being done during stream start. */ protected def checkReadIncompatibleSchemaChanges( metadata: Metadata, version: Long, batchStartVersion: Long, batchEndVersionOpt: Option[Long] = None, validatedDuringStreamStart: Boolean = false): Unit = { log.info(s"checking read incompatibility with schema at version $version, " + s"inside batch[$batchStartVersion, ${batchEndVersionOpt.getOrElse("latest")}]") val (newMetadata, oldMetadata) = if (version < snapshotAtSourceInit.version) { (snapshotAtSourceInit.metadata, metadata) } else { (metadata, snapshotAtSourceInit.metadata) } // Table ID has changed during streaming if (newMetadata.id != oldMetadata.id) { throw DeltaErrors.differentDeltaTableReadByStreamingSource( newTableId = newMetadata.id, oldTableId = oldMetadata.id) } checkNonAdditiveSchemaChanges(oldMetadata, newMetadata, validatedDuringStreamStart) // Other standard read compatibility changes if (!validatedDuringStreamStart || !schemaReadOptions. forceEnableStreamingReadOnReadIncompatibleSchemaChangesDuringStreamStart) { val schemaChange = if (options.readChangeFeed) { CDCReader.cdcReadSchema(metadata.schema) } else { metadata.schema } // There is a schema change. All of files after this commit will use `schemaChange`. Hence, we // check whether we can use `schema` (the fixed source schema we use in the same run of the // query) to read these new files safely. val backfilling = version < snapshotAtSourceInit.version // Partition column change will be ignored if user enable the unsafe flag val newPartitionColumns = if (schemaReadOptions.allowUnsafeStreamingReadOnPartitionColumnChanges) Seq.empty else newMetadata.partitionColumns val oldPartitionColumns = if (schemaReadOptions.allowUnsafeStreamingReadOnPartitionColumnChanges) Seq.empty else oldMetadata.partitionColumns val checkResult = DeltaStreamUtils.checkSchemaChangesWhenNoSchemaTracking( schemaChange, schema, newPartitionColumns, oldPartitionColumns, backfilling, schemaReadOptions) if (!DeltaStreamUtils.SchemaCompatibilityResult.isCompatible(checkResult)) { val isRetryable = DeltaStreamUtils.SchemaCompatibilityResult.isRetryableIncompatible(checkResult) recordDeltaEvent( deltaLog, "delta.streaming.source.schemaChanged", data = Map( "currentVersion" -> snapshotAtSourceInit.version, "newVersion" -> version, "retryable" -> isRetryable, "backfilling" -> backfilling, "readChangeDataFeed" -> options.readChangeFeed, "typeWideningEnabled" -> schemaReadOptions.typeWideningEnabled, "enableSchemaTrackingForTypeWidening" -> schemaReadOptions.enableSchemaTrackingForTypeWidening, "containsWideningTypeChanges" -> TypeWidening.containsWideningTypeChanges(schema, schemaChange) ) ) throw DeltaErrors.schemaChangedException( schema, schemaChange, retryable = isRetryable, Some(version), includeStartingVersionOrTimestampMessage = options.containsStartingVersionOrTimestamp) } } } /** * Checks for non-additive schema changes (column renames, drops, type widening) and blocks * the stream by throwing an exception if detected. * * Blocks when type widening tracking is enabled and widening changes exist, or when column * mapping changes (rename/drop) are detected, unless `allowUnsafeStreamingReadOnColumnMapping * SchemaChanges` is enabled. Upon blocking, the error requests the user to provide a schema * tracking location to enable schema tracking. On restart, users must acknowledge changes via * reader options or SQL confs. * See [[DeltaSourceMetadataEvolutionSupport.validateIfSchemaChangeCanBeUnblocked]]. * * Note: Should not be called when schema tracking is active (trackingMetadataChange = true). * * @param oldMetadata Previous metadata (typically from stream initialization) * @param newMetadata New metadata with potential schema changes * @param validatedDuringStreamStart Whether validating during stream start vs. execution, * which affects the error message. * @throws DeltaAnalysisException if non-additive schema changes require blocking */ private def checkNonAdditiveSchemaChanges( oldMetadata: Metadata, newMetadata: Metadata, validatedDuringStreamStart: Boolean): Unit = { val shouldTrackSchema: Boolean = if (schemaReadOptions.typeWideningEnabled && schemaReadOptions.enableSchemaTrackingForTypeWidening && TypeWidening.containsWideningTypeChanges(oldMetadata.schema, newMetadata.schema)) { // If schema tracking is enabled for type widening, we will detect widening type changes and // block the stream until the user sets `allowSourceColumnTypeChange` - similar to handling // DROP/RENAME for column mapping. true } else if (schemaReadOptions.allowUnsafeStreamingReadOnColumnMappingSchemaChanges) { false } else { // Column mapping schema changes assert(!trackingMetadataChange, "should not check schema change while tracking it") !DeltaColumnMapping.hasNoColumnMappingSchemaChanges(newMetadata, oldMetadata, schemaReadOptions.allowUnsafeStreamingReadOnPartitionColumnChanges) } if (shouldTrackSchema) { throw DeltaErrors.blockStreamingReadsWithIncompatibleNonAdditiveSchemaChanges( spark, oldMetadata.schema, newMetadata.schema, detectedDuringStreaming = !validatedDuringStreamStart) } } } /** * A streaming source for a Delta table. * * When a new stream is started, delta starts by constructing a * [[org.apache.spark.sql.delta.Snapshot]] at * the current version of the table. This snapshot is broken up into batches until * all existing data has been processed. Subsequent processing is done by tailing * the change log looking for new data. This results in the streaming query returning * the same answer as a batch query that had processed the entire dataset at any given point. */ case class DeltaSource( spark: SparkSession, deltaLog: DeltaLog, catalogTableOpt: Option[CatalogTable], options: DeltaOptions, snapshotAtSourceInit: SnapshotDescriptor, metadataPath: String, metadataTrackingLog: Option[DeltaSourceMetadataTrackingLog] = None, filters: Seq[Expression] = Nil) extends DeltaSourceBase with DeltaSourceCDCSupport with DeltaSourceMetadataEvolutionSupport { private val shouldValidateOffsets = spark.sessionState.conf.getConf(DeltaSQLConf.STREAMING_OFFSET_VALIDATION) // Deprecated. Please use `skipChangeCommits` from now on. private val ignoreFileDeletion = { if (options.ignoreFileDeletion) { logConsole(DeltaErrors.ignoreStreamingUpdatesAndDeletesWarning(spark)) recordDeltaEvent(deltaLog, "delta.deprecation.ignoreFileDeletion") } options.ignoreFileDeletion } /** A check on the source table that skips commits that contain removes from the * set of files. */ private val skipChangeCommits = options.skipChangeCommits protected val excludeRegex: Option[Regex] = options.excludeRegex // This was checked before creating ReservoirSource assert(schema.nonEmpty) protected val tableId = snapshotAtSourceInit.metadata.id // A metadata snapshot when starting the query. protected var initialState: DeltaSourceSnapshot = null protected var initialStateVersion: Long = -1L logInfo(log"Filters being pushed down: ${MDC(DeltaLogKeys.FILTER, filters)}") /** * Get the changes starting from (startVersion, startIndex). The start point should not be * included in the result. * * @param endOffset If defined, do not return changes beyond this offset. * If not defined, we must be scanning the log to find the next offset. * @param verifyMetadataAction If true, we will break the stream when we detect any * read-incompatible metadata changes. */ protected def getFileChanges( fromVersion: Long, fromIndex: Long, isInitialSnapshot: Boolean, endOffset: Option[DeltaSourceOffset] = None, verifyMetadataAction: Boolean = true ): ClosableIterator[IndexedFile] = { /** Returns matching files that were added on or after startVersion among delta logs. */ def filterAndIndexDeltaLogs(startVersion: Long): ClosableIterator[IndexedFile] = { // TODO: handle the case when failOnDataLoss = false and we are missing change log files // in that case, we need to recompute the start snapshot and evolve the schema if needed require(options.failOnDataLoss || !trackingMetadataChange, "Using schema from schema tracking log cannot tolerate missing commit files.") deltaLog.getChangeLogFiles( startVersion, catalogTableOpt, options.failOnDataLoss).flatMapWithClose { case (version, filestatus) => // First pass reads the whole commit and closes the iterator. val iter = DeltaSource.createRewindableActionIterator(spark, deltaLog, filestatus) val (shouldSkipCommit, metadataOpt, protocolOpt) = iter .processAndClose { actionsIter => validateCommitAndDecideSkipping( actionsIter, version, fromVersion, endOffset, verifyMetadataAction && !trackingMetadataChange ) } // Rewind the iterator to the beginning, if the actions are cached in memory, they will // be reused again. iter.rewind() // Second pass reads the commit lazily. iter.withClose { actionsIter => filterAndGetIndexedFiles( actionsIter, version, shouldSkipCommit, metadataOpt, protocolOpt) } } } val (result, duration) = Utils.timeTakenMs { var iter = if (isInitialSnapshot) { Iterator(1, 2).flatMapWithClose { // so that the filterAndIndexDeltaLogs call is lazy case 1 => getSnapshotAt(fromVersion)._1.toClosable case 2 => filterAndIndexDeltaLogs(fromVersion + 1) } } else { filterAndIndexDeltaLogs(fromVersion) } iter = iter.withClose { it => it.filter { file => file.version > fromVersion || file.index > fromIndex } } // If endOffset is provided, we are getting a batch on a constructed range so we should use // the endOffset as the limit. // Otherwise, we are looking for a new offset, so we try to use the latestOffset we found for // Trigger.availableNow() as limit. We know endOffset <= lastOffsetForTriggerAvailableNow. val lastOffsetForThisScan = endOffset.orElse(lastOffsetForTriggerAvailableNow) lastOffsetForThisScan.foreach { bound => iter = iter.withClose { it => it.takeWhile { file => file.version < bound.reservoirVersion || (file.version == bound.reservoirVersion && file.index <= bound.index) } } } iter } logInfo(log"Getting file changes for delta_log_path=" + log"${MDC(DeltaLogKeys.PATH, deltaLog.logPath)} with " + log"fromVersion=${MDC(DeltaLogKeys.START_VERSION, fromVersion)}, " + log"fromIndex=${MDC(DeltaLogKeys.START_INDEX, fromIndex)}, " + log"isInitialSnapshot=${MDC(DeltaLogKeys.IS_INIT_SNAPSHOT, isInitialSnapshot)} " + log"took timeMs=${MDC(DeltaLogKeys.DURATION, duration)} ms") result } /** * Adds dummy BEGIN_INDEX and END_INDEX IndexedFiles for @version before and after the * contents of the iterator. The contents of the iterator must be the IndexedFiles that correspond * to this version. */ protected def addBeginAndEndIndexOffsetsForVersion( version: Long, iterator: Iterator[IndexedFile]): Iterator[IndexedFile] = { Iterator.single(IndexedFile(version, DeltaSourceOffset.BASE_INDEX, add = null)) ++ iterator ++ Iterator.single(IndexedFile(version, DeltaSourceOffset.END_INDEX, add = null)) } /** * This method computes the initial snapshot to read when Delta Source was initialized on a fresh * stream. * @return A tuple where the first element is an iterator of IndexedFiles and the second element * is the in-commit timestamp of the initial snapshot if available. */ protected def getSnapshotAt(version: Long): (Iterator[IndexedFile], Option[Long]) = { if (initialState == null || version != initialStateVersion) { super[DeltaSourceBase].cleanUpSnapshotResources() val snapshot = getSnapshotFromDeltaLog(version) initialState = new DeltaSourceSnapshot(spark, snapshot, filters) initialStateVersion = version // This handle a special case for schema tracking log when it's initialized but the initial // snapshot's schema has changed, suppose: // 1. The stream starts and looks at the initial snapshot to compute the starting offset, say // at version 0 with schema // 2. User renames a column, creates version 1 with schema // 3. The read compatibility check fails during scanning version 1, initializes schema log // using the initial snapshot's schema (, because that's the safest thing to do as we // have not served any data from initial snapshot yet) and exits stream. // 4. Stream restarts, since no starting offset was generated, it will retry loading the // initial snapshot, which is now at version 1, but the tracked schema is now different // from the "new" initial snapshot schema! Worse, since schema tracking ignores any schema // changes inside initial snapshot, we will then be reading the files using a wrong schema! // The below logic allows us to detect any discrepancies when reading initial snapshot using // a tracked schema, and reinitialize the log if needed. if (trackingMetadataChange && initialState.snapshot.version >= readSnapshotDescriptor.version) { updateMetadataTrackingLogAndFailTheStreamIfNeeded( Some(initialState.snapshot.metadata), Some(initialState.snapshot.protocol), initialState.snapshot.version, // The new schema should replace the previous initialized schema for initial snapshot replace = true ) } } val inCommitTimestampOpt = Option.when( DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(initialState.snapshot.metadata)) { initialState.snapshot.timestamp } (addBeginAndEndIndexOffsetsForVersion(version, initialState.iterator()), inCommitTimestampOpt) } /** * Narrow-waist for generating snapshot from Delta Log within Delta Source */ protected def getSnapshotFromDeltaLog(version: Long): Snapshot = { try { deltaLog.getSnapshotAt(version, catalogTableOpt = catalogTableOpt) } catch { case e: FileNotFoundException => throw DeltaErrors.logFileNotFoundExceptionForStreamingSource(e) } } private def getStartingOffset( limits: Option[DeltaSource.AdmissionLimits]): Option[DeltaSourceOffset] = { val (version, isInitialSnapshot) = getStartingVersion match { case Some(v) => (v, false) case None => (snapshotAtSourceInit.version, true) } if (version < 0) { return None } getStartingOffsetFromSpecificDeltaVersion(version, isInitialSnapshot, limits) } override def getDefaultReadLimit: ReadLimit = { DeltaSource.AdmissionLimits.toReadLimit(options) } def toDeltaSourceOffset(offset: streaming.Offset): DeltaSourceOffset = { DeltaSourceOffset(tableId, offset) } /** * This should only be called by the engine. Call `latestOffsetInternal` instead if you need to * get the latest offset. */ override def latestOffset(startOffset: streaming.Offset, limit: ReadLimit): streaming.Offset = recordDeltaOperation( snapshotAtSourceInit.deltaLog, opType = "delta.streaming.source.latestOffset") { val deltaStartOffset = Option(startOffset).map(toDeltaSourceOffset) initForTriggerAvailableNowIfNeeded(deltaStartOffset) latestOffsetInternal(deltaStartOffset, limit).orNull } override protected def latestOffsetInternal( startOffset: Option[DeltaSourceOffset], limit: ReadLimit): Option[DeltaSourceOffset] = { val limits = DeltaSource.AdmissionLimits(options, limit) val endOffset = startOffset.map(getNextOffsetFromPreviousOffset(_, limits)) .getOrElse(getStartingOffset(limits)) val startVersion = startOffset.map(_.reservoirVersion).getOrElse(-1L) val endVersion = endOffset.map(_.reservoirVersion).getOrElse(-1L) lazy val offsetRangeInfo = "(latestOffsetInternal)startOffset -> endOffset:" + s" $startOffset -> $endOffset" if (endVersion - startVersion > 1000L) { // Improve the log level if the source is processing a large batch. logInfo(offsetRangeInfo) } else { logDebug(offsetRangeInfo) } if (shouldValidateOffsets && startOffset.isDefined) { endOffset.foreach { endOffset => DeltaSourceOffset.validateOffsets(startOffset.get, endOffset) } } endOffset } override def getOffset: Option[Offset] = { throw new UnsupportedOperationException( "latestOffset(Offset, ReadLimit) should be called instead of this method") } /** * Filter the iterator with only add files that contain data change and get indexed files. * @return indexed add files */ private def filterAndGetIndexedFiles( iterator: Iterator[Action], version: Long, shouldSkipCommit: Boolean, metadataOpt: Option[Metadata], protocolOpt: Option[Protocol]): Iterator[IndexedFile] = { val filteredIterator = if (shouldSkipCommit) { Iterator.empty } else { iterator.collect { case a: AddFile if a.dataChange => a } } var index = -1L val indexedFiles = new Iterator[IndexedFile] { override def hasNext: Boolean = filteredIterator.hasNext override def next(): IndexedFile = { index += 1 // pre-increment the index (so it starts from 0) val add = filteredIterator.next().copy(stats = null) IndexedFile(version, index, add) } } addBeginAndEndIndexOffsetsForVersion( version, getMetadataOrProtocolChangeIndexedFileIterator(metadataOpt, protocolOpt, version) ++ indexedFiles) } /** * Check stream for violating any constraints. * * If verifyMetadataAction = true, we will break the stream when we detect any read-incompatible * metadata changes. * * @return (true if commit should be skipped, a metadata action if found) */ protected def validateCommitAndDecideSkipping( actions: Iterator[Action], version: Long, batchStartVersion: Long, batchEndOffsetOpt: Option[DeltaSourceOffset] = None, verifyMetadataAction: Boolean = true ): (Boolean, Option[Metadata], Option[Protocol]) = { // If the batch end is at the beginning of this exact version, then we actually stop reading // just _before_ this version. So then we can ignore the version contents entirely. if (batchEndOffsetOpt.exists(end => end.reservoirVersion == version && end.index == DeltaSourceOffset.BASE_INDEX)) { return (false, None, None) } /** A check on the source table that disallows changes on the source data. */ val shouldAllowChanges = options.ignoreChanges || ignoreFileDeletion || skipChangeCommits /** A check on the source table that disallows commits that only include deletes to the data. */ val shouldAllowDeletes = shouldAllowChanges || options.ignoreDeletes || ignoreFileDeletion var seenFileAdd = false var skippedCommit = false var metadataAction: Option[Metadata] = None var protocolAction: Option[Protocol] = None var removeFileActionPath: Option[String] = None var operation: Option[String] = None actions.foreach { case a: AddFile if a.dataChange => seenFileAdd = true case r: RemoveFile if r.dataChange => skippedCommit = skipChangeCommits if (removeFileActionPath.isEmpty) { removeFileActionPath = Some(r.path) } case m: Metadata => if (verifyMetadataAction) { checkReadIncompatibleSchemaChanges( m, version, batchStartVersion, batchEndOffsetOpt.map(_.reservoirVersion)) } assert(metadataAction.isEmpty, "Should not encounter two metadata actions in the same commit") metadataAction = Some(m) case protocol: Protocol => deltaLog.protocolRead(protocol) assert(protocolAction.isEmpty, "Should not encounter two protocol actions in the same commit") protocolAction = Some(protocol) case commitInfo: CommitInfo => operation = Some(s"${commitInfo.operation} (${commitInfo.operationParameters})") case _ => () } if (removeFileActionPath.isDefined) { if (seenFileAdd && !shouldAllowChanges) { throw DeltaErrors.deltaSourceIgnoreChangesError( version, if (operation.nonEmpty) operation.get else removeFileActionPath.get, deltaLog.dataPath.toString ) } else if (!seenFileAdd && !shouldAllowDeletes) { throw DeltaErrors.deltaSourceIgnoreDeleteError( version, removeFileActionPath.get, deltaLog.dataPath.toString ) } } (skippedCommit, metadataAction, protocolAction) } override def getBatch(startOffsetOption: Option[Offset], end: Offset): DataFrame = recordDeltaOperation( snapshotAtSourceInit.deltaLog, opType = "delta.streaming.source.getBatch") { val endOffset = toDeltaSourceOffset(end) val startDeltaOffsetOption = startOffsetOption.map(toDeltaSourceOffset) val (startVersion, startIndex, isInitialSnapshot) = extractStartingState(startDeltaOffsetOption, endOffset) if (startOffsetOption.contains(endOffset)) { // This happens only if we recover from a failure and `MicroBatchExecution` tries to call // us with the previous offsets. The returned DataFrame will be dropped immediately, so we // can return any DataFrame. return emptyDataFrame } val offsetRangeInfo = s"(getBatch)start: $startDeltaOffsetOption end: $end" if (endOffset.reservoirVersion - startVersion > 1000L) { // Improve the log level if the source is processing a large batch. logInfo(offsetRangeInfo) } else { logDebug(offsetRangeInfo) } // Initialize schema tracking log if possible, no-op if already initialized. // This is one of the two places can initialize schema tracking. // This case specifically handles initialization when we are already working with an initialized // stream. // Here we may have two conditions: // 1. We are dealing with the recovery getBatch() that gives us the previous committed offset // where start and end corresponds to the previous batch. // In this case, we should initialize the schema at the previous committed offset (endOffset), // which can be done using the same `initializeMetadataTrackingAndExitStream` method. // This also means we are caught up with the stream and we can start schema tracking in the // next latestOffset call. // 2. We are running an already-constructed batch, we need the schema to be compatible // with the entire batch, so we also pass the batch end offset. The schema tracking log will // only be initialized if there exists a consistent read schema for the entire batch. If such // a consistent schema does not exist, the stream will be broken. This case will be rare: it can // only happen for streams where the schema tracking log was added after the stream has already // been running, *and* the stream was running on an older version of the DeltaSource that did // not detect non-additive schema changes, *and* it was stopped while processing a batch that // contained such a schema change. // In either world, the initialization logic would find the superset compatible schema for this // batch by scanning Delta log. validateAndInitMetadataLogForPlannedBatchesDuringStreamStart(startVersion, endOffset) val createdDf = createDataFrameBetweenOffsets( startVersion, startIndex, isInitialSnapshot, startDeltaOffsetOption, endOffset) createdDf } /** * Extracts the start state for a scan given an optional start offset and an end offset, so we * know exactly where we should scan from for a batch end at the `endOffset`, invoked when: * * 1. We are in `getBatch` given a startOffsetOption and endOffset from streaming engine. * 2. We are in the `init` method for every stream (re)start given a start offset for all pending * batches and the latest planned offset, and trying to figure out if this range contains any * non-additive schema changes. * * @param startOffsetOption Optional start offset, if not defined. This means we are trying to * scan the very first batch where endOffset is the very first offset * generated by `latestOffsets`, specifically `getStartingOffset` * @param endOffset The end offset for a batch. * @return (start commit version to scan from, * start offset index to scan from, * whether this version is part of the initial snapshot) */ private def extractStartingState( startOffsetOption: Option[DeltaSourceOffset], endOffset: DeltaSourceOffset): (Long, Long, Boolean) = { val (startVersion, startIndex, isInitialSnapshot) = if (startOffsetOption.isEmpty) { getStartingVersion match { case Some(v) => (v, DeltaSourceOffset.BASE_INDEX, false) case None => if (endOffset.isInitialSnapshot) { (endOffset.reservoirVersion, DeltaSourceOffset.BASE_INDEX, true) } else { assert( endOffset.reservoirVersion > 0, s"invalid reservoirVersion in endOffset: $endOffset") // Load from snapshot `endOffset.reservoirVersion - 1L` so that `index` in `endOffset` // is still valid. // It's OK to use the previous version as the updated initial snapshot, even if the // initial snapshot might have been different from the last time when this starting // offset was computed. (endOffset.reservoirVersion - 1L, DeltaSourceOffset.BASE_INDEX, true) } } } else { val startOffset = startOffsetOption.get if (!startOffset.isInitialSnapshot) { // unpersist `snapshot` because it won't be used any more. cleanUpSnapshotResources() } (startOffset.reservoirVersion, startOffset.index, startOffset.isInitialSnapshot) } (startVersion, startIndex, isInitialSnapshot) } /** * Centralized place for validating and initializing schema log for all pending batch(es). * This is called only during stream start. * * @param startVersion Start version of the pending batch range * @param endOffset End offset for the pending batch range. end offset >= start offset */ private def validateAndInitMetadataLogForPlannedBatchesDuringStreamStart( startVersion: Long, endOffset: DeltaSourceOffset): Unit = { // We don't have to include the end reservoir version when the end offset is a base index, i.e. // no data commit has been marked within a constructed batch, we can simply ignore end offset // version. This can help us avoid overblocking a potential ending offset right at a schema // change. val endVersionForMetadataLogInit = if (endOffset.index == DeltaSourceOffset.BASE_INDEX) { endOffset.reservoirVersion - 1 } else { endOffset.reservoirVersion } // For eager initialization, we initialize the log right now. if (readyToInitializeMetadataTrackingEagerly) { initializeMetadataTrackingAndExitStream(startVersion, Some(endVersionForMetadataLogInit)) } // Check for column mapping + streaming incompatible schema changes // Note for initial snapshot, the startVersion should be the same as the latestOffset's // version and therefore this check won't have any effect. // This method would also handle read-compatibility checks against the pending batch(es) // as well as lazy metadata log initialization. checkReadIncompatibleSchemaChangeOnStreamStartOnce( startVersion, Some(endVersionForMetadataLogInit) ) } override def stop(): Unit = { cleanUpSnapshotResources() } // Marks that the `end` offset is done and we can safely run any actions in response to that. // This happens AFTER `end` offset is committed by the streaming engine so we can safely fail this // if needed, e.g. for failing the stream to conduct schema evolution. override def commit(end: Offset): Unit = recordDeltaOperation(snapshotAtSourceInit.deltaLog, opType = "delta.streaming.source.commit") { super.commit(end) // IMPORTANT: for future developers, please place any work you would like to do in commit() // before `updateSchemaTrackingLogAndFailTheStreamIfNeeded(end)` as it may throw an exception. updateMetadataTrackingLogAndFailTheStreamIfNeeded(end) } override def toString(): String = s"DeltaSource[${deltaLog.dataPath}]" /** * Extracts whether users provided the option to time travel a relation. If a query restarts from * a checkpoint and the checkpoint has recorded the offset, this method should never been called. */ protected lazy val getStartingVersion: Option[Long] = { // Note: returning a version beyond latest snapshot version won't be a problem as callers // of this function won't use the version to retrieve snapshot(refer to [[getStartingOffset]]). val allowOutOfRange = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_CDF_ALLOW_OUT_OF_RANGE_TIMESTAMP) /** DeltaOption validates input and ensures that only one is provided. */ if (options.startingVersion.isDefined) { val v = options.startingVersion.get match { case StartingVersionLatest => deltaLog.update(catalogTableOpt = catalogTableOpt).version + 1 case StartingVersion(version) => if (!DeltaSource.validateProtocolAt(spark, deltaLog, catalogTableOpt, version)) { // When starting from a given version, we don't require that the snapshot of this // version can be reconstructed, even though the input table is technically in an // inconsistent state. If the snapshot cannot be reconstructed, then the protocol // check is skipped, so this is technically not safe, but we keep it this way for // historical reasons. deltaLog.history.checkVersionExists( version, catalogTableOpt = None, mustBeRecreatable = false, allowOutOfRange) } version } Some(v) } else if (options.startingTimestamp.isDefined) { val tt: DeltaTimeTravelSpec = DeltaTimeTravelSpec( timestamp = options.startingTimestamp.map(Literal(_)), version = None, creationSource = Some("deltaSource")) Some(DeltaSource .getStartingVersionFromTimestamp( spark, deltaLog, catalogTableOpt, tt.getTimestamp(spark.sessionState.conf), allowOutOfRange)) } else { None } } } object DeltaSource extends DeltaLogging { trait DeltaSourceAdmissionBase { self: AdmissionLimits => // This variable indicates whether a commit has already been processed by a batch or not. var commitProcessedInBatch = false protected def take(files: Int, bytes: Long): Unit = { filesToTake -= files bytesToTake -= bytes } /** * This overloaded method checks if all the FileActions for a commit can be accommodated by * the rate limit. */ def admit(admittableFiles: Seq[AdmittableFile]): Boolean = { def getSize(actions: Seq[AdmittableFile]): Long = { actions.filter(_.hasFileAction).foldLeft(0L) { (l, r) => l + r.getFileSize } } if (admittableFiles.isEmpty) { true } else { // if no files have been admitted, then admit all to avoid deadlock // else check if all of the files together satisfy the limit, only then admit val bytesInFiles = getSize(admittableFiles) val shouldAdmit = !commitProcessedInBatch || (filesToTake - admittableFiles.size >= 0 && bytesToTake - bytesInFiles >= 0) commitProcessedInBatch = true take(files = admittableFiles.size, bytes = bytesInFiles) shouldAdmit } } /** * Whether to admit the next file. Dummy IndexedFile entries with no attached file action are * always admitted. */ def admit(admittableFile: AdmittableFile): Boolean = { commitProcessedInBatch = true if (!admittableFile.hasFileAction) { // Don't count placeholders. They are not files. If we have empty commits, then we should // not count the placeholders as files, or else we'll end up with under-filled batches. return true } // We always admit a file if we still have capacity _before_ we take it. This ensures that we // will even admit a file when it is larger than the remaining capacity, and that we will // admit at least one file. val shouldAdmit = hasCapacity take(files = 1, bytes = admittableFile.getFileSize) shouldAdmit } /** Returns whether admission limits has capacity to accept files or bytes */ def hasCapacity: Boolean = { filesToTake > 0 && bytesToTake > 0 } } /** * Class that helps controlling how much data should be processed by a single micro-batch. */ case class AdmissionLimits( options: DeltaOptions, maxFiles: Option[Int] = None, maxBytes: Option[Long] = None ) extends DeltaSourceAdmissionBase { var bytesToTake = maxBytes.getOrElse(options.maxBytesPerTrigger.getOrElse(Long.MaxValue)) var filesToTake = maxFiles.getOrElse { if (options.maxBytesPerTrigger.isEmpty) { DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION_DEFAULT } else { Int.MaxValue - 8 // - 8 to prevent JVM Array allocation OOM } } } object AdmissionLimits { def toReadLimit(options: DeltaOptions): ReadLimit = { if (options.maxFilesPerTrigger.isDefined && options.maxBytesPerTrigger.isDefined) { CompositeLimit( ReadMaxBytes(options.maxBytesPerTrigger.get), ReadLimit.maxFiles(options.maxFilesPerTrigger.get).asInstanceOf[ReadMaxFiles]) } else if (options.maxBytesPerTrigger.isDefined) { ReadMaxBytes(options.maxBytesPerTrigger.get) } else { ReadLimit.maxFiles( options.maxFilesPerTrigger.getOrElse(DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION_DEFAULT)) } } def apply(options: DeltaOptions, limit: ReadLimit): Option[AdmissionLimits] = limit match { case _: ReadAllAvailable => None case maxFiles: ReadMaxFiles => Some(new AdmissionLimits( options = options, maxFiles = Some(maxFiles.maxFiles()), maxBytes = None)) case maxBytes: ReadMaxBytes => Some(new AdmissionLimits( options = options, maxFiles = None, maxBytes = Some(maxBytes.maxBytes))) case composite: CompositeLimit => Some(new AdmissionLimits( options = options, maxFiles = Some(composite.maxFiles.maxFiles()), maxBytes = Some(composite.bytes.maxBytes))) case other => throw DeltaErrors.unknownReadLimit(other.toString()) } } /** * Validate the protocol at a given version. If the snapshot reconstruction fails for any other * reason than table feature exception, we suppress it. This allows to fallback to previous * behavior where the starting version/timestamp was not mandatory to point to reconstructable * snapshot. * * Returns true when the validation was performed and succeeded. */ def validateProtocolAt( spark: SparkSession, deltaLog: DeltaLog, catalogTableOpt: Option[CatalogTable], version: Long): Boolean = { val alwaysValidateProtocol = spark.sessionState.conf.getConf( DeltaSQLConf.FAST_DROP_FEATURE_STREAMING_ALWAYS_VALIDATE_PROTOCOL) if (!alwaysValidateProtocol) return false try { // We attempt to construct a snapshot at the startingVersion in order to validate the // protocol. If snapshot reconstruction fails, fall back to the old behavior where the // only requirement was for the commit to exist. deltaLog.getSnapshotAt(version, catalogTableOpt = catalogTableOpt) return true } catch { case e: DeltaUnsupportedTableFeatureException => recordDeltaEvent( deltaLog = deltaLog, opType = "dropFeature.validateProtocolAt.unsupportedFeatureFound", data = Map("message" -> e.getMessage)) throw e case NonFatal(e) => // Suppress rest errors. logWarning(log"Protocol validation failed with '${MDC(DeltaLogKeys.EXCEPTION, e)}'.") recordDeltaEvent( deltaLog = deltaLog, opType = "dropFeature.validateProtocolAt.error", data = Map("message" -> e.getMessage)) } false } /** * Returns the earliest commit version whose timestamp is >= the provided timestamp. * * This method fetches the commit at the given timestamp via * [[DeltaLog.history.getActiveCommitAtTime]], computes the starting version using * [[DeltaStreamUtils.getStartingVersionFromCommitAtTimestamp]], and validates the protocol * at the returned version. * * @param spark - current spark session * @param deltaLog - Delta log of the table for which we find the version. * @param catalogTableOpt - The CatalogTable for the Delta table. * @param timestamp - user specified timestamp * @param canExceedLatest - if true, version can be greater than the latest snapshot commit * @return - corresponding version number for timestamp * @see [[DeltaStreamUtils.getStartingVersionFromCommitAtTimestamp]] for the core version * computation logic */ def getStartingVersionFromTimestamp( spark: SparkSession, deltaLog: DeltaLog, catalogTableOpt: Option[CatalogTable], timestamp: Timestamp, canExceedLatest: Boolean = false): Long = { val tz = spark.sessionState.conf.sessionLocalTimeZone val commit = deltaLog.history.getActiveCommitAtTime( timestamp, catalogTableOpt = catalogTableOpt, canReturnLastCommit = true, mustBeRecreatable = false, canReturnEarliestCommit = true) // Note: `getActiveCommitAtTime` has called `update`, so we don't need to call it again. val latestVersion = deltaLog.unsafeVolatileSnapshot.version val startingVersion = DeltaStreamUtils.getStartingVersionFromCommitAtTimestamp( timeZone = tz, commitTimestamp = commit.timestamp, commitVersion = commit.version, latestVersion = latestVersion, timestamp = timestamp, canExceedLatest = canExceedLatest ) if (startingVersion <= latestVersion) { validateProtocolAt(spark, deltaLog, catalogTableOpt, startingVersion) } startingVersion } /** * Read an [[ClosableIterator]] of Delta actions from file status, considering memory constraints */ def createRewindableActionIterator( spark: SparkSession, deltaLog: DeltaLog, fileStatus: FileStatus): ClosableIterator[Action] with SupportsRewinding[Action] = { val threshold = spark.sessionState.conf.getConf(DeltaSQLConf.LOG_SIZE_IN_MEMORY_THRESHOLD) lazy val actions = deltaLog.store.read(fileStatus, deltaLog.newDeltaHadoopConf()).map(Action.fromJson) // Return a new [[CloseableIterator]] over the commit. If the commit is smaller than the // threshold, we will read it into memory once and iterate over that every time. // Otherwise, we read it again every time. val shouldLoadIntoMemory = fileStatus.getLen < threshold def createClosableIterator(): ClosableIterator[Action] = if (shouldLoadIntoMemory) { // Reuse in the memory actions actions.toIterator.toClosable } else { deltaLog.store.readAsIterator(fileStatus, deltaLog.newDeltaHadoopConf()) .withClose { _.map(Action.fromJson) } } new ClosableIterator[Action] with SupportsRewinding[Action] { var delegatedIterator: ClosableIterator[Action] = createClosableIterator() override def hasNext: Boolean = delegatedIterator.hasNext override def next(): Action = delegatedIterator.next() override def close(): Unit = delegatedIterator.close() override def rewind(): Unit = delegatedIterator = createClosableIterator() } } /** * Scan and get the last item of the iterator. */ def iteratorLast[T](iter: ClosableIterator[T]): Option[T] = { try { var last: Option[T] = None while (iter.hasNext) { last = Some(iter.next()) } last } finally { iter.close() } } /** * Build the latest offset based on the last indexedFile. The function also checks if latest * version is valid by comparing with previous version. * Public for use by SparkMicroBatchStream. * @param tableId The table ID * @param fileVersion The version of the last indexed file. * @param fileIndex The index of the last indexed file. * @param previousVersion Previous offset reservoir version. * @param isInitialSnapshot Whether previous offset is starting version or not. * @return A DeltaSourceOffset representing the next offset to read from. */ def buildOffsetFromIndexedFile( tableId: String, fileVersion: Long, fileIndex: Long, previousVersion: Long, isInitialSnapshot: Boolean): DeltaSourceOffset = { val (v, i) = (fileVersion, fileIndex) assert(v >= previousVersion, s"buildOffsetFromIndexedFile returns an invalid version: $v " + s"(expected: >= $previousVersion), tableId: $tableId") // If the last file in previous batch is the end index of that version, automatically bump // to next version to skip accessing that version file altogether. The END_INDEX should never // be returned as an offset. val offset = if (i == DeltaSourceOffset.END_INDEX) { // isInitialSnapshot must be false here as we have bumped the version. DeltaSourceOffset( tableId, v + 1, index = DeltaSourceOffset.BASE_INDEX, isInitialSnapshot = false) } else { // isInitialSnapshot will be true only if previous isInitialSnapshot is true and the next file // is still at the same version. DeltaSourceOffset( tableId, v, i, isInitialSnapshot = v == previousVersion && isInitialSnapshot ) } offset } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSourceCDCSupport.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.sources import java.io.FileNotFoundException import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.actions.DomainMetadata import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.DeltaErrors import org.apache.spark.internal.MDC import org.apache.spark.sql.DataFrame import org.apache.spark.util.Utils /** * Helper functions for CDC-specific handling for DeltaSource. */ trait DeltaSourceCDCSupport { self: DeltaSource => ///////////////////////// // Nested helper class // ///////////////////////// /** * This class represents an iterator of Change metadata(AddFile, RemoveFile, AddCDCFile) * for a particular version. * @param fileActionsItr - Iterator of IndexedFiles for a particular commit. * @param isInitialSnapshot - Indicates whether the commit version is the initial snapshot or not. */ class IndexedChangeFileSeq( fileActionsItr: Iterator[IndexedFile], isInitialSnapshot: Boolean) { private def moreThanFrom( indexedFile: IndexedFile, fromVersion: Long, fromIndex: Long): Boolean = { // we need to filter out files so that we get only files after the startingOffset indexedFile.version > fromVersion || indexedFile.index > fromIndex } private def lessThanEnd( indexedFile: IndexedFile, endOffset: Option[DeltaSourceOffset]): Boolean = { // we need to filter out files so that they are within the end offsets. if (endOffset.isEmpty) { true } else { indexedFile.version < endOffset.get.reservoirVersion || (indexedFile.version <= endOffset.get.reservoirVersion && indexedFile.index <= endOffset.get.index) } } private def noMatchesRegex(indexedFile: IndexedFile): Boolean = { if (hasNoFileActionAndStartOrEndIndex(indexedFile)) return true excludeRegex.forall(_.findFirstIn(indexedFile.getFileAction.path).isEmpty) } private def hasFileAction(indexedFile: IndexedFile): Boolean = { indexedFile.getFileAction != null } private def hasNoFileActionAndStartOrEndIndex(indexedFile: IndexedFile): Boolean = { !indexedFile.hasFileAction && (indexedFile.index == DeltaSourceOffset.BASE_INDEX || indexedFile.index == DeltaSourceOffset.END_INDEX) } private def hasAddsOrRemoves(indexedFile: IndexedFile): Boolean = { indexedFile.add != null || indexedFile.remove != null } private def isSchemaChangeIndexedFile(indexedFile: IndexedFile): Boolean = { indexedFile.index == DeltaSourceOffset.METADATA_CHANGE_INDEX || indexedFile.index == DeltaSourceOffset.POST_METADATA_CHANGE_INDEX } private def isValidIndexedFile( indexedFile: IndexedFile, fromVersion: Long, fromIndex: Long, endOffset: Option[DeltaSourceOffset]): Boolean = { !indexedFile.shouldSkip && (hasFileAction(indexedFile) || hasNoFileActionAndStartOrEndIndex(indexedFile) || isSchemaChangeIndexedFile(indexedFile)) && moreThanFrom(indexedFile, fromVersion, fromIndex) && lessThanEnd(indexedFile, endOffset) && noMatchesRegex(indexedFile) && lessThanEnd(indexedFile, lastOffsetForTriggerAvailableNow) } /** * Returns the IndexedFiles for particular commit version after rate-limiting and filtering * out based on version boundaries. */ def filterFiles( fromVersion: Long, fromIndex: Long, limits: Option[DeltaSource.AdmissionLimits], endOffset: Option[DeltaSourceOffset] = None): Iterator[IndexedFile] = { if (limits.isEmpty) { return fileActionsItr.filter(isValidIndexedFile(_, fromVersion, fromIndex, endOffset)) } val admissionControl = limits.get if (isInitialSnapshot) { // NOTE: the initial snapshot can be huge hence we do not do a toSeq here. fileActionsItr .filter(isValidIndexedFile(_, fromVersion, fromIndex, endOffset)) .takeWhile { admissionControl.admit(_) } } else { // Change data for a commit can be either recorded by a Seq[AddCDCFiles] or // a Seq[AddFile]/ Seq[RemoveFile] val fileActions = fileActionsItr.toSeq // If there exists a stopping iterator for this version, we should return right-away fileActions.find(isSchemaChangeIndexedFile) match { case Some(schemaChangeBarrier) => return Seq(schemaChangeBarrier).toIterator case _ => } val cdcFiles = fileActions.filter(_.cdc != null) // get only cdc commits. if (cdcFiles.nonEmpty) { // CDC of commit is represented by AddCDCFile val filteredFiles = cdcFiles .filter(isValidIndexedFile(_, fromVersion, fromIndex, endOffset)) // For CDC commits we either admit the entire commit or nothing at all. // This is to avoid returning `update_preimage` and `update_postimage` in separate // batches. if (admissionControl.admit(filteredFiles)) { filteredFiles.toIterator } else { Iterator() } } else { // CDC is recorded as AddFile or RemoveFile // We allow entries with no file actions and index as [[DeltaSourceOffset.BASE_INDEX]] // that are used primarily to update latest offset when no other // file action based entries are present. val filteredFiles = fileActions .filter { indexedFile => hasAddsOrRemoves(indexedFile) || hasNoFileActionAndStartOrEndIndex(indexedFile) } .filter(isValidIndexedFile(_, fromVersion, fromIndex, endOffset)) val hasDeletionVectors = fileActions.filter(_.hasFileAction).map(_.getFileAction).exists { case add: AddFile => add.deletionVector != null case remove: RemoveFile => remove.deletionVector != null case _ => false } if (hasDeletionVectors) { // We cannot split up add/remove pairs with Deletion Vectors, because we will get the // wrong result. // So in this case we behave as above with CDC files and either admit all or none. if (admissionControl.admit(filteredFiles)) { filteredFiles.toIterator } else { Iterator() } } else { filteredFiles.takeWhile { admissionControl.admit(_) }.toIterator } } } } } /////////////////////////////// // Util methods for children // /////////////////////////////// /** * Get the changes from startVersion, startIndex to the end for CDC case. We need to call * CDCReader to get the CDC DataFrame. * * @param startVersion - calculated starting version * @param startIndex - calculated starting index * @param isInitialSnapshot - whether the stream has to return the initial snapshot or not * @param endOffset - Offset that signifies the end of the stream. * @return the DataFrame containing the file changes (AddFile, RemoveFile, AddCDCFile) */ protected def getCDCFileChangesAndCreateDataFrame( startVersion: Long, startIndex: Long, isInitialSnapshot: Boolean, endOffset: DeltaSourceOffset): DataFrame = { val changes = getFileChangesForCDC( startVersion, startIndex, isInitialSnapshot, limits = None, Some(endOffset)) val groupedFileAndCommitInfoActions = changes.map { case (v, indexFiles, commitInfoOpt) => (v, indexFiles.filter(_.hasFileAction).map(_.getFileAction).toSeq ++ commitInfoOpt) } val (result, duration) = Utils.timeTakenMs { // CDCReader calls getSnapshotAt directly instead of using DeltaSource's wrapper, which can // result in FileNotFoundExceptions from the Delta log. We wrap these to present a clearer // error message. try { CDCReader .changesToDF( readSnapshotDescriptor, startVersion, endOffset.reservoirVersion, groupedFileAndCommitInfoActions, spark, catalogTableOpt, isStreaming = true) .fileChangeDf } catch { case e: FileNotFoundException => throw DeltaErrors.logFileNotFoundExceptionForStreamingSource(e) } } logInfo(log"Getting CDC dataFrame for delta_log_path=" + log"${MDC(DeltaLogKeys.PATH, deltaLog.logPath)} with " + log"startVersion=${MDC(DeltaLogKeys.START_VERSION, startVersion)}, " + log"startIndex=${MDC(DeltaLogKeys.START_INDEX, startIndex)}, " + log"isInitialSnapshot=${MDC(DeltaLogKeys.IS_INIT_SNAPSHOT, isInitialSnapshot)}, " + log"endOffset=${MDC(DeltaLogKeys.END_OFFSET, endOffset)} took timeMs=" + log"${MDC(DeltaLogKeys.DURATION, duration)} ms") result } /** * Get the changes starting from (fromVersion, fromIndex). fromVersion is included. * It returns an iterator of (log_version, fileActions, Optional[CommitInfo]). The commit info * is needed later on so that the InCommitTimestamp of the log files can be determined. * * If verifyMetadataAction = true, we will break the stream when we detect any read-incompatible * metadata changes. */ protected def getFileChangesForCDC( fromVersion: Long, fromIndex: Long, isInitialSnapshot: Boolean, limits: Option[DeltaSource.AdmissionLimits], endOffset: Option[DeltaSourceOffset], verifyMetadataAction: Boolean = true ): Iterator[(Long, Iterator[IndexedFile], Option[CommitInfo])] = { /** Returns matching files that were added on or after startVersion among delta logs. */ def filterAndIndexDeltaLogs( startVersion: Long): Iterator[(Long, IndexedChangeFileSeq, Option[CommitInfo])] = { // TODO: handle the case when failOnDataLoss = false and we are missing change log files // in that case, we need to recompute the start snapshot and evolve the schema if needed require(options.failOnDataLoss || !trackingMetadataChange, "Using schema from schema tracking log cannot tolerate missing commit files.") deltaLog.getChanges( startVersion, catalogTableOpt, options.failOnDataLoss).map { case (version, actions) => // skipIndexedFile must be applied after creating IndexedFile so that // IndexedFile.index is consistent across all versions. val (fileActions, skipIndexedFile, metadataOpt, protocolOpt, commitInfoOpt) = filterCDCActions( actions, version, fromVersion, endOffset.map(_.reservoirVersion), verifyMetadataAction && !trackingMetadataChange) val itr = addBeginAndEndIndexOffsetsForVersion(version, getMetadataOrProtocolChangeIndexedFileIterator(metadataOpt, protocolOpt, version) ++ fileActions.zipWithIndex.map { case (action: AddFile, index) => IndexedFile( version, index.toLong, action, shouldSkip = skipIndexedFile) case (cdcFile: AddCDCFile, index) => IndexedFile( version, index.toLong, add = null, cdc = cdcFile, shouldSkip = skipIndexedFile) case (remove: RemoveFile, index) => IndexedFile( version, index.toLong, add = null, remove = remove, shouldSkip = skipIndexedFile) }) (version, new IndexedChangeFileSeq(itr, isInitialSnapshot = false), commitInfoOpt) } } /** Verifies that provided version is <= endOffset version, if defined. */ def versionLessThanEndOffset(version: Long, endOffset: Option[DeltaSourceOffset]): Boolean = { endOffset match { case Some(eo) => version <= eo.reservoirVersion case None => true } } val (result, duration) = Utils.timeTakenMs { val iter: Iterator[(Long, IndexedChangeFileSeq, Option[CommitInfo])] = if (isInitialSnapshot) { // If we are reading change data from the start of the table we need to // get the latest snapshot of the table as well. val (unprocessedSnapshot, snapshotInCommitTimestampOpt) = getSnapshotAt(fromVersion) val snapshot: Iterator[IndexedFile] = unprocessedSnapshot.map { m => // When we get the snapshot the dataChange is false for the AddFile actions // We need to set it to true for it to be considered by the CDCReader. if (m.add != null) { m.copy(add = m.add.copy(dataChange = true)) } else { m } } // This is a hack so that we can easily access the ICT later on. // This `CommitInfo` action is not useful for anything else and should be filtered // out later on. val ictOnlyCommitInfo = Some(CommitInfo.empty(Some(-1)) .copy(inCommitTimestamp = snapshotInCommitTimestampOpt)) val snapshotItr: Iterator[(Long, IndexedChangeFileSeq, Option[CommitInfo])] = Iterator(( fromVersion, new IndexedChangeFileSeq(snapshot, isInitialSnapshot = true), ictOnlyCommitInfo )) snapshotItr ++ filterAndIndexDeltaLogs(fromVersion + 1) } else { filterAndIndexDeltaLogs(fromVersion) } // In this case, filterFiles will consume the available capacity. We use takeWhile // to stop the iteration when we reach the limit or if endOffset is specified and the // endVersion is reached which will save us from reading unnecessary log files. iter.takeWhile { case (version, _, _) => limits.forall(_.hasCapacity) && versionLessThanEndOffset(version, endOffset) }.map { case (version, indexItr, ci) => (version, indexItr.filterFiles(fromVersion, fromIndex, limits, endOffset), ci) } } logInfo(log"Getting CDC file changes for delta_log_path=" + log"${MDC(DeltaLogKeys.PATH, deltaLog.logPath)} with " + log"fromVersion=${MDC(DeltaLogKeys.START_VERSION, fromVersion)}, fromIndex=" + log"${MDC(DeltaLogKeys.START_INDEX, fromIndex)}, " + log"isInitialSnapshot=${MDC(DeltaLogKeys.IS_INIT_SNAPSHOT, isInitialSnapshot)} took timeMs=" + log"${MDC(DeltaLogKeys.DURATION, duration)} ms") result } ///////////////////// // Private methods // ///////////////////// /** * Filter out non CDC actions and only return CDC ones. This will either be AddCDCFiles * or AddFile and RemoveFiles * * If verifyMetadataAction = true, we will break the stream when we detect any read-incompatible * metadata changes. */ private def filterCDCActions( actions: Seq[Action], version: Long, batchStartVersion: Long, batchEndVersionOpt: Option[Long] = None, verifyMetadataAction: Boolean = true ): (Seq[FileAction], Boolean, Option[Metadata], Option[Protocol], Option[CommitInfo]) = { var shouldSkipIndexedFile = false var metadataAction: Option[Metadata] = None var protocolAction: Option[Protocol] = None var commitInfoAction: Option[CommitInfo] = None def checkAndCacheMetadata(m: Metadata): Unit = { if (verifyMetadataAction) { checkReadIncompatibleSchemaChanges(m, version, batchStartVersion, batchEndVersionOpt) } assert(metadataAction.isEmpty, "Should not encounter two metadata actions in the same commit") metadataAction = Some(m) } if (actions.exists(_.isInstanceOf[AddCDCFile])) { (actions.filter { case _: AddCDCFile => true case commitInfo: CommitInfo => commitInfoAction = Some(commitInfo) false case m: Metadata => checkAndCacheMetadata(m) false case p: Protocol => protocolAction = Some(p) false case _ => false }.asInstanceOf[Seq[FileAction]], shouldSkipIndexedFile, metadataAction, protocolAction, commitInfoAction) } else { (actions.filter { case a: AddFile => a.dataChange case r: RemoveFile => r.dataChange case m: Metadata => checkAndCacheMetadata(m) false case protocol: Protocol => deltaLog.protocolRead(protocol) assert(protocolAction.isEmpty, "Should not encounter two protocol actions in the same commit") protocolAction = Some(protocol) false case commitInfo: CommitInfo => shouldSkipIndexedFile = CDCReader.shouldSkipFileActionsInCommit(commitInfo) commitInfoAction = Some(commitInfo) false case _: AddCDCFile | _: SetTransaction | _: DomainMetadata => false case null => // Some crazy future feature. Ignore false }.asInstanceOf[Seq[FileAction]], shouldSkipIndexedFile, metadataAction, protocolAction, commitInfoAction) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSourceMetadataEvolutionSupport.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.sources import java.util.Locale import scala.collection.mutable import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.{Action, Metadata, Protocol} import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.storage.ClosableIterator import org.apache.spark.sql.delta.storage.ClosableIterator._ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.execution.streaming.Offset import org.apache.spark.sql.types.StructType /** * Helper functions for metadata evolution related handling for DeltaSource. * A metadata change is one of: * 1. Schema change * 2. Delta table configuration change * 3. Delta protocol change * The documentation below will use schema change as example throughout. * * To achieve schema evolution, we intercept in different stages of the normal streaming process to: * 1. Capture all schema changes inside a stream * 2. Stop the latestOffset from crossing the schema change boundary * 3. Ensure the batch prior to the schema change can still be served correctly * 4. Ensure the stream fails if and only if the prior batch is served successfully * 5. Write the new schema to the schema tracking log prior to stream failure, so that next time when it restarts we will use the updated schema. * * Specifically, * 1. During latestOffset calls, if we detect schema change at version V, we generate a special * barrier [[DeltaSourceOffset]] X that has ver=V and index=INDEX_METADATA_CHANGE. * (We first generate an [[IndexedFile]] at this index, and that gets converted into an * equivalent [[DeltaSourceOffset]].) * [[INDEX_METADATA_CHANGE]] comes after [[INDEX_VERSION_BASE]] (the first * offset index that exists for any reservoir version) and before the offsets that represent data * changes. This ensures that we apply the schema change before processing the data * that uses that schema. * 2. When we see a schema change offset X, then this is treated as a barrier that ends the * current batch. The remaining data is effectively unavailable until all the source data before * the schema change has been committed. * 3. Then, when a [[commit]] is invoked on the offset schema change barrier offset X, we can * then officially write the new schema into the schema tracking log and fail the stream. * [[commit]] is only called after this batch ending at X is completed, so it would be safe to * fail there. * 4. In between when offset X is generated and when it is committed, there could be arbitrary * number of calls to [[latestOffset]], attempting to fetch new latestOffset. These calls mustn't * generate new offsets until the schema change barrier offset has been committed, the new schema * has been written to the schema tracking log, and the stream has been aborted and restarted. * A nuance here - streaming engine won't [[commit]] until it sees a new offset that is * semantically different, which is why we first generate an offset X with index * INDEX_METADATA_CHANGE, but another second barrier offset X' immediately following * it with index INDEX_POST_SCHEMA_CHANGE. * In this way, we could ensure: * a) Offset with index INDEX_METADATA_CHANGE is always committed (typically) * b) Even if streaming engine changed its behavior and ONLY offset with index * INDEX_POST_SCHEMA_CHANGE is committed, we can still see this is a * schema change barrier with a schema change ready to be evolved. * c) Whenever [[latestOffset]] sees a startOffset with a schema change barrier index, we can * easily tell that we should not progress past the schema change, unless the schema change * has actually happened. * When a stream is restarted post a schema evolution (not initialization), it is guaranteed to have * >= 2 entries in the schema log. To prevent users from shooting themselves in the foot while * blindly restart stream without considering implications to downstream tables, by default we would * not allow stream to restart without a magic SQL conf that user has to set to allow non-additive * schema changes to propagate. We detect such non-additive schema changes during stream start by * comparing the last schema log entry with the current one. */ trait DeltaSourceMetadataEvolutionSupport extends DeltaSourceBase { base: DeltaSource => /** * Whether this DeltaSource is utilizing a schema log entry as its read schema. * * If user explicitly turn on the flag to fall back to using latest schema to read (i.e. the * legacy mode), we will ignore the schema log. */ protected def trackingMetadataChange: Boolean = !schemaReadOptions.allowUnsafeStreamingReadOnColumnMappingSchemaChanges && metadataTrackingLog.flatMap(_.getCurrentTrackedMetadata).nonEmpty /** * Whether a schema tracking log is provided (and is empty), so we could initialize eagerly. * This should only be used for the first write to the schema log, after then, schema tracking * should not rely on this state any more. */ protected def readyToInitializeMetadataTrackingEagerly: Boolean = !schemaReadOptions.allowUnsafeStreamingReadOnColumnMappingSchemaChanges && metadataTrackingLog.exists { log => log.getCurrentTrackedMetadata.isEmpty && log.initMetadataLogEagerly } /** * This is called from getFileChangesWithRateLimit() during latestOffset(). */ protected def stopIndexedFileIteratorAtSchemaChangeBarrier( fileActionScanIter: ClosableIterator[IndexedFile]): ClosableIterator[IndexedFile] = { fileActionScanIter.withClose { iter => val (untilSchemaChange, fromSchemaChange) = iter.span { i => i.index != DeltaSourceOffset.METADATA_CHANGE_INDEX } // This will end at the schema change indexed file (inclusively) // If there are no schema changes, this is an no-op. untilSchemaChange ++ fromSchemaChange.take(1) } } /** * Check the table metadata or protocol changed since the initial read snapshot. We make sure: * 1. The schema is the same, except for internal metadata, AND * 2. The delta related table configurations are strictly equal, AND * 3. The incoming metadata change should not be considered a failure-causing change if we have * marked the persisted schema and the stream progress is behind that schema version. * This could happen when we've already merged consecutive schema changes during the analysis * phase and we are using the merged schema as the read schema. All the schema changes in * between can be safely ignored because they won't contribute any data. */ private def hasMetadataOrProtocolChangeComparedToStreamMetadata( metadataChangeOpt: Option[Metadata], protocolChangeOpt: Option[Protocol], newSchemaVersion: Long): Boolean = { if (persistedMetadataAtSourceInit.exists(_.deltaCommitVersion >= newSchemaVersion)) { false } else { protocolChangeOpt.exists(_ != readProtocolAtSourceInit) || metadataChangeOpt.exists { newMetadata => hasSchemaChangeComparedToStreamMetadata(newMetadata.schema) || newMetadata.partitionSchema != readPartitionSchemaAtSourceInit || newMetadata.configuration.filterKeys(_.startsWith("delta.")).toMap != readConfigurationsAtSourceInit.filterKeys(_.startsWith("delta.")).toMap } } } /** * Check that the give schema is the same as the schema from the initial read snapshot. */ private def hasSchemaChangeComparedToStreamMetadata(newSchema: StructType): Boolean = if (spark.conf.get(DeltaSQLConf.DELTA_STREAMING_IGNORE_INTERNAL_METADATA_FOR_SCHEMA_CHANGE)) { DeltaTableUtils.removeInternalWriterMetadata(spark, newSchema) != DeltaTableUtils.removeInternalWriterMetadata(spark, readSchemaAtSourceInit) } else { newSchema != readSchemaAtSourceInit } /** * If the current stream metadata is not equal to the metadata change in [[metadataChangeOpt]], * return a metadata change barrier [[IndexedFile]]. * Only returns something if [[trackingMetadataChange]]is true. */ protected def getMetadataOrProtocolChangeIndexedFileIterator( metadataChangeOpt: Option[Metadata], protocolChangeOpt: Option[Protocol], version: Long): ClosableIterator[IndexedFile] = { if (trackingMetadataChange && hasMetadataOrProtocolChangeComparedToStreamMetadata( metadataChangeOpt, protocolChangeOpt, version)) { // Create an IndexedFile with metadata change Iterator.single(IndexedFile(version, DeltaSourceOffset.METADATA_CHANGE_INDEX, null)) .toClosable } else { Iterator.empty.toClosable } } /** * Collect all actions between start and end version, both inclusive */ private def collectActions( startVersion: Long, endVersion: Long ): ClosableIterator[(Long, Action)] = { deltaLog.getChangeLogFiles(startVersion, catalogTableOpt, options.failOnDataLoss).takeWhile { case (version, _) => version <= endVersion }.flatMapWithClose { case (version, fileStatus) => DeltaSource.createRewindableActionIterator(spark, deltaLog, fileStatus) .map((version, _)) .toClosable } } /** * Given the version range for an ALREADY fetched batch, check if there are any * read-incompatible schema changes or protocol changes. * In this case, the streaming engine wants to getBatch(X,Y) on an existing Y that is already * loaded and saved in the offset log in the past before requesting new offsets. Therefore we * should verify if we could find a schema or protocol that is safe to read this constructed batch * , which then can be used to initialize the metadata log. * If not, there's not much we could do, even with metadata log, because unlike finding new * offsets, we don't have a chance to "split" this batch at schema change boundaries any more. The * streaming engine is not able to change the ranges of a batch after it has created it. * If there are no non-additive schema changes, or incompatible protocol changes, it is safe to * mark the metadata and protocol safe to read for all data files between startVersion and * endVersion. */ private def validateAndResolveMetadataForLogInitialization( startVersion: Long, endVersion: Long): (Metadata, Protocol) = { val metadataChanges = collectMetadataActions(startVersion, endVersion).map(_._2) val startSnapshot = getSnapshotFromDeltaLog(startVersion) val startMetadata = startSnapshot.metadata // Try to find rename or drop columns in between, or nullability/datatype changes by using // the last schema as the read schema and if so we cannot find a good read schema. // Otherwise, the most recent metadata change will be the most encompassing schema as well. val mostRecentMetadataChangeOpt = metadataChanges.lastOption mostRecentMetadataChangeOpt.foreach { mostRecentMetadataChange => val otherMetadataChanges = Seq(startMetadata) ++ metadataChanges.dropRight(1) otherMetadataChanges.foreach { potentialSchemaChangeMetadata => if (!DeltaColumnMapping.hasNoColumnMappingSchemaChanges( newMetadata = mostRecentMetadataChange, oldMetadata = potentialSchemaChangeMetadata) || !SchemaUtils.isReadCompatible( existingSchema = potentialSchemaChangeMetadata.schema, readSchema = mostRecentMetadataChange.schema, forbidTightenNullability = true)) { throw DeltaErrors.streamingMetadataLogInitFailedIncompatibleMetadataException( startVersion, endVersion) } } } // Check protocol changes and use the most supportive protocol val startProtocol = startSnapshot.protocol val protocolChanges = collectProtocolActions(startVersion, endVersion).map(_._2) var mostSupportiveProtocol = startProtocol protocolChanges.foreach { p => if (mostSupportiveProtocol.readerAndWriterFeatureNames .subsetOf(p.readerAndWriterFeatureNames)) { mostSupportiveProtocol = p } else { // TODO: or use protocol union instead? throw DeltaErrors.streamingMetadataLogInitFailedIncompatibleMetadataException( startVersion, endVersion) } } (mostRecentMetadataChangeOpt.getOrElse(startMetadata), mostSupportiveProtocol) } /** * Collect a metadata action at the commit version if possible. */ private def collectMetadataAtVersion(version: Long): Option[Metadata] = { collectActions(version, version).processAndClose { iter => iter.map(_._2).collectFirst { case a: Metadata => a } } } protected def collectMetadataActions( startVersion: Long, endVersion: Long): Seq[(Long, Metadata)] = { collectActions(startVersion, endVersion).processAndClose { iter => iter.collect { case (version, a: Metadata) => (version, a) }.toSeq } } /** * Collect a protocol action at the commit version if possible. */ private def collectProtocolAtVersion(version: Long): Option[Protocol] = { collectActions(version, version).processAndClose { iter => iter.map(_._2).collectFirst { case a: Protocol => a } } } protected def collectProtocolActions( startVersion: Long, endVersion: Long): Seq[(Long, Protocol)] = { collectActions(startVersion, endVersion).processAndClose { iter => iter.collect { case (version, a: Protocol) => (version, a) }.toSeq } } /** * If the given previous Delta source offset is a schema change offset, returns the appropriate * next offset. This should be called before trying any other means of determining the next * offset. * If this returns None, then there is no schema change, and the caller should determine the next * offset in the normal way. */ protected def getNextOffsetFromPreviousOffsetIfPendingSchemaChange( previousOffset: DeltaSourceOffset): Option[DeltaSourceOffset] = { // Check if we've generated a previous offset with schema change (i.e. offset X in class doc) // Then, we will generate offset X' as mentioned in the class doc. if (previousOffset.index == DeltaSourceOffset.METADATA_CHANGE_INDEX) { return Some(previousOffset.copy(index = DeltaSourceOffset.POST_METADATA_CHANGE_INDEX)) } // If the previous offset is already POST the schema change and schema evolution has not // occurred, simply block as no-op. if (previousOffset.index == DeltaSourceOffset.POST_METADATA_CHANGE_INDEX && hasMetadataOrProtocolChangeComparedToStreamMetadata( collectMetadataAtVersion(previousOffset.reservoirVersion), collectProtocolAtVersion(previousOffset.reservoirVersion), previousOffset.reservoirVersion)) { return Some(previousOffset) } // Otherwise, no special handling None } /** * Initialize the schema tracking log if an empty schema tracking log is provided. * This method also checks the range between batchStartVersion and batchEndVersion to ensure we * a safe schema to be initialized in the log. * @param batchStartVersion Start version of the batch of data to be proceed, it should typically * be the schema that is safe to process incoming data. * @param batchEndVersionOpt Optionally, if we are looking at a constructed batch with existing * end offset, we need to double verify to ensure no read-incompatible * within the batch range. * @param alwaysFailUponLogInitialized Whether we should always fail with the schema evolution * exception. */ protected def initializeMetadataTrackingAndExitStream( batchStartVersion: Long, batchEndVersionOpt: Option[Long] = None, alwaysFailUponLogInitialized: Boolean = false): Unit = { // If possible, initialize the metadata log with the desired start metadata instead of failing. // If a `batchEndVersion` is provided, we also need to verify if there are no incompatible // schema changes in a constructed batch, if so, we cannot find a proper schema to init the // schema log. val (version, metadata, protocol) = batchEndVersionOpt.map { endVersion => val (validMetadata, validProtocol) = validateAndResolveMetadataForLogInitialization(batchStartVersion, endVersion) // `endVersion` should be valid for initialization (endVersion, validMetadata, validProtocol) }.getOrElse { val startSnapshot = getSnapshotFromDeltaLog(batchStartVersion) (startSnapshot.version, startSnapshot.metadata, startSnapshot.protocol) } val newMetadata = PersistedMetadata(tableId, version, metadata, protocol, metadataPath) // Always initialize the metadata log metadataTrackingLog.get.writeNewMetadata(newMetadata) if (hasMetadataOrProtocolChangeComparedToStreamMetadata( Some(metadata), Some(protocol), version) || alwaysFailUponLogInitialized) { // But trigger evolution exception when there's a difference throw DeltaErrors.streamingMetadataEvolutionException( newMetadata.dataSchema, newMetadata.tableConfigurations.get, newMetadata.protocol.get ) } } /** * Update the current stream schema in the schema tracking log and fail the stream. * This is called during commit(). * It's ok to fail during commit() because in streaming's semantics, the batch with offset ending * at `end` should've already being processed completely. */ protected def updateMetadataTrackingLogAndFailTheStreamIfNeeded(end: Offset): Unit = { val offset = DeltaSourceOffset(tableId, end) if (trackingMetadataChange && (offset.index == DeltaSourceOffset.METADATA_CHANGE_INDEX || offset.index == DeltaSourceOffset.POST_METADATA_CHANGE_INDEX)) { // The offset must point to a metadata or protocol change action val changedMetadataOpt = collectMetadataAtVersion(offset.reservoirVersion) val changedProtocolOpt = collectProtocolAtVersion(offset.reservoirVersion) // Evolve the schema when the schema is indeed different from the current stream schema. We // need to check this because we could potentially generate two offsets before schema // evolution each with different indices. // Typically streaming engine will commit the first one and evolve the schema log, however, // to be absolutely safe, we also consider the case when the first is skipped and only the // second one is committed. // If the first one is committed (typically), the stream will fail and restart with the // evolved schema, then we should NOT fail/evolve again when we commit the second offset. updateMetadataTrackingLogAndFailTheStreamIfNeeded( changedMetadataOpt, changedProtocolOpt, offset.reservoirVersion) } } /** * Write a new potentially changed metadata into the metadata tracking log. Then fail the stream * to allow reanalysis if there are changes. * @param changedMetadataOpt Potentially changed metadata action * @param changedProtocolOpt Potentially changed protocol action * @param version The version of change */ protected def updateMetadataTrackingLogAndFailTheStreamIfNeeded( changedMetadataOpt: Option[Metadata], changedProtocolOpt: Option[Protocol], version: Long, replace: Boolean = false): Unit = { if (hasMetadataOrProtocolChangeComparedToStreamMetadata( changedMetadataOpt, changedProtocolOpt, version)) { val schemaToPersist = PersistedMetadata( deltaLog.unsafeVolatileTableId, version, changedMetadataOpt.getOrElse(readSnapshotDescriptor.metadata), changedProtocolOpt.getOrElse(readSnapshotDescriptor.protocol), metadataPath ) // Update schema log if (replace) { metadataTrackingLog.get.writeNewMetadata(schemaToPersist, replaceCurrent = true) } else { metadataTrackingLog.get.writeNewMetadata(schemaToPersist) } // Fail the stream with schema evolution exception throw DeltaErrors.streamingMetadataEvolutionException( schemaToPersist.dataSchema, schemaToPersist.tableConfigurations.get, schemaToPersist.protocol.get ) } } } object DeltaSourceMetadataEvolutionSupport { /** SQL configs that allow unblocking each type of schema changes. */ private val SQL_CONF_PREFIX = s"${DeltaSQLConf.SQL_CONF_PREFIX}.streaming" private final val SQL_CONF_UNBLOCK_RENAME_DROP = SQL_CONF_PREFIX + ".allowSourceColumnRenameAndDrop" private final val SQL_CONF_UNBLOCK_RENAME = SQL_CONF_PREFIX + ".allowSourceColumnRename" private final val SQL_CONF_UNBLOCK_DROP = SQL_CONF_PREFIX + ".allowSourceColumnDrop" private final val SQL_CONF_UNBLOCK_TYPE_CHANGE = SQL_CONF_PREFIX + ".allowSourceColumnTypeChange" /** * Defining the different combinations of non-additive schema changes to detect them and allow * users to vet and unblock them using a corresponding SQL conf or reader option: * - dropping columns * - renaming columns * - widening data types */ private sealed trait SchemaChangeType { val name: String val isRename: Boolean val isDrop: Boolean val isTypeWidening: Boolean val sqlConfsUnblock: Seq[String] val readerOptionsUnblock: Seq[String] val prettyColumnDetailsString: String protected def getRenamedColumnsPrettyString(renamedColumns: Seq[RenamedColumn]): String = { s"""Columns renamed: |${renamedColumns.map { case RenamedColumn(fromFieldPath, toFieldPath) => s"'${SchemaUtils.prettyFieldName(fromFieldPath)}' -> " + s"'${SchemaUtils.prettyFieldName(toFieldPath)}'" }.mkString("\n")} |""".stripMargin } protected def getDroppedColumnsPrettyString(droppedColumns: Seq[DroppedColumn]): String = { s"""Columns dropped: |${droppedColumns.map( c => s"'${SchemaUtils.prettyFieldName(c.fieldPath)}'").mkString(", ")} |""".stripMargin } protected def getWidenedColumnsPrettyString(widenedColumns: Seq[TypeChange]): String = { s"""Columns with widened types: |${widenedColumns.map { case TypeChange(_, fromType, toType, fieldPath) => s"'${SchemaUtils.prettyFieldName(fieldPath)}': ${fromType.sql} -> ${toType.sql}" }.mkString("\n")} |""".stripMargin } } // Single types of schema change, typically caused by a single ALTER TABLE operation. private case class SchemaChangeRename(renamedColumns: Seq[RenamedColumn]) extends SchemaChangeType { override val name = "RENAME COLUMN" override val (isRename, isDrop, isTypeWidening) = (true, false, false) override val sqlConfsUnblock: Seq[String] = Seq(SQL_CONF_UNBLOCK_RENAME) override val readerOptionsUnblock: Seq[String] = Seq(DeltaOptions.ALLOW_SOURCE_COLUMN_RENAME) override val prettyColumnDetailsString: String = getRenamedColumnsPrettyString(renamedColumns) } private case class SchemaChangeDrop(droppedColumns: Seq[DroppedColumn]) extends SchemaChangeType { override val name = "DROP COLUMN" override val (isRename, isDrop, isTypeWidening) = (false, true, false) override val sqlConfsUnblock: Seq[String] = Seq(SQL_CONF_UNBLOCK_DROP) override val readerOptionsUnblock: Seq[String] = Seq(DeltaOptions.ALLOW_SOURCE_COLUMN_DROP) override val prettyColumnDetailsString: String = getDroppedColumnsPrettyString(droppedColumns) } private case class SchemaChangeTypeWidening(widenedColumns: Seq[TypeChange]) extends SchemaChangeType { override val name = "TYPE WIDENING" override val (isRename, isDrop, isTypeWidening) = (false, false, true) override val sqlConfsUnblock: Seq[String] = Seq(SQL_CONF_UNBLOCK_TYPE_CHANGE) override val readerOptionsUnblock: Seq[String] = Seq(DeltaOptions.ALLOW_SOURCE_COLUMN_TYPE_CHANGE) override val prettyColumnDetailsString: String = getWidenedColumnsPrettyString(widenedColumns) } // Combinations of rename, drop and type change -> can be caused by a complete overwrite. private case class SchemaChangeRenameAndDrop( renamedColumns: Seq[RenamedColumn], droppedColumns: Seq[DroppedColumn]) extends SchemaChangeType { override val name = "RENAME AND DROP COLUMN" override val (isRename, isDrop, isTypeWidening) = (true, true, false) override val sqlConfsUnblock: Seq[String] = Seq(SQL_CONF_UNBLOCK_RENAME_DROP) override val readerOptionsUnblock: Seq[String] = Seq(DeltaOptions.ALLOW_SOURCE_COLUMN_RENAME, DeltaOptions.ALLOW_SOURCE_COLUMN_DROP) override val prettyColumnDetailsString: String = getRenamedColumnsPrettyString(renamedColumns) + getDroppedColumnsPrettyString(droppedColumns) } private case class SchemaChangeRenameAndTypeWidening( renamedColumns: Seq[RenamedColumn], widenedColumns: Seq[TypeChange]) extends SchemaChangeType { override val name = "RENAME AND TYPE WIDENING" override val (isRename, isDrop, isTypeWidening) = (true, false, true) override val sqlConfsUnblock: Seq[String] = Seq(SQL_CONF_UNBLOCK_RENAME, SQL_CONF_UNBLOCK_TYPE_CHANGE) override val readerOptionsUnblock: Seq[String] = Seq(DeltaOptions.ALLOW_SOURCE_COLUMN_RENAME, DeltaOptions.ALLOW_SOURCE_COLUMN_DROP) override val prettyColumnDetailsString: String = getRenamedColumnsPrettyString(renamedColumns) + getWidenedColumnsPrettyString(widenedColumns) } private case class SchemaChangeDropAndTypeWidening( droppedColumns: Seq[DroppedColumn], widenedColumns: Seq[TypeChange]) extends SchemaChangeType { override val name = "DROP AND TYPE WIDENING" override val (isRename, isDrop, isTypeWidening) = (false, true, true) override val sqlConfsUnblock: Seq[String] = Seq(SQL_CONF_UNBLOCK_DROP, SQL_CONF_UNBLOCK_TYPE_CHANGE) override val readerOptionsUnblock: Seq[String] = Seq(DeltaOptions.ALLOW_SOURCE_COLUMN_DROP, DeltaOptions.ALLOW_SOURCE_COLUMN_TYPE_CHANGE) override val prettyColumnDetailsString: String = getDroppedColumnsPrettyString(droppedColumns) + getWidenedColumnsPrettyString(widenedColumns) } private case class SchemaChangeRenameAndDropAndTypeWidening( renamedColumns: Seq[RenamedColumn], droppedColumns: Seq[DroppedColumn], widenedColumns: Seq[TypeChange]) extends SchemaChangeType { override val name = "RENAME, DROP AND TYPE WIDENING" override val (isRename, isDrop, isTypeWidening) = (true, true, true) override val sqlConfsUnblock: Seq[String] = Seq(SQL_CONF_UNBLOCK_RENAME_DROP, SQL_CONF_UNBLOCK_TYPE_CHANGE) override val readerOptionsUnblock: Seq[String] = Seq(DeltaOptions.ALLOW_SOURCE_COLUMN_DROP, DeltaOptions.ALLOW_SOURCE_COLUMN_TYPE_CHANGE) override val prettyColumnDetailsString: String = getRenamedColumnsPrettyString(renamedColumns) + getDroppedColumnsPrettyString(droppedColumns) + getWidenedColumnsPrettyString(widenedColumns) } /** * Build the final schema change descriptor after analyzing all possible schema changes. * @param renamedColumns The columns that have been renamed. * @param droppedColumns The columns that have been dropped. * @param widenedColumns The columns that have been widened. */ private def buildSchemaChangeDescriptor( renamedColumns: Seq[RenamedColumn], droppedColumns: Seq[DroppedColumn], widenedColumns: Seq[TypeChange]): Option[SchemaChangeType] = { (renamedColumns.nonEmpty, droppedColumns.nonEmpty, widenedColumns.nonEmpty) match { case (true, false, false) => Some(SchemaChangeRename(renamedColumns)) case (false, true, false) => Some(SchemaChangeDrop(droppedColumns)) case (false, false, true) => Some(SchemaChangeTypeWidening(widenedColumns)) case (true, true, false) => Some(SchemaChangeRenameAndDrop(renamedColumns, droppedColumns)) case (true, false, true) => Some(SchemaChangeRenameAndTypeWidening(renamedColumns, widenedColumns)) case (false, true, true) => Some(SchemaChangeDropAndTypeWidening(droppedColumns, widenedColumns)) case (true, true, true) => Some( SchemaChangeRenameAndDropAndTypeWidening(renamedColumns, droppedColumns, widenedColumns)) case _ => None } } /** * Determine the non-additive schema change type for an incoming schema change. None if it's * additive. */ private def determineNonAdditiveSchemaChangeType( spark: SparkSession, newSchema: StructType, oldSchema: StructType): Option[SchemaChangeType] = { val renamedColumns = DeltaColumnMapping.collectRenamedColumns(newSchema, oldSchema) val droppedColumns = DeltaColumnMapping.collectDroppedColumns(newSchema, oldSchema) // Use physical column names to identify type changes. Dropping a column and adding a new column // with a different type is historically allowed and is not considered a type change. val oldPhysicalSchema = DeltaColumnMapping.renameColumns(oldSchema) val newPhysicalSchema = DeltaColumnMapping.renameColumns(newSchema) // Check if there are widening type changes. This assumes [[checkIncompatibleSchemaChange]] was // already called before and failed if there were any non-widening type changes. The type change // checks - both widening and non-widening - can be disabled by flag to revert to historical // behavior where type changes are not considered a non-additive schema change and are allowed // to propagate without user action. val typeWideningChanges = if (allowTypeWidening(spark) && !bypassTypeChangeCheck(spark)) { TypeWideningMetadata.collectTypeChanges(oldPhysicalSchema, newPhysicalSchema) } else Seq.empty buildSchemaChangeDescriptor(renamedColumns, droppedColumns, typeWideningChanges) } /** * Returns whether the given type of non-additive schema change was unblocked by setting one of * the corresponding SQL confs or reader options. */ private def isChangeUnblocked( spark: SparkSession, change: SchemaChangeType, options: DeltaOptions, checkpointHash: Int, schemaChangeVersion: Long): Boolean = { def isUnblockedBySQLConf(sqlConf: String): Boolean = { def getConf(key: String): Option[String] = Option(spark.sessionState.conf.getConfString(key, null)) .map(_.toLowerCase(Locale.ROOT)) val validConfKeysValuePair = Seq( (sqlConf, "always"), (s"$sqlConf.ckpt_$checkpointHash", "always"), (s"$sqlConf.ckpt_$checkpointHash", schemaChangeVersion.toString) ) validConfKeysValuePair.exists(p => getConf(p._1).contains(p._2)) } def isUnblockedByReaderOption(readerOption: Option[String]): Boolean = { readerOption.contains("always") || readerOption.contains(schemaChangeVersion.toString) } val isBlockedRename = change.isRename && !isUnblockedByReaderOption(options.allowSourceColumnRename) && !isUnblockedBySQLConf(SQL_CONF_UNBLOCK_RENAME) && !isUnblockedBySQLConf(SQL_CONF_UNBLOCK_RENAME_DROP) val isBlockedDrop = change.isDrop && !isUnblockedByReaderOption(options.allowSourceColumnDrop) && !isUnblockedBySQLConf(SQL_CONF_UNBLOCK_DROP) && !isUnblockedBySQLConf(SQL_CONF_UNBLOCK_RENAME_DROP) val isBlockedTypeChange = change.isTypeWidening && !isUnblockedByReaderOption(options.allowSourceColumnTypeChange) && !isUnblockedBySQLConf(SQL_CONF_UNBLOCK_TYPE_CHANGE) !isBlockedRename && !isBlockedDrop && !isBlockedTypeChange } def getCheckpointHash(path: String): Int = path.hashCode /** * Whether to accept widening type changes: * - when true, widening type changes cause the stream to fail, requesting user to review and * unblock them via a SQL conf or reader option. * - when false, widening type changes are rejected without possibility to unblock, similar to * any other arbitrary type change. */ def allowTypeWidening(spark: SparkSession): Boolean = { spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_ALLOW_TYPE_WIDENING_STREAMING_SOURCE) } /** * We historically allowed any type changes to go through when schema tracking was enabled. This * config allows reverting to that behavior. */ def bypassTypeChangeCheck(spark: SparkSession): Boolean = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_TYPE_WIDENING_BYPASS_STREAMING_TYPE_CHANGE_CHECK) // scalastyle:off /** * Given a non-additive operation type from a previous schema evolution, check we can process * using the new schema given any SQL conf or dataframe reader option users have explicitly set to * unblock. * The SQL conf can take one of following formats: * 1. spark.databricks.delta.streaming.allowSourceColumn$action = "always" * -> allows non-additive schema change to propagate for all streams. * 2. spark.databricks.delta.streaming.allowSourceColumn$action.$checkpointHash = "always" * -> allows non-additive schema change to propagate for this particular stream. * 3. spark.databricks.delta.streaming.allowSourceColumn$action.$checkpointHash = $deltaVersion * -> allow non-additive schema change to propagate only for this particular stream source * table version. * The reader options can take one of the following format: * 1. .option("allowSourceColumn$action", "always") * -> allows non-additive schema change to propagate for this particular stream. * 2. .option("allowSourceColumn$action", "$deltaVersion") * -> allow non-additive schema change to propagate only for this particular stream source * table version. * where `allowSourceColumn$action` is one of: * 1. `allowSourceColumnRename` to allow column renames. * 2. `allowSourceColumnDrop` to allow column drops. * 3. `allowSourceColumnTypeChange` to allow widening type changes. * For SQL confs only, action can also be `allowSourceColumnRenameAndDrop` to allow both column * drops and renames. * * We will check for any of these configs given the non-additive operation, and throw a proper * error message to instruct the user to set the SQL conf / reader options if they would like to * unblock. * * @param metadataPath The path to the source-unique metadata location under checkpoint * @param currentSchema The current persisted schema * @param previousSchema The previous persisted schema */ // scalastyle:on protected[sources] def validateIfSchemaChangeCanBeUnblocked( spark: SparkSession, parameters: Map[String, String], metadataPath: String, currentSchema: PersistedMetadata, previousSchema: PersistedMetadata): Unit = { val options = new DeltaOptions(parameters, spark.sessionState.conf) val checkpointHash = getCheckpointHash(metadataPath) // The start version of a possible series of consecutive schema changes. val previousSchemaChangeVersion = previousSchema.deltaCommitVersion // The end version of a possible series of consecutive schema changes. val currentSchemaChangeVersion = currentSchema.deltaCommitVersion // Fail with a non-retryable exception if there are any type changes that we don't allow // unblocking, i.e. non-widening type changes. We do allow changes caused by columns being // dropped/renamed, e.g. dropping a column and adding it back with a different type. These were // historically allowed and will be surfaced to the user as column drop/rename. checkIncompatibleSchemaChange( spark, previousSchema = previousSchema.dataSchema, currentSchema = currentSchema.dataSchema, currentSchemaChangeVersion ) determineNonAdditiveSchemaChangeType( spark, currentSchema.dataSchema, previousSchema.dataSchema).foreach { change => if (!isChangeUnblocked( spark, change, options, checkpointHash, currentSchemaChangeVersion)) { // Throw error to prompt user to set the correct confs throw DeltaErrors.cannotContinueStreamingPostSchemaEvolution( change.name, previousSchemaChangeVersion, currentSchemaChangeVersion, checkpointHash, change.readerOptionsUnblock, change.sqlConfsUnblock, change.prettyColumnDetailsString) } } } /** * Checks that the new schema only contains column rename/drop and widening type changes compared * to the previous schema. That is, rejects any non-widening type changes. */ private def checkIncompatibleSchemaChange( spark: SparkSession, previousSchema: StructType, currentSchema: StructType, currentSchemaChangeVersion: Long): Unit = { if (bypassTypeChangeCheck(spark)) return val incompatibleSchema = !SchemaUtils.isReadCompatible( // We want to ignore renamed/dropped columns here and let the check for non-additive // schema changes handle them: we only check if an actual physical column had an // incompatible type change. existingSchema = DeltaColumnMapping.renameColumns(previousSchema), readSchema = DeltaColumnMapping.renameColumns(currentSchema), forbidTightenNullability = true, allowMissingColumns = true, typeWideningMode = if (allowTypeWidening(spark)) TypeWideningMode.AllTypeWidening else TypeWideningMode.NoTypeWidening ) if (incompatibleSchema) { throw DeltaErrors.schemaChangedException( previousSchema, currentSchema, retryable = false, Some(currentSchemaChangeVersion), includeStartingVersionOrTimestampMessage = false) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSourceMetadataTrackingLog.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.sources // scalastyle:off import.ordering.noEmptyLine import java.io.InputStream import scala.collection.JavaConverters._ import scala.util.control.NonFatal import org.apache.spark.sql.delta.streaming.{JsonSchemaSerializer, PartitionAndDataSchema, SchemaTrackingLog} import org.apache.spark.sql.delta.{DeltaErrors, DeltaLog, DeltaOptions, SnapshotDescriptor} import org.apache.spark.sql.delta.actions.{Action, FileAction, Metadata, Protocol} import org.apache.spark.sql.delta.storage.ClosableIterator._ import org.apache.spark.sql.delta.util.JsonUtils import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.databind.annotation.JsonDeserialize import org.apache.hadoop.fs.Path import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.types.{DataType, StructType} import org.apache.spark.sql.util.CaseInsensitiveStringMap // scalastyle:on import.ordering.noEmptyLine /** * A [[PersistedMetadata]] is an entry in Delta streaming source schema log, which can be used to * read data files during streaming. * * @param tableId Delta table id * @param deltaCommitVersion Delta commit version in which this change is captured. It does not * necessarily have to be the commit when there's an actual change, e.g. * during initialization. * The invariant is that the metadata must be read-compatible with the * table snapshot at this version. * @param dataSchemaJson Full schema json * @param partitionSchemaJson Partition schema json * @param sourceMetadataPath The checkpoint path that is unique to each source. * @param tableConfigurations The configurations of the table inside the metadata when the schema * change was detected. It is used to correctly create the right file * format when we use a particular schema to read. * Default to None for backward compatibility. * @param protocolJson JSON of the protocol change if any. * Default to None for backward compatibility. * @param previousMetadataSeqNum When defined, it points to the batch ID / seq num for the previous * metadata in the log sequence. It is used when we could not reliably * tell if the currentBatchId - 1 is indeed the previous schema evolution, * e.g. when we are merging consecutive schema changes during the analysis * phase and we are appending an extra schema after the merge to the log. * Default to None for backward compatibility. */ case class PersistedMetadata( tableId: String, deltaCommitVersion: Long, dataSchemaJson: String, partitionSchemaJson: String, sourceMetadataPath: String, tableConfigurations: Option[Map[String, String]] = None, protocolJson: Option[String] = None, @JsonDeserialize(contentAs = classOf[java.lang.Long]) previousMetadataSeqNum: Option[Long] = None) extends PartitionAndDataSchema { private def parseSchema(schemaJson: String): StructType = { try { DataType.fromJson(schemaJson).asInstanceOf[StructType] } catch { case NonFatal(_) => throw DeltaErrors.failToParseSchemaLog } } @JsonIgnore lazy val dataSchema: StructType = parseSchema(dataSchemaJson) @JsonIgnore lazy val partitionSchema: StructType = parseSchema(partitionSchemaJson) @JsonIgnore lazy val protocol: Option[Protocol] = protocolJson.map(Action.fromJson).map(_.asInstanceOf[Protocol]) def validateAgainstSnapshot(snapshot: SnapshotDescriptor): Unit = { if (snapshot.deltaLog.unsafeVolatileTableId != tableId) { throw DeltaErrors.incompatibleSchemaLogDeltaTable( tableId, snapshot.deltaLog.unsafeVolatileTableId) } } } object PersistedMetadata { val VERSION = 1 val EMPTY_JSON = "{}" def fromJson(json: String): PersistedMetadata = JsonUtils.fromJson[PersistedMetadata](json) def apply( tableId: String, deltaCommitVersion: Long, metadata: Metadata, protocol: Protocol, sourceMetadataPath: String): PersistedMetadata = { PersistedMetadata(tableId, deltaCommitVersion, metadata.schema.json, metadata.partitionSchema.json, // The schema is bound to the specific source sourceMetadataPath, // Table configurations come from the Metadata action Some(metadata.configuration), Some(protocol.json) ) } } /** * Tracks the metadata changes for a particular Delta streaming source in a particular stream, * it is utilized to save and lookup the correct metadata during streaming from a Delta table. * This schema log is NOT meant to be shared across different Delta streaming source instances. * * @param rootMetadataLocation Metadata log location * @param sourceSnapshot Delta source snapshot for the Delta streaming source * @param sourceMetadataPathOpt The source metadata path that is used during streaming execution. * @param initMetadataLogEagerly If true, initialize metadata log as early as possible, otherwise, * initialize only when detecting non-additive schema change. */ class DeltaSourceMetadataTrackingLog private( sparkSession: SparkSession, rootMetadataLocation: String, sourceSnapshot: SnapshotDescriptor, sourceMetadataPathOpt: Option[String] = None, val initMetadataLogEagerly: Boolean = true) { import org.apache.spark.sql.delta.streaming.SchemaTrackingExceptions._ protected val schemaSerializer = new JsonSchemaSerializer[PersistedMetadata](PersistedMetadata.VERSION) { override def deserialize(in: InputStream): PersistedMetadata = try super.deserialize(in) catch { case FailedToDeserializeException => throw DeltaErrors.failToDeserializeSchemaLog(rootMetadataLocation) } } protected val trackingLog = new SchemaTrackingLog[PersistedMetadata]( sparkSession, rootMetadataLocation, schemaSerializer) // Validate schema at log init trackingLog.getCurrentTrackedSchema.foreach(_.validateAgainstSnapshot(sourceSnapshot)) /** * Get the global latest metadata for this metadata location. * Visible for testing */ private[delta] def getLatestMetadata: Option[PersistedMetadata] = trackingLog.getLatest().map(_._2) /** * Get the current schema that is being tracked by this schema log. This is typically the latest * schema log entry to the best of this schema log's knowledge. */ def getCurrentTrackedMetadata: Option[PersistedMetadata] = trackingLog.getCurrentTrackedSchema /** * Get the current tracked seq num by this schema log or -1 if no schema has been tracked yet. */ def getCurrentTrackedSeqNum: Long = trackingLog.getCurrentTrackedSeqNum /** * Get the logically-previous tracked seq num by this schema log. * Considering the prev pointer from the latest entry if defined. */ private def getPreviousTrackedSeqNum: Long = { getCurrentTrackedMetadata.flatMap(_.previousMetadataSeqNum) match { case Some(previousSeqNum) => previousSeqNum case None => trackingLog.getCurrentTrackedSeqNum - 1 } } /** * Get the logically-previous tracked schema entry by this schema log. * DeltaSource requires it to compare the previous schema with the latest schema to determine if * an automatic stream restart is allowed. */ def getPreviousTrackedMetadata: Option[PersistedMetadata] = trackingLog.getTrackedSchemaAtSeqNum(getPreviousTrackedSeqNum) /** * Track a new schema to the log. * * @param newMetadata The incoming new metadata with schema. * @param replaceCurrent If true, we will set a previous seq num pointer on the incoming metadata * change pointing to the previous seq num of the current latest metadata. * So that once the new metadata is written, getPreviousTrackedMetadata() * will return the updated reference. * If a previous metadata does not exist, this is noop. */ def writeNewMetadata( newMetadata: PersistedMetadata, replaceCurrent: Boolean = false): PersistedMetadata = { try { trackingLog.addSchemaToLog( if (replaceCurrent && getCurrentTrackedMetadata.isDefined) { newMetadata.copy(previousMetadataSeqNum = Some(getPreviousTrackedSeqNum)) } else newMetadata ) } catch { case FailedToEvolveSchema => throw DeltaErrors.sourcesWithConflictingSchemaTrackingLocation( rootMetadataLocation, sourceSnapshot.deltaLog.dataPath.toString) } } } object DeltaSourceMetadataTrackingLog extends Logging { def fullMetadataTrackingLocation( rootSchemaTrackingLocation: String, tableId: String, sourceTrackingId: Option[String] = None): String = { val subdir = s"_schema_log_$tableId" + sourceTrackingId.map(n => s"_$n").getOrElse("") new Path(rootSchemaTrackingLocation, subdir).toString } /** * Create a schema log instance for a schema location. * The schema location is constructed as `$rootMetadataLocation/_schema_log_$tableId` * a suffix of `_$sourceTrackingId` is appended if provided to further differentiate the sources. * * @param mergeConsecutiveSchemaChanges Defined during analysis phase. * @param sourceMetadataPathOpt Defined during execution phase. */ def create( sparkSession: SparkSession, rootMetadataLocation: String, sourceSnapshot: SnapshotDescriptor, catalogTableOpt: Option[CatalogTable], parameters: Map[String, String], sourceMetadataPathOpt: Option[String] = None, mergeConsecutiveSchemaChanges: Boolean = false, initMetadataLogEagerly: Boolean = true): DeltaSourceMetadataTrackingLog = { val options = new CaseInsensitiveStringMap(parameters.asJava) val sourceTrackingId = Option(options.get(DeltaOptions.STREAMING_SOURCE_TRACKING_ID)) val metadataTrackingLocation = fullMetadataTrackingLocation( rootMetadataLocation, sourceSnapshot.deltaLog.unsafeVolatileTableId, sourceTrackingId) val log = new DeltaSourceMetadataTrackingLog( sparkSession, metadataTrackingLocation, sourceSnapshot, sourceMetadataPathOpt, initMetadataLogEagerly ) // During initialize schema log, validate against: // 1. table snapshot to check for partition and tahoe id mismatch // 2. source metadata path to ensure we are not using the wrong schema log for the source log.getCurrentTrackedMetadata.foreach { schema => schema.validateAgainstSnapshot(sourceSnapshot) if (sparkSession.sessionState.conf.getConf( DeltaSQLConf.DELTA_STREAMING_SCHEMA_TRACKING_METADATA_PATH_CHECK_ENABLED)) { sourceMetadataPathOpt.foreach { metadataPath => require(metadataPath == schema.sourceMetadataPath, s"The Delta source metadata path used for execution '${metadataPath}' is different " + s"from the one persisted for previous processing '${schema.sourceMetadataPath}'. " + s"Please check if the schema location has been reused across different streaming " + s"sources. Pick a new `${DeltaOptions.SCHEMA_TRACKING_LOCATION}` or use " + s"`${DeltaOptions.STREAMING_SOURCE_TRACKING_ID}` to " + s"distinguish between streaming sources.") } } } // The consecutive schema merging logic is run in the *analysis* phase, when we figure the final // schema to read for the streaming dataframe. if (mergeConsecutiveSchemaChanges && log.getCurrentTrackedMetadata.isDefined) { // If enable schema merging, skim ahead on consecutive schema changes and use the latest one // to update the log again if possible. // We add the prev pointer to the merged schema so that SQL conf validation logic later can // reliably fetch the previous read schema and the latest schema and then be able to determine // if it's OK for the stream to proceed. getMergedConsecutiveMetadataChanges( sparkSession, sourceSnapshot.deltaLog, catalogTableOpt, log.getCurrentTrackedMetadata.get ).foreach { mergedSchema => log.writeNewMetadata(mergedSchema, replaceCurrent = true) } } // The validation is ran in *execution* phase where the metadata path becomes available. // While loading the current persisted schema, validate against previous persisted schema // to check if the stream can move ahead with the custom SQL conf. (log.getPreviousTrackedMetadata, log.getCurrentTrackedMetadata, sourceMetadataPathOpt) match { case (Some(prev), Some(curr), Some(metadataPath)) => DeltaSourceMetadataEvolutionSupport .validateIfSchemaChangeCanBeUnblocked( sparkSession, parameters, metadataPath, curr, prev) case _ => } log } /** * Speculate ahead and find the next merged consecutive metadata change if possible. * A metadata change is either: * 1. A [[Metadata]] action change. OR * 2. A [[Protocol]] change. */ private def getMergedConsecutiveMetadataChanges( spark: SparkSession, deltaLog: DeltaLog, catalogTableOpt: Option[CatalogTable], currentMetadata: PersistedMetadata): Option[PersistedMetadata] = { val currentMetadataVersion = currentMetadata.deltaCommitVersion // We start from the currentSchemaVersion so that we can stop early in case the current // version still has file actions that potentially needs to be processed. val untilMetadataChange = deltaLog.getChangeLogFiles( currentMetadataVersion, catalogTableOpt).map { case (version, fileStatus) => var metadataAction: Option[Metadata] = None var protocolAction: Option[Protocol] = None var hasFileAction = false DeltaSource.createRewindableActionIterator(spark, deltaLog, fileStatus) .processAndClose { actionsIter => actionsIter.foreach { case m: Metadata => metadataAction = Some(m) case p: Protocol => protocolAction = Some(p) case _: FileAction => hasFileAction = true case _ => } } (!hasFileAction && (metadataAction.isDefined || protocolAction.isDefined), version, metadataAction, protocolAction) }.takeWhile(_._1) DeltaSource.iteratorLast(untilMetadataChange.toClosable) .flatMap { case (_, version, metadataOpt, protocolOpt) => if (version == currentMetadataVersion) { None } else { log.info(s"Looked ahead from version $currentMetadataVersion and " + s"will use metadata at version $version to read Delta stream.") Some( currentMetadata.copy( deltaCommitVersion = version, dataSchemaJson = metadataOpt.map(_.schema.json).getOrElse(currentMetadata.dataSchemaJson), partitionSchemaJson = metadataOpt.map(_.partitionSchema.json) .getOrElse(currentMetadata.partitionSchemaJson), tableConfigurations = metadataOpt.map(_.configuration) .orElse(currentMetadata.tableConfigurations), protocolJson = protocolOpt.map(_.json).orElse(currentMetadata.protocolJson) ) ) } } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSourceOffset.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.sources // scalastyle:off import.ordering.noEmptyLine import java.io.IOException import org.apache.spark.sql.delta.{DeltaErrors, DeltaLog} import org.apache.spark.sql.delta.util.JsonUtils import com.fasterxml.jackson.core.{JsonGenerator, JsonParseException, JsonParser, JsonProcessingException} import com.fasterxml.jackson.databind.{DeserializationContext, SerializerProvider} import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize} import com.fasterxml.jackson.databind.deser.std.StdDeserializer import com.fasterxml.jackson.databind.exc.InvalidFormatException import com.fasterxml.jackson.databind.ser.std.StdSerializer import org.apache.spark.internal.Logging import org.apache.spark.sql.connector.read.streaming.{Offset => OffsetV2} import org.apache.spark.sql.execution.streaming.Offset /** * Tracks how far we processed in when reading changes from the [[DeltaLog]]. * * Note this class retains the naming of `Reservoir` to maintain compatibility * with serialized offsets from the beta period. * * @param reservoirId The id of the table we are reading from. Used to detect * misconfiguration when restarting a query. * @param reservoirVersion The version of the table that we are current processing. * @param index The index in the sequence of AddFiles in this version. Used to * break large commits into multiple batches. This index is created by * sorting on modificationTimestamp and path. * @param isInitialSnapshot Whether this offset points into an initial full table snapshot at the * provided reservoir version rather than into the changes at that version. * When starting a new query, we first process all data present in the * table at the start and then move on to processing new data that has * arrived. */ @JsonDeserialize(using = classOf[DeltaSourceOffset.Deserializer]) @JsonSerialize(using = classOf[DeltaSourceOffset.Serializer]) case class DeltaSourceOffset private( reservoirId: String, reservoirVersion: Long, index: Long, isInitialSnapshot: Boolean ) extends Offset with Comparable[DeltaSourceOffset] { import DeltaSourceOffset._ assert(index != -1, "Index should never be -1, it should be set to the BASE_INDEX instead.") override def json: String = { JsonUtils.toJson(this) } /** * Compare two DeltaSourceOffsets which are on the same table. * @return 0 for equivalent offsets. negative if this offset is less than `otherOffset`. Positive * if this offset is greater than `otherOffset` */ def compare(otherOffset: DeltaSourceOffset): Int = { assert(reservoirId == otherOffset.reservoirId, "Comparing offsets that do not refer to the" + " same table is disallowed.") implicitly[Ordering[(Long, Long)]].compare((reservoirVersion, index), (otherOffset.reservoirVersion, otherOffset.index)) } override def compareTo(o: DeltaSourceOffset): Int = { compare(o) } } object DeltaSourceOffset extends Logging { private[DeltaSourceOffset] val VERSION_1 = 1 private[DeltaSourceOffset] val VERSION_2 = 2 // reserved // Serialization version 3 adds support for schema change index values. private[DeltaSourceOffset] val VERSION_3 = 3 private[DeltaSourceOffset] val CURRENT_VERSION = VERSION_3 // The base index within each reservoirVersion. This offset indicates the offset before all // changes in the reservoirVersion. All other offsets within the reservoirVersion have an index // that is higher than the base index. // // This index is for VERSION_3+. Unless there are other fields that force the version to be >=3, // it should NOT be serialized into offset log for backward compatibility. Instead, we serialize // this as INDEX_VERSION_BASE_V1, and set source version lower accordingly. It gets converted back // to the VERSION_3 value at deserialization time, so that we only use the V3 value in memory. private[DeltaSourceOffset] val BASE_INDEX_V3: Long = -100 // The V1 base index that should be serialized into the offset log private[DeltaSourceOffset] val BASE_INDEX_V1: Long = -1 // The base index version clients of DeltaSourceOffset should use val BASE_INDEX: Long = BASE_INDEX_V3 // The index for an IndexedFile that also contains a metadata change. (from VERSION_3) val METADATA_CHANGE_INDEX: Long = -20 // The index for an IndexedFile that is right after a metadata change. (from VERSION_3) val POST_METADATA_CHANGE_INDEX: Long = -19 // A value close to the end of the Long space. This is used to indicate that we are at the end of // a reservoirVersion and need to move on to the next one. This should never be serialized into // the offset log. val END_INDEX: Long = Long.MaxValue - 100 /** * The ONLY external facing constructor to create a DeltaSourceOffset in memory. * @param reservoirId Table id * @param reservoirVersion Table commit version * @param index File action index in the commit version * @param isInitialSnapshot Whether this offset is still in initial snapshot */ def apply( reservoirId: String, reservoirVersion: Long, index: Long, isInitialSnapshot: Boolean ): DeltaSourceOffset = { // TODO should we detect `reservoirId` changes when a query is running? new DeltaSourceOffset( reservoirId, reservoirVersion, index, isInitialSnapshot ) } /** * Validate and parse a DeltaSourceOffset from its JSON serialized format * @param reservoirId Table id * @param json Raw JSON string */ def apply(reservoirId: String, json: String): DeltaSourceOffset = { val o = JsonUtils.mapper.readValue[DeltaSourceOffset](json) if (o.reservoirId != reservoirId) { throw DeltaErrors.differentDeltaTableReadByStreamingSource( newTableId = reservoirId, oldTableId = o.reservoirId) } o } /** * Validate and parse a DeltaSourceOffset from its serialized format * @param reservoirId Table id * @param offset Raw streaming offset */ def apply(reservoirId: String, offset: OffsetV2): DeltaSourceOffset = { offset match { case o: DeltaSourceOffset => o case s => apply(reservoirId, s.json) } } /** * Validate offsets to make sure we always move forward. Moving backward may make the query * re-process data and cause data duplication. */ def validateOffsets(previousOffset: DeltaSourceOffset, currentOffset: DeltaSourceOffset): Unit = { if (previousOffset.isInitialSnapshot == false && currentOffset.isInitialSnapshot == true) { throw new IllegalStateException( s"Found invalid offsets: 'isInitialSnapshot' flipped incorrectly. " + s"Previous: $previousOffset, Current: $currentOffset") } if (previousOffset.reservoirVersion > currentOffset.reservoirVersion) { throw new IllegalStateException( s"Found invalid offsets: 'reservoirVersion' moved back. " + s"Previous: $previousOffset, Current: $currentOffset") } if (previousOffset.reservoirVersion == currentOffset.reservoirVersion && previousOffset.index > currentOffset.index) { throw new IllegalStateException( s"Found invalid offsets. 'index' moved back. " + s"Previous: $previousOffset, Current: $currentOffset") } } def isMetadataChangeIndex(index: Long): Boolean = index == METADATA_CHANGE_INDEX || index == POST_METADATA_CHANGE_INDEX /** * This is a 1:1 copy of [[DeltaSourceOffset]] used for JSON serialization. Our serializers only * want to adjust some field values and then serialize in the normal way. But we cannot access the * "default" serializers once we've overridden them. So instead, we use a separate case class that * gets serialized "as-is". */ private case class DeltaSourceOffsetForSerialization private( sourceVersion: Long, reservoirId: String, reservoirVersion: Long, index: Long, // This stores isInitialSnapshot. // This was confusingly called "starting version" in earlier versions, even though enabling // the option "startingVersion" actually causes this to be disabled. We still have to // serialize it using the old name for backward compatibility. isStartingVersion: Boolean ) class Deserializer extends StdDeserializer[DeltaSourceOffset](classOf[DeltaSourceOffset]) { @throws[IOException] @throws[JsonProcessingException] override def deserialize(p: JsonParser, ctxt: DeserializationContext): DeltaSourceOffset = { val o = try { p.readValueAs(classOf[DeltaSourceOffsetForSerialization]) } catch { case e: Throwable if e.isInstanceOf[JsonParseException] || e.isInstanceOf[InvalidFormatException] => // The version may be there with a different format, or something else might be off. throw DeltaErrors.invalidSourceOffsetFormat() } if (o.sourceVersion < VERSION_1) { throw DeltaErrors.invalidSourceVersion(o.sourceVersion.toString) } if (o.sourceVersion > CURRENT_VERSION) { throw DeltaErrors.invalidFormatFromSourceVersion(o.sourceVersion, CURRENT_VERSION) } if (o.sourceVersion == VERSION_2) { // Version 2 is reserved. throw DeltaErrors.invalidSourceVersion(o.sourceVersion.toString) } // Always upgrade to use the current latest INDEX_VERSION_BASE val offsetIndex = if (o.sourceVersion < VERSION_3 && o.index == BASE_INDEX_V1) { logDebug(s"upgrading offset to use latest version base index") BASE_INDEX } else { o.index } assert(offsetIndex != END_INDEX, "Should not deserialize END_INDEX") // Leverage the only external facing constructor to initialize with latest sourceVersion DeltaSourceOffset( reservoirId = o.reservoirId, reservoirVersion = o.reservoirVersion, index = offsetIndex, isInitialSnapshot = o.isStartingVersion ) } } class Serializer extends StdSerializer[DeltaSourceOffset](classOf[DeltaSourceOffset]) { @throws[IOException] override def serialize( o: DeltaSourceOffset, gen: JsonGenerator, provider: SerializerProvider): Unit = { assert(o.index != END_INDEX, "Should not serialize END_INDEX") // We handle a few backward compatibility scenarios during Serialization here: // 1. [Backward compatibility] If the source index is a schema changing base index, then // replace it with index = -1 and use VERSION_1. This allows older Delta to at least be // able to read the non-schema-changes stream offsets. // This needs to happen during serialization time so we won't be looking at a downgraded // index right away when we need to utilize this offset in memory. // 2. [Backward safety] If the source index is a new schema changing index, then use // VERSION_3. Older Delta would explode upon seeing this, but that's the safe thing to do. val minVersion = { if (DeltaSourceOffset.isMetadataChangeIndex(o.index)) { VERSION_3 } else { VERSION_1 } } val downgradedIndex = if (o.index == BASE_INDEX) { BASE_INDEX_V1 } else { o.index } gen.writeObject(DeltaSourceOffsetForSerialization( sourceVersion = minVersion, reservoirId = o.reservoirId, reservoirVersion = o.reservoirVersion, index = downgradedIndex, isStartingVersion = o.isInitialSnapshot )) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSourceUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.sources import java.util.Locale import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.sources import org.apache.spark.sql.sources.Filter object DeltaSourceUtils { val NAME = "delta" val ALT_NAME = "delta" // Batch relations don't pass partitioning columns to `CreatableRelationProvider`s, therefore // as a hack, we pass in the partitioning columns among the options. val PARTITIONING_COLUMNS_KEY = "__partition_columns" // The metadata key recording the generation expression in a generated column's `StructField`. val GENERATION_EXPRESSION_METADATA_KEY = "delta.generationExpression" val IDENTITY_INFO_ALLOW_EXPLICIT_INSERT = "delta.identity.allowExplicitInsert" val IDENTITY_INFO_START = "delta.identity.start" val IDENTITY_INFO_STEP = "delta.identity.step" val IDENTITY_INFO_HIGHWATERMARK = "delta.identity.highWaterMark" val IDENTITY_COMMITINFO_TAG = "delta.identity.schemaUpdate" def isDeltaDataSourceName(name: String): Boolean = { name.toLowerCase(Locale.ROOT) == NAME || name.toLowerCase(Locale.ROOT) == ALT_NAME } /** Check whether this table is a Delta table based on information from the Catalog. */ def isDeltaTable(provider: Option[String]): Boolean = { provider.exists(isDeltaDataSourceName) } /** Creates Spark literals from a value exposed by the public Spark API. */ private def createLiteral(value: Any): expressions.Literal = value match { case v: String => expressions.Literal.create(v) case v: Int => expressions.Literal.create(v) case v: Byte => expressions.Literal.create(v) case v: Short => expressions.Literal.create(v) case v: Long => expressions.Literal.create(v) case v: Double => expressions.Literal.create(v) case v: Float => expressions.Literal.create(v) case v: Boolean => expressions.Literal.create(v) case v: java.sql.Date => expressions.Literal.create(v) case v: java.sql.Timestamp => expressions.Literal.create(v) case v: java.time.Instant => expressions.Literal.create(v) case v: java.time.LocalDate => expressions.Literal.create(v) case v: BigDecimal => expressions.Literal.create(v) } /** Translates the public Spark Filter APIs into Spark internal expressions. */ def translateFilters(filters: Array[Filter]): Expression = filters.map { case sources.EqualTo(attribute, value) => expressions.EqualTo(UnresolvedAttribute(attribute), expressions.Literal.create(value)) case sources.EqualNullSafe(attribute, value) => expressions.EqualNullSafe(UnresolvedAttribute(attribute), expressions.Literal.create(value)) case sources.GreaterThan(attribute, value) => expressions.GreaterThan(UnresolvedAttribute(attribute), expressions.Literal.create(value)) case sources.GreaterThanOrEqual(attribute, value) => expressions.GreaterThanOrEqual( UnresolvedAttribute(attribute), expressions.Literal.create(value)) case sources.LessThan(attribute, value) => expressions.LessThan(UnresolvedAttribute(attribute), expressions.Literal.create(value)) case sources.LessThanOrEqual(attribute, value) => expressions.LessThanOrEqual(UnresolvedAttribute(attribute), expressions.Literal.create(value)) case sources.In(attribute, values) => expressions.In(UnresolvedAttribute(attribute), values.map(createLiteral)) case sources.IsNull(attribute) => expressions.IsNull(UnresolvedAttribute(attribute)) case sources.IsNotNull(attribute) => expressions.IsNotNull(UnresolvedAttribute(attribute)) case sources.Not(otherFilter) => expressions.Not(translateFilters(Array(otherFilter))) case sources.And(filter1, filter2) => expressions.And(translateFilters(Array(filter1)), translateFilters(Array(filter2))) case sources.Or(filter1, filter2) => expressions.Or(translateFilters(Array(filter1)), translateFilters(Array(filter2))) case sources.StringStartsWith(attribute, value) => new expressions.Like( UnresolvedAttribute(attribute), expressions.Literal.create(s"${value}%")) case sources.StringEndsWith(attribute, value) => new expressions.Like( UnresolvedAttribute(attribute), expressions.Literal.create(s"%${value}")) case sources.StringContains(attribute, value) => new expressions.Like( UnresolvedAttribute(attribute), expressions.Literal.create(s"%${value}%")) case sources.AlwaysTrue() => expressions.Literal.TrueLiteral case sources.AlwaysFalse() => expressions.Literal.FalseLiteral }.reduceOption(expressions.And).getOrElse(expressions.Literal.TrueLiteral) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaStreamUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.sources import java.sql.Timestamp import scala.collection.mutable import org.apache.hadoop.fs.Path import org.apache.spark.sql.delta.DataFrameUtils import org.apache.spark.sql.delta.DeltaErrors import org.apache.spark.sql.delta.Relocated._ import org.apache.spark.sql.delta.TypeWideningMode import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.util.{DateTimeUtils, TimestampFormatter} import org.apache.spark.sql.{Column, DataFrame, SparkSession} import org.apache.spark.sql.classic.ClassicConversions._ import org.apache.spark.sql.execution.QueryExecution import org.apache.spark.sql.types.StructType object DeltaStreamUtils { /** * Select `cols` from a micro batch DataFrame. Directly calling `select` won't work because it * will create a `QueryExecution` rather than inheriting `IncrementalExecution` from * the micro batch DataFrame. A streaming micro batch DataFrame to execute should use * `IncrementalExecution`. */ def selectFromStreamingDataFrame( incrementalExecution: IncrementalExecution, df: DataFrame, cols: Column*): DataFrame = { val newMicroBatch = df.select(cols: _*) val newIncrementalExecution = createIncrementalExecution( newMicroBatch.sparkSession, newMicroBatch.queryExecution.logical, incrementalExecution.outputMode, incrementalExecution.checkpointLocation, incrementalExecution.queryId, incrementalExecution.runId, incrementalExecution.currentBatchId, incrementalExecution.prevOffsetSeqMetadata, incrementalExecution.offsetSeqMetadata, incrementalExecution.watermarkPropagator, incrementalExecution.isFirstBatch) newIncrementalExecution.executedPlan // Force the lazy generation of execution plan DataFrameUtils.ofRows(newIncrementalExecution) } /** * Configuration options for schema compatibility validation during Delta streaming reads. * * This class encapsulates various flags and settings that control how Delta streaming handles * schema changes and compatibility checks. * * TODO(#5319): Clean up the configs that were intended as escape-hatches for behavior changes * if they aren't needed anymore. * * @param allowUnsafeStreamingReadOnColumnMappingSchemaChanges * Flag that allows user to force enable unsafe streaming read on Delta table with * column mapping enabled AND drop/rename actions. * @param allowUnsafeStreamingReadOnPartitionColumnChanges * Flag that allows user to force enable unsafe streaming read on Delta table with * column mapping enabled AND partition column changes. * @param forceEnableStreamingReadOnReadIncompatibleSchemaChangesDuringStreamStart * Flag that allows user to disable the read-compatibility check during stream start which * protects against a corner case in which verifyStreamHygiene could not detect. * This is a bug fix but yet a potential behavior change, so we add a flag to fallback. * @param forceEnableUnsafeReadOnNullabilityChange * Flag that allows user to fallback to the legacy behavior in which user can allow * nullable=false schema to read nullable=true data, which is incorrect but a behavior * change regardless. * @param isStreamingFromColumnMappingTable * Whether we are streaming from a table with column mapping enabled. * @param typeWideningEnabled * Whether we are streaming from a table that has the type widening table feature enabled. * @param enableSchemaTrackingForTypeWidening * Whether we should track widening type changes to allow users to accept them and resume * stream processing. */ case class SchemaReadOptions( allowUnsafeStreamingReadOnColumnMappingSchemaChanges: Boolean, allowUnsafeStreamingReadOnPartitionColumnChanges: Boolean, forceEnableStreamingReadOnReadIncompatibleSchemaChangesDuringStreamStart: Boolean, forceEnableUnsafeReadOnNullabilityChange: Boolean, isStreamingFromColumnMappingTable: Boolean, typeWideningEnabled: Boolean, enableSchemaTrackingForTypeWidening: Boolean ) object SchemaReadOptions { /** * Creates a SchemaReadOptions instance from SparkSession configuration settings. * * @param spark The SparkSession from which to read configuration values. * @param isStreamingFromColumnMappingTable Whether the source table has column mapping enabled. * @param isTypeWideningSupportedInProtocol Whether the table's protocol version supports * type widening. * @return A [[SchemaReadOptions]] instance containing all schema validation flags derived from * the session configuration and provided table state. */ def fromSparkSession( spark: SparkSession, isStreamingFromColumnMappingTable: Boolean, isTypeWideningSupportedInProtocol: Boolean): SchemaReadOptions = { val allowUnsafeStreamingReadOnColumnMappingSchemaChanges = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_COLUMN_MAPPING_SCHEMA_CHANGES) val allowUnsafeStreamingReadOnPartitionColumnChanges = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_STREAMING_UNSAFE_READ_ON_PARTITION_COLUMN_CHANGE) val forceEnableStreamingReadOnReadIncompatibleSchemaChangesDuringStreamStart = spark.sessionState.conf.getConf(DeltaSQLConf. DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_SCHEMA_CHANGES_DURING_STREAM_START) val forceEnableUnsafeReadOnNullabilityChange = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_STREAM_UNSAFE_READ_ON_NULLABILITY_CHANGE) val typeWideningEnabled = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_ALLOW_TYPE_WIDENING_STREAMING_SOURCE) && isTypeWideningSupportedInProtocol val enableSchemaTrackingForTypeWidening = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_TYPE_WIDENING_ENABLE_STREAMING_SCHEMA_TRACKING) new DeltaStreamUtils.SchemaReadOptions( allowUnsafeStreamingReadOnColumnMappingSchemaChanges = allowUnsafeStreamingReadOnColumnMappingSchemaChanges, allowUnsafeStreamingReadOnPartitionColumnChanges = allowUnsafeStreamingReadOnPartitionColumnChanges, forceEnableStreamingReadOnReadIncompatibleSchemaChangesDuringStreamStart = forceEnableStreamingReadOnReadIncompatibleSchemaChangesDuringStreamStart, forceEnableUnsafeReadOnNullabilityChange = forceEnableUnsafeReadOnNullabilityChange, isStreamingFromColumnMappingTable = isStreamingFromColumnMappingTable, typeWideningEnabled = typeWideningEnabled, enableSchemaTrackingForTypeWidening = enableSchemaTrackingForTypeWidening) } } sealed trait SchemaCompatibilityResult object SchemaCompatibilityResult { // Indicates that the schema change is compatible and can be applied safely case object Compatible extends SchemaCompatibilityResult // Indicates that the schema change is incompatible and would break the query, // but the change can be applied by recovering the query case object RetryableIncompatible extends SchemaCompatibilityResult // Indicates that the schema change is incompatible and would break the query, // but the change cannot be applied by recovering the query case object NonRetryableIncompatible extends SchemaCompatibilityResult // helper methods for java interop def isCompatible(result: SchemaCompatibilityResult): Boolean = result == Compatible def isRetryableIncompatible(result: SchemaCompatibilityResult): Boolean = result == RetryableIncompatible } /** * Validate schema compatibility between data schema and read schema. Checks for read * compatibility considering nullability, type widening, missing columns, and partition changes. * * @param dataSchema The actual schema of the data * @param readSchema The schema used by the reader to read data * @param newPartitionColumns The partition columns for new metadata * @param oldPartitionColumns The partition columns for old metadata * @param backfilling Whether the check is triggered during backfilling (processing old data) * @param readOptions Configuration options that control schema compatibility rules * * @return A [[SchemaCompatibilityResult]] on whether the data schema is compatible, and if not, * whether restarting the stream will allow processing data across the schema change. */ def checkSchemaChangesWhenNoSchemaTracking( dataSchema: StructType, readSchema: StructType, newPartitionColumns: Seq[String], oldPartitionColumns: Seq[String], backfilling: Boolean, readOptions: SchemaReadOptions): SchemaCompatibilityResult = { // We forbid the case when the data schema is nullable while the read schema is NOT // nullable, or in other words, `readSchema` should not tighten nullability from `dataSchema`, // because we don't ever want to read back any nulls when the read schema is non-nullable. val shouldForbidTightenNullability = !readOptions.forceEnableUnsafeReadOnNullabilityChange // If schema tracking is disabled for type widening, we allow widening type changes to go // through without requiring the user to set `allowSourceColumnTypeChange`. The schema change // will cause the stream to fail with a retryable exception, and the stream will restart using // the new schema. val allowWideningTypeChanges = readOptions.typeWideningEnabled && !readOptions.enableSchemaTrackingForTypeWidening // If a user is streaming from a column mapping table and enable the unsafe flag to ignore // column mapping schema changes, we can allow the standard check to allow missing columns // from the read schema in the data schema, because the only case that happens is when // user rename/drops column but they don't care so they enabled the flag to unblock. // This is only allowed when we are "backfilling", i.e. the stream progress is older than // the analyzed table version. Any schema change past the analysis should still throw // exception, because additive schema changes MUST be taken into account. val shouldAllowMissingColumns = readOptions.isStreamingFromColumnMappingTable && readOptions.allowUnsafeStreamingReadOnColumnMappingSchemaChanges && backfilling // When backfilling after a type change, allow processing the data using the new, wider // type. // typeWideningMode when using `readSchema` to read `dataSchema` val forwardTypeWideningMode = if (allowWideningTypeChanges && backfilling) { TypeWideningMode.AllTypeWidening } else { TypeWideningMode.NoTypeWidening } if (!SchemaUtils.isReadCompatible( existingSchema = dataSchema, readSchema = readSchema, forbidTightenNullability = shouldForbidTightenNullability, allowMissingColumns = shouldAllowMissingColumns, typeWideningMode = forwardTypeWideningMode, newPartitionColumns = newPartitionColumns, oldPartitionColumns = oldPartitionColumns )) { // Check for widening type changes that would succeed on retry when we backfill batches. // typeWideningMode when using `dataSchema` to read `readSchema` val backwardTypeWideningMode = if (allowWideningTypeChanges) { TypeWideningMode.AllTypeWidening } else { TypeWideningMode.NoTypeWidening } // Only schema change later than the current read snapshot/schema can be retried, in other // words, backfills could never be retryable, because we have no way to refresh // the latest schema to "catch up" when the schema change happens before than current read // schema version. // If not backfilling, we do another check to determine retryability, in which we assume // we will be reading using this later `dataSchema` back on the current outdated `readSchema`, // and if it works (including that `dataSchema` should not tighten the nullability // constraint from `readSchema`), it is a retryable exception. val retryable = !backfilling && SchemaUtils.isReadCompatible( existingSchema = readSchema, readSchema = dataSchema, forbidTightenNullability = shouldForbidTightenNullability, typeWideningMode = backwardTypeWideningMode ) if (retryable) { SchemaCompatibilityResult.RetryableIncompatible } else { SchemaCompatibilityResult.NonRetryableIncompatible } } else { SchemaCompatibilityResult.Compatible } } /** * - If commit's timestamp exactly matches the provided timestamp, we return it. * - Otherwise, we return the earliest commit version * with a timestamp greater than the provided one. * - If the provided timestamp is larger than the timestamp * of any committed version, and canExceedLatest is disabled we throw an error. * - If the provided timestamp is larger than the timestamp * of any committed version, and canExceedLatest is enabled we return a version that is greater * than commitVersion by one * * @param timeZone - time zone for formatting error messages * @param commitTimestamp - timestamp of the commit * @param commitVersion - version of the commit * @param latestVersion - latest snapshot version * @param timestamp - user specified timestamp * @param canExceedLatest - if true, version can be greater than the latest snapshot commit * @return - corresponding version number for timestamp */ def getStartingVersionFromCommitAtTimestamp( timeZone: String, commitTimestamp: Long, commitVersion: Long, latestVersion: Long, timestamp: Timestamp, canExceedLatest: Boolean = false): Long = { if (commitTimestamp >= timestamp.getTime) { // Find the commit at the `timestamp` or the earliest commit commitVersion } else { // commitTimestamp is not the same, so this commit is a commit before the timestamp and // the next version if exists should be the earliest commit after the timestamp. // // Note: In the use case of [[CDCReader]] timestamp passed in can exceed the latest commit // timestamp, caller doesn't expect exception, and can handle the non-existent version. val latestNotExceeded = commitVersion + 1 <= latestVersion if (latestNotExceeded || canExceedLatest) { commitVersion + 1 } else { val commitTs = new Timestamp(commitTimestamp) val timestampFormatter = TimestampFormatter(DateTimeUtils.getTimeZone(timeZone)) val tsString = DateTimeUtils.timestampToString( timestampFormatter, DateTimeUtils.fromJavaTimestamp(commitTs)) throw DeltaErrors.timestampGreaterThanLatestCommit(timestamp, commitTs, tsString) } } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/sources/limits.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.sources import org.apache.spark.sql.SparkSession import org.apache.spark.sql.connector.read.streaming.{ReadLimit, ReadMaxFiles} import org.apache.spark.sql.internal.SQLConf /** A read limit that admits a soft-max of `maxBytes` per micro-batch. */ case class ReadMaxBytes(maxBytes: Long) extends ReadLimit /** * A read limit that admits the given soft-max of `bytes` or max `maxFiles`, once `minFiles` * has been reached. Prior to that anything is admitted. */ case class CompositeLimit( bytes: ReadMaxBytes, maxFiles: ReadMaxFiles, minFiles: ReadMinFiles = ReadMinFiles(-1)) extends ReadLimit /** A read limit that admits a min of `minFiles` per micro-batch. */ case class ReadMinFiles(minFiles: Int) extends ReadLimit ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/stats/ArrayAccumulator.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.util.AccumulatorV2 /** * An accumulator that keeps arrays of counts. Counts from multiple partitions * are merged by index. -1 indicates a null and is handled using TVL (-1 + N = -1) */ class ArrayAccumulator(val size: Int) extends AccumulatorV2[(Int, Long), Array[Long]] { protected val counts = new Array[Long](size) override def isZero: Boolean = counts.forall(_ == 0) override def copy(): AccumulatorV2[(Int, Long), Array[Long]] = { val newCopy = new ArrayAccumulator(size) (0 until size).foreach(i => newCopy.counts(i) = counts(i)) newCopy } override def reset(): Unit = (0 until size).foreach(counts(_) = 0) override def add(v: (Int, Long)): Unit = { if (v._2 == -1 || counts(v._1) == -1) { counts(v._1) = -1 } else { counts(v._1) += v._2 } } override def merge(o: AccumulatorV2[(Int, Long), Array[Long]]): Unit = { val other = o.asInstanceOf[ArrayAccumulator] assert(size == other.size) (0 until size).foreach(i => { if (counts(i) == -1 || other.counts(i) == -1) { counts(i) = -1 } else { counts(i) += other.counts(i) } }) } override def value: Array[Long] = counts } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/stats/AutoCompactPartitionStats.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats import scala.collection.mutable import scala.util.control.NonFatal import org.apache.spark.sql.delta.actions.{Action, AddFile, FileAction, RemoveFile} import org.apache.spark.sql.delta.hooks.AutoCompactPartitionReserve import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.SparkSession /** * A collector used to aggregate auto-compaction stats for a single commit. The expectation * is to spin this up for a commit and then merging those local stats with the global stats. */ trait AutoCompactPartitionStatsCollector { def collectPartitionStatsForAdd(file: AddFile): Unit def collectPartitionStatsForRemove(file: RemoveFile): Unit def finalizeStats(tableId: String): Unit } /** * This singleton object collect the table partition statistic for each commit that creates * AddFile or RemoveFile objects. * To control the memory usage, there are `maxNumTablePartitions` per table and 'maxNumPartitions' * partition entries across all tables. * Note: * 1. Since the partition of each table is limited, if this limitation is reached, the least * recently used table partitions will be evicted. * 2. If all 'maxNumPartitions' are occupied, the partition stats of least recently used tables * will be evicted until the used partitions fall back below to 'maxNumPartitions'. * 3. The un-partitioned tables are treated as tables with single partition. * @param maxNumTablePartitions The hash space of partition key to reduce memory usage per table. * @param maxNumPartitions The maximum number of partition that can be occupied. */ class AutoCompactPartitionStats( private var maxNumTablePartitions: Int, private var maxNumPartitions: Int ) { /** * This class to store the states of one table partition. These state includes: * -- the number of small files, * -- the thread that assigned to compact this partition, and * -- whether the partition was compacted. * * Note: Since this class keeps tracking of the statistics of the table partition and the state of * the auto compaction thread that works on the table partition, any method that accesses any * attribute of this class needs to be protected by synchronized context. */ class PartitionStat( var numFiles: Long, var wasAutoCompacted: Boolean = false) { /** * Determine whether this partition can be autocompacted based on the number of small files or * if this [[AutoCompactPartitionStats]] instance has not auto compacted it yet. * @param minNumFiles The minimum number of files this table-partition should have to trigger * Auto Compaction in case it has already been compacted once. */ def hasSufficientSmallFilesOrHasNotBeenCompacted(minNumFiles: Long): Boolean = !wasAutoCompacted || hasSufficientFiles(minNumFiles) def hasSufficientFiles(minNumFiles: Long): Boolean = numFiles >= minNumFiles } /** * This hashtable is used to store all table partition states of a table, the key is the hashcode * of the partition, the value is [[PartitionStat]] object. */ type TablePartitionStats = mutable.LinkedHashMap[Int, PartitionStat] // The hash map to store the number of small files in each partition. // -- Key is the hash code of the partition value. // -- Values is the number of small files inside the corresponding partition. type PartitionFilesMap = mutable.LinkedHashMap[Int, Long] type PartitionKey = Map[String, String] type PartitionKeySet = Set[Map[String, String]] // This is a simple LRU to store the table partition statistics. // Workspace private to enable testing. private[delta] val tablePartitionStatsCache = new mutable.LinkedHashMap[String, TablePartitionStats]() // The number of partitions in this cache. private[delta] var numUsedPartitions = 0 /** * Helper class used to keep state regarding tracking auto-compaction stats of AddFile and * RemoveFile actions in a single run that are greater than a passed-in minimum file size. * If the collector runs into any non-fatal errors, it will invoke the error reporter on the error * and then skip further execution. * * @param minFileSize Minimum file size for files we track auto-compact stats * @param errorReporter Function that reports the first error, if any * @return A collector object that tracks the Add/Remove file actions of the current commit. */ def createStatsCollector( minFileSize: Long, errorReporter: Throwable => Unit): AutoCompactPartitionStatsCollector = new AutoCompactPartitionStatsCollector { private val inputPartitionFiles = new PartitionFilesMap() private var shouldCollect = true /** * If the file is less than the specified min file size, updates the partition file map * of stats with add or remove actions. If we encounter an error during stats collection, * the remainder of the files will not be collected as well. */ private def collectPartitionStatsForFile(file: FileAction, addSub: Int): Unit = { try { val minSizeThreshold = minFileSize if (shouldCollect && file.estLogicalFileSize.getOrElse(file.getFileSize) <= minSizeThreshold ) { updatePartitionFileCounter(inputPartitionFiles, file.partitionValues, addSub) } } catch { case NonFatal(e) => errorReporter(e) shouldCollect = false } } /** * Adds one file to all the appropriate partition counters. */ override def collectPartitionStatsForAdd(file: AddFile): Unit = { collectPartitionStatsForFile(file, addSub = 1) } /** * Removes one file from all the appropriate partition counters. */ override def collectPartitionStatsForRemove(file: RemoveFile): Unit = { collectPartitionStatsForFile(file, addSub = -1) } /** * Merges the current collector's stats with the global one. */ override def finalizeStats(tableId: String): Unit = { try { if (shouldCollect) merge(tableId, inputPartitionFiles.filter(_._2 != 0)) } catch { case NonFatal(e) => errorReporter(e) } } } /** * This method merges the `inputPartitionFiles` of current committed transaction to the * global cache of table partition stats. After merge is completed, tablePath will be moved * to most recently used position. If the number of occupied partitions exceeds * MAX_NUM_PARTITIONS, the least recently used tables will be evicted out. * * @param tableId The path of the table that contains `inputPartitionFiles`. * @param inputPartitionFiles The number of files, which are qualified for Auto Compaction, in * each partition. */ def merge(tableId: String, inputPartitionFiles: PartitionFilesMap): Unit = { if (inputPartitionFiles.isEmpty) return synchronized { tablePartitionStatsCache.get(tableId) match { case Some(cachedPartitionStates) => // If the table is already stored, merges inputPartitionFiles' content to // existing PartitionFilesMap. for ((partitionHashCode, numFilesDelta) <- inputPartitionFiles) { assert(numFilesDelta != 0) cachedPartitionStates.get(partitionHashCode) match { case Some(partitionState) => // If there is an entry of partitionHashCode, updates its number of files // and moves it to the most recently used slot. partitionState.numFiles += numFilesDelta moveAccessedPartitionToMru(cachedPartitionStates, partitionHashCode, partitionState) case None => if (numFilesDelta > 0) { // New table partition is always in the most recently used entry. cachedPartitionStates.put(partitionHashCode, new PartitionStat(numFilesDelta)) numUsedPartitions += 1 } } } // Move the accessed table to MRU position and evicts the LRU partitions from it // if necessary. moveAccessedTableToMru(tableId, cachedPartitionStates) case None => // If it is new table, just create new entry. val newPartitionStates = inputPartitionFiles .filter { case (_, numFiles) => numFiles > 0 } .map { case (partitionHashCode, numFiles) => (partitionHashCode, new PartitionStat(numFiles)) } tablePartitionStatsCache.put(tableId, newPartitionStates) numUsedPartitions += newPartitionStates.size moveAccessedTableToMru(tableId, newPartitionStates) } evictLruTablesIfNecessary() } } /** Move the accessed table partition to the most recently used position. */ private def moveAccessedPartitionToMru( cachedPartitionFiles: TablePartitionStats, partitionHashCode: Int, partitionState: PartitionStat): Unit = { cachedPartitionFiles.remove(partitionHashCode) if (partitionState.numFiles <= 0) { numUsedPartitions -= 1 } else { // If the newNumFiles is not empty, add it back and make it to be the // most recently used entry. cachedPartitionFiles.put(partitionHashCode, partitionState) } } /** Move the accessed table to the most recently used position. */ private def moveAccessedTableToMru( tableId: String, cachedPartitionFiles: TablePartitionStats): Unit = { // The tablePartitionStatsCache is insertion order preserved hash table. Thus, // removing and adding back the entry make this to be most recently used entry. // If cachedPartitionFiles's size is empty, no need to add it back to LRU. tablePartitionStatsCache.remove(tableId) numUsedPartitions -= cachedPartitionFiles.size if (cachedPartitionFiles.nonEmpty) { // Evict the least recently used partitions' statistics from table if necessary val numExceededPartitions = cachedPartitionFiles.size - maxNumTablePartitions if (numExceededPartitions > 0) { val newPartitionStats = cachedPartitionFiles.drop(numExceededPartitions) tablePartitionStatsCache.put(tableId, newPartitionStats) numUsedPartitions += newPartitionStats.size } else { tablePartitionStatsCache.put(tableId, cachedPartitionFiles) numUsedPartitions += cachedPartitionFiles.size } } } /** * Evicts the Lru tables from 'tablePartitionStatsCache' until the total number of partitions * is less than maxNumPartitions. */ private def evictLruTablesIfNecessary(): Unit = { // Keep removing the least recently used table until the used partition is lower than // threshold. while (numUsedPartitions > maxNumPartitions && tablePartitionStatsCache.nonEmpty) { // Pick the least recently accessed table and remove it. val (lruTable, tablePartitionStat) = tablePartitionStatsCache.head numUsedPartitions -= tablePartitionStat.size tablePartitionStatsCache.remove(lruTable) } } /** Update the file count of `PartitionFilesMap` according to the hash value of `partition`. */ private def updatePartitionFileCounter( partitionFileCounter: PartitionFilesMap, partition: PartitionKey, addSub: Int): Unit = { partitionFileCounter.get(partition.##) match { case Some(numFiles) => partitionFileCounter.update(partition.##, numFiles + addSub) case None => partitionFileCounter.put(partition.##, addSub) } } /** Get the maximum number of files among all partitions inside table `tableId`. */ def maxNumFilesInTable(tableId: String): Long = { synchronized { tablePartitionStatsCache.get(tableId) match { case Some(partitionFileCounter) => if (partitionFileCounter.isEmpty) { 0 } else { partitionFileCounter.map(_._2.numFiles).max } case None => 0 } } } /** * @return Filter partitions from targetPartitions that have not been auto-compacted or * that have enough small files. */ def filterPartitionsWithSmallFiles(tableId: String, targetPartitions: Set[PartitionKey], minNumFiles: Long): Set[PartitionKey] = synchronized { tablePartitionStatsCache.get(tableId).map { tablePartitionStates => targetPartitions.filter { partitionKey => tablePartitionStates.get(partitionKey.##).exists { partitionState => partitionState.hasSufficientSmallFilesOrHasNotBeenCompacted(minNumFiles) } } }.getOrElse(Set.empty) } def markPartitionsAsCompacted(tableId: String, compactedPartitions: Set[PartitionKey]) : Unit = synchronized { tablePartitionStatsCache.get(tableId).foreach { tablePartitionStats => compactedPartitions .foreach(partitionKey => tablePartitionStats.get(partitionKey.##) .foreach(_.wasAutoCompacted = true)) } } /** * Collect the number of files, which are less than minFileSize, added to or removed from each * partition from `actions`. * The stats collection is only complete when: * 1. The returned iterator has been consumed AND * 2. finalizeStats has been called on the collector. */ def collectPartitionStats( collector: AutoCompactPartitionStatsCollector, actions: Iterator[Action]): Iterator[Action] = { actions.map { action => action match { case addFile: AddFile => collector.collectPartitionStatsForAdd(addFile) case removeFile: RemoveFile => collector.collectPartitionStatsForRemove(removeFile) case _ => // do nothing } action } } /** This is test only code to reset the state of table partition statistics. */ private[delta] def resetTestOnly(newHashSpace: Int, newMaxNumPartitions: Int): Unit = { synchronized { tablePartitionStatsCache.clear() maxNumTablePartitions = newHashSpace maxNumPartitions = newMaxNumPartitions numUsedPartitions = 0 AutoCompactPartitionReserve.resetTestOnly() } } /** * This is test only code to reset all partition statistic information and keep current * configuration. */ private[delta] def resetTestOnly(): Unit = resetTestOnly(maxNumTablePartitions, maxNumPartitions) } object AutoCompactPartitionStats { private var _instance: AutoCompactPartitionStats = null /** The thread safe constructor of singleton. */ def instance(spark: SparkSession): AutoCompactPartitionStats = { synchronized { if (_instance == null) { val config = spark.conf val hashSpaceSize = config.get(DeltaSQLConf.DELTA_AUTO_COMPACT_MAX_TABLE_PARTITION_STATS) val maxNumPartitions = config.get(DeltaSQLConf.DELTA_AUTO_COMPACT_PARTITION_STATS_SIZE) _instance = new AutoCompactPartitionStats( hashSpaceSize, maxNumPartitions ) } } _instance } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingPredicateBuilder.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.stats.DeltaStatistics.{MAX, MIN} import org.apache.spark.sql.Column /** * A trait that defines interfaces for a data skipping predicate builder. * * Note that 'IsNull', 'IsNotNull' and 'StartsWith' are handled at a column (not expression) level * within [[DataSkippingReaderBase.DataFiltersBuilder.constructDataFilters]]. * * Note that the 'value' passed in for each of the interface should be [[SkippingEligibleLiteral]]. */ private [sql] trait DataSkippingPredicateBuilder { /** The predicate should match any file which contains the requested point. */ def equalTo(statsProvider: StatsProvider, colPath: Seq[String], value: Column) : Option[DataSkippingPredicate] /** The predicate should match any file which contains anything other than the rejected point. */ def notEqualTo(statsProvider: StatsProvider, colPath: Seq[String], value: Column) : Option[DataSkippingPredicate] /** * The predicate should match any file which contains values less than the requested upper bound. */ def lessThan(statsProvider: StatsProvider, colPath: Seq[String], value: Column) : Option[DataSkippingPredicate] /** * The predicate should match any file which contains values less than or equal to the requested * upper bound. */ def lessThanOrEqual(statsProvider: StatsProvider, colPath: Seq[String], value: Column) : Option[DataSkippingPredicate] /** * The predicate should match any file which contains values larger than the requested lower * bound. */ def greaterThan(statsProvider: StatsProvider, colPath: Seq[String], value: Column) : Option[DataSkippingPredicate] /** * The predicate should match any file which contains values larger than or equal to the requested * lower bound. */ def greaterThanOrEqual(statsProvider: StatsProvider, colPath: Seq[String], value: Column) : Option[DataSkippingPredicate] } /** * A collection of supported data skipping predicate builders. */ object DataSkippingPredicateBuilder { /** Predicate builder for skipping eligible columns. */ case object ColumnBuilder extends ColumnPredicateBuilder } /** * Predicate builder for skipping eligible columns. */ private [stats] class ColumnPredicateBuilder extends DataSkippingPredicateBuilder { def equalTo(statsProvider: StatsProvider, colPath: Seq[String], value: Column) : Option[DataSkippingPredicate] = { statsProvider.getPredicateWithStatTypesIfExists(colPath, value.expr.dataType, MIN, MAX) { (min, max) => min <= value && value <= max } } def notEqualTo(statsProvider: StatsProvider, colPath: Seq[String], value: Column) : Option[DataSkippingPredicate] = { statsProvider.getPredicateWithStatTypesIfExists(colPath, value.expr.dataType, MIN, MAX) { (min, max) => min < value || value < max } } def lessThan(statsProvider: StatsProvider, colPath: Seq[String], value: Column) : Option[DataSkippingPredicate] = statsProvider.getPredicateWithStatTypeIfExists(colPath, value.expr.dataType, MIN)(_ < value) def lessThanOrEqual(statsProvider: StatsProvider, colPath: Seq[String], value: Column) : Option[DataSkippingPredicate] = statsProvider.getPredicateWithStatTypeIfExists(colPath, value.expr.dataType, MIN)(_ <= value) def greaterThan(statsProvider: StatsProvider, colPath: Seq[String], value: Column) : Option[DataSkippingPredicate] = statsProvider.getPredicateWithStatTypeIfExists(colPath, value.expr.dataType, MAX)(_ > value) def greaterThanOrEqual(statsProvider: StatsProvider, colPath: Seq[String], value: Column) : Option[DataSkippingPredicate] = statsProvider.getPredicateWithStatTypeIfExists(colPath, value.expr.dataType, MAX)(_ >= value) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats // scalastyle:off import.ordering.noEmptyLine import java.io.Closeable import scala.collection.mutable.ArrayBuffer import org.apache.spark.sql.delta.skipping.clustering.{ClusteredTableUtils, ClusteringColumnInfo} import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.{DeltaColumnMapping, DeltaLog, DeltaTableUtils} import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions.{AddFile, Metadata} import org.apache.spark.sql.delta.expressions.DecodeNestedZ85EncodedVariant import org.apache.spark.sql.delta.implicits._ import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.DeltaDataSkippingType.DeltaDataSkippingType import org.apache.spark.sql.delta.stats.DeltaStatistics._ import org.apache.spark.sql.delta.util.StateCache import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.hadoop.fs.Path import org.apache.spark.sql.{DataFrame, _} import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral} import org.apache.spark.sql.catalyst.util.TypeUtils import org.apache.spark.sql.execution.InSubqueryExec import org.apache.spark.sql.execution.datasources.VariantMetadata import org.apache.spark.sql.expressions.SparkUserDefinedFunction import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{AtomicType, BooleanType, CalendarIntervalType, DataType, DateType, LongType, NumericType, StringType, StructField, StructType, TimestampNTZType, TimestampType, VariantType} import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} /** * Used to hold the list of files and scan stats after pruning files using the limit. */ case class ScanAfterLimit( files: Seq[AddFile], byteSize: Option[Long], numPhysicalRecords: Option[Long], numLogicalRecords: Option[Long]) /** * Used in deduplicateAndFilterRemovedLocally/getFilesAndNumRecords iterator for grouping * physical and logical number of records. * * @param numPhysicalRecords The number of records physically present in the file. * @param numLogicalRecords The physical number of records minus the Deletion Vector cardinality. */ case class NumRecords(numPhysicalRecords: java.lang.Long, numLogicalRecords: java.lang.Long) /** * Represents a stats column (MIN, MAX, etc) for a given (nested) user table column name. Used to * keep track of which stats columns a data skipping query depends on. * * The `pathToStatType` is path to a stats type accepted by `getStatsColumnOpt()` * (see object `DeltaStatistics`); * `pathToColumn` is the nested name of the user column whose stats are to be accessed. * `columnDataType` is the data type of the column. */ private[stats] case class StatsColumn private( pathToStatType: Seq[String], pathToColumn: Seq[String]) object StatsColumn { def apply(statType: String, pathToColumn: Seq[String], columnDataType: DataType): StatsColumn = { StatsColumn(Seq(statType), pathToColumn) } } /** * A data skipping predicate, which includes the expression itself, plus the set of stats columns * that expression depends on. The latter is required to correctly handle missing stats, which would * make the predicate unreliable; for details, see `DataSkippingReader.verifyStatsForFilter`. * * NOTE: It would be more accurate to call these "file keeping" predicates, because they specify the * set of files a query must examine, not the set of rows a query can safely skip. */ private [sql] case class DataSkippingPredicate( expr: Column, referencedStats: Set[StatsColumn] ) /** * Overloads the constructor for `DataSkippingPredicate`, allowing callers to pass referenced stats * as individual arguments, rather than wrapped up as a Set. * * For example, instead of this: * * DataSkippingPredicate(pred, Set(stat1, stat2)) * * We can just do: * * DataSkippingPredicate(pred, stat1, stat2) */ private [sql] object DataSkippingPredicate { def apply(filters: Column, referencedStats: StatsColumn*): DataSkippingPredicate = { DataSkippingPredicate(filters, referencedStats.toSet) } } /** * An extractor that matches on access of a skipping-eligible column. We only collect stats for leaf * columns, so internal columns of nested types are ineligible for skipping. * * NOTE: This check is sufficient for safe use of NULL_COUNT stats, but safe use of MIN and MAX * stats requires additional restrictions on column data type (see SkippingEligibleLiteral). * * @return The path to the column and the column's data type if it exists and is eligible. * Otherwise, return None. */ object SkippingEligibleColumn { def unapply(arg: Expression): Option[(Seq[String], DataType)] = { // Only atomic types are eligible for skipping, and args should always be resolved by now. // When `pushVariantIntoScan` is true, Variants in the read schema are transformed into Structs // to facilitate shredded reads. Therefore, filters like `v is not null` where `v` is a variant // column look like the filters on struct data. `VariantMetadata.isVariantStruct` helps in // distinguishing between "true structs" and "variant structs". val eligible = arg.resolved && (arg.dataType.isInstanceOf[AtomicType] || VariantMetadata.isVariantStruct(arg.dataType)) if (eligible) searchChain(arg).map(_ -> arg.dataType) else None } private def searchChain(arg: Expression): Option[Seq[String]] = arg match { case a: Attribute => Some(a.name :: Nil) case GetStructField(child, _, Some(name)) => searchChain(child).map(name +: _) case g @ GetStructField(child, ord, None) if g.resolved => searchChain(child).map(g.childSchema(ord).name +: _) case _ => None } } /** * An extractor that matches on access of a skipping-eligible Literal. Delta tables track min/max * stats for a limited set of data types, and only Literals of those types are skipping-eligible. * * @return The Literal, if it is eligible. Otherwise, return None. */ object SkippingEligibleLiteral { def unapply(arg: Literal): Option[Column] = { if (SkippingEligibleDataType(arg.dataType)) Some(Column(arg)) else None } } object SkippingEligibleDataType { // Call this directly, e.g. `SkippingEligibleDataType(dataType)` def apply(dataType: DataType): Boolean = dataType match { case _: NumericType | DateType | TimestampType | TimestampNTZType | StringType => true case _: VariantType => SQLConf.get.getConf(DeltaSQLConf.COLLECT_VARIANT_DATA_SKIPPING_STATS) case _ => false } // Use these in `match` statements def unapply(dataType: DataType): Option[DataType] = { if (SkippingEligibleDataType(dataType)) Some(dataType) else None } def unapply(f: StructField): Option[DataType] = unapply(f.dataType) } /** * An extractor that matches expressions that are eligible for data skipping predicates. * * @return A tuple of 1) column name referenced in the expression, 2) date type for the * expression, 3) [[DataSkippingPredicateBuilder]] that builds the data skipping * predicate for the expression, if the given expression is eligible. * Otherwise, return None. */ abstract class GenericSkippingEligibleExpression() { def unapply(arg: Expression): Option[(Seq[String], DataType, DataSkippingPredicateBuilder)] = { arg match { case SkippingEligibleColumn(c, dt) => Some((c, dt, DataSkippingPredicateBuilder.ColumnBuilder)) case _ => None } } } /** * This object is used to avoid referencing DataSkippingReader in DetlaConfig. * Otherwise, it might cause the cyclic import through SQLConf -> SparkSession -> DetlaConfig. */ private[delta] object DataSkippingReaderConf { /** * Default number of cols for which we should collect stats */ val DATA_SKIPPING_NUM_INDEXED_COLS_DEFAULT_VALUE = 32 } private[delta] object DataSkippingReader { private[this] def col(e: Expression): Column = Column(e) def fold(e: Expression): Column = col(new Literal(e.eval(), e.dataType)) // Literals often used in the data skipping reader expressions. val trueLiteral: Column = col(TrueLiteral) val falseLiteral: Column = col(FalseLiteral) val nullStringLiteral: Column = col(new Literal(null, StringType)) val nullBooleanLiteral: Column = col(new Literal(null, BooleanType)) val oneMillisecondLiteralExpr: Literal = { val oneMillisecond = new CalendarInterval(0, 0, 1000 /* micros */) new Literal(oneMillisecond, CalendarIntervalType) } lazy val sizeCollectorInputEncoders: Seq[Option[ExpressionEncoder[_]]] = Seq( Option(ExpressionEncoder[Boolean]()), Option(ExpressionEncoder[java.lang.Long]()), Option(ExpressionEncoder[java.lang.Long]()), Option(ExpressionEncoder[java.lang.Long]())) /** * For timestamps, JSON serialization will truncate to milliseconds. This means * that we must adjust 1 millisecond upwards for max stats, or we will incorrectly skip * records that differ only in microsecond precision. (For example, a file containing only * 01:02:03.456789 will be written with min == max == 01:02:03.456, so we must consider it * to contain the range from 01:02:03.456 to 01:02:03.457.) * * To avoid overflow when the timestamp is near Long.MAX_VALUE, we check if adding 1 * millisecond would overflow. If so, we saturate to Long.MAX_VALUE to ensure the max stat * is >= all actual values in the file while avoiding arithmetic overflow. */ def getAdjustedTimestamp(col: Column, tsType: DataType): Column = { val maxTimestampLiteral = Literal(Long.MaxValue, tsType) val overflowThresholdLiteral = Literal(Long.MaxValue - 1000, tsType) val adjustedExpr = If( GreaterThan(col.expr, overflowThresholdLiteral), maxTimestampLiteral, TimestampAdd("MILLISECOND", Literal(1L, LongType), col.expr)) Column(Cast(adjustedExpr, tsType)) } } /** * Adds the ability to use statistics to filter the set of files based on predicates * to a [[org.apache.spark.sql.delta.Snapshot]] of a given Delta table. */ trait DataSkippingReaderBase extends DeltaScanGenerator with StatisticsCollection with ReadsMetadataFields with StateCache with DeltaLogging { import DataSkippingReader._ def allFiles: Dataset[AddFile] def path: Path def version: Long def metadata: Metadata private[delta] def sizeInBytesIfKnown: Option[Long] def deltaLog: DeltaLog def schema: StructType private[delta] def numOfFilesIfKnown: Option[Long] def redactedPath: String private def useStats = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_STATS_SKIPPING) private lazy val limitPartitionLikeFiltersToClusteringColumns = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_DATASKIPPING_PARTITION_LIKE_FILTERS_CLUSTERING_COLUMNS_ONLY) private lazy val additionalPartitionLikeFilterSupportedExpressions = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_DATASKIPPING_PARTITION_LIKE_FILTERS_ADDITIONAL_SUPPORTED_EXPRESSIONS) .toSet.flatMap((exprs: String) => exprs.split(",")) /** Returns a DataFrame expression to obtain a list of files with parsed statistics. */ private def withStatsInternal0: DataFrame = { val parsedStats = from_json(col("stats"), statsSchema) // Only use DecodeNestedZ85EncodedVariant if the schema contains VariantType. // This avoids performance overhead for tables without variant columns. // `DecodeNestedZ85EncodedVariant` is a temporary workaround since the Spark 4.1 from_json // expression has no way to decode a VariantVal from an encoded Z85 string. // TODO: Add Z85 decoding to Variant in Spark 4.2 and use that from_json option here. val decodedStats = if (SchemaUtils.checkForVariantTypeColumnsRecursively(statsSchema)) { Column(DecodeNestedZ85EncodedVariant(parsedStats.expr)) } else { parsedStats } allFiles.withColumn("stats", decodedStats) } private lazy val withStatsCache = cacheDS(withStatsInternal0, s"Delta Table State with Stats #$version - $redactedPath") protected def withStatsInternal: DataFrame = withStatsCache.getDS /** All files with the statistics column dropped completely. */ def withNoStats: DataFrame = allFiles.drop("stats") /** * Returns a parsed and cached representation of files with statistics. * * * @return [[DataFrame]] */ final def withStats: DataFrame = { withStatsInternal } /** * Constructs a [[DataSkippingPredicate]] for isNotNull predicates. */ protected def constructNotNullFilter( statsProvider: StatsProvider, pathToColumn: Seq[String]): Option[DataSkippingPredicate] = { val nullCountCol = StatsColumn(NULL_COUNT, pathToColumn, LongType) val numRecordsCol = StatsColumn(NUM_RECORDS, pathToColumn = Nil, LongType) statsProvider.getPredicateWithStatsColumnsIfExists(nullCountCol, numRecordsCol) { (nullCount, numRecords) => nullCount < numRecords } } def withStatsDeduplicated: DataFrame = withStats /** * Builds the data filters for data skipping. */ class DataFiltersBuilder( protected val spark: SparkSession, protected val dataSkippingType: DeltaDataSkippingType) { protected val statsProvider: StatsProvider = new StatsProvider(getStatsColumnOpt) object SkippingEligibleExpression extends GenericSkippingEligibleExpression() // Main function for building data filters. def apply(dataFilter: Expression): Option[DataSkippingPredicate] = constructDataFilters(dataFilter, isNullExpansionDepth = 0) /** * Helper function to construct a [[DataSkippingPredicate]] for an IsNull predicate on * null-intolerant expressions that are guaranteed to return non-null results for non-null * inputs. This method is only valid *if and only if* the passed-in expression returns null * for any null children. That is, if all children are non-null, the expression *must* return * a non-null result. * @param expr Expression to push down IsNull into. * @return A [[DataSkippingPredicate]] that's the result of pushing IsNull down into expr's * children. */ protected def constructIsNullFilterForNullIntolerant( expr: Expression, isNullExpansionDepth: Int): Option[DataSkippingPredicate] = { val filters = expr.children.map { // Resolve literal children directly. constructDataFilters does not support skipping on // literal-only children. case l: Literal => if (l.value == null) { Some(DataSkippingPredicate(trueLiteral)) } else { Some(DataSkippingPredicate(falseLiteral)) } case c => constructDataFilters(IsNull(c), isNullExpansionDepth) } filters.reduceOption { (a, b) => (a, b) match { case (Some(a), Some(b)) => Some(DataSkippingPredicate(a.expr || b.expr, a.referencedStats ++ b.referencedStats)) case _ => None } }.flatten } // Helper method for expression types that represent an IN-list of literal values. // // // For excessively long IN-lists, we just test whether the file's min/max range overlaps the // range spanned by the list's smallest and largest elements. private def constructLiteralInListDataFilters( a: Expression, possiblyNullValues: Seq[Any], isNullExpansionDepth: Int): Option[DataSkippingPredicate] = { // The Ordering we use for sorting cannot handle null values, and these can anyway // be safely ignored because they will never cause an IN-list predicate to return TRUE. val values = possiblyNullValues.filter(_ != null) if (values.isEmpty) { // Handle the trivial empty case even for otherwise ineligible types. // NOTE: SQL forbids empty in-list, but InSubqueryExec could have an empty subquery result // or IN-list may contain only NULLs. return Some(DataSkippingPredicate(falseLiteral)) } val (pathToColumn, dt, builder) = SkippingEligibleExpression.unapply(a).getOrElse { // The expression is not eligible for skipping, and we can stop constructing data filters // for the expression by simply returning None. return None } lazy val ordering = TypeUtils.getInterpretedOrdering(dt) if (!SkippingEligibleDataType(dt)) { // Don't waste time building expressions for incompatible types None } else { // Emit filters for an imprecise range test that covers the entire entire list. val min = Literal(values.min(ordering), dt) val max = Literal(values.max(ordering), dt) constructDataFilters( And(GreaterThanOrEqual(max, a), LessThanOrEqual(min, a)), isNullExpansionDepth) } } /** * Returns a file skipping predicate expression, derived from the user query, which uses column * statistics to prune away files that provably contain no rows the query cares about. * * Specifically, the filter extraction code must obey the following rules: * * 1. Given a query predicate `e`, `constructDataFilters(e)` must return TRUE for a file unless * we can prove `e` will not return TRUE for any row the file might contain. For example, * given `a = 3` and min/max stat values [0, 100], this skipping predicate is safe: * * AND(minValues.a <= 3, maxValues.a >= 3) * * Because that condition must be true for any file that might possibly contain `a = 3`; the * skipping predicate could return FALSE only if the max is too low, or the min too high; it * could return NULL only if a is NULL in every row of the file. In both latter cases, it is * safe to skip the file because `a = 3` can never evaluate to TRUE. * * 2. It is unsafe to apply skipping to operators that can evaluate to NULL or produce an error * for non-NULL inputs. For example, consider this query predicate involving integer * addition: * * a + 1 = 3 * * It might be tempting to apply the standard equality skipping predicate: * * AND(minValues.a + 1 <= 3, 3 <= maxValues.a + 1) * * However, the skipping predicate would be unsound, because the addition operator could * trigger integer overflow (e.g. minValues.a = 0 and maxValues.a = INT_MAX), even though the * file could very well contain rows satisfying a + 1 = 3. * * 3. Predicates involving NOT are ineligible for skipping, because * `Not(constructDataFilters(e))` is seldom equivalent to `constructDataFilters(Not(e))`. * For example, consider the query predicate: * * NOT(a = 1) * * A simple inversion of the data skipping predicate would be: * * NOT(AND(minValues.a <= 1, maxValues.a >= 1)) * ==> OR(NOT(minValues.a <= 1), NOT(maxValues.a >= 1)) * ==> OR(minValues.a > 1, maxValues.a < 1) * * By contrast, if we first combine the NOT with = to obtain * * a != 1 * * We get a different skipping predicate: * * NOT(AND(minValues.a = 1, maxValues.a = 1)) * ==> OR(NOT(minValues.a = 1), NOT(maxValues.a = 1)) * ==> OR(minValues.a != 1, maxValues.a != 1) * * A truth table confirms that the first (naively inverted) skipping predicate is incorrect: * * minValues.a * | maxValues.a * | | OR(minValues.a > 1, maxValues.a < 1) * | | | OR(minValues.a != 1, maxValues.a != 1) * 0 0 T T * 0 1 F T !! first predicate wrongly skipped a = 0 * 1 1 F F * * Fortunately, we may be able to eliminate NOT from some (branches of some) predicates: * * a. It is safe to push the NOT into the children of AND and OR using de Morgan's Law, e.g. * * NOT(AND(a, b)) ==> OR(NOT(a), NOT(B)). * * b. It is safe to fold NOT into other operators, when a negated form of the operator * exists: * * NOT(NOT(x)) ==> x * NOT(a == b) ==> a != b * NOT(a > b) ==> a <= b * * NOTE: The skipping predicate must handle the case where min and max stats for a column are * both NULL -- which indicates that all values in the file are NULL. Fortunately, most of the * operators we support data skipping for are NULL intolerant, and thus trivially satisfy this * requirement because they never return TRUE for NULL inputs. The only NULL tolerant operator * we support -- IS [NOT] NULL -- is specifically NULL aware. AND and OR are also considered * null tolerant, and have special-cased handling of null pushdowns. * * NOTE: The skipping predicate does *NOT* need to worry about missing stats columns (which also * manifest as NULL). That case is handled separately by `verifyStatsForFilter` (which disables * skipping for any file that lacks the needed stats columns). * * @return An optional data skipping predicate, if this function returns None, then this means * that the dataFilter Expression is not eligible for data skipping, i.e. we cannot skip any * files. */ private[stats] def constructDataFilters( dataFilter: Expression, isNullExpansionDepth: Integer): Option[DataSkippingPredicate] = dataFilter match { // Expressions that contain only literals are not eligible for skipping. case cmp: Expression if cmp.children.forall(areAllLeavesLiteral) => None // Push skipping predicate generation through the AND: // // constructDataFilters(AND(a, b)) // ==> AND(constructDataFilters(a), constructDataFilters(b)) // // To see why this transformation is safe, consider that `constructDataFilters(a)` must // evaluate to TRUE *UNLESS* we can prove that `a` would not evaluate to TRUE for any row the // file might contain. Thus, if the rewritten form of the skipping predicate does not evaluate // to TRUE, at least one of the skipping predicates must not have evaluated to TRUE, which in // turn means we were able to prove that `a` and/or `b` will not evaluate to TRUE for any row // of the file. If that is the case, then `AND(a, b)` also cannot evaluate to TRUE for any row // of the file, which proves we have a valid data skipping predicate. // // NOTE: AND is special -- we can safely skip the file if one leg does not evaluate to TRUE, // even if we cannot construct a skipping filter for the other leg. case And(e1, e2) => val e1Filter = constructDataFilters(e1, isNullExpansionDepth) val e2Filter = constructDataFilters(e2, isNullExpansionDepth) if (e1Filter.isDefined && e2Filter.isDefined) { Some(DataSkippingPredicate( e1Filter.get.expr && e2Filter.get.expr, e1Filter.get.referencedStats ++ e2Filter.get.referencedStats)) } else if (e1Filter.isDefined) { e1Filter } else { e2Filter // possibly None } // Use deMorgan's law to push the NOT past the AND. This is safe even with SQL tri-valued // logic (see below), and is desirable because we cannot generally push predicate filters // through NOT, but we *CAN* push predicate filters through AND and OR: // // constructDataFilters(NOT(AND(a, b))) // ==> constructDataFilters(OR(NOT(a), NOT(b))) // ==> OR(constructDataFilters(NOT(a)), constructDataFilters(NOT(b))) // // Assuming we can push the resulting NOT operations all the way down to some leaf operation // it can fold into, the rewrite allows us to create a data skipping filter from the // expression. // // a b AND(a, b) // | | | NOT(AND(a, b)) // | | | | OR(NOT(a), NOT(b)) // T T T F F // T F F T T // T N N N N // F F F T T // F N F T T // N N N N N case Not(And(e1, e2)) => constructDataFilters(Or(Not(e1), Not(e2)), isNullExpansionDepth) // Push skipping predicate generation through OR (similar to AND case). // // constructDataFilters(OR(a, b)) // ==> OR(constructDataFilters(a), constructDataFilters(b)) // // Similar to AND case, if the rewritten predicate does not evaluate to TRUE, then it means // that neither `constructDataFilters(a)` nor `constructDataFilters(b)` evaluated to TRUE, // which in turn means that neither `a` nor `b` could evaluate to TRUE for any row the file // might contain, which proves we have a valid data skipping predicate. // // Unlike AND, a single leg of an OR expression provides no filtering power -- we can only // reject a file if both legs evaluate to false. case Or(e1, e2) => val e1Filter = constructDataFilters(e1, isNullExpansionDepth) val e2Filter = constructDataFilters(e2, isNullExpansionDepth) if (e1Filter.isDefined && e2Filter.isDefined) { Some(DataSkippingPredicate( e1Filter.get.expr || e2Filter.get.expr, e1Filter.get.referencedStats ++ e2Filter.get.referencedStats)) } else { None } // Similar to AND, we can (and want to) push the NOT past the OR using deMorgan's law. case Not(Or(e1, e2)) => constructDataFilters(And(Not(e1), Not(e2)), isNullExpansionDepth) // Match any file whose null count is larger than zero. // Note DVs might result in a redundant read of a file. // However, they cannot lead to a correctness issue. case IsNull(SkippingEligibleColumn(a, dt)) => statsProvider.getPredicateWithStatTypeIfExists(a, dt, NULL_COUNT) { nullCount => nullCount > Literal(0L) } // For these null-intolerant expressions, any null input resolves into a null output. In // addition, these expressions are special in that a null output is only possible if one of // the inputs was a NULL. Push down the IsNull operator to all children and return a // DataSkippingPredicate that's the Or of all child expressions. case IsNull(e @ (_: GreaterThan | _: GreaterThanOrEqual | _: LessThan | _: LessThanOrEqual | _: EqualTo | _: Not | _: StartsWith)) if spark.conf.get( DeltaSQLConf.DELTA_DATASKIPPING_ISNULL_PUSHDOWN_EXPRS_ENABLED) => constructIsNullFilterForNullIntolerant(e, isNullExpansionDepth) // And and Or necessitate custom pushdown logic for IsNull, as both expressions are not // considered null intolerant. Note that since the child expressions are duplicated in the // expanded expression, we need to track the depth of the expansion in this function's // signature to avoid exponential growth of the expression tree if the child expressions are // themselves And/Or expressions. case IsNull(And(left, right)) if spark.conf.get( DeltaSQLConf.DELTA_DATASKIPPING_ISNULL_PUSHDOWN_EXPRS_ENABLED) && (isNullExpansionDepth <= spark.conf.get(DeltaSQLConf.DELTA_DATASKIPPING_ISNULL_PUSHDOWN_EXPRS_MAX_DEPTH)) => // The result of an AND is only Null if either operand is a Null, and the other operand is // True. constructDataFilters( And( Or(IsNull(left), IsNull(right)), Not( Or( EqualNullSafe(left, FalseLiteral), EqualNullSafe(right, FalseLiteral) ) ) ), isNullExpansionDepth = isNullExpansionDepth + 1 ) case IsNull(Or(left, right)) if spark.conf.get( DeltaSQLConf.DELTA_DATASKIPPING_ISNULL_PUSHDOWN_EXPRS_ENABLED) && (isNullExpansionDepth <= spark.conf.get(DeltaSQLConf.DELTA_DATASKIPPING_ISNULL_PUSHDOWN_EXPRS_MAX_DEPTH)) => // The result of an OR is only Null if either operand is a Null, _and_ neither operand is // true. constructDataFilters( And( Or(IsNull(left), IsNull(right)), Not( Or( EqualNullSafe(left, TrueLiteral), EqualNullSafe(right, TrueLiteral) ) ) ), isNullExpansionDepth = isNullExpansionDepth + 1 ) case Not(IsNull(e)) => constructDataFilters(IsNotNull(e), isNullExpansionDepth) // Match any file whose null count is less than the row count. case IsNotNull(SkippingEligibleColumn(a, _)) => constructNotNullFilter(statsProvider, a) case Not(IsNotNull(e)) => constructDataFilters(IsNull(e), isNullExpansionDepth) // Match any file whose min/max range contains the requested point. case EqualTo(SkippingEligibleExpression(c, _, builder), SkippingEligibleLiteral(v)) => builder.equalTo(statsProvider, c, v) case EqualTo(v: Literal, a) => constructDataFilters(EqualTo(a, v), isNullExpansionDepth) // Match any file whose min/max range contains anything other than the rejected point. case Not(EqualTo(SkippingEligibleExpression(c, _, builder), SkippingEligibleLiteral(v))) => builder.notEqualTo(statsProvider, c, v) case Not(EqualTo(v: Literal, a)) => constructDataFilters(Not(EqualTo(a, v)), isNullExpansionDepth) // Rewrite `EqualNullSafe(a, NotNullLiteral)` as // `And(IsNotNull(a), EqualTo(a, NotNullLiteral))` and rewrite `EqualNullSafe(a, null)` as // `IsNull(a)` to let the existing logic handle it. case EqualNullSafe(a, v: Literal) => val rewrittenExpr = if (v.value != null) And(IsNotNull(a), EqualTo(a, v)) else IsNull(a) constructDataFilters(rewrittenExpr, isNullExpansionDepth) case EqualNullSafe(v: Literal, a) => constructDataFilters(EqualNullSafe(a, v), isNullExpansionDepth) case Not(EqualNullSafe(a, v: Literal)) => val rewrittenExpr = if (v.value != null) And(IsNotNull(a), EqualTo(a, v)) else IsNull(a) constructDataFilters(Not(rewrittenExpr), isNullExpansionDepth) case Not(EqualNullSafe(v: Literal, a)) => constructDataFilters(Not(EqualNullSafe(a, v)), isNullExpansionDepth) // Match any file whose min is less than the requested upper bound. case LessThan(SkippingEligibleExpression(c, _, builder), SkippingEligibleLiteral(v)) => builder.lessThan(statsProvider, c, v) case LessThan(v: Literal, a) => constructDataFilters(GreaterThan(a, v), isNullExpansionDepth) case Not(LessThan(a, b)) => constructDataFilters(GreaterThanOrEqual(a, b), isNullExpansionDepth) // Match any file whose min is less than or equal to the requested upper bound case LessThanOrEqual(SkippingEligibleExpression(c, _, builder), SkippingEligibleLiteral(v)) => builder.lessThanOrEqual(statsProvider, c, v) case LessThanOrEqual(v: Literal, a) => constructDataFilters(GreaterThanOrEqual(a, v), isNullExpansionDepth) case Not(LessThanOrEqual(a, b)) => constructDataFilters(GreaterThan(a, b), isNullExpansionDepth) // Match any file whose max is larger than the requested lower bound. case GreaterThan(SkippingEligibleExpression(c, _, builder), SkippingEligibleLiteral(v)) => builder.greaterThan(statsProvider, c, v) case GreaterThan(v: Literal, a) => constructDataFilters(LessThan(a, v), isNullExpansionDepth) case Not(GreaterThan(a, b)) => constructDataFilters(LessThanOrEqual(a, b), isNullExpansionDepth) // Match any file whose max is larger than or equal to the requested lower bound. case GreaterThanOrEqual( SkippingEligibleExpression(c, _, builder), SkippingEligibleLiteral(v)) => builder.greaterThanOrEqual(statsProvider, c, v) case GreaterThanOrEqual(v: Literal, a) => constructDataFilters(LessThanOrEqual(a, v), isNullExpansionDepth) case Not(GreaterThanOrEqual(a, b)) => constructDataFilters(LessThan(a, b), isNullExpansionDepth) // Similar to an equality test, except comparing against a prefix of the min/max stats, and // neither commutative nor invertible. case StartsWith(SkippingEligibleColumn(a, _), v @ Literal(s: UTF8String, dt: StringType)) => statsProvider.getPredicateWithStatTypesIfExists(a, dt, MIN, MAX) { (min, max) => val sLen = s.numChars() substring(min, 0, sLen) <= v && substring(max, 0, sLen) >= v } // We can only handle-IN lists whose values can all be statically evaluated to literals. case in @ In(a, values) if in.inSetConvertible => constructLiteralInListDataFilters( a, values.map(_.asInstanceOf[Literal].value), isNullExpansionDepth) // The optimizer automatically converts all but the shortest eligible IN-lists to InSet. case InSet(a, values) => constructLiteralInListDataFilters(a, values.toSeq, isNullExpansionDepth) // Treat IN(... subquery ...) as a normal IN-list, since the subquery already ran before now. case in: InSubqueryExec => // At this point the subquery has been materialized, but values() can return None if // the subquery was bypassed at runtime. in.values().flatMap(v => constructLiteralInListDataFilters(in.child, v.toSeq, isNullExpansionDepth)) // Remove redundant pairs of NOT case Not(Not(e)) => constructDataFilters(e, isNullExpansionDepth) // WARNING: NOT is dangerous, because `Not(constructDataFilters(e))` is seldom equivalent to // `constructDataFilters(Not(e))`. We must special-case every `Not(e)` we wish to support. case Not(_) => None // Unknown expression type... can't use it for data skipping. case _ => None } // Lightweight wrapper to represent a fully resolved reference to an attribute for // partition-like data filters. Contains the min/max/null count stats column expressions and // the referenced stats column for the attribute. private case class ResolvedPartitionLikeReference( referencedStatsCols: Seq[StatsColumn], minExpr: Expression, maxExpr: Expression, nullCountExpr: Expression) /** * Whitelist of expressions that can be rewritten as partition-like. * Set to a finite list to avoid having to silently introducing correctness issues as new * expressions that violate the assumptions of partition-like skipping are introduced. * There's no need to include [[SkippingEligibleColumn]] here - it's already handled explicitly. * * The following expressions have been intentionally excluded from the whitelist of supported * expressions: * - [[AttributeReference]]: Any non-skipping eligible column references can't be rewritten as * partition-like. * - Any nondeterministic expression: The value returned while skipping might be different when * the expression is evaluated again. For example, rand() > 0.5 would return ~25% of records * if used in data skipping, while the user would expect ~50% of records to be returned. * - [[UserDefinedExpression]]: Often nondeterministic, and may have side effects when executed * multiple times. * - [[RegExpReplace]], [[RegExpExtractBase]], [[Like]], [[MultiLikeBase]], [[InvokeLike]], and * [[JsonToStructs]]: These expressions might be very expensive to evalute more than once. */ private def shouldRewriteAsPartitionLike(expr: Expression): Boolean = expr match { // Expressions supported by traditional data skipping. // Boolean operators. case _: Not | _: Or | _: And => true // Comparison operators. case _: EqualNullSafe | _: EqualTo | _: GreaterThan | _: GreaterThanOrEqual | _: IsNull | _: IsNotNull | _: LessThan | _: LessThanOrEqual => true // String and set operators. InSubqueryExec is explicitly handled by the caller. case _: In | _: InSet | _: StartsWith => true case _: Literal => true // Expressions only supported for partition-like data skipping. // Date and time conversions. case _: ConvertTimezone | _: DateFormatClass | _: Extract | _: GetDateField | _: GetTimeField | _: IntegralToTimestampBase | _: MakeDate | _: MakeTimestamp | _: ParseToDate | _: ParseToTimestamp | _: ToTimestamp | _: TruncDate | _: TruncTimestamp | _: UTCTimestamp => true // Unix date and timestamp conversions. case _: DateFromUnixDate | _: FromUnixTime | _: TimestampToLongBase | _: ToUnixTimestamp | _: UnixDate | _: UnixTime | _: UnixTimestamp => true // Date and time arithmetic. case expr if DateTimeExpressionShims.isDateTimeArithmeticExpression(expr) => true // String expressions. case _: Base64 | _: BitLength | _: Chr | _: ConcatWs | _: Decode | _: Elt | _: Empty2Null | _: Encode | _: FormatNumber | _: FormatString | _: ILike | _: InitCap | _: Left | _: Length | _: Levenshtein | _: Luhncheck | _: OctetLength | _: Overlay | _: Right | _: Sentences | _: SoundEx | _: SplitPart | _: String2StringExpression | _: String2TrimExpression | _: StringDecode | _: StringInstr | _: StringLPad | _: StringLocate | _: StringPredicate | _: StringRPad | _: StringRepeat | _: StringReplace | _: StringSpace | _: StringSplit | _: StringSplitSQL | _: StringTranslate | _: StringTrimBoth | _: Substring | _: SubstringIndex | _: ToBinary | _: TryToBinary | _: UnBase64 => true // Arithmetic expressions. case _: Abs | _: BinaryArithmetic | _: Greatest | _: Least | _: UnaryMinus | _: UnaryPositive => true // Array expressions. case _: ArrayBinaryLike | _: ArrayCompact | _: ArrayContains | _: ArrayInsert | _: ArrayJoin | _: ArrayMax | _: ArrayMin | _: ArrayPosition | _: ArrayRemove | _: ArrayRepeat | _: ArraySetLike | _: ArraySize | _: ArraysZip | _: BinaryArrayExpressionWithImplicitCast | _: Concat | _: CreateArray | _: ElementAt | _: Flatten | _: Get | _: GetArrayItem | _: GetArrayStructFields | _: Reverse | _: Sequence | _: Size | _: Slice | _: SortArray | _: TryElementAt => true // Map expressions. case _: CreateMap | _: GetMapValue | _: MapConcat | _: MapContainsKey | _: MapEntries | _: MapFromArrays | _: MapFromEntries | _: MapKeys | _: MapValues | _: StringToMap => true // Struct expressions. case _: CreateNamedStruct | _: DropField | _: GetStructField | _: UpdateFields | _: WithField => true // Hash expressions. case _: Crc32 | _: HashExpression[_] | _: Md5 | _: Sha1 | _: Sha2 => true // URL expressions. case _: ParseUrl | _: UrlDecode | _: UrlEncode => true // NULL expressions. case _: AtLeastNNonNulls | _: Coalesce | _: IsNaN | _: NaNvl | _: NullIf | _: Nvl | _: Nvl2 => true // Cast expressions. case _: Cast | _: UpCast => true // Conditional expressions. case _: If | _: CaseWhen => true case _: Alias => true // Don't attempt partition-like skipping on any unknown expressions: there's no way to // guarantee it's safe to do so. case _ => additionalPartitionLikeFilterSupportedExpressions.contains( expr.getClass.getCanonicalName) } /** * Rewrites the references in an expression to point to the collected stats over that column * (if possible). * * This is generally equivalent to [[DeltaLog.rewritePartitionFilters]], with a few differences: * 1. This method checks the eligibility of the column datatype before rewriting it to point to * the stats column (which isn't needed for partition columns). * 2. There's no need to handle scalar subqueries (other than InSubqueryExec) here - subqueries * other than InSubqueryExec aren't eligible for data filtering. * 3. AND expressions may be partially rewritten as partition-like data filters if one branch * is eligible but the other is not. * * For example: * CAST(a AS DATE) = '2024-09-11' -> CAST(parsed_stats[minValues][a] AS DATE) = '2024-09-11' * * @param expr The expression to rewrite. * @param clusteringColumnPaths The logical paths to the clustering columns in the table. * @return If the expression is safe to rewrite, return the rewritten expression and a * set of referenced attributes (with both the logical path to the column and the * column type). */ private def rewriteDataFiltersAsPartitionLikeInternal( expr: Expression, clusteringColumnPaths: Set[Seq[String]]) : Option[(Expression, Set[ResolvedPartitionLikeReference])] = expr match { // The expression is an eligible reference to an attribute. // Do NOT allow partition-like filtering on timestamp columns because timestamps are truncated // to millisecond precision, meaning that we can't guarantee that the collected minVal and // maxVal are the same. // Applying these partition-like filters will generally only be beneficial if a large // percentage of files have the same min-max value. As a rough heuristic, only allow rewriting // expressions that reference only the clustering columns (since these columns are more likely // to have the same min-max values). case SkippingEligibleColumn(c, SkippingEligibleDataType(dt)) if dt != TimestampType && dt != TimestampNTZType && (!limitPartitionLikeFiltersToClusteringColumns || clusteringColumnPaths.exists(SchemaUtils.areLogicalNamesEqual(_, c.reverse))) => // Only rewrite the expression if all stats are collected for this column. val minStatsCol = StatsColumn(MIN, c, dt) val maxStatsCol = StatsColumn(MAX, c, dt) val nullCountStatsCol = StatsColumn(NULL_COUNT, c, dt) for { minCol <- getStatsColumnOpt(minStatsCol); maxCol <- getStatsColumnOpt(maxStatsCol); nullCol <- getStatsColumnOpt(nullCountStatsCol) } yield { val resolvedAttribute = ResolvedPartitionLikeReference( Seq(minStatsCol, maxStatsCol, nullCountStatsCol), minCol.expr, maxCol.expr, nullCol.expr) (minCol.expr, Set(resolvedAttribute)) } // For other attribute references, we can't safely rewrite the expression. case SkippingEligibleColumn(_, _) => None // Explicitly disallow rewriting nondeterministic expressions. Even though this check isn't // strictly necessary (there shouldn't be any nondeterministic expressions in the whitelist), // defensively keep it due to the extreme risk of correctness issues if any nondeterministic // expressions sneak into the whitelist. case other if !other.deterministic => None // Inline subquery results to support InSet. The subquery should generally have already been // evaluated. case in: InSubqueryExec => // Values may not be defined if the subquery has been skipped - we can't apply this filter. in.values().flatMap { possiblyNullValues => // Rewrite the children of InSubqueryExec, then replace the subquery with an InSet // containing the materialized values. rewriteDataFiltersAsPartitionLikeInternal(in.child, clusteringColumnPaths).flatMap { case (rewrittenChildren, referencedStats) => Some(InSet(rewrittenChildren, possiblyNullValues.toSet), referencedStats) } } // For all other eligible expressions, recursively rewrite the children. case other if shouldRewriteAsPartitionLike(other) => val childResults = other.children.map( rewriteDataFiltersAsPartitionLikeInternal(_, clusteringColumnPaths)) Option.whenNot (childResults.exists(_.isEmpty)) { val (children, stats) = childResults.map(_.get).unzip (other.withNewChildren(children), stats.flatten.toSet) } // Don't attempt rewriting any non-whitelisted expressions. case _ => None } /** * Returns an expression that returns true if a file must be read because of a mismatched * min-max value or partial nulls on a given column. For these files, it's not safe to apply * arbitrary partition-like filters. */ private def fileMustBeScanned( resolvedPartitionLikeReference: ResolvedPartitionLikeReference, numRecordsColOpt: Option[Column]): Expression = { // Construct an expression to determine if all records in the file are null. val nullCountExpr = resolvedPartitionLikeReference.nullCountExpr val allNulls = numRecordsColOpt match { case Some(physicalNumRecords) => EqualTo(nullCountExpr, physicalNumRecords.expr) case _ => Literal(false) } // Note that there are 2 other differences in behavior between unpartitioned and partitioned // tables: // 1. If the column is a timestamp, the min-max stats are truncated to millisecond precision. // We shouldn't apply partition-like filters in this case, but // rewriteDataFiltersAsPartitionLikeInternal validates the column is not a Timestamp, // so we don't have to check here. // 2. The min-max stats on a string column might be truncated for an unpartitioned table. // Note that just validating that the min and max are equal is enough to prevent this case // - if the string is truncated, the collected max value is guaranteed to be longer than // the min value due to the tiebreaker character(s) appended at the end of the max. Not( Or( allNulls, And( EqualTo( resolvedPartitionLikeReference.minExpr, resolvedPartitionLikeReference.maxExpr), EqualTo(resolvedPartitionLikeReference.nullCountExpr, Literal(0L)) ) ) ) } /** * Rewrites the given expression as a partition-like expression if possible: * 1. Rewrite the attribute references in the expression to reference the collected min stats * on the attribute reference's column. * 2. Construct an expression that returns true if any of the referenced columns are not * partition-like on a given file. * The rewritten expression is a union of the above expressions: a file is read if it's either * not partition-like on any of the columns or if the rewritten expression evaluates to true. * * @param clusteringColumns The columns that are used for clustering. * @param expr The data filtering expression to rewrite. * @return If the expression is safe to rewrite, return the rewritten * expression. Otherwise, return None. */ def rewriteDataFiltersAsPartitionLike( clusteringColumns: Seq[String], expr: Expression): Option[DataSkippingPredicate] = { val clusteringColumnPaths = clusteringColumns.map(UnresolvedAttribute.quotedString(_).nameParts).toSet rewriteDataFiltersAsPartitionLikeInternal(expr, clusteringColumnPaths).map { case (newExpr, referencedStats) => // Create an expression that returns true if a file must be read because it has mismatched // min-max values or partial nulls on any of the referenced columns. val numRecordsStatsCol = StatsColumn(NUM_RECORDS, pathToColumn = Nil, LongType) val numRecordsColOpt = getStatsColumnOpt(numRecordsStatsCol) val statsCols = referencedStats.flatMap(_.referencedStatsCols) + numRecordsStatsCol val mustScanFileExpression = referencedStats.map { resolvedReference => fileMustBeScanned(resolvedReference, numRecordsColOpt) }.toSeq.reduceLeftOption { (l, r) => Or(l, r) }.getOrElse(Literal(false)) // Only evaluate the rewritten expression if the file passes the validation expression, // ensuring that any non-partition-like input (that might cause a filter evaluation // exception) is skipped. Note that we cannot rely on short-circuiting here, since // common subexpression elimination during codegen may move the evaluation of the // condition before that of the file validation expression, so we need to explicitly use // a conditional expression to guarantee the correct evaluation order. val finalExpr = If(mustScanFileExpression, Literal(true), newExpr) // Create the final data skipping expression - read a file either if it's has nulls on any // referenced column, has mismatched stats on any referenced column, or the filter // expression evaluates to `true`. DataSkippingPredicate(Column(finalExpr), statsCols.toSet) } } // We are doing the iterative approach because of stack depth concerns. private[stats] def areAllLeavesLiteral(e: Expression): Boolean = { val stack = scala.collection.mutable.Stack[Expression]() def pushIfNonLiteral(e: Expression): Unit = e match { case _: Literal => case _ => stack.push(e) } pushIfNonLiteral(e) while (stack.nonEmpty) { val children = stack.pop().children if (children.isEmpty) { return false } children.foreach(pushIfNonLiteral) } true } } /** * Returns an expression to access the given statistics for a specific column, or None if that * stats column does not exist. * * @param pathToStatType Path components of one of the fields declared by the `DeltaStatistics` * object. For statistics of collated strings, this path contains the * versioned collation identifier. In all other cases the path only has one * element. The path is in reverse order. * @param pathToColumn The components of the nested column name to get stats for. The components * are in reverse order. */ final protected def getStatsColumnOpt( pathToStatType: Seq[String], pathToColumn: Seq[String]): Option[Column] = { require(pathToStatType.nonEmpty, "No path to stats type provided.") // First validate that pathToStatType is a valid path in the statsSchema. We start at the root // of the stats schema and then follow the path. Note that the path is stored in reverse order. // If one of the path components does not exist, the foldRight operation returns None. val (initialColumn, initialFieldType) = pathToStatType .foldRight(Option((getBaseStatsColumn, statsSchema.asInstanceOf[DataType]))) { case (statTypePathComponent: String, Some((column: Column, struct: StructType))) => // Find the field matching the current path component name or return None otherwise. struct.fields.collectFirst { case StructField(name, dataType: DataType, _, _) if name == statTypePathComponent => (column.getField(statTypePathComponent), dataType) } case _ => None } // If the requested stats type doesn't even exist, just return None right away. This can // legitimately happen if we have no stats at all, or if column stats are disabled (in which // case only the NUM_RECORDS stat type is available). .getOrElse { return None } // Given a set of path segments in reverse order, e.g. column a.b.c is Seq("c", "b", "a"), we // use a foldRight operation to build up the requested stats column, by successively applying // each new path step against both the table schema and the stats schema. We can't use the stats // schema alone, because the caller-provided path segments use logical column names, while the // stats schema requires physical column names. Instead, we must step into the table schema to // extract that field's physical column name, and use the result to step into the stats schema. // // We use a three-tuple to track state. The traversal starts with the base column for the // requested stat type, the stats schema for the requested stat type, and the table schema. Each // step of the traversal emits the updated column, along with the stats schema and table schema // elements corresponding to that column. val initialState: Option[(Column, DataType, DataType)] = Some((initialColumn, initialFieldType, metadata.schema)) pathToColumn .foldRight(initialState) { // NOTE: Only match on StructType, because we cannot traverse through other DataTypes. case (fieldName, Some((statCol, statsSchema: StructType, tableSchema: StructType))) => // First try to step into the table schema val tableFieldOpt = tableSchema.findNestedFieldIgnoreCase(Seq(fieldName)) // If that worked, try to step into the stats schema, using its its physical name val statsFieldOpt = tableFieldOpt .map(DeltaColumnMapping.getPhysicalName) .filter(physicalFieldName => statsSchema.exists(_.name == physicalFieldName)) .map(statsSchema(_)) // If all that succeeds, return the new stats column and the corresponding data types. statsFieldOpt.map(statsField => (statCol.getField(statsField.name), statsField.dataType, tableFieldOpt.get.dataType)) // Propagate failure if the above match failed (or if already None) case _ => None } // Filter out non-leaf columns -- they lack stats so skipping predicates can't use them. .filterNot(_._2.isInstanceOf[StructType]) .map { case (statCol, TimestampType, _) if pathToStatType.head == MAX => getAdjustedTimestamp(statCol, TimestampType) case (statCol, TimestampNTZType, _) if pathToStatType.head == MAX => getAdjustedTimestamp(statCol, TimestampNTZType) case (statCol, _, _) => statCol } } /** Convenience overload for single element stat type paths. */ final protected def getStatsColumnOpt( statType: String, pathToColumn: Seq[String] = Nil): Option[Column] = getStatsColumnOpt(Seq(statType), pathToColumn) /** * Returns an expression to access the given statistics for a specific column, or a NULL * literal expression if that column does not exist. */ final protected[delta] def getStatsColumnOrNullLiteral( statType: String, pathToColumn: Seq[String] = Nil) : Column = getStatsColumnOpt(Seq(statType), pathToColumn).getOrElse(lit(null)) /** Overload for convenience working with StatsColumn helpers */ final protected def getStatsColumnOpt(stat: StatsColumn): Option[Column] = getStatsColumnOpt(stat.pathToStatType, stat.pathToColumn) /** Overload for convenience working with StatsColumn helpers */ final protected[delta] def getStatsColumnOrNullLiteral(stat: StatsColumn): Column = getStatsColumnOpt(stat.pathToStatType, stat.pathToColumn).getOrElse(lit(null)) /** Overload for delta table property override */ override protected def getDataSkippingStringPrefixLength: Int = StatsCollectionUtils.getDataSkippingStringPrefixLength(spark, metadata) /** * Returns an expression that can be used to check that the required statistics are present for a * given file. If any required statistics are missing we must include the corresponding file. * * NOTE: We intentionally choose to disable skipping for any file if any required stat is missing, * because doing it that way allows us to check each stat only once (rather than once per * use). Checking per-use would anyway only help for tables where the number of indexed columns * has changed over time, producing add.stats_parsed records with differing schemas. That should * be a rare enough case to not worry about optimizing for, given that the fix requires more * complex skipping predicates that would penalize the common case. */ protected def verifyStatsForFilter(referencedStats: Set[StatsColumn]): Column = { recordFrameProfile("Delta", "DataSkippingReader.verifyStatsForFilter") { // The NULL checks for MIN and MAX stats depend on NULL_COUNT and NUM_RECORDS. Derive those // implied dependencies first, so the main pass can treat them like any other column. // // NOTE: We must include explicit NULL checks on all stats columns we access here, because our // caller will negate the expression we return. In case a stats column is NULL, `NOT(expr)` // must return `TRUE`, and without these NULL checks it would instead return // `NOT(NULL)` => `NULL`. referencedStats.flatMap { stat => stat match { case StatsColumn(MIN +: _, _) | StatsColumn(MAX +: _, _) => Seq(stat, StatsColumn(NULL_COUNT, stat.pathToColumn, LongType), StatsColumn(NUM_RECORDS, pathToColumn = Nil, LongType)) case _ => Seq(stat) }}.map{stat => stat match { // A usable MIN or MAX stat must be non-NULL, unless the column is provably all-NULL // // NOTE: We don't care about NULL/missing NULL_COUNT and NUM_RECORDS here, because the // separate NULL checks we emit for those columns will force the overall validation // predicate conjunction to FALSE in that case -- AND(FALSE, ) is FALSE. case StatsColumn(MIN +: _, _) | StatsColumn(MAX +: _, _) => getStatsColumnOrNullLiteral(stat).isNotNull || (getStatsColumnOrNullLiteral(NULL_COUNT, stat.pathToColumn) === getStatsColumnOrNullLiteral(NUM_RECORDS)) case _ => // Other stats, such as NULL_COUNT and NUM_RECORDS stat, merely need to be non-NULL getStatsColumnOrNullLiteral(stat).isNotNull }} .reduceLeftOption(_.and(_)) .getOrElse(trueLiteral) } } private def buildSizeCollectorFilter(): (ArrayAccumulator, Column => Column) = { val bytesCompressed = col("size") val rows = getStatsColumnOrNullLiteral(NUM_RECORDS) val dvCardinality = coalesce(col("deletionVector.cardinality"), lit(0L)) val logicalRows = (rows - dvCardinality).as("logicalRows") val accumulator = new ArrayAccumulator(4) spark.sparkContext.register(accumulator) // The arguments (order and datatype) must match the encoders defined in the // `sizeCollectorInputEncoders` value. val collector = (include: Boolean, bytesCompressed: java.lang.Long, logicalRows: java.lang.Long, rows: java.lang.Long) => { if (include) { accumulator.add((0, bytesCompressed)) /* count bytes of AddFiles */ accumulator.add((1, Option(rows).map(_.toLong).getOrElse(-1L))) /* count rows in AddFiles */ accumulator.add((2, 1)) /* count number of AddFiles */ accumulator.add((3, Option(logicalRows) .map(_.toLong).getOrElse(-1L))) /* count logical rows in AddFiles */ } include } val collectorUdf = SparkUserDefinedFunction( f = collector, dataType = BooleanType, inputEncoders = sizeCollectorInputEncoders, deterministic = false) (accumulator, collectorUdf(_: Column, bytesCompressed, logicalRows, rows)) } override def filesWithStatsForScan(partitionFilters: Seq[Expression]): DataFrame = { DeltaLog.filterFileList(metadata.partitionSchema, withStats, partitionFilters) } /** * Get all the files in this table. * * @param keepNumRecords Also select `stats.numRecords` in the query. * This may slow down the query as it has to parse json. */ protected def getAllFiles(keepNumRecords: Boolean): Seq[AddFile] = recordFrameProfile( "Delta", "DataSkippingReader.getAllFiles") { val ds = if (keepNumRecords) { withStats // use withStats instead of allFiles so the `stats` column is already parsed // keep only the numRecords field as a Json string in the stats field .withColumn("stats", to_json(struct(col("stats.numRecords") as "numRecords"))) } else { allFiles.withColumn("stats", nullStringLiteral) } convertDataFrameToAddFiles(ds.toDF()) } /** * Given the partition filters on the data, rewrite these filters by pointing to the metadata * columns. */ protected def constructPartitionFilters(filters: Seq[Expression]): Column = { recordFrameProfile("Delta", "DataSkippingReader.constructPartitionFilters") { val rewritten = DeltaLog.rewritePartitionFilters( metadata.partitionSchema, spark.sessionState.conf.resolver, filters) rewritten.reduceOption(And).map { expr => Column(expr) }.getOrElse(trueLiteral) } } /** * Get all the files in this table given the partition filter and the corresponding size of * the scan. * * @param keepNumRecords Also select `stats.numRecords` in the query. * This may slow down the query as it has to parse json. */ protected def filterOnPartitions( partitionFilters: Seq[Expression], keepNumRecords: Boolean): (Seq[AddFile], DataSize) = recordFrameProfile( "Delta", "DataSkippingReader.filterOnPartitions") { val forceCollectRowCount = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_ALWAYS_COLLECT_STATS) val shouldCollectStats = keepNumRecords || forceCollectRowCount val df = if (shouldCollectStats) { // use withStats instead of allFiles so the `stats` column is already parsed val filteredFiles = DeltaLog.filterFileList(metadata.partitionSchema, withStats, partitionFilters) filteredFiles // keep only the numRecords field as a Json string in the stats field .withColumn("stats", to_json(struct(col("stats.numRecords") as "numRecords"))) } else { val filteredFiles = DeltaLog.filterFileList(metadata.partitionSchema, allFiles.toDF(), partitionFilters) filteredFiles .withColumn("stats", nullStringLiteral) } val files = convertDataFrameToAddFiles(df) val sizeInBytesByPartitionFilters = files.map(_.size).sum // Compute row count if we have stats available and forceCollectRowCount is enabled val (rowCount, logicalRowCount) = if (forceCollectRowCount) { sumRowCounts(files) } else { (None, None) } files.toSeq -> DataSize(Some(sizeInBytesByPartitionFilters), rowCount, Some(files.size), logicalRowCount) } /** * Sums up the numPhysicalRecords and numLogicalRecords from the given AddFile objects. * Returns (None, None) if any file is missing physical record stats. * Returns (Some(physical), None) if any file is missing logical record stats. */ private def sumRowCounts(files: Seq[AddFile]): (Option[Long], Option[Long]) = { var physicalRows = 0L var logicalRows = 0L var physicalMissing = false var logicalMissing = false files.foreach { file => physicalMissing = physicalMissing || file.numPhysicalRecords.isEmpty logicalMissing = logicalMissing || file.numLogicalRecords.isEmpty physicalRows += file.numPhysicalRecords.getOrElse(0L) logicalRows += file.numLogicalRecords.getOrElse(0L) } ( if (physicalMissing) None else Some(physicalRows), if (logicalMissing) None else Some(logicalRows) ) } /** * Given the partition and data filters, leverage data skipping statistics to find the set of * files that need to be queried. Returns a tuple of the files and optionally the size of the * scan that's generated if there were no filters, if there were only partition filters, and * combined effect of partition and data filters respectively. */ protected def getDataSkippedFiles( partitionFilters: Column, dataFilters: DataSkippingPredicate, keepNumRecords: Boolean): (Seq[AddFile], Seq[DataSize]) = recordFrameProfile( "Delta", "DataSkippingReader.getDataSkippedFiles") { val (totalSize, totalFilter) = buildSizeCollectorFilter() val (partitionSize, partitionFilter) = buildSizeCollectorFilter() val (scanSize, scanFilter) = buildSizeCollectorFilter() // NOTE: If any stats are missing, the value of `dataFilters` is untrustworthy -- it could be // NULL or even just plain incorrect. We rely on `verifyStatsForFilter` to be FALSE in that // case, forcing the overall OR to evaluate as TRUE no matter what value `dataFilters` takes. val filteredFiles = withStats.where( totalFilter(trueLiteral) && partitionFilter(partitionFilters) && scanFilter(dataFilters.expr || !verifyStatsForFilter(dataFilters.referencedStats)) ) val statsColumn = if (keepNumRecords) { // keep only the numRecords field as a Json string in the stats field to_json(struct(col("stats.numRecords") as "numRecords")) } else nullStringLiteral val files = recordFrameProfile("Delta", "DataSkippingReader.getDataSkippedFiles.collectFiles") { val df = filteredFiles.withColumn("stats", statsColumn) convertDataFrameToAddFiles(df) } files.toSeq -> Seq(DataSize(totalSize), DataSize(partitionSize), DataSize(scanSize)) } private def getCorrectDataSkippingType( dataSkippingType: DeltaDataSkippingType): DeltaDataSkippingType = { dataSkippingType } /** * Gathers files that should be included in a scan based on the given predicates. * Statistics about the amount of data that will be read are gathered and returned. * Note, the statistics column that is added when keepNumRecords = true should NOT * take into account DVs. Consumers of this method might commit the file. The semantics * of the statistics need to be consistent across all files. */ override def filesForScan(filters: Seq[Expression], keepNumRecords: Boolean): DeltaScan = { val startTime = System.currentTimeMillis() if (filters == Seq(TrueLiteral) || filters.isEmpty || schema.isEmpty) { recordDeltaOperation(deltaLog, "delta.skipping.none") { // When there are no filters we can just return allFiles with no extra processing val forceCollectRowCount = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_ALWAYS_COLLECT_STATS) val shouldCollectStats = keepNumRecords || forceCollectRowCount lazy val files = getAllFiles(shouldCollectStats) // Compute row count if forceCollectRowCount is enabled val (rowCount, logicalRowCount) = if (forceCollectRowCount) { sumRowCounts(files) } else { (None, None) } val dataSize = DataSize( bytesCompressed = sizeInBytesIfKnown, rows = rowCount, files = numOfFilesIfKnown, logicalRows = logicalRowCount) return DeltaScan( version = version, files = files, total = dataSize, partition = dataSize, scanned = dataSize)( scannedSnapshot = snapshotToScan, partitionFilters = ExpressionSet(Nil), dataFilters = ExpressionSet(Nil), partitionLikeDataFilters = ExpressionSet(Nil), rewrittenPartitionLikeDataFilters = Set.empty, unusedFilters = ExpressionSet(Nil), scanDurationMs = System.currentTimeMillis() - startTime, dataSkippingType = getCorrectDataSkippingType(DeltaDataSkippingType.noSkippingV1) ) } } import DeltaTableUtils._ val partitionColumns = metadata.partitionColumns // For data skipping, avoid using the filters that either: // 1. involve subqueries. // 2. are non-deterministic. // 3. involve file metadata struct fields var (ineligibleFilters, eligibleFilters) = filters.partition { case f => containsSubquery(f) || !f.deterministic || f.exists { case MetadataAttribute(_) => true case _ => false } } val (partitionFilters, dataFilters) = eligibleFilters .partition(isPredicatePartitionColumnsOnly(_, partitionColumns, spark)) if (dataFilters.isEmpty) recordDeltaOperation(deltaLog, "delta.skipping.partition") { // When there are only partition filters we can scan allFiles // rather than withStats and thus we skip data skipping information. val (files, scanSize) = filterOnPartitions(partitionFilters, keepNumRecords) DeltaScan( version = version, files = files, total = DataSize(sizeInBytesIfKnown, None, numOfFilesIfKnown), partition = scanSize, scanned = scanSize)( scannedSnapshot = snapshotToScan, partitionFilters = ExpressionSet(partitionFilters), dataFilters = ExpressionSet(Nil), partitionLikeDataFilters = ExpressionSet(Nil), rewrittenPartitionLikeDataFilters = Set.empty, unusedFilters = ExpressionSet(ineligibleFilters), scanDurationMs = System.currentTimeMillis() - startTime, dataSkippingType = getCorrectDataSkippingType(DeltaDataSkippingType.partitionFilteringOnlyV1) ) } else recordDeltaOperation(deltaLog, "delta.skipping.data") { val finalPartitionFilters = constructPartitionFilters(partitionFilters) val dataSkippingType = if (partitionFilters.isEmpty) { DeltaDataSkippingType.dataSkippingOnlyV1 } else { DeltaDataSkippingType.dataSkippingAndPartitionFilteringV1 } var (skippingFilters, unusedFilters) = if (useStats) { val constructDataFilters = new DataFiltersBuilder(spark, dataSkippingType) dataFilters.map(f => (f, constructDataFilters(f))).partition(f => f._2.isDefined) } else { (Nil, dataFilters.map(f => (f, None))) } // If enabled, rewrite unused data filters to use partition-like data skipping for clustered // tables. Only rewrite filters if the table is expected to benefit from partition-like // data skipping: // 1. The table should be have a large portion of files with the same min-max values on the // referenced columns - as a rough heuristic, require the table to be a clustered table, as // many files often have the same min-max on the clustering columns. // 2. The table should be large enough to benefit from partition-like data skipping - as a // rough heuristic, require the table to no longer be considered a "small delta table." // 3. At least 1 data filter was not already used for data skipping. val shouldRewriteDataFiltersAsPartitionLike = spark.conf.get(DeltaSQLConf.DELTA_DATASKIPPING_PARTITION_LIKE_FILTERS_ENABLED) && ClusteredTableUtils.isSupported(snapshotToScan.protocol) && snapshotToScan.numOfFilesIfKnown.exists(_ >= spark.conf.get(DeltaSQLConf.DELTA_DATASKIPPING_PARTITION_LIKE_FILTERS_THRESHOLD)) && unusedFilters.nonEmpty val partitionLikeFilters = if (shouldRewriteDataFiltersAsPartitionLike) { val clusteringColumns = ClusteringColumnInfo.extractLogicalNames(snapshotToScan) val (rewrittenUsedFilters, rewrittenUnusedFilters) = { val constructDataFilters = new DataFiltersBuilder(spark, dataSkippingType) unusedFilters .map { case (expr, _) => val rewrittenExprOpt = constructDataFilters.rewriteDataFiltersAsPartitionLike( clusteringColumns, expr) (expr, rewrittenExprOpt) } .partition(_._2.isDefined) } skippingFilters = skippingFilters ++ rewrittenUsedFilters unusedFilters = rewrittenUnusedFilters rewrittenUsedFilters.map { case (orig, rewrittenOpt) => (orig, rewrittenOpt.get) } } else { Nil } val finalSkippingFilters = skippingFilters .map(_._2.get) .reduceOption((skip1, skip2) => DataSkippingPredicate( // Fold the filters into a conjunction, while unioning their referencedStats. skip1.expr && skip2.expr, skip1.referencedStats ++ skip2.referencedStats)) .getOrElse(DataSkippingPredicate(trueLiteral)) val (files, sizes) = { getDataSkippedFiles(finalPartitionFilters, finalSkippingFilters, keepNumRecords) } DeltaScan( version = version, files = files, total = sizes(0), partition = sizes(1), scanned = sizes(2))( scannedSnapshot = snapshotToScan, partitionFilters = ExpressionSet(partitionFilters), dataFilters = ExpressionSet(skippingFilters.map(_._1)), partitionLikeDataFilters = ExpressionSet(partitionLikeFilters.map(_._1)), rewrittenPartitionLikeDataFilters = partitionLikeFilters.map(_._2.expr.expr).toSet, unusedFilters = ExpressionSet(unusedFilters.map(_._1) ++ ineligibleFilters), scanDurationMs = System.currentTimeMillis() - startTime, dataSkippingType = getCorrectDataSkippingType(dataSkippingType) ) } } /** * Gathers files that should be included in a scan based on the given predicates and limit. * This will be called only when all predicates are on partitioning columns. * Statistics about the amount of data that will be read are gathered and returned. */ override def filesForScan(limit: Long, partitionFilters: Seq[Expression]): DeltaScan = recordDeltaOperation(deltaLog, "delta.skipping.filteredLimit") { val startTime = System.currentTimeMillis() val finalPartitionFilters = constructPartitionFilters(partitionFilters) val scan = { pruneFilesByLimit(withStats.where(finalPartitionFilters), limit) } val totalDataSize = new DataSize( sizeInBytesIfKnown, None, numOfFilesIfKnown, None ) val scannedDataSize = new DataSize( scan.byteSize, scan.numPhysicalRecords, Some(scan.files.size), scan.numLogicalRecords ) DeltaScan( version = version, files = scan.files, total = totalDataSize, partition = null, scanned = scannedDataSize)( scannedSnapshot = snapshotToScan, partitionFilters = ExpressionSet(partitionFilters), dataFilters = ExpressionSet(Nil), partitionLikeDataFilters = ExpressionSet(Nil), rewrittenPartitionLikeDataFilters = Set.empty, unusedFilters = ExpressionSet(Nil), scanDurationMs = System.currentTimeMillis() - startTime, dataSkippingType = DeltaDataSkippingType.filteredLimit ) } /** * Get AddFile (with stats) actions corresponding to given set of paths in the Snapshot. * If a path doesn't exist in snapshot, it will be ignored and no [[AddFile]] will be returned * for it. * @param paths Sequence of paths for which we want to get [[AddFile]] action * @return a sequence of addFiles for the given `paths` */ def getSpecificFilesWithStats(paths: Seq[String]): Seq[AddFile] = { recordFrameProfile("Delta", "DataSkippingReader.getSpecificFilesWithStats") { val right = paths.toDF(spark, "path") val df = allFiles.join(right, Seq("path"), "leftsemi") convertDataFrameToAddFiles(df) } } /** Get the files and number of records within each file, to perform limit pushdown. */ def getFilesAndNumRecords( df: DataFrame): Iterator[(AddFile, NumRecords)] with Closeable = recordFrameProfile( "Delta", "DataSkippingReaderEdge.getFilesAndNumRecords") { import org.apache.spark.sql.delta.implicits._ val dvCardinality = coalesce(col("deletionVector.cardinality"), lit(0L)) val numLogicalRecords = col("stats.numRecords") - dvCardinality val result = df.withColumn("numPhysicalRecords", col("stats.numRecords")) // Physical .withColumn("numLogicalRecords", numLogicalRecords) // Logical .withColumn("stats", nullStringLiteral) .select(struct(col("*")).as[AddFile], col("numPhysicalRecords").as[java.lang.Long], col("numLogicalRecords").as[java.lang.Long]) .collectAsList() new Iterator[(AddFile, NumRecords)] with Closeable { private val underlying = result.iterator override def hasNext: Boolean = underlying.hasNext override def next(): (AddFile, NumRecords) = { val next = underlying.next() (next._1, NumRecords(numPhysicalRecords = next._2, numLogicalRecords = next._3)) } override def close(): Unit = { } } } protected def convertDataFrameToAddFiles(df: DataFrame): Array[AddFile] = { df.as[AddFile].collect() } protected[delta] def pruneFilesByLimit(df: DataFrame, limit: Long): ScanAfterLimit = { val withNumRecords = { getFilesAndNumRecords(df) } pruneFilesWithIterator(withNumRecords, limit) } /** * Accepts an iterator of files with record counts and prunes them based on the limit. */ protected def pruneFilesWithIterator( withNumRecords: Iterator[(AddFile, NumRecords)] with Closeable, limit: Long): ScanAfterLimit = { var logicalRowsToScan = 0L var physicalRowsToScan = 0L var bytesToScan = 0L var bytesToIgnore = 0L var rowsUnknown = false val filesAfterLimit = try { val iter = withNumRecords val filesToScan = ArrayBuffer[AddFile]() val filesToIgnore = ArrayBuffer[AddFile]() while (iter.hasNext && logicalRowsToScan < limit) { val file = iter.next() if (file._2.numPhysicalRecords == null || file._2.numLogicalRecords == null) { // this file has no stats, ignore for now bytesToIgnore += file._1.size filesToIgnore += file._1 } else { physicalRowsToScan += file._2.numPhysicalRecords.toLong logicalRowsToScan += file._2.numLogicalRecords.toLong bytesToScan += file._1.size filesToScan += file._1 } } // If the files that have stats do not contain enough rows, fall back to reading all files if (logicalRowsToScan < limit && filesToIgnore.nonEmpty) { filesToScan ++= filesToIgnore bytesToScan += bytesToIgnore rowsUnknown = true } filesToScan.toSeq } finally { withNumRecords.close() } if (rowsUnknown) { ScanAfterLimit(filesAfterLimit, Some(bytesToScan), None, None) } else { ScanAfterLimit(filesAfterLimit, Some(bytesToScan), Some(physicalRowsToScan), Some(logicalRowsToScan)) } } } trait DataSkippingReader extends DataSkippingReaderBase ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingStatsTracker.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats import scala.collection.mutable import org.apache.spark.sql.delta.expressions.JoinedProjection import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate._ import org.apache.spark.sql.catalyst.expressions.codegen.GenerateMutableProjection import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.spark.util.SerializableConfiguration /** * A [[WriteTaskStats]] that contains a map from file name to the json representation * of the collected statistics. */ case class DeltaFileStatistics(stats: Map[String, String]) extends WriteTaskStats /** * A per-task (i.e. one instance per executor) [[WriteTaskStatsTracker]] that collects the * statistics defined by [[StatisticsCollection]] for files that are being written into a delta * table. * * @param dataCols Resolved data (i.e. non-partitionBy) columns of the dataframe to be written. * @param statsColExpr Resolved expression for computing all the statistics that we want to gather. * @param rootPath The Reservoir's root path. * @param hadoopConf Hadoop Config for being able to instantiate a [[FileSystem]]. */ class DeltaTaskStatisticsTracker( dataCols: Seq[Attribute], statsColExpr: Expression, rootPath: Path, hadoopConf: Configuration) extends WriteTaskStatsTracker { protected[this] val submittedFiles = mutable.HashMap[String, InternalRow]() // For example, when strings are involved, statsColExpr might look like // struct( // count(new Column("*")) as "numRecords" // struct( // substring(min(col), 0, stringPrefix)) // ) as "minValues", // struct( // udf(max(col)) // ) as "maxValues" // ) as "stats" // [[DeclarativeAggregate]] is the API to the Catalyst machinery for initializing and updating // the result of an aggregate function. We will be using it here the same way it's used during // query execution. // Given the example above, aggregates would hold: Seq(count, min, max) private val aggregates: Seq[DeclarativeAggregate] = statsColExpr.collect { case ae: AggregateExpression if ae.aggregateFunction.isInstanceOf[DeclarativeAggregate] => ae.aggregateFunction.asInstanceOf[DeclarativeAggregate] } // The fields of aggBuffer - see below protected val aggBufferAttrs: Seq[Attribute] = aggregates.flatMap(_.aggBufferAttributes) // This projection initializes aggBuffer with the neutral values for the agg fcns e.g. 0 for sum protected val initializeStats: MutableProjection = GenerateMutableProjection.generate( expressions = aggregates.flatMap(_.initialValues), inputSchema = Seq.empty, useSubexprElimination = false ) // This projection combines the intermediate results stored by aggBuffer with the values of the // currently processed row and updates aggBuffer in place. private val updateStats: MutableProjection = { val aggs = aggregates.flatMap(_.updateExpressions) val expressions = JoinedProjection.bind(aggBufferAttrs, dataCols, aggs) if (SQLConf.get.getConf( DeltaSQLConf.DELTA_STATS_COLLECTION_FALLBACK_TO_INTERPRETED_PROJECTION)) { MutableProjection.create( exprs = expressions, inputSchema = Nil ) } else { GenerateMutableProjection.generate( expressions = expressions, inputSchema = Nil, useSubexprElimination = true ) } } // This executes the whole statsColExpr in order to compute the final stats value for the file. // In order to evaluate it, we have to replace its aggregate functions with the corresponding // aggregates' evaluateExpressions that basically just return the results stored in aggBuffer. private val resultExpr: Expression = statsColExpr.transform { case ae: AggregateExpression if ae.aggregateFunction.isInstanceOf[DeclarativeAggregate] => ae.aggregateFunction.asInstanceOf[DeclarativeAggregate].evaluateExpression } // See resultExpr above private val getStats: Projection = UnsafeProjection.create( exprs = Seq(resultExpr), inputSchema = aggBufferAttrs ) // This serves as input to updateStats, with aggBuffer always on the left, while the right side // is every time replaced with the row currently being processed - see updateStats and newRow. private val extendedRow: GenericInternalRow = new GenericInternalRow(2) // file path to corresponding stats encoded as json protected val results = new collection.mutable.HashMap[String, String] // called once per file, executes the getStats projection override def closeFile(filePath: String): Unit = { // We assume file names are unique val fileName = new Path(filePath).getName assert(!results.contains(fileName), s"Stats already recorded for file: $filePath") // this is statsColExpr's output (json string) val jsonStats = getStats(submittedFiles(filePath)).getString(0) results += ((fileName, jsonStats)) submittedFiles.remove(filePath) } override def newPartition(partitionValues: InternalRow): Unit = { } protected def initializeAggBuf(buffer: SpecificInternalRow): InternalRow = initializeStats.target(buffer).apply(EmptyRow) override def newFile(newFilePath: String): Unit = { submittedFiles.getOrElseUpdate(newFilePath, { // `buffer` is a row that will start off by holding the initial values for the agg expressions // (see the initializeStats: Projection), will then be updated in place every time a new row // is processed (see updateStats: Projection), and will finally serve as an input for // computing the per-file result of statsColExpr (see getStats: Projection) val buffer = new SpecificInternalRow(aggBufferAttrs.map(_.dataType)) initializeAggBuf(buffer) }) } override def newRow(filePath: String, currentRow: InternalRow): Unit = { val aggBuffer = submittedFiles(filePath) extendedRow.update(0, aggBuffer) extendedRow.update(1, currentRow) updateStats.target(aggBuffer).apply(extendedRow) } override def getFinalStats(taskCommitTime: Long): DeltaFileStatistics = { submittedFiles.keys.foreach(closeFile) submittedFiles.clear() DeltaFileStatistics(results.toMap) } } /** * Serializable factory class that holds together all required parameters for being able to * instantiate a [[DeltaTaskStatisticsTracker]] on an executor. * * @param hadoopConf The Hadoop configuration object to use on an executor. * @param path Root Reservoir path * @param dataCols Resolved data (i.e. non-partitionBy) columns of the dataframe to be written. */ class DeltaJobStatisticsTracker( @transient private val hadoopConf: Configuration, @transient val path: Path, val dataCols: Seq[Attribute], val statsColExpr: Expression ) extends WriteJobStatsTracker with EvalHelper { var recordedStats: Map[String, String] = _ private val srlHadoopConf = new SerializableConfiguration(hadoopConf) private val rootUri = path.getFileSystem(hadoopConf).makeQualified(path).toUri() override def newTaskInstance(): WriteTaskStatsTracker = { val rootPath = new Path(rootUri) val hadoopConf = srlHadoopConf.value new DeltaTaskStatisticsTracker(dataCols, prepareForEval(statsColExpr), rootPath, hadoopConf) } override def processStats(stats: Seq[WriteTaskStats], jobCommitTime: Long): Unit = { recordedStats = stats.map(_.asInstanceOf[DeltaFileStatistics]).flatMap(_.stats).toMap } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/stats/DeletedRecordCountsHistogram.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats import java.util.Arrays import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.types.StructType /** * A Histogram class tracking the deleted record count distribution for all files in a table. * @param deletedRecordCounts An array with 10 bins where each slot represents the number of * files where the number of deleted records falls within the range * of the particular bin. The range of each bin is the following: * bin1 -> [0,0] * bin2 -> [1,9] * bin3 -> [10,99] * bin4 -> [100,999], * bin5 -> [1000,9999] * bin6 -> [10000,99999], * bin7 -> [100000,999999], * bin8 -> [1000000,9999999], * bin9 -> [10000000,Int.Max - 1], * bin10 -> [Int.Max,Long.Max]. */ case class DeletedRecordCountsHistogram(deletedRecordCounts: Array[Long]) { require(deletedRecordCounts.length == DeletedRecordCountsHistogramUtils.NUMBER_OF_BINS, s"There should be ${DeletedRecordCountsHistogramUtils.NUMBER_OF_BINS} bins in total") override def hashCode(): Int = 31 * Arrays.hashCode(deletedRecordCounts) + getClass.getCanonicalName.hashCode override def equals(that: Any): Boolean = that match { case DeletedRecordCountsHistogram(thatDP) => java.util.Arrays.equals(deletedRecordCounts, thatDP) case _ => false } /** * Insert a given value into the appropriate histogram bin. */ def insert(numDeletedRecords: Long): Unit = { if (numDeletedRecords >= 0) { val index = DeletedRecordCountsHistogramUtils.getHistogramBin(numDeletedRecords) deletedRecordCounts(index) += 1 } } /** * Remove a given value from the appropriate histogram bin. */ def remove(numDeletedRecords: Long): Unit = { if (numDeletedRecords >= 0) { val index = DeletedRecordCountsHistogramUtils.getHistogramBin(numDeletedRecords) deletedRecordCounts(index) -= 1 } } } private[delta] object DeletedRecordCountsHistogram { def apply(deletionPercentages: Array[Long]): DeletedRecordCountsHistogram = new DeletedRecordCountsHistogram(deletionPercentages) lazy val schema: StructType = ExpressionEncoder[DeletedRecordCountsHistogram]().schema } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/stats/DeletedRecordCountsHistogramUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats import org.apache.spark.sql.delta.{DeltaErrors, DeltaUDF} import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.expressions.aggregate.TypedImperativeAggregate import org.apache.spark.sql.catalyst.trees.UnaryLike import org.apache.spark.sql.catalyst.util.GenericArrayData import org.apache.spark.sql.functions.udf import org.apache.spark.sql.types.{ArrayType, DataType, LongType} import org.apache.spark.unsafe.Platform /** * This object contains helper functionality related to [[DeletedRecordCountsHistogram]]. */ object DeletedRecordCountsHistogramUtils { val BUCKET_BOUNDARIES = IndexedSeq( 0L, 1L, 10L, 100L, 1000L, 10000L, 100000L, 1000000L, 10000000L, Int.MaxValue, Long.MaxValue) val NUMBER_OF_BINS = BUCKET_BOUNDARIES.length - 1 def getDefaultBins: Array[Long] = Array.fill(NUMBER_OF_BINS)(0L) def emptyHistogram: DeletedRecordCountsHistogram = DeletedRecordCountsHistogram.apply(getDefaultBins) def getHistogramBin(dvCardinality: Long): Int = { import scala.collection.Searching._ require(dvCardinality >= 0) if (dvCardinality == Long.MaxValue) return NUMBER_OF_BINS - 1 BUCKET_BOUNDARIES.search(dvCardinality) match { case Found(index) => index case InsertionPoint(insertionPoint) => insertionPoint - 1 } } /** * An imperative aggregate implementation of DeletedRecordCountsHistogram. * * The return type of this Imperative Aggregate is of ArrayType(LongType). The array * represents a [[DeletedRecordCountsHistogram]]. * */ case class DeletedRecordCountsHistogramAgg( child: Expression, mutableAggBufferOffset: Int = 0, inputAggBufferOffset: Int = 0) extends TypedImperativeAggregate[Array[Long]] with UnaryLike[Expression] { override def createAggregationBuffer(): Array[Long] = getDefaultBins override val dataType: DataType = ArrayType(LongType) // This Aggregate doesn't return null. override val nullable: Boolean = false override protected def withNewChildInternal( newChild: Expression): DeletedRecordCountsHistogramAgg = copy(child = newChild) override def update(aggBuffer: Array[Long], input: InternalRow): Array[Long] = { val value = child.eval(input) if (value != null) { val dvCardinality = value.asInstanceOf[Long] val index = getHistogramBin(dvCardinality) aggBuffer(index) += 1 } aggBuffer } override def merge(buffer: Array[Long], input: Array[Long]): Array[Long] = { require(buffer.length == input.length) for (index <- buffer.indices) { buffer(index) += input(index) } buffer } override def eval(buffer: Array[Long]): Any = new GenericArrayData(buffer) /** Serializes the aggregation buffer to Array[Byte]. */ override def serialize(buffer: Array[Long]): Array[Byte] = { require(buffer.length < 128) val bytesPerLong = 8 // One 8bit value stores the number of elements, the remaining are bucket values. val serializedByteSize = (buffer.length * bytesPerLong) + 1 val byteArray = new Array[Byte](serializedByteSize) // Add buffer length for validation. Platform.putByte(byteArray, Platform.BYTE_ARRAY_OFFSET, buffer.length.toByte) for (index <- buffer.indices) { val offset = Platform.BYTE_ARRAY_OFFSET + 1 + index * bytesPerLong Platform.putLong(byteArray, offset, buffer(index)) } byteArray } /** De-serializes the serialized format Array[Byte], and produces aggregation buffer. */ override def deserialize(bytes: Array[Byte]): Array[Long] = { val bytesPerLong = 8 // One 8bit value stores the number of elements, the remaining are bucket values. val numElementsFromSerializedByteSize = (bytes.length - 1) / bytesPerLong val aggBuffer = new Array[Long](numElementsFromSerializedByteSize) // At the first byte we store the length of the deserialized buffer for validation purposes. val numElementsFromSerializedState = Platform.getByte(bytes, Platform.BYTE_ARRAY_OFFSET).toInt if (numElementsFromSerializedByteSize != numElementsFromSerializedState) { throw DeltaErrors.deletedRecordCountsHistogramDeserializationException() } for (index <- aggBuffer.indices) { val offset = Platform.BYTE_ARRAY_OFFSET + 1 + index * bytesPerLong aggBuffer(index) = Platform.getLong(bytes, offset) } aggBuffer } override def withNewMutableAggBufferOffset(offset: Int): DeletedRecordCountsHistogramAgg = copy(mutableAggBufferOffset = offset) override def withNewInputAggBufferOffset(offset: Int): DeletedRecordCountsHistogramAgg = copy(inputAggBufferOffset = offset) } /** * A UDF to convert a long array (returned by [[DeletedRecordCountsHistogramAgg]]) to * [[DeletedRecordCountsHistogram]]. */ private lazy val HistogramAggrToHistogramUDF = { DeltaUDF.deletedRecordCountsHistogramFromArrayLong { deletedRecordCountsHistogramArray => new DeletedRecordCountsHistogram(deletedRecordCountsHistogramArray) } } def histogramAggregate(dvCardinalityExpr: Column): Column = { val aggregate = Column(DeletedRecordCountsHistogramAgg(dvCardinalityExpr.expr).toAggregateExpression()) DeletedRecordCountsHistogramUtils.HistogramAggrToHistogramUDF(aggregate) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/stats/DeltaScan.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats // scalastyle:off import.ordering.noEmptyLine import scala.util.control.NonFatal import org.apache.spark.sql.delta.Snapshot import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.stats.DeltaDataSkippingType.DeltaDataSkippingType import com.fasterxml.jackson.databind.annotation.JsonDeserialize import org.apache.spark.sql.catalyst.expressions._ /** * DataSize describes following attributes for data that consists of a list of input files * @param bytesCompressed total size of the data * @param rows number of rows in the data * @param files number of input files * Note: Please don't add any new constructor to this class. `jackson-module-scala` always picks up * the first constructor returned by `Class.getConstructors` but the order of the constructors list * is non-deterministic. (SC-13343) */ case class DataSize( @JsonDeserialize(contentAs = classOf[java.lang.Long]) bytesCompressed: Option[Long] = None, @JsonDeserialize(contentAs = classOf[java.lang.Long]) rows: Option[Long] = None, @JsonDeserialize(contentAs = classOf[java.lang.Long]) files: Option[Long] = None, @JsonDeserialize(contentAs = classOf[java.lang.Long]) logicalRows: Option[Long] = None ) object DataSize { def apply(a: ArrayAccumulator): DataSize = { DataSize( Option(a.value(0)).filterNot(_ == -1), Option(a.value(1)).filterNot(_ == -1), Option(a.value(2)).filterNot(_ == -1), Option(a.value(3)).filterNot(_ == -1) ) } } object DeltaDataSkippingType extends Enumeration { type DeltaDataSkippingType = Value // V1: code path in DataSkippingReader.scala, which needs StateReconstruction // noSkipping: no skipping and get all files from the Delta table // partitionFiltering: filtering and skipping based on partition columns // dataSkipping: filtering and skipping based on stats columns // limit: skipping based on limit clause in DataSkippingReader.scala // filteredLimit: skipping based on limit clause and partition columns in DataSkippingReader.scala val noSkippingV1, noSkippingV2, partitionFilteringOnlyV1, partitionFilteringOnlyV2, dataSkippingOnlyV1, dataSkippingOnlyV2, dataSkippingAndPartitionFilteringV1, dataSkippingAndPartitionFilteringV2, limit, filteredLimit = Value } /** * Used to hold details the files and stats for a scan where we have already * applied filters and a limit. */ case class DeltaScan( version: Long, files: Seq[AddFile], total: DataSize, partition: DataSize, scanned: DataSize)( // Moved to separate argument list, to not be part of case class equals check - // expressions can differ by exprId or ordering, but as long as same files are scanned, the // PreparedDeltaFileIndex and HadoopFsRelation should be considered equal for reuse purposes. val scannedSnapshot: Snapshot, val partitionFilters: ExpressionSet, val dataFilters: ExpressionSet, val partitionLikeDataFilters: ExpressionSet, // We can't use an ExpressionSet here because the rewritten filters aren't yet resolved when the // DeltaScan is created. Since this is for logging only, it's OK to store the non-canonicalized // expressions instead. val rewrittenPartitionLikeDataFilters: Set[Expression], val unusedFilters: ExpressionSet, val scanDurationMs: Long, val dataSkippingType: DeltaDataSkippingType) { assert(version == scannedSnapshot.version) /** * For unresolved expressions, converting the expression to SQL may throw an exception (if the * conversion to SQL requires the child types to be resolved). This method safely handles these * cases by returning a placeholder string for unresolved expressions. */ def safeExprToSQL(expr: Expression): String = { try { expr.sql } catch { case NonFatal(_) => s"UNRESOLVED_EXPRESSION_(${expr.getClass.getSimpleName})" } } lazy val rewrittenPartitionLikeFilterSQL = rewrittenPartitionLikeDataFilters.map(safeExprToSQL) lazy val filtersUsedForSkipping: ExpressionSet = partitionFilters ++ dataFilters lazy val allFilters: ExpressionSet = filtersUsedForSkipping ++ unusedFilters } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/stats/DeltaScanGenerator.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats import org.apache.spark.sql.delta.{Snapshot, SnapshotDescriptor} import org.apache.spark.sql.DataFrame import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} /** Trait representing a class that can generate [[DeltaScan]] given filters and a limit. */ trait DeltaScanGenerator { /** The snapshot that the scan is being generated on. */ val snapshotToScan: Snapshot /** * Returns a DataFrame for the given partition filters. The schema of returned DataFrame is nearly * the same as `AddFile`, except that the `stats` field is parsed to a struct from a json string. */ def filesWithStatsForScan(partitionFilters: Seq[Expression]): DataFrame /** Returns a [[DeltaScan]] based on the given filters. */ def filesForScan(filters: Seq[Expression], keepNumRecords: Boolean = false): DeltaScan /** Returns a [[DeltaScan]] based on the given partition filters and limits. */ def filesForScan(limit: Long, partitionFilters: Seq[Expression]): DeltaScan } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/stats/FileSizeHistogram.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats import java.util.Arrays import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.types.StructType /** * A Histogram class tracking the file counts and total bytes in different size ranges * @param sortedBinBoundaries - a sorted list of bin boundaries where each element represents the * start of the bin (included) and the next element represents the end * of the bin (excluded) * @param fileCounts - an array of Int representing total number of files in different bins * @param totalBytes - an array of Long representing total number of bytes in different bins */ case class FileSizeHistogram( sortedBinBoundaries: IndexedSeq[Long], fileCounts: Array[Long], totalBytes: Array[Long]) extends FileStatsHistogram { /** * Not intended to be used for [[Map]] structure keys. Implemented for the sole purpose of having * an equals method, which requires overriding hashCode as well, so an incomplete hash is okay. * We only require a == b implies a.hashCode == b.hashCode */ override def hashCode(): Int = Arrays.hashCode(totalBytes) override def equals(that: Any): Boolean = that match { case h: FileSizeHistogram => equalsHistogram(h) case _ => false } /** * Insert a given value into the appropriate histogram bin */ def insert(fileSize: Long): Unit = { val index = FileSizeHistogram.getBinIndex(fileSize, sortedBinBoundaries) if (index >= 0) { fileCounts(index) += 1 totalBytes(index) += fileSize } } /** * Remove a given value from the appropriate histogram bin * @param fileSize to remove */ def remove(fileSize: Long): Unit = { val index = FileSizeHistogram.getBinIndex(fileSize, sortedBinBoundaries) if (index >= 0) { fileCounts(index) -= 1 totalBytes(index) -= fileSize } } } private[delta] object FileSizeHistogram { /** * Returns the index of the bin to which given fileSize belongs OR -1 if given fileSize doesn't * belongs to any bin */ def getBinIndex(fileSize: Long, sortedBinBoundaries: IndexedSeq[Long]): Int = { FileStatsHistogram.getBinIndex(fileSize, sortedBinBoundaries) } def apply(sortedBinBoundaries: IndexedSeq[Long]): FileSizeHistogram = { new FileSizeHistogram( sortedBinBoundaries, Array.fill(sortedBinBoundaries.size)(0), Array.fill(sortedBinBoundaries.size)(0) ) } lazy val schema: StructType = ExpressionEncoder[FileSizeHistogram]().schema } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/stats/FileStatsHistogram.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats import java.util.Arrays import scala.collection.mutable.ArrayBuffer import com.fasterxml.jackson.databind.annotation.JsonDeserialize import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.util.GenericArrayData import org.apache.spark.sql.types.{ArrayType, DataType, LongType} import org.apache.spark.unsafe.Platform /** * Base trait for histogram implementations tracking file counts and total bytes across bins. */ trait FileStatsHistogram { @JsonDeserialize(contentAs = classOf[java.lang.Long]) def sortedBinBoundaries: IndexedSeq[Long] def fileCounts: Array[Long] def totalBytes: Array[Long] require(sortedBinBoundaries.nonEmpty) require(sortedBinBoundaries.head == 0, "The first bin should start from 0") require(sortedBinBoundaries.length == fileCounts.length, "number of binBoundaries should be same as size of fileCounts") require(sortedBinBoundaries.length == totalBytes.length, "number of binBoundaries should be same as size of totalBytes") /** * Helper method for subclass equals implementations. Subclasses should override both * equals and hashCode together. */ protected def equalsHistogram(that: FileStatsHistogram): Boolean = { sortedBinBoundaries == that.sortedBinBoundaries && java.util.Arrays.equals(fileCounts, that.fileCounts) && java.util.Arrays.equals(totalBytes, that.totalBytes) } } /** * Companion object with utility functions for histograms. */ object FileStatsHistogram { /** * Returns the index of the bin to which given value belongs OR -1 if value doesn't belong * to any bin */ def getBinIndex(value: Long, sortedBinBoundaries: IndexedSeq[Long]): Int = { import scala.collection.Searching._ val searchResult = sortedBinBoundaries.search(value) searchResult match { case Found(index) => index case InsertionPoint(insertionPoint) => // insertionPoint=0 means that fileSize is lesser than min bucket of histogram insertionPoint - 1 } } /** * Returns a compacted version of a histogram where empty bins are merged together. */ def compress[H <: FileStatsHistogram]( h: H, constructor: (IndexedSeq[Long], Array[Long], Array[Long]) => H): H = { val newSortedBinBoundaries = ArrayBuffer.empty[Long] val newFileCounts = ArrayBuffer.empty[Long] val newTotalBytes = ArrayBuffer.empty[Long] if (h.sortedBinBoundaries.nonEmpty) { newSortedBinBoundaries.append(h.sortedBinBoundaries(0)) newFileCounts.append(h.fileCounts(0)) newTotalBytes.append(h.totalBytes(0)) for (index <- 1 until h.sortedBinBoundaries.length) { if (h.fileCounts(index) != 0 || h.fileCounts(index - 1) != 0) { newSortedBinBoundaries.append(h.sortedBinBoundaries(index)) newFileCounts.append(h.fileCounts(index)) newTotalBytes.append(h.totalBytes(index)) } } } constructor(newSortedBinBoundaries.toIndexedSeq, newFileCounts.toArray, newTotalBytes.toArray) } /** * Base class for imperative aggregate implementations of file statistics histograms. * This provides common functionality for both FileSizeHistogram and FileAgeHistogram aggregates. * * The return type of this Imperative Aggregate is of ArrayType(LongType). The array * represents a flattened histogram with following structure: * * -------------------------------------------------------------------------------- * | PART-1: sortedBinBoundaries | PART-2: fileCounts | PART-3: totalBytes | * -------------------------------------------------------------------------------- * * This Aggregate returns the flattened histogram and not the histogram object due to * the limitation that Imperative aggregates can only return primitive/Array/Map types. * * The intermediate aggregation buffer consists of only PART-2 + PART-3 and doesn't contain * the sortedBinBoundaries. sortedBinBoundaries are added only at the end when the aggregate * is finalized. */ abstract class FileStatsHistogramAggBase extends org.apache.spark.sql.catalyst.expressions.aggregate .TypedImperativeAggregate[Array[Long]] with org.apache.spark.sql.catalyst.trees.UnaryLike[ org.apache.spark.sql.catalyst.expressions.Expression] { import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.util.GenericArrayData import org.apache.spark.sql.types.{ArrayType, DataType, LongType} import org.apache.spark.unsafe.Platform def sortedBinBoundaries: IndexedSeq[Long] // Size of underlying buffer for the aggregate. We make buffer of 2 * totalBins to store // fileCount as well as totalBytes. lazy val underlyingBufferSize: Int = sortedBinBoundaries.size * 2 lazy val secondHalfStartIndex: Int = sortedBinBoundaries.size /** * The aggregation buffer of this aggregate is an Array of Longs of size - 2 * NumBins * 1. First half of the array represents fileCounts * 2. Second half of the array represents totalBytes * * -------------------------------------------------------------------------------- * | fileCounts related indices | totalBytes related indices | * -------------------------------------------------------------------------------- */ override def createAggregationBuffer(): Array[Long] = Array.fill(underlyingBufferSize)(0) override def dataType: DataType = ArrayType(LongType) override def nullable: Boolean = { // This Aggregate doesn't return null false } override def update(aggBuffer: Array[Long], input: InternalRow): Array[Long] = { val value = child.eval(input) if (value != null) { val metricValue = value.asInstanceOf[Long] val index = FileStatsHistogram.getBinIndex(metricValue, sortedBinBoundaries) if (index >= 0) { aggBuffer(index) += 1 aggBuffer(secondHalfStartIndex + index) += metricValue } } aggBuffer } override def merge(buffer: Array[Long], input: Array[Long]): Array[Long] = { buffer.indices.foreach { index => buffer(index) += input(index) } buffer } override def eval(buffer: Array[Long]): Any = { new GenericArrayData(sortedBinBoundaries ++ buffer) } /** Serializes the aggregation buffer to Array[Byte] */ override def serialize(buffer: Array[Long]): Array[Byte] = { val byteArray = new Array[Byte](buffer.length * 8) buffer.indices.foreach { index => Platform.putLong(byteArray, Platform.BYTE_ARRAY_OFFSET + index * 8, buffer(index)) } byteArray } /** De-serializes the serialized format Array[Byte], and produces aggregation buffer */ override def deserialize(bytes: Array[Byte]): Array[Long] = { val aggBuffer = new Array[Long](bytes.length / 8) aggBuffer.indices.foreach { index => aggBuffer(index) = Platform.getLong(bytes, Platform.BYTE_ARRAY_OFFSET + index * 8) } aggBuffer } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/stats/PrepareDeltaScan.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats import java.util.Objects import scala.collection.mutable import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.{AddFile, Protocol} import org.apache.spark.sql.delta.files.{TahoeFileIndex, TahoeFileIndexWithSnapshotDescriptor, TahoeLogFileIndex} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.perf.OptimizeMetadataOnlyDeltaQuery import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.hadoop.fs.Path import org.apache.spark.internal.MDC import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.planning.PhysicalOperation import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.trees.TreePattern.PROJECT import org.apache.spark.sql.execution.datasources.{FileIndex, LogicalRelation} import org.apache.spark.sql.types.StructType /** * Before query planning, we prepare any scans over delta tables by pushing * any projections or filters in allowing us to gather more accurate statistics * for CBO and metering. * * Note the following * - This rule also ensures that all reads from the same delta log use the same snapshot of log * thus providing snapshot isolation. * - If this rule is invoked within an active [[OptimisticTransaction]], then the scans are * generated using the transaction. */ trait PrepareDeltaScanBase extends Rule[LogicalPlan] with PredicateHelper with DeltaLogging with OptimizeMetadataOnlyDeltaQuery with SubqueryTransformerHelper { self: PrepareDeltaScan => /** * Tracks the first-access snapshots of other logs planned by this rule. The snapshots are * the keyed by the log's unique id. Note that the lifetime of this rule is a single * query, therefore, the map tracks the snapshots only within a query. */ private val scannedSnapshots = new java.util.concurrent.ConcurrentHashMap[(String, Path), Snapshot] /** * Gets the [[DeltaScanGenerator]] for the given log, which will be used to generate * [[DeltaScan]]s. Every time this method is called on a log within the lifetime of this * rule (i.e., the lifetime of the query for which this rule was instantiated), the returned * generator will read a snapshot that is pinned on the first access for that log. * * Internally, it will use the snapshot of the file index, the snapshot of the active transaction * (if any), or the latest snapshot of the given log. */ protected def getDeltaScanGenerator(index: TahoeLogFileIndex): DeltaScanGenerator = { // The first case means that we've fixed the table snapshot for time travel if (index.isTimeTravelQuery) return index.getSnapshot val scanGenerator = OptimisticTransaction.getActive() .map(_.getDeltaScanGenerator(index)) .getOrElse { // Will be called only when the log is accessed the first time scannedSnapshots.computeIfAbsent(index.deltaLog.compositeId, _ => index.getSnapshot) } import PrepareDeltaScanBase._ if (onGetDeltaScanGeneratorCallback != null) onGetDeltaScanGeneratorCallback(scanGenerator) scanGenerator } /** * Helper method to generate a [[PreparedDeltaFileIndex]] */ protected def getPreparedIndex( preparedScan: DeltaScan, fileIndex: TahoeLogFileIndex): PreparedDeltaFileIndex = { assert(fileIndex.partitionFilters.isEmpty, "Partition filters should have been extracted by DeltaAnalysis.") PreparedDeltaFileIndex( spark, fileIndex.deltaLog, fileIndex.path, fileIndex.catalogTableOpt, preparedScan, fileIndex.versionToUse) } /** * Scan files using the given `filters` and return `DeltaScan`. * * Note: when `limitOpt` is non empty, `filters` must contain only partition filters. Otherwise, * it can contain arbitrary filters. See `DeltaTableScan` for more details. */ protected def filesForScan( scanGenerator: DeltaScanGenerator, limitOpt: Option[Int], filters: Seq[Expression], delta: LogicalRelation): DeltaScan = { withStatusCode("DELTA", "Filtering files for query") { if (limitOpt.nonEmpty) { // If we trigger limit push down, the filters must be partition filters. Since // there are no data filters, we don't need to apply Generated Columns // optimization. See `DeltaTableScan` for more details. return scanGenerator.filesForScan(limitOpt.get, filters) } val filtersForScan = if (!GeneratedColumn.partitionFilterOptimizationEnabled(spark)) { filters } else { val generatedPartitionFilters = GeneratedColumn.generatePartitionFilters( spark, scanGenerator.snapshotToScan, filters, delta) filters ++ generatedPartitionFilters } scanGenerator.filesForScan(filtersForScan) } } /** * Prepares delta scans sequentially. */ protected def prepareDeltaScan(plan: LogicalPlan): LogicalPlan = { // A map from the canonicalized form of a DeltaTableScan operator to its corresponding delta // scan. This map is used to avoid fetching duplicate delta indexes for structurally-equal // delta scans. val deltaScans = new mutable.HashMap[LogicalPlan, DeltaScan]() transformWithSubqueries(plan) { case scan @ DeltaTableScan(planWithRemovedProjections, filters, fileIndex, limit, delta) => val scanGenerator = getDeltaScanGenerator(fileIndex) val preparedScan = deltaScans.getOrElseUpdate(planWithRemovedProjections.canonicalized, filesForScan(scanGenerator, limit, filters, delta)) val preparedIndex = getPreparedIndex(preparedScan, fileIndex) optimizeGeneratedColumns(scan, preparedIndex, filters, limit, delta) } } protected def optimizeGeneratedColumns( scan: LogicalPlan, preparedIndex: PreparedDeltaFileIndex, filters: Seq[Expression], limit: Option[Int], delta: LogicalRelation): LogicalPlan = { if (limit.nonEmpty) { // If we trigger limit push down, the filters must be partition filters. Since // there are no data filters, we don't need to apply Generated Columns // optimization. See `DeltaTableScan` for more details. return DeltaTableUtils.replaceFileIndex(scan, preparedIndex) } if (!GeneratedColumn.partitionFilterOptimizationEnabled(spark)) { DeltaTableUtils.replaceFileIndex(scan, preparedIndex) } else { val generatedPartitionFilters = GeneratedColumn.generatePartitionFilters(spark, preparedIndex, filters, delta) val scanWithFilters = if (generatedPartitionFilters.nonEmpty) { scan transformUp { case delta @ DeltaTable(_: TahoeLogFileIndex) => Filter(generatedPartitionFilters.reduceLeft(And), delta) } } else { scan } DeltaTableUtils.replaceFileIndex(scanWithFilters, preparedIndex) } } override def apply(_plan: LogicalPlan): LogicalPlan = { var plan = _plan val shouldPrepareDeltaScan = ( spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_STATS_SKIPPING) ) val updatedPlan = if (shouldPrepareDeltaScan) { // Should not be applied to subqueries to avoid duplicate delta jobs. val isSubquery = isSubqueryRoot(plan) // Should not be applied to DataSourceV2 write plans, because they'll be planned later // through a V1 fallback and only that later planning takes place within the transaction. val isDataSourceV2 = plan.isInstanceOf[V2WriteCommand] if (isSubquery || isDataSourceV2) { return plan } if (spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_METADATA_QUERY_ENABLED)) { plan = optimizeQueryWithMetadata(plan) } prepareDeltaScan(plan) } else { prepareDeltaScanWithoutFileSkipping(plan) } updatedPlan } protected def prepareDeltaScanWithoutFileSkipping(plan: LogicalPlan): LogicalPlan = { // If this query is running inside an active transaction and is touching the same table // as the transaction, then mark that the entire table as tainted to be safe. OptimisticTransaction.getActive().foreach { txn => val logsInPlan = plan.collect { case DeltaTable(fileIndex: TahoeFileIndex) => fileIndex.deltaLog } if (logsInPlan.exists(_.isSameLogAs(txn.deltaLog))) { txn.readWholeTable() } } // Just return the plan if statistics based skipping is off. // It will fall back to just partition pruning at planning time. plan } /** * This is an extractor object. See https://docs.scala-lang.org/tour/extractor-objects.html. */ object DeltaTableScan extends DeltaTableScan[TahoeLogFileIndex] { override def limitPushdownEnabled(plan: LogicalPlan): Boolean = spark.conf.get(DeltaSQLConf.DELTA_LIMIT_PUSHDOWN_ENABLED) override def getPartitionColumns(fileIndex: TahoeLogFileIndex): Seq[String] = fileIndex.snapshotAtAnalysis.metadata.partitionColumns override def getPartitionFilters(fileIndex: TahoeLogFileIndex): Seq[Expression] = fileIndex.partitionFilters } abstract class DeltaTableScan[FileIndexType <: FileIndex : scala.reflect.ClassTag] { /** * The components of DeltaTableScanType are: * - the plan with removed projections. We remove projections as a plan differentiator * because it does not affect file listing results. * - filter expressions collected by `PhysicalOperation` * - the `FileIndexType` of the matched DeltaTable` * - integer value of limit expression, if any * - matched `DeltaTable` */ protected type DeltaTableScanType = (LogicalPlan, Seq[Expression], FileIndexType, Option[Int], LogicalRelation) /** * This is an extractor method (basically, the opposite of a constructor) which takes in an * object `plan` and tries to give back the arguments as a [[DeltaTableScanType]]. */ def unapply(plan: LogicalPlan): Option[DeltaTableScanType] = { // Remove projections as a plan differentiator because it does not affect file listing // results. Plans with the same filters but different projections therefore will not have // duplicate delta indexes. def canonicalizePlanForDeltaFileListing(plan: LogicalPlan): LogicalPlan = { val planWithRemovedProjections = plan.transformWithPruning(_.containsPattern(PROJECT)) { case p: Project if p.projectList.forall(_.isInstanceOf[AttributeReference]) => p.child } planWithRemovedProjections } plan match { case LocalLimit(IntegerLiteral(limit), PhysicalOperation(_, filters, delta @ RelationFileIndex(fileIndex: FileIndexType))) if limitPushdownEnabled(plan) && containsPartitionFiltersOnly(filters, fileIndex) => Some((canonicalizePlanForDeltaFileListing(plan), filters, fileIndex, Some(limit), delta)) case PhysicalOperation( _, filters, delta @ RelationFileIndex(fileIndex: FileIndexType)) => val allFilters = getPartitionFilters(fileIndex) ++ filters Some((canonicalizePlanForDeltaFileListing(plan), allFilters, fileIndex, None, delta)) case _ => None } } protected def containsPartitionFiltersOnly( filters: Seq[Expression], fileIndex: FileIndexType): Boolean = { val partitionColumns = getPartitionColumns(fileIndex) import DeltaTableUtils._ filters.forall(expr => !containsSubquery(expr) && isPredicatePartitionColumnsOnly(expr, partitionColumns, spark)) } protected def limitPushdownEnabled(plan: LogicalPlan): Boolean protected def getPartitionColumns(fileIndex: FileIndexType): Seq[String] protected def getPartitionFilters(fileIndex: FileIndexType): Seq[Expression] } } class PrepareDeltaScan(protected val spark: SparkSession) extends PrepareDeltaScanBase object PrepareDeltaScanBase { /** * Optional callback function that is called after `getDeltaScanGenerator` is called * by the PrepareDeltaScan rule. This is primarily used for testing purposes. */ @volatile private var onGetDeltaScanGeneratorCallback: DeltaScanGenerator => Unit = _ /** * Run a thunk of code with the given callback function injected into the PrepareDeltaScan rule. * The callback function is called after `getDeltaScanGenerator` is called * by the PrepareDeltaScan rule. This is primarily used for testing purposes. */ private[delta] def withCallbackOnGetDeltaScanGenerator[T]( callback: DeltaScanGenerator => Unit)(thunk: => T): T = { try { onGetDeltaScanGeneratorCallback = callback thunk } finally { onGetDeltaScanGeneratorCallback = null } } } /** * A [[TahoeFileIndex]] that uses a prepared scan to return the list of relevant files. * This is injected into a query right before query planning by [[PrepareDeltaScan]] so that * CBO and metering can accurately understand how much data will be read. * * @param versionScanned The version of the table that is being scanned, if a specific version * has specifically been requested, e.g. by time travel. */ case class PreparedDeltaFileIndex( override val spark: SparkSession, override val deltaLog: DeltaLog, override val path: Path, catalogTableOpt: Option[CatalogTable], preparedScan: DeltaScan, versionScanned: Option[Long]) extends TahoeFileIndexWithSnapshotDescriptor(spark, deltaLog, path, preparedScan.scannedSnapshot) with DeltaLogging { /** * Returns all matching/valid files by the given `partitionFilters` and `dataFilters` */ override def matchingFiles( partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): Seq[AddFile] = { val currentFilters = ExpressionSet(partitionFilters ++ dataFilters) val (addFiles, eventData) = if (currentFilters == preparedScan.allFilters || currentFilters == preparedScan.filtersUsedForSkipping) { // [[DeltaScan]] was created using `allFilters` out of which only `filtersUsedForSkipping` // filters were used for skipping while creating the DeltaScan. // If currentFilters is same as allFilters, then no need to recalculate files and we can use // previous results. // If currentFilters is same as filtersUsedForSkipping, then also we don't need to recalculate // files as [[DeltaScan.files]] were calculates using filtersUsedForSkipping only. So if we // recalculate, we will get same result. So we should use previous result in this case also. val eventData = Map( "reused" -> true, "currentFiltersSameAsPreparedAllFilters" -> (currentFilters == preparedScan.allFilters), "currentFiltersSameAsPreparedFiltersUsedForSkipping" -> (currentFilters == preparedScan.filtersUsedForSkipping) ) (preparedScan.files.distinct, eventData) } else { logInfo( log""" |Prepared scan does not match actual filters. Reselecting files to query. |Prepared: ${MDC(DeltaLogKeys.FILTER, preparedScan.allFilters)} |Actual: ${MDC(DeltaLogKeys.FILTER2, currentFilters)} """.stripMargin) val eventData = Map( "reused" -> false, "preparedAllFilters" -> preparedScan.allFilters.mkString(","), "preparedFiltersUsedForSkipping" -> preparedScan.filtersUsedForSkipping.mkString(","), "currentFilters" -> currentFilters.mkString(",") ) val files = preparedScan.scannedSnapshot.filesForScan(partitionFilters ++ dataFilters).files (files, eventData) } recordDeltaEvent(deltaLog, opType = "delta.preparedDeltaFileIndex.reuseSkippingResult", data = eventData) addFiles } /** * Returns the list of files that will be read when scanning this relation. This call may be * very expensive for large tables. */ override def inputFiles: Array[String] = preparedScan.files.map(f => absolutePath(f.path).toString).toArray /** Refresh any cached file listings */ override def refresh(): Unit = { } /** Sum of table file sizes, in bytes */ override def sizeInBytes: Long = preparedScan.scanned.bytesCompressed .getOrElse(spark.sessionState.conf.defaultSizeInBytes) override def equals(other: Any): Boolean = other match { case p: PreparedDeltaFileIndex => p.deltaLog == deltaLog && p.path == path && p.preparedScan == preparedScan && p.partitionSchema == partitionSchema && p.versionScanned == versionScanned case _ => false } override def hashCode(): Int = { Objects.hash(deltaLog, path, preparedScan, partitionSchema, versionScanned) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/stats/ReadsMetadataFields.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats import org.apache.spark.sql.Column import org.apache.spark.sql.functions.col /** * A mixin trait that provides access to the stats fields in the transaction log. */ trait ReadsMetadataFields { /** Returns a Column that references the stats field data skipping should use */ def getBaseStatsColumn: Column = col(getBaseStatsColumnName) def getBaseStatsColumnName: String = "stats" } /** * A singleton of the Delta statistics field names. */ object DeltaStatistics { /* The total number of records in the file. */ val NUM_RECORDS = "numRecords" /* The smallest (possibly truncated) value for a column. */ val MIN = "minValues" /* The largest (possibly truncated) value for a column. */ val MAX = "maxValues" /* The number of null values present for a column. */ val NULL_COUNT = "nullCount" /* * Whether the column has tight or wide bounds. * This should only be present in tables with Deletion Vectors enabled. */ val TIGHT_BOUNDS = "tightBounds" val ALL_STAT_FIELDS = Seq(NUM_RECORDS, MIN, MAX, NULL_COUNT, TIGHT_BOUNDS) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/stats/StatisticsCollection.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats import java.util.Locale // scalastyle:off import.ordering.noEmptyLine import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer import scala.language.existentials import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.{Checkpoints, DeletionVectorsTableFeature, DeltaColumnMapping, DeltaColumnMappingMode, DeltaConfigs, DeltaErrors, DeltaIllegalArgumentException, DeltaLog, DeltaUDF, NoMapping} import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.DeltaColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY import org.apache.spark.sql.delta.DeltaOperations.ComputeStats import org.apache.spark.sql.delta.OptimisticTransaction import org.apache.spark.sql.delta.actions.{AddFile, Metadata, Protocol} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.commands.DeltaCommand import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.{SchemaMergingUtils, SchemaUtils} import org.apache.spark.sql.delta.schema.SchemaUtils.transformSchema import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.DeltaStatistics._ import org.apache.spark.sql.delta.stats.StatisticsCollection.getIndexedColumns import org.apache.spark.sql.delta.util.DeltaSqlParserUtils import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.parser.{AbstractSqlParser, AstBuilder, ParseException, ParserUtils} import org.apache.spark.sql.catalyst.parser.SqlBaseParser.MultipartIdentifierListContext import org.apache.spark.sql.catalyst.util.quoteIfNeeded import org.apache.spark.sql.functions._ import org.apache.spark.sql.functions.lit import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ /** * Used to report metrics on how predicates are used to prune the set of * files that are read by a query. * * @param predicate A user readable version of the predicate. * @param pruningType One of {partition, dataStats, none}. * @param filesMissingStats The number of files that were included due to missing statistics. * @param filesDropped The number of files that were dropped by this predicate. */ case class QueryPredicateReport( predicate: String, pruningType: String, filesMissingStats: Long, filesDropped: Long) /** Used to report details about prequery filtering of what data is scanned. */ case class FilterMetric(numFiles: Long, predicates: Seq[QueryPredicateReport]) /** * A helper trait that constructs expressions that can be used to collect global * and column level statistics for a collection of data, given its schema. * * Global statistics (such as the number of records) are stored as top level columns. * Per-column statistics (such as min/max) are stored in a struct that mirrors the * schema of the data. * * To illustrate, here is an example of a data schema along with the schema of the statistics * that would be collected. * * Data Schema: * {{{ * |-- a: struct (nullable = true) * | |-- b: struct (nullable = true) * | | |-- c: long (nullable = true) * }}} * * Collected Statistics: * {{{ * |-- stats: struct (nullable = true) * | |-- numRecords: long (nullable = false) * | |-- minValues: struct (nullable = false) * | | |-- a: struct (nullable = false) * | | | |-- b: struct (nullable = false) * | | | | |-- c: long (nullable = true) * | |-- maxValues: struct (nullable = false) * | | |-- a: struct (nullable = false) * | | | |-- b: struct (nullable = false) * | | | | |-- c: long (nullable = true) * | |-- nullCount: struct (nullable = false) * | | |-- a: struct (nullable = false) * | | | |-- b: struct (nullable = false) * | | | | |-- c: long (nullable = true) * }}} */ trait StatisticsCollection extends DeltaLogging { protected def spark: SparkSession /** The schema of the target table of this statistics collection. */ def tableSchema: StructType /** * The output attributes (`outputAttributeSchema`) that are replaced with table schema with * the physical mapping information. * NOTE: The partition columns' definitions are not included in this schema. */ def outputTableStatsSchema: StructType /** * The schema of the output attributes of the write queries that needs to collect statistics. * The partition columns' definitions are not included in this schema. */ def outputAttributeSchema: StructType /** The statistic indexed column specification of the target delta table. */ val statsColumnSpec: DeltaStatsColumnSpec /** The column mapping mode of the target delta table. */ def columnMappingMode: DeltaColumnMappingMode protected def protocol: Protocol lazy val deletionVectorsSupported = protocol.isFeatureSupported(DeletionVectorsTableFeature) private def effectiveSchema: StructType = if (statsColumnSpec.numIndexedColsOpt.isDefined) { outputTableStatsSchema } else { tableSchema } private lazy val explodedDataSchemaNames: Seq[String] = SchemaMergingUtils.explodeNestedFieldNames(outputAttributeSchema) /** * statCollectionPhysicalSchema is the schema that is composed of all the columns that have the * stats collected with our current table configuration. */ lazy val statCollectionPhysicalSchema: StructType = getIndexedColumns(explodedDataSchemaNames, statsColumnSpec, effectiveSchema, columnMappingMode) /** * statCollectionLogicalSchema is the logical schema that is composed of all the columns that have * the stats collected with our current table configuration. */ lazy val statCollectionLogicalSchema: StructType = getIndexedColumns(explodedDataSchemaNames, statsColumnSpec, effectiveSchema, NoMapping) /** * Traverses the [[statisticsSchema]] for the provided [[statisticsColumn]] * and applies [[function]] to leaves. * * Note, for values that are outside the domain of the partial function we keep the original * column. If the caller wants to drop the column needs to explicitly return None. */ def applyFuncToStatisticsColumn( statisticsSchema: StructType, statisticsColumn: Column)( function: PartialFunction[(Column, StructField), Option[Column]]): Seq[Column] = { statisticsSchema.flatMap { case StructField(name, s: StructType, _, _) => val column = statisticsColumn.getItem(name) applyFuncToStatisticsColumn(s, column)(function) match { case colSeq if colSeq.nonEmpty => Some(struct(colSeq: _*) as name) case _ => None } case structField@StructField(name, _, _, _) => val column = statisticsColumn.getItem(name) function.lift(column, structField).getOrElse(Some(column)).map(_.as(name)) } } /** * Sets the TIGHT_BOUNDS column to false and converts the logical nullCount * to a tri-state nullCount. The nullCount states are the following: * 1) For "all-nulls" columns we set the physical nullCount which is equal to the * physical numRecords. * 2) "no-nulls" columns remain unchanged, i.e. zero nullCount is the same for both * physical and logical representations. * 3) For "some-nulls" columns, we leave the existing value. In files with wide bounds, * the nullCount in SOME_NULLs columns is considered unknown. * * The file's state can transition back to tight when statistics are recomputed. In that case, * TIGHT_BOUNDS is set back to true and nullCount back to the logical value. * * Note, this function gets as input parsed statistics and returns a json document * similarly to allFiles. To further match the behavior of allFiles we always return * a column named `stats` instead of statsColName. * * @param withStats A dataFrame of actions with parsed statistics. * @param statsColName The name of the parsed statistics column. */ def updateStatsToWideBounds(withStats: DataFrame, statsColName: String): DataFrame = { val dvCardinalityCol = coalesce(col("deletionVector.cardinality"), lit(0)) val physicalNumRecordsCol = col(s"$statsColName.$NUM_RECORDS") val logicalNumRecordsCol = physicalNumRecordsCol - dvCardinalityCol val nullCountCol = col(s"$statsColName.$NULL_COUNT") val tightBoundsCol = col(s"$statsColName.$TIGHT_BOUNDS") val statsSchema = withStats.schema.apply(statsColName).dataType.asInstanceOf[StructType] val allStatCols = ALL_STAT_FIELDS.flatMap { case TIGHT_BOUNDS => Some(lit(false).as(TIGHT_BOUNDS)) case NULL_COUNT if statsSchema.names.contains(NULL_COUNT) => // Use the schema of the existing stats column. We only want to modify the existing // nullCount stats. Note, when the column mapping mode is enabled, the schema uses // the physical column names, not the logical names. val nullCountSchema = statsSchema .apply(NULL_COUNT).dataType.asInstanceOf[StructType] // When bounds are tight and we are about to transition to wide, store the physical null // count for ALL_NULLs columns. val nullCountColSeq = applyFuncToStatisticsColumn(nullCountSchema, nullCountCol) { case (c, _) => val allNullTightBounds = tightBoundsCol && (c === logicalNumRecordsCol) Some(when(allNullTightBounds, physicalNumRecordsCol).otherwise(c)) } Some(struct(nullCountColSeq: _*).as(NULL_COUNT)) case f if statsSchema.names.contains(f) => Some(col(s"${statsColName}.${f}")) case _ => // This stat is not present in the original stats schema, so we should not include it. None } // This may be very expensive because it is rewriting JSON. withStats .withColumn("stats", when(col(statsColName).isNotNull, to_json(struct(allStatCols: _*)))) .drop(col(Checkpoints.STRUCT_STATS_COL_NAME)) // Note: does not always exist. } /** * Returns the prefix length of strings that should be used for data skipping. * Intentionally left abstract to let implementation decide whether table property overrides * need to be included. */ protected def getDataSkippingStringPrefixLength: Int /** * Returns a struct column that can be used to collect statistics for the current * schema of the table. * The types we keep stats on must be consistent with DataSkippingReader.SkippingEligibleLiteral. * If a column is missing from dataSchema (which will be filled with nulls), we will only * collect the NULL_COUNT stats for it as the number of rows. */ lazy val statsCollector: Column = { val stringPrefix = getDataSkippingStringPrefixLength // On file initialization/stat recomputation TIGHT_BOUNDS is always set to true val tightBoundsColOpt = Option.when(deletionVectorsSupported && !spark.sessionState.conf.getConf(DeltaSQLConf.TIGHT_BOUND_COLUMN_ON_FILE_INIT_DISABLED)) { lit(true).as(TIGHT_BOUNDS) } val statCols = Seq( count(new Column("*")) as NUM_RECORDS, collectStats(MIN, statCollectionPhysicalSchema) { // Truncate string min values as necessary case (c, SkippingEligibleDataType(StringType), true) => substring(min(c), 0, stringPrefix) // Write null for min/max Variant stats because collecting variant stats is not supported // yet. case (c, SkippingEligibleDataType(_: VariantType), true) => lit(null).cast(VariantType) // Collect all numeric min values case (c, SkippingEligibleDataType(_), true) => min(c) }, collectStats(MAX, statCollectionPhysicalSchema) { // Truncate and pad string max values as necessary case (c, SkippingEligibleDataType(StringType), true) => val udfTruncateMax = DeltaUDF.stringFromString(StatisticsCollection.truncateMaxStringAgg(stringPrefix)_) udfTruncateMax(max(c)) // Write null for min/max Variant stats because collecting variant stats is not supported // yet. case (c, SkippingEligibleDataType(_: VariantType), true) => lit(null).cast(VariantType) // Collect all numeric max values case (c, SkippingEligibleDataType(_), true) => max(c) }, collectStats(NULL_COUNT, statCollectionPhysicalSchema) { case (c, _, true) => sum(when(c.isNull, 1).otherwise(0)) case (_, _, false) => count(new Column("*")) }) ++ tightBoundsColOpt struct(statCols: _*).as("stats") } /** Returns schema of the statistics collected. */ lazy val statsSchema: StructType = { // In order to get the Delta min/max stats schema from table schema, we do 1) replace field // name with physical name 2) set nullable to true 3) only keep stats eligible fields // 4) omits metadata in table schema as Delta stats schema does not need the metadata def getMinMaxStatsSchema(schema: StructType): Option[StructType] = { val fields = schema.fields.flatMap { case f@StructField(_, dataType: StructType, _, _) => getMinMaxStatsSchema(dataType).map { newDataType => StructField(DeltaColumnMapping.getPhysicalName(f), newDataType) } case f@StructField(_, SkippingEligibleDataType(dataType), _, _) => Some(StructField(DeltaColumnMapping.getPhysicalName(f), dataType)) case _ => None } if (fields.nonEmpty) Some(StructType(fields)) else None } // In order to get the Delta null count schema from table schema, we do 1) replace field name // with physical name 2) set nullable to true 3) use LongType for all fields // 4) omits metadata in table schema as Delta stats schema does not need the metadata def getNullCountSchema(schema: StructType): Option[StructType] = { val fields = schema.fields.flatMap { case f@StructField(_, dataType: StructType, _, _) => getNullCountSchema(dataType).map { newDataType => StructField(DeltaColumnMapping.getPhysicalName(f), newDataType) } case f: StructField => Some(StructField(DeltaColumnMapping.getPhysicalName(f), LongType)) } if (fields.nonEmpty) Some(StructType(fields)) else None } val minMaxStatsSchemaOpt = getMinMaxStatsSchema(statCollectionPhysicalSchema) val nullCountSchemaOpt = getNullCountSchema(statCollectionPhysicalSchema) val tightBoundsFieldOpt = Option.when(deletionVectorsSupported)(TIGHT_BOUNDS -> BooleanType) val fields = Array(NUM_RECORDS -> LongType) ++ minMaxStatsSchemaOpt.map(MIN -> _) ++ minMaxStatsSchemaOpt.map(MAX -> _) ++ nullCountSchemaOpt.map(NULL_COUNT -> _) ++ tightBoundsFieldOpt StructType(fields.map { case (name, dataType) => StructField(name, dataType) }) } /** * Recursively walks the given schema, constructing an expression to calculate * multiple statistics that mirrors structure of the data. When `function` is * defined for a given column, it return value is added to statistics structure. * When `function` is not defined, that column is skipped. * * @param name The name of the top level column for this statistic (i.e. minValues). * @param schema The schema of the data to collect statistics from. * @param function A partial function that is passed a tuple of (column, metadata about that * column, a flag that indicates whether the column is in the data schema). Based * on the metadata and flag, the function can decide if the given statistic should * be collected on the column by returning the correct aggregate expression. * @param includeAllColumns should statistics all the columns be included? */ private def collectStats( name: String, schema: StructType, includeAllColumns: Boolean = false)( function: PartialFunction[(Column, StructField, Boolean), Column]): Column = { def collectStats( schema: StructType, parent: Option[Column], parentFields: Seq[String], function: PartialFunction[(Column, StructField, Boolean), Column]): Seq[Column] = { schema.flatMap { case f @ StructField(name, s: StructType, _, _) => val column = parent.map(_.getItem(name)) .getOrElse(Column(UnresolvedAttribute.quoted(name))) val stats = collectStats(s, Some(column), parentFields :+ name, function) if (stats.nonEmpty) { Some(struct(stats: _*) as DeltaColumnMapping.getPhysicalName(f)) } else { None } case f @ StructField(name, _, _, _) => val fieldPath = UnresolvedAttribute(parentFields :+ name).name val column = parent.map(_.getItem(name)) .getOrElse(Column(UnresolvedAttribute.quoted(name))) // alias the column with its physical name // Note: explodedDataSchemaNames comes from dataSchema. In the read path, dataSchema comes // from the table's metadata.dataSchema, which is the same as tableSchema. In the // write path, dataSchema comes from the DataFrame schema. We then assume // TransactionWrite.writeFiles has normalized dataSchema, and // TransactionWrite.getStatsSchema has done the column mapping for tableSchema and // dropped the partition columns for both dataSchema and tableSchema. function.lift((column, f, explodedDataSchemaNames.contains(fieldPath))). map(_.as(DeltaColumnMapping.getPhysicalName(f))) } } val stats = collectStats(schema, None, Nil, function) if (stats.nonEmpty) { struct(stats: _*).as(name) } else { lit(null).as(name) } } } /** * Specifies the set of columns to be used for stats collection on a table. * The `deltaStatsColumnNamesOpt` has higher priority than `numIndexedColsOpt`. Thus, if * `deltaStatsColumnNamesOpt` is not None, StatisticsCollection would only collects file statistics * for all columns inside it. Otherwise, `numIndexedColsOpt` is used. */ case class DeltaStatsColumnSpec( deltaStatsColumnNamesOpt: Option[Seq[UnresolvedAttribute]], numIndexedColsOpt: Option[Int]) { require(deltaStatsColumnNamesOpt.isEmpty || numIndexedColsOpt.isEmpty) } object StatisticsCollection extends DeltaCommand { val ASCII_MAX_CHARACTER = '\u007F' val UTF8_MAX_CHARACTER = new String(Character.toChars(Character.MAX_CODE_POINT)) /** * This method is the wrapper method to validates the DATA_SKIPPING_STATS_COLUMNS value of * metadata. */ def validateDeltaStatsColumns(metadata: Metadata): Unit = { DeltaConfigs.DATA_SKIPPING_STATS_COLUMNS.fromMetaData(metadata).foreach { statsColumns => StatisticsCollection.validateDeltaStatsColumns( metadata.dataSchema, metadata.partitionColumns, statsColumns ) } } /** * This method validates that the data type of a data skipping column provided in * [[DeltaConfigs.DATA_SKIPPING_STATS_COLUMNS]] supports data skipping based on file statistics. * If a struct column is specified, all its children are considered valid. This helps users * who have complex nested types and wish to collect stats on all supported nested columns * without specifying each field individually. At stats collection time, unsupported types will * simply be skipped, so it is safe to allow those through. * @param name The name of the data skipping column for validating data type. * @param dataType The data type of the data skipping column. * @param columnPaths The column paths of all valid fields. * @param insideStruct Tracks if the field is inside a user-specified struct. Don't throw an * error on ineligible skipping types inside structs as the user didn't * specify them directly. Simply log a warning to let the user know * statistics won't be collected on that nested field. */ private def validateDataSkippingType( name: String, dataType: DataType, columnPaths: ArrayBuffer[String], insideStruct: Boolean = false): Unit = dataType match { case s: StructType => s.foreach { field => // we need to make sure we quote the field if needed otherwise we will not handle // column names with special characters correctly. validateDataSkippingType(name + "." + quoteIfNeeded(field.name), field.dataType, columnPaths, insideStruct = true) } case SkippingEligibleDataType(_) => if (insideStruct) { // If this is inside the struct we are already quoting the nested field name. columnPaths.append(name) } else { columnPaths.append(quoteIfNeeded(name)) } case _ if insideStruct => logWarning(s"Data skipping is not supported for column $name of type $dataType") columnPaths.append(name) case _ => throw new DeltaIllegalArgumentException( errorClass = "DELTA_COLUMN_DATA_SKIPPING_NOT_SUPPORTED_TYPE", messageParameters = Array(name, dataType.toString)) } /** * This method validates whether the DATA_SKIPPING_STATS_COLUMNS value satisfies following * conditions: * 1. Delta statistics columns must not be partitioned column. * 2. Delta statistics column must exist in delta table's schema. * 3. Delta statistics columns must be data skipping type. */ def validateDeltaStatsColumns( schema: StructType, partitionColumns: Seq[String], deltaStatsColumnsConfigs: String): Unit = { val partitionColumnSet = partitionColumns.map(_.toLowerCase(Locale.ROOT)).toSet val visitedColumns = ArrayBuffer.empty[String] DeltaSqlParserUtils.parseMultipartColumnList(deltaStatsColumnsConfigs).foreach { columns => columns.foreach { columnAttribute => val columnFullPath = columnAttribute.nameParts // Delta statistics columns must not be partitioned column. if (partitionColumnSet.contains(columnAttribute.name.toLowerCase(Locale.ROOT))) { throw new DeltaIllegalArgumentException( errorClass = "DELTA_COLUMN_DATA_SKIPPING_NOT_SUPPORTED_PARTITIONED_COLUMN", messageParameters = Array(columnAttribute.name)) } // Delta statistics column must exist in delta table's schema. SchemaUtils.findColumnPosition(columnFullPath, schema) // Delta statistics columns must be data skipping type. val (prefixPath, columnName) = columnFullPath.splitAt(columnFullPath.size - 1) transformSchema(schema, Some(columnName.head)) { case (`prefixPath`, struct @ StructType(_), _) => val columnField = struct(columnName.head) // We need to figure out if the column is top-level column // or a column inside a struct, we support collecting null count stats // on nested columns part of a struct. val fieldInsideStruct = prefixPath.size > 0 validateDataSkippingType( columnAttribute.name, columnField.dataType, visitedColumns, insideStruct = fieldInsideStruct) struct case (_, other, _) => other } } } val duplicatedColumnNames = visitedColumns .groupBy(identity) .collect { case (attribute, occurrences) if occurrences.size > 1 => attribute } .toSeq if (duplicatedColumnNames.size > 0) { throw new DeltaIllegalArgumentException( errorClass = "DELTA_DUPLICATE_DATA_SKIPPING_COLUMNS", messageParameters = Array(duplicatedColumnNames.mkString(",")) ) } } /** * Removes the dropped columns from delta statistics column list inside * DeltaConfigs.DATA_SKIPPING_STATS_COLUMNS. * Note: This method is matching the logical name of tables with the columns inside * DeltaConfigs.DATA_SKIPPING_STATS_COLUMNS. */ def dropDeltaStatsColumns( metadata: Metadata, columnsToDrop: Seq[Seq[String]]): Map[String, String] = { if (columnsToDrop.isEmpty) return Map.empty[String, String] val deltaStatsColumnSpec = configuredDeltaStatsColumnSpec(metadata) deltaStatsColumnSpec.deltaStatsColumnNamesOpt.map { deltaColumnsNames => val droppedColumnSet = columnsToDrop.toSet val deltaStatsColumnStr = deltaColumnsNames .map(_.nameParts) .filterNot { attributeNameParts => droppedColumnSet.filter { droppedColumnParts => val commonPrefix = droppedColumnParts.zip(attributeNameParts) .takeWhile { case (left, right) => left == right } .size commonPrefix == droppedColumnParts.size }.nonEmpty } .map(columnParts => UnresolvedAttribute(columnParts).name) .mkString(",") Map(DeltaConfigs.DATA_SKIPPING_STATS_COLUMNS.key -> deltaStatsColumnStr) }.getOrElse(Map.empty[String, String]) } /** * Rename the delta statistics column `oldColumnPath` of DeltaConfigs.DATA_SKIPPING_STATS_COLUMNS * to `newColumnPath`. * Note: This method is matching the logical name of tables with the columns inside * DeltaConfigs.DATA_SKIPPING_STATS_COLUMNS. */ def renameDeltaStatsColumn( metadata: Metadata, oldColumnPath: Seq[String], newColumnPath: Seq[String]): Map[String, String] = { if (oldColumnPath == newColumnPath) return Map.empty[String, String] val deltaStatsColumnSpec = configuredDeltaStatsColumnSpec(metadata) SchemaUtils.renameColumnForConfig( oldColumnPath, newColumnPath, deltaStatsColumnSpec.deltaStatsColumnNamesOpt, DeltaConfigs.DATA_SKIPPING_STATS_COLUMNS.key) } /** Returns the configured set of columns to be used for stats collection on a table */ def configuredDeltaStatsColumnSpec(metadata: Metadata): DeltaStatsColumnSpec = { val indexedColNamesOpt = DeltaConfigs.DATA_SKIPPING_STATS_COLUMNS.fromMetaData(metadata) val numIndexedCols = DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.fromMetaData(metadata) indexedColNamesOpt.map { indexedColNames => DeltaStatsColumnSpec(DeltaSqlParserUtils.parseMultipartColumnList(indexedColNames), None) }.getOrElse { DeltaStatsColumnSpec(None, Some(numIndexedCols)) } } /** * Convert the logical name of each field to physical name according to the column mapping mode. */ private[sql] def convertToPhysicalName( fullPath: String, field: StructField, schemaNames: Seq[String], mappingMode: DeltaColumnMappingMode): StructField = { // If mapping mode is NoMapping or the dataSchemaName already contains the mapped // column name, the schema mapping can be skipped. if (mappingMode == NoMapping || schemaNames.contains(fullPath)) return field // Check if the physical name exists. if (!DeltaColumnMapping.hasPhysicalName(field)) { throw DeltaErrors.missingPhysicalName(mappingMode, field.name) } // Get the physical column name from metadata. val physicalName = field.metadata.getString(COLUMN_MAPPING_PHYSICAL_NAME_KEY) field.dataType match { case structType: StructType => val newDataType = StructType( structType.map(child => convertToPhysicalName(fullPath, child, schemaNames, mappingMode)) ) field.copy(name = physicalName, dataType = newDataType) case _ => field.copy(name = physicalName) } } /** * Generates a filtered data schema for stats collection. * Note: This method is matching the logical name of tables with the columns inside * DeltaConfigs.DATA_SKIPPING_STATS_COLUMNS. The output of the filter schema is translated into * physical name. * * @param schemaNames the full name path of all columns inside `schema`. * @param schema the original data schema. * @param statsColPaths the specific set of columns to collect stats on. * @param mappingMode the column mapping mode of this statistics collection. * @param parentPath the parent column path of `schema`. * @return filtered schema */ private def filterSchema( schemaNames: Seq[String], schema: StructType, statsColPaths: Seq[Seq[String]], mappingMode: DeltaColumnMappingMode, parentPath: Seq[String] = Seq.empty): StructType = { // Find the unique column names at this nesting depth, each with its path remainders (if any) val cols = statsColPaths.groupBy(_.head).mapValues(_.map(_.tail)) val newSchema = schema.flatMap { field => val lowerCaseFieldName = field.name.toLowerCase(Locale.ROOT) cols.get(lowerCaseFieldName).flatMap { paths => field.dataType match { case _ if paths.forall(_.isEmpty) => // Convert full path to lower cases to avoid schema name contains upper case // characters. val fullPath = (parentPath :+ field.name).mkString(".").toLowerCase(Locale.ROOT) Some(convertToPhysicalName(fullPath, field, schemaNames, mappingMode)) case fieldSchema: StructType => // Convert full path to lower cases to avoid schema name contains upper case // characters. val fullPath = (parentPath :+ field.name).mkString(".").toLowerCase(Locale.ROOT) val physicalName = if (mappingMode == NoMapping || schemaNames.contains(fullPath)) { field.name } else { field.metadata.getString(COLUMN_MAPPING_PHYSICAL_NAME_KEY) } // Recurse into the child fields of this struct. val newSchema = filterSchema( schemaNames, fieldSchema, paths.filterNot(_.isEmpty), mappingMode, parentPath:+ field.name ) Some(field.copy(name = physicalName, dataType = newSchema)) case _ => // Filter expected a nested field and this isn't nested. No match None } } } StructType(newSchema.toArray) } /** * Computes the set of columns to be used for stats collection on a table. Specific named columns * take precedence, if provided; otherwise the first numIndexedColsOpt are extracted from the * schema. */ def getIndexedColumns( schemaNames: Seq[String], spec: DeltaStatsColumnSpec, schema: StructType, mappingMode: DeltaColumnMappingMode): StructType = { spec.deltaStatsColumnNamesOpt .map { indexedColNames => // convert all index columns to lower case characters to avoid user assigning any upper // case characters. val indexedColPaths = indexedColNames.map(_.nameParts.map(_.toLowerCase(Locale.ROOT))) filterSchema(schemaNames, schema, indexedColPaths, mappingMode) } .getOrElse { val numIndexedCols = spec.numIndexedColsOpt.get if (numIndexedCols < 0) { schema // negative means don't truncate the schema } else { truncateSchema(schema, numIndexedCols)._1 } } } /** * Generates a truncated data schema for stats collection. * @param schema the original data schema * @param indexedCols the maximum number of leaf columns to collect stats on * @return truncated schema and the number of leaf columns in this schema */ private def truncateSchema(schema: StructType, indexedCols: Int): (StructType, Int) = { var accCnt = 0 var i = 0 val fields = new ArrayBuffer[StructField]() while (i < schema.length && accCnt < indexedCols) { val field = schema.fields(i) val newField = field match { case StructField(name, st: StructType, nullable, metadata) => val (newSt, cnt) = truncateSchema(st, indexedCols - accCnt) accCnt += cnt StructField(name, newSt, nullable, metadata) case f => accCnt += 1 f } i += 1 fields += newField } (StructType(fields.toSeq), accCnt) } /** * Compute the AddFile entries with delta statistics entries by aggregating the data skipping * columns of each parquet file. */ private def computeNewAddFiles( deltaLog: DeltaLog, txn: OptimisticTransaction, files: Seq[AddFile]): Array[AddFile] = { val dataPath = deltaLog.dataPath val pathToAddFileMap = generateCandidateFileMap(dataPath, files) val persistentDVsReadable = DeletionVectorUtils.deletionVectorsReadable(txn.snapshot) // Throw error when the table contains DVs, because existing method of stats // recomputation doesn't work on tables with DVs. It needs to take into consideration of // DV files (TODO). if (persistentDVsReadable) { throw DeltaErrors.statsRecomputeNotSupportedOnDvTables() } val fileDataFrame = deltaLog .createDataFrame(txn.snapshot, addFiles = files, isStreaming = false) .withColumn("path", col("_metadata.file_path")) val newStats = fileDataFrame.groupBy(col("path")).agg(to_json(txn.statsCollector)) newStats.collect().map { r => val add = getTouchedFile(dataPath, r.getString(0), pathToAddFileMap) add.copy(dataChange = false, stats = r.getString(1)) } } /** * Recomputes statistics for a Delta table. This can be used to compute stats if they were never * collected or to recompute corrupted statistics. * @param deltaLog Delta log for the table to update. * @param predicates Which subset of the data to recompute stats for. Predicates must use only * partition columns. * @param fileFilter Filter for which AddFiles to recompute stats for. */ def recompute( spark: SparkSession, deltaLog: DeltaLog, catalogTable: Option[CatalogTable], predicates: Seq[Expression] = Seq(Literal(true)), fileFilter: AddFile => Boolean = af => true): Unit = { val txn = deltaLog.startTransaction(catalogTable) verifyPartitionPredicates(spark, txn.metadata.partitionColumns, predicates) // Save the current AddFiles that match the predicates so we can update their stats val files = txn.filterFiles(predicates).filter(fileFilter) val newAddFiles = computeNewAddFiles(deltaLog, txn, files) txn.commit(newAddFiles, ComputeStats(predicates)) } def truncateMinStringAgg(prefixLen: Int)(input: String): String = { if (input == null || input.length <= prefixLen) { return input } if (prefixLen <= 0) { return null } if (Character.isHighSurrogate(input.charAt(prefixLen - 1)) && Character.isLowSurrogate(input.charAt(prefixLen))) { // If the character at prefixLen - 1 is a high surrogate and the next character is a low // surrogate, we need to include the next character in the prefix to ensure that we don't // truncate the string in the middle of a surrogate pair. input.take(prefixLen + 1) } else { input.take(prefixLen) } } /** * Helper method to truncate the input string `input` to the given `prefixLen` length, while also * ensuring the any value in this column is less than or equal to the truncated max in UTF-8 * encoding. */ def truncateMaxStringAgg(prefixLen: Int)(originalMax: String): String = { // scalastyle:off nonascii if (originalMax == null || originalMax.length <= prefixLen) { return originalMax } if (prefixLen <= 0) { return null } // Grab the prefix. We want to append max Unicode code point `\uDBFF\uDFFF` as a tie-breaker, // but that is only safe if the character we truncated was smaller in UTF-8 encoded binary // comparison. Keep extending the prefix until that condition holds, or we run off the end of // the string. // We also try to use the ASCII max character `\u007F` as a tie-breaker if possible. val maxLen = getExpansionLimit(prefixLen) // Start with a valid prefix var currLen = truncateMinStringAgg(prefixLen)(originalMax).length while (currLen <= maxLen) { if (currLen >= originalMax.length) { // Return originalMax if we have reached the end of the string return originalMax } else if (currLen + 1 < originalMax.length && originalMax.substring(currLen, currLen + 2) == UTF8_MAX_CHARACTER) { // Skip the UTF-8 max character. It occupies two characters in a Scala string. currLen += 2 } else if (originalMax.charAt(currLen) < ASCII_MAX_CHARACTER) { return originalMax.take(currLen) + ASCII_MAX_CHARACTER } else { return originalMax.take(currLen) + UTF8_MAX_CHARACTER } } // Return null when the input string is too long to truncate. null // scalastyle:on nonascii } /** * Calculates the upper character limit when constructing a maximum is not possible with only * prefixLen chars. */ private def getExpansionLimit(prefixLen: Int): Int = 2 * prefixLen } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/stats/StatsCollectionUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats import scala.collection.JavaConverters._ import scala.collection.mutable import scala.concurrent.duration.Duration import scala.language.existentials import scala.util.control.NonFatal // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.{DeltaColumnMapping, DeltaColumnMappingMode, DeltaConfigs, DeltaErrors, DeltaLog, IdMapping, NameMapping, NoMapping, Snapshot} import org.apache.spark.sql.delta.actions.{AddFile, Metadata} import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.DeltaStatistics._ import org.apache.spark.sql.delta.util.{DeltaFileOperations, JsonUtils} import org.apache.spark.sql.delta.util.threads.DeltaThreadPool import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.FileSystem import org.apache.hadoop.fs.Path import org.apache.parquet.hadoop.ParquetFileReader import org.apache.parquet.hadoop.metadata.{BlockMetaData, ParquetMetadata} import org.apache.parquet.io.api.Binary import org.apache.parquet.schema.LogicalTypeAnnotation._ import org.apache.parquet.schema.PrimitiveType import org.apache.spark.internal.{Logging, MDC} import org.apache.spark.sql.{Dataset, SparkSession} import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.execution.datasources.DataSourceUtils import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{ArrayType, LongType, MapType, StructField, StructType} import org.apache.spark.util.SerializableConfiguration object StatsCollectionUtils extends Logging { /** * A helper function to compute stats of addFiles using StatsCollector. * * @param spark The SparkSession used to process data. * @param conf The Hadoop configuration used to access file system. * @param deltaLog The delta log of table, to which these AddFile(s) belong. * @param snapshot The snapshot of the table used to derive table schema information. We do not * derive it from deltaLog because a snapshot may not exist yet. * @param addFiles The list of target AddFile(s) to be processed. * @param numFilesOpt The number of AddFile(s) to process if known. Speeds up the query. * @param ignoreMissingStats Whether to ignore missing stats during computation. * @param setBoundsToWide Whether to set bounds to wide independently of whether or not * the files have DVs. * * @return A list of AddFile(s) with newly computed stats, please note the existing stats from * the input addFiles will be ignored regardless. */ def computeStats( spark: SparkSession, conf: Configuration, deltaLog: DeltaLog, snapshot: Snapshot, addFiles: Dataset[AddFile], numFilesOpt: Option[Long], stringTruncateLength: Int, ignoreMissingStats: Boolean = true, setBoundsToWide: Boolean = false): Dataset[AddFile] = { val useMultiThreadedStatsCollection = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_USE_MULTI_THREADED_STATS_COLLECTION) val preparedAddFiles = if (useMultiThreadedStatsCollection) { prepareRDDForMultiThreadedStatsCollection(spark, addFiles, numFilesOpt) } else { addFiles } val parquetRebaseMode = spark.sessionState.conf.getConf(SQLConf.PARQUET_REBASE_MODE_IN_READ).toString val statsCollector = StatsCollector( snapshot.columnMappingMode, snapshot.dataSchema, snapshot.statsSchema, parquetRebaseMode, ignoreMissingStats, Some(stringTruncateLength)) val serializableConf = new SerializableConfiguration(conf) val broadcastConf = spark.sparkContext.broadcast(serializableConf) val dataRootDir = deltaLog.dataPath.toString import org.apache.spark.sql.delta.implicits._ preparedAddFiles.mapPartitions { addFileIter => val defaultFileSystem = new Path(dataRootDir).getFileSystem(broadcastConf.value.value) if (useMultiThreadedStatsCollection) { ParallelFetchPool.parallelMap(spark, addFileIter.toSeq) { addFile => computeStatsForFile( addFile, dataRootDir, defaultFileSystem, broadcastConf.value, setBoundsToWide, statsCollector) }.toIterator } else { addFileIter.map { addFile => computeStatsForFile( addFile, dataRootDir, defaultFileSystem, broadcastConf.value, setBoundsToWide, statsCollector) } } } } /** * Prepares the underlying RDD of [[addFiles]] for multi-threaded stats collection by splitting * them up into more partitions if necessary. * If the number of partitions is too small, not every executor might * receive a partition, which reduces the achievable parallelism. By increasing the number of * partitions we can achieve more parallelism. */ private def prepareRDDForMultiThreadedStatsCollection( spark: SparkSession, addFiles: Dataset[AddFile], numFilesOpt: Option[Long]): Dataset[AddFile] = { val numFiles = numFilesOpt.getOrElse(addFiles.count()) val currNumPartitions = addFiles.rdd.getNumPartitions val numFilesPerPartition = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_STATS_COLLECTION_NUM_FILES_PARTITION) // We should not create more partitions than the cluster can currently handle. val minNumPartitions = Math.min( spark.sparkContext.defaultParallelism, numFiles / numFilesPerPartition + 1).toInt // Only repartition if it would increase the achievable parallelism if (currNumPartitions < minNumPartitions) { addFiles.repartition(minNumPartitions) } else { addFiles } } private def computeStatsForFile( addFile: AddFile, dataRootDir: String, defaultFileSystem: FileSystem, config: SerializableConfiguration, setBoundsToWide: Boolean, statsCollector: StatsCollector): AddFile = { val path = DeltaFileOperations.absolutePath(dataRootDir, addFile.path) val fileStatus = if (path.toString.startsWith(dataRootDir)) { defaultFileSystem.getFileStatus(path) } else { path.getFileSystem(config.value).getFileStatus(path) } val (stats, metric) = statsCollector.collect( ParquetFileReader.readFooter(config.value, fileStatus)) if (metric.totalMissingFields > 0 || metric.numMissingTypes > 0) { logWarning( log"StatsCollection of file `${MDC(DeltaLogKeys.PATH, path)}` " + log"misses fields/types: ${MDC(DeltaLogKeys.METRICS, JsonUtils.toJson(metric))}") } val statsWithTightBoundsCol = { val hasDeletionVector = addFile.deletionVector != null && !addFile.deletionVector.isEmpty stats + (TIGHT_BOUNDS -> !(setBoundsToWide || hasDeletionVector)) } addFile.copy(stats = JsonUtils.toJson(statsWithTightBoundsCol)) } /** * Get the string prefix length used for data skipping based on the following precedence: * 1. If the provided metadata is not null, and the delta table property is set inside, use it; * 2. Otherwise, use the Spark configuration. */ def getDataSkippingStringPrefixLength(spark: SparkSession, metadata: Metadata): Int = { Option(metadata) .flatMap(DeltaConfigs.DATA_SKIPPING_STRING_PREFIX_LENGTH.fromMetaData) .getOrElse(spark.sessionState.conf.getConf(DeltaSQLConf.DATA_SKIPPING_STRING_PREFIX_LENGTH)) } } object ParallelFetchPool { val NUM_THREADS_PER_CORE = 10 val MAX_THREADS = 1024 val NUM_THREADS = Math.min( Runtime.getRuntime.availableProcessors() * NUM_THREADS_PER_CORE, MAX_THREADS) lazy val threadPool = DeltaThreadPool("stats-collection", NUM_THREADS) def parallelMap[T, R]( spark: SparkSession, items: Iterable[T])( f: T => R): Iterable[R] = threadPool.parallelMap(spark, items)(f) } /** * A helper class to collect stats of parquet data files for Delta table and its equivalent (tables * that can be converted into Delta table like Parquet/Iceberg table). * * @param dataSchema The data schema from table metadata, which is the logical schema with logical * to physical mapping per schema field. It is used to map statsSchema to parquet * metadata. * @param statsSchema The schema of stats to be collected, statsSchema should follow the physical * schema and must be generated by StatisticsCollection. * @param parquetRebaseMode The parquet rebase mode used to parse date and timestamp. * @param ignoreMissingStats Indicate whether to return partial result by ignoring missing stats * or throw an exception. * @param stringTruncateLength The optional max length of string stats to be truncated into. * * Scala Example: * {{{ * import org.apache.spark.sql.delta.stats.StatsCollector * * val stringTruncateLength = * spark.sessionState.conf.getConf(DeltaSQLConf.DATA_SKIPPING_STRING_PREFIX_LENGTH) * * val statsCollector = StatsCollector( * snapshot.metadata.columnMappingMode, snapshot.metadata.dataSchema, snapshot.statsSchema, * ignoreMissingStats = false, Some(stringTruncateLength)) * * val filesWithStats = snapshot.allFiles.map { file => * val path = DeltaFileOperations.absolutePath(dataPath, file.path) * val fileSystem = path.getFileSystem(hadoopConf) * val fileStatus = fileSystem.listStatus(path).head * * val footer = ParquetFileReader.readFooter(hadoopConf, fileStatus) * val (stats, _) = statsCollector.collect(footer) * file.copy(stats = JsonUtils.toJson(stats)) * } * }}} */ abstract class StatsCollector( dataSchema: StructType, statsSchema: StructType, parquetRebaseMode: String, ignoreMissingStats: Boolean, stringTruncateLength: Option[Int]) extends Serializable { final val NUM_MISSING_TYPES = "numMissingTypes" /** * Used to report number of missing fields per supported type and number of missing unsupported * types in the collected statistics, currently the statistics collection supports 4 types of * stats: NUM_RECORDS, MAX, MIN, NULL_COUNT. * * @param numMissingMax The number of missing fields for MAX * @param numMissingMin The number of missing fields for MIN * @param numMissingNullCount The number of missing fields for NULL_COUNT * @param numMissingTypes The number of unsupported type being requested. */ case class StatsCollectionMetrics( numMissingMax: Long, numMissingMin: Long, numMissingNullCount: Long, numMissingTypes: Long) { val totalMissingFields: Long = Seq(numMissingMax, numMissingMin, numMissingNullCount).sum } object StatsCollectionMetrics { def apply(missingFieldCounts: Map[String, Long]): StatsCollectionMetrics = { StatsCollectionMetrics( missingFieldCounts.getOrElse(MAX, 0L), missingFieldCounts.getOrElse(MIN, 0L), missingFieldCounts.getOrElse(NULL_COUNT, 0L), missingFieldCounts.getOrElse(NUM_MISSING_TYPES, 0L)) } } /** * A list of schema physical path and corresponding struct field of leaf fields. Beside primitive * types, Map and Array (instead of their sub-columns) are also treated as leaf fields since we * only compute null count of them, and null is counted based on themselves instead of sub-fields. */ protected lazy val schemaPhysicalPathAndSchemaField: Seq[(Seq[String], StructField)] = { def explode(schema: StructType): Seq[(Seq[String], StructField)] = { schema.flatMap { field => val physicalName = DeltaColumnMapping.getPhysicalName(field) field.dataType match { case s: StructType => explode(s).map { case (path, field) => (Seq(physicalName) ++ path, field) } case _ => (Seq(physicalName), field) :: Nil } } } explode(dataSchema) } /** * Returns the map from schema physical field path (field for which to collect stats) to the * parquet metadata column index (where to collect stats). statsSchema generated by * StatisticsCollection always use physical field paths so physical field paths are the same as * to the ones used in statsSchema. Child class must implement this method based on delta column * mapping mode. */ def getSchemaPhysicalPathToParquetIndex(blockMetaData: BlockMetaData): Map[Seq[String], Int] /** * Collects the stats from [[ParquetMetadata]] * * @param parquetMetadata The metadata of parquet file following physical schema, it contains * statistics of row groups. * * @return A nested Map[String: Any] from requested stats field names to their stats field value * and [[StatsCollectionMetrics]] counting the number of missing fields/types. */ final def collect( parquetMetadata: ParquetMetadata): (Map[String, Any], StatsCollectionMetrics) = { val blocks = parquetMetadata.getBlocks.asScala.toSeq if (blocks.isEmpty) { return (Map(NUM_RECORDS -> 0L), StatsCollectionMetrics(Map.empty[String, Long])) } val schemaPhysicalPathToParquetIndex = getSchemaPhysicalPathToParquetIndex(blocks.head) val dateRebaseSpec = DataSourceUtils.datetimeRebaseSpec( parquetMetadata.getFileMetaData.getKeyValueMetaData.get, parquetRebaseMode) val dateRebaseFunc = DataSourceUtils.createDateRebaseFuncInRead(dateRebaseSpec.mode, "Parquet") val missingFieldCounts = mutable.Map(MAX -> 0L, MIN -> 0L, NULL_COUNT -> 0L, NUM_MISSING_TYPES -> 0L) // Collect the actual stats. // // The result of this operation is a tree of maps that matches the structure of the stats // schema. The stats schema is split by stats type at the top, and each type matches the // structure of the data schema (can be subset), so we collect per stats type. E.g. the MIN // values are under MIN.a, MIN.b.c, MIN.b.d etc., and then the MAX values are under MAX.a, // MAX.b.c etc. Note, we do omit here the tightBounds column and add it at a later stage. val collectedStats = statsSchema.filter(_.name != TIGHT_BOUNDS).map { case StructField(NUM_RECORDS, LongType, _, _) => val numRecords = blocks.map { block => block.getRowCount }.sum NUM_RECORDS -> numRecords case StructField(MIN, statsTypeSchema: StructType, _, _) => val (minValues, numMissingFields) = collectStats(Seq.empty[String], statsTypeSchema, blocks, schemaPhysicalPathToParquetIndex, ignoreMissingStats)(aggMaxOrMin(dateRebaseFunc, isMax = false)) missingFieldCounts(MIN) += numMissingFields MIN -> minValues case StructField(MAX, statsTypeSchema: StructType, _, _) => val (maxValues, numMissingFields) = collectStats(Seq.empty[String], statsTypeSchema, blocks, schemaPhysicalPathToParquetIndex, ignoreMissingStats)(aggMaxOrMin(dateRebaseFunc, isMax = true)) missingFieldCounts(MAX) += numMissingFields MAX -> maxValues case StructField(NULL_COUNT, statsTypeSchema: StructType, _, _) => val (nullCounts, numMissingFields) = collectStats(Seq.empty[String], statsTypeSchema, blocks, schemaPhysicalPathToParquetIndex, ignoreMissingStats)(aggNullCount) missingFieldCounts(NULL_COUNT) += numMissingFields NULL_COUNT -> nullCounts case field: StructField => if (ignoreMissingStats) { missingFieldCounts(NUM_MISSING_TYPES) += 1 field.name -> Map.empty[String, Any] } else { throw new UnsupportedOperationException(s"stats type not supported: ${field.name}") } }.toMap (collectedStats, StatsCollectionMetrics(missingFieldCounts.toMap)) } /** * Collects statistics by recurring through the structure of statsSchema and tracks the fields * that we have seen so far in parentPhysicalPath. * * @param parentPhysicalFieldPath The absolute path of parent field with physical names. * @param statsSchema The schema with physical names to collect stats recursively. * @param blocks The metadata of Parquet row groups, which contains the raw stats. * @param schemaPhysicalPathToParquetIndex Map from schema path to parquet metadata column index. * @param ignoreMissingStats Whether to ignore and log missing fields or throw an exception. * @param aggFunc The aggregation function used to aggregate stats across row. * * @return A nested Map[String: Any] from schema field name to stats value and a count of missing * fields. * * Here is an example of stats: * * stats schema: * | -- id: INT * | -- person: STRUCT * | name: STRUCT * | -- first: STRING * | -- last: STRING * | height: LONG * * The stats: * Map( * "id" -> 1003, * "person" -> Map( * "name" -> Map( * "first" -> "Chris", * "last" -> "Green" * ), * "height" -> 175L * ) * ) */ private def collectStats( parentPhysicalFieldPath: Seq[String], statsSchema: StructType, blocks: Seq[BlockMetaData], schemaPhysicalPathToParquetIndex: Map[Seq[String], Int], ignoreMissingStats: Boolean)( aggFunc: (Seq[BlockMetaData], Int) => Any): (Map[String, Any], Long) = { val stats = mutable.Map.empty[String, Any] var numMissingFields = 0L statsSchema.foreach { case StructField(name, dataType: StructType, _, _) => val (map, numMissingFieldsInSubtree) = collectStats(parentPhysicalFieldPath :+ name, dataType, blocks, schemaPhysicalPathToParquetIndex, ignoreMissingStats)(aggFunc) numMissingFields += numMissingFieldsInSubtree if (map.nonEmpty) { stats += name -> map } case StructField(name, _, _, _) => val physicalFieldPath = parentPhysicalFieldPath :+ name if (schemaPhysicalPathToParquetIndex.contains(physicalFieldPath)) { try { val value = aggFunc(blocks, schemaPhysicalPathToParquetIndex(physicalFieldPath)) // None value means the stats is undefined for this field (e.g., max/min of a field, // whose values are nulls in all blocks), we use null to be consistent with stats // generated from SQL. if (value != None) { stats += name -> value } else { stats += name -> null } } catch { case NonFatal(_) if ignoreMissingStats => numMissingFields += 1L case exception: Throwable => throw exception } } else if (ignoreMissingStats) { // Physical field path requested by stats is missing in the mapping, so it's missing from // the parquet metadata. numMissingFields += 1L } else { val columnPath = physicalFieldPath.mkString("[", ", ", "]") throw DeltaErrors.deltaStatsCollectionColumnNotFound("all", columnPath) } } (stats.toMap, numMissingFields) } /** * The aggregation function used to collect the max and min of a column across blocks, * dateRebaseFunc is used to adapt legacy date. */ private def aggMaxOrMin( dateRebaseFunc: Int => Int, isMax: Boolean)( blocks: Seq[BlockMetaData], index: Int): Any = { val columnMetadata = blocks.head.getColumns.get(index) val primitiveType = columnMetadata.getPrimitiveType val logicalType = primitiveType.getLogicalTypeAnnotation // Physical type of timestamp is INT96 in both Parquet and Delta. if (primitiveType.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT96 || logicalType.isInstanceOf[TimestampLogicalTypeAnnotation]) { throw new UnsupportedOperationException( s"max/min stats is not supported for timestamp: ${columnMetadata.getPath}") } var aggregatedValue: Any = None blocks.foreach { block => val column = block.getColumns.get(index) val statistics = column.getStatistics // Skip this block if the column has null for all rows, stats is defined as long as it exists // in even a single block. if (statistics.hasNonNullValue) { val currentValue = if (isMax) statistics.genericGetMax else statistics.genericGetMin if (currentValue == null) { throw DeltaErrors.deltaStatsCollectionColumnNotFound("max/min", column.getPath.toString) } if (aggregatedValue == None) { aggregatedValue = currentValue } else { // TODO: check NaN value for floating point columns. val compareResult = currentValue.asInstanceOf[Comparable[Any]].compareTo(aggregatedValue) if ((isMax && compareResult > 0) || (!isMax && compareResult < 0)) { aggregatedValue = currentValue } } } } // All blocks have null stats for this column, returns None to indicate the stats of this // column is undefined. if (aggregatedValue == None) return None aggregatedValue match { // String case bytes: Binary if logicalType.isInstanceOf[StringLogicalTypeAnnotation] => val rawString = bytes.toStringUsingUTF8 if (stringTruncateLength.isDefined && rawString.length > stringTruncateLength.get) { if (isMax) { // Append tie breakers to assure that any value in this column is less than or equal to // the max, check the helper function for more details. StatisticsCollection.truncateMaxStringAgg(stringTruncateLength.get)(rawString) } else { StatisticsCollection.truncateMinStringAgg(stringTruncateLength.get)(rawString) } } else { rawString } // Binary case _: Binary => throw new UnsupportedOperationException( s"max/min stats is not supported for binary other than string: ${columnMetadata.getPath}") // Date case date: Integer if logicalType.isInstanceOf[DateLogicalTypeAnnotation] => DateTimeUtils.toJavaDate(dateRebaseFunc(date)).toString // Byte, Short, Integer and Long case intValue @ (_: Integer | _: java.lang.Long) if logicalType.isInstanceOf[IntLogicalTypeAnnotation] => logicalType.asInstanceOf[IntLogicalTypeAnnotation].getBitWidth match { case 8 => intValue.asInstanceOf[Int].toByte case 16 => intValue.asInstanceOf[Int].toShort case 32 => intValue.asInstanceOf[Int] case 64 => intValue.asInstanceOf[Long] case other => throw new UnsupportedOperationException( s"max/min stats is not supported for $other-bits Integer: ${columnMetadata.getPath}") } // Decimal case _ if logicalType.isInstanceOf[DecimalLogicalTypeAnnotation] => throw new UnsupportedOperationException( s"max/min stats is not supported for decimal: ${columnMetadata.getPath}") // Integer, Long, Float and Double case primitive @ (_: Integer | _: java.lang.Long | _: java.lang.Float | _: java.lang.Double) if logicalType == null => primitive // Throw an exception on the other unknown types for safety. case unknown => throw new UnsupportedOperationException( s"max/min stats is not supported for ${unknown.getClass.getName} with $logicalType:" + columnMetadata.getPath.toString) } } /** The aggregation function used to count null of a column across blocks */ private def aggNullCount(blocks: Seq[BlockMetaData], index: Int): Any = { var count = 0L blocks.foreach { block => val column = block.getColumns.get(index) val statistics = column.getStatistics if (!statistics.isNumNullsSet) { throw DeltaErrors.deltaStatsCollectionColumnNotFound("nullCount", column.getPath.toString) } count += statistics.getNumNulls } count.asInstanceOf[Any] } } object StatsCollector { def apply( columnMappingMode: DeltaColumnMappingMode, dataSchema: StructType, statsSchema: StructType, parquetRebaseMode: String, ignoreMissingStats: Boolean = true, stringTruncateLength: Option[Int] = None): StatsCollector = { columnMappingMode match { case NoMapping | NameMapping => StatsCollectorNameMapping( dataSchema, statsSchema, parquetRebaseMode, ignoreMissingStats, stringTruncateLength) case IdMapping => StatsCollectorIdMapping( dataSchema, statsSchema, parquetRebaseMode, ignoreMissingStats, stringTruncateLength) case _ => throw new UnsupportedOperationException( s"$columnMappingMode mapping is currently not supported") } } private case class StatsCollectorNameMapping( dataSchema: StructType, statsSchema: StructType, parquetRebaseMode: String, ignoreMissingStats: Boolean, stringTruncateLength: Option[Int]) extends StatsCollector( dataSchema, statsSchema, parquetRebaseMode, ignoreMissingStats, stringTruncateLength) { /** * Maps schema physical field path to parquet metadata column index via parquet metadata column * path in NoMapping and NameMapping modes */ override def getSchemaPhysicalPathToParquetIndex( blockMetaData: BlockMetaData): Map[Seq[String], Int] = { val parquetColumnPathToIndex = getParquetColumnPathToIndex(blockMetaData) columnPathSchemaToParquet.collect { // Collect mapping of fields in physical schema that actually exist in parquet metadata, // parquet metadata can miss field due to schema evolution. In case stats collection is // requested on a column that is missing from parquet metadata, we will catch this in // collectStats when looking up in this map. case (schemaPath, parquetPath) if parquetColumnPathToIndex.contains(parquetPath) => schemaPath -> parquetColumnPathToIndex(parquetPath) } } /** * A map from schema field path (with physical names) to parquet metadata column path of schema * leaf fields with special handling of Array and Map. * * Here is an example: * * Data Schema (physical name in the parenthesis) * | -- id (a4def3): INT * | -- history (23aa42): STRUCT * | -- cost (23ddb0): DOUBLE * | -- events (23dda1): ARRAY[STRING] * | -- info (abb4d2): MAP[STRING, STRING] * * Block Metadata: * Columns: [ [a4def3], [23aa42, 23ddb0], [23ddb0, 23dda1, list, element], * [abb4d2, key_value, key], [abb4d2, key_value, value] ] * * The mapping: * [a4def3] -> [a4def3] * [23aa42, 23ddb0] -> [23aa42, 23ddb0] * [23ddb0, 23dda1] -> [23ddb0, 23dda1, list, element] * [abb4d2] -> [abb4d2, key_value, key] */ private lazy val columnPathSchemaToParquet: Map[Seq[String], Seq[String]] = { // Parquet metadata column path contains addition keywords for Array and Map. Here we only // support 2 cases below since stats is not available in the other cases: // 1. Array with non-null elements of primitive types // 2. Map with key of primitive types schemaPhysicalPathAndSchemaField.map { case(path, field) => field.dataType match { // Here we don't check array element type and map key type for primitive type since // parquet metadata column path always points to a primitive column. In other words, // the type is primitive if the column path can be found in parquet metadata later. case ArrayType(_, false) => path -> (path ++ Seq("list", "element")) case MapType(_, _, _) => path -> (path ++ Seq("key_value", "key")) case _ => path -> path } }.toMap } /** * Returns a map from parquet metadata column path to index. * * Here is an example: * * Data Schema: * |-- id : INT * |-- person : STRUCT * |-- name: STRING * |-- phone: INT * |-- eligible: BOOLEAN * * Block Metadata: * Columns: [ [id], [person, name], [person, phone], [eligible] ] * * The mapping: * [id] -> 0 * [person, name] -> 1 * [person, phone] -> 2 * [eligible] -> 3 */ private def getParquetColumnPathToIndex(block: BlockMetaData): Map[Seq[String], Int] = { block.getColumns.asScala.zipWithIndex.map { case (column, i) => column.getPath.toArray.toSeq -> i }.toMap } } private case class StatsCollectorIdMapping( dataSchema: StructType, statsSchema: StructType, parquetRebaseMode: String, ignoreMissingStats: Boolean, stringTruncateLength: Option[Int]) extends StatsCollector( dataSchema, statsSchema, parquetRebaseMode, ignoreMissingStats, stringTruncateLength) { // Define a FieldId type to better disambiguate between ids and indices in the code type FieldId = Int /** * Maps schema physical field path to parquet metadata column index via parquet metadata column * id in IdMapping mode. */ override def getSchemaPhysicalPathToParquetIndex( blockMetaData: BlockMetaData): Map[Seq[String], Int] = { val parquetColumnIdToIndex = getParquetColumnIdToIndex(blockMetaData) schemaPhysicalPathToColumnId.collect { // Collect mapping of fields in physical schema that actually exist in parquet metadata, // parquet metadata can miss field due to schema evolution and non-primitive types like Map // and Array. In case stats collection is requested on a column that is missing from // parquet metadata, we will catch this in collectStats when looking up in this map. case (schemaPath, columnId) if parquetColumnIdToIndex.contains(columnId) => schemaPath -> parquetColumnIdToIndex(columnId) } } /** * A map from schema field path (with physical names) to parquet metadata column id of schema * leaf fields. * * Here is an example: * * Data Schema (physical name, id in the parenthesis) * | -- id (a4def3, 1): INT * | -- history (23aa42, 2): STRUCT * | -- cost (23ddb0, 3): DOUBLE * | -- events (23dda1, 4): ARRAY[STRING] * | -- info (abb4d2, 5): MAP[STRING, STRING] * * The mapping: * [a4def3] -> 1 * [23aa42, 23ddb0] -> 3 * [23ddb0, 23dda1] -> 4 * [abb4d2] -> 5 */ private lazy val schemaPhysicalPathToColumnId: Map[Seq[String], FieldId] = { schemaPhysicalPathAndSchemaField.map { case (path, field) => path -> DeltaColumnMapping.getColumnId(field) }.toMap } /** * Returns a map from parquet metadata column id to column index by skipping columns without id. * E.g., subfields of ARRAY and MAP don't have id assigned. * * Here is an example: * * Data Schema (id in the parenthesis): * |-- id (1) : INT * |-- person (2) : STRUCT * |-- names (3) : ARRAY[STRING] * |-- phones (4) : MAP[STRING, INT] * |-- eligible (5) : BOOLEAN * * Block Metadata (id in the parenthesis): * Columns: [ [id](1), [person, names, list, element](null), * [person, phones, key_value, key](null), [person, phones, key_value, value](null), * [eligible](5) ] * * The mapping: 1 -> 0, 5 -> 4 */ private def getParquetColumnIdToIndex(block: BlockMetaData): Map[FieldId, Int] = { block.getColumns.asScala.zipWithIndex.collect { // Id of parquet metadata column is not guaranteed, subfields of Map and Array don't have // id assigned. In case id is missing and null, we skip the parquet metadata column here // and will catch this in collectStats when looking up in this map. case (column, i) if column.getPrimitiveType.getId != null => column.getPrimitiveType.getId.intValue() -> i }.toMap } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/stats/StatsProvider.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats import org.apache.spark.sql.Column import org.apache.spark.sql.types.DataType /** * A helper class that provides the functionalities to create [[DataSkippingPredicate]] with * the statistics for a column. * * @param getStat A function that returns an expression to access the given statistics for a * specific column, or None if that stats column does not exist. For example, * [[DataSkippingReaderBase.getStatsColumnOpt]] can be used here. */ private [stats] class StatsProvider(getStat: StatsColumn => Option[Column]) { /** * Given a [[StatsColumn]], which represents a stats column for a table column, returns a * [[DataSkippingPredicate]] which includes a data skipping expression (the result of running * `f` on the expression of accessing the given stats) and the stats column (which the data * skipping expression depends on), or None if the stats column does not exist. * * @param statCol A stats column (MIN, MAX, etc) for a table column name. * @param f A user-provided function that returns a data skipping expression given the expression * to access the statistics for `statCol`. * @return A [[DataSkippingPredicate]] with a data skipping expression, or None if the given * stats column does not exist. */ def getPredicateWithStatsColumnIfExists(statCol: StatsColumn) (f: Column => Column): Option[DataSkippingPredicate] = { for (stat <- getStat(statCol)) yield DataSkippingPredicate(f(stat), statCol) } /** A variant of [[getPredicateWithStatsColumnIfExists]] with two stats columns. */ def getPredicateWithStatsColumnsIfExists(statCol1: StatsColumn, statCol2: StatsColumn) (f: (Column, Column) => Column): Option[DataSkippingPredicate] = { for (stat1 <- getStat(statCol1); stat2 <- getStat(statCol2)) yield DataSkippingPredicate(f(stat1, stat2), statCol1, statCol2) } /** A variant of [[getPredicateWithStatsColumnIfExists]] with three stats columns. */ def getPredicateWithStatsColumnsIfExists( statCol1: StatsColumn, statCol2: StatsColumn, statCol3: StatsColumn) (f: (Column, Column, Column) => Column): Option[DataSkippingPredicate] = { for (stat1 <- getStat(statCol1); stat2 <- getStat(statCol2); stat3 <- getStat(statCol3)) yield DataSkippingPredicate(f(stat1, stat2, stat3), statCol1, statCol2, statCol3) } /** * Given a path to a table column and a stat type (MIN, MAX, etc.), returns a * [[DataSkippingPredicate]] which includes a data skipping expression (the result of running * `f` on the expression of accessing the given stats) and the stats column (which the data * skipping expression depends on), or None if the stats column does not exist. * * @param pathToColumn The name of a column whose stats are to be accessed. * @param statType The type of stats to access (MIN, MAX, etc.) * @param f A user-provided function that returns a data skipping expression given the expression * to access the statistics for `statCol`. * @return A [[DataSkippingPredicate]] with a data skipping expression, or None if the given * stats column does not exist. */ def getPredicateWithStatTypeIfExists( pathToColumn: Seq[String], columnDataType: DataType, statType: String) (f: Column => Column): Option[DataSkippingPredicate] = { getPredicateWithStatsColumnIfExists(StatsColumn(statType, pathToColumn, columnDataType))(f) } /** A variant of [[getPredicateWithStatTypeIfExists]] with two stat types. */ def getPredicateWithStatTypesIfExists( pathToColumn: Seq[String], columnDataType: DataType, statType1: String, statType2: String) (f: (Column, Column) => Column): Option[DataSkippingPredicate] = { getPredicateWithStatsColumnsIfExists( StatsColumn(statType1, pathToColumn, columnDataType), StatsColumn(statType2, pathToColumn, columnDataType))(f) } /** A variant of [[getPredicateWithStatTypeIfExists]] with three stat types. */ def getPredicateWithStatTypesIfExists( pathToColumn: Seq[String], columnDataType: DataType, statType1: String, statType2: String, statType3: String) (f: (Column, Column, Column) => Column): Option[DataSkippingPredicate] = { getPredicateWithStatsColumnsIfExists( StatsColumn(statType1, pathToColumn, columnDataType), StatsColumn(statType2, pathToColumn, columnDataType), StatsColumn(statType3, pathToColumn, columnDataType))(f) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/storage/AzureLogStore.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.storage import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf /** * LogStore implementation for Azure. * * We assume the following from Azure's [[FileSystem]] implementations: * - Rename without overwrite is atomic. * - List-after-write is consistent. * * Regarding file creation, this implementation: * - Uses atomic rename when overwrite is false; if the destination file exists or the rename * fails, throws an exception. * - Uses create-with-overwrite when overwrite is true. This does not make the file atomically * visible and therefore the caller must handle partial files. */ class AzureLogStore(sparkConf: SparkConf, hadoopConf: Configuration) extends HadoopFileSystemLogStore(sparkConf, hadoopConf) { override def write(path: Path, actions: Iterator[String], overwrite: Boolean = false): Unit = { writeWithRename(path, actions, overwrite) } override def write( path: Path, actions: Iterator[String], overwrite: Boolean, hadoopConf: Configuration): Unit = { writeWithRename(path, actions, overwrite, hadoopConf) } override def invalidateCache(): Unit = {} override def isPartialWriteVisible(path: Path): Boolean = true override def isPartialWriteVisible(path: Path, hadoopConf: Configuration): Boolean = true } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/storage/ClosableIterator.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.storage import java.io.Closeable trait SupportsRewinding[T] extends Iterator[T] { // Overrides if class supports rewinding the iterator to the beginning efficiently. def rewind(): Unit } trait ClosableIterator[T] extends Iterator[T] with Closeable { /** Calls f(this) and always closes the iterator afterwards. */ def processAndClose[R](f: Iterator[T] => R): R = { try { f(this) } finally { close() } } } object ClosableIterator { /** * An implicit class for applying a function to a [[ClosableIterator]] and returning the * resulting iterator as a [[ClosableIterator]] with the original `close()` method. */ implicit class IteratorCloseOps[A](val closableIter: ClosableIterator[A]) extends AnyVal { def withClose[B](f: Iterator[A] => Iterator[B]): ClosableIterator[B] = new ClosableIterator[B] { private val iter = try { f(closableIter) } catch { case e: Throwable => closableIter.close() throw e } override def next(): B = iter.next() override def hasNext: Boolean = iter.hasNext override def close(): Unit = closableIter.close() } } /** * An implicit class for a `flatMap` implementation that returns a [[ClosableIterator]] * which (a) closes inner iterators upon reaching their end, and (b) has a `close()` method * that closes any opened and unclosed inner iterators. */ implicit class IteratorFlatMapCloseOp[A](val closableIter: Iterator[A]) extends AnyVal { def flatMapWithClose[B](f: A => ClosableIterator[B]): ClosableIterator[B] = new ClosableIterator[B] { private var iter_curr: ClosableIterator[B] = null override def next(): B = { if (!hasNext) { throw new NoSuchElementException } iter_curr.next() } @scala.annotation.tailrec override def hasNext: Boolean = { if (iter_curr == null && closableIter.hasNext) { iter_curr = f(closableIter.next()) } if (iter_curr == null) { false } else if (iter_curr.hasNext) { true } else { iter_curr.close() if (closableIter.hasNext) { iter_curr = f(closableIter.next()) hasNext } else { iter_curr = null false } } } override def close(): Unit = { if (iter_curr != null) { iter_curr.close() } } } } /** * An implicit class for wrapping an iterator to be a [[ClosableIterator]] with a `close` method * that does nothing. */ implicit class ClosableWrapper[A](val iter: Iterator[A]) extends AnyVal { def toClosable: ClosableIterator[A] = new ClosableIterator[A] { override def next(): A = iter.next() override def hasNext: Boolean = iter.hasNext override def close(): Unit = () } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/storage/DelegatingLogStore.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.storage import java.util.Locale import scala.collection.mutable import org.apache.spark.sql.delta.DeltaErrors import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.SparkEnv import org.apache.spark.internal.MDC /** * A delegating LogStore used to dynamically resolve LogStore implementation based * on the scheme of paths. */ class DelegatingLogStore(hadoopConf: Configuration) extends LogStore with DeltaLogging { private val sparkConf = SparkEnv.get.conf // Map scheme to the corresponding LogStore resolved and created. Accesses to this map need // synchronization This could be accessed by multiple threads because it is shared through // shared DeltaLog instances. private val schemeToLogStoreMap = mutable.Map.empty[String, LogStore] private lazy val defaultLogStore = createLogStore(DelegatingLogStore.defaultHDFSLogStoreClassName) // Creates a LogStore with given LogStore class name. private def createLogStore(className: String): LogStore = { LogStore.createLogStoreWithClassName(className, sparkConf, hadoopConf) } // Create LogStore based on the scheme of `path`. private def schemeBasedLogStore(path: Path): LogStore = { val store = Option(path.toUri.getScheme) match { case Some(origScheme) => val scheme = origScheme.toLowerCase(Locale.ROOT) this.synchronized { if (schemeToLogStoreMap.contains(scheme)) { schemeToLogStoreMap(scheme) } else { // Resolve LogStore class based on the following order: // 1. Scheme conf if set. // 2. Defaults for scheme if exists. // 3. Default. val logStoreClassNameOpt = LogStore.getLogStoreConfValue( // we look for all viable keys LogStore.logStoreSchemeConfKey(scheme), sparkConf) .orElse(DelegatingLogStore.getDefaultLogStoreClassName(scheme)) val logStore = logStoreClassNameOpt.map(createLogStore(_)).getOrElse(defaultLogStore) schemeToLogStoreMap += scheme -> logStore val actualLogStoreClassName = logStore match { case lsa: LogStoreAdaptor => s"LogStoreAdapter(${lsa.logStoreImpl.getClass.getName})" case _ => logStore.getClass.getName } logInfo(log"LogStore ${MDC(DeltaLogKeys.SYSTEM_CLASS_NAME, actualLogStoreClassName)} " + log"is used for scheme ${MDC(DeltaLogKeys.FILE_SYSTEM_SCHEME, scheme)}") logStore } } case _ => defaultLogStore } store } def getDelegate(path: Path): LogStore = schemeBasedLogStore(path) ////////////////////////// // Public API Overrides // ////////////////////////// override def read(path: Path): Seq[String] = { getDelegate(path).read(path) } override def read(path: Path, hadoopConf: Configuration): Seq[String] = { getDelegate(path).read(path, hadoopConf) } override def readAsIterator(path: Path): ClosableIterator[String] = { getDelegate(path).readAsIterator(path) } override def readAsIterator(path: Path, hadoopConf: Configuration): ClosableIterator[String] = { getDelegate(path).readAsIterator(path, hadoopConf) } override def write( path: Path, actions: Iterator[String], overwrite: Boolean): Unit = { getDelegate(path).write(path, actions, overwrite) } override def write( path: Path, actions: Iterator[String], overwrite: Boolean, hadoopConf: Configuration): Unit = { getDelegate(path).write(path, actions, overwrite, hadoopConf) } override def listFrom(path: Path): Iterator[FileStatus] = { getDelegate(path).listFrom(path) } override def listFrom(path: Path, hadoopConf: Configuration): Iterator[FileStatus] = { getDelegate(path).listFrom(path, hadoopConf) } override def invalidateCache(): Unit = { this.synchronized { schemeToLogStoreMap.foreach { entry => entry._2.invalidateCache() } } defaultLogStore.invalidateCache() } override def resolvePathOnPhysicalStorage(path: Path): Path = { getDelegate(path).resolvePathOnPhysicalStorage(path) } override def resolvePathOnPhysicalStorage(path: Path, hadoopConf: Configuration): Path = { getDelegate(path).resolvePathOnPhysicalStorage(path, hadoopConf) } override def isPartialWriteVisible(path: Path): Boolean = { getDelegate(path).isPartialWriteVisible(path) } override def isPartialWriteVisible(path: Path, hadoopConf: Configuration): Boolean = { getDelegate(path).isPartialWriteVisible(path, hadoopConf) } } object DelegatingLogStore { try { // load any arbitrary delta-storage class to ensure the dependency has been included classOf[io.delta.storage.LogStore] } catch { case e: NoClassDefFoundError => throw DeltaErrors.missingDeltaStorageJar(e) } /** * Java LogStore (io.delta.storage) implementations are now the default. */ val defaultS3LogStoreClassName = classOf[io.delta.storage.S3SingleDriverLogStore].getName val defaultAzureLogStoreClassName = classOf[io.delta.storage.AzureLogStore].getName val defaultHDFSLogStoreClassName = classOf[io.delta.storage.HDFSLogStore].getName val defaultGCSLogStoreClassName = classOf[io.delta.storage.GCSLogStore].getName // Supported schemes with default. val s3Schemes = Set("s3", "s3a", "s3n") val azureSchemes = Set("abfs", "abfss", "adl", "wasb", "wasbs") val gsSchemes = Set("gs") // Returns the default LogStore class name for `scheme`. // None if we do not have a default for it. def getDefaultLogStoreClassName(scheme: String): Option[String] = { if (s3Schemes.contains(scheme)) { return Some(defaultS3LogStoreClassName) } else if (DelegatingLogStore.azureSchemes(scheme: String)) { return Some(defaultAzureLogStoreClassName) } else if (DelegatingLogStore.gsSchemes(scheme: String)) { return Some(defaultGCSLogStoreClassName) } None } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/storage/HDFSLogStore.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.storage import java.io.IOException import java.nio.charset.StandardCharsets.UTF_8 import java.util.EnumSet import scala.util.control.NonFatal import org.apache.spark.sql.delta.DeltaErrors import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs._ import org.apache.hadoop.fs.CreateFlag.CREATE import org.apache.hadoop.fs.Options.{ChecksumOpt, CreateOpts} import org.apache.spark.SparkConf import org.apache.spark.internal.Logging /** * The [[LogStore]] implementation for HDFS, which uses Hadoop [[FileContext]] API's to * provide the necessary atomic and durability guarantees: * * 1. Atomic visibility of files: `FileContext.rename` is used write files which is atomic for HDFS. * * 2. Consistent file listing: HDFS file listing is consistent. */ class HDFSLogStore(sparkConf: SparkConf, defaultHadoopConf: Configuration) extends HadoopFileSystemLogStore(sparkConf, defaultHadoopConf) with Logging{ @deprecated("call the method that asks for a Hadoop Configuration object instead") protected def getFileContext(path: Path): FileContext = { FileContext.getFileContext(path.toUri, getHadoopConfiguration) } protected def getFileContext(path: Path, hadoopConf: Configuration): FileContext = { FileContext.getFileContext(path.toUri, hadoopConf) } val noAbstractFileSystemExceptionMessage = "No AbstractFileSystem" override def write(path: Path, actions: Iterator[String], overwrite: Boolean = false): Unit = { write(path, actions, overwrite, getHadoopConfiguration) } override def write( path: Path, actions: Iterator[String], overwrite: Boolean, hadoopConf: Configuration): Unit = { val isLocalFs = path.getFileSystem(hadoopConf).isInstanceOf[RawLocalFileSystem] if (isLocalFs) { // We need to add `synchronized` for RawLocalFileSystem as its rename will not throw an // exception when the target file exists. Hence we must make sure `exists + rename` in // `writeInternal` for RawLocalFileSystem is atomic in our tests. synchronized { writeInternal(path, actions, overwrite, hadoopConf) } } else { // rename is atomic and also will fail when the target file exists. Not need to add the extra // `synchronized`. writeInternal(path, actions, overwrite, hadoopConf) } } private def writeInternal( path: Path, actions: Iterator[String], overwrite: Boolean, hadoopConf: Configuration): Unit = { val fc: FileContext = try { getFileContext(path, hadoopConf) } catch { case e: IOException if e.getMessage.contains(noAbstractFileSystemExceptionMessage) => val newException = DeltaErrors.incorrectLogStoreImplementationException(sparkConf, e) logError(newException.getMessage, newException.getCause) throw newException } if (!overwrite && fc.util.exists(path)) { // This is needed for the tests to throw error with local file system throw DeltaErrors.fileAlreadyExists(path.toString) } val tempPath = createTempPath(path) var streamClosed = false // This flag is to avoid double close var renameDone = false // This flag is to save the delete operation in most of cases. val stream = fc.create( tempPath, EnumSet.of(CREATE), CreateOpts.checksumParam(ChecksumOpt.createDisabled())) try { actions.map(_ + "\n").map(_.getBytes(UTF_8)).foreach(stream.write) stream.close() streamClosed = true try { val renameOpt = if (overwrite) Options.Rename.OVERWRITE else Options.Rename.NONE fc.rename(tempPath, path, renameOpt) renameDone = true // TODO: this is a workaround of HADOOP-16255 - remove this when HADOOP-16255 is resolved tryRemoveCrcFile(fc, tempPath) } catch { case e: org.apache.hadoop.fs.FileAlreadyExistsException => throw DeltaErrors.fileAlreadyExists(path.toString) } } finally { if (!streamClosed) { stream.close() } if (!renameDone) { fc.delete(tempPath, false) } } msyncIfSupported(path, hadoopConf) } /** * Normally when using HDFS with an Observer NameNode setup, there would be read after write * consistency within a single process, so the write would be guaranteed to be visible on the * next read. However, since we are using the FileContext API for writing (for atomic rename), * and the FileSystem API for reading (for more compatibility with various file systems), we * are essentially using two separate clients that are not guaranteed to be kept in sync. * Therefore we "msync" the FileSystem instance, which is cached across all uses of the same * protocol/host combination, to make sure the next read through the HDFSLogStore can see this * write. * Any underlying FileSystem that is not the DistributedFileSystem will simply throw an * UnsupportedOperationException, which can be ignored. Additionally, if an older version of * Hadoop is being used that does not include msync, a NoSuchMethodError will be thrown while * looking up the method, which can also be safely ignored. */ private def msyncIfSupported(path: Path, hadoopConf: Configuration): Unit = { try { val fs = path.getFileSystem(hadoopConf) val msync = fs.getClass.getMethod("msync") msync.invoke(fs) } catch { case NonFatal(_) => // ignore, calling msync is best effort } } private def tryRemoveCrcFile(fc: FileContext, path: Path): Unit = { try { val checksumFile = new Path(path.getParent, s".${path.getName}.crc") if (fc.util.exists(checksumFile)) { // checksum file exists, deleting it fc.delete(checksumFile, true) } } catch { case NonFatal(_) => // ignore, we are removing crc file as "best-effort" } } override def isPartialWriteVisible(path: Path): Boolean = true override def isPartialWriteVisible(path: Path, hadoopConf: Configuration): Boolean = true } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/storage/HadoopFileSystemLogStore.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.storage import java.io.{BufferedReader, InputStreamReader} import java.nio.charset.StandardCharsets.UTF_8 import java.nio.file.FileAlreadyExistsException import java.util.UUID import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.DeltaErrors import org.apache.commons.io.IOUtils import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, FileSystem, FSDataInputStream, Path} import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.sql.SparkSession /** * Default implementation of [[LogStore]] for Hadoop [[FileSystem]] implementations. */ abstract class HadoopFileSystemLogStore( sparkConf: SparkConf, hadoopConf: Configuration) extends LogStore { def this(sc: SparkContext) = this(sc.getConf, sc.hadoopConfiguration) protected def getHadoopConfiguration: Configuration = { // scalastyle:off deltahadoopconfiguration SparkSession.getActiveSession.map(_.sessionState.newHadoopConf()).getOrElse(hadoopConf) // scalastyle:on deltahadoopconfiguration } override def read(path: Path): Seq[String] = { read(path, getHadoopConfiguration) } override def read(path: Path, hadoopConf: Configuration): Seq[String] = { readStream(open(path, hadoopConf)) } override def readAsIterator(path: Path): ClosableIterator[String] = { readAsIterator(path, getHadoopConfiguration) } override def readAsIterator(path: Path, hadoopConf: Configuration): ClosableIterator[String] = readStreamAsIterator(open(path, hadoopConf)) private def open(path: Path, hadoopConf: Configuration): FSDataInputStream = path.getFileSystem(hadoopConf).open(path) private def readStream(stream: FSDataInputStream): Seq[String] = { try { val reader = new BufferedReader(new InputStreamReader(stream, UTF_8)) IOUtils.readLines(reader).asScala.map(_.trim).toSeq } finally { stream.close() } } private def readStreamAsIterator(stream: FSDataInputStream): ClosableIterator[String] = { val reader = new BufferedReader(new InputStreamReader(stream, UTF_8)) new LineClosableIterator(reader) } override def listFrom(path: Path): Iterator[FileStatus] = { listFrom(path, getHadoopConfiguration) } override def listFrom(path: Path, hadoopConf: Configuration): Iterator[FileStatus] = { val fs = path.getFileSystem(hadoopConf) if (!fs.exists(path.getParent)) { throw DeltaErrors.fileOrDirectoryNotFoundException(s"${path.getParent}") } val files = fs.listStatus(path.getParent) files.filter(_.getPath.getName >= path.getName).sortBy(_.getPath.getName).iterator } override def resolvePathOnPhysicalStorage(path: Path): Path = { resolvePathOnPhysicalStorage(path, getHadoopConfiguration) } override def resolvePathOnPhysicalStorage(path: Path, hadoopConf: Configuration): Path = { path.getFileSystem(hadoopConf).makeQualified(path) } /** * An internal write implementation that uses FileSystem.rename(). * * This implementation should only be used for the underlying file systems that support atomic * renames, e.g., Azure is OK but HDFS is not. */ @deprecated("call the method that asks for a Hadoop Configuration object instead") protected def writeWithRename( path: Path, actions: Iterator[String], overwrite: Boolean = false): Unit = { writeWithRename(path, actions, overwrite, getHadoopConfiguration) } /** * An internal write implementation that uses FileSystem.rename(). * * This implementation should only be used for the underlying file systems that support atomic * renames, e.g., Azure is OK but HDFS is not. */ protected def writeWithRename( path: Path, actions: Iterator[String], overwrite: Boolean, hadoopConf: Configuration): Unit = { val fs = path.getFileSystem(hadoopConf) if (!fs.exists(path.getParent)) { throw DeltaErrors.fileOrDirectoryNotFoundException(s"${path.getParent}") } if (overwrite) { val stream = fs.create(path, true) try { actions.map(_ + "\n").map(_.getBytes(UTF_8)).foreach(stream.write) } finally { stream.close() } } else { if (fs.exists(path)) { throw DeltaErrors.fileAlreadyExists(path.toString) } val tempPath = createTempPath(path) var streamClosed = false // This flag is to avoid double close var renameDone = false // This flag is to save the delete operation in most of cases. val stream = fs.create(tempPath) try { actions.map(_ + "\n").map(_.getBytes(UTF_8)).foreach(stream.write) stream.close() streamClosed = true try { if (fs.rename(tempPath, path)) { renameDone = true } else { if (fs.exists(path)) { throw DeltaErrors.fileAlreadyExists(path.toString) } else { throw DeltaErrors.cannotRenamePath(tempPath.toString, path.toString) } } } catch { case _: org.apache.hadoop.fs.FileAlreadyExistsException => throw DeltaErrors.fileAlreadyExists(path.toString) } } finally { if (!streamClosed) { stream.close() } if (!renameDone) { fs.delete(tempPath, false) } } } } protected def createTempPath(path: Path): Path = { new Path(path.getParent, s".${path.getName}.${UUID.randomUUID}.tmp") } override def invalidateCache(): Unit = {} } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/storage/LineClosableIterator.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.storage import java.io.Reader import org.apache.spark.sql.delta.DeltaErrors import org.apache.commons.io.IOUtils /** * Turn a `Reader` to `ClosableIterator` which can be read on demand. Each element is * a trimmed line. */ class LineClosableIterator(_reader: Reader) extends ClosableIterator[String] { private val reader = IOUtils.toBufferedReader(_reader) // Whether `nextValue` is valid. If it's invalid, we should try to read the next line. private var gotNext = false // The next value to return when `next` is called. This is valid only if `getNext` is true. private var nextValue: String = _ // Whether the reader is closed. private var closed = false // Whether we have consumed all data in the reader. private var finished = false override def hasNext: Boolean = { if (!finished) { // Check whether we have closed the reader before reading. Even if `nextValue` is valid, we // still don't return `nextValue` after a reader is closed. Otherwise, it would be confusing. if (closed) { throw DeltaErrors.iteratorAlreadyClosed() } if (!gotNext) { val nextLine = reader.readLine() if (nextLine == null) { finished = true close() } else { nextValue = nextLine.trim } gotNext = true } } !finished } override def next(): String = { if (!hasNext) { throw new NoSuchElementException("End of stream") } gotNext = false nextValue } override def close(): Unit = { if (!closed) { closed = true reader.close() } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/storage/LocalLogStore.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.storage import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf /** * Default [[LogStore]] implementation (should be used for testing only!). * * Production users should specify the appropriate [[[LogStore]] implementation in Spark properties. * * We assume the following from [[org.apache.hadoop.fs.FileSystem]] implementations: * - Rename without overwrite is atomic. * - List-after-write is consistent. * * Regarding file creation, this implementation: * - Uses atomic rename when overwrite is false; if the destination file exists or the rename * fails, throws an exception. * - Uses create-with-overwrite when overwrite is true. This does not make the file atomically * visible and therefore the caller must handle partial files. */ class LocalLogStore(sparkConf: SparkConf, hadoopConf: Configuration) extends HadoopFileSystemLogStore(sparkConf: SparkConf, hadoopConf: Configuration) { /** * This write implementation needs to wraps `writeWithRename` with `synchronized` as the rename() * for [[org.apache.hadoop.fs.RawLocalFileSystem]] doesn't throw an exception when the target file * exists. Hence we must make sure `exists + rename` in `writeWithRename` is atomic in our tests. */ override def write(path: Path, actions: Iterator[String], overwrite: Boolean = false): Unit = { synchronized { writeWithRename(path, actions, overwrite) } } /** * This write implementation needs to wraps `writeWithRename` with `synchronized` as the rename() * for [[org.apache.hadoop.fs.RawLocalFileSystem]] doesn't throw an exception when the target file * exists. Hence we must make sure `exists + rename` in `writeWithRename` is atomic in our tests. */ override def write( path: Path, actions: Iterator[String], overwrite: Boolean, hadoopConf: Configuration): Unit = { synchronized { writeWithRename(path, actions, overwrite, hadoopConf) } } override def invalidateCache(): Unit = {} override def isPartialWriteVisible(path: Path): Boolean = true override def isPartialWriteVisible(path: Path, hadoopConf: Configuration): Boolean = true } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/storage/LogStore.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.storage import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.{DeltaErrors, DeltaLog} import io.delta.storage.CloseableIterator import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession import org.apache.spark.util.Utils /** * General interface for all critical file system operations required to read and write the * [[DeltaLog]]. The correctness of the [[DeltaLog]] is predicated on the atomicity and * durability guarantees of the implementation of this interface. Specifically, * * 1. Atomic visibility of files: Any file written through this store must * be made visible atomically. In other words, this should not generate partial files. * * 2. Mutual exclusion: Only one writer must be able to create (or rename) a file at the final * destination. * * 3. Consistent listing: Once a file has been written in a directory, all future listings for * that directory must return that file. */ trait LogStore { /** * Load the given file and return a `Seq` of lines. The line break will be removed from each * line. This method will load the entire file into the memory. Call `readAsIterator` if possible * as its implementation may be more efficient. */ @deprecated("call the method that asks for a Hadoop Configuration object instead") final def read(path: String): Seq[String] = read(new Path(path)) /** * Load the given file and return a `Seq` of lines. The line break will be removed from each * line. This method will load the entire file into the memory. Call `readAsIterator` if possible * as its implementation may be more efficient. */ @deprecated("call the method that asks for a Hadoop Configuration object instead") def read(path: Path): Seq[String] /** * Load the given file and return a `Seq` of lines. The line break will be removed from each * line. This method will load the entire file into the memory. Call `readAsIterator` if possible * as its implementation may be more efficient. * * Note: The default implementation ignores the `hadoopConf` parameter to provide the backward * compatibility. Subclasses should override this method and use `hadoopConf` properly to support * passing Hadoop file system configurations through DataFrame options. */ def read(path: Path, hadoopConf: Configuration): Seq[String] = read(path) /** * Load the given file represented by `fileStatus` and return a `Seq` of lines. * The line break will be removed from each line. * * Note: Using a stale `FileStatus` may get an incorrect result. */ final def read(fileStatus: FileStatus, hadoopConf: Configuration): Seq[String] = { val iter = readAsIterator(fileStatus, hadoopConf) try { iter.toIndexedSeq } finally { iter.close() } } /** * Load the given file and return an iterator of lines. The line break will be removed from each * line. The default implementation calls `read` to load the entire file into the memory. * An implementation should provide a more efficient approach if possible. For example, the file * content can be loaded on demand. */ @deprecated("call the method that asks for a Hadoop Configuration object instead") final def readAsIterator(path: String): ClosableIterator[String] = { readAsIterator(new Path(path)) } /** * Load the given file and return an iterator of lines. The line break will be removed from each * line. The default implementation calls `read` to load the entire file into the memory. * An implementation should provide a more efficient approach if possible. For example, the file * content can be loaded on demand. * * Note: the returned [[ClosableIterator]] should be closed when it's no longer used to avoid * resource leak. */ @deprecated("call the method that asks for a Hadoop Configuration object instead") def readAsIterator(path: Path): ClosableIterator[String] = { val iter = read(path).iterator new ClosableIterator[String] { override def hasNext: Boolean = iter.hasNext override def next(): String = iter.next() override def close(): Unit = {} } } /** * Load the given file and return an iterator of lines. The line break will be removed from each * line. The default implementation calls `read` to load the entire file into the memory. * An implementation should provide a more efficient approach if possible. For example, the file * content can be loaded on demand. * * Note: the returned [[ClosableIterator]] should be closed when it's no longer used to avoid * resource leak. * * Note: The default implementation ignores the `hadoopConf` parameter to provide the backward * compatibility. Subclasses should override this method and use `hadoopConf` properly to support * passing Hadoop file system configurations through DataFrame options. */ def readAsIterator(path: Path, hadoopConf: Configuration): ClosableIterator[String] = { readAsIterator(path) } /** * Load the file represented by given fileStatus and return an iterator of lines. The line break * will be removed from each line. * * Note-1: the returned [[ClosableIterator]] should be closed when it's no longer used to avoid * resource leak. * * Note-2: Using a stale `FileStatus` may get an incorrect result. */ def readAsIterator( fileStatus: FileStatus, hadoopConf: Configuration): ClosableIterator[String] = { readAsIterator(fileStatus.getPath, hadoopConf) } /** * Write the given `actions` to the given `path` without overwriting any existing file. * Implementation must throw [[java.nio.file.FileAlreadyExistsException]] exception if the file * already exists. Furthermore, implementation must ensure that the entire file is made * visible atomically, that is, it should not generate partial files. */ @deprecated("call the method that asks for a Hadoop Configuration object instead") final def write(path: String, actions: Iterator[String]): Unit = write(new Path(path), actions) /** * Write the given `actions` to the given `path` with or without overwrite as indicated. * Implementation must throw [[java.nio.file.FileAlreadyExistsException]] exception if the file * already exists and overwrite = false. Furthermore, implementation must ensure that the * entire file is made visible atomically, that is, it should not generate partial files. */ @deprecated("call the method that asks for a Hadoop Configuration object instead") def write(path: Path, actions: Iterator[String], overwrite: Boolean = false): Unit /** * Write the given `actions` to the given `path` with or without overwrite as indicated. * Implementation must throw [[java.nio.file.FileAlreadyExistsException]] exception if the file * already exists and overwrite = false. Furthermore, implementation must ensure that the * entire file is made visible atomically, that is, it should not generate partial files. * * Note: The default implementation ignores the `hadoopConf` parameter to provide the backward * compatibility. Subclasses should override this method and use `hadoopConf` properly to support * passing Hadoop file system configurations through DataFrame options. */ def write( path: Path, actions: Iterator[String], overwrite: Boolean, hadoopConf: Configuration): Unit = { write(path, actions, overwrite) } /** * List the paths in the same directory that are lexicographically greater or equal to * (UTF-8 sorting) the given `path`. The result should also be sorted by the file name. */ @deprecated("call the method that asks for a Hadoop Configuration object instead") final def listFrom(path: String): Iterator[FileStatus] = listFrom(new Path(path)) /** * List the paths in the same directory that are lexicographically greater or equal to * (UTF-8 sorting) the given `path`. The result should also be sorted by the file name. */ @deprecated("call the method that asks for a Hadoop Configuration object instead") def listFrom(path: Path): Iterator[FileStatus] /** * List the paths in the same directory that are lexicographically greater or equal to * (UTF-8 sorting) the given `path`. The result should also be sorted by the file name. * * Note: The default implementation ignores the `hadoopConf` parameter to provide the backward * compatibility. Subclasses should override this method and use `hadoopConf` properly to support * passing Hadoop file system configurations through DataFrame options. */ def listFrom(path: Path, hadoopConf: Configuration): Iterator[FileStatus] = listFrom(path) /** Invalidate any caching that the implementation may be using */ def invalidateCache(): Unit /** Resolve the fully qualified path for the given `path`. */ @deprecated("call the method that asks for a Hadoop Configuration object instead") def resolvePathOnPhysicalStorage(path: Path): Path = { throw new UnsupportedOperationException() } /** * Resolve the fully qualified path for the given `path`. * * Note: The default implementation ignores the `hadoopConf` parameter to provide the backward * compatibility. Subclasses should override this method and use `hadoopConf` properly to support * passing Hadoop file system configurations through DataFrame options. */ def resolvePathOnPhysicalStorage(path: Path, hadoopConf: Configuration): Path = { resolvePathOnPhysicalStorage(path) } /** * Whether a partial write is visible when writing to `path`. * * As this depends on the underlying file system implementations, we require the input of `path` * here in order to identify the underlying file system, even though in most cases a log store * only deals with one file system. * * The default value is only provided here for legacy reasons, which will be removed. * Any LogStore implementation should override this instead of relying on the default. */ @deprecated("call the method that asks for a Hadoop Configuration object instead") def isPartialWriteVisible(path: Path): Boolean = true /** * Whether a partial write is visible when writing to `path`. * * As this depends on the underlying file system implementations, we require the input of `path` * here in order to identify the underlying file system, even though in most cases a log store * only deals with one file system. * * The default value is only provided here for legacy reasons, which will be removed. * Any LogStore implementation should override this instead of relying on the default. * * Note: The default implementation ignores the `hadoopConf` parameter to provide the backward * compatibility. Subclasses should override this method and use `hadoopConf` properly to support * passing Hadoop file system configurations through DataFrame options. */ def isPartialWriteVisible(path: Path, hadoopConf: Configuration): Boolean = { isPartialWriteVisible(path) } } object LogStore extends LogStoreProvider with Logging { def apply(spark: SparkSession): LogStore = { // scalastyle:off deltahadoopconfiguration // Ensure that the LogStore's hadoopConf has the values from the SQLConf. // This ensures that io.delta.storage LogStore (Java) hadoopConf's are configured correctly. apply(spark.sparkContext.getConf, spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration } def apply(sparkConf: SparkConf, hadoopConf: Configuration): LogStore = { createLogStore(sparkConf, hadoopConf) } // Creates a LogStore with the given LogStore class name and configurations. def createLogStoreWithClassName( className: String, sparkConf: SparkConf, hadoopConf: Configuration): LogStore = { if (className == classOf[DelegatingLogStore].getName) { new DelegatingLogStore(hadoopConf) } else { val logStoreClass = Utils.classForName(className) if (classOf[io.delta.storage.LogStore].isAssignableFrom(logStoreClass)) { new LogStoreAdaptor(logStoreClass.getConstructor(classOf[Configuration]) .newInstance(hadoopConf)) } else { logStoreClass.getConstructor(classOf[SparkConf], classOf[Configuration]) .newInstance(sparkConf, hadoopConf).asInstanceOf[LogStore] } } } } trait LogStoreProvider { val logStoreClassConfKey: String = "spark.delta.logStore.class" val defaultLogStoreClass: String = classOf[DelegatingLogStore].getName // The conf key for setting the LogStore implementation for `scheme`. def logStoreSchemeConfKey(scheme: String): String = s"spark.delta.logStore.${scheme}.impl" /** * We accept keys both with and without the `spark.` prefix to maintain compatibility across the * Delta ecosystem * @param key the spark-prefixed key to access */ def getLogStoreConfValue(key: String, sparkConf: SparkConf): Option[String] = { // verifyLogStoreConfs already validated that if both keys exist the values are the same when // the LogStore was instantiated sparkConf.getOption(key) .orElse(sparkConf.getOption(key.stripPrefix("spark."))) } def createLogStore(spark: SparkSession): LogStore = { LogStore(spark) } /** * Check for conflicting LogStore configs in the spark configuration. * * To maintain compatibility across the Delta ecosystem, we accept keys both with and without the * "spark." prefix. This means for setting the class conf, we accept both * "spark.delta.logStore.class" and "delta.logStore.class" and for scheme confs we accept both * "spark.delta.logStore.${scheme}.impl" and "delta.logStore.${scheme}.impl" * * If a conf is set both with and without the spark prefix, it must be set to the same value, * otherwise we throw an error. */ def verifyLogStoreConfs(sparkConf: SparkConf): Unit = { // check LogStore class conf key val classConf = sparkConf.getOption(logStoreClassConfKey.stripPrefix("spark.")) classConf.foreach { nonPrefixValue => sparkConf.getOption(logStoreClassConfKey).foreach { prefixValue => // Both the spark-prefixed and non-spark-prefixed key is present in the sparkConf. Check // that they store the same value, otherwise throw an error. if (prefixValue != nonPrefixValue) { throw DeltaErrors.inconsistentLogStoreConfs( Seq((logStoreClassConfKey.stripPrefix("spark."), nonPrefixValue), (logStoreClassConfKey, prefixValue))) } } } // check LogStore scheme conf keys val schemeConfs = sparkConf.getAllWithPrefix("delta.logStore.") .filter(_._1.endsWith(".impl")) schemeConfs.foreach { case (nonPrefixKey, nonPrefixValue) => val prefixKey = logStoreSchemeConfKey(nonPrefixKey.stripSuffix(".impl")) sparkConf.getOption(prefixKey).foreach { prefixValue => // Both the spark-prefixed and non-spark-prefixed key is present in the sparkConf. Check // that they store the same value, otherwise throw an error. if (prefixValue != nonPrefixValue) { throw DeltaErrors.inconsistentLogStoreConfs( Seq(("delta.logStore." + nonPrefixKey, nonPrefixValue), (prefixKey, prefixValue))) } } } } def checkLogStoreConfConflicts(sparkConf: SparkConf): Unit = { val sparkPrefixLogStoreConfs = sparkConf.getAllWithPrefix("spark.delta.logStore.") .map(kv => "spark.delta.logStore." + kv._1 -> kv._2) val nonSparkPrefixLogStoreConfs = sparkConf.getAllWithPrefix("delta.logStore.") .map(kv => "delta.logStore." + kv._1 -> kv._2) val (classConf, otherConf) = (sparkPrefixLogStoreConfs ++ nonSparkPrefixLogStoreConfs) .partition(v => v._1.endsWith("class")) val schemeConf = otherConf.filter(_._1.endsWith(".impl")) if (classConf.nonEmpty && schemeConf.nonEmpty) { throw DeltaErrors.logStoreConfConflicts(classConf, schemeConf) } } def createLogStore(sparkConf: SparkConf, hadoopConf: Configuration): LogStore = { checkLogStoreConfConflicts(sparkConf) verifyLogStoreConfs(sparkConf) val logStoreClassName = getLogStoreConfValue(logStoreClassConfKey, sparkConf) .getOrElse(defaultLogStoreClass) LogStore.createLogStoreWithClassName(logStoreClassName, sparkConf, hadoopConf) } } class LogStoreInverseAdaptor(val logStoreImpl: LogStore, override val initHadoopConf: Configuration) extends io.delta.storage.LogStore(initHadoopConf) { override def read( path: Path, hadoopConf: Configuration): CloseableIterator[String] = { val iter = logStoreImpl.readAsIterator(path, hadoopConf) new CloseableIterator[String] { override def close(): Unit = iter.close override def hasNext: Boolean = iter.hasNext override def next(): String = iter.next() } } override def write( path: Path, actions: java.util.Iterator[String], overwrite: java.lang.Boolean, hadoopConf: Configuration): Unit = { logStoreImpl.write(path, actions.asScala, overwrite, hadoopConf) } override def listFrom( path: Path, hadoopConf: Configuration): java.util.Iterator[FileStatus] = logStoreImpl.listFrom(path, hadoopConf).asJava override def resolvePathOnPhysicalStorage( path: Path, hadoopConf: Configuration): Path = logStoreImpl.resolvePathOnPhysicalStorage(path, hadoopConf) override def isPartialWriteVisible( path: Path, hadoopConf: Configuration): java.lang.Boolean = logStoreImpl.isPartialWriteVisible(path, hadoopConf) } object LogStoreInverseAdaptor { def apply(logStoreImpl: LogStore, initHadoopConf: Configuration): LogStoreInverseAdaptor = { new LogStoreInverseAdaptor(logStoreImpl, initHadoopConf) } } /** * An adaptor from the new public LogStore API to the old private LogStore API. The old LogStore * API is still used in most places. Before we move all of them to the new API, adapting from * the new API to the old API is a cheap way to ensure that implementations of both APIs work. * * @param logStoreImpl An implementation of the new public LogStore API. */ class LogStoreAdaptor(val logStoreImpl: io.delta.storage.LogStore) extends LogStore { private def getHadoopConfiguration: Configuration = { // scalastyle:off deltahadoopconfiguration SparkSession.getActiveSession.map(_.sessionState.newHadoopConf()) .getOrElse(logStoreImpl.initHadoopConf()) // scalastyle:on deltahadoopconfiguration } override def read(path: Path): Seq[String] = { read(path, getHadoopConfiguration) } override def read(path: Path, hadoopConf: Configuration): Seq[String] = { var iter: io.delta.storage.CloseableIterator[String] = null try { iter = logStoreImpl.read(path, hadoopConf) val contents = iter.asScala.toArray contents } finally { if (iter != null) { iter.close } } } override def readAsIterator(path: Path): ClosableIterator[String] = { readAsIterator(path, getHadoopConfiguration) } override def readAsIterator(path: Path, hadoopConf: Configuration): ClosableIterator[String] = { val iter = logStoreImpl.read(path, hadoopConf) new ClosableIterator[String] { override def close(): Unit = iter.close override def hasNext: Boolean = iter.hasNext override def next(): String = iter.next } } override def write(path: Path, actions: Iterator[String], overwrite: Boolean): Unit = { write(path, actions, overwrite, getHadoopConfiguration) } override def write( path: Path, actions: Iterator[String], overwrite: Boolean, hadoopConf: Configuration): Unit = { logStoreImpl.write(path, actions.asJava, overwrite, hadoopConf) } override def listFrom(path: Path): Iterator[FileStatus] = { listFrom(path, getHadoopConfiguration) } override def listFrom(path: Path, hadoopConf: Configuration): Iterator[FileStatus] = { logStoreImpl.listFrom(path, hadoopConf).asScala } override def invalidateCache(): Unit = {} override def resolvePathOnPhysicalStorage(path: Path): Path = { resolvePathOnPhysicalStorage(path, getHadoopConfiguration) } override def resolvePathOnPhysicalStorage(path: Path, hadoopConf: Configuration): Path = { logStoreImpl.resolvePathOnPhysicalStorage(path, hadoopConf) } override def isPartialWriteVisible(path: Path): Boolean = { isPartialWriteVisible(path, getHadoopConfiguration) } override def isPartialWriteVisible(path: Path, hadoopConf: Configuration): Boolean = { logStoreImpl.isPartialWriteVisible(path, hadoopConf) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/storage/S3SingleDriverLogStore.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.storage import java.io.FileNotFoundException import java.net.URI import java.nio.charset.StandardCharsets.UTF_8 import java.util.concurrent.{ConcurrentHashMap, TimeUnit} import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.DeltaErrors import org.apache.spark.sql.delta.util.FileNames import com.google.common.cache.CacheBuilder import com.google.common.io.CountingOutputStream import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs._ import org.apache.spark.SparkConf /** * Single Spark-driver/JVM LogStore implementation for S3. * * We assume the following from S3's [[FileSystem]] implementations: * - File writing on S3 is all-or-nothing, whether overwrite or not. * - List-after-write can be inconsistent. * * Regarding file creation, this implementation: * - Opens a stream to write to S3 (regardless of the overwrite option). * - Failures during stream write may leak resources, but may never result in partial writes. * * Regarding directory listing, this implementation: * - returns a list by merging the files listed from S3 and recently-written files from the cache. */ class S3SingleDriverLogStore( sparkConf: SparkConf, hadoopConf: Configuration) extends HadoopFileSystemLogStore(sparkConf, hadoopConf) { import S3SingleDriverLogStore._ private def resolved(path: Path, hadoopConf: Configuration): (FileSystem, Path) = { val fs = path.getFileSystem(hadoopConf) val resolvedPath = stripUserInfo(fs.makeQualified(path)) (fs, resolvedPath) } private def getPathKey(resolvedPath: Path): Path = { stripUserInfo(resolvedPath) } private def stripUserInfo(path: Path): Path = { val uri = path.toUri val newUri = new URI( uri.getScheme, null, uri.getHost, uri.getPort, uri.getPath, uri.getQuery, uri.getFragment) new Path(newUri) } /** * Merge two iterators of [[FileStatus]] into a single iterator ordered by file path name. * In case both iterators have [[FileStatus]]s for the same file path, keep the one from * `iterWithPrecedence` and discard that from `iter`. */ private def mergeFileIterators( iter: Iterator[FileStatus], iterWithPrecedence: Iterator[FileStatus]): Iterator[FileStatus] = { (iter.map(f => (f.getPath, f)).toMap ++ iterWithPrecedence.map(f => (f.getPath, f))) .values .toSeq .sortBy(_.getPath.getName) .iterator } /** * List files starting from `resolvedPath` (inclusive) in the same directory. */ private def listFromCache(fs: FileSystem, resolvedPath: Path) = { val pathKey = getPathKey(resolvedPath) writtenPathCache .asMap() .asScala .iterator .filter { case (path, _) => path.getParent == pathKey.getParent() && path.getName >= pathKey.getName } .map { case (path, fileMetadata) => new FileStatus( fileMetadata.length, false, 1, fs.getDefaultBlockSize(path), fileMetadata.modificationTime, path) } } /** * List files starting from `resolvedPath` (inclusive) in the same directory, which merges * the file system list and the cache list when `useCache` is on, otherwise * use file system list only. */ private def listFromInternal(fs: FileSystem, resolvedPath: Path, useCache: Boolean = true) = { val parentPath = resolvedPath.getParent if (!fs.exists(parentPath)) { throw DeltaErrors.fileOrDirectoryNotFoundException(parentPath.toString) } val listedFromFs = fs.listStatus(parentPath).filter(_.getPath.getName >= resolvedPath.getName).iterator val listedFromCache = if (useCache) listFromCache(fs, resolvedPath) else Iterator.empty // File statuses listed from file system take precedence mergeFileIterators(listedFromCache, listedFromFs) } override def listFrom(path: Path): Iterator[FileStatus] = { listFrom(path, getHadoopConfiguration) } /** * List files starting from `resolvedPath` (inclusive) in the same directory. */ override def listFrom(path: Path, hadoopConf: Configuration): Iterator[FileStatus] = { val (fs, resolvedPath) = resolved(path, hadoopConf) listFromInternal(fs, resolvedPath) } /** * Check if the path is an initial version of a Delta log. */ private def isInitialVersion(path: Path): Boolean = { FileNames.isDeltaFile(path) && FileNames.deltaVersion(path) == 0L } /** * Check if a path exists. Normally we check both the file system and the cache, but when the * path is the first version of a Delta log, we ignore the cache. */ private def exists(fs: FileSystem, resolvedPath: Path): Boolean = { // Ignore the cache for the first file of a Delta log listFromInternal(fs, resolvedPath, useCache = !isInitialVersion(resolvedPath)) .take(1) .exists(_.getPath.getName == resolvedPath.getName) } override def write(path: Path, actions: Iterator[String], overwrite: Boolean = false): Unit = { write(path, actions, overwrite, getHadoopConfiguration) } override def write( path: Path, actions: Iterator[String], overwrite: Boolean, hadoopConf: Configuration): Unit = { val (fs, resolvedPath) = resolved(path, hadoopConf) val lockedPath = getPathKey(resolvedPath) acquirePathLock(lockedPath) try { if (exists(fs, resolvedPath) && !overwrite) { throw new java.nio.file.FileAlreadyExistsException(resolvedPath.toUri.toString) } val stream = new CountingOutputStream(fs.create(resolvedPath, overwrite)) actions.map(_ + "\n").map(_.getBytes(UTF_8)).foreach(stream.write) stream.close() // When a Delta log starts afresh, all cached files in that Delta log become obsolete, // so we remove them from the cache. if (isInitialVersion(resolvedPath)) { val obsoleteFiles = writtenPathCache .asMap() .asScala .keys .filter(_.getParent == lockedPath.getParent()) .asJava writtenPathCache.invalidateAll(obsoleteFiles) } // Cache the information of written files to help fix the inconsistency in future listings writtenPathCache.put(lockedPath, FileMetadata(stream.getCount(), System.currentTimeMillis())) } catch { // Convert Hadoop's FileAlreadyExistsException to Java's FileAlreadyExistsException case e: org.apache.hadoop.fs.FileAlreadyExistsException => val converted = new java.nio.file.FileAlreadyExistsException(e.getMessage) converted.initCause(e) throw converted } finally { releasePathLock(lockedPath) } } override def isPartialWriteVisible(path: Path): Boolean = false override def isPartialWriteVisible(path: Path, hadoopConf: Configuration): Boolean = false override def invalidateCache(): Unit = { writtenPathCache.invalidateAll() } } object S3SingleDriverLogStore { /** * A global path lock to ensure that no concurrent writers writing to the same path in the same * JVM. */ private val pathLock = new ConcurrentHashMap[Path, AnyRef]() /** * A global cache that records the metadata of the files recently written. * As list-after-write may be inconsistent on S3, we can use the files in the cache * to fix the inconsistent file listing. */ private val writtenPathCache = CacheBuilder.newBuilder() .expireAfterAccess(120, TimeUnit.MINUTES) .build[Path, FileMetadata]() /** * Release the lock for the path after writing. * * Note: the caller should resolve the path to make sure we are locking the correct absolute path. */ private def releasePathLock(resolvedPath: Path): Unit = { val lock = pathLock.remove(resolvedPath) lock.synchronized { lock.notifyAll() } } /** * Acquire a lock for the path before writing. * * Note: the caller should resolve the path to make sure we are locking the correct absolute path. */ private def acquirePathLock(resolvedPath: Path): Unit = { while (true) { val lock = pathLock.putIfAbsent(resolvedPath, new Object) if (lock == null) return lock.synchronized { while (pathLock.get(resolvedPath) == lock) { lock.wait() } } } } } /** * The file metadata to be stored in the cache. */ case class FileMetadata(length: Long, modificationTime: Long) ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/storage/dv/DeletionVectorStore.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.storage.dv import java.io.{Closeable, DataInputStream} import java.net.URI import java.nio.charset.StandardCharsets.UTF_8 import java.util.UUID import java.util.zip.CRC32 import org.apache.spark.sql.delta.DeltaErrors import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.deletionvectors.{RoaringBitmapArray, StoredBitmap} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.util.PathWithFileSystem import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, FSDataOutputStream, Path} import org.apache.spark.paths.SparkPath import org.apache.spark.util.Utils trait DeletionVectorStore extends DeltaLogging { /** * Read a Deletion Vector and parse it as [[RoaringBitmapArray]]. */ def read( dvDescriptor: DeletionVectorDescriptor, tablePath: Path): RoaringBitmapArray = StoredBitmap.create(dvDescriptor, tablePath).load(this) /** * Read Deletion Vector and parse it as [[RoaringBitmapArray]]. */ def read(path: Path, offset: Int, size: Int): RoaringBitmapArray /** * Returns a writer that can be used to write multiple deletion vectors to the file at `path`. */ def createWriter(path: PathWithFileSystem): DeletionVectorStore.Writer /** * Returns full path for a DV with `filedId` UUID under `targetPath`. * * Optionally, prepend a `prefix` to the name. */ def generateFileNameInTable( targetPath: PathWithFileSystem, fileId: UUID, prefix: String = ""): PathWithFileSystem = { DeletionVectorStore.assembleDeletionVectorPathWithFileSystem(targetPath, fileId, prefix) } /** * Return a new unique path under `targetPath`. * * Optionally, prepend a `prefix` to the name. */ def generateUniqueNameInTable( targetPath: PathWithFileSystem, prefix: String = ""): PathWithFileSystem = generateFileNameInTable(targetPath, UUID.randomUUID(), prefix) /** * Creates a [[PathWithFileSystem]] instance * by using the configuration of this `DeletionVectorStore` instance */ def pathWithFileSystem(path: Path): PathWithFileSystem } /** * Trait containing the utility and constants needed for [[DeletionVectorStore]] */ trait DeletionVectorStoreUtils { final val DV_FILE_FORMAT_VERSION_ID_V1: Byte = 1 /** The length of a DV checksum. See [[calculateChecksum()]]. */ final val CHECKSUM_LEN = 4 /** The size of the stored length of a DV. */ final val DATA_SIZE_LEN = 4 // DV Format: def getTotalSizeOfDVFieldsInFile(bitmapDataSize: Int): Int = { DATA_SIZE_LEN + bitmapDataSize + CHECKSUM_LEN } /** Convert the given String path to a Hadoop Path. Please make sure the path is not escaped. */ def unescapedStringToPath(path: String): Path = SparkPath.fromPathString(path).toPath /** Convert the given String path to a Hadoop Path, Please make sure the path is escaped. */ def escapedStringToPath(path: String): Path = SparkPath.fromUrlString(path).toPath /** Convert the given Hadoop path to a String Path, handing special characters properly. */ def pathToEscapedString(path: Path): String = SparkPath.fromPath(path).urlEncoded /** * Calculate checksum of a serialized deletion vector. We are using CRC32 which has 4bytes size, * but CRC32 implementation conforms to Java Checksum interface which requires a long. However, * the high-order bytes are zero, so here is safe to cast to Int. This will result in negative * checksums, but this is not a problem because we only care about equality. */ def calculateChecksum(data: Array[Byte]): Int = { val crc = new CRC32() crc.update(data) crc.getValue.toInt } /** * Read a serialized deletion vector from a data stream. */ def readRangeFromStream(reader: DataInputStream, size: Int): Array[Byte] = { val sizeAccordingToFile = reader.readInt() if (size != sizeAccordingToFile) { throw DeltaErrors.deletionVectorSizeMismatch() } val buffer = new Array[Byte](size) reader.readFully(buffer) val expectedChecksum = reader.readInt() val actualChecksum = calculateChecksum(buffer) if (expectedChecksum != actualChecksum) { throw DeltaErrors.deletionVectorChecksumMismatch() } buffer } /** * Same as `assembleDeletionVectorPath`, but keeps the new path bundled with the fs. */ def assembleDeletionVectorPathWithFileSystem( targetParentPathWithFileSystem: PathWithFileSystem, id: UUID, prefix: String = ""): PathWithFileSystem = { targetParentPathWithFileSystem.copy(path = DeletionVectorDescriptor.assembleDeletionVectorPath( targetParentPathWithFileSystem.path, id, prefix)) } /** Descriptor for a serialized Deletion Vector in a file. */ case class DVRangeDescriptor(offset: Int, length: Int, checksum: Int) trait Writer extends Closeable { /** * Appends the serialized deletion vector in `data` to the file, and returns the offset in the * file that the deletion vector was written to and its checksum. */ def write(data: Array[Byte]): DVRangeDescriptor /** * Returns UTF-8 encoded path of the file that is being written by this writer. */ def serializedPath: Array[Byte] /** * Closes this writer. After calling this method it is no longer valid to call write (or close). * This method must always be called when the owner of this writer is done writing deletion * vectors. */ def close(): Unit } } object DeletionVectorStore extends DeletionVectorStoreUtils { /** Create a new instance of [[DeletionVectorStore]] from the given Hadoop configuration. */ private[delta] def createInstance( hadoopConf: Configuration): DeletionVectorStore = new HadoopFileSystemDVStore(hadoopConf) } /** * Default [[DeletionVectorStore]] implementation for Hadoop [[FileSystem]] implementations. * * Note: This class must be thread-safe, * because we sometimes write multiple deletion vectors in parallel through the same store. */ class HadoopFileSystemDVStore(hadoopConf: Configuration) extends DeletionVectorStore { override def read(path: Path, offset: Int, size: Int): RoaringBitmapArray = { val fs = path.getFileSystem(hadoopConf) val buffer = Utils.tryWithResource(fs.open(path)) { reader => reader.seek(offset) DeletionVectorStore.readRangeFromStream(reader, size) } DeletionVectorUtils.deserialize( buffer, debugInfo = Map("path" -> path, "offset" -> offset, "size" -> size)) } override def createWriter(path: PathWithFileSystem): DeletionVectorStore.Writer = { new DeletionVectorStore.Writer { // Lazily create the writer for the deletion vectors, so that we don't write an empty file // in case all deletion vectors are empty. private var outputStream: FSDataOutputStream = _ private var writtenBytes = 0L override def write(data: Array[Byte]): DeletionVectorStore.DVRangeDescriptor = { if (outputStream == null) { val overwrite = false // `create` Java API does not support named parameters outputStream = path.fs.create(path.path, overwrite) outputStream.writeByte(DeletionVectorStore.DV_FILE_FORMAT_VERSION_ID_V1) writtenBytes += 1 } val dvRange = DeletionVectorStore.DVRangeDescriptor( offset = outputStream.size(), length = data.length, checksum = DeletionVectorStore.calculateChecksum(data)) if (writtenBytes != dvRange.offset) { deltaAssert( writtenBytes == dvRange.offset, name = "dv.write.offsetMismatch", msg = s"Offset mismatch while writing deletion vector to file", data = Map( "path" -> path.path.toString, "reportedOffset" -> dvRange.offset, "calculatedOffset" -> writtenBytes) ) throw DeltaErrors.deletionVectorSizeMismatch() } log.debug(s"Writing DV range to file: Path=${path.path}, Range=${dvRange}") outputStream.writeInt(data.length) outputStream.write(data) outputStream.writeInt(dvRange.checksum) writtenBytes += DeletionVectorStore.getTotalSizeOfDVFieldsInFile(data.length) dvRange } override val serializedPath: Array[Byte] = DeletionVectorStore.pathToEscapedString(path.path).getBytes(UTF_8) override def close(): Unit = { if (outputStream != null) { outputStream.close() } } } } override def pathWithFileSystem(path: Path): PathWithFileSystem = PathWithFileSystem.withConf(path, hadoopConf) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/streaming/SchemaTrackingLog.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.streaming import java.io.{InputStream, OutputStream} import java.nio.charset.StandardCharsets._ import scala.io.{Source => IOSource} import scala.reflect.ClassTag import org.apache.spark.sql.delta.Relocated._ import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.util.JsonUtils import com.fasterxml.jackson.annotation.JsonIgnore import org.apache.spark.internal.{Logging, MDC} import org.apache.spark.sql.SparkSession import org.apache.spark.sql.execution.streaming.HDFSMetadataLog import org.apache.spark.sql.types.{DataType, StructType} /** * A serializable schema with a partition schema and a data schema. */ trait PartitionAndDataSchema { @JsonIgnore def dataSchema: DataType @JsonIgnore def partitionSchema: StructType } /** * A schema serializer handles the SerDe of a [[PartitionAndDataSchema]] */ sealed trait SchemaSerializer[T <: PartitionAndDataSchema] { def serdeVersion: Int def serialize(schema: T, outputStream: OutputStream): Unit def deserialize(in: InputStream): T } /** *A schema serializer that reads/writes schema using the following format: * {SERDE_VERSION} * {JSON of the serializable schema} */ class JsonSchemaSerializer[T <: PartitionAndDataSchema: ClassTag: Manifest] (override val serdeVersion: Int) extends SchemaSerializer[T] { import SchemaTrackingExceptions._ val EMPTY_JSON = "{}" /** * Deserializes the log entry from input stream. * @throws FailedToDeserializeException */ override def deserialize(in: InputStream): T = { // Called inside a try-finally where the underlying stream is closed in the caller val lines = IOSource.fromInputStream(in, UTF_8.name()).getLines() if (!lines.hasNext) { throw FailedToDeserializeException } MetadataVersionUtil.validateVersion(lines.next(), serdeVersion) val schemaJson = if (lines.hasNext) lines.next() else EMPTY_JSON JsonUtils.fromJson(schemaJson) } override def serialize(metadata: T, out: OutputStream): Unit = { // Called inside a try-finally where the underlying stream is closed in the caller out.write(s"v${serdeVersion}".getBytes(UTF_8)) out.write('\n') // Write metadata out.write(JsonUtils.toJson(metadata).getBytes(UTF_8)) } } /** * The underlying class for a streaming log that keeps track of a sequence of schema changes. * * It keeps tracks of the sequence of schema changes that this log is aware of, and it detects any * concurrent modifications to the schema log to prevent accidents on a best effort basis. */ class SchemaTrackingLog[T <: PartitionAndDataSchema: ClassTag: Manifest]( sparkSession: SparkSession, path: String, schemaSerializer: SchemaSerializer[T]) extends HDFSMetadataLog[T](sparkSession, path) with Logging { import SchemaTrackingExceptions._ // The schema and version detected when this log is initialized private val schemaAndSeqNumAtLogInit: Option[(Long, T)] = getLatest() // Next schema version to write, this should be updated after each schema evolution. // This allow HDFSMetadataLog to best detect concurrent schema log updates. private var currentSeqNum: Long = schemaAndSeqNumAtLogInit.map(_._1).getOrElse(-1L) private var nextSeqNumToWrite: Long = currentSeqNum + 1 // The current persisted schema this log has been tracking. Note that this does NOT necessarily // always equal to the globally latest schema. Attempting to commit to a schema version that // already exists is illegal. // Subclass can leverage this to compare the differences. private var currentTrackedSchema: Option[T] = schemaAndSeqNumAtLogInit.map(_._2) /** * Get the latest tracked schema entry by this schema log */ def getCurrentTrackedSchema: Option[T] = currentTrackedSchema /** * Get the latest tracked schema batch ID / seq num by this log */ def getCurrentTrackedSeqNum: Long = currentSeqNum /** * Get the tracked schema at specified seq num. */ def getTrackedSchemaAtSeqNum(seqNum: Long): Option[T] = get(seqNum) /** * Deserializes the log entry from input stream. * @throws FailedToDeserializeException */ override protected def deserialize(in: InputStream): T = schemaSerializer.deserialize(in).asInstanceOf[T] override protected def serialize(metadata: T, out: OutputStream): Unit = schemaSerializer.serialize(metadata, out) /** * Main API to actually write the log entry to the schema log. Clients can leverage this * to save their new schema to the log. * @throws FailedToEvolveSchema * @param newSchema New persisted schema */ def addSchemaToLog(newSchema: T): T = { // Write to schema log logInfo(log"Writing a new metadata version " + log"${MDC(DeltaLogKeys.VERSION, nextSeqNumToWrite)} in the metadata log") if (currentTrackedSchema.contains(newSchema)) { // Record a warning if schema has not changed logWarning(log"Schema didn't change after schema evolution. " + log"currentSchema = ${MDC(DeltaLogKeys.SCHEMA, currentTrackedSchema)}.") return newSchema } // Similar to how MicrobatchExecution detects concurrent checkpoint updates if (!add(nextSeqNumToWrite, newSchema)) { throw FailedToEvolveSchema } currentTrackedSchema = Some(newSchema) currentSeqNum = nextSeqNumToWrite nextSeqNumToWrite += 1 newSchema } } object SchemaTrackingExceptions { // Designated exceptions val FailedToDeserializeException = new RuntimeException("Failed to deserialize schema log") val FailedToEvolveSchema = new RuntimeException("Failed to add schema entry to log. Concurrent operations detected.") } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/tablefeatures/tableChanges.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.tablefeatures import org.apache.spark.sql.connector.catalog.TableChange /** * Change to remove a feature from a table. * @param featureName The name of the feature * @param truncateHistory When true we set the minimum log retention period and clean up metadata. */ case class DropFeature(featureName: String, truncateHistory: Boolean) extends TableChange {} ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/uniform/ParquetIcebergCompatV2Utils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.uniform import org.apache.parquet.format.converter.ParquetMetadataConverter import org.apache.parquet.hadoop.metadata.ParquetMetadata import org.apache.spark.sql.execution.datasources.parquet.ParquetFooterReaderShims /** * Contains utilities to check whether a specific parquet data file * is considered `IcebergCompatV2`. * See [[isParquetIcebergV2Compatible]] for details. */ object ParquetIcebergCompatV2Utils { // TIMESTAMP stored as INT96 is NOT considered `IcebergCompatV2`. // NOTE: `TIMESTAMP <-> INT96` is an exact *one-to-one* mapping // in default and `IcebergCompatV1` delta table. // this means we could confidently claim an `INT96` must be `TIMESTAMP` // if found in the parquet footer schema field. private val TIMESTAMP_AS_INT96 = org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT96 /** * Recursively traverse a specific parquet schema field, * check the following properties, i.e., * - for primitive type, * - check if TIMESTAMP is stored as INT96. * - check for the `field_id`. * - for group type, * - check for the `field_id`. * - iterate through all fields and check each field recursively in the same way. * * @param field the field to check, this corresponds to a specific parquet file. * @return whether the parquet field contains TIMESTAMP stored as INT96 or * lacking `field_id` for any (nested) field or not; * if so, return true; otherwise return false. */ private def hasTimestampAsInt96OrFieldIdNotExistForType( field: org.apache.parquet.schema.Type): Boolean = field match { // note: `getId` returns null indicates the field does not contain `field_id`. case p: org.apache.parquet.schema.PrimitiveType => (p.getPrimitiveTypeName == TIMESTAMP_AS_INT96) || (p.getId == null) case g: org.apache.parquet.schema.GroupType => if (g.getId != null) { val logicalAnnotation = g.getLogicalTypeAnnotation val fields = if (logicalAnnotation != null && (logicalAnnotation.toString == "LIST" || logicalAnnotation.toString == "MAP")) { // according to parquet's spec, // - for LIST: // - the outer-most level must be a group annotated with LIST // that contains a **single** field named list. // - for MAP: // - the outer-most level must be a group annotated with MAP // that contains a **single** field named key_value. // details could be found at // [[https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists]] and // [[https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#maps]] and g.getFields.get(0).asGroupType().getFields } else { g.getFields } fields.toArray.exists { case field: org.apache.parquet.schema.Type => hasTimestampAsInt96OrFieldIdNotExistForType(field) } } else { true } } /** * Check if the parquet file is `IcebergCompatV2` by inspecting the * provided parquet footer. * * note: icebergV2-compatible check refer to the following two properties. * 1. TIMESTAMP * - If TIMESTAMP is stored as `int96`, it's considered incompatible since * iceberg stores TIMESTAMP as `int64` according to the iceberg spec. * 2. field_id * - `field_id` is needed for *every* field in a parquet footer, this includes * the field of each column, and the potential nested fields for nested types * like LIST, MAP or STRUCT. * See [[https://github.com/apache/parquet-format/blob/master/LogicalTypes.md]] for details. * - This is checked by inspecting whether the `field_id` for each column * is null or not recursively. * * @param footer the parquet footer to be checked. * @return whether the parquet file is considered `IcebergCompatV2`. */ def isParquetIcebergCompatV2(footer: ParquetMetadata): Boolean = { // iterate through each column/field and check if there exists // any column/field that contains TIMESTAMP stored as INT96, // or lacking any `field_id` (included nested one as in LIST or MAP). !footer.getFileMetaData.getSchema.getFields.toArray.exists { case field: org.apache.parquet.schema.Type => hasTimestampAsInt96OrFieldIdNotExistForType(field) } } /** * Get the parquet footer based on the input `parquetPath`. * * @param parquetPath the absolute path to the parquet file. * @return the corresponding parquet metadata/footer. */ def getParquetFooter(parquetPath: String): ParquetMetadata = { val path = new org.apache.hadoop.fs.Path(parquetPath) val conf = new org.apache.hadoop.conf.Configuration val fs = path.getFileSystem(conf) val status = fs.getFileStatus(path) ParquetFooterReaderShims.readParquetFooter(conf, status, ParquetMetadataConverter.NO_FILTER) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/AnalysisHelper.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.{DataFrameUtils, DeltaAnalysisException, DeltaErrors} import org.apache.spark.sql.{AnalysisException, Dataset, Row, SparkSession} import org.apache.spark.sql.catalyst.ExtendedAnalysisException import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan trait AnalysisHelper { import AnalysisHelper._ // Keeping the following two methods for backward compatibility with previous Delta versions. protected def tryResolveReferences( sparkSession: SparkSession)( expr: Expression, planContainingExpr: LogicalPlan): Expression = tryResolveReferencesForExpressions(sparkSession)(Seq(expr), planContainingExpr.children).head protected def tryResolveReferencesForExpressions( sparkSession: SparkSession, exprs: Seq[Expression], planContainingExpr: LogicalPlan): Seq[Expression] = tryResolveReferencesForExpressions(sparkSession)(exprs, planContainingExpr.children) /** * Resolve expressions using the attributes provided by `planProvidingAttrs`. Throw an error if * failing to resolve any expressions. */ protected def resolveReferencesForExpressions( sparkSession: SparkSession, exprs: Seq[Expression], planProvidingAttrs: LogicalPlan): Seq[Expression] = { val resolvedExprs = tryResolveReferencesForExpressions(sparkSession)(exprs, Seq(planProvidingAttrs)) resolvedExprs.foreach { expr => if (!expr.resolved) { throw new ExtendedAnalysisException( new DeltaAnalysisException( errorClass = "_LEGACY_ERROR_TEMP_DELTA_0012", messageParameters = Array(expr.toString) ), planProvidingAttrs ) } } resolvedExprs } /** * Resolve expressions using the attributes provided by `planProvidingAttrs`, ignoring errors. */ protected def tryResolveReferencesForExpressions( sparkSession: SparkSession)( exprs: Seq[Expression], plansProvidingAttrs: Seq[LogicalPlan]): Seq[Expression] = { val newPlan = FakeLogicalPlan(exprs, plansProvidingAttrs) sparkSession.sessionState.analyzer.execute(newPlan) match { case FakeLogicalPlan(resolvedExprs, _) => // Return even if it did not successfully resolve resolvedExprs case _ => // This is unexpected throw new ExtendedAnalysisException( new DeltaAnalysisException( errorClass = "_LEGACY_ERROR_TEMP_DELTA_0012", messageParameters = Array(exprs.mkString(",")) ), newPlan ) } } protected def toDataset(sparkSession: SparkSession, logicalPlan: LogicalPlan): Dataset[Row] = { DataFrameUtils.ofRows(sparkSession, logicalPlan) } protected def improveUnsupportedOpError[T](f: => T): T = { val possibleErrorMsgs = Seq( "is only supported with v2 table", // full error: DELETE is only supported with v2 tables "is not supported temporarily", // full error: UPDATE TABLE is not supported temporarily "Table does not support read", "Table implementation does not support writes" ).map(_.toLowerCase()) def isExtensionOrCatalogError(error: Exception): Boolean = { possibleErrorMsgs.exists { m => error.getMessage != null && error.getMessage.toLowerCase().contains(m) } } try { f } catch { case e: Exception if isExtensionOrCatalogError(e) => throw DeltaErrors.configureSparkSessionWithExtensionAndCatalog(Some(e)) } } } object AnalysisHelper { /** LogicalPlan to help resolve the given expression */ case class FakeLogicalPlan( exprs: Seq[Expression], children: Seq[LogicalPlan]) extends LogicalPlan { override def output: Seq[Attribute] = Nil override protected def withNewChildrenInternal( newChildren: IndexedSeq[LogicalPlan]): FakeLogicalPlan = copy(children = newChildren) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/BinPackingIterator.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import scala.collection.generic.Sizing import scala.collection.mutable.ArrayBuffer /** * Iterator that packs objects in `inputIter` to create bins that have a total size of * 'targetSize'. Each [[T]] object may contain multiple inputs that are always packed into a * single bin. [[T]] instances must inherit from [[Sizing]] and define what is their size. */ class BinPackingIterator[T <: Sizing]( inputIter: Iterator[T], targetSize: Long) extends Iterator[Seq[T]] { private val currentBin = new ArrayBuffer[T]() private var sizeOfCurrentBin = 0L override def hasNext: Boolean = inputIter.hasNext || currentBin.nonEmpty override def next(): Seq[T] = { var resultBin: Seq[T] = null while (inputIter.hasNext && resultBin == null) { val input = inputIter.next() val sizeOfCurrentFile = input.size // Start a new bin if the deletion vectors for the current Parquet file corresponding to // `row` causes us to go over the target file size. if (currentBin.nonEmpty && sizeOfCurrentBin + sizeOfCurrentFile > targetSize) { resultBin = currentBin.toVector sizeOfCurrentBin = 0L currentBin.clear() } currentBin += input sizeOfCurrentBin += sizeOfCurrentFile } // Finish the last bin. if (resultBin == null && !inputIter.hasNext) { resultBin = currentBin.toVector currentBin.clear() } resultBin } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/BinPackingUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import scala.collection.mutable.ArrayBuffer object BinPackingUtils { /** * Takes a sequence of items and groups them such that the size of each group is * less than the specified maxBinSize. */ @inline def binPackBySize[I, V]( elements: Seq[I], sizeGetter: I => Long, valueGetter: I => V, maxBinSize: Long): Seq[Seq[V]] = { val bins = new ArrayBuffer[Seq[V]]() val currentBin = new ArrayBuffer[V]() var currentSize = 0L elements.sortBy(sizeGetter).foreach { element => val size = sizeGetter(element) // Generally, a bin is a group of existing files, whose total size does not exceed the // desired maxFileSize. They will be coalesced into a single output file. if ((currentSize >= maxBinSize) || size + currentSize > maxBinSize) { if (currentBin.nonEmpty) { bins += currentBin.toVector currentBin.clear() } currentBin += valueGetter(element) currentSize = size } else { currentBin += valueGetter(element) currentSize += size } } if (currentBin.nonEmpty) { bins += currentBin.toVector } bins.toSeq } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/Codec.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import java.nio.ByteBuffer import java.nio.charset.StandardCharsets.US_ASCII import java.util.UUID import com.google.common.primitives.UnsignedInteger /** Additional codecs not supported by Apache Commons Codecs. */ object Codec { def uuidToBytes(id: UUID): Array[Byte] = uuidToByteBuffer(id).array() def uuidFromBytes(bytes: Array[Byte]): UUID = { require(bytes.length == 16) uuidFromByteBuffer(ByteBuffer.wrap(bytes)) } def uuidToByteBuffer(id: UUID): ByteBuffer = { val buffer = ByteBuffer.allocate(16) buffer.putLong(id.getMostSignificantBits) buffer.putLong(id.getLeastSignificantBits) buffer.rewind() buffer } def uuidFromByteBuffer(buffer: ByteBuffer): UUID = { require(buffer.remaining() >= 16) val highBits = buffer.getLong val lowBits = buffer.getLong new UUID(highBits, lowBits) } /** * This implements Base85 using the 4 byte block aligned encoding and character set from Z85. * * @see https://rfc.zeromq.org/spec/32/ */ object Base85Codec { final val ENCODE_MAP: Array[Byte] = { val chars = ('0' to '9') ++ ('a' to 'z') ++ ('A' to 'Z') ++ ".-:+=^!/*?&<>()[]{}@%$#" chars.map(_.toByte).toArray } lazy val DECODE_MAP: Array[Byte] = { require(ENCODE_MAP.length - 1 <= Byte.MaxValue) // The bitmask is the same as largest possible value, so the length of the array must // be one greater. val map: Array[Byte] = Array.fill(ASCII_BITMASK + 1)(-1) for ((b, i) <- ENCODE_MAP.zipWithIndex) { map(b) = i.toByte } map } final val BASE: Long = 85L final val BASE_2ND_POWER: Long = 7225L // 85^2 final val BASE_3RD_POWER: Long = 614125L // 85^3 final val BASE_4TH_POWER: Long = 52200625L // 85^4 final val ASCII_BITMASK: Int = 0x7F // UUIDs always encode into 20 characters. final val ENCODED_UUID_LENGTH: Int = 20 /** Encode a 16 byte UUID. */ def encodeUUID(id: UUID): String = { val buffer = uuidToByteBuffer(id) encodeBlocks(buffer) } /** * Decode a 16 byte UUID. */ def decodeUUID(encoded: String): UUID = { val buffer = decodeBlocks(encoded) uuidFromByteBuffer(buffer) } /** * Encode an arbitrary byte array. * * Unaligned input will be padded to a multiple of 4 bytes. */ def encodeBytes(input: Array[Byte]): String = { if (input.length % 4 == 0) { encodeBlocks(ByteBuffer.wrap(input)) } else { val alignedLength = ((input.length + 4) / 4) * 4 val buffer = ByteBuffer.allocate(alignedLength) buffer.put(input) while (buffer.hasRemaining) { buffer.put(0.asInstanceOf[Byte]) } buffer.rewind() encodeBlocks(buffer) } } /** * Encode an arbitrary byte array using 4 byte blocks. * * Expects the input to be 4 byte aligned. */ private def encodeBlocks(buffer: ByteBuffer): String = { require(buffer.remaining() % 4 == 0) val numBlocks = buffer.remaining() / 4 // Every 4 byte block gets encoded into 5 bytes/chars val outputLength = numBlocks * 5 val output: Array[Byte] = Array.ofDim(outputLength) var outputIndex = 0 while (buffer.hasRemaining) { var sum: Long = buffer.getInt & 0x00000000ffffffffL output(outputIndex) = ENCODE_MAP((sum / BASE_4TH_POWER).toInt) sum %= BASE_4TH_POWER output(outputIndex + 1) = ENCODE_MAP((sum / BASE_3RD_POWER).toInt) sum %= BASE_3RD_POWER output(outputIndex + 2) = ENCODE_MAP((sum / BASE_2ND_POWER).toInt) sum %= BASE_2ND_POWER output(outputIndex + 3) = ENCODE_MAP((sum / BASE).toInt) output(outputIndex + 4) = ENCODE_MAP((sum % BASE).toInt) outputIndex += 5 } new String(output, US_ASCII) } /** * Decode an arbitrary byte array. * * Only `outputLength` bytes will be returned. * Any extra bytes, such as padding added because the input was unaligned, will be dropped. */ def decodeBytes(encoded: String, outputLength: Int): Array[Byte] = { val result = decodeBlocks(encoded) if (result.remaining() > outputLength) { // Only read the expected number of bytes. val output: Array[Byte] = Array.ofDim(outputLength) result.get(output) output } else { result.array() } } /** * Decode an arbitrary byte array. * * Output may contain padding bytes, if the input was not 4 byte aligned. * Use [[decodeBytes]] in that case and specify the expected number of output bytes * without padding. */ def decodeAlignedBytes(encoded: String): Array[Byte] = decodeBlocks(encoded).array() /** * Decode an arbitrary byte array. * * Output may contain padding bytes, if the input was not 4 byte aligned. */ private def decodeBlocks(encoded: String): ByteBuffer = { val input = encoded.toCharArray require(input.length % 5 == 0, "Input should be 5 character aligned.") val buffer = ByteBuffer.allocate(input.length / 5 * 4) // A mechanism to detect invalid characters in the input while decoding, that only has a // single conditional at the very end, instead of branching for every character. var canary: Int = 0 def decodeInputChar(i: Int): Long = { val c = input(i) canary |= c // non-ascii char has bits outside of ASCII_BITMASK val b = DECODE_MAP(c & ASCII_BITMASK) canary |= b // invalid char maps to -1, which has bits outside ASCII_BITMASK b.toLong } var inputIndex = 0 while (buffer.hasRemaining) { var sum = 0L sum += decodeInputChar(inputIndex) * BASE_4TH_POWER sum += decodeInputChar(inputIndex + 1) * BASE_3RD_POWER sum += decodeInputChar(inputIndex + 2) * BASE_2ND_POWER sum += decodeInputChar(inputIndex + 3) * BASE sum += decodeInputChar(inputIndex + 4) buffer.putInt(sum.toInt) inputIndex += 5 } require((canary & ~ASCII_BITMASK) == 0, s"Input is not valid Z85: $encoded") buffer.rewind() buffer } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/DatasetRefCache.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util // scalastyle:off import.ordering.noEmptyLine import java.util.concurrent.atomic.AtomicReference import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession} /** * A [[Dataset]] reference cache to automatically create new [[Dataset]] objects when the active * [[SparkSession]] changes. This is useful when sharing objects holding [[Dataset]] references * cross multiple sessions. Without this, using a [[Dataset]] that holds a stale session may change * the active session and cause multiple issues (e.g., if we switch to a stale session coming from a * notebook that has been detached, we may not be able to use built-in functions because those are * cleaned up). * * The `creator` function will be called to create a new [[Dataset]] object when the old one has a * different session than the current active session. Note that one MUST use SparkSession.active * in the creator() if creator() needs to use Spark session. * * Unlike [[StateCache]], this class only caches the [[Dataset]] reference and doesn't cache the * underlying `RDD`. * * WARNING: If there are many concurrent Spark sessions and each session calls 'get' multiple times, * then the cost of creator becomes more noticeable as everytime it switch the active * session, the older session needs to call creator again when it becomes active. * * @param creator a function to create [[Dataset]]. */ class DatasetRefCache[T] private[util](creator: () => Dataset[T]) { private val holder = new AtomicReference[Dataset[T]] private[delta] def invalidate() = holder.set(null) def get: Dataset[T] = Option(holder.get()) .filter(_.sparkSession eq SparkSession.active) .getOrElse { val df = creator() holder.set(df) df } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/DateFormatter.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import java.time.{Instant, ZoneId} import java.util.Locale import org.apache.spark.sql.delta.util.DateTimeUtils.instantToDays /** * Forked from [[org.apache.spark.sql.catalyst.util.DateFormatter]] */ sealed trait DateFormatter extends Serializable { def parse(s: String): Int // returns days since epoch def format(days: Int): String } class Iso8601DateFormatter( pattern: String, locale: Locale) extends DateFormatter with DateTimeFormatterHelper { @transient private lazy val formatter = getOrCreateFormatter(pattern, locale) private val UTC = ZoneId.of("UTC") private def toInstant(s: String): Instant = { val temporalAccessor = formatter.parse(s) toInstantWithZoneId(temporalAccessor, UTC) } override def parse(s: String): Int = instantToDays(toInstant(s)) override def format(days: Int): String = { val instant = Instant.ofEpochSecond(days * DateTimeUtils.SECONDS_PER_DAY) formatter.withZone(UTC).format(instant) } } object DateFormatter { val defaultPattern: String = "yyyy-MM-dd" val defaultLocale: Locale = Locale.US def apply(format: String, locale: Locale): DateFormatter = { new Iso8601DateFormatter(format, locale) } def apply(format: String): DateFormatter = apply(format, defaultLocale) def apply(): DateFormatter = apply(defaultPattern) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/DateTimeFormatterHelper.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import java.time._ import java.time.chrono.IsoChronology import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder, ResolverStyle} import java.time.temporal.{ChronoField, TemporalAccessor, TemporalQueries} import java.util.Locale import java.util.concurrent.Callable import org.apache.spark.sql.delta.util.DateTimeFormatterHelper._ import com.google.common.cache.CacheBuilder /** * Forked from [[org.apache.spark.sql.catalyst.util.DateTimeFormatterHelper]] */ trait DateTimeFormatterHelper { protected def toInstantWithZoneId(temporalAccessor: TemporalAccessor, zoneId: ZoneId): Instant = { val localTime = if (temporalAccessor.query(TemporalQueries.localTime) == null) { LocalTime.ofNanoOfDay(0) } else { LocalTime.from(temporalAccessor) } val localDate = LocalDate.from(temporalAccessor) val localDateTime = LocalDateTime.of(localDate, localTime) val zonedDateTime = ZonedDateTime.of(localDateTime, zoneId) Instant.from(zonedDateTime) } // Gets a formatter from the cache or creates new one. The buildFormatter method can be called // a few times with the same parameters in parallel if the cache does not contain values // associated to those parameters. Since the formatter is immutable, it does not matter. // In this way, synchronised is intentionally omitted in this method to make parallel calls // less synchronised. // The Cache.get method is not used here to avoid creation of additional instances of Callable. protected def getOrCreateFormatter(pattern: String, locale: Locale): DateTimeFormatter = { val key = (pattern, locale) cache.get(key, new Callable[DateTimeFormatter] { def call = buildFormatter(pattern, locale) }) } } private object DateTimeFormatterHelper { val cache = CacheBuilder.newBuilder() .maximumSize(128) .build[(String, Locale), DateTimeFormatter]() def createBuilder(): DateTimeFormatterBuilder = { new DateTimeFormatterBuilder().parseCaseInsensitive() } def toFormatter(builder: DateTimeFormatterBuilder, locale: Locale): DateTimeFormatter = { builder .parseDefaulting(ChronoField.ERA, 1) .parseDefaulting(ChronoField.MONTH_OF_YEAR, 1) .parseDefaulting(ChronoField.DAY_OF_MONTH, 1) .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0) .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0) .toFormatter(locale) .withChronology(IsoChronology.INSTANCE) .withResolverStyle(ResolverStyle.STRICT) } def buildFormatter(pattern: String, locale: Locale): DateTimeFormatter = { val builder = createBuilder().appendPattern(pattern) toFormatter(builder, locale) } lazy val fractionFormatter: DateTimeFormatter = { val builder = createBuilder() .append(DateTimeFormatter.ISO_LOCAL_DATE) .appendLiteral(' ') .appendValue(ChronoField.HOUR_OF_DAY, 2).appendLiteral(':') .appendValue(ChronoField.MINUTE_OF_HOUR, 2).appendLiteral(':') .appendValue(ChronoField.SECOND_OF_MINUTE, 2) .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) toFormatter(builder, TimestampFormatter.defaultLocale) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/DateTimeUtils.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import java.sql.Timestamp import java.time._ import java.util.TimeZone import java.util.concurrent.TimeUnit._ /** * Forked from [[org.apache.spark.sql.catalyst.util.DateTimeUtils]]. * Only included the methods that are used by Delta and added after Spark 2.4. */ /** * Helper functions for converting between internal and external date and time representations. * Dates are exposed externally as java.sql.Date and are represented internally as the number of * dates since the Unix epoch (1970-01-01). Timestamps are exposed externally as java.sql.Timestamp * and are stored internally as longs, which are capable of storing timestamps with microsecond * precision. */ object DateTimeUtils { // we use Int and Long internally to represent [[DateType]] and [[TimestampType]] type SQLDate = Int type SQLTimestamp = Long // Pre-calculated values can provide an opportunity of additional optimizations // to the compiler like constants propagation and folding. final val NANOS_PER_MICROS: Long = 1000 final val MICROS_PER_MILLIS: Long = 1000 final val MILLIS_PER_SECOND: Long = 1000 final val SECONDS_PER_DAY: Long = 24 * 60 * 60 final val MICROS_PER_SECOND: Long = MILLIS_PER_SECOND * MICROS_PER_MILLIS final val NANOS_PER_MILLIS: Long = NANOS_PER_MICROS * MICROS_PER_MILLIS final val NANOS_PER_SECOND: Long = NANOS_PER_MICROS * MICROS_PER_SECOND final val MICROS_PER_DAY: Long = SECONDS_PER_DAY * MICROS_PER_SECOND final val MILLIS_PER_MINUTE: Long = 60 * MILLIS_PER_SECOND final val MILLIS_PER_HOUR: Long = 60 * MILLIS_PER_MINUTE final val MILLIS_PER_DAY: Long = SECONDS_PER_DAY * MILLIS_PER_SECOND def defaultTimeZone(): TimeZone = TimeZone.getDefault def getTimeZone(timeZoneId: String): TimeZone = { val zoneId = ZoneId.of(timeZoneId, ZoneId.SHORT_IDS) TimeZone.getTimeZone(zoneId) } // Converts Timestamp to string according to Hive TimestampWritable convention. def timestampToString(tf: TimestampFormatter, us: SQLTimestamp): String = { tf.format(us) } def instantToMicros(instant: Instant): Long = { val us = Math.multiplyExact(instant.getEpochSecond, MICROS_PER_SECOND) val result = Math.addExact(us, NANOSECONDS.toMicros(instant.getNano)) result } def microsToInstant(us: Long): Instant = { val secs = Math.floorDiv(us, MICROS_PER_SECOND) val mos = Math.floorMod(us, MICROS_PER_SECOND) Instant.ofEpochSecond(secs, mos * NANOS_PER_MICROS) } def instantToDays(instant: Instant): Int = { val seconds = instant.getEpochSecond val days = Math.floorDiv(seconds, SECONDS_PER_DAY) days.toInt } /** * Returns the number of micros since epoch from java.sql.Timestamp. */ def fromJavaTimestamp(t: Timestamp): SQLTimestamp = { if (t != null) { MILLISECONDS.toMicros(t.getTime) + NANOSECONDS.toMicros(t.getNanos()) % NANOS_PER_MICROS } else { 0L } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/DeltaCommitFileProvider.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import org.apache.spark.sql.delta.Snapshot import org.apache.spark.sql.delta.util.FileNames._ import org.apache.hadoop.fs.Path /** * Provides access to resolve Delta commit files names based on the commit-version. * * This class is part of the changes introduced to accommodate the adoption of coordinated-commits * in Delta Lake. Previously, certain code paths assumed the existence of delta files for a specific * version at a predictable path `_delta_log/$version.json`. With coordinated-commits, delta files * may alternatively be located at `_delta_log/_staged_commits/$version.$uuid.json`. * DeltaCommitFileProvider attempts to locate the correct delta files from the Snapshot's * LogSegment. * * @param logPath The path to the Delta table log directory. * @param maxVersionInclusive The maximum version of the Delta table (inclusive). * @param uuids A map of version numbers to their corresponding UUIDs. */ case class DeltaCommitFileProvider( logPath: String, maxVersionInclusive: Long, uuids: Map[Long, String]) { // Ensure the Path object is reused across Delta Files but not stored as part of the object state // since it is not serializable. @transient lazy val resolvedPath: Path = new Path(logPath) lazy val minUnbackfilledVersion: Long = if (uuids.keys.isEmpty) { maxVersionInclusive + 1 } else { uuids.keys.min } def deltaFile(version: Long): Path = { if (version > maxVersionInclusive) { throw new IllegalStateException(s"Cannot resolve Delta table at version $version as the " + s"state is currently at version $maxVersionInclusive. The requested version may be " + s"incorrect or the state may be outdated. Please verify the requested version, update " + s"the state if necessary, and try again") } uuids.get(version) match { case Some(uuid) => FileNames.unbackfilledDeltaFile(resolvedPath, version, Some(uuid)) case _ => FileNames.unsafeDeltaFile(resolvedPath, version) } } /** * Lists unbackfilled delta files in a sorted order without incurring additional IO operations. */ def listSortedUnbackfilledDeltaFiles(startVersionOpt: Option[Long] = None): Seq[(Long, Path)] = { val minVersion = startVersionOpt.getOrElse(minUnbackfilledVersion) uuids .toSeq .sortBy(_._1) .collect { case (version, uuid) if version >= minVersion => (version, FileNames.unbackfilledDeltaFile(resolvedPath, version, Some(uuid))) } } } object DeltaCommitFileProvider { def apply(snapshot: Snapshot): DeltaCommitFileProvider = { val uuids = snapshot.logSegment.deltas .collect { case UnbackfilledDeltaFile(_, version, uuid) => version -> uuid } .toMap new DeltaCommitFileProvider(snapshot.path.toString, snapshot.version, uuids) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/DeltaEncoders.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import scala.reflect.runtime.universe.TypeTag import org.apache.spark.sql.delta.{DeltaHistory, DeltaHistoryManager, SerializableFileStatus, SnapshotState} import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.commands.convert.ConvertTargetFile import org.apache.spark.sql.delta.sources.IndexedFile import org.apache.spark.sql.Encoder import org.apache.spark.sql.catalyst.catalog.CatalogTypes import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder private[delta] class DeltaEncoder[T: TypeTag] { private lazy val _encoder = ExpressionEncoder[T]() def get: Encoder[T] = { _encoder.copy() } } /** * Define a few `Encoder`s to reuse in Delta in order to avoid touching Scala reflection after * warming up. This will be mixed into `org.apache.spark.sql.delta.implicits`. Use * `import org.apache.spark.sql.delta.implicits._` to use these `Encoder`s. */ private[delta] trait DeltaEncoders { private lazy val _BooleanEncoder = new DeltaEncoder[Boolean] implicit def booleanEncoder: Encoder[Boolean] = _BooleanEncoder.get private lazy val _IntEncoder = new DeltaEncoder[Int] implicit def intEncoder: Encoder[Int] = _IntEncoder.get private lazy val _longEncoder = new DeltaEncoder[Long] implicit def longEncoder: Encoder[Long] = _longEncoder.get private lazy val _stringEncoder = new DeltaEncoder[String] implicit def stringEncoder: Encoder[String] = _stringEncoder.get private lazy val _longLongEncoder = new DeltaEncoder[(Long, Long)] implicit def longLongEncoder: Encoder[(Long, Long)] = _longLongEncoder.get private lazy val _stringLongEncoder = new DeltaEncoder[(String, Long)] implicit def stringLongEncoder: Encoder[(String, Long)] = _stringLongEncoder.get private lazy val _stringStringEncoder = new DeltaEncoder[(String, String)] implicit def stringStringEncoder: Encoder[(String, String)] = _stringStringEncoder.get private lazy val _javaLongEncoder = new DeltaEncoder[java.lang.Long] implicit def javaLongEncoder: Encoder[java.lang.Long] = _javaLongEncoder.get private lazy val _singleActionEncoder = new DeltaEncoder[SingleAction] implicit def singleActionEncoder: Encoder[SingleAction] = _singleActionEncoder.get private lazy val _addFileEncoder = new DeltaEncoder[AddFile] implicit def addFileEncoder: Encoder[AddFile] = _addFileEncoder.get private lazy val _removeFileEncoder = new DeltaEncoder[RemoveFile] implicit def removeFileEncoder: Encoder[RemoveFile] = _removeFileEncoder.get private lazy val _addCdcFileEncoder = new DeltaEncoder[AddCDCFile] implicit def addCdcFileEncoder: Encoder[AddCDCFile] = _addCdcFileEncoder.get private lazy val _pmtvEncoder = new DeltaEncoder[(Protocol, Metadata, Option[Long], Long)] implicit def pmtvEncoder: Encoder[(Protocol, Metadata, Option[Long], Long)] = _pmtvEncoder.get private lazy val _v2CheckpointActionsEncoder = new DeltaEncoder[(CheckpointMetadata, SidecarFile)] implicit def v2CheckpointActionsEncoder: Encoder[(CheckpointMetadata, SidecarFile)] = _v2CheckpointActionsEncoder.get private lazy val _serializableFileStatusEncoder = new DeltaEncoder[SerializableFileStatus] implicit def serializableFileStatusEncoder: Encoder[SerializableFileStatus] = _serializableFileStatusEncoder.get private lazy val _indexedFileEncoder = new DeltaEncoder[IndexedFile] implicit def indexedFileEncoder: Encoder[IndexedFile] = _indexedFileEncoder.get private lazy val _addFileWithIndexEncoder = new DeltaEncoder[(AddFile, Long)] implicit def addFileWithIndexEncoder: Encoder[(AddFile, Long)] = _addFileWithIndexEncoder.get private lazy val _addFileWithSourcePathEncoder = new DeltaEncoder[(AddFile, String)] implicit def addFileWithSourcePathEncoder: Encoder[(AddFile, String)] = _addFileWithSourcePathEncoder.get private lazy val _deltaHistoryEncoder = new DeltaEncoder[DeltaHistory] implicit def deltaHistoryEncoder: Encoder[DeltaHistory] = _deltaHistoryEncoder.get private lazy val _historyCommitEncoder = new DeltaEncoder[DeltaHistoryManager.Commit] implicit def historyCommitEncoder: Encoder[DeltaHistoryManager.Commit] = _historyCommitEncoder.get private lazy val _snapshotStateEncoder = new DeltaEncoder[SnapshotState] implicit def snapshotStateEncoder: Encoder[SnapshotState] = _snapshotStateEncoder.get private lazy val _convertTargetFileEncoder = new DeltaEncoder[ConvertTargetFile] implicit def convertTargetFileEncoder: Encoder[ConvertTargetFile] = _convertTargetFileEncoder.get private lazy val _fsPartitionSpecEncoder = new DeltaEncoder[(SerializableFileStatus, CatalogTypes.TablePartitionSpec)] implicit def fsPartitionSpecEncoder : Encoder[(SerializableFileStatus, CatalogTypes.TablePartitionSpec)] = _fsPartitionSpecEncoder.get private lazy val _optionalHistoryCommitEncoder = new DeltaEncoder[Option[DeltaHistoryManager.Commit]] implicit def optionalHistoryCommitEncoder: Encoder[Option[DeltaHistoryManager.Commit]] = _optionalHistoryCommitEncoder.get } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/DeltaFileOperations.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import java.io.{FileNotFoundException, IOException} import java.net.URI import java.util.Locale import scala.util.Random import scala.util.control.NonFatal import org.apache.spark.sql.delta.Relocated._ import org.apache.spark.sql.delta.{DeltaErrors, SerializableFileStatus} import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.storage.LogStore import org.apache.commons.io.IOUtils import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileAlreadyExistsException, FileStatus, FileSystem, FSDataInputStream, Path} import org.apache.hadoop.io.IOUtils.copyBytes import org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS import org.apache.parquet.hadoop.{Footer, ParquetFileReader} import org.apache.spark.{SparkEnv, SparkException, TaskContext} import org.apache.spark.broadcast.Broadcast import org.apache.spark.internal.MDC import org.apache.spark.sql.{Dataset, SparkSession} import org.apache.spark.util.{SerializableConfiguration, ThreadUtils} /** * Some utility methods on files, directories, and paths. */ object DeltaFileOperations extends DeltaLogging { /** * Create an absolute path from `child` using the `basePath` if the child is a relative path. * Return `child` if it is an absolute path. * * @param basePath Base path to prepend to `child` if child is a relative path. * Note: It is assumed that the basePath do not have any escaped characters and * is directly readable by Hadoop APIs. * @param child Child path to append to `basePath` if child is a relative path. * Note: t is assumed that the child is escaped, that is, all special chars that * need escaping by URI standards are already escaped. * @return Absolute path without escaped chars that is directly readable by Hadoop APIs. */ def absolutePath(basePath: String, child: String): Path = { // scalastyle:off pathfromuri val p = new Path(new URI(child)) if (p.isAbsolute) { p } else { val merged = new Path(basePath, p) // URI resolution strips the final `/` in `p` if it exists val mergedUri = merged.toUri.toString if (child.endsWith("/") && !mergedUri.endsWith("/")) { new Path(new URI(mergedUri + "/")) } else { merged } } // scalastyle:on pathfromuri } /** * Given a path `child`: * 1. Returns `child` if the path is already relative * 2. Tries relativizing `child` with respect to `basePath` * a) If the `child` doesn't live within the same base path, returns `child` as is * b) If `child` lives in a different FileSystem, throws an exception * Note that `child` may physically be pointing to a path within `basePath`, but may logically * belong to a different FileSystem, e.g. DBFS mount points and direct S3 paths. */ def tryRelativizePath( fs: FileSystem, basePath: Path, child: Path, ignoreError: Boolean = false): Path = { // We can map multiple schemes to the same `FileSystem` class, but `FileSystem.getScheme` is // usually just a hard-coded string. Hence, we need to use the scheme of the URI that we use to // create the FileSystem here. if (child.isAbsolute) { try { new Path(fs.makeQualified(basePath).toUri.relativize(fs.makeQualified(child).toUri)) } catch { case _: IllegalArgumentException if ignoreError => // ES-85571: when the file system failed to make the child path qualified, // it means the child path exists in a different file system // (a different authority or schema). This usually happens when the file is coming // from the across buckets or across cloud storage system shallow clone. // When ignoreError being set to true, not try to relativize this path, // ignore the error and just return `child` as is. child case e: IllegalArgumentException => logError(log"Failed to relativize the path ${MDC(DeltaLogKeys.PATH, child)} " + log"with the base path ${MDC(DeltaLogKeys.PATH2, basePath)} " + log"and the file system URI ${MDC(DeltaLogKeys.URI, fs.getUri)}", e) throw DeltaErrors.failRelativizePath(child.toString) } } else { child } } /** Check if the thrown exception is a throttling error. */ private def isThrottlingError(t: Throwable): Boolean = { Option(t.getMessage).exists(_.toLowerCase(Locale.ROOT).contains("slow down")) } private def randomBackoff( opName: String, t: Throwable, base: Int = 100, jitter: Int = 1000): Unit = { val sleepTime = Random.nextInt(jitter) + base logWarning(log"Sleeping for ${MDC(DeltaLogKeys.TIME_MS, sleepTime.toLong)} ms to rate limit " + log"${MDC(DeltaLogKeys.OP_NAME, opName)}", t) Thread.sleep(sleepTime) } /** Iterate through the contents of directories. * * If `listAsDirectories` is enabled, then we consider each path in `subDirs` to be directories, * and we list files under that path. If, for example, "a/b" is provided, we would attempt to * list "a/b/1.txt", "a/b/c/2.txt", and so on. We would not list "a/c", since it's not the same * directory as "a/b". * If not, we consider that path to be a filename, and we list paths in the same directory with * names after that path. So, if "a/b" is provided, we would list "a/b/1.txt", "a/c", "a/d", and * so on. However a file like "a/a.txt" would not be listed, because lexically it appears before * "a/b". */ private def listUsingLogStore( logStore: LogStore, hadoopConf: Configuration, subDirs: Iterator[String], recurse: Boolean, hiddenDirNameFilter: String => Boolean, hiddenFileNameFilter: String => Boolean, listAsDirectories: Boolean = true): Iterator[SerializableFileStatus] = { def list(dir: String, tries: Int): Iterator[SerializableFileStatus] = { logInfo(log"Listing ${MDC(DeltaLogKeys.DIR, dir)}") try { val path = if (listAsDirectories) new Path(dir, "\u0000") else new Path(dir + "\u0000") logStore.listFrom(path, hadoopConf) .filterNot{ f => val name = f.getPath.getName if (f.isDirectory) hiddenDirNameFilter(name) else hiddenFileNameFilter(name) }.map(SerializableFileStatus.fromStatus) } catch { case NonFatal(e) if isThrottlingError(e) && tries > 0 => randomBackoff("listing", e) list(dir, tries - 1) case e: FileNotFoundException => // Can happen when multiple GCs are running concurrently or due to eventual consistency Iterator.empty } } val filesAndDirs = subDirs.flatMap { dir => list(dir, tries = 10) } if (recurse) { recurseDirectories( logStore, hadoopConf, filesAndDirs, hiddenDirNameFilter, hiddenFileNameFilter) } else { filesAndDirs } } /** Given an iterator of files and directories, recurse directories with its contents. */ private def recurseDirectories( logStore: LogStore, hadoopConf: Configuration, filesAndDirs: Iterator[SerializableFileStatus], hiddenDirNameFilter: String => Boolean, hiddenFileNameFilter: String => Boolean): Iterator[SerializableFileStatus] = { filesAndDirs.flatMap { case dir: SerializableFileStatus if dir.isDir => Iterator.single(dir) ++ listUsingLogStore( logStore, hadoopConf, Iterator.single(dir.path), recurse = true, hiddenDirNameFilter, hiddenFileNameFilter) case file => Iterator.single(file) } } /** * The default filter for hidden files. Files names beginning with _ or . are considered hidden. * @param fileName * @return true if the file is hidden */ def defaultHiddenFileFilter(fileName: String): Boolean = { fileName.startsWith("_") || fileName.startsWith(".") } /** * Recursively lists all the files and directories for the given `subDirs` in a scalable manner. * * @param spark The SparkSession * @param subDirs Absolute path of the subdirectories to list * @param hadoopConf The Hadoop Configuration to get a FileSystem instance * @param hiddenDirNameFilter A function that returns true when the directory should be considered * hidden and excluded from results. Defaults to checking for prefixes * of "." or "_". * @param hiddenFileNameFilter A function that returns true when the file should be considered * hidden and excluded from results. Defaults to checking for prefixes * of "." or "_". * @param listAsDirectories Whether to treat the paths in subDirs as directories, where all files * that are children to the path will be listed. If false, the paths are * treated as filenames, and files under the same folder with filenames * after the path will be listed instead. */ def recursiveListDirs( spark: SparkSession, subDirs: Seq[String], hadoopConf: Broadcast[SerializableConfiguration], hiddenDirNameFilter: String => Boolean = defaultHiddenFileFilter, hiddenFileNameFilter: String => Boolean = defaultHiddenFileFilter, fileListingParallelism: Option[Int] = None, listAsDirectories: Boolean = true): Dataset[SerializableFileStatus] = { import org.apache.spark.sql.delta.implicits._ if (subDirs.isEmpty) return spark.emptyDataset[SerializableFileStatus] val listParallelism = fileListingParallelism.getOrElse(spark.sparkContext.defaultParallelism) val subDirsParallelism = subDirs.length.min(spark.sparkContext.defaultParallelism) val dirsAndFiles = spark.sparkContext.parallelize( subDirs, subDirsParallelism).mapPartitions { dirs => val logStore = LogStore(SparkEnv.get.conf, hadoopConf.value.value) listUsingLogStore( logStore, hadoopConf.value.value, dirs, recurse = false, hiddenDirNameFilter, hiddenFileNameFilter, listAsDirectories) }.repartition(listParallelism) // Initial list of subDirs may be small val allDirsAndFiles = dirsAndFiles.mapPartitions { firstLevelDirsAndFiles => val logStore = LogStore(SparkEnv.get.conf, hadoopConf.value.value) recurseDirectories( logStore, hadoopConf.value.value, firstLevelDirsAndFiles, hiddenDirNameFilter, hiddenFileNameFilter) } spark.createDataset(allDirsAndFiles) } /** * Recursively and incrementally lists files with filenames after `listFilename` by alphabetical * order. Helpful if you only want to list new files instead of the entire directory. * * Files located within `topDir` with filenames lexically after `listFilename` will be included, * even if they may be located in parent/sibling folders of `listFilename`. * * @param spark The SparkSession * @param listFilename Absolute path to a filename from which new files are listed (exclusive) * @param topDir Absolute path to the original starting directory * @param hadoopConf The Hadoop Configuration to get a FileSystem instance * @param hiddenDirNameFilter A function that returns true when the directory should be considered * hidden and excluded from results. Defaults to checking for prefixes * of "." or "_". * @param hiddenFileNameFilter A function that returns true when the file should be considered * hidden and excluded from results. Defaults to checking for prefixes * of "." or "_". */ def recursiveListFrom( spark: SparkSession, listFilename: String, topDir: String, hadoopConf: Broadcast[SerializableConfiguration], hiddenDirNameFilter: String => Boolean = defaultHiddenFileFilter, hiddenFileNameFilter: String => Boolean = defaultHiddenFileFilter, fileListingParallelism: Option[Int] = None): Dataset[SerializableFileStatus] = { // Add folders from `listPath` to the depth before `topPath`, so as to ensure new folders/files // in the parent directories are also included in the listing. // If there are no new files, listing from parent directories are expected to be constant time. val subDirs = getAllTopComponents(new Path(listFilename), new Path(topDir)) recursiveListDirs(spark, subDirs, hadoopConf, hiddenDirNameFilter, hiddenFileNameFilter, fileListingParallelism, listAsDirectories = false) } /** * Lists the directory locally using LogStore without launching a spark job. Returns an iterator * from LogStore. */ def localListDirs( hadoopConf: Configuration, dirs: Seq[String], recursive: Boolean = true, dirFilter: String => Boolean = defaultHiddenFileFilter, fileFilter: String => Boolean = defaultHiddenFileFilter): Iterator[SerializableFileStatus] = { val logStore = LogStore(SparkEnv.get.conf, hadoopConf) listUsingLogStore( logStore, hadoopConf, dirs.toIterator, recurse = recursive, dirFilter, fileFilter) } /** * Incrementally lists files with filenames after `listDir` by alphabetical order. Helpful if you * only want to list new files instead of the entire directory. * Listed locally using LogStore without launching a spark job. Returns an iterator from LogStore. */ def localListFrom( hadoopConf: Configuration, listFilename: String, topDir: String, recursive: Boolean = true, dirFilter: String => Boolean = defaultHiddenFileFilter, fileFilter: String => Boolean = defaultHiddenFileFilter): Iterator[SerializableFileStatus] = { val logStore = LogStore(SparkEnv.get.conf, hadoopConf) val listDirs = getAllTopComponents(new Path(listFilename), new Path(topDir)) listUsingLogStore(logStore, hadoopConf, listDirs.toIterator, recurse = recursive, dirFilter, fileFilter, listAsDirectories = false) } /** * Tries deleting a file or directory non-recursively. If the file/folder doesn't exist, * that's fine, a separate operation may be deleting files/folders. If a directory is non-empty, * we shouldn't delete it. FileSystem implementations throw an `IOException` in those cases, * which we return as a "we failed to delete". * * Listing on S3 is not consistent after deletes, therefore in case the `delete` returns `false`, * because the file didn't exist, then we still return `true`. Retries on S3 rate limits up to 3 * times. */ def tryDeleteNonRecursive(fs: FileSystem, path: Path, tries: Int = 3): Boolean = { try fs.delete(path, false) catch { case _: FileNotFoundException => true case _: IOException => false case NonFatal(e) if isThrottlingError(e) && tries > 0 => randomBackoff("deletes", e) tryDeleteNonRecursive(fs, path, tries - 1) } } /** * Returns all the levels of sub directories that `path` has with respect to `base`. For example: * getAllSubDirectories("/base", "/base/a/b/c") => * (Iterator("/base/a", "/base/a/b"), "/base/a/b/c") */ def getAllSubDirectories(base: String, path: String): (Iterator[String], String) = { val baseSplits = base.split(Path.SEPARATOR) val pathSplits = path.split(Path.SEPARATOR).drop(baseSplits.length) val it = Iterator.tabulate(pathSplits.length - 1) { i => (baseSplits ++ pathSplits.take(i + 1)).mkString(Path.SEPARATOR) } (it, path) } /** Register a task failure listener to delete a temp file in our best effort. */ def registerTempFileDeletionTaskFailureListener( conf: Configuration, tempPath: Path): Unit = { val tc = TaskContext.get() if (tc == null) { throw DeltaErrors.sparkTaskThreadNotFound } tc.addTaskFailureListener { (_, _) => // Best effort to delete the temp file try { tempPath.getFileSystem(conf).delete(tempPath, false /* = recursive */) } catch { case NonFatal(e) => logError(log"Failed to delete ${MDC(DeltaLogKeys.PATH, tempPath)}", e) } () // Make the compiler happy } } /** * Reads Parquet footers in multi-threaded manner. * If the config "spark.sql.files.ignoreCorruptFiles" is set to true, we will ignore the corrupted * files when reading footers. */ def readParquetFootersInParallel( conf: Configuration, partFiles: Seq[FileStatus], ignoreCorruptFiles: Boolean): Seq[Footer] = { ThreadUtils.parmap(partFiles, "readingParquetFooters", 8) { currentFile => try { // Skips row group information since we only need the schema. // ParquetFileReader.readFooter throws RuntimeException, instead of IOException, // when it can't read the footer. Some(new Footer(currentFile.getPath(), ParquetFileReader.readFooter( conf, currentFile, SKIP_ROW_GROUPS))) } catch { case e: RuntimeException => if (ignoreCorruptFiles) { logWarning(log"Skipped the footer in the corrupted file: " + log"${MDC(DeltaLogKeys.FILE_STATUS, currentFile)}", e) None } else { throw DeltaErrors.failedReadFileFooter(currentFile.toString, e) } } }.flatten } /** * Get all parent directory paths from `listDir` until `topDir` (exclusive). * For example, if `topDir` is "/folder/" and `currDir` is "/folder/a/b/c", we would return * "/folder/a/b/c", "/folder/a/b" and "/folder/a". */ def getAllTopComponents(listDir: Path, topDir: Path): List[String] = { var ret: List[String] = List() var currDir = listDir while (currDir.depth() > topDir.depth()) { ret = ret :+ currDir.toString val parent = currDir.getParent currDir = parent } ret } /** Expose `org.apache.spark.util.ThreadUtils.runInNewThread` to use in Delta code. */ def runInNewThread[T]( threadName: String, isDaemon: Boolean = true)(body: => T): T = { ThreadUtils.runInNewThread(threadName, isDaemon)(body) } /** * Returns a `Dataset[AddFile]`, where all the `AddFile` actions have absolute paths. The files * may have already had absolute paths, in which case they are left unchanged. Else, they are * prepended with the `qualifiedSourcePath`. * * @param qualifiedTablePath Fully qualified path of Delta table root * @param files List of `AddFile` instances */ def makePathsAbsolute( qualifiedTablePath: String, files: Dataset[AddFile]): Dataset[AddFile] = { import org.apache.spark.sql.delta.implicits._ files.mapPartitions { fileList => fileList.map { addFile => val fileSource = DeltaFileOperations.absolutePath(qualifiedTablePath, addFile.path) if (addFile.deletionVector != null) { val absoluteDV = addFile.deletionVector.copyWithAbsolutePath(new Path(qualifiedTablePath)) addFile.copy(path = fileSource.toUri.toString, deletionVector = absoluteDV) } else { addFile.copy(path = fileSource.toUri.toString) } } } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/DeltaLogGroupingIterator.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import scala.collection.mutable.ArrayBuffer import org.apache.spark.sql.delta.util.FileNames.{CheckpointFile, DeltaFile} import org.apache.hadoop.fs.FileStatus /** * An iterator that groups same types of files by version. * Note that this class could handle only Checkpoints and Delta files. * For example for an input iterator: * - 11.checkpoint.0.1.parquet * - 11.checkpoint.1.1.parquet * - 11.json * - 12.checkpoint.parquet * - 12.json * - 13.json * - 14.json * - 15.checkpoint.0.1.parquet * - 15.checkpoint.1.1.parquet * - 15.checkpoint..parquet * - 15.json * This will return: * - (11, Seq(11.checkpoint.0.1.parquet, 11.checkpoint.1.1.parquet, 11.json)) * - (12, Seq(12.checkpoint.parquet, 12.json)) * - (13, Seq(13.json)) * - (14, Seq(14.json)) * - (15, Seq(15.checkpoint.0.1.parquet, 15.checkpoint.1.1.parquet, 15.checkpoint..parquet, * 15.json)) */ class DeltaLogGroupingIterator( checkpointAndDeltas: Iterator[FileStatus]) extends Iterator[(Long, ArrayBuffer[FileStatus])] { private val bufferedIterator = checkpointAndDeltas.buffered /** * Validates that the underlying file is a checkpoint/delta file and returns the corresponding * version. */ private def getFileVersion(file: FileStatus): Long = { file match { case DeltaFile(_, version) => version case CheckpointFile(_, version) => version case _ => throw new IllegalStateException( s"${file.getPath} is not a valid commit file / checkpoint file") } } override def hasNext: Boolean = bufferedIterator.hasNext override def next(): (Long, ArrayBuffer[FileStatus]) = { val first = bufferedIterator.next() val buffer = scala.collection.mutable.ArrayBuffer(first) val firstFileVersion = getFileVersion(first) while (bufferedIterator.headOption.exists(getFileVersion(_) == firstFileVersion)) { buffer += bufferedIterator.next() } firstFileVersion -> buffer } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/DeltaProgressReporter.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.SparkContext import org.apache.spark.internal.{Logging, MDC} import org.apache.spark.sql.SparkSession trait DeltaProgressReporter extends Logging { /** * Report a log to indicate some command is running. */ def withStatusCode[T]( statusCode: String, defaultMessage: String, data: Map[String, Any] = Map.empty)(body: => T): T = { logInfo(log"${MDC(DeltaLogKeys.STATUS, statusCode)}: " + log"${MDC(DeltaLogKeys.STATUS_MESSAGE, defaultMessage)}") val t = withJobDescription(defaultMessage)(body) logInfo(log"${MDC(DeltaLogKeys.STATUS, statusCode)}: Done") t } /** * Wrap various delta operations to provide a more meaningful name in Spark UI * This only has an effect if {{{body}}} actually runs a Spark job * @param jobDesc a short description of the operation */ private def withJobDescription[U](jobDesc: String)(body: => U): U = { val sc = SparkSession.active.sparkContext // will prefix jobDesc with whatever the user specified in the job description // of the higher level operation that triggered this delta operation val oldDesc = sc.getLocalProperty(SparkContext.SPARK_JOB_DESCRIPTION) val suffix = if (oldDesc == null) { "" } else { s" $oldDesc:" } try { sc.setJobDescription(s"Delta:$suffix $jobDesc") body } finally { sc.setJobDescription(oldDesc) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/DeltaSparkPlanUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import org.apache.spark.sql.delta.{DeltaTable, DeltaTableReadPredicate} import org.apache.spark.sql.DataFrame import org.apache.spark.sql.catalyst.expressions.{Exists, Expression, InSubquery, LateralSubquery, ScalarSubquery, SubqueryExpression => SparkSubqueryExpression, UserDefinedExpression} import org.apache.spark.sql.catalyst.plans.logical.{Distinct, Filter, LeafNode, LogicalPlan, OneRowRelation, Project, SubqueryAlias, Union} import org.apache.spark.sql.execution.columnar.InMemoryRelation import org.apache.spark.sql.execution.datasources.LogicalRelation trait DeltaSparkPlanUtils { import DeltaSparkPlanUtils._ protected def planContainsOnlyDeltaScans(source: LogicalPlan): Boolean = findFirstNonDeltaScan(source).isEmpty protected def findFirstNonDeltaScan(source: LogicalPlan): Option[LogicalPlan] = { (source match { case l: LogicalRelation => l match { case DeltaTable(_) => None case _ => Some(l) } case OneRowRelation() => None case leaf: LeafNode => Some(leaf) // Any other LeafNode is a non Delta scan. case node => collectFirst(node.children, findFirstNonDeltaScan) }).orElse( // If not found in main plan, look into subqueries. collectFirst(source.subqueries, findFirstNonDeltaScan) ) } /** Returns whether part of the plan was cached using df.cache() or similar. */ protected def planContainsCachedRelation(df: DataFrame): Boolean = df.queryExecution.withCachedData.exists(_.isInstanceOf[InMemoryRelation]) /** * Returns `true` if `plan` has a safe level of determinism. This is a conservative * approximation of `plan` being a truly deterministic query. * */ protected def planIsDeterministic( plan: LogicalPlan, checkDeterministicOptions: CheckDeterministicOptions): Boolean = findFirstNonDeterministicNode(plan, checkDeterministicOptions).isEmpty type PlanOrExpression = Either[LogicalPlan, Expression] /** * Returns a part of the `plan` that does not have a safe level of determinism. * This is a conservative approximation of `plan` being a truly deterministic query. */ protected def findFirstNonDeterministicNode( plan: LogicalPlan, checkDeterministicOptions: CheckDeterministicOptions): Option[PlanOrExpression] = { plan match { // This is very restrictive, allowing only deterministic filters and projections directly // on top of a Delta Table. case Distinct(child) => findFirstNonDeterministicNode(child, checkDeterministicOptions) case Project(projectList, child) => findFirstNonDeterministicChildNode(projectList, checkDeterministicOptions) orElse { findFirstNonDeterministicNode(child, checkDeterministicOptions) } case Filter(cond, child) => findFirstNonDeterministicNode(cond, checkDeterministicOptions) orElse { findFirstNonDeterministicNode(child, checkDeterministicOptions) } case Union(children, _, _) => collectFirst[LogicalPlan, PlanOrExpression]( children, c => findFirstNonDeterministicNode(c, checkDeterministicOptions)) case SubqueryAlias(_, child) => findFirstNonDeterministicNode(child, checkDeterministicOptions) case DeltaTable(_) => None case OneRowRelation() => None case node => Some(Left(node)) } } protected def planContainsUdf(plan: LogicalPlan): Boolean = { plan.collectWithSubqueries { case node if node.expressions.exists(_.exists(_.isInstanceOf[UserDefinedExpression])) => () }.nonEmpty } protected def findFirstNonDeterministicChildNode( children: Seq[Expression], checkDeterministicOptions: CheckDeterministicOptions): Option[PlanOrExpression] = collectFirst[Expression, PlanOrExpression]( children, c => findFirstNonDeterministicNode(c, checkDeterministicOptions)) protected def findFirstNonDeterministicNode( child: Expression, checkDeterministicOptions: CheckDeterministicOptions): Option[PlanOrExpression] = { child match { case SubqueryExpression(plan) => if (SparkSubqueryExpression.hasCorrelatedSubquery(child)) { // We consider joins potentially non-deterministic, and correlated subqueries are // flattened into joins, so they should also be considered potentially non-deterministic. Some(Right(child)) } else { findFirstNonDeterministicNode(plan, checkDeterministicOptions) } case _: UserDefinedExpression if !checkDeterministicOptions.allowDeterministicUdf => Some(Right(child)) case p => collectFirst[Expression, PlanOrExpression]( p.children, c => findFirstNonDeterministicNode(c, checkDeterministicOptions)) orElse { if (p.deterministic) None else Some(Right(p)) } } } protected def collectFirst[In, Out]( input: Iterable[In], recurse: In => Option[Out]): Option[Out] = { input.foldLeft(Option.empty[Out]) { case (acc, value) => acc.orElse(recurse(value)) } } /** Extractor object for the subquery plan of expressions that contain subqueries. */ object SubqueryExpression { def unapply(expr: Expression): Option[LogicalPlan] = expr match { case subquery: ScalarSubquery => Some(subquery.plan) case exists: Exists => Some(exists.plan) case subquery: InSubquery => Some(subquery.query.plan) case subquery: LateralSubquery => Some(subquery.plan) case _ => None } } /** Returns whether the read predicates of a transaction contain any deterministic UDFs. */ def containsDeterministicUDF( predicates: Seq[DeltaTableReadPredicate], partitionedOnly: Boolean): Boolean = { if (partitionedOnly) { predicates.exists { _.partitionPredicates.exists(containsDeterministicUDF) } } else { predicates.exists { p => p.dataPredicates.exists(containsDeterministicUDF) || p.partitionPredicates.exists(containsDeterministicUDF) } } } /** Returns whether an expression contains any deterministic UDFs. */ def containsDeterministicUDF(expr: Expression): Boolean = expr.exists { case udf: UserDefinedExpression => udf.deterministic case _ => false } } object DeltaSparkPlanUtils { /** * Options for deciding whether plans contain non-deterministic nodes and expressions. * * @param allowDeterministicUdf If true, allow UDFs that are marked by users as deterministic. * If false, always treat them as non-deterministic to be more * defensive against user bugs. */ case class CheckDeterministicOptions( allowDeterministicUdf: Boolean ) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/DeltaSqlParserUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import scala.collection.JavaConverters._ import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.parser.{AbstractSqlParser, AstBuilder, ParseException, ParserUtils, SqlBaseParser} /** * Utility functions for SQL parsing operations. */ object DeltaSqlParserUtils { /** * The SQL grammar already includes a `multipartIdentifierList` rule for parsing a string into a * list of multi-part identifiers. We just expose it here, with a custom parser and AstBuilder. */ private class MultipartIdentifierSqlParser extends AbstractSqlParser { override val astBuilder = new AstBuilder { override def visitMultipartIdentifierList(ctx: SqlBaseParser.MultipartIdentifierListContext) : Seq[UnresolvedAttribute] = ParserUtils.withOrigin(ctx) { ctx.multipartIdentifier.asScala.toSeq.map(typedVisit[Seq[String]]) .map(new UnresolvedAttribute(_)) } } def parseMultipartIdentifierList(sqlText: String): Seq[UnresolvedAttribute] = { parse(sqlText) { parser => astBuilder.visitMultipartIdentifierList(parser.multipartIdentifierList()) } } } private val multipartIdentifierSqlParser = new MultipartIdentifierSqlParser /** Parses a comma-separated list of column names; returns None if parsing fails. */ def parseMultipartColumnList(columns: String): Option[Seq[UnresolvedAttribute]] = { // The parser rejects empty lists, so handle that specially here. if (columns.trim.isEmpty) return Some(Nil) try { Some(multipartIdentifierSqlParser.parseMultipartIdentifierList(columns)) } catch { case _: ParseException => None } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/DeltaStatsJsonUtils.scala ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import org.apache.spark.sql.delta.shims.VariantStatsShims import org.apache.spark.sql.delta.util.Codec.Base85Codec import org.apache.spark.types.variant.{Variant, VariantUtil} import org.apache.spark.unsafe.types.VariantVal /** * Utility functions for encoding/decoding Variant values as Z85 strings. * This is used for storing variant statistics in Delta checkpoints and stats. */ object DeltaStatsJsonUtils { /** * Encode a Variant as a Z85 string. * The variant binary format stores metadata followed by value bytes. * This concatenates them and encodes as Z85. */ def encodeVariantAsZ85(v: Variant): String = { val metadata = v.getMetadata val value = v.getValue val combined = new Array[Byte](metadata.length + value.length) System.arraycopy(metadata, 0, combined, 0, metadata.length) System.arraycopy(value, 0, combined, metadata.length, value.length) Base85Codec.encodeBytes(combined) } /** * Decode a Z85-encoded string back to a VariantVal. */ def decodeVariantFromZ85(z85: String): VariantVal = { val decoded = Base85Codec.decodeBytes(z85, z85.length) val metadataSize = VariantStatsShims.metadataSize(decoded) val valueWithPadding = decoded.slice(metadataSize, decoded.length) val valueSize = VariantUtil.valueSize(valueWithPadding, 0) val value = valueWithPadding.slice(0, valueSize) val metadata = decoded.slice(0, metadataSize) new VariantVal(value, metadata) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/FileNames.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import java.util.UUID import org.apache.spark.sql.delta.DeltaLog import org.apache.hadoop.fs.{FileStatus, Path} /** Helper for creating file names for specific commits / checkpoints. */ object FileNames { val deltaFileRegex = raw"(\d+)\.json".r val uuidDeltaFileRegex = raw"(\d+)\.([^.]+)\.json".r val compactedDeltaFileRegex = raw"(\d+).(\d+).compacted.json".r val checksumFileRegex = raw"(\d+)\.crc".r val checkpointFileRegex = raw"(\d+)\.checkpoint((\.\d+\.\d+)?\.parquet|\.[^.]+\.(json|parquet))".r private val compactedDeltaFilePattern = compactedDeltaFileRegex.pattern private val checksumFilePattern = checksumFileRegex.pattern private val checkpointFilePattern = checkpointFileRegex.pattern /** * Returns the delta (json format) path for a given delta file. * WARNING: This API is unsafe and can resolve to incorrect paths if the table has * Coordinated Commits. * Use DeltaCommitFileProvider(snapshot).deltaFile instead to guarantee accurate paths. */ def unsafeDeltaFile(path: Path, version: Long): Path = new Path(path, f"$version%020d.json") /** * Returns the un-backfilled uuid formatted delta (json format) path for a given version. * * @param logPath The root path of the delta log. * @param version The version of the delta file. * @return The path to the un-backfilled delta file: * `/_staged_commits/..json` */ def unbackfilledDeltaFile( logPath: Path, version: Long, uuidString: Option[String] = None): Path = { val basePath = commitDirPath(logPath) val uuid = uuidString.getOrElse(UUID.randomUUID.toString) new Path(basePath, f"$version%020d.$uuid.json") } /** Returns the path for a given sample file */ def sampleFile(path: Path, version: Long): Path = new Path(path, f"$version%020d") /** Returns the path to the checksum file for the given version. */ def checksumFile(path: Path, version: Long): Path = new Path(path, f"$version%020d.crc") /** Returns the path to the compacted delta file for the given version range. */ def compactedDeltaFile( path: Path, fromVersion: Long, toVersion: Long): Path = { new Path(path, f"$fromVersion%020d.$toVersion%020d.compacted.json") } /** Returns the version for the given delta path. */ def deltaVersion(path: Path): Long = path.getName.split("\\.")(0).toLong def deltaVersion(file: FileStatus): Long = deltaVersion(file.getPath) /** Returns the version for the given checksum file. */ def checksumVersion(path: Path): Long = path.getName.stripSuffix(".crc").toLong def checksumVersion(file: FileStatus): Long = checksumVersion(file.getPath) def compactedDeltaVersions(path: Path): (Long, Long) = { val parts = path.getName.split("\\.") (parts(0).toLong, parts(1).toLong) } def compactedDeltaVersions(file: FileStatus): (Long, Long) = compactedDeltaVersions(file.getPath) /** * Returns the prefix of all delta log files for the given version. * * Intended for use with listFrom to get all files from this version onwards. The returned Path * will not exist as a file. */ def listingPrefix(path: Path, version: Long): Path = new Path(path, f"$version%020d.") /** * Returns the path for a singular checkpoint up to the given version. * * In a future protocol version this path will stop being written. */ def checkpointFileSingular(path: Path, version: Long): Path = new Path(path, f"$version%020d.checkpoint.parquet") /** * Returns the paths for all parts of the checkpoint up to the given version. * * In a future protocol version we will write this path instead of checkpointFileSingular. * * Example of the format: 00000000000000004915.checkpoint.0000000020.0000000060.parquet is * checkpoint part 20 out of 60 for the snapshot at version 4915. Zero padding is for * lexicographic sorting. */ def checkpointFileWithParts(path: Path, version: Long, numParts: Int): Seq[Path] = { Range(1, numParts + 1) .map(i => new Path(path, f"$version%020d.checkpoint.$i%010d.$numParts%010d.parquet")) } def numCheckpointParts(path: Path): Option[Int] = { val segments = path.getName.split("\\.") if (segments.size != 5) None else Some(segments(3).toInt) } def isCheckpointFile(path: Path): Boolean = checkpointFilePattern.matcher(path.getName).matches() def isCheckpointFile(file: FileStatus): Boolean = isCheckpointFile(file.getPath) def isDeltaFile(path: Path): Boolean = DeltaFile.unapply(path).isDefined def isDeltaFile(file: FileStatus): Boolean = isDeltaFile(file.getPath) def isUnbackfilledDeltaFile(path: Path): Boolean = UnbackfilledDeltaFile.unapply(path).isDefined def isUnbackfilledDeltaFile(file: FileStatus): Boolean = isUnbackfilledDeltaFile(file.getPath) def isBackfilledDeltaFile(path: Path): Boolean = BackfilledDeltaFile.unapply(path).isDefined def isBackfilledDeltaFile(file: FileStatus): Boolean = isBackfilledDeltaFile(file.getPath) def isChecksumFile(path: Path): Boolean = checksumFilePattern.matcher(path.getName).matches() def isChecksumFile(file: FileStatus): Boolean = isChecksumFile(file.getPath) def isCompactedDeltaFile(path: Path): Boolean = compactedDeltaFilePattern.matcher(path.getName).matches() def isCompactedDeltaFile(file: FileStatus): Boolean = isCompactedDeltaFile(file.getPath) def checkpointVersion(path: Path): Long = path.getName.split("\\.")(0).toLong def checkpointVersion(file: FileStatus): Long = checkpointVersion(file.getPath) object CompactedDeltaFile { def unapply(f: FileStatus): Option[(FileStatus, Long, Long)] = unapply(f.getPath).map { case (_, startVersion, endVersion) => (f, startVersion, endVersion) } def unapply(path: Path): Option[(Path, Long, Long)] = path.getName match { case compactedDeltaFileRegex(lo, hi) => Some(path, lo.toLong, hi.toLong) case _ => None } } /** * Get the version of the checkpoint, checksum or delta file. Returns None if an unexpected * file type is seen. */ def getFileVersionOpt(path: Path): Option[Long] = path match { case DeltaFile(_, version) => Some(version) case ChecksumFile(_, version) => Some(version) case CheckpointFile(_, version) => Some(version) case CompactedDeltaFile(_, _, endVersion) => Some(endVersion) case _ => None } /** * Get the version of the checkpoint, checksum or delta file. Throws an error if an unexpected * file type is seen. These unexpected files should be filtered out to ensure forward * compatibility in cases where new file types are added, but without an explicit protocol * upgrade. */ def getFileVersion(path: Path): Long = { getFileVersionOpt(path).getOrElse { // scalastyle:off throwerror throw new AssertionError( s"Unexpected file type found in transaction log: $path") // scalastyle:on throwerror } } def getFileVersion(file: FileStatus): Long = getFileVersion(file.getPath) object DeltaFile { def unapply(f: FileStatus): Option[(FileStatus, Long)] = unapply(f.getPath).map { case (_, version) => (f, version) } def unapply(path: Path): Option[(Path, Long)] = { val parentDirName = path.getParent.getName // If parent is `_staged_commits` dir, then match against unbackfilled commit file. val regex = if (parentDirName == COMMIT_SUBDIR) uuidDeltaFileRegex else deltaFileRegex regex.unapplySeq(path.getName).map(path -> _.head.toLong) } } object ChecksumFile { def unapply(f: FileStatus): Option[(FileStatus, Long)] = unapply(f.getPath).map { case (_, version) => (f, version) } def unapply(path: Path): Option[(Path, Long)] = checksumFileRegex.unapplySeq(path.getName).map(path -> _.head.toLong) } object CheckpointFile { def unapply(f: FileStatus): Option[(FileStatus, Long)] = unapply(f.getPath).map { case (_, version) => (f, version) } def unapply(path: Path): Option[(Path, Long)] = { checkpointFileRegex.unapplySeq(path.getName).map(path -> _.head.toLong) } } object BackfilledDeltaFile { def unapply(f: FileStatus): Option[(FileStatus, Long)] = unapply(f.getPath).map { case (_, version) => (f, version) } def unapply(path: Path): Option[(Path, Long)] = { // Don't match files in the `_staged_commits` subdirectory. if (path.getParent.getName == COMMIT_SUBDIR) { None } else { deltaFileRegex .unapplySeq(path.getName) .map(path -> _.head.toLong) } } } object UnbackfilledDeltaFile { def unapply(f: FileStatus): Option[(FileStatus, Long, String)] = unapply(f.getPath).map { case (_, version, uuidString) => (f, version, uuidString) } def unapply(path: Path): Option[(Path, Long, String)] = { // If parent is `_staged_commits` dir, then match against uuid commit file. if (path.getParent.getName == COMMIT_SUBDIR) { uuidDeltaFileRegex .unapplySeq(path.getName) .collect { case Seq(version, uuidString) => (path, version.toLong, uuidString) } } else { None } } } object FileType extends Enumeration { val DELTA, CHECKPOINT, CHECKSUM, COMPACTED_DELTA, OTHER = Value } /** File path for a new V2 Checkpoint Json file */ def newV2CheckpointJsonFile(path: Path, version: Long): Path = new Path(path, f"$version%020d.checkpoint.${UUID.randomUUID.toString}.json") /** File path for a new V2 Checkpoint Parquet file */ def newV2CheckpointParquetFile(path: Path, version: Long): Path = new Path(path, f"$version%020d.checkpoint.${UUID.randomUUID.toString}.parquet") /** File path for a V2 Checkpoint's Sidecar file */ def newV2CheckpointSidecarFile( logPath: Path, version: Long, numParts: Int, currentPart: Int): Path = { val basePath = sidecarDirPath(logPath) val uuid = UUID.randomUUID.toString new Path(basePath, f"$version%020d.checkpoint.$currentPart%010d.$numParts%010d.$uuid.parquet") } val SIDECAR_SUBDIR = "_sidecars" /** Returns path to the sidecar directory */ def sidecarDirPath(logPath: Path): Path = new Path(logPath, SIDECAR_SUBDIR) val COMMIT_SUBDIR = "_staged_commits" /** Returns path to the staged commit directory */ def commitDirPath(logPath: Path): Path = new Path(logPath, COMMIT_SUBDIR) } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/InCommitTimestampUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.actions.{Action, CommitInfo, Metadata} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.spark.sql.SparkSession object InCommitTimestampUtils { final val TABLE_PROPERTY_CONFS = Seq( DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED, DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION, DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP) final val TABLE_PROPERTY_KEYS: Seq[String] = TABLE_PROPERTY_CONFS.map(_.key) /** Returns true if the current transaction implicitly/explicitly enables ICT. */ def didCurrentTransactionEnableICT( currentTransactionMetadata: Metadata, readSnapshot: Snapshot): Boolean = { // If ICT is currently enabled, and the read snapshot did not have ICT enabled, // then the current transaction must have enabled it. // In case of a conflict, any winning transaction that enabled it after // our read snapshot would have caused a metadata conflict abort // (see [[ConflictChecker.checkNoMetadataUpdates]]), so we know that // all winning transactions' ICT enablement status must match the read snapshot. // // WARNING: The Metadata() of InitialSnapshot can enable ICT by default. To ensure that // this function returns true if ICT is enabled during the first commit, we explicitly handle // the case where the readSnapshot.version is -1. val isICTCurrentlyEnabled = DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(currentTransactionMetadata) val wasICTEnabledInReadSnapshot = readSnapshot.version != -1 && DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(readSnapshot.metadata) isICTCurrentlyEnabled && !wasICTEnabledInReadSnapshot } /** * Returns the updated [[Metadata]] with inCommitTimestamp enablement related info * (version and timestamp) correctly set. * This enablement info will be set to the current commit's timestamp and version if: * 1. If this transaction enables inCommitTimestamp. * 2. If the commit version is not 0. This is because we only need to persist * the enablement info if there are non-ICT commits in the Delta log. * For cases where ICT is enabled in both the current transaction and the read snapshot, * we will retain the enablement info from the read snapshot. Note that this can * happen for commands like REPLACE or CLONE, where we can end up dropping the enablement * info due to the belief that ICT was just enabled. * Note: This function must only be called after transaction conflicts have been resolved. */ def getUpdatedMetadataWithICTEnablementInfo( spark: SparkSession, inCommitTimestamp: Long, readSnapshot: Snapshot, metadata: Metadata, commitVersion: Long): Option[Metadata] = { if (didCurrentTransactionEnableICT(metadata, readSnapshot) && commitVersion != 0) { val enablementTrackingProperties = Map( DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.key -> commitVersion.toString, DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.key -> inCommitTimestamp.toString) Some(metadata.copy(configuration = metadata.configuration ++ enablementTrackingProperties)) } else if (DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(metadata) && !didCurrentTransactionEnableICT(metadata, readSnapshot) && // This check ensures that we don't make an unnecessary metadata update // even when ICT enablement properties are not being dropped. getValidatedICTEnablementInfo(readSnapshot.metadata).isDefined && getValidatedICTEnablementInfo(metadata).isEmpty && spark.conf.get(DeltaSQLConf.IN_COMMIT_TIMESTAMP_RETAIN_ENABLEMENT_INFO_FIX_ENABLED) ) { // If ICT was enabled in the readSnapshot and is still enabled, we should // retain the enablement info from the read snapshot. // This prevents enablement info from being dropped during REPLACE/CLONE. val existingICTConfigs = readSnapshot.metadata.configuration .filter { case (k, _) => TABLE_PROPERTY_KEYS.contains(k) } Some(metadata.copy(configuration = metadata.configuration ++ existingICTConfigs)) } else { None } } def getValidatedICTEnablementInfo(metadata: Metadata): Option[DeltaHistoryManager.Commit] = { val enablementTimestampOpt = DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.fromMetaData(metadata) val enablementVersionOpt = DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.fromMetaData(metadata) (enablementTimestampOpt, enablementVersionOpt) match { case (Some(enablementTimestamp), Some(enablementVersion)) => Some(DeltaHistoryManager.Commit(enablementVersion, enablementTimestamp)) case (None, None) => None case _ => throw new IllegalStateException( "Both enablement version and timestamp should be present or absent together.") } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/JsonUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import com.fasterxml.jackson.annotation.JsonInclude.Include import com.fasterxml.jackson.core.StreamReadConstraints import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper} /** Useful json functions used around the Delta codebase. */ object JsonUtils { /** Used to convert between classes and JSON. */ lazy val mapper = { val _mapper = new ObjectMapper with ScalaObjectMapper _mapper.setSerializationInclusion(Include.NON_ABSENT) _mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) _mapper.registerModule(DefaultScalaModule) // We do not want to limit the length of JSON strings in the Delta log or table data. Also note // that not having a limit was the default behavior before Jackson 2.15. val streamReadConstraints = StreamReadConstraints .builder() .maxStringLength(Int.MaxValue) .build() _mapper.getFactory.setStreamReadConstraints(streamReadConstraints) _mapper } def toJson[T: Manifest](obj: T): String = { mapper.writeValueAsString(obj) } def toPrettyJson[T: Manifest](obj: T): String = { mapper.writerWithDefaultPrettyPrinter().writeValueAsString(obj) } def fromJson[T: Manifest](json: String): T = { mapper.readValue[T](json) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/PartitionUtils.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import java.lang.{Double => JDouble, Long => JLong} import java.math.{BigDecimal => JBigDecimal} import java.time.ZoneId import java.util.{Locale, TimeZone} import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import scala.util.Try import org.apache.spark.sql.delta.{DeltaAnalysisException, DeltaErrors} import org.apache.hadoop.fs.Path import org.apache.spark.unsafe.types.UTF8String // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis._ import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, Literal} import org.apache.spark.sql.catalyst.types.DataTypeUtils import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.types._ /** * This file is forked from [[org.apache.spark.sql.execution.datasources.PartitioningUtils]]. */ // In open-source Apache Spark, PartitionPath is defined as // // case class PartitionPath(values: InternalRow, path: Path) // // but in Databricks we use a different representation where the Path is stored as a String // and converted back to a Path only when read. This significantly cuts memory consumption because // Hadoop Path objects are heavyweight. See SC-7591 for details. object PartitionPath { // Used only in tests: def apply(values: InternalRow, path: String): PartitionPath = { // Roundtrip through `new Path` to ensure any normalization done there is applied: apply(values, new Path(path)) } def apply(values: InternalRow, path: Path): PartitionPath = { new PartitionPath(values, path.toString) } } /** * Holds a directory in a partitioned collection of files as well as the partition values * in the form of a Row. Before scanning, the files at `path` need to be enumerated. */ class PartitionPath private (val values: InternalRow, val pathStr: String) { // Note: this isn't a case class because we don't want to have a public apply() method which // accepts a string. The goal is to force every value stored in `pathStr` to have gone through // a `new Path(...).toString` to ensure that canonicalization / normalization has taken place. def path: Path = new Path(pathStr) def withNewValues(newValues: InternalRow): PartitionPath = { new PartitionPath(newValues, pathStr) } override def equals(other: Any): Boolean = other match { case that: PartitionPath => values == that.values && pathStr == that.pathStr case _ => false } override def hashCode(): Int = { (values, pathStr).hashCode() } override def toString: String = { s"PartitionPath($values, $pathStr)" } } case class PartitionSpec( partitionColumns: StructType, partitions: Seq[PartitionPath]) object PartitionSpec { val emptySpec = PartitionSpec(StructType(Seq.empty[StructField]), Seq.empty[PartitionPath]) } private[delta] object PartitionUtils { lazy val timestampPartitionPattern = s"yyyy-MM-dd HH:mm:ss${precisionMatchPatterns(6)}" lazy val utcFormatter = TimestampFormatter(s"yyyy-MM-dd'T'HH:mm:ss.SSSSSSz", ZoneId.of("Z")) private def precisionMatchPatterns(maxDigits: Int): String = (maxDigits to 1 by -1) .map(n => "[." + ("S" * n) + "]") .mkString case class PartitionValues(columnNames: Seq[String], literals: Seq[Literal]) { require(columnNames.size == literals.size) } import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.{escapePathName, unescapePathName, DEFAULT_PARTITION_NAME} /** * Given a group of qualified paths, tries to parse them and returns a partition specification. * For example, given: * {{{ * hdfs://:/path/to/partition/a=1/b=hello/c=3.14 * hdfs://:/path/to/partition/a=2/b=world/c=6.28 * }}} * it returns: * {{{ * PartitionSpec( * partitionColumns = StructType( * StructField(name = "a", dataType = IntegerType, nullable = true), * StructField(name = "b", dataType = StringType, nullable = true), * StructField(name = "c", dataType = DoubleType, nullable = true)), * partitions = Seq( * Partition( * values = Row(1, "hello", 3.14), * path = "hdfs://:/path/to/partition/a=1/b=hello/c=3.14"), * Partition( * values = Row(2, "world", 6.28), * path = "hdfs://:/path/to/partition/a=2/b=world/c=6.28"))) * }}} */ def parsePartitions( paths: Seq[Path], typeInference: Boolean, basePaths: Set[Path], userSpecifiedSchema: Option[StructType], caseSensitive: Boolean, validatePartitionColumns: Boolean, timeZoneId: String): PartitionSpec = { parsePartitions(paths, typeInference, basePaths, userSpecifiedSchema, caseSensitive, validatePartitionColumns, DateTimeUtils.getTimeZone(timeZoneId)) } def parsePartitions( paths: Seq[Path], typeInference: Boolean, basePaths: Set[Path], userSpecifiedSchema: Option[StructType], caseSensitive: Boolean, validatePartitionColumns: Boolean, timeZone: TimeZone): PartitionSpec = { val userSpecifiedDataTypes = if (userSpecifiedSchema.isDefined) { val nameToDataType = mapNameToDataType(userSpecifiedSchema.get) if (!caseSensitive) { CaseInsensitiveMap(nameToDataType) } else { nameToDataType } } else { Map.empty[String, DataType] } // SPARK-26990: use user specified field names if case insensitive. val userSpecifiedNames = if (userSpecifiedSchema.isDefined && !caseSensitive) { CaseInsensitiveMap(userSpecifiedSchema.get.fields.map(f => f.name -> f.name).toMap) } else { Map.empty[String, String] } val dateFormatter = DateFormatter() val timestampFormatter = TimestampFormatter(timestampPartitionPattern, timeZone) // First, we need to parse every partition's path and see if we can find partition values. val (partitionValues, optDiscoveredBasePaths) = paths.map { path => parsePartition(path, typeInference, basePaths, userSpecifiedDataTypes, validatePartitionColumns, timeZone, dateFormatter, timestampFormatter) }.unzip // We create pairs of (path -> path's partition value) here // If the corresponding partition value is None, the pair will be skipped val pathsWithPartitionValues = paths.zip(partitionValues).flatMap(x => x._2.map(x._1 -> _)) if (pathsWithPartitionValues.isEmpty) { // This dataset is not partitioned. PartitionSpec.emptySpec } else { // This dataset is partitioned. We need to check whether all partitions have the same // partition columns and resolve potential type conflicts. // Check if there is conflicting directory structure. // For the paths such as: // var paths = Seq( // "hdfs://host:9000/invalidPath", // "hdfs://host:9000/path/a=10/b=20", // "hdfs://host:9000/path/a=10.5/b=hello") // It will be recognised as conflicting directory structure: // "hdfs://host:9000/invalidPath" // "hdfs://host:9000/path" // TODO: Selective case sensitivity. val discoveredBasePaths = optDiscoveredBasePaths.flatten.map(_.toString.toLowerCase()) assert( discoveredBasePaths.distinct.size == 1, "Conflicting directory structures detected. Suspicious paths:\b" + discoveredBasePaths.distinct.mkString("\n\t", "\n\t", "\n\n") + "If provided paths are partition directories, please set " + "\"basePath\" in the options of the data source to specify the " + "root directory of the table. If there are multiple root directories, " + "please load them separately and then union them.") val resolvedPartitionValues = resolvePartitions(pathsWithPartitionValues, caseSensitive, timeZone) // Creates the StructType which represents the partition columns. val fields = { val PartitionValues(columnNames, literals) = resolvedPartitionValues.head columnNames.zip(literals).map { case (name, Literal(_, dataType)) => // We always assume partition columns are nullable since we've no idea whether null values // will be appended in the future. val resultName = userSpecifiedNames.getOrElse(name, name) val resultDataType = userSpecifiedDataTypes.getOrElse(name, dataType) StructField(resultName, resultDataType, nullable = true) } } // Finally, we create `Partition`s based on paths and resolved partition values. val partitions = resolvedPartitionValues.zip(pathsWithPartitionValues).map { case (PartitionValues(_, literals), (path, _)) => PartitionPath(InternalRow.fromSeq(literals.map(_.value)), path) } PartitionSpec(StructType(fields), partitions) } } /** * Parses a single partition, returns column names and values of each partition column, also * the path when we stop partition discovery. For example, given: * {{{ * path = hdfs://:/path/to/partition/a=42/b=hello/c=3.14 * }}} * it returns the partition: * {{{ * PartitionValues( * Seq("a", "b", "c"), * Seq( * Literal.create(42, IntegerType), * Literal.create("hello", StringType), * Literal.create(3.14, DoubleType))) * }}} * and the path when we stop the discovery is: * {{{ * hdfs://:/path/to/partition * }}} */ def parsePartition( path: Path, typeInference: Boolean, basePaths: Set[Path], userSpecifiedDataTypes: Map[String, DataType], validatePartitionColumns: Boolean, timeZone: TimeZone, dateFormatter: DateFormatter, timestampFormatter: TimestampFormatter, useUtcNormalizedTimestamp: Boolean = false): (Option[PartitionValues], Option[Path]) = { val columns = ArrayBuffer.empty[(String, Literal)] // Old Hadoop versions don't have `Path.isRoot` var finished = path.getParent == null // currentPath is the current path that we will use to parse partition column value. var currentPath: Path = path while (!finished) { // Sometimes (e.g., when speculative task is enabled), temporary directories may be left // uncleaned. Here we simply ignore them. if (currentPath.getName.toLowerCase(Locale.ROOT) == "_temporary") { return (None, None) } if (basePaths.contains(currentPath)) { // If the currentPath is one of base paths. We should stop. finished = true } else { // Let's say currentPath is a path of "/table/a=1/", currentPath.getName will give us a=1. // Once we get the string, we try to parse it and find the partition column and value. val maybeColumn = parsePartitionColumn(currentPath.getName, typeInference, userSpecifiedDataTypes, validatePartitionColumns, timeZone, dateFormatter, timestampFormatter, useUtcNormalizedTimestamp) maybeColumn.foreach(columns += _) // Now, we determine if we should stop. // When we hit any of the following cases, we will stop: // - In this iteration, we could not parse the value of partition column and value, // i.e. maybeColumn is None, and columns is not empty. At here we check if columns is // empty to handle cases like /table/a=1/_temporary/something (we need to find a=1 in // this case). // - After we get the new currentPath, this new currentPath represent the top level dir // i.e. currentPath.getParent == null. For the example of "/table/a=1/", // the top level dir is "/table". finished = (maybeColumn.isEmpty && columns.nonEmpty) || currentPath.getParent == null if (!finished) { // For the above example, currentPath will be "/table/". currentPath = currentPath.getParent } } } if (columns.isEmpty) { (None, Some(path)) } else { val (columnNames, values) = columns.reverse.unzip (Some(PartitionValues(columnNames.toSeq, values.toSeq)), Some(currentPath)) } } private def parsePartitionColumn( columnSpec: String, typeInference: Boolean, userSpecifiedDataTypes: Map[String, DataType], validatePartitionColumns: Boolean, timeZone: TimeZone, dateFormatter: DateFormatter, timestampFormatter: TimestampFormatter, useUtcNormalizedTimestamp: Boolean = false): Option[(String, Literal)] = { val equalSignIndex = columnSpec.indexOf('=') if (equalSignIndex == -1) { None } else { val columnName = unescapePathName(columnSpec.take(equalSignIndex)) assert(columnName.nonEmpty, s"Empty partition column name in '$columnSpec'") val rawColumnValue = columnSpec.drop(equalSignIndex + 1) assert(rawColumnValue.nonEmpty, s"Empty partition column value in '$columnSpec'") val unescapedColumnValue = unescapePathName(rawColumnValue) val columnValue = if (unescapedColumnValue == DEFAULT_PARTITION_NAME) { null } else { unescapedColumnValue } // Workaround to maintain backward compatibility when UTC timestamp normalization is disabled. // The UTC timestamp partition values feature (enabled by default via // [[DeltaSQLConf.UTC_TIMESTAMP_PARTITION_VALUES]]) normalizes timestamp partition values to // UTC ISO 8601 format. When this feature is disabled, [[useUtcNormalizedTimestamp = false]], // we treat TimestampType partition columns as StringType to avoid timestamp parsing and // formatting operations that could change the original string representation. // This exists solely to sustain the existing behavior for legacy code paths that have // UTC normalization disabled. The partition value will be parsed as a string literal and // preserved exactly as provided, rather than being parsed as a timestamp and potentially // reformatted. val dataType = userSpecifiedDataTypes.get(columnName).map { case TimestampType if !useUtcNormalizedTimestamp => StringType case dt => dt } val literal = parsePartitionValue( columnName, columnValue, dataType, typeInference, timeZone, dateFormatter, timestampFormatter, validatePartitionColumns) Some(columnName -> literal) } } /** * Given a partition path fragment, e.g. `fieldOne=1/fieldTwo=2`, returns a parsed spec * for that fragment as a `TablePartitionSpec`, e.g. `Map(("fieldOne", "1"), ("fieldTwo", "2"))`. */ def parsePathFragment(pathFragment: String): TablePartitionSpec = { parsePathFragmentAsSeq(pathFragment).toMap } /** * Given a partition path fragment, e.g. `fieldOne=1/fieldTwo=2`, returns a parsed spec * for that fragment as a `Seq[(String, String)]`, e.g. * `Seq(("fieldOne", "1"), ("fieldTwo", "2"))`. */ def parsePathFragmentAsSeq(pathFragment: String): Seq[(String, String)] = { pathFragment.stripPrefix("data/").split("/").map { kv => val pair = kv.split("=", 2) (unescapePathName(pair(0)), unescapePathName(pair(1))) } } /** * This is the inverse of parsePathFragment(). */ def getPathFragment(spec: TablePartitionSpec, partitionSchema: StructType): String = { partitionSchema.map { field => escapePathName(field.name) + "=" + escapePathName(spec(field.name)) }.mkString("/") } def getPathFragment(spec: TablePartitionSpec, partitionColumns: Seq[Attribute]): String = { getPathFragment(spec, DataTypeUtils.fromAttributes(partitionColumns)) } /** * Normalize the column names in partition specification, w.r.t. the real partition column names * and case sensitivity. e.g., if the partition spec has a column named `monTh`, and there is a * partition column named `month`, and it's case insensitive, we will normalize `monTh` to * `month`. */ def normalizePartitionSpec[T]( partitionSpec: Map[String, T], partColNames: Seq[String], tblName: String, resolver: Resolver): Map[String, T] = { val normalizedPartSpec = partitionSpec.toSeq.map { case (key, value) => val normalizedKey = partColNames.find(resolver(_, key)).getOrElse { throw DeltaErrors.invalidPartitionColumn(key, tblName) } normalizedKey -> value } checkColumnNameDuplication( normalizedPartSpec.map(_._1), "in the partition schema", resolver) normalizedPartSpec.toMap } /** * Resolves possible type conflicts between partitions by up-casting "lower" types using * [[findWiderTypeForPartitionColumn]]. */ def resolvePartitions( pathsWithPartitionValues: Seq[(Path, PartitionValues)], caseSensitive: Boolean, timeZone: TimeZone): Seq[PartitionValues] = { if (pathsWithPartitionValues.isEmpty) { Seq.empty } else { val partColNames = if (caseSensitive) { pathsWithPartitionValues.map(_._2.columnNames) } else { pathsWithPartitionValues.map(_._2.columnNames.map(_.toLowerCase())) } assert( partColNames.distinct.size == 1, listConflictingPartitionColumns(pathsWithPartitionValues)) // Resolves possible type conflicts for each column val values = pathsWithPartitionValues.map(_._2) val columnCount = values.head.columnNames.size val resolvedValues = (0 until columnCount).map { i => resolveTypeConflicts(values.map(_.literals(i)), timeZone) } // Fills resolved literals back to each partition values.zipWithIndex.map { case (d, index) => d.copy(literals = resolvedValues.map(_(index))) } } } def listConflictingPartitionColumns( pathWithPartitionValues: Seq[(Path, PartitionValues)]): String = { val distinctPartColNames = pathWithPartitionValues.map(_._2.columnNames).distinct def groupByKey[K, V](seq: Seq[(K, V)]): Map[K, Iterable[V]] = seq.groupBy { case (key, _) => key }.mapValues(_.map { case (_, value) => value }).toMap val partColNamesToPaths = groupByKey(pathWithPartitionValues.map { case (path, partValues) => partValues.columnNames -> path }) val distinctPartColLists = distinctPartColNames.map(_.mkString(", ")).zipWithIndex.map { case (names, index) => s"Partition column name list #$index: $names" } // Lists out those non-leaf partition directories that also contain files val suspiciousPaths = distinctPartColNames.sortBy(_.length).flatMap(partColNamesToPaths) s"Conflicting partition column names detected:\n" + distinctPartColLists.mkString("\n\t", "\n\t", "\n\n") + "For partitioned table directories, data files should only live in leaf directories.\n" + "And directories at the same level should have the same partition column name.\n" + "Please check the following directories for unexpected files or " + "inconsistent partition column names:\n" + suspiciousPaths.map("\t" + _).mkString("\n", "\n", "") } // scalastyle:off line.size.limit /** * Converts a string to a [[Literal]] with automatic type inference. Currently only supports * [[NullType]], [[IntegerType]], [[LongType]], [[DoubleType]], [[DecimalType]], [[DateType]] * [[TimestampType]], and [[StringType]]. * * When resolving conflicts, it follows the table below: * * +--------------------+-------------------+-------------------+-------------------+--------------------+------------+---------------+---------------+------------+ * | InputA \ InputB | NullType | IntegerType | LongType | DecimalType(38,0)* | DoubleType | DateType | TimestampType | StringType | * +--------------------+-------------------+-------------------+-------------------+--------------------+------------+---------------+---------------+------------+ * | NullType | NullType | IntegerType | LongType | DecimalType(38,0) | DoubleType | DateType | TimestampType | StringType | * | IntegerType | IntegerType | IntegerType | LongType | DecimalType(38,0) | DoubleType | StringType | StringType | StringType | * | LongType | LongType | LongType | LongType | DecimalType(38,0) | StringType | StringType | StringType | StringType | * | DecimalType(38,0)* | DecimalType(38,0) | DecimalType(38,0) | DecimalType(38,0) | DecimalType(38,0) | StringType | StringType | StringType | StringType | * | DoubleType | DoubleType | DoubleType | StringType | StringType | DoubleType | StringType | StringType | StringType | * | DateType | DateType | StringType | StringType | StringType | StringType | DateType | TimestampType | StringType | * | TimestampType | TimestampType | StringType | StringType | StringType | StringType | TimestampType | TimestampType | StringType | * | StringType | StringType | StringType | StringType | StringType | StringType | StringType | StringType | StringType | * +--------------------+-------------------+-------------------+-------------------+--------------------+------------+---------------+---------------+------------+ * Note that, for DecimalType(38,0)*, the table above intentionally does not cover all other * combinations of scales and precisions because currently we only infer decimal type like * `BigInteger`/`BigInt`. For example, 1.1 is inferred as double type. * Note: [[columnValue]] is expected to be an column value unescaped from path escaping, * and [[DEFAULT_PARTITION_NAME]] should be replaced by null. */ // scalastyle:on line.size.limit def inferPartitionColumnValue( columnValue: String, typeInference: Boolean, timeZone: TimeZone, dateFormatter: DateFormatter, timestampFormatter: TimestampFormatter): Literal = { def decimalTry = Try { // `BigDecimal` conversion can fail when the `field` is not a form of number. val bigDecimal = new JBigDecimal(columnValue) // It reduces the cases for decimals by disallowing values having scale (eg. `1.1`). require(bigDecimal.scale <= 0) // `DecimalType` conversion can fail when // 1. The precision is bigger than 38. // 2. scale is bigger than precision. Literal(bigDecimal) } def dateTry = Try { // try and parse the date, if no exception occurs this is a candidate to be resolved as // DateType dateFormatter.parse(columnValue) // SPARK-23436: Casting the string to date may still return null if a bad Date is provided. // This can happen since DateFormat.parse may not use the entire text of the given string: // so if there are extra-characters after the date, it returns correctly. // We need to check that we can cast the raw string since we later can use Cast to get // the partition values with the right DataType (see // org.apache.spark.sql.execution.datasources.PartitioningAwareFileIndex.inferPartitioning) val dateValue = Cast(Literal(columnValue), DateType).eval() // Disallow DateType if the cast returned null require(dateValue != null) Literal.create(dateValue, DateType) } def timestampTry = Try { // try and parse the date, if no exception occurs this is a candidate to be resolved as // TimestampType timestampFormatter.parse(columnValue) // SPARK-23436: see comment for date val timestampValue = Cast(Literal(columnValue), TimestampType, Some(timeZone.getID)).eval() // Disallow TimestampType if the cast returned null require(timestampValue != null) Literal.create(timestampValue, TimestampType) } if (columnValue == null) { Literal.default(NullType) } else if (typeInference) { // First tries integral types Try(Literal.create(Integer.parseInt(columnValue), IntegerType)) .orElse(Try(Literal.create(JLong.parseLong(columnValue), LongType))) .orElse(decimalTry) // Then falls back to fractional types .orElse(Try(Literal.create(JDouble.parseDouble(columnValue), DoubleType))) // Then falls back to date/timestamp types .orElse(timestampTry) .orElse(dateTry) // Then falls back to string .getOrElse(Literal.create(columnValue, StringType)) } else { Literal.create(columnValue, StringType) } } def validatePartitionColumn( schema: StructType, partitionColumns: Seq[String], caseSensitive: Boolean): Unit = { checkColumnNameDuplication( partitionColumns, "in the partition columns", caseSensitive) partitionColumnsSchema(schema, partitionColumns, caseSensitive).foreach { field => field.dataType match { // Variant types are not orderable and thus cannot be partition columns. case a: AtomicType if !a.isInstanceOf[VariantType] => // OK case _ => throw DeltaErrors.cannotUseDataTypeForPartitionColumnError(field) } } if (partitionColumns.nonEmpty && partitionColumns.size == schema.fields.length) { throw new DeltaAnalysisException( errorClass = "DELTA_CANNOT_USE_ALL_COLUMNS_FOR_PARTITION", Array.empty) } } def partitionColumnsSchema( schema: StructType, partitionColumns: Seq[String], caseSensitive: Boolean): StructType = { val equality = columnNameEquality(caseSensitive) StructType(partitionColumns.map { col => schema.find(f => equality(f.name, col)).getOrElse { val schemaCatalog = schema.catalogString throw DeltaErrors.missingPartitionColumn(col, schemaCatalog) } }).asNullable } def mergeDataAndPartitionSchema( dataSchema: StructType, partitionSchema: StructType, caseSensitive: Boolean): (StructType, Map[String, StructField]) = { val overlappedPartCols = mutable.Map.empty[String, StructField] partitionSchema.foreach { partitionField => val partitionFieldName = getColName(partitionField, caseSensitive) if (dataSchema.exists(getColName(_, caseSensitive) == partitionFieldName)) { overlappedPartCols += partitionFieldName -> partitionField } } // When data and partition schemas have overlapping columns, the output // schema respects the order of the data schema for the overlapping columns, and it // respects the data types of the partition schema. // `HadoopFsRelation` will be mapped to `FileSourceScanExec`, which always output // all the partition columns physically. Here we need to make sure the final schema // contains all the partition columns. val fullSchema = StructType(dataSchema.map(f => overlappedPartCols.getOrElse(getColName(f, caseSensitive), f)) ++ partitionSchema.filterNot(f => overlappedPartCols.contains(getColName(f, caseSensitive)))) (fullSchema, overlappedPartCols.toMap) } def getColName(f: StructField, caseSensitive: Boolean): String = { if (caseSensitive) { f.name } else { f.name.toLowerCase(Locale.ROOT) } } private def columnNameEquality(caseSensitive: Boolean): (String, String) => Boolean = { if (caseSensitive) { org.apache.spark.sql.catalyst.analysis.caseSensitiveResolution } else { org.apache.spark.sql.catalyst.analysis.caseInsensitiveResolution } } /** * Given a collection of [[Literal]]s, resolves possible type conflicts by * [[findWiderTypeForPartitionColumn]]. */ private def resolveTypeConflicts(literals: Seq[Literal], timeZone: TimeZone): Seq[Literal] = { val litTypes = literals.map(_.dataType) val desiredType = litTypes.reduce(findWiderTypeForPartitionColumn) literals.map { case l @ Literal(_, dataType) => Literal.create(Cast(l, desiredType, Some(timeZone.getID)).eval(), desiredType) } } /** * Type widening rule for partition column types. It is similar to * [[TypeCoercion.findWiderTypeForTwo]] but the main difference is that here we disallow * precision loss when widening double/long and decimal, and fall back to string. */ private val findWiderTypeForPartitionColumn: (DataType, DataType) => DataType = { case (DoubleType, _: DecimalType) | (_: DecimalType, DoubleType) => StringType case (DoubleType, LongType) | (LongType, DoubleType) => StringType case (t1, t2) => TypeCoercion.findWiderTypeForTwo(t1, t2).getOrElse(StringType) } /** The methods below are forked from [[org.apache.spark.sql.util.SchemaUtils]] */ /** * Checks if input column names have duplicate identifiers. This throws an exception if * the duplication exists. * * @param columnNames column names to check * @param colType column type name, used in an exception message * @param resolver resolver used to determine if two identifiers are equal */ def checkColumnNameDuplication( columnNames: Seq[String], colType: String, resolver: Resolver): Unit = { checkColumnNameDuplication(columnNames, colType, isCaseSensitiveAnalysis(resolver)) } /** * Checks if input column names have duplicate identifiers. This throws an exception if * the duplication exists. * * @param columnNames column names to check * @param colType column type name, used in an exception message * @param caseSensitiveAnalysis whether duplication checks should be case sensitive or not */ def checkColumnNameDuplication( columnNames: Seq[String], colType: String, caseSensitiveAnalysis: Boolean): Unit = { // scalastyle:off caselocale val names = if (caseSensitiveAnalysis) columnNames else columnNames.map(_.toLowerCase) // scalastyle:on caselocale if (names.distinct.length != names.length) { val duplicateColumns = names.groupBy(identity).collect { case (x, ys) if ys.length > 1 => s"`$x`" } throw DeltaErrors.foundDuplicateColumnsException(colType, duplicateColumns.mkString(", ")) } } // Returns true if a given resolver is case-sensitive private def isCaseSensitiveAnalysis(resolver: Resolver): Boolean = { if (resolver == caseSensitiveResolution) { true } else if (resolver == caseInsensitiveResolution) { false } else { sys.error("A resolver to check if two identifiers are equal must be " + "`caseSensitiveResolution` or `caseInsensitiveResolution` in o.a.s.sql.catalyst.") } } /** * Converts a typed literal to a normalized string representation for correct comparisons. * @param literal The Literal to convert to a string. Must have the correct type set. * @param timeZoneId Optional timezone ID for timestamp types. * @param useUtcNormalizedTimestamp If true, formats timestamp types into UTC ISO 8601 format. * @return The string representation of the literal, or null if the literal is null. */ def literalToNormalizedString( literal: Literal, timeZoneId: Option[String] = None, useUtcNormalizedTimestamp: Boolean = false): String = { if (literal == null || literal.value == null) { return null } literal.dataType match { case TimestampType if useUtcNormalizedTimestamp => // Format timestamp in UTC ISO 8601 format: "2000-01-01T12:00:00.000000Z" utcFormatter.format(literal.value.asInstanceOf[Long]) case _ => // All other types can safely be converted to a string. val castedValue = Cast(literal, StringType, timeZoneId, ansiEnabled = false).eval() Option(castedValue).map(_.toString).orNull } } /** * Parses partition values (strings) to their corresponding Literal with the appropiate type * as defined in the partition schema. * * @param partValuesMap Map of partition column names to their string values. * @param partitionSchema Schema defining the data types for each partition column. * @param timeZoneId Time zone ID used for casting timestamp values. * @param validatePartitionColumns Throw an error when casting fails. * @return Map of partition column names to their parsed Literal values. */ def parsePartitionValues( partValuesMap: Map[String, String], partitionSchema: StructType, timeZoneId: String, validatePartitionColumns: Boolean = false): Map[String, Literal] = { val partSchemaNames = partitionSchema.names.toSet val partValuesMapKeys = partValuesMap.keySet if (partSchemaNames != partValuesMapKeys) { val errorMsg = s"Partition values map keys $partValuesMapKeys should match " + s"partition schema names $partSchemaNames." if (partSchemaNames.map(_.toLowerCase(Locale.ROOT)) == partValuesMapKeys.map(_.toLowerCase(Locale.ROOT))) { throw new IllegalStateException(s"CASE_MISMATCH: $errorMsg") } else if (partValuesMapKeys.isEmpty) { throw new IllegalStateException(s"PARTITION_VALUES_EMPTY: $errorMsg") } else { throw new IllegalStateException(errorMsg) } } val timeZone = DateTimeUtils.getTimeZone(timeZoneId) val dateFormatter = DateFormatter() val timestampFormatter = TimestampFormatter(timestampPartitionPattern, timeZone) val partColNameToDataType = mapNameToDataType(partitionSchema) partValuesMap.map { case (partColName, partValueStr) => val dataType = partColNameToDataType(partColName) val literal = parsePartitionValue( partColName, partValueStr, Some(dataType), typeInference = false, timeZone, dateFormatter, timestampFormatter, validatePartitionColumns) (partColName, literal) } } /** * Parses a single partition value string and returns a Literal with the appropriate type. * * @param columnName The name of the partition column (used for error messages). * @param rawValue The raw string value of the partition. * @param dataType Optional data type from the schema. If None, type inference is used. * @param typeInference Whether to infer the type when dataType is None. * @param timeZone Time zone for timestamp parsing. * @param dateFormatter Formatter for date parsing. * @param timestampFormatter Formatter for timestamp parsing. * @param validatePartitionColumns Throw an error when casting fails. * @return A Literal containing the parsed value. */ private def parsePartitionValue( columnName: String, columnValue: String, dataType: Option[DataType], typeInference: Boolean, timeZone: TimeZone, dateFormatter: DateFormatter, timestampFormatter: TimestampFormatter, validatePartitionColumns: Boolean = false): Literal = { dataType match { case Some(dt) => // If columnValue is null, return a null literal with the appropriate type. if (columnValue == null) { return Literal.create(null, dt) } // Fall back string literal val columnValueStringLiteral = Literal.create(columnValue, StringType) dt match { case TimestampType => Try { Literal.create( timestampFormatter.parse(columnValue), TimestampType) }.getOrElse { // If the timestamp is not in the expected format, cast it manually. val castedValue = Cast( columnValueStringLiteral, TimestampType, Option(timeZone.getID), ansiEnabled = false).eval() if (castedValue != null) { Literal.create(castedValue, TimestampType) } else if (validatePartitionColumns) { throw DeltaErrors.partitionColumnCastFailed( Option(columnValue).map(_.toString).getOrElse("null"), TimestampType.toString, columnName) } else { columnValueStringLiteral } } case _ => val castedValue = Cast( columnValueStringLiteral, dt, Option(timeZone.getID), ansiEnabled = false).eval() if (castedValue == null) { if (validatePartitionColumns) { throw DeltaErrors.partitionColumnCastFailed( Option(columnValue).map(_.toString).getOrElse("null"), dt.toString, columnName) } columnValueStringLiteral } else { Literal.create(castedValue, dt) } } case None => // No schema provided - use type inference inferPartitionColumnValue( columnValue, typeInference, timeZone, dateFormatter, timestampFormatter) } } // TODO: Use helpers from Spark when https://github.com/apache/spark/pull/54117 is available. private def mapNameToDataType(schema: StructType): Map[String, DataType] = schema.fields.map(f => f.name -> f.dataType).toMap def classifyPartitionValueParsingError(e: Throwable): String = e match { case ex if ex.getMessage.contains("CASE_MISMATCH") => ".caseMismatch" case ex if ex.getMessage.contains("PARTITION_VALUES_EMPTY") => ".partitionValuesEmpty" case _ => "" } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/PathWithFileSystem.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} /** * Bundling the `Path` with the `FileSystem` instance ensures * that we never pass the wrong file system with the path to a function * at compile time. */ case class PathWithFileSystem private (path: Path, fs: FileSystem) { /** * Extends the path with `s` * * The resulting path must be on the same filesystem. */ def withSuffix(s: String): PathWithFileSystem = new PathWithFileSystem(new Path(path, s), fs) /** * Qualify `path` using `fs` */ def makeQualified(): PathWithFileSystem = { val qualifiedPath = fs.makeQualified(path) PathWithFileSystem(qualifiedPath, fs) } } object PathWithFileSystem { /** * Create a new `PathWithFileSystem` instance by calling `getFileSystem` * on `path` with the given `hadoopConf`. */ def withConf(path: Path, hadoopConf: Configuration): PathWithFileSystem = { val fs = path.getFileSystem(hadoopConf) PathWithFileSystem(path, fs) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/SetAccumulator.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import org.apache.spark.util.AccumulatorV2 /** * Accumulator to collect distinct elements as a set. */ class SetAccumulator[T] extends AccumulatorV2[T, java.util.Set[T]] { private var _set: java.util.Set[T] = _ private def getOrCreate = { _set = Option(_set).getOrElse(java.util.Collections.synchronizedSet(new java.util.HashSet[T]())) _set } override def isZero: Boolean = this.synchronized(getOrCreate.isEmpty) override def reset(): Unit = this.synchronized { _set = null } override def add(v: T): Unit = this.synchronized(getOrCreate.add(v)) override def merge(other: AccumulatorV2[T, java.util.Set[T]]): Unit = other match { case o: SetAccumulator[T] => this.synchronized(getOrCreate.addAll(o.value)) case _ => throw new UnsupportedOperationException( s"Cannot merge ${this.getClass.getName} with ${other.getClass.getName}") } override def value: java.util.Set[T] = this.synchronized { java.util.Collections.unmodifiableSet(new java.util.HashSet[T](getOrCreate)) } override def copy(): AccumulatorV2[T, java.util.Set[T]] = { val newAcc = new SetAccumulator[T]() this.synchronized { newAcc.getOrCreate.addAll(getOrCreate) } newAcc } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/StateCache.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import scala.collection.mutable.ArrayBuffer import org.apache.spark.sql.delta.{DataFrameUtils, Snapshot} import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.rdd.RDD import org.apache.spark.sql.{DataFrame, Dataset, SparkSession} import org.apache.spark.sql.execution.{LogicalRDD, SQLExecution} import org.apache.spark.storage.StorageLevel /** * Machinery that caches the reconstructed state of a Delta table * using the RDD cache. The cache is designed so that the first access * will materialize the results. However, once uncache is called, * all data will be flushed and will not be cached again. */ trait StateCache extends DeltaLogging { protected def spark: SparkSession /** If state RDDs for this snapshot should still be cached. */ private var _isCached = true /** A list of RDDs that we need to uncache when we are done with this snapshot. */ private val cached = ArrayBuffer[RDD[_]]() private val cached_refs = ArrayBuffer[DatasetRefCache[_]]() /** Method to expose the value of _isCached for testing. */ private[delta] def isCached: Boolean = _isCached private val storageLevel = StorageLevel.fromString( spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_SNAPSHOT_CACHE_STORAGE_LEVEL)) class CachedDS[A] private[StateCache](ds: Dataset[A], name: String) { // While we cache RDD to avoid re-computation in different spark sessions, `Dataset` can only be // reused by the session that created it to avoid session pollution. So we use `DatasetRefCache` // to re-create a new `Dataset` when the active session is changed. This is an optimization for // single-session scenarios to avoid the overhead of `Dataset` creation which can take 100ms. private val cachedDs = cached.synchronized { if (isCached) { val qe = ds.queryExecution val rdd = SQLExecution.withNewExecutionId(qe, Some(s"Cache $name")) { val rdd = recordFrameProfile("Delta", "CachedDS.toRdd") { // toRdd should always trigger execution qe.toRdd.map(_.copy()) } rdd.setName(name) rdd.persist(storageLevel) } cached += rdd val dsCache = datasetRefCache { () => val logicalRdd = LogicalRDD(qe.analyzed.output, rdd)(spark) DataFrameUtils.ofRows(spark, logicalRdd) } Some(dsCache) } else { None } } /** * Retrieves the cached RDD in Dataframe form. * * If a RDD cache is available, * - return the cached DF if called from the same session in which the cached DF is created, or * - reconstruct the DF using the RDD cache if called from a different session. * * If no RDD cache is available, * - return a copy of the original DF with updated spark session. * * Since a cached DeltaLog can be accessed from multiple Spark sessions, this interface makes * sure that the original Spark session in the cached DF does not leak into the current active * sessions. */ def getDF: DataFrame = { if (cached.synchronized(isCached) && cachedDs.isDefined) { cachedDs.get.get } else { DataFrameUtils.ofRows(spark, ds.queryExecution.logical) } } /** * Retrieves the cached RDD as a strongly-typed Dataset. */ def getDS: Dataset[A] = getDF.as(ds.encoder) } /** * Create a CachedDS instance for the given Dataset and the name. */ def cacheDS[A](ds: Dataset[A], name: String): CachedDS[A] = recordFrameProfile( "Delta", "CachedDS.cacheDS") { new CachedDS[A](ds, name) } def datasetRefCache[A](creator: () => Dataset[A]): DatasetRefCache[A] = { val dsCache = new DatasetRefCache(creator) cached_refs += dsCache dsCache } /** Drop any cached data for this [[Snapshot]]. */ def uncache(): Unit = cached.synchronized { if (isCached) { _isCached = false cached.foreach(_.unpersist(blocking = false)) cached_refs.foreach(_.invalidate()) } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/TimestampFormatter.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import java.text.ParseException import java.time._ import java.time.format.DateTimeParseException import java.time.temporal.TemporalQueries import java.util.{Locale, TimeZone} import org.apache.spark.sql.delta.util.DateTimeUtils.instantToMicros /** * Forked from [[org.apache.spark.sql.catalyst.util.TimestampFormatter]] */ sealed trait TimestampFormatter extends Serializable { /** * Parses a timestamp in a string and converts it to microseconds. * * @param s - string with timestamp to parse * @return microseconds since epoch. * @throws ParseException can be thrown by legacy parser * @throws DateTimeParseException can be thrown by new parser * @throws DateTimeException unable to obtain local date or time */ @throws(classOf[ParseException]) @throws(classOf[DateTimeParseException]) @throws(classOf[DateTimeException]) def parse(s: String): Long def format(us: Long): String } class Iso8601TimestampFormatter( pattern: String, timeZone: ZoneId, locale: Locale) extends TimestampFormatter with DateTimeFormatterHelper { @transient protected lazy val formatter = getOrCreateFormatter(pattern, locale) private def toInstant(s: String): Instant = { val temporalAccessor = formatter.parse(s) if (temporalAccessor.query(TemporalQueries.offset()) == null) { toInstantWithZoneId(temporalAccessor, timeZone) } else { Instant.from(temporalAccessor) } } override def parse(s: String): Long = instantToMicros(toInstant(s)) override def format(us: Long): String = { val instant = DateTimeUtils.microsToInstant(us) formatter.withZone(timeZone).format(instant) } } /** * The formatter parses/formats timestamps according to the pattern `yyyy-MM-dd HH:mm:ss.[..fff..]` * where `[..fff..]` is a fraction of second up to microsecond resolution. The formatter does not * output trailing zeros in the fraction. For example, the timestamp `2019-03-05 15:00:01.123400` is * formatted as the string `2019-03-05 15:00:01.1234`. * * @param timeZone the time zone in which the formatter parses or format timestamps */ class FractionTimestampFormatter(timeZone: TimeZone) extends Iso8601TimestampFormatter("", timeZone.toZoneId, TimestampFormatter.defaultLocale) { @transient override protected lazy val formatter = DateTimeFormatterHelper.fractionFormatter } object TimestampFormatter { val defaultPattern: String = "yyyy-MM-dd HH:mm:ss" val defaultLocale: Locale = Locale.US def apply(format: String, zoneId: ZoneId): TimestampFormatter = { new Iso8601TimestampFormatter(format, zoneId, defaultLocale) } def apply(format: String, timeZone: TimeZone, locale: Locale): TimestampFormatter = { new Iso8601TimestampFormatter(format, timeZone.toZoneId, locale) } def apply(format: String, timeZone: TimeZone): TimestampFormatter = { apply(format, timeZone, defaultLocale) } def apply(timeZone: TimeZone): TimestampFormatter = { apply(defaultPattern, timeZone, defaultLocale) } def getFractionFormatter(timeZone: TimeZone): TimestampFormatter = { new FractionTimestampFormatter(timeZone) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/TransactionHelper.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import scala.util.control.NonFatal import org.apache.spark.sql.delta.skipping.clustering.{ClusteredTableUtils, ClusteringColumnInfo} import org.apache.spark.sql.delta.{CatalogOwnedTableFeature, CommitStats, CommittedTransaction, CoordinatedCommitsStats, CoordinatedCommitType, DeltaConfigs, DeltaLog, IsolationLevel, Serializable, Snapshot, WriteSerializable} import org.apache.spark.sql.delta.DeltaOperations.Operation import org.apache.spark.sql.delta.RowId.RowTrackingMetadataDomain import org.apache.spark.sql.delta.actions.{Action, AddCDCFile, AddFile, CommitInfo, DomainMetadata, FileAction, Metadata, Protocol, RemoveFile, SetTransaction} import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, TableCommitCoordinatorClient} import org.apache.spark.sql.delta.hooks.PostCommitHook import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.spark.internal.MDC import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.catalog.CatalogTable /** * Contains helper methods for Delta transactions. */ trait TransactionHelper extends DeltaLogging { def deltaLog: DeltaLog def catalogTable: Option[CatalogTable] def snapshot: Snapshot /** Unique identifier for the transaction */ def txnId: String /** * Returns the metadata for this transaction. The metadata refers to the metadata of the snapshot * at the transaction's read version unless updated during the transaction. */ def metadata: Metadata /** The protocol of the snapshot that this transaction is reading at. */ def protocol: Protocol /** * Default [[IsolationLevel]] as set in table metadata. */ private[delta] def getDefaultIsolationLevel(): IsolationLevel = { DeltaConfigs.ISOLATION_LEVEL.fromMetaData(metadata) } /** * Determines if a transaction can downgrade to SnapshotIsolation. * * Note-1: For no-data-change transactions such as OPTIMIZE/Auto Compaction/ZorderBY, we can * change the isolation level to SnapshotIsolation. SnapshotIsolation allows reduced conflict * detection by skipping the * [[ConflictChecker.checkForAddedFilesThatShouldHaveBeenReadByCurrentTxn]] check i.e. * don't worry about concurrent appends. * * Note-2: * We can also use SnapshotIsolation for empty transactions. e.g. consider a commit: * t0 - Initial state of table * t1 - Q1, Q2 starts * t2 - Q1 commits * t3 - Q2 is empty and wants to commit. * In this scenario, we can always allow Q2 to commit without worrying about new files * generated by Q1. * The final order which satisfies both Serializability and WriteSerializability is: Q2, Q1. * Note that Metadata only update transactions shouldn't be considered empty. If Q2 above has * a Metadata update (say schema change/identity column high watermark update), then Q2 can't * be moved above Q1 in the final SERIALIZABLE order. This is because if Q2 is moved above Q1, * then Q1 should see the updates from Q2 - which actually didn't happen. * * @param actions The sequence of actions being committed. * @param opChangesData Whether the operation changes data (from DeltaOperations.Operation). * @return true if the isolation level can be downgraded to SnapshotIsolation. */ private[delta] def canDowngradeToSnapshotIsolation( actions: Seq[Action], opChangesData: Boolean): Boolean = { var dataChanged = false var hasIncompatibleActions = false actions.foreach { case f: FileAction => if (f.dataChange) { dataChanged = true } // Row tracking is able to resolve write conflicts regardless of isolation level. case d: DomainMetadata if RowTrackingMetadataDomain.isSameDomain(d) => // Do nothing case _ => hasIncompatibleActions = true } if (hasIncompatibleActions) { // If incompatible actions are present (e.g. METADATA etc.), then don't downgrade the // isolation level to SnapshotIsolation. return false } val noDataChanged = !dataChanged val defaultIsolationLevel = getDefaultIsolationLevel() val allowFallbackToSnapshotIsolation = defaultIsolationLevel match { case Serializable => noDataChanged case WriteSerializable => noDataChanged && !opChangesData case _ => false // This case should never happen. } allowFallbackToSnapshotIsolation } /** * Return the user-defined metadata for the operation. */ def getUserMetadata(op: Operation): Option[String] = { // option wins over config if both are set op.userMetadata match { case data @ Some(_) => data case None => spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_USER_METADATA) } } /** The current spark session */ protected def spark: SparkSession = SparkSession.active private[delta] lazy val readSnapshotTableCommitCoordinatorClientOpt: Option[TableCommitCoordinatorClient] = { if (snapshot.isCatalogOwned) { // Catalog owned table's commit coordinator is always determined by the catalog. CatalogOwnedTableUtils.populateTableCommitCoordinatorFromCatalog( spark, catalogTable, snapshot) } else { // The commit-coordinator of a table shouldn't change. If it is changed by a concurrent // commit, then it will be detected as a conflict and the transaction will anyway fail. snapshot.getTableCommitCoordinatorForWrites } } def createCoordinatedCommitsStats(newProtocol: Option[Protocol]) : CoordinatedCommitsStats = { val (coordinatedCommitsType, metadataToUse) = readSnapshotTableCommitCoordinatorClientOpt match { // TODO: Capture the CO -> FS downgrade case when we start // supporting downgrade for CO. case Some(_) if snapshot.isCatalogOwned => // CO commit (CoordinatedCommitType.CO_COMMIT, snapshot.metadata) case Some(_) if metadata.coordinatedCommitsCoordinatorName.isEmpty => // CC -> FS (CoordinatedCommitType.CC_TO_FS_DOWNGRADE_COMMIT, snapshot.metadata) // Only the 0th commit to a table can be a FS -> CO upgrade for now. // Upgrading an existing FS table to CO through ALTER TABLE is not supported yet. case None if newProtocol.exists(_.readerAndWriterFeatureNames .contains(CatalogOwnedTableFeature.name)) => // FS -> CO (CoordinatedCommitType.FS_TO_CO_UPGRADE_COMMIT, metadata) case None if metadata.coordinatedCommitsCoordinatorName.isDefined => // FS -> CC (CoordinatedCommitType.FS_TO_CC_UPGRADE_COMMIT, metadata) case Some(_) => // CC commit (CoordinatedCommitType.CC_COMMIT, snapshot.metadata) case None => // FS commit (CoordinatedCommitType.FS_COMMIT, snapshot.metadata) // Errors out in rest of the cases. case _ => throw new IllegalStateException( "Unexpected state found when trying " + s"to generate CoordinatedCommitsStats for table ${deltaLog.logPath}. " + s"$readSnapshotTableCommitCoordinatorClientOpt, " + s"$metadata, $snapshot, $catalogTable") } CoordinatedCommitsStats( coordinatedCommitsType = coordinatedCommitsType.toString, commitCoordinatorName = if (Set(CoordinatedCommitType.CO_COMMIT, CoordinatedCommitType.FS_TO_CO_UPGRADE_COMMIT).contains(coordinatedCommitsType)) { // The catalog for FS -> CO upgrade commit would be // "CATALOG_EMPTY" because `catalogTable` is not available // for the 0th FS commit. catalogTable.flatMap { ct => CatalogOwnedTableUtils.getCatalogName( spark, identifier = ct.identifier) }.getOrElse("CATALOG_MISSING") } else { metadataToUse.coordinatedCommitsCoordinatorName.getOrElse("NONE") }, // For Catalog-Owned table, the coordinator conf for UC-CC is [[Map.empty]] // so we don't distinguish between CO/CC here. commitCoordinatorConf = metadataToUse.coordinatedCommitsCoordinatorConf) } /** * Determines if we should checkpoint the version that has just been committed. */ protected def isCheckpointNeeded( committedVersion: Long, postCommitSnapshot: Snapshot): Boolean = { def checkpointInterval = deltaLog.checkpointInterval(postCommitSnapshot.metadata) committedVersion != 0 && committedVersion % checkpointInterval == 0 } /** Runs a post-commit hook, handling any exceptions that occur. */ protected def runPostCommitHook( hook: PostCommitHook, committedTransaction: CommittedTransaction): Unit = { val version = committedTransaction.committedVersion try { hook.run(spark, committedTransaction) } catch { case NonFatal(e) => logWarning(log"Error when executing post-commit hook " + log"${MDC(DeltaLogKeys.HOOK_NAME, hook.name)} " + log"for commit ${MDC(DeltaLogKeys.VERSION, version)}", e) recordDeltaEvent(deltaLog, "delta.commit.hook.failure", data = Map( "hook" -> hook.name, "version" -> version, "exception" -> e.toString )) hook.handleError(spark, e, version) } } /** * Generates a timestamp which is greater than the commit timestamp * of the last snapshot. Note that this is only needed when the * feature `inCommitTimestamps` is enabled. */ protected[delta] def generateInCommitTimestampForFirstCommitAttempt( currentTimestamp: Long): Option[Long] = Option.when(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(metadata)) { val lastCommitTimestamp = snapshot.timestamp math.max(currentTimestamp, lastCommitTimestamp + 1) } /** * Computes and emits commit stats for the current transaction. */ class CommitStatsComputer { private var bytesNew: Long = 0L private var numAdd: Int = 0 private var numOfDomainMetadatas: Int = 0 private var numRemove: Int = 0 private var numSetTransaction: Int = 0 private var numCdcFiles: Int = 0 private var cdcBytesNew: Long = 0L private var numAbsolutePaths = 0 // We don't expect commits to have more than 2 billion actions private var numActions: Int = 0 private val partitionsAdded = mutable.HashSet.empty[Map[String, String]] private var newProtocolOpt = Option.empty[Protocol] private var newMetadataOpt = Option.empty[Metadata] private var inputActionsIteratorOpt = Option.empty[Iterator[Action]] private def assertStateBeforeFinalization(): Unit = { assert( inputActionsIteratorOpt.isDefined, "addToCommitStats must be called before finalizing commit stats") assert( !inputActionsIteratorOpt.get.hasNext, "The actions iterator must be consumed before finalizing commit stats") } /** * Takes in an iterator of actions and processes them to compute commit stats. * The commit stats are computed as a side effect of the iterator processing * and are only populated after the returned iterator is fully consumed. * Note that this function will not consume the input iterator. * @param actions An iterator of actions that are being committed in this transaction. * @return An iterator of actions. This is will return the same actions as the * input iterator, but with the commit stats computed as a side effect. */ def addToCommitStats(actions: Iterator[Action]): Iterator[Action] = { assert(inputActionsIteratorOpt.isEmpty, "addToCommitStats should only be called once per transaction") inputActionsIteratorOpt = Some(actions) actions.map { action => numActions += 1 action match { case a: AddFile => numAdd += 1 if (a.pathAsUri.isAbsolute) numAbsolutePaths += 1 partitionsAdded += a.partitionValues if (a.dataChange) bytesNew += a.size case r: RemoveFile => numRemove += 1 case c: AddCDCFile => numCdcFiles += 1 cdcBytesNew += c.size case _: SetTransaction => numSetTransaction += 1 case _: DomainMetadata => numOfDomainMetadatas += 1 case m: Metadata => newMetadataOpt = Some(m) case p: Protocol => newProtocolOpt = Some(p) case _ => () } action } } // scalastyle:off argcount /** * Finalizes the commit stats and emits them as a Delta event. * This must be called after * 1. [[addToCommitStats]] has been called on the actions iterator AND * 2. after the actions iterator returned by [[addToCommitStats]] * has been fully consumed. * @param spark The Spark session. * @param attemptVersion The version of the table which is being written to the log. * @param startVersion The version of the table which was read at the start of the transaction. * @param commitDurationMs The duration of the commit in milliseconds. * @param fsWriteDurationMs The duration of the file system write of the commit in milliseconds. * @param txnExecutionTimeMs The total execution time of the transaction in milliseconds. * @param stateReconstructionDurationMs The duration of post commit snapshot construction in * milliseconds. * @param postCommitSnapshot The snapshot constructed after the commit. * @param computedNeedsCheckpoint Whether a checkpoint needs to be created after this commit. * Computed in `setNeedsCheckpoint`. * @param isolationLevel The isolation level used for this transaction. * @param commitSizeBytes The total size of the commit in bytes, computed as the sum of * the JSON sizes of all actions in the commit. * @param commitInfo The commit info for this transaction. * @return A HashSet containing the partitions that were added in the transaction. */ def finalizeAndEmitCommitStats( spark: SparkSession, attemptVersion: Long, startVersion: Long, commitDurationMs: Long, fsWriteDurationMs: Long, txnExecutionTimeMs: Long, stateReconstructionDurationMs: Long, postCommitSnapshot: Snapshot, computedNeedsCheckpoint: Boolean, isolationLevel: IsolationLevel, commitInfoOpt: Option[CommitInfo], commitSizeBytes: Long): Unit = { assertStateBeforeFinalization() val doCollectCommitStats = computedNeedsCheckpoint || spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_FORCE_ALL_COMMIT_STATS) // Stats that force an expensive snapshot state reconstruction: val numFilesTotal = if (doCollectCommitStats) postCommitSnapshot.numOfFiles else -1L val sizeInBytesTotal = if (doCollectCommitStats) postCommitSnapshot.sizeInBytes else -1L val commitInfoToEmit = commitInfoOpt match { case Some(ci) => ci.copy(readVersion = None, isolationLevel = None) case None => null } val stats = CommitStats( startVersion = startVersion, commitVersion = attemptVersion, readVersion = postCommitSnapshot.version, txnDurationMs = txnExecutionTimeMs, commitDurationMs = commitDurationMs, fsWriteDurationMs = fsWriteDurationMs, stateReconstructionDurationMs = stateReconstructionDurationMs, numAdd = numAdd, numRemove = numRemove, numSetTransaction = numSetTransaction, bytesNew = bytesNew, numFilesTotal = numFilesTotal, sizeInBytesTotal = sizeInBytesTotal, numCdcFiles = numCdcFiles, cdcBytesNew = cdcBytesNew, protocol = postCommitSnapshot.protocol, commitSizeBytes = commitSizeBytes, checkpointSizeBytes = postCommitSnapshot.checkpointSizeInBytes(), totalCommitsSizeSinceLastCheckpoint = postCommitSnapshot.deltaFileSizeInBytes(), checkpointAttempt = computedNeedsCheckpoint, info = commitInfoToEmit, newMetadata = newMetadataOpt, numAbsolutePathsInAdd = numAbsolutePaths, numDistinctPartitionsInAdd = partitionsAdded.size, numPartitionColumnsInTable = postCommitSnapshot.metadata.partitionColumns.size, isolationLevel = isolationLevel.toString, coordinatedCommitsInfo = createCoordinatedCommitsStats(newProtocolOpt), numOfDomainMetadatas = numOfDomainMetadatas, txnId = Some(txnId)) recordDeltaEvent(deltaLog, DeltaLogging.DELTA_COMMIT_STATS_OPTYPE, data = stats) } /** * Returns the partitions that were added in this transaction. */ def getPartitionsAddedByTransaction: mutable.HashSet[Map[String, String]] = { assertStateBeforeFinalization() partitionsAdded } /** * Returns the number of actions that were added in this transaction. */ def getNumActions: Int = { assertStateBeforeFinalization() numActions } } // scalastyle:on argcount } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/Utils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import scala.util.Random import org.apache.spark.sql.delta.{DeltaConfigs, DeltaLog} import org.apache.spark.sql.delta.actions.Metadata import org.apache.hadoop.fs.Path import org.apache.spark.sql.{functions, Column, Dataset} import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.ElementAt /** * Various utility methods used by Delta. */ object Utils { /** Measures the time taken by function `f` */ def timedMs[T](f: => T): (T, Long) = { val start = System.currentTimeMillis() val res = f val duration = System.currentTimeMillis() - start (res, duration) } /** Returns the length of the random prefix to use for the data files of a Delta table. */ def getRandomPrefixLength(metadata: Metadata): Int = { if (DeltaConfigs.RANDOMIZE_FILE_PREFIXES.fromMetaData(metadata)) { DeltaConfigs.RANDOM_PREFIX_LENGTH.fromMetaData(metadata) } else { 0 } } /** Generates a string created of `randomPrefixLength` alphanumeric characters. */ def getRandomPrefix(numChars: Int): String = { Random.alphanumeric.take(numChars).mkString } /** * Construct a delta log from either the catalog table or a path. * * If catalogTableOpt is defined, use it to construct the delta log; otherwise, fall back to use * path-based delta log construction. */ def getDeltaLogFromTableOrPath( sparkSession: SparkSession, catalogTableOpt: Option[CatalogTable], path: Path, options: Map[String, String] = Map.empty): DeltaLog = { catalogTableOpt .map(catalogTable => DeltaLog.forTable(sparkSession, catalogTable, options)) .getOrElse(DeltaLog.forTable(sparkSession, path, options)) } /** * Indicates whether Delta is currently running unit tests. */ def isTesting: Boolean = { System.getenv("DELTA_TESTING") != null } /** * Returns value for the given key in value if column is a map and the key is present, NULL * otherwise. */ def try_element_at(mapColumn: Column, key: Any): Column = { functions.try_element_at(mapColumn, functions.lit(key)) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/threads/DeltaThreadPool.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util.threads import java.util.concurrent._ import scala.concurrent.duration.Duration import scala.util.control.NonFatal import org.apache.spark.sql.delta.DeltaErrors import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.SparkException import org.apache.spark.sql.SparkSession import org.apache.spark.util.ThreadUtils import org.apache.spark.util.ThreadUtils.namedThreadFactory /** A wrapper for [[ThreadPoolExecutor]] whose tasks run with the caller's [[SparkSession]]. */ private[delta] class DeltaThreadPool(tpe: ThreadPoolExecutor) { def getActiveCount: Int = tpe.getActiveCount def getMaximumPoolSize: Int = tpe.getMaximumPoolSize /** Submits a task for execution and returns a [[Future]] representing that task. */ def submit[T](spark: SparkSession)(body: => T): Future[T] = { tpe.submit { () => spark.withActive(body) } } /** * Executes `f` on each element of `items` as a task and returns the result. * Throws [[SparkException]] on error. */ def parallelMap[T, R]( spark: SparkSession, items: Iterable[T])( f: T => R): Iterable[R] = { // Materialize a list of futures, to ensure they all got submitted before we start waiting. val futures = items.map(i => submit(spark)(f(i))).toList futures.map(f => ThreadUtils.awaitResult(f, Duration.Inf)).toSeq } def submitNonFateSharing[T](f: SparkSession => T): NonFateSharingFuture[T] = new NonFateSharingFuture(this)(f) } /** Convenience constructor that creates a [[ThreadPoolExecutor]] with sensible defaults. */ private[delta] object DeltaThreadPool { def apply(prefix: String, numThreads: Int): DeltaThreadPool = new DeltaThreadPool(newDaemonCachedThreadPool(prefix, numThreads)) /** * Create a cached thread pool whose max number of threads is `maxThreadNumber`. Thread names * are formatted as prefix-ID, where ID is a unique, sequentially assigned integer. */ def newDaemonCachedThreadPool( prefix: String, maxThreadNumber: Int): ThreadPoolExecutor = { val keepAliveSeconds = 60 val queueSize = Integer.MAX_VALUE val threadFactory = namedThreadFactory(prefix) val threadPool = new SparkThreadLocalForwardingThreadPoolExecutor( maxThreadNumber, // corePoolSize: the max number of threads to create before queuing the tasks maxThreadNumber, // maximumPoolSize: because we use LinkedBlockingDeque, this one is not used keepAliveSeconds, TimeUnit.SECONDS, new LinkedBlockingQueue[Runnable](queueSize), threadFactory) threadPool.allowCoreThreadTimeOut(true) threadPool } } /** * A future invocation of `f` which avoids "fate sharing" of errors, in case multiple threads could * wait on the future's result. * * The future is only launched if a [[SparkSession]] is available. * * If the future succeeds, any thread can consume the result. * * If the future fails, threads will just invoke `f` directly -- except that fatal errors will * propagate (once) if the caller is from the same [[SparkSession]] that created the future. */ class NonFateSharingFuture[T](pool: DeltaThreadPool)(f: SparkSession => T) extends DeltaLogging { // Submit `f` as a future if a spark session is available @volatile private var futureOpt = SparkSession.getActiveSession.map { spark => spark -> pool.submit(spark) { f(spark) } } def get(timeout: Duration): T = { // Prefer to get a prefetched result from the future, but never fail because of it. val futureResult = futureOpt.flatMap { case (ownerSession, future) => try { val result = Some(ThreadUtils.awaitResult(future, timeout)) // no reason to keep the reference to the calling session anymore futureOpt = Some(null, future) result } catch { // NOTE: ThreadUtils.awaitResult wraps all non-fatal exceptions other than TimeoutException // with SparkException. Meanwhile, Java Future.get only throws four exceptions: // ExecutionException (non-fatal, wrapped, and itself wraps any Throwable from the task // itself), CancellationException (non-fatal, wrapped), InterruptedException (fatal, not // wrapped), and TimeoutException (non-fatal, but not wrapped). Thus, any "normal" failure // of the future will surface as SparkException(ExecutionException(OriginalException)). case outer: SparkException => outer.getCause match { case e: CancellationException => logWarning(log"Future was cancelled") futureOpt = None None case inner: ExecutionException if inner.getCause != null => inner.getCause match { case NonFatal(e) => logWarning(log"Future threw non-fatal exception", e) futureOpt = None None case e: Throwable => logWarning(log"Future threw fatal error", e) if (ownerSession eq SparkSession.active) { futureOpt = None throw e } None } } case e: TimeoutException => logWarning(log"Timed out waiting for future") None case NonFatal(e) => logWarning(log"Unknown failure while waiting for future", e) None } } futureResult.getOrElse { // Future missing or failed, so fall back to direct execution. SparkSession.getActiveSession match { case Some(spark) => f(spark) case _ => throw DeltaErrors.sparkSessionNotSetException() } } } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/util/threads/SparkThreadLocalForwardingThreadPoolExecutor.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util.threads import java.util.Properties import java.util.concurrent._ import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.{SparkContext, TaskContext} import org.apache.spark.internal.{Logging, MDC} import org.apache.spark.util.{Utils => SparkUtils} /** * Implementation of ThreadPoolExecutor that captures the Spark ThreadLocals present at submit time * and inserts them into the thread before executing the provided runnable. */ class SparkThreadLocalForwardingThreadPoolExecutor( corePoolSize: Int, maximumPoolSize: Int, keepAliveTime: Long, unit: TimeUnit, workQueue: BlockingQueue[Runnable], threadFactory: ThreadFactory, rejectedExecutionHandler: RejectedExecutionHandler = new ThreadPoolExecutor.AbortPolicy) extends ThreadPoolExecutor( corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, rejectedExecutionHandler) { override def execute(command: Runnable): Unit = super.execute(new SparkThreadLocalCapturingRunnable(command)) } trait SparkThreadLocalCapturingHelper extends Logging { // At the time of creating this instance we capture the task context and command context. val capturedTaskContext = TaskContext.get() val sparkContext = SparkContext.getActive // Capture an immutable threadsafe snapshot of the current local properties val capturedProperties = sparkContext .map(sc => CapturedSparkThreadLocals.toValuesArray( SparkUtils.cloneProperties(sc.getLocalProperties))) def runWithCaptured[T](body: => T): T = { // Save the previous contexts, overwrite them with the captured contexts, and then restore the // previous when execution completes. // This has the unfortunate side effect of writing nulls to these thread locals if they were // empty beforehand. val previousTaskContext = TaskContext.get() val previousProperties = sparkContext.map(_.getLocalProperties) TaskContext.setTaskContext(capturedTaskContext) for { p <- capturedProperties sc <- sparkContext } { sc.setLocalProperties(CapturedSparkThreadLocals.toProperties(p)) } try { body } catch { case t: Throwable => logError(log"Exception in thread " + log"${MDC(DeltaLogKeys.THREAD_NAME, Thread.currentThread().getName)}", t) throw t } finally { TaskContext.setTaskContext(previousTaskContext) for { p <- previousProperties sc <- sparkContext } { sc.setLocalProperties(p) } } } } class CapturedSparkThreadLocals extends SparkThreadLocalCapturingHelper object CapturedSparkThreadLocals { def apply(): CapturedSparkThreadLocals = { new CapturedSparkThreadLocals() } def toProperties(props: Array[(String, String)]): Properties = { val resultProps = new Properties() for ((key, value) <- props) { resultProps.put(key, value) } resultProps } def toValuesArray(props: Properties): Array[(String, String)] = { props.asScala.toArray } } class SparkThreadLocalCapturingRunnable(runnable: Runnable) extends Runnable with SparkThreadLocalCapturingHelper { override def run(): Unit = { runWithCaptured(runnable.run()) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/v2/interop/AbstractCommitInfo.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.v2.interop /** * Abstract trait for commit info actions in Delta. This trait provides a common * abstraction that can be implemented by both Spark's V1 CommitInfo and Kernel's CommitInfo * in V2 connector. The V2 connector will implement adapters for reusing V1 utilities. */ trait AbstractCommitInfo { /** * Get the in-commit timestamp of the commit as milliseconds after the epoch. * This is the timestamp recorded in the commit itself, used for time travel. */ def getCommitTimestamp: Long } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/v2/interop/AbstractMetadata.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.v2.interop import org.apache.spark.sql.types.StructType /** * Abstract trait for metadata actions in Delta. This trait provides a common * abstraction that can be implemented by both Spark's V1 Metadata and Kernel's MetadataV2 * in V2 connector. The V2 connector will implement adapters for reusing V1 utilities. */ trait AbstractMetadata { /** A unique table identifier. */ def id: String /** User-specified table identifier. */ def name: String /** User-specified table description. */ def description: String /** Returns the schema as a [[StructType]]. */ def schema: StructType /** List of partition column names. */ def partitionColumns: Seq[String] /** The table properties/configuration defined on the table. */ def configuration: Map[String, String] } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/v2/interop/AbstractProtocol.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.v2.interop /** * Abstract trait for protocol actions in Delta. This trait provides a common * abstraction that can be implemented by both Spark's V1 Protocol and Kernel's Protocol * in V2 connector. The V2 connector will implement adapters for reusing V1 utilities. */ trait AbstractProtocol { /** The minimum reader version required to read the table. */ def minReaderVersion: Int /** The minimum writer version required to write to the table. */ def minWriterVersion: Int /** * The reader features that need to be supported to read the table. * Returns None if table features are not enabled for readers. */ def readerFeatures: Option[Set[String]] /** * The writer features that need to be supported to write to the table. * Returns None if table features are not enabled for writers. */ def writerFeatures: Option[Set[String]] } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/v2/interop/README.md ================================================ # V2 Connector Interop This package contains abstract traits that enable interoperability between the Delta Spark connectors. ## Purpose These abstractions allow V1 utilities to be reused by the V2 connector through adapters. ## Usage For reusing V1 connector code in V2: 1. **Refactor V1 utilities** to depend on abstract traits (e.g., `AbstractMetadata`, `AbstractProtocol`) instead of the concrete V1 implementations. 2. **Implement adapters in V2 connector** that extend these abstractions, wrapping Kernel's action types (e.g., Kernel's `Metadata` → `AbstractMetadata`). ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/delta/zorder/ZCubeInfo.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.zorder import java.util.UUID import org.apache.spark.sql.delta.{DeltaErrors, DeltaLog, SnapshotIsolation} import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.actions.AddFile.Tags.{ZCUBE_ID, ZCUBE_ZORDER_BY, ZCUBE_ZORDER_CURVE} import org.apache.spark.sql.delta.commands.DeltaCommand import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.sql.delta.zorder.ZCubeInfo.ZCubeID import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.{Expression, Literal} /** * [[ZCube]] identifying information. * @param zCubeID unique identifier of a OPTIMIZE ZORDER BY command run * @param zOrderBy list of ZORDER BY columns for the run * @note This piece of info could be put in table Metadata or taken from CommitInfo * but is currently inlined here for simplicity. */ case class ZCubeInfo(zCubeID: ZCubeID, zOrderBy: Seq[String]) { require(zOrderBy.nonEmpty) } object ZCubeInfo extends DeltaCommand { type ZCubeID = String // Could be UUID, but there's no implicit encoding for that. /** * Preferred way of creating a ZCubeInfo for a new ZCube. * Automatically generates a unique zCubeID. */ def apply(zOrderBy: Seq[String]): ZCubeInfo = { val zCubeID = UUID.randomUUID.toString ZCubeInfo(zCubeID, zOrderBy) } private val ZCUBE_ID_KEY = AddFile.tag(ZCUBE_ID) private val ZORDER_BY_KEY = AddFile.tag(ZCUBE_ZORDER_BY) private val ZORDER_CURVE = AddFile.tag(ZCUBE_ZORDER_CURVE) /** * Serializes the given `zCubeInfo` to a Map[String, String] that can be used as or merged into * [[AddFile.tags]]. */ def toAddFileTags(zCubeInfo: ZCubeInfo): Map[String, String] = { Map( ZCUBE_ID_KEY -> zCubeInfo.zCubeID, ZORDER_BY_KEY -> JsonUtils.toJson(zCubeInfo.zOrderBy)) } /** * Deserializes a `ZCubeInfo` object from an [[AddFile.tags]] map, if present. */ def fromAddFileTags(tags: Map[String, String]): Option[ZCubeInfo] = { for { zCubeID <- tags.get(ZCUBE_ID_KEY) zOrderByColsAsJson <- tags.get(ZORDER_BY_KEY) } yield { val zOrderByCols = JsonUtils.fromJson[Seq[String]](zOrderByColsAsJson) ZCubeInfo(zCubeID, zOrderByCols) } } /** * If the given file was written by an OPTIMIZE ZORDER BY job, * return the corresponding [[ZCubeInfo]]. Otherwise return [[None]]. */ def getForFile(file: AddFile): Option[ZCubeInfo] = { for { tags <- Option(file.tags) zCubeInfo <- ZCubeInfo.fromAddFileTags(tags) } yield { zCubeInfo } } /** * Update the given file's metadata to make it part of the given zCubeInfo. */ def setForFile(file: AddFile, zCubeInfo: ZCubeInfo): AddFile = { val newTags = file.tagsOrEmpty ++ ZCubeInfo.toAddFileTags(zCubeInfo) file.copy(tags = newTags) } /** * Clears the ZCubeInfo metadata of the given file to make it appear as unoptimized. */ def unsetForFile(file: AddFile): AddFile = { val oldTags = file.tags val newTags = if (oldTags == null) null else oldTags -- Seq(ZCUBE_ID_KEY, ZORDER_BY_KEY, ZORDER_CURVE) file.copy(tags = newTags) } } ================================================ FILE: spark/src/main/scala/org/apache/spark/sql/util/ScalaExtensions.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.util /** Extension utility classes for built-in Scala functionality. */ object ScalaExtensions { implicit class OptionExt[T](opt: Option[T]) { /** * Execute `f` on the content of `opt`, if `opt.isDefined`. * * This is basically a rename of `opt.foreach`, but with better readability. */ def ifDefined(f: T => Unit): Unit = opt.foreach(f) } implicit class OptionExtCompanion(opt: Option.type) { /** * When a given condition is true, evaluates the a argument and returns Some(a). * When the condition is false, a is not evaluated and None is returned. */ def when[A](cond: Boolean)(a: => A): Option[A] = if (cond) Some(a) else None /** * When a given condition is false, evaluates the a argument and returns Some(a). * When the condition is true, a is not evaluated and None is returned. */ def whenNot[A](cond: Boolean)(a: => A): Option[A] = if (!cond) Some(a) else None /** Sum up all the `options`, substituting `default` for each `None`. */ def sum[N : Numeric](default: N)(options: Option[N]*): N = options.map(_.getOrElse(default)).sum } implicit class AnyExt(any: Any) { /** * Applies the partial function to any if it is defined and ignores the result if any. */ def condDo(pf: PartialFunction[Any, Unit]): Unit = scala.PartialFunction.condOpt(any)(pf) } } ================================================ FILE: spark/src/main/scala-shims/spark-4.0/CreateDeltaTableLikeShims.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands import org.apache.spark.sql.SaveMode import org.apache.spark.sql.delta.DeltaOptions object CreateDeltaTableLikeShims { /** * Differentiate between DataFrameWriterV1 and V2 so that we can decide * what to do with table metadata. In DataFrameWriterV1, mode("overwrite").saveAsTable, * behaves as a CreateOrReplace table, but we have asked for "overwriteSchema" as an * explicit option to overwrite partitioning or schema information. With DataFrameWriterV2, * the behavior asked for by the user is clearer: .createOrReplace(), which means that we * should overwrite schema and/or partitioning. Therefore we have this hack. * * In Spark 4.0 this horrible hack depends on the stack trace, where eager execution of the * command pointed to the calling API. * * TODO: Shim no longer needed once spark-4.0 is removed. */ def isV1WriterSaveAsTableOverwrite(options: DeltaOptions, mode: SaveMode): Boolean = { Thread.currentThread().getStackTrace.exists(_.toString.contains( classOf[org.apache.spark.sql.classic.DataFrameWriter[_]].getCanonicalName + ".")) && mode == SaveMode.Overwrite } } ================================================ FILE: spark/src/main/scala-shims/spark-4.0/DataSourceV2RelationShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.execution.datasources.v2 import org.apache.spark.sql.catalyst.expressions.AttributeReference import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.connector.catalog.{CatalogPlugin, Identifier, Table} import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation import org.apache.spark.sql.util.CaseInsensitiveStringMap /** * Shim for DataSourceV2Relation to handle API changes between Spark versions. * In Spark 4.0, DataSourceV2Relation has 5 constructor parameters. */ object DataSourceV2RelationShim { /** * Main extractor for DataSourceV2Relation that works across Spark versions. * Returns the common fields that exist in all versions. */ def unapply(plan: LogicalPlan): Option[ (Table, Seq[AttributeReference], Option[CatalogPlugin], Option[Identifier], CaseInsensitiveStringMap)] = { plan match { case r: DataSourceV2Relation => Some((r.table, r.output, r.catalog, r.identifier, r.options)) case _ => None } } } /** * Simplified extractor when only table and options are needed. */ object DataSourceV2RelationSimple { def unapply(plan: LogicalPlan): Option[(Table, CaseInsensitiveStringMap)] = { plan match { case r: DataSourceV2Relation => Some((r.table, r.options)) case _ => None } } } /** * Extractor for cases that only need the table. */ object DataSourceV2RelationTable { def unapply(plan: LogicalPlan): Option[Table] = { plan match { case r: DataSourceV2Relation => Some(r.table) case _ => None } } } ================================================ FILE: spark/src/main/scala-shims/spark-4.0/DateTimeExpressionShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.expressions /** * Shim for date/time expressions in Spark 4.0 */ object DateTimeExpressionShims { /** * Check if the given expression is a date/time arithmetic expression */ def isDateTimeArithmeticExpression(expr: Expression): Boolean = { expr match { case _: AddMonthsBase | _: DateAdd | _: DateAddInterval | _: DateDiff | _: DateSub | _: DatetimeSub | _: LastDay | _: MonthsBetween | _: NextDay | _: SubtractDates | _: SubtractTimestamps | _: TimeAdd | _: TimestampAdd | _: TimestampAddYMInterval | _: TimestampDiff | _: TruncInstant => true case _ => false } } } ================================================ FILE: spark/src/main/scala-shims/spark-4.0/LogKeyShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.logging import org.apache.spark.internal.LogKey /** * Shim for LogKey to handle API changes between Spark versions. * In Spark 4.0, LogKey is a Scala trait with a default implementation of `name`. * * DeltaLogKey is just a trait that extends LogKey, allowing case objects to extend it. */ trait DeltaLogKey extends LogKey ================================================ FILE: spark/src/main/scala-shims/spark-4.0/ParquetFooterReaderShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.execution.datasources.parquet import org.apache.hadoop.conf.Configuration import org.apache.parquet.format.converter.ParquetMetadataConverter import org.apache.hadoop.fs.FileStatus import org.apache.parquet.hadoop.metadata.ParquetMetadata import org.apache.spark.sql.execution.datasources.parquet.ParquetFooterReader object ParquetFooterReaderShims { def readParquetFooter( conf: Configuration, status: FileStatus, filter: ParquetMetadataConverter.MetadataFilter) : ParquetMetadata = { ParquetFooterReader.readFooter(conf, status, filter) } } ================================================ FILE: spark/src/main/scala-shims/spark-4.0/ParseExceptionShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.parser import org.antlr.v4.runtime.ParserRuleContext import org.apache.spark.sql.catalyst.parser.{ParseException, ParserUtils} import org.apache.spark.sql.catalyst.trees.Origin import org.apache.spark.sql.delta.DeltaThrowable /** * DeltaParseException for Spark 4.0 and earlier. * In these versions, ParseException takes both start and stop Origin parameters. */ class DeltaParseException( ctx: ParserRuleContext, errorClass: String, messageParameters: Map[String, String] = Map.empty) extends ParseException( Option(ParserUtils.command(ctx)), ParserUtils.position(ctx.getStart), ParserUtils.position(ctx.getStop), // In Spark 4.0, we have the stop parameter errorClass, messageParameters ) with DeltaThrowable { override def getErrorClass: String = errorClass } /** * Shim for ParseException to handle API changes between Spark versions. * In Spark 4.0 and earlier, ParseException has separate start and stop parameters. */ object ParseExceptionShims { /** * Create a ParseException with the appropriate constructor for this Spark version. * In Spark 4.0, we use both start and stop Origin parameters. */ def createParseException( command: Option[String], start: Origin, stop: Origin, errorClass: String, messageParameters: Map[String, String]): ParseException = { new ParseException(command, start, stop, errorClass, messageParameters) } } ================================================ FILE: spark/src/main/scala-shims/spark-4.0/QualifiedColTypeShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.plans.logical import org.apache.spark.sql.connector.catalog.TableChange.AddColumn /** * In Spark 4.0 QualifiedColType stores `default` as a String */ object QualifiedColTypeShims { def getDefaultValueArgFromAddColumn(col: AddColumn): Option[String] = { Option(col.defaultValue()).map(_.getSql()) } def getDefaultValueStr(col: QualifiedColType): Option[String] = { col.default } } ================================================ FILE: spark/src/main/scala-shims/spark-4.0/RelocatedStreamingClassesShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.UUID import org.apache.hadoop.fs.Path import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.execution.streaming.{OffsetSeqMetadata, WatermarkPropagator} import org.apache.spark.sql.streaming.OutputMode import org.apache.spark.sql.types.StructType import org.apache.spark.sql.execution.streaming.{ CheckpointFileManager => CheckpointFileManagerShim, IncrementalExecution => IncrementalExecutionShim, MetadataLogFileIndex => MetadataLogFileIndexShim, StreamExecution => StreamExecutionShim, StreamingRelation => StreamingRelationShim, MetadataVersionUtil => MetadataVersionUtilShim } import org.apache.spark.sql.execution.streaming.{FileStreamSink => FileStreamSinkShim} object Relocated { type CheckpointFileManager = CheckpointFileManagerShim val CheckpointFileManager: CheckpointFileManagerShim.type = CheckpointFileManagerShim type IncrementalExecution = IncrementalExecutionShim // scalastyle:off argcount def createIncrementalExecution( sparkSession: org.apache.spark.sql.classic.SparkSession, logicalPlan: LogicalPlan, outputMode: OutputMode, checkpointLocation: String, queryId: UUID, runId: UUID, currentBatchId: Long, prevOffsetSeqMetadata: Option[OffsetSeqMetadata], offsetSeqMetadata: OffsetSeqMetadata, watermarkPropagator: WatermarkPropagator, isFirstBatch: Boolean): IncrementalExecution = { // scalastyle:on argcount new IncrementalExecutionShim( sparkSession, logicalPlan, outputMode, checkpointLocation, queryId, runId, currentBatchId, prevOffsetSeqMetadata, offsetSeqMetadata, watermarkPropagator, isFirstBatch) } type StreamingRelation = StreamingRelationShim val StreamingRelation: StreamingRelationShim.type = StreamingRelationShim type MetadataLogFileIndex = MetadataLogFileIndexShim def createMetadataLogFileIndex( sparkSession: org.apache.spark.sql.SparkSession, path: Path, options: Map[String, String], userSpecifiedSchema: Option[StructType]): MetadataLogFileIndex = { new MetadataLogFileIndexShim(sparkSession, path, options, userSpecifiedSchema) } type FileStreamSink = FileStreamSinkShim val FileStreamSink: FileStreamSinkShim.type = FileStreamSinkShim type StreamExecution = StreamExecutionShim val StreamExecution: StreamExecutionShim.type = StreamExecutionShim val MetadataVersionUtil: MetadataVersionUtilShim.type = MetadataVersionUtilShim } ================================================ FILE: spark/src/main/scala-shims/spark-4.0/SupportsV1OverwriteWithSaveAsTable.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.connector.catalog /* Interface exists in Spark 4.1+. Is noop for Spark 4.0. */ trait SupportsV1OverwriteWithSaveAsTable extends TableProvider { def addV1OverwriteWithSaveAsTableOption(): Boolean = true } object SupportsV1OverwriteWithSaveAsTable { val OPTION_NAME: String = "__v1_save_as_table_overwrite" } ================================================ FILE: spark/src/main/scala-shims/spark-4.0/VariantShreddingShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.shims /** * Shim for variant shredding configs to handle API changes between Spark versions. * In Spark 4.0, VARIANT_INFER_SHREDDING_SCHEMA config does not exist. * * This shim provides a way to conditionally add the config to the options map * when writing files. */ object VariantShreddingShims { /** * Returns a Map containing variant shredding related configs for file writing. * In Spark 4.0, this returns an empty map since the config doesn't exist. */ def getVariantInferShreddingSchemaOptions(enableVariantShredding: Boolean) : Map[String, String] = { // In Spark 4.0, VARIANT_INFER_SHREDDING_SCHEMA does not exist, so return empty map Map.empty[String, String] } } ================================================ FILE: spark/src/main/scala-shims/spark-4.0/ViewShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.plans.logical import org.apache.spark.sql.catalyst.catalog.CatalogTable /** * Shim for View to handle API changes between Spark versions. * In Spark 4.0 and earlier, View has fewer constructor parameters. */ object ViewShims { /** * Extractor that matches View(desc, true, child) pattern. * Used in DeltaViewHelper for matching temp views with a specific structure. */ object TempViewWithChild { def unapply(plan: LogicalPlan): Option[(CatalogTable, LogicalPlan)] = plan match { case View(desc, isTempView, child) if isTempView => Some((desc, child)) case _ => None } } } ================================================ FILE: spark/src/main/scala-shims/spark-4.1/CreateDeltaTableLikeShims.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands import org.apache.spark.sql.SaveMode import org.apache.spark.sql.delta.DeltaOptions object CreateDeltaTableLikeShims { /** * Differentiate between DataFrameWriterV1 and V2 so that we can decide * what to do with table metadata. In DataFrameWriterV1, mode("overwrite").saveAsTable, * behaves as a CreateOrReplace table, but we have asked for "overwriteSchema" as an * explicit option to overwrite partitioning or schema information. With DataFrameWriterV2, * the behavior asked for by the user is clearer: .createOrReplace(), which means that we * should overwrite schema and/or partitioning. Therefore we have this hack. * * In Spark 4.1, DataFrameWriter provides the option "__v1_save_as_table_overwrite", because * the stack trace does not indicate the calling API anymore in connect mode - planning and * execution has been separated. * * TODO: Shim no longer needed once spark-4.0 is removed. */ def isV1WriterSaveAsTableOverwrite(options: DeltaOptions, mode: SaveMode): Boolean = { // Note: Spark is setting this only for SaveMode.Overwrite anyway, but we double check. // The 4.0 shim relies on stack trace analysis instead, so it has to check. // After 4.0 is dropped, we can simplify. options.isDataFrameWriterV1SaveAsTableOverwrite && mode == SaveMode.Overwrite } } ================================================ FILE: spark/src/main/scala-shims/spark-4.1/DataSourceV2RelationShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.execution.datasources.v2 import org.apache.spark.sql.catalyst.expressions.AttributeReference import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.connector.catalog.{CatalogPlugin, Identifier, Table} import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation import org.apache.spark.sql.util.CaseInsensitiveStringMap /** * Shim for DataSourceV2Relation to handle API changes between Spark versions. * In Spark 4.1, DataSourceV2Relation has 6 constructor parameters (added an extra parameter). */ object DataSourceV2RelationShim { /** * Main extractor for DataSourceV2Relation that works across Spark versions. * Returns the common fields that exist in all versions. */ def unapply(plan: LogicalPlan): Option[( Table, Seq[AttributeReference], Option[CatalogPlugin], Option[Identifier], CaseInsensitiveStringMap)] = { plan match { case r: DataSourceV2Relation => Some((r.table, r.output, r.catalog, r.identifier, r.options)) case _ => None } } } /** * Simplified extractor when only table and options are needed. */ object DataSourceV2RelationSimple { def unapply(plan: LogicalPlan): Option[(Table, CaseInsensitiveStringMap)] = { plan match { case r: DataSourceV2Relation => Some((r.table, r.options)) case _ => None } } } /** * Extractor for cases that only need the table. */ object DataSourceV2RelationTable { def unapply(plan: LogicalPlan): Option[Table] = { plan match { case r: DataSourceV2Relation => Some(r.table) case _ => None } } } ================================================ FILE: spark/src/main/scala-shims/spark-4.1/DateTimeExpressionShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.expressions /** * Shim for date/time expressions in Spark 4.1 * Note: TimeAdd is removed in Spark 4.1 */ object DateTimeExpressionShims { /** * Check if the given expression is a date/time arithmetic expression */ def isDateTimeArithmeticExpression(expr: Expression): Boolean = { expr match { case _: AddMonthsBase | _: DateAdd | _: DateAddInterval | _: DateDiff | _: DateSub | _: DatetimeSub | _: LastDay | _: MonthsBetween | _: NextDay | _: SubtractDates | _: SubtractTimestamps | _: TimestampAdd | _: TimestampAddYMInterval | _: TimestampDiff | _: TruncInstant => true case _ => false } } } ================================================ FILE: spark/src/main/scala-shims/spark-4.1/LogKeyShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.logging import java.util.Locale import org.apache.spark.internal.LogKey /** * Shim for LogKey to handle API changes between Spark versions. * In Spark 4.1, LogKey is a Java interface requiring explicit implementation of `name()`. * * DeltaLogKey provides the implementation of name() that case objects can inherit. */ abstract class DeltaLogKey extends LogKey { override def name(): String = getClass.getSimpleName.stripSuffix("$").toLowerCase(Locale.ROOT) } ================================================ FILE: spark/src/main/scala-shims/spark-4.1/ParquetFooterReaderShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.execution.datasources.parquet import org.apache.hadoop.conf.Configuration import org.apache.parquet.format.converter.ParquetMetadataConverter import org.apache.hadoop.fs.FileStatus import org.apache.parquet.hadoop.metadata.ParquetMetadata import org.apache.parquet.hadoop.util.HadoopInputFile object ParquetFooterReaderShims { def readParquetFooter( conf: Configuration, status: FileStatus, filter: ParquetMetadataConverter.MetadataFilter) : ParquetMetadata = { val inputFile = HadoopInputFile.fromStatus(status, conf) ParquetFooterReader.readFooter(inputFile, filter) } } ================================================ FILE: spark/src/main/scala-shims/spark-4.1/ParseExceptionShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.parser import org.antlr.v4.runtime.ParserRuleContext import org.apache.spark.sql.catalyst.parser.{ParseException, ParserUtils} import org.apache.spark.sql.catalyst.trees.Origin import org.apache.spark.sql.delta.DeltaThrowable /** * DeltaParseException for Spark 4.1. * In this version, ParseException only takes a single origin parameter (stop was removed). */ class DeltaParseException( ctx: ParserRuleContext, errorClass: String, messageParameters: Map[String, String] = Map.empty) extends ParseException( Option(ParserUtils.command(ctx)), ParserUtils.position(ctx.getStart), // In Spark 4.1, only start position is used // No stop parameter in Spark 4.1 errorClass, messageParameters ) with DeltaThrowable { override def getErrorClass: String = errorClass } /** * Shim for ParseException to handle API changes between Spark versions. * In Spark 4.1, ParseException only has a single origin parameter (stop was removed). */ object ParseExceptionShims { /** * Create a ParseException with the appropriate constructor for this Spark version. * In Spark 4.1, we only use the start Origin (stop parameter was removed). */ def createParseException( command: Option[String], start: Origin, stop: Origin, // This parameter is ignored in Spark 4.1 errorClass: String, messageParameters: Map[String, String]): ParseException = { // In Spark 4.1, ParseException only takes a single origin parameter new ParseException(command, start, errorClass, messageParameters) } } ================================================ FILE: spark/src/main/scala-shims/spark-4.1/QualifiedColTypeShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.plans.logical import org.apache.spark.sql.connector.catalog.TableChange.AddColumn /** * In Spark 4.1 QualifiedColType stores `default` as a DefaultValueExpression */ object QualifiedColTypeShims { def getDefaultValueArgFromAddColumn(col: AddColumn): Option[DefaultValueExpression] = { Option(col.defaultValue).map(v => DefaultValueExpression( org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parseExpression( v.getSql()), v.getSql())) } def getDefaultValueStr(col: QualifiedColType): Option[String] = { col.default.map { value => value match { case DefaultValueExpression(_, originalSQL, _) => originalSQL case _ => value.toString } } } } ================================================ FILE: spark/src/main/scala-shims/spark-4.1/RelocatedStreamingClassesShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.UUID import org.apache.hadoop.fs.Path import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.execution.streaming.runtime.WatermarkPropagator import org.apache.spark.sql.streaming.OutputMode import org.apache.spark.sql.types.StructType import org.apache.spark.sql.execution.streaming.checkpointing.{ CheckpointFileManager => CheckpointFileManagerShim, MetadataVersionUtil => MetadataVersionUtilShim, OffsetSeqMetadata } import org.apache.spark.sql.execution.streaming.runtime.{ IncrementalExecution => IncrementalExecutionShim, MetadataLogFileIndex => MetadataLogFileIndexShim, StreamExecution => StreamExecutionShim, StreamingRelation => StreamingRelationShim } import org.apache.spark.sql.execution.streaming.sinks.{FileStreamSink => FileStreamSinkShim} object Relocated { type CheckpointFileManager = CheckpointFileManagerShim val CheckpointFileManager: CheckpointFileManagerShim.type = CheckpointFileManagerShim type IncrementalExecution = IncrementalExecutionShim // scalastyle:off argcount def createIncrementalExecution( sparkSession: org.apache.spark.sql.classic.SparkSession, logicalPlan: LogicalPlan, outputMode: OutputMode, checkpointLocation: String, queryId: UUID, runId: UUID, currentBatchId: Long, prevOffsetSeqMetadata: Option[OffsetSeqMetadata], offsetSeqMetadata: OffsetSeqMetadata, watermarkPropagator: WatermarkPropagator, isFirstBatch: Boolean): IncrementalExecution = { // scalastyle:on argcount new IncrementalExecutionShim( sparkSession, logicalPlan, outputMode, checkpointLocation, queryId, runId, currentBatchId, prevOffsetSeqMetadata, offsetSeqMetadata, watermarkPropagator, isFirstBatch) } type StreamingRelation = StreamingRelationShim val StreamingRelation: StreamingRelationShim.type = StreamingRelationShim type MetadataLogFileIndex = MetadataLogFileIndexShim def createMetadataLogFileIndex( sparkSession: org.apache.spark.sql.SparkSession, path: Path, options: Map[String, String], userSpecifiedSchema: Option[StructType]): MetadataLogFileIndex = { new MetadataLogFileIndexShim(sparkSession, path, options, userSpecifiedSchema) } type FileStreamSink = FileStreamSinkShim val FileStreamSink: FileStreamSinkShim.type = FileStreamSinkShim type StreamExecution = StreamExecutionShim val StreamExecution: StreamExecutionShim.type = StreamExecutionShim val MetadataVersionUtil: MetadataVersionUtilShim.type = MetadataVersionUtilShim } ================================================ FILE: spark/src/main/scala-shims/spark-4.1/VariantShreddingShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.shims import org.apache.spark.sql.internal.SQLConf /** * Shim for variant shredding configs to handle API changes between Spark versions. * In Spark 4.1, VARIANT_INFER_SHREDDING_SCHEMA config exists. * * This shim provides a way to conditionally add the config to the options map * when writing files. */ object VariantShreddingShims { /** * Returns a Map containing variant shredding related configs for file writing. * In Spark 4.1, this returns the VARIANT_INFER_SHREDDING_SCHEMA config. */ def getVariantInferShreddingSchemaOptions(enableVariantShredding: Boolean) : Map[String, String] = { Map(SQLConf.VARIANT_INFER_SHREDDING_SCHEMA.key -> enableVariantShredding.toString) } } ================================================ FILE: spark/src/main/scala-shims/spark-4.1/ViewShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.plans.logical import org.apache.spark.sql.catalyst.catalog.CatalogTable /** * Shim for View to handle API changes between Spark versions. * In Spark 4.1, View has an additional constructor parameter. */ object ViewShims { /** * Extractor that matches View(desc, true, child) pattern. * Used in DeltaViewHelper for matching temp views with a specific structure. * In Spark 4.1, View has an additional parameter, so we use a wildcard to ignore it. */ object TempViewWithChild { def unapply(plan: LogicalPlan): Option[(CatalogTable, LogicalPlan)] = plan match { // In Spark 4.1, View has an additional parameter, we use _ to match it case View(desc, isTempView, child, _) if isTempView => Some((desc, child)) case _ => None } } } ================================================ FILE: spark/src/main/scala-shims/spark-4.2/CreateDeltaTableLikeShims.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands import org.apache.spark.sql.SaveMode import org.apache.spark.sql.delta.DeltaOptions object CreateDeltaTableLikeShims { /** * Differentiate between DataFrameWriterV1 and V2 so that we can decide * what to do with table metadata. In DataFrameWriterV1, mode("overwrite").saveAsTable, * behaves as a CreateOrReplace table, but we have asked for "overwriteSchema" as an * explicit option to overwrite partitioning or schema information. With DataFrameWriterV2, * the behavior asked for by the user is clearer: .createOrReplace(), which means that we * should overwrite schema and/or partitioning. Therefore we have this hack. * * In Spark 4.1, DataFrameWriter provides the option "__v1_save_as_table_overwrite", because * the stack trace does not indicate the calling API anymore in connect mode - planning and * execution has been separated. * * TODO: Shim no longer needed once spark-4.0 is removed. */ def isV1WriterSaveAsTableOverwrite(options: DeltaOptions, mode: SaveMode): Boolean = { // Note: Spark is setting this only for SaveMode.Overwrite anyway, but we double check. // The 4.0 shim relies on stack trace analysis instead, so it has to check. // After 4.0 is dropped, we can simplify. options.isDataFrameWriterV1SaveAsTableOverwrite && mode == SaveMode.Overwrite } } ================================================ FILE: spark/src/main/scala-shims/spark-4.2/DataSourceV2RelationShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.execution.datasources.v2 import org.apache.spark.sql.catalyst.expressions.AttributeReference import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.connector.catalog.{CatalogPlugin, Identifier, Table} import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation import org.apache.spark.sql.util.CaseInsensitiveStringMap /** * Shim for DataSourceV2Relation to handle API changes between Spark versions. * In Spark 4.2, DataSourceV2Relation has 6 constructor parameters (same as Spark 4.1). */ object DataSourceV2RelationShim { /** * Main extractor for DataSourceV2Relation that works across Spark versions. * Returns the common fields that exist in all versions. */ def unapply(plan: LogicalPlan): Option[( Table, Seq[AttributeReference], Option[CatalogPlugin], Option[Identifier], CaseInsensitiveStringMap)] = { plan match { case r: DataSourceV2Relation => Some((r.table, r.output, r.catalog, r.identifier, r.options)) case _ => None } } } /** * Simplified extractor when only table and options are needed. */ object DataSourceV2RelationSimple { def unapply(plan: LogicalPlan): Option[(Table, CaseInsensitiveStringMap)] = { plan match { case r: DataSourceV2Relation => Some((r.table, r.options)) case _ => None } } } /** * Extractor for cases that only need the table. */ object DataSourceV2RelationTable { def unapply(plan: LogicalPlan): Option[Table] = { plan match { case r: DataSourceV2Relation => Some(r.table) case _ => None } } } ================================================ FILE: spark/src/main/scala-shims/spark-4.2/DateTimeExpressionShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.expressions /** * Shim for date/time expressions in Spark 4.2 (same as Spark 4.1) * Note: TimeAdd is removed in Spark 4.1+ */ object DateTimeExpressionShims { /** * Check if the given expression is a date/time arithmetic expression */ def isDateTimeArithmeticExpression(expr: Expression): Boolean = { expr match { case _: AddMonthsBase | _: DateAdd | _: DateAddInterval | _: DateDiff | _: DateSub | _: DatetimeSub | _: LastDay | _: MonthsBetween | _: NextDay | _: SubtractDates | _: SubtractTimestamps | _: TimestampAdd | _: TimestampAddYMInterval | _: TimestampDiff | _: TruncInstant => true case _ => false } } } ================================================ FILE: spark/src/main/scala-shims/spark-4.2/LogKeyShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.logging import java.util.Locale import org.apache.spark.internal.LogKey /** * Shim for LogKey to handle API changes between Spark versions. * In Spark 4.2, LogKey is a Java interface requiring explicit implementation of `name()` * (same as Spark 4.1). * * DeltaLogKey provides the implementation of name() that case objects can inherit. */ abstract class DeltaLogKey extends LogKey { override def name(): String = getClass.getSimpleName.stripSuffix("$").toLowerCase(Locale.ROOT) } ================================================ FILE: spark/src/main/scala-shims/spark-4.2/ParquetFooterReaderShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.execution.datasources.parquet import org.apache.hadoop.conf.Configuration import org.apache.parquet.format.converter.ParquetMetadataConverter import org.apache.hadoop.fs.FileStatus import org.apache.parquet.hadoop.metadata.ParquetMetadata import org.apache.parquet.hadoop.util.HadoopInputFile object ParquetFooterReaderShims { def readParquetFooter( conf: Configuration, status: FileStatus, filter: ParquetMetadataConverter.MetadataFilter) : ParquetMetadata = { val inputFile = HadoopInputFile.fromStatus(status, conf) ParquetFooterReader.readFooter(inputFile, filter) } } ================================================ FILE: spark/src/main/scala-shims/spark-4.2/ParseExceptionShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.parser import org.antlr.v4.runtime.ParserRuleContext import org.apache.spark.sql.catalyst.parser.{ParseException, ParserUtils} import org.apache.spark.sql.catalyst.trees.Origin import org.apache.spark.sql.delta.DeltaThrowable /** * DeltaParseException for Spark 4.2 (same as Spark 4.1). * In this version, ParseException only takes a single origin parameter (stop was removed). */ class DeltaParseException( ctx: ParserRuleContext, errorClass: String, messageParameters: Map[String, String] = Map.empty) extends ParseException( Option(ParserUtils.command(ctx)), ParserUtils.position(ctx.getStart), // In Spark 4.2, only start position is used // No stop parameter in Spark 4.2 errorClass, messageParameters ) with DeltaThrowable { override def getErrorClass: String = errorClass } /** * Shim for ParseException to handle API changes between Spark versions. * In Spark 4.2, ParseException only has a single origin parameter (same as Spark 4.1, * stop was removed). */ object ParseExceptionShims { /** * Create a ParseException with the appropriate constructor for this Spark version. * In Spark 4.2, we only use the start Origin (same as Spark 4.1, stop parameter was removed). */ def createParseException( command: Option[String], start: Origin, stop: Origin, // This parameter is ignored in Spark 4.2 (same as 4.1) errorClass: String, messageParameters: Map[String, String]): ParseException = { // In Spark 4.2, ParseException only takes a single origin parameter (same as 4.1) new ParseException(command, start, errorClass, messageParameters) } } ================================================ FILE: spark/src/main/scala-shims/spark-4.2/QualifiedColTypeShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.plans.logical import org.apache.spark.sql.connector.catalog.TableChange.AddColumn /** * In Spark 4.2 QualifiedColType stores `default` as a DefaultValueExpression (same as Spark 4.1) */ object QualifiedColTypeShims { def getDefaultValueArgFromAddColumn(col: AddColumn): Option[DefaultValueExpression] = { Option(col.defaultValue).map(v => DefaultValueExpression( org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parseExpression( v.getSql()), v.getSql())) } def getDefaultValueStr(col: QualifiedColType): Option[String] = { col.default.map { value => value match { case DefaultValueExpression(_, originalSQL, _) => originalSQL case _ => value.toString } } } } ================================================ FILE: spark/src/main/scala-shims/spark-4.2/RelocatedStreamingClassesShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.UUID import org.apache.hadoop.fs.Path import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.execution.streaming.runtime.WatermarkPropagator import org.apache.spark.sql.streaming.OutputMode import org.apache.spark.sql.types.StructType import org.apache.spark.sql.execution.streaming.checkpointing.{ CheckpointFileManager => CheckpointFileManagerShim, MetadataVersionUtil => MetadataVersionUtilShim, OffsetSeqMetadataBase } import org.apache.spark.sql.execution.streaming.runtime.{ IncrementalExecution => IncrementalExecutionShim, MetadataLogFileIndex => MetadataLogFileIndexShim, StreamExecution => StreamExecutionShim, StreamingRelation => StreamingRelationShim } import org.apache.spark.sql.execution.streaming.sinks.{FileStreamSink => FileStreamSinkShim} object Relocated { type CheckpointFileManager = CheckpointFileManagerShim val CheckpointFileManager: CheckpointFileManagerShim.type = CheckpointFileManagerShim type IncrementalExecution = IncrementalExecutionShim // scalastyle:off argcount def createIncrementalExecution( sparkSession: org.apache.spark.sql.classic.SparkSession, logicalPlan: LogicalPlan, outputMode: OutputMode, checkpointLocation: String, queryId: UUID, runId: UUID, currentBatchId: Long, prevOffsetSeqMetadata: Option[OffsetSeqMetadataBase], offsetSeqMetadata: OffsetSeqMetadataBase, watermarkPropagator: WatermarkPropagator, isFirstBatch: Boolean): IncrementalExecution = { // scalastyle:on argcount new IncrementalExecutionShim( sparkSession, logicalPlan, outputMode, checkpointLocation, queryId, runId, currentBatchId, prevOffsetSeqMetadata, offsetSeqMetadata, watermarkPropagator, isFirstBatch) } type StreamingRelation = StreamingRelationShim val StreamingRelation: StreamingRelationShim.type = StreamingRelationShim type MetadataLogFileIndex = MetadataLogFileIndexShim def createMetadataLogFileIndex( sparkSession: org.apache.spark.sql.SparkSession, path: Path, options: Map[String, String], userSpecifiedSchema: Option[StructType]): MetadataLogFileIndex = { new MetadataLogFileIndexShim(sparkSession, path, options, userSpecifiedSchema) } type FileStreamSink = FileStreamSinkShim val FileStreamSink: FileStreamSinkShim.type = FileStreamSinkShim type StreamExecution = StreamExecutionShim val StreamExecution: StreamExecutionShim.type = StreamExecutionShim val MetadataVersionUtil: MetadataVersionUtilShim.type = MetadataVersionUtilShim } ================================================ FILE: spark/src/main/scala-shims/spark-4.2/VariantShreddingShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.shims import org.apache.spark.sql.internal.SQLConf /** * Shim for variant shredding configs to handle API changes between Spark versions. * In Spark 4.2, VARIANT_INFER_SHREDDING_SCHEMA config exists. * * This shim provides a way to conditionally add the config to the options map * when writing files. */ object VariantShreddingShims { /** * Returns a Map containing variant shredding related configs for file writing. * In Spark 4.2, this returns the VARIANT_INFER_SHREDDING_SCHEMA config. */ def getVariantInferShreddingSchemaOptions(enableVariantShredding: Boolean): Map[String, String] = { Map(SQLConf.VARIANT_INFER_SHREDDING_SCHEMA.key -> enableVariantShredding.toString) } } ================================================ FILE: spark/src/main/scala-shims/spark-4.2/ViewShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.plans.logical import org.apache.spark.sql.catalyst.catalog.CatalogTable /** * Shim for View to handle API changes between Spark versions. * In Spark 4.2, View has an additional constructor parameter (same as Spark 4.1). */ object ViewShims { /** * Extractor that matches View(desc, true, child) pattern. * Used in DeltaViewHelper for matching temp views with a specific structure. * In Spark 4.2, View has an additional parameter (same as 4.1), so we use a wildcard to ignore * it. */ object TempViewWithChild { def unapply(plan: LogicalPlan): Option[(CatalogTable, LogicalPlan)] = plan match { // In Spark 4.2, View has an additional parameter (same as 4.1), we use _ to match it case View(desc, isTempView, child, _) if isTempView => Some((desc, child)) case _ => None } } } ================================================ FILE: spark/src/test/java/io/delta/sql/JavaDeltaSparkSessionExtensionSuite.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sql; import org.apache.spark.sql.SparkSession; import org.apache.spark.util.Utils; import org.junit.Test; import java.io.IOException; public class JavaDeltaSparkSessionExtensionSuite { @Test public void testSQLConf() throws IOException { SparkSession spark = SparkSession.builder() .appName("JavaDeltaSparkSessionExtensionSuiteUsingSQLConf") .master("local[2]") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") .getOrCreate(); try { String input = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "input") .getCanonicalPath(); spark.range(1, 10).write().format("delta").save(input); spark.sql("vacuum delta.`" + input + "`"); } finally { spark.stop(); } } } ================================================ FILE: spark/src/test/java/io/delta/tables/JavaDeltaTableBuilderSuite.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.spark.sql.delta.DeltaLog; import org.apache.hadoop.fs.Path; import org.apache.spark.sql.*; import org.apache.spark.util.Utils; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.apache.spark.sql.delta.DeltaSQLCommandJavaTest; import static org.apache.spark.sql.types.DataTypes.*; public class JavaDeltaTableBuilderSuite implements DeltaSQLCommandJavaTest { private transient SparkSession spark; private transient String input; @Before public void setUp() { // Trigger static initializer of TestData spark = buildSparkSession(); } @After public void tearDown() { if (spark != null) { spark.stop(); spark = null; } } private DeltaTable buildTable(DeltaTableBuilder builder) { return builder.addColumn("c1", "int") .addColumn("c2", IntegerType) .addColumn("c3", "string", false) .addColumn("c4", StringType, true) .addColumn(DeltaTable.columnBuilder(spark, "c5") .dataType("bigint") .comment("foo") .nullable(false) .build() ) .addColumn(DeltaTable.columnBuilder(spark, "c6") .dataType(LongType) .generatedAlwaysAs("c5 + 10") .build() ).execute(); } private DeltaTable createTable(boolean ifNotExists, String tableName) { DeltaTableBuilder builder; if (ifNotExists) { builder = DeltaTable.createIfNotExists(); } else { builder = DeltaTable.create(); } if (tableName.startsWith("delta.`")) { tableName = tableName.substring("delta.`".length()); String location = tableName.substring(0, tableName.length() - 1); builder = builder.location(location); DeltaLog.forTable(spark, location).clearCache(); } else { builder = builder.tableName(tableName); DeltaLog.forTable(spark, new Path(tableName)).clearCache(); } return buildTable(builder); } private DeltaTable replaceTable(boolean orCreate, String tableName) { DeltaTableBuilder builder; if (orCreate) { builder = DeltaTable.createOrReplace(); } else { builder = DeltaTable.replace(); } if (tableName.startsWith("delta.`")) { tableName = tableName.substring("delta.`".length()); String location = tableName.substring(0, tableName.length() - 1); builder = builder.location(location); } else { builder = builder.tableName(tableName); } return buildTable(builder); } private void verifyGeneratedColumn(String tableName, DeltaTable deltaTable) { String cmd = String.format("INSERT INTO %s (c1, c2, c3, c4, c5, c6) %s", tableName, "VALUES (1, 2, 'a', 'c', 1, 11)"); spark.sql(cmd); Map set = new HashMap() {{ put("c5", "10"); }}; deltaTable.updateExpr("c6 = 11", set); assert(deltaTable.toDF().select("c6").collectAsList().get(0).getLong(0) == 20); } @Test public void testCreateTable() { try { // Test creating DeltaTable by name DeltaTable table = createTable(false, "deltaTable"); verifyGeneratedColumn("deltaTable", table); } finally { spark.sql("DROP TABLE IF EXISTS deltaTable"); } // Test creating DeltaTable by path. String input = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "input") .toString(); DeltaTable table2 = createTable(false, String.format("delta.`%s`", input)); verifyGeneratedColumn(String.format("delta.`%s`", input), table2); } @Test public void testCreateTableIfNotExists() { // Ignore table creation if already exsits. List data = Arrays.asList("hello", "world"); Dataset dataDF = spark.createDataset(data, Encoders.STRING()).toDF(); try { // Test creating DeltaTable by name - not exists. DeltaTable table = createTable(true, "deltaTable"); verifyGeneratedColumn("deltaTable", table); dataDF.write().format("delta").mode("overwrite").saveAsTable("deltaTable2"); // Table 2 should be the old table saved by path. DeltaTable table2 = DeltaTable.createIfNotExists().tableName("deltaTable2") .addColumn("value", "string") .execute(); QueryTest$.MODULE$.checkAnswer(table2.toDF(), dataDF.collectAsList()); } finally { spark.sql("DROP TABLE IF EXISTS deltaTable"); spark.sql("DROP TABLE IF EXISTS deltaTable2"); } // Test creating DeltaTable by path. String input = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "input") .toString(); dataDF.write().format("delta").mode("overwrite").save(input); DeltaTable table = createTable(true, String.format("delta.`%s`", input)); QueryTest$.MODULE$.checkAnswer(table.toDF(), dataDF.collectAsList()); } @Test public void testCreateTableWithExistingSchema() { try { // Test create table with an existing schema. List data = Arrays.asList("hello", "world"); Dataset dataDF = spark.createDataset(data, Encoders.STRING()).toDF(); DeltaLog.forTable(spark, new Path("deltaTable")).clearCache(); DeltaTable table = DeltaTable.create().tableName("deltaTable") .addColumns(dataDF.schema()) .execute(); dataDF.write().format("delta").mode("append").saveAsTable("deltaTable"); QueryTest$.MODULE$.checkAnswer(table.toDF(), dataDF.collectAsList()); } finally { spark.sql("DROP TABLE IF EXISTS deltaTable"); } } @Test public void testReplaceTable() { try { // create a table first spark.sql("CREATE TABLE deltaTable (col1 int) USING delta"); // Test replacing DeltaTable by name DeltaTable table = replaceTable(false, "deltaTable"); verifyGeneratedColumn("deltaTable", table); } finally { spark.sql("DROP TABLE IF EXISTS deltaTable"); } String input = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "input") .toString(); List data = Arrays.asList("hello", "world"); Dataset dataDF = spark.createDataset(data, Encoders.STRING()).toDF(); dataDF.write().format("delta").mode("overwrite").save(input); DeltaTable table = replaceTable(false, String.format("delta.`%s`", input)); verifyGeneratedColumn(String.format("delta.`%s`", input), table); } @Test public void testCreateOrReplaceTable() { try { // Test creating DeltaTable by name if table to be replaced does not exist. DeltaTable table = replaceTable(true, "deltaTable"); verifyGeneratedColumn("deltaTable", table); } finally { spark.sql("DROP TABLE IF EXISTS deltaTable"); } } } ================================================ FILE: spark/src/test/java/io/delta/tables/JavaDeltaTableSuite.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables; import java.util.Arrays; import java.util.List; import org.apache.spark.sql.test.*; import org.apache.spark.sql.*; import org.apache.spark.util.Utils; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.apache.spark.sql.delta.DeltaSQLCommandJavaTest; public class JavaDeltaTableSuite implements DeltaSQLCommandJavaTest { private transient SparkSession spark; private transient String input; @Before public void setUp() { // Trigger static initializer of TestData spark = buildSparkSession(); } @After public void tearDown() { if (spark != null) { spark.stop(); spark = null; } } @Test public void testAPI() { try { String input = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "input").toString(); List data = Arrays.asList("hello", "world"); Dataset dataDF = spark.createDataset(data, Encoders.STRING()).toDF(); List dataRows = dataDF.collectAsList(); dataDF.write().format("delta").mode("overwrite").save(input); // Test creating DeltaTable by path DeltaTable table1 = DeltaTable.forPath(spark, input); QueryTest$.MODULE$.checkAnswer(table1.toDF(), dataRows); // Test creating DeltaTable by path picks up active SparkSession DeltaTable table2 = DeltaTable.forPath(input); QueryTest$.MODULE$.checkAnswer(table2.toDF(), dataRows); dataDF.write().format("delta").mode("overwrite").saveAsTable("deltaTable"); // Test creating DeltaTable by name DeltaTable table3 = DeltaTable.forName(spark, "deltaTable"); QueryTest$.MODULE$.checkAnswer(table3.toDF(), dataRows); // Test creating DeltaTable by name DeltaTable table4 = DeltaTable.forName("deltaTable"); QueryTest$.MODULE$.checkAnswer(table4.toDF(), dataRows); // Test DeltaTable.as() creates subquery alias QueryTest$.MODULE$.checkAnswer(table2.as("tbl").toDF().select("tbl.value"), dataRows); // Test DeltaTable.isDeltaTable() is true for a Delta file path. Assert.assertTrue(DeltaTable.isDeltaTable(input)); } finally { spark.sql("DROP TABLE IF EXISTS deltaTable"); } } } ================================================ FILE: spark/src/test/java/org/apache/spark/sql/delta/DeleteJavaSuite.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import scala.Tuple2; import io.delta.tables.DeltaTable; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.apache.spark.sql.*; import org.apache.spark.util.Utils; public class DeleteJavaSuite implements DeltaSQLCommandJavaTest { private transient SparkSession spark; private transient String tempPath; @Before public void setUp() { spark = buildSparkSession(); tempPath = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "spark").toString(); } @After public void tearDown() { if (spark != null) { spark.stop(); spark = null; } } @Test public void testWithoutCondition() { Dataset targetTable = createKVDataSet( Arrays.asList(tuple2(1, 10), tuple2(2, 20), tuple2(3, 30), tuple2(4, 40)), "key", "value"); targetTable.write().format("delta").save(tempPath); DeltaTable target = DeltaTable.forPath(spark, tempPath); target.delete(); List expectedAnswer = new ArrayList<>(); QueryTest$.MODULE$.checkAnswer(target.toDF(), expectedAnswer); } @Test public void testWithCondition() { Dataset targetTable = createKVDataSet( Arrays.asList(tuple2(1, 10), tuple2(2, 20), tuple2(3, 30), tuple2(4, 40)), "key", "value"); targetTable.write().format("delta").save(tempPath); DeltaTable target = DeltaTable.forPath(spark, tempPath); target.delete("key = 1 or key = 2"); List expectedAnswer = createKVDataSet( Arrays.asList(tuple2(3, 30), tuple2(4, 40))).collectAsList(); QueryTest$.MODULE$.checkAnswer(target.toDF(), expectedAnswer); } @Test public void testWithColumnCondition() { Dataset targetTable = createKVDataSet( Arrays.asList(tuple2(1, 10), tuple2(2, 20), tuple2(3, 30), tuple2(4, 40)), "key", "value"); targetTable.write().format("delta").save(tempPath); DeltaTable target = DeltaTable.forPath(spark, tempPath); target.delete(functions.expr("key = 1 or key = 2")); List expectedAnswer = createKVDataSet( Arrays.asList(tuple2(3, 30), tuple2(4, 40))).collectAsList(); QueryTest$.MODULE$.checkAnswer(target.toDF(), expectedAnswer); } private Dataset createKVDataSet( List> data, String keyName, String valueName) { Encoder> encoder = Encoders.tuple(Encoders.INT(), Encoders.INT()); return spark.createDataset(data, encoder).toDF(keyName, valueName); } private Dataset createKVDataSet(List> data) { Encoder> encoder = Encoders.tuple(Encoders.INT(), Encoders.INT()); return spark.createDataset(data, encoder).toDF(); } private Tuple2 tuple2(T1 t1, T2 t2) { return new Tuple2<>(t1, t2); } } ================================================ FILE: spark/src/test/java/org/apache/spark/sql/delta/DeltaSQLCommandJavaTest.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta; import org.apache.spark.sql.SparkSession; public interface DeltaSQLCommandJavaTest { default SparkSession buildSparkSession() { // Set the configurations as DeltaSQLCommandTest return SparkSession.builder() .appName("JavaDeltaSparkSessionExtensionSuiteUsingSQLConf") .master("local[2]") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") .getOrCreate(); } } ================================================ FILE: spark/src/test/java/org/apache/spark/sql/delta/MergeIntoJavaSuite.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta; import java.io.Serializable; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import scala.Tuple2; import io.delta.tables.DeltaTable; import org.apache.spark.sql.*; import org.apache.spark.util.Utils; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.apache.spark.sql.test.TestSparkSession; import org.apache.spark.sql.delta.catalog.DeltaCatalog; import org.apache.spark.sql.internal.SQLConf; public class MergeIntoJavaSuite implements Serializable { private transient TestSparkSession spark; private transient String tempPath; @Before public void setUp() { spark = new TestSparkSession(); tempPath = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "spark").toString(); spark.sqlContext().conf().setConfString(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION().key(), DeltaCatalog.class.getCanonicalName()); } @After public void tearDown() { spark.stop(); spark = null; } @Test public void checkBasicApi() { Dataset targetTable = createKVDataSet( Arrays.asList(tuple2(1, 10), tuple2(2, 20)), "key1", "value1"); targetTable.write().format("delta").save(tempPath); Dataset sourceTable = createKVDataSet( Arrays.asList(tuple2(1, 100), tuple2(3, 30)), "key2", "value2"); DeltaTable target = DeltaTable.forPath(spark, tempPath); Map updateMap = new HashMap() {{ put("key1", "key2"); put("value1", "value2"); }}; Map insertMap = new HashMap() {{ put("key1", "key2"); put("value1", "value2"); }}; target.merge(sourceTable, "key1 = key2") .whenMatched() .updateExpr(updateMap) .whenNotMatched() .insertExpr(insertMap) .execute(); List expectedAnswer = createKVDataSet( Arrays.asList(tuple2(1, 100), tuple2(2, 20), tuple2(3, 30))).collectAsList(); QueryTest$.MODULE$.checkAnswer(target.toDF(), expectedAnswer); } @Test public void checkExtendedApi() { Dataset targetTable = createKVDataSet( Arrays.asList(tuple2(1, 10), tuple2(2, 20)), "key1", "value1"); targetTable.write().format("delta").save(tempPath); Dataset sourceTable = createKVDataSet( Arrays.asList(tuple2(1, 100), tuple2(3, 30)), "key2", "value2"); DeltaTable target = DeltaTable.forPath(spark, tempPath); Map updateMap = new HashMap() {{ put("key1", "key2"); put("value1", "value2"); }}; Map insertMap = new HashMap() {{ put("key1", "key2"); put("value1", "value2"); }}; target.merge(sourceTable, "key1 = key2") .whenMatched("key1 = 4").delete() .whenMatched("key2 = 1") .updateExpr(updateMap) .whenNotMatched("key2 = 3") .insertExpr(insertMap) .execute(); List expectedAnswer = createKVDataSet( Arrays.asList(tuple2(1, 100), tuple2(2, 20), tuple2(3, 30))).collectAsList(); QueryTest$.MODULE$.checkAnswer(target.toDF(), expectedAnswer); } @Test public void checkExtendedApiWithColumn() { Dataset targetTable = createKVDataSet( Arrays.asList(tuple2(1, 10), tuple2(2, 20), tuple2(4, 40)), "key1", "value1"); targetTable.write().format("delta").save(tempPath); Dataset sourceTable = createKVDataSet( Arrays.asList(tuple2(1, 100), tuple2(3, 30), tuple2(4, 41)), "key2", "value2"); DeltaTable target = DeltaTable.forPath(spark, tempPath); Map updateMap = new HashMap() {{ put("key1", functions.col("key2")); put("value1", functions.col("value2")); }}; Map insertMap = new HashMap() {{ put("key1", functions.col("key2")); put("value1", functions.col("value2")); }}; target.merge(sourceTable, functions.expr("key1 = key2")) .whenMatched(functions.expr("key1 = 4")).delete() .whenMatched(functions.expr("key2 = 1")) .update(updateMap) .whenNotMatched(functions.expr("key2 = 3")) .insert(insertMap) .execute(); List expectedAnswer = createKVDataSet( Arrays.asList(tuple2(1, 100), tuple2(2, 20), tuple2(3, 30))).collectAsList(); QueryTest$.MODULE$.checkAnswer(target.toDF(), expectedAnswer); } @Test public void checkUpdateAllAndInsertAll() { Dataset targetTable = createKVDataSet(Arrays.asList( tuple2(1, 10), tuple2(2, 20), tuple2(4, 40), tuple2(5, 50)), "key", "value"); targetTable.write().format("delta").save(tempPath); Dataset sourceTable = createKVDataSet(Arrays.asList( tuple2(1, 100), tuple2(3, 30), tuple2(4, 41), tuple2(5, 51), tuple2(6, 60)), "key", "value"); DeltaTable target = DeltaTable.forPath(spark, tempPath); target.as("t").merge(sourceTable.as("s"), functions.expr("t.key = s.key")) .whenMatched().updateAll() .whenNotMatched().insertAll() .execute(); List expectedAnswer = createKVDataSet(Arrays.asList(tuple2(1, 100), tuple2(2, 20), tuple2(3, 30), tuple2(4, 41), tuple2(5, 51), tuple2(6, 60))).collectAsList(); QueryTest$.MODULE$.checkAnswer(target.toDF(), expectedAnswer); } private Dataset createKVDataSet( List> data, String keyName, String valueName) { Encoder> encoder = Encoders.tuple(Encoders.INT(), Encoders.INT()); return spark.createDataset(data, encoder).toDF(keyName, valueName); } private Dataset createKVDataSet(List> data) { Encoder> encoder = Encoders.tuple(Encoders.INT(), Encoders.INT()); return spark.createDataset(data, encoder).toDF(); } private Tuple2 tuple2(T1 t1, T2 t2) { return new Tuple2<>(t1, t2); } } ================================================ FILE: spark/src/test/java/org/apache/spark/sql/delta/UpdateJavaSuite.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta; import java.util.*; import scala.Tuple2; import io.delta.tables.DeltaTable; import org.junit.*; import org.apache.spark.sql.*; import org.apache.spark.util.Utils; public class UpdateJavaSuite implements DeltaSQLCommandJavaTest { private transient SparkSession spark; private transient String tempPath; @Before public void setUp() { spark = buildSparkSession(); tempPath = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "spark").toString(); } @After public void tearDown() { if (spark != null) { spark.stop(); spark = null; } } @Test public void testWithoutCondition() { Dataset targetTable = createKVDataSet( Arrays.asList(tuple2(1, 10), tuple2(2, 20), tuple2(3, 30), tuple2(4, 40)), "key", "value"); targetTable.write().format("delta").save(tempPath); DeltaTable target = DeltaTable.forPath(spark, tempPath); Map set = new HashMap() {{ put("key", "100"); }}; target.updateExpr(set); List expectedAnswer = createKVDataSet(Arrays.asList( tuple2(100, 10), tuple2(100, 20), tuple2(100, 30), tuple2(100, 40))).collectAsList(); QueryTest$.MODULE$.checkAnswer(target.toDF(), expectedAnswer); } @Test public void testWithoutConditionUsingColumn() { Dataset targetTable = createKVDataSet( Arrays.asList(tuple2(1, 10), tuple2(2, 20), tuple2(3, 30), tuple2(4, 40)), "key", "value"); targetTable.write().format("delta").save(tempPath); DeltaTable target = DeltaTable.forPath(spark, tempPath); Map set = new HashMap() {{ put("key", functions.expr("100")); }}; target.update(set); List expectedAnswer = createKVDataSet(Arrays.asList( tuple2(100, 10), tuple2(100, 20), tuple2(100, 30), tuple2(100, 40))).collectAsList(); QueryTest$.MODULE$.checkAnswer(target.toDF(), expectedAnswer); } @Test public void testWithCondition() { Dataset targetTable = createKVDataSet( Arrays.asList(tuple2(1, 10), tuple2(2, 20), tuple2(3, 30), tuple2(4, 40)), "key", "value"); targetTable.write().format("delta").save(tempPath); DeltaTable target = DeltaTable.forPath(spark, tempPath); Map set = new HashMap() {{ put("key", "100"); }}; target.updateExpr("key = 1 or key = 2", set); List expectedAnswer = createKVDataSet(Arrays.asList( tuple2(100, 10), tuple2(100, 20), tuple2(3, 30), tuple2(4, 40))).collectAsList(); QueryTest$.MODULE$.checkAnswer(target.toDF(), expectedAnswer); } @Test public void testWithConditionUsingColumn() { Dataset targetTable = createKVDataSet( Arrays.asList(tuple2(1, 10), tuple2(2, 20), tuple2(3, 30), tuple2(4, 40)), "key", "value"); targetTable.write().format("delta").save(tempPath); DeltaTable target = DeltaTable.forPath(spark, tempPath); Map set = new HashMap() {{ put("key", functions.expr("100")); }}; target.update(functions.expr("key = 1 or key = 2"), set); List expectedAnswer = createKVDataSet(Arrays.asList( tuple2(100, 10), tuple2(100, 20), tuple2(3, 30), tuple2(4, 40))).collectAsList(); QueryTest$.MODULE$.checkAnswer(target.toDF(), expectedAnswer); } private Dataset createKVDataSet( List> data, String keyName, String valueName) { Encoder> encoder = Encoders.tuple(Encoders.INT(), Encoders.INT()); return spark.createDataset(data, encoder).toDF(keyName, valueName); } private Dataset createKVDataSet(List> data) { Encoder> encoder = Encoders.tuple(Encoders.INT(), Encoders.INT()); return spark.createDataset(data, encoder).toDF(); } private Tuple2 tuple2(T1 t1, T2 t2) { return new Tuple2<>(t1, t2); } } ================================================ FILE: spark/src/test/java/org/apache/spark/sql/delta/util/CatalogTableUtilsTest.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient; import java.util.Collections; import java.util.HashMap; import java.util.Map; import org.apache.spark.sql.catalyst.catalog.CatalogTable; import org.junit.Test; import scala.Option; /** Tests for {@link CatalogTableUtils}. */ public class CatalogTableUtilsTest { @Test public void testIsCatalogManaged_CatalogFlagEnabled_ReturnsTrue() { CatalogTable table = catalogTable( Collections.emptyMap(), Map.of(CatalogTableUtils.FEATURE_CATALOG_MANAGED, "supported")); assertTrue( "Catalog-managed flag should enable detection", CatalogTableUtils.isCatalogManaged(table)); } @Test public void testIsCatalogManaged_PreviewFlagEnabled_ReturnsTrue() { CatalogTable table = catalogTable( Collections.emptyMap(), Map.of(CatalogTableUtils.FEATURE_CATALOG_OWNED_PREVIEW, "SuPpOrTeD")); assertTrue( "Preview flag should enable detection ignoring case", CatalogTableUtils.isCatalogManaged(table)); } @Test public void testIsCatalogManaged_NoFlags_ReturnsFalse() { CatalogTable table = catalogTable(Collections.emptyMap(), Collections.emptyMap()); assertFalse( "No catalog flags should disable detection", CatalogTableUtils.isCatalogManaged(table)); } @Test public void testIsUnityCatalogManaged_FlagAndIdPresent_ReturnsTrue() { CatalogTable table = catalogTable( Collections.emptyMap(), Map.of( CatalogTableUtils.FEATURE_CATALOG_MANAGED, "supported", UCCommitCoordinatorClient.UC_TABLE_ID_KEY, "abc-123")); assertTrue( "Unity Catalog detection should require flag and identifier", CatalogTableUtils.isUnityCatalogManagedTable(table)); } @Test public void testIsUnityCatalogManaged_MissingId_ReturnsFalse() { CatalogTable table = catalogTable( Collections.emptyMap(), Map.of(CatalogTableUtils.FEATURE_CATALOG_MANAGED, "supported")); assertFalse( "Missing table identifier should break Unity detection", CatalogTableUtils.isUnityCatalogManagedTable(table)); } @Test public void testIsUnityCatalogManaged_PreviewFlagMissingId_ReturnsFalse() { CatalogTable table = catalogTable( Collections.emptyMap(), Map.of(CatalogTableUtils.FEATURE_CATALOG_OWNED_PREVIEW, "supported")); assertFalse( "Preview flag without ID should not be considered Unity managed", CatalogTableUtils.isUnityCatalogManagedTable(table)); } @Test public void testIsCatalogManaged_NullStorage_ReturnsFalse() { CatalogTable table = catalogTableWithNullStorage(Collections.emptyMap()); assertFalse( "Null storage should not be considered catalog managed", CatalogTableUtils.isCatalogManaged(table)); } @Test public void testIsUnityCatalogManaged_NullStorage_ReturnsFalse() { CatalogTable table = catalogTableWithNullStorage(Collections.emptyMap()); assertFalse( "Null storage should not be considered Unity managed", CatalogTableUtils.isUnityCatalogManagedTable(table)); } @Test public void testIsCatalogManaged_NullStorageProperties_ReturnsFalse() { CatalogTable table = catalogTableWithNullStorageProperties(Collections.emptyMap()); assertFalse( "Null storage properties should not be considered catalog managed", CatalogTableUtils.isCatalogManaged(table)); } @Test public void testIsUnityCatalogManaged_NullStorageProperties_ReturnsFalse() { CatalogTable table = catalogTableWithNullStorageProperties(Collections.emptyMap()); assertFalse( "Null storage properties should not be considered Unity managed", CatalogTableUtils.isUnityCatalogManagedTable(table)); } private static CatalogTable catalogTable( Map properties, Map storageProperties) { return CatalogTableTestUtils$.MODULE$.createCatalogTable( "tbl" /* tableName */, Option.empty() /* catalogName */, properties, storageProperties, Option.empty() /* locationUri */, false /* nullStorage */, false /* nullStorageProperties */); } private static CatalogTable catalogTableWithNullStorage(Map properties) { return CatalogTableTestUtils$.MODULE$.createCatalogTable( "tbl" /* tableName */, Option.empty() /* catalogName */, properties, new HashMap<>() /* storageProperties */, Option.empty() /* locationUri */, true /* nullStorage */, false /* nullStorageProperties */); } private static CatalogTable catalogTableWithNullStorageProperties( Map properties) { return CatalogTableTestUtils$.MODULE$.createCatalogTable( "tbl" /* tableName */, Option.empty() /* catalogName */, properties, new HashMap<>() /* storageProperties */, Option.empty() /* locationUri */, false /* nullStorage */, true /* nullStorageProperties */); } } ================================================ FILE: spark/src/test/resources/delta/dbr_8_0_non_generated_columns/_delta_log/00000000000000000000.crc ================================================ {"tableSizeBytes":422,"numFiles":1,"numMetadata":1,"numProtocol":1,"numTransactions":0} ================================================ FILE: spark/src/test/resources/delta/dbr_8_0_non_generated_columns/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1617557139648,"operation":"CREATE TABLE AS SELECT","operationParameters":{"isManaged":"false","description":null,"partitionBy":"[]","properties":"{}"},"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputBytes":"422","numOutputRows":"0"}}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"027fb01c-94aa-4cab-87cb-5aab6aec6d17","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"c1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c2\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.generationExpression\":\"c1 + 1\"}}]}","partitionColumns":[],"configuration":{},"createdTime":1617557137253}} {"add":{"path":"part-00000-74e02f0d-e727-46e5-8d74-779d2abd616e-c000.snappy.parquet","partitionValues":{},"size":422,"modificationTime":1617557139000,"dataChange":true,"stats":"{\"numRecords\":0,\"minValues\":{},\"maxValues\":{},\"nullCount\":{}}"}} ================================================ FILE: spark/src/test/resources/delta/dbr_8_1_generated_columns/_delta_log/00000000000000000000.crc ================================================ {"tableSizeBytes":0,"numFiles":0,"numMetadata":1,"numProtocol":1,"numTransactions":0} ================================================ FILE: spark/src/test/resources/delta/dbr_8_1_generated_columns/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1617556462951,"operation":"CREATE TABLE","operationParameters":{"isManaged":"false","description":null,"partitionBy":"[]","properties":"{}"},"isolationLevel":"SnapshotIsolation","isBlindAppend":true,"operationMetrics":{}}} {"protocol":{"minReaderVersion":1,"minWriterVersion":4}} {"metaData":{"id":"b406888a-3eb9-4dd5-a81a-ed0b0b535c00","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"c1\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"c2\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.generationExpression\":\"c1 + 1\"}}]}","partitionColumns":[],"configuration":{},"createdTime":1617556462734}} ================================================ FILE: spark/src/test/resources/delta/delta-0.1.0/_delta_log/00000000000000000000.json ================================================ {"protocol":{"minReaderVersion":1,"minWriterVersion":1}} {"metaData":{"id":"2edf2c02-bb63-44e9-a84c-517fad0db296","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"value\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{}}} {"add":{"path":"part-00000-f4aeebd0-a689-4e1b-bc7a-bbb0ec59dce5-c000.snappy.parquet","partitionValues":{},"size":525,"modificationTime":1501109075000,"dataChange":true}} {"add":{"path":"part-00001-f1cb1cf9-7a73-439c-b0ea-dcba5c2280a6-c000.snappy.parquet","partitionValues":{},"size":534,"modificationTime":1501109075000,"dataChange":true}} ================================================ FILE: spark/src/test/resources/delta/delta-0.1.0/_delta_log/00000000000000000001.json ================================================ {"remove":{"path":"part-00001-f1cb1cf9-7a73-439c-b0ea-dcba5c2280a6-c000.snappy.parquet","dataChange":true}} {"remove":{"path":"part-00000-f4aeebd0-a689-4e1b-bc7a-bbb0ec59dce5-c000.snappy.parquet","dataChange":true}} ================================================ FILE: spark/src/test/resources/delta/delta-0.1.0/_delta_log/00000000000000000002.json ================================================ {"txn":{"appId":"txnId","version":0}} {"add":{"path":"part-00000-348d7f43-38f6-4778-88c7-45f379471c49-c000.snappy.parquet","partitionValues":{},"size":525,"modificationTime":1501109075000,"dataChange":true}} {"add":{"path":"part-00001-6d252218-2632-416e-9e46-f32316ec314a-c000.snappy.parquet","partitionValues":{},"size":534,"modificationTime":1501109075000,"dataChange":true}} ================================================ FILE: spark/src/test/resources/delta/delta-0.1.0/_delta_log/00000000000000000003.json ================================================ {"metaData":{"id":"2edf2c02-bb63-44e9-a84c-517fad0db296","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"value\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["id"],"configuration":{}}} {"remove":{"path":"part-00001-6d252218-2632-416e-9e46-f32316ec314a-c000.snappy.parquet","dataChange":true}} {"remove":{"path":"part-00000-348d7f43-38f6-4778-88c7-45f379471c49-c000.snappy.parquet","dataChange":true}} {"add":{"path":"id=5/part-00000-f1e0b560-ca00-409e-a274-f1ab264bc412.c000.snappy.parquet","partitionValues":{"id":"5"},"size":362,"modificationTime":1501109076000,"dataChange":true}} {"add":{"path":"id=6/part-00000-adb59f54-6b8f-4bfd-9915-ae26bd0f0e2c.c000.snappy.parquet","partitionValues":{"id":"6"},"size":362,"modificationTime":1501109076000,"dataChange":true}} {"add":{"path":"id=4/part-00001-36c738bf-7836-479b-9cc1-7a4934207856.c000.snappy.parquet","partitionValues":{"id":"4"},"size":362,"modificationTime":1501109076000,"dataChange":true}} ================================================ FILE: spark/src/test/resources/delta/delta-0.1.0/_delta_log/_last_checkpoint ================================================ {"version":3,"size":6} ================================================ FILE: spark/src/test/resources/delta/delta-1.2.1/_delta_log/00000000000000000000.json ================================================ {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"fbfd25ac-9401-4dac-a644-ae543f02cc0f","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"value\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"col1\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"col2\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1657517977667}} {"add":{"path":"part-00000-87624dd4-c6dc-4163-a4e6-0e50caa28760-c000.snappy.parquet","partitionValues":{},"size":1124,"modificationTime":1657517977000,"dataChange":true,"stats":"{\"numRecords\":11,\"minValues\":{\"value\":0,\"col1\":0,\"col2\":0},\"maxValues\":{\"value\":10,\"col1\":6,\"col2\":2},\"nullCount\":{\"value\":0,\"col1\":0,\"col2\":0}}"}} {"commitInfo":{"timestamp":1657517977863,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"11","numOutputBytes":"1124"},"engineInfo":"Apache-Spark/3.2.1 Delta-Lake/1.2.1","txnId":"57be32c2-4b7d-415a-96a0-1499caf659e5"}} ================================================ FILE: spark/src/test/resources/delta/delta-1.2.1/_delta_log/00000000000000000001.json ================================================ {"metaData":{"id":"fbfd25ac-9401-4dac-a644-ae543f02cc0f","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"value\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"col1\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"col2\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2"},"createdTime":1657517977667}} {"commitInfo":{"timestamp":1657517989647,"operation":"SET TBLPROPERTIES","operationParameters":{"properties":"{\"delta.checkpointInterval\":\"2\"}"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.2.1 Delta-Lake/1.2.1","txnId":"b53af69e-b0aa-423b-af05-c3bff1c35a11"}} ================================================ FILE: spark/src/test/resources/delta/delta-1.2.1/_delta_log/00000000000000000002.json ================================================ {"add":{"path":"part-00000-59316e80-0f6c-491a-9716-5e0419434e46-c000.snappy.parquet","partitionValues":{},"size":1124,"modificationTime":1657517994000,"dataChange":true,"stats":"{\"numRecords\":11,\"minValues\":{\"value\":0,\"col1\":0,\"col2\":0},\"maxValues\":{\"value\":10,\"col1\":6,\"col2\":2},\"nullCount\":{\"value\":0,\"col1\":0,\"col2\":0}}"}} {"commitInfo":{"timestamp":1657517994301,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"11","numOutputBytes":"1124"},"engineInfo":"Apache-Spark/3.2.1 Delta-Lake/1.2.1","txnId":"6e6280cc-b8af-4e60-b2e1-766690b9faee"}} ================================================ FILE: spark/src/test/resources/delta/delta-1.2.1/_delta_log/00000000000000000003.json ================================================ {"add":{"path":"part-00000-635b7994-d3f9-4623-b032-8a9c8a7ca5b9-c000.snappy.parquet","partitionValues":{},"size":1124,"modificationTime":1657518013000,"dataChange":true,"stats":"{\"numRecords\":11,\"minValues\":{\"value\":0,\"col1\":0,\"col2\":0},\"maxValues\":{\"value\":10,\"col1\":6,\"col2\":2},\"nullCount\":{\"value\":0,\"col1\":0,\"col2\":0}}"}} {"commitInfo":{"timestamp":1657518013762,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"11","numOutputBytes":"1124"},"engineInfo":"Apache-Spark/3.2.1 Delta-Lake/1.2.1","txnId":"8e95e72f-dee7-4e0b-abb6-a47b4bcc46d2"}} ================================================ FILE: spark/src/test/resources/delta/delta-1.2.1/_delta_log/00000000000000000004.json ================================================ {"remove":{"path":"part-00000-635b7994-d3f9-4623-b032-8a9c8a7ca5b9-c000.snappy.parquet","deletionTimestamp":1657518515173,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1124}} {"remove":{"path":"part-00000-87624dd4-c6dc-4163-a4e6-0e50caa28760-c000.snappy.parquet","deletionTimestamp":1657518515173,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1124}} {"remove":{"path":"part-00000-59316e80-0f6c-491a-9716-5e0419434e46-c000.snappy.parquet","deletionTimestamp":1657518515173,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1124}} {"add":{"path":"part-00000-e107d259-11d5-4e5b-b472-62daa676743b-c000.snappy.parquet","partitionValues":{},"size":1124,"modificationTime":1657518515000,"dataChange":true,"stats":"{\"numRecords\":11,\"minValues\":{\"value\":0,\"col1\":0,\"col2\":0},\"maxValues\":{\"value\":10,\"col1\":8,\"col2\":2},\"nullCount\":{\"value\":0,\"col1\":0,\"col2\":0}}"}} {"add":{"path":"part-00001-91d10124-a73d-42c2-9ef0-75ed41ca73d8-c000.snappy.parquet","partitionValues":{},"size":1124,"modificationTime":1657518515000,"dataChange":true,"stats":"{\"numRecords\":11,\"minValues\":{\"value\":0,\"col1\":0,\"col2\":0},\"maxValues\":{\"value\":10,\"col1\":8,\"col2\":2},\"nullCount\":{\"value\":0,\"col1\":0,\"col2\":0}}"}} {"add":{"path":"part-00002-dca394a5-9d0a-4630-a90a-a8f7f675e4e4-c000.snappy.parquet","partitionValues":{},"size":1124,"modificationTime":1657518515000,"dataChange":true,"stats":"{\"numRecords\":11,\"minValues\":{\"value\":0,\"col1\":0,\"col2\":0},\"maxValues\":{\"value\":10,\"col1\":8,\"col2\":2},\"nullCount\":{\"value\":0,\"col1\":0,\"col2\":0}}"}} {"commitInfo":{"timestamp":1657518515749,"operation":"UPDATE","operationParameters":{"predicate":"(col2#477L = 2)"},"readVersion":3,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"3","numCopiedRows":"24","executionTimeMs":"2306","scanTimeMs":"1738","numAddedFiles":"3","numUpdatedRows":"9","rewriteTimeMs":"568"},"engineInfo":"Apache-Spark/3.2.1 Delta-Lake/1.2.1","txnId":"342d874b-a8e5-49a0-8641-7e5b2285d7cb"}} ================================================ FILE: spark/src/test/resources/delta/delta-1.2.1/_delta_log/_last_checkpoint ================================================ {"version":4,"size":8} ================================================ FILE: spark/src/test/resources/delta/history/delta-0.2.0/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1564524295023,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isBlindAppend":true}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"22ef18ba-191c-4c36-a606-3dad5cdf3830","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"value\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1564524294376}} {"add":{"path":"part-00000-b44fcdb0-8b06-4f3a-8606-f8311a96f6dc-c000.snappy.parquet","partitionValues":{},"size":396,"modificationTime":1564524294000,"dataChange":true}} {"add":{"path":"part-00001-185eca06-e017-4dea-ae49-fc48b973e37e-c000.snappy.parquet","partitionValues":{},"size":400,"modificationTime":1564524294000,"dataChange":true}} ================================================ FILE: spark/src/test/resources/delta/history/delta-0.2.0/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1564524296741,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isBlindAppend":true}} {"add":{"path":"part-00000-512e1537-8aaa-4193-b8b4-bef3de0de409-c000.snappy.parquet","partitionValues":{},"size":396,"modificationTime":1564524296000,"dataChange":true}} {"add":{"path":"part-00001-4327c977-2734-4477-9507-7ccf67924649-c000.snappy.parquet","partitionValues":{},"size":400,"modificationTime":1564524296000,"dataChange":true}} ================================================ FILE: spark/src/test/resources/delta/history/delta-0.2.0/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1564524298214,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":1,"isBlindAppend":false}} {"add":{"path":"part-00000-7c2deba3-1994-4fb8-bc07-d46c948aa415-c000.snappy.parquet","partitionValues":{},"size":396,"modificationTime":1564524297000,"dataChange":true}} {"add":{"path":"part-00001-c373a5bd-85f0-4758-815e-7eb62007a15c-c000.snappy.parquet","partitionValues":{},"size":400,"modificationTime":1564524297000,"dataChange":true}} {"remove":{"path":"part-00000-512e1537-8aaa-4193-b8b4-bef3de0de409-c000.snappy.parquet","deletionTimestamp":1564524298213,"dataChange":true}} {"remove":{"path":"part-00000-b44fcdb0-8b06-4f3a-8606-f8311a96f6dc-c000.snappy.parquet","deletionTimestamp":1564524298214,"dataChange":true}} {"remove":{"path":"part-00001-185eca06-e017-4dea-ae49-fc48b973e37e-c000.snappy.parquet","deletionTimestamp":1564524298214,"dataChange":true}} {"remove":{"path":"part-00001-4327c977-2734-4477-9507-7ccf67924649-c000.snappy.parquet","deletionTimestamp":1564524298214,"dataChange":true}} ================================================ FILE: spark/src/test/resources/delta/history/delta-0.2.0/_delta_log/00000000000000000003.json ================================================ {"commitInfo":{"timestamp":1564524299648,"operation":"STREAMING UPDATE","operationParameters":{"outputMode":"Append","queryId":"e4a20b59-dd0e-4c50-b074-e8ae4786df30","epochId":"0"},"readVersion":2,"isBlindAppend":true}} {"txn":{"appId":"e4a20b59-dd0e-4c50-b074-e8ae4786df30","version":0,"lastUpdated":1564524299648}} {"add":{"path":"part-00000-cb6b150b-30b8-4662-ad28-ff32ddab96d2-c000.snappy.parquet","partitionValues":{},"size":404,"modificationTime":1564524299000,"dataChange":true}} ================================================ FILE: spark/src/test/resources/delta/history/delta-0.2.0/_delta_log/_last_checkpoint ================================================ {"version":3,"size":10} ================================================ FILE: spark/src/test/resources/delta/identity_test_written_by_version_5/_delta_log/00000000000000000000.crc ================================================ {"tableSizeBytes":2303,"numFiles":2,"numMetadata":1,"numProtocol":1,"numTransactions":0,"protocol":{"minReaderVersion":1,"minWriterVersion":2},"metadata":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{\"delta.identity.start\":1,\"delta.identity.step\":1,\"delta.identity.highWaterMark\":4,\"delta.identity.allowExplicitInsert\":true}},{\"name\":\"part\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"value\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1638474481770}} ================================================ FILE: spark/src/test/resources/delta/identity_test_written_by_version_5/_delta_log/00000000000000000000.json ================================================ {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{\"delta.identity.start\":1,\"delta.identity.step\":1,\"delta.identity.highWaterMark\":4,\"delta.identity.allowExplicitInsert\":true}},{\"name\":\"part\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"value\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1638474481770}} {"add":{"path":"part-00000-1ec4087c-3109-48b4-9e1c-c44cad50f3d8-c000.snappy.parquet","partitionValues":{},"size":1149,"modificationTime":1638474496727,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"id\":1,\"part\":1,\"value\":\"one\"},\"maxValues\":{\"id\":2,\"part\":2,\"value\":\"two\"},\"nullCount\":{\"id\":0,\"part\":0,\"value\":0}}","tags":{"INSERTION_TIME":"1638474496727000","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"part-00001-77d98c61-0299-4a5a-b68d-305cab1a46f6-c000.snappy.parquet","partitionValues":{},"size":1154,"modificationTime":1638474496727,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":4,\"part\":3,\"value\":\"three\"},\"maxValues\":{\"id\":4,\"part\":3,\"value\":\"three\"},\"nullCount\":{\"id\":0,\"part\":0,\"value\":0}}","tags":{"INSERTION_TIME":"1638474496727001","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"commitInfo":{"timestamp":1638474497049,"operation":"CREATE TABLE AS SELECT","operationParameters":{"isManaged":"false","description":null,"partitionBy":"[]","properties":"{}"},"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"3","numOutputBytes":"2303"}}} ================================================ FILE: spark/src/test/resources/delta/partitioned-table-with-dv-large/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1675465305121,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"partCol\"]"},"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"10","numOutputRows":"2000","numOutputBytes":"13989"},"engineInfo":"","txnId":"ec179bfe-cc75-442f-bf1f-75a7a499d1ae"}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["deletionVectors"],"writerFeatures":["deletionVectors"]}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"partCol\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["partCol"],"configuration":{"delta.enableDeletionVectors":"true"},"createdTime":1675465301176}} {"add":{"path":"partCol=0/part-00000-757a3870-38dd-41ac-86f1-e1e6826df6bc.c000.snappy.parquet","partitionValues":{"partCol":"0"},"size":1399,"modificationTime":1675465304390,"dataChange":true,"stats":"{\"numRecords\":200,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":1990},\"nullCount\":{\"id\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1675465304390000","MIN_INSERTION_TIME":"1675465304390000","MAX_INSERTION_TIME":"1675465304390000","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"partCol=1/part-00000-ffe81e1a-1a1f-4803-bc2a-e68f7b2ea122.c000.snappy.parquet","partitionValues":{"partCol":"1"},"size":1399,"modificationTime":1675465304503,"dataChange":true,"stats":"{\"numRecords\":200,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":1991},\"nullCount\":{\"id\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1675465304390001","MIN_INSERTION_TIME":"1675465304390001","MAX_INSERTION_TIME":"1675465304390001","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"partCol=2/part-00000-5963000f-3e52-4c43-a106-d7e527f5722a.c000.snappy.parquet","partitionValues":{"partCol":"2"},"size":1399,"modificationTime":1675465304550,"dataChange":true,"stats":"{\"numRecords\":200,\"minValues\":{\"id\":2},\"maxValues\":{\"id\":1992},\"nullCount\":{\"id\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1675465304390002","MIN_INSERTION_TIME":"1675465304390002","MAX_INSERTION_TIME":"1675465304390002","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"partCol=3/part-00000-068d9a17-0362-43f9-ad68-6bfcbd27448d.c000.snappy.parquet","partitionValues":{"partCol":"3"},"size":1397,"modificationTime":1675465304596,"dataChange":true,"stats":"{\"numRecords\":200,\"minValues\":{\"id\":3},\"maxValues\":{\"id\":1993},\"nullCount\":{\"id\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1675465304390003","MIN_INSERTION_TIME":"1675465304390003","MAX_INSERTION_TIME":"1675465304390003","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"partCol=4/part-00000-c66868e5-d1e0-4f22-ae89-9cc4d2a133fa.c000.snappy.parquet","partitionValues":{"partCol":"4"},"size":1400,"modificationTime":1675465304641,"dataChange":true,"stats":"{\"numRecords\":200,\"minValues\":{\"id\":4},\"maxValues\":{\"id\":1994},\"nullCount\":{\"id\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1675465304390004","MIN_INSERTION_TIME":"1675465304390004","MAX_INSERTION_TIME":"1675465304390004","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"partCol=5/part-00000-70dbcf83-e5c0-4c91-8e1a-be86f08b98f4.c000.snappy.parquet","partitionValues":{"partCol":"5"},"size":1399,"modificationTime":1675465304685,"dataChange":true,"stats":"{\"numRecords\":200,\"minValues\":{\"id\":5},\"maxValues\":{\"id\":1995},\"nullCount\":{\"id\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1675465304390005","MIN_INSERTION_TIME":"1675465304390005","MAX_INSERTION_TIME":"1675465304390005","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"partCol=6/part-00000-34e763ec-3291-4cd0-9b90-fd2d24c68098.c000.snappy.parquet","partitionValues":{"partCol":"6"},"size":1399,"modificationTime":1675465304728,"dataChange":true,"stats":"{\"numRecords\":200,\"minValues\":{\"id\":6},\"maxValues\":{\"id\":1996},\"nullCount\":{\"id\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1675465304390006","MIN_INSERTION_TIME":"1675465304390006","MAX_INSERTION_TIME":"1675465304390006","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"partCol=7/part-00000-f43c32e8-3996-43ae-9b14-9b7f8fec6221.c000.snappy.parquet","partitionValues":{"partCol":"7"},"size":1399,"modificationTime":1675465304770,"dataChange":true,"stats":"{\"numRecords\":200,\"minValues\":{\"id\":7},\"maxValues\":{\"id\":1997},\"nullCount\":{\"id\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1675465304390007","MIN_INSERTION_TIME":"1675465304390007","MAX_INSERTION_TIME":"1675465304390007","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"partCol=8/part-00000-a1137e9e-5425-4589-b039-84378f061fc4.c000.snappy.parquet","partitionValues":{"partCol":"8"},"size":1399,"modificationTime":1675465304879,"dataChange":true,"stats":"{\"numRecords\":200,\"minValues\":{\"id\":8},\"maxValues\":{\"id\":1998},\"nullCount\":{\"id\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1675465304390008","MIN_INSERTION_TIME":"1675465304390008","MAX_INSERTION_TIME":"1675465304390008","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"partCol=9/part-00000-6bcf7302-8e23-4613-aec2-02856f8f1d05.c000.snappy.parquet","partitionValues":{"partCol":"9"},"size":1399,"modificationTime":1675465304928,"dataChange":true,"stats":"{\"numRecords\":200,\"minValues\":{\"id\":9},\"maxValues\":{\"id\":1999},\"nullCount\":{\"id\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1675465304390009","MIN_INSERTION_TIME":"1675465304390009","MAX_INSERTION_TIME":"1675465304390009","OPTIMIZE_TARGET_SIZE":"268435456"}}} ================================================ FILE: spark/src/test/resources/delta/partitioned-table-with-dv-large/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1675465322730,"operation":"DELETE","operationParameters":{"predicate":"[\"(spark_catalog.delta.`/private/var/folders/g3/hcd28y8s71s0yh7whh443wz00000gp/T/spark-2434260e-1ecd-45b0-b08a-62dd7928b9ae`.id IN (0, 180, 308, 225, 756, 1007, 1503))\"]"},"readVersion":0,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numCopiedRows":"0","numDeletionVectorsAdded":"6","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"0","executionTimeMs":"11013","numDeletionVectorsUpdated":"0","numDeletedRows":"7","scanTimeMs":"10438","numAddedFiles":"0","rewriteTimeMs":"557"},"engineInfo":"","txnId":"bf3a73e8-ad42-4a6a-8c7f-4430e1891c36"}} {"remove":{"path":"partCol=8/part-00000-a1137e9e-5425-4589-b039-84378f061fc4.c000.snappy.parquet","deletionTimestamp":1675465322727,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"partCol":"8"},"size":1399,"tags":{"INSERTION_TIME":"1675465304390008","MIN_INSERTION_TIME":"1675465304390008","MAX_INSERTION_TIME":"1675465304390008","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"remove":{"path":"partCol=5/part-00000-70dbcf83-e5c0-4c91-8e1a-be86f08b98f4.c000.snappy.parquet","deletionTimestamp":1675465322727,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"partCol":"5"},"size":1399,"tags":{"INSERTION_TIME":"1675465304390005","MIN_INSERTION_TIME":"1675465304390005","MAX_INSERTION_TIME":"1675465304390005","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"remove":{"path":"partCol=3/part-00000-068d9a17-0362-43f9-ad68-6bfcbd27448d.c000.snappy.parquet","deletionTimestamp":1675465322727,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"partCol":"3"},"size":1397,"tags":{"INSERTION_TIME":"1675465304390003","MIN_INSERTION_TIME":"1675465304390003","MAX_INSERTION_TIME":"1675465304390003","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"remove":{"path":"partCol=7/part-00000-f43c32e8-3996-43ae-9b14-9b7f8fec6221.c000.snappy.parquet","deletionTimestamp":1675465322727,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"partCol":"7"},"size":1399,"tags":{"INSERTION_TIME":"1675465304390007","MIN_INSERTION_TIME":"1675465304390007","MAX_INSERTION_TIME":"1675465304390007","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"remove":{"path":"partCol=6/part-00000-34e763ec-3291-4cd0-9b90-fd2d24c68098.c000.snappy.parquet","deletionTimestamp":1675465322727,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"partCol":"6"},"size":1399,"tags":{"INSERTION_TIME":"1675465304390006","MIN_INSERTION_TIME":"1675465304390006","MAX_INSERTION_TIME":"1675465304390006","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"remove":{"path":"partCol=0/part-00000-757a3870-38dd-41ac-86f1-e1e6826df6bc.c000.snappy.parquet","deletionTimestamp":1675465322727,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"partCol":"0"},"size":1399,"tags":{"INSERTION_TIME":"1675465304390000","MIN_INSERTION_TIME":"1675465304390000","MAX_INSERTION_TIME":"1675465304390000","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"partCol=8/part-00000-a1137e9e-5425-4589-b039-84378f061fc4.c000.snappy.parquet","partitionValues":{"partCol":"8"},"size":1399,"modificationTime":1675465304879,"dataChange":true,"stats":"{\"numRecords\":200,\"minValues\":{\"id\":8},\"maxValues\":{\"id\":1998},\"nullCount\":{\"id\":0},\"tightBounds\":false}","tags":{"INSERTION_TIME":"1675465304390008","MIN_INSERTION_TIME":"1675465304390008","MAX_INSERTION_TIME":"1675465304390008","OPTIMIZE_TARGET_SIZE":"268435456"},"deletionVector":{"storageType":"u","pathOrInlineDv":"24t","txnId":"c72d2694-23fb-4adc-a315-8ee8c30853b0"}} {"add":{"path":"partCol=6/part-00000-2dee959e-3d92-4c43-ac01-24d888ba82fd.c000.snappy.parquet","partitionValues":{"partCol":"6"},"size":586,"modificationTime":1675465324549,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":756},\"maxValues\":{\"id\":756},\"nullCount\":{\"id\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1675465324549000","MIN_INSERTION_TIME":"1675465324549000","MAX_INSERTION_TIME":"1675465324549000","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"partCol=8/part-00000-fe120a67-87dc-4997-8811-3ad9d8dc3743.c000.snappy.parquet","partitionValues":{"partCol":"8"},"size":586,"modificationTime":1675465324578,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":308},\"maxValues\":{\"id\":308},\"nullCount\":{\"id\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1675465324549001","MIN_INSERTION_TIME":"1675465324549001","MAX_INSERTION_TIME":"1675465324549001","OPTIMIZE_TARGET_SIZE":"268435456"}}} ================================================ FILE: spark/src/test/resources/delta/partitioned-table-with-dv-large/_delta_log/00000000000000000003.json ================================================ {"commitInfo":{"timestamp":1675465327086,"operation":"DELETE","operationParameters":{"predicate":"[\"(spark_catalog.delta.`/private/var/folders/g3/hcd28y8s71s0yh7whh443wz00000gp/T/spark-2434260e-1ecd-45b0-b08a-62dd7928b9ae`.id IN (300, 257, 399, 786, 1353, 1567, 1800))\"]"},"readVersion":2,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numCopiedRows":"0","numDeletionVectorsAdded":"1","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"0","executionTimeMs":"1484","numDeletionVectorsUpdated":"4","numDeletedRows":"7","scanTimeMs":"779","numAddedFiles":"0","rewriteTimeMs":"703"},"engineInfo":"","txnId":"67b81203-e0e8-4eca-bb04-5806f4b1cad5"}} {"remove":{"path":"partCol=0/part-00000-757a3870-38dd-41ac-86f1-e1e6826df6bc.c000.snappy.parquet","deletionTimestamp":1675465327084,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"partCol":"0"},"size":1399,"tags":{"INSERTION_TIME":"1675465304390000","MIN_INSERTION_TIME":"1675465304390000","MAX_INSERTION_TIME":"1675465304390000","OPTIMIZE_TARGET_SIZE":"268435456"},"deletionVector":{"storageType":"u","pathOrInlineDv":"24t","txnId":"14dd0cb9-4d96-487f-af5b-1e29a5c1fa70"}} {"add":{"path":"partCol=3/part-00000-8775b518-3470-41d4-8d7e-27596c48053e.c000.snappy.parquet","partitionValues":{"partCol":"3"},"size":585,"modificationTime":1675465328471,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1353},\"maxValues\":{\"id\":1353},\"nullCount\":{\"id\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1675465328471000","MIN_INSERTION_TIME":"1675465328471000","MAX_INSERTION_TIME":"1675465328471000","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"partCol=7/part-00000-156df4a5-759c-4b9f-82b1-9727a62b7990.c000.snappy.parquet","partitionValues":{"partCol":"7"},"size":586,"modificationTime":1675465328500,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1567},\"maxValues\":{\"id\":1567},\"nullCount\":{\"id\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1675465328471001","MIN_INSERTION_TIME":"1675465328471001","MAX_INSERTION_TIME":"1675465328471001","OPTIMIZE_TARGET_SIZE":"268435456"}}} ================================================ FILE: spark/src/test/resources/delta/table-with-dv-gigantic/_delta_log/00000000000000000000.json ================================================ {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["deletionVectors"],"writerFeatures":["deletionVectors"]}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"value\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableDeletionVectors":"true"},"createdTime":1682351914000}} {"add":{"path":"part-00000-2bc940f0-dd3f-461d-8581-136026bf6f95-c000.snappy.parquet","partitionValues":{},"size":8473865,"modificationTime":1682351914339,"dataChange":true,"stats":"{\"numRecords\":2147483658,\"minValues\":{\"value\":0},\"maxValues\":{\"value\":21},\"nullCount\":{\"value\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"o6J(G4p@f*QZS+b{khvI","offset":1,"sizeInBytes":4557136,"cardinality":2147484}}} ================================================ FILE: spark/src/test/resources/delta/table-with-dv-large/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1674064770682,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"20","numOutputRows":"2000","numOutputBytes":"20157"},"engineInfo":"","txnId":"f0ddc566-dfe6-4bd8-b264-ce100f9362ef"}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["deletionVectors"],"writerFeatures":["deletionVectors"]}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"value\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableDeletionVectors":"true"},"createdTime":1674064767118}} {"add":{"path":"part-00000-f5c18e7b-d1bf-4ba5-85dd-e63ddc5931bf-c000.snappy.parquet","partitionValues":{},"size":1008,"modificationTime":1674064769860,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":4},\"maxValues\":{\"value\":1967},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064769860000","MIN_INSERTION_TIME":"1674064769860000","MAX_INSERTION_TIME":"1674064769860000","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"part-00001-5dbf0ba2-220a-4770-8e26-18a77cf875f0-c000.snappy.parquet","partitionValues":{},"size":1008,"modificationTime":1674064769860,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":18},\"maxValues\":{\"value\":1988},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064769860001","MIN_INSERTION_TIME":"1674064769860001","MAX_INSERTION_TIME":"1674064769860001","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"part-00002-5459a52f-3fd3-4b79-83a6-e7f57db28650-c000.snappy.parquet","partitionValues":{},"size":1007,"modificationTime":1674064770019,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":16},\"maxValues\":{\"value\":1977},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064769860002","MIN_INSERTION_TIME":"1674064769860002","MAX_INSERTION_TIME":"1674064769860002","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"part-00003-0e842060-9e04-4896-ba21-029309ab8736-c000.snappy.parquet","partitionValues":{},"size":1008,"modificationTime":1674064770019,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":5},\"maxValues\":{\"value\":1982},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064769860003","MIN_INSERTION_TIME":"1674064769860003","MAX_INSERTION_TIME":"1674064769860003","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"part-00004-a72dbdec-2d0e-43d8-a756-4d0d63ef9fcb-c000.snappy.parquet","partitionValues":{},"size":1008,"modificationTime":1674064770100,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":1},\"maxValues\":{\"value\":1999},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064769860004","MIN_INSERTION_TIME":"1674064769860004","MAX_INSERTION_TIME":"1674064769860004","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"part-00005-0972979f-852d-4f3e-8f64-bf0bf072de5f-c000.snappy.parquet","partitionValues":{},"size":1008,"modificationTime":1674064770100,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":8},\"maxValues\":{\"value\":1914},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064769860005","MIN_INSERTION_TIME":"1674064769860005","MAX_INSERTION_TIME":"1674064769860005","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"part-00006-227c6a1e-0180-4feb-8816-19eccf7939f5-c000.snappy.parquet","partitionValues":{},"size":1008,"modificationTime":1674064770207,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":30},\"maxValues\":{\"value\":1992},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064769860006","MIN_INSERTION_TIME":"1674064769860006","MAX_INSERTION_TIME":"1674064769860006","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"part-00007-7c37e5e3-abb2-419e-8cba-eba4eeb3b11a-c000.snappy.parquet","partitionValues":{},"size":1008,"modificationTime":1674064770207,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":40},\"maxValues\":{\"value\":1990},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064769860007","MIN_INSERTION_TIME":"1674064769860007","MAX_INSERTION_TIME":"1674064769860007","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"part-00008-1a0b4375-bbcc-4f3c-8e51-ecb551c89430-c000.snappy.parquet","partitionValues":{},"size":1008,"modificationTime":1674064770265,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":13},\"maxValues\":{\"value\":1897},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064769860008","MIN_INSERTION_TIME":"1674064769860008","MAX_INSERTION_TIME":"1674064769860008","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"part-00009-52689115-1770-4f15-b98d-b942db5b7359-c000.snappy.parquet","partitionValues":{},"size":1008,"modificationTime":1674064770265,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":12},\"maxValues\":{\"value\":1987},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064769860009","MIN_INSERTION_TIME":"1674064769860009","MAX_INSERTION_TIME":"1674064769860009","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"part-00010-7f35fa1b-7993-4aff-8f60-2b76f1eb3f2c-c000.snappy.parquet","partitionValues":{},"size":1008,"modificationTime":1674064770319,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":19},\"maxValues\":{\"value\":1993},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064769860010","MIN_INSERTION_TIME":"1674064769860010","MAX_INSERTION_TIME":"1674064769860010","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"part-00011-fce7841f-be9a-43b8-b283-9e2308ef5487-c000.snappy.parquet","partitionValues":{},"size":1008,"modificationTime":1674064770319,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":11},\"maxValues\":{\"value\":1984},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064769860011","MIN_INSERTION_TIME":"1674064769860011","MAX_INSERTION_TIME":"1674064769860011","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"part-00012-9b83c213-31ff-4b2c-a5d9-be1a2bc2431d-c000.snappy.parquet","partitionValues":{},"size":1008,"modificationTime":1674064770372,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":33},\"maxValues\":{\"value\":1995},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064769860012","MIN_INSERTION_TIME":"1674064769860012","MAX_INSERTION_TIME":"1674064769860012","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"part-00013-c6b05dd2-0143-4e9f-a231-1a2d08a83a0e-c000.snappy.parquet","partitionValues":{},"size":1008,"modificationTime":1674064770372,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":20},\"maxValues\":{\"value\":1974},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064769860013","MIN_INSERTION_TIME":"1674064769860013","MAX_INSERTION_TIME":"1674064769860013","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"part-00014-41a4f51e-62cd-41f5-bb03-afba1e70ea29-c000.snappy.parquet","partitionValues":{},"size":1007,"modificationTime":1674064770427,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":3},\"maxValues\":{\"value\":1996},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064769860014","MIN_INSERTION_TIME":"1674064769860014","MAX_INSERTION_TIME":"1674064769860014","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"part-00015-f2f141bb-fa8f-4553-a5db-d1b8d682153b-c000.snappy.parquet","partitionValues":{},"size":1008,"modificationTime":1674064770427,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":0},\"maxValues\":{\"value\":1997},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064769860015","MIN_INSERTION_TIME":"1674064769860015","MAX_INSERTION_TIME":"1674064769860015","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"part-00016-d8f58ffc-8bff-4e12-b709-e628f9bf2553-c000.snappy.parquet","partitionValues":{},"size":1008,"modificationTime":1674064770477,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":2},\"maxValues\":{\"value\":1986},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064769860016","MIN_INSERTION_TIME":"1674064769860016","MAX_INSERTION_TIME":"1674064769860016","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"part-00017-45bac3c9-7eb8-42cb-bb51-fc5b4dd0be10-c000.snappy.parquet","partitionValues":{},"size":1008,"modificationTime":1674064770476,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":22},\"maxValues\":{\"value\":1998},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064769860017","MIN_INSERTION_TIME":"1674064769860017","MAX_INSERTION_TIME":"1674064769860017","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"part-00018-9d74a51b-b800-4e4d-a258-738e585a78a5-c000.snappy.parquet","partitionValues":{},"size":1008,"modificationTime":1674064770529,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":6},\"maxValues\":{\"value\":1983},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064769860018","MIN_INSERTION_TIME":"1674064769860018","MAX_INSERTION_TIME":"1674064769860018","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"part-00019-a9bb3ce8-afba-47ec-8451-13edcd855b15-c000.snappy.parquet","partitionValues":{},"size":1007,"modificationTime":1674064770528,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":36},\"maxValues\":{\"value\":1969},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064769860019","MIN_INSERTION_TIME":"1674064769860019","MAX_INSERTION_TIME":"1674064769860019","OPTIMIZE_TARGET_SIZE":"268435456"}}} ================================================ FILE: spark/src/test/resources/delta/table-with-dv-large/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1674064789962,"operation":"DELETE","operationParameters":{"predicate":"[\"(spark_catalog.delta.`/private/var/folders/g3/hcd28y8s71s0yh7whh443wz00000gp/T/spark-f3dd4a29-dc57-42eb-b752-84179135f5b8`.value IN (0, 180, 300, 700, 1800))\"]"},"readVersion":0,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numCopiedRows":"0","numDeletionVectorsAdded":"5","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"0","executionTimeMs":"12828","numDeletionVectorsUpdated":"0","numDeletedRows":"5","scanTimeMs":"12323","numAddedFiles":"0","rewriteTimeMs":"487"},"engineInfo":"","txnId":"5327cd46-c25b-4127-88fd-5b3c2402691b"}} {"remove":{"path":"part-00001-5dbf0ba2-220a-4770-8e26-18a77cf875f0-c000.snappy.parquet","deletionTimestamp":1674064789957,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1008,"tags":{"INSERTION_TIME":"1674064769860001","MIN_INSERTION_TIME":"1674064769860001","MAX_INSERTION_TIME":"1674064769860001","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"remove":{"path":"part-00003-0e842060-9e04-4896-ba21-029309ab8736-c000.snappy.parquet","deletionTimestamp":1674064789957,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1008,"tags":{"INSERTION_TIME":"1674064769860003","MIN_INSERTION_TIME":"1674064769860003","MAX_INSERTION_TIME":"1674064769860003","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"remove":{"path":"part-00012-9b83c213-31ff-4b2c-a5d9-be1a2bc2431d-c000.snappy.parquet","deletionTimestamp":1674064789957,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1008,"tags":{"INSERTION_TIME":"1674064769860012","MIN_INSERTION_TIME":"1674064769860012","MAX_INSERTION_TIME":"1674064769860012","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"remove":{"path":"part-00015-f2f141bb-fa8f-4553-a5db-d1b8d682153b-c000.snappy.parquet","deletionTimestamp":1674064789957,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1008,"tags":{"INSERTION_TIME":"1674064769860015","MIN_INSERTION_TIME":"1674064769860015","MAX_INSERTION_TIME":"1674064769860015","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"remove":{"path":"part-00019-a9bb3ce8-afba-47ec-8451-13edcd855b15-c000.snappy.parquet","deletionTimestamp":1674064789957,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1007,"tags":{"INSERTION_TIME":"1674064769860019","MIN_INSERTION_TIME":"1674064769860019","MAX_INSERTION_TIME":"1674064769860019","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"part-00001-5dbf0ba2-220a-4770-8e26-18a77cf875f0-c000.snappy.parquet","partitionValues":{},"size":1008,"modificationTime":1674064769860,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":18},\"maxValues\":{\"value\":1988},\"nullCount\":{\"value\":0},\"tightBounds\":false}","tags":{"INSERTION_TIME":"1674064769860001","MIN_INSERTION_TIME":"1674064769860001","MAX_INSERTION_TIME":"1674064769860001","OPTIMIZE_TARGET_SIZE":"268435456"},"deletionVector":{"storageType":"u","pathOrInlineDv":"m9JzgVlI!?Oy<+3x+y^b","offset":85,"sizeInBytes":34,"cardinality":1}}} {"add":{"path":"part-00003-0e842060-9e04-4896-ba21-029309ab8736-c000.snappy.parquet","partitionValues":{},"size":1008,"modificationTime":1674064770019,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":5},\"maxValues\":{\"value\":1982},\"nullCount\":{\"value\":0},\"tightBounds\":false}","tags":{"INSERTION_TIME":"1674064769860003","MIN_INSERTION_TIME":"1674064769860003","MAX_INSERTION_TIME":"1674064769860003","OPTIMIZE_TARGET_SIZE":"268435456"},"deletionVector":{"storageType":"u","pathOrInlineDv":"m9JzgVlI!?Oy<+3x+y^b","offset":169,"sizeInBytes":34,"cardinality":1}}} {"add":{"path":"part-00012-9b83c213-31ff-4b2c-a5d9-be1a2bc2431d-c000.snappy.parquet","partitionValues":{},"size":1008,"modificationTime":1674064770372,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":33},\"maxValues\":{\"value\":1995},\"nullCount\":{\"value\":0},\"tightBounds\":false}","tags":{"INSERTION_TIME":"1674064769860012","MIN_INSERTION_TIME":"1674064769860012","MAX_INSERTION_TIME":"1674064769860012","OPTIMIZE_TARGET_SIZE":"268435456"},"deletionVector":{"storageType":"u","pathOrInlineDv":"m9JzgVlI!?Oy<+3x+y^b","offset":1,"sizeInBytes":34,"cardinality":1}}} {"add":{"path":"part-00015-f2f141bb-fa8f-4553-a5db-d1b8d682153b-c000.snappy.parquet","partitionValues":{},"size":1008,"modificationTime":1674064770427,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":0},\"maxValues\":{\"value\":1997},\"nullCount\":{\"value\":0},\"tightBounds\":false}","tags":{"INSERTION_TIME":"1674064769860015","MIN_INSERTION_TIME":"1674064769860015","MAX_INSERTION_TIME":"1674064769860015","OPTIMIZE_TARGET_SIZE":"268435456"},"deletionVector":{"storageType":"u","pathOrInlineDv":"m9JzgVlI!?Oy<+3x+y^b","offset":43,"sizeInBytes":34,"cardinality":1}}} {"add":{"path":"part-00019-a9bb3ce8-afba-47ec-8451-13edcd855b15-c000.snappy.parquet","partitionValues":{},"size":1007,"modificationTime":1674064770528,"dataChange":true,"stats":"{\"numRecords\":100,\"minValues\":{\"value\":36},\"maxValues\":{\"value\":1969},\"nullCount\":{\"value\":0},\"tightBounds\":false}","tags":{"INSERTION_TIME":"1674064769860019","MIN_INSERTION_TIME":"1674064769860019","MAX_INSERTION_TIME":"1674064769860019","OPTIMIZE_TARGET_SIZE":"268435456"},"deletionVector":{"storageType":"u","pathOrInlineDv":"m9JzgVlI!?Oy<+3x+y^b","offset":127,"sizeInBytes":34,"cardinality":1}}} ================================================ FILE: spark/src/test/resources/delta/table-with-dv-large/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1674064791599,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"2","numOutputBytes":"600"},"engineInfo":"","txnId":"fb0a7015-0096-4d74-821b-3507163c17fa"}} {"add":{"path":"part-00000-51219d56-88a7-41cc-be5d-eada75aceb4f-c000.snappy.parquet","partitionValues":{},"size":600,"modificationTime":1674064791593,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"value\":300},\"maxValues\":{\"value\":700},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064791593000","MIN_INSERTION_TIME":"1674064791593000","MAX_INSERTION_TIME":"1674064791593000","OPTIMIZE_TARGET_SIZE":"268435456"}}} ================================================ FILE: spark/src/test/resources/delta/table-with-dv-large/_delta_log/00000000000000000003.json ================================================ {"commitInfo":{"timestamp":1674064797400,"operation":"DELETE","operationParameters":{"predicate":"[\"(spark_catalog.delta.`/private/var/folders/g3/hcd28y8s71s0yh7whh443wz00000gp/T/spark-f3dd4a29-dc57-42eb-b752-84179135f5b8`.value IN (300, 250, 350, 900, 1353, 1567, 1800))\"]"},"readVersion":2,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numCopiedRows":"0","numDeletionVectorsAdded":"3","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"0","executionTimeMs":"4726","numDeletionVectorsUpdated":"3","numDeletedRows":"6","scanTimeMs":"4057","numAddedFiles":"0","rewriteTimeMs":"667"},"engineInfo":"","txnId":"d50de74c-f8c8-4e68-b120-267504045e9d"}} {"remove":{"path":"part-00000-51219d56-88a7-41cc-be5d-eada75aceb4f-c000.snappy.parquet","deletionTimestamp":1674064797399,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":600,"tags":{"INSERTION_TIME":"1674064791593000","MIN_INSERTION_TIME":"1674064791593000","MAX_INSERTION_TIME":"1674064791593000","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"remove":{"path":"part-00001-5dbf0ba2-220a-4770-8e26-18a77cf875f0-c000.snappy.parquet","deletionTimestamp":1674064797399,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1008,"tags":{"INSERTION_TIME":"1674064769860001","MIN_INSERTION_TIME":"1674064769860001","MAX_INSERTION_TIME":"1674064769860001","OPTIMIZE_TARGET_SIZE":"268435456"},"deletionVector":{"storageType":"u","pathOrInlineDv":"m9JzgVlI!?Oy<+3x+y^b","offset":85,"sizeInBytes":34,"cardinality":1}}} {"remove":{"path":"part-00012-9b83c213-31ff-4b2c-a5d9-be1a2bc2431d-c000.snappy.parquet","deletionTimestamp":1674064797399,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1008,"tags":{"INSERTION_TIME":"1674064769860012","MIN_INSERTION_TIME":"1674064769860012","MAX_INSERTION_TIME":"1674064769860012","OPTIMIZE_TARGET_SIZE":"268435456"},"deletionVector":{"storageType":"u","pathOrInlineDv":"m9JzgVlI!?Oy<+3x+y^b","offset":1,"sizeInBytes":34,"cardinality":1}}} {"remove":{"path":"part-00014-41a4f51e-62cd-41f5-bb03-afba1e70ea29-c000.snappy.parquet","deletionTimestamp":1674064797399,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1007,"tags":{"INSERTION_TIME":"1674064769860014","MIN_INSERTION_TIME":"1674064769860014","MAX_INSERTION_TIME":"1674064769860014","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"remove":{"path":"part-00018-9d74a51b-b800-4e4d-a258-738e585a78a5-c000.snappy.parquet","deletionTimestamp":1674064797399,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1008,"tags":{"INSERTION_TIME":"1674064769860018","MIN_INSERTION_TIME":"1674064769860018","MAX_INSERTION_TIME":"1674064769860018","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"remove":{"path":"part-00019-a9bb3ce8-afba-47ec-8451-13edcd855b15-c000.snappy.parquet","deletionTimestamp":1674064797399,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1007,"tags":{"INSERTION_TIME":"1674064769860019","MIN_INSERTION_TIME":"1674064769860019","MAX_INSERTION_TIME":"1674064769860019","OPTIMIZE_TARGET_SIZE":"268435456"},"deletionVector":{"storageType":"u","pathOrInlineDv":"m9JzgVlI!?Oy<+3x+y^b","offset":127,"sizeInBytes":34,"cardinality":1}}} {"add":{"path":"part-00000-51219d56-88a7-41cc-be5d-eada75aceb4f-c000.snappy.parquet","partitionValues":{},"size":600,"modificationTime":1674064791593,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"value\":300},\"maxValues\":{\"value\":700},\"nullCount\":{\"value\":0},\"tightBounds\":false}","tags":{"INSERTION_TIME":"1674064791593000","MIN_INSERTION_TIME":"1674064791593000","MAX_INSERTION_TIME":"1674064791593000","OPTIMIZE_TARGET_SIZE":"268435456"},"deletionVector":{"storageType":"u","pathOrInlineDv":"UGM+pBY.mtVeP","txnId":"4016704a-babb-44a8-ae8b-c53303465742"}} {"add":{"path":"part-00000-7c52eadd-8da7-4782-a5d5-621cd92cab11-c000.snappy.parquet","partitionValues":{},"size":600,"modificationTime":1674064798704,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"value\":900},\"maxValues\":{\"value\":1567},\"nullCount\":{\"value\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1674064798704000","MIN_INSERTION_TIME":"1674064798704000","MAX_INSERTION_TIME":"1674064798704000","OPTIMIZE_TARGET_SIZE":"268435456"}}} ================================================ FILE: spark/src/test/resources/delta/table-with-dv-small/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1673461409137,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"10","numOutputBytes":"818"},"engineInfo":"","txnId":"d54c00f5-9500-4ed5-b1b5-9f463861f4d3"}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["deletionVectors","columnMapping"],"writerFeatures":["deletionVectors","columnMapping"]}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"value\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":1,\"delta.columnMapping.physicalName\":\"col-4f064e48-f371-433a-b851-9e73c78fa9fc\"}}]}","partitionColumns":[],"configuration":{"delta.columnMapping.mode":"name","delta.enableDeletionVectors":"true","delta.columnMapping.maxColumnId":"1"},"createdTime":1673461406485}} {"add":{"path":"r4/part-00000-5521fc5e-6e49-4437-8b2d-ce6a1a94a34a-c000.snappy.parquet","partitionValues":{},"size":818,"modificationTime":1673461408778,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"col-4f064e48-f371-433a-b851-9e73c78fa9fc\":0},\"maxValues\":{\"col-4f064e48-f371-433a-b851-9e73c78fa9fc\":9},\"nullCount\":{\"col-4f064e48-f371-433a-b851-9e73c78fa9fc\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1673461408778000","MIN_INSERTION_TIME":"1673461408778000","MAX_INSERTION_TIME":"1673461408778000","OPTIMIZE_TARGET_SIZE":"268435456"}}} ================================================ FILE: spark/src/test/resources/delta/table-with-dv-small/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1673461427387,"operation":"DELETE","operationParameters":{"predicate":"[\"(spark_catalog.delta.`/private/var/folders/g3/hcd28y8s71s0yh7whh443wz00000gp/T/spark-cb573b98-e75d-460f-9769-efd9e9bfeffc`.value IN (0, 9))\"]"},"readVersion":0,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numCopiedRows":"0","numDeletionVectorsAdded":"1","numDeletionVectorsRemoved":"0","numAddedChangeFiles":"0","executionTimeMs":"11114","numDeletionVectorsUpdated":"0","numDeletedRows":"2","scanTimeMs":"10589","numAddedFiles":"0","rewriteTimeMs":"508"},"engineInfo":"","txnId":"3943baa4-30a0-44a4-a4f4-e5e92d2ab08b"}} {"remove":{"path":"r4/part-00000-5521fc5e-6e49-4437-8b2d-ce6a1a94a34a-c000.snappy.parquet","deletionTimestamp":1673461427383,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":818,"tags":{"INSERTION_TIME":"1673461408778000","MIN_INSERTION_TIME":"1673461408778000","MAX_INSERTION_TIME":"1673461408778000","OPTIMIZE_TARGET_SIZE":"268435456"}}} {"add":{"path":"r4/part-00000-5521fc5e-6e49-4437-8b2d-ce6a1a94a34a-c000.snappy.parquet","partitionValues":{},"size":818,"modificationTime":1673461408778,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"col-4f064e48-f371-433a-b851-9e73c78fa9fc\":0},\"maxValues\":{\"col-4f064e48-f371-433a-b851-9e73c78fa9fc\":9},\"nullCount\":{\"col-4f064e48-f371-433a-b851-9e73c78fa9fc\":0},\"tightBounds\":false}","tags":{"INSERTION_TIME":"1673461408778000","MIN_INSERTION_TIME":"1673461408778000","MAX_INSERTION_TIME":"1673461408778000","OPTIMIZE_TARGET_SIZE":"268435456"},"deletionVector":{"storageType":"u","pathOrInlineDv":"WYbkwCTB$gH)J7t?$/sK","offset":1,"sizeInBytes":36,"cardinality":2}}} ================================================ FILE: spark/src/test/resources/delta/table-with-dv-special-char/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1708950820866,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"10","numOutputBytes":"539"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0-SNAPSHOT","txnId":"1586fe39-668f-48c6-ad0b-af4a10b38f22"}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableDeletionVectors":"true"},"createdTime":1708950819168}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["deletionVectors"],"writerFeatures":["deletionVectors"]}} {"add":{"path":"part-00000-8d24f407-08d3-49ab-9d1c-f7f6c129e882-c000.snappy.parquet","partitionValues":{},"size":539,"modificationTime":1708950820783,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"p","pathOrInlineDv":"file:{{FOLDER_WITH_SPECIAL_CHAR}}/test%25dv%25prefix-deletion_vector_67bc892e-2979-4760-a78c-856aba806564.bin","offset":1,"sizeInBytes":42,"cardinality":5}}} ================================================ FILE: spark/src/test/resources/delta/transaction_log_schema_evolvability/_delta_log/00000000000000000000.crc ================================================ {"tableSizeBytes":1594,"numFiles":2,"numMetadata":1,"numProtocol":1,"numTransactions":0} ================================================ FILE: spark/src/test/resources/delta/transaction_log_schema_evolvability/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1623255695348,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputBytes":"1594","numOutputRows":"9"}}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"key\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"value\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1623255692280}} {"add":{"path":"part-00000-dfb1dd9a-0fe2-420e-81d5-a84004aebcee-c000.snappy.parquet","partitionValues":{},"size":793,"modificationTime":1623255695000,"dataChange":true,"stats":"{\"numRecords\":4,\"minValues\":{\"key\":1,\"value\":1},\"maxValues\":{\"key\":4,\"value\":4},\"nullCount\":{\"key\":0,\"value\":0}}"}} {"add":{"path":"part-00001-d5da9c60-a615-4065-a3cb-4796d86fc797-c000.snappy.parquet","partitionValues":{},"size":801,"modificationTime":1623255695000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"key\":5,\"value\":5},\"maxValues\":{\"key\":9,\"value\":9},\"nullCount\":{\"key\":0,\"value\":0}}"}} ================================================ FILE: spark/src/test/resources/delta/transaction_log_schema_evolvability/_delta_log/00000000000000000001.crc ================================================ {"tableSizeBytes":1594,"numFiles":2,"numMetadata":1,"numProtocol":1,"numTransactions":0} ================================================ FILE: spark/src/test/resources/delta/transaction_log_schema_evolvability/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1623255703194,"operation":"SET TBLPROPERTIES","operationParameters":{"properties":"{\"delta.checkpoint.writeStatsAsStruct\":\"true\",\"delta.checkpoint.writeStatsAsJson\":\"false\"}"},"readVersion":0,"isolationLevel":"SnapshotIsolation","isBlindAppend":true,"operationMetrics":{}}} {"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"key\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"value\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpoint.writeStatsAsStruct":"true","delta.checkpoint.writeStatsAsJson":"false"},"createdTime":1623255692280}} ================================================ FILE: spark/src/test/resources/delta/transaction_log_schema_evolvability/_delta_log/00000000000000000002.crc ================================================ {"tableSizeBytes":1594,"numFiles":2,"numMetadata":1,"numProtocol":1,"numTransactions":0} ================================================ FILE: spark/src/test/resources/delta/transaction_log_schema_evolvability/_delta_log/00000000000000000002.json ================================================ {"commitInfo":{"timestamp":1623255706138,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":1,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"2","numOutputBytes":"1594","numOutputRows":"9"}}} {"add":{"path":"part-00000-f654b1f4-e1ea-40e5-a8cd-452f7c3359d8-c000.snappy.parquet","partitionValues":{},"size":793,"modificationTime":1623255705000,"dataChange":true,"stats":"{\"numRecords\":4,\"minValues\":{\"key\":1,\"value\":1},\"maxValues\":{\"key\":4,\"value\":4},\"nullCount\":{\"key\":0,\"value\":0}}"}} {"add":{"path":"part-00001-bfb08fc5-c967-40e4-a646-c8178d8b5e21-c000.snappy.parquet","partitionValues":{},"size":801,"modificationTime":1623255705000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"key\":5,\"value\":5},\"maxValues\":{\"key\":9,\"value\":9},\"nullCount\":{\"key\":0,\"value\":0}}"}} {"remove":{"path":"part-00000-dfb1dd9a-0fe2-420e-81d5-a84004aebcee-c000.snappy.parquet","deletionTimestamp":1623255706137,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":793}} {"remove":{"path":"part-00001-d5da9c60-a615-4065-a3cb-4796d86fc797-c000.snappy.parquet","deletionTimestamp":1623255706138,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":801}} ================================================ FILE: spark/src/test/resources/delta/transaction_log_schema_evolvability/_delta_log/00000000000000000003.crc ================================================ {"tableSizeBytes":3188,"numFiles":4,"numMetadata":1,"numProtocol":1,"numTransactions":0} ================================================ FILE: spark/src/test/resources/delta/transaction_log_schema_evolvability/_delta_log/00000000000000000003.json ================================================ {"commitInfo":{"timestamp":1623255724166,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":2,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputBytes":"1594","numOutputRows":"9"}}} {"add":{"path":"part-00000-9f483b95-3ea3-44f0-b54d-73199574be15-c000.snappy.parquet","partitionValues":{},"size":793,"modificationTime":1623255724000,"dataChange":true,"stats":"{\"numRecords\":4,\"minValues\":{\"key\":1,\"value\":1},\"maxValues\":{\"key\":4,\"value\":4},\"nullCount\":{\"key\":0,\"value\":0}}"}} {"add":{"path":"part-00001-d1030238-b55d-48f8-a4d6-89ef12e9d501-c000.snappy.parquet","partitionValues":{},"size":801,"modificationTime":1623255724000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"key\":5,\"value\":5},\"maxValues\":{\"key\":9,\"value\":9},\"nullCount\":{\"key\":0,\"value\":0}}"}} ================================================ FILE: spark/src/test/resources/delta/transaction_log_schema_evolvability/_delta_log/00000000000000000004.json ================================================ {"remove":{"path":"part-00000-9f483b95-3ea3-44f0-b54d-73199574be15-c000.snappy.parquet","deletionTimestamp":1623255727201,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":793}} {"remove":{"path":"part-00001-d1030238-b55d-48f8-a4d6-89ef12e9d501-c000.snappy.parquet","deletionTimestamp":1623255727201,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":801}} {"some_new_action":{"a":1}} ================================================ FILE: spark/src/test/resources/delta/transaction_log_schema_evolvability/_delta_log/00000000000000000005.json ================================================ {"commitInfo":{"timestamp":1623255724166,"operationMetrics":{"numFiles":"2","numOutputBytes":"1594","numOutputRows":"9"},"isolationLevel":"WriteSerializable","operationParameters":{"mode":"Append","partitionBy":"[]"},"operation":"WRITE","isBlindAppend":true,"readVersion":2},"some_new_action_alongside_add_action":["a","1"]} {"add":{"stats":"{\"numRecords\":4,\"minValues\":{\"key\":1,\"value\":1},\"maxValues\":{\"key\":4,\"value\":4},\"nullCount\":{\"key\":0,\"value\":0}}","path":"part-00000-9f483b95-3ea3-44f0-b54d-73199574be15-c000.snappy.parquet","size":793,"modificationTime":1623255724000,"dataChange":true,"some_new_column_in_add_action":1,"partitionValues":{}},"some_new_action_alongside_add_action":["a","1"]} {"add":{"stats":"{\"numRecords\":5,\"minValues\":{\"key\":5,\"value\":5},\"maxValues\":{\"key\":9,\"value\":9},\"nullCount\":{\"key\":0,\"value\":0}}","path":"part-00001-d1030238-b55d-48f8-a4d6-89ef12e9d501-c000.snappy.parquet","size":801,"modificationTime":1623255724000,"dataChange":true,"some_new_column_in_add_action":1,"partitionValues":{}},"some_new_action_alongside_add_action":["a","1"]} ================================================ FILE: spark/src/test/resources/delta/transaction_log_schema_evolvability/_delta_log/_last_checkpoint ================================================ {"version":2,"size":6} ================================================ FILE: spark/src/test/resources/delta/variant-stats-no-checkpoint/_delta_log/00000000000000000000.crc ================================================ {"txnId":"261694ff-57ea-40af-919a-eb1b606df973","tableSizeBytes":0,"numFiles":0,"numMetadata":1,"numProtocol":1,"setTransactions":[],"domainMetadata":[],"metadata":{"id":"5053928a-f69d-4fc1-906e-e427c8cfbaef","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"i\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}},{\"name\":\"nv\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpoint.writeStatsAsJson":"true","delta.enableVariantShredding":"true"},"createdTime":1769217561381},"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["variantType","variantShredding-preview"],"writerFeatures":["variantType","variantShredding-preview","appendOnly","invariants"]},"histogramOpt":{"sortedBinBoundaries":[0,8192,16384,32768,65536,131072,262144,524288,1048576,2097152,4194304,8388608,12582912,16777216,20971520,25165824,29360128,33554432,37748736,41943040,50331648,58720256,67108864,75497472,83886080,92274688,100663296,109051904,117440512,125829120,130023424,134217728,138412032,142606336,146800640,150994944,167772160,184549376,201326592,218103808,234881024,251658240,268435456,285212672,301989888,318767104,335544320,352321536,369098752,385875968,402653184,419430400,436207616,452984832,469762048,486539264,503316480,520093696,536870912,553648128,570425344,587202560,603979776,671088640,738197504,805306368,872415232,939524096,1006632960,1073741824,1140850688,1207959552,1275068416,1342177280,1409286144,1476395008,1610612736,1744830464,1879048192,2013265920,2147483648,2415919104,2684354560,2952790016,3221225472,3489660928,3758096384,4026531840,4294967296,8589934592,17179869184,34359738368,68719476736,137438953472,274877906944],"fileCounts":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"totalBytes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"allFiles":[]} ================================================ FILE: spark/src/test/resources/delta/variant-stats-no-checkpoint/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1769217561450,"operation":"CREATE OR REPLACE TABLE","operationParameters":{"partitionBy":"[]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{\"delta.checkpoint.writeStatsAsJson\":\"true\",\"delta.enableVariantShredding\":\"true\"}","statsOnLoad":false},"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/","txnId":"261694ff-57ea-40af-919a-eb1b606df973"}} {"metaData":{"id":"5053928a-f69d-4fc1-906e-e427c8cfbaef","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"i\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}},{\"name\":\"nv\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpoint.writeStatsAsJson":"true","delta.enableVariantShredding":"true"},"createdTime":1769217561381}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["variantType","variantShredding-preview"],"writerFeatures":["variantType","variantShredding-preview","appendOnly","invariants"]}} ================================================ FILE: spark/src/test/resources/delta/variant-stats-no-checkpoint/_delta_log/00000000000000000001.crc ================================================ {"txnId":"d49c9e08-79d2-4a65-ae2d-fbc5869eb8b4","tableSizeBytes":3390,"numFiles":1,"numMetadata":1,"numProtocol":1,"setTransactions":[],"domainMetadata":[],"metadata":{"id":"5053928a-f69d-4fc1-906e-e427c8cfbaef","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"i\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}},{\"name\":\"nv\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpoint.writeStatsAsJson":"true","delta.enableVariantShredding":"true"},"createdTime":1769217561381},"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["variantType","variantShredding-preview"],"writerFeatures":["variantType","variantShredding-preview","appendOnly","invariants"]},"histogramOpt":{"sortedBinBoundaries":[0,8192,16384,32768,65536,131072,262144,524288,1048576,2097152,4194304,8388608,12582912,16777216,20971520,25165824,29360128,33554432,37748736,41943040,50331648,58720256,67108864,75497472,83886080,92274688,100663296,109051904,117440512,125829120,130023424,134217728,138412032,142606336,146800640,150994944,167772160,184549376,201326592,218103808,234881024,251658240,268435456,285212672,301989888,318767104,335544320,352321536,369098752,385875968,402653184,419430400,436207616,452984832,469762048,486539264,503316480,520093696,536870912,553648128,570425344,587202560,603979776,671088640,738197504,805306368,872415232,939524096,1006632960,1073741824,1140850688,1207959552,1275068416,1342177280,1409286144,1476395008,1610612736,1744830464,1879048192,2013265920,2147483648,2415919104,2684354560,2952790016,3221225472,3489660928,3758096384,4026531840,4294967296,8589934592,17179869184,34359738368,68719476736,137438953472,274877906944],"fileCounts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"totalBytes":[3390,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"allFiles":[{"path":"part-00000-20135f43-a68e-4348-9a46-e6eeed704c0e-c000.snappy.parquet","partitionValues":{},"size":3390,"modificationTime":1769217565612,"dataChange":false,"stats":"{\"numRecords\":10,\"minValues\":{\"i\":100,\"v\":\"0rJlc5f$EczEEx[cTw04cUW9st(>6d00icd1U*?]0000000000\",\"nv\":{\"v\":\"0rJlc5f$EczEEx[cTw04cUW9st(>6d00rif2@adK3ig5a00000\"}},\"maxValues\":{\"i\":109,\"v\":\"0rJlc5f$EczEEx[cTw04cUW9st(>6d00icd1VWza0000000000\",\"nv\":{\"v\":\"0rJlc5f$EczEEx[cTw04cUW9st(>6d00rif2@jgK6951j00000\"}},\"nullCount\":{\"i\":0,\"v\":0,\"nv\":{\"v\":0}}}","tags":{"MAX_INSERTION_TIME":"1769217565612000","INSERTION_TIME":"1769217565612000","SHREDDING_STATE":"1","MIN_INSERTION_TIME":"1769217565612000","OPTIMIZE_TARGET_SIZE":"268435456"}}]} ================================================ FILE: spark/src/test/resources/delta/variant-stats-no-checkpoint/_delta_log/00000000000000000001.json ================================================ {"commitInfo":{"timestamp":1769217565748,"operation":"WRITE","operationParameters":{"mode":"Append","statsOnLoad":false,"partitionBy":"[]"},"readVersion":0,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"10","numOutputBytes":"3390"},"tags":{"noRowsCopied":"true","restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/","txnId":"d49c9e08-79d2-4a65-ae2d-fbc5869eb8b4"}} {"add":{"path":"part-00000-20135f43-a68e-4348-9a46-e6eeed704c0e-c000.snappy.parquet","partitionValues":{},"size":3390,"modificationTime":1769217565612,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"i\":100,\"v\":\"0rJlc5f$EczEEx[cTw04cUW9st(>6d00icd1U*?]0000000000\",\"nv\":{\"v\":\"0rJlc5f$EczEEx[cTw04cUW9st(>6d00rif2@adK3ig5a00000\"}},\"maxValues\":{\"i\":109,\"v\":\"0rJlc5f$EczEEx[cTw04cUW9st(>6d00icd1VWza0000000000\",\"nv\":{\"v\":\"0rJlc5f$EczEEx[cTw04cUW9st(>6d00rif2@jgK6951j00000\"}},\"nullCount\":{\"i\":0,\"v\":0,\"nv\":{\"v\":0}}}","tags":{"MAX_INSERTION_TIME":"1769217565612000","INSERTION_TIME":"1769217565612000","SHREDDING_STATE":"1","MIN_INSERTION_TIME":"1769217565612000","OPTIMIZE_TARGET_SIZE":"268435456"}}} ================================================ FILE: spark/src/test/resources/delta/variant-stats-state-reconstruction/_delta_log/00000000000000000000.crc ================================================ {"txnId":"56b6e637-4d75-4935-b186-205ed3ca5aff","tableSizeBytes":863,"numFiles":1,"numMetadata":1,"numProtocol":1,"setTransactions":[],"domainMetadata":[],"metadata":{"id":"9eebd0cb-ecd2-48fb-8b02-352d81209730","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableVariantShredding":"true","delta.checkpointInterval":"10000"},"createdTime":1769474649842},"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["variantType","variantShredding-preview"],"writerFeatures":["variantType","variantShredding-preview","appendOnly","invariants"]},"histogramOpt":{"sortedBinBoundaries":[0,8192,16384,32768,65536,131072,262144,524288,1048576,2097152,4194304,8388608,12582912,16777216,20971520,25165824,29360128,33554432,37748736,41943040,50331648,58720256,67108864,75497472,83886080,92274688,100663296,109051904,117440512,125829120,130023424,134217728,138412032,142606336,146800640,150994944,167772160,184549376,201326592,218103808,234881024,251658240,268435456,285212672,301989888,318767104,335544320,352321536,369098752,385875968,402653184,419430400,436207616,452984832,469762048,486539264,503316480,520093696,536870912,553648128,570425344,587202560,603979776,671088640,738197504,805306368,872415232,939524096,1006632960,1073741824,1140850688,1207959552,1275068416,1342177280,1409286144,1476395008,1610612736,1744830464,1879048192,2013265920,2147483648,2415919104,2684354560,2952790016,3221225472,3489660928,3758096384,4026531840,4294967296,8589934592,17179869184,34359738368,68719476736,137438953472,274877906944],"fileCounts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"totalBytes":[863,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"allFiles":[{"path":"part-00031-b7a56bf1-1672-47dc-b8e2-255d62f630ee-c000.snappy.parquet","partitionValues":{},"size":863,"modificationTime":1769474653660,"dataChange":false,"stats":"{\"numRecords\":1,\"minValues\":{\"v\":\"0rAf3bMW#D00%Fx0000000000\"},\"maxValues\":{\"v\":\"0rAf3bMW#D00%Fx0000000000\"},\"nullCount\":{\"v\":0}}","tags":{"MAX_INSERTION_TIME":"1769474653660000","INSERTION_TIME":"1769474653660000","SHREDDING_STATE":"0","MIN_INSERTION_TIME":"1769474653660000","OPTIMIZE_TARGET_SIZE":"268435456"}}]} ================================================ FILE: spark/src/test/resources/delta/variant-stats-state-reconstruction/_delta_log/00000000000000000000.json ================================================ {"commitInfo":{"timestamp":1769474654138,"operation":"CREATE TABLE AS SELECT","operationParameters":{"partitionBy":"[]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{\"delta.enableVariantShredding\":\"true\",\"delta.checkpointInterval\":\"10000\"}","statsOnLoad":false},"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"863"},"tags":{"noRowsCopied":"true","restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/","txnId":"56b6e637-4d75-4935-b186-205ed3ca5aff"}} {"metaData":{"id":"9eebd0cb-ecd2-48fb-8b02-352d81209730","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableVariantShredding":"true","delta.checkpointInterval":"10000"},"createdTime":1769474649842}} {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["variantType","variantShredding-preview"],"writerFeatures":["variantType","variantShredding-preview","appendOnly","invariants"]}} {"add":{"path":"part-00031-b7a56bf1-1672-47dc-b8e2-255d62f630ee-c000.snappy.parquet","partitionValues":{},"size":863,"modificationTime":1769474653660,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"v\":\"0rAf3bMW#D00%Fx0000000000\"},\"maxValues\":{\"v\":\"0rAf3bMW#D00%Fx0000000000\"},\"nullCount\":{\"v\":0}}","tags":{"MAX_INSERTION_TIME":"1769474653660000","INSERTION_TIME":"1769474653660000","SHREDDING_STATE":"0","MIN_INSERTION_TIME":"1769474653660000","OPTIMIZE_TARGET_SIZE":"268435456"}}} ================================================ FILE: spark/src/test/resources/hms/README.md ================================================ The file `hive-schema-3.1.0.derby.sql` is copied from the Hive official repository. Hive MetaStore uses this file to create database schema used by the metastore with Apache Derby. We use it for the same purpose as the EmbeddedHMS is back by Apache Derby. The original file can be found [here](https://github.com/apache/hive/blob/master/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-3.1.0.derby.sql). In the embedded HMS, we first create a derby instance, then load this script into derby to initialize the schema. See `org.apache.spark.sql.delta.uniform.ehms.EmbeddedHMS` for more details. ================================================ FILE: spark/src/test/resources/hms/hive-schema-3.1.0.derby.sql ================================================ -- Timestamp: 2011-09-22 15:32:02.024 -- Source database is: /home/carl/Work/repos/hive1/metastore/scripts/upgrade/derby/mdb -- Connection URL is: jdbc:derby:/home/carl/Work/repos/hive1/metastore/scripts/upgrade/derby/mdb -- Specified schema is: APP -- appendLogs: false -- ---------------------------------------------- -- DDL Statements for functions -- ---------------------------------------------- CREATE FUNCTION "APP"."NUCLEUS_ASCII" (C CHAR(1)) RETURNS INTEGER LANGUAGE JAVA PARAMETER STYLE JAVA READS SQL DATA CALLED ON NULL INPUT EXTERNAL NAME 'org.datanucleus.store.rdbms.adapter.DerbySQLFunction.ascii' ; CREATE FUNCTION "APP"."NUCLEUS_MATCHES" (TEXT VARCHAR(8000),PATTERN VARCHAR(8000)) RETURNS INTEGER LANGUAGE JAVA PARAMETER STYLE JAVA READS SQL DATA CALLED ON NULL INPUT EXTERNAL NAME 'org.datanucleus.store.rdbms.adapter.DerbySQLFunction.matches' ; -- ---------------------------------------------- -- DDL Statements for tables -- ---------------------------------------------- CREATE TABLE "APP"."DBS" ( "DB_ID" BIGINT NOT NULL, "DESC" VARCHAR(4000), "DB_LOCATION_URI" VARCHAR(4000) NOT NULL, "NAME" VARCHAR(128), "OWNER_NAME" VARCHAR(128), "OWNER_TYPE" VARCHAR(10), "CTLG_NAME" VARCHAR(256) NOT NULL DEFAULT 'hive' ); CREATE TABLE "APP"."TBL_PRIVS" ("TBL_GRANT_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "TBL_PRIV" VARCHAR(128), "TBL_ID" BIGINT, "AUTHORIZER" VARCHAR(128)); CREATE TABLE "APP"."DATABASE_PARAMS" ("DB_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(180) NOT NULL, "PARAM_VALUE" VARCHAR(4000)); CREATE TABLE "APP"."TBL_COL_PRIVS" ("TBL_COLUMN_GRANT_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(767), "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "TBL_COL_PRIV" VARCHAR(128), "TBL_ID" BIGINT, "AUTHORIZER" VARCHAR(128)); CREATE TABLE "APP"."SERDE_PARAMS" ("SERDE_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" CLOB); CREATE TABLE "APP"."COLUMNS_V2" ("CD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(4000), "COLUMN_NAME" VARCHAR(767) NOT NULL, "TYPE_NAME" CLOB, "INTEGER_IDX" INTEGER NOT NULL); CREATE TABLE "APP"."SORT_COLS" ("SD_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(767), "ORDER" INTEGER NOT NULL, "INTEGER_IDX" INTEGER NOT NULL); CREATE TABLE "APP"."CDS" ("CD_ID" BIGINT NOT NULL); CREATE TABLE "APP"."PARTITION_KEY_VALS" ("PART_ID" BIGINT NOT NULL, "PART_KEY_VAL" VARCHAR(256), "INTEGER_IDX" INTEGER NOT NULL); CREATE TABLE "APP"."DB_PRIVS" ("DB_GRANT_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "DB_PRIV" VARCHAR(128), "AUTHORIZER" VARCHAR(128)); CREATE TABLE "APP"."IDXS" ("INDEX_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DEFERRED_REBUILD" CHAR(1) NOT NULL, "INDEX_HANDLER_CLASS" VARCHAR(4000), "INDEX_NAME" VARCHAR(128), "INDEX_TBL_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "ORIG_TBL_ID" BIGINT, "SD_ID" BIGINT); CREATE TABLE "APP"."INDEX_PARAMS" ("INDEX_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(4000)); CREATE TABLE "APP"."PARTITIONS" ("PART_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "LAST_ACCESS_TIME" INTEGER NOT NULL, "PART_NAME" VARCHAR(767), "SD_ID" BIGINT, "TBL_ID" BIGINT); CREATE TABLE "APP"."SERDES" ("SERDE_ID" BIGINT NOT NULL, "NAME" VARCHAR(128), "SLIB" VARCHAR(4000), "DESCRIPTION" VARCHAR(4000), "SERIALIZER_CLASS" VARCHAR(4000), "DESERIALIZER_CLASS" VARCHAR(4000), SERDE_TYPE INTEGER); CREATE TABLE "APP"."PART_PRIVS" ("PART_GRANT_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PART_ID" BIGINT, "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "PART_PRIV" VARCHAR(128), "AUTHORIZER" VARCHAR(128)); CREATE TABLE "APP"."ROLE_MAP" ("ROLE_GRANT_ID" BIGINT NOT NULL, "ADD_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "ROLE_ID" BIGINT); CREATE TABLE "APP"."TYPES" ("TYPES_ID" BIGINT NOT NULL, "TYPE_NAME" VARCHAR(128), "TYPE1" VARCHAR(767), "TYPE2" VARCHAR(767)); CREATE TABLE "APP"."GLOBAL_PRIVS" ("USER_GRANT_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "USER_PRIV" VARCHAR(128), "AUTHORIZER" VARCHAR(128)); CREATE TABLE "APP"."PARTITION_PARAMS" ("PART_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(4000)); CREATE TABLE "APP"."PARTITION_EVENTS" ( "PART_NAME_ID" BIGINT NOT NULL, "CAT_NAME" VARCHAR(256), "DB_NAME" VARCHAR(128), "EVENT_TIME" BIGINT NOT NULL, "EVENT_TYPE" INTEGER NOT NULL, "PARTITION_NAME" VARCHAR(767), "TBL_NAME" VARCHAR(256) ); CREATE TABLE "APP"."COLUMNS" ("SD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(256), "COLUMN_NAME" VARCHAR(128) NOT NULL, "TYPE_NAME" VARCHAR(4000) NOT NULL, "INTEGER_IDX" INTEGER NOT NULL); CREATE TABLE "APP"."ROLES" ("ROLE_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "OWNER_NAME" VARCHAR(128), "ROLE_NAME" VARCHAR(128)); CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "OWNER_TYPE" VARCHAR(10), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(256), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "IS_REWRITE_ENABLED" CHAR(1) NOT NULL DEFAULT 'N'); CREATE TABLE "APP"."PARTITION_KEYS" ("TBL_ID" BIGINT NOT NULL, "PKEY_COMMENT" VARCHAR(4000), "PKEY_NAME" VARCHAR(128) NOT NULL, "PKEY_TYPE" VARCHAR(767) NOT NULL, "INTEGER_IDX" INTEGER NOT NULL); CREATE TABLE "APP"."PART_COL_PRIVS" ("PART_COLUMN_GRANT_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(767), "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PART_ID" BIGINT, "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "PART_COL_PRIV" VARCHAR(128), "AUTHORIZER" VARCHAR(128)); CREATE TABLE "APP"."SDS" ("SD_ID" BIGINT NOT NULL, "INPUT_FORMAT" VARCHAR(4000), "IS_COMPRESSED" CHAR(1) NOT NULL, "LOCATION" VARCHAR(4000), "NUM_BUCKETS" INTEGER NOT NULL, "OUTPUT_FORMAT" VARCHAR(4000), "SERDE_ID" BIGINT, "CD_ID" BIGINT, "IS_STOREDASSUBDIRECTORIES" CHAR(1) NOT NULL); CREATE TABLE "APP"."SEQUENCE_TABLE" ("SEQUENCE_NAME" VARCHAR(256) NOT NULL, "NEXT_VAL" BIGINT NOT NULL); CREATE TABLE "APP"."TAB_COL_STATS"( "CAT_NAME" VARCHAR(256) NOT NULL, "DB_NAME" VARCHAR(128) NOT NULL, "TABLE_NAME" VARCHAR(256) NOT NULL, "COLUMN_NAME" VARCHAR(767) NOT NULL, "COLUMN_TYPE" VARCHAR(128) NOT NULL, "LONG_LOW_VALUE" BIGINT, "LONG_HIGH_VALUE" BIGINT, "DOUBLE_LOW_VALUE" DOUBLE, "DOUBLE_HIGH_VALUE" DOUBLE, "BIG_DECIMAL_LOW_VALUE" VARCHAR(4000), "BIG_DECIMAL_HIGH_VALUE" VARCHAR(4000), "NUM_DISTINCTS" BIGINT, "NUM_NULLS" BIGINT NOT NULL, "AVG_COL_LEN" DOUBLE, "MAX_COL_LEN" BIGINT, "NUM_TRUES" BIGINT, "NUM_FALSES" BIGINT, "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, "TBL_ID" BIGINT NOT NULL, "BIT_VECTOR" BLOB ); CREATE TABLE "APP"."TABLE_PARAMS" ("TBL_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" CLOB); CREATE TABLE "APP"."BUCKETING_COLS" ("SD_ID" BIGINT NOT NULL, "BUCKET_COL_NAME" VARCHAR(256), "INTEGER_IDX" INTEGER NOT NULL); CREATE TABLE "APP"."TYPE_FIELDS" ("TYPE_NAME" BIGINT NOT NULL, "COMMENT" VARCHAR(256), "FIELD_NAME" VARCHAR(128) NOT NULL, "FIELD_TYPE" VARCHAR(767) NOT NULL, "INTEGER_IDX" INTEGER NOT NULL); CREATE TABLE "APP"."NUCLEUS_TABLES" ("CLASS_NAME" VARCHAR(128) NOT NULL, "TABLE_NAME" VARCHAR(128) NOT NULL, "TYPE" VARCHAR(4) NOT NULL, "OWNER" VARCHAR(2) NOT NULL, "VERSION" VARCHAR(20) NOT NULL, "INTERFACE_NAME" VARCHAR(256) DEFAULT NULL); CREATE TABLE "APP"."SD_PARAMS" ("SD_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" CLOB); CREATE TABLE "APP"."SKEWED_STRING_LIST" ("STRING_LIST_ID" BIGINT NOT NULL); CREATE TABLE "APP"."SKEWED_STRING_LIST_VALUES" ("STRING_LIST_ID" BIGINT NOT NULL, "STRING_LIST_VALUE" VARCHAR(256), "INTEGER_IDX" INTEGER NOT NULL); CREATE TABLE "APP"."SKEWED_COL_NAMES" ("SD_ID" BIGINT NOT NULL, "SKEWED_COL_NAME" VARCHAR(256), "INTEGER_IDX" INTEGER NOT NULL); CREATE TABLE "APP"."SKEWED_COL_VALUE_LOC_MAP" ("SD_ID" BIGINT NOT NULL, "STRING_LIST_ID_KID" BIGINT NOT NULL, "LOCATION" VARCHAR(4000)); CREATE TABLE "APP"."SKEWED_VALUES" ("SD_ID_OID" BIGINT NOT NULL, "STRING_LIST_ID_EID" BIGINT NOT NULL, "INTEGER_IDX" INTEGER NOT NULL); CREATE TABLE "APP"."MASTER_KEYS" ("KEY_ID" INTEGER NOT NULL generated always as identity (start with 1), "MASTER_KEY" VARCHAR(767)); CREATE TABLE "APP"."DELEGATION_TOKENS" ( "TOKEN_IDENT" VARCHAR(767) NOT NULL, "TOKEN" VARCHAR(767)); CREATE TABLE "APP"."PART_COL_STATS"( "CAT_NAME" VARCHAR(256) NOT NULL, "DB_NAME" VARCHAR(128) NOT NULL, "TABLE_NAME" VARCHAR(256) NOT NULL, "PARTITION_NAME" VARCHAR(767) NOT NULL, "COLUMN_NAME" VARCHAR(767) NOT NULL, "COLUMN_TYPE" VARCHAR(128) NOT NULL, "LONG_LOW_VALUE" BIGINT, "LONG_HIGH_VALUE" BIGINT, "DOUBLE_LOW_VALUE" DOUBLE, "DOUBLE_HIGH_VALUE" DOUBLE, "BIG_DECIMAL_LOW_VALUE" VARCHAR(4000), "BIG_DECIMAL_HIGH_VALUE" VARCHAR(4000), "NUM_DISTINCTS" BIGINT, "BIT_VECTOR" BLOB, "NUM_NULLS" BIGINT NOT NULL, "AVG_COL_LEN" DOUBLE, "MAX_COL_LEN" BIGINT, "NUM_TRUES" BIGINT, "NUM_FALSES" BIGINT, "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, "PART_ID" BIGINT NOT NULL ); CREATE TABLE "APP"."VERSION" ("VER_ID" BIGINT NOT NULL, "SCHEMA_VERSION" VARCHAR(127) NOT NULL, "VERSION_COMMENT" VARCHAR(255)); CREATE TABLE "APP"."FUNCS" ("FUNC_ID" BIGINT NOT NULL, "CLASS_NAME" VARCHAR(4000), "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "FUNC_NAME" VARCHAR(128), "FUNC_TYPE" INTEGER NOT NULL, "OWNER_NAME" VARCHAR(128), "OWNER_TYPE" VARCHAR(10)); CREATE TABLE "APP"."FUNC_RU" ("FUNC_ID" BIGINT NOT NULL, "RESOURCE_TYPE" INTEGER NOT NULL, "RESOURCE_URI" VARCHAR(4000), "INTEGER_IDX" INTEGER NOT NULL); CREATE TABLE "APP"."NOTIFICATION_LOG" ( "NL_ID" BIGINT NOT NULL, "CAT_NAME" VARCHAR(256), "DB_NAME" VARCHAR(128), "EVENT_ID" BIGINT NOT NULL, "EVENT_TIME" INTEGER NOT NULL, "EVENT_TYPE" VARCHAR(32) NOT NULL, "MESSAGE" CLOB, "TBL_NAME" VARCHAR(256), "MESSAGE_FORMAT" VARCHAR(16) ); CREATE TABLE "APP"."NOTIFICATION_SEQUENCE" ("NNI_ID" BIGINT NOT NULL, "NEXT_EVENT_ID" BIGINT NOT NULL); CREATE TABLE "APP"."KEY_CONSTRAINTS" ("CHILD_CD_ID" BIGINT, "CHILD_INTEGER_IDX" INTEGER, "CHILD_TBL_ID" BIGINT, "PARENT_CD_ID" BIGINT , "PARENT_INTEGER_IDX" INTEGER, "PARENT_TBL_ID" BIGINT NOT NULL, "POSITION" BIGINT NOT NULL, "CONSTRAINT_NAME" VARCHAR(400) NOT NULL, "CONSTRAINT_TYPE" SMALLINT NOT NULL, "UPDATE_RULE" SMALLINT, "DELETE_RULE" SMALLINT, "ENABLE_VALIDATE_RELY" SMALLINT NOT NULL, "DEFAULT_VALUE" VARCHAR(400)); CREATE TABLE "APP"."METASTORE_DB_PROPERTIES" ("PROPERTY_KEY" VARCHAR(255) NOT NULL, "PROPERTY_VALUE" VARCHAR(1000) NOT NULL, "DESCRIPTION" VARCHAR(1000)); CREATE TABLE "APP"."WM_RESOURCEPLAN" (RP_ID BIGINT NOT NULL, NAME VARCHAR(128) NOT NULL, QUERY_PARALLELISM INTEGER, STATUS VARCHAR(20) NOT NULL, DEFAULT_POOL_ID BIGINT); CREATE TABLE "APP"."WM_POOL" (POOL_ID BIGINT NOT NULL, RP_ID BIGINT NOT NULL, PATH VARCHAR(1024) NOT NULL, ALLOC_FRACTION DOUBLE, QUERY_PARALLELISM INTEGER, SCHEDULING_POLICY VARCHAR(1024)); CREATE TABLE "APP"."WM_TRIGGER" (TRIGGER_ID BIGINT NOT NULL, RP_ID BIGINT NOT NULL, NAME VARCHAR(128) NOT NULL, TRIGGER_EXPRESSION VARCHAR(1024), ACTION_EXPRESSION VARCHAR(1024), IS_IN_UNMANAGED INTEGER NOT NULL DEFAULT 0); CREATE TABLE "APP"."WM_POOL_TO_TRIGGER" (POOL_ID BIGINT NOT NULL, TRIGGER_ID BIGINT NOT NULL); CREATE TABLE "APP"."WM_MAPPING" (MAPPING_ID BIGINT NOT NULL, RP_ID BIGINT NOT NULL, ENTITY_TYPE VARCHAR(128) NOT NULL, ENTITY_NAME VARCHAR(128) NOT NULL, POOL_ID BIGINT, ORDERING INTEGER); CREATE TABLE "APP"."MV_CREATION_METADATA" ( "MV_CREATION_METADATA_ID" BIGINT NOT NULL, "CAT_NAME" VARCHAR(256) NOT NULL, "DB_NAME" VARCHAR(128) NOT NULL, "TBL_NAME" VARCHAR(256) NOT NULL, "TXN_LIST" CLOB, "MATERIALIZATION_TIME" BIGINT NOT NULL ); CREATE TABLE "APP"."MV_TABLES_USED" ( "MV_CREATION_METADATA_ID" BIGINT NOT NULL, "TBL_ID" BIGINT NOT NULL ); CREATE TABLE "APP"."CTLGS" ( "CTLG_ID" BIGINT NOT NULL, "NAME" VARCHAR(256) UNIQUE, "DESC" VARCHAR(4000), "LOCATION_URI" VARCHAR(4000) NOT NULL); -- Insert a default value. The location is TBD. Hive will fix this when it starts INSERT INTO "APP"."CTLGS" VALUES (1, 'hive', 'Default catalog for Hive', 'TBD'); -- ---------------------------------------------- -- DML Statements -- ---------------------------------------------- INSERT INTO "APP"."NOTIFICATION_SEQUENCE" ("NNI_ID", "NEXT_EVENT_ID") SELECT * FROM (VALUES (1,1)) tmp_table WHERE NOT EXISTS ( SELECT "NEXT_EVENT_ID" FROM "APP"."NOTIFICATION_SEQUENCE"); INSERT INTO "APP"."SEQUENCE_TABLE" ("SEQUENCE_NAME", "NEXT_VAL") SELECT * FROM (VALUES ('org.apache.hadoop.hive.metastore.model.MNotificationLog', 1)) tmp_table WHERE NOT EXISTS ( SELECT "NEXT_VAL" FROM "APP"."SEQUENCE_TABLE" WHERE "SEQUENCE_NAME" = 'org.apache.hadoop.hive.metastore.model.MNotificationLog'); -- ---------------------------------------------- -- DDL Statements for indexes -- ---------------------------------------------- CREATE UNIQUE INDEX "APP"."UNIQUEINDEX" ON "APP"."IDXS" ("INDEX_NAME", "ORIG_TBL_ID"); CREATE INDEX "APP"."TABLECOLUMNPRIVILEGEINDEX" ON "APP"."TBL_COL_PRIVS" ("AUTHORIZER", "TBL_ID", "COLUMN_NAME", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "TBL_COL_PRIV", "GRANTOR", "GRANTOR_TYPE"); CREATE UNIQUE INDEX "APP"."DBPRIVILEGEINDEX" ON "APP"."DB_PRIVS" ("AUTHORIZER", "DB_ID", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "DB_PRIV", "GRANTOR", "GRANTOR_TYPE"); CREATE INDEX "APP"."PCS_STATS_IDX" ON "APP"."PART_COL_STATS" ("CAT_NAME", "DB_NAME","TABLE_NAME","COLUMN_NAME","PARTITION_NAME"); CREATE INDEX "APP"."TAB_COL_STATS_IDX" ON "APP"."TAB_COL_STATS" ("CAT_NAME", "DB_NAME", "TABLE_NAME", "COLUMN_NAME"); CREATE INDEX "APP"."PARTPRIVILEGEINDEX" ON "APP"."PART_PRIVS" ("AUTHORIZER", "PART_ID", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "PART_PRIV", "GRANTOR", "GRANTOR_TYPE"); CREATE UNIQUE INDEX "APP"."ROLEENTITYINDEX" ON "APP"."ROLES" ("ROLE_NAME"); CREATE INDEX "APP"."TABLEPRIVILEGEINDEX" ON "APP"."TBL_PRIVS" ("AUTHORIZER", "TBL_ID", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "TBL_PRIV", "GRANTOR", "GRANTOR_TYPE"); CREATE UNIQUE INDEX "APP"."UNIQUETABLE" ON "APP"."TBLS" ("TBL_NAME", "DB_ID"); CREATE UNIQUE INDEX "APP"."UNIQUE_DATABASE" ON "APP"."DBS" ("NAME", "CTLG_NAME"); CREATE UNIQUE INDEX "APP"."USERROLEMAPINDEX" ON "APP"."ROLE_MAP" ("PRINCIPAL_NAME", "ROLE_ID", "GRANTOR", "GRANTOR_TYPE"); CREATE UNIQUE INDEX "APP"."GLOBALPRIVILEGEINDEX" ON "APP"."GLOBAL_PRIVS" ("AUTHORIZER", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "USER_PRIV", "GRANTOR", "GRANTOR_TYPE"); CREATE UNIQUE INDEX "APP"."UNIQUE_TYPE" ON "APP"."TYPES" ("TYPE_NAME"); CREATE INDEX "APP"."PARTITIONCOLUMNPRIVILEGEINDEX" ON "APP"."PART_COL_PRIVS" ("AUTHORIZER", "PART_ID", "COLUMN_NAME", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "PART_COL_PRIV", "GRANTOR", "GRANTOR_TYPE"); CREATE UNIQUE INDEX "APP"."UNIQUEPARTITION" ON "APP"."PARTITIONS" ("PART_NAME", "TBL_ID"); CREATE UNIQUE INDEX "APP"."UNIQUEFUNCTION" ON "APP"."FUNCS" ("FUNC_NAME", "DB_ID"); CREATE INDEX "APP"."FUNCS_N49" ON "APP"."FUNCS" ("DB_ID"); CREATE INDEX "APP"."FUNC_RU_N49" ON "APP"."FUNC_RU" ("FUNC_ID"); CREATE INDEX "APP"."CONSTRAINTS_PARENT_TBL_ID_INDEX" ON "APP"."KEY_CONSTRAINTS"("PARENT_TBL_ID"); CREATE INDEX "APP"."CONSTRAINTS_CONSTRAINT_TYPE_INDEX" ON "APP"."KEY_CONSTRAINTS"("CONSTRAINT_TYPE"); CREATE UNIQUE INDEX "APP"."UNIQUE_WM_RESOURCEPLAN" ON "APP"."WM_RESOURCEPLAN" ("NAME"); CREATE UNIQUE INDEX "APP"."UNIQUE_WM_POOL" ON "APP"."WM_POOL" ("RP_ID", "PATH"); CREATE UNIQUE INDEX "APP"."UNIQUE_WM_TRIGGER" ON "APP"."WM_TRIGGER" ("RP_ID", "NAME"); CREATE UNIQUE INDEX "APP"."UNIQUE_WM_MAPPING" ON "APP"."WM_MAPPING" ("RP_ID", "ENTITY_TYPE", "ENTITY_NAME"); CREATE UNIQUE INDEX "APP"."MV_UNIQUE_TABLE" ON "APP"."MV_CREATION_METADATA" ("TBL_NAME", "DB_NAME"); CREATE UNIQUE INDEX "APP"."UNIQUE_CATALOG" ON "APP"."CTLGS" ("NAME"); -- ---------------------------------------------- -- DDL Statements for keys -- ---------------------------------------------- -- primary/unique ALTER TABLE "APP"."IDXS" ADD CONSTRAINT "IDXS_PK" PRIMARY KEY ("INDEX_ID"); ALTER TABLE "APP"."TBL_COL_PRIVS" ADD CONSTRAINT "TBL_COL_PRIVS_PK" PRIMARY KEY ("TBL_COLUMN_GRANT_ID"); ALTER TABLE "APP"."CDS" ADD CONSTRAINT "SQL110922153006460" PRIMARY KEY ("CD_ID"); ALTER TABLE "APP"."DB_PRIVS" ADD CONSTRAINT "DB_PRIVS_PK" PRIMARY KEY ("DB_GRANT_ID"); ALTER TABLE "APP"."INDEX_PARAMS" ADD CONSTRAINT "INDEX_PARAMS_PK" PRIMARY KEY ("INDEX_ID", "PARAM_KEY"); ALTER TABLE "APP"."PARTITION_KEYS" ADD CONSTRAINT "PARTITION_KEY_PK" PRIMARY KEY ("TBL_ID", "PKEY_NAME"); ALTER TABLE "APP"."SEQUENCE_TABLE" ADD CONSTRAINT "SEQUENCE_TABLE_PK" PRIMARY KEY ("SEQUENCE_NAME"); ALTER TABLE "APP"."PART_PRIVS" ADD CONSTRAINT "PART_PRIVS_PK" PRIMARY KEY ("PART_GRANT_ID"); ALTER TABLE "APP"."SDS" ADD CONSTRAINT "SDS_PK" PRIMARY KEY ("SD_ID"); ALTER TABLE "APP"."SERDES" ADD CONSTRAINT "SERDES_PK" PRIMARY KEY ("SERDE_ID"); ALTER TABLE "APP"."COLUMNS" ADD CONSTRAINT "COLUMNS_PK" PRIMARY KEY ("SD_ID", "COLUMN_NAME"); ALTER TABLE "APP"."PARTITION_EVENTS" ADD CONSTRAINT "PARTITION_EVENTS_PK" PRIMARY KEY ("PART_NAME_ID"); ALTER TABLE "APP"."TYPE_FIELDS" ADD CONSTRAINT "TYPE_FIELDS_PK" PRIMARY KEY ("TYPE_NAME", "FIELD_NAME"); ALTER TABLE "APP"."ROLES" ADD CONSTRAINT "ROLES_PK" PRIMARY KEY ("ROLE_ID"); ALTER TABLE "APP"."TBL_PRIVS" ADD CONSTRAINT "TBL_PRIVS_PK" PRIMARY KEY ("TBL_GRANT_ID"); ALTER TABLE "APP"."SERDE_PARAMS" ADD CONSTRAINT "SERDE_PARAMS_PK" PRIMARY KEY ("SERDE_ID", "PARAM_KEY"); ALTER TABLE "APP"."NUCLEUS_TABLES" ADD CONSTRAINT "NUCLEUS_TABLES_PK" PRIMARY KEY ("CLASS_NAME"); ALTER TABLE "APP"."TBLS" ADD CONSTRAINT "TBLS_PK" PRIMARY KEY ("TBL_ID"); ALTER TABLE "APP"."SD_PARAMS" ADD CONSTRAINT "SD_PARAMS_PK" PRIMARY KEY ("SD_ID", "PARAM_KEY"); ALTER TABLE "APP"."DATABASE_PARAMS" ADD CONSTRAINT "DATABASE_PARAMS_PK" PRIMARY KEY ("DB_ID", "PARAM_KEY"); ALTER TABLE "APP"."DBS" ADD CONSTRAINT "DBS_PK" PRIMARY KEY ("DB_ID"); ALTER TABLE "APP"."ROLE_MAP" ADD CONSTRAINT "ROLE_MAP_PK" PRIMARY KEY ("ROLE_GRANT_ID"); ALTER TABLE "APP"."GLOBAL_PRIVS" ADD CONSTRAINT "GLOBAL_PRIVS_PK" PRIMARY KEY ("USER_GRANT_ID"); ALTER TABLE "APP"."BUCKETING_COLS" ADD CONSTRAINT "BUCKETING_COLS_PK" PRIMARY KEY ("SD_ID", "INTEGER_IDX"); ALTER TABLE "APP"."SORT_COLS" ADD CONSTRAINT "SORT_COLS_PK" PRIMARY KEY ("SD_ID", "INTEGER_IDX"); ALTER TABLE "APP"."PARTITION_KEY_VALS" ADD CONSTRAINT "PARTITION_KEY_VALS_PK" PRIMARY KEY ("PART_ID", "INTEGER_IDX"); ALTER TABLE "APP"."TYPES" ADD CONSTRAINT "TYPES_PK" PRIMARY KEY ("TYPES_ID"); ALTER TABLE "APP"."COLUMNS_V2" ADD CONSTRAINT "SQL110922153006740" PRIMARY KEY ("CD_ID", "COLUMN_NAME"); ALTER TABLE "APP"."PART_COL_PRIVS" ADD CONSTRAINT "PART_COL_PRIVS_PK" PRIMARY KEY ("PART_COLUMN_GRANT_ID"); ALTER TABLE "APP"."PARTITION_PARAMS" ADD CONSTRAINT "PARTITION_PARAMS_PK" PRIMARY KEY ("PART_ID", "PARAM_KEY"); ALTER TABLE "APP"."PARTITIONS" ADD CONSTRAINT "PARTITIONS_PK" PRIMARY KEY ("PART_ID"); ALTER TABLE "APP"."TABLE_PARAMS" ADD CONSTRAINT "TABLE_PARAMS_PK" PRIMARY KEY ("TBL_ID", "PARAM_KEY"); ALTER TABLE "APP"."SKEWED_STRING_LIST" ADD CONSTRAINT "SKEWED_STRING_LIST_PK" PRIMARY KEY ("STRING_LIST_ID"); ALTER TABLE "APP"."SKEWED_STRING_LIST_VALUES" ADD CONSTRAINT "SKEWED_STRING_LIST_VALUES_PK" PRIMARY KEY ("STRING_LIST_ID", "INTEGER_IDX"); ALTER TABLE "APP"."SKEWED_COL_NAMES" ADD CONSTRAINT "SKEWED_COL_NAMES_PK" PRIMARY KEY ("SD_ID", "INTEGER_IDX"); ALTER TABLE "APP"."SKEWED_COL_VALUE_LOC_MAP" ADD CONSTRAINT "SKEWED_COL_VALUE_LOC_MAP_PK" PRIMARY KEY ("SD_ID", "STRING_LIST_ID_KID"); ALTER TABLE "APP"."SKEWED_VALUES" ADD CONSTRAINT "SKEWED_VALUES_PK" PRIMARY KEY ("SD_ID_OID", "INTEGER_IDX"); ALTER TABLE "APP"."TAB_COL_STATS" ADD CONSTRAINT "TAB_COL_STATS_PK" PRIMARY KEY ("CS_ID"); ALTER TABLE "APP"."PART_COL_STATS" ADD CONSTRAINT "PART_COL_STATS_PK" PRIMARY KEY ("CS_ID"); ALTER TABLE "APP"."FUNCS" ADD CONSTRAINT "FUNCS_PK" PRIMARY KEY ("FUNC_ID"); ALTER TABLE "APP"."FUNC_RU" ADD CONSTRAINT "FUNC_RU_PK" PRIMARY KEY ("FUNC_ID", "INTEGER_IDX"); ALTER TABLE "APP"."NOTIFICATION_LOG" ADD CONSTRAINT "NOTIFICATION_LOG_PK" PRIMARY KEY ("NL_ID"); ALTER TABLE "APP"."NOTIFICATION_SEQUENCE" ADD CONSTRAINT "NOTIFICATION_SEQUENCE_PK" PRIMARY KEY ("NNI_ID"); ALTER TABLE "APP"."KEY_CONSTRAINTS" ADD CONSTRAINT "CONSTRAINTS_PK" PRIMARY KEY ("CONSTRAINT_NAME", "POSITION"); ALTER TABLE "APP"."METASTORE_DB_PROPERTIES" ADD CONSTRAINT "PROPERTY_KEY_PK" PRIMARY KEY ("PROPERTY_KEY"); ALTER TABLE "APP"."MV_CREATION_METADATA" ADD CONSTRAINT "MV_CREATION_METADATA_PK" PRIMARY KEY ("MV_CREATION_METADATA_ID"); ALTER TABLE "APP"."CTLGS" ADD CONSTRAINT "CTLG_PK" PRIMARY KEY ("CTLG_ID"); -- foreign ALTER TABLE "APP"."IDXS" ADD CONSTRAINT "IDXS_FK1" FOREIGN KEY ("ORIG_TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."IDXS" ADD CONSTRAINT "IDXS_FK2" FOREIGN KEY ("SD_ID") REFERENCES "APP"."SDS" ("SD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."IDXS" ADD CONSTRAINT "IDXS_FK3" FOREIGN KEY ("INDEX_TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."TBL_COL_PRIVS" ADD CONSTRAINT "TBL_COL_PRIVS_FK1" FOREIGN KEY ("TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."DB_PRIVS" ADD CONSTRAINT "DB_PRIVS_FK1" FOREIGN KEY ("DB_ID") REFERENCES "APP"."DBS" ("DB_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."INDEX_PARAMS" ADD CONSTRAINT "INDEX_PARAMS_FK1" FOREIGN KEY ("INDEX_ID") REFERENCES "APP"."IDXS" ("INDEX_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."PARTITION_KEYS" ADD CONSTRAINT "PARTITION_KEYS_FK1" FOREIGN KEY ("TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."PART_PRIVS" ADD CONSTRAINT "PART_PRIVS_FK1" FOREIGN KEY ("PART_ID") REFERENCES "APP"."PARTITIONS" ("PART_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."SDS" ADD CONSTRAINT "SDS_FK1" FOREIGN KEY ("SERDE_ID") REFERENCES "APP"."SERDES" ("SERDE_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."SDS" ADD CONSTRAINT "SDS_FK2" FOREIGN KEY ("CD_ID") REFERENCES "APP"."CDS" ("CD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."COLUMNS" ADD CONSTRAINT "COLUMNS_FK1" FOREIGN KEY ("SD_ID") REFERENCES "APP"."SDS" ("SD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."TYPE_FIELDS" ADD CONSTRAINT "TYPE_FIELDS_FK1" FOREIGN KEY ("TYPE_NAME") REFERENCES "APP"."TYPES" ("TYPES_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."TBL_PRIVS" ADD CONSTRAINT "TBL_PRIVS_FK1" FOREIGN KEY ("TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."SERDE_PARAMS" ADD CONSTRAINT "SERDE_PARAMS_FK1" FOREIGN KEY ("SERDE_ID") REFERENCES "APP"."SERDES" ("SERDE_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."TBLS" ADD CONSTRAINT "TBLS_FK2" FOREIGN KEY ("SD_ID") REFERENCES "APP"."SDS" ("SD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."TBLS" ADD CONSTRAINT "TBLS_FK1" FOREIGN KEY ("DB_ID") REFERENCES "APP"."DBS" ("DB_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."DBS" ADD CONSTRAINT "DBS_FK1" FOREIGN KEY ("CTLG_NAME") REFERENCES "APP"."CTLGS" ("NAME") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."SD_PARAMS" ADD CONSTRAINT "SD_PARAMS_FK1" FOREIGN KEY ("SD_ID") REFERENCES "APP"."SDS" ("SD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."DATABASE_PARAMS" ADD CONSTRAINT "DATABASE_PARAMS_FK1" FOREIGN KEY ("DB_ID") REFERENCES "APP"."DBS" ("DB_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."ROLE_MAP" ADD CONSTRAINT "ROLE_MAP_FK1" FOREIGN KEY ("ROLE_ID") REFERENCES "APP"."ROLES" ("ROLE_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."BUCKETING_COLS" ADD CONSTRAINT "BUCKETING_COLS_FK1" FOREIGN KEY ("SD_ID") REFERENCES "APP"."SDS" ("SD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."SORT_COLS" ADD CONSTRAINT "SORT_COLS_FK1" FOREIGN KEY ("SD_ID") REFERENCES "APP"."SDS" ("SD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."PARTITION_KEY_VALS" ADD CONSTRAINT "PARTITION_KEY_VALS_FK1" FOREIGN KEY ("PART_ID") REFERENCES "APP"."PARTITIONS" ("PART_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."COLUMNS_V2" ADD CONSTRAINT "COLUMNS_V2_FK1" FOREIGN KEY ("CD_ID") REFERENCES "APP"."CDS" ("CD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."PART_COL_PRIVS" ADD CONSTRAINT "PART_COL_PRIVS_FK1" FOREIGN KEY ("PART_ID") REFERENCES "APP"."PARTITIONS" ("PART_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."PARTITION_PARAMS" ADD CONSTRAINT "PARTITION_PARAMS_FK1" FOREIGN KEY ("PART_ID") REFERENCES "APP"."PARTITIONS" ("PART_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."PARTITIONS" ADD CONSTRAINT "PARTITIONS_FK1" FOREIGN KEY ("TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."PARTITIONS" ADD CONSTRAINT "PARTITIONS_FK2" FOREIGN KEY ("SD_ID") REFERENCES "APP"."SDS" ("SD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."TABLE_PARAMS" ADD CONSTRAINT "TABLE_PARAMS_FK1" FOREIGN KEY ("TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."SKEWED_STRING_LIST_VALUES" ADD CONSTRAINT "SKEWED_STRING_LIST_VALUES_FK1" FOREIGN KEY ("STRING_LIST_ID") REFERENCES "APP"."SKEWED_STRING_LIST" ("STRING_LIST_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."SKEWED_COL_NAMES" ADD CONSTRAINT "SKEWED_COL_NAMES_FK1" FOREIGN KEY ("SD_ID") REFERENCES "APP"."SDS" ("SD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."SKEWED_COL_VALUE_LOC_MAP" ADD CONSTRAINT "SKEWED_COL_VALUE_LOC_MAP_FK1" FOREIGN KEY ("SD_ID") REFERENCES "APP"."SDS" ("SD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."SKEWED_COL_VALUE_LOC_MAP" ADD CONSTRAINT "SKEWED_COL_VALUE_LOC_MAP_FK2" FOREIGN KEY ("STRING_LIST_ID_KID") REFERENCES "APP"."SKEWED_STRING_LIST" ("STRING_LIST_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."SKEWED_VALUES" ADD CONSTRAINT "SKEWED_VALUES_FK1" FOREIGN KEY ("SD_ID_OID") REFERENCES "APP"."SDS" ("SD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."SKEWED_VALUES" ADD CONSTRAINT "SKEWED_VALUES_FK2" FOREIGN KEY ("STRING_LIST_ID_EID") REFERENCES "APP"."SKEWED_STRING_LIST" ("STRING_LIST_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."TAB_COL_STATS" ADD CONSTRAINT "TAB_COL_STATS_FK" FOREIGN KEY ("TBL_ID") REFERENCES TBLS("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."PART_COL_STATS" ADD CONSTRAINT "PART_COL_STATS_FK" FOREIGN KEY ("PART_ID") REFERENCES PARTITIONS("PART_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."VERSION" ADD CONSTRAINT "VERSION_PK" PRIMARY KEY ("VER_ID"); ALTER TABLE "APP"."FUNCS" ADD CONSTRAINT "FUNCS_FK1" FOREIGN KEY ("DB_ID") REFERENCES "APP"."DBS" ("DB_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."FUNC_RU" ADD CONSTRAINT "FUNC_RU_FK1" FOREIGN KEY ("FUNC_ID") REFERENCES "APP"."FUNCS" ("FUNC_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."WM_RESOURCEPLAN" ADD CONSTRAINT "WM_RESOURCEPLAN_PK" PRIMARY KEY ("RP_ID"); ALTER TABLE "APP"."WM_POOL" ADD CONSTRAINT "WM_POOL_PK" PRIMARY KEY ("POOL_ID"); ALTER TABLE "APP"."WM_POOL" ADD CONSTRAINT "WM_POOL_FK1" FOREIGN KEY ("RP_ID") REFERENCES "APP"."WM_RESOURCEPLAN" ("RP_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."WM_RESOURCEPLAN" ADD CONSTRAINT "WM_RESOURCEPLAN_FK1" FOREIGN KEY ("DEFAULT_POOL_ID") REFERENCES "APP"."WM_POOL" ("POOL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."WM_TRIGGER" ADD CONSTRAINT "WM_TRIGGER_PK" PRIMARY KEY ("TRIGGER_ID"); ALTER TABLE "APP"."WM_TRIGGER" ADD CONSTRAINT "WM_TRIGGER_FK1" FOREIGN KEY ("RP_ID") REFERENCES "APP"."WM_RESOURCEPLAN" ("RP_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."WM_POOL_TO_TRIGGER" ADD CONSTRAINT "WM_POOL_TO_TRIGGER_FK1" FOREIGN KEY ("POOL_ID") REFERENCES "APP"."WM_POOL" ("POOL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."WM_POOL_TO_TRIGGER" ADD CONSTRAINT "WM_POOL_TO_TRIGGER_FK2" FOREIGN KEY ("TRIGGER_ID") REFERENCES "APP"."WM_TRIGGER" ("TRIGGER_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."WM_MAPPING" ADD CONSTRAINT "WM_MAPPING_PK" PRIMARY KEY ("MAPPING_ID"); ALTER TABLE "APP"."WM_MAPPING" ADD CONSTRAINT "WM_MAPPING_FK1" FOREIGN KEY ("RP_ID") REFERENCES "APP"."WM_RESOURCEPLAN" ("RP_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."WM_MAPPING" ADD CONSTRAINT "WM_MAPPING_FK2" FOREIGN KEY ("POOL_ID") REFERENCES "APP"."WM_POOL" ("POOL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."MV_TABLES_USED" ADD CONSTRAINT "MV_TABLES_USED_FK1" FOREIGN KEY ("MV_CREATION_METADATA_ID") REFERENCES "APP"."MV_CREATION_METADATA" ("MV_CREATION_METADATA_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."MV_TABLES_USED" ADD CONSTRAINT "MV_TABLES_USED_FK2" FOREIGN KEY ("TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE "APP"."DBS" ADD CONSTRAINT "DBS_CTLG_FK" FOREIGN KEY ("CTLG_NAME") REFERENCES "APP"."CTLGS" ("NAME") ON DELETE NO ACTION ON UPDATE NO ACTION; -- ---------------------------------------------- -- DDL Statements for checks -- ---------------------------------------------- ALTER TABLE "APP"."IDXS" ADD CONSTRAINT "SQL110318025504980" CHECK (DEFERRED_REBUILD IN ('Y','N')); ALTER TABLE "APP"."SDS" ADD CONSTRAINT "SQL110318025505550" CHECK (IS_COMPRESSED IN ('Y','N')); -- ---------------------------- -- Transaction and Lock Tables -- ---------------------------- CREATE TABLE TXNS ( TXN_ID bigint PRIMARY KEY, TXN_STATE char(1) NOT NULL, TXN_STARTED bigint NOT NULL, TXN_LAST_HEARTBEAT bigint NOT NULL, TXN_USER varchar(128) NOT NULL, TXN_HOST varchar(128) NOT NULL, TXN_AGENT_INFO varchar(128), TXN_META_INFO varchar(128), TXN_HEARTBEAT_COUNT integer, TXN_TYPE integer ); CREATE TABLE TXN_COMPONENTS ( TC_TXNID bigint NOT NULL REFERENCES TXNS (TXN_ID), TC_DATABASE varchar(128) NOT NULL, TC_TABLE varchar(128), TC_PARTITION varchar(767), TC_OPERATION_TYPE char(1) NOT NULL, TC_WRITEID bigint ); CREATE INDEX TC_TXNID_INDEX ON TXN_COMPONENTS (TC_TXNID); CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_TXNID bigint NOT NULL, CTC_DATABASE varchar(128) NOT NULL, CTC_TABLE varchar(256), CTC_PARTITION varchar(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, CTC_WRITEID bigint, CTC_UPDATE_DELETE char(1) NOT NULL ); CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); CREATE TABLE NEXT_TXN_ID ( NTXN_NEXT bigint NOT NULL ); INSERT INTO NEXT_TXN_ID VALUES(1); CREATE TABLE HIVE_LOCKS ( HL_LOCK_EXT_ID bigint NOT NULL, HL_LOCK_INT_ID bigint NOT NULL, HL_TXNID bigint NOT NULL, HL_DB varchar(128) NOT NULL, HL_TABLE varchar(128), HL_PARTITION varchar(767), HL_LOCK_STATE char(1) NOT NULL, HL_LOCK_TYPE char(1) NOT NULL, HL_LAST_HEARTBEAT bigint NOT NULL, HL_ACQUIRED_AT bigint, HL_USER varchar(128) NOT NULL, HL_HOST varchar(128) NOT NULL, HL_HEARTBEAT_COUNT integer, HL_AGENT_INFO varchar(128), HL_BLOCKEDBY_EXT_ID bigint, HL_BLOCKEDBY_INT_ID bigint, PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID) ); CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID); CREATE TABLE NEXT_LOCK_ID ( NL_NEXT bigint NOT NULL ); INSERT INTO NEXT_LOCK_ID VALUES(1); CREATE TABLE COMPACTION_QUEUE ( CQ_ID bigint PRIMARY KEY, CQ_DATABASE varchar(128) NOT NULL, CQ_TABLE varchar(128) NOT NULL, CQ_PARTITION varchar(767), CQ_STATE char(1) NOT NULL, CQ_TYPE char(1) NOT NULL, CQ_TBLPROPERTIES varchar(2048), CQ_WORKER_ID varchar(128), CQ_START bigint, CQ_RUN_AS varchar(128), CQ_HIGHEST_WRITE_ID bigint, CQ_META_INFO varchar(2048) for bit data, CQ_HADOOP_JOB_ID varchar(32) ); CREATE TABLE NEXT_COMPACTION_QUEUE_ID ( NCQ_NEXT bigint NOT NULL ); INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); CREATE TABLE COMPLETED_COMPACTIONS ( CC_ID bigint PRIMARY KEY, CC_DATABASE varchar(128) NOT NULL, CC_TABLE varchar(128) NOT NULL, CC_PARTITION varchar(767), CC_STATE char(1) NOT NULL, CC_TYPE char(1) NOT NULL, CC_TBLPROPERTIES varchar(2048), CC_WORKER_ID varchar(128), CC_START bigint, CC_END bigint, CC_RUN_AS varchar(128), CC_HIGHEST_WRITE_ID bigint, CC_META_INFO varchar(2048) for bit data, CC_HADOOP_JOB_ID varchar(32) ); CREATE TABLE AUX_TABLE ( MT_KEY1 varchar(128) NOT NULL, MT_KEY2 bigint NOT NULL, MT_COMMENT varchar(255), PRIMARY KEY(MT_KEY1, MT_KEY2) ); --1st 4 cols make up a PK but since WS_PARTITION is nullable we can't declare such PK --This is a good candidate for Index orgainzed table CREATE TABLE WRITE_SET ( WS_DATABASE varchar(128) NOT NULL, WS_TABLE varchar(128) NOT NULL, WS_PARTITION varchar(767), WS_TXNID bigint NOT NULL, WS_COMMIT_ID bigint NOT NULL, WS_OPERATION_TYPE char(1) NOT NULL ); CREATE TABLE TXN_TO_WRITE_ID ( T2W_TXNID bigint NOT NULL, T2W_DATABASE varchar(128) NOT NULL, T2W_TABLE varchar(256) NOT NULL, T2W_WRITEID bigint NOT NULL ); CREATE UNIQUE INDEX TBL_TO_TXN_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID); CREATE UNIQUE INDEX TBL_TO_WRITE_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_WRITEID); CREATE TABLE NEXT_WRITE_ID ( NWI_DATABASE varchar(128) NOT NULL, NWI_TABLE varchar(256) NOT NULL, NWI_NEXT bigint NOT NULL ); CREATE UNIQUE INDEX NEXT_WRITE_ID_IDX ON NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE); CREATE TABLE MIN_HISTORY_LEVEL ( MHL_TXNID bigint NOT NULL, MHL_MIN_OPEN_TXNID bigint NOT NULL, PRIMARY KEY(MHL_TXNID) ); CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( MRL_TXN_ID BIGINT NOT NULL, MRL_DB_NAME VARCHAR(128) NOT NULL, MRL_TBL_NAME VARCHAR(256) NOT NULL, MRL_LAST_HEARTBEAT BIGINT NOT NULL, PRIMARY KEY(MRL_TXN_ID) ); CREATE TABLE "APP"."I_SCHEMA" ( "SCHEMA_ID" bigint primary key, "SCHEMA_TYPE" integer not null, "NAME" varchar(256) unique, "DB_ID" bigint references "APP"."DBS" ("DB_ID"), "COMPATIBILITY" integer not null, "VALIDATION_LEVEL" integer not null, "CAN_EVOLVE" char(1) not null, "SCHEMA_GROUP" varchar(256), "DESCRIPTION" varchar(4000) ); CREATE TABLE "APP"."SCHEMA_VERSION" ( "SCHEMA_VERSION_ID" bigint primary key, "SCHEMA_ID" bigint references "APP"."I_SCHEMA" ("SCHEMA_ID"), "VERSION" integer not null, "CREATED_AT" bigint not null, "CD_ID" bigint references "APP"."CDS" ("CD_ID"), "STATE" integer not null, "DESCRIPTION" varchar(4000), "SCHEMA_TEXT" clob, "FINGERPRINT" varchar(256), "SCHEMA_VERSION_NAME" varchar(256), "SERDE_ID" bigint references "APP"."SERDES" ("SERDE_ID") ); CREATE UNIQUE INDEX "APP"."UNIQUE_SCHEMA_VERSION" ON "APP"."SCHEMA_VERSION" ("SCHEMA_ID", "VERSION"); CREATE TABLE REPL_TXN_MAP ( RTM_REPL_POLICY varchar(256) NOT NULL, RTM_SRC_TXN_ID bigint NOT NULL, RTM_TARGET_TXN_ID bigint NOT NULL, PRIMARY KEY (RTM_REPL_POLICY, RTM_SRC_TXN_ID) ); CREATE TABLE "APP"."RUNTIME_STATS" ( "RS_ID" bigint primary key, "CREATE_TIME" integer not null, "WEIGHT" integer not null, "PAYLOAD" BLOB ); CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script -- ----------------------------------------------------------------- INSERT INTO "APP"."VERSION" (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '3.1.0', 'Hive release version 3.1.0'); ================================================ FILE: spark/src/test/resources/log4j2.properties ================================================ # # Copyright (2021) The Delta Lake Project Authors. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Set everything to be logged to the file target/unit-tests.log rootLogger.level = warn rootLogger.appenderRef.file.ref = ${sys:test.appender:-File} appender.file.type = File appender.file.name = File appender.file.fileName = target/unit-tests.log appender.file.append = true appender.file.layout.type = PatternLayout appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n # Structured Logging Appender appender.structured.type = File appender.structured.name = structured appender.structured.fileName = target/structured.log appender.structured.layout.type = JsonTemplateLayout appender.structured.layout.eventTemplateUri = classpath:org/apache/spark/SparkLayout.json # Custom logger for testing structured logging with Spark 4.0+ logger.structured_logging.name = org.apache.spark.sql.delta.logging.DeltaStructuredLoggingSuite logger.structured_logging.level = trace logger.structured_logging.appenderRefs = structured logger.structured_logging.appenderRef.structured.ref = structured # Tests that launch java subprocesses can set the "test.appender" system property to # "console" to avoid having the child process's logs overwrite the unit test's # log file. appender.console.type = Console appender.console.name = console appender.console.target = SYSTEM_ERR appender.console.layout.type = PatternLayout appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n # Ignore messages below warning level from Jetty, because it's a bit verbose logger.jetty.name = org.sparkproject.jetty logger.jetty.level = warn ================================================ FILE: spark/src/test/scala/io/delta/exceptions/DeltaConcurrentExceptionsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.exceptions import org.apache.spark.SparkFunSuite import org.apache.spark.sql.test.SharedSparkSession class DeltaConcurrentExceptionsSuite extends SparkFunSuite with SharedSparkSession { test("test ConcurrentWriteException") { intercept[org.apache.spark.sql.delta.DeltaConcurrentModificationException] { throw org.apache.spark.sql.delta.DeltaErrors.concurrentWriteException(None) } intercept[io.delta.exceptions.DeltaConcurrentModificationException] { throw org.apache.spark.sql.delta.DeltaErrors.concurrentWriteException(None) } intercept[org.apache.spark.sql.delta.ConcurrentWriteException] { throw org.apache.spark.sql.delta.DeltaErrors.concurrentWriteException(None) } intercept[org.apache.spark.sql.delta.DeltaConcurrentModificationException] { throw new org.apache.spark.sql.delta.ConcurrentWriteException(None) } intercept[io.delta.exceptions.DeltaConcurrentModificationException] { throw new org.apache.spark.sql.delta.ConcurrentWriteException(None) } } test("test MetadataChangedException") { intercept[org.apache.spark.sql.delta.DeltaConcurrentModificationException] { throw org.apache.spark.sql.delta.DeltaErrors.metadataChangedException(None) } intercept[io.delta.exceptions.DeltaConcurrentModificationException] { throw org.apache.spark.sql.delta.DeltaErrors.metadataChangedException(None) } intercept[org.apache.spark.sql.delta.MetadataChangedException] { throw org.apache.spark.sql.delta.DeltaErrors.metadataChangedException(None) } intercept[org.apache.spark.sql.delta.DeltaConcurrentModificationException] { throw new org.apache.spark.sql.delta.MetadataChangedException(None) } intercept[io.delta.exceptions.DeltaConcurrentModificationException] { throw new org.apache.spark.sql.delta.MetadataChangedException(None) } } test("test ProtocolChangedException") { intercept[org.apache.spark.sql.delta.DeltaConcurrentModificationException] { throw org.apache.spark.sql.delta.DeltaErrors.protocolChangedException(None) } intercept[io.delta.exceptions.DeltaConcurrentModificationException] { throw org.apache.spark.sql.delta.DeltaErrors.protocolChangedException(None) } intercept[org.apache.spark.sql.delta.ProtocolChangedException] { throw org.apache.spark.sql.delta.DeltaErrors.protocolChangedException(None) } intercept[org.apache.spark.sql.delta.DeltaConcurrentModificationException] { throw new org.apache.spark.sql.delta.ProtocolChangedException(None) } intercept[io.delta.exceptions.DeltaConcurrentModificationException] { throw new org.apache.spark.sql.delta.ProtocolChangedException(None) } } test("test ConcurrentAppendException") { intercept[org.apache.spark.sql.delta.DeltaConcurrentModificationException] { throw org.apache.spark.sql.delta.DeltaErrors. concurrentAppendException(None, "t", -1, partitionOpt = None) } intercept[io.delta.exceptions.DeltaConcurrentModificationException] { throw org.apache.spark.sql.delta.DeltaErrors. concurrentAppendException(None, "t", -1, partitionOpt = None) } intercept[org.apache.spark.sql.delta.ConcurrentAppendException] { throw org.apache.spark.sql.delta.DeltaErrors. concurrentAppendException(None, "t", -1, partitionOpt = None) } } test("test ConcurrentDeleteReadException") { intercept[org.apache.spark.sql.delta.DeltaConcurrentModificationException] { throw org.apache.spark.sql.delta.DeltaErrors. concurrentDeleteReadException(None, "t", -1, partitionOpt = None) } intercept[io.delta.exceptions.DeltaConcurrentModificationException] { throw org.apache.spark.sql.delta.DeltaErrors .concurrentDeleteReadException(None, "t", -1, partitionOpt = None) } intercept[org.apache.spark.sql.delta.ConcurrentDeleteReadException] { throw org.apache.spark.sql.delta.DeltaErrors .concurrentDeleteReadException(None, "t", -1, partitionOpt = None) } } test("test ConcurrentDeleteDeleteException") { intercept[org.apache.spark.sql.delta.DeltaConcurrentModificationException] { throw org.apache.spark.sql.delta.DeltaErrors .concurrentDeleteDeleteException(None, "t", -1, partitionOpt = None) } intercept[io.delta.exceptions.DeltaConcurrentModificationException] { throw org.apache.spark.sql.delta.DeltaErrors .concurrentDeleteDeleteException(None, "t", -1, partitionOpt = None) } intercept[org.apache.spark.sql.delta.ConcurrentDeleteDeleteException] { throw org.apache.spark.sql.delta.DeltaErrors .concurrentDeleteDeleteException(None, "t", -1, partitionOpt = None) } } test("test ConcurrentTransactionException") { intercept[org.apache.spark.sql.delta.DeltaConcurrentModificationException] { throw org.apache.spark.sql.delta.DeltaErrors.concurrentTransactionException(None) } intercept[io.delta.exceptions.DeltaConcurrentModificationException] { throw org.apache.spark.sql.delta.DeltaErrors.concurrentTransactionException(None) } intercept[org.apache.spark.sql.delta.ConcurrentTransactionException] { throw org.apache.spark.sql.delta.DeltaErrors.concurrentTransactionException(None) } intercept[org.apache.spark.sql.delta.DeltaConcurrentModificationException] { throw new org.apache.spark.sql.delta.ConcurrentTransactionException(None) } intercept[io.delta.exceptions.DeltaConcurrentModificationException] { throw new org.apache.spark.sql.delta.ConcurrentTransactionException(None) } } } ================================================ FILE: spark/src/test/scala/io/delta/sql/DeltaExtensionAndCatalogSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sql import java.nio.file.Files import org.apache.spark.sql.delta.{DeltaAnalysisException, DeltaLog} import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.delta.sources.DeltaSQLConf import io.delta.tables.DeltaTable import org.apache.commons.io.FileUtils import org.apache.hadoop.fs.Path import org.apache.spark.SparkFunSuite import org.apache.spark.network.util.JavaUtils import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf class DeltaExtensionAndCatalogSuite extends SparkFunSuite { private def createTempDir(): String = { val dir = Files.createTempDirectory("DeltaSparkSessionExtensionSuite").toFile FileUtils.forceDeleteOnExit(dir) dir.getCanonicalPath } private def verifyDeltaSQLParserIsActivated(spark: SparkSession): Unit = { val input = Files.createTempDirectory("DeltaSparkSessionExtensionSuite").toFile try { spark.range(1, 10).write.format("delta").save(input.getCanonicalPath) spark.sql(s"vacuum delta.`${input.getCanonicalPath}`") } finally { JavaUtils.deleteRecursively(input) } } test("activate Delta SQL parser using SQL conf") { val spark = SparkSession.builder() .appName("DeltaSparkSessionExtensionSuiteUsingSQLConf") .master("local[2]") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") .getOrCreate() try { verifyDeltaSQLParserIsActivated(spark) } finally { spark.close() } } test("activate Delta SQL parser using withExtensions") { val spark = SparkSession.builder() .appName("DeltaSparkSessionExtensionSuiteUsingWithExtensions") .master("local[2]") .withExtensions(new io.delta.sql.DeltaSparkSessionExtension) .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") .getOrCreate() try { verifyDeltaSQLParserIsActivated(spark) } finally { spark.close() } } test("DeltaCatalog class should be initialized correctly") { withSparkSession( SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key -> classOf[org.apache.spark.sql.delta.catalog.DeltaCatalog].getName ) { spark => val v2Catalog = spark.sessionState.analyzer.catalogManager.catalog("spark_catalog") assert(v2Catalog.isInstanceOf[org.apache.spark.sql.delta.catalog.DeltaCatalog]) } } test("DeltaLog should not throw exception if spark.sql.catalog.spark_catalog is set") { withTempDir { dir => withSparkSession( SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key -> classOf[org.apache.spark.sql.delta.catalog.DeltaCatalog].getName ) { spark => val path = new Path(dir.getCanonicalPath) assert(DeltaLog.forTable(spark, path).tableExists == false) } } } test("DeltaLog should throw exception if spark.sql.catalog.spark_catalog " + "config is not found") { withTempDir { dir => withSparkSession("" -> "") { spark => val path = new Path(dir.getCanonicalPath) val e = intercept[DeltaAnalysisException] { DeltaLog.forTable(spark, path) } assert(e.isInstanceOf[DeltaAnalysisException]) assert(e.getErrorClass() == "DELTA_CONFIGURE_SPARK_SESSION_WITH_EXTENSION_AND_CATALOG") } } } test("DeltaLog should not throw exception if spark.sql.catalog.spark_catalog " + "config is not found and the check is disabled") { withTempDir { dir => withSparkSession(DeltaSQLConf.DELTA_REQUIRED_SPARK_CONFS_CHECK.key -> "false") { spark => val path = new Path(dir.getCanonicalPath) DeltaLog.forTable(spark, path) assert(DeltaLog.forTable(spark, path).tableExists == false) } } } private def withSparkSession(configs: (String, String)*)(f: SparkSession => Unit): Unit = { var builder = SparkSession.builder() .appName("DeltaSparkSessionExtensionSuite") .master("local[2]") .config("spark.sql.warehouse.dir", createTempDir()) configs.foreach { c => builder = builder.config(c._1, c._2) } val spark = builder.getOrCreate() try { f(spark) } finally { spark.close() } } private def checkErrorMessage(f: => Unit): Unit = { val e = intercept[AnalysisException](f) val expectedStrs = Seq( "Delta operation requires the SparkSession to be configured", "spark.sql.extensions", s"${classOf[DeltaSparkSessionExtension].getName}", SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key, s"${classOf[DeltaCatalog].getName}" ) expectedStrs.foreach { m => assert(e.getMessage().contains(m), "full exception: " + e) } } } ================================================ FILE: spark/src/test/scala/io/delta/sql/parser/DeltaSqlParserSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sql.parser import io.delta.tables.execution.VacuumTableCommand import org.apache.spark.sql.delta.skipping.clustering.ClusteredTableUtils import org.apache.spark.sql.delta.skipping.clustering.temp.ClusterByTransform import org.apache.spark.sql.delta.CloneTableSQLTestUtils import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.{UnresolvedPathBasedDeltaTable, UnresolvedPathBasedTable} import org.apache.spark.sql.delta.commands.{DeltaOptimizeContext, DescribeDeltaDetailCommand, DescribeDeltaHistory, OptimizeTableCommand, DeltaReorgTable} import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.{TableIdentifier, TimeTravel} import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedRelation, UnresolvedTable} import org.apache.spark.sql.catalyst.expressions.Literal import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.catalyst.plans.SQLHelper import org.apache.spark.sql.catalyst.plans.logical.{AlterTableDropFeature, CloneTableStatement, CreateTable, CreateTableAsSelect, LogicalPlan, ReplaceTable, ReplaceTableAsSelect, RestoreTableStatement} import org.apache.spark.sql.execution.SparkSqlParser class DeltaSqlParserSuite extends SparkFunSuite with SQLHelper { test("isValidDecimal should recognize a table identifier and not treat them as a decimal") { // Setting `delegate` to `null` is fine. The following tests don't need to touch `delegate`. val parser = new DeltaSqlParser(null) assert(parser.parsePlan("vacuum 123_") === VacuumTableCommand(UnresolvedTable(Seq("123_"), "VACUUM"), None, None, None, false, None)) assert(parser.parsePlan("vacuum 1a.123_") === VacuumTableCommand(UnresolvedTable(Seq("1a", "123_"), "VACUUM"), None, None, None, false, None)) assert(parser.parsePlan("vacuum a.123A") === VacuumTableCommand(UnresolvedTable(Seq("a", "123A"), "VACUUM"), None, None, None, false, None)) assert(parser.parsePlan("vacuum a.123E3_column") === VacuumTableCommand(UnresolvedTable(Seq("a", "123E3_column"), "VACUUM"), None, None, None, false, None)) assert(parser.parsePlan("vacuum a.123D_column") === VacuumTableCommand(UnresolvedTable(Seq("a", "123D_column"), "VACUUM"), None, None, None, false, None)) assert(parser.parsePlan("vacuum a.123BD_column") === VacuumTableCommand(UnresolvedTable(Seq("a", "123BD_column"), "VACUUM"), None, None, None, false, None)) assert(parser.parsePlan("vacuum delta.`/tmp/table`") === VacuumTableCommand(UnresolvedTable(Seq("delta", "/tmp/table"), "VACUUM"), None, None, None, false, None)) assert(parser.parsePlan("vacuum \"/tmp/table\"") === VacuumTableCommand( UnresolvedPathBasedDeltaTable("/tmp/table", Map.empty, "VACUUM"), None, None, None, false, None)) } test("Restore command is parsed as expected") { val parser = new DeltaSqlParser(null) var parsedCmd = parser.parsePlan("RESTORE catalog_foo.db.tbl TO VERSION AS OF 1;") assert(parsedCmd === RestoreTableStatement(TimeTravel( UnresolvedRelation(Seq("catalog_foo", "db", "tbl")), None, Some(1), Some("sql")))) parsedCmd = parser.parsePlan("RESTORE delta.`/tmp` TO VERSION AS OF 1;") assert(parsedCmd === RestoreTableStatement(TimeTravel( UnresolvedRelation(Seq("delta", "/tmp")), None, Some(1), Some("sql")))) } test("OPTIMIZE command is parsed as expected") { val parser = new DeltaSqlParser(null) var parsedCmd = parser.parsePlan("OPTIMIZE tbl") assert(parsedCmd === OptimizeTableCommand(None, Some(tblId("tbl")), Nil)(Nil)) assert(parsedCmd.asInstanceOf[OptimizeTableCommand].child === UnresolvedTable(Seq("tbl"), "OPTIMIZE")) parsedCmd = parser.parsePlan("OPTIMIZE db.tbl") assert(parsedCmd === OptimizeTableCommand(None, Some(tblId("tbl", "db")), Nil)(Nil)) assert(parsedCmd.asInstanceOf[OptimizeTableCommand].child === UnresolvedTable(Seq("db", "tbl"), "OPTIMIZE")) parsedCmd = parser.parsePlan("OPTIMIZE catalog_foo.db.tbl") assert(parsedCmd === OptimizeTableCommand(None, Some(tblId("tbl", "db", "catalog_foo")), Nil)(Nil)) assert(parsedCmd.asInstanceOf[OptimizeTableCommand].child === UnresolvedTable(Seq("catalog_foo", "db", "tbl"), "OPTIMIZE")) assert(parser.parsePlan("OPTIMIZE tbl_${system:spark.testing}") === OptimizeTableCommand(None, Some(tblId("tbl_true")), Nil)(Nil)) withSQLConf("tbl_var" -> "tbl") { assert(parser.parsePlan("OPTIMIZE ${tbl_var}") === OptimizeTableCommand(None, Some(tblId("tbl")), Nil)(Nil)) assert(parser.parsePlan("OPTIMIZE ${spark:tbl_var}") === OptimizeTableCommand(None, Some(tblId("tbl")), Nil)(Nil)) assert(parser.parsePlan("OPTIMIZE ${sparkconf:tbl_var}") === OptimizeTableCommand(None, Some(tblId("tbl")), Nil)(Nil)) assert(parser.parsePlan("OPTIMIZE ${hiveconf:tbl_var}") === OptimizeTableCommand(None, Some(tblId("tbl")), Nil)(Nil)) assert(parser.parsePlan("OPTIMIZE ${hivevar:tbl_var}") === OptimizeTableCommand(None, Some(tblId("tbl")), Nil)(Nil)) } parsedCmd = parser.parsePlan("OPTIMIZE '/path/to/tbl'") assert(parsedCmd === OptimizeTableCommand(Some("/path/to/tbl"), None, Nil)(Nil)) assert(parsedCmd.asInstanceOf[OptimizeTableCommand].child === UnresolvedPathBasedDeltaTable("/path/to/tbl", Map.empty, "OPTIMIZE")) parsedCmd = parser.parsePlan("OPTIMIZE delta.`/path/to/tbl`") assert(parsedCmd === OptimizeTableCommand(None, Some(tblId("/path/to/tbl", "delta")), Nil)(Nil)) assert(parsedCmd.asInstanceOf[OptimizeTableCommand].child === UnresolvedTable(Seq("delta", "/path/to/tbl"), "OPTIMIZE")) assert(parser.parsePlan("OPTIMIZE tbl WHERE part = 1") === OptimizeTableCommand(None, Some(tblId("tbl")), Seq("part = 1"))(Nil)) assert(parser.parsePlan("OPTIMIZE tbl ZORDER BY (col1)") === OptimizeTableCommand(None, Some(tblId("tbl")), Nil) (Seq(unresolvedAttr("col1")))) assert(parser.parsePlan("OPTIMIZE tbl WHERE part = 1 ZORDER BY col1, col2.subcol") === OptimizeTableCommand(None, Some(tblId("tbl")), Seq("part = 1"))( Seq(unresolvedAttr("col1"), unresolvedAttr("col2", "subcol")))) assert(parser.parsePlan("OPTIMIZE tbl WHERE part = 1 ZORDER BY (col1, col2.subcol)") === OptimizeTableCommand(None, Some(tblId("tbl")), Seq("part = 1"))( Seq(unresolvedAttr("col1"), unresolvedAttr("col2", "subcol")))) // Validate OPTIMIZE works correctly with FULL keyword. parsedCmd = parser.parsePlan("OPTIMIZE tbl FULL") assert(parsedCmd === OptimizeTableCommand(None, Some(tblId("tbl")), Nil, DeltaOptimizeContext(isFull = true))(Nil)) assert(parsedCmd.asInstanceOf[OptimizeTableCommand].child === UnresolvedTable(Seq("tbl"), "OPTIMIZE")) parsedCmd = parser.parsePlan("OPTIMIZE catalog_foo.db.tbl FULL") assert(parsedCmd === OptimizeTableCommand( None, Some(tblId("tbl", "db", "catalog_foo")), Nil, DeltaOptimizeContext(isFull = true))(Nil)) assert(parsedCmd.asInstanceOf[OptimizeTableCommand].child === UnresolvedTable(Seq("catalog_foo", "db", "tbl"), "OPTIMIZE")) parsedCmd = parser.parsePlan("OPTIMIZE '/path/to/tbl' FULL") assert(parsedCmd === OptimizeTableCommand( Some("/path/to/tbl"), None, Nil, DeltaOptimizeContext(isFull = true))(Nil)) assert(parsedCmd.asInstanceOf[OptimizeTableCommand].child === UnresolvedPathBasedDeltaTable("/path/to/tbl", Map.empty, "OPTIMIZE")) parsedCmd = parser.parsePlan("OPTIMIZE delta.`/path/to/tbl` FULL") assert(parsedCmd === OptimizeTableCommand( None, Some(tblId("/path/to/tbl", "delta")), Nil, DeltaOptimizeContext(isFull = true))(Nil)) assert(parsedCmd.asInstanceOf[OptimizeTableCommand].child === UnresolvedTable(Seq("delta", "/path/to/tbl"), "OPTIMIZE")) } test("OPTIMIZE command new tokens are non-reserved keywords") { // new keywords: OPTIMIZE, ZORDER val parser = new DeltaSqlParser(null) // Use the new keywords in table name assert(parser.parsePlan("OPTIMIZE optimize") === OptimizeTableCommand(None, Some(tblId("optimize")), Nil)(Nil)) assert(parser.parsePlan("OPTIMIZE zorder") === OptimizeTableCommand(None, Some(tblId("zorder")), Nil)(Nil)) assert(parser.parsePlan("OPTIMIZE full") === OptimizeTableCommand(None, Some(tblId("full")), Nil)(Nil)) // Use the new keywords in column name assert(parser.parsePlan("OPTIMIZE tbl WHERE zorder = 1 and optimize = 2 and full = 3") === OptimizeTableCommand(None, Some(tblId("tbl")) , Seq("zorder = 1 and optimize = 2 and full = 3"))(Nil)) assert(parser.parsePlan("OPTIMIZE tbl ZORDER BY (optimize, zorder, full)") === OptimizeTableCommand(None, Some(tblId("tbl")), Nil)( Seq(unresolvedAttr("optimize"), unresolvedAttr("zorder"), unresolvedAttr("full")))) } test("DESCRIBE DETAIL command is parsed as expected") { val parser = new DeltaSqlParser(null) // Desc detail on a table assert(parser.parsePlan("DESCRIBE DETAIL catalog_foo.db.tbl") === DescribeDeltaDetailCommand( UnresolvedTable(Seq("catalog_foo", "db", "tbl"), DescribeDeltaDetailCommand.CMD_NAME), Map.empty)) // Desc detail on a raw path assert(parser.parsePlan("DESCRIBE DETAIL \"/tmp/table\"") === DescribeDeltaDetailCommand( UnresolvedPathBasedTable("/tmp/table", Map.empty, DescribeDeltaDetailCommand.CMD_NAME), Map.empty)) // Desc detail on a delta raw path assert(parser.parsePlan("DESCRIBE DETAIL delta.`dummy_raw_path`") === DescribeDeltaDetailCommand( UnresolvedTable(Seq("delta", "dummy_raw_path"), DescribeDeltaDetailCommand.CMD_NAME), Map.empty)) } test("DESCRIBE HISTORY command is parsed as expected") { val parser = new DeltaSqlParser(null) var parsedCmd = parser.parsePlan("DESCRIBE HISTORY catalog_foo.db.tbl") assert(parsedCmd.asInstanceOf[DescribeDeltaHistory].child === UnresolvedTable(Seq("catalog_foo", "db", "tbl"), DescribeDeltaHistory.COMMAND_NAME)) parsedCmd = parser.parsePlan("DESCRIBE HISTORY delta.`/path/to/tbl`") assert(parsedCmd.asInstanceOf[DescribeDeltaHistory].child === UnresolvedTable(Seq("delta", "/path/to/tbl"), DescribeDeltaHistory.COMMAND_NAME)) parsedCmd = parser.parsePlan("DESCRIBE HISTORY '/path/to/tbl'") assert(parsedCmd.asInstanceOf[DescribeDeltaHistory].child === UnresolvedPathBasedDeltaTable("/path/to/tbl", Map.empty, DescribeDeltaHistory.COMMAND_NAME)) } private def targetPlanForTable(tableParts: String*): UnresolvedTable = UnresolvedTable(tableParts.toSeq, "REORG") test("REORG command is parsed as expected") { val parser = new DeltaSqlParser(null) assert(parser.parsePlan("REORG TABLE tbl APPLY (PURGE)") === DeltaReorgTable(targetPlanForTable("tbl"))(Nil)) assert(parser.parsePlan("REORG TABLE tbl_${system:spark.testing} APPLY (PURGE)") === DeltaReorgTable(targetPlanForTable("tbl_true"))(Nil)) withSQLConf("tbl_var" -> "tbl") { assert(parser.parsePlan("REORG TABLE ${tbl_var} APPLY (PURGE)") === DeltaReorgTable(targetPlanForTable("tbl"))(Nil)) assert(parser.parsePlan("REORG TABLE ${spark:tbl_var} APPLY (PURGE)") === DeltaReorgTable(targetPlanForTable("tbl"))(Nil)) assert(parser.parsePlan("REORG TABLE ${sparkconf:tbl_var} APPLY (PURGE)") === DeltaReorgTable(targetPlanForTable("tbl"))(Nil)) assert(parser.parsePlan("REORG TABLE ${hiveconf:tbl_var} APPLY (PURGE)") === DeltaReorgTable(targetPlanForTable("tbl"))(Nil)) assert(parser.parsePlan("REORG TABLE ${hivevar:tbl_var} APPLY (PURGE)") === DeltaReorgTable(targetPlanForTable("tbl"))(Nil)) } assert(parser.parsePlan("REORG TABLE delta.`/path/to/tbl` APPLY (PURGE)") === DeltaReorgTable(targetPlanForTable("delta", "/path/to/tbl"))(Nil)) assert(parser.parsePlan("REORG TABLE tbl WHERE part = 1 APPLY (PURGE)") === DeltaReorgTable(targetPlanForTable("tbl"))(Seq("part = 1"))) } test("REORG command new tokens are non-reserved keywords") { // new keywords: REORG, APPLY, PURGE val parser = new DeltaSqlParser(null) // Use the new keywords in table name assert(parser.parsePlan("REORG TABLE reorg APPLY (PURGE)") === DeltaReorgTable(targetPlanForTable("reorg"))(Nil)) assert(parser.parsePlan("REORG TABLE apply APPLY (PURGE)") === DeltaReorgTable(targetPlanForTable("apply"))(Nil)) assert(parser.parsePlan("REORG TABLE purge APPLY (PURGE)") === DeltaReorgTable(targetPlanForTable("purge"))(Nil)) // Use the new keywords in column name assert(parser.parsePlan( "REORG TABLE tbl WHERE reorg = 1 AND apply = 2 AND purge = 3 APPLY (PURGE)") === DeltaReorgTable(targetPlanForTable("tbl"))(Seq("reorg = 1 AND apply =2 AND purge = 3"))) } // scalastyle:off argcount private def checkCloneStmt( parser: DeltaSqlParser, source: String, target: String, sourceFormat: String = "delta", sourceIsTable: Boolean = true, sourceIs3LTable: Boolean = false, targetIsTable: Boolean = true, targetLocation: Option[String] = None, versionAsOf: Option[Long] = None, timestampAsOf: Option[String] = None, isCreate: Boolean = true, isReplace: Boolean = false, tableProperties: Map[String, String] = Map.empty): Unit = { assert { parser.parsePlan(CloneTableSQLTestUtils.buildCloneSqlString( source, target, sourceIsTable, targetIsTable, sourceFormat, targetLocation = targetLocation, versionAsOf = versionAsOf, timestampAsOf = timestampAsOf, isCreate = isCreate, isReplace = isReplace, tableProperties = tableProperties )) == { val sourceRelation = if (sourceIs3LTable) { new UnresolvedRelation(source.split('.')) } else { UnresolvedRelation(tblId(source, if (sourceIsTable) null else sourceFormat)) } CloneTableStatement( if (versionAsOf.isEmpty && timestampAsOf.isEmpty) { sourceRelation } else { TimeTravel( sourceRelation, timestampAsOf.map(Literal(_)), versionAsOf, Some("sql")) }, new UnresolvedRelation(target.split('.')), ifNotExists = false, isReplaceCommand = isReplace, isCreateCommand = isCreate, tablePropertyOverrides = tableProperties, targetLocation = targetLocation ) } } } // scalastyle:on argcount test("CLONE command is parsed as expected") { val parser = new DeltaSqlParser(null) // Standard shallow clone checkCloneStmt(parser, source = "t1", target = "t1") // Path based source table checkCloneStmt(parser, source = "/path/to/t1", target = "t1", sourceIsTable = false) // REPLACE checkCloneStmt(parser, source = "t1", target = "t1", isCreate = false, isReplace = true) // CREATE OR REPLACE checkCloneStmt(parser, source = "t1", target = "t1", isCreate = true, isReplace = true) // Clone with table properties checkCloneStmt(parser, source = "t1", target = "t1", tableProperties = Map("a" -> "a")) // Clone with external location checkCloneStmt(parser, source = "t1", target = "t1", targetLocation = Some("/new/path")) // Clone with time travel checkCloneStmt(parser, source = "t1", target = "t1", versionAsOf = Some(1L)) // Clone with 3L table (only useful for Iceberg table now) checkCloneStmt(parser, source = "local.iceberg.table", target = "t1", sourceIs3LTable = true) checkCloneStmt(parser, source = "local.iceberg.table", target = "delta.table", sourceIs3LTable = true) // Custom source format with path checkCloneStmt(parser, source = "/path/to/iceberg", target = "t1", sourceFormat = "iceberg", sourceIsTable = false) // Target table with 3L name checkCloneStmt(parser, source = "/path/to/iceberg", target = "a.b.t1", sourceFormat = "iceberg", sourceIsTable = false) checkCloneStmt( parser, source = "spark_catalog.tmp.table", target = "a.b.t1", sourceIs3LTable = true) checkCloneStmt(parser, source = "t2", target = "a.b.t1") } for (truncateHistory <- Seq(true, false)) test(s"DROP FEATURE command is parsed as expected - truncateHistory: $truncateHistory") { val parser = new DeltaSqlParser(null) val table = "tbl" val featureName = "feature_name" val sql = s"ALTER TABLE $table DROP FEATURE $featureName " + (if (truncateHistory) "TRUNCATE HISTORY" else "") val parsedCmd = parser.parsePlan(sql) assert(parsedCmd === AlterTableDropFeature( UnresolvedTable(Seq(table), "ALTER TABLE ... DROP FEATURE"), featureName, truncateHistory)) } private def unresolvedAttr(colName: String*): UnresolvedAttribute = { new UnresolvedAttribute(colName) } private def tblId( tblName: String, schema: String = null, catalog: String = null): TableIdentifier = { if (catalog == null) { if (schema == null) new TableIdentifier(tblName) else new TableIdentifier(tblName, Some(schema)) } else { assert(schema != null) new TableIdentifier(tblName, Some(schema), Some(catalog)) } } private def clusterByStatement( createOrReplaceClause: String, asSelect: Boolean, schema: String, clusterByClause: String): String = { val tableSchema = if (asSelect) { "" } else { s"($schema)" } val select = if (asSelect) { "AS SELECT * FROM tbl2" } else { "" } s"$createOrReplaceClause TABLE tbl $tableSchema USING DELTA $clusterByClause $select" } private def validateClusterByTransform( clause: String, asSelect: Boolean, plan: LogicalPlan, expectedColumns: Seq[Seq[String]]): Unit = { val partitioning = if (clause == "CREATE") { if (asSelect) { plan.asInstanceOf[CreateTableAsSelect].partitioning } else { plan.asInstanceOf[CreateTable].partitioning } } else { if (asSelect) { plan.asInstanceOf[ReplaceTableAsSelect].partitioning } else { plan.asInstanceOf[ReplaceTable].partitioning } } assert(partitioning.size === 1) val transform = partitioning.head val actualColumns = transform match { case ClusterByTransform(columnNames) => columnNames.map(_.fieldNames.toSeq) case _ => assert(false, "Should not reach here") } assert(actualColumns === expectedColumns) } for (asSelect <- BOOLEAN_DOMAIN) { Seq("CREATE", "REPLACE").foreach { clause => test(s"CLUSTER BY - $clause TABLE asSelect = $asSelect") { val parser = new DeltaSqlParser(new SparkSqlParser()) val sql = clusterByStatement(clause, asSelect, "a int, b string", "CLUSTER BY (a)") val parsedPlan = parser.parsePlan(sql) validateClusterByTransform(clause, asSelect, parsedPlan, Seq(Seq("a"))) } test(s"CLUSTER BY nested column - $clause TABLE asSelect = $asSelect") { val parser = new DeltaSqlParser(new SparkSqlParser()) val sql = clusterByStatement(clause, asSelect, "a struct", "CLUSTER BY (a.b, a.c)") val parsedPlan = parser.parsePlan(sql) validateClusterByTransform(clause, asSelect, parsedPlan, Seq(Seq("a", "b"), Seq("a", "c"))) } test(s"CLUSTER BY backquoted column - $clause TABLE asSelect = $asSelect") { val parser = new DeltaSqlParser(new SparkSqlParser()) val sql = clusterByStatement(clause, asSelect, "`a.b.c` int", "CLUSTER BY (`a.b.c`)") val parsedPlan = parser.parsePlan(sql) validateClusterByTransform(clause, asSelect, parsedPlan, Seq(Seq("a.b.c"))) } test(s"CLUSTER BY comma column - $clause TABLE asSelect = $asSelect") { val parser = new DeltaSqlParser(new SparkSqlParser()) val sql = clusterByStatement(clause, asSelect, "`a,b` int", "CLUSTER BY (`a,b`)") val parsedPlan = parser.parsePlan(sql) validateClusterByTransform(clause, asSelect, parsedPlan, Seq(Seq("a,b"))) } test(s"CLUSTER BY duplicated clauses - $clause TABLE asSelect = $asSelect") { val parser = new DeltaSqlParser(new SparkSqlParser()) val sql = clusterByStatement(clause, asSelect, "a int, b string", "CLUSTER BY (a) CLUSTER BY (b)") checkError(intercept[ParseException] { parser.parsePlan(sql) }, "DUPLICATE_CLAUSES", parameters = Map("clauseName" -> "CLUSTER BY")) } test("CLUSTER BY set clustering column property is ignored - " + s"$clause TABLE asSelect = $asSelect") { val parser = new DeltaSqlParser(new SparkSqlParser()) val sql = clusterByStatement( clause, asSelect, "a int, b string", "CLUSTER BY (a) " + s"TBLPROPERTIES ('${ClusteredTableUtils.PROP_CLUSTERING_COLUMNS}' = 'b')") val parsedPlan = parser.parsePlan(sql) validateClusterByTransform(clause, asSelect, parsedPlan, Seq(Seq("a"))) } test(s"CLUSTER BY with PARTITIONED BY - $clause TABLE asSelect = $asSelect") { val parser = new DeltaSqlParser(new SparkSqlParser()) val sql = clusterByStatement( clause, asSelect, "a int, b string", "CLUSTER BY (a) PARTITIONED BY (b)") val errorMsg = "Clustering and partitioning cannot both be specified. " + "Please remove PARTITIONED BY if you want to create a Delta table with clustering" checkError(intercept[ParseException] { parser.parsePlan(sql) }, "_LEGACY_ERROR_TEMP_0035", parameters = Map("message" -> errorMsg)) } test(s"CLUSTER BY with bucketing - $clause TABLE asSelect = $asSelect") { val parser = new DeltaSqlParser(new SparkSqlParser()) val sql = clusterByStatement( clause, asSelect, "a int, b string", "CLUSTER BY (a) CLUSTERED BY (b) INTO 2 BUCKETS") val errorMsg = "Clustering and bucketing cannot both be specified. " + "Please remove CLUSTERED BY INTO BUCKETS if you " + "want to create a Delta table with clustering" checkError(intercept[ParseException] { parser.parsePlan(sql) }, "_LEGACY_ERROR_TEMP_0035", parameters = Map("message" -> errorMsg)) } } } test("string coalescing") { val parser = new DeltaSqlParser(new SparkSqlParser()) val pathToTable = "/path/to/table" val partedPathes = Seq( "'/path/to/table'", "'/path/to' '/table'", "'/path' '/to' '/table'" ) partedPathes.foreach { path => // CLONE LOCATION val cloneCmd = parser.parsePlan( s"CREATE TABLE t1 SHALLOW CLONE source LOCATION $path") assert(cloneCmd.asInstanceOf[CloneTableStatement].targetLocation === Some(pathToTable)) // OPTIMIZE val optimizeCmd = parser.parsePlan(s"OPTIMIZE $path") assert(optimizeCmd === OptimizeTableCommand(Some(pathToTable), None, Nil)(Nil)) // DESCRIBE HISTORY var describeHistoryCmd = parser.parsePlan(s"DESCRIBE HISTORY $path") assert(describeHistoryCmd.asInstanceOf[DescribeDeltaHistory].child === UnresolvedPathBasedDeltaTable(pathToTable, Map.empty, DescribeDeltaHistory.COMMAND_NAME)) // DESCRIBE DETAIL val describeDetailCmd = parser.parsePlan(s"DESCRIBE DETAIL $path") assert(describeDetailCmd === DescribeDeltaDetailCommand( UnresolvedPathBasedTable(pathToTable, Map.empty, DescribeDeltaDetailCommand.CMD_NAME), Map.empty)) // VACUUM val vacuumCmd = parser.parsePlan(s"VACUUM $path") assert(vacuumCmd === VacuumTableCommand( UnresolvedPathBasedDeltaTable(pathToTable, Map.empty, "VACUUM"), None, None, None, false, None)) } } } ================================================ FILE: spark/src/test/scala/io/delta/tables/DeltaTableBuilderSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.skipping.ClusteredTableTestUtils import org.apache.spark.sql.delta.sources.DeltaSourceUtils.GENERATION_EXPRESSION_METADATA_KEY import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.{AnalysisException, QueryTest} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{IntegerType, LongType, MetadataBuilder, StringType, StructType} class DeltaTableBuilderSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with ClusteredTableTestUtils { // Define the information for a default test table used by many tests. protected val defaultTestTableSchema = "c1 int, c2 int, c3 string" protected val defaultTestTableGeneratedColumns = Map("c2" -> "c1 + 10") protected val defaultTestTablePartitionColumns = Seq("c1") protected val defaultTestTableColumnComments = Map("c1" -> "foo", "c3" -> "bar") protected val defaultTestTableComment = "tbl comment" protected val defaultTestTableNullableCols = Set("c1", "c3") protected val defaultTestTableProperty = ("foo", "bar") /** * Verify if the table metadata matches the test table. We use this to verify DDLs * write correct table metadata into the transaction logs. */ protected def verifyTestTableMetadata( table: String, schemaString: String, generatedColumns: Map[String, String] = Map.empty, colComments: Map[String, String] = Map.empty, colNullables: Set[String] = Set.empty, tableComment: Option[String] = None, partitionCols: Seq[String] = Seq.empty, tableProperty: Option[(String, String)] = None ): Unit = { val deltaLog = if (table.startsWith("delta.")) { DeltaLog.forTable(spark, table.stripPrefix("delta.`").stripSuffix("`")) } else { DeltaLog.forTable(spark, TableIdentifier(table)) } val schema = StructType.fromDDL(schemaString) val expectedSchema = StructType(schema.map { field => val newMetadata = new MetadataBuilder() .withMetadata(field.metadata) if (generatedColumns.contains(field.name)) { newMetadata.putString(GENERATION_EXPRESSION_METADATA_KEY, generatedColumns(field.name)) } if (colComments.contains(field.name)) { newMetadata.putString("comment", colComments(field.name)) } field.copy( nullable = colNullables.contains(field.name), metadata = newMetadata.build) }) val metadata = deltaLog.snapshot.metadata assert(metadata.schema == expectedSchema) assert(metadata.partitionColumns == partitionCols) if (tableProperty.nonEmpty) { assert(metadata.configuration(tableProperty.get._1).contentEquals(tableProperty.get._2)) } if (tableComment.nonEmpty) { assert(metadata.description.contentEquals(tableComment.get)) } } protected def testCreateTable(testName: String)(createFunc: String => Unit): Unit = { test(testName) { withTable(testName) { createFunc(testName) verifyTestTableMetadata( testName, defaultTestTableSchema, defaultTestTableGeneratedColumns, defaultTestTableColumnComments, defaultTestTableNullableCols, Some(defaultTestTableComment), defaultTestTablePartitionColumns, Some(defaultTestTableProperty) ) } } } protected def testCreateTableWithNameAndLocation( testName: String)(createFunc: (String, String) => Unit): Unit = { test(testName + ": external - with location and name") { withTempPath { path => withTable(testName) { createFunc(testName, path.getCanonicalPath) verifyTestTableMetadata( testName, defaultTestTableSchema, defaultTestTableGeneratedColumns, defaultTestTableColumnComments, defaultTestTableNullableCols, Some(defaultTestTableComment), defaultTestTablePartitionColumns, Some(defaultTestTableProperty) ) } } } } protected def testCreateTableWithLocationOnly( testName: String)(createFunc: String => Unit): Unit = { test(testName + ": external - location only") { withTempPath { path => withTable(testName) { createFunc(path.getCanonicalPath) verifyTestTableMetadata( s"delta.`${path.getCanonicalPath}`", defaultTestTableSchema, defaultTestTableGeneratedColumns, defaultTestTableColumnComments, defaultTestTableNullableCols, Some(defaultTestTableComment), defaultTestTablePartitionColumns, Some(defaultTestTableProperty) ) } } } } def defaultCreateTableBuilder( ifNotExists: Boolean, tableName: Option[String] = None, location: Option[String] = None): DeltaTableBuilder = { val tableBuilder = if (ifNotExists) { io.delta.tables.DeltaTable.createIfNotExists() } else { io.delta.tables.DeltaTable.create() } defaultTableBuilder(tableBuilder, tableName, location) } def defaultReplaceTableBuilder( orCreate: Boolean, tableName: Option[String] = None, location: Option[String] = None): DeltaTableBuilder = { var tableBuilder = if (orCreate) { io.delta.tables.DeltaTable.createOrReplace() } else { io.delta.tables.DeltaTable.replace() } defaultTableBuilder(tableBuilder, tableName, location) } private def defaultTableBuilder( builder: DeltaTableBuilder, tableName: Option[String], location: Option[String] ) = { var tableBuilder = builder if (tableName.nonEmpty) { tableBuilder = tableBuilder.tableName(tableName.get) } if (location.nonEmpty) { tableBuilder = tableBuilder.location(location.get) } tableBuilder.addColumn( io.delta.tables.DeltaTable.columnBuilder("c1").dataType("int").nullable(true).comment("foo") .build() ) tableBuilder.addColumn( io.delta.tables.DeltaTable.columnBuilder("c2").dataType("int") .nullable(false).generatedAlwaysAs("c1 + 10").build() ) tableBuilder.addColumn( io.delta.tables.DeltaTable.columnBuilder("c3").dataType("string").comment("bar").build() ) tableBuilder.partitionedBy("c1") tableBuilder.property("foo", "bar") tableBuilder.comment("tbl comment") tableBuilder } test("create table with existing schema and extra column") { withTable("table") { withTempDir { dir => spark.range(10).toDF("key").write.format("parquet").saveAsTable("table") val existingSchema = spark.read.format("parquet").table("table").schema io.delta.tables.DeltaTable.create() .location(dir.getAbsolutePath) .addColumns(existingSchema) .addColumn("value", "string", false) .execute() verifyTestTableMetadata(s"delta.`${dir.getAbsolutePath}`", "key bigint, value string", colNullables = Set("key")) } } } test("create table with variation of addColumns - with spark session") { withTable("test") { io.delta.tables.DeltaTable.create(spark) .tableName("test") .addColumn("c1", "int") .addColumn("c2", IntegerType) .addColumn("c3", "string", false) .addColumn("c4", StringType, true) .addColumn( io.delta.tables.DeltaTable.columnBuilder(spark, "c5") .dataType("bigint") .comment("foo") .nullable(false) .build ) .addColumn( io.delta.tables.DeltaTable.columnBuilder(spark, "c6") .dataType(LongType) .generatedAlwaysAs("c5 + 10") .build ).execute() verifyTestTableMetadata( "test", "c1 int, c2 int, c3 string, c4 string, c5 bigint, c6 bigint", generatedColumns = Map("c6" -> "c5 + 10"), colComments = Map("c5" -> "foo"), colNullables = Set("c1", "c2", "c4", "c6") ) } } test("test addColumn using columnBuilder, without dataType") { val e = intercept[AnalysisException] { DeltaTable.columnBuilder("value") .generatedAlwaysAs("true") .nullable(true) .build() } assert(e.getMessage.contains("The data type of the column `value` was not provided")) } testCreateTable("create_table") { table => defaultCreateTableBuilder(ifNotExists = false, Some(table)).execute() } testCreateTableWithNameAndLocation("create_table") { (name, path) => defaultCreateTableBuilder(ifNotExists = false, Some(name), Some(path)).execute() } testCreateTableWithLocationOnly("create_table") { path => defaultCreateTableBuilder(ifNotExists = false, location = Some(path)).execute() } test("create table - errors if already exists") { withTable("testTable") { sql(s"CREATE TABLE testTable (c1 int) USING DELTA") intercept[TableAlreadyExistsException] { defaultCreateTableBuilder(ifNotExists = false, Some("testTable")).execute() } } } test("create table - ignore if already exists") { withTable("testTable") { sql(s"CREATE TABLE testTable (c1 int) USING DELTA") defaultCreateTableBuilder(ifNotExists = true, Some("testTable")).execute() verifyTestTableMetadata("testTable", "c1 int", colNullables = Set("c1")) } } testCreateTable("create_table_if_not_exists") { table => defaultCreateTableBuilder(ifNotExists = true, Some(table)).execute() } testCreateTableWithNameAndLocation("create_table_if_not_exists") { (name, path) => defaultCreateTableBuilder(ifNotExists = true, Some(name), Some(path)).execute() } testCreateTableWithLocationOnly("create_table_if_not_exists") { path => defaultCreateTableBuilder(ifNotExists = true, location = Some(path)).execute() } test("replace table - errors if not exists") { intercept[AnalysisException] { defaultReplaceTableBuilder(orCreate = false, Some("testTable")).execute() } } testCreateTable("replace_table") { table => sql(s"CREATE TABLE replace_table(c1 int) USING DELTA") defaultReplaceTableBuilder(orCreate = false, Some(table)).execute() } testCreateTableWithNameAndLocation("replace_table") { (name, path) => sql(s"CREATE TABLE $name (c1 int) USING DELTA LOCATION '$path'") defaultReplaceTableBuilder(orCreate = false, Some(name), Some(path)).execute() } testCreateTableWithLocationOnly("replace_table") { path => sql(s"CREATE TABLE delta.`$path` (c1 int) USING DELTA") defaultReplaceTableBuilder(orCreate = false, location = Some(path)).execute() } testCreateTable("replace_or_create_table") { table => defaultReplaceTableBuilder(orCreate = true, Some(table)).execute() } testCreateTableWithNameAndLocation("replace_or_create_table") { (name, path) => defaultReplaceTableBuilder(orCreate = true, Some(name), Some(path)).execute() } testCreateTableWithLocationOnly("replace_or_create_table") { path => defaultReplaceTableBuilder(orCreate = true, location = Some(path)).execute() } test("test no identifier and no location") { val e = intercept[AnalysisException] { io.delta.tables.DeltaTable.create().addColumn("c1", "int").execute() } assert(e.getMessage.contains("Table name or location has to be specified")) } test("partitionedBy only should contain columns in the schema") { val e = intercept[AnalysisException] { io.delta.tables.DeltaTable.create().tableName("testTable") .addColumn("c1", "int") .partitionedBy("c2") .execute() } assert(e.getMessage.startsWith("Couldn't find column c2")) } test("errors if table name and location are different paths") { withTempDir { dir => val path = dir.getAbsolutePath val e = intercept[AnalysisException] { io.delta.tables.DeltaTable.create().tableName(s"delta.`$path`") .addColumn("c1", "int") .location("src/test/resources/delta/dbr_8_0_non_generated_columns") .execute() } assert(e.getMessage.contains( "Creating path-based Delta table with a different location isn't supported.")) } } test("table name and location are the same") { withTempDir { dir => val path = dir.getAbsolutePath io.delta.tables.DeltaTable.create().tableName(s"delta.`$path`") .addColumn("c1", "int") .location(path) .execute() } } test("errors if use parquet path as identifier") { withTempDir { dir => val path = dir.getAbsolutePath val e = intercept[AnalysisException] { io.delta.tables.DeltaTable.create().tableName(s"parquet.`$path`") .addColumn("c1", "int") .location(path) .execute() } assert(e.getMessage == "Database 'main.parquet' not found" || e.getMessage == "Database 'parquet' not found" || e.getMessage.contains("is not a valid name") || e.getMessage.contains("schema `parquet` cannot be found") ) } } test("delta table property case") { sealed trait DeltaTablePropertySetOperation { val preservedCaseConfig = Map("delta.appendOnly" -> "true", "Foo" -> "Bar", "foo" -> "Bar") val lowerCaseEnforcedConfig = Map("delta.appendOnly" -> "true", "foo" -> "Bar") def setTableProperty(tablePath: String): Unit def expectedConfig: Map[String, String] def description: String } trait CasePreservingTablePropertySetOperation extends DeltaTablePropertySetOperation { val expectedConfig = preservedCaseConfig } case object SetPropertyThroughCreate extends CasePreservingTablePropertySetOperation { def setTableProperty(tablePath: String): Unit = sql( s"CREATE TABLE delta.`$tablePath`(id INT) " + s"USING delta TBLPROPERTIES('delta.appendOnly'='true', 'Foo'='Bar', 'foo'='Bar' ) " ) val description = "Setting Table Property at Table Creation" } case object SetPropertyThroughAlter extends CasePreservingTablePropertySetOperation { def setTableProperty(tablePath: String): Unit = { spark.range(1, 10).write.format("delta").save(tablePath) sql(s"ALTER TABLE delta.`$tablePath` " + s"SET TBLPROPERTIES('delta.appendOnly'='true', 'Foo'='Bar', 'foo'='Bar')") } val description = "Setting Table Property via Table Alter" } case class SetPropertyThroughTableBuilder(backwardCompatible: Boolean) extends DeltaTablePropertySetOperation { def setTableProperty(tablePath: String): Unit = { withSQLConf(DeltaSQLConf.TABLE_BUILDER_FORCE_TABLEPROPERTY_LOWERCASE.key -> backwardCompatible.toString) { DeltaTable.create() .location(tablePath) .property("delta.appendOnly", "true") .property("Foo", "Bar") .property("foo", "Bar") .execute() } } override def expectedConfig : Map[String, String] = { if (backwardCompatible) { lowerCaseEnforcedConfig } else { preservedCaseConfig } } val description = s"Setting Table Property on DeltaTableBuilder." + s" Backward compatible enabled = ${backwardCompatible}" } val examples = Seq( SetPropertyThroughCreate, SetPropertyThroughAlter, SetPropertyThroughTableBuilder(backwardCompatible = true), SetPropertyThroughTableBuilder(backwardCompatible = false) ) for (example <- examples) { withClue(example.description) { withTempDir { dir => val path = dir.getCanonicalPath() example.setTableProperty(path) val config = DeltaLog.forTable(spark, path).snapshot.metadata.configuration assert( config == example.expectedConfig, s"$example's result is not correct: $config") } } } } test("create table with clustering") { withSQLConf( // Enable update catalog for verifyClusteringColumns. DeltaSQLConf.DELTA_UPDATE_CATALOG_ENABLED.key -> "true") { withTable("test") { io.delta.tables.DeltaTable.create().tableName("test") .addColumn("c1", "int") .clusterBy("c1") .execute() val deltaLog = DeltaLog.forTable(spark, TableIdentifier("test")) val metadata = deltaLog.snapshot.metadata verifyClusteringColumns(TableIdentifier("test"), Seq("c1")) } } } test("errors if partition and cluster columns are provided") { withTable("test") { val e = intercept[AnalysisException] { io.delta.tables.DeltaTable.create().tableName("test") .addColumn("c1", "int") .clusterBy("c1") .partitionedBy("c1") .execute() } checkError(e, "DELTA_CLUSTER_BY_WITH_PARTITIONED_BY") } } } ================================================ FILE: spark/src/test/scala/io/delta/tables/DeltaTableForNameSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables import scala.collection.mutable // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DummyCatalogWithNamespace} import org.apache.commons.io.FileUtils import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.sql.{AnalysisException, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.connector.catalog.CatalogNotFoundException import org.apache.spark.sql.test.SharedSparkSession class DeltaTableForNameSuite extends QueryTest with DeltaSQLCommandTest with SharedSparkSession { private val sparkCatalog = "spark_catalog" private val catalogName = "test_catalog" private val defaultSchema = "default" private val nonDefaultSchema = "non_default" private val commonTblName = "tbl" private val defaultSchemaUniqueTbl = "default_tbl" private val nonDefaultSchemaUniqueTbl = "non_default_tbl" private val nonSessionCatalogNonDefaultSchema = "non_default_session_schema" private val tableNameToId = mutable.Map.empty[String, String] override def sparkConf: SparkConf = super.sparkConf .set(s"spark.sql.catalog.$catalogName", classOf[DummyCatalogWithNamespace].getName) override def beforeAll(): Unit = { super.beforeAll() // General Test setup sql(s"CREATE SCHEMA $nonDefaultSchema;") sql(s"CREATE SCHEMA $catalogName.$defaultSchema;") sql(s"CREATE SCHEMA $catalogName.$nonSessionCatalogNonDefaultSchema;") // Setup custom catalog default schema tables sql(s"SET CATALOG $catalogName") setUpTable(catalogName, defaultSchema, commonTblName) setUpTable(catalogName, defaultSchema, defaultSchemaUniqueTbl) // Setup custom catalog non default schema tables setUpTable(catalogName, nonSessionCatalogNonDefaultSchema, commonTblName) setUpTable(catalogName, nonSessionCatalogNonDefaultSchema, nonDefaultSchemaUniqueTbl) // Setup session catalog default schema tables sql(s"SET CATALOG $sparkCatalog") setUpTable(sparkCatalog, defaultSchema, commonTblName) setUpTable(sparkCatalog, defaultSchema, defaultSchemaUniqueTbl) // Setup session catalog non default schema tables setUpTable(sparkCatalog, nonDefaultSchema, commonTblName) setUpTable(sparkCatalog, nonDefaultSchema, nonDefaultSchemaUniqueTbl) } override def afterAll(): Unit = { sql(s"DROP SCHEMA $sparkCatalog.$nonDefaultSchema CASCADE;") sql(s"DROP SCHEMA $catalogName.$defaultSchema CASCADE;") sql(s"DROP SCHEMA $catalogName.$nonSessionCatalogNonDefaultSchema CASCADE;") super.afterAll() } protected override def beforeEach(): Unit = { super.beforeEach() sql(s"SET CATALOG $sparkCatalog") } private def getTablePath(tableName: String): Path = { new Path(DummyCatalogWithNamespace.catalogDir + s"/$tableName") } private def setUpTable(catalog: String, schema: String, table: String): Unit = { val tableName = s"$catalog.$schema.$table" val path = getTablePath(tableName) sql(s"CREATE OR REPLACE TABLE $tableName (id int) USING delta LOCATION '$path';") tableNameToId += (tableName -> DeltaLog.forTable(spark, getTablePath(tableName)).tableId) } private def validateForNameTableId( tableName: String, expectedResult: Option[String] = None): Unit = { val table = DeltaTable.forName(spark, tableName) checkAnswer(table.detail().select("id"), Seq(Row(expectedResult.getOrElse( tableNameToId(tableName) )))) } test(s"forName resolves fully qualified tables in session catalog correctly") { validateForNameTableId(s"$sparkCatalog.$defaultSchema.$commonTblName") validateForNameTableId(s"$sparkCatalog.$defaultSchema.$defaultSchemaUniqueTbl") validateForNameTableId(s"$sparkCatalog.$nonDefaultSchema.$commonTblName") validateForNameTableId(s"$sparkCatalog.$nonDefaultSchema.$nonDefaultSchemaUniqueTbl") } test(s"forName resolves partially qualified tables in session catalog correctly") { validateForNameTableId(s"$defaultSchema.$commonTblName", Some(tableNameToId(s"$sparkCatalog.$defaultSchema.$commonTblName"))) validateForNameTableId(s"$defaultSchema.$defaultSchemaUniqueTbl", Some(tableNameToId(s"$sparkCatalog.$defaultSchema.$defaultSchemaUniqueTbl"))) validateForNameTableId(s"$nonDefaultSchema.$commonTblName", Some(tableNameToId(s"$sparkCatalog.$nonDefaultSchema.$commonTblName"))) validateForNameTableId(s"$nonDefaultSchema.$nonDefaultSchemaUniqueTbl", Some(tableNameToId(s"$sparkCatalog.$nonDefaultSchema.$nonDefaultSchemaUniqueTbl"))) } test(s"forName resolves fully qualified tables in non session catalog correctly") { validateForNameTableId(s"$catalogName.$defaultSchema.$commonTblName") validateForNameTableId(s"$catalogName.$defaultSchema.$defaultSchemaUniqueTbl") validateForNameTableId(s"$catalogName.$nonSessionCatalogNonDefaultSchema.$commonTblName") validateForNameTableId( s"$catalogName.$nonSessionCatalogNonDefaultSchema.$nonDefaultSchemaUniqueTbl") } for (table <- Seq(commonTblName, nonDefaultSchemaUniqueTbl)) test(s"forName fails for partially " + s"qualified tables in non session catalog with table=$table") { sql(s"SET CATALOG $catalogName") val e = intercept[AnalysisException] { DeltaTable.forName(spark, s"$nonSessionCatalogNonDefaultSchema.$table") } checkError(exception = e, "DELTA_MISSING_DELTA_TABLE", parameters = Map("tableName" -> s"`$nonSessionCatalogNonDefaultSchema`.`$table`")) } // forName currently doesn't resolve unqualified tables correctly for non session catalogs. // in this test, it resolves to the table `spark_catalog.default.tbl` but it adds an incorrect // identifier on top. test(s"forName resolves partially qualified tables in non session catalog incorrectly") { sql(s"SET CATALOG $catalogName") validateForNameTableId(s"$defaultSchema.$commonTblName", Some(tableNameToId(s"$catalogName.$defaultSchema.$commonTblName"))) validateForNameTableId(s"$defaultSchema.$defaultSchemaUniqueTbl", Some(tableNameToId(s"$catalogName.$defaultSchema.$defaultSchemaUniqueTbl"))) } test("forName with invalid non session catalog") { intercept[CatalogNotFoundException] { DeltaTable.forName(spark, "invalid_catalog.default.tbl") } } test("forName with unqualified non session catalog") { sql(s"SET CATALOG $sparkCatalog") val e = intercept[AnalysisException] { DeltaTable.forName(spark, s"$nonSessionCatalogNonDefaultSchema.$commonTblName") } checkError(exception = e, "DELTA_MISSING_DELTA_TABLE", parameters = Map("tableName" -> s"`$nonSessionCatalogNonDefaultSchema`.`$commonTblName`")) } test("forName fails with fully qualified non existent table") { val e = intercept[AnalysisException] { DeltaTable.forName(spark, s"$catalogName.$defaultSchema.invalid_table") } checkError(exception = e, "TABLE_OR_VIEW_NOT_FOUND", parameters = Map("relationName" -> s"`$catalogName`.`$defaultSchema`.`invalid_table`")) } } ================================================ FILE: spark/src/test/scala/io/delta/tables/DeltaTableSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables import java.io.File import java.sql.Timestamp import java.util.Locale import scala.concurrent.duration._ import scala.language.postfixOps // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.{AppendOnlyTableFeature, DeltaIllegalArgumentException, DeltaLog, DeltaTableFeatureException, FakeFileSystem, InvariantsTableFeature, TestReaderWriterFeature, TestRemovableReaderWriterFeature, TestRemovableWriterFeature, TestWriterFeature} import org.apache.spark.sql.delta.actions.{ Metadata, Protocol } import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.storage.LocalLogStore import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.FileNames import org.apache.commons.io.FileUtils import org.apache.hadoop.fs.{Path, UnsupportedFileSystemException} import org.apache.spark.SparkException import org.apache.spark.network.util.JavaUtils import org.apache.spark.sql.{functions, AnalysisException, DataFrame, Dataset, QueryTest, Row} import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.Utils class DeltaTableSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { test("forPath") { withTempDir { dir => testData.write.format("delta").save(dir.getAbsolutePath) checkAnswer( DeltaTable.forPath(spark, dir.getAbsolutePath).toDF, testData.collect().toSeq) checkAnswer( DeltaTable.forPath(dir.getAbsolutePath).toDF, testData.collect().toSeq) } } test("forPath - with non-Delta table path") { val msg = "not a delta table" withTempDir { dir => testData.write.format("parquet").mode("overwrite").save(dir.getAbsolutePath) testError(msg) { DeltaTable.forPath(spark, dir.getAbsolutePath) } testError(msg) { DeltaTable.forPath(dir.getAbsolutePath) } } } test("forName") { withTempDir { dir => withTable("deltaTable") { testData.write.format("delta").saveAsTable("deltaTable") checkAnswer( DeltaTable.forName(spark, "deltaTable").toDF, testData.collect().toSeq) checkAnswer( DeltaTable.forName("deltaTable").toDF, testData.collect().toSeq) } } } def testForNameOnNonDeltaName(tableName: String): Unit = { val msg = "not a Delta table" testError(msg) { DeltaTable.forName(spark, tableName) } testError(msg) { DeltaTable.forName(tableName) } } test("forName - with non-Delta table name") { withTempDir { dir => withTable("notADeltaTable") { testData.write.format("parquet").mode("overwrite").saveAsTable("notADeltaTable") testForNameOnNonDeltaName("notADeltaTable") } } } test("forName - with temp view name") { withTempDir { dir => withTempView("viewOnDeltaTable") { testData.write.format("delta").save(dir.getAbsolutePath) spark.read.format("delta").load(dir.getAbsolutePath) .createOrReplaceTempView("viewOnDeltaTable") testForNameOnNonDeltaName("viewOnDeltaTable") } } } test("forName - with delta.`path`") { // for name should work on Delta table paths withTempDir { dir => testData.write.format("delta").save(dir.getAbsolutePath) checkAnswer( DeltaTable.forName(spark, s"delta.`$dir`").toDF, testData.collect().toSeq) checkAnswer( DeltaTable.forName(s"delta.`$dir`").toDF, testData.collect().toSeq) } // using forName on non Delta Table paths should fail withTempDir { dir => testForNameOnNonDeltaName(s"delta.`$dir`") testData.write.format("parquet").mode("overwrite").save(dir.getAbsolutePath) testForNameOnNonDeltaName(s"delta.`$dir`") } } test("as") { withTempDir { dir => testData.write.format("delta").save(dir.getAbsolutePath) checkAnswer( DeltaTable.forPath(dir.getAbsolutePath).as("tbl").toDF.select("tbl.value"), testData.select("value").collect().toSeq) } } test("isDeltaTable - path - with _delta_log dir") { withTempDir { dir => testData.write.format("delta").save(dir.getAbsolutePath) assert(DeltaTable.isDeltaTable(dir.getAbsolutePath)) } } test("isDeltaTable - path - with empty _delta_log dir") { withTempDir { dir => new File(dir, "_delta_log").mkdirs() assert(!DeltaTable.isDeltaTable(dir.getAbsolutePath)) } } test("isDeltaTable - path - with no _delta_log dir") { withTempDir { dir => assert(!DeltaTable.isDeltaTable(dir.getAbsolutePath)) } } test("isDeltaTable - path - with non-existent dir") { withTempDir { dir => JavaUtils.deleteRecursively(dir) assert(!DeltaTable.isDeltaTable(dir.getAbsolutePath)) } } test("isDeltaTable - with non-Delta table path") { withTempDir { dir => testData.write.format("parquet").mode("overwrite").save(dir.getAbsolutePath) assert(!DeltaTable.isDeltaTable(dir.getAbsolutePath)) } } def testError(expectedMsg: String)(thunk: => Unit): Unit = { val e = intercept[AnalysisException] { thunk } assert(e.getMessage.toLowerCase(Locale.ROOT).contains(expectedMsg.toLowerCase(Locale.ROOT))) } test("DeltaTable is Java Serializable but cannot be used in executors") { import testImplicits._ // DeltaTable can be passed to executor without method calls. withTempDir { dir => testData.write.format("delta").mode("append").save(dir.getAbsolutePath) val dt: DeltaTable = DeltaTable.forPath(dir.getAbsolutePath) spark.range(5).as[Long].map{ row: Long => val foo = dt row + 3 }.count() } // DeltaTable can be passed to executor but method call causes exception. val e = intercept[Exception] { withTempDir { dir => testData.write.format("delta").mode("append").save(dir.getAbsolutePath) val dt: DeltaTable = DeltaTable.forPath(dir.getAbsolutePath) spark.range(5).as[Long].map{ row: Long => dt.toDF row + 3 }.count() } }.getMessage assert(e.contains("DeltaTable cannot be used in executors")) } } class DeltaTableHadoopOptionsSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { import testImplicits._ protected override def sparkConf = { // The drop feature test below is targeting the drop feature with history truncation // implementation. The fast drop feature implementation adds a new writer feature when dropping // a feature and also does not require any waiting time. The fast drop feature implementation // is tested extensively in the DeltaFastDropFeatureSuite. super.sparkConf .set(DeltaSQLConf.FAST_DROP_FEATURE_ENABLED.key, "false") .set("spark.delta.logStore.fake.impl", classOf[LocalLogStore].getName) } /** * Create Hadoop file system options for `FakeFileSystem`. If Delta doesn't pick up them, * it won't be able to read/write any files using `fake://`. */ private def fakeFileSystemOptions: Map[String, String] = { Map( "fs.fake.impl" -> classOf[FakeFileSystem].getName, "fs.fake.impl.disable.cache" -> "true" ) } /** Create a fake file system path to test from the dir path. */ private def fakeFileSystemPath(dir: File): String = s"fake://${dir.getCanonicalPath}" private def readDeltaTableByPath(path: String): DataFrame = { spark.read.options(fakeFileSystemOptions).format("delta").load(path) } // Ensure any new API from [[DeltaTable]] has to verify it can work with custom file system // options. private val publicMethods = scala.reflect.runtime.universe.typeTag[io.delta.tables.DeltaTable].tpe.decls .filter(_.isPublic) .map(_.name.toString).toSet private val ignoreMethods = Seq() private val testedMethods = Seq( "addFeatureSupport", "as", "alias", "clone", "cloneAtTimestamp", "cloneAtVersion", "delete", "detail", "dropFeatureSupport", "generate", "history", "merge", "optimize", "restoreToVersion", "restoreToTimestamp", "toDF", "update", "updateExpr", "upgradeTableProtocol", "vacuum" ) val untestedMethods = publicMethods -- ignoreMethods -- testedMethods assert( untestedMethods.isEmpty, s"Found new methods added to DeltaTable: $untestedMethods. " + "Please make sure you add a new test to verify it works with file system " + "options in this file, and update the `testedMethods` list. " + "If this new method doesn't need to support file system options, " + "you can add it to the `ignoredMethods` list") test("forPath: as/alias/toDF with filesystem options.") { withTempDir { dir => val path = fakeFileSystemPath(dir) val fsOptions = fakeFileSystemOptions testData.write.options(fsOptions).format("delta").save(path) checkAnswer( DeltaTable.forPath(spark, path, fsOptions).as("tbl").toDF.select("tbl.value"), testData.select("value").collect().toSeq) checkAnswer( DeltaTable.forPath(spark, path, fsOptions).alias("tbl").toDF.select("tbl.value"), testData.select("value").collect().toSeq) } } test("forPath with unsupported options") { withTempDir { dir => val path = fakeFileSystemPath(dir) val fsOptions = fakeFileSystemOptions testData.write.options(fsOptions).format("delta").save(path) val finalOptions = fsOptions + ("otherKey" -> "otherVal") assertThrows[DeltaIllegalArgumentException] { io.delta.tables.DeltaTable.forPath(spark, path, finalOptions) } } } test("forPath error out without filesystem options passed in.") { withTempDir { dir => val path = fakeFileSystemPath(dir) val fsOptions = fakeFileSystemOptions testData.write.options(fsOptions).format("delta").save(path) val e = intercept[UnsupportedFileSystemException] { io.delta.tables.DeltaTable.forPath(spark, path) }.getMessage assert(e.contains("""No FileSystem for scheme "fake"""")) } } test("forPath - with filesystem options") { withTempDir { dir => val path = fakeFileSystemPath(dir) val fsOptions = fakeFileSystemOptions testData.write.options(fsOptions).format("delta").save(path) val deltaTable = io.delta.tables.DeltaTable.forPath(spark, path, fsOptions) val testDataSeq = testData.collect().toSeq // verify table can be read checkAnswer(deltaTable.toDF, testDataSeq) // verify java friendly API. import scala.collection.JavaConverters._ val deltaTable2 = io.delta.tables.DeltaTable.forPath( spark, path, new java.util.HashMap[String, String](fsOptions.asJava)) checkAnswer(deltaTable2.toDF, testDataSeq) } } test("updateExpr - with filesystem options") { withTempDir { dir => val path = fakeFileSystemPath(dir) val fsOptions = fakeFileSystemOptions val df = Seq((1, 10), (2, 20), (3, 30), (4, 40)).toDF("key", "value") df.write.options(fsOptions).format("delta").save(path) val table = io.delta.tables.DeltaTable.forPath(spark, path, fsOptions) table.updateExpr(Map("key" -> "100")) checkAnswer(readDeltaTableByPath(path), Row(100, 10) :: Row(100, 20) :: Row(100, 30) :: Row(100, 40) :: Nil) } } test("update - with filesystem options") { withTempDir { dir => val path = fakeFileSystemPath(dir) val df = Seq((1, 10), (2, 20), (3, 30), (4, 40)).toDF("key", "value") df.write.options(fakeFileSystemOptions).format("delta").save(path) val table = io.delta.tables.DeltaTable.forPath(spark, path, fakeFileSystemOptions) table.update(Map("key" -> functions.expr("100"))) checkAnswer(readDeltaTableByPath(path), Row(100, 10) :: Row(100, 20) :: Row(100, 30) :: Row(100, 40) :: Nil) } } test("delete - with filesystem options") { withTempDir { dir => val path = fakeFileSystemPath(dir) val df = Seq((1, 10), (2, 20), (3, 30), (4, 40)).toDF("key", "value") df.write.options(fakeFileSystemOptions).format("delta").save(path) val table = io.delta.tables.DeltaTable.forPath(spark, path, fakeFileSystemOptions) table.delete(functions.expr("key = 1 or key = 2")) checkAnswer(readDeltaTableByPath(path), Row(3, 30) :: Row(4, 40) :: Nil) } } test("merge - with filesystem options") { withTempDir { dir => val path = fakeFileSystemPath(dir) val target = Seq((1, 10), (2, 20)).toDF("key1", "value1") target.write.options(fakeFileSystemOptions).format("delta").save(path) val source = Seq((1, 100), (3, 30)).toDF("key2", "value2") val table = io.delta.tables.DeltaTable.forPath(spark, path, fakeFileSystemOptions) table.merge(source, "key1 = key2") .whenMatched().updateExpr(Map("key1" -> "key2", "value1" -> "value2")) .whenNotMatched().insertExpr(Map("key1" -> "key2", "value1" -> "value2")) .execute() checkAnswer(readDeltaTableByPath(path), Row(1, 100) :: Row(2, 20) :: Row(3, 30) :: Nil) } } test("vacuum - with filesystem options") { // Note: verify that [DeltaTableUtils.findDeltaTableRoot] works when either // DELTA_FORMAT_CHECK_CACHE_ENABLED is on or off. Seq("true", "false").foreach{ deltaFormatCheckEnabled => withSQLConf( "spark.databricks.delta.formatCheck.cache.enabled" -> deltaFormatCheckEnabled) { withTempDir { dir => val path = fakeFileSystemPath(dir) testData.write.options(fakeFileSystemOptions).format("delta").save(path) val table = io.delta.tables.DeltaTable.forPath(spark, path, fakeFileSystemOptions) // create a uncommitted file. val notCommittedFile = "notCommittedFile.json" val file = new File(dir, notCommittedFile) FileUtils.write(file, "gibberish") // set to ancient time so that the file is eligible to be vacuumed. file.setLastModified(0) assert(file.exists()) table.vacuum() val file2 = new File(dir, notCommittedFile) assert(!file2.exists()) } } } } test("clone - with filesystem options") { withTempDir { dir => val baseDir = fakeFileSystemPath(dir) val srcDir = new File(baseDir, "source").getCanonicalPath val dstDir = new File(baseDir, "destination").getCanonicalPath spark.range(10).write.options(fakeFileSystemOptions).format("delta").save(srcDir) val srcTable = io.delta.tables.DeltaTable.forPath(spark, srcDir, fakeFileSystemOptions) srcTable.clone(dstDir, isShallow = true) val srcLog = DeltaLog.forTable(spark, new Path(srcDir), fakeFileSystemOptions) val dstLog = DeltaLog.forTable(spark, new Path(dstDir), fakeFileSystemOptions) checkAnswer( spark.baseRelationToDataFrame(srcLog.createRelation()), spark.baseRelationToDataFrame(dstLog.createRelation()) ) } } test("cloneAtVersion/timestamp - with filesystem options") { Seq(true, false).foreach { cloneWithVersion => withTempDir { dir => val baseDir = fakeFileSystemPath(dir) val fsOptions = fakeFileSystemOptions val srcDir = new File(baseDir, "source").getCanonicalPath val dstDir = new File(baseDir, "destination").getCanonicalPath val df1 = Seq(1, 2, 3).toDF("id") val df2 = Seq(4, 5).toDF("id") val df3 = Seq(6, 7).toDF("id") // version 0. df1.write.format("delta").options(fsOptions).save(srcDir) // version 1. df2.write.format("delta").options(fsOptions).mode("append").save(srcDir) // version 2. df3.write.format("delta").options(fsOptions).mode("append").save(srcDir) val srcTable = io.delta.tables.DeltaTable.forPath(spark, srcDir, fakeFileSystemOptions) if (cloneWithVersion) { srcTable.cloneAtVersion(0, dstDir, isShallow = true) } else { // clone with timestamp. // // set the time to first file with a early time and verify the delta table can be // restored to it. val desiredTime = new Timestamp(System.currentTimeMillis() - 5.days.toMillis) val logPath = new Path(srcDir, "_delta_log") val file = new File(FileNames.unsafeDeltaFile(logPath, 0).toString) assert(file.setLastModified(desiredTime.getTime)) srcTable.cloneAtTimestamp(desiredTime.toString, dstDir, isShallow = true) } val dstLog = DeltaLog.forTable(spark, new Path(dstDir), fakeFileSystemOptions) checkAnswer( df1, spark.baseRelationToDataFrame(dstLog.createRelation()) ) } } } test("optimize - with filesystem options") { withTempDir { dir => val path = fakeFileSystemPath(dir) val fsOptions = fakeFileSystemOptions Seq(1, 2, 3).toDF().write.options(fsOptions).format("delta").save(path) Seq(4, 5, 6) .toDF().write.options(fsOptions).format("delta").mode("append").save(path) val origData: DataFrame = spark.read.options(fsOptions).format("delta").load(path) val deltaLog = DeltaLog.forTable(spark, new Path(path), fsOptions) val table = io.delta.tables.DeltaTable.forPath(spark, path, fsOptions) val versionBeforeOptimize = deltaLog.snapshot.version table.optimize().executeCompaction() deltaLog.update() assert(deltaLog.snapshot.version == versionBeforeOptimize + 1) checkDatasetUnorderly(origData.as[Int], 1, 2, 3, 4, 5, 6) } } test("history - with filesystem options") { withTempDir { dir => val path = fakeFileSystemPath(dir) val fsOptions = fakeFileSystemOptions Seq(1, 2, 3).toDF().write.options(fsOptions).format("delta").save(path) val table = io.delta.tables.DeltaTable.forPath(spark, path, fsOptions) table.history().collect() } } test("generate - with filesystem options") { withSQLConf("spark.databricks.delta.symlinkFormatManifest.fileSystemCheck.enabled" -> "false") { withTempDir { dir => val path = fakeFileSystemPath(dir) val fsOptions = fakeFileSystemOptions Seq(1, 2, 3).toDF().write.options(fsOptions).format("delta").save(path) val table = io.delta.tables.DeltaTable.forPath(spark, path, fsOptions) table.generate("symlink_format_manifest") } } } test("restoreTable - with filesystem options") { withSQLConf("spark.databricks.service.checkSerialization" -> "false") { withTempDir { dir => val path = fakeFileSystemPath(dir) val fsOptions = fakeFileSystemOptions val df1 = Seq(1, 2, 3).toDF("id") val df2 = Seq(4, 5).toDF("id") val df3 = Seq(6, 7).toDF("id") // version 0. df1.write.format("delta").options(fsOptions).save(path) val deltaLog = DeltaLog.forTable(spark, new Path(path), fsOptions) assert(deltaLog.snapshot.version == 0) // version 1. df2.write.format("delta").options(fsOptions).mode("append").save(path) deltaLog.update() assert(deltaLog.snapshot.version == 1) // version 2. df3.write.format("delta").options(fsOptions).mode("append").save(path) deltaLog.update() assert(deltaLog.snapshot.version == 2) checkAnswer( spark.read.format("delta").options(fsOptions).load(path), df1.union(df2).union(df3)) val deltaTable = io.delta.tables.DeltaTable.forPath(spark, path, fsOptions) deltaTable.restoreToVersion(1) checkAnswer( spark.read.format("delta").options(fsOptions).load(path), df1.union(df2) ) // set the time to first file with a early time and verify the delta table can be restored // to it. val desiredTime = new Timestamp(System.currentTimeMillis() - 5.days.toMillis) val logPath = new Path(dir.getCanonicalPath, "_delta_log") val file = new File(FileNames.unsafeDeltaFile(logPath, 0).toString) assert(file.setLastModified(desiredTime.getTime)) val deltaTable2 = io.delta.tables.DeltaTable.forPath(spark, path, fsOptions) deltaTable2.restoreToTimestamp(desiredTime.toString) checkAnswer( spark.read.format("delta").options(fsOptions).load(path), df1 ) } } } test("upgradeTableProtocol - with filesystem options.") { withTempDir { dir => val path = fakeFileSystemPath(dir) val fsOptions = fakeFileSystemOptions // create a table with a default Protocol. val testSchema = spark.range(1).schema val log = DeltaLog.forTable(spark, new Path(path), fsOptions) log.createLogDirectoriesIfNotExists() log.store.write( FileNames.unsafeDeltaFile(log.logPath, 0), Iterator(Metadata(schemaString = testSchema.json).json, Protocol(0, 0).json), overwrite = false, log.newDeltaHadoopConf()) log.update() // update the protocol. val table = DeltaTable.forPath(spark, path, fsOptions) table.upgradeTableProtocol(1, 2) val expectedProtocol = Protocol(1, 2) assert(log.snapshot.protocol === expectedProtocol) } } test("addFeatureSupport - with filesystem options.") { withTempDir { dir => val path = fakeFileSystemPath(dir) val fsOptions = fakeFileSystemOptions // create a table with a default Protocol. val testSchema = spark.range(1).schema val log = DeltaLog.forTable(spark, new Path(path), fsOptions) log.createLogDirectoriesIfNotExists() log.store.write( FileNames.unsafeDeltaFile(log.logPath, 0), Iterator(Metadata(schemaString = testSchema.json).json, Protocol(1, 2).json), overwrite = false, log.newDeltaHadoopConf()) log.update() // update the protocol to support a writer feature. val table = DeltaTable.forPath(spark, path, fsOptions) table.addFeatureSupport(TestWriterFeature.name) assert(log.update().protocol === Protocol(1, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, TestWriterFeature))) table.addFeatureSupport(TestReaderWriterFeature.name) assert( log.update().protocol === Protocol(3, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, TestWriterFeature, TestReaderWriterFeature))) // update the protocol again with invalid feature name. assert(intercept[DeltaTableFeatureException] { table.addFeatureSupport("__invalid_feature__") }.getErrorClass === "DELTA_UNSUPPORTED_FEATURES_IN_CONFIG") } } test("dropFeatureSupport - with filesystem options.") { withTempDir { dir => val path = fakeFileSystemPath(dir) val fsOptions = fakeFileSystemOptions // create a table with a default Protocol. val testSchema = spark.range(1).schema val log = DeltaLog.forTable(spark, new Path(path), fsOptions) log.createLogDirectoriesIfNotExists() log.store.write( FileNames.unsafeDeltaFile(log.logPath, 0), Iterator(Metadata(schemaString = testSchema.json).json, Protocol(1, 2).json), overwrite = false, log.newDeltaHadoopConf()) log.update() // update the protocol to support a writer feature. val table = DeltaTable.forPath(spark, path, fsOptions) table.addFeatureSupport(TestRemovableWriterFeature.name) assert(log.update().protocol === Protocol(1, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, TestRemovableWriterFeature))) // Attempt truncating the history when dropping a feature that is not required. // This verifies the truncateHistory option was correctly passed. assert(intercept[DeltaTableFeatureException] { table.dropFeatureSupport("testRemovableWriter", truncateHistory = true) }.getErrorClass === "DELTA_FEATURE_DROP_HISTORY_TRUNCATION_NOT_ALLOWED") // Drop feature. table.dropFeatureSupport(TestRemovableWriterFeature.name) // After dropping the feature we should return back to the original protocol. assert(log.update().protocol === Protocol(1, 2)) table.addFeatureSupport(TestRemovableReaderWriterFeature.name) assert( log.update().protocol === Protocol(3, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, TestRemovableReaderWriterFeature))) // Drop feature. table.dropFeatureSupport(TestRemovableReaderWriterFeature.name) // After dropping the feature we should return back to the original protocol. assert(log.update().protocol === Protocol(1, 2)) // Try to drop an unsupported feature. assert(intercept[DeltaTableFeatureException] { table.dropFeatureSupport("__invalid_feature__") }.getErrorClass === "DELTA_FEATURE_DROP_UNSUPPORTED_CLIENT_FEATURE") // Try to drop a feature that is not present in the protocol. assert(intercept[DeltaTableFeatureException] { table.dropFeatureSupport(TestRemovableReaderWriterFeature.name) }.getErrorClass === "DELTA_FEATURE_DROP_FEATURE_NOT_PRESENT") table.addFeatureSupport(TestReaderWriterFeature.name) // Try to drop a non-removable feature. assert(intercept[DeltaTableFeatureException] { table.dropFeatureSupport(TestReaderWriterFeature.name) }.getErrorClass === "DELTA_FEATURE_DROP_NONREMOVABLE_FEATURE") } } test("details - with filesystem options.") { withTempDir{ dir => val path = fakeFileSystemPath(dir) val fsOptions = fakeFileSystemOptions Seq(1, 2, 3).toDF().write.format("delta").options(fsOptions).save(path) val deltaTable = DeltaTable.forPath(spark, path, fsOptions) checkAnswer( deltaTable.detail().select("format"), Seq(Row("delta")) ) } } } ================================================ FILE: spark/src/test/scala/io/delta/tables/DeltaTableTestUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.DataFrame object DeltaTableTestUtils { /** A utility method to access the private constructor of [[DeltaTable]] in tests. */ def createTable(df: DataFrame, deltaLog: DeltaLog): DeltaTable = { new DeltaTable(df, DeltaTableV2(df.sparkSession, deltaLog.dataPath)) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/ActionSerializerSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.UUID import org.apache.spark.sql.delta.DeltaOperations.ManualUpdate import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils.{TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import com.fasterxml.jackson.databind.annotation.JsonSerialize import org.apache.hadoop.fs.Path import org.apache.spark.sql.{QueryTest, SaveMode} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.StructType import org.apache.spark.util.Utils // scalastyle:off: removeFile class ActionSerializerSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { roundTripCompare("Add", AddFile("test", Map.empty, 1, 1, dataChange = true)) roundTripCompare("Add with partitions", AddFile("test", Map("a" -> "1"), 1, 1, dataChange = true)) roundTripCompare("Add with stats", AddFile("test", Map.empty, 1, 1, dataChange = true, stats = "stats")) roundTripCompare("Add with tags", AddFile("test", Map.empty, 1, 1, dataChange = true, tags = Map("a" -> "1"))) roundTripCompare("Add with empty tags", AddFile("test", Map.empty, 1, 1, dataChange = true, tags = Map.empty)) roundTripCompare("Remove", RemoveFile("test", Some(2))) test("AddFile tags") { val action1 = AddFile( path = "a", partitionValues = Map.empty, size = 1, modificationTime = 2, dataChange = false, stats = null, tags = Map("key1" -> "val1", "key2" -> "val2")) val json1 = """{ | "add": { | "path": "a", | "partitionValues": {}, | "size": 1, | "modificationTime": 2, | "dataChange": false, | "tags": { | "key1": "val1", | "key2": "val2" | } | } |}""".stripMargin assert(action1 === Action.fromJson(json1)) assert(action1.json === json1.replaceAll("\\s", "")) val json2 = """{ | "add": { | "path": "a", | "partitionValues": {}, | "size": 1, | "modificationTime": 2, | "dataChange": false, | "tags": {} | } |}""".stripMargin val action2 = AddFile( path = "a", partitionValues = Map.empty, size = 1, modificationTime = 2, dataChange = false, stats = null, tags = Map.empty) assert(action2 === Action.fromJson(json2)) assert(action2.json === json2.replaceAll("\\s", "")) } // This is the same test as "removefile" in OSS, but due to a Jackson library upgrade the behavior // has diverged between Spark 3.1 and Spark 3.2. // We don't believe this is a practical issue because all extant versions of Delta explicitly // write the dataChange field. test("remove file deserialization") { val removeJson = RemoveFile("a", Some(2L)).json assert(removeJson.contains(""""deletionTimestamp":2""")) assert(!removeJson.contains("""delTimestamp""")) val json1 = """{"remove":{"path":"a","deletionTimestamp":2,"dataChange":true}}""" val json2 = """{"remove":{"path":"a","dataChange":false}}""" val json4 = """{"remove":{"path":"a","deletionTimestamp":5}}""" assert(Action.fromJson(json1) === RemoveFile("a", Some(2L), dataChange = true)) assert(Action.fromJson(json2) === RemoveFile("a", None, dataChange = false)) assert(Action.fromJson(json4) === RemoveFile("a", Some(5L), dataChange = true)) } roundTripCompare("SetTransaction", SetTransaction("a", 1, Some(1234L))) roundTripCompare("SetTransaction without lastUpdated", SetTransaction("a", 1, None)) roundTripCompare("MetaData", Metadata( "id", "table", "testing", Format("parquet", Map.empty), new StructType().json, Seq("a"))) test("extra fields") { // TODO reading from checkpoint Action.fromJson("""{"txn": {"test": 1}}""") } test("deserialization of CommitInfo without tags") { val expectedCommitInfo = CommitInfo( time = 123L, operation = "CONVERT", inCommitTimestamp = Some(123L), operationParameters = Map.empty, commandContext = Map.empty, readVersion = Some(23), isolationLevel = Some("SnapshotIsolation"), isBlindAppend = Some(true), operationMetrics = Some(Map("m1" -> "v1", "m2" -> "v2")), userMetadata = Some("123"), tags = None, txnId = None).copy(engineInfo = None) // json of commit info actions without tag or engineInfo field val json1 = """{"commitInfo":{"inCommitTimestamp":123,"timestamp":123,"operation":"CONVERT",""" + """"operationParameters":{},"readVersion":23,""" + """"isolationLevel":"SnapshotIsolation","isBlindAppend":true,""" + """"operationMetrics":{"m1":"v1","m2":"v2"},"userMetadata":"123"}}""".stripMargin assert(Action.fromJson(json1) === expectedCommitInfo) } test("deserialization of CommitInfo without commitTime") { val expectedCommitInfo = CommitInfo( time = 123L, operation = "CONVERT", operationParameters = Map.empty, commandContext = Map.empty, readVersion = Some(23), isolationLevel = Some("SnapshotIsolation"), isBlindAppend = Some(true), operationMetrics = Some(Map("m1" -> "v1", "m2" -> "v2")), userMetadata = Some("123"), tags = None, txnId = None).copy(engineInfo = None) // json of commit info actions without tag or engineInfo field val json1 = """{"commitInfo":{"timestamp":123,"operation":"CONVERT",""" + """"operationParameters":{},"readVersion":23,""" + """"isolationLevel":"SnapshotIsolation","isBlindAppend":true,""" + """"operationMetrics":{"m1":"v1","m2":"v2"},"userMetadata":"123"}}""".stripMargin assert(Action.fromJson(json1) === expectedCommitInfo) } test("deserialization of CommitInfo with a very small ICT") { val json1 = """{"commitInfo":{"inCommitTimestamp":123,"timestamp":123,"operation":"CONVERT",""" + """"operationParameters":{},"readVersion":23,""" + """"isolationLevel":"SnapshotIsolation","isBlindAppend":true,""" + """"operationMetrics":{"m1":"v1","m2":"v2"},"userMetadata":"123"}}""".stripMargin assert(Action.fromJson(json1).asInstanceOf[CommitInfo].inCommitTimestamp.get == 123L) } test("deserialization of CommitInfo with a very large ICT") { val json1 = """{"commitInfo":{"inCommitTimestamp":123333333,"timestamp":123,"operation":"CONVERT",""" + """"operationParameters":{},"readVersion":23,""" + """"isolationLevel":"SnapshotIsolation","isBlindAppend":true,""" + """"operationMetrics":{"m1":"v1","m2":"v2"},"userMetadata":"123"}}""".stripMargin assert(Action.fromJson(json1).asInstanceOf[CommitInfo].inCommitTimestamp.get == 123333333L) } test("deserialization of CommitInfo with missing ICT") { val json1 = """{"commitInfo":{"timestamp":123,"operation":"CONVERT",""" + """"operationParameters":{},"readVersion":23,""" + """"isolationLevel":"SnapshotIsolation","isBlindAppend":true,""" + """"operationMetrics":{"m1":"v1","m2":"v2"},"userMetadata":"123"}}""".stripMargin val ictOpt: Option[Long] = Action.fromJson(json1).asInstanceOf[CommitInfo].inCommitTimestamp assert(ictOpt.isEmpty) } test("round trip of operation parameters: primitive types in operation parameters") { val rawOperationParameters: Map[String, Any] = Map( "catalogTable" -> "t1", "numFiles" -> 23L, "partitionedBy" -> JsonUtils.toJson(Seq("a", false)), "sourceFormat" -> "parquet", "collectStats" -> false, "k1" -> null, "" -> null) val operationParameters = rawOperationParameters.mapValues(JsonUtils.toJson(_)).toMap val expectedOperationParameters = Map( "catalogTable" -> "\"t1\"", "numFiles" -> "23", "partitionedBy" -> "\"[\\\"a\\\",false]\"", "sourceFormat" -> "\"parquet\"", "collectStats" -> "false", "k1" -> "null", "" -> "null") assert(operationParameters === expectedOperationParameters) val expectedLegacyOperationParameters = Map( "catalogTable" -> "t1", "numFiles" -> "23", "partitionedBy" -> "[\"a\",false]", "sourceFormat" -> "parquet", "collectStats" -> "false", "k1" -> null, "" -> null) val commitInfo = CommitInfo.empty().withTimestamp(1) .copy(operationParameters = operationParameters) // Try a couple rounds of round trips. val roundTrippedCommitInfo1 = JsonUtils.fromJson[CommitInfo](JsonUtils.toJson(commitInfo)) assert(roundTrippedCommitInfo1.operationParameters === expectedOperationParameters) assert(CommitInfo.getLegacyPostDeserializationOperationParameters( roundTrippedCommitInfo1.operationParameters) === expectedLegacyOperationParameters) val roundTrippedCommitInfo2 = JsonUtils.fromJson[CommitInfo](JsonUtils.toJson(roundTrippedCommitInfo1)) assert(roundTrippedCommitInfo2.operationParameters === expectedOperationParameters) assert(CommitInfo.getLegacyPostDeserializationOperationParameters( roundTrippedCommitInfo2.operationParameters) === expectedLegacyOperationParameters) } test("round trip of operation parameters: non-primitive types in operation parameters") { val rawOperationParameters: Map[String, Any] = Map( "k1" -> Seq(1, 2), "k2" -> Map("a" -> "x", "b" -> 1, "c" -> TestObject("f1", -1, None), "d" -> Seq(3, "e")), "k3" -> Seq.empty, "k4" -> TestObject("field1", 99, Some(Seq("v1", "v2")))) val operationParameters = rawOperationParameters.mapValues(JsonUtils.toJson(_)).toMap val expectedOperationParameters = Map( "k1" -> "[1,2]", "k2" -> "{\"a\":\"x\",\"b\":1,\"c\":{\"field1\":\"f1\",\"field2\":-1},\"d\":[3,\"e\"]}", "k3" -> "[]", "k4" -> "{\"field1\":\"field1\",\"field2\":99,\"field3\":[\"v1\",\"v2\"]}") assert(operationParameters === expectedOperationParameters) val expectedLegacyOperationParameters = Map( "k1" -> "[1,2]", "k2" -> "{\"a\":\"x\",\"b\":1,\"c\":{\"field1\":\"f1\",\"field2\":-1},\"d\":[3,\"e\"]}", "k3" -> "[]", "k4" -> "{\"field1\":\"field1\",\"field2\":99,\"field3\":[\"v1\",\"v2\"]}") val commitInfo = CommitInfo.empty().withTimestamp(1) .copy(operationParameters = operationParameters) // Try a couple rounds of round trips. val roundTrippedCommitInfo1 = JsonUtils.fromJson[CommitInfo](JsonUtils.toJson(commitInfo)) assert(roundTrippedCommitInfo1.operationParameters === expectedOperationParameters) intercept[com.fasterxml.jackson.databind.exc.MismatchedInputException] { // Non-primitive type values are not supported with legacy deserialization. // Note that this is not a quirk specific to getLegacyPostDeserializationOperationParameters // but rather a quirk of the legacy deserialization. CommitInfo.getLegacyPostDeserializationOperationParameters( roundTrippedCommitInfo1.operationParameters) } val roundTrippedCommitInfo2 = JsonUtils.fromJson[CommitInfo](JsonUtils.toJson(roundTrippedCommitInfo1)) assert(roundTrippedCommitInfo2.operationParameters === expectedOperationParameters) } test("getLegacyPostDeserializationOperationParameters is same as reading operation parameters " + "without custom deserialize") { val rawOperationParameters: Map[String, Any] = Map( "catalogTable" -> "t1", "numFiles" -> 23L, "partitionedBy" -> "[\"a\",\"b\"]", "sourceFormat" -> "parquet", "collectStats" -> false, "k1" -> JsonUtils.toJson(Seq(1, 2)), "k2" -> JsonUtils.toJson(Map("a" -> "x", "b" -> 1, "c" -> Seq(3, "e"))), "k3" -> null, "k4" -> JsonUtils.toJson(Seq.empty), "" -> null) val operationParameters = rawOperationParameters.mapValues(JsonUtils.toJson(_)).toMap val commitInfo = CommitInfo .empty() .withTimestamp(1) .copy(operationParameters = operationParameters) val testRawDeserialization = TestRawDeserialization( operationParameters = commitInfo.operationParameters) val expectedLegacyOperationParameters = JsonUtils.fromJson[TestRawDeserialization]( JsonUtils.toJson(testRawDeserialization)).operationParameters // Try a couple rounds of round trips. val roundTrippedCommitInfo1 = JsonUtils.fromJson[CommitInfo](JsonUtils.toJson(commitInfo)) assert(CommitInfo.getLegacyPostDeserializationOperationParameters( roundTrippedCommitInfo1.operationParameters) === expectedLegacyOperationParameters) val roundTrippedCommitInfo2 = JsonUtils.fromJson[CommitInfo](JsonUtils.toJson(roundTrippedCommitInfo1)) assert(CommitInfo.getLegacyPostDeserializationOperationParameters( roundTrippedCommitInfo2.operationParameters) === expectedLegacyOperationParameters) } testActionSerDe( "Protocol - json serialization/deserialization", Protocol(minReaderVersion = 1, minWriterVersion = 2), expectedJson = """{"protocol":{"minReaderVersion":1,"minWriterVersion":2}}""") testActionSerDe( "Protocol - json serialization/deserialization with writer features", Protocol(minReaderVersion = 1, minWriterVersion = TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(AppendOnlyTableFeature), expectedJson = """{"protocol":{"minReaderVersion":1,""" + s""""minWriterVersion":$TABLE_FEATURES_MIN_WRITER_VERSION,""" + """"writerFeatures":["appendOnly"]}}""") testActionSerDe( "Protocol - json serialization/deserialization with reader and writer features", Protocol( minReaderVersion = TABLE_FEATURES_MIN_READER_VERSION, minWriterVersion = TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestLegacyReaderWriterFeature), expectedJson = s"""{"protocol":{"minReaderVersion":$TABLE_FEATURES_MIN_READER_VERSION,""" + s""""minWriterVersion":$TABLE_FEATURES_MIN_WRITER_VERSION,""" + """"readerFeatures":["testLegacyReaderWriter"],""" + """"writerFeatures":["testLegacyReaderWriter"]}}""") testActionSerDe( "Protocol - json serialization/deserialization with several reader and writer features", Protocol( minReaderVersion = TABLE_FEATURES_MIN_READER_VERSION, minWriterVersion = TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestLegacyReaderWriterFeature) .withFeature(TestReaderWriterFeature), expectedJson = s"""{"protocol":{"minReaderVersion":$TABLE_FEATURES_MIN_READER_VERSION,""" + s""""minWriterVersion":$TABLE_FEATURES_MIN_WRITER_VERSION,""" + """"readerFeatures":["testLegacyReaderWriter","testReaderWriter"],""" + """"writerFeatures":["testLegacyReaderWriter","testReaderWriter"]}}""") testActionSerDe( "Protocol - json serialization/deserialization with empty reader and writer features", Protocol( minReaderVersion = TABLE_FEATURES_MIN_READER_VERSION, minWriterVersion = TABLE_FEATURES_MIN_WRITER_VERSION), expectedJson = s"""{"protocol":{"minReaderVersion":$TABLE_FEATURES_MIN_READER_VERSION,""" + s""""minWriterVersion":$TABLE_FEATURES_MIN_WRITER_VERSION,""" + """"readerFeatures":[],"writerFeatures":[]}}""") testActionSerDe( "SetTransaction (lastUpdated is None) - json serialization/deserialization", SetTransaction(appId = "app-1", version = 2L, lastUpdated = None), expectedJson = """{"txn":{"appId":"app-1","version":2}}""".stripMargin) testActionSerDe( "SetTransaction (lastUpdated is not None) - json serialization/deserialization", SetTransaction(appId = "app-2", version = 3L, lastUpdated = Some(4L)), expectedJson = """{"txn":{"appId":"app-2","version":3,"lastUpdated":4}}""".stripMargin) testActionSerDe( "AddFile (without tags) - json serialization/deserialization", AddFile("x=2/f1", partitionValues = Map("x" -> "2"), size = 10, modificationTime = 1, dataChange = true, stats = "{\"numRecords\": 2}"), expectedJson = """{"add":{"path":"x=2/f1","partitionValues":{"x":"2"},"size":10,""" + """"modificationTime":1,"dataChange":true,"stats":"{\"numRecords\": 2}"}}""".stripMargin) testActionSerDe( "AddFile (with tags) - json serialization/deserialization", AddFile("part=p1/f1", partitionValues = Map("x" -> "2"), size = 10, modificationTime = 1, dataChange = true, stats = "{\"numRecords\": 2}", tags = Map("TAG1" -> "23")), expectedJson = """{"add":{"path":"part=p1/f1","partitionValues":{"x":"2"},"size":10""" + ""","modificationTime":1,"dataChange":true,"stats":"{\"numRecords\": 2}",""" + """"tags":{"TAG1":"23"}}}""" ) testActionSerDe( "AddFile (with clusteringProvider) - json serialization/deserialization", AddFile( "clusteredFile.part", partitionValues = Map.empty[String, String], size = 10, modificationTime = 1, dataChange = true, stats = "{\"numRecords\": 2}", tags = Map("TAG1" -> "23"), clusteringProvider = Some("liquid")), expectedJson = """{"add":{"path":"clusteredFile.part","partitionValues":{},"size":10""" + ""","modificationTime":1,"dataChange":true,"stats":"{\"numRecords\": 2}",""" + """"tags":{"TAG1":"23"}""" + ""","clusteringProvider":"liquid"}}""") testActionSerDe( "RemoveFile (without tags) - json serialization/deserialization", AddFile("part=p1/f1", partitionValues = Map("x" -> "2"), size = 10, modificationTime = 1, dataChange = true, stats = "{\"numRecords\": 2}").removeWithTimestamp(timestamp = 11), expectedJson = """{"remove":{"path":"part=p1/f1","deletionTimestamp":11,"dataChange":true,""" + """"extendedFileMetadata":true,"partitionValues":{"x":"2"},"size":10,""" + """"stats":"{\"numRecords\": 2}"}}""") testActionSerDe( "RemoveFile (without tags and stats) - json serialization/deserialization", AddFile("part=p1/f1", partitionValues = Map("x" -> "2"), size = 10, modificationTime = 1, dataChange = true, stats = "{\"numRecords\": 2}") .removeWithTimestamp(timestamp = 11) .copy(stats = null), expectedJson = """{"remove":{"path":"part=p1/f1","deletionTimestamp":11,"dataChange":true,""" + """"extendedFileMetadata":true,"partitionValues":{"x":"2"},"size":10}}""") private def deletionVectorWithRelativePath: DeletionVectorDescriptor = DeletionVectorDescriptor.onDiskWithRelativePath( id = UUID.randomUUID(), randomPrefix = "a1", sizeInBytes = 10, cardinality = 2, offset = Some(10)) private def deletionVectorWithAbsolutePath: DeletionVectorDescriptor = DeletionVectorDescriptor.onDiskWithAbsolutePath( path = "/test.dv", sizeInBytes = 10, cardinality = 2, offset = Some(10)) private def deletionVectorInline: DeletionVectorDescriptor = DeletionVectorDescriptor.inlineInLog(Array(1, 2, 3, 4), 1) roundTripCompare("Add with deletion vector - relative path", AddFile( path = "test", partitionValues = Map.empty, size = 1, modificationTime = 1, dataChange = true, tags = Map.empty, deletionVector = deletionVectorWithRelativePath)) roundTripCompare("Add with deletion vector - absolute path", AddFile( path = "test", partitionValues = Map.empty, size = 1, modificationTime = 1, dataChange = true, tags = Map.empty, deletionVector = deletionVectorWithAbsolutePath)) roundTripCompare("Add with deletion vector - inline", AddFile( path = "test", partitionValues = Map.empty, size = 1, modificationTime = 1, dataChange = true, tags = Map.empty, deletionVector = deletionVectorInline)) roundTripCompare("Remove with deletion vector - relative path", RemoveFile( path = "test", deletionTimestamp = Some(1L), extendedFileMetadata = Some(true), partitionValues = Map.empty, dataChange = true, size = Some(1L), tags = Map.empty, deletionVector = deletionVectorWithRelativePath)) roundTripCompare("Remove with deletion vector - absolute path", RemoveFile( path = "test", deletionTimestamp = Some(1L), extendedFileMetadata = Some(true), partitionValues = Map.empty, dataChange = true, size = Some(1L), tags = Map.empty, deletionVector = deletionVectorWithAbsolutePath)) roundTripCompare("Remove with deletion vector - inline", RemoveFile( path = "test", deletionTimestamp = Some(1L), extendedFileMetadata = Some(true), partitionValues = Map.empty, dataChange = true, size = Some(1L), tags = Map.empty, deletionVector = deletionVectorInline)) // These make sure we don't accidentally serialise something we didn't mean to. testActionSerDe( name = "AddFile (with deletion vector) - json serialization/deserialization", action = AddFile( path = "test", partitionValues = Map.empty, size = 1, modificationTime = 1, dataChange = true, stats = """{"numRecords":3}""", tags = Map.empty, deletionVector = deletionVectorWithAbsolutePath), expectedJson = """ |{"add":{ |"path":"test", |"partitionValues":{}, |"size":1, |"modificationTime":1, |"dataChange":true, |"stats":"{\"numRecords\":3}", |"tags":{}, |"deletionVector":{ |"storageType":"p", |"pathOrInlineDv":"/test.dv", |"offset":10, |"sizeInBytes":10, |"cardinality":2}} |}""".stripMargin.replaceAll("\n", ""), extraSettings = Seq( // Skip the table property check, so this write doesn't fail. DeltaSQLConf.DELETION_VECTORS_COMMIT_CHECK_ENABLED.key -> "false") ) test("DomainMetadata action - json serialization/deserialization") { val table = "testTable" withTable(table) { sql( s""" | CREATE TABLE $table(id int) USING delta | tblproperties | ('${TableFeatureProtocolUtils.propertyKey(DomainMetadataTableFeature)}' = 'enabled') |""".stripMargin) val deltaTable = DeltaTableV2(spark, TableIdentifier(table)) val deltaLog = deltaTable.deltaLog val domainMetadatas = DomainMetadata( domain = "testDomain", configuration = JsonUtils.toJson(Map("key1" -> "value1")), removed = false) :: Nil val version = deltaTable.startTransactionWithInitialSnapshot() .commit(domainMetadatas, ManualUpdate) val committedActions = deltaLog.store.read( FileNames.unsafeDeltaFile(deltaLog.logPath, version), deltaLog.newDeltaHadoopConf()) assert(committedActions.size == 2) val serializedJson = committedActions.last val expectedJson = """ |{"domainMetadata":{ |"domain":"testDomain", |"configuration": |"{\"key1\":\"value1\"}", |"removed":false} |}""".stripMargin.replaceAll("\n", "") assert(serializedJson === expectedJson) val asObject = Action.fromJson(serializedJson) assert(domainMetadatas.head === asObject) } } test("CheckpointMetadata - serialize/deserialize") { val m1 = CheckpointMetadata(version = 1, tags = null) // tags are null val m2 = m1.copy(tags = Map()) // tags are empty val m3 = m1.copy( // tags are non empty tags = Map("k1" -> "v1", "schema" -> """{"type":"struct","fields":[]}""") ) assert(m1.json === """{"checkpointMetadata":{"version":1}}""") assert(m2.json === """{"checkpointMetadata":{"version":1,"tags":{}}}""") assert(m3.json === """{"checkpointMetadata":{"version":1,""" + """"tags":{"k1":"v1","schema":"{\"type\":\"struct\",\"fields\":[]}"}}}""") Seq(m1, m2, m3).foreach { metadata => assert(metadata === JsonUtils.fromJson[SingleAction](metadata.json).unwrap) } } test("SidecarFile - serialize/deserialize") { val f1 = // tags are null SidecarFile(path = "/t1/p1", sizeInBytes = 1L, modificationTime = 3, tags = null) val f2 = f1.copy(tags = Map()) // tags are empty val f3 = f2.copy( // tags are non empty tags = Map("k1" -> "v1", "schema" -> """{"type":"struct","fields":[]}""") ) assert(f1.json === """{"sidecar":{"path":"/t1/p1","sizeInBytes":1,"modificationTime":3}}""") assert(f2.json === """{"sidecar":{"path":"/t1/p1","sizeInBytes":1,""" + """"modificationTime":3,"tags":{}}}""") assert(f3.json === """{"sidecar":{"path":"/t1/p1","sizeInBytes":1,"modificationTime":3,""" + """"tags":{"k1":"v1","schema":"{\"type\":\"struct\",\"fields\":[]}"}}}""".stripMargin) Seq(f1, f2, f3).foreach { file => assert(file === JsonUtils.fromJson[SingleAction](file.json).unwrap) } } testActionSerDe( "AddCDCFile (without tags) - json serialization/deserialization", AddCDCFile("part=p1/f1", partitionValues = Map("x" -> "2"), size = 10), expectedJson = """{"cdc":{"path":"part=p1/f1","partitionValues":{"x":"2"},""" + """"size":10,"dataChange":false}}""".stripMargin) testActionSerDe( "AddCDCFile (with tags) - json serialization/deserialization", AddCDCFile("part=p2/f1", partitionValues = Map("x" -> "2"), size = 11, tags = Map("key1" -> "value1")), expectedJson = """{"cdc":{"path":"part=p2/f1","partitionValues":{"x":"2"},""" + """"size":11,"tags":{"key1":"value1"},"dataChange":false}}""".stripMargin) testActionSerDe( "AddCDCFile (without null value in partitionValues) - json serialization/deserialization", AddCDCFile("part=p1/f1", partitionValues = Map("x" -> null), size = 10), expectedJson = """{"cdc":{"path":"part=p1/f1","partitionValues":{"x":null},""" + """"size":10,"dataChange":false}}""".stripMargin) { val metadata = Metadata(id = "testId", createdTime = Some(2222)) testActionSerDe( "Metadata (with all defaults) - json serialization/deserialization", metadata, expectedJson = """{"metaData":{"id":"testId","format":{"provider":"parquet",""" + """"options":{}},"partitionColumns":[],"configuration":{},"createdTime":2222}}""") testActionSerDe( "Metadata (with all defaults and empty createdTime) - json serialization/deserialization", metadata.copy(createdTime = None), expectedJson = """{"metaData":{"id":"testId","format":{"provider":"parquet",""" + """"options":{}},"partitionColumns":[],"configuration":{}}}""") } { val schemaStr = new StructType().add("a", "long").json val metadata = Metadata( id = "testId", name = "t1", description = "desc", format = Format(provider = "parquet", options = Map("o1" -> "v1")), partitionColumns = Seq("a"), createdTime = Some(2222), configuration = Map("delta.enableXyz" -> "true"), schemaString = schemaStr) testActionSerDe( "Metadata - json serialization/deserialization", metadata, expectedJson = """{"metaData":{"id":"testId","name":"t1","description":"desc",""" + """"format":{"provider":"parquet","options":{"o1":"v1"}},""" + s""""schemaString":${JsonUtils.toJson(schemaStr)},"partitionColumns":["a"],""" + """"configuration":{"delta.enableXyz":"true"},"createdTime":2222}}""".stripMargin) testActionSerDe( "Metadata with empty createdTime- json serialization/deserialization", metadata.copy(createdTime = None), expectedJson = """{"metaData":{"id":"testId","name":"t1","description":"desc",""" + """"format":{"provider":"parquet","options":{"o1":"v1"}},""" + s""""schemaString":${JsonUtils.toJson(schemaStr)},"partitionColumns":["a"],""" + """"configuration":{"delta.enableXyz":"true"}}}""".stripMargin) } { // Test for CommitInfo val commitInfo = CommitInfo( time = 123L, operation = "CONVERT", inCommitTimestamp = Some(123L), operationParameters = Map.empty, commandContext = Map("clusterId" -> "23"), readVersion = Some(23), isolationLevel = Some("SnapshotIsolation"), isBlindAppend = Some(true), operationMetrics = Some(Map("m1" -> "v1", "m2" -> "v2")), userMetadata = Some("123"), tags = Some(Map("k1" -> "v1")), txnId = Some("123") ).copy(engineInfo = None) testActionSerDe( "CommitInfo (without operationParameters) - json serialization/deserialization", commitInfo, expectedJson = """{"commitInfo":{"inCommitTimestamp":123,"timestamp":123,"operation":"CONVERT",""" + """"operationParameters":{},"clusterId":"23","readVersion":23,""" + """"isolationLevel":"SnapshotIsolation","isBlindAppend":true,""" + """"operationMetrics":{"m1":"v1","m2":"v2"},"userMetadata":"123",""" + """"tags":{"k1":"v1"},"txnId":"123"}}""".stripMargin) test("CommitInfo (with operationParameters) - json serialization/deserialization") { val operation = DeltaOperations.Convert( numFiles = 23L, partitionBy = Seq("a", "b"), collectStats = false, catalogTable = Some("t1"), sourceFormat = Some("parquet")) val commitInfo1 = commitInfo.copy(operationParameters = operation.jsonEncodedValues) val expectedCommitInfoJson1 = // TODO JSON ordering differs between 2.12 and 2.13 if (scala.util.Properties.versionNumberString.startsWith("2.13")) { """{"commitInfo":{"inCommitTimestamp":123,""" + """"timestamp":123,"operation":"CONVERT","operationParameters"""" + """:{"catalogTable":"t1","numFiles":23,"partitionedBy":"[\"a\",\"b\"]",""" + """"sourceFormat":"parquet","collectStats":false},"clusterId":"23","readVersion"""" + """:23,"isolationLevel":"SnapshotIsolation","isBlindAppend":true,""" + """"operationMetrics":{"m1":"v1","m2":"v2"},""" + """"userMetadata":"123","tags":{"k1":"v1"},"txnId":"123"}}""" } else { """{"commitInfo":{"inCommitTimestamp":123,""" + """"timestamp":123,"operation":"CONVERT","operationParameters"""" + """:{"catalogTable":"t1","numFiles":23,"partitionedBy":"[\"a\",\"b\"]",""" + """"sourceFormat":"parquet","collectStats":false},"clusterId":"23","readVersion""" + """":23,"isolationLevel":"SnapshotIsolation","isBlindAppend":true,""" + """"operationMetrics":{"m1":"v1","m2":"v2"},""" + """"userMetadata":"123","tags":{"k1":"v1"},"txnId":"123"}}""" } assert(commitInfo1.json == expectedCommitInfoJson1) val newCommitInfo1 = Action.fromJson(expectedCommitInfoJson1).asInstanceOf[CommitInfo] assert(newCommitInfo1 == commitInfo1) } testActionSerDe( "CommitInfo (with engineInfo) - json serialization/deserialization", commitInfo.copy(engineInfo = Some("Apache-Spark/3.1.1 Delta-Lake/10.1.0")), expectedJson = """{"commitInfo":{"inCommitTimestamp":123,"timestamp":123,"operation":"CONVERT",""" + """"operationParameters":{},"clusterId":"23","readVersion":23,""" + """"isolationLevel":"SnapshotIsolation","isBlindAppend":true,""" + """"operationMetrics":{"m1":"v1","m2":"v2"},"userMetadata":"123",""" + """"tags":{"k1":"v1"},"engineInfo":"Apache-Spark/3.1.1 Delta-Lake/10.1.0",""" + """"txnId":"123"}}""".stripMargin) } test("CommitInfo operationParameters deserialization with primitive types") { // Test edge cases described in JsonMapDeserializer for primitive values // This test verifies that the custom deserializer correctly handles mixed primitive types // in operationParameters: strings, numbers, booleans, and null values val operationParameters = Map( "stringValue" -> "\"simpleString\"", "numberValue" -> "123", "booleanValue" -> "true", "floatValue" -> "45.67", "jsonEncodedString" -> "\"\\\"quoted string\\\"\"" ) val commitInfo = CommitInfo( time = 1234567890L, operation = "WRITE", operationParameters = operationParameters, commandContext = Map("clusterId" -> "test-cluster"), readVersion = Some(5), isolationLevel = Some("WriteSerializable"), isBlindAppend = Some(false), operationMetrics = Some(Map("numFiles" -> "10")), userMetadata = Some("test metadata"), tags = Some(Map("source" -> "test")), txnId = Some("txn-123") ) // Serialize and deserialize to test the JsonMapDeserializer val serialized = commitInfo.json val deserialized = Action.fromJson(serialized).asInstanceOf[CommitInfo] // Verify that operationParameters are correctly preserved after round-trip assert(deserialized.operationParameters == operationParameters) assert(deserialized.operation == "WRITE") assert(deserialized.readVersion == Some(5)) // Test that getLegacyPostDeserializationOperationParameters works correctly val legacyParams = CommitInfo.getLegacyPostDeserializationOperationParameters( deserialized.operationParameters) assert(legacyParams.nonEmpty) } test("CommitInfo operationParameters deserialization with complex operation") { // Test with actual DeltaOperation parameters to verify real-world usage // This focuses on the edge case where operation parameters contain JSON-encoded values val operation = DeltaOperations.Write( mode = SaveMode.Append, partitionBy = Some(Seq("year", "month")), predicate = Some("id > 100"), userMetadata = Some("batch write operation") ) val commitInfo = CommitInfo( time = 9876543210L, operation = operation.name, operationParameters = operation.jsonEncodedValues, commandContext = Map("clusterId" -> "prod-cluster"), readVersion = Some(42), isolationLevel = Some("WriteSerializable"), isBlindAppend = Some(true), operationMetrics = Some(Map("numFiles" -> "25", "numOutputRows" -> "1000")), userMetadata = operation.userMetadata, tags = Some(Map("environment" -> "production", "team" -> "data-eng")), txnId = Some("txn-write-456") ) // Test round-trip serialization/deserialization val serialized = commitInfo.json val deserialized = Action.fromJson(serialized).asInstanceOf[CommitInfo] // Verify that complex operationParameters are correctly handled assert(deserialized.operationParameters == operation.jsonEncodedValues) assert(deserialized.operation == operation.name) assert(deserialized.userMetadata == operation.userMetadata) // Verify the specific operation parameters that should be preserved val params = deserialized.operationParameters assert(params.contains("mode")) assert(params.contains("partitionBy")) assert(params.contains("predicate")) // Test legacy operation parameters for backward compatibility val legacyParams = CommitInfo.getLegacyPostDeserializationOperationParameters( deserialized.operationParameters) // Legacy parameters should be different due to the broken deserialization that the // JsonMapDeserializer fixes assert(legacyParams != deserialized.operationParameters) } private def roundTripCompare(name: String, actions: Action*) = { test(name) { val asJson = actions.map(_.json) val asObjects = asJson.map(Action.fromJson) assert(actions === asObjects) } } /** Test serialization/deserialization of [[Action]] by doing an actual commit */ private def testActionSerDe( name: String, action: => Action, expectedJson: String, extraSettings: Seq[(String, String)] = Seq.empty, testTags: Seq[org.scalatest.Tag] = Seq.empty): Unit = { import org.apache.spark.sql.delta.test.DeltaTestImplicits._ test(name, testTags: _*) { withTempDir { tempDir => val deltaLog = DeltaLog.forTable(spark, new Path(tempDir.getAbsolutePath)) // Disable different delta validations so that the passed action can be committed in // all cases. val settings = Seq( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> "1", DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "1", DeltaSQLConf.DELTA_COMMIT_VALIDATION_ENABLED.key -> "false") ++ extraSettings withSQLConf(settings: _*) { // Do one empty commit so that protocol gets committed. val protocol = Protocol( minReaderVersion = spark.conf.get(DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION), minWriterVersion = spark.conf.get(DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION)) deltaLog.startTransaction().commitManually(protocol, Metadata()) // Commit the actual action. val version = deltaLog.startTransaction().commit(Seq(action), ManualUpdate) // Read the commit file and get the serialized committed actions val committedActions = deltaLog.store.read( FileNames.unsafeDeltaFile(deltaLog.logPath, version), deltaLog.newDeltaHadoopConf()) assert(committedActions.size == 2) val serializedJson = committedActions.last assert(serializedJson === expectedJson) val asObject = Action.fromJson(serializedJson) assert(action === asObject) } } } } } // Both of these are used for CommitInfo serde tests case class TestObject(field1: String, field2: Int, field3: Option[Seq[String]]) /** * Test class to deserialize operation parameters without using custom * JsonMapDeserializer. */ private final case class TestRawDeserialization( @JsonSerialize(using = classOf[JsonMapSerializer]) operationParameters: Map[String, String]) ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/AutoCompactSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File // scalastyle:off import.ordering.noEmptyLine import com.databricks.spark.util.{Log4jUsageLogger, UsageRecord} import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.commands.optimize._ import org.apache.spark.sql.delta.hooks.{AutoCompact, AutoCompactType} import org.apache.spark.sql.delta.optimize.CompactionTestHelperForAutoCompaction import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.AutoCompactPartitionStats import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.JsonUtils import org.apache.hadoop.fs.Path import org.apache.spark.sql.DataFrame import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.expressions.Literal import org.apache.spark.sql.functions.lit import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.StringType import org.apache.spark.unsafe.types.UTF8String trait AutoCompactTestUtils { def captureOptimizeLogs(metrics: String)(f: => Unit): Seq[UsageRecord] = { val usageLogs = Log4jUsageLogger.track(f) usageLogs.filter { usageLog => usageLog.tags.get("opType") == Some(metrics) } } } /** * This class extends the [[CompactionSuiteBase]] and runs all the [[CompactionSuiteBase]] tests * with AutoCompaction. * * It also tests AutoCompaction specific behavior around configuration settings. */ class AutoCompactConfigurationSuite extends CompactionTestHelperForAutoCompaction with DeltaSQLCommandTest with SharedSparkSession with AutoCompactTestUtils { private def setTableProperty(log: DeltaLog, key: String, value: String): Unit = { spark.sql(s"ALTER TABLE delta.`${log.dataPath}` SET TBLPROPERTIES " + s"($key = $value)") } test("auto-compact-type: test table properties") { withTempDir { tempDir => val dir = tempDir.getCanonicalPath spark.range(0, 1).write.format("delta").mode("append").save(dir) val deltaLog = DeltaLog.forTable(spark, dir) val defaultAutoCompactType = AutoCompact.getAutoCompactType(conf, deltaLog.snapshot.metadata) Map( "true" -> Some(AutoCompactType.Enabled), "tRue" -> Some(AutoCompactType.Enabled), "'true'" -> Some(AutoCompactType.Enabled), "false" -> None, "fALse" -> None, "'false'" -> None ).foreach { case (propertyValue, expectedAutoCompactType) => setTableProperty(deltaLog, "delta.autoOptimize.autoCompact", propertyValue) assert(AutoCompact.getAutoCompactType(conf, deltaLog.snapshot.metadata) == expectedAutoCompactType) } } } test("auto-compact-type: test confs") { withTempDir { tempDir => val dir = tempDir.getCanonicalPath spark.range(0, 1).write.format("delta").mode("append").save(dir) val deltaLog = DeltaLog.forTable(spark, dir) val defaultAutoCompactType = AutoCompact.getAutoCompactType(conf, deltaLog.snapshot.metadata) Map( "true" -> Some(AutoCompactType.Enabled), "TrUE" -> Some(AutoCompactType.Enabled), "false" -> None, "FalsE" -> None ).foreach { case (confValue, expectedAutoCompactType) => withSQLConf(DeltaSQLConf.DELTA_AUTO_COMPACT_ENABLED.key -> confValue) { assert(AutoCompact.getAutoCompactType(conf, deltaLog.snapshot.metadata) == expectedAutoCompactType) } } } } } /** * This class extends the [[CompactionSuiteBase]] and runs all the [[CompactionSuiteBase]] tests * with AutoCompaction. * * It also tests AutoCompaction specific behavior around compaction execution. */ class AutoCompactExecutionSuite extends CompactionTestHelperForAutoCompaction with DeltaSQLCommandTest with SharedSparkSession with AutoCompactTestUtils { private def testBothModesViaProperty(testName: String)(f: String => Unit): Unit = { def runTest(autoCompactConfValue: String): Unit = { withTempDir { dir => withSQLConf( "spark.databricks.delta.properties.defaults.autoOptimize.autoCompact" -> s"$autoCompactConfValue", DeltaSQLConf.DELTA_AUTO_COMPACT_MIN_NUM_FILES.key -> "0", DeltaSQLConf.DELTA_AUTO_COMPACT_MODIFIED_PARTITIONS_ONLY_ENABLED.key -> "false") { f(dir.getCanonicalPath) } } } test(s"auto-compact-enabled-property: $testName") { runTest(autoCompactConfValue = "true") } } private def testBothModesViaConf(testName: String)(f: String => Unit): Unit = { def runTest(autoCompactConfValue: String): Unit = { withTempDir { dir => withSQLConf( DeltaSQLConf.DELTA_AUTO_COMPACT_ENABLED.key -> s"$autoCompactConfValue", DeltaSQLConf.DELTA_AUTO_COMPACT_MIN_NUM_FILES.key -> "0") { f(dir.getCanonicalPath) } } } test(s"auto-compact-enabled-conf: $testName") { runTest(autoCompactConfValue = "true") } } private def checkAutoOptimizeLogging(f: => Unit): Boolean = { val logs = Log4jUsageLogger.track { f } logs.exists(_.opType.map(_.typeName) === Some("delta.commit.hooks.autoOptimize")) } import testImplicits._ test("auto compact event log: inline AC") { withTempDir { dir => withSQLConf( DeltaSQLConf.DELTA_AUTO_COMPACT_ENABLED.key -> s"true", DeltaSQLConf.DELTA_AUTO_COMPACT_MIN_NUM_FILES.key -> "30") { val path = dir.getCanonicalPath // Append 1 file to each partition: record runOnModifiedPartitions event, as is first write var usageLogs = captureOptimizeLogs(AutoCompact.OP_TYPE) { createFilesToPartitions(numFilePartitions = 3, numFilesPerPartition = 1, path) } var log = JsonUtils.mapper.readValue[Map[String, String]](usageLogs.head.blob) assert(log("status") == "runOnModifiedPartitions" && log("partitions") == "3") // Append 10 more file to each partition: record skipInsufficientFilesInModifiedPartitions // event. usageLogs = captureOptimizeLogs(AutoCompact.OP_TYPE) { createFilesToPartitions(numFilePartitions = 3, numFilesPerPartition = 10, path) } log = JsonUtils.mapper.readValue[Map[String, String]](usageLogs.head.blob) assert(log("status") == "skipInsufficientFilesInModifiedPartitions") // Append 20 more files to each partition: record runOnModifiedPartitions on all 3 // partitions. usageLogs = captureOptimizeLogs(AutoCompact.OP_TYPE) { createFilesToPartitions(numFilePartitions = 3, numFilesPerPartition = 20, path) } log = JsonUtils.mapper.readValue[Map[String, String]](usageLogs.head.blob) assert(log("status") == "runOnModifiedPartitions" && log("partitions") == "3") // Append 30 more file to each partition and check OptimizeMetrics. usageLogs = captureOptimizeLogs(metrics = s"${AutoCompact.OP_TYPE}.execute.metrics") { createFilesToPartitions(numFilePartitions = 3, numFilesPerPartition = 30, path) } val metricsLog = JsonUtils.mapper.readValue[OptimizeMetrics](usageLogs.head.blob) assert(metricsLog.numBytesSkippedToReduceWriteAmplification === 0) assert(metricsLog.numFilesSkippedToReduceWriteAmplification === 0) assert(metricsLog.totalConsideredFiles === 93) assert(metricsLog.numFilesAdded == 3) assert(metricsLog.numFilesRemoved == 93) assert(metricsLog.numBins === 3) } } } /** * Writes `df` twice to the same location and checks that * 1. There is only one resultant file. * 2. The result is equal to `df` unioned with itself. */ private def checkAutoCompactionWorks(dir: String, df: DataFrame): Unit = { df.write.format("delta").mode("append").save(dir) val deltaLog = DeltaLog.forTable(spark, dir) val newSnapshot = deltaLog.update() assert(newSnapshot.version === 1) // 0 is the first commit, 1 is optimize assert(deltaLog.update().numOfFiles === 1) val isLogged = checkAutoOptimizeLogging { df.write.format("delta").mode("append").save(dir) } assert(isLogged) val lastEvent = deltaLog.history.getHistory(Some(1)).head assert(lastEvent.operation === "OPTIMIZE") assert(lastEvent.operationParameters("auto") === "true") assert(deltaLog.update().numOfFiles === 1, "Files should be optimized into a single one") checkAnswer( df.union(df).toDF(), spark.read.format("delta").load(dir) ) } testBothModesViaProperty("auto compact should kick in when enabled - table config") { dir => checkAutoCompactionWorks(dir, spark.range(10).toDF("id")) } testBothModesViaConf("auto compact should kick in when enabled - session config") { dir => checkAutoCompactionWorks(dir, spark.range(10).toDF("id")) } test("variant auto compact kicks in when enabled - table config") { withTempDir { dir => withSQLConf( "spark.databricks.delta.properties.defaults.autoOptimize.autoCompact" -> "true", DeltaSQLConf.DELTA_AUTO_COMPACT_MIN_NUM_FILES.key -> "0", DeltaSQLConf.DELTA_AUTO_COMPACT_MODIFIED_PARTITIONS_ONLY_ENABLED.key -> "false") { checkAutoCompactionWorks( dir.getCanonicalPath, spark.range(10).selectExpr("parse_json(cast(id as string)) as v")) } } } test("variant auto compact kicks in when enabled - session config") { withTempDir { dir => withSQLConf( DeltaSQLConf.DELTA_AUTO_COMPACT_ENABLED.key -> "true", DeltaSQLConf.DELTA_AUTO_COMPACT_MIN_NUM_FILES.key -> "0") { checkAutoCompactionWorks( dir.getCanonicalPath, spark.range(10).selectExpr("parse_json(cast(id as string)) as v")) } } } testBothModesViaProperty("auto compact should not kick in when session config is off") { dir => withSQLConf(DeltaSQLConf.DELTA_AUTO_COMPACT_ENABLED.key -> "false") { val isLogged = checkAutoOptimizeLogging { spark.range(10).write.format("delta").mode("append").save(dir) } val deltaLog = DeltaLog.forTable(spark, dir) val newSnapshot = deltaLog.update() assert(newSnapshot.version === 0) // 0 is the first commit assert(deltaLog.update().numOfFiles > 1) assert(!isLogged) } } test("auto compact should not kick in after optimize") { withTempDir { tempDir => val dir = tempDir.getCanonicalPath spark.range(0, 12, 1, 4).write.format("delta").mode("append").save(dir) val deltaLog = DeltaLog.forTable(spark, dir) val newSnapshot = deltaLog.update() assert(newSnapshot.version === 0) assert(deltaLog.update().numOfFiles === 4) spark.sql(s"ALTER TABLE delta.`${tempDir.getCanonicalPath}` SET TBLPROPERTIES " + "(delta.autoOptimize.autoCompact = true)") val isLogged = checkAutoOptimizeLogging { sql(s"optimize delta.`$dir`") } assert(!isLogged) val lastEvent = deltaLog.history.getHistory(Some(1)).head assert(lastEvent.operation === "OPTIMIZE") assert(lastEvent.operationParameters("auto") === "false") } } testBothModesViaProperty("auto compact should not kick in when there aren't " + "enough files") { dir => withSQLConf(DeltaSQLConf.DELTA_AUTO_COMPACT_MIN_NUM_FILES.key -> "5") { AutoCompactPartitionStats.instance(spark).resetTestOnly() spark.range(10).repartition(4).write.format("delta").mode("append").save(dir) val deltaLog = DeltaLog.forTable(spark, dir) val newSnapshot = deltaLog.update() assert(newSnapshot.version === 0) assert(deltaLog.update().numOfFiles === 4) val isLogged2 = checkAutoOptimizeLogging { spark.range(10).repartition(4).write.format("delta").mode("append").save(dir) } assert(isLogged2) val lastEvent = deltaLog.history.getHistory(Some(1)).head assert(lastEvent.operation === "OPTIMIZE") assert(lastEvent.operationParameters("auto") === "true") assert(deltaLog.update().numOfFiles === 1, "Files should be optimized into a single one") checkAnswer( spark.read.format("delta").load(dir), spark.range(10).union(spark.range(10)).toDF() ) } } testBothModesViaProperty("ensure no NPE in auto compact UDF with null " + "partition values") { dir => Seq(null, "", " ").zipWithIndex.foreach { case (partValue, i) => val path = new File(dir, i.toString).getCanonicalPath val df1 = spark.range(5).withColumn("part", lit(partValue)) val df2 = spark.range(5, 10).withColumn("part", lit("1")) val isLogged = checkAutoOptimizeLogging { // repartition to increase number of files written df1.union(df2).repartition(4) .write.format("delta").partitionBy("part").mode("append").save(path) } val deltaLog = DeltaLog.forTable(spark, path) val newSnapshot = deltaLog.update() assert(newSnapshot.version === 1) // 0 is the first commit, 1 and 2 are optimizes assert(newSnapshot.numOfFiles === 2) assert(isLogged) val lastEvent = deltaLog.history.getHistory(Some(1)).head assert(lastEvent.operation === "OPTIMIZE") assert(lastEvent.operationParameters("auto") === "true") } } testBothModesViaProperty("check auto compact recorded metrics") { dir => val logs = Log4jUsageLogger.track { spark.range(30).repartition(3).write.format("delta").save(dir) } val metrics = JsonUtils.mapper.readValue[OptimizeMetrics](logs.filter( _.tags.get("opType") == Some(s"${AutoCompact.OP_TYPE}.execute.metrics")).head.blob) assert(metrics.numFilesRemoved == 3) assert(metrics.numFilesAdded == 1) } private def setTableProperty(log: DeltaLog, key: String, value: String): Unit = { spark.sql(s"ALTER TABLE delta.`${log.dataPath}` SET TBLPROPERTIES " + s"($key = $value)") } } class AutoCompactConfigurationIdColumnMappingSuite extends AutoCompactConfigurationSuite with DeltaColumnMappingEnableIdMode { override def runAllTests: Boolean = true } class AutoCompactExecutionIdColumnMappingSuite extends AutoCompactExecutionSuite with DeltaColumnMappingEnableIdMode { override def runAllTests: Boolean = true } class AutoCompactConfigurationNameColumnMappingSuite extends AutoCompactConfigurationSuite with DeltaColumnMappingEnableNameMode { override def runAllTests: Boolean = true } class AutoCompactExecutionNameColumnMappingSuite extends AutoCompactExecutionSuite with DeltaColumnMappingEnableNameMode { override def runAllTests: Boolean = true } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/BlockWritesLocalFileSystem.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.net.URI import java.util.concurrent.CountDownLatch import org.apache.spark.sql.delta.BlockWritesLocalFileSystem.scheme import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{DelegateToFileSystem, FSDataOutputStream, Path, RawLocalFileSystem} import org.apache.hadoop.util.Progressable /** * This custom fs implementation is used for testing the execution multiple batches of Optimize. */ class BlockWritesLocalFileSystem extends RawLocalFileSystem { private var uri: URI = _ override def getScheme: String = scheme override def initialize(name: URI, conf: Configuration): Unit = { uri = URI.create(name.getScheme + ":///") super.initialize(name, conf) } override def getUri(): URI = if (uri == null) { // RawLocalFileSystem's constructor will call this one before `initialize` is called. // Just return the super's URI to avoid NPE. super.getUri } else { uri } override def create( f: Path, overwrite: Boolean, bufferSize: Int, replication: Short, blockSize: Long, progress: Progressable): FSDataOutputStream = { // called when data files and log files are written BlockWritesLocalFileSystem.blockLatch.countDown() BlockWritesLocalFileSystem.blockLatch.await() super.create(f, overwrite, bufferSize, replication, blockSize, progress) } } /** * An AbstractFileSystem implementation wrapper around [[BlockWritesLocalFileSystem]]. */ class BlockWritesAbstractFileSystem(uri: URI, conf: Configuration) extends DelegateToFileSystem( uri, new BlockWritesLocalFileSystem, conf, BlockWritesLocalFileSystem.scheme, false) /** * Singleton for BlockWritesLocalFileSystem used to initialize the file system countdown latch. */ object BlockWritesLocalFileSystem { val scheme = "block" /** latch that blocks writes */ private var blockLatch: CountDownLatch = _ /** * @param numWrites - writing is blocked until there are `numWrites` concurrent writes to * the file system. */ def blockUntilConcurrentWrites(numWrites: Integer): Unit = { blockLatch = new CountDownLatch(numWrites) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/CheckCDCAnswer.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.sql.Timestamp import org.apache.spark.sql.delta.commands.cdc.CDCReader.{CDC_COMMIT_TIMESTAMP, CDC_COMMIT_VERSION} import org.apache.spark.sql.{DataFrame, QueryTest, Row} trait CheckCDCAnswer extends QueryTest { /** * Check the result of a CDC operation. The expected answer should include only CDC type and * log version - the timestamp is nondeterministic, so we'll check just that it matches the * correct value in the Delta log. * * @param log The Delta log for the table CDC is being extracted from. * @param df The computed dataframe, which should match the default CDC result schema. * Callers doing projections on top should use checkAnswer directly. * @param expectedAnswer The expected results for the CDC query, excluding the CDC_LOG_TIMESTAMP * column which we handle inside this method. */ def checkCDCAnswer(log: DeltaLog, df: => DataFrame, expectedAnswer: Seq[Row]): Unit = { checkAnswer(df.drop(CDC_COMMIT_TIMESTAMP), expectedAnswer) val timestampsByVersion = df.select(CDC_COMMIT_VERSION, CDC_COMMIT_TIMESTAMP).collect() .map { row => val version = row.getLong(0) val ts = row.getTimestamp(1) (version -> ts) }.toMap val correctTimestampsByVersion = { // Results should match the fully monotonized commits. Note that this map will include // all versions of the table but only the ones in timestampsByVersion are checked for // correctness. val commits = log.history.getHistory(start = 0, end = None) // Note that the timestamps are in milliseconds since epoch and we don't need to deal // with timezones. commits.map(f => (f.getVersion -> f.timestamp)).toMap } timestampsByVersion.keySet.foreach { version => assert(timestampsByVersion(version) === correctTimestampsByVersion(version)) } } def checkCDCAnswer(log: DeltaLog, df: => DataFrame, expectedAnswer: DataFrame): Unit = { checkCDCAnswer(log, df, expectedAnswer.collect()) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/CheckpointInstanceSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.CheckpointInstance.Format import org.apache.spark.SparkFunSuite class CheckpointInstanceSuite extends SparkFunSuite { test("checkpoint instance comparisons") { val ci1_single_1 = CheckpointInstance(1, Format.SINGLE, numParts = None) val ci1_withparts_2 = CheckpointInstance(1, Format.WITH_PARTS, numParts = Some(2)) val ci1_sentinel = CheckpointInstance.sentinelValue(Some(1)) val ci2_single_1 = CheckpointInstance(2, Format.SINGLE, numParts = None) val ci2_withparts_4 = CheckpointInstance(2, Format.WITH_PARTS, numParts = Some(4)) val ci2_sentinel = CheckpointInstance.sentinelValue(Some(2)) val ci3_single_1 = CheckpointInstance(3, Format.SINGLE, numParts = None) val ci3_withparts_2 = CheckpointInstance(3, Format.WITH_PARTS, numParts = Some(2)) assert(ci1_single_1 < ci2_single_1) // version takes priority assert(ci1_single_1 < ci1_withparts_2) // parts takes priority when versions are same assert(ci2_withparts_4 < ci3_withparts_2) // version takes priority over parts // all checkpoint instances for version 1/2 are less than sentinel value for version 2. Seq(ci1_single_1, ci1_withparts_2, ci1_sentinel, ci2_single_1, ci2_withparts_4) .foreach(ci => assert(ci < ci2_sentinel)) // all checkpoint instances for version 3 are greater than sentinel value for version 2. Seq(ci3_single_1, ci3_withparts_2).foreach(ci => assert(ci > ci2_sentinel)) // Everything is less than CheckpointInstance.MaxValue Seq( ci1_single_1, ci1_withparts_2, ci1_sentinel, ci2_single_1, ci2_withparts_4, ci2_sentinel, ci3_single_1, ci3_withparts_2 ).foreach(ci => assert(ci < CheckpointInstance.MaxValue)) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/CheckpointProtectionTestUtilsMixin.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.util.concurrent.TimeUnit import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.FileNames import org.apache.spark.util.ManualClock trait CheckpointProtectionTestUtilsMixin extends DeltaSQLCommandTest { self: DeltaRetentionSuiteBase => // scalastyle:off argcount def testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod: Int, createNumCommitsWithinRetentionPeriod: Int, createCheckpoints: Set[Int], requireCheckpointProtectionBeforeVersion: Int, additionalFeatureToEnable: Option[TableFeature] = None, unsupportedFeature: TableFeature = TestUnsupportedNoHistoryProtectionReaderWriterFeature, unsupportedFeatureStartVersion: Option[Long] = None, unsupportedFeatureEndVersion: Option[Long] = None, incompleteCRCVersion: Option[Long] = None, missingCRCVersion: Option[Long] = None, expectedCommitsAfterCleanup: Seq[Int], expectedCheckpointsAfterCleanup: Set[Int]): Unit = { // scalastyle:on argcount withTempDir { dir => val currentTime = System.currentTimeMillis() val clock = new ManualClock(currentTime) val deltaLog = DeltaLog.forTable(spark, dir, clock) val fs = deltaLog.logPath.getFileSystem(deltaLog.newDeltaHadoopConf()) val propertyKey = DeltaConfigs.REQUIRE_CHECKPOINT_PROTECTION_BEFORE_VERSION.key val additionalFeatureEnablement = additionalFeatureToEnable.map(f => s"delta.feature.${f.name} = 'supported',") .getOrElse("") val featureEnablement = s"delta.feature.${unsupportedFeature.name} = 'supported'" val featureEnablementAtCreateTable = if (unsupportedFeatureStartVersion.exists(_ == 0)) s"$featureEnablement," else "" // Commit 0. sql( s"""CREATE TABLE delta.`${deltaLog.dataPath}` (id bigint) USING delta |TBLPROPERTIES ( |delta.feature.${CheckpointProtectionTableFeature.name} = 'supported', |$additionalFeatureEnablement |${featureEnablementAtCreateTable} |$propertyKey = $requireCheckpointProtectionBeforeVersion |)""".stripMargin) if (createCheckpoints.contains(0)) deltaLog.checkpoint(deltaLog.update()) setModificationTime(deltaLog, startTime = currentTime, version = 0, dayNum = 0, fs) def createCommit(version: Int): Unit = { if (unsupportedFeatureStartVersion.exists(_ == version)) { sql(s"ALTER TABLE delta.`${deltaLog.dataPath}` SET TBLPROPERTIES ($featureEnablement)") } else if (unsupportedFeatureEndVersion.exists(_ == version)) { sql( s"""ALTER TABLE delta.`${deltaLog.dataPath}` |DROP FEATURE ${unsupportedFeature.name} |""".stripMargin) } else { spark.range(version, version + 1) .write .format("delta") .mode("append") .save(dir.getCanonicalPath) } if (createCheckpoints.contains(version)) deltaLog.checkpoint(deltaLog.update()) } // Rest createNumCommitsOutsideRetentionPeriod - 1 commits. for (n <- 1 to createNumCommitsOutsideRetentionPeriod - 1) { createCommit(n) setModificationTime(deltaLog, startTime = currentTime, version = n, dayNum = 0, fs) } val millisToAdvance = intervalStringToMillis(DeltaConfigs.LOG_RETENTION.defaultValue) + TimeUnit.DAYS.toMillis(3) clock.advance(millisToAdvance) // Commits within retention period. val daysToAdvance = TimeUnit.MILLISECONDS.toDays(millisToAdvance).toInt for (n <- 0 to createNumCommitsWithinRetentionPeriod - 1) { val m = createNumCommitsOutsideRetentionPeriod + n createCommit(m) // Advance the timestamp of the commit/checkpoint we just created. setModificationTime( deltaLog, startTime = currentTime, version = m, // The files were created somewhere between day 32 and day 33. dayNum = daysToAdvance - 1, fs) } incompleteCRCVersion.foreach { version => val checksumFilePath = FileNames.checksumFile(deltaLog.logPath, version) removeProtocolAndMetadataFromChecksumFile(checksumFilePath) } missingCRCVersion.foreach { version => val checksumFilePath = FileNames.checksumFile(deltaLog.logPath, version) (new File(checksumFilePath.toUri)).delete() } deltaLog.cleanUpExpiredLogs(deltaLog.update()) val logPath = new File(deltaLog.logPath.toUri) assert(getDeltaVersions(logPath).toSeq.sorted === expectedCommitsAfterCleanup.sorted) assert(getCheckpointVersions(logPath) === expectedCheckpointsAfterCleanup) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/CheckpointProviderSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.actions.{Action} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.util.FileNames._ import org.apache.spark.sql.test.SharedSparkSession class CheckpointProviderSuite extends SharedSparkSession with DeltaSQLCommandTest { for (v2CheckpointFormat <- Seq("json", "parquet")) test(s"V2 Checkpoint compat file equivalency to normal V2 Checkpoint" + s" [v2CheckpointFormat: $v2CheckpointFormat]") { withSQLConf( DeltaConfigs.CHECKPOINT_POLICY.defaultTablePropertyKey -> CheckpointPolicy.V2.name, DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key -> v2CheckpointFormat ) { withTempDir { tempDir => spark.range(10).write.format("delta").save(tempDir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, tempDir.getAbsolutePath) spark.range(10).write.mode("append").format("delta").save(tempDir.getAbsolutePath) deltaLog.checkpoint() // Checkpoint 1 val snapshot = deltaLog.update() deltaLog.createSinglePartCheckpointForBackwardCompat( snapshot, new deltaLog.V2CompatCheckpointMetrics) // Compatibility Checkpoint 1 val fs = deltaLog.logPath.getFileSystem(deltaLog.newDeltaHadoopConf()) val v2CompatCheckpoint = fs.getFileStatus( checkpointFileSingular(deltaLog.logPath, snapshot.checkpointProvider.version)) val origCheckpoint = snapshot.checkpointProvider .asInstanceOf[LazyCompleteCheckpointProvider] .underlyingCheckpointProvider .asInstanceOf[V2CheckpointProvider] val compatCheckpoint = CheckpointProvider( spark, deltaLog.snapshot, None, UninitializedV2CheckpointProvider( 2L, v2CompatCheckpoint, deltaLog.logPath, deltaLog.newDeltaHadoopConf(), deltaLog.options, deltaLog.store, None)) .asInstanceOf[LazyCompleteCheckpointProvider] .underlyingCheckpointProvider .asInstanceOf[V2CheckpointProvider] // Check whether these checkpoints are equivalent after being loaded assert(compatCheckpoint.sidecarFiles.toSet === origCheckpoint.sidecarFiles.toSet) assert(compatCheckpoint.checkpointMetadata === origCheckpoint.checkpointMetadata) val compatDf = deltaLog.loadIndex(compatCheckpoint.topLevelFileIndex.get, Action.logSchema) // Check whether the manifest content is same or not val originalDf = deltaLog.loadIndex(origCheckpoint.topLevelFileIndex.get, Action.logSchema) assert(originalDf.sort().collect() === compatDf.sort().collect()) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/CheckpointsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.net.URI import java.util.UUID import scala.concurrent.duration._ // scalastyle:off import.ordering.noEmptyLine import com.databricks.spark.util.{Log4jUsageLogger, MetricDefinitions, UsageRecord} import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.deletionvectors.DeletionVectorsSuite import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.storage.LocalLogStore import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.shims.VariantStatsShims import org.apache.spark.sql.delta.util.{Codec, DeltaCommitFileProvider, DeltaStatsJsonUtils} import org.apache.spark.sql.delta.util.FileNames import org.apache.commons.io.FileUtils import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, FSDataOutputStream, Path, RawLocalFileSystem} import org.apache.hadoop.fs.permission.FsPermission import org.apache.hadoop.util.Progressable import org.apache.spark.SparkConf import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.col import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.StructType import org.apache.spark.types.variant.{Variant, VariantUtil} import org.apache.spark.unsafe.types.VariantVal class CheckpointsSuite extends QueryTest with SharedSparkSession with DeltaCheckpointTestUtils with DeltaSQLCommandTest with DeltaSQLTestUtils with CatalogOwnedTestBaseSuite { def testDifferentV2Checkpoints(testName: String)(f: => Unit): Unit = { for (checkpointFormat <- Seq(V2Checkpoint.Format.JSON.name, V2Checkpoint.Format.PARQUET.name)) { test(s"$testName [v2CheckpointFormat: $checkpointFormat]") { withSQLConf( DeltaConfigs.CHECKPOINT_POLICY.defaultTablePropertyKey -> CheckpointPolicy.V2.name, DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key -> checkpointFormat ) { f } } } } /** Get V2 [[CheckpointProvider]] from the underlying deltalog snapshot */ def getV2CheckpointProvider( deltaLog: DeltaLog, update: Boolean = true): V2CheckpointProvider = { val snapshot = if (update) deltaLog.update() else deltaLog.unsafeVolatileSnapshot snapshot.checkpointProvider match { case v2CheckpointProvider: V2CheckpointProvider => v2CheckpointProvider case provider : LazyCompleteCheckpointProvider if provider.underlyingCheckpointProvider.isInstanceOf[V2CheckpointProvider] => provider.underlyingCheckpointProvider.asInstanceOf[V2CheckpointProvider] case EmptyCheckpointProvider => throw new IllegalStateException("underlying snapshot doesn't have a checkpoint") case other => throw new IllegalStateException(s"The underlying checkpoint is not a v2 checkpoint. " + s"It is: ${other.getClass.getName}") } } protected override def sparkConf = { // Set the gs LogStore impl to `LocalLogStore` so that it will work with // `FakeGCSFileSystemValidatingCheckpoint`. // The default one is `HDFSLogStore` which requires a `FileContext` but we don't have one. super.sparkConf.set("spark.delta.logStore.gs.impl", classOf[LocalLogStore].getName) } test("checkpoint metadata - checkpoint schema above the configured threshold are not" + " written to LAST_CHECKPOINT") { withClassicCheckpointPolicyForCatalogOwned { withTempTable(createTable = false) { tableName => spark.range(10).write.format("delta").saveAsTable(tableName) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) deltaLog.checkpoint() val lastCheckpointOpt = deltaLog.readLastCheckpointFile() assert(lastCheckpointOpt.nonEmpty) assert(lastCheckpointOpt.get.checkpointSchema.nonEmpty) val expectedCheckpointSchema = Seq("txn", "add", "remove", "metaData", "protocol", "domainMetadata") assert(lastCheckpointOpt.get.checkpointSchema.get.fieldNames.toSeq === expectedCheckpointSchema) spark.range(10).write.mode("append").format("delta").saveAsTable(tableName) withSQLConf(DeltaSQLConf.CHECKPOINT_SCHEMA_WRITE_THRESHOLD_LENGTH.key -> "10") { deltaLog.checkpoint() val lastCheckpointOpt = deltaLog.readLastCheckpointFile() assert(lastCheckpointOpt.nonEmpty) assert(lastCheckpointOpt.get.checkpointSchema.isEmpty) } } } } testDifferentV2Checkpoints("checkpoint metadata - checkpoint schema not persisted in" + " json v2 checkpoints but persisted in parquet v2 checkpoints") { withTempDir { tempDir => spark.range(10).write.format("delta").save(tempDir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, tempDir.getAbsolutePath) deltaLog.checkpoint() val lastCheckpointOpt = deltaLog.readLastCheckpointFile() assert(lastCheckpointOpt.nonEmpty) val expectedFormat = spark.conf.getOption(DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key) assert(lastCheckpointOpt.get.checkpointSchema.isEmpty === (expectedFormat.contains(V2Checkpoint.Format.JSON.name))) } } testDifferentCheckpoints("test empty checkpoints") { (checkpointPolicy, _) => val tableName = "test_empty_table" withTable(tableName) { sql(s"CREATE TABLE `$tableName` (a INT) USING DELTA") sql(s"ALTER TABLE `$tableName` SET TBLPROPERTIES('comment' = 'A table comment')") val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) deltaLog.checkpoint() def validateSnapshot(snapshot: Snapshot): Unit = { assert(!snapshot.checkpointProvider.isEmpty) assert(snapshot.checkpointProvider.version === 1) val checkpointFile = snapshot.checkpointProvider.topLevelFiles.head.getPath val fileActions = getCheckpointDfForFilesContainingFileActions(deltaLog, checkpointFile) assert(fileActions.where("add is not null or remove is not null").collect().size === 0) if (checkpointPolicy == CheckpointPolicy.V2) { val v2CheckpointProvider = snapshot.checkpointProvider match { case lazyCompleteCheckpointProvider: LazyCompleteCheckpointProvider => lazyCompleteCheckpointProvider.underlyingCheckpointProvider .asInstanceOf[V2CheckpointProvider] case cp: V2CheckpointProvider => cp case _ => throw new IllegalStateException("Unexpected checkpoint provider") } assert(v2CheckpointProvider.sidecarFiles.size === 1) val sidecar = v2CheckpointProvider.sidecarFiles.head.toFileStatus(deltaLog.logPath) assert(spark.read.parquet(sidecar.getPath.toString).count() === 0) } } validateSnapshot(deltaLog.update()) DeltaLog.clearCache() validateSnapshot(DeltaLog.forTable(spark, TableIdentifier(tableName)).unsafeVolatileSnapshot) } } testDifferentV2Checkpoints(s"V2 Checkpoint write test" + s" - metadata, protocol, sidecar, checkpoint metadata actions") { withTempDir { tempDir => spark.range(10).write.format("delta").save(tempDir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, tempDir.getAbsolutePath) deltaLog.checkpoint() val checkpointFiles = deltaLog.listFrom(0).filter(FileNames.isCheckpointFile).toList assert(checkpointFiles.length == 1) val checkpoint = checkpointFiles.head val fileNameParts = checkpoint.getPath.getName.split("\\.") // The file name should be .checkpoint..parquet. assert(fileNameParts.length == 4) fileNameParts match { case Array(version, checkpointLiteral, _, format) => val expectedFormat = spark.conf.getOption(DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key).get assert(format == expectedFormat) assert(version.toLong == 0) assert(checkpointLiteral == "checkpoint") } def getCheckpointFileActions(checkpoint: FileStatus) : Seq[Action] = { if (checkpoint.getPath.toString.endsWith("json")) { deltaLog.store.read(checkpoint.getPath).map(Action.fromJson) } else { val fileIndex = DeltaLogFileIndex(DeltaLogFileIndex.CHECKPOINT_FILE_FORMAT_PARQUET, Seq(checkpoint)).get deltaLog.loadIndex(fileIndex, Action.logSchema) .as[SingleAction].collect().map(_.unwrap).toSeq } } val actions = getCheckpointFileActions(checkpoint) // V2 Checkpoints should contain exactly one action each of types // Metadata, CheckpointMetadata, and Protocol // In this particular case, we should only have one sidecar file val sidecarActions = actions.collect{ case s: SidecarFile => s} assert(sidecarActions.length == 1) val sidecarPath = sidecarActions.head.path assert(sidecarPath.endsWith("parquet")) val metadataActions = actions.collect { case m: Metadata => m } assert(metadataActions.length == 1) val checkpointMetadataActions = actions.collect { case cm: CheckpointMetadata => cm } assert(checkpointMetadataActions.length == 1) assert( DeltaConfigs.CHECKPOINT_POLICY.fromMetaData(metadataActions.head) .needsV2CheckpointSupport ) val protocolActions = actions.collect { case p: Protocol => p } assert(protocolActions.length == 1) assert(CheckpointProvider.isV2CheckpointEnabled(protocolActions.head)) } } test("SC-86940: isGCSPath") { val conf = new Configuration() assert(Checkpoints.isGCSPath(conf, new Path("gs://foo/bar"))) // Scheme is case insensitive assert(Checkpoints.isGCSPath(conf, new Path("Gs://foo/bar"))) assert(Checkpoints.isGCSPath(conf, new Path("GS://foo/bar"))) assert(Checkpoints.isGCSPath(conf, new Path("gS://foo/bar"))) assert(!Checkpoints.isGCSPath(conf, new Path("non-gs://foo/bar"))) assert(!Checkpoints.isGCSPath(conf, new Path("/foo"))) // Set the default file system and verify we can detect it conf.set("fs.defaultFS", "gs://foo/") conf.set("fs.gs.impl", classOf[FakeGCSFileSystemValidatingCheckpoint].getName) conf.set("fs.gs.impl.disable.cache", "true") assert(Checkpoints.isGCSPath(conf, new Path("/foo"))) } test("SC-86940: writing a GCS checkpoint should happen in a new thread") { withTempDir { tempDir => // Use `FakeGCSFileSystemValidatingCheckpoint` which will verify we write in a separate gcs // thread. withSQLConf( "fs.gs.impl" -> classOf[FakeGCSFileSystemValidatingCheckpoint].getName, "fs.gs.impl.disable.cache" -> "true") { val gsPath = s"gs://${tempDir.getCanonicalPath}" val writer = spark.range(1).write.format("delta") if (catalogOwnedDefaultCreationEnabledInTests) { // Setting checkpointPolicy=classic because this test is intended for v1 checkpoint only. writer.option(DeltaConfigs.CHECKPOINT_POLICY.key, "classic") } writer.save(gsPath) DeltaLog.clearCache() val deltaLog = DeltaLog.forTable(spark, new Path(gsPath)) deltaLog.checkpoint() } } } private def verifyCheckpoint( checkpoint: Option[LastCheckpointInfo], version: Int, parts: Option[Int]): Unit = { assert(checkpoint.isDefined) checkpoint.foreach { lastCheckpointInfo => assert(lastCheckpointInfo.version == version) assert(lastCheckpointInfo.parts == parts) } } test("multipart checkpoints") { withClassicCheckpointPolicyForCatalogOwned { withTempTable(createTable = false) { tableName => withSQLConf( DeltaSQLConf.DELTA_CHECKPOINT_PART_SIZE.key -> "10", DeltaConfigs.CHECKPOINT_INTERVAL.defaultTablePropertyKey -> "1") { // 1 file actions spark.range(1).repartition(1).write.format("delta").saveAsTable(tableName) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) // 2 file actions, 1 new file spark.range(1).repartition(1).write.format("delta").mode("append").saveAsTable(tableName) verifyCheckpoint(deltaLog.readLastCheckpointFile(), 1, None) val checkpointPath = FileNames.checkpointFileSingular(deltaLog.logPath, deltaLog.snapshot.version).toUri assert(new File(checkpointPath).exists()) // 11 total file actions, 9 new files spark.range(30).repartition(9).write.format("delta").mode("append").saveAsTable(tableName) verifyCheckpoint(deltaLog.readLastCheckpointFile(), 2, Some(2)) var checkpointPaths = FileNames.checkpointFileWithParts(deltaLog.logPath, deltaLog.snapshot.version, 2) checkpointPaths.foreach(p => assert(new File(p.toUri).exists())) // 20 total actions, 9 new files spark .range(100) .repartition(9) .write .format("delta") .mode("append") .saveAsTable(tableName) verifyCheckpoint(deltaLog.readLastCheckpointFile(), 3, Some(2)) assert(deltaLog.snapshot.version == 3) checkpointPaths = FileNames.checkpointFileWithParts(deltaLog.logPath, deltaLog.snapshot.version, 2) checkpointPaths.foreach(p => assert(new File(p.toUri).exists())) // 31 total actions, 11 new files spark .range(100) .repartition(11) .write .format("delta") .mode("append") .saveAsTable(tableName) verifyCheckpoint(deltaLog.readLastCheckpointFile(), 4, Some(4)) assert(deltaLog.snapshot.version == 4) checkpointPaths = FileNames.checkpointFileWithParts(deltaLog.logPath, deltaLog.snapshot.version, 4) checkpointPaths.foreach(p => assert(new File(p.toUri).exists())) } // Increase max actions withSQLConf(DeltaSQLConf.DELTA_CHECKPOINT_PART_SIZE.key -> "100") { val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) // 100 total actions, 69 new files spark.range(1000) .repartition(69).write.format("delta").mode("append").saveAsTable(tableName) verifyCheckpoint(deltaLog.readLastCheckpointFile(), 5, None) val checkpointPath = FileNames.checkpointFileSingular(deltaLog.logPath, deltaLog.snapshot.version).toUri assert(new File(checkpointPath).exists()) // 101 total actions, 1 new file spark.range(1).repartition(1).write.format("delta").mode("append").saveAsTable(tableName) verifyCheckpoint(deltaLog.readLastCheckpointFile(), 6, Some(2)) var checkpointPaths = FileNames.checkpointFileWithParts(deltaLog.logPath, deltaLog.snapshot.version, 2) checkpointPaths.foreach(p => assert(new File(p.toUri).exists())) } } } } testDifferentV2Checkpoints("multipart v2 checkpoint") { withTempDir { tempDir => val path = tempDir.getCanonicalPath withSQLConf( DeltaSQLConf.DELTA_CHECKPOINT_PART_SIZE.key -> "10", DeltaConfigs.CHECKPOINT_POLICY.defaultTablePropertyKey -> CheckpointPolicy.V2.name, DeltaConfigs.CHECKPOINT_INTERVAL.defaultTablePropertyKey -> "1") { // 1 file actions spark.range(1).repartition(1).write.format("delta").save(path) val deltaLog = DeltaLog.forTable(spark, path) def getNumFilesInSidecarDirectory(): Int = { val fs = deltaLog.sidecarDirPath.getFileSystem(deltaLog.newDeltaHadoopConf()) fs.listStatus(deltaLog.sidecarDirPath).size } // 2 file actions, 1 new file spark.range(1).repartition(1).write.format("delta").mode("append").save(path) assert(getV2CheckpointProvider(deltaLog).version == 1) assert(getV2CheckpointProvider(deltaLog).sidecarFileStatuses.size == 1) assert(getNumFilesInSidecarDirectory() == 1) // 11 total file actions, 9 new files spark.range(30).repartition(9).write.format("delta").mode("append").save(path) assert(getV2CheckpointProvider(deltaLog).version == 2) assert(getV2CheckpointProvider(deltaLog).sidecarFileStatuses.size == 2) assert(getNumFilesInSidecarDirectory() == 3) // 20 total actions, 9 new files spark.range(100).repartition(9).write.format("delta").mode("append").save(path) assert(getV2CheckpointProvider(deltaLog).version == 3) assert(getV2CheckpointProvider(deltaLog).sidecarFileStatuses.size == 2) assert(getNumFilesInSidecarDirectory() == 5) // 31 total actions, 11 new files spark.range(100).repartition(11).write.format("delta").mode("append").save(path) assert(getV2CheckpointProvider(deltaLog).version == 4) assert(getV2CheckpointProvider(deltaLog).sidecarFileStatuses.size == 4) assert(getNumFilesInSidecarDirectory() == 9) // Increase max actions withSQLConf(DeltaSQLConf.DELTA_CHECKPOINT_PART_SIZE.key -> "100") { // 100 total actions, 69 new files spark.range(1000).repartition(69).write.format("delta").mode("append").save(path) assert(getV2CheckpointProvider(deltaLog).version == 5) assert(getV2CheckpointProvider(deltaLog).sidecarFileStatuses.size == 1) assert(getNumFilesInSidecarDirectory() == 10) // 101 total actions, 1 new file spark.range(1).repartition(1).write.format("delta").mode("append").save(path) assert(getV2CheckpointProvider(deltaLog).version == 6) assert(getV2CheckpointProvider(deltaLog).sidecarFileStatuses.size == 2) assert(getNumFilesInSidecarDirectory() == 12) } } } } test("checkpoint does not contain CDC field") { withSQLConf( DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> "true" ) { withTempDir { tempDir => withTempView("src") { spark.range(10).write.format("delta").save(tempDir.getAbsolutePath) spark.range(5, 15).createOrReplaceTempView("src") sql( s""" |MERGE INTO delta.`$tempDir` t USING src s ON t.id = s.id |WHEN MATCHED THEN DELETE |WHEN NOT MATCHED THEN INSERT * |""".stripMargin) checkAnswer( spark.read.format("delta").load(tempDir.getAbsolutePath), Seq(0, 1, 2, 3, 4, 10, 11, 12, 13, 14).map { i => Row(i) }) // CDC should exist in the log as seen through getChanges, but it shouldn't be in the // snapshots and the checkpoint file shouldn't have a CDC column. val deltaLog = DeltaLog.forTable(spark, tempDir.getAbsolutePath) val deltaPath = DeltaCommitFileProvider(deltaLog.unsafeVolatileSnapshot) .deltaFile(version = 1) val deltaFileContent = deltaLog.store.read(deltaPath, deltaLog.newDeltaHadoopConf()) assert(deltaFileContent.map(Action.fromJson).exists(_.isInstanceOf[AddCDCFile])) assert(deltaLog.snapshot.stateDS.collect().forall { sa => sa.cdc == null }) deltaLog.checkpoint() val checkpointPathStr = DeltaLog.forTableWithSnapshot(spark, tempDir.getAbsolutePath)._2 .checkpointProvider.topLevelFiles.head.getPath.toString val checkpointFormat = checkpointPathStr.substring(checkpointPathStr.lastIndexOf('.') + 1) val checkpointSchema = spark.read.format(checkpointFormat).load(checkpointPathStr).schema var expectedCheckpointSchema = Seq( "txn", "add", "remove", "metaData", "protocol", "domainMetadata") // For CCv1.5 table, v2 checkpoints is enabled by default. if (catalogOwnedDefaultCreationEnabledInTests) { // V2 checkpoint's schema is shared by sidecar files (contains all file actions) // and the main v2 checkpoint file (contains all non-file actions). // So file actions (e.g. `txn`, `add`, `remove`) are not included in the main v2 // checkpoint file. expectedCheckpointSchema = Seq( "checkpointMetadata", "domainMetadata", "metaData", "protocol", "sidecar") } assert(checkpointSchema.fieldNames.toSeq == expectedCheckpointSchema) } } } } testDifferentV2Checkpoints("v2 checkpoint contains only addfile and removefile and" + " remove file does not contain remove.tags and remove.numRecords") { withSQLConf( DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> "true" ) { val expectedCheckpointSchema = Seq("add", "remove") val expectedRemoveFileSchema = Seq( "path", "deletionTimestamp", "dataChange", "extendedFileMetadata", "partitionValues", "size", "deletionVector", "baseRowId", "defaultRowCommitVersion") withTempDir { tempDir => withTempView("src") { val tablePath = tempDir.getAbsolutePath // Append rows [0, 9] to table and merge tablePath. spark.range(end = 10).write.format("delta").mode("overwrite").save(tablePath) spark.range(5, 15).createOrReplaceTempView("src") sql( s""" |MERGE INTO delta.`$tempDir` t USING src s ON t.id = s.id |WHEN MATCHED THEN DELETE |WHEN NOT MATCHED THEN INSERT * |""".stripMargin) checkAnswer( spark.read.format("delta").load(tempDir.getAbsolutePath), Seq(0, 1, 2, 3, 4, 10, 11, 12, 13, 14).map { i => Row(i) }) // CDC should exist in the log as seen through getChanges, but it shouldn't be in the // snapshots and the checkpoint file shouldn't have a CDC column. val deltaLog = DeltaLog.forTable(spark, tempDir.getAbsolutePath) val deltaPath = DeltaCommitFileProvider(deltaLog.unsafeVolatileSnapshot) .deltaFile(version = 1) val deltaFileContent = deltaLog.store.read(deltaPath, deltaLog.newDeltaHadoopConf()) assert(deltaFileContent.map(Action.fromJson).exists(_.isInstanceOf[AddCDCFile])) assert(deltaLog.snapshot.stateDS.collect().forall { sa => sa.cdc == null }) deltaLog.checkpoint() var sidecarCheckpointFiles = getV2CheckpointProvider(deltaLog).sidecarFileStatuses assert(sidecarCheckpointFiles.size == 1) var sidecarFile = sidecarCheckpointFiles.head.getPath.toString var checkpointSchema = spark.read.format("parquet").load(sidecarFile).schema var removeSchemaName = checkpointSchema("remove").dataType.asInstanceOf[StructType].fieldNames assert(checkpointSchema.fieldNames.toSeq == expectedCheckpointSchema) assert(removeSchemaName.toSeq === expectedRemoveFileSchema) // Append rows [0, 9] to table and merge one more time. spark.range(end = 10).write.format("delta").mode("append").save(tablePath) sql( s""" |MERGE INTO delta.`$tempDir` t USING src s ON t.id = s.id |WHEN MATCHED THEN DELETE |WHEN NOT MATCHED THEN INSERT * |""".stripMargin) deltaLog.checkpoint() sidecarCheckpointFiles = getV2CheckpointProvider(deltaLog).sidecarFileStatuses sidecarFile = sidecarCheckpointFiles.head.getPath.toString checkpointSchema = spark.read.format(source = "parquet").load(sidecarFile).schema removeSchemaName = checkpointSchema("remove").dataType.asInstanceOf[StructType].fieldNames assert(removeSchemaName.toSeq === expectedRemoveFileSchema) checkAnswer( spark.sql(s"select * from delta.`$tablePath`"), Seq(0, 0, 1, 1, 2, 2, 3, 3, 4, 4).map { i => Row(i) }) } } } } test("checkpoint does not contain remove.tags and remove.numRecords") { withClassicCheckpointPolicyForCatalogOwned { withTempDir { tempDir => val expectedRemoveFileSchema = Seq( "path", "deletionTimestamp", "dataChange", "extendedFileMetadata", "partitionValues", "size", "deletionVector", "baseRowId", "defaultRowCommitVersion") val tablePath = tempDir.getAbsolutePath // Append rows [0, 9] to table and merge tablePath. spark.range(end = 10).write.format("delta").mode("overwrite").save(tablePath) spark.range(5, 15).createOrReplaceTempView("src") sql( s""" |MERGE INTO delta.`$tempDir` t USING src s ON t.id = s.id |WHEN MATCHED THEN DELETE |WHEN NOT MATCHED THEN INSERT * |""".stripMargin) val deltaLog = DeltaLog.forTable(spark, tablePath) deltaLog.checkpoint() var checkpointFile = FileNames.checkpointFileSingular(deltaLog.logPath, 1).toString var checkpointSchema = spark.read.format(source = "parquet").load(checkpointFile).schema var removeSchemaName = checkpointSchema("remove").dataType.asInstanceOf[StructType].fieldNames assert(removeSchemaName.toSeq === expectedRemoveFileSchema) checkAnswer( spark.sql(s"select * from delta.`$tablePath`"), Seq(0, 1, 2, 3, 4, 10, 11, 12, 13, 14).map { i => Row(i) }) // Append rows [0, 9] to table and merge one more time. spark.range(end = 10).write.format("delta").mode("append").save(tablePath) sql( s""" |MERGE INTO delta.`$tempDir` t USING src s ON t.id = s.id |WHEN MATCHED THEN DELETE |WHEN NOT MATCHED THEN INSERT * |""".stripMargin) deltaLog.checkpoint() checkpointFile = FileNames.checkpointFileSingular(deltaLog.logPath, 1).toString checkpointSchema = spark.read.format(source = "parquet").load(checkpointFile).schema removeSchemaName = checkpointSchema("remove").dataType.asInstanceOf[StructType].fieldNames assert(removeSchemaName.toSeq === expectedRemoveFileSchema) checkAnswer( spark.sql(s"select * from delta.`$tablePath`"), Seq(0, 0, 1, 1, 2, 2, 3, 3, 4, 4).map { i => Row(i) }) } } } test("checkpoint with DVs") { for (v2Checkpoint <- Seq(true, false)) withTempDir { tempDir => val source = new File(DeletionVectorsSuite.table1Path) // this table has DVs in two versions val targetName = s"insertTest_${UUID.randomUUID().toString.replace("-", "")}" val target = new File(tempDir, targetName) // Copy the source2 DV table to a temporary directory, so that we do updates to it FileUtils.copyDirectory(source, target) if (v2Checkpoint) { spark.sql(s"ALTER TABLE delta.`${target.getAbsolutePath}` SET TBLPROPERTIES " + s"('${DeltaConfigs.CHECKPOINT_POLICY.key}' = 'v2')") } sql(s"ALTER TABLE delta.`${target.getAbsolutePath}` " + s"SET TBLPROPERTIES (${DeltaConfigs.CHECKPOINT_INTERVAL.key} = 10)") def insertData(data: String): Unit = { spark.sql(s"INSERT INTO TABLE delta.`${target.getAbsolutePath}` $data") } val newData = Seq.range(3000, 3010) newData.foreach { i => insertData(s"VALUES($i)") } // Check the target file has checkpoint generated val deltaLog = DeltaLog.forTable(spark, target.getAbsolutePath) verifyCheckpoint(deltaLog.readLastCheckpointFile(), version = 10, parts = None) // Delete the commit files 0-9, so that we are forced to read the checkpoint file val logPath = new Path(new File(target, "_delta_log").getAbsolutePath) for (i <- 0 to 9) { val file = new File(FileNames.unsafeDeltaFile(logPath, version = i).toString) file.delete() } // Make sure the contents are the same import testImplicits._ checkAnswer( spark.sql(s"SELECT * FROM delta.`${target.getAbsolutePath}`"), (DeletionVectorsSuite.expectedTable1DataV4 ++ newData).toSeq.toDF()) } } testDifferentV2Checkpoints(s"V2 Checkpoint compat file equivalency to normal V2 Checkpoint") { withTempDir { tempDir => spark.range(10).write.format("delta").save(tempDir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, tempDir.getAbsolutePath) spark.range(10, 20).write.mode("append").format("delta").save(tempDir.getAbsolutePath) deltaLog.checkpoint() // Checkpoint 1 val normalCheckpointSnapshot = deltaLog.update() deltaLog.createSinglePartCheckpointForBackwardCompat( // Compatibility Checkpoint 1 normalCheckpointSnapshot, new deltaLog.V2CompatCheckpointMetrics) val allFiles = normalCheckpointSnapshot.allFiles.collect().sortBy(_.path).toList val setTransactions = normalCheckpointSnapshot.setTransactions val numOfFiles = normalCheckpointSnapshot.numOfFiles val numOfRemoves = normalCheckpointSnapshot.numOfRemoves val numOfMetadata = normalCheckpointSnapshot.numOfMetadata val numOfProtocol = normalCheckpointSnapshot.numOfProtocol val actions = normalCheckpointSnapshot.stateDS.collect().toSet val fs = deltaLog.logPath.getFileSystem(deltaLog.newDeltaHadoopConf()) // Delete the normal V2 Checkpoint so that the snapshot can be initialized // using the compat checkpoint. fs.delete(normalCheckpointSnapshot.checkpointProvider.topLevelFiles.head.getPath) DeltaLog.clearCache() val deltaLog2 = DeltaLog.forTable(spark, tempDir.getAbsolutePath) val compatCheckpointSnapshot = deltaLog2.update() assert(!compatCheckpointSnapshot.checkpointProvider.isEmpty) assert(compatCheckpointSnapshot.checkpointProvider.version == normalCheckpointSnapshot.checkpointProvider.version) assert( compatCheckpointSnapshot.checkpointProvider.topLevelFiles.head.getPath.getName == FileNames.checkpointFileSingular( deltaLog2.logPath, normalCheckpointSnapshot.checkpointProvider.version).getName ) assert( compatCheckpointSnapshot.allFiles.collect().sortBy(_.path).toList == allFiles ) assert(compatCheckpointSnapshot.setTransactions == setTransactions) assert(compatCheckpointSnapshot.stateDS.collect().toSet == actions) assert(compatCheckpointSnapshot.numOfFiles == numOfFiles) assert(compatCheckpointSnapshot.numOfRemoves == numOfRemoves) assert(compatCheckpointSnapshot.numOfMetadata == numOfMetadata) assert(compatCheckpointSnapshot.numOfProtocol == numOfProtocol) val tableData = spark.sql(s"SELECT * FROM delta.`${deltaLog.dataPath}` ORDER BY id") .collect() .map(_.getLong(0)) assert(tableData.toSeq == (0 to 19)) } } testDifferentCheckpoints("last checkpoint contains correct schema for v1/v2" + " Checkpoints") { (checkpointPolicy, v2CheckpointFormatOpt) => withTempDir { tempDir => spark.range(10).write.format("delta").save(tempDir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, tempDir.getAbsolutePath) deltaLog.checkpoint() val lastCheckpointOpt = deltaLog.readLastCheckpointFile() assert(lastCheckpointOpt.nonEmpty) if (checkpointPolicy.needsV2CheckpointSupport) { if (v2CheckpointFormatOpt.contains(V2Checkpoint.Format.JSON)) { assert(lastCheckpointOpt.get.checkpointSchema.isEmpty) } else { assert(lastCheckpointOpt.get.checkpointSchema.nonEmpty) assert(lastCheckpointOpt.get.checkpointSchema.get.fieldNames.toSeq === Seq("txn", "add", "remove", "metaData", "protocol", "domainMetadata", "checkpointMetadata", "sidecar")) } } else { assert(lastCheckpointOpt.get.checkpointSchema.nonEmpty) assert(lastCheckpointOpt.get.checkpointSchema.get.fieldNames.toSeq === Seq("txn", "add", "remove", "metaData", "protocol", "domainMetadata")) } } } test("last checkpoint - v2 checkpoint fields threshold") { withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath spark.range(1).write.format("delta").save(tablePath) val deltaLog = DeltaLog.forTable(spark, tablePath) // Enable v2Checkpoint table feature. spark.sql(s"ALTER TABLE delta.`$tablePath` SET TBLPROPERTIES " + s"('${DeltaConfigs.CHECKPOINT_POLICY.key}' = 'v2')") def writeCheckpoint( adds: Int, nonFileActionThreshold: Int, sidecarActionThreshold: Int): LastCheckpointInfo = { withSQLConf( DeltaSQLConf.LAST_CHECKPOINT_NON_FILE_ACTIONS_THRESHOLD.key -> s"$nonFileActionThreshold", DeltaSQLConf.LAST_CHECKPOINT_SIDECARS_THRESHOLD.key -> s"$sidecarActionThreshold" ) { val addFiles = (1 to adds).map(_ => createTestAddFile( encodedPath = java.util.UUID.randomUUID.toString, partitionValues = Map(), size = 128L )) deltaLog.startTransaction().commit(addFiles, DeltaOperations.ManualUpdate) deltaLog.checkpoint() } val lastCheckpointInfoOpt = deltaLog.readLastCheckpointFile() assert(lastCheckpointInfoOpt.nonEmpty) lastCheckpointInfoOpt.get } // For CCv1.5 table, row tracking is enabled by default, there will be an extra // DomainMetadata added by RowTracking as a non file action. val domainMetadataAddedByRowTracking = if (catalogOwnedDefaultCreationEnabledInTests) 1 else 0 // Append 1 AddFile [AddFile-2] val lc1 = writeCheckpoint(adds = 1, nonFileActionThreshold = 10, sidecarActionThreshold = 10) assert(lc1.v2Checkpoint.nonEmpty) // 3 non file actions - protocol/metadata/checkpointMetadata, 1 sidecar assert( lc1.v2Checkpoint.get.nonFileActions.get.size === 3 + domainMetadataAddedByRowTracking ) assert(lc1.v2Checkpoint.get.sidecarFiles.get.size === 1) // Append 1 SetTxn, 8 more AddFiles [SetTxn-1, AddFile-10] deltaLog.startTransaction() .commit(Seq(SetTransaction("app-1", 2, None)), DeltaOperations.ManualUpdate) val lc2 = writeCheckpoint( adds = 8, sidecarActionThreshold = 10, nonFileActionThreshold = 4 + domainMetadataAddedByRowTracking ) assert(lc2.v2Checkpoint.nonEmpty) // 4 non file actions - protocol/metadata/checkpointMetadata/setTxn, 1 sidecar assert( lc2.v2Checkpoint.get.nonFileActions.get.size === 4 + domainMetadataAddedByRowTracking ) assert(lc2.v2Checkpoint.get.sidecarFiles.get.size === 1) // Append 10 more AddFiles [SetTxn-1, AddFile-20] val lc3 = writeCheckpoint(adds = 10, nonFileActionThreshold = 3, sidecarActionThreshold = 10) assert(lc3.v2Checkpoint.nonEmpty) // non-file actions exceeded threshold, 1 sidecar assert(lc3.v2Checkpoint.get.nonFileActions.isEmpty) assert(lc3.v2Checkpoint.get.sidecarFiles.get.size === 1) withSQLConf(DeltaSQLConf.DELTA_CHECKPOINT_PART_SIZE.key -> "5") { // Append 10 more AddFiles [SetTxn-1, AddFile-30] val lc4 = writeCheckpoint(adds = 10, nonFileActionThreshold = 3, sidecarActionThreshold = 10) assert(lc4.v2Checkpoint.nonEmpty) // non-file actions exceeded threshold // total 30 file actions, across 6 sidecar files (5 actions per file) assert(lc4.v2Checkpoint.get.nonFileActions.isEmpty) assert(lc4.v2Checkpoint.get.sidecarFiles.get.size === 6) } withSQLConf(DeltaSQLConf.DELTA_CHECKPOINT_PART_SIZE.key -> "2") { // Append 0 AddFiles [SetTxn-1, AddFile-30] val lc5 = writeCheckpoint(adds = 0, nonFileActionThreshold = 10, sidecarActionThreshold = 10) assert(lc5.v2Checkpoint.nonEmpty) // 4 non file actions - protocol/metadata/checkpointMetadata/setTxn // total 30 file actions, across 15 sidecar files (2 actions per file) assert( lc5.v2Checkpoint.get.nonFileActions.get.size === 4 + domainMetadataAddedByRowTracking ) assert(lc5.v2Checkpoint.get.sidecarFiles.isEmpty) } } } def checkIntermittentError( tempDir: File, lastCheckpointMissing: Boolean, crcMissing: Boolean): Unit = { // Create a table with commit version 0, 1 and a checkpoint. val tablePath = tempDir.getAbsolutePath spark.range(10).write.format("delta").save(tablePath) spark.sql(s"INSERT INTO delta.`$tablePath`" + s"SELECT * FROM delta.`$tablePath` WHERE id = 1").collect() val log = DeltaLog.forTable(spark, tablePath) val conf = log.newDeltaHadoopConf() log.checkpoint() // Delete _last_checkpoint based on test configuration. val fs = log.logPath.getFileSystem(conf) if (lastCheckpointMissing) { fs.delete(log.LAST_CHECKPOINT) } // Delete CRC file based on test configuration. if (crcMissing) { // Delete all CRC files (0L to log.update().version).foreach { version => fs.delete(FileNames.checksumFile(log.logPath, version)) } } // In order to trigger an intermittent failure while reading checkpoint, this test corrupts // the checkpoint temporarily so that json/parquet checkpoint reader fails. The corrupted // file is written with same length so that when the file is uncorrupted in future, then we // can test that delta is able to read that file and produce correct results. If the "bad" file // is not of same length, then the read with "good" file will also fail as parquet reader will // use the cache file status's getLen to find out where the footer is and will fail after not // finding the magic bytes. val checkpointFileStatus = log.listFrom(0).filter(FileNames.isCheckpointFile).toSeq.head // Rename the correct checkpoint to a temp path and create a checkpoint with character 'r' // repeated. val tempPath = checkpointFileStatus.getPath.suffix(".temp") fs.rename(checkpointFileStatus.getPath, tempPath) val randomContentToWrite = Seq("r" * (checkpointFileStatus.getLen.toInt - 1)) // + 1 (\n) log.store.write( checkpointFileStatus.getPath, randomContentToWrite.toIterator, overwrite = true, conf) assert(log.store.read(checkpointFileStatus.getPath, conf) === randomContentToWrite) assert(fs.getFileStatus(tempPath).getLen === checkpointFileStatus.getLen) DeltaLog.clearCache() if (!crcMissing) { // When CRC is present, then P&M will be taken from CRC and snapshot will be initialized // without needing a checkpoint. But the underlying checkpoint provider points to a // corrupted checkpoint and so any query/state reconstruction on this will fail. intercept[Exception] { sql(s"SELECT * FROM delta.`$tablePath`").collect() DeltaLog.forTable(spark, tablePath).unsafeVolatileSnapshot.validateChecksum() } val snapshot = DeltaLog.forTable(spark, tablePath).unsafeVolatileSnapshot intercept[Exception] { snapshot.allFiles.collect() } // Undo the corruption assert(fs.delete(checkpointFileStatus.getPath, true)) assert(fs.rename(tempPath, checkpointFileStatus.getPath)) // Once the corruption in undone, then the queries starts passing on top of same snapshot. // This tests that we have not caches the intermittent error in the underlying checkpoint // provider. sql(s"SELECT * FROM delta.`$tablePath`").collect() assert(DeltaLog.forTable(spark, tablePath).update() === snapshot) return } // When CRC is missing, then P&M will be taken from checkpoint which is temporarily // corrupted, so we will end up creating a new snapshot without using checkpoint and the // query will succeed. sql(s"SELECT * FROM delta.`$tablePath`").collect() val snapshot = DeltaLog.forTable(spark, tablePath).unsafeVolatileSnapshot snapshot.computeChecksum snapshot.validateChecksum() assert(snapshot.checkpointProvider.isEmpty) } /** * Writes all actions in the top-level file of a new V2 Checkpoint. No sidecar files are * written. */ private def writeAllActionsInV2Manifest( snapshot: Snapshot, v2CheckpointFormat: V2Checkpoint.Format): Path = { snapshot.ensureCommitFilesBackfilled() val checkpointMetadata = CheckpointMetadata(version = snapshot.version) val actionsDS = snapshot.stateDS .where("checkpointMetadata is null and " + "commitInfo is null and cdc is null and sidecar is null") .union(spark.createDataset(Seq(checkpointMetadata.wrap))) .toDF() val actionsToWrite = Checkpoints .buildCheckpoint(actionsDS, snapshot) .as[SingleAction] .collect() .toSeq .map(_.unwrap) val deltaLog = snapshot.deltaLog val (v2CheckpointPath, _) = if (v2CheckpointFormat == V2Checkpoint.Format.JSON) { val v2CheckpointPath = FileNames.newV2CheckpointJsonFile(deltaLog.logPath, snapshot.version) deltaLog.store.write( v2CheckpointPath, actionsToWrite.map(_.json).toIterator, overwrite = true, hadoopConf = deltaLog.newDeltaHadoopConf()) (v2CheckpointPath, None) } else if (v2CheckpointFormat == V2Checkpoint.Format.PARQUET) { val sparkSession = spark // scalastyle:off sparkimplicits import sparkSession.implicits._ // scalastyle:on sparkimplicits val dfToWrite = actionsToWrite.map(_.wrap).toDF() val v2CheckpointPath = FileNames.newV2CheckpointParquetFile(deltaLog.logPath, snapshot.version) val schemaOfDfWritten = Checkpoints.createCheckpointV2ParquetFile( spark, dfToWrite, v2CheckpointPath, deltaLog.newDeltaHadoopConf(), false) (v2CheckpointPath, Some(schemaOfDfWritten)) } else { throw DeltaErrors.assertionFailedError( s"Unrecognized checkpoint V2 format: $v2CheckpointFormat") } v2CheckpointPath } for (checkpointFormat <- V2Checkpoint.Format.ALL) test(s"All actions in V2 manifest [v2CheckpointFormat: ${checkpointFormat.name}]") { withSQLConf( DeltaConfigs.CHECKPOINT_POLICY.defaultTablePropertyKey -> CheckpointPolicy.V2.name) { withTempDir { dir => spark.range(10).write.format("delta").save(dir.getAbsolutePath) val log = DeltaLog.forTable(spark, dir) spark.sql(s"INSERT INTO delta.`${log.dataPath}` VALUES (2718);") log .startTransaction() .commit(Seq(SetTransaction("app-1", 2, None)), DeltaOperations.ManualUpdate) val snapshot = log.update() val allFiles = snapshot.allFiles.collect().toSet val setTransactions = snapshot.setTransactions.toSet val numOfFiles = snapshot.numOfFiles val numOfRemoves = snapshot.numOfRemoves val numOfMetadata = snapshot.numOfMetadata val numOfProtocol = snapshot.numOfProtocol val actions = snapshot.stateDS.collect().toSet assert(snapshot.version == 2) writeAllActionsInV2Manifest(snapshot, checkpointFormat) DeltaLog.clearCache() val checkpointSnapshot = log.update() assert(!checkpointSnapshot.checkpointProvider.isEmpty) assert(checkpointSnapshot.checkpointProvider.version == 2) // Check the integrity of the data in the checkpoint-backed table. val data = spark .sql(s"SELECT * FROM delta.`${log.dataPath}` ORDER BY ID;") .collect() .map(_.getLong(0)) val expectedData = ((0 to 9).toList :+ 2718).toArray assert(data sameElements expectedData) assert(checkpointSnapshot.setTransactions.toSet == setTransactions) assert(checkpointSnapshot.stateDS.collect().toSet == actions) assert(checkpointSnapshot.numOfFiles == numOfFiles) assert(checkpointSnapshot.numOfRemoves == numOfRemoves) assert(checkpointSnapshot.numOfMetadata == numOfMetadata) assert(checkpointSnapshot.numOfProtocol == numOfProtocol) assert(checkpointSnapshot.allFiles.collect().toSet == allFiles) } } } for (lastCheckpointMissing <- BOOLEAN_DOMAIN) testDifferentCheckpoints("intermittent error while reading checkpoint should not" + s" stick to snapshot [lastCheckpointMissing: $lastCheckpointMissing]") { (_, _) => withTempDir { tempDir => withSQLConf( DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_ENABLED.key -> "false" ) { checkIntermittentError(tempDir, lastCheckpointMissing, crcMissing = true) } } } test("validate metadata cleanup is not called with createCheckpointAtVersion API") { withTempDir { dir => val usageRecords1 = Log4jUsageLogger.track { spark.range(10).write.format("delta").save(dir.getAbsolutePath) val log = DeltaLog.forTable(spark, dir) log.createCheckpointAtVersion(0) } assert(filterUsageRecords(usageRecords1, "delta.log.cleanup").size === 0L) val usageRecords2 = Log4jUsageLogger.track { spark.range(10).write.mode("overwrite").format("delta").save(dir.getAbsolutePath) val log = DeltaLog.forTable(spark, dir) log.checkpoint() } assert(filterUsageRecords(usageRecords2, "delta.log.cleanup").size > 0) } } testDifferentCheckpoints("Ensure variant stats in checkpoint") { (policy, _) => // Test all combinations of (writeStatsAsJson, writeStatsAsStruct) // Skip (false, false) as that would have no stats at all val combinations = Seq( (true, false), (false, true), (true, true) ) // Test with collectVariantStats = false and true Seq(false, true).foreach { collectVariantStats => combinations.foreach { case (writeStatsAsJson, writeStatsAsStruct) => withClue(s"collectVariantStats=$collectVariantStats, " + s"writeStatsAsJson=$writeStatsAsJson, writeStatsAsStruct=$writeStatsAsStruct") { withSQLConf( DeltaSQLConf.COLLECT_VARIANT_DATA_SKIPPING_STATS.key -> collectVariantStats.toString ) { withTempDir { tempDir => // Load golden table with variant stats (no checkpoint) val source = new File("src/test/resources/delta/variant-stats-no-checkpoint") val target = new File(tempDir, "variant-stats-table") FileUtils.copyDirectory(source, target) val tablePath = target.getAbsolutePath // Set the stats configuration via ALTER TABLE spark.sql(s"ALTER TABLE delta.`$tablePath` SET TBLPROPERTIES " + s"('${DeltaConfigs.CHECKPOINT_WRITE_STATS_AS_JSON.key}' = '$writeStatsAsJson', " + s"'${DeltaConfigs.CHECKPOINT_WRITE_STATS_AS_STRUCT.key}' = '$writeStatsAsStruct')") if (policy == CheckpointPolicy.V2) { spark.sql(s"ALTER TABLE delta.`$tablePath` SET TBLPROPERTIES " + s"('${DeltaConfigs.CHECKPOINT_POLICY.key}' = 'v2')") } val deltaLog = DeltaLog.forTable(spark, tablePath) val snapshot = deltaLog.update() deltaLog.checkpoint(snapshot) val checkpointFile = if (policy.needsV2CheckpointSupport) { val provider = getV2CheckpointProvider(deltaLog) provider.sidecarFileStatuses.head.getPath } else { FileNames.checkpointFileSingular(deltaLog.logPath, deltaLog.snapshot.version) } val checkpointDf = spark.read.format("parquet").load(checkpointFile.toString) .filter(col("add").isNotNull) // Helper function to decode Z85 and get variant JSON def decodeZ85ToVariantJson(z85String: String): String = { val decoded = Codec.Base85Codec.decodeBytes(z85String, z85String.length) val metadataSize = VariantStatsShims.metadataSize(decoded) val value = decoded.slice(metadataSize, decoded.length) val variant = new Variant(value, decoded) variant.toJson(java.time.ZoneId.of("UTC")) } // Verify stats in add.stats (JSON format) when writeStatsAsJson=true if (writeStatsAsJson) { val checkpointStatsJson = checkpointDf .selectExpr( s"get_json_object(add.stats, '$$.minValues.v')", s"get_json_object(add.stats, '$$.maxValues.v')", s"get_json_object(add.stats, '$$.minValues.nv.v')", s"get_json_object(add.stats, '$$.maxValues.nv.v')").collect().head // Verify top-level variant column stats val actualMinTopLevel = decodeZ85ToVariantJson(checkpointStatsJson.getString(0)) val actualMaxTopLevel = decodeZ85ToVariantJson(checkpointStatsJson.getString(1)) assert(actualMinTopLevel == """{"$['id']":0,"$['name']":"1"}""") assert(actualMaxTopLevel == """{"$['id']":9,"$['name']":"9"}""") // Verify nested variant column stats val actualMinNested = decodeZ85ToVariantJson(checkpointStatsJson.getString(2)) val actualMaxNested = decodeZ85ToVariantJson(checkpointStatsJson.getString(3)) assert(actualMinNested == """{"$['id']":10,"$['name']":"11"}""") assert(actualMaxNested == """{"$['id']":19,"$['name']":"20"}""") } // Verify stats in add.stats_parsed (struct format) when writeStatsAsStruct=true if (writeStatsAsStruct) { if (collectVariantStats) { val checkpointStatsParsed = checkpointDf .selectExpr( "add.stats_parsed.minValues.v", "add.stats_parsed.maxValues.v", "add.stats_parsed.minValues.nv.v", "add.stats_parsed.maxValues.nv.v").collect().head // Verify top-level variant column stats val minVariantTopLevel = checkpointStatsParsed.getAs[VariantVal](0) val maxVariantTopLevel = checkpointStatsParsed.getAs[VariantVal](1) val minTopLevelVariant = new Variant(minVariantTopLevel.getValue, minVariantTopLevel.getMetadata) val maxTopLevelVariant = new Variant(maxVariantTopLevel.getValue, maxVariantTopLevel.getMetadata) assert(minTopLevelVariant.toJson(java.time.ZoneId.of("UTC")) == """{"$['id']":0,"$['name']":"1"}""") assert(maxTopLevelVariant.toJson(java.time.ZoneId.of("UTC")) == """{"$['id']":9,"$['name']":"9"}""") // Verify nested variant column stats val minVariantNested = checkpointStatsParsed.getAs[VariantVal](2) val maxVariantNested = checkpointStatsParsed.getAs[VariantVal](3) val minNestedVariant = new Variant(minVariantNested.getValue, minVariantNested.getMetadata) val maxNestedVariant = new Variant(maxVariantNested.getValue, maxVariantNested.getMetadata) assert(minNestedVariant.toJson(java.time.ZoneId.of("UTC")) == """{"$['id']":10,"$['name']":"11"}""") assert(maxNestedVariant.toJson(java.time.ZoneId.of("UTC")) == """{"$['id']":19,"$['name']":"20"}""") } else { // When collectVariantStats=false, variant columns should not be in stats_parsed val statsParsedSchema = checkpointDf .select("add.stats_parsed.minValues", "add.stats_parsed.maxValues") .schema val minValuesFields = statsParsedSchema("minValues").dataType .asInstanceOf[StructType].fieldNames val maxValuesFields = statsParsedSchema("maxValues").dataType .asInstanceOf[StructType].fieldNames assert(!minValuesFields.contains("v"), "minValues should not contain 'v' when collectVariantStats=false") assert(!maxValuesFields.contains("v"), "maxValues should not contain 'v' when collectVariantStats=false") } } } } } } } } testDifferentCheckpoints("Ensure variant stats are preserved during state reconstruction") { case (_, _) => // Test different combinations of (writeStatsAsJson, writeStatsAsStruct) // The golden table contains variant stats but NO checkpoint. // We set the checkpoint properties and create the checkpoint in this test. val combinations = Seq( ("true", "false"), ("false", "true"), ("true", "true") // Note: ("false", "false") would have no stats at all, so not testing it ) // Expected Z85-encoded value for variant `0` (the golden table contains `id::variant` // where id=0) // This is the Z85 encoding of the variant binary representation of integer 0 val expectedZ85 = "0rAf3bMW#D00%Fx0000000000" combinations.foreach { case (jsonStats, structStats) => withClue(s"writeStatsAsJson=$jsonStats, writeStatsAsStruct=$structStats") { withTempDir { tempDir => // Copy golden table to temp directory val source = new File("src/test/resources/delta/variant-stats-state-reconstruction") val target = new File(tempDir, "variant-stats-table") FileUtils.copyDirectory(source, target) val tablePath = target.getAbsolutePath // Set checkpoint properties spark.sql( s"""ALTER TABLE delta.`$tablePath` SET TBLPROPERTIES ( | 'delta.checkpoint.writeStatsAsJson' = '$jsonStats', | 'delta.checkpoint.writeStatsAsStruct' = '$structStats' |)""".stripMargin) // Create checkpoint with the new properties val deltaLog = DeltaLog.forTable(spark, tablePath) deltaLog.checkpoint(deltaLog.update()) // Clear cache to ensure fresh state reconstruction from checkpoint DeltaLog.clearCache() val snapshot = deltaLog.update() // Get the reconstructed state and verify variant stats are present val addFilesWithStats = snapshot.stateDS .filter("add IS NOT NULL") .filter("add.stats IS NOT NULL AND add.stats != ''") .collect() assert( addFilesWithStats.nonEmpty, s"Expected at least one AddFile with stats for " + s"writeStatsAsJson=$jsonStats, writeStatsAsStruct=$structStats") // Verify that the stats contain the expected Z85-encoded variant val statsContainZ85 = addFilesWithStats.exists { action => val stats = action.add.stats stats != null && stats.contains(expectedZ85) } assert( statsContainZ85, s"Expected stats to contain Z85-encoded variant '$expectedZ85' for " + s"writeStatsAsJson=$jsonStats, writeStatsAsStruct=$structStats. " + s"Actual stats: ${addFilesWithStats.map(_.add.stats).mkString(", ")}") } } } } } class OverwriteTrackingLogStore(sparkConf: SparkConf, hadoopConf: Configuration) extends LocalLogStore(sparkConf, hadoopConf) { var fileToOverwriteCount: Map[Path, Long] = Map[Path, Long]() private var isPartialWriteVisibleBool: Boolean = false override def isPartialWriteVisible(path: Path, hadoopConf: Configuration): Boolean = isPartialWriteVisibleBool override def write( path: Path, actions: Iterator[String], overwrite: Boolean, hadoopConf: Configuration): Unit = { val toAdd = if (overwrite) 1 else 0 fileToOverwriteCount += path -> (fileToOverwriteCount.getOrElse(path, 0L) + toAdd) super.write(path, actions, overwrite, hadoopConf) } def clearCounts(): Unit = { fileToOverwriteCount = Map[Path, Long]() } def setPartialWriteVisible(isPartialWriteVisibleBool: Boolean): Unit = { this.isPartialWriteVisibleBool = isPartialWriteVisibleBool } } class V2CheckpointManifestOverwriteSuite extends QueryTest with SharedSparkSession with DeltaCheckpointTestUtils with DeltaSQLCommandTest { protected override def sparkConf = { // Set the logStore to OverwriteTrackingLogStore. super.sparkConf .set("spark.delta.logStore.class", classOf[OverwriteTrackingLogStore].getName) .set(DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key, V2Checkpoint.Format.JSON.name) } for (isPartialWriteVisible <- BOOLEAN_DOMAIN) test("v2 checkpoint manifest write should use the logstore.write(overwrite) API correctly " + s"isPartialWriteVisible = $isPartialWriteVisible") { withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath // Create a simple table with V2 checkpoints enabled and json manifest. spark.range(10).write.format("delta").save(tablePath) val deltaLog = DeltaLog.forTable(spark, tablePath) spark.sql(s"ALTER TABLE delta.`$tablePath` SET TBLPROPERTIES " + s"('${DeltaConfigs.CHECKPOINT_POLICY.key}' = 'v2')") val store = deltaLog.store.asInstanceOf[OverwriteTrackingLogStore] store.clearCounts() store.setPartialWriteVisible(isPartialWriteVisible) deltaLog.checkpoint() val snapshot = deltaLog.update() assert(snapshot.checkpointProvider.version == 1) // Two writes will use logStore.write: // 1. Checkpoint Manifest // 2. LAST_CHECKPOINT. assert(store.fileToOverwriteCount.size == 2) val manifestWriteRecord = store.fileToOverwriteCount.find { case (path, _) => FileNames.isCheckpointFile(path) }.getOrElse(fail("expected checkpoint manifest write using logStore.write")) val numOverwritesExpected = if (isPartialWriteVisible) 0 else 1 assert(manifestWriteRecord._2 == numOverwritesExpected) } } } /** A fake GCS file system to verify delta checkpoints are written in a separate gcs thread. */ class FakeGCSFileSystemValidatingCheckpoint extends RawLocalFileSystem { override def getScheme: String = "gs" override def getUri: URI = URI.create("gs:/") protected def shouldValidateFilePattern(f: Path): Boolean = f.getName.contains(".checkpoint") protected def assertGCSThread(f: Path): Unit = { if (shouldValidateFilePattern(f)) { assert( Thread.currentThread().getName.contains("delta-gcs-"), s"writing $f was happening in non gcs thread: ${Thread.currentThread()}") } } override def create( f: Path, permission: FsPermission, overwrite: Boolean, bufferSize: Int, replication: Short, blockSize: Long, progress: Progressable): FSDataOutputStream = { assertGCSThread(f) super.create(f, permission, overwrite, bufferSize, replication, blockSize, progress) } override def create( f: Path, overwrite: Boolean, bufferSize: Int, replication: Short, blockSize: Long, progress: Progressable): FSDataOutputStream = { assertGCSThread(f) super.create(f, overwrite, bufferSize, replication, blockSize, progress) } } /** A fake GCS file system to verify delta commits are written in a separate gcs thread. */ class FakeGCSFileSystemValidatingCommits extends FakeGCSFileSystemValidatingCheckpoint { override protected def shouldValidateFilePattern(f: Path): Boolean = f.getName.contains(".json") } class CheckpointsWithCatalogOwnedBatch1Suite extends CheckpointsSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class CheckpointsWithCatalogOwnedBatch2Suite extends CheckpointsSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class CheckpointsWithCatalogOwnedBatch100Suite extends CheckpointsSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/ChecksumDVMetricsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.collection.mutable import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.{DeletedRecordCountsHistogram, DeletedRecordCountsHistogramUtils} import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.util.DeltaEncoder import org.apache.spark.sql.{DataFrame, Encoder, QueryTest, Row} import org.apache.spark.sql.functions.{coalesce, col, count, lit, sum} import org.apache.spark.sql.test.SharedSparkSession case class StatsSchema( numDeletedRecords: Long, numDeletionVectors: Long, deletedRecordCountsHistogramOpt: Option[DeletedRecordCountsHistogram]) class ChecksumDVMetricsSuite extends QueryTest with SharedSparkSession with DeletionVectorsTestUtils with DeltaSQLCommandTest { override def beforeAll(): Unit = { super.beforeAll() enableDeletionVectors(spark.conf) } protected implicit def statsSchemaEncoder: Encoder[StatsSchema] = (new DeltaEncoder[StatsSchema]).get /* * Compare statistics in the checksum by comparing to the log statistics. */ protected def validateChecksum( snapshot: Snapshot, statisticsExpected: Boolean = true, histogramEnabled: Boolean = true): Unit = { val checksum = snapshot.checksumOpt match { case Some(checksum) => checksum case None => snapshot.computeChecksum } if (statisticsExpected) { val histogramAggregationOpt = if (histogramEnabled) { Some(DeletedRecordCountsHistogramUtils.histogramAggregate( coalesce(col("deletionVector.cardinality"), lit(0L)) ).as("deletedRecordCountsHistogramOpt")) } else { Some(lit(null) .cast(DeletedRecordCountsHistogram.schema) .as("deletedRecordCountsHistogramOpt")) } val aggregations = Seq( sum(coalesce(col("deletionVector.cardinality"), lit(0))).as("numDeletedRecords"), count(col("deletionVector")).as("numDeletionVectors")) ++ histogramAggregationOpt val stats = snapshot.withStatsDeduplicated .select(aggregations: _*) .as[StatsSchema] .first() val numDeletedRecords = stats.numDeletedRecords val numDeletionVectors = stats.numDeletionVectors val deletionVectorHistogram = stats.deletedRecordCountsHistogramOpt assert(checksum.numDeletedRecordsOpt === Some(numDeletedRecords)) assert(checksum.numDeletionVectorsOpt === Some(numDeletionVectors)) assert(checksum.deletedRecordCountsHistogramOpt === deletionVectorHistogram) } else { assert(checksum.numDeletedRecordsOpt === None) assert(checksum.numDeletionVectorsOpt === None) assert(checksum.deletedRecordCountsHistogramOpt === None) } } protected def runMerge( target: io.delta.tables.DeltaTable, source: DataFrame, deleteFromID: Int): Unit = { target.as("t").merge(source.as("s"), "t.id = s.id") .whenMatched(s"s.id >= ${deleteFromID}").delete() .execute() } protected def runDelete( target: io.delta.tables.DeltaTable, source: DataFrame = null, deleteFromID: Int): Unit = { target.delete(s"id >= ${deleteFromID}") } protected def runUpdate( target: io.delta.tables.DeltaTable, source: DataFrame = null, deleteFromID: Int): Unit = { target.update(col("id") >= lit(deleteFromID), Map("v" -> lit(-1))) } for { enableDVsOnTableDefault <- BOOLEAN_DOMAIN enableDVCreation <- BOOLEAN_DOMAIN enableIncrementalCommit <- BOOLEAN_DOMAIN allowDVsOnOperation <- BOOLEAN_DOMAIN } test(s"Commit checksum captures DV statistics " + s"enableDVsOnTableDefault: ${enableDVsOnTableDefault} " + s"enableDVCreation: ${enableDVCreation} " + s"enableIncrementalCommit: ${enableIncrementalCommit} " + s"allowDVsOnOperation: ${allowDVsOnOperation}") { val targetDF = createTestDF(0, 100, 2) val sourceDF = targetDF val operations: Seq[(io.delta.tables.DeltaTable, DataFrame, Int) => Unit] = Seq(runMerge, runDelete, runUpdate) // We validate checksum validation for different feature combinations. for (runOperation <- operations) { withSQLConf( DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> enableDVsOnTableDefault.toString, DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key -> enableIncrementalCommit.toString) { withTempDeltaTable(targetDF, enableDVs = enableDVCreation) { (targetTable, targetLog) => validateChecksum(targetLog.update(), enableDVCreation) // The first operation only deletes half the records from the second file. runOperation(targetTable(), sourceDF, 75) validateChecksum(targetLog.update(), enableDVCreation) // The second operation deletes the remaining records from the second file. withSQLConf( DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS.key -> allowDVsOnOperation.toString, DeltaSQLConf.UPDATE_USE_PERSISTENT_DELETION_VECTORS.key -> allowDVsOnOperation.toString) { runOperation(targetTable(), sourceDF, 50) validateChecksum(targetLog.update(), enableDVCreation) } } } } } test(s"Verify checksum DV statistics are not produced when the relevant config is disabled") { val targetDF = createTestDF(0, 100, 2) withSQLConf(DeltaSQLConf.DELTA_DELETED_RECORD_COUNTS_HISTOGRAM_ENABLED.key -> false.toString) { withTempDeltaTable(targetDF, enableDVs = true) { (targetTable, targetLog) => runDelete(targetTable(), deleteFromID = 75) validateChecksum(targetLog.update(), histogramEnabled = false) } } } for { enableDVCreation <- BOOLEAN_DOMAIN enableIncrementalCommit <- BOOLEAN_DOMAIN } test(s"Checksum is backward compatible " + s"enableDVCreation: $enableDVCreation " + s"enableIncrementalCommit: $enableIncrementalCommit") { val targetDF = createTestDF(0, 100, 2) withSQLConf(DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key -> enableIncrementalCommit.toString) { withTempDeltaTable(targetDF, enableDVs = enableDVCreation) { (targetTable, targetLog) => validateChecksum(targetLog.update(), statisticsExpected = enableDVCreation) runDelete(targetTable(), deleteFromID = 75) validateChecksum(targetLog.update(), statisticsExpected = enableDVCreation) // Flip DV setting on an existing table. enableDeletionVectorsInTable(targetLog, enable = !enableDVCreation) runDelete(targetTable(), deleteFromID = 50) // When INCREMENTAL_COMMIT_ENABLED, enabling DVs midway would normally yield // empty stats. This is due to the incremental nature of the computation and due to the // fact we do not store stats for tables with no DVs. However, in this scenario we try // to take advantage any recent snapshot reconstruction and harvest the stats from there. // In the opposite scenario, disabling DVs midway, we maintain the previously computed // statistics so we do not lose incrementality if DVs are enabled again. // When incremental commit is disabled, both enabling and disabling DVs // midway is not an issue. When DVs are enabled we produce results and when DVs are // disabled we do not. validateChecksum(targetLog.update(), statisticsExpected = !(enableDVCreation && !enableIncrementalCommit)) } } } test("Checksum is computed in incremental commit when full state recomputation is triggered") { val targetDF = createTestDF(0, 100, 2) withSQLConf(DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key -> true.toString, DeltaSQLConf.INCREMENTAL_COMMIT_VERIFY.key -> false.toString, DeltaSQLConf.INCREMENTAL_COMMIT_FORCE_VERIFY_IN_TESTS.key -> false.toString, DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_VERIFICATION_MODE_ENABLED.key -> false.toString) { withTempDeltaTable(targetDF, enableDVs = false) { (targetTable, targetLog) => validateChecksum(targetLog.update(), statisticsExpected = false) runDelete(targetTable(), deleteFromID = 75) validateChecksum(targetLog.update(), statisticsExpected = false) // Flip DV setting on an existing table. enableDeletionVectorsInTable(targetLog) runDelete(targetTable(), deleteFromID = 60) validateChecksum(targetLog.update(), statisticsExpected = true) } } } for (enableIncrementalCommit <- BOOLEAN_DOMAIN) test(s"Verify checksum validation " + s"incrementalCommit: $enableIncrementalCommit") { val targetDF = createTestDF(0, 100, 2) withSQLConf(DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key -> enableIncrementalCommit.toString) { withTempDeltaTable(targetDF, enableDVs = true) { (targetTable, targetLog) => runDelete(targetTable(), deleteFromID = 75) verifyDVsExist(targetLog, 1) val snapshot = targetLog.update() assert(snapshot.validateChecksum()) } } } for (enableIncrementalCommit <- BOOLEAN_DOMAIN) test(s"Verify checksum validation when DVs are enabled on existing tables " + s"incrementalCommit: $enableIncrementalCommit") { val targetDF = createTestDF(0, 100, 2) withSQLConf(DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key -> enableIncrementalCommit.toString) { withTempDeltaTable(targetDF, enableDVs = false) { (targetTable, targetLog) => runDelete(targetTable(), deleteFromID = 75) // This validation should not take into account DV statistics. assert(targetLog.update().validateChecksum()) // Enable DVs, delete with DVs and validate checksum again. enableDeletionVectorsInTable(targetLog) runDelete(targetTable(), deleteFromID = 60) verifyDVsExist(targetLog, 1) // When incremental commit is enabled DV statistics should remain None since DVs were // enabled midway. Checksum validation should not include DV statistics in the // validation process. assert(targetLog.update().validateChecksum()) } } } test("Verify DeletedRecordsCountHistogram correctness") { val histogram1 = DeletedRecordCountsHistogramUtils.emptyHistogram // Initialize histogram with 100 files. (1 to 100).foreach(_ => histogram1.insert(0)) assert(histogram1.deletedRecordCounts === Seq(100, 0, 0, 0, 0, 0, 0, 0, 0, 0)) // Simulate record deletions from 10 files. This would generate 10 RemoveFile actions with // zero DV cardinality. Then we generate 10 add files, 5 in the range 1-9 and 5 more in // the range 1000-9999. (1 to 10).foreach(_ => histogram1.remove(0)) (5 to 9).foreach(n => histogram1.insert(n)) (1000 to 1004).foreach(n => histogram1.insert(n)) assert(histogram1.deletedRecordCounts === Seq(90, 5, 0, 0, 5, 0, 0, 0, 0, 0)) (1 to 5).foreach(_ => histogram1.remove(0)) (100000 to 100004).foreach(n => histogram1.insert(n)) assert(histogram1.deletedRecordCounts === Seq(85, 5, 0, 0, 5, 0, 5, 0, 0, 0)) // Negative values should be ignored. histogram1.insert(-123) histogram1.remove(-14) assert(histogram1.deletedRecordCounts === Seq(85, 5, 0, 0, 5, 0, 5, 0, 0, 0)) // Verify small numbers "catch all" bucket works. (1 to 5).foreach(_ => histogram1.remove(0)) histogram1.insert(10000000L) histogram1.insert(422290000L) histogram1.insert(300000999L) assert(histogram1.deletedRecordCounts === Seq(80, 5, 0, 0, 5, 0, 5, 0, 3, 0)) // Verify large numbers "catch all" bucket works. histogram1.insert(252763333339L) assert(histogram1.deletedRecordCounts === Seq(80, 5, 0, 0, 5, 0, 5, 0, 3, 1)) // Check edges. val histogram2 = DeletedRecordCountsHistogramUtils.emptyHistogram // Bin 1. histogram2.insert(0) assert(histogram2.deletedRecordCounts === Seq(1, 0, 0, 0, 0, 0, 0, 0, 0, 0)) // Bin 2. histogram2.insert(1) histogram2.insert(9) assert(histogram2.deletedRecordCounts === Seq(1, 2, 0, 0, 0, 0, 0, 0, 0, 0)) // Bin 3. histogram2.insert(10) histogram2.insert(99) assert(histogram2.deletedRecordCounts === Seq(1, 2, 2, 0, 0, 0, 0, 0, 0, 0)) // Bin 4. histogram2.insert(100) histogram2.insert(999) assert(histogram2.deletedRecordCounts === Seq(1, 2, 2, 2, 0, 0, 0, 0, 0, 0)) // Bin 5. histogram2.insert(1000) histogram2.insert(9999) assert(histogram2.deletedRecordCounts === Seq(1, 2, 2, 2, 2, 0, 0, 0, 0, 0)) // Bin 6. histogram2.insert(10000) histogram2.insert(99999) assert(histogram2.deletedRecordCounts === Seq(1, 2, 2, 2, 2, 2, 0, 0, 0, 0)) // Bin 7. histogram2.insert(100000) histogram2.insert(999999) assert(histogram2.deletedRecordCounts === Seq(1, 2, 2, 2, 2, 2, 2, 0, 0, 0)) // Bin 8. histogram2.insert(1000000) histogram2.insert(9999999) assert(histogram2.deletedRecordCounts === Seq(1, 2, 2, 2, 2, 2, 2, 2, 0, 0)) // Bin 9. histogram2.insert(10000000) histogram2.insert(100000000) histogram2.insert(1000000000) histogram2.insert(Int.MaxValue - 1) assert(histogram2.deletedRecordCounts === Seq(1, 2, 2, 2, 2, 2, 2, 2, 4, 0)) // Bin 10. histogram2.insert(Int.MaxValue) histogram2.insert(Long.MaxValue) assert(histogram2.deletedRecordCounts === Seq(1, 2, 2, 2, 2, 2, 2, 2, 4, 2)) } test("Verify DeletedRecordsCountHistogram aggregate correctness") { import org.apache.spark.sql.delta.implicits._ val data = Seq( 0L, 1L, 9L, 10L, 99L, 100L, 999L, 1000L, 9999L, 10000L, 99999L, 100000L, 999999L, 1000000L, 9999999L, 10000000L, Int.MaxValue - 1, Int.MaxValue, Long.MaxValue) val df = spark.createDataset(data).toDF("dvCardinality") val histogram = df .select(DeletedRecordCountsHistogramUtils.histogramAggregate(col("dvCardinality"))) .first() val deletedRecordCounts = histogram .getAs[Row](0) .getAs[mutable.WrappedArray[Long]]("deletedRecordCounts") assert(deletedRecordCounts === Seq(1, 2, 2, 2, 2, 2, 2, 2, 2, 2)) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/ChecksumSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.util.TimeZone import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta.DeltaTestUtils._ import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import org.apache.spark.SparkConf import org.apache.spark.sql.QueryTest import org.apache.spark.sql.SaveMode import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.col import org.apache.spark.sql.test.SharedSparkSession class ChecksumSuite extends QueryTest with SharedSparkSession with DeltaTestUtilsBase with DeltaSQLCommandTest with DeltaSQLTestUtils with CatalogOwnedTestBaseSuite { override def sparkConf: SparkConf = super.sparkConf .set(DeltaSQLConf.INCREMENTAL_COMMIT_FORCE_VERIFY_IN_TESTS, false) test(s"A Checksum should be written after every commit when " + s"${DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key} is true") { def testChecksumFile(writeChecksumEnabled: Boolean): Unit = { // Set up the log by explicitly creating the table otherwise we can't // construct the DeltaLog via the table name. withTempTable(createTable = true) { tableName => withSQLConf( DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> writeChecksumEnabled.toString) { def checksumExists(deltaLog: DeltaLog, version: Long): Boolean = { val checksumFile = new File(FileNames.checksumFile(deltaLog.logPath, version).toUri) checksumFile.exists() } // Commit the txn val log = DeltaLog.forTable(spark, TableIdentifier(tableName)) val txn = log.startTransaction(log.initialCatalogTable) val txnCommitVersion = txn.commit(Seq.empty, DeltaOperations.Truncate()) assert(checksumExists(log, txnCommitVersion) == writeChecksumEnabled) } } } testChecksumFile(writeChecksumEnabled = true) testChecksumFile(writeChecksumEnabled = false) } private def setTimeZone(timeZone: String): Unit = { spark.sql(s"SET spark.sql.session.timeZone = $timeZone") TimeZone.setDefault(TimeZone.getTimeZone(timeZone)) } test("Incremental checksums: post commit snapshot should have a checksum " + "without triggering state reconstruction") { for (incrementalCommitEnabled <- BOOLEAN_DOMAIN) { withSQLConf( DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> "false", DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key -> incrementalCommitEnabled.toString ) { withTempTable(createTable = false) { tableName => // Set the timezone to UTC to avoid triggering force verification of all files in CRC // for non utc environments. setTimeZone("UTC") val df = spark.range(1) df.write.format("delta").mode("append").saveAsTable(tableName) val log = DeltaLog.forTable(spark, TableIdentifier(tableName)) log .startTransaction() .commit(Seq(createTestAddFile()), DeltaOperations.Write(SaveMode.Append)) val postCommitSnapshot = log.snapshot assert(postCommitSnapshot.version == 1) assert(!postCommitSnapshot.stateReconstructionTriggered) assert(postCommitSnapshot.checksumOpt.isDefined == incrementalCommitEnabled) postCommitSnapshot.checksumOpt.foreach { incrementalChecksum => val checksumFromStateReconstruction = postCommitSnapshot.computeChecksum assert(incrementalChecksum.copy(txnId = None) == checksumFromStateReconstruction) } } } } } def testIncrementalChecksumWrites(tableMutationOperation: String => Unit): Unit = { withTempTable(createTable = false) { tableName => withSQLConf( DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> "true", DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key ->"true") { val df = spark.range(10).withColumn("id2", col("id") % 2) df.write .format("delta") .partitionBy("id") .mode("append") .saveAsTable(tableName) tableMutationOperation(tableName) val log = DeltaLog.forTable(spark, TableIdentifier(tableName)) val checksumOpt = log.snapshot.checksumOpt assert(checksumOpt.isDefined) val checksum = checksumOpt.get val computedChecksum = log.snapshot.computeChecksum assert(checksum.copy(txnId = None) === computedChecksum) } } } test("Incremental checksums: INSERT") { testIncrementalChecksumWrites { tableName => sql(s"INSERT INTO $tableName SELECT *, 1 FROM range(10, 20)") } } test("Incremental checksums: UPDATE") { testIncrementalChecksumWrites { tableName => sql(s"UPDATE $tableName SET id2 = id + 1 WHERE id % 2 = 0") } } test("Incremental checksums: DELETE") { testIncrementalChecksumWrites { tableName => sql(s"DELETE FROM $tableName WHERE id % 2 = 0") } } test("Checksum validation should happen on checkpoint") { withSQLConf( DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> "true", DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key -> "true", // Disabled for this test because with it enabled, a corrupted Protocol // or Metadata will trigger a failure earlier than the full validation. DeltaSQLConf.USE_PROTOCOL_AND_METADATA_FROM_CHECKSUM_ENABLED.key -> "false" ) { withTempTable(createTable = false) { tableName => spark .range(10) .write .format("delta") .saveAsTable(tableName) spark.range(1) .write .format("delta") .mode("append") .saveAsTable(tableName) val log = DeltaLog.forTable(spark, TableIdentifier(tableName)) val checksumOpt = log.readChecksum(1) assert(checksumOpt.isDefined) val checksum = checksumOpt.get // Corrupt the checksum file. val corruptedChecksum = checksum.copy( protocol = checksum.protocol.copy(minReaderVersion = checksum.protocol.minReaderVersion + 1), metadata = checksum.metadata.copy(description = "corrupted"), numProtocol = 2, numMetadata = 2, tableSizeBytes = checksum.tableSizeBytes + 1, numFiles = checksum.numFiles + 1) val corruptedChecksumJson = JsonUtils.toJson(corruptedChecksum) log.store.write( FileNames.checksumFile(log.logPath, 1), Seq(corruptedChecksumJson).toIterator, overwrite = true) DeltaLog.clearCache() val log2 = DeltaLog.forTable(spark, TableIdentifier(tableName)) val usageLogs = Log4jUsageLogger.track { intercept[DeltaIllegalStateException] { log2.checkpoint() } } val validationFailureLogs = filterUsageRecords(usageLogs, "delta.checksum.invalid") assert(validationFailureLogs.size == 1) validationFailureLogs.foreach { log => val usageLogBlob = JsonUtils.fromJson[Map[String, Any]](log.blob) val mismatchingFieldsOpt = usageLogBlob.get("mismatchingFields") assert(mismatchingFieldsOpt.isDefined) val mismatchingFieldsSet = mismatchingFieldsOpt.get.asInstanceOf[Seq[String]].toSet val expectedMismatchingFields = Set( "protocol", "metadata", "numOfProtocol", "numOfMetadata", "tableSizeBytes", "numFiles" ) assert(mismatchingFieldsSet === expectedMismatchingFields) } } } } test("incremental commit verify mode should always detect invalid .crc") { withSQLConf( DeltaSQLConf.INCREMENTAL_COMMIT_VERIFY.key -> "true", DeltaSQLConf.DELTA_CHECKSUM_MISMATCH_IS_FATAL.key -> "false", DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key -> "true", DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_ENABLED.key -> "false", DeltaSQLConf.USE_PROTOCOL_AND_METADATA_FROM_CHECKSUM_ENABLED.key -> "true" ) { // Explicitly create the table w/o any AddFile for the subsequent // DeltaLog construction. withTempTable(createTable = true) { tableName => import testImplicits._ val numAddFiles = 10 // Procedure: // 1. Populate the table with several files // 2. Start a new transaction // 3. Intentionally try to commit the same files again // a. Silently duplicated AddFile breaks incremental commit invariants // b. Incrementally computed .crc is thus invalid // c. Incremental commit verification should detect the "invalid" .crc // d. Post-commit snapshot should have empty checksumOpt // 4. Clear the delta log cache so we pick up the correct (fallback) .crc // 5. Create a new snapshot and manually validate the .crc val files = (1 to numAddFiles).map(i => createTestAddFile(encodedPath = i.toString)) DeltaLog .forTable(spark, TableIdentifier(tableName)) .startTransaction() .commitWriteAppend(files: _*) val log = DeltaLog.forTable(spark, TableIdentifier(tableName)) val txn = log.startTransaction() val expected = s"""Table size (bytes) - Expected: ${2*numAddFiles} Computed: $numAddFiles |Number of files - Expected: ${2*numAddFiles} Computed: $numAddFiles """.stripMargin.trim val Seq(corruptionReport) = collectUsageLogs("delta.checksum.invalid") { // Intentionally re-add the same files, without identifying them as duplicates txn.commitWriteAppend(files: _*) } val error = JsonUtils.fromJson[Map[String, Any]](corruptionReport.blob).get("error") assert(error.exists(_.asInstanceOf[String].contains(expected))) assert(log.snapshot.checksumOpt.isEmpty) } } } test("force checksum validation due to stale checkpoint") { withSQLConf( DeltaSQLConf.INCREMENTAL_COMMIT_VERIFY.key -> "false", // Set this to 0 to ensure that validation is not // skipped due the checkpoint not being old enough DeltaSQLConf.FORCED_CHECKSUM_VALIDATION_MIN_TIME_INTERVAL_MINUTES.key -> "0", DeltaSQLConf.DELTA_CHECKSUM_MISMATCH_IS_FATAL.key -> "true", DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key -> "true", DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_ENABLED.key -> "false", DeltaSQLConf.FORCED_CHECKSUM_VALIDATION_INTERVAL.key -> "999" ) { withTempTable(createTable = false) { tableName => spark .range(4) .write .format("delta") .saveAsTable(tableName) // Create checkpoint at version 0 DeltaLog.forTable(spark, TableIdentifier(tableName)).checkpoint() def validateAttemptedTransactionFails: Unit = { DeltaLog.clearCache() val usageLogs = Log4jUsageLogger.track { intercept[DeltaIllegalStateException] { DeltaLog .forTable(spark, TableIdentifier(tableName)) .startTransaction() } } val validationFailureLogs = filterUsageRecords(usageLogs, "delta.checksum.invalid") assert(validationFailureLogs.size == 1) validationFailureLogs.foreach { log => val usageLogBlob = JsonUtils.fromJson[Map[String, Any]](log.blob) val mismatchingFieldsOpt = usageLogBlob.get("mismatchingFields") assert(mismatchingFieldsOpt.isDefined) val mismatchingFieldsSet = mismatchingFieldsOpt.get.asInstanceOf[Seq[String]].toSet val expectedMismatchingFields = Set( "numOfProtocol", "numOfMetadata", "tableSizeBytes", "numFiles" ) assert(mismatchingFieldsSet === expectedMismatchingFields) } } // Write 4 commits. Also, corrupt every checksum file. (1 to 4).foreach { version => spark.range(1) .write .format("delta") .mode("append") .saveAsTable(tableName) // Corrupt the checksum file. val log = DeltaLog .forTable(spark, TableIdentifier(tableName)) val checksum = log.readChecksum(version).get val corruptedChecksum = checksum.copy( numProtocol = 2, numMetadata = 2, tableSizeBytes = checksum.tableSizeBytes + 1, numFiles = checksum.numFiles + 1) val corruptedChecksumJson = JsonUtils.toJson(corruptedChecksum) log.store.write( FileNames.checksumFile(log.logPath, version), Seq(corruptedChecksumJson).toIterator, overwrite = true) withSQLConf( // Set the forced checksum validation interval to the current version // so that validation is triggered DeltaSQLConf.FORCED_CHECKSUM_VALIDATION_INTERVAL.key -> version.toString ) { validateAttemptedTransactionFails } withSQLConf( // Set the validation interval to a value smaller than the current version // so that validation is triggered DeltaSQLConf.FORCED_CHECKSUM_VALIDATION_INTERVAL.key -> "0" ) { validateAttemptedTransactionFails } withSQLConf( DeltaSQLConf.FORCED_CHECKSUM_VALIDATION_INTERVAL.key -> "0", // Validation should only be triggered if the checkpoint was // created more than 999 minutes ago. Which should not be // the case here. DeltaSQLConf.FORCED_CHECKSUM_VALIDATION_MIN_TIME_INTERVAL_MINUTES.key -> "999" ) { DeltaLog.clearCache() DeltaLog .forTable(spark, TableIdentifier(tableName)) .startTransaction() } } } } } } class ChecksumWithCatalogOwnedBatch1Suite extends ChecksumSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class ChecksumWithCatalogOwnedBatch2Suite extends ChecksumSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class ChecksumWithCatalogOwnedBatch100Suite extends ChecksumSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/CloneParquetSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.test.DeltaExceptionTestUtils import org.apache.spark.{SparkException, SparkThrowable} import org.apache.spark.sql.DataFrame import org.apache.spark.sql.functions.col class CloneParquetByPathSuite extends CloneParquetSuiteBase with DeltaExceptionTestUtils { protected def withParquetTable( df: DataFrame, partCols: Seq[String] = Seq.empty[String])( func: ParquetIdent => Unit): Unit = { withTempDir { dir => val tempDir = dir.getCanonicalPath if (partCols.nonEmpty) { df.write.format("parquet").mode("overwrite").partitionBy(partCols: _*).save(tempDir) } else { df.write.format("parquet").mode("overwrite").save(tempDir) } func(ParquetIdent(tempDir, isTable = false)) } } // CLONE doesn't support partitioned parquet table using path since it requires customer to // provide the partition schema in the command like `CONVERT TO DELTA`, but such an option is not // available in CLONE yet. testClone("clone partitioned parquet to delta table") { mode => val df = spark.range(100) .withColumn("key1", col("id") % 4) .withColumn("key2", col("id") % 7 cast "String") withParquetTable(df, Seq("key1", "key2")) { sourceIdent => val tableName = "cloneTable" withTable(tableName) { val ex = interceptWithUnwrapping[DeltaAnalysisException] { sql(s"CREATE TABLE $tableName $mode CLONE $sourceIdent") } assert(ex.getMessage.contains("Expecting 0 partition column(s)")) } } } } class CloneParquetByNameSuite extends CloneParquetSuiteBase { protected def withParquetTable( df: DataFrame, partCols: Seq[String] = Seq.empty[String])( func: ParquetIdent => Unit): Unit = { val tableName = "parquet_table" withTable(tableName) { if (partCols.nonEmpty) { df.write.format("parquet").partitionBy(partCols: _*).saveAsTable(tableName) } else { df.write.format("parquet").saveAsTable(tableName) } func(ParquetIdent(tableName, isTable = true)) } } testClone("clone partitioned parquet to delta table") { mode => val df = spark.range(100) .withColumn("key1", col("id") % 4) .withColumn("key2", col("id") % 7 cast "String") withParquetTable(df, Seq("key1", "key2")) { sourceIdent => val tableName = "cloneTable" withTable(tableName) { sql(s"CREATE TABLE $tableName $mode CLONE $sourceIdent") checkAnswer(spark.table(tableName), df) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/CloneParquetSuiteBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta.commands.CloneParquetSource import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.sql.{DataFrame, QueryTest} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.col import org.apache.spark.sql.test.SharedSparkSession trait CloneParquetSuiteBase extends QueryTest with DeltaSQLCommandTest with SharedSparkSession { // Identifier to represent a Parquet source protected case class ParquetIdent(name: String, isTable: Boolean) { override def toString: String = if (isTable) name else s"parquet.`$name`" def toTableIdent: TableIdentifier = if (isTable) TableIdentifier(name) else TableIdentifier(name, Some("parquet")) def toCloneSource: CloneParquetSource = { val catalogTableOpt = if (isTable) Some(spark.sessionState.catalog.getTableMetadata(toTableIdent)) else None CloneParquetSource(toTableIdent, catalogTableOpt, spark) } } protected def supportedModes: Seq[String] = Seq("SHALLOW") protected def testClone(testName: String)(f: String => Unit): Unit = supportedModes.foreach { mode => test(s"$testName - $mode") { f(mode) } } protected def withParquetTable( df: DataFrame, partCols: Seq[String] = Seq.empty[String])(func: ParquetIdent => Unit): Unit protected def validateBlob( blob: Map[String, Any], mode: String, source: CloneParquetSource, target: DeltaLog): Unit = { // scalastyle:off deltahadoopconfiguration val hadoopConf = spark.sessionState.newHadoopConf() // scalastyle:on deltahadoopconfiguration val sourcePath = source.dataPath val sourceFs = sourcePath.getFileSystem(hadoopConf) val qualifiedSourcePath = sourceFs.makeQualified(sourcePath) val targetPath = target.dataPath val targetFs = targetPath.getFileSystem(hadoopConf) val qualifiedTargetPath = targetFs.makeQualified(targetPath) assert(blob("sourcePath") === qualifiedSourcePath.toString) assert(blob("target") === qualifiedTargetPath.toString) assert(blob("sourceTableSize") === source.sizeInBytes) assert(blob("sourceNumOfFiles") === source.numOfFiles) assert(blob("partitionBy") === source.metadata.partitionColumns) } testClone("validate clone metrics") { mode => val df = spark.range(100).withColumn("key", col("id") % 3) withParquetTable(df) { sourceIdent => val tableName = "cloneTable" withTable(tableName) { val allLogs = Log4jUsageLogger.track { sql(s"CREATE TABLE $tableName $mode CLONE $sourceIdent") } val source = sourceIdent.toCloneSource val target = DeltaLog.forTable(spark, TableIdentifier(tableName)) val blob = JsonUtils.fromJson[Map[String, Any]](allLogs .filter(_.metric == "tahoeEvent") .filter(_.tags.get("opType").contains("delta.clone")) .filter(_.blob.contains("source")) .map(_.blob).last) validateBlob(blob, mode, source, target) val sourceMetadata = source.metadata val targetMetadata = target.update().metadata assert(sourceMetadata.schema === targetMetadata.schema) assert(sourceMetadata.configuration === targetMetadata.configuration) assert(sourceMetadata.dataSchema === targetMetadata.dataSchema) assert(sourceMetadata.partitionColumns === targetMetadata.partitionColumns) } } } testClone("clone non-partitioned parquet to delta table") { mode => val df = spark.range(100) .withColumn("key1", col("id") % 4) .withColumn("key2", col("id") % 7 cast "String") withParquetTable(df) { sourceIdent => val tableName = "cloneTable" withTable(tableName) { sql(s"CREATE TABLE $tableName $mode CLONE $sourceIdent") checkAnswer(spark.table(tableName), df) } } } testClone("clone non-partitioned parquet to delta path") { mode => val df = spark.range(100) .withColumn("key1", col("id") % 4) .withColumn("key2", col("id") % 7 cast "String") withParquetTable(df) { sourceIdent => withTempDir { dir => val deltaDir = dir.getCanonicalPath sql(s"CREATE TABLE delta.`$deltaDir` $mode CLONE $sourceIdent") checkAnswer(spark.read.format("delta").load(deltaDir), df) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/CloneTableSQLSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.collection.immutable.NumericRange import org.apache.spark.sql.delta.actions.{AddFile, FileAction, RemoveFile} import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaExcludedTestMixin, DeltaSQLCommandTest} import org.apache.hadoop.fs.Path import org.apache.spark.sql.{AnalysisException, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.util.Utils class CloneTableSQLSuite extends CloneTableSuiteBase with CloneTableSQLTestMixin with DeltaColumnMappingTestUtils with CatalogOwnedTestBaseSuite { override def beforeAll(): Unit = { super.beforeAll() disableDeletionVectors(spark.conf) } testAllClones(s"table version as of syntax") { (_, target, isShallow) => val tbl = "source" testSyntax( tbl, target, s"CREATE TABLE delta.`$target` ${cloneTypeStr(isShallow)} CLONE $tbl VERSION AS OF 0", isShallow ) } testAllClones("CREATE OR REPLACE syntax when there is no existing table") { (_, clone, isShallow) => val tbl = "source" testSyntax( tbl, clone, s"CREATE OR REPLACE TABLE delta.`$clone` ${cloneTypeStr(isShallow)} CLONE $tbl", isShallow ) } cloneTest("REPLACE cannot be used with IF NOT EXISTS") { (shallow, _) => val tbl = "source" intercept[ParseException] { testSyntax(tbl, shallow, s"CREATE OR REPLACE TABLE IF NOT EXISTS delta.`$shallow` SHALLOW CLONE $tbl") } intercept[ParseException] { testSyntax(tbl, shallow, s"REPLACE TABLE IF NOT EXISTS delta.`$shallow` SHALLOW CLONE $tbl") } } testAllClones( "IF NOT EXISTS should not go through with CLONE if table exists") { (tblExt, _, isShallow) => val sourceTable = "source" val conflictingTable = "conflict" withTable(sourceTable, conflictingTable) { sql(s"CREATE TABLE $conflictingTable " + s"USING PARQUET LOCATION '$tblExt' TBLPROPERTIES ('abc'='def', 'def'='ghi') AS SELECT 1") spark.range(5).write.format("delta").saveAsTable(sourceTable) sql(s"CREATE TABLE IF NOT EXISTS " + s"$conflictingTable ${cloneTypeStr(isShallow)} CLONE $sourceTable") checkAnswer(sql(s"SELECT COUNT(*) FROM $conflictingTable"), Row(1)) } } testAllClones("IF NOT EXISTS should throw an error if path exists") { (_, target, isShallow) => spark.range(5).write.format("delta").save(target) val ex = intercept[AnalysisException] { sql(s"CREATE TABLE IF NOT EXISTS " + s"delta.`$target` ${cloneTypeStr(isShallow)} CLONE delta.`$target`") } assert(ex.getMessage.contains("is not empty")) } cloneTest("Negative test: REPLACE table where there is no existing table") { (shallow, _) => val tbl = "source" val ex = intercept[AnalysisException] { testSyntax(tbl, shallow, s"REPLACE TABLE delta.`$shallow` SHALLOW CLONE $tbl") } assert(ex.getMessage.contains("cannot be replaced as it does not exist.")) } cloneTest("cloning a table that doesn't exist") { (tblExt, _) => val ex = intercept[AnalysisException] { sql(s"CREATE TABLE delta.`$tblExt` SHALLOW CLONE not_exists") } assert(ex.getMessage.contains("Table not found") || ex.getMessage.contains("The table or view `not_exists` cannot be found")) val ex2 = intercept[AnalysisException] { sql(s"CREATE TABLE delta.`$tblExt` SHALLOW CLONE not_exists VERSION AS OF 0") } assert(ex2.getMessage.contains("Table not found") || ex2.getMessage.contains("The table or view `not_exists` cannot be found")) } cloneTest("cloning a view") { (tblExt, _) => withTempView("tmp") { sql("CREATE OR REPLACE TEMP VIEW tmp AS SELECT * FROM range(10)") val ex = intercept[AnalysisException] { sql(s"CREATE TABLE delta.`$tblExt` SHALLOW CLONE tmp") } assert(ex.errorClass === Some("DELTA_CLONE_UNSUPPORTED_SOURCE")) assert(ex.getMessage.contains("clone source 'tmp', whose format is View.")) } } cloneTest("Clone on table with delta statistics columns") { (source, target) => withTable("delta_table", "delta_table_shadow_clone", "delta_table_clone") { sql( "create table delta_table (c0 long, c1 long, c2 long) using delta " + "TBLPROPERTIES('delta.dataSkippingStatsColumns' = 'c1,c2', " + "'delta.columnMapping.mode' = 'name', " + "'delta.minReaderVersion' = '2', " + "'delta.minWriterVersion' = '5')" ) sql(s"CREATE TABLE delta_table_shadow_clone SHALLOW CLONE delta_table LOCATION '$source'") var dataSkippingStatsColumns = sql("SHOW TBLPROPERTIES delta_table_shadow_clone") .collect() .map { row => row.getString(0) -> row.getString(1) } .filter(_._1 == "delta.dataSkippingStatsColumns") .toSeq val result1 = Seq(("delta.dataSkippingStatsColumns", "c1,c2")) assert(dataSkippingStatsColumns == result1) } } cloneTest("Clone on table with nested delta statistics columns") { (source, target) => withTable("delta_table", "delta_table_shadow_clone", "delta_table_clone") { sql( "create table delta_table (c0 long, c1 long, c2 struct) using delta " + "TBLPROPERTIES('delta.dataSkippingStatsColumns' = 'c1,c2.a,c2.b', " + "'delta.columnMapping.mode' = 'name', " + "'delta.minReaderVersion' = '2', " + "'delta.minWriterVersion' = '5')" ) sql(s"CREATE TABLE delta_table_shadow_clone SHALLOW CLONE delta_table LOCATION '$source'") var dataSkippingStatsColumns = sql("SHOW TBLPROPERTIES delta_table_shadow_clone") .collect() .map { row => row.getString(0) -> row.getString(1) } .filter(_._1 == "delta.dataSkippingStatsColumns") .toSeq val result1 = Seq(("delta.dataSkippingStatsColumns", "c1,c2.a,c2.b")) assert(dataSkippingStatsColumns == result1) } } cloneTest("cloning a view over a Delta table") { (tblExt, _) => withTable("delta_table") { withView("tmp") { sql("CREATE TABLE delta_table USING delta AS SELECT * FROM range(10)") sql("CREATE VIEW tmp AS SELECT * FROM delta_table") val ex = intercept[AnalysisException] { sql(s"CREATE TABLE delta.`$tblExt` SHALLOW CLONE tmp") } assert(ex.errorClass === Some("DELTA_CLONE_UNSUPPORTED_SOURCE")) assert( ex.getMessage.contains("clone source") && ex.getMessage.contains("default.tmp', whose format is View.") ) } } } cloneTest("check metrics returned from shallow clone", TAG_HAS_SHALLOW_CLONE) { (_, _) => val source = "source" val target = "target" withTable(source, target) { spark.range(100).write.format("delta").saveAsTable(source) val res = sql(s"CREATE TABLE $target SHALLOW CLONE $source") // schema check val expectedColumns = Seq( "source_table_size", "source_num_of_files", "num_removed_files", "num_copied_files", "removed_files_size", "copied_files_size" ) assert(expectedColumns == res.columns.toSeq) // logic check assert(res.count() == 1) val returnedMetrics = res.first() assert(returnedMetrics.getAs[Long]("source_table_size") != 0L) assert(returnedMetrics.getAs[Long]("source_num_of_files") != 0L) // Delta-OSS doesn't support copied file metrics assert(returnedMetrics.getAs[Long]("num_copied_files") == 0L) assert(returnedMetrics.getAs[Long]("copied_files_size") == 0L) } } cloneTest("Negative test: Clone to target path and also have external location") { (deep, ext) => val sourceTable = "source" withTable(sourceTable) { spark.range(5).write.format("delta").saveAsTable(sourceTable) val ex = intercept[IllegalArgumentException] { runAndValidateClone( sourceTable, deep, isShallow = true, sourceIsTable = true, targetLocation = Some(ext))() } assert(ex.getMessage.contains("Two paths were provided as the CLONE target")) } } test("Clone should populate override table properties to catalog") { val source = "source" val target = "target" withTable(source, target) { withSQLConf(DeltaSQLConf.DELTA_UPDATE_CATALOG_ENABLED.key -> "false") { spark.range(100).write.format("delta").saveAsTable(source) sql(s"""CREATE TABLE $target SHALLOW CLONE $source tblproperties("abc" = "123")""") val targetCatalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(target)) targetCatalogTable.properties.get("abc").contains("123") } } } } class CloneTableSQLWithCatalogOwnedBatch1Suite extends CloneTableSQLSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class CloneTableSQLWithCatalogOwnedBatch2Suite extends CloneTableSQLSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class CloneTableSQLWithCatalogOwnedBatch100Suite extends CloneTableSQLSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } class CloneTableSQLIdColumnMappingSuite extends CloneTableSQLSuite with CloneTableColumnMappingSuiteBase with DeltaColumnMappingEnableIdMode { } class CloneTableSQLNameColumnMappingSuite extends CloneTableSQLSuite with CloneTableColumnMappingNameSuiteBase with DeltaColumnMappingEnableNameMode { } object CloneTableSQLTestUtils { // scalastyle:off argcount def buildCloneSqlString( source: String, target: String, sourceIsTable: Boolean = false, targetIsTable: Boolean = false, sourceFormat: String = "delta", targetLocation: Option[String] = None, versionAsOf: Option[Long] = None, timestampAsOf: Option[String] = None, isCreate: Boolean = true, isReplace: Boolean = false, tableProperties: Map[String, String] = Map.empty): String = { val header = if (isCreate && isReplace) { "CREATE OR REPLACE" } else if (isReplace) { "REPLACE" } else { "CREATE" } // e.g. CREATE TABLE targetTable val createTbl = if (targetIsTable) s"$header TABLE $target" else s"$header TABLE delta.`$target`" // e.g. CREATE TABLE targetTable SHALLOW CLONE val withMethod = createTbl + " SHALLOW CLONE " // e.g. CREATE TABLE targetTable SHALLOW CLONE delta.`/source/table` val withSource = if (sourceIsTable) { withMethod + s"$source " } else { withMethod + s"$sourceFormat.`$source` " } // e.g. CREATE TABLE targetTable SHALLOW CLONE delta.`/source/table` VERSION AS OF 0 val withVersion = if (versionAsOf.isDefined) { withSource + s"VERSION AS OF ${versionAsOf.get}" } else if (timestampAsOf.isDefined) { withSource + s"TIMESTAMP AS OF '${timestampAsOf.get}'" } else { withSource } // e.g. CREATE TABLE targetTable SHALLOW CLONE delta.`/source/table` VERSION AS OF 0 // LOCATION '/desired/target/location' val withLocation = if (targetLocation.isDefined) { s" $withVersion LOCATION '${targetLocation.get}'" } else { withVersion } val withProperties = if (tableProperties.nonEmpty) { val props = tableProperties.map(p => s"'${p._1}' = '${p._2}'").mkString(",") s" $withLocation TBLPROPERTIES ($props)" } else { withLocation } withProperties } // scalastyle:on argcount } class CloneTableScalaDeletionVectorSuite extends CloneTableScalaSuite with DeltaSQLCommandTest with DeltaExcludedTestMixin with DeletionVectorsTestUtils { override def excluded: Seq[String] = super.excluded ++ Seq( // These require the initial table protocol version to be low to work properly. "Cloning a table with new table properties that force protocol version upgrade -" + " delta.enableChangeDataFeed" , "Cloning a table with new table properties that force protocol version upgrade -" + " delta.enableDeletionVectors" , "Cloning a table without DV property should not upgrade protocol version" , "CLONE respects table features set by table property override, targetExists=true" , "CLONE ignores reader/writer session defaults") override def beforeAll(): Unit = { super.beforeAll() enableDeletionVectors(spark.conf) } testAllClones("Cloning table with persistent DVs") { (source, target, isShallow) => // Create source table writeMultiFileSourceTable( source, fileRanges = Seq(0L until 30L, 30L until 60L, 60L until 90L)) // Add DVs to 2 files, leave 1 file without DVs. spark.sql(s"DELETE FROM delta.`$source` WHERE id IN (24, 42)") runAndValidateCloneWithDVs( source, target, isShallow, expectedNumFilesWithDVs = 2) } testAllClones("Cloning table with persistent DVs and absolute parquet paths" ) { (source, target, isShallow) => withTempDir { originalSourceDir => val originalSource = originalSourceDir.getCanonicalPath // Create source table, by writing to an upstream table and then shallow cloning before // adding DVs. writeMultiFileSourceTable( source = originalSource, fileRanges = Seq(0L until 30L, 30L until 60L, 60L until 90L)) spark.sql(s"CREATE OR REPLACE TABLE delta.`$source` SHALLOW CLONE delta.`$originalSource`") // Add DVs to 2 files, leave 1 file without DVs. spark.sql(s"DELETE FROM delta.`$source` WHERE id IN (24, 42)") runAndValidateCloneWithDVs( source, target, isShallow, expectedNumFilesWithDVs = 2) } } testAllClones("Cloning table with persistent DVs and absolute DV file paths" ) { (source, target, isShallow) => withTempDir { originalSourceDir => val originalSource = originalSourceDir.getCanonicalPath // Create source table, by writing to an upstream table, adding DVs and then shallow cloning. writeMultiFileSourceTable( source = originalSource, fileRanges = Seq(0L until 30L, 30L until 60L, 60L until 90L)) // Add DVs to 2 files, leave 1 file without DVs. spark.sql(s"DELETE FROM delta.`$originalSource` WHERE id IN (24, 42)") val originalSourceTable = io.delta.tables.DeltaTable.forPath(spark, originalSource) spark.sql(s"CREATE OR REPLACE TABLE delta.`$source` SHALLOW CLONE delta.`$originalSource`") // Double check this clone was correct. checkAnswer( spark.read.format("delta").load(source), expectedAnswer = originalSourceTable.toDF) runAndValidateCloneWithDVs( source, target, isShallow, expectedNumFilesWithDVs = 2) } } cloneTest("Shallow clone round-trip with DVs") { (source, target) => // Create source table. writeMultiFileSourceTable( source = source, fileRanges = Seq( 0L until 30L, // file 1 30L until 60L, // file 2 60L until 90L, // file 3 90L until 120L)) // file 4 // Add DVs to files 1 and 2 and then shallow clone. spark.sql(s"DELETE FROM delta.`$source` WHERE id IN (24, 42)") runAndValidateCloneWithDVs( source = source, target = target, isShallow = true, expectedNumFilesWithDVs = 2) // Add a new DV to file 3 and update the DV file 2, // leaving file 4 without a DV and file 1 with the existing DV. // Then shallow clone back into source. spark.sql(s"DELETE FROM delta.`$target` WHERE id IN (43, 69)") runAndValidateCloneWithDVs( source = target, target = source, isShallow = true, expectedNumFilesWithDVs = 3, isReplaceOperation = true) } /** Write one file per range in `fileRanges`. */ private def writeMultiFileSourceTable( source: String, fileRanges: Seq[NumericRange.Exclusive[Long]]): Unit = { for (range <- fileRanges) { spark.range(start = range.start, end = range.end, step = 1L, numPartitions = 1).toDF("id") .write.format("delta").mode("append").save(source) } } private def tagAllFilesWithUniqueId(deltaLog: DeltaLog, tagName: String): Unit = { deltaLog.withNewTransaction { txn => val allFiles = txn.snapshot.allFiles.collect() val allFilesWithTags = allFiles.map { addFile => addFile.copyWithTags(Map(tagName -> java.util.UUID.randomUUID().toString)) } txn.commit(allFilesWithTags, DeltaOperations.ManualUpdate) } // Double check that the result is as expected. val snapshotWithTags = deltaLog.update() val filesWithTags = snapshotWithTags.allFiles.collect() assert(filesWithTags.forall(_.tags.get(tagName).isDefined)) assert(filesWithTags.map(_.tags(tagName)).toSet.size === filesWithTags.size) } private def runAndValidateCloneWithDVs( source: String, target: String, isShallow: Boolean, expectedNumFilesWithDVs: Int, isReplaceOperation: Boolean = false): Unit = { val sourceDeltaLog = DeltaLog.forTable(spark, source) // Add a unique tag to each file, so we can use this later to match up pre-/post-clone entries // without having to resolve all the possible combinations of relative vs. absolute paths. val uniqueIdTag = "unique-file-id" tagAllFilesWithUniqueId(sourceDeltaLog, uniqueIdTag) val targetDeltaLog = DeltaLog.forTable(spark, target) val filesWithDVsInSource = getFilesWithDeletionVectors(sourceDeltaLog) assert(filesWithDVsInSource.size === expectedNumFilesWithDVs) val numberOfUniqueDVFilesInSource = filesWithDVsInSource .map(_.deletionVector.pathOrInlineDv) .toSet .size runAndValidateClone( source, target, isShallow, isReplaceOperation = isReplaceOperation)() val filesWithDVsInTarget = getFilesWithDeletionVectors(targetDeltaLog) val numberOfUniqueDVFilesInTarget = filesWithDVsInTarget .map(_.deletionVector.pathOrInlineDv) .toSet .size // Make sure we didn't accidentally copy some file multiple times. assert(numberOfUniqueDVFilesInSource === numberOfUniqueDVFilesInTarget) // Check contents of the copied DV files. val filesWithDVsInTargetByUniqueId = filesWithDVsInTarget .map(addFile => addFile.tags(uniqueIdTag) -> addFile) .toMap // scalastyle:off deltahadoopconfiguration val hadoopConf = spark.sessionState.newHadoopConf() // scalastyle:on deltahadoopconfiguration for (sourceFile <- filesWithDVsInSource) { val targetFile = filesWithDVsInTargetByUniqueId(sourceFile.tags(uniqueIdTag)) if (sourceFile.deletionVector.isInline) { assert(targetFile.deletionVector.isInline) assert(sourceFile.deletionVector.inlineData === targetFile.deletionVector.inlineData) } else { def readDVData(path: Path): Array[Byte] = { val fs = path.getFileSystem(hadoopConf) val size = fs.getFileStatus(path).getLen val data = new Array[Byte](size.toInt) Utils.tryWithResource(fs.open(path)) { reader => reader.readFully(data) } data } val sourceDVPath = sourceFile.deletionVector.absolutePath(sourceDeltaLog.dataPath) val targetDVPath = targetFile.deletionVector.absolutePath(targetDeltaLog.dataPath) val sourceData = readDVData(sourceDVPath) val targetData = readDVData(targetDVPath) assert(sourceData === targetData) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/CloneTableScalaSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.time.LocalDate import org.apache.spark.sql.delta.util.AnalysisHelper import org.apache.hadoop.fs.Path class CloneTableScalaSuite extends CloneTableSuiteBase with CloneTableScalaTestMixin with AnalysisHelper with DeltaColumnMappingTestUtils { import testImplicits._ testAllClones("cloneAtVersion API") { (source, target, isShallow) => spark.range(5).write.format("delta").save(source) spark.range(5).write.format("delta").mode("append").save(source) spark.range(5).write.format("delta").mode("append").save(source) val sourceTbl = io.delta.tables.DeltaTable.forPath(source) assert(spark.read.format("delta").load(source).count() === 15) runAndValidateClone(source, target, isShallow, sourceVersion = Some(0)) { () => { sourceTbl.cloneAtVersion(0, target, isShallow) } } } test("deep clone not supported yet") { withSourceTargetDir { (source, clone) => checkError( intercept[DeltaIllegalArgumentException] { val df1 = Seq(1, 2, 3, 4, 5).toDF("id").withColumn("part", 'id % 2) val df2 = Seq(8, 9, 10).toDF("id").withColumn("part", 'id % 2) df1.write.format("delta").partitionBy("part").mode("append").save(source) df2.write.format("delta").mode("append").save(source) runAndValidateClone(source, clone, isShallow = false)() }, "DELTA_UNSUPPORTED_DEEP_CLONE" ) } } testAllClones("clone API") { (source, target, isShallow) => spark.range(5).write.format("delta").save(source) spark.range(5).write.format("delta").mode("append").save(source) spark.range(5).write.format("delta").mode("append").save(source) val sourceTbl = io.delta.tables.DeltaTable.forPath(source) assert(spark.read.format("delta").load(source).count() === 15) runAndValidateClone(source, target, isShallow) { () => { sourceTbl.clone(target, isShallow) } } } testAllClones("cloneAtTimestamp API") { (source, target, isShallow) => spark.range(5).write.format("delta").save(source) spark.range(5).write.format("delta").mode("append").save(source) spark.range(5).write.format("delta").mode("append").save(source) val sourceTbl = io.delta.tables.DeltaTable.forPath(source) assert(spark.read.format("delta").load(source).count() === 15) val desiredTime = LocalDate.now().minusDays(5).toString // Date as of 5 days old val format = new java.text.SimpleDateFormat("yyyy-MM-dd") val time = format.parse(desiredTime).getTime val path = new Path(source + "/_delta_log/00000000000000000000.json") // scalastyle:off deltahadoopconfiguration val fs = path.getFileSystem(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration fs.setTimes(path, time, 0) if (catalogOwnedDefaultCreationEnabledInTests) { InCommitTimestampTestUtils.overwriteICTInDeltaFile( DeltaLog.forTable(spark, source), path, Some(time)) } runAndValidateClone(source, target, isShallow, sourceTimestamp = Some(desiredTime)) { () => { sourceTbl.cloneAtTimestamp(desiredTime, target, isShallow) } } } } class CloneTableScalaIdColumnMappingSuite extends CloneTableScalaSuite with CloneTableColumnMappingSuiteBase with DeltaColumnMappingEnableIdMode { override protected def runOnlyTests: Seq[String] = super.runOnlyTests ++ Seq( "cloneAtVersion API", "clone API", "cloneAtTimestamp API" ) } class CloneTableScalaNameColumnMappingSuite extends CloneTableScalaSuite with CloneTableColumnMappingNameSuiteBase with DeltaColumnMappingEnableNameMode { override protected def runOnlyTests: Seq[String] = super.runOnlyTests ++ Seq( "cloneAtVersion API", "clone API", "cloneAtTimestamp API" ) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/CloneTableSuiteBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.time.LocalDate import org.apache.spark.sql.delta.actions.{AddFile, FileAction, Metadata, Protocol, RemoveFile, SetTransaction, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION import org.apache.spark.sql.delta.commands._ import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CatalogOwnedTestBaseSuite} import org.apache.spark.sql.delta.coordinatedcommits.CoordinatedCommitsTestUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaColumnMappingSelectedTestMixin, DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.util.FileNames.{isCheckpointFile, unsafeDeltaFile} import org.apache.hadoop.fs.Path import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.streaming.OutputMode import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{IntegerType, StructType} import org.apache.spark.util.Utils trait CloneTableSuiteBase extends QueryTest with SharedSparkSession with CloneTableTestMixin with DeltaColumnMappingTestUtils with DeltaSQLCommandTest with DeltaSQLTestUtils with CatalogOwnedTestBaseSuite with CoordinatedCommitsTestUtils with DeletionVectorsTestUtils { import testImplicits._ protected def deleteSourceAndCompareData( source: String, actual: => DataFrame, expected: DataFrame): Unit = { Utils.deleteRecursively(new File(source)) checkAnswer(actual, expected) } protected def verifyAllFilePaths( table: String, targetIsTable: Boolean = false, expectAbsolute: Boolean): Unit = { val targetLog = if (targetIsTable) { DeltaLog.forTable(spark, TableIdentifier(table)) } else { DeltaLog.forTable(spark, table) } assert(targetLog.unsafeVolatileSnapshot.allFiles.collect() .forall(p => new Path(p.pathAsUri).isAbsolute == expectAbsolute)) } protected def customConvertToDelta(internal: String, external: String): Unit = { ConvertToDeltaCommand( TableIdentifier(external, Some("parquet")), Option(new StructType().add("part", IntegerType)), collectStats = true, Some(internal)).run(spark) } // Test a basic clone with different syntaxes protected def testSyntax( source: String, target: String, sqlString: String, isShallow: Boolean = true, targetIsTable: Boolean = false): Unit = { withTable(source) { spark.range(5).write.format("delta").saveAsTable(source) runAndValidateClone( source, target, isShallow, sourceIsTable = true, targetIsTable = targetIsTable) { () => sql(sqlString) } } } cloneTest("simple shallow clone", TAG_HAS_SHALLOW_CLONE) { (source, clone) => val df1 = Seq(1, 2, 3, 4, 5).toDF("id").withColumn("part", 'id % 2) val df2 = Seq(8, 9, 10).toDF("id").withColumn("part", 'id % 2) df1.write.format("delta").partitionBy("part").mode("append").save(source) df2.write.format("delta").mode("append").save(source) runAndValidateClone( source, clone, isShallow = true )() // no files should be copied val cloneDir = new File(clone).list() assert(cloneDir.length === 1, s"There should only be a _delta_log directory but found:\n${cloneDir.mkString("\n")}") val cloneLog = DeltaLog.forTable(spark, clone) assert(cloneLog.snapshot.version === 0) assert(cloneLog.snapshot.metadata.partitionColumns === Seq("part")) val files = cloneLog.snapshot.allFiles.collect() assert(files.forall(_.pathAsUri.toString.startsWith("file:/")), "paths must be absolute") checkAnswer( spark.read.format("delta").load(clone), df1.union(df2) ) } cloneTest("shallow clone a shallow clone", TAG_HAS_SHALLOW_CLONE) { (source, clone) => val shallow1 = new File(clone, "shallow1").getCanonicalPath val shallow2 = new File(clone, "shallow2").getCanonicalPath val df1 = Seq(1, 2, 3, 4, 5).toDF("id").withColumn("part", 'id % 2) df1.write.format("delta").partitionBy("part").mode("append").save(source) runAndValidateClone( source, shallow1, isShallow = true )() runAndValidateClone( shallow1, shallow2, isShallow = true )() deleteSourceAndCompareData(shallow1, spark.read.format("delta").load(shallow2), df1) } testAllClones(s"validate commitLarge usage metrics") { (source, clone, isShallow) => val df1 = Seq(1, 2, 3, 4, 5).toDF("id").withColumn("part", 'id % 5) df1.write.format("delta").partitionBy("part").mode("append").save(source) val df2 = Seq(1, 2).toDF("id").withColumn("part", 'id % 5) df2.write.format("delta").partitionBy("part").mode("append").save(source) val numAbsolutePathsInAdd = if (isShallow) 7 else 0 val commitLargeMetricsMap = Map( "numAdd" -> "7", "numRemove" -> "0", "numFilesTotal" -> "7", "numCdcFiles" -> "0", "commitVersion" -> "0", "readVersion" -> "0", "numAbsolutePathsInAdd" -> s"$numAbsolutePathsInAdd", "startVersion" -> "-1", "numDistinctPartitionsInAdd" -> "5") runAndValidateClone( source, clone, isShallow, commitLargeMetricsMap = commitLargeMetricsMap)() checkAnswer( spark.read.format("delta").load(clone), df1.union(df2) ) } cloneTest("shallow clone across file systems", TAG_HAS_SHALLOW_CLONE) { (source, clone) => withSQLConf( "fs.s3.impl" -> classOf[S3LikeLocalFileSystem].getName, "fs.s3.impl.disable.cache" -> "true") { val df1 = Seq(1, 2, 3, 4, 5).toDF("id") df1.write.format("delta").mode("append").save(s"s3:$source") runAndValidateClone( s"s3:$source", s"file:$clone", isShallow = true )() checkAnswer( spark.read.format("delta").load(clone), df1 ) val cloneLog = DeltaLog.forTable(spark, clone) assert(cloneLog.snapshot.version === 0) val files = cloneLog.snapshot.allFiles.collect() assert(files.forall(_.pathAsUri.toString.startsWith("s3:/"))) } } testAllClones("Negative test: clone into a non-empty directory that has a path based " + "delta table") { (source, clone, isShallow) => // Create table to clone spark.range(5).write.format("delta").mode("append").save(source) // Table already exists at destination directory spark.range(5).write.format("delta").mode("append").save(clone) // Clone should fail since destination directory is non-empty val ex = intercept[AnalysisException] { runAndValidateClone( source, clone, isShallow )() } assert(ex.getMessage.contains("is not empty")) } cloneTest("Negative test: cloning into a non-empty parquet directory", TAG_HAS_SHALLOW_CLONE) { (source, clone) => // Create table to clone spark.range(5).write.format("delta").mode("append").save(source) // Table already exists at destination directory spark.range(5).write.format("parquet").mode("overwrite").save(clone) // Clone should fail since destination directory is non-empty val ex = intercept[AnalysisException] { sql(s"CREATE TABLE delta.`$clone` SHALLOW CLONE delta.`$source`") } assert(ex.getMessage.contains("is not empty and also not a Delta table")) } testAllClones( "Changes to clones only affect the cloned directory") { (source, target, isShallow) => // Create base directory Seq(1, 2, 3, 4, 5).toDF("id").write.format("delta").save(source) // Create a clone runAndValidateClone( source, target, isShallow )() // Write to clone should be visible Seq(6, 7, 8).toDF("id").write.format("delta").mode("append").save(target) assert(spark.read.format("delta").load(target).count() === 8) // Write to clone should not be visible in original table assert(spark.read.format("delta").load(source).count() === 5) } testAllClones("simple clone of source using table name") { (_, target, isShallow) => val tableName = "source" withTable(tableName) { spark.range(5).write.format("delta").saveAsTable(tableName) runAndValidateClone( tableName, target, isShallow, sourceIsTable = true)() } } testAllClones("clone a time traveled source using version") { (_, target, isShallow) => val tableName = "source" withTable(tableName) { spark.range(5).write.format("delta").saveAsTable(tableName) spark.range(5).write.format("delta").mode("append").saveAsTable(tableName) spark.range(5).write.format("delta").mode("append").saveAsTable(tableName) spark.range(5).write.format("delta").mode("append").saveAsTable(tableName) assert(DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName))._2.version === 3) runAndValidateClone( tableName, target, isShallow, sourceIsTable = true, sourceVersion = Some(2))() assert(spark.read.format("delta").load(target).count() === 15) } } Seq(true, false).foreach { isCreate => cloneTest(s"create or replace table - shallow, isCreate: $isCreate", TAG_HAS_SHALLOW_CLONE) { (_, _) => val tbl = "source" val target = "target" withTable(tbl, target) { spark.range(5).write.format("delta").saveAsTable(tbl) spark.range(25).write.format("delta").saveAsTable(target) runAndValidateClone( tbl, target, isShallow = true, sourceIsTable = true, targetIsTable = true, isCreate = isCreate, isReplaceOperation = true)() } } } Seq(true, false).foreach { isCreate => Seq("parquet", "json").foreach { format => cloneTest(s"create or replace non Delta table - shallow, isCreate: $isCreate, " + s"format: $format", TAG_HAS_SHALLOW_CLONE) { (_, _) => val tbl = "source" val target = "target" withTable(tbl, target) { spark.range(5).write.format("delta").saveAsTable(tbl) spark.range(25).write.format(format).saveAsTable(target) runAndValidateClone( tbl, target, isShallow = true, sourceIsTable = true, targetIsTable = true, isCreate = isCreate, isReplaceOperation = true, isReplaceDelta = false)() } } } } Seq(true, false).foreach { isCreate => cloneTest(s"shallow clone a table unto itself, isCreate: $isCreate", TAG_HAS_SHALLOW_CLONE) { (_, _) => val tbl = "source" withTable(tbl) { spark.range(5).write.format("delta").saveAsTable(tbl) runAndValidateClone( tbl, tbl, isShallow = true, sourceIsTable = true, targetIsTable = true, isCreate = isCreate, isReplaceOperation = true)() val allFiles = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tbl))._2.allFiles.collect() allFiles.foreach { file => assert(!file.pathAsUri.isAbsolute, "File paths should not be absolute") } } } } cloneTest("CLONE ignores reader/writer session defaults", TAG_HAS_SHALLOW_CLONE) { (source, clone) => if (catalogOwnedDefaultCreationEnabledInTests) { cancel("Expects base protocol version.") } withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> "1", DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "1") { // Create table without a default property setting. spark.range(1L).write.format("delta").mode("overwrite").save(source) val oldProtocol = DeltaLog.forTable(spark, source).update().protocol assert(oldProtocol === Protocol(1, 1)) // Just use something that can be default. withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> "2", DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "2", TableFeatureProtocolUtils.defaultPropertyKey(TestWriterFeature) -> "enabled") { // Clone in a session with default properties and check that they aren't merged // (i.e. target properties are identical to source properties). runAndValidateClone( source, clone, isShallow = true )() } val log = DeltaLog.forTable(spark, clone) val targetProtocol = log.update().protocol assert(targetProtocol === oldProtocol) } } testAllClones("clone a time traveled source using timestamp") { (source, clone, isShallow) => // Create source spark.range(5).write.format("delta").save(source) spark.range(5).write.format("delta").mode("append").save(source) spark.range(5).write.format("delta").mode("append").save(source) assert(spark.read.format("delta").load(source).count() === 15) // Get time corresponding to date val desiredTime = LocalDate.now().minusDays(5).toString // Date as of 5 days old val format = new java.text.SimpleDateFormat("yyyy-MM-dd") val time = format.parse(desiredTime).getTime // Change modification time of commit val path = new Path(source + "/_delta_log/00000000000000000000.json") // scalastyle:off deltahadoopconfiguration val fs = path.getFileSystem(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration fs.setTimes(path, time, 0) if (catalogOwnedDefaultCreationEnabledInTests) { InCommitTimestampTestUtils.overwriteICTInDeltaFile( DeltaLog.forTable(spark, source), path, Some(time)) } runAndValidateClone( source, clone, isShallow, sourceTimestamp = Some(desiredTime))() } cloneTest("clones take protocol from the source", TAG_HAS_SHALLOW_CLONE, TAG_MODIFY_PROTOCOL, TAG_CHANGE_COLUMN_MAPPING_MODE) { (source, clone) => if (catalogOwnedDefaultCreationEnabledInTests) { cancel("table needs to start with custom protocol versions but enabling " + "catalogOwned automatically upgrades table protocol version.") } // Change protocol versions of (read, write) = (2, 5). We cannot initialize this to (0, 0) // because min reader and writer versions are at least 1. val defaultNewTableProtocol = Protocol.forNewTable(spark, metadataOpt = None) val sourceProtocol = Protocol(2, 5) // Make sure this is actually an upgrade. Downgrades are not supported, and if it's the same // version, we aren't testing anything there. assert(sourceProtocol.minWriterVersion > defaultNewTableProtocol.minWriterVersion && sourceProtocol.minReaderVersion > defaultNewTableProtocol.minReaderVersion) val log = DeltaLog.forTable(spark, source) // make sure to have a dummy schema because we can't have empty schema table by default val newSchema = new StructType().add("id", IntegerType, nullable = true) log.createLogDirectoriesIfNotExists() log.store.write( unsafeDeltaFile(log.logPath, 0), Iterator(Metadata(schemaString = newSchema.json).json, sourceProtocol.json), overwrite = false, log.newDeltaHadoopConf()) log.update() // Validate that clone has the new protocol version runAndValidateClone( source, clone, isShallow = true )() } testAllClones("clones take the set transactions of the source") { (_, target, isShallow) => withTempDir { dir => // Create source val path = dir.getCanonicalPath spark.range(5).write.format("delta").save(path) // Add a Set Transaction val log = DeltaLog.forTable(spark, path) val txn = log.startTransaction() val setTxn = SetTransaction("app-id", 0, Some(0L)) :: Nil val op = DeltaOperations.StreamingUpdate(OutputMode.Complete(), "app-id", 0L) txn.commit(setTxn, op) log.update() runAndValidateClone( path, target, isShallow )() } } testAllClones("CLONE with table properties to disable DV") { (source, target, isShallow) => withSQLConf( DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> "true", DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS.key -> "true") { spark.range(10).write.format("delta").save(source) spark.sql(s"DELETE FROM delta.`$source` WHERE id = 1") } intercept[DeltaCommandUnsupportedWithDeletionVectorsException] { runAndValidateClone( source, target, isShallow, tableProperties = Map(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key -> "false"))() }.getErrorClass === "DELTA_ADDING_DELETION_VECTORS_DISALLOWED" } for(targetExists <- BOOLEAN_DOMAIN) testAllClones(s"CLONE respects table features set by table property override, " + s"targetExists=$targetExists", TAG_MODIFY_PROTOCOL) { (source, target, isShallow) => spark.range(10).write.format("delta").save(source) if (targetExists) { spark.range(0).write.format("delta").save(target) } val tblPropertyOverrides = Seq( s"delta.feature.${TestWriterFeature.name}" -> "enabled", "delta.minWriterVersion" -> s"$TABLE_FEATURES_MIN_WRITER_VERSION").toMap cloneTable( source, target, isShallow, isReplace = true, tableProperties = tblPropertyOverrides) val targetLog = DeltaLog.forTable(spark, target) assert(targetLog.update().protocol.isFeatureSupported(TestWriterFeature)) } case class TableFeatureWithProperty( feature: TableFeature, property: DeltaConfig[Boolean]) // Delta properties that automatically cause a version upgrade when enabled via ALTER TABLE. final val featuresWithAutomaticProtocolUpgrade: Seq[TableFeatureWithProperty] = Seq( TableFeatureWithProperty(ChangeDataFeedTableFeature, DeltaConfigs.CHANGE_DATA_FEED), TableFeatureWithProperty( DeletionVectorsTableFeature, DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION)) // This test ensures this upgrade also happens when enabled during a CLONE. for (featureWithProperty <- featuresWithAutomaticProtocolUpgrade) testAllClones("Cloning a table with new table properties" + s" that force protocol version upgrade - ${featureWithProperty.property.key}" ) { (source, target, isShallow) => if (catalogOwnedDefaultCreationEnabledInTests) { cancel("table needs to start with default protocol versions but enabling " + "catalogOwned upgrades table protocol version.") } import DeltaTestUtils.StrictProtocolOrdering spark.range(5).write.format("delta").save(source) val sourceDeltaLog = DeltaLog.forTable(spark, source) val sourceSnapshot = sourceDeltaLog.update() // This only works if the featureWithProperty is not enabled by default. assert(!featureWithProperty.property.fromMetaData(sourceSnapshot.metadata)) // Check that the original version is not already sufficient for the featureWithProperty. assert(!StrictProtocolOrdering.fulfillsVersionRequirements( actual = sourceSnapshot.protocol, requirement = featureWithProperty.feature.minProtocolVersion )) // Clone the table, enabling the featureWithProperty in an override. val tblProperties = Map(featureWithProperty.property.key -> "true") cloneTable( source, target, isShallow, isReplace = true, tableProperties = tblProperties) val targetDeltaLog = DeltaLog.forTable(spark, target) val targetSnapshot = targetDeltaLog.update() assert(targetSnapshot.metadata.configuration === sourceSnapshot.metadata.configuration ++ tblProperties) // Check that the protocol has been upgraded. assert(StrictProtocolOrdering.fulfillsVersionRequirements( actual = targetSnapshot.protocol, requirement = featureWithProperty.feature.minProtocolVersion )) } testAllClones("Cloning a table without DV property should not upgrade protocol version" ) { (source, target, isShallow) => if (catalogOwnedDefaultCreationEnabledInTests) { cancel("table needs to start with default protocol versions but enabling " + "catalogOwned upgrades table protocol version.") } import DeltaTestUtils.StrictProtocolOrdering spark.range(5).write.format("delta").save(source) withSQLConf(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> "true") { val sourceDeltaLog = DeltaLog.forTable(spark, source) val sourceSnapshot = sourceDeltaLog.update() // Should not be enabled, just because it's allowed. assert(!DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.fromMetaData(sourceSnapshot.metadata)) // Check that the original version is not already sufficient for the feature. assert(!StrictProtocolOrdering.fulfillsVersionRequirements( actual = sourceSnapshot.protocol, requirement = DeletionVectorsTableFeature.minProtocolVersion )) // Clone the table. cloneTable( source, target, isShallow, isReplace = true) val targetDeltaLog = DeltaLog.forTable(spark, target) val targetSnapshot = targetDeltaLog.update() // Protocol should not have been upgraded. assert(sourceSnapshot.protocol === targetSnapshot.protocol) } } } trait CloneTableColumnMappingSuiteBase extends CloneTableSuiteBase with DeltaColumnMappingSelectedTestMixin { override protected def runOnlyTests: Seq[String] = Seq( "simple shallow clone", "shallow clone a shallow clone", "create or replace table - shallow, isCreate: false", "create or replace table - shallow, isCreate: true", "shallow clone a table unto itself, isCreate: false", "shallow clone a table unto itself, isCreate: true", "clone a time traveled source using version", "clone a time traveled source using timestamp", "validate commitLarge usage metrics", "clones take the set transactions of the source", "block changing column mapping mode and modify max id modes under CLONE" ) import testImplicits._ testAllClones("block changing column mapping mode and modify max id modes under CLONE") { (_, _, isShallow) => val df1 = Seq(1, 2, 3, 4, 5).toDF("id").withColumn("part", 'id % 2) // block setting max id def validateModifyMaxIdError(f: => Any): Unit = { val e = intercept[UnsupportedOperationException] { f } assert(e.getMessage == DeltaErrors.cannotModifyTableProperty( DeltaConfigs.COLUMN_MAPPING_MAX_ID.key ).getMessage) } withSourceTargetDir { (source, target) => df1.write.format("delta").partitionBy("part").mode("append").save(source) // change max id w/ table property should be blocked validateModifyMaxIdError { cloneTable( source, target, isShallow, tableProperties = Map( DeltaConfigs.COLUMN_MAPPING_MAX_ID.key -> "123123" )) } // change max id w/ SQLConf should be blocked by table property guard validateModifyMaxIdError { withMaxColumnIdConf("123123") { cloneTable( source, target, isShallow ) } } } // block changing column mapping mode def validateChangeModeError(f: => Any): Unit = { val e = intercept[ColumnMappingUnsupportedException] { f } assert(e.getMessage.contains("Changing column mapping mode from")) } val currentMode = columnMappingModeString // currentMode to otherMode val otherMode = if (currentMode == "id") "name" else "id" withSourceTargetDir { (source, target) => df1.write.format("delta").partitionBy("part").mode("append").save(source) // change mode w/ table property should be blocked validateChangeModeError { cloneTable( source, target, isShallow, tableProperties = Map( DeltaConfigs.COLUMN_MAPPING_MODE.key -> otherMode )) } } withSourceTargetDir { (source, target) => df1.write.format("delta").partitionBy("part").mode("append").save(source) // change mode w/ SQLConf should have no effects withColumnMappingConf(otherMode) { cloneTable( source, target, isShallow ) } assert(DeltaLog.forTable(spark, target).snapshot.metadata.columnMappingMode.name == currentMode) } // currentMode to none withSourceTargetDir { (source, target) => df1.write.format("delta").partitionBy("part").mode("append").save(source) // change mode w/ table property validateChangeModeError { cloneTable( source, target, isShallow, tableProperties = Map( DeltaConfigs.COLUMN_MAPPING_MODE.key -> "none" )) } } withSourceTargetDir { (source, target) => df1.write.format("delta").partitionBy("part").mode("append").save(source) // change mode w/ SQLConf should have no effects withColumnMappingConf("none") { cloneTable( source, target, isShallow ) } assert(DeltaLog.forTable(spark, target).snapshot.metadata.columnMappingMode.name == currentMode) } } } trait CloneTableColumnMappingNameSuiteBase extends CloneTableColumnMappingSuiteBase { override protected def customConvertToDelta(internal: String, external: String): Unit = { withColumnMappingConf("none") { super.customConvertToDelta(internal, external) sql( s"""ALTER TABLE delta.`$internal` SET TBLPROPERTIES ( |${DeltaConfigs.COLUMN_MAPPING_MODE.key} = 'name', |${DeltaConfigs.MIN_READER_VERSION.key} = '2', |${DeltaConfigs.MIN_WRITER_VERSION.key} = '5' |)""".stripMargin) .collect() } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/CloneTableTestMixin.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.util.Locale import scala.jdk.CollectionConverters._ import com.databricks.spark.util.{Log4jUsageLogger, UsageRecord} import org.apache.spark.sql.delta.actions.{AddFile, FileAction, RemoveFile, SingleAction} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.{CloneDeltaSource, CloneSource, CloneSourceFormat} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.FileNames.unsafeDeltaFile import org.apache.spark.sql.delta.util.JsonUtils import org.apache.hadoop.fs.Path import org.scalatest.{BeforeAndAfterAll, Tag} import org.apache.spark.sql.{DataFrame, QueryTest} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.expressions.Literal import org.apache.spark.sql.connector.catalog.CatalogManager import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.Utils /** Common test setup and utils for CLONE TABLE tests. */ trait CloneTableTestMixin extends DeltaColumnMappingTestUtils with BeforeAndAfterAll with DeltaTestUtilsBase { self: QueryTest with SharedSparkSession => protected val TAG_HAS_SHALLOW_CLONE = new Tag("SHALLOW CLONE") protected val TAG_MODIFY_PROTOCOL = new Tag("CHANGES PROTOCOL") protected val TAG_CHANGE_COLUMN_MAPPING_MODE = new Tag("CHANGES COLUMN MAPPING MODE") protected val TAG_USES_CONVERT_TO_DELTA = new Tag("USES CONVERT TO DELTA") // scalastyle:off argcount protected def cloneTable( source: String, target: String, isShallow: Boolean, sourceIsTable: Boolean = false, targetIsTable: Boolean = false, targetLocation: Option[String] = None, versionAsOf: Option[Long] = None, timestampAsOf: Option[String] = None, isCreate: Boolean = true, isReplace: Boolean = false, tableProperties: Map[String, String] = Map.empty): Unit // scalastyle:on argcount protected def withSourceTargetDir(f: (String, String) => Unit): Unit = { withTempDir { dir => val firstDir = new File(dir, "source").getCanonicalPath val secondDir = new File(dir, "clone").getCanonicalPath f(firstDir, secondDir) } } protected def cloneTypeStr(isShallow: Boolean): String = { "SHALLOW" } /** * Run the given test function for SHALLOW clone. */ protected def testAllClones(testName: String, testTags: org.scalatest.Tag*) (testFunc: (String, String, Boolean) => Unit): Unit = { val tags = Seq(TAG_HAS_SHALLOW_CLONE) cloneTest(s"$testName", testTags ++ tags: _*) { (source, target) => testFunc(source, target, true) } } protected def cloneTest( testName: String, testTags: org.scalatest.Tag*)(f: (String, String) => Unit): Unit = { if (testTags.exists(_.name == TAG_CHANGE_COLUMN_MAPPING_MODE.name) && columnMappingMode != "none") { ignore(testName + " (not supporting changing column mapping mode)") { withSourceTargetDir(f) } } else { test(testName, testTags: _*) { withSourceTargetDir(f) } } } // Extracted function so it can be overriden in subclasses. protected def uniqueFileActionGroupBy(action: FileAction): String = { val filePath = action.pathAsUri.toString val dvId = action match { case add: AddFile => Option(add.deletionVector).map(_.uniqueId).getOrElse("") case remove: RemoveFile => Option(remove.deletionVector).map(_.uniqueId).getOrElse("") case _ => "" } filePath + dvId } import testImplicits._ // scalastyle:off protected def runAndValidateClone( source: String, target: String, isShallow: Boolean, sourceIsTable: Boolean = false, targetIsTable: Boolean = false, targetLocation: Option[String] = None, sourceVersion: Option[Long] = None, sourceTimestamp: Option[String] = None, isCreate: Boolean = true, // If we are doing a replace on an existing table isReplaceOperation: Boolean = false, // If we are doing a replace, whether it is on a Delta table isReplaceDelta: Boolean = true, tableProperties: Map[String, String] = Map.empty, commitLargeMetricsMap: Map[String, String] = Map.empty, expectedDataframe: DataFrame = spark.emptyDataFrame) (f: () => Unit = () => cloneTable( source, target, isShallow, sourceIsTable, targetIsTable, targetLocation, sourceVersion, sourceTimestamp, isCreate, isReplaceOperation, tableProperties)): Unit = { // scalastyle:on // Truncate table before REPLACE try { if (isReplaceOperation) { val targetTbl = if (targetIsTable) { target } else { s"delta.`$target`" } sql(s"DELETE FROM $targetTbl") } } catch { case _: Throwable => // ignore all } // Check logged blob for expected values val allLogs = Log4jUsageLogger.track { f() } verifyAllCloneOperationsEmitted(allLogs, isReplaceOperation && isReplaceDelta, commitLargeMetricsMap) val blob = JsonUtils.fromJson[Map[String, Any]](allLogs .filter(_.metric == "tahoeEvent") .filter(_.tags.get("opType").contains("delta.clone")) .filter(_.blob.contains("source")) .map(_.blob).last) val sourceIdent = resolveTableIdentifier(source, Some("delta"), sourceIsTable) val (cloneSource: CloneSource, sourceDf: DataFrame) = { val sourceLog = DeltaLog.forTable(spark, sourceIdent) val timeTravelSpec: Option[DeltaTimeTravelSpec] = if (sourceVersion.isDefined || sourceTimestamp.isDefined) { Some(DeltaTimeTravelSpec(sourceTimestamp.map(Literal(_)), sourceVersion, None)) } else { None } val deltaTable = DeltaTableV2(spark, sourceLog.dataPath, timeTravelOpt = timeTravelSpec) val sourceData = DataFrameUtils.ofRows( spark, LogicalRelation(sourceLog.createRelation( snapshotToUseOpt = Some(deltaTable.initialSnapshot), isTimeTravelQuery = sourceVersion.isDefined || sourceTimestamp.isDefined))) (new CloneDeltaSource(deltaTable), sourceData) } val targetLog = if (targetIsTable) { DeltaLog.forTable(spark, TableIdentifier(target)) } else { DeltaLog.forTable(spark, target) } val sourceSnapshot = cloneSource.snapshot val sourcePath = cloneSource.dataPath // scalastyle:off deltahadoopconfiguration val fs = sourcePath.getFileSystem(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration val qualifiedSourcePath = fs.makeQualified(sourcePath) val logSource = if (sourceIsTable) { val catalog = CatalogManager.SESSION_CATALOG_NAME s"$catalog.default.$source".toLowerCase(Locale.ROOT) } else { s"delta.`$qualifiedSourcePath`" } val rawTarget = new Path(targetLocation.getOrElse(targetLog.dataPath.toString)) // scalastyle:off deltahadoopconfiguration val targetFs = rawTarget.getFileSystem(targetLog.newDeltaHadoopConf()) // scalastyle:on deltahadoopconfiguration val qualifiedTarget = targetFs.makeQualified(rawTarget) // Check whether recordEvent operation is of correct form assert(blob("source") != null) val actualLogSource = blob("source").toString assert(actualLogSource === logSource) if (source != target) { assert(blob("sourceVersion") === sourceSnapshot.get.version) } val replacingDeltaTable = isReplaceOperation && isReplaceDelta assert(blob("sourcePath") === qualifiedSourcePath.toString) assert(blob("target") === qualifiedTarget.toString) assert(blob("isReplaceDelta") === replacingDeltaTable) assert(blob("sourceTableSize") === cloneSource.sizeInBytes) assert(blob("sourceNumOfFiles") === cloneSource.numOfFiles) assert(blob("partitionBy") === cloneSource.metadata.partitionColumns) // Check whether resulting metadata of target and source at version is the same compareMetadata( cloneSource, targetLog.unsafeVolatileSnapshot, targetLocation.isEmpty && targetIsTable, isReplaceOperation) val commit = unsafeDeltaFile(targetLog.logPath, targetLog.unsafeVolatileSnapshot.version) val hadoopConf = targetLog.newDeltaHadoopConf() val filePaths: Seq[FileAction] = targetLog.store.read(commit, hadoopConf).flatMap { line => JsonUtils.fromJson[SingleAction](line) match { case a if a.add != null => Some(a.add) case a if a.remove != null => Some(a.remove) case _ => None } } assert(filePaths.groupBy(uniqueFileActionGroupBy(_)).forall(_._2.length === 1), "A file was added and removed in the same commit") // Check whether the resulting datasets are the same val targetDf = DataFrameUtils.ofRows( spark, LogicalRelation(targetLog.createRelation())) checkAnswer( targetDf, sourceDf) } protected def verifyAllCloneOperationsEmitted( allLogs: Seq[UsageRecord], emitHandleExistingTable: Boolean, commitLargeMetricsMap: Map[String, String] = Map.empty): Unit = { val cloneLogs = allLogs .filter(_.metric === "sparkOperationDuration") .filter(_.opType.isDefined) .filter(_.opType.get.typeName.contains("delta.clone")) assert(cloneLogs.count(_.opType.get.typeName.equals("delta.clone.makeAbsolute")) == 1) val commitStatsUsageRecords = allLogs .filter(_.metric === "tahoeEvent") .filter(_.tags.get("opType") === Some("delta.commit.stats")) assert(commitStatsUsageRecords.length === 1) val commitStatsMap = JsonUtils.fromJson[Map[String, Any]](commitStatsUsageRecords.head.blob) commitLargeMetricsMap.foreach { case (name, expectedValue) => assert(commitStatsMap(name).toString == expectedValue, s"Expected value for $name metrics did not match with the captured value") } } private def compareMetadata( cloneSource: CloneSource, targetLog: Snapshot, targetIsTable: Boolean, isReplace: Boolean = false): Unit = { val sourceMetadata = cloneSource.metadata val targetMetadata = targetLog.metadata /** * Filter out row tracking properties from the configuration map. * These properties are not expected to be the same in source and target metadata. * * @param conf The configuration map to filter. * @return Filtered configuration map without row tracking properties. */ def filterOutRowTrackingProps(conf: Map[String, String]): Map[String, String] = { conf.filterNot { case (k, _) => k == MaterializedRowCommitVersion.MATERIALIZED_COLUMN_NAME_PROP || k == MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP } } val sourceConfigsWithoutRowTrackingProps = filterOutRowTrackingProps(conf = sourceMetadata.configuration) val targetConfigsWithoutRowTrackingProps = filterOutRowTrackingProps(conf = targetMetadata.configuration) assert(sourceMetadata.schema === targetMetadata.schema && sourceConfigsWithoutRowTrackingProps === targetConfigsWithoutRowTrackingProps && sourceMetadata.dataSchema === targetMetadata.dataSchema && sourceMetadata.partitionColumns === targetMetadata.partitionColumns && sourceMetadata.format === sourceMetadata.format) // Protocol should be changed, if source.protocol >= target.protocol, otherwise target must // retain it's existing protocol version (i.e. no downgrades). assert(cloneSource.protocol === targetLog.protocol || ( cloneSource.protocol.minReaderVersion <= targetLog.protocol.minReaderVersion && cloneSource.protocol.minWriterVersion <= targetLog.protocol.minWriterVersion)) assert(targetLog.setTransactions.isEmpty) if (!isReplace) { assert(sourceMetadata.id != targetMetadata.id && targetMetadata.name === null && targetMetadata.description === null) } } protected def resolveTableIdentifier( name: String, format: Option[String], isTable: Boolean): TableIdentifier = { if (isTable) { TableIdentifier(name) } else { TableIdentifier(name, format) } } } trait CloneTableSQLTestMixin extends CloneTableTestMixin { self: QueryTest with SharedSparkSession => // scalastyle:off argcount override protected def cloneTable( source: String, target: String, isShallow: Boolean, sourceIsTable: Boolean = false, targetIsTable: Boolean = false, targetLocation: Option[String] = None, versionAsOf: Option[Long] = None, timestampAsOf: Option[String] = None, isCreate: Boolean = true, isReplace: Boolean = false, tableProperties: Map[String, String] = Map.empty): Unit = { val commandSql = CloneTableSQLTestUtils.buildCloneSqlString( source, target, sourceIsTable, targetIsTable, "delta", targetLocation, versionAsOf, timestampAsOf, isCreate, isReplace, tableProperties) sql(commandSql) } // scalastyle:on argcount } trait CloneTableScalaTestMixin extends CloneTableTestMixin { self: QueryTest with SharedSparkSession => // scalastyle:off argcount override protected def cloneTable( source: String, target: String, isShallow: Boolean, sourceIsTable: Boolean = false, targetIsTable: Boolean = false, targetLocation: Option[String] = None, versionAsOf: Option[Long] = None, timestampAsOf: Option[String] = None, isCreate: Boolean = true, isReplace: Boolean = false, tableProperties: Map[String, String] = Map.empty): Unit = { val table = if (sourceIsTable) { io.delta.tables.DeltaTable.forName(spark, source) } else { io.delta.tables.DeltaTable.forPath(spark, source) } if (versionAsOf.isDefined) { table.cloneAtVersion(versionAsOf.get, target, isShallow = isShallow, replace = isReplace, tableProperties) } else if (timestampAsOf.isDefined) { table.cloneAtTimestamp(timestampAsOf.get, target, isShallow = isShallow, replace = isReplace, tableProperties) } else { table.clone(target, isShallow = isShallow, replace = isReplace, properties = new java.util.HashMap[String, String](tableProperties.asJava)) } } // scalastyle:on argcount } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/CommitInfoSerializerSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.sql.{QueryTest, SaveMode} import org.apache.spark.sql.catalyst.expressions.{EqualTo, Literal} import org.apache.spark.sql.streaming.OutputMode import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType} /** * Tests for the correct serialization and deserialization of CommitInfo. * The main focus is on the correct deserialization of operation parameters. * See [[JsonMapDeserializer]] for more details about how operation * parameter deserialization was broken before. */ class CommitInfoSerializerSuite extends QueryTest with SharedSparkSession { def testOperationSerialization(operation: DeltaOperations.Operation): Unit = { val commitInfo = CommitInfo( time = 123L, operation = operation.name, inCommitTimestamp = Some(123L), operationParameters = Map.empty, commandContext = Map("clusterId" -> "23"), readVersion = Some(23), isolationLevel = Some("SnapshotIsolation"), isBlindAppend = Some(true), operationMetrics = Some(Map("m1" -> "v1", "m2" -> "v2")), userMetadata = Some("123"), tags = Some(Map("k1" -> "v1")), txnId = Some("123") ).copy(engineInfo = None) val inMemoryCommitInfo = commitInfo.copy(operationParameters = operation.jsonEncodedValues) val commitInfoSerialized = inMemoryCommitInfo.json val roundTrippedCommitInfo = Action.fromJson(commitInfoSerialized).asInstanceOf[CommitInfo] assert(roundTrippedCommitInfo.operationParameters == inMemoryCommitInfo.operationParameters) assert(roundTrippedCommitInfo == inMemoryCommitInfo) // Also ensure that CommitInfo.getLegacyOperationParameters is correct val legacyPostDeserializationCommitInfo = JsonUtils.mapper.readValue[ActionWrapper](commitInfoSerialized).commitInfo val legacyOperationParametersActual = CommitInfo.getLegacyPostDeserializationOperationParameters( roundTrippedCommitInfo.operationParameters) assert( legacyOperationParametersActual == legacyPostDeserializationCommitInfo.operationParameters) } val testMetadata = Metadata( id = "test-id", name = "test_table", description = "Test table", format = Format(), schemaString = StructType(Seq( StructField("col1", StringType), StructField("col2", IntegerType) )).json, partitionColumns = Seq("col1"), configuration = Map("property1" -> "value1") ) val oldSchema = StructType(Seq(StructField("col1", StringType))) val newSchema = StructType(Seq(StructField("col1", StringType), StructField("col2", IntegerType))) val trackedOperationClasses = Map( "Convert" -> (() => DeltaOperations.Convert( numFiles = 23L, partitionBy = Seq("a", "b"), collectStats = false, catalogTable = Some("t1"), sourceFormat = Some("parquet"))), "DomainMetadataCleanup" -> (() => DeltaOperations.DomainMetadataCleanup(1)), "Write" -> (() => DeltaOperations.Write( mode = SaveMode.Append, partitionBy = Some(Seq("col1", "col2")), predicate = Some("col1 > 10"), userMetadata = Some("test metadata") )), "StreamingUpdate" -> (() => DeltaOperations.StreamingUpdate( outputMode = OutputMode.Append(), queryId = "query-123", epochId = 42L, userMetadata = Some("streaming metadata") )), "Delete" -> (() => DeltaOperations.Delete(Seq(EqualTo(Literal("col1"), Literal("value1"))))), "Truncate" -> (() => DeltaOperations.Truncate()), "Merge" -> (() => DeltaOperations.Merge( predicate = Some(EqualTo(Literal("source.id"), Literal("target.id"))), updatePredicate = Some("source.value > target.value"), deletePredicate = Some("source.flag = 'delete'"), insertPredicate = Some("source.id IS NOT NULL"), matchedPredicates = Seq(DeltaOperations.MergePredicate(Some("matched"), "update")), notMatchedPredicates = Seq(DeltaOperations.MergePredicate(Some("not matched"), "insert")), notMatchedBySourcePredicates = Seq( DeltaOperations.MergePredicate(Some("not matched by source"), "delete")) )), "Update" -> (() => DeltaOperations.Update(Some(EqualTo(Literal("col1"), Literal("value1"))))), "CreateTable" -> (() => DeltaOperations.CreateTable( metadata = testMetadata, isManaged = true, asSelect = true, clusterBy = Some(Seq("col1")) )), "ReplaceTable" -> (() => DeltaOperations.ReplaceTable( metadata = testMetadata, isManaged = true, orCreate = true, asSelect = true, userMetadata = Some("replace metadata"), clusterBy = Some(Seq("col2")))), "SetTableProperties" -> (() => DeltaOperations.SetTableProperties(Map("key1" -> "value1", "key2" -> "value2"))), "UnsetTableProperties" -> (() => DeltaOperations.UnsetTableProperties(Seq("key1", "key2"), ifExists = true)), "DropTableFeature" -> (() => DeltaOperations.DropTableFeature("testFeature", truncateHistory = true)), "AddColumns" -> (() => DeltaOperations.AddColumns(Seq( DeltaOperations.QualifiedColTypeWithPositionForLog( Seq("newCol"), StructField("newCol", StringType), Some("AFTER col1"))))), "DropColumns" -> (() => DeltaOperations.DropColumns(Seq(Seq("col1"), Seq("col2")))), "RenameColumn" -> (() => DeltaOperations.RenameColumn(Seq("oldCol"), Seq("newCol"))), "ChangeColumn" -> (() => DeltaOperations.ChangeColumn( columnPath = Seq("col1"), columnName = "col1", newColumn = StructField("col1", StringType), colPosition = Some("FIRST"))), "ChangeColumns" -> (() => DeltaOperations.ChangeColumns(Seq( DeltaOperations.ChangeColumn( columnPath = Seq("col1"), columnName = "col1", newColumn = StructField("col1", StringType), colPosition = Some("FIRST"))))), "ReplaceColumns" -> (() => DeltaOperations.ReplaceColumns(Seq( StructField("newCol1", StringType), StructField("newCol2", IntegerType)))), "UpgradeProtocol" -> (() => DeltaOperations.UpgradeProtocol(Protocol(minReaderVersion = 1, minWriterVersion = 2))), "UpdateColumnMetadata" -> (() => DeltaOperations.UpdateColumnMetadata( "UPDATE COLUMN METADATA", Seq((Seq("col1"), StructField("col1", StringType))))), "UpdateSchema" -> (() => DeltaOperations.UpdateSchema(oldSchema, newSchema)), "AddConstraint" -> (() => DeltaOperations.AddConstraint("check_positive", "col1 > 0")), "DropConstraint" -> (() => DeltaOperations.DropConstraint("check_positive", Some("col1 > 0"))), "ComputeStats" -> (() => DeltaOperations.ComputeStats(Seq(EqualTo(Literal("col1"), Literal("value1"))))), "Restore" -> (() => DeltaOperations.Restore(Some(5L), Some("2023-01-01T00:00:00Z"))), "Optimize" -> (() => DeltaOperations.Optimize( predicate = Seq(EqualTo(Literal("col1"), Literal("value1"))), zOrderBy = Seq("col1", "col2"), auto = true, clusterBy = Some(Seq("col3")), isFull = false)), "Clone" -> (() => DeltaOperations.Clone( source = "s3://bucket/path/to/table", sourceVersion = 10L)), "VacuumStart" -> (() => DeltaOperations.VacuumStart( retentionCheckEnabled = true, specifiedRetentionMillis = Some(604800000L), defaultRetentionMillis = 604800000L)), "VacuumEnd" -> (() => DeltaOperations.VacuumEnd("COMPLETED")), "Reorg" -> (() => DeltaOperations.Reorg( predicate = Seq(EqualTo(Literal("col1"), Literal("value1"))), applyPurge = true)), "ClusterBy" -> (() => DeltaOperations.ClusterBy( oldClusteringColumns = JsonUtils.toJson(Seq("oldCol1", "oldCol2")), newClusteringColumns = JsonUtils.toJson(Seq("newCol1", "newCol2")))), "RowTrackingBackfill" -> (() => DeltaOperations.RowTrackingBackfill(batchId = 3)), "RowTrackingUnBackfill" -> (() => DeltaOperations.RowTrackingUnBackfill(batchId = 4)), "UpgradeUniformProperties" -> (() => DeltaOperations.UpgradeUniformProperties(Map("uniform.property1" -> "value1"))), "RemoveColumnMapping" -> (() => DeltaOperations.RemoveColumnMapping(Some("remove column mapping metadata"))), "AddDeletionVectorsTombstones" -> (() => DeltaOperations.AddDeletionVectorsTombstones), "ManualUpdate" -> (() => DeltaOperations.ManualUpdate), "EmptyCommit" -> (() => DeltaOperations.EmptyCommit) ) trackedOperationClasses.foreach { case (operationName, operationGenerator) => test(s"$operationName operation serialization") { testOperationSerialization(operationGenerator()) } } val ignoredOperationClasses = Set( "TestOperation" ) test("all operations should be tested in this suite") { val allOperations = DeltaTestUtils.getAllDeltaOperations assert( (allOperations -- ignoredOperationClasses) == trackedOperationClasses.keySet, s"if you add a new operation, please add a new test case in this suite " + "for that operation and then add the operation name to the `trackedOperationClasses` " + "Map in this test. Missing operations: " + s"${allOperations -- ignoredOperationClasses -- trackedOperationClasses.keySet}" ) } } /** * A minimal CommitInfo with only operation parameters. This one * does not use the custom JsonMapDeserializer so we can * use it to test our ability to generate the legacy operation parameters. */ case class LegacyCommitInfoWithOperationParametersOnly( operationParameters: Map[String, String] ) case class ActionWrapper(commitInfo: LegacyCommitInfoWithOperationParametersOnly = null) ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/CommitSanityCheckSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta.DeltaTestUtils.{filterUsageRecords, BOOLEAN_DOMAIN} import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.SparkConf import org.apache.spark.sql.QueryTest import org.apache.spark.sql.test.SharedSparkSession /** * Tests for AddFile sanity checks during commit: * - Empty (0-byte) parquet file detection * - Null partition value validation for NOT NULL columns */ class CommitSanityCheckSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with DeltaSQLTestUtils { override protected def sparkConf: SparkConf = { super.sparkConf .set(DeltaSQLConf.DELTA_EMPTY_FILE_CHECK_THROW_ENABLED.key, "true") .set(DeltaSQLConf.DELTA_NULL_PARTITION_CHECK_THROW_ENABLED.key, "true") } private def createTable(tempDir: java.io.File): DeltaLog = { sql(s"CREATE TABLE delta.`${tempDir.getCanonicalPath}` " + s"(id Long, value String) USING delta") DeltaLog.forTable(spark, tempDir.getCanonicalPath) } /** * Creates a partitioned table with a NOT NULL partition column and returns the DeltaLog. */ private def createPartitionedTableWithNotNullColumn(tempDir: java.io.File): DeltaLog = { sql(s"CREATE TABLE delta.`${tempDir.getCanonicalPath()}` " + s"(part String NOT NULL, value Int) using delta PARTITIONED BY (part)" ) DeltaLog.forTable(spark, tempDir.getCanonicalPath()) } private def commit( deltaLog: DeltaLog, addFile: AddFile, isCommitLarge: Boolean): Unit = { val txn = deltaLog.startTransaction() if (isCommitLarge) { txn.commitLarge( spark, Seq(addFile).toIterator, newProtocolOpt = None, op = DeltaOperations.ManualUpdate, context = Map.empty, metrics = Map.empty ) } else { txn.commit(Seq(addFile), DeltaOperations.ManualUpdate) } } // --------------------------------------------------------------------------- // Empty file checks // --------------------------------------------------------------------------- BOOLEAN_DOMAIN.foreach { isCommitLarge => test(s"detect zero-byte AddFile [isCommitLarge: $isCommitLarge]") { withTempDir { tempDir => val deltaLog = createTable(tempDir) val addFile = AddFile( path = "part-00000.parquet", partitionValues = Map.empty, size = 0, modificationTime = System.currentTimeMillis(), dataChange = true, stats = """{"numRecords": 0}""" ) val e = intercept[IllegalStateException] { commit(deltaLog, addFile, isCommitLarge) } assert(e.getMessage.contains("zero-byte")) assert(e.getMessage.contains("part-00000.parquet")) } } test(s"no error when file size is positive [isCommitLarge: $isCommitLarge]") { withTempDir { tempDir => val deltaLog = createTable(tempDir) val addFile = AddFile( path = "part-00000.parquet", partitionValues = Map.empty, size = 100, modificationTime = System.currentTimeMillis(), dataChange = true, stats = """{"numRecords": 1}""" ) // Should not throw commit(deltaLog, addFile, isCommitLarge) } } test(s"empty file check - only log when throw is disabled " + s"[isCommitLarge: $isCommitLarge]") { withTempDir { tempDir => withSQLConf(DeltaSQLConf.DELTA_EMPTY_FILE_CHECK_THROW_ENABLED.key -> "false") { val deltaLog = createTable(tempDir) val addFile = AddFile( path = "part-00000.parquet", partitionValues = Map.empty, size = 0, modificationTime = System.currentTimeMillis(), dataChange = true, stats = """{"numRecords": 0}""" ) // Should not throw when flag is disabled, only log val events = Log4jUsageLogger.track { commit(deltaLog, addFile, isCommitLarge) } val violationEvents = filterUsageRecords(events, "delta.sanityCheck.emptyParquetFile") assert(violationEvents.size == 1) val eventBlob = JsonUtils.fromJson[Map[String, Any]](violationEvents.head.blob) assert(eventBlob.contains("addFile")) assert(eventBlob.contains("stackTrace")) } } } // --------------------------------------------------------------------------- // Null partition checks // --------------------------------------------------------------------------- test(s"detect null partition value for NOT NULL column with column mapping " + s"[isCommitLarge: $isCommitLarge]") { withTempDir { tempDir => sql(s"CREATE TABLE delta.`${tempDir.getCanonicalPath()}` " + s"(part String NOT NULL, value Int) USING delta PARTITIONED BY (part) " + s"TBLPROPERTIES('delta.columnMapping.mode'='name')") val deltaLog = DeltaLog.forTable(spark, tempDir.getCanonicalPath()) val physicalPartCol = deltaLog.snapshot.metadata.physicalPartitionColumns.head val addFile = AddFile( path = s"$physicalPartCol=__HIVE_DEFAULT_PARTITION__/file.parquet", partitionValues = Map(physicalPartCol -> null), size = 100, modificationTime = System.currentTimeMillis(), dataChange = true, stats = """{"numRecords": 1}""" ) val e = intercept[IllegalStateException] { commit(deltaLog, addFile, isCommitLarge) } assert(e.getMessage.contains("null partition value")) assert(e.getMessage.contains(s"NOT NULL column '$physicalPartCol'")) } } test(s"detect null partition value for NOT NULL column [isCommitLarge: $isCommitLarge]") { withTempDir { tempDir => val deltaLog = createPartitionedTableWithNotNullColumn(tempDir) // Create an AddFile with null partition value val addFile = AddFile( path = "part=__HIVE_DEFAULT_PARTITION__/file.parquet", partitionValues = Map("part" -> null), size = 100, modificationTime = System.currentTimeMillis(), dataChange = true, stats = """{"numRecords": 1}""" ) val e = intercept[IllegalStateException] { commit(deltaLog, addFile, isCommitLarge) } assert(e.getMessage.contains("null partition value")) assert(e.getMessage.contains("NOT NULL column 'part'")) } } test(s"no error when partition value is not null [isCommitLarge: $isCommitLarge]") { withTempDir { tempDir => val deltaLog = createPartitionedTableWithNotNullColumn(tempDir) // Create an AddFile with valid (non-null) partition value val addFile = AddFile( path = "part=valid_value/file.parquet", partitionValues = Map("part" -> "valid_value"), size = 100, modificationTime = System.currentTimeMillis(), dataChange = true, stats = """{"numRecords": 1}""" ) // Should not throw commit(deltaLog, addFile, isCommitLarge) } } test(s"null partition check - only log when throw is disabled " + s"[isCommitLarge: $isCommitLarge]") { withTempDir { tempDir => withSQLConf(DeltaSQLConf.DELTA_NULL_PARTITION_CHECK_THROW_ENABLED.key -> "false") { val deltaLog = createPartitionedTableWithNotNullColumn(tempDir) // Create an AddFile with null partition value val addFile = AddFile( path = "part=__HIVE_DEFAULT_PARTITION__/file.parquet", partitionValues = Map("part" -> null), size = 100, modificationTime = System.currentTimeMillis(), dataChange = true, stats = """{"numRecords": 1}""" ) // Should not throw when flag is disabled, only log val events = Log4jUsageLogger.track { commit(deltaLog, addFile, isCommitLarge) } // Validate that the null partition violation event was emitted val violationEvents = filterUsageRecords(events, "delta.constraints.nullPartitionViolation") assert(violationEvents.size == 1) val eventBlob = JsonUtils.fromJson[Map[String, Any]](violationEvents.head.blob) assert(eventBlob.contains("addFile")) assert(eventBlob.contains("notNullPartitionCols")) assert(eventBlob("notNullPartitionCols").toString == "part") assert(eventBlob.contains("stackTrace")) } } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/ConflictCheckerPredicateEliminationUnitSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.util.DeltaSparkPlanUtils import org.apache.spark.sql.{Column, QueryTest} import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.expressions.{Expression, Literal, Rand, ScalarSubquery} import org.apache.spark.sql.functions.{col, udf} import org.apache.spark.sql.test.SharedSparkSession /** * A set cheaper unit tests, that behave the same no matter if DVs, CDF, etc. are enabled * and do not need to be repeated in each conflict checker suite. */ class ConflictCheckerPredicateEliminationUnitSuite extends QueryTest with SharedSparkSession with ConflictCheckerPredicateElimination { val simpleExpressionA: Expression = $"a" === 1 val simpleExpressionB: Expression = $"b" === "test" val deterministicExpression: Expression = $"c" > 5L val nonDeterministicExpression: Expression = $"c" > rand(0) lazy val deterministicSubquery: Expression = { val df = spark.sql("SELECT 5") df.collect() $"c" > ScalarSubquery(df.queryExecution.analyzed) } lazy val nonDeterministicSubquery: Expression = { val df = spark.sql("SELECT rand()") df.collect() $"c" > ScalarSubquery(df.queryExecution.analyzed) } private def defaultEliminationFunction(e: Seq[Expression]): PredicateElimination = { val options = DeltaSparkPlanUtils.CheckDeterministicOptions(allowDeterministicUdf = false) eliminateNonDeterministicPredicates(e, options) } private def checkEliminationResult( predicate: Expression, expected: PredicateElimination, eliminationFunction: Seq[Expression] => PredicateElimination = defaultEliminationFunction) : Unit = { require(expected.newPredicates.size === 1) val actual = eliminationFunction(Seq(predicate)) assert(actual.newPredicates.size === 1) assert(actual.newPredicates.head.canonicalized == expected.newPredicates.head.canonicalized, s"actual=$actual\nexpected=$expected") assert(actual.eliminatedPredicates === expected.eliminatedPredicates) } for { deterministic <- BOOLEAN_DOMAIN subquery <- BOOLEAN_DOMAIN } { lazy val exprUnderTest = if (deterministic) { if (subquery) deterministicSubquery else deterministicExpression } else { if (subquery) nonDeterministicSubquery else nonDeterministicExpression } val testSuffix = s"deterministic $deterministic - subquery $subquery" def newPredicates(exprF: Expression => Expression): PredicateElimination = PredicateElimination( newPredicates = Seq(exprF(if (deterministic) exprUnderTest else Literal.TrueLiteral)), eliminatedPredicates = if (deterministic) Seq.empty else Seq("rand")) test(s"and expression - $testSuffix") { checkEliminationResult( predicate = simpleExpressionA && exprUnderTest, expected = newPredicates { eliminatedExprUnderTest => if (deterministic) { simpleExpressionA && eliminatedExprUnderTest } else { simpleExpressionA } } ) } test(s"or expression - $testSuffix") { checkEliminationResult( predicate = simpleExpressionA || exprUnderTest, expected = newPredicates { _ => if (deterministic) { simpleExpressionA || exprUnderTest } else { Literal.TrueLiteral } } ) } test(s"and or expression - $testSuffix") { checkEliminationResult( predicate = simpleExpressionA && (simpleExpressionB || exprUnderTest), expected = newPredicates { _ => if (deterministic) { simpleExpressionA && (simpleExpressionB || exprUnderTest) } else { simpleExpressionA } } ) } test(s"or and expression - $testSuffix") { checkEliminationResult( predicate = simpleExpressionA || (simpleExpressionB && exprUnderTest), expected = newPredicates { _ => if (deterministic) { simpleExpressionA || (simpleExpressionB && exprUnderTest) } else { simpleExpressionA || simpleExpressionB } } ) } test(s"or not and expression - $testSuffix") { checkEliminationResult( predicate = simpleExpressionA || !(simpleExpressionB && exprUnderTest), expected = newPredicates { _ => if (deterministic) { simpleExpressionA || !(simpleExpressionB && exprUnderTest) } else { Literal.TrueLiteral } } ) } test(s"and not or expression - $testSuffix") { checkEliminationResult( predicate = simpleExpressionA && !(simpleExpressionB || exprUnderTest), expected = newPredicates { _ => if (deterministic) { simpleExpressionA && !(simpleExpressionB || exprUnderTest) } else { simpleExpressionA } }) } } test("udf name is not exposed") { import testImplicits._ val random = udf(() => Math.random()) .asNondeterministic() .withName("sensitive_udf_name") checkEliminationResult( predicate = simpleExpressionA && (col("c") > random()).expr, expected = PredicateElimination( newPredicates = Seq(simpleExpressionA), eliminatedPredicates = Seq("scalaudf"))) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/ConflictResolutionTestUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.concurrent.{ExecutionException, ThreadPoolExecutor} import scala.collection.mutable.ArrayBuffer import scala.concurrent.Future import scala.concurrent.duration._ import org.apache.spark.sql.delta.concurrency.{PhaseLockingTestMixin, TransactionExecutionTestMixin} import org.apache.spark.sql.delta.fuzzer.{PhaseLockingTransactionExecutionObserver => TransactionObserver} import org.apache.spark.sql.delta.rowid.RowIdTestUtils import org.apache.spark.sql.util.ScalaExtensions.OptionExt import io.delta.tables.{DeltaTable => IODeltaTable} import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.functions.lit import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.{ThreadUtils, Utils} trait ConflictResolutionTestUtils extends QueryTest with SharedSparkSession with PhaseLockingTestMixin with TransactionExecutionTestMixin with DeletionVectorsTestUtils with RowIdTestUtils { import testImplicits._ final val ID_COLUMN = "idCol" final val PARTITION_COLUMN = "partitionCol" override val timeout: FiniteDuration = 120.seconds def abbreviate(str: String, abbrevMarker: String, len: Int): String = { if (str == null || abbrevMarker == null) { null } else if (str.length() <= len || str.length() <= abbrevMarker.length()) { str } else { str.substring(0, len - abbrevMarker.length()) + abbrevMarker } } abstract class TestTransaction(sqlConf: Map[String, String] = Map.empty) { val name: String val sqlConfStr: String = sqlConf.map { case (k, v) => s"$k=$v" }.mkString(",") def toSQL(tableName: String): String def execute(ctx: TestContext): Unit = { ctx.trackTransaction(this) { withSQLConf(sqlConf.toSeq: _*) { executeImpl(ctx) } } } def executeImpl(ctx: TestContext): Unit = { val sqlStr = toSQL(s"delta.`${ctx.deltaLog.dataPath.toUri.getPath}`") spark.sql(sqlStr).collect() } /** Whether this transaction is committing data change actions. */ def dataChange: Boolean /** Whether writing Deletion Vectors is enabled for this transaction. */ def deletionVectorsEnabled(deltaLog: DeltaLog): Boolean = false /** The executor thread to run this transaction. */ private lazy val executor: ThreadPoolExecutor = ThreadUtils.newDaemonSingleThreadExecutor(threadName = s"executor-$name") /** The transaction observer to step through the transaction phases. */ var observer: Option[TransactionObserver] = None /** The asynchronous future for the result of the transaction. */ private var future: Option[Future[Array[Row]]] = None /** Start transaction and unblock until precommit. */ def start(ctx: TestContext): Unit = { withSQLConf(sqlConf.toSeq: _*) { val (observer_, future_) = runFunctionWithObserver(name, executor, fn = () => { executeImpl(ctx) // DV tests do not use the results. We just return an empty array to conform with // function's signature. Array.empty[Row] }) unblockUntilPreCommit(observer_) busyWaitFor(observer_.phases.preparePhase.hasEntered, timeout) observer = Some(observer_) future = Some(future_) } } /** Commit the transaction. */ def commit(ctx: TestContext): Unit = { assert(observer.isDefined, "transaction not started") assert(future.isDefined, "transaction not started") val preCommitVersion = ctx.deltaLog.update().version withSQLConf(sqlConf.toSeq: _*) { ctx.trackTransaction(this) { unblockCommit(observer.get) waitForCommit(observer.get) ThreadUtils.awaitResult(future.get, Duration.Inf) } } // Ensure that the transaction actually commits something. val postCommitVersion = ctx.deltaLog.update().version assert(postCommitVersion > preCommitVersion, s"Transaction $this did not commit") } /** Run transaction and interleave fn() while transaction is stopped in precommit. */ def interleave[T](ctx: TestContext)(fn: => Unit): Unit = { start(ctx) fn commit(ctx) } } /** * Helper class containing the Delta log and committed transactions of a test. */ class TestContext(val deltaLog: DeltaLog) { /** The version of the Delta table after writing the initial data. */ val initialVersion: Long = deltaLog.update().version private val committedTransactions: ArrayBuffer[TestTransaction] = ArrayBuffer.empty /** Returns the transactions that successfully committed. */ def getCommittedTransactions: Seq[TestTransaction] = committedTransactions.toSeq /** Execute fn() and record the transaction if it successfully created a commit. */ def trackTransaction(transaction: TestTransaction)(fn: => Unit): Unit = { val preCommitVersion = deltaLog.update().version fn if (deltaLog.update().version > preCommitVersion) { committedTransactions.append(transaction) } } def deltaTable: IODeltaTable = IODeltaTable.forPath(deltaLog.dataPath.toString) } case class Insert( rows: Seq[Long], partitionColumn: Option[Long] = Some(0L), sqlConf: Map[String, String] = Map.empty) extends TestTransaction(sqlConf) { override val name: String = { val rowsStr = abbreviate(rows.mkString(","), "...", 10) s"INSERT($rowsStr)($sqlConfStr)" } override def toSQL(tableName: String): String = { throw new UnsupportedOperationException("toSQL for Insert is not implemented yet") } override def executeImpl(ctx: TestContext): Unit = { var df = rows.toDF(ID_COLUMN) partitionColumn.ifDefined { p => df = df.withColumn(PARTITION_COLUMN, lit(p)) } df.write.format("delta").mode("append").save(ctx.deltaLog.dataPath.toString) } override def dataChange: Boolean = true } case class Delete( rows: Seq[Long], sqlConf: Map[String, String] = Map.empty) extends TestTransaction(sqlConf) { override val name: String = { val rowsStr = abbreviate(rows.mkString(","), "...", 10) s"DELETE($rowsStr)($sqlConfStr)" } override def toSQL(tableName: String): String = { val inRowsStr = rows.mkString("(", ", ", ")") s"DELETE FROM $tableName WHERE $ID_COLUMN IN $inRowsStr" } override def dataChange: Boolean = true override def deletionVectorsEnabled(deltaLog: DeltaLog): Boolean = { var result = false withSQLConf(sqlConf.toSeq: _*) { result = deletionVectorsEnabledInDelete(spark, deltaLog) } result } } case class Update( rows: Seq[Long], setValue: Long = 42, sqlConf: Map[String, String] = Map.empty) extends TestTransaction(sqlConf) { override val name: String = { val rowsStr = abbreviate(rows.mkString(","), "...", 10) s"UPDATE($rowsStr)($sqlConfStr)" } override def toSQL(tableName: String): String = { val inRowsStr = rows.mkString("(", ", ", ")") // Dummy update. s"UPDATE $tableName SET $ID_COLUMN=$setValue WHERE $ID_COLUMN IN $inRowsStr" } override def dataChange: Boolean = true override def deletionVectorsEnabled(deltaLog: DeltaLog): Boolean = { var result = false withSQLConf(sqlConf.toSeq: _*) { result = deletionVectorsEnabledInUpdate(spark, deltaLog) } result } } // Delete-only MERGE. case class Merge( deleteRows: Seq[Long], sqlConf: Map[String, String] = Map.empty) extends TestTransaction(sqlConf) { override val name: String = { val rowsStr = abbreviate(deleteRows.mkString(","), "...", 10) s"MERGE($rowsStr)($sqlConfStr)" } override def toSQL(tableName: String): String = { val inRowsStr = deleteRows.mkString("(", ", ", ")") s""" |MERGE INTO $tableName t |USING $tableName s |ON t.$ID_COLUMN = s.$ID_COLUMN AND t.$ID_COLUMN IN $inRowsStr |WHEN MATCHED THEN DELETE |""".stripMargin } override def dataChange: Boolean = true override def deletionVectorsEnabled(deltaLog: DeltaLog): Boolean = { var result = false withSQLConf(sqlConf.toSeq: _*) { result = deletionVectorsEnabledInMerge(spark, deltaLog) } result } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/ConvertToDeltaSQLSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.execution.command.ExecutedCommandExec import org.apache.spark.sql.functions.{col, from_json} trait ConvertToDeltaSQLSuiteBase extends ConvertToDeltaSuiteBaseCommons with DeltaSQLCommandTest { override protected def convertToDelta( identifier: String, partitionSchema: Option[String] = None, collectStats: Boolean = true): Unit = { if (partitionSchema.isEmpty) { sql(s"convert to delta $identifier ${collectStatisticsStringOption(collectStats)}") } else { val stringSchema = partitionSchema.get sql(s"convert to delta $identifier ${collectStatisticsStringOption(collectStats)}" + s" partitioned by ($stringSchema)") } } // TODO: Move to ConvertToDeltaSuiteBaseCommons when DeltaTable API contains collectStats option test("convert with collectStats set to false") { withTempDir { dir => withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "true") { val tempDir = dir.getCanonicalPath writeFiles(tempDir, simpleDF) convertToDelta(s"parquet.`$tempDir`", collectStats = false) val deltaLog = DeltaLog.forTable(spark, tempDir) val history = io.delta.tables.DeltaTable.forPath(tempDir).history() checkAnswer( spark.read.format("delta").load(tempDir), simpleDF ) assert(history.count == 1) val statsDf = deltaLog.unsafeVolatileSnapshot.allFiles .select(from_json(col("stats"), deltaLog.unsafeVolatileSnapshot.statsSchema) .as("stats")).select("stats.*") assert(statsDf.filter(col("numRecords").isNotNull).count == 0) } } } for (numFiles <- Seq(1, 7)) { test(s"numConvertedFiles metric ($numFiles files)") { val testTableName = "test_table" withTable(testTableName) { spark.range(end = numFiles).toDF("part").withColumn("data", col("part")) .write.partitionBy("part").mode("overwrite").format("parquet").saveAsTable(testTableName) val plans = DeltaTestUtils.withPhysicalPlansCaptured(spark) { convertToDelta(testTableName, Some("part long")) } // Validate that the command node has the correct metrics. val commandNode = plans.collect { case exe: ExecutedCommandExec => exe.cmd }.head assert(commandNode.metrics("numConvertedFiles").value === numFiles) } } } } class ConvertToDeltaSQLSuite extends ConvertToDeltaSQLSuiteBase with ConvertToDeltaSuiteBase ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/ConvertToDeltaScalaSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.types.StructType class ConvertToDeltaScalaSuite extends ConvertToDeltaSuiteBase { override protected def convertToDelta( identifier: String, partitionSchema: Option[String] = None, collectStats: Boolean = true): Unit = { if (partitionSchema.isDefined) { io.delta.tables.DeltaTable.convertToDelta( spark, identifier, StructType.fromDDL(partitionSchema.get) ) } else { io.delta.tables.DeltaTable.convertToDelta( spark, identifier ) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/ConvertToDeltaSuiteBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.{File, FileNotFoundException} import org.apache.spark.sql.delta.files.TahoeLogFileIndex import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaExceptionTestUtils, DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.test.shims.StreamingTestShims.MemoryStream import org.apache.hadoop.fs.Path import org.apache.spark.SparkException import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.functions._ import org.apache.spark.sql.streaming.Trigger import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ import org.apache.spark.util.Utils /** * Common functions used across CONVERT TO DELTA test suites. We separate out these functions * so that we can re-use them in tests using Hive support. Tests that leverage Hive support cannot * extend the `SharedSparkSession`, therefore we keep this utility class as bare-bones as possible. */ trait ConvertToDeltaTestUtils extends QueryTest with DeltaExceptionTestUtils { self: DeltaSQLTestUtils => protected def collectStatisticsStringOption(collectStats: Boolean): String = Option(collectStats) .filterNot(identity).map(_ => "NO STATISTICS").getOrElse("") protected def simpleDF = spark.range(100) .withColumn("key1", col("id") % 2) .withColumn("key2", col("id") % 3 cast "String") protected def convertToDelta(identifier: String, partitionSchema: Option[String] = None, collectStats: Boolean = true): Unit protected val blockNonDeltaMsg = "A transaction log for Delta was found at" protected val parquetOnlyMsg = "CONVERT TO DELTA only supports parquet tables" protected val invalidParquetMsg = " not a Parquet file. Expected magic number at tail" // scalastyle:off deltahadoopconfiguration protected def sessionHadoopConf = spark.sessionState.newHadoopConf // scalastyle:on deltahadoopconfiguration protected def deltaRead(df: => DataFrame): Boolean = { val analyzed = df.queryExecution.analyzed analyzed.find { case DeltaTable(_: TahoeLogFileIndex) => true case _ => false }.isDefined } protected def writeFiles( dir: String, df: DataFrame, format: String = "parquet", partCols: Seq[String] = Nil, mode: String = "overwrite"): Unit = { if (partCols.nonEmpty) { df.write.partitionBy(partCols: _*).format(format).mode(mode).save(dir) } else { df.write.format(format).mode(mode).save(dir) } } } trait ConvertToDeltaSuiteBaseCommons extends ConvertToDeltaTestUtils with SharedSparkSession with DeltaSQLTestUtils with DeltaSQLCommandTest with DeltaTestUtilsForTempViews /** Tests for CONVERT TO DELTA that can be leveraged across SQL and Scala APIs. */ trait ConvertToDeltaSuiteBase extends ConvertToDeltaSuiteBaseCommons with ConvertToDeltaHiveTableTests { import org.apache.spark.sql.functions._ import testImplicits._ // Use different batch sizes to cover different merge schema code paths. protected def testSchemaMerging(testName: String)(block: => Unit): Unit = { Seq("1", "5").foreach { batchSize => test(s"$testName - batch size: $batchSize") { withSQLConf( DeltaSQLConf.DELTA_IMPORT_BATCH_SIZE_SCHEMA_INFERENCE.key -> batchSize) { block } } } } test("convert with collectStats true") { withTempDir { dir => val tempDir = dir.getCanonicalPath writeFiles(tempDir, simpleDF) convertToDelta(s"parquet.`$tempDir`", collectStats = true) val deltaLog = DeltaLog.forTable(spark, tempDir) val history = io.delta.tables.DeltaTable.forPath(tempDir).history() checkAnswer( spark.read.format("delta").load(tempDir), simpleDF ) assert(history.count == 1) val statsDf = deltaLog.unsafeVolatileSnapshot.allFiles .select(from_json($"stats", deltaLog.unsafeVolatileSnapshot.statsSchema) .as("stats")).select("stats.*") assert(statsDf.filter($"numRecords".isNull).count == 0) assert(statsDf.agg(sum("numRecords")).as[Long].head() == simpleDF.count) } } test("convert with collectStats true but config set to false -> Do not collect stats") { withTempDir { dir => withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false") { val tempDir = dir.getCanonicalPath writeFiles(tempDir, simpleDF) convertToDelta(s"parquet.`$tempDir`", collectStats = true) val deltaLog = DeltaLog.forTable(spark, tempDir) val history = io.delta.tables.DeltaTable.forPath(tempDir).history() checkAnswer( spark.read.format("delta").load(tempDir), simpleDF ) assert(history.count == 1) val statsDf = deltaLog.unsafeVolatileSnapshot.allFiles .select(from_json($"stats", deltaLog.unsafeVolatileSnapshot.statsSchema) .as("stats")).select("stats.*") assert(statsDf.filter($"numRecords".isNotNull).count == 0) } } } test("negative case: convert a non-delta path falsely claimed as parquet") { Seq("orc", "json", "csv").foreach { format => withTempDir { dir => val tempDir = dir.getCanonicalPath writeFiles(tempDir, simpleDF, format) // exception from executor reading parquet footer intercept[SparkException] { convertToDelta(s"parquet.`$tempDir`") } } } } test("negative case: convert non-parquet path to delta") { Seq("orc", "json", "csv").foreach { format => withTempDir { dir => val tempDir = dir.getCanonicalPath writeFiles(tempDir, simpleDF, format) val ae = intercept[AnalysisException] { convertToDelta(s"$format.`$tempDir`") } assert(ae.getMessage.contains(parquetOnlyMsg)) } } } test("negative case: convert non-parquet file to delta") { Seq("orc", "json", "csv").foreach { format => withTempDir { dir => val tempDir = dir.getCanonicalPath writeFiles(tempDir, simpleDF, format) val se = intercept[SparkException] { convertToDelta(s"parquet.`$tempDir`") } assert(se.getMessage.contains(invalidParquetMsg)) } } } test("filter non-parquet file for schema inference when not using catalog schema") { withTempDir { dir => val tempDir = dir.getCanonicalPath writeFiles(tempDir + "/part=1/", Seq(1).toDF("corrupted_id"), format = "orc") writeFiles(tempDir + "/part=2/", Seq(2).toDF("id")) val tableName = "pqtable" withTable(tableName) { // Create a catalog table on top of the parquet table with the wrong schema // The schema should be picked from the parquet data files sql(s"CREATE TABLE $tableName (key1 long, key2 string) " + s"USING PARQUET PARTITIONED BY (part string) LOCATION '$dir'") // Required for discovering partition of the table sql(s"MSCK REPAIR TABLE $tableName") withSQLConf( "spark.sql.files.ignoreCorruptFiles" -> "false", DeltaSQLConf.DELTA_CONVERT_USE_CATALOG_SCHEMA.key -> "false") { val se = intercept[SparkException] { convertToDelta(tableName) } assert(se.getMessage.contains(invalidParquetMsg)) } withSQLConf( "spark.sql.files.ignoreCorruptFiles" -> "true", DeltaSQLConf.DELTA_CONVERT_USE_CATALOG_SCHEMA.key -> "false") { convertToDelta(tableName) val tableId = TableIdentifier(tableName, Some("default")) val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, tableId) val expectedSchema = StructType( StructField("id", IntegerType, true) :: StructField("part", StringType, true) :: Nil) // Schema is inferred from the data assert(snapshot.schema.equals(expectedSchema)) } } } } test("filter non-parquet files during delta conversion") { withTempDir { dir => val tempDir = dir.getCanonicalPath writeFiles(tempDir + "/part=1/", Seq(1).toDF("id"), format = "json") writeFiles(tempDir + "/part=2/", Seq(2).toDF("id")) withSQLConf("spark.sql.files.ignoreCorruptFiles" -> "true") { convertToDelta(s"parquet.`$tempDir`", Some("part string")) checkAnswer(spark.read.format("delta").load(tempDir), Row(2, "2") :: Nil) } } } testQuietlyWithTempView("negative case: convert temp views to delta") { isSQLTempView => val tableName = "pqtbl" withTable(tableName) { // Create view simpleDF.write.format("parquet").saveAsTable(tableName) createTempViewFromTable(tableName, isSQLTempView, format = Some("parquet")) // Attempt to convert to delta val ae = intercept[AnalysisException] { convertToDelta("v") } assert(ae.getMessage.contains("Converting a view to a Delta table") || ae.getMessage.contains("Table default.v not found") || ae.getMessage.contains("Table or view 'v' not found in database 'default'") || ae.getMessage.contains("table or view `default`.`v` cannot be found") || ae.getMessage.contains("The table or view `spark_catalog`.`default`.`v` cannot be found")) } } test("negative case: missing data source name") { withTempDir { dir => val tempDir = dir.getCanonicalPath writeFiles(tempDir, simpleDF, "parquet", Seq("key1", "key2")) val ae = intercept[AnalysisException] { convertToDelta(s"`$tempDir`", None) } assert(ae.getMessage.contains(parquetOnlyMsg)) } } test("negative case: # partitions unmatched") { withTempDir { dir => val path = dir.getCanonicalPath writeFiles(path, simpleDF, partCols = Seq("key1", "key2")) val ae = intercept[AnalysisException] { convertToDelta(s"parquet.`$path`", Some("key1 long")) } assert(ae.getMessage.contains("Expecting 1 partition column(s)")) } } test("negative case: unmatched partition column names") { withTempDir { dir => val path = dir.getCanonicalPath writeFiles(path, simpleDF, partCols = Seq("key1", "key2")) val ae = intercept[AnalysisException] { convertToDelta(s"parquet.`$path`", Some("key1 long, key22 string")) } assert(ae.getMessage.contains("Expecting partition column ")) } } test("negative case: failed to cast partition value") { withTempDir { dir => val path = dir.getCanonicalPath val df = simpleDF.withColumn("partKey", lit("randomstring")) writeFiles(path, df, partCols = Seq("partKey")) val ae = intercept[RuntimeException] { convertToDelta(s"parquet.`$path`", Some("partKey int")) } assert(ae.getMessage.contains("Failed to cast partition value")) } } test("negative case: inconsistent directory structure") { withTempDir { dir => val tempDir = dir.getCanonicalPath writeFiles(tempDir, simpleDF) writeFiles(tempDir + "/key1=1/", simpleDF) var ae = intercept[AnalysisException] { convertToDelta(s"parquet.`$tempDir`") } assert(ae.getMessage.contains("Expecting 0 partition column")) ae = intercept[AnalysisException] { convertToDelta(s"parquet.`$tempDir`", Some("key1 string")) } assert(ae.getMessage.contains("Expecting 1 partition column")) } } test("negative case: empty and non-existent root dir") { withTempDir { dir => val tempDir = dir.getCanonicalPath val re = intercept[FileNotFoundException] { convertToDelta(s"parquet.`$tempDir`") } assert(re.getMessage.contains("No file found in the directory")) Utils.deleteRecursively(dir) val ae = intercept[FileNotFoundException] { convertToDelta(s"parquet.`$tempDir`") } assert(ae.getMessage.contains("doesn't exist")) } } testSchemaMerging("negative case: merge type conflict - string vs int") { withTempDir { dir => val tempDir = dir.getCanonicalPath writeFiles(tempDir + "/part=1/", Seq(1).toDF("id")) for (i <- 2 to 8 by 2) { writeFiles(tempDir + s"/part=$i/", Seq(1).toDF("id")) } for (i <- 3 to 9 by 2) { writeFiles(tempDir + s"/part=$i/", Seq("1").toDF("id")) } val ex = interceptWithUnwrapping[SparkException] { convertToDelta(s"parquet.`$tempDir`", Some("part string")) } assert(ex.getMessage.contains("Failed to merge")) assert(ex.getMessage.contains("/part="), "Error message should contain the file name") } } test("convert a streaming parquet path: use metadata") { val stream = MemoryStream[Int] val df = stream.toDS().toDF() withTempDir { outputDir => val checkpoint = new File(outputDir, "_check").toString val dataLocation = new File(outputDir, "data").toString val options = Map("checkpointLocation" -> checkpoint) // Add initial data to parquet file sink stream.addData(1, 2, 3) df.writeStream .options(options) .format("parquet") .trigger(Trigger.AvailableNow()) .start(dataLocation) .awaitTermination() // Add non-streaming data: this should be ignored in conversion. spark.range(10, 20).write.mode("append").parquet(dataLocation) sql(s"CONVERT TO DELTA parquet.`$dataLocation`") // Write data to delta stream.addData(4, 5, 6) df.writeStream .options(options) .format("delta") .trigger(Trigger.AvailableNow()) .start(dataLocation) .awaitTermination() // Should only read streaming data. checkAnswer( spark.read.format("delta").load(dataLocation), (1 to 6).map { Row(_) } ) } } test("convert a streaming parquet path: ignore metadata") { val stream = MemoryStream[Int] val df = stream.toDS().toDF("col1") withTempDir { outputDir => val checkpoint = new File(outputDir, "_check").toString val dataLocation = new File(outputDir, "data").toString val options = Map( "checkpointLocation" -> checkpoint ) // Add initial data to parquet file sink stream.addData(1 to 5) df.writeStream .options(options) .format("parquet") .trigger(Trigger.AvailableNow()) .start(dataLocation) .awaitTermination() // Add non-streaming data: this should not be ignored in conversion. spark.range(11, 21).select('id.cast("int") as "col1") .write.mode("append").parquet(dataLocation) withSQLConf(("spark.databricks.delta.convert.useMetadataLog", "false")) { sql(s"CONVERT TO DELTA parquet.`$dataLocation`") } // Write data to delta stream.addData(6 to 10) df.writeStream .options(options) .format("delta") .trigger(Trigger.AvailableNow()) .start(dataLocation) .awaitTermination() // Should read all data not just streaming data checkAnswer( spark.read.format("delta").load(dataLocation), (1 to 20).map { Row(_) } ) } } test("convert a parquet path") { withTempDir { dir => val tempDir = dir.getCanonicalPath writeFiles(tempDir, simpleDF, partCols = Seq("key1", "key2")) convertToDelta(s"parquet.`$tempDir`", Some("key1 long, key2 string")) // reads actually went through Delta assert(deltaRead(spark.read.format("delta").load(tempDir).select("id"))) // query through Delta is correct checkAnswer( spark.read.format("delta").load(tempDir).where("key1 = 0").select("id"), simpleDF.filter("id % 2 == 0").select("id")) // delta writers went through writeFiles( tempDir, simpleDF, format = "delta", partCols = Seq("key1", "key2"), mode = "append") checkAnswer( spark.read.format("delta").load(tempDir).where("key1 = 1").select("id"), simpleDF.union(simpleDF).filter("id % 2 == 1").select("id")) } } private def testSpecialCharactersInDirectoryNames(c: String, expectFailure: Boolean): Unit = { test(s"partition column names and values contain '$c'") { withTempDir { dir => val path = dir.getCanonicalPath val key1 = s"${c}key1${c}${c}" val key2 = s"${c}key2${c}${c}" val valueA = s"${c}some${c}${c}value${c}A" val valueB = s"${c}some${c}${c}value${c}B" val valueC = s"${c}some${c}${c}value${c}C" val valueD = s"${c}some${c}${c}value${c}D" val df1 = spark.range(3) .withColumn(key1, lit(valueA)) .withColumn(key2, lit(valueB)) val df2 = spark.range(4, 7) .withColumn(key1, lit(valueC)) .withColumn(key2, lit(valueD)) val df = df1.union(df2) writeFiles(path, df, format = "parquet", partCols = Seq(key1, key2)) if (expectFailure) { val e = intercept[AnalysisException] { convertToDelta(s"parquet.`$path`", Some(s"`$key1` string, `$key2` string")) } assert(e.getMessage.contains("invalid character")) } else { convertToDelta(s"parquet.`$path`", Some(s"`$key1` string, `$key2` string")) // missing one char from valueA, so no match checkAnswer( spark.read.format("delta").load(path).where(s"`$key1` = '${c}some${c}value${c}A'") .select("id"), Nil) checkAnswer( spark.read.format("delta").load(path) .where(s"`$key1` = '$valueA' and `$key2` = '$valueB'").select("id"), Row(0) :: Row(1) :: Row(2) :: Nil) checkAnswer( spark.read.format("delta").load(path).where(s"`$key2` = '$valueD' and id > 4") .select("id"), Row(5) :: Row(6) :: Nil) } } } } " ,;{}()\n\t=".foreach { char => testSpecialCharactersInDirectoryNames(char.toString, expectFailure = true) } testSpecialCharactersInDirectoryNames("%!@#$%^&*-", expectFailure = false) testSpecialCharactersInDirectoryNames("?.+<_>|/", expectFailure = false) test("can ignore empty sub-directories") { withTempDir { dir => val tempDir = dir.getCanonicalPath val fs = new Path(tempDir).getFileSystem(sessionHadoopConf) writeFiles(tempDir + "/key1=1/", Seq(1).toDF) assert(fs.mkdirs(new Path(tempDir + "/key1=2/"))) assert(fs.mkdirs(new Path(tempDir + "/random_dir/"))) convertToDelta(s"parquet.`$tempDir`", Some("key1 string")) checkAnswer(spark.read.format("delta").load(tempDir), Row(1, "1")) } } test("allow file names to have = character") { withTempDir { dir => val tempDir = dir.getCanonicalPath writeFiles(tempDir + "/part=1/", Seq(1).toDF("id")) val fs = new Path(tempDir).getFileSystem(sessionHadoopConf) // Rename the parquet file in partition "part=1" with something containing "=" val files = fs.listStatus(new Path(tempDir + "/part=1/")) .map(_.getPath) .filter(path => !path.getName.startsWith("_") && !path.getName.startsWith(".")) assert(files.length == 1) fs.rename( files.head, new Path(files.head.getParent.getName, "some-data-id=1.snappy.parquet")) convertToDelta(s"parquet.`$tempDir`", Some("part string")) checkAnswer(spark.read.format("delta").load(tempDir), Row(1, "1")) } } test("allow file names to not have .parquet suffix") { withTempDir { dir => val tempDir = dir.getCanonicalPath writeFiles(tempDir + "/part=1/", Seq(1).toDF("id")) writeFiles(tempDir + "/part=2/", Seq(2).toDF("id")) val fs = new Path(tempDir).getFileSystem(sessionHadoopConf) // Remove the suffix of the parquet file in partition "part=1" val files = fs.listStatus(new Path(tempDir + "/part=1/")) .map(_.getPath) .filter(path => !path.getName.startsWith("_") && !path.getName.startsWith(".")) assert(files.length == 1) fs.rename(files.head, new Path(files.head.getParent.toString, "unknown_suffix")) convertToDelta(s"parquet.`$tempDir`", Some("part string")) checkAnswer(spark.read.format("delta").load(tempDir), Row(1, "1") :: Row(2, "2") :: Nil) } } test("backticks") { withTempDir { dir => val tempDir = dir.getCanonicalPath writeFiles(tempDir, simpleDF) // wrap parquet with backticks should work convertToDelta(s"`parquet`.`$tempDir`", None) checkAnswer(spark.read.format("delta").load(tempDir), simpleDF) // path with no backticks should fail parsing intercept[ParseException] { convertToDelta(s"parquet.$tempDir") } } } test("overlapping partition and data columns") { withTempDir { dir => val tempDir = dir.getCanonicalPath val df = spark.range(1) .withColumn("partKey1", lit("1")) .withColumn("partKey2", lit("2")) df.write.parquet(tempDir + "/partKey1=1") convertToDelta(s"parquet.`$tempDir`", Some("partKey1 int")) // Same as in [[HadoopFsRelation]], for common columns, // respecting the order of data schema but the type of partition schema checkAnswer(spark.read.format("delta").load(tempDir), Row(0, 1, "2")) } } test("some partition value is null") { withTempDir { dir => val tempDir = dir.getCanonicalPath val df1 = Seq(0).toDF("id") .withColumn("key1", lit("A1")) .withColumn("key2", lit(null)) val df2 = Seq(1).toDF("id") .withColumn("key1", lit(null)) .withColumn("key2", lit(100)) writeFiles(tempDir, df1.union(df2), partCols = Seq("key1", "key2")) convertToDelta(s"parquet.`$tempDir`", Some("key1 string, key2 int")) checkAnswer( spark.read.format("delta").load(tempDir).where("key2 is null") .select("id"), Row(0)) checkAnswer( spark.read.format("delta").load(tempDir).where("key1 is null") .select("id"), Row(1)) checkAnswer( spark.read.format("delta").load(tempDir).where("key1 = 'A1'") .select("id"), Row(0)) checkAnswer( spark.read.format("delta").load(tempDir).where("key2 = 100") .select("id"), Row(1)) } } test("converting tables with dateType partition columns") { withTempDir { dir => val tempDir = dir.getCanonicalPath val df1 = Seq(0).toDF("id").withColumn("key1", lit("2019-11-22").cast("date")) val df2 = Seq(1).toDF("id").withColumn("key1", lit(null)) writeFiles(tempDir, df1.union(df2), partCols = Seq("key1")) convertToDelta(s"parquet.`$tempDir`", Some("key1 date")) checkAnswer( spark.read.format("delta").load(tempDir).where("key1 is null").select("id"), Row(1)) checkAnswer( spark.read.format("delta").load(tempDir).where("key1 = '2019-11-22'").select("id"), Row(0)) } } test("empty string partition value will be read back as null") { withTempDir { dir => val tempDir = dir.getCanonicalPath val df1 = Seq(0).toDF("id") .withColumn("key1", lit("A1")) .withColumn("key2", lit("")) val df2 = Seq(1).toDF("id") .withColumn("key1", lit("")) .withColumn("key2", lit("")) writeFiles(tempDir, df1.union(df2), partCols = Seq("key1", "key2")) convertToDelta(s"parquet.`$tempDir`", Some("key1 string, key2 string")) checkAnswer( spark.read.format("delta").load(tempDir).where("key1 is null and key2 is null") .select("id"), Row(1)) checkAnswer( spark.read.format("delta").load(tempDir).where("key1 = 'A1'") .select("id"), Row(0)) } } testSchemaMerging("can merge schema with different columns") { withTempDir { dir => val tempDir = dir.getCanonicalPath writeFiles(tempDir + "/part=1/", Seq(1).toDF("id1")) writeFiles(tempDir + "/part=2/", Seq(2).toDF("id2")) writeFiles(tempDir + "/part=3/", Seq(3).toDF("id3")) convertToDelta(s"parquet.`$tempDir`", Some("part string")) // spell out the columns as intra-batch and inter-batch merging logic may order // the columns differently val cols = Seq("id1", "id2", "id3", "part") checkAnswer( spark.read.format("delta").load(tempDir).where("id2 = 2") .select(cols.head, cols.tail: _*), Row(null, 2, null, "2") :: Nil) } } testSchemaMerging("can merge schema with different nullability") { withTempDir { dir => val tempDir = dir.getCanonicalPath writeFiles(tempDir + "/part=1/", Seq(1).toDF("id")) val schema = new StructType().add(StructField("id", IntegerType, false)) val df = spark.createDataFrame(spark.sparkContext.parallelize(Seq(Row(1))), schema) writeFiles(tempDir + "/part=2/", df) convertToDelta(s"parquet.`$tempDir`", Some("part string")) val fields = spark.read.format("delta").load(tempDir).schema.fields.toSeq assert(fields.map(_.name) === Seq("id", "part")) assert(fields.map(_.nullable) === Seq(true, true)) } } testSchemaMerging("can upcast in schema merging: short vs int") { withTempDir { dir => val tempDir = dir.getCanonicalPath writeFiles(tempDir + "/part=1/", Seq(1 << 20).toDF("id")) writeFiles(tempDir + "/part=2/", Seq(1).toDF("id").select(col("id") cast ShortType)) convertToDelta(s"parquet.`$tempDir`", Some("part string")) checkAnswer( spark.read.format("delta").load(tempDir), Row(1 << 20, "1") :: Row(1, "2") :: Nil) val expectedSchema = new StructType().add("id", IntegerType).add("part", StringType) val deltaLog = DeltaLog.forTable(spark, tempDir) assert(deltaLog.update().metadata.schema === expectedSchema) } } test("can fetch global configs") { withTempDir { dir => val path = dir.getCanonicalPath val deltaLog = DeltaLog.forTable(spark, path) withSQLConf("spark.databricks.delta.properties.defaults.appendOnly" -> "true") { writeFiles(path, simpleDF.coalesce(1)) convertToDelta(s"parquet.`$path`") } assert(deltaLog.snapshot.metadata.configuration("delta.appendOnly") === "true") } } test("convert to delta with string partition columns") { withTempDir { dir => val tempDir = dir.getCanonicalPath writeFiles(tempDir, simpleDF, partCols = Seq("key1", "key2")) convertToDelta(s"parquet.`$tempDir`", Some("key1 long, key2 string")) // reads actually went through Delta assert(deltaRead(spark.read.format("delta").load(tempDir).select("id"))) // query through Delta is correct checkAnswer( spark.read.format("delta").load(tempDir).where("key1 = 0").select("id"), simpleDF.filter("id % 2 == 0").select("id")) // delta writers went through writeFiles( tempDir, simpleDF, format = "delta", partCols = Seq("key1", "key2"), mode = "append") checkAnswer( spark.read.format("delta").load(tempDir).where("key1 = 1").select("id"), simpleDF.union(simpleDF).filter("id % 2 == 1").select("id")) } } test("convert a delta path falsely claimed as parquet") { withTempDir { dir => val tempDir = dir.getCanonicalPath writeFiles(tempDir, simpleDF, "delta") // Convert to delta convertToDelta(s"parquet.`$tempDir`") // Verify that table converted to delta checkAnswer( spark.read.format("delta").load(tempDir).where("key1 = 1").select("id"), simpleDF.filter("id % 2 == 1").select("id")) } } test("converting a delta path should not error for idempotency") { withTempDir { dir => val tempDir = dir.getCanonicalPath writeFiles(tempDir, simpleDF, "delta") convertToDelta(s"delta.`$tempDir`") checkAnswer( spark.read.format("delta").load(tempDir).where("key1 = 1").select("id"), simpleDF.filter("id % 2 == 1").select("id")) } } test("partition column name starting with underscore and dot") { withTempDir { dir => val df = spark.range(100) .withColumn("_key1", col("id") % 2) .withColumn(".key2", col("id") % 7 cast "String") val tempDir = dir.getCanonicalPath writeFiles(tempDir, df, partCols = Seq("_key1", ".key2")) convertToDelta(s"parquet.`$tempDir`", Some("_key1 long, `.key2` string")) checkAnswer(sql(s"SELECT * FROM delta.`$tempDir`"), df) } } } /** * Tests that involve tables defined in a Catalog such as Hive. We test in the sql as well as * hive package, where the hive package uses a proper HiveExternalCatalog to alter table definitions * in the HiveMetaStore. This test trait *should not* extend SharedSparkSession so that it can be * mixed in with the Hive test utilities. */ trait ConvertToDeltaHiveTableTests extends ConvertToDeltaTestUtils with DeltaSQLTestUtils { // Test conversion with and without the new CatalogFileManifest. protected def testCatalogFileManifest(testName: String)(block: (Boolean) => Unit): Unit = { Seq(true, false).foreach { useCatalogFileManifest => test(s"$testName - $useCatalogFileManifest") { withSQLConf( DeltaSQLConf.DELTA_CONVERT_USE_CATALOG_PARTITIONS.key -> useCatalogFileManifest.toString) { block(useCatalogFileManifest) } } } } protected def testCatalogSchema(testName: String)(testFn: (Boolean) => Unit): Unit = { Seq(true, false).foreach { useCatalogSchema => test(s"$testName - $useCatalogSchema") { withSQLConf( DeltaSQLConf.DELTA_CONVERT_USE_CATALOG_SCHEMA.key -> useCatalogSchema.toString) { testFn(useCatalogSchema) } } } } protected def getPathForTableName(tableName: String): String = { spark .sessionState .catalog .getTableMetadata(TableIdentifier(tableName, Some("default"))).location.getPath } protected def verifyExternalCatalogMetadata(tableName: String): Unit = { val catalog = spark.sessionState.catalog.externalCatalog.getTable("default", tableName) // Hive automatically adds some properties val cleanProps = catalog.properties.filterKeys(_ != "transient_lastDdlTime") assert(catalog.schema.isEmpty, s"Schema wasn't empty in the catalog for table $tableName: ${catalog.schema}") assert(catalog.partitionColumnNames.isEmpty, "Partition columns weren't empty in the " + s"catalog for table $tableName: ${catalog.partitionColumnNames}") assert(cleanProps.isEmpty, s"Table properties weren't empty for table $tableName: $cleanProps") } testQuietly("negative case: converting non-parquet table") { val tableName = "csvtable" withTable(tableName) { // Create a csv table simpleDF.write.partitionBy("key1", "key2").format("csv").saveAsTable(tableName) // Attempt to convert to delta val ae = intercept[AnalysisException] { convertToDelta(tableName, Some("key1 long, key2 string")) } // Get error message assert(ae.getMessage.contains(parquetOnlyMsg)) } } testQuietly("negative case: convert parquet path to delta when there is a database called " + "parquet but no table or path exists") { val dbName = "parquet" withDatabase(dbName) { withTempDir { dir => sql(s"CREATE DATABASE $dbName") val tempDir = dir.getCanonicalPath // Attempt to convert to delta val ae = intercept[FileNotFoundException] { convertToDelta(s"parquet.`$tempDir`") } // Get error message assert(ae.getMessage.contains("No file found in the directory")) } } } testQuietly("negative case: convert views to delta") { val viewName = "view" val tableName = "pqtbl" withTable(tableName) { // Create view simpleDF.write.format("parquet").saveAsTable(tableName) sql(s"CREATE VIEW $viewName as SELECT * from $tableName") // Attempt to convert to delta val ae = intercept[AnalysisException] { convertToDelta(viewName) } assert(ae.getMessage.contains("Converting a view to a Delta table")) } } testQuietly("negative case: converting a table that doesn't exist but the database does") { val dbName = "db" withDatabase(dbName) { sql(s"CREATE DATABASE $dbName") // Attempt to convert to delta val ae = intercept[AnalysisException] { convertToDelta(s"$dbName.faketable", Some("key1 long, key2 string")) } assert(ae.getMessage.contains("Table or view 'faketable' not found") || ae.getMessage.contains(s"table or view `$dbName`.`faketable` cannot be found")) } } testQuietly("negative case: unmatched partition schema") { val tableName = "pqtable" withTable(tableName) { // Create a partitioned parquet table simpleDF.write.partitionBy("key1", "key2").format("parquet").saveAsTable(tableName) // Check the partition schema in the catalog, key1's data type is original Long. assert(spark.sessionState.catalog.getTableMetadata( TableIdentifier(tableName, Some("default"))).partitionSchema .equals( (new StructType) .add(StructField("key1", LongType, true)) .add(StructField("key2", StringType, true)) )) // Convert to delta with partition schema mismatch on key1's data type, which is String. val ae = intercept[AnalysisException] { convertToDelta(tableName, Some("key1 string, key2 string")) } assert(ae.getMessage.contains("CONVERT TO DELTA was called with a partition schema " + "different from the partition schema inferred from the catalog")) } } testQuietly("convert two external tables pointing to same underlying files " + "with differing table properties should error if conf enabled otherwise merge properties") { val externalTblName = "extpqtbl" val secondExternalTbl = "othertbl" withTable(externalTblName, secondExternalTbl) { withTempDir { dir => val path = dir.getCanonicalPath // Create external table sql(s"CREATE TABLE $externalTblName " + s"USING PARQUET LOCATION '$path' TBLPROPERTIES ('abc'='def', 'def'='ghi') AS SELECT 1") // Create second external table with different table properties sql(s"CREATE TABLE $secondExternalTbl " + s"USING PARQUET LOCATION '$path' TBLPROPERTIES ('abc'='111', 'jkl'='mno')") // Convert first table to delta convertToDelta(externalTblName) // Verify that files converted to delta checkAnswer( sql(s"select * from delta.`$path`"), Row(1)) // Verify first table converted to delta assert(spark.sessionState.catalog.getTableMetadata( TableIdentifier(externalTblName, Some("default"))).provider.contains("delta")) // Attempt to convert second external table to delta val ae = intercept[AnalysisException] { convertToDelta(secondExternalTbl) } assert( ae.getMessage.contains("You are trying to convert a table which already has a delta") && ae.getMessage.contains("convert.metadataCheck.enabled")) // Disable convert metadata check withSQLConf(DeltaSQLConf.DELTA_CONVERT_METADATA_CHECK_ENABLED.key -> "false") { // Convert second external table to delta convertToDelta(secondExternalTbl) // Check delta table configuration has updated properties assert(DeltaLog.forTable(spark, path).startTransaction().metadata.configuration == Map("abc" -> "111", "def" -> "ghi", "jkl" -> "mno")) } } } } testQuietly("convert two external tables pointing to the same underlying files") { val externalTblName = "extpqtbl" val secondExternalTbl = "othertbl" withTable(externalTblName, secondExternalTbl) { withTempDir { dir => val path = dir.getCanonicalPath writeFiles(path, simpleDF, "delta") val deltaLog = DeltaLog.forTable(spark, path) // Create external table sql(s"CREATE TABLE $externalTblName (key1 long, key2 string) " + s"USING PARQUET LOCATION '$path'") // Create second external table sql(s"CREATE TABLE $secondExternalTbl (key1 long, key2 string) " + s"USING PARQUET LOCATION '$path'") assert(deltaLog.update().version == 0) // Convert first table to delta convertToDelta(externalTblName) // Convert should not update version since delta log metadata is not changing assert(deltaLog.update().version == 0) // Check that the metadata in the catalog was emptied and pushed to the delta log verifyExternalCatalogMetadata(externalTblName) // Convert second external table to delta convertToDelta(secondExternalTbl) verifyExternalCatalogMetadata(secondExternalTbl) // Verify that underlying files converted to delta checkAnswer( sql(s"select id from delta.`$path` where key1 = 1"), simpleDF.filter("id % 2 == 1").select("id")) // Verify catalog table provider is 'delta' for both tables assert(spark.sessionState.catalog.getTableMetadata( TableIdentifier(externalTblName, Some("default"))).provider.contains("delta")) assert(spark.sessionState.catalog.getTableMetadata( TableIdentifier(secondExternalTbl, Some("default"))).provider.contains("delta")) } } } testQuietly("convert an external parquet table") { val tableName = "pqtbl" val externalTblName = "extpqtbl" withTable(tableName) { simpleDF.write.format("parquet").saveAsTable(tableName) // Get where the table is stored and try to access it using parquet rather than delta val path = getPathForTableName(tableName) // Create external table sql(s"CREATE TABLE $externalTblName (key1 long, key2 string) " + s"USING PARQUET LOCATION '$path'") // Convert to delta sql(s"convert to delta $externalTblName") assert(spark.sessionState.catalog.getTableMetadata( TableIdentifier(externalTblName, Some("default"))).provider.contains("delta")) // Verify that table converted to delta checkAnswer( sql(s"select key2 from delta.`$path` where key1 = 1"), simpleDF.filter("id % 2 == 1").select("key2")) checkAnswer( sql(s"select key2 from $externalTblName where key1 = 1"), simpleDF.filter("id % 2 == 1").select("key2")) } } testCatalogSchema("convert a parquet table with catalog schema") { useCatalogSchema => { withTempDir { dir => // Create a parquet table with all 3 columns: id, key1 and key2 val tempDir = dir.getCanonicalPath writeFiles(tempDir, simpleDF) val tableName = "pqtable" withTable(tableName) { // Create a catalog table on top of the parquet table excluding column id sql(s"CREATE TABLE $tableName (key1 long, key2 string) " + s"USING PARQUET LOCATION '$dir'") convertToDelta(tableName) val tableId = TableIdentifier(tableName, Some("default")) val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, tableId) val catalog_columns = Seq[StructField]( StructField("key1", LongType, true), StructField("key2", StringType, true) ) if (useCatalogSchema) { // Catalog schema is used, column id is excluded. assert(snapshot.metadata.schema.equals(StructType(catalog_columns))) } else { // Schema is inferred from the data, all 3 columns are included. assert(snapshot.metadata.schema .equals(StructType(StructField("id", LongType, true) +: catalog_columns))) } } } } } testQuietly("converting a delta table should not error for idempotency") { val tableName = "deltatbl" val format = "delta" withTable(tableName) { simpleDF.write.partitionBy("key1", "key2").format(format).saveAsTable(tableName) convertToDelta(tableName) // reads actually went through Delta val path = getPathForTableName(tableName) checkAnswer( sql(s"select id from $format.`$path` where key1 = 1"), simpleDF.filter("id % 2 == 1").select("id")) } } testQuietly("convert to delta using table name without database name") { val tableName = "pqtable" withTable(tableName) { // Create a parquet table simpleDF.write.partitionBy("key1", "key2").format("parquet").saveAsTable(tableName) // Convert to delta using only table name convertToDelta(tableName, Some("key1 long, key2 string")) // reads actually went through Delta val path = getPathForTableName(tableName) checkAnswer( sql(s"select id from delta.`$path` where key1 = 1"), simpleDF.filter("id % 2 == 1").select("id")) } } testQuietly("convert a parquet table to delta with database name as parquet") { val dbName = "parquet" val tableName = "pqtbl" withDatabase(dbName) { withTable(dbName + "." + tableName) { sql(s"CREATE DATABASE $dbName") val table = TableIdentifier(tableName, Some(dbName)) simpleDF.write.partitionBy("key1", "key2") .format("parquet").saveAsTable(dbName + "." + tableName) convertToDelta(dbName + "." + tableName, Some("key1 long, key2 string")) // reads actually went through Delta val path = spark .sessionState .catalog .getTableMetadata(table).location.getPath checkAnswer( sql(s"select id from delta.`$path` where key1 = 1"), simpleDF.filter("id % 2 == 1").select("id")) } } } testQuietly("convert a parquet path to delta while database called parquet exists") { val dbName = "parquet" withDatabase(dbName) { withTempDir { dir => // Create a database called parquet sql(s"CREATE DATABASE $dbName") // Create a parquet table at given path val tempDir = dir.getCanonicalPath writeFiles(tempDir, simpleDF, partCols = Seq("key1", "key2")) // Convert should convert the path instead of trying to find a table in that database convertToDelta(s"parquet.`$tempDir`", Some("key1 long, key2 string")) // reads actually went through Delta checkAnswer( sql(s"select id from delta.`$tempDir` where key1 = 1"), simpleDF.filter("id % 2 == 1").select("id")) } } } testQuietly("convert a delta table where metadata does not reflect that the table is " + "already converted should update the metadata") { val tableName = "deltatbl" withTable(tableName) { simpleDF.write.partitionBy("key1", "key2").format("parquet").saveAsTable(tableName) // Get where the table is stored and try to access it using parquet rather than delta val path = getPathForTableName(tableName) // Convert using path so that metadata is not updated convertToDelta(s"parquet.`$path`", Some("key1 long, key2 string")) // Call convert again convertToDelta(s"default.$tableName", Some("key1 long, key2 string")) // Metadata should be updated so we can use table name checkAnswer( sql(s"select id from default.$tableName where key1 = 1"), simpleDF.filter("id % 2 == 1").select("id")) } } testQuietly("convert a parquet table using table name") { val tableName = "pqtable2" withTable(tableName) { // Create a parquet table simpleDF.write.partitionBy("key1", "key2").format("parquet").saveAsTable(tableName) // Convert to delta convertToDelta(s"default.$tableName", Some("key1 long, key2 string")) // Get where the table is stored and try to access it using parquet rather than delta val path = getPathForTableName(tableName) // reads actually went through Delta assert(deltaRead(sql(s"select id from default.$tableName"))) // query through Delta is correct checkAnswer( sql(s"select id from default.$tableName where key1 = 0"), simpleDF.filter("id % 2 == 0").select("id")) // delta writers went through writeFiles(path, simpleDF, format = "delta", partCols = Seq("key1", "key2"), mode = "append") checkAnswer( sql(s"select id from default.$tableName where key1 = 1"), simpleDF.union(simpleDF).filter("id % 2 == 1").select("id")) } } testQuietly("Convert a partitioned parquet table with partition schema autofill") { val tableName = "ppqtable" withTable(tableName) { // Create a partitioned parquet table simpleDF.write.partitionBy("key1", "key2").format("parquet").saveAsTable(tableName) // Convert to delta without partition schema, partition schema is autofill from catalog convertToDelta(tableName) // Verify that table is converted to delta assert(spark.sessionState.catalog.getTableMetadata( TableIdentifier(tableName, Some("default"))).provider.contains("delta")) // Check the partition schema in the transaction log val tableId = TableIdentifier(tableName, Some("default")) assert(DeltaLog.forTableWithSnapshot(spark, tableId)._2.metadata.partitionSchema.equals( (new StructType()) .add(StructField("key1", LongType, true)) .add(StructField("key2", StringType, true)) )) // Check data in the converted delta table. checkAnswer( sql(s"SELECT id from default.$tableName where key2 = '2'"), simpleDF.filter("id % 3 == 2").select("id")) } } testCatalogFileManifest("convert partitioned parquet table with catalog partitions") { useCatalogFileManifest => { val tableName = "ppqtable" withTable(tableName) { simpleDF.write.partitionBy("key1").format("parquet").saveAsTable(tableName) val path = getPathForTableName(tableName) // Create an orphan partition val df = spark.range(100, 200) .withColumn("key1", lit(2)) .withColumn("key2", col("id") % 4 cast "String") df.write.partitionBy("key1") .format("parquet") .mode("Append") .save(path) // The path should contains 3 partitions. val partitionDirs = new File(path).listFiles().filter(_.isDirectory) assert(partitionDirs.map(_.getName).sorted .sameElements(Array("key1=0", "key1=1", "key1=2"))) // Catalog only contains 2 partitions. assert(spark.sessionState.catalog .listPartitions(TableIdentifier(tableName, Some("default"))).size == 2) // Convert table to delta convertToDelta(tableName) // Verify that table is converted to delta assert(spark.sessionState.catalog.getTableMetadata( TableIdentifier(tableName, Some("default"))).provider.contains("delta")) // Check data in the converted delta table. if (useCatalogFileManifest) { // Partition "key1=2" is pruned. checkAnswer(sql(s"SELECT DISTINCT key1 from default.${tableName}"), spark.range(2).toDF()) } else { // All partitions are preserved. checkAnswer(sql(s"SELECT DISTINCT key1 from default.${tableName}"), spark.range(3).toDF()) } } } } test("external tables use correct path scheme") { withTempDir { dir => withTable("externalTable") { withSQLConf(("fs.s3.impl", classOf[S3LikeLocalFileSystem].getCanonicalName)) { sql(s"CREATE TABLE externalTable USING parquet LOCATION 's3://$dir' AS SELECT 1") // Ideally we would test a successful conversion with a remote filesystem, but there's // no good way to set one up in unit tests. So instead we delete the data, and let the // FileNotFoundException tell us which scheme it was using to look for it. Utils.deleteRecursively(dir) val ex = intercept[FileNotFoundException] { convertToDelta("default.externalTable", None) } // If the path incorrectly used the default scheme, this would be file: at the end. assert(ex.getMessage.contains(s"s3:$dir doesn't exist")) } } } } test("can convert a partition-like table path") { withTempDir { dir => val path = dir.getCanonicalPath writeFiles(path, simpleDF, partCols = Seq("key1", "key2")) val basePath = s"$path/key1=1/" convertToDelta(s"parquet.`$basePath`", Some("key2 string")) checkAnswer( sql(s"select id from delta.`$basePath` where key2 = '1'"), simpleDF.filter("id % 2 == 1").filter("id % 3 == 1").select("id")) } } test("can convert table with partition overwrite") { val tableName = "ppqtable" withTable(tableName) { // Create table with original partitions of "key1=0" and "key1=1". val df = spark.range(0, 100) .withColumn("key1", col("id") % 2) .withColumn("key2", col("id") % 3 cast "String") df.write.format("parquet").partitionBy("key1").mode("append").saveAsTable(tableName) checkAnswer(sql(s"SELECT id FROM $tableName"), df.select("id")) val dataDir = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName)).location.toString // Create orphan partition "key1=0;key2=3" with additional column. val df1 = spark.range(100, 120, 2) .withColumn("key1", col("id") % 2) .withColumn("key2", lit("3")) df1.write.format("parquet").partitionBy("key1", "key2").mode("append").save(dataDir) // Point table partition "key1=0" to the path of orphan partition "key1=0;key2=3" sql(s"ALTER TABLE $tableName PARTITION (key1=0) SET LOCATION '$dataDir/key1=0/key2=3/'") checkAnswer(sql(s"SELECT id FROM $tableName WHERE key1 = 0"), df1.select("id")) // ConvertToDelta should work without inferring the partition values from partition path. convertToDelta(tableName) // Verify that table is converted to delta assert(spark.sessionState.catalog.getTableMetadata( TableIdentifier(tableName, Some("default"))).provider.contains("delta")) // Check data in the converted delta table. checkAnswer(sql(s"SELECT id FROM $tableName WHERE key1 = 0"), df1.select("id")) } } test(s"catalog partition values contain special characters") { // Add interesting special characters here for test val specialChars = " ,;{}()\n\t=!@#$%^&*-?.+<_>|/" val tableName = "ppqtable" withTable(tableName) { val valueA = s"${specialChars}some${specialChars}${specialChars}value${specialChars}A" val valueB = s"${specialChars}some${specialChars}${specialChars}value${specialChars}B" val valueC = s"${specialChars}some${specialChars}${specialChars}value${specialChars}C" val valueD = s"${specialChars}some${specialChars}${specialChars}value${specialChars}D" val df1 = spark.range(3).withColumn("key1", lit(valueA)).withColumn("key2", lit(valueB)) val df2 = spark.range(4, 7).withColumn("key1", lit(valueC)).withColumn("key2", lit(valueD)) df1.union(df2).write.format("parquet").partitionBy("key1", "key2").saveAsTable(tableName) convertToDelta(tableName, Some("key1 string, key2 string")) // missing one char from valueA, so no match checkAnswer( spark.table(tableName) .where(s"key1 = '${specialChars}some${specialChars}value${specialChars}A'") .select("id"), Nil) checkAnswer( spark.table(tableName).where(s"key1 = '$valueA' and key2 = '$valueB'") .select("id"), Row(0) :: Row(1) :: Row(2) :: Nil) checkAnswer( spark.table(tableName).where(s"key2 = '$valueD' and id > 4") .select("id"), Row(5) :: Row(6) :: Nil) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/CustomCatalogSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands._ import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DummyCatalog, DummySessionCatalog, DummySessionCatalogInner} import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.analysis.ResolvedTable import org.apache.spark.sql.catalyst.plans.logical.{AppendData, SetTableProperties, UnaryNode, UnsetTableProperties} import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog} import org.apache.spark.sql.execution.datasources.v2.DataSourceV2RelationShim import org.apache.spark.sql.test.SharedSparkSession class CustomCatalogSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with DescribeDeltaDetailSuiteBase { override def sparkConf: SparkConf = super.sparkConf.set("spark.sql.catalog.dummy", classOf[DummyCatalog].getName) test("CatalogTable exists in DeltaTableV2 if use table identifier") { def catalogTableExists(sqlCmd: String): Unit = { val plan = spark.sql(sqlCmd).queryExecution.analyzed val catalogTable = plan match { case cmd: UnaryNode with DeltaCommand => cmd.getDeltaTable(cmd.child, "dummy").catalogTable case AppendData(DataSourceV2RelationShim(table: DeltaTableV2, _, _, _, _), _, _, _, _, _) => table.catalogTable case cmd: DeleteCommand => cmd.catalogTable case cmd: DescribeDeltaHistoryCommand => cmd.table.catalogTable case cmd: MergeIntoCommand => cmd.catalogTable case cmd: RestoreTableCommand => cmd.sourceTable.catalogTable case SetTableProperties(ResolvedTable(_, _, table: DeltaTableV2, _), _) => table.catalogTable case UnsetTableProperties(ResolvedTable(_, _, table: DeltaTableV2, _), _, _) => table.catalogTable case cmd: UpdateCommand => cmd.catalogTable case cmd: WriteIntoDelta => cmd.catalogTableOpt } assert(catalogTable.nonEmpty) } val mergeSrcTable = "merge_src_table" val tableName = "delta_commands_table" withTable(tableName, mergeSrcTable) { sql(f"CREATE TABLE $tableName (c1 int, c2 int) USING delta PARTITIONED BY (c1)") // DQL catalogTableExists(s"DESCRIBE DETAIL $tableName") catalogTableExists(s"DESCRIBE HISTORY $tableName") // DDL catalogTableExists(s"ALTER TABLE $tableName SET TBLPROPERTIES ('a' = 'b') ") catalogTableExists(s"ALTER TABLE $tableName UNSET TBLPROPERTIES ('a') ") // DML insert catalogTableExists(s"INSERT INTO $tableName VALUES (1, 1) ") // DML merge sql(s"CREATE TABLE $mergeSrcTable (c1 int, c2 int) USING delta PARTITIONED BY (c1)") sql(s"INSERT INTO $mergeSrcTable VALUES (1, 1) ") catalogTableExists(s"MERGE INTO $tableName USING $mergeSrcTable " + s"ON ${mergeSrcTable}.c1 = ${tableName}.c1 WHEN MATCHED THEN DELETE") // DML update catalogTableExists(s"UPDATE $tableName SET c1 = 4 WHERE true ") // DML delete catalogTableExists(s"DELETE FROM $tableName WHERE true ") // optimize sql(s"INSERT INTO $tableName VALUES (1, 1) ") sql(s"INSERT INTO $tableName VALUES (1, 1) ") catalogTableExists(s"OPTIMIZE $tableName") // vacuum catalogTableExists(s"VACUUM $tableName") } } test("DESC DETAIL a delta table from DummyCatalog") { val tableName = "desc_detail_table" withTable(tableName) { val dummyCatalog = spark.sessionState.catalogManager.catalog("dummy").asInstanceOf[DummyCatalog] val tablePath = dummyCatalog.getTablePath(tableName) sql("SET CATALOG dummy") sql(f"CREATE TABLE $tableName (id bigint) USING delta") sql("SET CATALOG spark_catalog") // Insert some data into the table in the dummy catalog. // To make it simple, here we insert data directly into the table path. sql(f"INSERT INTO delta.`$tablePath` VALUES (0)") sql("SET CATALOG dummy") // Test simple desc detail command under the dummy catalog checkResult( sql(f"DESC DETAIL $tableName"), Seq("delta", 1), Seq("format", "numFiles")) // Test 3-part identifier checkResult( sql(f"DESC DETAIL dummy.default.$tableName"), Seq("delta", 1), Seq("format", "numFiles")) // Test table path checkResult( sql(f"DESC DETAIL delta.`$tablePath`"), Seq("delta", 1), Seq("format", "numFiles")) // Test 3-part identifier when the current catalog is not dummy catalog sql("SET CATALOG spark_catalog") checkResult( sql(f"DESC DETAIL dummy.default.$tableName"), Seq("delta", 1), Seq("format", "numFiles")) } } test("RESTORE a table from DummyCatalog") { val dummyCatalog = spark.sessionState.catalogManager.catalog("dummy").asInstanceOf[DummyCatalog] val tableName = "restore_table" val tablePath = dummyCatalog.getTablePath(tableName) withTable(tableName) { sql("SET CATALOG dummy") sql(f"CREATE TABLE $tableName (id bigint) USING delta") sql("SET CATALOG spark_catalog") // Insert some data into the table in the dummy catalog. // To make it simple, here we insert data directly into the table path. sql(f"INSERT INTO delta.`$tablePath` VALUES (0)") sql(f"INSERT INTO delta.`$tablePath` VALUES (1)") // Test 3-part identifier when the current catalog is the default catalog sql(f"RESTORE TABLE dummy.default.$tableName VERSION AS OF 1") checkAnswer(spark.table(f"dummy.default.$tableName"), spark.range(1).toDF()) sql("SET CATALOG dummy") sql(f"RESTORE TABLE $tableName VERSION AS OF 0") checkAnswer(spark.table(tableName), Nil) sql(f"RESTORE TABLE $tableName VERSION AS OF 1") checkAnswer(spark.table(tableName), spark.range(1).toDF()) // Test 3-part identifier sql(f"RESTORE TABLE dummy.default.$tableName VERSION AS OF 2") checkAnswer(spark.table(tableName), spark.range(2).toDF()) // Test file path table sql(f"RESTORE TABLE delta.`$tablePath` VERSION AS OF 1") checkAnswer(spark.table(tableName), spark.range(1).toDF()) } } test("Shallow Clone a table with time travel") { val srcTable = "shallow_clone_src_table" val destTable1 = "shallow_clone_dest_table_1" val destTable2 = "shallow_clone_dest_table_2" val destTable3 = "shallow_clone_dest_table_3" val destTable4 = "shallow_clone_dest_table_4" val dummyCatalog = spark.sessionState.catalogManager.catalog("dummy").asInstanceOf[DummyCatalog] val tablePath = dummyCatalog.getTablePath(srcTable) withTable(srcTable) { sql("SET CATALOG dummy") sql(f"CREATE TABLE $srcTable (id bigint) USING delta") sql("SET CATALOG spark_catalog") // Insert some data into the table in the dummy catalog. // To make it simple, here we insert data directly into the table path. sql(f"INSERT INTO delta.`$tablePath` VALUES (0)") sql(f"INSERT INTO delta.`$tablePath` VALUES (1)") withTable(destTable1) { // Test 3-part identifier when the current catalog is the default catalog sql(f"CREATE TABLE $destTable1 SHALLOW CLONE dummy.default.$srcTable VERSION AS OF 1") checkAnswer(spark.table(destTable1), spark.range(1).toDF()) } sql("SET CATALOG dummy") Seq(true, false).foreach { createTableInDummy => val (dest2, dest3, dest4) = if (createTableInDummy) { (destTable2, destTable3, destTable4) } else { val prefix = "spark_catalog.default" (s"$prefix.$destTable2", s"$prefix.$destTable3", s"$prefix.$destTable4") } withTable(dest2, dest3, dest4) { // Test simple shallow clone command under the dummy catalog sql(f"CREATE TABLE $dest2 SHALLOW CLONE $srcTable") checkAnswer(spark.table(dest2), spark.range(2).toDF()) // Test time travel on the src table sql(f"CREATE TABLE $dest3 SHALLOW CLONE dummy.default.$srcTable VERSION AS OF 1") checkAnswer(spark.table(dest3), spark.range(1).toDF()) // Test time travel on the src table delta path sql(f"CREATE TABLE $dest4 SHALLOW CLONE delta.`$tablePath` VERSION AS OF 1") checkAnswer(spark.table(dest4), spark.range(1).toDF()) } } } } test("DESCRIBE HISTORY a delta table from DummyCatalog") { val tableName = "desc_history_table" withTable(tableName) { sql("SET CATALOG dummy") val dummyCatalog = spark.sessionState.catalogManager.catalog("dummy").asInstanceOf[DummyCatalog] val tablePath = dummyCatalog.getTablePath(tableName) sql(f"CREATE TABLE $tableName (column1 bigint) USING delta") sql("SET CATALOG spark_catalog") // Insert some data into the table in the dummy catalog. sql(f"INSERT INTO delta.`$tablePath` VALUES (0)") sql("SET CATALOG dummy") // Test simple desc detail command under the dummy catalog var result = sql(s"DESCRIBE HISTORY $tableName").collect() assert(result.length == 2) assert(result(0).getAs[Long]("version") == 1) // Test 3-part identifier result = sql(f"DESCRIBE HISTORY dummy.default.$tableName").collect() assert(result.length == 2) assert(result(0).getAs[Long]("version") == 1) // Test table path sql(f"DESC DETAIL delta.`$tablePath`").collect() assert(result.length == 2) assert(result(0).getAs[Long]("version") == 1) // Test 3-part identifier when the current catalog is not dummy catalog sql("SET CATALOG spark_catalog") result = sql(s"DESCRIBE HISTORY dummy.default.$tableName").collect() assert(result.length == 2) assert(result(0).getAs[Long]("version") == 1) } } test("SELECT Table Changes from DummyCatalog") { val dummyTableName = "dummy_table" val sparkTableName = "spark_catalog.default.spark_table" withTable(dummyTableName, sparkTableName) { sql("SET CATALOG spark_catalog") sql(f"CREATE TABLE $sparkTableName (id bigint, s string) USING delta" + f" TBLPROPERTIES(delta.enableChangeDataFeed=true)") sql(f"INSERT INTO $sparkTableName VALUES (0, 'a')") sql(f"INSERT INTO $sparkTableName VALUES (1, 'b')") sql("SET CATALOG dummy") // Since the dummy catalog doesn't pass through the TBLPROPERTIES 'delta.enableChangeDataFeed' // here we clone a table with the same schema as the spark table to test the table changes. sql(f"CREATE TABLE $dummyTableName SHALLOW CLONE $sparkTableName") // table_changes() should be able to read the table changes from the dummy catalog Seq(dummyTableName, f"dummy.default.$dummyTableName").foreach { name => val rows = sql(f"SELECT * from table_changes('$name', 1)").collect() assert(rows.length == 2) } } } test("custom catalog that adds additional table storage properties") { // Reset catalog manager so that the new `spark_catalog` implementation can apply. spark.sessionState.catalogManager.reset() withSQLConf("spark.sql.catalog.spark_catalog" -> classOf[DummySessionCatalog].getName) { withTable("t") { withTempPath { path => spark.range(10).write.format("delta").save(path.getCanonicalPath) sql(s"CREATE TABLE t (id LONG) USING delta LOCATION '${path.getCanonicalPath}'") val t = spark.sessionState.catalogManager.v2SessionCatalog.asInstanceOf[TableCatalog] .loadTable(Identifier.of(Array("default"), "t")).asInstanceOf[DeltaTableV2] assert(t.deltaLog.options("fs.myKey") == "val") } } } } test("custom catalog that generates location for managed tables") { // Reset catalog manager so that the new `spark_catalog` implementation can apply. spark.sessionState.catalogManager.reset() withSQLConf("spark.sql.catalog.spark_catalog" -> classOf[DummySessionCatalog].getName) { withTable("t") { withTempPath { path => sql(s"CREATE TABLE t (id LONG) USING delta TBLPROPERTIES (fakeLoc='$path')") val t = spark.sessionState.catalogManager.v2SessionCatalog.asInstanceOf[TableCatalog] .loadTable(Identifier.of(Array("default"), "t")) // It should be a managed table. assert(!t.properties().containsKey(TableCatalog.PROP_EXTERNAL)) } } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DDLTestUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.{QueryTest, SparkSession} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{DataType, LongType, StructField} /** * Base trait for specifying column definitions in tests in an API agnostic way. * * Note: we don't use StructField because StructField is defined in Spark. It's easier to * write tests with flexible helpers in our own project. */ trait ColumnSpec { /** Name of the column. */ def colName: String /** Spark logical type for the column. */ def dataType: DataType /** Returns a String which can be used to define the column in SQL. */ def ddl: String /** Return the specification as a StructField */ def structField(spark: SparkSession): StructField } case class GeneratedColumnSpec( colName: String, dataType: DataType, generatedExpression: String) extends ColumnSpec { override def ddl: String = s"$colName ${dataType.sql} GENERATED ALWAYS AS ($generatedExpression)" override def structField(spark: SparkSession): StructField = { io.delta.tables.DeltaTable.columnBuilder(spark, colName) .dataType(dataType) .generatedAlwaysAs(generatedExpression) .build() } } case class TestColumnSpec( colName: String, dataType: DataType) extends ColumnSpec { override def ddl: String = { s"$colName ${dataType.sql}" } override def structField(spark: SparkSession): StructField = { io.delta.tables.DeltaTable.columnBuilder(spark, colName) .dataType(dataType) .build() } } object GeneratedAsIdentityType extends Enumeration { type GeneratedAsIdentityType = Value val GeneratedAlways, GeneratedByDefault = Value } case class IdentityColumnSpec( generatedAsIdentityType: GeneratedAsIdentityType.GeneratedAsIdentityType, startsWith: Option[Long] = None, incrementBy: Option[Long] = None, colName: String = "id", dataType: DataType = LongType, comment: Option[String] = None, nullable: Boolean = true) extends ColumnSpec { override def ddl: String = { throw new UnsupportedOperationException( "DDL generation is not supported for identity columns yet") } override def structField(spark: SparkSession): StructField = { var col = io.delta.tables.DeltaTable.columnBuilder(spark, colName) .dataType(dataType) .nullable(nullable) val start = startsWith.getOrElse(IdentityColumn.defaultStart.toLong) val step = incrementBy.getOrElse(IdentityColumn.defaultStep.toLong) col = generatedAsIdentityType match { case GeneratedAsIdentityType.GeneratedAlways => col.generatedAlwaysAsIdentity(start, step) case GeneratedAsIdentityType.GeneratedByDefault => col.generatedByDefaultAsIdentity(start, step) } comment.foreach { c => col = col.comment(c) } col.build() } } trait DDLTestUtils extends QueryTest with SharedSparkSession with DeltaSQLTestUtils with DeltaSQLCommandTest { protected object DDLType extends Enumeration { val CREATE, REPLACE, CREATE_OR_REPLACE = Value } /** Interface (SQL, Scala) agnostic helper to execute the DDL statement. */ protected def runDDL( ddlType: DDLType.Value, tableName: String, columnSpecs: Seq[ColumnSpec], partitionedBy: Seq[String], tblProperties: Map[String, String]): Unit def createTable( tableName: String, columnSpecs: Seq[ColumnSpec], partitionedBy: Seq[String] = Nil, tblProperties: Map[String, String] = Map.empty): Unit = { runDDL(DDLType.CREATE, tableName, columnSpecs, partitionedBy, tblProperties) } def replaceTable( tableName: String, columnSpecs: Seq[ColumnSpec], partitionedBy: Seq[String] = Nil, tblProperties: Map[String, String] = Map.empty): Unit = { runDDL(DDLType.REPLACE, tableName, columnSpecs, partitionedBy, tblProperties) } def createOrReplaceTable( tableName: String, columnSpecs: Seq[ColumnSpec], partitionedBy: Seq[String] = Nil, tblProperties: Map[String, String] = Map.empty): Unit = { runDDL(DDLType.CREATE_OR_REPLACE, tableName, columnSpecs, partitionedBy, tblProperties) } } trait SQLDDLTestUtils extends DDLTestUtils { private def getPartitionByClause(partitionedBy: Seq[String]): String = { if (partitionedBy.nonEmpty) { s"PARTITIONED BY (${partitionedBy.mkString(", ")})" } else { "" } } protected def runDDL( ddlType: DDLType.Value, tableName: String, columnSpecs: Seq[ColumnSpec], partitionedBy: Seq[String], tblProperties: Map[String, String]): Unit = { val columnDefinitions = columnSpecs.map(_.ddl).mkString(",\n") val ddlClause = ddlType match { case DDLType.CREATE => "CREATE TABLE" case DDLType.REPLACE => "REPLACE TABLE" case DDLType.CREATE_OR_REPLACE => "CREATE OR REPLACE TABLE" } val tblPropertiesClause = if (tblProperties.nonEmpty) { val tblPropertiesStr = tblProperties.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ") s"TBLPROPERTIES ($tblPropertiesStr)" } else { "" } sql( s""" |$ddlClause $tableName( | $columnDefinitions |) USING delta |${getPartitionByClause(partitionedBy)} |$tblPropertiesClause |""".stripMargin) } } trait ScalaDDLTestUtils extends DDLTestUtils { protected def runDDL( ddlType: DDLType.Value, tableName: String, columnSpecs: Seq[ColumnSpec], partitionedBy: Seq[String], tblProperties: Map[String, String]): Unit = { val builder = ddlType match { case DDLType.CREATE => io.delta.tables.DeltaTable.create(spark) case DDLType.REPLACE => io.delta.tables.DeltaTable.replace(spark) case DDLType.CREATE_OR_REPLACE => io.delta.tables.DeltaTable.createOrReplace(spark) } builder.tableName(tableName) columnSpecs.foreach { columnSpec => val colAsStructField = columnSpec.structField(spark) builder.addColumn(colAsStructField) } if (partitionedBy.nonEmpty) { builder.partitionedBy(partitionedBy: _*) } for ((key, value) <- tblProperties) { builder.property(key, value) } builder.execute() } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DelegatingLogStoreSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.storage.{DelegatingLogStore, LogStore, LogStoreAdaptor} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.{SparkConf, SparkFunSuite} import org.apache.spark.sql.LocalSparkSession._ import org.apache.spark.sql.SparkSession class DelegatingLogStoreSuite extends SparkFunSuite { private val customLogStoreClassName = classOf[CustomPublicLogStore].getName private def fakeSchemeWithNoDefault = "fake" private def constructSparkConf(confs: Seq[(String, String)]): SparkConf = { val sparkConf = new SparkConf(loadDefaults = false).setMaster("local") confs.foreach { case (key, value) => sparkConf.set(key, value) } sparkConf } /** * Test DelegatingLogStore by directly creating a DelegatingLogStore and test LogStore * resolution based on input `scheme`. This is not an end-to-end test. * * @param scheme The scheme to be used for testing. * @param sparkConf The spark configuration to use. * @param expClassName Expected LogStore class name resolved by DelegatingLogStore. * @param expAdaptor True if DelegatingLogStore is expected to resolve to LogStore adaptor, for * which the actual implementation inside will be checked. This happens when * LogStore is set to subclass of the new [[io.delta.storage.LogStore]] API. */ private def testDelegatingLogStore( scheme: String, sparkConf: SparkConf, expClassName: String, expAdaptor: Boolean): Unit = { withSparkSession(SparkSession.builder.config(sparkConf).getOrCreate()) { spark => val sc = spark.sparkContext val delegatingLogStore = new DelegatingLogStore(sc.hadoopConfiguration) val actualLogStore = delegatingLogStore.getDelegate( new Path(s"${scheme}://dummy")) if (expAdaptor) { assert(actualLogStore.isInstanceOf[LogStoreAdaptor]) assert(actualLogStore.asInstanceOf[LogStoreAdaptor] .logStoreImpl.getClass.getName == expClassName) } else { assert(actualLogStore.getClass.getName == expClassName) } } } /** Test the default LogStore resolution for `scheme` */ private def testDefaultSchemeResolution(scheme: String, expClassName: String): Unit = { testDelegatingLogStore( scheme, constructSparkConf(Seq.empty), // we set no custom LogStore confs expClassName, expAdaptor = true // all default implementations are from delta-storage ) } /** Test LogStore resolution with a customized scheme conf */ private def testCustomSchemeResolution( scheme: String, className: String, expAdaptor: Boolean): Unit = { val sparkPrefixKey = LogStore.logStoreSchemeConfKey(scheme) val nonSparkPrefixKey = sparkPrefixKey.stripPrefix("spark.") // only set spark-prefixed key testDelegatingLogStore( scheme, constructSparkConf(Seq((sparkPrefixKey, className))), className, // we expect our custom-set LogStore class expAdaptor ) // only set non-spark-prefixed key testDelegatingLogStore( scheme, constructSparkConf(Seq((nonSparkPrefixKey, className))), className, // we expect our custom-set LogStore class expAdaptor ) // set both testDelegatingLogStore( scheme, constructSparkConf(Seq((nonSparkPrefixKey, className), (sparkPrefixKey, className))), className, // we expect our custom-set LogStore class expAdaptor ) } test("DelegatingLogStore resolution using default scheme confs") { for (scheme <- DelegatingLogStore.s3Schemes) { testDefaultSchemeResolution( scheme, expClassName = DelegatingLogStore.defaultS3LogStoreClassName) } for (scheme <- DelegatingLogStore.azureSchemes) { testDefaultSchemeResolution( scheme, expClassName = DelegatingLogStore.defaultAzureLogStoreClassName) } for (scheme <- DelegatingLogStore.gsSchemes) { testDefaultSchemeResolution( scheme, expClassName = DelegatingLogStore.defaultGCSLogStoreClassName) } testDefaultSchemeResolution( scheme = fakeSchemeWithNoDefault, expClassName = DelegatingLogStore.defaultHDFSLogStoreClassName) } test("DelegatingLogStore resolution using customized scheme confs") { val allTestSchemes = DelegatingLogStore.s3Schemes ++ DelegatingLogStore.azureSchemes + fakeSchemeWithNoDefault for (scheme <- allTestSchemes) { for (store <- Seq( // default (java) classes (in io.delta.storage) "io.delta.storage.S3SingleDriverLogStore", "io.delta.storage.AzureLogStore", "io.delta.storage.HDFSLogStore", // deprecated (scala) classes classOf[org.apache.spark.sql.delta.storage.S3SingleDriverLogStore].getName, classOf[org.apache.spark.sql.delta.storage.AzureLogStore].getName, classOf[org.apache.spark.sql.delta.storage.HDFSLogStore].getName, customLogStoreClassName)) { // we set spark.delta.logStore.${scheme}.impl -> $store testCustomSchemeResolution( scheme, store, expAdaptor = store.contains("io.delta.storage") || store == customLogStoreClassName) } } } } ////////////////// // Helper Class // ////////////////// class CustomPublicLogStore(initHadoopConf: Configuration) extends io.delta.storage.LogStore(initHadoopConf) { private val logStoreInternal = new io.delta.storage.HDFSLogStore(initHadoopConf) override def read( path: Path, hadoopConf: Configuration): io.delta.storage.CloseableIterator[String] = { logStoreInternal.read(path, hadoopConf) } override def write( path: Path, actions: java.util.Iterator[String], overwrite: java.lang.Boolean, hadoopConf: Configuration): Unit = { logStoreInternal.write(path, actions, overwrite, hadoopConf) } override def listFrom( path: Path, hadoopConf: Configuration): java.util.Iterator[FileStatus] = { logStoreInternal.listFrom(path, hadoopConf) } override def resolvePathOnPhysicalStorage( path: Path, hadoopConf: Configuration): Path = { logStoreInternal.resolvePathOnPhysicalStorage(path, hadoopConf) } override def isPartialWriteVisible(path: Path, hadoopConf: Configuration): java.lang.Boolean = { logStoreInternal.isPartialWriteVisible(path, hadoopConf) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeleteMetricsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import com.databricks.spark.util.DatabricksLogging import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.actions.{Action, AddFile, FileAction, RemoveFile} import org.apache.spark.sql.delta.commands.DeleteMetric import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.{Dataset, QueryTest} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.expr import org.apache.spark.sql.test.SharedSparkSession /** * Tests for metrics of Delta DELETE command. */ class DeleteMetricsSuite extends QueryTest with SharedSparkSession with DatabricksLogging with DeltaSQLCommandTest { /* * Case class to parameterize tests. */ case class TestConfiguration( partitioned: Boolean, cdfEnabled: Boolean ) case class TestMetricResults( operationMetrics: Map[String, Long], numAffectedRows: Long ) /* * Helper to generate tests for all configuration parameters. */ protected def testDeleteMetrics(name: String)(testFn: TestConfiguration => Unit): Unit = { for { partitioned <- BOOLEAN_DOMAIN cdfEnabled <- BOOLEAN_DOMAIN } { val testConfig = TestConfiguration( partitioned = partitioned, cdfEnabled = cdfEnabled ) var testName = s"delete-metrics: $name - Partitioned = $partitioned, cdfEnabled = $cdfEnabled" test(testName) { testFn(testConfig) } } } /* * Create a table from the provided dataset. * * If an partitioned table is needed, then we create one data partition per Spark partition, * i.e. every data partition will contain one file. * * Also an extra column is added to be used in non-partition filters. */ protected def createTempTable( table: Dataset[_], tableName: String, testConfig: TestConfiguration): Unit = { val numRows = table.count() val numPartitions = table.rdd.getNumPartitions val numRowsPerPart = if (numRows > 0 && numPartitions < numRows) numRows / numPartitions else 1 val partitionBy = if (testConfig.partitioned) Seq("partCol") else Seq() table.withColumn("partCol", expr(s"floor(id / $numRowsPerPart)")) .withColumn("extraCol", expr(s"$numRows - id")) .write .partitionBy(partitionBy: _*) .format("delta") .saveAsTable(tableName) } /* * Run a delete command, and capture number of affected rows, operation metrics from Delta * log and usage metrics. */ def runDeleteAndCaptureMetrics( table: Dataset[_], where: String, testConfig: TestConfiguration): TestMetricResults = { val tableName = "target" val whereClause = Option(where).map(c => s"WHERE $c").getOrElse("") var numAffectedRows = -1L var operationMetrics: Map[String, Long] = null withSQLConf( DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true", DeltaSQLConf.DELTA_SKIP_RECORDING_EMPTY_COMMITS.key -> "false", DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> testConfig.cdfEnabled.toString) { withTable(tableName) { createTempTable(table, tableName, testConfig) val resultDf = spark.sql(s"DELETE FROM $tableName $whereClause") assert(!resultDf.isEmpty) numAffectedRows = resultDf.take(1).head(0).toString.toLong operationMetrics = DeltaMetricsUtils.getLastOperationMetrics(tableName) // Check operation metrics against commit actions. val (deltaLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName)) DeltaMetricsUtils.checkOperationMetricsAgainstCommitActions( deltaLog, snapshot.version, operationMetrics) } } TestMetricResults( operationMetrics, numAffectedRows ) } /* * Run a delete command and check all available metrics. * We allow some metrics to be missing, by setting their value to -1. */ def runDeleteAndCheckMetrics( table: Dataset[_], where: String, expectedNumAffectedRows: Long, expectedOperationMetrics: Map[String, Long], testConfig: TestConfiguration): Unit = { // Run the delete capture and get all metrics. val testMetricResults = runDeleteAndCaptureMetrics(table, where, testConfig) val operationMetrics = testMetricResults.operationMetrics // Check the number of deleted rows. assert(testMetricResults.numAffectedRows === expectedNumAffectedRows) // Check operation metrics schema. val unknownKeys = operationMetrics.keySet -- DeltaOperationMetrics.DELETE -- DeltaOperationMetrics.WRITE assert(unknownKeys.isEmpty, s"Unknown operation metrics for DELETE command: ${unknownKeys.mkString(", ")}") // Check values of expected operation metrics. For all unspecified deterministic metrics, // we implicitly expect a zero value. val requiredMetrics = Set( "numCopiedRows", "numDeletedRows", "numAddedFiles", "numRemovedFiles", "numAddedChangeFiles") val expectedMetricsWithDefaults = requiredMetrics.map(k => k -> 0L).toMap ++ expectedOperationMetrics val expectedMetricsFiltered = expectedMetricsWithDefaults.filter(_._2 >= 0) DeltaMetricsUtils.checkOperationMetrics( expectedMetrics = expectedMetricsFiltered, operationMetrics = operationMetrics) // Check time operation metrics. val expectedTimeMetrics = Set("scanTimeMs", "rewriteTimeMs", "executionTimeMs").filter( k => expectedOperationMetrics.get(k).forall(_ >= 0) ) DeltaMetricsUtils.checkOperationTimeMetrics( operationMetrics = operationMetrics, expectedMetrics = expectedTimeMetrics) } val zeroDeleteMetrics: DeleteMetric = DeleteMetric( condition = "", numFilesTotal = 0, numTouchedFiles = 0, numRewrittenFiles = 0, numRemovedFiles = 0, numAddedFiles = 0, numAddedChangeFiles = 0, numFilesBeforeSkipping = 0, numBytesBeforeSkipping = -1, // We don't want to assert equality on bytes numFilesAfterSkipping = 0, numBytesAfterSkipping = -1, // We don't want to assert equality on bytes numPartitionsAfterSkipping = None, numPartitionsAddedTo = None, numPartitionsRemovedFrom = None, numCopiedRows = None, numDeletedRows = None, numBytesAdded = -1, // We don't want to assert equality on bytes numBytesRemoved = -1, // We don't want to assert equality on bytes changeFileBytes = -1, // We don't want to assert equality on bytes scanTimeMs = 0, rewriteTimeMs = 0, numDeletionVectorsAdded = 0, numDeletionVectorsRemoved = 0, numDeletionVectorsUpdated = 0 ) test("delete along partition boundary") { import testImplicits._ Seq(true, false).foreach { cdfEnabled => Seq(true, false).foreach { deltaCollectStatsEnabled => Seq(true, false).foreach { deltaDmlMetricsFromMetadataEnabled => withSQLConf( DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> cdfEnabled.toString, DeltaSQLConf.DELTA_COLLECT_STATS.key -> deltaCollectStatsEnabled.toString, DeltaSQLConf.DELTA_DML_METRICS_FROM_METADATA.key -> deltaDmlMetricsFromMetadataEnabled.toString ) { withTable("t1") { spark.range(100).withColumn("part", 'id % 10).toDF().write .partitionBy("part").format("delta").saveAsTable("t1") val result = spark.sql("DELETE FROM t1 WHERE part=1") .take(1).head(0).toString.toLong val opMetrics = DeltaMetricsUtils.getLastOperationMetrics("t1") assert(opMetrics("numRemovedFiles") > 0) if (deltaCollectStatsEnabled && deltaDmlMetricsFromMetadataEnabled) { assert(opMetrics("numDeletedRows") == 10) assert(result == 10) } else { assert(!opMetrics.contains("numDeletedRows")) assert(result == -1) } } } } } } } testDeleteMetrics("delete from empty table") { testConfig => for (where <- Seq("", "1 = 1", "1 != 1", "id > 50")) { def executeTest: Unit = runDeleteAndCheckMetrics( table = spark.range(0), where = where, expectedNumAffectedRows = 0, expectedOperationMetrics = Map( "numCopiedRows" -> 0, "numDeletedRows" -> 0, "numAddedFiles" -> 0, "numRemovedFiles" -> 0, "numAddedChangeFiles" -> 0, "scanTimeMs" -> -1, "rewriteTimeMs" -> -1, "executionTimeMs" -> -1 ), testConfig = testConfig ) executeTest } } for (whereClause <- Seq("", "1 = 1")) { testDeleteMetrics(s"delete all with where = '$whereClause'") { testConfig => runDeleteAndCheckMetrics( table = spark.range(start = 0, end = 100, step = 1, numPartitions = 5), where = whereClause, expectedNumAffectedRows = 100, expectedOperationMetrics = Map( "numCopiedRows" -> -1, "numDeletedRows" -> 100, "numOutputRows" -> -1, "numFiles" -> -1, "numAddedFiles" -> -1, "numRemovedFiles" -> 5, "numAddedChangeFiles" -> 0 ), testConfig = testConfig ) } } testDeleteMetrics("delete with false predicate") { testConfig => { runDeleteAndCheckMetrics( table = spark.range(start = 0, end = 100, step = 1, numPartitions = 5), where = "1 != 1", expectedNumAffectedRows = 0L, expectedOperationMetrics = Map( "numCopiedRows" -> 0, "numDeletedRows" -> 0, "numAddedFiles" -> 0, "numRemovedFiles" -> 0, "numAddedChangeFiles" -> 0, "scanTimeMs" -> -1, "rewriteTimeMs" -> -1, "executionTimeMs" -> -1 ), testConfig = testConfig ) }} testDeleteMetrics("delete with unsatisfied static predicate") { testConfig => { runDeleteAndCheckMetrics( table = spark.range(start = 0, end = 100, step = 1, numPartitions = 5), where = "id < 0 or id > 100", expectedNumAffectedRows = 0L, expectedOperationMetrics = Map( "numCopiedRows" -> 0, "numDeletedRows" -> 0, "numAddedFiles" -> 0, "numRemovedFiles" -> 0, "numAddedChangeFiles" -> 0, "scanTimeMs" -> -1, "rewriteTimeMs" -> -1, "executionTimeMs" -> -1 ), testConfig = testConfig ) }} testDeleteMetrics("delete with unsatisfied dynamic predicate") { testConfig => { runDeleteAndCheckMetrics( table = spark.range(start = 0, end = 100, step = 1, numPartitions = 5), where = "id / 200 > 1 ", expectedNumAffectedRows = 0L, expectedOperationMetrics = Map( "numCopiedRows" -> 0, "numDeletedRows" -> 0, "numAddedFiles" -> 0, "numRemovedFiles" -> 0, "numAddedChangeFiles" -> 0, "scanTimeMs" -> -1, "rewriteTimeMs" -> -1, "executionTimeMs" -> -1 ), testConfig = testConfig ) }} for (whereClause <- Seq("id = 0", "id >= 49 and id < 50")) { testDeleteMetrics(s"delete one row with where = `$whereClause`") { testConfig => var numAddedFiles = 1 var numRemovedFiles = 1 val numRemovedRows = 1 var numCopiedRows = 19 runDeleteAndCheckMetrics( table = spark.range(start = 0, end = 100, step = 1, numPartitions = 5), where = whereClause, expectedNumAffectedRows = 1L, expectedOperationMetrics = Map( "numCopiedRows" -> numCopiedRows, "numDeletedRows" -> numRemovedRows, "numAddedFiles" -> numAddedFiles, "numRemovedFiles" -> numRemovedFiles, "numAddedChangeFiles" -> { if (testConfig.cdfEnabled ) { 1 } else { 0 } } ), testConfig = testConfig ) } } testDeleteMetrics("delete one file") { testConfig => val numRemovedFiles = 1 val numRemovedRows = 20 def executeTest: Unit = runDeleteAndCheckMetrics( table = spark.range(start = 0, end = 100, step = 1, numPartitions = 5), where = "id < 20", expectedNumAffectedRows = 20L, expectedOperationMetrics = Map( "numCopiedRows" -> 0, "numDeletedRows" -> numRemovedRows, "numAddedFiles" -> 0, "numRemovedFiles" -> numRemovedFiles, "numAddedChangeFiles" -> { if (testConfig.cdfEnabled ) { 1 } else { 0 } } ), testConfig = testConfig ) executeTest } testDeleteMetrics("delete one row per file") { testConfig => var numRemovedFiles = 5 val numRemovedRows = 5 var numCopiedRows = 95 var numAddedFiles = if (testConfig.partitioned) 5 else 2 runDeleteAndCheckMetrics( table = spark.range(start = 0, end = 100, step = 1, numPartitions = 5), where = "id in (5, 25, 45, 65, 85)", expectedNumAffectedRows = 5L, expectedOperationMetrics = Map( "numCopiedRows" -> numCopiedRows, "numDeletedRows" -> numRemovedRows, "numAddedFiles" -> numAddedFiles, "numRemovedFiles" -> numRemovedFiles, "numAddedChangeFiles" -> { if (testConfig.cdfEnabled) numAddedFiles else 0 } ), testConfig = testConfig ) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeleteSQLSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaColumnMappingSelectedTestMixin, DeltaExcludedTestMixin, DeltaSQLCommandTest} import org.apache.spark.sql.Row trait DeleteSQLMixin extends DeleteBaseMixin with DeltaDMLTestUtils with DeltaSQLCommandTest { override protected def executeDelete(target: String, where: String = null): Unit = { val whereClause = Option(where).map(c => s"WHERE $c").getOrElse("") sql(s"DELETE FROM $target $whereClause") } } trait DeleteSQLTests extends DeleteSQLMixin { import testImplicits._ // For EXPLAIN, which is not supported in OSS test("explain") { append(Seq((2, 2)).toDF("key", "value")) val df = sql(s"EXPLAIN DELETE FROM $tableSQLIdentifier WHERE key = 2") val outputs = df.collect().map(_.mkString).mkString assert(outputs.contains("Delta")) assert(!outputs.contains("index") && !outputs.contains("ActionLog")) // no change should be made by explain checkAnswer(readDeltaTableByIdentifier(), Row(2, 2)) } test("delete from a temp view") { withTable("tab") { withTempView("v") { Seq((1, 1), (0, 3), (1, 5)).toDF("key", "value").write.format("delta").saveAsTable("tab") spark.table("tab").as("name").createTempView("v") sql("DELETE FROM v WHERE key = 1") checkAnswer(spark.table("tab"), Row(0, 3)) } } } test("delete from a SQL temp view") { withTable("tab") { withTempView("v") { Seq((1, 1), (0, 3), (1, 5)).toDF("key", "value").write.format("delta").saveAsTable("tab") sql("CREATE TEMP VIEW v AS SELECT * FROM tab") sql("DELETE FROM v WHERE key = 1 AND VALUE = 5") checkAnswer(spark.table("tab"), Seq(Row(1, 1), Row(0, 3))) } } } Seq(true, false).foreach { partitioned => test(s"User defined _change_type column doesn't get dropped - partitioned=$partitioned") { withTable("tab") { sql( s"""CREATE TABLE tab USING DELTA |${if (partitioned) "PARTITIONED BY (part) " else ""} |TBLPROPERTIES (delta.enableChangeDataFeed = false) |AS SELECT id, int(id / 10) AS part, 'foo' as _change_type |FROM RANGE(1000) |""".stripMargin) val rowsToDelete = (1 to 1000 by 42).mkString("(", ", ", ")") executeDelete("tab", s"id in $rowsToDelete") sql("SELECT id, _change_type FROM tab").collect().foreach { row => val _change_type = row.getString(1) assert(_change_type === "foo", s"Invalid _change_type for id=${row.get(0)}") } } } } } trait DeleteSQLNameColumnMappingMixin extends DeleteSQLMixin with DeltaColumnMappingSelectedTestMixin { protected override def runOnlyTests: Seq[String] = Seq(true, false).map { isPartitioned => s"basic case - delete from a Delta table - Partition=$isPartitioned" } ++ Seq(true, false).flatMap { isPartitioned => Seq( s"where key columns - Partition=$isPartitioned", s"where data columns - Partition=$isPartitioned") } } trait DeleteSQLWithDeletionVectorsMixin extends DeleteSQLMixin with DeltaExcludedTestMixin with DeletionVectorsTestUtils with DeltaDMLTestUtilsPathBased { override def beforeAll(): Unit = { super.beforeAll() enableDeletionVectors(spark, delete = true) } override def excluded: Seq[String] = super.excluded ++ Seq( // The following two tests must fail when DV is used. Covered by another test case: // "throw error when non-pinned TahoeFileIndex snapshot is used". "data and partition columns - Partition=true Skipping=false", "data and partition columns - Partition=false Skipping=false", // The scan schema contains additional row index filter columns. "nested schema pruning on data condition", // The number of records is not recomputed when using DVs "delete throws error if number of records increases", "delete logs error if number of records are missing in stats" ) // This works correctly with DVs, but fails in classic DELETE. override def testSuperSetColsTempView(): Unit = { testComplexTempViews("superset cols")( text = "SELECT key, value, 1 FROM tab", expectResult = Row(0, 3, 1) :: Nil) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeleteScalaSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.test.{DeltaExcludedTestMixin, DeltaSQLCommandTest} import org.apache.spark.sql.{functions, Row} trait DeleteScalaMixin extends DeleteBaseMixin with DeltaSQLCommandTest with DeltaDMLTestUtilsPathBased with DeltaExcludedTestMixin { override protected def executeDelete(target: String, where: String = null): Unit = { val deltaTable: io.delta.tables.DeltaTable = DeltaTestUtils.getDeltaTableForIdentifierOrPath( spark, DeltaTestUtils.getTableIdentifierOrPath(target)) if (where != null) { deltaTable.delete(where) } else { deltaTable.delete() } } } trait DeleteScalaTests extends DeleteScalaMixin { import testImplicits._ test("delete usage test - without condition") { append(Seq((1, 10), (2, 20), (3, 30), (4, 40)).toDF("key", "value")) val table = io.delta.tables.DeltaTable.forPath(tempPath) table.delete() checkAnswer(readDeltaTable(tempPath), Nil) } test("delete usage test - with condition") { append(Seq((1, 10), (2, 20), (3, 30), (4, 40)).toDF("key", "value")) val table = io.delta.tables.DeltaTable.forPath(tempPath) table.delete("key = 1 or key = 2") checkAnswer(readDeltaTable(tempPath), Row(3, 30) :: Row(4, 40) :: Nil) } test("delete usage test - with Column condition") { append(Seq((1, 10), (2, 20), (3, 30), (4, 40)).toDF("key", "value")) val table = io.delta.tables.DeltaTable.forPath(tempPath) table.delete(functions.expr("key = 1 or key = 2")) checkAnswer(readDeltaTable(tempPath), Row(3, 30) :: Row(4, 40) :: Nil) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeleteSuiteBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.{SparkThrowable, SparkUnsupportedOperationException} import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.execution.FileSourceScanExec import org.apache.spark.sql.functions.{lit, struct} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.StructType trait DeleteBaseMixin extends QueryTest with SharedSparkSession with DeltaDMLTestUtils with DeltaTestUtilsForTempViews { import testImplicits._ protected def executeDelete(target: String, where: String = null): Unit protected def checkDelete( condition: Option[String], expectedResults: Seq[Row], tableName: Option[String] = None): Unit = { val target = tableName.getOrElse(tableSQLIdentifier) executeDelete(target = target, where = condition.orNull) checkAnswer(readDeltaTableByIdentifier(target), expectedResults) } protected def testInvalidTempViews(name: String)( text: String, expectedErrorMsgForSQLTempView: String = null, expectedErrorMsgForDataSetTempView: String = null, expectedErrorClassForSQLTempView: String = null, expectedErrorClassForDataSetTempView: String = null): Unit = { testWithTempView(s"test delete on temp view - $name") { isSQLTempView => withTable("tab") { Seq((0, 3), (1, 2)).toDF("key", "value").write.format("delta").saveAsTable("tab") if (isSQLTempView) { sql(s"CREATE TEMP VIEW v AS $text") } else { sql(text).createOrReplaceTempView("v") } val ex = intercept[AnalysisException] { executeDelete( "v", "key >= 1 and value < 3" ) } testErrorMessageAndClass( isSQLTempView, ex, expectedErrorMsgForSQLTempView, expectedErrorMsgForDataSetTempView, expectedErrorClassForSQLTempView, expectedErrorClassForDataSetTempView) } } } // Need to be able to override this, because it works in some configurations. protected def testSuperSetColsTempView(): Unit = { testInvalidTempViews("superset cols")( text = "SELECT key, value, 1 FROM tab", // The analyzer can't tell whether the table originally had the extra column or not. expectedErrorMsgForSQLTempView = "Can't resolve column 1 in root", expectedErrorMsgForDataSetTempView = "Can't resolve column 1 in root" ) } protected def testComplexTempViews(name: String)( text: String, expectResult: Seq[Row]): Unit = { testWithTempView(s"test delete on temp view - $name") { isSQLTempView => withTable("tab") { Seq((0, 3), (1, 2)).toDF("key", "value").write.format("delta").saveAsTable("tab") createTempViewFromSelect(text, isSQLTempView) executeDelete( "v", "key >= 1 and value < 3" ) checkAnswer(spark.read.format("delta").table("v"), expectResult) } } } } trait DeleteTempViewTests extends DeleteBaseMixin with DeltaDMLTestUtilsPathBased { import testImplicits._ Seq(true, false).foreach { isPartitioned => val name = s"test delete on temp view - basic - Partition=$isPartitioned" testWithTempView(name) { isSQLTempView => val partitions = if (isPartitioned) "key" :: Nil else Nil append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value"), partitions) createTempViewFromTable(tableSQLIdentifier, isSQLTempView) checkDelete( condition = Some("key <= 1"), expectedResults = Row(2, 2) :: Nil, tableName = Some("v")) } } testInvalidTempViews("subset cols")( text = "SELECT key FROM tab", expectedErrorClassForSQLTempView = "UNRESOLVED_COLUMN.WITH_SUGGESTION", expectedErrorClassForDataSetTempView = "UNRESOLVED_COLUMN.WITH_SUGGESTION" ) testSuperSetColsTempView() testComplexTempViews("nontrivial projection")( text = "SELECT value as key, key as value FROM tab", expectResult = Row(3, 0) :: Nil ) testComplexTempViews("view with too many internal aliases")( text = "SELECT * FROM (SELECT * FROM tab AS t1) AS t2", expectResult = Row(0, 3) :: Nil ) } trait DeleteBaseTests extends DeleteBaseMixin { import testImplicits._ Seq(true, false).foreach { isPartitioned => test(s"basic case - Partition=$isPartitioned") { val partitions = if (isPartitioned) "key" :: Nil else Nil append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value"), partitions) checkDelete(condition = None, Nil) } } Seq(true, false).foreach { isPartitioned => test(s"basic case - delete from a Delta table - Partition=$isPartitioned") { withTable("deltaTable") { val partitions = if (isPartitioned) "key" :: Nil else Nil val input = Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value") append(input, partitions) checkDelete(Some("value = 4 and key = 3"), Row(2, 2) :: Row(1, 4) :: Row(1, 1) :: Row(0, 3) :: Nil) checkDelete(Some("value = 4 and key = 1"), Row(2, 2) :: Row(1, 1) :: Row(0, 3) :: Nil) checkDelete(Some("value = 2 or key = 1"), Row(0, 3) :: Nil) checkDelete(Some("key = 0 or value = 99"), Nil) } } } Seq(true, false).foreach { isPartitioned => test(s"basic key columns - Partition=$isPartitioned") { val input = Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value") val partitions = if (isPartitioned) "key" :: Nil else Nil append(input, partitions) checkDelete(Some("key > 2"), Row(2, 2) :: Row(1, 4) :: Row(1, 1) :: Row(0, 3) :: Nil) checkDelete(Some("key < 2"), Row(2, 2) :: Nil) checkDelete(Some("key = 2"), Nil) } } Seq(true, false).foreach { isPartitioned => test(s"where key columns - Partition=$isPartitioned") { val partitions = if (isPartitioned) "key" :: Nil else Nil append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value"), partitions) checkDelete(Some("key = 1"), Row(2, 2) :: Row(0, 3) :: Nil) checkDelete(Some("key = 2"), Row(0, 3) :: Nil) checkDelete(Some("key = 0"), Nil) } } Seq(true, false).foreach { isPartitioned => test(s"where data columns - Partition=$isPartitioned") { val partitions = if (isPartitioned) "key" :: Nil else Nil append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value"), partitions) checkDelete(Some("value <= 2"), Row(1, 4) :: Row(0, 3) :: Nil) checkDelete(Some("value = 3"), Row(1, 4) :: Nil) checkDelete(Some("value != 0"), Nil) } } test("where data columns and partition columns") { val input = Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value") append(input, Seq("key")) checkDelete(Some("value = 4 and key = 3"), Row(2, 2) :: Row(1, 4) :: Row(1, 1) :: Row(0, 3) :: Nil) checkDelete(Some("value = 4 and key = 1"), Row(2, 2) :: Row(1, 1) :: Row(0, 3) :: Nil) checkDelete(Some("value = 2 or key = 1"), Row(0, 3) :: Nil) checkDelete(Some("key = 0 or value = 99"), Nil) } Seq(true, false).foreach { skippingEnabled => Seq(true, false).foreach { isPartitioned => test(s"data and partition columns - Partition=$isPartitioned Skipping=$skippingEnabled") { withSQLConf(DeltaSQLConf.DELTA_STATS_SKIPPING.key -> skippingEnabled.toString) { val partitions = if (isPartitioned) "key" :: Nil else Nil val input = Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value") append(input, partitions) checkDelete(Some("value = 4 and key = 3"), Row(2, 2) :: Row(1, 4) :: Row(1, 1) :: Row(0, 3) :: Nil) checkDelete(Some("value = 4 and key = 1"), Row(2, 2) :: Row(1, 1) :: Row(0, 3) :: Nil) checkDelete(Some("value = 2 or key = 1"), Row(0, 3) :: Nil) checkDelete(Some("key = 0 or value = 99"), Nil) } } } } test("Negative case - non-Delta target") { writeTable( Seq((1, 1), (0, 3), (1, 5)).toDF("key1", "value") .write .mode("overwrite") .format("parquet"), tableSQLIdentifier) intercept[SparkThrowable] { executeDelete(target = tableSQLIdentifier) } match { // Thrown when running with path-based SQL case e: DeltaAnalysisException if e.getCondition == "DELTA_TABLE_NOT_FOUND" => checkError(e, "DELTA_TABLE_NOT_FOUND", parameters = Map("tableName" -> tableSQLIdentifier.stripPrefix("delta."))) case e: DeltaAnalysisException if e.getCondition == "DELTA_MISSING_TRANSACTION_LOG" => checkErrorMatchPVals(e, "DELTA_MISSING_TRANSACTION_LOG", parameters = Map("operation" -> "read from", "path" -> ".*", "docLink" -> "https://.*")) // Thrown when running with path-based Scala API case e: DeltaAnalysisException if e.getCondition == "DELTA_MISSING_DELTA_TABLE" => checkError(e, "DELTA_MISSING_DELTA_TABLE", parameters = Map("tableName" -> tableSQLIdentifier.stripPrefix("delta."))) // Thrown when running with name-based SQL case e: AnalysisException => checkErrorMatchPVals(e, "UNSUPPORTED_FEATURE.TABLE_OPERATION", parameters = Map( "tableName" -> s".*$tableSQLIdentifier.*", "operation" -> "DELETE")) } } test("Negative case - non-deterministic condition") { append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value")) val e = intercept[AnalysisException] { executeDelete(target = tableSQLIdentifier, where = "rand() > 0.5") }.getMessage assert(e.contains("nondeterministic expressions are only allowed in") || e.contains("The operator expects a deterministic expression")) } test("Negative case - DELETE the child directory", NameBasedAccessIncompatible) { withTempPath { tempDir => val tempPath = tempDir.getCanonicalPath val df = Seq((2, 2), (3, 2)).toDF("key", "value") df.write.format("delta").partitionBy("key").save(tempPath) val e = intercept[AnalysisException] { executeDelete(target = s"delta.`$tempPath/key=2`", where = "value = 2") }.getMessage assert(e.contains("Expect a full scan of Delta sources, but found a partial scan")) } } test("delete cached table by name") { withTable("cached_delta_table") { Seq((2, 2), (1, 4)).toDF("key", "value") .write.format("delta").saveAsTable("cached_delta_table") spark.table("cached_delta_table").cache() spark.table("cached_delta_table").collect() executeDelete(target = "cached_delta_table", where = "key = 2") checkAnswer(spark.table("cached_delta_table"), Row(1, 4) :: Nil) } } test("delete cached table") { append(Seq((2, 2), (1, 4)).toDF("key", "value")) readDeltaTableByIdentifier().cache() readDeltaTableByIdentifier().collect() executeDelete(tableSQLIdentifier, where = "key = 2") checkAnswer(readDeltaTableByIdentifier(), Row(1, 4) :: Nil) } Seq(true, false).foreach { isPartitioned => test(s"condition having current_date - Partition=$isPartitioned") { val partitions = if (isPartitioned) "key" :: Nil else Nil append( Seq((java.sql.Date.valueOf("1969-12-31"), 2), (java.sql.Date.valueOf("2099-12-31"), 4)) .toDF("key", "value"), partitions) checkDelete(Some("CURRENT_DATE > key"), Row(java.sql.Date.valueOf("2099-12-31"), 4) :: Nil) checkDelete(Some("CURRENT_DATE <= key"), Nil) } } test("condition having current_timestamp - Partition by Timestamp") { append( Seq((java.sql.Timestamp.valueOf("2012-12-31 16:00:10.011"), 2), (java.sql.Timestamp.valueOf("2099-12-31 16:00:10.011"), 4)) .toDF("key", "value"), Seq("key")) checkDelete(Some("CURRENT_TIMESTAMP > key"), Row(java.sql.Timestamp.valueOf("2099-12-31 16:00:10.011"), 4) :: Nil) checkDelete(Some("CURRENT_TIMESTAMP <= key"), Nil) } Seq(true, false).foreach { isPartitioned => test(s"foldable condition - Partition=$isPartitioned") { val partitions = if (isPartitioned) "key" :: Nil else Nil append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value"), partitions) val allRows = Row(2, 2) :: Row(1, 4) :: Row(1, 1) :: Row(0, 3) :: Nil checkDelete(Some("false"), allRows) checkDelete(Some("1 <> 1"), allRows) checkDelete(Some("1 > null"), allRows) checkDelete(Some("true"), Nil) checkDelete(Some("1 = 1"), Nil) } } test("SC-12232: should not delete the rows where condition evaluates to null") { append(Seq(("a", null), ("b", null), ("c", "v"), ("d", "vv")).toDF("key", "value").coalesce(1)) // "null = null" evaluates to null checkDelete(Some("value = null"), Row("a", null) :: Row("b", null) :: Row("c", "v") :: Row("d", "vv") :: Nil) // these expressions evaluate to null when value is null checkDelete(Some("value = 'v'"), Row("a", null) :: Row("b", null) :: Row("d", "vv") :: Nil) checkDelete(Some("value <> 'v'"), Row("a", null) :: Row("b", null) :: Nil) } test("SC-12232: delete rows with null values using isNull") { append(Seq(("a", null), ("b", null), ("c", "v"), ("d", "vv")).toDF("key", "value").coalesce(1)) // when value is null, this expression evaluates to true checkDelete(Some("value is null"), Row("c", "v") :: Row("d", "vv") :: Nil) } test("SC-12232: delete rows with null values using EqualNullSafe") { append(Seq(("a", null), ("b", null), ("c", "v"), ("d", "vv")).toDF("key", "value").coalesce(1)) // when value is null, this expression evaluates to true checkDelete(Some("value <=> null"), Row("c", "v") :: Row("d", "vv") :: Nil) } test("do not support subquery test") { append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value")) Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("c", "d").createOrReplaceTempView("source") // basic subquery val e0 = intercept[AnalysisException] { executeDelete(target = tableSQLIdentifier, "key < (SELECT max(c) FROM source)") }.getMessage assert(e0.contains("Subqueries are not supported")) // subquery with EXISTS val e1 = intercept[AnalysisException] { executeDelete(target = tableSQLIdentifier, "EXISTS (SELECT max(c) FROM source)") }.getMessage assert(e1.contains("Subqueries are not supported")) // subquery with NOT EXISTS val e2 = intercept[AnalysisException] { executeDelete(target = tableSQLIdentifier, "NOT EXISTS (SELECT max(c) FROM source)") }.getMessage assert(e2.contains("Subqueries are not supported")) // subquery with IN val e3 = intercept[AnalysisException] { executeDelete(target = tableSQLIdentifier, "key IN (SELECT max(c) FROM source)") }.getMessage assert(e3.contains("Subqueries are not supported")) // subquery with NOT IN val e4 = intercept[AnalysisException] { executeDelete(target = tableSQLIdentifier, "key NOT IN (SELECT max(c) FROM source)") }.getMessage assert(e4.contains("Subqueries are not supported")) } test("schema pruning on data condition") { val input = Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value") append(input, Nil) // Start from a cached snapshot state deltaLog.update().stateDF val executedPlans = DeltaTestUtils.withPhysicalPlansCaptured(spark) { checkDelete(Some("key = 2"), Row(1, 4) :: Row(1, 1) :: Row(0, 3) :: Nil) } val scans = executedPlans.flatMap(_.collect { case f: FileSourceScanExec => f }) // The first scan is for finding files to delete. We only are matching against the key // so that should be the only field in the schema assert(scans.head.schema.findNestedField(Seq("key")).nonEmpty) assert(scans.head.schema.findNestedField(Seq("value")).isEmpty) } test("nested schema pruning on data condition") { val input = Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value") .select(struct("key", "value").alias("nested")) append(input, Nil) // Start from a cached snapshot state deltaLog.update().stateDF val executedPlans = DeltaTestUtils.withPhysicalPlansCaptured(spark) { checkDelete(Some("nested.key = 2"), Row(Row(1, 4)) :: Row(Row(1, 1)) :: Row(Row(0, 3)) :: Nil) } val scans = executedPlans.flatMap(_.collect { case f: FileSourceScanExec => f }) assert(scans.head.schema == StructType.fromDDL("nested STRUCT")) } /** * @param function the unsupported function. * @param functionType The type of the unsupported expression to be tested. * @param data the data in the table. * @param where the where clause containing the unsupported expression. * @param expectException whether an exception is expected to be thrown * @param customErrorRegex customized error regex. */ private def testUnsupportedExpression( function: String, functionType: String, data: => DataFrame, where: String, expectException: Boolean, customErrorRegex: Option[String] = None) { test(s"$functionType functions in delete - expect exception: $expectException") { withTable("deltaTable") { data.write.format("delta").saveAsTable("deltaTable") val expectedErrorRegex = "(?s).*(?i)unsupported.*(?i).*Invalid expressions.*" var catchException = true var errorRegex = if (functionType.equals("Generate")) { ".*Subqueries are not supported in the DELETE.*" } else customErrorRegex.getOrElse(expectedErrorRegex) if (catchException) { val dataBeforeException = spark.read.format("delta").table("deltaTable").collect() val e = intercept[Exception] { executeDelete(target = "deltaTable", where = where) } val message = if (e.getCause != null) { e.getCause.getMessage } else e.getMessage assert(message.matches(errorRegex)) checkAnswer(spark.read.format("delta").table("deltaTable"), dataBeforeException) } else { executeDelete(target = "deltaTable", where = where) } } } } testUnsupportedExpression( function = "row_number", functionType = "Window", data = Seq((2, 2), (1, 4)).toDF("key", "value"), where = "row_number() over (order by value) > 1", expectException = true ) testUnsupportedExpression( function = "max", functionType = "Aggregate", data = Seq((2, 2), (1, 4)).toDF("key", "value"), where = "key > max(value)", expectException = true ) // Explode functions are supported in where if only one row generated. testUnsupportedExpression( function = "explode", functionType = "Generate", data = Seq((2, List(2))).toDF("key", "value"), where = "key = (select explode(value) from deltaTable)", expectException = false // generate only one row, no exception. ) // Explode functions are supported in where but if there's more than one row generated, // it will throw an exception. testUnsupportedExpression( function = "explode", functionType = "Generate", data = Seq((2, List(2)), (1, List(4, 5))).toDF("key", "value"), where = "key = (select explode(value) from deltaTable)", expectException = true, // generate more than one row. Exception expected. customErrorRegex = Some(".*More than one row returned by a subquery used as an expression(?s).*") ) test("Variant type") { val dstDf = sql( """SELECT parse_json(cast(id as string)) v, id i FROM range(3)""") append(dstDf) executeDelete(target = tableSQLIdentifier, where = "to_json(v) = '1'") checkAnswer(readDeltaTableByIdentifier().selectExpr("i", "to_json(v)"), Seq(Row(0, "0"), Row(2, "2"))) } test("delete on partitioned table with special chars") { val partValue = "part%one" append( spark.range(0, 3, 1, 1).toDF("key").withColumn("value", lit(partValue)), partitionBy = Seq("value")) checkDelete( condition = Some(s"value = '$partValue' and key = 1"), expectedResults = Row(0, partValue) :: Row(2, partValue) :: Nil) checkDelete( condition = Some(s"value = '$partValue' and key = 2"), expectedResults = Row(0, partValue) :: Nil) checkDelete( condition = Some(s"value = '$partValue'"), expectedResults = Nil) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeletionVectorsTestUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.util.UUID import org.apache.spark.sql.delta.DeltaOperations.Truncate import org.apache.spark.sql.delta.actions.{Action, AddFile, DeletionVectorDescriptor, RemoveFile} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.{AlterTableDropFeatureDeltaCommand, DeletionVectorUtils} import org.apache.spark.sql.delta.deletionvectors.{RoaringBitmapArray, RoaringBitmapArrayFormat} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.PathWithFileSystem import org.apache.commons.io.FileUtils import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.internal.config.ConfigEntry import org.apache.spark.sql.{DataFrame, QueryTest, RuntimeConfig, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.{col, lit} import org.apache.spark.sql.test.SharedSparkSession trait MergePersistentDVDisabled extends SharedSparkSession { override protected def sparkConf: SparkConf = super.sparkConf .set(DeltaSQLConf.MERGE_USE_PERSISTENT_DELETION_VECTORS.key, "false") } trait PersistentDVDisabled extends SharedSparkSession { override protected def sparkConf: SparkConf = super.sparkConf .set(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey, "false") } trait PersistentDVEnabled extends DeletionVectorsTestUtils { override def beforeAll(): Unit = { super.beforeAll() enableDeletionVectorsInNewTables(spark.conf) } } trait PredicatePushdownDisabled extends SharedSparkSession { override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX.key, "false") } } trait PredicatePushdownEnabled extends SharedSparkSession { override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX.key, "true") } } /** Collection of test utilities related with persistent Deletion Vectors. */ trait DeletionVectorsTestUtils extends QueryTest with SharedSparkSession with DeltaSQLTestUtils { def enableDeletionVectors( spark: SparkSession, delete: Boolean = false, update: Boolean = false, merge: Boolean = false): Unit = { val global = delete || update || merge spark.conf .set(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey, global.toString) spark.conf.set(DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS.key, delete.toString) spark.conf.set(DeltaSQLConf.UPDATE_USE_PERSISTENT_DELETION_VECTORS.key, update.toString) spark.conf.set(DeltaSQLConf.MERGE_USE_PERSISTENT_DELETION_VECTORS.key, merge.toString) } /** Disable persistent deletion vectors in new tables and all supported DML commands. */ def disableDeletionVectors(conf: RuntimeConfig): Unit = { conf.set(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey, false.toString) conf.set(DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS.key, false.toString) conf.set(DeltaSQLConf.UPDATE_USE_PERSISTENT_DELETION_VECTORS.key, false.toString) } def enableDeletionVectorsForAllSupportedOperations(spark: SparkSession): Unit = enableDeletionVectors(spark, delete = true, update = true) def deletionVectorsEnabledInCommand( sparkSession: SparkSession, deltaLog: DeltaLog, dmlConfig: ConfigEntry[Boolean]): Boolean = DeletionVectorUtils.deletionVectorsWritable(deltaLog.update()) && sparkSession.sessionState.conf.getConf(dmlConfig) /** Whether persistent Deletion Vectors are enabled in MERGE command. */ def deletionVectorsEnabledInMerge(spark: SparkSession, deltaLog: DeltaLog): Boolean = { deletionVectorsEnabledInCommand(spark, deltaLog, DeltaSQLConf.MERGE_USE_PERSISTENT_DELETION_VECTORS) } /** Whether persistent Deletion Vectors are enabled in UPDATE command. */ def deletionVectorsEnabledInUpdate(spark: SparkSession, deltaLog: DeltaLog): Boolean = deletionVectorsEnabledInCommand(spark, deltaLog, DeltaSQLConf.UPDATE_USE_PERSISTENT_DELETION_VECTORS) /** Whether persistent Deletion Vectors are enabled in DELETE command. */ def deletionVectorsEnabledInDelete(spark: SparkSession, deltaLog: DeltaLog): Boolean = { deletionVectorsEnabledInCommand( spark, deltaLog, DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS) } def testWithDVs(testName: String, testTags: org.scalatest.Tag*)(thunk: => Unit): Unit = { test(testName, testTags : _*) { withDeletionVectorsEnabled() { thunk } } } /** Run a thunk with Deletion Vectors enabled/disabled. */ def withDeletionVectorsEnabled(enabled: Boolean = true)(thunk: => Unit): Unit = { val enabledStr = enabled.toString withSQLConf( DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> enabledStr, DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS.key -> enabledStr, DeltaSQLConf.UPDATE_USE_PERSISTENT_DELETION_VECTORS.key -> enabledStr, DeltaSQLConf.MERGE_USE_PERSISTENT_DELETION_VECTORS.key -> enabledStr) { thunk } } def dropDVTableFeature( spark: SparkSession, log: DeltaLog, truncateHistory: Boolean): Unit = AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, log.dataPath), DeletionVectorsTableFeature.name, truncateHistory = truncateHistory).run(spark) /** Helper to run 'fn' with a temporary Delta table. */ def withTempDeltaTable( dataDF: DataFrame, partitionBy: Seq[String] = Seq.empty, enableDVs: Boolean = true, conf: Seq[(String, String)] = Nil, createNameBasedTable: Boolean = false) (fn: (() => io.delta.tables.DeltaTable, DeltaLog) => Unit): Unit = { def createTable(tableNameOpt: Option[String], tablePathOpt: Option[Path]): Unit = { assert((tableNameOpt.isDefined && tablePathOpt.isEmpty) || (tableNameOpt.isEmpty && tablePathOpt.isDefined)) withSQLConf(conf: _*) { val df = dataDF.write .option(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key, enableDVs.toString) .partitionBy(partitionBy: _*) .format("delta") (tableNameOpt, tablePathOpt) match { case (Some(tableName), None) => df.saveAsTable(tableName) case (None, Some(tablePath)) => df.save(tablePath.toString) } } // DeltaTable hangs on to the DataFrame it is created with for the entire object lifetime. // That means subsequent `targetTable.toDF` calls will return the same snapshot. // The DV tests are generally written assuming `targetTable.toDF` would return a new snapshot. // So create a function here instead of an instance, so `targetTable().toDF` // will actually provide a new snapshot. val targetTable = (tableNameOpt, tablePathOpt) match { case (Some(tableName), None) => () => io.delta.tables.DeltaTable.forName(tableName) case (None, Some(tablePath)) => () => io.delta.tables.DeltaTable.forPath(tablePath.toString) } val targetLog = (tableNameOpt, tablePathOpt) match { case (Some(tableName), None) => DeltaLog.forTable(spark, TableIdentifier(tableName)) case (None, Some(tablePath)) => DeltaLog.forTable(spark, tablePath) } fn(targetTable, targetLog) } if (createNameBasedTable) { withTempTable(createTable = false) { tableName => createTable(tableNameOpt = Some(tableName), tablePathOpt = None) } } else { withTempPath { path => val tablePath = new Path(path.getAbsolutePath) createTable(tableNameOpt = None, tablePathOpt = Some(tablePath)) } } } /** Create a temp path which contains special characters. */ override def withTempPath(f: File => Unit): Unit = { super.withTempPath(prefix = "s p a r k %2a")(f) } /** Create a temp path which contains special characters. */ override protected def withTempDir(f: File => Unit): Unit = { super.withTempDir(prefix = "s p a r k %2a")(f) } /** Helper that verifies whether a defined number of DVs exist */ def verifyDVsExist(targetLog: DeltaLog, filesWithDVsSize: Int): Unit = { val filesWithDVs = getFilesWithDeletionVectors(targetLog) assert(filesWithDVs.size === filesWithDVsSize) assertDeletionVectorsExist(targetLog, filesWithDVs) } /** Returns all [[AddFile]] actions of a Delta table that contain Deletion Vectors. */ def getFilesWithDeletionVectors(log: DeltaLog): Seq[AddFile] = log.update().allFiles.collect().filter(_.deletionVector != null).toSeq /** Lists the Deletion Vectors files of a table. */ def listDeletionVectors(log: DeltaLog): Seq[File] = { val dir = new File(log.dataPath.toUri.getPath) dir.listFiles().filter(_.getName.startsWith( DeletionVectorDescriptor.DELETION_VECTOR_FILE_NAME_CORE)) } /** Helper to check that the Deletion Vectors of the provided file actions exist on disk. */ def assertDeletionVectorsExist(log: DeltaLog, filesWithDVs: Seq[AddFile]): Unit = { val tablePath = new Path(log.dataPath.toUri.getPath) for (file <- filesWithDVs) { val dv = file.deletionVector assert(dv != null) assert(dv.isOnDisk && !dv.isInline) assert(dv.offset.isDefined) // Check that DV exists. val dvPath = dv.absolutePath(tablePath) assert(new File(dvPath.toString).exists(), s"DV not found $dvPath") // Check that cardinality is correct. val bitmap = newDVStore.read(dvPath, dv.offset.get, dv.sizeInBytes) assert(dv.cardinality === bitmap.cardinality) } } /** Enable persistent deletion vectors in new Delta tables. */ def enableDeletionVectorsInNewTables(conf: RuntimeConfig): Unit = conf.set(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey, "true") /** Enable persistent Deletion Vectors in a Delta table with table path. */ def enableDeletionVectorsInTable(tablePath: Path, enable: Boolean): Unit = enableDeletionVectorsInTable(tableName = s"delta.`$tablePath`", enable) /** Enable persistent Deletion Vectors in a Delta table with table name. */ def enableDeletionVectorsInTable(tableName: String, enable: Boolean): Unit = spark.sql( s"""ALTER TABLE $tableName |SET TBLPROPERTIES ('${DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key}' = '$enable') |""".stripMargin) /** Enable persistent Deletion Vectors in a Delta table. */ def enableDeletionVectorsInTable(deltaLog: DeltaLog, enable: Boolean = true): Unit = enableDeletionVectorsInTable(deltaLog.dataPath, enable) /** Enable persistent deletion vectors in new tables and DELETE DML commands. */ def enableDeletionVectors(conf: RuntimeConfig): Unit = { enableDeletionVectorsInNewTables(conf) conf.set(DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS.key, "true") } // ======== HELPER METHODS TO WRITE DVs ========== /** Helper method to remove the specified rows in the given file using DVs */ protected def removeRowsFromFileUsingDV( log: DeltaLog, addFile: AddFile, rowIds: Seq[Long]): Seq[Action] = { val dv = RoaringBitmapArray(rowIds: _*) writeFileWithDV(log, addFile, dv) } /** Utility method to remove a ratio of rows from the given file */ protected def deleteRows( log: DeltaLog, file: AddFile, approxPhyRows: Long, ratioOfRowsToDelete: Double): Unit = { val numRowsToDelete = Math.ceil(ratioOfRowsToDelete * file.numPhysicalRecords.getOrElse(approxPhyRows)).toInt removeRowsFromFile(log, file, Seq.range(0, numRowsToDelete)) } /** Utility method to remove the given rows from the given file using DVs */ protected def removeRowsFromFile( log: DeltaLog, addFile: AddFile, rowIndexesToRemove: Seq[Long]): Unit = { val txn = log.startTransaction() val actions = removeRowsFromFileUsingDV(log, addFile, rowIndexesToRemove) txn.commit(actions, Truncate()) } protected def getFileActionsInLastVersion(log: DeltaLog): (Seq[AddFile], Seq[RemoveFile]) = { val version = log.update().version val allFiles = log.getChanges(version).toSeq.head._2 val add = allFiles.collect { case a: AddFile => a } val remove = allFiles.collect { case r: RemoveFile => r } (add, remove) } protected def serializeRoaringBitmapArrayWithDefaultFormat( dv: RoaringBitmapArray): Array[Byte] = { val serializationFormat = RoaringBitmapArrayFormat.Portable dv.serializeAsByteArray(serializationFormat) } /** * Produce a new [[AddFile]] that will store `dv` in the log using default settings for choosing * inline or on-disk storage. * * Also returns the corresponding [[RemoveFile]] action for `currentFile`. * * TODO: Always on-disk for now. Inline support comes later. */ protected def writeFileWithDV( log: DeltaLog, currentFile: AddFile, dv: RoaringBitmapArray): Seq[Action] = { writeFileWithDVOnDisk(log, currentFile, dv) } /** Name of the partition column used by [[createTestDF()]]. */ val PARTITION_COL = "partitionColumn" def createTestDF( start: Long, end: Long, numFiles: Int, partitionColumn: Option[Int] = None): DataFrame = { val df = spark.range(start, end, 1, numFiles).withColumn("v", col("id")) if (partitionColumn.isEmpty) { df } else { df.withColumn(PARTITION_COL, lit(partitionColumn.get)) } } /** * Produce a new [[AddFile]] that will reference the `dv` in the log while storing it on-disk. * * Also returns the corresponding [[RemoveFile]] action for `currentFile`. */ protected def writeFileWithDVOnDisk( log: DeltaLog, currentFile: AddFile, dv: RoaringBitmapArray): Seq[Action] = writeFilesWithDVsOnDisk(log, Seq((currentFile, dv))) protected def withDVWriter[T]( log: DeltaLog, dvFileID: UUID)(fn: DeletionVectorStore.Writer => T): T = { val dvStore = newDVStore // scalastyle:off deltahadoopconfiguration val conf = spark.sessionState.newHadoopConf() // scalastyle:on deltahadoopconfiguration val tableWithFS = PathWithFileSystem.withConf(log.dataPath, conf) val dvPath = DeletionVectorStore.assembleDeletionVectorPathWithFileSystem(tableWithFS, dvFileID) val writer = dvStore.createWriter(dvPath) try { fn(writer) } finally { writer.close() } } /** * Produce new [[AddFile]] actions that will reference associated DVs in the log while storing * all DVs in the same file on-disk. * * Also returns the corresponding [[RemoveFile]] actions for the original file entries. */ protected def writeFilesWithDVsOnDisk( log: DeltaLog, filesWithDVs: Seq[(AddFile, RoaringBitmapArray)]): Seq[Action] = { val dvFileId = UUID.randomUUID() withDVWriter(log, dvFileId) { writer => filesWithDVs.flatMap { case (currentFile, dv) => val range = writer.write(serializeRoaringBitmapArrayWithDefaultFormat(dv)) val dvData = DeletionVectorDescriptor.onDiskWithRelativePath( id = dvFileId, sizeInBytes = range.length, cardinality = dv.cardinality, offset = Some(range.offset)) val (add, remove) = currentFile.removeRows( dvData, updateStats = true ) Seq(add, remove) } } } /** * Removes the `numRowsToRemovePerFile` from each file via DV. * Returns the total number of rows removed. */ protected def removeRowsFromAllFilesInLog( log: DeltaLog, numRowsToRemovePerFile: Long): Long = { var numFiles: Option[Int] = None // This is needed to make the manual commit work correctly, since we are not actually // running a command that produces metrics. withSQLConf(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "false") { val txn = log.startTransaction() val allAddFiles = txn.snapshot.allFiles.collect() numFiles = Some(allAddFiles.length) val bitmap = RoaringBitmapArray(0L until numRowsToRemovePerFile: _*) val actions = allAddFiles.flatMap { file => if (file.numPhysicalRecords.isDefined) { // Only when stats are enabled. Can't check when stats are disabled assert(file.numPhysicalRecords.get > numRowsToRemovePerFile) } writeFileWithDV(log, file, bitmap) } txn.commit(actions, DeltaOperations.Delete(predicate = Seq.empty)) } numFiles.get * numRowsToRemovePerFile } def newDVStore(): DeletionVectorStore = { // scalastyle:off deltahadoopconfiguration DeletionVectorStore.createInstance(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration } /** * Updates an [[AddFile]] with a [[DeletionVectorDescriptor]]. */ protected def updateFileDV( addFile: AddFile, dvDescriptor: DeletionVectorDescriptor): (AddFile, RemoveFile) = { addFile.removeRows( dvDescriptor, updateStats = true ) } /** Delete the DV file in the given [[AddFile]]. Assumes the [[AddFile]] has a valid DV. */ protected def deleteDVFile(tablePath: String, addFile: AddFile): Unit = { assert(addFile.deletionVector != null) val dvPath = addFile.deletionVector.absolutePath(new Path(tablePath)) FileUtils.delete(new File(dvPath.toString)) } /** * Creates a [[DeletionVectorDescriptor]] from an [[RoaringBitmapArray]] */ protected def writeDV( log: DeltaLog, bitmapArray: RoaringBitmapArray): DeletionVectorDescriptor = { val dvFileId = UUID.randomUUID() withDVWriter(log, dvFileId) { writer => val range = writer.write(serializeRoaringBitmapArrayWithDefaultFormat(bitmapArray)) DeletionVectorDescriptor.onDiskWithRelativePath( id = dvFileId, sizeInBytes = range.length, cardinality = bitmapArray.cardinality, offset = Some(range.offset)) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaAllFilesInCrcSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.util.TimeZone import java.util.concurrent.TimeUnit import scala.concurrent.duration.Duration import com.databricks.spark.util.{Log4jUsageLogger, UsageRecord} import org.apache.spark.sql.delta.DeltaTestUtils.{collectUsageLogs, BOOLEAN_DOMAIN} import org.apache.spark.sql.delta.concurrency._ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import org.apache.hadoop.fs.Path import org.apache.spark.SparkException import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.functions.col import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.ThreadUtils class DeltaAllFilesInCrcSuite extends QueryTest with SharedSparkSession with TransactionExecutionTestMixin with DeltaSQLCommandTest with PhaseLockingTestMixin { protected override def sparkConf = super.sparkConf .set(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key, "true") .set(DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_ENABLED.key, "true") // Set the threshold to a very high number so that this test suite continues to use all files // from CRC. .set(DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_THRESHOLD_FILES.key, "10000") // needed for DELTA_ALL_FILES_IN_CRC_ENABLED .set(DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key, "true") .set(DeltaSQLConf.USE_PROTOCOL_AND_METADATA_FROM_CHECKSUM_ENABLED.key, "true") // Turn on verification by default in the tests .set(DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_VERIFICATION_MODE_ENABLED.key, "true") // Turn off force verification for non-UTC timezones by default in the tests .set(DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_FORCE_VERIFICATION_MODE_FOR_NON_UTC_ENABLED.key, "false") private def setTimeZone(timeZone: String): Unit = { spark.sql(s"SET spark.sql.session.timeZone = $timeZone") TimeZone.setDefault(TimeZone.getTimeZone(timeZone)) } /** Filter usage records for specific `opType` */ protected def filterUsageRecords( usageRecords: Seq[UsageRecord], opType: String): Seq[UsageRecord] = { usageRecords.filter { r => r.tags.get("opType").contains(opType) || r.opType.map(_.typeName).contains(opType) } } /** Deletes all delta/crc/checkpoint files later that given `version` for the delta table */ private def deleteDeltaFilesLaterThanVersion(deltaLog: DeltaLog, version: Long): Unit = { val fs = deltaLog.logPath.getFileSystem(deltaLog.newDeltaHadoopConf()) deltaLog.listFrom(version + 1).filter { f => FileNames.isDeltaFile(f) || FileNames.isChecksumFile(f) || FileNames.isCheckpointFile(f) }.foreach(f => fs.delete(f.getPath, true)) DeltaLog.clearCache() assert(DeltaLog.forTable(spark, deltaLog.dataPath).update().version === version) } test("allFiles are written to CRC and different threshold configs are respected") { withTempDir { dir => val path = dir.getCanonicalPath // Helper method to perform a commit with 10 AddFile actions to the table. def writeToTable( version: Long, newFilesToWrite: Int, expectedFilesInCRCOption: Option[Long]): Unit = { spark.range(start = 1, end = 100, step = 1, numPartitions = newFilesToWrite) .toDF("c1") .withColumn("c2", col("c1")).withColumn("c3", col("c1")) .write.format("delta").mode("append").save(path) assert(deltaLog.update().version === version) assert(deltaLog.snapshot.checksumOpt.get.allFiles.map(_.size) === expectedFilesInCRCOption) assert(deltaLog.readChecksum(version).get.allFiles.map(_.size) === expectedFilesInCRCOption) } def deltaLog: DeltaLog = DeltaLog.forTable(spark, path) withSQLConf( DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_THRESHOLD_FILES.key -> "55", DeltaSQLConf.DELTA_AUTO_COMPACT_ENABLED.key -> "false") { // Commit-0: Add 10 new files to table. Total files (10) is less than threshold. writeToTable(version = 0, newFilesToWrite = 10, expectedFilesInCRCOption = Some(10)) withSQLConf(DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_ENABLED.key -> "false") { // Commit-1: Add 10 more files to table. Total files (20) is less than threshold. // Still these won't be written to CRC as the conf is explicitly disabled. writeToTable(version = 1, newFilesToWrite = 10, expectedFilesInCRCOption = None) } // Commit-2: Add 20 more files to table. Total files (40) is less than threshold. writeToTable(version = 2, newFilesToWrite = 20, expectedFilesInCRCOption = Some(40)) // Commit-3: Add 13 more files to table. Total files (53) is less than threshold. writeToTable(version = 3, newFilesToWrite = 13, expectedFilesInCRCOption = Some(53)) // Commit-4: Add 7 more files to table. Total files (60) is greater than the threshold (55). // So files won't be persisted to CRC. writeToTable(version = 4, newFilesToWrite = 7, expectedFilesInCRCOption = None) // Commit-5: Delete all rows except with value=1. After this step, very few files will // remain in table, still they won't be persisted to CRC as previous version had more than // 55 files. We write files to CRC if both previous commit and this commit has files <= 55. sql(s"DELETE FROM delta.`$path` WHERE c1 != 1").collect() assert(deltaLog.update().version === 5) assert(deltaLog.snapshot.checksumOpt.get.allFiles === None) val fileCountAfterDeleteCommand = deltaLog.snapshot.checksumOpt.get.numFiles assert(fileCountAfterDeleteCommand < 55) // Commit-6: Commit 1 new file again. Now previous-version also had < 55 files. This version // also has < 55 files. writeToTable(version = 6, newFilesToWrite = 1, expectedFilesInCRCOption = Some(fileCountAfterDeleteCommand + 1)) withSQLConf(DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_THRESHOLD_INDEXED_COLS.key -> "2") { // Table collects stats on 3 cols (col1/col2/col3) which is more than threshold. // So optimization should be disabled by default. writeToTable(version = 7, newFilesToWrite = 1, expectedFilesInCRCOption = None) } writeToTable(version = 8, newFilesToWrite = 1, expectedFilesInCRCOption = Some(fileCountAfterDeleteCommand + 3)) // Commit-7: Delete all rows from table sql(s"DELETE FROM delta.`$path` WHERE c1 >= 0").collect() assert(deltaLog.update().version === 9) assert(deltaLog.snapshot.checksumOpt.get.allFiles === Some(Seq())) } } } test("test all-files-in-crc verification failure also triggers and logs" + " incremental-commit verification result") { withTempDir { tempDir => withSQLConf( DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_THRESHOLD_FILES.key -> "100", // Disable incremental commit force verifications in UTs - to mimic prod behavior DeltaSQLConf.INCREMENTAL_COMMIT_FORCE_VERIFY_IN_TESTS.key -> "false", // Enable all-files-in-crc verification mode DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_VERIFICATION_MODE_ENABLED.key -> "true", DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false") { val df = spark.range(2).coalesce(1).toDF() df.write.format("delta").save(tempDir.toString()) val deltaLog = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) assert(deltaLog.update().allFiles.collect().map(_.numPhysicalRecords).forall(_.isEmpty)) val records = Log4jUsageLogger.track { val executor = ThreadUtils.newDaemonSingleThreadExecutor(threadName = "executor-txn-A") try { val query = s"DELETE from delta.`${tempDir.getAbsolutePath}` WHERE id >= 0" val (observer, future) = runQueryWithObserver(name = "A", executor, query) observer.phases.initialPhase.entryBarrier.unblock() observer.phases.preparePhase.entryBarrier.unblock() // Make sure that delete query has run the actual computation and has reached // the 'prepare commit' phase. i.e. it just wants to commit. busyWaitFor(observer.phases.preparePhase.hasLeft, timeout) // Now delete and recreate the complete table. deltaLog.logPath.getFileSystem(deltaLog.newDeltaHadoopConf()) .delete(deltaLog.dataPath, true) spark.range(3, 4).coalesce(1).toDF().write.format("delta").save(tempDir.toString()) // Allow the delete query to commit. unblockCommit(observer) waitForCommit(observer) // Query will fail due to incremental-state-reconstruction validation failure. // Note that this failure happens only in test. In prod, this would have just logged // the incremental-state-reconstruction failure and query would have passed. val ex = intercept[SparkException] { ThreadUtils.awaitResult(future, Duration.Inf) } val message = ex.getMessage + "\n" + ex.getCause.getMessage assert(message.contains("Incremental state reconstruction validation failed")) } finally { executor.shutdownNow() executor.awaitTermination(timeout.toMillis, TimeUnit.MILLISECONDS) } } // We will see all files in CRC verification failure. // This will trigger the incremental commit verification which will fail. assert(filterUsageRecords(records, "delta.assertions.mismatchedAction").size === 1) val allFilesInCrcValidationFailureRecords = filterUsageRecords(records, "delta.allFilesInCrc.checksumMismatch.differentAllFiles") assert(allFilesInCrcValidationFailureRecords.size === 1) val eventData = JsonUtils.fromJson[Map[String, String]](allFilesInCrcValidationFailureRecords.head.blob) assert(eventData("version").toLong === 1L) assert(eventData("mismatchWithStatsOnly").toBoolean === false) val expectedFilesCountFromCrc = 1L assert(eventData("filesCountFromCrc").toLong === expectedFilesCountFromCrc) assert(eventData("filesCountFromStateReconstruction").toLong === expectedFilesCountFromCrc + 1) assert(eventData("incrementalCommitCrcValidationPassed").toBoolean === false) assert(eventData("errorForIncrementalCommitCrcValidation").contains( "The metadata of your Delta table could not be recovered")) } } } test("schema changing metadata operations should disable putting AddFile" + " actions in crc but other metadata operations should not") { withTempDir { dir => val path = dir.getCanonicalPath spark.range(1, 5).toDF("c1").withColumn("c2", col("c1")) .write.format("delta").mode("append").save(path) def deltaLog: DeltaLog = DeltaLog.forTable(spark, path) assert(deltaLog.update().checksumOpt.get.allFiles.nonEmpty) sql(s"ALTER TABLE delta.`$dir` CHANGE COLUMN c2 FIRST") assert(deltaLog.update().checksumOpt.get.allFiles.isEmpty) sql(s"ALTER TABLE delta.`$dir` SET TBLPROPERTIES ('a' = 'b')") assert(deltaLog.update().checksumOpt.get.allFiles.nonEmpty) } } test("schema changing metadata operations on empty tables should not disable putting " + "AddFile actions in crc") { withTempDir { dir => val path = dir.getCanonicalPath def deltaLog: DeltaLog = DeltaLog.forTable(spark, path) def assertNoStateReconstructionTriggeredWhenPerfPackEnabled(f: => Unit): Unit = { val oldSnapshot = deltaLog.update() f val newSnapshot = deltaLog.update() } withSQLConf( // Disable test flags to make the behaviors verified in this test close to prod DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_VERIFICATION_MODE_ENABLED.key -> "false", DeltaSQLConf.INCREMENTAL_COMMIT_FORCE_VERIFY_IN_TESTS.key -> "false" ) { assertNoStateReconstructionTriggeredWhenPerfPackEnabled { // Create a table with an empty schema so that the next write will change the schema sql(s"CREATE TABLE delta.`$path` USING delta LOCATION '$path'") } assert(deltaLog.update().checksumOpt.get.allFiles == Option(Nil)) assertNoStateReconstructionTriggeredWhenPerfPackEnabled { // Write zero files but update the table schema spark.range(1, 5).filter("false").write.format("delta") .option("mergeSchema", "true").mode("append").save(path) } // Make sure writing zero files still make a Delta commit so that this test is valid assert(deltaLog.update().version == 1) assert(deltaLog.update().checksumOpt.get.allFiles == Option(Nil)) assertNoStateReconstructionTriggeredWhenPerfPackEnabled { // Write some files to the table spark.range(1, 5).write.format("delta").mode("append").save(path) } assert(deltaLog.update().checksumOpt.get.allFiles.nonEmpty) assert(deltaLog.update().checksumOpt.get.allFiles.get.size > 0) } } } private def withCrcVerificationEnabled(testCode: => Unit): Unit = { withSQLConf(DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_VERIFICATION_MODE_ENABLED.key -> "true") { testCode } } private def withCrcVerificationDisabled(testCode: => Unit): Unit = { withSQLConf(DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_VERIFICATION_MODE_ENABLED.key -> "false") { testCode } } private def write(deltaLog: DeltaLog, numFiles: Int, expectedFilesInCrc: Option[Int]): Unit = { spark .range(1, 100, 1, numPartitions = numFiles) .write .format("delta") .mode("append") .save(deltaLog.dataPath.toString) assert(deltaLog.snapshot.checksumOpt.get.allFiles.map(_.size) === expectedFilesInCrc) } private def corruptCRCNumFiles(deltaLog: DeltaLog, version: Int): Unit = { val crc = deltaLog.readChecksum(version).get assert(crc.allFiles.nonEmpty) val filesInCrc = crc.allFiles.get // Corrupt the CRC val corruptedCrc = crc.copy(allFiles = Some(filesInCrc.dropRight(1)), numFiles = crc.numFiles - 1) val checksumFilePath = FileNames.checksumFile(deltaLog.logPath, version) deltaLog.store.write( checksumFilePath, actions = Seq(JsonUtils.toJson(corruptedCrc)).toIterator, overwrite = true, hadoopConf = deltaLog.newDeltaHadoopConf()) } private def corruptCRCAddFilesModificationTime(deltaLog: DeltaLog, version: Int): Unit = { val crc = deltaLog.readChecksum(version).get assert(crc.allFiles.nonEmpty) val filesInCrc = crc.allFiles.get // Corrupt the CRC val corruptedCrc = crc.copy(allFiles = Some(filesInCrc.map(_.copy(modificationTime = 23)))) val checksumFilePath = FileNames.checksumFile(deltaLog.logPath, version) deltaLog.store.write( checksumFilePath, actions = Seq(JsonUtils.toJson(corruptedCrc)).toIterator, overwrite = true, hadoopConf = deltaLog.newDeltaHadoopConf()) } private def checkIfCrcModificationTimeCorrupted( deltaLog: DeltaLog, expectCorrupted: Boolean): Unit = { val crc = deltaLog.readChecksum(deltaLog.update().version).get assert(crc.allFiles.nonEmpty) assert(crc.allFiles.get.count(_.modificationTime == 23L) > 0 === expectCorrupted) } test("allFilesInCRC verification with flag manipulation for UTC timezone") { withSQLConf( DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_FORCE_VERIFICATION_MODE_FOR_NON_UTC_ENABLED.key -> "true") { setTimeZone("UTC") withTempDir { dir => var deltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) // Commit 0: Initial write with verification enabled withCrcVerificationEnabled { write(deltaLog, numFiles = 10, expectedFilesInCrc = Some(10)) } // Corrupt the CRC at Version 0 corruptCRCAddFilesModificationTime(deltaLog, version = 0) DeltaLog.clearCache() deltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) // Commit 1: Write with verification flag off and Verify Incremental CRC at version 1 is // also corrupted withCrcVerificationDisabled { write(deltaLog, numFiles = 10, expectedFilesInCrc = Some(20)) checkIfCrcModificationTimeCorrupted(deltaLog, expectCorrupted = true) } // Commit 2: Write with verification flag on and it should fail because the AddFiles from // base CRC at Version 1 are incorrect. withCrcVerificationEnabled { val usageRecords = collectUsageLogs("delta.allFilesInCrc.checksumMismatch.differentAllFiles") { intercept[IllegalStateException] { write(deltaLog, numFiles = 10, expectedFilesInCrc = None) } } assert(usageRecords.size === 1) } // Commit 3: Write with verification flag on and it should pass since the base CRC is not // corrupted anymore. withCrcVerificationEnabled { write(deltaLog, numFiles = 10, expectedFilesInCrc = Some(40)) checkIfCrcModificationTimeCorrupted(deltaLog, expectCorrupted = false) } } } } test("allFilesInCRC verification with flag manipulation for non-UTC timezone") { withSQLConf( DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_FORCE_VERIFICATION_MODE_FOR_NON_UTC_ENABLED.key -> "true") { setTimeZone("America/Los_Angeles") withTempDir { dir => var deltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) // Commit 0: Initial write with verification enabled withCrcVerificationEnabled { write(deltaLog, numFiles = 10, expectedFilesInCrc = Some(10)) } // Corrupt the CRC at Version 0 corruptCRCAddFilesModificationTime(deltaLog, version = 0) DeltaLog.clearCache() deltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) // Commit 1: Write with verification flag off and Verify Incremental CRC is still validated // because timezone is non-UTC. withCrcVerificationDisabled { val usageRecords = collectUsageLogs("delta.allFilesInCrc.checksumMismatch.differentAllFiles") { intercept[IllegalStateException] { write(deltaLog, numFiles = 10, expectedFilesInCrc = None) } } assert(usageRecords.size === 1) } // Commit 2: Write with verification flag on and it should pass since the base CRC is not // corrupted anymore. withCrcVerificationEnabled { write(deltaLog, numFiles = 10, expectedFilesInCrc = Some(30)) checkIfCrcModificationTimeCorrupted(deltaLog, expectCorrupted = false) } } } } test("Verify aggregate stats are matched even when allFilesInCrc " + "verification is disabled") { setTimeZone("UTC") withSQLConf(DeltaSQLConf.INCREMENTAL_COMMIT_FORCE_VERIFY_IN_TESTS.key -> "false") { withCrcVerificationDisabled { withTempDir { dir => var deltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) // Commit 0 write(deltaLog, numFiles = 10, expectedFilesInCrc = Some(10)) // Corrupt the CRC at Version 0 corruptCRCNumFiles(deltaLog, version = 0) DeltaLog.clearCache() deltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) // Commit 1: Verify aggregate stats are matched even when verification is off val usageRecords = collectUsageLogs("delta.allFilesInCrc.checksumMismatch.aggregated") { intercept[IllegalStateException] { write(deltaLog, numFiles = 10, expectedFilesInCrc = None) } } assert(usageRecords.size === 1) } } } } test("allFilesInCRC validation during checkpoint must be opposite of per-commit " + "validation") { withTempDir { dir => var deltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) withCrcVerificationDisabled { write(deltaLog, numFiles = 10, expectedFilesInCrc = Some(10)) write(deltaLog, numFiles = 10, expectedFilesInCrc = Some(20)) corruptCRCAddFilesModificationTime(deltaLog, version = 1) DeltaLog.clearCache() deltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) deltaLog.update() // Checkpoint should validate Checksum even when per-commit verification is disabled. val usageRecords = collectUsageLogs("delta.allFilesInCrc.checksumMismatch.differentAllFiles") { intercept[IllegalStateException] { deltaLog.checkpoint() } } assert(usageRecords.size === 1) assert(usageRecords.head.blob.contains("\"context\":" + "\"triggeredFromCheckpoint\""), usageRecords.head) write(deltaLog, numFiles = 10, expectedFilesInCrc = Some(30)) } // Checkpoint should not validate Checksum when per-commit verification is enabled. withCrcVerificationEnabled { val usageRecords = collectUsageLogs("delta.allFilesInCrc.checksumMismatch.differentAllFiles") { deltaLog.checkpoint() } assert(usageRecords.isEmpty) } } } test("allFilesInCrcVerificationForceEnabled works as expected") { // Test with the non-UTC force verification conf enabled. withSQLConf( DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_FORCE_VERIFICATION_MODE_FOR_NON_UTC_ENABLED.key -> "true") { setTimeZone("UTC") assert(!Snapshot.allFilesInCrcVerificationForceEnabled(spark)) setTimeZone("America/Los_Angeles") assert(Snapshot.allFilesInCrcVerificationForceEnabled(spark)) } // Test with the non-UTC force verification conf disabled. withSQLConf( DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_FORCE_VERIFICATION_MODE_FOR_NON_UTC_ENABLED.key -> "false") { assert(!Snapshot.allFilesInCrcVerificationForceEnabled(spark)) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaAlterTableReplaceTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.functions.{array, col, map, struct} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{ArrayType, MapType, StructType} trait DeltaAlterTableReplaceTests extends DeltaAlterTableTestBase { import testImplicits._ ddlTest("REPLACE COLUMNS - add a comment") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("s", struct("v1", "v2")) .withColumn("a", array("s")) .withColumn("m", map(col("s"), col("s"))) withDeltaTable(df) { tableName => sql(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int COMMENT 'a comment for v1', | v2 string COMMENT 'a comment for v2', | s STRUCT< | v1:int COMMENT 'a comment for s.v1', | v2:string COMMENT 'a comment for s.v2'> COMMENT 'a comment for s', | a ARRAY> COMMENT 'a comment for a', | m MAP, | STRUCT< | v1:int COMMENT 'a comment for m.value.v1', | v2:string COMMENT 'a comment for m.value.v2'>> COMMENT 'a comment for m' |)""".stripMargin) val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) val expectedSchema = new StructType() .add("v1", "integer", true, "a comment for v1") .add("v2", "string", true, "a comment for v2") .add("s", new StructType() .add("v1", "integer", true, "a comment for s.v1") .add("v2", "string", true, "a comment for s.v2"), true, "a comment for s") .add("a", ArrayType(new StructType() .add("v1", "integer", true, "a comment for a.v1") .add("v2", "string", true, "a comment for a.v2")), true, "a comment for a") .add("m", MapType( new StructType() .add("v1", "integer", true, "a comment for m.key.v1") .add("v2", "string", true, "a comment for m.key.v2"), new StructType() .add("v1", "integer", true, "a comment for m.value.v1") .add("v2", "string", true, "a comment for m.value.v2")), true, "a comment for m") assertEqual(snapshot.schema, expectedSchema) implicit val ordering = Ordering.by[ (Int, String, (Int, String), Seq[(Int, String)], Map[(Int, String), (Int, String)]), Int] { case (v1, _, _, _, _) => v1 } checkDatasetUnorderly( spark.table(tableName) .as[(Int, String, (Int, String), Seq[(Int, String)], Map[(Int, String), (Int, String)])], (1, "a", (1, "a"), Seq((1, "a")), Map((1, "a") -> ((1, "a")))), (2, "b", (2, "b"), Seq((2, "b")), Map((2, "b") -> ((2, "b"))))) // REPLACE COLUMNS doesn't remove metadata. sql(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin) assertEqual(deltaLog.snapshot.schema, expectedSchema) } } ddlTest("REPLACE COLUMNS - reorder") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("s", struct("v1", "v2")) .withColumn("a", array("s")) .withColumn("m", map(col("s"), col("s"))) withDeltaTable(df) { tableName => sql(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | m MAP, STRUCT>, | v2 string, | a ARRAY>, | v1 int, | s STRUCT |)""".stripMargin) val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("m", MapType( new StructType().add("v2", "string").add("v1", "integer"), new StructType().add("v2", "string").add("v1", "integer"))) .add("v2", "string") .add("a", ArrayType(new StructType().add("v2", "string").add("v1", "integer"))) .add("v1", "integer") .add("s", new StructType().add("v2", "string").add("v1", "integer"))) implicit val ordering = Ordering.by[ (Map[(String, Int), (String, Int)], String, Seq[(String, Int)], Int, (String, Int)), Int] { case (_, _, _, v1, _) => v1 } checkDatasetUnorderly( spark.table(tableName) .as[(Map[(String, Int), (String, Int)], String, Seq[(String, Int)], Int, (String, Int))], (Map(("a", 1) -> (("a", 1))), "a", Seq(("a", 1)), 1, ("a", 1)), (Map(("b", 2) -> (("b", 2))), "b", Seq(("b", 2)), 2, ("b", 2))) } } ddlTest("REPLACE COLUMNS - add columns") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("s", struct("v1", "v2")) .withColumn("a", array("s")) .withColumn("m", map(col("s"), col("s"))) withDeltaTable(df) { tableName => sql(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | v3 long, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin) val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer") .add("v2", "string") .add("v3", "long") .add("s", new StructType() .add("v1", "integer").add("v2", "string").add("v3", "long")) .add("a", ArrayType(new StructType() .add("v1", "integer").add("v2", "string").add("v3", "long"))) .add("m", MapType( new StructType().add("v1", "integer").add("v2", "string").add("v3", "long"), new StructType().add("v1", "integer").add("v2", "string").add("v3", "long")))) implicit val ordering = Ordering.by[ (Int, String, Option[Long], (Int, String, Option[Long]), Seq[(Int, String, Option[Long])], Map[(Int, String, Option[Long]), (Int, String, Option[Long])]), Int] { case (v1, _, _, _, _, _) => v1 } checkDatasetUnorderly( spark.table(tableName).as[ (Int, String, Option[Long], (Int, String, Option[Long]), Seq[(Int, String, Option[Long])], Map[(Int, String, Option[Long]), (Int, String, Option[Long])])], (1, "a", None, (1, "a", None), Seq((1, "a", None)), Map((1, "a", Option.empty[Long]) -> ((1, "a", None)))), (2, "b", None, (2, "b", None), Seq((2, "b", None)), Map((2, "b", Option.empty[Long]) -> ((2, "b", None))))) } } ddlTest("REPLACE COLUMNS - special column names") { val df = Seq((1, "a"), (2, "b")).toDF("x.x", "y.y") .withColumn("s.s", struct("`x.x`", "`y.y`")) .withColumn("a.a", array("`s.s`")) .withColumn("m.m", map(col("`s.s`"), col("`s.s`"))) withDeltaTable(df) { tableName => sql(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | `m.m` MAP, STRUCT<`y.y`:string, `x.x`:int>>, | `y.y` string, | `a.a` ARRAY>, | `x.x` int, | `s.s` STRUCT<`y.y`:string, `x.x`:int> |)""".stripMargin) val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("m.m", MapType( new StructType().add("y.y", "string").add("x.x", "integer"), new StructType().add("y.y", "string").add("x.x", "integer"))) .add("y.y", "string") .add("a.a", ArrayType(new StructType().add("y.y", "string").add("x.x", "integer"))) .add("x.x", "integer") .add("s.s", new StructType().add("y.y", "string").add("x.x", "integer"))) } } ddlTest("REPLACE COLUMNS - drop column") { // Column Mapping allows columns to be dropped def checkReplace( text: String, tableName: String, columnDropped: Seq[String], messages: String*): Unit = { if (columnMappingEnabled) { spark.sql(text) val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) val field = snapshot.schema.findNestedField(columnDropped, includeCollections = true) assert(field.isEmpty, "Column was not deleted") } else { assertNotSupported(text, messages: _*) } } val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("s", struct("v1", "v2")) .withColumn("a", array("s")) .withColumn("m", map(col("s"), col("s"))) withDeltaTable(df) { tableName => // trying to drop v1 of each struct, but it should fail because dropping column is // not supported unless column mapping is enabled checkReplace( s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, tableName, Seq("v1"), "dropping column(s)", "v1") // s.v1 checkReplace( s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, tableName, Seq("s", "v1"), "dropping column(s)", "v1", "from s") // a.v1 checkReplace( s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, tableName, Seq("a", "element", "v1"), "dropping column(s)", "v1", "from a") // m.key.v1 checkReplace( s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, tableName, Seq("m", "key", "v1"), "dropping column(s)", "v1", "from m.key") // m.value.v1 checkReplace( s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, tableName, Seq("m", "value", "v1"), "dropping column(s)", "v1", "from m.value") } } ddlTest("REPLACE COLUMNS - incompatible data type") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("s", struct("v1", "v2")) .withColumn("a", array("s")) .withColumn("m", map(col("s"), col("s"))) withDeltaTable(df) { tableName => // trying to change the data type of v1 of each struct to long, but it should fail because // changing data type is not supported. assertNotSupported(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 long, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, "changing data type", "v1", "from IntegerType to LongType") // s.v1 assertNotSupported(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, "changing data type", "s.v1", "from IntegerType to LongType") // a.element.v1 assertNotSupported(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, "changing data type", "a.element.v1", "from IntegerType to LongType") // m.key.v1 assertNotSupported(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, "changing data type", "m.key.v1", "from IntegerType to LongType") // m.value.v1 assertNotSupported(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, "changing data type", "m.value.v1", "from IntegerType to LongType") } } ddlTest("REPLACE COLUMNS - case insensitive") { withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("s", struct("v1", "v2")) .withColumn("a", array("s")) .withColumn("m", map(col("s"), col("s"))) withDeltaTable(df) { tableName => val (deltaLog, _) = getDeltaLogWithSnapshot(tableName) def checkSchema(command: String): Unit = { sql(command) assertEqual(deltaLog.update().schema, new StructType() .add("v1", "integer") .add("v2", "string") .add("s", new StructType().add("v1", "integer").add("v2", "string")) .add("a", ArrayType(new StructType().add("v1", "integer").add("v2", "string"))) .add("m", MapType( new StructType().add("v1", "integer").add("v2", "string"), new StructType().add("v1", "integer").add("v2", "string")))) } // trying to use V1 instead of v1 of each struct. checkSchema(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | V1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin) // s.V1 checkSchema(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin) // a.V1 checkSchema(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin) // m.key.V1 checkSchema(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin) // m.value.V1 checkSchema(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin) } } } ddlTest("REPLACE COLUMNS - case sensitive") { withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("s", struct("v1", "v2")) .withColumn("a", array("s")) .withColumn("m", map(col("s"), col("s"))) withDeltaTable(df) { tableName => // trying to use V1 instead of v1 of each struct, but it should fail because case sensitive. assertNotSupported(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | V1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, "ambiguous", "v1") // s.V1 assertNotSupported(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, "ambiguous", "data type of s") // a.V1 assertNotSupported(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, "ambiguous", "data type of a.element") // m.key.V1 assertNotSupported(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, "ambiguous", "data type of m.key") // m.value.V1 assertNotSupported(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, "ambiguous", "data type of m.value") } } } ddlTest("REPLACE COLUMNS - duplicate") { withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("s", struct("v1", "v2")) .withColumn("a", array("s")) .withColumn("m", map(col("s"), col("s"))) withDeltaTable(df) { tableName => def assertDuplicate(command: String): Unit = { val ex = intercept[AnalysisException] { sql(command) } assert(ex.getMessage.contains("duplicate column(s)")) } // trying to add a V1 column, but it should fail because Delta doesn't allow columns // at the same level of nesting that differ only by case. assertDuplicate(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | V1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin) // s.V1 assertDuplicate(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin) // a.V1 assertDuplicate(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin) // m.key.V1 assertDuplicate(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin) // m.value.V1 assertDuplicate(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin) } } } test("REPLACE COLUMNS - loosen nullability with unenforced allowed") { withSQLConf(("spark.databricks.delta.constraints.allowUnenforcedNotNull.enabled", "true")) { val schema = """ | v1 int NOT NULL, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> """.stripMargin withDeltaTable(schema) { tableName => sql( s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin) val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer") .add("v2", "string") .add("s", new StructType() .add("v1", "integer").add("v2", "string")) .add("a", ArrayType(new StructType() .add("v1", "integer").add("v2", "string"))) .add("m", MapType( new StructType().add("v1", "integer").add("v2", "string"), new StructType().add("v1", "integer").add("v2", "string")))) } } } test("REPLACE COLUMNS - loosen nullability") { val schema = """ | v1 int NOT NULL, | v2 string, | s STRUCT, | a ARRAY> NOT NULL, | m MAP, STRUCT> NOT NULL """.stripMargin withDeltaTable(schema) { tableName => sql(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin) val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer") .add("v2", "string") .add("s", new StructType() .add("v1", "integer").add("v2", "string")) .add("a", ArrayType(new StructType() .add("v1", "integer").add("v2", "string"))) .add("m", MapType( new StructType().add("v1", "integer").add("v2", "string"), new StructType().add("v1", "integer").add("v2", "string")))) } } test("REPLACE COLUMNS - add not-null column") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("s", struct("v1", "v2")) .withColumn("a", array("s")) .withColumn("m", map(col("s"), col("s"))) withDeltaTable(df) { tableName => // trying to add not-null column, but it should fail because adding not-null column is // not supported. assertNotSupported(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | v3 long NOT NULL, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, "NOT NULL is not supported in Hive-style REPLACE COLUMNS") // s.v3 assertNotSupported(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, "adding non-nullable column", "s.v3") // a.element.v3 assertNotSupported(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, "adding non-nullable column", "a.element.v3") // m.key.v3 assertNotSupported(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, "adding non-nullable column", "m.key.v3") // m.value.v3 assertNotSupported(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, "adding non-nullable column", "m.value.v3") } } test("REPLACE COLUMNS - incompatible nullability") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("s", struct("v1", "v2")) .withColumn("a", array("s")) .withColumn("m", map(col("s"), col("s"))) withDeltaTable(df) { tableName => // trying to change the data type of v1 of each struct to not null, but it should fail because // tightening nullability is not supported. assertNotSupported(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int NOT NULL, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, "NOT NULL is not supported in Hive-style REPLACE COLUMNS") // s.v1 assertNotSupported(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, "tightening nullability", "s.v1") // a.element.v1 assertNotSupported(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, "tightening nullability", "a.element.v1") // m.key.v1 assertNotSupported(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, "tightening nullability", "m.key.v1") // m.value.v1 assertNotSupported(s""" |ALTER TABLE $tableName REPLACE COLUMNS ( | v1 int, | v2 string, | s STRUCT, | a ARRAY>, | m MAP, STRUCT> |)""".stripMargin, "tightening nullability", "m.value.v1") } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaAlterTableTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.io.File import org.apache.spark.sql.delta.DeltaConfigs.CHECKPOINT_INTERVAL import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.delta.schema.{DeltaInvariantViolationException, SchemaUtils} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaColumnMappingSelectedTestMixin, DeltaSQLCommandTest} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.hadoop.fs.Path import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.catalyst.util.CharVarcharUtils import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ import org.apache.spark.util.Utils trait DeltaAlterTableTestBase extends QueryTest with SharedSparkSession with DeltaColumnMappingTestUtils with DeltaTestUtilsForTempViews { protected def createTable(schema: String, tblProperties: Map[String, String]): String protected def createTable(df: DataFrame, partitionedBy: Seq[String]): String protected def dropTable(identifier: String): Unit protected def getDeltaLogWithSnapshot(identifier: String): (DeltaLog, Snapshot) final protected def withDeltaTable(schema: String)(f: String => Unit): Unit = { withDeltaTable(schema, Map.empty[String, String])(i => f(i)) } final protected def withDeltaTable( schema: String, tblProperties: Map[String, String])(f: String => Unit): Unit = { val identifier = createTable(schema, tblProperties) try { f(identifier) } finally { dropTable(identifier) } } final protected def withDeltaTable(df: DataFrame)(f: String => Unit): Unit = { withDeltaTable(df, Seq.empty[String])(i => f(i)) } final protected def withDeltaTable( df: DataFrame, partitionedBy: Seq[String])(f: String => Unit): Unit = { val identifier = createTable(df, partitionedBy) try { f(identifier) } finally { dropTable(identifier) } } protected def ddlTest(testName: String)(f: => Unit): Unit = { testQuietly(testName)(f) } protected def assertNotSupported(command: String, messages: String*): Unit = { val ex = intercept[Exception] { sql(command) }.getMessage assert(ex.contains("not supported") || ex.contains("Unsupported") || ex.contains("Cannot")) messages.foreach(msg => assert(ex.contains(msg))) } } trait DeltaAlterTableTests extends DeltaAlterTableTestBase { import testImplicits._ /////////////////////////////// // SET/UNSET TBLPROPERTIES /////////////////////////////// ddlTest("SET/UNSET TBLPROPERTIES - simple") { withDeltaTable("v1 int, v2 string") { tableName => sql(s""" |ALTER TABLE $tableName |SET TBLPROPERTIES ( | 'delta.logRetentionDuration' = '2 weeks', | 'delta.checkpointInterval' = '20', | 'key' = 'value' |)""".stripMargin) val (deltaLog, snapshot1) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot1.metadata.configuration, Map( "delta.logRetentionDuration" -> "2 weeks", "delta.checkpointInterval" -> "20", "key" -> "value")) assert(deltaLog.deltaRetentionMillis(snapshot1.metadata) == 2 * 7 * 24 * 60 * 60 * 1000) assert(deltaLog.checkpointInterval(snapshot1.metadata) == 20) sql(s"ALTER TABLE $tableName UNSET TBLPROPERTIES ('delta.checkpointInterval', 'key')") val snapshot2 = deltaLog.update() assertEqual(snapshot2.metadata.configuration, Map("delta.logRetentionDuration" -> "2 weeks")) assert(deltaLog.deltaRetentionMillis(snapshot2.metadata) == 2 * 7 * 24 * 60 * 60 * 1000) assert(deltaLog.checkpointInterval(snapshot2.metadata) == CHECKPOINT_INTERVAL.fromString(CHECKPOINT_INTERVAL.defaultValue)) } } testQuietlyWithTempView("negative case - not supported on temp views") { isSQLTempView => withDeltaTable("v1 int, v2 string") { tableName => createTempViewFromTable(tableName, isSQLTempView) val e = intercept[AnalysisException] { sql( """ |ALTER TABLE v |SET TBLPROPERTIES ( | 'delta.logRetentionDuration' = '2 weeks', | 'delta.checkpointInterval' = '20', | 'key' = 'value' |)""".stripMargin) } assert(e.getMessage.contains("expects a table. Please use ALTER VIEW instead.") || e.getMessage.contains("EXPECT_TABLE_NOT_VIEW.USE_ALTER_VIEW")) } } ddlTest("SET/UNSET TBLPROPERTIES - case insensitivity") { withDeltaTable("v1 int, v2 string") { tableName => sql(s""" |ALTER TABLE $tableName |SET TBLPROPERTIES ( | 'dEltA.lOgrEteNtiOndURaTion' = '1 weeks', | 'DelTa.ChEckPoiNtinTervAl' = '5', | 'key' = 'value1' |)""".stripMargin) val (deltaLog, snapshot1) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot1.metadata.configuration, Map( "delta.logRetentionDuration" -> "1 weeks", "delta.checkpointInterval" -> "5", "key" -> "value1")) assert(deltaLog.deltaRetentionMillis(snapshot1.metadata) == 1 * 7 * 24 * 60 * 60 * 1000) assert(deltaLog.checkpointInterval(snapshot1.metadata) == 5) sql(s""" |ALTER TABLE $tableName |SET TBLPROPERTIES ( | 'dEltA.lOgrEteNtiOndURaTion' = '2 weeks', | 'DelTa.ChEckPoiNtinTervAl' = '20', | 'kEy' = 'value2' |)""".stripMargin) val snapshot2 = deltaLog.update() assertEqual(snapshot2.metadata.configuration, Map( "delta.logRetentionDuration" -> "2 weeks", "delta.checkpointInterval" -> "20", "key" -> "value1", "kEy" -> "value2")) assert(deltaLog.deltaRetentionMillis(snapshot2.metadata) == 2 * 7 * 24 * 60 * 60 * 1000) assert(deltaLog.checkpointInterval(snapshot2.metadata) == 20) sql(s"ALTER TABLE $tableName UNSET TBLPROPERTIES ('DelTa.ChEckPoiNtinTervAl', 'kEy')") val snapshot3 = deltaLog.update() assertEqual(snapshot3.metadata.configuration, Map("delta.logRetentionDuration" -> "2 weeks", "key" -> "value1")) assert(deltaLog.deltaRetentionMillis(snapshot3.metadata) == 2 * 7 * 24 * 60 * 60 * 1000) assert(deltaLog.checkpointInterval(snapshot3.metadata) == CHECKPOINT_INTERVAL.fromString(CHECKPOINT_INTERVAL.defaultValue)) } } ddlTest("SET/UNSET TBLPROPERTIES - set unknown config") { withDeltaTable("v1 int, v2 string") { tableName => val ex = intercept[AnalysisException] { sql(s"ALTER TABLE $tableName SET TBLPROPERTIES ('delta.key' = 'value')") } assert(ex.getMessage.contains("Unknown configuration was specified: delta.key")) } } ddlTest("SET/UNSET TBLPROPERTIES - set invalid value") { withDeltaTable("v1 int, v2 string") { tableName => val ex1 = intercept[Exception] { sql(s"ALTER TABLE $tableName SET TBLPROPERTIES ('delta.randomPrefixLength' = '-1')") } assert(ex1.getMessage.contains("randomPrefixLength needs to be greater than 0.")) val ex2 = intercept[Exception] { sql(s"ALTER TABLE $tableName SET TBLPROPERTIES ('delta.randomPrefixLength' = 'value')") } assert(ex2.getMessage.contains("randomPrefixLength needs to be greater than 0.")) } } ddlTest("SET TBLPROPERTIES - delta.randomizeFilePrefixes") { withDeltaTable("v1 int, v2 string") { tableName => // Initially, randomizeFilePrefixes should be false (default) val (deltaLog, initialSnapshot) = getDeltaLogWithSnapshot(tableName) assert(!DeltaConfigs.RANDOMIZE_FILE_PREFIXES.fromMetaData(initialSnapshot.metadata), "randomizeFilePrefixes should be false by default") // Set delta.randomizeFilePrefixes and delta.randomPrefixLength sql(s""" |ALTER TABLE $tableName |SET TBLPROPERTIES ( | 'delta.randomizeFilePrefixes' = 'true', | 'delta.randomPrefixLength' = '5' |)""".stripMargin) val snapshot1 = deltaLog.update() assertEqual(snapshot1.metadata.configuration, Map( "delta.randomizeFilePrefixes" -> "true", "delta.randomPrefixLength" -> "5" )) // Verify the configuration is properly parsed assert(DeltaConfigs.RANDOMIZE_FILE_PREFIXES.fromMetaData(snapshot1.metadata), "randomizeFilePrefixes should be enabled") assert(DeltaConfigs.RANDOM_PREFIX_LENGTH.fromMetaData(snapshot1.metadata) == 5, "randomPrefixLength should be 5") // Insert data to create files with random prefixes sql(s"INSERT INTO $tableName VALUES (1, 'test1'), (2, 'test2'), (3, 'test3')") val snapshot2 = deltaLog.update() val allFiles = snapshot2.allFiles.collect() // Verify that files exist and have random prefixes assert(allFiles.nonEmpty, "Table should have data files") // Check that file paths contain 5-character random prefix pattern val prefixLength = DeltaConfigs.RANDOM_PREFIX_LENGTH.fromMetaData(snapshot2.metadata) assert(prefixLength == 5, s"Expected prefix length of 5, but got $prefixLength") val pattern = s"[A-Za-z0-9]{$prefixLength}/.*part-.*parquet" allFiles.foreach { file => assert(file.path.matches(pattern), s"File path '${file.path}' does not match expected random prefix pattern '$pattern'") } } } ddlTest("UNSET TBLPROPERTIES - delta.randomizeFilePrefixes") { withDeltaTable("v1 int, v2 string") { tableName => // First, set the randomizeFilePrefixes properties sql(s""" |ALTER TABLE $tableName |SET TBLPROPERTIES ( | 'delta.randomizeFilePrefixes' = 'true', | 'delta.randomPrefixLength' = '8', | 'key' = 'value' |)""".stripMargin) val (deltaLog, snapshot1) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot1.metadata.configuration, Map( "delta.randomizeFilePrefixes" -> "true", "delta.randomPrefixLength" -> "8", "key" -> "value" )) // Verify the configuration is properly set assert(DeltaConfigs.RANDOMIZE_FILE_PREFIXES.fromMetaData(snapshot1.metadata), "randomizeFilePrefixes should be enabled") assert(DeltaConfigs.RANDOM_PREFIX_LENGTH.fromMetaData(snapshot1.metadata) == 8, "randomPrefixLength should be 8") // Insert data to create files with random prefixes sql(s"INSERT INTO $tableName VALUES (1, 'test1'), (2, 'test2')") val snapshot1WithData = deltaLog.update() val filesWithPrefixes = snapshot1WithData.allFiles.collect() // Verify files have random prefixes assert(filesWithPrefixes.nonEmpty, "Table should have data files") val pattern8 = s"[A-Za-z0-9]{8}/.*part-.*parquet" filesWithPrefixes.foreach { file => assert(file.path.matches(pattern8), s"File path '${file.path}' should have 8-character random prefix") } // Now UNSET the randomizeFilePrefixes property sql(s"ALTER TABLE $tableName UNSET TBLPROPERTIES ('delta.randomizeFilePrefixes', 'key')") val snapshot2 = deltaLog.update() assertEqual(snapshot2.metadata.configuration, Map("delta.randomPrefixLength" -> "8")) // Verify that randomizeFilePrefixes is now disabled (reverted to default) assert(!DeltaConfigs.RANDOMIZE_FILE_PREFIXES.fromMetaData(snapshot2.metadata), "randomizeFilePrefixes should be disabled after UNSET") assert(DeltaConfigs.RANDOM_PREFIX_LENGTH.fromMetaData(snapshot2.metadata) == 8, "randomPrefixLength should still be 8 (not unset)") // Insert more data to verify new files don't have random prefixes sql(s"INSERT INTO $tableName VALUES (3, 'test3'), (4, 'test4')") val snapshot3 = deltaLog.update() val allFiles = snapshot3.allFiles.collect() val newFiles = allFiles.filterNot(f => filesWithPrefixes.exists(_.path == f.path)) // Verify that new files don't have random prefixes (should be regular paths) assert(newFiles.nonEmpty, "Should have new files after second insert") newFiles.foreach { file => assert(!file.path.matches(pattern8), s"New file path '${file.path}' should NOT have random prefix after UNSET") // New files should have regular naming without random prefixes assert(file.path.matches(".*part-.*parquet"), s"New file path '${file.path}' should have regular parquet file naming") } } } ddlTest("SET/UNSET TBLPROPERTIES - delta.randomizeFilePrefixes - partitioned table") { withDeltaTable(Seq((1, "x", 100), (2, "y", 200)).toDF("id", "part", "value"), Seq("part")) { tableName => // First, set the randomizeFilePrefixes properties sql(s""" |ALTER TABLE $tableName |SET TBLPROPERTIES ( | 'delta.randomizeFilePrefixes' = 'true', | 'delta.randomPrefixLength' = '7', | 'key' = 'value' |)""".stripMargin) val (deltaLog, snapshot1) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot1.metadata.configuration, Map( "delta.randomizeFilePrefixes" -> "true", "delta.randomPrefixLength" -> "7", "key" -> "value" )) // Get initial files (created during table setup - should have partition structure) val initialFiles = deltaLog.update().allFiles.collect() // Insert data to create files with random prefixes sql(s"INSERT INTO $tableName VALUES (3, 'x', 300), (4, 'z', 400)") val snapshot1WithData = deltaLog.update() val filesInSnapshot1 = snapshot1WithData.allFiles.collect() // Separate initial files from new files with prefixes val filesWithPrefixes = filesInSnapshot1.filterNot(f => initialFiles.exists(_.path == f.path)) // Verify INITIAL files have partition directory structure // (created before random prefixes enabled) val initialPartitionPattern = s"part=[xyz]/.*part-.*parquet" initialFiles.foreach { file => assert(file.path.matches(initialPartitionPattern), s"Initial file path '${file.path}' should have partition directory structure") } // Verify NEW files have random prefixes (created after random prefixes enabled) assert(filesWithPrefixes.nonEmpty, "Should have new files with random prefixes") val pattern7 = s"[A-Za-z0-9]{7}/.*part-.*parquet" filesWithPrefixes.foreach { file => assert(file.path.matches(pattern7), s"New file path '${file.path}' should have 7-character random prefix") } // Now UNSET the randomizeFilePrefixes property sql(s"ALTER TABLE $tableName UNSET TBLPROPERTIES ('delta.randomizeFilePrefixes', 'key')") val snapshot2 = deltaLog.update() assertEqual(snapshot2.metadata.configuration, Map("delta.randomPrefixLength" -> "7")) // Verify that randomizeFilePrefixes is now disabled assert(!DeltaConfigs.RANDOMIZE_FILE_PREFIXES.fromMetaData(snapshot2.metadata), "randomizeFilePrefixes should be disabled after UNSET") assert(DeltaConfigs.RANDOM_PREFIX_LENGTH.fromMetaData(snapshot2.metadata) == 7, "randomPrefixLength should still be 7 (not unset)") // Insert more data to verify new files don't have random prefixes sql(s"INSERT INTO $tableName VALUES (5, 'x', 500), (6, 'y', 600)") val snapshot3 = deltaLog.update() val allFinalFiles = snapshot3.allFiles.collect() val filesAfterUnset = allFinalFiles.filterNot(f => filesInSnapshot1.exists(_.path == f.path)) // Verify that new files don't have random prefixes (should revert to partition structure) assert(filesAfterUnset.nonEmpty, "Should have new files after UNSET and second insert") val partitionPatternAfterUnset = s"part=[xy]/.*part-.*parquet" filesAfterUnset.foreach { file => assert(!file.path.matches(pattern7), s"File after UNSET '${file.path}' should NOT have random prefix") // New files should revert to partition directory structure assert(file.path.matches(partitionPatternAfterUnset), s"File after UNSET '${file.path}' should have partition directory structure") } } } test("SET/UNSET comment by TBLPROPERTIES") { withDeltaTable("v1 int, v2 string") { tableName => def assertCommentEmpty(): Unit = { val props = sql(s"DESC EXTENDED $tableName").collect() assert(!props.exists(_.getString(0) === "Comment"), "Comment should be empty") val desc = sql(s"DESCRIBE DETAIL $tableName").head() val fieldIndex = desc.fieldIndex("description") assert(desc.isNullAt(fieldIndex)) } assertCommentEmpty() sql(s"ALTER TABLE $tableName SET TBLPROPERTIES ('comment'='does it work?')") val props = sql(s"DESC EXTENDED $tableName").collect() assert(props.exists(r => r.getString(0) === "Comment" && r.getString(1) === "does it work?"), s"Comment not found in:\n${props.mkString("\n")}") val desc = sql(s"DESCRIBE DETAIL $tableName").head() assert(desc.getAs[String]("description") === "does it work?") sql(s"ALTER TABLE $tableName UNSET TBLPROPERTIES ('comment')") assertCommentEmpty() } } test("update comment by TBLPROPERTIES") { val tableName = "comment_table" def checkComment(expected: String): Unit = { val props = sql(s"DESC EXTENDED $tableName").collect() assert(props.exists(r => r.getString(0) === "Comment" && r.getString(1) === expected), s"Comment not found in:\n${props.mkString("\n")}") val desc = sql(s"DESCRIBE DETAIL $tableName").head() assert(desc.getAs[String]("description") === expected) } withTable(tableName) { sql(s"CREATE TABLE $tableName (id bigint) USING delta COMMENT 'x'") checkComment("x") sql(s"ALTER TABLE $tableName SET TBLPROPERTIES ('comment'='y')") checkComment("y") } } ddlTest("Invalid TBLPROPERTIES") { withDeltaTable("v1 int, v2 string") { tableName => // Handled by Spark intercept[ParseException] { sql(s"ALTER TABLE $tableName SET TBLPROPERTIES ('location'='/some/new/path')") } // Handled by Spark intercept[ParseException] { sql(s"ALTER TABLE $tableName SET TBLPROPERTIES ('provider'='json')") } // Illegal to add constraints val e3 = intercept[AnalysisException] { sql(s"ALTER TABLE $tableName SET TBLPROPERTIES ('delta.constraints.c1'='age >= 25')") } assert(e3.getMessage.contains("ALTER TABLE ADD CONSTRAINT")) } } private def setProps(table: String, kvs: (String, String)*): Unit = { val props = kvs.map { case (k, v) => s"'$k'='$v'" }.mkString(", ") val sqlString = s"ALTER TABLE $table SET TBLPROPERTIES ($props)" spark.sql(sqlString) } private def expectValidationError(f: => Unit): Unit = { val ex = intercept[Exception](f) assert( (ex.getMessage.contains("delta.logRetentionDuration") && ex.getMessage.contains("delta.deletedFileRetentionDuration")) && (ex.getMessage.contains("needs to be greater than or equal to") || ex.getMessage.contains("needs to be less than or equal to")) ) } /////////////////////////////// // logRetentionDuration and deletedFileRetentionDuration table property // compatibility tests /////////////////////////////// // cases where validation succeeds test("log > deleted (same units) succeeds") { withDeltaTable("v1 int, v2 string") { t => setProps(t, "delta.deletedFileRetentionDuration" -> "interval 7 days", "delta.logRetentionDuration" -> "interval 30 days" ) } } test("log > deleted (different units) succeeds") { withDeltaTable("v1 int, v2 string") { t => setProps(t, "delta.deletedFileRetentionDuration" -> "interval 4 days", "delta.logRetentionDuration" -> "interval 120 hours" ) } } test("log > deleted one after the other succeeds") { withDeltaTable("v1 int, v2 string") { t => setProps(t, "delta.deletedFileRetentionDuration" -> "interval 6 days" ) setProps(t, "delta.logRetentionDuration" -> "interval 10 days" ) } } test("key case-insensitivity still succeeds") { withDeltaTable("v1 int, v2 string") { t => setProps(t, "delta.deletedFileRETENTIONDuration" -> " interval 7 days ", "delta.logRetentionDURATION" -> " INTERVAL 30 DAYS " ) } } test("equal durations shouldn't fail") { withDeltaTable("v1 int, v2 string") { t => setProps(t, "delta.deletedFileRetentionDuration" -> "interval 7 days", "delta.logRetentionDuration" -> "interval 1 week" ) } } // cases where validation fails test("log < deleted should fail") { withDeltaTable("v1 int, v2 string") { t => expectValidationError( setProps(t, "delta.deletedFileRetentionDuration" -> "interval 10 days", "delta.logRetentionDuration" -> "interval 6 days" ) ) } } test("sequence that becomes invalid (raise deleted above log) should fail") { withDeltaTable("v1 int, v2 string") { t => setProps(t, "delta.deletedFileRetentionDuration" -> "interval 7 days", "delta.logRetentionDuration" -> "interval 30 days" ) expectValidationError( setProps(t, "delta.deletedFileRetentionDuration" -> "interval 60 days") ) } } test("default log vs explicit deleted that exceeds default should fail") { withDeltaTable("v1 int, v2 string") { t => // default log 30 days; setting deleted to 45 should fail expectValidationError( setProps(t, "delta.deletedFileRetentionDuration" -> "interval 45 days") ) } } test("default deletedRetention vs explicit log retention that exceeds default should fail") { withDeltaTable("v1 int, v2 string") { t => // default deletedFileRetention 7 days; setting log to 5 should fail expectValidationError( setProps(t, "delta.logRetentionDuration" -> "interval 5 days") ) } } test("key case-insensitivity still fails") { withDeltaTable("v1 int, v2 string") { t => expectValidationError( setProps(t, "DELTA.DELETEDFILERETENTIONDURATION" -> "interval 14 days", "delta.logRetentionDurATION" -> "interval 7 days" ) ) } } test("reset to defaults becomes valid") { withDeltaTable("v1 int, v2 string") { t => // Start invalid expectValidationError( setProps(t, "delta.deletedFileRetentionDuration" -> "interval 40 days", "delta.logRetentionDuration" -> "interval 30 days" ) ) // Reset deleted; expect success if defaults are valid spark.sql(s"ALTER TABLE $t UNSET TBLPROPERTIES ('delta.deletedFileRetentionDuration')") // Now set log to something valid relative to default deleted (7d) setProps(t, "delta.logRetentionDuration" -> "interval 30 days") } } test("property values are invalid before. Setting an unrelated property shouldn't error out") { withDeltaTable("v1 int, v2 string") { t => // Start invalid withSQLConf( DeltaSQLConf.ENFORCE_DELETED_FILE_AND_LOG_RETENTION_DURATION_COMPATIBILITY.key -> false.toString) { setProps(t, "delta.deletedFileRetentionDuration" -> "interval 40 days", "delta.logRetentionDuration" -> "interval 30 days" ) } // Now set unrelated table property setProps(t, "delta.checkpointInterval" -> "100") } } /////////////////////////////// // ADD COLUMNS /////////////////////////////// ddlTest("ADD COLUMNS - simple") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2")) { tableName => checkDatasetUnorderly( spark.table(tableName).as[(Int, String)], (1, "a"), (2, "b")) sql(s"ALTER TABLE $tableName ADD COLUMNS (v3 long, v4 double)") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("v3", "long").add("v4", "double")) checkDatasetUnorderly( spark.table(tableName).as[(Int, String, Option[Long], Option[Double])], (1, "a", None, None), (2, "b", None, None)) } } ddlTest("ADD COLUMNS into complex types - Array") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("a", array(struct("v1")))) { tableName => sql( s""" |ALTER TABLE $tableName ADD COLUMNS (a.element.v3 long) """.stripMargin) val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("a", ArrayType(new StructType() .add("v1", "integer") .add("v3", "long")))) sql( s""" |ALTER TABLE $tableName ADD COLUMNS (a.element.v4 struct) """.stripMargin) assertEqual(deltaLog.snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("a", ArrayType(new StructType() .add("v1", "integer") .add("v3", "long") .add("v4", new StructType().add("f1", "long"))))) sql( s""" |ALTER TABLE $tableName ADD COLUMNS (a.element.v4.f2 string) """.stripMargin) assertEqual(deltaLog.snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("a", ArrayType(new StructType() .add("v1", "integer") .add("v3", "long") .add("v4", new StructType() .add("f1", "long") .add("f2", "string"))))) } } ddlTest("ADD COLUMNS into complex types - Map with simple key") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("m", map('v1, struct("v2")))) { tableName => sql( s""" |ALTER TABLE $tableName ADD COLUMNS (m.value.mvv3 long) """.stripMargin) val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("m", MapType(IntegerType, new StructType() .add("v2", "string") .add("mvv3", "long")))) } } ddlTest("ADD COLUMNS into complex types - Map with simple value") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("m", map(struct("v1"), 'v2))) { tableName => sql( s""" |ALTER TABLE $tableName ADD COLUMNS (m.key.mkv3 long) """.stripMargin) val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("m", MapType( new StructType() .add("v1", "integer") .add("mkv3", "long"), StringType))) } } private def checkErrMsg(msg: String, field: Seq[String]): Unit = { val fieldStr = field.map(f => s"`$f`").mkString(".") val fieldParentStr = field.dropRight(1).map(f => s"`$f`").mkString(".") assert(msg.contains( s"Field name $fieldStr is invalid: $fieldParentStr is not a struct")) } ddlTest("ADD COLUMNS should not be able to add column to basic type key/value of " + "MapType") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("m", map('v1, 'v2))) { tableName => var ex = intercept[AnalysisException] { sql( s""" |ALTER TABLE $tableName ADD COLUMNS (m.key.mkv3 long) """.stripMargin) } checkErrMsg(ex.getMessage, Seq("m", "key", "mkv3")) ex = intercept[AnalysisException] { sql( s""" |ALTER TABLE $tableName ADD COLUMNS (m.value.mkv3 long) """.stripMargin) } checkErrMsg(ex.getMessage, Seq("m", "value", "mkv3")) } } ddlTest("ADD COLUMNS into complex types - Map") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("m", map(struct("v1"), struct("v2")))) { tableName => sql( s""" |ALTER TABLE $tableName ADD COLUMNS (m.key.mkv3 long, m.value.mvv3 long) """.stripMargin) val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("m", MapType( new StructType() .add("v1", "integer") .add("mkv3", "long"), new StructType() .add("v2", "string") .add("mvv3", "long")))) } } ddlTest("ADD COLUMNS into complex types - Map (nested)") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("m", map(struct("v1"), struct("v2")))) { tableName => sql( s""" |ALTER TABLE $tableName ADD COLUMNS |(m.key.mkv3 long, m.value.mvv3 struct>>) """.stripMargin) val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("m", MapType( new StructType() .add("v1", "integer") .add("mkv3", "long"), new StructType() .add("v2", "string") .add("mvv3", new StructType() .add("f1", "long") .add("f2", ArrayType(new StructType() .add("n", "long"))))))) sql( s""" |ALTER TABLE $tableName ADD COLUMNS |(m.value.mvv3.f2.element.p string) """.stripMargin) assertEqual(deltaLog.snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("m", MapType( new StructType() .add("v1", "integer") .add("mkv3", "long"), new StructType() .add("v2", "string") .add("mvv3", new StructType() .add("f1", "long") .add("f2", ArrayType(new StructType() .add("n", "long") .add("p", "string"))))))) } } ddlTest("ADD COLUMNS into Map should fail if key or value not specified") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("m", map(struct("v1"), struct("v2")))) { tableName => val ex = intercept[AnalysisException] { sql( s""" |ALTER TABLE $tableName ADD COLUMNS (m.mkv3 long) """.stripMargin) } checkErrMsg(ex.getMessage, Seq("m", "mkv3")) } } ddlTest("ADD COLUMNS into Array should fail if element is not specified") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("a", array(struct("v1")))) { tableName => intercept[AnalysisException] { sql( s""" |ALTER TABLE $tableName ADD COLUMNS (a.v3 long) """.stripMargin) } } } ddlTest("ADD COLUMNS - a partitioned table") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2"), Seq("v2")) { tableName => checkDatasetUnorderly( spark.table(tableName).as[(Int, String)], (1, "a"), (2, "b")) sql(s"ALTER TABLE $tableName ADD COLUMNS (v3 long, v4 double)") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("v3", "long").add("v4", "double")) checkDatasetUnorderly( spark.table(tableName).as[(Int, String, Option[Long], Option[Double])], (1, "a", None, None), (2, "b", None, None)) } } ddlTest("ADD COLUMNS - with a comment") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2")) { tableName => checkDatasetUnorderly( spark.table(tableName).as[(Int, String)], (1, "a"), (2, "b")) sql(s"ALTER TABLE $tableName ADD COLUMNS (v3 long COMMENT 'new column')") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("v3", "long", true, "new column")) checkDatasetUnorderly( spark.table(tableName).as[(Int, String, Option[Long])], (1, "a", None), (2, "b", None)) } } ddlTest("ADD COLUMNS - adding to a non-struct column") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2")) { tableName => val ex = intercept[AnalysisException] { sql(s"ALTER TABLE $tableName ADD COLUMNS (v2.x long)") } checkErrMsg(ex.getMessage, Seq("v2", "x")) } } ddlTest("ADD COLUMNS - a duplicate name") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2")) { tableName => intercept[AnalysisException] { sql(s"ALTER TABLE $tableName ADD COLUMNS (v2 long)") } } } ddlTest("ADD COLUMNS - a duplicate name (nested)") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("struct", struct("v1", "v2")) withDeltaTable(df) { tableName => intercept[AnalysisException] { sql(s"ALTER TABLE $tableName ADD COLUMNS (struct.v2 long)") } } } ddlTest("ADD COLUMNS - column name with spaces") { if (!columnMappingEnabled) { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2")) { tableName => val ex = intercept[AnalysisException] { sql(s"ALTER TABLE $tableName ADD COLUMNS (`a column name with spaces` long)") } assert(ex.getMessage.contains("invalid character(s)")) } } else { // column mapping mode supports arbitrary column names withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2")) { tableName => sql(s"ALTER TABLE $tableName ADD COLUMNS (`a column name with spaces` long)") } } } ddlTest("ADD COLUMNS - column name with spaces (nested)") { if (!columnMappingEnabled) { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("struct", struct("v1", "v2")) withDeltaTable(df) { tableName => val ex = intercept[AnalysisException] { sql(s"ALTER TABLE $tableName ADD COLUMNS (struct.`a column name with spaces` long)") } assert(ex.getMessage.contains("invalid character(s)")) } } else { // column mapping mode supports arbitrary column names val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("struct", struct("v1", "v2")) withDeltaTable(df) { tableName => sql(s"ALTER TABLE $tableName ADD COLUMNS (struct.`a column name with spaces` long)") } } } ddlTest("ADD COLUMNS - special column names") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("z.z", struct("v1", "v2")) withDeltaTable(df) { tableName => checkDatasetUnorderly( spark.table(tableName).as[(Int, String, (Int, String))], (1, "a", (1, "a")), (2, "b", (2, "b"))) sql(s"ALTER TABLE $tableName ADD COLUMNS (`x.x` long, `z.z`.`y.y` double)") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("z.z", new StructType() .add("v1", "integer").add("v2", "string").add("y.y", "double")) .add("x.x", "long")) checkDatasetUnorderly( spark.table(tableName).as[(Int, String, (Int, String, Option[Double]), Option[Long])], (1, "a", (1, "a", None), None), (2, "b", (2, "b", None), None)) } } test("ADD COLUMNS - with positions") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") withDeltaTable(df) { tableName => checkDatasetUnorderly( spark.table(tableName).as[(Int, String)], (1, "a"), (2, "b")) sql(s"ALTER TABLE $tableName ADD COLUMNS (v3 long FIRST, v4 long AFTER v1, v5 long)") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v3", "long").add("v1", "integer") .add("v4", "long").add("v2", "string").add("v5", "long")) checkDatasetUnorderly( spark.table(tableName).as[(Option[Long], Int, Option[Long], String, Option[Long])], (None, 1, None, "a", None), (None, 2, None, "b", None)) } } test("ADD COLUMNS - with positions using an added column") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") withDeltaTable(df) { tableName => checkDatasetUnorderly( spark.table("delta_test").as[(Int, String)], (1, "a"), (2, "b")) sql("ALTER TABLE delta_test ADD COLUMNS (v3 long FIRST, v4 long AFTER v3, v5 long AFTER v4)") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v3", "long").add("v4", "long").add("v5", "long") .add("v1", "integer").add("v2", "string")) checkDatasetUnorderly( spark.table("delta_test").as[(Option[Long], Option[Long], Option[Long], Int, String)], (None, None, None, 1, "a"), (None, None, None, 2, "b")) } } test("ADD COLUMNS - nested columns") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("struct", struct("v1", "v2")) withDeltaTable(df) { tableName => checkDatasetUnorderly( spark.table("delta_test").as[(Int, String, (Int, String))], (1, "a", (1, "a")), (2, "b", (2, "b"))) sql("ALTER TABLE delta_test ADD COLUMNS " + "(struct.v3 long FIRST, struct.v4 long AFTER v1, struct.v5 long)") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("struct", new StructType() .add("v3", "long").add("v1", "integer") .add("v4", "long").add("v2", "string").add("v5", "long"))) checkDatasetUnorderly( spark.table("delta_test") .as[(Int, String, (Option[Long], Int, Option[Long], String, Option[Long]))], (1, "a", (None, 1, None, "a", None)), (2, "b", (None, 2, None, "b", None))) } } test("ADD COLUMNS - special column names with positions") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("z.z", struct("v1", "v2")) withDeltaTable(df) { tableName => checkDatasetUnorderly( spark.table(tableName).as[(Int, String, (Int, String))], (1, "a", (1, "a")), (2, "b", (2, "b"))) sql(s"ALTER TABLE $tableName ADD COLUMNS (`x.x` long after v1, `z.z`.`y.y` double)") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("x.x", "long").add("v2", "string") .add("z.z", new StructType() .add("v1", "integer").add("v2", "string").add("y.y", "double")) ) checkDatasetUnorderly( spark.table(tableName).as[(Int, Option[Long], String, (Int, String, Option[Double]))], (1, None, "a", (1, "a", None)), (2, None, "b", (2, "b", None))) } } test("ADD COLUMNS - adding after an unknown column") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") withDeltaTable(df) { tableName => val ex = intercept[AnalysisException] { sql(s"ALTER TABLE $tableName ADD COLUMNS (v3 long AFTER unknown)") } assert( ex.getMessage.contains("Couldn't find") || ex.getMessage.contains("No such struct field")) } } test("ADD COLUMNS - case insensitive") { withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") withDeltaTable(df) { tableName => sql(s"ALTER TABLE $tableName ADD COLUMNS (v3 long AFTER V1)") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v3", "long").add("v2", "string")) } } } test("ADD COLUMNS - case sensitive") { withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") withDeltaTable(df) { tableName => val ex = intercept[AnalysisException] { sql(s"ALTER TABLE $tableName ADD COLUMNS (v3 long AFTER V1)") } assert( ex.getMessage.contains("Couldn't find") || ex.getMessage.contains("No such struct field")) } } } test("ADD COLUMNS - adding after an Array column") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("v3", array(map(col("v1"), col("v2")))) withDeltaTable(df) { tableName => sql(s"ALTER TABLE $tableName ADD COLUMNS (v4 string AFTER V3)") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", IntegerType) .add("v2", StringType) .add("v3", ArrayType( MapType(IntegerType, StringType))) .add("v4", StringType)) } } /////////////////////////////// // CHANGE COLUMN /////////////////////////////// ddlTest("CHANGE COLUMN - add a comment") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2")) { tableName => sql(s"ALTER TABLE $tableName CHANGE COLUMN v1 v1 integer COMMENT 'a comment'") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer", true, "a comment").add("v2", "string")) } } ddlTest("CHANGE COLUMN - add a comment to a partitioned table") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2"), Seq("v2")) { tableName => sql(s"ALTER TABLE $tableName CHANGE COLUMN v2 v2 string COMMENT 'a comment'") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string", true, "a comment")) } } ddlTest("CHANGE COLUMN - add a comment to special column names (nested)") { val df = Seq((1, "a"), (2, "b")).toDF("x.x", "y.y") .withColumn("z.z", struct("`x.x`", "`y.y`")) withDeltaTable(df) { tableName => sql(s"ALTER TABLE $tableName CHANGE COLUMN `z.z`.`x.x` `x.x` integer COMMENT 'a comment'") sql(s"ALTER TABLE $tableName CHANGE COLUMN `x.x` `x.x` integer COMMENT 'another comment'") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("x.x", "integer", true, "another comment") .add("y.y", "string") .add("z.z", new StructType() .add("x.x", "integer", true, "a comment").add("y.y", "string"))) checkDatasetUnorderly( spark.table(tableName).as[(Int, String, (Int, String))], (1, "a", (1, "a")), (2, "b", (2, "b"))) } } ddlTest("CHANGE COLUMN - add a comment to a MapType (nested)") { val table = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("a", array(struct(array(struct(map(struct("v1"), struct("v2"))))))) withDeltaTable(table) { tableName => sql( s""" |ALTER TABLE $tableName CHANGE COLUMN |a.element.col1.element.col1 col1 MAP, |STRUCT> COMMENT 'a comment' """.stripMargin) val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("a", ArrayType(new StructType() .add("col1", ArrayType(new StructType() .add("col1", MapType( new StructType() .add("v1", "integer"), new StructType() .add("v2", "string")), nullable = true, "a comment")))))) } } ddlTest("CHANGE COLUMN - add a comment to an ArrayType (nested)") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("m", map(struct("v1"), struct(array(struct(struct("v1"))))))) { tableName => sql( s""" |ALTER TABLE $tableName CHANGE COLUMN |m.value.col1.element.col1.v1 v1 integer COMMENT 'a comment' """.stripMargin) val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("m", MapType( new StructType() .add("v1", "integer"), new StructType() .add("col1", ArrayType(new StructType() .add("col1", new StructType() .add("v1", "integer", nullable = true, "a comment"))))))) } } ddlTest("CHANGE COLUMN - add a comment to an ArrayType") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("a", array('v1))) { tableName => sql(s"ALTER TABLE $tableName CHANGE COLUMN a a ARRAY COMMENT 'a comment'") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("a", ArrayType(IntegerType), nullable = true, "a comment")) } } ddlTest("CHANGE COLUMN - add a comment to a MapType") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("a", map('v1, 'v2))) { tableName => sql( s""" |ALTER TABLE $tableName CHANGE COLUMN |a a MAP COMMENT 'a comment' """.stripMargin) val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("a", MapType(IntegerType, StringType), nullable = true, "a comment")) } } ddlTest("CHANGE COLUMN - (unsupported) add a comment to key/value of a MapType") { val df = Seq((1, 1), (2, 2)).toDF("v1", "v2") .withColumn("a", map('v1, 'v2)) withDeltaTable(df) { tableName => checkError( intercept[DeltaAnalysisException] { sql(s"ALTER TABLE $tableName CHANGE COLUMN a.key COMMENT 'a comment'") }, "DELTA_UNSUPPORTED_COMMENT_MAP_ARRAY", parameters = Map("fieldPath" -> "a.key") ) checkError( intercept[DeltaAnalysisException] { sql(s"ALTER TABLE $tableName CHANGE COLUMN a.value COMMENT 'a comment'") }, "DELTA_UNSUPPORTED_COMMENT_MAP_ARRAY", parameters = Map("fieldPath" -> "a.value") ) } } ddlTest("CHANGE COLUMN - (unsupported) add a comment to element of an array") { val df = Seq(1, 2).toDF("v1") .withColumn("a", array('v1)) withDeltaTable(df) { tableName => checkError( intercept[DeltaAnalysisException] { sql(s"ALTER TABLE $tableName CHANGE COLUMN a.element COMMENT 'a comment'") }, "DELTA_UNSUPPORTED_COMMENT_MAP_ARRAY", parameters = Map("fieldPath" -> "a.element") ) } } ddlTest("RENAME COLUMN - (unsupported) rename key/value of a MapType") { val df = Seq((1, 1), (2, 2)).toDF("v1", "v2") .withColumn("a", map('v1, 'v2)) withDeltaTable(df) { tableName => checkError( intercept[AnalysisException] { sql(s"ALTER TABLE $tableName RENAME COLUMN a.key TO key2") }, "INVALID_FIELD_NAME", parameters = Map( "fieldName" -> "`a`.`key2`", "path" -> "`a`" ) ) checkError( intercept[AnalysisException] { sql(s"ALTER TABLE $tableName RENAME COLUMN a.value TO value2") }, "INVALID_FIELD_NAME", parameters = Map( "fieldName" -> "`a`.`value2`", "path" -> "`a`" ) ) } } ddlTest("RENAME COLUMN - (unsupported) rename element of an array") { val df = Seq(1, 2).toDF("v1") .withColumn("a", array('v1)) withDeltaTable(df) { tableName => checkError( intercept[AnalysisException] { sql(s"ALTER TABLE $tableName RENAME COLUMN a.element TO element2") }, "INVALID_FIELD_NAME", parameters = Map( "fieldName" -> "`a`.`element2`", "path" -> "`a`" ) ) } } ddlTest("CHANGE COLUMN - change name") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2")) { tableName => assertNotSupported(s"ALTER TABLE $tableName CHANGE COLUMN v2 v3 string") } } ddlTest("CHANGE COLUMN - incompatible") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2")) { tableName => checkError( intercept[DeltaAnalysisException] { sql(s"ALTER TABLE $tableName CHANGE COLUMN v1 v1 long") }, "DELTA_UNSUPPORTED_ALTER_TABLE_CHANGE_COL_OP", parameters = Map( "fieldPath" -> "v1", "oldField" -> "INT", "newField" -> "BIGINT" ) ) } } ddlTest("CHANGE COLUMN - incompatible (nested)") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("struct", struct("v1", "v2")) withDeltaTable(df) { tableName => checkError( intercept[DeltaAnalysisException] { sql(s"ALTER TABLE $tableName CHANGE COLUMN struct.v1 v1 long") }, "DELTA_UNSUPPORTED_ALTER_TABLE_CHANGE_COL_OP", parameters = Map( "fieldPath" -> "struct.v1", "oldField" -> "INT", "newField" -> "BIGINT" ) ) } } ddlTest("CHANGE COLUMN - (unsupported) change type of key of a MapType") { val df = Seq((1, 1), (2, 2)).toDF("v1", "v2") .withColumn("a", map('v1, 'v2)) withDeltaTable(df) { tableName => checkError( intercept[DeltaAnalysisException] { sql(s"ALTER TABLE $tableName CHANGE COLUMN a.key key long") }, "DELTA_UNSUPPORTED_ALTER_TABLE_CHANGE_COL_OP", parameters = Map( "fieldPath" -> "a.key", "oldField" -> "INT NOT NULL", "newField" -> "BIGINT NOT NULL" ) ) } } ddlTest("CHANGE COLUMN - (unsupported) change type of value of a MapType") { val df = Seq((1, 1), (2, 2)).toDF("v1", "v2") .withColumn("a", map('v1, 'v2)) withDeltaTable(df) { tableName => checkError( intercept[DeltaAnalysisException] { sql(s"ALTER TABLE $tableName CHANGE COLUMN a.value value long") }, "DELTA_UNSUPPORTED_ALTER_TABLE_CHANGE_COL_OP", parameters = Map( "fieldPath" -> "a.value", "oldField" -> "INT", "newField" -> "BIGINT" ) ) } } ddlTest("CHANGE COLUMN - (unsupported) change type of element of an ArrayType") { val df = Seq(1).toDF("v1") .withColumn("a", array('v1)) withDeltaTable(df) { tableName => checkError( intercept[DeltaAnalysisException] { sql(s"ALTER TABLE $tableName CHANGE COLUMN a.element element long") }, "DELTA_UNSUPPORTED_ALTER_TABLE_CHANGE_COL_OP", parameters = Map( "fieldPath" -> "a.element", "oldField" -> "INT", "newField" -> "BIGINT" ) ) } } test("CHANGE COLUMN - move to first") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") withDeltaTable(df) { tableName => sql(s"ALTER TABLE $tableName CHANGE COLUMN v2 v2 string FIRST") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v2", "string").add("v1", "integer")) checkDatasetUnorderly( spark.table(tableName).as[(String, Int)], ("a", 1), ("b", 2)) } } test("CHANGE COLUMN - move to first (nested)") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("struct", struct("v1", "v2")) withDeltaTable(df) { tableName => sql(s"ALTER TABLE $tableName CHANGE COLUMN struct.v2 v2 string FIRST") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("struct", new StructType() .add("v2", "string").add("v1", "integer"))) checkDatasetUnorderly( spark.table(tableName).as[(Int, String, (String, Int))], (1, "a", ("a", 1)), (2, "b", ("b", 2))) // Can't change the inner ordering assertNotSupported(s"ALTER TABLE $tableName CHANGE COLUMN struct struct " + "STRUCT FIRST") sql(s"ALTER TABLE $tableName CHANGE COLUMN struct struct " + "STRUCT FIRST") assertEqual(deltaLog.update().schema, new StructType() .add("struct", new StructType().add("v2", "string").add("v1", "integer")) .add("v1", "integer").add("v2", "string")) } } test("CHANGE COLUMN - move a partitioned column to first") { val df = Seq((1, "a", true), (2, "b", false)).toDF("v1", "v2", "v3") withDeltaTable(df, Seq("v2", "v3")) { tableName => sql(s"ALTER TABLE $tableName CHANGE COLUMN v3 v3 boolean FIRST") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v3", "boolean").add("v1", "integer").add("v2", "string")) checkDatasetUnorderly( spark.table(tableName).as[(Boolean, Int, String)], (true, 1, "a"), (false, 2, "b")) } } test("CHANGE COLUMN - move to after some column") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") withDeltaTable(df) { tableName => sql(s"ALTER TABLE $tableName CHANGE COLUMN v1 v1 integer AFTER v2") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v2", "string").add("v1", "integer")) checkDatasetUnorderly( spark.table(tableName).as[(String, Int)], ("a", 1), ("b", 2)) } } test("CHANGE COLUMN - move to after some column (nested)") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("struct", struct("v1", "v2")) withDeltaTable(df) { tableName => sql(s"ALTER TABLE $tableName CHANGE COLUMN struct.v1 v1 integer AFTER v2") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("struct", new StructType() .add("v2", "string").add("v1", "integer"))) checkDatasetUnorderly( spark.table(tableName).as[(Int, String, (String, Int))], (1, "a", ("a", 1)), (2, "b", ("b", 2))) // cannot change ordering within the struct assertNotSupported(s"ALTER TABLE $tableName CHANGE COLUMN struct struct " + "STRUCT AFTER v1") // can move the struct itself however sql(s"ALTER TABLE $tableName CHANGE COLUMN struct struct " + "STRUCT AFTER v1") assertEqual(deltaLog.update().schema, new StructType() .add("v1", "integer") .add("struct", new StructType().add("v2", "string").add("v1", "integer")) .add("v2", "string")) } } test("CHANGE COLUMN - move to after the same column") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") withDeltaTable(df) { tableName => sql(s"ALTER TABLE $tableName CHANGE COLUMN v1 v1 integer AFTER v1") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string")) checkDatasetUnorderly( spark.table(tableName).as[(Int, String)], (1, "a"), (2, "b")) } } test("CHANGE COLUMN - move to after the same column (nested)") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("struct", struct("v1", "v2")) withDeltaTable(df) { tableName => sql(s"ALTER TABLE $tableName CHANGE COLUMN struct.v1 v1 integer AFTER v1") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("struct", new StructType() .add("v1", "integer").add("v2", "string"))) checkDatasetUnorderly( spark.table(tableName).as[(Int, String, (Int, String))], (1, "a", (1, "a")), (2, "b", (2, "b"))) } } test("CHANGE COLUMN - move a partitioned column to after some column") { val df = Seq((1, "a", true), (2, "b", false)).toDF("v1", "v2", "v3") withDeltaTable(df, Seq("v2", "v3")) { tableName => sql(s"ALTER TABLE $tableName CHANGE COLUMN v3 v3 boolean AFTER v1") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v3", "boolean").add("v2", "string")) checkDatasetUnorderly( spark.table(tableName).as[(Int, Boolean, String)], (1, true, "a"), (2, false, "b")) } } test("CHANGE COLUMN - move to after the last column") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") withDeltaTable(df) { tableName => sql(s"ALTER TABLE $tableName CHANGE COLUMN v1 v1 integer AFTER v2") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v2", "string").add("v1", "integer")) } } test("CHANGE COLUMN - special column names with positions") { val df = Seq((1, "a"), (2, "b")).toDF("x.x", "y.y") withDeltaTable(df) { tableName => sql(s"ALTER TABLE $tableName CHANGE COLUMN `x.x` `x.x` integer AFTER `y.y`") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("y.y", "string").add("x.x", "integer")) checkDatasetUnorderly( spark.table(tableName).as[(String, Int)], ("a", 1), ("b", 2)) } } test("CHANGE COLUMN - special column names (nested) with positions") { val df = Seq((1, "a"), (2, "b")).toDF("x.x", "y.y") .withColumn("z.z", struct("`x.x`", "`y.y`")) withDeltaTable(df) { tableName => sql(s"ALTER TABLE $tableName CHANGE COLUMN `z.z`.`x.x` `x.x` integer AFTER `y.y`") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("x.x", "integer").add("y.y", "string") .add("z.z", new StructType() .add("y.y", "string").add("x.x", "integer"))) checkDatasetUnorderly( spark.table(tableName).as[(Int, String, (String, Int))], (1, "a", ("a", 1)), (2, "b", ("b", 2))) } } test("CHANGE COLUMN - move to after an unknown column") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") withDeltaTable(df) { tableName => val ex = intercept[AnalysisException] { sql(s"ALTER TABLE $tableName CHANGE COLUMN v1 v1 integer AFTER unknown") } checkExceptionMessage( ex, "Missing field unknown", "Couldn't resolve positional argument AFTER unknown", "A column, variable, or function parameter with name `unknown` cannot be resolved") } } test("CHANGE COLUMN - move to after an unknown column (nested)") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("struct", struct("v1", "v2")) withDeltaTable(df) { tableName => val ex = intercept[AnalysisException] { sql(s"ALTER TABLE $tableName CHANGE COLUMN struct.v1 v1 integer AFTER unknown") } checkExceptionMessage( ex, "Missing field struct.unknown", "Couldn't resolve positional argument AFTER unknown", "A column, variable, or function parameter with name `struct`.`unknown` cannot be resolved") } } test("CHANGE COLUMN - complex types nullability tests") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("s", struct("v1", "v2")) .withColumn("a", array("s")) .withColumn("m", map(col("s"), col("s"))) withDeltaTable(df) { tableName => // not supported to tighten nullabilities. assertNotSupported( s"ALTER TABLE $tableName CHANGE COLUMN s s STRUCT") assertNotSupported( s"ALTER TABLE $tableName CHANGE COLUMN a a " + "ARRAY>") assertNotSupported( s"ALTER TABLE $tableName CHANGE COLUMN m m " + "MAP, STRUCT>") assertNotSupported( s"ALTER TABLE $tableName CHANGE COLUMN m m " + "MAP, STRUCT>") // not supported to add not-null columns. assertNotSupported( s"ALTER TABLE $tableName CHANGE COLUMN s s " + "STRUCT") assertNotSupported( s"ALTER TABLE $tableName CHANGE COLUMN a a " + "ARRAY>") assertNotSupported( s"ALTER TABLE $tableName CHANGE COLUMN m m " + "MAP, " + "STRUCT>") assertNotSupported( s"ALTER TABLE $tableName CHANGE COLUMN m m " + "MAP, " + "STRUCT>") } } test("CHANGE COLUMN - (unsupported) change nullability of map key/value and array element") { val df = Seq((1, 1), (2, 2)) .toDF("key", "value") .withColumn("m", map(col("key"), col("value"))) .withColumn("a", array(col("value"))) withDeltaTable(df) { tableName => val schema = spark.read.table(tableName).schema assert(schema("m").dataType === MapType(IntegerType, IntegerType, valueContainsNull = true)) assert(schema("a").dataType === ArrayType(IntegerType, containsNull = true)) // No-op actions are allowed - map keys are always non-nullable. sql(s"ALTER TABLE $tableName CHANGE COLUMN m.key SET NOT NULL") sql(s"ALTER TABLE $tableName CHANGE COLUMN m.value DROP NOT NULL") sql(s"ALTER TABLE $tableName CHANGE COLUMN a.element DROP NOT NULL") // Changing the nullability of map/array fields is not allowed. var statement = s"ALTER TABLE $tableName CHANGE COLUMN m.key DROP NOT NULL" checkError( intercept[AnalysisException] { sql(statement) }, "DELTA_UNSUPPORTED_ALTER_TABLE_CHANGE_COL_OP", parameters = Map( "fieldPath" -> "m.key", "oldField" -> "INT NOT NULL", "newField" -> "INT" ) ) statement = s"ALTER TABLE $tableName CHANGE COLUMN m.value SET NOT NULL" checkError( intercept[AnalysisException] { sql(statement) }, "_LEGACY_ERROR_TEMP_2330", parameters = Map( "fieldName" -> "m.value" ), context = ExpectedContext(statement, 0, statement.length - 1) ) statement = s"ALTER TABLE $tableName CHANGE COLUMN a.element SET NOT NULL" checkError( intercept[AnalysisException] { sql(statement) }, "_LEGACY_ERROR_TEMP_2330", parameters = Map( "fieldName" -> "a.element" ), context = ExpectedContext(statement, 0, statement.length - 1) ) } } ddlTest("CHANGE COLUMN - set comment on a varchar column") { withDeltaTable(schema = "v varchar(1)") { tableName => sql(s"ALTER TABLE $tableName CHANGE COLUMN v COMMENT 'test comment'") val expectedResult = Row("v", "string", "test comment") :: Nil checkAnswer( sql(s"DESCRIBE $tableName").filter("col_name = 'v'"), expectedResult) checkColType(spark.table(tableName).schema.head, VarcharType(1)) val e = intercept[DeltaInvariantViolationException] { sql(s"INSERT into $tableName values ('12')") } assert(e.getMessage.contains("Value \"12\" exceeds char/varchar type length limitation. " + "Failed check: ((v IS NULL) OR (length(v) <= 1))")) } } ddlTest("CHANGE COLUMN - set comment on a char column") { withDeltaTable(schema = "v char(1)") { tableName => sql(s"ALTER TABLE $tableName CHANGE COLUMN v COMMENT 'test comment'") val expectedResult = Row("v", "string", "test comment") :: Nil checkAnswer( sql(s"DESCRIBE $tableName").filter("col_name = 'v'"), expectedResult) checkColType(spark.table(tableName).schema.head, CharType(1)) } } ddlTest("CHANGE COLUMN - set comment on a array/map/struct column") { val schema = """ |arr_c array, |map_cc map, |map_sc map, |map_cs map, |struct_c struct, |arr_v array, |map_vv map, |map_sv map, |map_vs map, |struct_v struct""".stripMargin def testCommentOnVarcharInContainer( colName: String, expectedType: String, goodInsertValue: String, badInsertValue: String ): Unit = { withDeltaTable(schema = schema) { tableName => sql(s"ALTER TABLE $tableName CHANGE COLUMN $colName COMMENT 'test comment'") val expectedResult = Row(colName, expectedType, "test comment") :: Nil checkAnswer( sql(s"DESCRIBE $tableName").filter(s"col_name = '$colName'"), expectedResult) sql(s"INSERT into $tableName($colName) values ($goodInsertValue)") val e = intercept[DeltaInvariantViolationException] { sql(s"INSERT into $tableName($colName) values ($badInsertValue)") } assert(e.getMessage.contains("exceeds char/varchar type length limitation")) } } testCommentOnVarcharInContainer( colName = "arr_c", expectedType = "array", goodInsertValue = "array('1')", badInsertValue = "array('12')") testCommentOnVarcharInContainer( colName = "map_cc", expectedType = "map", goodInsertValue = "map('1', '1')", badInsertValue = "map('12', '12')") testCommentOnVarcharInContainer( colName = "map_sc", expectedType = "map", goodInsertValue = "map('123', '1')", badInsertValue = "map('123', '12')") testCommentOnVarcharInContainer( colName = "map_cs", expectedType = "map", goodInsertValue = "map('1', '123')", badInsertValue = "map('12', '123')") testCommentOnVarcharInContainer( colName = "struct_c", expectedType = "struct", goodInsertValue = "named_struct('v', '1')", badInsertValue = "named_struct('v', '12')") testCommentOnVarcharInContainer( colName = "arr_v", expectedType = "array", goodInsertValue = "array('1')", badInsertValue = "array('12')") testCommentOnVarcharInContainer( colName = "map_vv", expectedType = "map", goodInsertValue = "map('1', '1')", badInsertValue = "map('12', '12')") testCommentOnVarcharInContainer( colName = "map_sv", expectedType = "map", goodInsertValue = "map('123', '1')", badInsertValue = "map('123', '12')") testCommentOnVarcharInContainer( colName = "map_vs", expectedType = "map", goodInsertValue = "map('1', '123')", badInsertValue = "map('12', '123')") testCommentOnVarcharInContainer( colName = "struct_v", expectedType = "struct", goodInsertValue = "named_struct('v', '1')", badInsertValue = "named_struct('v', '12')") } ddlTest("CHANGE COLUMN - set a default value for a varchar column") { withDeltaTable(schema = "v varchar(1)") { tableName => sql(s"ALTER TABLE $tableName " + s"SET TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported')") sql(s"ALTER TABLE $tableName CHANGE COLUMN v set default cast('a' as varchar(1))") val expectedResult = Row("v", "string", null) :: Nil checkAnswer( sql(s"DESCRIBE $tableName").filter("col_name = 'v'"), expectedResult) checkColType(spark.table(tableName).schema.head, VarcharType(1)) } } ddlTest("CHANGE COLUMN - change name (nested)") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("struct", struct("v1", "v2")) withDeltaTable(df) { tableName => assertNotSupported( s"ALTER TABLE $tableName CHANGE COLUMN struct.v2 v3 string") assertNotSupported( s"ALTER TABLE $tableName CHANGE COLUMN struct struct " + "STRUCT") } } ddlTest("CHANGE COLUMN - add a comment (nested)") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("struct", struct("v1", "v2")) withDeltaTable(df) { tableName => sql(s"ALTER TABLE $tableName CHANGE COLUMN struct.v1 v1 integer COMMENT 'a comment'") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("struct", new StructType() .add("v1", "integer", true, "a comment").add("v2", "string"))) assertNotSupported(s"ALTER TABLE $tableName CHANGE COLUMN struct struct " + "STRUCT") } } ddlTest("CHANGE COLUMN - complex types not supported because behavior is ambiguous") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("s", struct("v1", "v2")) .withColumn("a", array("s")) .withColumn("m", map(col("s"), col("s"))) withDeltaTable(df) { tableName => // not supported to add columns assertNotSupported( s"ALTER TABLE $tableName CHANGE COLUMN s s STRUCT") assertNotSupported( s"ALTER TABLE $tableName CHANGE COLUMN a a ARRAY>") assertNotSupported( s"ALTER TABLE $tableName CHANGE COLUMN m m " + "MAP, STRUCT>") // not supported to remove columns. assertNotSupported( s"ALTER TABLE $tableName CHANGE COLUMN s s STRUCT") assertNotSupported( s"ALTER TABLE $tableName CHANGE COLUMN a a ARRAY>") assertNotSupported( s"ALTER TABLE $tableName CHANGE COLUMN m m " + "MAP, STRUCT>") assertNotSupported( s"ALTER TABLE $tableName CHANGE COLUMN m m " + "MAP, STRUCT>") } } private def checkExceptionMessage(e: AnalysisException, messages: String*): Unit = { assert(messages.exists(e.getMessage.contains), s"${e.getMessage} did not contain $messages") } test("CHANGE COLUMN - move unknown column") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") withDeltaTable(df) { tableName => val ex = intercept[AnalysisException] { sql(s"ALTER TABLE $tableName CHANGE COLUMN unknown unknown string FIRST") } checkExceptionMessage( ex, "Missing field unknown", "Cannot update missing field unknown", "A column, variable, or function parameter with name `unknown` cannot be resolved") } } test("CHANGE COLUMN - move unknown column (nested)") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("struct", struct("v1", "v2")) withDeltaTable(df) { tableName => val ex = intercept[AnalysisException] { sql(s"ALTER TABLE $tableName CHANGE COLUMN struct.unknown unknown string FIRST") } checkExceptionMessage( ex, "Missing field struct.unknown", "Cannot update missing field struct.unknown", "A column, variable, or function parameter with name `struct`.`unknown` cannot be resolved") } } test("CHANGE COLUMN - case insensitive") { withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("s", struct("v1", "v2")) withDeltaTable(df) { tableName => val (deltaLog, _) = getDeltaLogWithSnapshot(tableName) sql(s"ALTER TABLE $tableName CHANGE COLUMN V1 v1 integer") assertEqual(deltaLog.update().schema, new StructType() .add("v1", "integer").add("v2", "string") .add("s", new StructType().add("v1", "integer").add("v2", "string"))) sql(s"ALTER TABLE $tableName CHANGE COLUMN v1 V1 integer") assertEqual(deltaLog.update().schema, new StructType() .add("v1", "integer").add("v2", "string") .add("s", new StructType().add("v1", "integer").add("v2", "string"))) sql(s"ALTER TABLE $tableName CHANGE COLUMN v1 v1 integer AFTER V2") assertEqual(deltaLog.update().schema, new StructType() .add("v2", "string").add("v1", "integer") .add("s", new StructType().add("v1", "integer").add("v2", "string"))) // Since the struct doesn't match the case this fails assertNotSupported( s"ALTER TABLE $tableName CHANGE COLUMN s s struct AFTER V2") sql( s"ALTER TABLE $tableName CHANGE COLUMN s s struct AFTER V2") assertEqual(deltaLog.update().schema, new StructType() .add("v2", "string") .add("s", new StructType().add("v1", "integer").add("v2", "string")) .add("v1", "integer")) } } } test("CHANGE COLUMN - case sensitive") { withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { val df = Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("s", struct("v1", "v2")) withDeltaTable(df) { tableName => val ex1 = intercept[AnalysisException] { sql(s"ALTER TABLE $tableName CHANGE COLUMN V1 V1 integer") } checkExceptionMessage( ex1, "Missing field V1", "Cannot update missing field V1", "A column, variable, or function parameter with name `V1` cannot be resolved.") val ex2 = intercept[ParseException] { sql(s"ALTER TABLE $tableName CHANGE COLUMN v1 V1 integer") } assert(ex2.getMessage.contains("Renaming column is not supported")) val ex3 = intercept[AnalysisException] { sql(s"ALTER TABLE $tableName CHANGE COLUMN v1 v1 integer AFTER V2") } checkExceptionMessage( ex2, "Missing field V2", "Couldn't resolve positional argument AFTER V2", "Renaming column is not supported in Hive-style ALTER COLUMN, " + "please run RENAME COLUMN instead") val ex4 = intercept[AnalysisException] { sql(s"ALTER TABLE $tableName CHANGE COLUMN s s struct AFTER v2") } assert(ex4.getMessage.contains("Cannot update")) } } } test("CHANGE COLUMN: allow to change change column from char to string type") { withTable("t") { sql("CREATE TABLE t(i STRING, c CHAR(4)) USING delta") sql("ALTER TABLE t CHANGE COLUMN c TYPE STRING") assert(spark.table("t").schema(1).dataType === StringType) } } test("CHANGE COLUMN: allow to change map key from char to string type") { withTable("t") { sql("CREATE TABLE t(i STRING, m map) USING delta") sql("ALTER TABLE t CHANGE COLUMN m.key TYPE STRING") assert(spark.table("t").schema(1).dataType === MapType(StringType, IntegerType)) } } test("CHANGE COLUMN: allow to change map value from char to string type") { withTable("t") { sql("CREATE TABLE t(i STRING, m map) USING delta") sql("ALTER TABLE t CHANGE COLUMN m.value TYPE STRING") assert(spark.table("t").schema(1).dataType === MapType(IntegerType, StringType)) } } test("CHANGE COLUMN: allow to change array element from char to string type") { withTable("t") { sql("CREATE TABLE t(i STRING, a array) USING delta") sql("ALTER TABLE t CHANGE COLUMN a.element TYPE STRING") assert(spark.table("t").schema(1).dataType === ArrayType(StringType)) } } private def checkColType(f: StructField, dt: DataType): Unit = { assert(f.dataType == CharVarcharUtils.replaceCharVarcharWithString(dt)) assert(CharVarcharUtils.getRawType(f.metadata).contains(dt)) } test("CHANGE COLUMN: allow to change column from char(x) to varchar(y) type x <= y") { withTable("t") { sql("CREATE TABLE t(i STRING, c CHAR(4)) USING delta") sql("ALTER TABLE t CHANGE COLUMN c TYPE VARCHAR(4)") checkColType(spark.table("t").schema(1), VarcharType(4)) } withTable("t") { sql("CREATE TABLE t(i STRING, c CHAR(4)) USING delta") sql("ALTER TABLE t CHANGE COLUMN c TYPE VARCHAR(5)") checkColType(spark.table("t").schema(1), VarcharType(5)) } } test("CHANGE COLUMN: allow to change column from varchar(x) to varchar(y) type x <= y") { withTable("t") { sql("CREATE TABLE t(i STRING, c VARCHAR(4)) USING delta") sql("ALTER TABLE t CHANGE COLUMN c TYPE VARCHAR(4)") checkColType(spark.table("t").schema(1), VarcharType(4)) sql("ALTER TABLE t CHANGE COLUMN c TYPE VARCHAR(5)") checkColType(spark.table("t").schema(1), VarcharType(5)) } } for (charVarcharMitigationDisabled <- BOOLEAN_DOMAIN) test(s"CHANGE COLUMN: allow change from char(x) to string type " + s"[charVarcharMitigationDisabled: $charVarcharMitigationDisabled]") { withSQLConf( DeltaSQLConf.DELTA_BYPASS_CHARVARCHAR_TO_STRING_FIX.key -> charVarcharMitigationDisabled.toString) { withTable("t") { sql("CREATE TABLE t(i VARCHAR(4)) USING delta") sql("ALTER TABLE t CHANGE COLUMN i TYPE STRING") val col = spark.table("t").schema.head assert(col.dataType == StringType) assert(CharVarcharUtils.getRawType(col.metadata).isDefined == charVarcharMitigationDisabled) if (!charVarcharMitigationDisabled) { sql("INSERT INTO t VALUES ('123456789')") } } } } } trait DeltaAlterTableByNameTests extends DeltaAlterTableTests { import testImplicits._ override protected def createTable(schema: String, tblProperties: Map[String, String]): String = { val props = tblProperties.map { case (key, value) => s"'$key' = '$value'" }.mkString(", ") val propsString = if (tblProperties.isEmpty) "" else s" TBLPROPERTIES ($props)" sql(s"CREATE TABLE delta_test ($schema) USING delta$propsString") "delta_test" } override protected def createTable(df: DataFrame, partitionedBy: Seq[String]): String = { df.write.partitionBy(partitionedBy: _*).format("delta").saveAsTable("delta_test") "delta_test" } override protected def dropTable(identifier: String): Unit = { sql(s"DROP TABLE IF EXISTS $identifier") } override protected def getDeltaLogWithSnapshot(identifier: String): (DeltaLog, Snapshot) = { DeltaLog.forTableWithSnapshot(spark, TableIdentifier(identifier)) } test("ADD COLUMNS - external table") { withTempDir { dir => withTable("delta_test") { val path = dir.getCanonicalPath Seq((1, "a"), (2, "b")).toDF("v1", "v2") .write .format("delta") .option("path", path) .saveAsTable("delta_test") checkDatasetUnorderly( spark.table("delta_test").as[(Int, String)], (1, "a"), (2, "b")) sql("ALTER TABLE delta_test ADD COLUMNS (v3 long, v4 double)") val (deltaLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, path) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("v3", "long").add("v4", "double")) checkDatasetUnorderly( spark.table("delta_test").as[(Int, String, Option[Long], Option[Double])], (1, "a", None, None), (2, "b", None, None)) checkDatasetUnorderly( spark.read.format("delta").load(path).as[(Int, String, Option[Long], Option[Double])], (1, "a", None, None), (2, "b", None, None)) } } } // LOCATION tests do not make sense for by path access testQuietly("SET LOCATION") { withTable("delta_table") { spark.range(1).write.format("delta").saveAsTable("delta_table") val catalog = spark.sessionState.catalog val table = catalog.getTableMetadata(TableIdentifier(tableName = "delta_table")) val oldLocation = table.location.toString withTempDir { dir => val path = dir.getCanonicalPath spark.range(1, 2).write.format("delta").save(path) checkAnswer(spark.table("delta_table"), Seq(Row(0))) sql(s"alter table delta_table set location '$path'") checkAnswer(spark.table("delta_table"), Seq(Row(1))) } Utils.deleteRecursively(new File(oldLocation.stripPrefix("file:"))) } } testQuietly("SET LOCATION: external delta table") { withTable("delta_table") { withTempDir { oldDir => spark.range(1).write.format("delta").save(oldDir.getCanonicalPath) sql(s"CREATE TABLE delta_table USING delta LOCATION '${oldDir.getCanonicalPath}'") withTempDir { dir => val path = dir.getCanonicalPath spark.range(1, 2).write.format("delta").save(path) checkAnswer(spark.table("delta_table"), Seq(Row(0))) sql(s"alter table delta_table set location '$path'") checkAnswer(spark.table("delta_table"), Seq(Row(1))) } } } } test( "SET LOCATION - negative cases") { withTable("delta_table") { spark.range(1).write.format("delta").saveAsTable("delta_table") withTempDir { dir => val path = dir.getCanonicalPath val catalog = spark.sessionState.catalog val table = catalog.getTableMetadata(TableIdentifier(tableName = "delta_table")) val oldLocation = table.location.toString // new location is not a delta table var e = intercept[AnalysisException] { sql(s"alter table delta_table set location '$path'") } assert(e.getMessage.contains("not a Delta table")) Seq("1").toDF("id").write.format("delta").save(path) // set location on specific partitions e = intercept[AnalysisException] { sql(s"alter table delta_table partition (id = 1) set location '$path'") } assert(Seq("partition", "not support").forall(e.getMessage.contains)) // schema mismatch e = intercept[AnalysisException] { sql(s"alter table delta_table set location '$path'") } assert(e.getMessage.contains("different than the current table schema")) withSQLConf(DeltaSQLConf.DELTA_ALTER_LOCATION_BYPASS_SCHEMA_CHECK.key -> "true") { checkAnswer(spark.table("delta_table"), Seq(Row(0))) // now we can bypass the schema mismatch check sql(s"alter table delta_table set location '$path'") checkAnswer(spark.table("delta_table"), Seq(Row("1"))) } Utils.deleteRecursively(new File(oldLocation.stripPrefix("file:"))) } } } } /** * For ByPath tests, we select a test case per ALTER TABLE command to simply test identifier * resolution. */ trait DeltaAlterTableByPathTests extends DeltaAlterTableTestBase { override protected def createTable(schema: String, tblProperties: Map[String, String]): String = { val tmpDir = Utils.createTempDir().getCanonicalPath val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tmpDir) // This is a path-based table so we don't need to pass the catalogTable here val txn = deltaLog.startTransaction(None, Some(snapshot)) val metadata = Metadata( schemaString = StructType.fromDDL(schema).json, configuration = tblProperties) txn.commit(metadata :: Nil, DeltaOperations.ManualUpdate) s"delta.`$tmpDir`" } override protected def createTable(df: DataFrame, partitionedBy: Seq[String]): String = { val tmpDir = Utils.createTempDir().getCanonicalPath df.write.format("delta").partitionBy(partitionedBy: _*).save(tmpDir) s"delta.`$tmpDir`" } override protected def dropTable(identifier: String): Unit = { Utils.deleteRecursively(new File(identifier.stripPrefix("delta.`").stripSuffix("`"))) } override protected def getDeltaLogWithSnapshot(identifier: String): (DeltaLog, Snapshot) = { DeltaLog.forTableWithSnapshot(spark, identifier.stripPrefix("delta.`").stripSuffix("`")) } override protected def ddlTest(testName: String)(f: => Unit): Unit = { super.ddlTest(testName)(f) testQuietly(testName + " with delta database") { withDatabase("delta") { spark.sql("CREATE DATABASE delta") f } } } import testImplicits._ ddlTest("SET/UNSET TBLPROPERTIES - simple") { withDeltaTable("v1 int, v2 string") { tableName => sql(s""" |ALTER TABLE $tableName |SET TBLPROPERTIES ( | 'delta.logRetentionDuration' = '2 weeks', | 'delta.checkpointInterval' = '20', | 'key' = 'value' |)""".stripMargin) val (deltaLog, snapshot1) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot1.metadata.configuration, Map( "delta.logRetentionDuration" -> "2 weeks", "delta.checkpointInterval" -> "20", "key" -> "value")) assert(deltaLog.deltaRetentionMillis(snapshot1.metadata) == 2 * 7 * 24 * 60 * 60 * 1000) assert(deltaLog.checkpointInterval(snapshot1.metadata) == 20) sql(s"ALTER TABLE $tableName UNSET TBLPROPERTIES ('delta.checkpointInterval', 'key')") val snapshot2 = deltaLog.update() assertEqual(snapshot2.metadata.configuration, Map("delta.logRetentionDuration" -> "2 weeks")) assert(deltaLog.deltaRetentionMillis(snapshot2.metadata) == 2 * 7 * 24 * 60 * 60 * 1000) assert(deltaLog.checkpointInterval(snapshot2.metadata) == CHECKPOINT_INTERVAL.fromString(CHECKPOINT_INTERVAL.defaultValue)) } } ddlTest("ADD COLUMNS - simple") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2")) { tableName => checkDatasetUnorderly( spark.table(tableName).as[(Int, String)], (1, "a"), (2, "b")) sql(s"ALTER TABLE $tableName ADD COLUMNS (v3 long, v4 double)") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer").add("v2", "string") .add("v3", "long").add("v4", "double")) checkDatasetUnorderly( spark.table(tableName).as[(Int, String, Option[Long], Option[Double])], (1, "a", None, None), (2, "b", None, None)) } } ddlTest("CHANGE COLUMN - add a comment") { withDeltaTable(Seq((1, "a"), (2, "b")).toDF("v1", "v2")) { tableName => sql(s"ALTER TABLE $tableName CHANGE COLUMN v1 v1 integer COMMENT 'a comment'") val (deltaLog, snapshot) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot.schema, new StructType() .add("v1", "integer", true, "a comment").add("v2", "string")) } } test("SET LOCATION is not supported for path based tables") { val df = spark.range(1).toDF() withDeltaTable(df) { identifier => withTempDir { dir => val path = dir.getCanonicalPath val e = intercept[DeltaAnalysisException] { sql(s"alter table $identifier set location '$path'") } assert(e.getErrorClass == "DELTA_CANNOT_SET_LOCATION_ON_PATH_IDENTIFIER") assert(e.getSqlState == "42613") assert(e.getMessage == "[DELTA_CANNOT_SET_LOCATION_ON_PATH_IDENTIFIER] " + "Cannot change the location of a path based table.") } } } } class DeltaAlterTableByNameSuite extends DeltaAlterTableByNameTests with DeltaSQLCommandTest { ddlTest("SET/UNSET TBLPROPERTIES - unset non-existent config value should still" + "unset the config if key matches") { val props = Map( "delta.randomizeFilePrefixes" -> "true", "delta.randomPrefixLength" -> "5", "key" -> "value" ) withDeltaTable("v1 int, v2 string", props) { tableName => sql(s"ALTER TABLE $tableName UNSET TBLPROPERTIES ('delta.randomizeFilePrefixes', 'kEy')") val (deltaLog, snapshot1) = getDeltaLogWithSnapshot(tableName) assertEqual(snapshot1.metadata.configuration, Map( "delta.randomPrefixLength" -> "5", "key" -> "value")) sql(s"ALTER TABLE $tableName UNSET TBLPROPERTIES IF EXISTS " + "('delta.randomizeFilePrefixes', 'kEy')") val snapshot2 = deltaLog.update() assertEqual(snapshot2.metadata.configuration, Map("delta.randomPrefixLength" -> "5", "key" -> "value")) } } } class DeltaAlterTableByPathSuite extends DeltaAlterTableByPathTests with DeltaSQLCommandTest with DeltaAlterTableReplaceTests trait DeltaAlterTableColumnMappingSelectedTests extends DeltaColumnMappingSelectedTestMixin { override protected def runOnlyTests = Seq( "ADD COLUMNS into complex types - Array", "CHANGE COLUMN - move to first (nested)", "CHANGE COLUMN - case insensitive") } class DeltaAlterTableByNameIdColumnMappingSuite extends DeltaAlterTableByNameSuite with DeltaColumnMappingEnableIdMode with DeltaAlterTableColumnMappingSelectedTests class DeltaAlterTableByPathIdColumnMappingSuite extends DeltaAlterTableByPathSuite with DeltaColumnMappingEnableIdMode with DeltaAlterTableColumnMappingSelectedTests class DeltaAlterTableByNameNameColumnMappingSuite extends DeltaAlterTableByNameSuite with DeltaColumnMappingEnableNameMode with DeltaAlterTableColumnMappingSelectedTests class DeltaAlterTableByPathNameColumnMappingSuite extends DeltaAlterTableByPathSuite with DeltaColumnMappingEnableNameMode with DeltaAlterTableColumnMappingSelectedTests ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaArbitraryColumnNameSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.collection.JavaConverters._ import org.scalatest.GivenWhenThen import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StringType, StructType} trait DeltaArbitraryColumnNameSuiteBase extends DeltaColumnMappingSuiteUtils { protected val simpleNestedSchema = new StructType() .add("a", StringType, true) .add("b", new StructType() .add("c", StringType, true) .add("d", IntegerType, true)) .add("map", MapType(StringType, StringType), true) .add("arr", ArrayType(IntegerType), true) protected val simpleNestedSchemaWithDuplicatedNestedColumnName = new StructType() .add("a", new StructType() .add("c", StringType, true) .add("d", IntegerType, true), true) .add("b", new StructType() .add("c", StringType, true) .add("d", IntegerType, true), true) .add("map", MapType(StringType, StringType), true) .add("arr", ArrayType(IntegerType), true) protected val nestedSchema = new StructType() .add(colName("a"), StringType, true) .add(colName("b"), new StructType() .add(colName("c"), StringType, true) .add(colName("d"), IntegerType, true)) .add(colName("map"), MapType(StringType, StringType), true) .add(colName("arr"), ArrayType(IntegerType), true) protected def simpleNestedData = spark.createDataFrame( Seq( Row("str1", Row("str1.1", 1), Map("k1" -> "v1"), Array(1, 11)), Row("str2", Row("str1.2", 2), Map("k2" -> "v2"), Array(2, 22))).asJava, simpleNestedSchema) protected def simpleNestedDataWithDuplicatedNestedColumnName = spark.createDataFrame( Seq( Row(Row("str1", 1), Row("str1.1", 1), Map("k1" -> "v1"), Array(1, 11)), Row(Row("str2", 2), Row("str1.2", 2), Map("k2" -> "v2"), Array(2, 22))).asJava, simpleNestedSchemaWithDuplicatedNestedColumnName) protected def nestedData = spark.createDataFrame( Seq( Row("str1", Row("str1.1", 1), Map("k1" -> "v1"), Array(1, 11)), Row("str2", Row("str1.2", 2), Map("k2" -> "v2"), Array(2, 22))).asJava, nestedSchema) // TODO: Refactor DeltaColumnMappingSuite and consolidate these table creation methods between // the two suites. protected def createTableWithSQLCreateOrReplaceAPI( tableName: String, data: DataFrame, props: Map[String, String] = Map.empty, partCols: Seq[String] = Nil): Unit = { withTable("source") { createTableWithDataFrameWriterV2API( "source", data, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode(props))) spark.sql( s""" |CREATE OR REPLACE TABLE $tableName |USING DELTA |${partitionStmt(partCols)} |${propString(props)} |AS SELECT * FROM source |""".stripMargin) } } protected def createTableWithSQLAPI( tableName: String, data: DataFrame, props: Map[String, String] = Map.empty, partCols: Seq[String] = Nil): Unit = { withTable("source") { spark.sql( s""" |CREATE TABLE $tableName (${data.schema.toDDL}) |USING DELTA |${partitionStmt(partCols)} |${propString(props)} |""".stripMargin) data.write.format("delta").mode("append").saveAsTable(tableName) } } protected def createTableWithCTAS( tableName: String, data: DataFrame, props: Map[String, String] = Map.empty, partCols: Seq[String] = Nil): Unit = { withTable("source") { createTableWithDataFrameWriterV2API( "source", data, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode(props))) spark.sql( s""" |CREATE TABLE $tableName |USING DELTA |${partitionStmt(partCols)} |${propString(props)} |AS SELECT * FROM source |""".stripMargin) } } protected def createTableWithDataFrameAPI( tableName: String, data: DataFrame, props: Map[String, String] = Map.empty, partCols: Seq[String]): Unit = { val sqlConfs = props.map { case (key, value) => "spark.databricks.delta.properties.defaults." + key.stripPrefix("delta.") -> value } withSQLConf(sqlConfs.toList: _*) { if (partCols.nonEmpty) { data.write.format("delta") .partitionBy(partCols.map(name => s"`$name`"): _*).saveAsTable(tableName) } else { data.write.format("delta").saveAsTable(tableName) } } } protected def createTableWithDataFrameWriterV2API( tableName: String, data: DataFrame, props: Map[String, String] = Map.empty, partCols: Seq[String] = Seq.empty): Unit = { val writer = data.writeTo(tableName).using("delta") props.foreach(prop => writer.tableProperty(prop._1, prop._2)) val partColumns = partCols.map(name => expr(s"`$name`")) if (partCols.nonEmpty) writer.partitionedBy(partColumns.head, partColumns.tail: _*) writer.create() } protected def assertException(message: String)(block: => Unit): Unit = { val e = intercept[Exception](block) assert(e.getMessage.contains(message)) } protected def assertExceptionOneOf(messages: Seq[String])(block: => Unit): Unit = { val e = intercept[Exception](block) assert(messages.exists(x => e.getMessage.contains(x))) } } class DeltaArbitraryColumnNameSuite extends QueryTest with DeltaArbitraryColumnNameSuiteBase with GivenWhenThen { private def testCreateTable(): Unit = { val allProps = supportedModes .map(mode => Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode)) ++ // none mode Seq(Map.empty[String, String]) def withProps(props: Map[String, String])(createFunc: => Unit) = { withTable("t1") { if (mode(props) != "none") { createFunc checkAnswer(spark.table("t1"), nestedData) } else { val e = intercept[AnalysisException] { createFunc } assert(e.getMessage.contains("Found invalid character(s)")) } } } allProps.foreach { props => withProps(props) { Given(s"with SQL CREATE TABLE API, mode ${mode(props)}") createTableWithSQLAPI("t1", nestedData, props, partCols = Seq(colName("a"))) } withProps(props) { Given(s"with SQL CTAS API, mode ${mode(props)}") createTableWithCTAS("t1", nestedData, props, partCols = Seq(colName("a")) ) } withProps(props) { Given(s"with SQL CREATE OR REPLACE TABLE API, mode ${mode(props)}") createTableWithSQLCreateOrReplaceAPI("t1", nestedData, props, partCols = Seq(colName("a"))) } withProps(props) { Given(s"with DataFrame API, mode ${mode(props)}") createTableWithDataFrameAPI("t1", nestedData, props, partCols = Seq(colName("a"))) } withProps(props) { Given(s"with DataFrameWriterV2 API, mode ${mode(props)}") createTableWithDataFrameWriterV2API("t1", nestedData, props, // TODO: make DataFrameWriterV2 work with arbitrary partition column names partCols = Seq.empty) } } } test("create table") { testCreateTable() } testColumnMapping("schema evolution and simple query") { mode => withTable("t1") { createTableWithSQLAPI("t1", nestedData, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode), partCols = Seq(colName("a")) ) val newNestedData = spark.createDataFrame( Seq(Row("str3", Row("str1.3", 3), Map("k3" -> "v3"), Array(3, 33), "new value")).asJava, nestedSchema.add(colName("e"), StringType)) newNestedData.write.format("delta") .option("mergeSchema", "true") .mode("append").saveAsTable("t1") checkAnswer( spark.table("t1"), Seq( Row("str1", Row("str1.1", 1), Map("k1" -> "v1"), Array(1, 11), null), Row("str2", Row("str1.2", 2), Map("k2" -> "v2"), Array(2, 22), null), Row("str3", Row("str1.3", 3), Map("k3" -> "v3"), Array(3, 33), "new value"))) val colA = colName("a") val colB = colName("b") val colC = colName("c") val colD = colName("d") checkAnswer( spark.table("t1") .where(s"`$colA` > 'str1'") .where(s"`$colB`.`$colD` < 3") .select(s"`$colB`.`$colC`"), Row("str1.2")) } } testColumnMapping("alter table add and replace columns") { mode => withTable("t1") { createTableWithSQLAPI("t1", nestedData, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode), partCols = Seq(colName("a")) ) spark.sql(s"alter table t1 add columns (`${colName("e")}` string)") spark.sql("insert into t1 " + "values ('str3', struct('str1.3', 3), map('k3', 'v3'), array(3, 33), 'new value')") checkAnswer( spark.table("t1"), Seq( Row("str1", Row("str1.1", 1), Map("k1" -> "v1"), Array(1, 11), null), Row("str2", Row("str1.2", 2), Map("k2" -> "v2"), Array(2, 22), null), Row("str3", Row("str1.3", 3), Map("k3" -> "v3"), Array(3, 33), "new value"))) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCColumnMappingSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.DeltaOperations.ManualUpdate import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils import org.apache.spark.sql.delta.commands.cdc.CDCReader._ import org.apache.spark.sql.delta.schema.SchemaMergingUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaColumnMappingSelectedTestMixin import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.{DataFrame, Row} import org.apache.spark.sql.functions._ import org.apache.spark.sql.types._ trait DeltaCDCColumnMappingSuiteBase extends DeltaCDCSuiteBase with DeltaColumnMappingTestUtils with DeltaColumnMappingSelectedTestMixin { import testImplicits._ implicit class DataFrameDropCDCFields(df: DataFrame) { def dropCDCFields: DataFrame = df.drop(CDC_COMMIT_TIMESTAMP) .drop(CDC_TYPE_COLUMN_NAME) .drop(CDC_COMMIT_VERSION) } test("upgrade to column mapping not blocked") { withTempDir { dir => setupInitialDeltaTable(dir, upgradeInNameMode = true) implicit val deltaLog: DeltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) val v1 = deltaLog.update().version checkAnswer( cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion(v1.toString), Some(BatchCDFSchemaEndVersion)).dropCDCFields, (0 until 10).map(_.toString).map(i => Row(i, i)) ) } } test("add column batch cdc read not blocked") { withTempDir { dir => // Set up an initial table with 10 records in schema setupInitialDeltaTable(dir) implicit val deltaLog: DeltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) // add column should not be blocked sql(s"ALTER TABLE delta.`${dir.getCanonicalPath}` ADD COLUMN (name string)") // write more data writeDeltaData((10 until 15)) // None of the schema mode should block this use case Seq(BatchCDFSchemaLegacy, BatchCDFSchemaLatest, BatchCDFSchemaEndVersion).foreach { mode => checkAnswer( cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion(deltaLog.update().version.toString), Some(mode)).dropCDCFields, (0 until 10).map(_.toString).toDF("id") .withColumn("value", col("id")) .withColumn("name", lit(null)) union (10 until 15).map(_.toString).toDF("id") .withColumn("value", col("id")) .withColumn("name", col("id"))) } } } test("data type and nullability change batch cdc read blocked") { withTempDir { dir => // Set up an initial table with 10 records in schema setupInitialDeltaTable(dir) implicit val deltaLog: DeltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) val s1 = deltaLog.update() val v1 = s1.version // Change the data type of column deltaLog.withNewTransaction { txn => // id was string val updatedSchema = SchemaMergingUtils.transformColumns( StructType.fromDDL("id INT, value STRING")) { (_, field, _) => val refField = s1.metadata.schema(field.name) field.copy(metadata = refField.metadata) } txn.commit(s1.metadata.copy(schemaString = updatedSchema.json) :: Nil, ManualUpdate) } val v2 = deltaLog.update().version // write more data in updated schema Seq((10, "10")).toDF("id", "value") .write.format("delta").mode("append").save(dir.getCanonicalPath) val v3 = deltaLog.update().version // query all changes using latest schema blocked assertBlocked( expectedIncompatSchemaVersion = 0, expectedReadSchemaVersion = v3, bySchemaChange = false) { cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion(v3.toString)).collect() } // query using end version also blocked if cross schema change assertBlocked( expectedIncompatSchemaVersion = 0, expectedReadSchemaVersion = v3, schemaMode = BatchCDFSchemaEndVersion, bySchemaChange = false) { cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion(v3.toString), Some(BatchCDFSchemaEndVersion)).collect() } // query using end version NOT blocked if NOT cross schema change checkAnswer( cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion(v3.toString), EndingVersion(v3.toString), Some(BatchCDFSchemaEndVersion)).dropCDCFields, Row(10, "10") :: Nil ) val s2 = deltaLog.update() // Change nullability unsafely deltaLog.withNewTransaction { txn => // the schema was nullable, but we want to make it non-nullable val updatedSchema = SchemaMergingUtils.transformColumns( StructType.fromDDL("id INT, value string").asNullable) { (_, field, _) => val refField = s1.metadata.schema(field.name) field.copy(metadata = refField.metadata, nullable = false) } txn.commit(s2.metadata.copy(schemaString = updatedSchema.json) :: Nil, ManualUpdate) } val v4 = deltaLog.update().version // write more data in updated schema Seq((11, "11")).toDF("id", "value") .write.format("delta").mode("append").save(dir.getCanonicalPath) val v5 = deltaLog.update().version // query changes using latest schema blocked // Note this is not detected as an illegal schema change, but a data violation, because // we attempt to read using latest schema @ v5 (nullable=false) to read some past data @ v3 // (nullable=true), which is unsafe. assertBlocked( expectedIncompatSchemaVersion = v3, expectedReadSchemaVersion = v5, bySchemaChange = false) { cdcRead( new TablePath(dir.getCanonicalPath), // v3 is the first version post the data type schema change StartingVersion(v3.toString), EndingVersion(v5.toString)).collect() } // query using end version also blocked if cross schema change assertBlocked( expectedIncompatSchemaVersion = v3, expectedReadSchemaVersion = v5, schemaMode = BatchCDFSchemaEndVersion, bySchemaChange = false) { cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion(v3.toString), EndingVersion(v5.toString), Some(BatchCDFSchemaEndVersion)).collect() } // query using end version NOT blocked if NOT cross schema change checkAnswer( cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion(v5.toString), EndingVersion(v5.toString), Some(BatchCDFSchemaEndVersion)).dropCDCFields, Row(11, "11") :: Nil ) } } test("overwrite table with invalid schema change in non-column mapping table is blocked") { withTempDir { dir => withColumnMappingConf("none") { // Create table action sequence Seq((1, "a")).toDF("id", "name").write.format("delta").save(dir.getCanonicalPath) implicit val log: DeltaLog = DeltaLog.forTable(spark, dir) val v1 = log.update().version // Overwrite with dropped column Seq(2).toDF("id") .write .format("delta") .mode("overwrite") .option("overwriteSchema", "true") .save(dir.getCanonicalPath) val v2 = log.update().version assertBlocked( expectedIncompatSchemaVersion = v1, expectedReadSchemaVersion = v2, bySchemaChange = false) { cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion(v1.toString), EndingVersion(v2.toString), schemaMode = Some(BatchCDFSchemaEndVersion)).collect() } // Overwrite with a renamed column Seq(3).toDF("id2") .write .format("delta") .mode("overwrite") .option("overwriteSchema", "true") .save(dir.getCanonicalPath) val v3 = log.update().version assertBlocked( expectedIncompatSchemaVersion = v2, expectedReadSchemaVersion = v3, bySchemaChange = false) { cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion(v2.toString), EndingVersion(v3.toString)).collect() } } } } test("drop column batch cdc read blocked") { withTempDir { dir => // Set up an initial table with 10 records in schema setupInitialDeltaTable(dir) implicit val deltaLog: DeltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) val v1 = deltaLog.update().version // drop column would cause CDC read to be blocked sql(s"ALTER TABLE delta.`${dir.getCanonicalPath}` DROP COLUMN value") val v2 = deltaLog.update().version // write more data writeDeltaData(Seq(10)) val v3 = deltaLog.update().version // query all changes using latest schema blocked assertBlocked( expectedIncompatSchemaVersion = 0, expectedReadSchemaVersion = v3, bySchemaChange = false) { cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion(v3.toString)).collect() } // query just first two versions which have more columns than latest schema is also blocked assertBlocked( expectedIncompatSchemaVersion = 0, expectedReadSchemaVersion = v3, bySchemaChange = false) { cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion("1")).collect() } // query unblocked if force enabled by user withSQLConf( DeltaSQLConf.DELTA_CDF_UNSAFE_BATCH_READ_ON_INCOMPATIBLE_SCHEMA_CHANGES.key -> "true") { checkAnswer( cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion(v3.toString)).dropCDCFields, // Note id is dropped because we are using latest schema (0 until 11).map(i => Row(i.toString)) ) } // querying changes using endVersion schema blocked if crossing schema boundary assertBlocked( expectedIncompatSchemaVersion = 0, expectedReadSchemaVersion = v3, schemaMode = BatchCDFSchemaEndVersion, bySchemaChange = false) { cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion(v3.toString), Some(BatchCDFSchemaEndVersion)).collect() } assertBlocked( expectedIncompatSchemaVersion = v1, expectedReadSchemaVersion = v3, schemaMode = BatchCDFSchemaEndVersion, bySchemaChange = false) { cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion(v1.toString), EndingVersion(v3.toString), Some(BatchCDFSchemaEndVersion)).collect() } // querying changes using endVersion schema NOT blocked if NOT crossing schema boundary // with schema checkAnswer( cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion(v1.toString), Some(BatchCDFSchemaEndVersion)).dropCDCFields, (0 until 10).map(_.toString).map(i => Row(i, i))) // with schema checkAnswer( cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion(v3.toString), EndingVersion(v3.toString), Some(BatchCDFSchemaEndVersion)).dropCDCFields, Row("10") :: Nil ) // let's add the column back... sql(s"ALTER TABLE delta.`${dir.getCanonicalPath}` ADD COLUMN (value string)") val v4 = deltaLog.update().version // write more data writeDeltaData(Seq(11)) val v5 = deltaLog.update().version // The read is still blocked, even schema @ 0 looks the "same" as the latest schema // but the added column now maps to a different physical column. // Note that this bypasses all the schema change actions in between because: // 1. The schema after dropping @ v2 is a subset of the read schema -> this is fine // 2. The schema after adding back @ v4 is the same as latest schema -> this is fine // but our final check against the starting schema would catch it. assertBlocked( expectedIncompatSchemaVersion = 0, expectedReadSchemaVersion = v5, bySchemaChange = false) { cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion(v5.toString)).collect() } // In this case, tho there aren't any read-incompat schema changes in the querying range, // the latest schema is not read-compat with the data files @ v0, so we still block. assertBlocked( expectedIncompatSchemaVersion = 0, expectedReadSchemaVersion = v5, bySchemaChange = false) { cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion("1")).collect() } } } test("rename column batch cdc read blocked") { withTempDir { dir => // Set up an initial table with 10 records in schema setupInitialDeltaTable(dir) implicit val deltaLog: DeltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) val v1 = deltaLog.update().version // Rename column sql(s"ALTER TABLE delta.`${dir.getCanonicalPath}` RENAME COLUMN id TO id2") val v2 = deltaLog.update().version // write more data writeDeltaData(Seq(10)) val v3 = deltaLog.update().version // query all versions using latest schema blocked assertBlocked( expectedIncompatSchemaVersion = 0, expectedReadSchemaVersion = v3, bySchemaChange = false) { cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion(v3.toString)).collect() } // query unblocked if force enabled by user withSQLConf( DeltaSQLConf.DELTA_CDF_UNSAFE_BATCH_READ_ON_INCOMPATIBLE_SCHEMA_CHANGES.key -> "true") { val df = cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion(v3.toString)).dropCDCFields checkAnswer(df, (0 until 11).map(i => Row(i.toString, i.toString))) // Note we serve the batch using the renamed column in the latest schema. assert(df.schema.fieldNames.sameElements(Array("id2", "value"))) } // query just the first few versions using latest schema also blocked assertBlocked( expectedIncompatSchemaVersion = 0, expectedReadSchemaVersion = v3, bySchemaChange = false) { cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion("1")).collect() } // query using endVersion schema across schema boundary also blocked assertBlocked( expectedIncompatSchemaVersion = 0, expectedReadSchemaVersion = v2, schemaMode = BatchCDFSchemaEndVersion, bySchemaChange = false) { cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion(v2.toString), Some(BatchCDFSchemaEndVersion)).collect() } // query using endVersion schema NOT blocked if NOT crossing schema boundary checkAnswer( cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion(v1.toString), Some(BatchCDFSchemaEndVersion)).dropCDCFields, (0 until 10).map(_.toString).map(i => Row(i, i)) ) checkAnswer( cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion(v2.toString), EndingVersion(v3.toString), Some(BatchCDFSchemaEndVersion)).dropCDCFields, Row("10", "10") :: Nil ) // Let's rename the column back sql(s"ALTER TABLE delta.`${dir.getCanonicalPath}` RENAME COLUMN id2 TO id") val v4 = deltaLog.update().version // write more data writeDeltaData(Seq(11)) val v5 = deltaLog.update().version // query all changes using latest schema would still block because we crossed an // intermediary action with a conflicting schema (the first rename). assertBlocked(expectedIncompatSchemaVersion = v2, expectedReadSchemaVersion = v5) { cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion(v5.toString)).collect() } // query all changes using LATEST schema would NOT block if we exclude the first // rename back, because the data schemas before that are now consistent with the latest. checkAnswer( cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion(v1.toString)).dropCDCFields, (0 until 10).map(_.toString).map(i => Row(i, i))) // query using endVersion schema is blocked if we cross schema boundary assertBlocked( expectedIncompatSchemaVersion = v3, expectedReadSchemaVersion = v5, schemaMode = BatchCDFSchemaEndVersion, bySchemaChange = false) { cdcRead( new TablePath(dir.getCanonicalPath), // v3 just pass the first schema change StartingVersion(v3.toString), EndingVersion(v5.toString), Some(BatchCDFSchemaEndVersion)).collect() } // Note how the conflictingVersion is v2 (the first rename), because v1 matches our end // version schema due to renaming back. assertBlocked( expectedIncompatSchemaVersion = v2, expectedReadSchemaVersion = v5, schemaMode = BatchCDFSchemaEndVersion) { cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion(v1.toString), EndingVersion(v5.toString), Some(BatchCDFSchemaEndVersion)).collect() } } } override def runOnlyTests: Seq[String] = Seq( "changes from table by name", "changes from table by path", "batch write: append, dynamic partition overwrite + CDF", // incompatible schema changes & schema mode tests "add column batch cdc read not blocked", "data type and nullability change batch cdc read blocked", "drop column batch cdc read blocked", "rename column batch cdc read blocked", "filters with special characters in name should be pushed down" ) protected def assertBlocked( expectedIncompatSchemaVersion: Long, expectedReadSchemaVersion: Long, schemaMode: DeltaBatchCDFSchemaMode = BatchCDFSchemaLegacy, timeTravel: Boolean = false, bySchemaChange: Boolean = true)(f: => Unit)(implicit log: DeltaLog): Unit = { val e = intercept[DeltaUnsupportedOperationException] { f } val (end, readSchemaJson) = if (bySchemaChange) { assert(e.getErrorClass == "DELTA_CHANGE_DATA_FEED_INCOMPATIBLE_SCHEMA_CHANGE") val Seq(_, end, readSchemaJson, readSchemaVersion, incompatibleVersion, _, _, _, _) = e.getMessageParametersArray.toSeq assert(incompatibleVersion.toLong == expectedIncompatSchemaVersion) assert(readSchemaVersion.toLong == expectedReadSchemaVersion) (end, readSchemaJson) } else { assert(e.getErrorClass == "DELTA_CHANGE_DATA_FEED_INCOMPATIBLE_DATA_SCHEMA") val Seq(_, end, readSchemaJson, readSchemaVersion, incompatibleVersion, config) = e.getMessageParametersArray.toSeq assert(incompatibleVersion.toLong == expectedIncompatSchemaVersion) assert(readSchemaVersion.toLong == expectedReadSchemaVersion) assert(config == DeltaSQLConf.DELTA_CDF_DEFAULT_SCHEMA_MODE_FOR_COLUMN_MAPPING_TABLE.key) (end, readSchemaJson) } val latestSnapshot = log.update() schemaMode match { case BatchCDFSchemaLegacy if timeTravel => // Read using time travelled schema, it can be arbitrary so nothing to check here case BatchCDFSchemaEndVersion => // Read using end version schema assert(expectedReadSchemaVersion == end.toLong && log.getSnapshotAt(expectedReadSchemaVersion).schema.json == readSchemaJson) case _ => // non time-travel legacy mode and latest mode should both read latest schema assert(expectedReadSchemaVersion == latestSnapshot.version && latestSnapshot.schema.json == readSchemaJson) } } /** * Write test delta data to test blocking column mapping for CDC batch queries, it takes a * sequence and write out as a row of strings, assuming the delta log's schema are all strings. */ protected def writeDeltaData( data: Seq[Int], userSpecifiedSchema: Option[StructType] = None)(implicit log: DeltaLog): Unit = { val schema = userSpecifiedSchema.getOrElse(log.update().schema) data.foreach { i => val data = Seq(Row(schema.map(_ => i.toString): _*)) spark.createDataFrame(data.asJava, schema) .write.format("delta").mode("append").save(log.dataPath.toString) } } /** * Set up initial table data, considering current column mapping mode * * The table contains 10 rows, with schema both are string */ protected def setupInitialDeltaTable(dir: File, upgradeInNameMode: Boolean = false): Unit = { require(columnMappingModeString != NoMapping.name) val tablePath = dir.getCanonicalPath implicit val deltaLog: DeltaLog = DeltaLog.forTable(spark, tablePath) if (upgradeInNameMode && columnMappingModeString == NameMapping.name) { // For name mode, we do an upgrade then write to test that behavior as well // init table with 5 versions without column mapping withColumnMappingConf("none") { writeDeltaData((0 until 5), userSpecifiedSchema = Some( new StructType().add("id", StringType, true).add("value", StringType, true) )) } // upgrade to name mode val protocol = deltaLog.snapshot.protocol val (r, w) = if (protocol.supportsReaderFeatures || protocol.supportsWriterFeatures) { (TableFeatureProtocolUtils.TABLE_FEATURES_MIN_READER_VERSION, TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION) } else { (ColumnMappingTableFeature.minReaderVersion, ColumnMappingTableFeature.minWriterVersion) } sql( s""" |ALTER TABLE delta.`${dir.getCanonicalPath}` |SET TBLPROPERTIES ( | ${DeltaConfigs.COLUMN_MAPPING_MODE.key} = "name", | ${DeltaConfigs.MIN_READER_VERSION.key} = "$r", | ${DeltaConfigs.MIN_WRITER_VERSION.key} = "$w")""".stripMargin) // write more data writeDeltaData((5 until 10)) } else { // For id mode and non-upgrade name mode, we could just create a table from scratch withColumnMappingConf(columnMappingModeString) { writeDeltaData((0 until 10), userSpecifiedSchema = Some( new StructType().add("id", StringType, true).add("value", StringType, true) )) } } checkAnswer( cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion(deltaLog.update().version.toString)).dropCDCFields, (0 until 10).map(_.toString).toDF("id").withColumn("value", col("id"))) } test("filters with special characters in name should be pushed down") { val tblName = "tbl" withTable(tblName) { spark.range(end = 10).withColumn("id with space", col("id")) .write.format("delta").saveAsTable(tblName) val plans = DeltaTestUtils.withAllPlansCaptured(spark) { val res = cdcRead(new TableName(tblName), StartingVersion("0"), EndingVersion("1")) .select("id with space", "_change_type") .where(col("id with space") < lit(5)) assert(res.columns === Seq("id with space", "_change_type")) checkAnswer( res, spark.range(end = 5) .withColumn("_change_type", lit("insert"))) } assert(plans.map(_.executedPlan).toString .contains("PushedFilters: [*IsNotNull(id with space), *LessThan(id with space,5)]")) } } } trait DeltaCDCColumnMappingScalaSuiteBase extends DeltaCDCColumnMappingSuiteBase { import testImplicits._ test("time travel with batch cdf is disbaled by default") { withTempDir { dir => Seq(1).toDF("id").write.format("delta").save(dir.getCanonicalPath) val e = intercept[DeltaAnalysisException] { cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion("1"), readerOptions = Map(DeltaOptions.VERSION_AS_OF -> "0")).collect() } assert(e.getErrorClass == "DELTA_UNSUPPORTED_TIME_TRAVEL_VIEWS") } } // NOTE: we do not support time travel option with SQL API, so we will just test Scala API suite test("cannot specify both time travel options and schema mode") { withSQLConf(DeltaSQLConf.DELTA_CDF_ALLOW_TIME_TRAVEL_OPTIONS.key -> "true") { withTempDir { dir => Seq(1).toDF("id").write.format("delta").save(dir.getCanonicalPath) val e = intercept[DeltaIllegalArgumentException] { cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion("1"), Some(BatchCDFSchemaEndVersion), readerOptions = Map(DeltaOptions.VERSION_AS_OF -> "0")).collect() } assert(e.getMessage.contains( DeltaSQLConf.DELTA_CDF_DEFAULT_SCHEMA_MODE_FOR_COLUMN_MAPPING_TABLE.key)) } } } test("time travel option is respected") { withSQLConf(DeltaSQLConf.DELTA_CDF_ALLOW_TIME_TRAVEL_OPTIONS.key -> "true") { withTempDir { dir => // Set up an initial table with 10 records in schema setupInitialDeltaTable(dir) implicit val deltaLog: DeltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) val v1 = deltaLog.update().version // Add a column sql(s"ALTER TABLE delta.`${dir.getCanonicalPath}` ADD COLUMN (prop string)") val v2 = deltaLog.update().version // write more data writeDeltaData(Seq(10)) val v3 = deltaLog.update().version // Rename a column sql(s"ALTER TABLE delta.`${dir.getCanonicalPath}` RENAME COLUMN id TO id2") val v4 = deltaLog.update().version // write more data writeDeltaData(Seq(11)) val v5 = deltaLog.update().version // query changes between version 0 - v1, not crossing schema boundary checkAnswer( cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion(v1.toString), readerOptions = Map(DeltaOptions.VERSION_AS_OF -> v1.toString)).dropCDCFields, (0 until 10).map(_.toString).map(i => Row(i, i))) // query across add column, but not cross the rename, not blocked checkAnswer( cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), EndingVersion(v3.toString), // v2 is the add column schema change readerOptions = Map(DeltaOptions.VERSION_AS_OF -> v2.toString)).dropCDCFields, // Note how the first 10 records now misses a column, but it's fine (0 until 10).map(_.toString).map(i => Row(i, i, null)) ++ Seq(Row("10", "10", "10"))) // query across rename is blocked, if we are still specifying an old version // note it failed at v4, because the initial schema does not conflict with schema @ v2 assertBlocked( expectedIncompatSchemaVersion = v4, expectedReadSchemaVersion = v2, timeTravel = true) { cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), // v5 cross the v4 rename column EndingVersion(v5.toString), // v2 is the add column schema change readerOptions = Map(DeltaOptions.VERSION_AS_OF -> v2.toString)).collect() } // Even the querying range has no schema change, the data files are still not // compatible with the read schema due to arbitrary time travel. assertBlocked( expectedIncompatSchemaVersion = 0, expectedReadSchemaVersion = v4, timeTravel = true, bySchemaChange = false) { cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion("0"), // v1 still uses the schema prior to the rename EndingVersion(v1.toString), // v4 is the rename column change readerOptions = Map(DeltaOptions.VERSION_AS_OF -> v4.toString)).collect() } // But without crossing schema change boundary (v4 - v5) using v4's renamed schema, // we can load the batch. checkAnswer( cdcRead( new TablePath(dir.getCanonicalPath), StartingVersion(v4.toString), EndingVersion(v5.toString), readerOptions = Map(DeltaOptions.VERSION_AS_OF -> v4.toString)).dropCDCFields, Seq(Row("11", "11", "11"))) } } } } class DeltaCDCIdColumnMappingSuite extends DeltaCDCScalaSuite with DeltaCDCColumnMappingScalaSuiteBase with DeltaColumnMappingEnableIdMode class DeltaCDCNameColumnMappingSuite extends DeltaCDCScalaSuite with DeltaCDCColumnMappingScalaSuiteBase with DeltaColumnMappingEnableNameMode class DeltaCDCSQLIdColumnMappingSuite extends DeltaCDCSQLSuite with DeltaCDCColumnMappingSuiteBase with DeltaColumnMappingEnableIdMode class DeltaCDCSQLNameColumnMappingSuite extends DeltaCDCSQLSuite with DeltaCDCColumnMappingSuiteBase with DeltaColumnMappingEnableNameMode ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSQLSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.Date import org.apache.spark.sql.delta.DeltaTestUtils.modifyCommitTimestamp import org.apache.spark.sql.delta.actions.Protocol import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTableUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.{AnalysisException, DataFrame} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._ import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.LongType class DeltaCDCSQLSuite extends DeltaCDCSuiteBase with DeltaColumnMappingTestUtils { /** Single method to do all kinds of CDC reads */ def cdcRead( tblId: TblId, start: Boundary, end: Boundary, schemaMode: Option[DeltaBatchCDFSchemaMode] = Some(BatchCDFSchemaLegacy), // SQL API does not support generic reader options, so it's a noop here readerOptions: Map[String, String] = Map.empty): DataFrame = { // Set the batch CDF schema mode using SQL conf if we specified it if (schemaMode.isDefined) { var result: DataFrame = null withSQLConf(DeltaSQLConf.DELTA_CDF_DEFAULT_SCHEMA_MODE_FOR_COLUMN_MAPPING_TABLE.key -> schemaMode.get.name) { result = cdcRead(tblId, start, end, None, readerOptions) } return result } val startPrefix: String = start match { case startingVersion: StartingVersion => s"""${startingVersion.value}""" case startingTimestamp: StartingTimestamp => s"""'${startingTimestamp.value}'""" case Unbounded => "" } val endPrefix: String = end match { case endingVersion: EndingVersion => s"""${endingVersion.value}""" case endingTimestamp: EndingTimestamp => s"""'${endingTimestamp.value}'""" case Unbounded => "" } val fnName = tblId match { case _: TablePath => DeltaTableValueFunctions.CDC_PATH_BASED case _: TableName => DeltaTableValueFunctions.CDC_NAME_BASED case _ => throw new IllegalArgumentException("No table name or path provided") } if (endPrefix === "") { sql(s"SELECT * FROM $fnName('${tblId.id}', $startPrefix)") } else { sql(s"SELECT * FROM $fnName('${tblId.id}', $startPrefix, $endPrefix) ") } } override def ctas( srcTbl: String, dstTbl: String, disableCDC: Boolean = false): Unit = { val prefix = s"CREATE TABLE ${dstTbl} USING DELTA" val suffix = s" AS SELECT * FROM table_changes('${srcTbl}', 0, 1)" if (disableCDC) { sql(prefix + s" TBLPROPERTIES (${DeltaConfigs.CHANGE_DATA_FEED.key} = false)" + suffix) } else { sql(prefix + suffix) } } private def testNullRangeBoundary(start: Boundary, end: Boundary): Unit = { test(s"range boundary cannot be null - start=$start end=$end") { val tblName = "tbl" withTable(tblName) { createTblWithThreeVersions(tblName = Some(tblName)) checkError(intercept[DeltaIllegalArgumentException] { cdcRead(new TableName(tblName), start, end) }, "DELTA_CDC_READ_NULL_RANGE_BOUNDARY") } } } for (end <- Seq( Unbounded, EndingVersion("null"), EndingVersion("0"), EndingTimestamp(dateFormat.format(new Date(1))) )) { testNullRangeBoundary(StartingVersion("null"), end) } for (start <- Seq(StartingVersion("0"), StartingTimestamp(dateFormat.format(new Date(1))))) { testNullRangeBoundary(start, EndingVersion("null")) } testNullRangeBoundary(StartingVersion("CAST(null AS INT)"), Unbounded) test("select individual column should push down filters") { val tblName = "tbl" withTable(tblName) { createTblWithThreeVersions(tblName = Some(tblName)) val plans = DeltaTestUtils.withAllPlansCaptured(spark) { val res = sql(s"SELECT id, _change_type FROM table_changes('$tblName', 0, 1)") .where(col("id") < lit(5)) assert(res.columns === Seq("id", "_change_type")) checkAnswer( res, spark.range(5) .withColumn("_change_type", lit("insert"))) } assert(plans.map(_.executedPlan).toString .contains("PushedFilters: [*IsNotNull(id), *LessThan(id,5)]")) } } test("use cdc query as a subquery") { val tblName = "tbl" withTable(tblName) { createTblWithThreeVersions(tblName = Some(tblName)) val res = sql(s""" SELECT * FROM RANGE(30) WHERE id > ( SELECT count(*) FROM table_changes('$tblName', 0, 1)) """) checkAnswer( res, spark.range(21, 30).toDF()) } } test("cdc table_changes is not case sensitive") { val tblName = "tbl" withTempDir { dir => withTable(tblName) { createTblWithThreeVersions(tblName = Some(tblName)) checkAnswer( spark.sql(s"SELECT * FROM tabLe_chAnges('$tblName', 0, 1)"), spark.sql(s"SELECT * FROM taBle_cHanges('$tblName', 0, 1)") ) } } } test("cdc table_changes_by_path are not case sensitive") { withTempDir { dir => createTblWithThreeVersions(path = Some(dir.getAbsolutePath)) checkAnswer( spark.sql(s"SELECT * FROM tabLe_chaNges_By_pAth('${dir.getAbsolutePath}', 0, 1)"), spark.sql(s"SELECT * FROM taBle_cHanges_bY_paTh('${dir.getAbsolutePath}', 0, 1)") ) } } test("parse multi part table name") { val tblName = "tbl" withTable(tblName) { createTblWithThreeVersions(tblName = Some(tblName)) checkAnswer( spark.sql(s"SELECT * FROM table_changes('$tblName', 0, 1)"), spark.sql(s"SELECT * FROM table_changes('default.`${tblName}`', 0, 1)") ) } } test("negative case - invalid number of args") { val tbl = "tbl" withTable(tbl) { spark.range(10).write.format("delta").saveAsTable(tbl) val invalidQueries = Seq( s"SELECT * FROM table_changes()", s"SELECT * FROM table_changes('tbl', 1, 2, 3)", s"SELECT * FROM table_changes('tbl')", s"SELECT * FROM table_changes_by_path()", s"SELECT * FROM table_changes_by_path('tbl', 1, 2, 3)", s"SELECT * FROM table_changes_by_path('tbl')" ) invalidQueries.foreach { q => val e = intercept[AnalysisException] { sql(q) } assert(e.getMessage.contains("requires at least 2 arguments and at most 3 arguments"), s"failed query: $q ") } } } test("negative case - invalid type of args") { val tbl = "tbl" withTable(tbl) { spark.range(10).write.format("delta").saveAsTable(tbl) val invalidQueries = Seq( s"SELECT * FROM table_changes(1, 1)", s"SELECT * FROM table_changes('$tbl', 1.0)", s"SELECT * FROM table_changes_by_path(1, 1)", s"SELECT * FROM table_changes_by_path('$tbl', 1.0)" ) invalidQueries.foreach { q => val e = intercept[AnalysisException] { sql(q) } assert(e.getMessage.contains("Unsupported expression type"), s"failed query: $q") } } } test("negative case - non-constant expressions in version/timestamp argument") { val tbl = "tbl" val otherTbl = "other_tbl" withTempDir { dir => withTable(tbl, otherTbl) { spark.range(10).write.format("delta").option("path", dir.getAbsolutePath).saveAsTable(tbl) spark.range(5).toDF("version").write.format("delta").saveAsTable(otherTbl) // (query, expectedFunctionName, expectedParamName, expectedPos, sqlExprPattern) val testCases = Seq( // Scalar subquery as starting arg (s"SELECT * FROM table_changes('$tbl', (SELECT MAX(version) FROM $otherTbl))", "table_changes", "starting", 2, "scalarsubquery.*"), // Scalar subquery as ending arg (s"SELECT * FROM table_changes('$tbl', 0, (SELECT MAX(version) FROM $otherTbl))", "table_changes", "ending", 3, "scalarsubquery.*"), // Scalar subquery in table_changes_by_path (s"SELECT * FROM table_changes_by_path('${dir.getAbsolutePath}'," + s" (SELECT MAX(version) FROM $otherTbl))", "table_changes_by_path", "starting", 2, "scalarsubquery.*"), // Aggregate expression as starting arg (s"SELECT * FROM table_changes('$tbl', MAX(1))", "table_changes", "starting", 2, ".*[Mm]ax.*"), // Aggregate expression as ending arg (s"SELECT * FROM table_changes('$tbl', 0, MAX(1))", "table_changes", "ending", 3, ".*[Mm]ax.*"), // Aggregate expression in table_changes_by_path (s"SELECT * FROM table_changes_by_path('${dir.getAbsolutePath}', MAX(1))", "table_changes_by_path", "starting", 2, ".*[Mm]ax.*") ) testCases.foreach { case (q, expectedFn, expectedParam, expectedPos, sqlExprPattern) => checkErrorMatchPVals( intercept[AnalysisException] { sql(q) }, "DELTA_CDC_NON_CONSTANT_ARGUMENT", parameters = Map( "argumentName" -> s"`$expectedParam`", "pos" -> expectedPos.toString, "functionName" -> s"`$expectedFn`", "sqlExpr" -> sqlExprPattern ) ) } } } } test("negative case - table_changes in correlated subquery with OuterReference") { // When table_changes() is used inside a correlated subquery with an expression that // wraps an OuterReference (e.g. `o.version + 0`), the top-level node is not Unevaluable // (Add is evaluable) but its child OuterReference is. The old isInstanceOf[Unevaluable] // check on the top-level expression misses this case and .eval() throws INTERNAL_ERROR. // We instead catch that SparkException and re-throw as DELTA_CDC_NON_CONSTANT_ARGUMENT. val tbl = "tbl" val otherTbl = "other_tbl" withTable(tbl, otherTbl) { spark.range(10).write.format("delta").saveAsTable(tbl) spark.range(5).toDF("version").write.format("delta").saveAsTable(otherTbl) val q = s""" SELECT * FROM $otherTbl o WHERE EXISTS ( SELECT 1 FROM table_changes('$tbl', o.version + 0) ) """ checkErrorMatchPVals( intercept[AnalysisException] { sql(q) }, "DELTA_CDC_NON_CONSTANT_ARGUMENT", parameters = Map( "argumentName" -> "`starting`", "pos" -> "2", "functionName" -> "`table_changes`", "sqlExpr" -> ".*" ) ) } } test("resolve expression for timestamp function") { val tbl = "tbl" withDefaultTimeZone(UTC) { withTable(tbl) { createTblWithThreeVersions(tblName = Some(tbl)) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tbl)) val currentTime = new Date().getTime modifyCommitTimestamp(deltaLog, 0, currentTime - 100000) modifyCommitTimestamp(deltaLog, 1, currentTime) modifyCommitTimestamp(deltaLog, 2, currentTime + 100000) // Make sure the snapshot used for the `table_changes` query is updated with the // new timestamps. The ICT changes in un-backfilled commits will not trigger the real // snapshot update, so we need to manually clear the cache and refresh the snapshot // to ensure the new timestamps are used. DeltaLog.clearCache() val readDf = sql(s"SELECT * FROM table_changes('$tbl', 0, now())") checkCDCAnswer( DeltaLog.forTable(spark, TableIdentifier("tbl")), readDf, spark.range(20) .withColumn("_change_type", lit("insert")) .withColumn("_commit_version", (col("id") / 10).cast(LongType)) ) // more complex expression val readDf2 = sql(s"SELECT * FROM table_changes('$tbl', 0, now() + interval 5 seconds)") checkCDCAnswer( DeltaLog.forTable(spark, TableIdentifier("tbl")), readDf2, spark.range(20) .withColumn("_change_type", lit("insert")) .withColumn("_commit_version", (col("id") / 10).cast(LongType)) ) val readDf3 = sql("SELECT * FROM table_changes" + s"('$tbl', string(date_sub(current_date(), 1)), string(now()))") checkCDCAnswer( DeltaLog.forTable(spark, TableIdentifier("tbl")), readDf3, spark.range(20) .withColumn("_change_type", lit("insert")) .withColumn("_commit_version", (col("id") / 10).cast(LongType)) ) } } } test("resolve invalid table name should throw error") { var e = intercept[AnalysisException] { sql(s"SELECT * FROM table_changes(now(), 1, 1)") } assert(e.getMessage.contains("Unsupported expression type(TimestampType) for table name." + " The supported types are [StringType literal]")) e = intercept[AnalysisException] { sql(s"SELECT * FROM table_changes('invalidtable', 1, 1)") } assert(e.getErrorClass === "TABLE_OR_VIEW_NOT_FOUND") withTable ("tbl") { spark.range(1).write.format("delta").saveAsTable("tbl") val e = intercept[AnalysisException] { sql(s"SELECT * FROM table_changes(concat('tb', 'l'), 1, 1)") } assert(e.getMessage.contains("Unsupported expression type(StringType) for table name." + " The supported types are [StringType literal]")) } } test("resolution of complex expression should throw an error") { val tbl = "tbl" withTable(tbl) { spark.range(10).write.format("delta").saveAsTable(tbl) checkError( intercept[AnalysisException] { sql(s"SELECT * FROM table_changes('$tbl', 0, id)") }, "UNRESOLVED_COLUMN.WITHOUT_SUGGESTION", parameters = Map("objectName" -> "`id`"), queryContext = Array(ExpectedContext( fragment = "id", start = 38, stop = 39))) } } test("protocol version") { if (catalogOwnedDefaultCreationEnabledInTests) { cancel("This test is intended to test the protocol version of `ChangeDataFeedTableFeature`." + "For CCv1.5 tables, the protocol version has different requirement and we already have " + "the corresponding coverage in `CatalogOwnedEnablementSuite` and " + "`CatalogOwnedPropertyEdgeSuite`.") } withTable("tbl") { spark.range(10).write.format("delta").saveAsTable("tbl") val log = DeltaLog.forTable(spark, TableIdentifier(tableName = "tbl")) // We set CDC to be enabled by default, so this should automatically bump the writer protocol // to the required version. if (columnMappingEnabled) { assert(log.update().protocol == Protocol(2, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, ChangeDataFeedTableFeature, ColumnMappingTableFeature))) } else { assert(log.update().protocol == Protocol(1, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, ChangeDataFeedTableFeature))) } } } test("table_changes and table_changes_by_path with a non-delta table") { withTempDir { dir => withTable("tbl") { spark.range(10).write.format("parquet") .option("path", dir.getAbsolutePath) .saveAsTable("tbl") var e = intercept[AnalysisException] { spark.sql(s"SELECT * FROM table_changes('tbl', 0, 1)") } assert(e.getErrorClass == "DELTA_TABLE_ONLY_OPERATION") assert(e.getMessage.contains("table_changes")) e = intercept[AnalysisException] { spark.sql(s"SELECT * FROM table_changes_by_path('${dir.getAbsolutePath}', 0, 1)") } assert(e.getErrorClass == "DELTA_MISSING_DELTA_TABLE") assert(e.getMessage.contains("not a Delta table")) } } } } class DeltaCDCSQLWithCatalogOwnedBatch1Suite extends DeltaCDCSQLSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class DeltaCDCSQLWithCatalogOwnedBatch2Suite extends DeltaCDCSQLSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class DeltaCDCSQLWithCatalogOwnedBatch100Suite extends DeltaCDCSQLSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCStreamSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.sql.Timestamp import java.text.SimpleDateFormat import java.util.Date import scala.language.implicitConversions import org.apache.spark.sql.delta.DeltaTestUtils.modifyCommitTimestamp import org.apache.spark.sql.delta.actions.AddCDCFile import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.sources.{DeltaSourceOffset, DeltaSQLConf} import org.apache.spark.sql.delta.test.{DeltaColumnMappingSelectedTestMixin, DeltaSQLCommandTest} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.JsonUtils import io.delta.tables._ import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.sql.functions._ import org.apache.spark.sql.streaming.{StreamingQuery, StreamingQueryException, StreamTest, Trigger} import org.apache.spark.sql.types.StructType trait DeltaCDCStreamSuiteBase extends StreamTest with DeltaSQLCommandTest with DeltaSourceSuiteBase with DeltaColumnMappingTestUtils { import testImplicits._ import io.delta.implicits._ /** * Returns the appropriate DeltaConfig */ protected def cdcConfig: DeltaConfig[Boolean] = DeltaConfigs.CHANGE_DATA_FEED override protected def sparkConf: SparkConf = super.sparkConf .set(cdcConfig.defaultTablePropertyKey, "true") /** * Create two tests for maxFilesPerTrigger and maxBytesPerTrigger */ protected def testRateLimit( name: String, maxFilesPerTrigger: String, maxBytesPerTrigger: String)(f: (String, String) => Unit): Unit = { Seq(("maxFilesPerTrigger", maxFilesPerTrigger), ("maxBytesPerTrigger", maxBytesPerTrigger)) .foreach { case (key: String, value: String) => test(s"rateLimit - $key - $name") { f(key, value) } } } testQuietly("no startingVersion should result fetch the entire snapshot") { withTempDir { inputDir => withSQLConf(cdcConfig.defaultTablePropertyKey -> "false") { // version 0 Seq(1, 9).toDF("value").write.format("delta").save(inputDir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(inputDir.getAbsolutePath) // version 1 deltaTable.delete("value = 9") // version 2 Seq(2).toDF("value").write.format("delta") .mode("append") .save(inputDir.getAbsolutePath) } // enable cdc - version 3 sql(s"ALTER TABLE delta.`${inputDir.getAbsolutePath}` SET TBLPROPERTIES " + s"(${cdcConfig.key}=true)") val df = spark.readStream .option(DeltaOptions.CDC_READ_OPTION, "true") .format("delta") .load(inputDir.getCanonicalPath) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) val version = 3 val deltaTable = io.delta.tables.DeltaTable.forPath(inputDir.getAbsolutePath) testStream(df) ( ProcessAllAvailable(), CheckAnswer((1, "insert", version), (2, "insert", version)), Execute { _ => deltaTable.delete("value = 1") // version 4 }, ProcessAllAvailable(), CheckAnswer((1, "insert", version), (2, "insert", version), (1, "delete", version + 1)) ) } } testQuietly("CDC initial snapshot should end at base index of next version") { withTempDir { inputDir => withSQLConf(cdcConfig.defaultTablePropertyKey -> "true") { // version 0 Seq(5, 6).toDF("value").write.format("delta").save(inputDir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(inputDir.getAbsolutePath) val df = spark.readStream .option(DeltaOptions.CDC_READ_OPTION, "true") .format("delta") .load(inputDir.getCanonicalPath) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) testStream(df)( ProcessAllAvailable(), CheckAnswer((5, "insert", 0), (6, "insert", 0)), AssertOnQuery { q => val offset = q.committedOffsets.iterator.next()._2.asInstanceOf[DeltaSourceOffset] // The initial snapshot (version 0) was completely processed, so we should now be at // the start of version 1. assert(offset.reservoirVersion === 1) assert(offset.index === DeltaSourceOffset.BASE_INDEX) true }, StopStream ) } } } test("startingVersion = latest") { withTempDir { inputDir => Seq(1, 2).toDF("value").write.format("delta").save(inputDir.getAbsolutePath) val df = spark.readStream .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingVersion", "latest") .format("delta") .load(inputDir.getCanonicalPath) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) testStream(df) ( ProcessAllAvailable(), CheckAnswer(), AddToReservoir(inputDir, Seq(3).toDF("value")), ProcessAllAvailable(), CheckAnswer((3, "insert", 1)) ) } } test("user provided startingVersion") { withTempDir { inputDir => // version 0 Seq(1, 2, 3).toDF("id").write.delta(inputDir.toString) // version 1 Seq(4, 5).toDF("id").write.mode("append").delta(inputDir.toString) // version 2 val deltaTable = io.delta.tables.DeltaTable.forPath(inputDir.getAbsolutePath) val df = spark.readStream .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingVersion", "1") .format("delta") .load(inputDir.toString) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) testStream(df) ( ProcessAllAvailable(), CheckAnswer((4, "insert", 1), (5, "insert", 1)), Execute { _ => deltaTable.delete("id = 3") // version 2 }, ProcessAllAvailable(), CheckAnswer((4, "insert", 1), (5, "insert", 1), (3, "delete", 2)) ) } } test("user provided startingTimestamp") { withTempDir { inputDir => // version 0 Seq(1, 2, 3).toDF("id").write.delta(inputDir.toString) val deltaLog = DeltaLog.forTable(spark, inputDir.getAbsolutePath) modifyCommitTimestamp(deltaLog, 0, 1000) // version 1 Seq(-1).toDF("id").write.mode("append").delta(inputDir.toString) modifyCommitTimestamp(deltaLog, 1, 2000) val deltaTable = io.delta.tables.DeltaTable.forPath(inputDir.getAbsolutePath) val startTs = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") .format(new Date(2000)) val df = spark.readStream .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingTimestamp", startTs) .format("delta") .load(inputDir.toString) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) testStream(df) ( ProcessAllAvailable(), CheckAnswer((-1, "insert", 1)), Execute { _ => deltaTable.update(expr("id == -1"), Map("id" -> lit("4"))) }, ProcessAllAvailable(), CheckAnswer((-1, "insert", 1), (-1, "update_preimage", 2), (4, "update_postimage", 2)) ) } } testQuietly("starting[Version/Timestamp] > latest version") { withTempDirs { (inputDir, outputDir, checkpointDir) => // version 0 Seq(1, 2, 3, 4, 5, 6).toDF("id").write.delta(inputDir.toString) val deltaLog = DeltaLog.forTable(spark, inputDir.getAbsolutePath) modifyCommitTimestamp(deltaLog, 0, 1000) val df1 = spark.readStream .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingVersion", 1) .format("delta") .load(inputDir.toString) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) val startTs = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") .format(new Date(3000)) val commitTs = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") .format(new Date(1000)) val df2 = spark.readStream .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingTimestamp", startTs) .format("delta") .load(inputDir.toString) val e1 = VersionNotFoundException(1, 0, 0).getMessage val e2 = DeltaErrors.timestampGreaterThanLatestCommit( new Timestamp(3000), new Timestamp(1000), commitTs).getMessage Seq((df1, e1), (df2, e2)).foreach { pair => val df = pair._1 val stream = df.select("id").writeStream .option("checkpointLocation", checkpointDir.toString) .outputMode("append") .format("delta") .start(outputDir.getAbsolutePath) val e = intercept[StreamingQueryException] { stream.processAllAvailable() } stream.stop() assert(e.cause.getMessage === pair._2) } } } test("check starting[Version/Timestamp] > latest version without error") { Seq("version", "timestamp").foreach { target => withTempDir { inputDir => withSQLConf(DeltaSQLConf.DELTA_CDF_ALLOW_OUT_OF_RANGE_TIMESTAMP.key -> "true") { // version 0 Seq(1, 2, 3).toDF("id").write.delta(inputDir.toString) val inputPath = inputDir.getAbsolutePath val deltaLog = DeltaLog.forTable(spark, inputPath) modifyCommitTimestamp(deltaLog, 0, 1000) val deltaTable = io.delta.tables.DeltaTable.forPath(inputPath) // Pick both the timestamp and version beyond latest commmit's version. val df = if (target == "timestamp") { // build dataframe with starting timestamp option. val startTs = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") .format(new Date(2000)) spark.readStream .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingTimestamp", startTs) .format("delta") .load(inputDir.toString) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) } else { assert(target == "version") // build dataframe with starting version option. spark.readStream .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingVersion", 1) .format("delta") .load(inputDir.toString) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) } testStream(df)( ProcessAllAvailable(), // Expect empty update from the read stream. CheckAnswer(), // Verify new updates after the start timestamp/version can be read. Execute { _ => deltaTable.update(expr("id == 1"), Map("id" -> lit("4"))) }, ProcessAllAvailable(), CheckAnswer((1, "update_preimage", 1), (4, "update_postimage", 1)) ) } } } } testQuietly("startingVersion and startingTimestamp are both set") { withTempDir { tableDir => val tablePath = tableDir.getCanonicalPath spark.range(10).write.format("delta").save(tableDir.getAbsolutePath) val q = spark.readStream .format("delta") .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingVersion", 0L) .option("startingTimestamp", "2020-07-15") .load(tablePath) .writeStream .format("console") .start() assert(intercept[StreamingQueryException] { q.processAllAvailable() }.getMessage.contains("Please either provide 'startingVersion' or 'startingTimestamp'")) q.stop() } } test("cdc streams should respect checkpoint") { withTempDirs { (inputDir, outputDir, checkpointDir) => // write 3 versions Seq(1, 2, 3).toDF("id").write.format("delta").save(inputDir.getAbsolutePath) Seq(4, 5, 6).toDF("id").write.format("delta") .mode("append") .save(inputDir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(inputDir.getAbsolutePath) deltaTable.delete("id = 5") val checkpointDir1 = new Path(checkpointDir.getAbsolutePath, "ck1") val checkpointDir2 = new Path(checkpointDir.getAbsolutePath, "ck2") def streamChanges( startingVersion: Long, checkpointLocation: String): Unit = { val q = spark.readStream .format("delta") .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingVersion", startingVersion) .load(inputDir.getCanonicalPath) .select("id") .writeStream .format("delta") .option("checkpointLocation", checkpointLocation) .start(outputDir.getCanonicalPath) try { q.processAllAvailable() } finally { q.stop() } } streamChanges(1, checkpointDir1.toString) checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq(4, 5, 5, 6).map(_.toLong).toDF("id")) // Second time streaming should not write the rows again streamChanges(1, checkpointDir1.toString) checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq(4, 5, 5, 6).map(_.toLong).toDF("id")) // new checkpoint location streamChanges(1, checkpointDir2.toString) checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq(4, 4, 5, 5, 5, 5, 6, 6).map(_.toLong).toDF("id")) } } test("cdc streams with noop merge") { withSQLConf( // When DeletionVectors are enabled (e.g., via CatalogManaged QoL features), a truly no-op // merge (all WHEN conditions false) produces empty actions (no FileActions) because // writeUnmodifiedRows=false in the DV path. The default table isolation level (Serializable) // allows canDowngradeToSnapshotIsolation to succeed (noDataChanged=true is sufficient), so // the transaction runs at SnapshotIsolation and skipRecordingEmptyCommitAllowed returns // true, causing commitIfNeeded to skip the commit entirely. This test requires version 1 // to exist for streaming, so we force the classic copy-on-write merge path. DeltaSQLConf.MERGE_USE_PERSISTENT_DELETION_VECTORS.key -> "false", cdcConfig.defaultTablePropertyKey -> "true" ) { withTempDirs { (srcDir, targetDir, checkpointDir) => // write source table Seq((1, "a"), (2, "b")) .toDF("key1", "val1") .write .format("delta") .save(srcDir.getCanonicalPath) // write target table Seq((1, "t"), (2, "u")) .toDF("key2", "val2") .write .format("delta") .save(targetDir.getCanonicalPath) val srcDF = spark.read.format("delta").load(srcDir.getCanonicalPath) val tgtTable = io.delta.tables.DeltaTable.forPath(targetDir.getCanonicalPath) // Perform the merge where all matching and non-matching conditions fail for // target rows. tgtTable .merge(srcDF, "key1 = key2") .whenMatched("key1 = 10") .updateExpr(Map("key2" -> "key1", "val2" -> "val1")) .whenNotMatched("key1 = 11") .insertExpr(Map("key2" -> "key1", "val2" -> "val1")) .execute() // Read the target dir with cdc read option and ensure that // data frame is empty. val q = spark.readStream .format("delta") .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingVersion", "1") .load(targetDir.getCanonicalPath) .writeStream .format("memory") .option("checkpointLocation", checkpointDir.getCanonicalPath) .queryName("testQuery") .start() try { q.processAllAvailable() } finally { q.stop() } assert(spark.table("testQuery").isEmpty) } } } Seq(true, false).foreach { readChangeFeed => test(s"streams updating latest offset with readChangeFeed=$readChangeFeed") { withTempDirs { (inputDir, checkpointDir, outputDir) => withSQLConf(cdcConfig.defaultTablePropertyKey -> "true") { sql(s"CREATE TABLE delta.`$inputDir` (id BIGINT, value STRING) USING DELTA") // save some rows to input table. spark.range(10).withColumn("value", lit("a")) .write.format("delta").mode("overwrite") .option("enableChangeDataFeed", "true").save(inputDir.getAbsolutePath) def runStreamingQuery(): StreamingQuery = { // process the input table in a CDC manner val df = spark.readStream .option(DeltaOptions.CDC_READ_OPTION, readChangeFeed) .format("delta") .load(inputDir.getAbsolutePath) val query = df .select("id") .writeStream .format("delta") .outputMode("append") .option("checkpointLocation", checkpointDir.toString) .start(outputDir.getAbsolutePath) query.processAllAvailable() query.stop() query.awaitTermination() query } var query = runStreamingQuery() val deltaLog = DeltaLog.forTable(spark, inputDir.toString) // Do three no-op updates to the table. These are tricky because the commits have no // changes, but the stream should still pick up the new versions and progress past them. for (i <- 0 to 2) { deltaLog.startTransaction().commit(Seq(), DeltaOperations.ManualUpdate) } // Read again from input table and no new data should be generated query = runStreamingQuery() // check that the last batch was committed and that the // reservoirVersion for the table was updated to latest // in both cdf and non-cdf cases. assert(query.lastProgress.batchId === 1) val endOffset = JsonUtils.fromJson[DeltaSourceOffset](query.lastProgress.sources.head.endOffset) assert(endOffset.reservoirVersion === 5, s"endOffset = $endOffset") assert(endOffset.index === DeltaSourceOffset.BASE_INDEX, s"endOffset = $endOffset") } } } } test("cdc streams should be able to get offset when there only RemoveFiles") { withTempDir { inputDir => // version 0 spark.range(2).withColumn("part", 'id % 2) .write .format("delta") .partitionBy("part") .save(inputDir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(inputDir.getAbsolutePath) val df = spark.readStream .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingVersion", 0) .format("delta") .load(inputDir.toString) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) testStream(df) ( ProcessAllAvailable(), CheckAnswer((0, 0, "insert", 0), (1, 1, "insert", 0)), Execute { _ => deltaTable.delete("part = 0") // version 2 }, ProcessAllAvailable(), CheckAnswer((0, 0, "insert", 0), (1, 1, "insert", 0), (0, 0, "delete", 1)) ) } } test("cdc streams should work starting from RemoveFile") { withTempDir { inputDir => // version 0 spark.range(2).withColumn("part", 'id % 2) .write .format("delta") .partitionBy("part") .save(inputDir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(inputDir.getAbsolutePath) deltaTable.delete("part = 0") val df = spark.readStream .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingVersion", 1) .format("delta") .load(inputDir.toString) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) testStream(df) ( ProcessAllAvailable(), CheckAnswer((0, 0, "delete", 1)) ) } } test("cdc streams should work starting from AddCDCFile") { withTempDir { inputDir => // version 0 spark.range(2).withColumn("col2", 'id % 2) .repartition(1) .write .format("delta") .save(inputDir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(inputDir.getAbsolutePath) deltaTable.delete("col2 = 0") val df = spark.readStream .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingVersion", 1) .format("delta") .load(inputDir.toString) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) testStream(df) ( ProcessAllAvailable(), CheckAnswer((0, 0, "delete", 1)), AddToReservoir(inputDir, spark.range(2, 3).withColumn("col2", 'id % 2)), ProcessAllAvailable(), CheckAnswer((0, 0, "delete", 1), (2, 0, "insert", 2)) ) } } testRateLimit(s"overall", "1", "1b") { (key, value) => withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) // write - version 0 - 2 AddFiles - Adds 4 rows spark.range(0, 4, 1, 1).toDF("id") .withColumn("part", col("id") % 2) // 2 partitions .write .format("delta") .partitionBy("part") .save(inputDir.getAbsolutePath) assert(deltaLog.snapshot.version == 0) assert(deltaLog.snapshot.numOfFiles == 2) // write - version 1 - 1 AddFile - Adds 1 row Seq(4L).toDF("id").withColumn("part", lit(-1L)) .write .format("delta") .mode("append") .partitionBy("part") .save(deltaLog.dataPath.toString) assert(deltaLog.snapshot.version == 1) assert(deltaLog.snapshot.numOfFiles == 3) // delete - version 2 - 1 RemoveFile - Removes 1 row val deltaTable = io.delta.tables.DeltaTable.forPath(inputDir.getAbsolutePath) deltaTable.delete("part = -1") assert(deltaLog.snapshot.version == 2) assert(deltaLog.snapshot.numOfFiles == 2) // update the table - version 3 - 2 cdc files - Updates 2 rows deltaTable.update(expr("id < 2"), Map("id" -> lit(0L))) // update the table - version 4 - 2 cdc files - Updates 2 rows deltaTable.update(expr("id > 1"), Map("id" -> lit(0L))) val rowsPerBatch = Seq( 2, // 2 rows from 1 AddFile 2, // 2 rows from the 2nd AddFile 1, // 1 row from the 3rd AddFile 1, // 1 row from the RemoveFile 4, // 4 rows(pre_image and post_image) from the 2 AddCDCFile 4 // 4 rows(pre_image and post_image) from the 2 AddCDCFile ) val q = spark.readStream .format("delta") .option(key, value) .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingVersion", "0") .load(inputDir.getCanonicalPath) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) testStream(q) ( ProcessAllAvailable(), CheckProgress(rowsPerBatch), CheckAnswer( (0, 0, "insert", 0), (1, 1, "insert", 0), (2, 0, "insert", 0), (3, 1, "insert", 0), (4, -1, "insert", 1), (4, -1, "delete", 2), (0, 0, "update_preimage", 3), (0, 0, "update_postimage", 3), (1, 1, "update_preimage", 3), (0, 1, "update_postimage", 3), (2, 0, "update_preimage", 4), (0, 0, "update_postimage", 4), (3, 1, "update_preimage", 4), (0, 1, "update_postimage", 4) ) ) } } testRateLimit(s"starting from initial snapshot", "1", "1b") { (key, value) => withTempDir { inputDir => // 3 commits - 3 AddFiles each (0 until 3).foreach { i => spark.range(i, i + 1, 1, 1) .write .mode("append") .format("delta") .save(inputDir.getAbsolutePath) } val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) assert(deltaLog.snapshot.numOfFiles === 3) // 1 commit - 2 AddFiles spark.range(3, 5, 1, 2) .write .mode("append") .format("delta") .save(inputDir.getAbsolutePath) assert(deltaLog.snapshot.numOfFiles === 5) val q = spark.readStream .format("delta") .option(key, value) .option(DeltaOptions.CDC_READ_OPTION, "true") .load(inputDir.getCanonicalPath) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) // 5 batches for the 5 commits split across commits and index number. val rowsPerBatch = Seq(1, 1, 1, 1, 1) testStream(q)( ProcessAllAvailable(), CheckProgress(rowsPerBatch), CheckAnswer( (0, "insert", 3), (1, "insert", 3), (2, "insert", 3), (3, "insert", 3), (4, "insert", 3) ) ) } } testRateLimit(s"should not deadlock", "1", "1b") { (key, value) => withTempDir { inputDir => // version 0 - 2 AddFiles spark.range(2) .withColumn("part", 'id % 2) .withColumn("col3", lit(0)) .repartition(1) .write .format("delta") .partitionBy("part") .save(inputDir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(inputDir.getAbsolutePath) // version 1 - 2 AddCDCFiles deltaTable.update(expr("col3 < 2"), Map("col3" -> lit("0"))) // version 2 - 2 AddCDCFiles deltaTable.update(expr("col3 < 2"), Map("col3" -> lit("1"))) val df = spark.readStream .format("delta") .option(key, value) .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingVersion", "1") .load(inputDir.getCanonicalPath) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) testStream(df)( ProcessAllAvailable(), CheckProgress(Seq(4, 4)),// 4 rows(2 pre- and 2 post-images) for each version CheckAnswer( (0, 0, 0, "update_preimage", 1), (0, 0, 0, "update_postimage", 1), (0, 0, 0, "update_preimage", 2), (0, 0, 1, "update_postimage", 2), (1, 1, 0, "update_preimage", 1), (1, 1, 0, "update_postimage", 1), (1, 1, 0, "update_preimage", 2), (1, 1, 1, "update_postimage", 2) ) ) } } test("maxFilesPerTrigger - 2 successive AddCDCFile commits") { withTempDir { inputDir => // version 0 - 2 AddFiles spark.range(2) .withColumn("part", 'id % 2) .withColumn("col3", lit(0)) .repartition(1) .write .format("delta") .partitionBy("part") .save(inputDir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(inputDir.getAbsolutePath) // version 1 - 2 AddCDCFiles deltaTable.update(expr("col3 < 2"), Map("col3" -> lit("0"))) // version 2 - 2 AddCDCFiles deltaTable.update(expr("col3 < 2"), Map("col3" -> lit("1"))) val df = spark.readStream .format("delta") .option(DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION, "3") .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingVersion", "0") .load(inputDir.getCanonicalPath) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) // test whether the AddCDCFile commits do not get split up. val rowsPerBatch = Seq( 2, // 2 rows from the 2 AddFile 4, // 4 rows(pre and post image) from the 2 AddCDCFiles 4 // 4 rows(pre and post image) from 2 AddCDCFiles ) testStream(df)( ProcessAllAvailable(), CheckProgress(rowsPerBatch), CheckAnswer( (0, 0, 0, "insert", 0), (1, 1, 0, "insert", 0), (0, 0, 0, "update_preimage", 1), (0, 0, 0, "update_postimage", 1), (1, 1, 0, "update_preimage", 1), (1, 1, 0, "update_postimage", 1), (0, 0, 0, "update_preimage", 2), (0, 0, 1, "update_postimage", 2), (1, 1, 0, "update_preimage", 2), (1, 1, 1, "update_postimage", 2) ) ) } } test("maxFilesPerTrigger with Trigger.AvailableNow respects read limits") { withTempDir { inputDir => // version 0 - 2 AddFiles spark.range(2) .withColumn("part", 'id % 2) .withColumn("col3", lit(0)) .repartition(1) .write .format("delta") .partitionBy("part") .save(inputDir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(inputDir.getAbsolutePath) // version 1 - 2 AddCDCFiles deltaTable.update(expr("col3 < 2"), Map("col3" -> lit("0"))) // version 2 - 2 AddCDCFiles deltaTable.update(expr("col3 < 2"), Map("col3" -> lit("1"))) val df = spark.readStream .format("delta") .option(DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION, "3") .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingVersion", "0") .load(inputDir.getCanonicalPath) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) // test whether the AddCDCFile commits do not get split up. val rowsPerBatch = Seq( 2, // 2 rows from the 2 AddFile 4, // 4 rows(pre and post image) from the 2 AddCDCFiles 4 // 4 rows(pre and post image) from 2 AddCDCFiles ) testStream(df)( StartStream(Trigger.AvailableNow), Execute { query => assert(query.awaitTermination(10000)) }, CheckProgress(rowsPerBatch), CheckAnswer( (0, 0, 0, "insert", 0), (1, 1, 0, "insert", 0), (0, 0, 0, "update_preimage", 1), (0, 0, 0, "update_postimage", 1), (1, 1, 0, "update_preimage", 1), (1, 1, 0, "update_postimage", 1), (0, 0, 0, "update_preimage", 2), (0, 0, 1, "update_postimage", 2), (1, 1, 0, "update_preimage", 2), (1, 1, 1, "update_postimage", 2) ) ) } } test("excludeRegex works with cdc") { withTempDir { inputDir => spark.range(2) .withColumn("part", 'id % 2) .repartition(1) .write .format("delta") .partitionBy("part") .save(inputDir.getAbsolutePath) val df = spark.readStream .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingVersion", "0") .option(DeltaOptions.EXCLUDE_REGEX_OPTION, "part=0") .format("delta") .load(inputDir.getCanonicalPath) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) testStream(df)( ProcessAllAvailable(), CheckAnswer((1, 1, "insert", 0)) // first file should get excluded ) } } test("excludeRegex on cdcPath should not return Add/RemoveFiles") { withTempDir { inputDir => // version 0 - 1 AddFile Seq(0).toDF("id") .withColumn("col2", lit("0")) .repartition(1) .write .format("delta") .save(inputDir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(inputDir.getAbsolutePath) // version 1 - 1 ChangeFile deltaTable.update(expr("col2 < 2"), Map("col2" -> lit("1"))) val deltaLog = DeltaLog.forTable(spark, inputDir.getAbsolutePath) val excludePath = deltaLog.getChanges(1).next()._2 .filter(_.isInstanceOf[AddCDCFile]) .head .asInstanceOf[AddCDCFile] .path val df = spark.readStream .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingVersion", "0") .option(DeltaOptions.EXCLUDE_REGEX_OPTION, excludePath) .format("delta") .load(inputDir.getCanonicalPath) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) testStream(df)( ProcessAllAvailable(), CheckAnswer((0, "0", "insert", 0)) // first file should get excluded ) } } test("schema check for cdc stream") { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 5).foreach { i => Seq(i).toDF.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } val df = spark.readStream .format("delta") .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingVersion", 0) .load(inputDir.getCanonicalPath) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) testStream(df)( AssertOnQuery { q => q.processAllAvailable(); true }, CheckAnswer( (0, "insert", 0), (1, "insert", 1), (2, "insert", 2), (3, "insert", 3), (4, "insert", 4) ), // no schema changed exception should be thrown. AssertOnQuery { _ => withMetadata(deltaLog, StructType.fromDDL("value int")) true }, // Force processing of stream to prevent race condition between DeltaSource.getBatch and // DeltaSource.checkReadIncompatibleSchemaChanges ProcessAllAvailable(), AssertOnQuery { _ => withMetadata(deltaLog, StructType.fromDDL("id int, value string")) true }, ExpectFailure[DeltaIllegalStateException](t => assert(t.getMessage.contains("Detected schema change"))) ) } } test("should not attempt to read a non exist version") { withTempDirs { (inputDir1, inputDir2, checkpointDir) => spark.range(1, 2).write.format("delta").save(inputDir1.getCanonicalPath) spark.range(1, 2).write.format("delta").save(inputDir2.getCanonicalPath) def startQuery(): StreamingQuery = { val df1 = spark.readStream .format("delta") .option("readChangeFeed", "true") .load(inputDir1.getCanonicalPath) val df2 = spark.readStream .format("delta") .option("readChangeFeed", "true") .load(inputDir2.getCanonicalPath) df1.union(df2).writeStream .format("noop") .option("checkpointLocation", checkpointDir.getCanonicalPath) .start() } var q = startQuery() try { q.processAllAvailable() // current offsets: // source1: DeltaSourceOffset(reservoirVersion=1,index=0,isInitialSnapshot=true) // source2: DeltaSourceOffset(reservoirVersion=1,index=0,isInitialSnapshot=true) spark.range(1, 2).write.format("delta").mode("append").save(inputDir1.getCanonicalPath) spark.range(1, 2).write.format("delta").mode("append").save(inputDir2.getCanonicalPath) q.processAllAvailable() // current offsets: // source1: DeltaSourceOffset(reservoirVersion=2,index=-1,isInitialSnapshot=false) // source2: DeltaSourceOffset(reservoirVersion=2,index=-1,isInitialSnapshot=false) // Note: version 2 doesn't exist in source1 spark.range(1, 2).write.format("delta").mode("append").save(inputDir2.getCanonicalPath) q.processAllAvailable() // current offsets: // source1: DeltaSourceOffset(reservoirVersion=2,index=-1,isInitialSnapshot=false) // source2: DeltaSourceOffset(reservoirVersion=3,index=-1,isInitialSnapshot=false) // Note: version 2 doesn't exist in source1 q.stop() // Restart the query. It will call `getBatch` on the previous two offsets of `source1` which // are both DeltaSourceOffset(reservoirVersion=2,index=-1,isInitialSnapshot=false) // As version 2 doesn't exist, we should not try to load version 2 in this case. q = startQuery() q.processAllAvailable() } finally { q.stop() } } } // LC-1281: Ensure that when we would split batches into one file at a time, we still produce // correct CDF even in cases where the CDF may need to compare multiple file actions from the // same commit to be correct, such as with persistent deletion vectors. test("double delete-only on the same file") { withTempDir { tableDir => val tablePath = tableDir.toString spark.range(start = 0L, end = 10L, step = 1L, numPartitions = 1).toDF("id") .write.format("delta").save(tablePath) spark.sql(s"DELETE FROM delta.`$tablePath` WHERE id IN (1, 3, 6)") spark.sql(s"DELETE FROM delta.`$tablePath` WHERE id IN (2, 4, 7)") val stream = spark.readStream .format("delta") .option(DeltaOptions.CDC_READ_OPTION, true) .option(DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION, 1) .option(DeltaOptions.STARTING_VERSION_OPTION, 1) .load(tablePath) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) testStream(stream)( ProcessAllAvailable(), CheckAnswer( (1L, "delete", 1L), (3L, "delete", 1L), (6L, "delete", 1L), (2L, "delete", 2L), (4L, "delete", 2L), (7L, "delete", 2L) ) ) } } } class DeltaCDCStreamDeletionVectorSuite extends DeltaCDCStreamSuite with DeletionVectorsTestUtils { override def beforeAll(): Unit = { super.beforeAll() enableDeletionVectorsForAllSupportedOperations(spark) } } class DeltaCDCStreamSuite extends DeltaCDCStreamSuiteBase // Batch sizes 1, 2, and 100 exercise different backfill behaviors in the commit coordinator. // Batch size 1 triggers a backfill on every commit (commitVersion % 1 == 0), testing the most // granular backfill path. Batch size 2 triggers backfill every other commit, testing the boundary // between backfilled and unbackfilled commits. Batch size 100 leaves most commits unbackfilled, // testing the production-like path where streaming must read from both the commit coordinator // and the filesystem. This follows the same pattern as other CatalogManaged (CCv2) test suites // (DeltaLogSuite, DeltaSourceSuite, etc.). class DeltaCDCStreamWithCatalogManagedBatch1Suite extends DeltaCDCStreamSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class DeltaCDCStreamWithCatalogManagedBatch2Suite extends DeltaCDCStreamSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class DeltaCDCStreamWithCatalogManagedBatch100Suite extends DeltaCDCStreamSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } abstract class DeltaCDCStreamColumnMappingSuiteBase extends DeltaCDCStreamSuite with ColumnMappingStreamingBlockedWorkflowSuiteBase with DeltaColumnMappingSelectedTestMixin { override protected def isCdcTest: Boolean = true override def runOnlyTests: Seq[String] = Seq( "no startingVersion should result fetch the entire snapshot", "user provided startingVersion", "maxFilesPerTrigger - 2 successive AddCDCFile commits", // streaming blocking semantics test "deltaLog snapshot should not be updated outside of the stream", "column mapping + streaming - allowed workflows - column addition", "column mapping + streaming - allowed workflows - upgrade to name mode", "column mapping + streaming: blocking workflow - drop column", "column mapping + streaming: blocking workflow - rename column" ) } class DeltaCDCStreamIdColumnMappingSuite extends DeltaCDCStreamColumnMappingSuiteBase with DeltaColumnMappingEnableIdMode { } class DeltaCDCStreamNameColumnMappingSuite extends DeltaCDCStreamColumnMappingSuiteBase with DeltaColumnMappingEnableNameMode { } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.text.SimpleDateFormat import java.util.Date import scala.collection.JavaConverters._ import scala.concurrent.duration._ // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.DeltaTestUtils.{modifyCommitTimestamp, BOOLEAN_DOMAIN} import org.apache.spark.sql.delta.cdc.CDCEnabled import org.apache.spark.sql.delta.commands.cdc.CDCReader._ import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.{DeltaCommitFileProvider, FileNames} import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.{col, current_timestamp, floor, lit, unix_timestamp} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.streaming.StreamingQueryException import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{LongType, StringType, StructType} abstract class DeltaCDCSuiteBase extends QueryTest with CDCEnabled with SharedSparkSession with CheckCDCAnswer with DeltaSQLCommandTest with CatalogOwnedTestBaseSuite with DeltaSQLTestUtils { import testImplicits._ /** Represents path or metastore table name */ abstract case class TblId(id: String) class TablePath(path: String) extends TblId(path) class TableName(name: String) extends TblId(name) /** Indicates either the starting or ending version/timestamp */ trait Boundary case class StartingVersion(value: String) extends Boundary case class StartingTimestamp(value: String) extends Boundary case class EndingVersion(value: String) extends Boundary case class EndingTimestamp(value: String) extends Boundary case object Unbounded extends Boundary // used to model situation when a boundary isn't provided val dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss Z") def createTblWithThreeVersions( tblName: Option[String] = None, path: Option[String] = None): Unit = { // version 0 if (tblName.isDefined && path.isDefined) { spark.range(10).write.format("delta") .option("path", path.get) .saveAsTable(tblName.get) } else if (tblName.isDefined) { spark.range(10).write.format("delta") .saveAsTable(tblName.get) } else if (path.isDefined) { spark.range(10).write.format("delta") .save(path.get) } if (tblName.isDefined) { // version 1 spark.range(10, 20).write.format("delta").mode("append").saveAsTable(tblName.get) // version 2 spark.range(20, 30).write.format("delta").mode("append").saveAsTable(tblName.get) } else if (path.isDefined) { // version 1 spark.range(10, 20).write.format("delta").mode("append").save(path.get) // version 2 spark.range(20, 30).write.format("delta").mode("append").save(path.get) } } /** Single method to do all kinds of CDC reads */ // By default, we use the `legacy` batch CDF schema mode, in which either latest schema is used // or the time-travelled schema is used. def cdcRead( tblId: TblId, start: Boundary, end: Boundary, schemaMode: Option[DeltaBatchCDFSchemaMode] = Some(BatchCDFSchemaLegacy), readerOptions: Map[String, String] = Map.empty): DataFrame /** Create table utility method */ def ctas(srcTbl: String, dstTbl: String, disableCDC: Boolean = false): Unit = { val readDf = cdcRead(new TableName(srcTbl), StartingVersion("0"), EndingVersion("1")) if (disableCDC) { withSQLConf(DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> "false") { readDf.write.format("delta") .saveAsTable(dstTbl) } } else { readDf.write.format("delta") .saveAsTable(dstTbl) } } private val validTimestampFormats = Seq("yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd") private val invalidTimestampFormats = Seq("yyyyMMddHHmmssSSS") (validTimestampFormats ++ invalidTimestampFormats).foreach { formatStr => val isValid = validTimestampFormats.contains(formatStr) val isValidStr = if (isValid) "valid" else "invalid" test(s"CDF timestamp format - $formatStr is $isValidStr") { withTable("src") { createTblWithThreeVersions(tblName = Some("src")) val timestamp = new SimpleDateFormat(formatStr).format(new Date(1)) def doRead(): Unit = { cdcRead(new TableName("src"), StartingTimestamp(timestamp), EndingVersion("1")) } if (isValid) { doRead() } else { val e = intercept[AnalysisException] { doRead() }.getMessage() assert(e.contains("The provided timestamp")) assert(e.contains("cannot be converted to a valid timestamp")) } } } } testQuietly("writes with metadata columns") { withTable("src", "dst") { // populate src table with CDC data createTblWithThreeVersions(tblName = Some("src")) // writing cdc data to a new table with cdc enabled should fail. the source table has columns // that are reserved for CDC only, and shouldn't be allowed into the target table. val e = intercept[IllegalStateException] { ctas("src", "dst") } val writeContainsCDCColumnsError = DeltaErrors.cdcColumnsInData( cdcReadSchema(new StructType()).fieldNames).getMessage val enablingCDCOnTableWithCDCColumns = DeltaErrors.tableAlreadyContainsCDCColumns( cdcReadSchema(new StructType()).fieldNames).getMessage assert(e.getMessage.contains(writeContainsCDCColumnsError)) // when cdc is disabled writes should work ctas("src", "dst", disableCDC = true) // write some more data withTable("more_data") { spark.range(20, 30) .withColumn(CDC_TYPE_COLUMN_NAME, lit("insert")) .withColumn("_commit_version", lit(2L)) .withColumn("_commit_timestamp", current_timestamp) .write.saveAsTable("more_data") spark.table("more_data").write.format("delta") .mode("append") .saveAsTable("dst") checkAnswer( spark.read.format("delta").table("dst"), cdcRead(new TableName("src"), StartingVersion("0"), EndingVersion("1")) .union(spark.table("more_data")) ) } // re-enabling cdc should be disallowed, since the dst table already contains column that are // reserved for CDC only. val e2 = intercept[IllegalStateException] { sql(s"ALTER TABLE dst SET TBLPROPERTIES " + s"(${DeltaConfigs.CHANGE_DATA_FEED.key}=true)") } assert(e2.getMessage.contains(enablingCDCOnTableWithCDCColumns)) } } // Test that schema evolution on a table with CDC enabled cannot add reserved columns. for (operation <- Seq("merge", "write")) { test(s"schema evolution with CDC reserved column names - op = $operation") { withTable("src", "dst") { // Create target table with CDC enabled. createTblWithThreeVersions(tblName = Some("dst")) // Create the source table containing the CDC of the destination table. ctas(srcTbl = "dst", dstTbl = "src", disableCDC = true) // Write the source back to the target table. val e = intercept[DeltaIllegalStateException] { withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { operation match { case "merge" => spark.sql( """ |MERGE INTO dst USING src |ON dst.id = src.id |WHEN MATCHED THEN UPDATE SET * |WHEN NOT MATCHED THEN INSERT * |""".stripMargin) case "write" => spark.table("src").write.format("delta") .option("mergeSchema", "true").mode("append").saveAsTable("dst") } } } assert(e.getErrorClass === "RESERVED_CDC_COLUMNS_ON_WRITE") } } } test("changes from table by name") { withTable("tbl") { createTblWithThreeVersions(tblName = Some("tbl")) val readDf = cdcRead(new TableName("tbl"), StartingVersion("0"), EndingVersion("1")) checkCDCAnswer( DeltaLog.forTable(spark, TableIdentifier("tbl")), readDf, spark.range(20) .withColumn("_change_type", lit("insert")) .withColumn("_commit_version", (col("id") / 10).cast(LongType)) ) } } test("changes from table by path") { withTempDir { dir => createTblWithThreeVersions(path = Some(dir.getAbsolutePath)) val readDf = cdcRead( new TablePath(dir.getAbsolutePath), StartingVersion("0"), EndingVersion("1")) checkCDCAnswer( DeltaLog.forTable(spark, dir.getAbsolutePath), readDf, spark.range(20) .withColumn("_change_type", lit("insert")) .withColumn("_commit_version", (col("id") / 10).cast(LongType)) ) } } test("changes - start and end are timestamps") { withTempTable(createTable = false) { tableName => createTblWithThreeVersions(tblName = Some(tableName)) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) // modify timestamps // version 0 val currentTime = System.currentTimeMillis() - 5.days.toMillis modifyCommitTimestamp(deltaLog, 0, currentTime + 0) val tsV0 = dateFormat.format(new Date(currentTime)) // version 1 modifyCommitTimestamp(deltaLog, 1, currentTime + 1000) val tsAfterV1 = dateFormat.format(new Date(currentTime + 2000)) modifyCommitTimestamp(deltaLog, 2, currentTime + 3000) val readDf = cdcRead( new TableName(tableName), StartingTimestamp(tsV0), EndingTimestamp(tsAfterV1)) // Answer should include version 0 and version 1, but not version 2. checkCDCAnswer( DeltaLog.forTable(spark, TableIdentifier(tableName)), readDf, spark.range(20) .withColumn("_change_type", lit("insert")) .withColumn("_commit_version", (col("id") / 10).cast(LongType))) } } test("changes - only start is a timestamp") { withTempTable(createTable = false) { tableName => createTblWithThreeVersions(tblName = Some(tableName)) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val currentTime = System.currentTimeMillis() - 5.days.toMillis modifyCommitTimestamp(deltaLog, 0, currentTime + 0) modifyCommitTimestamp(deltaLog, 1, currentTime + 10000) modifyCommitTimestamp(deltaLog, 2, currentTime + 20000) val ts0 = dateFormat.format(new Date(currentTime + 2000)) val readDf = cdcRead(new TableName(tableName), StartingTimestamp(ts0), EndingVersion("1")) checkCDCAnswer( DeltaLog.forTable(spark, TableIdentifier(tableName)), readDf, spark.range(10, 20) .withColumn("_change_type", lit("insert")) .withColumn("_commit_version", (col("id") / 10).cast(LongType))) } } test("changes - only start is a timestamp - inclusive behavior") { withTempTable(createTable = false) { tableName => createTblWithThreeVersions(tblName = Some(tableName)) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val currentTime = System.currentTimeMillis() - 5.days.toMillis modifyCommitTimestamp(deltaLog, 0, currentTime + 0) modifyCommitTimestamp(deltaLog, 1, currentTime + 1000) modifyCommitTimestamp(deltaLog, 2, currentTime + 2000) val ts0 = dateFormat.format(new Date(currentTime + 0)) val readDf = cdcRead(new TableName(tableName), StartingTimestamp(ts0), EndingVersion("1")) checkCDCAnswer( DeltaLog.forTable(spark, TableIdentifier(tableName)), readDf, spark.range(20) .withColumn("_change_type", lit("insert")) .withColumn("_commit_version", (col("id") / 10).cast(LongType))) } } test("version from timestamp - before the first version") { withTempTable(createTable = false) { tableName => createTblWithThreeVersions(tblName = Some(tableName)) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) modifyCommitTimestamp(deltaLog, 0, 4000) modifyCommitTimestamp(deltaLog, 1, 8000) modifyCommitTimestamp(deltaLog, 2, 12000) val ts0 = dateFormat.format(new Date(1000)) val ts1 = dateFormat.format(new Date(3000)) intercept[AnalysisException] { cdcRead(new TableName(tableName), StartingTimestamp(ts0), EndingTimestamp(ts1)).collect() }.getMessage.contains("before the earliest version") } } test("version from timestamp - between two valid versions") { withTempTable(createTable = false) { tableName => createTblWithThreeVersions(tblName = Some(tableName)) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) modifyCommitTimestamp(deltaLog, 0, 0) modifyCommitTimestamp(deltaLog, 1, 4000) modifyCommitTimestamp(deltaLog, 2, 8000) val ts0 = dateFormat.format(new Date(1000)) val ts1 = dateFormat.format(new Date(3000)) val readDf = cdcRead(new TableName(tableName), StartingTimestamp(ts0), EndingTimestamp(ts1)) checkCDCAnswer( DeltaLog.forTable(spark, TableIdentifier(tableName)), readDf, spark.range(0) .withColumn("_change_type", lit("insert")) .withColumn("_commit_version", (col("id") / 10).cast(LongType))) } } test("version from timestamp - one version in between") { withTempTable(createTable = false) { tableName => createTblWithThreeVersions(tblName = Some(tableName)) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val currentTime = System.currentTimeMillis() - 5.days.toMillis modifyCommitTimestamp(deltaLog, 0, currentTime + 0) modifyCommitTimestamp(deltaLog, 1, currentTime + 4000) modifyCommitTimestamp(deltaLog, 2, currentTime + 8000) val ts0 = dateFormat.format(new Date(currentTime + 3000)) val ts1 = dateFormat.format(new Date(currentTime + 5000)) val readDf = cdcRead(new TableName(tableName), StartingTimestamp(ts0), EndingTimestamp(ts1)) checkCDCAnswer( DeltaLog.forTable(spark, TableIdentifier(tableName)), readDf, spark.range(10, 20) .withColumn("_change_type", lit("insert")) .withColumn("_commit_version", (col("id") / 10).cast(LongType))) } } test("version from timestamp - end before start") { withTempTable(createTable = false) { tableName => createTblWithThreeVersions(tblName = Some(tableName)) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) modifyCommitTimestamp(deltaLog, 0, 0) modifyCommitTimestamp(deltaLog, 1, 4000) modifyCommitTimestamp(deltaLog, 2, 8000) val ts0 = dateFormat.format(new Date(3000)) val ts1 = dateFormat.format(new Date(1000)) intercept[DeltaIllegalArgumentException] { cdcRead(new TableName(tableName), StartingTimestamp(ts0), EndingTimestamp(ts1)) .collect() }.getErrorClass === "DELTA_INVALID_CDC_RANGE" } } test("version from timestamp - end before start with one version in between") { withTempTable(createTable = false) { tableName => createTblWithThreeVersions(tblName = Some(tableName)) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) modifyCommitTimestamp(deltaLog, 0, 0) modifyCommitTimestamp(deltaLog, 1, 4000) modifyCommitTimestamp(deltaLog, 2, 8000) val ts0 = dateFormat.format(new Date(5000)) val ts1 = dateFormat.format(new Date(3000)) intercept[DeltaIllegalArgumentException] { cdcRead(new TableName(tableName), StartingTimestamp(ts0), EndingTimestamp(ts1)) .collect() }.getErrorClass === "DELTA_INVALID_CDC_RANGE" } } test("start version and end version are the same") { val tblName = "tbl" withTable(tblName) { createTblWithThreeVersions(tblName = Some(tblName)) val readDf = cdcRead( new TableName(tblName), StartingVersion("0"), EndingVersion("0")) checkCDCAnswer( DeltaLog.forTable(spark, TableIdentifier("tbl")), readDf, spark.range(10) .withColumn("_change_type", lit("insert")) .withColumn("_commit_version", (col("id") / 10).cast(LongType))) } } for (readWithVersionNumber <- BOOLEAN_DOMAIN) test( s"CDC read respects timezone and DST - readWithVersionNumber=$readWithVersionNumber") { val tblName = "tbl" withTable(tblName) { createTblWithThreeVersions(tblName = Some(tblName)) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tblName)) val largeRetentionHours = 2 * System.currentTimeMillis().millis.toHours // Set the deletedFileRetentionDuration and logRetentionDuration to a large value so that // older versions can be accessed spark.sql(s"ALTER TABLE $tblName SET TBLPROPERTIES" + s" ('delta.deletedFileRetentionDuration' = 'interval $largeRetentionHours HOURS'," + s"'delta.logRetentionDuration' = 'interval $largeRetentionHours HOURS')") // Set commit time during Daylight savings time change. val restoreDate = "2022-11-06 01:42:44" val timestamp = dateFormat.parse(s"$restoreDate -0800").getTime modifyCommitTimestamp(deltaLog, 0, timestamp) // Verify DST is respected. val e = intercept[Exception] { cdcRead(new TableName(tblName), StartingTimestamp(s"$restoreDate -0700"), EndingTimestamp(s"$restoreDate -0700")) } assert(e.getMessage.contains("is before the earliest version available")) val readDf = if (readWithVersionNumber) { cdcRead(new TableName(tblName), StartingVersion("0"), EndingVersion("0")) } else { cdcRead( new TableName(tblName), StartingTimestamp(s"$restoreDate -0800"), EndingTimestamp(s"$restoreDate -0800")) } checkCDCAnswer( DeltaLog.forTable(spark, TableIdentifier(tblName)), readDf, spark.range(10) .withColumn("_change_type", lit("insert")) .withColumn("_commit_version", (col("id") / 10).cast(LongType))) } } test("CDC read's commit timestamps are correct under different timezones") { val tblName = "tbl" withTable(tblName) { spark.sql(s"CREATE OR REPLACE TABLE $tblName(id INT, name STRING, age INT) " + s"USING DELTA TBLPROPERTIES (delta.enableChangeDataFeed = true)") spark.sql(s"INSERT INTO $tblName(id, name, age) VALUES (1,'abc',20)") spark.sql(s"INSERT INTO $tblName(id, name, age) VALUES (2,'def',21)") spark.sql(s"UPDATE $tblName SET age = 19 WHERE id = 1") spark.sql(s"INSERT INTO $tblName(id, name, age) VALUES (3,'ghi',15)") spark.sql(s"DELETE FROM $tblName WHERE id = 3") // unix_timestamp() on a Timestamp column returns the UNIX timestamp of the specified // time under the given SESSION_LOCAL_TIMEZONE, while collect() on a timestamp column // always returns the Timestamp in UTC. // By using unix_timestamp() on the commit timestamp column, we can accurately determine // whether or not the timestamp under different timezones represent the same point in time. val startingVersion = StartingVersion("0") val endingVersion = EndingVersion("10") spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "America/Chicago") val readDfChicago = cdcRead(new TableName(tblName), startingVersion, endingVersion) .orderBy(CDC_COMMIT_VERSION, CDC_TYPE_COLUMN_NAME) .select(col(CDC_COMMIT_VERSION), col(CDC_TYPE_COLUMN_NAME), unix_timestamp(col(CDC_COMMIT_TIMESTAMP))) val readDfChicagoRows = readDfChicago.collect() spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "Asia/Ho_Chi_Minh") val readDfHCM = cdcRead(new TableName(tblName), startingVersion, endingVersion) .orderBy(CDC_COMMIT_VERSION, CDC_TYPE_COLUMN_NAME) .select(col(CDC_COMMIT_VERSION), col(CDC_TYPE_COLUMN_NAME), unix_timestamp(col(CDC_COMMIT_TIMESTAMP))) val readDfHCMRows = readDfHCM.collect() spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "UTC") val readDfUTC = cdcRead(new TableName(tblName), startingVersion, endingVersion) .orderBy(CDC_COMMIT_VERSION, CDC_TYPE_COLUMN_NAME) .select(col(CDC_COMMIT_VERSION), col(CDC_TYPE_COLUMN_NAME), unix_timestamp(col(CDC_COMMIT_TIMESTAMP))) val readDfUTCRows = readDfUTC.collect() def checkCDCTimestampEqual(firstRows: Array[Row], secondRows: Array[Row]): Boolean = { assert(firstRows.length === secondRows.length, "Number of rows from 2 DFs should be the same.") for ((firstRow, secondRow) <- firstRows.zip(secondRows)) { assert(firstRow.getLong(0) === secondRow.getLong(0), "Commit version should be the same for every rows.") assert(firstRow.getString(1) === secondRow.getString(1), "Change type should be the same for every rows.") if (firstRow.getLong(2) != secondRow.getLong(2)) { return false } } true } assert(checkCDCTimestampEqual(readDfChicagoRows, readDfHCMRows) === true) assert(checkCDCTimestampEqual(readDfChicagoRows, readDfUTCRows) === true) assert(checkCDCTimestampEqual(readDfHCMRows, readDfUTCRows) === true) } } test("start version is provided and no end version") { val tblName = "tbl" withTable(tblName) { createTblWithThreeVersions(tblName = Some(tblName)) val readDf = cdcRead( new TableName(tblName), StartingVersion("0"), Unbounded) checkCDCAnswer( DeltaLog.forTable(spark, TableIdentifier("tbl")), readDf, spark.range(30) .withColumn("_change_type", lit("insert")) .withColumn("_commit_version", (col("id") / 10).cast(LongType))) } } test("end timestamp < start timestamp") { withTempTable(createTable = false) { tableName => createTblWithThreeVersions(tblName = Some(tableName)) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) modifyCommitTimestamp(deltaLog, 0, 0) modifyCommitTimestamp(deltaLog, 1, 1000) modifyCommitTimestamp(deltaLog, 2, 2000) val ts0 = dateFormat.format(new Date(2000)) val ts1 = dateFormat.format(new Date(1)) val e = intercept[IllegalArgumentException] { cdcRead(new TableName(tableName), StartingTimestamp(ts0), EndingTimestamp(ts1)) } assert(e.getMessage.contains("End cannot be before start")) } } test("end version < start version") { val tblName = "tbl" withTable(tblName) { createTblWithThreeVersions(tblName = Some(tblName)) val e = intercept[IllegalArgumentException] { cdcRead(new TableName(tblName), StartingVersion("1"), EndingVersion("0")) } assert(e.getMessage.contains("End cannot be before start")) } } test("cdc result dataframe can be transformed further") { val tblName = "tbl" withTable(tblName) { createTblWithThreeVersions(tblName = Some(tblName)) val cdcResult = cdcRead(new TableName(tblName), StartingVersion("0"), EndingVersion("1")) val transformedDf = cdcResult .drop(CDC_COMMIT_TIMESTAMP) .withColumn("col3", lit(0)) .withColumn("still_there", col("_change_type")) checkAnswer( transformedDf, spark.range(20) .withColumn("_change_type", lit("insert")) .withColumn("_commit_version", (col("id") / 10).cast(LongType)) .withColumn("col3", lit(0)) .withColumn("still_there", col("_change_type")) ) } } test("multiple references on same table") { val tblName = "tbl" withTable(tblName) { createTblWithThreeVersions(tblName = Some(tblName)) val cdcResult0_1 = cdcRead(new TableName(tblName), StartingVersion("0"), EndingVersion("1")) val cdcResult0_2 = cdcRead(new TableName(tblName), StartingVersion("0"), EndingVersion("2")) val diff = cdcResult0_2.except(cdcResult0_1) checkCDCAnswer( DeltaLog.forTable(spark, TableIdentifier("tbl")), diff, spark.range(20, 30) .withColumn("_change_type", lit("insert")) .withColumn("_commit_version", (col("id") / 10).cast(LongType)) ) } } test("filtering cdc metadata columns") { val tblName = "tbl" withTable(tblName) { createTblWithThreeVersions(tblName = Some(tblName)) val deltaTable = io.delta.tables.DeltaTable.forName("tbl") deltaTable.delete("id > 20") val cdcResult = cdcRead(new TableName(tblName), StartingVersion("0"), EndingVersion("3")) checkCDCAnswer( DeltaLog.forTable(spark, TableIdentifier("tbl")), cdcResult.filter("_change_type != 'insert'"), spark.range(21, 30) .withColumn("_change_type", lit("delete")) .withColumn("_commit_version", lit(3)) ) checkCDCAnswer( DeltaLog.forTable(spark, TableIdentifier("tbl")), cdcResult.filter("_commit_version = 1"), spark.range(10, 20) .withColumn("_change_type", lit("insert")) .withColumn("_commit_version", lit(1)) ) } } test("aggregating non-numeric cdc data columns") { withTempTable(createTable = false) { tableName => spark.range(10).selectExpr("id", "'text' as text") .write.format("delta").saveAsTable(tableName) val deltaTable = io.delta.tables.DeltaTable.forName(tableName) deltaTable.delete("id > 5") val cdcResult = cdcRead(new TableName(tableName), StartingVersion("0"), EndingVersion("3")) checkAnswer( cdcResult.selectExpr("count(distinct text)"), Row(1) ) checkAnswer( cdcResult.selectExpr("first(text)"), Row("text") ) } } test("ending version not specified resolves to latest at execution time") { withTempTable(createTable = false) { tableName => spark.range(5).selectExpr("id", "'text' as text") .write.format("delta").saveAsTable(tableName) val cdcResult = cdcRead(new TableName(tableName), StartingVersion("0"), Unbounded) checkAnswer( cdcResult.selectExpr("id", "_change_type", "_commit_version"), Row(0, "insert", 0) :: Row(1, "insert", 0) :: Row(2, "insert", 0) :: Row(3, "insert", 0):: Row(4, "insert", 0) :: Nil ) // The next scan of `cdcResult` should include this delete even though the DF was defined // before it. val deltaTable = io.delta.tables.DeltaTable.forName(tableName) deltaTable.delete("id > 2") checkAnswer( cdcResult.selectExpr("id", "_change_type", "_commit_version"), Row(0, "insert", 0) :: Row(1, "insert", 0) :: Row(2, "insert", 0) :: Row(3, "insert", 0):: Row(4, "insert", 0) :: Row(3, "delete", 1):: Row(4, "delete", 1) :: Nil ) } } test("table schema changed after dataframe with ending specified") { withTempTable(createTable = false) { tableName => spark.range(5).selectExpr("id", "'text' as text") .write.format("delta").saveAsTable(tableName) val cdcResult = cdcRead(new TableName(tableName), StartingVersion("0"), EndingVersion("1")) sql(s"ALTER TABLE $tableName ADD COLUMN (newCol INT)") checkAnswer( cdcResult.selectExpr("id", "_change_type", "_commit_version"), Row(0, "insert", 0) :: Row(1, "insert", 0) :: Row(2, "insert", 0) :: Row(3, "insert", 0) :: Row(4, "insert", 0) :: Nil ) } } test("table schema changed after dataframe with ending not specified") { withTempTable(createTable = false) { tableName => spark.range(5).selectExpr("id", "'text' as text") .write.format("delta").saveAsTable(tableName) val cdcResult = cdcRead(new TableName(tableName), StartingVersion("0"), Unbounded) sql(s"ALTER TABLE $tableName ADD COLUMN (newCol STRING)") sql(s"INSERT INTO $tableName VALUES (5, 'text', 'newColVal')") // Just ignoring the new column is pretty weird, but it's what we do for non-CDC dataframes, // so we preserve the behavior rather than adding a special case. checkAnswer( cdcResult.selectExpr("id", "_change_type", "_commit_version"), Row(0, "insert", 0) :: Row(1, "insert", 0) :: Row(2, "insert", 0) :: Row(3, "insert", 0) :: Row(4, "insert", 0) :: Row(5, "insert", 2) :: Nil ) } } test("An error should be thrown when CDC is not enabled") { val tblName = "tbl" withTable(tblName) { withSQLConf(DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> "false") { // create version with cdc disabled - v0 spark.range(10).write.format("delta").saveAsTable(tblName) } val deltaTable = io.delta.tables.DeltaTable.forName(tblName) // v1 deltaTable.delete("id > 8") // v2 sql(s"ALTER TABLE ${tblName} SET TBLPROPERTIES " + s"(${DeltaConfigs.CHANGE_DATA_FEED.key}=true)") // v3 spark.range(10, 20).write.format("delta").mode("append").saveAsTable(tblName) // v4 deltaTable.delete("id > 18") // v5 sql(s"ALTER TABLE ${tblName} SET TBLPROPERTIES " + s"(${DeltaConfigs.CHANGE_DATA_FEED.key}=false)") var e = intercept[AnalysisException] { cdcRead(new TableName(tblName), StartingVersion("0"), EndingVersion("4")).collect() } assert(e.getMessage === DeltaErrors.changeDataNotRecordedException(0, 0, 4).getMessage) val cdcDf = cdcRead(new TableName(tblName), StartingVersion("2"), EndingVersion("4")) assert(cdcDf.count() == 11) // 10 rows inserted, 1 row deleted // Check that we correctly detect CDC is disabled and fail the query for multiple types of // ranges: // * disabled at the end but not start - (2, 5) // * disabled at the start but not end - (1, 4) // * disabled at both start and end (even though enabled in the middle) - (1, 5) for ((start, end, firstDisabledVersion) <- Seq((2, 5, 5), (1, 4, 1), (1, 5, 1))) { e = intercept[AnalysisException] { cdcRead( new TableName(tblName), StartingVersion(start.toString), EndingVersion(end.toString)).collect() } assert(e.getMessage === DeltaErrors.changeDataNotRecordedException( firstDisabledVersion, start, end).getMessage) } } } test("changes - start timestamp exceeding latest commit timestamp") { withTempTable(createTable = false) { tableName => withSQLConf(DeltaSQLConf.DELTA_CDF_ALLOW_OUT_OF_RANGE_TIMESTAMP.key -> "true") { createTblWithThreeVersions(tblName = Some(tableName)) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) // modify timestamps // version 0 modifyCommitTimestamp(deltaLog, 0, 0) // version 1 modifyCommitTimestamp(deltaLog, 1, 1000) // version 2 modifyCommitTimestamp(deltaLog, 2, 2000) val tsStart = dateFormat.format(new Date(3000)) val tsEnd = dateFormat.format(new Date(4000)) val readDf = cdcRead( new TableName(tableName), StartingTimestamp(tsStart), EndingTimestamp(tsEnd)) checkCDCAnswer( DeltaLog.forTable(spark, TableIdentifier(tableName)), readDf, sqlContext.emptyDataFrame) } } } test("changes - end timestamp exceeding latest commit timestamp") { withTempTable(createTable = false) { tableName => withSQLConf(DeltaSQLConf.DELTA_CDF_ALLOW_OUT_OF_RANGE_TIMESTAMP.key -> "true") { createTblWithThreeVersions(tblName = Some(tableName)) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val currentTime = System.currentTimeMillis() - 5.days.toMillis // modify timestamps // version 0 modifyCommitTimestamp(deltaLog, 0, currentTime + 0) // version 1 modifyCommitTimestamp(deltaLog, 1, currentTime + 1000) // version 2 modifyCommitTimestamp(deltaLog, 2, currentTime + 2000) val tsStart = dateFormat.format(new Date(currentTime + 0)) val tsEnd = dateFormat.format(new Date(currentTime + 4000)) val readDf = cdcRead( new TableName(tableName), StartingTimestamp(tsStart), EndingTimestamp(tsEnd)) checkCDCAnswer( DeltaLog.forTable(spark, TableIdentifier(tableName)), readDf, spark.range(30) .withColumn("_change_type", lit("insert")) .withColumn("_commit_version", (col("id") / 10).cast(LongType))) } } } test("batch write: append, dynamic partition overwrite + CDF") { withSQLConf( DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> "true", DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED.key -> "true") { withTempTable(createTable = false) { tableName => def data: DataFrame = spark.read.format("delta").table(tableName) Seq(("a", "x"), ("b", "y"), ("c", "x")).toDF("value", "part") .write .format("delta") .partitionBy("part") .mode("append") .saveAsTable(tableName) checkAnswer( cdcRead(new TableName(tableName), StartingVersion("0"), EndingVersion("0")) .drop(CDC_COMMIT_TIMESTAMP), Row("a", "x", "insert", 0) :: Row("b", "y", "insert", 0) :: Row("c", "x", "insert", 0) :: Nil ) // ovewrite nothing Seq(("d", "z")).toDF("value", "part") .write .format("delta") .partitionBy("part") .mode("overwrite") .option(DeltaOptions.PARTITION_OVERWRITE_MODE_OPTION, "dynamic") .saveAsTable(tableName) checkDatasetUnorderly(data.select("value", "part").as[(String, String)], ("a", "x"), ("b", "y"), ("c", "x"), ("d", "z")) checkAnswer( cdcRead(new TableName(tableName), StartingVersion("1"), EndingVersion("1")) .drop(CDC_COMMIT_TIMESTAMP), Row("d", "z", "insert", 1) :: Nil ) // overwrite partition `part`="x" Seq(("a", "x"), ("e", "x")).toDF("value", "part") .write .format("delta") .partitionBy("part") .mode("overwrite") .option(DeltaOptions.PARTITION_OVERWRITE_MODE_OPTION, "dynamic") .saveAsTable(tableName) checkDatasetUnorderly(data.select("value", "part").as[(String, String)], ("a", "x"), ("b", "y"), ("d", "z"), ("e", "x")) checkAnswer( cdcRead(new TableName(tableName), StartingVersion("2"), EndingVersion("2")) .drop(CDC_COMMIT_TIMESTAMP), Row("a", "x", "delete", 2) :: Row("c", "x", "delete", 2) :: Row("a", "x", "insert", 2) :: Row("e", "x", "insert", 2) :: Nil ) } } } } class DeltaCDCScalaSuite extends DeltaCDCSuiteBase { /** Single method to do all kinds of CDC reads */ def cdcRead( tblId: TblId, start: Boundary, end: Boundary, schemaMode: Option[DeltaBatchCDFSchemaMode] = Some(BatchCDFSchemaLegacy), readerOptions: Map[String, String] = Map.empty): DataFrame = { // Set the batch CDF schema mode using SQL conf if we specified it if (schemaMode.isDefined) { var result: DataFrame = null withSQLConf(DeltaSQLConf.DELTA_CDF_DEFAULT_SCHEMA_MODE_FOR_COLUMN_MAPPING_TABLE.key -> schemaMode.get.name) { result = cdcRead(tblId, start, end, None, readerOptions) } return result } val startPrefix: (String, String) = start match { case startingVersion: StartingVersion => ("startingVersion", startingVersion.value) case startingTimestamp: StartingTimestamp => ("startingTimestamp", startingTimestamp.value) case Unbounded => ("", "") } val endPrefix: (String, String) = end match { case endingVersion: EndingVersion => ("endingVersion", endingVersion.value) case endingTimestamp: EndingTimestamp => ("endingTimestamp", endingTimestamp.value) case Unbounded => ("", "") } var dfr = spark.read.format("delta") .option(DeltaOptions.CDC_READ_OPTION, "true") .option(startPrefix._1, startPrefix._2) .option(endPrefix._1, endPrefix._2) readerOptions.foreach { case (k, v) => dfr = dfr.option(k, v) } tblId match { case path: TablePath => dfr.load(path.id) case tblName: TableName => dfr.table(tblName.id) case _ => throw new IllegalArgumentException("No table name or path provided") } } private def testNullRangeBoundary(start: Boundary, end: Boundary): Unit = { test(s"range boundary cannot be null - start=$start end=$end") { val tblName = "tbl" withTable(tblName) { createTblWithThreeVersions(tblName = Some(tblName)) val expectedError = (start, end) match { case (StartingVersion(null), _) => "DELTA_VERSION_INVALID" case (StartingTimestamp(null), _) => "DELTA_TIMESTAMP_INVALID" case (_, EndingVersion(null)) => "DELTA_VERSION_INVALID" case (_, EndingTimestamp(null)) => "DELTA_TIMESTAMP_INVALID" } val expectedErrorParameters = expectedError match { case "DELTA_VERSION_INVALID" => Map("version" -> "null") case "DELTA_TIMESTAMP_INVALID" => Map("expr" -> "NULL") } checkError( intercept[DeltaAnalysisException] { cdcRead(new TableName(tblName), start, end) }, expectedError, parameters = expectedErrorParameters) } } } for { start <- Seq(StartingVersion("0"), StartingTimestamp(dateFormat.format(new Date(1)))) end <- Seq(EndingVersion(null), EndingTimestamp(null)) } { testNullRangeBoundary(start, end) } for { start <- Seq(StartingVersion(null), StartingTimestamp(null)) end <- Seq( Unbounded, EndingVersion(null), EndingTimestamp(null), EndingVersion("0"), EndingTimestamp(dateFormat.format(new Date(1)))) } { testNullRangeBoundary(start, end) } test("filters should be pushed down") { val tblName = "tbl" withTable(tblName) { createTblWithThreeVersions(tblName = Some(tblName)) val plans = DeltaTestUtils.withAllPlansCaptured(spark) { val res = spark.read.format("delta") .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingVersion", 0) .option("endingVersion", 1) .table(tblName) .select("id", "_change_type") .where(col("id") < lit(5)) assert(res.columns === Seq("id", "_change_type")) checkAnswer( res, spark.range(5) .withColumn("_change_type", lit("insert"))) } assert(plans.map(_.executedPlan).toString .contains("PushedFilters: [*IsNotNull(id), *LessThan(id,5)]")) } } test("start version or timestamp is not provided") { val tblName = "tbl" withTable(tblName) { createTblWithThreeVersions(tblName = Some(tblName)) val e = intercept[AnalysisException] { spark.read.format("delta") .option(DeltaOptions.CDC_READ_OPTION, "true") .option("endingVersion", 1) .table(tblName) .show() } assert(e.getMessage.contains(DeltaErrors.noStartVersionForCDC().getMessage)) } } test("Not having readChangeFeed will not output cdc columns") { val tblName = "tbl2" withTable(tblName) { spark.range(0, 10).write.format("delta").saveAsTable(tblName) checkAnswer(spark.read.format("delta").table(tblName), spark.range(0, 10).toDF("id")) checkAnswer( spark.read.format("delta") .option("startingVersion", "0") .option("endingVersion", "0") .table(tblName), spark.range(0, 10).toDF("id")) } } test("non-monotonic timestamps") { withTempTable(createTable = false) { tableName => var deltaLog: DeltaLog = null val currentTime = System.currentTimeMillis() - 5.days.toMillis (0 to 3).foreach { i => spark.range(i * 10, (i + 1) * 10).write.format("delta").mode("append") .saveAsTable(tableName) if (i == 0) { deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) } val file = new File(FileNames.unsafeDeltaFile(deltaLog.logPath, i).toUri) file.setLastModified(currentTime + 300 - i) } checkCDCAnswer( deltaLog, cdcRead(new TableName(tableName), StartingVersion("0"), EndingVersion("3")), spark.range(0, 40) .withColumn("_change_type", lit("insert")) .withColumn("_commit_version", floor(col("id") / 10))) } } test("Repeated delete") { withTempTable(createTable = false) { tableName => spark.range(0, 5, 1, numPartitions = 1).write.format("delta").saveAsTable(tableName) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) sql(s"DELETE FROM $tableName WHERE id = 3") // Version 1 sql(s"DELETE FROM $tableName WHERE id = 4") // Version 2 sql(s"DELETE FROM $tableName WHERE id IN (0, 1, 2)") // Version 3, remove the whole file val allChanges: Map[Int, Seq[Row]] = Map( 1 -> (Row(3, "delete", 1) :: Nil), 2 -> (Row(4, "delete", 2) :: Nil), 3 -> (Row(0, "delete", 3) :: Row(1, "delete", 3) :: Row(2, "delete", 3) :: Nil) ) for(start <- 1 to 3; end <- start to 3) { checkCDCAnswer( deltaLog, cdcRead( new TableName(tableName), StartingVersion(start.toString), EndingVersion(end.toString)), (start to end).flatMap(v => allChanges(v))) } } } test("reader should accept case insensitive option") { val tblName = "tbl" withTable(tblName) { createTblWithThreeVersions(tblName = Some(tblName)) val res = spark.read.format("delta") .option("ReadChangeFEED", "tRuE") .option("STARTINGVERSION", 0) .option("endingVersion", 1) .table(tblName) .select("id", "_change_type") assert(res.columns === Seq("id", "_change_type")) checkAnswer( res, spark.range(20).withColumn("_change_type", lit("insert"))) val resLegacy = spark.read.format("delta") .option("READCHANGEDATA", "TruE") .option("startingversion", 0) .option("ENDINGVERSION", 1) .table(tblName) .select("id", "_change_type") assert(resLegacy.columns === Seq("id", "_change_type")) checkAnswer( resLegacy, spark.range(20).withColumn("_change_type", lit("insert"))) } } } class DeltaCDCScalaWithDeletionVectorsSuite extends DeltaCDCScalaSuite with DeletionVectorsTestUtils { override def beforeAll(): Unit = { super.beforeAll() enableDeletionVectorsForAllSupportedOperations(spark) } } class DeltaCDCScalaWithCatalogOwnedBatch1Suite extends DeltaCDCScalaSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class DeltaCDCScalaWithCatalogOwnedBatch2Suite extends DeltaCDCScalaSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class DeltaCDCScalaWithCatalogOwnedBatch100Suite extends DeltaCDCScalaSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaCheckpointWithStructColsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import org.apache.spark.sql.delta.schema.SchemaMergingUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.FileNames import org.apache.spark.sql.{DataFrame, QueryTest, Row} import org.apache.spark.sql.functions.{col, lit, struct} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ class DeltaCheckpointWithStructColsSuite extends QueryTest with SharedSparkSession with DeltaColumnMappingTestUtils with DeltaSQLCommandTest { import testImplicits._ protected val checkpointFnsWithStructAndJsonStats: Seq[DeltaLog => Long] = Seq( checkpointWithProperty(writeStatsAsJson = Some(true)), checkpointWithProperty(writeStatsAsJson = None)) protected val checkpointFnsWithStructWithoutJsonStats: Seq[DeltaLog => Long] = Seq( checkpointWithProperty(writeStatsAsJson = Some(false))) protected val checkpointFnsWithoutStructWithJsonStats: Seq[DeltaLog => Long] = Seq( checkpointWithProperty(writeStatsAsJson = Some(true), writeStatsAsStruct = false), checkpointWithProperty(writeStatsAsJson = None, writeStatsAsStruct = false)) /** * Creates a table from the given DataFrame and partitioning. Then for each checkpointing * function, it runs the given validation function. */ protected def checkpointSchemaForTable(df: DataFrame, partitionBy: String*)( checkpointingFns: Seq[DeltaLog => Long], expectedCols: Seq[(String, DataType)], additionalValidationFn: Set[String] => Unit = _ => ()): Unit = { checkpointingFns.foreach { checkpointingFn => withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) df.write.format("delta").partitionBy(partitionBy: _*).save(dir.getCanonicalPath) val version = checkpointingFn(deltaLog) val f = spark.read.parquet( FileNames.checkpointFileSingular(deltaLog.logPath, version).toString) assert(f.schema.getFieldIndex("commitInfo").isEmpty, "commitInfo should not be written to the checkpoint") val baseCols = Set("add", "metaData", "protocol", "remove", "txn") baseCols.foreach { name => assert(f.schema.getFieldIndex(name).nonEmpty, s"Couldn't find required field $name " + s"among: ${f.schema.fieldNames.mkString("[", ", ", " ]")}") } val addSchema = f.schema("add").dataType.asInstanceOf[StructType] val addColumns = SchemaMergingUtils.explodeNestedFieldNames(addSchema).toSet val requiredCols = Seq( "path" -> StringType, "partitionValues" -> MapType(StringType, StringType), "size" -> LongType, "modificationTime" -> LongType, "dataChange" -> BooleanType, "tags" -> MapType(StringType, StringType) ) val schema = deltaLog.update().schema (requiredCols ++ expectedCols).foreach { case (expectedField, dataType) => // use physical name if possible val expectedPhysicalField = convertColumnNameToAttributeWithPhysicalName(expectedField, schema).name assert(addColumns.contains(expectedPhysicalField)) // Check data type assert(f.select(col(s"add.$expectedPhysicalField")).schema.head.dataType === dataType) } additionalValidationFn(addColumns) DeltaLog.clearCache() checkAnswer( spark.read.format("delta").load(dir.getCanonicalPath), df ) } } } test("unpartitioned table") { val df = spark.range(10).withColumn("part", ('id / 2).cast("int")) checkpointSchemaForTable(df)( checkpointingFns = checkpointFnsWithStructAndJsonStats, expectedCols = Nil, additionalValidationFn = addColumns => { checkFields( addColumns, statsAsJsonExists = true, unexpected = Seq(Checkpoints.STRUCT_PARTITIONS_COL_NAME)) } ) checkpointSchemaForTable(df)( checkpointingFns = checkpointFnsWithStructWithoutJsonStats, expectedCols = Nil, additionalValidationFn = addColumns => { checkFields( addColumns, statsAsJsonExists = false, unexpected = Seq(Checkpoints.STRUCT_PARTITIONS_COL_NAME)) } ) checkpointSchemaForTable(df)( checkpointingFns = checkpointFnsWithoutStructWithJsonStats, expectedCols = Nil, additionalValidationFn = addColumns => { checkFields( addColumns, statsAsJsonExists = true, unexpected = Seq(Checkpoints.STRUCT_PARTITIONS_COL_NAME)) } ) checkpointSchemaForTable(df)( checkpointingFns = Seq(checkpointWithoutStats), expectedCols = Nil, additionalValidationFn = addColumns => { checkFields( addColumns, statsAsJsonExists = false, Seq(Checkpoints.STRUCT_PARTITIONS_COL_NAME)) } ) } test("partitioned table") { val df = spark.range(10).withColumn("part", ('id / 2).cast("int")) // partitioned by "part" checkpointSchemaForTable(df, "part")( checkpointingFns = checkpointFnsWithStructAndJsonStats, expectedCols = Seq("partitionValues_parsed.part" -> IntegerType), additionalValidationFn = addColumns => { checkFields( addColumns, statsAsJsonExists = true, Nil) } ) checkpointSchemaForTable(df, "part")( checkpointingFns = checkpointFnsWithStructWithoutJsonStats, expectedCols = Seq("partitionValues_parsed.part" -> IntegerType), additionalValidationFn = addColumns => { checkFields( addColumns, statsAsJsonExists = false, Nil) } ) checkpointSchemaForTable(df, "part")( checkpointingFns = checkpointFnsWithoutStructWithJsonStats, expectedCols = Nil, additionalValidationFn = addColumns => { checkFields( addColumns, statsAsJsonExists = true, Seq(Checkpoints.STRUCT_PARTITIONS_COL_NAME)) } ) checkpointSchemaForTable(df, "part")( checkpointingFns = Seq(checkpointWithoutStats), expectedCols = Nil, additionalValidationFn = addColumns => { checkFields( addColumns, statsAsJsonExists = false, Seq(Checkpoints.STRUCT_PARTITIONS_COL_NAME)) } ) } test("special characters") { val weirdName1 = "part%!@#_$%^&*-" val weirdName2 = "part?_.+<>|/" val df = spark.range(10) .withColumn(weirdName1, ('id / 2).cast("int")) .withColumn(weirdName2, ('id / 3).cast("int")) .withColumn("struct", struct($"id", col(weirdName1), $"id".as(weirdName2))) val structColumns = Seq( s"partitionValues_parsed.$weirdName1" -> IntegerType, s"partitionValues_parsed.`$weirdName2`" -> IntegerType) // partitioned by weirdName1, weirdName2 checkpointSchemaForTable(df, weirdName1, weirdName2)( checkpointingFns = checkpointFnsWithStructAndJsonStats, expectedCols = structColumns, additionalValidationFn = addColumns => { checkFields( addColumns, statsAsJsonExists = true, Nil) } ) checkpointSchemaForTable(df, weirdName1, weirdName2)( checkpointingFns = checkpointFnsWithStructWithoutJsonStats, expectedCols = Nil, additionalValidationFn = addColumns => { checkFields( addColumns, statsAsJsonExists = false, Nil) } ) checkpointSchemaForTable(df, weirdName1, weirdName2)( checkpointingFns = checkpointFnsWithoutStructWithJsonStats, expectedCols = Nil, additionalValidationFn = addColumns => { checkFields( addColumns, statsAsJsonExists = true, structColumns.map(_._1)) } ) } test("timestamps as partition values") { withTempDir { dir => val df = Seq( (java.sql.Timestamp.valueOf("2012-12-31 16:00:10.011"), 2), (java.sql.Timestamp.valueOf("2099-12-31 16:00:10.011"), 4)).toDF("key", "value") df.write.format("delta").partitionBy("key").save(dir.getCanonicalPath) val deltaLog = DeltaLog.forTable(spark, dir) val version = checkpointWithProperty( writeStatsAsJson = Some(true), writeStatsAsStruct = true)(deltaLog) val f = spark.read.parquet( FileNames.checkpointFileSingular(deltaLog.logPath, version).toString) // use physical name val key = getPhysicalName("key", deltaLog.snapshot.schema) checkAnswer( f.select(s"add.partitionValues_parsed.`$key`"), Seq(Row(null), Row(null)) ++ df.select("key").collect() ) sql(s"DELETE FROM delta.`${dir.getCanonicalPath}` WHERE CURRENT_TIMESTAMP > key") checkAnswer( spark.read.format("delta").load(dir.getCanonicalPath), Row(java.sql.Timestamp.valueOf("2099-12-31 16:00:10.011"), 4) ) sql(s"DELETE FROM delta.`${dir.getCanonicalPath}` WHERE CURRENT_TIMESTAMP < key") } } /** * Creates a checkpoint by based on `writeStatsAsJson`/`writeStatsAsStruct` properties. */ protected def checkpointWithProperty( writeStatsAsJson: Option[Boolean], writeStatsAsStruct: Boolean = true)(deltaLog: DeltaLog): Long = { val asJson = writeStatsAsJson.map { v => s", delta.checkpoint.writeStatsAsJson = $v" }.getOrElse("") sql(s"ALTER TABLE delta.`${deltaLog.dataPath}` " + s"SET TBLPROPERTIES (delta.checkpoint.writeStatsAsStruct = ${writeStatsAsStruct}${asJson})") deltaLog.checkpoint() deltaLog.readLastCheckpointFile().get.version } /** A checkpoint that doesn't have any stats columns, i.e. `stats` and `stats_parsed`. */ protected def checkpointWithoutStats(deltaLog: DeltaLog): Long = { sql(s"ALTER TABLE delta.`${deltaLog.dataPath}` " + s"SET TBLPROPERTIES (delta.checkpoint.writeStatsAsStruct = false, " + "delta.checkpoint.writeStatsAsJson = false)") deltaLog.checkpoint() deltaLog.readLastCheckpointFile().get.version } /** * Check the existence of the stats field and also not existence of the `unexpected` fields. The * `addColumns` is a Set of column names that contain the entire tree of columns in the `add` * field of the schema. */ protected def checkFields( addColumns: Set[String], statsAsJsonExists: Boolean, unexpected: Seq[String]): Unit = { if (statsAsJsonExists) { assert(addColumns.contains("stats")) } else { assert(!addColumns.contains("stats")) } unexpected.foreach { colName => assert(!addColumns.contains(colName), s"$colName shouldn't be part of the " + "schema because it is of null type.") } } test("unpartitioned table - check stats") { val df = spark.range(10).withColumn("part", ('id / 2).cast("int")) val structStatsColumns = Seq( "stats_parsed.numRecords" -> LongType, "stats_parsed.minValues.id" -> LongType, "stats_parsed.maxValues.id" -> LongType, "stats_parsed.nullCount.id" -> LongType, "stats_parsed.minValues.part" -> IntegerType, "stats_parsed.maxValues.part" -> IntegerType, "stats_parsed.nullCount.part" -> LongType) checkpointSchemaForTable(df)( checkpointingFns = checkpointFnsWithStructAndJsonStats, expectedCols = structStatsColumns, additionalValidationFn = addColumns => { checkFields(addColumns, statsAsJsonExists = true, Nil) } ) checkpointSchemaForTable(df)( checkpointingFns = checkpointFnsWithStructWithoutJsonStats, expectedCols = structStatsColumns, additionalValidationFn = addColumns => { checkFields(addColumns, statsAsJsonExists = false, Nil) } ) checkpointSchemaForTable(df)( checkpointingFns = checkpointFnsWithoutStructWithJsonStats, expectedCols = Nil, additionalValidationFn = addColumns => { checkFields( addColumns, statsAsJsonExists = true, unexpected = Seq(Checkpoints.STRUCT_PARTITIONS_COL_NAME) ++ structStatsColumns.map(_._1)) } ) checkpointSchemaForTable(df)( checkpointingFns = Seq(checkpointWithoutStats), expectedCols = Nil, additionalValidationFn = addColumns => { checkFields(addColumns, statsAsJsonExists = false, structStatsColumns.map(_._1)) } ) } test("use kill switch to disable stats as struct in checkpoint") { withSQLConf(DeltaSQLConf.STATS_AS_STRUCT_IN_CHECKPOINT_FORCE_DISABLED.key -> "true") { val df = spark.range(10).withColumn("part", ('id / 2).cast("int")) val structStatsColumns = Seq( "stats_parsed.numRecords", "stats_parsed.minValues.id", "stats_parsed.maxValues.id", "stats_parsed.nullCount.id", "stats_parsed.minValues.part", "stats_parsed.maxValues.part", "stats_parsed.nullCount.part") checkpointSchemaForTable(df)( checkpointingFns = checkpointFnsWithStructAndJsonStats, expectedCols = Nil, additionalValidationFn = addColumns => { checkFields( addColumns, statsAsJsonExists = true, unexpected = structStatsColumns ) } ) } } test("partitioned table - check stats") { val df = spark.range(10).withColumn("part", ('id / 2).cast("int")) val structStatsColumns = Seq( "stats_parsed.numRecords" -> LongType, "stats_parsed.minValues.id" -> LongType, "stats_parsed.maxValues.id" -> LongType, "stats_parsed.nullCount.id" -> LongType) // partitioned by "part" checkpointSchemaForTable(df, "part")( checkpointingFns = checkpointFnsWithStructAndJsonStats, expectedCols = structStatsColumns, additionalValidationFn = addColumns => { checkFields(addColumns, statsAsJsonExists = true, Nil) } ) checkpointSchemaForTable(df, "part")( checkpointingFns = checkpointFnsWithStructWithoutJsonStats, expectedCols = structStatsColumns, additionalValidationFn = addColumns => { checkFields(addColumns, statsAsJsonExists = false, Nil) } ) checkpointSchemaForTable(df, "part")( checkpointingFns = checkpointFnsWithoutStructWithJsonStats, expectedCols = Nil, additionalValidationFn = addColumns => { checkFields( addColumns, statsAsJsonExists = true, structStatsColumns.map(_._1)) } ) checkpointSchemaForTable(df, "part")( checkpointingFns = Seq(checkpointWithoutStats), expectedCols = Nil, additionalValidationFn = addColumns => { checkFields(addColumns, statsAsJsonExists = false, structStatsColumns.map(_._1)) } ) } test("nested fields, dots, and boolean types") { val df = spark.range(10).withColumn("part", ('id / 2).cast("int")) .withColumn("struct", struct($"id", $"part", $"id".as("with.dot"))) .withColumn("bool", lit(true)) val structColumns = Seq( "partitionValues_parsed.part" -> IntegerType, "stats_parsed.numRecords" -> LongType, "stats_parsed.minValues.id" -> LongType, "stats_parsed.maxValues.id" -> LongType, "stats_parsed.nullCount.id" -> LongType, "stats_parsed.minValues.struct.id" -> LongType, "stats_parsed.maxValues.struct.id" -> LongType, "stats_parsed.nullCount.struct.id" -> LongType, "stats_parsed.minValues.struct.part" -> IntegerType, "stats_parsed.maxValues.struct.part" -> IntegerType, "stats_parsed.nullCount.struct.part" -> LongType, "stats_parsed.minValues.struct.`with.dot`" -> LongType, "stats_parsed.maxValues.struct.`with.dot`" -> LongType, "stats_parsed.nullCount.struct.`with.dot`" -> LongType, "stats_parsed.nullCount.bool" -> LongType) val unexpectedCols = Seq( "stats_parsed.minValues.bool", "stats_parsed.maxValues.bool" ) // partitioned by "part" checkpointSchemaForTable(df, "part")( checkpointingFns = checkpointFnsWithStructAndJsonStats, expectedCols = structColumns, additionalValidationFn = addColumns => { checkFields(addColumns, statsAsJsonExists = true, unexpectedCols) } ) checkpointSchemaForTable(df, "part")( checkpointingFns = checkpointFnsWithStructWithoutJsonStats, expectedCols = structColumns, additionalValidationFn = addColumns => { checkFields(addColumns, statsAsJsonExists = false, unexpectedCols) } ) checkpointSchemaForTable(df, "part")( checkpointingFns = checkpointFnsWithoutStructWithJsonStats, expectedCols = Nil, additionalValidationFn = addColumns => { checkFields( addColumns, statsAsJsonExists = true, unexpectedCols ++ structColumns.map(_._1)) } ) checkpointSchemaForTable(df, "part")( checkpointingFns = Seq(checkpointWithoutStats), expectedCols = Nil, additionalValidationFn = addColumns => { checkFields( addColumns, statsAsJsonExists = false, unexpectedCols ++ structColumns.map(_._1)) } ) } test("special characters - check stats") { val weirdName1 = "part%!@#_$%^&*-" val weirdName2 = "part?_.+<>|/" val df = spark.range(10) .withColumn(weirdName1, ('id / 2).cast("int")) .withColumn(weirdName2, ('id / 3).cast("int")) .withColumn("struct", struct($"id", col(weirdName1), $"id".as(weirdName2))) val structColumns = Seq( "stats_parsed.numRecords" -> LongType, "stats_parsed.minValues.id" -> LongType, "stats_parsed.maxValues.id" -> LongType, "stats_parsed.nullCount.id" -> LongType, "stats_parsed.minValues.struct.id" -> LongType, "stats_parsed.maxValues.struct.id" -> LongType, "stats_parsed.nullCount.struct.id" -> LongType, s"stats_parsed.minValues.struct.$weirdName1" -> IntegerType, s"stats_parsed.maxValues.struct.$weirdName1" -> IntegerType, s"stats_parsed.nullCount.struct.$weirdName1" -> LongType, s"stats_parsed.minValues.struct.`$weirdName2`" -> LongType, s"stats_parsed.maxValues.struct.`$weirdName2`" -> LongType, s"stats_parsed.nullCount.struct.`$weirdName2`" -> LongType, s"partitionValues_parsed.$weirdName1" -> IntegerType, s"partitionValues_parsed.`$weirdName2`" -> IntegerType) // partitioned by weirdName1, weirdName2 checkpointSchemaForTable(df, weirdName1, weirdName2)( checkpointingFns = checkpointFnsWithStructAndJsonStats, expectedCols = structColumns, additionalValidationFn = addColumns => { checkFields(addColumns, statsAsJsonExists = true, Nil) } ) checkpointSchemaForTable(df, weirdName1, weirdName2)( checkpointingFns = checkpointFnsWithStructWithoutJsonStats, expectedCols = structColumns, additionalValidationFn = addColumns => { checkFields(addColumns, statsAsJsonExists = false, Nil) } ) checkpointSchemaForTable(df, weirdName1, weirdName2)( checkpointingFns = checkpointFnsWithoutStructWithJsonStats, expectedCols = Nil, additionalValidationFn = addColumns => { checkFields( addColumns, statsAsJsonExists = true, structColumns.map(_._1)) } ) checkpointSchemaForTable(df, weirdName1, weirdName2)( checkpointingFns = Seq(checkpointWithoutStats), expectedCols = Nil, additionalValidationFn = addColumns => { checkFields( addColumns, statsAsJsonExists = false, structColumns.map(_._1)) } ) } test("no data column stats collected + unpartitioned") { val df = spark.range(10).withColumn("part", ('id / 2).cast("int")) .withColumn("struct", struct($"id", $"part", $"id".as("with.dot"))) .withColumn("bool", lit(true)) val expectedColumns = Seq("stats_parsed.numRecords" -> LongType) val unexpected = Seq( "stats_parsed.minValues", "stats_parsed.maxValues", "stats_parsed.nullCount", "stats_parsed.minValues.id", "stats_parsed.maxValues.id", "stats_parsed.nullCount.id", "stats_parsed.minValues.struct.id", "stats_parsed.maxValues.struct.id", "stats_parsed.nullCount.struct.id", "stats_parsed.minValues.struct.part", "stats_parsed.maxValues.struct.part", "stats_parsed.nullCount.struct.part", "stats_parsed.minValues.struct.`with.dot`", "stats_parsed.maxValues.struct.`with.dot`", "stats_parsed.nullCount.struct.`with.dot`", "stats_parsed.nullCount.bool", "stats_parsed.minValues.bool", "stats_parsed.maxValues.bool", "partitionValues_parsed") withSQLConf(s"${DeltaConfigs.sqlConfPrefix}dataSkippingNumIndexedCols" -> "0") { checkpointSchemaForTable(df)( checkpointingFns = checkpointFnsWithStructAndJsonStats, expectedCols = expectedColumns, additionalValidationFn = addColumns => { // None of the stats column should exist instead of numRecords checkFields(addColumns, statsAsJsonExists = true, unexpected) } ) checkpointSchemaForTable(df)( checkpointingFns = checkpointFnsWithStructWithoutJsonStats, expectedCols = expectedColumns, additionalValidationFn = addColumns => { // None of the stats column should exist instead of numRecords checkFields(addColumns, statsAsJsonExists = false, unexpected) } ) checkpointSchemaForTable(df)( checkpointingFns = checkpointFnsWithoutStructWithJsonStats, expectedCols = Nil, additionalValidationFn = addColumns => { checkFields( addColumns, statsAsJsonExists = true, unexpected :+ "stats_parsed.numRecords") } ) checkpointSchemaForTable(df)( checkpointingFns = Seq(checkpointWithoutStats), expectedCols = Nil, additionalValidationFn = addColumns => { checkFields( addColumns, statsAsJsonExists = false, unexpected :+ "stats_parsed.numRecords") } ) } } test("checkpoint read succeeds with column pruning disabled") { withTempDir { dir => // Populate the table with three commits, take a checkpoint at v1, and delete v0. Otherwise, // if the bug we test for caused snapshot construction to fail, Delta would silently retry // without the checkpoint, and the test would always appear to succeed. spark.range(1000).write.format("delta").save(dir.getCanonicalPath) spark.range(1000).write.format("delta").mode("append").save(dir.getCanonicalPath) val deltaLog = DeltaLog.forTable(spark, dir) deltaLog.checkpoint() spark.range(1000).write.format("delta").mode("append").save(dir.getCanonicalPath) val firstCommit = deltaLog.store .listFrom(FileNames.listingPrefix(deltaLog.logPath, 0), deltaLog.newDeltaHadoopConf()) .find(f => FileNames.isDeltaFile(f.getPath) && FileNames.deltaVersion(f.getPath) == 0) assert(new File(firstCommit.get.getPath.toUri).delete()) DeltaLog.clearCache() // Trigger both metadata reconstruction and state reconstruction queries with column pruning // disabled. We must also disable reading metadata from .crc file, for the former case. withSQLConf( SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> org.apache.spark.sql.catalyst.optimizer.ColumnPruning.ruleName, DeltaSQLConf.USE_PROTOCOL_AND_METADATA_FROM_CHECKSUM_ENABLED.key -> "false") { // NOTE: Just creating the snapshot should already trigger metadata reconstruction, but we // still access it directly just to be extra sure. logInfo("About to create a new snapshot") val snapshot = DeltaLog.forTable(spark, dir).snapshot logInfo("About to access metadata") snapshot.metadata logInfo("About to access withStats") snapshot.withStats.count() logInfo("About to trigger state reconstrution") snapshot.stateDF } } } Seq(Seq("part"), Nil).foreach { partitionBy => test("do not lose file stats after a checkpoint when writeStatsAsJson=false - isPartitioned: " + partitionBy.nonEmpty) { withTempDir { dir => var start = 0 def writeNewData(mode: String): Unit = { spark.range(start, start + 10).withColumn("part", 'id % 4) .write .format("delta") .mode(mode) .partitionBy(partitionBy: _*) .save(dir.getCanonicalPath) start += 10 } writeNewData("append") val deltaLog = DeltaLog.forTable(spark, dir) checkpointWithProperty(writeStatsAsJson = Some(false))(deltaLog) def checkpointAndCheck(): Unit = { deltaLog.checkpoint() val checkpoint = spark.read.parquet( FileNames.checkpointFileSingular(deltaLog.logPath, deltaLog.snapshot.version).toString) // use physical name if possible val id = getPhysicalName("id", deltaLog.snapshot.schema) val adds = checkpoint.where("add is not null").selectExpr("add.*") assert(adds.selectExpr(s"stats_parsed.minValues.`$id`") .collect().forall(r => !r.isNullAt(0)), "minValues was null for some values.\n" + adds.collect().mkString("\n")) assert(adds.selectExpr(s"stats_parsed.maxValues.`$id`") .collect().forall(r => !r.isNullAt(0)), "maxValues was null for some values.\n" + adds.collect().mkString("\n")) assert(adds.selectExpr(s"stats_parsed.nullCount.`$id`") .collect().forall(r => !r.isNullAt(0)), "nullCount was null for some values.\n" + adds.collect().mkString("\n")) checkAnswer( adds.select("path"), deltaLog.snapshot.allFiles.select("path") ) } writeNewData("append") checkpointAndCheck() writeNewData("overwrite") checkpointAndCheck() } } } test("switching between v1 and v2 checkpoints") { withTempDir { dir => spark.range(0, 10).withColumn("part", 'id % 4).write.format("delta").partitionBy("part") .save(dir.getCanonicalPath) val deltaLog = DeltaLog.forTable(spark, dir) sql(s"ALTER TABLE delta.`${dir.getCanonicalPath}` SET TBLPROPERTIES " + s"('delta.checkpoint.writeStatsAsStruct'='true')") deltaLog.checkpoint() def getLatestCheckpoint: DataFrame = spark.read.parquet( FileNames.checkpointFileSingular(deltaLog.logPath, deltaLog.snapshot.version).toString) // statsAsStruct=true, statsAsJson=true val withStructAndJson = getLatestCheckpoint sql(s"ALTER TABLE delta.`${dir.getCanonicalPath}` SET TBLPROPERTIES " + s"('delta.checkpoint.writeStatsAsStruct'='false')") deltaLog.checkpoint() // statsAsStruct=false, statsAsJson=true val noStructWithJson = getLatestCheckpoint // The columns should be the same, without the stats_parsed column in noStructWithJson checkAnswer( noStructWithJson.select("add.*") .select("path", "partitionValues", "modificationTime", "tags", "stats"), withStructAndJson.select("add.*") .select("path", "partitionValues", "modificationTime", "tags", "stats") ) sql(s"ALTER TABLE delta.`${dir.getCanonicalPath}` SET TBLPROPERTIES " + s"('delta.checkpoint.writeStatsAsStruct'='true'," + s"'delta.checkpoint.writeStatsAsJson'='false')") deltaLog.checkpoint() // statsAsStruct=true, statsAsJson=false val withStructNoJson = getLatestCheckpoint checkAnswer( withStructNoJson.select("add.*"), withStructAndJson.select("add.*").drop("stats") ) sql(s"ALTER TABLE delta.`${dir.getCanonicalPath}` SET TBLPROPERTIES " + s"('delta.checkpoint.writeStatsAsStruct'='false')") deltaLog.checkpoint() // statsAsStruct=false, statsAsJson=false val noStructNoJson = getLatestCheckpoint // should not have the stats column anymore checkAnswer( noStructNoJson.select("add.*"), noStructWithJson.select("add.*").drop("stats")) // going to a v2 checkpoint with the json stats sql(s"ALTER TABLE delta.`${dir.getCanonicalPath}` SET TBLPROPERTIES " + s"('delta.checkpoint.writeStatsAsStruct'='true'," + s"'delta.checkpoint.writeStatsAsJson'='true')") deltaLog.checkpoint() // statsAsStruct=true, statsAsJson=true val lostAllStats = getLatestCheckpoint // should be identical to withStructAndJson checkAnswer( lostAllStats.select("add.*"), withStructAndJson.select("add.*") .withColumn("stats", lit(null)) .withColumn("stats_parsed", lit(null)) ) } } } class DeltaCheckpointWithStructColsNameColumnMappingSuite extends DeltaCheckpointWithStructColsSuite with DeltaColumnMappingEnableNameMode { override protected def runOnlyTests = Seq( "unpartitioned table", "partitioned table" ) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.io.File import java.nio.file.Files import scala.collection.JavaConverters._ import scala.collection.mutable import org.apache.spark.sql.delta.DeltaOperations.ManualUpdate import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.actions.{Action, AddCDCFile, AddFile, Metadata => MetadataAction, Protocol, SetTransaction} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.schema.SchemaMergingUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.hadoop.fs.Path import org.apache.parquet.format.converter.ParquetMetadataConverter import org.apache.parquet.hadoop.ParquetFileReader import org.scalatest.GivenWhenThen import org.apache.spark.sql.{DataFrame, QueryTest, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ // scalastyle:on import.ordering.noEmptyLine trait DeltaColumnMappingSuiteUtils extends SharedSparkSession with DeltaSQLTestUtils with DeltaSQLCommandTest { protected def supportedModes: Seq[String] = Seq("id", "name") protected def colName(name: String) = s"$name with special chars ,;{}()\n\t=" protected def partitionStmt(partCols: Seq[String]): String = { if (partCols.nonEmpty) s"PARTITIONED BY (${partCols.map(name => s"`$name`").mkString(",")})" else "" } protected def propString(props: Map[String, String]) = if (props.isEmpty) "" else { props .map { case (key, value) => s"'$key' = '$value'" } .mkString("TBLPROPERTIES (", ",", ")") } protected def alterTableWithProps( tableName: String, props: Map[String, String]): Unit = spark.sql( s""" | ALTER TABLE $tableName SET ${propString(props)} |""".stripMargin) protected def mode(props: Map[String, String]): String = props.get(DeltaConfigs.COLUMN_MAPPING_MODE.key).getOrElse("none") protected def testColumnMapping( testName: String, enableSQLConf: Boolean = false, modes: Option[Seq[String]] = None)(testCode: String => Unit): Unit = { test(testName) { modes.getOrElse(supportedModes).foreach { mode => { withClue(s"Testing under mode: $mode") { if (enableSQLConf) { withSQLConf(DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey -> mode) { testCode(mode) } } else { testCode(mode) } } }} } } } class DeltaColumnMappingSuite extends QueryTest with GivenWhenThen with DeltaColumnMappingSuiteUtils { import testImplicits._ protected def withId(id: Long): Metadata = new MetadataBuilder() .putLong(DeltaColumnMapping.COLUMN_MAPPING_METADATA_ID_KEY, id) .build() protected def withPhysicalName(pname: String) = new MetadataBuilder() .putString(DeltaColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, pname) .build() protected def withIdAndPhysicalName(id: Long, pname: String): Metadata = new MetadataBuilder() .putLong(DeltaColumnMapping.COLUMN_MAPPING_METADATA_ID_KEY, id) .putString(DeltaColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, pname) .build() protected def assertEqual( actual: StructType, expected: StructType, ignorePhysicalName: Boolean = true): Unit = { var actualSchema = actual var expectedSchema = expected val fieldsToRemove = mutable.Set[String]() if (ignorePhysicalName) { fieldsToRemove.add(DeltaColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY) } def removeFields(metadata: Metadata): Metadata = { val metadataBuilder = new MetadataBuilder().withMetadata(metadata) fieldsToRemove.foreach { field => { if (metadata.contains(field)) { metadataBuilder.remove(field) } } } metadataBuilder.build() } // drop fields if needed actualSchema = SchemaMergingUtils.transformColumns(actual) { (_, field, _) => field.copy(metadata = removeFields(field.metadata)) } expectedSchema = SchemaMergingUtils.transformColumns(expected) { (_, field, _) => field.copy(metadata = removeFields(field.metadata)) } assert(expectedSchema === actualSchema, s""" |Schema mismatch: | |expected: |${expectedSchema.prettyJson} | |actual: |${actualSchema.prettyJson} |""".stripMargin) } protected def checkSchema( tableName: String, expectedSchema: StructType, ignorePhysicalName: Boolean = true): Unit = { // snapshot schema should have all the expected metadata val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) assertEqual(deltaLog.update().schema, expectedSchema, ignorePhysicalName) // table schema should not have any metadata assert(spark.table(tableName).schema === DeltaColumnMapping.dropColumnMappingMetadata(expectedSchema)) } // NOTE: // All attached metadata to the following sample inputs, if used in source dataframe, // will be CLEARED out after metadata is imported into the target table // See ImplicitMetadataOperation.updateMetadata() for how the old metadata is cleared protected val schema = new StructType() .add("a", StringType, true) .add("b", IntegerType, true) protected val schemaNested = new StructType() .add("a", StringType, true) .add("b", new StructType() .add("c", StringType, true) .add("d", IntegerType, true), true ) protected val schemaWithId = new StructType() .add("a", StringType, true, withId(1)) .add("b", IntegerType, true, withId(2)) protected val schemaWithIdRandom = new StructType() .add("a", StringType, true, withId(111)) .add("b", IntegerType, true, withId(222)) protected val schemaWithIdAndPhysicalNameRandom = new StructType() .add("a", StringType, true, withIdAndPhysicalName(111, "asjdklsajdkl")) .add("b", IntegerType, true, withIdAndPhysicalName(222, "iotiyoiopio")) protected val schemaWithDuplicatingIds = new StructType() .add("a", StringType, true, withId(1)) .add("b", IntegerType, true, withId(2)) .add("c", IntegerType, true, withId(2)) protected val schemaWithIdAndDuplicatingPhysicalNames = new StructType() .add("a", StringType, true, withIdAndPhysicalName(1, "aaa")) .add("b", IntegerType, true, withIdAndPhysicalName(2, "bbb")) .add("c", IntegerType, true, withIdAndPhysicalName(3, "bbb")) protected val schemaWithDuplicatingPhysicalNames = new StructType() .add("a", StringType, true, withPhysicalName("aaa")) .add("b", IntegerType, true, withPhysicalName("bbb")) .add("c", IntegerType, true, withPhysicalName("bbb")) protected val schemaWithDuplicatingPhysicalNamesNested = new StructType() .add("b", new StructType() .add("c", StringType, true, withPhysicalName("dupName")) .add("d", IntegerType, true, withPhysicalName("dupName")), true, withPhysicalName("b") ) protected val schemaWithIdNested = new StructType() .add("a", StringType, true, withId(1)) .add("b", new StructType() .add("c", StringType, true, withId(3)) .add("d", IntegerType, true, withId(4)), true, withId(2) ) protected val schemaWithPhysicalNamesNested = new StructType() .add("a", StringType, true, withIdAndPhysicalName(1, "aaa")) .add("b", // let's call this nested struct 'X'. new StructType() .add("c", StringType, true, withIdAndPhysicalName(2, "ccc")) .add("d", IntegerType, true, withIdAndPhysicalName(3, "ddd")) .add("foo.bar", new StructType().add("f", LongType, true, withIdAndPhysicalName(4, "fff")), true, withIdAndPhysicalName(5, "foo.foo.foo.bar.bar.bar")), true, withIdAndPhysicalName(6, "bbb") ) .add("g", // nested struct 'X' (see above) is repeated here. new StructType() .add("c", StringType, true, withIdAndPhysicalName(7, "ccc")) .add("d", IntegerType, true, withIdAndPhysicalName(8, "ddd")) .add("foo.bar", new StructType().add("f", LongType, true, withIdAndPhysicalName(9, "fff")), true, withIdAndPhysicalName(10, "foo.foo.foo.bar.bar.bar")), true, withIdAndPhysicalName(11, "ggg") ) .add("h", IntegerType, true, withIdAndPhysicalName(12, "hhh")) protected val schemaWithIdNestedRandom = new StructType() .add("a", StringType, true, withId(111)) .add("b", new StructType() .add("c", StringType, true, withId(333)) .add("d", IntegerType, true, withId(444)), true, withId(222) ) // This schema has both a.b and a . b as physical path for its columns, we would like to make sure // it shouldn't trigger the duplicated physical name check protected val schemaWithDottedColumnNames = new StructType() .add("a.b", StringType, true, withIdAndPhysicalName(1, "a.b")) .add("a", new StructType() .add("b", StringType, true, withIdAndPhysicalName(3, "b")), true, withIdAndPhysicalName(2, "a")) protected def dfWithoutIds(spark: SparkSession) = spark.createDataFrame(Seq(Row("str1", 1), Row("str2", 2)).asJava, schema) protected def dfWithoutIdsNested(spark: SparkSession) = spark.createDataFrame( Seq(Row("str1", Row("str1.1", 1)), Row("str2", Row("str1.2", 2))).asJava, schemaNested) protected def dfWithIds(spark: SparkSession, randomIds: Boolean = false) = spark.createDataFrame(Seq(Row("str1", 1), Row("str2", 2)).asJava, if (randomIds) schemaWithIdRandom else schemaWithId) protected def dfWithIdsNested(spark: SparkSession, randomIds: Boolean = false) = spark.createDataFrame( Seq(Row("str1", Row("str1.1", 1)), Row("str2", Row("str1.2", 2))).asJava, if (randomIds) schemaWithIdNestedRandom else schemaWithIdNested) protected def checkProperties( tableName: String, mode: Option[String] = None, readerVersion: Int = 1, writerVersion: Int = 2, curMaxId: Long = 0): Unit = { val props = spark.sql(s"SHOW TBLPROPERTIES $tableName").as[(String, String)].collect().toMap assert(props.get("delta.minReaderVersion").map(_.toInt) == Some(readerVersion)) assert(props.get("delta.minWriterVersion").map(_.toInt) == Some(writerVersion)) assert(props.get(DeltaConfigs.COLUMN_MAPPING_MODE.key) == mode) assert(props.get(DeltaConfigs.COLUMN_MAPPING_MAX_ID.key).map(_.toLong).getOrElse(0) == curMaxId) } protected def createTableWithDeltaTableAPI( tableName: String, props: Map[String, String] = Map.empty, withColumnIds: Boolean = false, isPartitioned: Boolean = false): Unit = { val schemaToUse = if (withColumnIds) schemaWithId else schema val builder = io.delta.tables.DeltaTable.createOrReplace(spark) .tableName(tableName) .addColumn(schemaToUse.fields(0)) .addColumn(schemaToUse.fields(1)) props.foreach { case (key, value) => builder.property(key, value) } if (isPartitioned) { builder.partitionedBy("a") } builder.execute() } protected def createTableWithSQLCreateOrReplaceAPI( tableName: String, props: Map[String, String] = Map.empty, withColumnIds: Boolean = false, isPartitioned: Boolean = false, nested: Boolean = false, randomIds: Boolean = false): Unit = { withTable("source") { val dfToWrite = if (withColumnIds) { if (nested) { dfWithIdsNested(spark, randomIds) } else { dfWithIds(spark, randomIds) } } else { if (nested) { dfWithoutIdsNested(spark) } else { dfWithoutIds(spark) } } dfToWrite.write.saveAsTable("source") val partitionStmt = if (isPartitioned) "PARTITIONED BY (a)" else "" spark.sql( s""" |CREATE OR REPLACE TABLE $tableName |USING DELTA |$partitionStmt |${propString(props)} |AS SELECT * FROM source |""".stripMargin) } } protected def createTableWithSQLAPI( tableName: String, props: Map[String, String] = Map.empty, withColumnIds: Boolean = false, isPartitioned: Boolean = false, nested: Boolean = false, randomIds: Boolean = false): Unit = { withTable("source") { val dfToWrite = if (withColumnIds) { if (nested) { dfWithIdsNested(spark, randomIds) } else { dfWithIds(spark, randomIds) } } else { if (nested) { dfWithoutIdsNested(spark) } else { dfWithoutIds(spark) } } dfToWrite.write.saveAsTable("source") val partitionStmt = if (isPartitioned) "PARTITIONED BY (a)" else "" spark.sql( s""" |CREATE TABLE $tableName |USING DELTA |$partitionStmt |${propString(props)} |AS SELECT * FROM source |""".stripMargin) } } protected def createTableWithDataFrameAPI( tableName: String, props: Map[String, String] = Map.empty, withColumnIds: Boolean = false, isPartitioned: Boolean = false, nested: Boolean = false, randomIds: Boolean = false): Unit = { val sqlConfs = props.map { case (key, value) => "spark.databricks.delta.properties.defaults." + key.stripPrefix("delta.") -> value } withSQLConf(sqlConfs.toList: _*) { val dfToWrite = if (withColumnIds) { if (nested) { dfWithIdsNested(spark, randomIds) } else { dfWithIds(spark, randomIds) } } else { if (nested) { dfWithoutIdsNested(spark) } else { dfWithoutIds(spark) } } if (isPartitioned) { dfToWrite.write.format("delta").partitionBy("a").saveAsTable(tableName) } else { dfToWrite.write.format("delta").saveAsTable(tableName) } } } protected def createTableWithDataFrameWriterV2API( tableName: String, props: Map[String, String] = Map.empty, withColumnIds: Boolean = false, isPartitioned: Boolean = false, nested: Boolean = false, randomIds: Boolean = false): Unit = { val dfToWrite = if (withColumnIds) { if (nested) { dfWithIdsNested(spark, randomIds) } else { dfWithIds(spark, randomIds) } } else { if (nested) { dfWithoutIdsNested(spark) } else { dfWithoutIds(spark) } } val writer = dfToWrite.writeTo(tableName).using("delta") props.foreach(prop => writer.tableProperty(prop._1, prop._2)) if (isPartitioned) writer.partitionedBy('a) writer.create() } protected def createStrictSchemaTableWithDeltaTableApi( tableName: String, schema: StructType, props: Map[String, String] = Map.empty, isPartitioned: Boolean = false): Unit = { val builder = io.delta.tables.DeltaTable.createOrReplace(spark) .tableName(tableName) builder.addColumns(schema) props.foreach(prop => builder.property(prop._1, prop._2)) if (isPartitioned) builder.partitionedBy("a") builder.execute() } protected def testCreateTableColumnMappingMode( tableName: String, expectedSchema: StructType, ignorePhysicalName: Boolean, mode: String, createNewTable: Boolean = true, tableFeaturesProtocolExpected: Boolean = true)(fn: => Unit): Unit = { withTable(tableName) { fn checkProperties(tableName, readerVersion = 2, writerVersion = if (tableFeaturesProtocolExpected) 7 else 5, mode = Some(mode), curMaxId = DeltaColumnMapping.findMaxColumnId(expectedSchema) ) checkSchema(tableName, expectedSchema, ignorePhysicalName) } } test("find max column id in existing columns") { assert(DeltaColumnMapping.findMaxColumnId(schemaWithId) == 2) assert(DeltaColumnMapping.findMaxColumnId(schemaWithIdNested) == 4) assert(DeltaColumnMapping.findMaxColumnId(schemaWithIdRandom) == 222) assert(DeltaColumnMapping.findMaxColumnId(schemaWithIdNestedRandom) == 444) assert(DeltaColumnMapping.findMaxColumnId(schema) == 0) assert(DeltaColumnMapping.findMaxColumnId(new StructType()) == 0) } test("Enable column mapping with schema change on table with no schema") { withTempDir { dir => val tablePath = dir.getCanonicalPath Seq((1, "a"), (2, "b")).toDF("id", "name") .write.mode("append").format("delta").save(tablePath) val deltaLog = DeltaLog.forTable(spark, tablePath) val txn = deltaLog.startTransaction() txn.commitManually(actions.Metadata()) // Whip the schema out val txn2 = deltaLog.startTransaction() txn2.commitManually(Protocol(2, 5)) txn2.updateMetadata(actions.Metadata( configuration = Map("delta.columnMapping.mode" -> "name"), schemaString = new StructType().add("a", StringType).json)) // Now ensure that it is not allowed to enable column mapping with schema change // on a table with a schema Seq((1, "a"), (2, "b")).toDF("id", "name") .write.mode("overwrite").format("delta") .option("overwriteSchema", "true") .save(tablePath) val txn3 = deltaLog.startTransaction() txn3.commitManually(Protocol(2, 5)) val e = intercept[DeltaColumnMappingUnsupportedException] { txn3.updateMetadata( actions.Metadata( configuration = Map("delta.columnMapping.mode" -> "name"), schemaString = new StructType().add("a", StringType).json)) } val msg = "Schema changes are not allowed during the change of column mapping mode." assert(e.getMessage.contains(msg)) } } // TODO: repurpose this once we roll out the proper semantics for CM + streaming testColumnMapping("isColumnMappingReadCompatible") { mode => // Set up table based on mode and return the initial metadata actions for comparison def setupInitialTable(deltaLog: DeltaLog): (MetadataAction, MetadataAction) = { val tablePath = deltaLog.dataPath.toString if (mode == NameMapping.name) { Seq((1, "a"), (2, "b")).toDF("id", "name") .write.mode("append").format("delta").save(tablePath) // schema: val m0 = deltaLog.update().metadata // add a column sql(s"ALTER TABLE delta.`$tablePath` ADD COLUMN (score long)") // schema: val m1 = deltaLog.update().metadata // column mapping not enabled -> not blocked at all assert(DeltaColumnMapping.hasNoColumnMappingSchemaChanges(m1, m0)) // upgrade to name mode alterTableWithProps(s"delta.`$tablePath`", Map( DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name", DeltaConfigs.MIN_READER_VERSION.key -> "2", DeltaConfigs.MIN_WRITER_VERSION.key -> "5")) (m0, m1) } else { // for id mode, just create the table withSQLConf(DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey -> "id") { Seq((1, "a"), (2, "b")).toDF("id", "name") .write.mode("append").format("delta").save(tablePath) } // schema: val m0 = deltaLog.update().metadata // add a column sql(s"ALTER TABLE delta.`$tablePath` ADD COLUMN (score long)") // schema: val m1 = deltaLog.update().metadata // add column shouldn't block assert(DeltaColumnMapping.hasNoColumnMappingSchemaChanges(m1, m0)) (m0, m1) } } withTempDir { dir => val tablePath = dir.getCanonicalPath val deltaLog = DeltaLog.forTable(spark, tablePath) val (m0, m1) = setupInitialTable(deltaLog) // schema: val m2 = deltaLog.update().metadata assert(DeltaColumnMapping.hasNoColumnMappingSchemaChanges(m2, m1)) assert(DeltaColumnMapping.hasNoColumnMappingSchemaChanges(m2, m0)) // rename column sql(s"ALTER TABLE delta.`$tablePath` RENAME COLUMN score TO age") // schema: val m3 = deltaLog.update().metadata assert(!DeltaColumnMapping.hasNoColumnMappingSchemaChanges(m3, m2)) assert(!DeltaColumnMapping.hasNoColumnMappingSchemaChanges(m3, m1)) // But IS read compatible with the initial schema, because the added column should not // be blocked by this column mapping check. assert(DeltaColumnMapping.hasNoColumnMappingSchemaChanges(m3, m0)) // drop a column sql(s"ALTER TABLE delta.`$tablePath` DROP COLUMN age") // schema: val m4 = deltaLog.update().metadata assert(!DeltaColumnMapping.hasNoColumnMappingSchemaChanges(m4, m3)) assert(!DeltaColumnMapping.hasNoColumnMappingSchemaChanges(m4, m2)) assert(!DeltaColumnMapping.hasNoColumnMappingSchemaChanges(m4, m1)) // but IS read compatible with the initial schema, because the added column is dropped assert(DeltaColumnMapping.hasNoColumnMappingSchemaChanges(m4, m0)) // add back the same column sql(s"ALTER TABLE delta.`$tablePath` ADD COLUMN (score long)") // schema: val m5 = deltaLog.update().metadata // It IS read compatible with the previous schema, because the added column should not // blocked by this column mapping check. assert(DeltaColumnMapping.hasNoColumnMappingSchemaChanges(m5, m4)) assert(!DeltaColumnMapping.hasNoColumnMappingSchemaChanges(m5, m3)) assert(!DeltaColumnMapping.hasNoColumnMappingSchemaChanges(m5, m2)) // But Since the new added column has a different physical name as all previous columns, // even it has the same logical name as say, m1.schema, we will still block assert(!DeltaColumnMapping.hasNoColumnMappingSchemaChanges(m5, m1)) // But it IS read compatible with the initial schema, because the added column should not // be blocked by this column mapping check. assert(DeltaColumnMapping.hasNoColumnMappingSchemaChanges(m5, m0)) } } testColumnMapping("create table through raw schema API should " + "auto bump the version and retain input metadata") { mode => // provides id only (let Delta generate physical name for me) testCreateTableColumnMappingMode( "t1", schemaWithIdRandom, ignorePhysicalName = true, mode = mode) { createStrictSchemaTableWithDeltaTableApi( "t1", schemaWithIdRandom, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode)) } // provides id and physical name (Delta shouldn't rebuild/override) // we use random ids as input, which shouldn't be changed too testCreateTableColumnMappingMode( "t1", schemaWithIdAndPhysicalNameRandom, ignorePhysicalName = false, mode = mode) { createStrictSchemaTableWithDeltaTableApi( "t1", schemaWithIdAndPhysicalNameRandom, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode)) } } testColumnMapping("create table through dataframe should " + "auto bumps the version and rebuild schema metadata/drop dataframe metadata") { mode => // existing ids should be dropped/ignored and ids should be regenerated // so for tests below even if we are ingesting dfs with random ids // we should still expect schema with normal sequential ids val expectedSchema = schemaWithId testCreateTableColumnMappingMode( "t1", expectedSchema, ignorePhysicalName = true, mode = mode) { createTableWithSQLAPI( "t1", Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode), withColumnIds = true, randomIds = true) } testCreateTableColumnMappingMode( "t1", expectedSchema, ignorePhysicalName = true, mode = mode) { createTableWithDataFrameAPI( "t1", Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode), withColumnIds = true, randomIds = true) } testCreateTableColumnMappingMode( "t1", expectedSchema, ignorePhysicalName = true, mode = mode) { createTableWithSQLCreateOrReplaceAPI( "t1", Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode), withColumnIds = true, randomIds = true) } testCreateTableColumnMappingMode( "t1", expectedSchema, ignorePhysicalName = true, mode = mode) { createTableWithDataFrameWriterV2API( "t1", Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode), withColumnIds = true, randomIds = true) } } test("create table with none mode") { withTable("t1") { // column ids will be dropped, having the options here to make sure such happens createTableWithSQLAPI( "t1", Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> "none"), withColumnIds = true, randomIds = true) // Should be still on old protocol, the schema shouldn't have any metadata checkProperties( "t1", mode = Some("none")) checkSchema("t1", schema, ignorePhysicalName = false) } } testColumnMapping("update column mapped table invalid max id property is blocked") { mode => withTable("t1") { createTableWithSQLAPI( "t1", Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode), withColumnIds = true ) val log = DeltaLog.forTable(spark, TableIdentifier("t1")) // Get rid of max column id prop assert { intercept[DeltaAnalysisException] { log.withNewTransaction { txn => val existingMetadata = log.update().metadata txn.commit(existingMetadata.copy(configuration = existingMetadata.configuration - DeltaConfigs.COLUMN_MAPPING_MAX_ID.key) :: Nil, DeltaOperations.ManualUpdate) } }.getErrorClass == "DELTA_COLUMN_MAPPING_MAX_COLUMN_ID_NOT_SET" } // Use an invalid max column id prop assert { intercept[DeltaAnalysisException] { log.withNewTransaction { txn => val existingMetadata = log.update().metadata txn.commit(existingMetadata.copy(configuration = existingMetadata.configuration ++ Map( // '1' is less than the current max DeltaConfigs.COLUMN_MAPPING_MAX_ID.key -> "1" )) :: Nil, DeltaOperations.ManualUpdate) } }.getErrorClass == "DELTA_COLUMN_MAPPING_MAX_COLUMN_ID_NOT_SET_CORRECTLY" } } } testColumnMapping( "create column mapped table with duplicated id/physical name should error" ) { mode => withTable("t1") { val e = intercept[ColumnMappingException] { createStrictSchemaTableWithDeltaTableApi( "t1", schemaWithDuplicatingIds, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode)) } assert( e.getMessage.contains( s"Found duplicated column id `2` in column mapping mode `$mode`")) assert(e.getMessage.contains(DeltaColumnMapping.COLUMN_MAPPING_METADATA_ID_KEY)) val e2 = intercept[ColumnMappingException] { createStrictSchemaTableWithDeltaTableApi( "t1", schemaWithIdAndDuplicatingPhysicalNames, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode)) } assert( e2.getMessage.contains( s"Found duplicated physical name `bbb` in column mapping mode `$mode`")) assert(e2.getMessage.contains(DeltaColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY)) } // for name mode specific, we would also like to check for name duplication if (mode == "name") { val e = intercept[ColumnMappingException] { createStrictSchemaTableWithDeltaTableApi( "t1", schemaWithDuplicatingPhysicalNames, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode)) } assert( e.getMessage.contains( s"Found duplicated physical name `bbb` in column mapping mode `$mode`") ) val e2 = intercept[ColumnMappingException] { createStrictSchemaTableWithDeltaTableApi( "t1", schemaWithDuplicatingPhysicalNamesNested, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode)) } assert( e2.getMessage.contains( s"Found duplicated physical name `b.dupName` in column mapping mode `$mode`") ) } } testColumnMapping( "create table in column mapping mode without defining ids explicitly" ) { mode => withTable("t1") { // column ids will be dropped, having the options here to make sure such happens createTableWithSQLAPI( "t1", Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode), withColumnIds = true, randomIds = true) checkSchema("t1", schemaWithId) checkProperties("t1", readerVersion = 2, writerVersion = 7, mode = Some(mode), curMaxId = DeltaColumnMapping.findMaxColumnId(schemaWithId) ) } } testColumnMapping("alter column order in schema on new protocol") { mode => withTable("t1") { // column ids will be dropped, having the options here to make sure such happens createTableWithSQLAPI("t1", Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode), withColumnIds = true, nested = true, randomIds = true) spark.sql( """ |ALTER TABLE t1 ALTER COLUMN a AFTER b |""".stripMargin ) checkProperties("t1", readerVersion = 2, writerVersion = 7, mode = Some(mode), curMaxId = DeltaColumnMapping.findMaxColumnId(schemaWithIdNested)) checkSchema( "t1", schemaWithIdNested.copy(fields = schemaWithIdNested.fields.reverse)) } } testColumnMapping("add column in schema on new protocol") { mode => def check(expectedSchema: StructType): Unit = { val curMaxId = DeltaColumnMapping.findMaxColumnId(expectedSchema) + 1 checkSchema("t1", expectedSchema) spark.sql( """ |ALTER TABLE t1 ADD COLUMNS (c STRING AFTER b) |""".stripMargin ) checkProperties("t1", readerVersion = 2, writerVersion = 7, mode = Some(mode), curMaxId = curMaxId) checkSchema("t1", expectedSchema.add("c", StringType, true, withId(curMaxId))) val curMaxId2 = DeltaColumnMapping.findMaxColumnId(expectedSchema) + 2 spark.sql( """ |ALTER TABLE t1 ADD COLUMNS (d STRING AFTER c) |""".stripMargin ) checkProperties("t1", readerVersion = 2, writerVersion = 7, mode = Some(mode), curMaxId = curMaxId2) checkSchema("t1", expectedSchema .add("c", StringType, true, withId(curMaxId)) .add("d", StringType, true, withId(curMaxId2))) } withTable("t1") { // column ids will be dropped, having the options here to make sure such happens createTableWithSQLAPI( "t1", Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode), withColumnIds = true, randomIds = true) check(schemaWithId) } withTable("t1") { // column ids will NOT be dropped, so future ids should update based on the current max createStrictSchemaTableWithDeltaTableApi( "t1", schemaWithIdRandom, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode) ) check(schemaWithIdRandom) } } testColumnMapping("add nested column in schema on new protocol") { mode => withTable("t1") { // column ids will be dropped, having the options here to make sure such happens createTableWithSQLAPI( "t1", Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode), withColumnIds = true, nested = true, randomIds = true) checkSchema("t1", schemaWithIdNested) val curMaxId = DeltaColumnMapping.findMaxColumnId(schemaWithIdNested) + 1 spark.sql( """ |ALTER TABLE t1 ADD COLUMNS (b.e STRING AFTER d) |""".stripMargin ) checkProperties("t1", readerVersion = 2, writerVersion = 7, mode = Some(mode), curMaxId = curMaxId) checkSchema("t1", schemaWithIdNested.merge( new StructType().add( "b", new StructType().add( "e", StringType, true, withId(5)), true, withId(2) )) ) val curMaxId2 = DeltaColumnMapping.findMaxColumnId(schemaWithIdNested) + 2 spark.sql( """ |ALTER TABLE t1 ADD COLUMNS (b.f STRING AFTER e) |""".stripMargin ) checkProperties("t1", readerVersion = 2, writerVersion = 7, mode = Some(mode), curMaxId = curMaxId2) checkSchema("t1", schemaWithIdNested.merge( new StructType().add( "b", new StructType().add( "e", StringType, true, withId(5)), true, withId(2) )).merge( new StructType().add( "b", new StructType() .add("f", StringType, true, withId(6)), true, withId(2)) )) } } testColumnMapping("write/merge df to table") { mode => withTable("t1") { // column ids will be dropped, having the options here to make sure such happens createTableWithDataFrameAPI("t1", Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode), withColumnIds = true, randomIds = true) val curMaxId = DeltaColumnMapping.findMaxColumnId(schemaWithId) val df1 = dfWithIds(spark) df1.write .format("delta") .mode("append") .saveAsTable("t1") checkProperties("t1", readerVersion = 2, writerVersion = 7, mode = Some(mode), curMaxId = curMaxId) checkSchema("t1", schemaWithId) val previousSchema = spark.table("t1").schema // ingest df with random id should not cause existing schema col id to change val df2 = dfWithIds(spark, randomIds = true) df2.write .format("delta") .mode("append") .saveAsTable("t1") checkProperties("t1", readerVersion = 2, writerVersion = 7, mode = Some(mode), curMaxId = curMaxId) // with checkPhysicalSchema check checkSchema("t1", schemaWithId) // compare with before assertEqual(spark.table("t1").schema, previousSchema, ignorePhysicalName = false) val df3 = spark.createDataFrame( Seq(Row("str3", 3, "str3.1"), Row("str4", 4, "str4.1")).asJava, schemaWithId.add("c", StringType, true, withId(3)) ) df3.write .option("mergeSchema", "true") .format("delta") .mode("append") .saveAsTable("t1") val curMaxId2 = DeltaColumnMapping.findMaxColumnId(schemaWithId) + 1 checkProperties("t1", readerVersion = 2, writerVersion = 7, mode = Some(mode), curMaxId = curMaxId2) checkSchema("t1", schemaWithId.add("c", StringType, true, withId(3))) } } testColumnMapping(s"try modifying restricted max id property should fail") { mode => withTable("t1") { val e = intercept[UnsupportedOperationException] { createTableWithSQLAPI( "t1", Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode, DeltaConfigs.COLUMN_MAPPING_MAX_ID.key -> "100"), withColumnIds = true, nested = true) } assert(e.getMessage.contains(s"The Delta table configuration " + s"${DeltaConfigs.COLUMN_MAPPING_MAX_ID.key} cannot be specified by the user")) } withTable("t1") { createTableWithSQLAPI( "t1", Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode), withColumnIds = true, nested = true) val e2 = intercept[UnsupportedOperationException] { alterTableWithProps("t1", Map(DeltaConfigs.COLUMN_MAPPING_MAX_ID.key -> "100")) } assert(e2.getMessage.contains(s"The Delta table configuration " + s"${DeltaConfigs.COLUMN_MAPPING_MAX_ID.key} cannot be specified by the user")) } withTable("t1") { val e = intercept[UnsupportedOperationException] { createTableWithDataFrameAPI( "t1", Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode, DeltaConfigs.COLUMN_MAPPING_MAX_ID.key -> "100"), withColumnIds = true, nested = true) } assert(e.getMessage.contains(s"The Delta table configuration " + s"${DeltaConfigs.COLUMN_MAPPING_MAX_ID.key} cannot be specified by the user")) } } testColumnMapping("physical data and partition schema") { mode => withTable("t1") { // column ids will be dropped, having the options here to make sure such happens createTableWithSQLAPI("t1", Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode), withColumnIds = true, randomIds = true) val metadata = DeltaLog.forTableWithSnapshot(spark, TableIdentifier("t1"))._2.metadata assertEqual(metadata.schema, schemaWithId) assertEqual(metadata.schema, StructType(metadata.partitionSchema ++ metadata.dataSchema)) } } testColumnMapping("block CONVERT TO DELTA") { mode => withSQLConf(DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey -> mode) { withTempDir { tablePath => val tempDir = tablePath.getCanonicalPath val df1 = Seq(0).toDF("id") .withColumn("key1", lit("A1")) .withColumn("key2", lit("A2")) df1.write .partitionBy(Seq("key1"): _*) .format("parquet") .mode("overwrite") .save(tempDir) val e = intercept[UnsupportedOperationException] { sql(s"convert to delta parquet.`$tempDir` partitioned by (key1 String)") } assert(e.getMessage.contains(s"cannot be set to `$mode` when using CONVERT TO DELTA")) } } } testColumnMapping( "column mapping batch scan should detect physical name changes", enableSQLConf = true ) { _ => withTempDir { dir => spark.range(10).toDF("id") .write.format("delta").save(dir.getCanonicalPath) // Analysis phase val df1 = spark.read.format("delta").load(dir.getCanonicalPath) val df2 = spark.read.format("delta").load(dir.getCanonicalPath) // Overwrite schema but with same logical schema withSQLConf(DeltaSQLConf.REUSE_COLUMN_MAPPING_METADATA_DURING_OVERWRITE.key -> "false") { spark.range(10).toDF("id") .write.format("delta").option("overwriteSchema", "true").mode("overwrite") .save(dir.getCanonicalPath) } // The previous analyzed DF no longer is able to read the data any more because it generates // new physical name for the underlying columns, so we should fail. assert { intercept[DeltaAnalysisException] { df1.collect() }.getErrorClass == "DELTA_SCHEMA_CHANGE_SINCE_ANALYSIS" } // See we can't read back the same data any more // Note: We need to use separate dataframe, because the error in df1 will be cached. withSQLConf(DeltaSQLConf.DELTA_SCHEMA_ON_READ_CHECK_ENABLED.key -> "false") { checkAnswer( df2, (0 until 10).map(_ => Row(null)) ) } } } protected def testPartitionPath(tableName: String)(createFunc: Boolean => Unit): Unit = { withTable(tableName) { Seq(true, false).foreach { isPartitioned => spark.sql(s"drop table if exists $tableName") createFunc(isPartitioned) val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName)) val prefixLen = DeltaConfigs.RANDOM_PREFIX_LENGTH.fromMetaData(snapshot.metadata) Seq(("str3", 3), ("str4", 4)).toDF(schema.fieldNames: _*) .write.format("delta").mode("append").saveAsTable(tableName) checkAnswer(spark.table(tableName), Row("str1", 1) :: Row("str2", 2) :: Row("str3", 3) :: Row("str4", 4) :: Nil) // both new table writes and appends should use prefix val pattern = s"[A-Za-z0-9]{$prefixLen}/.*part-.*parquet" for (file <- snapshot.allFiles.collect()) { assert(file.path.matches(pattern)) } } } } // Copied verbatim from the "valid replaceWhere" test in DeltaSuite protected def testReplaceWhere(): Unit = Seq(true, false).foreach { enabled => withSQLConf(DeltaSQLConf.REPLACEWHERE_DATACOLUMNS_ENABLED.key -> enabled.toString) { Seq(true, false).foreach { partitioned => // Skip when it's not enabled and not partitioned. if (enabled || partitioned) { withTempDir { dir => val writer = Seq(1, 2, 3, 4).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .withColumn("is_even", $"value" % 2 === 0) .write .format("delta") if (partitioned) { writer.partitionBy("is_odd").save(dir.toString) } else { writer.save(dir.toString) } def data: DataFrame = spark.read.format("delta").load(dir.toString) Seq(5, 7).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .withColumn("is_even", $"value" % 2 === 0) .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "is_odd = true") .save(dir.toString) checkAnswer( data, Seq(2, 4, 5, 7).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .withColumn("is_even", $"value" % 2 === 0)) // replaceWhere on non-partitioning columns if enabled. if (enabled) { Seq(6, 8).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .withColumn("is_even", $"value" % 2 === 0) .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "is_even = true") .save(dir.toString) checkAnswer( data, Seq(5, 6, 7, 8).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .withColumn("is_even", $"value" % 2 === 0)) } } } } } } testColumnMapping("valid replaceWhere", enableSQLConf = true) { _ => testReplaceWhere() } protected def verifyUpgradeAndTestSchemaEvolution(tableName: String): Unit = { checkProperties(tableName, readerVersion = 2, writerVersion = 5, mode = Some("name"), curMaxId = 4) checkSchema(tableName, schemaWithIdNested) val expectedSchema = new StructType() .add("a", StringType, true, withIdAndPhysicalName(1, "a")) .add("b", new StructType() .add("c", StringType, true, withIdAndPhysicalName(3, "c")) .add("d", IntegerType, true, withIdAndPhysicalName(4, "d")), true, withIdAndPhysicalName(2, "b")) assertEqual( DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName))._2.schema, expectedSchema, ignorePhysicalName = false) checkAnswer(spark.table(tableName), dfWithoutIdsNested(spark)) // test schema evolution val newNestedData = spark.createDataFrame( Seq(Row("str3", Row("str1.3", 3), "new value")).asJava, schemaNested.add("e", StringType)) newNestedData.write.format("delta") .option("mergeSchema", "true") .mode("append").saveAsTable(tableName) checkAnswer( spark.table(tableName), dfWithoutIdsNested(spark).withColumn("e", lit(null)).union(newNestedData)) val newTableSchema = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName))._2.schema val newPhysicalName = DeltaColumnMapping.getPhysicalName(newTableSchema("e")) // physical name of new column should be GUID, not display name assert(newPhysicalName.startsWith("col-")) assertEqual( newTableSchema, expectedSchema.add("e", StringType, true, withIdAndPhysicalName(5, newPhysicalName)), ignorePhysicalName = false) } test("change mode on new protocol table") { withTable("t1") { createTableWithSQLAPI( "t1", isPartitioned = true, nested = true, props = Map( DeltaConfigs.MIN_READER_VERSION.key -> "2", DeltaConfigs.MIN_WRITER_VERSION.key -> "5")) alterTableWithProps("t1", Map( DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name")) verifyUpgradeAndTestSchemaEvolution("t1") } } test("upgrade first and then change mode") { withTable("t1") { createTableWithSQLAPI("t1", isPartitioned = true, nested = true) alterTableWithProps("t1", Map( DeltaConfigs.MIN_READER_VERSION.key -> "2", DeltaConfigs.MIN_WRITER_VERSION.key -> "5")) alterTableWithProps("t1", Map( DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name")) verifyUpgradeAndTestSchemaEvolution("t1") } } test("upgrade and change mode in one ALTER TABLE cmd") { withTable("t1") { createTableWithSQLAPI("t1", isPartitioned = true, nested = true) alterTableWithProps("t1", Map( DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name", DeltaConfigs.MIN_READER_VERSION.key -> "2", DeltaConfigs.MIN_WRITER_VERSION.key -> "5")) verifyUpgradeAndTestSchemaEvolution("t1") } } test("illegal mode changes") { val oldModes = Seq("none") ++ supportedModes val newModes = Seq("none") ++ supportedModes val upgrade = Seq(true, false) val removalAllowed = Seq(true, false) for(oldMode <- oldModes; newMode <- newModes; ug <- upgrade; ra <- removalAllowed) { val oldProps = Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> oldMode) val newProps = Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> newMode) ++ (if (!ug) Map.empty else Map( DeltaConfigs.MIN_READER_VERSION.key -> "2", DeltaConfigs.MIN_WRITER_VERSION.key -> "5")) val isSupportedChange = { // No change. (oldMode == newMode) || // Downgrade allowed with a flag. (ra && oldMode != NoMapping.name && newMode == NoMapping.name) || // Upgrade always allowed. (oldMode == NoMapping.name && newMode == NameMapping.name) } if (!isSupportedChange) { Given(s"old mode: $oldMode, new mode: $newMode, upgrade: $ug, removalAllowed: $ra") val e = intercept[UnsupportedOperationException] { withTable("t1") { createTableWithSQLAPI("t1", props = oldProps) withSQLConf(DeltaSQLConf.ALLOW_COLUMN_MAPPING_REMOVAL.key -> ra.toString) { alterTableWithProps("t1", props = newProps) } } } assert(e.getMessage.contains("Changing column mapping mode from")) } } } test("getPhysicalNameFieldMap") { // To keep things simple, we use schema `schemaWithPhysicalNamesNested` such that the // physical name is just the logical name repeated three times. val actual = DeltaColumnMapping .getPhysicalNameFieldMap(schemaWithPhysicalNamesNested) .map { case (physicalPath, field) => (physicalPath, field.name) } val expected = Map[Seq[String], String]( Seq("aaa") -> "a", Seq("bbb") -> "b", Seq("bbb", "ccc") -> "c", Seq("bbb", "ddd") -> "d", Seq("bbb", "foo.foo.foo.bar.bar.bar") -> "foo.bar", Seq("bbb", "foo.foo.foo.bar.bar.bar", "fff") -> "f", Seq("ggg") -> "g", Seq("ggg", "ccc") -> "c", Seq("ggg", "ddd") -> "d", Seq("ggg", "foo.foo.foo.bar.bar.bar") -> "foo.bar", Seq("ggg", "foo.foo.foo.bar.bar.bar", "fff") -> "f", Seq("hhh") -> "h" ) assert(expected === actual, s""" |The actual physicalName -> logicalName map |${actual.mkString("\n")} |did not equal the expected map |${expected.mkString("\n")} |""".stripMargin) } testColumnMapping("is drop/rename column operation") { mode => import DeltaColumnMapping.{isDropColumnOperation, isRenameColumnOperation} withTable("t1") { def getMetadata(): MetadataAction = { DeltaLog.forTableWithSnapshot(spark, TableIdentifier("t1"))._2.metadata } createStrictSchemaTableWithDeltaTableApi( "t1", schemaWithPhysicalNamesNested, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode) ) // case 1: currentSchema compared with itself var currentMetadata = getMetadata() var newMetadata = getMetadata() def isBothCMEnabled: Boolean = newMetadata.columnMappingMode != NoMapping && currentMetadata.columnMappingMode != NoMapping assert( !isDropColumnOperation( newMetadata.schema, currentMetadata.schema, isBothCMEnabled) && !isRenameColumnOperation( newMetadata.schema, currentMetadata.schema, isBothCMEnabled) ) // case 2: add a top-level column sql("ALTER TABLE t1 ADD COLUMNS (ping INT)") currentMetadata = newMetadata newMetadata = getMetadata() assert( !isDropColumnOperation( newMetadata.schema, currentMetadata.schema, isBothCMEnabled) && !isRenameColumnOperation( newMetadata.schema, currentMetadata.schema, isBothCMEnabled) ) // case 3: add a nested column sql("ALTER TABLE t1 ADD COLUMNS (b.`foo.bar`.`my.new;col()` LONG)") currentMetadata = newMetadata newMetadata = getMetadata() assert( !isDropColumnOperation( newMetadata.schema, currentMetadata.schema, isBothCMEnabled) && !isRenameColumnOperation( newMetadata.schema, currentMetadata.schema, isBothCMEnabled) ) // case 4: drop a top-level column sql("ALTER TABLE t1 DROP COLUMN (ping)") currentMetadata = newMetadata newMetadata = getMetadata() assert( isDropColumnOperation( newMetadata.schema, currentMetadata.schema, isBothCMEnabled) && !isRenameColumnOperation( newMetadata.schema, currentMetadata.schema, isBothCMEnabled) ) // case 5: drop a nested column sql("ALTER TABLE t1 DROP COLUMN (g.`foo.bar`)") currentMetadata = newMetadata newMetadata = getMetadata() assert( isDropColumnOperation( newMetadata.schema, currentMetadata.schema, isBothCMEnabled) && !isRenameColumnOperation( newMetadata.schema, currentMetadata.schema, isBothCMEnabled) ) // case 6: rename a top-level column sql("ALTER TABLE t1 RENAME COLUMN a TO pong") currentMetadata = newMetadata newMetadata = getMetadata() assert( !isDropColumnOperation( newMetadata.schema, currentMetadata.schema, isBothCMEnabled) && isRenameColumnOperation( newMetadata.schema, currentMetadata.schema, isBothCMEnabled) ) // case 7: rename a nested column sql("ALTER TABLE t1 RENAME COLUMN b.c TO c2") currentMetadata = newMetadata newMetadata = getMetadata() assert( !isDropColumnOperation( newMetadata.schema, currentMetadata.schema, isBothCMEnabled) && isRenameColumnOperation( newMetadata.schema, currentMetadata.schema, isBothCMEnabled) ) } } Seq(true, false).foreach { cdfEnabled => var shouldBlock = cdfEnabled val shouldBlockStr = if (shouldBlock) "should block" else "should not block" def checkHelper( log: DeltaLog, newSchema: StructType, action: Action, shouldFail: Boolean = shouldBlock): Unit = { val txn = log.startTransaction() txn.updateMetadata(txn.metadata.copy(schemaString = newSchema.json)) if (shouldFail) { val e = intercept[DeltaUnsupportedOperationException] { txn.commit(Seq(action), DeltaOperations.ManualUpdate) }.getMessage assert(e == "[DELTA_BLOCK_COLUMN_MAPPING_AND_CDC_OPERATION] " + "Operation \"Manual Update\" is not allowed when the table has enabled " + "change data feed (CDF) and has undergone schema changes using DROP COLUMN or RENAME " + "COLUMN.") } else { txn.commit(Seq(action), DeltaOperations.ManualUpdate) } } val fileActions = Seq( AddFile("foo", Map.empty, 1L, 1L, dataChange = true), AddFile("foo", Map.empty, 1L, 1L, dataChange = true).remove) ++ (if (cdfEnabled) AddCDCFile("foo", Map.empty, 1L) :: Nil else Nil) testColumnMapping( s"CDF and Column Mapping: $shouldBlockStr when CDF=$cdfEnabled", enableSQLConf = true) { mode => def createTable(): Unit = { createStrictSchemaTableWithDeltaTableApi( "t1", schemaWithPhysicalNamesNested, Map( DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode, DeltaConfigs.CHANGE_DATA_FEED.key -> cdfEnabled.toString ) ) } Seq("h", "b.`foo.bar`.f").foreach { colName => // case 1: drop column with non-FileAction action should always pass withTable("t1") { createTable() val log = DeltaLog.forTable(spark, TableIdentifier("t1")) val droppedColumnSchema = sql("SELECT * FROM t1").drop(colName).schema checkHelper(log, droppedColumnSchema, SetTransaction("id", 1, None), shouldFail = false) } // case 2: rename column with FileAction should fail if $shouldBlock == true fileActions.foreach { fileAction => withTable("t1") { createTable() val log = DeltaLog.forTable(spark, TableIdentifier("t1")) withSQLConf( DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey -> mode) { withTable("t2") { sql("DROP TABLE IF EXISTS t2") sql("CREATE TABLE t2 USING DELTA AS SELECT * FROM t1") sql(s"ALTER TABLE t2 RENAME COLUMN $colName TO ii") val renamedColumnSchema = sql("SELECT * FROM t2").schema checkHelper(log, renamedColumnSchema, fileAction) } } } } // case 3: drop column with FileAction should fail if $shouldBlock == true fileActions.foreach { fileAction => { withTable("t1") { createTable() val log = DeltaLog.forTable(spark, TableIdentifier("t1")) val droppedColumnSchema = sql("SELECT * FROM t1").drop(colName).schema checkHelper(log, droppedColumnSchema, fileAction) } } } } } } testColumnMapping("id and name mode should write field_id in parquet schema", modes = Some(Seq("name", "id"))) { mode => withTable("t1") { createTableWithSQLAPI( "t1", Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode)) val (log, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier("t1")) val files = snapshot.allFiles.collect() files.foreach { f => val footer = ParquetFileReader.readFooter( log.newDeltaHadoopConf(), f.absolutePath(log), ParquetMetadataConverter.NO_FILTER) footer.getFileMetaData.getSchema.getFields.asScala.foreach(f => // getId.intValue will throw NPE if field id does not exist assert(f.getId.intValue >= 0) ) } } } test("should block CM upgrade when commit has FileActions and CDF enabled") { Seq(true, false).foreach { cdfEnabled => var shouldBlock = cdfEnabled withTable("t1") { createTableWithSQLAPI( "t1", props = Map(DeltaConfigs.CHANGE_DATA_FEED.key -> cdfEnabled.toString)) val table = DeltaTableV2(spark, TableIdentifier("t1")) val currMetadata = table.snapshot.metadata val upgradeMetadata = currMetadata.copy( configuration = currMetadata.configuration ++ Map( DeltaConfigs.MIN_READER_VERSION.key -> "2", DeltaConfigs.MIN_WRITER_VERSION.key -> "5", DeltaConfigs.COLUMN_MAPPING_MODE.key -> NameMapping.name ) ) val txn = table.startTransactionWithInitialSnapshot() txn.updateMetadata(upgradeMetadata) if (shouldBlock) { val e = intercept[DeltaUnsupportedOperationException] { txn.commit( AddFile("foo", Map.empty, 1L, 1L, dataChange = true) :: Nil, DeltaOperations.ManualUpdate) }.getMessage assert(e == "[DELTA_BLOCK_COLUMN_MAPPING_AND_CDC_OPERATION] " + "Operation \"Manual Update\" is not allowed when the table has enabled " + "change data feed (CDF) and has undergone schema changes using DROP COLUMN or RENAME " + "COLUMN.") } else { txn.commit( AddFile("foo", Map.empty, 1L, 1L, dataChange = true) :: Nil, DeltaOperations.ManualUpdate) } } } } test("upgrade with dot column name should not be blocked") { testCreateTableColumnMappingMode( "t1", schemaWithDottedColumnNames, false, "name", createNewTable = false, tableFeaturesProtocolExpected = false ) { sql(s"CREATE TABLE t1 (${schemaWithDottedColumnNames.toDDL}) USING DELTA") alterTableWithProps("t1", props = Map( DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name", DeltaConfigs.MIN_READER_VERSION.key -> "2", DeltaConfigs.MIN_WRITER_VERSION.key -> "5")) } } test("explicit id matching") { // Explicitly disable field id reading to test id mode reinitialization val requiredConfs = Seq( SQLConf.PARQUET_FIELD_ID_READ_ENABLED, SQLConf.PARQUET_FIELD_ID_WRITE_ENABLED) requiredConfs.foreach { conf => withSQLConf(conf.key -> "false") { val e = intercept[IllegalArgumentException] { withTable("t1") { createStrictSchemaTableWithDeltaTableApi( "t1", schemaWithIdNested, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> "id") ) val testData = spark.createDataFrame( Seq(Row("str3", Row("str1.3", 3))).asJava, schemaWithIdNested) testData.write.format("delta").mode("append").saveAsTable("t1") } } assert(e.getMessage.contains(conf.key)) } } // The above configs are enabled by default, so no need to explicitly enable. withTable("t1") { val testSchema = schemaWithIdNested.add("e", StringType, true, withId(5)) val testData = spark.createDataFrame( Seq(Row("str3", Row("str1.3", 3), "str4")).asJava, testSchema) createStrictSchemaTableWithDeltaTableApi( "t1", testSchema, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> "id") ) testData.write.format("delta").mode("append").saveAsTable("t1") def read: DataFrame = spark.read.format("delta").table("t1") val deltaLog = DeltaLog.forTable(spark, TableIdentifier("t1")) def updateFieldIdFor(fieldName: String, newId: Int): Unit = { val currentMetadata = deltaLog.update().metadata val currentSchema = currentMetadata.schema val field = currentSchema(fieldName) deltaLog.withNewTransaction { txn => val updated = field.copy(metadata = new MetadataBuilder().withMetadata(field.metadata) .putLong(DeltaColumnMapping.PARQUET_FIELD_ID_METADATA_KEY, newId) .putLong(DeltaColumnMapping.COLUMN_MAPPING_METADATA_ID_KEY, newId) .build()) val newSchema = StructType(Seq(updated) ++ currentSchema.filter(_.name != field.name)) txn.commit(currentMetadata.copy( schemaString = newSchema.json, configuration = currentMetadata.configuration ++ // Just a big id to bypass the check Map(DeltaConfigs.COLUMN_MAPPING_MAX_ID.key -> "10000")) :: Nil, ManualUpdate) } } // Case 1: manually modify the schema to read a non-existing id updateFieldIdFor("a", 100) // Reading non-existing id should return null checkAnswer(read.select("a"), Row(null) :: Nil) // Case 2: manually modify the schema to read another field's id // First let's drop e, because Delta detects duplicated field sql(s"ALTER TABLE t1 DROP COLUMN e") // point to the dropped field 's data updateFieldIdFor("a", 5) checkAnswer(read.select("a"), Row("str4")) } } test("drop and recreate external Delta table with name column mapping enabled") { withTempDir { dir => withTable("t1") { val createExternalTblCmd: String = s""" |CREATE EXTERNAL TABLE t1 (a long) |USING DELTA |LOCATION '${dir.getCanonicalPath}' |TBLPROPERTIES('delta.columnMapping.mode'='name')""".stripMargin sql(createExternalTblCmd) // Add column and drop the old one to increment max column ID sql(s"ALTER TABLE t1 ADD COLUMN (b long)") sql(s"ALTER TABLE t1 DROP COLUMN a") sql(s"ALTER TABLE t1 RENAME COLUMN b to a") val log = DeltaLog.forTable(spark, dir.getCanonicalPath) val configBeforeDrop = log.update().metadata.configuration assert(configBeforeDrop("delta.columnMapping.maxColumnId") == "2") sql(s"DROP TABLE t1") sql(createExternalTblCmd) // Configuration after recreating the external table should match the config right // before initially dropping it. assert(log.update().metadata.configuration == configBeforeDrop) // Adding another column picks up from the last maxColumnId and increments it sql(s"ALTER TABLE t1 ADD COLUMN (c string)") assert(log.update().metadata.configuration("delta.columnMapping.maxColumnId") == "3") } } } test("replace external Delta table with name column mapping enabled") { withTempDir { dir => withTable("t1") { val replaceExternalTblCmd: String = s""" |CREATE OR REPLACE TABLE t1 (a long) |USING DELTA |LOCATION '${dir.getCanonicalPath}' |TBLPROPERTIES('delta.columnMapping.mode'='name')""".stripMargin sql(replaceExternalTblCmd) // Add column and drop the old one to increment max column ID sql(s"ALTER TABLE t1 ADD COLUMN (b long)") sql(s"ALTER TABLE t1 DROP COLUMN a") sql(s"ALTER TABLE t1 RENAME COLUMN b to a") val log = DeltaLog.forTable(spark, dir.getCanonicalPath) assert(log.update().metadata.configuration("delta.columnMapping.maxColumnId") == "2") withSQLConf(DeltaSQLConf.REUSE_COLUMN_METADATA_DURING_REPLACE_TABLE.key -> "true") { sql(replaceExternalTblCmd) // Replace table doesn't reassign field id if column is unchanged assert(log.update().metadata.configuration("delta.columnMapping.maxColumnId") == "2") } withSQLConf(DeltaSQLConf.REUSE_COLUMN_METADATA_DURING_REPLACE_TABLE.key -> "false") { sql(replaceExternalTblCmd) // Replace table starts assigning field id from previous maxColumnId. assert(log.update().metadata.configuration("delta.columnMapping.maxColumnId") == "3") } } } } test("replace delta table will reuse the field id only when column name and type unchanged") { withTempDir { dir => withTable("t1") { sql(s""" |CREATE TABLE t1 (a long, b int) |USING DELTA |LOCATION '${dir.getCanonicalPath}' |TBLPROPERTIES('delta.columnMapping.mode'='name')""".stripMargin) // Check field IDs before replacement val logBefore = DeltaLog.forTable(spark, dir.getCanonicalPath) val colABefore = logBefore.update().metadata.schema.fields.find(_.name == "a").get val colBBefore = logBefore.update().metadata.schema.fields.find(_.name == "b").get assert(colABefore.metadata.getLong("delta.columnMapping.id") === 1L) assert(colBBefore.metadata.getLong("delta.columnMapping.id") === 2L) withSQLConf(DeltaSQLConf.REUSE_COLUMN_METADATA_DURING_REPLACE_TABLE.key -> "true") { sql(s""" |REPLACE TABLE t1 (a long, b long) |USING DELTA |LOCATION '${dir.getCanonicalPath}' |TBLPROPERTIES('delta.columnMapping.mode'='name')""".stripMargin) } // Check field IDs after replacement val log = DeltaLog.forTable(spark, dir.getCanonicalPath) val colA = log.update().metadata.schema.fields.find(_.name == "a").get val colB = log.update().metadata.schema.fields.find(_.name == "b").get assert(colA.metadata.getLong("delta.columnMapping.id") === 1L) assert(colABefore.metadata.getString("delta.columnMapping.physicalName") === colA.metadata.getString("delta.columnMapping.physicalName")) assert(colB.metadata.getLong("delta.columnMapping.id") === 3L) assert(colBBefore.metadata.getString("delta.columnMapping.physicalName") !== colB.metadata.getString("delta.columnMapping.physicalName")) } } } test("replace delta table will not reuse the field id when name mapping mode changed") { withSQLConf(DeltaSQLConf.REUSE_COLUMN_METADATA_DURING_REPLACE_TABLE.key -> "true") { Seq("id", "none").foreach { updatedNameMapping => withTempDir { dir => withTable("t1") { sql(s""" |CREATE TABLE t1 (a long, b int) |USING DELTA |LOCATION '${dir.getCanonicalPath}' |TBLPROPERTIES('delta.columnMapping.mode'='name')""".stripMargin) // Check field IDs before replacement val logBefore = DeltaLog.forTable(spark, dir.getCanonicalPath) val colABefore = logBefore.update().metadata.schema.fields.find(_.name == "a").get val colBBefore = logBefore.update().metadata.schema.fields.find(_.name == "b").get assert(colABefore.metadata.getLong("delta.columnMapping.id") === 1L) assert(colBBefore.metadata.getLong("delta.columnMapping.id") === 2L) // Replace table with different mapping mode sql(s""" |REPLACE TABLE t1 (a long, b long) |USING DELTA |LOCATION '${dir.getCanonicalPath}' |TBLPROPERTIES('delta.columnMapping.mode'='$updatedNameMapping')""".stripMargin) val log = DeltaLog.forTable(spark, dir.getCanonicalPath) val colA = log.update().metadata.schema.fields.find(_.name == "a").get val colB = log.update().metadata.schema.fields.find(_.name == "b").get if (updatedNameMapping == "id") { assert(colA.metadata.getLong("delta.columnMapping.id") === 3L) assert(colB.metadata.getLong("delta.columnMapping.id") === 4L) } else { assert(!colA.metadata.contains("delta.columnMapping.id")) assert(!colB.metadata.contains("delta.columnMapping.id")) } } } } } } test("restore Delta table with name column mapping enabled") { withTempDir { dir => withTable("t1") { sql(s""" |CREATE OR REPLACE TABLE t1 (a long) |USING DELTA |LOCATION '${dir.getCanonicalPath}' |TBLPROPERTIES('delta.columnMapping.mode'='name')""".stripMargin) // Add column and drop the old one to increment max column ID sql(s"ALTER TABLE t1 ADD COLUMN (b long)") sql(s"ALTER TABLE t1 DROP COLUMN a") sql(s"ALTER TABLE t1 RENAME COLUMN b to a") val log = DeltaLog.forTable(spark, dir.getCanonicalPath) assert(log.update().metadata.configuration("delta.columnMapping.maxColumnId") == "2") sql(s"RESTORE TABLE t1 TO VERSION AS OF 0") // Restore should not reduce the max field id, // but it should also not give out new field ids to the restored schema. assert(log.update().metadata.configuration("delta.columnMapping.maxColumnId") == "2") } } } test("verify internal table properties only if property exists in spec and existing metadata") { val withoutMaxColumnId = Map[String, String]("delta.columnMapping.mode" -> "name") val maxColumnIdOne = Map[String, String]( "delta.columnMapping.mode" -> "name", "delta.columnMapping.maxColumnId" -> "1" ) val maxColumnIdOneWithOthers = Map[String, String]( "delta.columnMapping.mode" -> "name", "delta.columnMapping.maxColumnId" -> "1", "dummy.property" -> "dummy" ) val maxColumnIdTwo = Map[String, String]( "delta.columnMapping.mode" -> "name", "delta.columnMapping.maxColumnId" -> "2" ) // Max column ID is missing in first set of configs. So don't block on verification. assert(DeltaColumnMapping.verifyInternalProperties(withoutMaxColumnId, maxColumnIdOne)) // Max column ID matches. assert(DeltaColumnMapping.verifyInternalProperties(maxColumnIdOne, maxColumnIdOneWithOthers)) // Max column IDs don't match assert(!DeltaColumnMapping.verifyInternalProperties(maxColumnIdOne, maxColumnIdTwo)) } testColumnMapping( "overwrite a column mapping table should preserve column mapping metadata", enableSQLConf = true) { _ => val data = spark.range(10).toDF("id").withColumn("value", lit(1)) def checkReadability( oldDf: DataFrame, expected: DataFrame, overwrite: () => Unit, // Whether the new data files are readable after applying the fix. readableWithFix: Boolean = true, // Whether the method can read the new data files out of box, regardless of the fix. readableOutOfBox: Boolean = false): Unit = { // Overwrite overwrite() if (readableWithFix) { // Previous analyzed DF is still readable // Apply a .select so the plan cache won't kick in. checkAnswer(oldDf.select("id"), expected.select("id").collect()) withSQLConf(DeltaSQLConf.REUSE_COLUMN_MAPPING_METADATA_DURING_OVERWRITE.key -> "false") { // Overwrite again overwrite() if (readableOutOfBox) { checkAnswer(oldDf.select("value"), expected.select("value").collect()) } else { // Without the fix, will fail assert { intercept[DeltaAnalysisException] { oldDf.select("value").collect() }.getErrorClass == "DELTA_SCHEMA_CHANGE_SINCE_ANALYSIS" } } } } else { // Not readable, just fail assert { intercept[DeltaAnalysisException] { oldDf.select("value").collect() }.getErrorClass == "DELTA_SCHEMA_CHANGE_SINCE_ANALYSIS" } } } // Readable - overwrite using DF val overwriteData1 = spark.range(10, 20).toDF("id").withColumn("value", lit(2)) withTempDir { dir => data.write.format("delta").save(dir.getCanonicalPath) val df = spark.read.format("delta").load(dir.getCanonicalPath) checkAnswer(df, data.collect()) checkReadability(df, overwriteData1, () => { overwriteData1.write.mode("overwrite") .option("overwriteSchema", "true") .format("delta") .save(dir.getCanonicalPath) }) } // Unreadable - data type changes val overwriteIncompatibleDatatType = spark.range(10, 20).toDF("id").withColumn("value", lit("name")) withTempDir { dir => data.write.format("delta").save(dir.getCanonicalPath) val df = spark.read.format("delta").load(dir.getCanonicalPath) checkAnswer(df, data.collect()) checkReadability(df, overwriteIncompatibleDatatType, () => { overwriteIncompatibleDatatType.write.mode("overwrite") .option("overwriteSchema", "true") .format("delta") .save(dir.getCanonicalPath) }, readableWithFix = false) } def withTestTable(f: (String, DataFrame) => Unit): Unit = { val tableName = s"cm_table" withTable(tableName) { data.createOrReplaceTempView("src_data") spark.sql(s"CREATE TABLE $tableName USING DELTA AS SELECT * FROM src_data") val df = spark.read.table(tableName) checkAnswer(df, data.collect()) f(tableName, df) } } withTestTable { (tableName, df) => // "overwrite" using REPLACE won't be covered by this fix because this is logically equivalent // to DROP and RECREATE a new table. Therefore this optimization won't kick in. overwriteData1.createOrReplaceTempView("overwrite_data") checkReadability(df, overwriteData1, () => { spark.sql(s"REPLACE TABLE $tableName USING DELTA AS SELECT * FROM overwrite_data") }, readableWithFix = false) } withTestTable { (tableName, df) => // "overwrite" using INSERT OVERWRITE actually works without this fix because it will NOT // trigger the overwriteSchema code path. In this case, the pre and post schema are exactly // the same, so in fact no schema updates would occur. val overwriteData2 = spark.range(20, 30).toDF("id").withColumn("value", lit(2)) overwriteData2.createOrReplaceTempView("overwrite_data2") checkReadability(df, overwriteData2, () => { spark.sql(s"INSERT OVERWRITE $tableName SELECT * FROM overwrite_data2") }, readableOutOfBox = true) } } test("column mapping upgrade with table features") { val testTableName = "columnMappingTestTable" withTable(testTableName) { val minReaderKey = DeltaConfigs.MIN_READER_VERSION.key val minWriterKey = DeltaConfigs.MIN_WRITER_VERSION.key sql( s"""CREATE TABLE $testTableName |USING DELTA |TBLPROPERTIES( |'${DeltaConfigs.ROW_TRACKING_ENABLED.key}' = 'true' |) |AS SELECT * FROM RANGE(1) |""".stripMargin) // [[DeltaColumnMapping.verifyAndUpdateMetadataChange]] should not throw an error. The table // does not need to support read table features too. val columnMappingMode = DeltaConfigs.COLUMN_MAPPING_MODE.key sql( s"""ALTER TABLE $testTableName SET TBLPROPERTIES( |'$columnMappingMode'='name' |)""".stripMargin) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(testTableName)) assert(deltaLog.update().protocol === Protocol(2, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, ColumnMappingTableFeature, RowTrackingFeature ))) } } test("DELTA_INVALID_CHARACTERS_IN_COLUMN_NAMES exception should include column names") { val testTableName = "columnMappingTestTable" withTable(testTableName) { val invalidColName1 = colName("col1") val invalidColName2 = colName("col2") // Make sure the error class stays the same for a single and multiple columns. testWithInvalidColumns(Seq(invalidColName1)) testWithInvalidColumns(Seq(invalidColName1, invalidColName2)) def testWithInvalidColumns(invalidColumns: Seq[String]): Unit = { val allColumns = (Seq("a", "b") ++ invalidColumns) .mkString("(`", "` int, `", "` int)") val e = intercept[DeltaAnalysisException] { sql( s"""CREATE TABLE $testTableName $allColumns |USING DELTA |TBLPROPERTIES('${DeltaConfigs.COLUMN_MAPPING_MODE.key}'='none') |""".stripMargin) } checkError(e, "DELTA_INVALID_CHARACTERS_IN_COLUMN_NAMES", "42K05", Map("invalidColumnNames" -> invalidColumns.mkString(", ")) ) } } } test("filters pushed down to parquet use physical names") { val tableName = "table_name" withTable(tableName) { // Create a table with column mapping **disabled** sql( s"""CREATE TABLE $tableName (a INT, b INT) |USING DELTA |TBLPROPERTIES ( | '${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'none', | '${DeltaConfigs.MIN_READER_VERSION.key}' = '2', | '${DeltaConfigs.MIN_WRITER_VERSION.key}' = '5' |) |""".stripMargin) sql(s"INSERT INTO $tableName VALUES (100, 1000)") sql( s"""ALTER TABLE $tableName |SET TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'name') |""".stripMargin) // Confirm that the physical names are equal to the logical names val schema = DeltaLog.forTable(spark, TableIdentifier(tableName)).update().schema assert(DeltaColumnMapping.getPhysicalName(schema("a")) == "a") assert(DeltaColumnMapping.getPhysicalName(schema("b")) == "b") // Rename the columns so that the logical name of the second column is equal to the physical // name of the first column. sql(s"ALTER TABLE $tableName RENAME COLUMN a TO c") sql(s"ALTER TABLE $tableName RENAME COLUMN b TO a") // Filter the table by the second column. This will return empty results if the filter was // (incorrectly) pushed down without translating the logical names to physical names. checkAnswer( sql(s"SELECT * FROM $tableName WHERE a = 1000"), Seq(Row(100, 1000)) ) } } testColumnMapping("stream read from column mapping does not leak metadata") { mode => withTempDir { dir => val (t1, t2, t3) = ( s"t1_${System.currentTimeMillis()}", s"t2_${System.currentTimeMillis()}", s"t3_${System.currentTimeMillis()}" ) withTable(t1, t2, t3) { // Create source table with column mapping mode and partitioning sql( s"""CREATE TABLE $t1 (a INT, b STRING) |USING DELTA |PARTITIONED BY (b) |TBLPROPERTIES ( | '${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = '$mode', | '${DeltaConfigs.MIN_READER_VERSION.key}' = '2', | '${DeltaConfigs.MIN_WRITER_VERSION.key}' = '5' |) |""".stripMargin) // Insert data into source table sql(s"INSERT INTO $t1 VALUES (1, 'a'), (2, 'b')") // Stream read from source table val streamDf = spark.readStream.format("delta").table(t1) // Should not contain column mapping metadata assert(streamDf.schema.forall(_.metadata.json == "{}")) // Create and write to another table // The streaming create-table path is what currently leaks the column mapping metadata // into the target table. If it was writing to an existing table via DeltaSink, it would not // leak because we pruned the column mapping metadata in [[ImplicitMetadataOperations]] when // we update the target metadata. val q = streamDf.writeStream .partitionBy("b") .trigger(org.apache.spark.sql.streaming.Trigger.AvailableNow()) .format("delta") .option("checkpointLocation", new File(dir, "_checkpoint1").getCanonicalPath) .toTable(t2) q.awaitTermination() // Check target table Delta log val deltaLog = DeltaLog.forTable(spark, TableIdentifier(t2)) assert(deltaLog.update().metadata.schema.forall(_.metadata.json == "{}")) assert(deltaLog.update().metadata.columnMappingMode == NoMapping) // Check target table data checkAnswer(spark.table(t2), Seq(Row(1, "a"), Row(2, "b"))) } } } for (txnIntroducesMetadata <- BOOLEAN_DOMAIN) { test("column mapping metadata are stripped when feature is disabled - " + s"txnIntroducesMetadata=$txnIntroducesMetadata") { withTempDir { dir => val tablePath = dir.getCanonicalPath val deltaLog = DeltaLog.forTable(spark, tablePath) // Create the original table. val schemaV0 = if (txnIntroducesMetadata) { new StructType().add("id", LongType, true) } else { new StructType().add("id", LongType, true, withIdAndPhysicalName(0, "col-0")) } withSQLConf(DeltaSQLConf.DELTA_COLUMN_MAPPING_STRIP_METADATA.key -> "false") { deltaLog.withNewTransaction(catalogTableOpt = None) { txn => val metadata = actions.Metadata( name = "testTable", schemaString = schemaV0.json, configuration = Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> NoMapping.name) ) txn.updateMetadata(metadata) txn.commit(Seq.empty, ManualUpdate) } } val metadataV0 = deltaLog.update().metadata assert(DeltaColumnMapping.schemaHasColumnMappingMetadata(metadataV0.schema) === !txnIntroducesMetadata) // Update the schema of the existing table. withSQLConf(DeltaSQLConf.DELTA_COLUMN_MAPPING_STRIP_METADATA.key -> "true") { deltaLog.withNewTransaction(catalogTableOpt = None) { txn => val schemaV1 = schemaV0.add("value", LongType, true, withIdAndPhysicalName(0, "col-0")) val metadata = metadataV0.copy(schemaString = schemaV1.json) txn.updateMetadata(metadata) txn.commit(Seq.empty, ManualUpdate) } val metadataV1 = deltaLog.update().metadata assert(DeltaColumnMapping.schemaHasColumnMappingMetadata(metadataV1.schema) === !txnIntroducesMetadata) } } } } test("Illegal null value specified for delta.columnMapping.mode option") { withTempPath { tempPath => val ex = intercept[DeltaIllegalArgumentException] { spark.range(10).write.mode("overwrite").format("delta"). option("delta.columnMapping.mode", null).save(tempPath.toString) } val supportedModes = DeltaColumnMapping.supportedModes.map(_.name).toSeq.mkString(", ") assert(ex.getErrorClass === "DELTA_MODE_NOT_SUPPORTED") assert(ex.getMessage.contains("Specified mode 'null' is not supported. " + s"Supported modes are: $supportedModes")) } } test("enabling column mapping disallowed if column mapping metadata already exists") { withSQLConf( // enabling this fixes the issue of committing invalid metadata in the first place DeltaSQLConf.DELTA_COLUMN_MAPPING_STRIP_METADATA.key -> "false" ) { withTempDir { dir => val path = dir.getCanonicalPath val deltaLog = DeltaLog.forTable(spark, path) deltaLog.withNewTransaction(catalogTableOpt = None) { txn => val schema = new StructType().add("id", IntegerType, true, withIdAndPhysicalName(0, "col-0")) val metadata = actions.Metadata( name = "test_table", schemaString = schema.json, configuration = Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> NoMapping.name) ) txn.updateMetadata(metadata) txn.commit(Seq.empty, DeltaOperations.ManualUpdate) // Enabling the config will disallow enabling column mapping. withSQLConf(DeltaSQLConf .DELTA_COLUMN_MAPPING_DISALLOW_ENABLING_WHEN_METADATA_ALREADY_EXISTS.key -> "true") { val e = intercept[DeltaColumnMappingUnsupportedException] { alterTableWithProps( s"delta.`$path`", Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> NameMapping.name)) } assert(e.getErrorClass == "DELTA_ENABLING_COLUMN_MAPPING_DISALLOWED_WHEN_COLUMN_MAPPING_METADATA_ALREADY_EXISTS") } // Disabling the config will allow enabling column mapping. withSQLConf(DeltaSQLConf .DELTA_COLUMN_MAPPING_DISALLOW_ENABLING_WHEN_METADATA_ALREADY_EXISTS.key -> "false") { alterTableWithProps( s"delta.`$path`", Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> NameMapping.name)) } } } } } test("unit test physical name assigning is case-insensitive") { val schema = new StructType() .add("A", IntegerType) .add("b", IntegerType) val fieldPathToPhysicalName = Map(Seq("a") -> "x", Seq("b") -> "y") val schemaWithPhysicalNames = DeltaColumnMapping.setPhysicalNames( schema = schema, fieldPathToPhysicalName = fieldPathToPhysicalName) assert(DeltaColumnMapping.getLogicalNameToPhysicalNameMap(schemaWithPhysicalNames) === Map( Seq("A") -> Seq("x"), Seq("b") -> Seq("y"))) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingTestUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import scala.collection.mutable import org.apache.spark.sql.delta.actions.{Metadata, Protocol, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaColumnMappingSelectedTestMixin import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import io.delta.tables.{DeltaTable => OSSDeltaTable} import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.sql.{Column, DataFrame, DataFrameWriter, Dataset, QueryTest, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{AtomicType, StructField, StructType} trait DeltaColumnMappingTestUtilsBase extends SharedSparkSession { import testImplicits._ protected def columnMappingMode: String = NoMapping.name private val PHYSICAL_NAME_REGEX = "col-[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}".r implicit class PhysicalNameString(s: String) { def phy(deltaLog: DeltaLog): String = { PHYSICAL_NAME_REGEX .findFirstIn(s) .getOrElse(getPhysicalName(s, deltaLog)) } } protected def columnMappingEnabled: Boolean = { columnMappingModeString != "none" } protected def columnMappingModeString: String = { spark.conf.getOption(DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey) .getOrElse("none") } /** * Check if two schemas are equal ignoring column mapping metadata * @param schema1 Schema * @param schema2 Schema */ protected def assertEqual(schema1: StructType, schema2: StructType): Unit = { if (columnMappingEnabled) { assert( DeltaColumnMapping.dropColumnMappingMetadata(schema1) == DeltaColumnMapping.dropColumnMappingMetadata(schema2) ) } else { assert(schema1 == schema2) } } /** * Check if two table configurations are equal ignoring column mapping metadata * @param config1 Table config * @param config2 Table config */ protected def assertEqual( config1: Map[String, String], config2: Map[String, String]): Unit = { if (columnMappingEnabled) { assert(dropColumnMappingConfigurations(config1) == dropColumnMappingConfigurations(config2)) } else { assert(config1 == config2) } } /** * Check if a partition with specific values exists. * Handles both column mapped and non-mapped cases * @param partCol Partition column name * @param partValue Partition value * @param deltaLog DeltaLog */ protected def assertPartitionWithValueExists( partCol: String, partValue: String, deltaLog: DeltaLog): Unit = { assert(getPartitionFilePathsWithValue(partCol, partValue, deltaLog).nonEmpty) } /** * Assert partition exists in an array of set of partition names/paths * @param partCol Partition column name * @param deltaLog Delta log * @param inputFiles Input files to scan for DF */ protected def assertPartitionExists( partCol: String, deltaLog: DeltaLog, inputFiles: Array[String]): Unit = { val physicalName = partCol.phy(deltaLog) val allFiles = deltaLog.update().allFiles.collect() // NOTE: inputFiles are *not* URL-encoded. val filesWithPartitions = inputFiles.map { f => allFiles.filter { af => f.contains(af.toPath.toString) }.flatMap(_.partitionValues.keys).toSet } assert(filesWithPartitions.forall(p => p.count(_ == physicalName) > 0)) // for non-column mapped mode, we can check the file paths as well if (!columnMappingEnabled) { assert(inputFiles.forall(path => path.contains(s"$physicalName=")), s"${inputFiles.toSeq.mkString("\n")}\ndidn't contain partition columns $physicalName") } } /** * Load Deltalog from path * @param pathOrIdentifier Location * @param isIdentifier Whether the previous argument is a metastore identifier * @return */ protected def loadDeltaLog(pathOrIdentifier: String, isIdentifier: Boolean = false): DeltaLog = { if (isIdentifier) { DeltaLog.forTable(spark, TableIdentifier(pathOrIdentifier)) } else { DeltaLog.forTable(spark, pathOrIdentifier) } } /** * Convert a (nested) column string to sequence of name parts * @param col Column string * @return Sequence of parts */ protected def columnNameToParts(col: String): Seq[String] = { UnresolvedAttribute.parseAttributeName(col) } /** * Get partition file paths for a specific partition value * @param partCol Logical or physical partition name * @param partValue Partition value * @param deltaLog DeltaLog * @return List of paths */ protected def getPartitionFilePathsWithValue( partCol: String, partValue: String, deltaLog: DeltaLog): Array[String] = { getPartitionFilePaths(partCol, deltaLog).getOrElse(partValue, Array.empty) } /** * Get the partition value for null */ protected def nullPartitionValue: String = { if (columnMappingEnabled) { null } else { ExternalCatalogUtils.DEFAULT_PARTITION_NAME } } /** * Get partition file paths grouped by partition value * @param partCol Logical or physical partition name * @param deltaLog DeltaLog * @return Partition value to paths */ protected def getPartitionFilePaths( partCol: String, deltaLog: DeltaLog): Map[String, Array[String]] = { if (columnMappingEnabled) { val colName = partCol.phy(deltaLog) deltaLog.update().allFiles.collect() .groupBy(_.partitionValues(colName)) .mapValues(_.map(deltaLog.dataPath.toUri.getPath + "/" + _.path)).toMap } else { val partColEscaped = s"${ExternalCatalogUtils.escapePathName(partCol)}" val dataPath = new File(deltaLog.dataPath.toUri.getPath) dataPath.listFiles().filter(_.getName.startsWith(s"$partColEscaped=")) .groupBy(_.getName.split("=").last).mapValues(_.map(_.getPath)).toMap } } /** * Group a list of input file paths by partition key-value pair w.r.t. delta log * @param inputFiles Input file paths * @param deltaLog Delta log * @return A mapped array each with the corresponding partition keys */ protected def groupInputFilesByPartition( inputFiles: Array[String], deltaLog: DeltaLog): Map[(String, String), Array[String]] = { if (columnMappingEnabled) { val allFiles = deltaLog.update().allFiles.collect() val grouped = inputFiles.flatMap { f => allFiles.find { af => f.contains(af.toPath.toString) }.head.partitionValues.map(entry => (f, entry)) }.groupBy(_._2) grouped.mapValues(_.map(_._1)).toMap } else { inputFiles.groupBy(p => { val nameParts = new Path(p).getParent.getName.split("=") (nameParts(0), nameParts(1)) }) } } /** * Drop column mapping configurations from Map * @param configuration Table configuration * @return Configuration */ protected def dropColumnMappingConfigurations( configuration: Map[String, String]): Map[String, String] = { configuration - DeltaConfigs.COLUMN_MAPPING_MODE.key - DeltaConfigs.COLUMN_MAPPING_MAX_ID.key } /** * Drop column mapping configurations from Dataset (e.g. sql("SHOW TBLPROPERTIES t1") * @param configs Table configuration * @return Configuration Dataset */ protected def dropColumnMappingConfigurations( configs: Dataset[(String, String)]): Dataset[(String, String)] = { spark.createDataset(configs.collect().filter(p => !Seq( DeltaConfigs.COLUMN_MAPPING_MAX_ID.key, DeltaConfigs.COLUMN_MAPPING_MODE.key ).contains(p._1) )) } /** Return KV pairs of Protocol-related stuff for checking the result of DESCRIBE TABLE. */ protected def buildProtocolProps(snapshot: Snapshot): Seq[(String, String)] = { val mergedConf = DeltaConfigs.mergeGlobalConfigs(spark.sessionState.conf, snapshot.metadata.configuration) val metadata = snapshot.metadata.copy(configuration = mergedConf) var props = Seq( (Protocol.MIN_READER_VERSION_PROP, Protocol.forNewTable(spark, Some(metadata)).minReaderVersion.toString), (Protocol.MIN_WRITER_VERSION_PROP, Protocol.forNewTable(spark, Some(metadata)).minWriterVersion.toString)) if (snapshot.protocol.supportsReaderFeatures || snapshot.protocol.supportsWriterFeatures) { props ++= Protocol.minProtocolComponentsFromAutomaticallyEnabledFeatures( spark, metadata, snapshot.protocol) ._3 .map(f => ( s"${TableFeatureProtocolUtils.FEATURE_PROP_PREFIX}${f.name}", TableFeatureProtocolUtils.FEATURE_PROP_SUPPORTED)) } props } /** * Convert (nested) column name string into physical name with reference from DeltaLog * If target field does not have physical name, display name is returned * @param col Logical column name * @param deltaLog Reference DeltaLog * @return Physical column name */ protected def getPhysicalName(col: String, deltaLog: DeltaLog): String = { val nameParts = UnresolvedAttribute.parseAttributeName(col) val realSchema = deltaLog.update().schema getPhysicalName(nameParts, realSchema) } protected def getPhysicalName(col: String, schema: StructType): String = { val nameParts = UnresolvedAttribute.parseAttributeName(col) getPhysicalName(nameParts, schema) } protected def getPhysicalName(nameParts: Seq[String], schema: StructType): String = { SchemaUtils.findNestedFieldIgnoreCase(schema, nameParts, includeCollections = true) .map(DeltaColumnMapping.getPhysicalName) .get } protected def withColumnMappingConf(mode: String)(f: => Any): Any = { withSQLConf(DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey -> mode) { f } } protected def withMaxColumnIdConf(maxId: String)(f: => Any): Any = { withSQLConf(DeltaConfigs.COLUMN_MAPPING_MAX_ID.defaultTablePropertyKey -> maxId) { f } } /** * Gets the physical names of a path. This is used for converting column paths in stats schema, * so it's ok to not support MapType and ArrayType. */ def getPhysicalPathForStats(path: Seq[String], schema: StructType): Option[Seq[String]] = { if (path.isEmpty) return Some(Seq.empty) val field = schema.fields.find(_.name.equalsIgnoreCase(path.head)) field match { case Some(f @ StructField(_, _: AtomicType, _, _ )) => if (path.size == 1) Some(Seq(DeltaColumnMapping.getPhysicalName(f))) else None case Some(f @ StructField(_, st: StructType, _, _)) => val tail = getPhysicalPathForStats(path.tail, st) tail.map(DeltaColumnMapping.getPhysicalName(f) +: _) case _ => None } } /** * Convert (nested) column name string into physical name. * Ignore parts of special paths starting with: * 1. stats columns: minValues, maxValues, numRecords * 2. stats df: stats_parsed * 3. partition values: partitionValues_parsed, partitionValues * @param col Logical column name (e.g. a.b.c) * @param schema Reference schema with metadata * @return Unresolved attribute with physical name paths */ protected def convertColumnNameToAttributeWithPhysicalName( col: String, schema: StructType): UnresolvedAttribute = { val parts = UnresolvedAttribute.parseAttributeName(col) val shouldIgnoreFirstPart = Set( "minValues", "maxValues", "numRecords", Checkpoints.STRUCT_PARTITIONS_COL_NAME, "partitionValues") val shouldIgnoreSecondPart = Set(Checkpoints.STRUCT_STATS_COL_NAME, "stats") val physical = if (shouldIgnoreFirstPart.contains(parts.head)) { parts.head +: getPhysicalPathForStats(parts.tail, schema).getOrElse(parts.tail) } else if (shouldIgnoreSecondPart.contains(parts.head)) { parts.take(2) ++ getPhysicalPathForStats(parts.slice(2, parts.length), schema) .getOrElse(parts.slice(2, parts.length)) } else { getPhysicalPathForStats(parts, schema).getOrElse(parts) } UnresolvedAttribute(physical) } /** * Convert a list of (nested) stats columns into physical name with reference from DeltaLog * @param columns Logical columns * @param deltaLog Reference DeltaLog * @return Physical columns */ protected def convertToPhysicalColumns( columns: Seq[Column], deltaLog: DeltaLog): Seq[Column] = { val schema = deltaLog.update().schema columns.map { col => val newExpr = col.expr.transform { case a: Attribute => convertColumnNameToAttributeWithPhysicalName(a.name, schema) } Column(newExpr) } } /** * Standard CONVERT TO DELTA * @param tableOrPath String */ protected def convertToDelta(tableOrPath: String): Unit = { sql(s"CONVERT TO DELTA $tableOrPath") } /** * Force enable streaming read (with possible data loss) on column mapping enabled table with * drop / rename schema changes. */ protected def withStreamingReadOnColumnMappingTableEnabled(f: => Unit): Unit = { if (columnMappingEnabled) { withSQLConf(DeltaSQLConf .DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_COLUMN_MAPPING_SCHEMA_CHANGES.key -> "true") { f } } else { f } } } trait DeltaColumnMappingTestUtils extends DeltaColumnMappingTestUtilsBase /** * Include this trait to enable Id column mapping mode for a suite */ trait DeltaColumnMappingEnableIdMode extends SharedSparkSession with DeltaColumnMappingTestUtils with DeltaColumnMappingSelectedTestMixin { protected override def columnMappingMode: String = IdMapping.name protected override def sparkConf: SparkConf = super.sparkConf.set(DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey, "id") /** * CONVERT TO DELTA blocked in id mode */ protected override def convertToDelta(tableOrPath: String): Unit = throw DeltaErrors.convertToDeltaWithColumnMappingNotSupported( DeltaColumnMappingMode(columnMappingModeString) ) } /** * Include this trait to enable Name column mapping mode for a suite */ trait DeltaColumnMappingEnableNameMode extends SharedSparkSession with DeltaColumnMappingTestUtils with DeltaColumnMappingSelectedTestMixin { protected override def columnMappingMode: String = NameMapping.name protected override def sparkConf: SparkConf = super.sparkConf.set(DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey, columnMappingMode) /** * CONVERT TO DELTA can be possible under name mode in tests */ protected override def convertToDelta(tableOrPath: String): Unit = { withColumnMappingConf("none") { super.convertToDelta(tableOrPath) } val (deltaPath, deltaLog) = if (tableOrPath.contains("parquet") && tableOrPath.contains("`")) { // parquet.`PATH` val plainPath = tableOrPath.split('.').last.drop(1).dropRight(1) (s"delta.`$plainPath`", DeltaLog.forTable(spark, plainPath)) } else { (tableOrPath, DeltaLog.forTable(spark, TableIdentifier(tableOrPath))) } val tableReaderVersion = deltaLog.unsafeVolatileSnapshot.protocol.minReaderVersion val tableWriterVersion = deltaLog.unsafeVolatileSnapshot.protocol.minWriterVersion val requiredReaderVersion = if (tableWriterVersion >= TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION) { // If the writer version of the table supports table features, we need to // bump the reader version to table features to enable column mapping. TableFeatureProtocolUtils.TABLE_FEATURES_MIN_READER_VERSION } else { ColumnMappingTableFeature.minReaderVersion } val readerVersion = spark.conf.get(DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION).max( requiredReaderVersion) val writerVersion = spark.conf.get(DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION).max( ColumnMappingTableFeature.minWriterVersion) val properties = mutable.ListBuffer(DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name") if (tableReaderVersion < readerVersion) { properties += DeltaConfigs.MIN_READER_VERSION.key -> readerVersion.toString } if (tableWriterVersion < writerVersion) { properties += DeltaConfigs.MIN_WRITER_VERSION.key -> writerVersion.toString } val propertiesStr = properties.map(kv => s"'${kv._1}' = '${kv._2}'").mkString(", ") sql(s"ALTER TABLE $deltaPath SET TBLPROPERTIES ($propertiesStr)") } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnRenameSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.scalatest.GivenWhenThen import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.types._ class DeltaColumnRenameSuite extends QueryTest with DeltaArbitraryColumnNameSuiteBase with GivenWhenThen { testColumnMapping("rename in column mapping mode") { mode => withTable("t1") { createTableWithSQLAPI("t1", simpleNestedData, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode), partCols = Seq("a")) spark.sql(s"Alter table t1 RENAME COLUMN b to b1") // insert data after rename spark.sql("insert into t1 " + "values ('str3', struct('str1.3', 3), map('k3', 'v3'), array(3, 33))") // some queries checkAnswer( spark.table("t1"), Seq( Row("str1", Row("str1.1", 1), Map("k1" -> "v1"), Array(1, 11)), Row("str2", Row("str1.2", 2), Map("k2" -> "v2"), Array(2, 22)), Row("str3", Row("str1.3", 3), Map("k3" -> "v3"), Array(3, 33)))) checkAnswer( spark.table("t1").select("b1"), Seq(Row(Row("str1.1", 1)), Row(Row("str1.2", 2)), Row(Row("str1.3", 3)))) checkAnswer( spark.table("t1").select("a", "b1.c").where("b1.c = 'str1.2'"), Seq(Row("str2", "str1.2"))) // b is no longer visible val e = intercept[AnalysisException] { spark.table("t1").select("b").collect() } // The error class is renamed in Spark 3.4 assert(e.getErrorClass == "UNRESOLVED_COLUMN.WITH_SUGGESTION" || e.getErrorClass == "MISSING_COLUMN" ) // rename partition column spark.sql(s"Alter table t1 RENAME COLUMN a to a1") // rename nested column spark.sql(s"Alter table t1 RENAME COLUMN b1.c to c1") // rename and verify rename history val renameHistoryDf = sql("DESCRIBE HISTORY t1") .where("operation = 'RENAME COLUMN'") .select("version", "operationParameters") checkAnswer(renameHistoryDf, Row(2, Map("oldColumnPath" -> "b", "newColumnPath" -> "b1")) :: Row(4, Map("oldColumnPath" -> "a", "newColumnPath" -> "a1")) :: Row(5, Map("oldColumnPath" -> "b1.c", "newColumnPath" -> "b1.c1")) :: Nil) // cannot rename column to the same name assert( intercept[AnalysisException] { spark.sql(s"Alter table t1 RENAME COLUMN map to map") }.getMessage.contains("already exists")) // cannot rename to a different casing assert( intercept[AnalysisException] { spark.sql("Alter table t1 RENAME COLUMN arr to Arr") }.getMessage.contains("already exists")) // a is no longer visible val e2 = intercept[AnalysisException] { spark.table("t1").select("a").collect() } // The error class is renamed in Spark 3.4 assert(e2.getErrorClass == "UNRESOLVED_COLUMN.WITH_SUGGESTION" || e2.getErrorClass == "MISSING_COLUMN" ) // b1.c is no longer visible val e3 = intercept[AnalysisException] { spark.table("t1").select("b1.c").collect() } assert(e3.getMessage.contains("No such struct field")) // insert data after rename spark.sql("insert into t1 " + "values ('str4', struct('str1.4', 4), map('k4', 'v4'), array(4, 44))") checkAnswer( spark.table("t1").select("a1", "b1.c1", "map") .where("b1.c1 = 'str1.4'"), Seq(Row("str4", "str1.4", Map("k4" -> "v4")))) } } test("rename workflow: error, upgrade to name mode and then rename") { // error when not in the correct protocol and mode withTable("t1") { createTableWithSQLAPI("t1", simpleNestedData, partCols = Seq("a")) val e = intercept[AnalysisException] { spark.sql(s"Alter table t1 RENAME COLUMN map to map1") } assert(e.getMessage.contains("enable Column Mapping") && e.getMessage.contains("mapping mode 'name'")) alterTableWithProps("t1", Map( DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name", DeltaConfigs.MIN_READER_VERSION.key -> "2", DeltaConfigs.MIN_WRITER_VERSION.key -> "5")) // rename a column to have arbitrary chars spark.sql(s"Alter table t1 RENAME COLUMN a to `${colName("a")}`") // rename a column that already has arbitrary chars spark.sql(s"Alter table t1" + s" RENAME COLUMN `${colName("a")}` to `${colName("a1")}`") // rename partition column spark.sql(s"Alter table t1 RENAME COLUMN map to `${colName("map")}`") // insert data after rename spark.sql("insert into t1 " + "values ('str3', struct('str1.3', 3), map('k3', 'v3'), array(3, 33))") checkAnswer( spark.table("t1").select(colName("a1"), "b.d", colName("map")) .where("b.c >= 'str1.2'"), Seq(Row("str2", 2, Map("k2" -> "v2")), Row("str3", 3, Map("k3" -> "v3")))) // add old column back? spark.sql(s"alter table t1 add columns (a string, map map)") // insert data after rename spark.sql("insert into t1 " + "values ('str4', struct('str1.4', 4), map('k4', 'v4'), array(4, 44)," + " 'new_str4', map('new_k4', 'new_v4'))") checkAnswer( spark.table("t1").select(colName("a1"), "a", colName("map"), "map") .where("b.c >= 'str1.2'"), Seq( Row("str2", null, Map("k2" -> "v2"), null), Row("str3", null, Map("k3" -> "v3"), null), Row("str4", "new_str4", Map("k4" -> "v4"), Map("new_k4" -> "new_v4")))) } } test("rename workflow: error, upgrade to name mode and then rename - " + "nested data with duplicated column name") { withTable("t1") { createTableWithSQLAPI("t1", simpleNestedDataWithDuplicatedNestedColumnName) val e = intercept[AnalysisException] { spark.sql(s"Alter table t1 RENAME COLUMN map to map1") } assert(e.getMessage.contains("enable Column Mapping") && e.getMessage.contains("mapping mode 'name'")) // Upgrading this schema shouldn't cause any errors even if there are leaf column name // duplications such as a.c, b.c. alterTableWithProps("t1", Map( DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name", DeltaConfigs.MIN_READER_VERSION.key -> "2", DeltaConfigs.MIN_WRITER_VERSION.key -> "5")) // rename shouldn't cause duplicates in column names Seq(("a", "b"), ("arr", "map")).foreach { case (from, to) => val e = intercept[AnalysisException] { spark.sql(s"Alter table t1 RENAME COLUMN $from to $to") } assert(e.getMessage.contains("Cannot rename column")) } // spice things up by changing name to arbitrary chars spark.sql(s"Alter table t1 RENAME COLUMN a to `${colName("a")}`") // rename partition column spark.sql(s"Alter table t1 RENAME COLUMN map to `${colName("map")}`") // insert data after rename spark.sql("insert into t1 " + "values (struct('str3', 3), struct('str1.3', 3), map('k3', 'v3'), array(3, 33))") checkAnswer( spark.table("t1").select(colName("a"), "b.d", colName("map")) .where("b.c >= 'str1.2'"), Seq(Row(Row("str2", 2), 2, Map("k2" -> "v2")), Row(Row("str3", 3), 3, Map("k3" -> "v3")))) // add old column back? spark.sql(s"alter table t1 add columns (a string, map map)") // insert data after rename spark.sql("insert into t1 " + "values (struct('str4', 4), struct('str1.4', 4), map('k4', 'v4'), array(4, 44)," + " 'new_str4', map('new_k4', 'new_v4'))") checkAnswer( spark.table("t1").select(colName("a"), "a", colName("map"), "map") .where("b.c >= 'str1.2'"), Seq( Row(Row("str2", 2), null, Map("k2" -> "v2"), null), Row(Row("str3", 3), null, Map("k3" -> "v3"), null), Row(Row("str4", 4), "new_str4", Map("k4" -> "v4"), Map("new_k4" -> "new_v4")))) } } test("rename with constraints") { withTable("t1") { val schemaWithNotNull = simpleNestedData.schema.toDDL.replace("c: STRING", "c: STRING NOT NULL") .replace("`c`: STRING", "`c`: STRING NOT NULL") withTable("source") { spark.sql( s""" |CREATE TABLE t1 ($schemaWithNotNull) |USING DELTA |${partitionStmt(Seq("a"))} |${propString(Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name"))} |""".stripMargin) simpleNestedData.write.format("delta").mode("append").saveAsTable("t1") } spark.sql("alter table t1 add constraint rangeABC check (concat(a, a) > 'str')") spark.sql("alter table t1 add constraint rangeBD check (`b`.`d` > 0)") spark.sql("alter table t1 add constraint arrValue check (arr[0] > 0)") assertException("Cannot alter column a") { spark.sql("alter table t1 rename column a to a1") } assertException("Cannot alter column arr") { spark.sql("alter table t1 rename column arr to arr1") } // cannot rename b because its child is referenced assertException("Cannot alter column b") { spark.sql("alter table t1 rename column b to b1") } // can still rename b.c because it's referenced by a null constraint spark.sql("alter table t1 rename column b.c to c1") spark.sql("insert into t1 " + "values ('str3', struct('str1.3', 3), map('k3', 'v3'), array(3, 33))") assertException("CHECK constraint rangeabc (concat(a, a) > 'str')") { spark.sql("insert into t1 " + "values ('fail constraint', struct('str1.3', 3), map('k3', 'v3'), array(3, 33))") } assertException("CHECK constraint rangebd (b.d > 0)") { spark.sql("insert into t1 " + "values ('str3', struct('str1.3', -1), map('k3', 'v3'), array(3, 33))") } assertException("NOT NULL constraint violated for column: b.c1") { spark.sql("insert into t1 " + "values ('str3', struct(null, 3), map('k3', 'v3'), array(3, 33))") } // this is a safety flag - it won't error when you turn it off withSQLConf(DeltaSQLConf.DELTA_ALTER_TABLE_CHANGE_COLUMN_CHECK_EXPRESSIONS.key -> "false") { spark.sql("alter table t1 rename column a to a1") spark.sql("alter table t1 rename column arr to arr1") spark.sql("alter table t1 rename column b to b1") } } } test("rename with constraints - map element") { withTable("t1") { val schemaWithNotNull = simpleNestedData.schema.toDDL.replace("c: STRING", "c: STRING NOT NULL") .replace("`c`: STRING", "`c`: STRING NOT NULL") withTable("source") { spark.sql( s""" |CREATE TABLE t1 ($schemaWithNotNull) |USING DELTA |${partitionStmt(Seq("a"))} |${propString(Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name"))} |""".stripMargin) simpleNestedData.write.format("delta").mode("append").saveAsTable("t1") } spark.sql("alter table t1 add constraint" + " mapValue check (not array_contains(map_keys(map), 'k1') or map['k1'] = 'v1')") assertException("Cannot alter column map") { spark.sql("alter table t1 rename column map to map1") } spark.sql("insert into t1 " + "values ('str3', struct('str1.3', 3), map('k3', 'v3'), array(3, 33))") } } test("rename with generated column") { withTable("t1") { val tableBuilder = io.delta.tables.DeltaTable.create(spark).tableName("t1") tableBuilder.property("delta.columnMapping.mode", "name") // add existing columns simpleNestedSchema.map(field => (field.name, field.dataType)).foreach(col => { val (colName, dataType) = col val columnBuilder = io.delta.tables.DeltaTable.columnBuilder(spark, colName) columnBuilder.dataType(dataType.sql) tableBuilder.addColumn(columnBuilder.build()) }) // add generated columns val genCol1 = io.delta.tables.DeltaTable.columnBuilder(spark, "genCol1") .dataType("int") .generatedAlwaysAs("length(a)") .build() val genCol2 = io.delta.tables.DeltaTable.columnBuilder(spark, "genCol2") .dataType("int") .generatedAlwaysAs("b.d * 100 + arr[0]") .build() val genCol3 = io.delta.tables.DeltaTable.columnBuilder(spark, "genCol3") .dataType("string") .generatedAlwaysAs("concat(a, a)") .build() tableBuilder .addColumn(genCol1) .addColumn(genCol2) .addColumn(genCol3) .partitionedBy("genCol2") .execute() simpleNestedData.write.format("delta").mode("append").saveAsTable("t1") assertException("Cannot alter column a") { spark.sql("alter table t1 rename column a to a1") } assertException("Cannot alter column b") { spark.sql("alter table t1 rename column b to b1") } assertException("Cannot alter column b.d") { spark.sql("alter table t1 rename column b.d to d1") } assertException("Cannot alter column arr") { spark.sql("alter table t1 rename column arr to arr1") } // you can still rename b.c spark.sql("alter table t1 rename column b.c to c1") // The following is just to show generated columns are actually there // add new data (without data for generated columns so that they are auto populated) spark.createDataFrame( Seq(Row("str3", Row("str1.3", 3), Map("k3" -> "v3"), Array(3, 33))).asJava, new StructType() .add("a", StringType, true) .add("b", new StructType() .add("c1", StringType, true) .add("d", IntegerType, true)) .add("map", MapType(StringType, StringType), true) .add("arr", ArrayType(IntegerType), true)) .write.format("delta").mode("append").saveAsTable("t1") checkAnswer(spark.table("t1"), Seq( Row("str1", Row("str1.1", 1), Map("k1" -> "v1"), Array(1, 11), 4, 101, "str1str1"), Row("str2", Row("str1.2", 2), Map("k2" -> "v2"), Array(2, 22), 4, 202, "str2str2"), Row("str3", Row("str1.3", 3), Map("k3" -> "v3"), Array(3, 33), 4, 303, "str3str3"))) // this is a safety flag - if you turn it off, it will still error but msg is not as helpful withSQLConf(DeltaSQLConf.DELTA_ALTER_TABLE_CHANGE_COLUMN_CHECK_EXPRESSIONS.key -> "false") { assertException("A generated column cannot use a non-existent column") { spark.sql("alter table t1 rename column arr to arr1") } assertExceptionOneOf(Seq("No such struct field d in c1, d1", "No such struct field `d` in `c1`, `d1`")) { spark.sql("alter table t1 rename column b.d to d1") } } } } /** * Covers renaming a nested field using the ALTER TABLE command. * @param initialColumnType Type of the single column used to create the initial test table. * @param fieldToRename Old and new name of the field to rename. * @param updatedColumnType Expected type of the single column after renaming the nested field. */ def testRenameNestedField(testName: String)( initialColumnType: String, fieldToRename: (String, String), updatedColumnType: String): Unit = testColumnMapping(s"ALTER TABLE RENAME COLUMN - nested $testName") { mode => withTempDir { dir => withTable("delta_test") { sql( s""" |CREATE TABLE delta_test (data $initialColumnType) |USING delta |TBLPROPERTIES (${DeltaConfigs.COLUMN_MAPPING_MODE.key} = '${mode}') |OPTIONS('path'='${dir.getCanonicalPath}')""".stripMargin) val expectedInitialType = initialColumnType.filterNot(_.isWhitespace) val expectedUpdatedType = updatedColumnType.filterNot(_.isWhitespace) val fieldName = s"data.${fieldToRename._1}" def columnType: DataFrame = sql("DESCRIBE TABLE delta_test") .filter("col_name = 'data'") .select("data_type") checkAnswer(columnType, Row(expectedInitialType)) sql(s"ALTER TABLE delta_test RENAME COLUMN $fieldName TO ${fieldToRename._2}") checkAnswer(columnType, Row(expectedUpdatedType)) } } } testRenameNestedField("struct in map key")( initialColumnType = "map, int>", fieldToRename = "key.b" -> "c", updatedColumnType = "map, int>") testRenameNestedField("struct in map value")( initialColumnType = "map>", fieldToRename = "value.b" -> "c", updatedColumnType = "map>") testRenameNestedField("struct in array")( initialColumnType = "array>", fieldToRename = "element.b" -> "c", updatedColumnType = "array>") testRenameNestedField("struct in nested map keys")( initialColumnType = "map, int>, int>", fieldToRename = "key.key.b" -> "c", updatedColumnType = "map, int>, int>") testRenameNestedField("struct in nested map values")( initialColumnType = "map>>", fieldToRename = "value.value.b" -> "c", updatedColumnType = "map>>") testRenameNestedField("struct in nested arrays")( initialColumnType = "array>>", fieldToRename = "element.element.b" -> "c", updatedColumnType = "array>>") testRenameNestedField("struct in nested array and map")( initialColumnType = "array>>", fieldToRename = "element.value.b" -> "c", updatedColumnType = "array>>") testRenameNestedField("struct in nested map key and array")( initialColumnType = "map>, int>", fieldToRename = "key.element.b" -> "c", updatedColumnType = "map>, int>") testRenameNestedField("struct in nested map value and array")( initialColumnType = "map>>", fieldToRename = "value.element.b" -> "c", updatedColumnType = "map>>") testColumnMapping("ALTER TABLE RENAME COLUMN - rename fields nested in maps") { mode => withTable("t1") { val rows = Seq( Row(Map(Row(1) -> Map(Row(10) -> Row(11)))), Row(Map(Row(2) -> Map(Row(20) -> Row(21))))) val df = spark.createDataFrame( rows = rows.asJava, schema = new StructType() .add("a", MapType( new StructType().add("x", IntegerType), MapType( new StructType().add("y", IntegerType), new StructType().add("z", IntegerType))))) createTableWithSQLAPI("t1", df, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode)) spark.sql(s"ALTER TABLE t1 RENAME COLUMN a.key.x to x1") checkAnswer(spark.table("t1"), rows) spark.sql(s"ALTER TABLE t1 RENAME COLUMN a.value.key.y to y1") checkAnswer(spark.table("t1"), rows) spark.sql(s"ALTER TABLE t1 RENAME COLUMN a.value.value.z to z1") checkAnswer(spark.table("t1"), rows) // Insert data after rename. spark.sql("INSERT INTO t1 " + "VALUES (map(named_struct('x', 3), map(named_struct('y', 30), named_struct('z', 31))))") checkAnswer(spark.table("t1"), rows :+ Row(Map(Row(3) -> Map(Row(30) -> Row(31))))) } } testColumnMapping("ALTER TABLE RENAME COLUMN - rename fields nested in arrays") { mode => withTable("t1") { val rows = Seq( Row(Array(Array(Row(10, 11), Row(12, 13)), Array(Row(14, 15), Row(16, 17)))), Row(Array(Array(Row(20, 21), Row(22, 23)), Array(Row(24, 25), Row(26, 27))))) val schema = new StructType() .add("a", ArrayType(ArrayType( new StructType() .add("x", IntegerType) .add("y", IntegerType)))) val df = spark.createDataFrame(rows.asJava, schema) createTableWithSQLAPI("t1", df, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> mode)) spark.sql(s"ALTER TABLE t1 RENAME COLUMN a.element.element.x to x1") checkAnswer(spark.table("t1"), df) spark.sql(s"ALTER TABLE t1 RENAME COLUMN a.element.element.y to y1") checkAnswer(spark.table("t1"), df) // Insert data after rename. spark.sql( """ |INSERT INTO t1 VALUES ( |array( | array(named_struct('x', 30, 'y', 31), named_struct('x', 32, 'y', 33)), | array(named_struct('x', 34, 'y', 35), named_struct('x', 36, 'y', 37)))) """.stripMargin) val expDf3 = spark.createDataFrame( (rows :+ Row(Array(Array(Row(30, 31), Row(32, 33)), Array(Row(34, 35), Row(36, 37))))) .asJava, schema) checkAnswer(spark.table("t1"), expDf3) } } testColumnMapping("rename column with special characters and data skipping stats") { mode => withSQLConf(DeltaSQLConf.DELTA_RENAME_COLUMN_ESCAPE_NAME.key -> "true") { withTable("t1") { spark.sql( s""" |CREATE TABLE t1 (c int, d string) |USING DELTA |TBLPROPERTIES ( | '${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = '$mode', | '${DeltaConfigs.DATA_SKIPPING_STATS_COLUMNS.key}' = 'c,d' |) |""".stripMargin) spark.sql("INSERT INTO t1 VALUES (1, 'value1'), (2, 'value2'), (3, 'value3')") // Verify stats are collected before rename val deltaLog = DeltaLog.forTable(spark, spark.sessionState.catalog.getTableMetadata( spark.sessionState.sqlParser.parseTableIdentifier("t1"))) val statsBefore = deltaLog.update().allFiles.collect().head.stats assert(statsBefore != null && statsBefore.contains("numRecords")) // Rename column c to a name with special characters spark.sql("ALTER TABLE t1 RENAME COLUMN c TO `c#2`") // Verify the rename worked checkAnswer( spark.table("t1"), Seq(Row(1, "value1"), Row(2, "value2"), Row(3, "value3"))) // Verify we can query using the new column name checkAnswer( spark.sql("SELECT `c#2` FROM t1 WHERE `c#2` > 1"), Seq(Row(2), Row(3))) // Insert data after rename to ensure stats collection still works spark.sql("INSERT INTO t1 VALUES (4, 'value4'), (5, 'value5')") checkAnswer( spark.table("t1"), Seq( Row(1, "value1"), Row(2, "value2"), Row(3, "value3"), Row(4, "value4"), Row(5, "value5"))) // Verify stats are still being collected after rename val statsAfter = deltaLog.update().allFiles.collect().last.stats assert(statsAfter != null && statsAfter.contains("numRecords")) // Verify the rename history includes the escaped column name val renameHistoryDf = sql("DESCRIBE HISTORY t1") .where("operation = 'RENAME COLUMN'") .select("operationParameters") val operationParams = renameHistoryDf.head().getMap[String, String](0) assert(operationParams("oldColumnPath") == "c") assert(operationParams("newColumnPath").contains("c#2")) // Rename c#2 back to c before renaming column d spark.sql("ALTER TABLE t1 RENAME COLUMN `c#2` TO c") // Verify rename back worked checkAnswer( spark.sql("SELECT c FROM t1 WHERE c = 3"), Seq(Row(3))) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaCommitLockSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.storage.{AzureLogStore, S3SingleDriverLogStore} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.SparkFunSuite import org.apache.spark.sql.{LocalSparkSession, SparkSession} import org.apache.spark.sql.catalyst.plans.SQLHelper import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.internal.SQLConf import org.apache.spark.util.Utils class DeltaCommitLockSuite extends SparkFunSuite with LocalSparkSession with SQLHelper { private def verifyIsCommitLockEnabled(path: File, expected: Boolean): Unit = { val deltaLog = DeltaLog.forTable(spark, path) val txn = deltaLog.startTransaction() assert(txn.isCommitLockEnabled == expected) } test("commit lock flag on Azure") { spark = SparkSession.builder() .config("spark.delta.logStore.class", classOf[AzureLogStore].getName) .master("local[2]") .config(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[DeltaCatalog].getName) .getOrCreate() val path = Utils.createTempDir() try { // Should lock by default on Azure verifyIsCommitLockEnabled(path, expected = true) // Should respect user config for (enabled <- true :: false :: Nil) { withSQLConf(DeltaSQLConf.DELTA_COMMIT_LOCK_ENABLED.key -> enabled.toString) { verifyIsCommitLockEnabled(path, expected = enabled) } } } finally { Utils.deleteRecursively(path) } } test("commit lock flag on S3") { spark = SparkSession.builder() .config("spark.delta.logStore.class", classOf[S3SingleDriverLogStore].getName) .master("local[2]") .config(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[DeltaCatalog].getName) .getOrCreate() val path = Utils.createTempDir() try { // Should not lock by default on S3 verifyIsCommitLockEnabled(path, expected = false) // Should respect user config for (enabled <- true :: false :: Nil) { withSQLConf(DeltaSQLConf.DELTA_COMMIT_LOCK_ENABLED.key -> enabled.toString) { verifyIsCommitLockEnabled(path, expected = enabled) } } } finally { Utils.deleteRecursively(path) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaConfigSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.concurrent.TimeUnit import org.apache.spark.sql.delta.DeltaConfigs.{getMilliSeconds, isValidIntervalConfigValue, parseCalendarInterval} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.SparkFunSuite import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.unsafe.types.CalendarInterval import org.apache.spark.util.ManualClock class DeltaConfigSuite extends SparkFunSuite with SharedSparkSession with DeltaSQLCommandTest { test("parseCalendarInterval") { for (input <- Seq("5 MINUTES", "5 minutes", "5 Minutes", "inTERval 5 minutes")) { assert(parseCalendarInterval(input) === new CalendarInterval(0, 0, TimeUnit.MINUTES.toMicros(5))) } for (input <- Seq(null, "", " ")) { val e = intercept[IllegalArgumentException] { parseCalendarInterval(input) } assert(e.getMessage.contains("cannot be null or blank")) } for (input <- Seq("interval", "interval1 day", "foo", "foo 1 day")) { val e = intercept[IllegalArgumentException] { parseCalendarInterval(input) } assert(e.getMessage.contains("not a valid INTERVAL")) } } test("isValidIntervalConfigValue") { for (input <- Seq( // Allow 0 microsecond because we always convert microseconds to milliseconds so 0 // microsecond is the same as 100 microseconds. "0 microsecond", "1 microsecond", "1 millisecond", "1 day", "-1 day 86400001 milliseconds", // This is 1 millisecond "1 day -1 microseconds")) { assert(isValidIntervalConfigValue(parseCalendarInterval(input))) } for (input <- Seq( "-1 microseconds", "-1 millisecond", "-1 day", "1 day -86400001 milliseconds", // This is -1 millisecond "1 month", "1 year")) { assert(!isValidIntervalConfigValue(parseCalendarInterval(input)), s"$input") } } test("Optional Calendar Interval config") { val clock = new ManualClock(System.currentTimeMillis()) // case 1: duration not specified withTempDir { dir => sql(s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta") val retentionTimestampOpt = DeltaLog.forTable(spark, dir, clock) .snapshot.minSetTransactionRetentionTimestamp assert(retentionTimestampOpt.isEmpty) } // case 2: valid duration specified withTempDir { dir => sql( s"""CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta |TBLPROPERTIES ('delta.setTransactionRetentionDuration' = 'interval 1 days') |""".stripMargin) DeltaLog.clearCache() // we want to ensure we can use the ManualClock we pass in val log = DeltaLog.forTable(spark, dir, clock) val retentionTimestampOpt = log.snapshot.minSetTransactionRetentionTimestamp assert(log.clock.getTimeMillis() == clock.getTimeMillis()) val expectedRetentionTimestamp = clock.getTimeMillis() - getMilliSeconds(parseCalendarInterval("interval 1 days")) assert(retentionTimestampOpt.contains(expectedRetentionTimestamp)) } // case 3: invalid duration specified withTempDir { dir => val e = intercept[IllegalArgumentException] { sql( s"""CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta |TBLPROPERTIES ('delta.setTransactionRetentionDuration' = 'interval 1 foo') |""".stripMargin) } assert(e.getMessage.contains("not a valid INTERVAL")) } } test("DeltaSQLConf.ALLOW_ARBITRARY_TABLE_PROPERTIES = true") { withSQLConf(DeltaSQLConf.ALLOW_ARBITRARY_TABLE_PROPERTIES.key -> "true") { // (1) we can set arbitrary table properties withTempDir { tempDir => sql( s"""CREATE TABLE delta.`${tempDir.getCanonicalPath}` (id bigint) USING delta |TBLPROPERTIES ('delta.autoOptimize.autoCompact' = true) |""".stripMargin) } // (2) we still validate matching properties withTempDir { tempDir => val e = intercept[IllegalArgumentException] { sql( s"""CREATE TABLE delta.`${tempDir.getCanonicalPath}` (id bigint) USING delta |TBLPROPERTIES ('delta.setTransactionRetentionDuration' = 'interval 1 foo') |""".stripMargin) } assert(e.getMessage.contains("not a valid INTERVAL")) } } } test("we don't allow arbitrary delta-prefixed table properties") { // standard behavior withSQLConf(DeltaSQLConf.ALLOW_ARBITRARY_TABLE_PROPERTIES.key -> "false") { val e = intercept[AnalysisException] { withTempDir { tempDir => sql( s"""CREATE TABLE delta.`${tempDir.getCanonicalPath}` (id bigint) USING delta |TBLPROPERTIES ('delta.foo' = true) |""".stripMargin) } } checkError(e, "DELTA_UNKNOWN_CONFIGURATION", "F0000", Map( "config" -> "delta.foo", "disableCheckConfig" -> DeltaSQLConf.ALLOW_ARBITRARY_TABLE_PROPERTIES.key)) } } test("allow setting valid and supported isolation level") { // currently only Serializable isolation level is supported withTempDir { dir => sql( s"""CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta |TBLPROPERTIES ('delta.isolationLevel' = 'Serializable') |""".stripMargin) val isolationLevel = DeltaLog.forTable(spark, dir.getCanonicalPath).startTransaction().getDefaultIsolationLevel() assert(isolationLevel == Serializable) } } test("do not allow setting valid but unsupported isolation level") { withTempDir { dir => val e = intercept[IllegalArgumentException] { sql( s"""CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta |TBLPROPERTIES ('delta.isolationLevel' = 'WriteSerializable') |""".stripMargin) } val msg = "requirement failed: delta.isolationLevel must be Serializable" assert(e.getMessage == msg) } } test("do not allow setting invalid isolation level") { withTempDir { dir => val e = intercept[DeltaIllegalArgumentException] { sql( s"""CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta |TBLPROPERTIES ('delta.isolationLevel' = 'InvalidSerializable') |""".stripMargin) } checkError(e, "DELTA_INVALID_ISOLATION_LEVEL", "25000", Map("isolationLevel" -> "InvalidSerializable")) } } test("getAllConfigs API") { assert(DeltaConfigs.getAllConfigs.contains("minreaderversion")) assert(!DeltaConfigs.getAllConfigs.contains("confignotexist")) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaCreateTableLikeSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.util.UUID import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.delta.commands.{ CreateDeltaTableCommand, CreateDeltaTableLike, TableCreationModes } import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.scalatest.exceptions.TestFailedException import org.apache.spark.sql.QueryTest import org.apache.spark.sql.SaveMode import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} import org.apache.spark.sql.functions.lit import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.StructType class DeltaCreateTableLikeSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with DeltaSQLTestUtils { def checkTableEmpty(tblName: String): Boolean = { val numRows = spark.sql(s"SELECT * FROM $tblName") numRows.count() == 0 } /** * This method checks if certain properties and fields of delta tables are the * same between the two delta tables. Boolean values can be passed in to check * or not to check (assert) the specific property. Note that for checkLocation * a boolean value is not passed in. If checkLocation argument is None, location * of target table will not be checked. * * @param checkTargetTableByPath when true, targetTbl must be a path not table name * @param checkSourceTableByPath when true, srcTbl must be a path not table name */ def checkTableCopyDelta( srcTbl: String, targetTbl: String, checkDesc: Boolean = true, checkSchemaString: Boolean = true, checkPartitionColumns: Boolean = true, checkConfiguration: Boolean = true, checkTargetTableByPath: Boolean = false, checkSourceTableByPath: Boolean = false, checkLocation: Option[String] = None): Unit = { val src = if (checkSourceTableByPath) { DeltaLog.forTable(spark, srcTbl) } else { DeltaLog.forTable(spark, TableIdentifier(srcTbl)) } val target = if (checkTargetTableByPath) { DeltaLog.forTable(spark, targetTbl) } else { DeltaLog.forTable(spark, TableIdentifier(targetTbl)) } assert(src.unsafeVolatileSnapshot.protocol == target.unsafeVolatileSnapshot.protocol, "protocol does not match") if (checkDesc) { assert(src.unsafeVolatileSnapshot.metadata.description == target.unsafeVolatileSnapshot.metadata.description, "description/comment does not match") } if (checkSchemaString) { assert(src.unsafeVolatileSnapshot.metadata.schemaString == target.unsafeVolatileSnapshot.metadata.schemaString, "schema does not match") } if (checkPartitionColumns) { assert(src.unsafeVolatileSnapshot.metadata.partitionColumns == target.unsafeVolatileSnapshot.metadata.partitionColumns, "partition columns do not match") } if (checkConfiguration) { // Checks Table properties and table constraints assert(src.unsafeVolatileSnapshot.metadata.configuration == target.unsafeVolatileSnapshot.metadata.configuration, "configuration does not match") } val catalog = spark.sessionState.catalog if(checkLocation.isDefined) { assert( catalog.getTableMetadata(TableIdentifier(targetTbl)).location.toString + "/" == checkLocation.get || catalog.getTableMetadata(TableIdentifier(targetTbl)).location.toString == checkLocation.get, "location does not match") } } /** * This method checks if certain properties and fields of a table are the * same between two tables. Boolean values can be passed in to check * or not to check (assert) the specific property. Note that for checkLocation * a boolean value is not passed in. If checkLocation argument is None, location * of target table will not be checked. */ def checkTableCopy( srcTbl: String, targetTbl: String, checkDesc: Boolean = true, checkSchemaString: Boolean = true, checkPartitionColumns: Boolean = true, checkConfiguration: Boolean = true, checkProvider: Boolean = true, checkLocation: Option[String] = None): Unit = { val srcTblDesc = spark.sessionState.catalog. getTempViewOrPermanentTableMetadata(TableIdentifier(srcTbl)) val targetTblDesc = DeltaLog.forTable(spark, TableIdentifier(targetTbl)) val targetTblMetadata = targetTblDesc.unsafeVolatileSnapshot.metadata if (checkDesc) { assert(srcTblDesc.comment == Some(targetTblMetadata.description), "description/comment does not match") } if (checkSchemaString) { assert(srcTblDesc.schema == targetTblDesc.unsafeVolatileSnapshot.metadata.schema, "schema does not match") } if (checkPartitionColumns) { assert(srcTblDesc.partitionColumnNames == targetTblMetadata.partitionColumns, "partition columns do not match") } if (checkConfiguration) { // Checks Table properties assert(srcTblDesc.properties == targetTblMetadata.configuration, "configuration does not match") } if (checkProvider) { val targetTblProvider = spark.sessionState.catalog. getTempViewOrPermanentTableMetadata(TableIdentifier(targetTbl)).provider assert(srcTblDesc.provider == targetTblProvider, "provider does not match") } val catalog = spark.sessionState.catalog if(checkLocation.isDefined) { assert( catalog.getTableMetadata(TableIdentifier(targetTbl)).location.toString + "/" == checkLocation.get || catalog.getTableMetadata(TableIdentifier(targetTbl)).location.toString == checkLocation.get) } } def createTable( srcTbl: String, format: String = "delta", addTblProperties: Boolean = true, addComment: Boolean = true): Unit = { spark.range(100) .withColumnRenamed("id", "key") .withColumn("newCol", lit(1)) .write .format(format) .partitionBy("key") .saveAsTable(srcTbl) if (addTblProperties) { spark.sql(s"ALTER TABLE $srcTbl" + " SET TBLPROPERTIES(this.is.my.key = 14, 'this.is.my.key2' = false)") } if (format == "delta") { spark.sql(s"ALTER TABLE $srcTbl SET TBLPROPERTIES('delta.minReaderVersion' = '2'," + " 'delta.minWriterVersion' = '5')") } if (addComment) { spark.sql(s"COMMENT ON TABLE $srcTbl IS 'srcTbl'") } } test("CREATE TABLE LIKE basic test") { val srcTbl = "srcTbl" val targetTbl = "targetTbl" withTable(srcTbl, targetTbl) { createTable(srcTbl) spark.sql(s"CREATE TABLE $targetTbl LIKE $srcTbl") checkTableCopyDelta(srcTbl, targetTbl) } } test("CREATE TABLE LIKE with no comment") { val srcTbl = "srcTbl" val targetTbl = "targetTbl" withTable(srcTbl, targetTbl) { createTable(srcTbl, addComment = false) spark.sql(s"CREATE TABLE $targetTbl LIKE $srcTbl") checkTableCopyDelta(srcTbl, targetTbl) } } test("CREATE TABLE LIKE with no added table properties") { val srcTbl = "srcTbl" val targetTbl = "targetTbl" withTable(srcTbl, targetTbl) { createTable(srcTbl, addTblProperties = false) spark.sql(s"CREATE TABLE $targetTbl LIKE $srcTbl") checkTableCopyDelta(srcTbl, targetTbl) } } test("CREATE TABLE LIKE where table has no schema") { val srcTbl = "srcTbl" val targetTbl = "targetTbl" withTable(srcTbl, targetTbl) { spark.sql(s"CREATE TABLE $srcTbl USING DELTA") spark.sql(s"CREATE TABLE $targetTbl LIKE $srcTbl") checkTableCopyDelta(srcTbl, targetTbl) } } test("CREATE TABLE LIKE with no added constraints") { val srcTbl = "srcTbl" val targetTbl = "targetTbl" withTable(srcTbl, targetTbl) { createTable(srcTbl ) spark.sql(s"CREATE TABLE $targetTbl LIKE $srcTbl") checkTableCopyDelta(srcTbl, targetTbl) } } test("CREATE TABLE LIKE with IF NOT EXISTS, given that targetTable does not exist") { val srcTbl = "srcTbl" val targetTbl = "targetTbl" withTable(srcTbl, targetTbl) { createTable(srcTbl) spark.sql(s"CREATE TABLE IF NOT EXISTS $targetTbl LIKE $srcTbl USING DELTA") checkTableCopyDelta(srcTbl, targetTbl) } } test("CREATE TABLE LIKE with IF NOT EXISTS, given that targetTable does exist") { val srcTbl = "srcTbl" val targetTbl = "targetTbl" withTable(srcTbl, targetTbl) { createTable(srcTbl) spark.sql(s"CREATE TABLE $targetTbl(key4 INT) USING DELTA") spark.sql(s"CREATE TABLE IF NOT EXISTS $targetTbl LIKE $srcTbl") val msg = intercept[TestFailedException] { checkTableCopyDelta(srcTbl, targetTbl) }.getMessage assert(msg.contains("protocol does not match")) } } test("CREATE TABLE LIKE without IF NOT EXISTS, given that targetTable does exist") { val srcTbl = "srcTbl" val targetTbl = "targetTbl" withTable(srcTbl, targetTbl) { createTable(srcTbl) spark.range(100).repartition(3) .withColumnRenamed("id4", "key4") .write .format("delta") .saveAsTable(targetTbl) val msg = intercept[DeltaAnalysisException] { spark.sql(s"CREATE TABLE $targetTbl LIKE $srcTbl") }.getMessage msg.contains("Table `default`.`targetTbl` already exists.") } } test("concurrent create Managed Catalog table commands should not fail") { withTempDir { dir => withTable("t") { def getCatalogTable: CatalogTable = { val storage = CatalogStorageFormat.empty.copy( locationUri = Some(dir.toPath.resolve(UUID.randomUUID().toString).toUri)) val catalogTableTarget = CatalogTable( identifier = TableIdentifier("t"), tableType = CatalogTableType.MANAGED, storage = storage, provider = Some("delta"), schema = new StructType().add("id", "long")) new DeltaCatalog() .verifyTableAndSolidify( tableDesc = catalogTableTarget, query = None, maybeClusterBySpec = None) } CreateDeltaTableCommand( getCatalogTable, existingTableOpt = None, mode = SaveMode.Ignore, query = None, operation = TableCreationModes.Create).run(spark) assert(spark.sessionState.catalog.tableExists(TableIdentifier("t"))) CreateDeltaTableCommand( getCatalogTable, existingTableOpt = None, // Set to None to simulate concurrent table creation commands. mode = SaveMode.Ignore, query = None, operation = TableCreationModes.Create).run(spark) assert(spark.sessionState.catalog.tableExists(TableIdentifier("t"))) } } } test("catalog-managed CREATE OR REPLACE creates missing tables") { withTempDir { dir => withTable("t") { val storage = CatalogStorageFormat.empty.copy( locationUri = Some(dir.toPath.resolve(UUID.randomUUID().toString).toUri)) val catalogTableTarget = CatalogTable( identifier = TableIdentifier("t"), tableType = CatalogTableType.MANAGED, storage = storage, provider = Some("delta"), schema = new StructType().add("id", "long")) val command = CreateDeltaTableCommand( new DeltaCatalog().verifyTableAndSolidify( tableDesc = catalogTableTarget, query = None, maybeClusterBySpec = None), existingTableOpt = None, mode = SaveMode.ErrorIfExists, query = None, operation = TableCreationModes.CreateOrReplace, allowCatalogManaged = true, createTableFunc = None) command.run(spark) assert(spark.sessionState.catalog.tableExists(TableIdentifier("t"))) } } } test("catalog-managed CREATE OR REPLACE skips catalog create callback " + "when metadata is unchanged") { withCatalogManagedTable(createTable = false) { tableName => spark.sql( s"""CREATE TABLE $tableName (id LONG) USING DELTA |TBLPROPERTIES ('delta.feature.catalogManaged' = 'supported') |""".stripMargin) val existingTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName)) val snapshot = DeltaLog.forTable(spark, existingTable).update() var createCallbackCalls = 0 val command = new CreateDeltaTableLike { override val table: CatalogTable = existingTable override val existingTableOpt: Option[CatalogTable] = Some(existingTable) override val operation: TableCreationModes.CreationMode = TableCreationModes.CreateOrReplace override val mode: SaveMode = SaveMode.ErrorIfExists override val allowCatalogManaged: Boolean = true def runUpdateCatalog(): Unit = { updateCatalog( spark, table, snapshot, query = None, didNotChangeMetadata = true, createTableFunc = Some((_: CatalogTable) => { createCallbackCalls += 1 })) } } command.runUpdateCatalog() assert(createCallbackCalls === 0) } } test("catalog-managed CREATE OR REPLACE rejects query-derived nullable schema") { withCatalogManagedTable(createTable = false) { tableName => withTable("source") { spark.sql( s"""CREATE TABLE $tableName (id LONG NOT NULL) USING DELTA |TBLPROPERTIES ('delta.feature.catalogManaged' = 'supported') |""".stripMargin) spark.sql(s"INSERT INTO $tableName VALUES (1)") spark.sql("CREATE TABLE source (id LONG) USING DELTA") spark.sql("INSERT INTO source VALUES (2)") val existingTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName)) val versionBefore = DeltaLog.forTable(spark, existingTable).update().version val query = spark.sql("SELECT id FROM source").logicalPlan val err = intercept[AssertionError] { new DeltaCatalog().verifyTableAndSolidify( tableDesc = existingTable.copy(schema = query.schema.asNullable), query = Some(query), maybeClusterBySpec = None) } assert(err.getMessage.contains("Can't specify table schema in CTAS.")) assert(DeltaLog.forTable(spark, existingTable).update().version === versionBefore) checkAnswer(spark.sql(s"SELECT * FROM $tableName"), Seq(org.apache.spark.sql.Row(1L))) } } } test("CREATE TABLE LIKE where sourceTable is a json table") { val srcTbl = "srcTbl" val targetTbl = "targetTbl" withTable(srcTbl, targetTbl) { createTable(srcTbl, format = "json") spark.sql(s"CREATE TABLE $targetTbl LIKE $srcTbl USING DELTA") // Provider should be different, expected exception to be thrown val msg = intercept[TestFailedException] { checkTableCopy(srcTbl, targetTbl, checkDesc = false) }.getMessage assert(msg.contains("provider does not match")) } } test("CREATE TABLE LIKE where sourceTable is a parquet table") { val srcTbl = "srcTbl" val targetTbl = "targetTbl" withTable(srcTbl, targetTbl) { createTable(srcTbl, format = "parquet") spark.sql(s"CREATE TABLE $targetTbl LIKE $srcTbl USING DELTA") // Provider should be different, expected exception to be thrown val msg = intercept[TestFailedException] { checkTableCopy(srcTbl, targetTbl, checkDesc = false) }.getMessage assert(msg.contains("provider does not match")) } } test("CREATE TABLE LIKE test where source table is an external table") { val srcTbl = "srcTbl" val targetTbl = "targetTbl" withTempDir { dir => val path = dir.toURI.toString new File(dir.getAbsolutePath, srcTbl).mkdir() withTable(srcTbl, targetTbl) { spark.sql(s"CREATE TABLE $srcTbl (key STRING) USING DELTA LOCATION '$path/$srcTbl'") spark.sql(s"ALTER TABLE $srcTbl" + s" SET TBLPROPERTIES(this.is.my.key = 14, 'this.is.my.key2' = false)") spark.sql(s"COMMENT ON TABLE $srcTbl IS 'srcTbl'") spark.sql(s"CREATE TABLE $targetTbl LIKE $srcTbl") checkTableCopyDelta(srcTbl, targetTbl) } } } test("CREATE TABLE LIKE where target table is a named external table") { val srcTbl = "srcTbl" val targetTbl = "targetTbl" withTempDir(prefix = "sparkdirprefix") { dir => withTable(srcTbl) { createTable(srcTbl) spark.sql(s"CREATE TABLE $targetTbl LIKE $srcTbl LOCATION '${dir.toURI.toString}'") checkTableCopyDelta(srcTbl, targetTbl, checkLocation = Some(dir.toURI.toString)) } } } test("CREATE TABLE LIKE where target table is a nameless table") { val srcTbl = "srcTbl" withTempDir { dir => withTable(srcTbl) { createTable(srcTbl) spark.sql(s"CREATE TABLE delta.`${dir.toURI.toString}` LIKE $srcTbl") checkTableCopyDelta(srcTbl, dir.toURI.toString, checkTargetTableByPath = true ) } } } test("CREATE TABLE LIKE where source is a view") { val srcTbl = "srcTbl" val targetTbl = "targetTbl" val srcView = "srcView" withTable(srcTbl, targetTbl) { withView(srcView) { createTable(srcTbl) spark.sql(s"DROP TABLE IF EXISTS $targetTbl") spark.sql(s"CREATE VIEW srcView AS SELECT * FROM $srcTbl") spark.sql(s"CREATE TABLE $targetTbl LIKE $srcView USING DELTA") val targetTableDesc = DeltaLog.forTable(spark, TableIdentifier(targetTbl)) val srcViewDesc = spark.sessionState.catalog. getTempViewOrPermanentTableMetadata(TableIdentifier(srcView)) assert(targetTableDesc.unsafeVolatileSnapshot.metadata.schema == srcViewDesc.schema) } } } test("CREATE TABLE LIKE where source is a temporary view") { val srcTbl = "srcTbl" val targetTbl = "targetTbl" val srcView = "srcView" withTable(srcTbl, targetTbl) { createTable(srcTbl) spark.sql(s"CREATE TEMPORARY VIEW srcView AS SELECT * FROM $srcTbl") spark.sql(s"CREATE TABLE $targetTbl LIKE $srcView USING DELTA") val targetTableDesc = DeltaLog.forTable(spark, TableIdentifier(targetTbl)) val srcViewDesc = spark.sessionState.catalog. getTempViewOrPermanentTableMetadata(TableIdentifier(srcView)) assert(targetTableDesc.unsafeVolatileSnapshot.metadata.schema == srcViewDesc.schema) } } test("CREATE TABLE LIKE where source table has a column mapping") { val srcTbl = "srcTbl" val targetTbl = "targetTbl" withTable(srcTbl, targetTbl) { createTable(srcTbl ) // Need to set minWriterVersion to 5 for column mappings to work spark.sql(s"ALTER TABLE $srcTbl SET TBLPROPERTIES('delta.minReaderVersion' = '2'," + " 'delta.minWriterVersion' = '5')") // Need to set delta.columnMapping.mode to 'name' for column mappings to work spark.sql(s"ALTER TABLE $srcTbl SET TBLPROPERTIES ('delta.columnMapping.mode' = 'name')") spark.sql(s"ALTER TABLE $srcTbl RENAME COLUMN key TO key2") spark.sql(s"CREATE TABLE $targetTbl LIKE $srcTbl USING DELTA") checkTableCopyDelta(srcTbl, targetTbl) } } test("CREATE TABLE LIKE where user explicitly provides table properties") { val srcTbl = "srcTbl" val targetTbl = "targetTbl" val expectedTbl = "expectedTbl" withTable(srcTbl, targetTbl, expectedTbl) { createTable(srcTbl, addTblProperties = false) createTable(expectedTbl, addTblProperties = false) spark.sql(s"ALTER TABLE $srcTbl" + " SET TBLPROPERTIES(this.is.my.key = 14, 'this.is.my.key2' = false," + "'delta.appendOnly' = 'false')") spark.sql(s"ALTER TABLE $expectedTbl" + " SET TBLPROPERTIES(this.is.my.key = 14, 'this.is.my.key2' = false, " + "'this.is.my.key3' = true, 'delta.appendOnly' = 'true')") spark.sql(s"CREATE TABLE $targetTbl LIKE $srcTbl TBLPROPERTIES('this.is.my.key3' = true, " + s"'delta.appendOnly' = 'true')") checkTableCopyDelta(expectedTbl, targetTbl) } } test("CREATE TABLE LIKE where sourceTable is a parquet table and " + "user explicitly provides table properties" ) { val srcTbl = "srcTbl" val targetTbl = "targetTbl" withTable(srcTbl, targetTbl) { createTable(srcTbl, format = "parquet") spark.sql(s"CREATE TABLE $targetTbl LIKE $srcTbl USING DELTA " + "TBLPROPERTIES('this.is.my.key3' = true)") spark.sql(s"ALTER TABLE $srcTbl SET TBLPROPERTIES('this.is.my.key3' = true)") val msg = intercept[TestFailedException] { checkTableCopy(srcTbl, targetTbl, checkDesc = false) }.getMessage assert(msg.contains("provider does not match")) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaDDLSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import scala.collection.JavaConverters._ import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient.UC_TABLE_ID_KEY import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient.UC_TABLE_ID_KEY_OLD import org.apache.spark.sql.delta.schema.InvariantViolationException import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.hadoop.fs.{Path, UnsupportedFileSystemException} import org.apache.spark.SparkEnv import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.NoSuchPartitionException import org.apache.spark.sql.catalyst.catalog.CatalogUtils import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructType} class DeltaDDLSuite extends DeltaDDLTestBase with SharedSparkSession with DeltaSQLCommandTest { override protected def verifyNullabilityFailure(exception: AnalysisException): Unit = { exception.getMessage.contains("Cannot change nullable column to non-nullable") } test("protocol-related properties are not considered during duplicate table creation") { def createTable(tableName: String, location: String): Unit = { sql(s""" |CREATE TABLE $tableName (id INT, val STRING) |USING DELTA |LOCATION '$location' |TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name', | 'delta.minReaderVersion' = '2', | 'delta.minWriterVersion' = '5' |)""".stripMargin ) } withTempDir { dir => val table1 = "t1" val table2 = "t2" withTable(table1, table2) { withSQLConf(DeltaSQLConf.DELTA_UPDATE_CATALOG_ENABLED.key -> "true") { createTable(table1, dir.getCanonicalPath) createTable(table2, dir.getCanonicalPath) val catalogTable1 = spark.sessionState.catalog.getTableMetadata(TableIdentifier(table1)) val catalogTable2 = spark.sessionState.catalog.getTableMetadata(TableIdentifier(table2)) assert(catalogTable1.properties("delta.columnMapping.mode") == "name") assert(catalogTable2.properties("delta.columnMapping.mode") == "name") } } } } test("table creation with ambiguous paths only allowed with legacy flag") { // ambiguous paths not allowed withTempDir { foo => withTempDir { bar => val fooPath = foo.getCanonicalPath() val barPath = bar.getCanonicalPath() val e = intercept[AnalysisException] { sql(s"CREATE TABLE delta.`$fooPath`(id LONG) USING delta LOCATION '$barPath'") } assert(e.message.contains("legacy.allowAmbiguousPathsInCreateTable")) } } // allowed with legacy flag withTempDir { foo => withTempDir { bar => val fooPath = foo.getCanonicalPath() val barPath = bar.getCanonicalPath() withSQLConf(DeltaSQLConf.DELTA_LEGACY_ALLOW_AMBIGUOUS_PATHS.key -> "true") { sql(s"CREATE TABLE delta.`$fooPath`(id LONG) USING delta LOCATION '$barPath'") assert(io.delta.tables.DeltaTable.isDeltaTable(fooPath)) assert(!io.delta.tables.DeltaTable.isDeltaTable(barPath)) } } } // allowed if paths are the same withTempDir { foo => val fooPath = foo.getCanonicalPath() sql(s"CREATE TABLE delta.`$fooPath`(id LONG) USING delta LOCATION '$fooPath'") assert(io.delta.tables.DeltaTable.isDeltaTable(fooPath)) } } test("append table when column name with special chars") { withTable("t") { val schema = new StructType().add("a`b", "int") val df = spark.createDataFrame(sparkContext.emptyRDD[Row], schema) df.write.format("delta").saveAsTable("t") df.write.format("delta").mode("append").saveAsTable("t") assert(spark.table("t").collect().isEmpty) } } test("CREATE TABLE with OPTIONS") { withTempPath { path => spark.range(10).write.format("delta").save(path.getCanonicalPath) withTable("t") { def createTableWithOptions(simulateUC: Boolean): Unit = { sql( s""" |CREATE TABLE t USING delta LOCATION 'fake://${path.getCanonicalPath}' |${if (simulateUC) "TBLPROPERTIES (test.simulateUC=true)" else ""} |OPTIONS ( | fs.fake.impl='${classOf[FakeFileSystem].getName}', | fs.fake.impl.disable.cache=true) |""".stripMargin) } intercept[UnsupportedFileSystemException](createTableWithOptions(false)) createTableWithOptions(true) } } } test("CREATE TABLE should translate old property `ucTableId` to `io.unitycatalog.tableId`") { for (withBothNewAndOldProperty <- Seq(false, true)) { withTempDir { dir => withTable("t") { val path = dir.getCanonicalPath if (withBothNewAndOldProperty) { // Create table with old and new property key using test.simulateUC to simulate Unity // Catalog sql(s""" |CREATE TABLE t (id INT) USING delta LOCATION '$path' |TBLPROPERTIES ( | test.simulateUC=true, | '$UC_TABLE_ID_KEY_OLD' = 'some-other-id', | '$UC_TABLE_ID_KEY' = 'correct-table-id' |) |""".stripMargin) } else { // Create table with old property key using test.simulateUC to simulate Unity Catalog sql(s""" |CREATE TABLE t (id INT) USING delta LOCATION '$path' |TBLPROPERTIES ( | test.simulateUC=true, | '$UC_TABLE_ID_KEY_OLD' = 'correct-table-id' |) |""".stripMargin) } val deltaLog = DeltaLog.forTable(spark, TableIdentifier("t")) val properties = deltaLog.snapshot.getProperties // Verify the new table id is present with the value from the old key assert(properties.contains(UC_TABLE_ID_KEY), s"New table id key '$UC_TABLE_ID_KEY' should be present in table properties") assert(properties(UC_TABLE_ID_KEY) == "correct-table-id", s"New table id key '$UC_TABLE_ID_KEY' should have value 'correct-table-id'") } } } } } class DeltaDDLNameColumnMappingSuite extends DeltaDDLSuite with DeltaColumnMappingEnableNameMode { override protected def runOnlyTests = Seq( "create table with NOT NULL - check violation through file writing", "ALTER TABLE CHANGE COLUMN with nullability change in struct type - relaxed" ) } abstract class DeltaDDLTestBase extends QueryTest with DeltaSQLTestUtils { import testImplicits._ protected def verifyDescribeTable(tblName: String): Unit = { val res = sql(s"DESCRIBE TABLE $tblName").collect() assert(res.takeRight(2).map(_.getString(0)) === Seq("name", "dept")) } protected def verifyNullabilityFailure(exception: AnalysisException): Unit protected def getDeltaLog(tableLocation: String): DeltaLog = { DeltaLog.forTable(spark, tableLocation) } testQuietly("create table with NOT NULL - check violation through file writing") { withTempDir { dir => withTable("delta_test") { sql(s""" |CREATE TABLE delta_test(a LONG, b String NOT NULL) |USING delta |OPTIONS('path'='${dir.getCanonicalPath}')""".stripMargin) val expectedSchema = new StructType() .add("a", LongType, nullable = true) .add("b", StringType, nullable = false) assert(spark.table("delta_test").schema === expectedSchema) val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("delta_test")) assert(table.location == makeQualifiedPath(dir.getAbsolutePath)) Seq((1L, "a")).toDF("a", "b") .write.format("delta").mode("append").save(table.location.toString) val read = spark.read.format("delta").load(table.location.toString) checkAnswer(read, Seq(Row(1L, "a"))) intercept[InvariantViolationException] { Seq((2L, null)).toDF("a", "b") .write.format("delta").mode("append").save(table.location.getPath) } } } } test("ALTER TABLE ADD COLUMNS with NOT NULL - not supported") { withTempDir { dir => val tableName = "delta_test_add_not_null" withTable(tableName) { sql(s""" |CREATE TABLE $tableName(a LONG) |USING delta |OPTIONS('path'='${dir.getCanonicalPath}')""".stripMargin) val expectedSchema = new StructType().add("a", LongType, nullable = true) assert(spark.table(tableName).schema === expectedSchema) val e = intercept[AnalysisException] { sql( s""" |ALTER TABLE $tableName |ADD COLUMNS (b String NOT NULL, c Int)""".stripMargin) } val msg = "`NOT NULL in ALTER TABLE ADD COLUMNS` is not supported for Delta tables" assert(e.getMessage.contains(msg)) } } } test("ALTER TABLE CHANGE COLUMN from nullable to NOT NULL - not supported") { withTempDir { dir => val tableName = "delta_test_from_nullable_to_not_null" withTable(tableName) { sql(s""" |CREATE TABLE $tableName(a LONG, b String) |USING delta |OPTIONS('path'='${dir.getCanonicalPath}')""".stripMargin) val expectedSchema = new StructType() .add("a", LongType, nullable = true) .add("b", StringType, nullable = true) assert(spark.table(tableName).schema === expectedSchema) val e = intercept[AnalysisException] { sql( s""" |ALTER TABLE $tableName |CHANGE COLUMN b b String NOT NULL""".stripMargin) } verifyNullabilityFailure(e) } } } test("ALTER TABLE CHANGE COLUMN from NOT NULL to nullable") { withTempDir { dir => val tableName = "delta_test_not_null_to_nullable" withTable(tableName) { sql( s""" |CREATE TABLE $tableName(a LONG NOT NULL, b String) |USING delta |OPTIONS('path'='${dir.getCanonicalPath}')""".stripMargin) val expectedSchema = new StructType() .add("a", LongType, nullable = false) .add("b", StringType, nullable = true) assert(spark.table(tableName).schema === expectedSchema) sql(s"INSERT INTO $tableName SELECT 1, 'a'") checkAnswer( sql(s"SELECT * FROM $tableName"), Seq(Row(1L, "a"))) sql( s""" |ALTER TABLE $tableName |ALTER COLUMN a DROP NOT NULL""".stripMargin) val expectedSchema2 = new StructType() .add("a", LongType, nullable = true) .add("b", StringType, nullable = true) assert(spark.table(tableName).schema === expectedSchema2) sql(s"INSERT INTO $tableName SELECT NULL, 'b'") checkAnswer( sql(s"SELECT * FROM $tableName"), Seq(Row(1L, "a"), Row(null, "b"))) } } } testQuietly("create table with NOT NULL - check violation through SQL") { withTempDir { dir => withTable("delta_test") { sql(s""" |CREATE TABLE delta_test(a LONG, b String NOT NULL) |USING delta |OPTIONS('path'='${dir.getCanonicalPath}')""".stripMargin) val expectedSchema = new StructType() .add("a", LongType, nullable = true) .add("b", StringType, nullable = false) assert(spark.table("delta_test").schema === expectedSchema) sql("INSERT INTO delta_test SELECT 1, 'a'") checkAnswer( sql("SELECT * FROM delta_test"), Seq(Row(1L, "a"))) val e = intercept[InvariantViolationException] { sql("INSERT INTO delta_test VALUES (2, null)") } if (!e.getMessage.contains("nullable values to non-null column")) { verifyInvariantViolationException(e) } } } } testQuietly("create table with NOT NULL in struct type - check violation") { withTempDir { dir => withTable("delta_test") { sql(s""" |CREATE TABLE delta_test |(x struct, y LONG) |USING delta |OPTIONS('path'='${dir.getCanonicalPath}')""".stripMargin) val expectedSchema = new StructType() .add("x", new StructType(). add("a", LongType, nullable = true) .add("b", StringType, nullable = false)) .add("y", LongType, nullable = true) assert(spark.table("delta_test").schema === expectedSchema) sql("INSERT INTO delta_test SELECT (1, 'a'), 1") checkAnswer( sql("SELECT * FROM delta_test"), Seq(Row(Row(1L, "a"), 1))) val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("delta_test")) assert(table.location == makeQualifiedPath(dir.getAbsolutePath)) val schema = new StructType() .add("x", new StructType() .add("a", "bigint") .add("b", "string")) .add("y", "bigint") val e = intercept[InvariantViolationException] { spark.createDataFrame( Seq(Row(Row(2L, null), 2L)).asJava, schema ).write.format("delta").mode("append").save(table.location.getPath) } verifyInvariantViolationException(e) } } } test("ALTER TABLE ADD COLUMNS with NOT NULL in struct type - not supported") { withTempDir { dir => val tableName = "delta_test_not_null_struct" withTable(tableName) { sql(s""" |CREATE TABLE $tableName |(y LONG) |USING delta |OPTIONS('path'='${dir.getCanonicalPath}')""".stripMargin) val expectedSchema = new StructType() .add("y", LongType, nullable = true) assert(spark.table(tableName).schema === expectedSchema) val e = intercept[AnalysisException] { sql( s""" |ALTER TABLE $tableName |ADD COLUMNS (x struct, z INT)""".stripMargin) } val msg = "Operation not allowed: " + "`NOT NULL in ALTER TABLE ADD COLUMNS` is not supported for Delta tables" assert(e.getMessage.contains(msg)) } } } test("ALTER TABLE ADD COLUMNS to table with existing NOT NULL fields") { withTempDir { dir => val tableName = "delta_test_existing_not_null" withTable(tableName) { sql( s""" |CREATE TABLE $tableName |(y LONG NOT NULL) |USING delta |OPTIONS('path'='${dir.getCanonicalPath}')""".stripMargin) val expectedSchema = new StructType() .add("y", LongType, nullable = false) assert(spark.table(tableName).schema === expectedSchema) sql( s""" |ALTER TABLE $tableName |ADD COLUMNS (x struct, z INT)""".stripMargin) val expectedSchema2 = new StructType() .add("y", LongType, nullable = false) .add("x", new StructType() .add("a", LongType) .add("b", StringType)) .add("z", IntegerType) assert(spark.table(tableName).schema === expectedSchema2) } } } /** * Covers adding and changing a nested field using the ALTER TABLE command. * @param initialColumnType Type of the single column used to create the initial test table. * @param fieldToAdd Tuple (name, type) of the nested field to add and change. * @param updatedColumnType Expected type of the single column after adding the nested field. */ def testAlterTableNestedFields(testName: String)( initialColumnType: String, fieldToAdd: (String, String), updatedColumnType: String): Unit = { // Remove spaces in test name so we can re-use it as a unique table name. val tableName = testName.replaceAll(" ", "") test(s"ALTER TABLE ADD/CHANGE COLUMNS - nested $testName") { withTempDir { dir => withTable(tableName) { sql( s""" |CREATE TABLE $tableName (data $initialColumnType) |USING delta |TBLPROPERTIES (${DeltaConfigs.COLUMN_MAPPING_MODE.key} = 'name') |OPTIONS('path'='${dir.getCanonicalPath}')""".stripMargin) val expectedInitialType = initialColumnType.filterNot(_.isWhitespace) val expectedUpdatedType = updatedColumnType.filterNot(_.isWhitespace) val fieldName = s"data.${fieldToAdd._1}" val fieldType = fieldToAdd._2 def columnType: DataFrame = sql(s"DESCRIBE TABLE $tableName") .where("col_name = 'data'") .select("data_type") checkAnswer(columnType, Row(expectedInitialType)) sql(s"ALTER TABLE $tableName ADD COLUMNS ($fieldName $fieldType)") checkAnswer(columnType, Row(expectedUpdatedType)) sql(s"ALTER TABLE $tableName CHANGE COLUMN $fieldName TYPE $fieldType") checkAnswer(columnType, Row(expectedUpdatedType)) } } } } testAlterTableNestedFields("struct in map key")( initialColumnType = "map, int>", fieldToAdd = "key.b" -> "string", updatedColumnType = "map, int>") testAlterTableNestedFields("struct in map value")( initialColumnType = "map>", fieldToAdd = "value.b" -> "string", updatedColumnType = "map>") testAlterTableNestedFields("struct in array")( initialColumnType = "array>", fieldToAdd = "element.b" -> "string", updatedColumnType = "array>") testAlterTableNestedFields("struct in nested map keys")( initialColumnType = "map, int>, int>", fieldToAdd = "key.key.b" -> "string", updatedColumnType = "map, int>, int>") testAlterTableNestedFields("struct in nested map values")( initialColumnType = "map>>", fieldToAdd = "value.value.b" -> "string", updatedColumnType = "map>>") testAlterTableNestedFields("struct in nested arrays")( initialColumnType = "array>>", fieldToAdd = "element.element.b" -> "string", updatedColumnType = "array>>") testAlterTableNestedFields("struct in nested array and map")( initialColumnType = "array>>", fieldToAdd = "element.value.b" -> "string", updatedColumnType = "array>>") testAlterTableNestedFields("struct in nested map key and array")( initialColumnType = "map>, int>", fieldToAdd = "key.element.b" -> "string", updatedColumnType = "map>, int>") testAlterTableNestedFields("struct in nested map value and array")( initialColumnType = "map>>", fieldToAdd = "value.element.b" -> "string", updatedColumnType = "map>>") test("ALTER TABLE CHANGE COLUMN with nullability change in struct type - not supported") { withTempDir { dir => val tableName = "not_supported_delta_test" withTable(tableName) { sql(s""" |CREATE TABLE $tableName |(x struct, y LONG) |USING delta |OPTIONS('path'='${dir.getCanonicalPath}')""".stripMargin) val expectedSchema = new StructType() .add("x", new StructType() .add("a", LongType) .add("b", StringType)) .add("y", LongType, nullable = true) assert(spark.table(tableName).schema === expectedSchema) val e1 = intercept[AnalysisException] { sql( s""" |ALTER TABLE $tableName |CHANGE COLUMN x x struct""".stripMargin) } assert(e1.getMessage.contains("Cannot update")) val e2 = intercept[AnalysisException] { sql( s""" |ALTER TABLE $tableName |CHANGE COLUMN x.b b String NOT NULL""".stripMargin) // this syntax may change } verifyNullabilityFailure(e2) } } } test("ALTER TABLE CHANGE COLUMN with nullability change in struct type - relaxed") { withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") { withTempDir { dir => val tblName = "delta_test2" withTable(tblName) { sql( s""" |CREATE TABLE $tblName |(x struct NOT NULL, y LONG) |USING delta |OPTIONS('path'='${dir.getCanonicalPath}')""".stripMargin) val expectedSchema = new StructType() .add("x", new StructType() .add("a", LongType) .add("b", StringType, nullable = false), nullable = false) .add("y", LongType) assert(spark.table(tblName).schema === expectedSchema) sql(s"INSERT INTO $tblName SELECT (1, 'a'), 1") checkAnswer( sql(s"SELECT * FROM $tblName"), Seq(Row(Row(1L, "a"), 1))) sql( s""" |ALTER TABLE $tblName |ALTER COLUMN x.b DROP NOT NULL""".stripMargin) // relax nullability sql(s"INSERT INTO $tblName SELECT (2, null), null") checkAnswer( sql(s"SELECT * FROM $tblName"), Seq( Row(Row(1L, "a"), 1), Row(Row(2L, null), null))) sql( s""" |ALTER TABLE $tblName |ALTER COLUMN x DROP NOT NULL""".stripMargin) sql(s"INSERT INTO $tblName SELECT null, 3") checkAnswer( sql(s"SELECT * FROM $tblName"), Seq( Row(Row(1L, "a"), 1), Row(Row(2L, null), null), Row(null, 3))) } } } } private def verifyInvariantViolationException(e: InvariantViolationException): Unit = { if (e == null) { fail("Didn't receive a InvariantViolationException.") } assert(e.getMessage.contains("NOT NULL constraint violated for column")) } test("ALTER TABLE RENAME TO") { withTable("tbl", "newTbl") { sql(s""" |CREATE TABLE tbl |USING delta |AS SELECT 1 as a, 'a' as b """.stripMargin) sql(s"ALTER TABLE tbl RENAME TO newTbl") checkDatasetUnorderly(sql("SELECT * FROM newTbl").as[(Long, String)], 1L -> "a") } } /** * Although Spark 3.2 adds the support for SHOW CREATE TABLE for v2 tables, it doesn't work * properly for Delta. For example, table properties, constraints and generated columns are not * showed properly. * * TODO Implement Delta's own ShowCreateTableCommand to show the Delta table definition correctly */ test("SHOW CREATE TABLE is not supported") { withTable("delta_test") { sql( s""" |CREATE TABLE delta_test(a LONG, b String) |USING delta """.stripMargin) val e = intercept[AnalysisException] { sql("SHOW CREATE TABLE delta_test").collect()(0).getString(0) } assert(e.message.contains("`SHOW CREATE TABLE` is not supported for Delta table")) } withTempDir { dir => withTable("delta_test") { val path = dir.getCanonicalPath() sql( s""" |CREATE TABLE delta_test(a LONG, b String) |USING delta |LOCATION '$path' """.stripMargin) val e = intercept[AnalysisException] { sql("SHOW CREATE TABLE delta_test").collect()(0).getString(0) } assert(e.message.contains("`SHOW CREATE TABLE` is not supported for Delta table")) } } } test("DESCRIBE TABLE for partitioned table") { withTempDir { dir => withTable("delta_test") { val path = dir.getCanonicalPath() val df = Seq( (1, "IT", "Alice"), (2, "CS", "Bob"), (3, "IT", "Carol")).toDF("id", "dept", "name") df.write.format("delta").partitionBy("name", "dept").save(path) sql(s"CREATE TABLE delta_test USING delta LOCATION '$path'") verifyDescribeTable("delta_test") verifyDescribeTable(s"delta.`$path`") assert(sql("DESCRIBE EXTENDED delta_test").collect().length > 0) } } } test("snapshot returned after a dropped managed table should be empty") { withTable("delta_test") { sql("CREATE TABLE delta_test USING delta AS SELECT 'foo' as a") val tableLocation = sql("DESC DETAIL delta_test").select("location").as[String].head() val snapshotBefore = getDeltaLog(tableLocation).update() sql("DROP TABLE delta_test") val snapshotAfter = getDeltaLog(tableLocation).update() assert(snapshotBefore ne snapshotAfter) assert(snapshotAfter.version === -1) } } test("snapshot returned after renaming a managed table should be empty") { val oldTableName = "oldTableName" val newTableName = "newTableName" withTable(oldTableName, newTableName) { sql(s"CREATE TABLE $oldTableName USING delta AS SELECT 'foo' as a") val tableLocation = sql(s"DESC DETAIL $oldTableName").select("location").as[String].head() val snapshotBefore = getDeltaLog(tableLocation).update() sql(s"ALTER TABLE $oldTableName RENAME TO $newTableName") val snapshotAfter = getDeltaLog(tableLocation).update() assert(snapshotBefore ne snapshotAfter) assert(snapshotAfter.version === -1) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaDDLUsingPathSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.actions.Protocol import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.hadoop.fs.Path import org.scalatest.Tag import org.apache.spark.SparkConf import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogV2Util, TableCatalog} import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.StructType import org.apache.spark.util.Utils trait DeltaDDLUsingPathTests extends QueryTest with SharedSparkSession with DeltaColumnMappingTestUtils { import testImplicits._ protected def catalogName: String = { CatalogManager.SESSION_CATALOG_NAME } protected def testUsingPath(command: String, tags: Tag*)(f: (String, String) => Unit): Unit = { test(s"$command - using path", tags: _*) { withTempDir { tempDir => withTable("delta_test") { val path = tempDir.getCanonicalPath Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("struct", struct((col("v1") * 10).as("x"), concat(col("v2"), col("v2")).as("y"))) .write .format("delta") .partitionBy("v1") .option("path", path) .saveAsTable("delta_test") f("`delta_test`", path) } } } test(s"$command - using path in 'delta' database", tags: _*) { withTempDir { tempDir => val path = tempDir.getCanonicalPath withDatabase("delta") { sql("CREATE DATABASE delta") withTable("delta.delta_test") { Seq((1, "a"), (2, "b")).toDF("v1", "v2") .withColumn("struct", struct((col("v1") * 10).as("x"), concat(col("v2"), col("v2")).as("y"))) .write .format("delta") .partitionBy("v1") .option("path", path) .saveAsTable("delta.delta_test") f("`delta`.`delta_test`", path) } } } } } protected def toQualifiedPath(path: String): String = { val hadoopPath = new Path(path) // scalastyle:off deltahadoopconfiguration val fs = hadoopPath.getFileSystem(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration fs.makeQualified(hadoopPath).toString } protected def checkDescribe(describe: String, keyvalues: (String, String)*): Unit = { val result = sql(describe).collect() keyvalues.foreach { case (key, value) => val row = result.find(_.getString(0) == key) assert(row.isDefined) if (key == "Location") { assert(toQualifiedPath(row.get.getString(1)) === toQualifiedPath(value)) } else { assert(row.get.getString(1) === value) } } } private def errorContains(errMsg: String, str: String): Unit = { assert(errMsg.contains(str)) } testUsingPath("SELECT") { (table, path) => Seq(table, s"delta.`$path`").foreach { tableOrPath => checkDatasetUnorderly( sql(s"SELECT * FROM $tableOrPath").as[(Int, String, (Int, String))], (1, "a", (10, "aa")), (2, "b", (20, "bb"))) checkDatasetUnorderly( spark.table(tableOrPath).as[(Int, String, (Int, String))], (1, "a", (10, "aa")), (2, "b", (20, "bb"))) } val ex = intercept[AnalysisException] { spark.table(s"delta.`/path/to/delta`") } assert(ex.getMessage.matches( ".*Path does not exist: (file:)?/path/to/delta.?.*"), "Found: " + ex.getMessage) withSQLConf(SQLConf.RUN_SQL_ON_FILES.key -> "false") { val ex = intercept[AnalysisException] { spark.table(s"delta.`/path/to/delta`") } assert(ex.getMessage.contains("Table or view not found: delta.`/path/to/delta`") || ex.getMessage.contains("table or view `delta`.`/path/to/delta` cannot be found")) } } testUsingPath("DESCRIBE TABLE") { (table, path) => val qualifiedPath = toQualifiedPath(path) Seq(table, s"delta.`$path`").foreach { tableOrPath => checkDescribe(s"DESCRIBE $tableOrPath", "v1" -> "int", "v2" -> "string", "struct" -> "struct") checkDescribe(s"DESCRIBE EXTENDED $tableOrPath", "v1" -> "int", "v2" -> "string", "struct" -> "struct", "Provider" -> "delta", "Location" -> qualifiedPath) } } testUsingPath("SHOW TBLPROPERTIES") { (table, path) => sql(s"ALTER TABLE $table SET TBLPROPERTIES " + "('delta.logRetentionDuration' = '2 weeks', 'key' = 'value')") val metadata = loadDeltaLog(path).snapshot.metadata Seq(table, s"delta.`$path`").foreach { tableOrPath => checkDatasetUnorderly( dropColumnMappingConfigurations( sql(s"SHOW TBLPROPERTIES $tableOrPath('delta.logRetentionDuration')") .as[(String, String)]), "delta.logRetentionDuration" -> "2 weeks") checkDatasetUnorderly( dropColumnMappingConfigurations( sql(s"SHOW TBLPROPERTIES $tableOrPath('key')").as[(String, String)]), "key" -> "value") } val protocol = Protocol.forNewTable(spark, Some(metadata)) val supportedFeatures = protocol .readerAndWriterFeatureNames .map(name => s"delta.feature.$name" -> "supported") val expectedProperties = Seq( "delta.logRetentionDuration" -> "2 weeks", "delta.minReaderVersion" -> protocol.minReaderVersion.toString, "delta.minWriterVersion" -> protocol.minWriterVersion.toString, "key" -> "value") ++ supportedFeatures checkDatasetUnorderly( dropColumnMappingConfigurations( sql(s"SHOW TBLPROPERTIES $table").as[(String, String)]), expectedProperties: _*) checkDatasetUnorderly( dropColumnMappingConfigurations( sql(s"SHOW TBLPROPERTIES delta.`$path`").as[(String, String)]), expectedProperties: _*) if (table == "`delta_test`") { val tableName = s"$catalogName.default.delta_test" checkDatasetUnorderly( dropColumnMappingConfigurations( sql(s"SHOW TBLPROPERTIES $table('dEltA.lOgrEteNtiOndURaTion')").as[(String, String)]), "dEltA.lOgrEteNtiOndURaTion" -> s"Table $tableName does not have property: dEltA.lOgrEteNtiOndURaTion") checkDatasetUnorderly( dropColumnMappingConfigurations( sql(s"SHOW TBLPROPERTIES $table('kEy')").as[(String, String)]), "kEy" -> s"Table $tableName does not have property: kEy") } else { checkDatasetUnorderly( dropColumnMappingConfigurations( sql(s"SHOW TBLPROPERTIES $table('kEy')").as[(String, String)]), "kEy" -> s"Table $catalogName.delta.delta_test does not have property: kEy") } checkDatasetUnorderly( dropColumnMappingConfigurations( sql(s"SHOW TBLPROPERTIES delta.`$path`('dEltA.lOgrEteNtiOndURaTion')") .as[(String, String)]), "dEltA.lOgrEteNtiOndURaTion" -> s"Table $catalogName.delta.`$path` does not have property: dEltA.lOgrEteNtiOndURaTion") checkDatasetUnorderly( dropColumnMappingConfigurations( sql(s"SHOW TBLPROPERTIES delta.`$path`('kEy')").as[(String, String)]), "kEy" -> s"Table $catalogName.delta.`$path` does not have property: kEy") val e = intercept[AnalysisException] { sql(s"SHOW TBLPROPERTIES delta.`/path/to/delta`").as[(String, String)] } assert(e.getMessage.contains(s"not a Delta table")) } testUsingPath("SHOW COLUMNS") { (table, path) => Seq(table, s"delta.`$path`").foreach { tableOrPath => checkDatasetUnorderly( sql(s"SHOW COLUMNS IN $tableOrPath").as[String], "v1", "v2", "struct") } if (table == "`delta_test`") { checkDatasetUnorderly( sql(s"SHOW COLUMNS IN $table").as[String], "v1", "v2", "struct") } else { checkDatasetUnorderly( sql(s"SHOW COLUMNS IN $table IN delta").as[String], "v1", "v2", "struct") } checkDatasetUnorderly( sql(s"SHOW COLUMNS IN `$path` IN delta").as[String], "v1", "v2", "struct") checkDatasetUnorderly( sql(s"SHOW COLUMNS IN delta.`$path` IN delta").as[String], "v1", "v2", "struct") val e = intercept[AnalysisException] { sql("SHOW COLUMNS IN delta.`/path/to/delta`") } assert(e.getMessage.contains(s"not a Delta table")) } testUsingPath("DESCRIBE COLUMN") { (table, path) => Seq(table, s"delta.`$path`").foreach { tableOrPath => checkDatasetUnorderly( sql(s"DESCRIBE $tableOrPath v1").as[(String, String)], "col_name" -> "v1", "data_type" -> "int", "comment" -> "NULL") checkDatasetUnorderly( sql(s"DESCRIBE $tableOrPath struct").as[(String, String)], "col_name" -> "struct", "data_type" -> "struct", "comment" -> "NULL") checkDatasetUnorderly( sql(s"DESCRIBE EXTENDED $tableOrPath v1").as[(String, String)], "col_name" -> "v1", "data_type" -> "int", "comment" -> "NULL" ) val ex1 = intercept[AnalysisException] { sql(s"DESCRIBE $tableOrPath unknown") } assert(ex1.getErrorClass() === "UNRESOLVED_COLUMN.WITH_SUGGESTION") val ex2 = intercept[AnalysisException] { sql(s"DESCRIBE $tableOrPath struct.x") } assert(ex2.getMessage.contains("DESC TABLE COLUMN does not support nested column: struct.x")) } val ex = intercept[AnalysisException] { sql("DESCRIBE delta.`/path/to/delta` v1") } assert(ex.getMessage.contains("not a Delta table"), s"Original message: ${ex.getMessage()}") } } class DeltaDDLUsingPathSuite extends DeltaDDLUsingPathTests with DeltaSQLCommandTest { } class DeltaDDLUsingPathNameColumnMappingSuite extends DeltaDDLUsingPathSuite with DeltaColumnMappingEnableNameMode { override protected def runOnlyTests = Seq( "create table with NOT NULL - check violation through file writing", "ALTER TABLE CHANGE COLUMN with nullability change in struct type - relaxed" ) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaDataFrameHadoopOptionsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.storage.LocalLogStore import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.hadoop.fs.Path import org.apache.spark.sql.QueryTest import org.apache.spark.sql.test.SharedSparkSession class DeltaDataFrameHadoopOptionsSuite extends QueryTest with DeltaSQLTestUtils with SharedSparkSession with DeltaSQLCommandTest { protected override def sparkConf = super.sparkConf.set("spark.delta.logStore.fake.impl", classOf[LocalLogStore].getName) /** * Create Hadoop file system options for `FakeFileSystem`. If Delta doesn't pick up them, * it won't be able to read/write any files using `fake://`. */ private def fakeFileSystemOptions: Map[String, String] = { Map( "fs.fake.impl" -> classOf[FakeFileSystem].getName, "fs.fake.impl.disable.cache" -> "true" ) } /** Create a fake file system path to test from the dir path. */ private def fakeFileSystemPath(dir: File): String = s"fake://${dir.getCanonicalPath}" /** Clear cache to make sure we don't reuse the cached snapshot */ private def clearCachedDeltaLogToForceReload(): Unit = { DeltaLog.clearCache() } // read/write parquet format check cache test("SC-86916: " + "read/write Delta paths using DataFrame should pick up Hadoop file system options") { withTempPath { dir => val path = fakeFileSystemPath(dir) spark.range(1, 10) .write .format("delta") .options(fakeFileSystemOptions) .save(path) clearCachedDeltaLogToForceReload() spark.read.format("delta").options(fakeFileSystemOptions).load(path).foreach(_ => {}) // Test time travel clearCachedDeltaLogToForceReload() spark.read.format("delta").options(fakeFileSystemOptions).load(path + "@v0").foreach(_ => {}) clearCachedDeltaLogToForceReload() spark.read.format("delta").options(fakeFileSystemOptions).option("versionAsOf", 0) .load(path).foreach(_ => {}) } } testQuietly("SC-86916: disabling the conf should not pick up Hadoop file system options") { withSQLConf(DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS.key -> "false") { withTempPath { dir => val path = fakeFileSystemPath(dir) intercept[Exception] { spark.read.format("delta").options(fakeFileSystemOptions).load(path) } } } } test("SC-86916: checkpoint should pick up Hadoop file system options") { withSQLConf(DeltaConfigs.CHECKPOINT_INTERVAL.defaultTablePropertyKey -> "1") { withTempPath { dir => val path = fakeFileSystemPath(dir) spark.range(1, 10).write.format("delta") .options(fakeFileSystemOptions) .mode("append") .save(path) spark.range(1, 10).write.format("delta") .options(fakeFileSystemOptions) .mode("append") .save(path) // Ensure we did write the checkpoint and read it back val deltaLog = DeltaLog.forTable(spark, new Path(path), fakeFileSystemOptions) assert(deltaLog.readLastCheckpointFile().get.version == 1) } } } test("SC-86916: invalidateCache should invalidate all DeltaLogs of the given path") { withTempPath { dir => val pathStr = fakeFileSystemPath(dir) val path = new Path(pathStr) spark.range(1, 10).write.format("delta") .options(fakeFileSystemOptions) .mode("append") .save(pathStr) val deltaLog = DeltaLog.forTable(spark, path, fakeFileSystemOptions) spark.range(1, 10).write.format("delta") .options(fakeFileSystemOptions) .mode("append") .save(pathStr) val cachedDeltaLog = DeltaLog.forTable(spark, path, fakeFileSystemOptions) assert(deltaLog eq cachedDeltaLog) withSQLConf(fakeFileSystemOptions.toSeq: _*) { DeltaLog.invalidateCache(spark, path) } spark.range(1, 10).write.format("delta") .options(fakeFileSystemOptions) .mode("append") .save(pathStr) val newDeltaLog = DeltaLog.forTable(spark, path, fakeFileSystemOptions) assert(deltaLog ne newDeltaLog) } } test("SC-86916: Delta log cache should respect options") { withTempPath { dir => val path = fakeFileSystemPath(dir) DeltaLog.clearCache() spark.range(1, 10).write.format("delta") .options(fakeFileSystemOptions) .mode("append") .save(path) assert(DeltaLog.cacheSize == 1) // Accessing the same table should not create a new entry in the cache spark.read.format("delta").options(fakeFileSystemOptions).load(path).foreach(_ => {}) assert(DeltaLog.cacheSize == 1) // Accessing the table with different options should create a new entry spark.read.format("delta") .options(fakeFileSystemOptions ++ Map("fs.foo" -> "foo")).load(path).foreach(_ => {}) assert(DeltaLog.cacheSize == 2) // Accessing the table without options should create a new entry withSQLConf(fakeFileSystemOptions.toSeq: _*) { spark.read.format("delta").load(path).foreach(_ => {}) } assert(DeltaLog.cacheSize == 3) // Make sure we don't break existing cache logic DeltaLog.clearCache() withSQLConf(fakeFileSystemOptions.toSeq: _*) { spark.read.format("delta").load(path).foreach(_ => {}) spark.read.format("delta").load(path).foreach(_ => {}) } assert(DeltaLog.cacheSize == 1) } } /** * Clears the DeltaLog cache, runs the operation, then verifies that * the resulting DeltaLog carries the expected fs.* options internally. */ private def withOptionsPropagationCheck(path: String, desc: String)(op: => Unit): Unit = { withClue(s"$desc: ") { clearCachedDeltaLogToForceReload() op val deltaLog = DeltaLog.forTable(spark, new Path(path), fakeFileSystemOptions) assert( deltaLog.options("fs.fake.impl") == classOf[FakeFileSystem].getName, "fs.fake.impl was not propagated to DeltaLog.options") assert( deltaLog.newDeltaHadoopConf().get("fs.fake.impl") == classOf[FakeFileSystem].getName, "fs.fake.impl was not propagated to Hadoop configuration") } } test("all operations should propagate Hadoop file system options") { withTempPaths(/* numPaths = */ 2) { case Seq(inputDir, checkpointDir) => val path = fakeFileSystemPath(inputDir) // Seed the table spark.range(2).write.format("delta") .options(fakeFileSystemOptions).save(path) withOptionsPropagationCheck(path, "batch write (overwrite)") { spark.range(3).write.format("delta") .options(fakeFileSystemOptions).mode("overwrite").save(path) } withOptionsPropagationCheck(path, "batch read") { assert(spark.read.format("delta") .options(fakeFileSystemOptions).load(path).count() == 3) } withOptionsPropagationCheck(path, "batch append") { spark.range(1).write.format("delta") .options(fakeFileSystemOptions).mode("append").save(path) } withOptionsPropagationCheck(path, "streaming read") { val query = spark.readStream.format("delta") .options(fakeFileSystemOptions) .load(path) .writeStream .format("memory") .queryName("options_propagation_test") .option("checkpointLocation", checkpointDir.getCanonicalPath) .start() try { query.processAllAvailable() assert(spark.table("options_propagation_test").count() == 4) } finally { query.stop() } } } } testQuietly("operations without Hadoop options should fail for fake:// filesystem") { withTempPaths(/* numPaths = */ 2) { case Seq(inputDir, checkpointDir) => val path = fakeFileSystemPath(inputDir) // Write data with options so the Delta table physically exists. spark.range(2).write.format("delta") .options(fakeFileSystemOptions).save(path) clearCachedDeltaLogToForceReload() // Batch read without options should fail val batchEx = intercept[Exception] { spark.read.format("delta").load(path).foreach(_ => {}) } assert(batchEx.getMessage.contains("""No FileSystem for scheme "fake"""")) clearCachedDeltaLogToForceReload() // Streaming read without options should fail val streamEx = intercept[Exception] { spark.readStream.format("delta") .load(path) .writeStream .format("memory") .queryName("options_failure_test") .option("checkpointLocation", checkpointDir.getCanonicalPath) .start() .processAllAvailable() } assert(streamEx.getMessage.contains("""No FileSystem for scheme "fake"""")) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaDataFrameWriterV2Suite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.actions.{Protocol, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.catalog.{DeltaCatalog, DeltaTableV2} import org.apache.spark.sql.delta.test.{DeltaColumnMappingSelectedTestMixin, DeltaSQLCommandTest} import org.scalatest.BeforeAndAfter import org.apache.spark.SparkConf import org.apache.spark.sql.{AnalysisException, CreateTableWriter, Dataset, QueryTest, Row} import org.apache.spark.sql.catalyst.analysis.{CannotReplaceMissingTableException, TableAlreadyExistsException} import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogV2Util, Identifier, Table, TableCatalog} import org.apache.spark.sql.connector.expressions._ import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{LongType, StringType, StructType} import org.apache.spark.util.Utils // These tests are copied from Apache Spark (minus partition by expressions) and should work exactly // the same with Delta minus some writer options trait OpenSourceDataFrameWriterV2Tests extends QueryTest with SharedSparkSession with BeforeAndAfter { import testImplicits._ before { val df = spark.createDataFrame(Seq((1L, "a"), (2L, "b"), (3L, "c"))).toDF("id", "data") df.createOrReplaceTempView("source") val df2 = spark.createDataFrame(Seq((4L, "d"), (5L, "e"), (6L, "f"))).toDF("id", "data") df2.createOrReplaceTempView("source2") } after { spark.sessionState.catalog.listTables("default").foreach { ti => spark.sessionState.catalog.dropTable(ti, ignoreIfNotExists = false, purge = false) } } def catalog: TableCatalog = { spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog] } protected def catalogPrefix: String = { s"${CatalogManager.SESSION_CATALOG_NAME}." } protected def getProperties(table: Table): Map[String, String] = { table.properties().asScala.toMap .filterKeys(!CatalogV2Util.TABLE_RESERVED_PROPERTIES.contains(_)) .filterKeys(!TableFeatureProtocolUtils.isTableProtocolProperty(_)) .toMap } test("Append: basic append") { spark.sql("CREATE TABLE table_name (id bigint, data string) USING delta") checkAnswer(spark.table("table_name"), Seq.empty) spark.table("source").writeTo("table_name").append() checkAnswer( spark.table("table_name"), Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"))) spark.table("source2").writeTo("table_name").append() checkAnswer( spark.table("table_name"), Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"), Row(4L, "d"), Row(5L, "e"), Row(6L, "f"))) } test("Append: by name not position") { spark.sql("CREATE TABLE table_name (id bigint, data string) USING delta") checkAnswer(spark.table("table_name"), Seq.empty) val exc = intercept[AnalysisException] { spark.table("source").withColumnRenamed("data", "d").writeTo("table_name").append() } assert(exc.getMessage.contains("schema mismatch")) checkAnswer( spark.table("table_name"), Seq()) } test("Append: fail if table does not exist") { val exc = intercept[AnalysisException] { spark.table("source").writeTo("table_name").append() } assert(exc.getMessage.contains("table_name")) } test("Overwrite: overwrite by expression: true") { spark.sql( "CREATE TABLE table_name (id bigint, data string) USING delta PARTITIONED BY (id)") checkAnswer(spark.table("table_name"), Seq.empty) spark.table("source").writeTo("table_name").append() checkAnswer( spark.table("table_name"), Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"))) spark.table("source2").writeTo("table_name").overwrite(lit(true)) checkAnswer( spark.table("table_name"), Seq(Row(4L, "d"), Row(5L, "e"), Row(6L, "f"))) } test("Overwrite: overwrite by expression: id = 3") { spark.sql( "CREATE TABLE table_name (id bigint, data string) USING delta PARTITIONED BY (id)") checkAnswer(spark.table("table_name"), Seq.empty) spark.table("source").writeTo("table_name").append() checkAnswer( spark.table("table_name"), Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"))) val e = intercept[AnalysisException] { spark.table("source2").writeTo("table_name").overwrite($"id" === 3) } assert(e.getErrorClass == "DELTA_REPLACE_WHERE_MISMATCH") assert(e.getMessage.startsWith( "[DELTA_REPLACE_WHERE_MISMATCH] Written data does not conform to partial table overwrite " + "condition or constraint")) checkAnswer( spark.table("table_name"), Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"))) } test("Overwrite: by name not position") { spark.sql("CREATE TABLE table_name (id bigint, data string) USING delta") checkAnswer(spark.table("table_name"), Seq.empty) val exc = intercept[AnalysisException] { spark.table("source").withColumnRenamed("data", "d") .writeTo("table_name").overwrite(lit(true)) } assert(exc.getMessage.contains("schema mismatch")) checkAnswer( spark.table("table_name"), Seq()) } test("Overwrite: fail if table does not exist") { val exc = intercept[AnalysisException] { spark.table("source").writeTo("table_name").overwrite(lit(true)) } assert(exc.getMessage.contains("table_name")) } test("OverwritePartitions: overwrite conflicting partitions") { spark.sql( "CREATE TABLE table_name (id bigint, data string) USING delta PARTITIONED BY (id)") checkAnswer(spark.table("table_name"), Seq.empty) spark.table("source").writeTo("table_name").append() checkAnswer( spark.table("table_name"), Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"))) spark.table("source2").withColumn("id", $"id" - 2) .writeTo("table_name").overwritePartitions() checkAnswer( spark.table("table_name"), Seq(Row(1L, "a"), Row(2L, "d"), Row(3L, "e"), Row(4L, "f"))) } test("OverwritePartitions: overwrite all rows if not partitioned") { spark.sql("CREATE TABLE table_name (id bigint, data string) USING delta") checkAnswer(spark.table("table_name"), Seq.empty) spark.table("source").writeTo("table_name").append() checkAnswer( spark.table("table_name"), Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"))) spark.table("source2").writeTo("table_name").overwritePartitions() checkAnswer( spark.table("table_name"), Seq(Row(4L, "d"), Row(5L, "e"), Row(6L, "f"))) } test("OverwritePartitions: by name not position") { spark.sql("CREATE TABLE table_name (id bigint, data string) USING delta") checkAnswer(spark.table("table_name"), Seq.empty) val e = intercept[AnalysisException] { spark.table("source").withColumnRenamed("data", "d") .writeTo("table_name").overwritePartitions() } assert(e.getMessage.contains("schema mismatch")) checkAnswer( spark.table("table_name"), Seq()) } test("OverwritePartitions: fail if table does not exist") { val exc = intercept[AnalysisException] { spark.table("source").writeTo("table_name").overwritePartitions() } assert(exc.getMessage.contains("table_name")) } test("Create: basic behavior") { spark.table("source").writeTo("table_name").using("delta").create() checkAnswer( spark.table("table_name"), Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"))) val table = catalog.loadTable(Identifier.of(Array("default"), "table_name")) assert(table.name === s"${catalogPrefix}default.table_name") assert(table.schema === new StructType().add("id", LongType).add("data", StringType)) assert(table.partitioning.isEmpty) assert(getProperties(table).isEmpty) } test("Create: with using") { spark.table("source").writeTo("table_name").using("delta").create() checkAnswer( spark.table("table_name"), Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"))) val table = catalog.loadTable(Identifier.of(Array("default"), "table_name")) assert(table.name === s"${catalogPrefix}default.table_name") assert(table.schema === new StructType().add("id", LongType).add("data", StringType)) assert(table.partitioning.isEmpty) assert(getProperties(table).isEmpty) } test("Create: with property") { spark.table("source").writeTo("table_name") .tableProperty("prop", "value").using("delta").create() checkAnswer( spark.table("table_name"), Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"))) val table = catalog.loadTable(Identifier.of(Array("default"), "table_name")) assert(table.name === s"${catalogPrefix}default.table_name") assert(table.schema === new StructType().add("id", LongType).add("data", StringType)) assert(table.partitioning.isEmpty) assert(getProperties(table) === Map("prop" -> "value")) } test("Create: identity partitioned table") { spark.table("source").writeTo("table_name").using("delta").partitionedBy($"id").create() checkAnswer( spark.table("table_name"), Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"))) val table = catalog.loadTable(Identifier.of(Array("default"), "table_name")) assert(table.name === s"${catalogPrefix}default.table_name") assert(table.schema === new StructType().add("id", LongType).add("data", StringType)) assert(table.partitioning === Seq(IdentityTransform(FieldReference("id")))) assert(getProperties(table).isEmpty) } test("Create: fail if table already exists") { spark.sql( "CREATE TABLE table_name (id bigint, data string) USING delta PARTITIONED BY (id)") val exc = intercept[TableAlreadyExistsException] { spark.table("source").writeTo("table_name").using("delta").create() } assert(exc.getMessage.contains("table_name")) val table = catalog.loadTable(Identifier.of(Array("default"), "table_name")) // table should not have been changed assert(table.name === s"${catalogPrefix}default.table_name") assert(table.schema === new StructType().add("id", LongType).add("data", StringType)) assert(table.partitioning === Seq(IdentityTransform(FieldReference("id")))) assert(getProperties(table).isEmpty) } test("Replace: basic behavior") { spark.sql( "CREATE TABLE table_name (id bigint, data string) USING delta PARTITIONED BY (id)") spark.sql("INSERT INTO TABLE table_name SELECT * FROM source") checkAnswer( spark.table("table_name"), Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"))) val table = catalog.loadTable(Identifier.of(Array("default"), "table_name")) // validate the initial table assert(table.name === s"${catalogPrefix}default.table_name") assert(table.schema === new StructType().add("id", LongType).add("data", StringType)) assert(table.partitioning === Seq(IdentityTransform(FieldReference("id")))) assert(getProperties(table).isEmpty) spark.table("source2") .withColumn("even_or_odd", when(($"id" % 2) === 0, "even").otherwise("odd")) .writeTo("table_name").using("delta") .tableProperty("deLta.aPpeNdonly", "true").replace() checkAnswer( spark.table("table_name"), Seq(Row(4L, "d", "even"), Row(5L, "e", "odd"), Row(6L, "f", "even"))) val replaced = catalog.loadTable(Identifier.of(Array("default"), "table_name")) // validate the replacement table assert(replaced.name === s"${catalogPrefix}default.table_name") assert(replaced.schema === new StructType() .add("id", LongType) .add("data", StringType) .add("even_or_odd", StringType)) assert(replaced.partitioning.isEmpty) assert(getProperties(replaced) === Map("delta.appendOnly" -> "true")) } test("Replace: partitioned table") { spark.sql("CREATE TABLE table_name (id bigint, data string) USING delta") spark.sql("INSERT INTO TABLE table_name SELECT * FROM source") checkAnswer( spark.table("table_name"), Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"))) val table = catalog.loadTable(Identifier.of(Array("default"), "table_name")) // validate the initial table assert(table.name === s"${catalogPrefix}default.table_name") assert(table.schema === new StructType().add("id", LongType).add("data", StringType)) assert(table.partitioning.isEmpty) assert(getProperties(table).isEmpty) spark.table("source2") .withColumn("even_or_odd", when(($"id" % 2) === 0, "even").otherwise("odd")) .writeTo("table_name").using("delta") .partitionedBy($"id") .replace() checkAnswer( spark.table("table_name"), Seq(Row(4L, "d", "even"), Row(5L, "e", "odd"), Row(6L, "f", "even"))) val replaced = catalog.loadTable(Identifier.of(Array("default"), "table_name")) // validate the replacement table assert(replaced.name === s"${catalogPrefix}default.table_name") assert(replaced.schema === new StructType() .add("id", LongType) .add("data", StringType) .add("even_or_odd", StringType)) assert(replaced.partitioning === Seq(IdentityTransform(FieldReference("id")))) assert(getProperties(replaced).isEmpty) } test("Replace: fail if table does not exist") { val exc = intercept[AnalysisException] { spark.table("source").writeTo("table_name").using("delta").replace() } checkError(exc, "TABLE_OR_VIEW_NOT_FOUND", Some("42P01"), Map("relationName" -> "`default`.`table_name`")) } test("CreateOrReplace: table does not exist") { spark.table("source2").writeTo("table_name").using("delta").createOrReplace() checkAnswer( spark.table("table_name"), Seq(Row(4L, "d"), Row(5L, "e"), Row(6L, "f"))) val replaced = catalog.loadTable(Identifier.of(Array("default"), "table_name")) // validate the replacement table assert(replaced.name === s"${catalogPrefix}default.table_name") assert(replaced.schema === new StructType().add("id", LongType).add("data", StringType)) assert(replaced.partitioning.isEmpty) assert(getProperties(replaced).isEmpty) } test("CreateOrReplace: table exists") { spark.sql( "CREATE TABLE table_name (id bigint, data string) USING delta PARTITIONED BY (id)") spark.sql("INSERT INTO TABLE table_name SELECT * FROM source") checkAnswer( spark.table("table_name"), Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"))) val table = catalog.loadTable(Identifier.of(Array("default"), "table_name")) // validate the initial table assert(table.name === s"${catalogPrefix}default.table_name") assert(table.schema === new StructType().add("id", LongType).add("data", StringType)) assert(table.partitioning === Seq(IdentityTransform(FieldReference("id")))) assert(getProperties(table).isEmpty) spark.table("source2") .withColumn("even_or_odd", when(($"id" % 2) === 0, "even").otherwise("odd")) .writeTo("table_name").using("delta").createOrReplace() checkAnswer( spark.table("table_name"), Seq(Row(4L, "d", "even"), Row(5L, "e", "odd"), Row(6L, "f", "even"))) val replaced = catalog.loadTable(Identifier.of(Array("default"), "table_name")) // validate the replacement table assert(replaced.name === s"${catalogPrefix}default.table_name") assert(replaced.schema === new StructType() .add("id", LongType) .add("data", StringType) .add("even_or_odd", StringType)) assert(replaced.partitioning.isEmpty) assert(getProperties(replaced).isEmpty) } test("Create: partitioned by years(ts) - not supported") { val e = intercept[AnalysisException] { spark.table("source") .withColumn("ts", lit("2019-06-01 10:00:00.000000").cast("timestamp")) .writeTo("table_name") .partitionedBy(years($"ts")) .using("delta") .create() } assert(e.getMessage.contains("Partitioning by expressions")) } test("Create: partitioned by months(ts) - not supported") { val e = intercept[AnalysisException] { spark.table("source") .withColumn("ts", lit("2019-06-01 10:00:00.000000").cast("timestamp")) .writeTo("table_name") .partitionedBy(months($"ts")) .using("delta") .create() } assert(e.getMessage.contains("Partitioning by expressions")) } test("Create: partitioned by days(ts) - not supported") { val e = intercept[AnalysisException] { spark.table("source") .withColumn("ts", lit("2019-06-01 10:00:00.000000").cast("timestamp")) .writeTo("table_name") .partitionedBy(days($"ts")) .using("delta") .create() } assert(e.getMessage.contains("Partitioning by expressions")) } test("Create: partitioned by hours(ts) - not supported") { val e = intercept[AnalysisException] { spark.table("source") .withColumn("ts", lit("2019-06-01 10:00:00.000000").cast("timestamp")) .writeTo("table_name") .partitionedBy(hours($"ts")) .using("delta") .create() } assert(e.getMessage.contains("Partitioning by expressions")) } test("Create: partitioned by bucket(4, id) - not supported") { val e = intercept[AnalysisException] { spark.table("source") .writeTo("table_name") .partitionedBy(bucket(4, $"id")) .using("delta") .create() } assert(e.getMessage.contains("is not supported for Delta tables")) } } class DeltaDataFrameWriterV2Suite extends OpenSourceDataFrameWriterV2Tests with DeltaSQLCommandTest { import testImplicits._ test("Append: basic append by path") { spark.sql("CREATE TABLE table_name (id bigint, data string) USING delta") checkAnswer(spark.table("table_name"), Seq.empty) val location = catalog.loadTable(Identifier.of(Array("default"), "table_name")) .asInstanceOf[DeltaTableV2].path spark.table("source").writeTo(s"delta.`$location`").append() checkAnswer( spark.table(s"delta.`$location`"), Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"))) // allows missing columns Seq(4L).toDF("id").writeTo(s"delta.`$location`").append() checkAnswer( spark.table(s"delta.`$location`"), Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"), Row(4L, null))) } test("Create: basic behavior by path") { withTempDir { tempDir => val dir = tempDir.getCanonicalPath spark.table("source").writeTo(s"delta.`$dir`").using("delta").create() checkAnswer( spark.read.format("delta").load(dir), Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"))) val table = catalog.loadTable(Identifier.of(Array("delta"), dir)) assert(table.name === s"delta.`file:$dir`") assert(table.schema === new StructType().add("id", LongType).add("data", StringType)) assert(table.partitioning.isEmpty) assert(getProperties(table).isEmpty) } } test("Create: using empty dataframe") { spark.table("source").where("false") .writeTo("table_name").using("delta") .tableProperty("delta.appendOnly", "true") .partitionedBy($"id").create() checkAnswer(spark.table("table_name"), Seq.empty[Row]) val table = catalog.loadTable(Identifier.of(Array("default"), "table_name")) assert(table.name === s"${catalogPrefix}default.table_name") assert(table.schema === new StructType().add("id", LongType).add("data", StringType)) assert(table.partitioning === Seq(IdentityTransform(FieldReference("id")))) assert(getProperties(table) === Map("delta.appendOnly" -> "true")) } test("Replace: basic behavior using empty df") { spark.sql( "CREATE TABLE table_name (id bigint, data string) USING delta PARTITIONED BY (id)") spark.sql("INSERT INTO TABLE table_name SELECT * FROM source") checkAnswer( spark.table("table_name"), Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"))) val table = catalog.loadTable(Identifier.of(Array("default"), "table_name")) // validate the initial table assert(table.name === s"${catalogPrefix}default.table_name") assert(table.schema === new StructType().add("id", LongType).add("data", StringType)) assert(table.partitioning === Seq(IdentityTransform(FieldReference("id")))) assert(getProperties(table).isEmpty) spark.table("source2").where("false") .withColumn("even_or_odd", when(($"id" % 2) === 0, "even").otherwise("odd")) .writeTo("table_name").using("delta") .tableProperty("deLta.aPpeNdonly", "true").replace() checkAnswer( spark.table("table_name"), Seq.empty[Row]) val replaced = catalog.loadTable(Identifier.of(Array("default"), "table_name")) // validate the replacement table assert(replaced.name === s"${catalogPrefix}default.table_name") assert(replaced.schema === new StructType() .add("id", LongType) .add("data", StringType) .add("even_or_odd", StringType)) assert(replaced.partitioning.isEmpty) assert(getProperties(replaced) === Map("delta.appendOnly" -> "true")) } test("throw error with createOrReplace and Replace if overwriteSchema=false") { spark.sql( "CREATE TABLE table_name (id bigint, data string) USING delta PARTITIONED BY (id)") spark.sql("INSERT INTO TABLE table_name SELECT * FROM source") checkAnswer( spark.table("table_name"), Seq(Row(1L, "a"), Row(2L, "b"), Row(3L, "c"))) def checkFailure( df: Dataset[_], errorMsg: String)( f: CreateTableWriter[_] => CreateTableWriter[_]): Unit = { val e = intercept[IllegalArgumentException] { val dfwV2 = df.writeTo("table_name") .using("delta") .option("overwriteSchema", "false") f(dfwV2).replace() } assert(e.getMessage.contains(errorMsg)) val e2 = intercept[IllegalArgumentException] { val dfwV2 = df.writeTo("table_name") .using("delta") .option("overwriteSchema", "false") f(dfwV2).createOrReplace() } assert(e2.getMessage.contains(errorMsg)) } // schema changes checkFailure( spark.table("table_name").withColumn("id2", 'id + 1), "overwriteSchema is not allowed when replacing")(a => a.partitionedBy($"id")) // partitioning changes // did not specify partitioning checkFailure(spark.table("table_name"), "overwriteSchema is not allowed when replacing")(a => a) // different partitioning column checkFailure(spark.table("table_name"), "overwriteSchema is not allowed when replacing")(a => a.partitionedBy($"data")) // different table Properties checkFailure(spark.table("table_name"), "overwriteSchema is not allowed when replacing")(a => a.partitionedBy($"id").tableProperty("delta.appendOnly", "true")) } test("append or overwrite mode should not do implicit casting") { val table = "not_implicit_casting" withTable(table) { spark.sql(s"CREATE TABLE $table(id bigint, p int) USING delta PARTITIONED BY (p)") def verifyNotImplicitCasting(f: => Unit): Unit = { val e = intercept[DeltaAnalysisException](f) checkError( e.getCause.asInstanceOf[DeltaAnalysisException], "DELTA_MERGE_INCOMPATIBLE_DATATYPE", parameters = Map("currentDataType" -> "LongType", "updateDataType" -> "IntegerType")) } verifyNotImplicitCasting { Seq(1 -> 1).toDF("id", "p").write.mode("append").format("delta").saveAsTable(table) } verifyNotImplicitCasting { Seq(1 -> 1).toDF("id", "p").write.mode("overwrite").format("delta").saveAsTable(table) } verifyNotImplicitCasting { Seq(1 -> 1).toDF("id", "p").writeTo(table).append() } verifyNotImplicitCasting { Seq(1 -> 1).toDF("id", "p").writeTo(table).overwrite($"p" === 1) } verifyNotImplicitCasting { Seq(1 -> 1).toDF("id", "p").writeTo(table).overwritePartitions() } } } test("append or overwrite mode allows missing columns") { val table = "allow_missing_columns" withTable(table) { spark.sql( s"CREATE TABLE $table(col1 int, col2 int, col3 int) USING delta PARTITIONED BY (col3)") // append Seq((0, 10)).toDF("col1", "col3").writeTo(table).append() checkAnswer( spark.table(table), Seq(Row(0, null, 10)) ) // overwrite by expression Seq((1, 11)).toDF("col1", "col3").writeTo(table).overwrite($"col3" === 11) checkAnswer( spark.table(table), Seq(Row(0, null, 10), Row(1, null, 11)) ) // dynamic partition overwrite Seq((2, 10)).toDF("col1", "col3").writeTo(table).overwritePartitions() checkAnswer( spark.table(table), Seq(Row(2, null, 10), Row(1, null, 11)) ) } } } trait DeltaDataFrameWriterV2ColumnMappingSuiteBase extends DeltaColumnMappingSelectedTestMixin { override protected def runOnlyTests = Seq( "Append: basic append", "Create: with using", "Overwrite: overwrite by expression: true", "Replace: partitioned table" ) } class DeltaDataFrameWriterV2IdColumnMappingSuite extends DeltaDataFrameWriterV2Suite with DeltaColumnMappingEnableIdMode with DeltaDataFrameWriterV2ColumnMappingSuiteBase { override protected def getProperties(table: Table): Map[String, String] = { // ignore column mapping configurations dropColumnMappingConfigurations(super.getProperties(table)) } } class DeltaDataFrameWriterV2NameColumnMappingSuite extends DeltaDataFrameWriterV2Suite with DeltaColumnMappingEnableNameMode with DeltaDataFrameWriterV2ColumnMappingSuiteBase { override protected def getProperties(table: Table): Map[String, String] = { // ignore column mapping configurations dropColumnMappingConfigurations(super.getProperties(table)) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaDropColumnSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.SparkConf import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StringType, StructType} class DeltaDropColumnSuite extends QueryTest with DeltaArbitraryColumnNameSuiteBase { override protected val sparkConf: SparkConf = super.sparkConf.set(DeltaSQLConf.DELTA_ALTER_TABLE_DROP_COLUMN_ENABLED.key, "true") protected def dropTest( testName: String, testTags: org.scalatest.Tag*)( f: ((String, Seq[String]) => Unit) => Unit): Unit = { test(testName, testTags: _*) { def drop(table: String, columns: Seq[String]): Unit = sql(s"alter table $table drop column (${columns.mkString(",")})") f(drop) } } dropTest("drop column disallowed with sql flag off") { drop => withSQLConf(DeltaSQLConf.DELTA_ALTER_TABLE_DROP_COLUMN_ENABLED.key -> "false") { withTable("t1") { createTableWithSQLAPI("t1", simpleNestedData, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name")) assertException("DROP COLUMN is not supported for your Delta table") { drop("t1", "arr" :: Nil) } } } } dropTest("drop column disallowed with no mapping mode") { drop => withTable("t1") { createTableWithSQLAPI("t1", simpleNestedData) assertException("DROP COLUMN is not supported for your Delta table") { drop("t1", "arr" :: Nil) } } } dropTest("drop column - basic") { drop => withTable("t1") { createTableWithSQLAPI("t1", simpleNestedData, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name")) // drop single column drop("t1", "arr" :: Nil) checkAnswer(spark.table("t1"), simpleNestedData.drop("arr")) // drop multiple columns drop("t1", "a" :: "b.c" :: Nil) checkAnswer(spark.table("t1"), Seq( Row(Row(1), Map("k1" -> "v1")), Row(Row(2), Map("k2" -> "v2")))) // check delta history checkAnswer( spark.sql("describe history t1") .select("operation", "operationParameters") .where("version = 3"), Seq(Row("DROP COLUMNS", Map("columns" -> """["a","b.c"]""")))) } } dropTest("drop column - basic - path based table") { drop => withTempDir { dir => simpleNestedData.write.mode("overwrite").format("delta").save(dir.getCanonicalPath) alterTableWithProps(s"delta.`${dir.getCanonicalPath}`", Map( DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name", DeltaConfigs.MIN_READER_VERSION.key -> "2", DeltaConfigs.MIN_WRITER_VERSION.key -> "5")) // drop single column drop(s"delta.`${dir.getCanonicalPath}`", "arr" :: Nil) checkAnswer(spark.read.format("delta").load(dir.getCanonicalPath), simpleNestedData.drop("arr")) } } dropTest("dropped columns can no longer be queried") { drop => withTable("t1") { createTableWithSQLAPI("t1", simpleNestedData, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name")) drop("t1", "a" :: "b.c" :: "arr" :: Nil) // dropped column cannot be queried anymore val err1 = intercept[AnalysisException] { spark.table("t1").where("a = 'str1'").collect() }.getMessage assert( err1.contains("cannot be resolved") || err1.contains("Column 'a' does not exist") || err1.contains("cannot resolve")) val err2 = intercept[AnalysisException] { spark.table("t1").select("min(a)").collect() }.getMessage assert( err2.contains("cannot be resolved") || err2.contains("Column '`min(a)`' does not exist") || err2.contains("cannot resolve")) } } dropTest("drop column - corner cases") { drop => withTable("t1") { createTableWithSQLAPI("t1", simpleNestedData, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name")) drop("t1", "a" :: "b.c" :: "arr" :: Nil) // cannot drop the last nested field val e = intercept[AnalysisException] { drop("t1", "b.d" :: Nil) } assert(e.getMessage.contains("Cannot drop column from a schema with a single column")) // can drop the parent column drop("t1", "b" :: Nil) // cannot drop the last top-level field val e2 = intercept[AnalysisException] { drop("t1", "map" :: Nil) } assert(e2.getMessage.contains("Cannot drop column from a schema with a single column")) spark.sql("alter table t1 add column (e struct)") // can drop a column with arbitrary chars spark.sql(s"alter table t1 rename column map to `${colName("map")}`") drop("t1", s"`${colName("map")}`" :: Nil) // only column e is left now assert(spark.table("t1").schema.map(_.name) == Seq("e")) // can drop a nested column when the top-level column is the only column drop("t1", "e.e1" :: Nil) val resultSchema = spark.table("t1").schema assert(resultSchema.findNestedField("e" :: "e2" :: Nil).isDefined) assert(resultSchema.findNestedField("e" :: "e1" :: Nil).isEmpty) } } dropTest("drop column with constraints") { drop => withTable("t1") { val schemaWithNotNull = simpleNestedData.schema.toDDL.replace("c: STRING", "c: STRING NOT NULL") withTable("source") { spark.sql( s""" |CREATE TABLE t1 ($schemaWithNotNull) |USING DELTA |${propString(Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name"))} |""".stripMargin) simpleNestedData.write.format("delta").mode("append").saveAsTable("t1") } spark.sql("alter table t1 add constraint rangeABC check (concat(a, a) > 'str')") spark.sql("alter table t1 add constraint rangeBD check (`b`.`d` > 0)") spark.sql("alter table t1 add constraint arrValue check (arr[0] > 0)") assertException("Cannot alter column a because this column is referenced by") { drop("t1", "a" :: Nil) } assertException("Cannot alter column arr because this column is referenced by") { drop("t1", "arr" :: Nil) } // cannot drop b because its child is referenced assertException("Cannot alter column b because this column is referenced by") { drop("t1", "b" :: Nil) } // can still drop b.c because it's referenced by a null constraint drop("t1", "b.c" :: Nil) // this is a safety flag - it won't error when you turn it off withSQLConf(DeltaSQLConf.DELTA_ALTER_TABLE_CHANGE_COLUMN_CHECK_EXPRESSIONS.key -> "false") { drop("t1", "b" :: "arr" :: Nil) } } } test("drop column with constraints - map element") { def drop(table: String, columns: Seq[String]): Unit = sql(s"alter table $table drop column (${columns.mkString(",")})") withTable("t1") { val schemaWithNotNull = simpleNestedData.schema.toDDL.replace("c: STRING", "c: STRING NOT NULL") withTable("source") { spark.sql( s""" |CREATE TABLE t1 ($schemaWithNotNull) |USING DELTA |${propString(Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name"))} |""".stripMargin) simpleNestedData.write.format("delta").mode("append").saveAsTable("t1") } spark.sql("alter table t1 add constraint" + " mapValue check (not array_contains(map_keys(map), 'k1') or map['k1'] = 'v1')") assertException("Cannot alter column map because this column is referenced by") { drop("t1", "map" :: Nil) } } } dropTest("drop with generated column") { drop => withTable("t1") { withSQLConf(DeltaSQLConf.DELTA_ALTER_TABLE_DROP_COLUMN_ENABLED.key -> "true") { val tableBuilder = io.delta.tables.DeltaTable.create(spark).tableName("t1") tableBuilder.property("delta.columnMapping.mode", "name") // add existing columns simpleNestedSchema.map(field => (field.name, field.dataType)).foreach(col => { val (colName, dataType) = col val columnBuilder = io.delta.tables.DeltaTable.columnBuilder(spark, colName) columnBuilder.dataType(dataType.sql) tableBuilder.addColumn(columnBuilder.build()) }) // add generated columns val genCol1 = io.delta.tables.DeltaTable.columnBuilder(spark, "genCol1") .dataType("int") .generatedAlwaysAs("length(a)") .build() val genCol2 = io.delta.tables.DeltaTable.columnBuilder(spark, "genCol2") .dataType("int") .generatedAlwaysAs("b.d * 100 + arr[0]") .build() tableBuilder .addColumn(genCol1) .addColumn(genCol2) .execute() simpleNestedData.write.format("delta").mode("append").saveAsTable("t1") assertException("Cannot alter column a because this column is referenced by") { drop("t1", "a" :: Nil) } assertException("Cannot alter column b because this column is referenced by") { drop("t1", "b" :: Nil) } assertException("Cannot alter column b.d because this column is referenced by") { drop("t1", "b.d" :: Nil) } assertException("Cannot alter column arr because this column is referenced by") { drop("t1", "arr" :: Nil) } // you can still drop b.c as it has no dependent gen col drop("t1", "b.c" :: Nil) // you can also drop a generated column itself drop("t1", "genCol1" :: Nil) // add new data after dropping spark.createDataFrame( Seq(Row("str3", Row(3), Map("k3" -> "v3"), Array(3, 33))).asJava, new StructType() .add("a", StringType, true) .add("b", new StructType() .add("d", IntegerType, true)) .add("map", MapType(StringType, StringType), true) .add("arr", ArrayType(IntegerType), true)) .write.format("delta").mode("append").saveAsTable("t1") checkAnswer(spark.table("t1"), Seq( Row("str1", Row(1), Map("k1" -> "v1"), Array(1, 11), 101), Row("str2", Row(2), Map("k2" -> "v2"), Array(2, 22), 202), Row("str3", Row(3), Map("k3" -> "v3"), Array(3, 33), 303))) // this is a safety flag - if you turn it off, it will still error but msg is not as helpful withSQLConf(DeltaSQLConf.DELTA_ALTER_TABLE_CHANGE_COLUMN_CHECK_EXPRESSIONS.key -> "false") { assertException("A generated column cannot use a non-existent column") { drop("t1", "arr" :: Nil) } } } } } dropTest("dropping all columns is not allowed") { drop => withTable("t1") { createTableWithSQLAPI("t1", simpleNestedData, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name") ) val e = intercept[AnalysisException] { drop("t1", "a" :: "b" :: "map" :: "arr" :: Nil) } assert(e.getMessage.contains("Cannot drop column")) } } dropTest("dropping partition columns is not allowed") { drop => withTable("t1") { createTableWithSQLAPI("t1", simpleNestedData, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name"), partCols = Seq("a") ) val e = intercept[AnalysisException] { drop("t1", "a" :: Nil) } assert(e.getMessage.contains("Dropping partition columns (a) is not allowed")) } } /** * Covers dropping a nested field using the ALTER TABLE command. * @param initialColumnType Type of the single column used to create the initial test table. * @param fieldToDrop Name of the field to drop from the initial column type. * @param updatedColumnType Expected type of the single column after dropping the nested field. */ def testDropNestedField(testName: String)( initialColumnType: String, fieldToDrop: String, updatedColumnType: String): Unit = testColumnMapping(s"ALTER TABLE DROP COLUMNS - nested $testName") { mode => withTempDir { dir => withTable("delta_test") { sql( s""" |CREATE TABLE delta_test (data $initialColumnType) |USING delta |TBLPROPERTIES (${DeltaConfigs.COLUMN_MAPPING_MODE.key} = '$mode') |OPTIONS('path'='${dir.getCanonicalPath}')""".stripMargin) val expectedInitialType = initialColumnType.filterNot(_.isWhitespace) val expectedUpdatedType = updatedColumnType.filterNot(_.isWhitespace) val fieldName = s"data.${fieldToDrop}" def columnType: DataFrame = sql("DESCRIBE TABLE delta_test") .filter("col_name = 'data'") .select("data_type") checkAnswer(columnType, Row(expectedInitialType)) sql(s"ALTER TABLE delta_test DROP COLUMNS ($fieldName)") checkAnswer(columnType, Row(expectedUpdatedType)) } } } testDropNestedField("struct in map key")( initialColumnType = "map, int>", fieldToDrop = "key.b", updatedColumnType = "map, int>") testDropNestedField("struct in map value")( initialColumnType = "map>", fieldToDrop = "value.b", updatedColumnType = "map>") testDropNestedField("struct in array")( initialColumnType = "array>", fieldToDrop = "element.b", updatedColumnType = "array>") testDropNestedField("struct in nested map keys")( initialColumnType = "map, int>, int>", fieldToDrop = "key.key.b", updatedColumnType = "map, int>, int>") testDropNestedField("struct in nested map values")( initialColumnType = "map>>", fieldToDrop = "value.value.b", updatedColumnType = "map>>") testDropNestedField("struct in nested arrays")( initialColumnType = "array>>", fieldToDrop = "element.element.b", updatedColumnType = "array>>") testDropNestedField("struct in nested array and map")( initialColumnType = "array>>", fieldToDrop = "element.value.b", updatedColumnType = "array>>") testDropNestedField("struct in nested map key and array")( initialColumnType = "map>, int>", fieldToDrop = "key.element.b", updatedColumnType = "map>, int>") testDropNestedField("struct in nested map value and array")( initialColumnType = "map>>", fieldToDrop = "value.element.b", updatedColumnType = "map>>") test("can't drop map key/value or array element") { withTable("delta_test") { sql( s""" |CREATE TABLE delta_test (m map, a array) |USING delta |TBLPROPERTIES (${DeltaConfigs.COLUMN_MAPPING_MODE.key} = 'name') """.stripMargin) for { field <- Seq("m.key", "m.value", "a.element") } checkError( intercept[AnalysisException] { sql(s"ALTER TABLE delta_test DROP COLUMN $field") }, "DELTA_UNSUPPORTED_DROP_NESTED_COLUMN_FROM_NON_STRUCT_TYPE", parameters = Map( "struct" -> "IntegerType" ) ) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaErrorsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.{FileNotFoundException, PrintWriter, StringWriter} import java.net.URI import java.sql.Timestamp import java.text.SimpleDateFormat import java.util.Locale import scala.sys.process.Process // scalastyle:off import.ordering.noEmptyLine // scalastyle:off line.size.limit import org.apache.spark.sql.delta.DeltaErrors.generateDocsLink import org.apache.spark.sql.delta.actions.{Action, Metadata, Protocol} import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils.{TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION} import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.delta.constraints.CharVarcharConstraint import org.apache.spark.sql.delta.constraints.Constraints import org.apache.spark.sql.delta.constraints.Constraints.NotNull import org.apache.spark.sql.delta.hooks.{AutoCompactType, PostCommitHook} import org.apache.spark.sql.delta.schema.{DeltaInvariantViolationException, InvariantViolationException, SchemaMergingUtils, SchemaUtils, UnsupportedDataTypeInfo} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import io.delta.sql.DeltaSparkSessionExtension import org.apache.hadoop.fs.Path import org.json4s.JString import org.scalatest.GivenWhenThen import org.apache.spark.{SparkContext, SparkThrowable} import org.apache.spark.sql.{AnalysisException, QueryTest, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable} import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.expressions.{AttributeReference, ExprId, Length, LessThanOrEqual, Literal, SparkVersion} import org.apache.spark.sql.catalyst.expressions.Uuid import org.apache.spark.sql.catalyst.parser.CatalystSqlParser import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.errors.QueryErrorsBase import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ trait DeltaErrorsSuiteBase extends QueryTest with SharedSparkSession with GivenWhenThen with DeltaSQLCommandTest with DeltaSQLTestUtils with QueryErrorsBase { val MAX_URL_ACCESS_RETRIES = 3 val path = "/sample/path" // Map of error function to the error // When adding a function... // (a) if the function is just a message: add the name of the message/function as the key, and an // error that uses that message as the value // (b) if the function is an error function: add the name of the function as the key, and the // value as the error being thrown def errorsToTest: Map[String, Throwable] = Map( "createExternalTableWithoutLogException" -> DeltaErrors.createExternalTableWithoutLogException(new Path(path), "tableName", spark), "createExternalTableWithoutSchemaException" -> DeltaErrors.createExternalTableWithoutSchemaException(new Path(path), "tableName", spark), "createManagedTableWithoutSchemaException" -> DeltaErrors.createManagedTableWithoutSchemaException("tableName", spark), "multipleSourceRowMatchingTargetRowInMergeException" -> DeltaErrors.multipleSourceRowMatchingTargetRowInMergeException(spark), "concurrentModificationExceptionMsg" -> new ConcurrentWriteException(None), "incorrectLogStoreImplementationException" -> DeltaErrors.incorrectLogStoreImplementationException(sparkConf, new Throwable()), "sourceNotDeterministicInMergeException" -> DeltaErrors.sourceNotDeterministicInMergeException(spark), "columnMappingAdviceMessage" -> DeltaErrors.columnRenameNotSupported, "icebergClassMissing" -> DeltaErrors.icebergClassMissing(sparkConf, new Throwable()), "tableFeatureReadRequiresWriteException" -> DeltaErrors.tableFeatureReadRequiresWriteException(requiredWriterVersion = 7), "tableFeatureRequiresHigherReaderProtocolVersion" -> DeltaErrors.tableFeatureRequiresHigherReaderProtocolVersion( feature = "feature", currentVersion = 1, requiredVersion = 7), "tableFeatureRequiresHigherWriterProtocolVersion" -> DeltaErrors.tableFeatureRequiresHigherReaderProtocolVersion( feature = "feature", currentVersion = 1, requiredVersion = 7), "blockStreamingReadsWithIncompatibleNonAdditiveSchemaChanges" -> DeltaErrors.blockStreamingReadsWithIncompatibleNonAdditiveSchemaChanges( spark, StructType.fromDDL("id int"), StructType.fromDDL("id2 int"), detectedDuringStreaming = true), "concurrentAppendException" -> DeltaErrors.concurrentAppendException(None, "t", -1, partitionOpt = None), "concurrentDeleteDeleteException" -> DeltaErrors.concurrentDeleteDeleteException(None, "t", -1, partitionOpt = None), "concurrentDeleteReadException" -> DeltaErrors.concurrentDeleteReadException(None, "t", -1, partitionOpt = None), "concurrentWriteException" -> DeltaErrors.concurrentWriteException(None), "concurrentTransactionException" -> DeltaErrors.concurrentTransactionException(None), "metadataChangedException" -> DeltaErrors.metadataChangedException(None), "protocolChangedException" -> DeltaErrors.protocolChangedException(None) ) def otherMessagesToTest: Map[String, String] = Map( "ignoreStreamingUpdatesAndDeletesWarning" -> DeltaErrors.ignoreStreamingUpdatesAndDeletesWarning(spark) ) def errorMessagesToTest: Map[String, String] = errorsToTest.mapValues(_.getMessage).toMap ++ otherMessagesToTest def checkIfValidResponse(url: String, response: String): Boolean = { response.contains("HTTP/1.1 200 OK") || response.contains("HTTP/2 200") } def getUrlsFromMessage(message: String): List[String] = { val regexToFindUrl = "https://[^\\s]+".r regexToFindUrl.findAllIn(message).toList } def testUrl(errName: String, url: String): Unit = { Given(s"*** Checking response for url: $url") val lastResponse = (1 to MAX_URL_ACCESS_RETRIES).map { attempt => if (attempt > 1) Thread.sleep(1000) val response = try { Process("curl -I -L " + url).!! } catch { case e: RuntimeException => val sw = new StringWriter e.printStackTrace(new PrintWriter(sw)) sw.toString } if (checkIfValidResponse(url, response)) { // The URL is correct. No need to retry. return } response }.last // None of the attempts resulted in a valid response. Fail the test. fail( s""" |A link to the URL: '$url' is broken in the error: $errName, accessing this URL |does not result in a valid response, received the following response: $lastResponse """.stripMargin) } def testUrls(): Unit = { errorMessagesToTest.foreach { case (errName, message) => getUrlsFromMessage(message).foreach { url => testUrl(errName, url) } } } def generateDocsLink(relativePath: String): String = DeltaErrors.generateDocsLink( spark.sparkContext.getConf, relativePath, skipValidation = true) test("Validate that links to docs in DeltaErrors are correct") { // verify DeltaErrors.errorsWithDocsLinks is consistent with DeltaErrorsSuite assert(errorsToTest.keySet ++ otherMessagesToTest.keySet == DeltaErrors.errorsWithDocsLinks.toSet ) testUrls() } protected def multipleSourceRowMatchingTargetRowInMergeUrl: String = "/delta-update.html#upsert-into-a-table-using-merge" test("test DeltaErrors methods -- part 1") { { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.tableAlreadyContainsCDCColumns(Seq("col1", "col2")) } checkError(e, "DELTA_TABLE_ALREADY_CONTAINS_CDC_COLUMNS", "42711", Map("columnList" -> "[col1,col2]")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.cdcColumnsInData(Seq("col1", "col2")) } checkError(e, "RESERVED_CDC_COLUMNS_ON_WRITE", "42939", Map("columnList" -> "[col1,col2]", "config" -> "delta.enableChangeDataFeed")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.multipleCDCBoundaryException("starting") } checkError(e, "DELTA_MULTIPLE_CDC_BOUNDARY", "42614", Map("startingOrEnding" -> "starting")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.failOnCheckpointRename(new Path("path-1"), new Path("path-2")) } checkError(e, "DELTA_CANNOT_RENAME_PATH", "22KD1", Map("currentPath" -> "path-1", "newPath" -> "path-2")) } { val e = intercept[DeltaInvariantViolationException] { throw DeltaErrors.notNullColumnMissingException(NotNull(Seq("c0", "c1"))) } checkError(e, "DELTA_MISSING_NOT_NULL_COLUMN_VALUE", "23502", Map("columnName" -> "c0.c1")) } { val parent = "parent" val nested = IntegerType val nestType = "nestType" val e = intercept[DeltaAnalysisException] { throw DeltaErrors.nestedNotNullConstraint(parent, nested, nestType) } checkError(e, "DELTA_NESTED_NOT_NULL_CONSTRAINT", "0AKDC", Map( "parent" -> parent, "nestedPrettyJson" -> nested.prettyJson, "nestType" -> nestType, "configKey" -> DeltaSQLConf.ALLOW_UNENFORCED_NOT_NULL_CONSTRAINTS.key)) } { val e = intercept[DeltaInvariantViolationException] { throw DeltaInvariantViolationException(Constraints.NotNull(Seq("col1"))) } checkError(e, "DELTA_NOT_NULL_CONSTRAINT_VIOLATED", "23502", Map("columnName" -> "col1")) } { val expr = UnresolvedAttribute("col") val e = intercept[DeltaInvariantViolationException] { throw DeltaInvariantViolationException( Constraints.Check(CharVarcharConstraint.INVARIANT_NAME, LessThanOrEqual(Length(expr), Literal(5))), Map("col" -> "Hello World")) } checkError(e, "DELTA_EXCEED_CHAR_VARCHAR_LIMIT", "22001", Map("value" -> "Hello World", "expr" -> "(length(col) <= 5)")) } { val e = intercept[DeltaInvariantViolationException] { throw DeltaInvariantViolationException( Constraints.Check("__dummy__", CatalystSqlParser.parseExpression("id < 0")), Map("a" -> "b")) } checkError(e, "DELTA_VIOLATE_CONSTRAINT_WITH_VALUES", "23001", Map( "constraintName" -> "__dummy__", "expression" -> "(id < 0)", "values" -> " - a : b")) } { val tableIdentifier = DeltaTableIdentifier(Some("tableName")) val e = intercept[DeltaAnalysisException] { throw DeltaErrors.notADeltaTableException(tableIdentifier) } checkError(e, "DELTA_MISSING_DELTA_TABLE", "42P01", Map("tableName" -> tableIdentifier.toString)) } { val tableIdentifier = DeltaTableIdentifier(Some("tableName")) val e = intercept[DeltaAnalysisException] { throw DeltaErrors.notADeltaTableException( operation = "delete", tableIdentifier) } checkError(e, "DELTA_TABLE_ONLY_OPERATION", "0AKDD", Map("tableName" -> tableIdentifier.toString, "operation" -> "delete")) } { val table = TableIdentifier("table") val e = intercept[DeltaAnalysisException] { throw DeltaErrors.cannotWriteIntoView(table) } checkError(e, "DELTA_CANNOT_WRITE_INTO_VIEW", "0A000", Map("table" -> table.toString)) } { val sourceType = IntegerType val targetType = DateType val columnName = "column_name" val e = intercept[DeltaArithmeticException] { throw DeltaErrors.castingCauseOverflowErrorInTableWrite(sourceType, targetType, columnName) } checkError(e, "DELTA_CAST_OVERFLOW_IN_TABLE_WRITE", "22003", Map( "sourceType" -> toSQLType(sourceType), "targetType" -> toSQLType(targetType), "columnName" -> toSQLId(columnName), "storeAssignmentPolicyFlag" -> SQLConf.STORE_ASSIGNMENT_POLICY.key, "updateAndMergeCastingFollowsAnsiEnabledFlag" -> DeltaSQLConf.UPDATE_AND_MERGE_CASTING_FOLLOWS_ANSI_ENABLED_FLAG.key, "ansiEnabledFlag" -> SQLConf.ANSI_ENABLED.key)) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.invalidColumnName(name = "col-1") } checkError(e, "DELTA_INVALID_CHARACTERS_IN_COLUMN_NAME", "42K05", Map("columnName" -> "col-1")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.updateSetColumnNotFoundException(col = "c0", colList = Seq("c1", "c2")) } checkError(e, "DELTA_MISSING_SET_COLUMN", "42703", Map("columnName" -> "`c0`", "columnList" -> "[`c1`, `c2`]")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.updateSetConflictException(cols = Seq("c1", "c2")) } checkError(e, "DELTA_CONFLICT_SET_COLUMN", "42701", Map("columnList" -> "[`c1`, `c2`]")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.bloomFilterOnNestedColumnNotSupportedException("c0") } checkError(e, "DELTA_UNSUPPORTED_NESTED_COLUMN_IN_BLOOM_FILTER", "0AKDC", Map("columnName" -> "c0")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.bloomFilterOnPartitionColumnNotSupportedException("c0") } checkError(e, "DELTA_UNSUPPORTED_PARTITION_COLUMN_IN_BLOOM_FILTER", "0AKDC", Map("columnName" -> "c0")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.bloomFilterDropOnNonIndexedColumnException("c0") } checkError(e, "DELTA_CANNOT_DROP_BLOOM_FILTER_ON_NON_INDEXED_COLUMN", "42703", Map("columnName" -> "c0")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.cannotRenamePath("a", "b") } checkError(e, "DELTA_CANNOT_RENAME_PATH", "22KD1", Map("currentPath" -> "a", "newPath" -> "b")) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.cannotSpecifyBothFileListAndPatternString() } checkError(e, "DELTA_FILE_LIST_AND_PATTERN_STRING_CONFLICT", "42613", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.cannotUpdateArrayField("t", "f") } checkError(e, "DELTA_CANNOT_UPDATE_ARRAY_FIELD", "429BQ", Map("tableName" -> "t", "fieldName" -> "f")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.cannotUpdateMapField("t", "f") } checkError(e, "DELTA_CANNOT_UPDATE_MAP_FIELD", "429BQ", Map("tableName" -> "t", "fieldName" -> "f")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.cannotUpdateStructField("t", "f") } checkError(e, "DELTA_CANNOT_UPDATE_STRUCT_FIELD", "429BQ", Map("tableName" -> "t", "fieldName" -> "f")) } { val tableName = "table" val e = intercept[DeltaAnalysisException] { throw DeltaErrors.cannotUpdateOtherField(tableName, IntegerType) } checkError(e, "DELTA_CANNOT_UPDATE_OTHER_FIELD", "429BQ", Map("tableName" -> tableName, "typeName" -> IntegerType.toString)) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.duplicateColumnsOnUpdateTable(originalException = new Exception("123")) } checkError(e, "DELTA_DUPLICATE_COLUMNS_ON_UPDATE_TABLE", "42701", Map("message" -> "123")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.maxCommitRetriesExceededException(0, 1, 2, 3, 4) } checkError(e, "DELTA_MAX_COMMIT_RETRIES_EXCEEDED", "40000", Map("failVersion" -> "1", "startVersion" -> "2", "timeSpent" -> "4", "numActions" -> "3", "numAttempts" -> "0")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.missingColumnsInInsertInto("c") } checkError(e, "DELTA_INSERT_COLUMN_MISMATCH", "42802", Map("columnName" -> "c")) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.invalidAutoCompactType("invalid") } val allowed = AutoCompactType.ALLOWED_VALUES.mkString("(", ",", ")") checkError(e, "DELTA_INVALID_AUTO_COMPACT_TYPE", "22023", Map("value" -> "invalid", "allowed" -> allowed)) } { val table = DeltaTableIdentifier(Some("path")) val e = intercept[DeltaAnalysisException] { throw DeltaErrors.nonExistentDeltaTable(table) } checkError(e, "DELTA_TABLE_NOT_FOUND", "42P01", Map("tableName" -> table.toString)) } { val newTableId = "027fb01c-94aa-4cab-87cb-5aab6aec6d17" val oldTableId = "2edf2c02-bb63-44e9-a84c-517fad0db296" val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.differentDeltaTableReadByStreamingSource( newTableId = newTableId, oldTableId = oldTableId) } checkError(e, "DIFFERENT_DELTA_TABLE_READ_BY_STREAMING_SOURCE", "55019", Map( "oldTableId" -> oldTableId, "newTableId" -> newTableId)) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.nonExistentColumnInSchema("c", "s") } checkError(e, "DELTA_COLUMN_NOT_FOUND_IN_SCHEMA", "42703", Map( "columnName" -> "c", "tableSchema" -> "s")) } { val ident = Identifier.of(Array("namespace"), "name") val e = intercept[DeltaNoSuchTableException] { throw DeltaErrors.noRelationTable(ident) } checkError(e, "DELTA_NO_RELATION_TABLE", "42P01", Map("tableIdent" -> ident.quoted)) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.notADeltaTable("t") } checkError(e, "DELTA_NOT_A_DELTA_TABLE", "0AKDD", Map("tableName" -> "t")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.notFoundFileToBeRewritten("f", Seq("a", "b")) } checkError(e, "DELTA_FILE_TO_OVERWRITE_NOT_FOUND", "42K03", Map("path" -> "f", "pathList" -> "a\nb")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.unsetNonExistentProperty("k", "t") } checkError(e, "DELTA_UNSET_NON_EXISTENT_PROPERTY", "42616", Map("property" -> "k", "tableName" -> "t")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.generatedColumnsReferToWrongColumns( new AnalysisException( errorClass = "INTERNAL_ERROR", messageParameters = Map("message" -> "internal test error msg")) ) } checkError(e, "DELTA_INVALID_GENERATED_COLUMN_REFERENCES", "42621", Map.empty[String, String]) checkError(e.getCause.asInstanceOf[AnalysisException], "INTERNAL_ERROR", None, Map("message" -> "internal test error msg")) } { val current = StructField("c0", IntegerType) val update = StructField("c0", StringType) val e = intercept[DeltaAnalysisException] { throw DeltaErrors.generatedColumnsUpdateColumnType(current, update) } checkError(e, "DELTA_GENERATED_COLUMN_UPDATE_TYPE_MISMATCH", "42K09", Map( "currentName" -> current.name, "currentDataType" -> current.dataType.sql, "updateDataType" -> update.dataType.sql)) } { val e = intercept[DeltaColumnMappingUnsupportedException] { throw DeltaErrors.changeColumnMappingModeNotSupported(oldMode = "old", newMode = "new") } checkError(e, "DELTA_UNSUPPORTED_COLUMN_MAPPING_MODE_CHANGE", "0AKDC", Map("oldMode" -> "old", "newMode" -> "new")) } { val e = intercept[DeltaColumnMappingUnsupportedException] { throw DeltaErrors.generateManifestWithColumnMappingNotSupported } checkError(e, "DELTA_UNSUPPORTED_MANIFEST_GENERATION_WITH_COLUMN_MAPPING", "0AKDC", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.convertToDeltaNoPartitionFound("testTable") } checkError(e, "DELTA_CONVERSION_NO_PARTITION_FOUND", "42KD6", Map("tableName" -> "testTable")) } { val e = intercept[DeltaColumnMappingUnsupportedException] { throw DeltaErrors.convertToDeltaWithColumnMappingNotSupported(IdMapping) } checkError(e, "DELTA_CONVERSION_UNSUPPORTED_COLUMN_MAPPING", "0AKDC", Map( "config" -> DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey, "mode" -> "id")) } { val oldSchema = StructType(Seq(StructField("c0", IntegerType))) val newSchema = StructType(Seq(StructField("c1", IntegerType))) val e = intercept[DeltaColumnMappingUnsupportedException] { throw DeltaErrors.schemaChangeDuringMappingModeChangeNotSupported( oldSchema = oldSchema, newSchema = newSchema) } checkError(e, "DELTA_UNSUPPORTED_COLUMN_MAPPING_SCHEMA_CHANGE", "0AKDC", Map("oldTableSchema" -> oldSchema.treeString, "newTableSchema" -> newSchema.treeString)) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.notEnoughColumnsInInsert( "table", 1, 2, Some("nestedField")) } checkError(e, "DELTA_INSERT_COLUMN_ARITY_MISMATCH", "42802", Map("tableName" -> "table", "columnName" -> "not enough nested fields in nestedField", "numColumns" -> "2", "insertColumns" -> "1")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.cannotInsertIntoColumn( "tableName", "source", "target", "targetType") } checkError(e, "DELTA_COLUMN_STRUCT_TYPE_MISMATCH", "2200G", Map("source" -> "source", "targetType" -> "targetType", "targetField" -> "target", "targetTable" -> "tableName")) } { val colName = "col1" val schema = Seq(UnresolvedAttribute("col2")) val e = intercept[DeltaAnalysisException] { throw DeltaErrors.partitionColumnNotFoundException(colName, schema) } checkError(e, "DELTA_PARTITION_COLUMN_NOT_FOUND", "42703", Map("columnName" -> DeltaErrors.formatColumn(colName), "schemaMap" -> schema.map(_.name).mkString(", "))) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.partitionPathParseException("fragment") } checkError(e, "DELTA_INVALID_PARTITION_PATH", "22KD1", Map("path" -> "fragment")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.replaceWhereMismatchException("replaceWhereArgValue", new InvariantViolationException("Invariant violated.")) } checkError(e, "DELTA_REPLACE_WHERE_MISMATCH", "44000", Map("replaceWhere" -> "replaceWhereArgValue", "message" -> "Invariant violated.")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.replaceWhereMismatchException("replaceWhere", "badPartitions") } checkError(e, "DELTA_REPLACE_WHERE_MISMATCH", "44000", Map("replaceWhere" -> "replaceWhere", "message" -> "Invalid data would be written to partitions badPartitions.")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.actionNotFoundException("action", 0) } checkError(e, "DELTA_STATE_RECOVER_ERROR", "XXKDS", Map("operation" -> "action", "version" -> "0")) } { val oldSchema = StructType(Seq(StructField("c0", IntegerType))) val newSchema = StructType(Seq(StructField("c0", StringType))) for (retryable <- DeltaTestUtils.BOOLEAN_DOMAIN) { val expectedClass: Class[_] = classOf[DeltaIllegalStateException] var e = intercept[Exception with SparkThrowable] { throw DeltaErrors.schemaChangedException(oldSchema, newSchema, retryable, None, false) } assert(expectedClass.isAssignableFrom(e.getClass)) checkError(e, "DELTA_SCHEMA_CHANGED", "KD007", Map( "readSchema" -> DeltaErrors.formatSchema(oldSchema), "dataSchema" -> DeltaErrors.formatSchema(newSchema) )) // Check the error message with version information e = intercept[Exception with SparkThrowable] { throw DeltaErrors.schemaChangedException(oldSchema, newSchema, retryable, Some(10), false) } assert(expectedClass.isAssignableFrom(e.getClass)) checkError(e, "DELTA_SCHEMA_CHANGED_WITH_VERSION", "KD007", Map( "version" -> "10", "readSchema" -> DeltaErrors.formatSchema(oldSchema), "dataSchema" -> DeltaErrors.formatSchema(newSchema) )) // Check the error message with startingVersion/Timestamp error message e = intercept[Exception with SparkThrowable] { throw DeltaErrors.schemaChangedException(oldSchema, newSchema, retryable, Some(10), true) } assert(expectedClass.isAssignableFrom(e.getClass)) checkError(e, "DELTA_SCHEMA_CHANGED_WITH_STARTING_OPTIONS", "KD007", Map( "version" -> "10", "readSchema" -> DeltaErrors.formatSchema(oldSchema), "dataSchema" -> DeltaErrors.formatSchema(newSchema) )) } } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.restoreVersionNotExistException(1, 2, 3) } checkError(e, "DELTA_CANNOT_RESTORE_TABLE_VERSION", "22003", Map("version" -> "1", "startVersion" -> "2", "endVersion" -> "3")) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.unsupportedColumnMappingModeException("modeName") } checkError(e, "DELTA_MODE_NOT_SUPPORTED", "0AKDC", Map( "mode" -> "modeName", "supportedModes" -> DeltaColumnMapping.supportedModes.map(_.name).toSeq.mkString(", ") )) } { import org.apache.spark.sql.delta.commands.DeltaGenerateCommand val supportedModes = DeltaGenerateCommand.modeNameToGenerationFunc.keys.toSeq.mkString(", ") val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.unsupportedGenerateModeException("modeName") } checkError(e, "DELTA_MODE_NOT_SUPPORTED", "0AKDC", Map( "mode" -> "modeName", "supportedModes" -> supportedModes)) } { import org.apache.spark.sql.delta.DeltaOptions.EXCLUDE_REGEX_OPTION val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.excludeRegexOptionException(EXCLUDE_REGEX_OPTION) } checkError(e, "DELTA_REGEX_OPT_SYNTAX_ERROR", "2201B", Map( "regExpOption" -> EXCLUDE_REGEX_OPTION )) } { val e = intercept[DeltaFileNotFoundException] { throw DeltaErrors.fileNotFoundException("somePath") } checkError(e, "DELTA_FILE_NOT_FOUND", "42K03", Map( "path" -> "somePath" )) } { val e = intercept[DeltaFileNotFoundException] { throw DeltaErrors.logFileNotFoundException(new Path("file://table"), None, 10) } checkError(e, "DELTA_LOG_FILE_NOT_FOUND", "42K03", Map( "version" -> "LATEST", "checkpointVersion" -> "10", "logPath" -> "file://table" )) } { val ex = new FileNotFoundException("reason") val e = intercept[DeltaFileNotFoundException] { throw DeltaErrors.logFileNotFoundExceptionForStreamingSource(ex) } checkError(e, "DELTA_LOG_FILE_NOT_FOUND_FOR_STREAMING_SOURCE", "42K03", parameters = Map.empty[String, String]) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.invalidIsolationLevelException("level") } checkError(e, "DELTA_INVALID_ISOLATION_LEVEL", "25000", Map( "isolationLevel" -> "level" )) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.columnNameNotFoundException("a", "b") } checkError(e, "DELTA_COLUMN_NOT_FOUND", "42703", Map( "columnName" -> "a", "columnList" -> "b" )) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.addColumnAtIndexLessThanZeroException("1", "a") } checkError(e, "DELTA_ADD_COLUMN_AT_INDEX_LESS_THAN_ZERO", "42KD3", Map( "columnIndex" -> "1", "columnName" -> "a" )) } { val pos = -1 val e = intercept[DeltaAnalysisException] { throw DeltaErrors.dropColumnAtIndexLessThanZeroException(pos) } checkError(e, "DELTA_DROP_COLUMN_AT_INDEX_LESS_THAN_ZERO", "42KD8", Map("columnIndex" -> pos.toString)) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.incorrectArrayAccess() } checkError(e, "DELTA_INCORRECT_ARRAY_ACCESS", "KD003", Map.empty[String, String]) } { val e = intercept[DeltaRuntimeException] { throw DeltaErrors.partitionColumnCastFailed("Value", "Type", "Name") } checkError(e, "DELTA_PARTITION_COLUMN_CAST_FAILED", "22525", Map("value" -> "Value", "dataType" -> "Type", "columnName" -> "Name")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.invalidTimestampFormat("ts", "someFormat") } checkError(e, "DELTA_INVALID_TIMESTAMP_FORMAT", "22007", Map("timestamp" -> "ts", "format" -> "someFormat")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.cannotChangeDataType("example message") } checkError(e, "DELTA_CANNOT_CHANGE_DATA_TYPE", "429BQ", Map("dataType" -> "example message")) } { val table = CatalogTable(TableIdentifier("my table"), null, null, null) val e = intercept[DeltaAnalysisException] { throw DeltaErrors.tableAlreadyExists(table) } checkError(e, "DELTA_TABLE_ALREADY_EXISTS", "42P07", Map("tableName" -> "`my table`")) } { val storage1 = CatalogStorageFormat(Option(new URI("loc1")), null, null, null, false, Map.empty) val storage2 = CatalogStorageFormat(Option(new URI("loc2")), null, null, null, false, Map.empty) val table = CatalogTable(TableIdentifier("table"), null, storage1, null) val existingTable = CatalogTable(TableIdentifier("existing table"), null, storage2, null) val e = intercept[DeltaAnalysisException] { throw DeltaErrors.tableLocationMismatch(table, existingTable) } checkError(e, "DELTA_TABLE_LOCATION_MISMATCH", "42613", Map( "tableName" -> "`table`", "tableLocation" -> "`loc1`", "existingTableLocation" -> "`loc2`" )) } { val ident = "ident" val e = intercept[DeltaNoSuchTableException] { throw DeltaErrors.nonSinglePartNamespaceForCatalog(ident) } checkError(e, "DELTA_NON_SINGLE_PART_NAMESPACE_FOR_CATALOG", "42K05", Map("identifier" -> ident)) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.targetTableFinalSchemaEmptyException() } checkError(e, "DELTA_TARGET_TABLE_FINAL_SCHEMA_EMPTY", "428GU", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.nonDeterministicNotSupportedException("op", Uuid()) } checkError(e, "DELTA_NON_DETERMINISTIC_FUNCTION_NOT_SUPPORTED", "0AKDC", Map("operation" -> "op", "expression" -> "(condition = uuid()).")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.tableNotSupportedException("someOp") } checkError(e, "DELTA_TABLE_NOT_SUPPORTED_IN_OP", "42809", Map("operation" -> "someOp")) } { val e = intercept[DeltaRuntimeException] { throw DeltaErrors.postCommitHookFailedException(new PostCommitHook() { override val name: String = "DummyPostCommitHook" override def run(spark: SparkSession, txn: CommittedTransaction): Unit = {} }, 0, "msg", null) } checkError(e, "DELTA_POST_COMMIT_HOOK_FAILED", "2DKD0", Map("version" -> "0", "name" -> "DummyPostCommitHook", "message" -> ": msg")) } { val e = intercept[DeltaRuntimeException] { throw DeltaErrors.postCommitHookFailedException(new PostCommitHook() { override val name: String = "DummyPostCommitHook" override def run(spark: SparkSession, txn: CommittedTransaction): Unit = {} }, 0, null, null) } checkError(e, "DELTA_POST_COMMIT_HOOK_FAILED", "2DKD0", Map( "version" -> "0", "name" -> "DummyPostCommitHook", "message" -> "")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.indexLargerThanStruct(1, StructField("col1", IntegerType), 1) } checkError(e, "DELTA_INDEX_LARGER_THAN_STRUCT", "42KD8", Map( "index" -> "1", "columnName" -> "StructField(col1,IntegerType,true)", "length" -> "1")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.indexLargerOrEqualThanStruct(pos = 1, len = 2) } checkError(e, "DELTA_INDEX_LARGER_OR_EQUAL_THAN_STRUCT", "42KD8", Map("index" -> "1", "length" -> "2") ) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.invalidV1TableCall("v1Table", "DeltaTableV2") } checkError(e, "DELTA_INVALID_V1_TABLE_CALL", "XXKDS", Map("callVersion" -> "v1Table", "tableVersion" -> "DeltaTableV2")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.cannotGenerateUpdateExpressions() } checkError(e, "DELTA_CANNOT_GENERATE_UPDATE_EXPRESSIONS", "XXKDS", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { val s1 = StructType(Seq(StructField("c0", IntegerType))) val s2 = StructType(Seq(StructField("c0", StringType))) SchemaMergingUtils.mergeSchemas(s1, s2) } checkError( e, "DELTA_FAILED_TO_MERGE_FIELDS", parameters = Map("currentField" -> "c0", "updateField" -> "c0")) checkError( e.getCause.asInstanceOf[DeltaAnalysisException], "DELTA_MERGE_INCOMPATIBLE_DATATYPE", parameters = Map("currentDataType" -> "IntegerType", "updateDataType" -> "StringType")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.describeViewHistory } checkError(e, "DELTA_CANNOT_DESCRIBE_VIEW_HISTORY", "42809", Map.empty[String, String]) } { val e = intercept[DeltaUnsupportedOperationException] { throw DeltaErrors.unrecognizedInvariant() } checkError(e, "DELTA_UNRECOGNIZED_INVARIANT", "56038", Map.empty[String, String]) } { val baseSchema = StructType(Seq(StructField("c0", StringType))) val field = StructField("id", IntegerType) val e = intercept[DeltaAnalysisException] { throw DeltaErrors.cannotResolveColumn(field.name, baseSchema) } checkError(e, "DELTA_CANNOT_RESOLVE_COLUMN", "42703", Map("schema" -> baseSchema.treeString, "columnName" -> "id")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.alterTableChangeColumnException( fieldPath = "a.b.c", oldField = StructField("c", IntegerType), newField = StructField("c", LongType)) } checkError( e, "DELTA_UNSUPPORTED_ALTER_TABLE_CHANGE_COL_OP", parameters = Map( "fieldPath" -> "a.b.c", "oldField" -> "INT", "newField" -> "BIGINT" ) ) } { val s1 = StructType(Seq(StructField("c0", IntegerType))) val s2 = StructType(Seq(StructField("c0", StringType))) val e = intercept[DeltaAnalysisException] { throw DeltaErrors.alterTableReplaceColumnsException(s1, s2, "incompatible") } checkError(e, "DELTA_UNSUPPORTED_ALTER_TABLE_REPLACE_COL_OP", "0AKDC", Map( "details" -> "incompatible", "oldSchema" -> s1.treeString, "newSchema" -> s2.treeString)) } { checkError( exception = intercept[DeltaUnsupportedOperationException] { throw DeltaErrors.unsupportedTypeChangeInPreview( fieldPath = Seq("origin", "country"), fromType = IntegerType, toType = LongType, feature = TypeWideningPreviewTableFeature ) }, "DELTA_UNSUPPORTED_TYPE_CHANGE_IN_PREVIEW", parameters = Map( "fieldPath" -> "origin.country", "fromType" -> "INT", "toType" -> "BIGINT", "typeWideningFeatureName" -> "typeWidening-preview" )) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.unsupportedTypeChangeInSchema(Seq("s", "a"), IntegerType, StringType) } checkError(e, "DELTA_UNSUPPORTED_TYPE_CHANGE_IN_SCHEMA", "0AKDC", Map("fieldName" -> "s.a", "fromType" -> "INT", "toType" -> "STRING")) } { val e = intercept[DeltaAnalysisException] { val classConf = Seq(("classKey", "classVal")) val schemeConf = Seq(("schemeKey", "schemeVal")) throw DeltaErrors.logStoreConfConflicts(classConf, schemeConf) } checkError(e, "DELTA_INVALID_LOGSTORE_CONF", "F0000", Map("classConfig" -> "classKey", "schemeConfig" -> "schemeKey")) } { val e = intercept[DeltaIllegalArgumentException] { val schemeConf = Seq(("key", "val")) throw DeltaErrors.inconsistentLogStoreConfs( Seq(("delta.key", "value1"), ("spark.delta.key", "value2"))) } checkError(e, "DELTA_INCONSISTENT_LOGSTORE_CONFS", "F0000", Map("setKeys" -> "delta.key = value1, spark.delta.key = value2")) } { val e = intercept[DeltaSparkException] { throw DeltaErrors.failedMergeSchemaFile( file = "someFile", schema = "someSchema", cause = null) } checkError(e, "DELTA_FAILED_MERGE_SCHEMA_FILE", "42KDA", Map("file" -> "someFile", "schema" -> "someSchema")) } { val id = TableIdentifier("id") val e = intercept[DeltaAnalysisException] { throw DeltaErrors.operationNotSupportedException("op", id) } checkError(e, "DELTA_OPERATION_NOT_ALLOWED_DETAIL", "0AKDC", Map("operation" -> "op", "tableName" -> "`id`")) } { val e = intercept[DeltaFileNotFoundException] { throw DeltaErrors.fileOrDirectoryNotFoundException("somePath") } checkError(e, "DELTA_FILE_OR_DIR_NOT_FOUND", "42K03", Map("path" -> "somePath")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.invalidPartitionColumn("col", "tbl") } checkError(e, "DELTA_INVALID_PARTITION_COLUMN", "42996", Map("columnName" -> "col", "tableName" -> "tbl")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.cannotFindSourceVersionException("someJson") } checkError(e, "DELTA_CANNOT_FIND_VERSION", "XXKDS", Map("json" -> "someJson")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.unknownConfigurationKeyException("confKey") } checkError(e, "DELTA_UNKNOWN_CONFIGURATION", "F0000", Map( "config" -> "confKey", "disableCheckConfig" -> DeltaSQLConf.ALLOW_ARBITRARY_TABLE_PROPERTIES.key )) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.pathNotExistsException("somePath") } checkError(e, "DELTA_PATH_DOES_NOT_EXIST", "42K03", Map("path" -> "somePath")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.failRelativizePath("somePath") } checkError(e, "DELTA_FAIL_RELATIVIZE_PATH", "XXKDS", Map( "path" -> "somePath", "config" -> DeltaSQLConf.DELTA_VACUUM_RELATIVIZE_IGNORE_ERROR.key )) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.illegalFilesFound("someFile") } checkError(e, "DELTA_ILLEGAL_FILE_FOUND", "XXKDS", Map("file" -> "someFile")) } { val name = "name" val input = "input" val explain = "explain" val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.illegalDeltaOptionException(name, input, explain) } checkError(e, "DELTA_ILLEGAL_OPTION", "42616", Map("name" -> name, "input" -> input, "explain" -> explain)) } { val version = "version" val timestamp = "timestamp" val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.startingVersionAndTimestampBothSetException(version, timestamp) } checkError(e, "DELTA_STARTING_VERSION_AND_TIMESTAMP_BOTH_SET", "42613", Map("version" -> version, "timestamp" -> timestamp)) } { val path = new Path("parent", "child") val specifiedSchema = StructType(Seq(StructField("a", IntegerType))) val existingSchema = StructType(Seq(StructField("b", StringType))) val diffs = Seq("a", "b") val e = intercept[DeltaAnalysisException] { throw DeltaErrors.createTableWithDifferentSchemaException(path, specifiedSchema, existingSchema, diffs) } checkError(e, "DELTA_CREATE_TABLE_SCHEME_MISMATCH", "42KD7", Map( "path" -> path.toString, "specifiedSchema" -> specifiedSchema.treeString, "existingSchema" -> existingSchema.treeString, "schemaDifferences" -> "- a\n- b")) } { val path = new Path("parent", "child") val e = intercept[DeltaAnalysisException] { throw DeltaErrors.noHistoryFound(path) } checkError(e, "DELTA_NO_COMMITS_FOUND", "KD006", Map("logPath" -> path.toString)) } { val path = new Path("parent", "child") val e = intercept[DeltaAnalysisException] { throw DeltaErrors.noRecreatableHistoryFound(path) } checkError(e, "DELTA_NO_RECREATABLE_HISTORY_FOUND", "KD006", Map("logPath" -> path.toString)) } { val e = intercept[DeltaRuntimeException] { throw DeltaErrors.castPartitionValueException("partitionValue", StringType) } checkError(e, "DELTA_FAILED_CAST_PARTITION_VALUE", "22018", Map("value" -> "partitionValue", "dataType" -> "StringType")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.sparkSessionNotSetException() } checkError(e, "DELTA_SPARK_SESSION_NOT_SET", "XXKDS", Map.empty[String, String]) } { val id = Identifier.of(Array("namespace"), "name") val e = intercept[DeltaAnalysisException] { throw DeltaErrors.cannotReplaceMissingTableException(id) } checkError(e, "DELTA_CANNOT_REPLACE_MISSING_TABLE", "42P01", Map("tableName" -> "namespace.name")) } { val e = intercept[DeltaIOException] { throw DeltaErrors.cannotCreateLogPathException("logPath") } checkError(e, "DELTA_CANNOT_CREATE_LOG_PATH", "42KD5", Map("path" -> "logPath")) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.protocolPropNotIntException( key = "someKey", value = "someValue") } checkError(e, "DELTA_PROTOCOL_PROPERTY_NOT_INT", "42K06", Map("key" -> "someKey", "value" -> "someValue")) } { val path = new Path("parent", "child") val e = intercept[DeltaAnalysisException] { throw DeltaErrors.createExternalTableWithoutLogException( path = path, tableName = "someTableName", spark = spark) } checkError(e, "DELTA_CREATE_EXTERNAL_TABLE_WITHOUT_TXN_LOG", "42K03", Map( "path" -> path.toString, "logPath" -> new Path(path, "_delta_log").toString, "tableName" -> "someTableName", "docLink" -> generateDocsLink("/index.html") )) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.ambiguousPathsInCreateTableException("id", "loc") } checkError(e, "DELTA_AMBIGUOUS_PATHS_IN_CREATE_TABLE", "42613", Map( "identifier" -> "id", "location" -> "loc", "config" -> DeltaSQLConf.DELTA_LEGACY_ALLOW_AMBIGUOUS_PATHS.key)) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.illegalUsageException("overwriteSchema", "replacing") } checkError(e, "DELTA_ILLEGAL_USAGE", "42601", Map( "option" -> "overwriteSchema", "operation" -> "replacing" )) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.expressionsNotFoundInGeneratedColumn("col1") } checkError(e, "DELTA_EXPRESSIONS_NOT_FOUND_IN_GENERATED_COLUMN", "XXKDS", Map( "columnName" -> "col1" )) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.activeSparkSessionNotFound() } checkError(e, "DELTA_ACTIVE_SPARK_SESSION_NOT_FOUND", "08003", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.operationOnTempViewWithGenerateColsNotSupported("UPDATE") } checkError(e, "DELTA_OPERATION_ON_TEMP_VIEW_WITH_GENERATED_COLS_NOT_SUPPORTED", "0A000", Map( "operation" -> "UPDATE" )) } { val property = "prop" val e = intercept[DeltaUnsupportedOperationException] { throw DeltaErrors.cannotModifyTableProperty(property) } checkError(e, "DELTA_CANNOT_MODIFY_TABLE_PROPERTY", "42939", Map("prop" -> property)) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.missingProviderForConvertException("parquet_path") } checkError(e, "DELTA_MISSING_PROVIDER_FOR_CONVERT", "0AKDC", Map( "path" -> "parquet_path" )) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.iteratorAlreadyClosed() } checkError(e, "DELTA_ITERATOR_ALREADY_CLOSED", "XXKDS", Map.empty[String, String]) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.activeTransactionAlreadySet() } checkError(e, "DELTA_ACTIVE_TRANSACTION_ALREADY_SET", "0B000", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.bloomFilterMultipleConfForSingleColumnException("col1") } checkError(e, "DELTA_MULTIPLE_CONF_FOR_SINGLE_COLUMN_IN_BLOOM_FILTER", "42614", Map("columnName" -> "col1")) } { val e = intercept[DeltaIOException] { throw DeltaErrors.incorrectLogStoreImplementationException(sparkConf, null) } checkError(e, "DELTA_INCORRECT_LOG_STORE_IMPLEMENTATION", "0AKDC", Map( "docLink" -> generateDocsLink("/delta-storage.html") )) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.invalidSourceVersion("xyz") } checkError(e, "DELTA_INVALID_SOURCE_VERSION", "XXKDS", Map("version" -> "xyz")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.invalidSourceOffsetFormat() } checkError(e, "DELTA_INVALID_SOURCE_OFFSET_FORMAT", "XXKDS", Map.empty[String, String]) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.invalidCommittedVersion(1L, 2L) } checkError(e, "DELTA_INVALID_COMMITTED_VERSION", "XXKDS", Map( "committedVersion" -> "1", "currentVersion" -> "2" )) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.nonPartitionColumnReference("col1", Seq("col2", "col3")) } checkError(e, "DELTA_NON_PARTITION_COLUMN_REFERENCE", "42P10", Map("columnName" -> "col1", "columnList" -> "col2, col3")) } { val e = intercept[DeltaAnalysisException] { val attr = UnresolvedAttribute("col1") val attrs = Seq(UnresolvedAttribute("col2"), UnresolvedAttribute("col3")) throw DeltaErrors.missingColumn(attr, attrs) } checkError(e, "DELTA_MISSING_COLUMN", "42703", Map("columnName" -> "col1", "columnList" -> "col2, col3")) } { val e = intercept[DeltaAnalysisException] { val schema = StructType(Seq(StructField("c0", IntegerType))) throw DeltaErrors.missingPartitionColumn("c1", schema.catalogString) } checkError(e, "DELTA_MISSING_PARTITION_COLUMN", "42KD6", Map("columnName" -> "c1", "columnList" -> "struct")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.aggsNotSupportedException("op", SparkVersion()) } checkError(e, "DELTA_AGGREGATION_NOT_SUPPORTED", "42903", Map("operation" -> "op", "predicate" -> "(condition = version())")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.cannotChangeProvider() } checkError(e, "DELTA_CANNOT_CHANGE_PROVIDER", "42939", Map.empty[String, String]) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.noNewAttributeId(AttributeReference("attr1", IntegerType)()) } checkError(e, "DELTA_NO_NEW_ATTRIBUTE_ID", "XXKDS", Map("columnName" -> "attr1")) } { val e = intercept[ProtocolDowngradeException] { val p1 = Protocol(1, 1) val p2 = Protocol(2, 2) throw new ProtocolDowngradeException(p1, p2) } checkError(e, "DELTA_INVALID_PROTOCOL_DOWNGRADE", "KD004", Map("oldProtocol" -> "1,1", "newProtocol" -> "2,2")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.generatedColumnsExprTypeMismatch("col1", IntegerType, StringType) } checkError(e, "DELTA_GENERATED_COLUMNS_EXPR_TYPE_MISMATCH", "42K09", Map("columnName" -> "col1", "expressionType" -> "STRING", "columnType" -> "INT")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.nonGeneratedColumnMissingUpdateExpression("attr1") } checkError(e, "DELTA_NON_GENERATED_COLUMN_MISSING_UPDATE_EXPR", "XXKDS", Map("columnName" -> "attr1")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.constraintDataTypeMismatch( columnPath = Seq("a", "x"), columnType = ByteType, dataType = IntegerType, constraints = Map("ck1" -> "a > 0", "ck2" -> "hash(b) > 0")) } checkError(e, "DELTA_CONSTRAINT_DATA_TYPE_MISMATCH", "42K09", Map( "columnName" -> "a.x", "columnType" -> "TINYINT", "dataType" -> "INT", "constraints" -> "ck1 -> a > 0\nck2 -> hash(b) > 0")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.generatedColumnsDataTypeMismatch( columnPath = Seq("a", "x"), columnType = ByteType, dataType = IntegerType, generatedColumns = Map( "gen1" -> "a . x + 1", "gen2" -> "3 + a . x" )) } checkError(e, "DELTA_GENERATED_COLUMNS_DATA_TYPE_MISMATCH", "42K09", Map( "columnName" -> "a.x", "columnType" -> "TINYINT", "dataType" -> "INT", "generatedColumns" -> "gen1 -> a . x + 1\ngen2 -> 3 + a . x")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.useSetLocation() } checkError(e, "DELTA_CANNOT_CHANGE_LOCATION", "42601", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.nonPartitionColumnAbsentException(false) } checkError(e, "DELTA_NON_PARTITION_COLUMN_ABSENT", "KD005", Map("details" -> "")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.nonPartitionColumnAbsentException(true) } checkError(e, "DELTA_NON_PARTITION_COLUMN_ABSENT", "KD005", Map("details" -> " Columns which are of NullType have been dropped.")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.constraintAlreadyExists("name", "oldExpr") } checkError(e, "DELTA_CONSTRAINT_ALREADY_EXISTS", "42710", Map("constraintName" -> "name", "oldConstraint" -> "oldExpr")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.timeTravelNotSupportedException } checkError(e, "DELTA_UNSUPPORTED_TIME_TRAVEL_VIEWS", "0AKDC", Map.empty[String, String]) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.addFilePartitioningMismatchException(Seq("col3"), Seq("col2")) } checkError(e, "DELTA_INVALID_PARTITIONING_SCHEMA", "XXKDS", Map( "neededPartitioning" -> "[`col2`]", "specifiedPartitioning" -> "[`col3`]", "config" -> DeltaSQLConf.DELTA_COMMIT_VALIDATION_ENABLED.key)) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.emptyCalendarInterval } checkError(e, "DELTA_INVALID_CALENDAR_INTERVAL_EMPTY", "2200P", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.createManagedTableWithoutSchemaException("table-1", spark) } checkError(e, "DELTA_INVALID_MANAGED_TABLE_SYNTAX_NO_SCHEMA", "42000", Map( "tableName" -> "table-1", "docLink" -> generateDocsLink("/index.html") )) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.generatedColumnsUnsupportedExpression("someExp".expr) } checkError(e, "DELTA_UNSUPPORTED_EXPRESSION_GENERATED_COLUMN", "42621", Map("expression" -> "'someExp'")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.unsupportedExpression("Merge", DataTypes.DateType, Seq("Integer", "Long")) } checkError(e, "DELTA_UNSUPPORTED_EXPRESSION", "0A000", Map( "expType" -> "DateType", "causedBy" -> "Merge", "supportedTypes" -> "Integer,Long" )) } { val expr = "someExp" val e = intercept[DeltaAnalysisException] { throw DeltaErrors.generatedColumnsUDF(expr.expr) } checkError(e, "DELTA_UDF_IN_GENERATED_COLUMN", "42621", Map("udfExpr" -> "'someExp'")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.bloomFilterOnColumnTypeNotSupportedException("col1", DateType) } checkError(e, "DELTA_UNSUPPORTED_COLUMN_TYPE_IN_BLOOM_FILTER", "0AKDC", Map( "columnName" -> "col1", "dataType" -> "date" )) } { val e = intercept[DeltaTableFeatureException] { throw DeltaErrors.tableFeatureDropHistoryTruncationNotAllowed() } checkError(e, "DELTA_FEATURE_DROP_HISTORY_TRUNCATION_NOT_ALLOWED", "42000", Map.empty[String, String]) } { val logRetention = DeltaConfigs.LOG_RETENTION val e = intercept[DeltaTableFeatureException] { throw DeltaErrors.dropTableFeatureWaitForRetentionPeriod( "test_feature", Metadata(configuration = Map(logRetention.key -> "30 days")) ) } checkError(e, "DELTA_FEATURE_DROP_WAIT_FOR_RETENTION_PERIOD", "22KD0", Map( "feature" -> "test_feature", "logRetentionPeriodKey" -> "delta.logRetentionDuration", "logRetentionPeriod" -> "30 days", "truncateHistoryLogRetentionPeriod" -> "24 hours")) } } test("test DeltaErrors methods -- part 2") { { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.unsupportedDataTypes( UnsupportedDataTypeInfo("foo", CalendarIntervalType), UnsupportedDataTypeInfo("bar", TimestampNTZType)) } checkError(e, "DELTA_UNSUPPORTED_DATA_TYPES", "0AKDC", Map( "dataTypeList" -> "[foo: CalendarIntervalType, bar: TimestampNTZType]", "config" -> DeltaSQLConf.DELTA_SCHEMA_TYPE_CHECK.key )) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.failOnDataLossException(12, 10) } checkError(e, "DELTA_MISSING_FILES_UNEXPECTED_VERSION", "XXKDS", Map("startVersion" -> "12", "earliestVersion" -> "10", "option" -> "failOnDataLoss")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.nestedFieldNotSupported("INSERT clause of MERGE operation", "col1") } checkError(e, "DELTA_UNSUPPORTED_NESTED_FIELD_IN_OPERATION", "0AKDC", Map("operation" -> "INSERT clause of MERGE operation", "fieldName" -> "col1")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.newCheckConstraintViolated(10, "table-1", "sample") } checkError(e, "DELTA_NEW_CHECK_CONSTRAINT_VIOLATION", "23512", Map( "checkConstraint" -> "sample", "tableName" -> "table-1", "numRows" -> "10" )) } { val e = intercept[DeltaRuntimeException] { throw DeltaErrors.failedInferSchema } checkError(e, "DELTA_FAILED_INFER_SCHEMA", "42KD9", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.unexpectedPartialScan(new Path("path-1")) } checkError(e, "DELTA_UNEXPECTED_PARTIAL_SCAN", "KD00A", Map("path" -> "path-1")) } { val e = intercept[DeltaUnsupportedOperationException] { throw DeltaErrors.unrecognizedLogFile(new Path("path-1")) } checkError(e, "DELTA_UNRECOGNIZED_LOGFILE", "KD00B", Map("filename" -> "path-1")) } { val e = intercept[DeltaUnsupportedOperationException] { throw DeltaErrors.unsupportedAbsPathAddFile("path-1") } checkError(e, "DELTA_UNSUPPORTED_ABS_PATH_ADD_FILE", "0AKDC", Map("path" -> "path-1")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.outputModeNotSupportedException("source1", "sample") } checkError(e, "DELTA_UNSUPPORTED_OUTPUT_MODE", "0AKDC", Map("dataSource" -> "source1", "mode" -> "sample")) } { val e = intercept[DeltaAnalysisException] { val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.US) throw DeltaErrors.timestampGreaterThanLatestCommit( new Timestamp(sdf.parse("2022-02-28 10:30:00").getTime), new Timestamp(sdf.parse("2022-02-28 10:00:00").getTime), "2022-02-28 10:00:00") } checkError(e, "DELTA_TIMESTAMP_GREATER_THAN_COMMIT", "42816", Map( "providedTimestamp" -> "2022-02-28 10:30:00.0", "tableName" -> "2022-02-28 10:00:00.0", "maximumTimestamp" -> "2022-02-28 10:00:00")) } { val e = intercept[DeltaAnalysisException] { val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.US) throw DeltaErrors.TimestampEarlierThanCommitRetentionException( new Timestamp(sdf.parse("2022-02-28 10:00:00").getTime), new Timestamp(sdf.parse("2022-02-28 11:00:00").getTime), "2022-02-28 11:00:00") } checkError(e, "DELTA_TIMESTAMP_EARLIER_THAN_COMMIT_RETENTION", "42816", Map( "userTimestamp" -> "2022-02-28 10:00:00.0", "commitTs" -> "2022-02-28 11:00:00.0", "timestampString" -> "2022-02-28 11:00:00")) } { val expr = "1".expr val e = intercept[DeltaAnalysisException] { throw DeltaErrors.timestampInvalid(expr) } checkError(e, "DELTA_TIMESTAMP_INVALID", "42816", Map("expr" -> expr.sql)) } { val version = "null" val e = intercept[DeltaAnalysisException] { throw DeltaErrors.versionInvalid(version) } checkError(e, "DELTA_VERSION_INVALID", "42815", Map("version" -> version)) } { val version = 2 val earliest = 0 val latest = 1 val e = intercept[DeltaAnalysisException] { throw VersionNotFoundException(version, earliest, latest) } checkError(e, "DELTA_VERSION_NOT_FOUND", "22003", Map( "userVersion" -> version.toString, "earliest" -> earliest.toString, "latest" -> latest.toString)) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.notADeltaSourceException("sample") } checkError(e, "DELTA_UNSUPPORTED_SOURCE", "0AKDD", Map("operation" -> "sample", "plan" -> "")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.restoreTimestampGreaterThanLatestException("2022-02-02 12:12:12", "2022-02-02 12:12:10") } checkError(e, "DELTA_CANNOT_RESTORE_TIMESTAMP_GREATER", "22003", Map( "requestedTimestamp" -> "2022-02-02 12:12:12", "latestTimestamp" -> "2022-02-02 12:12:10" )) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.addColumnStructNotFoundException("pos1") } checkError(e, "DELTA_ADD_COLUMN_STRUCT_NOT_FOUND", "42KD3", Map( "position" -> "pos1" )) } { val column = StructField("c0", IntegerType) val other = IntegerType val e = intercept[DeltaAnalysisException] { throw DeltaErrors.addColumnParentNotStructException(column, other) } checkError(e, "DELTA_ADD_COLUMN_PARENT_NOT_STRUCT", "42KD3", Map( "columnName" -> column.name, "other" -> other.toString )) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.updateNonStructTypeFieldNotSupportedException("col1", DataTypes.DateType) } checkError(e, "DELTA_UNSUPPORTED_FIELD_UPDATE_NON_STRUCT", "0AKDC", Map("columnName" -> "`col1`", "dataType" -> "DateType")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.extractReferencesFieldNotFound("struct1", DeltaErrors.updateSchemaMismatchExpression( StructType(Seq(StructField("c0", IntegerType))), StructType(Seq(StructField("c1", IntegerType))) )) } checkError(e, "DELTA_EXTRACT_REFERENCES_FIELD_NOT_FOUND", "XXKDS", Map("fieldName" -> "struct1")) } { val e = intercept[DeltaIndexOutOfBoundsException] { throw DeltaErrors.notNullColumnNotFoundInStruct("struct1") } checkError(e, "DELTA_NOT_NULL_COLUMN_NOT_FOUND_IN_STRUCT", "42K09", Map("struct" -> "struct1")) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.invalidIdempotentWritesOptionsException("someReason") } checkError(e, "DELTA_INVALID_IDEMPOTENT_WRITES_OPTIONS", "42616", Map("reason" -> "someReason")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.operationNotSupportedException("dummyOp") } checkError(e, "DELTA_OPERATION_NOT_ALLOWED", "0AKDC", Map("operation" -> "dummyOp")) } { val s1 = StructType(Seq(StructField("c0", IntegerType))) val s2 = StructType(Seq(StructField("c0", StringType))) val e = intercept[DeltaAnalysisException] { throw DeltaErrors.alterTableSetLocationSchemaMismatchException(s1, s2) } checkError(e, "DELTA_SET_LOCATION_SCHEMA_MISMATCH", "42KD7", Map( "original" -> s1.treeString, "destination" -> s2.treeString, "config" -> DeltaSQLConf.DELTA_ALTER_LOCATION_BYPASS_SCHEMA_CHECK.key)) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.foundDuplicateColumnsException("integer", "col1") } checkError(e, "DELTA_DUPLICATE_COLUMNS_FOUND", "42711", Map("coltype" -> "integer", "duplicateCols" -> "col1")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.subqueryNotSupportedException("dummyOp", "col1") } checkError(e, "DELTA_UNSUPPORTED_SUBQUERY", "0AKDC", Map("operation" -> "dummyOp", "cond" -> "'col1'")) } { val schema = StructType(Array(StructField("foo", IntegerType))) val e = intercept[DeltaAnalysisException] { throw DeltaErrors.foundMapTypeColumnException("dummyKey", "dummyVal", schema) } checkError(e, "DELTA_FOUND_MAP_TYPE_COLUMN", "KD003", Map( "key" -> "dummyKey", "value" -> "dummyVal", "schema" -> schema.treeString )) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.columnOfTargetTableNotFoundInMergeException("target", "dummyCol") } checkError(e, "DELTA_COLUMN_NOT_FOUND_IN_MERGE", "42703", Map("targetCol" -> "target", "colNames" -> "dummyCol")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.multiColumnInPredicateNotSupportedException("dummyOp") } checkError(e, "DELTA_UNSUPPORTED_MULTI_COL_IN_PREDICATE", "0AKDC", Map("operation" -> "dummyOp")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.newNotNullViolated(10L, "table1", UnresolvedAttribute("col1")) } checkError(e, "DELTA_NEW_NOT_NULL_VIOLATION", "23512", Map("numRows" -> "10", "tableName" -> "table1", "colName" -> "col1")) } { val e = intercept[DeltaUnsupportedOperationException] { throw DeltaErrors.modifyAppendOnlyTableException("dummyTable") } checkError(e, "DELTA_CANNOT_MODIFY_APPEND_ONLY", "42809", Map("table_name" -> "dummyTable", "config" -> DeltaConfigs.IS_APPEND_ONLY.key)) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.schemaNotConsistentWithTarget("dummySchema", "targetAttr") } checkError(e, "DELTA_SCHEMA_NOT_CONSISTENT_WITH_TARGET", "XXKDS", Map( "tableSchema" -> "dummySchema", "targetAttrs" -> "targetAttr" )) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.sparkTaskThreadNotFound } checkError(e, "DELTA_SPARK_THREAD_NOT_FOUND", "XXKDS", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.staticPartitionsNotSupportedException } checkError(e, "DELTA_UNSUPPORTED_STATIC_PARTITIONS", "0AKDD", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.unsupportedWriteStagedTable("table1") } checkError(e, "DELTA_UNSUPPORTED_WRITES_STAGED_TABLE", "42807", Map( "tableName" -> "table1" )) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.vacuumBasePathMissingException(new Path("path-1")) } checkError(e, "DELTA_UNSUPPORTED_VACUUM_SPECIFIC_PARTITION", "0AKDC", Map("baseDeltaPath" -> "path-1")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.bloomFilterCreateOnNonExistingColumnsException(Seq("col1", "col2")) } checkError(e, "DELTA_CANNOT_CREATE_BLOOM_FILTER_NON_EXISTING_COL", "42703", Map("unknownCols" -> "col1, col2")) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.zOrderingColumnDoesNotExistException("colName") } checkError(e, "DELTA_ZORDERING_COLUMN_DOES_NOT_EXIST", "42703", Map("columnName" -> "colName")) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.zOrderingOnPartitionColumnException("column1") } checkError(e, "DELTA_ZORDERING_ON_PARTITION_COLUMN", "42P10", Map("colName" -> "column1")) } { val colNames = Seq("col1", "col2") val e = intercept[DeltaAnalysisException] { throw DeltaErrors.zOrderingOnColumnWithNoStatsException(colNames, spark) } checkError(e, "DELTA_ZORDERING_ON_COLUMN_WITHOUT_STATS", "KD00D", Map( "cols" -> "[col1, col2]", "zorderColStatKey" -> DeltaSQLConf.DELTA_OPTIMIZE_ZORDER_COL_STAT_CHECK.key )) } { val e = intercept[DeltaIllegalStateException] { throw MaterializedRowId.missingMetadataException("table_name") } checkError(e, "DELTA_MATERIALIZED_ROW_TRACKING_COLUMN_NAME_MISSING", "22000", Map("rowTrackingColumn" -> "Row ID", "tableName" -> "table_name")) } { val e = intercept[DeltaIllegalStateException] { throw MaterializedRowCommitVersion.missingMetadataException("table_name") } checkError(e, "DELTA_MATERIALIZED_ROW_TRACKING_COLUMN_NAME_MISSING", "22000", Map( "rowTrackingColumn" -> "Row Commit Version", "tableName" -> "table_name" )) } { val path = new Path("a/b") val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.catalogManagedTablePathBasedAccessNotAllowed(path) } checkError(e, "DELTA_PATH_BASED_ACCESS_TO_CATALOG_MANAGED_TABLE_BLOCKED", "KD00G", Map("path" -> path.toString)) } { val e = intercept[DeltaUnsupportedOperationException] { throw DeltaErrors.operationBlockedOnCatalogManagedTable("OPTIMIZE") } checkError(e, "DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION", "0AKDC", Map("operation" -> "OPTIMIZE")) } } // The compiler complains the lambda function is too large if we put all tests in one lambda. test("test DeltaErrors OSS methods more") { { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.schemaNotSetException } checkError(e, "DELTA_SCHEMA_NOT_SET", "KD008", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.schemaNotProvidedException } checkError(e, "DELTA_SCHEMA_NOT_PROVIDED", "42908", Map.empty[String, String]) } { val st1 = StructType(Seq(StructField("a0", IntegerType))) val st2 = StructType(Seq(StructField("b0", IntegerType))) val schemaDiff = SchemaUtils.reportDifferences(st1, st2) .map(_.replace("Specified", "Latest")) val e = intercept[DeltaAnalysisException] { throw DeltaErrors.schemaChangedSinceAnalysis(st1, st2) } checkError(e, "DELTA_SCHEMA_CHANGE_SINCE_ANALYSIS", "KD007", Map( "schemaDiff" -> "Latest schema is missing field(s): a0\nLatest schema has additional field(s): b0", "legacyFlagMessage" -> "")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.generatedColumnsAggregateExpression("1".expr) } checkError(e, "DELTA_AGGREGATE_IN_GENERATED_COLUMN", "42621", Map("sqlExpr" -> "'1'")) } { val path = new Path("somePath") val specifiedColumns = Seq("col1", "col2") val existingColumns = Seq("col3", "col4") val e = intercept[DeltaAnalysisException] { throw DeltaErrors.createTableWithDifferentPartitioningException(path, specifiedColumns, existingColumns) } checkError(e, "DELTA_CREATE_TABLE_WITH_DIFFERENT_PARTITIONING", "42KD7", Map( "path" -> "somePath", "specifiedColumns" -> "col1, col2", "existingColumns" -> "col3, col4" )) } { val path = new Path("a/b") val smaps = Map("abc" -> "xyz") val emaps = Map("def" -> "hjk") val e = intercept[DeltaAnalysisException] { throw DeltaErrors.createTableWithDifferentPropertiesException(path, smaps, emaps) } checkError(e, "DELTA_CREATE_TABLE_WITH_DIFFERENT_PROPERTY", "42KD7", Map( "path" -> path.toString, "specifiedProperties" -> smaps.map { case (k, v) => s"$k=$v" }.mkString("\n"), "existingProperties" -> emaps.map { case (k, v) => s"$k=$v" }.mkString("\n") )) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.unsupportSubqueryInPartitionPredicates() } checkError(e, "DELTA_UNSUPPORTED_SUBQUERY_IN_PARTITION_PREDICATES", "0AKDC", Map.empty[String, String]) } { val e = intercept[DeltaFileNotFoundException] { throw DeltaErrors.emptyDirectoryException("dir") } checkError(e, "DELTA_EMPTY_DIRECTORY", "42K03", Map("directory" -> "dir")) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.replaceWhereUsedWithDynamicPartitionOverwrite() } checkError(e, "DELTA_REPLACE_WHERE_WITH_DYNAMIC_PARTITION_OVERWRITE", "42613", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.replaceWhereUsedInOverwrite() } checkError(e, "DELTA_REPLACE_WHERE_IN_OVERWRITE", "42613", Map.empty[String, String]) } { val schema = StructType(Array(StructField("foo", IntegerType))) val e = intercept[DeltaAnalysisException] { throw DeltaErrors.incorrectArrayAccessByName("right", "wrong", schema) } checkError(e, "DELTA_INCORRECT_ARRAY_ACCESS_BY_NAME", "KD003", Map( "rightName" -> "right", "wrongName" -> "wrong", "schema" -> schema.treeString)) } { val columnPath = "colPath" val other = IntegerType val column = Seq("col1", "col2") val schema = StructType(Array(StructField("foo", IntegerType))) val e = intercept[DeltaAnalysisException] { throw DeltaErrors.columnPathNotNested(columnPath, other, column, schema) } checkError(e, "DELTA_COLUMN_PATH_NOT_NESTED", "42704", Map( "columnPath" -> columnPath, "other" -> other.toString, "column" -> column.mkString("."), "schema" -> schema.treeString)) } { val e = intercept[DeltaUnsupportedOperationException] { throw DeltaErrors.multipleSourceRowMatchingTargetRowInMergeException(spark) } val docLink = generateDocsLink(multipleSourceRowMatchingTargetRowInMergeUrl) checkError(e, "DELTA_MULTIPLE_SOURCE_ROW_MATCHING_TARGET_ROW_IN_MERGE", "21506", Map("usageReference" -> docLink)) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.showPartitionInNotPartitionedTable("table") } checkError(e, "DELTA_SHOW_PARTITION_IN_NON_PARTITIONED_TABLE", "42809", Map("tableName" -> "table")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.showPartitionInNotPartitionedColumn(Set("col1", "col2")) } checkError(e, "DELTA_SHOW_PARTITION_IN_NON_PARTITIONED_COLUMN", "42P10", Map("badCols" -> "[col1, col2]")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.duplicateColumnOnInsert() } checkError(e, "DELTA_DUPLICATE_COLUMNS_ON_INSERT", "42701", Map.empty[String, String]) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.timeTravelInvalidBeginValue("key", new Throwable) } checkError(e, "DELTA_TIME_TRAVEL_INVALID_BEGIN_VALUE", "42604", Map("timeTravelKey" -> "key")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.metadataAbsentException() } checkError(e, "DELTA_METADATA_ABSENT", "XXKDS", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw new DeltaAnalysisException(errorClass = "DELTA_CANNOT_USE_ALL_COLUMNS_FOR_PARTITION", Array.empty) } checkError(e, "DELTA_CANNOT_USE_ALL_COLUMNS_FOR_PARTITION", "428FT", Map.empty[String, String]) } { val e = intercept[DeltaIOException] { throw DeltaErrors.failedReadFileFooter("test.txt", null) } checkError(e, "DELTA_FAILED_READ_FILE_FOOTER", "KD001", Map("currentFile" -> "test.txt")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.failedScanWithHistoricalVersion(123) } checkError(e, "DELTA_FAILED_SCAN_WITH_HISTORICAL_VERSION", "KD002", Map("historicalVersion" -> "123")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.failedRecognizePredicate("select ALL", new Throwable()) } checkError(e, "DELTA_FAILED_RECOGNIZE_PREDICATE", "42601", Map("predicate" -> "select ALL")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.failedFindAttributeInOutputColumns("col1", "col2,col3,col4") } checkError(e, "DELTA_FAILED_FIND_ATTRIBUTE_IN_OUTPUT_COLUMNS", "42703", Map("newAttributeName" -> "col1", "targetOutputColumns" -> "col2,col3,col4")) } { val col = "col1" val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.failedFindPartitionColumnInOutputPlan(col) } checkError(e, "DELTA_FAILED_FIND_PARTITION_COLUMN_IN_OUTPUT_PLAN", "XXKDS", Map("partitionColumn" -> col)) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.deltaTableFoundInExecutor() } checkError(e, "DELTA_TABLE_FOUND_IN_EXECUTOR", "XXKDS", Map.empty[String, String]) } { val e = intercept[DeltaFileAlreadyExistsException] { throw DeltaErrors.fileAlreadyExists("file.txt") } checkError(e, "DELTA_FILE_ALREADY_EXISTS", "42K04", Map("path" -> "file.txt")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.configureSparkSessionWithExtensionAndCatalog(Some(new Throwable())) } checkError(e, "DELTA_CONFIGURE_SPARK_SESSION_WITH_EXTENSION_AND_CATALOG", "56038", Map( "sparkSessionExtensionName" -> classOf[DeltaSparkSessionExtension].getName, "catalogKey" -> SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key, "catalogClassName" -> classOf[DeltaCatalog].getName )) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.cdcNotAllowedInThisVersion() } checkError(e, "DELTA_CDC_NOT_ALLOWED_IN_THIS_VERSION", "0AKDC", Map.empty[String, String]) } { val ident = TableIdentifier("view1") val e = intercept[DeltaAnalysisException] { throw DeltaErrors.convertNonParquetTablesException(ident, "source1") } checkError(e, "DELTA_CONVERT_NON_PARQUET_TABLE", "0AKDC", Map("sourceName" -> "source1", "tableId" -> "`view1`")) } { val from = StructType(Seq(StructField("c0", IntegerType))) val to = StructType(Seq(StructField("c1", IntegerType))) val e = intercept[DeltaAnalysisException] { throw DeltaErrors.updateSchemaMismatchExpression(from, to) } checkError(e, "DELTA_UPDATE_SCHEMA_MISMATCH_EXPRESSION", "42846", Map("fromCatalog" -> "struct", "toCatalog" -> "struct")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.removeFileCDCMissingExtendedMetadata("someFile") } checkError(e, "DELTA_REMOVE_FILE_CDC_MISSING_EXTENDED_METADATA", "XXKDS", Map("file" -> "someFile")) } { val columnName = "c0" val colMatches = Seq(StructField("c0", IntegerType)) val e = intercept[DeltaAnalysisException] { throw DeltaErrors.ambiguousPartitionColumnException(columnName, colMatches) } checkError(e, "DELTA_AMBIGUOUS_PARTITION_COLUMN", "42702", Map("column" -> "`c0`", "colMatches" -> "[`c0`]")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.truncateTablePartitionNotSupportedException } checkError(e, "DELTA_TRUNCATE_TABLE_PARTITION_NOT_SUPPORTED", "0AKDC", Map.empty[String, String]) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.invalidFormatFromSourceVersion(100, 10) } checkError(e, "DELTA_INVALID_FORMAT_FROM_SOURCE_VERSION", "XXKDS", Map("expectedVersion" -> "10", "realVersion" -> "100")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.emptyDataException } checkError(e, "DELTA_EMPTY_DATA", "428GU", Map.empty[String, String]) } { val path = "somePath" val parsedCol = "col1" val expectedCol = "col2" val e = intercept[DeltaAnalysisException] { throw DeltaErrors.unexpectedPartitionColumnFromFileNameException(path, parsedCol, expectedCol) } checkError(e, "DELTA_UNEXPECTED_PARTITION_COLUMN_FROM_FILE_NAME", "KD009", Map("expectedCol" -> "`col2`", "parsedCol" -> "`col1`", "path" -> "somePath")) } { val path = "somePath" val parsedCols = Seq("col1", "col2") val expectedCols = Seq("col3", "col4") val e = intercept[DeltaAnalysisException] { throw DeltaErrors.unexpectedNumPartitionColumnsFromFileNameException(path, parsedCols, expectedCols) } checkError(e, "DELTA_UNEXPECTED_NUM_PARTITION_COLUMNS_FROM_FILE_NAME", "KD009", Map( "expectedCols" -> "[`col3`, `col4`]", "path" -> "somePath", "parsedCols" -> "[`col1`, `col2`]", "parsedColsSize" -> "2", "expectedColsSize" -> "2")) } { val version = 100L val removedFile = "file" val dataPath = "tablePath" val e = intercept[DeltaUnsupportedOperationException] { throw DeltaErrors.deltaSourceIgnoreDeleteError(version, removedFile, dataPath) } checkError(e, "DELTA_SOURCE_IGNORE_DELETE", "0A000", Map("removedFile" -> "file", "version" -> "100", "dataPath" -> "tablePath")) } { val tableId = "someTableId" val tableLocation = "path" val e = intercept[DeltaAnalysisException] { throw DeltaErrors.createTableWithNonEmptyLocation(tableId, tableLocation) } checkError(e, "DELTA_CREATE_TABLE_WITH_NON_EMPTY_LOCATION", "42601", Map("tableId" -> "someTableId", "tableLocation" -> "path")) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.maxArraySizeExceeded() } checkError(e, "DELTA_MAX_ARRAY_SIZE_EXCEEDED", "42000", Map.empty[String, String]) } { val unknownColumns = Seq("col1", "col2") val e = intercept[DeltaAnalysisException] { throw DeltaErrors.bloomFilterDropOnNonExistingColumnsException(unknownColumns) } checkError(e, "DELTA_BLOOM_FILTER_DROP_ON_NON_EXISTING_COLUMNS", "42703", Map("unknownColumns" -> unknownColumns.mkString(", "))) } { val dataFilters = "filters" val e = intercept[DeltaAnalysisException] { throw DeltaErrors.replaceWhereWithFilterDataChangeUnset(dataFilters) } checkError(e, "DELTA_REPLACE_WHERE_WITH_FILTER_DATA_CHANGE_UNSET", "42613", Map("dataFilters" -> dataFilters)) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.missingTableIdentifierException("read") } checkError(e, "DELTA_OPERATION_MISSING_PATH", "42601", Map("operation" -> "read")) } { val column = StructField("c0", IntegerType) val e = intercept[DeltaAnalysisException] { throw DeltaErrors.cannotUseDataTypeForPartitionColumnError(column) } checkError(e, "DELTA_INVALID_PARTITION_COLUMN_TYPE", "42996", Map("name" -> "c0", "dataType" -> "IntegerType")) } { val catalogPartitionSchema = StructType(Seq(StructField("a", IntegerType))) val userPartitionSchema = StructType(Seq(StructField("b", StringType))) val e = intercept[DeltaAnalysisException] { throw DeltaErrors.unexpectedPartitionSchemaFromUserException(catalogPartitionSchema, userPartitionSchema) } checkError(e, "DELTA_UNEXPECTED_PARTITION_SCHEMA_FROM_USER", "KD009", Map( "catalogPartitionSchema" -> catalogPartitionSchema.treeString, "userPartitionSchema" -> userPartitionSchema.treeString)) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.invalidInterval("interval1") } checkError(e, "DELTA_INVALID_INTERVAL", "22006", Map("interval" -> "interval1")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.cdcWriteNotAllowedInThisVersion } checkError(e, "DELTA_CHANGE_TABLE_FEED_DISABLED", "42807", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.specifySchemaAtReadTimeException } checkError(e, "DELTA_UNSUPPORTED_SCHEMA_DURING_READ", "0AKDC", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.readSourceSchemaConflictException } checkError(e, "DELTA_READ_SOURCE_SCHEMA_CONFLICT", "42K07", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.unexpectedDataChangeException("operation1") } checkError(e, "DELTA_DATA_CHANGE_FALSE", "0AKDE", Map("op" -> "operation1")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.noStartVersionForCDC } checkError(e, "DELTA_NO_START_FOR_CDC_READ", "42601", Map.empty[String, String]) } { val e = intercept[DeltaUnsupportedOperationException] { throw DeltaErrors.unrecognizedColumnChange("change1") } checkError(e, "DELTA_UNRECOGNIZED_COLUMN_CHANGE", "42601", Map("otherClass" -> "change1")) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.nullRangeBoundaryInCDCRead() } checkError(e, "DELTA_CDC_READ_NULL_RANGE_BOUNDARY", "22004", Map.empty[String, String]) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.endBeforeStartVersionInCDC(2, 1) } checkError(e, "DELTA_INVALID_CDC_RANGE", "22003", Map("start" -> "2", "end" -> "1")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.unexpectedChangeFilesFound("a.parquet") } checkError(e, "DELTA_UNEXPECTED_CHANGE_FILES_FOUND", "XXKDS", Map("fileList" -> "a.parquet")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.logFailedIntegrityCheck(2, "option1") } checkError(e, "DELTA_TXN_LOG_FAILED_INTEGRITY", "XXKDS", Map( "version" -> "2", "mismatchStringOpt" -> "option1" )) } { val path = new Path("parent", "child") val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.checkpointNonExistTable(path) } checkError(e, "DELTA_CHECKPOINT_NON_EXIST_TABLE", "42K03", Map("path" -> path.toString)) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.unsupportedDeepCloneException() } checkError(e, "DELTA_UNSUPPORTED_DEEP_CLONE", "0A000", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.viewInDescribeDetailException(TableIdentifier("customer")) } checkError(e, "DELTA_UNSUPPORTED_DESCRIBE_DETAIL_VIEW", "42809", Map("view" -> "`customer`")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.pathAlreadyExistsException(new Path(path)) } checkError(e, "DELTA_PATH_EXISTS", "42K04", Map("path" -> "/sample/path")) } { val e = intercept[DeltaAnalysisException] { throw new DeltaAnalysisException( errorClass = "DELTA_MERGE_MISSING_WHEN", messageParameters = Array.empty ) } checkError(e, "DELTA_MERGE_MISSING_WHEN", "42601", Map.empty[String, String]) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.unrecognizedFileAction("invalidAction", "invalidClass") } checkError(e, "DELTA_UNRECOGNIZED_FILE_ACTION", "XXKDS", Map( "action" -> "invalidAction", "actionClass" -> "invalidClass" )) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.streamWriteNullTypeException } checkError(e, "DELTA_NULL_SCHEMA_IN_STREAMING_WRITE", "42P18", Map.empty[String, String]) } { val expr = "1".expr val e = intercept[DeltaIllegalArgumentException] { throw new DeltaIllegalArgumentException( errorClass = "DELTA_UNEXPECTED_ACTION_EXPRESSION", messageParameters = Array(s"$expr")) } checkError(e, "DELTA_UNEXPECTED_ACTION_EXPRESSION", "42601", Map("expression" -> "1")) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.unexpectedAlias("alias1") } checkError(e, "DELTA_UNEXPECTED_ALIAS", "XXKDS", Map("alias" -> "alias1")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.unexpectedProject("project1") } checkError(e, "DELTA_UNEXPECTED_PROJECT", "XXKDS", Map("project" -> "project1")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.nullableParentWithNotNullNestedField } checkError(e, "DELTA_NOT_NULL_NESTED_FIELD", "0A000", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.useAddConstraints } checkError(e, "DELTA_ADD_CONSTRAINTS", "0A000", Map.empty[String, String]) } { val e = intercept[DeltaUnsupportedOperationException] { throw DeltaErrors.deltaSourceIgnoreChangesError(10, "removedFile", "tablePath") } checkError(e, "DELTA_SOURCE_TABLE_IGNORE_CHANGES", "0A000", Map( "version" -> "10", "file" -> "removedFile", "dataPath" -> "tablePath" )) } { val limit = "someLimit" val e = intercept[DeltaUnsupportedOperationException] { throw DeltaErrors.unknownReadLimit(limit) } checkError(e, "DELTA_UNKNOWN_READ_LIMIT", "42601", Map("limit" -> limit)) } { val privilege = "unknown" val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.unknownPrivilege(privilege) } checkError(e, "DELTA_UNKNOWN_PRIVILEGE", "42601", Map("privilege" -> privilege)) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.deltaLogAlreadyExistsException("somePath") } checkError(e, "DELTA_LOG_ALREADY_EXISTS", "42K04", Map("path" -> "somePath")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.missingPartFilesException(10L, new FileNotFoundException("reason")) } checkError(e, "DELTA_MISSING_PART_FILES", "42KD6", Map("version" -> "10")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.checkConstraintNotBoolean("name1", "expr1") } checkError(e, "DELTA_NON_BOOLEAN_CHECK_CONSTRAINT", "42621", Map("name" -> "name1", "expr" -> "expr1")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.checkpointMismatchWithSnapshot } checkError(e, "DELTA_CHECKPOINT_SNAPSHOT_MISMATCH", "XXKDS", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.notADeltaTableException("operation1") } checkError(e, "DELTA_ONLY_OPERATION", "0AKDD", Map("operation" -> "operation1")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.dropNestedColumnsFromNonStructTypeException(StringType) } checkError(e, "DELTA_UNSUPPORTED_DROP_NESTED_COLUMN_FROM_NON_STRUCT_TYPE", "0AKDC", Map("struct" -> "StringType")) } { val locations = Seq("location1", "location2") val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.cannotSetLocationMultipleTimes(locations) } checkError(e, "DELTA_CANNOT_SET_LOCATION_MULTIPLE_TIMES", "XXKDS", Map("location" -> "List(location1, location2)")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.metadataAbsentForExistingCatalogTable("tblName", "file://path/to/table") } checkError(e, "DELTA_METADATA_ABSENT_EXISTING_CATALOG_TABLE", "XXKDS", Map( "tableName" -> "tblName", "tablePath" -> "file://path/to/table", "tableNameForDropCmd" -> "tblName" )) } { val e = intercept[DeltaStreamingNonAdditiveSchemaIncompatibleException] { throw DeltaErrors.blockStreamingReadsWithIncompatibleNonAdditiveSchemaChanges( spark, StructType.fromDDL("id int"), StructType.fromDDL("id2 int"), detectedDuringStreaming = true ) } val docLink = generateDocsLink("/versioning.html#column-mapping") checkError(e, "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE_USE_SCHEMA_LOG", "42KD4", Map( "docLink" -> docLink, "readSchema" -> StructType.fromDDL("id int").json, "incompatibleSchema" -> StructType.fromDDL("id2 int").json )) assert(e.additionalProperties("detectedDuringStreaming").toBoolean) } { val e = intercept[DeltaStreamingNonAdditiveSchemaIncompatibleException] { throw DeltaErrors.blockStreamingReadsWithIncompatibleNonAdditiveSchemaChanges( spark, StructType.fromDDL("id int"), StructType.fromDDL("id2 int"), detectedDuringStreaming = false ) } val docLink = generateDocsLink("/versioning.html#column-mapping") checkError(e, "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE_USE_SCHEMA_LOG", "42KD4", Map( "docLink" -> docLink, "readSchema" -> StructType.fromDDL("id int").json, "incompatibleSchema" -> StructType.fromDDL("id2 int").json )) assert(!e.additionalProperties("detectedDuringStreaming").toBoolean) } { val e = intercept[DeltaRuntimeException] { throw DeltaErrors.cannotContinueStreamingPostSchemaEvolution( nonAdditiveSchemaChangeOpType = "RENAME AND TYPE WIDENING", previousSchemaChangeVersion = 0, currentSchemaChangeVersion = 1, readerOptionsUnblock = Seq("allowSourceColumnRename", "allowSourceColumnTypeChange"), sqlConfsUnblock = Seq( "spark.databricks.delta.streaming.allowSourceColumnRename", "spark.databricks.delta.streaming.allowSourceColumnTypeChange"), checkpointHash = 15, prettyColumnChangeDetails = "some column details") } checkError(e, "DELTA_STREAMING_CANNOT_CONTINUE_PROCESSING_POST_SCHEMA_EVOLUTION", parameters = Map( "opType" -> "RENAME AND TYPE WIDENING", "previousSchemaChangeVersion" -> "0", "currentSchemaChangeVersion" -> "1", "columnChangeDetails" -> "some column details", "unblockChangeOptions" -> s""" .option("allowSourceColumnRename", "1") | .option("allowSourceColumnTypeChange", "1")""".stripMargin, "unblockStreamOptions" -> s""" .option("allowSourceColumnRename", "always") | .option("allowSourceColumnTypeChange", "always")""".stripMargin, "unblockChangeConfs" -> s""" SET spark.databricks.delta.streaming.allowSourceColumnRename.ckpt_15 = 1; | SET spark.databricks.delta.streaming.allowSourceColumnTypeChange.ckpt_15 = 1;""".stripMargin, "unblockStreamConfs" -> s""" SET spark.databricks.delta.streaming.allowSourceColumnRename.ckpt_15 = "always"; | SET spark.databricks.delta.streaming.allowSourceColumnTypeChange.ckpt_15 = "always";""".stripMargin, "unblockAllConfs" -> s""" SET spark.databricks.delta.streaming.allowSourceColumnRename = "always"; | SET spark.databricks.delta.streaming.allowSourceColumnTypeChange = "always";""".stripMargin ) ) } { val e = intercept[DeltaUnsupportedOperationException] { throw DeltaErrors.blockColumnMappingAndCdcOperation(DeltaOperations.ManualUpdate) } checkError(e, "DELTA_BLOCK_COLUMN_MAPPING_AND_CDC_OPERATION", "42KD4", Map("opName" -> "Manual Update")) } { val options = Map( "foo" -> "1", "bar" -> "2" ) val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.unsupportedDeltaTableForPathHadoopConf(options) } val prefixStr = DeltaTableUtils.validDeltaTableHadoopPrefixes.mkString("[", ",", "]") checkError(e, "DELTA_TABLE_FOR_PATH_UNSUPPORTED_HADOOP_CONF", "0AKDC", Map("allowedPrefixes" -> prefixStr, "unsupportedOptions" -> "foo -> 1,bar -> 2")) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.cloneOnRelativePath("somePath") } checkError(e, "DELTA_INVALID_CLONE_PATH", "22KD1", Map("path" -> "somePath")) } { val e = intercept[AnalysisException] { throw DeltaErrors.cloneFromUnsupportedSource( "table-0", "CSV") } checkError(e, "DELTA_CLONE_UNSUPPORTED_SOURCE", "0AKDC", Map("name" -> "table-0", "format" -> "CSV")) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.cloneReplaceUnsupported(TableIdentifier("customer")) } checkError(e, "DELTA_UNSUPPORTED_CLONE_REPLACE_SAME_TABLE", "0AKDC", Map("tableName" -> "`customer`")) } { val e = intercept[DeltaIllegalArgumentException] { throw DeltaErrors.cloneAmbiguousTarget("external-location", TableIdentifier("table1")) } checkError(e, "DELTA_CLONE_AMBIGUOUS_TARGET", "42613", Map("externalLocation" -> "external-location", "targetIdentifier" -> "`table1`")) } { DeltaTableValueFunctions.supportedFnNames.foreach { fnName => { val fnCall = s"${fnName}()" val e = intercept[DeltaAnalysisException] { sql(s"SELECT * FROM $fnCall").collect() } checkError(e, "INCORRECT_NUMBER_OF_ARGUMENTS", "42605", Map("failure" -> "not enough args", "functionName" -> fnName, "minArgs" -> "2", "maxArgs" -> "3"), ExpectedContext(fragment = fnCall, start = 14, stop = 14 + fnCall.length - 1)) } { val fnCall = s"${fnName}(1, 2, 3, 4, 5)" val e = intercept[DeltaAnalysisException] { sql(s"SELECT * FROM ${fnCall}").collect() } checkError(e, "INCORRECT_NUMBER_OF_ARGUMENTS", "42605", Map("failure" -> "too many args", "functionName" -> fnName, "minArgs" -> "2", "maxArgs" -> "3"), ExpectedContext(fragment = fnCall, start = 14, stop = 14 + fnCall.length - 1)) } } } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.invalidTableValueFunction("invalid1") } checkError(e, "DELTA_INVALID_TABLE_VALUE_FUNCTION", "22000", Map("function" -> "invalid1")) } { val e = intercept[DeltaAnalysisException] { throw new DeltaAnalysisException( errorClass = "WRONG_COLUMN_DEFAULTS_FOR_DELTA_FEATURE_NOT_ENABLED", messageParameters = Array("ALTER TABLE")) } checkError(e, "WRONG_COLUMN_DEFAULTS_FOR_DELTA_FEATURE_NOT_ENABLED", "0AKDE", Map("commandType" -> "ALTER TABLE")) } { val e = intercept[DeltaAnalysisException] { throw new DeltaAnalysisException( errorClass = "WRONG_COLUMN_DEFAULTS_FOR_DELTA_ALTER_TABLE_ADD_COLUMN_NOT_SUPPORTED", messageParameters = Array.empty) } checkError(e, "WRONG_COLUMN_DEFAULTS_FOR_DELTA_ALTER_TABLE_ADD_COLUMN_NOT_SUPPORTED", "0AKDC", Map.empty[String, String]) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.missingCommitInfo("featureName", "1225") } checkError(e, "DELTA_MISSING_COMMIT_INFO", "KD004", Map("featureName" -> "featureName", "version" -> "1225")) } { val e = intercept[DeltaIllegalStateException] { throw DeltaErrors.missingCommitTimestamp("1225") } checkError(e, "DELTA_MISSING_COMMIT_TIMESTAMP", "KD004", Map("featureName" -> "inCommitTimestamp", "version" -> "1225")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.invalidConstraintName("foo") } checkError(e, "_LEGACY_ERROR_TEMP_DELTA_0001", None, Map("name" -> "foo")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.bloomFilterInvalidParameterValueException("foo") } checkError(e, "_LEGACY_ERROR_TEMP_DELTA_0002", None, Map("message" -> "foo")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.convertMetastoreMetadataMismatchException( tableProperties = Map("delta.prop1" -> "foo"), deltaConfiguration = Map("delta.config1" -> "bar")) } checkError(e, "_LEGACY_ERROR_TEMP_DELTA_0003", None, Map( "tableProperties" -> "[delta.prop1=foo]", "configuration" -> "[delta.config1=bar]", "metadataCheckSqlConf" -> DeltaSQLConf.DELTA_CONVERT_METADATA_CHECK_ENABLED.key)) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.restoreTimestampBeforeEarliestException("2022-02-02 12:12:12", "2022-02-02 12:12:14") } checkError(e, "DELTA_CANNOT_RESTORE_TIMESTAMP_EARLIER", "22003", Map( "requestedTimestamp" -> "2022-02-02 12:12:12", "earliestTimestamp" -> "2022-02-02 12:12:14" )) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.viewNotSupported("FOO_OP") } checkError(e, "DELTA_OPERATION_ON_VIEW_NOT_ALLOWED", "0AKDC", Map("operation" -> "FOO_OP")) } { val expr = "1".expr val e = intercept[DeltaAnalysisException] { throw DeltaErrors.generatedColumnsNonDeterministicExpression(expr) } checkError(e, "DELTA_NON_DETERMINISTIC_EXPRESSION_IN_GENERATED_COLUMN", "42621", Map("expr" -> "'1'")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.columnBuilderMissingDataType("col1") } checkError(e, "DELTA_COLUMN_MISSING_DATA_TYPE", "42601", Map("colName" -> "`col1`")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.foundViolatingConstraintsForColumnChange( "col1", Map("foo" -> "bar")) } checkError(e, "DELTA_CONSTRAINT_DEPENDENT_COLUMN_CHANGE", "42K09", Map("columnName" -> "col1", "constraints" -> "foo -> bar")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.createTableMissingTableNameOrLocation() } checkError(e, "DELTA_CREATE_TABLE_MISSING_TABLE_NAME_OR_LOCATION", "42601", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.createTableIdentifierLocationMismatch("delta.`somePath1`", "somePath2") } checkError(e, "DELTA_CREATE_TABLE_IDENTIFIER_LOCATION_MISMATCH", "0AKDC", Map("identifier" -> "delta.`somePath1`", "location" -> "somePath2")) } { val schema = StructType(Seq(StructField("col1", IntegerType))) val e = intercept[DeltaAnalysisException] { throw DeltaErrors.dropColumnOnSingleFieldSchema(schema) } checkError(e, "DELTA_DROP_COLUMN_ON_SINGLE_FIELD_SCHEMA", "0AKDC", Map("schema" -> schema.treeString)) } { val schema = StructType(Seq(StructField("col1", IntegerType))) val e = intercept[DeltaAnalysisException] { throw DeltaErrors.errorFindingColumnPosition(Seq("col2"), schema, "foo") } checkError(e, "_LEGACY_ERROR_TEMP_DELTA_0008", None, Map( "column" -> "col2", "schema" -> schema.treeString, "message" -> "foo")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.foundViolatingGeneratedColumnsForColumnChange( columnName = "col1", generatedColumns = Map("col2" -> "col1 + 1", "col3" -> "col1 + 2")) } checkError(e, "DELTA_GENERATED_COLUMNS_DEPENDENT_COLUMN_CHANGE", "42K09", Map( "columnName" -> "col1", "generatedColumns" -> "col2 -> col1 + 1\ncol3 -> col1 + 2" )) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.identityColumnInconsistentMetadata("col1", true, true, true) } checkError(e, "_LEGACY_ERROR_TEMP_DELTA_0006", None, Map( "colName" -> "col1", "hasStart" -> "true", "hasStep" -> "true", "hasInsert" -> "true")) } { // Test MetadataMismatchErrorBuilder with single sub-error (schema mismatch) val errorBuilder = new MetadataMismatchErrorBuilder() val schema1 = StructType(Seq(StructField("c0", IntegerType))) val schema2 = StructType(Seq(StructField("c0", StringType))) errorBuilder.addSchemaMismatch(schema1, schema2, "id") val e = intercept[DeltaAnalysisException] { errorBuilder.finalizeAndThrow(spark.sessionState.conf) } checkError(e, "DELTA_METADATA_MISMATCH", "42KDG", Map.empty[String, String]) // Verify complete message format with main message + sub-error bullet val message = e.getMessage assert(message.contains( """[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table. |- A schema mismatch detected when writing to the Delta table (Table ID: id). |To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'. |For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details. | |Table schema: |root | |-- c0: integer (nullable = true) | | |Data schema: |root | |-- c0: string (nullable = true) |""".stripMargin)) } // Test with multiple sub-errors { val errorBuilder = new MetadataMismatchErrorBuilder() val schema1 = StructType(Seq(StructField("c0", IntegerType))) val schema2 = StructType(Seq(StructField("c0", StringType))) errorBuilder.addSchemaMismatch(schema1, schema2, "test-id") errorBuilder.addPartitioningMismatch(Seq("part1"), Seq("part2")) errorBuilder.addOverwriteBit() val e = intercept[DeltaAnalysisException] { errorBuilder.finalizeAndThrow(spark.sessionState.conf) } checkError(e, "DELTA_METADATA_MISMATCH", "42KDG", Map.empty[String, String]) // Verify complete message format with main message + three sub-error bullets val message = e.getMessage assert(message.contains( """[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table. |- A schema mismatch detected when writing to the Delta table (Table ID: test-id). |To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'. |For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details. | |Table schema: |root | |-- c0: integer (nullable = true) | | |Data schema: |root | |-- c0: string (nullable = true) | | |- Partition columns do not match the partition columns of the table. |Given: [`part2`] |Table: [`part1`] | |- To overwrite your schema or change partitioning, please set: '.option("overwriteSchema", "true")'. |Note that the schema can't be overwritten when using 'replaceWhere'.""".stripMargin)) } // Test with partitioning mismatch only { val errorBuilder = new MetadataMismatchErrorBuilder() errorBuilder.addPartitioningMismatch(Seq("year", "month"), Seq("date")) val e = intercept[DeltaAnalysisException] { errorBuilder.finalizeAndThrow(spark.sessionState.conf) } checkError(e, "DELTA_METADATA_MISMATCH", "42KDG", Map.empty[String, String]) // Verify complete message format with main message + one sub-error bullet val message = e.getMessage assert(message.contains( """[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table. |- Partition columns do not match the partition columns of the table. |Given: [`date`] |Table: [`year`, `month`] |""".stripMargin)) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.mergeAddVoidColumn("fooCol") } checkError(e, "DELTA_MERGE_ADD_VOID_COLUMN", "42K09", Map("newColumn" -> "`fooCol`")) } { val e = intercept[io.delta.exceptions.ConcurrentAppendException] { throw org.apache.spark.sql.delta.DeltaErrors .concurrentAppendException(None, "t", -1, partitionOpt = None) } checkError(e, "DELTA_CONCURRENT_APPEND.WITHOUT_HINT", "2D521", Map( "operation" -> "TRANSACTION", "tableName" -> "t", "version" -> "-1", "docLink" -> generateDocsLink("/concurrency-control.html") ) ) } { val e = intercept[io.delta.exceptions.ConcurrentAppendException] { throw org.apache.spark.sql.delta.DeltaErrors .concurrentAppendException(None, "t", -1, partitionOpt = Some("p1")) } checkError(e, "DELTA_CONCURRENT_APPEND.WITH_PARTITION_HINT", "2D521", Map("operation" -> "TRANSACTION", "tableName" -> "t", "version" -> "-1", "partitionValues" -> "p1", "docLink" -> generateDocsLink("/concurrency-control.html"))) } { val e = intercept[io.delta.exceptions.ConcurrentDeleteReadException] { throw org.apache.spark.sql.delta.DeltaErrors .concurrentDeleteReadException(None, "t", -1, partitionOpt = None) } checkError(e, "DELTA_CONCURRENT_DELETE_READ.WITHOUT_HINT", "2D521", Map("operation" -> "TRANSACTION", "tableName" -> "t", "version" -> "-1", "docLink" -> generateDocsLink("/concurrency-control.html"))) } { val e = intercept[io.delta.exceptions.ConcurrentDeleteReadException] { throw org.apache.spark.sql.delta.DeltaErrors .concurrentDeleteReadException(None, "t", -1, partitionOpt = Some("p1")) } checkError(e, "DELTA_CONCURRENT_DELETE_READ.WITH_PARTITION_HINT", "2D521", Map("operation" -> "TRANSACTION", "tableName" -> "t", "version" -> "-1", "partitionValues" -> "p1", "docLink" -> generateDocsLink("/concurrency-control.html"))) } { val e = intercept[io.delta.exceptions.ConcurrentDeleteDeleteException] { throw org.apache.spark.sql.delta.DeltaErrors .concurrentDeleteDeleteException(None, "t", -1, partitionOpt = None) } checkError(e, "DELTA_CONCURRENT_DELETE_DELETE.WITHOUT_HINT", "2D521", Map("operation" -> "TRANSACTION", "tableName" -> "t", "version" -> "-1", "docLink" -> generateDocsLink("/concurrency-control.html"))) } { val e = intercept[io.delta.exceptions.ConcurrentDeleteDeleteException] { throw org.apache.spark.sql.delta.DeltaErrors .concurrentDeleteDeleteException(None, "t", -1, partitionOpt = Some("p1")) } checkError(e, "DELTA_CONCURRENT_DELETE_DELETE.WITH_PARTITION_HINT", "2D521", Map("operation" -> "TRANSACTION", "tableName" -> "t", "version" -> "-1", "partitionValues" -> "p1", "docLink" -> generateDocsLink("/concurrency-control.html"))) } { val e = intercept[io.delta.exceptions.ConcurrentTransactionException] { throw org.apache.spark.sql.delta.DeltaErrors.concurrentTransactionException(None) } checkError(e, "DELTA_CONCURRENT_TRANSACTION", "2D521", Map.empty[String, String]) assert(e.getMessage.contains("This error occurs when multiple streaming queries are using " + "the same checkpoint to write into this table. Did you run multiple instances of the " + "same streaming query at the same time?")) } { val e = intercept[io.delta.exceptions.ConcurrentWriteException] { throw org.apache.spark.sql.delta.DeltaErrors.concurrentWriteException(None) } checkError(e, "DELTA_CONCURRENT_WRITE", "2D521", Map.empty[String, String]) assert(e.getMessage.contains("A concurrent transaction has written new data since the " + "current transaction read the table.")) } { val e = intercept[io.delta.exceptions.ProtocolChangedException] { throw org.apache.spark.sql.delta.DeltaErrors.protocolChangedException(None) } checkError(e, "DELTA_PROTOCOL_CHANGED", "2D521", Map.empty[String, String]) assert(e.getMessage.contains("The protocol version of the Delta table has been changed " + "by a concurrent update.")) } { val e = intercept[io.delta.exceptions.MetadataChangedException] { throw org.apache.spark.sql.delta.DeltaErrors.metadataChangedException(None) } checkError(e, "DELTA_METADATA_CHANGED", "2D521", Map.empty[String, String]) assert(e.getMessage.contains("The metadata of the Delta table has been changed by a " + "concurrent update.")) } { val e = intercept[DeltaAnalysisException] { throw new DeltaAnalysisException( errorClass = "_LEGACY_ERROR_TEMP_DELTA_0009", messageParameters = Array("prefixMsg - ")) } checkError(e, "_LEGACY_ERROR_TEMP_DELTA_0009", None, Map("optionalPrefixMessage" -> "prefixMsg - ")) } { val expr = "someExp".expr val e = intercept[DeltaAnalysisException] { throw new DeltaAnalysisException( errorClass = "_LEGACY_ERROR_TEMP_DELTA_0010", messageParameters = Array("prefixMsg - ", expr.sql)) } checkError(e, "_LEGACY_ERROR_TEMP_DELTA_0010", None, Map("optionalPrefixMessage" -> "prefixMsg - ", "expression" -> "'someExp'")) } { val exprs = Seq("1".expr, "2".expr) val e = intercept[DeltaAnalysisException] { throw new DeltaAnalysisException( errorClass = "_LEGACY_ERROR_TEMP_DELTA_0012", messageParameters = Array(exprs.mkString(","))) } checkError(e, "_LEGACY_ERROR_TEMP_DELTA_0012", None, Map("expression" -> exprs.mkString(","))) } { val unsupportedDataType = IntegerType val e = intercept[DeltaUnsupportedOperationException] { throw DeltaErrors.identityColumnDataTypeNotSupported(unsupportedDataType) } checkError(e, "DELTA_IDENTITY_COLUMNS_UNSUPPORTED_DATA_TYPE", "428H2", Map("dataType" -> "integer")) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.identityColumnIllegalStep() } checkError(e, "DELTA_IDENTITY_COLUMNS_ILLEGAL_STEP", "42611", Map.empty[String, String]) } { val e = intercept[DeltaAnalysisException] { throw DeltaErrors.identityColumnWithGenerationExpression() } checkError(e, "DELTA_IDENTITY_COLUMNS_WITH_GENERATED_EXPRESSION", "42613", Map.empty[String, String]) } { val e = intercept[DeltaUnsupportedOperationException] { throw DeltaErrors.unsupportedWritesWithMissingCoordinators("test") } checkError(e, "DELTA_UNSUPPORTED_WRITES_WITHOUT_COORDINATOR", "0AKDC", Map("coordinatorName" -> "test")) } { val exceptionWithContext = DeltaErrors.multipleSourceRowMatchingTargetRowInMergeException(spark) assert(exceptionWithContext.getMessage.contains("https") === true) val newSession = spark.newSession() setCustomContext(newSession, null) val exceptionWithoutContext = DeltaErrors.multipleSourceRowMatchingTargetRowInMergeException(newSession) assert(exceptionWithoutContext.getMessage.contains("https") === false) } } private def setCustomContext(session: SparkSession, context: SparkContext): Unit = { val scField = session.getClass.getDeclaredField("sparkContext") scField.setAccessible(true) try { scField.set(session, context) } finally { scField.setAccessible(false) } } } class DeltaErrorsSuite extends DeltaErrorsSuiteBase ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaFastDropFeatureSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.text.SimpleDateFormat import java.util.concurrent.TimeUnit // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.actions.{Action, AddFile, DeletionVectorDescriptor, Protocol, RemoveFile} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.AlterTableUnsetPropertiesDeltaCommand import org.apache.spark.sql.delta.commands.DeltaReorgTableCommand import org.apache.spark.sql.delta.deletionvectors.{RoaringBitmapArray, RoaringBitmapArrayFormat} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.DeltaFileOperations import org.apache.hadoop.fs.Path import org.apache.spark.SparkException import org.apache.spark.paths.SparkPath import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.analysis.ResolvedTable import org.apache.spark.sql.functions.{col, not} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.ManualClock class DeltaFastDropFeatureSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with DeletionVectorsTestUtils with DeltaRetentionSuiteBase { override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaSQLConf.FAST_DROP_FEATURE_ENABLED.key, true.toString) spark.conf.set(DeltaSQLConf.FAST_DROP_FEATURE_GENERATE_DV_TOMBSTONES.key, true.toString) enableDeletionVectors(spark, false, false, false) } val barrierVersionPropKey = DeltaConfigs.REQUIRE_CHECKPOINT_PROTECTION_BEFORE_VERSION.key protected def createTableWithFeature( deltaLog: DeltaLog, feature: TableFeature, featurePropertyEnablement: Option[String] = None): Unit = { val props = Seq(s"delta.feature.${feature.name} = 'supported'") ++ featurePropertyEnablement sql( s"""CREATE TABLE delta.`${deltaLog.dataPath}` (id bigint) USING delta |TBLPROPERTIES ( |${props.mkString(",")} |)""".stripMargin) assert(deltaLog.update().protocol.readerAndWriterFeatures.contains(feature)) } protected def dropTableFeature( deltaLog: DeltaLog, feature: TableFeature, truncateHistory: Boolean = false): Unit = { val dropFeatureSQL = s"""ALTER TABLE delta.`${deltaLog.dataPath}` |DROP FEATURE ${feature.name} |${if (truncateHistory) "TRUNCATE HISTORY" else ""}""".stripMargin sql(dropFeatureSQL) val snapshot = deltaLog.update() assert(!snapshot.protocol.readerAndWriterFeatures.contains(feature)) assert(truncateHistory || !feature.asInstanceOf[RemovableFeature].requiresHistoryProtection || snapshot.protocol.readerAndWriterFeatures.contains(CheckpointProtectionTableFeature)) } protected def getLogFiles(dir: File): Seq[File] = Nil protected def getDeltaVersions(dir: Path): Set[Long] = { getFileVersions(getDeltaFiles(new File(dir.toUri))) } protected def getCheckpointVersions(dir: Path): Set[Long] = { getFileVersions(getCheckpointFiles(new File(dir.toUri))) } protected def setModificationTimes( log: DeltaLog, startTime: Long = System.currentTimeMillis(), startVersion: Long, endVersion: Long, daysToAdd: Int): Unit = { val fs = log.logPath.getFileSystem(log.newDeltaHadoopConf()) for (version <- startVersion to endVersion) { setModificationTime(log, startTime, version.toInt, daysToAdd, fs) } } protected def addData(dir: File, start: Int, end: Int): Unit = spark.range(start, end).write.format("delta").mode("append").save(dir.getCanonicalPath) test("Dropping reader+writer feature") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) createTableWithFeature(deltaLog, TestRemovableReaderWriterFeature, Some(s"${TestRemovableReaderWriterFeature.TABLE_PROP_KEY} = 'true'")) // Add some data. This is optional to create a more realistic scenario. addData(dir, start = 0, end = 20) addData(dir, start = 20, end = 40) dropTableFeature(deltaLog, TestRemovableReaderWriterFeature) val snapshot = deltaLog.update() assert(snapshot.protocol.readerAndWriterFeatures.contains(CheckpointProtectionTableFeature)) assert(!snapshot.protocol.readerAndWriterFeatures.contains(TestRemovableReaderWriterFeature)) assert(snapshot.metadata.configuration.contains(barrierVersionPropKey)) assert(snapshot.metadata.configuration(barrierVersionPropKey).toInt === snapshot.version) assert(getCheckpointVersions(deltaLog.logPath).filter(_ <= snapshot.version).size === 4) } } test("Dropping writer feature") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) createTableWithFeature(deltaLog, TestRemovableWriterFeature, Some(s"${TestRemovableReaderWriterFeature.TABLE_PROP_KEY} = 'true'")) // Add some data. This is optional to create a more realistic scenario. addData(dir, start = 0, end = 20) addData(dir, start = 20, end = 40) dropTableFeature(deltaLog, TestRemovableWriterFeature) // Writer features do not require any checkpoint barriers. val snapshot = deltaLog.update() assert(!snapshot.protocol.readerAndWriterFeatures.contains(CheckpointProtectionTableFeature)) assert(!snapshot.protocol.readerAndWriterFeatures.contains(TestRemovableWriterFeature)) assert(!snapshot.metadata.configuration.contains(barrierVersionPropKey)) assert(getCheckpointVersions(deltaLog.logPath).size === 0) } } test("Dropping a legacy reader+writer feature") { withTempDir { dir => withSQLConf(DeltaSQLConf.TABLE_FEATURES_TEST_FEATURES_ENABLED.key -> false.toString) { val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) sql( s"""CREATE TABLE delta.`${deltaLog.dataPath}` (id bigint) USING delta |TBLPROPERTIES ( |delta.minReaderVersion=2, |delta.minWriterVersion=5 |)""".stripMargin) assert(deltaLog.update().protocol === Protocol(2, 5)) // Add some data. This is optional to create a more realistic scenario. addData(dir, start = 0, end = 20) addData(dir, start = 20, end = 40) dropTableFeature(deltaLog, ColumnMappingTableFeature) val snapshot = deltaLog.update() assert(deltaLog.update().protocol === Protocol(1, 7).withFeatures(Seq( InvariantsTableFeature, AppendOnlyTableFeature, CheckConstraintsTableFeature, ChangeDataFeedTableFeature, GeneratedColumnsTableFeature, CheckpointProtectionTableFeature))) assert(snapshot.metadata.configuration.contains(barrierVersionPropKey)) assert(snapshot.metadata.configuration(barrierVersionPropKey).toInt === snapshot.version) assert(getCheckpointVersions(deltaLog.logPath).filter(_ <= snapshot.version).size === 4) } } } test("Dropping multiple features") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) createTableWithFeature(deltaLog, TestRemovableReaderWriterFeature, Some(s"${TestRemovableReaderWriterFeature.TABLE_PROP_KEY} = 'true'")) // Add some data. This is optional to create a more realistic scenario. addData(dir, start = 0, end = 20) addData(dir, start = 20, end = 40) sql( s"""ALTER TABLE delta.`${dir.getCanonicalPath}` SET TBLPROPERTIES ( |delta.feature.${VacuumProtocolCheckTableFeature.name} = 'supported' |)""".stripMargin) dropTableFeature(deltaLog, TestRemovableReaderWriterFeature) // Add some data. This is optional to create a more realistic scenario. addData(dir, start = 40, end = 60) dropTableFeature(deltaLog, VacuumProtocolCheckTableFeature) // When multiple features are dropped, the barrier version must contain the version of the // last dropped feature. val snapshot = deltaLog.update() assert(snapshot.protocol.readerAndWriterFeatures.contains(CheckpointProtectionTableFeature)) assert(!snapshot.protocol.readerAndWriterFeatures.contains(TestRemovableReaderWriterFeature)) assert(!snapshot.protocol.readerAndWriterFeatures.contains(VacuumProtocolCheckTableFeature)) assert(snapshot.metadata.configuration.contains(barrierVersionPropKey)) assert(snapshot.metadata.configuration(barrierVersionPropKey).toInt === snapshot.version) assert(getCheckpointVersions(deltaLog.logPath).filter(_ <= snapshot.version).size === 8) } } test("Drop feature with history truncation option") { // When using the TRUNCATE HISTORY option we fallback to the legacy implementation. withTempDir { dir => val clock = new ManualClock(System.currentTimeMillis()) val deltaLog = DeltaLog.forTable(spark, new Path(dir.getAbsolutePath), clock) createTableWithFeature(deltaLog, TestRemovableReaderWriterFeature, Some( s"${TestRemovableReaderWriterFeature.TABLE_PROP_KEY} = 'true'")) // Add some data. This is optional to create a more realistic scenario. addData(dir, start = 0, end = 20) addData(dir, start = 20, end = 40) val e = intercept[DeltaTableFeatureException] { dropTableFeature(deltaLog, TestRemovableReaderWriterFeature, truncateHistory = true) } checkError( e, "DELTA_FEATURE_DROP_WAIT_FOR_RETENTION_PERIOD", parameters = Map( "feature" -> TestRemovableReaderWriterFeature.name, "logRetentionPeriodKey" -> "delta.logRetentionDuration", "logRetentionPeriod" -> "30 days", "truncateHistoryLogRetentionPeriod" -> "24 hours")) clock.advance(TimeUnit.HOURS.toMillis(24) + TimeUnit.MINUTES.toMillis(5)) dropTableFeature(deltaLog, TestRemovableReaderWriterFeature, truncateHistory = true) val snapshot = deltaLog.update() assert(!snapshot.protocol.readerAndWriterFeatures.contains(CheckpointProtectionTableFeature)) assert(!snapshot.protocol.readerAndWriterFeatures.contains(TestRemovableReaderWriterFeature)) assert(!snapshot.metadata.configuration.contains(barrierVersionPropKey)) } } test("Mixing drop feature implementations") { // When using the TRUNCATE HISTORY option we fallback to the legacy implementation. withTempDir { dir => val clock = new ManualClock(System.currentTimeMillis()) val deltaLog = DeltaLog.forTable(spark, new Path(dir.getAbsolutePath), clock) createTableWithFeature(deltaLog, TestRemovableReaderWriterFeature, Some(s"${TestRemovableReaderWriterFeature.TABLE_PROP_KEY} = 'true'")) // Add some data. This is optional to create a more realistic scenario. addData(dir, start = 0, end = 20) addData(dir, start = 20, end = 40) val e = intercept[DeltaTableFeatureException] { dropTableFeature(deltaLog, TestRemovableReaderWriterFeature, truncateHistory = true) } checkError( e, "DELTA_FEATURE_DROP_WAIT_FOR_RETENTION_PERIOD", parameters = Map( "feature" -> TestRemovableReaderWriterFeature.name, "logRetentionPeriodKey" -> "delta.logRetentionDuration", "logRetentionPeriod" -> "30 days", "truncateHistoryLogRetentionPeriod" -> "24 hours")) clock.advance(TimeUnit.HOURS.toMillis(24) + TimeUnit.MINUTES.toMillis(5)) // Adds the CheckpointProtectionTableFeature. dropTableFeature(deltaLog, TestRemovableReaderWriterFeature, truncateHistory = false) val snapshot = deltaLog.update() assert(snapshot.protocol.readerAndWriterFeatures.contains(CheckpointProtectionTableFeature)) assert(!snapshot.protocol.readerAndWriterFeatures.contains(TestRemovableReaderWriterFeature)) assert(snapshot.metadata.configuration.contains(barrierVersionPropKey)) assert(snapshot.metadata.configuration(barrierVersionPropKey).toInt === snapshot.version) // Two checkpoints were created in the first invocation of the legacy implementation. Four // more checkpoints were created in the second invocation. val expectedResult = 5 assert(getCheckpointVersions( deltaLog.logPath).filter(_ <= snapshot.version).size === expectedResult) } } for (withFastDropFeatureEnabled <- BOOLEAN_DOMAIN) test("Drop CheckpointProtectionTableFeature " + s"withFastDropFeatureEnabled: $withFastDropFeatureEnabled") { withTempDir { dir => val clock = new ManualClock(System.currentTimeMillis()) val deltaLog = DeltaLog.forTable(spark, new Path(dir.getAbsolutePath), clock) createTableWithFeature(deltaLog, TestRemovableReaderWriterFeature, Some(s"${TestRemovableReaderWriterFeature.TABLE_PROP_KEY} = 'true'")) // Add some data. This is optional to create a more realistic scenario. addData(dir, start = 0, end = 20) addData(dir, start = 20, end = 40) // Adds the CheckpointProtectionTableFeature. dropTableFeature(deltaLog, TestRemovableReaderWriterFeature) // More data. This is optional to create a more realistic scenario. addData(dir, start = 40, end = 60) val checkpointProtectionVersion = CheckpointProtectionTableFeature.getCheckpointProtectionVersion(deltaLog.update()) withSQLConf( DeltaSQLConf.FAST_DROP_FEATURE_ENABLED.key -> withFastDropFeatureEnabled.toString) { val e = intercept[DeltaTableFeatureException] { dropTableFeature(deltaLog, CheckpointProtectionTableFeature, truncateHistory = true) } checkError( e, "DELTA_FEATURE_DROP_CHECKPOINT_PROTECTION_WAIT_FOR_RETENTION_PERIOD", parameters = Map("truncateHistoryLogRetentionPeriod" -> "24 hours")) clock.advance(TimeUnit.HOURS.toMillis(48)) dropTableFeature(deltaLog, CheckpointProtectionTableFeature, truncateHistory = true) } val snapshot = deltaLog.update() val protocol = snapshot.protocol assert(!protocol.readerAndWriterFeatures.contains(CheckpointProtectionTableFeature)) assert(!protocol.readerAndWriterFeatures.contains(TestRemovableReaderWriterFeature)) assert(!snapshot.metadata.configuration.contains(barrierVersionPropKey)) assert(getDeltaVersions(deltaLog.logPath).min >= checkpointProtectionVersion) } } test("Drop CheckpointProtectionTableFeature with fast drop feature") { withTempDir { dir => val clock = new ManualClock(System.currentTimeMillis()) val deltaLog = DeltaLog.forTable(spark, new Path(dir.getAbsolutePath), clock) createTableWithFeature(deltaLog, TestRemovableReaderWriterFeature, Some(s"${TestRemovableReaderWriterFeature.TABLE_PROP_KEY} = 'true'")) // Add some data. This is optional to create a more realistic scenario. addData(dir, start = 0, end = 20) addData(dir, start = 20, end = 40) // Adds the CheckpointProtectionTableFeature. dropTableFeature(deltaLog, TestRemovableReaderWriterFeature) // This is optional since we won't be allowed to drop CheckpointProtectionTableFeature anyway. // However, we show that in a scenario were the feature would normally dropped, it did not // because we used the fast drop feature command. clock.advance(TimeUnit.HOURS.toMillis(48)) val e = intercept[DeltaTableFeatureException] { dropTableFeature(deltaLog, CheckpointProtectionTableFeature) } checkError( e, "DELTA_FEATURE_CAN_ONLY_DROP_CHECKPOINT_PROTECTION_WITH_HISTORY_TRUNCATION", parameters = Map.empty) } } test("Attempt dropping CheckpointProtectionTableFeature within the retention period") { withTempDir { dir => val clock = new ManualClock(System.currentTimeMillis()) val deltaLog = DeltaLog.forTable(spark, new Path(dir.getAbsolutePath), clock) createTableWithFeature(deltaLog, TestRemovableReaderWriterFeature, Some(s"${TestRemovableReaderWriterFeature.TABLE_PROP_KEY} = 'true'")) // Add some data. This is optional to create a more realistic scenario. addData(dir, start = 0, end = 20) addData(dir, start = 20, end = 40) // Adds the CheckpointProtectionTableFeature. dropTableFeature(deltaLog, TestRemovableReaderWriterFeature) // More data. This is optional to create a more realistic scenario. addData(dir, start = 40, end = 60) deltaLog.checkpoint(deltaLog.update()) val e1 = intercept[DeltaTableFeatureException] { dropTableFeature(deltaLog, CheckpointProtectionTableFeature, truncateHistory = true) } checkError( e1, "DELTA_FEATURE_DROP_CHECKPOINT_PROTECTION_WAIT_FOR_RETENTION_PERIOD", parameters = Map("truncateHistoryLogRetentionPeriod" -> "24 hours")) // TestRemovableReaderWriterFeature traces still exist in history. clock.advance(TimeUnit.HOURS.toMillis(15)) val e2 = intercept[DeltaTableFeatureException] { dropTableFeature(deltaLog, CheckpointProtectionTableFeature, truncateHistory = true) } checkError( e2, "DELTA_FEATURE_DROP_CHECKPOINT_PROTECTION_WAIT_FOR_RETENTION_PERIOD", parameters = Map("truncateHistoryLogRetentionPeriod" -> "24 hours")) } } test("Drop CheckpointProtectionTableFeature when history is already truncated") { withTempDir { dir => val startTS = System.currentTimeMillis() val clock = new ManualClock(startTS) val deltaLog = DeltaLog.forTable(spark, new Path(dir.getAbsolutePath), clock) createTableWithFeature(deltaLog, TestRemovableReaderWriterFeature, Some(s"${TestRemovableReaderWriterFeature.TABLE_PROP_KEY} = 'true'")) // Add some data. This is optional to create a more realistic scenario. addData(dir, start = 0, end = 20) addData(dir, start = 20, end = 40) // Adds the CheckpointProtectionTableFeature. dropTableFeature(deltaLog, TestRemovableReaderWriterFeature) val v1 = deltaLog.update().version // Default log retention is 30 days. clock.advance(TimeUnit.DAYS.toMillis(32)) // More data and checkpoints. Data is optional but the checkpoints are used // to cleanup the logs below. addData(dir, start = 40, end = 60) deltaLog.checkpoint(deltaLog.update()) addData(dir, start = 60, end = 80) deltaLog.checkpoint(deltaLog.update()) val v2 = deltaLog.update().version setModificationTimes(deltaLog, startVersion = v1 + 1, endVersion = v2, daysToAdd = 32) deltaLog.cleanUpExpiredLogs(deltaLog.update()) // Add some data. This is optional to create a more realistic scenario. addData(dir, start = 80, end = 100) addData(dir, start = 100, end = 120) val v3 = deltaLog.update().version setModificationTimes(deltaLog, startVersion = v2 + 1, endVersion = v3, daysToAdd = 32) clock.advance(TimeUnit.HOURS.toMillis(48)) // Add some data. This is optional to create a more realistic scenario. addData(dir, start = 120, end = 140) addData(dir, start = 140, end = 160) // At this point history before the atomic cleanup version should already be clean. val deltaVersionsBeforeDrop = getDeltaVersions(deltaLog.logPath) val atomicHistoryCleanupVersion = CheckpointProtectionTableFeature.getCheckpointProtectionVersion(deltaLog.update()) assert(deltaVersionsBeforeDrop.min >= atomicHistoryCleanupVersion) val v4 = deltaLog.update().version setModificationTimes(deltaLog, startVersion = v3 + 1, endVersion = v4, daysToAdd = 34) dropTableFeature(deltaLog, CheckpointProtectionTableFeature, truncateHistory = true) val snapshot = deltaLog.update() val protocol = snapshot.protocol assert(!protocol.readerAndWriterFeatures.contains(CheckpointProtectionTableFeature)) assert(!protocol.readerAndWriterFeatures.contains(TestRemovableReaderWriterFeature)) assert(!snapshot.metadata.configuration.contains(barrierVersionPropKey)) // No other commits should have been truncated. assert(getDeltaVersions(deltaLog.logPath).min === deltaVersionsBeforeDrop.min) } } for (timeTravelMethod <- Seq("restoreSQL", "restoreSQLTS", "selectSQL", "selectSQLTS", "getSnapshotAt", "restoreToVersion", "restoreToTS", "sparkVersion", "sparkTS")) test(s"Protocol is validated when time traveling - time-travel method: $timeTravelMethod") { withTempDir { dir => def getTimestampForVersion(version: Long): String = { val logPath = new Path(dir.getCanonicalPath, "_delta_log") val file = new File(new Path(logPath, f"$version%020d.json").toString) val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS") sdf.format(file.lastModified()) } val deltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) createTableWithFeature(deltaLog, TestUnsupportedReaderWriterFeature) // Add some data. This is optional to create a more realistic scenario. addData(dir, start = 0, end = 20) addData(dir, start = 20, end = 40) val versionBeforeRemoval = deltaLog.update().version val tsBeforeRemoval = getTimestampForVersion(versionBeforeRemoval) // Adds the CheckpointProtectionTableFeature. dropTableFeature(deltaLog, TestUnsupportedReaderWriterFeature) // More data. This is optional to create a more realistic scenario. addData(dir, start = 40, end = 60) addData(dir, start = 60, end = 80) withSQLConf(DeltaSQLConf.UNSUPPORTED_TESTING_FEATURES_ENABLED.key -> true.toString) { val e = intercept[DeltaUnsupportedTableFeatureException] { val table = io.delta.tables.DeltaTable.forPath(dir.toString) DeltaLog.clearCache() val tablePath = s"delta.`${dir.getCanonicalPath}`" timeTravelMethod match { case "restoreSQL" => sql(s"RESTORE $tablePath TO VERSION AS OF $versionBeforeRemoval") case "restoreSQLTS" => sql(s"RESTORE $tablePath TO TIMESTAMP AS OF '$tsBeforeRemoval'") case "selectSQL" => sql(s"SELECT * FROM $tablePath VERSION AS OF $versionBeforeRemoval") case "selectSQLTS" => sql(s"SELECT * FROM $tablePath TIMESTAMP AS OF '$tsBeforeRemoval'") case "getSnapshotAt" => deltaLog.getSnapshotAt(versionBeforeRemoval) case "restoreToVersion" => table.restoreToVersion(versionBeforeRemoval) case "restoreToTS" => table.restoreToTimestamp(tsBeforeRemoval) case "sparkVersion" => spark.read.format("delta") .option("versionAsOf", versionBeforeRemoval).load(dir.getCanonicalPath) case "sparkTS" => spark.read.format("delta") .option("timestampAsOf", tsBeforeRemoval).load(dir.getCanonicalPath) case _ => assert(false, "non existent time travel method.") } } assert(e.getErrorClass === "DELTA_UNSUPPORTED_FEATURES_FOR_READ") } } } for (downgradeAllowed <- BOOLEAN_DOMAIN) test(s"Restore table works with fast drop feature - downgradeAllowed: $downgradeAllowed") { withSQLConf( DeltaSQLConf.RESTORE_TABLE_PROTOCOL_DOWNGRADE_ALLOWED.key -> downgradeAllowed.toString) { withTempDir { dir => import org.apache.spark.sql.delta.implicits._ val deltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) createTableWithFeature(deltaLog, TestRemovableReaderWriterFeature) // Add some data. addData(dir, start = 0, end = 20) addData(dir, start = 20, end = 40) val versionBeforeRemoval = deltaLog.update().version // Adds the CheckpointProtectionTableFeature. dropTableFeature(deltaLog, TestRemovableReaderWriterFeature) // More data. This is optional to create a more realistic scenario. addData(dir, start = 40, end = 60) addData(dir, start = 60, end = 80) sql(s"RESTORE delta.`${dir.getCanonicalPath}` TO VERSION AS OF $versionBeforeRemoval") val protocol = deltaLog.update().protocol assert(protocol.readerAndWriterFeatures.contains(TestRemovableReaderWriterFeature)) assert(protocol.readerAndWriterFeatures .contains(CheckpointProtectionTableFeature) === !downgradeAllowed) val targetTable = io.delta.tables.DeltaTable.forPath(dir.getCanonicalPath) assert(targetTable.toDF.as[Long].collect().sorted === Seq.range(0, 40)) } } } private def createTableWithDeletionVectors(deltaLog: DeltaLog): Unit = { withSQLConf( DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> true.toString, DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS.key -> true.toString) { val dir = deltaLog.dataPath val targetDF = spark.range(start = 0, end = 100, step = 1, numPartitions = 4) targetDF.write.format("delta").save(dir.toString) val targetTable = io.delta.tables.DeltaTable.forPath(dir.toString) // Add some DVs. targetTable.delete("id < 5 or id >= 95") // Add some more DVs for the same set of files. targetTable.delete("id < 10 or id >= 90") // Assert that DVs exist. assert(deltaLog.update().numDeletionVectorsOpt === Some(2L)) } } private def dropDeletionVectors(deltaLog: DeltaLog, truncateHistory: Boolean = false): Unit = { sql(s"""ALTER TABLE delta.`${deltaLog.dataPath}` |DROP FEATURE deletionVectors |${if (truncateHistory) "TRUNCATE HISTORY" else ""} |""".stripMargin) val snapshot = deltaLog.update() val protocol = snapshot.protocol assert(snapshot.numDeletionVectorsOpt.getOrElse(0L) === 0) assert(snapshot.numDeletedRecordsOpt.getOrElse(0L) === 0) assert(truncateHistory || !protocol.readerFeatureNames.contains(DeletionVectorsTableFeature.name)) } private def validateTombstones( log: DeltaLog, expectedDVTombstoneCount: Option[Int] = None): Unit = { import org.apache.spark.sql.delta.implicits._ val snapshot = log.update() val dvPath = DeletionVectorDescriptor .urlEncodedRelativePathIfExists(col("deletionVector"), log.dataPath) val isDVTombstone = DeletionVectorDescriptor.isDeletionVectorPath(col("path")) val isInlineDeletionVector = DeletionVectorDescriptor.isInline(col("deletionVector")) val uniqueDvsFromParquetRemoveFiles = snapshot .tombstones .filter("deletionVector IS NOT NULL") .filter(not(isInlineDeletionVector)) .filter(not(isDVTombstone)) .select(dvPath.as("path")) .filter(col("path").isNotNull) .distinct() .as[String] val dvTombstones = snapshot .tombstones .filter(isDVTombstone) .select("path") .as[String] val dvTombstonesSet = dvTombstones.collect().toSet assert(dvTombstonesSet.nonEmpty || expectedDVTombstoneCount === Some(0)) assert(dvTombstonesSet.map(new Path(_)).forall(_.getParent.isRoot)) assert(uniqueDvsFromParquetRemoveFiles.collect().toSet === dvTombstonesSet) expectedDVTombstoneCount.foreach(expected => assert(dvTombstonesSet.size === expected)) } for (withCommitLarge <- BOOLEAN_DOMAIN) test("DV tombstones are created when dropping DVs" + s"withCommitLarge: $withCommitLarge") { val threshold = if (withCommitLarge) 0 else 10000 withSQLConf( DeltaSQLConf.FAST_DROP_FEATURE_DV_TOMBSTONE_COUNT_THRESHOLD.key -> threshold.toString) { withTempPath { dir => val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) createTableWithDeletionVectors(deltaLog) dropDeletionVectors(deltaLog) validateTombstones(deltaLog) val targetTable = io.delta.tables.DeltaTable.forPath(dir.toString) assert(targetTable.toDF.collect().length === 80) // DV Tombstones are recorded in the snapshot state. assert(deltaLog.update().numOfRemoves === 8) } } } test("DV tombstones are generated when no action is taken in pre-downgrade") { withTempPath { dir => val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) createTableWithDeletionVectors(deltaLog) // Remove all DV traces in advance. Table will look clean in DROP FEATURE. val table = DeltaTableV2(spark, deltaLog.dataPath) val properties = Seq(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key) AlterTableUnsetPropertiesDeltaCommand(table, properties, ifExists = true).run(spark) import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ val catalog = spark.sessionState.catalogManager.currentCatalog.asTableCatalog val tableId = Seq(table.name()).asIdentifier DeltaReorgTableCommand(ResolvedTable.create(catalog, tableId, table))(Nil).run(spark) assert(deltaLog.update().numDeletedRecordsOpt.forall(_ == 0)) dropDeletionVectors(deltaLog) validateTombstones(deltaLog) } } test("We only create missing DV tombstones when dropping DVs") { withTempPath { dir => val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) createTableWithDeletionVectors(deltaLog) dropDeletionVectors(deltaLog) validateTombstones(deltaLog) // Re enable the feature and add more DVs. The delete touches a new file as well as a file // that already contains a DV within the retention period. sql( s"""ALTER TABLE delta.`${dir.getAbsolutePath}` |SET TBLPROPERTIES ( |delta.feature.${DeletionVectorsTableFeature.name} = 'enabled', |${DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key} = 'true' |)""".stripMargin) withSQLConf(DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS.key -> true.toString) { val targetTable = io.delta.tables.DeltaTable.forPath(dir.toString) targetTable.delete("id > 20 and id <= 30") assert(deltaLog.update().numDeletionVectorsOpt === Some(2L)) } sql(s"ALTER TABLE delta.`${dir.getAbsolutePath}` DROP FEATURE deletionVectors") validateTombstones(deltaLog) } } for (isShallowClone <- Seq(true)) test(s"We do not create redundant DV tombstones after cloning " + s"isShallowClone: $isShallowClone") { withTempPaths(2) { case Seq(sourceDir, targetDir) => val sourceLog = DeltaLog.forTable(spark, sourceDir.getAbsolutePath) val targetLog = DeltaLog.forTable(spark, targetDir.getAbsolutePath) createTableWithDeletionVectors(sourceLog) io.delta.tables.DeltaTable.forPath(sourceLog.dataPath.toString).clone( target = targetDir.getCanonicalPath, isShallow = isShallowClone) // We should not create any DV tombstones at this point since the shallow cloned table // references the source table's data files. val expectedDVTombstoneCount1 = Some(0) dropDeletionVectors(targetLog) validateTombstones(targetLog, expectedDVTombstoneCount1) // Re-enable DVs in the target table. sql( s"""ALTER TABLE delta.`${targetDir.getAbsolutePath}` |SET TBLPROPERTIES ( |delta.feature.${DeletionVectorsTableFeature.name} = 'enabled', |${DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key} = 'true' |)""".stripMargin) // Deleting rows causes shallow clone tables to create local files. withSQLConf(DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS.key -> "true") { val targetTable = io.delta.tables.DeltaTable.forPath(targetDir.toString) targetTable.delete("id > 20 and id <= 30") assert(targetLog.update().numDeletionVectorsOpt === Some(2L)) } sql(s"ALTER TABLE delta.`${targetDir.getAbsolutePath}` DROP FEATURE deletionVectors") // Verify that the DV tombstones created exactly match the unique DV paths // found in files with DVs. val expectedDVTombstoneCount2 = Some(1) validateTombstones(targetLog, expectedDVTombstoneCount2) } } test("We do not create tombstones when there are no RemoveFiles within the retention period") { withTempPath { dir => val clock = new ManualClock(System.currentTimeMillis()) val deltaLog = DeltaLog.forTable(spark, new Path(dir.getAbsolutePath), clock) createTableWithDeletionVectors(deltaLog) // Pretend tombstone retention period has passed (default 1 week). val clockAdvanceMillis = DeltaLog.tombstoneRetentionMillis(deltaLog.update().metadata) clock.advance(clockAdvanceMillis + TimeUnit.DAYS.toMillis(3)) dropDeletionVectors(deltaLog) assert(deltaLog.update().tombstones.collect().forall(_.isDVTombstone == false)) } } test("We create DV tombstones when mixing drop feature implementations") { // When using the TRUNCATE HISTORY option we fallback to the legacy implementation. withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) createTableWithDeletionVectors(deltaLog) val e = intercept[DeltaTableFeatureException] { dropDeletionVectors(deltaLog, truncateHistory = true) } checkError( e, "DELTA_FEATURE_DROP_WAIT_FOR_RETENTION_PERIOD", parameters = Map( "feature" -> "deletionVectors", "logRetentionPeriodKey" -> "delta.logRetentionDuration", "logRetentionPeriod" -> "30 days", "truncateHistoryLogRetentionPeriod" -> "24 hours")) validateTombstones(deltaLog) dropDeletionVectors(deltaLog, truncateHistory = false) validateTombstones(deltaLog) } } test("DV tombstones are not created for inline DVs") { withSQLConf( DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> true.toString) { withTempPath { dir => val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) val targetTable = () => io.delta.tables.DeltaTable.forPath(dir.toString) spark.range(start = 0, end = 100, step = 1, numPartitions = 1) .write.format("delta").save(dir.toString) def removeRowsWithInlineDV(add: AddFile, markedRows: Long*): (AddFile, RemoveFile) = { val bitmap = RoaringBitmapArray(markedRows: _*) val serializedBitmap = bitmap.serializeAsByteArray(RoaringBitmapArrayFormat.Portable) val cardinality = markedRows.size val dv = DeletionVectorDescriptor.inlineInLog(serializedBitmap, cardinality) add.removeRows( deletionVector = dv, updateStats = true) } // There should be a single AddFile. val snapshot = deltaLog.update() val addFile = snapshot.allFiles.first() val (newAddFile, newRemoveFile) = removeRowsWithInlineDV(addFile, 3, 34, 67) val actionsToCommit: Seq[Action] = Seq(newAddFile, newRemoveFile) deltaLog.startTransaction(catalogTableOpt = None, snapshotOpt = Some(snapshot)) .commit(actionsToCommit, new DeltaOperations.TestOperation) // Verify the table is assert(deltaLog.update().numDeletedRecordsOpt.exists(_ === 3)) assert(deltaLog.update().numDeletionVectorsOpt.exists(_ === 1)) assert(targetTable().toDF.collect().length === 97) dropDeletionVectors(deltaLog) // No DV tombstones should be have been created. assert(deltaLog.update().tombstones.collect().forall(_.isDVTombstone == false)) assert(targetTable().toDF.collect().length === 97) assert(deltaLog.update().numDeletedRecordsOpt.forall(_ === 0)) assert(deltaLog.update().numDeletionVectorsOpt.forall(_ === 0)) } } } for (generateDVTombstones <- BOOLEAN_DOMAIN) test(s"Vacuum does not delete deletion vector files." + s"generateDVTombstones: $generateDVTombstones") { val targetDF = spark.range(start = 0, end = 100, step = 1, numPartitions = 4) withSQLConf( DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> true.toString, DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS.key -> true.toString, DeltaSQLConf.FAST_DROP_FEATURE_GENERATE_DV_TOMBSTONES.key -> generateDVTombstones.toString, // With this config we pretend the client does not support DVs. Therefore, it will not // discover DVs from the RemoveFile actions. DeltaSQLConf.FAST_DROP_FEATURE_DV_DISCOVERY_IN_VACUUM_DISABLED.key -> true.toString) { withTempPath { dir => val targetLog = DeltaLog.forTable(spark, dir.getAbsolutePath) targetDF.write.format("delta").save(dir.toString) val targetTable = io.delta.tables.DeltaTable.forPath(dir.toString) // Add some DVs. targetTable.delete("id < 5 or id >= 95") val versionWithDVs = targetLog.update().version // Unfortunately, there is no point in advancing the clock because the deletion timestamps // in the RemoveFiles do not use the clock. Instead, we set the creation time back 10 days // to all files created so far. These will be eligible for vacuum. val fs = targetLog.logPath.getFileSystem(targetLog.newDeltaHadoopConf()) val allFiles = DeltaFileOperations.localListDirs( hadoopConf = targetLog.newDeltaHadoopConf(), dirs = Seq(dir.getCanonicalPath), recursive = false) allFiles.foreach { p => fs.setTimes(p.getHadoopPath, System.currentTimeMillis() - TimeUnit.DAYS.toMillis(10), 0) } // Add new DVs for the same set of files. targetTable.delete("id < 10 or id >= 90") // Assert that DVs exist. assert(targetLog.update().numDeletionVectorsOpt === Some(2L)) sql(s"ALTER TABLE delta.`${dir.getAbsolutePath}` DROP FEATURE deletionVectors") val snapshot = targetLog.update() val protocol = snapshot.protocol assert(snapshot.numDeletionVectorsOpt.getOrElse(0L) === 0) assert(snapshot.numDeletedRecordsOpt.getOrElse(0L) === 0) assert(!protocol.readerFeatureNames.contains(DeletionVectorsTableFeature.name)) targetTable.delete("id < 15 or id >= 85") // The DV files are outside the retention period. However, the DVs are still referenced in // the history. Normally we should not delete any DVs. sql(s"VACUUM '${dir.getAbsolutePath}'") val query = sql(s"SELECT * FROM delta.`${dir.getAbsolutePath}` VERSION AS OF $versionWithDVs") if (generateDVTombstones) { // At version 1 we only deleted 10 rows. assert(query.collect().length === 90) } else { val e = intercept[SparkException] { query.collect() } val msg = e.getCause.getMessage assert(msg.contains("RowIndexFilterFileNotFoundException") || msg.contains(".bin does not exist")) } } } } test("DV tombstones do not generate CDC") { import org.apache.spark.sql.delta.commands.cdc.CDCReader withTempPath { dir => withSQLConf(DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> true.toString) { val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) createTableWithDeletionVectors(deltaLog) val versionBeforeDrop = deltaLog.update().version dropDeletionVectors(deltaLog) val deleteCountInDropFeature = CDCReader .changesToBatchDF(deltaLog, versionBeforeDrop + 1, deltaLog.update().version, spark) .filter(s"${CDCReader.CDC_TYPE_COLUMN_NAME} = '${CDCReader.CDC_TYPE_DELETE_STRING}'") .count() assert(deleteCountInDropFeature === 0) } } } for (incrementalCommitEnabled <- BOOLEAN_DOMAIN) test("Checksum computation does not take into account DV tombstones" + s"incrementalCommitEnabled: $incrementalCommitEnabled") { withTempPaths(2) { dirs => var checksumWithDVTombstones: VersionChecksum = null withSQLConf( DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key -> incrementalCommitEnabled.toString, DeltaSQLConf.FAST_DROP_FEATURE_GENERATE_DV_TOMBSTONES.key -> true.toString) { val deltaLog = DeltaLog.forTable(spark, dirs.head.getAbsolutePath) createTableWithDeletionVectors(deltaLog) dropDeletionVectors(deltaLog) val snapshot = deltaLog.update() checksumWithDVTombstones = snapshot.checksumOpt.getOrElse(snapshot.computeChecksum) } var checksumWithoutDVTombstones: VersionChecksum = null withSQLConf( DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key -> incrementalCommitEnabled.toString, DeltaSQLConf.FAST_DROP_FEATURE_GENERATE_DV_TOMBSTONES.key -> false.toString) { val deltaLog = DeltaLog.forTable(spark, dirs.last.getAbsolutePath) createTableWithDeletionVectors(deltaLog) dropDeletionVectors(deltaLog) val snapshot = deltaLog.update() checksumWithoutDVTombstones = snapshot.checksumOpt.getOrElse(snapshot.computeChecksum) } // DV tombstones do not affect the number of files. assert(checksumWithoutDVTombstones.numFiles === checksumWithDVTombstones.numFiles) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaGenerateSymlinkManifestSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.net.URI // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.DeltaOperations.Delete import org.apache.spark.sql.delta.commands.DeltaGenerateCommand import org.apache.spark.sql.delta.hooks.GenerateSymlinkManifest import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.DeltaFileOperations import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs._ import org.apache.hadoop.fs.permission.FsPermission import org.apache.hadoop.util.Progressable import org.apache.spark.SparkThrowable import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions._ import org.apache.spark.sql.test.SharedSparkSession // scalastyle:on import.ordering.noEmptyLine class DeltaGenerateSymlinkManifestSuite extends DeltaGenerateSymlinkManifestSuiteBase with DeltaSQLCommandTest trait DeltaGenerateSymlinkManifestSuiteBase extends DeltaGenerateSymlinkManifestTestHelper with DeletionVectorsTestUtils with DeltaTestUtilsForTempViews { import testImplicits._ test("basic case: SQL command - path-based table") { withTempDir { tablePath => tablePath.delete() spark.createDataset(spark.sparkContext.parallelize(1 to 100, 7)) .write.format("delta").mode("overwrite").save(tablePath.toString) assertManifest(tablePath, expectSameFiles = false, expectedNumFiles = 0) // Create a Delta table and call the scala api for generating manifest files spark.sql(s"GENERATE symlink_ForMat_Manifest FOR TABLE delta.`${tablePath.getAbsolutePath}`") assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 7) } } test("basic case: SQL command - name-based table") { val tableName = "deltaTable" withTable("deltaTable") { spark.createDataset(spark.sparkContext.parallelize(1 to 100, 7)) .write.format("delta").saveAsTable(tableName) assertManifest(tableName, expectSameFiles = false, expectedNumFiles = 0) spark.sql(s"GENERATE symlink_ForMat_Manifest FOR TABLE $tableName") assertManifest(tableName, expectSameFiles = true, expectedNumFiles = 7) } } test("basic case: SQL command - throw error on bad tables") { var e: Exception = intercept[AnalysisException] { spark.sql("GENERATE symlink_format_manifest FOR TABLE nonExistentTable") } assert(e.getMessage.contains("not found") || e.getMessage.contains("cannot be found")) withTable("nonDeltaTable") { spark.range(2).write.format("parquet").saveAsTable("nonDeltaTable") e = intercept[AnalysisException] { spark.sql("GENERATE symlink_format_manifest FOR TABLE nonDeltaTable") } assert(e.getMessage.contains("only supported for Delta")) } } test("basic case: SQL command - throw error on non delta table paths") { withTempDir { dir => var e = intercept[AnalysisException] { spark.sql(s"GENERATE symlink_format_manifest FOR TABLE delta.`$dir`") } assert(e.getMessage.contains("is not a Delta table")) spark.range(2).write.format("parquet").mode("overwrite").save(dir.toString) e = intercept[AnalysisException] { spark.sql(s"GENERATE symlink_format_manifest FOR TABLE delta.`$dir`") } assert(e.getMessage.contains("is not a Delta table")) e = intercept[AnalysisException] { spark.sql(s"GENERATE symlink_format_manifest FOR TABLE parquet.`$dir`") } assert(e.getMessage.contains("not found") || e.getMessage.contains("cannot be found")) } } testWithTempView("basic case: SQL command - throw error on temp views") { isSQLTempView => withTable("t1") { spark.range(2).write.format("delta").saveAsTable("t1") createTempViewFromTable("t1", isSQLTempView) val e = intercept[AnalysisException] { spark.sql(s"GENERATE symlink_format_manifest FOR TABLE v") } assert(e.getMessage.contains("'GENERATE' expects a table but `v` is a view.")) } } test("basic case: SQL command - throw error on unsupported mode") { withTempDir { tablePath => spark.range(2).write.format("delta").save(tablePath.getAbsolutePath) val e = intercept[IllegalArgumentException] { spark.sql(s"GENERATE xyz FOR TABLE delta.`${tablePath.getAbsolutePath}`") } assert(e.toString.contains("not supported")) DeltaGenerateCommand.modeNameToGenerationFunc.keys.foreach { modeName => assert(e.toString.contains(modeName)) } } } test("basic case: Scala API - path-based table") { withTempDir { tablePath => tablePath.delete() spark.createDataset(spark.sparkContext.parallelize(1 to 100, 7)) .write.format("delta").mode("overwrite").save(tablePath.toString) assertManifest(tablePath, expectSameFiles = false, expectedNumFiles = 0) // Create a Delta table and call the scala api for generating manifest files val deltaTable = io.delta.tables.DeltaTable.forPath(tablePath.getAbsolutePath) deltaTable.generate("symlink_format_manifest") assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 7) } } test("basic case: Scala API - name-based table") { val tableName = "deltaTable" withTable(tableName) { spark.createDataset(spark.sparkContext.parallelize(1 to 100, 7)) .write.format("delta").saveAsTable(tableName) assertManifest(tableName, expectSameFiles = false, expectedNumFiles = 0) val deltaTable = io.delta.tables.DeltaTable.forName(tableName) deltaTable.generate("symlink_format_manifest") assertManifest(tableName, expectSameFiles = true, expectedNumFiles = 7) } } test("full manifest: non-partitioned table") { withTempDir { tablePath => tablePath.delete() def write(parallelism: Int): Unit = { spark.createDataset(spark.sparkContext.parallelize(1 to 100, parallelism)) .write.format("delta").mode("overwrite").save(tablePath.toString) } write(7) assertManifest(tablePath, expectSameFiles = false, expectedNumFiles = 0) generateSymlinkManifest(tablePath.toString) assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 7) // Reduce files write(5) assertManifest(tablePath, expectSameFiles = false, expectedNumFiles = 7) generateSymlinkManifest(tablePath.toString) assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 5) // Remove all data spark.emptyDataset[Int].write.format("delta").mode("overwrite").save(tablePath.toString) assertManifest(tablePath, expectSameFiles = false, expectedNumFiles = 5) generateSymlinkManifest(tablePath.toString) assertManifest( tablePath, expectSameFiles = true, expectedNumFiles = 0) assert(spark.read.format("delta").load(tablePath.toString).count() == 0) // delete all data write(5) assertManifest( tablePath, expectSameFiles = false, expectedNumFiles = 0) val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tablePath.toString) deltaTable.delete() generateSymlinkManifest(tablePath.toString) assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 0) assert(spark.read.format("delta").load(tablePath.toString).count() == 0) } } test("full manifest: partitioned table") { withTempDir { tablePath => tablePath.delete() def write(parallelism: Int, partitions1: Int, partitions2: Int): Unit = { spark.createDataset(spark.sparkContext.parallelize(1 to 100, parallelism)).toDF("value") .withColumn("part1", $"value" % partitions1) .withColumn("part2", $"value" % partitions2) .write.format("delta").partitionBy("part1", "part2") .mode("overwrite").save(tablePath.toString) } write(10, 10, 10) assertManifest(tablePath, expectSameFiles = false, expectedNumFiles = 0) generateSymlinkManifest(tablePath.toString) // 10 files each in ../part1=X/part2=X/ for X = 0 to 9 assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 100) // Reduce # partitions on both dimensions write(1, 1, 1) assertManifest(tablePath, expectSameFiles = false, expectedNumFiles = 100) generateSymlinkManifest(tablePath.toString) assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 1) // Increase # partitions on both dimensions write(5, 5, 5) assertManifest(tablePath, expectSameFiles = false, expectedNumFiles = 1) generateSymlinkManifest(tablePath.toString) assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 25) // Increase # partitions on only one dimension write(5, 10, 5) assertManifest(tablePath, expectSameFiles = false, expectedNumFiles = 25) generateSymlinkManifest(tablePath.toString) assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 50) // Remove all data spark.emptyDataset[Int].toDF("value") .withColumn("part1", $"value" % 10) .withColumn("part2", $"value" % 10) .write.format("delta").mode("overwrite").save(tablePath.toString) assertManifest(tablePath, expectSameFiles = false, expectedNumFiles = 50) generateSymlinkManifest(tablePath.toString) assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 0) assert(spark.read.format("delta").load(tablePath.toString).count() == 0) // delete all data write(5, 5, 5) generateSymlinkManifest(tablePath.toString) assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 25) val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tablePath.toString) deltaTable.delete() generateSymlinkManifest(tablePath.toString) assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 0) assert(spark.read.format("delta").load(tablePath.toString).count() == 0) } } test("incremental manifest: table property controls post commit manifest generation") { withTempDir { tablePath => tablePath.delete() def writeWithIncrementalManifest(enabled: Boolean, numFiles: Int): Unit = { withIncrementalManifest(tablePath, enabled) { spark.createDataset(spark.sparkContext.parallelize(1 to 100, numFiles)) .write.format("delta").mode("overwrite").save(tablePath.toString) } } writeWithIncrementalManifest(enabled = false, numFiles = 1) assertManifest(tablePath, expectSameFiles = false, expectedNumFiles = 0) // Enabling it should automatically generate manifest files writeWithIncrementalManifest(enabled = true, numFiles = 2) assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 2) // Disabling it should stop updating existing manifest files writeWithIncrementalManifest(enabled = false, numFiles = 3) assertManifest(tablePath, expectSameFiles = false, expectedNumFiles = 2) } } test("incremental manifest: unpartitioned table") { withTempDir { tablePath => tablePath.delete() def write(numFiles: Int): Unit = withIncrementalManifest(tablePath, enabled = true) { spark.createDataset(spark.sparkContext.parallelize(1 to 100, numFiles)) .write.format("delta").mode("overwrite").save(tablePath.toString) } write(1) // first write won't generate automatic manifest as mode enable after first write assertManifest(tablePath, expectSameFiles = false, expectedNumFiles = 0) // Increase files write(7) assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 7) // Reduce files write(5) assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 5) // Remove all data spark.emptyDataset[Int].write.format("delta").mode("overwrite").save(tablePath.toString) assert(spark.read.format("delta").load(tablePath.toString).count() == 0) assertManifest( tablePath, expectSameFiles = true, expectedNumFiles = 0) } } test("incremental manifest: partitioned table") { withTempDir { tablePath => tablePath.delete() def writePartitioned(parallelism: Int, numPartitions1: Int, numPartitions2: Int): Unit = { withIncrementalManifest(tablePath, enabled = true) { val input = if (parallelism == 0) spark.emptyDataset[Int] else spark.createDataset(spark.sparkContext.parallelize(1 to 100, parallelism)) input.toDF("value") .withColumn("part1", $"value" % numPartitions1) .withColumn("part2", $"value" % numPartitions2) .write.format("delta").partitionBy("part1", "part2") .mode("overwrite").save(tablePath.toString) } } writePartitioned(1, 1, 1) // Manifests wont be generated in the first write because `withIncrementalManifest` will // enable manifest generation only after the first write defines the table log. assertManifest(tablePath, expectSameFiles = false, expectedNumFiles = 0) writePartitioned(10, 10, 10) // 10 files each in ../part1=X/part2=X/ for X = 0 to 9 (so only 10 subdirectories) assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 100) // Update such that 1 file is removed and 1 file is added in another partition val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tablePath.toString) deltaTable.updateExpr("value = 1", Map("part1" -> "0", "value" -> "-1")) assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 100) // Delete such that 1 file is removed deltaTable.delete("value = -1") assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 99) // Reduce # partitions on both dimensions writePartitioned(1, 1, 1) assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 1) // Increase # partitions on both dimensions writePartitioned(5, 5, 5) assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 25) // Increase # partitions on only one dimension writePartitioned(5, 10, 5) assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 50) // Remove all data writePartitioned(0, 1, 1) assert(spark.read.format("delta").load(tablePath.toString).count() == 0) assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 0) } } test("incremental manifest: generate full manifest if manifest did not exist") { withTempDir { tablePath => def write(numPartitions: Int): Unit = { spark.range(0, 100, 1, 1).toDF("value").withColumn("part", $"value" % numPartitions) .write.format("delta").partitionBy("part").mode("append").save(tablePath.toString) } write(10) assertManifest(tablePath, expectSameFiles = false, expectedNumFiles = 0) withIncrementalManifest(tablePath, enabled = true) { write(1) // update only one partition } // Manifests should be generated for all partitions assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 11) } } test("incremental manifest: failure to generate manifest throws exception") { withTempDir { tablePath => tablePath.delete() import SymlinkManifestFailureTestFileSystem._ withSQLConf( s"fs.$SCHEME.impl" -> classOf[SymlinkManifestFailureTestFileSystem].getName, s"fs.$SCHEME.impl.disable.cache" -> "true", s"fs.AbstractFileSystem.$SCHEME.impl" -> classOf[SymlinkManifestFailureTestAbstractFileSystem].getName, s"fs.AbstractFileSystem.$SCHEME.impl.disable.cache" -> "true") { def write(numFiles: Int): Unit = withIncrementalManifest(tablePath, enabled = true) { spark.createDataset(spark.sparkContext.parallelize(1 to 100, numFiles)) .write.format("delta").mode("overwrite").save(s"$SCHEME://$tablePath") } val manifestPath = new File(tablePath, GenerateSymlinkManifest.MANIFEST_LOCATION) require(!manifestPath.exists()) write(1) // first write enables the property does not write any file require(!manifestPath.exists()) val ex = catalyst.util.quietly { intercept[RuntimeException] { write(2) } } assert(ex.getMessage().contains(GenerateSymlinkManifest.name)) assert(ex.getCause().toString.contains("Test exception")) } } } test("special partition column names") { def assertColNames(inputStr: String): Unit = withClue(s"input: $inputStr") { withTempDir { tablePath => tablePath.delete() val inputLines = inputStr.trim.stripMargin.trim.split("\n").toSeq require(inputLines.size > 0) val input = spark.read.json(inputLines.toDS) val partitionCols = input.schema.fieldNames val inputWithValue = input.withColumn("value", lit(1)) inputWithValue.write.format("delta").partitionBy(partitionCols: _*).save(tablePath.toString) generateSymlinkManifest(tablePath.toString) assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = inputLines.size) } } intercept[AnalysisException] { assertColNames("""{ " " : 0 }""") } assertColNames("""{ "%" : 0 }""") assertColNames("""{ "a.b." : 0 }""") assertColNames("""{ "a/b." : 0 }""") assertColNames("""{ "a_b" : 0 }""") intercept[AnalysisException] { assertColNames("""{ "a b" : 0 }""") } } test("special partition column values") { withTempDir { tablePath => tablePath.delete() val inputStr = """ |{ "part1" : 1, "part2": "$0$", "value" : 1 } |{ "part1" : null, "part2": "_1_", "value" : 1 } |{ "part1" : 1, "part2": "", "value" : 1 } |{ "part1" : null, "part2": " ", "value" : 1 } |{ "part1" : 1, "part2": " ", "value" : 1 } |{ "part1" : null, "part2": "/", "value" : 1 } |{ "part1" : 1, "part2": null, "value" : 1 } |""" val input = spark.read.json(inputStr.trim.stripMargin.trim.split("\n").toSeq.toDS) input.write.format("delta").partitionBy("part1", "part2").save(tablePath.toString) generateSymlinkManifest(tablePath.toString) assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 7) } } test("root table path with escapable chars like space") { withTempDir { p => val tablePath = new File(p.toString, "path with space") spark.createDataset(spark.sparkContext.parallelize(1 to 100, 1)).toDF("value") .withColumn("part", $"value" % 2) .write.format("delta").partitionBy("part").save(tablePath.toString) generateSymlinkManifest(tablePath.toString) assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 2) } } test("block manifest generation with persistent DVs") { withDeletionVectorsEnabled() { val rowsToBeRemoved = Seq(1L, 42L, 43L) withTempDir { dir => val tablePath = dir.getAbsolutePath // Write in 2 files. spark.range(end = 50L).toDF("id").coalesce(1) .write.format("delta").mode("overwrite").save(tablePath) spark.range(start = 50L, end = 100L).toDF("id").coalesce(1) .write.format("delta").mode("append").save(tablePath) val deltaLog = DeltaLog.forTable(spark, tablePath) assert(deltaLog.snapshot.allFiles.count() === 2L) // Step 1: Make sure generation works on DV enabled tables without a DV in the snapshot. // Delete an entire file, which can't produce DVs. spark.sql(s"""DELETE FROM delta.`$tablePath` WHERE id BETWEEN 0 and 49""") val remainingFiles = deltaLog.snapshot.allFiles.collect() assert(remainingFiles.size === 1L) assert(remainingFiles(0).deletionVector === null) // Should work fine, since the snapshot doesn't contain DVs. spark.sql(s"""GENERATE symlink_format_manifest FOR TABLE delta.`$tablePath`""") // Step 2: Make sure generation fails if there are DVs in the snapshot. // This is needed to make the manual commit work correctly, since we are not actually // running a command that produces metrics. withSQLConf(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "false") { val txn = deltaLog.startTransaction() assert(txn.snapshot.allFiles.count() === 1) val file = txn.snapshot.allFiles.collect().head val actions = removeRowsFromFileUsingDV(deltaLog, file, rowIds = rowsToBeRemoved) txn.commit(actions, Delete(predicate = Seq.empty)) } val e = intercept[DeltaCommandUnsupportedWithDeletionVectorsException] { spark.sql(s"""GENERATE symlink_format_manifest FOR TABLE delta.`$tablePath`""") } checkErrorHelper( exception = e, errorClass = "DELTA_UNSUPPORTED_GENERATE_WITH_DELETION_VECTORS") } } } private def setEnabledIncrementalManifest(tablePath: String, enabled: Boolean): Unit = { spark.sql(s"ALTER TABLE delta.`$tablePath` " + s"SET TBLPROPERTIES('${DeltaConfigs.SYMLINK_FORMAT_MANIFEST_ENABLED.key}'='$enabled')") } test("block incremental manifest generation with persistent DVs") { import DeltaTablePropertyValidationFailedSubClass._ def expectConstraintViolation(subClass: DeltaTablePropertyValidationFailedSubClass) (thunk: => Unit): Unit = { val e = intercept[DeltaTablePropertyValidationFailedException] { thunk } checkErrorHelper( exception = e, errorClass = "DELTA_VIOLATE_TABLE_PROPERTY_VALIDATION_FAILED." + subClass.tag ) } withDeletionVectorsEnabled() { val rowsToBeRemoved = Seq(1L, 42L, 43L) withTempDir { dir => val tablePath = dir.getAbsolutePath spark.range(end = 100L).toDF("id").coalesce(1) .write.format("delta").mode("overwrite").save(tablePath) val deltaLog = DeltaLog.forTable(spark, tablePath) // Make sure both properties can't be enabled together. enableDeletionVectorsInTable(new Path(tablePath), enable = true) expectConstraintViolation( subClass = PersistentDeletionVectorsWithIncrementalManifestGeneration) { setEnabledIncrementalManifest(tablePath, enabled = true) } // Or in the other order. enableDeletionVectorsInTable(new Path(tablePath), enable = false) setEnabledIncrementalManifest(tablePath, enabled = true) expectConstraintViolation( subClass = PersistentDeletionVectorsWithIncrementalManifestGeneration) { enableDeletionVectorsInTable(new Path(tablePath), enable = true) } setEnabledIncrementalManifest(tablePath, enabled = false) // Or both at once. expectConstraintViolation( subClass = PersistentDeletionVectorsWithIncrementalManifestGeneration) { spark.sql(s"ALTER TABLE delta.`$tablePath` " + s"SET TBLPROPERTIES('${DeltaConfigs.SYMLINK_FORMAT_MANIFEST_ENABLED.key}'='true'," + s" '${DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key}' = 'true')") } // If DVs were allowed at some point and are still present in the table, // enabling incremental manifest generation must still fail. enableDeletionVectorsInTable(new Path(tablePath), enable = true) withSQLConf(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "false") { val txn = deltaLog.startTransaction() assert(txn.snapshot.allFiles.count() === 1) val file = txn.snapshot.allFiles.collect().head val actions = removeRowsFromFileUsingDV(deltaLog, file, rowIds = rowsToBeRemoved) txn.commit(actions, Delete(predicate = Seq.empty)) } assert(getFilesWithDeletionVectors(deltaLog).nonEmpty) enableDeletionVectorsInTable(new Path(tablePath), enable = false) expectConstraintViolation( subClass = ExistingDeletionVectorsWithIncrementalManifestGeneration) { setEnabledIncrementalManifest(tablePath, enabled = true) } // Purge spark.sql(s"REORG TABLE delta.`$tablePath` APPLY (PURGE)") assert(getFilesWithDeletionVectors(deltaLog).isEmpty) // Now it should work. setEnabledIncrementalManifest(tablePath, enabled = true) // As a last fallback, in case some other writer put the table into an illegal state, // we still need to fail the manifest generation if there are DVs. // Reset table. setEnabledIncrementalManifest(tablePath, enabled = false) enableDeletionVectorsInTable(new Path(tablePath), enable = false) spark.range(end = 100L).toDF("id").coalesce(1) .write.format("delta").mode("overwrite").save(tablePath) // Add DVs enableDeletionVectorsInTable(new Path(tablePath), enable = true) withSQLConf(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "false") { val txn = deltaLog.startTransaction() assert(txn.snapshot.allFiles.count() === 1) val file = txn.snapshot.allFiles.collect().head val actions = removeRowsFromFileUsingDV(deltaLog, file, rowIds = rowsToBeRemoved) txn.commit(actions, Delete(predicate = Seq.empty)) } // Force enable manifest generation. withSQLConf(DeltaSQLConf.DELTA_TABLE_PROPERTY_CONSTRAINTS_CHECK_ENABLED.key -> "false") { setEnabledIncrementalManifest(tablePath, enabled = true) } val e2 = intercept[DeltaCommandUnsupportedWithDeletionVectorsException] { spark.range(10).write.format("delta").mode("append").save(tablePath) } checkErrorHelper( exception = e2, errorClass = "DELTA_UNSUPPORTED_GENERATE_WITH_DELETION_VECTORS") // This is fine, since the new snapshot won't contain DVs. spark.range(10).write.format("delta").mode("overwrite").save(tablePath) // Make sure we can get the table back into a consistent state, as well setEnabledIncrementalManifest(tablePath, enabled = false) // No more exception. spark.range(10).write.format("delta").mode("append").save(tablePath) } } } private def checkErrorHelper( exception: SparkThrowable, errorClass: String ): Unit = { assert(exception.getErrorClass === errorClass, s"Expected errorClass $errorClass, but got $exception") } Seq(true, false).foreach { useIncremental => test(s"delete partition column with special char - incremental=$useIncremental") { def writePartition(dir: File, partName: String): Unit = { spark.range(10) .withColumn("part", lit(partName)) .repartition(1) .write .format("delta") .mode("append") .partitionBy("part") .save(dir.toString) } withTempDir { dir => // create table and write first manifest writePartition(dir, "noSpace") generateSymlinkManifest(dir.toString) withIncrementalManifest(dir, useIncremental) { // 1. test paths with spaces writePartition(dir, "yes space") if (!useIncremental) { generateSymlinkManifest(dir.toString) } assertManifest(dir, expectSameFiles = true, expectedNumFiles = 2) // delete partition sql(s"""DELETE FROM delta.`${dir.toString}` WHERE part="yes space";""") if (!useIncremental) { generateSymlinkManifest(dir.toString) } assertManifest(dir, expectSameFiles = true, expectedNumFiles = 1) // 2. test special characters // scalastyle:off nonascii writePartition(dir, "库尔 勒") if (!useIncremental) { generateSymlinkManifest(dir.toString) } assertManifest(dir, expectSameFiles = true, expectedNumFiles = 2) // delete partition sql(s"""DELETE FROM delta.`${dir.toString}` WHERE part="库尔 勒";""") // scalastyle:on nonascii if (!useIncremental) { generateSymlinkManifest(dir.toString) } assertManifest(dir, expectSameFiles = true, expectedNumFiles = 1) } } } } } trait DeltaGenerateSymlinkManifestTestHelper extends QueryTest with SharedSparkSession { import testImplicits._ protected def assertManifest( tablePath: File, expectSameFiles: Boolean, expectedNumFiles: Int): Unit = { val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, tablePath.toString) assertManifest(snapshot, tablePath, expectSameFiles, expectedNumFiles) } protected def assertManifest( tableName: String, expectSameFiles: Boolean, expectedNumFiles: Int): Unit = { val (log, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName)) assertManifest(snapshot, new File(log.dataPath.toUri), expectSameFiles, expectedNumFiles) } /** * Assert that the manifest files in the table meet the expectations. * @param deltaSnapshot Snapshot of the Delta table to check against * @param tablePath Path of the Delta table * @param expectSameFiles Expect that the manifest files contain the same data files * as the latest version of the table * @param expectedNumFiles Expected number of manifest files */ private def assertManifest( deltaSnapshot: Snapshot, tablePath: File, expectSameFiles: Boolean, expectedNumFiles: Int): Unit = { val manifestPath = new File(tablePath, GenerateSymlinkManifest.MANIFEST_LOCATION) if (!manifestPath.exists) { assert(expectedNumFiles == 0 && !expectSameFiles) return } // Validate the expected number of files are present in the manifest val filesInManifest = spark.read.text(manifestPath.toString) .select("value") .as[String] .collect() .map(_.stripPrefix("file:")) .toSeq .toDF("file") assert(filesInManifest.count() == expectedNumFiles) // Validate that files in the latest version of DeltaLog is same as those in the manifest val filesInLog = deltaSnapshot.allFiles .collect() .map { addFile => // Note: this unescapes the relative path in `addFile` DeltaFileOperations.absolutePath(tablePath.toString, addFile.path).toString } .toSeq .toDF("file") if (expectSameFiles) { checkAnswer(filesInManifest, filesInLog) // Validate that each file in the manifest is actually present in table. This mainly checks // whether the file names in manifest are not escaped and therefore are readable directly // by Hadoop APIs. val fs = new Path(manifestPath.toString) .getFileSystem(deltaSnapshot.deltaLog.newDeltaHadoopConf()) spark.read.text(manifestPath.toString).select("value").as[String].collect().foreach { p => assert(fs.exists(new Path(p)), s"path $p in manifest not found in file system") } } else { assert(filesInManifest.as[String].collect().toSet != filesInLog.as[String].collect().toSet) } // If there are partitioned files, make sure the partitions values read from them are the // same as those in the table. val partitionCols = deltaSnapshot.metadata.partitionColumns.map(x => s"`$x`") if (partitionCols.nonEmpty && expectSameFiles && expectedNumFiles > 0) { val partitionsInManifest = spark.read.text(manifestPath.toString) .selectExpr(partitionCols: _*).distinct() val partitionsInData = spark.read.format("delta").load(tablePath.toString) .selectExpr(partitionCols: _*).distinct() checkAnswer(partitionsInManifest, partitionsInData) } } protected def withIncrementalManifest(tablePath: File, enabled: Boolean)(func: => Unit): Unit = { if (tablePath.exists()) { val latestMetadata = DeltaLog.forTable(spark, tablePath).update().metadata if (DeltaConfigs.SYMLINK_FORMAT_MANIFEST_ENABLED.fromMetaData(latestMetadata) != enabled) { spark.sql(s"ALTER TABLE delta.`$tablePath` " + s"SET TBLPROPERTIES(${DeltaConfigs.SYMLINK_FORMAT_MANIFEST_ENABLED.key}=$enabled)") } } func } protected def generateSymlinkManifest(tablePath: String): Unit = { val deltaLog = DeltaLog.forTable(spark, tablePath) GenerateSymlinkManifest.generateFullManifest(spark, deltaLog, catalogTableOpt = None) } } class SymlinkManifestFailureTestAbstractFileSystem( uri: URI, conf: org.apache.hadoop.conf.Configuration) extends org.apache.hadoop.fs.DelegateToFileSystem( uri, new SymlinkManifestFailureTestFileSystem, conf, SymlinkManifestFailureTestFileSystem.SCHEME, false) { // Implementation copied from RawLocalFs import org.apache.hadoop.fs.local.LocalConfigKeys import org.apache.hadoop.fs._ override def getUriDefaultPort(): Int = -1 override def getServerDefaults(): FsServerDefaults = LocalConfigKeys.getServerDefaults() override def isValidName(src: String): Boolean = true } class SymlinkManifestFailureTestFileSystem extends RawLocalFileSystem { private var uri: URI = _ override def getScheme: String = SymlinkManifestFailureTestFileSystem.SCHEME override def initialize(name: URI, conf: Configuration): Unit = { uri = URI.create(name.getScheme + ":///") super.initialize(name, conf) } override def getUri(): URI = if (uri == null) { // RawLocalFileSystem's constructor will call this one before `initialize` is called. // Just return the super's URI to avoid NPE. super.getUri } else { uri } // Override both create() method defined in RawLocalFileSystem such that any file creation // throws error. override def create( path: Path, overwrite: Boolean, bufferSize: Int, replication: Short, blockSize: Long, progress: Progressable): FSDataOutputStream = { if (path.toString.contains(GenerateSymlinkManifest.MANIFEST_LOCATION)) { throw new RuntimeException("Test exception") } super.create(path, overwrite, bufferSize, replication, blockSize, null) } override def create( path: Path, permission: FsPermission, overwrite: Boolean, bufferSize: Int, replication: Short, blockSize: Long, progress: Progressable): FSDataOutputStream = { if (path.toString.contains(GenerateSymlinkManifest.MANIFEST_LOCATION)) { throw new RuntimeException("Test exception") } super.create(path, permission, overwrite, bufferSize, replication, blockSize, progress) } } object SymlinkManifestFailureTestFileSystem { val SCHEME = "testScheme" } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaHistoryManagerSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.{File, FileNotFoundException} import java.net.URI import java.nio.charset.StandardCharsets.UTF_8 import java.sql.Timestamp import java.text.SimpleDateFormat import java.util.{Date, Locale} import scala.concurrent.duration._ import scala.language.implicitConversions import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta.DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED import org.apache.spark.sql.delta.DeltaTestUtils.{createTestAddFile, modifyCommitTimestamp} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.StatsUtils import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.FileNames import org.scalatest.GivenWhenThen import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.catalyst.util.quietly import org.apache.spark.sql.connector.catalog.CatalogManager import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.Utils /** A set of tests which we can open source after Spark 3.0 is released. */ trait DeltaTimeTravelTests extends QueryTest with SharedSparkSession with GivenWhenThen with DeltaSQLCommandTest with StatsUtils with CatalogOwnedTestBaseSuite { protected implicit def durationToLong(duration: FiniteDuration): Long = { duration.toMillis } protected implicit def longToTimestamp(ts: Long): Timestamp = new Timestamp(ts) protected val timeFormatter = new SimpleDateFormat("yyyyMMddHHmmssSSS") protected def versionAsOf(table: String, version: Long): String = { s"$table version as of $version" } protected def timestampAsOf(table: String, expr: String): String = { s"$table timestamp as of $expr" } protected def verifyLogging( tableVersion: Long, queriedVersion: Long, accessType: String, apiUsed: String)(f: => Unit): Unit = { // TODO: would be great to verify our logging metrics } protected def getTableLocation(table: String): String = { spark.sessionState.catalog.getTableMetadata(TableIdentifier(table)).location.toString } /** Generate commits with the given timestamp in millis. */ protected def generateCommitsCheap( deltaLog: DeltaLog, commits: Long*): Unit = { var startVersion = deltaLog.snapshot.version + 1 commits.foreach { ts => val action = createTestAddFile(encodedPath = startVersion.toString, modificationTime = startVersion) deltaLog.startTransaction().commitManually(action) modifyCommitTimestamp(deltaLog, startVersion, ts) startVersion += 1 } } protected def generateCommitsAtPath(table: String, path: String, commits: Long*): Unit = { generateCommitsBase(table, Some(path), commits: _*) } /** Generate commits with the given timestamp in millis. */ protected def generateCommits(table: String, commits: Long*): Unit = { generateCommitsBase(table, None, commits: _*) } private def generateCommitsBase(table: String, path: Option[String], commits: Long*): Unit = { var commitList = commits.toSeq if (commitList.isEmpty) return if (!spark.sessionState.catalog.tableExists(TableIdentifier(table))) { if (path.isDefined) { spark.range(0, 10).write.format("delta") .mode("append") .option("path", path.get) .saveAsTable(table) } else { spark.range(0, 10).write.format("delta") .mode("append") .saveAsTable(table) } val deltaLog = DeltaLog.forTable(spark, new TableIdentifier(table)) modifyCommitTimestamp(deltaLog, 0, commitList.head) commitList = commits.slice(1, commits.length) // we already wrote the first commit here var startVersion = deltaLog.snapshot.version + 1 commitList.foreach { ts => val rangeStart = startVersion * 10 val rangeEnd = rangeStart + 10 spark.range(rangeStart, rangeEnd).write.format("delta").mode("append").saveAsTable(table) modifyCommitTimestamp(deltaLog, startVersion, ts) startVersion += 1 } } } /** Alternate for `withTables` as we leave some tables in an unusable state for clean up */ protected def withTable(tableName: String, dir: String)(f: => Unit): Unit = { try f finally { try { Utils.deleteRecursively(new File(dir.toString)) } catch { case _: Throwable => Nil // do nothing, this can fail if the table was deleted by the test. } finally { try { sql(s"DROP TABLE IF EXISTS $tableName") } catch { case _: Throwable => // There is one test that fails the drop table as well // we ignore this exception as that test uses a path based location. Nil } } } } protected implicit def longToTimestampExpr(value: Long): String = { s"cast($value / 1000 as timestamp)" } import testImplicits._ test("time travel with partition changes and data skipping - should instantiate old schema") { withTempDir { dir => val tblLoc = dir.getCanonicalPath val v0 = spark.range(10).withColumn("part5", 'id % 5) v0.write.format("delta").partitionBy("part5").mode("append").save(tblLoc) val deltaLog = DeltaLog.forTable(spark, tblLoc) val schemaString = spark.range(10, 20).withColumn("part2", 'id % 2).schema.json deltaLog.startTransaction().commit( Seq(deltaLog.snapshot.metadata.copy( schemaString = schemaString, partitionColumns = Seq("part2"))), DeltaOperations.ManualUpdate ) checkAnswer( spark.read.option("versionAsOf", 0).format("delta").load(tblLoc).where("part5 = 1"), v0.where("part5 = 1")) } } test("can't provide both version and timestamp in DataFrameReader") { val e = intercept[IllegalArgumentException] { spark.read.option("versionaSof", 1) .option("timestampAsOF", "fake").format("delta").load("/some/fake") } assert(e.getMessage.contains("either provide 'timestampAsOf' or 'versionAsOf'")) } test("don't time travel a valid non-delta path with @ syntax") { val format = "json" withTempDir { dir => val path = new File(dir, "base@v0").getCanonicalPath spark.range(10).write.format(format).mode("append").save(path) spark.range(10).write.format(format).mode("append").save(path) checkAnswer( spark.read.format(format).load(path), spark.range(10).union(spark.range(10)).toDF() ) checkAnswer( spark.table(s"$format.`$path`"), spark.range(10).union(spark.range(10)).toDF() ) intercept[AnalysisException] { spark.read.format(format).load(path + "@v0").count() } intercept[AnalysisException] { spark.table(s"$format.`$path@v0`").count() } } } /////////////////////////// // Time Travel SQL Tests // /////////////////////////// test("AS OF support does not impact non-delta tables") { withTable("t1") { spark.range(10).write.format("parquet").mode("append").saveAsTable("t1") spark.range(10, 20).write.format("parquet").mode("append").saveAsTable("t1") // We should still use the default, non-delta code paths for a non-delta table. // For parquet, that means to fail with QueryCompilationErrors::tableNotSupportTimeTravelError val e = intercept[Exception] { spark.sql("SELECT * FROM t1 VERSION AS OF 0") }.getMessage assert(e.contains("does not support time travel") || e.contains("The feature is not supported: Time travel on the relation")) } } // scalastyle:off line.size.limit test("as of timestamp in between commits should use commit before timestamp") { // scalastyle:off line.size.limit val tblName = "delta_table" withTable(tblName) { val start = System.currentTimeMillis() - 5.days.toMillis generateCommits(tblName, start, start + 20.minutes, start + 40.minutes) verifyLogging(2L, 0L, "timestamp", "sql") { checkAnswer( sql(s"select count(*) from ${timestampAsOf(tblName, start + 10.minutes)}"), Row(10L) ) } verifyLogging(2L, 0L, "timestamp", "sql") { checkAnswer( sql("select count(*) from " + s"${timestampAsOf(s"delta.`${getTableLocation(tblName)}`", start + 10.minutes)}"), Row(10L) ) } checkAnswer( sql(s"select count(*) from ${timestampAsOf(tblName, start + 30.minutes)}"), Row(20L) ) } } test("as of timestamp on exact timestamp") { val tblName = "delta_table" withTable(tblName) { val start = System.currentTimeMillis() - 5.days.toMillis generateCommits(tblName, start, start + 20.minutes) // Simulate getting the timestamp directly from Spark SQL val ts = Seq(new Timestamp(start), new Timestamp(start + 20.minutes)).toDF("ts") .select($"ts".cast("string")).as[String].collect() .map(i => s"'$i'") checkAnswer( sql(s"select count(*) from ${timestampAsOf(tblName, ts(0))}"), Row(10L) ) checkAnswer( sql(s"select count(*) from ${timestampAsOf(tblName, start)}"), Row(10L) ) checkAnswer( sql(s"select count(*) from ${timestampAsOf(tblName, start + 20.minutes)}"), Row(20L) ) checkAnswer( sql(s"select count(*) from ${timestampAsOf(tblName, ts(1))}"), Row(20L) ) } } test("as of with versions") { val tblName = s"delta_table" withTempDir { dir => withTable(tblName, dir.toString) { val start = System.currentTimeMillis() - 5.days.toMillis generateCommitsAtPath(tblName, dir.toString, start, start + 20.minutes, start + 40.minutes) verifyLogging(2L, 0L, "version", "sql") { checkAnswer( sql(s"select count(*) from ${versionAsOf(tblName, 0)}"), Row(10L) ) } verifyLogging(2L, 0L, "version", "dfReader") { checkAnswer( spark.read.format("delta").option("versionAsOf", "0") .load(getTableLocation(tblName)).groupBy().count(), Row(10) ) } checkAnswer( sql(s"select count(*) from ${versionAsOf(tblName, 1)}"), Row(20L) ) checkAnswer( spark.read.format("delta").option("versionAsOf", 1) .load(getTableLocation(tblName)).groupBy().count(), Row(20) ) checkAnswer( sql(s"select count(*) from ${versionAsOf(tblName, 2)}"), Row(30L) ) val e1 = intercept[AnalysisException] { sql(s"select count(*) from ${versionAsOf(tblName, 3)}").collect() } assert(e1.getMessage.contains("[0, 2]")) val deltaLog = DeltaLog.forTable(spark, getTableLocation(tblName)) new File(FileNames.unsafeDeltaFile(deltaLog.logPath, 0).toUri).delete() // Delta Lake will create a DeltaTableV2 explicitly with time travel options in the catalog. // These options will be verified by DeltaHistoryManager, which will throw an // AnalysisException. val e2 = intercept[AnalysisException] { sql(s"select count(*) from ${versionAsOf(tblName, 0)}").collect() } if (catalogOwnedCoordinatorBackfillBatchSize.exists(_ > 2)) { assert(e2.getMessage.contains("No commits found at")) } else { assert(e2.getMessage.contains("No recreatable commits found at")) } } } } test("as of exact timestamp after last commit should fail") { val tblName = "delta_table" withTable(tblName) { val start = 1540415658000L generateCommits(tblName, start) // Simulate getting the timestamp directly from Spark SQL val ts = Seq(new Timestamp(start + 10.minutes)).toDF("ts") .select($"ts".cast("string")).as[String].collect() .map(i => s"'$i'") val e1 = intercept[DeltaErrors.TemporallyUnstableInputException] { sql(s"select count(*) from ${timestampAsOf(tblName, ts(0))}").collect() } checkError( e1, "DELTA_TIMESTAMP_GREATER_THAN_COMMIT", sqlState = "42816", parameters = Map( "providedTimestamp" -> "2018-10-24 14:24:18.0", "tableName" -> "2018-10-24 14:14:18.0", "maximumTimestamp" -> "2018-10-24 14:14:18") ) val e2 = intercept[DeltaErrors.TemporallyUnstableInputException] { sql(s"select count(*) from ${timestampAsOf(tblName, start + 10.minutes)}").collect() } checkError( e2, "DELTA_TIMESTAMP_GREATER_THAN_COMMIT", sqlState = "42816", parameters = Map( "providedTimestamp" -> "2018-10-24 14:24:18.0", "tableName" -> "2018-10-24 14:14:18.0", "maximumTimestamp" -> "2018-10-24 14:14:18") ) checkAnswer( sql(s"select count(*) from ${timestampAsOf(tblName, "'2018-10-24 14:14:18'")}"), Row(10) ) verifyLogging(0L, 0L, "timestamp", "dfReader") { checkAnswer( spark.read.format("delta").option("timestampAsOf", "2018-10-24 14:14:18") .load(getTableLocation(tblName)).groupBy().count(), Row(10) ) } } } test("time travelling with adjusted timestamps") { if (isICTEnabledForNewTablesCatalogOwned) { // ICT Timestamps are always monotonically increasing. Therefore, // this test is not needed when ICT is enabled. cancel("This test is not compatible with InCommitTimestamps.") } val tblName = "delta_table" withTable(tblName) { val start = System.currentTimeMillis() - 5.days.toMillis generateCommits(tblName, start, start - 5.seconds, start + 3.minutes) checkAnswer( sql(s"select count(*) from ${timestampAsOf(tblName, start)}"), Row(10L) ) checkAnswer( sql(s"select count(*) from ${timestampAsOf(tblName, start + 1.milli)}"), Row(20L) ) checkAnswer( sql(s"select count(*) from ${timestampAsOf(tblName, start + 119.seconds)}"), Row(20L) ) val e = intercept[AnalysisException] { sql(s"select count(*) from ${timestampAsOf(tblName, start - 3.seconds)}").collect() } assert(e.getMessage.contains("before the earliest version")) } } test("Time travel with schema changes") { val tblName = "delta_table" withTable(tblName) { spark.range(10).write.format("delta").mode("append").saveAsTable(tblName) sql(s"ALTER TABLE $tblName ADD COLUMNS (part bigint)") spark.range(10, 20).withColumn("part", 'id) .write.format("delta").mode("append").saveAsTable(tblName) val tableLoc = getTableLocation(tblName) checkAnswer( sql(s"select * from ${versionAsOf(tblName, 0)}"), spark.range(10).toDF()) checkAnswer( sql(s"select * from ${versionAsOf(s"delta.`$tableLoc`", 0)}"), spark.range(10).toDF()) checkAnswer( spark.read.option("versionAsOf", 0).format("delta").load(tableLoc), spark.range(10).toDF()) } } test("data skipping still works with time travel") { val tblName = "delta_table" withTable(tblName) { val start = System.currentTimeMillis() - 5.days.toMillis generateCommits(tblName, start, start + 20.minutes) def testScan(df: DataFrame): Unit = { val scan = getStats(df) assert(scan.scanned.bytesCompressed.get < scan.total.bytesCompressed.get) } testScan(sql(s"select * from ${versionAsOf(tblName, 0)} where id = 2")) testScan(spark.read.format("delta").option("versionAsOf", 0).load(getTableLocation(tblName)) .where("id = 2")) } } test("fail to time travel a different relation than Delta") { withTempDir { output => val dir = output.getCanonicalPath spark.range(10).write.mode("append").parquet(dir) spark.range(10).write.mode("append").parquet(dir) def assertFormatFailure(f: => Unit): Unit = { val e = intercept[AnalysisException] { f } assert( e.getMessage.contains("path-based tables") || e.message.contains("[UNSUPPORTED_FEATURE.TIME_TRAVEL] The feature is not supported"), s"Returned instead:\n$e") } assertFormatFailure { sql(s"select * from ${versionAsOf(s"parquet.`$dir`", 0)}").collect() } assertFormatFailure { sql(s"select * from ${versionAsOf(s"parquet.`$dir`", 0)}").collect() } checkAnswer( spark.read.option("versionAsOf", 0).parquet(dir), // do not time travel other relations spark.range(10).union(spark.range(10)).toDF() ) checkAnswer( // do not time travel other relations spark.read.option("timestampAsOf", "2018-10-12 01:01:01").parquet(dir), spark.range(10).union(spark.range(10)).toDF() ) val tblName = "parq_table" withTable(tblName) { sql(s"create table $tblName using parquet as select * from parquet.`$dir`") val e = intercept[Exception] { sql(s"select * from ${versionAsOf(tblName, 0)}").collect() } val catalogName = CatalogManager.SESSION_CATALOG_NAME val catalogPrefix = catalogName + "." assert(e.getMessage.contains( s"Table ${catalogPrefix}default.parq_table does not support time travel") || e.getMessage.contains(s"Time travel on the relation: `$catalogName`.`default`.`parq_table`")) } val viewName = "parq_view" assertFormatFailure { sql(s"create temp view $viewName as select * from ${versionAsOf(s"parquet.`$dir`", 0)}") } } } } abstract class DeltaHistoryManagerBase extends DeltaTimeTravelTests { test("cannot time travel target tables of insert/delete/update/merge") { val tblName = "delta_table" withTable(tblName) { val start = 1540415658000L generateCommits(tblName, start, start + 20.minutes) // These all actually fail parsing intercept[ParseException] { sql(s"insert into ${versionAsOf(tblName, 0)} values (11, 12, 13)") } intercept[ParseException] { sql(s"update ${versionAsOf(tblName, 0)} set id = id - 1 where id < 10") } intercept[ParseException] { sql(s"delete from ${versionAsOf(tblName, 0)} id < 10") } intercept[ParseException] { sql(s"""merge into ${versionAsOf(tblName, 0)} old |using $tblName new |on old.id = new.id |when not matched then insert * """.stripMargin) } } } test("vacuumed version") { assume(!catalogOwnedDefaultCreationEnabledInTests, "VACUUM is blocked on catalog-managed tables") quietly { val tblName = "delta_table" withTable(tblName) { val start = System.currentTimeMillis() - 5.days.toMillis generateCommits(tblName, start, start + 20.minutes) sql(s"optimize $tblName") withSQLConf( // Disable query rewrite or else the parquet files are not scanned. DeltaSQLConf.DELTA_OPTIMIZE_METADATA_QUERY_ENABLED.key -> "false", DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED.key -> "false") { sql(s"vacuum $tblName retain 0 hours") intercept[SparkException] { sql(s"select * from ${versionAsOf(tblName, 0)}").collect() } intercept[SparkException] { sql(s"select count(*) from ${versionAsOf(tblName, 1)}").collect() } } } } } test("as of with table API") { val tblName = "delta_table" withTable(tblName) { val start = System.currentTimeMillis() - 5.days.toMillis generateCommits(tblName, start, start + 20.minutes, start + 40.minutes) assert(spark.read.format("delta").option("versionAsOf", "0").table(tblName).count() == 10) assert(spark.read.format("delta").option("versionAsOf", 1).table(tblName).count() == 20) assert(spark.read.format("delta").option("versionAsOf", 2).table(tblName).count() == 30) val e1 = intercept[AnalysisException] { spark.read.format("delta").option("versionAsOf", 3).table(tblName).collect() } assert(e1.getMessage.contains("[0, 2]")) val e2 = intercept[org.apache.spark.sql.AnalysisException] { spark.read.format("delta") .option("versionAsOf", 3) .option("timestampAsOf", "2020-10-22 23:20:11") .table(tblName).collect() } assert(e2.getMessage.contains("Cannot specify both version and timestamp")) } } test("getHistory returns the correct set of commits") { val tblName = "delta_table" withTable(tblName) { val start = 1540415658000L generateCommits(tblName, start, start + 20.minutes, start + 40.minutes, start + 60.minutes) val table = DeltaTableV2(spark, TableIdentifier(tblName)) def testGetHistory( start: Long, endOpt: Option[Long], versions: Seq[Long], expectedLogUpdates: Int): Unit = { val usageRecords = Log4jUsageLogger.track { val history = table.deltaLog.history.getHistory(start, endOpt, table.catalogTable) assert(history.map(_.getVersion) == versions) } assert(filterUsageRecords(usageRecords, "deltaLog.update").size === expectedLogUpdates) } testGetHistory(start = 0, endOpt = Some(2), versions = Seq(2, 1, 0), expectedLogUpdates = 0) testGetHistory(start = 1, endOpt = Some(1), versions = Seq(1), expectedLogUpdates = 0) testGetHistory(start = 2, endOpt = None, versions = Seq(3, 2), expectedLogUpdates = 1) testGetHistory(start = 1, endOpt = Some(5), versions = Seq(3, 2, 1), expectedLogUpdates = 1) testGetHistory(start = 4, endOpt = None, versions = Seq.empty, expectedLogUpdates = 1) testGetHistory(start = 2, endOpt = Some(1), versions = Seq.empty, expectedLogUpdates = 0) } } test("getCommitFromNonICTRange should handle empty history by throwing proper error") { val tblName = "delta_table" withTable(tblName) { val start = 1540415658000L generateCommits(tblName, start) val deltaLog = DeltaLog.forTable(spark, getTableLocation(tblName)) val deltaFile = new File(FileNames.unsafeDeltaFile(deltaLog.logPath, 0).toUri) assert(deltaFile.delete(), "Failed to delete delta log file") val e = intercept[DeltaAnalysisException] { deltaLog.history.getCommitFromNonICTRange(0, 1, start) } assert(e.getMessage.contains("DELTA_NO_COMMITS_FOUND")) assert(e.getMessage.contains(deltaLog.logPath.toString)) } } test("parallel search handles empty commits in a partition correctly") { if (catalogOwnedDefaultCreationEnabledInTests) { cancel("This test is not compatible with coordinated commits backfill timestamps.") } val tblName = "delta_table" withTable(tblName) { // Small threshold to trigger parallel search withSQLConf( DeltaSQLConf.DELTA_HISTORY_PAR_SEARCH_THRESHOLD.key -> "3", IN_COMMIT_TIMESTAMPS_ENABLED.key -> "false") { val start = 1540415658000L // Generate 10 commits which will be processed in parallel due to threshold=3 val timestamps = (0 to 9).map(i => start + (i * 20).minutes) generateCommits(tblName, timestamps: _*) val table = DeltaTableV2(spark, TableIdentifier(tblName)) val deltaLog = table.deltaLog // Delete all files in first partition to simulate concurrent metadata cleanup val deltaFiles = (0 to 4).map { version => new File(FileNames.unsafeDeltaFile(deltaLog.logPath, version).toUri) } deltaFiles.foreach(f => assert(f.delete(), s"Failed to delete delta log file ${f.getPath}")) assert( deltaLog.history.getCommitFromNonICTRange(0, 9, start + (7 * 20).minutes).version == 7) } } } } /** Uses V2 resolution code paths */ class DeltaHistoryManagerSuite extends DeltaHistoryManagerBase { override protected def sparkConf: SparkConf = { super.sparkConf.set(SQLConf.USE_V1_SOURCE_LIST.key, "parquet,json") } } class DeltaHistoryManagerWithCatalogOwnedBatch1Suite extends DeltaHistoryManagerSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class DeltaHistoryManagerWithCatalogOwnedBatch2Suite extends DeltaHistoryManagerSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class DeltaHistoryManagerWithCatalogOwnedBatch100Suite extends DeltaHistoryManagerSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaImplicitsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.SparkFunSuite import org.apache.spark.sql.test.SharedSparkSession class DeltaImplicitsSuite extends SparkFunSuite with SharedSparkSession { private def testImplict(name: String, func: => Unit): Unit = { test(name) { func } } import org.apache.spark.sql.delta.implicits._ testImplict("int", intEncoder) testImplict("long", longEncoder) testImplict("string", stringEncoder) testImplict("longLong", longLongEncoder) testImplict("stringLong", stringLongEncoder) testImplict("stringString", stringStringEncoder) testImplict("javaLong", javaLongEncoder) testImplict("singleAction", singleActionEncoder) testImplict("addFile", addFileEncoder) testImplict("removeFile", removeFileEncoder) testImplict("serializableFileStatus", serializableFileStatusEncoder) testImplict("indexedFile", indexedFileEncoder) testImplict("addFileWithIndex", addFileWithIndexEncoder) testImplict("addFileWithSourcePath", addFileWithSourcePathEncoder) testImplict("deltaHistoryEncoder", deltaHistoryEncoder) testImplict("historyCommitEncoder", historyCommitEncoder) testImplict("snapshotStateEncoder", snapshotStateEncoder) testImplict("RichAddFileSeq: toDF", Seq(AddFile("foo", Map.empty, 0, 0, true)).toDF(spark)) testImplict("RichAddFileSeq: toDS", Seq(AddFile("foo", Map.empty, 0, 0, true)).toDS(spark)) testImplict("RichStringSeq: toDF", Seq("foo").toDF(spark)) testImplict("RichStringSeq: toDF(col)", Seq("foo").toDF(spark, "str")) testImplict("RichIntSeq: toDF", Seq(1).toDF(spark)) testImplict("RichIntSeq: toDF(col)", Seq(1).toDF(spark, "int")) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaIncrementalSetTransactionsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.util.UUID // scalastyle:off import.ordering.noEmptyLine import com.databricks.spark.util.UsageRecord import org.apache.spark.sql.delta.DeltaTestUtils.{collectUsageLogs, createTestAddFile, BOOLEAN_DOMAIN} import org.apache.spark.sql.delta.actions.{AddFile, SetTransaction, SingleAction} import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.sql.{QueryTest, SaveMode} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.col import org.apache.spark.sql.test.SharedSparkSession class DeltaIncrementalSetTransactionsSuite extends QueryTest with DeltaSQLCommandTest with SharedSparkSession with CatalogOwnedTestBaseSuite { protected override def sparkConf = super.sparkConf .set(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key, "true") .set(DeltaSQLConf.DELTA_WRITE_SET_TRANSACTIONS_IN_CRC.key, "true") // needed for DELTA_WRITE_SET_TRANSACTIONS_IN_CRC .set(DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key, "true") // This test suite is sensitive to stateReconstruction we do at different places. So we disable // [[INCREMENTAL_COMMIT_FORCE_VERIFY_IN_TESTS]] to simulate prod behaviour. .set(DeltaSQLConf.INCREMENTAL_COMMIT_FORCE_VERIFY_IN_TESTS.key, "false") /** * Validates the result of [[Snapshot.setTransactions]] API for the latest snapshot of this * [[DeltaLog]]. */ private def assertSetTransactions( deltaLog: DeltaLog, expectedTxns: Map[String, Long], viaCRC: Boolean = false ): Unit = { val snapshot = deltaLog.update() if (viaCRC) { assert(snapshot.checksumOpt.flatMap(_.setTransactions).isDefined) snapshot.checksumOpt.flatMap(_.setTransactions).foreach { setTxns => assert(setTxns.map(txn => (txn.appId, txn.version)).toMap === expectedTxns) } } assert(snapshot.setTransactions.map(txn => (txn.appId, txn.version)).toMap === expectedTxns) assert(snapshot.numOfSetTransactions === expectedTxns.size) assert(expectedTxns === snapshot.transactions) } /** Commit given [[SetTransaction]] to `deltaLog`` */ private def commitSetTxn( deltaLog: DeltaLog, appId: String, version: Long, lastUpdated: Long): Unit = { commitSetTxn(deltaLog, Seq(SetTransaction(appId, version, Some(lastUpdated)))) } /** Commit given [[SetTransaction]]s to `deltaLog`` */ private def commitSetTxn( deltaLog: DeltaLog, setTransactions: Seq[SetTransaction]): Unit = { deltaLog.startTransaction().commit( // Use createTestAddFile to create addFile with default stats for RowTracking. setTransactions :+ createTestAddFile(encodedPath = s"file-${UUID.randomUUID().toString}"), DeltaOperations.Write(SaveMode.Append) ) } test( "set-transaction tracking starts from 0th commit in CRC" ) { withSQLConf( DeltaSQLConf.DELTA_WRITE_SET_TRANSACTIONS_IN_CRC.key -> "true" ) { val tbl = "test_table" withTable(tbl) { sql(s"CREATE TABLE $tbl USING delta as SELECT 1 as value") // 0th commit val log = DeltaLog.forTable(spark, TableIdentifier(tbl)) log.update() // CRC for 0th commit has SetTransactions defined and are empty Seq. assert(log.unsafeVolatileSnapshot.checksumOpt.flatMap(_.setTransactions).isDefined) assert(log.unsafeVolatileSnapshot.checksumOpt.flatMap(_.setTransactions).get.isEmpty) assertSetTransactions(log, expectedTxns = Map()) commitSetTxn(log, "app-1", version = 1, lastUpdated = 1) // 1st commit assertSetTransactions(log, expectedTxns = Map("app-1" -> 1)) commitSetTxn(log, "app-1", version = 3, lastUpdated = 2) // 2nd commit assertSetTransactions(log, expectedTxns = Map("app-1" -> 3)) commitSetTxn(log, "app-2", version = 100, lastUpdated = 3) // 3rd commit assertSetTransactions(log, expectedTxns = Map("app-1" -> 3, "app-2" -> 100)) commitSetTxn(log, "app-1", version = 4, lastUpdated = 4) // 4th commit assertSetTransactions(log, expectedTxns = Map("app-1" -> 4, "app-2" -> 100)) // 5th commit - Commit multiple [[SetTransaction]] in single commit commitSetTxn( log, setTransactions = Seq( SetTransaction("app-1", version = 100, lastUpdated = Some(4)), SetTransaction("app-3", version = 300, lastUpdated = Some(4)) )) assertSetTransactions( log, expectedTxns = Map("app-1" -> 100, "app-2" -> 100, "app-3" -> 300)) } } } test("set-transaction tracking starts for old tables after new commits") { withSQLConf(DeltaSQLConf.DELTA_WRITE_SET_TRANSACTIONS_IN_CRC.key -> "false") { val tbl = "test_table" withTable(tbl) { // Create a table with feature disabled. So 0th/1st commit won't do SetTransaction // tracking in CRC. sql(s"CREATE TABLE $tbl USING delta as SELECT 1 as value") // 0th commit def deltaLog: DeltaLog = DeltaLog.forTable(spark, TableIdentifier(tbl)) assert(deltaLog.update().checksumOpt.get.setTransactions.isEmpty) commitSetTxn(deltaLog, "app-1", version = 1, lastUpdated = 1) // 1st commit assert(deltaLog.update().checksumOpt.get.setTransactions.isEmpty) // Enable the SetTransaction tracking config and do more commits in the table. withSQLConf(DeltaSQLConf.DELTA_WRITE_SET_TRANSACTIONS_IN_CRC.key -> "true") { DeltaLog.clearCache() commitSetTxn(deltaLog, "app-1", version = 2, lastUpdated = 2) // 2nd commit if (catalogOwnedDefaultCreationEnabledInTests) { // For CatalogOwned tables with Row Tracking enabled (QoL feature), the commit DOES // trigger state reconstruction via RowId.assignFreshRowIds -> extractHighWatermark -> // domainMetadata access. This sets _computedStateTriggered=true, causing // setTransactions to be included in the CRC immediately (not lazily as in the // non-CatalogOwned case). assert(deltaLog.update().checksumOpt.get.setTransactions.nonEmpty) // crc has set-txn assertSetTransactions(deltaLog, expectedTxns = Map("app-1" -> 2), viaCRC = true) } else { // By default, commit doesn't trigger stateReconstruction and so the // incremental CRC won't have setTransactions present until `setTransactions` API is // explicitly invoked before the commit. assert(deltaLog.update().checksumOpt.get.setTransactions.isEmpty) // crc has no set-txn assertSetTransactions(deltaLog, expectedTxns = Map("app-1" -> 2), viaCRC = false) } DeltaLog.clearCache() // Do commit after forcing computeState. Now SetTransaction tracking will start. deltaLog.snapshot.setTransactions // This triggers computeState. commitSetTxn(deltaLog, "app-2", version = 100, lastUpdated = 3) // 3rd commit assert(deltaLog.update().checksumOpt.get.setTransactions.nonEmpty) // crc has set-txn } } } } test("validate that crc doesn't contain SetTransaction when tracking is disabled") { withSQLConf(DeltaSQLConf.DELTA_WRITE_SET_TRANSACTIONS_IN_CRC.key -> "false") { val tbl = "test_table" withTable(tbl) { sql(s"CREATE TABLE $tbl (value Int) USING delta") val log = DeltaLog.forTable(spark, TableIdentifier(tbl)) // CRC for 0th commit should not have SetTransactions defined if conf is disabled. assert(log.unsafeVolatileSnapshot.checksumOpt.flatMap(_.setTransactions).isEmpty) assertSetTransactions(log, expectedTxns = Map(), viaCRC = false) commitSetTxn(log, "app-1", version = 1, lastUpdated = 1) assert(log.update().checksumOpt.flatMap(_.setTransactions).isEmpty) assertSetTransactions(log, expectedTxns = Map("app-1" -> 1), viaCRC = false) commitSetTxn(log, "app-1", version = 3, lastUpdated = 2) assert(log.update().checksumOpt.flatMap(_.setTransactions).isEmpty) assertSetTransactions(log, expectedTxns = Map("app-1" -> 3), viaCRC = false) commitSetTxn(log, "app-2", version = 100, lastUpdated = 3) assert(log.update().checksumOpt.flatMap(_.setTransactions).isEmpty) assertSetTransactions(log, expectedTxns = Map("app-1" -> 3, "app-2" -> 100), viaCRC = false) commitSetTxn(log, "app-1", version = 4, lastUpdated = 4) assert(log.update().checksumOpt.flatMap(_.setTransactions).isEmpty) assertSetTransactions(log, expectedTxns = Map("app-1" -> 4, "app-2" -> 100), viaCRC = false) } } } for(computeStatePreloaded <- BOOLEAN_DOMAIN) { test("set-transaction tracking should start if computeState is pre-loaded before" + s" commit [computeState preloaded: $computeStatePreloaded]") { // Enable INCREMENTAL COMMITS and disable verification - to make sure that we // don't trigger state reconstruction after a commit. withSQLConf( DeltaSQLConf.DELTA_WRITE_SET_TRANSACTIONS_IN_CRC.key -> "false", DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key -> "true", DeltaSQLConf.INCREMENTAL_COMMIT_VERIFY.key -> "false" ) { val tbl = "test_table" def log: DeltaLog = DeltaLog.forTable(spark, TableIdentifier(tbl)) withTable(tbl) { sql(s"CREATE TABLE $tbl (value Int) USING delta") // After 0th commit - CRC shouldn't have SetTransactions as feature is disabled. assertSetTransactions(log, expectedTxns = Map(), viaCRC = false) DeltaLog.clearCache() withSQLConf(DeltaSQLConf.DELTA_WRITE_SET_TRANSACTIONS_IN_CRC.key -> "true") { commitSetTxn(log, "app-1", version = 1, lastUpdated = 1) if (catalogOwnedDefaultCreationEnabledInTests) { // For CatalogOwned tables with Row Tracking enabled (QoL feature), the commit DOES // trigger state reconstruction via RowId.assignFreshRowIds, which sets // _computedStateTriggered=true, causing setTransactions to be included in CRC. assert(log.update().checksumOpt.flatMap(_.setTransactions).nonEmpty) } else { // During 1st commit, the feature is enabled. But still the new commit crc shouldn't // contain the [[SetTransaction]] actions as we don't have an estimate of how many // [[SetTransaction]] actions might be already part of this table till now. // So incremental computation of [[SetTransaction]] won't trigger. assert(log.update().checksumOpt.flatMap(_.setTransactions).isEmpty) } if (computeStatePreloaded) { // Calling `validateChecksum` will pre-load the computeState log.update().validateChecksum() } commitSetTxn(log, "app-1", version = 100, lastUpdated = 1) if (catalogOwnedDefaultCreationEnabledInTests) { // For CatalogOwned tables with Row Tracking enabled, state reconstruction is // triggered during every commit (via RowId.assignFreshRowIds), so setTransactions are // always included in CRC regardless of whether computeState was preloaded. assert(log.update().checksumOpt.flatMap(_.setTransactions).nonEmpty) } else { // During 2nd commit, we have following 2 cases: // 1. If `computeStatePreloaded` is set, then the Snapshot has already calculated // computeState, and so we have estimate of number of SetTransactions till this // point. So next commit will trigger incremental computation of [[SetTransaction]] // 2. If `computeStatePreloaded` is not set, then Snapshot doesn't have computeState // pre-computed. So next commit will not trigger incremental computation of // [[SetTransaction]]. assert(log.update().checksumOpt.flatMap(_.setTransactions).nonEmpty === computeStatePreloaded) } } } } } } test("set-transaction tracking in CRC should stop once threshold is crossed") { withSQLConf( DeltaSQLConf.DELTA_MAX_SET_TRANSACTIONS_IN_CRC.key -> "2", DeltaSQLConf.DELTA_WRITE_SET_TRANSACTIONS_IN_CRC.key -> "true") { val tbl = "test_table" withTable(tbl) { sql(s"CREATE TABLE $tbl (value Int) USING delta") def log: DeltaLog = DeltaLog.forTable(spark, TableIdentifier(tbl)) assertSetTransactions(log, expectedTxns = Map()) commitSetTxn(log, "app-1", version = 1, lastUpdated = 1) assert(log.update().checksumOpt.flatMap(_.setTransactions).isDefined) assertSetTransactions(log, expectedTxns = Map("app-1" -> 1)) commitSetTxn(log, "app-1", version = 3, lastUpdated = 2) assert(log.update().checksumOpt.flatMap(_.setTransactions).isDefined) assertSetTransactions(log, expectedTxns = Map("app-1" -> 3)) commitSetTxn(log, "app-2", version = 100, lastUpdated = 3) assert(log.update().checksumOpt.flatMap(_.setTransactions).isDefined) assertSetTransactions( log, expectedTxns = Map("app-1" -> 3, "app-2" -> 100)) commitSetTxn(log, "app-1", version = 4, lastUpdated = 4) assert(log.update().checksumOpt.flatMap(_.setTransactions).isDefined) assertSetTransactions( log, expectedTxns = Map("app-1" -> 4, "app-2" -> 100)) commitSetTxn(log, "app-3", version = 1000, lastUpdated = 5) assert(log.update().checksumOpt.flatMap(_.setTransactions).isEmpty) assertSetTransactions( log, expectedTxns = Map("app-1" -> 4, "app-2" -> 100, "app-3" -> 1000), viaCRC = false) } } } test("set-transaction tracking in CRC should stop once setTxn retention conf is set") { withSQLConf( DeltaSQLConf.DELTA_MAX_SET_TRANSACTIONS_IN_CRC.key -> "2", DeltaSQLConf.DELTA_WRITE_SET_TRANSACTIONS_IN_CRC.key -> "true") { val tbl = "test_table" withTable(tbl) { sql(s"CREATE TABLE $tbl (value Int) USING delta") def log: DeltaLog = DeltaLog.forTable(spark, TableIdentifier(tbl)) assert(log.update().checksumOpt.flatMap(_.setTransactions).isDefined) // Do 1 commit to table - set-transaction tracking continue to happen. commitSetTxn(log, "app-1", version = 1, lastUpdated = 1) assert(log.update().checksumOpt.flatMap(_.setTransactions).isDefined) // Set any random table property - set-transaction tracking continue to happen. sql(s"ALTER TABLE $tbl SET TBLPROPERTIES ('randomProp1' = 'value1')") assert(log.update().checksumOpt.flatMap(_.setTransactions).isDefined) // Set the `setTransactionRetentionDuration` table property - set-transaction tracking will // stop. sql(s"ALTER TABLE $tbl SET TBLPROPERTIES " + s"('delta.setTransactionRetentionDuration' = 'interval 1 days')") assert(log.update().checksumOpt.flatMap(_.setTransactions).isEmpty) commitSetTxn(log, "app-1", version = 1, lastUpdated = 1) log.update().setTransactions commitSetTxn(log, "app-1", version = 1, lastUpdated = 1) assert(log.update().checksumOpt.flatMap(_.setTransactions).isEmpty) } } } for(checksumVerificationFailureIsFatal <- BOOLEAN_DOMAIN) { // In this test we check that verification failed usage-logs are triggered when // there is an issue in incremental computation and verification is explicitly enabled. test("incremental set-transaction verification failures" + s" [checksumVerificationFailureIsFatal: $checksumVerificationFailureIsFatal]") { withSQLConf( DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key -> "true", DeltaSQLConf.DELTA_WRITE_SET_TRANSACTIONS_IN_CRC.key -> "true", // Enable verification explicitly as it is disabled by default. DeltaSQLConf.INCREMENTAL_COMMIT_VERIFY.key -> true.toString, DeltaSQLConf.DELTA_CHECKSUM_MISMATCH_IS_FATAL.key -> s"$checksumVerificationFailureIsFatal" ) { withTempDir { tempDir => // Procedure: // 1. Populate the table with 2 [[SetTransaction]]s and create a checkpoint, validate that // CRC has setTransactions present. // 2. Intentionally corrupt the checkpoint - Remove one SetTransaction from it. // 3. Clear the delta log cache so we pick up the checkpoint // 4. Start a new transaction and attempt to commit the transaction // a. Incremental SetTransaction verification should fail // b. Post-commit snapshot should have checksumOpt with no [[SetTransaction]]s // Step-1 val txn0 = SetTransaction("app-0", version = 1, lastUpdated = Some(1)) val txn1 = SetTransaction("app-1", version = 888, lastUpdated = Some(2)) def log: DeltaLog = DeltaLog.forTable(spark, tempDir) // commit-0 val actions0 = (1 to 10).map(i => createTestAddFile(encodedPath = i.toString)) :+ txn0 log.startTransaction().commitWriteAppend(actions0: _*) // commit-1 val actions1 = (11 to 20).map(i => createTestAddFile(encodedPath = i.toString)) :+ txn1 log.startTransaction().commitWriteAppend(actions1: _*) assert(log.readChecksum(version = 1).get.setTransactions.nonEmpty) log.checkpoint() // Step-2 dropOneSetTransactionFromCheckpoint(log) // Step-3 DeltaLog.clearCache() assert(!log.update().logSegment.checkpointProvider.isEmpty) // Step-4 // Create the txn with [[DELTA_CHECKSUM_MISMATCH_IS_FATAL]] as false so that pre-commit // CRC validation doesn't fail. Our goal is to capture that post-commit verification // catches any issues. var txn: OptimisticTransactionImpl = null withSQLConf(DeltaSQLConf.DELTA_CHECKSUM_MISMATCH_IS_FATAL.key -> "false") { txn = log.startTransaction() } val Seq(corruptionReport) = collectSetTransactionCorruptionReport { if (checksumVerificationFailureIsFatal) { val e = intercept[DeltaIllegalStateException] { withSQLConf(DeltaSQLConf.INCREMENTAL_COMMIT_VERIFY.key -> "true") { txn.commit(Seq(), DeltaOperations.Write(SaveMode.Append)) } } assert(e.getMessage.contains("SetTransaction mismatch")) } else { txn.commit(Seq(), DeltaOperations.Write(SaveMode.Append)) } } val eventData = JsonUtils.fromJson[Map[String, Any]](corruptionReport.blob) val expectedErrorEventData = Map( "unmatchedSetTransactionsCRC" -> Seq(txn1), "unmatchedSetTransactionsComputedState" -> Seq.empty, "version" -> 2, "minSetTransactionRetentionTimestamp" -> None, "repeatedEntriesForSameAppId" -> Seq.empty, "exactMatchFailed" -> true) val observedMismatchingFields = eventData("mismatchingFields").asInstanceOf[Seq[String]] val observedErrorMessage = eventData("error").asInstanceOf[String] val observedDetailedErrorMap = eventData("detailedErrorMap").asInstanceOf[Map[String, String]] assert(observedMismatchingFields === Seq("setTransactions")) assert(observedErrorMessage.contains("SetTransaction mismatch")) assert(observedDetailedErrorMap("setTransactions") === JsonUtils.toJson(expectedErrorEventData)) if (checksumVerificationFailureIsFatal) { // Due to failure, post-commit snapshot couldn't be updated assert(log.snapshot.version === 1) assert(log.readChecksum(version = 2).isEmpty) } else { assert(log.snapshot.version === 2) assert(log.readChecksum(version = 2).get.setTransactions.isEmpty) } } } } } /** Drops one [[SetTransaction]] operation from checkpoint - the one with max appId */ private def dropOneSetTransactionFromCheckpoint(log: DeltaLog): Unit = { import testImplicits._ val checkpointPath = log.update().checkpointProvider.topLevelFiles.head.getPath val checkpointPathStr = checkpointPath.toString // Get the checkpoint file format to read and write. Specify format to be compatible // with v2 checkpoint b/c v2 checkpoint format could be either json/parquet. // Test suites that extend w/ CC will enable v2 checkpoint by default. val checkpointFormat = checkpointPathStr.substring(checkpointPathStr.lastIndexOf('.') + 1) withTempDir { tmpCheckpoint => // count total rows in checkpoint val checkpointDf = spark.read .schema(SingleAction.encoder.schema) .format(checkpointFormat).load(checkpointPathStr) val initialActionCount = checkpointDf.count().toInt val corruptedCheckpointData = checkpointDf .orderBy(col("txn.appId").asc_nulls_first) // force non setTransaction actions to front .as[SingleAction].take(initialActionCount - 1) // Drop 1 action corruptedCheckpointData.toSeq.toDS().coalesce(1).write .mode("overwrite").format(checkpointFormat).save(tmpCheckpoint.toString) assert( spark.read.format(checkpointFormat).load(tmpCheckpoint.toString).count() === initialActionCount - 1) val writtenCheckpoint = tmpCheckpoint.listFiles().toSeq.filter(_.getName.startsWith("part")).head val checkpointFile = new File(checkpointPath.toUri) new File(log.logPath.toUri).listFiles().toSeq.foreach { file => if (file.getName.startsWith(".0")) { // we need to delete checksum files, otherwise trying to replace our incomplete // checkpoint file fails due to the LocalFileSystem's checksum checks. assert(file.delete(), "Failed to delete checksum file") } } assert(checkpointFile.delete(), "Failed to delete old checkpoint") assert(writtenCheckpoint.renameTo(checkpointFile), "Failed to rename corrupt checkpoint") val newCheckpoint = spark.read.format(checkpointFormat).load(checkpointFile.toString) assert(newCheckpoint.count() === initialActionCount - 1, "Checkpoint file incorrect:\n" + newCheckpoint.collect().mkString("\n")) } } private def collectSetTransactionCorruptionReport(f: => Unit): Seq[UsageRecord] = { collectUsageLogs("delta.checksum.invalid")(f).toSeq } } class DeltaIncrementalSetTransactionsWithCatalogOwnedBatch1Suite extends DeltaIncrementalSetTransactionsSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class DeltaIncrementalSetTransactionsWithCatalogOwnedBatch2Suite extends DeltaIncrementalSetTransactionsSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class DeltaIncrementalSetTransactionsWithCatalogOwnedBatch100Suite extends DeltaIncrementalSetTransactionsSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoColumnOrderSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.SaveMode import org.apache.spark.sql.internal.SQLConf /** * Test suite covering INSERT operations with columns or struct fields ordered differently than in * the table schema. */ class DeltaInsertIntoColumnOrderSuite extends DeltaInsertIntoTest { override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaSQLConf.DELTA_STREAMING_SINK_ALLOW_IMPLICIT_CASTS.key, "true") spark.conf.set(SQLConf.ANSI_ENABLED.key, "true") } /** Collects inserts that don't support implicit casting and will fail if the input data type * doesn't match the expected column type. * These are all dataframe inserts that use by name resolution, except for streaming writes. */ private val insertsWithoutImplicitCastSupport: Set[Insert] = insertsByName.intersect(insertsDataframe) - StreamingInsert test("all test cases are implemented") { checkAllTestCasesImplemented() } // Inserting using a different ordering for top-level columns behaves as one would expect: // inserts by position resolve columns based on position, inserts by name resolve based on name. // Whether additional handling is required to add implicit casts doesn't impact this behavior. for { (inserts, expectedAnswer) <- Seq( insertsByPosition.intersect(insertsAppend) -> TestData("a int, b int, c int", Seq("""{ "a": 1, "b": 2, "c": 3 }""", """{ "a": 1, "b": 4, "c": 5 }""")), insertsByPosition.intersect(insertsOverwrite) -> TestData("a int, b int, c int", Seq("""{ "a": 1, "b": 4, "c": 5 }""")), insertsByName.intersect(insertsAppend) -> TestData("a int, b int, c int", Seq("""{ "a": 1, "b": 2, "c": 3 }""", """{ "a": 1, "b": 5, "c": 4 }""")), insertsByName.intersect(insertsOverwrite) -> TestData("a int, b int, c int", Seq("""{ "a": 1, "b": 5, "c": 4 }""")) ) } { testInserts(s"insert with different top-level column ordering")( initialData = TestData("a int, b int, c int", Seq("""{ "a": 1, "b": 2, "c": 3 }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, c int, b int", Seq("""{ "a": 1, "c": 4, "b": 5 }""")), expectedResult = ExpectedResult.Success(expectedAnswer), includeInserts = inserts ) testInserts(s"insert with implicit cast and different top-level column ordering")( initialData = TestData("a int, b int, c int", Seq("""{ "a": 1, "b": 2, "c": 3 }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a long, c int, b int", Seq("""{ "a": 1, "c": 4, "b": 5 }""")), expectedResult = ExpectedResult.Success(expectedAnswer), // Inserts that don't support implicit cast are failing, these are covered in the test below. includeInserts = inserts -- insertsWithoutImplicitCastSupport ) } testInserts(s"insert with implicit cast and different top-level column ordering")( initialData = TestData("a int, b int, c int", Seq("""{ "a": 1, "b": 2, "c": 3 }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a long, c int, b int", Seq("""{ "a": 1, "c": 4, "b": 4 }""")), expectedResult = ExpectedResult.Failure(ex => { checkError( ex, "DELTA_FAILED_TO_MERGE_FIELDS", parameters = Map( "currentField" -> "a", "updateField" -> "a" ))}), includeInserts = insertsWithoutImplicitCastSupport ) // Inserting using a different ordering for struct fields is full of surprises... for { (inserts: Set[Insert], expectedAnswer) <- Seq( // Most inserts use name based resolution for struct fields when there's no implicit cast // required due to mismatching data types, except for `INSERT INTO/OVERWRITE (columns)` and // `INSERT OVERWRITE PARTITION (partition) (columns)` which use position based resolution - even // though these are by name inserts. insertsAppend - SQLInsertColList(SaveMode.Append) -> TestData("a int, s struct ", Seq("""{ "a": 1, "s": { "x": 2, "y": 3 } }""", """{ "a": 1, "s": { "x": 4, "y": 5 } }""")), insertsOverwrite - SQLInsertColList(SaveMode.Overwrite) - SQLInsertOverwritePartitionColList -> TestData("a int, s struct ", Seq("""{ "a": 1, "s": { "x": 4, "y": 5 } }""")), Set(SQLInsertColList(SaveMode.Append)) -> TestData("a int, s struct ", Seq("""{ "a": 1, "s": { "x": 2, "y": 3 } }""", """{ "a": 1, "s": { "x": 5, "y": 4 } }""")), Set(SQLInsertColList(SaveMode.Overwrite), SQLInsertOverwritePartitionColList) -> TestData("a int, s struct ", Seq("""{ "a": 1, "s": { "x": 5, "y": 4 } }""")) ) } { testInserts(s"insert with different struct fields ordering")( initialData = TestData( "a int, s struct ", Seq("""{ "a": 1, "s": { "x": 2, "y": 3 } }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, s struct ", Seq("""{ "a": 1, "s": { "y": 5, "x": 4 } }""")), expectedResult = ExpectedResult.Success(expectedAnswer), includeInserts = inserts ) } for { (inserts: Set[Insert], expectedAnswer) <- Seq( // When there's a type mismatch and an implicit cast is required, then all inserts use position // based resolution for struct fields, except for `INSERT OVERWRITE PARTITION (partition)` and // streaming insert which use name based resolution, and dataframe inserts by name which don't // support implicit cast and fail - see negative test below. insertsAppend - StreamingInsert -> TestData("a int, s struct ", Seq("""{ "a": 1, "s": { "x": 2, "y": 3 } }""", """{ "a": 1, "s": { "x": 5, "y": 4 } }""")), insertsOverwrite - SQLInsertOverwritePartitionByPosition -> TestData("a int, s struct ", Seq("""{ "a": 1, "s": { "x": 5, "y": 4 } }""")), Set(StreamingInsert) -> TestData("a int, s struct ", Seq("""{ "a": 1, "s": { "x": 2, "y": 3 } }""", """{ "a": 1, "s": { "x": 4, "y": 5 } }""")), Set(SQLInsertOverwritePartitionByPosition) -> TestData("a int, s struct ", Seq("""{ "a": 1, "s": { "x": 4, "y": 5 } }""")) ) } { testInserts(s"insert with implicit cast and different struct fields ordering")( initialData = TestData( "a int, s struct ", Seq("""{ "a": 1, "s": { "x": 2, "y": 3 } }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a long, s struct ", Seq("""{ "a": 1, "s": { "y": 5, "x": 4 } }""")), expectedResult = ExpectedResult.Success(expectedAnswer), // Inserts that don't support implicit cast are failing, these are covered in the test below. includeInserts = inserts -- insertsWithoutImplicitCastSupport ) } testInserts(s"insert with implicit cast and different struct fields ordering")( initialData = TestData( "a int, s struct ", Seq("""{ "a": 1, "s": { "x": 2, "y": 3 } }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a long, s struct ", Seq("""{ "a": 1, "s": { "y": 5, "x": 4 } }""")), expectedResult = ExpectedResult.Failure(ex => { checkError( ex, "DELTA_FAILED_TO_MERGE_FIELDS", parameters = Map( "currentField" -> "a", "updateField" -> "a" ))}), includeInserts = insertsWithoutImplicitCastSupport ) for { preserveNullSourceStructs <- BOOLEAN_DOMAIN (inserts: Set[Insert], expectedAnswer) <- Seq( insertsAppend -> TestData("a int, s struct ", Seq("""{ "a": 1, "s": { "x": 2, "y": 3 } }""", """{ "a": 1, "s": null }""")), insertsOverwrite -> TestData("a int, s struct ", Seq("""{ "a": 1, "s": null }""")) ) } { testInserts(s"null struct with different field order, " + s"preserveNullSourceStructs=$preserveNullSourceStructs")( initialData = TestData( "a int, s struct ", Seq("""{ "a": 1, "s": { "x": 2, "y": 3 } }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, s struct ", Seq("""{ "a": 1, "s": null }""")), expectedResult = ExpectedResult.Success(expectedAnswer), includeInserts = inserts, confs = Seq( // Implicit casts in streaming writes would cause the `null` struct to be incorrectly // expanded. This conf allows skipping adding casts since there are no actual data type // mismatch. DeltaSQLConf.DELTA_STREAMING_SINK_IMPLICIT_CAST_FOR_TYPE_MISMATCH_ONLY.key -> "true", DeltaSQLConf.DELTA_INSERT_PRESERVE_NULL_SOURCE_STRUCTS.key -> preserveNullSourceStructs.toString ) ) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoImplicitCastSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.SaveMode import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ /** * Test suite covering implicit casting in INSERT operations when the type of the data to insert * doesn't match the type in Delta table. * * The casting behavior is (unfortunately) dependent on the API used to run the INSERT, e.g. * Dataframe V1 insertInto() vs V2 saveAsTable() or using SQL. * This suite intends to exhaustively cover all the ways INSERT can be run on a Delta table. See * [[DeltaInsertIntoTest]] for a list of these INSERT operations covered. */ trait DeltaInsertIntoImplicitCastBase extends DeltaInsertIntoTest { override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaSQLConf.DELTA_STREAMING_SINK_ALLOW_IMPLICIT_CASTS.key, "true") // Enable the null expansion fix by preserving NULL source structs in INSERT operations. // Without this fix, NULL source structs are incorrectly expanded to structs with NULL fields. spark.conf.set(DeltaSQLConf.DELTA_MERGE_PRESERVE_NULL_SOURCE_STRUCTS.key, "true") spark.conf.set(SQLConf.ANSI_ENABLED.key, "true") } protected val ignoredTestCases: Map[String, Set[Insert]] = Map.empty test("all test cases are implemented") { checkAllTestCasesImplemented(ignoredTestCases) } } trait DeltaInsertIntoImplicitCastTests extends DeltaInsertIntoImplicitCastBase { for (schemaEvolution <- BOOLEAN_DOMAIN) { testInserts("insert with implicit up and down cast on top-level fields, " + s"schemaEvolution=$schemaEvolution")( initialData = TestData("a long, b int", Seq("""{ "a": 1, "b": 2 }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, b long", Seq("""{ "a": 1, "b": 4 }""")), expectedResult = ExpectedResult.Success( expected = new StructType() .add("a", LongType) .add("b", IntegerType)), // The following insert operations don't implicitly cast the data but fail instead - see // following test covering failure for these cases. We should change this to offer consistent // behavior across all inserts. excludeInserts = insertsDataframe.intersect(insertsByName) - StreamingInsert, confs = Seq(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> schemaEvolution.toString) ) testInserts("insert with implicit up and down cast on top-level fields, " + s"schemaEvolution=$schemaEvolution")( initialData = TestData("a long, b int", Seq("""{ "a": 1, "b": 2 }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, b long", Seq("""{ "a": 1, "b": 4 }""")), expectedResult = ExpectedResult.Failure { ex => checkError( ex, "DELTA_FAILED_TO_MERGE_FIELDS", parameters = Map( "currentField" -> "a", "updateField" -> "a" )) }, includeInserts = insertsDataframe.intersect(insertsByName) - StreamingInsert, confs = Seq(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> schemaEvolution.toString) ) testInserts("insert with implicit up and down cast on fields nested in array, " + s"schemaEvolution=$schemaEvolution")( initialData = TestData("key int, a array>", Seq("""{ "key": 1, "a": [ { "x": 1, "y": 2 } ] }""")), partitionBy = Seq("key"), overwriteWhere = "key" -> 1, insertData = TestData("key int, a array>", Seq("""{ "key": 1, "a": [ { "x": 3, "y": 4 } ] }""")), expectedResult = ExpectedResult.Success( expected = new StructType() .add("key", IntegerType) .add("a", ArrayType(new StructType() .add("x", LongType) .add("y", IntegerType, nullable = true)))), // The following insert operations don't implicitly cast the data but fail instead - see // following test covering failure for these cases. We should change this to offer consistent // behavior across all inserts. excludeInserts = insertsDataframe.intersect(insertsByName) - StreamingInsert, confs = Seq(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> schemaEvolution.toString) ) testInserts("insert with implicit up and down cast on fields nested in array, " + s"schemaEvolution=$schemaEvolution")( initialData = TestData("key int, a array>", Seq("""{ "key": 1, "a": [ { "x": 1, "y": 2 } ] }""")), partitionBy = Seq("key"), overwriteWhere = "key" -> 1, insertData = TestData("key int, a array>", Seq("""{ "key": 1, "a": [ { "x": 3, "y": 4 } ] }""")), expectedResult = ExpectedResult.Failure { ex => checkError( ex, "DELTA_FAILED_TO_MERGE_FIELDS", parameters = Map( "currentField" -> "a", "updateField" -> "a" )) }, includeInserts = insertsDataframe.intersect(insertsByName) - StreamingInsert, confs = Seq(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> schemaEvolution.toString) ) testInserts("insert with implicit up and down cast on fields nested in map, " + s"schemaEvolution=$schemaEvolution")( initialData = TestData("key int, m map>", Seq("""{ "key": 1, "m": { "a": { "x": 1, "y": 2 } } }""")), partitionBy = Seq("key"), overwriteWhere = "key" -> 1, insertData = TestData("key int, m map>", Seq("""{ "key": 1, "m": { "a": { "x": 3, "y": 4 } } }""")), expectedResult = ExpectedResult.Success( expected = new StructType() .add("key", IntegerType) .add("m", MapType(StringType, new StructType() .add("x", LongType) .add("y", IntegerType)))), // The following insert operations don't implicitly cast the data but fail instead - see // following test covering failure for these cases. We should change this to offer consistent // behavior across all inserts. excludeInserts = insertsDataframe.intersect(insertsByName) - StreamingInsert, confs = Seq(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> schemaEvolution.toString) ) testInserts("insert with implicit up and down cast on fields nested in map, " + s"schemaEvolution=$schemaEvolution")( initialData = TestData("key int, m map>", Seq("""{ "key": 1, "m": { "a": { "x": 1, "y": 2 } } }""")), partitionBy = Seq("key"), overwriteWhere = "key" -> 1, insertData = TestData("key int, m map>", Seq("""{ "key": 1, "m": { "a": { "x": 3, "y": 4 } } }""")), expectedResult = ExpectedResult.Failure { ex => checkError( ex, "DELTA_FAILED_TO_MERGE_FIELDS", parameters = Map( "currentField" -> "m", "updateField" -> "m" )) }, includeInserts = insertsDataframe.intersect(insertsByName) - StreamingInsert, confs = Seq(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> schemaEvolution.toString) ) } } trait DeltaInsertIntoImplicitCastStreamingWriteTests extends DeltaInsertIntoImplicitCastBase { override protected val ignoredTestCases: Map[String, Set[Insert]] = Map( "null struct with different field order, preserveNullSourceStructs=true" -> (insertsDataframe.intersect(insertsByName) - StreamingInsert), "null struct with different field order, preserveNullSourceStructs=false" -> (insertsDataframe.intersect(insertsByName) - StreamingInsert), "cast with dot in column name" -> (insertsDataframe.intersect(insertsByName) - StreamingInsert) ) for { preserveNullSourceStructs <- BOOLEAN_DOMAIN (inserts: Set[Insert], expectedAnswer) <- Seq( Set(SQLInsertColList(SaveMode.Append), StreamingInsert) -> TestData("a long, s struct ", Seq("""{ "a": 1, "s": { "x": 2, "y": 3 } }""", """{ "a": 1, "s": null }""")), Set(SQLInsertColList(SaveMode.Overwrite), SQLInsertOverwritePartitionByPosition, SQLInsertOverwritePartitionColList) -> TestData("a long, s struct ", Seq("""{ "a": 1, "s": null }""")), // For all other INSERT types, the null struct gets incorrectly expanded to // `struct unless preserveNullSourceStructs is true. insertsAppend - SQLInsertColList(SaveMode.Append) - StreamingInsert -> TestData("a long, s struct ", Seq("""{ "a": 1, "s": { "x": 2, "y": 3 } }""", if (preserveNullSourceStructs) { """{ "a": 1, "s": null }""" } else { """{ "a": 1, "s": { "x": null, "y": null } }""" } ) ), insertsOverwrite - SQLInsertColList(SaveMode.Overwrite) - SQLInsertOverwritePartitionByPosition - SQLInsertOverwritePartitionColList -> TestData("a long, s struct ", Seq( if (preserveNullSourceStructs) { """{ "a": 1, "s": null }""" } else { """{ "a": 1, "s": { "x": null, "y": null } }""" } ) ) ) } { testInserts("null struct with different field order, " + s"preserveNullSourceStructs=$preserveNullSourceStructs")( initialData = TestData( "a long, s struct ", Seq("""{ "a": 1, "s": { "x": 2, "y": 3 } }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, s struct ", Seq("""{ "a": 1, "s": null }""")), expectedResult = ExpectedResult.Success(expectedAnswer), includeInserts = inserts, // Dataframe INSERTs by name don't support implicit casting except for streaming // writes, no point in testing them. excludeInserts = insertsDataframe.intersect(insertsByName) - StreamingInsert, confs = Seq(DeltaSQLConf.DELTA_INSERT_PRESERVE_NULL_SOURCE_STRUCTS.key -> preserveNullSourceStructs.toString) ) } for { (inserts: Set[Insert], expectedAnswer) <- Seq( insertsAppend -> TestData("`s.a` long, s struct ", Seq("""{ "s.a": 1, "s": { "x": 2, "y": 3 } }""", """{ "s.a": 1, "s": { "x": 4, "y": 5 } }""")), insertsOverwrite -> TestData("`s.a` long, s struct ", Seq("""{ "s.a": 1, "s": { "x": 4, "y": 5 } }""")) ) } { testInserts(s"cast with dot in column name")( initialData = TestData( "`s.a` long, s struct ", Seq("""{ "s.a": 1, "s": { "x": 2, "y": 3 } }""")), partitionBy = Seq("`s.a`"), overwriteWhere = "`s.a`" -> 1, insertData = TestData("`s.a` int, s struct ", Seq("""{ "s.a": 1, "s": { "x": 4, "y": 5 } }""")), expectedResult = ExpectedResult.Success(expectedAnswer), includeInserts = inserts, // Dataframe INSERTs by name don't support implicit casting except for streaming // writes, no point in testing them. excludeInserts = insertsDataframe.intersect(insertsByName) - StreamingInsert, confs = Seq(DeltaSQLConf.DELTA_STREAMING_SINK_IMPLICIT_CAST_ESCAPE_COLUMN_NAMES.key -> "true") ) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoMissingColumnSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ /** * Test suite covering behavior of INSERT operations with missing top-level columns or nested struct * fields. * This suite intends to exhaustively cover all the ways INSERT can be run on a Delta table. See * [[DeltaInsertIntoTest]] for a list of these INSERT operations covered. */ class DeltaInsertIntoMissingColumnSuite extends DeltaInsertIntoTest { override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaSQLConf.DELTA_STREAMING_SINK_ALLOW_IMPLICIT_CASTS.key, "true") spark.conf.set(SQLConf.ANSI_ENABLED.key, "true") } test("all test cases are implemented") { checkAllTestCasesImplemented() } for (schemaEvolution <- BOOLEAN_DOMAIN) { // Missing top-level columns are allowed for all inserts by name (SQL+dataframe) but missing // nested fields are only allowed for dataframe inserts by name. testInserts(s"insert with missing top-level column, schemaEvolution=$schemaEvolution")( initialData = TestData("a int, b int, c int", Seq("""{ "a": 1, "b": 2, "c": 3 }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, b int", Seq("""{ "a": 1, "b": 4 }""")), expectedResult = ExpectedResult.Success( expected = new StructType() .add("a", IntegerType) .add("b", IntegerType) .add("c", IntegerType)), includeInserts = insertsByName, withSchemaEvolution = schemaEvolution ) testInserts(s"insert with missing nested field, schemaEvolution=$schemaEvolution")( initialData = TestData("a int, s struct", Seq("""{ "a": 1, "s": { "x": 2, "y": 3 } }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, s struct", Seq("""{ "a": 1, "s": { "y": 5 } }""")), expectedResult = ExpectedResult.Success( expected = new StructType() .add("a", IntegerType) .add("s", new StructType() .add("x", IntegerType) .add("y", IntegerType) )), includeInserts = insertsByName.intersect(insertsDataframe), withSchemaEvolution = schemaEvolution ) // Missing columns for all inserts by name and missing nested fields for dataframe inserts by // name are also allowed when the insert includes type mismatches, with the difference that // dataframe inserts by name don't support implicit casting and will fail due to the type // mismatch (but not the missing column/field per se). testInserts(s"insert with implicit cast and missing top-level column," + s"schemaEvolution=$schemaEvolution")( initialData = TestData("a long, b int, c int", Seq("""{ "a": 1, "b": 2, "c": 3 }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, b long", Seq("""{ "a": 1, "b": 4 }""")), expectedResult = ExpectedResult.Success( expected = new StructType() .add("a", LongType) .add("b", IntegerType) .add("c", IntegerType)), includeInserts = insertsByName.intersect(insertsSQL) + StreamingInsert, withSchemaEvolution = schemaEvolution ) testInserts(s"insert with implicit cast and missing top-level column," + s"schemaEvolution=$schemaEvolution")( initialData = TestData("a long, b int, c int", Seq("""{ "a": 1, "b": 2, "c": 3 }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, b long", Seq("""{ "a": 1, "b": 4 }""")), expectedResult = ExpectedResult.Failure(ex => { // The missing column isn't an issue, but dataframe inserts by name (except streaming) don't // support implicit casting to reconcile the type mismatch. checkError( ex, "DELTA_FAILED_TO_MERGE_FIELDS", parameters = Map( "currentField" -> "a", "updateField" -> "a" )) }), includeInserts = insertsByName.intersect(insertsDataframe) - StreamingInsert, withSchemaEvolution = schemaEvolution ) testInserts(s"insert with implicit cast and missing nested field," + s"schemaEvolution=$schemaEvolution")( initialData = TestData("a int, s struct", Seq("""{ "a": 1, "s": { "x": 2, "y": 3 } }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, s struct", Seq("""{ "a": 1, "s": { "y": 5 } }""")), expectedResult = ExpectedResult.Failure(ex => { // The missing column isn't an issue, but dataframe inserts by name (except streaming) don't // support implicit casting to reconcile the type mismatch. checkError( ex, "DELTA_FAILED_TO_MERGE_FIELDS", parameters = Map( "currentField" -> "s", "updateField" -> "s" )) }), includeInserts = insertsByName.intersect(insertsDataframe) - StreamingInsert, withSchemaEvolution = schemaEvolution ) testInserts(s"insert with implicit cast and missing nested field," + s"schemaEvolution=$schemaEvolution")( initialData = TestData("a int, s struct", Seq("""{ "a": 1, "s": { "x": 2, "y": 3 } }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, s struct", Seq("""{ "a": 1, "s": { "y": 5 } }""")), // Missing nested fields are allowed when writing to a delta streaming sink when there's a // type mismatch, same as when there's no type mismatch. expectedResult = ExpectedResult.Success( expected = new StructType() .add("a", IntegerType) .add("s", new StructType() .add("x", IntegerType) .add("y", IntegerType))), includeInserts = Set(StreamingInsert), withSchemaEvolution = schemaEvolution ) // Missing columns for all inserts by position and missing nested fields for all inserts by // position or SQL inserts are rejected. Whether the insert also includes a type mismatch // doesn't play a role. testInserts(s"insert with missing top-level column, schemaEvolution=$schemaEvolution")( initialData = TestData("a int, b int, c int", Seq("""{ "a": 1, "b": 2, "c": 3 }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, b int", Seq("""{ "a": 1, "b": 4 }""")), expectedResult = ExpectedResult.Failure(ex => { checkError( ex, "DELTA_INSERT_COLUMN_ARITY_MISMATCH", parameters = Map( "tableName" -> s"$catalogName.default.target", "columnName" -> "not enough data columns", "numColumns" -> "3", "insertColumns" -> "2" )) }), includeInserts = insertsByPosition, withSchemaEvolution = schemaEvolution ) testInserts(s"insert with implicit cast and missing top-level column," + s"schemaEvolution=$schemaEvolution")( initialData = TestData("a long, b int, c int", Seq("""{ "a": 1, "b": 2, "c": 3 }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, b long", Seq("""{ "a": 1, "b": 4 }""")), expectedResult = ExpectedResult.Failure(ex => { checkError( ex, "DELTA_INSERT_COLUMN_ARITY_MISMATCH", parameters = Map( "tableName" -> s"$catalogName.default.target", "columnName" -> "not enough data columns", "numColumns" -> "3", "insertColumns" -> "2" )) }), includeInserts = insertsByPosition, withSchemaEvolution = schemaEvolution ) testInserts(s"insert with missing nested field, schemaEvolution=$schemaEvolution")( initialData = TestData("a int, s struct", Seq("""{ "a": 1, "s": { "x": 2, "y": 3 } }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, s struct", Seq("""{ "a": 1, "s": { "y": 5 } }""")), expectedResult = ExpectedResult.Failure(ex => { checkErrorMatchPVals( ex, "DELTA_INSERT_COLUMN_ARITY_MISMATCH", parameters = Map( "tableName" -> s"$catalogName\\.default\\.target", "columnName" -> s"not enough nested fields in ($catalogName\\.default\\.source\\.)?s", "numColumns" -> "2", "insertColumns" -> "1" )) }), includeInserts = insertsByPosition ++ insertsSQL, withSchemaEvolution = schemaEvolution ) testInserts(s"insert with implicit cast and missing nested field," + s"schemaEvolution=$schemaEvolution")( initialData = TestData("a int, s struct", Seq("""{ "a": 1, "s": { "x": 2, "y": 3 } }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, s struct", Seq("""{ "a": 1, "s": { "y": 5 } }""")), expectedResult = ExpectedResult.Failure(ex => { checkErrorMatchPVals( ex, "DELTA_INSERT_COLUMN_ARITY_MISMATCH", parameters = Map( "tableName" -> s"$catalogName\\.default\\.target", "columnName" -> s"not enough nested fields in ($catalogName\\.default\\.source\\.)?s", "numColumns" -> "2", "insertColumns" -> "1" )) }), includeInserts = insertsByPosition ++ insertsSQL, withSchemaEvolution = schemaEvolution ) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoSchemaEvolutionSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.SparkThrowable import org.apache.spark.sql.SaveMode import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ /** * Test suite covering behavior of INSERT operations with extra top-level columns or nested struct * fields in the input data. * This suite intends to exhaustively cover all the ways INSERT can be run on a Delta table. See * [[DeltaInsertIntoTest]] for a list of these INSERT operations covered. */ class DeltaInsertIntoSchemaEvolutionSuite extends DeltaInsertIntoTest { override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaSQLConf.DELTA_STREAMING_SINK_ALLOW_IMPLICIT_CASTS.key, "true") spark.conf.set(SQLConf.ANSI_ENABLED.key, "true") } test("all test cases are implemented") { // we don't cover SQL INSERT with an explicit column list in this suite as it's not possible to // specify a column that doesn't exist in the target table that way. val ignoredTestCases = testCases.map { case (name, _) => name -> Set( SQLInsertColList(SaveMode.Append), SQLInsertColList(SaveMode.Overwrite), SQLInsertOverwritePartitionColList) }.toMap checkAllTestCasesImplemented(ignoredTestCases) } for (enableAutoMergeSQLConf <- BOOLEAN_DOMAIN) { val testMsg = s"enableAutoMergeSQLConf=$enableAutoMergeSQLConf" testInserts("WITH SCHEMA EVOLUTION or .option always take precedence over the SQL Conf, " + testMsg)( initialData = TestData("a int, s struct ", Seq("""{ "a": 1, "s": { "x": 2 } }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, s struct ", Seq("""{ "a": 1, "s": { "x": 4, "y": 5 } }""")), expectedResult = ExpectedResult.Success( expected = new StructType() .add("a", IntegerType) .add("s", new StructType() .add("x", IntegerType) .add("y", IntegerType) )), confs = Seq( DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> enableAutoMergeSQLConf.toString), withSchemaEvolution = true ) } for (schemaEvolution <- BOOLEAN_DOMAIN) { // We allow adding new top-level columns with schema evolution for all inserts. testInserts(s"insert with extra top-level column, schemaEvolution=$schemaEvolution")( initialData = TestData("a int, b int", Seq("""{ "a": 1, "b": 2 }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, b int, c int", Seq("""{ "a": 1, "b": 4, "c": 5 }""")), expectedResult = if (schemaEvolution) { ExpectedResult.Success( expected = new StructType() .add("a", IntegerType) .add("b", IntegerType) .add("c", IntegerType)) } else { ExpectedResult.Failure(ex => { checkError(ex, "DELTA_METADATA_MISMATCH", "42KDG", Map.empty[String, String]) }) }, excludeInserts = Set( SQLInsertColList(SaveMode.Append), SQLInsertColList(SaveMode.Overwrite), SQLInsertOverwritePartitionColList ), withSchemaEvolution = schemaEvolution ) // Adding new top-level columns with schema evolution is allowed for all inserts, but dataframe // inserts by name don't support implicit casting and will fail due to the type mismatch. testInserts(s"insert with extra top-level column and implicit cast," + s"schemaEvolution=$schemaEvolution")( initialData = TestData("a int, b int", Seq("""{ "a": 1, "b": 2 }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, b long, c int", Seq("""{ "a": 1, "b": 4, "c": 5 }""")), expectedResult = if (schemaEvolution) { ExpectedResult.Success( expected = new StructType() .add("a", IntegerType) .add("b", IntegerType) .add("c", IntegerType)) } else { ExpectedResult.Failure(ex => { checkError(ex, "DELTA_METADATA_MISMATCH", "42KDG", Map.empty[String, String]) }) }, includeInserts = insertsByPosition + StreamingInsert, withSchemaEvolution = schemaEvolution ) testInserts(s"insert with extra top-level column and implicit cast," + s"schemaEvolution=$schemaEvolution")( initialData = TestData("a int, b int", Seq("""{ "a": 1, "b": 2 }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, b long, c int", Seq("""{ "a": 1, "b": 4, "c": 5 }""")), expectedResult = ExpectedResult.Failure(ex => { checkError( ex, "DELTA_FAILED_TO_MERGE_FIELDS", parameters = Map( "currentField" -> "b", "updateField" -> "b" )) }), includeInserts = insertsDataframe.intersect(insertsByName) - StreamingInsert, withSchemaEvolution = schemaEvolution ) // SQL inserts by name fail with a different error in the analysis when there's an extra column // and schema evolution is disabled. testInserts(s"insert with extra top-level column and implicit cast," + s"schemaEvolution=$schemaEvolution")( initialData = TestData("a int, b int", Seq("""{ "a": 1, "b": 2 }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, b long, c int", Seq("""{ "a": 1, "b": 4, "c": 5 }""")), expectedResult = if (schemaEvolution) { ExpectedResult.Success( expected = new StructType() .add("a", IntegerType) .add("b", IntegerType) .add("c", IntegerType)) } else { ExpectedResult.Failure(ex => { checkError( ex, "INSERT_COLUMN_ARITY_MISMATCH.TOO_MANY_DATA_COLUMNS", parameters = Map( "tableName" -> s"`$catalogName`.`default`.`target`", "tableColumns" -> "`a`, `b`", "dataColumns" -> "`a`, `b`, `c`" )) }) }, includeInserts = insertsSQL.intersect(insertsByName) -- Set( // It's not possible to specify a column that doesn't exist in the target using SQL with an // explicit column list. SQLInsertColList(SaveMode.Append), SQLInsertColList(SaveMode.Overwrite), SQLInsertOverwritePartitionColList ), withSchemaEvolution = schemaEvolution ) // We allow adding new nested struct fields for all inserts, including SQL inserts by name. testInserts(s"insert with extra nested field, schemaEvolution=$schemaEvolution")( initialData = TestData("a int, s struct ", Seq("""{ "a": 1, "s": { "x": 2 } }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, s struct ", Seq("""{ "a": 1, "s": { "x": 4, "y": 5 } }""")), expectedResult = if (schemaEvolution) { ExpectedResult.Success( expected = new StructType() .add("a", IntegerType) .add("s", new StructType() .add("x", IntegerType) .add("y", IntegerType) )) } else { ExpectedResult.Failure(ex => { checkError(ex, "DELTA_METADATA_MISMATCH", "42KDG", Map.empty[String, String]) }) }, withSchemaEvolution = schemaEvolution ) } for { preserveNullSourceStructs <- BOOLEAN_DOMAIN (inserts: Set[Insert], expectedAnswer) <- Seq( insertsAppend -> TestData("a int, s struct ", Seq("""{ "a": 1, "s": { "x": 2, "y": null } }""", """{ "a": 1, "s": null }""")), insertsOverwrite -> TestData("a int, s struct ", Seq("""{ "a": 1, "s": null }""")) ) } { testInserts(s"insert with extra nested field, null struct, " + s"preserveNullSourceStructs=$preserveNullSourceStructs")( initialData = TestData("a int, s struct ", Seq("""{ "a": 1, "s": { "x": 2 } }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, s struct ", Seq("""{ "a": 1, "s": null }""")), expectedResult = ExpectedResult.Success(expected = expectedAnswer), includeInserts = inserts, confs = Seq( DeltaSQLConf.DELTA_INSERT_PRESERVE_NULL_SOURCE_STRUCTS.key -> preserveNullSourceStructs.toString ), withSchemaEvolution = true ) } for (schemaEvolution <- BOOLEAN_DOMAIN) { // Adding new nested struct fields with schema evolution is allowed for all inserts, but // dataframe inserts by name don't support implicit casting and will fail due to the type // mismatch. testInserts(s"insert with extra nested field and implicit cast," + s"schemaEvolution=$schemaEvolution")( initialData = TestData("a int, s struct ", Seq("""{ "a": 1, "s": { "x": 2 } }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, s struct ", Seq("""{ "a": 1, "s": { "x": 4, "y": 5 } }""")), expectedResult = if (schemaEvolution) { ExpectedResult.Success( expected = new StructType() .add("a", IntegerType) .add("s", new StructType() .add("x", IntegerType) .add("y", IntegerType) )) } else { ExpectedResult.Failure(ex => { checkError(ex, "DELTA_METADATA_MISMATCH", "42KDG", Map.empty[String, String]) }) }, includeInserts = insertsSQL ++ insertsByPosition + StreamingInsert -- Seq( // It's not possible to specify a column that doesn't exist in the target using SQL with an // explicit column list. SQLInsertColList(SaveMode.Append), SQLInsertColList(SaveMode.Overwrite), SQLInsertOverwritePartitionColList ), withSchemaEvolution = schemaEvolution ) testInserts(s"insert with extra nested field and implicit cast," + s"schemaEvolution=$schemaEvolution")( initialData = TestData("a int, s struct ", Seq("""{ "a": 1, "s": { "x": 2 } }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, s struct ", Seq("""{ "a": 1, "s": { "x": 4, "y": 5 } }""")), expectedResult = ExpectedResult.Failure(ex => { checkError( ex, "DELTA_FAILED_TO_MERGE_FIELDS", parameters = Map( "currentField" -> "s", "updateField" -> "s" )) }), includeInserts = insertsDataframe.intersect(insertsByName) - StreamingInsert, withSchemaEvolution = schemaEvolution ) } // When DELTA_INSERT_BY_NAME_SCHEMA_EVOLUTION_ENABLED is disabled, SQL INSERT BY NAME with extra // top-level columns should fail even when schema evolution is enabled. test("insert by name with extra top-level column and implicit cast fails " + "when byNameSchemaEvolution is disabled") { withTable("target") { sql("CREATE TABLE target (a INT, b INT) USING DELTA") withSQLConf( DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true", DeltaSQLConf.DELTA_INSERT_BY_NAME_SCHEMA_EVOLUTION_ENABLED.key -> "false") { val ex = intercept[SparkThrowable] { sql("INSERT INTO target BY NAME SELECT 1 AS a, 2L AS b, 3 AS c") } checkError( ex, "INSERT_COLUMN_ARITY_MISMATCH.TOO_MANY_DATA_COLUMNS", parameters = Map( "tableName" -> s"`$catalogName`.`default`.`target`", "tableColumns" -> "`a`, `b`", "dataColumns" -> "`a`, `b`, `c`" )) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoTableSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.io.File import java.util.TimeZone import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.schema.InvariantViolationException import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaColumnMappingSelectedTestMixin, DeltaSQLCommandTest} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.test.shims.InvalidDefaultValueErrorShims import org.scalatest.BeforeAndAfter import org.apache.spark.{SparkConf, SparkContext, SparkException, SparkThrowable} import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row, SaveMode} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.functions.{lit, struct} import org.apache.spark.sql.internal.{LegacyBehaviorPolicy, SQLConf} import org.apache.spark.sql.internal.SQLConf.{LEAF_NODE_DEFAULT_PARALLELISM, PARTITION_OVERWRITE_MODE, PartitionOverwriteMode} import org.apache.spark.sql.test.{SharedSparkSession, TestSparkSession} import org.apache.spark.sql.types._ import org.apache.spark.util.Utils class DeltaInsertIntoSQLSuite extends DeltaInsertIntoTestsWithTempViews( supportsDynamicOverwrite = true, includeSQLOnlyTests = true) with DeltaSQLCommandTest { import testImplicits._ override protected def doInsert(tableName: String, insert: DataFrame, mode: SaveMode): Unit = { val tmpView = "tmp_view" withTempView(tmpView) { insert.createOrReplaceTempView(tmpView) val overwrite = if (mode == SaveMode.Overwrite) "OVERWRITE" else "INTO" sql(s"INSERT $overwrite TABLE $tableName SELECT * FROM $tmpView") } } test("Variant type") { withTable("t") { sql("CREATE TABLE t (id LONG, v VARIANT) USING delta") sql("INSERT INTO t (id, v) VALUES (1, parse_json('{\"a\": 1}'))") sql("INSERT INTO t (id, v) VALUES (2, parse_json('{\"b\": 2}'))") sql( "INSERT INTO t SELECT id, parse_json(cast(id as string)) v FROM range(2)") checkAnswer(sql("select * from t").selectExpr("id", "to_json(v)"), Seq(Row(1, "{\"a\":1}"), Row(2, "{\"b\":2}"), Row(0, "0"), Row(1, "1"))) } } test("insert overwrite should work with selecting constants") { withTable("t1") { sql("CREATE TABLE t1 (a int, b int, c int) USING delta PARTITIONED BY (b, c)") sql("INSERT OVERWRITE TABLE t1 PARTITION (c=3) SELECT 1, 2") checkAnswer( sql("SELECT * FROM t1"), Row(1, 2, 3) :: Nil ) sql("INSERT OVERWRITE TABLE t1 PARTITION (b=2, c=3) SELECT 1") checkAnswer( sql("SELECT * FROM t1"), Row(1, 2, 3) :: Nil ) sql("INSERT OVERWRITE TABLE t1 PARTITION (b=2, c) SELECT 1, 3") checkAnswer( sql("SELECT * FROM t1"), Row(1, 2, 3) :: Nil ) } } test("insertInto: append by name") { import testImplicits._ val t1 = "tbl" withTable(t1) { sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format") val df = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") sql(s"INSERT INTO $t1(id, data) VALUES(1L, 'a')") // Can be in a different order sql(s"INSERT INTO $t1(data, id) VALUES('b', 2L)") // Can be casted automatically sql(s"INSERT INTO $t1(data, id) VALUES('c', 3)") verifyTable(t1, df) withSQLConf(SQLConf.USE_NULLS_FOR_MISSING_DEFAULT_COLUMN_VALUES.key -> "false") { // Missing columns assert(intercept[AnalysisException] { sql(s"INSERT INTO $t1(data) VALUES(4)") }.getMessage.contains("Column id is not specified in INSERT")) // Missing columns with matching dataType assert(intercept[AnalysisException] { sql(s"INSERT INTO $t1(data) VALUES('b')") }.getMessage.contains("Column id is not specified in INSERT")) } // Duplicate columns assert(intercept[AnalysisException]( sql(s"INSERT INTO $t1(data, data) VALUES(5)")).getMessage.nonEmpty) } } test("insertInto: overwrite by name") { import testImplicits._ val t1 = "tbl" withTable(t1) { sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format") sql(s"INSERT OVERWRITE $t1(id, data) VALUES(1L, 'a')") verifyTable(t1, Seq((1L, "a")).toDF("id", "data")) // Can be in a different order sql(s"INSERT OVERWRITE $t1(data, id) VALUES('b', 2L)") verifyTable(t1, Seq((2L, "b")).toDF("id", "data")) // Can be casted automatically sql(s"INSERT OVERWRITE $t1(data, id) VALUES('c', 3)") verifyTable(t1, Seq((3L, "c")).toDF("id", "data")) withSQLConf(SQLConf.USE_NULLS_FOR_MISSING_DEFAULT_COLUMN_VALUES.key -> "false") { // Missing columns assert(intercept[AnalysisException] { sql(s"INSERT OVERWRITE $t1(data) VALUES(4)") }.getMessage.contains("Column id is not specified in INSERT")) // Missing columns with matching datatype assert(intercept[AnalysisException] { sql(s"INSERT OVERWRITE $t1(data) VALUES(4L)") }.getMessage.contains("Column id is not specified in INSERT")) } // Duplicate columns assert(intercept[AnalysisException]( sql(s"INSERT OVERWRITE $t1(data, data) VALUES(5)")).getMessage.nonEmpty) } } test("insertInto should throw an AnalysisError on name mismatch") { def testInsertByNameError(targetSchema: String, expectedErrorClass: String): Unit = { val sourceTableName = "source" val targetTableName = "target" val format = "delta" withTable(sourceTableName, targetTableName) { sql(s"CREATE TABLE $sourceTableName (a int, b int) USING $format") sql(s"CREATE TABLE $targetTableName $targetSchema USING $format") val e = intercept[AnalysisException] { sql(s"INSERT INTO $targetTableName BY NAME SELECT * FROM $sourceTableName") } assert(e.getErrorClass === expectedErrorClass) } } // NOTE: We use upper case in the target schema so that needsSchemaAdjustmentByName returns // true (due to case sensitivity) so that we call resolveQueryColumnsByName and hit the right // code path. // when the number of columns does not match and schema evolution is disabled, throw // an arity mismatch error. testInsertByNameError( targetSchema = "(A int)", expectedErrorClass = "INSERT_COLUMN_ARITY_MISMATCH.TOO_MANY_DATA_COLUMNS") // when the number of columns matches, but the names do not, throw a missing column error. testInsertByNameError( targetSchema = "(A int, c int)", expectedErrorClass = "DELTA_MISSING_COLUMN") } dynamicOverwriteTest("insertInto: dynamic overwrite by name") { import testImplicits._ val t1 = "tbl" withTable(t1) { sql(s"CREATE TABLE $t1 (id bigint, data string, data2 string) " + s"USING $v2Format PARTITIONED BY (id)") sql(s"INSERT OVERWRITE $t1(id, data, data2) VALUES(1L, 'a', 'b')") verifyTable(t1, Seq((1L, "a", "b")).toDF("id", "data", "data2")) // Can be in a different order sql(s"INSERT OVERWRITE $t1(data, data2, id) VALUES('b', 'd', 2L)") verifyTable(t1, Seq((1L, "a", "b"), (2L, "b", "d")).toDF("id", "data", "data2")) // Can be casted automatically sql(s"INSERT OVERWRITE $t1(data, data2, id) VALUES('c', 'e', 1)") verifyTable(t1, Seq((1L, "c", "e"), (2L, "b", "d")).toDF("id", "data", "data2")) withSQLConf(SQLConf.USE_NULLS_FOR_MISSING_DEFAULT_COLUMN_VALUES.key -> "false") { // Missing columns assert(intercept[AnalysisException] { sql(s"INSERT OVERWRITE $t1(data, id) VALUES('c', 1)") }.getMessage.contains("Column data2 is not specified in INSERT")) // Missing columns with matching datatype assert(intercept[AnalysisException] { sql(s"INSERT OVERWRITE $t1(data, id) VALUES('c', 1L)") }.getMessage.contains("Column data2 is not specified in INSERT")) } // Duplicate columns assert(intercept[AnalysisException]( sql(s"INSERT OVERWRITE $t1(data, data) VALUES(5)")).getMessage.nonEmpty) } } test("insertInto: static partition column name should not be used in the column list") { withTable("t") { sql(s"CREATE TABLE t(i STRING, c string) USING $v2Format PARTITIONED BY (c)") checkError( intercept[AnalysisException] { sql("INSERT OVERWRITE t PARTITION (c='1') (c) VALUES ('2')") }, "STATIC_PARTITION_COLUMN_IN_INSERT_COLUMN_LIST", parameters = Map("staticName" -> "c")) } } Seq(("ordinal", ""), ("name", "(id, col2, col)")).foreach { case (testName, values) => test(s"INSERT OVERWRITE schema evolution works for array struct types - $testName") { val sourceSchema = "id INT, col2 STRING, col ARRAY>" val sourceRecord = "1, '2022-11-01', array(struct('s1', 's2', DATE'2022-11-01'))" val targetSchema = "id INT, col2 DATE, col ARRAY>" val targetRecord = "1, DATE'2022-11-02', array(struct('t1', 't2'))" runInsertOverwrite(sourceSchema, sourceRecord, targetSchema, targetRecord) { (sourceTable, targetTable) => sql(s"INSERT OVERWRITE $targetTable $values SELECT * FROM $sourceTable") // make sure table is still writeable sql(s"""INSERT INTO $targetTable VALUES (2, DATE'2022-11-02', | array(struct('s3', 's4', DATE'2022-11-02')))""".stripMargin) sql(s"""INSERT INTO $targetTable VALUES (3, DATE'2022-11-03', |array(struct('s5', 's6', NULL)))""".stripMargin) val df = spark.sql( """SELECT 1 as id, DATE'2022-11-01' as col2, | array(struct('s1', 's2', DATE'2022-11-01')) as col UNION | SELECT 2 as id, DATE'2022-11-02' as col2, | array(struct('s3', 's4', DATE'2022-11-02')) as col UNION | SELECT 3 as id, DATE'2022-11-03' as col2, | array(struct('s5', 's6', NULL)) as col""".stripMargin) verifyTable(targetTable, df) } } } Seq(("ordinal", ""), ("name", "(id, col2, col)")).foreach { case (testName, values) => test(s"INSERT OVERWRITE schema evolution works for array nested types - $testName") { val sourceSchema = "id INT, col2 STRING, " + "col ARRAY, f3: STRUCT>>" val sourceRecord = "1, '2022-11-01', " + "array(struct(1, struct('s1', DATE'2022-11-01'), struct('s1')))" val targetSchema = "id INT, col2 DATE, col ARRAY>>" val targetRecord = "2, DATE'2022-11-02', array(struct(2, struct('s2')))" runInsertOverwrite(sourceSchema, sourceRecord, targetSchema, targetRecord) { (sourceTable, targetTable) => sql(s"INSERT OVERWRITE $targetTable $values SELECT * FROM $sourceTable") // make sure table is still writeable sql(s"""INSERT INTO $targetTable VALUES (2, DATE'2022-11-02', | array(struct(2, struct('s2', DATE'2022-11-02'), struct('s2'))))""".stripMargin) sql(s"""INSERT INTO $targetTable VALUES (3, DATE'2022-11-03', | array(struct(3, struct('s3', NULL), struct(NULL))))""".stripMargin) val df = spark.sql( """SELECT 1 as id, DATE'2022-11-01' as col2, | array(struct(1, struct('s1', DATE'2022-11-01'), struct('s1'))) as col UNION | SELECT 2 as id, DATE'2022-11-02' as col2, | array(struct(2, struct('s2', DATE'2022-11-02'), struct('s2'))) as col UNION | SELECT 3 as id, DATE'2022-11-03' as col2, | array(struct(3, struct('s3', NULL), struct(NULL))) as col |""".stripMargin) verifyTable(targetTable, df) } } } // Schema evolution for complex map type test("insertInto schema evolution with map type - append mode: field renaming + new field") { withTable("map_schema_evolution") { val tableName = "map_schema_evolution" val initialSchema = StructType(Seq( StructField("key", IntegerType, nullable = false), StructField("metrics", MapType(StringType, StructType(Seq( StructField("id", IntegerType, nullable = false), StructField("value", IntegerType, nullable = false) )))) )) val initialData = Seq( Row(1, Map("event" -> Row(1, 1))) ) val initialRdd = spark.sparkContext.parallelize(initialData) val initialDf = spark.createDataFrame(initialRdd, initialSchema) // Write initial data initialDf.write .option("overwriteSchema", "true") .mode("overwrite") .format("delta") .saveAsTable(tableName) // Evolved schema with field renamed and additional field in map struct val evolvedSchema = StructType(Seq( StructField("renamed_key", IntegerType, nullable = false), StructField("metrics", MapType(StringType, StructType(Seq( StructField("id", IntegerType, nullable = false), StructField("value", IntegerType, nullable = false), StructField("comment", StringType, nullable = true) )))) )) val evolvedData = Seq( Row(1, Map("event" -> Row(1, 1, "deprecated"))) ) val evolvedRdd = spark.sparkContext.parallelize(evolvedData) val evolvedDf = spark.createDataFrame(evolvedRdd, evolvedSchema) // insert data without schema evolution val err = intercept[AnalysisException] { evolvedDf.write .mode("append") .format("delta") .insertInto(tableName) } checkError(err, "DELTA_METADATA_MISMATCH", "42KDG", Map.empty[String, String]) // insert data with schema evolution withSQLConf("spark.databricks.delta.schema.autoMerge.enabled" -> "true") { evolvedDf.write .mode("append") .format("delta") .insertInto(tableName) checkAnswer( spark.sql(s"SELECT * FROM $tableName"), Seq( Row(1, Map("event" -> Row(1, 1, null))), Row(1, Map("event" -> Row(1, 1, "deprecated"))) )) } } } test("not enough column in source to insert in nested map types") { withTable("source", "target") { sql( """CREATE TABLE source ( | id INT, | metrics MAP> |) USING delta""".stripMargin) sql( """CREATE TABLE target ( | id INT, | metrics MAP> |) USING delta""".stripMargin) sql("INSERT INTO source VALUES (1, map('event', struct(1, 1)))") val e = intercept[AnalysisException] { sql("INSERT INTO target SELECT * FROM source") } checkError( exception = e, "DELTA_INSERT_COLUMN_ARITY_MISMATCH", parameters = Map( "tableName" -> "spark_catalog.default.target", "columnName" -> "not enough nested fields in value", "numColumns" -> "3", "insertColumns" -> "2" ) ) } } // not enough nested fields in value test("more columns in source to insert in nested map types") { withTable("source", "target") { sql( """CREATE TABLE source ( | id INT, | metrics MAP> |) USING delta""".stripMargin) sql( """CREATE TABLE target ( | id INT, | metrics MAP> |) USING delta""".stripMargin) sql("INSERT INTO source VALUES (1, map('event', struct(1, 1, 'deprecated')))") val e = intercept[AnalysisException] { sql("INSERT INTO target SELECT * FROM source") } checkError(e, "DELTA_METADATA_MISMATCH", "42KDG", Map.empty[String, String]) withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { sql("INSERT INTO target SELECT * FROM source") checkAnswer( spark.sql(s"SELECT * FROM source"), Seq( Row(1, Map("event" -> Row(1, 1, "deprecated"))) )) } } } test("more columns in source to insert in nested 2-level deep map types") { withTable("source", "target") { sql( """CREATE TABLE source ( | id INT, | metrics MAP>> |) USING delta""".stripMargin) sql( """CREATE TABLE target ( | id INT, | metrics MAP>> |) USING delta""".stripMargin) sql( """INSERT INTO source VALUES | (1, map('event', map('subEvent', struct(1, 1, 'deprecated')))) """.stripMargin) val e = intercept[AnalysisException] { sql("INSERT INTO target SELECT * FROM source") } checkError(e, "DELTA_METADATA_MISMATCH", "42KDG", Map.empty[String, String]) withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { sql("INSERT INTO target SELECT * FROM source") checkAnswer( spark.sql(s"SELECT * FROM source"), Seq( Row(1, Map("event" -> Map("subEvent" -> Row(1, 1, "deprecated")))) )) } } } test("insert map type with different data type in key") { withTable("source", "target") { sql( """CREATE TABLE source ( | id INT, | metrics MAP> |) USING delta""".stripMargin) sql( """CREATE TABLE target ( | id INT, | metrics MAP> |) USING delta""".stripMargin) sql("INSERT INTO source VALUES (1, map('1', struct(2, 3)))") sql("INSERT INTO target SELECT * FROM source") checkAnswer( spark.sql("SELECT * FROM target"), Seq( Row(1, Map(1 -> Row(2, 3))) )) } } test("insert map type with different data type in value") { withTable("source", "target") { sql( """CREATE TABLE source ( | id INT, | metrics MAP> |) USING delta""".stripMargin) sql( """CREATE TABLE target ( | id INT, | metrics MAP> |) USING delta""".stripMargin) sql("INSERT INTO source VALUES (1, map('m1', struct(2, 3L)))") sql("INSERT INTO target SELECT * FROM source") checkAnswer( spark.sql("SELECT * FROM target"), Seq( Row(1, Map("m1" -> Row(2, 3))) )) } } def runInsertOverwrite( sourceSchema: String, sourceRecord: String, targetSchema: String, targetRecord: String)( runAndVerify: (String, String) => Unit): Unit = { val sourceTable = "source" val targetTable = "target" withTable(sourceTable) { withTable(targetTable) { withSQLConf("spark.databricks.delta.schema.autoMerge.enabled" -> "true") { // prepare source table sql(s"""CREATE TABLE $sourceTable ($sourceSchema) | USING DELTA""".stripMargin) sql(s"INSERT INTO $sourceTable VALUES ($sourceRecord)") // prepare target table sql(s"""CREATE TABLE $targetTable ($targetSchema) | USING DELTA""".stripMargin) sql(s"INSERT INTO $targetTable VALUES ($targetRecord)") runAndVerify(sourceTable, targetTable) } } } } } class DeltaInsertIntoSQLByPathSuite extends DeltaInsertIntoTests(supportsDynamicOverwrite = true, includeSQLOnlyTests = true) with DeltaSQLCommandTest { override protected def doInsert(tableName: String, insert: DataFrame, mode: SaveMode): Unit = { val tmpView = "tmp_view" withTempView(tmpView) { insert.createOrReplaceTempView(tmpView) val overwrite = if (mode == SaveMode.Overwrite) "OVERWRITE" else "INTO" val ident = spark.sessionState.sqlParser.parseTableIdentifier(tableName) val catalogTable = spark.sessionState.catalog.getTableMetadata(ident) sql(s"INSERT $overwrite TABLE delta.`${catalogTable.location}` SELECT * FROM $tmpView") } } testQuietly("insertInto: cannot insert into a table that doesn't exist") { import testImplicits._ Seq(SaveMode.Append, SaveMode.Overwrite).foreach { mode => withTempDir { dir => val t1 = s"delta.`${dir.getCanonicalPath}`" val tmpView = "tmp_view" withTempView(tmpView) { val overwrite = if (mode == SaveMode.Overwrite) "OVERWRITE" else "INTO" val df = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") df.createOrReplaceTempView(tmpView) intercept[AnalysisException] { sql(s"INSERT $overwrite TABLE $t1 SELECT * FROM $tmpView") } assert(new File(dir, "_delta_log").mkdirs(), "Failed to create a _delta_log directory") intercept[AnalysisException] { sql(s"INSERT $overwrite TABLE $t1 SELECT * FROM $tmpView") } } } } } } class DeltaInsertIntoDataFrameSuite extends DeltaInsertIntoTestsWithTempViews( supportsDynamicOverwrite = true, includeSQLOnlyTests = false) with DeltaSQLCommandTest { override protected def doInsert(tableName: String, insert: DataFrame, mode: SaveMode): Unit = { val dfw = insert.write.format(v2Format) if (mode != null) { dfw.mode(mode) } dfw.insertInto(tableName) } } class DeltaInsertIntoDataFrameByPathSuite extends DeltaInsertIntoTests(supportsDynamicOverwrite = true, includeSQLOnlyTests = false) with DeltaSQLCommandTest { override protected def doInsert(tableName: String, insert: DataFrame, mode: SaveMode): Unit = { val dfw = insert.write.format(v2Format) if (mode != null) { dfw.mode(mode) } val ident = spark.sessionState.sqlParser.parseTableIdentifier(tableName) val catalogTable = spark.sessionState.catalog.getTableMetadata(ident) dfw.insertInto(s"delta.`${catalogTable.location}`") } testQuietly("insertInto: cannot insert into a table that doesn't exist") { import testImplicits._ Seq(SaveMode.Append, SaveMode.Overwrite).foreach { mode => withTempDir { dir => val t1 = s"delta.`${dir.getCanonicalPath}`" val df = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") intercept[AnalysisException] { df.write.mode(mode).insertInto(t1) } assert(new File(dir, "_delta_log").mkdirs(), "Failed to create a _delta_log directory") intercept[AnalysisException] { df.write.mode(mode).insertInto(t1) } // Test DataFrameWriterV2 as well val dfW2 = df.writeTo(t1) if (mode == SaveMode.Append) { intercept[AnalysisException] { dfW2.append() } } else { intercept[AnalysisException] { dfW2.overwrite(lit(true)) } } } } } } trait DeltaInsertIntoColumnMappingSelectedTests extends DeltaColumnMappingSelectedTestMixin { override protected def runOnlyTests = Seq( "InsertInto: overwrite - mixed clause reordered - static mode", "InsertInto: overwrite - multiple static partitions - dynamic mode" ) } class DeltaInsertIntoSQLNameColumnMappingSuite extends DeltaInsertIntoSQLSuite with DeltaColumnMappingEnableNameMode with DeltaInsertIntoColumnMappingSelectedTests { override protected def runOnlyTests: Seq[String] = super.runOnlyTests :+ "insert overwrite should work with selecting constants" } class DeltaInsertIntoSQLByPathNameColumnMappingSuite extends DeltaInsertIntoSQLByPathSuite with DeltaColumnMappingEnableNameMode with DeltaInsertIntoColumnMappingSelectedTests class DeltaInsertIntoDataFrameNameColumnMappingSuite extends DeltaInsertIntoDataFrameSuite with DeltaColumnMappingEnableNameMode with DeltaInsertIntoColumnMappingSelectedTests class DeltaInsertIntoDataFrameByPathNameColumnMappingSuite extends DeltaInsertIntoDataFrameByPathSuite with DeltaColumnMappingEnableNameMode with DeltaInsertIntoColumnMappingSelectedTests abstract class DeltaInsertIntoTestsWithTempViews( supportsDynamicOverwrite: Boolean, includeSQLOnlyTests: Boolean) extends DeltaInsertIntoTests(supportsDynamicOverwrite, includeSQLOnlyTests) with DeltaTestUtilsForTempViews { protected def testComplexTempViews(name: String)(text: String, expectedResult: Seq[Row]): Unit = { testWithTempView(s"insertInto a temp view created on top of a table - $name") { isSQLTempView => import testImplicits._ val t1 = "tbl" sql(s"CREATE TABLE $t1 (key int, value int) USING $v2Format") Seq(SaveMode.Append, SaveMode.Overwrite).foreach { mode => createTempViewFromSelect(text, isSQLTempView) val df = Seq((0, 3), (1, 2)).toDF("key", "value") try { doInsert("v", df, mode) checkAnswer(spark.table("v"), expectedResult) } catch { case e: AnalysisException => assert( e.getMessage.contains("[EXPECT_TABLE_NOT_VIEW.NO_ALTERNATIVE]") || e.getMessage.contains("Inserting into an RDD-based table is not allowed") || e.getMessage.contains("Table default.v not found") || e.getMessage.contains("Table or view 'v' not found in database 'default'") || e.getMessage.contains("The table or view `default`.`v` cannot be found") || e.getMessage.contains( "[UNSUPPORTED_INSERT.RDD_BASED] Can't insert into the target.") || e.getMessage.contains( "The table or view `spark_catalog`.`default`.`v` cannot be found")) } } } } testComplexTempViews("basic") ( "SELECT * FROM tbl", Seq(Row(0, 3), Row(1, 2)) ) testComplexTempViews("subset cols")( "SELECT key FROM tbl", Seq(Row(0), Row(1)) ) testComplexTempViews("superset cols")( "SELECT key, value, 1 FROM tbl", Seq(Row(0, 3, 1), Row(1, 2, 1)) ) testComplexTempViews("nontrivial projection")( "SELECT value as key, key as value FROM tbl", Seq(Row(3, 0), Row(2, 1)) ) testComplexTempViews("view with too many internal aliases")( "SELECT * FROM (SELECT * FROM tbl AS t1) AS t2", Seq(Row(0, 3), Row(1, 2)) ) } class DeltaColumnDefaultsInsertSuite extends InsertIntoSQLOnlyTests with DeltaSQLCommandTest { import testImplicits._ override val supportsDynamicOverwrite = true override val includeSQLOnlyTests = true val tblPropertiesAllowDefaults = """tblproperties ( | 'delta.feature.allowColumnDefaults' = 'enabled', | 'delta.columnMapping.mode' = 'name' |)""".stripMargin test("Column DEFAULT value support with Delta Lake, positive tests") { Seq( PartitionOverwriteMode.STATIC.toString, PartitionOverwriteMode.DYNAMIC.toString ).foreach { partitionOverwriteMode => withSQLConf( SQLConf.ENABLE_DEFAULT_COLUMNS.key -> "true", SQLConf.PARTITION_OVERWRITE_MODE.key -> partitionOverwriteMode, // Set these configs to allow writing test values like timestamps of Jan. 1, year 1, etc. SQLConf.PARQUET_REBASE_MODE_IN_WRITE.key -> LegacyBehaviorPolicy.LEGACY.toString, SQLConf.PARQUET_INT96_REBASE_MODE_IN_WRITE.key -> LegacyBehaviorPolicy.LEGACY.toString) { withTable("t1", "t2", "t3", "t4") { // Positive tests: // Create some columns with default values and then insert into them. sql("create table t1(" + s"a int default 42, b boolean default true, c string default 'abc') using $v2Format " + s"partitioned by (a) $tblPropertiesAllowDefaults") sql("insert into t1 values (1, false, default)") sql("insert into t1 values (1, default, default)") sql("alter table t1 alter column c set default 'def'") sql("insert into t1 values (default, default, default)") sql("alter table t1 alter column c drop default") // Exercise INSERT INTO commands with VALUES lists mapping columns positionally. sql("insert into t1 values (default, default, default)") // Write the data in the table 't1' to new table 't4' and then perform an INSERT OVERWRITE // back to 't1' here, to exercise static and dynamic partition overwrites. sql(f"create table t4(a int, b boolean, c string) using $v2Format " + s"partitioned by (a) $tblPropertiesAllowDefaults") // Exercise INSERT INTO commands with SELECT queries mapping columns by name. sql("insert into t4(a, b, c) select a, b, c from t1") sql("insert overwrite table t1 select * from t4") checkAnswer(spark.table("t1"), Seq( Row(1, false, "abc"), Row(1, true, "abc"), Row(42, true, "def"), Row(42, true, null) )) // Insert default values with all supported types. sql("create table t2(" + "s boolean default true, " + "t byte default cast(null as byte), " + "u short default cast(42 as short), " + "v float default 0, " + "w double default 0, " + "x date default date'0000', " + "y timestamp default timestamp'0000', " + "z decimal(5, 2) default 123.45," + "a1 bigint default 43," + "a2 smallint default cast(5 as smallint)," + s"a3 tinyint default cast(6 as tinyint)) using $v2Format " + tblPropertiesAllowDefaults) sql("insert into t2 values (default, default, default, default, default, default, " + "default, default, default, default, default)") val result: Array[Row] = spark.table("t2").collect() assert(result.length == 1) val row: Row = result(0) assert(row.length == 11) assert(row(0) == true) assert(row(1) == null) assert(row(2) == 42) assert(row(3) == 0.0f) assert(row(4) == 0.0d) assert(row(5).toString == "0001-01-01") assert(row(6).toString == "0001-01-01 00:00:00.0") assert(row(7).toString == "123.45") assert(row(8) == 43L) assert(row(9) == 5) assert(row(10) == 6) } withTable("t3") { // Set a default value for a partitioning column. sql(s"create table t3(i boolean, s bigint, q int default 42) using $v2Format " + s"partitioned by (i) $tblPropertiesAllowDefaults") sql("alter table t3 alter column i set default true") sql("insert into t3(i, s, q) values (default, default, default)") checkAnswer(spark.table("t3"), Seq( Row(true, null, 42))) // Drop the column and add it again without the default. Querying the column now returns // NULL. sql("alter table t3 drop column q") sql("alter table t3 add column q int") checkAnswer(spark.table("t3"), Seq( Row(true, null, null))) } } } } test("Column DEFAULT value support with Delta Lake, negative tests") { withSQLConf(SQLConf.ENABLE_DEFAULT_COLUMNS.key -> "true") { // The table feature is not enabled via TBLPROPERTIES. withTable("createTableWithDefaultFeatureNotEnabled") { checkError( intercept[DeltaAnalysisException] { sql(s"create table createTableWithDefaultFeatureNotEnabled(" + s"i boolean, s bigint, q int default 42) using $v2Format " + "partitioned by (i)") }, "WRONG_COLUMN_DEFAULTS_FOR_DELTA_FEATURE_NOT_ENABLED", parameters = Map("commandType" -> "CREATE TABLE") ) } withTable("alterTableSetDefaultFeatureNotEnabled") { sql(s"create table alterTableSetDefaultFeatureNotEnabled(a int) using $v2Format") checkError( intercept[DeltaAnalysisException] { sql("alter table alterTableSetDefaultFeatureNotEnabled alter column a set default 42") }, "WRONG_COLUMN_DEFAULTS_FOR_DELTA_FEATURE_NOT_ENABLED", parameters = Map("commandType" -> "ALTER TABLE") ) } // Adding a new column with a default value to an existing table is not allowed. withTable("alterTableTest") { sql(s"create table alterTableTest(i boolean, s bigint, q int default 42) using $v2Format " + s"partitioned by (i) $tblPropertiesAllowDefaults") checkError( intercept[DeltaAnalysisException] { sql("alter table alterTableTest add column z int default 42") }, "WRONG_COLUMN_DEFAULTS_FOR_DELTA_ALTER_TABLE_ADD_COLUMN_NOT_SUPPORTED" ) } // The default value fails to analyze. checkError( intercept[AnalysisException] { sql(s"create table t4 (s int default badvalue) using $v2Format " + s"$tblPropertiesAllowDefaults") }, InvalidDefaultValueErrorShims.INVALID_DEFAULT_VALUE_ERROR_CODE, parameters = Map( "statement" -> "CREATE TABLE", "colName" -> "`s`", "defaultValue" -> "badvalue")) // The default value analyzes to a table not in the catalog. // The error message reports that we failed to execute the command because subquery // expressions are not allowed in DEFAULT values. checkError( intercept[AnalysisException] { sql(s"create table t4 (s int default (select min(x) from badtable)) using $v2Format " + tblPropertiesAllowDefaults) }, "INVALID_DEFAULT_VALUE.SUBQUERY_EXPRESSION", parameters = Map( "statement" -> "CREATE TABLE", "colName" -> "`s`", "defaultValue" -> "(select min(x) from badtable)")) // The default value has an explicit alias. It fails to evaluate when inlined into the // VALUES list at the INSERT INTO time. // The error message reports that we failed to execute the command because subquery // expressions are not allowed in DEFAULT values. checkError( intercept[AnalysisException] { sql(s"create table t4 (s int default (select 42 as alias)) using $v2Format " + tblPropertiesAllowDefaults) }, "INVALID_DEFAULT_VALUE.SUBQUERY_EXPRESSION", parameters = Map( "statement" -> "CREATE TABLE", "colName" -> "`s`", "defaultValue" -> "(select 42 as alias)")) // The default value parses but the type is not coercible. checkError( intercept[AnalysisException] { sql(s"create table t4 (s bigint default false) " + s"using $v2Format $tblPropertiesAllowDefaults") }, "INVALID_DEFAULT_VALUE.DATA_TYPE", parameters = Map( "statement" -> "CREATE TABLE", "colName" -> "`s`", "expectedType" -> "\"BIGINT\"", "actualType" -> "\"BOOLEAN\"", "defaultValue" -> "false")) // It is possible to create a table with NOT NULL constraint and a DEFAULT value of NULL. // However, future inserts into that table will fail. withTable("t4") { sql(s"create table t4(i boolean, s bigint, q int default null not null) using $v2Format " + s"partitioned by (i) $tblPropertiesAllowDefaults") // The InvariantViolationException is not a SparkThrowable, so just check we receive one. assert(intercept[InvariantViolationException] { sql("insert into t4 values (default, default, default)") }.getMessage.nonEmpty) } // It is possible to create a table with a check constraint and a DEFAULT value that does not // conform. However, future inserts into that table will fail. withTable("t4") { sql(s"create table t4(i boolean, s bigint, q int default 42) using $v2Format " + s"partitioned by (i) $tblPropertiesAllowDefaults") sql("alter table t4 add constraint smallq check (q < 10)") assert(intercept[InvariantViolationException] { sql("insert into t4 values (default, default, default)") }.getMessage.nonEmpty) } } // Column default values are disabled per configuration in general. withSQLConf(SQLConf.ENABLE_DEFAULT_COLUMNS.key -> "false") { checkError( intercept[ParseException] { sql(s"create table t4 (s int default 41 + 1) using $v2Format " + tblPropertiesAllowDefaults) }, "UNSUPPORTED_DEFAULT_VALUE.WITH_SUGGESTION", parameters = Map.empty, context = ExpectedContext(fragment = "s int default 41 + 1", start = 17, stop = 36)) } } test("Exercise column defaults with dataframe writes") { // There are three column types exercising various combinations of implicit and explicit // default column value references in the 'insert into' statements. Note these tests depend on // enabling the configuration to use NULLs for missing DEFAULT column values. withSQLConf(SQLConf.USE_NULLS_FOR_MISSING_DEFAULT_COLUMN_VALUES.key -> "true") { for (useDataFrames <- Seq(false, true)) { withTable("t1", "t2") { sql(s"create table t1(j int, s bigint default 42, x bigint default 43) using $v2Format " + tblPropertiesAllowDefaults) if (useDataFrames) { // Use 'saveAsTable' to exercise mapping columns by name. Note that we have to specify // values for all columns of the target table here whether we use 'saveAsTable' or // 'insertInto', since the DataFrame generates a LogicalPlan equivalent to a SQL INSERT // INTO command without any explicit user-specified column list. For example, if we // used Seq((1)).toDF("j", "s", "x").write.mode("append") here instead, it would // generate an unresolved LogicalPlan equivalent to the SQL query // "INSERT INTO t1 VALUES (1)". This would fail with an error reporting the VALUES // list is not long enough, since the analyzer would consider this equivalent to // "INSERT INTO t1 (j, s, x) VALUES (1)". Seq((1, 42L, 43L)).toDF("j", "s", "x").write.mode("append") .format("delta").saveAsTable("t1") Seq((2, 42L, 43L)).toDF("j", "s", "x").write.mode("append") .format("delta").saveAsTable("t1") Seq((3, 42L, 43L)).toDF("j", "s", "x").write.mode("append") .format("delta").saveAsTable("t1") Seq((4, 44L, 43L)).toDF("j", "s", "x").write.mode("append") .format("delta").saveAsTable("t1") Seq((5, 44L, 45L)).toDF("j", "s", "x") .write.mode("append").format("delta").saveAsTable("t1") } else { sql("insert into t1(j) values(1)") sql("insert into t1(j, s) values(2, default)") sql("insert into t1(j, s, x) values(3, default, default)") sql("insert into t1(j, s) values(4, 44)") sql("insert into t1(j, s, x) values(5, 44, 45)") } sql(s"create table t2(j int, s bigint default 42, x bigint default 43) using $v2Format " + tblPropertiesAllowDefaults) if (useDataFrames) { // Use 'insertInto' to exercise mapping columns positionally. spark.table("t1").where("j = 1").write.insertInto("t2") spark.table("t1").where("j = 2").write.insertInto("t2") spark.table("t1").where("j = 3").write.insertInto("t2") spark.table("t1").where("j = 4").write.insertInto("t2") spark.table("t1").where("j = 5").write.insertInto("t2") } else { sql("insert into t2(j) select j from t1 where j = 1") sql("insert into t2(j, s) select j, default from t1 where j = 2") sql("insert into t2(j, s, x) select j, default, default from t1 where j = 3") sql("insert into t2(j, s) select j, s from t1 where j = 4") sql("insert into t2(j, s, x) select j, s, 45L from t1 where j = 5") } checkAnswer( spark.table("t2"), Row(1, 42L, 43L) :: Row(2, 42L, 43L) :: Row(3, 42L, 43L) :: Row(4, 44L, 43L) :: Row(5, 44L, 45L) :: Nil) // Also exercise schema evolution with DataFrames. if (useDataFrames) { Seq((5, 44L, 45L, 46L)).toDF("j", "s", "x", "y") .write.mode("append").format("delta").option("mergeSchema", "true") .saveAsTable("t2") checkAnswer( spark.table("t2"), Row(1, 42L, 43L, null) :: Row(2, 42L, 43L, null) :: Row(3, 42L, 43L, null) :: Row(4, 44L, 43L, null) :: Row(5, 44L, 45L, null) :: Row(5, 44L, 45L, 46L) :: Nil) } } } } } test("ReplaceWhere with column defaults with dataframe writes") { withTable("t1", "t2", "t3") { sql(s"create table t1(j int, s bigint default 42, x bigint default 43) using $v2Format " + tblPropertiesAllowDefaults) Seq((1, 42L, 43L)).toDF.write.insertInto("t1") Seq((2, 42L, 43L)).toDF.write.insertInto("t1") Seq((3, 42L, 43L)).toDF.write.insertInto("t1") Seq((4, 44L, 43L)).toDF.write.insertInto("t1") Seq((5, 44L, 45L)).toDF.write.insertInto("t1") spark.table("t1") .write.format("delta") .mode("overwrite") .option("replaceWhere", "j = default and s = default and x = default") .saveAsTable("t2") Seq("t1", "t2").foreach { t => checkAnswer( spark.table(t), Row(1, 42L, 43L) :: Row(2, 42L, 43L) :: Row(3, 42L, 43L) :: Row(4, 44L, 43L) :: Row(5, 44L, 45L) :: Nil) } } } test("DESCRIBE and SHOW CREATE TABLE with column defaults") { withTable("t") { spark.sql(s"CREATE TABLE t (id bigint default 42) " + s"using $v2Format $tblPropertiesAllowDefaults") val descriptionDf = spark.sql(s"DESCRIBE TABLE EXTENDED t") assert(descriptionDf.schema.map { field => (field.name, field.dataType) } === Seq( ("col_name", StringType), ("data_type", StringType), ("comment", StringType))) QueryTest.checkAnswer( descriptionDf.filter( "!(col_name in ('Catalog', 'Created Time', 'Created By', 'Database', " + "'index', 'Is_managed_location', 'Location', 'Name', 'Owner', 'Partition Provider'," + "'Provider', 'Table', 'Table Properties', 'Type', '_partition', 'Last Access', " + "'Statistics', ''))"), Seq( Row("# Column Default Values", "", ""), Row("# Detailed Table Information", "", ""), Row("id", "bigint", "42"), Row("id", "bigint", null) )) } withTable("t") { sql( s""" |CREATE TABLE t ( | a bigint NOT NULL, | b bigint DEFAULT 42, | c string DEFAULT 'abc, "def"' COMMENT 'comment' |) |USING parquet |COMMENT 'This is a comment' |$tblPropertiesAllowDefaults """.stripMargin) val currentCatalog = spark.sessionState.catalogManager.currentCatalog.name() QueryTest.checkAnswer(sql("SHOW CREATE TABLE T"), Seq( Row( s"""CREATE TABLE ${currentCatalog}.default.T ( | a BIGINT, | b BIGINT DEFAULT 42, | c STRING DEFAULT 'abc, "def"' COMMENT 'comment') |USING parquet |COMMENT 'This is a comment' |TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name', | 'delta.feature.allowColumnDefaults' = 'enabled') |""".stripMargin))) } } } /** These tests come from Apache Spark with some modifications to match Delta behavior. */ abstract class DeltaInsertIntoTests( override protected val supportsDynamicOverwrite: Boolean, override protected val includeSQLOnlyTests: Boolean) extends InsertIntoSQLOnlyTests { import testImplicits._ override def afterEach(): Unit = { spark.catalog.listTables().collect().foreach(t => sql(s"drop table ${t.name}")) super.afterEach() } // START Apache Spark tests /** * Insert data into a table using the insertInto statement. Implementations can be in SQL * ("INSERT") or using the DataFrameWriter (`df.write.insertInto`). Insertions will be * by column ordinal and not by column name. */ protected def doInsert(tableName: String, insert: DataFrame, mode: SaveMode = null): Unit test("insertInto: append") { val t1 = "tbl" sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format") val df = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") doInsert(t1, df) verifyTable(t1, df) } test("insertInto: append by position") { val t1 = "tbl" sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format") val df = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") val dfr = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("data", "id") doInsert(t1, dfr) verifyTable(t1, df) } test("insertInto: append cast automatically") { val t1 = "tbl" sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format") val df = Seq((1, "a"), (2, "b"), (3, "c")).toDF("id", "data") doInsert(t1, df) verifyTable(t1, df) } test("insertInto: append partitioned table") { val t1 = "tbl" withTable(t1) { sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format PARTITIONED BY (id)") val df = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") doInsert(t1, df) verifyTable(t1, df) } } test("insertInto: overwrite non-partitioned table") { val t1 = "tbl" sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format") val df = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") val df2 = Seq((4L, "d"), (5L, "e"), (6L, "f")).toDF("id", "data") doInsert(t1, df) doInsert(t1, df2, SaveMode.Overwrite) verifyTable(t1, df2) } test("insertInto: overwrite partitioned table in static mode") { withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.STATIC.toString) { val t1 = "tbl" sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format PARTITIONED BY (id)") val init = Seq((2L, "dummy"), (4L, "keep")).toDF("id", "data") doInsert(t1, init) val df = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") doInsert(t1, df, SaveMode.Overwrite) verifyTable(t1, df) } } test("insertInto: overwrite by position") { withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.STATIC.toString) { val t1 = "tbl" withTable(t1) { sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format PARTITIONED BY (id)") val init = Seq((2L, "dummy"), (4L, "keep")).toDF("id", "data") doInsert(t1, init) val dfr = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("data", "id") doInsert(t1, dfr, SaveMode.Overwrite) val df = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") verifyTable(t1, df) } } } test("insertInto: overwrite cast automatically") { val t1 = "tbl" sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format") val df = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") val df2 = Seq((4L, "d"), (5L, "e"), (6L, "f")).toDF("id", "data") val df2c = Seq((4, "d"), (5, "e"), (6, "f")).toDF("id", "data") doInsert(t1, df) doInsert(t1, df2c, SaveMode.Overwrite) verifyTable(t1, df2) } test("insertInto: fails when missing a column") { val t1 = "tbl" sql(s"CREATE TABLE $t1 (id bigint, data string, missing string) USING $v2Format") val df1 = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") // mismatched datatype val df2 = Seq((1, "a"), (2, "b"), (3, "c")).toDF("id", "data") for (df <- Seq(df1, df2)) { val exc = intercept[AnalysisException] { doInsert(t1, df) } verifyTable(t1, Seq.empty[(Long, String, String)].toDF("id", "data", "missing")) assert(exc.getMessage.contains("not enough data columns")) } } test("insertInto: overwrite fails when missing a column") { val t1 = "tbl" sql(s"CREATE TABLE $t1 (id bigint, data string, missing string) USING $v2Format") val df1 = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") // mismatched datatype val df2 = Seq((1, "a"), (2, "b"), (3, "c")).toDF("id", "data") for (df <- Seq(df1, df2)) { val exc = intercept[AnalysisException] { doInsert(t1, df, SaveMode.Overwrite) } verifyTable(t1, Seq.empty[(Long, String, String)].toDF("id", "data", "missing")) assert(exc.getMessage.contains("not enough data columns")) } } // This behavior is specific to Delta test("insertInto: fails when an extra column is present but can evolve schema") { val t1 = "tbl" withTable(t1) { sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format") val df = Seq((1L, "a", "mango")).toDF("id", "data", "fruit") val exc = intercept[AnalysisException] { doInsert(t1, df) } verifyTable(t1, Seq.empty[(Long, String)].toDF("id", "data")) assert(exc.getMessage.contains(s"mergeSchema")) withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { doInsert(t1, df) } verifyTable(t1, Seq((1L, "a", "mango")).toDF("id", "data", "fruit")) } } test("insertInto: UTC timestamp partition values round trip across different session TZ") { val t1 = "utc_timestamp_partitioned_values" withTable(t1) { withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") { sql(s"CREATE TABLE $t1 (data int, ts timestamp) USING delta PARTITIONED BY (ts)") sql(s"INSERT INTO $t1 VALUES (1, timestamp'2024-06-15T04:00:00UTC')") sql(s"INSERT INTO $t1 VALUES (2, timestamp'2024-06-15T4:00:00UTC+8')") sql(s"INSERT INTO $t1 VALUES (3, timestamp'2024-06-15T5:00:00 UTC+01:00')") sql(s"INSERT INTO $t1 VALUES (4, timestamp'2024-06-16T5:00:00.123456UTC')") sql(s"INSERT INTO $t1 VALUES (5, timestamp'1903-12-28T5:00:00')") } withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "GMT-8") { val deltaLog = DeltaLog.forTable( spark, TableIdentifier(t1)) val allFiles = deltaLog.unsafeVolatileSnapshot.allFiles val partitionColName = deltaLog.unsafeVolatileMetadata.physicalPartitionColumns.head checkAnswer( allFiles.select("partitionValues").orderBy("modificationTime"), Seq( Row(Map(partitionColName -> "2024-06-15T04:00:00.000000Z")), Row(Map(partitionColName -> "2024-06-14T20:00:00.000000Z")), Row(Map(partitionColName -> "2024-06-15T04:00:00.000000Z")), Row(Map(partitionColName -> "2024-06-16T05:00:00.123456Z")), Row(Map(partitionColName -> "1903-12-28T05:00:00.000000Z")) )) checkAnswer( sql(s"SELECT data FROM $t1 where ts = timestamp'2024-06-15T4:00:00UTC+8'"), Seq(Row(2))) checkAnswer( sql(s"SELECT data FROM $t1 where ts = timestamp'2024-06-14T20:00:00UTC-08'"), Seq(Row(1), Row(3))) checkAnswer( sql(s"SELECT data FROM $t1 where ts = timestamp'1903-12-27T21:00:00UTC-08'"), Seq(Row(5))) checkAnswer(sql(s"SELECT count(distinct(ts)) from $t1"), Seq(Row(4))) } } } test("insertInto: timestamp partition values across different" + " non-UTC session timezones round-trip when UTC adjusted") { val t1 = "utc_write_and_read_non_utc_tz" withTable(t1) { withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "America/Los_Angeles") { sql(s"CREATE TABLE $t1 (data int, ts timestamp) USING delta PARTITIONED BY (ts)") sql(s"INSERT INTO $t1 VALUES (1, timestamp'2025-11-26T12:00:00')") sql(s"INSERT INTO $t1 VALUES (2, timestamp'2025-11-27T4:00:00UTC+8')") sql(s"INSERT INTO $t1 VALUES (3, timestamp'2025-11-28T5:00:00 UTC+01:00')") } withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "Europe/Berlin") { val deltaLog = DeltaLog.forTable( spark, TableIdentifier(t1)) val allFiles = deltaLog.unsafeVolatileSnapshot.allFiles val partitionColName = deltaLog.unsafeVolatileMetadata.physicalPartitionColumns.head checkAnswer( allFiles.select("partitionValues").orderBy("modificationTime"), Seq( Row(Map(partitionColName -> "2025-11-26T20:00:00.000000Z")), Row(Map(partitionColName -> "2025-11-26T20:00:00.000000Z")), Row(Map(partitionColName -> "2025-11-28T04:00:00.000000Z")))) // Berlin is UTC+1 checkAnswer( sql(s"SELECT data FROM $t1 where ts = timestamp'2025-11-26T21:00:00' order by data"), Seq(Row(1), Row(2))) checkAnswer( sql(s"SELECT data FROM $t1 where ts = timestamp'2025-11-28T5:00:00'"), Seq(Row(3))) checkAnswer(sql(s"SELECT count(distinct(ts)) from $t1"), Seq(Row(2))) } } } test("insertInto: partition and non-partitioned timestamps have some behavior across timezones") { val t1 = "utc_partition_and_non_partitioned_ts" withTable(t1) { withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "Asia/Kolkata") { sql(s"CREATE TABLE $t1 (data int, ts_partition timestamp, ts_value timestamp) " + s"USING delta PARTITIONED BY (ts_partition)") sql(s"INSERT INTO $t1 VALUES " + s"(1, timestamp'2025-11-27T01:30:00', timestamp'2025-11-27T01:30:00')") sql(s"INSERT INTO $t1 VALUES " + s"(2, timestamp'2025-11-27T4:00:00UTC+8', timestamp'2025-11-27T4:00:00UTC+8')") sql(s"INSERT INTO $t1 VALUES " + s"(3, timestamp'2025-11-28T5:00:00 UTC+01:00', timestamp'2025-11-28T5:00:00 UTC+01:00')") } withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "America/Los_Angeles") { val deltaLog = DeltaLog.forTable( spark, TableIdentifier(t1)) val allFiles = deltaLog.unsafeVolatileSnapshot.allFiles val partitionColName = deltaLog.unsafeVolatileMetadata.physicalPartitionColumns.head checkAnswer( allFiles.select("partitionValues").orderBy("modificationTime"), Seq( Row(Map(partitionColName -> "2025-11-26T20:00:00.000000Z")), Row(Map(partitionColName -> "2025-11-26T20:00:00.000000Z")), Row(Map(partitionColName -> "2025-11-28T04:00:00.000000Z")))) // America/Los_Angeles is UTC-8 checkAnswer( sql(s"SELECT data FROM $t1 where " + s"ts_partition = timestamp'2025-11-26T12:00:00' and ts_value='2025-11-26T12:00:00' " + s"order by data"), Seq(Row(1), Row(2))) checkAnswer( sql(s"SELECT data FROM $t1 where " + s"ts_partition = timestamp'2025-11-27T20:00:00' " + s"and ts_value = timestamp'2025-11-27T20:00:00'"), Seq(Row(3))) checkAnswer(sql(s"SELECT count(distinct(ts_partition)) from $t1"), Seq(Row(2))) checkAnswer(sql(s"SELECT count(distinct(ts_value)) from $t1"), Seq(Row(2))) } } } test("insertInto: Non-UTC and UTC partition values round trip same session TZ") { val t1 = "utc_timestamp_partitioned_values" withTable(t1) { withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "GMT-8") { sql(s"CREATE TABLE $t1 (data int, ts timestamp) USING delta PARTITIONED BY (ts)") sql(s"INSERT INTO $t1 VALUES (1, timestamp'2024-06-15T4:00:00UTC')") sql(s"INSERT INTO $t1 VALUES (2, timestamp'2024-06-15T4:00:00UTC+8')") } withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "GMT-8") { withSQLConf(DeltaSQLConf.UTC_TIMESTAMP_PARTITION_VALUES.key -> "false") { sql(s"INSERT INTO $t1 VALUES (3, timestamp'2024-06-15T5:00:00 UTC+01:00')") sql(s"INSERT INTO $t1 VALUES (4, timestamp'1903-12-28T5:00:00')") } val deltaLog = DeltaLog.forTable( spark, TableIdentifier(t1)) val allFiles = deltaLog.unsafeVolatileSnapshot.allFiles val partitionColName = deltaLog.unsafeVolatileMetadata.physicalPartitionColumns.head checkAnswer( allFiles.select("partitionValues").orderBy("modificationTime"), Seq( Row(Map(partitionColName -> "2024-06-15T04:00:00.000000Z")), Row(Map(partitionColName -> "2024-06-14T20:00:00.000000Z")), Row(Map(partitionColName -> "2024-06-14 20:00:00")), Row(Map(partitionColName -> "1903-12-28 05:00:00")) )) } withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "GMT-8") { checkAnswer( sql(s"SELECT data FROM $t1 where ts = timestamp'2024-06-15T4:00:00UTC+8'"), Seq(Row(2))) checkAnswer( sql(s"SELECT data FROM $t1 where ts = timestamp'2024-06-14T20:00:00'"), Seq(Row(1), Row(3))) checkAnswer( sql(s"SELECT data FROM $t1 where ts = timestamp'1903-12-28T05:00:00'"), Seq(Row(4))) checkAnswer(sql(s"SELECT count(distinct(ts)) from $t1"), Seq(Row(3))) } } } test("insertInto: Timestamp No Timezone can be interpreted across timezones") { val t1 = "timestamp_ntz" withTable(t1) { withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "GMT-8") { sql(s"CREATE TABLE $t1 (data int, ts timestamp_ntz) USING delta PARTITIONED BY (ts)") sql(s"INSERT INTO $t1 VALUES (1, timestamp'2024-06-15T4:00:00')") sql(s"INSERT INTO $t1 VALUES (2, timestamp'2024-06-16T5:00:00')") sql(s"INSERT INTO $t1 VALUES (3, timestamp'1903-12-28T5:00:00')") val deltaLog = DeltaLog.forTable( spark, TableIdentifier(t1)) val allFiles = deltaLog.unsafeVolatileSnapshot.allFiles val partitionColName = deltaLog.unsafeVolatileMetadata.physicalPartitionColumns.head checkAnswer( allFiles.select("partitionValues").orderBy("modificationTime"), Seq( Row(Map(partitionColName -> "2024-06-15 04:00:00")), Row(Map(partitionColName -> "2024-06-16 05:00:00")), Row(Map(partitionColName -> "1903-12-28 05:00:00")) )) } withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC-03:00") { checkAnswer( sql(s"SELECT data FROM $t1 where ts = timestamp'2024-06-15T4:00:00'"), Seq(Row(1))) checkAnswer( sql(s"SELECT data FROM $t1 where ts = timestamp'2024-06-16T05:00:00'"), Seq(Row(2))) checkAnswer( sql(s"SELECT data FROM $t1 where ts = timestamp'1903-12-28T05:00:00'"), Seq(Row(3))) checkAnswer(sql(s"SELECT count(distinct(ts)) from $t1"), Seq(Row(3))) } } } test("insertInto: Timestamp round trips across same session time zone: UTC normalized") { val t1 = "utc_timestamp_partitioned_values" withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "GMT-8") { sql(s"CREATE TABLE $t1 (data int, ts timestamp) USING delta PARTITIONED BY (ts)") sql(s"INSERT INTO $t1 VALUES (1, timestamp'2024-06-15 04:00:00UTC')") sql(s"INSERT INTO $t1 VALUES (2, timestamp'2024-06-15T4:00:00UTC+8')") sql(s"INSERT INTO $t1 VALUES (3, timestamp'2024-06-15T5:00:00 UTC+01:00')") val deltaLog = DeltaLog.forTable( spark, TableIdentifier(t1)) val allFiles = deltaLog.unsafeVolatileSnapshot.allFiles val partitionColName = deltaLog.unsafeVolatileMetadata.physicalPartitionColumns.head checkAnswer( allFiles.select("partitionValues").orderBy("modificationTime"), Seq( Row(Map(partitionColName -> "2024-06-15T04:00:00.000000Z")), Row(Map(partitionColName -> "2024-06-14T20:00:00.000000Z")), Row(Map(partitionColName -> "2024-06-15T04:00:00.000000Z")) )) checkAnswer( sql(s"SELECT data FROM $t1 where ts = timestamp'2024-06-15T04:00:00UTC'"), Seq(Row(1), Row(3))) checkAnswer( sql(s"SELECT data FROM $t1 where ts = timestamp'2024-06-14T20:00:00UTC'"), Seq(Row(2))) checkAnswer(sql(s"SELECT count(distinct(ts)) from $t1"), Seq(Row(2))) } } test("insertInto: Timestamp round trips across same session time zone: session time normalized") { val t1 = "utc_timestamp_partitioned_values" withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") { withSQLConf(DeltaSQLConf.UTC_TIMESTAMP_PARTITION_VALUES.key -> "false") { sql(s"CREATE TABLE $t1 (data int, ts timestamp) USING delta PARTITIONED BY (ts)") sql(s"INSERT INTO $t1 VALUES (1, timestamp'2024-06-15 04:00:00UTC+08:00')") sql(s"INSERT INTO $t1 VALUES (2, timestamp'2024-06-15T4:00:00UTC-08:00')") sql(s"INSERT INTO $t1 VALUES (3, timestamp'2024-06-15T5:00:00UTC+09:00')") val deltaLog = DeltaLog.forTable( spark, TableIdentifier(t1)) val allFiles = deltaLog.unsafeVolatileSnapshot.allFiles val partitionColName = deltaLog.unsafeVolatileMetadata.physicalPartitionColumns.head checkAnswer( allFiles.select("partitionValues").orderBy("modificationTime"), Seq( Row(Map(partitionColName -> "2024-06-14 20:00:00")), Row(Map(partitionColName -> "2024-06-15 12:00:00")), Row(Map(partitionColName -> "2024-06-14 20:00:00")) )) checkAnswer( sql(s"SELECT data FROM $t1 where ts = timestamp'2024-06-14T20:00:00'"), Seq(Row(1), Row(3))) checkAnswer( sql(s"SELECT data FROM $t1 where ts = timestamp'2024-06-15T12:00:00'"), Seq(Row(2))) checkAnswer(sql(s"SELECT count(distinct(ts)) from $t1"), Seq(Row(2))) } } } test("insertInto: timestamp partition values with different precisions") { val t1 = "utc_timestamp_partitioned_values_different_precisions" withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "GMT-8") { sql(s"CREATE TABLE $t1 (data int, ts timestamp) USING delta PARTITIONED BY (ts)") sql(s"INSERT INTO $t1 VALUES (1, timestamp'2025-11-26 04:00:00.1')") sql(s"INSERT INTO $t1 VALUES (2, timestamp'2025-11-26 04:00:00.12')") sql(s"INSERT INTO $t1 VALUES (3, timestamp'2025-11-26 04:00:00.123')") sql(s"INSERT INTO $t1 VALUES (4, timestamp'2025-11-26 04:00:00.1234')") sql(s"INSERT INTO $t1 VALUES (5, timestamp'2025-11-26 04:00:00.12345')") sql(s"INSERT INTO $t1 VALUES (6, timestamp'2025-11-26 04:00:00.123456')") checkAnswer( sql(s"SELECT data FROM $t1 where ts = timestamp'2025-11-26 04:00:00.1'"), Seq(Row(1))) checkAnswer( sql(s"SELECT data FROM $t1 where ts = timestamp'2025-11-26 04:00:00.12'"), Seq(Row(2))) checkAnswer( sql(s"SELECT data FROM $t1 where ts = timestamp'2025-11-26 04:00:00.123'"), Seq(Row(3))) checkAnswer( sql(s"SELECT data FROM $t1 where ts = timestamp'2025-11-26 04:00:00.1234'"), Seq(Row(4))) checkAnswer( sql(s"SELECT data FROM $t1 where ts = timestamp'2025-11-26 04:00:00.12345'"), Seq(Row(5))) checkAnswer( sql(s"SELECT data FROM $t1 where ts = timestamp'2025-11-26 04:00:00.123456'"), Seq(Row(6))) checkAnswer(sql(s"SELECT count(distinct(ts)) from $t1"), Seq(Row(6))) } } // FIXME: Documenting existing behaviour. Fixing this should be a bugfix and not behavior change. test("insertInto: __HIVE_DEFAULT_PARTITION__ results in null partition column") { val t1 = "tbl" withTable(t1) { sql(s"CREATE TABLE $t1 (part string, data string) USING $v2Format PARTITIONED BY (part)") // Insert with __HIVE_DEFAULT_PARTITION__ as partition value // __HIVE_DEFAULT_PARTITION__ is a tombstone value for null partition column sql(s"INSERT INTO $t1 VALUES ('__HIVE_DEFAULT_PARTITION__', 'test')") // Verify that the partition column is null checkAnswer( sql(s"SELECT part, data FROM $t1"), Seq(Row(null, "test")) ) } } // This behavior is specific to Delta testQuietly("insertInto: schema enforcement") { val t1 = "tbl" sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format") val df = Seq(("a", 1L)).toDF("id", "data") // reverse order def getDF(rows: Row*): DataFrame = { spark.createDataFrame(spark.sparkContext.parallelize(rows), spark.table(t1).schema) } withSQLConf(SQLConf.STORE_ASSIGNMENT_POLICY.key -> "strict") { intercept[AnalysisException] { doInsert(t1, df, SaveMode.Overwrite) } verifyTable(t1, Seq.empty[(Long, String)].toDF("id", "data")) intercept[AnalysisException] { doInsert(t1, df) } verifyTable(t1, Seq.empty[(Long, String)].toDF("id", "data")) } withSQLConf(SQLConf.STORE_ASSIGNMENT_POLICY.key -> "ansi") { intercept[SparkException] { doInsert(t1, df, SaveMode.Overwrite) } verifyTable(t1, Seq.empty[(Long, String)].toDF("id", "data")) intercept[SparkException] { doInsert(t1, df) } verifyTable(t1, Seq.empty[(Long, String)].toDF("id", "data")) } withSQLConf(SQLConf.STORE_ASSIGNMENT_POLICY.key -> "legacy") { doInsert(t1, df, SaveMode.Overwrite) verifyTable( t1, getDF(Row(null, "1"))) doInsert(t1, df) verifyTable( t1, getDF(Row(null, "1"), Row(null, "1"))) } } testQuietly("insertInto: struct types and schema enforcement") { val t1 = "tbl" withTable(t1) { sql( s"""CREATE TABLE $t1 ( | id bigint, | point struct |) |USING delta""".stripMargin) val init = Seq((1L, (0.0, 1.0))).toDF("id", "point") doInsert(t1, init) doInsert(t1, Seq((2L, (1.0, 0.0))).toDF("col1", "col2")) // naming doesn't matter // can handle null types doInsert(t1, Seq((3L, (1.0, null))).toDF("col1", "col2")) doInsert(t1, Seq((4L, (null, 1.0))).toDF("col1", "col2")) val expected = Seq( Row(1L, Row(0.0, 1.0)), Row(2L, Row(1.0, 0.0)), Row(3L, Row(1.0, null)), Row(4L, Row(null, 1.0))) verifyTable( t1, spark.createDataFrame(expected.asJava, spark.table(t1).schema)) // schema enforcement val complexSchema = Seq((5L, (0.5, 0.5), (2.5, 2.5, 1.0), "a", (0.5, "b"))) .toDF("long", "struct", "newstruct", "string", "badstruct") .select( $"long", $"struct", struct( $"newstruct._1".as("x"), $"newstruct._2".as("y"), $"newstruct._3".as("z")) as "newstruct", $"string", $"badstruct") // new column in root intercept[AnalysisException] { doInsert(t1, complexSchema.select("long", "struct", "string")) } // new column in struct not accepted intercept[AnalysisException] { doInsert(t1, complexSchema.select("long", "newstruct")) } withSQLConf(SQLConf.STORE_ASSIGNMENT_POLICY.key -> "strict") { // bad data type not accepted intercept[AnalysisException] { doInsert(t1, complexSchema.select("string", "struct")) } // nested bad data type in struct not accepted intercept[AnalysisException] { doInsert(t1, complexSchema.select("long", "badstruct")) } } // missing column in struct intercept[AnalysisException] { doInsert(t1, complexSchema.select($"long", struct(lit(0.1)))) } // wrong ordering intercept[AnalysisException] { doInsert(t1, complexSchema.select("struct", "long")) } // schema evolution withSQLConf( DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true", SQLConf.STORE_ASSIGNMENT_POLICY.key -> "strict") { // ordering should still match intercept[AnalysisException] { doInsert(t1, complexSchema.select("struct", "long")) } intercept[AnalysisException] { doInsert(t1, complexSchema.select("struct", "long", "string")) } // new column to the end works doInsert(t1, complexSchema.select($"long", $"struct", $"string".as("letter"))) // still cannot insert missing column intercept[AnalysisException] { doInsert(t1, complexSchema.select("long", "struct")) } intercept[AnalysisException] { doInsert(t1, complexSchema.select($"long", struct(lit(0.1)), $"string")) } // still perform nested data type checks intercept[AnalysisException] { doInsert(t1, complexSchema.select("long", "badstruct", "string")) } // bad column within struct intercept[AnalysisException] { doInsert(t1, complexSchema.select( $"long", struct(lit(0.1), lit("a"), lit(0.2)), $"string")) } // Add column to nested field doInsert(t1, complexSchema.select($"long", $"newstruct", lit(null))) // cannot insert missing field into struct now intercept[AnalysisException] { doInsert(t1, complexSchema.select("long", "struct", "string")) } } val expected2 = Seq( Row(1L, Row(0.0, 1.0, null), null), Row(2L, Row(1.0, 0.0, null), null), Row(3L, Row(1.0, null, null), null), Row(4L, Row(null, 1.0, null), null), Row(5L, Row(0.5, 0.5, null), "a"), Row(5L, Row(2.5, 2.5, 1.0), null)) verifyTable( t1, spark.createDataFrame(expected2.asJava, spark.table(t1).schema)) val expectedSchema = new StructType() .add("id", LongType) .add("point", new StructType() .add("x", DoubleType) .add("y", DoubleType) .add("z", DoubleType)) .add("letter", StringType) val diff = SchemaUtils.reportDifferences(spark.table(t1).schema, expectedSchema) if (diff.nonEmpty) { fail(diff.mkString("\n")) } } } dynamicOverwriteTest("insertInto: overwrite partitioned table in dynamic mode") { val t1 = "tbl" withTable(t1) { sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format PARTITIONED BY (id)") val init = Seq((2L, "dummy"), (4L, "keep")).toDF("id", "data") doInsert(t1, init) val df = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") doInsert(t1, df, SaveMode.Overwrite) verifyTable(t1, df.union(sql("SELECT 4L, 'keep'"))) } } dynamicOverwriteTest("insertInto: overwrite partitioned table in dynamic mode by position") { val t1 = "tbl" withTable(t1) { sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format PARTITIONED BY (id)") val init = Seq((2L, "dummy"), (4L, "keep")).toDF("id", "data") doInsert(t1, init) val dfr = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("data", "id") doInsert(t1, dfr, SaveMode.Overwrite) val df = Seq((1L, "a"), (2L, "b"), (3L, "c"), (4L, "keep")).toDF("id", "data") verifyTable(t1, df) } } dynamicOverwriteTest( "insertInto: overwrite partitioned table in dynamic mode automatic casting") { val t1 = "tbl" withTable(t1) { sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format PARTITIONED BY (id)") val init = Seq((2L, "dummy"), (4L, "keep")).toDF("id", "data") doInsert(t1, init) val df = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") val dfc = Seq((1, "a"), (2, "b"), (3, "c")).toDF("id", "data") doInsert(t1, df, SaveMode.Overwrite) verifyTable(t1, df.union(sql("SELECT 4L, 'keep'"))) } } dynamicOverwriteTest("insertInto: overwrite fails when missing a column in dynamic mode") { val t1 = "tbl" sql(s"CREATE TABLE $t1 (id bigint, data string, missing string) USING $v2Format") val df1 = Seq((1L, "a"), (2L, "b"), (3L, "c")).toDF("id", "data") // mismatched datatype val df2 = Seq((1, "a"), (2, "b"), (3, "c")).toDF("id", "data") for (df <- Seq(df1, df2)) { val exc = intercept[AnalysisException] { doInsert(t1, df, SaveMode.Overwrite) } verifyTable(t1, Seq.empty[(Long, String, String)].toDF("id", "data", "missing")) assert(exc.getMessage.contains("not enough data columns")) } } test("insert nested struct from view into delta") { withTable("testNestedStruct") { sql(s"CREATE TABLE testNestedStruct " + s" (num INT, text STRING, s STRUCT, b:STRING>)" + s" USING DELTA") val data = sql(s"SELECT 1, 'a', struct('a', struct('c', 'd'), 'b')") doInsert("testNestedStruct", data) verifyTable("testNestedStruct", sql(s"SELECT 1 AS num, 'a' AS text, struct('a', struct('c', 'd') AS s2, 'b') AS s")) } } } trait InsertIntoSQLOnlyTests extends QueryTest with SharedSparkSession with BeforeAndAfter { import testImplicits._ /** Check that the results in `tableName` match the `expected` DataFrame. */ protected def verifyTable(tableName: String, expected: DataFrame): Unit = { checkAnswer(spark.table(tableName), expected) } protected val v2Format: String = "delta" /** * Whether dynamic partition overwrites are supported by the `Table` definitions used in the * test suites. Tables that leverage the V1 Write interface do not support dynamic partition * overwrites. */ protected val supportsDynamicOverwrite: Boolean /** Whether to include the SQL specific tests in this trait within the extending test suite. */ protected val includeSQLOnlyTests: Boolean private def withTableAndData(tableName: String)(testFn: String => Unit): Unit = { withTable(tableName) { val viewName = "tmp_view" val df = spark.createDataFrame(Seq((1L, "a"), (2L, "b"), (3L, "c"))).toDF("id", "data") df.createOrReplaceTempView(viewName) withTempView(viewName) { testFn(viewName) } } } protected def dynamicOverwriteTest(testName: String)(f: => Unit): Unit = { test(testName) { try { withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.DYNAMIC.toString) { f } if (!supportsDynamicOverwrite) { fail("Expected failure from test, because the table doesn't support dynamic overwrites") } } catch { case a: AnalysisException if !supportsDynamicOverwrite => assert(a.getMessage.contains("does not support dynamic overwrite")) } } } if (includeSQLOnlyTests) { test("InsertInto: when the table doesn't exist") { val t1 = "tbl" val t2 = "tbl2" withTableAndData(t1) { _ => sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format") val e = intercept[AnalysisException] { sql(s"INSERT INTO $t2 VALUES (2L, 'dummy')") } assert(e.getMessage.contains(t2)) assert(e.getMessage.contains("Table not found") || e.getMessage.contains(s"table or view `$t2` cannot be found") ) } } test("InsertInto: append to partitioned table - static clause") { val t1 = "tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format PARTITIONED BY (id)") sql(s"INSERT INTO $t1 PARTITION (id = 23) SELECT data FROM $view") verifyTable(t1, sql(s"SELECT 23, data FROM $view")) } } test("InsertInto: static PARTITION clause fails with non-partition column") { val t1 = "tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format PARTITIONED BY (data)") val exc = intercept[AnalysisException] { sql(s"INSERT INTO TABLE $t1 PARTITION (id=1) SELECT data FROM $view") } verifyTable(t1, spark.emptyDataFrame) assert( exc.getMessage.contains("PARTITION clause cannot contain a non-partition column") || exc.getMessage.contains("PARTITION clause cannot contain the non-partition column") || exc.getMessage.contains( "[NON_PARTITION_COLUMN] PARTITION clause cannot contain the non-partition column")) assert(exc.getMessage.contains("id")) } } test("InsertInto: dynamic PARTITION clause fails with non-partition column") { val t1 = "tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format PARTITIONED BY (id)") val exc = intercept[AnalysisException] { sql(s"INSERT INTO TABLE $t1 PARTITION (data) SELECT * FROM $view") } verifyTable(t1, spark.emptyDataFrame) assert( exc.getMessage.contains("PARTITION clause cannot contain a non-partition column") || exc.getMessage.contains("PARTITION clause cannot contain the non-partition column") || exc.getMessage.contains( "[NON_PARTITION_COLUMN] PARTITION clause cannot contain the non-partition column")) assert(exc.getMessage.contains("data")) } } test("InsertInto: overwrite - dynamic clause - static mode") { withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.STATIC.toString) { val t1 = "tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format PARTITIONED BY (id)") sql(s"INSERT INTO $t1 VALUES (2L, 'dummy'), (4L, 'also-deleted')") sql(s"INSERT OVERWRITE TABLE $t1 PARTITION (id) SELECT * FROM $view") verifyTable(t1, Seq( (1, "a"), (2, "b"), (3, "c")).toDF()) } } } dynamicOverwriteTest("InsertInto: overwrite - dynamic clause - dynamic mode") { val t1 = "tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format PARTITIONED BY (id)") sql(s"INSERT INTO $t1 VALUES (2L, 'dummy'), (4L, 'keep')") sql(s"INSERT OVERWRITE TABLE $t1 PARTITION (id) SELECT * FROM $view") verifyTable(t1, Seq( (1, "a"), (2, "b"), (3, "c"), (4, "keep")).toDF("id", "data")) } } test("InsertInto: overwrite - missing clause - static mode") { withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.STATIC.toString) { val t1 = "tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format PARTITIONED BY (id)") sql(s"INSERT INTO $t1 VALUES (2L, 'dummy'), (4L, 'also-deleted')") sql(s"INSERT OVERWRITE TABLE $t1 SELECT * FROM $view") verifyTable(t1, Seq( (1, "a"), (2, "b"), (3, "c")).toDF("id", "data")) } } } dynamicOverwriteTest("InsertInto: overwrite - missing clause - dynamic mode") { val t1 = "tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format PARTITIONED BY (id)") sql(s"INSERT INTO $t1 VALUES (2L, 'dummy'), (4L, 'keep')") sql(s"INSERT OVERWRITE TABLE $t1 SELECT * FROM $view") verifyTable(t1, Seq( (1, "a"), (2, "b"), (3, "c"), (4, "keep")).toDF("id", "data")) } } test("InsertInto: overwrite - static clause") { val t1 = "tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string, p1 int) " + s"USING $v2Format PARTITIONED BY (p1)") sql(s"INSERT INTO $t1 VALUES (2L, 'dummy', 23), (4L, 'keep', 2)") verifyTable(t1, Seq( (2L, "dummy", 23), (4L, "keep", 2)).toDF("id", "data", "p1")) sql(s"INSERT OVERWRITE TABLE $t1 PARTITION (p1 = 23) SELECT * FROM $view") verifyTable(t1, Seq( (1, "a", 23), (2, "b", 23), (3, "c", 23), (4, "keep", 2)).toDF("id", "data", "p1")) } } test("InsertInto: overwrite - mixed clause - static mode") { withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.STATIC.toString) { val t1 = "tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string, p int) " + s"USING $v2Format PARTITIONED BY (id, p)") sql(s"INSERT INTO $t1 VALUES (2L, 'dummy', 2), (4L, 'also-deleted', 2)") sql(s"INSERT OVERWRITE TABLE $t1 PARTITION (id, p = 2) SELECT * FROM $view") verifyTable(t1, Seq( (1, "a", 2), (2, "b", 2), (3, "c", 2)).toDF("id", "data", "p")) } } } test("InsertInto: overwrite - mixed clause reordered - static mode") { withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.STATIC.toString) { val t1 = "tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string, p int) " + s"USING $v2Format PARTITIONED BY (id, p)") sql(s"INSERT INTO $t1 VALUES (2L, 'dummy', 2), (4L, 'also-deleted', 2)") sql(s"INSERT OVERWRITE TABLE $t1 PARTITION (p = 2, id) SELECT * FROM $view") verifyTable(t1, Seq( (1, "a", 2), (2, "b", 2), (3, "c", 2)).toDF("id", "data", "p")) } } } test("InsertInto: overwrite - implicit dynamic partition - static mode") { withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.STATIC.toString) { val t1 = "tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string, p int) " + s"USING $v2Format PARTITIONED BY (id, p)") sql(s"INSERT INTO $t1 VALUES (2L, 'dummy', 2), (4L, 'also-deleted', 2)") sql(s"INSERT OVERWRITE TABLE $t1 PARTITION (p = 2) SELECT * FROM $view") verifyTable(t1, Seq( (1, "a", 2), (2, "b", 2), (3, "c", 2)).toDF("id", "data", "p")) } } } dynamicOverwriteTest("InsertInto: overwrite - mixed clause - dynamic mode") { val t1 = "tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string, p int) " + s"USING $v2Format PARTITIONED BY (id, p)") sql(s"INSERT INTO $t1 VALUES (2L, 'dummy', 2), (4L, 'keep', 2)") sql(s"INSERT OVERWRITE TABLE $t1 PARTITION (p = 2, id) SELECT * FROM $view") verifyTable(t1, Seq( (1, "a", 2), (2, "b", 2), (3, "c", 2), (4, "keep", 2)).toDF("id", "data", "p")) } } dynamicOverwriteTest("InsertInto: overwrite - mixed clause reordered - dynamic mode") { val t1 = "tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string, p int) " + s"USING $v2Format PARTITIONED BY (id, p)") sql(s"INSERT INTO $t1 VALUES (2L, 'dummy', 2), (4L, 'keep', 2)") sql(s"INSERT OVERWRITE TABLE $t1 PARTITION (id, p = 2) SELECT * FROM $view") verifyTable(t1, Seq( (1, "a", 2), (2, "b", 2), (3, "c", 2), (4, "keep", 2)).toDF("id", "data", "p")) } } dynamicOverwriteTest("InsertInto: overwrite - implicit dynamic partition - dynamic mode") { val t1 = "tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string, p int) " + s"USING $v2Format PARTITIONED BY (id, p)") sql(s"INSERT INTO $t1 VALUES (2L, 'dummy', 2), (4L, 'keep', 2)") sql(s"INSERT OVERWRITE TABLE $t1 PARTITION (p = 2) SELECT * FROM $view") verifyTable(t1, Seq( (1, "a", 2), (2, "b", 2), (3, "c", 2), (4, "keep", 2)).toDF("id", "data", "p")) } } test("insert nested struct literal into delta") { withTable("insertNestedTest") { sql(s"CREATE TABLE insertNestedTest " + s" (num INT, text STRING, s STRUCT, b:STRING>)" + s" USING DELTA") sql(s"INSERT INTO insertNestedTest VALUES (1, 'a', struct('a', struct('c', 'd'), 'b'))") } } dynamicOverwriteTest("InsertInto: overwrite - multiple static partitions - dynamic mode") { val t1 = "tbl" withTableAndData(t1) { view => sql(s"CREATE TABLE $t1 (id bigint, data string, p int) " + s"USING $v2Format PARTITIONED BY (id, p)") sql(s"INSERT INTO $t1 VALUES (2L, 'dummy', 2), (4L, 'keep', 2)") sql(s"INSERT OVERWRITE TABLE $t1 PARTITION (id = 2, p = 2) SELECT data FROM $view") verifyTable(t1, Seq( (2, "a", 2), (2, "b", 2), (2, "c", 2), (4, "keep", 2)).toDF("id", "data", "p")) } } test("InsertInto: overwrite - dot in column names - static mode") { import testImplicits._ val t1 = "tbl" withTable(t1) { sql(s"CREATE TABLE $t1 (`a.b` string, `c.d` string) USING $v2Format PARTITIONED BY (`a.b`)") sql(s"INSERT OVERWRITE $t1 PARTITION (`a.b` = 'a') VALUES('b')") verifyTable(t1, Seq("a" -> "b").toDF("id", "data")) } } } // END Apache Spark tests } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoTest.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.collection.mutable import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.hadoop.fs.Path import org.apache.spark.{DebugFilesystem, SparkThrowable} import org.apache.spark.sql.{DataFrame, QueryTest, SaveMode} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.util.QuotingUtils import org.apache.spark.sql.functions.{col, lit} import org.apache.spark.sql.streaming.{StreamingQueryException, Trigger} import org.apache.spark.sql.types.StructType /** * There are **many** different ways to run an insert: * - Using SQL, the dataframe v1 and v2 APIs or the streaming API. * - Append vs. Overwrite / Partition overwrite. * - Position-based vs. name-based resolution. * * Each take a unique path through analysis. The abstractions below captures these different * inserts to allow more easily running tests with all or a subset of them. */ trait DeltaInsertIntoTest extends QueryTest with DeltaDMLTestUtilsPathBased with DeltaSQLCommandTest { val catalogName = "spark_catalog" /** * Represents one way of inserting data into a Delta table. * @param name A human-readable name for the insert type displayed in the test names. * @param mode Append or Overwrite. This dictates in particular what the expected result after the * insert should be. * @param byName Whether the insert uses name-based resolution or position-based resolution. * @param isSQL Whether the insert is done using SQL or the dataframe API (includes streaming * write). */ trait Insert { val name: String val mode: SaveMode val byName: Boolean val isSQL: Boolean /** * The method that tests will call to run the insert. Each type of insert must implement its * specific way to run insert. */ def runInsert( columns: Seq[String], whereCol: String, whereValue: Int, withSchemaEvolution: Boolean): Unit /** SQL keyword for this type of insert. */ def intoOrOverwrite: String = if (mode == SaveMode.Append) "INTO" else "OVERWRITE" /** The expected content of the table after the insert. */ def expectedResult(initialDF: DataFrame, insertedDF: DataFrame): DataFrame = { // Always union with the initial data even if we're overwriting it to ensure the resulting // schema contains all columns from the table in case some are missing in `insertedDF`. val initial = if (mode == SaveMode.Overwrite) initialDF.limit(0) else initialDF initial.unionByName(insertedDF, allowMissingColumns = true) } } /** INSERT INTO/OVERWRITE */ case class SQLInsertByPosition(mode: SaveMode) extends Insert { val name: String = s"INSERT $intoOrOverwrite" val byName: Boolean = false val isSQL: Boolean = true def runInsert( columns: Seq[String], whereCol: String, whereValue: Int, withSchemaEvolution: Boolean): Unit = { withSQLConf(DeltaSQLConf. DELTA_SCHEMA_AUTO_MIGRATE.key -> withSchemaEvolution.toString) { sql(s"INSERT $intoOrOverwrite target SELECT * FROM source") } } } /** INSERT INTO/OVERWRITE (a, b) */ case class SQLInsertColList(mode: SaveMode) extends Insert { val name: String = s"INSERT $intoOrOverwrite (columns) - $mode" val byName: Boolean = true val isSQL: Boolean = true def runInsert( columns: Seq[String], whereCol: String, whereValue: Int, withSchemaEvolution: Boolean): Unit = { val colList = columns.mkString(", ") withSQLConf(DeltaSQLConf. DELTA_SCHEMA_AUTO_MIGRATE.key -> withSchemaEvolution.toString) { sql(s"INSERT $intoOrOverwrite target ($colList) SELECT $colList FROM source") } } } /** INSERT INTO/OVERWRITE BY NAME */ case class SQLInsertByName(mode: SaveMode) extends Insert { val name: String = s"INSERT $intoOrOverwrite BY NAME - $mode" val byName: Boolean = true val isSQL: Boolean = true def runInsert( columns: Seq[String], whereCol: String, whereValue: Int, withSchemaEvolution: Boolean): Unit = { withSQLConf(DeltaSQLConf. DELTA_SCHEMA_AUTO_MIGRATE.key -> withSchemaEvolution.toString) { sql(s"INSERT $intoOrOverwrite target BY NAME " + s"SELECT ${columns.mkString(", ")} FROM source") } } } /** INSERT INTO REPLACE WHERE */ object SQLInsertOverwriteReplaceWhere extends Insert { val name: String = s"INSERT INTO REPLACE WHERE" val mode: SaveMode = SaveMode.Overwrite val byName: Boolean = false val isSQL: Boolean = true def runInsert( columns: Seq[String], whereCol: String, whereValue: Int, withSchemaEvolution: Boolean): Unit = { withSQLConf(DeltaSQLConf. DELTA_SCHEMA_AUTO_MIGRATE.key -> withSchemaEvolution.toString) { sql(s"INSERT INTO target REPLACE WHERE $whereCol = $whereValue " + s"SELECT ${columns.mkString(", ")} FROM source") } } } /** INSERT OVERWRITE PARTITION (part = 1) */ object SQLInsertOverwritePartitionByPosition extends Insert { val name: String = s"INSERT OVERWRITE PARTITION (partition)" val mode: SaveMode = SaveMode.Overwrite val byName: Boolean = false val isSQL: Boolean = true def runInsert( columns: Seq[String], whereCol: String, whereValue: Int, withSchemaEvolution: Boolean): Unit = { val assignments = columns.filterNot(_ == whereCol).mkString(", ") withSQLConf(DeltaSQLConf. DELTA_SCHEMA_AUTO_MIGRATE.key -> withSchemaEvolution.toString) { sql(s"INSERT OVERWRITE target PARTITION ($whereCol = $whereValue) " + s"SELECT $assignments FROM source") } } } /** INSERT OVERWRITE PARTITION (part = 1) (a, b) */ object SQLInsertOverwritePartitionColList extends Insert { val name: String = s"INSERT OVERWRITE PARTITION (partition) (columns)" val mode: SaveMode = SaveMode.Overwrite val byName: Boolean = true val isSQL: Boolean = true def runInsert( columns: Seq[String], whereCol: String, whereValue: Int, withSchemaEvolution: Boolean): Unit = { val assignments = columns.filterNot(_ == whereCol).mkString(", ") withSQLConf(DeltaSQLConf. DELTA_SCHEMA_AUTO_MIGRATE.key -> withSchemaEvolution.toString) { sql(s"INSERT OVERWRITE target " + s"PARTITION ($whereCol = $whereValue) ($assignments) " + s"SELECT $assignments FROM source") } } } /** df.write.mode(mode).insertInto() */ case class DFv1InsertInto(mode: SaveMode) extends Insert { val name: String = s"DFv1 insertInto() - $mode" val byName: Boolean = false val isSQL: Boolean = false def runInsert( columns: Seq[String], whereCol: String, whereValue: Int, withSchemaEvolution: Boolean): Unit = spark.read.table("source").write.mode(mode) .option("mergeSchema", withSchemaEvolution.toString) .format("delta") .insertInto("target") } /** df.write.mode(mode).saveAsTable() */ case class DFv1SaveAsTable(mode: SaveMode) extends Insert { val name: String = s"DFv1 saveAsTable() - $mode" val byName: Boolean = true val isSQL: Boolean = false def runInsert( columns: Seq[String], whereCol: String, whereValue: Int, withSchemaEvolution: Boolean): Unit = { spark.read.table("source").write.mode(mode) .option("mergeSchema", withSchemaEvolution.toString) .format("delta") .saveAsTable("target") } } /** df.write.mode(mode).save() */ case class DFv1Save(mode: SaveMode) extends Insert { val name: String = s"DFv1 save() - $mode" val byName: Boolean = true val isSQL: Boolean = false def runInsert( columns: Seq[String], whereCol: String, whereValue: Int, withSchemaEvolution: Boolean): Unit = { val deltaLog = DeltaLog.forTable(spark, TableIdentifier("target")) spark.read.table("source").write.mode(mode) .option("mergeSchema", withSchemaEvolution.toString) .format("delta") .save(deltaLog.dataPath.toString) } } /** df.write.mode(mode).option("partitionOverwriteMode", "dynamic").insertInto() */ object DFv1InsertIntoDynamicPartitionOverwrite extends Insert { val name: String = s"DFv1 insertInto() - dynamic partition overwrite" val mode: SaveMode = SaveMode.Overwrite val byName: Boolean = false val isSQL: Boolean = false def runInsert( columns: Seq[String], whereCol: String, whereValue: Int, withSchemaEvolution: Boolean): Unit = spark.read.table("source").write .mode(mode) .option("partitionOverwriteMode", "dynamic") .option("mergeSchema", withSchemaEvolution.toString) .format("delta") .insertInto("target") } /** df.writeTo.append() */ object DFv2Append extends Insert { self: Insert => val name: String = "DFv2 append()" val mode: SaveMode = SaveMode.Append val byName: Boolean = true val isSQL: Boolean = false def runInsert( columns: Seq[String], whereCol: String, whereValue: Int, withSchemaEvolution: Boolean): Unit = { spark.read.table("source") .writeTo("target") .option("mergeSchema", withSchemaEvolution.toString) .append() } } /** df.writeTo.overwrite() */ object DFv2Overwrite extends Insert { self: Insert => val name: String = s"DFv2 overwrite()" val mode: SaveMode = SaveMode.Overwrite val byName: Boolean = true val isSQL: Boolean = false def runInsert( columns: Seq[String], whereCol: String, whereValue: Int, withSchemaEvolution: Boolean): Unit = { spark.read.table("source") .writeTo("target") .option("mergeSchema", withSchemaEvolution.toString) .overwrite(col(whereCol) === lit(whereValue)) } } /** df.writeTo.overwritePartitions() */ object DFv2OverwritePartition extends Insert { self: Insert => val name: String = s"DFv2 overwritePartitions()" override val mode: SaveMode = SaveMode.Overwrite val byName: Boolean = true val isSQL: Boolean = false def runInsert( columns: Seq[String], whereCol: String, whereValue: Int, withSchemaEvolution: Boolean): Unit = { spark.read.table("source") .writeTo("target") .option("mergeSchema", withSchemaEvolution.toString) .overwritePartitions() } } /** df.writeStream.toTable() */ object StreamingInsert extends Insert { self: Insert => val name: String = s"Streaming toTable()" override val mode: SaveMode = SaveMode.Append val byName: Boolean = true val isSQL: Boolean = false def runInsert( columns: Seq[String], whereCol: String, whereValue: Int, withSchemaEvolution: Boolean): Unit = { val tablePath = DeltaLog.forTable(spark, TableIdentifier("target")).dataPath val checkpointLocation = new Path(tablePath, "_checkpoint") val query = spark.readStream .table("source") .writeStream .option("checkpointLocation", checkpointLocation.toString) .option("mergeSchema", withSchemaEvolution.toString) .format("delta") .trigger(Trigger.AvailableNow()) .toTable("target") query.processAllAvailable() } } /** Collects all the types of insert previously defined. */ protected lazy val allInsertTypes: Set[Insert] = Set( SQLInsertOverwriteReplaceWhere, SQLInsertOverwritePartitionByPosition, SQLInsertOverwritePartitionColList, DFv1InsertIntoDynamicPartitionOverwrite, DFv2Append, DFv2Overwrite, DFv2OverwritePartition, StreamingInsert ) ++ (for { mode: SaveMode <- Seq(SaveMode.Append, SaveMode.Overwrite) insert: Insert <- Seq( SQLInsertByPosition(mode), SQLInsertColList(mode), SQLInsertByName(mode), DFv1InsertInto(mode), DFv1SaveAsTable(mode), DFv1Save(mode) ) } yield insert).toSet /** Collects inserts using resolution by name and by position respectively. */ protected lazy val (insertsByName, insertsByPosition): (Set[Insert], Set[Insert]) = allInsertTypes.partition(_.byName) /** Collects inserts run through SQL and the dataframe API respectively. */ protected lazy val (insertsSQL, insertsDataframe): (Set[Insert], Set[Insert]) = allInsertTypes.partition(_.isSQL) /** Collects append inserts vs. overwrite. */ protected lazy val (insertsAppend, insertsOverwrite): (Set[Insert], Set[Insert]) = allInsertTypes.partition(_.mode == SaveMode.Append) /** Collects all test cases defined, aggregated by test name. Used in * [[checkAllTestCasesImplemented]] below to ensure each test covers all existing insert types. */ protected val testCases: mutable.Map[String, Set[Insert]] = mutable.HashMap.empty.withDefaultValue(Set.empty) /** Tests should cover all insert types but it's easy to miss some cases. This method checks * that each test cover all insert types. */ def checkAllTestCasesImplemented(ignoredTestCases: Map[String, Set[Insert]] = Map.empty): Unit = { val ignoredTests = ignoredTestCases.withDefaultValue(Set.empty) val missingTests = testCases.map { case (name, inserts) => name -> (allInsertTypes -- inserts -- ignoredTests(name)) }.collect { case (name, missingInserts) if missingInserts.nonEmpty => s"Test '$name' is not covering all insert types, missing: $missingInserts" } if (missingTests.nonEmpty) { fail("Missing test cases:\n" + missingTests) } } /** Convenience wrapper define test data using a SQL schema and a JSON string for each row. */ case class TestData(schemaDDL: String, data: Seq[String]) { val schema: StructType = StructType.fromDDL(schemaDDL) def toDF: DataFrame = readFromJSON(data, schema) } /** * Test runner to cover INSERT operations defined above. * @param name Test name * @param initialData Initial data used to create the table. * @param partitionBy Partition columns for the initial table. * @param insertData Additional data to be inserted. * @param overwriteWhere Where clause for overwrite PARTITION / REPLACE WHERE (as * colName -> value) * @param expectedResult Expected result, see [[ExpectedResult]] above. * @param includeInserts List of insert types to run the test with. * Defaults to all inserts. * @param excludeInserts List of insert types to exclude when running the test. * Defaults to no inserts excluded. * @param confs Custom spark confs to set before running the insert * operation. * @param withSchemaEvolution Whether to enable Automatic Schema Evolution. */ def testInserts[T](name: String)( initialData: TestData, partitionBy: Seq[String] = Seq.empty, insertData: TestData, overwriteWhere: (String, Int), expectedResult: ExpectedResult[T], includeInserts: Set[Insert] = allInsertTypes, excludeInserts: Set[Insert] = Set.empty, confs: Seq[(String, String)] = Seq.empty, withSchemaEvolution: Boolean = false): Unit = { val inserts = includeInserts.filterNot(excludeInserts) assert(inserts.nonEmpty, s"Test '$name' doesn't cover any inserts. Please check the " + "includeInserts/excludeInserts sets and ensure at least one insert is included.") testCases(name) ++= inserts for (insert <- inserts) { test(s"${insert.name} - $name") { withTable("source", "target") { val writer = initialData.toDF.write.format("delta") if (partitionBy.nonEmpty) { writer.partitionBy(partitionBy: _*) } writer.saveAsTable("target") // Write the data to insert to a table so that we can use it in both SQL and dataframe // writer inserts. insertData.toDF.write.format("delta").saveAsTable("source") def runInsert(): Unit = insert.runInsert( columns = insertData.schema.map(f => QuotingUtils.quoteIfNeeded(f.name)), whereCol = overwriteWhere._1, whereValue = overwriteWhere._2, withSchemaEvolution = withSchemaEvolution ) withSQLConf(confs: _*) { expectedResult match { case ExpectedResult.Success(expectedSchema: StructType) => runInsert() val target = spark.read.table("target") assert(target.schema === expectedSchema) checkAnswer(target, insert.expectedResult(initialData.toDF, insertData.toDF)) case ExpectedResult.Success(expectedData: TestData) => runInsert() val target = spark.read.table("target") assert(target.schema === expectedData.schema) checkAnswer(spark.read.table("target"), expectedData.toDF) case ExpectedResult.Failure(checkError) => val ex = if (insert == StreamingInsert) { intercept[StreamingQueryException] { runInsert() }.getCause.asInstanceOf[SparkThrowable] } else { intercept[SparkThrowable] { runInsert() } } checkError(ex) } } } } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaLimitPushDownSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import com.databricks.spark.util.DatabricksLogging import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.StatsUtils import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, ScanReportHelper} import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.sql.{DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.col import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.Utils trait DeltaLimitPushDownTests extends QueryTest with SharedSparkSession with DatabricksLogging with ScanReportHelper with DeletionVectorsTestUtils with StatsUtils with DeltaSQLCommandTest with CatalogOwnedTestBaseSuite { import testImplicits._ test("no filter or projection") { withTempTable(createTable = false) { tableName => val ds = Seq(1, 1, 2, 2, 3, 3).toDS().repartition(5, $"value") ds.write.format("delta").saveAsTable(tableName) val Seq(deltaScan, deltaScanWithLimit) = getScanReport { spark.read.format("delta").table(tableName).collect() val res = spark.read.format("delta").table(tableName).limit(3).collect() assert(res.size == 3) } assert(deltaScan.size("total").bytesCompressed === deltaScanWithLimit.size("total").bytesCompressed) assert(deltaScan.size("scanned").bytesCompressed != deltaScanWithLimit.size("scanned").bytesCompressed) assert(deltaScanWithLimit.size("scanned").rows === Some(4L)) } } test("limit larger than total") { withTempTable(createTable = false) { tableName => val data = Seq(1, 1, 2, 2) val ds = data.toDS().repartition($"value") ds.write.format("delta").saveAsTable(tableName) val Seq(deltaScan, deltaScanWithLimit) = getScanReport { spark.read.format("delta").table(tableName).collect() checkAnswer(spark.read.format("delta").table(tableName).limit(5), data.toDF()) } assert(deltaScan.size("total").bytesCompressed === deltaScanWithLimit.size("total").bytesCompressed) assert(deltaScan.size("scanned").bytesCompressed === deltaScanWithLimit.size("scanned").bytesCompressed) } } test("limit 0") { val records = getScanReport { withTempTable(createTable = false) { tableName => val ds = Seq(1, 1, 2, 2, 3, 3).toDS().repartition($"value") ds.write.format("delta").saveAsTable(tableName) val res = spark.read.format("delta") .table(tableName) .limit(0) checkAnswer(res, Seq()) } } } test("insufficient rows have stats") { withSQLConf(DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey -> "false") { withTempTable(createTable = false) { tableName => val file = Seq(1, 2).toDS().coalesce(1) withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false") { file.write.format("delta").mode("append").saveAsTable(tableName) } withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false") { file.write.format("delta").mode("append").saveAsTable(tableName) } withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "true") { file.write.format("delta").mode("append").saveAsTable(tableName) } val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val deltaScan = deltaLog.snapshot.filesForScan(limit = 3, partitionFilters = Seq.empty) assert(deltaScan.scanned.bytesCompressed === deltaScan.total.bytesCompressed) } } } test("sufficient rows have stats") { withSQLConf(DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey -> "false") { withTempTable(createTable = false) { tableName => val file = Seq(1, 2).toDS().coalesce(1) withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false") { file.write.format("delta").mode("append").saveAsTable(tableName) } withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "true") { file.write.format("delta").mode("append").saveAsTable(tableName) } withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "true") { file.write.format("delta").mode("append").saveAsTable(tableName) } val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val deltaScan = deltaLog.snapshot.filesForScan(limit = 3, partitionFilters = Seq.empty) assert(deltaScan.scanned.rows === Some(4)) assert(deltaScan.scanned.bytesCompressed != deltaScan.total.bytesCompressed) } } } test("with projection only") { withTempTable(createTable = false) { tableName => val ds = Seq((1, 1), (2, 1), (3, 1)).toDF("key", "value").as[(Int, Int)] ds.write.format("delta").partitionBy("key").saveAsTable(tableName) val Seq(deltaScan) = getScanReport { val res = spark.read.format("delta").table(tableName).select("value").limit(1).collect() assert(res === Seq(Row(1))) } assert(deltaScan.size("scanned").rows === Some(1L)) } } test("with partition filter only") { withTempTable(createTable = false) { tableName => val ds = Seq((1, 4), (2, 5), (3, 6)).toDF("key", "value").as[(Int, Int)] ds.write.format("delta").partitionBy("key").saveAsTable(tableName) val Seq(deltaScan, deltaScanWithLimit, deltaScanWithLimit2) = getScanReport { spark.read.format("delta").table(tableName).where("key > 1").collect() val res1 = spark.read.format("delta").table(tableName).where("key > 1").limit(1).collect() assert(res1 === Seq(Row(2, 5)) || res1 === Seq(Row(3, 6))) val res2 = spark.read.format("delta").table(tableName).where("key == 1").limit(2).collect() assert(res2 === Seq(Row(1, 4))) } assert(deltaScan.size("total").bytesCompressed === deltaScanWithLimit.size("total").bytesCompressed) assert(deltaScan.size("scanned").bytesCompressed != deltaScanWithLimit.size("scanned").bytesCompressed) assert(deltaScan.size("scanned").bytesCompressed.get < deltaScan.size("total").bytesCompressed.get) assert(deltaScanWithLimit.size("scanned").rows === Some(1L)) assert(deltaScanWithLimit2.size("scanned").rows === Some(1L)) } } test("with non-partition filter") { withTempTable(createTable = false) { tableName => val ds = Seq((1, 4), (2, 5), (3, 6)).toDF("key", "value").as[(Int, Int)] ds.write.format("delta").partitionBy("key").saveAsTable(tableName) val Seq(deltaScan) = getScanReport { // this query should not trigger limit push-down spark.read.format("delta").table(tableName) .where("key > 1") .where("value > 4") .limit(1) .collect() } assert(deltaScan.size("scanned").rows === Some(2L)) } } test("limit push-down flag") { withTempTable(createTable = false) { tableName => val ds = Seq((1, 4), (2, 5), (3, 6)).toDF("key", "value").as[(Int, Int)] ds.write.format("delta").partitionBy("key").saveAsTable(tableName) val Seq(baseline, scan, scan2) = getScanReport { withSQLConf(DeltaSQLConf.DELTA_LIMIT_PUSHDOWN_ENABLED.key -> "true") { spark.read.format("delta").table(tableName).where("key > 1").limit(1).collect() } withSQLConf(DeltaSQLConf.DELTA_LIMIT_PUSHDOWN_ENABLED.key -> "false") { spark.read.format("delta").table(tableName).where("key > 1").limit(1).collect() spark.read.format("delta").table(tableName).limit(2).collect() } } assert(scan.size("scanned").bytesCompressed.get > baseline.size("scanned").bytesCompressed.get) assert(scan2.size("scanned").bytesCompressed === scan2.size("total").bytesCompressed) } } test("GlobalLimit should be kept") { withTempTable(createTable = false) { tableName => (1 to 10).toDF.repartition(5).write.format("delta").saveAsTable(tableName) assert(spark.read.format("delta").table(tableName).limit(5).collect().size == 5) } } test("Works with union") { withTempTable(createTable = false) { tableName => (1 to 10).toDF.repartition(5).write.format("delta").saveAsTable(tableName) val t1 = spark.read.format("delta").table(tableName) val t2 = spark.read.format("delta").table(tableName) val union = t1.union(t2) withSQLConf(DeltaSQLConf.DELTA_LIMIT_PUSHDOWN_ENABLED.key -> "true") { val Seq(scanFull1, scanFull2) = getScanReport { union.collect() } val Seq(scanLimit1, scanLimit2) = getScanReport { union.limit(1).collect() } assert(scanFull1.size("scanned").bytesCompressed.get > scanLimit1.size("scanned").bytesCompressed.get) assert(scanFull2.size("scanned").bytesCompressed.get > scanLimit2.size("scanned").bytesCompressed.get) } } } private def withDVSettings(thunk: => Unit): Unit = { withSQLConf( DeltaSQLConf.DELTA_OPTIMIZE_METADATA_QUERY_ENABLED.key -> "false" ) { withDeletionVectorsEnabled() { thunk } } } test(s"Verify limit correctness in the presence of DVs") { withDVSettings { val targetDF = spark.range(start = 0, end = 100, step = 1, numPartitions = 2) .withColumn("value", col("id")) withTempDeltaTable(targetDF, createNameBasedTable = true) { (targetTable, targetLog) => removeRowsFromAllFilesInLog(targetLog, numRowsToRemovePerFile = 10) verifyDVsExist(targetLog, 2) val targetDF = targetTable().toDF // We have 2 files 50 rows each. We deleted 10 rows from the first file. The first file // now contains 50 physical rows and 40 logical. Failing to take into account the DVs in // the first file results into prematurely terminating the scan and returning an // incorrect result. Note, the corner case in terms of correctness is when the limit is // set to 50. When statistics collection is disabled, we read both files. val limitToExpectedNumberOfFilesReadSeq = Range(10, 90, 10) .map(n => (n, if (n < 50) 1 else 2)) for ((limit, expectedNumberOfFilesRead) <- limitToExpectedNumberOfFilesReadSeq) { val df = targetDF.limit(limit) // Assess correctness. assert(df.count === limit) val scanStats = getStats(df) // Check we do not read more files than needed. assert(scanStats.scanned.files === Some(expectedNumberOfFilesRead)) // Verify physical and logical rows are updated correctly. val numDeletedRows = 10 val numPhysicalRowsPerFile = 50 val numTotalPhysicalRows = numPhysicalRowsPerFile * expectedNumberOfFilesRead val numTotalLogicalRows = numTotalPhysicalRows - (numDeletedRows * expectedNumberOfFilesRead) val expectedNumTotalPhysicalRows = Some(numTotalPhysicalRows) val expectedNumTotalLogicalRows = Some(numTotalLogicalRows) assert(scanStats.scanned.rows === expectedNumTotalPhysicalRows) assert(scanStats.scanned.logicalRows === expectedNumTotalLogicalRows) } } } } } class DeltaLimitPushDownV1Suite extends DeltaLimitPushDownTests class DeltaLimitPushDownWithCatalogOwnedBatch1Suite extends DeltaLimitPushDownTests { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class DeltaLimitPushDownWithCatalogOwnedBatch2Suite extends DeltaLimitPushDownTests { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class DeltaLimitPushDownWithCatalogOwnedBatch100Suite extends DeltaLimitPushDownTests { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaLogMinorCompactionSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.DeltaOperations.ManualUpdate import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CatalogOwnedTestBaseSuite} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.{DeltaCommitFileProvider, FileNames, JsonUtils} import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.sql._ import org.apache.spark.sql.functions.col import org.apache.spark.sql.test.SharedSparkSession // scalastyle:off: removeFile class DeltaLogMinorCompactionSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with DeltaSQLTestUtils with CatalogOwnedTestBaseSuite { /** Helper method to do minor compaction of [[DeltaLog]] from [startVersion, endVersion] */ private def minorCompactDeltaLog( tablePath: String, startVersion: Long, endVersion: Long): Unit = { val deltaLog = DeltaLog.forTable(spark, tablePath) val logReplay = new InMemoryLogReplay( minFileRetentionTimestamp = None, minSetTransactionRetentionTimestamp = None) val hadoopConf = deltaLog.newDeltaHadoopConf() (startVersion to endVersion).foreach { versionToRead => val file = FileNames.unsafeDeltaFile(deltaLog.logPath, versionToRead) val actionsIterator = deltaLog.store.readAsIterator(file, hadoopConf).map(Action.fromJson) logReplay.append(versionToRead, actionsIterator) } deltaLog.store.write( path = FileNames.compactedDeltaFile(deltaLog.logPath, startVersion, endVersion), actions = logReplay.checkpoint.map(_.json).toIterator, overwrite = true, hadoopConf = hadoopConf) } // Helper method to validate a commit. protected def validateCommit( log: DeltaLog, version: Long, numAdds: Int = 0, numRemoves: Int = 0, numMetadata: Int = 0): Unit = { assert(log.update().version === version) val filePath = DeltaCommitFileProvider(log.update()).deltaFile(version) val actions = log.store.read(filePath, log.newDeltaHadoopConf()).map(Action.fromJson) assert(actions.head.isInstanceOf[CommitInfo]) assert(actions.tail.count(_.isInstanceOf[AddFile]) === numAdds) assert(actions.tail.count(_.isInstanceOf[RemoveFile]) === numRemoves) assert(actions.tail.count(_.isInstanceOf[Metadata]) === numMetadata) } // Helper method to validate a compacted delta. private def validateCompactedDelta( log: DeltaLog, filePath: Path, expectedCompactedDelta: CompactedDelta): Unit = { val actions = log.store.read(filePath, log.newDeltaHadoopConf()).map(Action.fromJson) val observedCompactedDelta = CompactedDelta( versionWindow = FileNames.compactedDeltaVersions(filePath), numAdds = actions.count(_.isInstanceOf[AddFile]), numRemoves = actions.count(_.isInstanceOf[RemoveFile]), numMetadata = actions.count(_.isInstanceOf[Metadata]) ) assert(expectedCompactedDelta === observedCompactedDelta) } case class CompactedDelta( versionWindow: (Long, Long), numAdds: Int = 0, numRemoves: Int = 0, numMetadata: Int = 0) def createTestAddFile( path: String = "foo", partitionValues: Map[String, String] = Map.empty, size: Long = 1L, modificationTime: Long = 1L, dataChange: Boolean = true, stats: String = "{\"numRecords\": 1}"): AddFile = { AddFile(path, partitionValues, size, modificationTime, dataChange, stats) } def generateData(tableDir: String, checkpoints: Set[Int]): Unit = { val files = (1 to 21).map( index => createTestAddFile(s"f${index}")) // commit version 0 - AddFile: 4 val deltaLog = DeltaLog.forTable(spark, tableDir) import org.apache.spark.sql.delta.test.DeltaTestImplicits._ val metadata = Metadata() val tableMetadata = metadata.copy( configuration = DeltaConfigs.mergeGlobalConfigs(conf, metadata.configuration)) deltaLog.startTransaction().commitManually( files(1), files(2), files(3), files(4), tableMetadata) validateCommit(deltaLog, version = 0, numAdds = 4, numRemoves = 0, numMetadata = 1) if (checkpoints.contains(0)) deltaLog.checkpoint() // commit version 1 - AddFile: 1 deltaLog.startTransaction().commit(files(5) :: Nil, ManualUpdate) validateCommit(deltaLog, version = 1, numAdds = 1, numRemoves = 0) if (checkpoints.contains(1)) deltaLog.checkpoint() // commit version 2 - RemoveFile: 1, AddFile: 1 deltaLog.startTransaction().commit(Seq(files(5).remove, files(6)), ManualUpdate) validateCommit(deltaLog, version = 2, numAdds = 1, numRemoves = 1) if (checkpoints.contains(2)) deltaLog.checkpoint() // commit version 3 - empty commit deltaLog.startTransaction().commit(Seq(), ManualUpdate) validateCommit(deltaLog, version = 3, numAdds = 0, numRemoves = 0) if (checkpoints.contains(3)) deltaLog.checkpoint() // commit version 4 - empty commit deltaLog.startTransaction().commit(Seq(), ManualUpdate) validateCommit(deltaLog, version = 4, numAdds = 0, numRemoves = 0) if (checkpoints.contains(4)) deltaLog.checkpoint() // commit version 5 - AddFile: 1, RemoveFile: 5 deltaLog.startTransaction().commit( (1 to 4).map(i => files(i).remove) ++ Seq(files(6).remove, files(7)), ManualUpdate) validateCommit(deltaLog, version = 5, numAdds = 1, numRemoves = 5) if (checkpoints.contains(5)) deltaLog.checkpoint() // commit version 6 - AddFile: 10, RemoveFile: 0 deltaLog.startTransaction().commit((8 to 17).map(i => files(i)), ManualUpdate) validateCommit(deltaLog, version = 6, numAdds = 10, numRemoves = 0) if (checkpoints.contains(6)) deltaLog.checkpoint() // commit version 7 - AddFile: 2, RemoveFile: 6 deltaLog.startTransaction().commit( (10 to 15).map(i => files(i).remove) ++ Seq(files(18), files(19)), ManualUpdate) validateCommit(deltaLog, version = 7, numAdds = 2, numRemoves = 6) if (checkpoints.contains(7)) deltaLog.checkpoint() // commit version 8 - Metadata: 1 deltaLog.startTransaction().commit(Seq(deltaLog.unsafeVolatileSnapshot.metadata), ManualUpdate) validateCommit(deltaLog, version = 8, numMetadata = 1) if (checkpoints.contains(8)) deltaLog.checkpoint() // commit version 9 - AddFile: 7 deltaLog.startTransaction().commit( Seq(files(16), files(17), files(18), files(19), files(7), files(8), files(9)) .map(af => af.copy(dataChange = false)), ManualUpdate) validateCommit(deltaLog, version = 9, numAdds = 7) if (checkpoints.contains(9)) deltaLog.checkpoint() // commit version 10 - AddFiles: 1 deltaLog.startTransaction().commit(files(20) :: Nil, ManualUpdate) validateCommit(deltaLog, version = 10, numAdds = 1, numRemoves = 0) } /** * This test creates a Delta table with 11 commits (0, 1, ..., 10) and also creates compacted * deltas based on the provided `compactionRange` tuples. * * At the end, we create a Snapshot and see if the Snapshot is initialized properly using the * right compacted delta files instead of regular delta files. We also compare the * `computeState`, `stateDF`, `allFiles` of this compacted delta backed Snapshot against a * regular Snapshot backed by single delta files. */ def testSnapshotCreation( compactionWindows: Seq[(Long, Long)], checkpoints: Set[Int] = Set.empty, postDataGenerationFunc: Option[DeltaLog => Unit] = None, postSetupFunc: Option[DeltaLog => Unit] = None, expectedCompactedDeltas: Seq[CompactedDelta], expectedDeltas: Seq[Long], expectedCheckpoint: Long = -1L, expectError: Boolean = false, additionalConfs: Seq[(String, String)] = Seq.empty): Unit = { val confs = Seq( DeltaSQLConf.DELTALOG_MINOR_COMPACTION_USE_FOR_READS.key -> "true", DeltaSQLConf.DELTA_SKIP_RECORDING_EMPTY_COMMITS.key -> "false", // Set CHECKPOINT_INTERVAL to high number so that we could checkpoint whenever we need as per // test setup. DeltaConfigs.CHECKPOINT_INTERVAL.defaultTablePropertyKey -> "1000" ) ++ additionalConfs withSQLConf(confs: _*) { withTempDir { tmpDir => val tableDir = tmpDir.getAbsolutePath generateData(tableDir, checkpoints) val (deltaLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, tableDir) // Ensure all commits are backfilled after data generation. CatalogOwnedTableUtils.populateTableCommitCoordinatorFromCatalog( spark, catalogTableOpt = None, snapshot = snapshot).foreach { tcc => tcc.backfillToVersion(snapshot.version) } // Data generation complete - run post data generation function postDataGenerationFunc.foreach(_.apply(deltaLog)) compactionWindows.foreach { case (startV, endV) => minorCompactDeltaLog(tableDir, startV, endV) } // Setup complete - run post setup function postSetupFunc.foreach(_.apply(deltaLog)) DeltaLog.clearCache() if (expectError) { intercept[DeltaIllegalStateException] { DeltaLog.forTable(spark, tableDir).unsafeVolatileSnapshot } return } val snapshot1 = DeltaLog.forTable(spark, tableDir).unsafeVolatileSnapshot val (compactedDeltas1, deltas1) = snapshot1.logSegment.deltas.map(_.getPath).partition(FileNames.isCompactedDeltaFile) assert(compactedDeltas1.size === expectedCompactedDeltas.size) compactedDeltas1.sorted .zip(expectedCompactedDeltas.sortBy(_.versionWindow)) .foreach { case (compactedDeltaPath, expectedCompactedDelta) => validateCompactedDelta(deltaLog, compactedDeltaPath, expectedCompactedDelta) } assert(deltas1.sorted.map(FileNames.deltaVersion) === expectedDeltas) assert(snapshot1.logSegment.checkpointProvider.version === expectedCheckpoint) // Disable the conf and create a new Snapshot. The new snapshot should not use the comoacted // deltas. withSQLConf(DeltaSQLConf.DELTALOG_MINOR_COMPACTION_USE_FOR_READS.key -> "false") { DeltaLog.clearCache() val snapshot2 = DeltaLog.forTable(spark, tableDir).unsafeVolatileSnapshot val (compactedDeltas2, _) = snapshot2.logSegment.deltas.map(_.getPath).partition(FileNames.isCompactedDeltaFile) assert(compactedDeltas2.isEmpty) // Compare checksum, state reconstruction result of these 2 different snapshots. assert(snapshot2.computeChecksum === snapshot1.computeChecksum) checkAnswer(snapshot2.stateDF, snapshot1.stateDF) checkAnswer(snapshot2.allFiles.toDF(), snapshot1.allFiles.toDF()) } } } } /////////////////////// // Without Checkpoints ////////////////////// test("smallest interval is chosen first for Snapshot creation") { testSnapshotCreation( compactionWindows = Seq((1, 3), (2, 3), (3, 8)), expectedCompactedDeltas = Seq(CompactedDelta((1, 3), numAdds = 1, numRemoves = 1)), expectedDeltas = Seq(0, 4, 5, 6, 7, 8, 9, 10) ) } test("Snapshot backed by single compacted delta") { testSnapshotCreation( compactionWindows = Seq((0, 10)), expectedCompactedDeltas = Seq(CompactedDelta((0, 10), numAdds = 8, numRemoves = 12, numMetadata = 1)), expectedDeltas = Seq() ) } test("empty compacted delta, compacted delta covers the beginning part") { testSnapshotCreation( compactionWindows = Seq((0, 2), (3, 4), (4, 5)), expectedCompactedDeltas = Seq( CompactedDelta((0, 2), numAdds = 5, numRemoves = 1, numMetadata = 1), CompactedDelta((3, 4), numAdds = 0, numRemoves = 0) // empty compacted delta ), expectedDeltas = Seq(5, 6, 7, 8, 9, 10) ) } test("compacted delta covers the end part of LogSegment") { testSnapshotCreation( compactionWindows = Seq((7, 10), (8, 10)), expectedCompactedDeltas = Seq( CompactedDelta((7, 10), numAdds = 8, numRemoves = 6, numMetadata = 1) ), expectedDeltas = Seq(0, 1, 2, 3, 4, 5, 6) ) } test("multiple compacted delta covers full LogSegment") { testSnapshotCreation( compactionWindows = Seq((0, 2), (3, 5), (6, 8), (9, 10)), expectedCompactedDeltas = Seq( CompactedDelta((0, 2), numAdds = 5, numRemoves = 1, numMetadata = 1), CompactedDelta((3, 5), numAdds = 1, numRemoves = 5, numMetadata = 0), CompactedDelta((6, 8), numAdds = 6, numRemoves = 6, numMetadata = 1), CompactedDelta((9, 10), numAdds = 8, numRemoves = 0, numMetadata = 0) ), expectedDeltas = Seq() ) } /////////////////////// // With Checkpoints ////////////////////// test("smallest interval after last checkpoint is chosen for Snapshot creation") { testSnapshotCreation( compactionWindows = Seq((1, 3), (2, 3), (3, 8), (4, 9), (3, 10)), checkpoints = Set(0, 2), expectedCompactedDeltas = Seq(CompactedDelta((3, 8), numAdds = 7, numRemoves = 11, numMetadata = 1)), expectedDeltas = Seq(9, 10), expectedCheckpoint = 2, // Disable DELTA_CHECKPOINT_V2_ENABLED conf so that we don't forcefully checkpoint at // commit version 8 where we change `delta.dataSkippingNumIndexedCols` to 0. additionalConfs = Seq(DeltaConfigs.CHECKPOINT_WRITE_STATS_AS_STRUCT.defaultTablePropertyKey -> "false") ) } test("Snapshot backed by single compacted delta after LAST_CHECKPOINT") { testSnapshotCreation( compactionWindows = Seq((0, 10), (5, 10)), checkpoints = Set(2, 4), expectedCompactedDeltas = Seq(CompactedDelta((5, 10), numAdds = 8, numRemoves = 11, numMetadata = 1)), expectedDeltas = Seq(), expectedCheckpoint = 4, // Disable DELTA_CHECKPOINT_V2_ENABLED conf so that we don't forcefully checkpoint at // commit version 8 where we change `delta.dataSkippingNumIndexedCols` to 0. additionalConfs = Seq(DeltaConfigs.CHECKPOINT_WRITE_STATS_AS_STRUCT.defaultTablePropertyKey -> "false") ) } test("empty compacted delta, compacted delta covers the beginning part after LAST_CHECKPOINT") { testSnapshotCreation( compactionWindows = Seq((1, 2), (3, 4), (4, 5)), checkpoints = Set(0), expectedCompactedDeltas = Seq( CompactedDelta((1, 2), numAdds = 1, numRemoves = 1), CompactedDelta((3, 4), numAdds = 0, numRemoves = 0) // empty compacted delta ), expectedDeltas = Seq(5, 6, 7, 8, 9, 10), expectedCheckpoint = 0, // Disable DELTA_CHECKPOINT_V2_ENABLED conf so that we don't forcefully checkpoint at // commit version 8 where we change `delta.dataSkippingNumIndexedCols` to 0. additionalConfs = Seq(DeltaConfigs.CHECKPOINT_WRITE_STATS_AS_STRUCT.defaultTablePropertyKey -> "false") ) } test("compacted delta covers the end part of LogSegment (with Checkpoint)") { testSnapshotCreation( compactionWindows = Seq((7, 10), (8, 10)), checkpoints = Set(0, 2, 5), expectedCompactedDeltas = Seq( CompactedDelta((7, 10), numAdds = 8, numRemoves = 6, numMetadata = 1) ), expectedDeltas = Seq(6), expectedCheckpoint = 5, // Disable DELTA_CHECKPOINT_V2_ENABLED conf so that we don't forcefully checkpoint at // commit version 8 where we change `delta.dataSkippingNumIndexedCols` to 0. additionalConfs = Seq(DeltaConfigs.CHECKPOINT_WRITE_STATS_AS_STRUCT.defaultTablePropertyKey -> "false") ) } test("multiple compacted delta covers full LogSegment (with Checkpoint)") { testSnapshotCreation( compactionWindows = Seq((0, 2), (3, 5), (3, 6), (9, 10)), checkpoints = Set(0, 2), expectedCompactedDeltas = Seq( CompactedDelta((3, 5), numAdds = 1, numRemoves = 5, numMetadata = 0), CompactedDelta((9, 10), numAdds = 8, numRemoves = 0, numMetadata = 0) ), expectedDeltas = Seq(6, 7, 8), expectedCheckpoint = 2, // Disable DELTA_CHECKPOINT_V2_ENABLED conf so that we don't forcefully checkpoint at // commit version 8 where we change `delta.dataSkippingNumIndexedCols` to 0. additionalConfs = Seq(DeltaConfigs.CHECKPOINT_WRITE_STATS_AS_STRUCT.defaultTablePropertyKey -> "false") ) } ///////////////////////////////////////////////////// // negative scenarios where deltaLog is manipulated ///////////////////////////////////////////////////// test("when compacted delta is available till version 11 but actual delta files are" + " till version 10") { testSnapshotCreation( compactionWindows = Seq((0, 2), (3, 5), (3, 6), (9, 10)), checkpoints = Set(0, 2), postSetupFunc = Some( (deltaLog: DeltaLog) => { val logPath = deltaLog.logPath val fromName = FileNames.compactedDeltaFile(logPath, fromVersion = 9, toVersion = 10) val toName = FileNames.compactedDeltaFile(logPath, fromVersion = 9, toVersion = 11) logPath.getFileSystem(deltaLog.newDeltaHadoopConf()).rename(fromName, toName) } ), expectedCompactedDeltas = Seq( CompactedDelta((3, 5), numAdds = 1, numRemoves = 5, numMetadata = 0) ), expectedDeltas = Seq(6, 7, 8, 9, 10), expectedCheckpoint = 2, // Disable DELTA_CHECKPOINT_V2_ENABLED conf so that we don't forcefully checkpoint at // commit version 8 where we change `delta.dataSkippingNumIndexedCols` to 0. additionalConfs = Seq(DeltaConfigs.CHECKPOINT_WRITE_STATS_AS_STRUCT.defaultTablePropertyKey -> "false") ) } test("compacted deltas should not be used when there are holes in deltas") { testSnapshotCreation( compactionWindows = Seq((0, 2), (3, 5), (3, 6)), checkpoints = Set(0, 2), postSetupFunc = Some( (deltaLog: DeltaLog) => { val logPath = deltaLog.logPath val deltaFileToDelete = FileNames.unsafeDeltaFile(logPath, version = 4) logPath.getFileSystem(deltaLog.newDeltaHadoopConf()).delete(deltaFileToDelete, true) } ), expectError = true, expectedCompactedDeltas = Seq(), expectedDeltas = Seq(), expectedCheckpoint = -1L, // Disable DELTA_CHECKPOINT_V2_ENABLED conf so that we don't forcefully checkpoint at // commit version 8 where we change `delta.dataSkippingNumIndexedCols` to 0. additionalConfs = Seq(DeltaConfigs.CHECKPOINT_WRITE_STATS_AS_STRUCT.defaultTablePropertyKey -> "false") ) } test("compacted deltas should include RemoveFiles that do not have deletionTimestamp") { testSnapshotCreation( compactionWindows = Seq((1, 3), (4, 6), (7, 10)), checkpoints = Set.empty, postDataGenerationFunc = Some( (deltaLog: DeltaLog) => { val hadoopConf = deltaLog.newDeltaHadoopConf() // Remove deletionTimestamp from RemoveFile in versions 1 to 10. (1 to 10).foreach { versionToRead => val file = FileNames.unsafeDeltaFile(deltaLog.logPath, versionToRead) val actions = deltaLog.store.readAsIterator(file, hadoopConf).map(Action.fromJson) val actionsWithoutDeletionTimestamp = actions .map { case r: RemoveFile if r.deletionTimestamp.isDefined => r.copy(deletionTimestamp = None) case other => other } .map(_.json) .toSeq // The iterator is already consumed above so that we don't run into the issue of // overwriting the file while reading it. deltaLog.store.write( path = file, actions = actionsWithoutDeletionTimestamp.toIterator, overwrite = true, hadoopConf = hadoopConf) } } ), expectedCompactedDeltas = Seq( CompactedDelta((1, 3), numAdds = 1, numRemoves = 1, numMetadata = 0), CompactedDelta((4, 6), numAdds = 11, numRemoves = 5, numMetadata = 0), CompactedDelta((7, 10), numAdds = 8, numRemoves = 6, numMetadata = 1) ), expectedDeltas = Seq(0) ) } } class DeltaLogMinorCompactionWithCatalogOwnedBatch1Suite extends DeltaLogMinorCompactionSuite { override val catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) override protected def sparkConf: SparkConf = super.sparkConf .set(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey, "false") } class DeltaLogMinorCompactionWithCatalogOwnedBatch2Suite extends DeltaLogMinorCompactionSuite { override val catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) override protected def sparkConf: SparkConf = super.sparkConf .set(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey, "false") } class DeltaLogMinorCompactionWithCatalogOwnedBatch100Suite extends DeltaLogMinorCompactionSuite { override val catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) override protected def sparkConf: SparkConf = super.sparkConf .set(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey, "false") } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaLogSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.{BufferedReader, File, InputStreamReader, IOException} import java.nio.charset.StandardCharsets import java.util.{Locale, Optional} import scala.collection.JavaConverters._ import scala.language.postfixOps import org.apache.spark.sql.delta.DeltaOperations.Truncate import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CatalogOwnedTestBaseSuite, InMemoryCommitCoordinator, TrackingCommitCoordinatorClient} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.scala.DefaultScalaModule import io.delta.storage.commit.{CommitCoordinatorClient, TableDescriptor} import org.apache.hadoop.fs.Path import org.apache.hadoop.fs.permission.FsPermission import org.apache.spark.rdd.RDD import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.JsonToStructs import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.NullType import org.apache.spark.unsafe.types.UTF8String import org.apache.spark.util.Utils // scalastyle:off: removeFile class DeltaLogSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with CatalogOwnedTestBaseSuite with DeltaCheckpointTestUtils with DeltaSQLTestUtils { protected val testOp = Truncate() testDifferentCheckpoints("checkpoint", quiet = true) { (_, _) => val tempDir = Utils.createTempDir() val log1 = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) (1 to 15).foreach { i => val txn = log1.startTransaction() val file = createTestAddFile(encodedPath = i.toString) :: Nil val delete: Seq[Action] = if (i > 1) { RemoveFile(i - 1 toString, Some(System.currentTimeMillis()), true) :: Nil } else { Nil } txn.commitManually(delete ++ file: _*) } DeltaLog.clearCache() val log2 = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) assert(log2.snapshot.version == log1.snapshot.version) assert(log2.snapshot.allFiles.count == 1) } testDifferentCheckpoints("update deleted directory", quiet = true) { (_, _) => withTempDir { dir => val path = new Path(dir.getCanonicalPath) val log = DeltaLog.forTable(spark, path) // Commit data so the in-memory state isn't consistent with an empty log. val txn = log.startTransaction() val files = (1 to 10).map(f => createTestAddFile(encodedPath = f.toString)) txn.commitManually(files: _*) log.checkpoint() val fs = path.getFileSystem(log.newDeltaHadoopConf()) fs.delete(path, true) val snapshot = log.update() assert(snapshot.version === -1) } } testDifferentCheckpoints( "checkpoint write should use the correct Hadoop configuration") { (_, _) => withTempDir { dir => withSQLConf( "fs.AbstractFileSystem.fake.impl" -> classOf[FakeAbstractFileSystem].getName, "fs.fake.impl" -> classOf[FakeFileSystem].getName, "fs.fake.impl.disable.cache" -> "true") { val path = s"fake://${dir.getCanonicalPath}" val log = DeltaLog.forTable(spark, path) val txn = log.startTransaction() txn.commitManually(createTestAddFile()) log.checkpoint() } } } testDifferentCheckpoints("update should pick up checkpoints", quiet = true) { (_, _) => withTempDir { tempDir => val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) val checkpointInterval = log.checkpointInterval() for (f <- 0 until (checkpointInterval * 2)) { val txn = log.startTransaction() txn.commitManually(createTestAddFile(encodedPath = f.toString)) } def collectReservoirStateRDD(rdd: RDD[_]): Seq[RDD[_]] = { if (rdd.name != null && rdd.name.startsWith("Delta Table State")) { Seq(rdd) ++ rdd.dependencies.flatMap(d => collectReservoirStateRDD(d.rdd)) } else { rdd.dependencies.flatMap(d => collectReservoirStateRDD(d.rdd)) } } val numOfStateRDDs = collectReservoirStateRDD(log.snapshot.stateDS.rdd).size assert(numOfStateRDDs >= 1, "collectReservoirStateRDD may not work properly") assert(numOfStateRDDs < checkpointInterval) } } testDifferentCheckpoints( "update shouldn't pick up delta files earlier than checkpoint") { (_, _) => val tempDir = Utils.createTempDir() val log1 = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) (1 to 5).foreach { i => val txn = log1.startTransaction() val file = if (i > 1) { createTestAddFile(encodedPath = i.toString) :: Nil } else { Metadata(configuration = Map(DeltaConfigs.CHECKPOINT_INTERVAL.key -> "10")) :: Nil } val delete: Seq[Action] = if (i > 1) { RemoveFile(i - 1 toString, Some(System.currentTimeMillis()), true) :: Nil } else { Nil } txn.commitManually(delete ++ file: _*) } DeltaLog.clearCache() val log2 = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) (6 to 15).foreach { i => val txn = log1.startTransaction() val file = createTestAddFile(encodedPath = i.toString) :: Nil val delete: Seq[Action] = if (i > 1) { RemoveFile(i - 1 toString, Some(System.currentTimeMillis()), true) :: Nil } else { Nil } txn.commitManually(delete ++ file: _*) } // Since log2 is a separate instance, it shouldn't be updated to version 15 assert(log2.snapshot.version == 4) val updateLog2 = log2.update() assert(updateLog2.version == log1.snapshot.version, "Did not update to correct version") val deltas = log2.snapshot.logSegment.deltas assert(deltas.length === 4, "Expected 4 files starting at version 11 to 14") val versions = deltas.map(FileNames.deltaVersion).sorted assert(versions === Seq[Long](11, 12, 13, 14), "Received the wrong files for update") } testQuietly("ActionLog cache should use the normalized path as key") { withTempDir { tempDir => val dir = tempDir.getAbsolutePath.stripSuffix("/") assert(dir.startsWith("/")) // scalastyle:off deltahadoopconfiguration val fs = new Path("/").getFileSystem(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration val samePaths = Seq( new Path(dir + "/foo"), new Path(dir + "/foo/"), new Path(fs.getScheme + ":" + dir + "/foo"), new Path(fs.getScheme + "://" + dir + "/foo") ) val logs = samePaths.map(DeltaLog.forTable(spark, _)) logs.foreach { log => assert(log eq logs.head) } } } testDifferentCheckpoints( "handle corrupted '_last_checkpoint' file", quiet = true) { (checkpointPolicy, format) => withTempDir { tempDir => val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) val checkpointInterval = log.checkpointInterval() for (f <- 0 to checkpointInterval) { val txn = log.startTransaction() txn.commitManually(createTestAddFile(encodedPath = f.toString)) } val lastCheckpointOpt = log.readLastCheckpointFile() assert(lastCheckpointOpt.isDefined) val lastCheckpoint = lastCheckpointOpt.get import CheckpointInstance.Format._ val expectedCheckpointFormat = if (checkpointPolicy == CheckpointPolicy.V2) V2 else SINGLE assert(CheckpointInstance(lastCheckpoint).format === expectedCheckpointFormat) // Create an empty "_last_checkpoint" (corrupted) val fs = log.LAST_CHECKPOINT.getFileSystem(log.newDeltaHadoopConf()) fs.create(log.LAST_CHECKPOINT, true /* overwrite */).close() // Create a new DeltaLog DeltaLog.clearCache() val log2 = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) // Make sure we create a new DeltaLog in order to test the loading logic. assert(log ne log2) // We should get the same metadata even if "_last_checkpoint" is corrupted. assert(CheckpointInstance(log2.readLastCheckpointFile().get) === CheckpointInstance(lastCheckpoint.version, SINGLE)) } } testQuietly("paths should be canonicalized") { Seq("file:", "file://").foreach { scheme => withTempDir { dir => val log = DeltaLog.forTable(spark, dir) assert(new File(log.logPath.toUri).mkdirs()) val path = "/some/unqualified/absolute/path" val add = AddFile( path, Map.empty, 100L, 10L, dataChange = true) val rm = RemoveFile( s"$scheme$path", Some(200L), dataChange = false) log.store.write( FileNames.unsafeDeltaFile(log.logPath, 0L), Iterator(Action.supportedProtocolVersion( featuresToExclude = Seq(CatalogOwnedTableFeature)), Metadata(), add) .map(a => JsonUtils.toJson(a.wrap)), overwrite = false, log.newDeltaHadoopConf()) log.store.write( FileNames.unsafeDeltaFile(log.logPath, 1L), Iterator(JsonUtils.toJson(rm.wrap)), overwrite = false, log.newDeltaHadoopConf()) assert(log.update().version === 1) assert(log.snapshot.numOfFiles === 0) } } } testQuietly("paths should be canonicalized - special characters") { Seq("file:", "file://").foreach { scheme => withTempDir { dir => val log = DeltaLog.forTable(spark, dir) assert(new File(log.logPath.toUri).mkdirs()) val path = new Path("/some/unqualified/with space/p@#h").toUri.toString val add = AddFile( path, Map.empty, 100L, 10L, dataChange = true) val rm = RemoveFile( s"$scheme$path", Some(200L), dataChange = false) log.store.write( FileNames.unsafeDeltaFile(log.logPath, 0L), Iterator(Action.supportedProtocolVersion( featuresToExclude = Seq(CatalogOwnedTableFeature)), Metadata(), add) .map(a => JsonUtils.toJson(a.wrap)), overwrite = false, log.newDeltaHadoopConf()) log.store.write( FileNames.unsafeDeltaFile(log.logPath, 1L), Iterator(JsonUtils.toJson(rm.wrap)), overwrite = false, log.newDeltaHadoopConf()) assert(log.update().version === 1) assert(log.snapshot.numOfFiles === 0) } } } test("Reject read from Delta if no path is passed") { val e = intercept[IllegalArgumentException](spark.read.format("delta").load()).getMessage assert(e.contains("'path' is not specified")) } test("do not relativize paths in RemoveFiles") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir) assert(new File(log.logPath.toUri).mkdirs()) val path = new File(dir, "a/b%c/d").toURI.toString val rm = RemoveFile(path, Some(System.currentTimeMillis()), dataChange = true) log.startTransaction().commitManually(rm) val committedRemove = log.update(stalenessAcceptable = false).tombstones.collect().head assert(committedRemove.path === path) } } test("delete and re-add the same file in different transactions") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir) assert(new File(log.logPath.toUri).mkdirs()) val add1 = createTestAddFile(modificationTime = System.currentTimeMillis()) log.startTransaction().commitManually(add1) val rm = add1.remove log.startTransaction().commit(rm :: Nil, DeltaOperations.ManualUpdate) val add2 = createTestAddFile(modificationTime = System.currentTimeMillis()) log.startTransaction().commit(add2 :: Nil, DeltaOperations.ManualUpdate) // Add a new transaction to replay logs using the previous snapshot. If it contained // AddFile("foo") and RemoveFile("foo"), "foo" would get removed and fail this test. val otherAdd = createTestAddFile(encodedPath = "bar", modificationTime = System.currentTimeMillis()) log.startTransaction().commit(otherAdd :: Nil, DeltaOperations.ManualUpdate) assert(log.update().allFiles.collect().find(_.path == "foo") // `dataChange` is set to `false` after replaying logs. === Some(add2.copy( dataChange = false, baseRowId = Some(1), defaultRowCommitVersion = Some(2)))) } } test("error - versions not contiguous") { if (catalogOwnedCoordinatorBackfillBatchSize.contains(100L)) { cancel("Backfill size of 100 is not compatible w/ the test.") } withTempDir { dir => val staleLog = DeltaLog.forTable(spark, dir) DeltaLog.clearCache() val log = DeltaLog.forTable(spark, dir) assert(new File(log.logPath.toUri).mkdirs()) val metadata = Metadata( // Needs to manually enable ICT during manual Metadata update for CC. configuration = Map(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key -> "true")) val add1 = AddFile("foo", Map.empty, 1L, System.currentTimeMillis(), dataChange = true) log.startTransaction().commit(metadata :: add1 :: Nil, DeltaOperations.ManualUpdate) val add2 = AddFile("bar", Map.empty, 1L, System.currentTimeMillis(), dataChange = true) log.startTransaction().commit(add2 :: Nil, DeltaOperations.ManualUpdate) val add3 = AddFile("baz", Map.empty, 1L, System.currentTimeMillis(), dataChange = true) log.startTransaction().commit(add3 :: Nil, DeltaOperations.ManualUpdate) new File(new Path(log.logPath, "00000000000000000001.json").toUri).delete() val ex = intercept[IllegalStateException] { staleLog.update() } assert(ex.getMessage.contains("Versions (0, 2) are not contiguous.")) } } Seq("protocol", "metadata").foreach { action => test(s"state reconstruction without $action should fail") { withTempDir { tempDir => val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) assert(new File(log.logPath.toUri).mkdirs()) val selectedAction = if (action == "metadata") { Protocol() } else { Metadata() } val file = AddFile("abc", Map.empty, 1, 1, true) log.store.write( FileNames.unsafeDeltaFile(log.logPath, 0L), Iterator(selectedAction, file).map(a => JsonUtils.toJson(a.wrap)), overwrite = false, log.newDeltaHadoopConf()) val e = intercept[IllegalStateException] { log.update() } assert(e.getMessage === DeltaErrors.actionNotFoundException(action, 0).getMessage) } } } Seq("protocol", "metadata").foreach { action => testDifferentCheckpoints(s"state reconstruction from checkpoint with" + s" missing $action should fail", quiet = true) { (_, _) => withTempDir { tempDir => import testImplicits._ val staleLog = DeltaLog.forTable(spark, tempDir) DeltaLog.clearCache() val log = DeltaLog.forTable(spark, tempDir) assert (staleLog != log) val checkpointInterval = log.checkpointInterval() // Create a checkpoint regularly for (f <- 0 to checkpointInterval) { val txn = log.startTransaction() val addFile = createTestAddFile(encodedPath = f.toString) if (f == 0) { txn.commitManually(addFile) } else { txn.commit(Seq(addFile), testOp) } } val checksumFilePath = FileNames.checksumFile(log.logPath, log.snapshot.version) removeProtocolAndMetadataFromChecksumFile(checksumFilePath) { // Create an incomplete checkpoint without the action and overwrite the // original checkpoint val checkpointPathOpt = log.listFrom(log.snapshot.version).find(FileNames.isCheckpointFile).map(_.getPath) assert(checkpointPathOpt.nonEmpty) assert(FileNames.checkpointVersion(checkpointPathOpt.get) === log.snapshot.version) val checkpointPath = checkpointPathOpt.get def removeActionFromParquetCheckpoint(tmpCheckpoint: File): Unit = { val takeAction = if (action == "metadata") { "protocol" } else { "metadata" } val corruptedCheckpointData = spark.read.schema(SingleAction.encoder.schema) .parquet(checkpointPath.toString) .where(s"add is not null or $takeAction is not null") .as[SingleAction].collect() // Keep the add files and also filter by the additional condition corruptedCheckpointData.toSeq.toDS().coalesce(1).write .mode("overwrite").parquet(tmpCheckpoint.toString) val writtenCheckpoint = tmpCheckpoint.listFiles().toSeq.filter(_.getName.startsWith("part")).head val checkpointFile = new File(checkpointPath.toUri) new File(log.logPath.toUri).listFiles().toSeq.foreach { file => if (file.getName.startsWith(".0")) { // we need to delete checksum files, otherwise trying to replace our incomplete // checkpoint file fails due to the LocalFileSystem's checksum checks. require(file.delete(), "Failed to delete checksum file") } } require(checkpointFile.delete(), "Failed to delete old checkpoint") require(writtenCheckpoint.renameTo(checkpointFile), "Failed to rename corrupt checkpoint") } if (checkpointPath.getName.endsWith("json")) { val conf = log.newDeltaHadoopConf() val filteredActions = log.store .read(checkpointPath, log.newDeltaHadoopConf()) .map(Action.fromJson) .filter { case _: Protocol => action != "protocol" case _: Metadata => action != "metadata" case _ => true }.map(_.json) log.store.write(checkpointPath, filteredActions.toIterator, overwrite = true, conf) } else { withTempDir { f => removeActionFromParquetCheckpoint(f) } } } // Verify if the state reconstruction from the checkpoint fails. val e = intercept[IllegalStateException] { staleLog.update() } assert(e.getMessage === DeltaErrors.actionNotFoundException(action, checkpointInterval).getMessage) } } } testDifferentCheckpoints("deleting and recreating a directory should" + " cause the snapshot to be recomputed", quiet = true) { (_, _) => withTempDir { dir => val path = dir.getCanonicalPath spark.range(10).write.format("delta").mode("append").save(path) spark.range(10, 20).write.format("delta").mode("append").save(path) val deltaLog = DeltaLog.forTable(spark, path) deltaLog.checkpoint() spark.range(20, 30).write.format("delta").mode("append").save(path) // Store these for later usage val actions = deltaLog.snapshot.stateDS.collect() val commitTimestamp = deltaLog.snapshot.logSegment.lastCommitFileModificationTimestamp checkAnswer( spark.read.format("delta").load(path), spark.range(30).toDF() ) val fs = deltaLog.logPath.getFileSystem(deltaLog.newDeltaHadoopConf()) // Now let's delete the last version deltaLog.store .listFrom( FileNames.listingPrefix(deltaLog.logPath, deltaLog.snapshot.version), deltaLog.newDeltaHadoopConf()) .filter(!_.getPath.getName.startsWith("_")) .foreach(f => fs.delete(f.getPath, true)) if (catalogOwnedDefaultCreationEnabledInTests) { // For Catalog Owned table with a commit that is not backfilled, we can't use // 00000000002.json yet. Contact commit coordinator to get uuid file path to malform json // file. val oc = getCatalogOwnedCommitCoordinatorClient( catalogName = CatalogOwnedTableUtils.DEFAULT_CATALOG_NAME_FOR_TESTING) val tableDesc = new TableDescriptor(deltaLog.logPath, Optional.empty(), Map.empty[String, String].asJava) val commitResponse = oc.getCommits(tableDesc, 2, null) if (!commitResponse.getCommits.isEmpty) { val path = commitResponse.getCommits.asScala.last.getFileStatus.getPath fs.delete(path, true) } // Also deletes it from in-memory commit coordinator. oc.asInstanceOf[TrackingCommitCoordinatorClient] .delegatingCommitCoordinatorClient .asInstanceOf[InMemoryCommitCoordinator] .removeCommitTestOnly(deltaLog.logPath, 2) } // Should show up to 20 checkAnswer( spark.read.format("delta").load(path), spark.range(20).toDF() ) // Now let's delete the checkpoint and json file for version 1. We will try to list from // version 1, but since we can't find anything, we should start listing from version 0 deltaLog.store .listFrom( FileNames.listingPrefix(deltaLog.logPath, 1), deltaLog.newDeltaHadoopConf()) .filter(!_.getPath.getName.startsWith("_")) .foreach(f => fs.delete(f.getPath, true)) if (catalogOwnedDefaultCreationEnabledInTests) { val oc = getCatalogOwnedCommitCoordinatorClient( catalogName = CatalogOwnedTableUtils.DEFAULT_CATALOG_NAME_FOR_TESTING) oc.asInstanceOf[TrackingCommitCoordinatorClient] .delegatingCommitCoordinatorClient .asInstanceOf[InMemoryCommitCoordinator] .removeCommitTestOnly(deltaLog.logPath, 1) } checkAnswer( spark.read.format("delta").load(path), spark.range(10).toDF() ) // Now let's delete that commit as well, and write a new first version deltaLog.listFrom(0) .filter(!_.getPath.getName.startsWith("_")) .foreach(f => fs.delete(f.getPath, false)) assert(deltaLog.snapshot.version === 0) deltaLog.store.write( FileNames.unsafeDeltaFile(deltaLog.logPath, 0), actions.map(_.unwrap.json).iterator, overwrite = false, deltaLog.newDeltaHadoopConf()) // To avoid flakiness, we manually set the modification timestamp of the file to a later // second new File(FileNames.unsafeDeltaFile(deltaLog.logPath, 0).toUri) .setLastModified(commitTimestamp + 5000) checkAnswer( spark.read.format("delta").load(path), spark.range(30).toDF() ) } } test("forTableWithSnapshot should always return the latest snapshot") { withTempDir { dir => val path = dir.getCanonicalPath spark.range(10).write.format("delta").mode("append").save(path) val deltaLog = DeltaLog.forTable(spark, path) assert(deltaLog.snapshot.version === 0) val (_, snapshot) = DeltaLog.withFreshSnapshot { _ => // This update is necessary to advance the lastUpdatedTs beyond the start time of // withFreshSnapshot call. deltaLog.update() // Manually add a commit. However, the deltaLog should now be fresh enough // that we don't trigger another update, and thus don't find the commit. val add = AddFile(path, Map.empty, 100L, 10L, dataChange = true) deltaLog.store.write( FileNames.unsafeDeltaFile(deltaLog.logPath, 1L), Iterator(JsonUtils.toJson(add.wrap)), overwrite = false, deltaLog.newDeltaHadoopConf()) (deltaLog, None) } assert(snapshot.version === 0) val deltaLog2 = DeltaLog.forTable(spark, path) assert(deltaLog2.snapshot.version === 0) // This shouldn't update val (_, snapshot2) = DeltaLog.forTableWithSnapshot(spark, path) assert(snapshot2.version === 1) // This should get the latest snapshot } } test("Delta log should handle malformed json") { val mapper = new ObjectMapper() mapper.registerModule(DefaultScalaModule) def testJsonCommitParser( path: String, func: Map[String, Map[String, String]] => String): Unit = { spark.range(10).write.format("delta").mode("append").save(path) spark.range(1).write.format("delta").mode("append").save(path) val log = DeltaLog.forTable(spark, path) var commitFilePath = FileNames.unsafeDeltaFile(log.logPath, 1L) if (catalogOwnedDefaultCreationEnabledInTests) { // For Catalog Owned table with a commit that is not backfilled, we can't use // 00000000001.json yet. Contact commit coordinator to get uuid file path to malform json // file. val oc = getCatalogOwnedCommitCoordinatorClient( catalogName = CatalogOwnedTableUtils.DEFAULT_CATALOG_NAME_FOR_TESTING) val tableDesc = new TableDescriptor(log.logPath, Optional.empty(), Map.empty[String, String].asJava) val commitResponse = oc.getCommits(tableDesc, 1, null) if (!commitResponse.getCommits.isEmpty) { commitFilePath = commitResponse.getCommits.asScala.head.getFileStatus.getPath } } val fs = log.logPath.getFileSystem(log.newDeltaHadoopConf()) val stream = fs.open(commitFilePath) val reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)) val commitInfo = reader.readLine() + "\n" val addFile = reader.readLine() stream.close() val map = mapper.readValue(addFile, classOf[Map[String, Map[String, String]]]) val output = fs.create(commitFilePath, true) output.write(commitInfo.getBytes(StandardCharsets.UTF_8)) output.write(func(map).getBytes(StandardCharsets.UTF_8)) output.close() DeltaLog.clearCache() val parser = JsonToStructs( schema = Action.logSchema, options = DeltaLog.jsonCommitParseOption, child = null, timeZoneId = Some(spark.sessionState.conf.sessionLocalTimeZone)) val it = log.store.readAsIterator(commitFilePath, log.newDeltaHadoopConf()) try { it.foreach { json => val utf8json = UTF8String.fromString(json) parser.nullSafeEval(utf8json).asInstanceOf[InternalRow] } } finally { it.close() } } // Parser should succeed when AddFile in json commit has missing fields withTempDir { dir => testJsonCommitParser(dir.toString, (content: Map[String, Map[String, String]]) => { mapper.writeValueAsString(Map("add" -> content("add").-("path").-("size"))) + "\n" }) } // Parser should succeed when AddFile in json commit has extra fields withTempDir { dir => testJsonCommitParser(dir.toString, (content: Map[String, Map[String, String]]) => { mapper.writeValueAsString(Map("add" -> content("add"). +("random" -> "field"))) + "\n" }) } // Parser should succeed when AddFile in json commit has mismatched schema withTempDir { dir => val json = """{"x": 1, "y": 2, "z": [10, 20]}""" testJsonCommitParser(dir.toString, (content: Map[String, Map[String, String]]) => { mapper.writeValueAsString(Map("add" -> content("add").updated("path", json))) + "\n" }) } // Parser should throw exception when AddFile is a bad json withTempDir { dir => val e = intercept[Throwable] { testJsonCommitParser(dir.toString, (content: Map[String, Map[String, String]]) => { "bad json{{{" }) } assert(e.getMessage.contains("FAILFAST")) } } test("DeltaLog cache size should honor config limit") { def assertCacheSize(expected: Long): Unit = { for (_ <- 1 to 6) { withTempDir(dir => { val path = dir.getCanonicalPath spark.range(10).write.format("delta").mode("append").save(path) }) } assert(DeltaLog.cacheSize === expected) } DeltaLog.unsetCache() withSQLConf(DeltaSQLConf.DELTA_LOG_CACHE_SIZE.key -> "4") { assertCacheSize(4) DeltaLog.unsetCache() // the larger of SQLConf and env var is adopted try { System.getProperties.setProperty("delta.log.cacheSize", "5") assertCacheSize(5) } finally { System.getProperties.remove("delta.log.cacheSize") } } // assert timeconf returns correct value withSQLConf(DeltaSQLConf.DELTA_LOG_CACHE_RETENTION_MINUTES.key -> "100") { assert(spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_LOG_CACHE_RETENTION_MINUTES) === 100) } } test("DeltaLog should create log directory when ensureLogDirectory is called") { withTempDir { dir => val path = dir.getCanonicalPath val log = DeltaLog.forTable(spark, new Path(path)) log.createLogDirectoriesIfNotExists() val logPath = log.logPath val fs = logPath.getFileSystem(log.newDeltaHadoopConf()) assert(fs.exists(logPath), "Log path should exist.") assert(fs.getFileStatus(logPath).isDirectory, "Log path should be a directory") val commitPath = FileNames.commitDirPath(logPath) assert(fs.exists(commitPath), "Commit path should exist.") assert(fs.getFileStatus(commitPath).isDirectory, "Commit path should be a directory") } } test("DeltaLog should throw exception when unable to create log directory " + "with filesystem IO Exception") { withTempDir { dir => val path = dir.getCanonicalPath val log = DeltaLog.forTable(spark, new Path(path)) val fs = log.logPath.getFileSystem(log.newDeltaHadoopConf()) // create a file in place of what should be the directory. // Attempting to create a child file/directory should fail and throw an IOException. fs.create(log.logPath) val e = intercept[DeltaIOException] { log.createLogDirectoriesIfNotExists() } checkError(e, "DELTA_CANNOT_CREATE_LOG_PATH", "42KD5", Map("path" -> log.logPath.toString)) e.getCause match { case e: IOException => assert(e.getMessage.contains("Parent path is not a directory")) case _ => fail(s"Expected IOException, got ${e.getCause}") } } } test("DeltaFileProviderUtils.getDeltaFilesInVersionRange") { withTempDir { dir => val path = dir.getCanonicalPath spark.range(0, 1).write.format("delta").mode("overwrite").save(path) spark.range(0, 1).write.format("delta").mode("overwrite").save(path) spark.range(0, 1).write.format("delta").mode("overwrite").save(path) spark.range(0, 1).write.format("delta").mode("overwrite").save(path) val log = DeltaLog.forTable(spark, new Path(path)) val result = DeltaFileProviderUtils.getDeltaFilesInVersionRange( spark, log, startVersion = 1, endVersion = 3, catalogTableOpt = None) assert(result.map(FileNames.getFileVersion) === Seq(1, 2, 3)) val filesAreUnbackfilledArray = result.map(FileNames.isUnbackfilledDeltaFile) val (fileV1, fileV2, fileV3) = (result(0), result(1), result(2)) assert(FileNames.getFileVersion(fileV1) === 1) assert(FileNames.getFileVersion(fileV2) === 2) assert(FileNames.getFileVersion(fileV3) === 3) val backfillInterval = catalogOwnedCoordinatorBackfillBatchSize.getOrElse(0L) if (backfillInterval == 0 || backfillInterval == 1) { assert(filesAreUnbackfilledArray === Seq(false, false, false)) } else if (backfillInterval == 2) { assert(filesAreUnbackfilledArray === Seq(false, false, true)) } else { assert(filesAreUnbackfilledArray === Seq(true, true, true)) } } } test("checksum file should contain protocol and metadata") { withSQLConf( DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> "true", DeltaSQLConf.USE_PROTOCOL_AND_METADATA_FROM_CHECKSUM_ENABLED.key -> "true" ) { withTempDir { dir => val path = new Path("file://" + dir.getAbsolutePath) val log = DeltaLog.forTable(spark, path) val txn = log.startTransaction() val files = (1 to 10).map(f => createTestAddFile(encodedPath = f.toString)) txn.commitManually(files: _*) val metadata = log.snapshot.metadata val protocol = log.snapshot.protocol DeltaLog.clearCache() val readLog = DeltaLog.forTable(spark, path) val checksum = readLog.snapshot.checksumOpt.get assert(checksum.metadata != null) assert(checksum.protocol != null) assert(checksum.metadata.equals(metadata)) assert(checksum.protocol.equals(protocol)) } } } test("checksum reader should be able to read incomplete checksum file without " + "protocol and metadata") { withSQLConf( DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> "true", DeltaSQLConf.USE_PROTOCOL_AND_METADATA_FROM_CHECKSUM_ENABLED.key -> "true" ) { withTempDir { dir => val path = new Path("file://" + dir.getAbsolutePath) val log = DeltaLog.forTable(spark, path) val txn = log.startTransaction() val files = (1 to 10).map(f => createTestAddFile(encodedPath = f.toString)) txn.commitManually(files: _*) val metadata = log.snapshot.metadata val protocol = log.snapshot.protocol DeltaLog.clearCache() val checksumFilePath = FileNames.checksumFile(log.logPath, 0L) removeProtocolAndMetadataFromChecksumFile(checksumFilePath) val readLog = DeltaLog.forTable(spark, path) val checksum = readLog.snapshot.checksumOpt.get assert(checksum.metadata == null) assert(checksum.protocol == null) // check we are still able to read protocol and metadata from checkpoint assert(readLog.snapshot.metadata.equals(metadata)) assert(readLog.snapshot.protocol.equals(protocol)) } } } private def testCreateDataFrame(shouldDropNullTypeColumns: Boolean): Unit = { withSQLConf(DeltaSQLConf.DELTA_CREATE_DATAFRAME_DROP_NULL_COLUMNS.key -> shouldDropNullTypeColumns.toString) { withTempDir { tempDir => spark.sql("select CAST(null as VOID) as nullTypeCol, id from range(10)") .write .format("delta") .mode("append") .save(tempDir.getCanonicalPath) val deltaLog = DeltaLog.forTable(spark, tempDir) val df = deltaLog.createDataFrame(deltaLog.update(), Seq.empty, isStreaming = false) val nullTypeFields = df.schema.filter(_.dataType == NullType) if (shouldDropNullTypeColumns) { assert(nullTypeFields.isEmpty) } else { assert(nullTypeFields.size == 1) } } } } test("DeltaLog.createDataFrame should drop null columns with feature flag") { testCreateDataFrame(shouldDropNullTypeColumns = true) } test("DeltaLog.createDataFrame should not drop null columns without feature flag") { testCreateDataFrame(shouldDropNullTypeColumns = false) } } class DeltaLogWithCatalogOwnedBatch1Suite extends DeltaLogSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class DeltaLogWithCatalogOwnedBatch2Suite extends DeltaLogSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class DeltaLogWithCatalogOwnedBatch100Suite extends DeltaLogSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaMetricsUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.collection.mutable.ArrayBuffer import org.apache.spark.sql.delta.actions.{AddCDCFile, AddFile, CommitInfo, RemoveFile} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.scalatest.Assertions._ /** * Various helper methods to for metric tests. */ object DeltaMetricsUtils { /** * Get operation metrics of the last operation of a table. * * @param table The Delta table to query * @return The operation metrics of the last command. */ def getLastOperationMetrics(table: io.delta.tables.DeltaTable): Map[String, Long] = { table.history().select("operationMetrics").take(1).head.getMap(0) .asInstanceOf[Map[String, String]].mapValues(_.toLong).toMap } def getLastOperationMetrics(tableName: String): Map[String, Long] = { getLastOperationMetrics(io.delta.tables.DeltaTable.forName(tableName)) } /** * Assert that metrics of a Delta operation have the expected values. * * @param expectedMetrics The expected metrics the values of which to check. * @param operationMetrics The operation metrics that were collected from Delta log. */ def checkOperationMetrics( expectedMetrics: Map[String, Long], operationMetrics: Map[String, Long]): Unit = { val sep = System.lineSeparator() * 2 val failMessages = expectedMetrics.flatMap { case (metric, expectedValue) => // Check missing metrics. var errMsg = if (!operationMetrics.contains(metric)) { Some( s"""The recorded operation metrics does not contain metric: $metric" | ExpectedMetrics = $expectedMetrics | ActualMetrics = $operationMetrics |""".stripMargin) } else { None } // Check negative values. errMsg = errMsg.orElse { if (operationMetrics(metric) < 0) { Some(s"Invalid non-positive value for metric $metric: ${operationMetrics(metric)}") } else { None } } // Check unexpected values. errMsg = errMsg.orElse { if (expectedValue != operationMetrics(metric)) { Some( s"""The recorded metric for $metric does not equal the expected value. | Expected = ${expectedMetrics(metric)} | Actual = ${operationMetrics(metric)} | ExpectedMetrics = $expectedMetrics | ActualMetrics = $operationMetrics |""".stripMargin) } else { None } } errMsg }.mkString(sep, sep, sep).trim assert(failMessages.isEmpty) } /** * Check that time metrics for a Delta operation are valid. * * @param operationMetrics The collected operation metrics from the Delta log. * @param expectedMetrics The keys of the expected time metrics. Set to None to check for * common time metrics. */ def checkOperationTimeMetrics( operationMetrics: Map[String, Long], expectedMetrics: Set[String]): Unit = { // Validate that all time metrics exist and have a non-negative value. for (key <- expectedMetrics) { assert(operationMetrics.contains(key), s"Missing operation metric $key") val value: Long = operationMetrics(key) assert(value >= 0, s"Invalid non-positive value for metric $key: $value") } // Validate that if 'executionTimeMs' exists, is larger than all other time metrics. if (expectedMetrics.contains("executionTimeMs")) { val executionTimeMs = operationMetrics("executionTimeMs") val maxTimeMs = operationMetrics.filterKeys(k => expectedMetrics.contains(k)) .valuesIterator.max assert(executionTimeMs == maxTimeMs) } } /** * Computes the expected operation metrics from the actions in a Delta commit. * * @param deltaLog The Delta log of the table. * @param version The version of the commit. * @return A map with the expected operation metrics. */ def getOperationMetricsFromCommitActions( deltaLog: DeltaLog, version: Long): Map[String, Long] = { val (_, changes) = deltaLog.getChanges(version).next() val commitInfo = changes.collect { case ci: CommitInfo => ci }.head val operationName = commitInfo.operation var filesAdded = ArrayBuffer.empty[AddFile] var filesRemoved = ArrayBuffer.empty[RemoveFile] val changeFilesAdded = ArrayBuffer.empty[AddCDCFile] changes.foreach { case a: AddFile => filesAdded.append(a) case r: RemoveFile => filesRemoved.append(r) case c: AddCDCFile => changeFilesAdded.append(c) case _ => // Nothing } // Filter-out DV updates from files added and removed. val pathsWithDvUpdate = filesAdded.map(_.path).toSet & filesRemoved.map(_.path).toSet filesAdded = filesAdded.filter(a => !pathsWithDvUpdate.contains(a.path)) val numFilesAdded = filesAdded.size val numBytesAdded = filesAdded.map(_.size).sum filesRemoved = filesRemoved.filter(r => !pathsWithDvUpdate.contains(r.path)) val numFilesRemoved = filesRemoved.size val numBytesRemoved = filesRemoved.map(_.size.getOrElse(0L)).sum val numChangeFilesAdded = changeFilesAdded.size operationName match { case "MERGE" => Map( "numTargetFilesAdded" -> numFilesAdded, "numTargetFilesRemoved" -> numFilesRemoved, "numTargetBytesAdded" -> numBytesAdded, "numTargetBytesRemoved" -> numBytesRemoved, "numTargetChangeFilesAdded" -> numChangeFilesAdded ) case "UPDATE" | "DELETE" => Map( "numAddedFiles" -> numFilesAdded, "numRemovedFiles" -> numFilesRemoved, "numAddedBytes" -> numBytesAdded, "numRemovedBytes" -> numBytesRemoved, "numAddedChangeFiles" -> numChangeFilesAdded ) case _ => throw new UnsupportedOperationException(s"Unsupported operation: $operationName") } } /** * Checks the provided operation metrics against the actions in a Delta commit. * * @param deltaLog The Delta log of the table. * @param version The version of the commit. * @param operationMetrics The operation metrics that were collected from Delta log. */ def checkOperationMetricsAgainstCommitActions( deltaLog: DeltaLog, version: Long, operationMetrics: Map[String, Long]): Unit = { checkOperationMetrics( expectedMetrics = getOperationMetricsFromCommitActions(deltaLog, version), operationMetrics = operationMetrics) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaNotSupportedDDLSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.Locale import scala.util.control.NonFatal import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.{AnalysisException, QueryTest} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession class DeltaNotSupportedDDLSuite extends DeltaNotSupportedDDLBase with SharedSparkSession with DeltaSQLCommandTest abstract class DeltaNotSupportedDDLBase extends QueryTest with DeltaSQLTestUtils { val format = "delta" val nonPartitionedTableName = "deltaTbl" val partitionedTableName = "partitionedTahoeTbl" protected override def beforeAll(): Unit = { super.beforeAll() try { sql(s""" |CREATE TABLE $nonPartitionedTableName |USING $format |AS SELECT 1 as a, 'a' as b """.stripMargin) sql(s""" |CREATE TABLE $partitionedTableName (a INT, b STRING, p1 INT) |USING $format |PARTITIONED BY (p1) """.stripMargin) sql(s"INSERT INTO $partitionedTableName SELECT 1, 'A', 2") } catch { case NonFatal(e) => afterAll() throw e } } protected override def afterAll(): Unit = { try { sql(s"DROP TABLE IF EXISTS $nonPartitionedTableName") sql(s"DROP TABLE IF EXISTS $partitionedTableName") } finally { super.afterAll() } } def assertUnsupported(query: String, messages: String*): Unit = { val allErrMessages = "operation not allowed" +: messages val e = intercept[AnalysisException] { sql(query) } assert(allErrMessages.exists(err => e.getMessage.toLowerCase(Locale.ROOT).contains(err))) } private def assertIgnored(query: String): Unit = { val outputStream = new java.io.ByteArrayOutputStream() Console.withOut(outputStream) { sql(query) } assert(outputStream.toString.contains("The request is ignored")) } test("bucketing is not supported for delta tables") { withTable("tbl") { assertUnsupported( s""" |CREATE TABLE tbl(a INT, b INT) |USING $format |CLUSTERED BY (a) INTO 5 BUCKETS """.stripMargin) } } test("ANALYZE TABLE PARTITION") { assertUnsupported( s"ANALYZE TABLE $partitionedTableName PARTITION (p1) COMPUTE STATISTICS", "not supported for v2 tables") } test("ALTER TABLE ADD PARTITION") { assertUnsupported( s"ALTER TABLE $partitionedTableName ADD PARTITION (p1=3)", "does not support partition management") } test("ALTER TABLE DROP PARTITION") { assertUnsupported( s"ALTER TABLE $partitionedTableName DROP PARTITION (p1=2)", "does not support partition management") } test("ALTER TABLE RECOVER PARTITIONS") { assertUnsupported( s"ALTER TABLE $partitionedTableName RECOVER PARTITIONS", "alter table ... recover partitions is not supported for v2 tables") assertUnsupported( s"MSCK REPAIR TABLE $partitionedTableName", "msck repair table is not supported for v2 tables") } test("ALTER TABLE SET SERDEPROPERTIES") { assertUnsupported( s"ALTER TABLE $nonPartitionedTableName SET SERDEPROPERTIES (s1=3)", "alter table ... set [serde|serdeproperties] is not supported for v2 tables") } test("LOAD DATA") { assertUnsupported( s"""LOAD DATA LOCAL INPATH '/path/to/home' INTO TABLE $nonPartitionedTableName""", "not supported for v2 tables") } test("INSERT OVERWRITE DIRECTORY") { assertUnsupported(s"INSERT OVERWRITE DIRECTORY '/path/to/home' USING $format VALUES (1, 'a')") } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaOptionSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.Locale // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.DeltaOptions.{OVERWRITE_SCHEMA_OPTION, PARTITION_OVERWRITE_MODE_OPTION} import org.apache.spark.sql.delta.actions.{Action, FileAction} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.FileNames import org.apache.commons.io.FileUtils import org.apache.parquet.format.CompressionCodec import org.apache.spark.sql.{AnalysisException, QueryTest} import org.apache.spark.sql.functions.lit import org.apache.spark.sql.internal.SQLConf.PARTITION_OVERWRITE_MODE import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.Utils class DeltaOptionSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { import testImplicits._ test("support for setting dataChange to false") { val tempDir = Utils.createTempDir() spark.range(100) .write .format("delta") .save(tempDir.toString) val df = spark.read.format("delta").load(tempDir.toString) df .write .format("delta") .mode("overwrite") .option("dataChange", "false") .save(tempDir.toString) val deltaLog = DeltaLog.forTable(spark, tempDir) val version = deltaLog.snapshot.version val commitActions = deltaLog.store .read(FileNames.unsafeDeltaFile(deltaLog.logPath, version), deltaLog.newDeltaHadoopConf()) .map(Action.fromJson) val fileActions = commitActions.collect { case a: FileAction => a } assert(fileActions.forall(!_.dataChange)) } test("dataChange is by default set to true") { val tempDir = Utils.createTempDir() spark.range(100) .write .format("delta") .save(tempDir.toString) val df = spark.read.format("delta").load(tempDir.toString) df .write .format("delta") .mode("overwrite") .save(tempDir.toString) val deltaLog = DeltaLog.forTable(spark, tempDir) val version = deltaLog.snapshot.version val commitActions = deltaLog.store .read(FileNames.unsafeDeltaFile(deltaLog.logPath, version), deltaLog.newDeltaHadoopConf()) .map(Action.fromJson) val fileActions = commitActions.collect { case a: FileAction => a } assert(fileActions.forall(_.dataChange)) } test("dataChange is set to false on metadata changing operation") { withTempDir { tempDir => // Initialize a table while having dataChange set to false. val e = intercept[AnalysisException] { spark.range(100) .write .format("delta") .option("dataChange", "false") .save(tempDir.getAbsolutePath) } assert(e.getMessage === DeltaErrors.unexpectedDataChangeException("Create a Delta table").getMessage) spark.range(100) .write .format("delta") .save(tempDir.getAbsolutePath) // Adding a new column to the existing table while having dataChange set to false. val e2 = intercept[AnalysisException] { val df = spark.read.format("delta").load(tempDir.getAbsolutePath) df.withColumn("id2", 'id + 1) .write .format("delta") .mode("overwrite") .option("mergeSchema", "true") .option("dataChange", "false") .save(tempDir.getAbsolutePath) } assert(e2.getMessage === DeltaErrors.unexpectedDataChangeException("Change the Delta table schema").getMessage) // Overwriting the schema of the existing table while having dataChange as false. val e3 = intercept[AnalysisException] { spark.range(50) .withColumn("id3", 'id + 1) .write .format("delta") .mode("overwrite") .option("dataChange", "false") .option("overwriteSchema", "true") .save(tempDir.getAbsolutePath) } assert(e3.getMessage === DeltaErrors.unexpectedDataChangeException("Overwrite the Delta table schema or " + "change the partition schema").getMessage) } } test("support the maxRecordsPerFile write option: path") { withTempDir { tempDir => val path = tempDir.getCanonicalPath withTable("maxRecordsPerFile") { spark.range(100) .write .format("delta") .option("maxRecordsPerFile", 5) .save(path) assert(FileUtils.listFiles(tempDir, Array("parquet"), false).size === 20) } } } test("support the maxRecordsPerFile write option: external table") { withTempDir { tempDir => val path = tempDir.getCanonicalPath withTable("maxRecordsPerFile") { spark.range(100) .write .format("delta") .option("maxRecordsPerFile", 5) .option("path", path) .saveAsTable("maxRecordsPerFile") assert(FileUtils.listFiles(tempDir, Array("parquet"), false).size === 20) } } } test("support the maxRecordsPerFile write option: v2 write") { withTempDir { tempDir => val path = tempDir.getCanonicalPath withTable("maxRecordsPerFile") { spark.range(100) .writeTo("maxRecordsPerFile") .using("delta") .option("maxRecordsPerFile", 5) .tableProperty("location", path) .create() assert(FileUtils.listFiles(tempDir, Array("parquet"), false).size === 20) } } } test("support no compression write option (defaults to snappy)") { withTempDir { tempDir => val path = tempDir.getCanonicalPath withTable("compression") { spark.range(100) .writeTo("compression") .using("delta") .tableProperty("location", path) .create() assert(FileUtils.listFiles(tempDir, Array("snappy.parquet"), false).size > 0) } } } // LZO and BROTLI left out as additional library dependencies needed val codecsAndSubExtensions = Seq( CompressionCodec.UNCOMPRESSED -> "", CompressionCodec.SNAPPY -> "snappy.", CompressionCodec.GZIP -> "gz.", CompressionCodec.LZ4 -> "lz4hadoop.", // CompressionCodec.LZ4_RAW -> "lz4raw.", // Support is not yet available in Spark 3.5 CompressionCodec.ZSTD -> "zstd." ) codecsAndSubExtensions.foreach { case (codec, subExt) => val codecName = codec.name().toLowerCase(Locale.ROOT) test(s"support compression codec '$codecName' as write option") { withTempDir { tempDir => val path = tempDir.getCanonicalPath withTable(s"compression_$codecName") { spark.range(100) .writeTo(s"compression_$codecName") .using("delta") .option("compression", codecName) .tableProperty("location", path) .create() assert(FileUtils.listFiles(tempDir, Array(s"${subExt}parquet"), false).size > 0) } } } } test("invalid compression write option") { withTempDir { tempDir => val path = tempDir.getCanonicalPath withTable("compression") { assert( intercept[java.lang.IllegalArgumentException] { spark.range(100) .writeTo("compression") .using("delta") .option("compression", "???") .tableProperty("location", path) .create() }.getMessage.nonEmpty) } } } for { invalidMode <- Seq("ADAPTIVE", null) } { test("DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED = true: " + s"partitionOverwriteMode is set to invalid value in options invalidMode=$invalidMode") { withSQLConf(DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED.key -> "true") { withTempDir { tempDir => val e = intercept[IllegalArgumentException] { Seq(1, 2, 3).toDF .withColumn("part", $"value" % 2) .write .format("delta") .partitionBy("part") .option("partitionOverwriteMode", invalidMode) .save(tempDir.getAbsolutePath) } assert(e.getMessage === DeltaErrors.illegalDeltaOptionException( PARTITION_OVERWRITE_MODE_OPTION, invalidMode, "must be 'STATIC' or 'DYNAMIC'" ).getMessage ) } } } } for { invalidMode <- Seq("ADAPTIVE", null) } { test("DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED = false: " + s"partitionOverwriteMode is set to invalid value in options invalidMode=$invalidMode") { // partitionOverwriteMode is ignored and no error is thrown withSQLConf(DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED.key -> "false") { withTempDir { tempDir => Seq(1, 2, 3).toDF .withColumn("part", $"value" % 2) .write .format("delta") .partitionBy("part") .option("partitionOverwriteMode", invalidMode) .save(tempDir.getAbsolutePath) } } } } test("overwriteSchema=true should be invalid with partitionOverwriteMode=dynamic") { withTempDir { tempDir => val e = intercept[DeltaIllegalArgumentException] { withSQLConf(DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED.key -> "true") { Seq(1, 2, 3).toDF .withColumn("part", $"value" % 2) .write .mode("overwrite") .format("delta") .partitionBy("part") .option(OVERWRITE_SCHEMA_OPTION, "true") .option(PARTITION_OVERWRITE_MODE_OPTION, "dynamic") .save(tempDir.getAbsolutePath) } } assert(e.getErrorClass == "DELTA_OVERWRITE_SCHEMA_WITH_DYNAMIC_PARTITION_OVERWRITE") } } test("overwriteSchema=true should be invalid with partitionOverwriteMode=dynamic, " + "saveAsTable") { withTable("temp") { val e = intercept[DeltaIllegalArgumentException] { withSQLConf(DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED.key -> "true") { Seq(1, 2, 3).toDF .withColumn("part", $"value" % 2) .write .mode("overwrite") .format("delta") .partitionBy("part") .option(OVERWRITE_SCHEMA_OPTION, "true") .option(PARTITION_OVERWRITE_MODE_OPTION, "dynamic") .saveAsTable("temp") } } assert(e.getErrorClass == "DELTA_OVERWRITE_SCHEMA_WITH_DYNAMIC_PARTITION_OVERWRITE") } } test("Prohibit spark.databricks.delta.dynamicPartitionOverwrite.enabled=false in " + "dynamic partition overwrite mode") { withTempDir { tempDir => withSQLConf(DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED.key -> "false") { var e = intercept[DeltaIllegalArgumentException] { Seq(1, 2, 3).toDF .withColumn("part", $"value" % 2) .write .format("delta") .partitionBy("part") .option("partitionOverwriteMode", "dynamic") .save(tempDir.getAbsolutePath) } assert(e.getErrorClass == "DELTA_DYNAMIC_PARTITION_OVERWRITE_DISABLED") withSQLConf(PARTITION_OVERWRITE_MODE.key -> "dynamic") { e = intercept[DeltaIllegalArgumentException] { Seq(1, 2, 3).toDF .withColumn("part", $"value" % 2) .write .format("delta") .partitionBy("part") .save(tempDir.getAbsolutePath) } } assert(e.getErrorClass == "DELTA_DYNAMIC_PARTITION_OVERWRITE_DISABLED") } } } for (createOrReplace <- Seq("CREATE OR REPLACE", "REPLACE")) { test(s"$createOrReplace table command should not respect " + "dynamic partition overwrite mode") { withTempDir { tempDir => Seq(0, 1).toDF .withColumn("key", $"value" % 2) .withColumn("stringColumn", lit("string")) .withColumn("part", $"value" % 2) .write .format("delta") .partitionBy("part") .save(tempDir.getAbsolutePath) withSQLConf(PARTITION_OVERWRITE_MODE.key -> "dynamic") { // Write only to one partition with a different schema type of stringColumn. sql( s""" |$createOrReplace TABLE delta.`${tempDir.getAbsolutePath}` |USING delta |PARTITIONED BY (part) |LOCATION '${tempDir.getAbsolutePath}' |AS SELECT -1 as value, 0 as part, 0 as stringColumn |""".stripMargin) assert(spark.read.format("delta").load(tempDir.getAbsolutePath).count() == 1, "Table should be fully replaced even with DPO mode enabled") } } } } // Same test as above but using DeltaWriter V2. test("create or replace table V2 should not respect dynamic partition overwrite mode") { withTable("temp") { Seq(0, 1).toDF .withColumn("part", $"value" % 2) .write .format("delta") .partitionBy("part") .saveAsTable("temp") withSQLConf(PARTITION_OVERWRITE_MODE.key -> "dynamic") { // Write to one partition only. Seq(0).toDF .withColumn("part", $"value" % 2) .writeTo("temp") .using("delta") .createOrReplace() assert(spark.read.format("delta").table("temp").count() == 1, "Table should be fully replaced even with DPO mode enabled") } } } // Same test as above but using saveAsTable. test("saveAsTable with overwrite should respect dynamic partition overwrite mode") { withTable("temp") { Seq(0, 1).toDF .withColumn("part", $"value" % 2) .write .format("delta") .partitionBy("part") .saveAsTable("temp") // Write to one partition only. Seq(0).toDF .withColumn("part", $"value" % 2) .write .mode("overwrite") .option(PARTITION_OVERWRITE_MODE_OPTION, "dynamic") .partitionBy("part") .format("delta") .saveAsTable("temp") assert(spark.read.format("delta").table("temp").count() == 2, "Table should keep the original partition with DPO mode enabled.") } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaParquetFileFormatSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.DataFrameUtils import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.files.TahoeLogFileIndex import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.parquet.format.converter.ParquetMetadataConverter import org.apache.parquet.hadoop.ParquetFileReader import org.apache.spark.sql.{DataFrame, Dataset, QueryTest} import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} import org.apache.spark.sql.test.SharedSparkSession trait DeltaParquetFileFormatSuiteBase extends QueryTest with SharedSparkSession with DeletionVectorsTestUtils with DeltaSQLCommandTest { import testImplicits._ /** Helper method to run the test with vectorized and non-vectorized Parquet readers */ protected def testWithBothParquetReaders(name: String)(f: => Any): Unit = { for { enableVectorizedParquetReader <- BOOLEAN_DOMAIN readColumnarBatchAsRows <- BOOLEAN_DOMAIN // don't run for the combination (vectorizedReader=false, readColumnarBathAsRows = false) // as the non-vectorized reader always generates and returns rows, unlike the vectorized // reader which internally generates columnar batches but can returns either columnar batches // or rows from the columnar batch depending upon the config. if enableVectorizedParquetReader || readColumnarBatchAsRows } { test(s"$name, with vectorized Parquet reader=$enableVectorizedParquetReader, " + s"with readColumnarBatchAsRows=$readColumnarBatchAsRows") { // Set the max code gen fields to 0 to force the vectorized Parquet reader generate rows // from columnar batches. val codeGenMaxFields = if (readColumnarBatchAsRows) "0" else "100" withSQLConf( "spark.sql.parquet.enableVectorizedReader" -> enableVectorizedParquetReader.toString, "spark.sql.codegen.maxFields" -> codeGenMaxFields) { f } } } } /** Helper method to generate a table with single Parquet file with multiple rowgroups */ protected def generateData(tablePath: String): Unit = { // This is to generate a Parquet file with two row groups hadoopConf().set("parquet.block.size", (1024 * 50).toString) // Keep the number of partitions to 1 to generate a single Parquet data file val df = Seq.range(0, 20000).toDF().repartition(1) df.write.format("delta").mode("append").save(tablePath) // Set DFS block size to be less than Parquet rowgroup size, to allow // the file split logic to kick-in, but gets turned off due to the // disabling of file splitting in DeltaParquetFileFormat when DVs are present. hadoopConf().set("dfs.block.size", (1024 * 20).toString) } protected def assertParquetHasMultipleRowGroups(filePath: Path): Unit = { val parquetMetadata = ParquetFileReader.readFooter( hadoopConf, filePath, ParquetMetadataConverter.NO_FILTER) assert(parquetMetadata.getBlocks.size() > 1) } protected def hadoopConf(): Configuration = { // scalastyle:off hadoopconfiguration // This is to generate a Parquet file with two row groups spark.sparkContext.hadoopConfiguration // scalastyle:on hadoopconfiguration } lazy val dvStore: DeletionVectorStore = DeletionVectorStore.createInstance(hadoopConf) } class DeltaParquetFileFormatSuite extends DeltaParquetFileFormatSuiteBase { import testImplicits._ override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX.key, "false") } // Read with deletion vectors has separate code paths based on vectorized Parquet // reader is enabled or not. Test both the combinations for { readIsRowDeletedCol <- BOOLEAN_DOMAIN readRowIndexCol <- BOOLEAN_DOMAIN enableDVs <- BOOLEAN_DOMAIN if (enableDVs && readIsRowDeletedCol) || !enableDVs } { testWithBothParquetReaders( s"isDeletionVectorsEnabled=$enableDVs, read DV metadata columns: " + s"with isRowDeletedCol=$readIsRowDeletedCol, " + s"with rowIndexCol=$readRowIndexCol") { withSQLConf(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> enableDVs.toString) { withTempDir { tempDir => val tablePath = tempDir.toString // Generate a table with one parquet file containing multiple row groups. generateData(tablePath) val deltaLog = DeltaLog.forTable(spark, tempDir) val metadata = deltaLog.snapshot.metadata // Add additional field that has the deleted row flag to existing data schema var readingSchema = metadata.schema if (readIsRowDeletedCol) { readingSchema = readingSchema.add(DeltaParquetFileFormat.IS_ROW_DELETED_STRUCT_FIELD) } if (readRowIndexCol) { readingSchema = readingSchema.add(DeltaParquetFileFormat.ROW_INDEX_STRUCT_FIELD) } // Fetch the only file in the DeltaLog snapshot val addFile = deltaLog.snapshot.allFiles.collect()(0) if (enableDVs) { removeRowsFromFile(deltaLog, addFile, Seq(0, 200, 300, 756, 10352, 19999)) } val addFilePath = addFile.absolutePath(deltaLog) assertParquetHasMultipleRowGroups(addFilePath) val deltaParquetFormat = new DeltaParquetFileFormat( deltaLog.snapshot.protocol, metadata, nullableRowTrackingConstantFields = false, nullableRowTrackingGeneratedFields = false, optimizationsEnabled = false, if (enableDVs) Some(tablePath) else None) val fileIndex = TahoeLogFileIndex(spark, deltaLog) val relation = HadoopFsRelation( fileIndex, fileIndex.partitionSchema, readingSchema, bucketSpec = None, deltaParquetFormat, options = Map.empty)(spark) val plan = LogicalRelation(relation) if (readIsRowDeletedCol) { val (deletedColumnValue, notDeletedColumnValue) = (1, 0) if (enableDVs) { // Select some rows that are deleted and some rows not deleted // Deleted row `value`: 0, 200, 300, 756, 10352, 19999 // Not deleted row `value`: 7, 900 checkDatasetUnorderly( DataFrameUtils.ofRows(spark, plan) .filter("value in (0, 7, 200, 300, 756, 900, 10352, 19999)") .select("value", DeltaParquetFileFormat.IS_ROW_DELETED_COLUMN_NAME) .as[(Int, Int)], (0, deletedColumnValue), (7, notDeletedColumnValue), (200, deletedColumnValue), (300, deletedColumnValue), (756, deletedColumnValue), (900, notDeletedColumnValue), (10352, deletedColumnValue), (19999, deletedColumnValue)) } else { checkDatasetUnorderly( DataFrameUtils.ofRows(spark, plan) .filter("value in (0, 7, 200, 300, 756, 900, 10352, 19999)") .select("value", DeltaParquetFileFormat.IS_ROW_DELETED_COLUMN_NAME) .as[(Int, Int)], (0, notDeletedColumnValue), (7, notDeletedColumnValue), (200, notDeletedColumnValue), (300, notDeletedColumnValue), (756, notDeletedColumnValue), (900, notDeletedColumnValue), (10352, notDeletedColumnValue), (19999, notDeletedColumnValue)) } } if (readRowIndexCol) { def rowIndexes(df: DataFrame): Set[Long] = { val colIndex = if (readIsRowDeletedCol) 2 else 1 df.collect().map(_.getLong(colIndex)).toSet } val df = DataFrameUtils.ofRows(spark, plan) assert(rowIndexes(df) === Seq.range(0, 20000).toSet) assert( rowIndexes( df.filter("value in (0, 7, 200, 300, 756, 900, 10352, 19999)")) === Seq(0, 7, 200, 300, 756, 900, 10352, 19999).toSet) } } } } } } class DeltaParquetFileFormatWithPredicatePushdownSuite extends DeltaParquetFileFormatSuiteBase { import testImplicits._ override def beforeAll(): Unit = { super.beforeAll() enableDeletionVectorsForAllSupportedOperations(spark) spark.conf.set(DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX.key, "true") } for { rowIndexFilterType <- Seq(RowIndexFilterType.IF_CONTAINED, RowIndexFilterType.IF_NOT_CONTAINED) } testWithBothParquetReaders("read DV metadata columns: " + s"with rowIndexFilterType=$rowIndexFilterType") { withTempDir { tempDir => val tablePath = tempDir.toString // Generate a table with one parquet file containing multiple row groups. generateData(tablePath) val deltaLog = DeltaLog.forTable(spark, tempDir) val metadata = deltaLog.update().metadata // Add additional field that has the deleted row flag to existing data schema val readingSchema = metadata.schema.add(DeltaParquetFileFormat.IS_ROW_DELETED_STRUCT_FIELD) // Fetch the only file in the DeltaLog snapshot val addFile = deltaLog.update().allFiles.collect()(0) removeRowsFromFile(deltaLog, addFile, Seq(0, 200, 300, 756, 10352, 19999)) val addFilePath = addFile.absolutePath(deltaLog) assertParquetHasMultipleRowGroups(addFilePath) val deltaParquetFormat = new DeltaParquetFileFormat( deltaLog.update().protocol, metadata, nullableRowTrackingConstantFields = false, nullableRowTrackingGeneratedFields = false, optimizationsEnabled = true, Some(tablePath)) val fileIndex = TahoeLogFileIndex(spark, deltaLog) val relation = HadoopFsRelation( fileIndex, fileIndex.partitionSchema, readingSchema, bucketSpec = None, deltaParquetFormat, options = Map.empty)(spark) val plan = LogicalRelation(relation) val planWithMetadataCol = plan.copy(output = plan.output :+ deltaParquetFormat.createFileMetadataCol()) val (deletedColumnValue, notDeletedColumnValue) = (1, 0) // Select some rows that are deleted and some rows not deleted // Deleted row `value`: 0, 200, 300, 756, 10352, 19999 // Not deleted row `value`: 7, 900 checkDatasetUnorderly( DataFrameUtils.ofRows(spark, planWithMetadataCol) .filter("value in (0, 7, 200, 300, 756, 900, 10352, 19999)") .select("value", DeltaParquetFileFormat.IS_ROW_DELETED_COLUMN_NAME) .as[(Int, Int)], (0, deletedColumnValue), (7, notDeletedColumnValue), (200, deletedColumnValue), (300, deletedColumnValue), (756, deletedColumnValue), (900, notDeletedColumnValue), (10352, deletedColumnValue), (19999, deletedColumnValue)) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaProtocolTransitionsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.actions.Protocol import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils.{TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.AlterTableDropFeatureDeltaCommand import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.QueryTest import org.apache.spark.sql.test.SharedSparkSession trait DeltaProtocolTransitionsBaseSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey, "false") } protected def testProtocolTransition( createTableColumns: Seq[(String, String)] = Seq.empty, createTableGeneratedColumns: Seq[(String, String, String)] = Seq.empty, createTableProperties: Seq[(String, String)] = Seq.empty, alterTableProperties: Seq[(String, String)] = Seq.empty, dropFeatures: Seq[TableFeature] = Seq.empty, expectedProtocol: Protocol): Unit = { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) val tableBuilder = io.delta.tables.DeltaTable.create(spark) tableBuilder.tableName(s"delta.`$dir`") createTableColumns.foreach { c => tableBuilder.addColumn(c._1, c._2) } createTableGeneratedColumns.foreach { c => val columnBuilder = io.delta.tables.DeltaTable.columnBuilder(spark, c._1) columnBuilder.dataType(c._2) columnBuilder.generatedAlwaysAs(c._3) tableBuilder.addColumn(columnBuilder.build()) } createTableProperties.foreach { p => tableBuilder.property(p._1, p._2) } tableBuilder.location(dir.getCanonicalPath) tableBuilder.execute() if (alterTableProperties.nonEmpty) { sql( s"""ALTER TABLE delta.`${deltaLog.dataPath}` |SET TBLPROPERTIES ( |${alterTableProperties.map(p => s"'${p._1}' = '${p._2}'").mkString(",")} |)""".stripMargin) } // Drop features. dropFeatures.foreach { f => sql(s"ALTER TABLE delta.`${deltaLog.dataPath}` DROP FEATURE ${f.name}") } assert(deltaLog.update().protocol === expectedProtocol) } } } class DeltaProtocolTransitionsSuite extends DeltaProtocolTransitionsBaseSuite { test("CREATE TABLE default protocol versions") { testProtocolTransition( expectedProtocol = Protocol(1, 2)) // Setting table versions overrides protocol versions. testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 1.toString)), expectedProtocol = Protocol(1, 1)) } test("CREATE TABLE normalization") { // Table features protocols without features are normalized to (1, 1). testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 3.toString), ("delta.minWriterVersion", 7.toString)), expectedProtocol = Protocol(1, 1)) // Default protocol is taken into account. testProtocolTransition( createTableProperties = Seq( (s"delta.feature.${TestRemovableWriterFeature.name}", "supported")), expectedProtocol = Protocol(1, 7).withFeatures(Seq( InvariantsTableFeature, AppendOnlyTableFeature, TestRemovableWriterFeature))) // Default protocol is not taken into account because we explicitly set the protocol versions. testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 3.toString), ("delta.minWriterVersion", 7.toString), (s"delta.feature.${TestRemovableWriterFeature.name}", "supported")), expectedProtocol = Protocol(1, 7).withFeature(TestRemovableWriterFeature)) // Reader version normalizes correctly. testProtocolTransition( createTableProperties = Seq( (s"delta.feature.${TestRemovableWriterFeature.name}", "supported"), (s"delta.feature.${ColumnMappingTableFeature.name}", "supported")), expectedProtocol = Protocol(2, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, TestRemovableWriterFeature, ColumnMappingTableFeature))) // Reader version denormalizes correctly. testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 7.toString), (s"delta.feature.${TestRemovableReaderWriterFeature.name}", "supported")), expectedProtocol = Protocol(3, 7).withFeature(TestRemovableReaderWriterFeature)) // Reader version denormalizes correctly. testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 2.toString), ("delta.minWriterVersion", 7.toString), (s"delta.feature.${TestRemovableReaderWriterFeature.name}", "supported")), expectedProtocol = Protocol(3, 7).withFeature(TestRemovableReaderWriterFeature)) } test("Setting partial versions") { testProtocolTransition( createTableProperties = Seq( ("delta.minWriterVersion", 3.toString)), expectedProtocol = Protocol(1, 3)) testProtocolTransition( alterTableProperties = Seq( ("delta.minWriterVersion", 3.toString)), expectedProtocol = Protocol(1, 3)) testProtocolTransition( createTableProperties = Seq( ("delta.minWriterVersion", 3.toString), (s"delta.feature.${DeletionVectorsTableFeature.name}", "supported")), expectedProtocol = Protocol(3, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, CheckConstraintsTableFeature, DeletionVectorsTableFeature))) testProtocolTransition( alterTableProperties = Seq( ("delta.minWriterVersion", 3.toString), (s"delta.feature.${DeletionVectorsTableFeature.name}", "supported")), expectedProtocol = Protocol(3, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, CheckConstraintsTableFeature, DeletionVectorsTableFeature))) } for ((readerVersion, writerVersion) <- Seq((2, 1), (2, 2), (2, 3), (2, 4), (1, 5))) test("Invalid legacy protocol normalization" + s" - invalidProtocol($readerVersion, $writerVersion)") { val expectedReaderVersion = 1 val expectedWriterVersion = Math.min(writerVersion, 4) withSQLConf(DeltaSQLConf.TABLE_FEATURES_TEST_FEATURES_ENABLED.key -> false.toString) { // Base case. testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", readerVersion.toString), ("delta.minWriterVersion", writerVersion.toString)), expectedProtocol = Protocol(expectedReaderVersion, expectedWriterVersion)) // Invalid legacy versions are normalized in default confs. withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> readerVersion.toString, DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> writerVersion.toString) { testProtocolTransition( expectedProtocol = Protocol(expectedReaderVersion, expectedWriterVersion)) } // Invalid legacy versions are normalized in alter table. testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 1.toString)), alterTableProperties = Seq( ("delta.minReaderVersion", readerVersion.toString), ("delta.minWriterVersion", writerVersion.toString)), expectedProtocol = Protocol(expectedReaderVersion, expectedWriterVersion)) } } test("ADD FEATURE normalization") { testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 1.toString)), alterTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 4.toString)), expectedProtocol = Protocol(1, 4)) testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 2.toString)), alterTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 4.toString)), expectedProtocol = Protocol(1, 4)) // Setting lower legacy versions is noop. testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 4.toString)), alterTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 2.toString)), expectedProtocol = Protocol(1, 4)) // Setting the same legacy versions is noop. testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 4.toString)), alterTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 4.toString)), expectedProtocol = Protocol(1, 4)) // Setting legacy versions is an ADD operation. testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 6.toString)), alterTableProperties = Seq( ("delta.minReaderVersion", 2.toString), ("delta.minWriterVersion", 5.toString)), expectedProtocol = Protocol(2, 6)) // The inverse of the above test. testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 2.toString), ("delta.minWriterVersion", 5.toString)), alterTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 6.toString)), expectedProtocol = Protocol(2, 6)) // Adding a legacy protocol to a table features protocol adds the features // of the former to the later. testProtocolTransition( createTableProperties = Seq( (s"delta.feature.${TestWriterFeature.name}", "supported")), alterTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 3.toString)), expectedProtocol = Protocol(1, 7).withFeatures(Seq( AppendOnlyTableFeature, CheckConstraintsTableFeature, InvariantsTableFeature, TestWriterFeature))) // Variation of the above. testProtocolTransition( createTableProperties = Seq( (s"delta.feature.${TestWriterFeature.name}", "supported"), (s"delta.feature.${IdentityColumnsTableFeature.name}", "supported")), alterTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 3.toString)), expectedProtocol = Protocol(1, 7).withFeatures(Seq( AppendOnlyTableFeature, CheckConstraintsTableFeature, InvariantsTableFeature, IdentityColumnsTableFeature, TestWriterFeature))) // New feature is added to the table protocol features. testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 3.toString)), alterTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 7.toString), (DeltaConfigs.CHANGE_DATA_FEED.key, true.toString)), expectedProtocol = Protocol(1, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, CheckConstraintsTableFeature, ChangeDataFeedTableFeature))) // Addition result is normalized. testProtocolTransition( createTableProperties = Seq( (s"delta.feature.${InvariantsTableFeature.name}", "supported")), alterTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 2.toString)), expectedProtocol = Protocol(1, 2)) // Variation of the above. testProtocolTransition( createTableProperties = Seq( (s"delta.feature.${CheckConstraintsTableFeature.name}", "supported")), alterTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 2.toString)), expectedProtocol = Protocol(1, 3)) testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 2.toString)), alterTableProperties = Seq( (s"delta.feature.${CheckConstraintsTableFeature.name}", "supported")), expectedProtocol = Protocol(1, 3)) withSQLConf(DeltaSQLConf.TABLE_FEATURES_TEST_FEATURES_ENABLED.key -> false.toString) { testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 4.toString)), alterTableProperties = Seq( (s"delta.feature.${ColumnMappingTableFeature.name}", "supported")), expectedProtocol = Protocol(2, 5)) testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 4.toString)), alterTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 7.toString), (DeltaConfigs.COLUMN_MAPPING_MODE.key, "name")), expectedProtocol = Protocol(2, 5)) } } for (withFastDropFeature <- BOOLEAN_DOMAIN) test(s"DROP FEATURE normalization. withFastDropFeature: $withFastDropFeature") { withSQLConf(DeltaSQLConf.FAST_DROP_FEATURE_ENABLED.key -> withFastDropFeature.toString) { // Can drop features on legacy protocols and the result is normalized. testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 3.toString)), dropFeatures = Seq(CheckConstraintsTableFeature), expectedProtocol = Protocol(1, 2)) // If the removal result does not match a legacy version use the denormalized form. testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 4.toString)), dropFeatures = Seq(CheckConstraintsTableFeature), expectedProtocol = Protocol(1, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, GeneratedColumnsTableFeature, ChangeDataFeedTableFeature))) // Normalization after dropping a table feature. testProtocolTransition( createTableProperties = Seq( (s"delta.feature.${TestRemovableWriterFeature.name}", "supported")), dropFeatures = Seq(TestRemovableWriterFeature), expectedProtocol = Protocol(1, 2)) // Variation of the above. Because the default protocol is overwritten the result // is normalized to (1, 1). testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 7.toString), (s"delta.feature.${TestRemovableWriterFeature.name}", "supported")), dropFeatures = Seq(TestRemovableWriterFeature), expectedProtocol = Protocol(1, 1)) // Reader version is normalized correctly to 2 after dropping the reader feature. val checkpointProtectionTableFeatureOpt = if (withFastDropFeature) Some(CheckpointProtectionTableFeature) else None testProtocolTransition( createTableProperties = Seq( (s"delta.feature.${ColumnMappingTableFeature.name}", "supported"), (s"delta.feature.${TestRemovableWriterFeature.name}", "supported"), (s"delta.feature.${TestRemovableReaderWriterFeature.name}", "supported")), dropFeatures = Seq(TestRemovableReaderWriterFeature), expectedProtocol = Protocol(2, 7).withFeatures(Seq( InvariantsTableFeature, AppendOnlyTableFeature, ColumnMappingTableFeature, TestRemovableWriterFeature) ++ checkpointProtectionTableFeatureOpt)) testProtocolTransition( createTableProperties = Seq( (s"delta.feature.${TestRemovableWriterFeature.name}", "supported"), (s"delta.feature.${TestRemovableReaderWriterFeature.name}", "supported")), dropFeatures = Seq(TestRemovableReaderWriterFeature), expectedProtocol = Protocol(1, 7).withFeatures(Seq( InvariantsTableFeature, AppendOnlyTableFeature, TestRemovableWriterFeature) ++ checkpointProtectionTableFeatureOpt)) val expectedProtocol = if (withFastDropFeature) { Protocol(1, 7).withFeatures(Seq( InvariantsTableFeature, AppendOnlyTableFeature, CheckConstraintsTableFeature, ChangeDataFeedTableFeature, GeneratedColumnsTableFeature, CheckpointProtectionTableFeature)) } else { Protocol(1, 4) } withSQLConf(DeltaSQLConf.TABLE_FEATURES_TEST_FEATURES_ENABLED.key -> false.toString) { testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 2.toString), ("delta.minWriterVersion", 5.toString)), dropFeatures = Seq(ColumnMappingTableFeature), expectedProtocol = expectedProtocol) } } } test("Default Enabled native features") { withSQLConf(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> "true") { // Table protocol is taken into account when default table features exist. testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 4.toString)), expectedProtocol = Protocol(3, 7).withFeatures(Seq( DeletionVectorsTableFeature, InvariantsTableFeature, AppendOnlyTableFeature, CheckConstraintsTableFeature, ChangeDataFeedTableFeature, GeneratedColumnsTableFeature))) // Default protocol versions are taken into account when default features exist. testProtocolTransition( expectedProtocol = Protocol(3, 7).withFeatures(Seq( DeletionVectorsTableFeature, InvariantsTableFeature, AppendOnlyTableFeature))) } withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> 1.toString, DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> 7.toString, DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> "true") { testProtocolTransition( expectedProtocol = Protocol(3, 7).withFeature(DeletionVectorsTableFeature)) } } test("Default Enabled legacy features") { testProtocolTransition( createTableProperties = Seq((DeltaConfigs.CHANGE_DATA_FEED.key, true.toString)), expectedProtocol = Protocol(1, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, ChangeDataFeedTableFeature))) testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 3.toString), (DeltaConfigs.CHANGE_DATA_FEED.key, true.toString)), expectedProtocol = Protocol(1, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, CheckConstraintsTableFeature, ChangeDataFeedTableFeature))) testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 4.toString), (DeltaConfigs.CHANGE_DATA_FEED.key, true.toString)), expectedProtocol = Protocol(1, 4)) testProtocolTransition( alterTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 4.toString), (DeltaConfigs.CHANGE_DATA_FEED.key, true.toString)), expectedProtocol = Protocol(1, 4)) withSQLConf(DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> "true") { testProtocolTransition( expectedProtocol = Protocol(1, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, ChangeDataFeedTableFeature))) } testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 7.toString), (DeltaConfigs.CHANGE_DATA_FEED.key, true.toString)), expectedProtocol = Protocol(1, 7).withFeature(ChangeDataFeedTableFeature)) withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> 1.toString, DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> 7.toString, DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> "true") { testProtocolTransition( expectedProtocol = Protocol(1, 7).withFeature(ChangeDataFeedTableFeature)) } withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> 1.toString, DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> 7.toString) { testProtocolTransition( createTableProperties = Seq((DeltaConfigs.CHANGE_DATA_FEED.key, true.toString)), expectedProtocol = Protocol(1, 7).withFeature(ChangeDataFeedTableFeature)) } withSQLConf(DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> "true") { testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 7.toString)), expectedProtocol = Protocol(1, 7).withFeature(ChangeDataFeedTableFeature)) } } test("Enabling legacy features on a table") { testProtocolTransition( createTableColumns = Seq(("id", "INT")), createTableGeneratedColumns = Seq(("id2", "INT", "id + 1")), expectedProtocol = Protocol(1, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, GeneratedColumnsTableFeature))) testProtocolTransition( createTableColumns = Seq(("id", "INT")), createTableGeneratedColumns = Seq(("id2", "INT", "id + 1")), createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 7.toString)), expectedProtocol = Protocol(1, 7).withFeature(GeneratedColumnsTableFeature)) withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> 1.toString, DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> 7.toString) { testProtocolTransition( createTableColumns = Seq(("id", "INT")), createTableGeneratedColumns = Seq(("id2", "INT", "id + 1")), expectedProtocol = Protocol(1, 7).withFeature(GeneratedColumnsTableFeature)) } testProtocolTransition( alterTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 7.toString), (DeltaConfigs.CHANGE_DATA_FEED.key, true.toString)), expectedProtocol = Protocol(1, 7).withFeatures(Seq( InvariantsTableFeature, AppendOnlyTableFeature, ChangeDataFeedTableFeature))) } test("Column Mapping does not require a manual protocol versions upgrade") { testProtocolTransition( createTableProperties = Seq((DeltaConfigs.COLUMN_MAPPING_MODE.key, "name")), expectedProtocol = Protocol(2, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, ColumnMappingTableFeature))) withSQLConf(DeltaSQLConf.TABLE_FEATURES_TEST_FEATURES_ENABLED.key -> false.toString) { testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 4.toString), (DeltaConfigs.COLUMN_MAPPING_MODE.key, "name")), expectedProtocol = Protocol(2, 5)) testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 4.toString)), alterTableProperties = Seq( (DeltaConfigs.COLUMN_MAPPING_MODE.key, "name")), expectedProtocol = Protocol(2, 5)) } testProtocolTransition( createTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 7.toString), (DeltaConfigs.COLUMN_MAPPING_MODE.key, "name")), expectedProtocol = Protocol(2, 7).withFeature(ColumnMappingTableFeature)) withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> 1.toString, DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> 7.toString) { testProtocolTransition( createTableProperties = Seq((DeltaConfigs.COLUMN_MAPPING_MODE.key, "name")), expectedProtocol = Protocol(2, 7).withFeature(ColumnMappingTableFeature)) } testProtocolTransition( alterTableProperties = Seq((DeltaConfigs.COLUMN_MAPPING_MODE.key, "name")), expectedProtocol = Protocol(2, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, ColumnMappingTableFeature))) testProtocolTransition( alterTableProperties = Seq( ("delta.minReaderVersion", 1.toString), ("delta.minWriterVersion", 7.toString), (DeltaConfigs.COLUMN_MAPPING_MODE.key, "name")), expectedProtocol = Protocol(2, 7).withFeatures(Seq( InvariantsTableFeature, AppendOnlyTableFeature, ColumnMappingTableFeature))) } private def validVersions = Seq((1, 1), (1, 2), (1, 3), (1, 4), (2, 5), (1, 7), (3, 7)) private def invalidVersions = Seq((2, 2), (2, 3)) for ((readerVersion, writerVersion) <- validVersions ++ invalidVersions) test("Legacy features are added when setting legacy versions: " + s"readerVersionToSet = $readerVersion, writerVersionToSet = $writerVersion") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) // Creates a table with (1, 7) versions with the given table feature. sql( s"""CREATE TABLE delta.`${deltaLog.dataPath}` (id bigint) USING delta |TBLPROPERTIES ( |delta.feature.${TestRemovableWriterFeature.name} = 'supported' |)""".stripMargin) sql( s""" |ALTER TABLE delta.`${deltaLog.dataPath}` SET TBLPROPERTIES ( | 'delta.minReaderVersion' = $readerVersion, | 'delta.minWriterVersion' = $writerVersion |)""".stripMargin) val expected = Protocol(readerVersion, writerVersion).implicitlySupportedFeatures ++ Set(InvariantsTableFeature, AppendOnlyTableFeature, TestRemovableWriterFeature) assert(deltaLog.update().protocol.readerAndWriterFeatureNames === expected.map(_.name)) } } for { tableFeatureToAdd <- Seq(TestRemovableWriterFeature, TestRemovableReaderWriterFeature) downgradeVersionToSet <- Seq(1, 2, 3, 4, 5, 6) preemptiveVersionDowngrade <- BOOLEAN_DOMAIN } test("Protocol versions are always downgraded to the minimum required " + s"tableFeatureToAdd: ${tableFeatureToAdd.name}, " + s"downgradeVersionToSet: $downgradeVersionToSet, " + s"preemptiveVersionDowngrade: $preemptiveVersionDowngrade") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) sql( s"""CREATE TABLE delta.`${deltaLog.dataPath}` (id bigint) USING delta |TBLPROPERTIES ( |delta.minReaderVersion = ${Math.max(tableFeatureToAdd.minReaderVersion, 1)}, |delta.minWriterVersion = $TABLE_FEATURES_MIN_WRITER_VERSION, |delta.feature.${tableFeatureToAdd.name} = 'supported', |delta.feature.${ChangeDataFeedTableFeature.name} = 'supported' |)""".stripMargin) val downgradeProtocolVersionsSQL = s""" |ALTER TABLE delta.`${deltaLog.dataPath}` SET TBLPROPERTIES ( | 'delta.minReaderVersion' = 1, | 'delta.minWriterVersion' = $downgradeVersionToSet |)""".stripMargin if (preemptiveVersionDowngrade) sql(downgradeProtocolVersionsSQL) AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), tableFeatureToAdd.name, truncateHistory = tableFeatureToAdd.isReaderWriterFeature).run(spark) if (!preemptiveVersionDowngrade) sql(downgradeProtocolVersionsSQL) val expectedProtocol = if (downgradeVersionToSet < 4) { Protocol(tableFeatureToAdd.minReaderVersion, 7).withFeature(ChangeDataFeedTableFeature) .merge(Protocol(1, downgradeVersionToSet)) } else { Protocol(1, downgradeVersionToSet) } assert(deltaLog.update().protocol === expectedProtocol) } } for { tableFeatureToAdd <- Seq(TestRemovableWriterFeature, TestRemovableReaderWriterFeature) setLegacyVersions <- BOOLEAN_DOMAIN downgradeAfterDrop <- if (setLegacyVersions) BOOLEAN_DOMAIN else Seq(false) } test("SOP for downgrading to legacy protocol versions for tables created with features. " + s"tableFeatureToAdd: ${tableFeatureToAdd.name}, setLegacyVersions: $setLegacyVersions, " + s"downgradeAfterDrop: ${downgradeAfterDrop}") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) sql( s"""CREATE TABLE delta.`${deltaLog.dataPath}` (id bigint) USING delta |TBLPROPERTIES ( |delta.minReaderVersion = $TABLE_FEATURES_MIN_READER_VERSION, |delta.minWriterVersion = $TABLE_FEATURES_MIN_WRITER_VERSION, |delta.feature.${tableFeatureToAdd.name} = 'supported', |delta.feature.${ChangeDataFeedTableFeature.name} = 'supported' |)""".stripMargin) val downgradeProtocolVersionsSQL = s""" |ALTER TABLE delta.`${deltaLog.dataPath}` SET TBLPROPERTIES ( | 'delta.minReaderVersion' = 1, | 'delta.minWriterVersion' = 4 |)""".stripMargin if (setLegacyVersions && !downgradeAfterDrop) sql(downgradeProtocolVersionsSQL) AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), tableFeatureToAdd.name, truncateHistory = tableFeatureToAdd.isReaderWriterFeature).run(spark) if (setLegacyVersions && downgradeAfterDrop) sql(downgradeProtocolVersionsSQL) val expectedProtocol = if (setLegacyVersions) { Protocol(1, 4) } else { Protocol(1, TABLE_FEATURES_MIN_WRITER_VERSION).withFeature(ChangeDataFeedTableFeature) } assert(deltaLog.update().protocol === expectedProtocol) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaProtocolVersionSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.io.File import java.nio.file.{Files, Paths, StandardOpenOption} import java.util.Locale import java.util.concurrent.TimeUnit import scala.collection.JavaConverters._ import com.databricks.spark.util.{Log4jUsageLogger, MetricDefinitions, UsageRecord} import org.apache.spark.sql.delta.DeltaOperations.ManualUpdate import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils._ import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.{AlterTableDropFeatureDeltaCommand, AlterTableSetPropertiesDeltaCommand, AlterTableUnsetPropertiesDeltaCommand} import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.coordinatedcommits._ import org.apache.spark.sql.delta.redirect.{PathBasedRedirectSpec, RedirectReaderWriter, RedirectWriterOnly, TableRedirect} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.FileNames import org.apache.spark.sql.delta.util.FileNames.{unsafeDeltaFile, DeltaFile} import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.sql.delta.test.shims.StreamingTestShims.MemoryStream import io.delta.storage.LogStore import io.delta.storage.commit.TableDescriptor import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.sql.{AnalysisException, QueryTest, SaveMode} import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.StructType import org.apache.spark.unsafe.types.CalendarInterval import org.apache.spark.util.ManualClock trait DeltaProtocolVersionSuiteBase extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with DeletionVectorsTestUtils { // `.schema` generates NOT NULL columns which requires writer protocol 2. We convert all to // NULLable to avoid silent writer protocol version bump. private lazy val testTableSchema = spark.range(1).schema.asNullable override protected def sparkConf: SparkConf = { // All the drop feature tests below are targeting the drop feature with history truncation // implementation. The fast drop feature implementation is tested extensively in // DeltaFastDropFeatureSuite. super.sparkConf.set(DeltaSQLConf.FAST_DROP_FEATURE_ENABLED.key, "false") } // This is solely a test hook. Users cannot create new Delta tables with protocol lower than // that of their current version. protected def createTableWithProtocol( protocol: Protocol, path: File, schema: StructType = testTableSchema): DeltaLog = { val log = DeltaLog.forTable(spark, path) log.createLogDirectoriesIfNotExists() log.store.write( unsafeDeltaFile(log.logPath, 0), Iterator(Metadata(schemaString = schema.json).json, protocol.json), overwrite = false, log.newDeltaHadoopConf()) log.update() log } test("protocol for empty folder") { def testEmptyFolder( readerVersion: Int, writerVersion: Int, features: Iterable[TableFeature] = Seq.empty, sqlConfs: Iterable[(String, String)] = Seq.empty, expectedProtocol: Protocol): Unit = { withTempDir { path => val configs = Seq( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> readerVersion.toString, DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> writerVersion.toString) ++ features.map(defaultPropertyKey(_) -> FEATURE_PROP_ENABLED) ++ sqlConfs withSQLConf(configs: _*) { val log = DeltaLog.forTable(spark, path) assert(log.update().protocol === expectedProtocol) } } } testEmptyFolder(1, 1, expectedProtocol = Protocol(1, 1)) testEmptyFolder(1, 2, expectedProtocol = Protocol(1, 2)) testEmptyFolder( readerVersion = 1, writerVersion = 1, sqlConfs = Seq((DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey, "true")), expectedProtocol = Protocol(1, 7).withFeature(ChangeDataFeedTableFeature)) testEmptyFolder( readerVersion = 1, writerVersion = 1, features = Seq(TestLegacyReaderWriterFeature), expectedProtocol = Protocol(2, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestLegacyReaderWriterFeature)) testEmptyFolder( readerVersion = 1, writerVersion = 1, features = Seq(TestWriterFeature), expectedProtocol = Protocol(1, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestWriterFeature)) testEmptyFolder( readerVersion = TABLE_FEATURES_MIN_READER_VERSION, writerVersion = TABLE_FEATURES_MIN_WRITER_VERSION, features = Seq(TestLegacyReaderWriterFeature), expectedProtocol = Protocol(2, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestLegacyReaderWriterFeature)) testEmptyFolder( readerVersion = 1, writerVersion = 1, features = Seq(TestWriterFeature), sqlConfs = Seq((DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey, "true")), expectedProtocol = Protocol(1, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(Seq(TestWriterFeature, ChangeDataFeedTableFeature))) testEmptyFolder( readerVersion = 1, writerVersion = 1, features = Seq(TestLegacyReaderWriterFeature), sqlConfs = Seq((DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey, "true")), expectedProtocol = Protocol(2, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(Seq(TestLegacyReaderWriterFeature, ChangeDataFeedTableFeature))) testEmptyFolder( readerVersion = 1, writerVersion = 1, features = Seq(TestWriterFeature, TestLegacyReaderWriterFeature), expectedProtocol = Protocol(2, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(Seq(TestWriterFeature, TestLegacyReaderWriterFeature))) } test("upgrade to current version") { withTempDir { path => val log = createTableWithProtocol(Protocol(1, 1), path) assert(log.snapshot.protocol === Protocol(1, 1)) log.upgradeProtocol(Action.supportedProtocolVersion( featuresToExclude = Seq(CatalogOwnedTableFeature))) assert(log.snapshot.protocol === Action.supportedProtocolVersion( featuresToExclude = Seq(CatalogOwnedTableFeature))) } } test("upgrade to a version with DeltaTable API") { withTempDir { path => val log = createTableWithProtocol(Protocol(0, 0), path) assert(log.snapshot.protocol === Protocol(0, 0)) val table = io.delta.tables.DeltaTable.forPath(spark, path.getCanonicalPath) table.upgradeTableProtocol(1, 1) assert(log.snapshot.protocol === Protocol(1, 1)) table.upgradeTableProtocol(1, 2) assert(log.snapshot.protocol === Protocol(1, 2)) table.upgradeTableProtocol(1, 3) assert(log.snapshot.protocol === Protocol(1, 3)) intercept[DeltaTableFeatureException] { table.upgradeTableProtocol( TABLE_FEATURES_MIN_READER_VERSION, writerVersion = 1) } intercept[IllegalArgumentException] { table.upgradeTableProtocol( TABLE_FEATURES_MIN_READER_VERSION + 1, TABLE_FEATURES_MIN_WRITER_VERSION) } intercept[IllegalArgumentException] { table.upgradeTableProtocol( TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION + 1) } } } test("upgrade to support table features - no feature") { // Setting a table feature versions to a protocol without table features is a noop. withTempDir { path => val log = createTableWithProtocol(Protocol(1, 1), path) assert(log.update().protocol === Protocol(1, 1)) val table = io.delta.tables.DeltaTable.forPath(spark, path.getCanonicalPath) table.upgradeTableProtocol(1, TABLE_FEATURES_MIN_WRITER_VERSION) assert(log.update().protocol === Protocol(1, 1)) table.upgradeTableProtocol( TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) assert(log.update().protocol === Protocol(1, 1)) } } test("upgrade to support table features - writer-only feature") { // Setting table feature versions to a protocol without table features is a noop. withTempDir { path => val log = createTableWithProtocol(Protocol(1, 2), path) assert(log.update().protocol === Protocol(1, 2)) val table = io.delta.tables.DeltaTable.forPath(spark, path.getCanonicalPath) table.upgradeTableProtocol(1, TABLE_FEATURES_MIN_WRITER_VERSION) assert(log.update().protocol === Protocol(1, 2)) table.upgradeTableProtocol( TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) assert(log.update().protocol === Protocol(1, 2)) } } test("upgrade to support table features - many features") { withTempDir { path => val log = createTableWithProtocol(Protocol(2, 5), path) assert(log.update().protocol === Protocol(2, 5)) val table = io.delta.tables.DeltaTable.forPath(spark, path.getCanonicalPath) table.upgradeTableProtocol(2, TABLE_FEATURES_MIN_WRITER_VERSION) // Setting table feature versions to a protocol without table features is a noop. assert(log.update().protocol === Protocol(2, 5)) spark.sql( s"ALTER TABLE delta.`${path.getPath}` SET TBLPROPERTIES (" + s" delta.feature.${TestWriterFeature.name}='enabled'" + s")") table.upgradeTableProtocol( TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) assert( log.snapshot.protocol === Protocol( minReaderVersion = 2, minWriterVersion = TABLE_FEATURES_MIN_WRITER_VERSION, readerFeatures = None, writerFeatures = Some( Set( AppendOnlyTableFeature, ChangeDataFeedTableFeature, CheckConstraintsTableFeature, ColumnMappingTableFeature, GeneratedColumnsTableFeature, InvariantsTableFeature, TestLegacyWriterFeature, TestRemovableLegacyWriterFeature, TestLegacyReaderWriterFeature, TestRemovableLegacyReaderWriterFeature, TestWriterFeature) .map(_.name)))) } } test("protocol upgrade using SQL API") { withTempDir { path => val log = createTableWithProtocol(Protocol(1, 2), path) assert(log.update().protocol === Protocol(1, 2)) sql( s"ALTER TABLE delta.`${path.getCanonicalPath}` " + "SET TBLPROPERTIES (delta.minWriterVersion = 3)") assert(log.update().protocol === Protocol(1, 3)) assertPropertiesAndShowTblProperties(log) sql(s"ALTER TABLE delta.`${path.getCanonicalPath}` " + s"SET TBLPROPERTIES (delta.minWriterVersion=$TABLE_FEATURES_MIN_WRITER_VERSION)") assert(log.update().protocol === Protocol(1, 3)) assertPropertiesAndShowTblProperties(log, tableHasFeatures = false) sql(s"""ALTER TABLE delta.`${path.getCanonicalPath}` SET TBLPROPERTIES ( |delta.minReaderVersion=$TABLE_FEATURES_MIN_READER_VERSION, |delta.minWriterVersion=$TABLE_FEATURES_MIN_WRITER_VERSION |)""".stripMargin) assert(log.update().protocol === Protocol(1, 3)) assertPropertiesAndShowTblProperties(log, tableHasFeatures = false) } } test("overwrite keeps the same protocol version") { withTempDir { path => val log = createTableWithProtocol(Protocol(0, 0), path) spark.range(1) .write .format("delta") .mode("overwrite") .save(path.getCanonicalPath) log.update() assert(log.snapshot.protocol === Protocol(0, 0)) } } test("overwrite keeps the same table properties") { withTempDir { path => val log = createTableWithProtocol(Protocol(0, 0), path) spark.sql( s"ALTER TABLE delta.`${path.getCanonicalPath}` SET TBLPROPERTIES ('myProp'='true')") spark .range(1) .write .format("delta") .option("anotherProp", "true") .mode("overwrite") .save(path.getCanonicalPath) log.update() assert(log.snapshot.metadata.configuration.size === 1) assert(log.snapshot.metadata.configuration("myProp") === "true") } } test("overwrite keeps the same protocol version and features") { withTempDir { path => val protocol = Protocol(0, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(AppendOnlyTableFeature) val log = createTableWithProtocol(protocol, path) spark .range(1) .write .format("delta") .mode("overwrite") .save(path.getCanonicalPath) log.update() assert(log.snapshot.protocol === protocol) } } test("overwrite with additional configs keeps the same protocol version and features") { withTempDir { path => val protocol = Protocol(1, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(AppendOnlyTableFeature) val log = createTableWithProtocol(protocol, path) spark .range(1) .write .format("delta") .option("delta.feature.testWriter", "enabled") .option("delta.feature.testReaderWriter", "enabled") .mode("overwrite") .save(path.getCanonicalPath) log.update() assert(log.snapshot.protocol === protocol) } } test("overwrite with additional session defaults keeps the same protocol version and features") { withTempDir { path => val protocol = Protocol(1, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(AppendOnlyTableFeature) val log = createTableWithProtocol(protocol, path) withSQLConf( s"$DEFAULT_FEATURE_PROP_PREFIX${TestLegacyWriterFeature.name}" -> "enabled") { spark .range(1) .write .format("delta") .option("delta.feature.testWriter", "enabled") .option("delta.feature.testReaderWriter", "enabled") .mode("overwrite") .save(path.getCanonicalPath) } log.update() assert(log.snapshot.protocol === protocol) } } test("access with protocol too high") { withTempDir { path => val log = DeltaLog.forTable(spark, path) log.createLogDirectoriesIfNotExists() log.store.write( unsafeDeltaFile(log.logPath, 0), Iterator(Metadata().json, Protocol(Integer.MAX_VALUE, Integer.MAX_VALUE).json), overwrite = false, log.newDeltaHadoopConf()) intercept[InvalidProtocolVersionException] { spark.range(1).write.format("delta").save(path.getCanonicalPath) } } } test("Vacuum checks the write protocol") { withTempDir { path => spark.range(10).write.format("delta").save(path.getCanonicalPath) val log = DeltaLog.forTable(spark, path) sql(s"INSERT INTO delta.`${path.getCanonicalPath}` VALUES (10)") val vacuumCommandsToTry = Seq( s"vacuum delta.`${path.getCanonicalPath}` RETAIN 10000 HOURS", s"vacuum delta.`${path.getCanonicalPath}` RETAIN 10000 HOURS DRY RUN" ) // Both vacuum and vacuum dry run works as expected vacuumCommandsToTry.foreach(spark.sql(_).collect()) val snapshot = log.update() val newProtocol = Protocol( TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION).withWriterFeatures(Seq("newUnsupportedWriterFeature")) log.store.write( unsafeDeltaFile(log.logPath, snapshot.version + 1), Iterator(Metadata().json, newProtocol.json), overwrite = false, log.newDeltaHadoopConf()) // Both vacuum and vacuum dry run fails as expected vacuumCommandsToTry.foreach { command => intercept[DeltaUnsupportedTableFeatureException] { spark.sql(command).collect() } } } } test("InvalidProtocolVersionException - error message with protocol too high - table path") { withTempDir { path => spark.range(1).write.format("delta").save(path.getCanonicalPath) val (deltaLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, path.getCanonicalPath) var tableReaderVersion = 4 var tableWriterVersion = 7 var version = snapshot.version + 1 untrackedChangeProtocolVersion(deltaLog, version, tableReaderVersion, tableWriterVersion) val exceptionRead = intercept[InvalidProtocolVersionException] { spark.read.format("delta").load(path.getCanonicalPath) } validateInvalidProtocolVersionException( exceptionRead, deltaLog.dataPath.toString, tableReaderVersion, tableWriterVersion) tableReaderVersion = 3 tableWriterVersion = 8 version = version + 1 untrackedChangeProtocolVersion(deltaLog, version, tableReaderVersion, tableWriterVersion) val exceptionWrite = intercept[InvalidProtocolVersionException] { spark.range(1).write .mode("append") .option("mergeSchema", "true") .format("delta") .save(path.getCanonicalPath) } validateInvalidProtocolVersionException( exceptionWrite, deltaLog.dataPath.toString, tableReaderVersion, tableWriterVersion) } } def testInvalidProtocolErrorMessageWithTableName(warm: Boolean): Unit = { val protocolTableName = "mytableprotocoltoohigh" withTable(protocolTableName) { spark.range(1).write.format("delta").saveAsTable(protocolTableName) val (deltaLog, snapshot) = DeltaLog.forTableWithSnapshot( spark, TableIdentifier(protocolTableName)) var tableReaderVersion = 4 var tableWriterVersion = 7 var version = snapshot.version + 1 untrackedChangeProtocolVersion(deltaLog, version, tableReaderVersion, tableWriterVersion) if (!warm) { DeltaLog.clearCache() } val exceptionRead = intercept[InvalidProtocolVersionException] { spark.read.format("delta").table(protocolTableName) } var pathInErrorMessage = "default." + protocolTableName validateInvalidProtocolVersionException( exceptionRead, pathInErrorMessage, tableReaderVersion, tableWriterVersion) tableReaderVersion = 3 tableWriterVersion = 8 version = version + 1 untrackedChangeProtocolVersion(deltaLog, version, tableReaderVersion, tableWriterVersion) if (!warm) { DeltaLog.clearCache() } val exceptionWrite = intercept[InvalidProtocolVersionException] { spark.range(1).write .mode("append") .option("mergeSchema", "true") .format("delta") .saveAsTable(protocolTableName) } validateInvalidProtocolVersionException( exceptionWrite, pathInErrorMessage, tableReaderVersion, tableWriterVersion) // Restore the protocol version or the clean-up fails version = version + 1 untrackedChangeProtocolVersion(deltaLog, version, 1, 2) } } test("InvalidProtocolVersionException - error message with table name - warm") { testInvalidProtocolErrorMessageWithTableName(true) } test("InvalidProtocolVersionException - error message with table name - cold") { testInvalidProtocolErrorMessageWithTableName(false) } test("InvalidProtocolVersionException - " + "incompatible protocol change during the transaction - table name") { for (incompatibleProtocol <- Seq( Protocol(minReaderVersion = Int.MaxValue), Protocol(minWriterVersion = Int.MaxValue), Protocol(minReaderVersion = Int.MaxValue, minWriterVersion = Int.MaxValue) )) { val tableName = "mytableprotocoltoohigh" withTable(tableName) { spark.range(0).write.format("delta").saveAsTable(tableName) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val catalogTable = DeltaTableV2(spark, TableIdentifier(tableName)).catalogTable val txn = deltaLog.startTransaction(catalogTable) val currentVersion = txn.snapshot.version untrackedChangeProtocolVersion(deltaLog, currentVersion + 1, incompatibleProtocol) // Should detect the above incompatible protocol change and fail val exception = intercept[InvalidProtocolVersionException] { txn.commit(AddFile("test", Map.empty, 1, 1, dataChange = true) :: Nil, ManualUpdate) } var pathInErrorMessage = "default." + tableName validateInvalidProtocolVersionException( exception, pathInErrorMessage, incompatibleProtocol.minReaderVersion, incompatibleProtocol.minWriterVersion) } } } private def untrackedChangeProtocolVersion( log: DeltaLog, version: Long, tableProtocolReaderVersion: Int, tableProtocolWriterVersion: Int) { untrackedChangeProtocolVersion( log, version, Protocol(tableProtocolReaderVersion, tableProtocolWriterVersion)) } private def untrackedChangeProtocolVersion( log: DeltaLog, version: Long, protocol: Protocol): Unit = { log.store.write( unsafeDeltaFile(log.logPath, version), Iterator( Metadata().json, protocol.json), overwrite = false, log.newDeltaHadoopConf()) } def validateInvalidProtocolVersionException( exception: InvalidProtocolVersionException, tableNameOrPath: String, readerRequiredVersion: Int, writerRequiredVersion: Int): Unit = { assert(exception.getErrorClass == "DELTA_INVALID_PROTOCOL_VERSION") assert(exception.tableNameOrPath == tableNameOrPath) assert(exception.readerRequiredVersion == readerRequiredVersion) assert(exception.writerRequiredVersion == writerRequiredVersion) } test("DeltaUnsupportedTableFeatureException - error message - table path") { withTempDir { path => spark.range(1).write.format("delta").save(path.getCanonicalPath) val (deltaLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, path.getCanonicalPath) var version = snapshot.version + 1 val invalidReaderFeatures = Seq("NonExistingReaderFeature1", "NonExistingReaderFeature2") val protocolReaderFeatures = Protocol( TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withReaderFeatures(invalidReaderFeatures) untrackedChangeProtocolVersion(deltaLog, version, protocolReaderFeatures) val exceptionRead = intercept[DeltaUnsupportedTableFeatureException] { spark.read.format("delta").load(path.getCanonicalPath) } validateUnsupportedTableReadFeatureException( exceptionRead, deltaLog.dataPath.toString, invalidReaderFeatures) version = version + 1 val invalidWriterFeatures = Seq("NonExistingWriterFeature1", "NonExistingWriterFeature2") val protocolWriterFeatures = Protocol( TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withWriterFeatures(invalidWriterFeatures) untrackedChangeProtocolVersion(deltaLog, version, protocolWriterFeatures) val exceptionWrite = intercept[DeltaUnsupportedTableFeatureException] { spark.range(1).write .mode("append") .option("mergeSchema", "true") .format("delta") .save(path.getCanonicalPath) } validateUnsupportedTableWriteFeatureException( exceptionWrite, deltaLog.dataPath.toString, invalidWriterFeatures) } } def testTableFeatureErrorMessageWithTableName(warm: Boolean): Unit = { val featureTable = "mytablefeaturesnotsupported" withTable(featureTable) { spark.range(1).write.format("delta").saveAsTable(featureTable) val (deltaLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(featureTable)) var version = snapshot.version + 1 val invalidReaderFeatures = Seq("NonExistingReaderFeature1", "NonExistingReaderFeature2") val protocolReaderFeatures = Protocol( TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withReaderFeatures(invalidReaderFeatures) untrackedChangeProtocolVersion(deltaLog, version, protocolReaderFeatures) if (!warm) { DeltaLog.clearCache() } val exceptionRead = intercept[DeltaUnsupportedTableFeatureException] { spark.read.format("delta").table(featureTable) } val pathInErrorMessage = "default." + featureTable validateUnsupportedTableReadFeatureException( exceptionRead, pathInErrorMessage, invalidReaderFeatures) version = version + 1 val invalidWriterFeatures = Seq("NonExistingWriterFeature1", "NonExistingWriterFeature2") val protocolWriterFeatures = Protocol( TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withWriterFeatures(invalidWriterFeatures) untrackedChangeProtocolVersion(deltaLog, version, protocolWriterFeatures) if (!warm) { DeltaLog.clearCache() } val exceptionWrite = intercept[DeltaUnsupportedTableFeatureException] { spark.range(1).write .mode("append") .option("mergeSchema", "true") .format("delta") .saveAsTable(featureTable) } validateUnsupportedTableWriteFeatureException( exceptionWrite, pathInErrorMessage, invalidWriterFeatures) // Restore the protocol version or the clean-up fails version = version + 1 untrackedChangeProtocolVersion(deltaLog, version, 1, 2) } } test("DeltaUnsupportedTableFeatureException - error message with table name - warm") { testTableFeatureErrorMessageWithTableName(warm = true) } test("DeltaUnsupportedTableFeatureException - error message with table name - cold") { testTableFeatureErrorMessageWithTableName(warm = false) } test("DeltaUnsupportedTableFeatureException - " + "incompatible protocol change during the transaction - table name") { for ((incompatibleProtocol, read) <- Seq( (Protocol( TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withReaderFeatures(Seq("NonExistingReaderFeature1", "NonExistingReaderFeature2")), true), (Protocol( TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withWriterFeatures(Seq("NonExistingWriterFeature1", "NonExistingWriterFeature2")), false) )) { val tableName = "mytablefeaturesnotsupported" withTable(tableName) { spark.range(0).write.format("delta").saveAsTable(tableName) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val catalogTable = DeltaTableV2(spark, TableIdentifier(tableName)).catalogTable val txn = deltaLog.startTransaction(catalogTable) val currentVersion = txn.snapshot.version untrackedChangeProtocolVersion(deltaLog, currentVersion + 1, incompatibleProtocol) // Should detect the above incompatible feature and fail val exception = intercept[DeltaUnsupportedTableFeatureException] { txn.commit(AddFile("test", Map.empty, 1, 1, dataChange = true) :: Nil, ManualUpdate) } var pathInErrorMessage = "default." + tableName read match { case true => validateUnsupportedTableReadFeatureException( exception, pathInErrorMessage, incompatibleProtocol.readerFeatures.get) case false => validateUnsupportedTableWriteFeatureException( exception, pathInErrorMessage, incompatibleProtocol.writerFeatures.get) } } } } def validateUnsupportedTableReadFeatureException( exception: DeltaUnsupportedTableFeatureException, tableNameOrPath: String, unsupportedFeatures: Iterable[String]): Unit = { validateUnsupportedTableFeatureException( exception, "DELTA_UNSUPPORTED_FEATURES_FOR_READ", tableNameOrPath, unsupportedFeatures) } def validateUnsupportedTableWriteFeatureException( exception: DeltaUnsupportedTableFeatureException, tableNameOrPath: String, unsupportedFeatures: Iterable[String]): Unit = { validateUnsupportedTableFeatureException( exception, "DELTA_UNSUPPORTED_FEATURES_FOR_WRITE", tableNameOrPath, unsupportedFeatures) } def validateUnsupportedTableFeatureException( exception: DeltaUnsupportedTableFeatureException, errorClass: String, tableNameOrPath: String, unsupportedFeatures: Iterable[String]): Unit = { assert(exception.getErrorClass == errorClass) assert(exception.tableNameOrPath == tableNameOrPath) assert(exception.unsupported.toSeq.sorted == unsupportedFeatures.toSeq.sorted) } test("protocol downgrade is a no-op") { withTempDir { path => val log = createTableWithProtocol(Protocol(2, 5), path) assert(log.update().protocol === Protocol(2, 5)) { // DeltaLog API. This API is internal-only and will fail when downgrade. val e = intercept[ProtocolDowngradeException] { log.upgradeProtocol(Protocol(1, 2)) } assert(log.update().protocol == Protocol(2, 5)) assert(e.getErrorClass.contains("DELTA_INVALID_PROTOCOL_DOWNGRADE")) } { // DeltaTable API val table = io.delta.tables.DeltaTable.forPath(spark, path.getCanonicalPath) val events = Log4jUsageLogger.track { table.upgradeTableProtocol(1, 2) } assert(log.update().protocol == Protocol(2, 5)) assert(events.count(_.tags.get("opType").contains("delta.protocol.downgradeIgnored")) === 1) } { // SQL API val events = Log4jUsageLogger.track { sql(s"ALTER TABLE delta.`${path.getCanonicalPath}` " + "SET TBLPROPERTIES (delta.minWriterVersion = 2)") } assert(log.update().protocol == Protocol(2, 5)) assert(events.count(_.tags.get("opType").contains("delta.protocol.downgradeIgnored")) === 1) } } } private case class SessionAndTableConfs(name: String, session: Seq[String], table: Seq[String]) for (confs <- Seq( SessionAndTableConfs( "session", session = Seq(DeltaConfigs.CREATE_TABLE_IGNORE_PROTOCOL_DEFAULTS.defaultTablePropertyKey), table = Seq.empty[String]), SessionAndTableConfs( "table", session = Seq.empty[String], table = Seq(DeltaConfigs.CREATE_TABLE_IGNORE_PROTOCOL_DEFAULTS.key)))) test(s"CREATE TABLE can ignore protocol defaults, configured in ${confs.name}") { withTempDir { path => withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> "3", DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "7", defaultPropertyKey(ChangeDataFeedTableFeature) -> FEATURE_PROP_SUPPORTED) { withSQLConf(confs.session.map(_ -> "true"): _*) { spark .range(10) .write .format("delta") .options(confs.table.map(_ -> "true").toMap) .save(path.getCanonicalPath) } } val snapshot = DeltaLog.forTable(spark, path).update() assert(snapshot.protocol === Protocol(1, 1)) assert( !snapshot.metadata.configuration .contains(DeltaConfigs.CREATE_TABLE_IGNORE_PROTOCOL_DEFAULTS.key)) } } for (ignoreProtocolDefaults <- BOOLEAN_DOMAIN) for (op <- Seq( "ALTER TABLE", "SHALLOW CLONE", "RESTORE")) { test(s"$op always ignore protocol defaults (flag = $ignoreProtocolDefaults)" ) { withTempDir { path => val expectedProtocol = if (ignoreProtocolDefaults) { Protocol(1, 1) } else { Protocol( spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION), spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION)) } val cPath = path.getCanonicalPath spark .range(10) .write .format("delta") .option( DeltaConfigs.CREATE_TABLE_IGNORE_PROTOCOL_DEFAULTS.key, ignoreProtocolDefaults.toString) .save(cPath) val snapshot = DeltaLog.forTable(spark, path).update() assert(snapshot.protocol === expectedProtocol) assert( !snapshot.metadata.configuration .contains(DeltaConfigs.CREATE_TABLE_IGNORE_PROTOCOL_DEFAULTS.key)) withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> "3", DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "7", defaultPropertyKey(ChangeDataFeedTableFeature) -> FEATURE_PROP_SUPPORTED) { val snapshotAfter = op match { case "ALTER TABLE" => sql(s"ALTER TABLE delta.`$cPath` ALTER COLUMN id COMMENT 'hallo'") DeltaLog.forTable(spark, path).update() case "SHALLOW CLONE" => var s: Snapshot = null withTempDir { cloned => sql( s"CREATE TABLE delta.`${cloned.getCanonicalPath}` " + s"SHALLOW CLONE delta.`$cPath`") s = DeltaLog.forTable(spark, cloned).update() } s case "RESTORE" => sql(s"INSERT INTO delta.`$cPath` VALUES (99)") // version 2 sql(s"RESTORE TABLE delta.`$cPath` TO VERSION AS OF 1") DeltaLog.forTable(spark, path).update() case _ => throw new RuntimeException("OP is invalid. Add a match!") } assert(snapshotAfter.protocol === expectedProtocol) assert( !snapshotAfter.metadata.configuration .contains(DeltaConfigs.CREATE_TABLE_IGNORE_PROTOCOL_DEFAULTS.key)) } } } } test("concurrent upgrade") { withTempDir { path => val newProtocol = Protocol() val log = createTableWithProtocol(Protocol(0, 0), path) // We have to copy out the internals of upgradeProtocol to induce the concurrency. val txn = log.startTransaction() log.upgradeProtocol(newProtocol) intercept[ProtocolChangedException] { txn.commit(Seq(newProtocol), DeltaOperations.UpgradeProtocol(newProtocol)) } } } test("incompatible protocol change during the transaction") { for (incompatibleProtocol <- Seq( Protocol(minReaderVersion = Int.MaxValue), Protocol(minWriterVersion = Int.MaxValue), Protocol(minReaderVersion = Int.MaxValue, minWriterVersion = Int.MaxValue) )) { withTempDir { path => spark.range(0).write.format("delta").save(path.getCanonicalPath) val deltaLog = DeltaLog.forTable(spark, path) val hadoopConf = deltaLog.newDeltaHadoopConf() val txn = deltaLog.startTransaction() val currentVersion = txn.snapshot.version deltaLog.store.write( unsafeDeltaFile(deltaLog.logPath, currentVersion + 1), Iterator(incompatibleProtocol.json), overwrite = false, hadoopConf) // Should detect the above incompatible protocol change and fail intercept[InvalidProtocolVersionException] { txn.commit(AddFile("test", Map.empty, 1, 1, dataChange = true) :: Nil, ManualUpdate) } // Make sure we didn't commit anything val p = unsafeDeltaFile(deltaLog.logPath, currentVersion + 2) assert( !p.getFileSystem(hadoopConf).exists(p), s"$p should not be committed") } } } import testImplicits._ /** Creates a Delta table and checks the expected protocol version */ private def testCreation(tableName: String, writerVersion: Int, tableInitialized: Boolean = false) (fn: String => Unit): Unit = { withTempDir { dir => withSQLConf(DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "1") { withTable(tableName) { fn(dir.getCanonicalPath) val deltaLog = DeltaLog.forTable(spark, dir) assert((deltaLog.snapshot.version != 0) == tableInitialized) assert(deltaLog.snapshot.protocol.minWriterVersion === writerVersion) assert(deltaLog.snapshot.protocol.minReaderVersion === 1) } } } } test("can create table using features configured in session") { val readerVersion = Action.supportedProtocolVersion().minReaderVersion val writerVersion = Action.supportedProtocolVersion().minWriterVersion withTempDir { dir => withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> writerVersion.toString, DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> readerVersion.toString, s"$DEFAULT_FEATURE_PROP_PREFIX${AppendOnlyTableFeature.name}" -> "enabled", s"$DEFAULT_FEATURE_PROP_PREFIX${TestReaderWriterFeature.name}" -> "enabled") { sql(s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta") val deltaLog = DeltaLog.forTable(spark, dir) assert( deltaLog.snapshot.protocol === Action .supportedProtocolVersion(withAllFeatures = false) .withFeatures(Set(AppendOnlyTableFeature, TestReaderWriterFeature))) } } } test("can create table using features configured in table properties and session") { withTempDir { dir => withSQLConf( s"$DEFAULT_FEATURE_PROP_PREFIX${TestWriterFeature.name}" -> "enabled") { sql( s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta " + "TBLPROPERTIES (" + s" delta.feature.${AppendOnlyTableFeature.name}='enabled'," + s" delta.feature.${TestLegacyReaderWriterFeature.name}='enabled'" + s")") val deltaLog = DeltaLog.forTable(spark, dir) assert(deltaLog.snapshot.protocol.minReaderVersion === 2, "reader protocol version should support table features because we used the " + "'delta.feature.' config.") assert( deltaLog.snapshot.protocol.minWriterVersion === TABLE_FEATURES_MIN_WRITER_VERSION, "reader protocol version should support table features because we used the " + "'delta.feature.' config.") assert( deltaLog.snapshot.protocol.readerAndWriterFeatureNames === Set( AppendOnlyTableFeature, InvariantsTableFeature, TestLegacyReaderWriterFeature, TestWriterFeature).map(_.name)) } } } test("creating a new table with default protocol") { val tableName = "delta_test" def testTableCreation(fn: String => Unit, tableInitialized: Boolean = false): Unit = { testCreation(tableName, 1, tableInitialized) { dir => fn(dir) } } testTableCreation { dir => spark.range(10).write.format("delta").save(dir) } testTableCreation { dir => spark.range(10).write.format("delta").option("path", dir).saveAsTable(tableName) } testTableCreation { dir => spark.range(10).writeTo(tableName).using("delta").tableProperty("location", dir).create() } testTableCreation { dir => sql(s"CREATE TABLE $tableName (id bigint) USING delta LOCATION '$dir'") } testTableCreation { dir => sql(s"CREATE TABLE $tableName USING delta LOCATION '$dir' AS SELECT * FROM range(10)") } testTableCreation(dir => { val stream = MemoryStream[Int] stream.addData(1 to 10) val q = stream.toDF().writeStream.format("delta") .option("checkpointLocation", new File(dir, "_checkpoint").getCanonicalPath) .start(dir) q.processAllAvailable() q.stop() } ) testTableCreation { dir => spark.range(10).write.mode("append").parquet(dir) sql(s"CONVERT TO DELTA parquet.`$dir`") } } test( "creating a new table with default protocol - requiring more recent protocol version") { val tableName = "delta_test" def testTableCreation(fn: String => Unit, tableInitialized: Boolean = false): Unit = testCreation(tableName, 7, tableInitialized)(fn) testTableCreation { dir => spark.range(10).writeTo(tableName).using("delta") .tableProperty("location", dir) .tableProperty("delta.appendOnly", "true") .create() } testTableCreation { dir => sql(s"CREATE TABLE $tableName (id bigint) USING delta LOCATION '$dir' " + s"TBLPROPERTIES (delta.appendOnly = 'true')") } testTableCreation { dir => sql(s"CREATE TABLE $tableName USING delta TBLPROPERTIES (delta.appendOnly = 'true') " + s"LOCATION '$dir' AS SELECT * FROM range(10)") } testTableCreation { dir => sql(s"CREATE TABLE $tableName (id bigint NOT NULL) USING delta LOCATION '$dir'") } withSQLConf("spark.databricks.delta.properties.defaults.appendOnly" -> "true") { testTableCreation { dir => spark.range(10).write.format("delta").save(dir) } testTableCreation { dir => spark.range(10).write.format("delta").option("path", dir).saveAsTable(tableName) } testTableCreation { dir => spark.range(10).writeTo(tableName).using("delta").tableProperty("location", dir).create() } testTableCreation { dir => sql(s"CREATE TABLE $tableName (id bigint) USING delta LOCATION '$dir'") } testTableCreation { dir => sql(s"CREATE TABLE $tableName USING delta LOCATION '$dir' AS SELECT * FROM range(10)") } testTableCreation(dir => { val stream = MemoryStream[Int] stream.addData(1 to 10) val q = stream.toDF().writeStream.format("delta") .option("checkpointLocation", new File(dir, "_checkpoint").getCanonicalPath) .start(dir) q.processAllAvailable() q.stop() } ) testTableCreation { dir => spark.range(10).write.mode("append").parquet(dir) sql(s"CONVERT TO DELTA parquet.`$dir`") } } } test("replacing a new table with default protocol") { withTempDir { dir => // In this test we go back and forth through protocol versions, testing the various syntaxes // of replacing tables val tbl = "delta_test" withTable(tbl) { withSQLConf(DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "1") { sql(s"CREATE TABLE $tbl (id bigint) USING delta LOCATION '${dir.getCanonicalPath}'") } val deltaLog = DeltaLog.forTable(spark, dir) assert(deltaLog.update().protocol.minWriterVersion === 1, "Should've picked up the protocol from the configuration") // Replace the table and make sure the config is picked up withSQLConf(DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "2") { spark.range(10).writeTo(tbl).using("delta") .tableProperty("location", dir.getCanonicalPath).replace() } assert(deltaLog.update().protocol.minWriterVersion === 2, "Should've picked up the protocol from the configuration") // Will not downgrade without special flag. withSQLConf(DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "1") { sql(s"REPLACE TABLE $tbl (id bigint) USING delta LOCATION '${dir.getCanonicalPath}'") assert(deltaLog.update().protocol.minWriterVersion === 2, "Should not pick up the protocol from the configuration") } // Replace with the old writer again withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "1", DeltaSQLConf.REPLACE_TABLE_PROTOCOL_DOWNGRADE_ALLOWED.key -> "true") { sql(s"REPLACE TABLE $tbl (id bigint) USING delta LOCATION '${dir.getCanonicalPath}'") assert(deltaLog.update().protocol.minWriterVersion === 1, "Should've created a new protocol") sql(s"CREATE OR REPLACE TABLE $tbl (id bigint NOT NULL) USING delta " + s"LOCATION '${dir.getCanonicalPath}'") assert(deltaLog.update().protocol === Protocol(1, 7).withFeature(InvariantsTableFeature), "Invariant should require the higher protocol") // Go back to version 1 sql(s"REPLACE TABLE $tbl (id bigint) USING delta LOCATION '${dir.getCanonicalPath}'") assert(deltaLog.update().protocol.minWriterVersion === 1, "Should've created a new protocol") // Check table properties with different syntax spark.range(10).writeTo(tbl).tableProperty("location", dir.getCanonicalPath) .tableProperty("delta.appendOnly", "true").using("delta").createOrReplace() assert(deltaLog.update().protocol === Protocol(1, 7).withFeature(AppendOnlyTableFeature), "appendOnly should require the higher protocol") } } } } test("create a table with no protocol") { withTempDir { path => val log = DeltaLog.forTable(spark, path) log.createLogDirectoriesIfNotExists() log.store.write( unsafeDeltaFile(log.logPath, 0), Iterator(Metadata().json), overwrite = false, log.newDeltaHadoopConf()) assert(intercept[DeltaIllegalStateException] { log.update() }.getErrorClass == "DELTA_STATE_RECOVER_ERROR") assert(intercept[DeltaIllegalStateException] { spark.read.format("delta").load(path.getCanonicalPath) }.getErrorClass == "DELTA_STATE_RECOVER_ERROR") assert(intercept[DeltaIllegalStateException] { spark.range(1).write.format("delta").mode(SaveMode.Overwrite).save(path.getCanonicalPath) }.getErrorClass == "DELTA_STATE_RECOVER_ERROR") } } test("bad inputs for default protocol versions") { val readerVersion = Action.supportedProtocolVersion().minReaderVersion val writerVersion = Action.supportedProtocolVersion().minWriterVersion withTempDir { path => val dir = path.getCanonicalPath Seq("abc", "", "0", (readerVersion + 1).toString).foreach { conf => val e = intercept[IllegalArgumentException] { withSQLConf(DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> conf) { spark.range(10).write.format("delta").save(dir) } } } Seq("abc", "", "0", (writerVersion + 1).toString).foreach { conf => intercept[IllegalArgumentException] { withSQLConf(DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> conf) { spark.range(10).write.format("delta").save(dir) } } } } } test("table creation with protocol as table property") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) withSQLConf(DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "1") { sql(s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta " + "TBLPROPERTIES (delta.minWriterVersion=3)") assert(deltaLog.snapshot.protocol.minReaderVersion === 1) assert(deltaLog.snapshot.protocol.minWriterVersion === 3) assertPropertiesAndShowTblProperties(deltaLog) } } } test("table creation with writer-only features as table property") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) sql( s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta " + "TBLPROPERTIES (" + " DeLtA.fEaTurE.APPendONly='eNAbled'," + " delta.feature.testWriter='enabled'" + ")") assert(deltaLog.snapshot.protocol.minReaderVersion === 1) assert( deltaLog.snapshot.protocol.minWriterVersion === TABLE_FEATURES_MIN_WRITER_VERSION) assert( deltaLog.snapshot.protocol.readerAndWriterFeatureNames === Set( AppendOnlyTableFeature, InvariantsTableFeature, TestWriterFeature).map(_.name)) assertPropertiesAndShowTblProperties(deltaLog, tableHasFeatures = true) } } test("table creation with legacy reader-writer features as table property") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) sql( s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta " + "TBLPROPERTIES (DeLtA.fEaTurE.testLEGACYReaderWritER='eNAbled')") assert( deltaLog.update().protocol === Protocol(2, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, TestLegacyReaderWriterFeature))) } } test("table creation with native writer-only features as table property") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) sql( s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta " + "TBLPROPERTIES (DeLtA.fEaTurE.testWritER='eNAbled')") assert( deltaLog.snapshot.protocol.minReaderVersion === 1) assert( deltaLog.snapshot.protocol.minWriterVersion === TABLE_FEATURES_MIN_WRITER_VERSION) assert( deltaLog.snapshot.protocol.readerAndWriterFeatureNames === Set(AppendOnlyTableFeature.name, InvariantsTableFeature.name, TestWriterFeature.name)) assertPropertiesAndShowTblProperties(deltaLog, tableHasFeatures = true) } } test("table creation with reader-writer features as table property") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) sql( s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta " + "TBLPROPERTIES (" + " DeLtA.fEaTurE.testLEGACYReaderWritER='eNAbled'," + " DeLtA.fEaTurE.testReaderWritER='enabled'" + ")") assert( deltaLog.snapshot.protocol.minReaderVersion === TABLE_FEATURES_MIN_READER_VERSION) assert( deltaLog.snapshot.protocol.minWriterVersion === TABLE_FEATURES_MIN_WRITER_VERSION) assert( deltaLog.snapshot.protocol.readerAndWriterFeatureNames === Set( InvariantsTableFeature, AppendOnlyTableFeature, TestLegacyReaderWriterFeature, TestReaderWriterFeature).map(_.name)) assertPropertiesAndShowTblProperties(deltaLog, tableHasFeatures = true) } } test("table creation with feature as table property and supported protocol version") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) sql( s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta " + "TBLPROPERTIES (" + s" DEltA.MINREADERversion='$TABLE_FEATURES_MIN_READER_VERSION'," + s" DEltA.MINWRITERversion='$TABLE_FEATURES_MIN_WRITER_VERSION'," + " DeLtA.fEaTurE.testLEGACYReaderWriter='eNAbled'" + ")") assert( deltaLog.snapshot.protocol === Protocol( minReaderVersion = 2, minWriterVersion = TABLE_FEATURES_MIN_WRITER_VERSION, readerFeatures = None, writerFeatures = Some(Set(TestLegacyReaderWriterFeature.name)))) assertPropertiesAndShowTblProperties(deltaLog, tableHasFeatures = true) } } test("table creation with feature as table property and supported writer protocol version") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) sql( s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta " + s"TBLPROPERTIES (" + s" delta.minWriterVersion='$TABLE_FEATURES_MIN_WRITER_VERSION'," + s" delta.feature.testLegacyWriter='enabled'" + s")") assert( deltaLog.snapshot.protocol === Protocol( minReaderVersion = 1, minWriterVersion = TABLE_FEATURES_MIN_WRITER_VERSION, readerFeatures = None, writerFeatures = Some(Set(TestLegacyWriterFeature.name)))) assertPropertiesAndShowTblProperties(deltaLog, tableHasFeatures = true) } } test("table creation with automatically-enabled features") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) sql( s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta TBLPROPERTIES (" + s" ${TestReaderWriterMetadataAutoUpdateFeature.TABLE_PROP_KEY}='true'" + ")") assert( deltaLog.snapshot.protocol === Protocol( minReaderVersion = TABLE_FEATURES_MIN_READER_VERSION, minWriterVersion = TABLE_FEATURES_MIN_WRITER_VERSION, readerFeatures = Some(Set(TestReaderWriterMetadataAutoUpdateFeature.name)), writerFeatures = Some(Set( TestReaderWriterMetadataAutoUpdateFeature.name, AppendOnlyTableFeature.name, InvariantsTableFeature.name)))) assertPropertiesAndShowTblProperties(deltaLog, tableHasFeatures = true) } } test("table creation with automatically-enabled legacy feature and unsupported protocol") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) sql( s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta TBLPROPERTIES (" + " delta.minReaderVersion='1'," + " delta.minWriterVersion='2'," + " delta.enableChangeDataFeed='true'" + ")") assert(deltaLog.update().protocol === Protocol(1, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, ChangeDataFeedTableFeature))) } } test("table creation with automatically-enabled native feature and unsupported protocol") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) sql( s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta TBLPROPERTIES (" + " delta.minReaderVersion='1'," + " delta.minWriterVersion='2'," + s" ${TestReaderWriterMetadataAutoUpdateFeature.TABLE_PROP_KEY}='true'" + ")") assert( deltaLog.snapshot.protocol === Protocol( minReaderVersion = TABLE_FEATURES_MIN_READER_VERSION, minWriterVersion = TABLE_FEATURES_MIN_WRITER_VERSION, readerFeatures = Some(Set(TestReaderWriterMetadataAutoUpdateFeature.name)), writerFeatures = Some(Set( TestReaderWriterMetadataAutoUpdateFeature.name, InvariantsTableFeature.name, AppendOnlyTableFeature.name)))) assertPropertiesAndShowTblProperties(deltaLog, tableHasFeatures = true) } } test("table creation with feature as table property and unsupported protocol version") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) sql( s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta TBLPROPERTIES (" + " delta.minReaderVersion='1'," + " delta.minWriterVersion='2'," + " delta.feature.testWriter='enabled'" + ")") assert( deltaLog.snapshot.protocol === Protocol( minReaderVersion = 1, minWriterVersion = TABLE_FEATURES_MIN_WRITER_VERSION, readerFeatures = None, writerFeatures = Some(Set( InvariantsTableFeature.name, AppendOnlyTableFeature.name, TestWriterFeature.name)))) assertPropertiesAndShowTblProperties(deltaLog, tableHasFeatures = true) } } def testCreateTable( name: String, props: Map[String, String], expectedExceptionClass: Option[String] = None, expectedFinalProtocol: Option[Protocol] = None): Unit = { test(s"create table - $name") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir) val propString = props.map(kv => s"'${kv._1}'='${kv._2}'").mkString(",") if (expectedExceptionClass.isDefined) { assert(intercept[DeltaTableFeatureException] { sql( s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta " + s"TBLPROPERTIES ($propString)") }.getErrorClass === expectedExceptionClass.get) } else { sql( s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta " + s"TBLPROPERTIES ($propString)") } expectedFinalProtocol match { case Some(p) => assert(log.update().protocol === p) case None => // Do nothing } } } } testCreateTable( "legacy protocol, legacy feature, metadata", Map("delta.appendOnly" -> "true"), expectedFinalProtocol = Some(Protocol(1, 2))) testCreateTable( "legacy protocol, legacy feature, feature property", Map(s"delta.feature.${TestLegacyReaderWriterFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(2, TABLE_FEATURES_MIN_WRITER_VERSION).withFeatures(Seq( TestLegacyReaderWriterFeature, AppendOnlyTableFeature, InvariantsTableFeature)))) testCreateTable( "legacy protocol, legacy writer feature, feature property", Map(s"delta.feature.${TestLegacyWriterFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(1, TABLE_FEATURES_MIN_WRITER_VERSION).withFeatures(Seq( TestLegacyWriterFeature, AppendOnlyTableFeature, InvariantsTableFeature )))) testCreateTable( "legacy protocol, native auto-update feature, metadata", Map(TestReaderWriterMetadataAutoUpdateFeature.TABLE_PROP_KEY -> "true"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(Seq( TestReaderWriterMetadataAutoUpdateFeature, AppendOnlyTableFeature, InvariantsTableFeature)))) testCreateTable( "legacy protocol, native non-auto-update feature, metadata", Map(TestReaderWriterMetadataNoAutoUpdateFeature.TABLE_PROP_KEY -> "true"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(Seq( TestReaderWriterMetadataNoAutoUpdateFeature, AppendOnlyTableFeature, InvariantsTableFeature)))) testCreateTable( "legacy protocol, native auto-update feature, feature property", Map(s"delta.feature.${TestReaderWriterMetadataAutoUpdateFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(Seq( TestReaderWriterMetadataAutoUpdateFeature, AppendOnlyTableFeature, InvariantsTableFeature)))) testCreateTable( "legacy protocol, native non-auto-update feature, feature property", Map(s"delta.feature.${TestReaderWriterMetadataNoAutoUpdateFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(Seq( TestReaderWriterMetadataNoAutoUpdateFeature, AppendOnlyTableFeature, InvariantsTableFeature)))) testCreateTable( "legacy protocol with supported version props, legacy feature, feature property", Map( DeltaConfigs.MIN_READER_VERSION.key -> TestLegacyReaderWriterFeature.minReaderVersion.toString, DeltaConfigs.MIN_WRITER_VERSION.key -> TestLegacyReaderWriterFeature.minWriterVersion.toString, s"delta.feature.${TestLegacyReaderWriterFeature.name}" -> "enabled"), expectedFinalProtocol = Some(Protocol( TestLegacyReaderWriterFeature.minReaderVersion, TestLegacyReaderWriterFeature.minWriterVersion))) testCreateTable( "legacy protocol with table feature version props, legacy feature, feature property", Map( DeltaConfigs.MIN_READER_VERSION.key -> TABLE_FEATURES_MIN_READER_VERSION.toString, DeltaConfigs.MIN_WRITER_VERSION.key -> TABLE_FEATURES_MIN_WRITER_VERSION.toString, s"delta.feature.${TestLegacyReaderWriterFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(2, TABLE_FEATURES_MIN_WRITER_VERSION).withFeature(TestLegacyReaderWriterFeature))) testCreateTable( "legacy protocol with supported version props, native feature, feature property", Map( DeltaConfigs.MIN_READER_VERSION.key -> TABLE_FEATURES_MIN_READER_VERSION.toString, DeltaConfigs.MIN_WRITER_VERSION.key -> TABLE_FEATURES_MIN_WRITER_VERSION.toString, s"delta.feature.${TestReaderWriterMetadataAutoUpdateFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestReaderWriterMetadataAutoUpdateFeature))) testCreateTable( "table features protocol, legacy feature, metadata", Map( DeltaConfigs.MIN_READER_VERSION.key -> TABLE_FEATURES_MIN_READER_VERSION.toString, DeltaConfigs.MIN_WRITER_VERSION.key -> TABLE_FEATURES_MIN_WRITER_VERSION.toString, "delta.appendOnly" -> "true"), expectedFinalProtocol = Some( Protocol(1, TABLE_FEATURES_MIN_WRITER_VERSION).withFeature(AppendOnlyTableFeature))) testCreateTable( "table features protocol, legacy feature, feature property", Map( DeltaConfigs.MIN_READER_VERSION.key -> TABLE_FEATURES_MIN_READER_VERSION.toString, DeltaConfigs.MIN_WRITER_VERSION.key -> TABLE_FEATURES_MIN_WRITER_VERSION.toString, s"delta.feature.${TestLegacyReaderWriterFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(2, TABLE_FEATURES_MIN_WRITER_VERSION).withFeature(TestLegacyReaderWriterFeature))) testCreateTable( "table features protocol, native auto-update feature, metadata", Map( DeltaConfigs.MIN_READER_VERSION.key -> TABLE_FEATURES_MIN_READER_VERSION.toString, DeltaConfigs.MIN_WRITER_VERSION.key -> TABLE_FEATURES_MIN_WRITER_VERSION.toString, TestReaderWriterMetadataAutoUpdateFeature.TABLE_PROP_KEY -> "true"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestReaderWriterMetadataAutoUpdateFeature))) testCreateTable( "table features protocol, native non-auto-update feature, metadata", Map( DeltaConfigs.MIN_READER_VERSION.key -> TABLE_FEATURES_MIN_READER_VERSION.toString, DeltaConfigs.MIN_WRITER_VERSION.key -> TABLE_FEATURES_MIN_WRITER_VERSION.toString, TestReaderWriterMetadataNoAutoUpdateFeature.TABLE_PROP_KEY -> "true"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestReaderWriterMetadataNoAutoUpdateFeature))) testCreateTable( "table features protocol, native auto-update feature, feature property", Map( DeltaConfigs.MIN_READER_VERSION.key -> TABLE_FEATURES_MIN_READER_VERSION.toString, DeltaConfigs.MIN_WRITER_VERSION.key -> TABLE_FEATURES_MIN_WRITER_VERSION.toString, s"delta.feature.${TestReaderWriterMetadataAutoUpdateFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestReaderWriterMetadataAutoUpdateFeature))) testCreateTable( "table features protocol, native non-auto-update feature, feature property", Map( DeltaConfigs.MIN_READER_VERSION.key -> TABLE_FEATURES_MIN_READER_VERSION.toString, DeltaConfigs.MIN_WRITER_VERSION.key -> TABLE_FEATURES_MIN_WRITER_VERSION.toString, s"delta.feature.${TestReaderWriterMetadataNoAutoUpdateFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestReaderWriterMetadataNoAutoUpdateFeature))) testCreateTable( name = "feature with a dependency", props = Map( DeltaConfigs.MIN_READER_VERSION.key -> TABLE_FEATURES_MIN_READER_VERSION.toString, DeltaConfigs.MIN_WRITER_VERSION.key -> TABLE_FEATURES_MIN_WRITER_VERSION.toString, s"delta.feature.${TestFeatureWithDependency.name}" -> "supported"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(Seq(TestFeatureWithDependency, TestReaderWriterFeature)))) testCreateTable( name = "feature with a dependency, enabled using a feature property", props = Map( DeltaConfigs.MIN_READER_VERSION.key -> TABLE_FEATURES_MIN_READER_VERSION.toString, DeltaConfigs.MIN_WRITER_VERSION.key -> TABLE_FEATURES_MIN_WRITER_VERSION.toString, TestFeatureWithDependency.TABLE_PROP_KEY -> "true"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(Seq(TestFeatureWithDependency, TestReaderWriterFeature)))) testCreateTable( name = "feature with a dependency that has a dependency", props = Map( DeltaConfigs.MIN_READER_VERSION.key -> TABLE_FEATURES_MIN_READER_VERSION.toString, DeltaConfigs.MIN_WRITER_VERSION.key -> TABLE_FEATURES_MIN_WRITER_VERSION.toString, s"delta.feature.${TestFeatureWithTransitiveDependency.name}" -> "supported"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(Seq( TestFeatureWithTransitiveDependency, TestFeatureWithDependency, TestReaderWriterFeature)))) def testAlterTable( name: String, props: Map[String, String], expectedExceptionClass: Option[String] = None, expectedFinalProtocol: Option[Protocol] = None, tableProtocol: Protocol = Protocol(1, 1)): Unit = { test(s"alter table - $name") { withTempDir { dir => val log = createTableWithProtocol(tableProtocol, dir) val propString = props.map(kv => s"'${kv._1}'='${kv._2}'").mkString(",") if (expectedExceptionClass.isDefined) { assert(intercept[DeltaTableFeatureException] { sql(s"ALTER TABLE delta.`${dir.getCanonicalPath}` SET TBLPROPERTIES ($propString)") }.getErrorClass === expectedExceptionClass.get) } else { sql(s"ALTER TABLE delta.`${dir.getCanonicalPath}` SET TBLPROPERTIES ($propString)") } expectedFinalProtocol match { case Some(p) => assert(log.update().protocol === p) case None => // Do nothing } } } } testAlterTable( name = "downgrade reader version is a no-op", tableProtocol = Protocol(2, 5), props = Map(DeltaConfigs.MIN_READER_VERSION.key -> "1"), expectedFinalProtocol = Some(Protocol(2, 5))) testAlterTable( name = "downgrade writer version is a no-op", tableProtocol = Protocol(1, 3), props = Map(DeltaConfigs.MIN_WRITER_VERSION.key -> "1"), expectedFinalProtocol = Some(Protocol(1, 3))) testAlterTable( name = "downgrade both reader and versions version is a no-op", tableProtocol = Protocol(2, 5), props = Map( DeltaConfigs.MIN_READER_VERSION.key -> "1", DeltaConfigs.MIN_WRITER_VERSION.key -> "1"), expectedFinalProtocol = Some(Protocol(2, 5))) testAlterTable( name = "downgrade reader but upgrade writer versions (legacy protocol)", tableProtocol = Protocol(2, 2), props = Map( DeltaConfigs.MIN_READER_VERSION.key -> "1", DeltaConfigs.MIN_WRITER_VERSION.key -> "5"), expectedFinalProtocol = Some(Protocol(2, 5))) testAlterTable( name = "downgrade reader but upgrade writer versions (table features protocol)", tableProtocol = Protocol(2, 2), props = Map( DeltaConfigs.MIN_READER_VERSION.key -> "1", DeltaConfigs.MIN_WRITER_VERSION.key -> "7"), // There is no (2, 2) feature. Protocol versions are downgraded (1, 2). expectedFinalProtocol = Some(Protocol(1, 2))) testAlterTable( name = "downgrade while enabling a feature will become an upgrade", tableProtocol = Protocol(1, 2), props = Map( DeltaConfigs.MIN_READER_VERSION.key -> "1", DeltaConfigs.MIN_WRITER_VERSION.key -> "1", DeltaConfigs.CHANGE_DATA_FEED.key -> "true"), expectedFinalProtocol = Some(Protocol(1, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, ChangeDataFeedTableFeature)))) testAlterTable( "legacy protocol, legacy feature, metadata", Map("delta.appendOnly" -> "true"), expectedFinalProtocol = Some(Protocol(1, 7).withFeature(AppendOnlyTableFeature))) testAlterTable( "legacy protocol, legacy feature, feature property", Map(s"delta.feature.${TestLegacyReaderWriterFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(2, TABLE_FEATURES_MIN_WRITER_VERSION).withFeature(TestLegacyReaderWriterFeature))) testAlterTable( "legacy protocol, legacy writer feature, feature property", Map(s"delta.feature.${TestLegacyWriterFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(1, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestLegacyWriterFeature) .merge(Protocol(1, 2))), tableProtocol = Protocol(1, 2)) testAlterTable( "legacy protocol, native auto-update feature, metadata", Map(TestReaderWriterMetadataAutoUpdateFeature.TABLE_PROP_KEY -> "true"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestReaderWriterMetadataAutoUpdateFeature))) testAlterTable( "legacy protocol, native non-auto-update feature, metadata", Map(TestReaderWriterMetadataNoAutoUpdateFeature.TABLE_PROP_KEY -> "true"), expectedExceptionClass = Some("DELTA_FEATURES_REQUIRE_MANUAL_ENABLEMENT")) testAlterTable( "legacy protocol, native non-auto-update feature, metadata and feature property", Map( TestReaderWriterMetadataNoAutoUpdateFeature.TABLE_PROP_KEY -> "true", s"delta.feature.${TestReaderWriterMetadataNoAutoUpdateFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestReaderWriterMetadataNoAutoUpdateFeature))) testAlterTable( "legacy protocol, native auto-update feature, feature property", Map(s"delta.feature.${TestReaderWriterMetadataAutoUpdateFeature.name}" -> "supported"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestReaderWriterMetadataAutoUpdateFeature))) testAlterTable( "legacy protocol, native non-auto-update feature, feature property", Map(s"delta.feature.${TestReaderWriterMetadataNoAutoUpdateFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestReaderWriterMetadataNoAutoUpdateFeature))) testAlterTable( "legacy protocol with supported version props, legacy feature, feature property", Map( DeltaConfigs.MIN_READER_VERSION.key -> TestLegacyReaderWriterFeature.minReaderVersion.toString, DeltaConfigs.MIN_WRITER_VERSION.key -> TestLegacyReaderWriterFeature.minWriterVersion.toString, s"delta.feature.${TestLegacyReaderWriterFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .merge(TestLegacyReaderWriterFeature.minProtocolVersion))) testAlterTable( "legacy protocol with table feature version props, legacy feature, feature property", Map( DeltaConfigs.MIN_READER_VERSION.key -> TABLE_FEATURES_MIN_READER_VERSION.toString, DeltaConfigs.MIN_WRITER_VERSION.key -> TABLE_FEATURES_MIN_WRITER_VERSION.toString, s"delta.feature.${TestLegacyReaderWriterFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(2, TABLE_FEATURES_MIN_WRITER_VERSION).withFeature(TestLegacyReaderWriterFeature))) testAlterTable( "legacy protocol with supported version props, native feature, feature property", Map( DeltaConfigs.MIN_READER_VERSION.key -> TABLE_FEATURES_MIN_READER_VERSION.toString, DeltaConfigs.MIN_WRITER_VERSION.key -> TABLE_FEATURES_MIN_WRITER_VERSION.toString, s"delta.feature.${TestReaderWriterMetadataAutoUpdateFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestReaderWriterMetadataAutoUpdateFeature))) testAlterTable( "table features protocol, legacy feature, metadata", Map("delta.appendOnly" -> "true"), expectedFinalProtocol = Some( Protocol(1, TABLE_FEATURES_MIN_WRITER_VERSION).withFeature(AppendOnlyTableFeature)), tableProtocol = Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION)) testAlterTable( "table features protocol, legacy feature, feature property", Map(s"delta.feature.${TestLegacyReaderWriterFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(2, TABLE_FEATURES_MIN_WRITER_VERSION).withFeature(TestLegacyReaderWriterFeature)), tableProtocol = Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION)) testAlterTable( "table features protocol, native auto-update feature, metadata", Map(TestReaderWriterMetadataAutoUpdateFeature.TABLE_PROP_KEY -> "true"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestReaderWriterMetadataAutoUpdateFeature)), tableProtocol = Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION)) testAlterTable( "table features protocol, native non-auto-update feature, metadata", Map(TestReaderWriterMetadataNoAutoUpdateFeature.TABLE_PROP_KEY -> "true"), tableProtocol = Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION), expectedExceptionClass = Some("DELTA_FEATURES_REQUIRE_MANUAL_ENABLEMENT")) testAlterTable( "table features protocol, native non-auto-update feature, metadata and feature property", Map( TestReaderWriterMetadataNoAutoUpdateFeature.TABLE_PROP_KEY -> "true", s"delta.feature.${TestReaderWriterMetadataNoAutoUpdateFeature.name}" -> "enabled"), tableProtocol = Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestReaderWriterMetadataNoAutoUpdateFeature))) testAlterTable( "table features protocol, native auto-update feature, feature property", Map(s"delta.feature.${TestReaderWriterMetadataAutoUpdateFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestReaderWriterMetadataAutoUpdateFeature)), tableProtocol = Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION)) testAlterTable( "table features protocol, native non-auto-update feature, feature property", Map(s"delta.feature.${TestReaderWriterMetadataNoAutoUpdateFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestReaderWriterMetadataNoAutoUpdateFeature)), tableProtocol = Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION)) testAlterTable( "feature property merges the old protocol", Map(s"delta.feature.${TestReaderWriterMetadataAutoUpdateFeature.name}" -> "enabled"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestReaderWriterMetadataAutoUpdateFeature).merge(Protocol(1, 2))), tableProtocol = Protocol(1, 2)) testAlterTable( name = "feature with a dependency", tableProtocol = Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION), props = Map(s"delta.feature.${TestFeatureWithDependency.name}" -> "supported"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(Seq(TestFeatureWithDependency, TestReaderWriterFeature)))) testAlterTable( name = "feature with a dependency, enabled using a feature property", tableProtocol = Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION), props = Map(TestFeatureWithDependency.TABLE_PROP_KEY -> "true"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(Seq(TestFeatureWithDependency, TestReaderWriterFeature)))) testAlterTable( name = "feature with a dependency that has a dependency", tableProtocol = Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION), props = Map(s"delta.feature.${TestFeatureWithTransitiveDependency.name}" -> "supported"), expectedFinalProtocol = Some( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(Seq( TestFeatureWithTransitiveDependency, TestFeatureWithDependency, TestReaderWriterFeature)))) test("non-auto-update capable feature requires manual enablement (via feature prop)") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> "1", DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "1") { spark.range(10).writeTo(s"delta.`${dir.getCanonicalPath}`").using("delta").create() } val expectedProtocolOnCreation = Protocol(1, 1) assert(deltaLog.update().protocol === expectedProtocolOnCreation) assert(intercept[DeltaTableFeatureException] { withSQLConf(defaultPropertyKey(TestWriterMetadataNoAutoUpdateFeature) -> "supported") { sql( s"ALTER TABLE delta.`${dir.getCanonicalPath}` SET TBLPROPERTIES (" + s" '${TestWriterMetadataNoAutoUpdateFeature.TABLE_PROP_KEY}' = 'true')") } }.getErrorClass === "DELTA_FEATURES_REQUIRE_MANUAL_ENABLEMENT", "existing tables should ignore session defaults.") sql( s"ALTER TABLE delta.`${dir.getCanonicalPath}` SET TBLPROPERTIES (" + s" '${propertyKey(TestWriterMetadataNoAutoUpdateFeature)}' = 'supported'," + s" '${TestWriterMetadataNoAutoUpdateFeature.TABLE_PROP_KEY}' = 'true')") assert( deltaLog.update().protocol === Protocol(1, 7).withFeature(TestWriterMetadataNoAutoUpdateFeature) .merge(TestWriterMetadataNoAutoUpdateFeature.minProtocolVersion)) } } test("non-auto-update capable error message is correct") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> "1", DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "1") { spark.range(10).writeTo(s"delta.`${dir.getCanonicalPath}`") .tableProperty("delta.appendOnly", "true") .using("delta") .create() val protocolOfNewTable = Protocol(1, 7).withFeature(AppendOnlyTableFeature) assert(deltaLog.update().protocol === protocolOfNewTable) val e = intercept[DeltaTableFeatureException] { // ALTER TABLE must not consider this SQL config withSQLConf(defaultPropertyKey(TestWriterFeature) -> "supported") { sql( s"ALTER TABLE delta.`${dir.getCanonicalPath}` SET TBLPROPERTIES (" + s" 'delta.appendOnly' = 'false'," + s" 'delta.enableChangeDataFeed' = 'true'," + s" '${TestReaderWriterMetadataAutoUpdateFeature.TABLE_PROP_KEY}' = 'true'," + s" '${TestWriterMetadataNoAutoUpdateFeature.TABLE_PROP_KEY}' = 'true')") } } val unsupportedFeatures = TestWriterMetadataNoAutoUpdateFeature.name val supportedFeatures = (protocolOfNewTable.implicitlyAndExplicitlySupportedFeatures + ChangeDataFeedTableFeature + TestReaderWriterMetadataAutoUpdateFeature).map(_.name).toSeq.sorted.mkString(", ") assert(e.getErrorClass === "DELTA_FEATURES_REQUIRE_MANUAL_ENABLEMENT") // `getMessageParameters` is available starting from Spark 3.4. // For now we have to check for substrings. assert(e.getMessage.contains(s" $unsupportedFeatures.")) assert(e.getMessage.contains(s" $supportedFeatures.")) } } } test("table creation with protocol as table property - property wins over conf") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) withSQLConf(DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "3") { sql(s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta " + "TBLPROPERTIES (delta.MINwriterVERsion=2)") assert(deltaLog.snapshot.protocol.minWriterVersion === 2) assertPropertiesAndShowTblProperties(deltaLog) } } } test("table creation with protocol as table property - feature requirements win SQL") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) withSQLConf(DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "1") { sql(s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta " + "TBLPROPERTIES (delta.minWriterVersion=1, delta.appendOnly=true)") assert(deltaLog.update().protocol === Protocol(1, 7).withFeature(AppendOnlyTableFeature)) assertPropertiesAndShowTblProperties(deltaLog, tableHasFeatures = true) } } } test("table creation with protocol as table property - feature requirements win DF") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) withSQLConf(DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "1") { spark.range(10).writeTo(s"delta.`${dir.getCanonicalPath}`") .tableProperty("delta.minWriterVersion", "1") .tableProperty("delta.appendOnly", "true") .using("delta") .create() assert(deltaLog.update().protocol === Protocol(1, 7).withFeature(AppendOnlyTableFeature)) assertPropertiesAndShowTblProperties(deltaLog, tableHasFeatures = true) } } } test("table creation with protocol as table property - default table properties") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) withSQLConf((DeltaConfigs.sqlConfPrefix + "minWriterVersion") -> "3") { spark.range(10).writeTo(s"delta.`${dir.getCanonicalPath}`") .using("delta") .create() assert(deltaLog.snapshot.protocol.minWriterVersion === 3) assertPropertiesAndShowTblProperties(deltaLog) } } } test("table creation with protocol as table property - explicit wins over conf") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) withSQLConf((DeltaConfigs.sqlConfPrefix + "minWriterVersion") -> "3") { spark.range(10).writeTo(s"delta.`${dir.getCanonicalPath}`") .tableProperty("delta.minWriterVersion", "2") .using("delta") .create() assert(deltaLog.snapshot.protocol.minWriterVersion === 2) assertPropertiesAndShowTblProperties(deltaLog) } } } test("table creation with protocol as table property - bad input") { withTempDir { dir => val e = intercept[IllegalArgumentException] { sql(s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta " + "TBLPROPERTIES (delta.minWriterVersion='delta rulz')") } assert(e.getMessage.contains(" one of ")) val e2 = intercept[AnalysisException] { sql(s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta " + "TBLPROPERTIES (delta.minWr1terVersion=2)") // Typo in minWriterVersion } assert(e2.getMessage.contains("Unknown configuration")) val e3 = intercept[IllegalArgumentException] { sql(s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta " + "TBLPROPERTIES (delta.minWriterVersion='-1')") } assert(e3.getMessage.contains(" one of ")) } } test("protocol as table property - desc table") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) withSQLConf(DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "2") { spark.range(10).writeTo(s"delta.`${dir.getCanonicalPath}`") .using("delta") .tableProperty("delta.minWriterVersion", "3") .createOrReplace() } assert(deltaLog.snapshot.protocol.minWriterVersion === 3) val output = spark.sql(s"DESC EXTENDED delta.`${dir.getCanonicalPath}`").collect() assert(output.exists(_.toString.contains("delta.minWriterVersion")), s"minWriterVersion not found in: ${output.mkString("\n")}") assert(output.exists(_.toString.contains("delta.minReaderVersion")), s"minReaderVersion not found in: ${output.mkString("\n")}") } } test("auto upgrade protocol version - version 2") { withTempDir { path => val log = createTableWithProtocol(Protocol(1, 1), path) spark.sql(s""" |ALTER TABLE delta.`${log.dataPath.toString}` |SET TBLPROPERTIES ('delta.appendOnly' = 'true') """.stripMargin) assert(log.update().protocol === Protocol(1, 7).withFeature(AppendOnlyTableFeature)) } } test("auto upgrade protocol version - version 3") { withTempDir { path => val log = DeltaLog.forTable(spark, path) sql(s"CREATE TABLE delta.`${path.getCanonicalPath}` (id bigint) USING delta " + "TBLPROPERTIES (delta.minWriterVersion=2)") assert(log.update().protocol.minWriterVersion === 2) spark.sql(s""" |ALTER TABLE delta.`${path.getCanonicalPath}` |ADD CONSTRAINT test CHECK (id < 5) """.stripMargin) assert(log.update().protocol.minWriterVersion === 3) } } test("auto upgrade protocol version even with explicit protocol version configs") { withTempDir { path => val log = createTableWithProtocol(Protocol(1, 1), path) spark.sql(s""" |ALTER TABLE delta.`${log.dataPath.toString}` SET TBLPROPERTIES ( | 'delta.minWriterVersion' = '2', | 'delta.enableChangeDataFeed' = 'true' |)""".stripMargin) assert(log.update().protocol === Protocol(1, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, ChangeDataFeedTableFeature))) } } test("legacy feature can be listed during alter table with silent protocol upgrade") { withTempDir { path => val log = createTableWithProtocol(Protocol(1, 1), path) spark.sql(s""" |ALTER TABLE delta.`${log.dataPath.toString}` SET TBLPROPERTIES ( | 'delta.feature.testLegacyReaderWriter' = 'enabled' |)""".stripMargin) assert( log.update().protocol === Protocol(2, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestLegacyReaderWriterFeature)) } } test("legacy feature can be explicitly listed during alter table") { withTempDir { path => val log = createTableWithProtocol(Protocol(2, TABLE_FEATURES_MIN_WRITER_VERSION), path) spark.sql(s""" |ALTER TABLE delta.`${log.dataPath.toString}` SET TBLPROPERTIES ( | 'delta.feature.testLegacyReaderWriter' = 'enabled' |)""".stripMargin) assert(log.snapshot.protocol === Protocol( 2, TABLE_FEATURES_MIN_WRITER_VERSION, readerFeatures = None, writerFeatures = Some(Set(TestLegacyReaderWriterFeature.name)))) } } test("native feature can be explicitly listed during alter table with silent protocol upgrade") { withTempDir { path => val log = createTableWithProtocol(Protocol(1, 2), path) spark.sql(s""" |ALTER TABLE delta.`${log.dataPath.toString}` SET TBLPROPERTIES ( | 'delta.feature.testReaderWriter' = 'enabled' |)""".stripMargin) assert( log.snapshot.protocol === TestReaderWriterFeature.minProtocolVersion .withFeature(TestReaderWriterFeature) .merge(Protocol(1, 2))) } } test("all active features are enabled in protocol") { withTempDir { path => spark.range(10).write.format("delta").save(path.getCanonicalPath) val log = DeltaLog.forTable(spark, path) val snapshot = log.unsafeVolatileSnapshot val p = Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) val m = snapshot.metadata.copy(configuration = snapshot.metadata.configuration ++ Map( DeltaConfigs.IS_APPEND_ONLY.key -> "false", DeltaConfigs.CHANGE_DATA_FEED.key -> "true")) log.store.write( unsafeDeltaFile(log.logPath, snapshot.version + 1), Iterator(m.json, p.json), overwrite = false, log.newDeltaHadoopConf()) val e = intercept[DeltaTableFeatureException] { spark.read.format("delta").load(path.getCanonicalPath).collect() } assert(e.getMessage.contains("enabled in metadata but not listed in protocol")) assert(e.getMessage.contains(": changeDataFeed.")) } } test("table feature status") { withTempDir { path => withSQLConf( defaultPropertyKey(ChangeDataFeedTableFeature) -> FEATURE_PROP_SUPPORTED, defaultPropertyKey(GeneratedColumnsTableFeature) -> FEATURE_PROP_ENABLED) { spark.range(10).write.format("delta").save(path.getCanonicalPath) val log = DeltaLog.forTable(spark, path) val protocol = log.update().protocol assert(protocol.isFeatureSupported(ChangeDataFeedTableFeature)) assert(protocol.isFeatureSupported(GeneratedColumnsTableFeature)) } } } private def replaceTableAs(path: File): Unit = { val p = path.getCanonicalPath sql(s"REPLACE TABLE delta.`$p` USING delta AS (SELECT * FROM delta.`$p`)") } test("REPLACE AS updates protocol when defaults are higher") { withTempDir { path => spark .range(10) .write .format("delta") .option(DeltaConfigs.MIN_READER_VERSION.key, 1) .option(DeltaConfigs.MIN_WRITER_VERSION.key, 2) .mode("append") .save(path.getCanonicalPath) val log = DeltaLog.forTable(spark, path) assert(log.update().protocol === Protocol(1, 2)) withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> "2", DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "5") { replaceTableAs(path) } assert(log.update().protocol === Protocol(2, 5)) withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> "3", DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "7", TableFeatureProtocolUtils.defaultPropertyKey(TestReaderWriterFeature) -> "enabled") { replaceTableAs(path) } assert( log.update().protocol === Protocol(2, 5).merge(Protocol(3, 7).withFeature(TestReaderWriterFeature))) } } for (p <- Seq(Protocol(2, 5), Protocol(3, 7).withFeature(TestReaderWriterFeature))) test(s"REPLACE AS keeps protocol when defaults are lower ($p)") { withTempDir { path => spark .range(10) .write .format("delta") .option(DeltaConfigs.MIN_READER_VERSION.key, p.minReaderVersion) .option(DeltaConfigs.MIN_WRITER_VERSION.key, p.minWriterVersion) .options( p.readerAndWriterFeatureNames .flatMap(TableFeature.featureNameToFeature) .map(f => TableFeatureProtocolUtils.propertyKey(f) -> "enabled") .toMap) .mode("append") .save(path.getCanonicalPath) val log = DeltaLog.forTable(spark, path) assert(log.update().protocol === p) withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> "1", DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "2") { replaceTableAs(path) } assert(log.update().protocol === p.merge(Protocol(1, 2))) withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> "1", DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "2", TableFeatureProtocolUtils.defaultPropertyKey(TestReaderWriterFeature) -> "enabled") { replaceTableAs(path) } assert( log.update().protocol === p .merge(Protocol(1, 2)) .merge( TestReaderWriterFeature.minProtocolVersion.withFeature(TestReaderWriterFeature))) } } test("REPLACE AS can ignore protocol defaults") { withTempDir { path => withSQLConf( DeltaConfigs.CREATE_TABLE_IGNORE_PROTOCOL_DEFAULTS.defaultTablePropertyKey -> "true") { spark.range(10).write.format("delta").save(path.getCanonicalPath) } val log = DeltaLog.forTable(spark, path) assert(log.update().protocol === Protocol(1, 1)) withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> "3", DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "7", defaultPropertyKey(ChangeDataFeedTableFeature) -> FEATURE_PROP_SUPPORTED, DeltaConfigs.CREATE_TABLE_IGNORE_PROTOCOL_DEFAULTS.defaultTablePropertyKey -> "true") { replaceTableAs(path) } assert(log.update().protocol === Protocol(1, 1)) assert( !log.update().metadata.configuration .contains(DeltaConfigs.CREATE_TABLE_IGNORE_PROTOCOL_DEFAULTS.key)) } } test("protocol change logging") { withTempDir { path => val dir = path.getCanonicalPath withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> "1", DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "2") { assert( captureProtocolChangeEventBlob { sql(s"CREATE TABLE delta.`$dir` (id INT) USING delta") } === Map( "toProtocol" -> Map( "minReaderVersion" -> 1, "minWriterVersion" -> 2, "supportedFeatures" -> List("appendOnly", "invariants") ), "operationName" -> "CREATE TABLE")) } // Upgrade protocol assert(captureProtocolChangeEventBlob { sql( s"ALTER TABLE delta.`$dir` " + s"SET TBLPROPERTIES (${DeltaConfigs.MIN_WRITER_VERSION.key} = '3')") } === Map( "fromProtocol" -> Map( "minReaderVersion" -> 1, "minWriterVersion" -> 2, "supportedFeatures" -> List("appendOnly", "invariants") ), "toProtocol" -> Map( "minReaderVersion" -> 1, "minWriterVersion" -> 3, "supportedFeatures" -> List("appendOnly", "checkConstraints", "invariants") ), "operationName" -> "SET TBLPROPERTIES")) // Add feature assert(captureProtocolChangeEventBlob { sql( s"ALTER TABLE delta.`$dir` " + s"SET TBLPROPERTIES (${DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key} = 'true')") } === Map( "fromProtocol" -> Map( "minReaderVersion" -> 1, "minWriterVersion" -> 3, "supportedFeatures" -> List("appendOnly", "checkConstraints", "invariants") ), "toProtocol" -> Map( "minReaderVersion" -> 3, "minWriterVersion" -> 7, "supportedFeatures" -> List("appendOnly", "checkConstraints", "deletionVectors", "invariants") ), "operationName" -> "SET TBLPROPERTIES")) } } test("protocol change logging using commitLarge") { withTempDir { path => withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> "1", DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "2") { assert( captureProtocolChangeEventBlob { sql(s"CREATE TABLE delta.`${path.getCanonicalPath}` (id INT) USING delta") } === Map( "toProtocol" -> Map( "minReaderVersion" -> 1, "minWriterVersion" -> 2, "supportedFeatures" -> List("appendOnly", "invariants") ), "operationName" -> "CREATE TABLE")) } // Clone table to invoke commitLarge withTempDir { clonedPath => assert( captureProtocolChangeEventBlob { sql(s"CREATE TABLE delta.`${clonedPath.getCanonicalPath}` " + s"SHALLOW CLONE delta.`${path.getCanonicalPath}` " + s"TBLPROPERTIES (${DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key} = 'true')") } === Map( "toProtocol" -> Map( "minReaderVersion" -> 3, "minWriterVersion" -> 7, "supportedFeatures" -> List("appendOnly", "deletionVectors", "invariants") ), "operationName" -> "CREATE TABLE")) } } } def protocolWithFeatures( readerFeatures: Seq[TableFeature] = Seq.empty, writerFeatures: Seq[TableFeature] = Seq.empty): Protocol = { val readerFeaturesEnabled = readerFeatures.nonEmpty val writerFeaturesEnabled = readerFeatures.nonEmpty || writerFeatures.nonEmpty val minReaderVersion = if (readerFeaturesEnabled) TABLE_FEATURES_MIN_READER_VERSION else 1 val minWriterVersion = if (writerFeaturesEnabled) TABLE_FEATURES_MIN_WRITER_VERSION else 1 val readerFeatureNames = if (readerFeaturesEnabled) Some(readerFeatures.map(_.name).toSet) else None val writerFeatureNames = if (writerFeaturesEnabled) { Some((readerFeatures ++ writerFeatures).map(_.name).toSet) } else { None } Protocol( minReaderVersion = minReaderVersion, minWriterVersion = minWriterVersion, readerFeatures = readerFeatureNames, writerFeatures = writerFeatureNames) } def protocolWithReaderFeature(readerFeature: TableFeature): Protocol = { protocolWithFeatures(readerFeatures = Seq(readerFeature)) } def protocolWithWriterFeature(writerFeature: TableFeature): Protocol = { protocolWithFeatures(writerFeatures = Seq(writerFeature)) } def emptyProtocolWithWriterFeatures: Protocol = Protocol( minReaderVersion = 1, minWriterVersion = TABLE_FEATURES_MIN_WRITER_VERSION, readerFeatures = None, writerFeatures = Some(Set.empty)) def emptyProtocolWithReaderFeatures: Protocol = Protocol( minReaderVersion = TABLE_FEATURES_MIN_READER_VERSION, minWriterVersion = TABLE_FEATURES_MIN_WRITER_VERSION, readerFeatures = Some(Set.empty), writerFeatures = Some(Set.empty)) protected def createTableWithFeature( deltaLog: DeltaLog, feature: TableFeature, featureProperty: String): Unit = { sql(s"""CREATE TABLE delta.`${deltaLog.dataPath}` (id bigint) USING delta |TBLPROPERTIES ( |delta.feature.${feature.name} = 'supported', |$featureProperty = "true" |)""".stripMargin) val readerVersion = Math.max(feature.minReaderVersion, 1) val expectedWriterFeatures = Some(Set(feature.name, InvariantsTableFeature.name, AppendOnlyTableFeature.name)) val expectedReaderFeatures: Option[Set[String]] = if (supportsReaderFeatures(readerVersion)) Some(Set(feature.name)) else None assert( deltaLog.update().protocol === Protocol( minReaderVersion = Math.max(feature.minReaderVersion, 1), minWriterVersion = TABLE_FEATURES_MIN_WRITER_VERSION, readerFeatures = expectedReaderFeatures, writerFeatures = expectedWriterFeatures)) } /** Assumes there is at least 1 commit. */ def getEarliestCommitVersion(deltaLog: DeltaLog): Long = deltaLog.listFrom(0L).collectFirst { case DeltaFile(_, v) => v }.get def testWriterFeatureRemoval( feature: TableFeature, featurePropertyKey: String): Unit = { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) createTableWithFeature(deltaLog, feature, featurePropertyKey) AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), feature.name).run(spark) // Writer feature is removed from the writer features set. val snapshot = deltaLog.update() assert(snapshot.protocol === Protocol(1, 2)) assert(!snapshot.metadata.configuration.contains(featurePropertyKey)) assertPropertiesAndShowTblProperties(deltaLog) } } def truncateHistoryDefaultLogRetention: CalendarInterval = DeltaConfigs.parseCalendarInterval( DeltaConfigs.TABLE_FEATURE_DROP_TRUNCATE_HISTORY_LOG_RETENTION.defaultValue) def testReaderFeatureRemoval( feature: TableFeature, featurePropertyKey: String, advanceClockPastRetentionPeriod: Boolean = true, truncateHistory: Boolean = false, truncateHistoryRetentionOpt: Option[String] = None): Unit = { withTempDir { dir => val truncateHistoryRetention = truncateHistoryRetentionOpt .map(DeltaConfigs.parseCalendarInterval) .getOrElse(truncateHistoryDefaultLogRetention) val clock = new ManualClock(System.currentTimeMillis()) val deltaLog = DeltaLog.forTable(spark, dir, clock) createTableWithFeature(deltaLog, feature, featurePropertyKey) if (truncateHistoryRetentionOpt.nonEmpty) { val propertyKey = DeltaConfigs.TABLE_FEATURE_DROP_TRUNCATE_HISTORY_LOG_RETENTION.key AlterTableSetPropertiesDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), Map(propertyKey -> truncateHistoryRetention.toString)).run(spark) } // First attempt should cleanup feature traces but fail with a message due to historical // log entries containing the feature. val e1 = intercept[DeltaTableFeatureException] { AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), feature.name).run(spark) } checkError( e1, "DELTA_FEATURE_DROP_WAIT_FOR_RETENTION_PERIOD", parameters = Map( "feature" -> feature.name, "logRetentionPeriodKey" -> "delta.logRetentionDuration", "logRetentionPeriod" -> "30 days", "truncateHistoryLogRetentionPeriod" -> truncateHistoryRetention.toString)) // Add some more commits. spark.range(0, 100).write.format("delta").mode("append").save(dir.getCanonicalPath) spark.range(100, 120).write.format("delta").mode("append").save(dir.getCanonicalPath) // Table still contains historical data with the feature. Attempt should fail. val e2 = intercept[DeltaTableFeatureException] { AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), feature.name).run(spark) } checkError( e2, "DELTA_FEATURE_DROP_HISTORICAL_VERSIONS_EXIST", parameters = Map( "feature" -> feature.name, "logRetentionPeriodKey" -> "delta.logRetentionDuration", "logRetentionPeriod" -> "30 days", "truncateHistoryLogRetentionPeriod" -> truncateHistoryRetention.toString)) // Generate commit. spark.range(120, 140).write.format("delta").mode("append").save(dir.getCanonicalPath) // Pretend retention period has passed. if (advanceClockPastRetentionPeriod) { val clockAdvanceMillis = if (truncateHistory) { DeltaConfigs.getMilliSeconds(truncateHistoryRetention) + TimeUnit.HOURS.toMillis(24) } else { deltaLog.deltaRetentionMillis(deltaLog.update().metadata) + TimeUnit.DAYS.toMillis(3) } clock.advance(clockAdvanceMillis) } val dropCommand = AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), feature.name, truncateHistory = truncateHistory) if (advanceClockPastRetentionPeriod) { // History is now clean. We should be able to remove the feature. dropCommand.run(spark) // Reader+writer feature is removed from the features set. val snapshot = deltaLog.update() assert(snapshot.protocol === Protocol(1, 2)) assert(!snapshot.metadata.configuration.contains(featurePropertyKey)) assertPropertiesAndShowTblProperties(deltaLog) } else { // When the clock did not advance the logs are not cleaned. We should detect there // are still versions that contain traces of the feature. val e3 = intercept[DeltaTableFeatureException] { dropCommand.run(spark) } checkError( e3, "DELTA_FEATURE_DROP_HISTORICAL_VERSIONS_EXIST", parameters = Map( "feature" -> feature.name, "logRetentionPeriodKey" -> "delta.logRetentionDuration", "logRetentionPeriod" -> "30 days", "truncateHistoryLogRetentionPeriod" -> truncateHistoryRetention.toString)) } // Verify commits before the checkpoint are cleaned. val earliestExpectedCommitVersion = if (advanceClockPastRetentionPeriod) { deltaLog.findEarliestReliableCheckpoint.get } else { 0L } assert(getEarliestCommitVersion(deltaLog) === earliestExpectedCommitVersion) // Validate extra commits. val table = io.delta.tables.DeltaTable.forPath(deltaLog.dataPath.toString) assert(table.toDF.count() == 140) } } test("Remove writer feature") { testWriterFeatureRemoval( TestRemovableWriterFeature, TestRemovableWriterFeature.TABLE_PROP_KEY) } test("Remove legacy writer feature") { testWriterFeatureRemoval( TestRemovableLegacyWriterFeature, TestRemovableLegacyWriterFeature.TABLE_PROP_KEY) } for { advanceClockPastRetentionPeriod <- BOOLEAN_DOMAIN truncateHistory <- if (advanceClockPastRetentionPeriod) BOOLEAN_DOMAIN else Seq(false) retentionOpt <- if (truncateHistory) Seq(Some("12 hours"), None) else Seq(None) } test(s"Remove reader+writer feature " + s"advanceClockPastRetentionPeriod: $advanceClockPastRetentionPeriod " + s"truncateHistory: $truncateHistory " + s"retentionOpt: ${retentionOpt.getOrElse("None")}") { testReaderFeatureRemoval( TestRemovableReaderWriterFeature, TestRemovableReaderWriterFeature.TABLE_PROP_KEY, advanceClockPastRetentionPeriod, truncateHistory, retentionOpt) } test("Remove legacy reader+writer feature") { testReaderFeatureRemoval( TestRemovableLegacyReaderWriterFeature, TestRemovableLegacyReaderWriterFeature.TABLE_PROP_KEY) } test("Remove writer feature when table protocol does not support reader features") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) sql(s"""CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta |TBLPROPERTIES ( |delta.feature.${TestWriterFeature.name} = 'supported', |delta.feature.${TestRemovableWriterFeature.name} = 'supported' |)""".stripMargin) val protocol = deltaLog.update().protocol assert(protocol === protocolWithFeatures( writerFeatures = Seq( AppendOnlyTableFeature, InvariantsTableFeature, TestWriterFeature, TestRemovableWriterFeature))) val command = AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), TestRemovableWriterFeature.name) command.run(spark) assert( deltaLog.update().protocol === Protocol( minReaderVersion = 1, minWriterVersion = TABLE_FEATURES_MIN_WRITER_VERSION, readerFeatures = None, writerFeatures = Some(Set( TestWriterFeature.name, AppendOnlyTableFeature.name, InvariantsTableFeature.name)))) } } test("Remove a non-removable feature") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) sql(s"""CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta |TBLPROPERTIES ( |delta.feature.${TestWriterMetadataNoAutoUpdateFeature.name} = 'supported' |)""".stripMargin) val expectedProtocol = protocolWithFeatures(writerFeatures = Seq( TestWriterMetadataNoAutoUpdateFeature, AppendOnlyTableFeature, InvariantsTableFeature)) assert(deltaLog.update().protocol === expectedProtocol) val command = AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), TestWriterMetadataNoAutoUpdateFeature.name) val e = intercept[DeltaTableFeatureException] { command.run(spark) } checkError( e, "DELTA_FEATURE_DROP_NONREMOVABLE_FEATURE", parameters = Map("feature" -> TestWriterMetadataNoAutoUpdateFeature.name)) } } test("Remove an implicit writer feature") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) sql(s"""CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta |TBLPROPERTIES ( |delta.minWriterVersion = 2)""".stripMargin) assert(deltaLog.update().protocol === Protocol(minReaderVersion = 1, minWriterVersion = 2)) // Try removing AppendOnly which is an implicitly supported feature (writer version 2). val command = AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), AppendOnlyTableFeature.name) val e = intercept[DeltaTableFeatureException] { command.run(spark) } checkError( e, "DELTA_FEATURE_DROP_NONREMOVABLE_FEATURE", parameters = Map("feature" -> AppendOnlyTableFeature.name)) } } test("Remove a feature not supported by the client") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) sql(s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta") assert( deltaLog.update().protocol === Protocol( minReaderVersion = 1, minWriterVersion = 2, readerFeatures = None, writerFeatures = None)) val command = AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), "NonSupportedFeature") val e = intercept[DeltaTableFeatureException] { command.run(spark) } checkError( e, "DELTA_FEATURE_DROP_UNSUPPORTED_CLIENT_FEATURE", parameters = Map("feature" -> "NonSupportedFeature")) } } for (withTableFeatures <- BOOLEAN_DOMAIN) test(s"Remove a feature not present in the protocol - withTableFeatures: $withTableFeatures") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) sql(s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta") assert(deltaLog.update().protocol === Protocol( minReaderVersion = 1, minWriterVersion = 2, readerFeatures = None, writerFeatures = None)) val command = AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), TestRemovableWriterFeature.name) val e = intercept[DeltaTableFeatureException] { command.run(spark) } checkError( e, "DELTA_FEATURE_DROP_FEATURE_NOT_PRESENT", parameters = Map("feature" -> TestRemovableWriterFeature.name)) } } test("Reintroduce a feature after removing it") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) sql(s"""CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta |TBLPROPERTIES ( |delta.feature.${TestRemovableWriterFeature.name} = 'supported' |)""".stripMargin) val expectedFeatures = Seq(AppendOnlyTableFeature, InvariantsTableFeature, TestRemovableWriterFeature) val protocol = deltaLog.update().protocol assert(protocol === protocolWithFeatures(writerFeatures = expectedFeatures)) val command = AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), TestRemovableWriterFeature.name) command.run(spark) assert(deltaLog.update().protocol === Protocol(1, 2)) sql(s"""ALTER TABLE delta.`${dir.getCanonicalPath}` SET TBLPROPERTIES ( |delta.feature.${TestRemovableWriterFeature.name} = 'supported' |)""".stripMargin) val expectedProtocolAfterReintroduction = protocolWithFeatures(writerFeatures = expectedFeatures) assert(deltaLog.update().protocol === expectedProtocolAfterReintroduction) } } test("Remove a feature which is a dependency of other features") { // TestRemovableWriterFeatureWithDependency has two dependencies: // 1. TestRemovableReaderWriterFeature // 2. TestRemovableWriterFeature withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) // Scenario-1: Create a table with `TestRemovableWriterFeature` feature and validate that we // can drop it. sql( s"""CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta |TBLPROPERTIES ( |delta.feature.${TestRemovableWriterFeature.name} = 'supported' |)""".stripMargin) var protocol = deltaLog.update().protocol assert(protocol === protocolWithFeatures(writerFeatures = Seq( AppendOnlyTableFeature, InvariantsTableFeature, TestRemovableWriterFeature))) AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), TestRemovableWriterFeature.name).run(spark) assert(deltaLog.update().protocol === Protocol(1, 2)) // Scenario-2: Create a table with `TestRemovableWriterFeatureWithDependency` feature. This // will enable 2 dependent features also. sql( s"""ALTER TABLE delta.`${dir.getCanonicalPath}` SET TBLPROPERTIES ( |delta.feature.${TestRemovableWriterFeatureWithDependency.name} = 'supported' |)""".stripMargin) protocol = deltaLog.update().protocol Seq( TestRemovableWriterFeatureWithDependency, TestRemovableReaderWriterFeature, TestRemovableWriterFeature ).foreach(f => assert(protocol.isFeatureSupported(f))) // Now we should not be able to drop `TestRemovableWriterFeature` as it is a dependency of // `TestRemovableWriterFeatureWithDependency`. // Although we should be able to drop `TestRemovableReaderWriterFeature` as it is not a // dependency of any other feature. val e1 = intercept[DeltaTableFeatureException] { AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), TestRemovableWriterFeature.name).run(spark) } checkError( e1, "DELTA_FEATURE_DROP_DEPENDENT_FEATURE", parameters = Map( "feature" -> TestRemovableWriterFeature.name, "dependentFeatures" -> TestRemovableWriterFeatureWithDependency.name)) AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), TestRemovableWriterFeatureWithDependency.name).run(spark) protocol = deltaLog.update().protocol assert(!protocol.isFeatureSupported(TestRemovableWriterFeatureWithDependency)) assert(protocol.isFeatureSupported(TestRemovableWriterFeature)) assert(protocol.isFeatureSupported(TestRemovableReaderWriterFeature)) // Once the dependent feature is removed, we should be able to drop // `TestRemovableWriterFeature` also. AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), TestRemovableWriterFeature.name).run(spark) protocol = deltaLog.update().protocol assert(!protocol.isFeatureSupported(TestRemovableWriterFeatureWithDependency)) assert(!protocol.isFeatureSupported(TestRemovableWriterFeature)) assert(protocol.isFeatureSupported(TestRemovableReaderWriterFeature)) } } test(s"Truncate history while dropping a writer feature") { withTempDir { dir => val table = s"delta.`${dir.getCanonicalPath}`" val deltaLog = DeltaLog.forTable(spark, dir) createTableWithFeature( deltaLog, feature = TestRemovableWriterFeature, featureProperty = TestRemovableWriterFeature.TABLE_PROP_KEY) val e = intercept[DeltaTableFeatureException] { sql(s"""ALTER TABLE $table |DROP FEATURE ${TestRemovableWriterFeature.name} |TRUNCATE HISTORY""".stripMargin) } checkError( e, "DELTA_FEATURE_DROP_HISTORY_TRUNCATION_NOT_ALLOWED", parameters = Map.empty) } } test("Try removing reader+writer feature but re-enable feature after disablement") { withTempDir { dir => val clock = new ManualClock(System.currentTimeMillis()) val deltaLog = DeltaLog.forTable(spark, dir, clock) createTableWithFeature( deltaLog, feature = TestRemovableReaderWriterFeature, featureProperty = TestRemovableReaderWriterFeature.TABLE_PROP_KEY) // Add some more commits. spark.range(0, 100).write.format("delta").mode("append").save(dir.getCanonicalPath) spark.range(100, 120).write.format("delta").mode("append").save(dir.getCanonicalPath) // First attempt should cleanup feature traces but fail with a message due to historical // log entries containing the feature. val e1 = intercept[DeltaTableFeatureException] { AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), TestRemovableReaderWriterFeature.name).run(spark) } checkError( e1, "DELTA_FEATURE_DROP_WAIT_FOR_RETENTION_PERIOD", parameters = Map( "feature" -> TestRemovableReaderWriterFeature.name, "logRetentionPeriodKey" -> "delta.logRetentionDuration", "logRetentionPeriod" -> "30 days", "truncateHistoryLogRetentionPeriod" -> truncateHistoryDefaultLogRetention.toString)) // Advance clock. clock.advance(TimeUnit.DAYS.toMillis(1) + TimeUnit.HOURS.toMillis(24)) // Generate commit. spark.range(120, 140).write.format("delta").mode("append").save(dir.getCanonicalPath) // Add feature property again. val v2Table = DeltaTableV2(spark, deltaLog.dataPath) AlterTableSetPropertiesDeltaCommand( v2Table, Map(TestRemovableReaderWriterFeature.TABLE_PROP_KEY -> true.toString)) .run(spark) // Feature was enabled again in the middle of the timeframe. The feature traces are // are cleaned up again and we get a new "Wait for retention period message." val e2 = intercept[DeltaTableFeatureException] { AlterTableDropFeatureDeltaCommand( table = DeltaTableV2(spark, deltaLog.dataPath), featureName = TestRemovableReaderWriterFeature.name, truncateHistory = true).run(spark) } checkError( e2, "DELTA_FEATURE_DROP_WAIT_FOR_RETENTION_PERIOD", parameters = Map( "feature" -> TestRemovableReaderWriterFeature.name, "logRetentionPeriodKey" -> "delta.logRetentionDuration", "logRetentionPeriod" -> "30 days", "truncateHistoryLogRetentionPeriod" -> truncateHistoryDefaultLogRetention.toString)) } } test("Remove reader+writer feature with shortened retention period") { withTempDir { dir => val clock = new ManualClock(System.currentTimeMillis()) val deltaLog = DeltaLog.forTable(spark, dir, clock) createTableWithFeature( deltaLog, feature = TestRemovableReaderWriterFeature, featureProperty = TestRemovableReaderWriterFeature.TABLE_PROP_KEY) // First attempt should cleanup feature traces but fail with a message due to historical // log entries containing the feature. val e1 = intercept[DeltaTableFeatureException] { AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), TestRemovableReaderWriterFeature.name).run(spark) } checkError( e1, "DELTA_FEATURE_DROP_WAIT_FOR_RETENTION_PERIOD", parameters = Map( "feature" -> TestRemovableReaderWriterFeature.name, "logRetentionPeriodKey" -> "delta.logRetentionDuration", "logRetentionPeriod" -> "30 days", "truncateHistoryLogRetentionPeriod" -> truncateHistoryDefaultLogRetention.toString)) // Set retention period to a day. AlterTableSetPropertiesDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), Map(DeltaConfigs.LOG_RETENTION.key -> "1 DAY")).run(spark) // Metadata is not cleaned yet. Attempt should fail. val e2 = intercept[DeltaTableFeatureException] { AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), TestRemovableReaderWriterFeature.name).run(spark) } checkError( e2, "DELTA_FEATURE_DROP_HISTORICAL_VERSIONS_EXIST", parameters = Map( "feature" -> TestRemovableReaderWriterFeature.name, "logRetentionPeriodKey" -> "delta.logRetentionDuration", "logRetentionPeriod" -> "1 days", "truncateHistoryLogRetentionPeriod" -> truncateHistoryDefaultLogRetention.toString)) spark.range(1, 100).write.format("delta").mode("append").save(dir.getCanonicalPath) // Pretend retention period has passed. clock.advance( deltaLog.deltaRetentionMillis(deltaLog.update().metadata) + TimeUnit.DAYS.toMillis(3)) // History is now clean. We should be able to remove the feature. AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), TestRemovableReaderWriterFeature.name).run(spark) // Verify commits before the checkpoint are cleaned. val earliestExpectedCommitVersion = deltaLog.findEarliestReliableCheckpoint.get assert(getEarliestCommitVersion(deltaLog) === earliestExpectedCommitVersion) // Reader+writer feature is removed from the features set. val snapshot = deltaLog.update() assert(snapshot.protocol === Protocol(1, 2)) assert(!snapshot.metadata.configuration .contains(TestRemovableReaderWriterFeature.TABLE_PROP_KEY)) assertPropertiesAndShowTblProperties(deltaLog) } } test("Try removing reader+writer feature after restore") { withTempDir { dir => val clock = new ManualClock(System.currentTimeMillis()) val deltaLog = DeltaLog.forTable(spark, dir, clock) createTableWithFeature( deltaLog, feature = TestRemovableReaderWriterFeature, featureProperty = TestRemovableReaderWriterFeature.TABLE_PROP_KEY) val preRemovalVersion = deltaLog.update().version // Cleanup feature traces and throw message to wait retention period to expire. val e1 = intercept[DeltaTableFeatureException] { AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), TestRemovableReaderWriterFeature.name).run(spark) } checkError( e1, "DELTA_FEATURE_DROP_WAIT_FOR_RETENTION_PERIOD", parameters = Map( "feature" -> TestRemovableReaderWriterFeature.name, "logRetentionPeriodKey" -> "delta.logRetentionDuration", "logRetentionPeriod" -> "30 days", "truncateHistoryLogRetentionPeriod" -> truncateHistoryDefaultLogRetention.toString)) // Add some more commits. spark.range(0, 100).write.format("delta").mode("append").save(dir.getCanonicalPath) spark.range(100, 120).write.format("delta").mode("append").save(dir.getCanonicalPath) // Restore table to an older version with feature traces. sql(s"RESTORE delta.`${deltaLog.dataPath}` TO VERSION AS OF $preRemovalVersion") // Drop command should detect that latest version has feature traces and run // preDowngrade again. val e2 = intercept[DeltaTableFeatureException] { AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), TestRemovableReaderWriterFeature.name).run(spark) } checkError( e2, "DELTA_FEATURE_DROP_WAIT_FOR_RETENTION_PERIOD", parameters = Map( "feature" -> TestRemovableReaderWriterFeature.name, "logRetentionPeriodKey" -> "delta.logRetentionDuration", "logRetentionPeriod" -> "30 days", "truncateHistoryLogRetentionPeriod" -> truncateHistoryDefaultLogRetention.toString)) } } test("Remove reader+writer feature after unrelated metadata change") { withTempDir { dir => val clock = new ManualClock(System.currentTimeMillis()) val deltaLog = DeltaLog.forTable(spark, dir, clock) createTableWithFeature( deltaLog, feature = TestRemovableReaderWriterFeature, featureProperty = TestRemovableReaderWriterFeature.TABLE_PROP_KEY) // First attempt should cleanup feature traces but fail with a message due to historical // log entries containing the feature. val e1 = intercept[DeltaTableFeatureException] { AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), TestRemovableReaderWriterFeature.name).run(spark) } checkError( e1, "DELTA_FEATURE_DROP_WAIT_FOR_RETENTION_PERIOD", parameters = Map( "feature" -> TestRemovableReaderWriterFeature.name, "logRetentionPeriodKey" -> "delta.logRetentionDuration", "logRetentionPeriod" -> "30 days", "truncateHistoryLogRetentionPeriod" -> truncateHistoryDefaultLogRetention.toString)) // Add some more commits. spark.range(0, 100).write.format("delta").mode("append").save(dir.getCanonicalPath) spark.range(100, 120).write.format("delta").mode("append").save(dir.getCanonicalPath) // Pretend retention period has passed. clock.advance( deltaLog.deltaRetentionMillis(deltaLog.update().metadata) + TimeUnit.DAYS.toMillis(3)) // Perform an unrelated metadata change. sql(s"ALTER TABLE delta.`${deltaLog.dataPath}` ADD COLUMN (value INT)") // The unrelated metadata change should not interfere with validation and we should // be able to downgrade the protocol. AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), TestRemovableReaderWriterFeature.name).run(spark) // Verify commits before the checkpoint are cleaned. val earliestExpectedCommitVersion = deltaLog.findEarliestReliableCheckpoint.get assert(getEarliestCommitVersion(deltaLog) === earliestExpectedCommitVersion) } } for { withCatalog <- BOOLEAN_DOMAIN quoteWith <- if (withCatalog) Seq ("none", "single", "backtick") else Seq("none") } test(s"Drop feature DDL - withCatalog=$withCatalog, quoteWith=$quoteWith") { withTempDir { dir => val table = if (withCatalog) "table" else s"delta.`${dir.getCanonicalPath}`" if (withCatalog) sql(s"DROP TABLE IF EXISTS $table") sql( s"""CREATE TABLE $table (id bigint) USING delta |TBLPROPERTIES ( |delta.feature.${TestRemovableWriterFeature.name} = 'supported' |)""".stripMargin) val deltaLog = if (withCatalog) { DeltaLog.forTable(spark, TableIdentifier(table)) } else { DeltaLog.forTable(spark, dir) } AlterTableSetPropertiesDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), Map(TestRemovableWriterFeature.TABLE_PROP_KEY -> "true")).run(spark) val protocol = deltaLog.update().protocol assert(protocol === protocolWithFeatures(writerFeatures = Seq( AppendOnlyTableFeature, InvariantsTableFeature, TestRemovableWriterFeature))) val logs = Log4jUsageLogger.track { val featureName = quoteWith match { case "none" => s"${TestRemovableWriterFeature.name}" case "single" => s"'${TestRemovableWriterFeature.name}'" case "backtick" => s"`${TestRemovableWriterFeature.name}`" } sql(s"ALTER TABLE $table DROP FEATURE $featureName") assert(deltaLog.update().protocol === Protocol(1, 2)) } // Test that the write downgrade command was invoked. val expectedOpType = "delta.test.TestWriterFeaturePreDowngradeCommand" val blob = logs.collectFirst { case r if r.metric == MetricDefinitions.EVENT_TAHOE.name && r.tags.get("opType").contains(expectedOpType) => r.blob } assert(blob.nonEmpty, s"Expecting an '$expectedOpType' event but didn't see any.") } } for { withCatalog <- BOOLEAN_DOMAIN quoteWith <- if (withCatalog) Seq("none", "single", "backtick") else Seq("none") } test(s"Drop feature DDL TRUNCATE HISTORY - withCatalog=$withCatalog, quoteWith=$quoteWith") { withTempDir { dir => val table: String = if (withCatalog) { s"${spark.sessionState.catalog.getCurrentDatabase}.table" } else { s"delta.`${dir.getCanonicalPath}`" } if (withCatalog) sql(s"DROP TABLE IF EXISTS $table") sql( s"""CREATE TABLE $table (id bigint) USING delta |TBLPROPERTIES ( |delta.feature.${TestRemovableReaderWriterFeature.name} = 'supported', |${TestRemovableReaderWriterFeature.TABLE_PROP_KEY} = "true", |${DeltaConfigs.TABLE_FEATURE_DROP_TRUNCATE_HISTORY_LOG_RETENTION.key} = "0 hours" |)""".stripMargin) // We need to use a Delta log object with the ManualClock created in this test instead of // the default SystemClock. However, we can't pass the Delta log to use directly in the SQL // command. Instead, we will // 1. Clear the Delta log cache to remove the log associated with table creation. // 2. Populate the Delta log cache with the Delta log object that has the ManualClock we // want to use // TODO(c27kwan): Refactor this and provide a better way to control clocks in Delta tests. val clock = new ManualClock(System.currentTimeMillis()) val deltaLog = if (withCatalog) { val tableIdentifier = TableIdentifier("table", Some(spark.sessionState.catalog.getCurrentDatabase)) // We need to hack the Delta log cache with path based access to setup the right key. val path = DeltaLog.forTable(spark, tableIdentifier, clock).dataPath DeltaLog.clearCache() DeltaLog.forTable(spark, path, clock) } else { DeltaLog.clearCache() DeltaLog.forTable(spark, dir, clock) } val protocol = deltaLog.update().protocol assert(protocol === protocolWithFeatures( readerFeatures = Seq(TestRemovableReaderWriterFeature), writerFeatures = Seq( AppendOnlyTableFeature, InvariantsTableFeature, TestRemovableReaderWriterFeature))) val logs = Log4jUsageLogger.track { val featureName = quoteWith match { case "none" => s"${TestRemovableReaderWriterFeature.name}" case "single" => s"'${TestRemovableReaderWriterFeature.name}'" case "backtick" => s"`${TestRemovableReaderWriterFeature.name}`" } // Expect an exception when dropping a reader writer feature on a table that // still has traces of the feature. intercept[DeltaTableFeatureException] { sql(s"ALTER TABLE $table DROP FEATURE $featureName") } // Move past retention period. clock.advance(TimeUnit.HOURS.toMillis(48)) sql(s"ALTER TABLE $table DROP FEATURE $featureName TRUNCATE HISTORY") assert(deltaLog.update().protocol === Protocol(1, 2)) } // Validate the correct downgrade command was invoked. val expectedOpType = "delta.test.TestReaderWriterFeaturePreDowngradeCommand" val blob = logs.collectFirst { case r if r.metric == MetricDefinitions.EVENT_TAHOE.name && r.tags.get("opType").contains(expectedOpType) => r.blob } assert(blob.nonEmpty, s"Expecting an '$expectedOpType' event but didn't see any.") } } for { propertyName <- Seq("delta.enableRowTracking", "DELTA.enableRowTracking", "delta.ENABLEROWTRACKING", "DELTA.ENABLEROWTRACKING") } test(s"Drop a table property using drop feature should fail" + s" - with propertyName=$propertyName") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) sql(s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta") val command = AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), propertyName) val e = intercept[DeltaTableFeatureException] { command.run(spark) } checkError( e, "DELTA_FEATURE_DROP_FEATURE_IS_DELTA_PROPERTY", parameters = Map("property" -> propertyName) ) } } protected def testProtocolVersionDowngrade( initialMinReaderVersion: Int, initialMinWriterVersion: Int, featuresToAdd: Seq[TableFeature], featuresToRemove: Seq[TableFeature], expectedDowngradedProtocol: Protocol, truncateHistory: Boolean = false): Unit = { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) spark.sql(s"""CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta |TBLPROPERTIES ( |delta.minReaderVersion = $initialMinReaderVersion, |delta.minWriterVersion = $initialMinWriterVersion |)""".stripMargin) // Upgrade protocol to table features. val newTBLProperties = featuresToAdd .map(f => s"delta.feature.${f.name}='supported'") .reduce(_ + ", " + _) spark.sql( s"""ALTER TABLE delta.`${dir.getPath}` |SET TBLPROPERTIES ( |$newTBLProperties |)""".stripMargin) for (feature <- featuresToRemove) { AlterTableDropFeatureDeltaCommand( table = DeltaTableV2(spark, deltaLog.dataPath), featureName = feature.name, truncateHistory = truncateHistory).run(spark) } assert(deltaLog.update().protocol === expectedDowngradedProtocol) } } test("Downgrade protocol version (1, 4)") { testProtocolVersionDowngrade( initialMinReaderVersion = 1, initialMinWriterVersion = 4, featuresToAdd = Seq(TestRemovableWriterFeature), featuresToRemove = Seq(TestRemovableWriterFeature), expectedDowngradedProtocol = Protocol(1, 4)) } // Initial minReader version is (2, 4), however, there are no legacy features that require // reader version 2. Therefore, the protocol version is downgraded to (1, 4). test("Downgrade protocol version (2, 4)") { testProtocolVersionDowngrade( initialMinReaderVersion = 2, initialMinWriterVersion = 4, featuresToAdd = Seq(TestRemovableWriterFeature), featuresToRemove = Seq(TestRemovableWriterFeature), expectedDowngradedProtocol = Protocol(1, 4)) } // Version (2, 5) enables column mapping which is a reader+writer feature and requires (2, 5). // Therefore, to downgrade from table features we need at least (2, 5). test("Downgrade protocol version (2, 5)") { testProtocolVersionDowngrade( initialMinReaderVersion = 2, initialMinWriterVersion = 5, featuresToAdd = Seq(TestRemovableWriterFeature), featuresToRemove = Seq(TestRemovableWriterFeature), expectedDowngradedProtocol = Protocol(2, 5)) } test("Downgrade protocol version (1, 1)") { testProtocolVersionDowngrade( initialMinReaderVersion = 1, initialMinWriterVersion = 1, featuresToAdd = Seq(TestRemovableWriterFeature), featuresToRemove = Seq(TestRemovableWriterFeature), expectedDowngradedProtocol = Protocol(1, 1)) } test("Downgrade protocol version on table created with (3, 7)") { // When the table is initialized with table features there are no active (implicit) legacy // features. After removing the last table feature we downgrade back to (1, 1). testProtocolVersionDowngrade( initialMinReaderVersion = 3, initialMinWriterVersion = 7, featuresToAdd = Seq(TestRemovableWriterFeature), featuresToRemove = Seq(TestRemovableWriterFeature), expectedDowngradedProtocol = Protocol(1, 1)) } test("Downgrade protocol version on table created with (1, 7)") { testProtocolVersionDowngrade( initialMinReaderVersion = 1, initialMinWriterVersion = 7, featuresToAdd = Seq(TestRemovableWriterFeature), featuresToRemove = Seq(TestRemovableWriterFeature), expectedDowngradedProtocol = Protocol(1, 1)) } test("Protocol version downgrade on a table with table features and added legacy feature") { // Added legacy feature should be removed and the protocol should be downgraded to (2, 5). testProtocolVersionDowngrade( initialMinReaderVersion = 3, initialMinWriterVersion = 7, featuresToAdd = Seq(TestRemovableWriterFeature) ++ Protocol(2, 5).implicitlySupportedFeatures, featuresToRemove = Seq(TestRemovableWriterFeature), expectedDowngradedProtocol = Protocol(2, 5)) // Added legacy feature should not be removed and the protocol should stay on (1, 7). testProtocolVersionDowngrade( initialMinReaderVersion = 1, initialMinWriterVersion = 7, featuresToAdd = Seq(TestRemovableWriterFeature, TestRemovableLegacyWriterFeature), featuresToRemove = Seq(TestRemovableWriterFeature), expectedDowngradedProtocol = Protocol(1, 7) .withFeature(TestRemovableLegacyWriterFeature)) // Legacy feature was manually removed. Protocol should be downgraded to (1, 1). testProtocolVersionDowngrade( initialMinReaderVersion = 1, initialMinWriterVersion = 7, featuresToAdd = Seq(TestRemovableWriterFeature, TestRemovableLegacyWriterFeature), featuresToRemove = Seq(TestRemovableLegacyWriterFeature, TestRemovableWriterFeature), expectedDowngradedProtocol = Protocol(1, 1)) // Start with writer table features and add a legacy reader+writer feature. testProtocolVersionDowngrade( initialMinReaderVersion = 1, initialMinWriterVersion = 7, featuresToAdd = Seq(TestRemovableWriterFeature, ColumnMappingTableFeature), featuresToRemove = Seq(TestRemovableWriterFeature), expectedDowngradedProtocol = Protocol(2, 7).withFeature(ColumnMappingTableFeature)) // Remove reader+writer legacy feature as well. testProtocolVersionDowngrade( initialMinReaderVersion = 1, initialMinWriterVersion = 7, featuresToAdd = Seq(TestRemovableLegacyReaderWriterFeature, TestRemovableWriterFeature), featuresToRemove = Seq(TestRemovableLegacyReaderWriterFeature, TestRemovableWriterFeature), expectedDowngradedProtocol = Protocol(1, 1)) } test("Protocol version is not downgraded when writer features exist") { testProtocolVersionDowngrade( initialMinReaderVersion = 1, initialMinWriterVersion = 7, featuresToAdd = Seq(TestRemovableWriterFeature, DomainMetadataTableFeature), featuresToRemove = Seq(TestRemovableWriterFeature), expectedDowngradedProtocol = protocolWithWriterFeature(DomainMetadataTableFeature)) } test("Protocol version is not downgraded when multiple reader+writer features exist") { testProtocolVersionDowngrade( initialMinReaderVersion = 3, initialMinWriterVersion = 7, featuresToAdd = Seq(TestRemovableReaderWriterFeature, DeletionVectorsTableFeature), featuresToRemove = Seq(TestRemovableReaderWriterFeature), expectedDowngradedProtocol = protocolWithReaderFeature(DeletionVectorsTableFeature)) } test("Protocol version is not downgraded when reader+writer features exist") { testProtocolVersionDowngrade( initialMinReaderVersion = 3, initialMinWriterVersion = 7, featuresToAdd = Seq(TestRemovableReaderWriterFeature, TestRemovableWriterFeature), featuresToRemove = Seq(TestRemovableWriterFeature), expectedDowngradedProtocol = protocolWithReaderFeature(TestRemovableReaderWriterFeature)) } for { truncateHistory <- BOOLEAN_DOMAIN enableCDF <- if (truncateHistory) Seq(false) else BOOLEAN_DOMAIN } test(s"Remove Deletion Vectors feature " + s"truncateHistory: $truncateHistory, enableCDF: $enableCDF") { val targetDF = spark.range(start = 0, end = 100, step = 1, numPartitions = 2) withSQLConf( DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> "true", DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> enableCDF.toString) { withTempPath { dir => val clock = new ManualClock(System.currentTimeMillis()) val targetLog = DeltaLog.forTable(spark, dir, clock) val defaultRetentionPeriod = DeltaConfigs.LOG_RETENTION.fromMetaData(targetLog.update().metadata).toString targetDF.write.format("delta").save(dir.toString) val targetTable = io.delta.tables.DeltaTable.forPath(dir.toString) // Add some DVs. targetTable.delete("id >= 90") // Assert that DVs exist. val preDowngradeSnapshot = targetLog.update() assert(DeletionVectorUtils.deletionVectorsWritable(preDowngradeSnapshot)) assert(preDowngradeSnapshot.numDeletionVectorsOpt === Some(1L)) // Attempting to drop Deletion Vectors feature will prohibit adding new DVs and remove // all DVs from the latest snapshot, but ultimately fail, because history will still // contain traces of the feature. For this reason, we have to wait for the retention period // to be over before we can downgrade the protocol. val e1 = intercept[DeltaTableFeatureException] { dropDVTableFeature(spark, targetLog, truncateHistory = false) } checkError( e1, "DELTA_FEATURE_DROP_WAIT_FOR_RETENTION_PERIOD", parameters = Map( "feature" -> DeletionVectorsTableFeature.name, "logRetentionPeriodKey" -> "delta.logRetentionDuration", "logRetentionPeriod" -> defaultRetentionPeriod, "truncateHistoryLogRetentionPeriod" -> truncateHistoryDefaultLogRetention.toString)) val postCleanupSnapshot = targetLog.update() assert(!DeletionVectorUtils.deletionVectorsWritable(postCleanupSnapshot)) assert(postCleanupSnapshot.numDeletionVectorsOpt.getOrElse(0L) === 0) assert(postCleanupSnapshot.numDeletedRecordsOpt.getOrElse(0L) === 0) spark.range(100, 120).write.format("delta").mode("append").save(dir.getCanonicalPath) spark.range(120, 140).write.format("delta").mode("append").save(dir.getCanonicalPath) // Table still contains historical data with DVs. Attempt should fail. val e2 = intercept[DeltaTableFeatureException] { dropDVTableFeature(spark, targetLog, truncateHistory = false) } checkError( e2, "DELTA_FEATURE_DROP_HISTORICAL_VERSIONS_EXIST", parameters = Map( "feature" -> DeletionVectorsTableFeature.name, "logRetentionPeriodKey" -> "delta.logRetentionDuration", "logRetentionPeriod" -> defaultRetentionPeriod, "truncateHistoryLogRetentionPeriod" -> truncateHistoryDefaultLogRetention.toString)) // Pretend retention period has passed. val clockAdvanceMillis = if (truncateHistory) { DeltaConfigs.getMilliSeconds(truncateHistoryDefaultLogRetention) + TimeUnit.HOURS.toMillis(24) } else { targetLog.deltaRetentionMillis(targetLog.update().metadata) + TimeUnit.DAYS.toMillis(3) } clock.advance(clockAdvanceMillis) // Cleanup logs. targetLog.cleanUpExpiredLogs(targetLog.update()) // History is now clean. We should be able to remove the feature. dropDVTableFeature(spark, targetLog, truncateHistory) val postDowngradeSnapshot = targetLog.update() val protocol = postDowngradeSnapshot.protocol assert(!DeletionVectorUtils.deletionVectorsWritable(postDowngradeSnapshot)) assert(postDowngradeSnapshot.numDeletionVectorsOpt.getOrElse(0L) === 0) assert(postDowngradeSnapshot.numDeletedRecordsOpt.getOrElse(0L) === 0) assert(!protocol.readerFeatureNames.contains(DeletionVectorsTableFeature.name)) } } } test("Can drop reader+writer feature when there is nothing to clean") { withTempPath { dir => val clock = new ManualClock(System.currentTimeMillis()) val targetLog = DeltaLog.forTable(spark, dir, clock) createTableWithFeature( targetLog, TestRemovableReaderWriterFeature, TestRemovableReaderWriterFeature.TABLE_PROP_KEY) sql( s"""ALTER TABLE delta.`${dir.getPath}` SET TBLPROPERTIES ( |'${TestRemovableReaderWriterFeature.TABLE_PROP_KEY}'='false' |)""".stripMargin) // Pretend retention period has passed. val clockAdvanceMillis = DeltaConfigs.getMilliSeconds(truncateHistoryDefaultLogRetention) clock.advance(clockAdvanceMillis + TimeUnit.HOURS.toMillis(24)) // History is now clean. We should be able to remove the feature. AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, targetLog.dataPath), TestRemovableReaderWriterFeature.name, truncateHistory = true).run(spark) assert(targetLog.update().protocol == Protocol(1, 2)) } } for (truncateHistory <- BOOLEAN_DOMAIN) test(s"Protocol version downgrade with Table Features - Basic test " + s"truncateHistory: ${truncateHistory}") { val expectedFeatures = Seq(RowTrackingFeature, DomainMetadataTableFeature) testProtocolVersionDowngrade( initialMinReaderVersion = 3, initialMinWriterVersion = 7, featuresToAdd = expectedFeatures :+ TestRemovableReaderWriterFeature, featuresToRemove = Seq(TestRemovableReaderWriterFeature), expectedDowngradedProtocol = Protocol(1, 7).withFeatures(expectedFeatures), truncateHistory = truncateHistory) } for (truncateHistory <- BOOLEAN_DOMAIN) test(s"Protocol version downgrade with Table Features - include legacy writer features: " + s"truncateHistory: ${truncateHistory}") { val expectedFeatures = Seq(DomainMetadataTableFeature, ChangeDataFeedTableFeature, AppendOnlyTableFeature) testProtocolVersionDowngrade( initialMinReaderVersion = 3, initialMinWriterVersion = 7, featuresToAdd = expectedFeatures :+ TestRemovableReaderWriterFeature, featuresToRemove = Seq(TestRemovableReaderWriterFeature), expectedDowngradedProtocol = Protocol(1, 7).withFeatures(expectedFeatures), truncateHistory = truncateHistory) } for (truncateHistory <- BOOLEAN_DOMAIN) test(s"Protocol version downgrade with Table Features - include legacy reader features: " + s"truncateHistory: ${truncateHistory}") { val expectedFeatures = Seq(DomainMetadataTableFeature, ChangeDataFeedTableFeature, ColumnMappingTableFeature) testProtocolVersionDowngrade( initialMinReaderVersion = 3, initialMinWriterVersion = 7, featuresToAdd = expectedFeatures :+ TestRemovableReaderWriterFeature, featuresToRemove = Seq(TestRemovableReaderWriterFeature), expectedDowngradedProtocol = Protocol(2, 7).withFeatures(expectedFeatures), truncateHistory = truncateHistory) } for (truncateHistory <- BOOLEAN_DOMAIN) test("Writer features that require history validation/truncation." + s" - truncateHistory: $truncateHistory") { withTempDir { dir => val clock = new ManualClock(System.currentTimeMillis()) val deltaLog = DeltaLog.forTable(spark, dir, clock) createTableWithFeature(deltaLog, TestRemovableWriterWithHistoryTruncationFeature, TestRemovableWriterWithHistoryTruncationFeature.TABLE_PROP_KEY) // Add some data. spark.range(100).write.format("delta").mode("overwrite").save(dir.getCanonicalPath) val e1 = intercept[DeltaTableFeatureException] { AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), TestRemovableWriterWithHistoryTruncationFeature.name).run(spark) } checkError( e1, "DELTA_FEATURE_DROP_WAIT_FOR_RETENTION_PERIOD", parameters = Map( "feature" -> TestRemovableWriterWithHistoryTruncationFeature.name, "logRetentionPeriodKey" -> "delta.logRetentionDuration", "logRetentionPeriod" -> "30 days", "truncateHistoryLogRetentionPeriod" -> "24 hours")) // Pretend retention period has passed. val clockAdvanceMillis = if (truncateHistory) { DeltaConfigs.getMilliSeconds(truncateHistoryDefaultLogRetention) + TimeUnit.HOURS.toMillis(24) } else { deltaLog.deltaRetentionMillis(deltaLog.update().metadata) + TimeUnit.DAYS.toMillis(3) } clock.advance(clockAdvanceMillis) AlterTableDropFeatureDeltaCommand( table = DeltaTableV2(spark, deltaLog.dataPath), featureName = TestRemovableWriterWithHistoryTruncationFeature.name, truncateHistory = truncateHistory).run(spark) assert(deltaLog.update().protocol === Protocol(1, 2)) } } private def dropV2CheckpointsTableFeature(spark: SparkSession, log: DeltaLog): Unit = { spark.sql(s"ALTER TABLE delta.`${log.dataPath}` DROP FEATURE " + s"`${V2CheckpointTableFeature.name}`") } private def testV2CheckpointTableFeatureDrop( v2CheckpointFormat: V2Checkpoint.Format, withInitialV2Checkpoint: Boolean, forceMultiPartCheckpoint: Boolean = false): Unit = { var confs = Seq( DeltaConfigs.CHECKPOINT_POLICY.defaultTablePropertyKey -> CheckpointPolicy.V2.name, DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key -> v2CheckpointFormat.name ) val expectedClassicCheckpointType = if (forceMultiPartCheckpoint) { confs :+= DeltaSQLConf.DELTA_CHECKPOINT_PART_SIZE.key -> "1" CheckpointInstance.Format.WITH_PARTS } else { CheckpointInstance.Format.SINGLE } withSQLConf(confs: _*) { withTempPath { dir => val clock = new ManualClock(System.currentTimeMillis()) val targetLog = DeltaLog.forTable(spark, dir, clock) val defaultRetentionPeriod = DeltaConfigs.LOG_RETENTION.fromMetaData(targetLog.update().metadata).toString val targetDF = spark.range(start = 0, end = 100, step = 1, numPartitions = 2) targetDF.write.format("delta").save(dir.toString) val initialCheckpointCount = if (withInitialV2Checkpoint) 1 else 0 if (withInitialV2Checkpoint) { // Create a v2 checkpoint. targetLog.checkpoint() } // Assert that the current checkpointing policy requires v2 checkpoint support. val preDowngradeSnapshot = targetLog.update() assert( DeltaConfigs.CHECKPOINT_POLICY .fromMetaData(preDowngradeSnapshot.metadata) .needsV2CheckpointSupport) val checkpointFiles = targetLog.listFrom(0).filter(FileNames.isCheckpointFile) assert(checkpointFiles.length == initialCheckpointCount) checkpointFiles.foreach { f => assert(CheckpointInstance(f.getPath).format == CheckpointInstance.Format.V2) } // Dropping the feature should fail because // 1. The checkpointing policy in metadata requires v2 checkpoint support. // 2. Also, when initialCheckpointCount = true, there is a v2 checkpoint. val e1 = intercept[DeltaTableFeatureException] { dropV2CheckpointsTableFeature(spark, targetLog) } checkError( e1, "DELTA_FEATURE_DROP_WAIT_FOR_RETENTION_PERIOD", parameters = Map( "feature" -> V2CheckpointTableFeature.name, "logRetentionPeriodKey" -> "delta.logRetentionDuration", "logRetentionPeriod" -> defaultRetentionPeriod, "truncateHistoryLogRetentionPeriod" -> truncateHistoryDefaultLogRetention.toString)) val postCleanupCheckpointFiles = targetLog.listFrom(0).filter(FileNames.isCheckpointFile).toList // Assert that a new classic checkpoint has been created. val uniqueCheckpointCount = postCleanupCheckpointFiles .drop(initialCheckpointCount) .map { checkpointFile => val checkpointInstance = CheckpointInstance(checkpointFile.getPath) assert(checkpointInstance.format == expectedClassicCheckpointType) checkpointInstance.version } // Count a multi-part checkpoint as a single checkpoint. .toSet.size // Drop feature command generates one classic checkpoints after v2 checkpoint cleanup. val expectedClassicCheckpointCount = 1 assert(uniqueCheckpointCount == expectedClassicCheckpointCount) spark.range(100, 120).write.format("delta").mode("append").save(dir.getCanonicalPath) // V2 Checkpoint related traces have not been cleaned up yet. Attempt should fail. val e2 = intercept[DeltaTableFeatureException] { dropV2CheckpointsTableFeature(spark, targetLog) } checkError( e2, "DELTA_FEATURE_DROP_HISTORICAL_VERSIONS_EXIST", parameters = Map( "feature" -> V2CheckpointTableFeature.name, "logRetentionPeriodKey" -> "delta.logRetentionDuration", "logRetentionPeriod" -> defaultRetentionPeriod, "truncateHistoryLogRetentionPeriod" -> truncateHistoryDefaultLogRetention.toString)) // Pretend retention period has passed. clock.advance( targetLog.deltaRetentionMillis(targetLog.update().metadata) + TimeUnit.DAYS.toMillis(3)) // History is now clean. We should be able to remove the feature. dropV2CheckpointsTableFeature(spark, targetLog) val postDowngradeSnapshot = targetLog.update() val protocol = postDowngradeSnapshot.protocol assert(!protocol.readerFeatureNames.contains(V2CheckpointTableFeature.name)) assert( !DeltaConfigs.CHECKPOINT_POLICY .fromMetaData(postDowngradeSnapshot.metadata) .needsV2CheckpointSupport) assert(targetLog.listFrom(0).filter(FileNames.isCheckpointFile).forall { f => CheckpointInstance(f.getPath).format == expectedClassicCheckpointType }) } } } for ( v2CheckpointFormat <- V2Checkpoint.Format.ALL; withInitialV2Checkpoint <- BOOLEAN_DOMAIN) test(s"Remove v2 Checkpoints Feature [v2CheckpointFormat: ${v2CheckpointFormat.name}; " + s"withInitialV2Checkpoint: $withInitialV2Checkpoint; forceMultiPartCheckpoint: false]") { testV2CheckpointTableFeatureDrop(v2CheckpointFormat, withInitialV2Checkpoint) } test( s"Remove v2 Checkpoints Feature [v2CheckpointFormat: ${V2Checkpoint.Format.PARQUET.name}; " + s"withInitialV2Checkpoint: true; forceMultiPartCheckpoint: true]") { testV2CheckpointTableFeatureDrop(V2Checkpoint.Format.PARQUET, true, true) } private def testRemoveVacuumProtocolCheckTableFeature( enableFeatureInitially: Boolean, additionalTableProperties: Seq[(String, String)] = Seq.empty, downgradeFailsWithException: Option[String] = None, featureExpectedAtTheEnd: Boolean = false): Unit = { val featureName = VacuumProtocolCheckTableFeature.name withTempTable(createTable = false) { tableName => // Register a temporary InMemory-CC builder to support CatalogOwned table creation. CatalogOwnedCommitCoordinatorProvider.clearBuilders() CatalogOwnedCommitCoordinatorProvider.registerBuilder( catalogName = CatalogOwnedTableUtils.DEFAULT_CATALOG_NAME_FOR_TESTING, TrackingInMemoryCommitCoordinatorBuilder(batchSize = 1) ) val finalAdditionalTableProperty = if (enableFeatureInitially) { additionalTableProperties ++ Seq((s"$FEATURE_PROP_PREFIX${featureName}", "supported")) } else { additionalTableProperties } var additionalTablePropertyString = finalAdditionalTableProperty.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ") if (additionalTablePropertyString.nonEmpty) { additionalTablePropertyString = s", $additionalTablePropertyString" } sql( s"""CREATE TABLE $tableName (id bigint) USING delta |TBLPROPERTIES ( | delta.minReaderVersion = $TABLE_FEATURES_MIN_READER_VERSION, | delta.minWriterVersion = $TABLE_FEATURES_MIN_WRITER_VERSION | $additionalTablePropertyString |)""".stripMargin) val (deltaLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName)) val protocol = snapshot.protocol assert(protocol.minReaderVersion == (if (enableFeatureInitially) TABLE_FEATURES_MIN_READER_VERSION else 1)) assert(protocol.minWriterVersion == (if (enableFeatureInitially) TABLE_FEATURES_MIN_WRITER_VERSION else 1)) assert(protocol.readerFeatures.isDefined === enableFeatureInitially) downgradeFailsWithException match { case Some(exceptionClass) => val e = intercept[DeltaTableFeatureException] { AlterTableDropFeatureDeltaCommand(DeltaTableV2(spark, deltaLog.dataPath), featureName) .run(spark) } assert(e.getErrorClass == exceptionClass) case None => AlterTableDropFeatureDeltaCommand(DeltaTableV2(spark, deltaLog.dataPath), featureName) .run(spark) } val latestProtocolReaderFeatures = deltaLog.update().protocol.readerFeatures.getOrElse(Set()) assert( latestProtocolReaderFeatures.contains(VacuumProtocolCheckTableFeature.name) === featureExpectedAtTheEnd) assertPropertiesAndShowTblProperties(deltaLog, tableHasFeatures = featureExpectedAtTheEnd) } } test("Remove VacuumProtocolCheckTableFeature when it was enabled") { testRemoveVacuumProtocolCheckTableFeature(enableFeatureInitially = true) } test("Removing VacuumProtocolCheckTableFeature should fail when dependent feature " + "Catalog Owned is enabled") { testRemoveVacuumProtocolCheckTableFeature( enableFeatureInitially = true, additionalTableProperties = Seq( (s"$FEATURE_PROP_PREFIX${CatalogOwnedTableFeature.name}", "supported")), downgradeFailsWithException = Some("DELTA_FEATURE_DROP_DEPENDENT_FEATURE"), featureExpectedAtTheEnd = true) } test("Removing VacuumProtocolCheckTableFeature should fail when it is not enabled") { testRemoveVacuumProtocolCheckTableFeature( enableFeatureInitially = false, downgradeFailsWithException = Some("DELTA_FEATURE_DROP_FEATURE_NOT_PRESENT") ) } private def validateICTRemovalMetrics( usageLogs: Seq[UsageRecord], expectEnablementProperty: Boolean, expectProvenanceTimestampProperty: Boolean, expectProvenanceVersionProperty: Boolean): Unit = { val dropFeatureBlob = usageLogs .find(_.tags.get("opType").contains("delta.inCommitTimestampFeatureRemovalMetrics")) .getOrElse(fail("Expected a log for inCommitTimestampFeatureRemovalMetrics")) val blob = JsonUtils.fromJson[Map[String, String]](dropFeatureBlob.blob) assert(blob.contains("downgradeTimeMs")) val traceRemovalNeeded = expectEnablementProperty || expectProvenanceTimestampProperty || expectProvenanceVersionProperty assert(blob.get("traceRemovalNeeded").contains(traceRemovalNeeded.toString)) assert(blob .get(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key) .contains(expectEnablementProperty.toString)) assert(blob .get(DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.key) .contains(expectProvenanceTimestampProperty.toString)) assert(blob .get(DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.key) .contains(expectProvenanceVersionProperty.toString)) } test("drop InCommitTimestamp -- ICT enabled from commit 0") { withTempDir { dir => val featureEnablementKey = DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key spark.sql(s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta" + s" TBLPROPERTIES ('${featureEnablementKey}' = 'true')") val deltaLog = DeltaLog.forTable(spark, dir) val featurePropertyKey = InCommitTimestampTableFeature.name val usageLogs = Log4jUsageLogger.track { AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), featurePropertyKey) .run(spark) } val snapshot = deltaLog.update() // Writer feature is removed from the writer features set. assert(!snapshot.protocol.writerFeatureNames.contains(featurePropertyKey)) assert(!DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(snapshot.metadata)) validateICTRemovalMetrics( usageLogs, expectEnablementProperty = true, expectProvenanceTimestampProperty = false, expectProvenanceVersionProperty = false) // Running the command again should throw an exception. val e = intercept[DeltaTableFeatureException] { AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), featurePropertyKey) .run(spark) } assert(e.getErrorClass == "DELTA_FEATURE_DROP_FEATURE_NOT_PRESENT") } } test("drop InCommitTimestamp -- ICT enabled after commit 0") { withTempDir { dir => val featureEnablementKey = DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key val featurePropertyKey = InCommitTimestampTableFeature.name sql(s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta " + s"TBLPROPERTIES ('${featureEnablementKey}' = 'false')") val deltaLog = DeltaLog.forTable(spark, dir) assert(!deltaLog.snapshot.metadata.configuration.contains(featurePropertyKey)) sql(s"ALTER TABLE delta.`${dir.getCanonicalPath}` " + s"SET TBLPROPERTIES ('${featureEnablementKey}' = 'true')") val snapshotV1 = deltaLog.update() assert(snapshotV1.protocol.writerFeatureNames.contains(featurePropertyKey)) assert(snapshotV1.metadata.configuration.contains(featureEnablementKey)) val ictProvenanceProperties = Seq( DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.key, DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.key) ictProvenanceProperties.foreach(prop => assert(snapshotV1.metadata.configuration.contains(prop))) val usageLogs = Log4jUsageLogger.track { AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), featurePropertyKey) .run(spark) } val snapshot = deltaLog.update() // Writer feature is removed from the writer features set. assert(!snapshot.protocol.writerFeatureNames.contains(featurePropertyKey)) assert(!DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(snapshot.metadata)) // The provenance properties should also have been removed. ictProvenanceProperties.foreach(prop => assert(!snapshot.metadata.configuration.contains(prop))) validateICTRemovalMetrics( usageLogs, expectEnablementProperty = true, expectProvenanceTimestampProperty = true, expectProvenanceVersionProperty = true) } } test("drop InCommitTimestamp --- only one table property") { withTempDir { dir => // Dropping the ICT table feature should also remove any ICT provenance // table properties even when the ICT enablement table property is not present. spark.sql( s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta" + s" TBLPROPERTIES ('${DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key}' = 'true')") val deltaLog = DeltaLog.forTable(spark, dir) // Remove the enablement property. AlterTableUnsetPropertiesDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), Seq(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key), ifExists = true).run(spark) // Set the IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION property. AlterTableSetPropertiesDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), Map(DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.key -> "1")).run(spark) val snapshot1 = deltaLog.update() assert(snapshot1.protocol.writerFeatureNames.contains(InCommitTimestampTableFeature.name)) // Ensure that the enablement property is not set. assert(!DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(snapshot1.metadata)) assert(snapshot1.metadata.configuration.contains( DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.key)) val usageLogs = Log4jUsageLogger.track { AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), InCommitTimestampTableFeature.name) .run(spark) } val snapshot2 = deltaLog.update() assert(!snapshot2.protocol.writerFeatureNames.contains(InCommitTimestampTableFeature.name)) assert(!DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(snapshot2.metadata)) assert(!snapshot2.metadata.configuration.contains( DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.key)) validateICTRemovalMetrics( usageLogs, expectEnablementProperty = false, expectProvenanceTimestampProperty = false, expectProvenanceVersionProperty = true) } } test("drop InCommitTimestamp --- no table property") { withTempDir { dir => spark.sql( s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta" + s" TBLPROPERTIES ('${DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key}' = 'true')") val deltaLog = DeltaLog.forTable(spark, dir) // Remove the enablement property. AlterTableUnsetPropertiesDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), Seq(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key), ifExists = true).run(spark) val usageLogs = Log4jUsageLogger.track { AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, deltaLog.dataPath), InCommitTimestampTableFeature.name) .run(spark) } val snapshot = deltaLog.update() assert(!snapshot.protocol.writerFeatureNames.contains(InCommitTimestampTableFeature.name)) assert(!DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(snapshot.metadata)) validateICTRemovalMetrics( usageLogs, expectEnablementProperty = false, expectProvenanceTimestampProperty = false, expectProvenanceVersionProperty = false) } } // ---- Coordinated Commits Drop Feature Tests ---- private def setUpCoordinatedCommitsTable(dir: File, mcBuilder: CommitCoordinatorBuilder): Unit = { CommitCoordinatorProvider.clearNonDefaultBuilders() CommitCoordinatorProvider.registerBuilder(mcBuilder) val tablePath = dir.getAbsolutePath val log = DeltaLog.forTable(spark, tablePath) val fs = log.logPath.getFileSystem(log.newDeltaHadoopConf()) val commitCoordinatorConf = Map(DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.key -> mcBuilder.getName) val newMetadata = Metadata().copy(configuration = commitCoordinatorConf) log.startTransaction().commitManually(newMetadata) assert(log.unsafeVolatileSnapshot.version === 0) assert(log.unsafeVolatileSnapshot.metadata.coordinatedCommitsCoordinatorName === Some(mcBuilder.getName)) assert(log.unsafeVolatileSnapshot.tableCommitCoordinatorClientOpt.nonEmpty) assert(log.unsafeVolatileSnapshot.metadata.coordinatedCommitsTableConf === Map.empty) // upgrade commit always filesystem based assert(fs.exists(FileNames.unsafeDeltaFile(log.logPath, 0))) // Do a couple of commits on the coordinated-commits table (1 to 2).foreach { version => log.startTransaction() .commitManually(DeltaTestUtils.createTestAddFile(s"$version")) assert(log.unsafeVolatileSnapshot.version === version) assert(log.unsafeVolatileSnapshot.tableCommitCoordinatorClientOpt.nonEmpty) assert(log.unsafeVolatileSnapshot.metadata.coordinatedCommitsCoordinatorName.nonEmpty) assert(log.unsafeVolatileSnapshot.metadata.coordinatedCommitsCoordinatorConf === Map.empty) assert(log.unsafeVolatileSnapshot.metadata.coordinatedCommitsTableConf === Map.empty) } } private def validateCoordinatedCommitsDropLogs( usageLogs: Seq[UsageRecord], expectTablePropertiesPresent: Boolean, expectUnbackfilledCommitsPresent: Boolean, exceptionMessageOpt: Option[String] = None): Unit = { val dropFeatureBlob = usageLogs .find(_.tags.get("opType").contains("delta.coordinatedCommitsFeatureRemovalMetrics")) .getOrElse(fail("Expected a log for coordinatedCommitsFeatureRemovalMetrics")) val blob = JsonUtils.fromJson[Map[String, String]](dropFeatureBlob.blob) assert(blob.contains("downgradeTimeMs")) val expectTraceRemovalNeeded = expectTablePropertiesPresent || expectUnbackfilledCommitsPresent assert(blob.get("traceRemovalNeeded").contains(expectTraceRemovalNeeded.toString)) Seq( DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.key, DeltaConfigs.COORDINATED_COMMITS_TABLE_CONF.key).foreach { prop => assert(blob.get(prop).contains(expectTablePropertiesPresent.toString)) } // COORDINATED_COMMITS_COORDINATOR_CONF is not used by "in-memory" commit coordinator. assert(blob .get("postDisablementUnbackfilledCommitsPresent") .contains(expectUnbackfilledCommitsPresent.toString)) assert( blob.get(DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF.key).contains("false")) assert(blob.get("traceRemovalSuccess").contains(exceptionMessageOpt.isEmpty.toString)) exceptionMessageOpt.foreach { exceptionMessage => assert(blob.get("traceRemovalException").contains(exceptionMessage)) } } test("basic coordinated commits feature drop") { withTempDir { dir => val mcBuilder = TrackingInMemoryCommitCoordinatorBuilder(batchSize = 1000) setUpCoordinatedCommitsTable(dir, mcBuilder) val log = DeltaLog.forTable(spark, dir) val usageLogs = Log4jUsageLogger.track { AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, log.dataPath), CoordinatedCommitsTableFeature.name) .run(spark) } val snapshot = log.update() assert(!CoordinatedCommitsUtils.TABLE_PROPERTY_KEYS.exists( snapshot.metadata.configuration.contains(_))) assert(!snapshot.protocol.writerFeatures.exists( _.contains(CoordinatedCommitsTableFeature.name))) validateCoordinatedCommitsDropLogs( usageLogs, expectTablePropertiesPresent = true, expectUnbackfilledCommitsPresent = false) } } test("backfill failure during coordinated commits feature drop") { withTempDir { dir => var shouldFailBackfill = true val alternatingFailureBackfillClient = new TrackingCommitCoordinatorClient(new InMemoryCommitCoordinator(1000) { override def backfillToVersion( logStore: LogStore, hadoopConf: Configuration, tableDesc: TableDescriptor, startVersion: Long, endVersionOpt: java.lang.Long): Unit = { // Backfill fails on every other attempt. if (shouldFailBackfill) { shouldFailBackfill = !shouldFailBackfill throw new IllegalStateException("backfill failed") } else { super.backfillToVersion( logStore, hadoopConf, tableDesc, startVersion, endVersionOpt) } } }) val mcBuilder = TrackingInMemoryCommitCoordinatorBuilder(100, Some(alternatingFailureBackfillClient)) setUpCoordinatedCommitsTable(dir, mcBuilder) val log = DeltaLog.forTable(spark, dir) val usageLogs = Log4jUsageLogger.track { val e = intercept[IllegalStateException] { AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, log.dataPath), CoordinatedCommitsTableFeature.name) .run(spark) } assert(e.getMessage.contains("backfill failed")) } validateCoordinatedCommitsDropLogs( usageLogs, expectTablePropertiesPresent = true, expectUnbackfilledCommitsPresent = false, exceptionMessageOpt = Some("backfill failed")) def backfilledCommitExists(v: Long): Boolean = { val fs = log.logPath.getFileSystem(log.newDeltaHadoopConf()) fs.exists(FileNames.unsafeDeltaFile(log.logPath, v)) } // Backfill of the commit which disables coordinated commits failed. assert(!backfilledCommitExists(3)) // The commit coordinator still tracks the commit that disables it. val commitsFromCommitCoordinator = log.snapshot.tableCommitCoordinatorClientOpt.get.getCommits(Some(3L)) assert(commitsFromCommitCoordinator.getCommits.asScala.exists(_.getVersion == 3)) // The next drop attempt will also trigger an explicit backfill. val usageLogs2 = Log4jUsageLogger.track { AlterTableDropFeatureDeltaCommand( DeltaTableV2(spark, log.dataPath), CoordinatedCommitsTableFeature.name) .run(spark) } validateCoordinatedCommitsDropLogs( usageLogs2, expectTablePropertiesPresent = false, expectUnbackfilledCommitsPresent = true) val snapshot = log.update() assert(snapshot.version === 4) assert(backfilledCommitExists(3)) // The protocol downgrade commit is performed through logstore directly. assert(backfilledCommitExists(4)) assert(!CoordinatedCommitsUtils.TABLE_PROPERTY_KEYS.exists( snapshot.metadata.configuration.contains(_))) assert(!snapshot.protocol.writerFeatures.exists( _.contains(CoordinatedCommitsTableFeature.name))) } } // ---- End Coordinated Commits Drop Feature Tests ---- private def testRedirectFeature( redirectFeature: TableFeature, tableRedirect: TableRedirect, enableFastDrop: Boolean, unsetTableProperty: Boolean): Unit = { withSQLConf(DeltaSQLConf.FAST_DROP_FEATURE_ENABLED.key -> enableFastDrop.toString) { test(s"drop ${redirectFeature.name} with fast drop - " + s"enableFastDrop=$enableFastDrop, unsetTableProperty=$unsetTableProperty") { withTempDir { dir => spark.sql(s"CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta") val deltaLog = DeltaLog.forTable(spark, dir) val redirectSpec = new PathBasedRedirectSpec("sourcePath", "targetPath") tableRedirect.add( deltaLog, catalogTableOpt = None, PathBasedRedirectSpec.REDIRECT_TYPE, redirectSpec) if (unsetTableProperty) { sql(s"ALTER TABLE delta.`${dir.getCanonicalPath}` UNSET TBLPROPERTIES " + s"('${tableRedirect.config.key}')") } val featureName = redirectFeature.name // Both RedirectReaderWriterFeature and RedirectWriterOnlyFeature can be immediately // dropped as they don't require history truncation. This is because there is no // associated action with the features. AlterTableDropFeatureDeltaCommand(DeltaTableV2(spark, deltaLog.dataPath), featureName) .run(spark) val snapshot = deltaLog.update() // Writer feature is removed from the writer features set. assert(!snapshot.protocol.writerFeatureNames.contains(featureName)) // Reader feature is removed from the reader features set. assert(!snapshot.protocol.readerFeatureNames.contains(featureName)) assert(tableRedirect.config.fromMetaData(snapshot.metadata).isEmpty) assertPropertiesAndShowTblProperties(deltaLog) // Running the command again should throw an exception. val e = intercept[DeltaTableFeatureException] { AlterTableDropFeatureDeltaCommand(DeltaTableV2(spark, deltaLog.dataPath), featureName) .run(spark) } assert(e.getErrorClass == "DELTA_FEATURE_DROP_FEATURE_NOT_PRESENT") } } } } BOOLEAN_DOMAIN.foreach { unsetTableProperty => BOOLEAN_DOMAIN.foreach { enableFastDrop => // Test both writer-only and reader writer redirect feature. testRedirectFeature( RedirectWriterOnlyFeature, RedirectWriterOnly, enableFastDrop, unsetTableProperty) testRedirectFeature( RedirectReaderWriterFeature, RedirectReaderWriter, enableFastDrop, unsetTableProperty) } } // Create a table for testing that has an unsupported feature. private def withTestTableWithUnsupportedWriterFeature( emptyTable: Boolean)(testCode: String => Unit): Unit = { val tableName = "test_table" withTable(tableName) { if (emptyTable) { sql(s"CREATE TABLE $tableName(id INT) USING DELTA") } else { sql(s"CREATE TABLE $tableName USING DELTA AS SELECT 1 AS id") } sql(s"""ALTER TABLE $tableName SET TBLPROPERTIES ('delta.minReaderVersion' = '3', 'delta.minWriterVersion' = '7')""") val deltaLogPath = DeltaLog.forTable(spark, TableIdentifier(tableName)).logPath .toString.stripPrefix("file:") // scalastyle:off val commitJson = """{"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1702304249309}} |{"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":[],"writerFeatures":["unsupportedWriter"]}}""".stripMargin // scalastyle:on Files.write(Paths.get(deltaLogPath, "00000000000000000002.json"), commitJson.getBytes) testCode(tableName) } } // Test that write commands error out when unsupported features in the table protocol. private def testUnsupportedFeature( commandName: String, emptyTable: Boolean)(command: String => Unit): Unit = { test(s"Writes using $commandName error out when unsupported writer features are present") { withTestTableWithUnsupportedWriterFeature(emptyTable) { tableName => intercept[DeltaUnsupportedTableFeatureException] { command(tableName) } } } } testUnsupportedFeature("INSERT", emptyTable = true) { testTableName => sql(s"INSERT INTO $testTableName VALUES (2)") } testUnsupportedFeature("UPDATE", emptyTable = false) { testTableName => sql(s"UPDATE $testTableName SET id = 2") } testUnsupportedFeature("DELETE", emptyTable = false) { testTableName => sql(s"DELETE FROM $testTableName WHERE id > 0") } testUnsupportedFeature("MERGE", emptyTable = false) { testTableName => sql(s"""MERGE INTO $testTableName t |USING $testTableName s |ON s.id = t.id + 100 |WHEN NOT MATCHED THEN INSERT *""".stripMargin) } testUnsupportedFeature("CREATE OR REPLACE TABLE", emptyTable = false) { testTableName => sql(s"CREATE OR REPLACE TABLE $testTableName (other_column INT) USING DELTA") } testUnsupportedFeature("ManualUpdate commit", emptyTable = true) { testTableName => val deltaLog = DeltaLog.forTable(spark, TableIdentifier(testTableName)) deltaLog.startTransaction(None) .commit(Seq(DeltaTestUtils.createTestAddFile()), DeltaOperations.ManualUpdate) } testUnsupportedFeature("SHALLOW CLONE", emptyTable = true) { testTableName => val cloneSourceTableName = "clone_source_table" withTable(cloneSourceTableName) { sql(s"DELETE FROM $testTableName") sql(s"CREATE TABLE $cloneSourceTableName USING delta AS SELECT 1337 as id") sql(s"CREATE OR REPLACE TABLE $testTableName SHALLOW CLONE $cloneSourceTableName") } } private def assertPropertiesAndShowTblProperties( deltaLog: DeltaLog, tableHasFeatures: Boolean = false): Unit = { val configs = deltaLog.snapshot.metadata.configuration.map { case (k, v) => k.toLowerCase(Locale.ROOT) -> v } assert(!configs.contains(Protocol.MIN_READER_VERSION_PROP)) assert(!configs.contains(Protocol.MIN_WRITER_VERSION_PROP)) assert(!configs.exists(_._1.startsWith(FEATURE_PROP_PREFIX))) val tblProperties = sql(s"SHOW TBLPROPERTIES delta.`${deltaLog.dataPath.toString}`").collect() assert( tblProperties.exists(row => row.getAs[String]("key") == Protocol.MIN_READER_VERSION_PROP)) assert( tblProperties.exists(row => row.getAs[String]("key") == Protocol.MIN_WRITER_VERSION_PROP)) assert(tableHasFeatures === tblProperties.exists(row => row.getAs[String]("key").startsWith(FEATURE_PROP_PREFIX))) val rows = tblProperties.filter(row => row.getAs[String]("key").startsWith(FEATURE_PROP_PREFIX)) for (row <- rows) { val name = row.getAs[String]("key").substring(FEATURE_PROP_PREFIX.length) val status = row.getAs[String]("value") assert(TableFeature.featureNameToFeature(name).isDefined) assert(status == FEATURE_PROP_SUPPORTED) } } private def captureProtocolChangeEventBlob(f: => Unit): Map[String, Any] = { val logs = Log4jUsageLogger.track(f) val blob = logs.collectFirst { case r if r.metric == MetricDefinitions.EVENT_TAHOE.name && r.tags.get("opType").contains("delta.protocol.change") => r.blob } require(blob.nonEmpty, "Expecting a delta.protocol.change event but didn't see any.") blob.map(JsonUtils.fromJson[Map[String, Any]]).head } } class DeltaProtocolVersionSuite extends DeltaProtocolVersionSuiteBase ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaRestartSessionSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.SparkFunSuite import org.apache.spark.sql.SparkSession import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.internal.SQLConf class DeltaRestartSessionSuite extends SparkFunSuite { test("restart Spark session should work") { withTempDir { dir => var spark = SparkSession.builder().master("local[2]") .config(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[DeltaCatalog].getName) .getOrCreate() try { val path = dir.getCanonicalPath spark.range(10).write.format("delta").mode("overwrite").save(path) spark.read.format("delta").load(path).count() spark.stop() spark = SparkSession.builder().master("local[2]") .config(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[DeltaCatalog].getName) .getOrCreate() spark.range(10).write.format("delta").mode("overwrite").save(path) spark.read.format("delta").load(path).count() } finally { spark.stop() } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaRetentionSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import scala.concurrent.duration._ import scala.language.postfixOps import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile import org.apache.spark.sql.delta.actions.{Action, AddFile, RemoveFile, SetTransaction} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.FileNames import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, FileSystem, Path, RawLocalFileSystem} import org.apache.spark.SparkConf import org.apache.spark.sql.QueryTest import org.apache.spark.util.ManualClock // scalastyle:off: removeFile class DeltaRetentionSuite extends QueryTest with DeltaRetentionSuiteBase with DeltaSQLTestUtils with DeltaSQLCommandTest with CheckpointProtectionTestUtilsMixin { protected override def sparkConf: SparkConf = super.sparkConf override protected def getLogFiles(dir: File): Seq[File] = getDeltaFiles(dir) ++ getUnbackfilledDeltaFiles(dir) ++ getCheckpointFiles(dir)++ getCrcFiles(dir) test("startTxnWithManualLogCleanup") { withTempDir { tempDir => val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) startTxnWithManualLogCleanup(log).commit(Nil, testOp) assert(!log.enableExpiredLogCleanup()) } } test("delete expired logs") { withTempDir { tempDir => val startTime = getStartTimeForRetentionTest val clock = new ManualClock(startTime) val actualTestStartTime = System.currentTimeMillis() val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath), clock) val logPath = new File(log.logPath.toUri) (1 to 5).foreach { i => val txn = if (i == 1) startTxnWithManualLogCleanup(log) else log.startTransaction() val file = createTestAddFile(encodedPath = i.toString) :: Nil val delete: Seq[Action] = if (i > 1) { val timestamp = startTime + (System.currentTimeMillis()-actualTestStartTime) RemoveFile(i - 1 toString, Some(timestamp), true) :: Nil } else { Nil } txn.commit(delete ++ file, testOp) } val initialFiles = getLogFiles(logPath) // Shouldn't clean up, no checkpoint, no expired files log.cleanUpExpiredLogs(log.snapshot) assert(initialFiles === getLogFiles(logPath)) clock.advance(intervalStringToMillis(DeltaConfigs.LOG_RETENTION.defaultValue) + intervalStringToMillis("interval 1 day")) // Shouldn't clean up, no checkpoint, although all files have expired log.cleanUpExpiredLogs(log.snapshot) assert(initialFiles === getLogFiles(logPath)) log.checkpoint() // With V2 checkpoints (QoL feature for CatalogOwned tables), checkpoint files have UUIDs // and may be .json or .parquet (e.g., "04.checkpoint..json" instead of // "04.checkpoint.parquet"). We check for the commit log and CRC, and verify that at least // one checkpoint file exists for version 4. log.cleanUpExpiredLogs(log.snapshot) val afterCleanup = getLogFiles(logPath) assert(initialFiles !== afterCleanup) val afterCleanupNames = afterCleanup.map(_.getName) assert(afterCleanupNames.exists(_.contains("00000000000000000004.json")), s"Missing 04.json in: ${afterCleanupNames.mkString("\n")}") assert(afterCleanupNames.exists(name => name.contains("00000000000000000004.checkpoint")), s"Missing 04.checkpoint file in: ${afterCleanupNames.mkString("\n")}") assert(afterCleanupNames.exists(_.contains("00000000000000000004.crc")), s"Missing 04.crc in: ${afterCleanupNames.mkString("\n")}") } } test("log files being already deleted shouldn't fail log deletion job") { withTempDir { tempDir => val startTime = getStartTimeForRetentionTest val clock = new ManualClock(startTime) val actualTestStartTime = System.currentTimeMillis() val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath), clock) val logPath = new File(log.logPath.toUri) val iterationCount = (log.checkpointInterval() * 2) + 1 (1 to iterationCount).foreach { i => val txn = if (i == 1) startTxnWithManualLogCleanup(log) else log.startTransaction() val file = createTestAddFile(encodedPath = i.toString) :: Nil val delete: Seq[Action] = if (i > 1) { val timestamp = startTime + (System.currentTimeMillis()-actualTestStartTime) RemoveFile(i - 1 toString, Some(timestamp), true) :: Nil } else { Nil } val version = txn.commit(delete ++ file, testOp) val deltaFile = new File(FileNames.unsafeDeltaFile(log.logPath, version).toUri) deltaFile.setLastModified(clock.getTimeMillis() + i * 10000) val crcFile = new File(FileNames.checksumFile(log.logPath, version).toUri) crcFile.setLastModified(clock.getTimeMillis() + i * 10000) val chk = new File(FileNames.checkpointFileSingular(log.logPath, version).toUri) if (chk.exists()) { chk.setLastModified(clock.getTimeMillis() + i * 10000) } } // delete some files in the middle val middleStartIndex = log.checkpointInterval() / 2 getDeltaFiles(logPath).sortBy(_.getName).slice( middleStartIndex, middleStartIndex + log.checkpointInterval()).foreach(_.delete()) clock.advance(intervalStringToMillis(DeltaConfigs.LOG_RETENTION.defaultValue) + intervalStringToMillis("interval 2 day")) log.cleanUpExpiredLogs(log.snapshot) val minDeltaFile = getDeltaFiles(logPath).map(f => FileNames.deltaVersion(new Path(f.toString))).min val maxChkFile = getCheckpointFiles(logPath).map(f => FileNames.checkpointVersion(new Path(f.toString))).max assert(maxChkFile === minDeltaFile, "Delta files before the last checkpoint version should have been deleted") // With V2 checkpoints (QoL feature for CatalogOwned tables), cleanup behavior may retain // additional checkpoint files for safety. We check that there are no more than 2 checkpoint // files remaining (classic behavior expects 1, V2 may have up to 2). assert(getCheckpointFiles(logPath).length <= 2, s"There should be at most 2 checkpoint files, but found: " + s"${getCheckpointFiles(logPath).length}") } } testQuietly( "RemoveFiles persist across checkpoints as tombstones if retention time hasn't expired") { withTempDir { tempDir => val clock = new ManualClock(getStartTimeForRetentionTest) val log1 = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath), clock) val txn = startTxnWithManualLogCleanup(log1) val files1 = (1 to 10).map(f => createTestAddFile(encodedPath = f.toString)) txn.commit(files1, testOp) val txn2 = log1.startTransaction() val files2 = (1 to 4).map(f => RemoveFile(f.toString, Some(clock.getTimeMillis()))) txn2.commit(files2, testOp) log1.checkpoint() DeltaLog.clearCache() val log2 = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath), clock) assert(log2.snapshot.tombstones.count() === 4) assert(log2.snapshot.allFiles.count() === 6) } } def removeFileCountFromUnderlyingCheckpoint(snapshot: Snapshot): Long = { val df = snapshot.checkpointProvider .allActionsFileIndexes() .map(snapshot.deltaLog.loadIndex(_)) .reduce(_.union(_)) df.where("remove is not null").count() } testQuietly("retention timestamp is picked properly by the cold snapshot initialization") { withTempDir { dir => val clock = new ManualClock(getStartTimeForRetentionTest) def deltaLog: DeltaLog = DeltaLog.forTable(spark, new Path(dir.getCanonicalPath), clock) // Create table with 30 day tombstone retention. sql( s"""CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta |TBLPROPERTIES ('delta.deletedFileRetentionDuration' = 'interval 30 days') """.stripMargin) // 1st day - commit 10 new files and remove them also same day. clock.advance(intervalStringToMillis("interval 1 days")) val files1 = (1 to 4).map(f => createTestAddFile(encodedPath = f.toString)) deltaLog.startTransaction().commit(files1, testOp) val files2 = (1 to 4).map(f => RemoveFile(f.toString, Some(clock.getTimeMillis()))) deltaLog.startTransaction().commit(files2, testOp) // Advance clock by 10 days. clock.advance(intervalStringToMillis("interval 10 days")) DeltaLog.clearCache() deltaLog.checkpoint() DeltaLog.clearCache() // Clear cache and reinitialize snapshot with latest checkpoint. assert(removeFileCountFromUnderlyingCheckpoint(deltaLog.unsafeVolatileSnapshot) === 4) // Advance clock by 21 more days. Now checkpoint should stop tracking remove tombstones. clock.advance(intervalStringToMillis("interval 21 days")) deltaLog.startTransaction().commit(Seq.empty, testOp) DeltaLog.clearCache() deltaLog.checkpoint(deltaLog.unsafeVolatileSnapshot) DeltaLog.clearCache() // Clear cache and reinitialize snapshot with latest checkpoint. assert(removeFileCountFromUnderlyingCheckpoint(deltaLog.unsafeVolatileSnapshot) === 0) } } testQuietly("retention timestamp is lesser than the default value") { withTempDir { dir => val clock = new ManualClock(getStartTimeForRetentionTest) def deltaLog: DeltaLog = DeltaLog.forTable(spark, new Path(dir.getCanonicalPath), clock) // Create table with 2 day tombstone retention. sql( s"""CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta |TBLPROPERTIES ('delta.deletedFileRetentionDuration' = 'interval 2 days') """.stripMargin) // 1st day - commit 10 new files and remove them also same day. { clock.advance(intervalStringToMillis("interval 1 days")) val txn = deltaLog.startTransaction() val files1 = (1 to 4).map(f => createTestAddFile(encodedPath = f.toString)) txn.commit(files1, testOp) val txn2 = deltaLog.startTransaction() val files2 = (1 to 4).map(f => RemoveFile(f.toString, Some(clock.getTimeMillis()))) txn2.commit(files2, testOp) } // Advance clock by 4 days. clock.advance(intervalStringToMillis("interval 4 days")) DeltaLog.clearCache() deltaLog.checkpoint(deltaLog.unsafeVolatileSnapshot) DeltaLog.clearCache() // Clear cache and reinitialize snapshot with latest checkpoint. assert(removeFileCountFromUnderlyingCheckpoint(deltaLog.unsafeVolatileSnapshot) === 0) } } testQuietly("RemoveFiles get deleted during checkpoint if retention time has passed") { withTempDir { tempDir => val clock = new ManualClock(getStartTimeForRetentionTest) val log1 = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath), clock) val txn = startTxnWithManualLogCleanup(log1) val files1 = (1 to 10).map(f => createTestAddFile(encodedPath = f.toString)) txn.commit(files1, testOp) val txn2 = log1.startTransaction() val files2 = (1 to 4).map(f => RemoveFile(f.toString, Some(clock.getTimeMillis()))) txn2.commit(files2, testOp) clock.advance( intervalStringToMillis(DeltaConfigs.TOMBSTONE_RETENTION.defaultValue) + 1000000L) log1.checkpoint() DeltaLog.clearCache() val log2 = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath), clock) assert(log2.snapshot.tombstones.count() === 0) assert(log2.snapshot.allFiles.count() === 6) } } test("the checkpoint and checksum for version 0 should be cleaned") { withTempDir { tempDir => val clock = new ManualClock(getStartTimeForRetentionTest) val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath), clock) val logPath = new File(log.logPath.toUri) startTxnWithManualLogCleanup(log).commit(createTestAddFile(encodedPath = "0") :: Nil, testOp) log.checkpoint() val initialFiles = getLogFiles(logPath) clock.advance(intervalStringToMillis(DeltaConfigs.LOG_RETENTION.defaultValue) + intervalStringToMillis("interval 1 day")) // Create new checkpoints so that the previous version can be deleted. // With V2 checkpoints (QoL feature for CatalogOwned tables), we need to create version 2 // before version 0 can be cleaned up, as V2 checkpoints have stricter retention policies // that require more checkpoint history before allowing cleanup. log.startTransaction().commit(createTestAddFile(encodedPath = "1") :: Nil, testOp) log.checkpoint() log.startTransaction().commit(createTestAddFile(encodedPath = "2") :: Nil, testOp) log.checkpoint() // despite our clock time being set in the future, this doesn't change the FileStatus // lastModified time. this can cause some flakiness during log cleanup. setting it fixes that. getLogFiles(logPath) .filterNot(f => initialFiles.contains(f)) .foreach(f => f.setLastModified(clock.getTimeMillis())) log.cleanUpExpiredLogs(log.snapshot) val afterCleanup = getLogFiles(logPath) initialFiles.foreach { file => assert(!afterCleanup.contains(file)) } // With V2 checkpoints, version 0 should be cleaned, but versions 1 and 2 may be retained assert(!getCrcVersions(logPath).contains(0), "Version 0 checksum should be deleted") assert(!getFileVersions(getDeltaFiles(logPath)).contains(0), "Version 0 commit should be deleted") assert(!getFileVersions(getCheckpointFiles(logPath)).contains(0), "Version 0 checkpoint should be deleted") } } test("allow users to expire transaction identifiers from checkpoints") { withTempDir { dir => val clock = new ManualClock(getStartTimeForRetentionTest) val log = DeltaLog.forTable(spark, new Path(dir.getCanonicalPath), clock) sql( s"""CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) USING delta |TBLPROPERTIES ('delta.setTransactionRetentionDuration' = 'interval 1 days') """.stripMargin) // commit at time < TRANSACTION_ID_RETENTION_DURATION log.startTransaction().commitManually(SetTransaction("app", 1, Some(clock.getTimeMillis()))) assert(log.update().transactions == Map("app" -> 1)) assert(log.update().numOfSetTransactions == 1) clock.advance(intervalStringToMillis("interval 1 days")) // query at time == TRANSACTION_ID_RETENTION_DURATION & NO new commit // No new commit has been made, so we will see expired transactions (this is not ideal, but // it's a tradeoff we've accepted) assert(log.update().transactions == Map("app" -> 1)) assert(log.snapshot.numOfSetTransactions == 1) clock.advance(1) // query at time > TRANSACTION_ID_RETENTION_DURATION & NO new commit // we continue to see expired transactions assert(log.update().transactions == Map("app" -> 1)) assert(log.snapshot.numOfSetTransactions == 1) // query at time > TRANSACTION_ID_RETENTION_DURATION & there IS a new commit // We will only filter expired transactions when time is >= TRANSACTION_ID_RETENTION_DURATION // and a new commit has been made val addFile = createTestAddFile(encodedPath = "fake/path/1") log.startTransaction().commitManually(addFile) assert(log.update().transactions.isEmpty) assert(log.snapshot.numOfSetTransactions == 0) } } protected def cleanUpExpiredLogs(log: DeltaLog): Unit = { val snapshot = log.update() val checkpointVersion = snapshot.logSegment.checkpointProvider.version logInfo(s"snapshot version: ${snapshot.version} checkpoint: $checkpointVersion") log.cleanUpExpiredLogs(snapshot) } for (v2CheckpointFormat <- V2Checkpoint.Format.ALL_AS_STRINGS) test(s"sidecar file cleanup [v2CheckpointFormat: $v2CheckpointFormat]") { val checkpointPolicy = CheckpointPolicy.V2.name withSQLConf((DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key -> v2CheckpointFormat)) { withTempDir { tempDir => val startTime = getStartTimeForRetentionTest val clock = new ManualClock(startTime) val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath), clock) val logPath = new File(log.logPath.toUri) val visitedFiles = scala.collection.mutable.Set.empty[String] spark.sql(s"""CREATE TABLE delta.`${tempDir.toString()}` (id Int) USING delta | TBLPROPERTIES( |-- Disable the async log cleanup as this test needs to manually trigger log |-- clean up. |'delta.enableExpiredLogCleanup' = 'false', |'${DeltaConfigs.CHECKPOINT_POLICY.key}' = '$checkpointPolicy', |'${DeltaConfigs.CHECKPOINT_WRITE_STATS_AS_STRUCT.key}' = 'false', |'delta.checkpointInterval' = '100000', |'delta.deletedFileRetentionDuration' = 'interval 1 days', |'delta.logRetentionDuration' = 'interval 6 days') """.stripMargin) // day-1. Create a commit with 4 AddFiles. clock.setTime(day(startTime, day = 1)) val file = (1 to 4).map(i => createTestAddFile(i.toString)) log.startTransaction().commit(file, testOp) setModificationTimeOfNewFiles(log, clock, visitedFiles) // Trigger 1 commit and 1 checkpoint daily for next 8 days val sidecarFiles = scala.collection.mutable.Map.empty[Long, String] val oddCommitSidecarFile_1 = createSidecarFile(log, Seq(1)) val evenCommitSidecarFile_1 = createSidecarFile(log, Seq(1)) def commitAndCheckpoint(dayNumber: Int): Unit = { clock.setTime(day(startTime, dayNumber)) // Write a new commit on each day log.startTransaction().commit(Seq(log.unsafeVolatileSnapshot.metadata), testOp) setModificationTimeOfNewFiles(log, clock, visitedFiles) // Write a new checkpoint on each day. Each checkpoint has 2 sodecars: // 1. Common sidecar - one of oddCommitSidecarFile_1/evenCommitSidecarFile_1 // 2. A new sidecar just created for this checkpoint. val sidecarFile1 = if (dayNumber % 2 == 0) evenCommitSidecarFile_1 else oddCommitSidecarFile_1 val sidecarFile2 = createSidecarFile(log, Seq(2, 3, 4)) val checkpointVersion = log.update().version createV2CheckpointWithSidecarFile( log, checkpointVersion, sidecarFileNames = Seq(sidecarFile1, sidecarFile2)) setModificationTimeOfNewFiles(log, clock, visitedFiles) sidecarFiles.put(checkpointVersion, sidecarFile2) } (2 to 9).foreach { dayNumber => commitAndCheckpoint(dayNumber) } clock.setTime(day(startTime, day = 10)) log.update() // Assert all log files are present. compareVersions(getCheckpointVersions(logPath), "checkpoint", 2 to 9) compareVersions(getDeltaVersions(logPath), "delta", 0 to 9) assert( getSidecarFiles(log) === Set( evenCommitSidecarFile_1, oddCommitSidecarFile_1) ++ sidecarFiles.values.toIndexedSeq) // Trigger metadata cleanup and validate that only last 6 days of deltas and checkpoints // have been retained. cleanUpExpiredLogs(log) compareVersions(getCheckpointVersions(logPath), "checkpoint", 4 to 9) compareVersions(getDeltaVersions(logPath), "delta", 4 to 9) // Check that all active sidecars are retained and expired ones are deleted. assert( getSidecarFiles(log) === Set(evenCommitSidecarFile_1, oddCommitSidecarFile_1) ++ (4 to 9).map(sidecarFiles(_))) // Advance 1 day and again run metadata cleanup. clock.setTime(day(startTime, day = 11)) cleanUpExpiredLogs(log) setModificationTimeOfNewFiles(log, clock, visitedFiles) // Commit 4 and checkpoint 4 have expired and were deleted. compareVersions(getCheckpointVersions(logPath), "checkpoint", 5 to 9) compareVersions(getDeltaVersions(logPath), "delta", 5 to 9) assert( getSidecarFiles(log) === Set(evenCommitSidecarFile_1, oddCommitSidecarFile_1) ++ (5 to 9).map(sidecarFiles(_))) // do 1 more commit and checkpoint on day 13 and run metadata cleanup. commitAndCheckpoint(dayNumber = 13) // commit and checkpoint 10 compareVersions(getCheckpointVersions(logPath), "checkpoint", 5 to 10) compareVersions(getDeltaVersions(logPath), "delta", 5 to 10) cleanUpExpiredLogs(log) setModificationTimeOfNewFiles(log, clock, visitedFiles) // Version 5 and 6 checkpoints and deltas have expired and were deleted. compareVersions(getCheckpointVersions(logPath), "checkpoint", 7 to 10) compareVersions(getDeltaVersions(logPath), "delta", 7 to 10) assert( getSidecarFiles(log) === Set(evenCommitSidecarFile_1, oddCommitSidecarFile_1) ++ (7 to 10).map(sidecarFiles(_))) } } } for (v2CheckpointFormat <- V2Checkpoint.Format.ALL_AS_STRINGS) test( s"compat file created with metadata cleanup when checkpoints are deleted" + s" [v2CheckpointFormat: $v2CheckpointFormat]") { val checkpointPolicy = CheckpointPolicy.V2.name withSQLConf((DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key -> v2CheckpointFormat)) { withTempDir { tempDir => val startTime = getStartTimeForRetentionTest val clock = new ManualClock(startTime) val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath), clock) val logPath = new File(log.logPath.toUri) val visitedFiles = scala.collection.mutable.Set.empty[String] spark.sql(s"""CREATE TABLE delta.`${tempDir.toString()}` (id Int) USING delta | TBLPROPERTIES( |-- Disable the async log cleanup as this test needs to manually trigger log |-- clean up. |'delta.enableExpiredLogCleanup' = 'false', |'${DeltaConfigs.CHECKPOINT_POLICY.key}' = '$checkpointPolicy', |'${DeltaConfigs.CHECKPOINT_WRITE_STATS_AS_STRUCT.key}' = 'false', |'delta.checkpointInterval' = '100000', |'delta.deletedFileRetentionDuration' = 'interval 1 days', |'delta.logRetentionDuration' = 'interval 6 days') """.stripMargin) (1 to 10).foreach { dayNum => clock.setTime(day(startTime, dayNum)) log.startTransaction().commit(Seq(), testOp) setModificationTimeOfNewFiles(log, clock, visitedFiles) clock.setTime(day(startTime, dayNum) + 10) log.checkpoint(log.update()) setModificationTimeOfNewFiles(log, clock, visitedFiles) } clock.setTime(day(startTime, 11)) log.update() compareVersions(getCheckpointVersions(logPath), "checkpoint", 1 to 10) compareVersions(getDeltaVersions(logPath), "delta", 0 to 10) // 11th day Run metadata cleanup. clock.setTime(day(startTime, 11)) cleanUpExpiredLogs(log) compareVersions(getCheckpointVersions(logPath), "checkpoint", 5 to 10) compareVersions(getDeltaVersions(logPath), "delta", 5 to 10) val checkpointInstancesForV10 = getCheckpointFiles(logPath) .filter(f => getFileVersions(Seq(f)).head == 10) .map(f => new Path(f.getAbsolutePath)) .sortBy(_.getName) .map(CheckpointInstance.apply) assert(checkpointInstancesForV10.size == 2) assert( checkpointInstancesForV10.map(_.format) === Seq(CheckpointInstance.Format.V2, CheckpointInstance.Format.SINGLE)) } } } (Seq(("Default", Seq.empty[(String, String)])) ++ CheckpointPolicy.ALL.map { case CheckpointPolicy.Classic => Seq( ("Classic", Seq( DeltaConfigs.CHECKPOINT_POLICY.defaultTablePropertyKey -> CheckpointPolicy.Classic.name)), ("Multipart", Seq( DeltaConfigs.CHECKPOINT_POLICY.defaultTablePropertyKey -> CheckpointPolicy.Classic.name, DeltaSQLConf.DELTA_CHECKPOINT_PART_SIZE.key -> "1")) ) case CheckpointPolicy.V2 => V2Checkpoint.Format.ALL_AS_STRINGS.map { v2CheckpointFormat => (s"V2 $v2CheckpointFormat", Seq(DeltaConfigs.CHECKPOINT_POLICY.defaultTablePropertyKey -> CheckpointPolicy.V2.name, DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key -> v2CheckpointFormat)) } }.flatten).foreach { case (chkConfigName, chkConfig) => test(s"cleanup does not delete the checkpoint if it is required by non-expired versions. " + s"Config: $chkConfigName.") { withSQLConf(chkConfig: _*) { // Disable the following check as the test relies on time travel beyond // deletedFileRetentionDuration withSQLConf( DeltaSQLConf.ENFORCE_TIME_TRAVEL_WITHIN_DELETED_FILE_RETENTION_DURATION.key -> "false" ) { withTempDir { tempDir => val startTime = getStartTimeForRetentionTest val clock = new ManualClock(startTime) val actualTestStartTime = System.currentTimeMillis() val tableReference = s"delta.`${tempDir.getCanonicalPath()}`" val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath), clock) val logPath = new File(log.logPath.toUri) val minChksCount = if (chkConfigName == "Multipart") { 2 } else { 1 } // commit 0 spark.sql( s"""CREATE TABLE $tableReference (id Int) USING delta TBLPROPERTIES( | 'delta.enableChangeDataFeed' = true) """.stripMargin) // Set time for commit 0 to ensure that the commits don't need timestamp adjustment. val commit0Time = clock.getTimeMillis() new File(FileNames.unsafeDeltaFile(log.logPath, 0).toUri).setLastModified(commit0Time) new File(FileNames.checksumFile(log.logPath, 0).toUri).setLastModified(commit0Time) def commitNewVersion(version: Long): Unit = { spark.sql(s"INSERT INTO $tableReference VALUES (1)") val deltaFile = new File(FileNames.unsafeDeltaFile(log.logPath, version).toUri) val time = clock.getTimeMillis() + version * 1000 deltaFile.setLastModified(time) val crcFile = new File(FileNames.checksumFile(log.logPath, version).toUri) crcFile.setLastModified(time) val chks = getCheckpointFiles(logPath) .filter(f => FileNames.checkpointVersion(new Path(f.getCanonicalPath)) == version) if (version % 10 == 0) { assert(chks.length >= minChksCount) chks.foreach { chk => assert(chk.exists()) chk.setLastModified(time) } } else { assert(chks.isEmpty) } } // Day 0: Add commits 1 to 15 --> creates 1 checkpoint at Day 0 for version 10 (1L to 15L).foreach(commitNewVersion) // ensure that the checkpoint at version 10 exists val checkpoint10Files = getCheckpointFiles(logPath) .filter(f => FileNames.checkpointVersion(new Path(f.getCanonicalPath)) == 10) assert(checkpoint10Files.length >= minChksCount) assert(checkpoint10Files.forall(_.exists)) val deltaFiles = (0 to 15).map { i => new File(FileNames.unsafeDeltaFile(log.logPath, i).toUri) } deltaFiles.foreach { f => assert(f.exists()) } // Day 35: Add commits 16 to 25 --> creates a checkpoint at Day 35 for version 20 clock.setTime(day(startTime, 35)) (16L to 25L).foreach(commitNewVersion) assert(checkpoint10Files.forall(_.exists)) deltaFiles.foreach { f => assert(f.exists()) } // auto cleanup is disabled in DeltaRetentionSuiteBase so tests have control when it // happens cleanUpExpiredLogs(log) // assert that the checkpoint from day 0 (at version 10) and all the commits after // that are still there assert(checkpoint10Files.forall(_.exists)) deltaFiles.foreach { f => val version = FileNames.deltaVersion(new Path(f.toString())) if (version < 10) { assert(!f.exists, version) } else { assert(f.exists, version) } } // Validate we can time travel to version >=10 val earliestExpectedChkVersion = 10 (0 to 25).map { version => val sqlCommand = s"SELECT * FROM $tableReference VERSION AS OF $version" if (version < earliestExpectedChkVersion) { val ex = intercept[org.apache.spark.sql.delta.VersionNotFoundException] { spark.sql(sqlCommand).collect() } checkError( ex, "DELTA_VERSION_NOT_FOUND", sqlState = "22003", parameters = Map( "userVersion" -> version.toString, "earliest" -> earliestExpectedChkVersion.toString, "latest" -> "25")) } else { spark.sql(sqlCommand).collect() } } // Validate CDF - SELECT * FROM table_changes_by_path('table', X, Y) (0 to 24).map { version => val sqlCommand = s"SELECT * FROM " + s"table_changes_by_path('${tempDir.getCanonicalPath}', $version, 25)" if (version < earliestExpectedChkVersion) { if (catalogOwnedDefaultCreationEnabledInTests) { intercept[IllegalStateException] { spark.sql(sqlCommand).collect() } } else { intercept[org.apache.spark.sql.delta.DeltaFileNotFoundException] { spark.sql(sqlCommand).collect() } } } else { spark.sql(sqlCommand).collect() } } } } } } } test(s"cleanup does not delete the JSON logs if the multi-part checkpoint is incomplete.") { withSQLConf(DeltaSQLConf.DELTA_CHECKPOINT_PART_SIZE.key -> "1") { withTempDir { tempDir => val startTime = getStartTimeForRetentionTest val clock = new ManualClock(startTime) val actualTestStartTime = System.currentTimeMillis() val tableReference = s"delta.`${tempDir.getCanonicalPath()}`" val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath), clock) val logPath = new File(log.logPath.toUri) // commit 0 spark.sql( s"""CREATE TABLE $tableReference (id Int) USING delta | TBLPROPERTIES('delta.enableChangeDataFeed' = true) """.stripMargin) // Set time for commit 0 to ensure that the commits don't need timestamp adjustment. val commit0Time = clock.getTimeMillis() new File(FileNames.unsafeDeltaFile(log.logPath, 0).toUri).setLastModified(commit0Time) new File(FileNames.checksumFile(log.logPath, 0).toUri).setLastModified(commit0Time) def commitNewVersion(version: Long): Unit = { spark.sql(s"INSERT INTO $tableReference VALUES (1)") val deltaFile = new File(FileNames.unsafeDeltaFile(log.logPath, version).toUri) val time = clock.getTimeMillis() + version * 1000 deltaFile.setLastModified(time) val crcFile = new File(FileNames.checksumFile(log.logPath, version).toUri) crcFile.setLastModified(time) val chks = getCheckpointFiles(logPath) .filter(f => FileNames.checkpointVersion(new Path(f.getCanonicalPath)) == version) if (version % 10 == 0) { // With V2 checkpoints (QoL feature for CatalogOwned), checkpoints may be single-file. // Classic checkpoints with DELTA_CHECKPOINT_PART_SIZE=1 are multi-part (>= 2 files). assert(chks.length >= 1) // At least one checkpoint file chks.foreach { chk => assert(chk.exists()) chk.setLastModified(time) } } else { assert(chks.isEmpty) } } // Day 0: Add commits 1 to 15 --> creates 1 checkpoint at Day 0 for version 10 (1L to 15L).foreach(commitNewVersion) // ensure that the checkpoint at version 10 exists val checkpoint10Files = getCheckpointFiles(logPath) .filter(f => FileNames.checkpointVersion(new Path(f.getCanonicalPath)) == 10) // With V2 checkpoints (QoL feature for CatalogOwned), checkpoints may be single-file assert(checkpoint10Files.length >= 1) // At least one checkpoint file assert(checkpoint10Files.forall(_.exists)) val deltaFiles = (0 to 15).map { i => new File(FileNames.unsafeDeltaFile(log.logPath, i).toUri) } deltaFiles.foreach { f => assert(f.exists()) } // Day 35: Add commits 16 to 25 --> creates a checkpoint at Day 35 for version 20 clock.setTime(day(startTime, 35)) (16L to 25L).foreach(commitNewVersion) assert(checkpoint10Files.forall(_.exists)) deltaFiles.foreach { f => assert(f.exists()) } checkpoint10Files.lastOption.foreach { lastPart => lastPart.delete() // delete the last part to simulate incomplete checkpoint } // auto cleanup is disabled in DeltaRetentionSuiteBase so tests have control when it happens cleanUpExpiredLogs(log) // assert that delta logs are not deleted due to missing checkpoint part deltaFiles.foreach { f => val version = FileNames.deltaVersion(new Path(f.toString())) assert(f.exists, s"version $version should not be deleted") } } } } test("Metadata cleanup respects requireCheckpointProtectionBeforeVersion") { withSQLConf( DeltaSQLConf.ALLOW_METADATA_CLEANUP_WHEN_ALL_PROTOCOLS_SUPPORTED.key -> "false", DeltaSQLConf.ALLOW_METADATA_CLEANUP_CHECKPOINT_EXISTENCE_CHECK_DISABLED.key -> "true") { // Commits should be cleaned up to the latest checkpoint. testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, createCheckpoints = Set(6, 8), requireCheckpointProtectionBeforeVersion = 2, expectedCommitsAfterCleanup = (8 to 15), // Α checkpoint is automatically created every 10 commits. expectedCheckpointsAfterCleanup = Set(8, 10)) // Commits should be cleaned up to the latest checkpoint. testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, createCheckpoints = Set(6, 8), requireCheckpointProtectionBeforeVersion = 6, expectedCommitsAfterCleanup = (8 to 15), // Α checkpoint is automatically created every 10 commits. expectedCheckpointsAfterCleanup = Set(8, 10)) // Commits should be cleaned up to the latest checkpoint. testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, createCheckpoints = Set(6, 8), requireCheckpointProtectionBeforeVersion = 7, expectedCommitsAfterCleanup = (8 to 15), // Α checkpoint is automatically created every 10 commits. expectedCheckpointsAfterCleanup = Set(8, 10)) // Commits should be cleaned up to the latest checkpoint. testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, createCheckpoints = Set(6, 8), requireCheckpointProtectionBeforeVersion = 8, expectedCommitsAfterCleanup = (8 to 15), // Α checkpoint is automatically created every 10 commits. expectedCheckpointsAfterCleanup = Set(8, 10)) // Commits should be cleaned up to the checkpoint 10. testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 10, createNumCommitsWithinRetentionPeriod = 10, createCheckpoints = Set(6, 8), requireCheckpointProtectionBeforeVersion = 9, expectedCommitsAfterCleanup = (10 to 19), // Α checkpoint is automatically created every 10 commits. expectedCheckpointsAfterCleanup = Set(10)) // Checkpoint 8 is within the retention period. // Cleanup should be skipped. testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, createCheckpoints = Set(6, 8), requireCheckpointProtectionBeforeVersion = 9, expectedCommitsAfterCleanup = (0 to 15), // Α checkpoint is automatically created every 10 commits. expectedCheckpointsAfterCleanup = Set(6, 8, 10)) // Cleanup should be skipped. testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, createCheckpoints = Set(6, 8), requireCheckpointProtectionBeforeVersion = 20, expectedCommitsAfterCleanup = (0 to 15), // Α checkpoint is automatically created every 10 commits. expectedCheckpointsAfterCleanup = Set(6, 8, 10)) // With multiple checkpoints (8, 12, 14) within the retention period. // None of these should be cleaned up. testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, createCheckpoints = Set(6, 8, 12, 14), requireCheckpointProtectionBeforeVersion = 8, expectedCommitsAfterCleanup = (8 to 15), // Α checkpoint is automatically created every 10 commits. expectedCheckpointsAfterCleanup = Set(8, 10, 12, 14)) // With multiple checkpoints (8, 12, 14) within the retention period. // requireCheckpointProtectionBeforeVersion = 9 is within the retention period. // Cleanup should be skipped. testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, createCheckpoints = Set(6, 8, 12, 14), requireCheckpointProtectionBeforeVersion = 9, expectedCommitsAfterCleanup = (0 to 15), // Α checkpoint is automatically created every 10 commits. expectedCheckpointsAfterCleanup = Set(6, 8, 10, 12, 14)) // Corner cases. testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 1, createNumCommitsWithinRetentionPeriod = 15, createCheckpoints = Set(1), requireCheckpointProtectionBeforeVersion = 0, expectedCommitsAfterCleanup = (1 to 15), // Α checkpoint is automatically created every 10 commits. expectedCheckpointsAfterCleanup = Set(1, 10)) testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 1, createNumCommitsWithinRetentionPeriod = 15, createCheckpoints = Set(1), requireCheckpointProtectionBeforeVersion = 1, expectedCommitsAfterCleanup = (1 to 15), // Α checkpoint is automatically created every 10 commits. expectedCheckpointsAfterCleanup = Set(1, 10)) // v1 can't be deleted because it is the only checkpoint before version 2. // v0 can't be deleted because of the checkpoint protection, v0 and v1 needs // to be deleted together. testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 1, createNumCommitsWithinRetentionPeriod = 15, createCheckpoints = Set(1), requireCheckpointProtectionBeforeVersion = 2, expectedCommitsAfterCleanup = (0 to 15), // Α checkpoint is automatically created every 10 commits. expectedCheckpointsAfterCleanup = Set(1, 10)) testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 2, createNumCommitsWithinRetentionPeriod = 14, createCheckpoints = Set(1), requireCheckpointProtectionBeforeVersion = 3, expectedCommitsAfterCleanup = (0 to 15), // Α checkpoint is automatically created every 10 commits. expectedCheckpointsAfterCleanup = Set(1, 10)) } } test("Cleanup is allowed if a checkpoint already exists at the boundary") { withSQLConf(DeltaSQLConf.ALLOW_METADATA_CLEANUP_WHEN_ALL_PROTOCOLS_SUPPORTED.key -> "false") { testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, // Metadata cleanup should attempt to clean before version 8. createCheckpoints = Set(0, 8), requireCheckpointProtectionBeforeVersion = 10, unsupportedFeatureStartVersion = Some(8), expectedCommitsAfterCleanup = (8 to 15), expectedCheckpointsAfterCleanup = Set(8, 10)) } } test("Metadata cleanup protocol validation positive tests.") { withSQLConf( DeltaSQLConf.ALLOW_METADATA_CLEANUP_CHECKPOINT_EXISTENCE_CHECK_DISABLED.key -> "true") { // In all tests below, we cannot satisfy the version requirement and thus fallback // to protocol validations. We identify we support all features and proceed to // metadata cleanup. testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, createCheckpoints = Set(8), requireCheckpointProtectionBeforeVersion = 10, expectedCommitsAfterCleanup = (8 to 15), expectedCheckpointsAfterCleanup = Set(8, 10)) // The protocol contains unsupported feature but at requireCheckpointProtectionBeforeVersion. testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, createCheckpoints = Set(8), requireCheckpointProtectionBeforeVersion = 10, unsupportedFeatureStartVersion = Some(10), expectedCommitsAfterCleanup = (8 to 15), expectedCheckpointsAfterCleanup = Set(8, 10)) // The protocol contains unsupported feature but after // requireCheckpointProtectionBeforeVersion. testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, createCheckpoints = Set(8), requireCheckpointProtectionBeforeVersion = 10, unsupportedFeatureStartVersion = Some(11), expectedCommitsAfterCleanup = (8 to 15), expectedCheckpointsAfterCleanup = Set(8, 10)) // The protocol contains unsupported feature before requireCheckpointProtectionBeforeVersion // but right after the boundary version where the cleanup ends. testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, // Metadata cleanup should attempt to clean before version 8. createCheckpoints = Set(0, 8), requireCheckpointProtectionBeforeVersion = 10, unsupportedFeatureStartVersion = Some(9), expectedCommitsAfterCleanup = (8 to 15), expectedCheckpointsAfterCleanup = Set(8, 10)) // Other corner cases. testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, createCheckpoints = Set(1, 8), requireCheckpointProtectionBeforeVersion = 10, expectedCommitsAfterCleanup = (8 to 15), expectedCheckpointsAfterCleanup = Set(8, 10)) testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, createCheckpoints = Set(0, 8), requireCheckpointProtectionBeforeVersion = 10, expectedCommitsAfterCleanup = (8 to 15), expectedCheckpointsAfterCleanup = Set(8, 10)) } } test("Metadata cleanup protocol validation negative tests.") { withSQLConf( DeltaSQLConf.ALLOW_METADATA_CLEANUP_CHECKPOINT_EXISTENCE_CHECK_DISABLED.key -> "true") { // In all tests below, we cannot satisfy the version requirement and thus fallback // to protocol validations. We should detect the start version version includes a // non-supported feature and skip the cleanup. testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, createCheckpoints = Set(8), requireCheckpointProtectionBeforeVersion = 10, // Unsupported feature in the first version. unsupportedFeatureStartVersion = Some(0), unsupportedFeatureEndVersion = Some(1), expectedCommitsAfterCleanup = (0 to 15), expectedCheckpointsAfterCleanup = Set(8, 10)) testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, createCheckpoints = Set(0, 8), requireCheckpointProtectionBeforeVersion = 10, // Unsupported feature right before the boundary version where the cleanup ends. unsupportedFeatureStartVersion = Some(7), expectedCommitsAfterCleanup = (0 to 15), expectedCheckpointsAfterCleanup = Set(0, 8, 10)) testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, createCheckpoints = Set(8), requireCheckpointProtectionBeforeVersion = 10, // Unsupported feature in intermediate versions. unsupportedFeatureStartVersion = Some(4), unsupportedFeatureEndVersion = Some(7), expectedCommitsAfterCleanup = (0 to 15), expectedCheckpointsAfterCleanup = Set(8, 10)) testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, createCheckpoints = Set(8), requireCheckpointProtectionBeforeVersion = 10, // Unsupported feature in dropped at the boundary version. unsupportedFeatureStartVersion = Some(4), unsupportedFeatureEndVersion = Some(8), expectedCommitsAfterCleanup = (0 to 15), expectedCheckpointsAfterCleanup = Set(8, 10)) // The protocol contains unsupported feature before requireCheckpointProtectionBeforeVersion // but at the boundary version where the cleanup ends. testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, // Metadata cleanup should attempt to clean before version 8. createCheckpoints = Set(0, 8), requireCheckpointProtectionBeforeVersion = 10, unsupportedFeatureStartVersion = Some(8), expectedCommitsAfterCleanup = (0 to 15), expectedCheckpointsAfterCleanup = Set(0, 8, 10)) testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, // Metadata cleanup should attempt to clean before version 8. createCheckpoints = Set(0, 8), requireCheckpointProtectionBeforeVersion = 10, // Make sure we correctly validate the protocol of the checkpoint version. unsupportedFeature = TestUnsupportedWriterFeature, unsupportedFeatureStartVersion = Some(8), expectedCommitsAfterCleanup = (0 to 15), expectedCheckpointsAfterCleanup = Set(0, 8, 10)) } } test("Metadata cleanup protocol validation with incomplete CRCs.") { withSQLConf( DeltaSQLConf.ALLOW_METADATA_CLEANUP_CHECKPOINT_EXISTENCE_CHECK_DISABLED.key -> "true") { // We fall back to protocol validations which cannot be completed due to missing // protocol in one of the CRCs. testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, createCheckpoints = Set(8), requireCheckpointProtectionBeforeVersion = 10, incompleteCRCVersion = Some(3), expectedCommitsAfterCleanup = (0 to 15), expectedCheckpointsAfterCleanup = Set(8, 10)) // Similar to above but a CRC file is missing. testRequireCheckpointProtectionBeforeVersion( createNumCommitsOutsideRetentionPeriod = 8, createNumCommitsWithinRetentionPeriod = 8, createCheckpoints = Set(8), requireCheckpointProtectionBeforeVersion = 10, missingCRCVersion = Some(3), expectedCommitsAfterCleanup = (0 to 15), expectedCheckpointsAfterCleanup = Set(8, 10)) } } } class DeltaRetentionWithCatalogOwnedBatch1Suite extends DeltaRetentionSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } /** * This test suite does not extend other tests of DeltaRetentionSuiteEdge because * DeltaRetentionSuiteEdge contain tests that rely on setting the file modification time for delta * files. However, in this suite, delta files might be backfilled asynchronously, which means * setting the modification time will not work as expected. */ class DeltaRetentionWithCatalogOwnedBatch2Suite extends QueryTest with DeltaSQLCommandTest with DeltaRetentionSuiteBase { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) override def getLogFiles(dir: File): Seq[File] = getDeltaFiles(dir) ++ getUnbackfilledDeltaFiles(dir) ++ getCheckpointFiles(dir) /** * This test verifies that unbackfilled versions, i.e., versions for which backfilled deltas do * not exist yet, are never considered for deletion, even if they fall outside the retention * window. The primary reason for not deleting these versions is that the CommitCoordinator might * be actively tracking those files, and currently, MetadataCleanup does not communicate with the * CommitCoordinator. * * Although the fact that they are unbackfilled is somewhat redundant since these versions are * currently already protected due to two additional reasons: * 1.They will always be part of the latest snapshot. * 2.They don't have two checkpoints after them. * However, this test helps ensure that unbackfilled deltas remain protected in the future, even * if the above two conditions are no longer triggered. * * Note: This test is too slow for batchSize = 100 and wouldn't necessarily work for batchSize = 1 */ test("unbackfilled expired commits are always retained") { withTempDir { tempDir => val startTime = getStartTimeForRetentionTest val clock = new ManualClock(startTime) val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath), clock) val logPath = new File(log.logPath.toUri.getPath) val fs = new RawLocalFileSystem() fs.initialize(tempDir.toURI, new Configuration()) log.startTransaction().commitManually(createTestAddFile("1")) log.checkpoint() spark.sql(s"""ALTER TABLE delta.`${tempDir.toString}` |SET TBLPROPERTIES( |-- Trigger log clean up manually. |'delta.enableExpiredLogCleanup' = 'false', |'delta.checkpointInterval' = '10000', |'delta.checkpointRetentionDuration' = 'interval 2 days', |'delta.logRetentionDuration' = 'interval 30 days', |'delta.enableFullRetentionRollback' = 'true') """.stripMargin) log.checkpoint() setModificationTime(log, startTime, 0, 0, fs) setModificationTime(log, startTime, 1, 0, fs) // Create commits [2, 6] with a checkpoint per commit 2 to 6 foreach { i => log.startTransaction().commitManually(createTestAddFile(s"$i")) log.checkpoint() setModificationTime(log, startTime, i, 0, fs) } // Create unbackfilled commit [7] with no checkpoints log.startTransaction().commitManually(createTestAddFile("7")) setModificationTime(log, startTime, 7, 0, fs) // Everything is eligible for deletion but we don't consider the unbackfilled commit, // i.e. [7], for deletion because it is part of the current LogSegment. clock.setTime(day(startTime, 100)) log.cleanUpExpiredLogs(log.update()) // Since we also need a checkpoint, [6] is also protected. val firstProtectedVersion = 6 compareVersions( getDeltaVersions(logPath), "backfilled delta", firstProtectedVersion to 6) compareVersions( getUnbackfilledDeltaVersions(logPath), "unbackfilled delta", firstProtectedVersion to 7) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaRetentionSuiteBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.util.{Calendar, TimeZone} import scala.collection.mutable import org.apache.spark.sql.delta.DeltaOperations.Truncate import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile import org.apache.spark.sql.delta.actions.{CheckpointMetadata, Metadata, SidecarFile} import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.FileNames import org.apache.spark.sql.delta.util.FileNames.{newV2CheckpointJsonFile, newV2CheckpointParquetFile} import org.apache.commons.lang3.time.DateUtils import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.spark.SparkConf import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.util.IntervalUtils import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.unsafe.types.UTF8String import org.apache.spark.util.ManualClock trait DeltaRetentionSuiteBase extends QueryTest with SharedSparkSession with CatalogOwnedTestBaseSuite { protected val testOp = Truncate() protected override def sparkConf: SparkConf = super.sparkConf // Disable the log cleanup because it runs asynchronously and causes test flakiness .set("spark.databricks.delta.properties.defaults.enableExpiredLogCleanup", "false") protected def intervalStringToMillis(str: String): Long = { DeltaConfigs.getMilliSeconds( IntervalUtils.safeStringToInterval(UTF8String.fromString(str))) } /** * Returns milliseconds since epoch at 1:00am UTC of current day. * * Context: * Most DeltaRetentionSuite tests rely on ManualClock to time travel and * trigger metadata cleanup. Cleanup boundaries are determined by * finding files that were modified before 00:00 of the day on which * currentTime-LOG_RETENTION_PERIOD falls. This means that for a long running * test started at 23:59, the number of expired files would jump suddenly * in 1 minute (the expiration boundary would move by a day as soon as * system clock hits 00:00 of the next day). By fixing the start time of the * test to 01:00, we avoid these scenarios. * * This would still break if the test runs for more than 23 hours. */ protected def getStartTimeForRetentionTest: Long = { val currentTime = System.currentTimeMillis() val date = Calendar.getInstance(TimeZone.getTimeZone("UTC")) date.setTimeInMillis(currentTime) val dayStartTimeStamp = DateUtils.truncate(date, Calendar.DAY_OF_MONTH) dayStartTimeStamp.add(Calendar.HOUR_OF_DAY, 1); dayStartTimeStamp.getTimeInMillis } protected def getDeltaFiles(dir: File): Seq[File] = dir.listFiles().filter(f => FileNames.isDeltaFile(new Path(f.getCanonicalPath))) protected def getCheckpointFiles(dir: File): Seq[File] = dir.listFiles().filter(f => FileNames.isCheckpointFile(new Path(f.getCanonicalPath))) protected def getLogFiles(dir: File): Seq[File] protected def getFileVersions(files: Seq[File]): Set[Long] = { files.map(f => f.getName()).map(s => s.substring(0, s.indexOf(".")).toLong).toSet } protected def getCrcFiles(dir: File): Seq[File] = dir.listFiles().filter(f => FileNames.isChecksumFile(new Path(f.getCanonicalPath))) protected def getCrcVersions(dir: File): Set[Long] = getFileVersions(getCrcFiles(dir)) protected def getDeltaAndCrcFiles(dir: File): Seq[File] = getDeltaFiles(dir) ++ getCrcFiles(dir) protected def getDeltaVersions(dir: File): Set[Long] = { val backfilledDeltaVersions = getFileVersions(getDeltaFiles(dir)) val unbackfilledDeltaVersions = getUnbackfilledDeltaVersions(dir) if (catalogOwnedDefaultCreationEnabledInTests) { // The unbackfilled commit files (except commit 0) should be a superset of the backfilled // commit files since they're always deleted together in this suite. assert( unbackfilledDeltaVersions.toArray.sorted.startsWith( backfilledDeltaVersions.filter(_ != 0).toArray.sorted)) } backfilledDeltaVersions } protected def getUnbackfilledDeltaFiles(dir: File): Seq[File] = { val commitDirPath = FileNames.commitDirPath(new Path(dir.toURI)) getDeltaFiles(new File(commitDirPath.toUri)) } protected def getUnbackfilledDeltaVersions(dir: File): Set[Long] = getFileVersions(getUnbackfilledDeltaFiles(dir)) protected def getSidecarFiles(log: DeltaLog): Set[String] = { new java.io.File(log.sidecarDirPath.toUri) .listFiles() .filter(_.getName.endsWith(".parquet")) .map(_.getName) .toSet } protected def getCheckpointVersions(dir: File): Set[Long] = { getFileVersions(getCheckpointFiles(dir)) } /** Compares the given versions with expected and generates a nice error message. */ protected def compareVersions( versions: Set[Long], logType: String, expected: Iterable[Int]): Unit = { val expectedSet = expected.map(_.toLong).toSet val deleted = expectedSet -- versions val notDeleted = versions -- expectedSet if (!(deleted.isEmpty && notDeleted.isEmpty)) { fail(s"""Mismatch in log clean up for ${logType}s: |Shouldn't be deleted but deleted: ${deleted.toArray.sorted.mkString("[", ", ", "]")} |Should be deleted but not: ${notDeleted.toArray.sorted.mkString("[", ", ", "]")} """.stripMargin) } } // Set modification time of the new files in _delta_log directory and mark them as visited. def setModificationTimeOfNewFiles( log: DeltaLog, clock: ManualClock, visitedFiled: mutable.Set[String]): Unit = { val fs = log.logPath.getFileSystem(log.newDeltaHadoopConf()) val allFiles = fs.listFiles(log.logPath, true) while (allFiles.hasNext) { val file = allFiles.next() if (!visitedFiled.contains(file.getPath.toString)) { visitedFiled += file.getPath.toString fs.setTimes(file.getPath, clock.getTimeMillis(), 0) } } } protected def setModificationTime( log: DeltaLog, startTime: Long, version: Int, dayNum: Int, fs: FileSystem, checkpointOnly: Boolean = false): Unit = { val paths = log .listFrom(version) .collect { case FileNames.CheckpointFile(f, v) if v == version => f.getPath } .toSeq paths.foreach { cpPath => // Add some second offset so that we don't have files with same timestamps fs.setTimes(cpPath, day(startTime, dayNum) + version * 1000, 0) } if (!checkpointOnly) { val deltaPath = new Path(log.logPath, new Path(f"$version%020d.json")) if (fs.exists(deltaPath)) { // Add some second offset so that we don't have files with same timestamps fs.setTimes(deltaPath, day(startTime, dayNum) + version * 1000, 0) } // Add the same timestamp for unbackfilled delta files as well fs.listStatus(FileNames.commitDirPath(log.logPath)) .find(_.getPath.getName.startsWith(f"$version%020d")) .foreach(f => fs.setTimes(f.getPath, day(startTime, dayNum) + version * 1000, 0)) } } protected def day(startTime: Long, day: Int): Long = startTime + intervalStringToMillis(s"interval $day days") /** * Creates a sidecar file with the given AddFiles. * * @param log The DeltaLog to which the sidecar file will be added. * @param files A sequence of integers representing the AddFile indices. * @return The name of the created sidecar file. */ protected def createSidecarFile(log: DeltaLog, files: Seq[Int]): String = { val sparkSession = spark // scalastyle:off sparkimplicits import sparkSession.implicits._ // scalastyle:on sparkimplicits var sidecarFileName: String = "" withTempDir { dir => val snapshot = log.unsafeVolatileSnapshot val isRowTrackingEnabled = RowTracking.isEnabled(snapshot.protocol, snapshot.metadata) val adds = files.map { i => val baseAddFile = createTestAddFile(i.toString) if (isRowTrackingEnabled) { // When [[RowTrackingFeature]] is enabled, assign `baseRowId` and // `defaultRowCommitVersion` to match what would happen during actual commit. // Otherwise, CRC validation will fail for subsequent commits after the first // checkpoint, since the AddFiles from state reconstruction (checkpoint) differ // from the incremental ones. baseAddFile.copy( baseRowId = Some((i - 1).toLong), // Use 1L as the default row commit version for the mock AddFiles in the sidecar file. defaultRowCommitVersion = Some(1L)) } else { baseAddFile } } adds.map(_.wrap).toDF.repartition(1).write.mode("overwrite").parquet(dir.getAbsolutePath) val srcPath = new Path(dir.listFiles().filter(_.getName.endsWith("parquet")).head.getAbsolutePath) val dstPath = new Path(log.sidecarDirPath, srcPath.getName) val fs = srcPath.getFileSystem(log.newDeltaHadoopConf()) fs.mkdirs(log.sidecarDirPath) fs.rename(srcPath, dstPath) sidecarFileName = fs.getFileStatus(dstPath).getPath.getName } sidecarFileName } // Create a V2 Checkpoint at given version with given sidecar files. protected def createV2CheckpointWithSidecarFile( log: DeltaLog, version: Long, sidecarFileNames: Seq[String]): Unit = { val hadoopConf = log.newDeltaHadoopConf() val fs = log.logPath.getFileSystem(hadoopConf) val sidecarFiles = sidecarFileNames.map { fileName => val sidecarPath = new Path(log.sidecarDirPath, fileName) val fileStatus = SerializableFileStatus.fromStatus(fs.getFileStatus(sidecarPath)) SidecarFile(fileStatus) } val snapshot = log.getSnapshotAt(version) val actionsForCheckpoint = snapshot.nonFileActions ++ sidecarFiles :+ CheckpointMetadata(version) val v2CheckpointFormat = spark.conf.getOption(DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key) v2CheckpointFormat match { case Some(V2Checkpoint.Format.JSON.name) | None => log.store.write( newV2CheckpointJsonFile(log.logPath, version), actionsForCheckpoint.map(_.json).toIterator, overwrite = true, hadoopConf = hadoopConf) case Some(V2Checkpoint.Format.PARQUET.name) => val parquetFile = newV2CheckpointParquetFile(log.logPath, version) val sparkSession = spark // scalastyle:off sparkimplicits import sparkSession.implicits._ // scalastyle:on sparkimplicits val dfToWrite = actionsForCheckpoint.map(_.wrap).toDF Checkpoints.createCheckpointV2ParquetFile( spark, dfToWrite, parquetFile, hadoopConf, useRename = false) case _ => assert(false, "Invalid v2 checkpoint format") } log.writeLastCheckpointFile( log, LastCheckpointInfo(version, -1, None, None, None, None), false) } /** * Start a txn that disables automatic log cleanup. Some tests may need to manually clean up logs * to get deterministic behaviors. */ protected def startTxnWithManualLogCleanup(log: DeltaLog): OptimisticTransaction = { val txn = log.startTransaction() // This will pick up `spark.databricks.delta.properties.defaults.enableExpiredLogCleanup` to // disable log cleanup. txn.updateMetadata(Metadata()) txn } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaSinkImplicitCastSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.sql.{Date, Timestamp} import scala.concurrent.duration._ import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.Relocated.StreamExecution import org.apache.spark.sql.delta.sources.{DeltaSink, DeltaSQLConf} import org.apache.spark.sql.delta.test.shims.StreamingTestShims.MemoryStream import org.apache.spark.{SparkArithmeticException, SparkConf, SparkThrowable} import org.apache.spark.sql.{DataFrame, Encoder, Row} import org.apache.spark.sql.errors.QueryExecutionErrors.toSQLType import org.apache.spark.sql.functions.{col, lit} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy import org.apache.spark.sql.streaming.{OutputMode, StreamingQueryException, Trigger} import org.apache.spark.sql.types._ /** * Defines helper class & methods to test writing to a Delta streaming sink using data types that * don't match the corresponding column type in the table schema. */ abstract class DeltaSinkImplicitCastSuiteBase extends DeltaSinkTest { override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaSQLConf.DELTA_STREAMING_SINK_ALLOW_IMPLICIT_CASTS.key, "true") spark.conf.set(SQLConf.ANSI_ENABLED.key, "true") } /** * Helper to write to and read from a Delta sink. Creates and runs a streaming query for each call * to `write`. */ class TestDeltaStream[T: Encoder]( outputDir: File, checkpointDir: File) { private val source = MemoryStream[T] def write(data: T*)(selectExpr: String*): Unit = write( outputMode = OutputMode.Append, timeout = streamingTimeout, extraOptions = Map.empty)( data: _*)( selectExpr: _*) def write( outputMode: OutputMode, timeout: Duration, extraOptions: Map[String, String])( data: T*)( selectExpr: String*): Unit = { source.addData(data) val query = source.toDF() .selectExpr(selectExpr: _*) .writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .outputMode(outputMode) .options(extraOptions) .format("delta") .trigger(Trigger.AvailableNow()) .start(outputDir.getCanonicalPath) try { failAfter(timeout) { query.processAllAvailable() } } finally { query.stop() } } def currentSchema: StructType = spark.read.format("delta").load(outputDir.getCanonicalPath).schema def read(): DataFrame = spark.read.format("delta").load(outputDir.getCanonicalPath) def deltaLog: DeltaLog = DeltaLog.forTable(spark, outputDir.getCanonicalPath) } /** Sets up a new [[TestDeltaStream]] to write to and read from a test Delta sink. */ def withDeltaStream[T: Encoder](f: TestDeltaStream[T] => Unit): Unit = withTempDirs { (outputDir, checkpointDir) => f(new TestDeltaStream[T](outputDir, checkpointDir)) } /** * Validates that the table history for the test Delta sink matches the given list of operations. */ def checkOperationHistory[T](stream: TestDeltaStream[T], expectedOperations: Seq[String]) : Unit = { val history = sql(s"DESCRIBE HISTORY delta.`${stream.deltaLog.dataPath}`") .sort("version") .select("operation") checkAnswer(history, expectedOperations.map(Row(_))) } } /** * Covers handling implicit casting to handle type mismatches when writing data to a Delta sink. */ class DeltaSinkImplicitCastSuite extends DeltaSinkImplicitCastSuiteBase with CatalogOwnedTestBaseSuite { import testImplicits._ test(s"write wider type - long -> int") { withDeltaStream[Long] { stream => // This is the first write in this test suite, use a larger timeout to allow for the initial // streaming setup to take place. stream.write( outputMode = OutputMode.Append, timeout = 600.seconds, extraOptions = Map.empty)(17)("CAST(value AS INT)") assert(stream.currentSchema("value").dataType === IntegerType) checkAnswer(stream.read(), Row(17)) stream.write(23)("CAST(value AS LONG)") assert(stream.currentSchema("value").dataType === IntegerType) checkAnswer(stream.read(), Row(17) :: Row(23) :: Nil) checkOperationHistory(stream, expectedOperations = Seq( "STREAMING UPDATE", // First write "STREAMING UPDATE" // Second write )) } } test("write wider type - long -> int - overflow with " + s"storeAssignmentPolicy=${StoreAssignmentPolicy.STRICT}") { withDeltaStream[Long] { stream => stream.write(17)("CAST(value AS INT)") assert(stream.currentSchema("value").dataType === IntegerType) checkAnswer(stream.read(), Row(17)) withSQLConf(SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.STRICT.toString) { val ex = intercept[StreamingQueryException] { stream.write(Long.MaxValue)("CAST(value AS LONG)") } checkError( ex.getCause.asInstanceOf[SparkThrowable], "CANNOT_UP_CAST_DATATYPE", parameters = Map( "expression" -> "value", "sourceType" -> toSQLType("BIGINT"), "targetType" -> toSQLType("INT"), "details" -> ("The type path of the target object is:\n\nYou can either add an " + "explicit cast to the input data or choose a higher precision type of the field in " + "the target object") ) ) } } } test("write wider type - long -> int - overflow with " + s"storeAssignmentPolicy=${StoreAssignmentPolicy.ANSI}") { withDeltaStream[Long] { stream => stream.write(17)("CAST(value AS INT)") assert(stream.currentSchema("value").dataType === IntegerType) checkAnswer(stream.read(), Row(17)) withSQLConf(SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.ANSI.toString) { val ex = intercept[StreamingQueryException] { stream.write(Long.MaxValue)("CAST(value AS LONG)") } def getSparkArithmeticException(ex: Throwable): SparkArithmeticException = ex match { case e: SparkArithmeticException => e case e: Throwable if e.getCause != null => getSparkArithmeticException(e.getCause) case e => fail(s"Unexpected exception: $e") } checkError( getSparkArithmeticException(ex), "CAST_OVERFLOW_IN_TABLE_INSERT", parameters = Map( "sourceType" -> "\"BIGINT\"", "targetType" -> "\"INT\"", "columnName" -> "`value`") ) } } } test("write wider type - long -> int - overflow with " + s"storeAssignmentPolicy=${StoreAssignmentPolicy.LEGACY}") { withDeltaStream[Long] { stream => stream.write(17)("CAST(value AS INT)") assert(stream.currentSchema("value").dataType === IntegerType) checkAnswer(stream.read(), Row(17)) withSQLConf(SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.LEGACY.toString) { stream.write(Long.MaxValue)("CAST(value AS LONG)") // LEGACY allows the value to silently overflow. checkAnswer(stream.read(), Row(17) :: Row(-1) :: Nil) } } } test("write wider type - Decimal(10, 4) -> Decimal(6, 2)") { withDeltaStream[BigDecimal] { stream => stream.write(BigDecimal(123456L, scale = 2))("CAST(value AS DECIMAL(6, 2))") assert(stream.currentSchema("value").dataType === DecimalType(6, 2)) checkAnswer(stream.read(), Row(BigDecimal(123456L, scale = 2))) stream.write(BigDecimal(987654L, scale = 4))("CAST(value AS DECIMAL(10, 4))") assert(stream.currentSchema("value").dataType === DecimalType(6, 2)) checkAnswer(stream.read(), Row(BigDecimal(123456L, scale = 2)) :: Row(BigDecimal(9877L, scale = 2)) :: Nil ) } } test("write narrower type - int -> long") { withDeltaStream[Long] { stream => stream.write(Long.MinValue)("CAST(value AS LONG)") assert(stream.currentSchema("value").dataType === LongType) checkAnswer(stream.read(), Row(Long.MinValue)) stream.write(23)("CAST(value AS INT)") assert(stream.currentSchema("value").dataType === LongType) checkAnswer(stream.read(), Row(Long.MinValue) :: Row(23) :: Nil) } } test("write different type - date -> string") { withDeltaStream[String] { stream => stream.write("abc")("CAST(value AS STRING)") assert(stream.currentSchema("value").dataType === StringType) checkAnswer(stream.read(), Row("abc")) stream.write("2024-07-25")("CAST(value AS DATE)") assert(stream.currentSchema("value").dataType === StringType) checkAnswer(stream.read(), Row("abc") :: Row("2024-07-25") :: Nil) } } test("implicit cast in nested struct/array/map") { withDeltaStream[Int] { stream => stream.write(17)("named_struct('a', value) AS s") assert(stream.currentSchema("s").dataType === new StructType().add("a", IntegerType)) checkAnswer(stream.read(), Row(Row(17))) stream.write(-12)("named_struct('a', CAST(value AS LONG)) AS s") assert(stream.currentSchema("s").dataType === new StructType().add("a", IntegerType)) checkAnswer(stream.read(), Row(Row(17)) :: Row(Row(-12)) :: Nil) } withDeltaStream[(Int, Int)] { stream => stream.write((17, 57))("map(_1, _2) AS m") assert(stream.currentSchema("m").dataType === MapType(IntegerType, IntegerType)) checkAnswer(stream.read(), Row(Map(17 -> 57))) stream.write((-12, 3))("map(CAST(_1 AS LONG), CAST(_2 AS STRING)) AS m") assert(stream.currentSchema("m").dataType === MapType(IntegerType, IntegerType)) checkAnswer(stream.read(), Row(Map(17 -> 57)) :: Row(Map(-12 -> 3)) :: Nil) } withDeltaStream[(Int, Int)] { stream => stream.write((17, 57))("array(_1, _2) AS a") assert(stream.currentSchema("a").dataType === ArrayType(IntegerType)) checkAnswer(stream.read(), Row(Seq(17, 57)) :: Nil) stream.write((-12, 3))("array(_1, _2) AS a") assert(stream.currentSchema("a").dataType === ArrayType(IntegerType)) checkAnswer(stream.read(), Row(Seq(17, 57)) :: Row(Seq(-12, 3)) :: Nil) } } test("write invalid nested type - array -> struct") { withDeltaStream[Int] { stream => stream.write(17)("named_struct('a', value) AS s") assert(stream.currentSchema("s").dataType === new StructType().add("a", IntegerType)) checkAnswer(stream.read(), Row(Row(17))) val ex = intercept[StreamingQueryException] { stream.write(-12)("array(value) AS s") } checkError( ex.getCause.asInstanceOf[SparkThrowable], "DELTA_FAILED_TO_MERGE_FIELDS", parameters = Map( "currentField" -> "s", "updateField" -> "s") ) } } test("implicit cast on partition value") { withDeltaStream[(String, Int)] { stream => sql( s""" |CREATE TABLE delta.`${stream.deltaLog.dataPath}` (day date, value int) |USING DELTA |PARTITIONED BY (day) """.stripMargin) stream.write(("2024-07-26", 1))("CAST(_1 AS DATE) AS day", "_2 AS value") assert(stream.currentSchema === new StructType() .add("day", DateType) .add("value", IntegerType)) checkAnswer(stream.read(), Row(Date.valueOf("2024-07-26"), 1)) stream.write(("2024-07-27", 2))( "CAST(_1 AS TIMESTAMP) AS day", "CAST(_2 AS DECIMAL(4, 1)) AS value") assert(stream.currentSchema === new StructType() .add("day", DateType) .add("value", IntegerType)) checkAnswer(stream.read(), Row(Date.valueOf("2024-07-26"), 1) :: Row(Date.valueOf("2024-07-27"), 2) :: Nil) } } test("implicit cast with schema evolution") { withDeltaStream[(Long, String)] { stream => stream.write((123, "unused"))("CAST(_1 AS DECIMAL(6, 3)) AS a") assert(stream.currentSchema === new StructType() .add("a", DecimalType(6, 3))) checkAnswer(stream.read(), Row(BigDecimal(123000, scale = 3))) withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { stream.write((678, "abc"))("CAST(_1 AS LONG) AS a", "_2 AS b") assert(stream.currentSchema === new StructType() .add("a", DecimalType(6, 3)) .add("b", StringType)) checkAnswer(stream.read(), Row(BigDecimal(123000, scale = 3), null) :: Row(BigDecimal(678000, scale = 3), "abc") :: Nil) } } } test("implicit cast with schema overwrite") { withTempDirs { (outputDir, checkpointDir) => val source = MemoryStream[Long] def write(streamingDF: DataFrame, data: Long*): Unit = { val query = streamingDF.writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .outputMode(OutputMode.Complete) .option(DeltaOptions.OVERWRITE_SCHEMA_OPTION, "true") .format("delta") .start(outputDir.getCanonicalPath) try { source.addData(data: _*) failAfter(streamingTimeout) { query.processAllAvailable() } } finally { query.stop() } } // Initial write to the sink with columns a, count, b, c. val initialDF = source.toDF() .selectExpr("CAST(value AS DECIMAL(6, 3)) AS a") .groupBy("a") .count() .withColumn("b", col("count").cast("INT")) .withColumn("c", lit(11).cast("STRING")) write(initialDF, 10) val initialResult = spark.read.format("delta").load(outputDir.getCanonicalPath) assert(initialResult.schema === new StructType() .add("a", DecimalType(6, 3)) .add("count", LongType) .add("b", IntegerType) .add("c", StringType)) checkAnswer(initialResult, Row(BigDecimal(10000, scale = 3), 1, 1, "11")) // Second write with overwrite schema: change type of column b and replace c with d. val overwriteDF = source.toDF() .selectExpr("CAST(value AS DECIMAL(6, 3)) AS a") .groupBy("a") .count() .withColumn("b", col("count").cast("LONG")) .withColumn("d", lit(21).cast("STRING")) write(overwriteDF, 20) val overwriteResult = spark.read.format("delta").load(outputDir.getCanonicalPath) assert(overwriteResult.schema === new StructType() .add("a", DecimalType(6, 3)) .add("count", LongType) .add("b", LongType) .add("d", StringType)) checkAnswer(overwriteResult, Row(BigDecimal(10000, scale = 3), 1, 1, "21") :: Row(BigDecimal(20000, scale = 3), 1, 1, "21") :: Nil ) } } // Writing to a delta sink is always case insensitive and ignores the value of // 'spark.sql.caseSensitive'. for (caseSensitive <- Seq(true, false)) test(s"implicit cast with case sensitivity, caseSensitive=$caseSensitive") { withDeltaStream[Long] { stream => stream.write(17)("CAST(value AS LONG) AS value") assert(stream.currentSchema === new StructType().add("value", LongType)) checkAnswer(stream.read(), Row(17)) withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) { stream.write(23)("CAST(value AS INT) AS VALUE") assert(stream.currentSchema === new StructType().add("value", LongType)) checkAnswer(stream.read(), Row(17) :: Row(23) :: Nil) } } } test("implicit cast and missing column") { withDeltaStream[(String, String)] { stream => stream.write(("2024-07-28 12:00:00", "abc"))("CAST(_1 AS TIMESTAMP) AS a", "_2 AS b") assert(stream.currentSchema === new StructType() .add("a", TimestampType) .add("b", StringType)) checkAnswer(stream.read(), Row(Timestamp.valueOf("2024-07-28 12:00:00"), "abc")) stream.write(("2024-07-29", "unused"))("CAST(_1 AS DATE) AS a") assert(stream.currentSchema === new StructType() .add("a", TimestampType) .add("b", StringType)) checkAnswer(stream.read(), Row(Timestamp.valueOf("2024-07-28 12:00:00"), "abc") :: Row(Timestamp.valueOf("2024-07-29 00:00:00"), null) :: Nil) checkOperationHistory(stream, expectedOperations = Seq( "STREAMING UPDATE", // First write "STREAMING UPDATE" // Second write )) } } test("implicit cast after renaming/dropping columns with column mapping") { withDeltaStream[(Int, Int)] { stream => stream.write((1, 100))("_1 AS a", "CAST(_2 AS LONG) AS b") assert(stream.currentSchema === new StructType() .add("a", IntegerType) .add("b", LongType)) checkAnswer(stream.read(), Row(1, 100)) sql( s""" |ALTER TABLE delta.`${stream.deltaLog.dataPath}` SET TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name', | 'delta.minReaderVersion' = '2', | 'delta.minWriterVersion' = '5' |) """.stripMargin) sql(s"ALTER TABLE delta.`${stream.deltaLog.dataPath}` DROP COLUMN a") sql(s"ALTER TABLE delta.`${stream.deltaLog.dataPath}` RENAME COLUMN b to a") assert(stream.currentSchema === new StructType() .add("a", LongType)) stream.write((17, -1))("CAST(_1 AS STRING) AS a") assert(stream.currentSchema === new StructType() .add("a", LongType)) checkAnswer(stream.read(), Row(100) :: Row(17) :: Nil) checkOperationHistory(stream, expectedOperations = Seq( "STREAMING UPDATE", // First write "SET TBLPROPERTIES", // Enable column mapping "DROP COLUMNS", // Drop column "RENAME COLUMN", // Rename Column "STREAMING UPDATE" // Second write )) } } test("disallow implicit cast with spark.databricks.delta.streaming.sink.allowImplicitCasts") { withSQLConf(DeltaSQLConf.DELTA_STREAMING_SINK_ALLOW_IMPLICIT_CASTS.key -> "false") { withDeltaStream[Long] { stream => stream.write(17)("CAST(value AS INT)") assert(stream.currentSchema("value").dataType === IntegerType) checkAnswer(stream.read(), Row(17)) val ex = intercept[StreamingQueryException] { stream.write(23)("CAST(value AS LONG)") } checkError( ex.getCause.asInstanceOf[SparkThrowable], "DELTA_FAILED_TO_MERGE_FIELDS", parameters = Map( "currentField" -> "value", "updateField" -> "value") ) } } } for (allowImplicitCasts <- Seq(true, false)) test(s"schema evolution with case sensitivity and without type mismatch, " + s"allowImplicitCasts=$allowImplicitCasts") { withSQLConf( DeltaSQLConf.DELTA_STREAMING_SINK_ALLOW_IMPLICIT_CASTS.key -> allowImplicitCasts.toString, SQLConf.CASE_SENSITIVE.key -> "true", DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true" ) { withDeltaStream[(Long, Long)] { stream => stream.write((17, -1))("CAST(_1 AS INT) AS a") assert(stream.currentSchema == new StructType().add("a", IntegerType)) checkAnswer(stream.read(), Row(17)) stream.write((21, 22))("CAST(_1 AS INT) AS A", "_2 AS b") assert(stream.currentSchema == new StructType() .add("a", IntegerType) .add("b", LongType)) checkAnswer(stream.read(), Row(17, null) :: Row(21, 22) :: Nil) } } } test("handling type mismatch in addBatch") { withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath val deltaLog = DeltaLog.forTable(spark, tablePath) sqlContext.sparkContext.setLocalProperty(StreamExecution.QUERY_ID_KEY, "streaming_query") val sink = DeltaSink( sqlContext, path = deltaLog.dataPath, partitionColumns = Seq.empty, outputMode = OutputMode.Append(), options = new DeltaOptions(options = Map.empty, conf = spark.sessionState.conf) ) val schema = new StructType().add("value", IntegerType) { val data = Seq(0, 1).toDF("value").selectExpr("CAST(value AS INT)") sink.addBatch(0, data) val df = spark.read.format("delta").load(tablePath) assert(df.schema === schema) checkAnswer(df, Row(0) :: Row(1) :: Nil) } { val data = Seq(2, 3).toDF("value").selectExpr("CAST(value AS LONG)") sink.addBatch(1, data) val df = spark.read.format("delta").load(tablePath) assert(df.schema === schema) checkAnswer(df, Row(0) :: Row(1) :: Row(2) :: Row(3) :: Nil) } } } } class DeltaSinkImplicitCastWithCatalogManagedBatch1Suite extends DeltaSinkImplicitCastSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class DeltaSinkImplicitCastWithCatalogManagedBatch2Suite extends DeltaSinkImplicitCastSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class DeltaSinkImplicitCastWithCatalogManagedBatch100Suite extends DeltaSinkImplicitCastSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaSinkSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.util.Locale // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.actions.CommitInfo import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.sources.{DeltaSink, DeltaSQLConf} import org.apache.spark.sql.delta.test.{DeltaColumnMappingSelectedTestMixin, DeltaSQLCommandTest} import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.test.shims.StreamingTestShims.{MemoryStream, MicroBatchExecution, StreamingQueryWrapper} import org.apache.commons.io.FileUtils import org.scalatest.time.SpanSugar._ import org.apache.spark.SparkConf import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.DataSourceScanExec import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.execution.streaming.sources.WriteToMicroBatchDataSourceV1 import org.apache.spark.sql.functions._ import org.apache.spark.sql.streaming._ import org.apache.spark.sql.types._ import org.apache.spark.util.Utils abstract class DeltaSinkTest extends StreamTest with DeltaSQLCommandTest { override val streamingTimeout = 60.seconds import testImplicits._ // Before we start running the tests in this suite, we should let Spark perform all necessary set // up that needs to be done for streaming. Without this, the first test in the suite may be flaky // as its running time can exceed the timeout for the test due to Spark setup. See: ES-235735 override def beforeAll(): Unit = { super.beforeAll() withTempDirs { (outputDir, checkpointDir) => val inputData = MemoryStream[Int].toDF() val query = inputData.writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("delta") .start(outputDir.getCanonicalPath) query.stop() } } protected def withTempDirs(f: (File, File) => Unit): Unit = { withTempDir { file1 => withTempDir { file2 => f(file1, file2) } } } } class DeltaSinkSuite extends DeltaSinkTest with DeltaColumnMappingTestUtils with CatalogOwnedTestBaseSuite with DeltaSQLTestUtils { import testImplicits._ test("append mode") { failAfter(streamingTimeout) { withTempDirs { (outputDir, checkpointDir) => val inputData = MemoryStream[Int] val df = inputData.toDF() val query = df.writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("delta") .start(outputDir.getCanonicalPath) val log = DeltaLog.forTable(spark, outputDir.getCanonicalPath) try { inputData.addData(1) query.processAllAvailable() val outputDf = spark.read.format("delta").load(outputDir.getCanonicalPath) checkDatasetUnorderly(outputDf.as[Int], 1) assert(log.update().transactions.head == (query.id.toString -> 0L)) inputData.addData(2) query.processAllAvailable() checkDatasetUnorderly(outputDf.as[Int], 1, 2) assert(log.update().transactions.head == (query.id.toString -> 1L)) inputData.addData(3) query.processAllAvailable() checkDatasetUnorderly(outputDf.as[Int], 1, 2, 3) assert(log.update().transactions.head == (query.id.toString -> 2L)) } finally { query.stop() } } } } test("complete mode") { failAfter(streamingTimeout) { withTempDirs { (outputDir, checkpointDir) => val inputData = MemoryStream[Int] val df = inputData.toDF() val query = df.groupBy().count() .writeStream .outputMode("complete") .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("delta") .start(outputDir.getCanonicalPath) val log = DeltaLog.forTable(spark, outputDir.getCanonicalPath) try { inputData.addData(1) query.processAllAvailable() val outputDf = spark.read.format("delta").load(outputDir.getCanonicalPath) checkDatasetUnorderly(outputDf.as[Long], 1L) assert(log.update().transactions.head == (query.id.toString -> 0L)) inputData.addData(2) query.processAllAvailable() checkDatasetUnorderly(outputDf.as[Long], 2L) assert(log.update().transactions.head == (query.id.toString -> 1L)) inputData.addData(3) query.processAllAvailable() checkDatasetUnorderly(outputDf.as[Long], 3L) assert(log.update().transactions.head == (query.id.toString -> 2L)) } finally { query.stop() } } } } test("update mode: not supported") { failAfter(streamingTimeout) { withTempDirs { (outputDir, checkpointDir) => val inputData = MemoryStream[Int] val df = inputData.toDF() val e = intercept[AnalysisException] { df.writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .outputMode("update") .format("delta") .start(outputDir.getCanonicalPath) } Seq("update", "not support").foreach { msg => assert(e.getMessage.toLowerCase(Locale.ROOT).contains(msg)) } } } } test("path not specified") { failAfter(streamingTimeout) { withTempDir { checkpointDir => val inputData = MemoryStream[Int] val df = inputData.toDF() val e = intercept[IllegalArgumentException] { df.writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("delta") .start() } Seq("path", " not specified").foreach { msg => assert(e.getMessage.toLowerCase(Locale.ROOT).contains(msg)) } } } } test("SPARK-21167: encode and decode path correctly") { withTempDirs { (outputDir, checkpointDir) => val inputData = MemoryStream[String] val query = inputData.toDS() .map(s => (s, s.length)) .toDF("value", "len") .writeStream .partitionBy("value") .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("delta") .start(outputDir.getCanonicalPath) try { // The output is partitioned by "value", so the value will appear in the file path. // This is to test if we handle spaces in the path correctly. inputData.addData("hello world") failAfter(streamingTimeout) { query.processAllAvailable() } val outputDf = spark.read.format("delta").load(outputDir.getCanonicalPath) checkDatasetUnorderly(outputDf.as[(String, Int)], ("hello world", "hello world".length)) } finally { query.stop() } } } test("partitioned writing and batch reading") { withTempDirs { (outputDir, checkpointDir) => val inputData = MemoryStream[Int] val ds = inputData.toDS() val query = ds.map(i => (i, i * 1000)) .toDF("id", "value") .writeStream .partitionBy("id") .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("delta") .start(outputDir.getCanonicalPath) try { inputData.addData(1, 2, 3) failAfter(streamingTimeout) { query.processAllAvailable() } val outputDf = spark.read.format("delta").load(outputDir.getCanonicalPath) val expectedSchema = new StructType() .add(StructField("id", IntegerType)) .add(StructField("value", IntegerType)) assert(outputDf.schema === expectedSchema) // Verify the correct partitioning schema has been inferred val hadoopFsRelations = outputDf.queryExecution.analyzed.collect { case LogicalRelationWithTable(baseRelation, _) if baseRelation.isInstanceOf[HadoopFsRelation] => baseRelation.asInstanceOf[HadoopFsRelation] } assert(hadoopFsRelations.size === 1) assert(hadoopFsRelations.head.partitionSchema.exists(_.name == "id")) assert(hadoopFsRelations.head.dataSchema.exists(_.name == "value")) // Verify the data is correctly read checkDatasetUnorderly( outputDf.as[(Int, Int)], (1, 1000), (2, 2000), (3, 3000)) /** Check some condition on the partitions of the FileScanRDD generated by a DF */ def checkFileScanPartitions(df: DataFrame)(func: Seq[FilePartition] => Unit): Unit = { val filePartitions = df.queryExecution.executedPlan.collect { case scan: DataSourceScanExec if scan.inputRDDs().head.isInstanceOf[FileScanRDD] => scan.inputRDDs().head.asInstanceOf[FileScanRDD].filePartitions }.flatten if (filePartitions.isEmpty) { fail(s"No FileScan in query\n${df.queryExecution}") } func(filePartitions) } // Read without pruning checkFileScanPartitions(outputDf) { partitions => // There should be as many distinct partition values as there are distinct ids assert(partitions.flatMap(_.files.map(_.partitionValues)).distinct.size === 3) } // Read with pruning, should read only files in partition dir id=1 checkFileScanPartitions(outputDf.filter("id = 1")) { partitions => // use physical name val filesToBeRead = partitions.flatMap(_.files) assert(filesToBeRead.forall(_.partitionValues.getInt(0) == 1)) assert(filesToBeRead.map(_.partitionValues).distinct.size === 1) } // Read with pruning, should read only files in partition dir id=1 and id=2 checkFileScanPartitions(outputDf.filter("id in (1,2)")) { partitions => val filesToBeRead = partitions.flatMap(_.files) assert(filesToBeRead.forall(_.partitionValues.getInt(0) != 3)) assert(filesToBeRead.map(_.partitionValues).distinct.size === 2) } } finally { if (query != null) { query.stop() } } } } test("work with aggregation + watermark") { withTempDirs { (outputDir, checkpointDir) => val inputData = MemoryStream[Long] val inputDF = inputData.toDF.toDF("time") val outputDf = inputDF .selectExpr("CAST(time AS timestamp) AS timestamp") .withWatermark("timestamp", "10 seconds") .groupBy(window($"timestamp", "5 seconds")) .count() .select("window.start", "window.end", "count") val query = outputDf.writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("delta") .start(outputDir.getCanonicalPath) try { def addTimestamp(timestampInSecs: Int*): Unit = { inputData.addData(timestampInSecs.map(_ * 1L): _*) failAfter(streamingTimeout) { query.processAllAvailable() } } def check(expectedResult: ((Long, Long), Long)*): Unit = { val outputDf = spark.read.format("delta").load(outputDir.getCanonicalPath) .selectExpr( "CAST(start as BIGINT) AS start", "CAST(end as BIGINT) AS end", "count") checkDatasetUnorderly( outputDf.as[(Long, Long, Long)], expectedResult.map(x => (x._1._1, x._1._2, x._2)): _*) } addTimestamp(100) // watermark = None before this, watermark = 100 - 10 = 90 after this addTimestamp(104, 123) // watermark = 90 before this, watermark = 123 - 10 = 113 after this addTimestamp(140) // wm = 113 before this, emit results on 100-105, wm = 130 after this check((100L, 105L) -> 2L, (120L, 125L) -> 1L) // no-data-batch emits results on 120-125 } finally { if (query != null) { query.stop() } } } } test("throw exception when users are trying to write in batch with different partitioning") { withTempDirs { (outputDir, checkpointDir) => val inputData = MemoryStream[Int] val ds = inputData.toDS() val query = ds.map(i => (i, i * 1000)) .toDF("id", "value") .writeStream .partitionBy("id") .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("delta") .start(outputDir.getCanonicalPath) try { inputData.addData(1, 2, 3) failAfter(streamingTimeout) { query.processAllAvailable() } val e = intercept[AnalysisException] { spark.range(100) .select('id.cast("integer"), 'id % 4 as "by4", 'id.cast("integer") * 1000 as "value") .write .format("delta") .partitionBy("id", "by4") .mode("append") .save(outputDir.getCanonicalPath) } assert(e.getMessage.contains("Partition columns do not match")) } finally { query.stop() } } } testQuietly("incompatible schema merging throws errors - first streaming then batch") { withTempDirs { (outputDir, checkpointDir) => val inputData = MemoryStream[Int] val ds = inputData.toDS() val query = ds.map(i => (i, i * 1000)) .toDF("id", "value") .writeStream .partitionBy("id") .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("delta") .start(outputDir.getCanonicalPath) try { inputData.addData(1, 2, 3) failAfter(streamingTimeout) { query.processAllAvailable() } val e = intercept[AnalysisException] { spark.range(100).select('id, ('id * 3).cast("string") as "value") .write .partitionBy("id") .format("delta") .mode("append") .save(outputDir.getCanonicalPath) } checkError( e, "DELTA_FAILED_TO_MERGE_FIELDS", parameters = Map("currentField" -> "id", "updateField" -> "id")) } finally { query.stop() } } } test("incompatible schema merging throws errors - first batch then streaming") { withTempDirs { (outputDir, checkpointDir) => val inputData = MemoryStream[Int] val ds = inputData.toDS() val dsWriter = ds.map(i => (i, i * 1000)) .toDF("id", "value") .writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("delta") spark.range(100).select('id, ('id * 3).cast("string") as "value") .write .format("delta") .mode("append") .save(outputDir.getCanonicalPath) // More tests covering type changes can be found in [[DeltaSinkImplicitCastSuite]]. This only // covers type changes disabled. withSQLConf(DeltaSQLConf.DELTA_STREAMING_SINK_ALLOW_IMPLICIT_CASTS.key -> "false") { val wrapperException = intercept[StreamingQueryException] { val q = dsWriter.start(outputDir.getCanonicalPath) inputData.addData(1, 2, 3) q.processAllAvailable() } assert(wrapperException.cause.isInstanceOf[AnalysisException]) checkError( wrapperException.cause.asInstanceOf[AnalysisException], "DELTA_FAILED_TO_MERGE_FIELDS", parameters = Map("currentField" -> "id", "updateField" -> "id")) } } } private def verifyDeltaSinkCatalog(f: DataStreamWriter[_] => StreamingQuery): Unit = { // Create a Delta sink whose target table is defined by our caller. val input = MemoryStream[Int] val streamWriter = input.toDF .writeStream .format("delta") .option( "checkpointLocation", Utils.createTempDir(namePrefix = "tahoe-test").getCanonicalPath) val q = f(streamWriter).asInstanceOf[StreamingQueryWrapper] // WARNING: Only the query execution thread is allowed to initialize the logical plan (enforced // by an assertion in MicroBatchExecution.scala). To avoid flaky failures, run the stream to // completion, to guarantee the query execution thread ran before we try to access the plan. try { input.addData(1, 2, 3) q.processAllAvailable() } finally { q.stop() } val plan = q.streamingQuery.logicalPlan val WriteToMicroBatchDataSourceV1(catalogTable, sink: DeltaSink, _, _, _, _, _) = plan assert(catalogTable === sink.catalogTable) } test("DeltaSink.catalogTable is correctly populated - catalog-based table") { withTable("tab") { verifyDeltaSinkCatalog(_.toTable("tab")) } } test("DeltaSink.catalogTable is correctly populated - path-based table") { withTempDir { tempDir => if (tempDir.exists()) { assert(tempDir.delete()) } verifyDeltaSinkCatalog(_.start(tempDir.getCanonicalPath)) } } test("can't write out with all columns being partition columns") { withTempDirs { (outputDir, checkpointDir) => val inputData = MemoryStream[Int] val ds = inputData.toDS() val query = ds.map(i => (i, i * 1000)) .toDF("id", "value") .writeStream .partitionBy("id", "value") .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("delta") .start(outputDir.getCanonicalPath) val e = intercept[StreamingQueryException] { inputData.addData(1) query.awaitTermination(30000) } assert(e.cause.isInstanceOf[AnalysisException]) } } test("streaming write correctly sets isBlindAppend in CommitInfo") { withTempDirs { (outputDir, checkpointDir) => val input = MemoryStream[Int] val inputDataStream = input.toDF().toDF("value") def tableData: DataFrame = spark.read.format("delta").load(outputDir.toString) def appendToTable(df: DataFrame): Unit = failAfter(streamingTimeout) { var q: StreamingQuery = null try { input.addData(0) q = df.writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) .start(outputDir.toString) q.processAllAvailable() } finally { if (q != null) q.stop() } } var lastCheckedVersion = -1L def isLastCommitBlindAppend: Boolean = { val log = DeltaLog.forTable(spark, outputDir.toString) val lastVersion = log.update().version assert(lastVersion > lastCheckedVersion, "no new commit was made") lastCheckedVersion = lastVersion val lastCommitChanges = log.getChanges(lastVersion).toSeq.head._2 lastCommitChanges.collectFirst { case c: CommitInfo => c }.flatMap(_.isBlindAppend).get } // Simple streaming write should have isBlindAppend = true appendToTable(inputDataStream) assert( isLastCommitBlindAppend, "simple write to target table should have isBlindAppend = true") // Join with the table should have isBlindAppend = false appendToTable(inputDataStream.join(tableData, "value")) assert( !isLastCommitBlindAppend, "joining with target table in the query should have isBlindAppend = false") } } test("do not trust user nullability, so that parquet files aren't corrupted") { val jsonRec = """{"s": "ss", "b": {"s": "ss"}}""" val schema = new StructType() .add("s", StringType) .add("b", new StructType() .add("s", StringType) .add("i", IntegerType, nullable = false)) .add("c", IntegerType, nullable = false) withTempDir { base => val sourceDir = new File(base, "source").getCanonicalPath val tableDir = new File(base, "output").getCanonicalPath val chkDir = new File(base, "checkpoint").getCanonicalPath FileUtils.write(new File(sourceDir, "a.json"), jsonRec) val q = spark.readStream .format("json") .schema(schema) .load(sourceDir) .withColumn("file", input_file_name()) // Not sure why needs this to reproduce .writeStream .format("delta") .trigger(org.apache.spark.sql.streaming.Trigger.Once) .option("checkpointLocation", chkDir) .start(tableDir) q.awaitTermination() checkAnswer( spark.read.format("delta").load(tableDir).drop("file"), Seq(Row("ss", Row("ss", null), null))) } } test("history includes user-defined metadata for DataFrame.writeStream API") { failAfter(streamingTimeout) { withTempDirs { (outputDir, checkpointDir) => val inputData = MemoryStream[Int] val df = inputData.toDF() val query = df.writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .option("userMetadata", "testMeta!") .format("delta") .start(outputDir.getCanonicalPath) val log = DeltaLog.forTable(spark, outputDir.getCanonicalPath) inputData.addData(1) query.processAllAvailable() val lastCommitInfo = io.delta.tables.DeltaTable.forPath(spark, outputDir.getCanonicalPath) .history(1).as[DeltaHistory].head assert(lastCommitInfo.userMetadata === Some("testMeta!")) query.stop() } } } test( "DeltaSink.deltaLog is not initialized in DeltaSink constructor" ) { withTempTable(createTable = true) { tableName => val outputDir = DeltaLog.forTable(spark, TableIdentifier(tableName)).dataPath // Create a DeltaSink instance directly val deltaSink = new DeltaSink( spark.sqlContext, outputDir, partitionColumns = Seq.empty[String], outputMode = OutputMode.Append, options = new DeltaOptions(Map( "checkpointlocation" -> outputDir.toString, "path" -> outputDir.toString ), spark.sessionState.conf) ) // Helper function to check if deltaLog is initialized using reflection def isDeltaLogInitialized(sink: DeltaSink): Boolean = { val fieldOpt = classOf[DeltaSink].getDeclaredFields.find( f => f.getName.contains("deltaLog") && f.getType == classOf[DeltaLog]) assert(fieldOpt.isDefined, "deltaLog field not found") fieldOpt.exists { field => field.setAccessible(true) field.get(sink) != null } } // Test that deltaLog is NOT initialized after constructor assert(!isDeltaLogInitialized(deltaSink), "deltaLog should not be initialized after constructor") } } test("DeltaSink rejects DataFrame with UDT containing NullType") { failAfter(streamingTimeout) { withTempDirs { (outputDir, checkpointDir) => val inputData = MemoryStream[Int] val ds = inputData.toDS() val dsWriter = ds.map(i => (i, new NullData())) .toDF("id", "value") .writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("delta") val wrapperException = intercept[StreamingQueryException] { val q = dsWriter.start(outputDir.getCanonicalPath) inputData.addData(42) q.processAllAvailable() } assert(wrapperException.cause.isInstanceOf[AnalysisException]) checkError( wrapperException.cause.asInstanceOf[AnalysisException], "DELTA_NULL_SCHEMA_IN_STREAMING_WRITE") } } } } // Batch sizes 1, 2, and 100 exercise different backfill behaviors in the commit coordinator. // Batch size 1 triggers a backfill on every commit (commitVersion % 1 == 0), testing the most // granular backfill path. Batch size 2 triggers backfill every other commit, testing the boundary // between backfilled and unbackfilled commits. Batch size 100 leaves most commits unbackfilled, // testing the production-like path where streaming must read from both the commit coordinator // and the filesystem. This follows the same pattern as other CatalogManaged (CCv2) test suites // (DeltaLogSuite, DeltaSourceSuite, etc.). class DeltaSinkWithCatalogManagedBatch1Suite extends DeltaSinkSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class DeltaSinkWithCatalogManagedBatch2Suite extends DeltaSinkSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class DeltaSinkWithCatalogManagedBatch100Suite extends DeltaSinkSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } abstract class DeltaSinkColumnMappingSuiteBase extends DeltaSinkSuite with DeltaColumnMappingSelectedTestMixin { import testImplicits._ override protected def runOnlyTests = Seq( "append mode", "complete mode", "partitioned writing and batch reading", "work with aggregation + watermark" ) test("allow schema evolution after renaming column") { Seq(true, false).foreach { schemaMergeEnabled => withClue(s"Schema merge enabled: $schemaMergeEnabled") { withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> schemaMergeEnabled.toString) { failAfter(streamingTimeout) { withTempDirs { (outputDir, checkpointDir) => val sourceDir = Utils.createTempDir() def addData(df: DataFrame): Unit = df.coalesce(1).write.mode("append").save(sourceDir.getCanonicalPath) // save data to target dir Seq(100).toDF("value").write.format("delta").save(outputDir.getCanonicalPath) // use parquet stream as MemoryStream doesn't support recovering failed batches val df = spark.readStream .schema(new StructType().add("value", IntegerType, true)) .parquet(sourceDir.getCanonicalPath) // start writing into Delta sink def queryGen(df: DataFrame): StreamingQuery = df.writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("delta") .start(outputDir.getCanonicalPath) val query = queryGen(df) val log = DeltaLog.forTable(spark, outputDir.getCanonicalPath) // delta sink contains [100, 1] addData(Seq(1).toDF("value")) query.processAllAvailable() def outputDf: DataFrame = spark.read.format("delta").load(outputDir.getCanonicalPath) checkDatasetUnorderly(outputDf.as[Int], 100, 1) require(log.update().transactions.head == (query.id.toString -> 0L)) sql(s"ALTER TABLE delta.`${outputDir.getAbsolutePath}` " + s"RENAME COLUMN value TO new_value") if (!schemaMergeEnabled) { // schema has changed, we can't automatically migrate the schema val e = intercept[StreamingQueryException] { addData(Seq(2).toDF("value")) query.processAllAvailable() } assert(e.cause.isInstanceOf[AnalysisException]) assert(e.cause.getMessage.contains("A schema mismatch detected when writing")) // restart using the same query would still fail val query2 = queryGen(df) val e2 = intercept[StreamingQueryException] { addData(Seq(2).toDF("value")) query2.processAllAvailable() } assert(e2.cause.isInstanceOf[AnalysisException]) assert(e2.cause.getMessage.contains("A schema mismatch detected when writing")) // but reingest using new schema should work val df2 = spark.readStream .schema(new StructType().add("value", IntegerType, true)) .parquet(sourceDir.getCanonicalPath) .withColumnRenamed("value", "new_value") val query3 = queryGen(df2) // delta sink contains [100, 1, 2] + [2, 2] due to recovering the failed batched addData(Seq(2).toDF("value")) query3.processAllAvailable() checkAnswer(outputDf, Row(100) :: Row(1) :: Row(2) :: Row(2) :: Row(2) :: Nil) assert(outputDf.schema == new StructType().add("new_value", IntegerType, true)) query3.stop() } else { // we allow auto schema migration, delta sink contains [100, 1, 2] addData(Seq(2).toDF("value")) query.processAllAvailable() // Since the incoming `value` column is now merged as a new column (even though it // has the same value as the original name) in which only the 3rd record has data. checkAnswer(outputDf, Row(100, null) :: Row(1, null) :: Row(null, 2) :: Nil) assert(outputDf.schema == new StructType().add("new_value", IntegerType, true) .add("value", IntegerType, true)) query.stop() } } } } } } } test("allow schema evolution after dropping column") { Seq(true, false).foreach { schemaMergeEnabled => withClue(s"Schema merge enabled: $schemaMergeEnabled") { withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> schemaMergeEnabled.toString) { failAfter(streamingTimeout) { withTempDirs { (outputDir, checkpointDir) => val sourceDir = Utils.createTempDir() def addData(df: DataFrame): Unit = df.coalesce(1).write.mode("append").save(sourceDir.getCanonicalPath) // save data to target dir Seq((1, 100)).toDF("id", "value").write.format("delta") .save(outputDir.getCanonicalPath) // use parquet stream as MemoryStream doesn't support recovering failed batches val df = spark.readStream .schema(new StructType().add("id", IntegerType, true) .add("value", IntegerType, true)) .parquet(sourceDir.getCanonicalPath) // start writing into Delta sink def queryGen(df: DataFrame): StreamingQuery = df.writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("delta") .start(outputDir.getCanonicalPath) val query = queryGen(df) val log = DeltaLog.forTable(spark, outputDir.getCanonicalPath) // delta sink contains [(1, 100), (2, 200)] addData(Seq((2, 200)).toDF("id", "value")) query.processAllAvailable() def outputDf: DataFrame = spark.read.format("delta").load(outputDir.getCanonicalPath) checkDatasetUnorderly(outputDf.as[(Int, Int)], (1, 100), (2, 200)) assert(log.update().transactions.head == (query.id.toString -> 0L)) withSQLConf(DeltaSQLConf.DELTA_ALTER_TABLE_DROP_COLUMN_ENABLED.key -> "true") { sql(s"ALTER TABLE delta.`${outputDir.getAbsolutePath}` DROP COLUMN value") } if (!schemaMergeEnabled) { // schema changed, we can't automatically migrate the schema val e = intercept[StreamingQueryException] { addData(Seq((3, 300)).toDF("id", "value")) query.processAllAvailable() } assert(e.cause.isInstanceOf[AnalysisException]) assert(e.cause.getMessage.contains("A schema mismatch detected when writing")) // restart using the same query would still fail val query2 = queryGen(df) val e2 = intercept[StreamingQueryException] { addData(Seq((3, 300)).toDF("id", "value")) query2.processAllAvailable() } assert(e2.cause.isInstanceOf[AnalysisException]) assert(e2.cause.getMessage.contains("A schema mismatch detected when writing")) // but reingest using new schema should work val df2 = spark.readStream .schema(new StructType().add("id", IntegerType, true)) .parquet(sourceDir.getCanonicalPath) val query3 = queryGen(df2) // delta sink contains [1, 2, 3] + [3, 3] due to // recovering failed batches addData(Seq((3, 300)).toDF("id", "value")) query3.processAllAvailable() checkAnswer(outputDf, Row(1) :: Row(2) :: Row(3) :: Row(3) :: Row(3) :: Nil) assert(outputDf.schema == new StructType().add("id", IntegerType, true)) query3.stop() } else { addData(Seq((3, 300)).toDF("id", "value")) query.processAllAvailable() // None/null value appears because even though the added column has the same // logical name (`value`) as the dropped column, the physical name has been // changed so the old data could not be loaded. checkAnswer(outputDf, Row(1, null) :: Row(2, null) :: Row(3, 300) :: Nil) assert(outputDf.schema == new StructType().add("id", IntegerType, true).add("value", IntegerType, true)) query.stop() } } } } } } } } class DeltaSinkIdColumnMappingSuite extends DeltaSinkColumnMappingSuiteBase with DeltaColumnMappingEnableIdMode with DeltaColumnMappingTestUtils class DeltaSinkNameColumnMappingSuite extends DeltaSinkColumnMappingSuiteBase with DeltaColumnMappingEnableNameMode ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceColumnMappingSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.util.UUID import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.Relocated.StreamExecution import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.sources.{DeltaSource, DeltaSQLConf} import org.apache.spark.sql.delta.test.DeltaColumnMappingSelectedTestMixin import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.JsonUtils import org.apache.commons.io.FileUtils import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.sql.{DataFrame, Row} import org.apache.spark.sql.delta.test.shims.StreamingTestShims.StreamingExecutionRelation import org.apache.spark.sql.streaming.{DataStreamReader, StreamTest} import org.apache.spark.sql.types.{StringType, StructType} import org.apache.spark.util.Utils trait ColumnMappingStreamingTestUtils extends StreamTest with DeltaColumnMappingTestUtils { // Whether we are requesting CDC streaming changes protected def isCdcTest: Boolean protected val ProcessAllAvailableIgnoreError = Execute { q => try { q.processAllAvailable() } catch { case _: Throwable => // swallow the errors so we could check answer and failure on the query later } } protected def isColumnMappingSchemaIncompatibleFailure( t: Throwable, detectedDuringStreaming: Boolean): Boolean = t match { case e: DeltaStreamingNonAdditiveSchemaIncompatibleException => e.additionalProperties.get("detectedDuringStreaming") .exists(_.toBoolean == detectedDuringStreaming) case _ => false } protected val ExpectStreamStartInCompatibleSchemaFailure = ExpectFailure[DeltaStreamingNonAdditiveSchemaIncompatibleException] { t => assert(isColumnMappingSchemaIncompatibleFailure(t, detectedDuringStreaming = false)) } protected val ExpectInStreamSchemaChangeFailure = ExpectFailure[DeltaStreamingNonAdditiveSchemaIncompatibleException] { t => assert(isColumnMappingSchemaIncompatibleFailure(t, detectedDuringStreaming = true)) } protected val ExpectGenericSchemaIncompatibleFailure = ExpectFailure[DeltaStreamingNonAdditiveSchemaIncompatibleException]() // Failure thrown by the current DeltaSource schema change incompatible check protected val ExistingRetryableInStreamSchemaChangeFailure = Execute { q => // Similar to ExpectFailure but allows more fine-grained checking of exceptions failAfter(streamingTimeout) { try { q.awaitTermination() } catch { case _: Throwable => // swallow the exception } val cause = ExceptionUtils.getRootCause(q.exception.get) assert(cause.getMessage.contains("Detected schema change")) } } protected def getLatestCommittedDeltaVersion(q: StreamExecution): Long = JsonUtils.fromJson[Map[String, Any]]( q.committedOffsets.values.head.json() ).apply("reservoirVersion").asInstanceOf[Number].longValue() // Drop CDC fields because they are not useful for testing the blocking behavior protected def dropCDCFields(df: DataFrame): DataFrame = df.drop(CDCReader.CDC_COMMIT_TIMESTAMP) .drop(CDCReader.CDC_TYPE_COLUMN_NAME) .drop(CDCReader.CDC_COMMIT_VERSION) } trait ColumnMappingStreamingBlockedWorkflowSuiteBase extends ColumnMappingStreamingTestUtils { import testImplicits._ // DataStreamReader to use // Set a small max file per trigger to ensure we could catch failures ASAP private def dsr: DataStreamReader = if (isCdcTest) { spark.readStream.format("delta") .option(DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION, "1") .option(DeltaOptions.CDC_READ_OPTION, "true") } else { spark.readStream.format("delta") .option(DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION, "1") } private def checkStreamStartBlocked( df: DataFrame, ckpt: File, expectedFailure: StreamAction): Unit = { // Restart the stream from the same checkpoint will pick up the dropped schema and our // column mapping check will kick in and error out. testStream(df)( StartStream(checkpointLocation = ckpt.getCanonicalPath), ProcessAllAvailableIgnoreError, // No batches have been served CheckLastBatch(Nil: _*), expectedFailure ) } protected def writeDeltaData( data: Seq[Int], deltaLog: DeltaLog, userSpecifiedSchema: Option[StructType] = None): Unit = { val schema = userSpecifiedSchema.getOrElse(deltaLog.update().schema) data.foreach { i => val data = Seq(Row(schema.map(_ => i.toString): _*)) spark.createDataFrame(data.asJava, schema) .write.format("delta").mode("append").save(deltaLog.dataPath.toString) } } test("deltaLog snapshot should not be updated outside of the stream") { withTempDir { dir => val tablePath = dir.getCanonicalPath // write initial data Seq(1).toDF("id").write.format("delta").mode("overwrite").save(tablePath) // record initial snapshot version and warm DeltaLog cache val initialDeltaLog = DeltaLog.forTable(spark, tablePath) // start streaming val df = spark.readStream.format("delta").load(tablePath) testStream(df)( StartStream(), ProcessAllAvailable(), AssertOnQuery { q => // write more data Seq(2).toDF("id").write.format("delta").mode("append").save(tablePath) // update deltaLog externally initialDeltaLog.update() assert(initialDeltaLog.snapshot.version == 1) // query start snapshot should not change val source = q.logicalPlan.collectFirst { case r: StreamingExecutionRelation => r.source.asInstanceOf[DeltaSource] }.get // same delta log but stream start version not affected source.snapshotAtSourceInit.deltaLog == initialDeltaLog && source.snapshotAtSourceInit.version == 0 } ) } } test("column mapping + streaming - allowed workflows - column addition") { // column addition schema evolution should not be blocked upon restart withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) writeDeltaData(0 until 5, deltaLog, Some(StructType.fromDDL("id string, value string"))) val checkpointDir = new File(inputDir, "_checkpoint") def loadDf(): DataFrame = dropCDCFields(dsr.load(inputDir.getCanonicalPath)) testStream(loadDf())( StartStream(checkpointLocation = checkpointDir.getCanonicalPath), ProcessAllAvailable(), CheckAnswer((0 until 5).map(i => (i.toString, i.toString)): _*), Execute { _ => sql(s"ALTER TABLE delta.`${inputDir.getCanonicalPath}` ADD COLUMN (value2 string)") }, Execute { _ => writeDeltaData(5 until 10, deltaLog) }, ExistingRetryableInStreamSchemaChangeFailure ) testStream(loadDf())( StartStream(checkpointLocation = checkpointDir.getCanonicalPath), ProcessAllAvailable(), // Sink is reinitialized, only 5-10 are ingested CheckAnswer( (5 until 10).map(i => (i.toString, i.toString, i.toString)): _*) ) } } test("column mapping + streaming - allowed workflows - upgrade to name mode") { // upgrade should not blocked both during the stream AND during stream restart withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) withColumnMappingConf("none") { writeDeltaData(0 until 5, deltaLog, Some(StructType.fromDDL("id string, name string"))) } def createNewDf(): DataFrame = dropCDCFields(dsr.load(inputDir.getCanonicalPath)) val checkpointDir = new File(inputDir, "_checkpoint") testStream(createNewDf())( StartStream(checkpointLocation = checkpointDir.getCanonicalPath), ProcessAllAvailable(), CheckAnswer((0 until 5).map(i => (i.toString, i.toString)): _*), Execute { _ => sql( s""" |ALTER TABLE delta.`${inputDir.getCanonicalPath}` |SET TBLPROPERTIES ( | ${DeltaConfigs.COLUMN_MAPPING_MODE.key} = "name", | ${DeltaConfigs.MIN_READER_VERSION.key} = "2", | ${DeltaConfigs.MIN_WRITER_VERSION.key} = "5")""".stripMargin) }, Execute { _ => writeDeltaData(5 until 10, deltaLog) }, ProcessAllAvailable(), CheckAnswer((0 until 10).map(i => (i.toString, i.toString)): _*), // add column schema evolution should fail the stream Execute { _ => sql(s"ALTER TABLE delta.`${inputDir.getCanonicalPath}` ADD COLUMN (value2 string)") }, Execute { _ => writeDeltaData(10 until 15, deltaLog) }, ExistingRetryableInStreamSchemaChangeFailure ) // but should not block after restarting, now in column mapping mode testStream(createNewDf())( StartStream(checkpointLocation = checkpointDir.getCanonicalPath), ProcessAllAvailable(), // Sink is reinitialized, only 10-15 are ingested CheckAnswer( (10 until 15).map(i => (i.toString, i.toString, i.toString)): _*) ) // use a different checkpoint to simulate a clean stream restart val checkpointDir2 = new File(inputDir, "_checkpoint2") testStream(createNewDf())( StartStream(checkpointLocation = checkpointDir2.getCanonicalPath), ProcessAllAvailable(), // Since the latest schema contain the additional column, it is null for previous batches. // This is fine as it is consistent with the current semantics. CheckAnswer((0 until 10).map(i => (i.toString, i.toString, null)) ++ (10 until 15).map(i => (i.toString, i.toString, i.toString)): _*), StopStream ) // Refresh delta log so we could catch the latest schema with column mapping mode deltaLog.update() // test read prior to upgrade batches with latest metadata should also work val checkpointDir3 = new File(inputDir, "_checkpoint3") testStream(dropCDCFields(dsr.option("startingVersion", 0).load(inputDir.getCanonicalPath)))( StartStream(checkpointLocation = checkpointDir3.getCanonicalPath), ProcessAllAvailable(), // Since the latest schema contain the additional column, it is null for previous batches. // This is fine as it is consistent with the current semantics. CheckAnswer((0 until 10).map(i => (i.toString, i.toString, null)) ++ (10 until 15).map(i => (i.toString, i.toString, i.toString)): _*), StopStream ) } } /** * Setup the test table for testing blocked workflow, this will create a id or name mode table * based on which tests it is run. */ protected def setupTestTable(deltaLog: DeltaLog): Unit = { require(columnMappingModeString != NoMapping.name) val tablePath = deltaLog.dataPath.toString // For name mapping, we use upgrade to stir things up a little if (columnMappingModeString == NameMapping.name) { // initialize with no column mapping withColumnMappingConf("none") { writeDeltaData(0 until 5, deltaLog, Some(StructType.fromDDL("id string, value string"))) } // upgrade to name mode val protocol = deltaLog.snapshot.protocol val (r, w) = if (protocol.supportsReaderFeatures || protocol.supportsWriterFeatures) { (TableFeatureProtocolUtils.TABLE_FEATURES_MIN_READER_VERSION, TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION) } else { (spark.conf .get(DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION) .max(ColumnMappingTableFeature.minReaderVersion), spark.conf .get(DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION) .max(ColumnMappingTableFeature.minWriterVersion)) } sql( s""" |ALTER TABLE delta.`${tablePath}` |SET TBLPROPERTIES ( | ${DeltaConfigs.COLUMN_MAPPING_MODE.key} = "name", | ${DeltaConfigs.MIN_READER_VERSION.key} = "$r", | ${DeltaConfigs.MIN_WRITER_VERSION.key} = "$w")""".stripMargin) // write more data post upgrade writeDeltaData(5 until 10, deltaLog) } // For id mapping, we could only create the table from scratch else if (columnMappingModeString == IdMapping.name) { withColumnMappingConf("id") { writeDeltaData(0 until 10, deltaLog, Some(StructType.fromDDL("id string, value string"))) } } } test( "column mapping + streaming: blocking workflow - drop column" ) { val schemaAlterQuery = "DROP COLUMN value" val schemaRestoreQuery = "ADD COLUMN (value string)" withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) setupTestTable(deltaLog) // change schema sql(s"ALTER TABLE delta.`${inputDir.getCanonicalPath}` $schemaAlterQuery") // write more data post change schema writeDeltaData(10 until 15, deltaLog) // Test the two code paths below // Case 1 - Restart did not specify a start version, this will successfully serve the initial // entire existing data based on the initial snapshot's schema, which is basically // the stream schema, all schema changes in between are ignored. // But once the initial snapshot is served, all subsequent batches will fail if // encountering a schema change during streaming, and all restart effort should fail. val checkpointDir = new File(inputDir, "_checkpoint") val df = dropCDCFields(dsr.load(inputDir.getCanonicalPath)) testStream(df)( StartStream(checkpointLocation = checkpointDir.getCanonicalPath), ProcessAllAvailable(), // Initial data (pre + post upgrade + post change schema) all served CheckAnswer((0 until 15).map(i => i.toString): _*), Execute { _ => // write more data in new schema during streaming writeDeltaData(15 until 20, deltaLog) }, ProcessAllAvailable(), // can still work because the schema is still compatible CheckAnswer((0 until 20).map(i => i.toString): _*), // But a new schema change would cause stream to fail // Note here we are restoring back the original schema, see next case for how we test // some extra special cases when schemas are reverted. Execute { _ => sql(s"ALTER TABLE delta.`${inputDir.getCanonicalPath}` $schemaRestoreQuery") }, // write more data in updated schema again Execute { _ => writeDeltaData(20 until 25, deltaLog) }, // The last batch should not be processed and stream should fail ProcessAllAvailableIgnoreError, // sink data did not change CheckAnswer((0 until 20).map(i => i.toString): _*), // The schemaRestoreQuery for DROP column is ADD column so it fails a more benign error ExistingRetryableInStreamSchemaChangeFailure ) val df2 = dropCDCFields(dsr.load(inputDir.getCanonicalPath)) // Since the initial snapshot ignores all schema changes, the most recent schema change // is just ADD COLUMN, which can be retried. testStream(df2)( StartStream(checkpointLocation = checkpointDir.getCanonicalPath), // but an additional drop should fail the stream as we are capturing data changes now Execute { _ => sql(s"ALTER TABLE delta.`${inputDir.getCanonicalPath}` $schemaAlterQuery") }, ProcessAllAvailableIgnoreError, ExpectInStreamSchemaChangeFailure ) // The latest DROP columns blocks the stream. if (isCdcTest) { checkStreamStartBlocked(df2, checkpointDir, ExpectGenericSchemaIncompatibleFailure) } else { val expectedError = ExpectStreamStartInCompatibleSchemaFailure checkStreamStartBlocked(df2, checkpointDir, expectedError) } // Case 2 - Specifically we use startingVersion=0 to simulate serving the entire table's data // in a streaming fashion, ignoring the initialSnapshot. // Here we test the special case when the latest schema is "restored". val checkpointDir2 = new File(inputDir, "_checkpoint2") val dfStartAtZero = dropCDCFields(dsr .option(DeltaOptions.STARTING_VERSION_OPTION, "0") .load(inputDir.getCanonicalPath)) if (isCdcTest) { checkStreamStartBlocked( dfStartAtZero, checkpointDir2, ExpectGenericSchemaIncompatibleFailure) } else { // In the case when we drop and add a column back // the restart should still fail directly because all the historical batches with the same // old logical name now will have a different physical name we would have data loss // lets add back the column we just dropped before sql(s"ALTER TABLE delta.`${inputDir.getCanonicalPath}` $schemaRestoreQuery") assert(DeltaLog.forTable(spark, inputDir.getCanonicalPath).snapshot.schema.size == 2) // restart should block right away checkStreamStartBlocked( dfStartAtZero, checkpointDir, ExpectStreamStartInCompatibleSchemaFailure) } } } test("column mapping + streaming: blocking workflow - rename column") { val schemaAlterQuery = "RENAME COLUMN value TO value2" val schemaRestoreQuery = "RENAME COLUMN value2 TO value" withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) setupTestTable(deltaLog) // change schema sql(s"ALTER TABLE delta.`${inputDir.getCanonicalPath}` $schemaAlterQuery") // write more data post change schema writeDeltaData(10 until 15, deltaLog) // Test the two code paths below // Case 1 - Restart did not specify a start version, this will successfully serve the initial // entire existing data based on the initial snapshot's schema, which is basically // the stream schema, all schema changes in between are ignored. // But once the initial snapshot is served, all subsequent batches will fail if // encountering a schema change during streaming, and all restart effort should fail. val checkpointDir = new File(inputDir, "_checkpoint") def df: DataFrame = dropCDCFields(dsr.load(inputDir.getCanonicalPath)) testStream(df)( StartStream(checkpointLocation = checkpointDir.getCanonicalPath), ProcessAllAvailable(), // Initial data (pre + post upgrade + post change schema) all served CheckAnswer((0 until 15).map(i => (i.toString, i.toString)): _*), Execute { _ => // write more data in new schema during streaming writeDeltaData(15 until 20, deltaLog) }, ProcessAllAvailable(), // can still work because the schema is still compatible CheckAnswer((0 until 20).map(i => (i.toString, i.toString)): _*), // stop stream to allow schema change + data update to start in a batch StopStream, // But a new schema change would cause stream to fail // Note here we are restoring back the original schema, see next case for how we test // some extra special cases when schemas are reverted. Execute { _ => writeDeltaData(20 until 25, deltaLog) sql(s"ALTER TABLE delta.`${inputDir.getCanonicalPath}` $schemaRestoreQuery") } ) val df2 = dropCDCFields(dsr.load(inputDir.getCanonicalPath)) testStream(df2)( // Restart stream StartStream(checkpointLocation = checkpointDir.getCanonicalPath), // the last batch should not be processed because the batch cross an incompatible // schema change. ProcessAllAvailableIgnoreError, // no data processed CheckAnswer(Nil: _*), // detected schema change while trying to generate the next offset ExpectStreamStartInCompatibleSchemaFailure ) // Case 2 - Specifically we use startingVersion=0 to simulate serving the entire table's data // in a streaming fashion, ignoring the initialSnapshot. // Here we test the special case when the latest schema is "restored". if (isCdcTest) { val checkpointDir2 = new File(inputDir, "_checkpoint2") val dfStartAtZero = dropCDCFields(dsr .option(DeltaOptions.STARTING_VERSION_OPTION, "0") .load(inputDir.getCanonicalPath)) testStream(dfStartAtZero)( StartStream(checkpointLocation = checkpointDir2.getCanonicalPath), ProcessAllAvailableIgnoreError, ExpectGenericSchemaIncompatibleFailure ) } else { // In the trickier case when we rename a column and rename back, we could not // immediately detect the schema incompatibility at stream start, so we will move on. // This is fine because the batches served will be compatible until the in-stream check // finds another schema change action and fail. val checkpointDir2 = new File(inputDir, s"_checkpoint_${UUID.randomUUID.toString}") val dfStartAtZero = dropCDCFields(dsr .option(DeltaOptions.STARTING_VERSION_OPTION, "0") .load(inputDir.getCanonicalPath)) testStream(dfStartAtZero)( // The stream could not move past version 10, because batches after which // will be incompatible with the latest schema. StartStream(checkpointLocation = checkpointDir2.getCanonicalPath), ProcessAllAvailableIgnoreError, AssertOnQuery { q => val latestCommittedVersion = getLatestCommittedDeltaVersion(q) latestCommittedVersion <= 10 }, ExpectInStreamSchemaChangeFailure ) // restart won't move forward either val df2 = dropCDCFields(dsr.load(inputDir.getCanonicalPath)) checkStreamStartBlocked(df2, checkpointDir2, ExpectInStreamSchemaChangeFailure) } } } test("column mapping + streaming: blocking workflow - " + "should not generate latestOffset past schema change") { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) writeDeltaData(0 until 5, deltaLog, userSpecifiedSchema = Some( new StructType() .add("id", StringType, true) .add("value", StringType, true))) // rename column sql(s"ALTER TABLE delta.`${inputDir.getCanonicalPath}` RENAME COLUMN value TO value2") val renameVersion = deltaLog.update().version // write more data writeDeltaData(5 until 10, deltaLog) // Case 1 - Stream start failure should not progress new latestOffset // Since we had a rename, the data files prior to that should not be served with the renamed // schema , but the original schema . latestOffset() should not create // a new offset moves past the schema change. val df1 = dropCDCFields( dsr.option("startingVersion", "1") // start from 1 to ignore the initial schema change .load(inputDir.getCanonicalPath)) testStream(df1)( StartStream(), // fresh checkpoint ProcessAllAvailableIgnoreError, AssertOnQuery { q => // This should come from the latestOffset checker q.availableOffsets.isEmpty && q.latestOffsets.isEmpty && q.exception.get.cause.getStackTrace.exists(_.toString.contains("latestOffset")) }, ExpectStreamStartInCompatibleSchemaFailure ) // try drop column now sql(s"ALTER TABLE delta.`${inputDir.getCanonicalPath}` DROP COLUMN value2") val dropVersion = deltaLog.update().version // write more data writeDeltaData(10 until 15, deltaLog) val df2 = dropCDCFields( dsr.option("startingVersion", renameVersion + 1) // so we could detect drop column .load(inputDir.getCanonicalPath)) testStream(df2)( StartStream(), // fresh checkpoint ProcessAllAvailableIgnoreError, AssertOnQuery { q => // This should come from the latestOffset stream start checker q.availableOffsets.isEmpty && q.latestOffsets.isEmpty && q.exception.get.cause.getStackTrace.exists(_.toString.contains("latestOffset")) }, ExpectStreamStartInCompatibleSchemaFailure ) // Case 2 - in stream failure should not progress latest offset too // This is the handle prior to SC-111607, which should cover the major cases. def loadDf(): DataFrame = dropCDCFields( dsr.option("startingVersion", dropVersion + 1) // so we could move on to in stream failure .load(inputDir.getCanonicalPath)) val ckpt = Utils.createTempDir().getCanonicalPath var latestAvailableOffsets: Seq[String] = null testStream(loadDf())( StartStream(checkpointLocation = ckpt), // fresh checkpoint ProcessAllAvailable(), CheckAnswer((10 until 15).map(i => (i.toString)): _*), Execute { q => latestAvailableOffsets = q.availableOffsets.values.map(_.json()).toSeq }, // add more data and rename column Execute { _ => sql(s"ALTER TABLE delta.`${inputDir.getCanonicalPath}` RENAME COLUMN id TO id2") writeDeltaData(15 until 16, deltaLog) }, ProcessAllAvailableIgnoreError, CheckAnswer((10 until 15).map(i => (i.toString)): _*), // no data processed AssertOnQuery { q => // Available offsets should not change // This should come from the latestOffset in-stream checker q.availableOffsets.values.map(_.json()) == latestAvailableOffsets && q.latestOffsets.isEmpty && q.exception.get.cause.getStackTrace.exists(_.toString.contains("latestOffset")) }, ExpectInStreamSchemaChangeFailure ) // Case 3 - resuming from existing checkpoint, note that getBatch's stream start check // should be called instead of latestOffset for recovery. // This is also the handle prior to SC-111607, which should cover the major cases. testStream(loadDf())( StartStream(checkpointLocation = ckpt), // existing checkpoint ProcessAllAvailableIgnoreError, CheckAnswer(Nil: _*), AssertOnQuery { q => // This should come from the latestOffset in-stream checker q.availableOffsets.values.map(_.json()) == latestAvailableOffsets && q.latestOffsets.isEmpty && q.exception.get.cause.getStackTrace.exists(_.toString.contains("getBatch")) }, ExpectStreamStartInCompatibleSchemaFailure ) } } test("unsafe flag can unblock drop or rename column") { // upgrade should not blocked both during the stream AND during stream restart withTempDir { inputDir => Seq( s"ALTER TABLE delta.`${inputDir.getCanonicalPath}` DROP COLUMN value", s"ALTER TABLE delta.`${inputDir.getCanonicalPath}` RENAME COLUMN value TO value2" ).foreach { schemaChangeQuery => FileUtils.deleteDirectory(inputDir) val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) withColumnMappingConf("none") { writeDeltaData(0 until 5, deltaLog, Some(StructType.fromDDL("id string, value string"))) } def createNewDf(): DataFrame = dropCDCFields(dsr.load(inputDir.getCanonicalPath)) val checkpointDir = new File(inputDir, s"_checkpoint_${schemaChangeQuery.hashCode}") val isRename = schemaChangeQuery.contains("RENAME") testStream(createNewDf())( StartStream(checkpointLocation = checkpointDir.getCanonicalPath), ProcessAllAvailable(), CheckAnswer((0 until 5).map(i => (i.toString, i.toString)): _*), Execute { _ => sql( s""" |ALTER TABLE delta.`${inputDir.getCanonicalPath}` |SET TBLPROPERTIES ( | ${DeltaConfigs.COLUMN_MAPPING_MODE.key} = "name", | ${DeltaConfigs.MIN_READER_VERSION.key} = "2", | ${DeltaConfigs.MIN_WRITER_VERSION.key} = "5")""".stripMargin) // Add another schema change to ensure even after enable the flag, we would still hit // a schema change with more columns than read schema so `verifySchemaChange` would see // that can complain. sql(s"ALTER TABLE delta.`${inputDir.getCanonicalPath}` ADD COLUMN (random STRING)") sql(schemaChangeQuery) writeDeltaData(5 until 10, deltaLog) }, ProcessAllAvailableIgnoreError, ExistingRetryableInStreamSchemaChangeFailure ) // Without the flag it would still fail testStream(createNewDf())( StartStream(checkpointLocation = checkpointDir.getCanonicalPath), ProcessAllAvailableIgnoreError, CheckAnswer(Nil: _*), ExpectStreamStartInCompatibleSchemaFailure ) val checkExpectedResult = if (isRename) { CheckAnswer((5 until 10).map(i => (i.toString, i.toString, i.toString)): _*) } else { CheckAnswer((5 until 10).map(i => (i.toString, i.toString)): _*) } withSQLConf(DeltaSQLConf .DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_COLUMN_MAPPING_SCHEMA_CHANGES .key -> "true") { testStream(createNewDf())( StartStream(checkpointLocation = checkpointDir.getCanonicalPath), // The processing will pass, ignoring any schema column missing in the backfill. ProcessAllAvailable(), // Show up as dropped column checkExpectedResult, Execute { _ => // But any schema change post the stream analysis would still cause exceptions // as usual, which is critical to avoid data loss. sql(s"ALTER TABLE delta.`${inputDir.getCanonicalPath}` ADD COLUMN (random2 STRING)") }, ProcessAllAvailableIgnoreError, ExistingRetryableInStreamSchemaChangeFailure ) } } } } } trait DeltaSourceColumnMappingSuiteBase extends DeltaColumnMappingSelectedTestMixin { override protected def runOnlyTests = Seq( "basic", "maxBytesPerTrigger: metadata checkpoint", "maxFilesPerTrigger: metadata checkpoint", "allow to change schema before starting a streaming query", // streaming blocking semantics test "deltaLog snapshot should not be updated outside of the stream", "column mapping + streaming - allowed workflows - column addition", "column mapping + streaming - allowed workflows - upgrade to name mode", "column mapping + streaming: blocking workflow - drop column", "column mapping + streaming: blocking workflow - rename column", "column mapping + streaming: blocking workflow - " + "should not generate latestOffset past schema change" ) } class DeltaSourceIdColumnMappingSuite extends DeltaSourceSuite with ColumnMappingStreamingBlockedWorkflowSuiteBase with DeltaColumnMappingEnableIdMode with DeltaSourceColumnMappingSuiteBase { override protected def isCdcTest: Boolean = false } class DeltaSourceNameColumnMappingSuite extends DeltaSourceSuite with ColumnMappingStreamingBlockedWorkflowSuiteBase with DeltaColumnMappingEnableNameMode with DeltaSourceColumnMappingSuiteBase { override protected def isCdcTest: Boolean = false } // Batch sizes 1, 2, and 100 exercise different backfill behaviors in the commit coordinator. // Batch size 1 triggers a backfill on every commit (commitVersion % 1 == 0), testing the most // granular backfill path. Batch size 2 triggers backfill every other commit, testing the boundary // between backfilled and unbackfilled commits. Batch size 100 leaves most commits unbackfilled, // testing the production-like path where streaming must read from both the commit coordinator // and the filesystem. This follows the same pattern as other CatalogManaged (CCv2) test suites // (DeltaLogSuite, DeltaCDCStreamSuite, etc.). class DeltaSourceIdColumnMappingWithCatalogManagedBatch1Suite extends DeltaSourceIdColumnMappingSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class DeltaSourceIdColumnMappingWithCatalogManagedBatch2Suite extends DeltaSourceIdColumnMappingSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class DeltaSourceIdColumnMappingWithCatalogManagedBatch100Suite extends DeltaSourceIdColumnMappingSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } class DeltaSourceNameColumnMappingWithCatalogManagedBatch1Suite extends DeltaSourceNameColumnMappingSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class DeltaSourceNameColumnMappingWithCatalogManagedBatch2Suite extends DeltaSourceNameColumnMappingSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class DeltaSourceNameColumnMappingWithCatalogManagedBatch100Suite extends DeltaSourceNameColumnMappingSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceDeletionVectorsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import scala.util.control.NonFatal import org.apache.spark.sql.delta.Relocated.StreamExecution import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.hadoop.fs.Path import org.scalatest.concurrent.Eventually import org.scalatest.concurrent.PatienceConfiguration.Timeout import org.apache.spark.sql.streaming.{StreamTest, Trigger} import org.apache.spark.sql.streaming.util.StreamManualClock trait DeltaSourceDeletionVectorTests extends StreamTest with DeletionVectorsTestUtils { self: DeltaSourceConnectorTrait => import testImplicits._ /** * Executes a DML SQL statement (DELETE, INSERT, etc.). * Overridable so that V2 suites can route DML through the V1 connector, * since SparkTable (V2) is read-only and does not support writes. */ protected def executeDml(sqlText: String): Unit = sql(sqlText) test("allow to delete files before starting a streaming query") { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 5).foreach { i => val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } executeDml(s"DELETE FROM delta.`$inputDir`") (5 until 10).foreach { i => val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } deltaLog.checkpoint() assert(deltaLog.readLastCheckpointFile().nonEmpty, "this test requires a checkpoint") val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) testStream(df)( AssertOnQuery { q => q.processAllAvailable() true }, CheckAnswer((5 until 10).map(_.toString): _*)) } } test("allow to delete files before staring a streaming query without checkpoint") { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 5).foreach { i => val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } executeDml(s"DELETE FROM delta.`$inputDir`") (5 until 7).foreach { i => val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } assert(deltaLog.readLastCheckpointFile().isEmpty, "this test requires no checkpoint") val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) testStream(df)( AssertOnQuery { q => q.processAllAvailable() true }, CheckAnswer((5 until 7).map(_.toString): _*)) } } /** * If deletion vectors are expected here, return true if they are present. If none are expected, * return true if none are present. */ protected def deletionVectorsPresentIfExpected( inputDir: String, expectDVs: Boolean): Boolean = { val deltaLog = DeltaLog.forTable(spark, inputDir) val filesWithDVs = getFilesWithDeletionVectors(deltaLog) logWarning(s"Expecting DVs=$expectDVs - found ${filesWithDVs.size}") if (expectDVs) { filesWithDVs.nonEmpty } else { filesWithDVs.isEmpty } } private def ignoreOperationsTest( inputDir: String, sourceOptions: Seq[(String, String)], sqlCommand: String, commandShouldProduceDVs: Option[Boolean] = None)(expectations: StreamAction*): Unit = { (0 until 10 by 2).foreach { i => Seq(i, i + 1).toDF().coalesce(1).write.format("delta").mode("append").save(inputDir) } val df = loadStreamWithOptions(inputDir, sourceOptions.toMap) val expectDVs = commandShouldProduceDVs.getOrElse( sqlCommand.toUpperCase().startsWith("DELETE")) val base = Seq( AssertOnQuery { q => q.processAllAvailable() true }, CheckAnswer((0 until 10): _*), AssertOnQuery { q => executeDml(sqlCommand) deletionVectorsPresentIfExpected(inputDir, expectDVs) }) testStream(df)((base ++ expectations): _*) } private def ignoreOperationsTestWithManualClock( inputDir: String, sourceOptions: Seq[(String, String)], sqlCommand1: String, sqlCommand2: String, command1ShouldProduceDVs: Option[Boolean] = None, command2ShouldProduceDVs: Option[Boolean] = None, expectations: List[StreamAction]): Unit = { val clock = new StreamManualClock (0 until 15 by 3).foreach { i => Seq(i, i + 1, i + 2).toDF().coalesce(1).write.format("delta").mode("append").save(inputDir) } val log = DeltaLog.forTable(spark, inputDir) val commitVersionBeforeDML = log.update().version val df = loadStreamWithOptions(inputDir, sourceOptions.toMap) def expectDVsInCommand(shouldProduceDVs: Option[Boolean], command: String): Boolean = { shouldProduceDVs.getOrElse(command.toUpperCase().startsWith("DELETE")) } val expectDVsInCommand1 = expectDVsInCommand(command1ShouldProduceDVs, sqlCommand1) val expectDVsInCommand2 = expectDVsInCommand(command2ShouldProduceDVs, sqlCommand2) // If it's expected to fail we must be sure not to actually process it in here, // or it'll fail too early instead of being caught by ExpectFailure. val shouldFailAfterCommands = expectations.exists(_.isInstanceOf[ExpectFailure[_]]) val baseActions: Seq[StreamAction] = Seq( StartStream(Trigger.ProcessingTime(1000), clock), AdvanceManualClock(1000L), CheckAnswer((0 until 15): _*), AssertOnQuery { q => // Ensure we only processed a single batch since the initial data load. q.commitLog.getLatestBatchId().get == 0 }, AssertOnQuery { q => eventually("Stream never stopped processing") { // Wait until the stream stops processing, so we aren't racing with the next two // commands on whether or not they end up in the same batch. assert(!q.status.isTriggerActive) assert(!q.status.isDataAvailable) } true }, AssertOnQuery { q => executeDml(sqlCommand1) deletionVectorsPresentIfExpected(inputDir, expectDVsInCommand1) }, AssertOnQuery { q => executeDml(sqlCommand2) deletionVectorsPresentIfExpected(inputDir, expectDVsInCommand2) }, AssertOnQuery { q => // Ensure we still didn't process the DML commands. q.commitLog.getLatestBatchId().get == 0 }, // Advance the clock, so that we process the two DML commands. AdvanceManualClock(2000L)) ++ (if (shouldFailAfterCommands) { Seq.empty[StreamAction] } else { Seq( // This makes it move to the next batch. AssertOnQuery { q => eventually("Next batch was never processed") { // Ensure we only processed a single batch with the DML commands. assert(q.commitLog.getLatestBatchId().get === 1) } true }) }) testStream(df)((baseActions ++ expectations): _*) } protected def eventually[T](message: String)(func: => T): T = { try { Eventually.eventually(Timeout(streamingTimeout)) { func } } catch { case NonFatal(e) => fail(message, e) } } testQuietly(s"deleting files fails query if ignoreDeletes = false") { withTempDir { inputDir => ignoreOperationsTest( inputDir.getAbsolutePath, sourceOptions = Nil, sqlCommand = s"DELETE FROM delta.`$inputDir`", // Whole table deletes do not produce DVs. commandShouldProduceDVs = Some(false))(ExpectFailure[DeltaUnsupportedOperationException] { e => for (msg <- Seq("Detected deleted data", "not supported", "ignoreDeletes", "true")) { assert(e.getMessage.contains(msg)) } }) } } Seq("ignoreFileDeletion", DeltaOptions.IGNORE_DELETES_OPTION).foreach { ignoreDeletes => testQuietly( s"allow to delete files after staring a streaming query when $ignoreDeletes is true") { withTempDir { inputDir => ignoreOperationsTest( inputDir.getAbsolutePath, sourceOptions = Seq(ignoreDeletes -> "true"), sqlCommand = s"DELETE FROM delta.`$inputDir`", // Whole table deletes do not produce DVs. commandShouldProduceDVs = Some(false))( AssertOnQuery { q => Seq(10).toDF().write.format("delta").mode("append").save(inputDir.getAbsolutePath) q.processAllAvailable() true }, CheckAnswer((0 to 10): _*)) } } } case class SourceChangeVariant( label: String, query: File => String, answerWithIgnoreChanges: Seq[Int]) val sourceChangeVariants: Seq[SourceChangeVariant] = Seq( // A partial file delete is treated like an update by the Source. SourceChangeVariant( label = "DELETE", query = inputDir => s"DELETE FROM delta.`$inputDir` WHERE value = 3", // 2 occurs in the same file as 3, so it gets duplicated during processing. answerWithIgnoreChanges = (0 to 10) :+ 2)) for (variant <- sourceChangeVariants) testQuietly( "updating the source table causes failure when ignoreChanges = false" + s" - using ${variant.label}") { withTempDir { inputDir => ignoreOperationsTest( inputDir.getAbsolutePath, sourceOptions = Nil, sqlCommand = variant.query(inputDir))( ExpectFailure[DeltaUnsupportedOperationException] { e => for (msg <- Seq("data update", "not supported", "skipChangeCommits", "true")) { assert(e.getMessage.contains(msg)) } }) } } for (variant <- sourceChangeVariants) testQuietly( "allow to update the source table when ignoreChanges = true" + s" - using ${variant.label}") { withTempDir { inputDir => ignoreOperationsTest( inputDir.getAbsolutePath, sourceOptions = Seq(DeltaOptions.IGNORE_CHANGES_OPTION -> "true"), sqlCommand = variant.query(inputDir))( AssertOnQuery { q => Seq(10).toDF().write.format("delta").mode("append").save(inputDir.getAbsolutePath) q.processAllAvailable() true }, CheckAnswer(variant.answerWithIgnoreChanges: _*)) } } testQuietly("deleting files when ignoreChanges = true doesn't fail the query") { withTempDir { inputDir => ignoreOperationsTest( inputDir.getAbsolutePath, sourceOptions = Seq(DeltaOptions.IGNORE_CHANGES_OPTION -> "true"), sqlCommand = s"DELETE FROM delta.`$inputDir`", // Whole table deletes do not produce DVs. commandShouldProduceDVs = Some(false))( AssertOnQuery { q => Seq(10).toDF().write.format("delta").mode("append").save(inputDir.getAbsolutePath) q.processAllAvailable() true }, CheckAnswer((0 to 10): _*)) } } for (variant <- sourceChangeVariants) testQuietly("updating source table when ignoreDeletes = true fails the query" + s" - using ${variant.label}") { withTempDir { inputDir => ignoreOperationsTest( inputDir.getAbsolutePath, sourceOptions = Seq(DeltaOptions.IGNORE_DELETES_OPTION -> "true"), sqlCommand = variant.query(inputDir))( ExpectFailure[DeltaUnsupportedOperationException] { e => for (msg <- Seq("data update", "not supported", "skipChangeCommits", "true")) { assert(e.getMessage.contains(msg)) } }) } } private val allSourceOptions = Seq( Nil, List(DeltaOptions.IGNORE_DELETES_OPTION), List(DeltaOptions.IGNORE_CHANGES_OPTION), List(DeltaOptions.SKIP_CHANGE_COMMITS_OPTION)) .map { options => options.map(key => key -> "true") } for (sourceOption <- allSourceOptions) testQuietly( "subsequent DML commands are processed correctly in a batch - DELETE->DELETE" + s" - $sourceOption") { val expectations: List[StreamAction] = sourceOption.map(_._1) match { case List(DeltaOptions.IGNORE_DELETES_OPTION) | Nil => // These two do not allow updates. ExpectFailure[DeltaUnsupportedOperationException] { e => for (msg <- Seq("data update", "not supported", "skipChangeCommits", "true")) { assert(e.getMessage.contains(msg)) } } :: Nil case List(DeltaOptions.IGNORE_CHANGES_OPTION) => // The 4 and 5 are in the same file as 3, so the first DELETE is going to duplicate them. // 5 is still in the same file as 4 after the first DELETE, so the second DELETE is going // to duplicate it again. CheckAnswer((0 until 15) ++ Seq(4, 5, 5): _*) :: Nil case List(DeltaOptions.SKIP_CHANGE_COMMITS_OPTION) => // This will completely ignore the DELETEs. CheckAnswer((0 until 15): _*) :: Nil } withTempDir { inputDir => ignoreOperationsTestWithManualClock( inputDir.getAbsolutePath, sourceOptions = sourceOption, sqlCommand1 = s"DELETE FROM delta.`$inputDir` WHERE value == 3", sqlCommand2 = s"DELETE FROM delta.`$inputDir` WHERE value == 4", expectations = expectations) } } for (sourceOption <- allSourceOptions) testQuietly("subsequent DML commands are processed correctly in a batch - INSERT->DELETE" + s" - $sourceOption") { val expectations: List[StreamAction] = sourceOption.map(_._1) match { case List(DeltaOptions.IGNORE_DELETES_OPTION) | Nil => // These two do not allow updates. ExpectFailure[DeltaUnsupportedOperationException] { e => for (msg <- Seq("data update", "not supported", "skipChangeCommits", "true")) { assert(e.getMessage.contains(msg)) } } :: Nil case List(DeltaOptions.IGNORE_CHANGES_OPTION) => // 15 and 16 are in the same file, so 16 will get duplicated by the DELETE. CheckAnswer((0 to 16) ++ Seq(16): _*) :: Nil case List(DeltaOptions.SKIP_CHANGE_COMMITS_OPTION) => // This will completely ignore the DELETE. CheckAnswer((0 to 16): _*) :: Nil } withTempDir { inputDir => ignoreOperationsTestWithManualClock( inputDir.getAbsolutePath, sourceOptions = sourceOption, sqlCommand1 = s"INSERT INTO delta.`$inputDir` SELECT /*+ COALESCE(1) */ * FROM VALUES 15, 16", sqlCommand2 = s"DELETE FROM delta.`$inputDir` WHERE value == 15", expectations = expectations) } } test("multiple deletion vectors per file with initial snapshot") { withTempDir { inputDir => val path = inputDir.getAbsolutePath val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) // V0: 10 rows in a single file (0 until 10).toDF("value").coalesce(1).write.format("delta").save(path) // V1: Delete row 0 executeDml(s"DELETE FROM delta.`$path` WHERE value = 0") // V2: Delete row 1 executeDml(s"DELETE FROM delta.`$path` WHERE value = 1") // V3: Delete row 2 executeDml(s"DELETE FROM delta.`$path` WHERE value = 2") // Verify DVs are present assert(getFilesWithDeletionVectors(deltaLog).nonEmpty, "This test requires deletion vectors to be present") val df = loadStreamWithOptions(path, Map.empty) testStream(df)( // Process the initial snapshot AssertOnQuery { q => q.processAllAvailable() true }, CheckAnswer((3 until 10): _*) ) } } private val multiDVSourceOptions = Seq( List(DeltaOptions.IGNORE_FILE_DELETION_OPTION), List(DeltaOptions.IGNORE_CHANGES_OPTION)) .map(options => options.map(key => key -> "true")) for (sourceOptions <- multiDVSourceOptions) test(s"multiple deletion vectors per file - $sourceOptions") { withTempDir { inputDir => val path = inputDir.getAbsolutePath val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) // V0: 10 rows in a single file (0 until 10).toDF("value").coalesce(1).write.format("delta").save(path) val df = loadStreamWithOptions(path, sourceOptions.toMap) testStream(df)( AssertOnQuery { q => q.processAllAvailable() true }, CheckAnswer((0 until 10): _*), AssertOnQuery { q => // V1: Delete row 0 - creates first DV (version 1) executeDml(s"DELETE FROM delta.`$path` WHERE value = 0") true }, AssertOnQuery { q => // V2: Delete row 1 - updates DV (version 2). DV is cumulative: {0, 1} executeDml(s"DELETE FROM delta.`$path` WHERE value = 1") true }, AssertOnQuery { q => // Verify DVs are present assert(getFilesWithDeletionVectors(deltaLog).nonEmpty, "This test requires deletion vectors to be present") true }, AssertOnQuery { q => q.processAllAvailable() true }, // One file is read out 3 times! // This matches the expectation for ignoreChanges & ignoreFileDeletion: // After a data changing operation, unchanged rows are re-emitted. // - v0: rows 0-9 // - v1: file re-added with rows 1-9 (DV excludes 0) // - v2: file re-added with rows 2-9 (DV excludes 0,1) CheckAnswer((0 until 10) ++ (1 until 10) ++ (2 until 10): _*)) } } } class DeltaSourceDeletionVectorsSuite extends DeltaSourceSuiteBase with DeltaSQLCommandTest with DeltaSourceDeletionVectorTests with PersistentDVEnabled ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceFastDropFeatureSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.text.SimpleDateFormat import org.apache.spark.sql.delta.cdc.CDCEnabled import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.FileNames import org.apache.hadoop.fs.Path import org.apache.spark.sql.Row import org.apache.spark.sql.streaming.{DataStreamWriter, StreamingQuery, StreamingQueryException} class DeltaSourceFastDropFeatureSuite extends DeltaSourceSuiteBase with DeltaColumnMappingTestUtils with DeltaSQLCommandTest { import testImplicits._ override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaSQLConf.FAST_DROP_FEATURE_ENABLED.key, "true") } protected def dropUnsupportedFeature(dir: File): Unit = sql( s"""ALTER TABLE delta.`${dir.getCanonicalPath}` |DROP FEATURE ${TestUnsupportedReaderWriterFeature.name} |""".stripMargin) protected def addUnsupportedFeature(dir: File): Unit = sql( s"""ALTER TABLE delta.`${dir.getCanonicalPath}` SET TBLPROPERTIES ( |delta.feature.${TestUnsupportedReaderWriterFeature.name} = 'supported' |)""".stripMargin) protected def getReadOnlyStream( dir: File, cdcReadEnabled: Boolean = false): DataStreamWriter[Row] = spark.readStream .option(DeltaOptions.CDC_READ_OPTION, cdcReadEnabled) .format("delta") .load(dir.getCanonicalPath) .writeStream .format("noop") protected def addData(dir: File, value: Int): Unit = Seq(value).toDF.write.mode("append").format("delta").save(dir.getCanonicalPath) protected lazy val cdcReadEnabled = spark.conf.getOption(DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey) .map(_.toBoolean) .getOrElse(false) test("Latest protocol is checked for unsupported features") { withTempDir { inputDir => addData(inputDir, value = 1) addUnsupportedFeature(inputDir) withSQLConf(DeltaSQLConf.UNSUPPORTED_TESTING_FEATURES_ENABLED.key -> true.toString) { DeltaLog.clearCache() val e = intercept[DeltaUnsupportedTableFeatureException] { getReadOnlyStream(inputDir, cdcReadEnabled).start() } assert(e.getErrorClass === "DELTA_UNSUPPORTED_FEATURES_FOR_READ") } } } for (useStartingTS <- DeltaTestUtils.BOOLEAN_DOMAIN) test(s"Protocol is checked when using startingVersion - useStartingTS: $useStartingTS.") { withTempDir { inputDir => def getTimestampForVersion(version: Long): String = { val logPath = new Path(inputDir.getCanonicalPath, "_delta_log") val file = new File(new Path(logPath, f"$version%020d.json").toString) val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS") sdf.format(file.lastModified()) } addData(inputDir, value = 1) addUnsupportedFeature(inputDir) addData(inputDir, value = 2) // More data. val versionAfterProtocolUpgrade = DeltaLog.forTable(spark, inputDir).update().version dropUnsupportedFeature(inputDir) withSQLConf(DeltaSQLConf.UNSUPPORTED_TESTING_FEATURES_ENABLED.key -> true.toString) { // No problem loading from the latest version. Feature is dropped. DeltaLog.clearCache() getReadOnlyStream(inputDir, cdcReadEnabled).start() // Start a stream to a version the feature was active. val e = intercept[StreamingQueryException] { val stream = spark.readStream .option(DeltaOptions.CDC_READ_OPTION, cdcReadEnabled) .format("delta") if (useStartingTS) { stream.option("startingTimestamp", getTimestampForVersion(versionAfterProtocolUpgrade)) } else { stream.option("startingVersion", versionAfterProtocolUpgrade) } val q = stream .load(inputDir.getCanonicalPath) .writeStream .format("noop") .start() // At initialization get attempt to get a snapshot at the starting version. // This will validate whether the client supports the protocol at that version. // Note, the protocol upgrade happened before the startingVersion. Therefore, // we are certain the exception here does not stem from coming across the protocol // bump while processing the stream. q.processAllAvailable() } assert(e.getCause.getMessage.contains("DELTA_UNSUPPORTED_FEATURES_FOR_READ")) } } } test("Protocol check at startingVersion is skipped when config is disabled") { withTempDir { inputDir => addData(inputDir, value = 1) addUnsupportedFeature(inputDir) addData(inputDir, value = 2) // More data. val versionAfterProtocolUpgrade = DeltaLog.forTable(spark, inputDir).update().version dropUnsupportedFeature(inputDir) withSQLConf( DeltaSQLConf.FAST_DROP_FEATURE_STREAMING_ALWAYS_VALIDATE_PROTOCOL.key -> false.toString, DeltaSQLConf.UNSUPPORTED_TESTING_FEATURES_ENABLED.key -> true.toString) { // Start a stream to a version the feature was active. val q = spark.readStream .option(DeltaOptions.CDC_READ_OPTION, cdcReadEnabled) .format("delta") .option("startingVersion", versionAfterProtocolUpgrade) .load(inputDir.getCanonicalPath) .writeStream .format("noop") .start() try { // Should had produced an exception but the check is disabled. q.processAllAvailable() } finally { q.stop() } } } } test("Protocol is checked when coming across an action with a protocol upgrade") { withTempDir { inputDir => addData(inputDir, value = 1) addData(inputDir, value = 2) // More data. Optional. val versionBeforeProtocolUpgrade = DeltaLog.forTable(spark, inputDir).update().version addUnsupportedFeature(inputDir) dropUnsupportedFeature(inputDir) // Latest version looks clean. Feature is dropped. val stream = spark.readStream .option(DeltaOptions.CDC_READ_OPTION, cdcReadEnabled) .format("delta") .option("startingVersion", versionBeforeProtocolUpgrade) .load(inputDir.getCanonicalPath) .writeStream .format("noop") withSQLConf(DeltaSQLConf.UNSUPPORTED_TESTING_FEATURES_ENABLED.key -> true.toString) { val q = stream.start() val e = intercept[StreamingQueryException] { // We come across the protocol upgrade commit and fail. q.processAllAvailable() } q.stop() assert(e.getCause.getMessage.contains("DELTA_UNSUPPORTED_FEATURES_FOR_READ")) } } } test("Protocol validations after restarting from a checkpoint") { withTempDirs { (inputDir, outputDir, checkpointDir) => addData(inputDir, value = 1) addData(inputDir, value = 2) // More data. Optional. addUnsupportedFeature(inputDir) val stream = spark.readStream .option(DeltaOptions.CDC_READ_OPTION, cdcReadEnabled) .format("delta") .option(DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION, "1") .load(inputDir.getCanonicalPath) .drop(CDCReader.CDC_TYPE_COLUMN_NAME) .drop(CDCReader.CDC_COMMIT_VERSION) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) .writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("delta") val q = stream.start(outputDir.getCanonicalPath) q.processAllAvailable() // Validate progress so far. val progress = q.recentProgress.filter(_.numInputRows != 0) assert(progress.length === 2) progress.foreach { p => assert(p.numInputRows === 1) } checkAnswer( spark.read.format("delta").load(outputDir.getAbsolutePath), (1 until 3).toDF()) q.stop() // More stuff happened since the stream stopped. addData(inputDir, value = 3) // More data. Optional. addData(inputDir, value = 4) // More data. Optional. addData(inputDir, value = 5) // More data. Optional. dropUnsupportedFeature(inputDir) // Query is restarted from checkpoint. Latest protocol looks clean because we dropped the // unsupported feature. Furthermore, the protocol upgrade is before the checkpoint, thus // we cannot come across it while streaming. // The initial state of the stream is null because it was stopped. As a result, the client // attempts to create a snapshot at the checkpoint version. This version contains the // unsupported feature and fails. withSQLConf(DeltaSQLConf.UNSUPPORTED_TESTING_FEATURES_ENABLED.key -> true.toString) { DeltaLog.clearCache() val q2 = stream.start(outputDir.getCanonicalPath) val e = intercept[StreamingQueryException] { // We come across the protocol upgrade commit and fail. q2.processAllAvailable() } assert(e.getCause.getMessage.contains("DELTA_UNSUPPORTED_FEATURES_FOR_READ")) q2.stop() } } } test("Protocol validations supress errors when snapshot cannot be reconstructed") { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, inputDir) // Add some data. addData(inputDir, value = 0) // Version 0. addData(inputDir, value = 1) // Version 1. addData(inputDir, value = 2) // Version 2. addData(inputDir, value = 3) // Version 3. deltaLog.checkpoint(deltaLog.update()) // Version 3. addData(inputDir, value = 4) // Version 4. // Delete version 1. new File(FileNames.unsafeDeltaFile(deltaLog.logPath, 1).toUri).delete() withSQLConf( DeltaSQLConf.FAST_DROP_FEATURE_STREAMING_ALWAYS_VALIDATE_PROTOCOL.key -> "true") { DeltaLog.clearCache() val q = spark.readStream .option(DeltaOptions.CDC_READ_OPTION, cdcReadEnabled) .format("delta") // Starting version exists but we cannot reconstruct a snapshot because version 1 // is missing. .option("startingVersion", 2) .load(inputDir.getCanonicalPath) .writeStream .format("noop") .start() try { if (cdcReadEnabled) { // With CDC enabled, this scenario always produces an exception. In that sense, // CDC is more restrictive. This exception is produced in changesToDF when trying // to construct a snapshot at the starting version. This is existing // behaviour. assert(intercept[StreamingQueryException] { q.processAllAvailable() }.getCause.getMessage.contains("DELTA_VERSIONS_NOT_CONTIGUOUS")) } else { q.processAllAvailable() } } finally { q.stop() } } } } } class DeltaSourceFastDropFeatureCDCSuite extends DeltaSourceFastDropFeatureSuite with CDCEnabled { override protected def excluded: Seq[String] = super.excluded ++ Seq( // Excluded because in CDC streaming the current behaviour is to always check the protocol at // the starting version. "Protocol check at startingVersion is skipped when config is disabled") } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceLargeLogSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.SparkConf class DeltaSourceLargeLogSuite extends DeltaSourceSuite { protected override def sparkConf = { super.sparkConf.set(DeltaSQLConf.LOG_SIZE_IN_MEMORY_THRESHOLD.key, "0") } } // Batch sizes 1, 2, and 100 exercise different backfill behaviors in the commit coordinator. // Batch size 1 triggers a backfill on every commit (commitVersion % 1 == 0), testing the most // granular backfill path. Batch size 2 triggers backfill every other commit, testing the boundary // between backfilled and unbackfilled commits. Batch size 100 leaves most commits unbackfilled, // testing the production-like path where streaming must read from both the commit coordinator // and the filesystem. This follows the same pattern as other CatalogManaged (CCv2) test suites // (DeltaLogSuite, DeltaCDCStreamSuite, etc.). class DeltaSourceLargeLogWithCatalogManagedBatch1Suite extends DeltaSourceLargeLogSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class DeltaSourceLargeLogWithCatalogManagedBatch2Suite extends DeltaSourceLargeLogSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class DeltaSourceLargeLogWithCatalogManagedBatch100Suite extends DeltaSourceLargeLogSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceOffsetSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.UUID import org.apache.spark.sql.delta.sources.DeltaSourceOffset import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.SparkFunSuite import org.apache.spark.SparkThrowable import org.apache.spark.sql.execution.streaming.SerializedOffset class DeltaSourceOffsetSuite extends SparkFunSuite { test("unknown sourceVersion value") { // Set unknown sourceVersion as the max allowed version plus 1. val unknownVersion = 4 // Note: "isStartingVersion" corresponds to DeltaSourceOffset.isInitialSnapshot. val json = s""" |{ | "sourceVersion": $unknownVersion, | "reservoirVersion": 1, | "index": 1, | "isStartingVersion": true |} """.stripMargin val e = intercept[SparkThrowable] { DeltaSourceOffset( UUID.randomUUID().toString, SerializedOffset(json)) } assert(e.getErrorClass == "DELTA_INVALID_FORMAT_FROM_SOURCE_VERSION") assert(e.toString.contains("Please upgrade to newer version of Delta")) } test("invalid sourceVersion value") { // Note: "isStartingVersion" corresponds to DeltaSourceOffset.isInitialSnapshot. val json = """ |{ | "sourceVersion": "foo", | "reservoirVersion": 1, | "index": 1, | "isStartingVersion": true |} """.stripMargin val e = intercept[SparkThrowable] { DeltaSourceOffset( UUID.randomUUID().toString, SerializedOffset(json)) } assert(e.getErrorClass == "DELTA_INVALID_SOURCE_OFFSET_FORMAT") assert(e.toString.contains("source offset format is invalid")) } test("missing sourceVersion") { // Note: "isStartingVersion" corresponds to DeltaSourceOffset.isInitialSnapshot. val json = """ |{ | "reservoirVersion": 1, | "index": 1, | "isStartingVersion": true |} """.stripMargin val e = intercept[SparkThrowable] { DeltaSourceOffset( UUID.randomUUID().toString, SerializedOffset(json)) } assert(e.getErrorClass == "DELTA_INVALID_SOURCE_VERSION") for (msg <- "is invalid") { assert(e.toString.contains(msg)) } } test("unmatched reservoir id") { // Note: "isStartingVersion" corresponds to DeltaSourceOffset.isInitialSnapshot. val json = s""" |{ | "reservoirId": "${UUID.randomUUID().toString}", | "sourceVersion": 1, | "reservoirVersion": 1, | "index": 1, | "isStartingVersion": true |} """.stripMargin val e = intercept[SparkThrowable] { DeltaSourceOffset( UUID.randomUUID().toString, SerializedOffset(json)) } assert(e.getErrorClass == "DIFFERENT_DELTA_TABLE_READ_BY_STREAMING_SOURCE") for (msg <- Seq("delete", "checkpoint", "restart")) { assert(e.toString.contains(msg)) } } test("isInitialSnapshot serializes as isStartingVersion") { for (isStartingVersion <- Seq(false, true)) { // From serialized to object val reservoirId = UUID.randomUUID().toString val json = s""" |{ | "reservoirId": "$reservoirId", | "sourceVersion": 1, | "reservoirVersion": 1, | "index": 1, | "isStartingVersion": $isStartingVersion |} """.stripMargin val offsetDeserialized = DeltaSourceOffset(reservoirId, SerializedOffset(json)) assert(offsetDeserialized.isInitialSnapshot === isStartingVersion) // From object to serialized val offset = DeltaSourceOffset( reservoirId = reservoirId, reservoirVersion = 7, index = 13, isInitialSnapshot = isStartingVersion) assert(offset.json.contains(s""""isStartingVersion":$isStartingVersion""")) } } test("DeltaSourceOffset deserialization") { // Source version 1 with BASE_INDEX_V1 val reservoirId = UUID.randomUUID().toString val jsonV1 = s""" |{ | "reservoirId": "$reservoirId", | "sourceVersion": 1, | "reservoirVersion": 3, | "index": -1, | "isStartingVersion": false |} """.stripMargin val offsetDeserializedV1 = JsonUtils.fromJson[DeltaSourceOffset](jsonV1) assert(offsetDeserializedV1 == DeltaSourceOffset(reservoirId, 3, DeltaSourceOffset.BASE_INDEX, false)) // Source version 3 with BASE_INDEX_V3 val jsonV3 = s""" |{ | "reservoirId": "$reservoirId", | "sourceVersion": 3, | "reservoirVersion": 7, | "index": -100, | "isStartingVersion": false |} """.stripMargin val offsetDeserializedV3 = JsonUtils.fromJson[DeltaSourceOffset](jsonV3) assert(offsetDeserializedV3 == DeltaSourceOffset(reservoirId, 7, DeltaSourceOffset.BASE_INDEX, false)) // Source version 3 with METADATA_CHANGE_INDEX val jsonV3metadataChange = s""" |{ | "reservoirId": "$reservoirId", | "sourceVersion": 3, | "reservoirVersion": 7, | "index": -20, | "isStartingVersion": false |} """.stripMargin val offsetDeserializedV3metadataChange = JsonUtils.fromJson[DeltaSourceOffset](jsonV3metadataChange) assert(offsetDeserializedV3metadataChange == DeltaSourceOffset(reservoirId, 7, DeltaSourceOffset.METADATA_CHANGE_INDEX, false)) // Source version 3 with regular index and isStartingVersion = true val jsonV3start = s""" |{ | "reservoirId": "$reservoirId", | "sourceVersion": 3, | "reservoirVersion": 9, | "index": 23, | "isStartingVersion": true |} """.stripMargin val offsetDeserializedV3start = JsonUtils.fromJson[DeltaSourceOffset](jsonV3start) assert(offsetDeserializedV3start == DeltaSourceOffset(reservoirId, 9, 23, true)) } test("DeltaSourceOffset deserialization error") { val reservoirId = UUID.randomUUID().toString // This is missing a double quote so it's unbalanced. val jsonV1 = s""" |{ | "reservoirId": "$reservoirId", | "sourceVersion": 23x, | "reservoirVersion": 3, | "index": -1, | "isStartingVersion": false |} """.stripMargin val e = intercept[SparkThrowable] { JsonUtils.fromJson[DeltaSourceOffset](jsonV1) } assert(e.getErrorClass == "DELTA_INVALID_SOURCE_OFFSET_FORMAT") } test("DeltaSourceOffset serialization") { val reservoirId = UUID.randomUUID().toString // BASE_INDEX is always serialized as V1. val offsetV1 = DeltaSourceOffset(reservoirId, 3, DeltaSourceOffset.BASE_INDEX, false) assert(JsonUtils.toJson(offsetV1) === s"""{"sourceVersion":1,"reservoirId":"$reservoirId","reservoirVersion":3,"index":-1,""" + s""""isStartingVersion":false}""") // The same serializer should be used by both methods. assert(JsonUtils.toJson(offsetV1) === offsetV1.json) // METADATA_CHANGE_INDEX is always serialized as V3 val offsetV3metadataChange = DeltaSourceOffset(reservoirId, 7, DeltaSourceOffset.METADATA_CHANGE_INDEX, false) assert(JsonUtils.toJson(offsetV3metadataChange) === s"""{"sourceVersion":3,"reservoirId":"$reservoirId","reservoirVersion":7,"index":-20,""" + s""""isStartingVersion":false}""") // The same serializer should be used by both methods. assert(JsonUtils.toJson(offsetV3metadataChange) === offsetV3metadataChange.json) // Regular index and isStartingVersion = true, serialized as V1 val offsetV1start = DeltaSourceOffset(reservoirId, 9, 23, true) assert(JsonUtils.toJson(offsetV1start) === s"""{"sourceVersion":1,"reservoirId":"$reservoirId","reservoirVersion":9,"index":23,""" + s""""isStartingVersion":true}""") // The same serializer should be used by both methods. assert(JsonUtils.toJson(offsetV1start) === offsetV1start.json) } test("DeltaSourceOffset.validateOffsets") { DeltaSourceOffset.validateOffsets( previousOffset = DeltaSourceOffset( reservoirId = "foo", reservoirVersion = 4, index = 10, isInitialSnapshot = false), currentOffset = DeltaSourceOffset( reservoirId = "foo", reservoirVersion = 4, index = 10, isInitialSnapshot = false)) DeltaSourceOffset.validateOffsets( previousOffset = DeltaSourceOffset( reservoirId = "foo", reservoirVersion = 4, index = 10, isInitialSnapshot = false), currentOffset = DeltaSourceOffset( reservoirId = "foo", reservoirVersion = 5, index = 1, isInitialSnapshot = false)) assert(intercept[IllegalStateException] { DeltaSourceOffset.validateOffsets( previousOffset = DeltaSourceOffset( reservoirId = "foo", reservoirVersion = 4, index = 10, isInitialSnapshot = false), currentOffset = DeltaSourceOffset( reservoirId = "foo", reservoirVersion = 4, index = 10, isInitialSnapshot = true)) }.getMessage.contains("Found invalid offsets: 'isInitialSnapshot' flipped incorrectly.")) assert(intercept[IllegalStateException] { DeltaSourceOffset.validateOffsets( previousOffset = DeltaSourceOffset( reservoirId = "foo", reservoirVersion = 4, index = 10, isInitialSnapshot = false), currentOffset = DeltaSourceOffset( reservoirId = "foo", reservoirVersion = 1, index = 10, isInitialSnapshot = false)) }.getMessage.contains("Found invalid offsets: 'reservoirVersion' moved back.")) assert(intercept[IllegalStateException] { DeltaSourceOffset.validateOffsets( previousOffset = DeltaSourceOffset( reservoirId = "foo", reservoirVersion = 4, index = 10, isInitialSnapshot = false), currentOffset = DeltaSourceOffset( reservoirId = "foo", reservoirVersion = 4, index = 9, isInitialSnapshot = false)) }.getMessage.contains("Found invalid offsets. 'index' moved back.")) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSchemaEvolutionSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.nio.charset.Charset import scala.collection.JavaConverters._ import scala.util.Try import org.apache.spark.sql.delta.actions.{Metadata, Protocol} import org.apache.spark.sql.delta.sources._ import org.apache.spark.sql.delta.test.{DeltaColumnMappingSelectedTestMixin, DeltaSQLCommandTest} import org.apache.spark.sql.delta.util.JsonUtils import org.apache.commons.io.FileUtils import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.hadoop.fs.Path import org.apache.logging.log4j.Level import org.apache.spark.SparkConf import org.apache.spark.sql.{DataFrame, Row} import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} import org.apache.spark.sql.execution.streaming.Offset import org.apache.spark.sql.functions.lit import org.apache.spark.sql.streaming.{StreamingQueryException, Trigger} import org.apache.spark.sql.types.{StringType, StructType} import org.apache.spark.util.Utils trait StreamingSchemaEvolutionSuiteBase extends ColumnMappingStreamingTestUtils with DeltaSourceSuiteBase with DeltaColumnMappingSelectedTestMixin with DeltaSQLCommandTest { override protected def runOnlyTests: Seq[String] = Seq( "schema log initialization with additive schema changes", "detect incompatible schema change while streaming", "trigger.Once with deferred commit should work", "trigger.AvailableNow should work", "consecutive schema evolutions", "latestOffset should not progress before schema evolved" ) override protected def sparkConf: SparkConf = { val conf = super.sparkConf // Enable for testing conf.set(DeltaSQLConf.DELTA_STREAMING_ENABLE_SCHEMA_TRACKING.key, "true") conf.set( DeltaSQLConf.DELTA_STREAMING_ENABLE_SCHEMA_TRACKING_MERGE_CONSECUTIVE_CHANGES.key, "true") conf.set( s"${DeltaSQLConf.SQL_CONF_PREFIX}.streaming.allowSourceColumnRenameAndDrop", "always") if (isCdcTest) { conf.set(DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey, "true") } else { conf } } protected def withoutAllowStreamRestart(f: => Unit): Unit = { withSQLConf(s"${DeltaSQLConf.SQL_CONF_PREFIX}.streaming" + s".allowSourceColumnRenameAndDrop" -> "false") { f } } protected def testWithoutAllowStreamRestart(testName: String)(f: => Unit): Unit = { test(testName) { withoutAllowStreamRestart(f) } } import testImplicits._ protected val ExpectSchemaLogInitializationFailedException = ExpectFailure[DeltaRuntimeException](e => assert( e.asInstanceOf[DeltaRuntimeException].getErrorClass == "DELTA_STREAMING_SCHEMA_LOG_INIT_FAILED_INCOMPATIBLE_METADATA" && // Does NOT come from the stream start check which is for lazy initialization ... !e.getStackTrace.exists( _.toString.contains("checkReadIncompatibleSchemaChangeOnStreamStartOnce")) && // Coming from the check against constructed batches e.getStackTrace.exists( _.toString.contains("validateAndInitMetadataLogForPlannedBatchesDuringStreamStart")) ) ) protected val ExpectMetadataEvolutionException = ExpectFailure[DeltaRuntimeException](e => assert( e.asInstanceOf[DeltaRuntimeException].getErrorClass == "DELTA_STREAMING_METADATA_EVOLUTION" && e.getStackTrace.exists( _.toString.contains("updateMetadataTrackingLogAndFailTheStreamIfNeeded")) ) ) protected val ExpectMetadataEvolutionExceptionFromInitialization = ExpectFailure[DeltaRuntimeException](e => assert( e.asInstanceOf[DeltaRuntimeException].getErrorClass == "DELTA_STREAMING_METADATA_EVOLUTION" && !e.getStackTrace.exists(_.toString.contains("checkReadIncompatibleSchemaChanges")) && e.getStackTrace.exists(_.toString.contains("initializeMetadataTrackingAndExitStream")) ) ) protected val indexWhenSchemaLogIsUpdated = DeltaSourceOffset.POST_METADATA_CHANGE_INDEX protected val AwaitTermination = AssertOnQuery { q => q.awaitTermination(600 * 1000) // 600 seconds true } protected val AwaitTerminationIgnoreError = AssertOnQuery { q => try { q.awaitTermination(600 * 1000) // 600 seconds } catch { case _: Throwable => // ignore } true } protected def allowSchemaLocationOutsideCheckpoint(f: => Unit): Unit = { val allowSchemaLocationOutSideCheckpointConf = DeltaSQLConf.DELTA_STREAMING_ALLOW_SCHEMA_LOCATION_OUTSIDE_CHECKPOINT_LOCATION.key withSQLConf(allowSchemaLocationOutSideCheckpointConf -> "true") { f } } protected def testSchemaEvolution( testName: String, columnMapping: Boolean = true, tags: Seq[org.scalatest.Tag] = Seq.empty)(f: DeltaLog => Unit): Unit = { super.test(testName, tags: _*) { if (columnMapping) { withStarterTable { log => f(log) } } else { withColumnMappingConf("none") { withStarterTable { log => f(log) } } } } } /** * Initialize a starter table with 6 rows and schema STRUCT */ protected def withStarterTable(f: DeltaLog => Unit): Unit = { withTempDir { dir => val tablePath = dir.getCanonicalPath // Write 6 versions, the first version 0 will contain data -1 and will come with the default // schema initialization actions. (-1 until 5).foreach { i => Seq((i.toString, i.toString)).toDF("a", "b") .write.mode("append").format("delta") .save(tablePath) } val deltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) deltaLog.update() f(deltaLog) } } protected def addData( data: Seq[Int], userSpecifiedSchema: Option[StructType] = None)(implicit log: DeltaLog): Unit = { val schema = userSpecifiedSchema.getOrElse(log.update().schema) data.foreach { i => val data = Seq(Row(schema.map(_ => i.toString): _*)) spark.createDataFrame(data.asJava, schema) .write.format("delta").mode("append").save(log.dataPath.toString) } } protected def readStream( schemaLocation: Option[String] = None, sourceTrackingId: Option[String] = None, startingVersion: Option[Long] = None, maxFilesPerTrigger: Option[Int] = None, ignoreDeletes: Option[Boolean] = None)(implicit log: DeltaLog): DataFrame = { var dsr = spark.readStream.format("delta") if (isCdcTest) { dsr = dsr.option(DeltaOptions.CDC_READ_OPTION, "true") } schemaLocation.foreach { loc => dsr = dsr.option(DeltaOptions.SCHEMA_TRACKING_LOCATION, loc) } sourceTrackingId.foreach { name => dsr = dsr.option(DeltaOptions.STREAMING_SOURCE_TRACKING_ID, name) } startingVersion.foreach { v => dsr = dsr.option("startingVersion", v) } maxFilesPerTrigger.foreach { f => dsr = dsr.option("maxFilesPerTrigger", f) } ignoreDeletes.foreach{ i => dsr.option("ignoreDeletes", i) } val df = { dsr.load(log.dataPath.toString) } if (isCdcTest) { dropCDCFields(df) } else { df } } protected def getDefaultSchemaLog( sourceTrackingId: Option[String] = None, initializeEagerly: Boolean = true )(implicit log: DeltaLog): DeltaSourceMetadataTrackingLog = DeltaSourceMetadataTrackingLog.create( spark, getDefaultSchemaLocation.toString, log.update(), catalogTableOpt = None, parameters = sourceTrackingId.map(DeltaOptions.STREAMING_SOURCE_TRACKING_ID -> _).toMap, initMetadataLogEagerly = initializeEagerly) protected def getDefaultCheckpoint(implicit log: DeltaLog): Path = new Path(log.dataPath, "_checkpoint") protected def getDefaultSchemaLocation(implicit log: DeltaLog): Path = new Path(getDefaultCheckpoint, "_schema_location") protected def addColumn(column: String, dt: String = "STRING")(implicit log: DeltaLog): Unit = { sql(s"ALTER TABLE delta.`${log.dataPath}` ADD COLUMN ($column $dt)") } protected def renameColumn(oldColumn: String, newColumn: String)(implicit log: DeltaLog): Unit = { sql(s"ALTER TABLE delta.`${log.dataPath}` RENAME COLUMN $oldColumn TO $newColumn") } protected def dropColumn(column: String)(implicit log: DeltaLog): Unit = { sql(s"ALTER TABLE delta.`${log.dataPath}` DROP COLUMN $column") } protected def overwriteSchema( schema: StructType, partitionColumns: Seq[String] = Nil)(implicit log: DeltaLog): Unit = { spark.sqlContext.internalCreateDataFrame(spark.sparkContext.emptyRDD[InternalRow], schema) .write.format("delta") .mode("overwrite") .partitionBy(partitionColumns: _*) .option("overwriteSchema", "true") .save(log.dataPath.toString) } protected def upgradeToNameMode(implicit log: DeltaLog): Unit = { sql( s"""ALTER TABLE delta.`${log.dataPath}` SET TBLPROPERTIES ( |'delta.columnMapping.mode' = "name", |'delta.minReaderVersion' = '2', |'delta.minWriterVersion' = '5' |) |""".stripMargin) } protected def makeMetadata( schema: StructType, partitionSchema: StructType)(implicit log: DeltaLog): Metadata = { log.update().metadata.copy( schemaString = schema.json, partitionColumns = partitionSchema.fieldNames ) } protected def testSchemasLocationMustBeUnderCheckpoint( checkpointLocation: String, schemaLocation: String, expectValid: Boolean, verify: DeltaAnalysisException => Boolean = _ => true)(implicit log: DeltaLog): Unit = { val dest = Utils.createTempDir().getCanonicalPath if (!expectValid) { // By default it should fail val e = intercept[DeltaAnalysisException] { readStream(schemaLocation = Some(schemaLocation)) .writeStream.option("checkpointLocation", checkpointLocation).start(dest) } assert(e.getErrorClass == "DELTA_STREAMING_SCHEMA_LOCATION_NOT_UNDER_CHECKPOINT") assert(verify(e)) // But can be lifted with the flag allowSchemaLocationOutsideCheckpoint { testStream(readStream(schemaLocation = Some(schemaLocation)))( StartStream(checkpointLocation = checkpointLocation), ProcessAllAvailable(), CheckAnswer((-1 until 5).map(i => (i.toString, i.toString)): _*) ) } } else { // Should just work testStream(readStream(schemaLocation = Some(schemaLocation)))( StartStream(checkpointLocation = checkpointLocation), ProcessAllAvailable(), CheckAnswer((-1 until 5).map(i => (i.toString, i.toString)): _*) ) } } testSchemaEvolution("schema location not under checkpoint") { implicit log => testSchemasLocationMustBeUnderCheckpoint( getDefaultCheckpoint.toString, Utils.createTempDir().getCanonicalPath, expectValid = false, verify = e => { val Array(schemaLocation, checkpointLocation) = e.getMessageParametersArray // Make sure paths with interchangeable schemes are handled schemaLocation.startsWith("/") && checkpointLocation.startsWith("file:") } ) } testSchemaEvolution("schema location same as checkpoint") { implicit log => testSchemasLocationMustBeUnderCheckpoint( getDefaultCheckpoint.toString, getDefaultCheckpoint.toString, expectValid = true ) } testSchemaEvolution("schema location using a different file system") { implicit log => withSQLConf( "fs.s3.impl" -> classOf[S3LikeLocalFileSystem].getCanonicalName, "fs.s3.impl.disable.cache" -> "true") { testSchemasLocationMustBeUnderCheckpoint( getDefaultCheckpoint.toString, s"s3:${Utils.createTempDir().getCanonicalPath}", expectValid = false ) } } private case class SchemaLocationUnderCheckpointUnitTest( checkpointLocation: String, schemaLocation: String, expectValid: Boolean, sqlConfs: Map[String, String] = Map.empty) private val schemaLocationUnderCheckpointUnitTests = Map( "checkpoint location and schema location are the same" -> { val path = Utils.createTempDir().getCanonicalPath SchemaLocationUnderCheckpointUnitTest(path, path, expectValid = true) }, "schema location is under checkpoint location" -> { val checkpoint = Utils.createTempDir().getCanonicalPath val schema = new File(checkpoint, "schema").getCanonicalPath // Also test that file:/ scheme is treated the same. SchemaLocationUnderCheckpointUnitTest(checkpoint, s"file:$schema", expectValid = true) }, "schema location is not under checkpoint location" -> { val checkpoint = Utils.createTempDir().getCanonicalPath val schema = Utils.createTempDir().getCanonicalPath SchemaLocationUnderCheckpointUnitTest( checkpoint, schema, expectValid = false ) }, "schema location and checkpoint location are on different file systems" -> { val checkpoint = Utils.createTempDir().getCanonicalPath val schema = s"s3:${Utils.createTempDir().getCanonicalPath}" SchemaLocationUnderCheckpointUnitTest( checkpoint, schema, sqlConfs = Map( "fs.s3.impl" -> classOf[S3LikeLocalFileSystem].getCanonicalName, "fs.s3.impl.disable.cache" -> "true" ), expectValid = false) }, "schema location and checkpoint location are the same but with explicit file scheme" -> { val path = Utils.createTempDir().getCanonicalPath SchemaLocationUnderCheckpointUnitTest( path, s"file:$path", expectValid = true ) }, "special characters in schema location" -> { val checkpoint = Utils.createTempDir().getCanonicalPath val schema = s"$checkpoint/a % ^ * _ b" SchemaLocationUnderCheckpointUnitTest( checkpoint, schema, expectValid = true ) } ) schemaLocationUnderCheckpointUnitTests.foreach { case (testName, testCase) => test(s"schema / checkpoint location unit tests - $testName") { val analysis = new DeltaAnalysis(spark) withSQLConf(testCase.sqlConfs.toSeq: _*) { val resultTry = Try( analysis.assertSchemaTrackingLocationUnderCheckpoint( testCase.checkpointLocation, testCase.schemaLocation ) ) if (testCase.expectValid) { assert(resultTry.isSuccess) } else { assert(resultTry.isFailure) val e = resultTry.failed.get logInfo("Expected exception", e) assert(e.isInstanceOf[DeltaAnalysisException]) checkError( e.asInstanceOf[DeltaAnalysisException], "DELTA_STREAMING_SCHEMA_LOCATION_NOT_UNDER_CHECKPOINT", "22000", Map( "schemaTrackingLocation" -> testCase.schemaLocation, "checkpointLocation" -> testCase.checkpointLocation ) ) } } } } testSchemaEvolution("multiple delta source sharing same schema log is blocked") { implicit log => allowSchemaLocationOutsideCheckpoint { val dest = Utils.createTempDir().getCanonicalPath val ckpt = getDefaultCheckpoint.toString val schemaLocation = getDefaultSchemaLocation.toString // Two INSTANCES of Delta sources sharing same schema location should be blocked val df1 = readStream(schemaLocation = Some(schemaLocation)) val df2 = readStream(schemaLocation = Some(schemaLocation)) val sdf = df1 union df2 val e = intercept[DeltaAnalysisException] { sdf.writeStream.option("checkpointLocation", ckpt).start(dest) } assert(e.getErrorClass == "DELTA_STREAMING_SCHEMA_LOCATION_CONFLICT") // But providing an additional source name can differentiate val df3 = readStream(schemaLocation = Some(schemaLocation), sourceTrackingId = Some("a")) val df4 = readStream(schemaLocation = Some(schemaLocation), sourceTrackingId = Some("b")) val sdf2 = df3 union df4 testStream(sdf2)( StartStream(checkpointLocation = ckpt), ProcessAllAvailable(), CheckAnswer(((-1 until 5) union (-1 until 5)).map(i => (i.toString, i.toString)): _*) ) // But if they are the same instance it should not be blocked, because they will be // unified to the same source during execution. val sdf3 = df1 union df1 testStream(sdf3)( StartStream(checkpointLocation = ckpt), ProcessAllAvailable(), AssertOnQuery { q => // Just one source being executed q.committedOffsets.size == 1 } ) } } // Disable column mapping for this test so we could save some schema metadata manipulation hassle testSchemaEvolution("schema log is applied", columnMapping = false) { implicit log => withSQLConf( DeltaSQLConf.DELTA_STREAMING_SCHEMA_TRACKING_METADATA_PATH_CHECK_ENABLED.key -> "false") { // Schema log's schema is respected val schemaLog = getDefaultSchemaLog() val newSchema = PersistedMetadata(log.unsafeVolatileTableId, 0, makeMetadata( new StructType().add("a", StringType, true) .add("b", StringType, true) .add("c", StringType, true), partitionSchema = new StructType() ), log.update().protocol, sourceMetadataPath = "" ) schemaLog.writeNewMetadata(newSchema) testStream( readStream(schemaLocation = Some(getDefaultSchemaLocation.toString), // Ignore initial snapshot startingVersion = Some(1L)))( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailable(), // See how the schema returns one more dimension for `c` CheckAnswer((0 until 5).map(_.toString).map(i => (i, i, null)): _*) ) // Cannot use schema from another table val newSchemaWithTableId = PersistedMetadata( "some_random_id", 0, makeMetadata( new StructType().add("a", StringType, true) .add("b", StringType, true), partitionSchema = new StructType() ), log.update().protocol, sourceMetadataPath = "" ) schemaLog.writeNewMetadata(newSchemaWithTableId) assert { val e = intercept[DeltaAnalysisException] { val q = readStream( schemaLocation = Some(getDefaultSchemaLocation.toString), // Ignore initial snapshot startingVersion = Some(1L)) .writeStream .option("checkpointLocation", getDefaultCheckpoint.toString) .outputMode("append") .format("console") .start() q.processAllAvailable() q.stop() } ExceptionUtils.getRootCause(e).asInstanceOf[DeltaAnalysisException] .getErrorClass == "DELTA_STREAMING_SCHEMA_LOG_INCOMPATIBLE_DELTA_TABLE_ID" } } } test("concurrent schema log modification should be detected") { withStarterTable { implicit log => // Note: this test assumes schema log files are written one after another, which is majority // of the case; True concurrent execution would require commit service to protected against. val schemaLocation = getDefaultSchemaLocation.toString val snapshot = log.update() val schemaLog1 = DeltaSourceMetadataTrackingLog.create( spark, schemaLocation, snapshot, catalogTableOpt = None, parameters = Map.empty) val schemaLog2 = DeltaSourceMetadataTrackingLog.create( spark, schemaLocation, snapshot, catalogTableOpt = None, Map.empty) val newSchema = PersistedMetadata("1", 1, makeMetadata(new StructType(), partitionSchema = new StructType()), Protocol(), sourceMetadataPath = "") schemaLog1.writeNewMetadata(newSchema) val e = intercept[DeltaAnalysisException] { schemaLog2.writeNewMetadata(newSchema) } assert(e.getErrorClass == "DELTA_STREAMING_SCHEMA_LOCATION_CONFLICT") } } /** * Manually create a new offset with targeted reservoirVersion by copying it from the previous * offset. * @param checkpoint Checkpoint location * @param version Target version * @param index Target index fle. * @return The raw content for the updated offset file */ protected def manuallyCreateLatestStreamingOffsetUntilReservoirVersion( checkpoint: String, version: Long, index: Long = DeltaSourceOffset.BASE_INDEX): String = { // manually create another offset to latest version val offsetDir = new File(checkpoint.stripPrefix("file:") + "/offsets") val previousOffset = offsetDir.listFiles().filter(!_.getName.endsWith(".crc")) .maxBy(_.getName.toInt) val previousOffsetContent = FileUtils .readFileToString(previousOffset, Charset.defaultCharset()) val reservoirVersionRegex = """"reservoirVersion":[0-9]+""".r val indexRegex = """"index":-?\d+""".r var updated = reservoirVersionRegex .replaceAllIn(previousOffsetContent, s""""reservoirVersion":$version""") updated = indexRegex.replaceAllIn(updated, s""""index":$index""") val newOffsetFile = new File(previousOffset.getParent, (previousOffset.getName.toInt + 1).toString) FileUtils.writeStringToFile(newOffsetFile, updated, Charset.defaultCharset()) updated } /** * Write serialized offset content as a batch id for a particular checkpoint. * @param checkpoint Checkpoint location * @param batchId Target batch ID to write to * @param offsetContent Offset content */ protected def manuallyCreateStreamingOffsetAtBatchId( checkpoint: String, batchId: Long, offsetContent: String): Unit = { // manually create another offset to latest version val offsetDir = new File(checkpoint.stripPrefix("file:") + "/offsets") val newOffsetFile = new File(offsetDir, batchId.toString) FileUtils.writeStringToFile(newOffsetFile, offsetContent, Charset.defaultCharset()) } /** * Manually delete the latest offset * @param checkpoint Checkpoint location */ protected def manuallyDeleteLatestBatchId(checkpoint: String): Unit = { // manually create another offset to latest version val offsetDir = new File(checkpoint.stripPrefix("file:") + "/offsets") val latestOffsetFile = offsetDir.listFiles().filter(!_.getName.endsWith(".crc")) .maxBy(_.getName.toInt) latestOffsetFile.delete() } testSchemaEvolution("schema log initialization with additive schema changes") { implicit log => // Provide a schema log by default def createNewDf(): DataFrame = readStream(schemaLocation = Some(getDefaultSchemaLocation.toString)) // Initialize snapshot schema same as latest, no need to fail stream testStream(createNewDf())( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailable(), CheckAnswer((-1 until 5).map(_.toString).map(i => (i, i)): _*) ) val v0 = log.update().version // And schema log is initialized already, even though there aren't schema evolution exceptions assert(getDefaultSchemaLog().getCurrentTrackedMetadata.get.deltaCommitVersion == v0) // Add a column and some data addColumn("c") val v1 = log.update().version addData(5 until 10) // Update schema log to v1 testStream(createNewDf())( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, CheckAnswer(Nil: _*), ExpectMetadataEvolutionException ) assert(getDefaultSchemaLog().getCurrentTrackedMetadata.get.deltaCommitVersion == v1) var v2: Long = -1 testStream(createNewDf())( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailable(), // Process successfully CheckAnswer((5 until 10).map(_.toString).map(i => (i, i, i)): _*), // Trigger additive schema change would evolve schema as well Execute { _ => addColumn("d") v2 = log.update().version }, Execute { _ => addData(10 until 15) }, ExpectMetadataEvolutionException, AssertOnQuery { q => val offset = DeltaSourceOffset(log.unsafeVolatileTableId, q.availableOffsets.values.last) offset.index == indexWhenSchemaLogIsUpdated } ) assert(getDefaultSchemaLog().getCurrentTrackedMetadata.get.deltaCommitVersion == v2) testStream(createNewDf())( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailable(), CheckAnswer((10 until 15).map(_.toString).map(i => (i, i, i, i)): _*) ) } testSchemaEvolution("detect incompatible schema change while streaming") { implicit log => // Rename as part of initial snapshot renameColumn("b", "c") // Write more data addData(5 until 10) // Source df without schema location val df = readStream() var schemaChangeDeltaVersion: Long = -1 testStream(df)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailable(), // schema change inside initial snapshot should not throw error CheckAnswer((-1 until 10).map(i => (i.toString, i.toString)): _*), // This new rename should throw the legacy error because we have not provided a schema // location Execute { _ => renameColumn("c", "d") schemaChangeDeltaVersion = log.update().version }, // Add some data in new schema Execute { _ => addData(10 until 15) }, ProcessAllAvailableIgnoreError, // No more data should've been processed CheckAnswer((-1 until 10).map(i => (i.toString, i.toString)): _*), // Detected by the in stream check ExpectInStreamSchemaChangeFailure ) // Start the stream again with a schema location val df2 = readStream(schemaLocation = Some(getDefaultSchemaLocation.toString)) assert(getDefaultSchemaLog().getLatestMetadata.isEmpty) testStream(df2)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, // No data should've been processed CheckAnswer(Nil: _*), // Schema evolution exception! ExpectMetadataEvolutionExceptionFromInitialization ) // We should've updated the schema to the version just before the schema change version // because that's the previous version's schema we left with. To be safe and in case there // are more file actions to process, we saved that schema instead of the renamed schema. // Also, since the previous batch was still on initial snapshot, the last file action was not // bumped to the next version, so the schema initialization effectively did not consider the // rename column schema change's version. assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == schemaChangeDeltaVersion - 1) // Start the stream again with the same schema location val df3 = readStream(schemaLocation = Some(getDefaultSchemaLocation.toString)) testStream(df3)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, // Again, no data should've been processed because the next version has a rename CheckAnswer(Nil: _*), // And schema will be evolved again ExpectMetadataEvolutionException ) // Now finally the schema log is up to date assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == schemaChangeDeltaVersion) // Start the stream again should process the rest of the data without a problem val df4 = readStream(schemaLocation = Some(getDefaultSchemaLocation.toString)) val v1 = log.update().version testStream(df4)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailable(), CheckAnswer((10 until 15).map(i => (i.toString, i.toString)): _*), AssertOnQuery { q => val offset = DeltaSourceOffset(log.unsafeVolatileTableId, q.availableOffsets.values.last) // bumped from file action, no pending schema change offset.reservoirVersion == v1 + 1 && offset.index == DeltaSourceOffset.BASE_INDEX && // BASE_INDEX is -100 but serialized form should use version 1 & index -1 for backward // compatibility offset.json.contains(s""""sourceVersion":1""") && offset.json.contains(s""""index":-1""") }, // Trigger another schema change Execute { _ => addColumn("e") addData(15 until 20) }, ProcessAllAvailableIgnoreError, // No more new data CheckAnswer((10 until 15).map(i => (i.toString, i.toString)): _*), AssertOnQuery { q => val offset = DeltaSourceOffset(log.unsafeVolatileTableId, q.availableOffsets.values.last) // latest offset should have a schema attached and evolved set to true // note the reservoir version has not changed offset.reservoirVersion == v1 + 1 && offset.index == indexWhenSchemaLogIsUpdated }, ExpectMetadataEvolutionException ) assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == v1 + 1) val df5 = readStream(schemaLocation = Some(getDefaultSchemaLocation.toString)) // Process the rest testStream(df5)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailable(), CheckAnswer((15 until 20).map(i => (i.toString, i.toString, i.toString)): _*) ) } testSchemaEvolution("detect incompatible schema change during first getBatch") { implicit log => renameColumn("b", "c") val schemaChangeVersion = log.update().version // Source df without schema location, and start at version 1 to ignore initial snapshot // We also use maxFilePerTrigger=1 so that the first getBatch will conduct the check instead // of latestOffset() scanning far ahead and throw the In-Stream version of the exception. val df = readStream(startingVersion = Some(1), maxFilesPerTrigger = Some(1)) testStream(df)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), // Add more data Execute { _ => addData(5 until 10) }, // Try processing ProcessAllAvailableIgnoreError, // No data should've been processed :) CheckAnswer(Nil: _*), // The first getBatch should fail if (isCdcTest) { ExpectGenericSchemaIncompatibleFailure } else { ExpectStreamStartInCompatibleSchemaFailure } ) // Restart with a schema location, note that maxFilePerTrigger is not needed now // because a schema location is provided and any exception would evolve the schema. val df2 = readStream(startingVersion = Some(1), schemaLocation = Some(getDefaultSchemaLocation.toString)) assert(getDefaultSchemaLog().getLatestMetadata.isEmpty) testStream(df2)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, // Again, no data is processed CheckLastBatch(Nil: _*), // Schema evolution exception! ExpectMetadataEvolutionExceptionFromInitialization ) // Since the error happened during the first getBatch, we initialize schema log to schema@v1 assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == 1) // Restart again with a schema location val df3 = readStream(startingVersion = Some(1), schemaLocation = Some(getDefaultSchemaLocation.toString)) testStream(df3)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, // Note that the default maxFilePerTrigger is 1000, so this shows that the batch has been // split and the available data prior to schema change should've been served. // Also since we started at v1, -1 is not included. CheckAnswer((0 until 5).map(i => (i.toString, i.toString)): _*), // Schema evolution exception! ExpectMetadataEvolutionException ) // Now the schema is up to date assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == schemaChangeVersion) // Restart again should pick up the new schema and process the rest without a problem. // Note that startingVersion is ignored when we have existing progress to work with. val df4 = readStream(startingVersion = Some(1), schemaLocation = Some(getDefaultSchemaLocation.toString)) testStream(df4)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailable(), CheckAnswer((5 until 10).map(i => (i.toString, i.toString)): _*) ) } test("identity columns shouldn't cause schema mismatches") { withTable("source") { sql( s""" |CREATE TABLE source (key INT, id LONG GENERATED ALWAYS AS IDENTITY) |USING DELTA """.stripMargin ) val deltaLog = DeltaLog.forTable(spark, TableIdentifier("source")) deltaLog.update() val schemaLocation = getDefaultSchemaLocation(deltaLog).toString val checkpointLocation = getDefaultCheckpoint(deltaLog).toString def addData(values: Seq[Int]): Unit = spark.createDataFrame(values.map(Row(_)).asJava, StructType.fromDDL("key INT")) .write.format("delta").mode("append").saveAsTable("source") def readStream(): DataFrame = spark.readStream .format("delta") .option(DeltaOptions.SCHEMA_TRACKING_LOCATION, schemaLocation) .table("source") // Check fix disabled: writing to the table updates the identity column's high-water mark // stored in the table schema, causing a schema change to be detected. addData(values = 0 until 5) withSQLConf( DeltaSQLConf.DELTA_STREAMING_IGNORE_INTERNAL_METADATA_FOR_SCHEMA_CHANGE.key -> "false" ) { testStream(readStream())( StartStream(checkpointLocation = checkpointLocation), ProcessAllAvailable(), Execute { _ => addData(values = 10 until 15) }, ExpectMetadataEvolutionException ) } // Check fix enabled: high-water mark updates are ignored when checking for schema changes. addData(values = 15 until 20) withSQLConf( DeltaSQLConf.DELTA_STREAMING_IGNORE_INTERNAL_METADATA_FOR_SCHEMA_CHANGE.key -> "true" ) { testStream(readStream())( StartStream(checkpointLocation = checkpointLocation), ProcessAllAvailable(), Execute { _ => addData(values = 20 until 25) }, ProcessAllAvailable() ) // No schema change detected. Note that the identity column metadata is still present in the // tracked schema val field = getDefaultSchemaLog()(deltaLog).getLatestMetadata.get.dataSchema("id") assert(field.metadata.contains(DeltaSourceUtils.IDENTITY_INFO_HIGHWATERMARK)) } } } /** * This test manually generates Delta source offsets that crosses non-additive schema change * boundaries to test if the schema log initialization check logic can detect those changes and * error out. */ protected def testDetectingInvalidOffsetDuringLogInit( invalidAction: String, readStreamWithSchemaLocation: => DataFrame, expectedLogInitException: StreamAction)(implicit log: DeltaLog): Unit = { // start a stream to initialize checkpoint val ckpt = getDefaultCheckpoint.toString val schemaLoc = getDefaultSchemaLocation.toString val df = readStream(startingVersion = Some(1)) testStream(df)( StartStream(checkpointLocation = ckpt), ProcessAllAvailable(), CheckAnswer((0 until 5).map(i => (i.toString, i.toString)): _*), StopStream ) // Add more data to create room for data offsets, so when the stream resumes, the latest // committed offset if still in the old schema. addData(Seq(6)) if (invalidAction == "rename") { renameColumn("b", "c") } else if (invalidAction == "drop") { addColumn("c") } // write more data addData(Seq(7)) // Add a rename or drop commit that reverses the previous change, to ensure that our check // has validated all the schema changes, instead of just checking the start schema. if (invalidAction == "rename") { renameColumn("c", "b") } else if (invalidAction == "drop") { dropColumn("c") } else { assert(false, s"unexpected action ${invalidAction}") } // write more data addData(Seq(8)) val latestVersion = log.update().version // Manually create another offset to latest version to simulate the situation that an end // offset is somehow generated that bypasses the block, e.g. they were upgrading from a // super old version that did not have the block logic, and is left with a constructed // batch that bypasses a schema change. // There should be at MOST one such trailing batch as of today's streaming engine semantics. val offsetContent = manuallyCreateLatestStreamingOffsetUntilReservoirVersion(ckpt, latestVersion) // rerun the stream should detect that and fail, even with schema location testStream(readStreamWithSchemaLocation)( StartStream(checkpointLocation = ckpt), ProcessAllAvailableIgnoreError, CheckAnswer(Nil: _*), expectedLogInitException ) // Let's also test the case when we only have one offset in the checkpoint without any committed // Delete everything except the metadata file to avoid triggering metadata validation error val ckptDir = new File(ckpt.stripPrefix("file:")) val metadataFile = new File(ckptDir, "metadata") // Delete all checkpoint subdirectories and files except metadata ckptDir.listFiles().foreach { f => if (f.getAbsolutePath != metadataFile.getAbsolutePath) { if (f.isDirectory) { FileUtils.deleteDirectory(f) } else { f.delete() } } } FileUtils.deleteDirectory(new File(schemaLoc.stripPrefix("file:"))) // Create a single offset that points to the latest version of the table. manuallyCreateStreamingOffsetAtBatchId(ckpt, 0, offsetContent) // One more non additive schema change if (invalidAction == "rename") { renameColumn("a", "x") } else if (invalidAction == "drop") { dropColumn("b") } addData(Seq(9)) val latestVersion2 = log.update().version // Create another offset point to the updated latest version manuallyCreateLatestStreamingOffsetUntilReservoirVersion(ckpt, latestVersion2) // This should also fail because it crossed the new non-additive schema change above, note that // since we didn't have a committed offset nor a user specified startingVersion, the first // offset will re-read using latestVersion2 - 1 as the initial snapshot now. // Without this new non-additive schema change the validation would actually pass. testStream(readStreamWithSchemaLocation)( StartStream(checkpointLocation = ckpt), ProcessAllAvailableIgnoreError, CheckAnswer(Nil: _*), expectedLogInitException ) } Seq("rename", "drop").foreach { invalidAction => testSchemaEvolution(s"detect invalid offset during initialization before " + s"initializing schema log - $invalidAction") { implicit log => def provideStreamingDf: DataFrame = readStream(schemaLocation = Some(getDefaultSchemaLocation.toString)) testDetectingInvalidOffsetDuringLogInit( invalidAction, provideStreamingDf, ExpectSchemaLogInitializationFailedException ) } } /** * This test checks a corner case on the initialization of the schema log. * When a log is initialized, we would check over ALL pending batches and their delta versions * to ensure we have a safe schema to read all of them (i.e. no non-additive schema changes) * within the range. * This test checks the case when the last version of the range is a non-additive schema change, * but it does not need to be blocked because there's no data to be read during initialization. */ protected def testLogInitializationWithoutBlockingOnSchemaChangeInTheEnd( readStreamWithSchemaLocation: => DataFrame, expectLogInitException: StreamAction)(implicit log: DeltaLog): Unit = { // Start a stream to initialize checkpoint val ckpt = getDefaultCheckpoint.toString val df = readStream(startingVersion = Some(1)) testStream(df)( StartStream(checkpointLocation = ckpt), ProcessAllAvailable(), CheckAnswer((0 until 5).map(i => (i.toString, i.toString)): _*), StopStream ) val v0 = log.update().version // The previous committed offset ends at (v0 + 1, -100). // Add more data addData(Seq(5)) // Non-additive schema change renameColumn("b", "c") val v1 = log.update().version // Manually create another offset ending on [v1, -100] manuallyCreateLatestStreamingOffsetUntilReservoirVersion(ckpt, v1) // Start stream again would attempt to run the constructed batch first. // Since the ending offset does not yet contain the metadata action, we won't need to block // the schema log initialization testStream(readStreamWithSchemaLocation)( StartStream(checkpointLocation = ckpt), ProcessAllAvailableIgnoreError, CheckAnswer(Nil: _*), expectLogInitException ) assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == v0 + 1) testStream(readStreamWithSchemaLocation)( StartStream(checkpointLocation = ckpt), ProcessAllAvailableIgnoreError, // Data processed CheckAnswer(("5", "5")), ExpectMetadataEvolutionException ) assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == v1) } testSchemaEvolution(s"no need to block schema log initialization if " + s"constructed batch ends on schema change") { implicit log => def provideStreamingDf: DataFrame = readStream(schemaLocation = Some(getDefaultSchemaLocation.toString)) testLogInitializationWithoutBlockingOnSchemaChangeInTheEnd( provideStreamingDf, ExpectMetadataEvolutionExceptionFromInitialization ) } testSchemaEvolution("resolve the most encompassing schema during getBatch " + "to initialize schema log") { implicit log => // start a stream to initialize checkpoint val ckpt = getDefaultCheckpoint.toString val df = readStream(startingVersion = Some(1)) testStream(df)( StartStream(checkpointLocation = ckpt), ProcessAllAvailable() ) val v1 = log.update().version // add a new column addColumn("c") // write more data addData(5 until 6) // add another column addColumn("d") val secondAddColumnVersion = log.update().version addData(6 until 10) // add an invalid commit so we could fail directly renameColumn("d", "d2") val renamedVersion = log.update().version // v2 should include the two add column change but not the renamed version val v2 = v1 + 5 // manually create another offset to latest version manuallyCreateLatestStreamingOffsetUntilReservoirVersion(ckpt, v2, -1) // rerun the stream should detect rename with the stream start check, but since within the // offsets the schema changes are all additive, we could use the encompassing schema . val schemaLocation = getDefaultSchemaLocation.toString testStream(readStream(schemaLocation = Some(schemaLocation)))( StartStream(checkpointLocation = ckpt), ProcessAllAvailableIgnoreError, CheckAnswer(Nil: _*), // Schema can be evolved ExpectMetadataEvolutionExceptionFromInitialization ) // Schema log is ready and populated with assert(getDefaultSchemaLog().getLatestMetadata.get.dataSchema.fieldNames .sameElements(Array("a", "b", "c", "d"))) // ... which is the schema that should be valid until v2 - 1 (the batch end version). // It is v2 - 1 because the latest offset sits on the BASE_INDEX of v2, which does not contain // any data, so there's no need to consider that for schema change initialization. assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == v2 - 1) // Keep going until rename is found testStream(readStream(schemaLocation = Some(schemaLocation)))( StartStream(checkpointLocation = ckpt), ProcessAllAvailableIgnoreError, CheckAnswer((Seq(5).map(i => (i.toString, i.toString, i.toString, null)) ++ (6 until 10).map(i => (i.toString, i.toString, i.toString, i.toString))): _*), ExpectMetadataEvolutionException ) // Schema log is evolved with assert(getDefaultSchemaLog().getLatestMetadata.get.dataSchema.fieldNames .sameElements(Array("a", "b", "c", "d2"))) // ... which is the renamed version assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == renamedVersion) } test("trigger.Once with deferred commit should work") { withStarterTable { implicit log => dropColumn("b") val schemaChangeVersion = log.update().version addData(5 until 10) val ckpt = getDefaultCheckpoint.toString val schemaLoc = getDefaultSchemaLocation.toString // Use starting version to ignore initial snapshot def read: DataFrame = readStream(schemaLocation = Some(schemaLoc), startingVersion = Some(1)) // Use once trigger to execute streaming one step a time val StartThisStream = StartStream(trigger = Trigger.Once, checkpointLocation = ckpt) // This trigger: // 1. The stream starts with an uninitialized schema log. // 2. The stream schema is taken from the latest version of the Delta table. // 3. The schema tracking log must initialized immediately, in this case from latestOffset // because this is the first time the stream starts. The schema is initialized to the // schema at version 1. // 4. Because the schema at version 1 is not equal to the stream schema, the stream must be // restarted. testStream(read)( StartThisStream, AwaitTerminationIgnoreError, CheckAnswer(Nil: _*), ExpectMetadataEvolutionExceptionFromInitialization ) // Latest schema in schema log has been initialized assert(getDefaultSchemaLog().getLatestMetadata.exists(_.deltaCommitVersion == 1)) // This trigger: // 1. Finds the latest offset that ends with the schema change // 2. Serve all batches prior to the schema change // Note that the schema has NOT evolved yet because the batch ending at the schema change has // not being committed, and thus we have not triggered the schema evolution and will need an // extra restart. testStream(read)( StartThisStream, AwaitTerminationIgnoreError, CheckAnswer((0 until 5).map(i => (i.toString, i.toString)): _*), AssertOnQuery { q => val offset = DeltaSourceOffset(log.unsafeVolatileTableId, q.availableOffsets.values.last) // bumped from file action offset.reservoirVersion == schemaChangeVersion && offset.index == DeltaSourceOffset.METADATA_CHANGE_INDEX && // serialized as version 3 because METADATA_CHANGE_INDEX is only available in v3 offset.json.contains(s""""sourceVersion":3""") } ) assert(getDefaultSchemaLog().getLatestMetadata.exists(_.deltaCommitVersion == 1)) // This trigger: // 1. Finds a NEW latest offset that sets the dummy offset index post schema change // 2. The previous valid batch can be committed // 3. The commit evolves the schema and exit the stream. testStream(read)( StartThisStream, AwaitTerminationIgnoreError, CheckAnswer(Nil: _*), AssertOnQuery { q => val offset = DeltaSourceOffset(log.unsafeVolatileTableId, q.availableOffsets.values.last) // still stuck, but the pending schema change is marked as evolved offset.reservoirVersion == schemaChangeVersion && offset.index == DeltaSourceOffset.POST_METADATA_CHANGE_INDEX && // serialized as version 3 because POST_METADATA_CHANGE_INDEX is only available in v3 offset.json.contains(s""""sourceVersion":3""") }, ExpectMetadataEvolutionException ) assert(getDefaultSchemaLog().getLatestMetadata .exists(_.deltaCommitVersion == schemaChangeVersion)) // This trigger: // 1. GetBatch for the empty batch because it was constructed and now no schema mismatches testStream(read)( StartThisStream, AwaitTermination, CheckAnswer(Nil: _*) ) // This trigger: // 1. Find the latest offset till end of data // 2. Commits the previous empty batch (with no schema change), so no schema evolution // 3. GetBatch of all data val v2 = log.update().version testStream(read)( StartThisStream, AwaitTermination, CheckAnswer((5 until 10).map(i => (i.toString)): _*), AssertOnQuery { q => val offset = DeltaSourceOffset(log.unsafeVolatileTableId, q.availableOffsets.values.last) // bumped by file action, and since it's an non schema change, just clear schema change offset.reservoirVersion == v2 + 1 && offset.index == DeltaSourceOffset.BASE_INDEX } ) // Create a new schema change addColumn("b") val v3 = log.update().version addData(10 until 11) // This trigger: // 1. Finds a new offset ending with the schema change index // 2. Commits previous batch (no schema change, thus no schema evolution) // 3. GetBatch of this empty batch testStream(read)( StartThisStream, AwaitTermination, CheckAnswer(Nil: _*), AssertOnQuery { q => val offset = DeltaSourceOffset(log.unsafeVolatileTableId, q.availableOffsets.values.last) offset.reservoirVersion == v2 + 1 && offset.index == DeltaSourceOffset.METADATA_CHANGE_INDEX } ) // This trigger: // 1. Again, finds an empty batch but now ending at the dummy post schema change index. // 2. Commits the previous batch, evolve the schema and fail the stream. testStream(read)( StartThisStream, AwaitTerminationIgnoreError, CheckAnswer(Nil: _*), AssertOnQuery { q => val offset = DeltaSourceOffset(log.unsafeVolatileTableId, q.availableOffsets.values.last) offset.reservoirVersion == v3 && offset.index == DeltaSourceOffset.POST_METADATA_CHANGE_INDEX }, ExpectMetadataEvolutionException ) } } test("trigger.AvailableNow should work") { withStarterTable { implicit log => dropColumn("b") val schemaChangeVersion = log.update().version addData(5 until 10) val ckpt = getDefaultCheckpoint.toString val schemaLoc = getDefaultSchemaLocation.toString // Use starting version to ignore initial snapshot def read: DataFrame = readStream(schemaLocation = Some(schemaLoc), startingVersion = Some(1)) // Use trigger available now val StartThisStream = StartStream(trigger = Trigger.AvailableNow(), checkpointLocation = ckpt) // Similar to once trigger, this: // 1. Detects the schema change right-away from computing latest offset // 2. Initialize the schema log and exit stream testStream(read)( StartThisStream, AwaitTerminationIgnoreError, CheckAnswer(Nil: _*), ExpectMetadataEvolutionExceptionFromInitialization ) // Latest schema in schema log has been updated assert(getDefaultSchemaLog().getLatestMetadata.exists(_.deltaCommitVersion == 1)) // Now, this trigger: // 1. Finds the latest offset RIGHT AT the schema change ending at schema change index // 2. GetBatch till that offset // 3. Finds ANOTHER the latest offset ending at the dummy post schema change index // 4. GetBatch for this empty batch // 5. Commits the previous batch // 6. Triggers schema evolution testStream(read)( StartThisStream, AwaitTerminationIgnoreError, CheckAnswer((0 until 5).map(_.toString).map(i => (i, i)): _*), AssertOnQuery { q => val offset = DeltaSourceOffset(log.unsafeVolatileTableId, q.availableOffsets.values.last) offset.reservoirVersion == schemaChangeVersion && // schema change marked as evolved offset.index == DeltaSourceOffset.POST_METADATA_CHANGE_INDEX }, ExpectMetadataEvolutionException ) assert(getDefaultSchemaLog().getLatestMetadata .exists(_.deltaCommitVersion == schemaChangeVersion)) // This trigger: // 1. Finds the next latest offset, which is the end of data // 2. Commit previous empty batch with no pending schema change // 3. GetBatch with the remaining data val latestVersion = log.update().version testStream(read)( StartThisStream, AwaitTermination, CheckAnswer((5 until 10).map(i => (i.toString)): _*), AssertOnQuery { q => val offset = DeltaSourceOffset(log.unsafeVolatileTableId, q.availableOffsets.values.last) // schema change cleared because it's a non-schema change offset offset.reservoirVersion == latestVersion + 1 && offset.index == DeltaSourceOffset.BASE_INDEX } ) // Create a new schema change addColumn("b") val v3 = log.update().version addData(10 until 11) // This trigger: // 1. Finds the latest offset, again ending at the schema change index // 2. Commits previous batch // 3. GetBatch with empty data and schema change ending offset // 4. Finds another latest offset, ending at the dummy post schema change index // 5. Commits the empty batch at 3, evolves schema log and restart stream. testStream(read)( StartThisStream, AwaitTerminationIgnoreError, CheckAnswer(Nil: _*), AssertOnQuery { q => val offset = DeltaSourceOffset(log.unsafeVolatileTableId, q.availableOffsets.values.last) offset.reservoirVersion == v3 && offset.index == DeltaSourceOffset.POST_METADATA_CHANGE_INDEX }, ExpectMetadataEvolutionException ) // Finish the rest testStream(read)( StartThisStream, AwaitTermination, CheckAnswer((10 until 11).map(_.toString).map(i => (i, i)): _*) ) } } testSchemaEvolution("consecutive schema evolutions without schema merging") { implicit log => withSQLConf( DeltaSQLConf.DELTA_STREAMING_ENABLE_SCHEMA_TRACKING_MERGE_CONSECUTIVE_CHANGES.key -> "false") { val v5 = log.update().version // v5 has an ADD file action with value (4, 4) renameColumn("b", "c") // v6 renameColumn("c", "b") // v7 dropColumn("b") // v9 addColumn("b") // v10 def df: DataFrame = readStream( schemaLocation = Some(getDefaultSchemaLocation.toString), startingVersion = Some(v5)) // The schema log initializes @ v1 with schema testStream(df)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, AssertOnQuery { q => // initialization does not generate any offsets q.availableOffsets.isEmpty }, ExpectMetadataEvolutionExceptionFromInitialization ) assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == v5) assert(getDefaultSchemaLog().getLatestMetadata.get.dataSchema.fieldNames .sameElements(Array("a", "b"))) // Encounter next schema change testStream(df)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, CheckAnswer(Seq(4).map(_.toString).map(i => (i, i)): _*), AssertOnQuery { q => q.availableOffsets.size == 1 && { val offset = DeltaSourceOffset( log.unsafeVolatileTableId, q.availableOffsets.values.head) offset.reservoirVersion == v5 + 1 && offset.index == indexWhenSchemaLogIsUpdated } }, ExpectMetadataEvolutionException ) assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == v5 + 1) assert(getDefaultSchemaLog().getLatestMetadata.get.dataSchema.fieldNames .sameElements(Array("a", "c"))) // Encounter next schema change again testStream(df)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, AssertOnQuery { q => // size is 1 because commit removes previous offset q.availableOffsets.size == 1 && { val offset = DeltaSourceOffset( log.unsafeVolatileTableId, q.availableOffsets.values.head) offset.reservoirVersion == v5 + 2 && offset.index == indexWhenSchemaLogIsUpdated } }, ExpectMetadataEvolutionException ) assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == v5 + 2) assert(getDefaultSchemaLog().getLatestMetadata.get.dataSchema.fieldNames .sameElements(Array("a", "b"))) // Encounter next schema change testStream(df)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, AssertOnQuery { q => q.availableOffsets.size == 1 && { val offset = DeltaSourceOffset( log.unsafeVolatileTableId, q.availableOffsets.values.head) offset.reservoirVersion == v5 + 3 && offset.index == indexWhenSchemaLogIsUpdated } }, ExpectMetadataEvolutionException ) assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == v5 + 3) assert(getDefaultSchemaLog().getLatestMetadata.get.dataSchema.fieldNames .sameElements(Array("a"))) // Encounter next schema change again testStream(df)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, AssertOnQuery { q => q.availableOffsets.size == 1 && { val offset = DeltaSourceOffset( log.unsafeVolatileTableId, q.availableOffsets.values.head) offset.reservoirVersion == v5 + 4 && offset.index == indexWhenSchemaLogIsUpdated } }, ExpectMetadataEvolutionException ) assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == v5 + 4) assert(getDefaultSchemaLog().getLatestMetadata.get.dataSchema.fieldNames .sameElements(Array("a", "b"))) } } testSchemaEvolution("consecutive schema evolutions") { implicit log => // By default we have consecutive schema merging turned on val v5 = log.update().version // v5 has an ADD file action with value (4, 4) renameColumn("b", "c") // v6 renameColumn("c", "b") // v7 dropColumn("b") // v9 addColumn("b") // v10 val v10 = log.update().version // Write some more data post the consecutive schema changes addData(5 until 6) def df: DataFrame = readStream( schemaLocation = Some(getDefaultSchemaLocation.toString), startingVersion = Some(v5)) // The schema log initializes @ v1 with schema testStream(df)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, AssertOnQuery { q => // initialization does not generate any offsets q.availableOffsets.isEmpty }, ExpectMetadataEvolutionExceptionFromInitialization ) assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == v5) assert(getDefaultSchemaLog().getLatestMetadata.get.dataSchema.fieldNames .sameElements(Array("a", "b"))) // Encounter next schema change // This still fails schema evolution exception and won't scan ahead testStream(df)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, CheckAnswer(Seq(4).map(_.toString).map(i => (i, i)): _*), AssertOnQuery { q => q.availableOffsets.size == 1 && { val offset = DeltaSourceOffset(log.unsafeVolatileTableId, q.availableOffsets.values.head) offset.reservoirVersion == v5 + 1 && offset.index == indexWhenSchemaLogIsUpdated } }, ExpectMetadataEvolutionException ) assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == v5 + 1) assert(getDefaultSchemaLog().getLatestMetadata.get.dataSchema.fieldNames .sameElements(Array("a", "c"))) // Now the next restart would scan over the consecutive schema changes and use the last one // to initialize the schema again. val latestDf = df assert(latestDf.schema.fieldNames.sameElements(Array("a", "b"))) // The analysis phase should've already updated schema log assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == v10) // Processing should ignore the intermediary schema changes and process the data using the // merged schema. testStream(latestDf)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailable(), CheckAnswer((5 until 6).map(i => (i.toString, i.toString)): _*) ) } testSchemaEvolution("upgrade and downgrade") { implicit log => val ckpt = getDefaultCheckpoint.toString val df = readStream(startingVersion = Some(1)) val v0 = log.update().version // Initialize a stream testStream(df)( StartStream(checkpointLocation = ckpt), ProcessAllAvailable(), CheckAnswer((0 until 5).map(_.toString).map(i => (i, i)): _*), AssertOnQuery { q => assert(q.availableOffsets.size == 1) val offset = DeltaSourceOffset(log.unsafeVolatileTableId, q.availableOffsets.values.last) offset.reservoirVersion == v0 + 1 && offset.index == DeltaSourceOffset.BASE_INDEX } ) addData(Seq(5)) val v1 = log.update().version dropColumn("b") val v2 = log.update().version // Restart with schema location should initialize val df2 = readStream(schemaLocation = Some(getDefaultSchemaLocation.toString)) testStream(df2)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, AssertOnQuery { q => // initialization does not generate any more offsets q.availableOffsets.size <= 1 }, ExpectMetadataEvolutionExceptionFromInitialization ) // The schema should be valid until v1 (the batch end version). // It is v1 - 1 because the latest offset sits on the BASE_INDEX of v1, which does not contain // any data, so there's no need to consider that for schema change initialization. assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == v1 - 1) // Restart again should be able to use the new offset version val df3 = readStream(schemaLocation = Some(getDefaultSchemaLocation.toString)) val logAppenderUpgrade = new LogAppender("Should convert legacy offset", maxEvents = 1e6.toInt) logAppenderUpgrade.setThreshold(Level.DEBUG) withLogAppender(logAppenderUpgrade, level = Some(Level.DEBUG)) { testStream(df3)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, CheckAnswer(("5", "5")), AssertOnQuery { q => val offset = DeltaSourceOffset(log.unsafeVolatileTableId, q.availableOffsets.values.last) offset.reservoirVersion == v2 && offset.index == indexWhenSchemaLogIsUpdated }, ExpectMetadataEvolutionException ) } assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == v2) // Should've upgraded the legacy offset val target = logAppenderUpgrade.loggingEvents.find( _.getMessage.toString.contains("upgrading offset ")) assert(target.isDefined) // Add more data addData(Seq(6)) // Suppose now the user doesn't want to use schema tracking any more, and whats to downgrade // to use latest schema again, it should be able to do that. val df4 = readStream() // without schema location val logAppenderDowngrade = new LogAppender("Should convert new offset", maxEvents = 1e6.toInt) logAppenderDowngrade.setThreshold(Level.DEBUG) withSQLConf( DeltaSQLConf.DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_SCHEMA_CHANGES_DURING_STREAM_START .key -> "true", DeltaSQLConf.DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_COLUMN_MAPPING_SCHEMA_CHANGES .key -> "true") { withLogAppender(logAppenderDowngrade, level = Some(Level.DEBUG)) { testStream(df4)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailable(), // See the next read just falls back to use latest schema CheckAnswer(("6")) ) } } } testSchemaEvolution("multiple sources with schema evolution" ) { implicit log => val v5 = log.update().version // v5 has an ADD file action with value (4, 4) renameColumn("b", "c") addData(5 until 10) val schemaLog1Location = new Path(getDefaultCheckpoint, "_schema_log1").toString val schemaLog2Location = new Path(getDefaultCheckpoint, "_schema_log2").toString // Join two individual sources with two schema log // Each source should return an identical batch and therefore the output batch should also be // identical, we are just using join to create a multi-source situation. def df: DataFrame = readStream(schemaLocation = Some(schemaLog1Location), startingVersion = Some(v5)) .unionByName( readStream(schemaLocation = Some(schemaLog2Location), startingVersion = Some(v5)), allowMissingColumns = true) // Both schema log initialized def schemaLog1: DeltaSourceMetadataTrackingLog = DeltaSourceMetadataTrackingLog.create( spark, schemaLog1Location, log.update(), catalogTableOpt = None, parameters = Map.empty) def schemaLog2: DeltaSourceMetadataTrackingLog = DeltaSourceMetadataTrackingLog.create( spark, schemaLog2Location, log.update(), catalogTableOpt = None, parameters = Map.empty) // The schema log initializes @ v5 with schema testStream(df)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, AssertOnQuery { q => // initialization does not generate any offsets q.availableOffsets.isEmpty }, ExpectMetadataEvolutionExceptionFromInitialization ) // But takes another restart for the other Delta source testStream(df)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, AssertOnQuery { q => // initialization does not generate any offsets q.availableOffsets.isEmpty }, ExpectMetadataEvolutionExceptionFromInitialization ) // Both schema log should be initialized assert(schemaLog1.getCurrentTrackedMetadata.map(_.deltaCommitVersion) == schemaLog2.getCurrentTrackedMetadata.map(_.deltaCommitVersion)) // One of the source will commit and fail testStream(df)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, // The data prior to schema change is served // Two rows in schema [a, b] CheckAnswer(("4", "4"), ("4", "4")), ExpectMetadataEvolutionException ) // Restart should fail the other commit testStream(df)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, CheckAnswer(Nil: _*), ExpectMetadataEvolutionException ) assert(schemaLog1.getCurrentTrackedMetadata.map(_.deltaCommitVersion) == schemaLog2.getCurrentTrackedMetadata.map(_.deltaCommitVersion)) // Restart stream should proceed on loading the rest of data testStream(df)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailable(), // Unioned data is served // 10 rows in schema [a, c] CheckAnswer((5 until 10).map(_.toString).flatMap(i => Seq((i, i), (i, i))): _*) ) // Attempt to use the wrong schema log for each source will be detected val wrongDf = readStream(schemaLocation = // instead of using schemaLog1Location Some(schemaLog2Location), startingVersion = Some(v5)) .unionByName( readStream(schemaLocation = // instead of using schemaLog2Location Some(schemaLog1Location), startingVersion = Some(v5)), allowMissingColumns = true) testStream(wrongDf)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, ExpectFailure[IllegalArgumentException](t => assert(t.getMessage.contains("The Delta source metadata path used for execution"))) ) } testSchemaEvolution("schema evolution with Delta sink") { implicit log => val v5 = log.update().version // v5 has an ADD file action with value (4) renameColumn("b", "c") val renameVersion1 = log.update().version addData(5 until 10) renameColumn("c", "b") val renameVersion2 = log.update().version addData(10 until 15) dropColumn("b") val dropVersion = log.update().version addData(15 until 20) addColumn("b") val addVersion = log.update().version addData(20 until 25) withTempDir { sink => def writeStream(df: DataFrame): Unit = { val q = df.writeStream .format("delta") .option("checkpointLocation", getDefaultCheckpoint.toString) .option("mergeSchema", "true") // for automatically adding columns .start(sink.getCanonicalPath) q.processAllAvailable() q.stop() } def df: DataFrame = readStream( schemaLocation = Some(getDefaultSchemaLocation.toString), startingVersion = Some(v5)) def readSink: DataFrame = spark.read.format("delta").load(sink.getCanonicalPath) val e1 = ExceptionUtils.getRootCause { intercept[StreamingQueryException] { writeStream(df) } } ExpectMetadataEvolutionExceptionFromInitialization.assertFailure(e1) assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == v5) val e2 = ExceptionUtils.getRootCause { intercept[StreamingQueryException] { writeStream(df) } } assert(readSink.schema.fieldNames sameElements Array("a", "b")) checkAnswer(readSink, Seq(4).map(_.toString).map(i => Row(i, i))) ExpectMetadataEvolutionException.assertFailure(e2) assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == renameVersion1) val e3 = ExceptionUtils.getRootCause { intercept[StreamingQueryException] { writeStream(df) } } // c added as a new column assert(readSink.schema.fieldNames sameElements Array("a", "b", "c")) checkAnswer(readSink, Seq(4).map(_.toString).map(i => Row(i, i, null)) ++ (5 until 10).map(_.toString).map(i => Row(i, null, i))) ExpectMetadataEvolutionException.assertFailure(e3) assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == renameVersion2) val e4 = ExceptionUtils.getRootCause { intercept[StreamingQueryException] { writeStream(df) } } // c was renamed to b, new data now writes to b assert(readSink.schema.fieldNames sameElements Array("a", "b", "c")) checkAnswer(readSink, Seq(4).map(_.toString).map(i => Row(i, i, null)) ++ (5 until 10).map(_.toString).map(i => Row(i, null, i)) ++ (10 until 15).map(_.toString).map(i => Row(i, i, null))) ExpectMetadataEvolutionException.assertFailure(e4) assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == dropVersion) val e5 = ExceptionUtils.getRootCause { intercept[StreamingQueryException] { writeStream(df) } } // b was dropped, but sink remains the same assert(readSink.schema.fieldNames sameElements Array("a", "b", "c")) checkAnswer(readSink, Seq(4).map(_.toString).map(i => Row(i, i, null)) ++ (5 until 10).map(_.toString).map(i => Row(i, null, i)) ++ (10 until 15).map(_.toString).map(i => Row(i, i, null)) ++ (15 until 20).map(_.toString).map(i => Row(i, null, null))) ExpectMetadataEvolutionException.assertFailure(e5) assert(getDefaultSchemaLog().getLatestMetadata.get.deltaCommitVersion == addVersion) // Finish the stream without errors writeStream(df) // b was added back, sink remains the same assert(readSink.schema.fieldNames sameElements Array("a", "b", "c")) checkAnswer(readSink, Seq(4).map(_.toString).map(i => Row(i, i, null)) ++ (5 until 10).map(_.toString).map(i => Row(i, null, i)) ++ (10 until 15).map(_.toString).map(i => Row(i, i, null)) ++ (15 until 20).map(_.toString).map(i => Row(i, null, null)) ++ (20 until 25).map(_.toString).map(i => Row(i, i, null))) } } testSchemaEvolution("latestOffset should not progress before schema evolved") { implicit log => val s0 = log.update() // Change schema renameColumn("b", "c") val v0 = log.update().version addData(Seq(5)) val v1 = log.update().version // Manually construct a Delta source since it's hard to test multiple (2+) latestOffset() calls // with the current streaming engine without incurring the schema evolution failure. def getSource: DeltaSource = DeltaSource( spark, log, catalogTableOpt = None, new DeltaOptions(Map("startingVersion" -> "0"), spark.sessionState.conf), log.update(), metadataPath = "", Some(getDefaultSchemaLog())) def getLatestOffset(source: DeltaSource, start: Option[Offset] = None): DeltaSourceOffset = DeltaSourceOffset(log.unsafeVolatileTableId, source.latestOffset(start.orNull, source.getDefaultReadLimit)) // Initialize the schema log to skip initialization failure getDefaultSchemaLog().writeNewMetadata( PersistedMetadata( log.unsafeVolatileTableId, 0L, s0.metadata, s0.protocol, sourceMetadataPath = "" ) ) val source1 = getSource // 1st call, land at INDEX_SCHEMA_CHANGE val ofs1 = getLatestOffset(source1) assert(ofs1.index == DeltaSourceOffset.METADATA_CHANGE_INDEX) source1.getBatch(startOffsetOption = None, ofs1) // 2nd call, land at INDEX_POST_SCHEMA_CHANGE val ofs2 = getLatestOffset(source1, Some(ofs1)) assert(ofs2.index == DeltaSourceOffset.POST_METADATA_CHANGE_INDEX) source1.getBatch(Some(ofs1), ofs2) // 3rd call, still land at INDEX_POST_SCHEMA_CHANGE, because schema evolution has not happened val ofs3 = getLatestOffset(source1, Some(ofs2)) assert(ofs3.index == DeltaSourceOffset.POST_METADATA_CHANGE_INDEX) // Commit and restart val e = intercept[DeltaRuntimeException] { source1.commit(ofs2) } ExpectMetadataEvolutionException.assertFailure(e) assert(getDefaultSchemaLog().getCurrentTrackedMetadata.get.deltaCommitVersion == v0) val source2 = getSource // restore previousOffset source2.getBatch(Some(ofs3), ofs3) // 4th call, should move on to latest version + 1 (bumped by file action) val ofs4 = getLatestOffset(source2, Some(ofs3)) assert(ofs4.index == DeltaSourceOffset.BASE_INDEX && ofs4.reservoirVersion == v1 + 1) } protected def expectSqlConfException( opType: String, ver: Long, columnChangeDetails: String, checkpointHash: Int) = { ExpectFailure[DeltaRuntimeException] { e => val se = e.asInstanceOf[DeltaRuntimeException] assert { se.getErrorClass == "DELTA_STREAMING_CANNOT_CONTINUE_PROCESSING_POST_SCHEMA_EVOLUTION" && se.messageParameters(0) == opType && se.messageParameters(2) == ver.toString && se.messageParameters(3).contains(columnChangeDetails) && se.messageParameters.exists(_.contains(checkpointHash.toString)) } } } /** * Initialize a simple streaming DF for a simple table with just one (0, 0) entry for schema * We also prepare an initialized schema log to skip the initialization phase. */ protected def withSimpleStreamingDf(f: (() => DataFrame, DeltaLog) => Unit): Unit = { withTempDir { dir => val tablePath = dir.getCanonicalPath Seq(("0", "0")).toDF("a", "b") .write.mode("append").format("delta").save(tablePath) implicit val log = DeltaLog.forTable(spark, dir.getCanonicalPath) val s0 = log.update() val schemaLog = getDefaultSchemaLog() schemaLog.writeNewMetadata( PersistedMetadata(log.unsafeVolatileTableId, s0.version, s0.metadata, s0.protocol, sourceMetadataPath = "") ) def read(): DataFrame = readStream( Some(getDefaultSchemaLocation.toString), startingVersion = Some(s0.version)) // Initialize checkpoint withSQLConf( DeltaSQLConf.DELTA_STREAMING_SCHEMA_TRACKING_METADATA_PATH_CHECK_ENABLED.key -> "false") { testStream(read())( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailable(), CheckAnswer(("0", "0")), StopStream ) f(read, log) } } } testWithoutAllowStreamRestart("unblock with sql conf") { def testStreamFlow( changeSchema: DeltaLog => Unit, schemaChangeType: String, columnChangeDetails: String, getConfKV: (Int, Long) => (String, String)): Unit = { withSimpleStreamingDf { (readDf, log) => val ckptHash = (getDefaultCheckpoint(log).toString + "/sources/0").hashCode changeSchema(log) val v1 = log.update().version addData(Seq(1))(log) // Encounter schema evolution exception testStream(readDf())( StartStream(checkpointLocation = getDefaultCheckpoint(log).toString), ProcessAllAvailableIgnoreError, CheckAnswer(Nil: _*), ExpectMetadataEvolutionException ) // Restart would fail due to SQL conf validation testStream(readDf())( StartStream(checkpointLocation = getDefaultCheckpoint(log).toString), ProcessAllAvailableIgnoreError, CheckAnswer(Nil: _*), expectSqlConfException(schemaChangeType, v1, columnChangeDetails, ckptHash) ) // Another restart still fails testStream(readDf())( StartStream(checkpointLocation = getDefaultCheckpoint(log).toString), ProcessAllAvailableIgnoreError, CheckAnswer(Nil: _*), expectSqlConfException(schemaChangeType, v1, columnChangeDetails, ckptHash) ) // With SQL Conf set we can move on val (k, v) = getConfKV(ckptHash, v1) withSQLConf(k -> v) { testStream(readDf())( StartStream(checkpointLocation = getDefaultCheckpoint(log).toString), ProcessAllAvailable() ) } } } // Test drop column Seq("allowSourceColumnRenameAndDrop", "allowSourceColumnDrop").foreach { allow => Seq( ( (log: DeltaLog) => { dropColumn("a")(log) // Revert the drop to test consecutive schema changes won't affect sql conf validation // the new column will show up with different physical name so it can trigger the // DROP COLUMN detection logic addColumn("a")(log) }, (ckptHash: Int, _: Long) => (s"${DeltaSQLConf.SQL_CONF_PREFIX}.streaming.$allow.ckpt_$ckptHash", "always") ), ( (log: DeltaLog) => { dropColumn("a")(log) // Ditto addColumn("a")(log) }, (ckptHash: Int, ver: Long) => (s"${DeltaSQLConf.SQL_CONF_PREFIX}.streaming.$allow.ckpt_$ckptHash", ver.toString) ) ).foreach { case (changeSchema, getConfKV) => testStreamFlow( changeSchema, schemaChangeType = "DROP COLUMN", columnChangeDetails = s"""Columns dropped: |'a' |""".stripMargin, getConfKV) } } // Test rename column Seq("allowSourceColumnRenameAndDrop", "allowSourceColumnRename").foreach { allow => Seq( ( (log: DeltaLog) => { renameColumn("b", "c")(log) }, (ckptHash: Int, _: Long) => (s"${DeltaSQLConf.SQL_CONF_PREFIX}.streaming.$allow.ckpt_$ckptHash", "always") ), ( (log: DeltaLog) => { renameColumn("b", "c")(log) }, (ckptHash: Int, ver: Long) => (s"${DeltaSQLConf.SQL_CONF_PREFIX}.streaming.$allow.ckpt_$ckptHash", ver.toString) ) ).foreach { case (changeSchema, getConfKV) => testStreamFlow( changeSchema, schemaChangeType = "RENAME COLUMN", columnChangeDetails = s"""Columns renamed: |'b' -> 'c' |""".stripMargin, getConfKV ) } } } testSchemaEvolution( "schema tracking interacting with unsafe escape flag") { implicit log => renameColumn("b", "c") // Even when schema location is provided, it won't be initialized because the unsafe // flag is turned on. val df = readStream( schemaLocation = Some(getDefaultSchemaLocation.toString), startingVersion = Some(1L)) withSQLConf( DeltaSQLConf.DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_COLUMN_MAPPING_SCHEMA_CHANGES.key -> "true") { testStream(df)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailable(), CheckAnswer((0 until 5).map(_.toString).map(i => (i, i)): _*) ) } assert(getDefaultSchemaLog().getCurrentTrackedMetadata.isEmpty) } testSchemaEvolution( "streaming with a column mapping upgrade", columnMapping = false) { implicit log => upgradeToNameMode val v0 = log.update().version renameColumn("b", "c") val v1 = log.update().version addData(5 until 10) // Start schema tracking from prior to upgrade // Initialize schema tracking log def readDf(): DataFrame = readStream( schemaLocation = Some(getDefaultSchemaLocation.toString), startingVersion = Some(1)) testStream(readDf())( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, CheckAnswer(Nil: _*), ExpectMetadataEvolutionExceptionFromInitialization ) assert { val schemaEntry = getDefaultSchemaLog().getCurrentTrackedMetadata.get schemaEntry.deltaCommitVersion == 1 && // no physical name entry !DeltaColumnMapping.hasPhysicalName(schemaEntry.dataSchema.head) } testStream(readDf())( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, CheckAnswer((0 until 5).map(_.toString).map(i => (i, i)): _*), ExpectMetadataEvolutionException ) assert { val schemaEntry = getDefaultSchemaLog().getCurrentTrackedMetadata.get // stopped at the upgrade commit schemaEntry.deltaCommitVersion == v0 && // now with physical name entry DeltaColumnMapping.hasPhysicalName(schemaEntry.dataSchema.head) } // Note that since we have schema merging, we won't need to fail again at the rename column // schema change, the rest of the data can be served altogether. testStream(readDf())( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailable(), CheckAnswer((5 until 10).map(_.toString).map(i => (i, i)): _*) ) assert { val schemaEntry = getDefaultSchemaLog().getCurrentTrackedMetadata.get // schema log updated implicitly schemaEntry.deltaCommitVersion == v1 && schemaEntry.dataSchema.fieldNames.sameElements(Array("a", "c")) } } test("backward-compat: latest version can read back older JSON") { val serialized = JsonUtils.toJson { OldPersistedSchema( tableId = "test", deltaCommitVersion = 1, StructType.fromDDL("a INT").json, StructType.fromDDL("a INT").json, sourceMetadataPath = "" ) } val schemaFromJson = PersistedMetadata.fromJson(serialized) assert(schemaFromJson == PersistedMetadata( tableId = "test", deltaCommitVersion = 1, StructType.fromDDL("a INT").json, StructType.fromDDL("a INT").json, sourceMetadataPath = "", tableConfigurations = None, protocolJson = None, previousMetadataSeqNum = None )) } test("forward-compat: older version can read back newer JSON") { val newSchema = PersistedMetadata( tableId = "test", deltaCommitVersion = 1, StructType.fromDDL("a INT").json, StructType.fromDDL("a INT").json, sourceMetadataPath = "/path", tableConfigurations = Some(Map("a" -> "b")), protocolJson = Some(Protocol(1, 2).json), previousMetadataSeqNum = Some(1L) ) assert { JsonUtils.fromJson[OldPersistedSchema](JsonUtils.toJson(newSchema)) == OldPersistedSchema( tableId = "test", deltaCommitVersion = 1, StructType.fromDDL("a INT").json, StructType.fromDDL("a INT").json, sourceMetadataPath = "/path" ) } } testSchemaEvolution("partition evolution") { implicit log => // Same schema but different partition overwriteSchema(log.update().schema, partitionColumns = Seq("a")) val v0 = log.update().version addData(5 until 10) overwriteSchema(log.update().schema, partitionColumns = Seq("b")) val v1 = log.update().version def readDf: DataFrame = readStream(schemaLocation = Some(getDefaultSchemaLocation.toString), startingVersion = Some(1), // ignoreDeletes because overwriteSchema would generate RemoveFiles. ignoreDeletes = Some(true)) // Init schema log testStream(readDf)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), AwaitTerminationIgnoreError, CheckAnswer(Nil: _*), ExpectMetadataEvolutionExceptionFromInitialization ) // Latest schema in schema log has been updated assert(getDefaultSchemaLog().getLatestMetadata.exists(_.deltaCommitVersion == 1)) // Process the first batch before overwrite testStream(readDf)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, CheckAnswer((0 until 5).map(_.toString).map(i => (i, i)): _*), ExpectMetadataEvolutionException ) assert(getDefaultSchemaLog().getLatestMetadata.exists(_.deltaCommitVersion == v0)) // Process until the next overwrite testStream(readDf)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, CheckAnswer( // TODO: since we did an overwrite, the previous RemoveFiles are also captured, but they are // using the old physical schema, we cannot read them back correctly. This is a corner case // with schema overwrite + CDC, although technically CDC should not worry about overwrite // because that means the downstream table needs to be truncated after applying CDC. // Note that since we support reuse physical name across overwrite, the value of partition // can still be read. (if (isCdcTest) (-1 until 5).map(_.toString).map(i => (null, i)) else Nil) ++ (5 until 10).map(_.toString).map(i => (i, i)): _*), ExpectMetadataEvolutionException ) assert(getDefaultSchemaLog().getLatestMetadata.exists(_.deltaCommitVersion == v1)) } testSchemaEvolution("schema log replace current", columnMapping = false) { implicit log => withSQLConf( DeltaSQLConf.DELTA_STREAMING_SCHEMA_TRACKING_METADATA_PATH_CHECK_ENABLED.key -> "false") { // Schema log's schema is respected val schemaLog = getDefaultSchemaLog() val s0 = PersistedMetadata(log.unsafeVolatileTableId, 0, makeMetadata( new StructType().add("a", StringType, true) .add("b", StringType, true) .add("c", StringType, true), partitionSchema = new StructType() ), log.update().protocol, sourceMetadataPath = "" ) // The `replaceCurrent` is noop because there is no previous schema. schemaLog.writeNewMetadata(s0, replaceCurrent = true) assert(schemaLog.getCurrentTrackedMetadata.contains(s0)) assert(schemaLog.getPreviousTrackedMetadata.isEmpty) val s1 = s0.copy(deltaCommitVersion = 1L) schemaLog.writeNewMetadata(s1) assert(schemaLog.getCurrentTrackedMetadata.contains(s1)) assert(schemaLog.getPreviousTrackedMetadata.contains(s0)) val s2 = s1.copy(deltaCommitVersion = 2L) schemaLog.writeNewMetadata(s2, replaceCurrent = true) assert(schemaLog.getCurrentTrackedMetadata.contains( s2.copy(previousMetadataSeqNum = Some(0L)))) assert(schemaLog.getPreviousTrackedMetadata.contains(s0)) val s3 = s2.copy(deltaCommitVersion = 3L) schemaLog.writeNewMetadata(s3, replaceCurrent = true) assert(schemaLog.getCurrentTrackedMetadata.contains( s3.copy(previousMetadataSeqNum = Some(0L)))) assert(schemaLog.getPreviousTrackedMetadata.contains(s0)) val s4 = s3.copy(deltaCommitVersion = 4L) schemaLog.writeNewMetadata(s4) assert(schemaLog.getCurrentTrackedMetadata.contains(s4)) assert(schemaLog.getPreviousTrackedMetadata.contains( s3.copy(previousMetadataSeqNum = Some(0L)))) val s5 = s4.copy(deltaCommitVersion = 5L) schemaLog.writeNewMetadata(s5, replaceCurrent = true) assert(schemaLog.getCurrentTrackedMetadata.contains( s5.copy(previousMetadataSeqNum = Some(3L)))) assert(schemaLog.getPreviousTrackedMetadata.contains( s3.copy(previousMetadataSeqNum = Some(0L)))) } } } // Needs to be top-level for serialization to work. case class OldPersistedSchema( tableId: String, deltaCommitVersion: Long, dataSchemaJson: String, partitionSchemaJson: String, sourceMetadataPath: String ) class DeltaSourceSchemaEvolutionNameColumnMappingSuite extends StreamingSchemaEvolutionSuiteBase with DeltaColumnMappingEnableNameMode { override def isCdcTest: Boolean = false } class DeltaSourceSchemaEvolutionIdColumnMappingSuite extends StreamingSchemaEvolutionSuiteBase with DeltaColumnMappingEnableIdMode { override def isCdcTest: Boolean = false } trait CDCStreamingSchemaEvolutionSuiteBase extends StreamingSchemaEvolutionSuiteBase { override def isCdcTest: Boolean = true import testImplicits._ // This test will generate AddCDCFiles test("CDC streaming with schema evolution") { withTempDir { dir => spark.range(10).toDF("id").write.format("delta").save(dir.getCanonicalPath) implicit val log: DeltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) { withTable("merge_source") { spark.range(10).filter(_ % 2 == 0) .toDF("id").withColumn("age", lit("string")) .createOrReplaceTempView("data") spark.sql(s"CREATE TABLE merge_source USING delta AS SELECT * FROM data") // Use merge to trigger schema evolution as well (add column age) withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { spark.sql( s""" |MERGE INTO delta.`${log.dataPath}` t |USING merge_source s |ON t.id = s.id |WHEN MATCHED | THEN UPDATE SET * |WHEN NOT MATCHED | THEN INSERT * |""".stripMargin) } } } val v1 = log.update().version def readDf: DataFrame = readStream(schemaLocation = Some(getDefaultSchemaLocation.toString), startingVersion = Some(0)) // Init schema log testStream(readDf)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, CheckAnswer(Nil: _*), ExpectMetadataEvolutionExceptionFromInitialization ) assert(getDefaultSchemaLog().getCurrentTrackedMetadata.get.deltaCommitVersion == 0L) // Streaming CDC until the MERGE invoked schema change testStream(readDf)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, // The first 10 inserts CheckAnswer((0L until 10L): _*), ExpectMetadataEvolutionException ) assert(getDefaultSchemaLog().getCurrentTrackedMetadata.get.deltaCommitVersion == v1 && getDefaultSchemaLog().getCurrentTrackedMetadata.get.dataSchema.fieldNames.sameElements( Array("id", "age"))) // Streaming CDC of the MERGE testStream(readDf)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailable(), CheckAnswer( // odd numbers have UPDATE actions (preimage and postimage) (0L until 10L).filter(_ % 2 == 0).flatMap(i => Seq((i, null), (i, "string"))): _* ) ) } } testSchemaEvolution( "protocol and configuration evolution", columnMapping = false) { implicit log => // Updates table properties / protocol spark.sql( s""" |ALTER TABLE delta.`${log.dataPath}` |SET TBLPROPERTIES ( | 'delta.minReaderVersion' = 2, | 'delta.minWriterVersion' = 5 |) |""".stripMargin) val v1 = log.update().version addData(5 until 10) // Update just delta table property spark.sql( s""" |ALTER TABLE delta.`${log.dataPath}` |SET TBLPROPERTIES ( | 'delta.isolationLevel' = 'SERIALIZABLE' |) |""".stripMargin ) val v2 = log.update().version addData(10 until 13) // Update non-delta property won't need stream stop spark.sql( s""" |ALTER TABLE delta.`${log.dataPath}` |SET TBLPROPERTIES ( | 'hello' = 'its me' |) |""".stripMargin ) addData(13 until 15) def readDf: DataFrame = readStream(schemaLocation = Some(getDefaultSchemaLocation.toString), startingVersion = Some(1L)) // Init schema log testStream(readDf)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, CheckAnswer(Nil: _*), ExpectMetadataEvolutionExceptionFromInitialization ) assert(getDefaultSchemaLog().getCurrentTrackedMetadata.get.deltaCommitVersion == 1L) // Reaching the first protocol change testStream(readDf)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, CheckAnswer((0 until 5).map(_.toString).map(i => (i, i)): _*), ExpectMetadataEvolutionException ) assert(getDefaultSchemaLog().getCurrentTrackedMetadata.get.deltaCommitVersion == v1) assert(getDefaultSchemaLog().getCurrentTrackedMetadata.get.protocol.contains(Protocol(2, 5))) // Reaching the second property change testStream(readDf)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailableIgnoreError, CheckAnswer((5 until 10).map(_.toString).map(i => (i, i)): _*), ExpectMetadataEvolutionException ) assert(getDefaultSchemaLog().getCurrentTrackedMetadata.get.deltaCommitVersion == v2) assert(getDefaultSchemaLog().getCurrentTrackedMetadata.get.tableConfigurations .get.contains("delta.isolationLevel")) // The final property change won't stop stream because it's non delta testStream(readDf)( StartStream(checkpointLocation = getDefaultCheckpoint.toString), ProcessAllAvailable(), CheckAnswer((10 until 15).map(_.toString).map(i => (i, i)): _*) ) } } class DeltaSourceSchemaEvolutionCDCNameColumnMappingSuite extends CDCStreamingSchemaEvolutionSuiteBase with DeltaColumnMappingEnableNameMode { override def isCdcTest: Boolean = true } class DeltaSourceSchemaEvolutionCDCIdColumnMappingSuite extends CDCStreamingSchemaEvolutionSuiteBase with DeltaColumnMappingEnableIdMode { override def isCdcTest: Boolean = true } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.{File, FileInputStream, OutputStream, PrintWriter, StringWriter} import java.net.URI import java.sql.Timestamp import java.util.UUID import java.util.concurrent.TimeoutException import scala.concurrent.duration._ import scala.language.implicitConversions import org.apache.spark.sql.delta.DataFrameUtils import org.apache.spark.sql.delta.DeltaTestUtils.modifyCommitTimestamp import org.apache.spark.sql.delta.Relocated import org.apache.spark.sql.delta.actions.{AddFile, Protocol} import org.apache.spark.sql.delta.sources.{DeltaDataSource, DeltaSQLConf, DeltaSource, DeltaSourceOffset} import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.test.shims.StreamingTestShims.{MemoryStream, OffsetSeqLog} import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import org.apache.commons.io.FileUtils import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.hadoop.fs.{FileStatus, Path, RawLocalFileSystem} import org.scalatest.time.{Seconds, Span} import org.apache.spark.{SparkConf, SparkThrowable} import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.util.IntervalUtils import org.apache.spark.sql.execution.streaming._ import org.apache.spark.sql.functions.when import org.apache.spark.sql.streaming.{OutputMode, StreamingQuery, StreamingQueryException, Trigger} import org.apache.spark.sql.streaming.util.StreamManualClock import org.apache.spark.sql.types.{IntegerType, LongType, NullType, StringType, StructField, StructType} import org.apache.spark.unsafe.types.UTF8String import org.apache.spark.util.{ManualClock, Utils} class DeltaSourceSuite extends DeltaSourceSuiteBase with DeltaColumnMappingTestUtils with DeltaSQLCommandTest { import testImplicits._ def testNullTypeColumn(shouldDropNullTypeColumns: Boolean): Unit = { withTempPaths(3) { case Seq(sourcePath, sinkPath, checkpointPath) => withSQLConf( DeltaSQLConf.DELTA_STREAMING_CREATE_DATAFRAME_DROP_NULL_COLUMNS.key -> shouldDropNullTypeColumns.toString) { spark.sql("select CAST(null as VOID) as nullTypeCol, id from range(10)") .write .format("delta") .mode("append") .save(sourcePath.getCanonicalPath) def runStream() = { loadStreamWithOptions(sourcePath.getCanonicalPath, Map.empty) // Need to drop null type columns because it's not supported by the writer. .drop("nullTypeCol") .writeStream .option("checkpointLocation", checkpointPath.getCanonicalPath) .format("delta") .start(sinkPath.getCanonicalPath) .processAllAvailable() } if (shouldDropNullTypeColumns) { val e = intercept[StreamingQueryException] { runStream() } assert(e.getErrorClass == "STREAM_FAILED") // This assertion checks the schema of the source did not change while processing a batch. assert(e.getMessage.contains("assertion failed: Invalid batch: nullTypeCol")) } else { runStream() } } } } test("streaming delta source should not drop null columns") { testNullTypeColumn(shouldDropNullTypeColumns = false) } test("streaming delta source should drop null columns without feature flag") { testNullTypeColumn(shouldDropNullTypeColumns = true) } test("no schema should throw an exception") { withTempDir { inputDir => new File(inputDir, "_delta_log").mkdir() val e = intercept[AnalysisException] { loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) } for (msg <- Seq("Table schema is not set", "CREATE TABLE")) { assert(e.getMessage.contains(msg)) } } } test("disallow user specified schema") { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) withMetadata(deltaLog, StructType.fromDDL("value STRING")) val e = intercept[AnalysisException] { spark.readStream .schema(StructType.fromDDL("a INT, b STRING")) .format("delta") .load(inputDir.getCanonicalPath) } for ( msg <- Seq( "The schema provided for the source read doesn't match the schema of the Delta table") ) { assert(e.getMessage.contains(msg)) } val e2 = intercept[Exception] { spark.readStream .schema(StructType.fromDDL("value STRING")) .format("delta") .load(inputDir.getCanonicalPath) } assert(e2.getMessage.contains("does not support user-specified schema")) } } test("allow user specified schema if consistent: v1 source") { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) withMetadata(deltaLog, StructType.fromDDL("value STRING")) import org.apache.spark.sql.execution.datasources.DataSource // User-specified schema is allowed if it's consistent with the actual Delta table schema. // Here we use Spark internal APIs to trigger v1 source code path. That being said, we // are not fixing end-user behavior, but advanced Spark plugins. val v1DataSource = DataSource( spark, userSpecifiedSchema = Some(StructType.fromDDL("value STRING")), className = "delta", options = Map("path" -> inputDir.getCanonicalPath)) DataFrameUtils.ofRows(spark, Relocated.StreamingRelation(v1DataSource)) } } test("createSource should create source with empty or matching table schema provided") { withTempDir { tempDir => val path = tempDir.getCanonicalPath sql(s"CREATE TABLE delta.`$path` (id INT NOT NULL, name STRING) USING delta") val deltaSource = new DeltaDataSource() val parameters = Map("path" -> path) val metadataPath = tempDir.getCanonicalPath + "/_metadata" val tableSchema = StructType(Seq( StructField("id", IntegerType, false), StructField("name", StringType, true) )) val emptySchema = new StructType() val allowedCreationSchemas = Seq(emptySchema, tableSchema) for (schema <- allowedCreationSchemas) { val source = deltaSource.createSource( sqlContext, metadataPath = metadataPath, schema = Some(schema), providerName = "delta", parameters = parameters ) val actualSchema = source.asInstanceOf[DeltaSource].schema assert(actualSchema.fields.map(_.name).toSet == Set("id", "name")) } val conflictingSchemas = Seq( StructType(Seq( StructField("id", IntegerType, true) // missing field "name" )), StructType(Seq( StructField("id", IntegerType, false), StructField("name", StringType, true), StructField("age", IntegerType, true) // extra field )) ) for (schema <- conflictingSchemas) { val e = intercept[Exception] { deltaSource.createSource( sqlContext, metadataPath = metadataPath, schema = Some(schema), providerName = "delta", parameters = parameters ) } assert(e.getMessage.contains( "[DELTA_READ_SOURCE_SCHEMA_CONFLICT] " + "The schema provided for the source read doesn't match the schema of the Delta table")) } } } test("basic") { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) withMetadata(deltaLog, StructType.fromDDL("value STRING")) val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) .filter($"value" contains "keep") testStream(df)( AddToReservoir(inputDir, Seq("keep1", "keep2", "drop3").toDF), AssertOnQuery { q => q.processAllAvailable(); true }, CheckAnswer("keep1", "keep2"), StopStream, AddToReservoir(inputDir, Seq("drop4", "keep5", "keep6").toDF), StartStream(), AssertOnQuery { q => q.processAllAvailable(); true }, CheckAnswer("keep1", "keep2", "keep5", "keep6"), AddToReservoir(inputDir, Seq("keep7", "drop8", "keep9").toDF), AssertOnQuery { q => q.processAllAvailable(); true }, CheckAnswer("keep1", "keep2", "keep5", "keep6", "keep7", "keep9") ) } } test("initial snapshot ends at base index of next version") { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) withMetadata(deltaLog, StructType.fromDDL("value STRING")) // Add data before creating the stream, so that it becomes part of the initial snapshot. Seq("keep1", "keep2", "drop3").toDF.write .format("delta").mode("append").save(inputDir.getAbsolutePath) val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) .filter($"value" contains "keep") testStream(df)( AssertOnQuery { q => q.processAllAvailable(); true }, AssertOnQuery { q => val offset = q.committedOffsets.iterator.next()._2.asInstanceOf[DeltaSourceOffset] assert(offset.reservoirVersion === 2) assert(offset.index === DeltaSourceOffset.BASE_INDEX) true }, CheckAnswer("keep1", "keep2"), StopStream ) } } test("allow to change schema before starting a streaming query") { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 5).foreach { i => val v = Seq(i.toString).toDF("id") v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } withMetadata(deltaLog, StructType.fromDDL("id STRING, value STRING")) (5 until 10).foreach { i => val v = Seq(i.toString -> i.toString).toDF("id", "value") v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) val expected = ( (0 until 5).map(_.toString -> null) ++ (5 until 10).map(_.toString).map(x => x -> x) ).toDF("id", "value").collect() testStream(df)( AssertOnQuery { q => q.processAllAvailable(); true }, CheckAnswer(expected: _*) ) } } testQuietly("disallow to change schema after starting a streaming query") { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 5).foreach { i => val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) testStream(df)( AssertOnQuery { q => q.processAllAvailable(); true }, CheckAnswer((0 until 5).map(_.toString): _*), AssertOnQuery { _ => withMetadata(deltaLog, StructType.fromDDL("id int, value int")) true }, ExpectFailure[DeltaIllegalStateException](t => assert(t.getMessage.contains("Detected schema change"))) ) } } test("maxFilesPerTrigger") { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 5).foreach { i => val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } val q = loadStreamWithOptions( inputDir.getCanonicalPath, Map(DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION -> "1")) .writeStream .format("memory") .queryName("maxFilesPerTriggerTest") .start() try { q.processAllAvailable() val progress = q.recentProgress.filter(_.numInputRows != 0) assert(progress.length === 5) progress.foreach { p => assert(p.numInputRows === 1) } checkAnswer(sql("SELECT * from maxFilesPerTriggerTest"), (0 until 5).map(_.toString).toDF) } finally { q.stop() } } } test("maxFilesPerTrigger: metadata checkpoint") { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 20).foreach { i => val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } val q = loadStreamWithOptions( inputDir.getCanonicalPath, Map(DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION -> "1")) .writeStream .format("memory") .queryName("maxFilesPerTriggerTest") .start() try { q.processAllAvailable() val progress = q.recentProgress.filter(_.numInputRows != 0) assert(progress.length === 20) progress.foreach { p => assert(p.numInputRows === 1) } checkAnswer(sql("SELECT * from maxFilesPerTriggerTest"), (0 until 20).map(_.toString).toDF) } finally { q.stop() } } } test("maxFilesPerTrigger: change and restart") { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 10).foreach { i => val v = Seq(i.toString).toDF() v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } val q = loadStreamWithOptions( inputDir.getCanonicalPath, Map(DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION -> "1")) .writeStream .format("delta") .option("checkpointLocation", checkpointDir.getCanonicalPath) .start(outputDir.getCanonicalPath) try { q.processAllAvailable() val progress = q.recentProgress.filter(_.numInputRows != 0) assert(progress.length === 10) progress.foreach { p => assert(p.numInputRows === 1) } checkAnswer( spark.read.format("delta").load(outputDir.getAbsolutePath), (0 until 10).map(_.toString).toDF()) } finally { q.stop() } (10 until 20).foreach { i => val v = Seq(i.toString).toDF() v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } val q2 = loadStreamWithOptions( inputDir.getCanonicalPath, Map(DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION -> "2")) .writeStream .format("delta") .option("checkpointLocation", checkpointDir.getCanonicalPath) .start(outputDir.getCanonicalPath) try { q2.processAllAvailable() val progress = q2.recentProgress.filter(_.numInputRows != 0) assert(progress.length === 5) progress.foreach { p => assert(p.numInputRows === 2) } checkAnswer( spark.read.format("delta").load(outputDir.getAbsolutePath), (0 until 20).map(_.toString).toDF()) } finally { q2.stop() } } } testQuietly("maxFilesPerTrigger: invalid parameter") { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) withMetadata(deltaLog, StructType.fromDDL("value STRING")) Seq(0, -1, "string").foreach { invalidMaxFilesPerTrigger => val e = intercept[StreamingQueryException] { loadStreamWithOptions( inputDir.getCanonicalPath, Map(DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION -> invalidMaxFilesPerTrigger.toString)) .writeStream .format("console") .start() .processAllAvailable() } assert(e.getCause.isInstanceOf[IllegalArgumentException]) for (msg <- Seq("Invalid", DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION, "positive")) { assert(e.getCause.getMessage.contains(msg)) } } } } test("maxFilesPerTrigger: ignored when using Trigger.Once") { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 5).foreach { i => val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } def runTriggerOnceAndVerifyResult(expected: Seq[Int]): Unit = { val q = loadStreamWithOptions( inputDir.getCanonicalPath, Map(DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION -> "1")) .writeStream .format("memory") .trigger(Trigger.Once) .queryName("triggerOnceTest") .start() try { assert(q.awaitTermination(streamingTimeout.toMillis)) assert(q.recentProgress.count(_.numInputRows != 0) == 1) // only one trigger was run checkAnswer(sql("SELECT * from triggerOnceTest"), expected.map(_.toString).toDF) } finally { q.stop() } } runTriggerOnceAndVerifyResult(0 until 5) // Write more data and start a second batch. (5 until 10).foreach { i => val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } // Verify we can see all of latest data. runTriggerOnceAndVerifyResult(0 until 10) } } test("maxFilesPerTrigger: Trigger.AvailableNow respects read limits") { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaLog = DeltaLog.forTable(spark, inputDir) // Write versions 0, 1, 2, 3, 4. (0 to 4).foreach { i => val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } val stream = loadStreamWithOptions( inputDir.getCanonicalPath, Map(DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION -> "1")) .writeStream .format("delta") .option("checkpointLocation", checkpointDir.getCanonicalPath) .trigger(Trigger.AvailableNow) .queryName("maxFilesPerTriggerTest") var q = stream.start(outputDir.getCanonicalPath) try { assert(q.awaitTermination(streamingTimeout.toMillis)) assert(q.recentProgress.length === 5) // The first 5 versions each contain one file. They are processed as part of the initial // snapshot (reservoir version 4) with one index per file. (0 to 3).foreach { i => val p = q.recentProgress(i) assert(p.numInputRows === 1) val endOffset = JsonUtils.fromJson[DeltaSourceOffset](p.sources.head.endOffset) assert(endOffset == DeltaSourceOffset( endOffset.reservoirId, reservoirVersion = 4, index = i, isInitialSnapshot = true)) } // The last batch ends at the base index of the next reservoir version (5). val p4 = q.recentProgress(4) assert(p4.numInputRows === 1) val endOffset = JsonUtils.fromJson[DeltaSourceOffset](p4.sources.head.endOffset) assert(endOffset == DeltaSourceOffset( endOffset.reservoirId, reservoirVersion = 5, index = DeltaSourceOffset.BASE_INDEX, isInitialSnapshot = false)) checkAnswer( sql(s"SELECT * from delta.`${outputDir.getCanonicalPath}`"), (0 to 4).map(_.toString).toDF) // Restarting the stream should immediately terminate with no progress because no more data q = stream.start(outputDir.getCanonicalPath) assert(q.awaitTermination(streamingTimeout.toMillis)) // The streaming engine always reports one batch, even if it's empty. assert(q.recentProgress.length === 1) assert(q.recentProgress(0).sources.head.startOffset == q.recentProgress(0).sources.head.endOffset) // Write versions 5, 6, 7. (5 to 7).foreach { i => val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } q = stream.start(outputDir.getCanonicalPath) assert(q.awaitTermination(streamingTimeout.toMillis)) // These versions are processed one by one outside the initial snapshot. assert(q.recentProgress.length === 3) (5 to 7).foreach { i => val p = q.recentProgress(i - 5) assert(p.numInputRows === 1) val endOffset = JsonUtils.fromJson[DeltaSourceOffset](p.sources.head.endOffset) assert(endOffset == DeltaSourceOffset( endOffset.reservoirId, reservoirVersion = i + 1, index = DeltaSourceOffset.BASE_INDEX, isInitialSnapshot = false)) } // Restarting the stream should immediately terminate with no progress because no more data q = stream.start(outputDir.getCanonicalPath) assert(q.awaitTermination(streamingTimeout.toMillis)) assert(q.recentProgress.length === 1) assert(q.recentProgress(0).sources.head.startOffset == q.recentProgress(0).sources.head.endOffset) } finally { q.stop() } } } test("Trigger.AvailableNow with an empty table") { withTempDirs { (inputDir, outputDir, checkpointDir) => sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (value STRING) USING delta") val stream = loadStreamWithOptions( inputDir.getCanonicalPath, Map(DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION -> "1")) .writeStream .format("memory") .option("checkpointLocation", checkpointDir.getCanonicalPath) .trigger(Trigger.AvailableNow) .queryName("emptyTableTriggerAvailableNow") var q = stream.start() try { assert(q.awaitTermination(10000)) val progress = q.recentProgress.filter(_.numInputRows != 0) assert(progress.length === 0) } finally { q.stop() } } } test("maxBytesPerTrigger: process at least one file") { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 5).foreach { i => val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } val q = loadStreamWithOptions( inputDir.getCanonicalPath, Map(DeltaOptions.MAX_BYTES_PER_TRIGGER_OPTION -> "1b")) .writeStream .format("memory") .option("checkpointLocation", checkpointDir.getCanonicalPath) .queryName("maxBytesPerTriggerTest") .start() try { q.processAllAvailable() val progress = q.recentProgress.filter(_.numInputRows != 0) assert(progress.length === 5) progress.foreach { p => assert(p.numInputRows === 1) } checkAnswer(sql("SELECT * from maxBytesPerTriggerTest"), (0 until 5).map(_.toString).toDF) } finally { q.stop() } } } test("maxBytesPerTrigger: metadata checkpoint") { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 20).foreach { i => val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } val q = loadStreamWithOptions( inputDir.getCanonicalPath, Map(DeltaOptions.MAX_BYTES_PER_TRIGGER_OPTION -> "1b")) .writeStream .format("memory") .option("checkpointLocation", checkpointDir.getCanonicalPath) .queryName("maxBytesPerTriggerTest") .start() try { q.processAllAvailable() val progress = q.recentProgress.filter(_.numInputRows != 0) assert(progress.length === 20) progress.foreach { p => assert(p.numInputRows === 1) } checkAnswer(sql("SELECT * from maxBytesPerTriggerTest"), (0 until 20).map(_.toString).toDF) } finally { q.stop() } } } test("maxBytesPerTrigger: change and restart") { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 10).foreach { i => val v = Seq(i.toString).toDF() v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } val q = loadStreamWithOptions( inputDir.getCanonicalPath, Map(DeltaOptions.MAX_BYTES_PER_TRIGGER_OPTION -> "1b")) .writeStream .format("delta") .option("checkpointLocation", checkpointDir.getCanonicalPath) .start(outputDir.getCanonicalPath) try { q.processAllAvailable() val progress = q.recentProgress.filter(_.numInputRows != 0) assert(progress.length === 10) progress.foreach { p => assert(p.numInputRows === 1) } checkAnswer( spark.read.format("delta").load(outputDir.getAbsolutePath), (0 until 10).map(_.toString).toDF()) } finally { q.stop() } (10 until 20).foreach { i => val v = Seq(i.toString).toDF() v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } val q2 = loadStreamWithOptions( inputDir.getCanonicalPath, Map(DeltaOptions.MAX_BYTES_PER_TRIGGER_OPTION -> "100g")) .writeStream .format("delta") .option("checkpointLocation", checkpointDir.getCanonicalPath) .start(outputDir.getCanonicalPath) try { q2.processAllAvailable() val progress = q2.recentProgress.filter(_.numInputRows != 0) assert(progress.length === 1) progress.foreach { p => assert(p.numInputRows === 10) } checkAnswer( spark.read.format("delta").load(outputDir.getAbsolutePath), (0 until 20).map(_.toString).toDF()) } finally { q2.stop() } } } testQuietly("maxBytesPerTrigger: invalid parameter") { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) withMetadata(deltaLog, StructType.fromDDL("value STRING")) Seq(0, -1, "string").foreach { invalidMaxBytesPerTrigger => val e = intercept[StreamingQueryException] { loadStreamWithOptions( inputDir.getCanonicalPath, Map(DeltaOptions.MAX_BYTES_PER_TRIGGER_OPTION -> invalidMaxBytesPerTrigger.toString)) .writeStream .format("console") .option("checkpointLocation", checkpointDir.getCanonicalPath) .start() .processAllAvailable() } assert(e.getCause.isInstanceOf[IllegalArgumentException]) for (msg <- Seq("Invalid", DeltaOptions.MAX_BYTES_PER_TRIGGER_OPTION, "size")) { assert(e.getCause.getMessage.contains(msg)) } } } } test("maxBytesPerTrigger: Trigger.AvailableNow respects read limits") { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaLog = DeltaLog.forTable(spark, inputDir) (0 until 5).foreach { i => val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } val stream = loadStreamWithOptions( inputDir.getCanonicalPath, Map(DeltaOptions.MAX_BYTES_PER_TRIGGER_OPTION -> "1b")) .writeStream .format("delta") .option("checkpointLocation", checkpointDir.getCanonicalPath) .trigger(Trigger.AvailableNow) .queryName("maxBytesPerTriggerTest") var q = stream.start(outputDir.getCanonicalPath) try { assert(q.awaitTermination(streamingTimeout.toMillis)) val progress = q.recentProgress.filter(_.numInputRows != 0) assert(progress.length === 5) progress.foreach { p => assert(p.numInputRows === 1) } checkAnswer( sql(s"SELECT * from delta.`${outputDir.getCanonicalPath}`"), (0 until 5).map(_.toString).toDF) // Restarting the stream should immediately terminate with no progress because no more data q = stream.start(outputDir.getCanonicalPath) assert(q.awaitTermination(streamingTimeout.toMillis)) assert(q.recentProgress.length === 1) assert(q.recentProgress(0).sources.head.startOffset == q.recentProgress(0).sources.head.endOffset) } finally { q.stop() } } } test("maxBytesPerTrigger: max bytes and max files together") { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 5).foreach { i => val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } // should process a file at a time due to MAX_FILES_PER_TRIGGER_OPTION val q = loadStreamWithOptions( inputDir.getCanonicalPath, Map( DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION -> "1", DeltaOptions.MAX_BYTES_PER_TRIGGER_OPTION -> "100gb")) .writeStream .format("memory") .queryName("maxBytesPerTriggerTest") .start() try { q.processAllAvailable() val progress = q.recentProgress.filter(_.numInputRows != 0) assert(progress.length === 5) progress.foreach { p => assert(p.numInputRows === 1) } checkAnswer(sql("SELECT * from maxBytesPerTriggerTest"), (0 until 5).map(_.toString).toDF) } finally { q.stop() } val q2 = loadStreamWithOptions( inputDir.getCanonicalPath, Map( DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION -> "2", DeltaOptions.MAX_BYTES_PER_TRIGGER_OPTION -> "1b")) .writeStream .format("memory") .queryName("maxBytesPerTriggerTest") .start() try { q2.processAllAvailable() val progress = q2.recentProgress.filter(_.numInputRows != 0) assert(progress.length === 5) progress.foreach { p => assert(p.numInputRows === 1) } checkAnswer(sql("SELECT * from maxBytesPerTriggerTest"), (0 until 5).map(_.toString).toDF) } finally { q2.stop() } } } // DeltaSourceOffset unit tests have been moved to DeltaSourceOffsetSuite. testQuietly("streaming query should fail when table is deleted and recreated with new id") { withTempDir { inputDir => val tablePath = new Path(inputDir.toURI) val deltaLog = DeltaLog.forTable(spark, tablePath) withMetadata(deltaLog, StructType.fromDDL("value STRING")) val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) .filter($"value" contains "keep") testStream(df)( AddToReservoir(inputDir, Seq("keep1", "keep2", "drop3").toDF), AssertOnQuery { q => q.processAllAvailable(); true }, CheckAnswer("keep1", "keep2"), StopStream, AssertOnQuery { _ => Utils.deleteRecursively(inputDir) // This test deletes and recreates a table at the same path. The InMemoryCommitCoordinator // keys on logPath, so stale coordinator state from the old table must be cleared before // the new table is created. In production, UC handles table lifecycle management, so // explicit coordinator cleanup is not needed. if (catalogOwnedDefaultCreationEnabledInTests) { deleteCatalogOwnedTableFromCommitCoordinator(tablePath) } val deltaLog = DeltaLog.forTable(spark, tablePath) // All Delta tables in tests use the same tableId by default. Here we pass a new tableId // to simulate a new table creation in production withMetadata(deltaLog, StructType.fromDDL("value STRING"), tableId = Some("tableId-1234")) true }, StartStream(), ExpectFailure[DeltaIllegalStateException] { e => for (msg <- Seq("delete", "checkpoint", "restart")) { assert(e.getMessage.contains(msg)) } } ) } } test("excludeRegex works and doesn't mess up offsets across restarts - parquet version") { withTempDir { inputDir => val chk = new File(inputDir, "_checkpoint").toString def excludeReTest(s: Option[String], expected: String*): Unit = { val options = s.map(regex => Map(DeltaOptions.EXCLUDE_REGEX_OPTION -> regex)).getOrElse(Map.empty) val df = loadStreamWithOptions(inputDir.getCanonicalPath, options).groupBy('value).count testStream(df, OutputMode.Complete())( StartStream(checkpointLocation = chk), AssertOnQuery { sq => sq.processAllAvailable(); true }, CheckLastBatch(expected.map((_, 1)): _*), StopStream ) } val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) def writeFile(name: String, content: String): AddFile = { FileUtils.write(new File(inputDir, name), content) // CatalogManaged (CCv2) tables auto-enable row tracking, which requires numRecords in // AddFile stats to assign base row IDs. Without numRecords, the commit fails with // DELTA_ROW_ID_ASSIGNMENT_WITHOUT_STATS. AddFile(name, Map.empty, content.length, System.currentTimeMillis(), dataChange = true, stats = s"""{"numRecords": 1}""") } def commitFiles(files: AddFile*): Unit = { deltaLog.startTransaction().commit(files, DeltaOperations.ManualUpdate) } Seq("abc", "def").toDF().write.format("delta").save(inputDir.getAbsolutePath) commitFiles( writeFile("batch1-ignore-file1", "ghi"), writeFile("batch1-ignore-file2", "jkl") ) excludeReTest(Some("ignore"), "abc", "def") } } testQuietly("excludeRegex throws good error on bad regex pattern") { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) withMetadata(deltaLog, StructType.fromDDL("value STRING")) val e = intercept[StreamingQueryException] { loadStreamWithOptions( inputDir.getCanonicalPath, Map(DeltaOptions.EXCLUDE_REGEX_OPTION -> "[abc")) .writeStream .format("console") .start() .awaitTermination() }.cause assert(e.isInstanceOf[IllegalArgumentException]) assert(e.getMessage.contains(DeltaOptions.EXCLUDE_REGEX_OPTION)) } } test("a fast writer should not starve a Delta source") { val deltaPath = Utils.createTempDir().getCanonicalPath val checkpointPath = Utils.createTempDir().getCanonicalPath val writer = spark.readStream .format("rate") .load() .writeStream .format("delta") .option("checkpointLocation", checkpointPath) .start(deltaPath) try { eventually(timeout(streamingTimeout)) { assert(spark.read.format("delta").load(deltaPath).count() > 0) } val testTableName = "delta_source_test" withTable(testTableName) { val reader = loadStreamWithOptions(deltaPath, Map.empty) .writeStream .format("memory") .queryName(testTableName) .start() try { eventually(timeout(streamingTimeout)) { assert(spark.table(testTableName).count() > 0) } } finally { reader.stop() } } } finally { writer.stop() } } test("start from corrupt checkpoint") { withTempDir { inputDir => val path = inputDir.getAbsolutePath for (i <- 1 to 5) { Seq(i).toDF("id").write.mode("append").format("delta").save(path) } val deltaLog = DeltaLog.forTable(spark, path) deltaLog.checkpoint() Seq(6).toDF("id").write.mode("append").format("delta").save(path) val checkpoints = new File(deltaLog.logPath.toUri).listFiles() .filter(f => FileNames.isCheckpointFile(new Path(f.getAbsolutePath))) checkpoints.last.delete() val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) testStream(df)( AssertOnQuery { q => q.processAllAvailable(); true }, CheckAnswer(1, 2, 3, 4, 5, 6), StopStream ) } } test("SC-11561: can consume new data without update") { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) withMetadata(deltaLog, StructType.fromDDL("value STRING")) val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) // clear the cache so that the writer creates its own DeltaLog instead of reusing the reader's DeltaLog.clearCache() (0 until 3).foreach { i => Seq(i.toString).toDF("value") .write.mode("append").format("delta").save(deltaLog.dataPath.toString) } // check that reader consumed new data without updating its DeltaLog testStream(df)( AssertOnQuery { q => q.processAllAvailable(); true }, CheckAnswer("0", "1", "2") ) assert(deltaLog.snapshot.version == 0) (3 until 5).foreach { i => Seq(i.toString).toDF("value") .write.mode("append").format("delta").save(deltaLog.dataPath.toString) } // check that reader consumed new data without update despite checkpoint val writersLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) writersLog.checkpoint() testStream(df)( AssertOnQuery { q => q.processAllAvailable(); true }, CheckAnswer("0", "1", "2", "3", "4") ) assert(deltaLog.snapshot.version == 0) } } test("startingVersion specific version: new commits arrive after stream initialization") { withTempDirs { (inputDir, outputDir, checkpointDir) => // Add version 0 and version 1 Seq(1, 2, 3).toDF("value").write.format("delta").save(inputDir.getCanonicalPath) Seq(4, 5, 6).toDF("value").write .format("delta").mode("append").save(inputDir.getCanonicalPath) // Start streaming from version 1 val df = loadStreamWithOptions( inputDir.getCanonicalPath, Map( "startingVersion" -> "1", DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION -> "1" ) ) val q = df.writeStream .format("delta") .option("checkpointLocation", checkpointDir.getCanonicalPath) .start(outputDir.getCanonicalPath) try { // Process version 1 only q.processAllAvailable() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Seq(4, 5, 6).toDF("value")) // Add version 2 and version 3 (after snapshotAtSourceInit was captured) Seq(7, 8, 9).toDF("value").write .format("delta").mode("append").save(inputDir.getCanonicalPath) Seq(10, 11, 12).toDF("value").write .format("delta").mode("append").save(inputDir.getCanonicalPath) q.processAllAvailable() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), (4 to 12).toDF("value")) } finally { q.stop() } } } test( "can delete old files of a snapshot without update" ) { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) withMetadata(deltaLog, StructType.fromDDL("value STRING")) val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) // clear the cache so that the writer creates its own DeltaLog instead of reusing the reader's DeltaLog.clearCache() val clock = new ManualClock(System.currentTimeMillis()) val writersLog = DeltaLog.forTable(spark, new Path(inputDir.toURI), clock) (0 until 3).foreach { i => Seq(i.toString).toDF("value") .write.mode("append").format("delta").save(inputDir.getCanonicalPath) } // Create a checkpoint so that logs before checkpoint can be expired and deleted writersLog.checkpoint() // This isn't stable, but it shouldn't change during the test. val tahoeId = deltaLog.unsafeVolatileTableId testStream(df)( StartStream(Trigger.ProcessingTime("10 seconds"), new StreamManualClock), AdvanceManualClock(10 * 1000L), CheckLastBatch("0", "1", "2"), Assert { val defaultLogRetentionMillis = DeltaConfigs.getMilliSeconds( IntervalUtils.safeStringToInterval( UTF8String.fromString(DeltaConfigs.LOG_RETENTION.defaultValue))) clock.advance(defaultLogRetentionMillis + 100000000L) // Delete all logs before checkpoint writersLog.cleanUpExpiredLogs(writersLog.snapshot) // Check that the first few log files have been deleted val logPath = new File(inputDir, "_delta_log") val logVersions = logPath.listFiles().map(_.getName) .filter(_.endsWith(".json")) .map(_.stripSuffix(".json").toInt) !logVersions.contains(0) && !logVersions.contains(1) }, Assert { (3 until 5).foreach { i => Seq(i.toString).toDF("value") .write.mode("append").format("delta").save(inputDir.getCanonicalPath) } true }, // can process new data without update, despite that previous log files have been deleted AdvanceManualClock(10 * 1000L), AdvanceManualClock(10 * 1000L), CheckNewAnswer("3", "4") ) assert(deltaLog.snapshot.version == 0) } } test("Delta sources don't write offsets with null json") { withTempDirs { (inputDir, outputDir, checkpointDir) => Seq(1, 2, 3).toDF("x").write.format("delta").save(inputDir.toString) val df = loadStreamWithOptions(inputDir.toString, Map.empty) val stream = df.writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) .start(outputDir.toString) stream.processAllAvailable() val offsetFile = checkpointDir.toString + "/offsets/0" // Make sure JsonUtils doesn't serialize it as null val deltaSourceOffsetLine = scala.io.Source.fromFile(offsetFile).getLines.toSeq.last val deltaSourceOffset = JsonUtils.fromJson[DeltaSourceOffset](deltaSourceOffsetLine) assert(deltaSourceOffset.json != null, "Delta sources shouldn't write null json field") // Make sure OffsetSeqLog won't choke on the offset we wrote withTempDir { logPath => new OffsetSeqLog(spark, logPath.toString) { val offsetSeq = this.deserialize(new FileInputStream(offsetFile)) val out = new OutputStream() { override def write(b: Int): Unit = { } } this.serialize(offsetSeq, out) } } stream.stop() } } test("Delta source advances with non-data inserts and generates empty dataframe for " + "non-data operations") { withTempDirs { (inputDir, outputDir, checkpointDir) => // Version 0 Seq(1L, 2L, 3L).toDF("x").write.format("delta").save(inputDir.toString) val df = loadStreamWithOptions(inputDir.toString, Map.empty) val stream = df .writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) .foreachBatch( (outputDf: DataFrame, bid: Long) => { // Apart from first batch, rest of batches work with non-data operations // for which we expect an empty dataframe to be generated. if (bid > 0) { assert(outputDf.isEmpty) } outputDf .write .format("delta") .mode("append") .save(outputDir.toString) } ) .start() val deltaLog = DeltaLog.forTable(spark, inputDir.toString) def expectLatestOffset(offset: DeltaSourceOffset) { val lastOffset = DeltaSourceOffset( deltaLog.unsafeVolatileTableId, SerializedOffset(stream.lastProgress.sources.head.endOffset) ) assert(lastOffset == offset) } try { stream.processAllAvailable() expectLatestOffset(DeltaSourceOffset( deltaLog.unsafeVolatileTableId, reservoirVersion = 1, DeltaSourceOffset.BASE_INDEX, isInitialSnapshot = false)) deltaLog.startTransaction().commit(Seq(), DeltaOperations.ManualUpdate) stream.processAllAvailable() expectLatestOffset(DeltaSourceOffset( deltaLog.unsafeVolatileTableId, reservoirVersion = 2, DeltaSourceOffset.BASE_INDEX, isInitialSnapshot = false)) deltaLog.startTransaction().commit(Seq(), DeltaOperations.ManualUpdate) stream.processAllAvailable() expectLatestOffset(DeltaSourceOffset( deltaLog.unsafeVolatileTableId, reservoirVersion = 3, DeltaSourceOffset.BASE_INDEX, isInitialSnapshot = false)) } finally { stream.stop() } } } test("Rate limited Delta source advances with non-data inserts") { withTempDirs { (inputDir, outputDir, checkpointDir) => // Version 0 Seq(1L, 2L, 3L).toDF("x").write.format("delta").save(inputDir.toString) val df = loadStreamWithOptions(inputDir.toString, Map.empty) val stream = df.writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) .option("maxFilesPerTrigger", 2) .start(outputDir.toString) try { val deltaLog = DeltaLog.forTable(spark, inputDir.toString) def waitForOffset(offset: DeltaSourceOffset) { eventually(timeout(streamingTimeout)) { val lastOffset = DeltaSourceOffset( deltaLog.unsafeVolatileTableId, SerializedOffset(stream.lastProgress.sources.head.endOffset) ) assert(lastOffset == offset) } } // Process the initial snapshot (version 0) and end up at the start of version 1 which // does not exist yet. stream.processAllAvailable() waitForOffset(DeltaSourceOffset( deltaLog.unsafeVolatileTableId, 1, DeltaSourceOffset.BASE_INDEX, false)) // Add Versions 1, 2, 3, and 4 for(i <- 1 to 4) { deltaLog.startTransaction().commit(Seq(), DeltaOperations.ManualUpdate) } // The manual commits don't have any files in them, but they do have indexes: BASE_INDEX // and END_INDEX. Neither of those indexes are counted for rate limiting. We end up at // v4[END_INDEX] which is then rounded up to v5[BASE_INDEX] even though v5 does not exist // yet. stream.processAllAvailable() waitForOffset( DeltaSourceOffset(deltaLog.unsafeVolatileTableId, 5, DeltaSourceOffset.BASE_INDEX, false)) // Add Version 5 deltaLog.startTransaction().commit(Seq(), DeltaOperations.ManualUpdate) // The stream progresses to v5[END_INDEX] which is rounded up to v6[BASE_INDEX]. (In prior // versions of the code we did not have END_INDEX. In that case the stream would not have // moved forward from v5, because there were no indexes after v5[BASE_INDEX]. stream.processAllAvailable() waitForOffset( DeltaSourceOffset(deltaLog.unsafeVolatileTableId, 6, DeltaSourceOffset.BASE_INDEX, false)) } finally { stream.stop() } } } testQuietly("Delta sources should verify the protocol reader version") { withTempDir { tempDir => spark.range(0).write.format("delta").save(tempDir.getCanonicalPath) val df = loadStreamWithOptions(tempDir.getCanonicalPath, Map.empty) val stream = df.writeStream .format("console") .start() try { stream.processAllAvailable() val deltaLog = DeltaLog.forTable(spark, tempDir) deltaLog.store.write( FileNames.unsafeDeltaFile(deltaLog.logPath, deltaLog.snapshot.version + 1), // Write a large reader version to fail the streaming query Iterator(Protocol(minReaderVersion = Int.MaxValue).json), overwrite = false, deltaLog.newDeltaHadoopConf()) // The streaming query should fail because its version is too old val e = intercept[StreamingQueryException] { stream.processAllAvailable() } val cause = e.getCause val sw = new StringWriter() cause.printStackTrace(new PrintWriter(sw)) assert( cause.isInstanceOf[InvalidProtocolVersionException] || // When coordinated commits are enabled, the following assertion error coming from // CoordinatedCommitsUtils.getCommitCoordinatorClient may get hit (cause.isInstanceOf[AssertionError] && e.getCause.getMessage.contains("coordinated commits table feature is not supported")), s"Caused by: ${sw.toString}") } finally { stream.stop() } } } /** Generate commits with the given timestamp in millis. */ private def generateCommits(location: String, commits: Long*): Unit = { val deltaLog = DeltaLog.forTable(spark, location) var startVersion = deltaLog.snapshot.version + 1 commits.foreach { ts => val rangeStart = startVersion * 10 val rangeEnd = rangeStart + 10 spark.range(rangeStart, rangeEnd).write.format("delta").mode("append").save(location) modifyCommitTimestamp(deltaLog, startVersion, ts) startVersion += 1 } } private implicit def durationToLong(duration: FiniteDuration): Long = { duration.toMillis } /** Disable log cleanup to avoid deleting logs we are testing. */ protected def disableLogCleanup(tablePath: String): Unit = { sql(s"alter table delta.`$tablePath` " + s"set tblproperties (${DeltaConfigs.ENABLE_EXPIRED_LOG_CLEANUP.key} = false)") } testQuietly("startingVersion") { withTempDir { tableDir => val tablePath = tableDir.getCanonicalPath val start = 1594795800000L generateCommits(tablePath, start, start + 20.minutes) def testStartingVersion(startingVersion: Long): Unit = { val df = loadStreamWithOptions( tablePath, Map("startingVersion" -> startingVersion.toString)) val q = df.writeStream .format("memory") .queryName("startingVersion_test") .start() try { q.processAllAvailable() } finally { q.stop() } } for ((startingVersion, expected) <- Seq( 0 -> (0 until 20), 1 -> (10 until 20)) ) { withTempView("startingVersion_test") { testStartingVersion(startingVersion) checkAnswer( spark.table("startingVersion_test"), expected.map(_.toLong).toDF()) } } assert(intercept[StreamingQueryException] { testStartingVersion(-1) }.getMessage.contains("Invalid value '-1' for option 'startingVersion'")) assert(intercept[StreamingQueryException] { testStartingVersion(2) }.getMessage.contains("Cannot time travel Delta table to version 2")) // Create a checkpoint at version 2 and delete version 0 disableLogCleanup(tablePath) val deltaLog = DeltaLog.forTable(spark, tablePath) assert(deltaLog.update().version == 2) deltaLog.checkpoint() new File(FileNames.unsafeDeltaFile(deltaLog.logPath, 0).toUri).delete() // Cannot start from version 0 assert(intercept[StreamingQueryException] { testStartingVersion(0) }.getMessage.contains("Cannot time travel Delta table to version 0")) // Can start from version 1 even if it's not recreatable // TODO: currently we would error out if we couldn't construct the snapshot to check column // mapping enable tables. Unblock this once we roll out the proper semantics. withStreamingReadOnColumnMappingTableEnabled { withTempView("startingVersion_test") { testStartingVersion(1L) checkAnswer( spark.table("startingVersion_test"), (10 until 20).map(_.toLong).toDF()) } } } } // Row tracking forces actions to appear after AddFiles within commits. This will verify that // we correctly skip processed commits, even when an AddFile is not the last action within a // commit. Seq(true, false).foreach { withRowTracking => testQuietly(s"startingVersion should be ignored when restarting from a checkpoint, " + s"withRowTracking = $withRowTracking") { withTempDirs { (inputDir, outputDir, checkpointDir) => val start = 1594795800000L withSQLConf( DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey -> withRowTracking.toString) { generateCommits(inputDir.getCanonicalPath, start, start + 20.minutes) } def testStartingVersion( startingVersion: Long, checkpointLocation: String = checkpointDir.getCanonicalPath): Unit = { val q = loadStreamWithOptions( inputDir.getCanonicalPath, Map("startingVersion" -> startingVersion.toString)) .writeStream .format("delta") .option("checkpointLocation", checkpointLocation) .start(outputDir.getCanonicalPath) try { q.processAllAvailable() } finally { q.stop() } } testStartingVersion(1L) checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), (10 until 20).map(_.toLong).toDF()) // Add two new commits generateCommits(inputDir.getCanonicalPath, start + 40.minutes) disableLogCleanup(inputDir.getCanonicalPath) val deltaLog = DeltaLog.forTable(spark, inputDir.getCanonicalPath) assert(deltaLog.update().version == 3) deltaLog.checkpoint() // Make the streaming query move forward. When we restart here, we still need to touch // `DeltaSource.getStartingVersion` because the engine will call `getBatch` // that was committed (start is None) during the restart. testStartingVersion(1L) checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), (10 until 30).map(_.toLong).toDF()) // Add one commit and delete version 0 and version 1 generateCommits(inputDir.getCanonicalPath, start + 60.minutes) (0 to 1).foreach { v => new File(FileNames.unsafeDeltaFile(deltaLog.logPath, v).toUri).delete() } // Although version 1 has been deleted, restarting the query should still work as we have // processed files in version 1. // In other words, query restart should ignore "startingVersion" // TODO: currently we would error out if we couldn't construct the snapshot to check column // mapping enable tables. Unblock this once we roll out the proper semantics. withStreamingReadOnColumnMappingTableEnabled { testStartingVersion(1L) checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), // the gap caused by "alter table" ((10 until 30) ++ (40 until 50)).map(_.toLong).toDF()) // But if we start a new query, it should fail. val newCheckpointDir = Utils.createTempDir() try { assert(intercept[StreamingQueryException] { testStartingVersion(1L, newCheckpointDir.getCanonicalPath) }.getMessage.contains("[2, 4]")) } finally { Utils.deleteRecursively(newCheckpointDir) } } } } } testQuietly("startingTimestamp") { withTempDir { tableDir => val tablePath = tableDir.getCanonicalPath val start = 1594795800000L // 2020-07-14 23:50:00 PDT generateCommits(tablePath, start, start + 20.minutes) def testStartingTimestamp(startingTimestamp: String): Unit = { val q = loadStreamWithOptions( tablePath, Map("startingTimestamp" -> startingTimestamp)) .writeStream .format("memory") .queryName("startingTimestamp_test") .start() try { q.processAllAvailable() } finally { q.stop() } } for ((startingTimestamp, expected) <- Seq( "2020-07-14" -> (0 until 20), "2020-07-14 23:40:00" -> (0 until 20), "2020-07-14 23:50:00" -> (0 until 20), // the timestamp of version 0 "2020-07-14 23:50:01" -> (10 until 20), "2020-07-15" -> (10 until 20), "2020-07-15 00:00:00" -> (10 until 20), "2020-07-15 00:10:00" -> (10 until 20)) // the timestamp of version 1 ) { withTempView("startingTimestamp_test") { testStartingTimestamp(startingTimestamp) checkAnswer( spark.table("startingTimestamp_test"), expected.map(_.toLong).toDF()) } } assert(intercept[StreamingQueryException] { testStartingTimestamp("2020-07-15 00:10:01") }.getMessage.contains("The provided timestamp (2020-07-15 00:10:01.0) " + "is after the latest version")) assert(intercept[StreamingQueryException] { testStartingTimestamp("2020-07-16") }.getMessage.contains("The provided timestamp (2020-07-16 00:00:00.0) " + "is after the latest version")) assert(intercept[StreamingQueryException] { testStartingTimestamp("i am not a timestamp") }.getMessage.contains("The provided timestamp ('i am not a timestamp') " + "cannot be converted to a valid timestamp")) // With non-strict parsing this produces null when casted to a timestamp and then parses // to 1970-01-01 (unix time 0). withSQLConf(DeltaSQLConf.DELTA_TIME_TRAVEL_STRICT_TIMESTAMP_PARSING.key -> "false") { withTempView("startingTimestamp_test") { testStartingTimestamp("i am not a timestamp") checkAnswer( spark.table("startingTimestamp_test"), (0L until 20L).toDF()) } } // Create a checkpoint at version 2 and delete version 0 disableLogCleanup(tablePath) val deltaLog = DeltaLog.forTable(spark, tablePath) assert(deltaLog.update().version == 2) deltaLog.checkpoint() new File(FileNames.unsafeDeltaFile(deltaLog.logPath, 0).toUri).delete() // Can start from version 1 even if it's not recreatable // TODO: currently we would error out if we couldn't construct the snapshot to check column // mapping enable tables. Unblock this once we roll out the proper semantics. withStreamingReadOnColumnMappingTableEnabled { withTempView("startingTimestamp_test") { testStartingTimestamp("2020-07-14") checkAnswer( spark.table("startingTimestamp_test"), (10 until 20).map(_.toLong).toDF()) } } } } testQuietly("startingVersion and startingTimestamp are both set") { withTempDir { tableDir => val tablePath = tableDir.getCanonicalPath generateCommits(tablePath, 0) val q = loadStreamWithOptions( tablePath, Map("startingVersion" -> "0", "startingTimestamp" -> "2020-07-15")) .writeStream .format("console") .start() try { assert(intercept[StreamingQueryException] { q.processAllAvailable() }.getMessage.contains("Please either provide 'startingVersion' or 'startingTimestamp'")) } finally { q.stop() } } } test("startingVersion: user defined start works with mergeSchema") { withTempDir { inputDir => withTempView("startingVersionTest") { spark.range(10) .write .format("delta") .mode("append") .save(inputDir.getCanonicalPath) // Change schema at version 1 spark.range(10, 20) .withColumn("id2", 'id) .write .option("mergeSchema", "true") .format("delta") .mode("append") .save(inputDir.getCanonicalPath) // Change schema at version 2 spark.range(20, 30) .withColumn("id2", 'id) .withColumn("id3", 'id) .write .option("mergeSchema", "true") .format("delta") .mode("append") .save(inputDir.getCanonicalPath) // check answer from version 1 val q = loadStreamWithOptions( inputDir.getCanonicalPath, Map("startingVersion" -> "1") ).writeStream .format("memory") .queryName("startingVersionTest") .start() try { q.processAllAvailable() checkAnswer( sql("select * from startingVersionTest"), ((10 until 20).map(x => (x.toLong, x.toLong, "null")) ++ (20 until 30).map(x => (x.toLong, x.toLong, x.toString))) .toDF("id", "id2", "id3") .selectExpr( "id", "id2", "CASE WHEN id3 = 'null' THEN NULL ELSE cast(id3 as long) END as id3") ) } finally { q.stop() } } } } test("startingVersion latest") { withTempDir { dir => withTempView("startingVersionTest") { val path = dir.getAbsolutePath spark.range(0, 10).write.format("delta").save(path) val q = loadStreamWithOptions(path, Map("startingVersion" -> "latest")) .writeStream .format("memory") .queryName("startingVersionLatest") .start() try { // Starting from latest shouldn't include any data at first, even the most recent version. q.processAllAvailable() checkAnswer(sql("select * from startingVersionLatest"), Seq.empty) // After we add some batches the stream should continue as normal. spark.range(10, 15).write.format("delta").mode("append").save(path) q.processAllAvailable() checkAnswer(sql("select * from startingVersionLatest"), (10 until 15).map(Row(_))) spark.range(15, 20).write.format("delta").mode("append").save(path) spark.range(20, 25).write.format("delta").mode("append").save(path) q.processAllAvailable() checkAnswer(sql("select * from startingVersionLatest"), (10 until 25).map(Row(_))) } finally { q.stop() } } } } test("startingVersion latest defined before started") { withTempDir { dir => withTempView("startingVersionTest") { val path = dir.getAbsolutePath spark.range(0, 10).write.format("delta").save(path) // Define the stream, but don't start it, before a second write. The startingVersion // latest should be resolved when the query *starts*, so there'll be no data even though // some was added after the stream was defined. val streamDef = loadStreamWithOptions(path, Map("startingVersion" -> "latest")) .writeStream .format("memory") .queryName("startingVersionLatest") spark.range(10, 20).write.format("delta").mode("append").save(path) val q = streamDef.start() try { q.processAllAvailable() checkAnswer(sql("select * from startingVersionLatest"), Seq.empty) spark.range(20, 25).write.format("delta").mode("append").save(path) q.processAllAvailable() checkAnswer(sql("select * from startingVersionLatest"), (20 until 25).map(Row(_))) } finally { q.stop() } } } } test("startingVersion latest works on defined but empty table") { withTempDir { dir => withTempView("startingVersionTest") { val path = dir.getAbsolutePath spark.range(0).write.format("delta").save(path) val streamDef = loadStreamWithOptions(path, Map("startingVersion" -> "latest")) .writeStream .format("memory") .queryName("startingVersionLatest") val q = streamDef.start() try { q.processAllAvailable() checkAnswer(sql("select * from startingVersionLatest"), Seq.empty) spark.range(0, 5).write.format("delta").mode("append").save(path) q.processAllAvailable() checkAnswer(sql("select * from startingVersionLatest"), (0 until 5).map(Row(_))) } finally { q.stop() } } } } test("startingVersion latest calls update when starting") { withTempDir { dir => withTempView("startingVersionTest") { val path = dir.getAbsolutePath spark.range(0).write.format("delta").save(path) val streamDef = loadStreamWithOptions( path, Map("startingVersion" -> "latest") ).writeStream .format("memory") .queryName("startingVersionLatest") val log = DeltaLog.forTable(spark, path) val originalSnapshot = log.snapshot val timestamp = System.currentTimeMillis() // We write out some new data, and then do a dirty reflection hack to produce an un-updated // Delta log. The stream should still update when started and not produce any data. spark.range(10).write.format("delta").mode("append").save(path) // The field is actually declared in the SnapshotManagement trait, but because traits don't // exist in the JVM DeltaLog is where it ends up in reflection. val snapshotField = classOf[DeltaLog].getDeclaredField("currentSnapshot") snapshotField.setAccessible(true) snapshotField.set(log, CapturedSnapshot(originalSnapshot, timestamp)) val q = streamDef.start() try { q.processAllAvailable() checkAnswer(sql("select * from startingVersionLatest"), Seq.empty) } finally { q.stop() } } } } test("startingVersion should work with rate time") { withTempDir { dir => withTempView("startingVersionWithRateLimit") { val path = dir.getAbsolutePath // Create version 0 and version 1 and each version has two files spark.range(0, 5).repartition(2).write.mode("append").format("delta").save(path) spark.range(5, 10).repartition(2).write.mode("append").format("delta").save(path) val q = loadStreamWithOptions( path, Map( "startingVersion" -> "1", "maxFilesPerTrigger" -> "1" ) ).writeStream .format("memory") .queryName("startingVersionWithRateLimit") .start() try { q.processAllAvailable() checkAnswer(sql("select * from startingVersionWithRateLimit"), (5 until 10).map(Row(_))) val id = DeltaLog.forTable(spark, path).snapshot.metadata.id val endOffsets = q.recentProgress .map(_.sources(0).endOffset) .map(offsetJson => DeltaSourceOffset( id, SerializedOffset(offsetJson) )) assert(endOffsets.toList == DeltaSourceOffset(id, 1, 0, isInitialSnapshot = false) // When we reach the end of version 1, we will jump to version 2 with index -1 :: DeltaSourceOffset(id, 2, DeltaSourceOffset.BASE_INDEX, isInitialSnapshot = false) :: Nil) } finally { q.stop() } } } } testQuietly("deltaSourceIgnoreChangesError contains removeFile, version, tablePath") { withTempDirs { (inputDir, outputDir, checkpointDir) => Seq(1, 2, 3).toDF("x").write.format("delta").save(inputDir.toString) val df = loadStreamWithOptions(inputDir.toString, Map.empty) df.writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) .start(outputDir.toString) .processAllAvailable() // Overwrite values, causing AddFile & RemoveFile actions to be triggered Seq(1, 2, 3).toDF("x") .write .mode("overwrite") .format("delta") .save(inputDir.toString) val e = intercept[StreamingQueryException] { val q = df.writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) // DeltaOptions.IGNORE_CHANGES_OPTION is false by default .start(outputDir.toString) try { q.processAllAvailable() } finally { q.stop() } } assert(e.getCause.isInstanceOf[UnsupportedOperationException]) assert(e.getCause.getMessage.contains( "This is currently not supported. If this is going to happen regularly and you are okay" + " to skip changes, set the option 'skipChangeCommits' to 'true'." )) assert(e.getCause.getMessage.contains("for example")) assert(e.getCause.getMessage.contains("version")) assert(e.getCause.getMessage.matches(s".*$inputDir.*")) } } testQuietly("deltaSourceIgnoreDeleteError contains removeFile, version, tablePath") { withTempDirs { (inputDir, outputDir, checkpointDir) => Seq(1, 2, 3).toDF("x").write.format("delta").save(inputDir.toString) val df = loadStreamWithOptions(inputDir.toString, Map.empty) df.writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) .start(outputDir.toString) .processAllAvailable() // Delete the table, causing only RemoveFile (not AddFile) actions to be triggered io.delta.tables.DeltaTable.forPath(spark, inputDir.getAbsolutePath).delete() val e = intercept[StreamingQueryException] { val q = df.writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) // DeltaOptions.IGNORE_DELETES_OPTION is false by default .start(outputDir.toString) try { q.processAllAvailable() } finally { q.stop() } } assert(e.getCause.isInstanceOf[UnsupportedOperationException]) assert(e.getCause.getMessage.contains( "This is currently not supported. If you'd like to ignore deletes, set the option " + "'ignoreDeletes' to 'true'.")) assert(e.getCause.getMessage.contains("for example")) assert(e.getCause.getMessage.contains("version")) assert(e.getCause.getMessage.matches(s".*$inputDir.*")) } } test("streaming with ignoreDeletes = true skips delete-only commits") { withTempDirs { (inputDir, outputDir, checkpointDir) => // Write initial data Seq(1, 2, 3).toDF("x").write.format("delta").save(inputDir.toString) val df = loadStreamWithOptions( inputDir.toString, Map(DeltaOptions.IGNORE_DELETES_OPTION -> "true", "startingVersion" -> "0")) val q = df.writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) .start(outputDir.toString) try { q.processAllAvailable() checkAnswer( spark.read.format("delta").load(outputDir.toString), Seq(1, 2, 3).map(Row(_))) // Delete all rows: produces only RemoveFile actions io.delta.tables.DeltaTable.forPath(spark, inputDir.getAbsolutePath).delete() // Append new data after the delete Seq(4, 5).toDF("x").write.format("delta").mode("append").save(inputDir.toString) q.processAllAvailable() // The delete commit should be silently skipped; only inserts are processed checkAnswer( spark.read.format("delta").load(outputDir.toString), Seq(1, 2, 3, 4, 5).map(Row(_))) } finally { q.stop() } } } testQuietly("streaming with ignoreDeletes = true still fails on change commits") { withTempDirs { (inputDir, outputDir, checkpointDir) => Seq(1, 2, 3).toDF("x").write.format("delta").save(inputDir.toString) val df = loadStreamWithOptions( inputDir.toString, Map(DeltaOptions.IGNORE_DELETES_OPTION -> "true")) df.writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) .start(outputDir.toString) .processAllAvailable() // Overwrite produces both AddFile and RemoveFile actions (a change commit) Seq(4, 5, 6).toDF("x") .write .mode("overwrite") .format("delta") .save(inputDir.toString) val e = intercept[StreamingQueryException] { val q = df.writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) .start(outputDir.toString) try { q.processAllAvailable() } finally { q.stop() } } assert(e.getCause.isInstanceOf[UnsupportedOperationException]) assert(e.getCause.getMessage.contains( "This is currently not supported. If this is going to happen regularly and you are okay" + " to skip changes, set the option 'skipChangeCommits' to 'true'." )) } } test("streaming with skipChangeCommits = true skips both delete and change commits") { withTempDirs { (inputDir, outputDir, checkpointDir) => Seq(1, 2, 3).toDF("x").write.format("delta").save(inputDir.toString) val df = loadStreamWithOptions( inputDir.toString, Map(DeltaOptions.SKIP_CHANGE_COMMITS_OPTION -> "true")) val q = df.writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) .start(outputDir.toString) try { q.processAllAvailable() checkAnswer( spark.read.format("delta").load(outputDir.toString), Seq(1, 2, 3).map(Row(_))) // Delete all rows: produces only RemoveFile actions (delete-only commit) io.delta.tables.DeltaTable.forPath(spark, inputDir.getAbsolutePath).delete() Seq(4, 5).toDF("x").write.format("delta").mode("append").save(inputDir.toString) // Overwrite produces both AddFile and RemoveFile actions (change commit) Seq(6, 7, 8).toDF("x") .write .mode("overwrite") .format("delta") .save(inputDir.toString) Seq(9, 10).toDF("x").write.format("delta").mode("append").save(inputDir.toString) q.processAllAvailable() // Both the delete and overwrite commits are silently skipped; only inserts are processed checkAnswer( spark.read.format("delta").load(outputDir.toString), Seq(1, 2, 3, 4, 5, 9, 10).map(Row(_))) } finally { q.stop() } } } test("fail on data loss - starting from missing files") { withTempDirs { (srcData, targetData, chkLocation) => def addData(): Unit = { spark.range(10).write.format("delta").mode("append").save(srcData.getCanonicalPath) } addData() val df = loadStreamWithOptions(srcData.getCanonicalPath, Map.empty) val q = df.writeStream.format("delta") .option("checkpointLocation", chkLocation.getCanonicalPath) .start(targetData.getCanonicalPath) q.processAllAvailable() q.stop() addData() addData() addData() val srcLog = DeltaLog.forTable(spark, srcData) // Create a checkpoint so that we can create a snapshot without json files before version 3 srcLog.checkpoint() // Delete the first file assert(new File(FileNames.unsafeDeltaFile(srcLog.logPath, 1).toUri).delete()) val e = intercept[StreamingQueryException] { val q = df.writeStream.format("delta") .option("checkpointLocation", chkLocation.getCanonicalPath) .start(targetData.getCanonicalPath) q.processAllAvailable() } assert(e.getCause.getMessage === DeltaErrors.failOnDataLossException(1L, 2L).getMessage) } } test("fail on data loss - gaps of files") { withTempDirs { (srcData, targetData, chkLocation) => def addData(): Unit = { spark.range(10).write.format("delta").mode("append").save(srcData.getCanonicalPath) } addData() val df = loadStreamWithOptions(srcData.getCanonicalPath, Map.empty) val q = df.writeStream.format("delta") .option("checkpointLocation", chkLocation.getCanonicalPath) .start(targetData.getCanonicalPath) q.processAllAvailable() q.stop() addData() addData() addData() val srcLog = DeltaLog.forTable(spark, srcData) // Create a checkpoint so that we can create a snapshot without json files before version 3 srcLog.checkpoint() // Delete the second file assert(new File(FileNames.unsafeDeltaFile(srcLog.logPath, 2).toUri).delete()) val e = intercept[StreamingQueryException] { val q = df.writeStream.format("delta") .option("checkpointLocation", chkLocation.getCanonicalPath) .start(targetData.getCanonicalPath) q.processAllAvailable() } assert(e.getCause.getMessage === DeltaErrors.failOnDataLossException(2L, 3L).getMessage) } } test("fail on data loss - starting from missing files with option off") { withTempDirs { (srcData, targetData, chkLocation) => def addData(): Unit = { spark.range(10).write.format("delta").mode("append").save(srcData.getCanonicalPath) } addData() val df = loadStreamWithOptions( srcData.getCanonicalPath, Map("failOnDataLoss" -> "false")) val q = df.writeStream.format("delta") .option("checkpointLocation", chkLocation.getCanonicalPath) .start(targetData.getCanonicalPath) q.processAllAvailable() q.stop() addData() addData() addData() val srcLog = DeltaLog.forTable(spark, srcData) // Create a checkpoint so that we can create a snapshot without json files before version 3 srcLog.checkpoint() // Delete the first file assert(new File(FileNames.unsafeDeltaFile(srcLog.logPath, 1).toUri).delete()) val q2 = df.writeStream.format("delta") .option("checkpointLocation", chkLocation.getCanonicalPath) .start(targetData.getCanonicalPath) q2.processAllAvailable() q2.stop() assert(spark.read.format("delta").load(targetData.getCanonicalPath).count() === 30) } } test("fail on data loss - gaps of files with option off") { withTempDirs { (srcData, targetData, chkLocation) => def addData(): Unit = { spark.range(10).write.format("delta").mode("append").save(srcData.getCanonicalPath) } addData() val df = loadStreamWithOptions( srcData.getCanonicalPath, Map("failOnDataLoss" -> "false")) val q = df.writeStream.format("delta") .option("checkpointLocation", chkLocation.getCanonicalPath) .start(targetData.getCanonicalPath) q.processAllAvailable() q.stop() addData() addData() addData() val srcLog = DeltaLog.forTable(spark, srcData) // Create a checkpoint so that we can create a snapshot without json files before version 3 srcLog.checkpoint() // Delete the second file assert(new File(FileNames.unsafeDeltaFile(srcLog.logPath, 2).toUri).delete()) val q2 = df.writeStream.format("delta") .option("checkpointLocation", chkLocation.getCanonicalPath) .start(targetData.getCanonicalPath) q2.processAllAvailable() q2.stop() assert(spark.read.format("delta").load(targetData.getCanonicalPath).count() === 30) } } test("make sure that the delta sources works fine") { withTempDirs { (inputDir, outputDir, checkpointDir) => import io.delta.implicits._ Seq(1, 2, 3).toDF().write.delta(inputDir.toString) val df = spark.readStream.delta(inputDir.toString) val stream = df.writeStream .option("checkpointLocation", checkpointDir.toString) .delta(outputDir.toString) stream.processAllAvailable() stream.stop() val writtenStreamDf = spark.read.delta(outputDir.toString) val expectedRows = Seq(Row(1), Row(2), Row(3)) checkAnswer(writtenStreamDf, expectedRows) } } test("should not attempt to read a non exist version") { withTempDirs { (inputDir1, inputDir2, checkpointDir) => spark.range(1, 2).write.format("delta").save(inputDir1.getCanonicalPath) spark.range(1, 2).write.format("delta").save(inputDir2.getCanonicalPath) def startQuery(): StreamingQuery = { val df1 = loadStreamWithOptions(inputDir1.getCanonicalPath, Map.empty) val df2 = loadStreamWithOptions(inputDir2.getCanonicalPath, Map.empty) df1.union(df2).writeStream .format("noop") .option("checkpointLocation", checkpointDir.getCanonicalPath) .start() } var q = startQuery() try { q.processAllAvailable() // current offsets: // source1: DeltaSourceOffset(reservoirVersion=1,index=0,isInitialSnapshot=true) // source2: DeltaSourceOffset(reservoirVersion=1,index=0,isInitialSnapshot=true) spark.range(1, 2).write.format("delta").mode("append").save(inputDir1.getCanonicalPath) spark.range(1, 2).write.format("delta").mode("append").save(inputDir2.getCanonicalPath) q.processAllAvailable() // current offsets: // source1: DeltaSourceOffset(reservoirVersion=2,index=-1,isInitialSnapshot=false) // source2: DeltaSourceOffset(reservoirVersion=2,index=-1,isInitialSnapshot=false) // Note: version 2 doesn't exist in source1 spark.range(1, 2).write.format("delta").mode("append").save(inputDir2.getCanonicalPath) q.processAllAvailable() // current offsets: // source1: DeltaSourceOffset(reservoirVersion=2,index=-1,isInitialSnapshot=false) // source2: DeltaSourceOffset(reservoirVersion=3,index=-1,isInitialSnapshot=false) // Note: version 2 doesn't exist in source1 q.stop() // Restart the query. It will call `getBatch` on the previous two offsets of `source1` which // are both DeltaSourceOffset(reservoirVersion=2,index=-1,isInitialSnapshot=false) // As version 2 doesn't exist, we should not try to load version 2 in this case. q = startQuery() q.processAllAvailable() } finally { q.stop() } } } test("self union a Delta table should pass the catalog table assert") { withTable("self_union_delta") { spark.range(10).write.format("delta").saveAsTable("self_union_delta") val df = spark.readStream.format("delta").table("self_union_delta") val q = df.union(df).writeStream.format("noop").start() try { q.processAllAvailable() } finally { q.stop() } } } test("ES-445863: delta source should not hang or reprocess data when using AvailableNow") { withTempDirs { (inputDir, outputDir, checkpointDir) => def runQuery(): Unit = { val q = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) // Require a partition filter. The max index of files matching the partition filter must // be less than the number of files in the second commit. .where("part = 0") .writeStream .format("delta") .trigger(Trigger.AvailableNow) .option("checkpointLocation", checkpointDir.getCanonicalPath) .start(outputDir.getCanonicalPath) try { if (!q.awaitTermination(60000)) { throw new TimeoutException("the query didn't stop in 60 seconds") } } finally { q.stop() } } spark.range(0, 1) .selectExpr("id", "id as part") .repartition(10) .write .partitionBy("part") .format("delta") .mode("append") .save(inputDir.getCanonicalPath) runQuery() spark.range(1, 10) .selectExpr("id", "id as part") .repartition(9) .write .partitionBy("part") .format("delta") .mode("append") .save(inputDir.getCanonicalPath) runQuery() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), Row(0, 0) :: Nil) } } test("add column: restarting with new DataFrame should recover") { withTempDirs { (inputDir, outputDir, checkpointDir) => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 10).foreach { i => val v = Seq(i.toString).toDF("id") v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } def startQuery(): StreamingQuery = { loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) .writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .option("mergeSchema", "true") // use delta sink because we need to check the result .format("delta") .start(outputDir.getCanonicalPath) } var q = startQuery() try { q.processAllAvailable() checkAnswer( spark.read.format("delta").load(outputDir.getCanonicalPath), (0 until 10).map(i => Row(i.toString))) // Clear delta log cache DeltaLog.clearCache() // Change the table schema using the non-cached `DeltaLog` to mimic the case that the // table schema change happens on a different cluster withMetadata(deltaLog, StructType.fromDDL("id STRING, newcol STRING")) Seq(("10", "a")).toDF("id", "newcol") .write.mode("append").format("delta").save(deltaLog.dataPath.toString) // The streaming query should fail when detecting a schema change val e = intercept[StreamingQueryException] { q.processAllAvailable() } assert(e.getMessage.contains("Detected schema change")) // Restarting the query with a new DataFrame should recover from the schema change q = startQuery() q.processAllAvailable() // Verify the output schema includes the new column val outputDf = spark.read.format("delta").load(outputDir.getCanonicalPath) assert(outputDf.schema.fieldNames.toSet == Set("id", "newcol"), s"Expected schema with {id, newcol} but got ${outputDf.schema.fieldNames.mkString(", ")}") checkAnswer(outputDf, (0 until 10).map(i => Row(i.toString, null)) :+ Row("10", "a")) } finally { q.stop() } } } test("add column: restarting with stale DataFrame should fail") { withTempDir { inputDir => withTempDir { checkpointDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 2).foreach { i => val v = Seq(i.toString).toDF("id") v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) // First run: stream detects schema change and fails testStream(df)( StartStream(checkpointLocation = checkpointDir.getCanonicalPath), ProcessAllAvailable(), CheckAnswer("0", "1"), Execute { _ => withMetadata(deltaLog, StructType.fromDDL("id STRING, newcol STRING")) Seq(("2", "a")).toDF("id", "newcol") .write.mode("append").format("delta").save(deltaLog.dataPath.toString) }, ExpectFailure[DeltaIllegalStateException](t => assert(t.getMessage.contains("Detected schema change"))), Execute { q => assert(!q.isActive) } ) // Restarting with the same DataFrame cannot recover from column addition because the // plan's read schema is fixed at analysis time. User must create a new DataFrame. val e = intercept[StreamingQueryException] { val q = df.writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("noop") .start() try q.processAllAvailable() finally q.stop() } // V1 fails with an internal batch schema assertion ("Invalid batch"), // V2 fails with DELTA_STREAMING_SCHEMA_MISMATCH_ON_RESTART. assert( e.getMessage.contains("Invalid batch") || e.getMessage.contains("DELTA_STREAMING_SCHEMA_MISMATCH_ON_RESTART"), s"Expected schema mismatch error but got: ${e.getMessage}" ) } } } test("relax nullability: restarting with new DataFrame should recover") { withTempDirs { (inputDir, outputDir, checkpointDir) => sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` " + "(id STRING NOT NULL) USING DELTA") val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 2).foreach { i => Seq(i.toString).toDF("id") .write.mode("append").format("delta").save(deltaLog.dataPath.toString) } def startQuery(): StreamingQuery = { loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) .writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .option("mergeSchema", "true") // use delta sink because we need to check the result .format("delta") .start(outputDir.getCanonicalPath) } var q = startQuery() try { q.processAllAvailable() // Clear delta log cache DeltaLog.clearCache() // Relax nullability from "id STRING NOT NULL" to "id STRING" using the non-cached // `DeltaLog` to mimic the case that the schema change happens on a different cluster withMetadata(deltaLog, StructType.fromDDL("id STRING")) // Insert a null row Seq(Option.empty[String]).toDF("id") .write.mode("append").format("delta").save(deltaLog.dataPath.toString) // The streaming query should fail when detecting a schema change val e = intercept[StreamingQueryException] { q.processAllAvailable() } assert(e.getMessage.contains("Detected schema change")) // Restarting the query with a new DataFrame should recover from the schema change q = startQuery() q.processAllAvailable() val outputDf = spark.read.format("delta").load(outputDir.getCanonicalPath) assert(outputDf.schema("id").nullable, "Expected 'id' column to be nullable after relaxing nullability") checkAnswer(outputDf.orderBy("id"), Seq(Row(null), Row("0"), Row("1"))) } finally { q.stop() } } } test("relax nullability: restarting with stale DataFrame should recover") { withTempDirs { (inputDir, outputDir, checkpointDir) => sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` " + "(id STRING NOT NULL) USING DELTA") val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 2).foreach { i => Seq(i.toString).toDF("id") .write.mode("append").format("delta").save(deltaLog.dataPath.toString) } val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) var q = df.writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .option("mergeSchema", "true") // use delta sink because we need to check the result .format("delta") .start(outputDir.getCanonicalPath) try { q.processAllAvailable() // Clear delta log cache DeltaLog.clearCache() // Relax nullability from "id STRING NOT NULL" to "id STRING" using the non-cached // `DeltaLog` to mimic the case that the schema change happens on a different cluster withMetadata(deltaLog, StructType.fromDDL("id STRING")) // Insert a null row Seq(Option.empty[String]).toDF("id") .write.mode("append").format("delta").save(deltaLog.dataPath.toString) // The streaming query should fail when detecting a schema change val e = intercept[StreamingQueryException] { q.processAllAvailable() } assert(e.getMessage.contains("Detected schema change")) // Restarting the query with the stale DataFrame should still recover q = df.writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .option("mergeSchema", "true") .format("delta") .start(outputDir.getCanonicalPath) q.processAllAvailable() val outputDf = spark.read.format("delta").load(outputDir.getCanonicalPath) assert(outputDf.schema("id").nullable, "Expected 'id' column to be nullable after relaxing nullability") checkAnswer(outputDf.orderBy("id"), Seq(Row(null), Row("0"), Row("1"))) } finally { q.stop() } } } test("type widening: restarting with new DataFrame should recover") { withSQLConf( DeltaSQLConf.DELTA_TYPE_WIDENING_ENABLE_STREAMING_SCHEMA_TRACKING.key -> "false", DeltaConfigs.ENABLE_TYPE_WIDENING.defaultTablePropertyKey -> "true", DeltaSQLConf.DELTA_STREAMING_SINK_ALLOW_IMPLICIT_CASTS.key -> "true") { withTempDirs { (inputDir, outputDir, checkpointDir) => sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (id INT) " + "USING DELTA TBLPROPERTIES ('delta.enableTypeWidening' = 'true')") val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 2).foreach { i => Seq(i).toDF("id") .write.mode("append").format("delta").save(deltaLog.dataPath.toString) } def startQuery(): StreamingQuery = { loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) .writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .option("mergeSchema", "true") // use delta sink because we need to check the result .format("delta") .start(outputDir.getCanonicalPath) } var q = startQuery() try { q.processAllAvailable() // Clear delta log cache DeltaLog.clearCache() // Widen the column type from INT to BIGINT using the non-cached `DeltaLog` to mimic // the case that the schema change happens on a different cluster withMetadata(deltaLog, StructType.fromDDL("id BIGINT")) // 2^31 cannot be represented as an int, so it will be inserted as a bigint Seq(2147483648L).toDF("id") .write.mode("append").format("delta").save(deltaLog.dataPath.toString) // The streaming query should fail when detecting a schema change val e = intercept[StreamingQueryException] { q.processAllAvailable() } assert(e.getMessage.contains("Detected schema change")) // Restarting the query with a new DataFrame should recover from the schema change q = startQuery() q.processAllAvailable() val outputDf = spark.read.format("delta").load(outputDir.getCanonicalPath) assert(outputDf.schema("id").dataType === LongType, s"Expected 'id' column to be LongType after type widening but got " + s"${outputDf.schema("id").dataType}") checkAnswer(outputDf.orderBy("id"), Seq(Row(0L), Row(1L), Row(2147483648L))) } finally { q.stop() } } } } test("type widening: restarting with stale DataFrame should recover") { withSQLConf( DeltaSQLConf.DELTA_TYPE_WIDENING_ENABLE_STREAMING_SCHEMA_TRACKING.key -> "false", DeltaConfigs.ENABLE_TYPE_WIDENING.defaultTablePropertyKey -> "true", DeltaSQLConf.DELTA_STREAMING_SINK_ALLOW_IMPLICIT_CASTS.key -> "true") { withTempDirs { (inputDir, outputDir, checkpointDir) => sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (id INT) " + "USING DELTA TBLPROPERTIES ('delta.enableTypeWidening' = 'true')") val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 2).foreach { i => Seq(i).toDF("id") .write.mode("append").format("delta").save(deltaLog.dataPath.toString) } val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) var q = df.writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .option("mergeSchema", "true") // use delta sink because we need to check the result .format("delta") .start(outputDir.getCanonicalPath) try { q.processAllAvailable() // Clear delta log cache DeltaLog.clearCache() // Widen the column type from INT to BIGINT using the non-cached `DeltaLog` to mimic // the case that the schema change happens on a different cluster withMetadata(deltaLog, StructType.fromDDL("id BIGINT")) // 2^31 cannot be represented as an int, so it will be inserted as a bigint Seq(2147483648L).toDF("id") .write.mode("append").format("delta").save(deltaLog.dataPath.toString) // The streaming query should fail when detecting a schema change val e = intercept[StreamingQueryException] { q.processAllAvailable() } assert(e.getMessage.contains("Detected schema change")) // Restarting the query with the stale DataFrame should still recover q = df.writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .option("mergeSchema", "true") .format("delta") .start(outputDir.getCanonicalPath) q.processAllAvailable() val outputDf = spark.read.format("delta").load(outputDir.getCanonicalPath) assert(outputDf.schema("id").dataType === LongType, s"Expected 'id' column to be LongType after type widening but got " + s"${outputDf.schema("id").dataType}") checkAnswer(outputDf.orderBy("id"), Seq(Row(0L), Row(1L), Row(2147483648L))) } finally { q.stop() } } } } test("drop column: should fail with non-additive schema change error") { withTempDir { inputDir => withTempDir { checkpointDir => // Create a table with column mapping enabled (required for drop/rename column) sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (id STRING, value STRING) " + "USING DELTA " + "TBLPROPERTIES ('delta.columnMapping.mode' = 'name')") val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 5).foreach { i => Seq((i.toString, s"val$i")).toDF("id", "value") .write.mode("append").format("delta").save(deltaLog.dataPath.toString) } def startQuery(): StreamingQuery = { loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) .writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("noop") .start() } val q = startQuery() try { q.processAllAvailable() DeltaLog.clearCache() // Simulate dropping "value" column by committing new metadata with only "id". // This is a non-additive schema change (column removal). withMetadata(deltaLog, StructType.fromDDL("id STRING")) val e = intercept[StreamingQueryException] { q.processAllAvailable() } assert(e.getMessage.contains( "Streaming read is not supported on tables with read-incompatible schema changes")) } finally { q.stop() } } } } test("drop column: should succeed with unsafe column mapping schema change flag enabled") { withTempDirs { (inputDir, outputDir, checkpointDir) => // Enable the unsafe flag that allows streaming to continue past column mapping // schema changes (drop/rename) instead of failing. withSQLConf( DeltaSQLConf .DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_COLUMN_MAPPING_SCHEMA_CHANGES .key -> "true" ) { sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (id STRING, value STRING) " + "USING DELTA " + "TBLPROPERTIES ('delta.columnMapping.mode' = 'name')") val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 5).foreach { i => Seq((i.toString, s"val$i")).toDF("id", "value") .write.mode("append").format("delta").save(deltaLog.dataPath.toString) } val q = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) .writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("delta") .start(outputDir.getCanonicalPath) try { q.processAllAvailable() DeltaLog.clearCache() // Simulate dropping "value" column by committing new metadata with only "id". withMetadata(deltaLog, StructType.fromDDL("id STRING")) // Write more data after the drop column schema change (5 until 10).foreach { i => Seq(i.toString).toDF("id") .write.mode("append").format("delta").save(deltaLog.dataPath.toString) } // With the unsafe flag, the stream should process all data without failing. // Post-drop rows have null for the removed "value" column. q.processAllAvailable() checkAnswer( spark.read.format("delta").load(outputDir.getAbsolutePath), (0 until 5).map(i => (i.toString, s"val$i")).toDF("id", "value") union (5 until 10).map(i => (i.toString, null: String)).toDF("id", "value")) } finally { q.stop() } } } } test("rename column: should fail with non-additive schema change error") { withTempDir { inputDir => withTempDir { checkpointDir => sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (id STRING, value STRING) " + "USING DELTA " + "TBLPROPERTIES ('delta.columnMapping.mode' = 'name')") val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 5).foreach { i => Seq((i.toString, s"val$i")).toDF("id", "value") .write.mode("append").format("delta").save(deltaLog.dataPath.toString) } def startQuery(): StreamingQuery = { loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) .writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("noop") .start() } var q = startQuery() try { q.processAllAvailable() DeltaLog.clearCache() // Simulate renaming "value" -> "renamed_value" by committing new metadata. // Column rename is a non-additive schema change under column mapping. withMetadata(deltaLog, StructType.fromDDL("id STRING, renamed_value STRING")) val e = intercept[StreamingQueryException] { q.processAllAvailable() } assert(e.getMessage.contains( "Streaming read is not supported on tables with read-incompatible schema changes")) } finally { q.stop() } } } } test("rename column: should throw schema change error with unsafe flag enabled") { withTempDir { inputDir => withTempDir { checkpointDir => // The unsafe flag only helps with drop-column; rename is still blocked because // it changes logical column identity, which can silently corrupt data. withSQLConf( DeltaSQLConf .DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_COLUMN_MAPPING_SCHEMA_CHANGES .key -> "true" ) { sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (id STRING, value STRING) " + "USING DELTA " + "TBLPROPERTIES ('delta.columnMapping.mode' = 'name')") val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 5).foreach { i => Seq((i.toString, s"val$i")).toDF("id", "value") .write.mode("append").format("delta").save(deltaLog.dataPath.toString) } def startQuery(): StreamingQuery = { loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) .writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("noop") .start() } var q = startQuery() try { q.processAllAvailable() DeltaLog.clearCache() // Simulate renaming "value" -> "renamed_value" by committing new metadata. withMetadata(deltaLog, StructType.fromDDL("id STRING, renamed_value STRING")) // Even with the unsafe flag, rename still fails - but with a different error // ("Detected schema change") rather than the non-additive schema change error. val e = intercept[StreamingQueryException] { q.processAllAvailable() } assert(e.getMessage.contains("Detected schema change in version 6")) } finally { q.stop() } } } } } test("type widening: should fail with non-additive schema change error " + "when enable schema tracking") { withTempDir { inputDir => withTempDir { checkpointDir => withSQLConf( DeltaSQLConf.DELTA_TYPE_WIDENING_ENABLE_STREAMING_SCHEMA_TRACKING.key -> "true" ) { // Table with type widening enabled so that type changes are allowed but tracked. sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (id INT, value STRING) " + "USING DELTA " + "TBLPROPERTIES ('delta.enableTypeWidening' = 'true')") val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) (0 until 5).foreach { i => Seq((i, s"val$i")).toDF("id", "value") .write.mode("append").format("delta").save(deltaLog.dataPath.toString) } def startQuery(): StreamingQuery = { loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) .writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("noop") .start() } var q = startQuery() try { q.processAllAvailable() DeltaLog.clearCache() // Simulate widening "id" from INT -> BIGINT by committing new metadata. // Type widening is a non-additive schema change for streaming reads. withMetadata(deltaLog, StructType.fromDDL("id BIGINT, value STRING")) val e = intercept[StreamingQueryException] { q.processAllAvailable() } assert(e.getMessage.contains( "Streaming read is not supported on tables with read-incompatible schema changes")) } finally { q.stop() } } } } } test("handling nullability schema changes") { withTable("srcTable") { withTempDirs { (srcTblDir, checkpointDir, checkpointDir2) => def readStream(startingVersion: Option[Long] = None): DataFrame = { var dsr = spark.readStream startingVersion.foreach { v => dsr = dsr.option("startingVersion", v) } dsr.table("srcTable") } sql(s""" |CREATE TABLE srcTable ( | a STRING NOT NULL, | b STRING NOT NULL |) USING DELTA LOCATION '${srcTblDir.getCanonicalPath}' |""".stripMargin) sql(""" |INSERT INTO srcTable | VALUES ("a", "b") |""".stripMargin) // Initialize the stream to pass the initial snapshot testStream(readStream())( StartStream(checkpointLocation = checkpointDir.getCanonicalPath), ProcessAllAvailable(), CheckAnswer(("a", "b")) ) // It is ok to relax nullability during streaming post analysis, and restart would fix it. var v1 = 0L val clock = new StreamManualClock(System.currentTimeMillis()) testStream(readStream())( StartStream(checkpointLocation = checkpointDir.getCanonicalPath, trigger = ProcessingTimeTrigger(1000), triggerClock = clock), ProcessAllAvailable(), // Write more data and drop NOT NULL constraint Execute { _ => // A batch of Delta actions sql(""" |INSERT INTO srcTable |VALUES ("c", "d") |""".stripMargin) sql("ALTER TABLE srcTable ALTER COLUMN a DROP NOT NULL") sql(""" |INSERT INTO srcTable |VALUES ("e", "f") |""".stripMargin) v1 = DeltaLog.forTable(spark, TableIdentifier("srcTable")).update().version }, // Process next trigger AdvanceManualClock(1 * 1000L), // The query would fail because the read schema has nullable=false but the schema change // tries to relax it, we cannot automatically move ahead with it. ExpectFailure[DeltaIllegalStateException](t => assert(t.getMessage.contains("Detected schema change"))), Execute { q => assert(!q.isActive) }, // Upon restart, the backfill can work with relaxed nullability read schema StartStream(checkpointLocation = checkpointDir.getCanonicalPath), ProcessAllAvailable(), // See how it loads data from across the nullability change without a problem CheckAnswer(("c", "d"), ("e", "f")) ) // However, it is NOT ok to read data with relaxed nullability during backfill, and restart // would NOT fix it. val deltaLog = DeltaLog.forTable(spark, TableIdentifier("srcTable")) deltaLog.withNewTransaction { txn => val schema = txn.snapshot.metadata.schema val newSchema = StructType(schema("a").copy(nullable = false) :: schema("b") :: Nil) txn.commit(txn.metadata.copy(schemaString = newSchema.json) :: Nil, DeltaOperations.ManualUpdate) } sql(""" |INSERT INTO srcTable |VALUES ("g", "h") |""".stripMargin) // Backfill from the ADD file action prior to the nullable=false, the latest schema has // nullable = false, but the ADD file has nullable = true, which is not allowed as we don't // want to show any nulls. // It queries [INSERT (e, f), nullable=false schema change, INSERT (g, h)] testStream(readStream(startingVersion = Some(v1)))( StartStream(checkpointLocation = checkpointDir2.getCanonicalPath), // See how it is: // 1. a non-retryable exception as it is a backfill. // 2. it comes from the new stream start check we added, before this, verifyStreamHygiene // could not detect because the most recent schema change looks exactly like the latest // schema. ExpectFailure[DeltaIllegalStateException](t => assert(t.getMessage.contains("Detected schema change") && t.getStackTrace.exists( _.toString.contains("checkReadIncompatibleSchemaChangeOnStreamStartOnce")))) ) } } } } /** * A FileSystem implementation that returns monotonically increasing timestamps for file creation. * Note that we may return a different timestamp for the same file. This is okay for the tests * where we use this though. */ class MonotonicallyIncreasingTimestampFS extends RawLocalFileSystem { private var time: Long = System.currentTimeMillis() override def getScheme: String = MonotonicallyIncreasingTimestampFS.scheme override def getUri: URI = { URI.create(s"$getScheme:///") } override def getFileStatus(f: Path): FileStatus = { val original = super.getFileStatus(f) time += 1000L new FileStatus(original.getLen, original.isDirectory, 0, 0, time, f) } } object MonotonicallyIncreasingTimestampFS { val scheme = s"MonotonicallyIncreasingTimestampFS" } // Batch sizes 1, 2, and 100 exercise different backfill behaviors in the commit coordinator. // Batch size 1 triggers a backfill on every commit (commitVersion % 1 == 0), testing the most // granular backfill path. Batch size 2 triggers backfill every other commit, testing the boundary // between backfilled and unbackfilled commits. Batch size 100 leaves most commits unbackfilled, // testing the production-like path where streaming must read from both the commit coordinator // and the filesystem. This follows the same pattern as other CatalogManaged (CCv2) test suites // (DeltaLogSuite, DeltaCDCStreamSuite, etc.). class DeltaSourceWithCatalogManagedBatch1Suite extends DeltaSourceSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class DeltaSourceWithCatalogManagedBatch2Suite extends DeltaSourceSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class DeltaSourceWithCatalogManagedBatch100Suite extends DeltaSourceSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuiteBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import org.apache.spark.sql.delta.actions.Format import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.schema.{SchemaMergingUtils, SchemaUtils} import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.{Column, DataFrame} import org.apache.spark.sql.streaming.StreamTest import org.apache.spark.sql.types.StructType import org.scalactic.source.Position import org.scalatest.Tag /** * Trait that provides abstraction for testing both DSv1 and DSv2 connectors. */ trait DeltaSourceConnectorTrait { self: DeltaSQLTestUtils => protected def useDsv2: Boolean = false protected def loadStreamWithOptions(path: String, options: Map[String, String]): DataFrame = { val reader = spark.readStream options.foreach { case (k, v) => reader.option(k, v) } if (useDsv2) { // This will route through DeltaCatalog which checks V2_ENABLE_MODE reader.table(s"delta.`$path`") } else { reader.format("delta").load(path) } } } trait DeltaSourceSuiteBase extends StreamTest with DeltaSQLTestUtils with CatalogOwnedTestBaseSuite with DeltaSourceConnectorTrait { /** * Creates 3 temporary directories for use within a function. * @param f function to be run with created temp directories */ protected def withTempDirs(f: (File, File, File) => Unit): Unit = { withTempDir { file1 => withTempDir { file2 => withTempDir { file3 => f(file1, file2, file3) } } } } /** * Creates 3 temporary directories for use within a function using a given prefix. * @param f function to be run with created temp directories */ protected def withTempDirs(prefix: String)(f: (File, File, File) => Unit): Unit = { withTempDir(prefix) { file1 => withTempDir(prefix) { file2 => withTempDir(prefix) { file3 => f(file1, file2, file3) } } } } /** * Copy metadata for fields in newSchema from currentSchema * @param newSchema new schema * @param currentSchema current schema to reference * @param columnMappingMode mode for column mapping * @return updated new schema */ protected def copyOverMetadata( newSchema: StructType, currentSchema: StructType, columnMappingMode: DeltaColumnMappingMode): StructType = { SchemaMergingUtils.transformColumns(newSchema) { (path, field, _) => val fullName = path :+ field.name val inSchema = SchemaUtils.findNestedFieldIgnoreCase( currentSchema, fullName, includeCollections = true ) inSchema.map { refField => val sparkMetadata = DeltaColumnMapping.getColumnMappingMetadata(refField, columnMappingMode) field.copy(metadata = sparkMetadata) }.getOrElse { field } } } protected def withMetadata( deltaLog: DeltaLog, schema: StructType, format: String = "parquet", tableId: Option[String] = None): Unit = { val txn = deltaLog.startTransaction() val baseMetadata = tableId.map { tId => txn.metadata.copy(id = tId) }.getOrElse(txn.metadata) // We need to fill up the missing id/physical name in column mapping mode // while maintaining existing metadata if there is any val updatedSchema = copyOverMetadata( schema, baseMetadata.schema, baseMetadata.columnMappingMode) // Configure CatalogManaged (CCv2) table settings. val updatedConfiguration = if (catalogOwnedDefaultCreationEnabledInTests) { // This withMetadata helper calls txn.commit(Metadata, ManualUpdate) directly, bypassing // the normal CREATE TABLE path (CreateDeltaTableCommand, DeltaTestImplicits.commitActions) // that populates newProtocol with CatalogOwnedTableFeature and auto-enables ICT. Without // ICT, the CatalogManaged commit coordinator rejects the commit because it requires // commitTimestamp on every version. This is a test-only issue: txn.commit is not a // user-facing table creation API, and production tables always go through the full DDL // path. We enable ICT manually here as the simplest fix. baseMetadata.configuration + (DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key -> "true") } else { baseMetadata.configuration } txn.commit( DeltaColumnMapping.assignColumnIdAndPhysicalName( baseMetadata.copy( schemaString = updatedSchema.json, configuration = updatedConfiguration, format = Format(format)), baseMetadata, isChangingModeOnExistingTable = false, isOverwritingSchema = false) :: Nil, DeltaOperations.ManualUpdate) } object AddToReservoir { def apply(path: File, data: DataFrame): AssertOnQuery = AssertOnQuery { _ => data.write.format("delta").mode("append").save(path.getAbsolutePath) true } } object UpdateReservoir { def apply(path: File, updateExpression: Map[String, Column]): AssertOnQuery = AssertOnQuery { _ => io.delta.tables.DeltaTable.forPath(path.getAbsolutePath).update(updateExpression) true } } object DeleteFromReservoir { def apply(path: File, deleteCondition: Column): AssertOnQuery = AssertOnQuery { _ => io.delta.tables.DeltaTable.forPath(path.getAbsolutePath).delete(deleteCondition) true } } object MergeIntoReservoir { def apply(path: File, dfToMerge: DataFrame, mergeCondition: Column, updateExpression: Map[String, Column]): AssertOnQuery = AssertOnQuery { _ => io.delta.tables.DeltaTable .forPath(path.getAbsolutePath) .as("table") .merge(dfToMerge, mergeCondition) .whenMatched() .update(updateExpression) .whenNotMatched() .insertAll() .execute() true } } object CheckProgress { def apply(rowsPerBatch: Seq[Int]): AssertOnQuery = Execute { q => val progress = q.recentProgress.filter(_.numInputRows != 0) assert(progress.length === rowsPerBatch.size, "Expected batches don't match") progress.zipWithIndex.foreach { case (p, i) => assert(p.numInputRows === rowsPerBatch(i), s"Expected rows in batch $i does not match ") } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceTableAPISuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import scala.language.implicitConversions import org.apache.spark.sql.delta.coordinatedcommits.CoordinatedCommitsBaseSuite import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.shims.StreamingTestShims.MemoryStream import org.apache.spark.sql.{AnalysisException, Dataset} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.SessionCatalog.DEFAULT_DATABASE import org.apache.spark.sql.execution.streaming._ import org.apache.spark.sql.streaming.{StreamingQuery, StreamTest} import org.apache.spark.util.Utils class DeltaSourceTableAPISuite extends StreamTest with DeltaSQLCommandTest with CoordinatedCommitsBaseSuite { override def beforeAll(): Unit = { super.beforeAll() } import testImplicits._ test("table API") { withTempDir { tempDir => val tblName = "my_table" val dir = tempDir.getAbsolutePath withTable(tblName) { spark.range(3).write.format("delta").option("path", dir).saveAsTable(tblName) testStream(spark.readStream.table(tblName))( ProcessAllAvailable(), CheckAnswer(0, 1, 2) ) } } } test("table API with database") { withTempDir { tempDir => val tblName = "my_table" val dir = tempDir.getAbsolutePath withTempDatabase { db => withTable(tblName) { spark.sql(s"USE $db") spark.range(3).write.format("delta").option("path", dir).saveAsTable(tblName) spark.sql(s"USE $DEFAULT_DATABASE") testStream(spark.readStream.table(s"$db.$tblName"))( ProcessAllAvailable(), CheckAnswer(0, 1, 2) ) } } } } private def startTableStream( ds: Dataset[_], tableName: String, baseDir: Option[File] = None, partitionColumns: Seq[String] = Nil, format: String = "delta"): StreamingQuery = { val checkpoint = baseDir.map(new File(_, "_checkpoint")) .getOrElse(Utils.createTempDir().getCanonicalFile) val dsw = ds.writeStream.format(format).partitionBy(partitionColumns: _*) baseDir.foreach { output => dsw.option("path", output.getCanonicalPath) } dsw.option("checkpointLocation", checkpoint.getCanonicalPath).toTable(tableName) } test("writeStream.table - create new external table") { withTempDir { dir => val memory = MemoryStream[Int] val tableName = "stream_test" withTable(tableName) { val sq = startTableStream(memory.toDS(), tableName, Some(dir)) memory.addData(1, 2, 3) sq.processAllAvailable() checkDatasetUnorderly( spark.table(tableName).as[Int], 1, 2, 3) checkDatasetUnorderly( spark.read.format("delta").load(dir.getCanonicalPath).as[Int], 1, 2, 3) } } } test("writeStream.table - create new managed table") { val memory = MemoryStream[Int] val tableName = "stream_test" withTable(tableName) { val sq = startTableStream(memory.toDS(), tableName, None) memory.addData(1, 2, 3) sq.processAllAvailable() checkDatasetUnorderly( spark.table(tableName).as[Int], 1, 2, 3) val path = spark.sessionState.catalog.getTableRawMetadata(TableIdentifier(tableName)).location checkDatasetUnorderly( spark.read.format("delta").load(new File(path).getCanonicalPath).as[Int], 1, 2, 3) } } test("writeStream.table - create new managed table with database") { val memory = MemoryStream[Int] val db = "my_db" val tableName = s"$db.stream_test" withDatabase(db) { sql(s"create database $db") withTable(tableName) { val sq = startTableStream(memory.toDS(), tableName, None) memory.addData(1, 2, 3) sq.processAllAvailable() checkDatasetUnorderly( spark.table(tableName).as[Int], 1, 2, 3) val path = spark.sessionState.catalog.getTableRawMetadata( spark.sessionState.sqlParser.parseTableIdentifier(tableName)).location checkDatasetUnorderly( spark.read.format("delta").load(new File(path).getCanonicalPath).as[Int], 1, 2, 3) } } } test("writeStream.table - create table from existing output") { withTempDir { dir => Seq(4, 5, 6).toDF("value").write.format("delta").save(dir.getCanonicalPath) val memory = MemoryStream[Int] val tableName = "stream_test" withTable(tableName) { val sq = startTableStream(memory.toDS(), tableName, Some(dir)) memory.addData(1, 2, 3) sq.processAllAvailable() checkDatasetUnorderly( spark.table(tableName).as[Int], 1, 2, 3, 4, 5, 6) checkDatasetUnorderly( spark.read.format("delta").load(dir.getCanonicalPath).as[Int], 1, 2, 3, 4, 5, 6) } } } test("writeStream.table - fail writing into a view") { val memory = MemoryStream[Int] val tableName = "stream_test" withTable(tableName) { val viewName = tableName + "_view" withView(viewName) { Seq(4, 5, 6).toDF("value").write.saveAsTable(tableName) sql(s"create view $viewName as select * from $tableName") val e = intercept[AnalysisException] { startTableStream(memory.toDS(), viewName, None) } assert(e.getMessage.contains("views")) } } } test("writeStream.table - fail due to different schema than existing Delta table") { withTempDir { dir => Seq(4, 5, 6).toDF("id").write.format("delta").save(dir.getCanonicalPath) val memory = MemoryStream[Int] val tableName = "stream_test" withTable(tableName) { val e = intercept[Exception] { val sq = startTableStream(memory.toDS(), tableName, Some(dir)) memory.addData(1, 2, 3) sq.processAllAvailable() } assert(e.getMessage.contains("The specified schema does not match the existing schema")) } } } test("writeStream.table - fail due to different partitioning on existing Delta table") { withTempDir { dir => Seq(4 -> "a").toDF("id", "key").write.format("delta").save(dir.getCanonicalPath) val memory = MemoryStream[(Int, String)] val tableName = "stream_test" withTable(tableName) { val e = intercept[Exception] { val sq = startTableStream( memory.toDS().toDF("id", "key"), tableName, Some(dir), Seq("key")) memory.addData(1 -> "a") sq.processAllAvailable() } assert(e.getMessage.contains( "The specified partitioning does not match the existing partitioning")) } } } test("writeStream.table - fail writing into an external nonDelta table") { withTempDir { dir => val memory = MemoryStream[(Int, String)] val tableName = "stream_test" withTable(tableName) { Seq(1).toDF("value").write.format("parquet") .option("path", dir.getCanonicalPath).saveAsTable(tableName) val e = intercept[AnalysisException] { startTableStream(memory.toDS(), tableName, Some(dir)) } assert(e.getMessage.contains("delta")) } } } test("writeStream.table - fail writing into an external nonDelta path") { withTempDir { dir => val memory = MemoryStream[Int] val tableName = "stream_test" withTable(tableName) { Seq(1).toDF("value").write.mode("append").parquet(dir.getCanonicalPath) val e = intercept[AnalysisException] { startTableStream(memory.toDS(), tableName, Some(dir)) } assert(e.getMessage.contains("Delta")) } } } } class DeltaSourceTableAPIWithCoordinatedCommitsBatch1Suite extends DeltaSourceTableAPISuite { override def coordinatedCommitsBackfillBatchSize: Option[Int] = Some(1) } class DeltaSourceTableAPIWithCoordinatedCommitsBatch100Suite extends DeltaSourceTableAPISuite { override def coordinatedCommitsBackfillBatchSize: Option[Int] = Some(100) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaStreamUtilsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.sql.Timestamp import org.apache.spark.SparkFunSuite import org.apache.spark.sql.delta.sources.DeltaStreamUtils class DeltaStreamUtilsSuite extends SparkFunSuite { // ========== getStartingVersionFromCommitAtTimestamp ========== test("getStartingVersionFromCommitAtTimestamp - " + "commit at timestamp returns commitVersion") { val timeZone = "UTC" val commitTs = 1000L val commitVersion = 2L val latestVersion = 5L val timestamp = new Timestamp(1000) val result = DeltaStreamUtils.getStartingVersionFromCommitAtTimestamp( timeZone, commitTs, commitVersion, latestVersion, timestamp) assert(result == 2L) } test("getStartingVersionFromCommitAtTimestamp - " + "commit after timestamp returns commitVersion") { val timeZone = "UTC" val commitTs = 2000L val commitVersion = 2L val latestVersion = 5L val timestamp = new Timestamp(1000) val result = DeltaStreamUtils.getStartingVersionFromCommitAtTimestamp( timeZone, commitTs, commitVersion, latestVersion, timestamp) assert(result == 2L) } test("getStartingVersionFromCommitAtTimestamp - " + "commit before timestamp returns commitVersion+1") { val timeZone = "UTC" val commitTs = 1000L val commitVersion = 2L val latestVersion = 5L val timestamp = new Timestamp(2000) val result = DeltaStreamUtils.getStartingVersionFromCommitAtTimestamp( timeZone, commitTs, commitVersion, latestVersion, timestamp) assert(result == 3L) } test("getStartingVersionFromCommitAtTimestamp - " + "timestamp after latest throws when canExceedLatest false") { val timeZone = "UTC" val commitTs = 1000L val commitVersion = 5L val latestVersion = 5L val timestamp = new Timestamp(2000) val e = intercept[Exception] { DeltaStreamUtils.getStartingVersionFromCommitAtTimestamp( timeZone, commitTs, commitVersion, latestVersion, timestamp, canExceedLatest = false) } assert(e.getMessage.contains("DELTA_TIMESTAMP_GREATER_THAN_COMMIT")) } test("getStartingVersionFromCommitAtTimestamp - " + "timestamp after latest returns commitVersion+1 when canExceedLatest true") { val timeZone = "UTC" val commitTs = 1000L val commitVersion = 5L val latestVersion = 5L val timestamp = new Timestamp(2000) val result = DeltaStreamUtils.getStartingVersionFromCommitAtTimestamp( timeZone, commitTs, commitVersion, latestVersion, timestamp, canExceedLatest = true) assert(result == 6L) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.{File, FileNotFoundException} import java.util.concurrent.atomic.AtomicInteger // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.actions.{Action, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CatalogOwnedTestBaseSuite} import org.apache.spark.sql.delta.files.TahoeLogFileIndex import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.test.shims.StreamingTestShims.MemoryStream import org.apache.spark.sql.delta.util.{DeltaFileOperations, FileNames} import org.apache.spark.sql.delta.util.FileNames.unsafeDeltaFile import org.apache.hadoop.fs.{FileSystem, FSDataInputStream, Path, PathHandle} import org.apache.spark.SparkException import org.apache.spark.scheduler.{SparkListener, SparkListenerJobStart} import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} import org.apache.spark.sql.connector.catalog.TableCatalog import org.apache.spark.sql.catalyst.expressions.InSet import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral import org.apache.spark.sql.catalyst.plans.logical.Filter import org.apache.spark.sql.execution.FileSourceScanExec import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelationWithTable} import org.apache.spark.sql.functions.{asc, col, expr, lit, map_values, struct} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.streaming.StreamingQuery import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ import org.apache.spark.util.Utils class DeltaSuite extends QueryTest with SharedSparkSession with DeltaColumnMappingTestUtils with DeltaSQLTestUtils with DeltaSQLCommandTest with CatalogOwnedTestBaseSuite { import testImplicits._ private def tryDeleteNonRecursive(fs: FileSystem, path: Path): Boolean = { try fs.delete(path, false) catch { case _: FileNotFoundException => true } } test("handle partition filters and data filters") { withTempDir { inputDir => val testPath = inputDir.getCanonicalPath spark.range(10) .map(_.toInt) .withColumn("part", $"value" % 2) .write .format("delta") .partitionBy("part") .mode("append") .save(testPath) val ds = spark.read.format("delta").load(testPath).as[(Int, Int)] // partition filter checkDatasetUnorderly( ds.where("part = 1"), 1 -> 1, 3 -> 1, 5 -> 1, 7 -> 1, 9 -> 1) checkDatasetUnorderly( ds.where("part = 0"), 0 -> 0, 2 -> 0, 4 -> 0, 6 -> 0, 8 -> 0) // data filter checkDatasetUnorderly( ds.where("value >= 5"), 5 -> 1, 6 -> 0, 7 -> 1, 8 -> 0, 9 -> 1) checkDatasetUnorderly( ds.where("value < 5"), 0 -> 0, 1 -> 1, 2 -> 0, 3 -> 1, 4 -> 0) // partition filter + data filter checkDatasetUnorderly( ds.where("part = 1 and value >= 5"), 5 -> 1, 7 -> 1, 9 -> 1) checkDatasetUnorderly( ds.where("part = 1 and value < 5"), 1 -> 1, 3 -> 1) } } test("query with predicates should skip partitions") { withTempDir { tempDir => val testPath = tempDir.getCanonicalPath // Generate two files in two partitions spark.range(2) .withColumn("part", $"id" % 2) .write .format("delta") .partitionBy("part") .mode("append") .save(testPath) // Read only one partition val query = spark.read.format("delta").load(testPath).where("part = 1") val fileScans = query.queryExecution.executedPlan.collect { case f: FileSourceScanExec => f } // Force the query to read files and generate metrics query.queryExecution.executedPlan.execute().count() // Verify only one file was read assert(fileScans.size == 1) val numFilesAferPartitionSkipping = fileScans.head.metrics.get("numFiles") assert(numFilesAferPartitionSkipping.nonEmpty) assert(numFilesAferPartitionSkipping.get.value == 1) checkAnswer(query, Seq(Row(1, 1))) } } test("partition column location should not impact table schema") { val tableColumns = Seq("c1", "c2") for (partitionColumn <- tableColumns) { withTempDir { inputDir => val testPath = inputDir.getCanonicalPath Seq(1 -> "a", 2 -> "b").toDF(tableColumns: _*) .write .format("delta") .partitionBy(partitionColumn) .save(testPath) val ds = spark.read.format("delta").load(testPath).as[(Int, String)] checkDatasetUnorderly(ds, 1 -> "a", 2 -> "b") } } } test("SC-8078: read deleted directory") { val tempDir = Utils.createTempDir() val path = new Path(tempDir.getCanonicalPath) Seq(1).toDF().write.format("delta").save(tempDir.toString) val df = spark.read.format("delta").load(tempDir.toString) // scalastyle:off deltahadoopconfiguration val fs = path.getFileSystem(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration fs.delete(path, true) val e = intercept[AnalysisException] { withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "0s") { checkAnswer(df, Row(1) :: Nil) } }.getMessage assert(e.contains("The schema of your Delta table has changed")) val e2 = intercept[AnalysisException] { withSQLConf(DeltaSQLConf.DELTA_ASYNC_UPDATE_STALENESS_TIME_LIMIT.key -> "0s") { // Define new DataFrame spark.read.format("delta").load(tempDir.toString).collect() } }.getMessage assert(e2.contains("Path does not exist")) } test("SC-70676: directory deleted before first DataFrame is defined") { val tempDir = Utils.createTempDir() val path = new Path(tempDir.getCanonicalPath) Seq(1).toDF().write.format("delta").save(tempDir.toString) // scalastyle:off deltahadoopconfiguration val fs = path.getFileSystem(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration fs.delete(path, true) val e = intercept[AnalysisException] { spark.read.format("delta").load(tempDir.toString).collect() }.getMessage assert(e.contains("Path does not exist")) } test("append then read") { val tempDir = Utils.createTempDir() Seq(1).toDF().write.format("delta").save(tempDir.toString) Seq(2, 3).toDF().write.format("delta").mode("append").save(tempDir.toString) def data: DataFrame = spark.read.format("delta").load(tempDir.toString) checkAnswer(data, Row(1) :: Row(2) :: Row(3) :: Nil) // append more Seq(4, 5, 6).toDF().write.format("delta").mode("append").save(tempDir.toString) checkAnswer(data.toDF(), Row(1) :: Row(2) :: Row(3) :: Row(4) :: Row(5) :: Row(6) :: Nil) } test("null struct with NullType field kept as null") { withTempTable(createTable = false) { tableName => Seq(((null, 2), 1), (null, 2)).toDF("key", "value") .write.format("delta").saveAsTable(tableName) // Evolve the schema because tables with NullType columns cannot be read currently. Seq(((10, 10), 10)).toDF("key", "value") .write .format("delta") .option("mergeSchema", "true") .mode("append") .saveAsTable(tableName) // Confirm struct value stays as null (fields are not set to null). val rowWithNullStruct = spark.read.format("delta").table(tableName).filter($"value" === 2) checkAnswer(rowWithNullStruct, Row(null, 2) :: Nil) } } test("null struct with NullType field, with backticks in the column name, kept as null") { withTempTable(createTable = false) { tableName => Seq(((null, 2), 1), (null, 2)).toDF("key`", "val`ue") .write.format("delta").saveAsTable(tableName) // Evolve the schema because tables with NullType columns cannot be read currently. Seq(((10, 10), 10)).toDF("key`", "val`ue") .write .format("delta") .option("mergeSchema", "true") .mode("append") .saveAsTable(tableName) // Confirm struct value stays as null (fields are not set to null). val rowWithNullStruct = spark.read.format("delta").table(tableName).filter($"`val``ue`" === 2) checkAnswer(rowWithNullStruct, Row(null, 2) :: Nil) } } test("Cannot create table with NullType UDT column") { val table_name = "test_table" withTable(table_name) { checkError( intercept[DeltaAnalysisException] { Seq((1, new NullData())).toDF("id", "value") .write.format("delta").saveAsTable(table_name) }, "DELTA_USER_DEFINED_TYPE_COLUMN_CONTAINS_NULL_TYPE", sqlState = Some("22005"), parameters = Map("columnName" -> "value", "userClass" -> classOf[NullData].getName) ) } } test("Cannot create table with NullType in a complex UDT column") { val table_name = "test_table" withTable(table_name) { checkError( intercept[DeltaAnalysisException] { Seq((1, new ComplexData())).toDF("id", "value") .write.format("delta").saveAsTable(table_name) }, "DELTA_USER_DEFINED_TYPE_COLUMN_CONTAINS_NULL_TYPE", sqlState = Some("22005"), parameters = Map("columnName" -> "value", "userClass" -> classOf[ComplexData].getName) ) } } test("partitioned append - nulls") { val tempDir = Utils.createTempDir() Seq(Some(1), None).toDF() .withColumn("is_odd", $"value" % 2 === 1) .write .format("delta") .partitionBy("is_odd") .save(tempDir.toString) val df = spark.read.format("delta").load(tempDir.toString) // Verify the correct partitioning schema is picked up val hadoopFsRelations = df.queryExecution.analyzed.collect { case LogicalRelationWithTable(h: HadoopFsRelation, _) => h } assert(hadoopFsRelations.size === 1) assert(hadoopFsRelations.head.partitionSchema.exists(_.name == "is_odd")) assert(hadoopFsRelations.head.dataSchema.exists(_.name == "value")) checkAnswer(df.where("is_odd = true"), Row(1, true) :: Nil) checkAnswer(df.where("is_odd IS NULL"), Row(null, null) :: Nil) } test("input files should be absolute paths") { withTempDir { dir => val basePath = dir.getAbsolutePath spark.range(10).withColumn("part", 'id % 3) .write.format("delta").partitionBy("part").save(basePath) val df1 = spark.read.format("delta").load(basePath) val df2 = spark.read.format("delta").load(basePath).where("part = 1") val df3 = spark.read.format("delta").load(basePath).where("part = 1").limit(3) assert(df1.inputFiles.forall(_.contains(basePath))) assert(df2.inputFiles.forall(_.contains(basePath))) assert(df3.inputFiles.forall(_.contains(basePath))) } } test("invalid replaceWhere") { Seq(true, false).foreach { enabled => withSQLConf(DeltaSQLConf.REPLACEWHERE_DATACOLUMNS_ENABLED.key -> enabled.toString) { val tempDir = Utils.createTempDir() Seq(1, 2, 3, 4).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .write .format("delta") .partitionBy("is_odd") .save(tempDir.toString) val e1 = intercept[AnalysisException] { Seq(6).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "is_odd = true") .save(tempDir.toString) }.getMessage assert(e1.contains("does not conform to partial table overwrite condition or constraint")) val e2 = intercept[AnalysisException] { Seq(true).toDF("is_odd") .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "is_odd = true") .save(tempDir.toString) }.getMessage assert(e2.contains( "Data written into Delta needs to contain at least one non-partitioned")) val e3 = intercept[AnalysisException] { Seq(6).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "not_a_column = true") .save(tempDir.toString) }.getMessage if (enabled) { assert(e3.contains( "or function parameter with name `not_a_column` cannot be resolved") || e3.contains("Column 'not_a_column' does not exist. Did you mean one of " + "the following? [value, is_odd]")) } else { assert(e3.contains( "Predicate references non-partition column 'not_a_column'. Only the " + "partition columns may be referenced: [is_odd]")) } val e4 = intercept[AnalysisException] { Seq(6).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "value = 1") .save(tempDir.toString) }.getMessage if (enabled) { assert(e4.contains( "Written data does not conform to partial table overwrite condition " + "or constraint 'value = 1'")) } else { assert(e4.contains("Predicate references non-partition column 'value'. Only the " + "partition columns may be referenced: [is_odd]")) } val e5 = intercept[AnalysisException] { Seq(6).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "") .save(tempDir.toString) }.getMessage assert(e5.contains("Cannot recognize the predicate ''")) } } } test("replaceWhere with rearrangeOnly") { withTempDir { dir => Seq(1, 2, 3, 4).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .write .format("delta") .partitionBy("is_odd") .save(dir.toString) // dataFilter non empty val e = intercept[AnalysisException] { Seq(9).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "is_odd = true and value < 2") .option(DeltaOptions.DATA_CHANGE_OPTION, "false") .save(dir.toString) }.getMessage assert(e.contains( "'replaceWhere' cannot be used with data filters when 'dataChange' is set to false")) Seq(9).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "is_odd = true") .option(DeltaOptions.DATA_CHANGE_OPTION, "false") .save(dir.toString) checkAnswer( spark.read.format("delta").load(dir.toString), Seq(2, 4, 9).toDF().withColumn("is_odd", $"value" % 2 =!= 0)) } } test("valid replaceWhere") { Seq(true, false).foreach { enabled => withSQLConf(DeltaSQLConf.REPLACEWHERE_DATACOLUMNS_ENABLED.key -> enabled.toString) { Seq(true, false).foreach { partitioned => // Skip when it's not enabled and not partitioned. if (enabled || partitioned) { withTempDir { dir => val writer = Seq(1, 2, 3, 4).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .withColumn("is_even", $"value" % 2 === 0) .write .format("delta") if (partitioned) { writer.partitionBy("is_odd").save(dir.toString) } else { writer.save(dir.toString) } def data: DataFrame = spark.read.format("delta").load(dir.toString) Seq(5, 7).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .withColumn("is_even", $"value" % 2 === 0) .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "is_odd = true") .save(dir.toString) checkAnswer( data, Seq(2, 4, 5, 7).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .withColumn("is_even", $"value" % 2 === 0)) // replaceWhere on non-partitioning columns if enabled. if (enabled) { Seq(6, 8).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .withColumn("is_even", $"value" % 2 === 0) .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "is_even = true") .save(dir.toString) checkAnswer( data, Seq(5, 6, 7, 8).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .withColumn("is_even", $"value" % 2 === 0)) // nothing to be replaced because the condition is false. Seq(10, 12).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .withColumn("is_even", $"value" % 2 === 0) .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "1 = 2") .save(dir.toString) checkAnswer( data, Seq(5, 6, 7, 8, 10, 12).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .withColumn("is_even", $"value" % 2 === 0) ) // replace the whole thing because the condition is true. Seq(10, 12).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .withColumn("is_even", $"value" % 2 === 0) .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "1 = 1") .save(dir.toString) checkAnswer( data, Seq(10, 12).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .withColumn("is_even", $"value" % 2 === 0) ) } } } } } } } Seq(false, true).foreach { replaceWhereInDataColumn => test(s"valid replaceWhere with cdf enabled, " + s"replaceWhereInDataColumn = $replaceWhereInDataColumn") { testReplaceWhereWithCdf( replaceWhereInDataColumn) } } def testReplaceWhereWithCdf( replaceWhereInDataColumn: Boolean): Unit = { withSQLConf( DeltaSQLConf.REPLACEWHERE_DATACOLUMNS_ENABLED.key -> replaceWhereInDataColumn.toString, DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> "true") { withTempDir { dir => Seq(1, 2, 3, 4).map(i => (i, i + 2)).toDF("key", "value.1") .withColumn("is_odd", $"`value.1`" % 2 =!= 0) .withColumn("is_even", $"`value.1`" % 2 === 0) .coalesce(1) .write .format("delta") .partitionBy("is_odd").save(dir.toString) checkAnswer( CDCReader.changesToBatchDF(DeltaLog.forTable(spark, dir), 0, 0, spark) .drop(CDCReader.CDC_COMMIT_TIMESTAMP), Row(1, 3, true, false, "insert", 0) :: Row(3, 5, true, false, "insert", 0) :: Row(2, 4, false, true, "insert", 0) :: Row(4, 6, false, true, "insert", 0) :: Nil) def data: DataFrame = spark.read.format("delta").load(dir.toString) Seq(5, 7).map(i => (i, i + 2)).toDF("key", "value.1") .withColumn("is_odd", $"`value.1`" % 2 =!= 0) .withColumn("is_even", $"`value.1`" % 2 === 0) .coalesce(1) .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "is_odd = true") .save(dir.toString) checkAnswer( data, Seq(2, 4, 5, 7).map(i => (i, i + 2)).toDF("key", "value.1") .withColumn("is_odd", $"`value.1`" % 2 =!= 0) .withColumn("is_even", $"`value.1`" % 2 === 0)) checkAnswer( CDCReader.changesToBatchDF(DeltaLog.forTable(spark, dir), 1, 1, spark) .drop(CDCReader.CDC_COMMIT_TIMESTAMP), Row(1, 3, true, false, "delete", 1) :: Row(3, 5, true, false, "delete", 1) :: Row(5, 7, true, false, "insert", 1) :: Row(7, 9, true, false, "insert", 1) :: Nil) if (replaceWhereInDataColumn) { // replaceWhere on non-partitioning columns if enabled. Seq((4, 8)).toDF("key", "value.1") .withColumn("is_odd", $"`value.1`" % 2 =!= 0) .withColumn("is_even", $"`value.1`" % 2 === 0) .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "key = 4") .save(dir.toString) checkAnswer( data, Seq((2, 4), (4, 8), (5, 7), (7, 9)).toDF("key", "value.1") .withColumn("is_odd", $"`value.1`" % 2 =!= 0) .withColumn("is_even", $"`value.1`" % 2 === 0)) checkAnswer( CDCReader.changesToBatchDF(DeltaLog.forTable(spark, dir), 2, 2, spark) .drop(CDCReader.CDC_COMMIT_TIMESTAMP), Row(4, 6, false, true, "delete", 2) :: Row(4, 8, false, true, "insert", 2) :: Nil) } } } } test("replace arbitrary with multiple references") { withTempDir { dir => def data: DataFrame = spark.read.format("delta").load(dir.toString) Seq((1, 3, 8), (1, 5, 9)).toDF("a", "b", "c") .write .format("delta") .mode("overwrite") .save(dir.toString) Seq((2, 4, 6)).toDF("a", "b", "c") .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "a + c < 10") .save(dir.toString) checkAnswer( data, Seq((1, 5, 9), (2, 4, 6)).toDF("a", "b", "c")) } } test("replaceWhere with constraint check disabled") { withSQLConf(DeltaSQLConf.REPLACEWHERE_CONSTRAINT_CHECK_ENABLED.key -> "false") { withTempDir { dir => Seq(1, 2, 3, 4).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .write .format("delta") .partitionBy("is_odd") .save(dir.toString) def data: DataFrame = spark.read.format("delta").load(dir.toString) Seq(6).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "is_odd = true") .save(dir.toString) checkAnswer(data, Seq(2, 4, 6).toDF().withColumn("is_odd", $"value" % 2 =!= 0)) } } } Seq(true, false).foreach { p => test(s"replaceWhere user defined _change_type column doesn't get dropped - partitioned=$p") { withTable("tab") { sql( s"""CREATE TABLE tab USING DELTA |${if (p) "PARTITIONED BY (part) " else ""} |TBLPROPERTIES (delta.enableChangeDataFeed = false) |AS SELECT id, floor(id / 10) AS part, 'foo' as _change_type |FROM RANGE(1000) |""".stripMargin) Seq(33L).map(id => id * 42).toDF("id") .withColumn("part", expr("floor(id / 10)")) .withColumn("_change_type", lit("bar")) .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "id % 7 = 0") .saveAsTable("tab") sql("SELECT id, _change_type FROM tab").collect().foreach { row => val _change_type = row.getString(1) assert(_change_type === "foo" || _change_type === "bar", s"Invalid _change_type for id=${row.get(0)}") } } } } test("move delta table") { val tempDir = Utils.createTempDir() Seq(1, 2, 3).toDS().write.format("delta").mode("append").save(tempDir.toString) def data: DataFrame = spark.read.format("delta").load(tempDir.toString) checkAnswer(data.toDF(), Row(1) :: Row(2) :: Row(3) :: Nil) // Append files in log path should use relative paths and should work with file renaming. val targetDir = new File(Utils.createTempDir(), "target") assert(tempDir.renameTo(targetDir)) def data2: DataFrame = spark.read.format("delta").load(targetDir.toString) checkDatasetUnorderly(data2.toDF().as[Int], 1, 2, 3) } test("append table to itself") { val tempDir = Utils.createTempDir() Seq(1, 2, 3).toDS().write.format("delta").mode("append").save(tempDir.toString) def data: DataFrame = spark.read.format("delta").load(tempDir.toString) checkDatasetUnorderly(data.toDF.as[Int], 1, 2, 3) data.write.format("delta").mode("append").save(tempDir.toString) checkDatasetUnorderly(data.toDF.as[Int], 1, 1, 2, 2, 3, 3) } test("missing partition columns") { val tempDir = Utils.createTempDir() Seq(1, 2, 3).toDF() .withColumn("part", $"value" % 2) .write .format("delta") .partitionBy("part") .save(tempDir.toString) val e = intercept[Exception] { Seq(1, 2, 3).toDF() .write .format("delta") .mode("append") .save(tempDir.toString) } assert(e.getMessage contains "Partition column") assert(e.getMessage contains "part") assert(e.getMessage contains "not found") } test("batch write: append, overwrite") { withTempDir { tempDir => def data: DataFrame = spark.read.format("delta").load(tempDir.toString) Seq(1, 2, 3).toDF .write .format("delta") .mode("append") .save(tempDir.getCanonicalPath) checkDatasetUnorderly(data.toDF.as[Int], 1, 2, 3) Seq(4, 5, 6).toDF .write .format("delta") .mode("overwrite") .save(tempDir.getCanonicalPath) checkDatasetUnorderly(data.toDF.as[Int], 4, 5, 6) } } test("batch write: overwrite an empty directory with replaceWhere") { withTempDir { tempDir => def data: DataFrame = spark.read.format("delta").load(tempDir.toString) Seq (1, 3, 5).toDF .withColumn("part", $"value" % 2) .write .format("delta") .mode("overwrite") .partitionBy("part") .option(DeltaOptions.REPLACE_WHERE_OPTION, "part = 1") .save(tempDir.getCanonicalPath) checkDatasetUnorderly(data.toDF.as[(Int, Int)], 1 -> 1, 3 -> 1, 5 -> 1) } } test("batch write: append, overwrite where") { withTempDir { tempDir => def data: DataFrame = spark.read.format("delta").load(tempDir.toString) Seq (1, 2, 3).toDF .withColumn("part", $"value" % 2) .write .format("delta") .partitionBy("part") .mode("append") .save(tempDir.getCanonicalPath) Seq(1, 5).toDF .withColumn("part", $"value" % 2) .write .format("delta") .partitionBy("part") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "part=1") .save(tempDir.getCanonicalPath) checkDatasetUnorderly(data.toDF.select($"value".as[Int]), 1, 2, 5) } } test("batch write: append, dynamic partition overwrite integer partition column") { withSQLConf(DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED.key -> "true") { withTempDir { tempDir => def data: DataFrame = spark.read.format("delta").load(tempDir.toString) Seq(1, 2, 3).toDF .withColumn("part", $"value" % 2) .write .format("delta") .partitionBy("part") .mode("append") .save(tempDir.getCanonicalPath) Seq(1, 5).toDF .withColumn("part", $"value" % 2) .write .format("delta") .partitionBy("part") .mode("overwrite") .option(DeltaOptions.PARTITION_OVERWRITE_MODE_OPTION, "dynamic") .save(tempDir.getCanonicalPath) checkDatasetUnorderly(data.select("value").as[Int], 1, 2, 5) } } } test("batch write: append, dynamic partition overwrite string partition column") { withSQLConf(DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED.key -> "true") { withTempDir { tempDir => def data: DataFrame = spark.read.format("delta").load(tempDir.toString) Seq(("a", "x"), ("b", "y"), ("c", "x")).toDF("value", "part") .write .format("delta") .partitionBy("part") .mode("append") .save(tempDir.getCanonicalPath) Seq(("a", "x"), ("d", "x")).toDF("value", "part") .write .format("delta") .partitionBy("part") .mode("overwrite") .option(DeltaOptions.PARTITION_OVERWRITE_MODE_OPTION, "dynamic") .save(tempDir.getCanonicalPath) checkDatasetUnorderly(data.select("value").as[String], "a", "b", "d") } } } test("batch write: append, dynamic partition overwrite string and integer partition column") { withSQLConf(DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED.key -> "true") { withTempDir { tempDir => def data: DataFrame = spark.read.format("delta").load(tempDir.toString) Seq((1, "x"), (2, "y"), (3, "z")).toDF("value", "part2") .withColumn("part1", $"value" % 2) .write .format("delta") .partitionBy("part1", "part2") .mode("append") .save(tempDir.getCanonicalPath) Seq((5, "x"), (7, "y")).toDF("value", "part2") .withColumn("part1", $"value" % 2) .write .format("delta") .partitionBy("part1", "part2") .mode("overwrite") .option(DeltaOptions.PARTITION_OVERWRITE_MODE_OPTION, "dynamic") .save(tempDir.getCanonicalPath) checkDatasetUnorderly(data.select("value").as[Int], 2, 3, 5, 7) } } } test("batch write: append, dynamic partition overwrite overwrites nothing") { withSQLConf(DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED.key -> "true") { withTempDir { tempDir => def data: DataFrame = spark.read.format("delta").load(tempDir.toString) Seq(("a", "x"), ("b", "y"), ("c", "x")).toDF("value", "part") .write .format("delta") .partitionBy("part") .mode("append") .save(tempDir.getCanonicalPath) Seq(("d", "z")).toDF("value", "part") .write .format("delta") .partitionBy("part") .mode("overwrite") .option(DeltaOptions.PARTITION_OVERWRITE_MODE_OPTION, "dynamic") .save(tempDir.getCanonicalPath) checkDatasetUnorderly(data.select("value", "part").as[(String, String)], ("a", "x"), ("b", "y"), ("c", "x"), ("d", "z")) } } } test("batch write: append, dynamic partition overwrite multiple partition columns") { withSQLConf(DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED.key -> "true") { withTempDir { tempDir => def data: DataFrame = spark.read.format("delta").load(tempDir.toString) Seq(("a", "x", 1), ("b", "y", 2), ("c", "x", 3)).toDF("part1", "part2", "value") .write .format("delta") .partitionBy("part1", "part2") .mode("append") .save(tempDir.getCanonicalPath) Seq(("a", "x", 4), ("d", "x", 5)).toDF("part1", "part2", "value") .write .format("delta") .partitionBy("part1", "part2") .mode("overwrite") .option(DeltaOptions.PARTITION_OVERWRITE_MODE_OPTION, "dynamic") .save(tempDir.getCanonicalPath) checkDatasetUnorderly(data.select("part1", "part2", "value").as[(String, String, Int)], ("a", "x", 4), ("b", "y", 2), ("c", "x", 3), ("d", "x", 5)) } } } test("batch write: append, dynamic partition overwrite without partitionBy") { withSQLConf(DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED.key -> "true") { withTempDir { tempDir => def data: DataFrame = spark.read.format("delta").load(tempDir.toString) Seq(1, 2, 3).toDF .withColumn("part", $"value" % 2) .write .format("delta") .partitionBy("part") .mode("append") .save(tempDir.getCanonicalPath) Seq(1, 5).toDF .withColumn("part", $"value" % 2) .write .format("delta") .mode("overwrite") .option(DeltaOptions.PARTITION_OVERWRITE_MODE_OPTION, "dynamic") .save(tempDir.getCanonicalPath) checkDatasetUnorderly(data.select("value").as[Int], 1, 2, 5) } } } test("batch write: append, dynamic partition overwrite conf, replaceWhere takes precedence") { // when dynamic partition overwrite mode is enabled in the spark configuration, and a // replaceWhere expression is provided, we delete data according to the replaceWhere expression withSQLConf( DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED.key -> "true", SQLConf.PARTITION_OVERWRITE_MODE.key -> "dynamic") { withTempDir { tempDir => def data: DataFrame = spark.read.format("delta").load(tempDir.toString) Seq((1, "x"), (2, "y"), (3, "z")).toDF("value", "part2") .withColumn("part1", $"value" % 2) .write .format("delta") .partitionBy("part1", "part2") .mode("append") .save(tempDir.getCanonicalPath) Seq((5, "x")).toDF("value", "part2") .withColumn("part1", $"value" % 2) .write .format("delta") .partitionBy("part1", "part2") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "part1 = 1") .save(tempDir.getCanonicalPath) checkDatasetUnorderly(data.select($"value").as[Int], 2, 5) } } } test("batch write: append, replaceWhere + dynamic partition overwrite enabled in options") { // when dynamic partition overwrite mode is enabled in the DataFrameWriter options, and // a replaceWhere expression is provided, we throw an error withSQLConf(DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED.key -> "true") { withTempDir { tempDir => Seq((1, "x"), (2, "y"), (3, "z")).toDF("value", "part2") .withColumn("part1", $"value" % 2) .write .format("delta") .partitionBy("part1", "part2") .mode("append") .save(tempDir.getCanonicalPath) val e = intercept[IllegalArgumentException] { Seq((3, "x"), (5, "x")).toDF("value", "part2") .withColumn("part1", $"value" % 2) .write .format("delta") .partitionBy("part1", "part2") .mode("overwrite") .option(DeltaOptions.PARTITION_OVERWRITE_MODE_OPTION, "dynamic") .option(DeltaOptions.REPLACE_WHERE_OPTION, "part1 = 1") .save(tempDir.getCanonicalPath) } assert(e.getMessage === "[DELTA_REPLACE_WHERE_WITH_DYNAMIC_PARTITION_OVERWRITE] " + "A 'replaceWhere' expression and " + "'partitionOverwriteMode'='dynamic' cannot both be set in the DataFrameWriter options.") } } } test("batch write: append, dynamic partition overwrite set via conf") { withSQLConf( DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED.key -> "true", SQLConf.PARTITION_OVERWRITE_MODE.key -> "dynamic") { withTempDir { tempDir => def data: DataFrame = spark.read.format("delta").load(tempDir.toString) Seq(1, 2, 3).toDF .withColumn("part", $"value" % 2) .write .format("delta") .partitionBy("part") .mode("append") .save(tempDir.getCanonicalPath) Seq(1, 5).toDF .withColumn("part", $"value" % 2) .write .format("delta") .partitionBy("part") .mode("overwrite") .save(tempDir.getCanonicalPath) checkDatasetUnorderly(data.select("value").as[Int], 1, 2, 5) } } } test("batch write: append, dynamic partition overwrite set via conf and overridden via option") { withSQLConf( DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED.key -> "true", SQLConf.PARTITION_OVERWRITE_MODE.key -> "dynamic") { withTempDir { tempDir => def data: DataFrame = spark.read.format("delta").load(tempDir.toString) Seq(1, 2, 3).toDF .withColumn("part", $"value" % 2) .write .format("delta") .partitionBy("part") .mode("append") .save(tempDir.getCanonicalPath) Seq(1, 5).toDF .withColumn("part", $"value" % 2) .write .format("delta") .partitionBy("part") .option(DeltaOptions.PARTITION_OVERWRITE_MODE_OPTION, "static") .mode("overwrite") .save(tempDir.getCanonicalPath) checkDatasetUnorderly(data.select("value").as[Int], 1, 5) } } } test("batch write: append, overwrite without partitions should ignore partition overwrite mode") { withSQLConf(DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED.key -> "true") { withTempDir { tempDir => def data: DataFrame = spark.read.format("delta").load(tempDir.toString) Seq(1, 2, 3).toDF .withColumn("part", $"value" % 2) .write .format("delta") .mode("append") .save(tempDir.getCanonicalPath) Seq(1, 5).toDF .withColumn("part", $"value" % 2) .write .format("delta") .mode("overwrite") .option(DeltaOptions.PARTITION_OVERWRITE_MODE_OPTION, "dynamic") .save(tempDir.getCanonicalPath) checkDatasetUnorderly(data.select("value").as[Int], 1, 5) } } } test("batch write: append, overwrite non-partitioned table with replaceWhere ignores partition " + "overwrite mode option") { // we check here that setting both replaceWhere and dynamic partition overwrite in the // DataFrameWriter options is allowed for a non-partitioned table withSQLConf(DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED.key -> "true") { withTempDir { tempDir => def data: DataFrame = spark.read.format("delta").load(tempDir.toString) Seq(1, 2, 3).toDF .withColumn("part", $"value" % 2) .write .format("delta") .mode("append") .save(tempDir.getCanonicalPath) Seq(1, 5).toDF .withColumn("part", $"value" % 2) .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "part = 1") .option(DeltaOptions.PARTITION_OVERWRITE_MODE_OPTION, "dynamic") .save(tempDir.getCanonicalPath) checkDatasetUnorderly(data.select("value").as[Int], 1, 2, 5) } } } test("batch write: append, dynamic partition with 'partitionValues' column") { withSQLConf(DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED.key -> "true") { withTempDir { tempDir => def data: DataFrame = spark.read.format("delta").load(tempDir.toString) Seq(1, 2, 3).toDF .withColumn("partitionValues", $"value" % 2) .write .format("delta") .partitionBy("partitionValues") .mode("append") .save(tempDir.getCanonicalPath) Seq(1, 5).toDF .withColumn("partitionValues", $"value" % 2) .write .format("delta") .partitionBy("partitionValues") .mode("overwrite") .option(DeltaOptions.PARTITION_OVERWRITE_MODE_OPTION, "dynamic") .save(tempDir.getCanonicalPath) checkDatasetUnorderly(data.select("value").as[Int], 1, 2, 5) } } } test("batch write: ignore") { withTempDir { tempDir => def data: DataFrame = spark.read.format("delta").load(tempDir.toString) Seq(1, 2, 3).toDF .write .format("delta") .mode("ignore") .save(tempDir.getCanonicalPath) checkDatasetUnorderly(data.toDF.as[Int], 1, 2, 3) // The following data will be ignored Seq(4, 5, 6).toDF .write .format("delta") .mode("ignore") .save(tempDir.getCanonicalPath) checkDatasetUnorderly(data.toDF.as[Int], 1, 2, 3) } } test("batch write: error") { withTempDir { tempDir => def data: DataFrame = spark.read.format("delta").load(tempDir.toString) Seq(1, 2, 3).toDF .write .format("delta") .mode("error") .save(tempDir.getCanonicalPath) checkDatasetUnorderly(data.toDF.as[Int], 1, 2, 3) val e = intercept[AnalysisException] { Seq(4, 5, 6).toDF .write .format("delta") .mode("error") .save(tempDir.getCanonicalPath) } assert(e.getMessage.contains("Cannot write to already existent path")) } } testQuietly("creating log should not create the log directory") { withTempDir { tempDir => if (tempDir.exists()) { assert(tempDir.delete()) } // Creating an empty log should not create the directory assert(!tempDir.exists()) // Writing to table should create the directory Seq(1, 2, 3).toDF .write .format("delta") .save(tempDir.getCanonicalPath) def data: DataFrame = spark.read.format("delta").load(tempDir.toString) checkDatasetUnorderly(data.toDF.as[Int], 1, 2, 3) } } test("read via data source API when the directory doesn't exist") { withTempDir { tempDir => if (tempDir.exists()) { assert(tempDir.delete()) } // a batch query should fail at once var e = intercept[AnalysisException] { spark.read .format("delta") .load(tempDir.getCanonicalPath) .show() } assert(e.getMessage.contains("Path does not exist")) assert(e.getMessage.contains(tempDir.getCanonicalPath)) assert(!tempDir.exists()) // a streaming query will also fail but it's because there is no schema e = intercept[AnalysisException] { spark.readStream .format("delta") .load(tempDir.getCanonicalPath) } assert(e.getMessage.contains("Table schema is not set")) assert(e.getMessage.contains("CREATE TABLE")) } } test("write via data source API when the directory doesn't exist") { withTempDir { tempDir => if (tempDir.exists()) { assert(tempDir.delete()) } // a batch query should create the output directory automatically Seq(1, 2, 3).toDF .write .format("delta").save(tempDir.getCanonicalPath) checkDatasetUnorderly( spark.read.format("delta").load(tempDir.getCanonicalPath).as[Int], 1, 2, 3) Utils.deleteRecursively(tempDir) assert(!tempDir.exists()) // a streaming query should create the output directory automatically val input = MemoryStream[Int] val q = input.toDF .writeStream .format("delta") .option( "checkpointLocation", Utils.createTempDir(namePrefix = "tahoe-test").getCanonicalPath) .start(tempDir.getCanonicalPath) try { input.addData(1, 2, 3) q.processAllAvailable() checkDatasetUnorderly( spark.read.format("delta").load(tempDir.getCanonicalPath).as[Int], 1, 2, 3) } finally { q.stop() } } } test("support partitioning with batch data source API - append") { withTempDir { tempDir => if (tempDir.exists()) { assert(tempDir.delete()) } spark.range(100).select('id, 'id % 4 as "by4", 'id % 8 as "by8") .write .format("delta") .partitionBy("by4", "by8") .save(tempDir.toString) val files = spark.read.format("delta").load(tempDir.toString).inputFiles val deltaLog = loadDeltaLog(tempDir.getAbsolutePath) assertPartitionExists("by4", deltaLog, files) assertPartitionExists("by8", deltaLog, files) } } test("support removing partitioning") { withTempDir { tempDir => if (tempDir.exists()) { assert(tempDir.delete()) } spark.range(100).select('id, 'id % 4 as "by4") .write .format("delta") .partitionBy("by4") .save(tempDir.toString) val deltaLog = DeltaLog.forTable(spark, tempDir) assert(deltaLog.snapshot.metadata.partitionColumns === Seq("by4")) spark.read.format("delta").load(tempDir.toString).write .option(DeltaOptions.OVERWRITE_SCHEMA_OPTION, "true") .format("delta") .mode(SaveMode.Overwrite) .save(tempDir.toString) assert(deltaLog.snapshot.metadata.partitionColumns === Nil) } } test("columns with commas as partition columns") { withTempDir { tempDir => if (tempDir.exists()) { assert(tempDir.delete()) } val dfw = spark.range(100).select('id, 'id % 4 as "by,4") .write .format("delta") .partitionBy("by,4") // if in column mapping mode, we should not expect invalid character errors if (!columnMappingEnabled) { val e = intercept[AnalysisException] { dfw.save(tempDir.toString) } assert(e.getMessage.contains("invalid character(s)")) } withSQLConf(DeltaSQLConf.DELTA_PARTITION_COLUMN_CHECK_ENABLED.key -> "false") { dfw.save(tempDir.toString) } // Note: although we are able to write, we cannot read the table with Spark 3.2+ with // OSS Delta 1.1.0+ because SPARK-36271 adds a column name check in the read path. } } test("throw exception when users are trying to write in batch with different partitioning") { withTempDir { tempDir => if (tempDir.exists()) { assert(tempDir.delete()) } spark.range(100).select('id, 'id % 4 as "by4", 'id % 8 as "by8") .write .format("delta") .partitionBy("by4", "by8") .save(tempDir.toString) val e = intercept[AnalysisException] { spark.range(100).select('id, 'id % 4 as "by4") .write .format("delta") .partitionBy("by4") .mode("append") .save(tempDir.toString) } assert(e.getMessage.contains("Partition columns do not match")) } } test("incompatible schema merging throws errors") { withTempDir { tempDir => if (tempDir.exists()) { assert(tempDir.delete()) } spark.range(100).select('id, ('id * 3).cast("string") as "value") .write .format("delta") .save(tempDir.toString) val e = intercept[AnalysisException] { spark.range(100).select('id, 'id * 3 as "value") .write .format("delta") .mode("append") .save(tempDir.toString) } checkError( e, "DELTA_FAILED_TO_MERGE_FIELDS", parameters = Map("currentField" -> "value", "updateField" -> "value")) } } test("support partitioning with batch data source API - overwrite") { withTempDir { tempDir => if (tempDir.exists()) { assert(tempDir.delete()) } spark.range(100).select('id, 'id % 4 as "by4") .write .format("delta") .partitionBy("by4") .save(tempDir.toString) val files = spark.read.format("delta").load(tempDir.toString).inputFiles val deltaLog = loadDeltaLog(tempDir.getAbsolutePath) assertPartitionExists("by4", deltaLog, files) spark.range(101, 200).select('id, 'id % 4 as "by4", 'id % 8 as "by8") .write .format("delta") .option(DeltaOptions.MERGE_SCHEMA_OPTION, "true") .mode("overwrite") .save(tempDir.toString) checkAnswer( spark.read.format("delta").load(tempDir.toString), spark.range(101, 200).select('id, 'id % 4 as "by4", 'id % 8 as "by8")) } } test("overwrite and replaceWhere should check partitioning compatibility") { withTempDir { tempDir => if (tempDir.exists()) { assert(tempDir.delete()) } spark.range(100).select('id, 'id % 4 as "by4") .write .format("delta") .partitionBy("by4") .save(tempDir.toString) val files = spark.read.format("delta").load(tempDir.toString).inputFiles val deltaLog = loadDeltaLog(tempDir.getAbsolutePath) assertPartitionExists("by4", deltaLog, files) val e = intercept[AnalysisException] { spark.range(101, 200).select('id, 'id % 4 as "by4", 'id % 8 as "by8") .write .format("delta") .partitionBy("by4", "by8") .option(DeltaOptions.REPLACE_WHERE_OPTION, "by4 > 0") .mode("overwrite") .save(tempDir.toString) } assert(e.getMessage.contains("Partition columns do not match")) } } test("can't write out with all columns being partition columns") { withTempDir { tempDir => SaveMode.values().foreach { mode => if (tempDir.exists()) { assert(tempDir.delete()) } val e = intercept[AnalysisException] { spark.range(100).select('id, 'id % 4 as "by4") .write .format("delta") .partitionBy("by4", "id") .mode(mode) .save(tempDir.toString) } assert(e.getMessage.contains("Cannot use all columns for partition columns")) } } } test("SC-8727 - default snapshot num partitions") { withTempDir { tempDir => spark.range(10).write.format("delta").save(tempDir.toString) val deltaLog = DeltaLog.forTable(spark, tempDir) val numParts = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_SNAPSHOT_PARTITIONS).get assert(deltaLog.snapshot.stateDS.rdd.getNumPartitions == numParts) } } test("SC-8727 - can't set negative num partitions") { withTempDir { tempDir => val caught = intercept[IllegalArgumentException] { withSQLConf(("spark.databricks.delta.snapshotPartitions", "-1")) {} } assert(caught.getMessage.contains("Delta snapshot partition number must be positive.")) } } test("SC-8727 - reconfigure num partitions") { withTempDir { tempDir => withSQLConf(("spark.databricks.delta.snapshotPartitions", "410")) { spark.range(10).write.format("delta").save(tempDir.toString) val deltaLog = DeltaLog.forTable(spark, tempDir) assert(deltaLog.snapshot.stateDS.rdd.getNumPartitions == 410) } } } test("SC-8727 - can't set zero num partitions") { withTempDir { tempDir => val caught = intercept[IllegalArgumentException] { withSQLConf(("spark.databricks.delta.snapshotPartitions", "0")) {} } assert(caught.getMessage.contains("Delta snapshot partition number must be positive.")) } } testQuietly("SC-8810: skip deleted file") { withSQLConf( ("spark.sql.files.ignoreMissingFiles", "true")) { withTempDir { tempDir => val tempDirPath = new Path(tempDir.getCanonicalPath) Seq(1).toDF().write.format("delta").mode("append").save(tempDir.toString) Seq(2, 2).toDF().write.format("delta").mode("append").save(tempDir.toString) Seq(4).toDF().write.format("delta").mode("append").save(tempDir.toString) Seq(5).toDF().write.format("delta").mode("append").save(tempDir.toString) def data: DataFrame = spark.read.format("delta").load(tempDir.toString) val deltaLog = DeltaLog.forTable(spark, tempDir) // The file names are opaque. To identify which one we're deleting, we ensure that only one // append has 2 partitions, and give them the same value so we know what was deleted. val inputFiles = TahoeLogFileIndex(spark, deltaLog).inputFiles.toSeq assert(inputFiles.size == 5) val filesToDelete = inputFiles.filter(_.split("/").last.contains("part-00001")) assert(filesToDelete.size == 1) filesToDelete.foreach { f => val deleted = tryDeleteNonRecursive( tempDirPath.getFileSystem(deltaLog.newDeltaHadoopConf()), new Path(tempDirPath, f)) assert(deleted) } // The single 2 that we deleted should be missing, with the rest of the data still present. checkAnswer(data.toDF(), Row(1) :: Row(2) :: Row(4) :: Row(5) :: Nil) } } } testQuietly("SC-8810: skipping deleted file still throws on corrupted file") { withSQLConf(("spark.sql.files.ignoreMissingFiles", "true")) { withTempDir { tempDir => val tempDirPath = new Path(tempDir.getCanonicalPath) Seq(1).toDF().write.format("delta").mode("append").save(tempDir.toString) Seq(2, 2).toDF().write.format("delta").mode("append").save(tempDir.toString) Seq(4).toDF().write.format("delta").mode("append").save(tempDir.toString) Seq(5).toDF().write.format("delta").mode("append").save(tempDir.toString) def data: DataFrame = spark.read.format("delta").load(tempDir.toString) val deltaLog = DeltaLog.forTable(spark, tempDir) // The file names are opaque. To identify which one we're deleting, we ensure that only one // append has 2 partitions, and give them the same value so we know what was deleted. val inputFiles = TahoeLogFileIndex(spark, deltaLog).inputFiles.toSeq assert(inputFiles.size == 5) val filesToCorrupt = inputFiles.filter(_.split("/").last.contains("part-00001")) assert(filesToCorrupt.size == 1) val fs = tempDirPath.getFileSystem(deltaLog.newDeltaHadoopConf()) filesToCorrupt.foreach { f => val filePath = new Path(tempDirPath, f) fs.create(filePath, true).close() } val thrown = intercept[SparkException] { data.toDF().collect() } assert(thrown.getMessage.contains("[FAILED_READ_FILE.NO_HINT]")) } } } testQuietly("SC-8810: skip multiple deleted files") { withSQLConf(("spark.sql.files.ignoreMissingFiles", "true")) { withTempDir { tempDir => val tempDirPath = new Path(tempDir.getCanonicalPath) def data: DataFrame = spark.read.format("delta").load(tempDir.toString) val deltaLog = DeltaLog.forTable(spark, tempDir) Range(0, 10).foreach(n => Seq(n).toDF().write.format("delta").mode("append").save(tempDir.toString)) val inputFiles = TahoeLogFileIndex(spark, deltaLog).inputFiles.toSeq val filesToDelete = inputFiles.take(4) filesToDelete.foreach { f => val deleted = tryDeleteNonRecursive( tempDirPath.getFileSystem(deltaLog.newDeltaHadoopConf()), new Path(tempDirPath, f)) assert(deleted) } // We don't have a good way to tell which specific values got deleted, so just check that // the right number remain. (Note that this works because there's 1 value per append, which // means 1 value per file.) assert(data.toDF().collect().size == 6) } } } testQuietly("deleted files cause failure by default") { withTempDir { tempDir => val tempDirPath = new Path(tempDir.getCanonicalPath) def data: DataFrame = spark.read.format("delta").load(tempDir.toString) val deltaLog = DeltaLog.forTable(spark, tempDir) Range(0, 10).foreach(n => Seq(n).toDF().write.format("delta").mode("append").save(tempDir.toString)) val inputFiles = TahoeLogFileIndex(spark, deltaLog).inputFiles.toSeq val fileToDelete = inputFiles.head val pathToDelete = new Path(tempDirPath, fileToDelete) val deleted = tryDeleteNonRecursive( tempDirPath.getFileSystem(deltaLog.newDeltaHadoopConf()), pathToDelete) assert(deleted) val thrown = intercept[SparkException] { data.toDF().collect() } assert(thrown.getMessage.contains("[FAILED_READ_FILE.FILE_NOT_EXIST]")) } } test("ES-4716: Delta shouldn't be broken when users turn on case sensitivity") { withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") { withTempDir { tempDir => // We use a column with the weird name just to make sure that customer configurations still // work. The original bug was within the `Snapshot` code, where we referred to `metaData` // as `metadata`. Seq(1, 2, 3).toDF("aBc").write.format("delta").mode("append").save(tempDir.toString) def testDf(columnName: Symbol): Unit = { DeltaLog.clearCache() val df = spark.read.format("delta").load(tempDir.getCanonicalPath).select(columnName) checkDatasetUnorderly(df.as[Int], 1, 2, 3) } withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { testDf('aBc) intercept[AnalysisException] { testDf('abc) } } testDf('aBc) testDf('abc) } } } test("special chars in base path") { withTempDir { dir => val basePath = new File(new File(dir, "some space"), "and#spec*al+ch@rs") spark.range(10).write.format("delta").save(basePath.getCanonicalPath) checkAnswer( spark.read.format("delta").load(basePath.getCanonicalPath), spark.range(10).toDF() ) } } test("get touched files for update, delete and merge") { withTempDir { dir => val directory = new File(dir, "test with space") val df = Seq((1, 10), (2, 20), (3, 30), (4, 40)).toDF("key", "value") val writer = df.write.format("delta").mode("append") writer.save(directory.getCanonicalPath) spark.sql(s"UPDATE delta.`${directory.getCanonicalPath}` SET value = value + 10") spark.sql(s"DELETE FROM delta.`${directory.getCanonicalPath}` WHERE key = 4") Seq((3, 30)).toDF("key", "value").createOrReplaceTempView("inbound") spark.sql(s"""|MERGE INTO delta.`${directory.getCanonicalPath}` AS base |USING inbound |ON base.key = inbound.key |WHEN MATCHED THEN UPDATE SET base.value = |base.value+inbound.value""".stripMargin) spark.sql(s"UPDATE delta.`${directory.getCanonicalPath}` SET value = 40 WHERE key = 1") spark.sql(s"DELETE FROM delta.`${directory.getCanonicalPath}` WHERE key = 2") checkAnswer( spark.read.format("delta").load(directory.getCanonicalPath), Seq((1, 40), (3, 70)).toDF("key", "value") ) } } test("support Java8 API for DATE type") { withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") { val tableName = "my_table" withTable(tableName) { spark.sql(s"CREATE TABLE $tableName (id STRING, date DATE) USING DELTA;") spark.sql( s""" |INSERT INTO $tableName REPLACE |where (DATE IN (DATE('2024-03-11'), DATE('2024-03-13'))) |VALUES ('2', DATE('2024-03-13')), ('3', DATE('2024-03-11')) |""".stripMargin) } } } test("support Java8 API for TIMESTAMP type") { withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") { val tableName = "my_table" withTable(tableName) { spark.sql(s"CREATE TABLE $tableName (id STRING, timestamp TIMESTAMP) USING DELTA;") spark.sql( s""" |INSERT INTO $tableName REPLACE |where | (timestamp IN (TIMESTAMP('2022-12-22 15:50:00'), TIMESTAMP('2022-12-23 15:50:00'))) | VALUES | ('2', TIMESTAMP('2022-12-22 15:50:00')), ('3', TIMESTAMP('2022-12-23 15:50:00')) |""".stripMargin) } } } test("all operations with special characters in path") { withTempDir { dir => val directory = new File(dir, "test with space") val df = Seq((1, 10), (2, 20), (3, 30), (4, 40)).toDF("key", "value") val writer = df.write.format("delta").mode("append") writer.save(directory.getCanonicalPath) // UPDATE and DELETE spark.sql(s"UPDATE delta.`${directory.getCanonicalPath}` SET value = 99") spark.sql(s"DELETE FROM delta.`${directory.getCanonicalPath}` WHERE key = 4") spark.sql(s"DELETE FROM delta.`${directory.getCanonicalPath}` WHERE key = 3") checkAnswer( spark.read.format("delta").load(directory.getCanonicalPath), Seq((1, 99), (2, 99)).toDF("key", "value") ) // INSERT spark.sql(s"INSERT INTO delta.`${directory.getCanonicalPath}` VALUES (5, 50)") spark.sql(s"INSERT INTO delta.`${directory.getCanonicalPath}` VALUES (5, 50)") checkAnswer( spark.read.format("delta").load(directory.getCanonicalPath), Seq((1, 99), (2, 99), (5, 50), (5, 50)).toDF("key", "value") ) // MERGE Seq((1, 1), (3, 88), (5, 88)).toDF("key", "value").createOrReplaceTempView("inbound") spark.sql( s"""|MERGE INTO delta.`${directory.getCanonicalPath}` AS base |USING inbound |ON base.key = inbound.key |WHEN MATCHED THEN DELETE |WHEN NOT MATCHED THEN INSERT * |""".stripMargin) checkAnswer( spark.read.format("delta").load(directory.getCanonicalPath), Seq((2, 99), (3, 88)).toDF("key", "value") ) // DELETE and INSERT again spark.sql(s"DELETE FROM delta.`${directory.getCanonicalPath}` WHERE key = 3") spark.sql(s"INSERT INTO delta.`${directory.getCanonicalPath}` VALUES (5, 99)") checkAnswer( spark.read.format("delta").load(directory.getCanonicalPath), Seq((2, 99), (5, 99)).toDF("key", "value") ) assume(!catalogOwnedDefaultCreationEnabledInTests, "VACUUM is blocked on catalog-managed tables") // VACUUM withSQLConf(DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED.key -> "false") { spark.sql(s"VACUUM delta.`${directory.getCanonicalPath}` RETAIN 0 HOURS") } checkAnswer( spark.sql(s"SELECT * FROM delta.`${directory.getCanonicalPath}@v8`"), Seq((2, 99), (5, 99)).toDF("key", "value") ) // Version 0 should be lost, as version 1 rewrites the whole file val ex = intercept[Exception] { checkAnswer( spark.sql(s"SELECT * FROM delta.`${directory.getCanonicalPath}@v0`"), spark.emptyDataFrame ) } var cause = ex.getCause while (cause.getCause != null) { cause = cause.getCause } assert(cause.getMessage.contains(".parquet does not exist")) } } test("can't create zero-column table with a write") { withTempDir { dir => intercept[AnalysisException] { Seq(1).toDF("a").drop("a").write.format("delta").save(dir.getAbsolutePath) } } } test("SC-10573: InSet operator prunes partitions properly") { withTempDir { dir => val path = dir.getCanonicalPath Seq((1, 1L, "1")).toDS() .write .format("delta") .partitionBy("_2", "_3") .save(path) val df = spark.read.format("delta").load(path) .where("_2 IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)").select("_1") val condition = df.queryExecution.optimizedPlan.collectFirst { case f: Filter => f.condition } assert(condition.exists(_.isInstanceOf[InSet])) checkAnswer(df, Row(1)) } } test("SC-24886: partition columns have correct datatype in metadata scans") { withTempDir { inputDir => Seq(("foo", 2019)).toDF("name", "y") .write.format("delta").partitionBy("y").mode("overwrite") .save(inputDir.getAbsolutePath) // Before the fix, this query would fail because it tried to read strings from the metadata // partition values as the LONG type that the actual partition columns are. This works now // because we added a cast. val df = spark.read.format("delta") .load(inputDir.getAbsolutePath) .where( """cast(format_string("%04d-01-01 12:00:00", y) as timestamp) is not null""".stripMargin) assert(df.collect().length == 1) } } test("SC-11332: session isolation for cached delta logs") { withTempDir { tempDir => val path = tempDir.getCanonicalPath val oldSession = spark val deltaLog = DeltaLog.forTable(spark, path) val maxSLL = deltaLog.maxSnapshotLineageLength val activeSession = oldSession.newSession() SparkSession.setActiveSession(activeSession) activeSession.sessionState.conf.setConf( DeltaSQLConf.DELTA_MAX_SNAPSHOT_LINEAGE_LENGTH, maxSLL + 1) // deltaLog fetches conf from active session assert(deltaLog.maxSnapshotLineageLength == maxSLL + 1) // new session confs don't propagate to old session assert(maxSLL == oldSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_MAX_SNAPSHOT_LINEAGE_LENGTH)) } } test("SC-11198: global configs - save to path") { withTempDir { dir => val path = dir.getCanonicalPath withSQLConf("spark.databricks.delta.properties.defaults.dataSkippingNumIndexedCols" -> "1") { spark.range(5).write.format("delta").save(path) val tableConfigs = DeltaLog.forTable(spark, path).update().metadata.configuration assert(tableConfigs.get("delta.dataSkippingNumIndexedCols") == Some("1")) } } } test("SC-24982 - initial snapshot has zero partitions") { withTempDir { tempDir => val deltaLog = DeltaLog.forTable(spark, tempDir) assert(deltaLog.snapshot.stateDS.rdd.getNumPartitions == 0) } } test("SC-24982 - initial snapshot does not trigger jobs") { val jobCount = new AtomicInteger(0) val listener = new SparkListener { override def onJobStart(jobStart: SparkListenerJobStart): Unit = { // Spark will always log a job start/end event even when the job does not launch any task. if (jobStart.stageInfos.exists(_.numTasks > 0)) { jobCount.incrementAndGet() } } } sparkContext.listenerBus.waitUntilEmpty(15000) sparkContext.addSparkListener(listener) try { withTempDir { tempDir => val files = DeltaLog.forTable(spark, tempDir).snapshot.stateDS.collect() assert(files.isEmpty) } sparkContext.listenerBus.waitUntilEmpty(15000) assert(jobCount.get() == 0) } finally { sparkContext.removeSparkListener(listener) } } def lastDeltaHistory(dir: String): DeltaHistory = io.delta.tables.DeltaTable.forPath(spark, dir).history(1).as[DeltaHistory].head test("history includes user-defined metadata for DataFrame.Write API") { val tempDir = Utils.createTempDir().toString val df = Seq(2).toDF().write.format("delta").mode("overwrite") df.option("userMetadata", "meta1") .save(tempDir) assert(lastDeltaHistory(tempDir).userMetadata === Some("meta1")) df.option("userMetadata", "meta2") .save(tempDir) assert(lastDeltaHistory(tempDir).userMetadata === Some("meta2")) } test("history includes user-defined metadata for SQL API") { val tempDir = Utils.createTempDir().toString val tblName = "tblName" withTable(tblName) { withSQLConf(DeltaSQLConf.DELTA_USER_METADATA.key -> "meta1") { spark.sql(s"CREATE TABLE $tblName (data STRING) USING delta LOCATION '$tempDir';") } assert(lastDeltaHistory(tempDir).userMetadata === Some("meta1")) withSQLConf(DeltaSQLConf.DELTA_USER_METADATA.key -> "meta2") { spark.sql(s"INSERT INTO $tblName VALUES ('test');") } assert(lastDeltaHistory(tempDir).userMetadata === Some("meta2")) withSQLConf(DeltaSQLConf.DELTA_USER_METADATA.key -> "meta3") { spark.sql(s"INSERT INTO $tblName VALUES ('test2');") } assert(lastDeltaHistory(tempDir).userMetadata === Some("meta3")) } } test("history includes user-defined metadata for DF.Write API and config setting") { val tempDir = Utils.createTempDir().toString val df = Seq(2).toDF().write.format("delta").mode("overwrite") withSQLConf(DeltaSQLConf.DELTA_USER_METADATA.key -> "meta1") { df.save(tempDir) } assert(lastDeltaHistory(tempDir).userMetadata === Some("meta1")) withSQLConf(DeltaSQLConf.DELTA_USER_METADATA.key -> "meta2") { df.option("userMetadata", "optionMeta2") .save(tempDir) } assert(lastDeltaHistory(tempDir).userMetadata === Some("optionMeta2")) } test("history includes user-defined metadata for SQL + DF.Write API") { val tempDir = Utils.createTempDir().toString val df = Seq(2).toDF().write.format("delta").mode("overwrite") // metadata given in `option` should beat config withSQLConf(DeltaSQLConf.DELTA_USER_METADATA.key -> "meta1") { df.option("userMetadata", "optionMeta1") .save(tempDir) } assert(lastDeltaHistory(tempDir).userMetadata === Some("optionMeta1")) withSQLConf(DeltaSQLConf.DELTA_USER_METADATA.key -> "meta2") { df.option("userMetadata", "optionMeta2") .save(tempDir) } assert(lastDeltaHistory(tempDir).userMetadata === Some("optionMeta2")) } test("SC-77958 - history includes user-defined metadata for createOrReplace") { withTable("tbl") { spark.range(10).writeTo("tbl").using("delta").option("userMetadata", "meta").createOrReplace() val history = sql("DESCRIBE HISTORY tbl LIMIT 1").as[DeltaHistory].head() assert(history.userMetadata === Some("meta")) } } test("SC-77958 - history includes user-defined metadata for saveAsTable") { withTable("tbl") { spark.range(10).write.format("delta").option("userMetadata", "meta1") .mode("overwrite").saveAsTable("tbl") val history = sql("DESCRIBE HISTORY tbl LIMIT 1").as[DeltaHistory].head() assert(history.userMetadata === Some("meta1")) } } test("lastCommitVersionInSession - init") { spark.sessionState.conf.unsetConf(DeltaSQLConf.DELTA_LAST_COMMIT_VERSION_IN_SESSION) withTempDir { tempDir => assert(spark.conf.get(DeltaSQLConf.DELTA_LAST_COMMIT_VERSION_IN_SESSION) === None) Seq(1).toDF .write .format("delta") .save(tempDir.getCanonicalPath) assert(spark.conf.get(DeltaSQLConf.DELTA_LAST_COMMIT_VERSION_IN_SESSION) === Some(0)) } } test("lastCommitVersionInSession - SQL") { spark.sessionState.conf.unsetConf(DeltaSQLConf.DELTA_LAST_COMMIT_VERSION_IN_SESSION) withTempDir { tempDir => val k = DeltaSQLConf.DELTA_LAST_COMMIT_VERSION_IN_SESSION.key assert(sql(s"SET $k").head().get(1) === "") Seq(1).toDF .write .format("delta") .save(tempDir.getCanonicalPath) assert(sql(s"SET $k").head().get(1) === "0") } } test("lastCommitVersionInSession - SQL only") { spark.sessionState.conf.unsetConf(DeltaSQLConf.DELTA_LAST_COMMIT_VERSION_IN_SESSION) withTable("test_table") { val k = DeltaSQLConf.DELTA_LAST_COMMIT_VERSION_IN_SESSION.key assert(sql(s"SET $k").head().get(1) === "") sql("CREATE TABLE test_table USING delta AS SELECT * FROM range(10)") assert(sql(s"SET $k").head().get(1) === "0") } } test("lastCommitVersionInSession - CONVERT TO DELTA") { withTempDir { tempDir => val path = tempDir.getCanonicalPath + "/table" spark.range(10).write.format("parquet").save(path) convertToDelta(s"parquet.`$path`") // In column mapping (name mode), we perform convertToDelta with a CONVERT and an ALTER, // so the version has been updated val commitVersion = if (columnMappingEnabled) 1 else 0 assert(spark.conf.get(DeltaSQLConf.DELTA_LAST_COMMIT_VERSION_IN_SESSION) === Some(commitVersion)) } } test("lastCommitVersionInSession - many writes") { withTempDir { tempDir => for (i <- 0 until 10) { Seq(i).toDF .write .mode("overwrite") .format("delta") .save(tempDir.getCanonicalPath) } Seq(10).toDF .write .format("delta") .mode("append") .save(tempDir.getCanonicalPath) assert(spark.conf.get(DeltaSQLConf.DELTA_LAST_COMMIT_VERSION_IN_SESSION) === Some(10)) } } test("lastCommitVersionInSession - new thread writes") { withTempDir { tempDir => Seq(1).toDF .write .format("delta") .mode("overwrite") .save(tempDir.getCanonicalPath) val t = new Thread { override def run(): Unit = { Seq(2).toDF .write .format("delta") .mode("overwrite") .save(tempDir.getCanonicalPath) } } t.start t.join assert(spark.conf.get(DeltaSQLConf.DELTA_LAST_COMMIT_VERSION_IN_SESSION) === Some(1)) } } // This test is only compatible w/ backfill batch size = 1. testWithCatalogOwned(backfillBatchSize = 1)( "An external write should be reflected during analysis of a path based query") { val tempDir = Utils.createTempDir().toString spark.range(10).coalesce(1).write.format("delta").mode("append").save(tempDir) spark.range(10, 20).coalesce(1).write.format("delta").mode("append").save(tempDir) val deltaLog = DeltaLog.forTable(spark, tempDir) val hadoopConf = deltaLog.newDeltaHadoopConf() val snapshot = deltaLog.snapshot val files = snapshot.allFiles.collect() // assign physical name to new schema val newMetadata = if (columnMappingEnabled) { DeltaColumnMapping.assignColumnIdAndPhysicalName( snapshot.metadata.copy(schemaString = new StructType().add("data", "bigint").json), snapshot.metadata, isChangingModeOnExistingTable = false, isOverwritingSchema = false) } else { snapshot.metadata.copy(schemaString = new StructType().add("data", "bigint").json) } // Now make a commit that comes from an "external" writer that deletes existing data and // changes the schema val actions = Seq(Action.supportedProtocolVersion( featuresToExclude = Seq(CatalogOwnedTableFeature)), newMetadata) ++ files.map(_.remove) deltaLog.store.write( FileNames.unsafeDeltaFile(deltaLog.logPath, snapshot.version + 1), actions.map(_.json).iterator, overwrite = false, hadoopConf) deltaLog.store.write( FileNames.unsafeDeltaFile(deltaLog.logPath, snapshot.version + 2), files.take(1).map(_.json).iterator, overwrite = false, hadoopConf) // Since the column `data` doesn't exist in our old files, we read it as null. checkAnswer( spark.read.format("delta").load(tempDir), Seq.fill(10)(Row(null)) ) } test("isBlindAppend with save and saveAsTable") { withTempDir { tempDir => val path = tempDir.getCanonicalPath withTable("blind_append") { sql(s"CREATE TABLE blind_append(value INT) USING delta LOCATION '$path'") // version = 0 sql("INSERT INTO blind_append VALUES(1)") // version = 1 spark.read.format("delta").load(path) .where("value = 1") .write.mode("append").format("delta").save(path) // version = 2 checkAnswer(spark.table("blind_append"), Row(1) :: Row(1) :: Nil) assert(sql("desc history blind_append") .select("version", "isBlindAppend").head == Row(2, false)) spark.table("blind_append").where("value = 1").write.mode("append").format("delta") .saveAsTable("blind_append") // version = 3 checkAnswer(spark.table("blind_append"), Row(1) :: Row(1) :: Row(1) :: Row(1) :: Nil) assert(sql("desc history blind_append") .select("version", "isBlindAppend").head == Row(3, false)) } } } test("isBlindAppend with DataFrameWriterV2") { withTempDir { tempDir => val path = tempDir.getCanonicalPath withTable("blind_append") { sql(s"CREATE TABLE blind_append(value INT) USING delta LOCATION '$path'") // version = 0 sql("INSERT INTO blind_append VALUES(1)") // version = 1 spark.read.format("delta").load(path) .where("value = 1") .writeTo("blind_append").append() // version = 2 checkAnswer(spark.table("blind_append"), Row(1) :: Row(1) :: Nil) assert(sql("desc history blind_append") .select("version", "isBlindAppend").head == Row(2, false)) } } } test("isBlindAppend with RTAS") { withTempDir { tempDir => val path = tempDir.getCanonicalPath withTable("blind_append") { sql(s"CREATE TABLE blind_append(value INT) USING delta LOCATION '$path'") // version = 0 sql("INSERT INTO blind_append VALUES(1)") // version = 1 sql("REPLACE TABLE blind_append USING delta AS SELECT * FROM blind_append") // version = 2 checkAnswer(spark.table("blind_append"), Row(1) :: Nil) assert(sql("desc history blind_append") .select("version", "isBlindAppend").head == Row(2, false)) } } } test("replaceWhere should support backtick when flag is disabled") { val table = "replace_where_backtick" withSQLConf(DeltaSQLConf.REPLACEWHERE_DATACOLUMNS_ENABLED.key -> "false") { withTable(table) { // The STRUCT column is added to prevent us from introducing any ambiguity in future sql(s"CREATE TABLE $table(`a.b` STRING, `c.d` STRING, a STRUCT)" + s"USING delta PARTITIONED BY (`a.b`)") Seq(("a", "b", "c")) .toDF("a.b", "c.d", "ab") .withColumn("a", struct($"ab".alias("b"))) .drop("ab") .write .format("delta") // "replaceWhere" should support backtick and remove it correctly. Technically, // "a.b" is not correct, but some users may already use it, // so we keep supporting both. This is not ambiguous since "replaceWhere" only // supports partition columns and it doesn't support struct type or map type. .option("replaceWhere", "`a.b` = 'a' AND a.b = 'a'") .mode("overwrite") .saveAsTable(table) checkAnswer(sql(s"SELECT `a.b`, `c.d`, a.b from $table"), Row("a", "b", "c") :: Nil) } } } test("replaceArbitrary should enforce proper usage of backtick") { val table = "replace_where_backtick" withTable(table) { sql(s"CREATE TABLE $table(`a.b` STRING, `c.d` STRING, a STRUCT)" + s"USING delta PARTITIONED BY (`a.b`)") // User has to use backtick properly. If they want to use a.b to match on `a.b`, // error will be thrown if `a.b` doesn't have the value. val e = intercept[AnalysisException] { Seq(("a", "b", "c")) .toDF("a.b", "c.d", "ab") .withColumn("a", struct($"ab".alias("b"))) .drop("ab") .write .format("delta") .option("replaceWhere", "a.b = 'a' AND `a.b` = 'a'") .mode("overwrite") .saveAsTable(table) } assert(e.getMessage.startsWith("[DELTA_REPLACE_WHERE_MISMATCH] " + "Written data does not conform to partial table overwrite condition or constraint")) Seq(("a", "b", "c"), ("d", "e", "f")) .toDF("a.b", "c.d", "ab") .withColumn("a", struct($"ab".alias("b"))) .drop("ab") .write .format("delta") .mode("overwrite") .saveAsTable(table) // Use backtick properly for `a.b` Seq(("a", "h", "c")) .toDF("a.b", "c.d", "ab") .withColumn("a", struct($"ab".alias("b"))) .drop("ab") .write .format("delta") .option("replaceWhere", "`a.b` = 'a'") .mode("overwrite") .saveAsTable(table) checkAnswer(sql(s"SELECT `a.b`, `c.d`, a.b from $table"), Row("a", "h", "c") :: Row("d", "e", "f") :: Nil) // struct field can only be referred by "a.b". Seq(("a", "b", "c")) .toDF("a.b", "c.d", "ab") .withColumn("a", struct($"ab".alias("b"))) .drop("ab") .write .format("delta") .option("replaceWhere", "a.b = 'c'") .mode("overwrite") .saveAsTable(table) checkAnswer(sql(s"SELECT `a.b`, `c.d`, a.b from $table"), Row("a", "b", "c") :: Row("d", "e", "f") :: Nil) } } test("need to update DeltaLog on DataFrameReader.load() code path") { // Due to possible race conditions (like in mounting/unmounting paths) there might be an initial // snapshot that gets cached for a table that should have a valid (non-initial) snapshot. In // such a case we need to call deltaLog.update() in the DataFrame read paths to update the // initial snapshot to a valid one. // // We simulate a cached InitialSnapshot + valid delta table by creating an empty DeltaLog // (which creates an InitialSnapshot cached for that path) then move an actual Delta table's // transaction log into the path for the empty log. val dir1 = Utils.createTempDir() val dir2 = Utils.createTempDir() val log = DeltaLog.forTable(spark, dir1) assert(!log.tableExists) spark.range(10).write.format("delta").save(dir2.getCanonicalPath) // rename dir2 to dir1 then read dir2.renameTo(dir1) checkAnswer(spark.read.format("delta").load(dir1.getCanonicalPath), spark.range(10).toDF) } test("set metadata upon write") { withSQLConf(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> "false") { withTempDir { inputDir => val testPath = inputDir.getCanonicalPath spark.range(10) .map(_.toInt) .withColumn("part", $"value" % 2) .write .format("delta") .option("delta.logRetentionDuration", "123 days") .option("mergeSchema", "true") .partitionBy("part") .mode("append") .save(testPath) val deltaLog = DeltaLog.forTable(spark, testPath) val metadata = deltaLog.snapshot.metadata // We need to drop default properties set by subclasses to make this test pass in them // We need to drop `enableDeletionVectors` property b/c it is explicitly set to false. assert( metadata.configuration .filter { case (k, _) => !k.startsWith("delta.columnMapping.") && !k.startsWith("delta.enableDeletionVectors")} === Map("delta.logRetentionDuration" -> "123 days") ++ extractCatalogOwnedSpecificPropertiesIfEnabled(metadata)) } } } test("idempotent write: idempotent DataFrame insert") { withTempDir { tableDir => spark.conf.set("spark.databricks.delta.write.txnAppId", "insertTest") io.delta.tables.DeltaTable.createOrReplace(spark) .addColumn("col1", "INT") .addColumn("col2", "INT") .location(tableDir.getCanonicalPath) .execute() val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tableDir.getCanonicalPath) def runInsert(data: (Int, Int)): Unit = { Seq(data).toDF("col1", "col2") .write .format("delta") .mode("append") .save(tableDir.getCanonicalPath) } def assertTable(numRows: Int): Unit = { val count = deltaTable.toDF.count() assert(count == numRows) } // run insert (1,1), table should have 1 row (1,1) spark.conf.set("spark.databricks.delta.write.txnVersion", "1") runInsert((1, 1)) assertTable(1) // run insert (2,2), table should have 2 rows (1,1),(2,2) spark.conf.set("spark.databricks.delta.write.txnVersion", "2") runInsert((2, 2)) assertTable(2) // retry update 2, table should have 2 rows (1,1),(2,2) spark.conf.set("spark.databricks.delta.write.txnVersion", "2") runInsert((2, 2)) assertTable(2) // run insert (3,3), table should have 3 rows (1,1),(2,2),(3,3) spark.conf.set("spark.databricks.delta.write.txnVersion", "3") runInsert((3, 3)) assertTable(3) // clean up spark.conf.unset("spark.databricks.delta.write.txnAppId") spark.conf.unset("spark.databricks.delta.write.txnVersion") } } test("idempotent write: idempotent SQL insert") { withTempDir { tableDir => val tableName = "myInsertTable" spark.conf.set("spark.databricks.delta.write.txnAppId", "insertTestSQL") spark.sql(s"CREATE TABLE $tableName (col1 INT, col2 INT) USING DELTA LOCATION '" + tableDir.getCanonicalPath + "'") def runInsert(data: (Int, Int)): Unit = { spark.sql(s"INSERT INTO $tableName (col1, col2) VALUES (${data._1}, ${data._2})") } def assertTable(numRows: Int): Unit = { val count = spark.sql(s"SELECT * FROM $tableName").count() assert(count == numRows) } // run insert (1,1), table should have 1 row (1,1) spark.conf.set("spark.databricks.delta.write.txnVersion", "1") runInsert((1, 1)) assertTable(1) // run insert (2,2), table should have 2 rows (1,1),(2,2) spark.conf.set("spark.databricks.delta.write.txnVersion", "2") runInsert((2, 2)) assertTable(2) // retry update 2, table should have 2 rows (1,1),(2,2) spark.conf.set("spark.databricks.delta.write.txnVersion", "2") runInsert((2, 2)) assertTable(2) // run insert (3,3), table should have 3 rows (1,1),(2,2),(3,3) spark.conf.set("spark.databricks.delta.write.txnVersion", "3") runInsert((3, 3)) assertTable(3) // clean up spark.conf.unset("spark.databricks.delta.write.txnAppId") spark.conf.unset("spark.databricks.delta.write.txnVersion") } } test("idempotent write: idempotent DeltaTable merge") { withTempDir { tableDir => spark.conf.set("spark.databricks.delta.write.txnAppId", "mergeTest") io.delta.tables.DeltaTable.createOrReplace(spark) .addColumn("col1", "INT") .addColumn("col2", "INT") .location(tableDir.getCanonicalPath) .execute() val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tableDir.getCanonicalPath) def runMerge(data: (Int, Int)): Unit = { val df = Seq(data).toDF("col1", "col2") deltaTable.as("t") .merge( df.as("s"), "t.col1 = s.col1") .whenMatched.updateExpr(Map("t.col2" -> "t.col2 + s.col2")) .whenNotMatched().insertAll() .execute() } def assertTable(col2Val: Int, numRows: Int): Unit = { val res1 = deltaTable.toDF.select("col2").where("col1 = 1").collect() assert(res1.length == numRows) assert(res1(0).getInt(0) == col2Val) } // merge (1,0) into empty table, table should have 1 row (1,0) spark.conf.set("spark.databricks.delta.write.txnVersion", "1") runMerge((1, 0)) assertTable(0, 1) // merge (1,2) into table, table should have 1 row (1,2) spark.conf.set("spark.databricks.delta.write.txnVersion", "2") runMerge((1, 2)) assertTable(2, 1) // retry merge 2, table should have 1 row (1,2) spark.conf.set("spark.databricks.delta.write.txnVersion", "2") runMerge((1, 2)) assertTable(2, 1) // merge (1,3) into table, table should have 1 row (1,5) spark.conf.set("spark.databricks.delta.write.txnVersion", "3") runMerge((1, 3)) assertTable(5, 1) // clean up spark.conf.unset("spark.databricks.delta.write.txnAppId") spark.conf.unset("spark.databricks.delta.write.txnVersion") } } test("idempotent write: idempotent SQL merge") { def withTempDirs(f: (File, File) => Unit): Unit = { withTempDir { file1 => withTempDir { file2 => f(file1, file2) } } } withTempDirs { (tableDir, updateTableDir) => val targetTableName = "myMergeTable" val sourceTableName = "updates" spark.conf.set("spark.databricks.delta.write.txnAppId", "mergeTestSQL") spark.sql(s"CREATE TABLE $targetTableName (col1 INT, col2 INT) USING DELTA LOCATION '" + tableDir.getCanonicalPath + "'") spark.sql(s"CREATE TABLE $sourceTableName (col1 INT, col2 INT) USING DELTA LOCATION '" + updateTableDir.getCanonicalPath + "'") def runMerge(data: (Int, Int), txnVersion: Int): Unit = { val df = Seq(data).toDF("col1", "col2") spark.conf.set("spark.databricks.delta.write.txnVersion", s"$txnVersion") df.write.format("delta").mode("overwrite").save(updateTableDir.getCanonicalPath) spark.conf.set("spark.databricks.delta.write.txnVersion", s"$txnVersion") spark.sql(s""" |MERGE INTO $targetTableName AS t USING $sourceTableName AS s | ON t.col1 = s.col1 | WHEN MATCHED THEN UPDATE SET t.col2 = t.col2 + s.col2 | WHEN NOT MATCHED THEN INSERT (col1, col2) VALUES (col1, col2) |""".stripMargin) } def assertTable(col2Val: Int, numRows: Int): Unit = { val res1 = spark.sql(s"SELECT col2 FROM $targetTableName WHERE col1 = 1").collect() assert(res1.length == numRows) assert(res1(0).getInt(0) == col2Val) } // merge (1,0) into empty table, table should have 1 row (1,0) runMerge((1, 0), 1) assertTable(0, 1) // merge (1,2) into table, table should have 1 row (1,2) runMerge((1, 2), 2) assertTable(2, 1) // retry merge 2, table should have 1 row (1,2) runMerge((1, 2), 2) assertTable( 2, 1) // merge (1,3) into table, table should have 1 row (1,5) runMerge((1, 3), 3) assertTable(5, 1) // clean up spark.conf.unset("spark.databricks.delta.write.txnAppId") spark.conf.unset("spark.databricks.delta.write.txnVersion") } } test("idempotent write: idempotent DeltaTable update") { withTempDir { tableDir => spark.conf.set("spark.databricks.delta.write.txnAppId", "updateTest") io.delta.tables.DeltaTable.createOrReplace(spark) .addColumn("col1", "INT") .addColumn("col2", "INT") .location(tableDir.getCanonicalPath) .execute() val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tableDir.getCanonicalPath) spark.conf.set("spark.databricks.delta.write.txnVersion", "0") Seq((1, 0)).toDF("col1", "col2") .write.format("delta").mode("append").save(tableDir.getCanonicalPath) def runUpdate(data: (Int, Int)): Unit = { deltaTable.update( condition = expr(s"col1 == ${data._1}"), set = Map("col2" -> expr(s"col2 + ${data._2}")) ) } def assertTable(col2Val: Int, numRows: Int): Unit = { val res1 = deltaTable.toDF.select("col2").where("col1 = 1").collect() assert(res1.length == numRows) assert(res1(0).getInt(0) == col2Val) } // run update (1,1), table should have 1 row (1,1) spark.conf.set("spark.databricks.delta.write.txnVersion", "1") runUpdate((1, 1)) assertTable(1, 1) // run update (1,2), table should have 1 row (1,3) spark.conf.set("spark.databricks.delta.write.txnVersion", "2") runUpdate((1, 2)) assertTable(3, 1) // retry update 2, table should have 1 row (1,3) spark.conf.set("spark.databricks.delta.write.txnVersion", "2") runUpdate((1, 2)) assertTable(3, 1) // retry update 1, table should have 1 row (1,3) spark.conf.set("spark.databricks.delta.write.txnVersion", "1") runUpdate((1, 1)) assertTable(3, 1) // run update (1,3) into table, table should have 1 row (1,6) spark.conf.set("spark.databricks.delta.write.txnVersion", "3") runUpdate((1, 3)) assertTable(6, 1) // clean up spark.conf.unset("spark.databricks.delta.write.txnAppId") spark.conf.unset("spark.databricks.delta.write.txnVersion") } } test("idempotent write: idempotent SQL update") { withTempDir { tableDir => val tableName = "myUpdateTable" spark.conf.set("spark.databricks.delta.write.txnAppId", "updateTestSQL") spark.sql(s"CREATE TABLE $tableName (col1 INT, col2 INT) USING DELTA LOCATION '" + tableDir.getCanonicalPath + "'") spark.conf.set("spark.databricks.delta.write.txnVersion", "0") spark.sql(s"INSERT INTO $tableName (col1, col2) VALUES (1, 0)") def runUpdate(data: (Int, Int)): Unit = { spark.sql(s""" |UPDATE $tableName SET | col2 = col2 + ${data._2} WHERE col1 = ${data._1} """.stripMargin) } def assertTable(col2Val: Int, numRows: Int): Unit = { val res1 = spark.sql(s"SELECT col2 FROM $tableName WHERE col1 = 1").collect() assert(res1.length == numRows) assert(res1(0).getInt(0) == col2Val) } // run update (1,1), table should have 1 row (1,1) spark.conf.set("spark.databricks.delta.write.txnVersion", "1") runUpdate((1, 1)) assertTable(1, 1) // run update (1,2), table should have 1 row (1,3) spark.conf.set("spark.databricks.delta.write.txnVersion", "2") runUpdate((1, 2)) assertTable(3, 1) // retry update 2, table should have 1 row (1,3) spark.conf.set("spark.databricks.delta.write.txnVersion", "2") runUpdate((1, 2)) assertTable(3, 1) // retry update 1, table should have 1 row (1,3) spark.conf.set("spark.databricks.delta.write.txnVersion", "1") runUpdate((1, 1)) assertTable(3, 1) // run update (1,3) into table, table should have 1 row (1,6) spark.conf.set("spark.databricks.delta.write.txnVersion", "3") runUpdate((1, 3)) assertTable(6, 1) // clean up spark.conf.unset("spark.databricks.delta.write.txnAppId") spark.conf.unset("spark.databricks.delta.write.txnVersion") } } test("idempotent write: idempotent DeltaTable delete") { withTempDir { tableDir => spark.conf.set("spark.databricks.delta.write.txnAppId", "deleteTest") io.delta.tables.DeltaTable.createOrReplace(spark) .addColumn("col1", "INT") .addColumn("col2", "INT") .location(tableDir.getCanonicalPath) .execute() val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tableDir.getCanonicalPath) spark.conf.set("spark.databricks.delta.write.txnVersion", "0") Seq((1, 0), (2, 0), (3, 0), (4, 0)).toDF("col1", "col2") .write.format("delta").mode("append").save(tableDir.getCanonicalPath) def runDelete(toDelete: Int): Unit = { deltaTable.delete(s"col1 = $toDelete") } def assertTable(numRows: Int): Unit = { val rows = deltaTable.toDF.count() assert(rows == numRows) } // run delete (1), table should have 3 rows (2,0),(3,0),(4,0) spark.conf.set("spark.databricks.delta.write.txnVersion", "1") runDelete(1) assertTable(3) // add (1,0) back to table spark.conf.set("spark.databricks.delta.write.txnVersion", "2") Seq((1, 0)).toDF("col1", "col2") .write.format("delta").mode("append").save(tableDir.getCanonicalPath) assertTable(4) // retry delete 1, table should have 4 rows (2,0),(3,0),(4,0) spark.conf.set("spark.databricks.delta.write.txnVersion", "1") runDelete(1) assertTable(4) // run delete (1), table should have 3 rows (2,0),(3,0),(4,0) spark.conf.set("spark.databricks.delta.write.txnVersion", "3") runDelete(1) assertTable(3) // clean up spark.conf.unset("spark.databricks.delta.write.txnAppId") spark.conf.unset("spark.databricks.delta.write.txnVersion") } } test("idempotent write: idempotent SQL delete") { withTempDir { tableDir => val tableName = "myDeleteTable" spark.conf.set("spark.databricks.delta.write.txnAppId", "deleteTestSQL") spark.sql(s"CREATE TABLE $tableName (col1 INT, col2 INT) USING DELTA LOCATION '" + tableDir.getCanonicalPath + "'") spark.conf.set("spark.databricks.delta.write.txnVersion", "0") spark.sql(s"INSERT INTO $tableName (col1, col2) VALUES (1, 0), (2, 0), (3, 0), (4, 0)") def runDelete(toDelete: Int): Unit = { spark.sql(s"DELETE FROM $tableName WHERE col1 = $toDelete") } def assertTable(numRows: Long): Unit = { val res1 = spark.sql(s"SELECT COUNT(*) FROM $tableName").collect() assert(res1.length == 1) assert(res1(0).getLong(0) == numRows) } // run delete (1), table should have 3 rows (2,0),(3,0),(4,0) spark.conf.set("spark.databricks.delta.write.txnVersion", "1") runDelete(1) assertTable(3) // add (1,0) back to table spark.conf.set("spark.databricks.delta.write.txnVersion", "2") spark.sql(s"INSERT INTO $tableName (col1, col2) VALUES (1, 0)") assertTable(4) // retry delete (1), table should have 4 rows (2,0),(3,0),(4,0) spark.conf.set("spark.databricks.delta.write.txnVersion", "1") runDelete(1) assertTable(4) // run delete (1), table should have 3 rows (2,0),(3,0),(4,0) spark.conf.set("spark.databricks.delta.write.txnVersion", "3") runDelete(1) assertTable(3) // clean up spark.conf.unset("spark.databricks.delta.write.txnAppId") spark.conf.unset("spark.databricks.delta.write.txnVersion") } } test("idempotent write: valid txnVersion") { spark.conf.set("spark.databricks.delta.write.txnAppId", "deleteTestSQL") val e = intercept[IllegalArgumentException] { spark.sessionState.conf.setConfString( "spark.databricks.delta.write.txnVersion", "someVersion") } assert(e.getMessage == "spark.databricks.delta.write.txnVersion should be long, but was someVersion" || e.getMessage.contains("INVALID_CONF_VALUE.TYPE_MISMATCH")) // clean up spark.conf.unset("spark.databricks.delta.write.txnAppId") spark.conf.unset("spark.databricks.delta.write.txnVersion") } Seq("REPLACE", "CREATE OR REPLACE").foreach { command => test(s"Idempotent $command command") { withTempDir { tableDir => val tableName = "myIdempotentReplaceTable" withTable(tableName) { spark.conf.set("spark.databricks.delta.write.txnAppId", "replaceTestSQL") spark.sql(s"CREATE TABLE $tableName(c1 INT, c2 INT, c3 INT)" + s"USING DELTA LOCATION '" + tableDir.getCanonicalPath + "'") def runReplace(data: (Int, Int, Int)): Unit = { spark.sql(s"$command table $tableName USING DELTA " + s"as SELECT ${data._1} as c1, ${data._2} as c2, ${data._3} as c3") } def assertTable(numRows: Int, commitVersion: Int, data: (Int, Int, Int)): Unit = { val count = spark.sql(s"SELECT * FROM $tableName").count() assert(count == numRows) val snapshot = DeltaLog.forTable(spark, tableDir.getCanonicalPath).update() assert(snapshot.version == commitVersion) val tableContent = spark.sql(s"SELECT * FROM $tableName").collect().head assert(tableContent.getInt(0) == data._1) assert(tableContent.getInt(1) == data._2) assert(tableContent.getInt(2) == data._3) } // run replace (1,1,1) with version 1, table should have 1 row (1,1,1). spark.conf.set("spark.databricks.delta.write.txnVersion", "1") runReplace((1, 1, 1)) assertTable(1, 1, (1, 1, 1)) // run replace (2,2,2) with version 2, table should have 1 row (2,2,2) spark.conf.set("spark.databricks.delta.write.txnVersion", "2") runReplace((2, 2, 2)) assertTable(1, 2, (2, 2, 2)) // retry replace (3,3,3) with version 2, table should have 1 row (2,2,2). spark.conf.set("spark.databricks.delta.write.txnVersion", "2") runReplace((3, 3, 3)) assertTable(1, 2, (2, 2, 2)) // run replace (4,4,4) with version 3, table should have 1 row (4,4,4). spark.conf.set("spark.databricks.delta.write.txnVersion", "3") runReplace((4, 4, 4)) assertTable(1, 3, (4, 4, 4)) // run replace (5,5,5) with version 3, table should have 1 row (4,4,4). spark.conf.set("spark.databricks.delta.write.txnVersion", "3") runReplace((5, 5, 5)) assertTable(1, 3, (4, 4, 4)) // clean up spark.conf.unset("spark.databricks.delta.write.txnAppId") spark.conf.unset("spark.databricks.delta.write.txnVersion") } } } } test("idempotent write: auto reset txnVersion") { withTempDir { tableDir => val tableName = "myAutoResetTable" spark.conf.set("spark.databricks.delta.write.txnAppId", "autoReset") spark.sql(s"CREATE TABLE $tableName (col1 INT, col2 INT) USING DELTA LOCATION '" + tableDir.getCanonicalPath + "'") // this write is done with txn version 0 spark.conf.set("spark.databricks.delta.write.txnVersion", "0") spark.sql(s"INSERT INTO $tableName (col1, col2) VALUES (1, 0)") // this write should be skipped as the version is not reset so it will be applied // with the same version spark.sql(s"INSERT INTO $tableName (col1, col2) VALUES (2, 0)") assert(spark.sql(s"SELECT * FROM $tableName").count() == 1) // now enable auto reset spark.conf.set("spark.databricks.delta.write.txnVersion.autoReset.enabled", "true") // this write should be skipped as it is using the same txnVersion as the first write spark.conf.set("spark.databricks.delta.write.txnVersion", "0") spark.sql(s"INSERT INTO $tableName (col1, col2) VALUES (3, 0)") // this should throw an exception as the txn version is automatically reset val e1 = intercept[DeltaIllegalArgumentException] { spark.sql(s"INSERT INTO $tableName (col1, col2) VALUES (4, 0)") } checkError(e1, "DELTA_INVALID_IDEMPOTENT_WRITES_OPTIONS", "42616", Map("reason" -> ( "Both spark.databricks.delta.write.txnAppId and spark.databricks.delta.write.txnVersion " + "must be specified for idempotent Delta writes") )) // this write should succeed as it's using a newer version than the latest spark.conf.set("spark.databricks.delta.write.txnVersion", "10") spark.sql(s"INSERT INTO $tableName (col1, col2) VALUES (2, 0)") // this should throw an exception as the txn version is automatically reset val e2 = intercept[DeltaIllegalArgumentException] { spark.sql(s"INSERT INTO $tableName (col1, col2) VALUES (3, 0)") } checkError(e2, "DELTA_INVALID_IDEMPOTENT_WRITES_OPTIONS", "42616", Map("reason" -> ( "Both spark.databricks.delta.write.txnAppId and spark.databricks.delta.write.txnVersion " + "must be specified for idempotent Delta writes") )) val res = spark.sql(s"SELECT col1 FROM $tableName") .orderBy(asc("col1")) .collect() assert(res.length == 2) assert(res(0).getInt(0) == 1) assert(res(1).getInt(0) == 2) // clean up spark.conf.unset("spark.databricks.delta.write.txnAppId") spark.conf.unset("spark.databricks.delta.write.txnVersion") } } def idempotentWrite( mode: String, appId: String, seq: DataFrame, path: String, name: String, version: Long, expectedCount: Long, commitVersion: Int, isSaveAsTable: Boolean = true): Unit = { val df = seq.write.format("delta") .option(DeltaOptions.TXN_VERSION, version) .option(DeltaOptions.TXN_APP_ID, appId) .mode(mode) if (isSaveAsTable) { df.option("path", path).saveAsTable(name) } else { df.save(path) } val i = spark.read.format("delta").load(path).count() assert(i == expectedCount) val snapshot = DeltaLog.forTable(spark, path).update() assert(snapshot.version == (commitVersion - 1)) } Seq((true, true), (true, false), (false, true), (false, false)) .foreach {case (isSaveAsTable, isLegacy) => val op = if (isSaveAsTable) "saveAsTable" else "save" val version = if (isLegacy) "legacy" else "non-legacy" val appId1 = "myAppId1" val appId2 = "myAppId2" val confs = if (isLegacy) Seq(SQLConf.USE_V1_SOURCE_LIST.key -> "tahoe,delta") else Seq.empty if (!(isSaveAsTable && isLegacy)) { test(s"Idempotent $version Dataframe $op: append") { withSQLConf(confs: _*) { withTempDir { dir => val path = dir.getCanonicalPath val name = "append_table_t1" val mode = "append" sql("DROP TABLE IF EXISTS append_table_t1") val df = Seq((1, 2, 3), (4, 5, 6), (7, 8, 9)).toDF("a", "b", "c") // The first 2 runs must succeed increasing the expected count. idempotentWrite(mode, appId1, df, path, name, 1, 3, 1, isSaveAsTable) idempotentWrite(mode, appId1, df, path, name, 2, 6, 2, isSaveAsTable) // Even if the version is not consecutive, higher versions should commit successfully. idempotentWrite(mode, appId1, df, path, name, 5, 9, 3, isSaveAsTable) // This run should be ignored because it uses an older version. idempotentWrite(mode, appId1, df, path, name, 5, 9, 3, isSaveAsTable) // Use a different app ID, but same version. This should succeed. idempotentWrite(mode, appId2, df, path, name, 5, 12, 4, isSaveAsTable) idempotentWrite(mode, appId2, df, path, name, 5, 12, 4, isSaveAsTable) // Verify that specifying only one of the options -- either appId or version -- fails. val e1 = intercept[Exception] { val stage = df.write.format("delta").option(DeltaOptions.TXN_APP_ID, 1).mode(mode) if (isSaveAsTable) { stage.option("path", path).saveAsTable(name) } else { stage.save(path) } } assert(e1.getMessage.contains("Invalid options for idempotent Dataframe writes")) val e2 = intercept[Exception] { val stage = df.write.format("delta").option(DeltaOptions.TXN_VERSION, 1).mode(mode) if (isSaveAsTable) { stage.option("path", path).saveAsTable(name) } else { stage.save(path) } } assert(e2.getMessage.contains("Invalid options for idempotent Dataframe writes")) } } } } test(s"Idempotent $version Dataframe $op: overwrite") { withSQLConf(confs: _*) { withTempDir { dir => val path = dir.getCanonicalPath val name = "overwrite_table_t1" val mode = "overwrite" sql("DROP TABLE IF EXISTS overwrite_table_t1") val df = Seq((1, 2, 3), (4, 5, 6), (7, 8, 9)).toDF("a", "b", "c") // The first 2 runs must succeed increasing the expected count. idempotentWrite(mode, appId1, df, path, name, 1, 3, 1, isSaveAsTable) idempotentWrite(mode, appId1, df, path, name, 2, 3, 2, isSaveAsTable) // Even if the version is not consecutive, higher versions should commit successfully. idempotentWrite(mode, appId1, df, path, name, 5, 3, 3, isSaveAsTable) // This run should be ignored because it uses an older version. idempotentWrite(mode, appId1, df, path, name, 5, 3, 3, isSaveAsTable) // Use a different app ID, but same version. This should succeed. idempotentWrite(mode, appId2, df, path, name, 5, 3, 4, isSaveAsTable) idempotentWrite(mode, appId2, df, path, name, 5, 3, 4, isSaveAsTable) // Verify that specifying only one of the options -- either appId or version -- fails. val e1 = intercept[Exception] { val stage = df.write.format("delta").option(DeltaOptions.TXN_APP_ID, 1).mode(mode) if (isSaveAsTable) stage.option("path", path).saveAsTable(name) else stage.save(path) } assert(e1.getMessage.contains("Invalid options for idempotent Dataframe writes")) val e2 = intercept[Exception] { val stage = df.write.format("delta").option(DeltaOptions.TXN_VERSION, 1).mode(mode) if (isSaveAsTable) stage.option("path", path).saveAsTable(name) else stage.save(path) } assert(e2.getMessage.contains("Invalid options for idempotent Dataframe writes")) } } } } test("idempotent writes in streaming foreachBatch") { // Function to get a checkpoint location and 2 table locations. def withTempDirs(f: (File, File, File) => Unit): Unit = { withTempDir { file1 => withTempDir { file2 => withTempDir { file3 => f(file1, file2, file3) } } } } // In this test, we are going to run a streaming query in a deterministic way. // This streaming query uses foreachBatch to append data to two tables, and // depending on a boolean flag, the query can fail between the two table writes. // By setting this flag, we will test whether both tables are consistenly updated // when query resumes after failure - no duplicates, no data missing. withTempDirs { (checkpointDir, table1Dir, table2Dir) => @volatile var shouldFail = false /* Function to write a batch's data to 2 tables */ def runBatch(batch: DataFrame, appId: String, batchId: Long): Unit = { // Append to table 1 batch.write.format("delta") .option(DeltaOptions.TXN_VERSION, batchId) .option(DeltaOptions.TXN_APP_ID, appId) .mode("append").save(table1Dir.getCanonicalPath) if (shouldFail) { throw new Exception("Terminating execution") } else { // Append to table 2 batch.write.format("delta") .option(DeltaOptions.TXN_VERSION, batchId) .option(DeltaOptions.TXN_APP_ID, appId) .mode("append").save(table2Dir.getCanonicalPath) } } @volatile var query: StreamingQuery = null // Prepare a streaming query val inputData = MemoryStream[Int] val df = inputData.toDF() val streamWriter = df.writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .foreachBatch { (batch: DataFrame, id: Long) => { runBatch(batch, query.id.toString, id) } } /* Add data and run streaming query, then verify # rows in 2 tables */ def runQuery(dataToAdd: Int, expectedTable1Count: Int, expectedTable2Count: Int): Unit = { inputData.addData(dataToAdd) query = streamWriter.start() try { query.processAllAvailable() } catch { case e: Exception => assert(e.getMessage.contains("Terminating execution")) } finally { query.stop() } val t1Count = spark.read.format("delta").load(table1Dir.getCanonicalPath).count() assert(t1Count == expectedTable1Count) val t2Count = spark.read.format("delta").load(table2Dir.getCanonicalPath).count() assert(t2Count == expectedTable2Count) } // Run the query 3 times. First time without failure, both the output tables are updated. shouldFail = false runQuery(dataToAdd = 0, expectedTable1Count = 1, expectedTable2Count = 1) // Second time with failure. Only one of the tables should be updated. shouldFail = true runQuery(dataToAdd = 1, expectedTable1Count = 2, expectedTable2Count = 1) // Third time without failure. Both the tables should be consistently updated. shouldFail = false runQuery(dataToAdd = 2, expectedTable1Count = 3, expectedTable2Count = 3) } } test("parsing table name and alias using test helper") { import DeltaTestUtils.parseTableAndAlias // Parse table name from path and optional alias. assert(parseTableAndAlias("delta.`store_sales`") === "delta.`store_sales`" -> None) assert(parseTableAndAlias("delta.`store sales`") === "delta.`store sales`" -> None) assert(parseTableAndAlias("delta.`store_sales` s") === "delta.`store_sales`" -> Some("s")) assert(parseTableAndAlias("delta.`store sales` as s") === "delta.`store sales`" -> Some("s")) assert(parseTableAndAlias("delta.`store%sales` AS s") === "delta.`store%sales`" -> Some("s")) // Parse table name and optional alias. assert(parseTableAndAlias("store_sales") === "store_sales" -> None) assert(parseTableAndAlias("store sales") === "store" -> Some("sales")) assert(parseTableAndAlias("store_sales s") === "store_sales" -> Some("s")) assert(parseTableAndAlias("'store sales' as s") === "'store sales'" -> Some("s")) assert(parseTableAndAlias("'store%sales' AS s") === "'store%sales'" -> Some("s")) // Not properly supported: ambiguous without special handling for escaping. assert(parseTableAndAlias("'store sales'") === "'store" -> Some("sales'")) } test("DeltaTableV2.properties() filters fs.* storage properties injected by catalogs") { withTempDir { dir => spark.range(1).write.format("delta").save(dir.getAbsolutePath) val tablePath = new Path(dir.toURI) // Simulate catalog (e.g., Unity Catalog) injecting fs.* credentials and metadata // into CatalogTable.storage.properties at table-load time. val injectedFsProps = Map( "fs.s3a.fake-endpoint" -> "s3.us-west-2.amazonaws.com", "fs.unitycatalog.uri" -> "https://uc.example.com", "fs.unitycatalog.auth.fake-token" -> "dapi_secret_token" ) val otherStorageProps = Map( "nonFsProp" -> "visible_value", "path" -> dir.getAbsolutePath ) val allStorageProps = injectedFsProps ++ otherStorageProps val catalogTable = CatalogTable( identifier = TableIdentifier("test_fs_filter"), tableType = CatalogTableType.EXTERNAL, storage = CatalogStorageFormat( locationUri = Some(dir.toURI), inputFormat = None, outputFormat = None, serde = None, compressed = false, properties = allStorageProps ), schema = new StructType().add("id", "long"), provider = Some("delta") ) val deltaTable = DeltaTableV2(spark, tablePath, Some(catalogTable)) val v2Props = deltaTable.properties() injectedFsProps.keys.foreach { fsKey => assert(!v2Props.containsKey(TableCatalog.OPTION_PREFIX + fsKey), s"DeltaTableV2.properties() should hide '${TableCatalog.OPTION_PREFIX}$fsKey'") } injectedFsProps.keys.foreach { fsKey => assert(!v2Props.containsKey(fsKey), s"DeltaTableV2.properties() should also hide '$fsKey'") } assert(v2Props.get(TableCatalog.OPTION_PREFIX + "nonFsProp") === "visible_value", "Non-fs storage properties should remain visible in DeltaTableV2.properties()") } } } class DeltaNameColumnMappingSuite extends DeltaSuite with DeltaColumnMappingEnableNameMode { import testImplicits._ override protected def runOnlyTests = Seq( "handle partition filters and data filters", "query with predicates should skip partitions", "valid replaceWhere", "batch write: append, overwrite where", "get touched files for update, delete and merge", "isBlindAppend with save and saveAsTable" ) test( "dynamic partition overwrite with conflicting logical vs. physical named partition columns") { // It isn't sufficient to just test with column mapping enabled because the physical names are // generated automatically and thus are unique w.r.t. the logical names. // Instead we need to have: ColA.logicalName = ColB.physicalName, // which means we need to start with columnMappingMode=None, and then upgrade to // columnMappingMode=name and rename our columns withSQLConf(DeltaSQLConf.DYNAMIC_PARTITION_OVERWRITE_ENABLED.key -> "true", DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey-> NoMapping.name) { withTempDir { tempDir => def data: DataFrame = spark.read.format("delta").load(tempDir.toString) Seq(("a", "x", 1), ("b", "y", 2), ("c", "x", 3)).toDF("part1", "part2", "value") .write .format("delta") .partitionBy("part1", "part2") .mode("append") .save(tempDir.getCanonicalPath) val protocol = DeltaLog.forTable(spark, tempDir).snapshot.protocol val (r, w) = if (protocol.supportsReaderFeatures || protocol.supportsWriterFeatures) { (TableFeatureProtocolUtils.TABLE_FEATURES_MIN_READER_VERSION, TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION) } else { (ColumnMappingTableFeature.minReaderVersion, ColumnMappingTableFeature.minWriterVersion) } spark.sql( s""" |ALTER TABLE delta.`${tempDir.getCanonicalPath}` SET TBLPROPERTIES ( | 'delta.minReaderVersion' = '$r', | 'delta.minWriterVersion' = '$w', | 'delta.columnMapping.mode' = 'name' |) |""".stripMargin) spark.sql( s""" |ALTER TABLE delta.`${tempDir.getCanonicalPath}` RENAME COLUMN part1 TO temp |""".stripMargin) spark.sql( s""" |ALTER TABLE delta.`${tempDir.getCanonicalPath}` RENAME COLUMN part2 TO part1 |""".stripMargin) spark.sql( s""" |ALTER TABLE delta.`${tempDir.getCanonicalPath}` RENAME COLUMN temp TO part2 |""".stripMargin) Seq(("a", "x", 4), ("d", "x", 5)).toDF("part2", "part1", "value") .write .format("delta") .partitionBy("part2", "part1") .mode("overwrite") .option(DeltaOptions.PARTITION_OVERWRITE_MODE_OPTION, "dynamic") .save(tempDir.getCanonicalPath) checkDatasetUnorderly(data.select("part2", "part1", "value").as[(String, String, Int)], ("a", "x", 4), ("b", "y", 2), ("c", "x", 3), ("d", "x", 5)) } } } test("replaceWhere dataframe V2 API with less than predicate") { withTempDir { dir => val insertedDF = spark.range(10).toDF() insertedDF.write.format("delta").save(dir.toString) val otherDF = spark.range(start = 0, end = 4).toDF() otherDF.writeTo(s"delta.`${dir.toString}`").overwrite(col("id") < 6) checkAnswer(spark.read.load(dir.toString), insertedDF.filter(col("id") >= 6).union(otherDF)) } } test("replaceWhere SQL - partition column - dynamic filter") { withTempDir { dir => // create partitioned table spark.range(100).withColumn("part", 'id % 10) .write .format("delta") .partitionBy("part") .save(dir.toString) // ans will be used to replace the entire contents of the table val ans = spark.range(10) .withColumn("part", lit(0)) ans.createOrReplaceTempView("replace") sql(s"INSERT INTO delta.`${dir.toString}` REPLACE WHERE part >=0 SELECT * FROM replace") checkAnswer(spark.read.format("delta").load(dir.toString), ans) } } test("replaceWhere SQL - partition column - static filter") { withTable("tbl") { // create partitioned table spark.range(100).withColumn("part", lit(0)) .write .format("delta") .partitionBy("part") .saveAsTable("tbl") val partEq1DF = spark.range(10, 20) .withColumn("part", lit(1)) partEq1DF.write.format("delta").mode("append").saveAsTable("tbl") val replacer = spark.range(10) .withColumn("part", lit(0)) replacer.createOrReplaceTempView("replace") sql(s"INSERT INTO tbl REPLACE WHERE part=0 SELECT * FROM replace") checkAnswer(spark.read.format("delta").table("tbl"), replacer.union(partEq1DF)) } } test("replaceWhere SQL - data column - dynamic") { withTable("tbl") { // write table spark.range(100).withColumn("col", lit(1)) .write .format("delta") .saveAsTable("tbl") val colGt2DF = spark.range(100, 200) .withColumn("col", lit(3)) colGt2DF.write .format("delta") .mode("append") .saveAsTable("tbl") val replacer = spark.range(10) .withColumn("col", lit(1)) replacer.createOrReplaceTempView("replace") sql(s"INSERT INTO tbl REPLACE WHERE col < 2 SELECT * FROM replace") checkAnswer( spark.read.format("delta").table("tbl"), replacer.union(colGt2DF) ) } } test("replaceWhere SQL - data column - static") { withTempDir { dir => // write table spark.range(100).withColumn("col", lit(2)) .write .format("delta") .save(dir.toString) val colEq2DF = spark.range(100, 200) .withColumn("col", lit(1)) colEq2DF.write .format("delta") .mode("append") .save(dir.toString) val replacer = spark.range(10) .withColumn("col", lit(2)) replacer.createOrReplaceTempView("replace") sql(s"INSERT INTO delta.`${dir.toString}` REPLACE WHERE col = 2 SELECT * FROM replace") checkAnswer( spark.read.format("delta").load(dir.toString), replacer.union(colEq2DF) ) } } test("replaceWhere SQL - multiple predicates - static") { withTempDir { dir => // write table spark.range(100).withColumn("col", lit(2)) .write .format("delta") .save(dir.toString) spark.range(100, 200).withColumn("col", lit(5)) .write .format("delta") .mode("append") .save(dir.toString) val colEq2DF = spark.range(100, 200) .withColumn("col", lit(1)) colEq2DF.write .format("delta") .mode("append") .save(dir.toString) val replacer = spark.range(10) .withColumn("col", lit(2)) replacer.createOrReplaceTempView("replace") sql(s"INSERT INTO delta.`${dir.toString}` REPLACE WHERE col = 2 OR col = 5 " + s"SELECT * FROM replace") checkAnswer( spark.read.format("delta").load(dir.toString), replacer.union(colEq2DF) ) } } test("replaceWhere with less than predicate") { withTempDir { dir => val insertedDF = spark.range(10).toDF() insertedDF.write.format("delta").save(dir.toString) val otherDF = spark.range(start = 0, end = 4).toDF() otherDF.write.format("delta").mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "id < 6") .save(dir.toString) checkAnswer(spark.read.load(dir.toString), insertedDF.filter(col("id") >= 6).union(otherDF)) } } test("replaceWhere SQL with less than predicate") { withTempDir { dir => val insertedDF = spark.range(10).toDF() insertedDF.write.format("delta").save(dir.toString) val otherDF = spark.range(start = 0, end = 4).toDF() otherDF.createOrReplaceTempView("replace") sql( s""" |INSERT INTO delta.`${dir.getAbsolutePath}` |REPLACE WHERE id < 6 |SELECT * FROM replace |""".stripMargin) checkAnswer(spark.read.load(dir.toString), insertedDF.filter(col("id") >= 6).union(otherDF)) } } } class DeltaWithCatalogOwnedBatch1Suite extends DeltaSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class DeltaWithCatalogOwnedBatch2Suite extends DeltaSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class DeltaWithCatalogOwnedBatch100Suite extends DeltaSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } @SQLUserDefinedType(udt = classOf[NullUDT]) class NullData extends Serializable class NullUDT extends UserDefinedType[NullData] { override def sqlType: DataType = NullType override def userClass: Class[NullData] = classOf[NullData] override def serialize(obj: NullData): Any = null override def deserialize(datum: Any): NullData = new NullData() } @SQLUserDefinedType(udt = classOf[ComplexUDT]) class ComplexData extends Serializable class ComplexUDT extends UserDefinedType[ComplexData] { override def sqlType: DataType = new MapType( StringType, new ArrayType( new StructType().add("a", IntegerType).add("b", new NullUDT), containsNull = true), valueContainsNull = true) override def userClass: Class[ComplexData] = classOf[ComplexData] override def serialize(obj: ComplexData): Any = null override def deserialize(datum: Any): ComplexData = new ComplexData() } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaTableCreationTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.util.Locale // scalastyle:off import.ordering.noEmptyLine import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer import scala.language.implicitConversions import org.apache.spark.sql.delta.DeltaOperations.ManualUpdate import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.delta.actions.Protocol import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaColumnMappingSelectedTestMixin, DeltaSQLCommandTest} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.commons.io.FileUtils import org.apache.hadoop.fs.Path import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row, SaveMode} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.NoSuchTableException import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType, ExternalCatalogUtils, SessionCatalog} import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.catalyst.util.ResolveDefaultColumnsUtils import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Identifier, Table, TableCatalog} import org.apache.spark.sql.functions.col import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{MetadataBuilder, StructType} import org.apache.spark.util.Utils trait DeltaTableCreationTests extends QueryTest with SharedSparkSession with DeltaColumnMappingTestUtils { import testImplicits._ val format = "delta" override protected def sparkConf: SparkConf = { super.sparkConf // to make compatible with existing empty schema fail tests .set(DeltaSQLConf.DELTA_ALLOW_CREATE_EMPTY_SCHEMA_TABLE.key, "false") } private def createDeltaTableByPath( path: File, df: DataFrame, tableName: String, partitionedBy: Seq[String] = Nil): Unit = { df.write .partitionBy(partitionedBy: _*) .mode(SaveMode.Append) .format(format) .save(path.getCanonicalPath) sql(s""" |CREATE TABLE delta_test |USING delta |LOCATION '${path.getCanonicalPath}' """.stripMargin) } private implicit def toTableIdentifier(tableName: String): TableIdentifier = { spark.sessionState.sqlParser.parseTableIdentifier(tableName) } protected def getTablePath(tableName: String): String = { new Path(spark.sessionState.catalog.getTableMetadata(tableName).location).toString } protected def getDefaultTablePath(tableName: String): String = { new Path(spark.sessionState.catalog.defaultTablePath(tableName)).toString } protected def getPartitioningColumns(tableName: String): Seq[String] = { spark.sessionState.catalog.getTableMetadata(tableName).partitionColumnNames } protected def getSchema(tableName: String): StructType = { spark.sessionState.catalog.getTableMetadata(tableName).schema } protected def getTableProperties(tableName: String): Map[String, String] = { spark.sessionState.catalog.getTableMetadata(tableName).properties } private def getDeltaLog(table: CatalogTable): DeltaLog = { getDeltaLog(new Path(table.storage.locationUri.get)) } private def getDeltaLog(tableName: String): DeltaLog = { getDeltaLog(spark.sessionState.catalog.getTableMetadata(tableName)) } protected def getDeltaLog(path: Path): DeltaLog = { DeltaLog.forTable(spark, path) } protected def verifyTableInCatalog(catalog: SessionCatalog, table: String): Unit = { val externalTable = catalog.externalCatalog.getTable("default", table) assertEqual(externalTable.schema, new StructType()) assert(externalTable.partitionColumnNames.isEmpty) } protected def checkResult( result: DataFrame, expected: Seq[Any], columns: Seq[String]): Unit = { checkAnswer( result.select(columns.head, columns.tail: _*), Seq(Row(expected: _*)) ) } Seq("partitioned" -> Seq("v2"), "non-partitioned" -> Nil).foreach { case (isPartitioned, cols) => SaveMode.values().foreach { saveMode => test(s"saveAsTable to a new table (managed) - $isPartitioned, saveMode: $saveMode") { val tbl = "delta_test" withTable(tbl) { Seq(1L -> "a").toDF("v1", "v2") .write .partitionBy(cols: _*) .mode(saveMode) .format(format) .saveAsTable(tbl) checkDatasetUnorderly(spark.table(tbl).as[(Long, String)], 1L -> "a") assert(getTablePath(tbl) === getDefaultTablePath(tbl), "Table path is wrong") assert(getPartitioningColumns(tbl) === cols, "Partitioning columns don't match") } } test(s"saveAsTable to a new table (managed) - $isPartitioned," + s" saveMode: $saveMode (empty df)") { val tbl = "delta_test" withTable(tbl) { Seq(1L -> "a").toDF("v1", "v2").where("false") .write .partitionBy(cols: _*) .mode(saveMode) .format(format) .saveAsTable(tbl) checkDatasetUnorderly(spark.table(tbl).as[(Long, String)]) assert(getTablePath(tbl) === getDefaultTablePath(tbl), "Table path is wrong") assert(getPartitioningColumns(tbl) === cols, "Partitioning columns don't match") } } } SaveMode.values().foreach { saveMode => test(s"saveAsTable to a new table (external) - $isPartitioned, saveMode: $saveMode") { withTempDir { dir => val tbl = "delta_test" withTable(tbl) { Seq(1L -> "a").toDF("v1", "v2") .write .partitionBy(cols: _*) .mode(saveMode) .format(format) .option("path", dir.getCanonicalPath) .saveAsTable(tbl) checkDatasetUnorderly(spark.table(tbl).as[(Long, String)], 1L -> "a") assert(getTablePath(tbl) === new Path(dir.toURI).toString.stripSuffix("/"), "Table path is wrong") assert(getPartitioningColumns(tbl) === cols, "Partitioning columns don't match") } } } test(s"saveAsTable to a new table (external) - $isPartitioned," + s" saveMode: $saveMode (empty df)") { withTempDir { dir => val tbl = "delta_test" withTable(tbl) { Seq(1L -> "a").toDF("v1", "v2").where("false") .write .partitionBy(cols: _*) .mode(saveMode) .format(format) .option("path", dir.getCanonicalPath) .saveAsTable(tbl) checkDatasetUnorderly(spark.table(tbl).as[(Long, String)]) assert(getTablePath(tbl) === new Path(dir.toURI).toString.stripSuffix("/"), "Table path is wrong") assert(getPartitioningColumns(tbl) === cols, "Partitioning columns don't match") } } } } test(s"saveAsTable (append) to an existing table - $isPartitioned") { withTempDir { dir => val tbl = "delta_test" withTable(tbl) { createDeltaTableByPath(dir, Seq(1L -> "a").toDF("v1", "v2"), tbl, cols) Seq(2L -> "b").toDF("v1", "v2") .write .partitionBy(cols: _*) .mode(SaveMode.Append) .format(format) .saveAsTable(tbl) checkDatasetUnorderly(spark.table(tbl).as[(Long, String)], 1L -> "a", 2L -> "b") } } } test(s"saveAsTable (overwrite) to an existing table - $isPartitioned") { withTempDir { dir => val tbl = "delta_test" withTable(tbl) { createDeltaTableByPath(dir, Seq(1L -> "a").toDF("v1", "v2"), tbl, cols) Seq(2L -> "b").toDF("v1", "v2") .write .partitionBy(cols: _*) .mode(SaveMode.Overwrite) .format(format) .saveAsTable(tbl) checkDatasetUnorderly(spark.table(tbl).as[(Long, String)], 2L -> "b") } } } test(s"saveAsTable (ignore) to an existing table - $isPartitioned") { withTempDir { dir => val tbl = "delta_test" withTable(tbl) { createDeltaTableByPath(dir, Seq(1L -> "a").toDF("v1", "v2"), tbl, cols) Seq(2L -> "b").toDF("v1", "v2") .write .partitionBy(cols: _*) .mode(SaveMode.Ignore) .format(format) .saveAsTable(tbl) checkDatasetUnorderly(spark.table(tbl).as[(Long, String)], 1L -> "a") } } } test(s"saveAsTable (error if exists) to an existing table - $isPartitioned") { withTempDir { dir => val tbl = "delta_test" withTable(tbl) { createDeltaTableByPath(dir, Seq(1L -> "a").toDF("v1", "v2"), tbl, cols) val e = intercept[AnalysisException] { Seq(2L -> "b").toDF("v1", "v2") .write .partitionBy(cols: _*) .mode(SaveMode.ErrorIfExists) .format(format) .saveAsTable(tbl) } assert(e.getMessage.contains(tbl)) assert(e.getMessage.contains("already exists")) checkDatasetUnorderly(spark.table(tbl).as[(Long, String)], 1L -> "a") } } } } test("saveAsTable (append) + insert to a table created without a schema") { withTempDir { dir => withTable("delta_test") { Seq(1L -> "a").toDF("v1", "v2") .write .mode(SaveMode.Append) .partitionBy("v2") .format(format) .option("path", dir.getCanonicalPath) .saveAsTable("delta_test") // Out of order Seq("b" -> 2L).toDF("v2", "v1") .write .partitionBy("v2") .mode(SaveMode.Append) .format(format) .saveAsTable("delta_test") Seq(3L -> "c").toDF("v1", "v2") .write .format(format) .insertInto("delta_test") checkDatasetUnorderly( spark.table("delta_test").as[(Long, String)], 1L -> "a", 2L -> "b", 3L -> "c") } } } test("saveAsTable to a table created with an invalid partitioning column") { withTempDir { dir => withTable("delta_test") { Seq(1L -> "a").toDF("v1", "v2") .write .mode(SaveMode.Append) .partitionBy("v2") .format(format) .option("path", dir.getCanonicalPath) .saveAsTable("delta_test") checkDatasetUnorderly(spark.table("delta_test").as[(Long, String)], 1L -> "a") var ex = intercept[Exception] { Seq("b" -> 2L).toDF("v2", "v1") .write .partitionBy("v1") .mode(SaveMode.Append) .format(format) .saveAsTable("delta_test") }.getMessage assert(ex.contains("not match")) assert(ex.contains("partition")) checkDatasetUnorderly(spark.table("delta_test").as[(Long, String)], 1L -> "a") ex = intercept[Exception] { Seq("b" -> 2L).toDF("v3", "v1") .write .partitionBy("v1") .mode(SaveMode.Append) .format(format) .saveAsTable("delta_test") }.getMessage assert(ex.contains("not match")) assert(ex.contains("partition")) checkDatasetUnorderly(spark.table("delta_test").as[(Long, String)], 1L -> "a") Seq("b" -> 2L).toDF("v1", "v3") .write .partitionBy("v1") .mode(SaveMode.Ignore) .format(format) .saveAsTable("delta_test") checkDatasetUnorderly(spark.table("delta_test").as[(Long, String)], 1L -> "a") ex = intercept[AnalysisException] { Seq("b" -> 2L).toDF("v1", "v3") .write .partitionBy("v1") .mode(SaveMode.ErrorIfExists) .format(format) .saveAsTable("delta_test") }.getMessage assert(ex.contains("delta_test")) assert(ex.contains("already exists")) checkDatasetUnorderly(spark.table("delta_test").as[(Long, String)], 1L -> "a") } } } testQuietly("create delta table with spaces in column names") { val tableName = "delta_test" val tableLoc = new File(spark.sessionState.catalog.defaultTablePath(TableIdentifier(tableName))) Utils.deleteRecursively(tableLoc) def createTableUsingDF: Unit = { Seq(1, 2, 3).toDF("a column name with spaces") .write .format(format) .mode(SaveMode.Overwrite) .saveAsTable(tableName) } def createTableUsingSQL: DataFrame = { sql(s"CREATE TABLE $tableName(`a column name with spaces` LONG, b String) USING delta") } withTable(tableName) { if (!columnMappingEnabled) { val ex = intercept[AnalysisException] { createTableUsingDF } assert( ex.getMessage.contains("[INVALID_COLUMN_NAME_AS_PATH]") || ex.getMessage.contains("invalid character(s)") ) assert(!tableLoc.exists()) } else { // column mapping modes support creating table with arbitrary col names createTableUsingDF assert(tableLoc.exists()) } } withTable(tableName) { if (!columnMappingEnabled) { val ex2 = intercept[AnalysisException] { createTableUsingSQL } assert( ex2.getMessage.contains("[INVALID_COLUMN_NAME_AS_PATH]") || ex2.getMessage.contains("invalid character(s)") ) assert(!tableLoc.exists()) } else { // column mapping modes support creating table with arbitrary col names createTableUsingSQL assert(tableLoc.exists()) } } } testQuietly("cannot create delta table when using buckets") { withTable("bucketed_table") { val e = intercept[AnalysisException] { Seq(1L -> "a").toDF("i", "j").write .format(format) .partitionBy("i") .bucketBy(numBuckets = 8, "j") .saveAsTable("bucketed_table") } assert(e.getMessage.toLowerCase(Locale.ROOT).contains( "is not supported for delta tables")) } } test("save without a path") { val e = intercept[IllegalArgumentException] { Seq(1L -> "a").toDF("i", "j").write .format(format) .partitionBy("i") .save() } assert(e.getMessage.toLowerCase(Locale.ROOT).contains("'path' is not specified")) } test("save with an unknown partition column") { withTempDir { dir => val path = dir.getCanonicalPath val e = intercept[AnalysisException] { Seq(1L -> "a").toDF("i", "j").write .format(format) .partitionBy("unknownColumn") .save(path) } assert(e.getMessage.contains("unknownColumn")) } } test("create a table with special column names") { withTable("t") { Seq(1 -> "a").toDF("x.x", "y.y").write.format(format).saveAsTable("t") Seq(2 -> "b").toDF("x.x", "y.y").write.format(format).mode("append").saveAsTable("t") checkAnswer(spark.table("t"), Row(1, "a") :: Row(2, "b") :: Nil) } } testQuietly("saveAsTable (overwrite) to a non-partitioned table created with different paths") { withTempDir { dir1 => withTempDir { dir2 => withTable("delta_test") { Seq(1L -> "a").toDF("v1", "v2") .write .mode(SaveMode.Append) .format(format) .option("path", dir1.getCanonicalPath) .saveAsTable("delta_test") val ex = intercept[AnalysisException] { Seq((3L, "c")).toDF("v1", "v2") .write .mode(SaveMode.Overwrite) .format(format) .option("path", dir2.getCanonicalPath) .saveAsTable("delta_test") }.getMessage assert(ex.contains("The location of the existing table")) assert(ex.contains("`default`.`delta_test`")) checkAnswer( spark.table("delta_test"), Row(1L, "a") :: Nil) } } } } test("saveAsTable (append) to a non-partitioned table created without path") { withTempDir { dir => withTable("delta_test") { Seq(1L -> "a").toDF("v1", "v2") .write .mode(SaveMode.Overwrite) .format(format) .option("path", dir.getCanonicalPath) .saveAsTable("delta_test") Seq((3L, "c")).toDF("v1", "v2") .write .mode(SaveMode.Append) .format(format) .saveAsTable("delta_test") checkAnswer( spark.table("delta_test"), Row(1L, "a") :: Row(3L, "c") :: Nil) } } } test("saveAsTable (append) to a non-partitioned table created with identical paths") { withTempDir { dir => withTable("delta_test") { Seq(1L -> "a").toDF("v1", "v2") .write .mode(SaveMode.Overwrite) .format(format) .option("path", dir.getCanonicalPath) .saveAsTable("delta_test") Seq((3L, "c")).toDF("v1", "v2") .write .mode(SaveMode.Append) .format(format) .option("path", dir.getCanonicalPath) .saveAsTable("delta_test") checkAnswer( spark.table("delta_test"), Row(1L, "a") :: Row(3L, "c") :: Nil) } } } test("overwrite mode saveAsTable without path shouldn't create managed table") { withTempDir { dir => withTable("delta_test") { sql( s"""CREATE TABLE delta_test |USING delta |LOCATION '${dir.getAbsolutePath}' |AS SELECT 1 as a """.stripMargin) val deltaLog = DeltaLog.forTable(spark, dir) assert(deltaLog.snapshot.version === 0, "CTAS should be a single commit") checkAnswer(spark.table("delta_test"), Row(1) :: Nil) Seq((2, "key")).toDF("a", "b") .write .mode(SaveMode.Overwrite) .option(DeltaOptions.OVERWRITE_SCHEMA_OPTION, "true") .format(format) .saveAsTable("delta_test") assert(deltaLog.snapshot.version === 1, "Overwrite mode shouldn't create new managed table") checkAnswer(spark.table("delta_test"), Row(2, "key") :: Nil) } } } testQuietly("reject table creation with column names that only differ by case") { withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { withTempDir { dir => withTable("delta_test") { intercept[AnalysisException] { sql( s"""CREATE TABLE delta_test |USING delta |LOCATION '${dir.getAbsolutePath}' |AS SELECT 1 as a, 2 as A """.stripMargin) } intercept[AnalysisException] { sql( s"""CREATE TABLE delta_test( | a string, | A string |) |USING delta |LOCATION '${dir.getAbsolutePath}' """.stripMargin) } intercept[ParseException] { sql( s"""CREATE TABLE delta_test( | a string, | b string |) |partitioned by (a, a) |USING delta |LOCATION '${dir.getAbsolutePath}' """.stripMargin) } } } } } testQuietly("saveAsTable into a view throws exception around view definition") { withTempDir { dir => val viewName = "delta_test" withView(viewName) { Seq((1, "key")).toDF("a", "b").write.format(format).save(dir.getCanonicalPath) sql(s"create view $viewName as select * from delta.`${dir.getCanonicalPath}`") val e = intercept[AnalysisException] { Seq((2, "key")).toDF("a", "b").write.format(format).mode("append").saveAsTable(viewName) } assert(e.getMessage.contains("a view")) } } } testQuietly("saveAsTable into a parquet table throws exception around format") { withTempPath { dir => val tabName = "delta_test" withTable(tabName) { Seq((1, "key")).toDF("a", "b").write.format("parquet") .option("path", dir.getCanonicalPath).saveAsTable(tabName) intercept[AnalysisException] { Seq((2, "key")).toDF("a", "b").write.format("delta").mode("append").saveAsTable(tabName) } } } } test("create table with schema and path") { withTempDir { dir => withTable("delta_test") { sql( s""" |CREATE TABLE delta_test(a LONG, b String) |USING delta |OPTIONS('path'='${dir.getCanonicalPath}')""".stripMargin) sql("INSERT INTO delta_test SELECT 1, 'a'") checkDatasetUnorderly( sql("SELECT * FROM delta_test").as[(Long, String)], 1L -> "a") } } } protected def createTableWithEmptySchemaQuery( tableName: String, provider: String = "delta", location: Option[String] = None): String = { var query = s"CREATE TABLE $tableName USING $provider" if (location.nonEmpty) { query = s"$query LOCATION '${location.get}'" } query } testQuietly("failed to create a table and then able to recreate it") { withTable("delta_test") { val createEmptySchemaQuery = createTableWithEmptySchemaQuery("delta_test") val e = intercept[AnalysisException] { sql(createEmptySchemaQuery) }.getMessage assert(e.contains("but the schema is not specified")) sql("CREATE TABLE delta_test(a LONG, b String) USING delta") sql("INSERT INTO delta_test SELECT 1, 'a'") checkDatasetUnorderly( sql("SELECT * FROM delta_test").as[(Long, String)], 1L -> "a") } } test("create external table without schema") { withTempDir { dir => withTable("delta_test", "delta_test1") { Seq(1L -> "a").toDF() .selectExpr("_1 as v1", "_2 as v2") .write .mode("append") .partitionBy("v2") .format("delta") .save(dir.getCanonicalPath) sql(s""" |CREATE TABLE delta_test |USING delta |OPTIONS('path'='${dir.getCanonicalPath}') """.stripMargin) spark.catalog.createTable("delta_test1", dir.getCanonicalPath, "delta") checkDatasetUnorderly( sql("SELECT * FROM delta_test").as[(Long, String)], 1L -> "a") checkDatasetUnorderly( sql("SELECT * FROM delta_test1").as[(Long, String)], 1L -> "a") } } } testQuietly("create managed table without schema") { withTable("delta_test") { val createEmptySchemaQuery = createTableWithEmptySchemaQuery("delta_test") val e = intercept[AnalysisException] { sql(createEmptySchemaQuery) }.getMessage assert(e.contains("but the schema is not specified")) } } testQuietly("reject creating a delta table pointing to non-delta files") { withTempPath { dir => withTable("delta_test") { val path = dir.getCanonicalPath Seq(1L -> "a").toDF("col1", "col2").write.parquet(path) val e = intercept[AnalysisException] { sql( s""" |CREATE TABLE delta_test (col1 int, col2 string) |USING delta |LOCATION '$path' """.stripMargin) }.getMessage var catalogPrefix = "" assert(e.contains( s"Cannot create table ('$catalogPrefix`default`.`delta_test`'). The associated location")) } } } testQuietly("create external table without schema but using non-delta files") { withTempDir { dir => withTable("delta_test") { Seq(1L -> "a").toDF().selectExpr("_1 as v1", "_2 as v2").write .mode("append").partitionBy("v2").format("parquet").save(dir.getCanonicalPath) val createEmptySchemaQuery = createTableWithEmptySchemaQuery( "delta_test", location = Some(dir.getCanonicalPath)) val e = intercept[AnalysisException] { sql(createEmptySchemaQuery) }.getMessage assert(e.contains("but there is no transaction log")) } } } testQuietly("create external table without schema and input files") { withTempDir { dir => withTable("delta_test") { val createEmptySchemaQuery = createTableWithEmptySchemaQuery( "delta_test", location = Some(dir.getCanonicalPath)) val e = intercept[AnalysisException] { sql(createEmptySchemaQuery) }.getMessage assert(e.contains("but the schema is not specified") && e.contains("input path is empty")) } } } test("create and drop delta table - external") { val catalog = spark.sessionState.catalog withTempDir { tempDir => withTable("delta_test") { sql("CREATE TABLE delta_test(a LONG, b String) USING delta " + s"OPTIONS (path='${tempDir.getCanonicalPath}')") val table = catalog.getTableMetadata(TableIdentifier("delta_test")) assert(table.tableType == CatalogTableType.EXTERNAL) assert(table.provider.contains("delta")) // Query the data and the metadata directly via the DeltaLog val deltaLog = getDeltaLog(table) assertEqual( deltaLog.snapshot.schema, new StructType().add("a", "long").add("b", "string")) assertEqual( deltaLog.snapshot.metadata.partitionSchema, new StructType()) assertEqual(deltaLog.snapshot.schema, getSchema("delta_test")) assert(getPartitioningColumns("delta_test").isEmpty) // External catalog does not contain the schema and partition column names. verifyTableInCatalog(catalog, "delta_test") sql("INSERT INTO delta_test SELECT 1, 'a'") checkDatasetUnorderly( sql("SELECT * FROM delta_test").as[(Long, String)], 1L -> "a") sql("DROP TABLE delta_test") intercept[NoSuchTableException](catalog.getTableMetadata(TableIdentifier("delta_test"))) // Verify that the underlying location is not deleted for an external table checkAnswer(spark.read.format("delta") .load(new Path(tempDir.getCanonicalPath).toString), Seq(Row(1L, "a"))) } } } test("create and drop delta table - managed") { val catalog = spark.sessionState.catalog withTable("delta_test") { sql("CREATE TABLE delta_test(a LONG, b String) USING delta") val table = catalog.getTableMetadata(TableIdentifier("delta_test")) assert(table.tableType == CatalogTableType.MANAGED) assert(table.provider.contains("delta")) // Query the data and the metadata directly via the DeltaLog val deltaLog = getDeltaLog(table) assertEqual( deltaLog.snapshot.schema, new StructType().add("a", "long").add("b", "string")) assertEqual( deltaLog.snapshot.metadata.partitionSchema, new StructType()) assertEqual(deltaLog.snapshot.schema, getSchema("delta_test")) assert(getPartitioningColumns("delta_test").isEmpty) assertEqual(getSchema("delta_test"), new StructType().add("a", "long").add("b", "string")) // External catalog does not contain the schema and partition column names. verifyTableInCatalog(catalog, "delta_test") sql("INSERT INTO delta_test SELECT 1, 'a'") checkDatasetUnorderly( sql("SELECT * FROM delta_test").as[(Long, String)], 1L -> "a") sql("DROP TABLE delta_test") intercept[NoSuchTableException](catalog.getTableMetadata(TableIdentifier("delta_test"))) // Verify that the underlying location is deleted for a managed table assert(!new File(table.location).exists()) } } test("create table using - with partitioned by") { val catalog = spark.sessionState.catalog withTable("delta_test") { sql("CREATE TABLE delta_test(a LONG, b String) USING delta PARTITIONED BY (a)") val table = catalog.getTableMetadata(TableIdentifier("delta_test")) assert(table.tableType == CatalogTableType.MANAGED) assert(table.provider.contains("delta")) // Query the data and the metadata directly via the DeltaLog val deltaLog = getDeltaLog(table) assertEqual( deltaLog.snapshot.schema, new StructType().add("a", "long").add("b", "string")) assertEqual( deltaLog.snapshot.metadata.partitionSchema, new StructType().add("a", "long")) assertEqual(deltaLog.snapshot.schema, getSchema("delta_test")) assert(getPartitioningColumns("delta_test") == Seq("a")) assertEqual(getSchema("delta_test"), new StructType().add("a", "long").add("b", "string")) // External catalog does not contain the schema and partition column names. verifyTableInCatalog(catalog, "delta_test") sql("INSERT INTO delta_test SELECT 1, 'a'") assertPartitionWithValueExists("a", "1", deltaLog) checkDatasetUnorderly( sql("SELECT * FROM delta_test").as[(Long, String)], 1L -> "a") } } test("CTAS a managed table with the existing empty directory") { val tableLoc = new File(spark.sessionState.catalog.defaultTablePath(TableIdentifier("tab1"))) try { tableLoc.mkdir() withTable("tab1") { sql("CREATE TABLE tab1 USING delta AS SELECT 2, 'b'") checkAnswer(spark.table("tab1"), Row(2, "b")) } } finally { waitForTasksToFinish() Utils.deleteRecursively(tableLoc) } } test("create a managed table with the existing empty directory") { val tableLoc = new File(spark.sessionState.catalog.defaultTablePath(TableIdentifier("tab1"))) try { tableLoc.mkdir() withTable("tab1") { sql("CREATE TABLE tab1 (col1 int, col2 string) USING delta") sql("INSERT INTO tab1 VALUES (2, 'B')") checkAnswer(spark.table("tab1"), Row(2, "B")) } } finally { waitForTasksToFinish() Utils.deleteRecursively(tableLoc) } } testQuietly( "create a managed table with the existing non-empty directory") { withTable("tab1") { val tableLoc = new File(spark.sessionState.catalog.defaultTablePath(TableIdentifier("tab1"))) try { // create an empty hidden file tableLoc.mkdir() val hiddenGarbageFile = new File(tableLoc.getCanonicalPath, ".garbage") hiddenGarbageFile.createNewFile() var ex = intercept[AnalysisException] { sql("CREATE TABLE tab1 USING delta AS SELECT 2, 'b'") }.getMessage assert(ex.contains("Cannot create table")) ex = intercept[AnalysisException] { sql("CREATE TABLE tab1 (col1 int, col2 string) USING delta") }.getMessage assert(ex.contains("Cannot create table")) } finally { waitForTasksToFinish() Utils.deleteRecursively(tableLoc) } } } test("create table with table properties") { withTable("delta_test") { sql(s""" |CREATE TABLE delta_test(a LONG, b String) |USING delta |TBLPROPERTIES( | 'delta.logRetentionDuration' = '2 weeks', | 'delta.checkpointInterval' = '20', | 'key' = 'value' |) """.stripMargin) val deltaLog = getDeltaLog("delta_test") val snapshot = deltaLog.update() assertEqual(snapshot.metadata.configuration, Map( "delta.logRetentionDuration" -> "2 weeks", "delta.checkpointInterval" -> "20", "key" -> "value")) assert(deltaLog.deltaRetentionMillis(snapshot.metadata) == 2 * 7 * 24 * 60 * 60 * 1000) assert(deltaLog.checkpointInterval(snapshot.metadata) == 20) } } test("create table with table properties - case insensitivity") { withTable("delta_test") { sql(s""" |CREATE TABLE delta_test(a LONG, b String) |USING delta |TBLPROPERTIES( | 'dEltA.lOgrEteNtiOndURaTion' = '2 weeks', | 'DelTa.ChEckPoiNtinTervAl' = '20' |) """.stripMargin) val deltaLog = getDeltaLog("delta_test") val snapshot = deltaLog.update() assertEqual(snapshot.metadata.configuration, Map("delta.logRetentionDuration" -> "2 weeks", "delta.checkpointInterval" -> "20")) assert(deltaLog.deltaRetentionMillis(snapshot.metadata) == 2 * 7 * 24 * 60 * 60 * 1000) assert(deltaLog.checkpointInterval(snapshot.metadata) == 20) } } test( "create table with table properties - case insensitivity with existing configuration") { withTempDir { tempDir => withTable("delta_test") { val path = tempDir.getCanonicalPath val deltaLog = getDeltaLog(new Path(path)) val txn = deltaLog.startTransaction() txn.commit(Seq(Metadata( schemaString = new StructType().add("a", "long").add("b", "string").json, configuration = Map( "delta.logRetentionDuration" -> "2 weeks", "delta.checkpointInterval" -> "20", "key" -> "value"))), ManualUpdate) sql(s""" |CREATE TABLE delta_test(a LONG, b String) |USING delta LOCATION '$path' |TBLPROPERTIES( | 'dEltA.lOgrEteNtiOndURaTion' = '2 weeks', | 'DelTa.ChEckPoiNtinTervAl' = '20', | 'key' = "value" |) """.stripMargin) val snapshot = deltaLog.update() assertEqual(snapshot.metadata.configuration, Map( "delta.logRetentionDuration" -> "2 weeks", "delta.checkpointInterval" -> "20", "key" -> "value")) assert(deltaLog.deltaRetentionMillis(snapshot.metadata) == 2 * 7 * 24 * 60 * 60 * 1000) assert(deltaLog.checkpointInterval(snapshot.metadata) == 20) } } } test("create table with table properties - delta.randomizeFilePrefixes") { withTable("delta_test") { sql(s""" |CREATE TABLE delta_test(a LONG, b String) |USING delta |TBLPROPERTIES( | 'delta.randomizeFilePrefixes' = 'true', | 'delta.randomPrefixLength' = '5' |) """.stripMargin) val deltaLog = getDeltaLog("delta_test") val snapshot = deltaLog.update() // Verify the properties are set correctly assertEqual(snapshot.metadata.configuration, Map( "delta.randomizeFilePrefixes" -> "true", "delta.randomPrefixLength" -> "5" )) // Insert some data to create files sql("INSERT INTO delta_test VALUES (1, 'test1'), (2, 'test2'), (3, 'test3')") val updatedSnapshot = deltaLog.update() val allFiles = updatedSnapshot.allFiles.collect() // Verify that files exist and have random prefixes assert(allFiles.nonEmpty, "Table should have data files") // Check that file paths contain 5-character random prefix pattern val prefixLength = DeltaConfigs.RANDOM_PREFIX_LENGTH.fromMetaData(updatedSnapshot.metadata) assert(prefixLength == 5, s"Expected prefix length of 5, but got $prefixLength") val pattern = s"[A-Za-z0-9]{$prefixLength}/.*part-.*parquet" allFiles.foreach { file => assert(file.path.matches(pattern), s"File path '${file.path}' does not match expected random prefix pattern '$pattern'") } } } test("create partitioned table with table properties - delta.randomizeFilePrefixes") { withTable("delta_test") { sql(s""" |CREATE TABLE delta_test(id LONG, part String, value INT) |USING delta |PARTITIONED BY (part) |TBLPROPERTIES( | 'delta.randomizeFilePrefixes' = 'true', | 'delta.randomPrefixLength' = '4' |) """.stripMargin) val deltaLog = getDeltaLog("delta_test") val snapshot = deltaLog.update() // Verify the properties are set correctly assertEqual(snapshot.metadata.configuration, Map( "delta.randomizeFilePrefixes" -> "true", "delta.randomPrefixLength" -> "4" )) // Verify the configuration is properly parsed assert(DeltaConfigs.RANDOMIZE_FILE_PREFIXES.fromMetaData(snapshot.metadata), "randomizeFilePrefixes should be enabled") assert(DeltaConfigs.RANDOM_PREFIX_LENGTH.fromMetaData(snapshot.metadata) == 4, "randomPrefixLength should be 4") // Verify table is partitioned correctly assert(snapshot.metadata.partitionColumns == Seq("part"), "Table should be partitioned by 'part' column") // Insert data to create files with random prefixes across multiple partitions sql("""INSERT INTO delta_test VALUES |(1, 'A', 100), (2, 'B', 200), (3, 'A', 300), (4, 'C', 400)""".stripMargin) val updatedSnapshot = deltaLog.update() val allFiles = updatedSnapshot.allFiles.collect() // Verify that files exist and have random prefixes assert(allFiles.nonEmpty, "Partitioned table should have data files") // Check that file paths contain 4-character random prefix pattern val prefixLength = DeltaConfigs.RANDOM_PREFIX_LENGTH.fromMetaData(updatedSnapshot.metadata) assert(prefixLength == 4, s"Expected prefix length of 4, but got $prefixLength") // For partitioned tables, files still use random prefix pattern (same as non-partitioned) // Partition information is stored separately in metadata val pattern = s"[A-Za-z0-9]{$prefixLength}/.*part-.*parquet" allFiles.foreach { file => assert(file.path.matches(pattern), s"Partitioned file path '${file.path}' does not match expected random prefix pattern " + s"'$pattern'") } // Verify we have files for multiple partitions (by checking partition values in metadata) val partitionValues = allFiles.map(_.partitionValues("part")).distinct assert(partitionValues.length >= 2, s"Expected files in multiple partitions, but only found partitions: " + s"${partitionValues.mkString(", ")}") // Verify we have the expected partition values val expectedPartitions = Set("A", "B", "C") assert(partitionValues.toSet.subsetOf(expectedPartitions), s"Found unexpected partition values: ${partitionValues.toSet}") } } test("schema mismatch between DDL and table location should throw an error") { withTempDir { tempDir => withTable("delta_test") { val deltaLog = getDeltaLog(new Path(tempDir.getCanonicalPath)) val txn = deltaLog.startTransaction() txn.commit( Seq(Metadata(schemaString = new StructType().add("a", "long").add("b", "long").json)), DeltaOperations.ManualUpdate) val ex = intercept[AnalysisException] { sql("CREATE TABLE delta_test(a LONG, b String)" + s" USING delta OPTIONS (path '${tempDir.getCanonicalPath}')") } assert(ex.getMessage.contains("The specified schema does not match the existing schema")) assert(ex.getMessage.contains("Specified type for b is different")) val ex1 = intercept[AnalysisException] { sql("CREATE TABLE delta_test(a LONG)" + s" USING delta OPTIONS (path '${tempDir.getCanonicalPath}')") } assert(ex1.getMessage.contains("The specified schema does not match the existing schema")) assert(ex1.getMessage.contains("Specified schema is missing field")) val ex2 = intercept[AnalysisException] { sql("CREATE TABLE delta_test(a LONG, b String, c INT, d LONG)" + s" USING delta OPTIONS (path '${tempDir.getCanonicalPath}')") } assert(ex2.getMessage.contains("The specified schema does not match the existing schema")) assert(ex2.getMessage.contains("Specified schema has additional field")) assert(ex2.getMessage.contains("Specified type for b is different")) } } } test( "schema metadata mismatch between DDL and table location should throw an error") { withTempDir { tempDir => withTable("delta_test") { val deltaLog = getDeltaLog(new Path(tempDir.getCanonicalPath)) val txn = deltaLog.startTransaction() txn.commit( Seq(Metadata(schemaString = new StructType().add("a", "long") .add("b", "string", nullable = true, new MetadataBuilder().putBoolean("pii", value = true).build()).json)), DeltaOperations.ManualUpdate) val ex = intercept[AnalysisException] { sql("CREATE TABLE delta_test(a LONG, b String)" + s" USING delta OPTIONS (path '${tempDir.getCanonicalPath}')") } assert(ex.getMessage.contains("The specified schema does not match the existing schema")) assert(ex.getMessage.contains("metadata for field b is different")) } } } test( "partition schema mismatch between DDL and table location should throw an error") { withTempDir { tempDir => withTable("delta_test") { val deltaLog = getDeltaLog(new Path(tempDir.getCanonicalPath)) val txn = deltaLog.startTransaction() txn.commit( Seq(Metadata( schemaString = new StructType().add("a", "long").add("b", "string").json, partitionColumns = Seq("a"))), DeltaOperations.ManualUpdate) val ex = intercept[AnalysisException](sql("CREATE TABLE delta_test(a LONG, b String)" + s" USING delta PARTITIONED BY(b) LOCATION '${tempDir.getCanonicalPath}'")) assert(ex.getMessage.contains( "The specified partitioning does not match the existing partitioning")) } } } testQuietly("create table with unknown table properties should throw an error") { withTempDir { tempDir => withTable("delta_test") { val ex = intercept[AnalysisException](sql( s""" |CREATE TABLE delta_test(a LONG, b String) |USING delta LOCATION '${tempDir.getCanonicalPath}' |TBLPROPERTIES('delta.key' = 'value') """.stripMargin)) assert(ex.getMessage.contains( "Unknown configuration was specified: delta.key")) } } } testQuietly("create table with invalid table properties should throw an error") { withTempDir { tempDir => withTable("delta_test") { val ex1 = intercept[IllegalArgumentException](sql( s""" |CREATE TABLE delta_test(a LONG, b String) |USING delta LOCATION '${tempDir.getCanonicalPath}' |TBLPROPERTIES('delta.randomPrefixLength' = '-1') """.stripMargin)) assert(ex1.getMessage.contains( "randomPrefixLength needs to be greater than 0.")) val ex2 = intercept[IllegalArgumentException](sql( s""" |CREATE TABLE delta_test(a LONG, b String) |USING delta LOCATION '${tempDir.getCanonicalPath}' |TBLPROPERTIES('delta.randomPrefixLength' = 'value') """.stripMargin)) assert(ex2.getMessage.contains( "randomPrefixLength needs to be greater than 0.")) } } } test( "table properties mismatch between DDL and table location should throw an error") { withTempDir { tempDir => withTable("delta_test") { val deltaLog = getDeltaLog(new Path(tempDir.getCanonicalPath)) val txn = deltaLog.startTransaction() txn.commit( Seq(Metadata( schemaString = new StructType().add("a", "long").add("b", "string").json)), DeltaOperations.ManualUpdate) val ex = intercept[AnalysisException] { sql( s""" |CREATE TABLE delta_test(a LONG, b String) |USING delta LOCATION '${tempDir.getCanonicalPath}' |TBLPROPERTIES('delta.randomizeFilePrefixes' = 'true') """.stripMargin) } assert(ex.getMessage.contains( "The specified properties do not match the existing properties")) } } } test("create table on an existing table location") { val catalog = spark.sessionState.catalog withTempDir { tempDir => withTable("delta_test") { val deltaLog = getDeltaLog(new Path(tempDir.getCanonicalPath)) val txn = deltaLog.startTransaction() txn.commit( Seq(Metadata( schemaString = new StructType().add("a", "long").add("b", "string").json, partitionColumns = Seq("b"))), DeltaOperations.ManualUpdate) sql("CREATE TABLE delta_test(a LONG, b String) USING delta " + s"OPTIONS (path '${tempDir.getCanonicalPath}') PARTITIONED BY(b)") val table = catalog.getTableMetadata(TableIdentifier("delta_test")) assert(table.tableType == CatalogTableType.EXTERNAL) assert(table.provider.contains("delta")) // Query the data and the metadata directly via the DeltaLog val deltaLog2 = getDeltaLog(table) // Since we manually committed Metadata without schema, we won't have column metadata in // the latest deltaLog snapshot assert( deltaLog2.snapshot.schema == new StructType().add("a", "long").add("b", "string")) assert( deltaLog2.snapshot.metadata.partitionSchema == new StructType().add("b", "string")) assert(getSchema("delta_test") === deltaLog2.snapshot.schema) assert(getPartitioningColumns("delta_test") === Seq("b")) // External catalog does not contain the schema and partition column names. verifyTableInCatalog(catalog, "delta_test") } } } test("create datasource table with a non-existing location") { withTempPath { dir => withTable("t") { spark.sql(s"CREATE TABLE t(a int, b int) USING delta LOCATION '${dir.getAbsolutePath}'") val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t")) assert(table.location == makeQualifiedPath(dir.getAbsolutePath)) spark.sql("INSERT INTO TABLE t SELECT 1, 2") assert(dir.exists()) checkDatasetUnorderly( sql("SELECT * FROM t").as[(Int, Int)], 1 -> 2) } } // partition table withTempPath { dir => withTable("t1") { spark.sql( s""" |CREATE TABLE t1(a int, b int) USING delta PARTITIONED BY(a) |LOCATION '${dir.getAbsolutePath}' |""".stripMargin) val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t1")) assert(table.location == makeQualifiedPath(dir.getAbsolutePath)) Seq((1, 2)).toDF("a", "b") .write.format("delta").mode("append").save(table.location.getPath) val read = spark.read.format("delta").load(table.location.getPath) checkAnswer(read, Seq(Row(1, 2))) val deltaLog = loadDeltaLog(table.location.getPath) assert(deltaLog.update().version > 0) assertPartitionWithValueExists("a", "1", deltaLog) } } } Seq(true, false).foreach { shouldDelete => val tcName = if (shouldDelete) "non-existing" else "existing" test(s"CTAS for external data source table with $tcName location") { val catalog = spark.sessionState.catalog withTable("t", "t1") { withTempDir { dir => if (shouldDelete) dir.delete() spark.sql( s""" |CREATE TABLE t |USING delta |LOCATION '${dir.getAbsolutePath}' |AS SELECT 3 as a, 4 as b, 1 as c, 2 as d """.stripMargin) val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t")) assert(table.tableType == CatalogTableType.EXTERNAL) assert(table.provider.contains("delta")) assert(table.location == makeQualifiedPath(dir.getAbsolutePath)) // Query the data and the metadata directly via the DeltaLog val deltaLog = getDeltaLog(table) assert(deltaLog.update().version >= 0) assertEqual(deltaLog.snapshot.schema, new StructType() .add("a", "integer").add("b", "integer") .add("c", "integer").add("d", "integer")) assertEqual( deltaLog.snapshot.metadata.partitionSchema, new StructType()) assertEqual(getSchema("t"), deltaLog.snapshot.schema) assert(getPartitioningColumns("t").isEmpty) // External catalog does not contain the schema and partition column names. verifyTableInCatalog(catalog, "t") // Query the table checkAnswer(spark.table("t"), Row(3, 4, 1, 2)) // Directly query the reservoir checkAnswer(spark.read.format("delta") .load(new Path(table.storage.locationUri.get).toString), Seq(Row(3, 4, 1, 2))) } // partition table withTempDir { dir => if (shouldDelete) dir.delete() spark.sql( s""" |CREATE TABLE t1 |USING delta |PARTITIONED BY(a, b) |LOCATION '${dir.getAbsolutePath}' |AS SELECT 3 as a, 4 as b, 1 as c, 2 as d """.stripMargin) val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t1")) assert(table.tableType == CatalogTableType.EXTERNAL) assert(table.provider.contains("delta")) assert(table.location == makeQualifiedPath(dir.getAbsolutePath)) // Query the data and the metadata directly via the DeltaLog val deltaLog = getDeltaLog(table) assertEqual(deltaLog.snapshot.schema, new StructType() .add("a", "integer").add("b", "integer") .add("c", "integer").add("d", "integer")) assertEqual( deltaLog.snapshot.metadata.partitionSchema, new StructType() .add("a", "integer").add("b", "integer")) assertEqual(getSchema("t1"), deltaLog.snapshot.schema) assert(getPartitioningColumns("t1") == Seq("a", "b")) // External catalog does not contain the schema and partition column names. verifyTableInCatalog(catalog, "t1") // Query the table checkAnswer(spark.table("t1"), Row(3, 4, 1, 2)) // Directly query the reservoir checkAnswer(spark.read.format("delta") .load(new Path(table.storage.locationUri.get).toString), Seq(Row(3, 4, 1, 2))) } } } } test("CTAS with table properties") { withTable("delta_test") { sql( s""" |CREATE TABLE delta_test |USING delta |TBLPROPERTIES( | 'delta.logRetentionDuration' = '2 weeks', | 'delta.checkpointInterval' = '20', | 'key' = 'value' |) |AS SELECT 3 as a, 4 as b, 1 as c, 2 as d """.stripMargin) val deltaLog = getDeltaLog("delta_test") val snapshot = deltaLog.update() assertEqual(snapshot.metadata.configuration, Map( "delta.logRetentionDuration" -> "2 weeks", "delta.checkpointInterval" -> "20", "key" -> "value")) assert(deltaLog.deltaRetentionMillis(snapshot.metadata) == 2 * 7 * 24 * 60 * 60 * 1000) assert(deltaLog.checkpointInterval(snapshot.metadata) == 20) } } test("CTAS with table properties - case insensitivity") { withTable("delta_test") { sql( s""" |CREATE TABLE delta_test |USING delta |TBLPROPERTIES( | 'dEltA.lOgrEteNtiOndURaTion' = '2 weeks', | 'DelTa.ChEckPoiNtinTervAl' = '20' |) |AS SELECT 3 as a, 4 as b, 1 as c, 2 as d """.stripMargin) val deltaLog = getDeltaLog("delta_test") val snapshot = deltaLog.update() assertEqual(snapshot.metadata.configuration, Map("delta.logRetentionDuration" -> "2 weeks", "delta.checkpointInterval" -> "20")) assert(deltaLog.deltaRetentionMillis(snapshot.metadata) == 2 * 7 * 24 * 60 * 60 * 1000) assert(deltaLog.checkpointInterval(snapshot.metadata) == 20) } } testQuietly("CTAS external table with existing data should fail") { withTable("t") { withTempDir { dir => dir.delete() Seq((3, 4)).toDF("a", "b") .write.format("delta") .save(dir.getAbsolutePath) val ex = intercept[AnalysisException](spark.sql( s""" |CREATE TABLE t |USING delta |LOCATION '${dir.getAbsolutePath}' |AS SELECT 1 as a, 2 as b """.stripMargin)) assert(ex.getMessage.contains("Cannot create table")) } } withTable("t") { withTempDir { dir => dir.delete() Seq((3, 4)).toDF("a", "b").write.format("parquet").save(dir.getCanonicalPath) val ex = intercept[AnalysisException](spark.sql( s""" |CREATE TABLE t |USING delta |LOCATION '${dir.getAbsolutePath}' |AS SELECT 1 as a, 2 as b """.stripMargin)) assert(ex.getMessage.contains("Cannot create table")) } } } testQuietly("CTAS with unknown table properties should throw an error") { withTempDir { tempDir => withTable("delta_test") { val ex = intercept[AnalysisException] { sql( s""" |CREATE TABLE delta_test |USING delta |LOCATION '${tempDir.getCanonicalPath}' |TBLPROPERTIES('delta.key' = 'value') |AS SELECT 3 as a, 4 as b, 1 as c, 2 as d """.stripMargin) } assert(ex.getMessage.contains( "Unknown configuration was specified: delta.key")) } } } testQuietly("CTAS with invalid table properties should throw an error") { withTempDir { tempDir => withTable("delta_test") { val ex1 = intercept[IllegalArgumentException] { sql( s""" |CREATE TABLE delta_test |USING delta |LOCATION '${tempDir.getCanonicalPath}' |TBLPROPERTIES('delta.randomPrefixLength' = '-1') |AS SELECT 3 as a, 4 as b, 1 as c, 2 as d """.stripMargin) } assert(ex1.getMessage.contains( "randomPrefixLength needs to be greater than 0.")) val ex2 = intercept[IllegalArgumentException] { sql( s""" |CREATE TABLE delta_test |USING delta |LOCATION '${tempDir.getCanonicalPath}' |TBLPROPERTIES('delta.randomPrefixLength' = 'value') |AS SELECT 3 as a, 4 as b, 1 as c, 2 as d """.stripMargin) } assert(ex2.getMessage.contains( "randomPrefixLength needs to be greater than 0.")) } } } Seq("a:b", "a%b").foreach { specialChars => test(s"data source table:partition column name containing $specialChars") { // On Windows, it looks colon in the file name is illegal by default. See // https://support.microsoft.com/en-us/help/289627 assume(!Utils.isWindows || specialChars != "a:b") withTable("t") { withTempDir { dir => spark.sql( s""" |CREATE TABLE t(a string, `$specialChars` string) |USING delta |PARTITIONED BY(`$specialChars`) |LOCATION '${dir.getAbsolutePath}' """.stripMargin) assert(dir.listFiles().forall(_.toString.contains("_delta_log"))) spark.sql(s"INSERT INTO TABLE t SELECT 1, 2") val deltaLog = loadDeltaLog(dir.getAbsolutePath) assert(deltaLog.update().version > 0) assertPartitionWithValueExists(specialChars, "2", deltaLog) checkAnswer(spark.table("t"), Row("1", "2") :: Nil) } } } } Seq("a b", "a:b", "a%b").foreach { specialChars => test(s"location uri contains $specialChars for datasource table") { // On Windows, it looks colon in the file name is illegal by default. See // https://support.microsoft.com/en-us/help/289627 assume(!Utils.isWindows || specialChars != "a:b") withTable("t", "t1") { withTempDir { dir => val loc = new File(dir, specialChars) loc.mkdir() // The parser does not recognize the backslashes on Windows as they are. // These currently should be escaped. val escapedLoc = loc.getAbsolutePath.replace("\\", "\\\\") spark.sql( s""" |CREATE TABLE t(a string) |USING delta |LOCATION '$escapedLoc' """.stripMargin) val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t")) assert(table.location == makeQualifiedPath(loc.getAbsolutePath)) assert(new Path(table.location).toString.contains(specialChars)) assert(loc.listFiles().forall(_.toString.contains("_delta_log"))) spark.sql("INSERT INTO TABLE t SELECT 1") assert(!loc.listFiles().forall(_.toString.contains("_delta_log"))) checkAnswer(spark.table("t"), Row("1") :: Nil) } withTempDir { dir => val loc = new File(dir, specialChars) loc.mkdir() // The parser does not recognize the backslashes on Windows as they are. // These currently should be escaped. val escapedLoc = loc.getAbsolutePath.replace("\\", "\\\\") spark.sql( s""" |CREATE TABLE t1(a string, b string) |USING delta |PARTITIONED BY(b) |LOCATION '$escapedLoc' """.stripMargin) val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t1")) assert(table.location == makeQualifiedPath(loc.getAbsolutePath)) assert(new Path(table.location).toString.contains(specialChars)) assert(loc.listFiles().forall(_.toString.contains("_delta_log"))) spark.sql("INSERT INTO TABLE t1 SELECT 1, 2") checkAnswer(spark.table("t1"), Row("1", "2") :: Nil) if (columnMappingEnabled) { // column mapping always use random file prefixes so we can't compare path val deltaLog = loadDeltaLog(loc.getCanonicalPath) val partPaths = getPartitionFilePathsWithValue("b", "2", deltaLog) assert(partPaths.nonEmpty) assert(partPaths.forall { p => val parentPath = new File(p).getParentFile !parentPath.listFiles().forall(_.toString.contains("_delta_log")) }) // In column mapping mode, as we are using random file prefixes, // this partition value is valid spark.sql("INSERT INTO TABLE t1 SELECT 1, '2017-03-03 12:13%3A14'") assertPartitionWithValueExists("b", "2017-03-03 12:13%3A14", deltaLog) checkAnswer( spark.table("t1"), Row("1", "2") :: Row("1", "2017-03-03 12:13%3A14") :: Nil) } else { val partFile = new File(loc, "b=2") assert(!partFile.listFiles().forall(_.toString.contains("_delta_log"))) spark.sql("INSERT INTO TABLE t1 SELECT 1, '2017-03-03 12:13%3A14'") val partFile1 = new File(loc, "b=2017-03-03 12:13%3A14") assert(!partFile1.exists()) if (!Utils.isWindows) { // Actual path becomes "b=2017-03-03%2012%3A13%253A14" on Windows. val partFile2 = new File(loc, "b=2017-03-03 12%3A13%253A14") assert(!partFile2.listFiles().forall(_.toString.contains("_delta_log"))) checkAnswer( spark.table("t1"), Row("1", "2") :: Row("1", "2017-03-03 12:13%3A14") :: Nil) } } } } } } test("the qualified path of a delta table is stored in the catalog") { withTempDir { dir => withTable("t", "t1") { assert(!dir.getAbsolutePath.startsWith("file:/")) // The parser does not recognize the backslashes on Windows as they are. // These currently should be escaped. val escapedDir = dir.getAbsolutePath.replace("\\", "\\\\") spark.sql( s""" |CREATE TABLE t(a string) |USING delta |LOCATION '$escapedDir' """.stripMargin) val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t")) assert(table.location.toString.startsWith("file:/")) } } withTempDir { dir => withTable("t", "t1") { assert(!dir.getAbsolutePath.startsWith("file:/")) // The parser does not recognize the backslashes on Windows as they are. // These currently should be escaped. val escapedDir = dir.getAbsolutePath.replace("\\", "\\\\") spark.sql( s""" |CREATE TABLE t1(a string, b string) |USING delta |PARTITIONED BY(b) |LOCATION '$escapedDir' """.stripMargin) val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t1")) assert(table.location.toString.startsWith("file:/")) } } } testQuietly("CREATE TABLE with existing data path") { // Re-use `filterV2TableProperties()` from `SQLTestUtils` as soon as it will be released. def isReservedProperty(propName: String): Boolean = { CatalogV2Util.TABLE_RESERVED_PROPERTIES.contains(propName) || propName.startsWith(TableCatalog.OPTION_PREFIX) || propName == TableCatalog.PROP_EXTERNAL } def filterV2TableProperties(properties: Map[String, String]): Map[String, String] = { properties.filterNot(kv => isReservedProperty(kv._1)) } withTempPath { path => withTable("src", "t1", "t2", "t3", "t4", "t5", "t6") { sql("CREATE TABLE src(i int, p string) USING delta PARTITIONED BY (p) " + "TBLPROPERTIES('delta.randomizeFilePrefixes' = 'true') " + s"LOCATION '${path.getAbsolutePath}'") sql("INSERT INTO src SELECT 1, 'a'") // CREATE TABLE without specifying anything works sql(s"CREATE TABLE t1 USING delta LOCATION '${path.getAbsolutePath}'") checkAnswer(spark.table("t1"), Row(1, "a")) // CREATE TABLE with the same schema and partitioning but no properties works sql(s"CREATE TABLE t2(i int, p string) USING delta PARTITIONED BY (p) " + s"LOCATION '${path.getAbsolutePath}'") checkAnswer(spark.table("t2"), Row(1, "a")) // Table properties should not be changed to empty. assert(filterV2TableProperties(getTableProperties("t2")) == Map("delta.randomizeFilePrefixes" -> "true")) // CREATE TABLE with the same schema but no partitioning fails. val e0 = intercept[AnalysisException] { sql(s"CREATE TABLE t3(i int, p string) USING delta LOCATION '${path.getAbsolutePath}'") } assert(e0.message.contains("The specified partitioning does not match the existing")) // CREATE TABLE with different schema fails val e1 = intercept[AnalysisException] { sql(s"CREATE TABLE t4(j int, p string) USING delta LOCATION '${path.getAbsolutePath}'") } assert(e1.message.contains("The specified schema does not match the existing")) // CREATE TABLE with different partitioning fails val e2 = intercept[AnalysisException] { sql(s"CREATE TABLE t5(i int, p string) USING delta PARTITIONED BY (i) " + s"LOCATION '${path.getAbsolutePath}'") } assert(e2.message.contains("The specified partitioning does not match the existing")) // CREATE TABLE with different table properties fails val e3 = intercept[AnalysisException] { sql(s"CREATE TABLE t6 USING delta " + "TBLPROPERTIES ('delta.randomizeFilePrefixes' = 'false') " + s"LOCATION '${path.getAbsolutePath}'") } assert(e3.message.contains("The specified properties do not match the existing")) } } } test("CREATE TABLE on existing data should not commit metadata") { withTempDir { tempDir => val path = tempDir.getCanonicalPath() val df = Seq(1, 2, 3, 4, 5).toDF() df.write.format("delta").save(path) val deltaLog = getDeltaLog(new Path(path)) val oldVersion = deltaLog.snapshot.version sql(s"CREATE TABLE table USING delta LOCATION '$path'") assert(oldVersion == deltaLog.snapshot.version) } } } class DeltaTableCreationSuite extends DeltaTableCreationTests with DeltaSQLCommandTest { private def loadTable(tableName: String): Table = { val ti = spark.sessionState.sqlParser.parseMultipartIdentifier(tableName) val namespace = if (ti.length == 1) Array("default") else ti.init.toArray spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog] .loadTable(Identifier.of(namespace, ti.last)) } override protected def getPartitioningColumns(tableName: String): Seq[String] = { loadTable(tableName).partitioning() .map(_.references().head.fieldNames().mkString(".")) } override def getSchema(tableName: String): StructType = { loadTable(tableName).schema() } override protected def getTableProperties(tableName: String): Map[String, String] = { loadTable(tableName).properties().asScala.toMap .filterKeys(!CatalogV2Util.TABLE_RESERVED_PROPERTIES.contains(_)) .filterKeys(!TableFeatureProtocolUtils.isTableProtocolProperty(_)) .toMap } testQuietly("REPLACE TABLE") { withTempDir { dir => withTable("delta_test") { sql( s"""CREATE TABLE delta_test |USING delta |LOCATION '${dir.getAbsolutePath}' |AS SELECT 1 as a """.stripMargin) val deltaLog = DeltaLog.forTable(spark, dir) assert(deltaLog.snapshot.version === 0, "CTAS should be a single commit") sql( s"""REPLACE TABLE delta_test (col string) |USING delta |LOCATION '${dir.getAbsolutePath}' """.stripMargin) assert(deltaLog.snapshot.version === 1) assertEqual( deltaLog.snapshot.schema, new StructType().add("col", "string")) val e2 = intercept[AnalysisException] { sql( s"""REPLACE TABLE delta_test |USING delta |LOCATION '${dir.getAbsolutePath}' """.stripMargin) } assert(e2.getMessage.contains("schema is not provided")) } } } testQuietly("CREATE OR REPLACE TABLE on table without schema") { withTempDir { dir => withTable("delta_test") { spark.range(10).write.format("delta").option("path", dir.getCanonicalPath) .saveAsTable("delta_test") // We need the schema val e = intercept[AnalysisException] { sql(s"""CREATE OR REPLACE TABLE delta_test |USING delta |LOCATION '${dir.getAbsolutePath}' """.stripMargin) } assert(e.getMessage.contains("schema is not provided")) } } } testQuietly("CREATE OR REPLACE TABLE on non-empty directory") { withTempDir { dir => spark.range(10).write.format("delta").save(dir.getCanonicalPath) withTable("delta_test") { // We need the schema val e = intercept[AnalysisException] { sql(s"""CREATE OR REPLACE TABLE delta_test |USING delta |LOCATION '${dir.getAbsolutePath}' """.stripMargin) } assert(e.getMessage.contains("schema is not provided")) } } } testQuietly( "REPLACE TABLE on non-empty directory") { withTempDir { dir => spark.range(10).write.format("delta").save(dir.getCanonicalPath) withTable("delta_test") { val e = intercept[AnalysisException] { sql( s"""REPLACE TABLE delta_test |USING delta |LOCATION '${dir.getAbsolutePath}' """.stripMargin) } assert(e.getMessage.contains("cannot be replaced as it did not exist") || e.getMessage.contains(s"table or view `default`.`delta_test` cannot be found")) } } } test("Create a table without comment") { withTempDir { dir => val table = "delta_without_comment" withTable(table) { sql(s"CREATE TABLE $table (col string) USING delta LOCATION '${dir.getAbsolutePath}'") checkResult( sql(s"DESCRIBE DETAIL $table"), Seq("delta", null), Seq("format", "description")) } } } protected def withEmptySchemaTable(emptyTableName: String)(f: => Unit): Unit = { def getDeltaLog: DeltaLog = DeltaLog.forTable(spark, TableIdentifier(emptyTableName)) // create using SQL API withTable(emptyTableName) { sql(s"CREATE TABLE $emptyTableName USING delta") assert(getDeltaLog.snapshot.schema.isEmpty) f // just make sure this statement runs sql(s"CREATE TABLE IF NOT EXISTS $emptyTableName USING delta") } // create using Delta table API (creates v1 table) withTable(emptyTableName) { io.delta.tables.DeltaTable .create(spark) .tableName(emptyTableName) .execute() assert(getDeltaLog.snapshot.schema.isEmpty) f io.delta.tables.DeltaTable .createIfNotExists(spark) .tableName(emptyTableName) .execute() } } test("Create an empty table without schema - unsupported cases") { import testImplicits._ withSQLConf(DeltaSQLConf.DELTA_ALLOW_CREATE_EMPTY_SCHEMA_TABLE.key -> "true") { val emptySchemaTableName = "t1" // TODO: support CREATE OR REPLACE code path if needed in the future intercept[AnalysisException] { sql(s"CREATE OR REPLACE TABLE $emptySchemaTableName USING delta") } // similarly blocked using Delta Table API withTable(emptySchemaTableName) { intercept[AnalysisException] { io.delta.tables.DeltaTable .createOrReplace(spark) .tableName(emptySchemaTableName) .execute() } } withTable(emptySchemaTableName) { io.delta.tables.DeltaTable .create(spark) .tableName(emptySchemaTableName) .execute() intercept[AnalysisException] { io.delta.tables.DeltaTable .replace(spark) .tableName(emptySchemaTableName) .execute() } } // external table with an invalid location it shouldn't work (e.g. no transaction log present) withTable(emptySchemaTableName) { withTempDir { dir => Seq(1, 2, 3).toDF().write.format("delta").save(dir.getAbsolutePath) Utils.deleteRecursively(new File(dir, "_delta_log")) val e = intercept[AnalysisException] { sql(s"CREATE TABLE $emptySchemaTableName USING delta LOCATION '${dir.getAbsolutePath}'") } assert(e.getErrorClass == "DELTA_CREATE_EXTERNAL_TABLE_WITHOUT_TXN_LOG") } } // CTAS from an empty schema dataframe should be blocked intercept[AnalysisException] { withTable(emptySchemaTableName) { val df = spark.emptyDataFrame df.createOrReplaceTempView("empty_df") sql(s"CREATE TABLE $emptySchemaTableName USING delta AS SELECT * FROM empty_df") } } // create empty schema table using dataframe api should be blocked intercept[AnalysisException] { withTable(emptySchemaTableName) { spark.emptyDataFrame .write.format("delta") .saveAsTable(emptySchemaTableName) } } intercept[AnalysisException] { withTable(emptySchemaTableName) { spark.emptyDataFrame .writeTo(emptySchemaTableName) .using("delta") .create() } } def assertFailToRead(f: => Any): Unit = { try f catch { case e: AnalysisException => assert(e.getMessage.contains("that does not have any columns.")) } } def assertSchemaEvolutionRequired(f: => Any): Unit = { val e = intercept[AnalysisException] { f } assert(e.getMessage.contains("A schema mismatch detected when writing to the Delta")) } // data reading or writing without mergeSchema should fail withEmptySchemaTable(emptySchemaTableName) { assertFailToRead { spark.read.table(emptySchemaTableName).collect() } assertFailToRead { sql(s"SELECT * FROM $emptySchemaTableName").collect() } assertSchemaEvolutionRequired { sql(s"INSERT INTO $emptySchemaTableName VALUES (1,2,3)") } // but enabling auto merge should make insert work withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { sql(s"INSERT INTO $emptySchemaTableName VALUES (1,2,3)") checkAnswer(spark.read.table(emptySchemaTableName), Seq(Row(1, 2, 3))) } } // allows drop and recreate the same table with empty schema withTempDir { dir => withTable(emptySchemaTableName) { sql(s"CREATE TABLE $emptySchemaTableName USING delta LOCATION '${dir.getCanonicalPath}'") val snapshot = DeltaLog.forTable(spark, TableIdentifier(emptySchemaTableName)).update() assert(snapshot.schema.isEmpty && snapshot.version == 0) assertFailToRead { sql(s"SELECT * FROM $emptySchemaTableName") } // drop the table sql(s"DROP TABLE $emptySchemaTableName") // recreate the table again should work sql(s"CREATE TABLE $emptySchemaTableName USING delta LOCATION '${dir.getCanonicalPath}'") assertFailToRead { sql(s"SELECT * FROM $emptySchemaTableName") } // write some data to it withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { sql(s"INSERT INTO $emptySchemaTableName VALUES (1,2,3)") checkAnswer(spark.read.table(emptySchemaTableName), Seq(Row(1, 2, 3))) } // drop again sql(s"DROP TABLE $emptySchemaTableName") // recreate the table again should work sql(s"CREATE TABLE $emptySchemaTableName USING delta LOCATION '${dir.getCanonicalPath}'") checkAnswer(spark.read.table(emptySchemaTableName), Seq(Row(1, 2, 3))) } } } } test("Create an empty table without schema - supported cases") { import testImplicits._ withSQLConf(DeltaSQLConf.DELTA_ALLOW_CREATE_EMPTY_SCHEMA_TABLE.key -> "true") { val emptyTableName = "t1" def getDeltaLog: DeltaLog = DeltaLog.forTable(spark, TableIdentifier(emptyTableName)) // yet CTAS should be allowed withTable(emptyTableName) { sql(s"CREATE TABLE $emptyTableName USING delta AS SELECT 1") assert(getDeltaLog.snapshot.schema.size == 1) } // and create Delta table using existing valid location should work without () withTable(emptyTableName) { withTempDir { dir => Seq(1, 2, 3).toDF().write.format("delta").save(dir.getAbsolutePath) sql(s"CREATE TABLE $emptyTableName USING delta LOCATION '${dir.getAbsolutePath}'") assert(getDeltaLog.snapshot.schema.size == 1) } } // checkpointing should work withEmptySchemaTable(emptyTableName) { getDeltaLog.checkpoint() assert(getDeltaLog.readLastCheckpointFile().exists(_.version == 0)) // run some operations withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { sql(s"INSERT INTO $emptyTableName VALUES (1,2,3)") checkAnswer(spark.read.table(emptyTableName), Seq(Row(1, 2, 3))) } getDeltaLog.checkpoint() assert(getDeltaLog.readLastCheckpointFile().exists(_.version == 1)) } withEmptySchemaTable(emptyTableName) { // TODO: possibly support MERGE into the future try { val source = "t2" withTable(source) { sql(s"CREATE TABLE $source USING delta AS SELECT 1") sql( s""" |MERGE INTO $emptyTableName |USING $source |ON FALSE |WHEN NOT MATCHED | THEN INSERT * |""".stripMargin) } } catch { case _: AssertionError | _: SparkException => } } // Delta specific DMLs should work, though they should basically be noops withEmptySchemaTable(emptyTableName) { sql(s"OPTIMIZE $emptyTableName") sql(s"VACUUM $emptyTableName") assert(getDeltaLog.snapshot.schema.isEmpty) } // metadata DDL should work withEmptySchemaTable(emptyTableName) { sql(s"ALTER TABLE $emptyTableName SET TBLPROPERTIES ('a' = 'b')") assert(DeltaLog.forTable(spark, TableIdentifier(emptyTableName)).snapshot.metadata.configuration.contains("a")) checkAnswer( sql(s"COMMENT ON TABLE $emptyTableName IS 'My Empty Cool Table'"), Nil) assert(sql(s"DESCRIBE TABLE $emptyTableName").collect().length == 0) // create table, alter tbl property, tbl comment assert(sql(s"DESCRIBE HISTORY $emptyTableName").collect().length == 3) checkAnswer(sql(s"SHOW COLUMNS IN $emptyTableName"), Nil) } // schema evolution ddl should work withEmptySchemaTable(emptyTableName) { sql(s"ALTER TABLE $emptyTableName ADD COLUMN (id long COMMENT 'haha')") assert(getDeltaLog.snapshot.schema.size == 1) } withEmptySchemaTable(emptyTableName) { sql(s"ALTER TABLE $emptyTableName ADD COLUMNS (id long, id2 long)") assert(getDeltaLog.snapshot.schema.size == 2) } // schema evolution through df should work // - v1 api withEmptySchemaTable(emptyTableName) { Seq(1, 2, 3).toDF() .write.format("delta") .mode("append") .option("mergeSchema", "true") .saveAsTable(emptyTableName) assert(getDeltaLog.snapshot.schema.size == 1) } withEmptySchemaTable(emptyTableName) { Seq(1, 2, 3).toDF() .write.format("delta") .mode("overwrite") .option("overwriteSchema", "true") .saveAsTable(emptyTableName) assert(getDeltaLog.snapshot.schema.size == 1) } // - v2 api withEmptySchemaTable(emptyTableName) { Seq(1, 2, 3).toDF() .writeTo(emptyTableName) .option("mergeSchema", "true") .append() assert(getDeltaLog.snapshot.schema.size == 1) } withEmptySchemaTable(emptyTableName) { Seq(1, 2, 3).toDF() .writeTo(emptyTableName) .using("delta") .replace() assert(getDeltaLog.snapshot.schema.size == 1) } } } test("Create a table with comment") { val table = "delta_with_comment" withTempDir { dir => withTable(table) { sql( s""" |CREATE TABLE $table (col string) |USING delta |COMMENT 'This is my table' |LOCATION '${dir.getAbsolutePath}' """.stripMargin) checkResult( sql(s"DESCRIBE DETAIL $table"), Seq("delta", "This is my table"), Seq("format", "description")) } } } test("Replace a table without comment") { withTempDir { dir => val table = "replace_table_without_comment" val location = dir.getAbsolutePath withTable(table) { sql(s"CREATE TABLE $table (col string) USING delta COMMENT 'Table' LOCATION '$location'") sql(s"REPLACE TABLE $table (col string) USING delta LOCATION '$location'") checkResult( sql(s"DESCRIBE DETAIL $table"), Seq("delta", null), Seq("format", "description")) } } } test("Replace a table with comment") { withTempDir { dir => val table = "replace_table_with_comment" val location = dir.getAbsolutePath withTable(table) { sql(s"CREATE TABLE $table (col string) USING delta LOCATION '$location'") sql( s""" |REPLACE TABLE $table (col string) |USING delta |COMMENT 'This is my table' |LOCATION '$location' """.stripMargin) checkResult( sql(s"DESCRIBE DETAIL $table"), Seq("delta", "This is my table"), Seq("format", "description")) } } } test("CTAS a table without comment") { val table = "ctas_without_comment" withTable(table) { sql(s"CREATE TABLE $table USING delta AS SELECT * FROM range(10)") checkResult( sql(s"DESCRIBE DETAIL $table"), Seq("delta", null), Seq("format", "description")) } } test("CTAS a table with comment") { val table = "ctas_with_comment" withTable(table) { sql( s"""CREATE TABLE $table |USING delta |COMMENT 'This table is created with existing data' |AS SELECT * FROM range(10) """.stripMargin) checkResult( sql(s"DESCRIBE DETAIL $table"), Seq("delta", "This table is created with existing data"), Seq("format", "description")) } } test("Replace CTAS a table without comment") { val table = "replace_ctas_without_comment" withTable(table) { sql( s"""CREATE TABLE $table |USING delta |COMMENT 'This table is created with existing data' |AS SELECT * FROM range(10) """.stripMargin) sql(s"REPLACE TABLE $table USING delta AS SELECT * FROM range(10)") checkResult( sql(s"DESCRIBE DETAIL $table"), Seq("delta", null), Seq("format", "description")) } } test("Replace CTAS a table with comment") { val table = "replace_ctas_with_comment" withTable(table) { sql(s"CREATE TABLE $table USING delta COMMENT 'a' AS SELECT * FROM range(10)") sql( s"""REPLACE TABLE $table |USING delta |COMMENT 'This table is created with existing data' |AS SELECT * FROM range(10) """.stripMargin) checkResult( sql(s"DESCRIBE DETAIL $table"), Seq("delta", "This table is created with existing data"), Seq("format", "description")) } } /** * Verifies that the correct table properties are stored in the transaction log as well as the * catalog. */ private def verifyTableProperties( tableName: String, deltaLogPropertiesContains: Seq[String], deltaLogPropertiesMissing: Seq[String], catalogStorageProps: Seq[String] = Nil): Unit = { val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName)) if (catalogStorageProps.isEmpty) { assert(table.storage.properties.isEmpty) } else { assert(catalogStorageProps.forall(table.storage.properties.contains), s"Catalog didn't contain properties: ${catalogStorageProps}.\n" + s"Catalog: ${table.storage.properties}") } val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) deltaLogPropertiesContains.foreach { prop => assert(deltaLog.snapshot.getProperties.contains(prop)) } deltaLogPropertiesMissing.foreach { prop => assert(!deltaLog.snapshot.getProperties.contains(prop)) } } test("do not store write options in the catalog - DataFrameWriter") { withTempDir { dir => withTable("t") { spark.range(10).write.format("delta") .option("path", dir.getCanonicalPath) .option("mergeSchema", "true") .option("delta.appendOnly", "true") .saveAsTable("t") verifyTableProperties( "t", // Still allow delta prefixed confs Seq("delta.appendOnly"), Seq("mergeSchema") ) // Sanity check that table is readable checkAnswer(spark.table("t"), spark.range(10).toDF()) } } } test("do not store write options in the catalog - DataFrameWriterV2") { withTempDir { dir => withTable("t") { spark.range(10).writeTo("t").using("delta") .option("path", dir.getCanonicalPath) .option("mergeSchema", "true") .option("delta.appendOnly", "true") .tableProperty("key", "value") .create() verifyTableProperties( "t", Seq( "delta.appendOnly", // Still allow delta prefixed confs "key" // Explicit properties should work ), Seq("mergeSchema") ) // Sanity check that table is readable checkAnswer(spark.table("t"), spark.range(10).toDF()) } } } test( "do not store write options in the catalog - legacy flag") { withTempDir { dir => withTable("t") { withSQLConf(DeltaSQLConf.DELTA_LEGACY_STORE_WRITER_OPTIONS_AS_PROPS.key -> "true") { spark.range(10).write.format("delta") .option("path", dir.getCanonicalPath) .option("mergeSchema", "true") .option("delta.appendOnly", "true") .saveAsTable("t") verifyTableProperties( "t", // Everything gets stored in the transaction log Seq("delta.appendOnly", "mergeSchema"), Nil, // Things get stored in the catalog props as well Seq("delta.appendOnly", "mergeSchema") ) checkAnswer(spark.table("t"), spark.range(10).toDF()) } } } } test("create table using varchar at the same location should succeed") { withTempDir { location => withTable("t1", "t2") { sql(s""" |create table t1 |(colourID string, colourName varchar(128), colourGroupID string) |USING delta LOCATION '$location'""".stripMargin) sql( s""" |insert into t1 (colourID, colourName, colourGroupID) |values ('1', 'RED', 'a'), ('2', 'BLUE', 'b') |""".stripMargin) sql(s""" |create table t2 |(colourID string, colourName varchar(128), colourGroupID string) |USING delta LOCATION '$location'""".stripMargin) // Verify that select from the second table should be the same as inserted val readout = sql( s""" |select * from t2 order by colourID |""".stripMargin).collect() assert(readout.length == 2) assert(readout(0).get(0) == "1") assert(readout(0).get(1) == "RED") assert(readout(1).get(0) == "2") assert(readout(1).get(1) == "BLUE") } } } test("CREATE OR REPLACE TABLE on a catalog table where the backing " + "directory has been deleted") { val tbl = "delta_tbl" withTempDir { dir => withTable(tbl) { val subdir = new File(dir, "subdir") sql(s"CREATE OR REPLACE table $tbl (id String) USING delta " + s"LOCATION '${subdir.getCanonicalPath}'") val tableIdentifier = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tbl)).identifier val tableName = tableIdentifier.copy(catalog = None).toString val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tbl)) sql(s"INSERT INTO $tbl VALUES ('1')") FileUtils.deleteDirectory(subdir) val e = intercept[DeltaIllegalStateException] { sql( s"CREATE OR REPLACE table $tbl (id String) USING delta" + s" LOCATION '${subdir.getCanonicalPath}'") } checkError( e, "DELTA_METADATA_ABSENT_EXISTING_CATALOG_TABLE", parameters = Map( "tableName" -> tableName, "tablePath" -> deltaLog.logPath.toString, "tableNameForDropCmd" -> tableName )) // Table creation should work after running DROP TABLE. sql(s"DROP table ${e.getMessageParameters().get("tableNameForDropCmd")}") sql(s"CREATE OR REPLACE table $tbl (id String) USING delta " + s"LOCATION '${subdir.getCanonicalPath}'") sql(s"INSERT INTO $tbl VALUES ('21')") val data = sql(s"SELECT * FROM $tbl").collect() assert(data.length == 1) } } } private def schemaContainsExistsDefaultKey(testTableName: String): Boolean = { val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(testTableName)) snapshot.metadata.schema.fields.exists( _.metadata.contains(ResolveDefaultColumnsUtils.EXISTS_DEFAULT_COLUMN_METADATA_KEY)) } private def withDeltaTableUsingExistsDefault(testFun: String => Unit): Unit = { val testTableName = "test_table" withTable(testTableName) { withSQLConf(DeltaSQLConf.REMOVE_EXISTS_DEFAULT_FROM_SCHEMA.key -> "false") { sql(s"""CREATE TABLE $testTableName (id INT, column_with_default INT DEFAULT 1) |USING DELTA |TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported')""".stripMargin) assert(schemaContainsExistsDefaultKey(testTableName)) } testFun(testTableName) } } test("Default column values: Writes do not remove EXISTS_DEFAULT from a table") { val testTableName = "test_table" val writeDF = spark.range(end = 1) .withColumn("id", col("id").cast("int")) .withColumn("column_with_default", col("id")) .write .format("delta") val writeOperations = Seq( () => { sql(s"ALTER TABLE $testTableName ALTER COLUMN id SET DEFAULT 2") }, () => { sql(s"ALTER TABLE $testTableName CLUSTER BY (id)") }, () => { sql(s"COMMENT ON TABLE $testTableName IS 'test comment'") }, () => { sql(s"INSERT INTO $testTableName VALUES (1, 1)") }, () => { writeDF.mode("append").saveAsTable(testTableName) }, () => { writeDF.mode("overwrite").saveAsTable(testTableName) }, () => { writeDF.mode("append") .save(DeltaLog.forTable(spark, TableIdentifier(testTableName)).dataPath.toString) } ) writeOperations.foreach { writeOperation => withDeltaTableUsingExistsDefault { testTableName => // Execute the operation and assert that it keep EXISTS_DEFAULT in the schema. withSQLConf(DeltaSQLConf.REMOVE_EXISTS_DEFAULT_FROM_SCHEMA.key -> "true") { writeOperation() assert(schemaContainsExistsDefaultKey(testTableName), s"Operation '$writeOperation' did remove EXISTS_DEFAULT from the schema.") } } } } for ((shortName, createOperation) <- Seq( "CREATE TABLE" -> (() => { sql(s"""CREATE TABLE test_table(int_with_default INT DEFAULT 2) |USING DELTA |TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported')""".stripMargin) }), "CREATE OR REPLACE TABLE that CREATES" -> (() => { sql(s"""CREATE OR REPLACE TABLE test_table(int_with_default INT DEFAULT 2) |USING DELTA |TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported')""".stripMargin) }), "REPLACE TABLE" -> (() => { withSQLConf(DeltaSQLConf.REMOVE_EXISTS_DEFAULT_FROM_SCHEMA.key -> "false") { sql("CREATE TABLE test_table(id INT) USING DELTA") } sql(s"""REPLACE TABLE test_table(int_with_default INT DEFAULT 2) |USING DELTA |TBLPROPERTIES ('delta.feature.allowColumnDefaults'= 'supported')""".stripMargin) }), "CREATE OR REPLACE TABLE that REPLACES" -> (() => { withSQLConf(DeltaSQLConf.REMOVE_EXISTS_DEFAULT_FROM_SCHEMA.key -> "false") { sql("CREATE TABLE test_table(id INT) USING DELTA") } sql(s"""CREATE OR REPLACE TABLE test_table(int_with_default INT DEFAULT 2) |USING DELTA |TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported')""".stripMargin) })) ) { test(s"Default column values: Storing 'EXISTS_DEFAULT' in $shortName with column defaults") { val testTableName = "test_table" for (removeExistsDefault <- Seq(true, false)) { withTable(testTableName) { withSQLConf(DeltaSQLConf.REMOVE_EXISTS_DEFAULT_FROM_SCHEMA.key -> removeExistsDefault.toString) { createOperation() assert(schemaContainsExistsDefaultKey(testTableName) != removeExistsDefault) } } } } } test("Default column values: SHALLOW CLONE keeps EXISTS_DEFAULT") { withDeltaTableUsingExistsDefault { testTableName => val targetTableName = "target_table" withTable(targetTableName) { sql(s"CREATE TABLE $targetTableName SHALLOW CLONE $testTableName") assert(schemaContainsExistsDefaultKey(targetTableName), s"SHALLOW CLONE did remove EXISTS_DEFAULT from the schema.") } } } test("Default column values: CONVERT TO DELTA keeps EXISTS_DEFAULT") { withTable("test_table") { withSQLConf("spark.databricks.delta.properties.defaults.columnMapping.mode" -> "none") { spark.range(end = 1).write.format("parquet").saveAsTable("test_table") // EXISTS_DEFAULT is used for the existing row. sql("ALTER TABLE test_table ADD COLUMN new_column_with_a_default INT DEFAULT 1") sql("CONVERT TO DELTA test_table") checkAnswer(spark.table("test_table"), Row(0, 1) :: Nil) } } } test("Default column values: CREATE TABLE AS SELECT from a table with column defaults") { for (sourceTableSchemaContainsKey <- Seq(true, false)) { withTable("test_table", "test_table_2", "test_table_3", "test_table_4") { // To test with the 'EXISTS_DEFAULT' key present in the source table, we disable removal. withSQLConf(DeltaSQLConf.REMOVE_EXISTS_DEFAULT_FROM_SCHEMA.key -> (!sourceTableSchemaContainsKey).toString) { // Defaults are only possible for top level columns. sql("""CREATE TABLE test_table(int_col INT DEFAULT 2) |USING DELTA |TBLPROPERTIES ('delta.feature.allowColumnDefaults' = 'supported')""".stripMargin) } def schemaContainsExistsKey(tableName: String): Boolean = { val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName)) snapshot.schema.fields.exists { field => field.metadata.contains(ResolveDefaultColumnsUtils.EXISTS_DEFAULT_COLUMN_METADATA_KEY) } } def schemaContainsCurrentDefaultKey(tableName: String): Boolean = { val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName)) snapshot.schema.fields.exists { field => field.metadata.contains( ResolveDefaultColumnsUtils.CURRENT_DEFAULT_COLUMN_METADATA_KEY) } } def defaultsTableFeatureEnabled(tableName: String): Boolean = { val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName)) val isEnabled = snapshot.protocol.writerFeatureNames.contains(AllowColumnDefaultsTableFeature.name) assert(schemaContainsCurrentDefaultKey(tableName) === isEnabled) isEnabled } assert(schemaContainsExistsKey("test_table") == sourceTableSchemaContainsKey) assert(defaultsTableFeatureEnabled("test_table")) // It is not possible to add a column with a default to a Delta table. assertThrows[DeltaAnalysisException] { sql("ALTER TABLE test_table ADD COLUMN new_column_with_a_default INT DEFAULT 0") } // @TODO: It is currently not possible to CTAS from a table with an active column default // without explicitly enabling the table feature. assertThrows[AnalysisException] { sql("CREATE TABLE test_table_2 USING DELTA AS SELECT * FROM test_table") } // @TODO: It is possible to CTAS from a table with an active column default when the table // feature is explicitly enabled. This copies the default values setting, which is // probably not the desired behaviour. sql("""CREATE TABLE test_table_3 |USING DELTA |TBLPROPERTIES ('delta.feature.allowColumnDefaults' = 'supported') |AS SELECT * FROM test_table""".stripMargin) assert(schemaContainsCurrentDefaultKey("test_table_3")) assert(schemaContainsExistsKey("test_table_3") === false) assert(defaultsTableFeatureEnabled("test_table_3")) // Remove the active column default from the source table and CTAS from it. sql("ALTER TABLE test_table ALTER COLUMN int_col DROP DEFAULT") sql("CREATE TABLE test_table_4 USING DELTA AS SELECT * FROM test_table") assert(schemaContainsCurrentDefaultKey("test_table_4") === false) assert(schemaContainsExistsKey("test_table_4") === false) assert(defaultsTableFeatureEnabled("test_table_4") === false) } } } } trait DeltaTableCreationColumnMappingSuiteBase extends DeltaColumnMappingSelectedTestMixin { override protected def runOnlyTests: Seq[String] = Seq( "create table with schema and path", "create external table without schema", "REPLACE TABLE", "CREATE OR REPLACE TABLE on non-empty directory" ) ++ Seq("partitioned" -> Seq("v2"), "non-partitioned" -> Nil) .flatMap { case (isPartitioned, cols) => SaveMode.values().flatMap { saveMode => Seq( s"saveAsTable to a new table (managed) - $isPartitioned, saveMode: $saveMode", s"saveAsTable to a new table (external) - $isPartitioned, saveMode: $saveMode") } } ++ Seq("a b", "a:b", "a%b").map { specialChars => s"location uri contains $specialChars for datasource table" } } class DeltaTableCreationIdColumnMappingSuite extends DeltaTableCreationSuite with DeltaColumnMappingEnableIdMode { override protected def getTableProperties(tableName: String): Map[String, String] = { // ignore comparing column mapping properties dropColumnMappingConfigurations(super.getTableProperties(tableName)) } } class DeltaTableCreationNameColumnMappingSuite extends DeltaTableCreationSuite with DeltaColumnMappingEnableNameMode { override protected def getTableProperties(tableName: String): Map[String, String] = { // ignore comparing column mapping properties dropColumnMappingConfigurations(super.getTableProperties(tableName)) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaTableFeatureSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import scala.collection.mutable import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils._ import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.coordinatedcommits.{CommitCoordinatorProvider, InMemoryCommitCoordinatorBuilder} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.FileNames.unsafeDeltaFile import org.apache.spark.SparkConf import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.StructType class DeltaTableFeatureSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { private lazy val testTableSchema = spark.range(1).schema override protected def sparkConf: SparkConf = { // All the drop feature tests below are targeting the drop feature with history truncation // implementation. The fast drop feature implementation adds a new writer feature when dropping // a feature and also does not require any waiting time. The fast drop feature implementation // is tested extensively in the DeltaFastDropFeatureSuite. super.sparkConf.set(DeltaSQLConf.FAST_DROP_FEATURE_ENABLED.key, "false") } // This is solely a test hook. Users cannot create new Delta tables with protocol lower than // that of their current version. protected def createTableWithProtocol( protocol: Protocol, path: File, schema: StructType = testTableSchema): DeltaLog = { val log = DeltaLog.forTable(spark, path) log.createLogDirectoriesIfNotExists() log.store.write( unsafeDeltaFile(log.logPath, 0), Iterator(Metadata(schemaString = schema.json).json, protocol.json), overwrite = false, log.newDeltaHadoopConf()) log.update() log } test("all defined table features are registered") { import scala.reflect.runtime.{universe => ru} val subClassNames = mutable.Set[String]() def collect(clazz: ru.Symbol): Unit = { val collected = clazz.asClass.knownDirectSubclasses // add only table feature objects to the result set subClassNames ++= collected.filter(_.isModuleClass).map(_.name.toString) collected.filter(_.isAbstract).foreach(collect) } collect(ru.typeOf[TableFeature].typeSymbol) val registeredFeatures = TableFeature.allSupportedFeaturesMap.values .map(_.getClass.getSimpleName.stripSuffix("$")) // remove '$' from object names .toSet val notRegisteredFeatures = subClassNames.diff(registeredFeatures) assert( notRegisteredFeatures.isEmpty, "Expecting all defined table features are registered (either as prod or testing-only) " + s"but the followings are not: $notRegisteredFeatures") } test("adding feature requires supported protocol version") { assert( intercept[DeltaTableFeatureException] { Protocol(1, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestLegacyReaderWriterFeature) }.getMessage.contains("Unable to enable table feature testLegacyReaderWriter because it " + "requires a higher reader protocol version")) assert(intercept[DeltaTableFeatureException] { Protocol(TABLE_FEATURES_MIN_READER_VERSION, 6) }.getMessage.contains("Unable to upgrade only the reader protocol version")) assert( Protocol(2, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(AppendOnlyTableFeature) .readerAndWriterFeatureNames === Set(AppendOnlyTableFeature.name)) assert( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestReaderWriterFeature) .readerAndWriterFeatureNames === Set(TestReaderWriterFeature.name)) } test("adding feature automatically adds all dependencies") { assert( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestFeatureWithDependency) .readerAndWriterFeatureNames === Set(TestFeatureWithDependency.name, TestReaderWriterFeature.name)) assert( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(TestFeatureWithTransitiveDependency) .readerAndWriterFeatureNames === Set( TestFeatureWithTransitiveDependency.name, TestFeatureWithDependency.name, TestReaderWriterFeature.name)) // Validate new protocol has required features enabled when a writer feature requires a // reader/write feature. val metadata = Metadata( configuration = Map( TableFeatureProtocolUtils.propertyKey(TestWriterFeatureWithTransitiveDependency) -> TableFeatureProtocolUtils.FEATURE_PROP_SUPPORTED)) assert( Protocol .forNewTable( spark, Some(metadata)) .readerAndWriterFeatureNames === Set( AppendOnlyTableFeature.name, InvariantsTableFeature.name, TestWriterFeatureWithTransitiveDependency.name, TestFeatureWithDependency.name, TestReaderWriterFeature.name)) } test("implicitly-enabled features") { assert( Protocol(2, 6).implicitlySupportedFeatures === Set( AppendOnlyTableFeature, ColumnMappingTableFeature, InvariantsTableFeature, CheckConstraintsTableFeature, ChangeDataFeedTableFeature, GeneratedColumnsTableFeature, IdentityColumnsTableFeature, TestLegacyWriterFeature, TestLegacyReaderWriterFeature, TestRemovableLegacyWriterFeature, TestRemovableLegacyReaderWriterFeature)) assert( Protocol(2, 5).implicitlySupportedFeatures === Set( AppendOnlyTableFeature, ColumnMappingTableFeature, InvariantsTableFeature, CheckConstraintsTableFeature, ChangeDataFeedTableFeature, GeneratedColumnsTableFeature, TestLegacyWriterFeature, TestLegacyReaderWriterFeature, TestRemovableLegacyWriterFeature, TestRemovableLegacyReaderWriterFeature)) assert(Protocol(2, TABLE_FEATURES_MIN_WRITER_VERSION).implicitlySupportedFeatures === Set()) assert( Protocol( TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION).implicitlySupportedFeatures === Set()) } test("implicit feature listing") { assert( intercept[DeltaTableFeatureException] { Protocol(1, 4).withFeature(TestLegacyReaderWriterFeature) }.getMessage.contains( "Unable to enable table feature testLegacyReaderWriter because it requires a higher " + "reader protocol version (current 1)")) assert( intercept[DeltaTableFeatureException] { Protocol(2, 4).withFeature(TestLegacyReaderWriterFeature) }.getMessage.contains( "Unable to enable table feature testLegacyReaderWriter because it requires a higher " + "writer protocol version (current 4)")) assert( intercept[DeltaTableFeatureException] { Protocol(1, TABLE_FEATURES_MIN_WRITER_VERSION).withFeature(TestLegacyReaderWriterFeature) }.getMessage.contains( "Unable to enable table feature testLegacyReaderWriter because it requires a higher " + "reader protocol version (current 1)")) val protocol = Protocol(2, TABLE_FEATURES_MIN_WRITER_VERSION).withFeature(TestLegacyReaderWriterFeature) assert(!protocol.readerFeatures.isDefined) assert( protocol.writerFeatures.get === Set(TestLegacyReaderWriterFeature.name)) } test("merge protocols") { val tfProtocol1 = Protocol(1, TABLE_FEATURES_MIN_WRITER_VERSION) val tfProtocol2 = Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) assert(tfProtocol1.merge(Protocol(1, 2)) === Protocol(1, 2)) assert(tfProtocol2.merge(Protocol(2, 6)) === Protocol(2, 6)) } test("protocol upgrade compatibility") { assert(Protocol(1, 1).canUpgradeTo(Protocol(1, 1))) assert(Protocol(1, 1).canUpgradeTo(Protocol(2, 1))) assert( Protocol(1, 1).canUpgradeTo( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION))) assert( !Protocol(2, 3).canUpgradeTo( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION))) assert( !Protocol(2, 6).canUpgradeTo( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures( Seq( // With one feature not referenced, `canUpgradeTo` must be `false`. // AppendOnlyTableFeature, InvariantsTableFeature, CheckConstraintsTableFeature, ChangeDataFeedTableFeature, GeneratedColumnsTableFeature, IdentityColumnsTableFeature, ColumnMappingTableFeature, TestLegacyWriterFeature, TestLegacyReaderWriterFeature, TestRemovableLegacyWriterFeature, TestRemovableLegacyReaderWriterFeature)))) assert( Protocol(2, 6).canUpgradeTo( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, CheckConstraintsTableFeature, ChangeDataFeedTableFeature, GeneratedColumnsTableFeature, IdentityColumnsTableFeature, ColumnMappingTableFeature, TestLegacyWriterFeature, TestLegacyReaderWriterFeature, TestRemovableLegacyWriterFeature, TestRemovableLegacyReaderWriterFeature)))) } test("protocol downgrade compatibility") { val tableFeatureProtocol = Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) assert(Protocol(1, 7).withFeature(TestWriterFeature) .canDowngradeTo(Protocol(1, 7), droppedFeatureName = TestWriterFeature.name)) // When there are no explicit features the protocol versions need to be downgraded // below table features. The new protocol versions need to match exactly the supported // legacy features. for (n <- 1 to 3) { assert( !Protocol(n, 7) .withFeatures(Seq(TestWriterFeature, AppendOnlyTableFeature)) .canDowngradeTo(Protocol(1, 2), droppedFeatureName = TestWriterFeature.name)) assert( Protocol(n, 7) .withFeatures(Seq(TestWriterFeature, AppendOnlyTableFeature, InvariantsTableFeature)) .canDowngradeTo(Protocol(1, 2), droppedFeatureName = TestWriterFeature.name)) } assert(tableFeatureProtocol.withFeatures(Seq(TestReaderWriterFeature)) .canDowngradeTo(Protocol(1, 1), droppedFeatureName = TestReaderWriterFeature.name)) assert( tableFeatureProtocol .withFeatures(Seq(TestReaderWriterFeature, TestRemovableLegacyReaderWriterFeature)) .merge(Protocol(2, 5)) .canDowngradeTo(Protocol(2, 5), droppedFeatureName = TestReaderWriterFeature.name)) // Downgraded protocol must be able to support all legacy table features. assert( !tableFeatureProtocol .withFeatures(Seq(TestWriterFeature, AppendOnlyTableFeature, ColumnMappingTableFeature)) .canDowngradeTo(Protocol(2, 4), droppedFeatureName = TestWriterFeature.name)) assert( tableFeatureProtocol .withFeatures(Seq(TestWriterFeature, AppendOnlyTableFeature, ColumnMappingTableFeature)) .merge(Protocol(2, 5)) .canDowngradeTo(Protocol(2, 5), droppedFeatureName = TestWriterFeature.name)) } test("add reader and writer feature descriptors") { var p = Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) val name = AppendOnlyTableFeature.name p = p.withReaderFeatures(Seq(name)) assert(p.readerFeatures === Some(Set(name))) assert(p.writerFeatures === Some(Set.empty)) p = p.withWriterFeatures(Seq(name)) assert(p.readerFeatures === Some(Set(name))) assert(p.writerFeatures === Some(Set(name))) } test("native automatically-enabled feature can't be implicitly enabled") { val p = Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) assert(p.implicitlySupportedFeatures.isEmpty) } test("Table features are not automatically enabled by default table property settings") { withTable("tbl") { spark.range(10).write.format("delta").saveAsTable("tbl") val snapshot = DeltaLog.forTable(spark, TableIdentifier("tbl")).update() TableFeature.allSupportedFeaturesMap.values.foreach { case feature: FeatureAutomaticallyEnabledByMetadata => assert( !feature.metadataRequiresFeatureToBeEnabled( snapshot.protocol, snapshot.metadata, spark), s""" |${feature.name} is automatically enabled by the default metadata. This will lead to |the inability of reading existing tables that do not have the feature enabled and |should not reach production! If this is only for testing purposes, ignore this test. """.stripMargin) case _ => } } } test("Can enable legacy metadata table feature by setting default table property key") { withSQLConf( s"$DEFAULT_FEATURE_PROP_PREFIX${TestWriterFeature.name}" -> "enabled", DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey -> "name") { withTable("tbl") { spark.range(10).write.format("delta").saveAsTable("tbl") val log = DeltaLog.forTable(spark, TableIdentifier("tbl")) val protocol = log.update().protocol assert(protocol.readerAndWriterFeatureNames === Set( AppendOnlyTableFeature.name, InvariantsTableFeature.name, ColumnMappingTableFeature.name, TestWriterFeature.name)) } } } test("CLONE does not take into account default table features") { withTable("tbl") { spark.range(0).write.format("delta").saveAsTable("tbl") val log = DeltaLog.forTable(spark, TableIdentifier("tbl")) val protocolBefore = log.update().protocol withSQLConf(defaultPropertyKey(TestWriterFeature) -> "enabled") { sql(buildTablePropertyModifyingCommand( commandName = "CLONE", targetTableName = "tbl", sourceTableName = "tbl") ) } val protocolAfter = log.update().protocol assert(protocolBefore === protocolAfter) } } test("CLONE only enables enabled metadata table features") { withTable("src", "target") { withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> TABLE_FEATURES_MIN_WRITER_VERSION.toString, DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> TABLE_FEATURES_MIN_READER_VERSION.toString, DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey -> "name") { spark.range(0).write.format("delta").saveAsTable("src") } sql(buildTablePropertyModifyingCommand( commandName = "CLONE", targetTableName = "target", sourceTableName = "src")) val targetLog = DeltaLog.forTable(spark, TableIdentifier("target")) val protocol = targetLog.update().protocol assert(protocol.readerAndWriterFeatureNames === Set( ColumnMappingTableFeature.name)) } } for(commandName <- Seq("ALTER", "REPLACE", "CREATE OR REPLACE", "CLONE")) { test(s"Can enable legacy metadata table feature during $commandName TABLE") { withSQLConf( s"${defaultPropertyKey(TestWriterFeature)}" -> "enabled") { withTable("tbl") { spark.range(0).write.format("delta").saveAsTable("tbl") val log = DeltaLog.forTable(spark, TableIdentifier("tbl")) val tblProperties = Seq("'delta.enableChangeDataFeed' = true") sql(buildTablePropertyModifyingCommand( commandName, targetTableName = "tbl", sourceTableName = "tbl", tblProperties)) val protocol = log.update().protocol assert(protocol.readerAndWriterFeatureNames === Set( AppendOnlyTableFeature.name, InvariantsTableFeature.name, ChangeDataFeedTableFeature.name, TestWriterFeature.name)) } } } } for(commandName <- Seq("ALTER", "CLONE", "REPLACE", "CREATE OR REPLACE")) { test("Enabling table feature on already existing table enables all table features " + s"up to the table's protocol version during $commandName TABLE") { withSQLConf(DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey -> "name") { withTable("tbl") { spark.range(0).write.format("delta").saveAsTable("tbl") val log = DeltaLog.forTable(spark, TableIdentifier("tbl")) assert(log.update().protocol === Protocol(2, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, ColumnMappingTableFeature))) val tblProperties = Seq(s"'$FEATURE_PROP_PREFIX${TestWriterFeature.name}' = 'enabled'", s"'delta.minWriterVersion' = $TABLE_FEATURES_MIN_WRITER_VERSION") sql(buildTablePropertyModifyingCommand( commandName, targetTableName = "tbl", sourceTableName = "tbl", tblProperties)) val newProtocol = log.update().protocol assert(newProtocol.readerAndWriterFeatureNames === Set( AppendOnlyTableFeature.name, InvariantsTableFeature.name, ColumnMappingTableFeature.name, TestWriterFeature.name)) } } } } for(commandName <- Seq("ALTER", "CLONE", "REPLACE", "CREATE OR REPLACE")) { test(s"Vacuum Protocol Check is disabled by default but can be enabled during $commandName") { val table = "tbl" withTable(table) { spark.range(0).write.format("delta").saveAsTable(table) val log = DeltaLog.forTable(spark, TableIdentifier(table)) val protocol = log.update().protocol assert(!protocol.readerAndWriterFeatureNames.contains(VacuumProtocolCheckTableFeature.name)) val tblProperties1 = Seq(s"'delta.minWriterVersion' = $TABLE_FEATURES_MIN_WRITER_VERSION") sql(buildTablePropertyModifyingCommand( commandName, targetTableName = table, sourceTableName = table, tblProperties1)) val newProtocol1 = log.update().protocol assert(!newProtocol1.readerAndWriterFeatureNames.contains( VacuumProtocolCheckTableFeature.name)) val tblProperties2 = Seq(s"'$FEATURE_PROP_PREFIX${VacuumProtocolCheckTableFeature.name}' " + s"= 'supported', 'delta.minWriterVersion' = $TABLE_FEATURES_MIN_WRITER_VERSION") sql(buildTablePropertyModifyingCommand( commandName, targetTableName = table, sourceTableName = table, tblProperties2)) val newProtocol2 = log.update().protocol assert(newProtocol2.readerAndWriterFeatureNames.contains( VacuumProtocolCheckTableFeature.name)) } } } test("drop table feature works with coordinated commits") { val table = "tbl" withTable(table) { spark.range(0).write.format("delta").saveAsTable(table) val log = DeltaLog.forTable(spark, TableIdentifier(table)) val featureName = TestRemovableReaderWriterFeature.name assert(!log.update().protocol.readerAndWriterFeatureNames.contains(featureName)) // Add coordinated commits table feature to the table CommitCoordinatorProvider.registerBuilder(InMemoryCommitCoordinatorBuilder(batchSize = 100)) val tblProperties1 = Seq(s"'${DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.key}' = 'in-memory'", s"'${DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF.key}' = '{}'") sql(buildTablePropertyModifyingCommand( "ALTER", targetTableName = table, sourceTableName = table, tblProperties1)) // Add TestRemovableReaderWriterFeature to the table in unbackfilled delta files val tblProperties2 = Seq(s"'$FEATURE_PROP_PREFIX$featureName' = 'supported', " + s"'delta.minWriterVersion' = $TABLE_FEATURES_MIN_WRITER_VERSION, " + s"'${TestRemovableReaderWriterFeature.TABLE_PROP_KEY}' = 'true'") sql(buildTablePropertyModifyingCommand( "ALTER", targetTableName = table, sourceTableName = table, tblProperties2)) assert(log.update().protocol.readerAndWriterFeatureNames.contains(featureName)) // Disable feature on the latest snapshot val tblProperties3 = Seq(s"'${TestRemovableReaderWriterFeature.TABLE_PROP_KEY}' = 'false'") sql(buildTablePropertyModifyingCommand( "ALTER", targetTableName = table, sourceTableName = table, tblProperties3)) val tableFeature = TableFeature.featureNameToFeature(featureName).get.asInstanceOf[RemovableFeature] assert(tableFeature.historyContainsFeature( spark, DeltaTableV2(spark, log.dataPath), log.update())) // Dropping feature should fail because the feature still has traces in deltas. val e = intercept[DeltaTableFeatureException] { sql(s"ALTER TABLE $table DROP FEATURE $featureName") } assert(e.getMessage.contains("DELTA_FEATURE_DROP_HISTORICAL_VERSIONS_EXIST"), e) // Add in a checkpoint and cleanUp up older logs containing feature traces log.startTransaction().commitManually() log.checkpoint() log.cleanUpExpiredLogs(log.update(), deltaRetentionMillisOpt = Some(-1000000000000L)) sql(s"ALTER TABLE $table DROP FEATURE $featureName") assert(!log.update().protocol.readerAndWriterFeatureNames.contains(featureName)) } } private def buildTablePropertyModifyingCommand( commandName: String, targetTableName: String, sourceTableName: String, tblProperties: Seq[String] = Seq.empty): String = { val commandStr = if (commandName == "CLONE") { "CREATE OR REPLACE" } else { commandName } val cloneClause = if (commandName == "CLONE") { s"SHALLOW CLONE $sourceTableName" } else { "" } val (usingDeltaClause, dataSourceClause) = if ("ALTER" != commandName && "CLONE" != commandName) { ("USING DELTA", s"AS SELECT * FROM $sourceTableName") } else { ("", "") } var tblPropertiesClause = "" if (tblProperties.nonEmpty) { if (commandName == "ALTER") { tblPropertiesClause += "SET " } tblPropertiesClause += s"TBLPROPERTIES ${tblProperties.mkString("(", ",", ")")}" } s"""$commandStr TABLE $targetTableName |$usingDeltaClause |$cloneClause |$tblPropertiesClause |$dataSourceClause |""".stripMargin } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaTableUtilsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.net.URI import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.hadoop.fs.{Path, RawLocalFileSystem} import org.apache.spark.SparkConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ class DeltaTableUtilsSuite extends SharedSparkSession with DeltaSQLCommandTest { override protected def sparkConf: SparkConf = super.sparkConf .set("spark.hadoop.fs.s3.impl", classOf[MockS3FileSystem].getCanonicalName) test("findDeltaTableRoot correctly combines paths") { val path1 = new Path("s3://my-bucket") assert(DeltaTableUtils.findDeltaTableRoot(spark, path1).isEmpty) val path2 = new Path("s3://my-bucket/") assert(DeltaTableUtils.findDeltaTableRoot(spark, path2).isEmpty) withTempDir { dir => sql(s"CREATE TABLE myTable (id INT) USING DELTA LOCATION '${dir.getAbsolutePath}'") val path = new Path(s"file://${dir.getAbsolutePath}") assert(DeltaTableUtils.findDeltaTableRoot(spark, path).contains(path)) } } test("safeConcatPaths") { val basePath = new Path("s3://my-bucket/subfolder") val basePathEmpty = new Path("s3://my-bucket") assert(DeltaTableUtils.safeConcatPaths(basePath, "_delta_log") == new Path("s3://my-bucket/subfolder/_delta_log")) assert(DeltaTableUtils.safeConcatPaths(basePathEmpty, "_delta_log") == new Path("s3://my-bucket/_delta_log")) assert(DeltaTableUtils.safeConcatPaths(basePath, "_delta/_log") == new Path("s3://my-bucket/subfolder/_delta/_log")) assert(DeltaTableUtils.safeConcatPaths(basePathEmpty, "_delta/_log") == new Path("s3://my-bucket/_delta/_log")) withSQLConf(DeltaSQLConf.DELTA_WORK_AROUND_COLONS_IN_HADOOP_PATHS.key -> "false") { assert(intercept[IllegalArgumentException] { DeltaTableUtils.safeConcatPaths(basePath, "part-2024-03-05T16:08:53.002.csv") }.getMessage.contains("Relative path in absolute URI")) } withSQLConf(DeltaSQLConf.DELTA_WORK_AROUND_COLONS_IN_HADOOP_PATHS.key -> "true") { assert(DeltaTableUtils.safeConcatPaths(basePath, "part-2024-03-05T16:08:53.002.csv") == new Path("s3://my-bucket/subfolder/part-2024-03-05T16:08:53.002.csv")) assert(DeltaTableUtils.safeConcatPaths(basePathEmpty, "part-2024-03-05T16:08:53.002.csv") == new Path("s3://my-bucket/part-2024-03-05T16:08:53.002.csv")) assert(DeltaTableUtils.safeConcatPaths(basePath, "part/2024-03-05T16:08:53.002.csv") == new Path("s3://my-bucket/subfolder/part/2024-03-05T16:08:53.002.csv")) assert(DeltaTableUtils.safeConcatPaths(basePathEmpty, "part/2024-03-05T16:08:53.002.csv") == new Path("s3://my-bucket/part/2024-03-05T16:08:53.002.csv")) } } test("removeInternalWriterMetadata") { for (flag <- BOOLEAN_DOMAIN) { withSQLConf(DeltaSQLConf.DELTA_SCHEMA_REMOVE_SPARK_INTERNAL_METADATA.key -> flag.toString) { for (internalMetadataKey <- DeltaTableUtils.SPARK_INTERNAL_METADATA_KEYS) { val metadata = new MetadataBuilder() .putString(internalMetadataKey, "foo") .putString("other", "bar") .build() val schema = StructType(Seq(StructField("foo", StringType, metadata = metadata))) val newSchema = DeltaTableUtils.removeInternalWriterMetadata(spark, schema) newSchema.foreach { f => if (flag) { // Flag on: should remove internal metadata assert(!f.metadata.contains(internalMetadataKey)) // Should reserve non internal metadata assert(f.metadata.contains("other")) } else { // Flag off: no-op assert(f.metadata == metadata) } } } } } } } private class MockS3FileSystem extends RawLocalFileSystem { override def getScheme: String = "s3" override def getUri: URI = URI.create("s3://my-bucket") } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaTestUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.{BufferedReader, File, InputStreamReader} import java.nio.charset.StandardCharsets.UTF_8 import java.util.{Locale, TimeZone} import java.util.concurrent.{ConcurrentHashMap, TimeUnit} import scala.collection.JavaConverters._ import scala.collection.concurrent import scala.reflect.ClassTag import scala.reflect.runtime.universe._ import scala.util.matching.Regex import com.databricks.spark.util.{Log4jUsageLogger, UsageRecord} import org.apache.spark.sql.delta.DeltaTestUtils.Plans import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.{DeltaCommitFileProvider, FileNames} import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.scala.DefaultScalaModule import io.delta.tables.{DeltaTable => IODeltaTable} import org.apache.hadoop.fs.FileStatus import org.apache.hadoop.fs.Path import org.scalactic.source.Position import org.scalatest.{BeforeAndAfterEach, Tag} import org.apache.spark.{SparkConf, SparkContext, SparkFunSuite, SparkThrowable} import org.apache.spark.scheduler.{JobFailed, SparkListener, SparkListenerJobEnd, SparkListenerJobStart} import org.apache.spark.sql.{AnalysisException, DataFrame, DataFrameWriter, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.util.{quietly, FailFastMode} import org.apache.spark.sql.execution.{FileSourceScanExec, QueryExecution, RDDScanExec, SparkPlan, WholeStageCodegenExec} import org.apache.spark.sql.execution.aggregate.HashAggregateExec import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.StructType import org.apache.spark.sql.util.QueryExecutionListener import org.apache.spark.util.{ManualClock, SystemClock, Utils} object DeltaTestUtilsBase { final val BOOLEAN_DOMAIN: Seq[Boolean] = Seq(true, false) } trait CDCTestMixin extends SharedSparkSession { // Setting the spark Conf is left to the test implementation. def computeCDC( spark: SparkSession, deltaLog: DeltaLog, startVersion: Long, endVersion: Long, predicates: Seq[Expression] = Seq.empty): DataFrame = { CDCReader.changesToBatchDF(deltaLog, startVersion, endVersion, spark) } } trait DeltaTestUtilsBase { import DeltaTestUtils.TableIdentifierOrPath // Re-define here to avoid the need to import it before using final def BOOLEAN_DOMAIN: Seq[Boolean] = DeltaTestUtilsBase.BOOLEAN_DOMAIN class PlanCapturingListener() extends QueryExecutionListener { private[this] var capturedPlans = List.empty[Plans] def plans: Seq[Plans] = capturedPlans.reverse override def onSuccess(funcName: String, qe: QueryExecution, durationNs: Long): Unit = { capturedPlans ::= Plans( qe.analyzed, qe.optimizedPlan, qe.sparkPlan, qe.executedPlan) } override def onFailure( funcName: String, qe: QueryExecution, error: Exception): Unit = {} } /** * Run a thunk with physical plans for all queries captured and passed into a provided buffer. */ def withLogicalPlansCaptured[T]( spark: SparkSession, optimizedPlan: Boolean)( thunk: => Unit): Seq[LogicalPlan] = { val planCapturingListener = new PlanCapturingListener spark.sparkContext.listenerBus.waitUntilEmpty(15000) spark.listenerManager.register(planCapturingListener) try { thunk spark.sparkContext.listenerBus.waitUntilEmpty(15000) planCapturingListener.plans.map { plans => if (optimizedPlan) plans.optimized else plans.analyzed } } finally { spark.listenerManager.unregister(planCapturingListener) } } /** * Run a thunk with physical plans for all queries captured and passed into a provided buffer. */ def withPhysicalPlansCaptured[T]( spark: SparkSession)( thunk: => Unit): Seq[SparkPlan] = { val planCapturingListener = new PlanCapturingListener spark.sparkContext.listenerBus.waitUntilEmpty(15000) spark.listenerManager.register(planCapturingListener) try { thunk spark.sparkContext.listenerBus.waitUntilEmpty(15000) planCapturingListener.plans.map(_.sparkPlan) } finally { spark.listenerManager.unregister(planCapturingListener) } } /** * Run a thunk with logical and physical plans for all queries captured and passed * into a provided buffer. */ def withAllPlansCaptured[T]( spark: SparkSession)( thunk: => Unit): Seq[Plans] = { val planCapturingListener = new PlanCapturingListener spark.sparkContext.listenerBus.waitUntilEmpty(15000) spark.listenerManager.register(planCapturingListener) try { thunk spark.sparkContext.listenerBus.waitUntilEmpty(15000) planCapturingListener.plans } finally { spark.listenerManager.unregister(planCapturingListener) } } def countSparkJobs(sc: SparkContext, f: => Unit): Int = { val jobs: concurrent.Map[Int, Long] = new ConcurrentHashMap[Int, Long]().asScala val listener = new SparkListener { override def onJobStart(jobStart: SparkListenerJobStart): Unit = { jobs.put(jobStart.jobId, jobStart.stageInfos.map(_.numTasks).sum) } override def onJobEnd(jobEnd: SparkListenerJobEnd): Unit = jobEnd.jobResult match { case JobFailed(_) => jobs.remove(jobEnd.jobId) case _ => // On success, do nothing. } } sc.addSparkListener(listener) try { sc.listenerBus.waitUntilEmpty(15000) f sc.listenerBus.waitUntilEmpty(15000) } finally { sc.removeSparkListener(listener) } // Spark will always log a job start/end event even when the job does not launch any task. jobs.values.count(_ > 0) } /** Filter `usageRecords` by the `opType` tag or field. */ def filterUsageRecords(usageRecords: Seq[UsageRecord], opType: String): Seq[UsageRecord] = { usageRecords.filter { r => r.tags.get("opType").contains(opType) || r.opType.map(_.typeName).contains(opType) } } def collectUsageLogs(opType: String)(f: => Unit): collection.Seq[UsageRecord] = { Log4jUsageLogger.track(f).filter { r => r.metric == "tahoeEvent" && r.tags.get("opType").contains(opType) } } /** * Remove protocol and metadata fields from checksum file of json format */ def removeProtocolAndMetadataFromChecksumFile(checksumFilePath : Path): Unit = { // scalastyle:off deltahadoopconfiguration val fs = checksumFilePath.getFileSystem( SparkSession.getActiveSession.map(_.sessionState.newHadoopConf()).get ) // scalastyle:on deltahadoopconfiguration if (!fs.exists(checksumFilePath)) return val stream = fs.open(checksumFilePath) val reader = new BufferedReader(new InputStreamReader(stream, UTF_8)) val content = reader.readLine() stream.close() val mapper = new ObjectMapper() mapper.registerModule(DefaultScalaModule) val map = mapper.readValue(content, classOf[Map[String, String]]) val partialContent = mapper.writeValueAsString(map.-("protocol").-("metadata")) + "\n" val output = fs.create(checksumFilePath, true) output.write(partialContent.getBytes(UTF_8)) output.close() } protected def getfindTouchedFilesJobPlans(plans: Seq[Plans]): SparkPlan = { // The expected plan for touched file computation is of the format below. // The data column should be pruned from both leaves. // HashAggregate(output=[count#3463L]) // +- HashAggregate(output=[count#3466L]) // +- Project // +- Filter (isnotnull(count#3454L) AND (count#3454L > 1)) // +- HashAggregate(output=[count#3454L]) // +- HashAggregate(output=[_row_id_#3418L, sum#3468L]) // +- Project [_row_id_#3418L, UDF(_file_name_#3422) AS one#3448] // +- BroadcastHashJoin [id#3342L], [id#3412L], Inner, BuildLeft // :- Project [id#3342L] // : +- Filter isnotnull(id#3342L) // : +- FileScan parquet [id#3342L,part#3343L] // +- Filter isnotnull(id#3412L) // +- Project [...] // +- Project [...] // +- FileScan parquet [id#3412L,part#3413L] // Note: It can be RDDScanExec instead of FileScan if the source was materialized. // We pick the first plan starting from FileScan and ending in HashAggregate as a // stable heuristic for the one we want. plans.map(_.executedPlan) .filter { case WholeStageCodegenExec(hash: HashAggregateExec) => hash.collectLeaves().size == 2 && hash.collectLeaves() .forall { s => s.isInstanceOf[FileSourceScanExec] || s.isInstanceOf[RDDScanExec] } case _ => false }.head } /** * Separate name- from path-based SQL table identifiers. */ def getTableIdentifierOrPath(sqlIdentifier: String): TableIdentifierOrPath = { // Match: delta.`path`[[ as] alias] or tahoe.`path`[[ as] alias] val pathMatcher: Regex = raw"(?:delta|tahoe)\.`([^`]+)`(?:(?: as)? (.+))?".r // Match: db.table[[ as] alias] val qualifiedDbMatcher: Regex = raw"`?([^\.` ]+)`?\.`?([^\.` ]+)`?(?:(?: as)? (.+))?".r // Match: table[[ as] alias] val unqualifiedNameMatcher: Regex = raw"([^ ]+)(?:(?: as)? (.+))?".r sqlIdentifier match { case pathMatcher(path, alias) => TableIdentifierOrPath.Path(path, Option(alias)) case qualifiedDbMatcher(dbName, tableName, alias) => TableIdentifierOrPath.Identifier(TableIdentifier(tableName, Some(dbName)), Option(alias)) case unqualifiedNameMatcher(tableName, alias) => TableIdentifierOrPath.Identifier(TableIdentifier(tableName), Option(alias)) } } /** * Produce a DeltaTable instance given a `TableIdentifierOrPath` instance. */ def getDeltaTableForIdentifierOrPath( spark: SparkSession, identifierOrPath: TableIdentifierOrPath): IODeltaTable = { identifierOrPath match { case TableIdentifierOrPath.Identifier(id, optionalAlias) => val table = IODeltaTable.forName(spark, id.unquotedString) optionalAlias.map(table.as(_)).getOrElse(table) case TableIdentifierOrPath.Path(path, optionalAlias) => val table = IODeltaTable.forPath(spark, path) optionalAlias.map(table.as(_)).getOrElse(table) } } @deprecated("Use checkError() instead") protected def errorContains(errMsg: String, str: String): Unit = { assert(errMsg.toLowerCase(Locale.ROOT).contains(str.toLowerCase(Locale.ROOT))) } /** * Helper types to define the expected result of a test case. * Either: * - Success: include an expected value to check, e.g. expected schema or result as a DF or rows. * - Failure: an exception is thrown and the caller passes a function to check that it matches an * expected error, typ. `checkError()` or `checkErrorMatchPVals()`. */ sealed trait ExpectedResult[-T] object ExpectedResult { case class Success[T](expected: T) extends ExpectedResult[T] case class Failure[T](checkError: SparkThrowable => Unit = _ => ()) extends ExpectedResult[T] } /** Utility method to check exception `e` is of type `E` or a cause of it is of type `E` */ def findIfResponsible[E <: Throwable: ClassTag](e: Throwable): Option[E] = e match { case culprit: E => Some(culprit) case _ => val children = Option(e.getCause).iterator ++ e.getSuppressed.iterator children .map(findIfResponsible[E](_)) .collectFirst { case Some(culprit) => culprit } } def verifyBackfilled(file: FileStatus): Unit = { val unbackfilled = file.getPath.getName.matches(FileNames.uuidDeltaFileRegex.toString) assert(!unbackfilled, s"File $file was not backfilled") } def verifyUnbackfilled(file: FileStatus): Unit = { val unbackfilled = file.getPath.getName.matches(FileNames.uuidDeltaFileRegex.toString) assert(unbackfilled, s"File $file was backfilled") } } trait DeltaCheckpointTestUtils extends DeltaTestUtilsBase { self: SparkFunSuite with SharedSparkSession => def testDifferentCheckpoints(testName: String, quiet: Boolean = false) (f: (CheckpointPolicy.Policy, Option[V2Checkpoint.Format]) => Unit): Unit = { test(s"$testName [Checkpoint V1]") { def testFunc(): Unit = { withSQLConf(DeltaConfigs.CHECKPOINT_POLICY.defaultTablePropertyKey -> CheckpointPolicy.Classic.name) { f(CheckpointPolicy.Classic, None) } } if (quiet) quietly { testFunc() } else testFunc() } for (checkpointFormat <- V2Checkpoint.Format.ALL) test(s"$testName [Checkpoint V2, format: ${checkpointFormat.name}]") { def testFunc(): Unit = { withSQLConf( DeltaConfigs.CHECKPOINT_POLICY.defaultTablePropertyKey -> CheckpointPolicy.V2.name, DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key -> checkpointFormat.name ) { f(CheckpointPolicy.V2, Some(checkpointFormat)) } } if (quiet) quietly { testFunc() } else testFunc() } } /** * Helper method to get the dataframe corresponding to the files which has the file actions for a * given checkpoint. */ def getCheckpointDfForFilesContainingFileActions( log: DeltaLog, checkpointFile: Path): DataFrame = { val ci = CheckpointInstance.apply(checkpointFile) val allCheckpointFiles = log .listFrom(ci.version) .filter(FileNames.isCheckpointFile) .filter(f => CheckpointInstance(f.getPath) == ci) .toSeq val fileActionsFileIndex = ci.format match { case CheckpointInstance.Format.V2 => val incompleteCheckpointProvider = ci.getCheckpointProvider(log, allCheckpointFiles) val df = log.loadIndex(incompleteCheckpointProvider.topLevelFileIndex.get, Action.logSchema) val sidecarFileStatuses = df.as[SingleAction].collect().map(_.unwrap).collect { case sf: SidecarFile => sf }.map(sf => sf.toFileStatus(log.logPath)) DeltaLogFileIndex(DeltaLogFileIndex.CHECKPOINT_FILE_FORMAT_PARQUET, sidecarFileStatuses) case CheckpointInstance.Format.SINGLE | CheckpointInstance.Format.WITH_PARTS => DeltaLogFileIndex(DeltaLogFileIndex.CHECKPOINT_FILE_FORMAT_PARQUET, allCheckpointFiles.toArray) case _ => throw new Exception(s"Unexpected checkpoint format for file $checkpointFile") } fileActionsFileIndex.files .map(fileStatus => spark.read.parquet(fileStatus.getPath.toString)) .reduce(_.union(_)) } } object DeltaTestUtils extends DeltaTestUtilsBase { sealed trait TableIdentifierOrPath object TableIdentifierOrPath { case class Identifier(id: TableIdentifier, alias: Option[String]) extends TableIdentifierOrPath case class Path(path: String, alias: Option[String]) extends TableIdentifierOrPath } case class Plans( analyzed: LogicalPlan, optimized: LogicalPlan, sparkPlan: SparkPlan, executedPlan: SparkPlan) /** * Creates an AddFile that can be used for tests where the exact parameters do not matter. */ def createTestAddFile( encodedPath: String = "foo", partitionValues: Map[String, String] = Map.empty, size: Long = 1L, modificationTime: Long = 1L, dataChange: Boolean = true, stats: String = "{\"numRecords\": 1}"): AddFile = { AddFile(encodedPath, partitionValues, size, modificationTime, dataChange, stats) } /** * Discovers all DeltaOperations.Operation subclasses using reflection. * Returns a Set of operation class names. * * This is useful for tests that need to ensure exhaustive coverage of all operations. */ def getAllDeltaOperations: Set[String] = { val mirror = runtimeMirror(getClass.getClassLoader) val moduleSymbol = mirror.staticModule("org.apache.spark.sql.delta.DeltaOperations") val moduleMirror = mirror.reflectModule(moduleSymbol) val instance = moduleMirror.instance val instanceMirror = mirror.reflect(instance) val symbol = instanceMirror.symbol val traitOperation = typeOf[org.apache.spark.sql.delta.DeltaOperations.Operation].typeSymbol symbol.typeSignature.members.flatMap { case cls: ClassSymbol if cls.isCaseClass && cls.isPublic && cls.toType.baseClasses.contains(traitOperation) => Some(cls.name.toString) case obj: ModuleSymbol if obj.isPublic && obj.moduleClass.asType.toType.baseClasses.contains(traitOperation) => Some(obj.name.toString) case _ => None }.toSet } /** * Extracts the table name and alias (if any) from the given string. Correctly handles whitespaces * in table name but doesn't support whitespaces in alias. */ def parseTableAndAlias(table: String): (String, Option[String]) = { // Matches 'delta.`path` AS alias' (case insensitive). val deltaPathWithAsAlias = raw"(?i)(delta\.`.+`)(?: AS) (\S+)".r // Matches 'delta.`path` alias'. val deltaPathWithAlias = raw"(delta\.`.+`) (\S+)".r // Matches 'delta.`path`'. val deltaPath = raw"(delta\.`.+`)".r // Matches 'tableName AS alias' (case insensitive). val tableNameWithAsAlias = raw"(?i)(.+)(?: AS) (\S+)".r // Matches 'tableName alias'. val tableNameWithAlias = raw"(.+) (.+)".r table match { case deltaPathWithAsAlias(tableName, alias) => tableName -> Some(alias) case deltaPathWithAlias(tableName, alias) => tableName -> Some(alias) case deltaPath(tableName) => tableName -> None case tableNameWithAsAlias(tableName, alias) => tableName -> Some(alias) case tableNameWithAlias(tableName, alias) => tableName -> Some(alias) case tableName => tableName -> None } } /** * Implements an ordering where `x < y` iff both reader and writer versions of * `x` are strictly less than those of `y`. * * Can be used to conveniently check that this relationship holds in tests/assertions * without having to write out the conjunction of the two subconditions every time. */ case object StrictProtocolOrdering extends PartialOrdering[Protocol] { override def tryCompare(x: Protocol, y: Protocol): Option[Int] = { if (x.minReaderVersion == y.minReaderVersion && x.minWriterVersion == y.minWriterVersion) { Some(0) } else if (x.minReaderVersion < y.minReaderVersion && x.minWriterVersion < y.minWriterVersion) { Some(-1) } else if (x.minReaderVersion > y.minReaderVersion && x.minWriterVersion > y.minWriterVersion) { Some(1) } else { None } } override def lteq(x: Protocol, y: Protocol): Boolean = x.minReaderVersion <= y.minReaderVersion && x.minWriterVersion <= y.minWriterVersion // Just a more readable version of `lteq`. def fulfillsVersionRequirements(actual: Protocol, requirement: Protocol): Boolean = lteq(requirement, actual) } def modifyCommitTimestamp(deltaLog: DeltaLog, version: Long, ts: Long): Unit = { val filePath = DeltaCommitFileProvider(deltaLog.update()).deltaFile(version) val file = new File(filePath.toUri) InCommitTimestampTestUtils.overwriteICTInDeltaFile( deltaLog, new Path(file.getPath), Some(ts)) file.setLastModified(ts) if (FileNames.isUnbackfilledDeltaFile(filePath)) { // Also change the ICT in the backfilled file if it exists. val backfilledFilePath = FileNames.unsafeDeltaFile(deltaLog.logPath, version) val fs = backfilledFilePath.getFileSystem(deltaLog.newDeltaHadoopConf()) if (fs.exists(backfilledFilePath)) { InCommitTimestampTestUtils.overwriteICTInDeltaFile(deltaLog, backfilledFilePath, Some(ts)) } } val crc = new File(FileNames.checksumFile(deltaLog.logPath, version).toUri) if (crc.exists()) { InCommitTimestampTestUtils.overwriteICTInCrc(deltaLog, version, Some(ts)) crc.setLastModified(ts) } } def withTimeZone(zone: String)(f: => Unit): Unit = { val currentDefault = TimeZone.getDefault try { TimeZone.setDefault(TimeZone.getTimeZone(zone)) f } finally { TimeZone.setDefault(currentDefault) } } } trait DeltaTestUtilsForTempViews extends SharedSparkSession with DeltaTestUtilsBase { def testWithTempView(testName: String)(testFun: Boolean => Any): Unit = { Seq(true, false).foreach { isSQLTempView => val tempViewUsed = if (isSQLTempView) "SQL TempView" else "Dataset TempView" test(s"$testName - $tempViewUsed") { withTempView("v") { testFun(isSQLTempView) } } } } def testQuietlyWithTempView(testName: String)(testFun: Boolean => Any): Unit = { Seq(true, false).foreach { isSQLTempView => val tempViewUsed = if (isSQLTempView) "SQL TempView" else "Dataset TempView" testQuietly(s"$testName - $tempViewUsed") { withTempView("v") { testFun(isSQLTempView) } } } } def createTempViewFromTable( tableName: String, isSQLTempView: Boolean, format: Option[String] = None): Unit = { if (isSQLTempView) { sql(s"CREATE OR REPLACE TEMP VIEW v AS SELECT * from $tableName") } else { spark.read.format(format.getOrElse("delta")).table(tableName).createOrReplaceTempView("v") } } def createTempViewFromSelect(text: String, isSQLTempView: Boolean): Unit = { if (isSQLTempView) { sql(s"CREATE OR REPLACE TEMP VIEW v AS $text") } else { sql(text).createOrReplaceTempView("v") } } def testErrorMessageAndClass( isSQLTempView: Boolean, ex: AnalysisException, expectedErrorMsgForSQLTempView: String = null, expectedErrorMsgForDataSetTempView: String = null, expectedErrorClassForSQLTempView: String = null, expectedErrorClassForDataSetTempView: String = null): Unit = { if (isSQLTempView) { if (expectedErrorMsgForSQLTempView != null) { errorContains(ex.getMessage, expectedErrorMsgForSQLTempView) } if (expectedErrorClassForSQLTempView != null) { assert(ex.getErrorClass == expectedErrorClassForSQLTempView) } } else { if (expectedErrorMsgForDataSetTempView != null) { errorContains(ex.getMessage, expectedErrorMsgForDataSetTempView) } if (expectedErrorClassForDataSetTempView != null) { assert(ex.getErrorClass == expectedErrorClassForDataSetTempView, ex.getMessage) } } } } /** * Trait collecting helper methods for DML tests e.p. creating a test table for each test and * cleaning it up after each test. */ trait DeltaDMLTestUtils extends DeltaSQLTestUtils with DeltaTestUtilsBase with BeforeAndAfterEach with CDCTestMixin { self: SharedSparkSession => import testImplicits._ protected def tableSQLIdentifier: String protected def tableIdentifier: TableIdentifier protected def dropTable(): Unit /** * Clock used for [[deltaLog]]. [[SystemClock]] is used if not set via [[setupManualClock]]. */ protected var clock: ManualClock = _ protected def setupManualClock(): Unit = { clock = new ManualClock(System.currentTimeMillis()) // Override the (cached) delta log with one using our manual clock. DeltaLog.clearCache() deltaLog } /** * Use this to artificially move the current time to after the table retention period. */ protected def advancePastRetentionPeriod(): Unit = { assert(clock != null, "Must call setupManualClock in tests that are using this method.") clock.advance( deltaLog.deltaRetentionMillis(deltaLog.update().metadata) + TimeUnit.DAYS.toMillis(3)) } // No need to cache deltaLog here as it is already cached protected def deltaLog: DeltaLog = { if (clock != null) { DeltaLog.forTable(spark, tableIdentifier, clock) } else { DeltaLog.forTable(spark, tableIdentifier) } } override protected def afterEach(): Unit = { try { dropTable() } finally { super.afterEach() } } protected def append(df: DataFrame, partitionBy: Seq[String] = Nil): Unit = { val dfw = df.write.format("delta").mode("append") if (partitionBy.nonEmpty) { dfw.partitionBy(partitionBy: _*) } writeTable(dfw, tableSQLIdentifier) } protected def withKeyValueData( source: Seq[(Int, Int)], target: Seq[(Int, Int)], isKeyPartitioned: Boolean = false, sourceKeyValueNames: (String, String) = ("key", "value"), targetKeyValueNames: (String, String) = ("key", "value"))( thunk: (String, String) => Unit = null): Unit = { import testImplicits._ append(target.toDF(targetKeyValueNames._1, targetKeyValueNames._2).coalesce(2), if (isKeyPartitioned) Seq(targetKeyValueNames._1) else Nil) withTempView("source") { source.toDF(sourceKeyValueNames._1, sourceKeyValueNames._2).createOrReplaceTempView("source") thunk("source", tableSQLIdentifier) } } /** * Parse the input JSON data into a dataframe, one row per input element. * Throws an exception on malformed inputs or records that don't comply with the provided schema. */ protected def readFromJSON(data: Seq[String], schema: StructType = null): DataFrame = { if (schema != null) { spark.read .schema(schema) .option("mode", FailFastMode.name) .json(data.toDS) } else { spark.read .option("mode", FailFastMode.name) .json(data.toDS) } } /** * Reads a delta table by its identifier. The identifier can either be the table name or table * path that is in the form of delta.`tablePath`. */ protected def readDeltaTableByIdentifier( tableIdentifier: String = tableSQLIdentifier): DataFrame = { spark.read.format("delta").table(tableIdentifier) } protected def writeTable[T](dfw: DataFrameWriter[T], tableName: String): Unit = { import DeltaTestUtils.TableIdentifierOrPath getTableIdentifierOrPath(tableName) match { case TableIdentifierOrPath.Identifier(id, _) => dfw.saveAsTable(id.toString) // A cleaner way to write this is to just use `saveAsTable` where the // table name is delta.`path`. However, it will throw an error when // we use "append" mode and the table does not exist, so we use `save` // here instead. case TableIdentifierOrPath.Path(path, _) => dfw.save(path) } } /** * Finds the latest operation of the given type that ran on the test table and returns the * dataframe with the changes of the corresponding table version. * * @param operation Delta operation name, see [[DeltaOperations]]. */ protected def getCDCForLatestOperation(deltaLog: DeltaLog, operation: String): DataFrame = { val latestOperation = deltaLog.history .getHistory(None) .find(_.operation == operation) assert(latestOperation.nonEmpty, s"Couldn't find a ${operation} operation to check CDF") val latestOperationVersion = latestOperation.get.version assert(latestOperationVersion.nonEmpty, s"Latest ${operation} operation doesn't have a version associated with it") computeCDC( spark, deltaLog, latestOperationVersion.get, latestOperationVersion.get ) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) .drop(CDCReader.CDC_COMMIT_VERSION) } } trait DeltaDMLTestUtilsPathBased extends DeltaDMLTestUtils { self: SharedSparkSession => protected var tempDir: File = _ protected def tempPath: String = tempDir.getCanonicalPath override protected def tableIdentifier: TableIdentifier = TableIdentifier(tempPath, Some("delta")) override protected def beforeEach(): Unit = { super.beforeEach() // Using a space in path to provide coverage for special characters. tempDir = Utils.createTempDir(namePrefix = "spark test") } override protected def tableSQLIdentifier: String = s"delta.`$tempPath`" protected def readDeltaTable(path: String): DataFrame = { spark.read.format("delta").load(path) } override protected def dropTable(): Unit = { Utils.deleteRecursively(tempDir) DeltaLog.clearCache() } } /** * Represents a test that is incompatible with name-based table access */ case object NameBasedAccessIncompatible extends Tag("NameBasedAccessIncompatible") trait DeltaDMLTestUtilsNameBased extends DeltaDMLTestUtils { self: SharedSparkSession => override protected def test(testName: String, testTags: Tag*)(testFun: => Any)( implicit pos: Position): Unit = { if (testTags.contains(NameBasedAccessIncompatible)) { super.ignore(testName, testTags: _*)(testFun) } else { super.test(testName, testTags: _*)(testFun) } } override protected def tableIdentifier: TableIdentifier = TableIdentifier(tableSQLIdentifier) override protected def append(df: DataFrame, partitionBy: Seq[String] = Nil): Unit = { super.append(df, partitionBy) } // Keep this all lowercase. Otherwise, for tests with spark.sql.caseSensitive set to // true, the table name used for dropping the table will not match the created table // name, causing the table not being dropped. override protected def tableSQLIdentifier: String = "test_delta_table" override protected def dropTable(): Unit = { spark.sql(s"DROP TABLE IF EXISTS $tableSQLIdentifier") DeltaLog.clearCache() } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaThrowableSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.nio.charset.StandardCharsets import java.nio.file.Files import scala.collection.immutable.SortedMap import org.apache.spark.sql.delta.DeltaThrowableHelper.{deltaErrorClassSource, sparkErrorClassSource} import com.fasterxml.jackson.annotation.JsonInclude.Include import com.fasterxml.jackson.core.JsonParser.Feature.STRICT_DUPLICATE_DETECTION import com.fasterxml.jackson.core.`type`.TypeReference import com.fasterxml.jackson.core.util.{DefaultIndenter, DefaultPrettyPrinter} import com.fasterxml.jackson.databind.SerializationFeature import com.fasterxml.jackson.databind.json.JsonMapper import com.fasterxml.jackson.module.scala.DefaultScalaModule import org.apache.commons.io.{FileUtils, IOUtils} import org.apache.spark.{ErrorClassesJsonReader, ErrorInfo, SparkFunSuite} /** Test suite for Delta Throwables. */ class DeltaThrowableSuite extends SparkFunSuite { private lazy val sparkErrorClassesMap = { new ErrorClassesJsonReader(Seq(sparkErrorClassSource)).errorInfoMap } private lazy val deltaErrorClassToInfoMap = { new ErrorClassesJsonReader(Seq(deltaErrorClassSource)).errorInfoMap } /* Used to regenerate the error class file. Run: {{{ SPARK_GENERATE_GOLDEN_FILES=1 build/sbt \ "sql/testOnly *DeltaThrowableSuite -- -t \"Error classes are correctly formatted\"" }}} */ def checkIfUnique(ss: Seq[Any]): Unit = { val duplicatedKeys = ss.groupBy(identity).mapValues(_.size).filter(_._2 > 1).keys.toSeq assert(duplicatedKeys.isEmpty) } def checkCondition(ss: Seq[String], fx: String => Boolean): Unit = { ss.foreach { s => assert(fx(s)) } } test("No duplicate error classes in Delta") { // Enabling this feature incurs performance overhead (20-30%) val mapper = JsonMapper.builder() .addModule(DefaultScalaModule) .enable(STRICT_DUPLICATE_DETECTION) .build() mapper.readValue(deltaErrorClassSource, new TypeReference[Map[String, ErrorInfo]]() {}) } test("No error classes are shared by Delta and Spark") { assert(deltaErrorClassToInfoMap.keySet.intersect(sparkErrorClassesMap.keySet).isEmpty) } test("No word 'databricks' in OSS Delta errors") { val errorClasses = deltaErrorClassToInfoMap.keys.toSeq val errorMsgs = deltaErrorClassToInfoMap.values.toSeq.flatMap(_.message) checkCondition(errorClasses ++ errorMsgs, s => !s.toLowerCase().contains("databricks")) } test("Delta error classes are correctly formatted with keys in alphabetical order") { lazy val ossDeltaErrorFile = new File(getWorkspaceFilePath( "delta", "core", "src", "main", "resources", "error").toFile, "delta-error-classes.json") val errorClassFileContents = { IOUtils.toString(deltaErrorClassSource.openStream()) } val mapper = JsonMapper.builder() .addModule(DefaultScalaModule) .enable(SerializationFeature.INDENT_OUTPUT) .build() val prettyPrinter = new DefaultPrettyPrinter() .withArrayIndenter(DefaultIndenter.SYSTEM_LINEFEED_INSTANCE) val rewrittenString = { val writer = mapper.configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true) .setSerializationInclusion(Include.NON_ABSENT) .writer(prettyPrinter) writer.writeValueAsString(deltaErrorClassToInfoMap) } if (regenerateGoldenFiles) { if (rewrittenString.trim != errorClassFileContents.trim) { logInfo(s"Regenerating error class file $ossDeltaErrorFile") Files.delete(ossDeltaErrorFile.toPath) FileUtils.writeStringToFile(ossDeltaErrorFile, rewrittenString, StandardCharsets.UTF_8) } } else { assert(rewrittenString.trim == errorClassFileContents.trim) } } test("Delta message format invariants") { val messageFormats = deltaErrorClassToInfoMap.values.toSeq.flatMap { i => i.subClass match { // Has sub error class: the message template should be: base + sub case Some(subs) => subs.values.toSeq.map(sub => s"${i.messageTemplate} ${sub.messageTemplate}") // Does not have any sub error class: the message template is itself case None => Seq(i.messageTemplate) } } checkCondition(messageFormats, s => s != null) checkIfUnique(messageFormats) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaTimeTravelSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.nio.charset.StandardCharsets.UTF_8 import java.sql.Timestamp import java.text.SimpleDateFormat import java.util.Date import scala.concurrent.duration._ import scala.language.implicitConversions import org.apache.spark.sql.delta.DeltaHistoryManager.BufferingLogDeletionIterator import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile import org.apache.spark.sql.delta.actions.{Action, CommitInfo, SingleAction} import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.{DateTimeUtils, DeltaCommitFileProvider, FileNames, JsonUtils, TimestampFormatter} import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.sql.{functions, AnalysisException, QueryTest, Row} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.ManualClock class DeltaTimeTravelSuite extends QueryTest with SharedSparkSession with DeltaSQLTestUtils with DeltaSQLCommandTest with CatalogOwnedTestBaseSuite { import testImplicits._ private val timeFormatter = new SimpleDateFormat("yyyyMMddHHmmssSSS") private implicit def durationToLong(duration: FiniteDuration): Long = { duration.toMillis } private implicit def longToTimestamp(ts: Long): Timestamp = new Timestamp(ts) private def modifyCommitTimestamp(deltaLog: DeltaLog, version: Long, ts: Long): Unit = { val file = new File(DeltaCommitFileProvider(deltaLog.update()).deltaFile(version).toUri) file.setLastModified(ts) val crc = new File(FileNames.checksumFile(deltaLog.logPath, version).toUri) if (crc.exists()) { crc.setLastModified(ts) } } private def modifyCheckpointTimestamp(deltaLog: DeltaLog, version: Long, ts: Long): Unit = { val file = new File(FileNames.checkpointFileSingular(deltaLog.logPath, version).toUri) file.setLastModified(ts) } /** Generate commits with the given timestamp in millis. */ private def generateCommitsCheap(deltaLog: DeltaLog, clock: ManualClock, commits: Long*): Unit = { var startVersion = deltaLog.unsafeVolatileSnapshot.version + 1 commits.foreach { ts => val action = createTestAddFile(encodedPath = startVersion.toString, modificationTime = startVersion) clock.setTime(ts) deltaLog.startTransaction().commitManually(action) modifyCommitTimestamp(deltaLog, startVersion, ts) startVersion += 1 } } /** Generate commits with the given timestamp in millis. */ private def generateCommits(location: String, commits: Long*): Unit = { var deltaLog = DeltaLog.forTable(spark, dataPath = location) var startVersion = deltaLog.unsafeVolatileSnapshot.version + 1 commits.foreach { ts => val rangeStart = startVersion * 10 val rangeEnd = rangeStart + 10 spark.range(rangeStart, rangeEnd).write.format("delta").mode("append").save(location) // Construct a new delta log here before calling `DeltaCommitFileProvider` to get the commit // file path. This is b/c [[Snapshot.logSegment.deltas]] will *not* be automatically updated // after triggering backfill. // We will then overwrite the commit timestamp for unbackilled commits even if we have already // backfilled them. This leads to the failure in certain UT where we manually modify/overwrite // the commit timestamps. // To correctly update the deltas in [[LogSegment]], we construct a fresh delta log. DeltaLog.clearCache() deltaLog = DeltaLog.forTable(spark, dataPath = location) val filePath = DeltaCommitFileProvider .apply(snapshot = deltaLog.unsafeVolatileSnapshot) .deltaFile(version = startVersion) if (isICTEnabledForNewTablesCatalogOwned) { InCommitTimestampTestUtils.overwriteICTInDeltaFile(deltaLog, filePath, Some(ts)) InCommitTimestampTestUtils.overwriteICTInCrc(deltaLog, startVersion, Some(ts)) } else { val file = new File(filePath.toUri) file.setLastModified(ts) } startVersion += 1 } } private def identifierWithTimestamp(identifier: String, ts: Long): String = { s"$identifier@${timeFormatter.format(new Date(ts))}" } private def identifierWithVersion(identifier: String, v: Long): String = { s"$identifier@v$v" } private implicit def longToTimestampExpr(value: Long): String = { s"cast($value / 1000 as timestamp)" } private def getSparkFormattedTimestamps(values: Long*): Seq[String] = { // Simulates getting timestamps directly from Spark SQL values.map(new Timestamp(_)).toDF("ts") .select($"ts".cast("string")).as[String].collect() .map(i => s"$i") } private def historyTest(testName: String)(f: (DeltaLog, ManualClock) => Unit): Unit = { testQuietly(testName) { val clock = new ManualClock() withTempDir { dir => f(DeltaLog.forTable(spark, dir, clock), clock) } } } historyTest("getCommits should monotonize timestamps") { (deltaLog, clock) => if (catalogOwnedDefaultCreationEnabledInTests) { // This is fine for CC tables since ICT should've been enabled from the beginning. // Hence, we should *never* call [[DeltaHistoryManager.getCommitsWithNonIctTimestamps]]. cancel("This test is not compatible with CC since ICT should've been enabled from the " + "beginning for CC tables.") } val start = 1540415658000L // Make the commits out of order generateCommitsCheap(deltaLog, clock, start, start - 5.seconds, // adjusts to start + 1 ms start + 1.milli, // adjusts to start + 2 ms start + 2.millis, // adjusts to start + 3 ms start - 2.seconds, // adjusts to start + 4 ms start + 10.seconds) val commits = DeltaHistoryManager.getCommitsWithNonIctTimestamps( deltaLog.store, deltaLog.logPath, 0, None, deltaLog.newDeltaHadoopConf()) // Note that when InCommitTimestamps are enabled, the monotization of timestamps is not // performed by getCommits. Instead, the timestamps are already monotonized before they // are written in the commit. assert(commits.map(_.timestamp) === Seq(start, start + 1.millis, start + 2.millis, start + 3.millis, start + 4.millis, start + 10.seconds)) } historyTest("describe history timestamps are adjusted according to file timestamp") { (deltaLog, clock) => if (isICTEnabledForNewTablesCatalogOwned) { // File timestamp adjustment is not needed when ICT is enabled. cancel("This test is not compatible with InCommitTimestamps.") } // this is in '2018-10-24', so earlier than today. The recorded timestamps in commitInfo will // be much after this val start = 1540415658000L // Make the commits out of order generateCommitsCheap(deltaLog, clock, start, start - 5.seconds, // adjusts to start + 1 ms start + 1.milli // adjusts to start + 2 ms ) val history = new DeltaHistoryManager(deltaLog) val commits = history.getHistory(None) assert(commits.map(_.timestamp.getTime) === Seq(start + 2.millis, start + 1.milli, start)) } historyTest("should filter only delta files when computing earliest version") { (deltaLog, clock) => val start = 1540415658000L clock.setTime(start) generateCommitsCheap(deltaLog, clock, start, start + 10.seconds, start + 20.seconds) val history = new DeltaHistoryManager(deltaLog) assert(history.getActiveCommitAtTime(start + 15.seconds, false).version === 1) val commits2 = history.getHistory(Some(10)) assert(commits2.last.version === Some(0)) assert(new File(FileNames.unsafeDeltaFile(deltaLog.logPath, 0L).toUri).delete()) val e = intercept[AnalysisException] { history.getActiveCommitAtTime(start + 15.seconds, false).version } if (catalogOwnedDefaultCreationEnabledInTests && // Since we are creating a table w/ three initial commits, the table would have // unbackfilled commits if the backfill batch size is greater or equal to three. // See [[generateCommitsCheap]] for details. catalogOwnedCoordinatorBackfillBatchSize.exists(_ >= 3)) { // We throw an "incorrect" exception for CC tables if there exist any unbackfilled commits // and the backfilled commits have been manually deleted. E.g., the 0.json we are deleting // in this UT. // // Please see the comment in [[DeltaHistoryManager.getEarliestRecreatableCommit]] for the // detailed rationale. assert(e.getMessage.contains("[DELTA_NO_COMMITS_FOUND]")) } else { assert(e.getMessage.contains("recreatable")) } } historyTest("resolving commits should return commit before timestamp") { (deltaLog, clock) => val start = 1540415658000L clock.setTime(start) // Make a commit every 20 minutes val commits = Seq.tabulate(10)(i => start + (i * 20).minutes) generateCommitsCheap(deltaLog, clock, commits: _*) // When maxKeys is 2, we will use the parallel search algorithm, when it is 1000, we will // use the linear search method Seq(1, 2, 1000).foreach { maxKeys => val history = new DeltaHistoryManager(deltaLog, maxKeys) (0 until 10).foreach { i => assert(history.getActiveCommitAtTime(start + (i * 20 + 10).minutes, true).version === i) } val e = intercept[DeltaErrors.TemporallyUnstableInputException] { // This is 20 minutes after the last commit history.getActiveCommitAtTime(start + 200.minutes, false) } checkError( e, "DELTA_TIMESTAMP_GREATER_THAN_COMMIT", sqlState = "42816", parameters = Map( "providedTimestamp" -> "2018-10-24 17:34:18.0", "tableName" -> "2018-10-24 17:14:18.0", "maximumTimestamp" -> "2018-10-24 17:14:18") ) assert(history.getActiveCommitAtTime(start + 180.minutes, true).version === 9) val e2 = intercept[AnalysisException] { history.getActiveCommitAtTime(start - 10.minutes, true) } assert(e2.getMessage.contains("before the earliest version")) } } /** * Creates FileStatus objects, where the name is the version of a commit, and the modification * timestamps come from the input. */ private def createFileStatuses(modTimes: Long*): Iterator[FileStatus] = { modTimes.zipWithIndex.map { case (time, version) => new FileStatus( 10L, false, 1, 10L, time, FileNames.checkpointFileSingular(new Path("/foo"), version)) }.iterator } /** * Creates a log deletion iterator with a retention `maxTimestamp` and `maxVersion` (both * inclusive). The input iterator takes the original file timestamps, and the deleted output will * return the adjusted timestamps of files that would actually be consumed by the iterator. */ private def testBufferingLogDeletionIterator( maxTimestamp: Long, maxVersion: Long)(inputTimestamps: Seq[Long], deleted: Seq[Long]): Unit = { val i = new BufferingLogDeletionIterator(createFileStatuses(inputTimestamps: _*), maxTimestamp, maxVersion, FileNames.getFileVersion) deleted.foreach { ts => assert(i.hasNext, s"Was supposed to delete $ts, but iterator returned hasNext: false") assert(i.next().getModificationTime === ts, "Returned files out of order!") } assert(!i.hasNext, "Iterator should be consumed") } test("BufferingLogDeletionIterator: iterator behavior") { val i1 = new BufferingLogDeletionIterator(Iterator.empty, 100, 100, _ => 1) intercept[NoSuchElementException](i1.next()) assert(!i1.hasNext) testBufferingLogDeletionIterator(maxTimestamp = 100, maxVersion = 100)( inputTimestamps = Seq(10, 11), deleted = Seq(10) ) testBufferingLogDeletionIterator(maxTimestamp = 100, maxVersion = 100)( inputTimestamps = Seq(10, 15, 25, 26), deleted = Seq(10, 15, 25) ) } test("BufferingLogDeletionIterator: " + "early exit while handling adjusted timestamps due to timestamp") { // only should return 5 because 5 < 7 testBufferingLogDeletionIterator(maxTimestamp = 7, maxVersion = 100)( inputTimestamps = Seq(5, 10, 8, 12), deleted = Seq(5) ) // Should only return 5, because 10 is used to adjust the following 8 to 11 testBufferingLogDeletionIterator(maxTimestamp = 10, maxVersion = 100)( inputTimestamps = Seq(5, 10, 8, 12), deleted = Seq(5) ) // When it is 11, we can delete both 10 and 8 testBufferingLogDeletionIterator(maxTimestamp = 11, maxVersion = 100)( inputTimestamps = Seq(5, 10, 8, 12), deleted = Seq(5, 10, 11) ) // When it is 12, we can return all, except last one testBufferingLogDeletionIterator(maxTimestamp = 12, maxVersion = 100)( inputTimestamps = Seq(5, 10, 8, 12, 13), deleted = Seq(5, 10, 11, 12) ) // Should only return 5, because 10 is used to adjust the following 8 to 11 testBufferingLogDeletionIterator(maxTimestamp = 10, maxVersion = 100)( inputTimestamps = Seq(5, 10, 8), deleted = Seq(5) ) // When it is 11, we can delete both 10 and 8 testBufferingLogDeletionIterator(maxTimestamp = 11, maxVersion = 100)( inputTimestamps = Seq(5, 10, 8, 12), deleted = Seq(5, 10, 11) ) } test("BufferingLogDeletionIterator: " + "early exit while handling adjusted timestamps due to version") { // only should return 5 because we can delete only up to version 0 testBufferingLogDeletionIterator(maxTimestamp = 100, maxVersion = 0)( inputTimestamps = Seq(5, 10, 8, 12), deleted = Seq(5) ) // Should only return 5, because 10 is used to adjust the following 8 to 11 testBufferingLogDeletionIterator(maxTimestamp = 100, maxVersion = 1)( inputTimestamps = Seq(5, 10, 8, 12), deleted = Seq(5) ) // When we can delete up to version 2, we can return up to version 2 testBufferingLogDeletionIterator(maxTimestamp = 100, maxVersion = 2)( inputTimestamps = Seq(5, 10, 8, 12), deleted = Seq(5, 10, 11) ) // When it is version 3, we can return all, except last one testBufferingLogDeletionIterator(maxTimestamp = 100, maxVersion = 3)( inputTimestamps = Seq(5, 10, 8, 12, 13), deleted = Seq(5, 10, 11, 12) ) // Should only return 5, because 10 is used to adjust the following 8 to 11 testBufferingLogDeletionIterator(maxTimestamp = 100, maxVersion = 1)( inputTimestamps = Seq(5, 10, 8), deleted = Seq(5) ) // When we can delete up to version 2, we can return up to version 2 testBufferingLogDeletionIterator(maxTimestamp = 100, maxVersion = 2)( inputTimestamps = Seq(5, 10, 8, 12), deleted = Seq(5, 10, 11) ) } test("BufferingLogDeletionIterator: multiple adjusted timestamps") { Seq(9, 10, 11).foreach { retentionTimestamp => // Files should be buffered but not deleted, because of the file 11, which has adjusted ts 12 testBufferingLogDeletionIterator(maxTimestamp = retentionTimestamp, maxVersion = 100)( inputTimestamps = Seq(5, 10, 8, 11, 14), deleted = Seq(5) ) } // Safe to delete everything before (including) file: 11 which has adjusted timestamp 12 testBufferingLogDeletionIterator(maxTimestamp = 12, maxVersion = 100)( inputTimestamps = Seq(5, 10, 8, 11, 14), deleted = Seq(5, 10, 11, 12) ) Seq(0, 1, 2).foreach { retentionVersion => testBufferingLogDeletionIterator(maxTimestamp = 100, maxVersion = retentionVersion)( inputTimestamps = Seq(5, 10, 8, 11, 14), deleted = Seq(5) ) } testBufferingLogDeletionIterator(maxTimestamp = 100, maxVersion = 3)( inputTimestamps = Seq(5, 10, 8, 11, 14), deleted = Seq(5, 10, 11, 12) ) // Test when the last element is adjusted with both timestamp and version Seq(9, 10, 11).foreach { retentionTimestamp => testBufferingLogDeletionIterator(maxTimestamp = retentionTimestamp, maxVersion = 100)( inputTimestamps = Seq(5, 10, 8, 9), deleted = Seq(5) ) } testBufferingLogDeletionIterator(maxTimestamp = 12, maxVersion = 100)( inputTimestamps = Seq(5, 10, 8, 9, 13), deleted = Seq(5, 10, 11, 12) ) Seq(0, 1, 2).foreach { retentionVersion => testBufferingLogDeletionIterator(maxTimestamp = 100, maxVersion = retentionVersion)( inputTimestamps = Seq(5, 10, 8, 9), deleted = Seq(5) ) } testBufferingLogDeletionIterator(maxTimestamp = 100, maxVersion = 3)( inputTimestamps = Seq(5, 10, 8, 9, 13), deleted = Seq(5, 10, 11, 12) ) Seq(9, 10, 11).foreach { retentionTimestamp => testBufferingLogDeletionIterator(maxTimestamp = retentionTimestamp, maxVersion = 100)( inputTimestamps = Seq(10, 8, 9), deleted = Nil ) } // Test the first element causing cascading adjustments testBufferingLogDeletionIterator(maxTimestamp = 12, maxVersion = 100)( inputTimestamps = Seq(10, 8, 9, 13), deleted = Seq(10, 11, 12) ) Seq(0, 1).foreach { retentionVersion => testBufferingLogDeletionIterator(maxTimestamp = 100, maxVersion = retentionVersion)( inputTimestamps = Seq(10, 8, 9), deleted = Nil ) } testBufferingLogDeletionIterator(maxTimestamp = 100, maxVersion = 2)( inputTimestamps = Seq(10, 8, 9, 13), deleted = Seq(10, 11, 12) ) // Test multiple batches of time adjustments testBufferingLogDeletionIterator(maxTimestamp = 12, maxVersion = 100)( inputTimestamps = Seq(5, 10, 8, 9, 12, 15, 14, 14), // 5, 10, 11, 12, 13, 15, 16, 17 deleted = Seq(5) ) Seq(13, 14, 15, 16).foreach { retentionTimestamp => testBufferingLogDeletionIterator(maxTimestamp = retentionTimestamp, maxVersion = 100)( inputTimestamps = Seq(5, 10, 8, 9, 12, 15, 14, 14), // 5, 10, 11, 12, 13, 15, 16, 17 deleted = Seq(5, 10, 11, 12, 13) ) } testBufferingLogDeletionIterator(maxTimestamp = 17, maxVersion = 100)( inputTimestamps = Seq(5, 10, 8, 9, 12, 15, 14, 14, 18), // 5, 10, 11, 12, 13, 15, 16, 17, 18 deleted = Seq(5, 10, 11, 12, 13, 15, 16, 17) ) } test("[SPARK-45383] Time travel on a non-existing table should throw AnalysisException") { intercept[AnalysisException] { spark.sql("SELECT * FROM not_existing VERSION AS OF 0") } } test("as of timestamp in between commits should use commit before timestamp") { withTempDir { dir => val tblLoc = dir.getCanonicalPath val start = System.currentTimeMillis() - 5.days.toMillis generateCommits(tblLoc, start, start + 20.minutes, start + 40.minutes) val tablePathUri = identifierWithTimestamp(tblLoc, start + 10.minutes) val df1 = spark.read.format("delta").load(tablePathUri) checkAnswer(df1.groupBy().count(), Row(10L)) // 2 minutes after start val timeTwoMinutesAfterStart = new Timestamp(start + 2.minutes) val df2 = spark.read.format("delta") .option("timestampAsOf", timeTwoMinutesAfterStart.toString).load(tblLoc) checkAnswer(df2.groupBy().count(), Row(10L)) } } test("as of timestamp on exact timestamp") { withTempDir { dir => val tblLoc = dir.getCanonicalPath val start = System.currentTimeMillis() - 5.days.toMillis generateCommits(tblLoc, start, start + 20.minutes) // Simulate getting the timestamp directly from Spark SQL val ts = getSparkFormattedTimestamps(start, start + 20.minutes) checkAnswer( spark.read.format("delta").option("timestampAsOf", ts.head).load(tblLoc).groupBy().count(), Row(10L) ) checkAnswer( spark.read.format("delta").option("timestampAsOf", ts(1)).load(tblLoc).groupBy().count(), Row(20L) ) checkAnswer( spark.read.format("delta").load(identifierWithTimestamp(tblLoc, start)).groupBy().count(), Row(10L) ) checkAnswer( spark.read.format("delta").load(identifierWithTimestamp(tblLoc, start + 20.minutes)) .groupBy().count(), Row(20L) ) } } test("as of timestamp on invalid timestamp") { withTempDir { dir => val tblLoc = dir.getCanonicalPath val start = 1540415658000L generateCommits(tblLoc, start, start + 20.minutes) val ex = intercept[AnalysisException] { spark.read.format("delta").option("timestampAsOf", "i am not a timestamp") .load(tblLoc).groupBy().count() } assert(ex.getMessage.contains( "The provided timestamp ('i am not a timestamp') cannot be converted to a valid timestamp")) } } test("as of exact timestamp after last commit should fail") { withTempDir { dir => val tblLoc = dir.getCanonicalPath val start = 1540415658000L generateCommits(tblLoc, start) // Simulate getting the timestamp directly from Spark SQL val ts = getSparkFormattedTimestamps(start + 10.minutes) val e1 = intercept[DeltaErrors.TemporallyUnstableInputException] { spark.read.format("delta").option("timestampAsOf", ts.head).load(tblLoc).collect() } checkError( e1, "DELTA_TIMESTAMP_GREATER_THAN_COMMIT", sqlState = "42816", parameters = Map( "providedTimestamp" -> "2018-10-24 14:24:18.0", "tableName" -> "2018-10-24 14:14:18.0", "maximumTimestamp" -> "2018-10-24 14:14:18") ) val e2 = intercept[DeltaErrors.TemporallyUnstableInputException] { spark.read.format("delta").load(identifierWithTimestamp(tblLoc, start + 10.minutes)) .collect() } checkError( e2, "DELTA_TIMESTAMP_GREATER_THAN_COMMIT", sqlState = "42816", parameters = Map( "providedTimestamp" -> "2018-10-24 14:24:18.0", "tableName" -> "2018-10-24 14:14:18.0", "maximumTimestamp" -> "2018-10-24 14:14:18") ) checkAnswer( spark.read.format("delta").option("timestampAsOf", "2018-10-24 14:14:18") .load(tblLoc).groupBy().count(), Row(10) ) } } test("as of with versions") { withTempDir { dir => val tblLoc = dir.getCanonicalPath val start = System.currentTimeMillis() - 5.days.toMillis generateCommits(tblLoc, start, start + 20.minutes, start + 40.minutes) val df = spark.read.format("delta").load(identifierWithVersion(tblLoc, 0)) checkAnswer(df.groupBy().count(), Row(10L)) checkAnswer( spark.read.format("delta").option("versionAsOf", "0").load(tblLoc).groupBy().count(), Row(10) ) checkAnswer( spark.read.format("delta").option("versionAsOf", 1).load(tblLoc).groupBy().count(), Row(20) ) val e1 = intercept[AnalysisException] { spark.read.format("delta").option("versionAsOf", 3).load(tblLoc).collect() } assert(e1.getMessage.contains("[0, 2]")) val deltaLog = DeltaLog.forTable(spark, tblLoc) new File(FileNames.unsafeDeltaFile(deltaLog.logPath, 0).toUri).delete() val e2 = intercept[AnalysisException] { spark.read.format("delta").option("versionAsOf", 0).load(tblLoc).collect() } if (catalogOwnedDefaultCreationEnabledInTests && // Since we are creating a table w/ three initial commits, the table would have // unbackfilled commits if the backfill batch size is greater or equal to three. // See [[generateCommits]] for details. catalogOwnedCoordinatorBackfillBatchSize.exists(_ >= 3)) { // We throw an "incorrect" exception for CC tables if there exist any unbackfilled commits // and the backfilled commits have been manually deleted. E.g., the 0.json we are deleting // in this UT. // // Please see the comment in [[DeltaHistoryManager.getEarliestRecreatableCommit]] for the // detailed rationale. assert(e2.getMessage.contains("[DELTA_NO_COMMITS_FOUND]")) } else { assert(e2.getMessage.contains("recreatable")) } } } test("time travelling with adjusted timestamps") { if (isICTEnabledForNewTablesCatalogOwned) { // ICT Timestamps are always monotonically increasing. Therefore, // this test is not needed when ICT is enabled. cancel("This test is not compatible with InCommitTimestamps.") } withTempDir { dir => val tblLoc = dir.getCanonicalPath val start = System.currentTimeMillis() - 5.days.toMillis generateCommits(tblLoc, start, start - 5.seconds, start + 3.minutes) val ts = getSparkFormattedTimestamps( start, start + 1.milli, start + 119.seconds, start - 3.seconds) checkAnswer( spark.read.option("timestampAsOf", ts.head).format("delta").load(tblLoc).groupBy().count(), Row(10L) ) checkAnswer( spark.read.option("timestampAsOf", ts(1)).format("delta").load(tblLoc).groupBy().count(), Row(20L) ) checkAnswer( spark.read.option("timestampAsOf", ts(2)).format("delta").load(tblLoc).groupBy().count(), Row(20L) ) val e = intercept[AnalysisException] { spark.read.option("timestampAsOf", ts(3)).format("delta").load(tblLoc).collect() } assert(e.getMessage.contains("before the earliest version")) } } test("can't provide both version and timestamp in DataFrameReader") { val e = intercept[IllegalArgumentException] { spark.read.option("versionaSof", 1) .option("timestampAsOF", "fake").format("delta").load("/some/fake") } assert(e.getMessage.contains("either provide 'timestampAsOf' or 'versionAsOf'")) } test("don't time travel a valid delta path with @ syntax") { withTempDir { dir => val path = new File(dir, "base@v0").getCanonicalPath spark.range(10).write.format("delta").mode("append").save(path) spark.range(10).write.format("delta").mode("append").save(path) checkAnswer( spark.read.format("delta").load(path), spark.range(10).union(spark.range(10)).toDF() ) checkAnswer( spark.read.format("delta").load(path + "@v0"), spark.range(10).toDF() ) } } test("don't time travel a valid non-delta path with @ syntax") { val format = "json" withTempDir { dir => val path = new File(dir, "base@v0").getCanonicalPath spark.range(10).write.format(format).mode("append").save(path) spark.range(10).write.format(format).mode("append").save(path) checkAnswer( spark.read.format(format).load(path), spark.range(10).union(spark.range(10)).toDF() ) checkAnswer( spark.table(s"$format.`$path`"), spark.range(10).union(spark.range(10)).toDF() ) intercept[AnalysisException] { spark.read.format(format).load(path + "@v0").count() } intercept[AnalysisException] { spark.table(s"$format.`$path@v0`").count() } } } test("scans on different versions of same table are executed correctly") { withTempDir { dir => val path = dir.getCanonicalPath spark.range(5).selectExpr("id as key", "id * 10 as value").write.format("delta").save(path) spark.range(5, 10).selectExpr("id as key", "id * 10 as value") .write.format("delta").mode("append").save(path) val df = spark.read.format("delta").option("versionAsOf", "0").load(path).as("a").join( spark.read.format("delta").option("versionAsOf", "1").load(path).as("b"), functions.expr("a.key == b.key"), "fullOuter" ).where("a.key IS NULL") // keys 5 to 9 should be null assert(df.count() == 5) } } test("timestamp as of expression for table in database") { withDatabase("testDb") { sql("CREATE DATABASE testDb") withTable("tbl") { spark.range(10).write.format("delta").saveAsTable("testDb.tbl") val ts = sql("DESCRIBE HISTORY testDb.tbl").select("timestamp").head().getTimestamp(0) sql(s"SELECT * FROM testDb.tbl TIMESTAMP AS OF " + s"coalesce(CAST ('$ts' AS TIMESTAMP), current_date())") } } } test("time travel with schema changes - should instantiate old schema") { withTempDir { dir => val tblLoc = dir.getCanonicalPath spark.range(10).write.format("delta").mode("append").save(tblLoc) spark.range(10, 20).withColumn("part", 'id) .write.format("delta").mode("append").option("mergeSchema", true).save(tblLoc) checkAnswer( spark.read.option("versionAsOf", 0).format("delta").load(tblLoc), spark.range(10).toDF()) checkAnswer( spark.read.format("delta").load(identifierWithVersion(tblLoc, 0)), spark.range(10).toDF()) } } test("time travel with partition changes - should instantiate old schema") { withTempDir { dir => val tblLoc = dir.getCanonicalPath val v0 = spark.range(10).withColumn("part5", 'id % 5) v0.write.format("delta").partitionBy("part5").mode("append").save(tblLoc) spark.range(10, 20).withColumn("part2", 'id % 2) .write .format("delta") .partitionBy("part2") .mode("overwrite") .option("overwriteSchema", true) .save(tblLoc) checkAnswer( spark.read.option("versionAsOf", 0).format("delta").load(tblLoc), v0) checkAnswer( spark.read.format("delta").load(identifierWithVersion(tblLoc, 0)), v0) } } test("time travel support in SQL") { withTempDir { dir => val tblLoc = dir.getCanonicalPath val start = System.currentTimeMillis() - 5.days.toMillis generateCommits(tblLoc, start, start + 20.minutes) val tableName = "testTable" withTable(tableName) { spark.sql(s"create table $tableName(id long) using delta location '$tblLoc'") checkAnswer( spark.sql(s"SELECT * from $tableName FOR VERSION AS OF 0"), spark.read.option("versionAsOf", 0).format("delta").load(tblLoc)) checkAnswer( spark.sql(s"SELECT * from $tableName VERSION AS OF 1"), spark.read.option("versionAsOf", 1).format("delta").load(tblLoc)) val ex = intercept[VersionNotFoundException] { spark.sql(s"SELECT * from $tableName FOR VERSION AS OF 2") } checkError( ex, "DELTA_VERSION_NOT_FOUND", sqlState = "22003", parameters = Map("userVersion" -> "2", "earliest" -> "0", "latest" -> "1")) val timeAtVersion0 = new Timestamp(start).toString val timeAtVersion1 = new Timestamp(start + 20.minutes).toString val timeAfterVersion2 = new Timestamp(start + 6.hours).toString checkAnswer( spark.sql(s"SELECT * from $tableName FOR TIMESTAMP AS OF '$timeAtVersion0'"), spark.read.option("versionAsOf", 0).format("delta").load(tblLoc)) checkAnswer( spark.sql(s"SELECT * from $tableName TIMESTAMP AS OF '$timeAtVersion1'"), spark.read.option("versionAsOf", 1).format("delta").load(tblLoc)) val ex2 = intercept[DeltaErrors.TemporallyUnstableInputException] { spark.sql(s"SELECT * from $tableName FOR TIMESTAMP AS OF '$timeAfterVersion2'") } checkError( ex2, "DELTA_TIMESTAMP_GREATER_THAN_COMMIT", sqlState = "42816", parameters = Map( "providedTimestamp" -> s"$timeAfterVersion2", "tableName" -> s"$timeAtVersion1", "maximumTimestamp" -> s"${timeAtVersion1.replaceFirst("\\.\\d+$", "")}") // exclude ms ) } } } test("SPARK-41154: Correct relation caching for queries with time travel spec") { val tblName = "tab" withTable(tblName) { sql(s"CREATE TABLE $tblName USING DELTA AS SELECT 1 as c") sql(s"INSERT INTO $tblName SELECT 2 as c") checkAnswer( sql(s""" |SELECT * FROM $tblName VERSION AS OF '0' |UNION ALL |SELECT * FROM $tblName VERSION AS OF '1' |""".stripMargin), Row(1) :: Row(1) :: Row(2) :: Nil) } } test("Dataframe-based time travel works with different timestamp precisions") { val tblName = "test_tab" withTable(tblName) { sql(s"CREATE TABLE spark_catalog.default.$tblName (a int) USING DELTA") // Ensure that the current timestamp is different from the one in the table. Thread.sleep(1000) // Microsecond precision timestamp. val current_time_micros = spark.sql("SELECT current_timestamp() as ts") .select($"ts".cast("string")) .head().getString(0) // Millisecond precision timestamp. val current_time_millis = new Timestamp(System.currentTimeMillis()) // Second precision timestamp. val sdf = new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss") val current_time_seconds = sdf.format(new java.sql.Timestamp(System.currentTimeMillis())) sql(s"INSERT INTO spark_catalog.default.$tblName VALUES (1)") checkAnswer(spark.read.option("timestampAsOf", current_time_micros) .table(s"spark_catalog.default.$tblName"), Seq.empty) checkAnswer(spark.read.option("timestampAsOf", current_time_millis.toString) .table(s"spark_catalog.default.$tblName"), Seq.empty) checkAnswer(spark.read.option("timestampAsOf", current_time_seconds) .table(s"spark_catalog.default.$tblName"), Seq.empty) } } // Helper to generate a unique test table, commits, and timestamps for time travel blocking tests def withTestTable(testBody: (String, String, String) => Unit): Unit = { withTempDir { dir => val tblLoc = dir.getCanonicalPath val tableName = "testTable" withTable(tableName) { // create six versions spaced 1 day apart val start = System.currentTimeMillis() - 10.days.toMillis generateCommits( tblLoc, start, // v0 start + 0.5.days, // v1 start + 1.days, // v2 start + 2.days, // v3 start + 2.5.days, // v4 start + 4.days, // v5 start + 5.days // v6 ) // timestamps for v4 and v6 val t4 = new Timestamp(start + 2.5.days.toMillis).toString val t6 = new Timestamp(start + 5.days.toMillis).toString spark.sql(s"CREATE TABLE $tableName(id LONG) USING delta LOCATION '$tblLoc'") spark.sql(s"ALTER TABLE $tableName" + s" SET TBLPROPERTIES ('delta.enableChangeDataFeed' = 'true')") testBody(tableName, t4, t6) } } } // Helper to assert whether a given SQL should or should not throw the retention exception def assertBlocked(sql: String, shouldThrow: Boolean): Unit = { val msg = "Cannot time travel beyond delta.deletedFileRetentionDuration" if (shouldThrow) { val ex = intercept[Exception]( spark.sql(sql) ) assert(ex.getMessage.contains(msg)) } else { spark.sql(sql) // must succeed } } // 1) SELECT ... AS OF test("Block time travel beyond deletedFileRetention") { withTestTable { (tbl, t4, t6) => Seq( s"SELECT * FROM $tbl VERSION AS OF 2" -> true, s"SELECT * FROM $tbl TIMESTAMP AS OF '$t4'" -> true, s"SELECT * FROM $tbl VERSION AS OF 5" -> false, s"SELECT * FROM $tbl TIMESTAMP AS OF '$t6'" -> false ).foreach { case (sql, fail) => assertBlocked(sql, fail) } spark.sql(s"ALTER TABLE $tbl " + s"SET TBLPROPERTIES ('delta.deletedFileRetentionDuration' = 'interval 0 HOURS')") // Even after lowering retention to zero, a simple select * should still work // which references the latest version assertBlocked(s"SELECT * FROM $tbl", false) // After setting it to zero, any time travel will fail Seq( s"SELECT * FROM $tbl VERSION AS OF 5" -> true, s"SELECT * FROM $tbl TIMESTAMP AS OF '$t6'" -> true ).foreach { case (sql, fail) => assertBlocked(sql, fail) } } } // 2) SELECT ... CHANGES AS OF test("Block CDC beyond deletedFileRetention") { withTestTable { (tbl, t4, t6) => Seq( s"SELECT * FROM table_changes('$tbl', 2)" -> true, s"SELECT * FROM table_changes('$tbl', '$t4')" -> true, s"SELECT * FROM table_changes('$tbl', 5)" -> false, s"SELECT * FROM table_changes('$tbl', '$t6')" -> false ).foreach { case (sql, fail) => assertBlocked(sql, fail) } spark.sql(s"ALTER TABLE $tbl " + s"SET TBLPROPERTIES ('delta.deletedFileRetentionDuration' = 'interval 0 HOURS')") // After setting it to zero, any previous version will fail Seq( s"SELECT * FROM table_changes('$tbl', 5)" -> true, s"SELECT * FROM table_changes('$tbl', '$t6')" -> true ).foreach { case (sql, fail) => assertBlocked(sql, fail) } } } // 3) RESTORE ... AS OF test("Block restore table beyond deletedFileRetention") { withTestTable { (tbl, t4, t6) => Seq( s"RESTORE TABLE $tbl TO VERSION AS OF 2" -> true, s"RESTORE TABLE $tbl TO TIMESTAMP AS OF '$t4'" -> true, s"RESTORE TABLE $tbl TO VERSION AS OF 5" -> false, s"RESTORE TABLE $tbl TO TIMESTAMP AS OF '$t6'" -> false ).foreach { case (sql, fail) => assertBlocked(sql, fail) } spark.sql(s"ALTER TABLE $tbl" + s" SET TBLPROPERTIES ('delta.deletedFileRetentionDuration' = 'interval 0 HOURS')") // After setting it to zero, any previous version will fail Seq( s"RESTORE TABLE $tbl TO VERSION AS OF 5" -> true, s"RESTORE TABLE $tbl TO TIMESTAMP AS OF '$t6'" -> true ).foreach { case (sql, fail) => assertBlocked(sql, fail) } } } // 4) CLONE ... AS OF test("Block clone table beyond deletedFileRetention") { withTestTable { (tbl, t4, t6) => val targets = Seq("targetTable1", "targetTable2", "targetTable3") Seq( s"CREATE TABLE ${targets(0)} SHALLOW CLONE $tbl VERSION AS OF 2" -> true, s"CREATE TABLE ${targets(0)} SHALLOW CLONE $tbl TIMESTAMP AS OF '$t4'" -> true, s"CREATE TABLE ${targets(0)} SHALLOW CLONE $tbl VERSION AS OF 5" -> false, s"CREATE TABLE ${targets(1)} SHALLOW CLONE $tbl TIMESTAMP AS OF '$t6'" -> false ).foreach { case (sql, fail) => assertBlocked(sql, fail) } spark.sql(s"ALTER TABLE $tbl" + s" SET TBLPROPERTIES ('delta.deletedFileRetentionDuration' = 'interval 0 HOURS')") // After setting it to zero, any previous version will fail Seq( s"CREATE TABLE ${targets(0)} SHALLOW CLONE $tbl VERSION AS OF 5" -> true, s"CREATE TABLE ${targets(0)} SHALLOW CLONE $tbl TIMESTAMP AS OF '$t6'" -> true ).foreach { case (sql, fail) => assertBlocked(sql, fail) } } } } class DeltaTimeTravelWithCatalogOwnedBatch1Suite extends DeltaTimeTravelSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class DeltaTimeTravelWithCatalogOwnedBatch2Suite extends DeltaTimeTravelSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class DeltaTimeTravelWithCatalogOwnedBatch100Suite extends DeltaTimeTravelSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaTimestampNTZSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.sql.Timestamp import java.time.LocalDateTime import org.apache.spark.sql.delta.actions.Protocol import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.SparkThrowable import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.StructType class DeltaTimestampNTZSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { private def getProtocolForTable(table: String): Protocol = { val deltaLog = DeltaLog.forTable(spark, TableIdentifier(table)) deltaLog.unsafeVolatileSnapshot.protocol } test("create a new table with TIMESTAMP_NTZ, higher protocol and feature should be picked.") { withTable("tbl") { sql("CREATE TABLE tbl(c1 STRING, c2 TIMESTAMP, c3 TIMESTAMP_NTZ) USING DELTA") sql( """INSERT INTO tbl VALUES |('foo','2022-01-02 03:04:05.123456','2022-01-02 03:04:05.123456')""".stripMargin) assert(spark.table("tbl").head == Row( "foo", new Timestamp(2022 - 1900, 0, 2, 3, 4, 5, 123456000), LocalDateTime.of(2022, 1, 2, 3, 4, 5, 123456000))) assert(getProtocolForTable("tbl") == TimestampNTZTableFeature.minProtocolVersion.withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, TimestampNTZTableFeature)) ) } } test("creating a table without TIMESTAMP_NTZ should use the usual minimum protocol") { withTable("tbl") { sql("CREATE TABLE tbl(c1 STRING, c2 TIMESTAMP, c3 TIMESTAMP) USING DELTA") assert(getProtocolForTable("tbl") == Protocol(1, 2)) val deltaLog = DeltaLog.forTable(spark, TableIdentifier("tbl")) assert( !deltaLog.unsafeVolatileSnapshot.protocol.isFeatureSupported(TimestampNTZTableFeature), s"Table tbl contains TimestampNTZFeature descriptor when its not supposed to" ) } } test("add a new column using TIMESTAMP_NTZ should upgrade to the correct protocol versions") { withTable("tbl") { sql("CREATE TABLE tbl(c1 STRING, c2 TIMESTAMP) USING delta") assert(getProtocolForTable("tbl") == Protocol(1, 2)) // Should throw error val e = intercept[SparkThrowable] { sql("ALTER TABLE tbl ADD COLUMN c3 TIMESTAMP_NTZ") } // add table feature sql(s"ALTER TABLE tbl " + s"SET TBLPROPERTIES('delta.feature.timestampNtz' = 'supported')") sql("ALTER TABLE tbl ADD COLUMN c3 TIMESTAMP_NTZ") sql( """INSERT INTO tbl VALUES |('foo','2022-01-02 03:04:05.123456','2022-01-02 03:04:05.123456')""".stripMargin) assert(spark.table("tbl").head == Row( "foo", new Timestamp(2022 - 1900, 0, 2, 3, 4, 5, 123456000), LocalDateTime.of(2022, 1, 2, 3, 4, 5, 123456000))) assert(getProtocolForTable("tbl") == TimestampNTZTableFeature.minProtocolVersion .withFeature(TimestampNTZTableFeature) .withFeature(InvariantsTableFeature) .withFeature(AppendOnlyTableFeature) ) } } test("use TIMESTAMP_NTZ in a partition column") { withTable("delta_test") { sql( """CREATE TABLE delta_test(c1 STRING, c2 TIMESTAMP, c3 TIMESTAMP_NTZ) |USING delta |PARTITIONED BY (c3)""".stripMargin) sql( """INSERT INTO delta_test VALUES |('foo','2022-01-02 03:04:05.123456','2022-01-02 03:04:05.123456')""".stripMargin) assert(spark.table("delta_test").head == Row( "foo", new Timestamp(2022 - 1900, 0, 2, 3, 4, 5, 123456000), LocalDateTime.of(2022, 1, 2, 3, 4, 5, 123456000))) assert(getProtocolForTable("delta_test") == TimestampNTZTableFeature.minProtocolVersion.withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, TimestampNTZTableFeature)) ) } } test("min/max stats collection should apply on TIMESTAMP_NTZ") { withTable("delta_test") { val schemaString = "c1 STRING, c2 TIMESTAMP, c3 TIMESTAMP_NTZ" sql(s"CREATE TABLE delta_test($schemaString) USING delta") val statsSchema = DeltaLog.forTable(spark, TableIdentifier("delta_test")) .unsafeVolatileSnapshot.statsSchema assert(statsSchema("minValues").dataType == StructType .fromDDL(schemaString)) assert(statsSchema("maxValues").dataType == StructType .fromDDL(schemaString)) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaUDFSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.{Encoder, QueryTest, Row} import org.apache.spark.sql.expressions.UserDefinedFunction import org.apache.spark.sql.functions._ import org.apache.spark.sql.test.SharedSparkSession class DeltaUDFSuite extends QueryTest with SharedSparkSession { import testImplicits._ private def testUDF( name: String, testResultFunc: => Unit): Unit = { test(name) { // Verify the returned UDF function is working correctly testResultFunc } } private def testUDF( name: String, func: => UserDefinedFunction, expected: Any): Unit = { testUDF( name, checkAnswer(Seq("foo").toDF.select(func()), Row(expected)) ) } private def testUDF[T: Encoder]( name: String, func: => UserDefinedFunction, input: T, expected: Any): Unit = { testUDF( name, checkAnswer(Seq(input).toDF.select(func(col("value"))), Row(expected)) ) } private def testUDF[T1: Encoder, T2: Encoder]( name: String, func: => UserDefinedFunction, input1: T1, input2: T2, expected: Any): Unit = { testUDF( name, { val df = Seq(input1) .toDF("value1") .withColumn("value2", lit(input2).as[T2]) .select(func(col("value1"), col("value2"))) checkAnswer(df, Row(expected)) } ) } testUDF( name = "stringFromString", func = DeltaUDF.stringFromString(x => x), input = "foo", expected = "foo") testUDF( name = "intFromString", func = DeltaUDF.intFromString(x => x.toInt), input = "100", expected = 100) testUDF( name = "intFromStringBoolean", func = DeltaUDF.intFromStringBoolean((x, y) => 1), input1 = "foo", input2 = true, expected = 1) testUDF(name = "boolean", func = DeltaUDF.boolean(() => true), expected = true) testUDF( name = "stringFromMap", func = DeltaUDF.stringFromMap(x => x.toString), input = Map("foo" -> "bar"), expected = "Map(foo -> bar)") testUDF( name = "booleanFromMap", func = DeltaUDF.booleanFromMap(x => x.isEmpty), input = Map("foo" -> "bar"), expected = false) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaUpdateCatalogSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import scala.util.control.NonFatal import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta.hooks.UpdateCatalog import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaHiveTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.test.shims.StreamingTestShims.MemoryStream import com.fasterxml.jackson.core.JsonParseException import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.ExternalCatalogWithListener import org.apache.spark.sql.functions.{lit, struct} import org.apache.spark.sql.hive.HiveExternalCatalog import org.apache.spark.sql.types.{ArrayType, DoubleType, IntegerType, LongType, MapType, StringType, StructField, StructType} import org.apache.spark.util.{ThreadUtils, Utils} class DeltaUpdateCatalogSuite extends DeltaUpdateCatalogSuiteBase with DeltaHiveTest { import testImplicits._ override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaSQLConf.DELTA_UPDATE_CATALOG_ENABLED.key, "true") } override def beforeEach(): Unit = { super.beforeEach() cleanupDefaultTable() } override def afterEach(): Unit = { if (!UpdateCatalog.awaitCompletion(10000)) { logWarning(s"There are active catalog udpate requests after 10 seconds") } cleanupDefaultTable() super.afterEach() } /** Remove Hive specific table properties. */ override protected def filterProperties(properties: Map[String, String]): Map[String, String] = { properties.filterKeys(_ != "transient_lastDdlTime").toMap } test("streaming") { withTable(tbl) { implicit val sparkSession: SparkSession = spark implicit val _sqlContext = spark.sqlContext val stream = MemoryStream[Long] val df1 = stream.toDF().toDF("id") withTempDir { dir => try { val q = df1.writeStream .option("checkpointLocation", dir.getCanonicalPath) .format("delta") .toTable(tbl) verifyTableMetadata(expectedSchema = df1.schema.asNullable) stream.addData(1, 2, 3) q.processAllAvailable() q.stop() val q2 = df1.withColumn("id2", 'id) .writeStream .format("delta") .option("mergeSchema", "true") .option("checkpointLocation", dir.getCanonicalPath) .toTable(tbl) stream.addData(4, 5, 6) q2.processAllAvailable() verifyTableMetadataAsync(expectedSchema = df1.schema.asNullable.add("id2", LongType)) } finally { spark.streams.active.foreach(_.stop()) } } } } test("streaming - external location") { withTempDir { dir => withTable(tbl) { implicit val sparkSession: SparkSession = spark implicit val _sqlContext = spark.sqlContext val stream = MemoryStream[Long] val df1 = stream.toDF().toDF("id") val chk = new File(dir, "chkpoint").getCanonicalPath val data = new File(dir, "data").getCanonicalPath try { val q = df1.writeStream .option("checkpointLocation", chk) .format("delta") .option("path", data) .toTable(tbl) verifyTableMetadata(expectedSchema = df1.schema.asNullable) stream.addData(1, 2, 3) q.processAllAvailable() q.stop() val q2 = df1.withColumn("id2", 'id) .writeStream .format("delta") .option("mergeSchema", "true") .option("checkpointLocation", chk) .toTable(tbl) stream.addData(4, 5, 6) q2.processAllAvailable() verifyTableMetadataAsync(expectedSchema = df1.schema.add("id2", LongType).asNullable) } finally { spark.streams.active.foreach(_.stop()) } } } } test("streaming - external table that already exists") { withTable(tbl) { implicit val sparkSession: SparkSession = spark implicit val _sqlContext = spark.sqlContext val stream = MemoryStream[Long] val df1 = stream.toDF().toDF("id") withTempDir { dir => val chk = new File(dir, "chkpoint").getCanonicalPath val data = new File(dir, "data").getCanonicalPath spark.range(10).write.format("delta").save(data) try { val q = df1.writeStream .option("checkpointLocation", chk) .format("delta") .option("path", data) .toTable(tbl) verifyTableMetadataAsync(expectedSchema = df1.schema.asNullable) stream.addData(1, 2, 3) q.processAllAvailable() q.stop() val q2 = df1.withColumn("id2", 'id) .writeStream .format("delta") .option("mergeSchema", "true") .option("checkpointLocation", chk) .toTable(tbl) stream.addData(4, 5, 6) q2.processAllAvailable() verifyTableMetadataAsync(expectedSchema = df1.schema.add("id2", LongType).asNullable) } finally { spark.streams.active.foreach(_.stop()) } } } } val MAX_CATALOG_TYPE_DDL_LENGTH: Long = DeltaSQLConf.DELTA_UPDATE_CATALOG_LONG_FIELD_TRUNCATION_THRESHOLD.defaultValue.get test("convert to delta with partitioning change") { withTable(tbl) { val df = spark.range(10).withColumn("part", 'id / 2).withColumn("id2", 'id) df.writeTo(tbl) .partitionedBy('part) .using("parquet") .create() // Partitioning columns go to the end for parquet tables val tableSchema = new StructType().add("id", LongType).add("id2", LongType).add("part", DoubleType) verifyTableMetadata( expectedSchema = tableSchema, expectedProperties = Map.empty, partitioningCols = Seq("part") ) sql(s"CONVERT TO DELTA $tbl PARTITIONED BY (part double)") // Information is duplicated for now verifyTableMetadata( expectedSchema = tableSchema, expectedProperties = Map.empty, partitioningCols = Seq("part") ) // Remove partitioning of table df.writeTo(tbl).using("delta").replace() assert(snapshot.metadata.partitionColumns === Nil, "Table is unpartitioned") // Hive does not allow for the removal of the partition column once it has // been added. Spark keeps the partition columns towards the end if it // finds them in Hive. So, for converted tables with partitions, // Hive schema != df.schema val expectedSchema = tableSchema // Schema converts to Delta's format verifyTableMetadata( expectedSchema = expectedSchema, expectedProperties = getBaseProperties(snapshot), partitioningCols = Seq("part") // The partitioning information cannot be removed... ) // table is still usable checkAnswer(spark.table(tbl), df) val df2 = spark.range(10).withColumn("id2", 'id) // Gets rid of partition column "part" from the schema df2.writeTo(tbl).using("delta").replace() val expectedSchema2 = new StructType() .add("id", LongType).add("id2", LongType).add("part", DoubleType) verifyTableMetadataAsync( expectedSchema = expectedSchema2, expectedProperties = getBaseProperties(snapshot), partitioningCols = Seq("part") // The partitioning information cannot be removed... ) // table is still usable checkAnswer(spark.table(tbl), df2) } } test("partitioned table + add column") { withTable(tbl) { val df = spark.range(10).withColumn("part", 'id / 2).withColumn("id2", 'id) df.writeTo(tbl) .partitionedBy('part) .using("delta") .create() val tableSchema = new StructType().add("id", LongType).add("part", DoubleType).add("id2", LongType) verifyTableMetadata( expectedSchema = tableSchema, expectedProperties = getBaseProperties(snapshot), partitioningCols = Seq()) sql(s"ALTER TABLE $tbl ADD COLUMNS (id3 bigint)") verifyTableMetadataAsync( expectedSchema = tableSchema.add("id3", LongType), expectedProperties = getBaseProperties(snapshot), partitioningCols = Seq()) } } test("partitioned convert to delta with schema change") { withTable(tbl) { val df = spark.range(10).withColumn("part", 'id / 2).withColumn("id2", 'id) df.writeTo(tbl) .partitionedBy('part) .using("parquet") .create() // Partitioning columns go to the end val tableSchema = new StructType().add("id", LongType).add("id2", LongType).add("part", DoubleType) verifyTableMetadata( expectedSchema = tableSchema, expectedProperties = Map.empty, partitioningCols = Seq("part") ) sql(s"CONVERT TO DELTA $tbl PARTITIONED BY (part double)") // Information is duplicated for now verifyTableMetadata( expectedSchema = tableSchema, expectedProperties = Map.empty, partitioningCols = Seq("part") ) sql(s"ALTER TABLE $tbl ADD COLUMNS (id3 bigint)") // Hive does not allow for the removal of the partition column once it has // been added. Spark keeps the partition columns towards the end if it // finds them in Hive. So, for converted tables with partitions, // Hive schema != df.schema val expectedSchema = new StructType() .add("id", LongType) .add("id2", LongType) .add("id3", LongType) .add("part", DoubleType) verifyTableMetadataAsync( expectedSchema = expectedSchema, partitioningCols = Seq("part") ) // Table is still queryable checkAnswer( spark.table(tbl), // Ordering of columns are different than df due to Hive semantics spark.range(10).withColumn("id2", 'id) .withColumn("part", 'id / 2) .withColumn("id3", lit(null))) } } test("Very long schemas can be stored in the catalog") { withTable(tbl) { val schema = StructType(Seq.tabulate(1000)(i => StructField(s"col$i", StringType))) require(schema.toDDL.length >= MAX_CATALOG_TYPE_DDL_LENGTH, s"The length of the schema should be over $MAX_CATALOG_TYPE_DDL_LENGTH " + "characters for this test") sql(s"CREATE TABLE $tbl (${schema.toDDL}) USING delta") verifyTableMetadata(expectedSchema = schema) } } for (truncationThreshold <- Seq(99999, MAX_CATALOG_TYPE_DDL_LENGTH, 4020)) test(s"Schemas that contain very long fields cannot be stored in the catalog " + " when longer than the truncation threshold " + s" [DELTA_UPDATE_CATALOG_LONG_FIELD_TRUNCATION_THRESHOLD = $truncationThreshold]") { withSQLConf( DeltaSQLConf.DELTA_UPDATE_CATALOG_LONG_FIELD_TRUNCATION_THRESHOLD.key -> truncationThreshold.toString) { withTable(tbl) { val schema = new StructType() .add("i", StringType) .add("struct", StructType(Seq.tabulate(1000)(i => StructField(s"col$i", StringType)))) require( schema.toDDL.length >= 4020, s"The length of the schema should be over 4020 " + s"characters for this test") sql(s"CREATE TABLE $tbl (${schema.toDDL}) USING delta") if (truncationThreshold > 4020) { verifyTableMetadata(expectedSchema = schema) } else { verifySchemaInCatalog() } } } } for (truncationThreshold <- Seq(99999, MAX_CATALOG_TYPE_DDL_LENGTH)) test(s"Schemas that contain very long fields cannot be stored in the catalog - array" + " when longer than the truncation threshold " + s" [DELTA_UPDATE_CATALOG_LONG_FIELD_TRUNCATION_THRESHOLD = $truncationThreshold]") { withSQLConf( DeltaSQLConf.DELTA_UPDATE_CATALOG_LONG_FIELD_TRUNCATION_THRESHOLD.key -> truncationThreshold.toString) { withTable(tbl) { val struct = StructType(Seq.tabulate(1000)(i => StructField(s"col$i", StringType))) val schema = new StructType() .add("i", StringType) .add("array", ArrayType(struct)) require(schema.toDDL.length >= MAX_CATALOG_TYPE_DDL_LENGTH, s"The length of the schema should be over $MAX_CATALOG_TYPE_DDL_LENGTH " + s"characters for this test") sql(s"CREATE TABLE $tbl (${schema.toDDL}) USING delta") if (truncationThreshold == 99999) { verifyTableMetadata(expectedSchema = schema) } else { verifySchemaInCatalog() } } } } for (truncationThreshold <- Seq(99999, MAX_CATALOG_TYPE_DDL_LENGTH)) test(s"Schemas that contain very long fields cannot be stored in the catalog - map" + " when longer than the truncation threshold " + s" [DELTA_UPDATE_CATALOG_LONG_FIELD_TRUNCATION_THRESHOLD = $truncationThreshold]") { withSQLConf( DeltaSQLConf.DELTA_UPDATE_CATALOG_LONG_FIELD_TRUNCATION_THRESHOLD.key -> truncationThreshold.toString) { withTable(tbl) { val struct = StructType(Seq.tabulate(1000)(i => StructField(s"col$i", StringType))) val schema = new StructType() .add("i", StringType) .add("map", MapType(StringType, struct)) require(schema.toDDL.length >= MAX_CATALOG_TYPE_DDL_LENGTH, s"The length of the schema should be over $MAX_CATALOG_TYPE_DDL_LENGTH " + s"characters for this test") sql(s"CREATE TABLE $tbl (${schema.toDDL}) USING delta") if (truncationThreshold == 99999) { verifyTableMetadata(expectedSchema = schema) } else { verifySchemaInCatalog() } } } } for (truncationThreshold <- Seq(99999, MAX_CATALOG_TYPE_DDL_LENGTH)) test(s"Very long nested fields cannot be stored in the catalog - partitioned" + " when longer than the truncation threshold " + s" [DELTA_UPDATE_CATALOG_LONG_FIELD_TRUNCATION_THRESHOLD = $truncationThreshold]") { withSQLConf( DeltaSQLConf.DELTA_UPDATE_CATALOG_LONG_FIELD_TRUNCATION_THRESHOLD.key -> truncationThreshold.toString) { withTable(tbl) { val schema = new StructType() .add("i", StringType) .add("part", StringType) .add("struct", StructType(Seq.tabulate(1000)(i => StructField(s"col$i", StringType)))) require( schema.toDDL.length >= MAX_CATALOG_TYPE_DDL_LENGTH, "The length of the schema should be over 4000 characters for this test") sql(s"CREATE TABLE $tbl (${schema.toDDL}) USING delta PARTITIONED BY (part)") if (truncationThreshold == 99999) { verifyTableMetadata(expectedSchema = schema) } else { verifySchemaInCatalog() } } } } test("Very long schemas can be stored in the catalog - partitioned") { withTable(tbl) { val schema = StructType(Seq.tabulate(1000)(i => StructField(s"col$i", StringType))) .add("part", StringType) require(schema.toDDL.length >= MAX_CATALOG_TYPE_DDL_LENGTH, "The length of the schema should be over 4000 characters for this test") sql(s"CREATE TABLE $tbl (${schema.toDDL}) USING delta PARTITIONED BY (part)") verifyTableMetadata(expectedSchema = schema) } } // scalastyle:off nonascii test("Schema containing non-latin characters cannot be stored - top-level") { withTable(tbl) { val schema = new StructType().add("今天", "string") sql(s"CREATE TABLE $tbl (${schema.toDDL}) USING delta") verifySchemaInCatalog(expectedErrorMessage = UpdateCatalog.NON_LATIN_CHARS_ERROR) } } test("Schema containing non-latin characters cannot be stored - struct") { withTable(tbl) { val schema = new StructType().add("struct", new StructType().add("今天", "string")) sql(s"CREATE TABLE $tbl (${schema.toDDL}) USING delta") verifySchemaInCatalog(expectedErrorMessage = UpdateCatalog.NON_LATIN_CHARS_ERROR) } } test("Schema containing non-latin characters cannot be stored - array") { withTable(tbl) { val schema = new StructType() .add("i", StringType) .add("array", ArrayType(new StructType().add("今天", "string"))) sql(s"CREATE TABLE $tbl (${schema.toDDL}) USING delta") verifySchemaInCatalog(expectedErrorMessage = UpdateCatalog.NON_LATIN_CHARS_ERROR) } } test("Schema containing non-latin characters cannot be stored - map") { withTable(tbl) { val schema = new StructType() .add("i", StringType) .add("map", MapType(StringType, new StructType().add("今天", "string"))) sql(s"CREATE TABLE $tbl (${schema.toDDL}) USING delta") verifySchemaInCatalog(expectedErrorMessage = UpdateCatalog.NON_LATIN_CHARS_ERROR) } } // scalastyle:on nonascii /** * Verifies that the schema stored in the catalog explicitly is empty, however the getTablesByName * method still correctly returns the actual schema. */ private def verifySchemaInCatalog( table: String = tbl, catalogPartitionCols: Seq[String] = Nil, expectedErrorMessage: String = UpdateCatalog.LONG_SCHEMA_ERROR): Unit = { val cat = spark.sessionState.catalog.externalCatalog.getTable("default", table) assert(cat.schema.isEmpty, s"Schema wasn't empty") assert(cat.partitionColumnNames === catalogPartitionCols) getBaseProperties(snapshot).foreach { case (k, v) => assert(cat.properties.get(k) === Some(v), s"Properties didn't match for table: $table. Expected: ${getBaseProperties(snapshot)}, " + s"Got: ${cat.properties}") } assert(cat.properties(UpdateCatalog.ERROR_KEY) === expectedErrorMessage) // Make sure table is readable checkAnswer(spark.table(table), Nil) } def testAddRemoveProperties(): Unit = { withTable(tbl) { val df = spark.range(10).toDF("id") df.writeTo(tbl) .using("delta") .create() var initialProperties: Map[String, String] = Map.empty val logs = Log4jUsageLogger.track { sql(s"ALTER TABLE $tbl SET TBLPROPERTIES(some.key = 1, another.key = 2)") initialProperties = getBaseProperties(snapshot) verifyTableMetadataAsync( expectedSchema = df.schema.asNullable, expectedProperties = Map("some.key" -> "1", "another.key" -> "2") ++ initialProperties ) } val updateLogged = logs.filter(_.metric == "tahoeEvent") .filter(_.tags.get("opType").exists(_.startsWith("delta.catalog.update.properties"))) assert(updateLogged.nonEmpty, "Ensure that the schema update in the MetaStore is logged") // The UpdateCatalog hook only checks if new properties have been // added. If properties have been removed only, no metadata update will be triggered. val logs2 = Log4jUsageLogger.track { sql(s"ALTER TABLE $tbl UNSET TBLPROPERTIES(another.key)") verifyTableMetadataAsync( expectedSchema = df.schema.asNullable, expectedProperties = Map("some.key" -> "1", "another.key" -> "2") ++ initialProperties ) } val updateLogged2 = logs2.filter(_.metric == "tahoeEvent") .filter(_.tags.get("opType").exists(_.startsWith("delta.catalog.update.properties"))) assert(updateLogged2.size == 0, "Ensure that the schema update in the MetaStore is logged") // Adding a new property will trigger an update val logs3 = Log4jUsageLogger.track { sql(s"ALTER TABLE $tbl SET TBLPROPERTIES(a.third.key = 3)") verifyTableMetadataAsync( expectedSchema = df.schema.asNullable, expectedProperties = Map("some.key" -> "1", "a.third.key" -> "3") ++ getBaseProperties(snapshot) ) } val updateLogged3 = logs3.filter(_.metric == "tahoeEvent") .filter(_.tags.get("opType").exists(_.startsWith("delta.catalog.update.properties"))) assert(updateLogged3.nonEmpty, "Ensure that the schema update in the MetaStore is logged") } } test("add and remove properties") { testAddRemoveProperties() } test("alter table commands update the catalog") { runAlterTableTests { (tableName, expectedSchema) => verifyTableMetadataAsync( expectedSchema = expectedSchema, // The ALTER TABLE statements in runAlterTableTests create table version 7. // However, version 7 is created by dropping a CHECK constraint, which currently // *does not* trigger a catalog update. For Hive tables, only *adding* properties // causes a catalog update, not *removing*. Hence, the metadata in the catalog should // still be at version 6. expectedProperties = getBaseProperties(snapshotAt(6)) ++ Map("some" -> "thing", "delta.constraints.id_3" -> "id3 > 10"), table = tableName ) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaUpdateCatalogSuiteBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import scala.util.control.NonFatal import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils import org.apache.spark.sql.delta.hooks.UpdateCatalog import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.scalatest.time.SpanSugar import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.{lit, struct} import org.apache.spark.sql.types.{BooleanType, DoubleType, IntegerType, LongType, StringType, StructField, StructType} import org.apache.spark.util.{ThreadUtils, Utils} abstract class DeltaUpdateCatalogSuiteBase extends QueryTest with DeltaSQLTestUtils with SpanSugar { protected val tbl = "delta_table" import testImplicits._ protected def cleanupDefaultTable(): Unit = disableUpdates { spark.sql(s"DROP TABLE IF EXISTS $tbl") val path = spark.sessionState.catalog.defaultTablePath(TableIdentifier(tbl)) try Utils.deleteRecursively(new File(path)) catch { case NonFatal(e) => // do nothing } } /** Turns off the storing of metadata (schema + properties) in the catalog. */ protected def disableUpdates(f: => Unit): Unit = { withSQLConf(DeltaSQLConf.DELTA_UPDATE_CATALOG_ENABLED.key -> "false") { f } } protected def deltaLog: DeltaLog = DeltaLog.forTable(spark, TableIdentifier(tbl)) protected def snapshot: Snapshot = deltaLog.unsafeVolatileSnapshot protected def snapshotAt(v: Long): Snapshot = deltaLog.getSnapshotAt(v) protected def getBaseProperties(snapshot: Snapshot): Map[String, String] = { Map( DeltaConfigs.METASTORE_LAST_UPDATE_VERSION -> snapshot.version.toString, DeltaConfigs.METASTORE_LAST_COMMIT_TIMESTAMP -> snapshot.timestamp.toString, DeltaConfigs.MIN_READER_VERSION.key -> snapshot.protocol.minReaderVersion.toString, DeltaConfigs.MIN_WRITER_VERSION.key -> snapshot.protocol.minWriterVersion.toString) ++ snapshot.protocol.readerAndWriterFeatureNames.map { name => s"${TableFeatureProtocolUtils.FEATURE_PROP_PREFIX}$name" -> TableFeatureProtocolUtils.FEATURE_PROP_SUPPORTED } ++ snapshot.metadata.configuration.get("delta.enableDeletionVectors") .map("delta.enableDeletionVectors" -> _).toMap } /** * Verifies that the table metadata in the catalog are eventually up-to-date. Updates to the * catalog are generally asynchronous, except explicit DDL operations, e.g. CREATE/REPLACE. */ protected def verifyTableMetadataAsync( expectedSchema: StructType, expectedProperties: Map[String, String] = getBaseProperties(snapshot), table: String = tbl, partitioningCols: Seq[String] = Nil): Unit = { // We unfortunately need an eventually, because the updates can be async eventually(timeout(10.seconds)) { verifyTableMetadata(expectedSchema, expectedProperties, table, partitioningCols) } // Ensure that no other threads will later revert us back to the state we just checked if (!UpdateCatalog.awaitCompletion(10000)) { logWarning(s"There are active catalog udpate requests after 10 seconds") } } protected def filterProperties(properties: Map[String, String]): Map[String, String] /** Verifies that the table metadata in the catalog are up-to-date. */ protected def verifyTableMetadata( expectedSchema: StructType, expectedProperties: Map[String, String] = getBaseProperties(snapshot), table: String = tbl, partitioningCols: Seq[String] = Nil): Unit = { DeltaLog.clearCache() val cat = spark.sessionState.catalog.externalCatalog.getTable("default", table) assert(cat.schema === expectedSchema, s"Schema didn't match for table: $table") assert(cat.partitionColumnNames === partitioningCols) assert(filterProperties(cat.properties) === expectedProperties, s"Properties didn't match for table: $table") val tables = spark.sessionState.catalog.getTablesByName(Seq(TableIdentifier(table))) assert(tables.head.schema === expectedSchema) assert(tables.head.partitionColumnNames === partitioningCols) assert(filterProperties(tables.head.properties) === expectedProperties) } test("mergeSchema") { withTable(tbl) { val df = spark.range(10).withColumn("part", 'id / 2) df.writeTo(tbl).using("delta").create() verifyTableMetadata(expectedSchema = df.schema.asNullable) val df2 = spark.range(10).withColumn("part", 'id / 2).withColumn("id2", 'id) df2.writeTo(tbl) .option("mergeSchema", "true") .append() verifyTableMetadataAsync(expectedSchema = df2.schema.asNullable) } } test("mergeSchema - nested data types") { withTable(tbl) { val df = spark.range(10).withColumn("part", 'id / 2) .withColumn("str", struct('id.cast("int") as "int")) df.writeTo(tbl).using("delta").create() verifyTableMetadata(expectedSchema = df.schema.asNullable) val df2 = spark.range(10).withColumn("part", 'id / 2) .withColumn("str", struct('id as "id2", 'id.cast("int") as "int")) df2.writeTo(tbl) .option("mergeSchema", "true") .append() val schema = new StructType() .add("id", LongType) .add("part", DoubleType) .add("str", new StructType() .add("int", IntegerType) .add("id2", LongType)) // New columns go to the end verifyTableMetadataAsync(expectedSchema = schema) } } test("merge") { val tmp = "tmpView" withDeltaTable { df => withTempView(tmp) { withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { df.withColumn("id2", 'id).createOrReplaceTempView(tmp) sql( s"""MERGE INTO $tbl t |USING $tmp s |ON t.id = s.id |WHEN NOT MATCHED THEN INSERT * """.stripMargin) verifyTableMetadataAsync(df.withColumn("id2", 'id).schema.asNullable) } } } } test("creating and replacing a table puts the schema and table properties in the metastore") { withTable(tbl) { val df = spark.range(10).withColumn("part", 'id / 2).withColumn("id2", 'id) df.writeTo(tbl) .tableProperty("delta.checkpointInterval", "5") .tableProperty("some", "thing") .partitionedBy('part) .using("delta") .create() verifyTableMetadata( expectedSchema = df.schema.asNullable, expectedProperties = getBaseProperties(snapshot) ++ Map( "delta.checkpointInterval" -> "5", "some" -> "thing") ) val df2 = spark.range(10).withColumn("part", 'id / 2) df2.writeTo(tbl) .tableProperty("other", "thing") .using("delta") .replace() verifyTableMetadata( expectedSchema = df2.schema.asNullable, expectedProperties = getBaseProperties(snapshot) ++ Map("other" -> "thing") ) } } test("creating table in metastore over existing path") { withTempDir { dir => withTable(tbl) { val df = spark.range(10).withColumn("part", 'id % 2).withColumn("id2", 'id) df.write.format("delta").partitionBy("part").save(dir.getCanonicalPath) sql(s"CREATE TABLE $tbl USING delta LOCATION '${dir.getCanonicalPath}'") verifyTableMetadata(df.schema.asNullable) } } } test("replacing non-Delta table") { withTable(tbl) { val df = spark.range(10).withColumn("part", 'id / 2).withColumn("id2", 'id) df.writeTo(tbl) .tableProperty("delta.checkpointInterval", "5") .tableProperty("some", "thing") .partitionedBy('part) .using("parquet") .create() val e = intercept[AnalysisException] { df.writeTo(tbl).using("delta").replace() } assert(e.getMessage.contains("not a Delta table")) } } test("alter table add columns") { withDeltaTable { df => sql(s"ALTER TABLE $tbl ADD COLUMNS (id2 bigint)") verifyTableMetadataAsync(df.withColumn("id2", 'id).schema.asNullable) } } protected def runAlterTableTests(f: (String, StructType) => Unit): Unit = { // We set the default minWriterVersion to the version required to ADD/DROP CHECK constraints // to prevent an automatic protocol upgrade (i.e. an implicit property change) when adding // the CHECK constraint below. withSQLConf( "spark.databricks.delta.properties.defaults.minReaderVersion" -> "1", "spark.databricks.delta.properties.defaults.minWriterVersion" -> "3") { withDeltaTable { _ => sql(s"ALTER TABLE $tbl SET TBLPROPERTIES ('some' = 'thing', 'other' = 'thing')") sql(s"ALTER TABLE $tbl UNSET TBLPROPERTIES ('other')") sql(s"ALTER TABLE $tbl ADD COLUMNS (id2 bigint, id3 bigint)") sql(s"ALTER TABLE $tbl CHANGE COLUMN id2 id2 bigint FIRST") sql(s"ALTER TABLE $tbl REPLACE COLUMNS (id3 bigint, id2 bigint, id bigint)") sql(s"ALTER TABLE $tbl ADD CONSTRAINT id_3 CHECK (id3 > 10)") sql(s"ALTER TABLE $tbl DROP CONSTRAINT id_3") val expectedSchema = StructType(Seq( StructField("id3", LongType, true), StructField("id2", LongType, true), StructField("id", LongType, true)) ) f(tbl, expectedSchema) } } } /** * Creates a table with the name `tbl` and executes a function that takes a representative * DataFrame with the schema of the table. Performs cleanup of the table afterwards. */ protected def withDeltaTable(f: DataFrame => Unit): Unit = { // Turn off async updates so that we don't update the catalog during table cleanup disableUpdates { withTable(tbl) { withSQLConf(DeltaSQLConf.DELTA_UPDATE_CATALOG_ENABLED.key -> "true") { sql(s"CREATE TABLE $tbl (id bigint) USING delta") val df = spark.range(10) verifyTableMetadata(df.schema.asNullable) f(df.toDF()) } } } } test("skip update when flag is not set") { withDeltaTable(df => { withSQLConf(DeltaSQLConf.DELTA_UPDATE_CATALOG_ENABLED.key -> "false") { val propertiesAtV1 = getBaseProperties(snapshot) sql(s"ALTER TABLE $tbl SET TBLPROPERTIES(some.key = 1)") verifyTableMetadataAsync( expectedSchema = df.schema.asNullable, expectedProperties = propertiesAtV1) } }) } test(s"REORG TABLE does not perform catalog update") { val tableName = "myTargetTable" withDeltaTable { df => sql(s"REORG TABLE $tbl APPLY (PURGE)") verifyTableMetadataAsync(df.schema.asNullable) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaUsageLogsOpsTypes.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta object DeltaUsageLogsOpTypes { final val BACKFILL_COMMAND = "delta.rowTracking.backfill.stats" final val BACKFILL_BATCH = "delta.rowTracking.backfill.batch.stats" } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaVacuumSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.util.Locale import java.util.concurrent.TimeUnit import scala.collection.mutable.ArrayBuffer import scala.language.implicitConversions import org.apache.spark.sql.delta.DeltaOperations.{Delete, Write} import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile import org.apache.spark.sql.delta.DeltaUnsupportedOperationException import org.apache.spark.sql.delta.actions.{AddCDCFile, AddFile, Metadata, RemoveFile} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.VacuumCommand import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.{DeltaCommitFileProvider, DeltaFileOperations, FileNames} import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.commons.io.FileUtils import org.apache.hadoop.fs.FileSystem import org.apache.hadoop.fs.Path import org.scalatest.GivenWhenThen import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row, SaveMode, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTableType import org.apache.spark.sql.catalyst.expressions.Literal import org.apache.spark.sql.catalyst.util.IntervalUtils import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric import org.apache.spark.sql.functions.{col, expr, lit} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String import org.apache.spark.util.ManualClock trait DeltaVacuumSuiteBase extends QueryTest with SharedSparkSession with GivenWhenThen with DeltaSQLTestUtils with DeletionVectorsTestUtils with DeltaTestUtilsForTempViews with CatalogOwnedTestBaseSuite { private def executeWithEnvironment(file: File)(f: (File, ManualClock) => Unit): Unit = { val clock = new ManualClock() withSQLConf(DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED.key -> "false") { f(file, clock) } } protected def isLiteVacuum: Boolean = false protected def testFullVacuumOnly( testName: String, testTags: org.scalatest.Tag*)( testFun: => Any): Unit = { // Certain tests are not valid for lite vacuum as lite vacuum doesn't care about // the files not tracked by the delta log. if (isLiteVacuum) { ignore(testName + " (full Vacuum only)", testTags: _*)(testFun) } else { test(testName, testTags: _*)(testFun) } } protected def withEnvironment(f: (File, ManualClock) => Unit): Unit = withTempDir(file => executeWithEnvironment(file)(f)) protected def withEnvironment(prefix: String)(f: (File, ManualClock) => Unit): Unit = withTempDir(prefix)(file => executeWithEnvironment(file)(f)) protected def defaultTombstoneInterval: Long = { DeltaConfigs.getMilliSeconds( IntervalUtils.safeStringToInterval( UTF8String.fromString(DeltaConfigs.TOMBSTONE_RETENTION.defaultValue))) } /** Lists the data files in a given dir recursively. */ protected def listDataFiles(spark: SparkSession, tableDir: String): Seq[String] = { val result = ArrayBuffer.empty[String] // scalastyle:off deltahadoopconfiguration val fs = FileSystem.get(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration val iterator = fs.listFiles(fs.makeQualified(new Path(tableDir)), true) while (iterator.hasNext) { val path = iterator.next().getPath.toUri.toString if (path.endsWith(".parquet") && !path.contains(".checkpoint")) { result += path } } result.toSeq } protected def assertNumFiles( deltaLog: DeltaLog, addFiles: Int, addFilesWithDVs: Int, dvFiles: Int, dataFiles: Int): Unit = { assert(deltaLog.update().allFiles.count() === addFiles) assert(getFilesWithDeletionVectors(deltaLog).size === addFilesWithDVs) assert(listDeletionVectors(deltaLog).size === dvFiles) assert(listDataFiles(spark, deltaLog.dataPath.toString).size === dataFiles) } implicit def fileToPathString(f: File): String = new Path(f.getAbsolutePath).toString trait Operation /** * Write a file to the given absolute or relative path. Could be inside or outside the Reservoir * base path. The file can be committed to the action log to be tracked, or left out for deletion. */ case class CreateFile( path: String, commitToActionLog: Boolean, partitionValues: Map[String, String] = Map.empty) extends Operation /** Create a directory at the given path. */ case class CreateDirectory(path: String) extends Operation /** * Logically deletes a file in the action log. Paths can be absolute or relative paths, and can * point to files inside and outside a reservoir. */ case class LogicallyDeleteFile(path: String) extends Operation /** Check that the given paths exist. */ case class CheckFiles(paths: Seq[String], exist: Boolean = true) extends Operation /** Garbage collect the reservoir. */ case class GC( dryRun: Boolean, expectedDf: Seq[String], retentionHours: Option[Double] = None) extends Operation case class GCByInventory(dryRun: Boolean, expectedDf: Seq[String], retentionHours: Option[Double] = None, inventory: Option[DataFrame] = Option.empty[DataFrame]) extends Operation /** Garbage collect the reservoir. */ case class ExecuteVacuumInScala( deltaTable: io.delta.tables.DeltaTable, expectedDf: Seq[String], retentionHours: Option[Double] = None) extends Operation /** Advance the time. */ case class AdvanceClock(timeToAdd: Long) extends Operation /** Execute SQL command */ case class ExecuteVacuumInSQL( identifier: String, expectedDf: Seq[String], retentionHours: Option[Long] = None, dryRun: Boolean = false) extends Operation { def sql: String = { val retainStr = retentionHours.map { h => s"RETAIN $h HOURS"}.getOrElse("") val dryRunStr = if (dryRun) "DRY RUN" else "" s"VACUUM $identifier $retainStr $dryRunStr" } } /** * Expect a failure with the given exception type. Expect the given `msg` fragments as the error * message. */ case class ExpectFailure[T <: Throwable]( action: Operation, expectedError: Class[T], msg: Seq[String]) extends Operation private final val RANDOM_FILE_CONTENT = "gibberish" protected def createFile( reservoirBase: String, filePath: String, file: File, clock: ManualClock, partitionValues: Map[String, String] = Map.empty): AddFile = { FileUtils.write(file, RANDOM_FILE_CONTENT) file.setLastModified(clock.getTimeMillis()) createTestAddFile( encodedPath = filePath, partitionValues = partitionValues, modificationTime = clock.getTimeMillis()) } protected def gcTest(table: DeltaTableV2, clock: ManualClock)(actions: Operation*): Unit = { import testImplicits._ val basePath = table.deltaLog.dataPath.toString val fs = new Path(basePath).getFileSystem(table.deltaLog.newDeltaHadoopConf()) actions.foreach { case CreateFile(path, commit, partitionValues) => Given(s"*** Writing file to $path. Commit to log: $commit") val sanitizedPath = new Path(path).toUri.toString val file = new File( fs.makeQualified(DeltaFileOperations.absolutePath(basePath, sanitizedPath)).toUri) if (commit) { if (!DeltaTableUtils.isDeltaTable(spark, new Path(basePath))) { // initialize the table val version = table.startTransaction().commitManually() setCommitClock(table, version, clock) } val txn = table.startTransaction() val action = createFile(basePath, sanitizedPath, file, clock, partitionValues) val version = txn.commit(Seq(action), Write(SaveMode.Append)) setCommitClock(table, version, clock) } else { createFile(basePath, path, file, clock) } case CreateDirectory(path) => Given(s"*** Creating directory at $path") val dir = new File(DeltaFileOperations.absolutePath(basePath, path).toUri) assert(dir.mkdir(), s"Couldn't create directory at $path") assert(dir.setLastModified(clock.getTimeMillis())) case LogicallyDeleteFile(path) => Given(s"*** Removing files") val txn = table.startTransaction() // scalastyle:off val metrics = Map[String, SQLMetric]( "numRemovedFiles" -> createMetric(sparkContext, "number of files removed."), "numAddedFiles" -> createMetric(sparkContext, "number of files added."), "numDeletedRows" -> createMetric(sparkContext, "number of rows deleted."), "numCopiedRows" -> createMetric(sparkContext, "total number of rows.") ) txn.registerSQLMetrics(spark, metrics) val encodedPath = new Path(path).toUri.toString val size = Some(RANDOM_FILE_CONTENT.length.toLong) val version = txn.commit( Seq(RemoveFile(encodedPath, Option(clock.getTimeMillis()), size = size)), Delete(Seq(Literal.TrueLiteral))) setCommitClock(table, version, clock) // scalastyle:on case e: ExecuteVacuumInSQL => Given(s"*** Executing SQL: ${e.sql}") val qualified = e.expectedDf.map(p => fs.makeQualified(new Path(p)).toString) val df = spark.sql(e.sql).as[String] checkDatasetUnorderly(df, qualified: _*) case CheckFiles(paths, exist) => Given(s"*** Checking files exist=$exist") paths.foreach { p => val sp = new Path(p).toUri.toString val f = new File(fs.makeQualified(DeltaFileOperations.absolutePath(basePath, sp)).toUri) val res = if (exist) f.exists() else !f.exists() assert(res, s"Expectation: exist=$exist, paths: $p") } case GC(dryRun, expectedDf, retention) => Given("*** Garbage collecting Reservoir") val result = VacuumCommand.gc(spark, table, dryRun, retention, clock = clock) val qualified = expectedDf.map(p => fs.makeQualified(new Path(p)).toString) checkDatasetUnorderly(result.as[String], qualified: _*) case GCByInventory(dryRun, expectedDf, retention, inventory) => Given("*** Garbage collecting using inventory") val result = VacuumCommand.gc(spark, table, dryRun, retention, inventory, clock = clock) val qualified = expectedDf.map(p => fs.makeQualified(new Path(p)).toString) checkDatasetUnorderly(result.as[String], qualified: _*) case ExecuteVacuumInScala(deltaTable, expectedDf, retention) => Given("*** Garbage collecting Reservoir using Scala") val result = if (retention.isDefined) { deltaTable.vacuum(retention.get) } else { deltaTable.vacuum() } if(expectedDf == Seq()) { assert(result === spark.emptyDataFrame) } else { val qualified = expectedDf.map(p => fs.makeQualified(new Path(p)).toString) checkDatasetUnorderly(result.as[String], qualified: _*) } case AdvanceClock(timeToAdd: Long) => Given(s"*** Advancing clock by $timeToAdd millis") clock.advance(timeToAdd) case ExpectFailure(action, failure, msg) => Given(s"*** Expecting failure of ${failure.getName} for action: $action") val e = intercept[Exception](gcTest(table, clock)(action)) assert(e.getClass === failure) assert( msg.forall(m => e.getMessage.toLowerCase(Locale.ROOT).contains(m.toLowerCase(Locale.ROOT))), e.getMessage + "didn't contain: " + msg.mkString("[", ", ", "]")) } } protected def vacuumSQLTest(table: DeltaTableV2, tableName: String) { val committedFile = "committedFile.txt" val notCommittedFile = "notCommittedFile.txt" val expectedDf = Option.when(!isLiteVacuum)(new Path(table.path, notCommittedFile).toString) gcTest(table, new ManualClock())( // Prepare the table with files with timestamp of epoch-time 0 (i.e. 01-01-1970 00:00) CreateFile(committedFile, commitToActionLog = true), CreateFile(notCommittedFile, commitToActionLog = false), CheckFiles(Seq(committedFile, notCommittedFile)), // Dry run should return the not committed file and but not delete files ExecuteVacuumInSQL(tableName, expectedDf = expectedDf.toSeq, dryRun = true), CheckFiles(Seq(committedFile, notCommittedFile)), // Actual run should not delete the committed file but delete the not-committed file ExecuteVacuumInSQL(tableName, Seq(table.path.toString)), CheckFiles(Seq(committedFile)), // File ts older than default retention // However, non committed files are not deleted by lite vacuum. CheckFiles(Seq(notCommittedFile), exist = isLiteVacuum), // Logically delete the file. LogicallyDeleteFile(committedFile), CheckFiles(Seq(committedFile)), // Vacuum with 0 retention should actually delete the file. ExecuteVacuumInSQL(tableName, Seq(table.path.toString), Some(0)), CheckFiles(Seq(committedFile), exist = false)) } protected def vacuumScalaTest(deltaTable: io.delta.tables.DeltaTable, tablePath: String) { val table = DeltaTableV2(spark, new Path(tablePath), options = Map.empty, "test") val committedFile = "committedFile.txt" val notCommittedFile = "notCommittedFile.txt" gcTest(table, new ManualClock())( // Prepare the table with files with timestamp of epoch-time 0 (i.e. 01-01-1970 00:00) CreateFile(committedFile, commitToActionLog = true), CreateFile(notCommittedFile, commitToActionLog = false), CheckFiles(Seq(committedFile, notCommittedFile)), // Actual run should delete the not committed file and but not delete files ExecuteVacuumInScala(deltaTable, Seq()), CheckFiles(Seq(committedFile)), // File ts older than default retention // However, non committed files are not deleted by lite vacuum. CheckFiles(Seq(notCommittedFile), exist = isLiteVacuum), // Logically delete the file. LogicallyDeleteFile(committedFile), CheckFiles(Seq(committedFile)), // Vacuum with 0 retention should actually delete the file. ExecuteVacuumInScala(deltaTable, Seq(), Some(0)), CheckFiles(Seq(committedFile), exist = false)) } /** * Helper method to tell us if the given filePath exists. Thus, it can be used to detect if a * file has been deleted. */ protected def pathExists(deltaLog: DeltaLog, filePath: String): Boolean = { val fs = deltaLog.logPath.getFileSystem(deltaLog.newDeltaHadoopConf()) fs.exists(DeltaFileOperations.absolutePath(deltaLog.dataPath.toString, filePath)) } protected def deleteCommitFile(table: DeltaTableV2, version: Long) = { new File(DeltaCommitFileProvider(table.update()).deltaFile(version).toUri).delete() } /** * Helper method to get all of the [[AddCDCFile]]s that exist in the delta table */ protected def getCDCFiles(deltaLog: DeltaLog): Seq[AddCDCFile] = { val changes = deltaLog.getChanges( startVersion = 0, catalogTableOpt = None, failOnDataLoss = true) changes.flatMap(_._2).collect { case a: AddCDCFile => a }.toList } protected def setCommitClock(table: DeltaTableV2, version: Long, clock: ManualClock) = { val f = new File(DeltaCommitFileProvider(table.update()).deltaFile(version).toUri) f.setLastModified(clock.getTimeMillis()) } protected def testCDCVacuumForUpdateMerge(): Unit = { withSQLConf( DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> "true", DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED.key -> "false" ) { withTempDir { dir => // create table - version 0 spark.range(10) .repartition(1) .write .format("delta") .save(dir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(dir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) // update table - version 1 deltaTable.update(expr("id == 0"), Map("id" -> lit("11"))) // merge table - version 2 deltaTable.as("target") .merge( spark.range(0, 12).toDF().as("src"), "src.id = target.id") .whenMatched() .updateAll() .whenNotMatched() .insertAll() .execute() val df1 = sql(s"SELECT * FROM delta.`${dir.getAbsolutePath}`").collect() val changes = getCDCFiles(deltaLog) // vacuum will not delete the cdc files if they are within retention sql(s"VACUUM '${dir.getAbsolutePath}' RETAIN 100 HOURS") changes.foreach { change => assert(pathExists(deltaLog, change.path)) // cdc file exists } // vacuum will delete the cdc files if they are outside retention sql(s"VACUUM '${dir.getAbsolutePath}' RETAIN 0 HOURS") changes.foreach { change => assert(!pathExists(deltaLog, change.path)) // cdc file has been removed } // try reading the table checkAnswer(sql(s"SELECT * FROM delta.`${dir.getAbsolutePath}`"), df1) // try reading cdc data val e = intercept[SparkException] { spark.read .format("delta") .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingVersion", 1) .option("endingVersion", 2) .load(dir.getAbsolutePath) .count() } // QueryExecutionErrors.readCurrentFileNotFoundError var expectedErrorMessage = "It is possible the underlying files have been updated." assert(e.getMessage.contains(expectedErrorMessage)) } } } protected def testCDCVacuumForTombstones(): Unit = { withSQLConf( DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> "true", DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED.key -> "false" ) { withTempDir { dir => // create table - version 0 spark.range(0, 10, 1, 1) .withColumn("part", col("id") % 2) .write .format("delta") .partitionBy("part") .save(dir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(dir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) // create version 1 - delete single row should generate one cdc file deltaTable.delete(col("id") === lit(9)) val changes = getCDCFiles(deltaLog) assert(changes.size === 1) val cdcPath = changes.head.path assert(pathExists(deltaLog, cdcPath)) val df1 = sql(s"SELECT * FROM delta.`${dir.getAbsolutePath}`").collect() // vacuum will not delete the cdc files if they are within retention sql(s"VACUUM '${dir.getAbsolutePath}' RETAIN 100 HOURS") assert(pathExists(deltaLog, cdcPath)) // cdc path exists // vacuum will delete the cdc files when they are outside retention // one cdc file and one RemoveFile should be deleted by vacuum sql(s"VACUUM '${dir.getAbsolutePath}' RETAIN 0 HOURS") assert(!pathExists(deltaLog, cdcPath)) // cdc file is removed // try reading the table checkAnswer(sql(s"SELECT * FROM delta.`${dir.getAbsolutePath}`"), df1) // create version 2 - partition delete - does not create new cdc files deltaTable.delete(col("part") === lit(0)) assert(getCDCFiles(deltaLog).size === 1) // still just the one cdc file from before. // try reading cdc data val e = intercept[SparkException] { spark.read .format("delta") .option(DeltaOptions.CDC_READ_OPTION, "true") .option("startingVersion", 1) .option("endingVersion", 2) .load(dir.getAbsolutePath) .count() } // QueryExecutionErrors.readCurrentFileNotFoundError var expectedErrorMessage = "It is possible the underlying files have been updated." assert(e.getMessage.contains(expectedErrorMessage)) } } } } class DeltaVacuumSuite extends DeltaVacuumSuiteBase with DeltaSQLCommandTest { import testImplicits._ override def sparkConf: SparkConf = { super.sparkConf.set("spark.sql.sources.parallelPartitionDiscovery.parallelism", "2") } testQuietly("basic case - SQL command on path-based tables with direct 'path'") { withEnvironment { (tempDir, _) => val table = DeltaTableV2(spark, tempDir) vacuumSQLTest(table, tableName = s"'$tempDir'") } } testQuietly("basic case - SQL command on path-based table with delta.`path`") { withEnvironment { (tempDir, _) => val table = DeltaTableV2(spark, tempDir) vacuumSQLTest(table, tableName = s"delta.`$tempDir`") } } testQuietly("basic case - SQL command on name-based table") { val tableName = "deltaTable" withEnvironment { (_, _) => withTable(tableName) { import testImplicits._ spark.emptyDataset[Int].write.format("delta").saveAsTable(tableName) val table = DeltaTableV2(spark, new TableIdentifier(tableName)) vacuumSQLTest(table, tableName) } } } testQuietlyWithTempView("basic case - SQL command on temp view not supported") { isSQLTempView => val tableName = "deltaTable" val viewName = "v" withEnvironment { (_, _) => withTable(tableName) { import testImplicits._ spark.emptyDataset[Int].write.format("delta").saveAsTable(tableName) createTempViewFromTable(tableName, isSQLTempView) val table = DeltaTableV2(spark, new TableIdentifier(tableName)) val e = intercept[AnalysisException] { vacuumSQLTest(table, viewName) } assert(e.getMessage.contains("'VACUUM' expects a table but `v` is a view")) } } } test("basic case - Scala on path-based table") { withEnvironment { (tempDir, _) => import testImplicits._ spark.emptyDataset[Int].write.format("delta").save(tempDir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(tempDir.getAbsolutePath) vacuumScalaTest(deltaTable, tempDir.getAbsolutePath) } } test("basic case - Scala on name-based table") { val tableName = "deltaTable" withEnvironment { (tempDir, _) => withTable(tableName) { // Initialize the table so that we can create the DeltaTable object import testImplicits._ spark.emptyDataset[Int].write.format("delta").saveAsTable(tableName) val deltaTable = io.delta.tables.DeltaTable.forName(tableName) val tablePath = new File(spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName)).location) vacuumScalaTest(deltaTable, tablePath) } } } test("don't delete data in a non-reservoir") { withEnvironment { (tempDir, clock) => val table = DeltaTableV2(spark, tempDir, clock) gcTest(table, clock)( CreateFile("file1.txt", commitToActionLog = false), CreateDirectory("abc"), ExpectFailure( GC(dryRun = false, Nil), classOf[IllegalArgumentException], Seq("no state defined")) ) } } test("invisible files and dirs") { withEnvironment { (tempDir, clock) => val table = DeltaTableV2(spark, tempDir, clock) gcTest(table, clock)( CreateFile("file1.txt", commitToActionLog = true), CreateFile("_hidden_dir/000001.text", commitToActionLog = false), CreateFile(".hidden.txt", commitToActionLog = false), AdvanceClock(defaultTombstoneInterval + 1000), GC(dryRun = false, Seq(tempDir)), CheckFiles(Seq( "file1.txt", "_delta_log", "_hidden_dir", "_hidden_dir/000001.text", ".hidden.txt")) ) } } test("partition column name starting with underscore") { // We should be able to see inside partition directories to GC them, even if they'd normally // be considered invisible because of their name. withEnvironment { (tempDir, clock) => val table = DeltaTableV2(spark, tempDir, clock) val txn = table.startTransaction() val schema = new StructType().add("_underscore_col_", IntegerType).add("n", IntegerType) val metadata = Metadata(schemaString = schema.json, partitionColumns = Seq("_underscore_col_")) val version = txn.commitActions( DeltaOperations.CreateTable(metadata, isManaged = true), metadata) setCommitClock(table, version, clock) gcTest(table, clock)( CreateFile("file1.txt", commitToActionLog = true, Map("_underscore_col_" -> "10")), CreateFile("_underscore_col_=10/test.txt", true, Map("_underscore_col_" -> "10")), CheckFiles(Seq("file1.txt", "_underscore_col_=10")), LogicallyDeleteFile("_underscore_col_=10/test.txt"), AdvanceClock(defaultTombstoneInterval + 1000), GC(dryRun = false, Seq(tempDir)), CheckFiles(Seq("file1.txt")), CheckFiles(Seq("_underscore_col_=10/test.txt"), exist = false) ) } } test("schema validation for vacuum by using inventory dataframe") { withEnvironment { (tempDir, clock) => val table = DeltaTableV2(spark, tempDir, clock) val txn = table.startTransaction() val schema = new StructType().add("_underscore_col_", IntegerType).add("n", IntegerType) val metadata = Metadata(schemaString = schema.json, partitionColumns = Seq("_underscore_col_")) val version = txn.commitActions( DeltaOperations.CreateTable(metadata, isManaged = true), metadata) setCommitClock(table, version, clock) val inventorySchema = StructType( Seq( StructField("file", StringType), StructField("size", LongType), StructField("isDir", BooleanType), StructField("modificationTime", LongType) )) val inventory = spark.createDataFrame( spark.sparkContext.parallelize(Seq.empty[Row]), inventorySchema) gcTest(table, clock)( ExpectFailure( GCByInventory(dryRun = false, expectedDf = Seq(tempDir), inventory = Some(inventory)), classOf[DeltaAnalysisException], Seq( "The schema for the specified INVENTORY", "does not contain all of the required fields.", "Required fields are:", s"${VacuumCommand.INVENTORY_SCHEMA.treeString}") ) ) } } test("run vacuum by using inventory dataframe") { withEnvironment { (tempDir, clock) => val table = DeltaTableV2(spark, tempDir, clock) val txn = table.startTransaction() val schema = new StructType().add("_underscore_col_", IntegerType).add("n", IntegerType) // Vacuum should consider partition folders even for clean up even though it starts with `_` val metadata = Metadata(schemaString = schema.json, partitionColumns = Seq("_underscore_col_")) val version = txn.commitActions( DeltaOperations.CreateTable(metadata, isManaged = true), metadata) setCommitClock(table, version, clock) val dataPath = table.deltaLog.dataPath // Create a Seq of Rows containing the data val data = Seq( Row(s"${dataPath}", 300000L, true, 0L), Row(s"${dataPath}/file1.txt", 300000L, false, 0L), Row(s"${dataPath}/file2.txt", 300000L, false, 0L), Row(s"${dataPath}/_underscore_col_=10/test.txt", 300000L, false, 0L), Row(s"${dataPath}/_underscore_col_=10/test2.txt", 300000L, false, 0L), // Below file is not within Delta table path and should be ignored by vacuum Row(s"/tmp/random/_underscore_col_=10/test2.txt", 300000L, false, 0L), // Below are Delta table root location and vacuum must safely handle them Row(s"${dataPath}", 300000L, true, 0L) ) val inventory = spark.createDataFrame(spark.sparkContext.parallelize(data), VacuumCommand.INVENTORY_SCHEMA) gcTest(table, clock)( CreateFile("file1.txt", commitToActionLog = true, Map("_underscore_col_" -> "10")), CreateFile("file2.txt", commitToActionLog = false, Map("_underscore_col_" -> "10")), CreateFile("_underscore_col_=10/test.txt", true, Map("_underscore_col_" -> "10")), CreateFile("_underscore_col_=10/test2.txt", false, Map("_underscore_col_" -> "10")), CheckFiles(Seq("file1.txt", "_underscore_col_=10", "file2.txt")), LogicallyDeleteFile("_underscore_col_=10/test.txt"), AdvanceClock(defaultTombstoneInterval + 1000), GCByInventory(dryRun = true, expectedDf = Seq( s"${dataPath}/file2.txt", s"${dataPath}/_underscore_col_=10/test.txt", s"${dataPath}/_underscore_col_=10/test2.txt" ), inventory = Some(inventory)), GCByInventory(dryRun = false, expectedDf = Seq(tempDir), inventory = Some(inventory)), CheckFiles(Seq("file1.txt")), CheckFiles(Seq("file2.txt", "_underscore_col_=10/test.txt", "_underscore_col_=10/test2.txt"), exist = false) ) } } test("vacuum using inventory delta table and should not touch hidden files") { withSQLConf(DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED.key -> "false") { withEnvironment { (tempDir, clock) => import testImplicits._ val path = s"""${tempDir.getCanonicalPath}_data""" val inventoryPath = s"""${tempDir.getCanonicalPath}_inventory""" // Define test delta table val data = Seq( (10, 1, "a"), (10, 2, "a"), (10, 3, "a"), (10, 4, "a"), (10, 5, "a") ) data.toDF("v1", "v2", "v3") .write .partitionBy("v1", "v2") .format("delta") .save(path) val table = DeltaTableV2(spark, new File(path), clock) val dataPath = table.deltaLog.dataPath val reservoirData = Seq( Row(s"${dataPath}/file1.txt", 300000L, false, 0L), Row(s"${dataPath}/file2.txt", 300000L, false, 0L), Row(s"${dataPath}/_underscore_col_=10/test.txt", 300000L, false, 0L), Row(s"${dataPath}/_underscore_col_=10/test2.txt", 300000L, false, 0L) ) spark.createDataFrame( spark.sparkContext.parallelize(reservoirData), VacuumCommand.INVENTORY_SCHEMA) .write .format("delta") .save(inventoryPath) gcTest(table, clock)( CreateFile("file1.txt", commitToActionLog = false), CreateFile("file2.txt", commitToActionLog = false), // Delta marks dirs starting with `_` as hidden unless specified as partition folder CreateFile("_underscore_col_=10/test.txt", false), CreateFile("_underscore_col_=10/test2.txt", false), AdvanceClock(defaultTombstoneInterval + 1000) ) sql(s"vacuum delta.`$path` using inventory delta.`$inventoryPath` retain 0 hours") gcTest(table, clock)( CheckFiles(Seq("file1.txt", "file2.txt"), exist = false), // hidden files must not be dropped CheckFiles(Seq("_underscore_col_=10/test.txt", "_underscore_col_=10/test2.txt")) ) } } } test("vacuum using inventory query and should not touch hidden files") { withSQLConf(DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED.key -> "false") { withEnvironment { (tempDir, clock) => import testImplicits._ val path = s"""${tempDir.getCanonicalPath}_data""" val reservoirPath = s"""${tempDir.getCanonicalPath}_reservoir""" // Define test delta table val data = Seq( (10, 1, "a"), (10, 2, "a"), (10, 3, "a"), (10, 4, "a"), (10, 5, "a") ) data.toDF("v1", "v2", "v3") .write .partitionBy("v1", "v2") .format("delta") .save(path) val table = DeltaTableV2(spark, new File(path), clock) val dataPath = table.deltaLog.dataPath val reservoirData = Seq( Row(s"${dataPath}/file1.txt", 300000L, false, 0L), Row(s"${dataPath}/file2.txt", 300000L, false, 0L), Row(s"${dataPath}/_underscore_col_=10/test.txt", 300000L, false, 0L), Row(s"${dataPath}/_underscore_col_=10/test2.txt", 300000L, false, 0L) ) spark.createDataFrame( spark.sparkContext.parallelize(reservoirData), VacuumCommand.INVENTORY_SCHEMA) .write .format("delta") .save(reservoirPath) gcTest(table, clock)( CreateFile("file1.txt", commitToActionLog = false), CreateFile("file2.txt", commitToActionLog = false), // Delta marks dirs starting with `_` as hidden unless specified as partition folder CreateFile("_underscore_col_=10/test.txt", false), CreateFile("_underscore_col_=10/test2.txt", false) ) sql(s"""vacuum delta.`$path` |using inventory (select * from delta.`$reservoirPath`) |retain 0 hours""".stripMargin) gcTest(table, clock)( AdvanceClock(defaultTombstoneInterval + 1000), CheckFiles(Seq("file1.txt", "file2.txt"), exist = false), // hidden files must not be dropped CheckFiles(Seq("_underscore_col_=10/test.txt", "_underscore_col_=10/test2.txt")) ) } } } // Since lite vacuum uses delta log, it doesn't delete empty directories. testFullVacuumOnly("multiple levels of empty directory deletion") { withEnvironment { (tempDir, clock) => val table = DeltaTableV2(spark, tempDir, clock) gcTest(table, clock)( CreateFile("file1.txt", commitToActionLog = true), CreateFile("abc/def/file2.txt", commitToActionLog = false), AdvanceClock(defaultTombstoneInterval + 1000), GC(dryRun = false, Seq(tempDir)), CheckFiles(Seq("file1.txt", "abc", "abc/def")), CheckFiles(Seq("abc/def/file2.txt"), exist = false), GC(dryRun = false, Seq(tempDir)), // we need two GCs to guarantee the deletion of the directories GC(dryRun = false, Seq(tempDir)), CheckFiles(Seq("file1.txt")), CheckFiles(Seq("abc", "abc/def"), exist = false) ) } } test("gc doesn't delete base path") { withEnvironment { (tempDir, clock) => val table = DeltaTableV2(spark, tempDir, clock) gcTest(table, clock)( CreateFile("file1.txt", commitToActionLog = true), AdvanceClock(100), LogicallyDeleteFile("file1.txt"), AdvanceClock(defaultTombstoneInterval + 1000), GC(dryRun = false, Seq(tempDir.toString)), CheckFiles(Seq("file1.txt"), exist = false), GC(dryRun = false, Seq(tempDir.toString)) // shouldn't throw an error ) } } testQuietly("correctness test") { withEnvironment { (tempDir, clock) => val reservoirDir = new File(tempDir.getAbsolutePath, "reservoir") assert(reservoirDir.mkdirs()) val externalDir = new File(tempDir.getAbsolutePath, "external") assert(externalDir.mkdirs()) val table = DeltaTableV2(spark, reservoirDir, clock) val externalFile = new File(externalDir, "file4.txt").getAbsolutePath gcTest(table, clock)( // Create initial state CreateFile("file1.txt", commitToActionLog = true), CreateDirectory("abc"), CreateFile("abc/file2.txt", commitToActionLog = true), CheckFiles(Seq("file1.txt", "abc", "abc/file2.txt")), // Nothing should be deleted here, since we didn't logically delete any file AdvanceClock(defaultTombstoneInterval + 1000), GC(dryRun = false, Seq(reservoirDir.toString)), CheckFiles(Seq("file1.txt", "abc", "abc/file2.txt")), // Create an untracked file CreateFile("file3.txt", commitToActionLog = false), CheckFiles(Seq("file3.txt")), GC(dryRun = false, Seq(reservoirDir.toString)), CheckFiles(Seq("file3.txt")), AdvanceClock(defaultTombstoneInterval - 1000), // file is still new GC(dryRun = false, Seq(reservoirDir.toString)), CheckFiles(Seq("file3.txt")), AdvanceClock(2000), // Since file3.txt is not committed, it's not tracked by lite vacuum. GC(dryRun = true, Option.when(!isLiteVacuum)(new File(reservoirDir, "file3.txt").toString).toSeq), // nothing should be deleted CheckFiles(Seq("file1.txt", "abc", "abc/file2.txt", "file3.txt")), GC(dryRun = false, Seq(reservoirDir.toString)), CheckFiles(Seq("file1.txt", "abc", "abc/file2.txt")), // Since file3.txt is not committed, it would be deleted only if it's non-lite-vacuum CheckFiles(Seq("file3.txt"), exist = isLiteVacuum), // Verify tombstones LogicallyDeleteFile("abc/file2.txt"), GC(dryRun = false, Seq(reservoirDir.toString)), CheckFiles(Seq("file1.txt", "abc", "abc/file2.txt")), AdvanceClock(defaultTombstoneInterval - 1000), GC(dryRun = false, Seq(reservoirDir.toString)), CheckFiles(Seq("file1.txt", "abc", "abc/file2.txt")), AdvanceClock(2000), // tombstone should expire GC(dryRun = false, Seq(reservoirDir.toString)), CheckFiles(Seq("file1.txt", "abc")), CheckFiles(Seq("abc/file2.txt"), exist = false), // Second gc should clear empty directory if it's not lite vacuum GC(dryRun = false, Seq(reservoirDir.toString)), CheckFiles(Seq("file1.txt")), CheckFiles(Seq("abc"), exist = isLiteVacuum), // Make sure that files outside the reservoir are not affected CreateFile(externalFile, commitToActionLog = true), AdvanceClock(100), CheckFiles(Seq("file1.txt", externalFile)), LogicallyDeleteFile(externalFile), AdvanceClock(defaultTombstoneInterval * 2), CheckFiles(Seq("file1.txt", externalFile)) ) } } test("parallel file delete") { withEnvironment { (tempDir, clock) => val table = DeltaTableV2(spark, tempDir, clock) withSQLConf("spark.databricks.delta.vacuum.parallelDelete.enabled" -> "true") { gcTest(table, clock)( CreateFile("file1.txt", commitToActionLog = true), CreateFile("file2.txt", commitToActionLog = true), LogicallyDeleteFile("file1.txt"), CheckFiles(Seq("file1.txt", "file2.txt")), AdvanceClock(defaultTombstoneInterval + 1000), GC(dryRun = false, Seq(tempDir)), CheckFiles(Seq("file1.txt"), exist = false), CheckFiles(Seq("file2.txt")), GC(dryRun = false, Seq(tempDir)), // shouldn't throw an error with no files to delete CheckFiles(Seq("file2.txt")) ) } } } test("retention duration must be greater than 0") { withSQLConf("spark.databricks.delta.vacuum.retentionWindowIgnore.enabled" -> "false") { withEnvironment { (tempDir, clock) => val table = DeltaTableV2(spark, tempDir, clock) gcTest(table, clock)( CreateFile("file1.txt", commitToActionLog = true), CheckFiles(Seq("file1.txt")), ExpectFailure( GC(false, Seq(tempDir), Some(-2)), classOf[DeltaIllegalArgumentException], Seq("Retention", "less than", "0")) ) val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tempDir.getAbsolutePath) gcTest(table, clock)( CreateFile("file2.txt", commitToActionLog = true), CheckFiles(Seq("file2.txt")), ExpectFailure( ExecuteVacuumInScala(deltaTable, Seq(), Some(-2)), classOf[DeltaIllegalArgumentException], Seq("Retention", "less than", "0")) ) } } } test("deleting directories") { withEnvironment { (tempDir, clock) => val table = DeltaTableV2(spark, tempDir, clock) gcTest(table, clock)( CreateFile("abc/def/file1.txt", commitToActionLog = true), CreateFile("abc/def/file2.txt", commitToActionLog = true), CreateDirectory("ghi"), CheckFiles(Seq("abc", "abc/def", "ghi")), // Since "ghi" is a empty directory not tracked by the delta log, // lite Vacuum won't delete it. GC(dryRun = true, Option.when(!isLiteVacuum)(new File(tempDir, "ghi").toString).toSeq), GC(dryRun = false, Seq(tempDir)), CheckFiles(Seq("abc", "abc/def")), CheckFiles(Seq("ghi"), exist = isLiteVacuum) ) } } test("deleting files with special characters in path") { withEnvironment { (tempDir, clock) => val table = DeltaTableV2(spark, tempDir, clock) // Non committed files are not deleted by lite vacuum. val expected = Option.when(!isLiteVacuum)(new File(tempDir, "abc def/#1/file2.txt").toString) gcTest(table, clock)( CreateFile("abc def/#1/file1.txt", commitToActionLog = true), CreateFile("abc def/#1/file2.txt", commitToActionLog = false), CheckFiles(Seq("abc def", "abc def/#1")), AdvanceClock(defaultTombstoneInterval + 1000), GC(dryRun = true, expected.toSeq), GC(dryRun = false, Seq(tempDir)), CheckFiles(Seq("abc def/#1", "abc def/#1/file1.txt")), CheckFiles(Seq("abc def/#1/file2.txt"), exist = isLiteVacuum) ) } } testQuietly("additional retention duration check with vacuum command") { withEnvironment { (tempDir, clock) => val table = DeltaTableV2(spark, tempDir, clock) withSQLConf("spark.databricks.delta.retentionDurationCheck.enabled" -> "true") { gcTest(table, clock)( CreateFile("file1.txt", commitToActionLog = true), CheckFiles(Seq("file1.txt")), ExpectFailure( GC(false, Nil, Some(0)), classOf[DeltaIllegalArgumentException], Seq("delta.retentionDurationCheck.enabled = false", "168 hours")) ) } gcTest(table, clock)( CreateFile("file2.txt", commitToActionLog = true), CheckFiles(Seq("file2.txt")), GC(false, Seq(tempDir.toString), Some(0)) ) } } test("vacuum for a partition path") { withEnvironment { (tempDir, _) => import testImplicits._ val path = tempDir.getCanonicalPath Seq((1, "a"), (2, "b")).toDF("v1", "v2") .write .format("delta") .partitionBy("v2") .save(path) val ex = intercept[AnalysisException] { sql(s"vacuum '$path/v2=a' retain 0 hours") } assert( ex.getMessage.contains( s"`$path/v2=a` is not a Delta table. VACUUM is only supported for Delta tables.")) } } test(s"vacuum table with DVs and zero retention policy throws exception by default") { val targetDF = spark.range(0, 100, 1, 2) .withColumn("value", col("id")) withTempDeltaTable(targetDF, enableDVs = true) { (targetTable, targetLog) => // Add some DVs. targetTable().delete("id < 10") val e = intercept[DeltaIllegalArgumentException] { spark.sql(s"VACUUM delta.`${targetLog.dataPath}` RETAIN 0 HOURS") } assert(e.getMessage.contains( "The specified VACUUM retention period is too low")) } } test(s"vacuum after purge with zero retention policy") { val tableName = "testTable" withDeletionVectorsEnabled() { withSQLConf( DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED.key -> "false") { withTable(tableName) { // Create a Delta Table with 5 files of 10 rows, and delete half rows from first 4 files. spark.range(0, 50, step = 1, numPartitions = 5) .write.format("delta").saveAsTable(tableName) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) spark.sql(s"DELETE from $tableName WHERE ID % 2 = 0 and ID < 40") assertNumFiles(deltaLog, addFiles = 5, addFilesWithDVs = 4, dvFiles = 1, dataFiles = 5) purgeDVs(tableName) assertNumFiles(deltaLog, addFiles = 5, addFilesWithDVs = 0, dvFiles = 1, dataFiles = 9) spark.sql(s"VACUUM $tableName RETAIN 0 HOURS") assertNumFiles(deltaLog, addFiles = 5, addFilesWithDVs = 0, dvFiles = 0, dataFiles = 5) checkAnswer( spark.read.table(tableName), Seq.range(0, 50).filterNot(x => x < 40 && x % 2 == 0).toDF) } } } } // Since lite vacuum uses delta log, it doesn't delete uniform metadata directories // as they are not reachable through delta log. testFullVacuumOnly("gc metadata dir when uniform disabled") { withEnvironment { (tempDir, clock) => spark.emptyDataset[Int].write.format("delta").save(tempDir) val table = DeltaTableV2(spark, tempDir, clock) gcTest(table, clock)( CreateDirectory("metadata"), CreateFile("metadata/file1.json", false), AdvanceClock(defaultTombstoneInterval + 1000), GC(dryRun = false, Seq(tempDir)), CheckFiles(Seq("metadata/file1.json"), exist = false), GC(dryRun = false, Seq(tempDir)), // Second GC clears empty dir CheckFiles(Seq("metadata"), exist = false) ) } } test("hudi metadata dir") { withEnvironment { (tempDir, clock) => spark.emptyDataset[Int].write.format("delta").save(tempDir) val table = DeltaTableV2(spark, tempDir, clock) gcTest(table, clock)( CreateDirectory(".hoodie"), CreateFile(".hoodie/00001.commit", false), AdvanceClock(defaultTombstoneInterval + 1000), GC(dryRun = false, Seq(tempDir)), CheckFiles(Seq(".hoodie", ".hoodie/00001.commit")) ) } } // Helper method to remove the DVs in Delta table and rewrite the data files def purgeDVs(tableName: String): Unit = { withSQLConf( // Set the max file size to low so that we always rewrite the single file without DVs // and not combining with other data files. DeltaSQLConf.DELTA_OPTIMIZE_MAX_FILE_SIZE.key -> "2") { spark.sql(s"REORG TABLE $tableName APPLY (PURGE)") } } test(s"vacuum after purging deletion vectors") { import org.apache.spark.sql.delta.test.DeltaTestImplicits.DeltaTableV2ObjectTestHelper val tableName = "testTable" withDeletionVectorsEnabled() { withSQLConf( DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED.key -> "false", // Disable the following check since the test relies on time travel beyond // deletedFileRetentionDuration. DeltaSQLConf.ENFORCE_TIME_TRAVEL_WITHIN_DELETED_FILE_RETENTION_DURATION.key -> "false") { withTable(tableName) { // Create Delta table with 5 files of 10 rows. spark.range(0, 50, step = 1, numPartitions = 5) .write .format("delta") .option("delta.deletedFileRetentionDuration", "interval 1 hours") .saveAsTable(tableName) // The following is done to ensure deltaLog object uses the same clock that Vacuum // logic uses. val deltaLogThrowaway = DeltaLog.forTable(spark, TableIdentifier(tableName)) val tablePath = deltaLogThrowaway.dataPath DeltaLog.clearCache() val clock = new ManualClock(System.currentTimeMillis()) val deltaLog = DeltaLog.forTable(spark, tablePath, clock) val deltaTable = DeltaTableV2(spark, TableIdentifier(tableName)) assertNumFiles(deltaLog, addFiles = 5, addFilesWithDVs = 0, dvFiles = 0, dataFiles = 5) // Delete 1 row from each file. DVs will be packed to one DV file. val deletedRows1 = Seq(0, 10, 20, 30, 40) val deletedRowsStr1 = deletedRows1.mkString("(", ",", ")") spark.sql(s"DELETE FROM $tableName WHERE id IN $deletedRowsStr1") val snapshotV1 = deltaTable.update() // We retrieve both timestamp and file modification time b/c when ICT is enabled, // timestamp represents ICT instead of file modification time. Lite vacuum relies on // both the ICT and the file modification time to determine cleanup behavior. val timestampV1 = snapshotV1.timestamp val fileModificationTimeV1 = snapshotV1.logSegment.lastCommitFileModificationTimestamp assertNumFiles(deltaLog, addFiles = 5, addFilesWithDVs = 5, dvFiles = 1, dataFiles = 5) // Delete all rows from the first file. An ephemeral DV will still be created. // We need to add 1000 ms for local filesystems that only write modificationTimes to the // second precision. Thread.sleep(1000) // Ensure it's been at least 1000 ms since V1 // Assign clock to the current system time so that the ICT falls within // (fileModificationTimeV(X-1) + 1000, fileModificationTimeV(X+1)]. This ensures we can // later manually adjust the clock time to test both full and lite vacuum behavior. clock.setTime(System.currentTimeMillis()) spark.sql(s"DELETE FROM $tableName WHERE id < 10") val snapshotV2 = deltaTable.update() val timestampV2 = snapshotV2.timestamp val fileModificationTimeV2 = snapshotV2.logSegment.lastCommitFileModificationTimestamp assertNumFiles(deltaLog, addFiles = 4, addFilesWithDVs = 4, dvFiles = 2, dataFiles = 5) val expectedAnswerV2 = Seq.range(0, 50).filterNot(deletedRows1.contains).filterNot(_ < 10) // Delete 1 more row from each file. Thread.sleep(1000) // Ensure it's been at least 1000 ms since V2 clock.setTime(System.currentTimeMillis()) val deletedRows2 = Seq(11, 21, 31, 41) val deletedRowsStr2 = deletedRows2.mkString("(", ",", ")") spark.sql(s"DELETE FROM $tableName WHERE id IN $deletedRowsStr2") val snapshotV3 = deltaTable.update() val timestampV3 = snapshotV3.timestamp val fileModificationTimeV3 = snapshotV3.logSegment.lastCommitFileModificationTimestamp assertNumFiles(deltaLog, addFiles = 4, addFilesWithDVs = 4, dvFiles = 3, dataFiles = 5) val expectedAnswerV3 = expectedAnswerV2.filterNot(deletedRows2.contains) // Delete DVs by rewriting the data files with DVs. Thread.sleep(1000) // Ensure it's been at least 1000 ms since V3 clock.setTime(System.currentTimeMillis()) purgeDVs(tableName) val numFilesAfterPurge = 4 val snapshotV4 = deltaTable.update() val timestampV4 = snapshotV4.timestamp val fileModificationTimeV4 = snapshotV4.logSegment.lastCommitFileModificationTimestamp assertNumFiles(deltaLog, addFiles = numFilesAfterPurge, addFilesWithDVs = 0, dvFiles = 3, dataFiles = 9) // Run VACUUM with nothing expired. It should not delete anything. clock.setTime(System.currentTimeMillis()) VacuumCommand.gc( spark, deltaTable, retentionHours = Some(1), clock = clock, dryRun = false) assertNumFiles(deltaLog, addFiles = numFilesAfterPurge, addFilesWithDVs = 0, dvFiles = 3, dataFiles = 9) val oneHour = TimeUnit.HOURS.toMillis(1) // Run VACUUM @ V1. // The clock time must be set such that: (X is the version where we run VACUUM) // 1. (clock time - retention time) falls within // (fileModificationTimeV(X), fileModificationTimeV(X+1)] for both lite and full // vacuum, since both use file modification time to determine if the files are valid // for cleanup. // 2. (clock time - retention time) falls within [timestampV(X), timestampV(X+1)) for // lite vacuum, since it uses [[DeltaHistoryManager(deltaLog).getActiveCommitAtTime]] // which depends on timestamp to capture files of commit-X as candidates for cleanup. clock.setTime(Math.max(fileModificationTimeV1 + 1, timestampV1) + oneHour) VacuumCommand.gc( spark, deltaTable, retentionHours = Some(1), clock = clock, dryRun = false) assertNumFiles(deltaLog, addFiles = numFilesAfterPurge, addFilesWithDVs = 0, dvFiles = 3, dataFiles = 9) // Run VACUUM @ V2. It should delete the ephemeral DV and the removed Parquet file. // Since ephemeral DV is not GC'ed by Lite Vacuum, the number of DVs we expect will be // one more in case of lite Vacuum val numDVstoAdd = if (isLiteVacuum) 1 else 0 clock.setTime(Math.max(fileModificationTimeV2 + 1, timestampV2) + oneHour) VacuumCommand.gc( spark, deltaTable, retentionHours = Some(1), clock = clock, dryRun = false) assertNumFiles(deltaLog, addFiles = numFilesAfterPurge, addFilesWithDVs = 0, dvFiles = 2 + numDVstoAdd, dataFiles = 8) checkAnswer( spark.sql(s"SELECT * FROM $tableName VERSION AS OF 2"), expectedAnswerV2.toDF) // Run VACUUM @ V3. It should delete the persistent DVs from V1. clock.setTime(Math.max(fileModificationTimeV3 + 1, timestampV3) + oneHour) VacuumCommand.gc( spark, deltaTable, retentionHours = Some(1), clock = clock, dryRun = false) assertNumFiles(deltaLog, addFiles = numFilesAfterPurge, addFilesWithDVs = 0, dvFiles = 1 + numDVstoAdd, dataFiles = 8) checkAnswer( spark.sql(s"SELECT * FROM $tableName VERSION AS OF 3"), expectedAnswerV3.toDF) // Run VACUUM @ V4. It should delete the Parquet files and DVs of V3. clock.setTime(Math.max(fileModificationTimeV4 + 1, timestampV4) + oneHour) VacuumCommand.gc( spark, deltaTable, retentionHours = Some(1), clock = clock, dryRun = false) assertNumFiles(deltaLog, addFiles = numFilesAfterPurge, addFilesWithDVs = 0, dvFiles = 0 + numDVstoAdd, dataFiles = 4) checkAnswer( spark.sql(s"SELECT * FROM $tableName VERSION AS OF 4"), expectedAnswerV3.toDF) // Run VACUUM with zero retention period. It should not delete anything. clock.setTime(Math.max(fileModificationTimeV4 + 1, timestampV4) + oneHour) VacuumCommand.gc( spark, deltaTable, retentionHours = Some(0), clock = clock, dryRun = false) assertNumFiles(deltaLog, addFiles = numFilesAfterPurge, addFilesWithDVs = 0, dvFiles = 0 + numDVstoAdd, dataFiles = 4) // Last version should still be readable. checkAnswer(spark.sql(s"SELECT * FROM $tableName"), expectedAnswerV3.toDF) } } } } for (partitioned <- DeltaTestUtils.BOOLEAN_DOMAIN) { test(s"delete persistent deletion vectors - partitioned = $partitioned") { val targetDF = spark.range(0, 100, 1, 10).toDF .withColumn("v", col("id")) .withColumn("partCol", lit(0)) val partitionBy = if (partitioned) Seq("partCol") else Seq.empty withSQLConf( DeltaSQLConf.DELETION_VECTOR_PACKING_TARGET_SIZE.key -> "0", DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED.key -> "false") { withDeletionVectorsEnabled() { withTempDeltaTable( targetDF, partitionBy = partitionBy) { (targetTable, targetLog) => val targetDir = targetLog.dataPath // Add a DV to all files and check that DVs are not deleted. targetTable().delete("id % 2 == 0") assert(listDeletionVectors(targetLog).size == 10) targetTable().vacuum(0) assert(listDeletionVectors(targetLog).size == 10) checkAnswer(sql(s"select count(*) from delta.`$targetDir`"), Row(50)) // Update the DV of the first file by deleting two rows and check that previous DV is // deleted. targetTable().delete("id < 10 AND id % 3 == 0") assert(listDeletionVectors(targetLog).size == 11) targetTable().vacuum(0) assert(listDeletionVectors(targetLog).size == 10) checkAnswer(sql(s"select count(*) from delta.`$targetDir`"), Row(48)) // Delete all rows in first 5 files and check that DVs are not deleted due to // the retention period, but deleted after that. // with lite vacuum ephemeral dvs are not going to be GC'ed. So, the dvs we expect // will be 5 more for lite vacuum val dvsToAdd = if (isLiteVacuum) 5 else 0 targetTable().delete("id < 50") assert(listDeletionVectors(targetLog).size == 15) targetTable().vacuum(10) assert(listDeletionVectors(targetLog).size == 15) targetTable().vacuum(0) assert(listDeletionVectors(targetLog).size == 5 + dvsToAdd) checkAnswer(sql(s"select count(*) from delta.`$targetDir`"), Row(25)) } } } } } test("vacuum a non-existent path and a non Delta table") { def assertNotADeltaTableException(path: String): Unit = { for (table <- Seq(s"'$path'", s"delta.`$path`")) { val e = intercept[AnalysisException] { sql(s"vacuum $table") } assert(e.getMessage.contains("is not a Delta table.")) } } withTempPath { tempDir => assert(!tempDir.exists()) assertNotADeltaTableException(tempDir.getCanonicalPath) } withTempPath { tempDir => spark.range(1, 10).write.parquet(tempDir.getCanonicalPath) assertNotADeltaTableException(tempDir.getCanonicalPath) } } test("vacuum for cdc - update/merge") { testCDCVacuumForUpdateMerge() } test("vacuum for cdc - delete tombstones") { testCDCVacuumForTombstones() } private def getFromHistory(history: DataFrame, key: String, pos: Integer): Map[String, String] = { val op = history.select(key).take(pos + 1) if (pos == 0) { op.head.getMap(0).asInstanceOf[Map[String, String]] } else { op.tail.head.getMap(0).asInstanceOf[Map[String, String]] } } private def testEventLogging( isDryRun: Boolean, loggingEnabled: Boolean, retentionHours: Long, timeGapHours: Long): Unit = { test(s"vacuum event logging dryRun=$isDryRun loggingEnabled=$loggingEnabled" + s" retentionHours=$retentionHours timeGap=$timeGapHours") { withSQLConf(DeltaSQLConf.DELTA_VACUUM_LOGGING_ENABLED.key -> loggingEnabled.toString) { withEnvironment { (dir, clock) => clock.setTime(System.currentTimeMillis()) spark .range(2) .write .format("delta") .option("delta.deletedFileRetentionDuration", s"interval $retentionHours hours") .save(dir.getAbsolutePath) // The following is done to ensure deltaLog object uses the same clock that Vacuum // logic uses. DeltaLog.clearCache() val table = DeltaTableV2(spark, dir, clock) setCommitClock(table, 0L, clock) val expectedReturn = if (isDryRun) { // dry run returns files that will be deleted Seq(new Path(dir.getAbsolutePath, "file1.txt").toString) } else { Seq(dir.getAbsolutePath) } gcTest(table, clock)( CreateFile("file1.txt", commitToActionLog = true), CreateFile("file2.txt", commitToActionLog = true), LogicallyDeleteFile("file1.txt"), AdvanceClock(timeGapHours * 1000 * 60 * 60), GC(dryRun = isDryRun, expectedReturn, Some(retentionHours)) ) val deltaTable = io.delta.tables.DeltaTable.forPath(table.deltaLog.dataPath.toString) val history = deltaTable.history() if (isDryRun || !loggingEnabled) { // We do not record stats when logging is disabled or dryRun assert(history.select("operation").head() == Row("DELETE")) } else { assert(history.select("operation").head() == Row("VACUUM END")) assert(history.select("operation").collect()(1) == Row("VACUUM START")) val operationParamsBegin = getFromHistory(history, "operationParameters", 1) val operationParamsEnd = getFromHistory(history, "operationParameters", 0) val operationMetricsBegin = getFromHistory(history, "operationMetrics", 1) val operationMetricsEnd = getFromHistory(history, "operationMetrics", 0) val filesDeleted = if (retentionHours > timeGapHours) { 0 } else { 1 } assert(operationParamsBegin("retentionCheckEnabled") === "false") assert(operationMetricsBegin("numFilesToDelete") === filesDeleted.toString) assert(operationMetricsBegin("sizeOfDataToDelete") === (filesDeleted * 9).toString) if (retentionHours == 0) { assert( operationParamsBegin("specifiedRetentionMillis") === (retentionHours * 60 * 60 * 1000).toString) } assert( operationParamsBegin("defaultRetentionMillis") === DeltaLog.tombstoneRetentionMillis(table.initialSnapshot.metadata).toString) assert(operationParamsEnd === Map("status" -> "COMPLETED")) assert(operationMetricsEnd === Map("numDeletedFiles" -> filesDeleted.toString, "numVacuumedDirectories" -> "1")) } } } } } testEventLogging( isDryRun = false, loggingEnabled = true, retentionHours = 0, timeGapHours = 10 ) testEventLogging( isDryRun = true, // dry run will not record the vacuum loggingEnabled = true, retentionHours = 5, timeGapHours = 10 ) testEventLogging( isDryRun = false, loggingEnabled = false, retentionHours = 5, timeGapHours = 0 ) testEventLogging( isDryRun = false, loggingEnabled = true, retentionHours = 20, // vacuum will not delete any files timeGapHours = 10 ) test(s"vacuum sql syntax checks") { val tableName = "testTable" withTable(tableName) { withDeletionVectorsEnabled() { withSQLConf( DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED.key -> "false", DeltaSQLConf.LITE_VACUUM_ENABLED.key -> "false" ) { spark.range(0, 50, step = 1, numPartitions = 5).write.format("delta") .saveAsTable(tableName) var e = intercept[AnalysisException] { spark.sql(s"Vacuum $tableName DRY RUN DRY RUN") } assert(e.getMessage.contains("Found duplicate clauses: DRY RUN")) e = intercept[AnalysisException] { spark.sql(s"Vacuum $tableName RETAIN 200 HOURS RETAIN 200 HOURS") } assert(e.getMessage.contains("Found duplicate clauses: RETAIN")) e = intercept[AnalysisException] { spark.sql(s"Vacuum $tableName FULL LITE") } assert(e.getMessage.contains("Found duplicate clauses: LITE/FULL")) e = intercept[AnalysisException] { spark.sql(s"Vacuum $tableName USING INVENTORY $tableName INVENTORY $tableName") } assert(e.getMessage.contains("Syntax error at or near")) e = intercept[AnalysisException] { spark.sql(s"Vacuum $tableName USING INVENTORY $tableName LITE") } assert(e.getMessage.contains("Inventory option is not compatible with LITE")) // create an uncommitted file. Presence or lack of this file will help us // validate that we ran the right type of Vacuum. val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val basePath = deltaLog.dataPath.toString val clock = new ManualClock() val fs = new Path(basePath).getFileSystem(deltaLog.newDeltaHadoopConf()) val sanitizedPath = new Path("UnCommittedFile.parquet").toUri.toString val file = new File( fs.makeQualified(DeltaFileOperations.absolutePath(basePath, sanitizedPath)).toUri) createFile(basePath, sanitizedPath, file, clock) spark.sql(s"DELETE from $tableName WHERE ID % 2 = 0 and ID < 40") assertNumFiles(deltaLog, addFiles = 5, addFilesWithDVs = 4, dvFiles = 1, dataFiles = 6) purgeDVs(tableName) assertNumFiles(deltaLog, addFiles = 5, addFilesWithDVs = 0, dvFiles = 1, dataFiles = 10) // 9 file actions + one uncommitted file spark.sql(s"Vacuum $tableName LITE DRY RUN RETAIN 0 HOURS") // DRY RUN option doesn't change anything. assertNumFiles(deltaLog, addFiles = 5, addFilesWithDVs = 0, dvFiles = 1, dataFiles = 10) // LITE will be able to GC 4 files removed by DELETE. spark.sql(s"Vacuum $tableName LITE RETAIN 0 HOURS") assertNumFiles(deltaLog, addFiles = 5, addFilesWithDVs = 0, dvFiles = 0, dataFiles = 6) // Default is full and it's able to delete the 'notCommittedFile.parquet' spark.sql(s"Vacuum $tableName RETAIN 0 HOURS") assertNumFiles(deltaLog, addFiles = 5, addFilesWithDVs = 0, dvFiles = 0, dataFiles = 5) // Create the uncommittedFile file again to make sure explicit vacuum full works as // expected. createFile(basePath, sanitizedPath, file, clock) assertNumFiles(deltaLog, addFiles = 5, addFilesWithDVs = 0, dvFiles = 0, dataFiles = 6) spark.sql(s"Vacuum $tableName FULL RETAIN 0 HOURS") assertNumFiles(deltaLog, addFiles = 5, addFilesWithDVs = 0, dvFiles = 0, dataFiles = 5) } } } } test("running vacuum on a catalog managed table should fail") { withCatalogManagedTable() { tableName => checkError( intercept[DeltaUnsupportedOperationException] { spark.sql(s"VACUUM $tableName") }, "DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION", parameters = Map("operation" -> "VACUUM") ) checkError( intercept[DeltaUnsupportedOperationException] { spark.sql(s"VACUUM $tableName DRY RUN") }, "DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION", parameters = Map("operation" -> "VACUUM") ) } } } class DeltaLiteVacuumSuite extends DeltaVacuumSuite { override def isLiteVacuum: Boolean = true private var oldValue: Boolean = false override def beforeAll(): Unit = { super.beforeAll() oldValue = spark.conf.get(DeltaSQLConf.LITE_VACUUM_ENABLED) spark.conf.set(DeltaSQLConf.LITE_VACUUM_ENABLED.key, "true") } override def afterAll(): Unit = { spark.conf.set(DeltaSQLConf.LITE_VACUUM_ENABLED.key, oldValue) super.afterAll() } test("lite vacuum not possible - commit 0 is missing") { withSQLConf( DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED.key -> "false" ) { withTempDir { dir => // create table versions 0 and 1 spark.range(10) .write .format("delta") .save(dir.getAbsolutePath) spark.range(10) .write .format("delta") .mode("append") .save(dir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(dir.getAbsolutePath) val table = DeltaTableV2(spark, new Path(dir.getAbsolutePath)) deltaTable.delete() // Checkpoints will allow us to construct the table snapshot table.deltaLog.createCheckpointAtVersion(2L) deleteCommitFile(table, 0L) // delete version 0 val e = intercept[DeltaIllegalStateException] { VacuumCommand.gc(spark, table, dryRun = true, retentionHours = Some(0)) } assert(e.getMessage.contains("VACUUM LITE cannot delete all eligible files as some files" + " are not referenced by the Delta log. Please run VACUUM FULL.")) } } } test("lite vacuum not possible - commits since last vacuum is missing") { withSQLConf( DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED.key -> "false" ) { withTempDir { dir => // create table - version 0 spark.range(10) .write .format("delta") .save(dir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(dir.getAbsolutePath) val table = DeltaTableV2(spark, new Path(dir.getAbsolutePath)) deltaTable.delete() // version 1 // The following Vacuum saves latestCommitVersionOutsideOfRetentionWindow as 1 VacuumCommand.gc(spark, table, dryRun = false, retentionHours = Some(0)) spark.range(10) .write .format("delta") .mode("append") .save(dir.getAbsolutePath) // version 2 deltaTable.delete() // version 3 // Checkpoint will allow us to construct the table snapshot table.deltaLog.createCheckpointAtVersion(3L) // Deleting version 0 shouldn't fail the vacuum since // latestCommitVersionOutsideOfRetentionWindow is already at 1 deleteCommitFile(table, 0L)// delete version 0. VacuumCommand.gc(spark, table, dryRun = true, retentionHours = Some(0)) // Since commit versions 1 and 2 are required for lite vacuum, deleting them will // fail the command. for (i <- 1 to 2) { deleteCommitFile(table, i) } val e = intercept[DeltaIllegalStateException] { VacuumCommand.gc(spark, table, dryRun = true, retentionHours = Some(0)) } assert(e.getMessage.contains("VACUUM LITE cannot delete all eligible files as some files" + " are not referenced by the Delta log. Please run VACUUM FULL.")) } } } test("repeated invocations for lite vacuum is a no-op and doesn't throw any exception") { withEnvironment { (tempDir, clock) => val reservoirDir = new File(tempDir.getAbsolutePath, "reservoir") val table = DeltaTableV2(spark, reservoirDir, clock) gcTest(table, clock)( // create 2 files CreateFile("file1.txt", commitToActionLog = true), CreateFile("file2.txt", commitToActionLog = true), LogicallyDeleteFile("file1.txt"), LogicallyDeleteFile("file2.txt"), CheckFiles(Seq("file1.txt", "file2.txt")), AdvanceClock(defaultTombstoneInterval + 1000), GC(dryRun = true, Seq(reservoirDir.toString + "/file1.txt", reservoirDir.toString + "/file2.txt")), GC(dryRun = false, Seq(reservoirDir.toString)), CheckFiles(Seq("file1.txt", "file2.txt"), exist = false), AdvanceClock(defaultTombstoneInterval + 1000), GC(dryRun = true, Seq()), GC(dryRun = false, Seq(reservoirDir.toString)), AdvanceClock(defaultTombstoneInterval + 1000), GC(dryRun = true, Seq()), GC(dryRun = false, Seq(reservoirDir.toString)) ) } } test("Vacuum retain argument is ignored if it's not 0 hours") { withEnvironment { (tempDir, clock) => val reservoirDir = new File(tempDir.getAbsolutePath, "reservoir") val table = DeltaTableV2(spark, reservoirDir, clock) gcTest(table, clock)( // create 2 files CreateFile("file1.txt", commitToActionLog = true), CreateFile("file2.txt", commitToActionLog = true), LogicallyDeleteFile("file1.txt"), AdvanceClock(defaultTombstoneInterval + 1000), LogicallyDeleteFile("file2.txt"), CheckFiles(Seq("file1.txt", "file2.txt")), AdvanceClock((24 * 60 * 60 * 1000) + 1000), // 24 hours + 1000 ms // 24 hours retain argument is ignored and only file1.txt is eligible for GC GC(dryRun = true, Seq(reservoirDir.toString + "/file1.txt"), retentionHours = Some(24)), GC(dryRun = false, Seq(reservoirDir.toString), retentionHours = Some(24)), CheckFiles(Seq("file2.txt")) ) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaVariantShreddingSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import scala.collection.JavaConverters._ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.spark.SparkException import org.apache.spark.sql.{QueryTest, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils, TestsStatistics} import org.apache.spark.sql.delta.test.shims.VariantShreddingTestShims import org.apache.spark.sql.delta.util.DeltaFileOperations import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.execution.datasources.parquet.{ParquetToSparkSchemaConverter, SparkShreddingUtils} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ import org.apache.parquet.hadoop.ParquetFileReader import org.apache.parquet.hadoop.metadata.ParquetMetadata import org.apache.parquet.schema.{GroupType, MessageType, Type} import org.scalatest.Ignore class DeltaVariantShreddingSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with DeltaSQLTestUtils with TestsStatistics { import testImplicits._ private def numShreddedFiles(path: String, validation: GroupType => Boolean = _ => true): Int = { def listParquetFilesRecursively(dir: String): Seq[String] = { val deltaLog = DeltaLog.forTable(spark, dir) val files = deltaLog.snapshot.allFiles files.collect().map { file: AddFile => file.absolutePath(deltaLog).toString } } val parquetFiles = listParquetFilesRecursively(path) def hasStructWithFieldNamesInternal(schema: List[Type], fieldNames: Set[String]): Boolean = { schema.exists { case group: GroupType if group.getFields.asScala.map(_.getName).toSet == fieldNames => true case group: GroupType => hasStructWithFieldNamesInternal(group.getFields.asScala.toList, fieldNames) case _ => false } } def hasStructWithFieldNames(schema: MessageType, fieldNames: Set[String]): Boolean = { schema.getFields.asScala.exists { case group: GroupType if group.getFields.asScala.map(_.getName).toSet == fieldNames && validation(group) => true case group: GroupType => hasStructWithFieldNamesInternal(group.getFields.asScala.toList, fieldNames) case _ => false } } val requiredFieldNames = Set("value", "metadata", "typed_value") val conf = new Configuration() parquetFiles.count { p => val reader = ParquetFileReader.open(conf, new Path(p)) val footer: ParquetMetadata = reader.getFooter val isShredded = hasStructWithFieldNames(footer.getFileMetaData().getSchema, requiredFieldNames) reader.close() isShredded } } test("variant shredding table property") { withTable("tbl") { sql("CREATE TABLE tbl(s STRING, i INTEGER) USING DELTA") val (deltaLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier("tbl")) assert(!snapshot.protocol .isFeatureSupported(VariantShreddingPreviewTableFeature), s"Table tbl contains ShreddedVariantTableFeature descriptor when its not supposed to" ) sql(s"ALTER TABLE tbl " + s"SET TBLPROPERTIES('${DeltaConfigs.ENABLE_VARIANT_SHREDDING.key}' = 'true')") assert(getProtocolForTable("tbl") .readerAndWriterFeatures.contains(VariantShreddingPreviewTableFeature)) } withTable("tbl") { sql(s"CREATE TABLE tbl(s STRING, i INTEGER) USING DELTA " + s"TBLPROPERTIES('${DeltaConfigs.ENABLE_VARIANT_SHREDDING.key}' = 'true')") assert(getProtocolForTable("tbl") .readerAndWriterFeatures.contains(VariantShreddingPreviewTableFeature)) } assert(DeltaConfigs.ENABLE_VARIANT_SHREDDING.key == "delta.enableVariantShredding") } test("Spark can read shredded table containing the shredding table feature") { withTable("tbl") { withTempDir { dir => val schema = "a int, b string, c decimal(15, 1)" val df = spark.sql( """ | select id i, case | when id = 0 then parse_json('{"a": 1, "b": "2", "c": 3.3, "d": 4.4}') | when id = 1 then parse_json('{"a": [1,2,3], "b": "hello", "c": {"x": 0}}') | when id = 2 then parse_json('{"A": 1, "c": 1.23}') | end v from range(0, 3, 1, 1) |""".stripMargin) sql("CREATE TABLE tbl (i long, v variant) USING DELTA " + s"TBLPROPERTIES ('${DeltaConfigs.ENABLE_VARIANT_SHREDDING.key}' = 'true') " + s"LOCATION '${dir.getAbsolutePath}'") assert(getProtocolForTable("tbl") .readerAndWriterFeatures.contains(VariantShreddingPreviewTableFeature)) withSQLConf(SQLConf.VARIANT_WRITE_SHREDDING_ENABLED.key -> true.toString, SQLConf.VARIANT_ALLOW_READING_SHREDDED.key -> true.toString, SQLConf.VARIANT_FORCE_SHREDDING_SCHEMA_FOR_TEST.key -> schema) { df.write.format("delta").mode("append").saveAsTable("tbl") // Make sure the actual parquet files are shredded assert(numShreddedFiles(dir.getAbsolutePath, validation = { field: GroupType => field.getName == "v" && (field.getType("typed_value") match { case t: GroupType => t.getFields.asScala.map(_.getName).toSet == Set("a", "b", "c") case _ => false }) }) == 1) checkAnswer( spark.read.format("delta").load(dir.getAbsolutePath).selectExpr("i", "to_json(v)"), df.selectExpr("i", "to_json(v)").collect() ) } } } } test("Test shredding property controls shredded writes") { val schema = "a int, b string, c decimal(15, 1)" val df = spark.sql( """ | select id i, case | when id = 0 then parse_json('{"a": 1, "b": "2", "c": 3.3, "d": 4.4}') | when id = 1 then parse_json('{"a": [1,2,3], "b": "hello", "c": {"x": 0}}') | when id = 2 then parse_json('{"A": 1, "c": 1.23}') | end v from range(0, 3, 1, 1) |""".stripMargin) // Table property not present or false Seq("", s"TBLPROPERTIES ('${DeltaConfigs.ENABLE_VARIANT_SHREDDING.key}' = 'false') ") .foreach { tblProperties => withTable("tbl") { withTempDir { dir => sql("CREATE TABLE tbl (i long, v variant) USING DELTA " + tblProperties + s"LOCATION '${dir.getAbsolutePath}'") withSQLConf(SQLConf.VARIANT_WRITE_SHREDDING_ENABLED.key -> true.toString, SQLConf.VARIANT_ALLOW_READING_SHREDDED.key -> true.toString, SQLConf.VARIANT_FORCE_SHREDDING_SCHEMA_FOR_TEST.key -> schema) { val e = intercept[DeltaSparkException] { df.write.format("delta").mode("append").saveAsTable("tbl") } checkError(e, "DELTA_SHREDDING_TABLE_PROPERTY_DISABLED", parameters = Map()) assert(e.getMessage.contains( "Attempted to write shredded Variants but the table does not support shredded " + "writes. Consider setting the table property enableVariantShredding to true.")) assert(numShreddedFiles(dir.getAbsolutePath, validation = { field: GroupType => field.getName == "v" && (field.getType("typed_value") match { case t: GroupType => t.getFields.asScala.map(_.getName).toSet == Set("a", "b", "c") case _ => false }) }) == 0) checkAnswer( spark.read.format("delta").load(dir.getAbsolutePath).selectExpr("i", "to_json(v)"), Seq() ) } } } } } test("Set table property to invalid value") { withTable("tbl") { sql("CREATE TABLE tbl(s STRING, i INTEGER) USING DELTA") val (deltaLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier("tbl")) assert(!snapshot.protocol .isFeatureSupported(VariantShreddingPreviewTableFeature), s"Table tbl contains ShreddedVariantTableFeature descriptor when its not supposed to" ) checkError( intercept[SparkException] { sql(s"ALTER TABLE tbl " + s"SET TBLPROPERTIES('${DeltaConfigs.ENABLE_VARIANT_SHREDDING.key}' = 'bla')") }, "_LEGACY_ERROR_TEMP_2045", parameters = Map( "message" -> "For input string: \"bla\"" ) ) assert(!getProtocolForTable("tbl") .readerAndWriterFeatures.contains(VariantShreddingPreviewTableFeature)) } } test("Infer schema for Delta table") { // Skip this test if VARIANT_INFER_SHREDDING_SCHEMA is not supported (Spark 4.0) assume(VariantShreddingTestShims.variantInferShreddingSchemaSupported, "VARIANT_INFER_SHREDDING_SCHEMA is not supported in this Spark version") // make sure top level conf has no effect and table property is respected. Seq(false, true).foreach { inferShreddingSchema => withSQLConf(VariantShreddingTestShims.variantInferShreddingSchemaKey -> inferShreddingSchema.toString) { Seq(false, true).foreach { enable => val tbl = s"tbl_$enable" withTable(tbl) { withTempDir { dir => val query = """select parse_json('{"a": ' || id || ', "b": "' || id || '"}') as v | from range(0, 3, 1, 1)""".stripMargin val properties = if (enable) { "tblproperties ('delta.enableVariantShredding' = 'true')" } else { "" } spark.sql( s""" | create table $tbl using delta | $properties | location '${dir.getAbsolutePath}' | as | $query |""".stripMargin) if (enable) { assert(numShreddedFiles( dir.getAbsolutePath, validation = { field: GroupType => field.getName == "v" && (field.getType("typed_value") match { case t: GroupType => t.getFields.asScala.map(_.getName).toSet == Set("a", "b") case _ => false }) }) == 1) } else { assert(numShreddedFiles(dir.getAbsolutePath) == 0) } checkAnswer(spark.table(tbl), spark.sql(query).collect()) } } } } } } test("creating table with preview feature does not add stable feature (and vice versa)") { Seq("-preview", "").foreach { featureSuffix => withTable("tbl") { sql(s"""CREATE TABLE tbl(i INT) USING delta TBLPROPERTIES( 'delta.enableVariantShredding' = 'true', 'delta.feature.variantShredding$featureSuffix' = 'supported' )""") DeltaVariantShreddingSuite.assertVariantShreddingTableFeatures( spark, "tbl", expectPreviewFeature = featureSuffix.nonEmpty, expectStableFeature = featureSuffix.isEmpty) } } } test("manually enabling preview and stable table feature") { Seq(false, true).foreach { forcePreview => withSQLConf(DeltaSQLConf.FORCE_USE_PREVIEW_SHREDDING_FEATURE.key -> forcePreview.toString) { withTable("tbl") { sql("""CREATE TABLE tbl(i INT) USING delta TBLPROPERTIES( 'delta.feature.variantShredding' = 'supported', 'delta.feature.variantShredding-preview' = 'supported' )""") DeltaVariantShreddingSuite.assertVariantShreddingTableFeatures( spark, "tbl", expectPreviewFeature = true, expectStableFeature = true) sql("""ALTER TABLE tbl SET TBLPROPERTIES ('delta.enableVariantShredding' = 'true')""") DeltaVariantShreddingSuite.assertVariantShreddingTableFeatures( spark, "tbl", expectPreviewFeature = true, expectStableFeature = true) } } } } test("enabling 'FORCE_USE_PREVIEW_SHREDDING_FEATURE' adds preview table feature for new table") { Seq(false, true).foreach { forcePreview => withSQLConf(DeltaSQLConf.FORCE_USE_PREVIEW_SHREDDING_FEATURE.key -> forcePreview.toString) { withTable("tbl") { sql("CREATE TABLE tbl(s STRING) USING DELTA TBLPROPERTIES " + "('delta.enableVariantShredding' = 'true')") DeltaVariantShreddingSuite.assertVariantShreddingTableFeatures( spark, "tbl", expectPreviewFeature = forcePreview, expectStableFeature = !forcePreview) } } } } test("enabling 'FORCE_USE_PREVIEW_SHREDDING_FEATURE' and setting shredding table property " + "adds the preview table feature") { Seq(false, true).foreach { forcePreview => withSQLConf(DeltaSQLConf.FORCE_USE_PREVIEW_SHREDDING_FEATURE.key -> forcePreview.toString) { withTable("tbl") { sql("CREATE TABLE tbl(s STRING) USING DELTA") DeltaVariantShreddingSuite.assertVariantShreddingTableFeatures( spark, "tbl", expectPreviewFeature = false, expectStableFeature = false) sql("ALTER TABLE tbl SET TBLPROPERTIES ('delta.enableVariantShredding' = 'true')") DeltaVariantShreddingSuite.assertVariantShreddingTableFeatures( spark, "tbl", expectPreviewFeature = forcePreview, expectStableFeature = !forcePreview) } } } } test("enabling 'FORCE_USE_PREVIEW_VARIANT_FEATURE' on table with stable feature does not " + "require adding preview feature") { Seq(false, true).foreach { forcePreview => withSQLConf(DeltaSQLConf.FORCE_USE_PREVIEW_SHREDDING_FEATURE.key -> forcePreview.toString) { withTable("tbl") { sql("CREATE TABLE tbl(s STRING) USING DELTA TBLPROPERTIES " + "('delta.enableVariantShredding' = 'true')") DeltaVariantShreddingSuite.assertVariantShreddingTableFeatures( spark, "tbl", expectPreviewFeature = forcePreview, expectStableFeature = !forcePreview) withSQLConf(DeltaSQLConf.FORCE_USE_PREVIEW_VARIANT_FEATURE.key -> (!forcePreview).toString) { // Reset the table property and set it again to see if it modifies to protocol sql("ALTER TABLE tbl SET " + "TBLPROPERTIES ('delta.enableVariantShredding' = 'false')") DeltaVariantShreddingSuite.assertVariantShreddingTableFeatures( spark, "tbl", expectPreviewFeature = forcePreview, expectStableFeature = !forcePreview) sql("ALTER TABLE tbl SET " + "TBLPROPERTIES ('delta.enableVariantShredding' = 'true')") DeltaVariantShreddingSuite.assertVariantShreddingTableFeatures( spark, "tbl", expectPreviewFeature = forcePreview, expectStableFeature = !forcePreview) } } } } } } object DeltaVariantShreddingSuite { def assertVariantShreddingTableFeatures( spark: SparkSession, tableName: String, expectPreviewFeature: Boolean, expectStableFeature: Boolean): Unit = { val features = DeltaLog.forTable(spark, TableIdentifier(tableName)).update().protocol .readerAndWriterFeatures assert(expectPreviewFeature == features.contains(VariantShreddingPreviewTableFeature)) assert(expectStableFeature == features.contains(VariantShreddingTableFeature)) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaVariantSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.collection.mutable.{Seq => MutableSeq} import scala.io.Source import io.delta.tables.DeltaTable import org.apache.spark.sql.delta.actions.Protocol import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils import org.apache.spark.sql.delta.commands.optimize.OptimizeMetrics import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils, TestsStatistics} import org.apache.spark.{SparkException, SparkThrowable} import org.apache.spark.sql.{AnalysisException, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.delta.DeltaAnalysisException import org.apache.spark.sql.delta.schema.DeltaInvariantViolationException import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.StructType import org.scalatest.Ignore class DeltaVariantSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with DeltaSQLTestUtils with TestsStatistics { import testImplicits._ private def assertVariantTypeTableFeatures( tableName: String, expectPreviewFeature: Boolean, expectStableFeature: Boolean): Unit = { val features = getProtocolForTable("tbl").readerAndWriterFeatures if (expectPreviewFeature) { assert(features.contains(VariantTypePreviewTableFeature)) } else { assert(!features.contains(VariantTypePreviewTableFeature)) } if (expectStableFeature) { assert(features.contains(VariantTypeTableFeature)) } else { assert(!features.contains(VariantTypeTableFeature)) } } test("create a new table with Variant, higher protocol and feature should be picked.") { withTable("tbl") { sql("CREATE TABLE tbl(s STRING, v VARIANT) USING DELTA") sql("INSERT INTO tbl (SELECT 'foo', parse_json(cast(id + 99 as string)) FROM range(1))") assert(spark.table("tbl").selectExpr("v::int").head == Row(99)) assertVariantTypeTableFeatures( "tbl", expectPreviewFeature = false, expectStableFeature = true) } } test("creating a table without Variant should use the usual minimum protocol") { withTable("tbl") { sql("CREATE TABLE tbl(s STRING, i INTEGER) USING DELTA") assert(getProtocolForTable("tbl") == Protocol(1, 2)) val deltaLog = DeltaLog.forTable(spark, TableIdentifier("tbl")) assert( !deltaLog.unsafeVolatileSnapshot.protocol.isFeatureSupported( VariantTypePreviewTableFeature) && !deltaLog.unsafeVolatileSnapshot.protocol.isFeatureSupported( VariantTypeTableFeature), s"Table tbl contains VariantTypeFeature descriptor when its not supposed to" ) } } test("add a new Variant column should upgrade to the correct protocol versions") { withTable("tbl") { sql("CREATE TABLE tbl(s STRING) USING delta") assert(getProtocolForTable("tbl") == Protocol(1, 2)) sql("ALTER TABLE tbl ADD COLUMN v VARIANT") sql("INSERT INTO tbl (SELECT 'foo', parse_json(cast(id + 99 as string)) FROM range(1))") assert(spark.table("tbl").selectExpr("v::int").head == Row(99)) sql("ALTER TABLE tbl ADD COLUMN v2 STRUCT") sql("INSERT INTO tbl (SELECT 'bar', " + "parse_json(cast(id + 100 as string)), struct(parse_json(cast(id + 101 as string))) " + "FROM range(1))") checkAnswer(spark.table("tbl").selectExpr("v::int"), Seq(Row(99), Row(100))) checkAnswer(spark.table("tbl").selectExpr("v2.v21::int"), Seq(Row(null), Row(101))) assert( getProtocolForTable("tbl") == VariantTypeTableFeature.minProtocolVersion .withFeature(VariantTypeTableFeature) .withFeature(InvariantsTableFeature) .withFeature(AppendOnlyTableFeature) ) } } test("variant stable and preview features can be supported simultaneously and read") { withTable("tbl") { sql("CREATE TABLE tbl(v VARIANT) USING delta") sql("INSERT INTO tbl (SELECT parse_json(cast(id + 99 as string)) FROM range(1))") assert(spark.table("tbl").selectExpr("v::int").head == Row(99)) assertVariantTypeTableFeatures( "tbl", expectPreviewFeature = false, expectStableFeature = true) sql( s"ALTER TABLE tbl " + s"SET TBLPROPERTIES('delta.feature.variantType-preview' = 'supported')" ) assertVariantTypeTableFeatures( "tbl", expectPreviewFeature = true, expectStableFeature = true) assert(spark.table("tbl").selectExpr("v::int").head == Row(99)) } } test("creating a new variant table uses only the stable table feature") { withTable("tbl") { sql("CREATE TABLE tbl(s STRING, v VARIANT) USING DELTA") sql("INSERT INTO tbl (SELECT 'foo', parse_json(cast(id + 99 as string)) FROM range(1))") assert(spark.table("tbl").selectExpr("v::int").head == Row(99)) assertVariantTypeTableFeatures( "tbl", expectPreviewFeature = false, expectStableFeature = true) } } test("manually adding preview table feature does not require adding stable table feature") { withTable("tbl") { sql("CREATE TABLE tbl(s STRING) USING delta") sql( s"ALTER TABLE tbl " + s"SET TBLPROPERTIES('delta.feature.variantType-preview' = 'supported')" ) sql("ALTER TABLE tbl ADD COLUMN v VARIANT") sql("INSERT INTO tbl (SELECT 'foo', parse_json(cast(id + 99 as string)) FROM range(1))") assert(spark.table("tbl").selectExpr("v::int").head == Row(99)) assertVariantTypeTableFeatures( "tbl", expectPreviewFeature = true, expectStableFeature = false ) } } test("creating table with preview feature does not add stable feature") { withTable("tbl") { sql(s"""CREATE TABLE tbl(v VARIANT) USING delta TBLPROPERTIES('delta.feature.variantType-preview' = 'supported')""" ) sql("INSERT INTO tbl (SELECT parse_json(cast(id + 99 as string)) FROM range(1))") assertVariantTypeTableFeatures( "tbl", expectPreviewFeature = true, expectStableFeature = false ) } } test("enabling 'FORCE_USE_PREVIEW_VARIANT_FEATURE' adds preview table feature for new table") { withSQLConf(DeltaSQLConf.FORCE_USE_PREVIEW_VARIANT_FEATURE.key -> "true") { withTable("tbl") { sql("CREATE TABLE tbl(s STRING, v VARIANT) USING DELTA") sql("INSERT INTO tbl (SELECT 'foo', parse_json(cast(id + 99 as string)) FROM range(1))") assert(spark.table("tbl").selectExpr("v::int").head == Row(99)) assertVariantTypeTableFeatures( "tbl", expectPreviewFeature = true, expectStableFeature = false) } } } test("enabling 'FORCE_USE_PREVIEW_VARIANT_FEATURE' and adding a variant column adds the " + "preview table feature") { withSQLConf(DeltaSQLConf.FORCE_USE_PREVIEW_VARIANT_FEATURE.key -> "true") { withTable("tbl") { sql("CREATE TABLE tbl(s STRING) USING delta") sql("ALTER TABLE tbl ADD COLUMN v VARIANT") sql("INSERT INTO tbl (SELECT 'foo', parse_json(cast(id + 99 as string)) FROM range(1))") assert(spark.table("tbl").selectExpr("v::int").head == Row(99)) sql("ALTER TABLE tbl ADD COLUMN v2 STRUCT") sql("INSERT INTO tbl (SELECT 'bar', " + "parse_json(cast(id + 100 as string)), struct(parse_json(cast(id + 101 as string))) " + "FROM range(1))") checkAnswer(spark.table("tbl").selectExpr("v::int"), Seq(Row(99), Row(100))) checkAnswer(spark.table("tbl").selectExpr("v2.v21::int"), Seq(Row(null), Row(101))) assertVariantTypeTableFeatures( "tbl", expectPreviewFeature = true, expectStableFeature = false ) } } } test("enabling 'FORCE_USE_PREVIEW_VARIANT_FEATURE' on table with stable feature does not " + "require adding preview feature") { withTable("tbl") { sql("CREATE TABLE tbl(s STRING, v VARIANT) USING DELTA") sql("INSERT INTO tbl (SELECT 'foo', parse_json(cast(id + 99 as string)) FROM range(1))") assert(spark.table("tbl").selectExpr("v::int").head == Row(99)) assertVariantTypeTableFeatures( "tbl", expectPreviewFeature = false, expectStableFeature = true) withSQLConf(DeltaSQLConf.FORCE_USE_PREVIEW_VARIANT_FEATURE.key -> "true") { sql("INSERT INTO tbl (SELECT 'foo', parse_json(cast(id + 99 as string)) FROM range(1))") assert(spark.table("tbl").selectExpr("v::int").count == 2) assertVariantTypeTableFeatures( "tbl", expectPreviewFeature = false, expectStableFeature = true) } } } test("VariantType may not be used as a partition column") { withTable("delta_test") { checkError( intercept[AnalysisException] { sql( """CREATE TABLE delta_test(s STRING, v VARIANT) |USING delta |PARTITIONED BY (v)""".stripMargin) }, "INVALID_PARTITION_COLUMN_DATA_TYPE", parameters = Map("type" -> "\"VARIANT\"") ) } } test("streaming variant delta table") { withTempDir { dir => val path = dir.getAbsolutePath spark.range(100) .selectExpr("parse_json(cast(id as string)) v") .write .format("delta") .mode("overwrite") .save(path) val streamingDf = spark.readStream .format("delta") .load(path) .selectExpr("v::int as extractedVal") val q = streamingDf.writeStream .format("memory") .queryName("test_table") .start() q.processAllAvailable() q.stop() val actual = spark.sql("select extractedVal from test_table") val expected = spark.sql("select id from range(100)") checkAnswer(actual, expected.collect()) } } test("variant works with schema evolution for INSERT") { withTempDir { dir => val path = dir.getAbsolutePath spark.range(0, 100, 1, 1) .selectExpr("id", "parse_json(cast(id as string)) v") .write .format("delta") .mode("overwrite") .save(path) spark.range(100, 200, 1, 1) .selectExpr( "id", "parse_json(cast(id as string)) v", "parse_json(cast(id as string)) v_two" ) .write .format("delta") .mode("append") .option("mergeSchema", "true") .save(path) val expected = spark.range(0, 200, 1, 1).selectExpr( "id", "parse_json(cast(id as string)) v", "case when id >= 100 then parse_json(cast(id as string)) else null end v_two" ) val read = spark.read.format("delta").load(path) checkAnswer(read, expected.collect()) } } test("variant works with schema evolution for MERGE") { withTempDir { dir => withSQLConf("spark.databricks.delta.schema.autoMerge.enabled" -> "true") { val path = dir.getAbsolutePath spark.range(0, 100, 1, 1) .selectExpr("id", "parse_json(cast(id as string)) v") .write .format("delta") .mode("overwrite") .save(path) val sourceDf = spark.range(50, 200, 1, 1) .selectExpr( "id", "parse_json(cast(id as string)) v", "parse_json(cast(id as string)) v_two" ) DeltaTable.forPath(spark, path) .as("source") .merge(sourceDf.as("target"), "source.id = target.id") .whenMatched() .updateAll() .whenNotMatched() .insertAll() .execute() val expected = spark.range(0, 200, 1, 1).selectExpr( "id", "parse_json(cast(id as string)) v", "case when id >= 50 then parse_json(cast(id as string)) else null end v_two" ) val read = spark.read.format("delta").load(path) checkAnswer(read, expected.collect()) } } } test("variant cannot be used as a clustering column") { withTable("tbl") { val e = intercept[DeltaAnalysisException] { sql("CREATE TABLE tbl(v variant) USING DELTA CLUSTER BY (v)") } checkError( e, "DELTA_CLUSTERING_COLUMNS_DATATYPE_NOT_SUPPORTED", parameters = Map("columnsWithDataTypes" -> "v : VARIANT") ) } } test("describe history works with variant column") { withTable("tbl") { sql("CREATE TABLE tbl(s STRING, v VARIANT) USING DELTA") sql("INSERT INTO tbl (SELECT 'foo', parse_json(cast(id + 99 as string)) FROM range(1))") // Create and insert should result in two table versions. assert(sql("DESCRIBE HISTORY tbl").count() == 2) } } test("describe detail works with variant column") { withTable("tbl") { sql("CREATE TABLE tbl(s STRING, v VARIANT) USING DELTA") sql("INSERT INTO tbl (SELECT 'foo', parse_json(cast(id + 99 as string)) FROM range(1))") val tableFeatures = sql("DESCRIBE DETAIL tbl") .selectExpr("tableFeatures") .collect()(0) .getAs[MutableSeq[String]](0) assert(tableFeatures.find(f => f == VariantTypePreviewTableFeature.name).isEmpty) assert(tableFeatures.find(f => f == VariantTypeTableFeature.name).nonEmpty) } } test("time travel with variant column works") { withTempDir { dir => val path = dir.getAbsolutePath val initialDf = spark.range(0, 100, 1, 1).selectExpr("parse_json(cast(id as string)) v") initialDf .write .format("delta") .mode("overwrite") .save(path) spark.range(100, 150, 1, 1).selectExpr("parse_json(cast(id as string)) v") .write .format("delta") .mode("append") .save(path) val timeTravelDf = spark.read.format("delta").option("versionAsOf", "0").load(path) checkAnswer(timeTravelDf, initialDf.collect()) } } statsTest("optimize variant") { withTable("tbl") { spark.range(0, 100) .selectExpr("case when id % 2 = 0 then parse_json(cast(id as string)) else null end as v") .repartition(100) .write .format("delta") .mode("overwrite") .saveAsTable("tbl") val deltaLog = DeltaLog.forTable(spark, TableIdentifier("tbl")) val res = sql("OPTIMIZE tbl") val metrics = res.select($"metrics.*").as[OptimizeMetrics].head() assert(metrics.numFilesAdded > 0) assert(metrics.numTableColumnsWithStats == 1) val statsDf = getStatsDf(deltaLog, Seq($"numRecords", $"nullCount")) checkAnswer(statsDf, Row(100, Row(50))) } } test("Zorder is not supported for Variant") { withTable("tbl") { sql("CREATE TABLE tbl USING DELTA AS SELECT id, cast(null as variant) v from range(100)") val e = intercept[SparkException](sql("optimize tbl zorder by (v)")) checkError( e.getCause.asInstanceOf[SparkThrowable], "DATATYPE_MISMATCH.TYPE_CHECK_FAILURE_WITH_HINT", parameters = Map( "msg" -> "cannot sort data type variant", "hint" -> "", "sqlExpr" -> "\"rangepartitionid(v)\"")) } } test("Table with variant type can use CDF") { withTable("tbl") { sql("""CREATE TABLE tbl USING DELTA TBLPROPERTIES (delta.enableChangeDataFeed = true) AS SELECT parse_json(cast(id as string)) v from range(100)""") sql("INSERT INTO tbl (SELECT parse_json(cast(id as string)) as v from range(0, 1))") sql("DELETE FROM tbl WHERE v::int = 0") sql("UPDATE tbl SET v = parse_json('-2') WHERE v::int = 50") checkAnswer( sql("""select _change_type, v::int from table_changes('tbl', 0) where _change_type = 'delete'"""), Seq(Row("delete", 0), Row("delete", 0)) ) checkAnswer( sql("""select _change_type, v::int from table_changes('tbl', 0) where _change_type = 'update_preimage'"""), Seq(Row("update_preimage", 50)) ) checkAnswer( sql("""select _change_type, v::int from table_changes('tbl', 0) where _change_type = 'update_postimage'"""), Seq(Row("update_postimage", -2)) ) } } test("Existing table with variant type can enable CDF") { withTable("tbl") { sql("CREATE TABLE tbl(v variant) USING DELTA") sql("ALTER TABLE tbl SET TBLPROPERTIES (delta.enableChangeDataFeed = true)") sql("INSERT INTO tbl (SELECT parse_json(cast(id as string)) as v from range(0, 100))") sql("DELETE FROM tbl WHERE v::string = '0'") sql("UPDATE tbl SET v = parse_json('-2') WHERE v::int = 50") checkAnswer( sql("""select _change_type, v::int from table_changes('tbl', 1) where _change_type = 'delete'"""), Seq(Row("delete", 0)) ) checkAnswer( sql("""select _change_type, v::int from table_changes('tbl', 1) where _change_type = 'update_preimage'"""), Seq(Row("update_preimage", 50)) ) checkAnswer( sql("""select _change_type, v::int from table_changes('tbl', 1) where _change_type = 'update_postimage'"""), Seq(Row("update_postimage", -2)) ) } } test(s"shallow cloning table with variant") { withTable("tbl", "clone_tbl") { sql("""CREATE TABLE tbl USING DELTA AS SELECT parse_json(cast(id as string)) v FROM range(100)""") sql("INSERT INTO tbl (SELECT parse_json(cast(id as string)) as v from range(0, 10))") sql(s"CREATE TABLE IF NOT EXISTS clone_tbl SHALLOW CLONE tbl") sql("INSERT INTO tbl (SELECT parse_json(cast(id as string)) as v from range(0, 10))") sql("INSERT INTO clone_tbl (SELECT parse_json(cast(id as string)) as v from range(0, 10))") val origTable = spark.sql("select * from tbl") val clonedTable = spark.sql("select * from clone_tbl") checkAnswer(clonedTable, origTable.collect()) } } Seq("", "NO STATISTICS").foreach { statsClause => test(s"Convert to Delta from parquet - $statsClause") { withTempDir { dir => val path = dir.getAbsolutePath spark.range(0, 100).selectExpr("parse_json(cast(id as string)) as v") .write .format("parquet") .mode("overwrite") .save(path) sql(s"CONVERT TO DELTA parquet.`$path` $statsClause") // Ensure Delta feature like column renaming works. sql(s"ALTER TABLE delta.`$path` SET TBLPROPERTIES ('delta.columnMapping.mode' = 'name')") sql(s"ALTER TABLE delta.`$path` RENAME COLUMN v TO new_v") val expectedDf = spark.range(0, 100).selectExpr("parse_json(cast(id as string)) as new_v") val actualDf = spark.sql(s"select * from delta.`$path`") checkAnswer(actualDf, expectedDf.collect()) } } } Seq("name", "id").foreach { mode => Seq(false, true).foreach { pushVariantIntoScan => withSQLConf( SQLConf.PUSH_VARIANT_INTO_SCAN.key -> pushVariantIntoScan.toString ) { test(s"column mapping works - $mode - $pushVariantIntoScan") { withTable("tbl") { sql(s"""CREATE TABLE tbl USING DELTA TBLPROPERTIES ( 'delta.columnMapping.mode' = '$mode', 'delta.enableVariantShredding' = 'true' ) AS SELECT parse_json(cast(id as string)) v, parse_json(cast(id as string)) v_two FROM range(5)""") val expectedAnswer = spark.sql("select v from tbl").collect() sql("ALTER TABLE tbl RENAME COLUMN v TO new_v") checkAnswer(spark.sql("select new_v from tbl"), expectedAnswer) sql("ALTER TABLE tbl DROP COLUMN new_v") // 'SELECT *' from the test table should return the same as `expectedAnswer` because `v` // and `v_two` are initially identical and `v` is dropped, resulting in a single column. checkAnswer(spark.sql("select * from tbl"), expectedAnswer) } } } } } test("Variant can have default value set") { withTable("tbl") { sql("""CREATE TABLE tbl USING DELTA TBLPROPERTIES ('delta.feature.allowColumnDefaults' = 'enabled') AS SELECT parse_json(cast(id as string)) v from range(5)""") sql("INSERT INTO tbl VALUES (DEFAULT)") val nullCount = spark.sql("SELECT * FROM tbl WHERE v is null").count() // Default DEFAULT value is null. assert(nullCount == 1) sql("ALTER TABLE tbl ALTER COLUMN v SET DEFAULT (parse_json('{\"k\": \"v\"}'))") sql("INSERT INTO tbl VALUES (DEFAULT)") checkAnswer( sql("SELECT v FROM tbl WHERE variant_get(v, '$.k', 'STRING') = 'v'"), sql("select parse_json('{\"k\": \"v\"}')").collect ) } } test("Variant can be used as a source for generated columns") { withTable("tbl") { DeltaTable.create(spark) .tableName("tbl") .addColumn("v", "VARIANT") .addColumn( DeltaTable.columnBuilder(spark, "vInt") .dataType("INT") .generatedAlwaysAs("v::int") .build() ) .execute() spark.range(0, 100) .selectExpr("parse_json(cast(id as string)) as v") .write .mode("append") .format("delta") .saveAsTable("tbl") val expectedDf = spark.range(0, 100).selectExpr( "parse_json(cast(id as string)) v", "cast(id as int) vInt" ) val actualDf = spark.sql("select * from tbl") checkAnswer(actualDf, expectedDf.collect()) } } test("Variant cannot be created as a generated column") { withTable("tbl") { val e = intercept[DeltaAnalysisException] { DeltaTable.create(spark) .tableName("tbl") .addColumn("id", "INT") .addColumn( DeltaTable.columnBuilder(spark, "v") .dataType("VARIANT") .generatedAlwaysAs("parse_json(cast(id as string))") .build() ) .execute() } checkError( e, "DELTA_UNSUPPORTED_DATA_TYPE_IN_GENERATED_COLUMN", parameters = Map("dataType" -> "VARIANT") ) } } test("Variant respects Delta table IS NOT NULL constraints") { withTable("tbl") { sql("CREATE TABLE tbl(v variant NOT NULL) USING DELTA") sql("INSERT INTO tbl (SELECT parse_json(cast(id as string)) from range(0, 100))") val insertException = intercept[DeltaInvariantViolationException] { sql("INSERT INTO tbl VALUES (cast(null as variant))") } checkError( insertException, "DELTA_NOT_NULL_CONSTRAINT_VIOLATED", parameters = Map("columnName" -> "v") ) sql("ALTER TABLE tbl ALTER COLUMN v DROP NOT NULL") // Inserting null value should work now. sql("INSERT INTO tbl VALUES (cast(null as variant))") val nullCount = spark.sql("select * from tbl where v is null").count() assert(nullCount == 1) } } test("Variant respects Delta table CHECK constraints") { withTable("tbl") { sql("CREATE TABLE tbl(v variant) USING DELTA") sql("ALTER TABLE tbl ADD CONSTRAINT variantGTEZero CHECK (variant_get(v, '$', 'INT') >= 0)") sql("INSERT INTO tbl (SELECT parse_json(cast(id as string)) from range(0, 100))") val insertException = intercept[DeltaInvariantViolationException] { sql("INSERT INTO tbl (select parse_json(cast(id as string)) from range(-1, 0))") } checkError( insertException, "DELTA_VIOLATE_CONSTRAINT_WITH_VALUES", parameters = Map( "constraintName" -> "variantgtezero", "expression" -> "(variant_get(v, '$', 'INT') >= 0)", "values" -> " - v : -1" ) ) sql("ALTER TABLE tbl DROP CONSTRAINT variantGTEZero") sql("INSERT INTO tbl (select parse_json(cast(id as string)) from range(-1, 0))") val lessThanZeroCount = spark.sql("select * from tbl where v::int < 0").count() // Inserting variant with value less than zero should work after dropping constraint. assert(lessThanZeroCount == 1) val addConstraintException = intercept[DeltaAnalysisException] { sql("ALTER TABLE tbl ADD CONSTRAINT variantGTEZero CHECK (variant_get(v, '$', 'INT') >= 0)") } checkError( addConstraintException, "DELTA_NEW_CHECK_CONSTRAINT_VIOLATION", parameters = Map( "numRows" -> "1", "tableName" -> "spark_catalog.default.tbl", "checkConstraint" -> "variant_get ( v , '$' , 'INT' ) >= 0" ) ) sql("DELETE FROM tbl WHERE variant_get(v, '$', 'INT') < 0") // Adding the constraint should work after deleting the variant that is less than zero. sql("ALTER TABLE tbl ADD CONSTRAINT variantGTEZero CHECK (variant_get(v, '$', 'INT') >= 0)") val newLessThanZeroCount = spark.sql("select * from tbl where v::int < 0").count() assert(newLessThanZeroCount == 0) } } test("column mapping with pushVariantIntoScan") { withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> "true") { withTable("t1") { sql( """create table t1 (v variant) using delta |tblproperties ( | 'delta.columnMapping.mode' = 'name', | 'delta.enableVariantShredding' = 'true' |)""".stripMargin) sql("""insert into t1 (v) select parse_json('{"a": 1}')""") checkAnswer(sql("select to_json(v) from t1"), Seq(Row("""{"a":1}"""))) checkAnswer(sql("select variant_get(v,'$.a','int') from t1"), Seq(Row(1))) } // Ensure it also works when the variant is nested in a struct. withTable("t2") { sql( """create table t2 (s struct) using delta |tblproperties ( | 'delta.columnMapping.mode' = 'name', | 'delta.enableVariantShredding' = 'true' |)""".stripMargin) sql("""insert into t2 (s) select named_struct('v', parse_json('{"a": 2}'))""") checkAnswer(sql("select to_json(s) from t2"), Seq(Row("""{"v":{"a":2}}"""))) checkAnswer(sql("select to_json(s.v) from t2"), Seq(Row("""{"a":2}"""))) checkAnswer(sql("select variant_get(s.v, '$.a', 'int') from t2"), Seq(Row(2))) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaWithNewTransactionSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.DeltaTestUtils._ import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.{DataFrame, Dataset, QueryTest} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.{ThreadUtils, Utils} trait DeltaWithNewTransactionSuiteBase extends QueryTest with SharedSparkSession with DeltaColumnMappingTestUtils with DeltaSQLCommandTest with CatalogOwnedTestBaseSuite { /** * Test whether `withNewTransaction` captures all delta read made within it and correctly * detects conflicts in transaction table and provides snapshot isolation for other table reads. * * The order in which the given thunks are executed is as follows. * - Txn started using `withNewTransaction`. The following are executed while the txn is active. * - currentThreadReadOp - Read operations performed in current thread. * - concurrentUpdateOp - Update operations performed in different thread to * simulate concurrent modification. This is synchronously completed * before moving on. * - currentThreadCommitOperation - Attempt to commit changes in the txn. */ protected def testWithNewTransaction( name: String, partitionedTableKeys: Seq[Int], preTxnSetup: DeltaLog => Unit = null, currentThreadReadOp: DataFrame => Unit, concurrentUpdateOp: String => Unit, currentThreadCommitOperation: OptimisticTransaction => Unit, shouldFail: Boolean, confs: Map[String, String] = Map.empty, partitionTablePath: String = Utils.createTempDir().getAbsolutePath): Unit = { val tableName = "NewTransactionTest" require(currentThreadCommitOperation != null) import testImplicits._ test(s"withNewTransaction - $name") { withSQLConf(confs.toSeq: _*) { withTable(tableName) { sql(s"CREATE TABLE NewTransactionTest(key int, value int) " + s"USING delta partitioned by (key) LOCATION '$partitionTablePath'") partitionedTableKeys.toDS.select('value as "key", 'value) .write.mode("append").partitionBy("key").format("delta").saveAsTable(tableName) val log = DeltaLog.forTable(spark, partitionTablePath) assert(OptimisticTransaction.getActive().isEmpty, "active txn already set") if (preTxnSetup != null) preTxnSetup(log) log.withNewTransaction { txn => assert(OptimisticTransaction.getActive().nonEmpty, "active txn not set") currentThreadReadOp(spark.table(tableName)) ThreadUtils.runInNewThread(s"withNewTransaction test - $name") { concurrentUpdateOp(tableName) } if (shouldFail) { intercept[DeltaConcurrentModificationException] { currentThreadCommitOperation(txn) } } else { currentThreadCommitOperation(txn) } } assert(OptimisticTransaction.getActive().isEmpty, "active txn not cleared") }} } } testWithNewTransaction( name = "capture reads on txn table with no filters (i.e. full scan)", partitionedTableKeys = Seq(1, 2, 3), currentThreadReadOp = txnTable => { txnTable.count() }, concurrentUpdateOp = txnTableName => { sql(s"DELETE FROM $txnTableName WHERE key = 1") }, currentThreadCommitOperation = txn => { txn.commit(Seq.empty, DeltaOperations.ManualUpdate) }, shouldFail = true) testWithNewTransaction( name = "capture reads on txn table with partition filter + conflicting concurrent updates", partitionedTableKeys = Seq(1, 2, 3), currentThreadReadOp = txnTable => { txnTable.filter("key == 1").count() }, concurrentUpdateOp = txnTableName => { sql(s"DELETE FROM $txnTableName WHERE key = 1") }, currentThreadCommitOperation = txn => { // Concurrent delete op touches the same partition as those read in the active txn. txn.commit(Seq.empty, DeltaOperations.ManualUpdate) }, shouldFail = true) testWithNewTransaction( name = "snapshot isolation for query that can leverage metadata query optimization", partitionedTableKeys = Seq(1, 2, 3), currentThreadReadOp = txnTable => { txnTable.count() }, concurrentUpdateOp = txnTableName => { sql(s"DELETE FROM $txnTableName WHERE key = 1") }, currentThreadCommitOperation = txn => { txn.commit(Seq.empty, DeltaOperations.ManualUpdate) }, shouldFail = true) testWithNewTransaction( name = "snapshot isolation for query that can leverage metadata query optimization " + "with partition filter + conflicting concurrent updates", partitionedTableKeys = Seq(1, 2, 3), currentThreadReadOp = txnTable => { txnTable.filter("key == 1").count() }, concurrentUpdateOp = txnTableName => { sql(s"DELETE FROM $txnTableName WHERE key = 1") }, currentThreadCommitOperation = txn => { // Concurrent delete op touches the same partition as those read in the active txn. txn.commit(Seq.empty, DeltaOperations.ManualUpdate) }, shouldFail = true) testWithNewTransaction( name = "capture reads on txn table with data filter + conflicting concurrent updates", partitionedTableKeys = Seq(1, 2, 3), // will generate (key, value) = (1, 1), (2, 2), (3, 3) currentThreadReadOp = txnTable => { txnTable.filter("value == 1").count() // pure data filter that touches one file }, concurrentUpdateOp = txnTableName => { sql(s"DELETE FROM $txnTableName WHERE key = 1") // deletes the one file read above }, currentThreadCommitOperation = txn => { txn.commit(Seq.empty, DeltaOperations.ManualUpdate) }, shouldFail = true) testWithNewTransaction( name = "capture reads on txn table with partition filter + non-conflicting concurrent updates", partitionedTableKeys = Seq(1, 2, 3), currentThreadReadOp = txnTable => { txnTable.filter("key == 1").count() }, concurrentUpdateOp = txnTableName => { sql(s"DELETE FROM $txnTableName WHERE key = 2") sql(s"INSERT INTO $txnTableName SELECT 4, 4") }, currentThreadCommitOperation = txn => { // Concurrent delete op touches the different files as those read in the active txn. txn.commit(Seq.empty, DeltaOperations.ManualUpdate) }, shouldFail = false) testWithNewTransaction( name = "snapshot isolation for metadata optimizable query with partition filter +" + " non-conflicting concurrent updates", partitionedTableKeys = Seq(1, 2, 3), currentThreadReadOp = txnTable => { txnTable.filter("key == 1").count() }, concurrentUpdateOp = txnTableName => { sql(s"DELETE FROM $txnTableName WHERE key = 2") sql(s"INSERT INTO $txnTableName SELECT 4, 4") }, currentThreadCommitOperation = txn => { // Concurrent delete op touches the different files as those read in the active txn. txn.commit(Seq.empty, DeltaOperations.ManualUpdate) }, shouldFail = false) testWithNewTransaction( name = "capture reads on txn table with filter+limit and conflicting concurrent updates", partitionedTableKeys = Seq(1, 2, 3), currentThreadReadOp = txnTable => { txnTable.filter("key == 1").limit(1).collect() }, concurrentUpdateOp = txnTableName => { sql(s"DELETE FROM $txnTableName WHERE key = 1") }, currentThreadCommitOperation = txn => { // Concurrent delete op touches the same files as those read in the active txn. txn.commit(Seq.empty, DeltaOperations.ManualUpdate) }, shouldFail = true) testWithNewTransaction( name = "capture reads on txn table with filter+limit and non-conflicting concurrent updates", partitionedTableKeys = Seq(1, 2, 3), currentThreadReadOp = txnTable => { txnTable.filter("key == 1").limit(1).collect() }, concurrentUpdateOp = txnTableName => { sql(s"DELETE FROM $txnTableName WHERE key = 2") sql(s"INSERT INTO $txnTableName SELECT 4, 4") }, currentThreadCommitOperation = txn => { // Concurrent delete op touches the different files as those read in the active txn. txn.commit(Seq.empty, DeltaOperations.ManualUpdate) }, shouldFail = false) testWithNewTransaction( name = "capture reads on txn table with limit + conflicting concurrent updates", partitionedTableKeys = Seq(1, 2, 3), currentThreadReadOp = txnTable => { txnTable.limit(1).collect() }, concurrentUpdateOp = txnTableName => { sql(s"DELETE FROM $txnTableName WHERE true") }, currentThreadCommitOperation = txn => { txn.commit(Seq.empty, DeltaOperations.ManualUpdate) }, shouldFail = true) testWithNewTransaction( name = "capture reads on txn table even when limit pushdown is disabled", confs = Map(DeltaSQLConf.DELTA_LIMIT_PUSHDOWN_ENABLED.key -> "false"), partitionedTableKeys = Seq(1, 2, 3), currentThreadReadOp = txnTable => { txnTable.limit(1).collect() }, concurrentUpdateOp = txnTableName => { sql(s"UPDATE $txnTableName SET key = 2 WHERE key = 3") }, currentThreadCommitOperation = txn => { // Any concurrent change (even if its seemingly non-conflicting) should fail the filter as // the whole table will be scanned by the filter when data skipping is disabled txn.commit(Seq.empty, DeltaOperations.ManualUpdate) }, shouldFail = true) test("withNewTransaction - nesting withNewTransaction is not supported") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir.getCanonicalPath) log.withNewTransaction { txn => assert(OptimisticTransaction.getActive() === Some(txn)) intercept[IllegalStateException] { log.withNewTransaction { txn2 => } } assert(OptimisticTransaction.getActive() === Some(txn)) } assert(OptimisticTransaction.getActive().isEmpty) } } test("withActiveTxn idempotency") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir.getCanonicalPath) val txn = log.startTransaction() assert(OptimisticTransaction.getActive().isEmpty) OptimisticTransaction.withActive(txn) { assert(OptimisticTransaction.getActive() === Some(txn)) OptimisticTransaction.withActive(txn) { assert(OptimisticTransaction.getActive() === Some(txn)) } assert(OptimisticTransaction.getActive() === Some(txn)) val txn2 = log.startTransaction() intercept[IllegalStateException] { OptimisticTransaction.withActive(txn2) { } } intercept[IllegalStateException] { OptimisticTransaction.setActive(txn2) } assert(OptimisticTransaction.getActive() === Some(txn)) } assert(OptimisticTransaction.getActive().isEmpty) } } testWithNewTransaction( name = "capture reads on txn table even when data skipping is disabled", confs = Map(DeltaSQLConf.DELTA_STATS_SKIPPING.key -> "false"), partitionedTableKeys = Seq(1, 2, 3), currentThreadReadOp = txnTable => { txnTable.filter("key == 1").count() }, concurrentUpdateOp = txnTableName => { sql(s"UPDATE $txnTableName SET key = 2 WHERE key = 3") }, currentThreadCommitOperation = txn => { // use physical name val key = getPhysicalName("key", txn.metadata.schema) // Any concurrent change (even if its seemingly non-conflicting) should fail the filter as // the whole table will be scanned by the filter when data skipping is disabled. // Note: Adding a file to avoid snapshot isolation level for the commit. txn.commit( Seq(createTestAddFile(encodedPath = "a", partitionValues = Map(key -> "2"))), DeltaOperations.ManualUpdate ) }, shouldFail = true) def testSnapshotIsolation(): Unit = { val txnTablePath = Utils.createTempDir().getCanonicalPath val nonTxnTablePath = Utils.createTempDir().getCanonicalPath def txnTable: DataFrame = spark.read.format("delta").load(txnTablePath) def nonTxnTable: DataFrame = spark.read.format("delta").load(nonTxnTablePath) def writeToNonTxnTable(ds: Dataset[java.lang.Long]): Unit = { import testImplicits._ ds.toDF("key").select('key, 'key as "value") .write.format("delta").mode("append").partitionBy("key").save(nonTxnTablePath) DeltaLog.forTable(spark, nonTxnTablePath).update(stalenessAcceptable = false) } testWithNewTransaction( name = s"snapshot isolation uses first-access snapshots when enabled", partitionTablePath = txnTablePath, partitionedTableKeys = Seq(1, 2, 3, 4, 5), // Prepare txn-table preTxnSetup = _ => { writeToNonTxnTable(spark.range(3)) // Prepare non-txn table }, currentThreadReadOp = txnTable => { // First read on tables require(txnTable.count() == 5) require(nonTxnTable.count() === 3) }, concurrentUpdateOp = txnTableName => { // Update tables in a different thread and make sure the DeltaLog gets updated sql(s"INSERT INTO $txnTableName SELECT 6, 6") DeltaLog.forTable(spark, txnTablePath).update(stalenessAcceptable = false) require(txnTable.count() == 6) writeToNonTxnTable(spark.range(3, 10)) require(nonTxnTable.count() == 10) }, currentThreadCommitOperation = _ => { // Second read on concurrently updated tables should read old snapshots assert(txnTable.count() == 5, "snapshot isolation failed on txn table") assert(nonTxnTable.count() == 3, "snapshot isolation failed on non-txn table") }, shouldFail = false) } testSnapshotIsolation() } class DeltaWithNewTransactionSuite extends DeltaWithNewTransactionSuiteBase class DeltaWithNewTransactionIdColumnMappingSuite extends DeltaWithNewTransactionSuite with DeltaColumnMappingEnableIdMode class DeltaWithNewTransactionNameColumnMappingSuite extends DeltaWithNewTransactionSuite with DeltaColumnMappingEnableNameMode class DeltaWithNewTransactionWithCatalogOwnedBatch1Suite extends DeltaWithNewTransactionSuite { override val catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class DeltaWithNewTransactionWithCatalogOwnedBatch2Suite extends DeltaWithNewTransactionSuite { override val catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class DeltaWithNewTransactionWithCatalogOwnedBatch100Suite extends DeltaWithNewTransactionSuite { override val catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DeltaWriteConfigsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.Locale import scala.collection.mutable.ListBuffer import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.StringType /** * This test suite tests all (or nearly-all) combinations of ways to write configs to a delta table. * * At a high level, it tests the following matrix of conditions: * * - DataFrameWriter or DataStreamWriter or DataFrameWriterV2 or DeltaTableBuilder or SQL API * X * - option is / is not prefixed with 'delta' * X * - using table name or table path * X * - CREATE or REPLACE or CREATE OR REPLACE (table already exists) OR CREATE OR REPLACE (table * doesn't already exist) * * At the end of the test suite, it prints out summary tables all of the cases above. */ class DeltaWriteConfigsSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { val config_no_prefix = "dataSkippingNumIndexedCols" val config_no_prefix_value = "33" val config_prefix = "delta.deletedFileRetentionDuration" val config_prefix_value = "interval 2 weeks" val config_no_prefix_2 = "logRetentionDuration" val config_no_prefix_2_value = "interval 60 days" val config_prefix_2 = "delta.checkpointInterval" val config_prefix_2_value = "20" override def afterAll(): Unit = { import testImplicits._ // scalastyle:off println println("DataFrameWriter Test Output") dfw_output.toSeq .toDF("Output Location", "Output Mode", s"Contains No-Prefix Option", "Contains Prefix-Option", "Config") .show(100, false) println("DataStreamWriter Test Output") dsw_output.toSeq .toDF("Output Location", "Output Mode", s"Contains No-Prefix Option", "Contains Prefix-Option", "Config") .show(100, false) println("DataFrameWriterV2 Test Output") dfw_v2_output.toSeq .toDF("Output Location", "Output Mode", s"Contains No-Prefix Option", "Contains Prefix-Option", "Config") .show(100, false) println("DeltaTableBuilder Test Output") dtb_output.toSeq .toDF("Output Location", "Output Mode", s"Contains No-Prefix Option (lowercase)", s"Contains No-Prefix Option", "Contains Prefix-Option", "ERROR", "Config") .show(100, false) println("SQL Test Output") sql_output.toSeq .toDF("Output Location", "Config Input", s"SQL Operation", "AS SELECT", "Contains OPTION no-prefix", "Contains OPTION prefix", "Contains TBLPROPERTIES no-prefix", "Contains TBLPROPERTIES prefix", "Config") .show(100, false) // scalastyle:on println super.afterAll() } private val dfw_output = new ListBuffer[DeltaFrameStreamAPITestOutput] private val dsw_output = new ListBuffer[DeltaFrameStreamAPITestOutput] private val dfw_v2_output = new ListBuffer[DeltaFrameStreamAPITestOutput] private val dtb_output = new ListBuffer[DeltaTableBuilderAPITestOutput] private val sql_output = new ListBuffer[SQLAPIOutput] // scalastyle:off line.size.limit /* DataFrameWriter Test Output +---------------+-----------+-------------------------+----------------------+------------------------------------------------------+ |Output Location|Output Mode|Contains No-Prefix Option|Contains Prefix-Option|Config | +---------------+-----------+-------------------------+----------------------+------------------------------------------------------+ |path |create |false |true |delta.deletedFileRetentionDuration -> interval 2 weeks| |path |overwrite |false |true |delta.deletedFileRetentionDuration -> interval 2 weeks| |path |append |false |true |delta.deletedFileRetentionDuration -> interval 2 weeks| |table |create |false |true |delta.deletedFileRetentionDuration -> interval 2 weeks| |table |overwrite |false |true |delta.deletedFileRetentionDuration -> interval 2 weeks| |table |append |false |true |delta.deletedFileRetentionDuration -> interval 2 weeks| +---------------+-----------+-------------------------+----------------------+------------------------------------------------------+ */ // scalastyle:on line.size.limit Seq("path", "table").foreach { outputLoc => Seq("create", "overwrite", "append").foreach { outputMode => val testName = s"DataFrameWriter - outputLoc=$outputLoc & mode=$outputMode" test(testName) { withTempDir { dir => withTable("tbl") { var data = spark.range(10).write.format("delta") .option(config_no_prefix, config_no_prefix_value) .option(config_prefix, config_prefix_value) if (outputMode != "create") { data = data.mode(outputMode) } val log = outputLoc match { case "path" => data.save(dir.getCanonicalPath) DeltaLog.forTable(spark, dir) case "table" => data.saveAsTable("tbl") DeltaLog.forTable(spark, TableIdentifier("tbl")) } val config = log.snapshot.metadata.configuration val answer_no_prefix = config.contains(config_no_prefix) val answer_prefix = config.contains(config_prefix) assert(!answer_no_prefix) assert(answer_prefix) assert(config.size == 1) dfw_output += DeltaFrameStreamAPITestOutput( outputLocation = outputLoc, outputMode = outputMode, containsNoPrefixOption = answer_no_prefix, containsPrefixOption = answer_prefix, config = config.mkString(",") ) } } } } } // scalastyle:off line.size.limit /* DataStreamWriter Test Output +---------------+-----------+-------------------------+----------------------+------+ |Output Location|Output Mode|Contains No-Prefix Option|Contains Prefix-Option|Config| +---------------+-----------+-------------------------+----------------------+------+ |path |create |false |false | | |path |append |false |false | | |path |complete |false |false | | |table |create |false |false | | |table |append |false |false | | |table |complete |false |false | | +---------------+-----------+-------------------------+----------------------+------+ */ // scalastyle:on line.size.limit // Data source DeltaDataSource does not support Update output mode Seq("path", "table").foreach { outputLoc => Seq("create", "append", "complete").foreach { outputMode => val testName = s"DataStreamWriter - outputLoc=$outputLoc & outputMode=$outputMode" test(testName) { withTempDir { dir => withTempDir { checkpointDir => withTable("src", "tbl") { spark.range(10).write.format("delta").saveAsTable("src") var data = spark.readStream.format("delta").table("src") // Needed to resolve error: Complete output mode not supported when there are no // streaming aggregations on streaming DataFrames/Datasets if (outputMode == "complete") { data = data.groupBy().count() } var stream = data.writeStream .format("delta") .option("checkpointLocation", checkpointDir.getCanonicalPath) .option(config_no_prefix, config_no_prefix_value) .option(config_prefix, config_prefix_value) if (outputMode != "create") { stream = stream.outputMode(outputMode) } val log = outputLoc match { case "path" => stream.start(dir.getCanonicalPath).stop() DeltaLog.forTable(spark, dir) case "table" => stream.toTable("tbl").stop() DeltaLog.forTable(spark, TableIdentifier("tbl")) } val config = log.snapshot.metadata.configuration val answer_no_prefix = config.contains(config_no_prefix) val answer_prefix = config.contains(config_prefix) assert(config.isEmpty) assert(!answer_no_prefix) assert(!answer_prefix) dsw_output += DeltaFrameStreamAPITestOutput( outputLocation = outputLoc, outputMode = outputMode, containsNoPrefixOption = answer_no_prefix, containsPrefixOption = answer_prefix, config = config.mkString(",") ) } } } } } } // scalastyle:off line.size.limit /* DataFrameWriterV2 Test Output +---------------+--------------+-------------------------+----------------------+------------------------------------------------------+ |Output Location|Output Mode |Contains No-Prefix Option|Contains Prefix-Option|Config | +---------------+--------------+-------------------------+----------------------+------------------------------------------------------+ |path |create |false |true |delta.deletedFileRetentionDuration -> interval 2 weeks| |path |replace |false |true |delta.deletedFileRetentionDuration -> interval 2 weeks| |path |c_or_r_create |false |true |delta.deletedFileRetentionDuration -> interval 2 weeks| |path |c_or_r_replace|false |true |delta.deletedFileRetentionDuration -> interval 2 weeks| |table |create |false |true |delta.deletedFileRetentionDuration -> interval 2 weeks| |table |replace |false |true |delta.deletedFileRetentionDuration -> interval 2 weeks| |table |c_or_r_create |false |true |delta.deletedFileRetentionDuration -> interval 2 weeks| |table |c_or_r_replace|false |true |delta.deletedFileRetentionDuration -> interval 2 weeks| +---------------+--------------+-------------------------+----------------------+------------------------------------------------------+ */ // scalastyle:on line.size.limit Seq("path", "table").foreach { outputLoc => Seq("create", "replace", "c_or_r_create", "c_or_r_replace").foreach { outputMode => val testName = s"DataFrameWriterV2 - outputLoc=$outputLoc & outputMode=$outputMode" test(testName) { withTempDir { dir => withTable("tbl") { val table = outputLoc match { case "path" => s"delta.`${dir.getCanonicalPath}`" case "table" => "tbl" } val data = spark.range(10).writeTo(table).using("delta") .option(config_no_prefix, config_no_prefix_value) .option(config_prefix, config_prefix_value) if (outputMode.contains("replace")) { spark.range(100).writeTo(table).using("delta").create() } outputMode match { case "create" => data.create() case "replace" => data.replace() case "c_or_r_create" | "c_or_r_replace" => data.createOrReplace() } val log = outputLoc match { case "path" => DeltaLog.forTable(spark, dir) case "table" => DeltaLog.forTable(spark, TableIdentifier("tbl")) } val config = log.snapshot.metadata.configuration val answer_no_prefix = config.contains(config_no_prefix) val answer_prefix = config.contains(config_prefix) assert(!answer_no_prefix) assert(answer_prefix) assert(config.size == 1) dfw_v2_output += DeltaFrameStreamAPITestOutput( outputLocation = outputLoc, outputMode = outputMode, containsNoPrefixOption = answer_no_prefix, containsPrefixOption = answer_prefix, config = config.mkString(",") ) } } } } } // scalastyle:off line.size.limit /* DeltaTableBuilder Test Output +---------------+--------------+-------------------------------------+-------------------------+----------------------+-----+---------------------------------------------------------------------------------------+ |Output Location|Output Mode |Contains No-Prefix Option (lowercase)|Contains No-Prefix Option|Contains Prefix-Option|ERROR|Config | +---------------+--------------+-------------------------------------+-------------------------+----------------------+-----+---------------------------------------------------------------------------------------+ |path |create |true |false |true |false|delta.deletedFileRetentionDuration -> interval 2 weeks,dataSkippingNumIndexedCols -> 33| |path |replace |true |false |true |false|delta.deletedFileRetentionDuration -> interval 2 weeks,dataSkippingNumIndexedCols -> 33| |path |c_or_r_create |true |false |true |false|delta.deletedFileRetentionDuration -> interval 2 weeks,dataSkippingNumIndexedCols -> 33| |path |c_or_r_replace|false |false |false |true | | |table |create |true |false |true |false|dataSkippingNumIndexedCols -> 33,delta.deletedFileRetentionDuration -> interval 2 weeks| |table |replace |true |false |true |false|delta.deletedFileRetentionDuration -> interval 2 weeks,dataSkippingNumIndexedCols -> 33| |table |c_or_r_create |true |false |true |false|delta.deletedFileRetentionDuration -> interval 2 weeks,dataSkippingNumIndexedCols -> 33| |table |c_or_r_replace|true |false |true |false|delta.deletedFileRetentionDuration -> interval 2 weeks,dataSkippingNumIndexedCols -> 33| +---------------+--------------+-------------------------------------+-------------------------+----------------------+-----+---------------------------------------------------------------------------------------+ */ // scalastyle:on line.size.limit Seq("path", "table").foreach { outputLoc => Seq("create", "replace", "c_or_r_create", "c_or_r_replace").foreach { outputMode => val testName = s"DeltaTableBuilder - outputLoc=$outputLoc & outputMode=$outputMode" test(testName) { withTempDir { dir => withTable("tbl") { if (outputMode.contains("replace")) { outputLoc match { case "path" => io.delta.tables.DeltaTable.create() .addColumn("bar", StringType).location(dir.getCanonicalPath).execute() case "table" => io.delta.tables.DeltaTable.create() .addColumn("bar", StringType).tableName("tbl").execute() } } var tblBuilder = outputMode match { case "create" => io.delta.tables.DeltaTable.create() case "replace" => io.delta.tables.DeltaTable.replace() case "c_or_r_create" | "c_or_r_replace" => io.delta.tables.DeltaTable.createOrReplace() } tblBuilder.addColumn("foo", StringType) tblBuilder = tblBuilder.property(config_no_prefix, config_no_prefix_value) tblBuilder = tblBuilder.property(config_prefix, config_prefix_value) val log = (outputLoc, outputMode) match { case ("path", "c_or_r_replace") => intercept[DeltaAnalysisException] { tblBuilder.location(dir.getCanonicalPath).execute() } null case ("path", _) => tblBuilder.location(dir.getCanonicalPath).execute() DeltaLog.forTable(spark, dir) case ("table", _) => tblBuilder.tableName("tbl").execute() DeltaLog.forTable(spark, TableIdentifier("tbl")) } log match { case null => // CREATE OR REPLACE seems broken when using path and the table already exists // with a different schema. // DeltaAnalysisException: The specified schema does not match the existing schema // ... // Specified schema is missing field(s): bar // Specified schema has additional field(s): foo assert(outputLoc == "path" && outputMode == "c_or_r_replace") dtb_output += DeltaTableBuilderAPITestOutput( outputLocation = outputLoc, outputMode = outputMode, containsNoPrefixOptionLowerCase = false, containsNoPrefixOption = false, containsPrefixOption = false, error = true, config = "" ) case _ => val config = log.snapshot.metadata.configuration val answer_no_prefix_lowercase = config.contains(config_no_prefix.toLowerCase(Locale.ROOT)) val answer_no_prefix = config.contains(config_no_prefix) val answer_prefix = config.contains(config_prefix) assert(!answer_no_prefix_lowercase) assert(answer_no_prefix) assert(answer_prefix) assert(config.size == 2) dtb_output += DeltaTableBuilderAPITestOutput( outputLocation = outputLoc, outputMode = outputMode, containsNoPrefixOptionLowerCase = answer_no_prefix_lowercase, containsNoPrefixOption = answer_no_prefix, containsPrefixOption = answer_prefix, error = false, config = config.mkString(",") ) } } } } } } // scalastyle:off line.size.limit /* SQL Test Output +---------------+-------------------------+--------------+---------+-------------------------+----------------------+--------------------------------+-----------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |Output Location|Config Input |SQL Operation |AS SELECT|Contains OPTION no-prefix|Contains OPTION prefix|Contains TBLPROPERTIES no-prefix|Contains TBLPROPERTIES prefix|Config | +---------------+-------------------------+--------------+---------+-------------------------+----------------------+--------------------------------+-----------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |path |options |create |true |false |true |N/A |N/A |delta.deletedFileRetentionDuration -> interval 2 weeks | |path |options |create |false |true |true |N/A |N/A |delta.deletedFileRetentionDuration -> interval 2 weeks,dataSkippingNumIndexedCols -> 33,option.delta.deletedFileRetentionDuration -> interval 2 weeks,option.dataSkippingNumIndexedCols -> 33 | |path |options |replace |true |false |true |N/A |N/A |delta.deletedFileRetentionDuration -> interval 2 weeks | |path |options |replace |false |false |true |N/A |N/A |delta.deletedFileRetentionDuration -> interval 2 weeks | |path |options |c_or_r_create |true |false |true |N/A |N/A |delta.deletedFileRetentionDuration -> interval 2 weeks | |path |options |c_or_r_create |false |false |true |N/A |N/A |delta.deletedFileRetentionDuration -> interval 2 weeks | |path |options |c_or_r_replace|true |false |true |N/A |N/A |delta.deletedFileRetentionDuration -> interval 2 weeks | |path |options |c_or_r_replace|false |false |true |N/A |N/A |delta.deletedFileRetentionDuration -> interval 2 weeks | |path |tblproperties |create |true |N/A |N/A |true |true |logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |path |tblproperties |create |false |N/A |N/A |true |true |logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |path |tblproperties |replace |true |N/A |N/A |true |true |logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |path |tblproperties |replace |false |N/A |N/A |true |true |logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |path |tblproperties |c_or_r_create |true |N/A |N/A |true |true |logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |path |tblproperties |c_or_r_create |false |N/A |N/A |true |true |logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |path |tblproperties |c_or_r_replace|true |N/A |N/A |true |true |logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |path |tblproperties |c_or_r_replace|false |N/A |N/A |true |true |logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |path |options_and_tblproperties|create |true |false |true |true |true |delta.deletedFileRetentionDuration -> interval 2 weeks,logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |path |options_and_tblproperties|create |false |true |true |true |true |delta.deletedFileRetentionDuration -> interval 2 weeks,dataSkippingNumIndexedCols -> 33,logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20,option.delta.deletedFileRetentionDuration -> interval 2 weeks,option.dataSkippingNumIndexedCols -> 33| |path |options_and_tblproperties|replace |true |false |true |true |true |delta.deletedFileRetentionDuration -> interval 2 weeks,logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |path |options_and_tblproperties|replace |false |false |true |true |true |delta.deletedFileRetentionDuration -> interval 2 weeks,logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |path |options_and_tblproperties|c_or_r_create |true |false |true |true |true |delta.deletedFileRetentionDuration -> interval 2 weeks,logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |path |options_and_tblproperties|c_or_r_create |false |false |true |true |true |delta.deletedFileRetentionDuration -> interval 2 weeks,logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |path |options_and_tblproperties|c_or_r_replace|true |false |true |true |true |delta.deletedFileRetentionDuration -> interval 2 weeks,logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |path |options_and_tblproperties|c_or_r_replace|false |false |true |true |true |delta.deletedFileRetentionDuration -> interval 2 weeks,logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |table |options |create |true |false |true |N/A |N/A |delta.deletedFileRetentionDuration -> interval 2 weeks | |table |options |create |false |true |true |N/A |N/A |delta.deletedFileRetentionDuration -> interval 2 weeks,dataSkippingNumIndexedCols -> 33,option.delta.deletedFileRetentionDuration -> interval 2 weeks,option.dataSkippingNumIndexedCols -> 33 | |table |options |replace |true |false |true |N/A |N/A |delta.deletedFileRetentionDuration -> interval 2 weeks | |table |options |replace |false |false |true |N/A |N/A |delta.deletedFileRetentionDuration -> interval 2 weeks | |table |options |c_or_r_create |true |false |true |N/A |N/A |delta.deletedFileRetentionDuration -> interval 2 weeks | |table |options |c_or_r_create |false |false |true |N/A |N/A |delta.deletedFileRetentionDuration -> interval 2 weeks | |table |options |c_or_r_replace|true |false |true |N/A |N/A |delta.deletedFileRetentionDuration -> interval 2 weeks | |table |options |c_or_r_replace|false |false |true |N/A |N/A |delta.deletedFileRetentionDuration -> interval 2 weeks | |table |tblproperties |create |true |N/A |N/A |true |true |logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |table |tblproperties |create |false |N/A |N/A |true |true |logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |table |tblproperties |replace |true |N/A |N/A |true |true |logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |table |tblproperties |replace |false |N/A |N/A |true |true |logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |table |tblproperties |c_or_r_create |true |N/A |N/A |true |true |logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |table |tblproperties |c_or_r_create |false |N/A |N/A |true |true |logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |table |tblproperties |c_or_r_replace|true |N/A |N/A |true |true |logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |table |tblproperties |c_or_r_replace|false |N/A |N/A |true |true |logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |table |options_and_tblproperties|create |true |false |true |true |true |delta.deletedFileRetentionDuration -> interval 2 weeks,logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |table |options_and_tblproperties|create |false |true |true |true |true |delta.deletedFileRetentionDuration -> interval 2 weeks,dataSkippingNumIndexedCols -> 33,logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20,option.delta.deletedFileRetentionDuration -> interval 2 weeks,option.dataSkippingNumIndexedCols -> 33| |table |options_and_tblproperties|replace |true |false |true |true |true |delta.deletedFileRetentionDuration -> interval 2 weeks,logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |table |options_and_tblproperties|replace |false |false |true |true |true |delta.deletedFileRetentionDuration -> interval 2 weeks,logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |table |options_and_tblproperties|c_or_r_create |true |false |true |true |true |delta.deletedFileRetentionDuration -> interval 2 weeks,logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |table |options_and_tblproperties|c_or_r_create |false |false |true |true |true |delta.deletedFileRetentionDuration -> interval 2 weeks,logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |table |options_and_tblproperties|c_or_r_replace|true |false |true |true |true |delta.deletedFileRetentionDuration -> interval 2 weeks,logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | |table |options_and_tblproperties|c_or_r_replace|false |false |true |true |true |delta.deletedFileRetentionDuration -> interval 2 weeks,logRetentionDuration -> interval 60 days,delta.checkpointInterval -> 20 | +---------------+-------------------------+--------------+---------+-------------------------+----------------------+--------------------------------+-----------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ */ // scalastyle:on line.size.limit Seq("path", "table").foreach { outputLoc => Seq("options", "tblproperties", "options_and_tblproperties").foreach { configInput => Seq("create", "replace", "c_or_r_create", "c_or_r_replace").foreach { sqlOp => Seq(true, false).foreach { useAsSelectStmt => val testName = s"SQL - outputLoc=$outputLoc & configInput=$configInput & sqlOp=$sqlOp" + s" & useAsSelectStmt=$useAsSelectStmt" test(testName) { withTempDir { dir => withTable("tbl", "other") { if (sqlOp.contains("replace")) { var stmt = "CREATE TABLE tbl (ID INT) USING DELTA" if (outputLoc == "path") { stmt = stmt + s" LOCATION '${dir.getCanonicalPath}'" } sql(stmt) } val sqlOpStr = sqlOp match { case "c_or_r_create" | "c_or_r_replace" => "CREATE OR REPLACE" case _ => sqlOp.toUpperCase(Locale.ROOT) } val schemaStr = if (useAsSelectStmt) "" else "(id INT) " var stmt = sqlOpStr + " TABLE tbl " + schemaStr + "USING DELTA\n" if (configInput.contains("options")) { stmt = stmt + s"OPTIONS(" + s"'$config_no_prefix'=$config_no_prefix_value," + s"'$config_prefix'='$config_prefix_value')\n" } if (outputLoc == "path") { stmt = stmt + s"LOCATION '${dir.getCanonicalPath}'\n" } if (configInput.contains("tblproperties")) { stmt = stmt + s"TBLPROPERTIES(" + s"'$config_no_prefix_2'='$config_no_prefix_2_value'," + s"'$config_prefix_2'=$config_prefix_2_value)\n" } if (useAsSelectStmt) { sql("CREATE TABLE other (id INT) USING DELTA") stmt = stmt + "AS SELECT * FROM other\n" } // scalastyle:off println println(stmt) // scalastyle:on println sql(stmt) val log = DeltaLog.forTable(spark, TableIdentifier("tbl")) val config = log.snapshot.metadata.configuration val option_was_set = configInput.contains("options") val tblproperties_was_set = configInput.contains("tblproperties") val option_no_prefix = config.contains(config_no_prefix) val option_prefix = config.contains(config_prefix) val tblproperties_no_prefix = config.contains(config_no_prefix_2) val tblproperties_prefix = config.contains(config_prefix_2) var expectedSize = 0 if (option_was_set) { assert(option_prefix) expectedSize += 1 if (sqlOp == "create" && !useAsSelectStmt) { assert(option_no_prefix) assert(config.contains(s"option.$config_prefix")) assert(config.contains(s"option.$config_no_prefix")) expectedSize += 3 } } if (tblproperties_was_set) { assert(tblproperties_prefix) assert(tblproperties_no_prefix) expectedSize += 2 } assert(config.size == expectedSize) sql_output += SQLAPIOutput( outputLoc, configInput, sqlOp, useAsSelectStmt, if (option_was_set) option_no_prefix.toString else "N/A", if (option_was_set) option_prefix.toString else "N/A", if (tblproperties_was_set) tblproperties_no_prefix.toString else "N/A", if (tblproperties_was_set) tblproperties_prefix.toString else "N/A", config.mkString(",") ) } } } } } } } } // Need to be outside to be stable references for Spark to generate the case classes case class DeltaFrameStreamAPITestOutput( outputLocation: String, outputMode: String, containsNoPrefixOption: Boolean, containsPrefixOption: Boolean, config: String) case class DeltaTableBuilderAPITestOutput( outputLocation: String, outputMode: String, containsNoPrefixOptionLowerCase: Boolean, containsNoPrefixOption: Boolean, containsPrefixOption: Boolean, error: Boolean, config: String) case class SQLAPIOutput( outputLocation: String, confiInput: String, sqlOperation: String, asSelect: Boolean, containsOptionNoPrefix: String, containsOptionPrefix: String, containsTblPropertiesNoPrefix: String, containsTblPropertiesPrefix: String, config: String) ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DescribeDeltaDetailSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.io.FileNotFoundException // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils.{TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION} import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.Utils trait DescribeDeltaDetailSuiteBase extends QueryTest with SharedSparkSession with CatalogOwnedTestBaseSuite with DeltaTestUtilsForTempViews { import testImplicits._ val catalogAndSchema = { s"$SESSION_CATALOG_NAME.default." } protected def checkResult( result: DataFrame, expected: Seq[Any], columns: Seq[String]): Unit = { checkAnswer( result.select(columns.head, columns.tail: _*), Seq(Row(expected: _*)) ) } def describeDeltaDetailTest(f: File => String): Unit = { val tempDir = Utils.createTempDir() Seq(1 -> 1).toDF("column1", "column2") .write .format("delta") .partitionBy("column1") .save(tempDir.toString()) // Check SQL details checkResult( sql(s"DESCRIBE DETAIL ${f(tempDir)}"), Seq("delta", Array("column1"), 1), Seq("format", "partitionColumns", "numFiles")) // Check Scala details val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tempDir.toString) checkResult( deltaTable.detail(), Seq("delta", Array("column1"), 1), Seq("format", "partitionColumns", "numFiles")) } test("delta table: Scala details using table name") { withTable("delta_test") { Seq(1, 2, 3).toDF().write.format("delta").saveAsTable("delta_test") val deltaTable = io.delta.tables.DeltaTable.forName(spark, "delta_test") checkAnswer( deltaTable.detail().select("format"), Seq(Row("delta")) ) } } test("delta table: path") { describeDeltaDetailTest(f => s"'${f.toString()}'") } test("delta table: delta table identifier") { describeDeltaDetailTest(f => s"delta.`${f.toString()}`") } test("non-delta table: SQL details using table name") { withTable("describe_detail") { sql( """ |CREATE TABLE describe_detail(column1 INT, column2 INT) |USING parquet |PARTITIONED BY (column1) |COMMENT "this is a table comment" """.stripMargin) sql( """ |INSERT INTO describe_detail VALUES(1, 1) """.stripMargin ) checkResult( sql("DESCRIBE DETAIL describe_detail"), Seq("parquet", Array("column1")), Seq("format", "partitionColumns")) } } test("non-delta table: SQL details using table path") { val tempDir = Utils.createTempDir().toString Seq(1 -> 1).toDF("column1", "column2") .write .format("parquet") .partitionBy("column1") .mode("overwrite") .save(tempDir) checkResult( sql(s"DESCRIBE DETAIL '$tempDir'"), Seq(tempDir), Seq("location")) } test("non-delta table: SQL details when table path doesn't exist") { val tempDir = Utils.createTempDir() tempDir.delete() val e = intercept[FileNotFoundException] { sql(s"DESCRIBE DETAIL '$tempDir'") } assert(e.getMessage.contains(tempDir.toString)) } test("delta table: SQL details using table name") { withTable("describe_detail") { sql( """ |CREATE TABLE describe_detail(column1 INT, column2 INT) |USING delta |PARTITIONED BY (column1) |COMMENT "describe a non delta table" """.stripMargin) sql( """ |INSERT INTO describe_detail VALUES(1, 1) """.stripMargin ) checkResult( sql("DESCRIBE DETAIL describe_detail"), Seq("delta", Array("column1"), 1), Seq("format", "partitionColumns", "numFiles")) } } test("delta table: create table on an existing delta log") { val tempDir = Utils.createTempDir().toString Seq(1 -> 1).toDF("column1", "column2") .write .format("delta") .partitionBy("column1") .mode("overwrite") .save(tempDir) val tblName1 = "tbl_name1" val tblName2 = "tbl_name2" withTable(tblName1, tblName2) { sql(s"CREATE TABLE $tblName1 USING DELTA LOCATION '$tempDir'") sql(s"CREATE TABLE $tblName2 USING DELTA LOCATION '$tempDir'") checkResult( sql(s"DESCRIBE DETAIL $tblName1"), Seq(s"$catalogAndSchema$tblName1"), Seq("name")) checkResult( sql(s"DESCRIBE DETAIL $tblName2"), Seq(s"$catalogAndSchema$tblName2"), Seq("name")) checkResult( sql(s"DESCRIBE DETAIL delta.`$tempDir`"), Seq(null), Seq("name")) checkResult( sql(s"DESCRIBE DETAIL '$tempDir'"), Seq(null), Seq("name")) } } testWithTempView(s"SC-37296: describe detail on temp view") { isSQLTempView => withTable("t1") { Seq(1, 2, 3).toDF().write.format("delta").saveAsTable("t1") val viewName = "v" createTempViewFromTable("t1", isSQLTempView) val e = intercept[AnalysisException] { sql(s"DESCRIBE DETAIL $viewName") } assert(e.getMessage.contains("'DESCRIBE DETAIL' expects a table")) } } test("SC-37296: describe detail on permanent view") { val view = "detailTestView" withView(view) { sql(s"CREATE VIEW $view AS SELECT 1") val e = intercept[AnalysisException] { sql(s"DESCRIBE DETAIL $view") } assert(e.getMessage.contains("'DESCRIBE DETAIL' expects a table")) } } test("delta table: describe detail always run on the latest snapshot") { val tableName = "tbl_name_on_latest_snapshot" withTable(tableName) { val tempDir = Utils.createTempDir().toString sql(s"CREATE TABLE $tableName USING DELTA LOCATION '$tempDir'") val deltaLog = DeltaLog.forTable(spark, tempDir) DeltaLog.clearCache() // Cache a new DeltaLog sql(s"DESCRIBE DETAIL $tableName") val txn = deltaLog.startTransaction() val metadata = txn.snapshot.metadata val newMetadata = metadata.copy(configuration = metadata.configuration ++ Map("foo" -> "bar") ) txn.commit(newMetadata :: Nil, DeltaOperations.ManualUpdate) val catalogOwnedProperties = constructCatalogOwnedSpecificTableProperties( spark, newMetadata) checkResult(sql(s"DESCRIBE DETAIL $tableName"), Seq(Map("foo" -> "bar") ++ catalogOwnedProperties), Seq("properties") ) } } test("delta table: describe detail shows table features") { if (catalogOwnedDefaultCreationEnabledInTests) { cancel("CatalogOwned is not compatible w/ the test since protocol version would be " + "set to (3, 7) by default for CC tables.") } withTable("t1") { withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> "1", DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "2" ) { Seq(1, 2, 3).toDF().write.format("delta").saveAsTable("t1") } val p = DeltaLog.forTable(spark, TableIdentifier("t1")).snapshot.protocol checkResult( sql(s"DESCRIBE DETAIL t1"), Seq( p.minReaderVersion, p.minWriterVersion, p.implicitlySupportedFeatures.map(_.name).toArray.sorted), Seq("minReaderVersion", "minWriterVersion", "tableFeatures")) val features = p.readerAndWriterFeatureNames ++ p.implicitlySupportedFeatures.map(_.name) sql(s"""ALTER TABLE t1 SET TBLPROPERTIES ( | delta.minReaderVersion = $TABLE_FEATURES_MIN_READER_VERSION, | delta.minWriterVersion = $TABLE_FEATURES_MIN_WRITER_VERSION, | delta.feature.${TestReaderWriterFeature.name} = 'enabled' |)""".stripMargin) checkResult( sql(s"DESCRIBE DETAIL t1"), Seq( TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION, (features + TestReaderWriterFeature.name).toArray.sorted), Seq("minReaderVersion", "minWriterVersion", "tableFeatures")) } } test("describe detail contains table name") { val tblName = "test_table" withTable(tblName) { spark.sql(s"CREATE TABLE $tblName(id INT) USING delta") val deltaTable = io.delta.tables.DeltaTable.forName(tblName) checkResult( deltaTable.detail(), Seq(s"$catalogAndSchema$tblName"), Seq("name") ) } } private def withTempTableOrDir(useTable: Boolean = true)(f: String => Unit): Unit = { if (useTable) { val testTable = "test_table" withTable(testTable) { f(testTable) } } else { withTempDir { dir => f(s"delta.`$dir`") } } } private def checkResultForClusteredTable( table: String, clusteringColumns: Array[String]): Unit = { // Check SQL API. checkResult( sql(s"DESCRIBE DETAIL $table"), Seq("delta", Array.empty, clusteringColumns, 0), Seq("format", "partitionColumns", "clusteringColumns", "numFiles")) // Check DeltaTable APIs. val isPathBased = table.startsWith("delta.") val deltaTable = if (isPathBased) { val path = table.replace("delta.`", "").dropRight(1) io.delta.tables.DeltaTable.forPath(path) } else { io.delta.tables.DeltaTable.forName(table) } checkResult( deltaTable.detail(), Seq("delta", Array.empty, clusteringColumns, 0), Seq("format", "partitionColumns", "clusteringColumns", "numFiles")) } Seq(true -> "", false -> " - path based").foreach { case (useTable, testSuffix) => test(s"describe liquid table$testSuffix") { withTempTableOrDir(useTable) { testTable => sql(s"CREATE TABLE $testTable(a STRUCT, d INT) USING DELTA " + "CLUSTER BY (a.b, d)") checkResultForClusteredTable(testTable, Array("a.b", "d")) sql(s"ALTER TABLE $testTable CLUSTER BY NONE") checkResultForClusteredTable(testTable, Array.empty) } } test(s"describe liquid table - column mapping$testSuffix") { withTempTableOrDir(useTable) { testTable => sql(s"CREATE TABLE $testTable (col1 STRING, col2 INT) USING delta CLUSTER BY (col1, col2)") sql(s"ALTER TABLE $testTable SET TBLPROPERTIES ('delta.columnMapping.mode' = 'name'," + "'delta.minReaderVersion' = '3'," + "'delta.minWriterVersion'= '7')") sql(s"ALTER TABLE $testTable RENAME COLUMN col2 TO new_col_name") checkResultForClusteredTable(testTable, Array("col1", "new_col_name")) } } } // TODO: run it with OSS Delta after it's supported } class DescribeDeltaDetailSuite extends DescribeDeltaDetailSuiteBase with DeltaSQLCommandTest class DescribeDeltaDetailWithCatalogOwnedBatch1Suite extends DescribeDeltaDetailSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class DescribeDeltaDetailWithCatalogOwnedBatch2Suite extends DescribeDeltaDetailSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class DescribeDeltaDetailWithCatalogOwnedBatch100Suite extends DescribeDeltaDetailSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DescribeDeltaHistorySuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.io.File import org.apache.spark.sql.delta.actions.{Action, AddCDCFile, AddFile, Metadata, Protocol, RemoveFile} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.DescribeDeltaHistoryCommand import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CatalogOwnedTestBaseSuite} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.test.shims.StreamingTestShims.MemoryStream import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import org.scalactic.source.Position import org.scalatest.Tag import org.apache.spark.SparkConf import org.apache.spark.sql.{AnalysisException, Column, DataFrame, QueryTest, Row, SaveMode} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{BooleanType, LongType, MapType, StringType, StructField, StructType, TimestampType} import org.apache.spark.util.Utils trait DescribeDeltaHistorySuiteBase extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with DeltaTestUtilsForTempViews with MergeIntoMetricsBase with CatalogOwnedTestBaseSuite with WriteOptionsTestBase { import testImplicits._ protected val evolvabilityResource = { new File("src/test/resources/delta/history/delta-0.2.0").getAbsolutePath() } protected val evolvabilityLastOp = Seq("STREAMING UPDATE", null, null) protected def deleteMetricsSchema(partitioned: Boolean) = if (partitioned) DeltaOperationMetrics.DELETE_PARTITIONS else DeltaOperationMetrics.DELETE protected val updateMetricsSchema = DeltaOperationMetrics.UPDATE protected val mergeMetricsSchema = DeltaOperationMetrics.MERGE protected val replaceWhereMetricsSchema = DeltaOperationMetrics.WRITE_REPLACE_WHERE protected def testWithFlag(name: String, tags: Tag*)(f: => Unit): Unit = { test(name, tags: _*) { f } } protected def checkLastOperation( basePath: String, expectedOperationParameters: Seq[String], expectedColVals: Seq[String], columns: Seq[Column] = Seq($"operation", $"operationParameters.mode"), removeExpressionId: Boolean = false): Unit = { var df = io.delta.tables.DeltaTable.forPath(spark, basePath).history(1) val operationParametersRow = df.select("operationParameters").collect()(0) assert(operationParametersRow.getAs[Map[String, String]](0).keys.toSeq === expectedOperationParameters) df = df.select(columns: _*) if (removeExpressionId) { // As the expression ID is written as part of the column predicate (in the form of col#expId) // but it is non-deterministic, we remove it here so that any comparison can just go against // the column name df = df.withColumn("predicate", regexp_replace(col("predicate"), "#[0-9]+", "")) } checkAnswer(df, Seq(Row(expectedColVals: _*))) df = spark.sql(s"DESCRIBE HISTORY delta.`$basePath` LIMIT 1") df = df.select(columns: _*) if (removeExpressionId) { df = df.withColumn("predicate", regexp_replace(col("predicate"), "#[0-9]+", "")) } checkAnswer(df, Seq(Row(expectedColVals: _*))) } /** * a separate check on properties is needed because order inside properties * is determined by order in Map and can differ between scala versions * Thus, we want to make sure check on properties can ignore orders and * check if all (key, value) property-pairs are expected */ protected def checkLastOperationProperties( basePath: String, expectedProperties: Map[String, String]): Unit = { def checkFirstRowPropertyCol(df: DataFrame): Unit = { val propertyDf = df.select(Seq($"operationParameters.properties"): _*) val actualPropertiesJson = propertyDf.take(1).head.getString(0) val actualProperties = JsonUtils.fromJson[Map[String, String]](actualPropertiesJson) if (catalogOwnedDefaultCreationEnabledInTests) { // We need to filter out the following two properties b/c // they are generated as part of [[RowTrackingFeature]] enablement, // the values of which are non-deterministic so we only verify the // existence. assert(actualProperties.contains(MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP) && actualProperties.contains(MaterializedRowCommitVersion.MATERIALIZED_COLUMN_NAME_PROP), "RowTracking should be enabled as part of CatalogOwned QoL features, " + s"expecting ${MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP} and " + s"${MaterializedRowCommitVersion.MATERIALIZED_COLUMN_NAME_PROP} to be present. " + s"The `actualProperties`: $actualProperties") val actualPropertiesForCO = actualProperties.filterNot { case (k, v) => k == MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP || k == MaterializedRowCommitVersion.MATERIALIZED_COLUMN_NAME_PROP } assert(actualPropertiesForCO == expectedProperties) } else { assert(actualProperties == expectedProperties) } } var df = io.delta.tables.DeltaTable.forPath(spark, basePath).history(1) checkFirstRowPropertyCol(df) // double verification df = spark.sql(s"DESCRIBE HISTORY delta.`$basePath` LIMIT 1") checkFirstRowPropertyCol(df) } protected def checkOperationMetrics( expectedMetrics: Map[String, String], operationMetrics: Map[String, String], metricsSchema: Set[String]): Unit = { if (metricsSchema != operationMetrics.keySet) { fail( s"""The collected metrics does not match the defined schema for the metrics. | Expected : $metricsSchema | Actual : ${operationMetrics.keySet} """.stripMargin) } expectedMetrics.keys.foreach { key => if (!operationMetrics.contains(key)) { fail(s"The recorded operation metrics does not contain key: $key") } if (expectedMetrics(key) != operationMetrics(key)) { fail( s"""The recorded metric for $key does not equal the expected value. | expected = ${expectedMetrics(key)} , | But actual = ${operationMetrics(key)} """.stripMargin ) } } } /** * Check all expected metrics exist and executime time (if expected to exist) is the largest time * metric. */ protected def checkOperationTimeMetricsInvariant( expectedMetrics: Set[String], operationMetrics: Map[String, String]): Unit = { expectedMetrics.foreach { m => assert(operationMetrics.contains(m)) } if (expectedMetrics.contains("executionTimeMs")) { val executionTimeMs = operationMetrics("executionTimeMs").toLong val maxTimeMs = operationMetrics.filterKeys(expectedMetrics.contains(_)) .mapValues(v => v.toLong).valuesIterator.max assert(executionTimeMs == maxTimeMs) } } protected def getOperationMetrics(history: DataFrame): Map[String, String] = { history.select("operationMetrics") .take(1) .head .getMap(0) .asInstanceOf[Map[String, String]] } // Returns necessary delta property json expected for the test. If Catalog-Owned is enabled, // a few properties will be automatically populated, and this method will take care of it. protected def getProperties( extraProperty: Option[Map[String, String]] = None): Map[String, String] = { val catalogOwnedProperty = if (catalogOwnedDefaultCreationEnabledInTests) { CatalogOwnedTableUtils.QOL_TABLE_FEATURES_AND_PROPERTIES.collect { case (feature, config, value) => config.key -> value }.toMap ++ // DV is explicitly disabled here b/c the current suite is incompatible // w/ DV, and we automatically enable it as part of CatalogOwned QoL features. Map(s"${DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key}" -> "false") } else { Map.empty[String, String] } // For history command, the output omits the empty config value, so we also need to // manually omit the value here. val properties = catalogOwnedProperty.filterNot { case (_, value) => value == "{}" } val finalProperties = extraProperty.map(properties ++ _).getOrElse(properties) finalProperties.asInstanceOf[Map[String, String]] } testWithFlag("basic case - Scala history with path-based table") { val tempDir = Utils.createTempDir().toString Seq(1, 2, 3).toDF().write.format("delta").save(tempDir) Seq(4, 5, 6).toDF().write.format("delta").mode("overwrite").save(tempDir) val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tempDir) // Full History checkAnswer( deltaTable.history().select("operation", "operationParameters.mode"), Seq(Row("WRITE", "Overwrite"), Row("WRITE", "ErrorIfExists"))) // History with limit checkAnswer( deltaTable.history(1).select("operation", "operationParameters.mode"), Seq(Row("WRITE", "Overwrite"))) } test("basic case - Scala history with name-based table") { withTable("delta_test") { Seq(1, 2, 3).toDF().write.format("delta").saveAsTable("delta_test") Seq(4, 5, 6).toDF().write.format("delta").mode("overwrite").saveAsTable("delta_test") val deltaTable = io.delta.tables.DeltaTable.forName(spark, "delta_test") // Full History checkAnswer( deltaTable.history().select("operation"), Seq(Row("CREATE OR REPLACE TABLE AS SELECT"), Row("CREATE TABLE AS SELECT"))) // History with limit checkAnswer( deltaTable.history(1).select("operation"), Seq(Row("CREATE OR REPLACE TABLE AS SELECT"))) } } testWithFlag("basic case - SQL describe history with path-based table") { val tempDir = Utils.createTempDir().toString Seq(1, 2, 3).toDF().write.format("delta").save(tempDir) Seq(4, 5, 6).toDF().write.format("delta").mode("overwrite").save(tempDir) // With delta.`path` format checkAnswer( sql(s"DESCRIBE HISTORY delta.`$tempDir`").select("operation", "operationParameters.mode"), Seq(Row("WRITE", "Overwrite"), Row("WRITE", "ErrorIfExists"))) checkAnswer( sql(s"DESCRIBE HISTORY delta.`$tempDir` LIMIT 1") .select("operation", "operationParameters.mode"), Seq(Row("WRITE", "Overwrite"))) // With direct path format checkAnswer( sql(s"DESCRIBE HISTORY '$tempDir'").select("operation", "operationParameters.mode"), Seq(Row("WRITE", "Overwrite"), Row("WRITE", "ErrorIfExists"))) checkAnswer( sql(s"DESCRIBE HISTORY '$tempDir' LIMIT 1") .select("operation", "operationParameters.mode"), Seq(Row("WRITE", "Overwrite"))) } testWithFlag("basic case - SQL describe history with name-based table") { withTable("delta_test") { Seq(1, 2, 3).toDF().write.format("delta").saveAsTable("delta_test") Seq(4, 5, 6).toDF().write.format("delta").mode("overwrite").saveAsTable("delta_test") checkAnswer( sql(s"DESCRIBE HISTORY delta_test").select("operation"), Seq(Row("CREATE OR REPLACE TABLE AS SELECT"), Row("CREATE TABLE AS SELECT"))) checkAnswer( sql(s"DESCRIBE HISTORY delta_test LIMIT 1").select("operation"), Seq(Row("CREATE OR REPLACE TABLE AS SELECT"))) } } testWithFlag("describe history command passes catalogTable to getHistory") { withTable("delta_catalog_test") { Seq(1, 2, 3).toDF().write.format("delta").saveAsTable("delta_catalog_test") val table = DeltaTableV2(spark, TableIdentifier("delta_catalog_test")) assert(table.catalogTable.isDefined, "Managed table should have catalogTable defined") val deltaLog = table.deltaLog val originalHistory = deltaLog.history var catalogTableWasPassed = false // Create a wrapper that tracks if catalogTable is passed to getHistory. // Note: getHistory(limitOpt) delegates to getHistory(limitOpt, None), so we only // need to override the two-parameter version to detect whether catalogTable is passed. val trackingHistory = new DeltaHistoryManager( deltaLog, spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_HISTORY_PAR_SEARCH_THRESHOLD)) { override def getHistory( limitOpt: Option[Int], catalogTableOpt: Option[CatalogTable]): Seq[DeltaHistory] = { catalogTableWasPassed = catalogTableOpt.isDefined originalHistory.getHistory(limitOpt, catalogTableOpt) } } // Replace history field using reflection val historyField = deltaLog.getClass.getDeclaredField("history") historyField.setAccessible(true) historyField.set(deltaLog, trackingHistory) // Run the command DescribeDeltaHistoryCommand( table = table, limit = Some(10), output = toAttributes(ExpressionEncoder[DeltaHistory]().schema) ).run(spark) assert(catalogTableWasPassed, "DescribeDeltaHistoryCommand should pass table.catalogTable to getHistory") } } testWithFlag("describe history fails on views") { val tempDir = Utils.createTempDir().toString Seq(1, 2, 3).toDF().write.format("delta").save(tempDir) val viewName = "delta_view" withView(viewName) { sql(s"create view $viewName as select * from delta.`$tempDir`") val e = intercept[AnalysisException] { sql(s"DESCRIBE HISTORY $viewName").collect() } assert(e.getMessage.contains( "'DESCRIBE HISTORY' expects a table but `spark_catalog`.`default`.`delta_view` is a view.")) } } testWithTempView("describe history fails on temp views") { isSQLTempView => withTable("t1") { Seq(1, 2, 3).toDF().write.format("delta").saveAsTable("t1") val viewName = "v" createTempViewFromTable("t1", isSQLTempView) val e = intercept[AnalysisException] { sql(s"DESCRIBE HISTORY $viewName").collect() } assert(e.getMessage.contains("'DESCRIBE HISTORY' expects a table but `v` is a view.")) } } private val expectedCreateOperationParameters = Seq("partitionBy", "clusterBy", "description", "isManaged", "properties") testWithFlag("operations - create table") { withTable("delta_test") { sql( s"""create table delta_test ( | a int, | b string |) |using delta |partitioned by (b) |comment 'this is my table' |tblproperties (delta.appendOnly=true) """.stripMargin) val basePath = spark.sessionState.catalog.getTableMetadata(TableIdentifier("delta_test")).location.getPath val appendOnlyTableProperty = Map("delta.appendOnly" -> "true") checkLastOperation( basePath, expectedOperationParameters = expectedCreateOperationParameters, expectedColVals = Seq("CREATE TABLE", "true", """["b"]""", """[]""", "this is my table"), columns = Seq( $"operation", $"operationParameters.isManaged", $"operationParameters.partitionBy", $"operationParameters.clusterBy", $"operationParameters.description")) checkLastOperationProperties(basePath, getProperties(Some(appendOnlyTableProperty))) } } testWithFlag("operations - ctas (saveAsTable)") { val tempDir = Utils.createTempDir().toString withTable("delta_test") { Seq((1, "a"), (2, "3")).toDF("id", "data").write.format("delta") .option("path", tempDir).saveAsTable("delta_test") checkLastOperation( tempDir, expectedOperationParameters = expectedCreateOperationParameters, expectedColVals = Seq("CREATE TABLE AS SELECT", "false", """[]""", """[]""", null), columns = Seq($"operation", $"operationParameters.isManaged", $"operationParameters.partitionBy", $"operationParameters.clusterBy", $"operationParameters.description")) checkLastOperationProperties(tempDir, getProperties()) } } testWithFlag("operations - ctas (sql)") { val tempDir = Utils.createTempDir().toString withTable("delta_test") { sql( s"""create table delta_test |using delta |location '$tempDir' |tblproperties (delta.appendOnly=true) |partitioned by (b) |as select 1 as a, 'x' as b """.stripMargin) val appendOnlyProperty = Map[String, String]("delta.appendOnly" -> "true") checkLastOperation( tempDir, expectedOperationParameters = expectedCreateOperationParameters, expectedColVals = Seq("CREATE TABLE AS SELECT", "false", """["b"]""", """[]""", null), columns = Seq($"operation", $"operationParameters.isManaged", $"operationParameters.partitionBy", $"operationParameters.clusterBy", $"operationParameters.description")) checkLastOperationProperties(tempDir, getProperties(Some(appendOnlyProperty))) } val tempDir2 = Utils.createTempDir().toString withTable("delta_test") { sql( s"""create table delta_test |using delta |location '$tempDir2' |comment 'this is my table' |as select 1 as a, 'x' as b """.stripMargin) // TODO(burak): Fix comments for CTAS checkLastOperation( tempDir2, expectedOperationParameters = expectedCreateOperationParameters, expectedColVals = Seq("CREATE TABLE AS SELECT", "false", """[]""", """[]""", "this is my table"), columns = Seq($"operation", $"operationParameters.isManaged", $"operationParameters.partitionBy", $"operationParameters.clusterBy", $"operationParameters.description")) checkLastOperationProperties(tempDir2, getProperties()) } } testWithFlag("operations - [un]set tbproperties") { withTable("delta_test") { sql("CREATE TABLE delta_test (v1 int, v2 string) USING delta") sql(""" |ALTER TABLE delta_test |SET TBLPROPERTIES ( | 'delta.checkpointInterval' = '20', | 'key' = 'value' |)""".stripMargin) checkLastOperation( spark.sessionState.catalog.getTableMetadata(TableIdentifier("delta_test")).location.getPath, expectedOperationParameters = Seq("properties"), expectedColVals = Seq("SET TBLPROPERTIES", """{"delta.checkpointInterval":"20","key":"value"}"""), columns = Seq($"operation", $"operationParameters.properties")) sql("ALTER TABLE delta_test UNSET TBLPROPERTIES ('key')") checkLastOperation( spark.sessionState.catalog.getTableMetadata(TableIdentifier("delta_test")).location.getPath, expectedOperationParameters = Seq("properties", "ifExists"), expectedColVals = Seq("UNSET TBLPROPERTIES", """["key"]""", "true"), columns = Seq($"operation", $"operationParameters.properties", $"operationParameters.ifExists")) } } testWithFlag("operations - add columns") { withTable("delta_test") { sql("CREATE TABLE delta_test (v1 int, v2 string) USING delta") sql("ALTER TABLE delta_test ADD COLUMNS (v3 long, v4 int AFTER v1)") val column3 = """{"name":"v3","type":"long","nullable":true,"metadata":{}}""" val column4 = """{"name":"v4","type":"integer","nullable":true,"metadata":{}}""" checkLastOperation( spark.sessionState.catalog.getTableMetadata(TableIdentifier("delta_test")).location.getPath, expectedOperationParameters = Seq("columns"), expectedColVals = Seq("ADD COLUMNS", s"""[{"column":$column3},{"column":$column4,"position":"AFTER v1"}]"""), columns = Seq($"operation", $"operationParameters.columns")) } } testWithFlag("operations - change column") { withTable("delta_test") { sql("CREATE TABLE delta_test (v1 int, v2 string) USING delta") sql("ALTER TABLE delta_test CHANGE COLUMN v1 v1 integer AFTER v2") checkLastOperation( spark.sessionState.catalog.getTableMetadata(TableIdentifier("delta_test")).location.getPath, expectedOperationParameters = Seq("column", "position"), expectedColVals = Seq("CHANGE COLUMN", s"""{"name":"v1","type":"integer","nullable":true,"metadata":{}}""", "AFTER v2"), columns = Seq($"operation", $"operationParameters.column", $"operationParameters.position")) } } test("operations - upgrade protocol") { val readerVersion = Action.supportedProtocolVersion().minReaderVersion val writerVersion = Action.supportedProtocolVersion().minWriterVersion withTempDir { path => val log = DeltaLog.forTable(spark, path) log.createLogDirectoriesIfNotExists() log.store.write( FileNames.unsafeDeltaFile(log.logPath, 0), Iterator( Metadata(schemaString = spark.range(1).schema.asNullable.json).json, Protocol(1, 1).json), overwrite = false, log.newDeltaHadoopConf()) log.update() log.upgradeProtocol( Action.supportedProtocolVersion(withAllFeatures = false) .withFeature(TestLegacyReaderWriterFeature)) // scalastyle:off line.size.limit checkLastOperation( path.toString, expectedOperationParameters = Seq("newProtocol"), expectedColVals = Seq("UPGRADE PROTOCOL", s"""{"minReaderVersion":$readerVersion,""" + s""""minWriterVersion":$writerVersion,""" + s""""readerFeatures":["${TestLegacyReaderWriterFeature.name}"],""" + s""""writerFeatures":["${TestLegacyReaderWriterFeature.name}"]}"""), columns = Seq($"operation", $"operationParameters.newProtocol")) // scalastyle:on line.size.limit } } val expectedInsertOperationParameters = Seq("mode", "partitionBy") testWithFlag("operations - insert append with partition columns") { val tempDir = Utils.createTempDir().toString Seq((1, "a"), (2, "3")).toDF("id", "data") .write .format("delta") .mode("append") .partitionBy("id") .save(tempDir) checkLastOperation( tempDir, expectedOperationParameters = expectedInsertOperationParameters, expectedColVals = Seq("WRITE", "Append", """["id"]"""), columns = Seq($"operation", $"operationParameters.mode", $"operationParameters.partitionBy")) } testWithFlag("operations - insert append without partition columns") { val tempDir = Utils.createTempDir().toString Seq((1, "a"), (2, "3")).toDF("id", "data").write.format("delta").save(tempDir) checkLastOperation( tempDir, expectedOperationParameters = expectedInsertOperationParameters, expectedColVals = Seq("WRITE", "ErrorIfExists", """[]"""), columns = Seq($"operation", $"operationParameters.mode", $"operationParameters.partitionBy")) } testWithFlag("operations - insert error if exists with partitions") { val tempDir = Utils.createTempDir().toString Seq((1, "a"), (2, "3")).toDF("id", "data") .write .format("delta") .partitionBy("id") .mode("errorIfExists") .save(tempDir) checkLastOperation( tempDir, expectedOperationParameters = expectedInsertOperationParameters, expectedColVals = Seq("WRITE", "ErrorIfExists", """["id"]"""), columns = Seq($"operation", $"operationParameters.mode", $"operationParameters.partitionBy")) } testWithFlag("operations - insert error if exists without partitions") { val tempDir = Utils.createTempDir().toString Seq((1, "a"), (2, "3")).toDF("id", "data") .write .format("delta") .mode("errorIfExists") .save(tempDir) checkLastOperation( tempDir, expectedOperationParameters = expectedInsertOperationParameters, expectedColVals = Seq("WRITE", "ErrorIfExists", """[]"""), columns = Seq($"operation", $"operationParameters.mode", $"operationParameters.partitionBy")) } test("operations - streaming append with transaction ids") { val tempDir = Utils.createTempDir().toString val checkpoint = Utils.createTempDir().toString val data = MemoryStream[Int] data.addData(1, 2, 3) val stream = data.toDF() .writeStream .format("delta") .option("checkpointLocation", checkpoint) .start(tempDir) stream.processAllAvailable() stream.stop() checkLastOperation( tempDir, expectedOperationParameters = Seq("outputMode", "queryId", "epochId"), expectedColVals = Seq("STREAMING UPDATE", "Append", "0"), columns = Seq($"operation", $"operationParameters.outputMode", $"operationParameters.epochId")) } testWithFlag("operations - insert overwrite with predicate") { val tempDir = Utils.createTempDir().toString Seq((1, "a"), (2, "3")).toDF("id", "data").write.format("delta").partitionBy("id").save(tempDir) Seq((1, "b")).toDF("id", "data").write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "id = 1") .save(tempDir) checkLastOperation( tempDir, expectedOperationParameters = expectedInsertOperationParameters ++ Seq("predicate"), expectedColVals = Seq("WRITE", "Overwrite", """id = 1"""), columns = Seq($"operation", $"operationParameters.mode", $"operationParameters.predicate")) } testWithFlag("operations - delete with predicate") { val tempDir = Utils.createTempDir().toString Seq((1, "a"), (2, "3")).toDF("id", "data").write.format("delta").partitionBy("id").save(tempDir) val deltaLog = DeltaLog.forTable(spark, tempDir) val deltaTable = io.delta.tables.DeltaTable.forPath(spark, deltaLog.dataPath.toString) deltaTable.delete("id = 1") checkLastOperation( tempDir, expectedOperationParameters = Seq("predicate"), expectedColVals = Seq("DELETE", """["(id = 1)"]"""), columns = Seq($"operation", $"operationParameters.predicate"), removeExpressionId = true) } testWithFlag("old and new writers") { val tempDir = Utils.createTempDir().toString Seq(1, 2, 3).toDF().write.format("delta").save(tempDir.toString) checkLastOperation(tempDir, expectedOperationParameters = expectedInsertOperationParameters, expectedColVals = Seq("WRITE", "ErrorIfExists"), columns = Seq($"operation", $"operationParameters.mode")) Seq(1, 2, 3).toDF().write.format("delta").mode("append").save(tempDir.toString) assert(spark.sql(s"DESCRIBE HISTORY delta.`$tempDir`").count() === 2) checkLastOperation(tempDir, expectedOperationParameters = expectedInsertOperationParameters, expectedColVals = Seq("WRITE", "Append"), columns = Seq($"operation", $"operationParameters.mode")) } testWithFlag("order history by version") { val tempDir = Utils.createTempDir().toString Seq(0).toDF().write.format("delta").save(tempDir) Seq(1).toDF().write.format("delta").mode("overwrite").save(tempDir) Seq(2).toDF().write.format("delta").mode("append").save(tempDir) Seq(3).toDF().write.format("delta").mode("overwrite").save(tempDir) Seq(4).toDF().write.format("delta").mode("overwrite").save(tempDir) val ans = io.delta.tables.DeltaTable.forPath(spark, tempDir) .history().as[DeltaHistory].collect() assert(ans.map(_.version) === Seq(Some(4), Some(3), Some(2), Some(1), Some(0))) val ans2 = sql(s"DESCRIBE HISTORY delta.`$tempDir`").as[DeltaHistory].collect() assert(ans2.map(_.version) === Seq(Some(4), Some(3), Some(2), Some(1), Some(0))) } test("read version") { val tempDir = Utils.createTempDir().toString Seq(0).toDF().write.format("delta").save(tempDir) // readVersion = None as first commit Seq(1).toDF().write.format("delta").mode("overwrite").save(tempDir) // readVersion = Some(0) val log = DeltaLog.forTable(spark, tempDir) val txn = log.startTransaction() // should read snapshot version 1 Seq(2).toDF().write.format("delta").mode("append").save(tempDir) // readVersion = Some(1) Seq(3).toDF().write.format("delta").mode("append").save(tempDir) // readVersion = Some(2) txn.commit(Seq.empty, DeltaOperations.Truncate()) // readVersion = Some(1) Seq(5).toDF().write.format("delta").mode("append").save(tempDir) // readVersion = Some(4) val ans = sql(s"DESCRIBE HISTORY delta.`$tempDir`").as[DeltaHistory].collect() assert(ans.map(x => x.version.get -> x.readVersion) === Seq(5 -> Some(4), 4 -> Some(1), 3 -> Some(2), 2 -> Some(1), 1 -> Some(0), 0 -> None)) } testWithFlag("evolvability test") { checkLastOperation( evolvabilityResource, expectedOperationParameters = Seq("outputMode", "queryId", "epochId"), expectedColVals = evolvabilityLastOp, columns = Seq($"operation", $"operationParameters.mode", $"operationParameters.partitionBy")) } test("using on non delta") { withTempDir { basePath => val e = intercept[AnalysisException] { sql(s"describe history '$basePath'").collect() } assert(Seq("supported", "Delta").forall(e.getMessage.contains)) } } test("describe history a non-existent path and a non Delta table") { def assertNotADeltaTableException(path: String): Unit = { for (table <- Seq(s"'$path'", s"delta.`$path`")) { val e = intercept[AnalysisException] { sql(s"describe history $table").show() } Seq("is not a Delta table").foreach { msg => assert(e.getMessage.contains(msg)) } } } withTempPath { tempDir => assert(!tempDir.exists()) assertNotADeltaTableException(tempDir.getCanonicalPath) } withTempPath { tempDir => spark.range(1, 10).write.parquet(tempDir.getCanonicalPath) assertNotADeltaTableException(tempDir.getCanonicalPath) } } test("operation metrics - write metrics") { withSQLConf(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true") { withTempDir { tempDir => // create table spark.range(100).repartition(5).write.format("delta").save(tempDir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(tempDir.getAbsolutePath) // get last command history val operationMetrics = getOperationMetrics(deltaTable.history(1)) val expectedMetrics = Map( "numFiles" -> "5", "numOutputRows" -> "100" ) // Check if operation metrics from history are accurate checkOperationMetrics(expectedMetrics, operationMetrics, DeltaOperationMetrics.WRITE) assert(operationMetrics("numOutputBytes").toLong > 0) } } } test("operation metrics - merge") { withSQLConf(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true") { withTempDir { tempDir => // create target spark.range(100).write.format("delta").save(tempDir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(tempDir.getAbsolutePath) // run merge deltaTable.as("t") .merge(spark.range(50, 150).toDF().as("s"), "s.id = t.id") .whenMatched() .updateAll() .whenNotMatched() .insertAll() .execute() // Get operation metrics val operationMetrics: Map[String, String] = getOperationMetrics(deltaTable.history(1)) val expectedMetrics = Map( "numTargetRowsInserted" -> "50", "numTargetRowsUpdated" -> "50", "numTargetRowsDeleted" -> "0", "numOutputRows" -> "100", "numSourceRows" -> "100" ) val copiedRows = operationMetrics("numTargetRowsCopied").toInt assert(0 <= copiedRows && copiedRows <= 50) checkOperationMetrics( expectedMetrics, operationMetrics, mergeMetricsSchema) val expectedTimeMetrics = Set("executionTimeMs", "scanTimeMs", "rewriteTimeMs") checkOperationTimeMetricsInvariant(expectedTimeMetrics, operationMetrics) } } } test("operation metrics - streaming update") { withSQLConf(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true") { withTempDir { tempDir => val memoryStream = MemoryStream[Long] val df = memoryStream.toDF() val tbl = tempDir.getAbsolutePath + "tbl1" spark.range(10).write.format("delta").save(tbl) // ensure that you are writing out a single file per batch val q = df.coalesce(1) .withColumnRenamed("value", "id") .writeStream .format("delta") .option("checkpointLocation", tempDir + "checkpoint") .start(tbl) memoryStream.addData(1) q.processAllAvailable() val deltaTable = io.delta.tables.DeltaTable.forPath(tbl) var operationMetrics: Map[String, String] = getOperationMetrics(deltaTable.history(1)) val expectedMetrics = Map( "numAddedFiles" -> "1", "numRemovedFiles" -> "0", "numOutputRows" -> "1" ) checkOperationMetrics( expectedMetrics, operationMetrics, DeltaOperationMetrics.STREAMING_UPDATE) // check if second batch also returns correct metrics. memoryStream.addData(1, 2, 3) q.processAllAvailable() operationMetrics = getOperationMetrics(deltaTable.history(1)) val expectedMetrics2 = Map( "numAddedFiles" -> "1", "numRemovedFiles" -> "0", "numOutputRows" -> "3" ) checkOperationMetrics( expectedMetrics2, operationMetrics, DeltaOperationMetrics.STREAMING_UPDATE) assert(operationMetrics("numOutputBytes").toLong > 0) q.stop() } } } test("operation metrics - streaming update - complete mode") { withSQLConf(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true") { withTempDir { tempDir => val memoryStream = MemoryStream[Long] val df = memoryStream.toDF() val tbl = tempDir.getAbsolutePath + "tbl1" Seq(1L -> 1L, 2L -> 2L).toDF("value", "count") .coalesce(1) .write .format("delta") .save(tbl) // ensure that you are writing out a single file per batch val q = df.groupBy("value").count().coalesce(1) .writeStream .format("delta") .outputMode("complete") .option("checkpointLocation", tempDir + "checkpoint") .start(tbl) memoryStream.addData(1) q.processAllAvailable() val deltaTable = io.delta.tables.DeltaTable.forPath(tbl) val operationMetrics = getOperationMetrics(deltaTable.history(1)) val expectedMetrics = Map( "numAddedFiles" -> "1", "numRemovedFiles" -> "1", "numOutputRows" -> "1" ) checkOperationMetrics( expectedMetrics, operationMetrics, DeltaOperationMetrics.STREAMING_UPDATE) } } } def getLastCommitNumAddedAndRemovedBytes(deltaLog: DeltaLog): (Long, Long) = { val changes = deltaLog.getChanges(deltaLog.update().version).flatMap(_._2).toSeq val addedBytes = changes.collect { case a: AddFile => a.size }.sum val removedBytes = changes.collect { case r: RemoveFile => r.getFileSize }.sum (addedBytes, removedBytes) } def metricsUpdateTest : Unit = withTempDir { tempDir => // Create the initial table as a single file Seq(1, 2, 5, 11, 21, 3, 4, 6, 9, 7, 8, 0).toDF("key") .withColumn("value", 'key % 2) .write .format("delta") .save(tempDir.getAbsolutePath) // append additional data with the same number range to the table. // This data is saved as a separate file as well Seq(15, 16, 17).toDF("key") .withColumn("value", 'key % 2) .repartition(1) .write .format("delta") .mode("append") .save(tempDir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tempDir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, tempDir.getAbsolutePath) deltaLog.snapshot.numOfFiles // update the table deltaTable.update(col("key") === lit("16"), Map("value" -> lit("1"))) // The file from the append gets updated but the file from the initial table gets scanned // as well. We want to make sure numCopied rows is calculated from written files and not // scanned files[SC-33980] // get operation metrics val operationMetrics = getOperationMetrics(deltaTable.history(1)) val (addedBytes, removedBytes) = getLastCommitNumAddedAndRemovedBytes(deltaLog) val expectedMetrics = Map( "numAddedFiles" -> "1", "numRemovedFiles" -> "1", "numUpdatedRows" -> "1", "numCopiedRows" -> "2", // There should be only three rows in total(updated + copied) "numAddedBytes" -> addedBytes.toString, "numRemovedBytes" -> removedBytes.toString ) checkOperationMetrics( expectedMetrics, operationMetrics, updateMetricsSchema) val expectedTimeMetrics = Set("executionTimeMs", "scanTimeMs", "rewriteTimeMs") checkOperationTimeMetricsInvariant(expectedTimeMetrics, operationMetrics) } test("operation metrics - update") { withSQLConf((DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true")) { metricsUpdateTest } } def metricsUpdatePartitionedColumnTest : Unit = { val numRows = 100 val numPartitions = 5 withTempDir { tempDir => spark.range(numRows) .withColumn("c1", 'id + 1) .withColumn("c2", 'id % numPartitions) .write .partitionBy("c2") .format("delta") .save(tempDir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(tempDir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, tempDir.getAbsolutePath) val numFilesBeforeUpdate = deltaLog.snapshot.numOfFiles deltaTable.update(col("c2") < 1, Map("c2" -> lit("1"))) val numFilesAfterUpdate = deltaLog.snapshot.numOfFiles val operationMetrics = getOperationMetrics(deltaTable.history(1)) val newFiles = numFilesAfterUpdate - numFilesBeforeUpdate val oldFiles = numFilesBeforeUpdate / numPartitions val addedFiles = newFiles + oldFiles val (addedBytes, removedBytes) = getLastCommitNumAddedAndRemovedBytes(deltaLog) val expectedMetrics = Map( "numUpdatedRows" -> (numRows / numPartitions).toString, "numCopiedRows" -> "0", "numAddedFiles" -> addedFiles.toString, "numRemovedFiles" -> (numFilesBeforeUpdate / numPartitions).toString, "numAddedBytes" -> addedBytes.toString, "numRemovedBytes" -> removedBytes.toString ) checkOperationMetrics( expectedMetrics, operationMetrics, updateMetricsSchema) } } test("operation metrics - update - partitioned column") { withSQLConf((DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true")) { metricsUpdatePartitionedColumnTest } } test("operation metrics - delete") { withSQLConf(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true") { withTempDir { tempDir => // Create the initial table as a single file Seq(1, 2, 5, 11, 21, 3, 4, 6, 9, 7, 8, 0).toDF("key") .withColumn("value", 'key % 2) .repartition(1) .write .format("delta") .save(tempDir.getAbsolutePath) // Append to the initial table additional data in the same numerical range Seq(15, 16, 17).toDF("key") .withColumn("value", 'key % 2) .repartition(1) .write .format("delta") .mode("append") .save(tempDir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tempDir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, tempDir.getAbsolutePath) deltaLog.snapshot.numOfFiles // delete the table deltaTable.delete(col("key") === lit("16")) // The file from the append gets deleted but the file from the initial table gets scanned // as well. We want to make sure numCopied rows is calculated from the written files instead // of the scanned files.[SC-33980] // get operation metrics val operationMetrics = getOperationMetrics(deltaTable.history(1)) // get expected byte level metrics val (numAddedBytesExpected, numRemovedBytesExpected) = getLastCommitNumAddedAndRemovedBytes(deltaLog) val expectedMetrics = Map( "numAddedFiles" -> "1", "numAddedBytes" -> numAddedBytesExpected.toString, "numRemovedFiles" -> "1", "numRemovedBytes" -> numRemovedBytesExpected.toString, "numDeletedRows" -> "1", "numCopiedRows" -> "2" // There should be only three rows in total(deleted + copied) ) checkOperationMetrics( expectedMetrics, operationMetrics, deleteMetricsSchema(partitioned = false)) val expectedTimeMetrics = Set("executionTimeMs", "scanTimeMs", "rewriteTimeMs") checkOperationTimeMetricsInvariant(expectedTimeMetrics, operationMetrics) } } } test("operation metrics - delete - partition column") { withSQLConf(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true") { val numRows = 100 val numPartitions = 5 withTempDir { tempDir => spark.range(numRows) .withColumn("c1", 'id % numPartitions) .write .format("delta") .partitionBy("c1") .save(tempDir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, tempDir.getAbsolutePath) val numFilesBeforeDelete = deltaLog.snapshot.numOfFiles val deltaTable = io.delta.tables.DeltaTable.forPath(tempDir.getAbsolutePath) deltaTable.delete("c1 = 1") val operationMetrics = getOperationMetrics(deltaTable.history(1)) // get expected byte level metrics val (numAddedBytesExpected, numRemovedBytesExpected) = getLastCommitNumAddedAndRemovedBytes(deltaLog) val expectedMetrics = Map[String, String]( "numRemovedFiles" -> (numFilesBeforeDelete / numPartitions).toString, "numAddedBytes" -> numAddedBytesExpected.toString, "numRemovedBytes" -> numRemovedBytesExpected.toString ) // row level metrics are not collected for deletes with parition columns checkOperationMetrics( expectedMetrics, operationMetrics, deleteMetricsSchema(partitioned = true)) val expectedTimeMetrics = Set("executionTimeMs", "scanTimeMs", "rewriteTimeMs") checkOperationTimeMetricsInvariant(expectedTimeMetrics, operationMetrics) } } } test("operation metrics - delete - full") { withSQLConf(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true") { val numRows = 100 val numPartitions = 5 withTempDir { tempDir => spark.range(numRows) .withColumn("c1", 'id % numPartitions) .write .format("delta") .partitionBy("c1") .save(tempDir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, tempDir.getAbsolutePath) val numFilesBeforeDelete = deltaLog.snapshot.numOfFiles val deltaTable = io.delta.tables.DeltaTable.forPath(tempDir.getAbsolutePath) deltaTable.delete() // get expected byte level metrics val (numAddedBytesExpected, numRemovedBytesExpected) = getLastCommitNumAddedAndRemovedBytes(deltaLog) val operationMetrics = getOperationMetrics(deltaTable.history(1)) val expectedMetrics = Map[String, String]( "numRemovedFiles" -> numFilesBeforeDelete.toString, "numAddedBytes" -> numAddedBytesExpected.toString, "numRemovedBytes" -> numRemovedBytesExpected.toString ) checkOperationMetrics( expectedMetrics, operationMetrics, deleteMetricsSchema(partitioned = true)) val expectedTimeMetrics = Set("executionTimeMs", "scanTimeMs", "rewriteTimeMs") checkOperationTimeMetricsInvariant(expectedTimeMetrics, operationMetrics) } } } test("operation metrics - convert to delta") { withSQLConf(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true") { val numPartitions = 5 withTempDir { tempDir => // Create a parquet table val dir = tempDir.getAbsolutePath() spark.range(10) .withColumn("col2", 'id % numPartitions) .write .format("parquet") .mode("overwrite") .partitionBy("col2") .save(dir) // convert to delta val deltaTable = io.delta.tables.DeltaTable.convertToDelta(spark, s"parquet.`$dir`", "col2 long") val deltaLog = DeltaLog.forTable(spark, dir) val expectedMetrics = Map( "numConvertedFiles" -> deltaLog.snapshot.numOfFiles.toString ) val operationMetrics = getOperationMetrics(deltaTable.history(1)) checkOperationMetrics(expectedMetrics, operationMetrics, DeltaOperationMetrics.CONVERT) } } } test("sort and collect the DESCRIBE HISTORY result") { withTempDir { tempDir => val path = tempDir.getCanonicalPath Seq(1, 2, 3).toDF().write.format("delta").save(path) val rows = sql(s"DESCRIBE HISTORY delta.`$path`") .orderBy("version") .collect() assert(rows.map(_.getAs[Long]("version")).toList == 0L :: Nil) withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false") { val rows = sql(s"DESCRIBE HISTORY delta.`$path`") .filter("version >= 0") .orderBy("version") .collect() assert(rows.map(_.getAs[Long]("version")).toList == 0L :: Nil) } } } test("operation metrics - create table") { withSQLConf(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true") { val tblName = "tblName" val numRows = 10 withTable(tblName) { sql(s"CREATE TABLE $tblName USING DELTA SELECT * from range($numRows)") val deltaTable = io.delta.tables.DeltaTable.forName(tblName) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tblName)) val numFiles = deltaLog.snapshot.numOfFiles val expectedMetrics = Map( "numFiles" -> numFiles.toString, "numOutputRows" -> numRows.toString ) val operationMetrics = getOperationMetrics(deltaTable.history(1)) assert(operationMetrics("numOutputBytes").toLong > 0) checkOperationMetrics(expectedMetrics, operationMetrics, DeltaOperationMetrics.WRITE) } } } test("operation metrics - create table - without data") { withSQLConf(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true") { val tblName = s"tbl_${System.currentTimeMillis()}" // unique name withTable(tblName) { sql(s"CREATE TABLE $tblName(id bigint) USING DELTA") val deltaTable = io.delta.tables.DeltaTable.forName(tblName) val operationMetrics = getOperationMetrics(deltaTable.history(1)) assert(operationMetrics === Map.empty) } } } def testReplaceWhere(testName: String)(f: (Boolean, Boolean) => Unit): Unit = { Seq(true, false).foreach { enableCDF => Seq(true, false).foreach { enableStats => test(testName + s"enableCDF=${enableCDF} - enableStats ${enableStats}") { withSQLConf( DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> enableCDF.toString, DeltaSQLConf.DELTA_COLLECT_STATS.key ->enableStats.toString, DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true") { if (!enableStats) { // Row IDs assignment needs row count statistics. So we need to disable RowTracking // here for CCv1.5's QoL features if we are not enabling [[DELTA_COLLECT_STATS]]. spark.conf.set(DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey, "false") } f(enableCDF, enableStats) if (!enableStats && spark.sessionState.conf.contains( DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey)) { spark.conf.unset(DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey) } } } } } } testReplaceWhere("replaceWhere on data column") { (enableCDF, enableStats) => withTable("tbl") { // create a table with one row spark.range(10) .repartition(1) // 1 file table .withColumn("b", lit(1)) .write .format("delta") .saveAsTable("tbl") val deltaTable = io.delta.tables.DeltaTable.forName("tbl") val deltaLog = DeltaLog.forTable(spark, TableIdentifier("tbl")) // replace where spark.range(20) .withColumn("b", lit(1)) .repartition(1) // write 1 file .write .format("delta") .option("replaceWhere", "b = 1") .mode("overwrite") .saveAsTable("tbl") val numWrittenFiles = deltaLog.getChanges(1).flatMap { case (a, v) => v }.filter(_.isInstanceOf[AddFile]) .toSeq .size val numAddedChangeFiles = if (enableCDF) { deltaLog.getChanges(1).flatMap { case (a, v) => v }.filter(_.isInstanceOf[AddCDCFile]) .toSeq .size } else { 0 } // get expected byte level metrics val (numAddedBytesExpected, numRemovedBytesExpected) = getLastCommitNumAddedAndRemovedBytes(deltaLog) if (enableStats) { checkOperationMetrics( Map( "numFiles" -> (numWrittenFiles).toString, "numOutputRows" -> "20", "numCopiedRows" -> "0", "numOutputBytes" -> numAddedBytesExpected.toString, "numRemovedBytes" -> numRemovedBytesExpected.toString, "numAddedChangeFiles" -> numAddedChangeFiles.toString, "numDeletedRows" -> "10", "numRemovedFiles" -> "1" ), getOperationMetrics(deltaTable.history(1)), replaceWhereMetricsSchema ) } else { checkOperationMetrics( Map( "numFiles" -> (numWrittenFiles).toString, "numOutputBytes" -> numAddedBytesExpected.toString, "numRemovedBytes" -> numRemovedBytesExpected.toString, "numAddedChangeFiles" -> numAddedChangeFiles.toString, "numRemovedFiles" -> "1" ), getOperationMetrics(deltaTable.history(1)), replaceWhereMetricsSchema.filter(!_.contains("Rows")) ) } } } testReplaceWhere(s"replaceWhere on data column - partial rewrite") { (enableCDF, enableStats) => // Whats different from the above test // replace where has a append + delete. // make the delete also write new files withTable("tbl") { // create a table with one row spark.range(10) .repartition(1) // 1 file table .withColumn("b", 'id % 2) // 1 file contains 2 values .write .format("delta") .saveAsTable("tbl") val deltaTable = io.delta.tables.DeltaTable.forName("tbl") // replace where spark.range(20) .withColumn("b", lit(1L)) .repartition(3) // write 3 files .write .format("delta") .option("replaceWhere", "b = 1") // partial match .mode("overwrite") .saveAsTable("tbl") val deltaLog = DeltaLog.forTable(spark, TableIdentifier("tbl")) val numAddedChangeFiles = if (enableCDF) { deltaLog.getChanges(1).flatMap { case (a, v) => v }.filter(_.isInstanceOf[AddCDCFile]) .toSeq .size } else { 0 } // get expected byte level metrics val (numAddedBytesExpected, numRemovedBytesExpected) = getLastCommitNumAddedAndRemovedBytes(deltaLog) if (enableStats) { checkOperationMetrics( Map( "numFiles" -> "4", // 3(append) + 1(delete) "numOutputRows" -> "25", // 20 + 5 "numCopiedRows" -> "5", "numAddedChangeFiles" -> numAddedChangeFiles.toString, "numDeletedRows" -> "5", "numOutputBytes" -> numAddedBytesExpected.toString, "numRemovedBytes" -> numRemovedBytesExpected.toString, "numRemovedFiles" -> "1" ), getOperationMetrics(deltaTable.history(1)), replaceWhereMetricsSchema ) } else { checkOperationMetrics( Map( "numFiles" -> "4", // 3(append) + 1(delete) "numAddedChangeFiles" -> numAddedChangeFiles.toString, "numOutputBytes" -> numAddedBytesExpected.toString, "numRemovedBytes" -> numRemovedBytesExpected.toString, "numRemovedFiles" -> "1" ), getOperationMetrics(deltaTable.history(1)), replaceWhereMetricsSchema.filter(!_.contains("Rows")) ) } } } Seq("true", "false").foreach { enableArbitraryRW => testReplaceWhere(s"replaceWhere on partition column " + s"- arbitraryReplaceWhere=${enableArbitraryRW}") { (enableCDF, enableStats) => withSQLConf( DeltaSQLConf.REPLACEWHERE_DATACOLUMNS_ENABLED.key -> enableArbitraryRW, DeltaSQLConf.OVERWRITE_REMOVE_METRICS_ENABLED.key -> "true") { withTable("tbl") { // create a table with one row spark.range(10) .repartition(1) // 1 file table .withColumn("b", lit(1)) .write .format("delta") .partitionBy("b") .saveAsTable("tbl") val deltaTable = io.delta.tables.DeltaTable.forName("tbl") // replace where spark.range(20) .repartition(2) // write 2 files .withColumn("b", lit(1)) .write .format("delta") .option("replaceWhere", "b = 1") // partial match .mode("overwrite") .saveAsTable("tbl") val deltaLog = DeltaLog.forTable(spark, TableIdentifier("tbl")) // get expected byte level metrics val (numAddedBytesExpected, numRemovedBytesExpected) = getLastCommitNumAddedAndRemovedBytes(deltaLog) // metrics are a subset here as it would involve a partition delete if (enableArbitraryRW.toBoolean) { if (enableStats) { checkOperationMetrics( Map( "numFiles" -> "2", "numOutputRows" -> "20", "numAddedChangeFiles" -> "0", "numRemovedFiles" -> "1", "numCopiedRows" -> "0", "numOutputBytes" -> numAddedBytesExpected.toString, "numRemovedBytes" -> numRemovedBytesExpected.toString, "numDeletedRows" -> "10" ), getOperationMetrics(deltaTable.history(1)), replaceWhereMetricsSchema ) } else { checkOperationMetrics( Map( "numFiles" -> "2", "numAddedChangeFiles" -> "0", "numOutputBytes" -> numAddedBytesExpected.toString, "numRemovedBytes" -> numRemovedBytesExpected.toString, "numRemovedFiles" -> "1" ), getOperationMetrics(deltaTable.history(1)), replaceWhereMetricsSchema.filter(!_.contains("Rows")) ) } } else { // legacy replace where mentioned output rows regardless of stats or not. checkOperationMetrics( Map( "numFiles" -> "2", "numOutputRows" -> "20", "numOutputBytes" -> numAddedBytesExpected.toString ), getOperationMetrics(deltaTable.history(1)), DeltaOperationMetrics.WRITE ++ DeltaOperationMetrics.OVERWRITE_REMOVES ) } } } } } test("replaceWhere metrics turned off - reverts to old behavior") { withSQLConf(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true", DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false", // We need to turn RowTracking off b/c it needs the row count // statistics w/ [[DELTA_COLLECT_STATS]] enabled. // We automatically enable [[RowTracking]] as part // of CCv1.5's QoL features enablement. DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey -> "false", DeltaSQLConf.REPLACEWHERE_METRICS_ENABLED.key -> "false", DeltaSQLConf.OVERWRITE_REMOVE_METRICS_ENABLED.key -> "false") { withTable("tbl") { // create a table with one row spark.range(10) .repartition(1) // 1 file table .withColumn("b", lit(1)) .write .format("delta") .partitionBy("b") .saveAsTable("tbl") val deltaTable = io.delta.tables.DeltaTable.forName("tbl") // replace where spark.range(20) .repartition(2) // write 2 files .withColumn("b", lit(1)) .write .format("delta") .option("replaceWhere", "b = 1") // partial match .mode("overwrite") .saveAsTable("tbl") checkOperationMetrics( Map( "numFiles" -> "2", "numOutputRows" -> "20" ), getOperationMetrics(deltaTable.history(1)), DeltaOperationMetrics.WRITE ) } } } test("enable remove metrics in insert with overwrite") { withSQLConf(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true", DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false", // We need to turn RowTracking off b/c it needs the row count // statistics w/ [[DELTA_COLLECT_STATS]] enabled. // We automatically enable [[RowTracking]] as part // of CCv1.5's QoL features enablement. DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey -> "false", DeltaSQLConf.REPLACEWHERE_METRICS_ENABLED.key -> "false", DeltaSQLConf.OVERWRITE_REMOVE_METRICS_ENABLED.key -> "true") { withTable("tbl") { spark.range(10).repartition(4).write.format("delta").saveAsTable("tbl") spark.range(20).repartition(2).write.format("delta").mode("overwrite").saveAsTable("tbl") val deltaTable = io.delta.tables.DeltaTable.forName("tbl") val operationMetrics = getOperationMetrics(deltaTable.history(1)) checkOperationMetrics( Map( "numFiles" -> "2", "numOutputRows" -> "20", "numRemovedFiles" -> "4" ), operationMetrics, DeltaOperationMetrics.WRITE ++ DeltaOperationMetrics.OVERWRITE_REMOVES ) assert(operationMetrics("numRemovedBytes").toLong > 0) } } } test("operation metrics - create table - v2") { withSQLConf( DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true", DeltaSQLConf.OVERWRITE_REMOVE_METRICS_ENABLED.key -> "true") { val tblName = "tblName" withTable(tblName) { // Create spark.range(100).writeTo(tblName).using("delta").create() val deltaTable = io.delta.tables.DeltaTable.forName(spark, tblName) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tblName)) var operationMetrics = getOperationMetrics(deltaTable.history(1)) var expectedMetrics = Map( "numFiles" -> deltaLog.snapshot.numOfFiles.toString, "numOutputRows" -> "100" ) assert(operationMetrics("numOutputBytes").toLong > 0) checkOperationMetrics(expectedMetrics, operationMetrics, DeltaOperationMetrics.WRITE) // replace spark.range(50).writeTo(tblName).using("delta").replace() deltaLog.update() expectedMetrics = Map( "numFiles" -> deltaLog.snapshot.numOfFiles.toString, "numOutputRows" -> "50" ) operationMetrics = getOperationMetrics(deltaTable.history(1)) assert(operationMetrics("numOutputBytes").toLong > 0) checkOperationMetrics( expectedMetrics, operationMetrics, DeltaOperationMetrics.WRITE ++ DeltaOperationMetrics.OVERWRITE_REMOVES ) // create or replace spark.range(70).writeTo(tblName).using("delta").createOrReplace() deltaLog.update() expectedMetrics = Map( "numFiles" -> deltaLog.snapshot.numOfFiles.toString, "numOutputRows" -> "70" ) operationMetrics = getOperationMetrics(deltaTable.history(1)) assert(operationMetrics("numOutputBytes").toLong > 0) checkOperationMetrics( expectedMetrics, operationMetrics, DeltaOperationMetrics.WRITE ++ DeltaOperationMetrics.OVERWRITE_REMOVES ) } } } test("operation metrics for RESTORE") { withTempDir { dir => // version 0 spark.range(5).write.format("delta").save(dir.getCanonicalPath) val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(spark, dir.getAbsolutePath) val numFilesV0 = deltaLog.snapshot.numOfFiles val sizeBytesV0 = deltaLog.snapshot.sizeInBytes // version 1 spark.range(10, 12).write.format("delta").mode("append").save(dir.getCanonicalPath) val numFilesV1 = deltaLog.snapshot.numOfFiles val sizeBytesV1 = deltaLog.snapshot.sizeInBytes // version 2 - RESTORE table to version 0 sql(s"RESTORE TABLE delta.`${dir.getAbsolutePath}` VERSION AS OF 0") val expectedMetrics = Map( "tableSizeAfterRestore" -> sizeBytesV0, "numOfFilesAfterRestore" -> numFilesV0, "numRemovedFiles" -> (numFilesV1 - numFilesV0), "numRestoredFiles" -> 0, "removedFilesSize" -> (sizeBytesV1 - sizeBytesV0), "restoredFilesSize" -> 0).mapValues(_.toString).toMap val operationMetrics = getOperationMetrics(deltaTable.history(1)) checkOperationMetrics( expectedMetrics, operationMetrics, DeltaOperationMetrics.RESTORE) // check operation parameters checkLastOperation( dir.getAbsolutePath, expectedOperationParameters = Seq("version", "timestamp"), expectedColVals = Seq("RESTORE", "0"), columns = Seq($"operation", $"operationParameters.version")) // we can check metrics for a case where we restore files as well. // version 3 spark.range(10, 12).write.format("delta").mode("append").save(dir.getCanonicalPath) // version 4 - delete all rows sql(s"DELETE FROM delta.`${dir.getAbsolutePath}`") val numFilesV4 = deltaLog.update().numOfFiles val sizeBytesV4 = deltaLog.update().sizeInBytes // version 5 - RESTORE table to version 3 sql(s"RESTORE TABLE delta.`${dir.getAbsolutePath}` VERSION AS OF 3") val numFilesV5 = deltaLog.update().numOfFiles val sizeBytesV5 = deltaLog.update().sizeInBytes val expectedMetrics2 = Map( "tableSizeAfterRestore" -> sizeBytesV5, "numOfFilesAfterRestore" -> numFilesV5, "numRemovedFiles" -> 0, "numRestoredFiles" -> (numFilesV5 - numFilesV4), "removedFilesSize" -> 0, "restoredFilesSize" -> (sizeBytesV5 - sizeBytesV4)).mapValues(_.toString).toMap val operationMetrics2 = getOperationMetrics(deltaTable.history(1)) checkOperationMetrics( expectedMetrics2, operationMetrics2, DeltaOperationMetrics.RESTORE) } } test("test output schema of describe delta history command") { val tblName = "tbl" withTable(tblName) { sql(s"CREATE TABLE $tblName(id bigint) USING DELTA") val deltaTable = io.delta.tables.DeltaTable.forName(tblName) val expectedSchema = StructType(Seq( StructField("version", LongType, nullable = true), StructField("timestamp", TimestampType, nullable = true), StructField("userId", StringType, nullable = true), StructField("userName", StringType, nullable = true), StructField("operation", StringType, nullable = true), StructField("operationParameters", MapType(StringType, StringType, valueContainsNull = true), nullable = true), StructField("job", StructType(Seq( StructField("jobId", StringType, nullable = true), StructField("jobName", StringType, nullable = true), StructField("jobRunId", StringType, nullable = true), StructField("runId", StringType, nullable = true), StructField("jobOwnerId", StringType, nullable = true), StructField("triggerType", StringType, nullable = true))), nullable = true), StructField("notebook", StructType(Seq(StructField("notebookId", StringType, nullable = true))), nullable = true), StructField("clusterId", StringType, nullable = true), StructField("readVersion", LongType, nullable = true), StructField("isolationLevel", StringType, nullable = true), StructField("isBlindAppend", BooleanType, nullable = true), StructField("operationMetrics", MapType(StringType, StringType, valueContainsNull = true), nullable = true), StructField("userMetadata", StringType, nullable = true), StructField("engineInfo", StringType, nullable = true))) // Test schema from [[io.delta.tables.DeltaTable.history]] api val df1 = deltaTable.history(1) assert(df1.schema == expectedSchema) // Test schema from SQL api val df2 = spark.sql(s"DESCRIBE HISTORY $tblName LIMIT 1") assert(df2.schema == expectedSchema) } } testPathWrite("DPO and replaceWhere conflict throws exception") { path => val ex = intercept[DeltaIllegalArgumentException] { testData(Seq(10), Seq(1)).write.format("delta") .mode(SaveMode.Overwrite) .option("replaceWhere", "part = 1") .option("partitionOverwriteMode", "dynamic") .save(path) } assert(ex.getErrorClass === "DELTA_REPLACE_WHERE_WITH_DYNAMIC_PARTITION_OVERWRITE") }(WriteOptionsAssertion()) override def executePathWriteTest( write: String => Unit)(assertions: WriteOptionsAssertion): Unit = { withTempDir { tempDir => val path = createPartitionedTable(tempDir) write(path) assertWriteOptions(path, assertions) } } /** * Execute a write operation and run assertions on a name-based table. */ protected def executeTableWriteTest( write: String => Unit)(assertions: WriteOptionsAssertion): Unit = { withTable("test_table") { // Create initial partitioned table Seq((1, 1, "event1"), (2, 2, "event2"), (3, 1, "event3")) .toDF("id", "part", "event_name") .write.format("delta").partitionBy("part").saveAsTable("test_table") // Execute the write operation write("test_table") // Assert write options using table name val opParams = getTableCommitInfo("test_table") assertWriteOptionsFromParams(opParams, assertions) } } def assertWriteOptions( path: String, assertions: WriteOptionsAssertion): Unit = { val opParams = getCommitOpParams(path) assertWriteOptionsFromParams(opParams, assertions) } private def assertWriteOptionsFromParams( opParams: Map[String, String], asserts: WriteOptionsAssertion): Unit = { val expected = Map.newBuilder[String, String] if (asserts.isDynamicPartitionOverwrite) expected += ("isDynamicPartitionOverwrite" -> "true") if (asserts.canOverwriteSchema) expected += ("canOverwriteSchema" -> "true") if (asserts.canMergeSchema) expected += ("canMergeSchema" -> "true") asserts.predicate.foreach(pred => expected += ("predicate" -> pred)) assertInHistory(opParams, expected.result()) } def assertInHistory(opParams: Map[String, String], expected: Map[String, String]): Unit = { val allParams = Seq( "isDynamicPartitionOverwrite", "predicate", "canOverwriteSchema", "canMergeSchema" ) expected.foreach { case (key, value) => assert(opParams.get(key).exists(_.contains(value)), s"Expected $key=$value in DESCRIBE HISTORY, got ${opParams.get(key)}") } assertNotInHistory(opParams, (allParams.toSet -- expected.keySet).toSeq: _*) } def assertNotInHistory(opParams: Map[String, String], keys: String*): Unit = { keys.foreach { key => assert(!opParams.contains(key), s"Expected $key not in DESCRIBE HISTORY") } } def getCommitOpParams(tablePath: String): Map[String, String] = { val recentCommits = DeltaLog.forTable(spark, tablePath).history.getHistory(Some(1)) DeltaLog.forTable(spark, tablePath).history.getHistory(Some(1)).head.operationParameters } def getTableCommitInfo(tableName: String): Map[String, String] = { val deltaTable = io.delta.tables.DeltaTable.forName(spark, tableName) deltaTable.history(1).select("operationParameters") .head() .getMap(0) .asInstanceOf[Map[String, String]] } test("isV1SaveAsTableOverwrite logged only for v1 saveAsTable overwrite") { withTable("tbl") { // Initial create via saveAsTable - not an overwrite, flag should be absent spark.range(5).write.format("delta").saveAsTable("tbl") assert(!getTableCommitInfo("tbl").contains("isV1SaveAsTableOverwrite")) // v1 saveAsTable overwrite -> ReplaceTable with isV1SaveAsTableOverwrite = true spark.range(10).write.format("delta").mode("overwrite").saveAsTable("tbl") assert(getTableCommitInfo("tbl").get("isV1SaveAsTableOverwrite").contains("true")) // SQL REPLACE TABLE AS SELECT -> ReplaceTable but not v1 saveAsTable, flag should be absent sql("CREATE OR REPLACE TABLE tbl USING delta AS SELECT id FROM range(3)") assert(!getTableCommitInfo("tbl").contains("isV1SaveAsTableOverwrite")) // V2 writeTo.createOrReplace -> ReplaceTable but not v1 saveAsTable, flag should be absent spark.range(5).writeTo("tbl").using("delta").createOrReplace() assert(!getTableCommitInfo("tbl").contains("isV1SaveAsTableOverwrite")) // V2 writeTo.replace -> ReplaceTable but not v1 saveAsTable, flag should be absent spark.range(5).writeTo("tbl").using("delta").replace() assert(!getTableCommitInfo("tbl").contains("isV1SaveAsTableOverwrite")) } } } case class WriteOptionsAssertion( mode: String = "", isDynamicPartitionOverwrite: Boolean = false, canOverwriteSchema: Boolean = false, canMergeSchema: Boolean = false, predicate: Option[String] = None ) /** * Shared test utilities for validating write options in DESCRIBE HISTORY / commit stats. */ trait WriteOptionsTestBase { this: QueryTest with SharedSparkSession => import testImplicits._ // Execute a write operation and run assertions. protected def executePathWriteTest(write: String => Unit)(assertions: WriteOptionsAssertion): Unit // Execute a table based write operation and run assertions. protected def executeTableWriteTest( write: String => Unit)(assertions: WriteOptionsAssertion): Unit // Data generation helpers def createPartitionedTable(tempDir: java.io.File): String = { val tablePath = tempDir.getAbsolutePath Seq((1, 1, "event1"), (2, 2, "event2"), (3, 1, "event3")) .toDF("id", "part", "event_name") .write.format("delta").partitionBy("part").save(tablePath) tablePath } def testData(ids: Seq[Int], parts: Seq[Int]): DataFrame = { ids.zip(parts).map { case (id, part) => (id, part, s"event$id") } .toDF("id", "part", "event_name") } def testDataWithCols(id: Int, part: Int, extraCols: (String, String)*): DataFrame = { var df = Seq((id, part, s"event$id")).toDF("id", "part", "event_name") extraCols.foreach { case (name, value) => df = df.withColumn(name, lit(value)) } df } def testPathWrite( testName: String)(testBody: String => Unit)(assertions: WriteOptionsAssertion): Unit = { test(testName) { executePathWriteTest(testBody)(assertions) } } def testTableWrite( testName: String)(testBody: String => Unit)(assertions: WriteOptionsAssertion): Unit = { test(testName) { executeTableWriteTest(testBody)(assertions) } } def testWriteVariants(testName: String)( writeVariants: Seq[(String, Boolean, String => Unit)])( assertions: WriteOptionsAssertion): Unit = { writeVariants.foreach { case (variantName, isPathBased, writeFunc) => test(s"$testName via $variantName") { if (isPathBased) { executePathWriteTest(writeFunc)(assertions) } else { executeTableWriteTest(writeFunc)(assertions) } } } } testWriteVariants("write options for dynamic partition overwrite")( Seq( ("SQL", true, { path: String => withSQLConf(SQLConf.PARTITION_OVERWRITE_MODE.key -> "dynamic") { spark.sql(s"INSERT OVERWRITE TABLE delta.`$path` " + s"SELECT 5 as id, 1 as part, 'event5' as event_name") } }), ("DFv1", true, { path: String => testData(Seq(4), Seq(1)).write.format("delta") .mode(SaveMode.Overwrite) .option("partitionOverwriteMode", "dynamic") .save(path) }), ("DFv1 saveAsTable", false, { tableName: String => withSQLConf(SQLConf.PARTITION_OVERWRITE_MODE.key -> "dynamic") { testData(Seq(7), Seq(1)) .write .format("delta") .mode(SaveMode.Overwrite) .option("partitionOverwriteMode", "dynamic") .partitionBy("part") .saveAsTable(tableName) } }), ("DFv2", true, { path: String => testData(Seq(5), Seq(1)).writeTo(s"delta.`$path`").overwritePartitions() }) ) )(WriteOptionsAssertion(isDynamicPartitionOverwrite = true)) testWriteVariants("write options for replaceWhere")( Seq( ("SQL", true, { path: String => spark.sql(s"INSERT INTO TABLE delta.`$path` " + s"REPLACE WHERE part = 1 SELECT 6 as id, 1 as part, 'event6' as event_name") }), ("DFv1", true, { path: String => testData(Seq(5, 6), Seq(1, 1)).write.format("delta") .mode("overwrite") .option("replaceWhere", "part = 1") .save(path) }), ("DFv1 saveAsTable", false, { tableName: String => testData(Seq(7, 8), Seq(1, 1)) .write .format("delta") .mode(SaveMode.Overwrite) .option("replaceWhere", "part = 1") .partitionBy("part") .saveAsTable(tableName) }), ("DFv2", true, { path: String => testData(Seq(5, 6), Seq(1, 1)).writeTo(s"delta.`$path`") .overwrite($"part" === 1) }) ) )(WriteOptionsAssertion(predicate = Some("part = 1"))) testPathWrite("explicitly false option not persisted in commit info") { path => testData(Seq(9), Seq(1)).write.format("delta") .mode(SaveMode.Append) .option("mergeSchema", "false") .save(path) }(WriteOptionsAssertion()) testPathWrite("multiple false options not persisted in commit info") { path => testData(Seq(13), Seq(1)).write.format("delta") .mode(SaveMode.Overwrite) .option("mergeSchema", "false") .option("overwriteSchema", "false") .save(path) }(WriteOptionsAssertion()) testPathWrite("write options for overwriteSchema") { path => testDataWithCols(7, 1, "newcol" -> "extra").write.format("delta") .mode(SaveMode.Overwrite) .option("overwriteSchema", "true") .save(path) }(WriteOptionsAssertion(canOverwriteSchema = true)) testWriteVariants("write options for DFv2 replace with overwriteSchema")( Seq( ("replace()", false, { path: String => testDataWithCols(14, 1, "newcol" -> "extra") .writeTo(path) .using("delta") .option("overwriteSchema", "true") .replace() }), ("createOrReplace()", false, { path: String => testDataWithCols(15, 1, "newcol" -> "extra") .writeTo(path) .using("delta") .option("overwriteSchema", "true") .createOrReplace() }) ) )(WriteOptionsAssertion(canOverwriteSchema = true)) testWriteVariants("write options for DFv2 replace with mergeSchema")( Seq( ("replace()", false, { path: String => testDataWithCols(16, 1, "newcol" -> "extra") .writeTo(path) .using("delta") .option("mergeSchema", "true") .replace() }), ("createOrReplace()", false, { path: String => testDataWithCols(17, 1, "newcol" -> "extra") .writeTo(path) .using("delta") .option("mergeSchema", "true") .createOrReplace() }) ) )(WriteOptionsAssertion(canMergeSchema = true)) testWriteVariants("write options for mergeSchema")( Seq( ("DFv1 option", true, { path: String => testDataWithCols(8, 1, "newcol" -> "extra", "anothercol" -> "extra2") .write.format("delta") .mode(SaveMode.Append) .option("mergeSchema", "true") .save(path) }), ("saveAsTable", false, { tableName: String => testDataWithCols(14, 1, "newcol" -> "extra") .write .format("delta") .mode(SaveMode.Append) .option("mergeSchema", "true") .saveAsTable(tableName) }), ("SQL config", true, { path: String => withSQLConf("spark.databricks.delta.schema.autoMerge.enabled" -> "true") { testDataWithCols(13, 1, "newcol" -> "extra").write.format("delta") .mode(SaveMode.Append).save(path) } }) ) )(WriteOptionsAssertion(mode = "Append", canMergeSchema = true)) testPathWrite("write options - both replaceWhere and overwriteSchema logged " + "even though replaceWhere takes precedence") { path => testData(Seq(12), Seq(1)).write.format("delta") .mode(SaveMode.Overwrite) .option("replaceWhere", "part = 1") .option("overwriteSchema", "true") .save(path) }(WriteOptionsAssertion( canOverwriteSchema = true, predicate = Some("part = 1") )) testPathWrite("write options - mergeSchema and overwriteSchema combination") { path => testDataWithCols(11, 1, "newcol" -> "extra").write.format("delta") .mode(SaveMode.Overwrite) .option("overwriteSchema", "true") .option("mergeSchema", "true") .save(path) }(WriteOptionsAssertion(canOverwriteSchema = true, canMergeSchema = true)) testPathWrite("write options - DPO with mergeSchema") { path => testDataWithCols(14, 1, "newcol" -> "extra").write.format("delta") .mode(SaveMode.Overwrite) .option("partitionOverwriteMode", "dynamic") .option("mergeSchema", "true") .save(path) }(WriteOptionsAssertion(isDynamicPartitionOverwrite = true, canMergeSchema = true)) testPathWrite("write options - DFv2 overwriteSchema option") { path => testDataWithCols(7, 1) .writeTo(s"delta.`$path`") .option("overwriteSchema", "true") .append() }(WriteOptionsAssertion(mode = "Append", canOverwriteSchema = true)) testPathWrite("write options - DFv2 mergeSchema option") { path => testDataWithCols(8, 1, "newcol" -> "extra") .writeTo(s"delta.`$path`") .option("mergeSchema", "true") .append() }(WriteOptionsAssertion(mode = "Append", canMergeSchema = true)) testPathWrite("write options - REPLACE TABLE with DPO") { path => withSQLConf(SQLConf.PARTITION_OVERWRITE_MODE.key -> "dynamic") { spark.sql(s""" CREATE OR REPLACE TABLE delta.`$path` USING delta PARTITIONED BY (part) AS SELECT 7 as id, 1 as part, 'event7' as event_name """) } }(WriteOptionsAssertion(isDynamicPartitionOverwrite = true)) } class DescribeDeltaHistorySuite extends DescribeDeltaHistorySuiteBase with DeltaSQLCommandTest class DescribeDeltaHistoryWithCatalogOwnedBatch100Suite extends DescribeDeltaHistorySuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) override def sparkConf: SparkConf = super.sparkConf .set(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey, "false") } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DomainMetadataRemovalSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.SparkConf import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.test.SharedSparkSession class DomainMetadataRemovalSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { val testTableName = "test_domain_metadata_removal_table" def addData(deltaLog: DeltaLog, start: Long, end: Long): Unit = { spark.range(start, end, step = 1, numPartitions = 2) .write .format("delta") .mode("append") .save(deltaLog.dataPath.toString) } def createTableWithDomainMetadata(): DeltaLog = { sql(s"DROP TABLE IF EXISTS $testTableName") sql( s"""CREATE TABLE $testTableName (id LONG) |USING DELTA |TBLPROPERTIES( |'delta.feature.${DomainMetadataTableFeature.name}' = 'supported' |)""".stripMargin) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(testTableName)) assert(deltaLog.update().protocol.isFeatureSupported(DomainMetadataTableFeature)) addData(deltaLog, 0, 100) deltaLog } def dropFeature( feature: TableFeature, truncateHistory: Boolean = false): Unit = { val sqlText = s""" |ALTER TABLE $testTableName |DROP FEATURE ${feature.name} |${if (truncateHistory) "TRUNCATE HISTORY" else ""} |""".stripMargin sql(sqlText) } def validateDomainMetadataRemoval(deltaLog: DeltaLog): Unit = { val snapshot = deltaLog.update() assert(!snapshot.protocol.isFeatureSupported(DomainMetadataTableFeature)) assert(snapshot.domainMetadata.isEmpty) } test("DomainMetadata can be dropped") { val deltaLog = createTableWithDomainMetadata() dropFeature(DomainMetadataTableFeature) validateDomainMetadataRemoval(deltaLog) } test("Drop DomainMetadata feature when leaked domain metadata exist in the table") { case class TestMetadataDomain() extends JsonMetadataDomain[TestMetadataDomain] { override val domainName: String = "delta.test" } val deltaLog = createTableWithDomainMetadata() // Add a domainMetadata action. deltaLog .startTransaction(catalogTableOpt = None) .commit(Seq(TestMetadataDomain().toDomainMetadata), DeltaOperations.ManualUpdate) dropFeature(DomainMetadataTableFeature) validateDomainMetadataRemoval(deltaLog) } test("Cannot drop DomainMetadata if there is a dependent feature on the table") { createTableWithDomainMetadata() // Enable row tracking on the table. sql( s"""ALTER TABLE $testTableName |SET TBLPROPERTIES( |${DeltaConfigs.ROW_TRACKING_ENABLED.key} = 'true' |)""".stripMargin) val e = intercept[DeltaTableFeatureException] { dropFeature(DomainMetadataTableFeature) } checkError( e, "DELTA_FEATURE_DROP_DEPENDENT_FEATURE", parameters = Map( "feature" -> "domainMetadata", "dependentFeatures" -> "rowTracking")) } test("Drop domainMetadata after dropping a dependent feature") { val deltaLog = createTableWithDomainMetadata() addData(deltaLog, 0, 100) // Enable row tracking on the table. This also enables the domainMetadata feature. sql( s"""ALTER TABLE $testTableName |SET TBLPROPERTIES( |${DeltaConfigs.ROW_TRACKING_ENABLED.key} = 'true' |)""".stripMargin) assert(deltaLog.update().protocol.isFeatureSupported(DomainMetadataTableFeature)) dropFeature(RowTrackingFeature) dropFeature(DomainMetadataTableFeature) validateDomainMetadataRemoval(deltaLog) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DomainMetadataSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.nio.charset.StandardCharsets.UTF_8 import java.util.concurrent.ExecutionException import scala.util.{Failure, Success, Try} import org.apache.spark.sql.delta.DeltaOperations.{ManualUpdate, Truncate} import org.apache.spark.sql.delta.Relocated.CheckpointFileManager import org.apache.spark.sql.delta.actions.{DomainMetadata, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import org.junit.Assert._ import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.test.SharedSparkSession class DomainMetadataSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { import testImplicits._ private def sortByDomain(domainMetadata: Seq[DomainMetadata]): Seq[DomainMetadata] = domainMetadata.sortBy(_.domain) /** * A helper to validate the [[DomainMetadata]] actions can be retained during the delta state * reconstruction. * * @param doCheckpoint: Explicitly create a delta log checkpoint if marked as true. * @param doChecksum: Disable writting checksum file if marked as false. */ private def validateStateReconstructionHelper( doCheckpoint: Boolean, doChecksum: Boolean): Unit = { val table = "testTable" withTable(table) { withSQLConf( DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> doChecksum.toString) { sql( s""" | CREATE TABLE $table(id int) USING delta | tblproperties | ('${TableFeatureProtocolUtils.propertyKey(DomainMetadataTableFeature)}' = 'enabled') |""".stripMargin) (1 to 100).toDF("id").write.format("delta").mode("append").saveAsTable(table) var deltaTable = DeltaTableV2(spark, TableIdentifier(table)) def deltaLog = deltaTable.deltaLog assert(deltaTable.snapshot.domainMetadata.isEmpty) val domainMetadata = DomainMetadata("testDomain1", "", false) :: DomainMetadata("testDomain2", "{\"key1\":\"value1\"", false) :: Nil deltaTable.startTransactionWithInitialSnapshot().commit(domainMetadata, Truncate()) assertEquals(sortByDomain(domainMetadata), sortByDomain(deltaLog.update().domainMetadata)) assert(deltaLog.update().logSegment.checkpointProvider.version === -1) if (doCheckpoint) { deltaLog.checkpoint(deltaLog.unsafeVolatileSnapshot) // Clear the DeltaLog cache to force creating a new DeltaLog instance which will build // the Snapshot from the checkpoint file. DeltaLog.clearCache() deltaTable = DeltaTableV2(spark, TableIdentifier(table)) assert(!deltaTable.snapshot.logSegment.checkpointProvider.isEmpty) assertEquals( sortByDomain(domainMetadata), sortByDomain(deltaTable.snapshot.domainMetadata)) } DeltaLog.clearCache() deltaTable = DeltaTableV2(spark, TableIdentifier(table)) val checksumOpt = deltaTable.snapshot.checksumOpt if (doChecksum) { assertEquals( sortByDomain(checksumOpt.get.domainMetadata.get), sortByDomain(domainMetadata)) } else { assert(checksumOpt.isEmpty) } assert(deltaLog.update().validateChecksum()) } } } // A helper to validate [[DomainMetadata]] actions can be deleted. private def validateDeletionHelper(doCheckpoint: Boolean, doChecksum: Boolean): Unit = { val table = "testTable" withTable(table) { withSQLConf( DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> doChecksum.toString ) { sql( s""" | CREATE TABLE $table(id int) USING delta | tblproperties | ('${TableFeatureProtocolUtils.propertyKey(DomainMetadataTableFeature)}' = 'enabled') |""".stripMargin) (1 to 100).toDF("id").write.format("delta").mode("append").saveAsTable(table) DeltaLog.clearCache() val deltaTable = DeltaTableV2(spark, TableIdentifier(table)) val deltaLog = deltaTable.deltaLog assert(deltaTable.snapshot.domainMetadata.isEmpty) val domainMetadata = DomainMetadata("testDomain1", "", false) :: DomainMetadata("testDomain2", "{\"key1\":\"value1\"}", false) :: Nil deltaTable.startTransactionWithInitialSnapshot().commit(domainMetadata, Truncate()) assertEquals(sortByDomain(domainMetadata), sortByDomain(deltaLog.update().domainMetadata)) assert(deltaLog.update().logSegment.checkpointProvider.version === -1) // Delete testDomain1. deltaTable.startTransaction().commit( DomainMetadata("testDomain1", "", true) :: Nil, Truncate()) val domainMetadatasAfterDeletion = DomainMetadata( "testDomain2", "{\"key1\":\"value1\"}", false) :: Nil assertEquals( sortByDomain(domainMetadatasAfterDeletion), sortByDomain(deltaLog.update().domainMetadata)) // Create a new commit and validate the incrementally built snapshot state respects the // DomainMetadata deletion. deltaTable.startTransaction().commit(Nil, ManualUpdate) var snapshot = deltaLog.update() assertEquals(sortByDomain(domainMetadatasAfterDeletion), snapshot.domainMetadata) if (doCheckpoint) { deltaLog.checkpoint(snapshot) assertEquals( sortByDomain(domainMetadatasAfterDeletion), deltaLog.update().domainMetadata) } // force state reconstruction and validate it respects the DomainMetadata retention. DeltaLog.clearCache() snapshot = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(table))._2 assertEquals(sortByDomain(domainMetadatasAfterDeletion), snapshot.domainMetadata) } } } test("DomainMetadata actions tracking in CRC should stop once threshold is crossed") { def assertDomainMetadatas( deltaLog: DeltaLog, expectedDomainMetadatas: Seq[DomainMetadata], expectedInCrc: Boolean): Unit = { val snapshot = deltaLog.update() assert(snapshot.validateChecksum()) assertEquals(sortByDomain(expectedDomainMetadatas), sortByDomain(snapshot.domainMetadata)) assert(snapshot.checksumOpt.nonEmpty) if (expectedInCrc) { assert(snapshot.checksumOpt.get.domainMetadata.nonEmpty) assertEquals( sortByDomain(expectedDomainMetadatas), sortByDomain(snapshot.checksumOpt.get.domainMetadata.get)) } else { assert(snapshot.checksumOpt.get.domainMetadata.isEmpty) } } val table = "testTable" withSQLConf( DeltaSQLConf.DELTA_MAX_DOMAIN_METADATAS_IN_CRC.key -> "2") { withTable(table) { sql( s""" | CREATE TABLE $table(id int) USING delta | tblproperties | ('${TableFeatureProtocolUtils.propertyKey(DomainMetadataTableFeature)}' = 'enabled') |""".stripMargin) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(table)) assertDomainMetadatas(deltaLog, Seq.empty, true) deltaLog .startTransaction() .commit(DomainMetadata("testDomain1", "", false) :: Nil, Truncate()) assertDomainMetadatas( deltaLog, DomainMetadata("testDomain1", "", false) :: Nil, true) deltaLog .startTransaction() .commit(DomainMetadata("testDomain2", "", false) :: Nil, Truncate()) assertDomainMetadatas( deltaLog, DomainMetadata("testDomain1", "", false) :: DomainMetadata("testDomain2", "", false) :: Nil, true) deltaLog .startTransaction() .commit(DomainMetadata("testDomain3", "", false) :: Nil, Truncate()) assertDomainMetadatas( deltaLog, DomainMetadata("testDomain1", "", false) :: DomainMetadata("testDomain2", "", false) :: DomainMetadata("testDomain3", "", false) :: Nil, false) } } } test("Validate crc can be read when domainMetadata is missing") { val table = "testTable" withTable(table) { sql( s""" | CREATE TABLE $table(id int) USING delta | tblproperties | ('${TableFeatureProtocolUtils.propertyKey(DomainMetadataTableFeature)}' = 'enabled') |""".stripMargin) val deltaTable = DeltaTableV2(spark, TableIdentifier(table)) val deltaLog = deltaTable.deltaLog val version = deltaTable .startTransactionWithInitialSnapshot() .commit(DomainMetadata("testDomain1", "", false) :: Nil, Truncate()) val snapshot = deltaLog.update() assert(snapshot.checksumOpt.nonEmpty) assert(snapshot.checksumOpt.get.domainMetadata.nonEmpty) val originalChecksum = snapshot.checksumOpt.get // Write out a checksum without domainMetadata. val checksumWithoutDomainMetadata = originalChecksum.copy(domainMetadata = None) val writer = CheckpointFileManager.create(deltaLog.logPath, deltaLog.newDeltaHadoopConf()) val toWrite = JsonUtils.toJson(checksumWithoutDomainMetadata) + "\n" val stream = writer.createAtomic( FileNames.checksumFile(deltaLog.logPath, version + 1), overwriteIfPossible = false) stream.write(toWrite.getBytes(UTF_8)) stream.close() // Make sure the read is not broken. val content = deltaLog .store .read( FileNames.checksumFile(deltaLog.logPath, version + 1), deltaLog.newDeltaHadoopConf()) val checksumFromFile = JsonUtils.mapper.readValue[VersionChecksum](content.head) assert(checksumWithoutDomainMetadata == checksumFromFile) } } test("DomainMetadata action survives state reconstruction [w/o checkpoint, w/o checksum]") { validateStateReconstructionHelper(doCheckpoint = false, doChecksum = false) } test("DomainMetadata action survives state reconstruction [w/ checkpoint, w/ checksum]") { validateStateReconstructionHelper(doCheckpoint = true, doChecksum = true) } test("DomainMetadata action survives state reconstruction [w/ checkpoint, w/o checksum]") { validateStateReconstructionHelper(doCheckpoint = true, doChecksum = false) } test("DomainMetadata action survives state reconstruction [w/o checkpoint, w/ checksum]") { validateStateReconstructionHelper(doCheckpoint = false, doChecksum = true) } test("DomainMetadata deletion [w/o checkpoint, w/o checksum]") { validateDeletionHelper(doCheckpoint = false, doChecksum = false) } test("DomainMetadata deletion [w/ checkpoint, w/o checksum]") { validateDeletionHelper(doCheckpoint = true, doChecksum = false) } test("DomainMetadata deletion [w/o checkpoint, w/ checksum]") { validateDeletionHelper(doCheckpoint = false, doChecksum = true) } test("DomainMetadata deletion [w/ checkpoint, w/ checksum]") { validateDeletionHelper(doCheckpoint = true, doChecksum = true) } test("Multiple DomainMetadatas with the same domain should fail in single transaction") { val table = "testTable" withTable(table) { sql( s""" | CREATE TABLE $table(id int) USING delta | tblproperties | ('${TableFeatureProtocolUtils.propertyKey(DomainMetadataTableFeature)}' = 'enabled') |""".stripMargin) (1 to 100).toDF("id").write.format("delta").mode("append").saveAsTable(table) val deltaTable = DeltaTableV2(spark, TableIdentifier(table)) val domainMetadata = DomainMetadata("testDomain1", "", false) :: DomainMetadata("testDomain1", "", false) :: Nil val e = intercept[DeltaIllegalArgumentException] { deltaTable.startTransactionWithInitialSnapshot().commit(domainMetadata, Truncate()) } checkError(e, "DELTA_DUPLICATE_DOMAIN_METADATA_INTERNAL_ERROR", "42601", Map("domainName" -> "testDomain1")) } } test("Validate the failure when table feature is not enabled") { withTempDir { dir => (1 to 100).toDF().write.format("delta").save(dir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, dir) val domainMetadata = DomainMetadata("testDomain1", "", false) :: Nil val e = intercept[DeltaIllegalArgumentException] { deltaLog.startTransaction().commit(domainMetadata, Truncate()) } checkError(e, "DELTA_DOMAIN_METADATA_NOT_SUPPORTED", "0A000", Map("domainNames" -> "[testDomain1]")) } } test("Validate the lifespan of metadata domains for the REPLACE TABLE operation") { val existingDomainMetadatas = DomainMetadata("testDomain1", "", false) :: DomainMetadata("testDomain2", "", false) :: Nil val newDomainMetadatas = DomainMetadata("testDomain2", "key=val", false) :: DomainMetadata("testDomain3", "", false) :: Nil val result = DomainMetadataUtils.handleDomainMetadataForReplaceTable( existingDomainMetadatas, newDomainMetadatas) // testDomain1: survives by default (not in the final list since it already // exists in the snapshot). // testDomain2: overwritten by new domain metadata // testDomain3: added to the final list since it only appears in the new set. assert(result === DomainMetadata("testDomain2", "key=val", false) :: // Overwritten DomainMetadata("testDomain3", "", false) :: // New metadata domain Nil) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/DuplicatingListLogStoreSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.{File} import java.nio.charset.StandardCharsets import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.storage.{HDFSLogStore} import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.util.DeltaFileOperations import com.google.common.io.Files import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.SparkConf import org.apache.spark.sql.test.SharedSparkSession class DuplicatingListLogStore(sparkConf: SparkConf, defaultHadoopConf: Configuration) extends HDFSLogStore(sparkConf, defaultHadoopConf) { override def listFrom(path: Path, hadoopConf: Configuration): Iterator[FileStatus] = { val list = super.listFrom(path, hadoopConf).toSeq // The first listing if directory will be listed twice to mimic the WASBS Log Store if (!list.isEmpty && list.head.isDirectory) { (Seq(list.head) ++ list).toIterator } else { list.toIterator } } } class DuplicatingListLogStoreSuite extends SharedSparkSession with DeltaSQLCommandTest { override def sparkConf: SparkConf = { super.sparkConf.set("spark.databricks.tahoe.logStore.class", classOf[DuplicatingListLogStore].getName) } def pathExists(deltaLog: DeltaLog, filePath: String): Boolean = { val fs = deltaLog.logPath.getFileSystem(deltaLog.newDeltaHadoopConf()) fs.exists(DeltaFileOperations.absolutePath(deltaLog.dataPath.toString, filePath)) } test("vacuum should handle duplicate listing") { withTempDir { dir => // create cdc file (lexicographically < _delta_log) spark.range(10).write.format("delta").save(dir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(dir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) val cdcDir = new File(new Path(dir.getAbsolutePath, "_change_data").toString) cdcDir.mkdir() val cdcPath = new File( new Path(cdcDir.getAbsolutePath, "dupFile").toString) Files.write("test", cdcPath, StandardCharsets.UTF_8) require(pathExists(deltaLog, cdcPath.toString)) require(pathExists(deltaLog, cdcDir.toString)) withSQLConf(DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED.key -> "false") { deltaTable.vacuum(0) // check if path doesn't exists assert(!pathExists(deltaLog, cdcPath.toString)) // to delete directories deltaTable.vacuum(0) assert(!pathExists(deltaLog, cdcDir.toString)) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/EvolvabilitySuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.test.shims.StreamingTestShims.MemoryStream import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import org.apache.hadoop.fs.Path // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.catalyst.expressions.Literal import org.apache.spark.sql.functions.lit import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.streaming.Trigger import org.apache.spark.sql.types.StringType import org.apache.spark.util.Utils class EvolvabilitySuite extends EvolvabilitySuiteBase with DeltaSQLCommandTest { import testImplicits._ test("delta 0.1.0") { testEvolvability("src/test/resources/delta/delta-0.1.0") } test("delta 0.1.0 - case sensitivity enabled") { withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { testEvolvability("src/test/resources/delta/delta-0.1.0") } } test("serialized partition values must contain null values") { val tempDir = Utils.createTempDir().toString val df1 = spark.range(5).withColumn("part", lit(null).cast(StringType)) val df2 = spark.range(5).withColumn("part", lit("1")) df1.union(df2).coalesce(1).write.partitionBy("part").format("delta").save(tempDir) // Clear the cache DeltaLog.clearCache() val deltaLog = DeltaLog.forTable(spark, tempDir) val dataThere = deltaLog.snapshot.allFiles.collect().forall { addFile => if (!addFile.partitionValues.contains("part")) { fail(s"The partition values: ${addFile.partitionValues} didn't contain the column 'part'.") } val value = addFile.partitionValues("part") value === null || value === "1" } assert(dataThere, "Partition values didn't match with null or '1'") // Check serialized JSON as well val contents = deltaLog.store.read( FileNames.unsafeDeltaFile(deltaLog.logPath, 0L), deltaLog.newDeltaHadoopConf()) assert(contents.exists(_.contains(""""part":null""")), "null value should be written in json") } testQuietly("parse old version LastCheckpointInfo") { assert(JsonUtils.mapper.readValue[LastCheckpointInfo]("""{"version":1,"size":1}""") === LastCheckpointInfo(1, 1, None, None, None, None)) } test("parse partial version LastCheckpointInfo") { assert(JsonUtils.mapper.readValue[LastCheckpointInfo]( """{"version":1,"size":1,"parts":100}""") === LastCheckpointInfo(1, 1, Some(100), None, None, None)) } // Following tests verify that operations on Delta table won't fail when there is an // unknown column in Delta files and checkpoints. // The modified Delta files and checkpoints with an extra column is generated by // `EvolvabilitySuiteBase.generateTransactionLogWithExtraColumn()` test("transaction log schema evolvability - batch change data read") { withTempDir { dir => withSQLConf( DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> "true", // All files verification will always fail in this test since we the extra column // will not be present in the `allFiles` of the CRC. DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_VERIFICATION_MODE_ENABLED.key -> "false", DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_FORCE_VERIFICATION_MODE_FOR_NON_UTC_ENABLED.key -> "false" ) { EvolvabilitySuiteBase.generateTransactionLogWithExtraColumn(spark, dir.getAbsolutePath) spark.sql(s"UPDATE delta.`${dir.getAbsolutePath}` SET value = 10") spark.read.format("delta").option("readChangeFeed", "true") .option("startingVersion", 0).load(dir.getAbsolutePath).collect() val expectedPreimage = (1 until 10).flatMap(x => Seq(x, x)).toSeq val expectedPostimage = Seq.fill(18)(10) testCdfUpdate(dir.getAbsolutePath, 6, expectedPreimage, expectedPostimage) } } } test("transaction log schema evolvability - streaming change data read") { withTempDir { dir => withSQLConf( DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> "true", // All files verification will always fail in this test since we the extra column // will not be present in the `allFiles` of the CRC. DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_VERIFICATION_MODE_ENABLED.key -> "false", DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_FORCE_VERIFICATION_MODE_FOR_NON_UTC_ENABLED.key -> "false" ) { EvolvabilitySuiteBase.generateTransactionLogWithExtraColumn(spark, dir.getAbsolutePath) spark.sql(s"UPDATE delta.`${dir.getAbsolutePath}` SET value = 10") val query = spark.readStream.format("delta") .option("readChangeFeed", "true") .option("startingVersion", 0) .load(dir.getAbsolutePath) .writeStream.format("noop").start() try { query.processAllAvailable() } finally { query.stop() } val expectedPreimage = (1 until 10).flatMap(x => Seq(x, x)).toSeq val expectedPostimage = Seq.fill(18)(10) testCdfUpdate(dir.getAbsolutePath, 6, expectedPreimage, expectedPostimage, true) } } } test("transaction log schema evolvability - batch read") { testLogSchemaEvolvability( (path: String) => { spark.read.format("delta").load(path).collect() } ) } test("transaction log schema evolvability - batch write") { testLogSchemaEvolvability( (path: String) => { (10 until 20).map(num => (num, num)).toDF("key", "value") .write.format("delta").mode("append").save(path) spark.read.format("delta").load(path).collect() } ) } test("transaction log schema evolvability - streaming read") { testLogSchemaEvolvability( (path: String) => { val query = spark.readStream.format("delta").load(path).writeStream.format("noop").start() try { query.processAllAvailable() } finally { query.stop() } } ) } test("transaction log schema evolvability - streaming write") { testLogSchemaEvolvability( (path: String) => { withTempDir { tempDir => val memStream = MemoryStream[(Int, Int)] memStream.addData((11, 11), (12, 12)) val stream = memStream.toDS().toDF("key", "value") .coalesce(1).writeStream .format("delta") .trigger(Trigger.Once) .outputMode("append") .option("checkpointLocation", tempDir.getCanonicalPath + "/cp") .start(path) try { stream.processAllAvailable() } finally { stream.stop() } } } ) } test("transaction log schema evolvability - describe commands") { testLogSchemaEvolvability( (path: String) => { spark.sql(s"DESCRIBE delta.`$path`") spark.sql(s"DESCRIBE HISTORY delta.`$path`") spark.sql(s"DESCRIBE DETAIL delta.`$path`") } ) } test("transaction log schema evolvability - vacuum") { testLogSchemaEvolvability( (path: String) => { sql(s"VACUUM delta.`$path`") } ) } test("transaction log schema evolvability - alter table") { testLogSchemaEvolvability( (path: String) => { sql(s"ALTER TABLE delta.`$path` ADD COLUMNS (col int)") } ) } test("transaction log schema evolvability - delete") { testLogSchemaEvolvability( (path: String) => { sql(s"DELETE FROM delta.`$path` WHERE key = 1") } ) } test("transaction log schema evolvability - update") { testLogSchemaEvolvability( (path: String) => { sql(s"UPDATE delta.`$path` set value = 100 WHERE key = 1") } ) } test("transaction log schema evolvability - merge") { testLogSchemaEvolvability( (path: String) => { withTable("source") { Seq((1, 5), (11, 12)) .toDF("key", "value") .write .mode("overwrite") .format("delta") .saveAsTable("source") sql( s""" |MERGE INTO delta.`$path` tgrt |USING source src |ON src.key = tgrt.key |WHEN MATCHED THEN | UPDATE SET key = 20 + src.key, value = 20 + src.value |WHEN NOT MATCHED THEN | INSERT (key, value) VALUES (src.key + 5, src.value + 10) """.stripMargin ) } } ) } test("Delta Lake issue 1229: able to read a checkpoint containing `numRecords`") { // table created using Delta 1.2.1 which has additional field `numRecords` in // checkpoint schema. It is removed in version after 1.2.1. // Make sure we are able to read the Delta table in the latest version. val tablePath = "src/test/resources/delta/delta-1.2.1" assert( spark.read.format("delta") .load(tablePath).where("col1 = 8").count() === 9L) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/EvolvabilitySuiteBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import org.apache.spark.sql.delta.actions.{Action, AddFile, FileAction, SingleAction} import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.test.shims.StreamingTestShims import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import org.apache.spark.sql.delta.util.JsonUtils import org.apache.commons.io.FileUtils import org.apache.hadoop.fs.Path import org.apache.spark.sql.{QueryTest, Row, SparkSession} import org.apache.spark.sql.functions.lit import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.StructType import org.apache.spark.util.Utils abstract class EvolvabilitySuiteBase extends QueryTest with SharedSparkSession with DeltaSQLTestUtils { import testImplicits._ protected def testEvolvability(tablePath: String): Unit = { // Check we can load everything from a log checkpoint val deltaLog = DeltaLog.forTable(spark, new Path(tablePath)) val path = deltaLog.dataPath.toString checkDatasetUnorderly( spark.read.format("delta").load(path).select("id", "value").as[(Int, String)], 4 -> "d", 5 -> "e", 6 -> "f") assert(deltaLog.snapshot.metadata.schema === StructType.fromDDL("id INT, value STRING")) assert(deltaLog.snapshot.metadata.partitionSchema === StructType.fromDDL("id INT")) // Check we can load LastCheckpointInfo val lastCheckpointOpt = deltaLog.readLastCheckpointFile() assert(lastCheckpointOpt.get.version === 3) assert(lastCheckpointOpt.get.size === 6L) assert(lastCheckpointOpt.get.checkpointSchema.isEmpty) // Check we can parse all `Action`s in delta files. It doesn't check correctness. deltaLog.getChanges(0L).toList.map(_._2.toList) } /** * This tests the evolution of the schema at delta file and checkpoint file. * Operations on the Delta table shouldn't fail when there is an unknown column * in delta file and checkpoint file. * * Table Schema: StructType(StructField("key", StringType), StructField("value", StringType)) * Overwritten Delta file: {"some_new_feature":{"a":1}} * Overwritten checkpoint file with a new column called `unknown` with boolean type. * * The delta file and checkpoint file with an unknown column are generated by * `EvolvabilitySuiteBase.generateTransactionLogWithExtraColumn()`. */ protected def testLogSchemaEvolvability(operation: String => Unit): Unit = { withTempDir { tempDir => // copy the existing dir to the temp data dir. FileUtils.copyDirectory( new File("src/test/resources/delta/transaction_log_schema_evolvability"), tempDir) makeWritable(tempDir) DeltaLog.clearCache() operation(tempDir.getAbsolutePath) } } /** * Recursively make all files in a directory writable. */ private def makeWritable(directory: File): Unit = { if (!directory.isDirectory) return directory.listFiles().foreach { file => if (file.isDirectory) { makeWritable(file) } else { file.setWritable(true) } } } /** * Read from a table's CDF and check for the expected preimage/postimage after applying an update */ protected def testCdfUpdate( tablePath: String, commitVersion: Long, expectedPreimage: Seq[Int], expectedPostimage: Seq[Int], streaming: Boolean = false): Unit = { val df = if (streaming) { val q = spark.readStream.format("delta") .option("readChangeFeed", "true") .option("startingVersion", commitVersion) .option("endingVersion", commitVersion) .load(tablePath) .writeStream .option("checkpointLocation", tablePath + "-checkpoint") .toTable("streaming"); try { q.processAllAvailable() } finally { q.stop() } spark.read.table("streaming") } else { spark.read.format("delta") .option("readChangeFeed", "true") .option("startingVersion", commitVersion) .option("endingVersion", commitVersion) .load(tablePath) } val preimage = df.where("_change_type = 'update_preimage'").select("value") val postimage = df.where("_change_type = 'update_postimage'").select("value") checkAnswer(preimage, expectedPreimage.map(Row(_))) checkAnswer(postimage, expectedPostimage.map(Row(_))) } } // scalastyle:off /*** * A tool to generate data and transaction log for evolvability tests. * * Here are the steps to generate data. * * 1. Update `EvolvabilitySuite.generateData` if there are new [[Action]] types. * 2. Change the following command with the right path and run it. Note: the working directory is "[delta_project_root]". * * scalastyle:off * ``` * build/sbt "core/test:runMain org.apache.spark.sql.delta.EvolvabilitySuite src/test/resources/delta/delta-0.1.0 generateData" * ``` * * You can also use this tool to generate DeltaLog that contains a checkpoint a json log with a new column. * * scalastyle:off * ``` * build/sbt "core/test:runMain org.apache.spark.sql.delta.EvolvabilitySuite /path/src/test/resources/delta/transaction_log_schema_evolvability generateTransactionLogWithExtraColumn" * ``` */ // scalastyle:on object EvolvabilitySuiteBase { def generateData( spark: SparkSession, path: String, tblProps: Map[DeltaConfig[_], String] = Map.empty): Unit = { import org.apache.spark.sql.delta.implicits._ implicit val sparkSession: SparkSession = spark implicit val s = spark.sqlContext Seq(1, 2, 3).toDF(spark).write.format("delta").save(path) if (tblProps.nonEmpty) { val tblPropsStr = tblProps.map { case (k, v) => s"'${k.key}' = '$v'" }.mkString(", ") spark.sql(s"CREATE TABLE test USING DELTA LOCATION '$path'") spark.sql(s"ALTER TABLE test SET TBLPROPERTIES($tblPropsStr)") } Seq(1, 2, 3).toDF(spark).write.format("delta").mode("append").save(path) Seq(1, 2, 3).toDF(spark).write.format("delta").mode("overwrite").save(path) val checkpoint = Utils.createTempDir().toString val data = StreamingTestShims.MemoryStream[Int] data.addData(1, 2, 3) val stream = data.toDF() .writeStream .format("delta") .option("checkpointLocation", checkpoint) .start(path) stream.processAllAvailable() stream.stop() DeltaLog.forTable(spark, path).checkpoint() } /** Validate the generated data contains all [[Action]] types */ def validateData(spark: SparkSession, path: String): Unit = { import org.apache.spark.sql.delta.util.FileNames._ import scala.reflect.runtime.{universe => ru} import org.apache.spark.sql.delta.implicits._ val mirror = ru.runtimeMirror(this.getClass.getClassLoader) val tpe = ru.typeOf[Action] val clazz = tpe.typeSymbol.asClass assert(clazz.isSealed, s"${classOf[Action]} must be sealed") val deltaLog = DeltaLog.forTable(spark, new Path(path)) val deltas = 0L to deltaLog.snapshot.version val deltaFiles = deltas.map(unsafeDeltaFile(deltaLog.logPath, _)).map(_.toString) val actionsTypesInLog = spark.read.schema(Action.logSchema).json(deltaFiles: _*) .as[SingleAction] .collect() .map(_.unwrap.getClass.asInstanceOf[Class[_]]) .toSet val allActionTypes = clazz.knownDirectSubclasses .flatMap { case t if t == ru.typeOf[FileAction].typeSymbol => t.asClass.knownDirectSubclasses case t => Set(t) } .map(t => mirror.runtimeClass(t.asClass)) val missingTypes = allActionTypes -- actionsTypesInLog val unknownTypes = actionsTypesInLog -- allActionTypes assert( missingTypes.isEmpty, s"missing types: $missingTypes. " + "Please update EvolveabilitySuite.generateData to include them in the log.") assert( unknownTypes.isEmpty, s"unknown types: $unknownTypes. " + s"Please make sure they inherit ${classOf[Action]} or ${classOf[FileAction]} directly.") } /** Generate the transaction log with extra column in checkpoint and json. */ def generateTransactionLogWithExtraColumn(spark: SparkSession, path: String): Unit = { // scalastyle:off sparkimplicits import spark.implicits._ // scalastyle:on sparkimplicits implicit val s = spark.sqlContext val absPath = new File(path).getAbsolutePath (1 until 10).map(num => (num, num)).toDF("key", "value").write.format("delta").save(path) // Enable struct-only stats spark.sql(s"ALTER TABLE delta.`$absPath` " + s"SET TBLPROPERTIES (delta.checkpoint.writeStatsAsStruct = true, " + "delta.checkpoint.writeStatsAsJson = false)") (1 until 10).map(num => (num, num)).toDF("key", "value").write .format("delta").mode("overwrite").save(path) val deltaLog = DeltaLog.forTable(spark, new Path(path)) deltaLog.checkpoint() // Create an incomplete checkpoint without the action and overwrite the // original checkpoint val checkpointPath = FileNames.checkpointFileSingular(deltaLog.logPath, deltaLog.snapshot.version) val tmpCheckpoint = Utils.createTempDir() val checkpointDataWithNewCol = spark.read.parquet(checkpointPath.toString) .withColumn("unknown", lit(true)) // Keep the add files and also filter by the additional condition checkpointDataWithNewCol.coalesce(1).write .mode("overwrite").parquet(tmpCheckpoint.toString) val writtenCheckpoint = tmpCheckpoint.listFiles().toSeq.filter(_.getName.startsWith("part")).head val checkpointFile = new File(checkpointPath.toUri) new File(deltaLog.logPath.toUri).listFiles().toSeq.foreach { file => if (file.getName.startsWith(".0")) { // we need to delete checksum files, // otherwise trying to replace our incomplete // checkpoint file fails due to the LocalFileSystem's checksum checks. require(file.delete(), "Failed to delete checksum file") } } require(checkpointFile.delete(), "Failed to delete old checkpoint") require(writtenCheckpoint.renameTo(checkpointFile), "Failed to rename corrupt checkpoint") (1 until 10).map(num => (num, num)).toDF("key", "value").write .format("delta").mode("append").save(path) // Shouldn't fail here deltaLog.update() val version = deltaLog.snapshot.version // We want to have a delta log with a new column after a checkpoint, to test out operations // against both checkpoint with unknown column and delta log with unkown column. // manually remove AddFile in the previous commit and append a new column. val records = deltaLog.store.read( FileNames.unsafeDeltaFile(deltaLog.logPath, version), deltaLog.newDeltaHadoopConf()) val actions = records.map(Action.fromJson).filter(action => action.isInstanceOf[AddFile]) .map { action => action.asInstanceOf[AddFile].remove} .toIterator val recordsWithNewAction = actions.map(_.json) ++ Iterator("""{"some_new_action":{"a":1}}""") deltaLog.store.write( FileNames.unsafeDeltaFile(deltaLog.logPath, version + 1), recordsWithNewAction, overwrite = false, deltaLog.newDeltaHadoopConf()) // manually add those files back and add a unknown field to it. val newRecords = records.map{ record => val recordMap = JsonUtils.fromJson[Map[String, Any]](record) val newRecordMap = if (recordMap.contains("add")) { // add a unknown column inside action fields. val actionFields = recordMap("add").asInstanceOf[Map[String, Any]] + ("some_new_column_in_add_action" -> 1) recordMap + ("add" -> actionFields) } else recordMap // add a unknown column outside action fields. JsonUtils.toJson(newRecordMap + ("some_new_action_alongside_add_action" -> ("a" -> "1"))) }.toIterator deltaLog.store.write( FileNames.unsafeDeltaFile(deltaLog.logPath, version + 2), newRecords, overwrite = false, deltaLog.newDeltaHadoopConf()) // Shouldn't fail here deltaLog.update() DeltaLog.clearCache() } def main(args: Array[String]): Unit = { val spark = SparkSession.builder().master("local[2]").getOrCreate() val path = new File(args(0)) if (path.exists()) { // Don't delete automatically in case the user types a wrong path. // scalastyle:off throwerror throw new AssertionError(s"${path.getCanonicalPath} exists. Please delete it and retry.") // scalastyle:on throwerror } args(1) match { case "generateData" => generateData(spark, path.toString) validateData(spark, path.toString) case "generateTransactionLogWithExtraColumn" => generateTransactionLogWithExtraColumn(spark, path.toString) case _ => throw new RuntimeException("Unrecognized (or omitted) argument. " + "Please try again (no data generated).") } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/FakeFileSystem.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.net.URI import org.apache.hadoop.fs.RawLocalFileSystem /** A fake file system to test whether session Hadoop configuration will be picked up. */ class FakeFileSystem extends RawLocalFileSystem { override def getScheme: String = FakeFileSystem.scheme override def getUri: URI = FakeFileSystem.uri } object FakeFileSystem { val scheme = "fake" val uri = URI.create(s"$scheme:///") } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/FeatureEnablementConcurrencySuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.actions.{AddFile, Format, Metadata} import org.apache.spark.sql.delta.fuzzer.{OptimisticTransactionPhases, PhaseLockingTransactionExecutionObserver => TransactionObserver} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.util.{DeltaFileOperations, FileNames} import org.apache.hadoop.conf.Configuration import org.apache.parquet.hadoop.ParquetFileReader import org.apache.parquet.hadoop.metadata.ParquetMetadata import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.execution.datasources.parquet.ParquetReadSupport import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.ThreadUtils class FeatureEnablementConcurrencySuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with ConflictResolutionTestUtils { val testTableName = "test_feature_enablement_table" /** Represents a transaction that alters a table property. */ case class AlterTableProperty( property: String, value: String) extends TestTransaction(Map.empty) { override val name: String = s"ALTER TABLE($property $value)" override def dataChange: Boolean = false override def toSQL(tableName: String): String = { s"ALTER TABLE $tableName SET TBLPROPERTIES ('$property' = '$value')" } } /** Represents a transaction that unsets a table property. */ case class UnsetTableProperty(property: String) extends TestTransaction(Map.empty) { override val name: String = s"UNSET PROPERTY($property)" override def dataChange: Boolean = false override def toSQL(tableName: String): String = { s"ALTER TABLE $tableName UNSET TBLPROPERTIES ('$property')" } } private def createTestTable( properties: Seq[String] = Seq.empty, numPartitions: Int = 2): (DeltaLog, CatalogTable) = { sql(s"DROP TABLE IF EXISTS $testTableName") val propertiesString = if (properties.nonEmpty) properties.mkString(",") + "," else "" sql( s"""CREATE TABLE $testTableName (idCol bigint) |USING delta |TBLPROPERTIES ( |$propertiesString |'${DeltaConfigs.ROW_TRACKING_ENABLED.key}' = 'false', |'${DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key}' = 'false' |)""".stripMargin) spark.range(start = 0, end = 100, step = 1, numPartitions) .withColumnRenamed("id", "idCol") .write .format("delta") .mode("append") .saveAsTable(testTableName) val catalogTable = spark.sessionState.catalog.getTableMetadata( TableIdentifier(testTableName)) (DeltaLog.forTable(spark, catalogTable), catalogTable) } private def getParquetFooter(deltaLog: DeltaLog, file: AddFile): ParquetMetadata = { val dataPath = deltaLog.dataPath.toString val filePath = file.path val hadoopConf = new Configuration() val path = DeltaFileOperations.absolutePath(dataPath, filePath) val fileSystem = path.getFileSystem(hadoopConf) val fileStatus = fileSystem.listStatus(path).head ParquetFileReader.readFooter(hadoopConf, fileStatus) } private def validateFooter(footer: ParquetMetadata, expected: Boolean): Unit = { val footerMetadata = footer.getFileMetaData.getKeyValueMetaData assert(footerMetadata.containsKey(ParquetReadSupport.SPARK_METADATA_KEY)) val fieldMetadata = footerMetadata.get(ParquetReadSupport.SPARK_METADATA_KEY) assert( fieldMetadata.contains(DeltaColumnMapping.PARQUET_FIELD_ID_METADATA_KEY) === expected && fieldMetadata.contains(DeltaColumnMapping.COLUMN_MAPPING_METADATA_ID_KEY) === expected && fieldMetadata.contains(DeltaColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY) === expected) } test("Validate Metadata diff") { val metadataA = Metadata( id = "idA", name = "nameA", description = "descriptionA", format = Format(options = Map("OptionA" -> "A")), schemaString = "schemaA", partitionColumns = Seq("colA1", "colA2"), configuration = Map("propA1" -> "valueA1", "propA2" -> "valueA2"), createdTime = Some(1L)) val metadataB = Metadata( id = "idB", name = "nameB", description = "descriptionB", format = Format(), schemaString = "schemaB", partitionColumns = Seq("colB1"), configuration = Map("propB1" -> "valueB1"), createdTime = Some(2L)) // Diff should output all properties in correct order. val expectedDiff = Set( "id", "name", "description", "format", "schemaString", "partitionColumns", "configuration", "createdTime") assert(metadataA.diffFieldNames(metadataB) === expectedDiff, """The Metadata properties do not match the expected diff. |If you are extending Metadata please check Metadata.diff as well as |ConflictChecker.attemptToResolveMetadataConflicts""".stripMargin) val metadataA_1 = Metadata(id = "idA") val metadataB_1 = Metadata(id = "idB") assert(metadataA_1.diffFieldNames(metadataB_1) === Set("id")) val metadataA_2 = Metadata(id = "id", name = "nameA") val metadataB_2 = Metadata(id = "id", name = "nameB") assert(metadataA_2.diffFieldNames(metadataB_2) === Set("name")) val metadataA_3 = Metadata(id = "id", description = "descriptionA") val metadataB_3 = Metadata(id = "id", description = "descriptionB") assert(metadataA_3.diffFieldNames(metadataB_3) === Set("description")) val metadataA_4 = Metadata(id = "id", format = Format(options = Map("OptionA" -> "A"))) val metadataB_4 = Metadata(id = "id", format = Format()) assert(metadataA_4.diffFieldNames(metadataB_4) === Set("format")) val metadataA_5 = Metadata(id = "id", schemaString = "schemaA") val metadataB_5 = Metadata(id = "id", schemaString = "schemaB") assert(metadataA_5.diffFieldNames(metadataB_5) === Set("schemaString")) val metadataA_6 = Metadata(id = "id", partitionColumns = Seq("colA1")) val metadataB_6 = Metadata(id = "id", partitionColumns = Seq("colB1")) assert(metadataA_6.diffFieldNames(metadataB_6) === Set("partitionColumns")) val metadataA_7 = Metadata(id = "id", configuration = Map.empty) val metadataB_7 = Metadata(id = "id", configuration = Map("propB1" -> "valueB1")) assert(metadataA_7.diffFieldNames(metadataB_7) === Set("configuration")) val metadataA_8 = Metadata(id = "id", createdTime = Some(1L)) val metadataB_8 = Metadata(id = "id", createdTime = Some(2L)) assert(metadataA_8.diffFieldNames(metadataB_8) === Set("createdTime")) val metadataA_9 = Metadata(id = "idA", createdTime = Some(1L)) val metadataB_9 = Metadata(id = "idB", createdTime = Some(2L)) assert(metadataA_9.diffFieldNames(metadataB_9) === Set("id", "createdTime")) } test("checkConfigurationChangesForConflicts") { val (deltaLog, catalogTable) = createTestTable() val snapshot = deltaLog.update(catalogTableOpt = Some(catalogTable)) val dummyTransactionInfo = CurrentTransactionInfo( txnId = "txn 1", readPredicates = Vector.empty, readFiles = Set.empty, readWholeTable = false, readAppIds = Set.empty, metadata = Metadata(), protocol = snapshot.protocol, actions = Seq.empty[AddFile], readSnapshot = snapshot, commitInfo = None, readRowIdHighWatermark = 0L, catalogTable = Some(catalogTable), domainMetadata = Seq.empty, op = DeltaOperations.ManualUpdate) val lastVersion = snapshot.version val dummyCommit = deltaLog .getChangeLogFiles( startVersion = lastVersion, endVersion = lastVersion, catalogTableOpt = Some(catalogTable), failOnDataLoss = false) .map { case (_, file) => file } .filter(FileNames.isDeltaFile) .take(1) .toList .last val dummySummary = WinningCommitSummary.createFromFileStatus(deltaLog, dummyCommit) val conflictChecker = new ConflictChecker( spark, initialCurrentTransactionInfo = dummyTransactionInfo, winningCommitSummary = dummySummary, isolationLevel = WriteSerializable) // Test 1: Change 2 configs. One is allowed, the other is not. val current = Metadata(configuration = Map("prop1" -> "value1", "prop2" -> "value2")) val winning = Metadata(configuration = Map("prop1" -> "newValue1", "prop2" -> "newValue2")) val result1 = conflictChecker.checkConfigurationChangesForConflicts( current, winning, allowList = Set("prop1")) val expected1 = conflictChecker.ConfigurationChanges( areValid = false, changed = Map("prop1" -> "newValue1", "prop2" -> "newValue2")) assert(result1 === expected1) // Test 2: Change 2 configs. Both allowed. val result2 = conflictChecker.checkConfigurationChangesForConflicts( current, winning, allowList = Set("prop1", "prop2")) val expected2 = conflictChecker.ConfigurationChanges( areValid = true, changed = Map("prop1" -> "newValue1", "prop2" -> "newValue2")) assert(result2 === expected2) // Test 3: Same as previous but one property is added instead of changed. val result3 = conflictChecker.checkConfigurationChangesForConflicts( currentMetadata = Metadata(configuration = Map("prop1" -> "value1")), winningMetadata = Metadata(configuration = Map("prop1" -> "newValue1", "prop2" -> "newValue2")), allowList = Set("prop1", "prop2")) val expected3 = conflictChecker.ConfigurationChanges( areValid = true, added = Map("prop2" -> "newValue2"), changed = Map("prop1" -> "newValue1")) assert(result3 === expected3) // Test 4: Removals are not allowed. val result4 = conflictChecker.checkConfigurationChangesForConflicts( currentMetadata = Metadata(configuration = Map("prop1" -> "value1")), winningMetadata = Metadata(configuration = Map("prop2" -> "newValue2")), allowList = Set("prop1", "prop2")) val expected4 = conflictChecker.ConfigurationChanges( areValid = false, removed = Set("prop1"), added = Map("prop2" -> "newValue2")) assert(result4 === expected4) } def testFeatureDisablement(property: String, withUnset: Boolean): Unit = { val (deltaLog, _) = createTestTable() val ctx = new TestContext(deltaLog) AlterTableProperty(property, value = "true") .execute(ctx) val businessTxn = Delete(rows = Seq(90L)) val disableTxn = if (withUnset) { UnsetTableProperty(property) } else { AlterTableProperty(property, value = "false") } businessTxn.start(ctx) disableTxn.execute(ctx) val e = intercept[org.apache.spark.SparkException] { businessTxn.commit(ctx) } assert(e.getCause.asInstanceOf[DeltaThrowable].getErrorClass() === "DELTA_METADATA_CHANGED") } /* * -------------------------------------------> TIME -------------------------------------------> * * Row tracking Enablement: * protocol ---------- Unbackfill ------------------------ Metadata ------------ * Upgrade Batch 1 Update * prep+commit prep+commit prep+commit * * * Concurrent Txn prep commit * * -------------------------------------------> TIME -------------------------------------------> */ for { concurrentTxnName <- Seq("alterTableProperty", "delete") } test("Enable row tracking feature " + s"concurrent txn: $concurrentTxnName") { val (deltaLog, catalogTable) = createTestTable() val ctx = new TestContext(deltaLog) val enableFeatureFn = () => { AlterTableProperty(property = DeltaConfigs.ROW_TRACKING_ENABLED.key, value = "true") .execute(ctx) Array.empty[Row] } val Seq(enableFuture) = runFunctionsWithOrderingFromObserver(Seq(enableFeatureFn)) { case (protocolUpgradeObserver :: Nil) => val backfillObserver = new TransactionObserver( OptimisticTransactionPhases.forName("Backfill")) val metadataUpdateObserver = new TransactionObserver( OptimisticTransactionPhases.forName("Metadata Update")) protocolUpgradeObserver.setNextObserver(backfillObserver, autoAdvance = true) backfillObserver.setNextObserver(metadataUpdateObserver, autoAdvance = true) prepareAndCommitWithNextObserverSet(protocolUpgradeObserver) prepareAndCommitWithNextObserverSet(backfillObserver) val concurrentTxn = if (concurrentTxnName == "alterTableProperty") { AlterTableProperty( property = DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key, value = "true") } else { Delete(rows = Seq(90L)) } concurrentTxn.start(ctx) concurrentTxn .observer .foreach(o => busyWaitFor(o.phases.commitPhase.hasReached, timeout)) prepareAndCommit(metadataUpdateObserver) val expectException = concurrentTxnName == "alterTableProperty" if (expectException) { val e = intercept[org.apache.spark.SparkException] { concurrentTxn.commit(ctx) }.getCause.asInstanceOf[DeltaThrowable] assert(e.getErrorClass() === "DELTA_METADATA_CHANGED") } else { concurrentTxn.commit(ctx) } } ThreadUtils.awaitResult(enableFuture, timeout) assert(DeltaConfigs.ROW_TRACKING_ENABLED.fromMetaData( deltaLog.update(catalogTableOpt = Some(catalogTable)).metadata)) } for (withUnset <- BOOLEAN_DOMAIN) test(s"Disable row tracking feature - withUnset: $withUnset") { testFeatureDisablement(DeltaConfigs.ROW_TRACKING_ENABLED.key, withUnset) } test("Validate column metadata schema") { val (deltaLog, catalogTable) = createTestTable() val schema = deltaLog.update(catalogTableOpt = Some(catalogTable)).metadata.schema assert(schema.fields.head.productArity === 4, """ |Got a non expected field column arity. |If extending the StructField schema please check validateSchemaChanges. |""".stripMargin) } for (mode <- Seq(NameMapping, IdMapping)) test(s"Create table with column mapping - mode: ${mode.name}") { val (deltaLog, catalogTable) = createTestTable( properties = Seq(s"'${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = '${mode.name}'")) deltaLog.update(catalogTableOpt = Some(catalogTable)).allFiles.collect().foreach { addFile => val footer = getParquetFooter(deltaLog, addFile) validateFooter(footer, expected = true) } } for (txnInterleaved <- BOOLEAN_DOMAIN) test(s"Enable column mapping feature - txnInterleaved: $txnInterleaved") { val (deltaLog, catalogTable) = createTestTable() val ctx = new TestContext(deltaLog) val columnMappingEnablementTxn = AlterTableProperty( property = DeltaConfigs.COLUMN_MAPPING_MODE.key, value = NameMapping.name) val businessTxn = Update(rows = Seq(90L), setValue = -1L) if (txnInterleaved) { businessTxn.interleave(ctx) { columnMappingEnablementTxn.execute(ctx) } } else { columnMappingEnablementTxn.execute(ctx) } val metadata = deltaLog.update(catalogTableOpt = Some(catalogTable)).metadata assert(metadata.columnMappingMode === NameMapping) assert(metadata.schema.fields.map(_.metadata).forall { m => m.contains("delta.columnMapping.id") && m.contains("delta.columnMapping.physicalName") }) val tableDf = io.delta.tables.DeltaTable.forName(testTableName).toDF val expectedResult = if (txnInterleaved) { Seq.range(0L, 100L).filterNot(_ == 90L) :+ -1L } else { Seq.range(0L, 100L) } assert(tableDf.orderBy("idCol").collect() === expectedResult.sorted.map(Row(_))) // When column mapping is enabled on an existing table we do not expect any metadata in the // parquet footer. deltaLog.update(catalogTableOpt = Some(catalogTable)).allFiles.collect().foreach { addFile => val footer = getParquetFooter(deltaLog, addFile) validateFooter(footer, expected = false) } } for (startMode <- Seq(NameMapping, IdMapping, NoMapping)) test(s"Verify invalid column mapping transitions - startMode: ${startMode.name}") { val (deltaLog, _) = createTestTable( properties = Seq(s"'${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = '${startMode.name}'")) val ctx = new TestContext(deltaLog) // Not allowed transitions. val newMode = startMode match { case NameMapping => IdMapping case IdMapping => NameMapping case NoMapping => IdMapping } val e = intercept[DeltaColumnMappingUnsupportedException] { AlterTableProperty(property = DeltaConfigs.COLUMN_MAPPING_MODE.key, value = newMode.name) .execute(ctx) } checkError( e, "DELTA_UNSUPPORTED_COLUMN_MAPPING_MODE_CHANGE", parameters = Map("oldMode" -> startMode.name, "newMode" -> newMode.name)) } for (startMode <- Seq(NameMapping, IdMapping)) test(s"Removing column mapping mode produces conflict - startMode: ${startMode.name}") { val (deltaLog, catalogTable) = createTestTable( properties = Seq(s"'${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = '${startMode.name}'")) val ctx = new TestContext(deltaLog) val columnMappingDisablementTxn = AlterTableProperty( property = DeltaConfigs.COLUMN_MAPPING_MODE.key, value = NoMapping.name) val businessTxn = Delete(rows = Seq(90L)) businessTxn.start(ctx) columnMappingDisablementTxn.execute(ctx) val e = intercept[org.apache.spark.SparkException] { businessTxn.commit(ctx) } assert(e.getCause.asInstanceOf[DeltaThrowable].getErrorClass() === "DELTA_METADATA_CHANGED") assert(deltaLog.update( catalogTableOpt = Some(catalogTable)).metadata.columnMappingMode === NoMapping) } test("Column mapping enablement with RESTORE") { val (deltaLog, catalogTable) = createTestTable( properties = Seq(s"'${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = '${IdMapping.name}'")) val ctx = new TestContext(deltaLog) val columnMappingEnabledVersion = deltaLog.update(catalogTableOpt = Some(catalogTable)).version // Disable column mapping. AlterTableProperty(property = DeltaConfigs.COLUMN_MAPPING_MODE.key, value = NoMapping.name) .execute(ctx) // Cannot re-enable column mapping with RESTORE. val e = intercept[DeltaColumnMappingUnsupportedException] { sql(s"RESTORE TABLE $testTableName TO VERSION AS OF $columnMappingEnabledVersion") } checkError( e, "DELTA_UNSUPPORTED_COLUMN_MAPPING_MODE_CHANGE", parameters = Map("oldMode" -> NoMapping.name, "newMode" -> IdMapping.name)) } test("Enable deletion vectors feature") { val (deltaLog, _) = createTestTable() val ctx = new TestContext(deltaLog) val dvEnablementTxn = AlterTableProperty( property = DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key, value = "true") val businessTxn = Delete(rows = Seq(90L)) businessTxn.interleave(ctx) { dvEnablementTxn.execute(ctx) } assert(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.fromMetaData(deltaLog.update().metadata)) } for (withUnset <- BOOLEAN_DOMAIN) test(s"Disable Deletion Vectors feature - withUnset: $withUnset") { testFeatureDisablement(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key, withUnset) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/FileMetadataMaterializationTrackerSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.concurrent.Semaphore import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.scalatest.concurrent.TimeLimits import org.scalatest.time.{Seconds, Span} import org.apache.spark.SparkFunSuite import org.apache.spark.sql.test.SharedSparkSession class FileMetadataMaterializationTrackerSuite extends SparkFunSuite with TimeLimits with SharedSparkSession { test("tracker - unit test") { def acquireForTask(tracker: FileMetadataMaterializationTracker, numPermits: Int): Unit = { val taskLevelPermitAllocator = tracker.createTaskLevelPermitAllocator() for (i <- 1 to numPermits) { taskLevelPermitAllocator.acquirePermit() } } // Initialize the semaphore for tests val totalAvailablePermits = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_COMMAND_FILE_MATERIALIZATION_LIMIT) val semaphore = new Semaphore(totalAvailablePermits) FileMetadataMaterializationTracker.initializeSemaphoreForTests(semaphore) val tracker = new FileMetadataMaterializationTracker() // test that acquiring a permit should work and decrement the available permits. acquireForTask(tracker, 1) assert(semaphore.availablePermits() === totalAvailablePermits - 1) // releasing the permit should increment the semaphore's count tracker.releasePermits(1) assert(semaphore.availablePermits() === totalAvailablePermits) // test overallocation acquireForTask(tracker, totalAvailablePermits + 1) // allowed to over allocate assert(semaphore.availablePermits() === 0) assert(semaphore.availablePermits() === 0) tracker.releasePermits(totalAvailablePermits + 1) assert(semaphore.availablePermits() === totalAvailablePermits) // make sure we don't overflow // test - wait for other task to release overallocation lock acquireForTask(tracker, totalAvailablePermits + 1) val acquireThread = new Thread() { override def run(): Unit = { val taskLevelPermitAllocator = tracker.createTaskLevelPermitAllocator() taskLevelPermitAllocator.acquirePermit() } } // we acquire in a separate thread so that we can make sure the acquiring is blocked // until another thread(main thread here) releases a permit. acquireThread.start() Thread.sleep(2000) // Sleep for 2 seconds to make sure the acquireThread is blocked assert(acquireThread.isAlive) // acquire thread is actually blocked tracker.releasePermits(totalAvailablePermits + 1) failAfter(Span(2, Seconds)) { acquireThread.join() // acquire thread should get unblocked } // test releaseAllPermits assert(semaphore.availablePermits() === totalAvailablePermits - 1) tracker.releaseAllPermits() assert(semaphore.availablePermits() === totalAvailablePermits) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/FileNamesSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.hadoop.fs.Path import org.apache.spark.SparkFunSuite class FileNamesSuite extends SparkFunSuite { import org.apache.spark.sql.delta.util.FileNames._ test("isDeltaFile") { assert(isDeltaFile(new Path("/a/_delta_log/123.json"))) assert(isDeltaFile(new Path("/a/123.json"))) assert(!isDeltaFile(new Path("/a/_delta_log/123ajson"))) assert(!isDeltaFile(new Path("/a/_delta_log/123.jso"))) assert(!isDeltaFile(new Path("/a/_delta_log/123a.json"))) assert(!isDeltaFile(new Path("/a/_delta_log/a123.json"))) // UUID Files assert(!isDeltaFile(new Path("/a/123.uuid.json"))) assert(!isDeltaFile(new Path("/a/_delta_log/123.uuid.json"))) assert(isDeltaFile(new Path("/a/_delta_log/_staged_commits/123.uuid.json"))) assert(!isDeltaFile(new Path("/a/_delta_log/_staged_commits/123.uuid1.uuid2.json"))) } test("DeltaFile.unapply") { assert(DeltaFile.unapply(new Path("/a/_delta_log/123.json")) === Some((new Path("/a/_delta_log/123.json"), 123))) assert(DeltaFile.unapply(new Path("/a/123.json")) === Some((new Path("/a/123.json"), 123))) assert(DeltaFile.unapply(new Path("/a/_delta_log/123ajson")).isEmpty) assert(DeltaFile.unapply(new Path("/a/_delta_log/123.jso")).isEmpty) assert(DeltaFile.unapply(new Path("/a/_delta_log/123a.json")).isEmpty) assert(DeltaFile.unapply(new Path("/a/_delta_log/a123.json")).isEmpty) // UUID Files assert(DeltaFile.unapply(new Path("/a/123.uuid.json")).isEmpty) assert(DeltaFile.unapply(new Path("/a/_delta_log/123.uuid.json")).isEmpty) assert(DeltaFile.unapply(new Path("/a/_delta_log/_staged_commits/123.uuid.json")) === Some((new Path("/a/_delta_log/_staged_commits/123.uuid.json"), 123))) assert(DeltaFile.unapply( new Path("/a/_delta_log/_staged_commits/123.uuid1.uuid2.json")).isEmpty) } test("isCheckpointFile") { assert(isCheckpointFile(new Path("/a/123.checkpoint.parquet"))) assert(isCheckpointFile(new Path("/a/123.checkpoint.0000000001.0000000087.parquet"))) assert(!isCheckpointFile(new Path("/a/123.json"))) } test("checkpointVersion") { assert(checkpointVersion(new Path("/a/123.checkpoint.parquet")) == 123) assert(checkpointVersion(new Path("/a/0.checkpoint.parquet")) == 0) assert(checkpointVersion(new Path("/a/00000000000000000151.checkpoint.parquet")) == 151) assert(checkpointVersion(new Path("/a/999.checkpoint.0000000090.0000000099.parquet")) == 999) } test("listingPrefix") { assert(listingPrefix(new Path("/a"), 1234) == new Path("/a/00000000000000001234.")) } test("checkpointFileWithParts") { assert(checkpointFileWithParts(new Path("/a"), 1, 1) == Seq( new Path("/a/00000000000000000001.checkpoint.0000000001.0000000001.parquet"))) assert(checkpointFileWithParts(new Path("/a"), 1, 2) == Seq( new Path("/a/00000000000000000001.checkpoint.0000000001.0000000002.parquet"), new Path("/a/00000000000000000001.checkpoint.0000000002.0000000002.parquet"))) assert(checkpointFileWithParts(new Path("/a"), 1, 5) == Seq( new Path("/a/00000000000000000001.checkpoint.0000000001.0000000005.parquet"), new Path("/a/00000000000000000001.checkpoint.0000000002.0000000005.parquet"), new Path("/a/00000000000000000001.checkpoint.0000000003.0000000005.parquet"), new Path("/a/00000000000000000001.checkpoint.0000000004.0000000005.parquet"), new Path("/a/00000000000000000001.checkpoint.0000000005.0000000005.parquet"))) } test("numCheckpointParts") { assert(numCheckpointParts(new Path("/a/00000000000000000099.checkpoint.parquet")).isEmpty) assert( numCheckpointParts( new Path("/a/00000000000000000099.checkpoint.0000000078.0000000092.parquet")) .contains(92)) } test("commitDirPath") { assert(commitDirPath(logPath = new Path("/a/_delta_log")) === new Path("/a/_delta_log/_staged_commits")) assert(commitDirPath(logPath = new Path("/a/_delta_log/")) === new Path("/a/_delta_log/_staged_commits")) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/FindLastCompleteCheckpointSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta.CheckpointInstance.Format import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.coordinatedcommits.CoordinatedCommitsBaseSuite import org.apache.spark.sql.delta.storage.LocalLogStore import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.SparkConf import org.apache.spark.sql.QueryTest import org.apache.spark.sql.test.SharedSparkSession class FindLastCompleteCheckpointSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with CoordinatedCommitsBaseSuite { protected override def sparkConf: SparkConf = { super.sparkConf .set("spark.delta.logStore.class", classOf[CustomListingLogStore].getName) } private def pathToFileStatus(path: Path, length: Long = 20): FileStatus = { SerializableFileStatus(path.toString, length, isDir = true, modificationTime = 0L).toFileStatus } private def commitFiles(logPath: Path, versions: Seq[Long]): Seq[FileStatus] = { versions.map { version => pathToFileStatus(FileNames.unsafeDeltaFile(logPath, version)) } } private def singleCheckpointFiles( logPath: Path, versions: Seq[Long], length: Long = 20): Seq[FileStatus] = { versions.map { v => pathToFileStatus(FileNames.checkpointFileSingular(logPath, v), length) } } private def multipartCheckpointFiles( logPath: Path, versions: Seq[Long], numParts: Int, length: Long = 20): Seq[FileStatus] = { versions.flatMap { version => FileNames.checkpointFileWithParts(logPath, version, numParts).map(pathToFileStatus(_, length)) } } private def checksumFiles(logPath: Path, versions: Seq[Long]): Seq[FileStatus] = { versions.map { version => pathToFileStatus(FileNames.checksumFile(logPath, version)) } } def getLastCompleteCheckpointUsageLog(f: => Unit): Map[String, String] = { val usageRecords = Log4jUsageLogger.track { f } val opType = "delta.findLastCompleteCheckpointBefore" val records = usageRecords.filter { r => r.tags.get("opType").contains(opType) || r.opType.map(_.typeName).contains(opType) } assert(records.size === 1) JsonUtils.fromJson[Map[String, String]](records.head.blob) } test("findLastCompleteCheckpoint without any argument") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir.getAbsolutePath) val logPath = log.logPath val logStore = log.store.asInstanceOf[CustomListingLogStore] // Case-1: Multiple checkpoint exists in table dir logStore.customListingResult = Some( commitFiles(logPath, 0L to 3000) ++ singleCheckpointFiles(logPath, Seq(100, 200, 1000, 2000)) ) val eventData1 = getLastCompleteCheckpointUsageLog { assert(log.findLastCompleteCheckpointBefore().contains(CheckpointInstance(version = 2000))) } assert(!eventData1.contains("iterations")) assert(logStore.listFromCount == 1) assert(logStore.elementsConsumedFromListFromIter == 3005) logStore.reset() // Case-2: No checkpoint exists in table dir logStore.customListingResult = Some(commitFiles(logPath, 0L to 3000)) val eventData2 = getLastCompleteCheckpointUsageLog { assert(log.findLastCompleteCheckpointBefore().isEmpty) } assert(!eventData2.contains("iterations")) assert(logStore.listFromCount == 1) assert(logStore.elementsConsumedFromListFromIter == 3001) logStore.reset() // Case-3: Multiple checkpoints for same version exists in table dir logStore.customListingResult = Some( commitFiles(logPath, 0L to 3000) ++ singleCheckpointFiles(logPath, Seq(100, 200, 1000, 2000)) ++ multipartCheckpointFiles(logPath, Seq(300, 2000), numParts = 4) ) val eventData3 = getLastCompleteCheckpointUsageLog { assert(log.findLastCompleteCheckpointBefore().contains( CheckpointInstance(version = 2000, Format.WITH_PARTS, numParts = Some(4)))) } assert(!eventData2.contains("iterations")) assert(logStore.listFromCount == 1) assert(logStore.elementsConsumedFromListFromIter == 3013) logStore.reset() } } test("findLastCompleteCheckpoint with an upperBound which exists") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir.getAbsolutePath) val logPath = log.logPath val logStore = log.store.asInstanceOf[CustomListingLogStore] logStore.reset() // Case-1: The upperBound exists and it should not be returned logStore.customListingResult = Some( commitFiles(logPath, 0L to 3000) ++ singleCheckpointFiles(logPath, Seq(100, 200, 1000, 2000)) ) val eventData1 = getLastCompleteCheckpointUsageLog { assert( log.findLastCompleteCheckpointBefore(Some(CheckpointInstance(version = 2000))) .contains(CheckpointInstance(version = 1000))) } assert(logStore.listFromCount == 1) assert(logStore.elementsConsumedFromListFromIter == 1002 + 2) // commits + checkpoint assert(eventData1("iterations") == "1") assert(eventData1("numFilesScanned") == "1004") logStore.reset() // Case-2: The exact upperBound (a multi-part checkpoint) doesn't exist but another single // part checkpoint for same version exists. logStore.customListingResult = Some( commitFiles(logPath, 0L to 3000) ++ singleCheckpointFiles(logPath, Seq(100, 200, 1000, 2000)) ) var sentinelCheckpoint = CheckpointInstance(version = 2000, Format.WITH_PARTS, numParts = Some(4)) val eventData2 = getLastCompleteCheckpointUsageLog { assert(log.findLastCompleteCheckpointBefore(Some(sentinelCheckpoint)) .contains(CheckpointInstance(version = 2000))) } assert(logStore.listFromCount == 1) assert(logStore.elementsConsumedFromListFromIter == 1002 + 2) // commits + checkpoint assert(eventData2("iterations") == "1") assert(eventData2("numFilesScanned") == "1004") logStore.reset() // Case-3: The last complete checkpoint doesn't exist in last 1000 elements and needs // multiple iterations. logStore.customListingResult = Some( commitFiles(logPath, 0L to 2500) ++ singleCheckpointFiles(logPath, Seq(100, 150)) ) val eventData3 = getLastCompleteCheckpointUsageLog { assert( log.findLastCompleteCheckpointBefore(2200).contains(CheckpointInstance(version = 150))) } assert(logStore.listFromCount == 3) // the first listing will consume 1000 elements from 1200 to 2201 => 1002 commits // the second listing will consume 1000 elements from 200 to 1201 => 1002 commits // the third listing will consume 501 elements from 0 to 201 => 202 commits + 2 checkpoints assert(logStore.elementsConsumedFromListFromIter == 2208) // commits + checkpoint assert(eventData3("iterations") == "3") assert(eventData3("numFilesScanned") == "2208") logStore.reset() } } for (passSentinelInstance <- BOOLEAN_DOMAIN) test("findLastCompleteCheckpoint ignores 0B files " + s"[passSentinelInstance: $passSentinelInstance]") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir.getAbsolutePath) val logPath = log.logPath val logStore = log.store.asInstanceOf[CustomListingLogStore] logStore.reset() val lastCommitVersion = 1400 val sentinelInstance = if (passSentinelInstance) Some(CheckpointInstance(version = 1200)) else None val expectedListCount = if (passSentinelInstance) 2 else 1 def getExpectedFileCount(filesPerCheckpoint: Int): Int = { if (passSentinelInstance) { // commits and checkpoints from 200 to 1201 => 1002 + 1-checkpoint // commits and checkpoints from 0 to 201 => 202 + 2-checkpoint 1204 + 3 * filesPerCheckpoint } else { val totalCommits = lastCommitVersion + 1 // commit starts from 0 totalCommits + 2 * filesPerCheckpoint } } // Case-1: `findLastCompleteCheckpointBefore` invoked without upperBound, with 0B single part // checkpoint. logStore.customListingResult = Some( commitFiles(logPath, 0L to lastCommitVersion) ++ singleCheckpointFiles(logPath, Seq(100), length = 20) ++ singleCheckpointFiles(logPath, Seq(200), length = 0)) val eventData1 = getLastCompleteCheckpointUsageLog { assert( log.findLastCompleteCheckpointBefore(sentinelInstance) .contains(CheckpointInstance(version = 100))) } assert(logStore.listFromCount == expectedListCount) assert(logStore.elementsConsumedFromListFromIter === getExpectedFileCount(filesPerCheckpoint = 1)) if (passSentinelInstance) { assert(eventData1("iterations") == expectedListCount.toString) assert(eventData1("numFilesScanned") == getExpectedFileCount(filesPerCheckpoint = 1).toString) } else { assert(Seq("iterations", "numFilesScanned").forall(!eventData1.contains(_))) } logStore.reset() // Case-2: `findLastCompleteCheckpointBefore` invoked with upperBound, with a multi-part // checkpoint having one of the part as 0B. val badCheckpointV200 = { val checkpointV200 = multipartCheckpointFiles(logPath, Seq(200), numParts = 4) SerializableFileStatus.fromStatus(checkpointV200.head).copy(length = 0).toFileStatus +: checkpointV200.tail } logStore.customListingResult = Some( commitFiles(logPath, 0L to lastCommitVersion) ++ multipartCheckpointFiles(logPath, Seq(100), numParts = 4) ++ badCheckpointV200 ) val eventData2 = getLastCompleteCheckpointUsageLog { assert(log.findLastCompleteCheckpointBefore(sentinelInstance) .contains(CheckpointInstance(version = 100, Format.WITH_PARTS, numParts = Some(4)))) } if (passSentinelInstance) { assert(eventData2("iterations") == expectedListCount.toString) assert(eventData2("numFilesScanned") == getExpectedFileCount(filesPerCheckpoint = 4).toString) } else { assert(Seq("iterations", "numFilesScanned").forall(!eventData2.contains(_))) } assert(logStore.listFromCount == expectedListCount) assert(logStore.elementsConsumedFromListFromIter === getExpectedFileCount(filesPerCheckpoint = 4)) logStore.reset() } } for (passSentinelInstance <- BOOLEAN_DOMAIN) test("findLastCompleteCheckpoint ignores incomplete multi-part checkpoint " + s"[passSentinelInstance: $passSentinelInstance]") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir.getAbsolutePath) val logPath = log.logPath val logStore = log.store.asInstanceOf[CustomListingLogStore] logStore.reset() val lastCommitVersion = 1400 val sentinelInstance = if (passSentinelInstance) Some(CheckpointInstance(version = 1200)) else None val expectedListCount = if (passSentinelInstance) 2 else 1 def getExpectedFileCount(fileInCheckpointV200: Int, filesInCheckpointV100: Int): Int = { if (passSentinelInstance) { // commits and checkpoints from 200 to 1201 => 1002 + 1-checkpoint // commits and checkpoints from 0 to 201 => 202 + 2-checkpoint 1204 + (fileInCheckpointV200 + fileInCheckpointV200 + filesInCheckpointV100) } else { val totalCommits = lastCommitVersion + 1 // commit starts from 0 totalCommits + (fileInCheckpointV200 + filesInCheckpointV100) } } // Case-1: `findLastCompleteCheckpointBefore` invoked, with 0B single part checkpoint. logStore.customListingResult = Some( commitFiles(logPath, 0L to lastCommitVersion) ++ multipartCheckpointFiles(logPath, Seq(100), numParts = 4, length = 20) ++ multipartCheckpointFiles(logPath, Seq(200), numParts = 4, length = 20).take(3)) val eventData1 = getLastCompleteCheckpointUsageLog { assert( log.findLastCompleteCheckpointBefore(sentinelInstance) .contains(CheckpointInstance(100, Format.WITH_PARTS, numParts = Some(4)))) } assert(logStore.listFromCount == expectedListCount) assert(logStore.elementsConsumedFromListFromIter === getExpectedFileCount(fileInCheckpointV200 = 3, filesInCheckpointV100 = 4)) if (passSentinelInstance) { assert(eventData1("iterations") == expectedListCount.toString) assert(eventData1("numFilesScanned") == getExpectedFileCount(fileInCheckpointV200 = 3, filesInCheckpointV100 = 4).toString) } else { assert(Seq("iterations", "numFilesScanned").forall(!eventData1.contains(_))) } logStore.reset() } } test("findLastCompleteCheckpoint with CheckpointInstance.MAX value") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir.getAbsolutePath) val logPath = log.logPath val logStore = log.store.asInstanceOf[CustomListingLogStore] logStore.reset() logStore.customListingResult = Some( commitFiles(logPath, 0L to 3000) ++ singleCheckpointFiles(logPath, Seq(100, 200, 1000, 1200)) ) val eventData = getLastCompleteCheckpointUsageLog { assert( log.findLastCompleteCheckpointBefore(Some(CheckpointInstance.MaxValue)) .contains(CheckpointInstance(version = 1200))) } assert(!eventData.contains("iterations")) assert(!eventData.contains("upperBoundVersion")) assert(eventData("totalTimeTakenMs").toLong > 0) assert(logStore.listFromCount == 1) assert(logStore.elementsConsumedFromListFromIter == 3001 + 4) // commits + checkpoint logStore.reset() } } } /** * A custom log store that allows to provide custom listing results. This is useful to test * `DeltaLog.findLastCompleteCheckpointBefore` method. */ class CustomListingLogStore( sparkConf: SparkConf, hadoopConf: Configuration) extends LocalLogStore(sparkConf, hadoopConf) { var listFromCount = 0 var elementsConsumedFromListFromIter = 0 // The custom listing result that will be returned by `listFrom` method. If this is None, then // the default listing result from the actual filesystem will be returned. var customListingResult: Option[Seq[FileStatus]] = None override def listFrom(path: Path, hadoopConf: Configuration): Iterator[FileStatus] = { customListingResult.map { results => listFromCount += 1 results .sortBy(_.getPath) .dropWhile(_.getPath.toString < path.toString) .toIterator .map { file => elementsConsumedFromListFromIter += 1 file } }.getOrElse(super.listFrom(path, hadoopConf)) } def reset(): Unit = { listFromCount = 0 elementsConsumedFromListFromIter = 0 customListingResult = None } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/GenerateIdentityValuesSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta.IdentityColumn.IdentityInfo import org.apache.spark.SparkException import org.apache.spark.sql.{Column, DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.expressions.{GreaterThan, If, Literal} import org.apache.spark.sql.functions._ import org.apache.spark.sql.test.SharedSparkSession class GenerateIdentityValuesSuite extends QueryTest with SharedSparkSession { import testImplicits._ private val colName = "id" /** * Verify the generated IDENTITY values are correct. * * @param df A DataFrame with a single column containing all the generated IDENTITY values. * @param identityInfo IDENTITY information used for verification. * @param rowCount Expected row count. */ private def verifyIdentityValues( df: => DataFrame, identityInfo: IdentityInfo, rowCount: Long): Unit = { // Check row count is expected. checkAnswer(df.select(count(col(colName))), Row(rowCount)) // Check there is no duplicate. checkAnswer(df.select(count_distinct(Column(colName))), Row(rowCount)) // Check every value follows start and step configuration val condViolateConfig = s"($colName - ${identityInfo.start}) % ${identityInfo.step} != 0" checkAnswer(df.where(condViolateConfig), Seq.empty) // Check every value is after high watermark OR >= start. val highWaterMark = identityInfo.highWaterMark.getOrElse(identityInfo.start - identityInfo.step) val condViolateHighWaterMark = s"(($colName - $highWaterMark)/${identityInfo.step}) < 0" checkAnswer(df.where(condViolateHighWaterMark), Seq.empty) // When high watermark is empty, the first value should be start. if (identityInfo.highWaterMark.isEmpty) { val agg = if (identityInfo.step > 0) min(Column(colName)) else max(Column(colName)) checkAnswer(df.select(agg), Row(identityInfo.start)) } } test("basic") { val sizes = Seq(100, 1000, 10000) val slices = Seq(2, 7, 15) val starts = Seq(-3, 0, 1, 5, 43) val steps = Seq(-3, -2, -1, 1, 2, 3) for (size <- sizes; slice <- slices; start <- starts; step <- steps) { val highWaterMarks = Seq(None, Some((start + 100 * step).toLong)) val df = spark.range(1, size + 1, 1, slice).toDF(colName) highWaterMarks.foreach { highWaterMark => verifyIdentityValues( df.select(Column(GenerateIdentityValues(start, step, highWaterMark)).alias(colName)), IdentityInfo(start, step, highWaterMark), size ) } } } test("shared state") { val size = 10000 val slice = 7 val start = -1 val step = 3 val highWaterMarks = Seq(None, Some((start + 100 * step).toLong)) val df = spark.range(1, size + 1, 1, slice).toDF(colName) highWaterMarks.foreach { highWaterMark => // Create two GenerateIdentityValues expressions that share the same state. They should // generate distinct values. val gev = GenerateIdentityValues(start, step, highWaterMark) val gev2 = gev.copy() verifyIdentityValues( df.select(Column( If(GreaterThan(col(colName).expr, right = Literal(10)), gev, gev2)).alias(colName)), IdentityInfo(start, step, highWaterMark), size ) } } test("bigint value range") { val size = 1000 val slice = 32 val start = Integer.MAX_VALUE.toLong + 1 val step = 10 val highWaterMark = start - step val df = spark.range(1, size + 1, 1, slice).toDF(colName) verifyIdentityValues( df.select( Column(GenerateIdentityValues(start, step, Some(highWaterMark))).alias(colName)), IdentityInfo(start, step, Some(highWaterMark)), size ) } test("overflow initial value") { val events = Log4jUsageLogger.track { val df = spark.range(1, 10, 1, 5).toDF(colName) .select(Column(GenerateIdentityValues( start = 2, step = Long.MaxValue, highWaterMarkOpt = Some(2 - Long.MaxValue)))) val ex = intercept[SparkException] { df.collect() } assert(ex.getMessage.contains("java.lang.ArithmeticException: long overflow")) } val filteredEvents = events.filter { e => e.tags.get("opType").exists(_ == "delta.identityColumn.overflow") } assert(filteredEvents.size > 0) } test("overflow next") { val events = Log4jUsageLogger.track { val df = spark.range(1, 10, 1, 5).toDF(colName) .select(Column(GenerateIdentityValues( start = Long.MaxValue - 1, step = 2, highWaterMarkOpt = Some(Long.MaxValue - 3)))) val ex = intercept[SparkException] { df.collect() } assert(ex.getMessage.contains("java.lang.ArithmeticException: long overflow")) } val filteredEvents = events.filter { e => e.tags.get("opType").exists(_ == "delta.identityColumn.overflow") } assert(filteredEvents.size > 0) } test("invalid high water mark") { val df = spark.range(1, 10, 1, 5).toDF(colName) intercept[IllegalArgumentException] { df.select(Column(GenerateIdentityValues( start = 1, step = 2, highWaterMarkOpt = Some(4))) ).collect() } } test("invalid step") { val df = spark.range(1, 10, 1, 5).toDF(colName) intercept[IllegalArgumentException] { df.select(Column(GenerateIdentityValues( start = 1, step = 0, highWaterMarkOpt = Some(4))) ).collect() } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/GeneratedColumnCompatibilitySuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import org.apache.spark.sql.delta.actions.Protocol import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.commons.io.FileUtils import org.apache.spark.sql.AnalysisException /** * We store the generation expressions in column's metadata. As Spark will propagate column metadata * to downstream operations when reading a table, old versions may create tables that have * generation expressions with an old writer version. For such tables, this test suite will verify * it behaves as a normal table. In other words, the generation expressions should be ignored in * new versions that understand generated columns so that all versions will have the same behaviors. */ class GeneratedColumnCompatibilitySuite extends GeneratedColumnTest { import GeneratedColumn._ import testImplicits._ /** * This test uses a special table generated by the following steps: * * 1. Run the following command using DBR 8.1 to generate a generated column table. * * ``` * spark.sql("""CREATE TABLE generated_columns_table( * |c1 INT, * |c2 INT GENERATED ALWAYS AS ( c1 + 1 ) * |) USING DELTA * |LOCATION 'sql/core/src/test/resources/delta/dbr_8_1_generated_columns' * |""".stripMargin) * ``` * * 2. Run the following command using DBR 8.0 to read the above table and create a new one. * * ``` * spark.sql("""CREATE TABLE delta_non_generated_columns * |USING DELTA * |LOCATION 'sql/core/src/test/resources/delta/dbr_8_0_non_generated_columns' * |AS SELECT * FROM * |delta.`sql/core/src/test/resources/delta/dbr_8_1_generated_columns` * |""".stripMargin) * ``` * * Now the schema of `dbr_8_0_non_generated_columns` will contain generation expressions but it * has an old writer version. This test will verify this test is treated as a non generated column * table, which means new versions will have the exact behaviors as the old versions when reading * or writing this table. */ def withDBR8_0Table(func: String => Unit): Unit = { val resourcePath = "src/test/resources/delta/dbr_8_0_non_generated_columns" withTempDir { tempDir => // Prepare a table that has the old writer version and generation expressions FileUtils.copyDirectory(new File(resourcePath), tempDir) val path = tempDir.getCanonicalPath val deltaLog = DeltaLog.forTable(spark, path) // Verify the test table has the old writer version and generation expressions assert(hasGeneratedColumns(deltaLog.snapshot.metadata.schema)) assert(!enforcesGeneratedColumns(deltaLog.snapshot.protocol, deltaLog.snapshot.metadata)) func(path) } } test("dbr 8_0") { withDBR8_0Table { path => withTempDir { normalTableDir => // Prepare a normal table val normalTablePath = normalTableDir.getCanonicalPath spark.sql( s"""CREATE TABLE generated_columns_table( |c1 INT, |c2 INT |) USING DELTA |LOCATION '$normalTablePath' |""".stripMargin) // Now we are going to verify commands on `path` and `normalTablePath` should be the same. // Update `path` and `normalTablePath` using the same func and verify they have the // same result def updateTableAndCheckAnswer(func: String => Unit): Unit = { func(path) func(normalTablePath) checkAnswer( spark.read.format("delta").load(path), spark.read.format("delta").load(normalTablePath) ) } // Insert values that violate the generation expression should be okay because the table // should not be treated as a generated column table. updateTableAndCheckAnswer { tablePath => sql(s"INSERT INTO delta.`$tablePath`VALUES(1, 10)") } updateTableAndCheckAnswer { tablePath => sql(s"INSERT INTO delta.`$tablePath`(c2, c1) VALUES(11, 1)") } updateTableAndCheckAnswer { tablePath => sql(s"INSERT OVERWRITE delta.`$tablePath`VALUES(1, 13)") } updateTableAndCheckAnswer { tablePath => sql(s"INSERT OVERWRITE delta.`$tablePath`(c2, c1) VALUES(14, 1)") } updateTableAndCheckAnswer { tablePath => // Append (1, null) to the table Seq(1).toDF("c1").write.format("delta").mode("append").save(tablePath) } updateTableAndCheckAnswer { tablePath => Seq(1 -> 15).toDF("c1", "c2").write.format("delta").mode("append").save(tablePath) } updateTableAndCheckAnswer { tablePath => // Overwrite the table with (2, null) Seq(2).toDF("c1").write.format("delta").mode("overwrite").save(tablePath) } } } } test("adding a new column should not enable generated columns") { withDBR8_0Table { path => val deltaLog = DeltaLog.forTable(spark, path) val protocolBeforeUpdate = deltaLog.snapshot.protocol sql(s"ALTER TABLE delta.`$path` ADD COLUMNS (c3 INT)") deltaLog.update() // The generation expressions should be dropped assert(!hasGeneratedColumns(deltaLog.snapshot.metadata.schema)) assert(deltaLog.snapshot.protocol == protocolBeforeUpdate) assert(!enforcesGeneratedColumns(deltaLog.snapshot.protocol, deltaLog.snapshot.metadata)) } } test("specifying a min writer version should not enable generated column") { withDBR8_0Table { path => val deltaLog = DeltaLog.forTable(spark, path) sql(s"ALTER TABLE delta.`$path` SET TBLPROPERTIES ('delta.minWriterVersion'='4')") deltaLog.update() // The generation expressions should be dropped assert(!hasGeneratedColumns(deltaLog.snapshot.metadata.schema)) assert(deltaLog.snapshot.protocol == Protocol(1, 4)) assert(!enforcesGeneratedColumns(deltaLog.snapshot.protocol, deltaLog.snapshot.metadata)) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/GeneratedColumnSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off typedlit import java.sql.{Date, Timestamp} import java.util.UUID import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.actions.Protocol import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.schema.{DeltaInvariantViolationException, InvariantViolationException} import org.apache.spark.sql.delta.sources.DeltaSourceUtils.GENERATION_EXPRESSION_METADATA_KEY import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.sources.DeltaSQLConf.GeneratedColumnValidateOnWriteMode import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.test.shims.StreamingTestShims.MemoryStream import org.apache.spark.sql.delta.util.FileNames import org.apache.spark.sql.{AnalysisException, Column, DataFrame, Dataset, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.util.quietly import org.apache.spark.sql.functions.{lit, make_dt_interval, struct, typedLit} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.streaming.{StreamingQueryException, Trigger} import org.apache.spark.sql.types.{ArrayType, DataType, DateType, IntegerType, LongType, MetadataBuilder, ShortType, StringType, StructField, StructType, TimestampType} trait GeneratedColumnSuiteBase extends GeneratedColumnTest { import GeneratedColumn._ import testImplicits._ protected def replaceTable( tableName: String, path: Option[String], schemaString: String, generatedColumns: Map[String, String], partitionColumns: Seq[String], notNullColumns: Set[String] = Set.empty, comments: Map[String, String] = Map.empty, properties: Map[String, String] = Map.empty, orCreate: Option[Boolean] = None): Unit = { var tableBuilder = if (orCreate.getOrElse(false)) { io.delta.tables.DeltaTable.createOrReplace(spark) } else { io.delta.tables.DeltaTable.replace(spark) } buildTable(tableBuilder, tableName, path, schemaString, generatedColumns, partitionColumns, notNullColumns, comments, properties).execute() } // Define the information for a default test table used by many tests. protected val defaultTestTableSchema = "c1 bigint, c2_g bigint, c3_p string, c4_g_p date, c5 timestamp, c6 int, c7_g_p int, c8 date" protected val defaultTestTableGeneratedColumns = Map( "c2_g" -> "c1 + 10", "c4_g_p" -> "cast(c5 as date)", "c7_g_p" -> "c6 * 10" ) protected val defaultTestTablePartitionColumns = "c3_p, c4_g_p, c7_g_p".split(", ").toList protected def createDefaultTestTable(tableName: String, path: Option[String] = None): Unit = { createTable( tableName, path, defaultTestTableSchema, defaultTestTableGeneratedColumns, defaultTestTablePartitionColumns ) } /** * @param updateFunc A function that's called with the table information (tableName, path). It * should execute update operations, and return the expected data after * updating. */ protected def testTableUpdate( testName: String, isStreaming: Boolean = false)(updateFunc: (String, String) => Seq[Row]): Unit = { def testBody(): Unit = { val table = testName withTempDir { path => withTable(table) { createDefaultTestTable(tableName = table, path = Some(path.getCanonicalPath)) val expected = updateFunc(testName, path.getCanonicalPath) checkAnswer(sql(s"select * from $table"), expected) } } } if (isStreaming) { test(testName) { testBody() } } else { test(testName) { testBody() } } } private def errorContains(errMsg: String, str: String): Unit = { assert(errMsg.contains(str)) } protected def testTableUpdateDPO( testName: String)(updateFunc: (String, String) => Seq[Row]): Unit = { withSQLConf(SQLConf.PARTITION_OVERWRITE_MODE.key -> SQLConf.PartitionOverwriteMode.DYNAMIC.toString) { testTableUpdate("dpo_" + testName)(updateFunc) } } testTableUpdate("append_data") { (table, path) => Seq( Tuple5(1L, "foo", "2020-10-11 12:30:30", 100, "2020-11-12") ).toDF("c1", "c3_p", "c5", "c6", "c8") .withColumn("c5", $"c5".cast(TimestampType)) .withColumn("c8", $"c8".cast(DateType)) .write .format("delta") .mode("append") .save(path) Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdate("append_data_in_different_column_order") { (table, path) => Seq( Tuple5("2020-10-11 12:30:30", 100, "2020-11-12", 1L, "foo") ).toDF("c5", "c6", "c8", "c1", "c3_p") .withColumn("c5", $"c5".cast(TimestampType)) .withColumn("c8", $"c8".cast(DateType)) .write .format("delta") .mode("append") .save(path) Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdate("append_data_v2") { (table, _) => Seq( Tuple5(1L, "foo", "2020-10-11 12:30:30", 100, "2020-11-12") ).toDF("c1", "c3_p", "c5", "c6", "c8") .withColumn("c5", $"c5".cast(TimestampType)) .withColumn("c8", $"c8".cast(DateType)) .writeTo(table) .append() Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdate("append_data_in_different_column_order_v2") { (table, _) => Seq( Tuple5("2020-10-11 12:30:30", 100, "2020-11-12", 1L, "foo") ).toDF("c5", "c6", "c8", "c1", "c3_p") .withColumn("c5", $"c5".cast(TimestampType)) .withColumn("c8", $"c8".cast(DateType)) .writeTo(table) .append() Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdate("insert_into_values_provide_all_columns") { (table, path) => sql(s"INSERT INTO $table VALUES" + s"(1, 11, 'foo', '2020-10-11', '2020-10-11 12:30:30', 100, 1000, '2020-11-12')") Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdate("insert_into_by_name_provide_all_columns") { (table, _) => sql(s"INSERT INTO $table (c5, c6, c7_g_p, c8, c1, c2_g, c3_p, c4_g_p) VALUES" + s"('2020-10-11 12:30:30', 100, 1000, '2020-11-12', 1, 11, 'foo', '2020-10-11')") Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdate("insert_into_by_name_not_provide_generated_columns") { (table, _) => sql(s"INSERT INTO $table (c6, c8, c1, c3_p, c5) VALUES" + s"(100, '2020-11-12', 1L, 'foo', '2020-10-11 12:30:30')") Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdate("insert_into_by_name_with_some_generated_columns") { (table, _) => sql(s"INSERT INTO $table (c5, c6, c8, c1, c3_p, c4_g_p) VALUES" + s"('2020-10-11 12:30:30', 100, '2020-11-12', 1L, 'foo', '2020-10-11')") Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdate("insert_into_select_provide_all_columns") { (table, path) => sql(s"INSERT INTO $table SELECT " + s"1, 11, 'foo', '2020-10-11', '2020-10-11 12:30:30', 100, 1000, '2020-11-12'") Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdate("insert_into_by_name_not_provide_normal_columns") { (table, _) => val e = intercept[AnalysisException] { withSQLConf(SQLConf.USE_NULLS_FOR_MISSING_DEFAULT_COLUMN_VALUES.key -> "false") { sql(s"INSERT INTO $table (c6, c8, c1, c3_p) VALUES" + s"(100, '2020-11-12', 1L, 'foo')") } } errorContains(e.getMessage, "Column c5 is not specified in INSERT") Nil } testTableUpdate("insert_overwrite_values_provide_all_columns") { (table, path) => sql(s"INSERT OVERWRITE TABLE $table VALUES" + s"(1, 11, 'foo', '2020-10-11', '2020-10-11 12:30:30', 100, 1000, '2020-11-12')") Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdate("insert_overwrite_select_provide_all_columns") { (table, path) => sql(s"INSERT OVERWRITE TABLE $table SELECT " + s"1, 11, 'foo', '2020-10-11', '2020-10-11 12:30:30', 100, 1000, '2020-11-12'") Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdate("insert_overwrite_by_name_provide_all_columns") { (table, _) => sql(s"INSERT OVERWRITE $table (c5, c6, c7_g_p, c8, c1, c2_g, c3_p, c4_g_p) VALUES" + s"('2020-10-11 12:30:30', 100, 1000, '2020-11-12', 1, 11, 'foo', '2020-10-11')") Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdate("insert_overwrite_by_name_not_provide_generated_columns") { (table, _) => sql(s"INSERT OVERWRITE $table (c6, c8, c1, c3_p, c5) VALUES" + s"(100, '2020-11-12', 1L, 'foo', '2020-10-11 12:30:30')") Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdate("insert_overwrite_by_name_with_some_generated_columns") { (table, _) => sql(s"INSERT OVERWRITE $table (c5, c6, c8, c1, c3_p, c4_g_p) VALUES" + s"('2020-10-11 12:30:30', 100, '2020-11-12', 1L, 'foo', '2020-10-11')") Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdate("insert_overwrite_by_name_not_provide_normal_columns") { (table, _) => val e = intercept[AnalysisException] { withSQLConf(SQLConf.USE_NULLS_FOR_MISSING_DEFAULT_COLUMN_VALUES.key -> "false") { sql(s"INSERT OVERWRITE $table (c6, c8, c1, c3_p) VALUES" + s"(100, '2020-11-12', 1L, 'foo')") } } errorContains(e.getMessage, "Column c5 is not specified in INSERT") Nil } testTableUpdateDPO("insert_overwrite_values_provide_all_columns") { (table, path) => sql(s"INSERT OVERWRITE TABLE $table VALUES" + s"(1, 11, 'foo', '2020-10-11', '2020-10-11 12:30:30', 100, 1000, '2020-11-12')") Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdateDPO("insert_overwrite_select_provide_all_columns") { (table, path) => sql(s"INSERT OVERWRITE TABLE $table SELECT " + s"1, 11, 'foo', '2020-10-11', '2020-10-11 12:30:30', 100, 1000, '2020-11-12'") Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdateDPO("insert_overwrite_by_name_values_provide_all_columns") { (table, _) => sql(s"INSERT OVERWRITE $table (c5, c6, c7_g_p, c8, c1, c2_g, c3_p, c4_g_p) VALUES" + s"(CAST('2020-10-11 12:30:30' AS TIMESTAMP), 100, 1000, CAST('2020-11-12' AS DATE), " + s"1L, 11L, 'foo', CAST('2020-10-11' AS DATE))") Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdateDPO( "insert_overwrite_by_name_not_provide_generated_columns") { (table, _) => sql(s"INSERT OVERWRITE $table (c6, c8, c1, c3_p, c5) VALUES" + s"(100, CAST('2020-11-12' AS DATE), 1L, 'foo', CAST('2020-10-11 12:30:30' AS TIMESTAMP))") Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdateDPO("insert_overwrite_by_name_with_some_generated_columns") { (table, _) => sql(s"INSERT OVERWRITE $table (c5, c6, c8, c1, c3_p, c4_g_p) VALUES" + s"(CAST('2020-10-11 12:30:30' AS TIMESTAMP), 100, CAST('2020-11-12' AS DATE), 1L, " + s"'foo', CAST('2020-10-11' AS DATE))") Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdateDPO("insert_overwrite_by_name_not_provide_normal_columns") { (table, _) => val e = intercept[AnalysisException] { sql(s"INSERT OVERWRITE $table (c6, c8, c1, c3_p) VALUES" + s"(100, '2020-11-12', 1L, 'foo')") } assert(e.getMessage.contains("with name `c5` cannot be resolved") || e.getMessage.contains("Column c5 is not specified in INSERT")) Nil } testTableUpdate("delete") { (table, path) => Seq( Tuple5(1L, "foo", "2020-10-11 12:30:30", 100, "2020-11-12"), Tuple5(2L, "foo", "2020-10-11 13:30:30", 100, "2020-12-12") ).toDF("c1", "c3_p", "c5", "c6", "c8") .withColumn("c5", $"c5".cast(TimestampType)) .withColumn("c8", $"c8".cast(DateType)) .coalesce(1) .write .format("delta") .mode("append") .save(path) // Make sure we create only one file so that we will trigger file rewriting. assert(DeltaLog.forTable(spark, path).snapshot.allFiles.count == 1) sql(s"DELETE FROM $table WHERE c1 = 2") Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdate("update_generated_column_with_correct_value") { (table, path) => sql(s"INSERT INTO $table SELECT " + s"1, 11, 'foo', '2020-10-11', '2020-10-11 12:30:30', 100, 1000, '2020-11-12'") sql(s"UPDATE $table SET c2_g = 11 WHERE c1 = 1") Row(1, 11, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdate("update_generated_column_with_incorrect_value") { (table, path) => sql(s"INSERT INTO $table SELECT " + s"1, 11, 'foo', '2020-10-11', '2020-10-11 12:30:30', 100, 1000, '2020-11-12'") val e = intercept[InvariantViolationException] { quietly { sql(s"UPDATE $table SET c2_g = 12 WHERE c1 = 1") } } errorContains(e.getMessage, "CHECK constraint Generated Column (c2_g <=> (c1 + 10)) violated by row with values") Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdate("update_source_column_used_by_generated_column") { (table, _) => sql(s"INSERT INTO $table SELECT " + s"1, 11, 'foo', '2020-10-11', '2020-10-11 12:30:30', 100, 1000, '2020-11-12'") sql(s"UPDATE $table SET c1 = 2 WHERE c1 = 1") Row(2, 12, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdate("update_source_and_generated_columns_with_correct_value") { (table, _) => sql(s"INSERT INTO $table SELECT " + s"1, 11, 'foo', '2020-10-11', '2020-10-11 12:30:30', 100, 1000, '2020-11-12'") sql(s"UPDATE $table SET c2_g = 12, c1 = 2 WHERE c1 = 1") Row(2, 12, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdate("update_source_and_generated_columns_with_incorrect_value") { (table, _) => sql(s"INSERT INTO $table SELECT " + s"1, 11, 'foo', '2020-10-11', '2020-10-11 12:30:30', 100, 1000, '2020-11-12'") val e = intercept[InvariantViolationException] { quietly { sql(s"UPDATE $table SET c2_g = 12, c1 = 3 WHERE c1 = 1") } } errorContains(e.getMessage, "CHECK constraint Generated Column (c2_g <=> (c1 + 10)) violated by row with values") Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } test("various update commands") { withTempDir { tempDir => val path = tempDir.getCanonicalPath withTableName("update_commands") { table => createTable(table, Some(path), "c INT, g INT", Map("g" -> "c + 10"), Nil) sql(s"INSERT INTO $table VALUES(10, 20)") sql(s"UPDATE $table SET c = 20") checkAnswer(spark.table(table), Row(20, 30) :: Nil) sql(s"UPDATE delta.`$path` SET c = 30") checkAnswer(spark.table(table), Row(30, 40) :: Nil) io.delta.tables.DeltaTable.forName(table).updateExpr(Map("c" -> "40")) checkAnswer(spark.table(table), Row(40, 50) :: Nil) io.delta.tables.DeltaTable.forPath(path).updateExpr(Map("c" -> "50")) checkAnswer(spark.table(table), Row(50, 60) :: Nil) } } } test("update with various column references") { withTableName("update_with_various_references") { table => createTable(table, None, "c1 INT, c2 INT, g INT", Map("g" -> "c1 + 10"), Nil) sql(s"INSERT INTO $table VALUES(10, 50, 20)") sql(s"UPDATE $table SET c1 = 20") checkAnswer(spark.table(table), Row(20, 50, 30) :: Nil) sql(s"UPDATE $table SET c1 = c2 + 100, c2 = 1000") checkAnswer(spark.table(table), Row(150, 1000, 160) :: Nil) sql(s"UPDATE $table SET c1 = c2 + g") checkAnswer(spark.table(table), Row(1160, 1000, 1170) :: Nil) sql(s"UPDATE $table SET c1 = g") checkAnswer(spark.table(table), Row(1170, 1000, 1180) :: Nil) } } test("update a struct source column") { withTableName("update_struct_column") { table => createTable(table, None, "s STRUCT, g INT", Map("g" -> "s.s1 + 10"), Nil) sql(s"INSERT INTO $table VALUES(struct(10, 'foo'), 20)") sql(s"UPDATE $table SET s.s1 = 20 WHERE s.s1 = 10") checkAnswer(spark.table(table), Row(Row(20, "foo"), 30) :: Nil) } } test("updating a temp view is not supported") { withTableName("update_temp_view") { table => createTable(table, None, "c1 INT, c2 INT", Map("c2" -> "c1 + 10"), Nil) withTempView("test_view") { sql(s"CREATE TEMP VIEW test_view AS SELECT * FROM $table") val e = intercept[AnalysisException] { sql(s"UPDATE test_view SET c1 = 2 WHERE c1 = 1") } assert(e.getMessage.contains("a temp view")) } } } testTableUpdate("streaming_write", isStreaming = true) { (table, path) => withTempDir { checkpointDir => val stream = MemoryStream[Int] val q = stream.toDF .map(_ => Tuple5(1L, "foo", "2020-10-11 12:30:30", 100, "2020-11-12")) .toDF("c1", "c3_p", "c5", "c6", "c8") .withColumn("c5", $"c5".cast(TimestampType)) .withColumn("c8", $"c8".cast(DateType)) .writeStream .format("delta") .outputMode("append") .option("checkpointLocation", checkpointDir.getCanonicalPath) .start(path) stream.addData(1) q.processAllAvailable() q.stop() } Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdate("streaming_write_with_different_case", isStreaming = true) { (table, path) => withTempDir { checkpointDir => val stream = MemoryStream[Int] val q = stream.toDF .map(_ => Tuple5(1L, "foo", "2020-10-11 12:30:30", 100, "2020-11-12")) .toDF("C1", "c3_p", "c5", "c6", "c8") // C1 is using upper case .withColumn("c5", $"c5".cast(TimestampType)) .withColumn("c8", $"c8".cast(DateType)) .writeStream .format("delta") .outputMode("append") .option("checkpointLocation", checkpointDir.getCanonicalPath) .start(path) stream.addData(1) q.processAllAvailable() q.stop() } Row(1L, 11L, "foo", sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:30:30"), 100, 1000, sqlDate("2020-11-12")) :: Nil } testTableUpdate("streaming_write_incorrect_value", isStreaming = true) { (table, path) => withTempDir { checkpointDir => quietly { val stream = MemoryStream[Int] val q = stream.toDF // 2L is an incorrect value. The correct value should be 11L .map(_ => Tuple6(1L, 2L, "foo", "2020-10-11 12:30:30", 100, "2020-11-12")) .toDF("c1", "c2_g", "c3_p", "c5", "c6", "c8") .withColumn("c5", $"c5".cast(TimestampType)) .withColumn("c8", $"c8".cast(DateType)) .writeStream .format("delta") .outputMode("append") .option("checkpointLocation", checkpointDir.getCanonicalPath) .start(path) stream.addData(1) val e = intercept[StreamingQueryException] { q.processAllAvailable() } errorContains(e.getMessage, "CHECK constraint Generated Column (c2_g <=> (c1 + 10)) violated by row with values") q.stop() } } Nil } testQuietly("write to a generated column with an incorrect value") { withTableName("write_incorrect_value") { table => createTable(table, None, "id INT, id2 INT", Map("id2" -> "id + 10"), partitionColumns = Nil) val e = intercept[InvariantViolationException] { sql(s"INSERT INTO $table VALUES(1, 12)") } errorContains(e.getMessage, "CHECK constraint Generated Column (id2 <=> (id + 10)) violated by row with values") } } test("dot in the column name") { withTableName("dot_in_column_name") { table => createTable(table, None, "`a.b` INT, `x.y` INT", Map("x.y" -> "`a.b` + 10"), Nil) sql(s"INSERT INTO $table VALUES(1, 11)") sql(s"INSERT INTO $table VALUES(2, 12)") checkAnswer(sql(s"SELECT * FROM $table"), Row(1, 11) :: Row(2, 12) :: Nil) } } test("validateGeneratedColumns: generated columns should not refer to non-existent columns") { val f1 = StructField("c1", IntegerType) val f2 = withGenerationExpression(StructField("c2", IntegerType), "c10 + 10") val schema = StructType(f1 :: f2 :: Nil) val e = intercept[DeltaAnalysisException](validateGeneratedColumns(spark, schema)) errorContains(e.getMessage, "A generated column cannot use a non-existent column or another generated column") } test("validateGeneratedColumns: no generated columns") { val f1 = StructField("c1", IntegerType) val f2 = StructField("c2", IntegerType) val schema = StructType(f1 :: f2 :: Nil) validateGeneratedColumns(spark, schema) } test("validateGeneratedColumns: all generated columns") { val f1 = withGenerationExpression(StructField("c1", IntegerType), "1 + 2") val f2 = withGenerationExpression(StructField("c1", IntegerType), "3 + 4") val schema = StructType(f1 :: f2 :: Nil) validateGeneratedColumns(spark, schema) } test("validateGeneratedColumns: generated columns should not refer to other generated columns") { val f1 = StructField("c1", IntegerType) val f2 = withGenerationExpression(StructField("c2", IntegerType), "c1 + 10") val f3 = withGenerationExpression(StructField("c3", IntegerType), "c2 + 10") val schema = StructType(f1 :: f2 :: f3 :: Nil) val e = intercept[DeltaAnalysisException](validateGeneratedColumns(spark, schema)) errorContains(e.getMessage, "A generated column cannot use a non-existent column or another generated column") } test("validateGeneratedColumns: supported expressions") { for (exprString <- Seq( // Generated column should support timestamp to date "to_date(foo, \"yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'\")")) { val f1 = StructField("foo", TimestampType) val f2 = withGenerationExpression(StructField("bar", DateType), exprString) val schema = StructType(Seq(f1, f2)) validateGeneratedColumns(spark, schema) } } test("validateGeneratedColumns: unsupported expressions") { spark.udf.register("myudf", (s: Array[Int]) => s) for ((exprString, error) <- Seq( "myudf(foo)" -> "Found myudf(foo). A generated column cannot use a user-defined function", "rand()" -> "Found rand(). A generated column cannot use a non deterministic expression", "max(foo)" -> "Found max(foo). A generated column cannot use an aggregate expression", "explode(foo)" -> "explode(foo) cannot be used in a generated column", "current_timestamp" -> "current_timestamp() cannot be used in a generated column" )) { val f1 = StructField("foo", ArrayType(IntegerType, true)) val f2 = withGenerationExpression(StructField("bar", IntegerType), exprString) val schema = StructType(f1 :: f2 :: Nil) val e = intercept[AnalysisException](validateGeneratedColumns(spark, schema)) errorContains(e.getMessage, error) } } protected def testTypeMismatch( generatedColumnType: DataType, generatedColumnNullable: Boolean, generateAsExpression: Column, expectSuccess: Boolean ): Unit = { val verb = if (expectSuccess) "matches" else "doesn't match" val columnTypeString = if (generatedColumnNullable) { generatedColumnType.sql } else { s"${generatedColumnType.sql} NOT NULL" } test(s"validateGeneratedColumns: column type ${columnTypeString}" + s" $verb expression type $generateAsExpression") { val f1 = StructField("nullableIntCol", IntegerType, nullable = true) val f2 = withGenerationExpression( StructField("genCol", generatedColumnType, nullable = generatedColumnNullable), generateAsExpression.expr.sql) val schema = StructType(f1 :: f2 :: Nil) if (expectSuccess) { validateGeneratedColumns(spark, schema) } else { val e = intercept[AnalysisException](validateGeneratedColumns(spark, schema)) val expressionTypeString = if (generateAsExpression.expr.resolved) { generateAsExpression.expr.dataType.sql } else { val df1 = spark.createDataFrame(spark.emptyDataFrame.rdd, schema) .select(generateAsExpression) df1.schema.fields.head.dataType.sql } checkErrorMatchPVals(e, "DELTA_GENERATED_COLUMNS_EXPR_TYPE_MISMATCH", parameters = Map( "columnName" -> "genCol", "expressionType" -> s".*${expressionTypeString}.*", "columnType" -> s".*${generatedColumnType.sql}.*" )) } } } testTypeMismatch( generatedColumnType = IntegerType, generatedColumnNullable = true, generateAsExpression = $"nullableIntCol", expectSuccess = true ) testTypeMismatch( generatedColumnType = IntegerType, generatedColumnNullable = false, generateAsExpression = $"nullableIntCol", // Even though foo is nullable, we allow this and fail at runtime when foo actually contains // a NULL value. expectSuccess = true ) testTypeMismatch( generatedColumnType = IntegerType, generatedColumnNullable = false, generateAsExpression = lit(5), expectSuccess = true ) testTypeMismatch( generatedColumnType = IntegerType, generatedColumnNullable = false, // We need to force this to be INT NULL not a VOID NULL. generateAsExpression = typedLit[java.lang.Integer](null), // Even though the expression is clearly nullable, we allow this and fail at runtime // when actually generating a NULL value. expectSuccess = true ) testTypeMismatch( generatedColumnType = IntegerType, generatedColumnNullable = true, // We need to force this to be INT NULL not a VOID NULL. generateAsExpression = typedLit[java.lang.Integer](null), expectSuccess = true ) for (generatedColumnNullable <- BOOLEAN_DOMAIN) { testTypeMismatch( generatedColumnType = IntegerType, generatedColumnNullable = generatedColumnNullable, generateAsExpression = $"nullableIntCol".cast(StringType), expectSuccess = false) testTypeMismatch( generatedColumnType = IntegerType, generatedColumnNullable = generatedColumnNullable, generateAsExpression = $"nullableIntCol".cast(LongType), expectSuccess = false) testTypeMismatch( generatedColumnType = IntegerType, generatedColumnNullable = generatedColumnNullable, generateAsExpression = $"nullableIntCol".cast(ShortType), expectSuccess = false) } testTypeMismatch( generatedColumnType = StructType(Array(StructField("first", IntegerType, nullable = false))), generatedColumnNullable = true, generateAsExpression = struct($"nullableIntCol".as("first")), // Even though foo is nullable, we allow this and fail at runtime when foo actually contains // a NULL value. expectSuccess = true ) testTypeMismatch( generatedColumnType = StructType(Array(StructField("firstNullable", IntegerType, nullable = true))), generatedColumnNullable = true, generateAsExpression = struct($"nullableIntCol".as("firstNullable")), expectSuccess = true ) test("nullability mismatch fails at runtime") { withTableName("tbl") { tbl => createTable( tableName = tbl, path = None, schemaString = "base STRING, gen STRING", generatedColumns = Map("gen" -> "concat(base, '-generated')"), partitionColumns = Seq.empty, // base is nullable, but gen isn't even though it's derived from base. notNullColumns = Set("gen")) // Perform a legal write. Seq("1").toDF("base") .write.format("delta").mode("append").saveAsTable(tbl) // Perform an illegal write. val e = intercept[DeltaInvariantViolationException] { Seq(null.asInstanceOf[String]).toDF("base") .write.format("delta").mode("append").saveAsTable(tbl) } checkError(e, "DELTA_NOT_NULL_CONSTRAINT_VIOLATED", parameters = Map("columnName" -> "gen")) // Ensure the result is correct. checkAnswer( spark.read.table(tbl), Row("1", "1-generated") :: Nil ) } } test("test partition transform expressions end to end") { withTableName("partition_transform_expressions") { table => createTable(table, None, "time TIMESTAMP, year DATE, month DATE, day DATE, hour TIMESTAMP", Map( "year" -> "make_date(year(time), 1, 1)", "month" -> "make_date(year(time), month(time), 1)", "day" -> "make_date(year(time), month(time), day(time))", "hour" -> "make_timestamp(year(time), month(time), day(time), hour(time), 0, 0)" ), partitionColumns = Nil) Seq("2020-10-11 12:30:30") .toDF("time") .withColumn("time", $"time".cast(TimestampType)) .write .format("delta") .mode("append") .saveAsTable(table) checkAnswer( sql(s"SELECT * from $table"), Row(sqlTimestamp("2020-10-11 12:30:30"), sqlDate("2020-01-01"), sqlDate("2020-10-01"), sqlDate("2020-10-11"), sqlTimestamp("2020-10-11 12:00:00")) ) } } test("the generation expression constraint should support null values") { withTableName("null") { table => createTable(table, None, "c1 STRING, c2 STRING", Map("c2" -> "CONCAT(c1, 'y')"), Nil) sql(s"INSERT INTO $table VALUES('x', 'xy')") sql(s"INSERT INTO $table VALUES(null, null)") checkAnswer( sql(s"SELECT * from $table"), Row("x", "xy") :: Row(null, null) :: Nil ) quietly { val e = intercept[InvariantViolationException](sql(s"INSERT INTO $table VALUES('foo', null)")) errorContains(e.getMessage, "CHECK constraint Generated Column (c2 <=> CONCAT(c1, 'y')) " + "violated by row with values") } quietly { val e = intercept[InvariantViolationException](sql(s"INSERT INTO $table VALUES(null, 'foo')")) errorContains(e.getMessage, "CHECK constraint Generated Column (c2 <=> CONCAT(c1, 'y')) " + "violated by row with values") } } } test("complex type extractors") { withTableName("struct_field") { table => createTable( table, None, "`a.b` STRING, a STRUCT, array ARRAY, " + "c1 STRING, c2 INT, c3 INT", Map("c1" -> "CONCAT(`a.b`, 'b')", "c2" -> "a.b + 100", "c3" -> "array[1]"), Nil) sql(s"INSERT INTO $table VALUES(" + s"'a', struct(100, 'foo'), array(1000, 1001), " + s"'ab', 200, 1001)") checkAnswer( spark.table(table), Row("a", Row(100, "foo"), Array(1000, 1001), "ab", 200, 1001) :: Nil) } } test("getGeneratedColumnsAndColumnsUsedByGeneratedColumns") { def testSchema(schema: Seq[StructField], expected: Set[String]): Unit = { assert(getGeneratedColumnsAndColumnsUsedByGeneratedColumns(StructType(schema)) == expected) } val f1 = StructField("c1", IntegerType) val f2 = withGenerationExpression(StructField("c2", IntegerType), "c1 + 10") val f3 = StructField("c3", IntegerType) val f4 = withGenerationExpression(StructField("c4", IntegerType), "hash(c3 + 10)") val f5 = withGenerationExpression(StructField("c5", IntegerType), "hash(C1 + 10)") val f6 = StructField("c6", StructType(StructField("x", IntegerType) :: Nil)) val f6x = StructField("c6.x", IntegerType) val f7x = withGenerationExpression(StructField("c7.x", IntegerType), "`c6.x` + 10") val f8 = withGenerationExpression(StructField("c8", IntegerType), "c6.x + 10") testSchema(Seq(f1, f2), Set("c1", "c2")) testSchema(Seq(f1, f2, f3), Set("c1", "c2")) testSchema(Seq(f1, f2, f3, f4), Set("c1", "c2", "c3", "c4")) testSchema(Seq(f1, f2, f5), Set("c1", "c2", "c5")) testSchema(Seq(f6x, f7x), Set("c6.x", "c7.x")) testSchema(Seq(f6, f6x, f7x), Set("c6.x", "c7.x")) testSchema(Seq(f6, f6x, f8), Set("c6", "c8")) } test("disallow column type evolution") { withTableName("disallow_column_type_evolution") { table => // "HASH(c1)" returns different results for INT and LONG. For example, "SELECT hash(32767)" // returns 1249274084, but "SELECT hash(32767L)" returns -860381306. Hence we should // not allow updating column type from INT to LONG. createTable(table, None, "c1 INT, c2 INT", Map("c2" -> "HASH(c1)"), Nil) val tableSchema = spark.table(table).schema Seq(32767).toDF("c1").write.format("delta").mode("append").saveAsTable(table) assert(tableSchema == spark.table(table).schema) // Insert a LONG to `c1` should fail rather than changing the `c1` type to LONG. checkError( intercept[AnalysisException] { Seq(32767.toLong).toDF("c1").write.format("delta").mode("append") .option("mergeSchema", "true") .saveAsTable(table) }, "DELTA_GENERATED_COLUMNS_DATA_TYPE_MISMATCH", parameters = Map( "columnName" -> "c1", "columnType" -> "INT", "dataType" -> "BIGINT", "generatedColumns" -> "c2 -> HASH(c1)" )) checkAnswer(spark.table(table), Row(32767, 1249274084) :: Nil) } } test("disallow column type evolution - nesting") { withTableName("disallow_column_type_evolution") { table => createTable(table, None, "a SMALLINT, c1 STRUCT, c2 INT", Map("c2" -> "HASH(a)"), Nil) val tableSchema = spark.table(table).schema Seq(32767.toShort).toDF("a") .selectExpr("a", "named_struct('a', a) as c1") .write.format("delta").mode("append").saveAsTable(table) assert(tableSchema == spark.table(table).schema) // INSERT an INT to `c1.a` should not fail Seq((32767.toShort, 32767)).toDF("a", "c1a") .selectExpr("a", "named_struct('a', c1a) as c1") .write.format("delta").mode("append") .option("mergeSchema", "true") .saveAsTable(table) // Insert an INT to `a` should fail rather than changing the `a` type to INT checkError( intercept[AnalysisException] { Seq((32767, 32767)).toDF("a", "c1a") .selectExpr("a", "named_struct('a', c1a) as c1") .write.format("delta").mode("append") .option("mergeSchema", "true") .saveAsTable(table) }, "DELTA_GENERATED_COLUMNS_DATA_TYPE_MISMATCH", parameters = Map( "columnName" -> "a", "columnType" -> "SMALLINT", "dataType" -> "INT", "generatedColumns" -> "c2 -> HASH(a)" ) ) } } test("changing the type of a nested field named the same as the generated column") { withTableName("disallow_column_type_evolution") { table => createTable(table, None, "a INT, t STRUCT, gen INT", Map("gen" -> "HASH(a)"), Nil) // Changing the type of `t.gen` should succeed since it's not actually the generated column. Seq((32767, 32767)).toDF("a", "gen") .selectExpr("a", "named_struct('gen', gen) as t") .write.format("delta").mode("append") .option("mergeSchema", "true") .saveAsTable(table) checkAnswer(spark.table(table), Row(32767, Row(32767), 1249274084) :: Nil) } } test("changing the type of nested field not referenced by a generated col") { withTableName("disallow_column_type_evolution") { table => createTable(table, None, "t STRUCT, gen INT", Map("gen" -> "HASH(t.a)"), Nil) // changing the type of `t.b` should succeed since it is not being // referenced by any CHECK constraints or generated columns. Seq((32767.toShort, 32767)).toDF("a", "b") .selectExpr("named_struct('a', a, 'b', b) as t") .write.format("delta").mode("append") .option("mergeSchema", "true") .saveAsTable(table) checkAnswer(spark.table(table), Row(Row(32767, 32767), 1249274084) :: Nil) } } test("reading from a Delta table should not see generation expressions") { def verifyNoGenerationExpression(df: Dataset[_]): Unit = { assert(!hasGeneratedColumns(df.schema)) } withTableName("test_source") { table => createTable(table, None, "c1 INT, c2 INT", Map("c1" -> "c2 + 1"), Nil) sql(s"INSERT INTO $table VALUES(2, 1)") val path = DeltaLog.forTable(spark, TableIdentifier(table)).dataPath.toString verifyNoGenerationExpression(spark.table(table)) verifyNoGenerationExpression(spark.sql(s"select * from $table")) verifyNoGenerationExpression(spark.sql(s"select * from delta.`$path`")) verifyNoGenerationExpression(spark.read.format("delta").load(path)) verifyNoGenerationExpression(spark.read.format("delta").table(table)) verifyNoGenerationExpression(spark.readStream.format("delta").load(path)) verifyNoGenerationExpression(spark.readStream.format("delta").table(table)) withTempDir { checkpointDir => val q = spark.readStream.format("delta").table(table).writeStream .trigger(Trigger.Once) .option("checkpointLocation", checkpointDir.getCanonicalPath) .foreachBatch { (ds: DataFrame, _: Long) => verifyNoGenerationExpression(ds) }.start() try { q.processAllAvailable() } finally { q.stop() } } withTempDir { outputDir => withTempDir { checkpointDir => val q = spark.readStream.format("delta").table(table).writeStream .trigger(Trigger.Once) .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("delta") .start(outputDir.getCanonicalPath) try { q.processAllAvailable() } finally { q.stop() } val deltaLog = DeltaLog.forTable(spark, outputDir) assert(deltaLog.snapshot.version >= 0) assert(!hasGeneratedColumns(deltaLog.snapshot.schema)) } } } } /** * Verify if the table metadata matches the default test table. We use this to verify DDLs * write correct table metadata into the transaction logs. */ protected def verifyDefaultTestTableMetadata(table: String): Unit = { val (deltaLog, snapshot) = if (table.startsWith("delta.")) { DeltaLog.forTableWithSnapshot(spark, table.stripPrefix("delta.`").stripSuffix("`")) } else { DeltaLog.forTableWithSnapshot(spark, TableIdentifier(table)) } val schema = StructType.fromDDL(defaultTestTableSchema) val expectedSchema = StructType(schema.map { field => defaultTestTableGeneratedColumns.get(field.name).map { expr => withGenerationExpression(field, expr) }.getOrElse(field) }) val partitionColumns = defaultTestTablePartitionColumns val metadata = snapshot.metadata assert(metadata.schema == expectedSchema) assert(metadata.partitionColumns == partitionColumns) } protected def testCreateTable(testName: String)(createFunc: String => Unit): Unit = { test(testName) { withTable(testName) { createFunc(testName) verifyDefaultTestTableMetadata(testName) } } } protected def testCreateTableWithLocation( testName: String)(createFunc: (String, String) => Unit): Unit = { test(testName + ": external") { withTempPath { path => withTable(testName) { createFunc(testName, path.getCanonicalPath) verifyDefaultTestTableMetadata(testName) verifyDefaultTestTableMetadata(s"delta.`${path.getCanonicalPath}`") } } } } testCreateTable("create_table") { table => createTable( table, None, defaultTestTableSchema, defaultTestTableGeneratedColumns, defaultTestTablePartitionColumns ) } testCreateTable("replace_table") { table => createTable(table, None, "id bigint", Map.empty, Seq.empty) replaceTable( table, None, defaultTestTableSchema, defaultTestTableGeneratedColumns, defaultTestTablePartitionColumns ) } testCreateTable("create_or_replace_table_non_exist") { table => replaceTable( table, None, defaultTestTableSchema, defaultTestTableGeneratedColumns, defaultTestTablePartitionColumns, orCreate = Some(true) ) } testCreateTable("create_or_replace_table_exist") { table => createTable(table, None, "id bigint", Map.empty, Seq.empty) replaceTable( table, None, defaultTestTableSchema, defaultTestTableGeneratedColumns, defaultTestTablePartitionColumns, orCreate = Some(true) ) } testCreateTableWithLocation("create_table") { (table, path) => createTable( table, Some(path), defaultTestTableSchema, defaultTestTableGeneratedColumns, defaultTestTablePartitionColumns ) } testCreateTableWithLocation("replace_table") { (table, path) => createTable( table, Some(path), defaultTestTableSchema, defaultTestTableGeneratedColumns, defaultTestTablePartitionColumns ) replaceTable( table, Some(path), defaultTestTableSchema, defaultTestTableGeneratedColumns, defaultTestTablePartitionColumns ) } testCreateTableWithLocation("create_or_replace_table_non_exist") { (table, path) => replaceTable( table, Some(path), defaultTestTableSchema, defaultTestTableGeneratedColumns, defaultTestTablePartitionColumns, orCreate = Some(true) ) } testCreateTableWithLocation("create_or_replace_table_exist") { (table, path) => createTable( table, Some(path), "id bigint", Map.empty, Seq.empty ) replaceTable( table, Some(path), defaultTestTableSchema, defaultTestTableGeneratedColumns, defaultTestTablePartitionColumns, orCreate = Some(true) ) } test("using generated columns should upgrade the protocol") { withTableName("upgrade_protocol") { table => // Use the default protocol versions when not using computed partitions. createTable(table, None, "i INT", Map.empty, Seq.empty) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName = table)) assert(deltaLog.update().protocol == Protocol(1, 2)) assert(DeltaLog.forTable(spark, TableIdentifier(tableName = table)).snapshot.version == 0) // Protocol versions should be upgraded when using computed partitions. replaceTable( table, None, defaultTestTableSchema, defaultTestTableGeneratedColumns, defaultTestTablePartitionColumns) assert(deltaLog.update().protocol == Protocol(1, 7).withFeatures(Seq( AppendOnlyTableFeature, InvariantsTableFeature, GeneratedColumnsTableFeature))) // Make sure we did overwrite the table rather than deleting and re-creating. assert(DeltaLog.forTable(spark, TableIdentifier(tableName = table)).update().version == 1) } } test("creating a table with a different schema should fail") { withTempPath { path => // Currently SQL is the only way to define a table using generated columns. So we create a // temp table and drop it to get a path for such table. withTableName("temp_generated_column_table") { table => createTable( null, Some(path.toString), defaultTestTableSchema, defaultTestTableGeneratedColumns, defaultTestTablePartitionColumns ) } withTableName("table_with_no_schema") { table => createTable( table, Some(path.toString), "", Map.empty, Seq.empty ) verifyDefaultTestTableMetadata(table) } withTableName("table_with_different_expr") { table => val e = intercept[AnalysisException]( createTable( table, Some(path.toString), defaultTestTableSchema, Map( "c2_g" -> "c1 + 11", // use a different generated expr "c4_g_p" -> "CAST(c5 AS date)", "c7_g_p" -> "c6 * 10" ), defaultTestTablePartitionColumns ) ) assert(e.getMessage.contains( "Specified generation expression for field c2_g is different from existing schema")) assert(e.getMessage.contains("Specified: c1 + 11")) assert(e.getMessage.contains("Existing: c1 + 10")) } withTableName("table_add_new_expr") { table => val e = intercept[AnalysisException]( createTable( table, Some(path.toString), defaultTestTableSchema, Map( "c2_g" -> "c1 + 10", "c3_p" -> "CAST(c1 AS string)", // add a generated expr "c4_g_p" -> "CAST(c5 AS date)", "c7_g_p" -> "c6 * 10" ), defaultTestTablePartitionColumns ) ) assert(e.getMessage.contains( "Specified generation expression for field c3_p is different from existing schema")) assert(e.getMessage.contains("CAST(c1 AS string)")) assert(e.getMessage.contains("Existing: \n")) } } } test("use the generation expression, column comment and NOT NULL at the same time") { withTableName("generation_expression_comment") { table => createTable( table, None, "c1 INT, c2 INT, c3 INT", Map("c2" -> "c1 + 10", "c3" -> "c1 + 10"), Seq.empty, Set("c3"), Map("c2" -> "foo", "c3" -> "foo") ) // Verify schema val f1 = StructField("c1", IntegerType, nullable = true) val fieldMetadata = new MetadataBuilder() .putString(GENERATION_EXPRESSION_METADATA_KEY, "c1 + 10") .putString("comment", "foo") .build() val f2 = StructField("c2", IntegerType, nullable = true, metadata = fieldMetadata) val f3 = StructField("c3", IntegerType, nullable = false, metadata = fieldMetadata) val expectedSchema = StructType(f1 :: f2 :: f3 :: Nil) val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(table)) assert(snapshot.metadata.schema == expectedSchema) // Verify column comment val comments = sql(s"DESC $table") .where("col_name = 'c2'") .select("comment") .as[String] .collect() .toSeq assert("foo" :: Nil == comments) } } test("generation expression allows timestampdiff & timestampadd") { withTableName("generation_expression_timestamp_diff_add") { tableName => createTable( tableName, path = None, schemaString = "c1 TIMESTAMP, c2 TIMESTAMP, c3 BIGINT, c4 TIMESTAMP", generatedColumns = Map("c3" -> "timestampdiff(MONTH, c1, c2)", "c4" -> "timestampadd(MONTH, 1, c1)"), partitionColumns = Seq.empty) } } test("MERGE UPDATE basic") { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c3 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (1, 3, 4);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN MATCHED THEN UPDATE SET ${tgt}.c2 = ${src}.c2 |""".stripMargin) checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq(Row(1, 3, 4)) ) } } } test("MERGE UPDATE set both generated column and its input") { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c3 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (1, 3, 4);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN MATCHED THEN UPDATE SET ${tgt}.c2 = ${src}.c2, ${tgt}.c3 = ${src}.c3 |""".stripMargin) checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq(Row(1, 3, 4)) ) } } } test("MERGE UPDATE set star") { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c3 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (1, 4, 5);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN MATCHED THEN UPDATE SET * |""".stripMargin) checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq(Row(1, 4, 5)) ) } } } test("MERGE UPDATE set star add column") { withSQLConf(("spark.databricks.delta.schema.autoMerge.enabled", "true")) { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c4 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (1, 20, 40);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") sql( s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN MATCHED THEN UPDATE SET * |""".stripMargin) checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq(Row(1, 20, 21, 40)) ) } } } } test("MERGE UPDATE using value from target") { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c3 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (1, 3, 4);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN MATCHED THEN UPDATE SET ${tgt}.c2 = ${tgt}.c3 |""".stripMargin) checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq(Row(1, 3, 4)) ) } } } test("MERGE UPDATE using value from both target and source") { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c3 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (1, 3, 4);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN MATCHED THEN UPDATE SET ${tgt}.c2 = ${tgt}.c3 + ${src}.c3 |""".stripMargin) checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq(Row(1, 7, 8)) ) } } } test("MERGE UPDATE set to null") { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c3 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (1, 3, 4);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN MATCHED THEN UPDATE SET ${tgt}.c2 = null |""".stripMargin) checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq(Row(1, null, null)) ) } } } test("MERGE UPDATE multiple columns") { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c3 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (1, 3, 4);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN MATCHED THEN UPDATE | SET ${tgt}.c2 = ${src}.c1 * 10, ${tgt}.c1 = ${tgt}.c1 * 100 |""".stripMargin) checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq(Row(100, 10, 11)) ) } } } test("MERGE UPDATE source is a query") { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c3 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (1, 3, 4);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") sql(s""" |MERGE INTO ${tgt} |USING (SELECT c1, max(c3) + min(c2) AS m FROM ${src} GROUP BY c1) source |on ${tgt}.c1 = source.c1 |WHEN MATCHED THEN UPDATE SET ${tgt}.c2 = source.m |""".stripMargin) checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq(Row(1, 7, 8)) ) } } } test("MERGE UPDATE temp view is not supported") { withTableName("source") { src => withTableName("target") { tgt => withTempView("test_temp_view") { createTable(src, None, "c1 INT, c2 INT, c3 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (1, 3, 4);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") sql(s"CREATE TEMP VIEW test_temp_view AS SELECT c1 as c2, c2 as c1, c3 FROM ${tgt}") val e = intercept[AnalysisException] { sql(s""" |MERGE INTO test_temp_view |USING ${src} |on test_temp_view.c2 = ${src}.c1 |WHEN MATCHED THEN UPDATE SET test_temp_view.c1 = ${src}.c2 |""".stripMargin) } assert(e.getMessage.contains("a temp view")) } } } } test("MERGE INSERT star satisfies constraint") { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c3 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (2, 3, 4);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN NOT MATCHED THEN INSERT * |""".stripMargin) checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq(Row(1, 2, 3), Row(2, 3, 4)) ) } } } test("MERGE INSERT star violates constraint") { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c3 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (2, 3, 5);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") val e = intercept[InvariantViolationException]( sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN NOT MATCHED THEN INSERT * |""".stripMargin) ) assert(e.getMessage.contains("CHECK constraint Generated Column")) } } } test("MERGE INSERT star add column") { withSQLConf(("spark.databricks.delta.schema.autoMerge.enabled", "true")) { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c4 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (2, 3, 5);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN NOT MATCHED THEN INSERT * |""".stripMargin) checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq(Row(1, 2, 3, null), Row(2, 3, 4, 5)) ) } } } } test("MERGE INSERT star add column violates constraint") { withSQLConf(("spark.databricks.delta.schema.autoMerge.enabled", "true")) { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c3 INT, c4 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (2, 3, 5);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") val e = intercept[InvariantViolationException]( sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN NOT MATCHED THEN INSERT * |""".stripMargin) ) assert(e.getMessage.contains("CHECK constraint Generated Column")) } } } } test("MERGE INSERT star add column unrelated to generated columns") { withSQLConf(("spark.databricks.delta.schema.autoMerge.enabled", "true")) { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c4 INT, c5 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (2, 3, 5);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN NOT MATCHED THEN INSERT * |""".stripMargin) checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq(Row(1, 2, 3, null, null), Row(2, null, null, 3, 5)) ) } } } } test("MERGE INSERT unrelated columns") { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c3 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (2, 3, 4);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN NOT MATCHED THEN INSERT (c1) VALUES (${src}.c1) |""".stripMargin) checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq(Row(1, 2, 3), Row(2, null, null)) ) } } } test("MERGE INSERT unrelated columns with const") { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c3 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (2, 3, 4);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN NOT MATCHED THEN INSERT (c1) VALUES (3) |""".stripMargin) checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq(Row(1, 2, 3), Row(3, null, null)) ) } } } test("MERGE INSERT referenced column only") { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c3 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (2, 3, 4);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN NOT MATCHED THEN INSERT (c2) VALUES (10) |""".stripMargin) checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq(Row(1, 2, 3), Row(null, 10, 11)) ) } } } test("MERGE INSERT referenced column with null") { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c3 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (2, 3, 4);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN NOT MATCHED THEN INSERT (c2) VALUES (null) |""".stripMargin) checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq(Row(1, 2, 3), Row(null, null, null)) ) } } } test("MERGE INSERT not all referenced column inserted") { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c3 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (2, 3, 4);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + c1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN NOT MATCHED THEN INSERT (c2) VALUES (5) |""".stripMargin) checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq(Row(1, 2, 3), Row(null, 5, null)) ) } } } test("MERGE INSERT generated column only") { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c3 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (2, 3, 4);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") val e = intercept[InvariantViolationException]( sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN NOT MATCHED THEN INSERT (c3) VALUES (10) |""".stripMargin) ) assert(e.getMessage.contains("CHECK constraint Generated Column")) } } } test("MERGE INSERT referenced and generated columns satisfies constraint") { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c3 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (2, 3, 4);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN NOT MATCHED THEN INSERT (c2, c3) VALUES (${src}.c2, ${src}.c3) |""".stripMargin) checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq(Row(1, 2, 3), Row(null, 3, 4)) ) } } } test("MERGE INSERT referenced and generated columns violates constraint") { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c3 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (2, 3, 5);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3);") val e = intercept[InvariantViolationException]( sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN NOT MATCHED THEN INSERT (c2, c3) VALUES (${src}.c2, ${src}.c3) |""".stripMargin) ) assert(e.getMessage.contains("CHECK constraint Generated Column")) } } } test("MERGE INSERT and UPDATE") { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c3 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (1, 11, 12), (2, 3, 4), (3, 20, 21), (4, 5, 6), (5, 6, 7);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3), (2, 100, 101);") sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN MATCHED AND ${src}.c1 = 1 THEN UPDATE SET ${tgt}.c2 = 100 |WHEN MATCHED THEN UPDATE SET * |WHEN NOT MATCHED AND ${src}.c1 = 4 THEN INSERT (c1, c2) values (${src}.c1, 22) |WHEN NOT MATCHED AND ${src}.c1 = 5 THEN INSERT (c1, c2) values (5, ${src}.c3) |WHEN NOT MATCHED THEN INSERT * |""".stripMargin) checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq(Row(1, 100, 101), Row(2, 3, 4), Row(3, 20, 21), Row(4, 22, 23), Row(5, 7, 8)) ) } } } test("MERGE INSERT and UPDATE schema evolution") { withSQLConf(("spark.databricks.delta.schema.autoMerge.enabled", "true")) { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c4 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (1, 11, 12), (2, 3, 4), (3, 20, 21), " + "(4, 5, 6), (5, 6, 7);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c2 + 1"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 3), (2, 100, 101);") sql( s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN MATCHED AND ${src}.c1 = 1 THEN UPDATE SET ${tgt}.c2 = 100 |WHEN MATCHED THEN UPDATE SET * |WHEN NOT MATCHED AND ${src}.c1 = 4 THEN INSERT (c1, c2) values (${src}.c1, 22) |WHEN NOT MATCHED AND ${src}.c1 = 5 THEN INSERT (c1, c2) values (5, ${src}.c4) |WHEN NOT MATCHED THEN INSERT * |""".stripMargin) checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq( Row(1, 100, 101, null), Row(2, 3, 4, 4), Row(3, 20, 21, 21), Row(4, 22, 23, null), Row(5, 7, 8, null) ) ) } } } } test("MERGE INSERT and UPDATE schema evolution multiple referenced columns") { withSQLConf(("spark.databricks.delta.schema.autoMerge.enabled", "true")) { withTableName("source") { src => withTableName("target") { tgt => createTable(src, None, "c1 INT, c2 INT, c4 INT", Map.empty, Seq.empty) sql(s"INSERT INTO ${src} values (1, 11, 12), (2, null, 4), (3, 20, 21), " + "(4, 5, 6), (5, 6, 7);") createTable(tgt, None, "c1 INT, c2 INT, c3 INT", Map("c3" -> "c1 + CAST(ISNULL(c2) AS INT)"), Seq.empty) sql(s"INSERT INTO ${tgt} values (1, 2, 1), (2, 100, 2);") sql( s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN MATCHED AND ${src}.c1 = 1 THEN UPDATE SET ${tgt}.c2 = 100 |WHEN MATCHED THEN UPDATE SET * |WHEN NOT MATCHED AND ${src}.c1 = 4 THEN INSERT (c1, c2) values (${src}.c1, 22) |WHEN NOT MATCHED AND ${src}.c1 = 5 THEN INSERT (c1) values (5) |WHEN NOT MATCHED THEN INSERT * |""".stripMargin) checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq( Row(1, 100, 1, null), Row(2, null, 3, 4), Row(3, 20, 3, 21), Row(4, 22, 4, null), Row(5, null, 6, null) ) ) } } } } test("MERGE INSERT with schema evolution on different name case") { withTableName("source") { src => withTableName("target") { tgt => createTable( tableName = src, path = None, schemaString = "c1 INT, c2 INT", generatedColumns = Map.empty, partitionColumns = Seq.empty ) sql(s"INSERT INTO ${src} values (2, 4);") createTable( tableName = tgt, path = None, schemaString = "c1 INT, c3 INT", generatedColumns = Map("c3" -> "c1 + 1"), partitionColumns = Seq.empty ) sql(s"INSERT INTO ${tgt} values (1, 2);") withSQLConf(("spark.databricks.delta.schema.autoMerge.enabled", "true")) { sql(s""" |MERGE INTO ${tgt} |USING ${src} |on ${tgt}.c1 = ${src}.c1 |WHEN NOT MATCHED THEN INSERT (c1, C2) VALUES (${src}.c1, ${src}.c2) |""".stripMargin) } checkAnswer( sql(s"SELECT * FROM ${tgt}"), Seq(Row(1, 2, null), Row(2, 3, 4)) ) } } } test("generated columns with cdf") { val tableName1 = "gcEnabledCDCOn" val tableName2 = "gcEnabledCDCOff" withTable(tableName1, tableName2) { def readCdf(startingVersion: Long): DataFrame = { spark.read.format("delta").option("readChangeData", "true") .option("startingVersion", startingVersion) .table(tableName1) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) } createTable( tableName1, None, schemaString = "id LONG, timeCol TIMESTAMP, dateCol DATE", generatedColumns = Map( "dateCol" -> "CAST(timeCol AS DATE)" ), partitionColumns = Seq("dateCol"), properties = Map( "delta.enableChangeDataFeed" -> "true" ) ) checkAnswer(readCdf(startingVersion = 0), Seq()) spark.range(100).repartition(10) .withColumn( "timeCol", lit(sqlTimestamp("1970-01-01 00:00:00")) + make_dt_interval($"id")) .write .format("delta") .mode("append") .saveAsTable(tableName1) spark.sql(s"DELETE FROM $tableName1 WHERE id < 3") checkAnswer( readCdf(startingVersion = 2), Seq( Row(0, sqlTimestamp("1970-01-01 00:00:00"), sqlDate("1970-01-01"), "delete", 2), Row(1, sqlTimestamp("1970-01-02 00:00:00"), sqlDate("1970-01-02"), "delete", 2), Row(2, sqlTimestamp("1970-01-03 00:00:00"), sqlDate("1970-01-03"), "delete", 2) ) ) // Now write out the data frame of cdc to another table that has generated columns but not // cdc enabled. createTable( tableName2, None, schemaString = "id LONG, _change_type STRING, timeCol TIMESTAMP, dateCol DATE", generatedColumns = Map( "dateCol" -> "CAST(timeCol AS DATE)" ), partitionColumns = Seq("dateCol"), properties = Map( "delta.enableChangeDataFeed" -> "false" ) ) val cdcRead = spark.read.format("delta").option("readChangeData", "true") .option("startingVersion", "2") .table(tableName1) .select("id", CDCReader.CDC_TYPE_COLUMN_NAME, "timeCol") cdcRead .write .format("delta") .mode("append") .saveAsTable(tableName2) checkAnswer( cdcRead, spark.table(tableName2).drop("dateCol") ) } } test("not null should be enforced with generated columns") { withTableName("tbl") { tbl => createTable(tbl, None, "c1 INT, c2 STRING, c3 INT", Map("c3" -> "c1 + 1"), Seq.empty, Set("c1", "c2", "c3")) // try to write data without c2 in the DF val schemaWithoutColumnC2 = StructType( Seq(StructField("c1", IntegerType, true))) val data1 = List(Row(3)) val df1 = spark.createDataFrame(data1.asJava, schemaWithoutColumnC2) val e1 = intercept[DeltaInvariantViolationException] { df1.write.format("delta").mode("append").saveAsTable("tbl") } assert(e1.getMessage.contains("Column c2, which has a NOT NULL constraint," + " is missing from the data being written into the table.")) } } Seq(true, false).foreach { allowNullInsert => test("nullable column should work with generated columns - " + "allowNullInsert enabled=" + allowNullInsert) { withTableName("tbl") { tbl => withSQLConf(DeltaSQLConf.GENERATED_COLUMN_ALLOW_NULLABLE.key -> allowNullInsert.toString) { createTable( tbl, None, "c1 INT, c2 STRING, c3 INT", Map("c3" -> "c1 + 1"), Seq.empty) // create data frame that matches the table's schema val data1 = List(Row(1, "a1"), Row(2, "a2")) val schema = StructType( Seq(StructField("c1", IntegerType, true), StructField("c2", StringType, true))) val df1 = spark.createDataFrame(data1.asJava, schema) df1.write.format("delta").mode("append").saveAsTable("tbl") // create a data frame that does not have c2 val schemaWithoutOptionalColumnC2 = StructType( Seq(StructField("c1", IntegerType, true))) val data2 = List(Row(3)) val df2 = spark.createDataFrame(data2.asJava, schemaWithoutOptionalColumnC2) if (allowNullInsert) { df2.write.format("delta").mode("append").saveAsTable("tbl") // check correctness val expectedDF = df1 .union(df2.withColumn("c2", lit(null).cast(StringType))) .withColumn("c3", 'c1 + 1) checkAnswer(spark.read.table(tbl), expectedDF) } else { // when allow null insert is not enabled. val e = intercept[AnalysisException] { df2.write.format("delta").mode("append").saveAsTable("tbl") } e.getMessage.contains( "A column, variable, or function parameter with name `c2` cannot be resolved") } } } } } test("generated column metadata is not exposed in schema") { val tableName = "table" withTable(tableName) { createDefaultTestTable(tableName) Seq((1L, "foo", Timestamp.valueOf("2020-10-11 12:30:30"), 100, Date.valueOf("2020-11-12"))) .toDF("c1", "c3_p", "c5", "c6", "c8") .write.format("delta").mode("append").saveAsTable(tableName) val expectedSchema = new StructType() .add("c1", LongType) .add("c2_g", LongType) .add("c3_p", StringType) .add("c4_g_p", DateType) .add("c5", TimestampType) .add("c6", IntegerType) .add("c7_g_p", IntegerType) .add("c8", DateType) assert(spark.read.table(tableName).schema === expectedSchema) val ttDf = spark.read.option(DeltaOptions.VERSION_AS_OF, 0).table(tableName) assert(ttDf.schema === expectedSchema) val cdcDf = spark.read .option(DeltaOptions.CDC_READ_OPTION, true) .option(DeltaOptions.STARTING_VERSION_OPTION, 0) .table(tableName) assert(cdcDf.schema === expectedSchema .add("_change_type", StringType) .add("_commit_version", LongType) .add("_commit_timestamp", TimestampType) ) assert(spark.readStream.table(tableName).schema === expectedSchema) val cdcStreamDf = spark.readStream .option(DeltaOptions.CDC_READ_OPTION, true) .option(DeltaOptions.STARTING_VERSION_OPTION, 0) .table(tableName) assert(cdcStreamDf.schema === expectedSchema .add("_change_type", StringType) .add("_commit_version", LongType) .add("_commit_timestamp", TimestampType) ) } } test("DML into table with generated column, char column and readSideCharPadding=true") { val tableName = "table" withTable(tableName) { withSQLConf(SQLConf.READ_SIDE_CHAR_PADDING.key -> "true") { createTable(tableName, None, "c1 INT, c2 CHAR(5), c3 INT", Map("c3" -> "c1 + 1"), Nil) spark.sql( s""" |MERGE INTO $tableName AS TARGET |USING (SELECT id as c1, cast(id AS CHAR(5)) as c2 FROM RANGE(10)) AS SOURCE |ON TARGET.c1 = SOURCE.c1 |WHEN MATCHED THEN UPDATE SET c1 = SOURCE.c1, c2 = SOURCE.c2 |WHEN NOT MATCHED THEN INSERT (c1, c2) VALUES (SOURCE.c1, SOURCE.c2) |""".stripMargin) spark.sql(s"UPDATE $tableName SET c2 = 'upd' WHERE c1 = 1") } } } } class GeneratedColumnSuite extends GeneratedColumnSuiteBase ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/GeneratedColumnTest.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.io.PrintWriter import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.schema.{DeltaInvariantViolationException, InvariantViolationException, SchemaUtils} import org.apache.spark.sql.delta.sources.DeltaSourceUtils.GENERATION_EXPRESSION_METADATA_KEY import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.test.shims.StreamingTestShims.MemoryStream import io.delta.tables.DeltaTableBuilder import org.apache.spark.SparkConf import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, stringToDate, stringToTimestamp, toJavaDate, toJavaTimestamp} import org.apache.spark.sql.catalyst.util.quietly import org.apache.spark.sql.functions.{current_timestamp, lit} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.streaming.{StreamingQueryException, Trigger} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{ArrayType, DateType, IntegerType, MetadataBuilder, StringType, StructField, StructType, TimestampType} import org.apache.spark.unsafe.types.UTF8String trait GeneratedColumnTest extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with DeltaSQLTestUtils { protected def sqlDate(date: String): java.sql.Date = { toJavaDate(stringToDate(UTF8String.fromString(date)).get) } protected def sqlTimestamp(timestamp: String): java.sql.Timestamp = { toJavaTimestamp(stringToTimestamp( UTF8String.fromString(timestamp), getZoneId(SQLConf.get.sessionLocalTimeZone)).get) } protected def withTableName[T](tableName: String)(func: String => T): Unit = { withTable(tableName) { func(tableName) } } /** Create a new field with the given generation expression. */ def withGenerationExpression(field: StructField, expr: String): StructField = { val newMetadata = new MetadataBuilder() .withMetadata(field.metadata) .putString(GENERATION_EXPRESSION_METADATA_KEY, expr) .build() field.copy(metadata = newMetadata) } protected def buildTable( builder: DeltaTableBuilder, tableName: String, path: Option[String], schemaString: String, generatedColumns: Map[String, String], partitionColumns: Seq[String], notNullColumns: Set[String], comments: Map[String, String], properties: Map[String, String]): DeltaTableBuilder = { val schema = if (schemaString.nonEmpty) { StructType.fromDDL(schemaString) } else { new StructType() } val cols = schema.map(field => (field.name, field.dataType)) if (tableName != null) { builder.tableName(tableName) } cols.foreach(col => { val (colName, dataType) = col val nullable = !notNullColumns.contains(colName) var columnBuilder = io.delta.tables.DeltaTable.columnBuilder(spark, colName) columnBuilder.dataType(dataType.sql) columnBuilder.nullable(nullable) if (generatedColumns.contains(colName)) { columnBuilder.generatedAlwaysAs(generatedColumns(colName)) } if (comments.contains(colName)) { columnBuilder.comment(comments(colName)) } builder.addColumn(columnBuilder.build()) }) if (partitionColumns.nonEmpty) { builder.partitionedBy(partitionColumns: _*) } if (path.nonEmpty) { builder.location(path.get) } properties.foreach { case (key, value) => builder.property(key, value) } builder } protected def createTable( tableName: String, path: Option[String], schemaString: String, generatedColumns: Map[String, String], partitionColumns: Seq[String], notNullColumns: Set[String] = Set.empty, comments: Map[String, String] = Map.empty, properties: Map[String, String] = Map.empty): Unit = { var tableBuilder = io.delta.tables.DeltaTable.create(spark) buildTable(tableBuilder, tableName, path, schemaString, generatedColumns, partitionColumns, notNullColumns, comments, properties) .execute() } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/HiveConvertToDeltaSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaHiveTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.{AnalysisException, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.{col, from_json} import org.apache.spark.sql.hive.test.TestHiveSingleton abstract class HiveConvertToDeltaSuiteBase extends ConvertToDeltaHiveTableTests with DeltaSQLTestUtils { override protected def convertToDelta( identifier: String, partitionSchema: Option[String] = None, collectStats: Boolean = true): Unit = { if (partitionSchema.isEmpty) { sql(s"convert to delta $identifier ${collectStatisticsStringOption(collectStats)} ") } else { val stringSchema = partitionSchema.get sql(s"convert to delta $identifier ${collectStatisticsStringOption(collectStats)}" + s" partitioned by ($stringSchema) ") } } override protected def verifyExternalCatalogMetadata(tableName: String): Unit = { val catalogTable = spark.sessionState.catalog.externalCatalog.getTable("default", tableName) // Hive automatically adds some properties val cleanProps = catalogTable.properties.filterKeys(_ != "transient_lastDdlTime") // We can't alter the schema in the catalog at the moment :( assert(cleanProps.isEmpty, s"Table properties weren't empty for table $tableName: $cleanProps") } test("convert with statistics") { val tbl = "hive_parquet" withTable(tbl) { sql( s""" |CREATE TABLE $tbl (id int, str string) |PARTITIONED BY (part string) |STORED AS PARQUET """.stripMargin) sql(s"insert into $tbl VALUES (1, 'a', 1)") val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tbl)) convertToDelta(tbl, Some("part string"), collectStats = true) val deltaLog = DeltaLog.forTable(spark, catalogTable) val statsDf = deltaLog.unsafeVolatileSnapshot.allFiles .select( from_json(col("stats"), deltaLog.unsafeVolatileSnapshot.statsSchema).as("stats")) .select("stats.*") assert(statsDf.filter(col("numRecords").isNull).count == 0) val history = io.delta.tables.DeltaTable.forPath(catalogTable.location.getPath).history() assert(history.count == 1) } } test("convert without statistics") { val tbl = "hive_parquet" withTable(tbl) { sql( s""" |CREATE TABLE $tbl (id int, str string) |PARTITIONED BY (part string) |STORED AS PARQUET """.stripMargin) sql(s"insert into $tbl VALUES (1, 'a', 1)") val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tbl)) convertToDelta(tbl, Some("part string"), collectStats = false) val deltaLog = DeltaLog.forTable(spark, catalogTable) val statsDf = deltaLog.unsafeVolatileSnapshot.allFiles .select(from_json(col("stats"), deltaLog.unsafeVolatileSnapshot.statsSchema).as("stats")) .select("stats.*") assert(statsDf.filter(col("numRecords").isNotNull).count == 0) val history = io.delta.tables.DeltaTable.forPath(catalogTable.location.getPath).history() assert(history.count == 1) } } test("convert a Hive based parquet table") { val tbl = "hive_parquet" withTable(tbl) { sql( s""" |CREATE TABLE $tbl (id int, str string) |PARTITIONED BY (part string) |STORED AS PARQUET """.stripMargin) sql(s"insert into $tbl VALUES (1, 'a', 1)") val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tbl)) assert(catalogTable.provider === Some("hive")) assert(catalogTable.storage.serde.exists(_.contains("parquet"))) convertToDelta(tbl, Some("part string")) checkAnswer( sql(s"select * from delta.`${getPathForTableName(tbl)}`"), Row(1, "a", "1")) verifyExternalCatalogMetadata(tbl) val updatedTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tbl)) assert(updatedTable.provider === Some("delta")) } } test("convert a Hive based external parquet table") { val tbl = "hive_parquet" withTempDir { dir => withTable(tbl) { sql( s""" |CREATE EXTERNAL TABLE $tbl (id int, str string) |PARTITIONED BY (part string) |STORED AS PARQUET |LOCATION '${dir.getCanonicalPath}' """.stripMargin) sql(s"insert into $tbl VALUES (1, 'a', 1)") val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tbl)) assert(catalogTable.provider === Some("hive")) assert(catalogTable.storage.serde.exists(_.contains("parquet"))) convertToDelta(tbl, Some("part string")) checkAnswer( sql(s"select * from delta.`${dir.getCanonicalPath}`"), Row(1, "a", "1")) verifyExternalCatalogMetadata(tbl) val updatedTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tbl)) assert(updatedTable.provider === Some("delta")) } } } test("negative case: convert empty partitioned parquet table") { val tbl = "hive_parquet" withTempDir { dir => withTable(tbl) { sql( s""" |CREATE EXTERNAL TABLE $tbl (id int, str string) |PARTITIONED BY (part string) |STORED AS PARQUET |LOCATION '${dir.getCanonicalPath}' """.stripMargin) val ae = intercept[AnalysisException] { convertToDelta(tbl, Some("part string")) } assert(ae.getErrorClass == "DELTA_CONVERSION_NO_PARTITION_FOUND") assert(ae.getSqlState == "42KD6") assert(ae.getMessage.contains(tbl)) } } } } class HiveConvertToDeltaSuite extends HiveConvertToDeltaSuiteBase with DeltaHiveTest ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/HiveDeltaDDLSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaHiveTest import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.hive.test.TestHiveSingleton abstract class HiveDeltaDDLSuiteBase extends DeltaDDLTestBase { import testImplicits._ override protected def verifyNullabilityFailure(exception: AnalysisException): Unit = { exception.getMessage.contains("not supported for changing column") } } class HiveDeltaDDLSuite extends HiveDeltaDDLSuiteBase with DeltaHiveTest ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/HiveDeltaNotSupportedDDLSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.test.DeltaHiveTest import org.apache.spark.sql.hive.test.TestHiveSingleton class HiveDeltaNotSupportedDDLSuite extends DeltaNotSupportedDDLBase with DeltaHiveTest ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/IcebergCompatUtilsBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.UUID import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.{DataFrame, QueryTest, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier /** * The shared utils base to be extended by corresponding suites/traits. */ trait IcebergCompatUtilsBase extends QueryTest { override protected def spark: SparkSession protected val compatObject: IcebergCompatBase = null protected def compatVersion: Int = Option(compatObject).map(_.version.toInt).getOrElse(-1) protected def enableCompatTableProperty: String = compatObject.config.key protected val compatColumnMappingMode: String = "name" protected def compatTableFeature: TableFeature = compatObject.tableFeature protected val allReaderWriterVersions: Seq[(Int, Int)] = (1 to 3) .flatMap { r => (1 to 7).map(w => (r, w)) } // can only be at minReaderVersion >= 3 if minWriterVersion is >= 7 .filterNot { case (r, w) => w < 7 && r >= 3 } protected val defaultSchemaName: String = "default" protected val defaultCatalogName: String = "main" def getRndTableId: TableIdentifier = { val rndTableName = s"testTable${UUID.randomUUID()}" TableIdentifier(rndTableName, Some(defaultSchemaName), Some(defaultCatalogName)) } /** * Executes `f` with params (tableId, tempPath). * * We want to use a temp directory in addition to a unique temp table so that when the async * iceberg conversion runs and completes, the parent folder is still removed. */ protected def withTempTableAndDir(f: (String, String) => Unit): Unit protected def executeSql(sqlStr: String): DataFrame protected def getProperties(tableId: String): Map[String, String] = { val table = DeltaTableV2(spark, TableIdentifier(tableId)) table.update.getProperties.toMap } protected def assertIcebergCompatProtocolAndProperties( tableId: String, compatObj: IcebergCompatBase = compatObject): Unit = { val table = DeltaTableV2(spark, TableIdentifier(tableId)) val snapshot = table.update val protocol = snapshot.protocol val tblProperties = snapshot.getProperties val tableFeature = compatObj.tableFeature val expectedMinReaderVersion = Math.max( ColumnMappingTableFeature.minReaderVersion, tableFeature.minReaderVersion ) val expectedMinWriterVersion = Math.max( ColumnMappingTableFeature.minWriterVersion, tableFeature.minWriterVersion ) assert(protocol.minReaderVersion >= expectedMinReaderVersion) assert(protocol.minWriterVersion >= expectedMinWriterVersion) assert(protocol.writerFeatures.get.contains(tableFeature.name)) assert(tblProperties(compatObj.config.key) === "true") assert(Seq("name", "id").contains(tblProperties("delta.columnMapping.mode"))) } protected def parseIcebergVersion(metadataLocation: String): Int = { val versionStart = metadataLocation.lastIndexOf('/') + 1 val versionEnd = metadataLocation.indexOf('-', versionStart) if (versionEnd < 0) throw new RuntimeException( s"No version end found in $metadataLocation: $versionEnd") Integer.valueOf(metadataLocation.substring(versionStart, versionEnd)) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/IdentityColumnAdmissionSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.{File, FileNotFoundException, PrintWriter} import org.apache.spark.sql.delta.GeneratedAsIdentityType.GeneratedAlways import org.apache.spark.sql.delta.actions.RemoveFile import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.test.shims.StreamingTestShims.MemoryStream import org.apache.hadoop.fs.Path import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.streaming.{StreamingQueryException, Trigger} import org.apache.spark.sql.types.{DoubleType, IntegerType, LongType} import org.apache.spark.util.Utils // Test command that should be allowed and disallowed on IDENTITY columns. trait IdentityColumnAdmissionSuiteBase extends IdentityColumnTestUtils { import testImplicits._ protected override def sparkConf: SparkConf = { super.sparkConf .set(DeltaSQLConf.DELTA_IDENTITY_COLUMN_ENABLED.key, "true") } test("alter table change column type") { for { generatedAsIdentityType <- GeneratedAsIdentityType.values keyword <- Seq("ALTER", "CHANGE") targetType <- Seq(IntegerType, DoubleType) } { val tblName = getRandomTableName withIdentityColumnTable(generatedAsIdentityType, tblName) { targetType match { case IntegerType => // Long -> Integer (downcast) is rejected early during analysis by Spark. val ex = intercept[AnalysisException] { sql(s"ALTER TABLE $tblName $keyword COLUMN id TYPE ${targetType.sql}") } assert(ex.getErrorClass === "NOT_SUPPORTED_CHANGE_COLUMN") case DoubleType => // Long -> Double (upcast) is rejected in Delta when altering data type of an // identity column. val ex = intercept[DeltaAnalysisException] { sql(s"ALTER TABLE $tblName $keyword COLUMN id TYPE ${targetType.sql}") } assert(ex.getErrorClass === "DELTA_IDENTITY_COLUMNS_ALTER_COLUMN_NOT_SUPPORTED") case _ => fail("unexpected targetType") } } } } test("alter table change column comment") { for { generatedAsIdentityType <- GeneratedAsIdentityType.values keyword <- Seq("ALTER", "CHANGE") } { val tblName = getRandomTableName withIdentityColumnTable(generatedAsIdentityType, tblName) { sql(s"ALTER TABLE $tblName $keyword COLUMN id COMMENT 'comment'") } } } test("identity columns can be renamed") { for { generatedAsIdentityType <- GeneratedAsIdentityType.values } { val tblName = getRandomTableName withIdentityColumnTable(generatedAsIdentityType, tblName) { sql(s"ALTER TABLE $tblName SET TBLPROPERTIES ('delta.columnMapping.mode' = 'name')") sql(s"INSERT INTO $tblName (value) VALUES (1)") sql(s"INSERT INTO $tblName (value) VALUES (2)") checkAnswer(sql(s"SELECT id, value FROM $tblName"), Seq(Row(1, 1), Row(2, 2))) sql(s"ALTER TABLE $tblName RENAME COLUMN id TO id2") sql(s"INSERT INTO $tblName (value) VALUES (0)") checkAnswer(sql(s"SELECT id2, value FROM $tblName"), Seq(Row(1, 1), Row(2, 2), Row(3, 0))) } } } test("cannot set default value for identity column") { for (generatedAsIdentityType <- GeneratedAsIdentityType.values) { val tblName = getRandomTableName withIdentityColumnTable(generatedAsIdentityType, tblName) { val ex = intercept[DeltaAnalysisException] { sql(s"ALTER TABLE $tblName ALTER COLUMN id SET DEFAULT 1") } assert(ex.getMessage.contains("ALTER TABLE ALTER COLUMN is not supported")) } } } test("position of identity column can be moved") { for (generatedAsIdentityType <- GeneratedAsIdentityType.values) { val tblName = getRandomTableName withIdentityColumnTable(generatedAsIdentityType, tblName) { sql(s"ALTER TABLE $tblName ALTER COLUMN id AFTER value") sql(s"INSERT INTO $tblName (value) VALUES (1)") sql(s"INSERT INTO $tblName (value) VALUES (2)") checkAnswer(sql(s"SELECT id, value FROM $tblName"), Seq(Row(1, 1), Row(2, 2))) sql(s"ALTER TABLE $tblName ALTER COLUMN id FIRST") sql(s"INSERT INTO $tblName (value) VALUES (3)") checkAnswer(sql(s"SELECT id, value FROM $tblName"), Seq(Row(1, 1), Row(2, 2), Row(3, 3))) } } } test("alter table replace columns") { for (generatedAsIdentityType <- GeneratedAsIdentityType.values) { val tblName = getRandomTableName withIdentityColumnTable(generatedAsIdentityType, tblName) { val ex = intercept[DeltaAnalysisException] { sql(s"ALTER TABLE $tblName REPLACE COLUMNS (id BIGINT, value INT)") } assert(ex.getMessage.contains("ALTER TABLE REPLACE COLUMNS is not supported")) } } } test("create table partitioned by identity column") { for (generatedAsIdentityType <- GeneratedAsIdentityType.values) { val tblName = getRandomTableName withTable(tblName) { val ex1 = intercept[DeltaAnalysisException] { createTable( tblName, Seq( IdentityColumnSpec(generatedAsIdentityType), TestColumnSpec("value1", dataType = IntegerType), TestColumnSpec("value2", dataType = DoubleType) ), partitionedBy = Seq("id") ) } assert(ex1.getMessage.contains("PARTITIONED BY IDENTITY column")) val ex2 = intercept[DeltaAnalysisException] { createTable( tblName, Seq( IdentityColumnSpec(generatedAsIdentityType), TestColumnSpec("value1", dataType = IntegerType), TestColumnSpec("value2", dataType = DoubleType) ), partitionedBy = Seq("id", "value1") ) } assert(ex2.getMessage.contains("PARTITIONED BY IDENTITY column")) } } } test("replace with table partitioned by identity column") { for (generatedAsIdentityType <- GeneratedAsIdentityType.values) { val tblName = getRandomTableName withTable(tblName) { // First create a table with no identity column and no partitions. createTable( tblName, Seq( TestColumnSpec("id", dataType = LongType), TestColumnSpec("value1", dataType = IntegerType), TestColumnSpec("value2", dataType = DoubleType) ) ) // CREATE OR REPLACE should not allow a table using identity column with partition. val ex1 = intercept[DeltaAnalysisException] { createOrReplaceTable( tblName, Seq( IdentityColumnSpec(generatedAsIdentityType), TestColumnSpec("value1", dataType = IntegerType), TestColumnSpec("value2", dataType = DoubleType) ), partitionedBy = Seq("id") ) } assert(ex1.getMessage.contains("PARTITIONED BY IDENTITY column")) // REPLACE should also not allow a table using identity column as partition. val ex2 = intercept[DeltaAnalysisException] { replaceTable( tblName, Seq( IdentityColumnSpec(generatedAsIdentityType), TestColumnSpec("value1", dataType = IntegerType), TestColumnSpec("value2", dataType = DoubleType) ), partitionedBy = Seq("id", "value1") ) } assert(ex2.getMessage.contains("PARTITIONED BY IDENTITY column")) } } } test("CTAS does not inherit IDENTITY column") { for (generatedAsIdentityType <- GeneratedAsIdentityType.values) { val tblName = getRandomTableName val ctasTblName = getRandomTableName withIdentityColumnTable(generatedAsIdentityType, tblName) { withTable(ctasTblName) { sql(s"INSERT INTO $tblName (value) VALUES (1), (2)") sql( s""" |CREATE TABLE $ctasTblName USING delta AS SELECT * FROM $tblName |""".stripMargin) val dl = DeltaLog.forTable(spark, TableIdentifier(ctasTblName)) assert(!dl.snapshot.metadata.schemaString.contains(DeltaSourceUtils.IDENTITY_INFO_START)) } } } } test("insert generated always as") { val tblName = getRandomTableName withIdentityColumnTable(GeneratedAlways, tblName) { // Test SQLs. val blockedStmts = Seq( s"INSERT INTO $tblName VALUES (1,1)", s"INSERT INTO $tblName (value, id) VALUES (1,1)", s"INSERT OVERWRITE $tblName VALUES (1,1)", s"INSERT OVERWRITE $tblName (value, id) VALUES (1,1)" ) for (stmt <- blockedStmts) { val ex = intercept[DeltaAnalysisException](sql(stmt)) assert(ex.getMessage.contains("Providing values for GENERATED ALWAYS AS IDENTITY")) } // Test DataFrame V1 and V2 API. val df = (1 to 10).map(v => (v.toLong, v)).toDF("id", "value") val path = DeltaLog.forTable(spark, TableIdentifier(tblName)).dataPath.toString val exV1 = intercept[DeltaAnalysisException](df.write.format("delta").mode("append").save(path)) assert(exV1.getMessage.contains("Providing values for GENERATED ALWAYS AS IDENTITY")) val exV2 = intercept[DeltaAnalysisException](df.writeTo(tblName).append()) assert(exV2.getMessage.contains("Providing values for GENERATED ALWAYS AS IDENTITY")) } } test("streaming") { val tblName = getRandomTableName withIdentityColumnTable(GeneratedAlways, tblName) { val path = DeltaLog.forTable(spark, TableIdentifier(tblName)).dataPath.toString withTempDir { checkpointDir => val ex = intercept[StreamingQueryException] { val stream = MemoryStream[Int] val q = stream .toDF .map(_ => Tuple2(1L, 1)) .toDF("id", "value") .writeStream .format("delta") .outputMode("append") .option("checkpointLocation", checkpointDir.getCanonicalPath) .trigger(Trigger.AvailableNow) .start(path) stream.addData(1 to 10) q.processAllAvailable() q.stop() } assert(ex.getMessage.contains("Providing values for GENERATED ALWAYS AS IDENTITY")) } } } test("update") { for (generatedAsIdentityType <- GeneratedAsIdentityType.values) { val tblName = getRandomTableName withIdentityColumnTable(generatedAsIdentityType, tblName) { sql(s"INSERT INTO $tblName (value) VALUES (1), (2)") val blockedStatements = Seq( // Unconditional UPDATE. s"UPDATE $tblName SET id = 1", // Conditional UPDATE. s"UPDATE $tblName SET id = 1 WHERE value = 2" ) for (stmt <- blockedStatements) { val ex = intercept[DeltaAnalysisException](sql(stmt)) assert(ex.getMessage.contains("UPDATE on IDENTITY column")) } } } } test("merge") { for (generatedAsIdentityType <- GeneratedAsIdentityType.values) { val source = s"${getRandomTableName}_source" val target = s"${getRandomTableName}_target" withIdentityColumnTable(generatedAsIdentityType, target) { withTable(source) { sql( s""" |CREATE TABLE $source ( | value INT, | id BIGINT |) USING delta |""".stripMargin) sql( s""" |INSERT INTO $source VALUES (1, 100), (2, 200), (3, 300) |""".stripMargin) sql( s""" |INSERT INTO $target(value) VALUES (2), (3), (4) |""".stripMargin) val updateStmt = s""" |MERGE INTO $target | USING $source on $target.value = $source.value | WHEN MATCHED THEN UPDATE SET * |""".stripMargin val updateEx = intercept[DeltaAnalysisException](sql(updateStmt)) assert(updateEx.getMessage.contains("UPDATE on IDENTITY column")) val insertStmt = s""" |MERGE INTO $target | USING $source on $target.value = $source.value | WHEN NOT MATCHED THEN INSERT * |""".stripMargin if (generatedAsIdentityType == GeneratedAlways) { val insertEx = intercept[DeltaAnalysisException](sql(insertStmt)) assert( insertEx.getMessage.contains("Providing values for GENERATED ALWAYS AS IDENTITY")) } else { sql(insertStmt) } } } } } } class IdentityColumnAdmissionScalaSuite extends IdentityColumnAdmissionSuiteBase with ScalaDDLTestUtils ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/IdentityColumnConflictSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.concurrent.duration.Duration import scala.util.control.NonFatal import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils import org.apache.spark.sql.delta.concurrency.{PhaseLockingTestMixin, TransactionExecutionTestMixin} import org.apache.spark.sql.delta.fuzzer.{OptimisticTransactionPhases, PhaseLockingTransactionExecutionObserver} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.SparkConf import org.apache.spark.sql.execution.QueryExecution import org.apache.spark.util.ThreadUtils /** * Helper class used in this test suite for describing different transaction conflict scenarios. */ sealed trait TransactionConflictTestCase { /** label for this transaction scenario. */ def name: String /** SQL command to be executed. */ def sqlCommand: String /** Boolean indicating whether this transaction does a metadata update. */ def hasMetadataUpdate: Boolean /** Boolean indicating whether the SQL command appends data (add files) to the table. */ def isAppend: Boolean } case class NoMetadataUpdateTestCase( name: String, sqlCommand: String, isAppend: Boolean) extends TransactionConflictTestCase { val hasMetadataUpdate = false } /** * A transaction that will do a metadata update but will not be tagged as identity column only nor * row tracking enablement only. */ case class GenericMetadataUpdateTestCase( name: String, sqlCommand: String, isAppend: Boolean) extends TransactionConflictTestCase { val hasMetadataUpdate = true } /** A transaction that will be tagged as a metadata update only for identity column. */ case class IdentityOnlyMetadataUpdateTestCase( name: String, sqlCommand: String, isAppend: Boolean) extends TransactionConflictTestCase { val hasMetadataUpdate = true } /** A transaction that will be tagged as a metadata update only for row tracking enablement. */ case class RowTrackingEnablementOnlyTestCase( name: String, sqlCommand: String, isAppend: Boolean) extends TransactionConflictTestCase { val hasMetadataUpdate = true } trait IdentityColumnConflictSuiteBase extends IdentityColumnTestUtils with TransactionExecutionTestMixin with PhaseLockingTestMixin { override def sparkConf: SparkConf = super.sparkConf .set(DeltaSQLConf.DELTA_ROW_TRACKING_BACKFILL_ENABLED.key, "true") .set(DeltaSQLConf.FEATURE_ENABLEMENT_CONFLICT_RESOLUTION_ENABLED.key, "true") val colName = "id" private def setupEmptyTableWithRowTrackingTableFeature( tblIsoLevel: Option[IsolationLevel], tblName: String): Unit = { val tblPropertiesMap: Map[String, String] = Map( TableFeatureProtocolUtils.propertyKey(RowTrackingFeature) -> "supported", DeltaConfigs.ROW_TRACKING_ENABLED.key -> "false", DeltaConfigs.ISOLATION_LEVEL.key -> tblIsoLevel.map(_.toString).getOrElse(DeltaConfigs.ISOLATION_LEVEL.defaultValue) ) createTableWithIdColAndIntValueCol( tableName = tblName, generatedAsIdentityType = GeneratedAsIdentityType.GeneratedByDefault, startsWith = Some(1), incrementBy = Some(1), tblProperties = tblPropertiesMap ) } /** * Returns the expected exception class for the test case. * Returns None if no exception is expected. */ protected def expectedExceptionClass( currentTxn: TransactionConflictTestCase, winningTxn: TransactionConflictTestCase): Option[Class[_ <: RuntimeException]] = { val currentTxnShouldAbortDueToMetadataUpdate = winningTxn match { case _: NoMetadataUpdateTestCase => false case _: IdentityOnlyMetadataUpdateTestCase if !currentTxn.hasMetadataUpdate => false case _: RowTrackingEnablementOnlyTestCase if !currentTxn.hasMetadataUpdate => false case _ => true } // Metadata update is checked before concurrent append in ConflictChecker. if (currentTxnShouldAbortDueToMetadataUpdate) { return Some(classOf[io.delta.exceptions.MetadataChangedException]) } val currentTxnShouldAbortDueToConcurrentAppend = winningTxn.isAppend && currentTxn.isInstanceOf[IdentityOnlyMetadataUpdateTestCase] && !currentTxn.isAppend if (currentTxnShouldAbortDueToConcurrentAppend) { return Some(classOf[io.delta.exceptions.ConcurrentAppendException]) } None } /** Executes the winning transaction SQL. Overridable for custom RPC assertions. */ protected def sqlWithTotalRpcBound(sqlText: String): Unit = sql(sqlText) /** * Helper function to test two concurrently running commands. Winning transaction commits before * current transaction commits. */ protected def transactionIdentityConflictHelper( currentTxn: TransactionConflictTestCase, winningTxn: TransactionConflictTestCase, tblIsoLevel: Option[IsolationLevel]): Unit = { val tblName = getRandomTableName withTable(tblName) { // We start with an empty table that has row tracking table feature support and row tracking // table property disabled. This way, when we set the table property to true, it will not // also do a protocol upgrade and we don't need any backfill commit. setupEmptyTableWithRowTrackingTableFeature(tblIsoLevel, tblName) val threadPool = ThreadUtils.newDaemonSingleThreadExecutor(threadName = "identity-column-thread-pool") val (txnObserver, future) = runQueryWithObserver( name = "current", threadPool, currentTxn.sqlCommand.replace("{tblName}", tblName)) unblockUntilPreCommit(txnObserver) busyWaitFor(txnObserver.phases.preparePhase.hasEntered, timeout) sqlWithTotalRpcBound(winningTxn.sqlCommand.replace("{tblName}", tblName)) val expectedException = expectedExceptionClass(currentTxn, winningTxn) val events = Log4jUsageLogger.track { try { unblockCommit(txnObserver) ThreadUtils.awaitResult(future, Duration.Inf) assert(expectedException.isEmpty, "Expected txn to fail, but no exception was thrown") } catch { case NonFatal(e) => expectedException match { case None => fail("Expecting no exception, but an exception was thrown", e) case Some(expected) if (e.getCause == null) || e.getCause.getClass != expected => fail(s"Expected exception of type ${expected.getName}, " + "but got a different exception", e) case Some(_) => // Expected exception was thrown, test passes. } } } // We should log if the txn is aborted due to identity column only metadata update. if (currentTxn.hasMetadataUpdate && winningTxn.isInstanceOf[IdentityOnlyMetadataUpdateTestCase]) { val identityColumnAbortEvents = events .filter(_.tags.get("opType").contains(IdentityColumn.opTypeAbort)) assert(identityColumnAbortEvents.size === 1) } } } // scalastyle:off line.size.limit /** * We are testing the following combinations (see [[ConflictChecker.checkNoMetadataUpdates]] * for details). * * | | Winning Metadata (id) | Winning Metadata Row Tracking Enablement Only | Winning Metadata (other) | Winning No Metadata | * | --------------------------------------------- | --------------------- | --------------------------------------------- | ------------------------ | ------------------- | * | Current Metadata (id) | Conflict | Conflict | Conflict | No conflict | * | Current Metadata Row Tracking Enablement Only | Conflict | Conflict | Conflict | No conflict | * | Current Metadata (other) | Conflict | Conflict | Conflict | No conflict | * | Current No Metadata | No conflict | No conflict | Conflict | No conflict | */ // scalastyle:on line.size.limit // System generated IDENTITY value will have a metadata update for IDENTITY high water marks. private val generatedIdTestCase = IdentityOnlyMetadataUpdateTestCase( name = "generatedId", sqlCommand = s"INSERT INTO {tblName}(value) VALUES (1)", isAppend = true ) // SYNC IDENTITY updates the high water mark based on the values in the IDENTITY column. private val syncIdentityTestCase = IdentityOnlyMetadataUpdateTestCase( name = "syncIdentity", sqlCommand = s"ALTER TABLE {tblName} ALTER COLUMN $colName SYNC IDENTITY", isAppend = false ) // Explicitly provided IDENTITY value will not generate a metadata update. private val noMetadataUpdateTestCase = NoMetadataUpdateTestCase( name = "noMetadataUpdate", sqlCommand = s"INSERT INTO {tblName} VALUES (1, 1)", isAppend = true ) private val rowTrackingEnablementTestCase = RowTrackingEnablementOnlyTestCase( name = "rowTrackingEnablement", sqlCommand = s"""ALTER TABLE {tblName} |SET TBLPROPERTIES( |'${DeltaConfigs.ROW_TRACKING_ENABLED.key}' = 'true' |)""".stripMargin, isAppend = false ) private val otherMetadataUpdateTestCase = GenericMetadataUpdateTestCase( name = "otherMetadataUpdate", sqlCommand = s"ALTER TABLE {tblName} ADD COLUMN value2 STRING", isAppend = false ) protected val conflictTestCases: Seq[TransactionConflictTestCase] = Seq( generatedIdTestCase, syncIdentityTestCase, noMetadataUpdateTestCase, rowTrackingEnablementTestCase, otherMetadataUpdateTestCase ) for { currentTxn <- conflictTestCases winningTxn <- conflictTestCases } { val testName = s"identity conflict test: [currentTxn: ${currentTxn.name}, winningTxn: ${winningTxn.name}]" test(testName) { transactionIdentityConflictHelper( currentTxn, winningTxn, tblIsoLevel = None ) } } test("ALTER TABLE SYNC IDENTITY conflict on serializable table") { transactionIdentityConflictHelper( syncIdentityTestCase, noMetadataUpdateTestCase, tblIsoLevel = Some(Serializable) ) } test("high watermark changes after analysis but before execution of merge") { val tblName = getRandomTableName withIdentityColumnTable(GeneratedAsIdentityType.GeneratedAlways, tblName) { // Create a QueryExecution object for a MERGE statement, and it forces the command to be // analyzed, but does not execute the command yet. val parsedMerge = spark.sessionState.sqlParser.parsePlan( s"""MERGE INTO $tblName t |USING (SELECT * FROM range(1000)) s |ON t.id = s.id |WHEN NOT MATCHED THEN INSERT (value) VALUES (s.id)""".stripMargin) val qeMerge = new QueryExecution(spark, parsedMerge) qeMerge.analyzed // Insert a row, forcing the high watermark to be updated. sql(s"INSERT INTO $tblName (value) VALUES (0)") // Force merge to be executed. This should fail, as MERGE is still using the old high // watermark in its insert action. intercept[MetadataChangedException] { qeMerge.commandExecuted } } } } class IdentityColumnConflictScalaSuite extends IdentityColumnConflictSuiteBase with ScalaDDLTestUtils ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/IdentityColumnDMLSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import com.databricks.spark.util.{Log4jUsageLogger, MetricDefinitions} import org.apache.spark.sql.delta.GeneratedAsIdentityType.{GeneratedAlways, GeneratedByDefault} import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.commands.merge.MergeStats import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.sql.{AnalysisException, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode import org.apache.spark.sql.functions.col import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ /** * Identity Column test suite for DML operations, including INSERT REPLACE WHERE. */ trait IdentityColumnDMLSuiteBase extends IdentityColumnTestUtils { import testImplicits._ test("delete") { for (generatedAsIdentityType <- GeneratedAsIdentityType.values) { val tblName = getRandomTableName withIdentityColumnTable(generatedAsIdentityType, tblName) { sql(s"INSERT INTO $tblName (value) VALUES (1), (2)") val prevMax = sql(s"SELECT MAX(id) FROM $tblName").collect().head.getLong(0) sql(s"DELETE FROM $tblName WHERE value = 1") checkAnswer( sql(s"SELECT COUNT(*) FROM $tblName"), Row(1L) ) sql(s"DELETE FROM $tblName") checkAnswer( sql(s"SELECT COUNT(*) FROM $tblName"), Row(0L) ) sql(s"INSERT INTO $tblName (value) VALUES (1), (2)") checkAnswer( sql(s"SELECT COUNT(*) FROM $tblName where id <= $prevMax"), Row(0L) ) } } } test("merge with insert and update") { val start = 1L val step = 2L withSQLConf( DeltaSQLConf.MERGE_USE_PERSISTENT_DELETION_VECTORS.key -> "false") { val source = s"${getRandomTableName}_src" val target = s"${getRandomTableName}_tgt" withTable(source, target) { var highWaterMark = start - step createTable( source, Seq( TestColumnSpec(colName = "value", dataType = IntegerType), TestColumnSpec(colName = "value2", dataType = IntegerType) ) ) sql( s""" |INSERT INTO $source VALUES (1, 100), (2, 200), (3, 300) |""".stripMargin) createTableWithIdColAndIntValueCol( target, GeneratedAlways, startsWith = Some(start), incrementBy = Some(step)) sql( s""" |INSERT INTO $target(value) VALUES (2), (3), (4) |""".stripMargin) highWaterMark = validateIdentity(target, 3, start, step, 2, 4, highWaterMark) val idBeforeMerge1 = sql(s"SELECT id FROM $target WHERE value in (2, 3)").collect() sql( s""" |MERGE INTO $target | USING $source on $target.value = $source.value | WHEN MATCHED THEN UPDATE SET $target.value = $source.value2 | WHEN NOT MATCHED THEN INSERT (value) VALUES ($source.value2) |""".stripMargin) highWaterMark = validateIdentity(target, 4, start, step, 100, 100, highWaterMark) // IDENTITY values for updated rows shouldn't change. checkAnswer( sql(s"SELECT id FROM $target WHERE value in (200, 300)"), idBeforeMerge1 ) val idBeforeMerge2 = sql(s"SELECT id FROM $target WHERE value in (100, 300, 4)").collect() sql(s"INSERT OVERWRITE $source VALUES(200, 2000), (4, 400), (5, 500)") sql( s""" |MERGE INTO $target | USING $source on $target.value = $source.value | WHEN MATCHED AND $source.value = 200 THEN DELETE | WHEN MATCHED THEN UPDATE SET $target.value = $source.value2 | WHEN NOT MATCHED THEN INSERT (value) VALUES ($source.value2) |""".stripMargin) highWaterMark = validateIdentity(target, 4, start, step, 500, 500, highWaterMark) // IDENTITY values for updated rows shouldn't change. checkAnswer( sql(s"SELECT id FROM $target WHERE value in (100, 300, 400)"), idBeforeMerge2 ) } } } test("merge with insert and update and schema evolution") { val start = 1L val step = 3L withSQLConf( "spark.databricks.delta.schema.autoMerge.enabled"-> "true", DeltaSQLConf.MERGE_USE_PERSISTENT_DELETION_VECTORS.key -> "false") { val source = s"${getRandomTableName}_src" val target = s"${getRandomTableName}_tgt" withTable(source, target) { var highWaterMark = start - step createTable( source, Seq( TestColumnSpec(colName = "id2", dataType = LongType), TestColumnSpec(colName = "value2", dataType = IntegerType) ) ) sql(s"INSERT INTO $source VALUES (4, 44), (9, 99)") createTable( target, Seq( IdentityColumnSpec( GeneratedAlways, startsWith = Some(start), incrementBy = Some(step) ), TestColumnSpec(colName = "id2", dataType = LongType), TestColumnSpec(colName = "value", dataType = IntegerType) ) ) sql(s"INSERT INTO $target (id2, value) VALUES(1, 1), (4, 4), (7, 7), (10, 10)") highWaterMark = validateIdentity(target, 4, start, step, 1, 10, highWaterMark) val idBeforeMerge1 = sql(s"SELECT id FROM $target WHERE id2 in (1, 4, 7, 10)") sql( s""" |MERGE INTO $target | USING $source on $target.id2 = $source.id2 | WHEN NOT MATCHED THEN INSERT * |""".stripMargin) checkAnswer( sql(s"SELECT id FROM $target WHERE id2 in (1, 4, 7, 10)"), idBeforeMerge1 ) checkAnswer( sql(s"SELECT COUNT(DISTINCT id) == COUNT(*) FROM $target"), Row(true) ) val idBeforeMerge2 = sql(s"SELECT id FROM $target WHERE id2 in (1, 4, 7, 9, 10)") sql(s"INSERT OVERWRITE $source VALUES(9, 999), (11, 1100)") val events = Log4jUsageLogger.track { sql( s""" |MERGE INTO $target | USING $source on $target.id2 = $source.id2 | WHEN MATCHED THEN UPDATE SET $target.value = $source.value2 | WHEN NOT MATCHED THEN INSERT * |""".stripMargin) } checkAnswer( sql(s"SELECT id FROM $target WHERE id2 in (1, 4, 7, 9, 10)"), idBeforeMerge2 ) checkAnswer( sql(s"SELECT COUNT(DISTINCT id) == COUNT(*) FROM $target"), Row(true) ) val mergeStats = events.filter { e => e.metric == MetricDefinitions.EVENT_TAHOE.name && e.tags.get("opType").contains("delta.dml.merge.stats") } assert(mergeStats.size == 1) } } } test("MERGE/UPDATE/DELETE which does not INSERT any new data but just touches old rows" + " should not change the HIGH WATERMARK") { for { increment <- Seq(1, -1) } { withSQLConf( DeltaSQLConf.MERGE_USE_PERSISTENT_DELETION_VECTORS.key -> "false") { val src = s"${getRandomTableName}_src" val tgt = s"${getRandomTableName}_tgt" withTable(src, tgt) { sql(s"DROP TABLE IF EXISTS $tgt") createTable( tgt, Seq( IdentityColumnSpec( GeneratedAlways, startsWith = Some(0), incrementBy = Some(increment) ), TestColumnSpec(colName = "col1", dataType = IntegerType) ) ) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tgt)) assert(deltaLog.snapshot.version === 0L) // INSERT 10 rows where each row is inserted into different file. (1 to 10) .toDF("col1") .repartition(10) .write.mode("overwrite").format("delta").saveAsTable(tgt) assert(deltaLog.snapshot.version === 1L) assert(highWaterMark(deltaLog.snapshot, "id") === increment * 9L) // Create src table with only 1 row having value 5. Seq(5).toDF("col1").write.saveAsTable(src) // The MERGE query will just UPDATE only one file that has one row only. sql( s""" | MERGE INTO $tgt tgt USING $src src | ON src.col1 = tgt.col1 | WHEN MATCHED | THEN UPDATE SET tgt.col1 = 100 | WHEN NOT MATCHED THEN INSERT (tgt.col1) VALUES (src.col1) | """.stripMargin).collect() assert(deltaLog.snapshot.version === 2L) // The MERGE query shouldn't change the high watermark as it has not INSERTED any new // data. assert(highWaterMark(deltaLog.snapshot, "id") === increment * 9L) // Write 10 more rows to the table using single task and make sure that HIGH WATERMARK is // moved 10 units in either direction. val newDfToWrite = (11 to 20).toDF("col1").repartition(1) newDfToWrite.write.format("delta").mode("append").saveAsTable(tgt) assert(highWaterMark(deltaLog.snapshot, "id") === increment * 19L) // validate no duplicate identity values checkAnswer(sql(s"SELECT COUNT(DISTINCT id) == COUNT(*) FROM $tgt"), Row(true)) } } } } // Helper function to test multiple "WHEN NOT MATCHED THEN INSERT" clauses in a single MERGE with // different variations - enable/disable WSCG, vary num partitions in the identity column // generation stage. private def testMergeWithMultipleWhenNotMatchedClauses( numPartitions: Int = 2, codegenEnabled: Boolean = true): Unit = { val codegenFactoryMode = if (codegenEnabled) { CodegenObjectFactoryMode.CODEGEN_ONLY } else { CodegenObjectFactoryMode.NO_CODEGEN } withSQLConf( SQLConf.CODEGEN_FACTORY_MODE.key -> codegenFactoryMode.toString, DeltaSQLConf.MERGE_USE_PERSISTENT_DELETION_VECTORS.key -> "false", SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> s"$codegenEnabled") { val src = s"${getRandomTableName}_src" val tgt = s"${getRandomTableName}_tgt" withTable(src, tgt) { (1 to 10) .map(i => (i, i % 2)) .toDF("col1", "col2") .repartition(numPartitions) .write.mode("overwrite") .saveAsTable(src) sql(s"DROP TABLE IF EXISTS $tgt") createTable( tgt, Seq( IdentityColumnSpec( GeneratedAlways, startsWith = Some(5), incrementBy = Some(5), colName = "id1" ), IdentityColumnSpec( GeneratedAlways, startsWith = Some(0), incrementBy = Some(3), colName = "id2" ), TestColumnSpec(colName = "col1", dataType = LongType), TestColumnSpec(colName = "col2", dataType = LongType) ) ) sql(s"INSERT INTO $tgt (col1, col2) VALUES (5, 100), (6, 101)") sql( s""" | MERGE INTO $tgt tgt USING $src src | ON src.col1 = tgt.col1 | WHEN MATCHED AND tgt.col1 == 5 | THEN UPDATE SET tgt.col2 = src.col2 | WHEN NOT MATCHED AND src.col1 % 3 != 0 | THEN INSERT (tgt.col1, tgt.col2) VALUES (src.col1, src.col2) | WHEN NOT MATCHED AND src.col1 % 3 == 0 | THEN INSERT (tgt.col1, tgt.col2) VALUES (src.col1, src.col2) | """.stripMargin).collect() Seq("id1", "id2").foreach { idCol => checkAnswer( sql(s"SELECT COUNT(DISTINCT $idCol) == COUNT(*) FROM $tgt"), Row(true)) } assert(sql(s"SELECT * FROM $tgt WHERE id1 % 5 != 0").count() === 0) assert(sql(s"SELECT * FROM $tgt WHERE id2 % 3 != 0").count() === 0) checkAnswer( sql(s"SELECT col1, col2 FROM $tgt"), Seq(Row(1, 1), Row(2, 0), Row(3, 1), Row(4, 0), Row(5, 1), Row(6, 101), Row(7, 1), Row(8, 0), Row(9, 1), Row(10, 0)) ) } } } test(s"MERGE with multiple WHEN NOT MATCHED THEN INSERT clauses") { testMergeWithMultipleWhenNotMatchedClauses() } test(s"MERGE with multiple WHEN NOT MATCHED THEN INSERT clauses + WholeStageCodeGen disabled") { testMergeWithMultipleWhenNotMatchedClauses(codegenEnabled = false) } test(s"MERGE with multiple WHEN NOT MATCHED THEN INSERT clauses + single partition") { testMergeWithMultipleWhenNotMatchedClauses(numPartitions = 1) } private def testReplaceWhereWithCDF(isPartitioned: Boolean): Unit = { val start = 1L val step = 2L withSQLConf(DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> "true") { val table = getRandomTableName withTable(table) { var highWaterMarkFromData = start - step createTable( table, Seq( IdentityColumnSpec( GeneratedAlways, startsWith = Some(start), incrementBy = Some(step) ), TestColumnSpec(colName = "value", dataType = IntegerType), TestColumnSpec(colName = "is_odd", dataType = BooleanType), TestColumnSpec(colName = "is_even", dataType = BooleanType) ), partitionedBy = if (isPartitioned) Seq("is_odd") else Nil ) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(table)) def highWatermarkFromDeltaLog(): Long = highWaterMark(deltaLog.update(), "id") Seq(1, 2, 3, 4).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .withColumn("is_even", $"value" % 2 === 0) .coalesce(1) .write .format("delta") .mode("append") .saveAsTable(table) highWaterMarkFromData = validateIdentity( table, expectedRowCount = 4, start = start, step = step, minValue = 1, maxValue = 4, oldHighWaterMark = highWaterMarkFromData) assert(highWaterMarkFromData === highWatermarkFromDeltaLog()) val idBeforeReplaceWhere1 = sql(s"SELECT id FROM $table WHERE is_even = true").collect() Seq(5, 7).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .withColumn("is_even", $"value" % 2 === 0) .coalesce(1) .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "is_odd = true") .saveAsTable(table) highWaterMarkFromData = validateIdentity( table, expectedRowCount = 4, start = start, step = step, minValue = 5, maxValue = 7, oldHighWaterMark = highWaterMarkFromData) assert(highWaterMarkFromData === highWatermarkFromDeltaLog()) // IDENTITY values for not-updated shouldn't change. checkAnswer( sql(s"SELECT id FROM $table WHERE is_even = true"), idBeforeReplaceWhere1 ) // IDENTITY VALUES for inserted change records and new data should be consistent. checkAnswer( sql(s"SELECT id FROM table_changes('$table', 2, 2) " + "WHERE is_odd = true and _change_type = 'insert'"), sql(s"SELECT id FROM $table WHERE is_odd = true") ) val idBeforeReplaceWhere2 = sql(s"SELECT id FROM $table WHERE is_odd = true").collect() Seq(10, 12).toDF() .withColumn("is_odd", $"value" % 2 =!= 0) .withColumn("is_even", $"value" % 2 === 0) .coalesce(1) .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "is_even = true") .saveAsTable(table) highWaterMarkFromData = validateIdentity( table, expectedRowCount = 4, start = start, step = step, minValue = 10, maxValue = 12, oldHighWaterMark = highWaterMarkFromData) assert(highWaterMarkFromData === highWatermarkFromDeltaLog()) // IDENTITY values for not-updated shouldn't change. checkAnswer( sql(s"SELECT id FROM $table WHERE is_odd = true"), idBeforeReplaceWhere2 ) // IDENTITY VALUES for inserted change records and data should be consistent. checkAnswer( sql(s"SELECT id FROM table_changes('$table', 3, 3) " + "WHERE is_even = true and _change_type = 'insert'"), sql(s"SELECT id FROM $table WHERE is_even = true") ) // ReplaceWhere source data contains an Identity Column will be blocked. val e = intercept[AnalysisException] { Seq((15, 14), (17, 16)).toDF("id", "value") .withColumn("is_odd", $"value" % 2 =!= 0) .withColumn("is_even", $"value" % 2 === 0) .coalesce(1) .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "is_even = true") .saveAsTable(table) } assert(e.getMessage.contains( "Providing values for GENERATED ALWAYS AS IDENTITY column id is not supported.")) } } } Seq(true, false).foreach { isPartitioned => test(s"replaceWhere with CDF enabled [isPartitioned: $isPartitioned]") { testReplaceWhereWithCDF(isPartitioned) } } test("CDF for tables with identity column - MERGE") { val src = s"${getRandomTableName}_src" val tgt = s"${getRandomTableName}_tgt" withTable(tgt) { generateTableWithIdentityColumn(tgt) sql(s"ALTER TABLE $tgt SET TBLPROPERTIES (${DeltaConfigs.CHANGE_DATA_FEED.key}=true)") withTempView(src) { (1 :: 20 :: 30 :: Nil).toDF("value").createOrReplaceTempView(src) sql( s"""MERGE INTO $tgt target |USING $src src ON target.value = src.value |WHEN MATCHED THEN UPDATE SET target.value = src.value |WHEN NOT MATCHED THEN INSERT (target.value) VALUES (src.value)""".stripMargin) } val sortedResult = sql( s"""SELECT id, value |FROM $tgt |ORDER BY id""".stripMargin) .as[IdentityColumnTestTableRow] .collect() assert(sortedResult.length === 8) checkGeneratedIdentityValues( sortedRows = sortedResult, start = 0, step = 1, expectedLowerBound = 0, expectedUpperBound = 20, expectedDistinctCount = 8) def getIdForValue(value: Int): Long = { val rowWithValue = sortedResult.filter(_.value == value.toString) assert( rowWithValue.length === 1, s"Expected 1 row for value $value, found ${rowWithValue.length}") rowWithValue.head.id } // Validate the ids in CDCReader match those in logical data. checkAnswer(sql( s"""SELECT id, value, ${CDCReader.CDC_TYPE_COLUMN_NAME} |FROM table_changes('$tgt', 8) |ORDER BY value, id, _change_type""".stripMargin), Seq( Row(1, 1, CDCReader.CDC_TYPE_UPDATE_POSTIMAGE), Row(1, 1, CDCReader.CDC_TYPE_UPDATE_PREIMAGE), Row(getIdForValue(value = 20), 20, CDCReader.CDC_TYPE_INSERT), Row(getIdForValue(value = 30), 30, CDCReader.CDC_TYPE_INSERT))) } } test("CDF for tables with identity column - UPDATE") { val tgt = s"${getRandomTableName}_tgt" withTable(tgt) { generateTableWithIdentityColumn(tgt) sql(s"ALTER TABLE $tgt SET TBLPROPERTIES (${DeltaConfigs.CHANGE_DATA_FEED.key}=true)") sql( s"""UPDATE $tgt |SET value = value + 100 |WHERE id < 3""".stripMargin) checkAnswer(sql( s"""SELECT * |FROM $tgt |ORDER BY id, value""".stripMargin), Seq( Row(0, 100), Row(1, 101), Row(2, 102), Row(3, 3), Row(4, 4), Row(5, 5))) checkAnswer(sql( s"""SELECT id, value, ${CDCReader.CDC_TYPE_COLUMN_NAME} |FROM table_changes('$tgt', 8) |ORDER BY id, value, _change_type""".stripMargin), Seq( Row(0, 0, CDCReader.CDC_TYPE_UPDATE_PREIMAGE), Row(0, 100, CDCReader.CDC_TYPE_UPDATE_POSTIMAGE), Row(1, 1, CDCReader.CDC_TYPE_UPDATE_PREIMAGE), Row(1, 101, CDCReader.CDC_TYPE_UPDATE_POSTIMAGE), Row(2, 2, CDCReader.CDC_TYPE_UPDATE_PREIMAGE), Row(2, 102, CDCReader.CDC_TYPE_UPDATE_POSTIMAGE))) } } test("UPDATE cannot lead to bad high watermarks") { val tblName = getRandomTableName withTable(tblName) { createTable( tblName, Seq( IdentityColumnSpec( GeneratedByDefault, startsWith = Some(1), incrementBy = Some(1)), TestColumnSpec(colName = "value", dataType = IntegerType) ), partitionedBy = Seq("value"), tblProperties = Map( DeltaConfigs.CHANGE_DATA_FEED.key -> "true", DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key -> "false" ) ) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tblName)) sql(s"INSERT INTO $tblName(id, value) VALUES (-5, -5), (-3, -3), (-1, -1)") val valuesStr = (-999 to -900).map(id => s"($id, -3)").mkString(", ") sql(s"INSERT INTO $tblName(id, value) VALUES $valuesStr") sql(s"INSERT INTO $tblName(id, value) VALUES (-1, -1)") assert(getHighWaterMark(deltaLog.update(), colName = "id").isEmpty, "High watermark should not be set for user inserted values") Seq((-1000L, -3)).toDF("id", "value") .write .format("delta") .mode("overwrite") .option(DeltaOptions.REPLACE_WHERE_OPTION, "value = -3 and id <= -987") .saveAsTable(tblName) assert(getHighWaterMark(deltaLog.update(), colName = "id").isEmpty, "High watermark should not be set for user inserted values") sql(s"UPDATE $tblName SET value = -3 WHERE id = -1") assert(getHighWaterMark(deltaLog.update(), colName = "id").isEmpty, "Updates should not update high watermark") } } } class IdentityColumnDMLScalaSuite extends IdentityColumnDMLSuiteBase with ScalaDDLTestUtils class IdentityColumnDMLScalaIdColumnMappingSuite extends IdentityColumnDMLSuiteBase with ScalaDDLTestUtils with DeltaColumnMappingEnableIdMode class IdentityColumnDMLScalaNameColumnMappingSuite extends IdentityColumnDMLSuiteBase with ScalaDDLTestUtils with DeltaColumnMappingEnableNameMode ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/IdentityColumnIngestionSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.PrintWriter import org.apache.spark.sql.delta.GeneratedAsIdentityType.{GeneratedAlways, GeneratedByDefault} import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf} import org.apache.spark.sql.delta.test.shims.StreamingTestShims.MemoryStream import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.types._ /** * Identity Column test suite for ingestion, including insert-only MERGE. * Tests with identity columns where MERGE does data modification should be * in IdentityColumnDMLSuiteBase. */ trait IdentityColumnIngestionSuiteBase extends IdentityColumnTestUtils { import testImplicits._ private val tempCsvFileName = "test.csv" /** Helper function to write a single 'value' column into `sourcePath`. */ private def setupSimpleCsvFiles(sourcePath: String, start: Int, end: Int): Unit = { val writer = new PrintWriter(s"$sourcePath/$tempCsvFileName") // Write header. writer.write("value\n") // Write values. (start to end).foreach { v => writer.write(s"$v\n") } writer.close() } object IngestMode extends Enumeration { // Ingest using data frame append v1. val appendV1 = Value // Ingest using data frame append v2. val appendV2 = Value // Ingest using "INSERT INTO ... VALUES". val insertIntoValues = Value // Ingest using "INSERT INTO ... SELECT ...". val insertIntoSelect = Value // Ingest using "INSERT OVERWRITE ... VALUES". val insertOverwriteValues = Value // Ingest using "INSERT OVERWRITE ... SELECT ...". val insertOverwriteSelect = Value // Ingest using streaming query. val streaming = Value // Ingest using MERGE INTO ... WHEN NOT MATCHED INSERT val mergeInsert = Value } case class IngestTestCase(start: Long, step: Long, iteration: Int, batchSize: Int) /** * Helper function to test ingesting data to delta table with IDENTITY columns. * * @param start IDENTITY start configuration. * @param step IDENTITY step configuration. * @param iteration How many batch to ingest. * @param batchSize How many rows to ingest in each batch. * @param mode Specifies what command to use to ingest data. */ private def testIngestData( start: Long, step: Long, iteration: Int, batchSize: Int, mode: IngestMode.Value): Unit = { var highWaterMark = start - step val tblName = getRandomTableName withTable(tblName) { createTableWithIdColAndIntValueCol( tblName, GeneratedAlways, startsWith = Some(start), incrementBy = Some(step)) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tblName)) for (iter <- 0 to iteration - 1) { val batchStart = iter * batchSize + 1 val batchEnd = (iter + 1) * batchSize // Used by data frame append v1 and append v2. val df = (batchStart to batchEnd).toDF("value") // Used by insertInto, insertIntoSelect, insertOverwrite, insertOverwriteSelect val insertValues = (batchStart to batchEnd).map(v => s"($v)").mkString(",") val tempTblName = s"${getRandomTableName}_temp" mode match { case IngestMode.appendV1 => df.write.format("delta").mode("append").save(deltaLog.dataPath.toString) case IngestMode.appendV2 => df.writeTo(tblName).append() case IngestMode.insertIntoValues => val insertStmt = s"INSERT INTO $tblName(value) VALUES $insertValues;" sql(insertStmt) case IngestMode.insertIntoSelect => withTable(tempTblName) { // Insert values into a separate table, then select into the destination table. createTable( tempTblName, Seq(TestColumnSpec(colName = "value", dataType = IntegerType))) sql(s"INSERT INTO $tempTblName VALUES $insertValues") sql(s"INSERT INTO $tblName(value) SELECT value FROM $tempTblName") } case IngestMode.insertOverwriteSelect => withTable(tempTblName) { // Insert values into a separate table, then select into the destination table. createTable( tempTblName, Seq(TestColumnSpec(colName = "value", dataType = IntegerType))) sql(s"INSERT INTO $tempTblName VALUES $insertValues") sql(s"INSERT OVERWRITE $tblName(value) SELECT value FROM $tempTblName") } case IngestMode.insertOverwriteValues => val insertStmt = s"INSERT OVERWRITE $tblName(value) VALUES $insertValues" sql(insertStmt) case IngestMode.streaming => withTempDir { checkpointDir => val stream = MemoryStream[Int] val q = stream .toDF .toDF("value") .writeStream .format("delta") .outputMode("append") .option("checkpointLocation", checkpointDir.getCanonicalPath) .start(deltaLog.dataPath.toString) stream.addData(batchStart to batchEnd) q.processAllAvailable() q.stop() } case IngestMode.mergeInsert => withTable(tempTblName) { // Insert values into a separate table, then merge into the destination table. createTable( tempTblName, Seq(TestColumnSpec(colName = "value", dataType = IntegerType))) sql(s"INSERT INTO $tempTblName VALUES $insertValues") sql( s""" |MERGE INTO $tblName | USING $tempTblName ON $tblName.value = $tempTblName.value | WHEN NOT MATCHED THEN INSERT (value) VALUES ($tempTblName.value) |""".stripMargin) } case _ => assert(false, "Unrecognized ingestion mode") } val expectedRowCount = mode match { case _@(IngestMode.insertOverwriteValues | IngestMode.insertOverwriteSelect) => // These modes keep the row count unchanged. batchSize case _ => batchSize * (iter + 1) } highWaterMark = validateIdentity(tblName, expectedRowCount, start, step, batchStart, batchEnd, highWaterMark) } } } test("append v1") { val testCases = Seq( IngestTestCase(1, 1, 4, 250), IngestTestCase(1, -3, 10, 23) ) for (tc <- testCases) { testIngestData(tc.start, tc.step, tc.iteration, tc.batchSize, IngestMode.appendV1) } } test("append v2") { val testCases = Seq( IngestTestCase(100, 100, 3, 300), IngestTestCase(Integer.MAX_VALUE.toLong + 1, -1000, 10, 23) ) for (tc <- testCases) { testIngestData(tc.start, tc.step, tc.iteration, tc.batchSize, IngestMode.appendV2) } } test("insert into values") { val testCases = Seq( IngestTestCase(100, -100, 4, 201), IngestTestCase(Integer.MAX_VALUE.toLong + 1, 1000, 10, 37) ) for (tc <- testCases) { testIngestData(tc.start, tc.step, tc.iteration, tc.batchSize, IngestMode.insertIntoValues) } } test("insert into select") { val testCases = Seq( IngestTestCase(23, 102, 3, 77), IngestTestCase(Integer.MAX_VALUE.toLong - 12345, 99, 8, 25) ) for (tc <- testCases) { testIngestData(tc.start, tc.step, tc.iteration, tc.batchSize, IngestMode.insertIntoSelect) } } test("insert overwrite values") { val testCases = Seq( IngestTestCase(-10, 3, 5, 30), IngestTestCase(Integer.MIN_VALUE.toLong - 1000, -18, 2, 100) ) for (tc <- testCases) { testIngestData(tc.start, tc.step, tc.iteration, tc.batchSize, IngestMode.insertOverwriteValues) } } test("insert overwrite select") { val testCases = Seq( IngestTestCase(-15, 20, 4, 35), IngestTestCase(200, 50, 3, 7) ) for (tc <- testCases) { testIngestData(tc.start, tc.step, tc.iteration, tc.batchSize, IngestMode.insertOverwriteSelect) } } test("streaming") { val testCases = Seq( IngestTestCase(-2000, 19, 5, 20), IngestTestCase(10, 10, 4, 17) ) for (tc <- testCases) { testIngestData(tc.start, tc.step, tc.iteration, tc.batchSize, IngestMode.streaming) } } test("merge insert") { val testCases = Seq( IngestTestCase(10, 20, 5, 8), IngestTestCase(-5000, 37, 7, 99) ) for (tc <- testCases) { testIngestData(tc.start, tc.step, tc.iteration, tc.batchSize, IngestMode.mergeInsert) } } test("explicit insert not allowed") { val tblName = getRandomTableName withIdentityColumnTable(GeneratedAlways, tblName) { val ex = intercept[AnalysisException](sql(s"INSERT INTO $tblName values(1,1);")) assert(ex.getMessage.contains("Providing values for GENERATED ALWAYS AS IDENTITY")) } } test("explicit insert should not update high water mark") { val tblName = getRandomTableName withIdentityColumnTable(GeneratedByDefault, tblName) { val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tblName)) val schema1 = deltaLog.update().metadata.schemaString // System generated IDENTITY value - should update schema. sql(s"INSERT INTO $tblName(value) VALUES (1);") val snapshot2 = deltaLog.update() val highWatermarkAfterGeneration = getHighWaterMark(snapshot2, "id") assert(highWatermarkAfterGeneration.isDefined) val schema2 = snapshot2.metadata.schemaString assert(schema1 != schema2) // Explicitly provided IDENTITY value - should not update schema. sql(s"INSERT INTO $tblName VALUES (1,1);") val snapshot3 = deltaLog.update() val schema3 = snapshot3.metadata.schemaString val highWatermarkAfterUserInsert = getHighWaterMark(snapshot3, "id") assert(highWatermarkAfterUserInsert == highWatermarkAfterGeneration) assert(schema2 == schema3) } } test("merge command with nondeterministic functions in conditions") { val source = "identity_merge_source" val target = "identity_merge_target" withIdentityColumnTable(GeneratedByDefault, target) { withTable(source) { createTable( source, Seq( TestColumnSpec(colName = "id2", dataType = LongType), TestColumnSpec(colName = "value2", dataType = LongType) ) ) val ex1 = intercept[AnalysisException] { sql( s""" |MERGE INTO $target | USING $source ON $target.value = $source.value2 + rand() | WHEN NOT MATCHED THEN INSERT (value) VALUES ($source.value2) |""".stripMargin) } assert(ex1.getMessage.contains("Non-deterministic functions are not supported")) val ex2 = intercept[AnalysisException] { sql( s""" |MERGE INTO $target | USING $source ON $target.value = $source.value2 | WHEN NOT MATCHED AND $source.value2 = rand() | THEN INSERT (value) VALUES ($source.value2) |""".stripMargin) } assert(ex2.getMessage.contains("Non-deterministic functions are not supported")) } } } /** * Creates a source and destination table with the same schema such that if it is a positive step, * the source table has identity column values < the target table's start value. If it's * a negative step, the source table has identity column values > the target table's start value. * @param isSrcDataSubsetOfTgt Whether the source data is a subset of the target data. If false, * some data is inserted into the target table below the start of * the identity column value. * @param positiveStep Whether the identity column values are generated in a positive step. * @param expectValidHighWaterMark Whether the high water mark is expected to be set to a valid * value in the target table after running `insertDataFn`. If so, * we check that it respects the start value of the column. * @param insertDataFn Function that inserts data from the source table to the target table. */ private def withSrcAndDestTables( isSrcDataSubsetOfTgt: Boolean, positiveStep: Boolean, expectValidHighWaterMark: Boolean)( insertDataFn: (String, String) => Unit): Unit = { import testImplicits._ val srcTblName = s"${getRandomTableName}_src" val tgtTblName = s"${getRandomTableName}_tgt" withTable(srcTblName, tgtTblName) { val targetTableStartWith = if (positiveStep) 100000 else -100000 val targetTableIncrementBy = if (positiveStep) 53 else -53 // Create a generated always source table with (id, value) // starting with 0 and incrementing by targetTableIncrementBy. generateTableWithIdentityColumn(srcTblName, step = targetTableIncrementBy) val srcDeltaLog = DeltaLog.forTable(spark, TableIdentifier(srcTblName)) assert(getHighWaterMark(srcDeltaLog.update(), colName = "id").isDefined) // While id col values generation is nondeterministic, the high water mark // should really not exceed this value. if (positiveStep) { assert(highWaterMark(srcDeltaLog.update(), colName = "id") < targetTableStartWith) } else { assert(highWaterMark(srcDeltaLog.update(), colName = "id") > targetTableStartWith) } // Create a generated by default target table with (id, value) createTableWithIdColAndIntValueCol( tgtTblName, GeneratedByDefault, startsWith = Some(targetTableStartWith), incrementBy = Some(targetTableIncrementBy), tblProperties = Map.empty) val tgtDeltaLog = DeltaLog.forTable(spark, TableIdentifier(tgtTblName)) assert(getHighWaterMark(tgtDeltaLog.update(), colName = "id").isEmpty, "High watermark should not be set if the table is empty.") if (isSrcDataSubsetOfTgt) { sql(s"INSERT INTO $tgtTblName(id, value) SELECT * FROM $srcTblName") } else { // Manually insert some data into the target table below the startWith. if (positiveStep) { sql(s"INSERT INTO $tgtTblName(id, value) VALUES (1, 100), (2, 101)") } else { sql(s"INSERT INTO $tgtTblName(id, value) VALUES (-1, 100), (-2, 101)") } } assert(getHighWaterMark(tgtDeltaLog.update(), colName = "id").isEmpty, "High watermark should not be set for user inserted data.") if (positiveStep) { assert(sql(s"SELECT max(id) FROM $tgtTblName").as[Long].head < targetTableStartWith) } else { assert(sql(s"SELECT min(id) FROM $tgtTblName").as[Long].head > targetTableStartWith) } insertDataFn(srcTblName, tgtTblName) if (expectValidHighWaterMark) { if (positiveStep) { assert(highWaterMark(tgtDeltaLog.update(), colName = "id") >= targetTableStartWith) } else { assert(highWaterMark(tgtDeltaLog.update(), colName = "id") <= targetTableStartWith) } } } } test("Appending from a source table with a high water mark should not update" + " the target table's high water mark") { withSrcAndDestTables( isSrcDataSubsetOfTgt = false, positiveStep = true, expectValidHighWaterMark = false) { (srcTblName, tgtTblName) => val tgtDeltaLog = DeltaLog.forTable(spark, TableIdentifier(tgtTblName)) // dataframe v2 spark.table(srcTblName).writeTo(tgtTblName).append() assert(getHighWaterMark(tgtDeltaLog.update(), colName = "id").isEmpty, "High watermark should not be set for user inserted data.") // v1 spark.table(srcTblName).write.format("delta").mode("append").saveAsTable(tgtTblName) assert(getHighWaterMark(tgtDeltaLog.update(), colName = "id").isEmpty, "High watermark should not be set for user inserted data.") spark.table(srcTblName).write.insertInto(tgtTblName) assert(getHighWaterMark(tgtDeltaLog.update(), colName = "id").isEmpty, "High watermark should not be set for user inserted data.") // SQL sql(s"INSERT INTO $tgtTblName SELECT * FROM $srcTblName") assert(getHighWaterMark(tgtDeltaLog.update(), colName = "id").isEmpty, "High watermark should not be set for user inserted data.") sql(s"INSERT INTO $tgtTblName BY NAME SELECT * FROM $srcTblName") assert(getHighWaterMark(tgtDeltaLog.update(), colName = "id").isEmpty, "High watermark should not be set for user inserted data.") sql(s"INSERT INTO $tgtTblName(id, value) SELECT id, value FROM $srcTblName") assert(getHighWaterMark(tgtDeltaLog.update(), colName = "id").isEmpty, "High watermark should not be set for user inserted data.") } } for { cdfEnabled <- DeltaTestUtils.BOOLEAN_DOMAIN isSrcDataSubsetOfTgt <- DeltaTestUtils.BOOLEAN_DOMAIN positiveStep <- DeltaTestUtils.BOOLEAN_DOMAIN statementWithOnlyUpdates <- DeltaTestUtils.BOOLEAN_DOMAIN } test( s"MERGE UPSERT with source on identity column, cdfEnabled=$cdfEnabled, " + s"isSrcDataSubsetOfTgt=$isSrcDataSubsetOfTgt, " + s"positiveStep=$positiveStep, statementWithOnlyUpdates=$statementWithOnlyUpdates") { val expectValidHighWaterMark = !statementWithOnlyUpdates && !isSrcDataSubsetOfTgt withSrcAndDestTables( isSrcDataSubsetOfTgt, positiveStep, expectValidHighWaterMark) { (srcTblName, tgtTblName) => if (cdfEnabled) { val cdfPropKey = DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey sql(s"ALTER TABLE $tgtTblName SET TBLPROPERTIES('$cdfPropKey' = 'true')") } // Merge into the target table from the source table. // The target table will generate values starting from targetTableStartWith. // The high water mark from the source should not interfere. if (statementWithOnlyUpdates) { sql( s""" |MERGE INTO $tgtTblName tgt |USING $srcTblName src ON tgt.id = src.id |WHEN MATCHED THEN UPDATE SET tgt.value = src.value |""".stripMargin) } else { sql( s""" |MERGE INTO $tgtTblName tgt |USING $srcTblName src ON tgt.id = src.id |WHEN MATCHED THEN UPDATE SET tgt.value = src.value |WHEN NOT MATCHED THEN INSERT (value) VALUES (src.value) |""".stripMargin) } if (!expectValidHighWaterMark) { val tgtDeltaLog = DeltaLog.forTable(spark, TableIdentifier(tgtTblName)) assert(getHighWaterMark(tgtDeltaLog.update(), colName = "id").isEmpty) } } } for (positiveStep <- DeltaTestUtils.BOOLEAN_DOMAIN) test(s"MERGE UPSERT into a table with a bad watermark, positiveStep=$positiveStep") { // Suppose that a table has a bad watermark (for whatever reason), the system should still // have a sensible behavior and be robust to these bad watermark. withSrcAndDestTables( isSrcDataSubsetOfTgt = false, positiveStep, expectValidHighWaterMark = false) { (srcTblName, tgtTblName) => val tgtDeltaLog = DeltaLog.forTable(spark, TableIdentifier(tgtTblName)) forceBadWaterMark(tgtDeltaLog) val badWaterMark = highWaterMark(tgtDeltaLog.update(), colName = "id") sql( s""" |MERGE INTO $tgtTblName tgt |USING $srcTblName src ON tgt.id = src.id |WHEN MATCHED THEN UPDATE SET tgt.value = src.value |WHEN NOT MATCHED THEN INSERT (value) VALUES (src.value) |""".stripMargin) // Even though the high water mark is invalid, we don't want to prevent updates to the high // water mark as this would lead to us generating the same values over and over. val newHighWaterMark = highWaterMark(tgtDeltaLog.update(), colName = "id") assert(newHighWaterMark !== badWaterMark, "New data was inserted. The high water mark should have updated") if (positiveStep) { assert(newHighWaterMark > badWaterMark) } else { assert(newHighWaterMark < badWaterMark) } } } } class IdentityColumnIngestionScalaSuite extends IdentityColumnIngestionSuiteBase with ScalaDDLTestUtils class IdentityColumnIngestionScalaIdColumnMappingSuite extends IdentityColumnIngestionSuiteBase with ScalaDDLTestUtils with DeltaColumnMappingEnableIdMode class IdentityColumnIngestionScalaNameColumnMappingSuite extends IdentityColumnIngestionSuiteBase with ScalaDDLTestUtils with DeltaColumnMappingEnableNameMode ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/IdentityColumnSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import scala.collection.mutable.ListBuffer import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta.GeneratedAsIdentityType.{GeneratedAlways, GeneratedAsIdentityType, GeneratedByDefault} import org.apache.spark.sql.delta.actions.Protocol import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf} import org.apache.spark.sql.delta.util.JsonUtils import org.apache.commons.io.FileUtils import org.apache.spark.SparkConf import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.streaming.Trigger import org.apache.spark.sql.types._ /** * General test suite for identity columns. */ trait IdentityColumnSuiteBase extends IdentityColumnTestUtils { import testImplicits._ test("Don't allow IDENTITY column in the schema if the feature is disabled") { val tblName = getRandomTableName withSQLConf(DeltaSQLConf.DELTA_IDENTITY_COLUMN_ENABLED.key -> "false") { withTable(tblName) { val e = intercept[DeltaUnsupportedTableFeatureException] { createTableWithIdColAndIntValueCol( tblName, GeneratedByDefault, startsWith = None, incrementBy = None) } val errorMsg = e.getMessage assert(errorMsg.contains("requires writer table feature(s) that are unsupported")) assert(errorMsg.contains(IdentityColumnsTableFeature.name)) } } } // Build expected schema of the following table definition for verification: // CREATE TABLE tableName ( // id BIGINT IDENTITY (START WITH INCREMENT BY ), // value INT // ); private def expectedSchema( generatedAsIdentityType: GeneratedAsIdentityType, start: Long = IdentityColumn.defaultStart, step: Long = IdentityColumn.defaultStep): StructType = { val colFields = new ListBuffer[StructField] val allowExplicitInsert = generatedAsIdentityType == GeneratedByDefault val builder = new MetadataBuilder() builder.putBoolean(DeltaSourceUtils.IDENTITY_INFO_ALLOW_EXPLICIT_INSERT, allowExplicitInsert) builder.putLong(DeltaSourceUtils.IDENTITY_INFO_START, start) builder.putLong(DeltaSourceUtils.IDENTITY_INFO_STEP, step) colFields += StructField("id", LongType, true, builder.build()) colFields += StructField("value", IntegerType) StructType(colFields.toSeq) } test("various configuration") { val starts = Seq( Long.MinValue, Integer.MIN_VALUE.toLong, -100L, 0L, 1000L, Integer.MAX_VALUE.toLong, Long.MaxValue ) val steps = Seq( Long.MinValue, Integer.MIN_VALUE.toLong, -100L, 1000L, Integer.MAX_VALUE.toLong, Long.MaxValue ) for { generatedAsIdentityType <- GeneratedAsIdentityType.values startsWith <- starts incrementBy <- steps } { val tblName = getRandomTableName withTable(tblName) { createTableWithIdColAndIntValueCol( tblName, generatedAsIdentityType, Some(startsWith), Some(incrementBy)) val table = DeltaLog.forTable(spark, TableIdentifier(tblName)) val actualSchema = DeltaColumnMapping.dropColumnMappingMetadata(table.snapshot.metadata.schema) assert(actualSchema === expectedSchema(generatedAsIdentityType, startsWith, incrementBy)) } } } test("default configuration") { for { generatedAsIdentityType <- GeneratedAsIdentityType.values startsWith <- Seq(Some(1L), None) incrementBy <- Seq(Some(1L), None) } { val tblName = getRandomTableName withTable(tblName) { createTableWithIdColAndIntValueCol( tblName, generatedAsIdentityType, startsWith, incrementBy) val table = DeltaLog.forTable(spark, TableIdentifier(tblName)) val actualSchema = DeltaColumnMapping.dropColumnMappingMetadata(table.snapshot.metadata.schema) assert(actualSchema === expectedSchema(generatedAsIdentityType)) } } } test("logging") { val tblName = getRandomTableName withTable(tblName) { val eventsDefinition = Log4jUsageLogger.track { createTable( tblName, Seq( IdentityColumnSpec( GeneratedByDefault, startsWith = Some(1), incrementBy = Some(1), colName = "id1" ), IdentityColumnSpec( GeneratedAlways, startsWith = Some(1), incrementBy = Some(1), colName = "id2" ), IdentityColumnSpec( GeneratedAlways, startsWith = Some(1), incrementBy = Some(1), colName = "id3" ), TestColumnSpec(colName = "value", dataType = IntegerType) ) ) }.filter { e => e.tags.get("opType").exists(_ == IdentityColumn.opTypeDefinition) } assert(eventsDefinition.size == 1) assert(JsonUtils.fromJson[Map[String, String]](eventsDefinition.head.blob) .get("numIdentityColumns").exists(_ == "3")) val eventsWrite = Log4jUsageLogger.track { sql(s"INSERT INTO $tblName (id1, value) VALUES (1, 10), (2, 20)") }.filter { e => e.tags.get("opType").exists(_ == IdentityColumn.opTypeWrite) } assert(eventsWrite.size == 1) val data = JsonUtils.fromJson[Map[String, String]](eventsWrite.head.blob) assert(data.get("numInsertedRows").exists(_ == "2")) assert(data.get("generatedIdentityColumnNames").exists(_ == "id2,id3")) assert(data.get("generatedIdentityColumnCount").exists(_ == "2")) assert(data.get("explicitIdentityColumnNames").exists(_ == "id1")) assert(data.get("explicitIdentityColumnCount").exists(_ == "1")) } } test("reading table should not see identity column properties") { def verifyNoIdentityColumn(id: Int, f: () => Dataset[_]): Unit = { assert(!ColumnWithDefaultExprUtils.hasIdentityColumn(f().schema), s"test $id failed") } val tblName = getRandomTableName withTable(tblName) { createTable( tblName, Seq( IdentityColumnSpec(GeneratedByDefault), TestColumnSpec(colName = "part", dataType = LongType), TestColumnSpec(colName = "value", dataType = StringType) ), partitionedBy = Seq("part") ) sql( s""" |INSERT INTO $tblName (part, value) VALUES | (1, "one"), | (2, "two"), | (3, "three") |""".stripMargin) val path = DeltaLog.forTable(spark, TableIdentifier(tblName)).dataPath.toString val commands: Seq[() => Dataset[_]] = Seq( () => spark.table(tblName), () => sql(s"SELECT * FROM $tblName"), () => sql(s"SELECT * FROM delta.`$path`"), () => spark.read.format("delta").load(path), () => spark.read.format("delta").table(tblName), () => spark.readStream.format("delta").load(path), () => spark.readStream.format("delta").table(tblName) ) commands.zipWithIndex.foreach { case (f, id) => verifyNoIdentityColumn(id, f) } withTempDir { checkpointDir => val q = spark.readStream.format("delta").table(tblName).writeStream .trigger(Trigger.Once) .option("checkpointLocation", checkpointDir.getCanonicalPath) .foreachBatch { (df: DataFrame, _: Long) => assert(!ColumnWithDefaultExprUtils.hasIdentityColumn(df.schema)) () }.start() try { q.processAllAvailable() } finally { q.stop() } } withTempDir { outputDir => withTempDir { checkpointDir => val q = spark.readStream.format("delta").table(tblName).writeStream .trigger(Trigger.Once) .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("delta") .start(outputDir.getCanonicalPath) try { q.processAllAvailable() } finally { q.stop() } val deltaLog = DeltaLog.forTable(spark, outputDir.getCanonicalPath) assert(deltaLog.snapshot.version >= 0) assert(!ColumnWithDefaultExprUtils.hasIdentityColumn(deltaLog.snapshot.schema)) } } } } private def withWriterVersion5Table(func: String => Unit): Unit = { // The table on the following path is created with the following steps: // (1) Create a table with IDENTITY column using writer version 6 // CREATE TABLE $tblName ( // id BIGINT GENERATED BY DEFAULT AS IDENTITY, // part INT, // value STRING // ) USING delta // PARTITIONED BY (part) // (2) CTAS from the above table using writer version 5. // This will result in a table created using protocol (1, 2) with IDENTITY columns. val resourcePath = "src/test/resources/delta/identity_test_written_by_version_5" withTempDir { tempDir => // Prepare a table that has the old writer version and identity columns. FileUtils.copyDirectory(new File(resourcePath), tempDir) val path = tempDir.getCanonicalPath val deltaLog = DeltaLog.forTable(spark, path) // Verify the table has old writer version and identity columns. assert(ColumnWithDefaultExprUtils.hasIdentityColumn(deltaLog.snapshot.schema)) val writerVersionOnTable = deltaLog.snapshot.protocol.minWriterVersion assert(writerVersionOnTable < IdentityColumnsTableFeature.minWriterVersion) func(path) } } test("compatibility") { withWriterVersion5Table { v5TablePath => // Verify initial data. checkAnswer( sql(s"SELECT * FROM delta.`$v5TablePath`"), Row(1, 1, "one") :: Row(2, 2, "two") :: Row(4, 3, "three") :: Nil ) // Insert new data should generate correct IDENTITY values. sql(s"""INSERT INTO delta.`$v5TablePath` VALUES (5, 5, "five")""") checkAnswer( sql(s"SELECT COUNT(DISTINCT id) FROM delta.`$v5TablePath`"), Row(4L) ) val deltaLog = DeltaLog.forTable(spark, v5TablePath) val protocolBeforeUpdate = deltaLog.snapshot.protocol // ALTER TABLE should drop the IDENTITY columns and keeps the protocol version unchanged. sql(s"ALTER TABLE delta.`$v5TablePath` ADD COLUMNS (value2 DOUBLE)") deltaLog.update() assert(deltaLog.snapshot.protocol == protocolBeforeUpdate) assert(!ColumnWithDefaultExprUtils.hasIdentityColumn(deltaLog.snapshot.schema)) // Specifying a min writer version should not enable IDENTITY column. sql(s"ALTER TABLE delta.`$v5TablePath` SET TBLPROPERTIES ('delta.minWriterVersion'='4')") deltaLog.update() assert(deltaLog.snapshot.protocol == Protocol(1, 4)) assert(!ColumnWithDefaultExprUtils.hasIdentityColumn(deltaLog.snapshot.schema)) } } for { generatedAsIdentityType <- GeneratedAsIdentityType.values } { test( "replace table with identity column should upgrade protocol, " + s"identityType: $generatedAsIdentityType") { val tblName = getRandomTableName def getProtocolVersions: (Int, Int) = { sql(s"DESC DETAIL $tblName") .select("minReaderVersion", "minWriterVersion") .as[(Int, Int)] .head() } withTable(tblName) { createTable( tblName, Seq( TestColumnSpec(colName = "id", dataType = LongType), TestColumnSpec(colName = "value", dataType = IntegerType)) ) assert(getProtocolVersions == (1, 2) || getProtocolVersions == (2, 7)) assert(DeltaLog.forTable(spark, TableIdentifier(tblName)).snapshot.version == 0) replaceTable( tblName, Seq( IdentityColumnSpec( generatedAsIdentityType, startsWith = Some(1), incrementBy = Some(1) ), TestColumnSpec(colName = "value", dataType = IntegerType) ) ) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tblName)) val snapshot = deltaLog.update() val protocol = snapshot.protocol assert(getProtocolVersions == (1, 7) || protocol.readerAndWriterFeatures.contains(IdentityColumnsTableFeature)) assert(snapshot.version == 1) assert(getHighWaterMark(snapshot, "id").isEmpty) } } } test("ctas/rtas does not produce an identity column") { def assertIdentityColumn(tblName: String, idColExpected: Boolean): Unit = { val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tblName)) val snapshot = deltaLog.update() assert(ColumnWithDefaultExprUtils.hasIdentityColumn(snapshot.tableSchema) === idColExpected) assert(getHighWaterMark(snapshot, colName = "id").isDefined === idColExpected) } val tblName = getRandomTableName val ctasTblName = s"ctas_$tblName" withTable(tblName) { generateTableWithIdentityColumn(tblName) assertIdentityColumn(tblName, idColExpected = true) withTable(ctasTblName) { sql( s""" |CREATE TABLE $ctasTblName |USING DELTA |AS SELECT * FROM $tblName |""".stripMargin) assertIdentityColumn(ctasTblName, idColExpected = false) sql( s""" |CREATE OR REPLACE TABLE $ctasTblName |USING DELTA |AS SELECT * FROM $tblName |""".stripMargin) assertIdentityColumn(ctasTblName, idColExpected = false) } } } test("create or replace on a table resets high watermark") { val tblName = getRandomTableName val initialStartsWith = 100L val increment = 1L for { generatedAsIdentityType <- GeneratedAsIdentityType.values isPartitioned <- DeltaTestUtils.BOOLEAN_DOMAIN } { val partitionedBy = if (isPartitioned) Seq("value") else Nil withTable(tblName) { createTable( tblName, Seq( IdentityColumnSpec( generatedAsIdentityType, Some(initialStartsWith), Some(increment) ), TestColumnSpec(colName = "value", dataType = IntegerType) ), partitionedBy = partitionedBy ) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tblName)) // A column that has not yet generated values does not have a high watermark. assert(getHighWaterMark(deltaLog.update(), colName = "id").isEmpty) // The system generates one row for the identity column . // The high watermark should now be set to the start value. sql(s"INSERT INTO $tblName (value) VALUES (1)") assert(highWaterMark(deltaLog.update(), colName = "id") === initialStartsWith) for { replaceType <- Seq(DDLType.REPLACE, DDLType.CREATE_OR_REPLACE) } { // After a REPLACE or CREATE OR REPLACE TABLE, there should be no high watermark. val newStartsWith = 50000L runDDL( replaceType, tblName, Seq( IdentityColumnSpec( generatedAsIdentityType, startsWith = Some(newStartsWith), incrementBy = Some(increment)), TestColumnSpec(colName = "value", dataType = StringType) ), partitionedBy = partitionedBy, tblProperties = Map.empty ) assert(getHighWaterMark(deltaLog.update(), colName = "id").isEmpty) // Sanity check that the new table is using the new start for the next high watermark. sql(s"INSERT INTO $tblName (value) VALUES (-1)") assert(highWaterMark(deltaLog.update(), colName = "id") === newStartsWith) } } } } ignore("identity value start at boundaries") { val starts = Seq(Long.MinValue, Long.MaxValue) val steps = Seq(1, 2, -1, -2) for { generatedAsIdentityType <- GeneratedAsIdentityType.values start <- starts step <- steps } { val tblName = getRandomTableName withTable(tblName) { createTableWithIdColAndIntValueCol( tblName, generatedAsIdentityType, Some(start), Some(step)) val table = DeltaLog.forTable(spark, TableIdentifier(tblName)) val actualSchema = DeltaColumnMapping.dropColumnMappingMetadata(table.snapshot.metadata.schema) assert(actualSchema === expectedSchema(generatedAsIdentityType, start, step)) if ((start < 0L) == (step < 0L)) { // test long underflow and overflow val ex = intercept[org.apache.spark.SparkException]( sql(s"INSERT INTO $tblName(value) SELECT 1 UNION ALL SELECT 2") ) assert(ex.getMessage.contains("long overflow")) } else { sql(s"INSERT INTO $tblName(value) SELECT 1 UNION ALL SELECT 2") checkAnswer(sql(s"SELECT COUNT(DISTINCT id) == COUNT(*) FROM $tblName"), Row(true)) sql(s"INSERT INTO $tblName(value) SELECT 1 UNION ALL SELECT 2") checkAnswer(sql(s"SELECT COUNT(DISTINCT id) == COUNT(*) FROM $tblName"), Row(true)) assert(highWaterMark(table.update(), "id") === (start + (3 * step))) } } } } test("restore - positive step") { val tblName = getRandomTableName withTable(tblName) { generateTableWithIdentityColumn(tblName) sql(s"RESTORE TABLE $tblName TO VERSION AS OF 3") sql(s"INSERT INTO $tblName (value) VALUES (6)") checkAnswer( sql(s"SELECT id, value FROM $tblName ORDER BY value ASC"), Seq(Row(0, 0), Row(1, 1), Row(2, 2), Row(6, 6)) ) } } test("restore - negative step") { val tblName = getRandomTableName withTable(tblName) { generateTableWithIdentityColumn(tblName, step = -1) sql(s"RESTORE TABLE $tblName TO VERSION AS OF 3") sql(s"INSERT INTO $tblName (value) VALUES (6)") checkAnswer( sql(s"SELECT id, value FROM $tblName ORDER BY value ASC"), Seq(Row(0, 0), Row(-1, 1), Row(-2, 2), Row(-6, 6)) ) } } test("restore - on partitioned table") { for (generatedAsIdentityType <- GeneratedAsIdentityType.values) { val tblName = getRandomTableName withTable(tblName) { // v0. createTable( tblName, Seq( IdentityColumnSpec(generatedAsIdentityType), TestColumnSpec(colName = "value", dataType = IntegerType) ), partitionedBy = Seq("value") ) // v1. sql(s"INSERT INTO $tblName (value) VALUES (1), (2)") val v1Content = sql(s"SELECT * FROM $tblName").collect() // v2. sql(s"INSERT INTO $tblName (value) VALUES (3), (4)") // v3: RESTORE to v1. sql(s"RESTORE TABLE $tblName TO VERSION AS OF 1") checkAnswer( sql(s"SELECT COUNT(DISTINCT id) FROM $tblName"), Row(2L) ) checkAnswer( sql(s"SELECT * FROM $tblName"), v1Content ) // v4. sql(s"INSERT INTO $tblName (value) VALUES (5), (6)") checkAnswer( sql(s"SELECT COUNT(DISTINCT id) FROM $tblName"), Row(4L) ) } } } test("clone") { for { generatedAsIdentityType <- GeneratedAsIdentityType.values } { val oldTbl = s"${getRandomTableName}_old" val newTbl = s"${getRandomTableName}_new" withIdentityColumnTable(generatedAsIdentityType, oldTbl) { withTable(newTbl) { sql(s"INSERT INTO $oldTbl (value) VALUES (1), (2)") val oldSchema = DeltaLog.forTable(spark, TableIdentifier(oldTbl)).snapshot.schema sql( s""" |CREATE TABLE $newTbl | SHALLOW CLONE $oldTbl |""".stripMargin) val newSchema = DeltaLog.forTable(spark, TableIdentifier(newTbl)).snapshot.schema assert(newSchema("id").metadata.getLong(DeltaSourceUtils.IDENTITY_INFO_START) == 1L) assert(newSchema("id").metadata.getLong(DeltaSourceUtils.IDENTITY_INFO_STEP) == 1L) assert(oldSchema == newSchema) sql(s"INSERT INTO $newTbl (value) VALUES (1), (2)") checkAnswer( sql(s"SELECT COUNT(DISTINCT id) FROM $newTbl"), Row(4L) ) } } } } } class IdentityColumnScalaSuite extends IdentityColumnSuiteBase with ScalaDDLTestUtils { test("unsupported column type") { for (unsupportedType <- unsupportedDataTypes) { val tblName = getRandomTableName withTable(tblName) { val ex = intercept[DeltaUnsupportedOperationException] { createTable( tblName, Seq( IdentityColumnSpec(GeneratedAlways, dataType = unsupportedType), TestColumnSpec(colName = "value", dataType = StringType) ) ) } assert(ex.getErrorClass === "DELTA_IDENTITY_COLUMNS_UNSUPPORTED_DATA_TYPE") assert(ex.getMessage.contains("is not supported for IDENTITY columns")) } } } test("unsupported step") { for { generatedAsIdentityType <- GeneratedAsIdentityType.values startsWith <- Seq(Some(1L), None) } { val tblName = getRandomTableName withTable(tblName) { val ex = intercept[DeltaAnalysisException] { createTableWithIdColAndIntValueCol( tblName, generatedAsIdentityType, startsWith, incrementBy = Some(0)) } assert(ex.getErrorClass === "DELTA_IDENTITY_COLUMNS_ILLEGAL_STEP") assert(ex.getMessage.contains("step cannot be 0.")) } } } test("cannot specify generatedAlwaysAs with identity columns") { def expectColumnBuilderError(f: => StructField): Unit = { val ex = intercept[DeltaAnalysisException] { f } assert(ex.getErrorClass === "DELTA_IDENTITY_COLUMNS_WITH_GENERATED_EXPRESSION") ex.getMessage.contains( "Identity column cannot be specified with a generated column expression.") } val generatedColumn = io.delta.tables.DeltaTable.columnBuilder(spark, "id") .dataType(LongType) .generatedAlwaysAs("id + 1") expectColumnBuilderError { generatedColumn.generatedAlwaysAsIdentity().build() } expectColumnBuilderError { generatedColumn.generatedByDefaultAsIdentity().build() } } } class IdentityColumnScalaIdColumnMappingSuite extends IdentityColumnSuiteBase with ScalaDDLTestUtils with DeltaColumnMappingEnableIdMode class IdentityColumnScalaNameColumnMappingSuite extends IdentityColumnSuiteBase with ScalaDDLTestUtils with DeltaColumnMappingEnableNameMode ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/IdentityColumnSyncSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.GeneratedAsIdentityType.GeneratedByDefault import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf} import org.apache.spark.sql.{AnalysisException, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.types._ case class IdentityColumnTestTableRow(id: Long, value: String) /** * Identity Column test suite for the SYNC IDENTITY command. */ trait IdentityColumnSyncSuiteBase extends IdentityColumnTestUtils { import testImplicits._ /** * Create and manage a table with a single identity column "id" generated by default and a single * String "value" column. */ private def withSimpleGeneratedByDefaultTable( tblName: String, startsWith: Long, incrementBy: Long)(f: => Unit): Unit = { withTable(tblName) { createTable( tblName, Seq( IdentityColumnSpec( GeneratedByDefault, startsWith = Some(startsWith), incrementBy = Some(incrementBy)), TestColumnSpec(colName = "value", dataType = StringType) ) ) f } } test("alter table sync identity on delta table") { val starts = Seq(-1, 1) val steps = Seq(-3, 3) val alterKeywords = Seq("ALTER", "CHANGE") for (start <- starts; step <- steps; alterKeyword <- alterKeywords) { val tblName = getRandomTableName withSimpleGeneratedByDefaultTable(tblName, start, step) { // Test empty table. val oldSchema = DeltaLog.forTable(spark, TableIdentifier(tblName)).snapshot.schema sql(s"ALTER TABLE $tblName $alterKeyword COLUMN id SYNC IDENTITY") assert(DeltaLog.forTable(spark, TableIdentifier(tblName)).snapshot.schema === oldSchema) // Test a series of values that are not all following start and step configurations. for (i <- start to (start + step * 10)) { sql(s"INSERT INTO $tblName VALUES($i, 'v')") sql(s"ALTER TABLE $tblName $alterKeyword COLUMN id SYNC IDENTITY") val expected = start + (((i - start) + (step - 1)) / step) * step val schema = DeltaLog.forTable(spark, TableIdentifier(tblName)).snapshot.schema assert(schema("id").metadata.getLong(DeltaSourceUtils.IDENTITY_INFO_HIGHWATERMARK) === expected) } } } } test("sync identity with values before start") { val tblName = getRandomTableName withSimpleGeneratedByDefaultTable(tblName, startsWith = 100L, incrementBy = 2L) { val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tblName)) assert(getHighWaterMark(deltaLog.update(), "id").isEmpty, "an empty table does not have an identity high watermark") sql(s"INSERT INTO $tblName (id, value) VALUES (1, 'a'), (2, 'b'), (99, 'c')") assert(getHighWaterMark(deltaLog.update(), "id").isEmpty, "user inserted values do not update the high watermark") sql(s"ALTER TABLE $tblName ALTER COLUMN id SYNC IDENTITY") assert(getHighWaterMark(deltaLog.update(), "id").isEmpty, "sync identity must not add a high watermark that is lower " + "than the start value when it has positive increment") sql(s"INSERT INTO $tblName (value) VALUES ('d'), ('e'), ('f')") val result = spark.read.table(tblName) .as[IdentityColumnTestTableRow] .collect() .sortBy(_.id) assert(result.length === 6) assert(result.take(3) === Seq(IdentityColumnTestTableRow(1, "a"), IdentityColumnTestTableRow(2, "b"), IdentityColumnTestTableRow(99, "c"))) checkGeneratedIdentityValues( sortedRows = result.takeRight(3), start = 100, step = 2, expectedLowerBound = 100, expectedUpperBound = highWaterMark(DeltaLog.forTable(spark, TableIdentifier(tblName)).snapshot, "id"), expectedDistinctCount = 3) } } test("sync identity with start in table") { val tblName = getRandomTableName withSimpleGeneratedByDefaultTable(tblName, startsWith = 100L, incrementBy = 2L) { sql(s"INSERT INTO $tblName (id, value) VALUES (1, 'a'), (2, 'b'), (100, 'c')") sql(s"ALTER TABLE $tblName ALTER COLUMN id SYNC IDENTITY") sql(s"INSERT INTO $tblName (value) VALUES ('d'), ('e'), ('f')") val result = spark.read.table(tblName) .as[IdentityColumnTestTableRow] .collect() .sortBy(_.id) assert(result.length === 6) assert(result.take(3) === Seq(IdentityColumnTestTableRow(1, "a"), IdentityColumnTestTableRow(2, "b"), IdentityColumnTestTableRow(100, "c"))) checkGeneratedIdentityValues( sortedRows = result.takeRight(3), start = 100, step = 2, expectedLowerBound = 101, expectedUpperBound = highWaterMark(DeltaLog.forTable(spark, TableIdentifier(tblName)).snapshot, "id"), expectedDistinctCount = 3) } } test("sync identity with values before and after start") { val tblName = getRandomTableName withSimpleGeneratedByDefaultTable(tblName, startsWith = 100L, incrementBy = 2L) { sql(s"INSERT INTO $tblName (id, value) VALUES (1, 'a'), (2, 'b'), (101, 'c')") sql(s"ALTER TABLE $tblName ALTER COLUMN id SYNC IDENTITY") sql(s"INSERT INTO $tblName (value) VALUES ('d'), ('e'), ('f')") val result = spark.read.table(tblName) .as[IdentityColumnTestTableRow] .collect() .sortBy(_.id) assert(result.length === 6) assert(result.take(3) === Seq(IdentityColumnTestTableRow(1, "a"), IdentityColumnTestTableRow(2, "b"), IdentityColumnTestTableRow(101, "c"))) checkGeneratedIdentityValues( sortedRows = result.takeRight(3), start = 100, step = 2, expectedLowerBound = 102, expectedUpperBound = highWaterMark(DeltaLog.forTable(spark, TableIdentifier(tblName)).snapshot, "id"), expectedDistinctCount = 3) } } test("sync identity with values before start and negative step") { val tblName = getRandomTableName withSimpleGeneratedByDefaultTable(tblName, startsWith = -10L, incrementBy = -2L) { val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tblName)) assert(getHighWaterMark(deltaLog.update(), "id").isEmpty, "an empty table does not have an identity high watermark") sql(s"INSERT INTO $tblName (id, value) VALUES (1, 'a'), (2, 'b'), (-9, 'c')") assert(getHighWaterMark(deltaLog.update(), "id").isEmpty, "user inserted values do not update the high watermark") sql(s"ALTER TABLE $tblName ALTER COLUMN id SYNC IDENTITY") assert(getHighWaterMark(deltaLog.update(), "id").isEmpty, "sync identity must not add a high watermark that is higher " + "than the start value when it has negative increment") sql(s"INSERT INTO $tblName (value) VALUES ('d'), ('e'), ('f')") val result = spark.read.table(tblName) .as[IdentityColumnTestTableRow] .collect() .sortBy(_.id) assert(result.length === 6) assert(result.takeRight(3) === Seq(IdentityColumnTestTableRow(-9, "c"), IdentityColumnTestTableRow(1, "a"), IdentityColumnTestTableRow(2, "b"))) checkGeneratedIdentityValues( sortedRows = result.take(3), start = -10, step = -2, expectedLowerBound = highWaterMark(DeltaLog.forTable(spark, TableIdentifier(tblName)).snapshot, "id"), expectedUpperBound = -10, expectedDistinctCount = 3) } } test("alter table sync identity - deleting high watermark rows followed by sync identity" + " brings down the highWatermark only with a flag") { for (generatedAsIdentityType <- GeneratedAsIdentityType.values) { val tblName = getRandomTableName withTable(tblName) { createTableWithIdColAndIntValueCol(tblName, generatedAsIdentityType, Some(1L), Some(10L)) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tblName)) (0 to 4).foreach { v => sql(s"INSERT INTO $tblName(value) VALUES ($v)") } checkAnswer(sql(s"SELECT max(id) FROM $tblName"), Row(41)) sql(s"DELETE FROM $tblName WHERE value IN (0, 3, 4)") assert(highWaterMark(deltaLog.snapshot, "id") === 41L) // Unless this flag is enabled, the high watermark is not updated if it is lower // than the previous high watermark. withSQLConf( DeltaSQLConf.DELTA_IDENTITY_ALLOW_SYNC_IDENTITY_TO_LOWER_HIGH_WATER_MARK.key -> "false" ) { sql(s"ALTER TABLE $tblName ALTER COLUMN id SYNC IDENTITY") assert(highWaterMark(deltaLog.update(), "id") === 41L) } // With the flag enabled, the high watermark is updated even if it is lower, // than the previous high watermark, as long as it is higher than the defined start. withSQLConf( DeltaSQLConf.DELTA_IDENTITY_ALLOW_SYNC_IDENTITY_TO_LOWER_HIGH_WATER_MARK.key -> "true" ) { sql(s"ALTER TABLE $tblName ALTER COLUMN id SYNC IDENTITY") assert(highWaterMark(deltaLog.update(), "id") === 21L) } sql(s"INSERT INTO $tblName(value) VALUES (8)") checkAnswer(sql(s"SELECT max(id) FROM $tblName"), Row(31)) checkAnswer(sql(s"SELECT COUNT(DISTINCT id) == COUNT(*) FROM $tblName"), Row(true)) } } } test("alter table sync identity overflow error") { val tblName = getRandomTableName withSimpleGeneratedByDefaultTable(tblName, startsWith = 1L, incrementBy = 10L) { sql(s"INSERT INTO $tblName VALUES (${Long.MaxValue}, 'a')") assertThrows[ArithmeticException] { sql(s"ALTER TABLE $tblName ALTER COLUMN id SYNC IDENTITY") } } } test("alter table sync identity on non delta table error") { val tblName = getRandomTableName withTable(tblName) { sql( s""" |CREATE TABLE $tblName ( | id BIGINT, | value INT |) USING parquet; |""".stripMargin) val ex = intercept[AnalysisException] { sql(s"ALTER TABLE $tblName ALTER COLUMN id SYNC IDENTITY") } assert(ex.getMessage.contains( "ALTER TABLE ALTER COLUMN SYNC IDENTITY is only supported by Delta.")) } } test("alter table sync identity on non identity column error") { val tblName = getRandomTableName withTable(tblName) { createTable( tblName, Seq( TestColumnSpec(colName = "id", dataType = LongType), TestColumnSpec(colName = "value", dataType = IntegerType) ) ) val ex = intercept[AnalysisException] { sql(s"ALTER TABLE $tblName ALTER COLUMN id SYNC IDENTITY") } assert(ex.getMessage.contains( "ALTER TABLE ALTER COLUMN SYNC IDENTITY cannot be called on non IDENTITY columns.")) } } for (positiveStep <- DeltaTestUtils.BOOLEAN_DOMAIN) test(s"SYNC IDENTITY on table with bad water mark. positiveStep = $positiveStep") { val tblName = getRandomTableName withTable(tblName) { val incrementBy = if (positiveStep) 48 else -48 createTableWithIdColAndIntValueCol( tblName, GeneratedByDefault, startsWith = Some(100), incrementBy = Some(incrementBy) ) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tblName)) // Insert data that don't respect the start. if (positiveStep) { sql(s"INSERT INTO $tblName(id, value) VALUES (4, 4)") } else { sql(s"INSERT INTO $tblName(id, value) VALUES (196, 196)") } forceBadWaterMark(deltaLog) val badWaterMark = highWaterMark(deltaLog.snapshot, "id") // Even though the candidate high water mark and the existing high water mark is invalid, // we don't want to prevent updates to the high water mark as this would lead to us // generating the same values over and over. sql(s"ALTER TABLE $tblName ALTER COLUMN id SYNC IDENTITY") val newHighWaterMark = highWaterMark(deltaLog.update(), colName = "id") assert(newHighWaterMark !== badWaterMark, "Sync identity should update the high water mark based on the data.") if (positiveStep) { assert(newHighWaterMark > badWaterMark) } else { assert(newHighWaterMark < badWaterMark) } } } for { allowExplicitInsert <- DeltaTestUtils.BOOLEAN_DOMAIN allowLoweringHighWatermarkForSyncIdentity <- DeltaTestUtils.BOOLEAN_DOMAIN } test(s"IdentityColumn.updateToValidHighWaterMark - allowExplicitInsert = $allowExplicitInsert," + s" allowLoweringHighWatermarkForSyncIdentity = $allowLoweringHighWatermarkForSyncIdentity") { /** * Unit test for the updateToValidHighWaterMark function by creating a StructField with the * specified start, step, and existing high water mark. After calling the function, we verify * the StructField's metadata has the expect high water mark. */ def testUpdateToValidHighWaterMark( start: Long, step: Long, allowExplicitInsert: Boolean, allowLoweringHighWatermarkForSyncIdentity: Boolean, existingHighWaterMark: Option[Long], candidateHighWaterMark: Long, expectedHighWaterMark: Option[Long]): Unit = { /** Creates a MetadataBuilder for Struct Metadata. */ def getMetadataBuilder(highWaterMarkOpt: Option[Long]): MetadataBuilder = { var metadataBuilder = new MetadataBuilder() .putLong(DeltaSourceUtils.IDENTITY_INFO_START, start) .putLong(DeltaSourceUtils.IDENTITY_INFO_STEP, step) .putBoolean(DeltaSourceUtils.IDENTITY_INFO_ALLOW_EXPLICIT_INSERT, allowExplicitInsert) highWaterMarkOpt match { case Some(oldHighWaterMark) => metadataBuilder = metadataBuilder.putLong( DeltaSourceUtils.IDENTITY_INFO_HIGHWATERMARK, oldHighWaterMark) case None => () } metadataBuilder } val initialStructField = StructField( name = "id", LongType, nullable = false, metadata = getMetadataBuilder(existingHighWaterMark).build()) val (updatedStructField, _) = IdentityColumn.updateToValidHighWaterMark( initialStructField, candidateHighWaterMark, allowLoweringHighWatermarkForSyncIdentity) val expectedMetadata = getMetadataBuilder(expectedHighWaterMark).build() assert(updatedStructField.metadata === expectedMetadata) } // existingHighWaterMark = None, positive step testUpdateToValidHighWaterMark( start = 1L, step = 3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = None, candidateHighWaterMark = 2L, expectedHighWaterMark = Some(4L) // rounded up ) testUpdateToValidHighWaterMark( start = 1L, step = 3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = None, candidateHighWaterMark = 0L, expectedHighWaterMark = None // below start ) testUpdateToValidHighWaterMark( start = 1L, step = 3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = None, candidateHighWaterMark = 1L, expectedHighWaterMark = Some(1L) // equal to start ) testUpdateToValidHighWaterMark( start = 1L, step = 3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = None, candidateHighWaterMark = 7L, expectedHighWaterMark = Some(7L) // respects start and step ) // existingHighWaterMark = None, negative step testUpdateToValidHighWaterMark( start = 1L, step = -3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = None, candidateHighWaterMark = -1L, expectedHighWaterMark = Some(-2L) // rounded up ) testUpdateToValidHighWaterMark( start = 1L, step = -3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = None, candidateHighWaterMark = 2L, expectedHighWaterMark = None // above start ) testUpdateToValidHighWaterMark( start = 1L, step = -3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = None, candidateHighWaterMark = 1L, expectedHighWaterMark = Some(1L) // equal to start ) testUpdateToValidHighWaterMark( start = 1L, step = -3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = None, candidateHighWaterMark = -5L, expectedHighWaterMark = Some(-5L) // respects start and step ) // existingHighWaterMark = Some, positive step testUpdateToValidHighWaterMark( start = 1L, step = 3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = Some(4L), candidateHighWaterMark = 5L, expectedHighWaterMark = Some(7L) // rounded up ) testUpdateToValidHighWaterMark( start = 1L, step = 3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = Some(4L), candidateHighWaterMark = 0L, expectedHighWaterMark = Some(4L) // below start, preserve existing high watermark ) testUpdateToValidHighWaterMark( start = 1L, step = 3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = Some(-5L), candidateHighWaterMark = -2L, expectedHighWaterMark = Some(-2L) // below start, bad existing water mark, update to candidate ) testUpdateToValidHighWaterMark( start = 1L, step = 3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = Some(-5L), candidateHighWaterMark = 0L, expectedHighWaterMark = Some(1L) // below start, bad existing water mark, update rounded up ) testUpdateToValidHighWaterMark( start = 1L, step = 3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = Some(-5L), candidateHighWaterMark = -9L, expectedHighWaterMark = if (allowLoweringHighWatermarkForSyncIdentity) { // below start, bad existing water mark, allow lowering, rounded down Some(-8L) } else { // below start, bad existing water mark, keep existing Some(-5L) } ) testUpdateToValidHighWaterMark( start = 1L, step = 3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = Some(4L), candidateHighWaterMark = 1L, expectedHighWaterMark = if (allowLoweringHighWatermarkForSyncIdentity) { Some(1L) // allow lowering } else { Some(4L) // below existing high watermark } ) testUpdateToValidHighWaterMark( start = 1L, step = 3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = Some(4L), candidateHighWaterMark = 7L, expectedHighWaterMark = Some(7L) // respects start and step ) // existingHighWaterMark = Some, negative step testUpdateToValidHighWaterMark( start = 1L, step = -3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = Some(-2L), candidateHighWaterMark = -3L, expectedHighWaterMark = Some(-5L) // rounded up ) testUpdateToValidHighWaterMark( start = 1L, step = -3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = Some(-2L), candidateHighWaterMark = 2L, expectedHighWaterMark = Some(-2L) // above start, preserve existing high water mark ) testUpdateToValidHighWaterMark( start = 1L, step = -3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = Some(7L), candidateHighWaterMark = 4L, expectedHighWaterMark = Some(4L) // above start, bad existing water mark, update to candidate ) testUpdateToValidHighWaterMark( start = 1L, step = -3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = Some(7L), candidateHighWaterMark = 6L, expectedHighWaterMark = Some(4L) // above start, bad existing water mark, update rounded down ) testUpdateToValidHighWaterMark( start = 1L, step = -3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = Some(7L), candidateHighWaterMark = 11L, expectedHighWaterMark = if (allowLoweringHighWatermarkForSyncIdentity) { // above start, bad existing water mark, allow lowering, rounded down Some(10L) } else { // above start, bad existing water mark, keep existing Some(7L) } ) testUpdateToValidHighWaterMark( start = 1L, step = -3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = Some(-2L), candidateHighWaterMark = 1L, expectedHighWaterMark = if (allowLoweringHighWatermarkForSyncIdentity) { Some(1L) // allow lowering } else { Some(-2L) // higher than high watermark } ) testUpdateToValidHighWaterMark( start = 1L, step = -3L, allowExplicitInsert = allowExplicitInsert, allowLoweringHighWatermarkForSyncIdentity = allowLoweringHighWatermarkForSyncIdentity, existingHighWaterMark = Some(-2L), candidateHighWaterMark = -5L, expectedHighWaterMark = Some(-5L) // respects start and step ) } test("IdentityColumn.roundToNext") { val posStart = 7L val negStart = -7L val posLargeStart = Long.MaxValue - 10000 val negLargeStart = Long.MinValue + 10000 for (start <- Seq(posStart, negStart, posLargeStart, negLargeStart)) { assert(IdentityColumn.roundToNext(start = start, step = 3L, value = start) === start) assert(IdentityColumn.roundToNext( start = start, step = 3L, value = start + 5L) === start + 6L) assert(IdentityColumn.roundToNext( start = start, step = 3L, value = start + 6L) === start + 6L) assert(IdentityColumn.roundToNext( start = start, step = 3L, value = start - 5L) === start - 3L) // bad watermark assert(IdentityColumn.roundToNext( start = start, step = 3L, value = start - 7L) === start - 6L) // bad watermark assert(IdentityColumn.roundToNext( start = start, step = 3L, value = start - 6L) === start - 6L) // bad watermark assert(IdentityColumn.roundToNext(start = start, step = -3L, value = start) === start) assert(IdentityColumn.roundToNext( start = start, step = -3L, value = start - 5L) === start - 6L) assert(IdentityColumn.roundToNext( start = start, step = -3L, value = start - 6L) === start - 6L) assert(IdentityColumn.roundToNext( start = start, step = -3L, value = start + 5L) === start + 3L) // bad watermark assert(IdentityColumn.roundToNext( start = start, step = -3L, value = start + 7L) === start + 6L) // bad watermark assert(IdentityColumn.roundToNext( start = start, step = -3L, value = start + 6L) === start + 6L) // bad watermark } } } class IdentityColumnSyncScalaSuite extends IdentityColumnSyncSuiteBase with ScalaDDLTestUtils class IdentityColumnSyncScalaIdColumnMappingSuite extends IdentityColumnSyncSuiteBase with ScalaDDLTestUtils with DeltaColumnMappingEnableIdMode class IdentityColumnSyncScalaNameColumnMappingSuite extends IdentityColumnSyncSuiteBase with ScalaDDLTestUtils with DeltaColumnMappingEnableNameMode ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/IdentityColumnTestUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.UUID import org.apache.spark.sql.delta.GeneratedAsIdentityType.{GeneratedAlways, GeneratedAsIdentityType} import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf} import org.apache.spark.SparkConf import org.apache.spark.sql.Row import org.apache.spark.sql.types._ trait IdentityColumnTestUtils extends DDLTestUtils { protected override def sparkConf: SparkConf = { super.sparkConf .set(DeltaSQLConf.DELTA_IDENTITY_COLUMN_ENABLED.key, "true") } protected def getRandomTableName: String = s"identity_test_${UUID.randomUUID()}".replaceAll("-", "_") protected val unsupportedDataTypes: Seq[DataType] = Seq( BooleanType, ByteType, ShortType, IntegerType, DoubleType, DateType, TimestampType, StringType, BinaryType, DecimalType(precision = 5, scale = 2), YearMonthIntervalType(startField = 0, endField = 0) // Interval Year ) def createTableWithIdColAndIntValueCol( tableName: String, generatedAsIdentityType: GeneratedAsIdentityType, startsWith: Option[Long], incrementBy: Option[Long], tblProperties: Map[String, String] = Map.empty): Unit = { createTable( tableName, Seq( IdentityColumnSpec( generatedAsIdentityType, startsWith, incrementBy ), TestColumnSpec(colName = "value", dataType = IntegerType) ), tblProperties = tblProperties ) } /** * Creates and manages a simple identity column table with one other column "value" of type int */ protected def withIdentityColumnTable( generatedAsIdentityType: GeneratedAsIdentityType, tableName: String)(f: => Unit): Unit = { withTable(tableName) { createTableWithIdColAndIntValueCol(tableName, generatedAsIdentityType, None, None) f } } protected def generateTableWithIdentityColumn(tableName: String, step: Long = 1): Unit = { createTableWithIdColAndIntValueCol( tableName, GeneratedAlways, startsWith = Some(0), incrementBy = Some(step) ) // Insert numRows and make sure they assigned sequential IDs val numRows = 6 for (i <- 0 until numRows) { sql(s"INSERT INTO $tableName (value) VALUES ($i)") } val expectedAnswer = for (i <- 0 until numRows) yield Row(i * step, i) checkAnswer(sql(s"SELECT * FROM $tableName ORDER BY value ASC"), expectedAnswer) } /** * Retrieves the high watermark information for the given `colName` in the metadata of * given `snapshot`, if it's present. Returns None if the high watermark has not been set yet. */ protected def getHighWaterMark(snapshot: Snapshot, colName: String): Option[Long] = { val metadata = snapshot.schema(colName).metadata if (metadata.contains(DeltaSourceUtils.IDENTITY_INFO_HIGHWATERMARK)) { Some(metadata.getLong(DeltaSourceUtils.IDENTITY_INFO_HIGHWATERMARK)) } else { None } } /** * Retrieves the high watermark information for the given `colName` in the metadata of * given `snapshot` */ protected def highWaterMark(snapshot: Snapshot, colName: String): Long = { getHighWaterMark(snapshot, colName).get } /** * Helper function to validate values of IDENTITY column `id` in table `tableName`. Returns the * new high water mark. We use minValue and maxValue to filter column `value` to get the set of * values we are checking in this batch. */ protected def validateIdentity( tableName: String, expectedRowCount: Long, start: Long, step: Long, minValue: Long, maxValue: Long, oldHighWaterMark: Long): Long = { // Check row count. checkAnswer( sql(s"SELECT COUNT(*) FROM $tableName"), Row(expectedRowCount) ) // Check values are unique. checkAnswer( sql(s"SELECT COUNT(DISTINCT id) FROM $tableName"), Row(expectedRowCount) ) // Check values follow start and step configuration. checkAnswer( sql(s"SELECT COUNT(*) FROM $tableName WHERE (id - $start) % $step != 0"), Row(0) ) // Check values generated in this batch are after previous high water mark. checkAnswer( sql( s""" |SELECT COUNT(*) FROM $tableName | WHERE (value BETWEEN $minValue and $maxValue) | AND ((id - $oldHighWaterMark) / $step < 0) |""".stripMargin), Row(0) ) // Update high water mark. val func = if (step > 0) "MAX" else "MIN" sql(s"SELECT $func(id) FROM $tableName").collect().head.getLong(0) } /** * Helper function to validate generated identity values in sortedRows. * * @param sortedRows rows of the table sorted by id * @param start start value of the identity column * @param step step value of the identity column * @param expectedLowerBound expected lower bound of the generated values * @param expectedUpperBound expected upper bound of the generated values * @param expectedDistinctCount expected distinct count of the generated values */ protected def checkGeneratedIdentityValues( sortedRows: Seq[IdentityColumnTestTableRow], start: Long, step: Long, expectedLowerBound: Long, expectedUpperBound: Long, expectedDistinctCount: Long): Unit = { assert(sortedRows.head.id >= expectedLowerBound) for (row <- sortedRows) { assert((row.id - start) % step === 0) } assert(sortedRows.last.id <= expectedUpperBound) assert(sortedRows.map(_.id).distinct.size === expectedDistinctCount) } /** Force a bad high water mark on all identity columns in the table with a manual commit. */ def forceBadWaterMark(deltaLog: DeltaLog): Unit = { deltaLog.withNewTransaction { txn => // Manually corrupt the high water mark. val tblSchema = txn.snapshot.schema val badTblSchema = StructType(tblSchema.map { case tblIdCol if ColumnWithDefaultExprUtils.isIdentityColumn(tblIdCol) => val identityInfo = IdentityColumn.getIdentityInfo(tblIdCol) // This bad water mark needs to follow the step and start, // otherwise we fail the requirement in GenerateIdentityValues val badWaterMark = identityInfo.start - identityInfo.step * 1000 val tblColMetadata = tblIdCol.metadata val badMetadata = new MetadataBuilder().withMetadata(tblColMetadata) .putLong(DeltaSourceUtils.IDENTITY_INFO_HIGHWATERMARK, badWaterMark).build() tblIdCol.copy(metadata = badMetadata) case f => f }) val updatedMetadata = txn.snapshot.metadata.copy(schemaString = badTblSchema.json) txn.updateMetadata(updatedMetadata, ignoreDefaultProperties = false) txn.commit(Nil, DeltaOperations.ManualUpdate) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/ImplicitDMLCastingSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.annotation.tailrec import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaExceptionTestUtils, DeltaSQLCommandTest} import org.apache.spark.{SparkConf, SparkException, SparkThrowable} import org.apache.spark.sql.{DataFrame, QueryTest} import org.apache.spark.sql.Row import org.apache.spark.sql.internal.SQLConf /** * Tests for casts that are implicitly added in DML commands modifying Delta tables. * These casts are added to convert values to the schema of a table. * INSERT operations are excluded as they are covered by InsertSuite and InsertSuiteEdge. */ abstract class ImplicitDMLCastingSuite extends QueryTest with DeltaExceptionTestUtils with DeltaSQLCommandTest { /** Implement the actual test for a specific DML command in subclasses. */ protected def commandTest(sqlConfig: SqlConfiguration, testConfig: TestConfiguration): Unit protected case class TestConfiguration( sourceType: String, sourceTypeInErrorMessage: String, targetType: String, targetTypeInErrorMessage: String, validValue: String, overflowValue: String, // String because SparkArithmeticException is private and cannot be used for matching. exceptionAnsiCast: String ) { override def toString: String = s"sourceType: $sourceType, targetType: $targetType" } protected case class SqlConfiguration( followAnsiEnabled: Boolean, ansiEnabled: Boolean, storeAssignmentPolicy: SQLConf.StoreAssignmentPolicy.Value) { def withSqlSettings(f: => Unit): Unit = withSQLConf( DeltaSQLConf.UPDATE_AND_MERGE_CASTING_FOLLOWS_ANSI_ENABLED_FLAG.key -> followAnsiEnabled.toString, SQLConf.STORE_ASSIGNMENT_POLICY.key -> storeAssignmentPolicy.toString, SQLConf.ANSI_ENABLED.key -> ansiEnabled.toString)(f) override def toString: String = s"followAnsiEnabled: $followAnsiEnabled, ansiEnabled: $ansiEnabled," + s" storeAssignmentPolicy: $storeAssignmentPolicy" } protected def expectLegacyCastingBehaviour(sqlConfig: SqlConfiguration): Boolean = { (sqlConfig.followAnsiEnabled && !sqlConfig.ansiEnabled) || (!sqlConfig.followAnsiEnabled && sqlConfig.storeAssignmentPolicy == SQLConf.StoreAssignmentPolicy.LEGACY) } // Note that DATE to TIMESTAMP casts are not in this list as they always throw an error on // overflow no matter if ANSI is enabled or not. private val testConfigurations = Seq( TestConfiguration(sourceType = "INT", sourceTypeInErrorMessage = "INT", targetType = "TINYINT", targetTypeInErrorMessage = "TINYINT", validValue = "1", overflowValue = Int.MaxValue.toString, exceptionAnsiCast = "SparkArithmeticException"), TestConfiguration(sourceType = "INT", sourceTypeInErrorMessage = "INT", targetType = "SMALLINT", targetTypeInErrorMessage = "SMALLINT", validValue = "1", overflowValue = Int.MaxValue.toString, exceptionAnsiCast = "SparkArithmeticException"), TestConfiguration(sourceType = "BIGINT", sourceTypeInErrorMessage = "BIGINT", targetType = "INT", targetTypeInErrorMessage = "INT", validValue = "1", overflowValue = Long.MaxValue.toString, exceptionAnsiCast = "SparkArithmeticException"), TestConfiguration(sourceType = "DOUBLE", sourceTypeInErrorMessage = "DOUBLE", targetType = "BIGINT", targetTypeInErrorMessage = "BIGINT", validValue = "1", overflowValue = "12345678901234567890D", exceptionAnsiCast = "SparkArithmeticException"), TestConfiguration(sourceType = "BIGINT", sourceTypeInErrorMessage = "BIGINT", targetType = "DECIMAL(7,2)", targetTypeInErrorMessage = "DECIMAL(7,2)", validValue = "1", overflowValue = Long.MaxValue.toString, exceptionAnsiCast = "SparkArithmeticException"), TestConfiguration(sourceType = "Struct", sourceTypeInErrorMessage = "BIGINT", targetType = "Struct", targetTypeInErrorMessage = "INT", validValue = "named_struct('value', 1)", overflowValue = s"named_struct('value', ${Long.MaxValue.toString})", exceptionAnsiCast = "SparkArithmeticException"), TestConfiguration(sourceType = "ARRAY", sourceTypeInErrorMessage = "ARRAY", targetType = "ARRAY", targetTypeInErrorMessage = "ARRAY", validValue = "ARRAY(1)", overflowValue = s"ARRAY(${Long.MaxValue.toString})", exceptionAnsiCast = "SparkArithmeticException"), TestConfiguration(sourceType = "STRING", sourceTypeInErrorMessage = "STRING", targetType = "INT", targetTypeInErrorMessage = "INT", validValue = "'1'", overflowValue = s"'${Long.MaxValue.toString}'", exceptionAnsiCast = "SparkNumberFormatException"), TestConfiguration(sourceType = "MAP", sourceTypeInErrorMessage = "MAP", targetType = "MAP", targetTypeInErrorMessage = "MAP", validValue = "map('abc', 1)", overflowValue = s"map('abc', ${Long.MaxValue.toString})", exceptionAnsiCast = "SparkArithmeticException"), TestConfiguration(sourceType = "DECIMAL(3,1)", sourceTypeInErrorMessage = "DECIMAL(3,1)", targetType = "DECIMAL(3,2)", targetTypeInErrorMessage = "DECIMAL(3,2)", validValue = "CAST(1 AS DECIMAL(3,1))", overflowValue = s"CAST(12.3 AS DECIMAL(3,1))", exceptionAnsiCast = "SparkArithmeticException") ) /** Returns cast failure exception if present in the cause chain. None otherwise. */ @tailrec private def castFailureCause(exception: Throwable): Option[Throwable] = { exception match { case arithmeticException: ArithmeticException => Some(arithmeticException) case numberFormatException: NumberFormatException => Some(numberFormatException) case _ if exception.getCause != null => castFailureCause(exception.getCause) case _ => None } } /** * Validate that a custom error is throws in case ansi.enabled is false, or a different * overflow error is case ansi.enabled is true. */ protected def validateException( exception: Throwable, sqlConfig: SqlConfiguration, testConfig: TestConfiguration): Unit = { // Validate that the type of error matches the expected error type. castFailureCause(exception) match { case Some(failureCause) if sqlConfig.followAnsiEnabled => assert(sqlConfig.ansiEnabled) assert(failureCause.toString.contains(testConfig.exceptionAnsiCast)) val sparkThrowable = failureCause.asInstanceOf[SparkThrowable] assert(Seq( "CAST_OVERFLOW", "NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION", "CAST_INVALID_INPUT" ).contains(sparkThrowable.getErrorClass)) case Some(failureCause) if !sqlConfig.followAnsiEnabled => assert(sqlConfig.storeAssignmentPolicy === SQLConf.StoreAssignmentPolicy.ANSI) val sparkThrowable = failureCause.asInstanceOf[SparkThrowable] // Only arithmetic exceptions get a custom error message. if (testConfig.exceptionAnsiCast == "SparkArithmeticException") { assert(sparkThrowable.getErrorClass == "DELTA_CAST_OVERFLOW_IN_TABLE_WRITE") assert(sparkThrowable.getMessageParameters == Map("sourceType" -> ("\"" + testConfig.sourceTypeInErrorMessage + "\""), "targetType" -> ("\"" + testConfig.targetTypeInErrorMessage + "\""), "columnName" -> "`value`", "storeAssignmentPolicyFlag" -> SQLConf.STORE_ASSIGNMENT_POLICY.key, "updateAndMergeCastingFollowsAnsiEnabledFlag" -> DeltaSQLConf.UPDATE_AND_MERGE_CASTING_FOLLOWS_ANSI_ENABLED_FLAG.key, "ansiEnabledFlag" -> SQLConf.ANSI_ENABLED.key).asJava) } else { assert(sparkThrowable.getErrorClass == "CAST_INVALID_INPUT") assert(sparkThrowable.getMessageParameters.get("sourceType") == "\"STRING\"") } case None => assert(false, s"No arithmetic exception thrown: $exception") } } for { followAnsiEnabled <- BOOLEAN_DOMAIN ansiEnabled <- BOOLEAN_DOMAIN storeAssignmentPolicy <- Seq(SQLConf.StoreAssignmentPolicy.LEGACY, SQLConf.StoreAssignmentPolicy.ANSI) sqlConfiguration <- Some(SqlConfiguration(followAnsiEnabled, ansiEnabled, storeAssignmentPolicy)) testConfiguration <- testConfigurations } commandTest(sqlConfiguration, testConfiguration) test("Details are part of the error message") { val sourceTableName = "source_table_name" val sourceValueType = "INT" val targetTableName = "target_table_name" val targetValueType = "LONG" val valueColumnName = "value" withTable(sourceTableName, targetTableName) { sql(s"CREATE OR REPLACE TABLE $targetTableName(id LONG, $valueColumnName $sourceValueType) " + "USING DELTA") sql(s"CREATE OR REPLACE TABLE $sourceTableName(id LONG, $valueColumnName $targetValueType) " + "USING DELTA") sql(s"INSERT INTO $sourceTableName VALUES(0, 9223372036854775807)") val userFacingError = interceptWithUnwrapping[DeltaArithmeticException] { sql(s"""MERGE INTO $targetTableName t |USING $sourceTableName s |ON s.id = t.id |WHEN NOT MATCHED THEN INSERT *""".stripMargin) } val expectedDetails = Seq("DELTA_CAST_OVERFLOW_IN_TABLE_WRITE", sourceValueType, valueColumnName) for (detail <- expectedDetails) { assert(userFacingError.toString.contains(detail)) } } } } class ImplicitUpdateCastingSuite extends ImplicitDMLCastingSuite { /** Test an UPDATE that requires to cast the update value that is part of the SET clause. */ override protected def commandTest( sqlConfig: SqlConfiguration, testConfig: TestConfiguration): Unit = { val testName = s"UPDATE overflow $testConfig $sqlConfig" test(testName) { sqlConfig.withSqlSettings { val tableName = "overflowTable" withTable(tableName) { sql(s"""CREATE TABLE $tableName USING DELTA |AS SELECT cast(${testConfig.validValue} AS ${testConfig.targetType}) AS value |""".stripMargin) val updateCommand = s"UPDATE $tableName SET value = ${testConfig.overflowValue}" if (expectLegacyCastingBehaviour(sqlConfig)) { sql(updateCommand) } else { val exception = intercept[Throwable] { sql(updateCommand) } validateException(exception, sqlConfig, testConfig) } } } } } for (preserveNullSourceStructs <- BOOLEAN_DOMAIN) { test(s"Implicit cast with NULL struct, preserveNullSourceStructs=$preserveNullSourceStructs") { withSQLConf(DeltaSQLConf.DELTA_MERGE_PRESERVE_NULL_SOURCE_STRUCTS.key -> preserveNullSourceStructs.toString) { val tableName = "struct_null_expansion_test" withTable(tableName) { sql( s"""CREATE TABLE $tableName ( | col1 STRUCT, | col2 STRUCT |) USING DELTA""".stripMargin) sql(s"INSERT INTO $tableName VALUES (NULL, NULL)") // col1: cast col1.x from INT to LONG // col2: reorder col2.x and col2.y sql( s"""UPDATE $tableName SET | col1 = CAST(NULL AS STRUCT), | col2 = CAST(NULL AS STRUCT) |""".stripMargin) val expectedRow = if (preserveNullSourceStructs) { // The entire structs should be null Row(null, null) } else { // Results are structs with null fields Row(Row(null), Row(null, null)) } checkAnswer(spark.table(tableName), Seq(expectedRow)) } } } } } class ImplicitMergeCastingSuite extends ImplicitDMLCastingSuite { /** Tests for MERGE with overflows cause by the different conditions. */ override protected def commandTest( sqlConfig: SqlConfiguration, testConfig: TestConfiguration): Unit = { mergeTest(matchedCondition = s"WHEN MATCHED THEN UPDATE SET t.value = s.value", sqlConfig, testConfig) mergeTest(matchedCondition = s"WHEN NOT MATCHED THEN INSERT *", sqlConfig, testConfig) mergeTest(matchedCondition = s"WHEN NOT MATCHED BY SOURCE THEN UPDATE SET t.value = ${testConfig.overflowValue}", sqlConfig, testConfig) } private def mergeTest( matchedCondition: String, sqlConfig: SqlConfiguration, testConfig: TestConfiguration): Unit = { val testName = s"MERGE overflow in $matchedCondition $testConfig $sqlConfig" test(testName) { sqlConfig.withSqlSettings { val targetTableName = "target_table" val sourceViewName = "source_view" withTable(targetTableName) { withTempView(sourceViewName) { val numRows = 10 sql(s"""CREATE TABLE $targetTableName USING DELTA |AS SELECT col as key, | cast(${testConfig.validValue} AS ${testConfig.targetType}) AS value |FROM explode(sequence(0, $numRows))""".stripMargin) // The view maps the key space such that we get matched, not matched by source, and // not match by target rows. sql(s"""CREATE TEMPORARY VIEW $sourceViewName |AS SELECT key + ($numRows / 2) AS key, | cast(${testConfig.overflowValue} AS ${testConfig.sourceType}) AS value |FROM $targetTableName""".stripMargin) val mergeCommand = s"""MERGE INTO $targetTableName t |USING $sourceViewName s |ON s.key = t.key |$matchedCondition |""".stripMargin if (expectLegacyCastingBehaviour(sqlConfig)) { sql(mergeCommand) } else { val exception = intercept[Throwable] { sql(mergeCommand) } validateException(exception, sqlConfig, testConfig) } } } } } } } class ImplicitStreamingMergeCastingSuite extends ImplicitDMLCastingSuite { /** A merge that is executed for each batch of a stream and has to cast values before insert. */ override protected def commandTest( sqlConfig: SqlConfiguration, testConfig: TestConfiguration): Unit = { val testName = s"Streaming MERGE overflow $testConfig $sqlConfig" test(testName) { sqlConfig.withSqlSettings { val targetTableName = "target_table" val sourceTableName = "source_table" withTable(sourceTableName, targetTableName) { sql(s"CREATE TABLE $targetTableName (key INT, value ${testConfig.targetType})" + " USING DELTA") sql(s"CREATE TABLE $sourceTableName (key INT, value ${testConfig.sourceType})" + " USING DELTA") def upsertToDelta(microBatchOutputDF: DataFrame, batchId: Long): Unit = { microBatchOutputDF.createOrReplaceTempView("micro_batch_output") microBatchOutputDF.sparkSession.sql(s"""MERGE INTO $targetTableName t |USING micro_batch_output s |ON s.key = t.key |WHEN NOT MATCHED THEN INSERT * |""".stripMargin) } val sourceStream = spark.readStream.table(sourceTableName) val streamWriter = sourceStream .writeStream .format("delta") .foreachBatch(upsertToDelta _) .outputMode("update") .start() sql(s"INSERT INTO $sourceTableName(key, value) VALUES(0, ${testConfig.overflowValue})") if (expectLegacyCastingBehaviour(sqlConfig)) { streamWriter.processAllAvailable() } else { val exception = intercept[Throwable] { streamWriter.processAllAvailable() } validateException(exception, sqlConfig, testConfig) } } } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/InCommitTimestampSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.sql.Timestamp import scala.collection.JavaConverters._ import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta.DeltaOperations.ManualUpdate import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile import org.apache.spark.sql.delta.actions.{Action, CommitInfo} import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedCommitCoordinatorProvider, CatalogOwnedTableUtils, CatalogOwnedTestBaseSuite, CommitCoordinatorProvider, CommitCoordinatorUtilBase, TrackingInMemoryCommitCoordinatorBuilder} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.{DateTimeUtils, DeltaCommitFileProvider, FileNames, JsonUtils, TimestampFormatter} import org.apache.hadoop.fs.Path import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.col import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.streaming.StreamTest import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.ManualClock class InCommitTimestampSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with DeltaSQLTestUtils with DeltaTestUtilsBase with CatalogOwnedTestBaseSuite with StreamTest { import InCommitTimestampTestUtils._ override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey, "true") spark.conf.set(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key, "true") spark.conf.set(DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key, "true") } /** * Create a delta table with 10 rows and a single commit with an AddFile. * This is used to create the initial state of the delta table for testing. * * @param tableName The name of the delta table. * @return The DeltaLog for the created table. */ private def createInitialTable(tableName: String): DeltaLog = { spark.range(10).write.format("delta").saveAsTable(tableName) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) deltaLog.startTransaction().commit(Seq(createTestAddFile("1")), ManualUpdate) deltaLog } /** * Construct a delta log w/ a specific manual clock. * * @param tableName The name of the Delta table. * @param clock The manual clock to use for the DeltaLog. */ private def getDeltaLogWithClock(tableName: String, clock: ManualClock): DeltaLog = { // Construct a delta log by name first. val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) // Get the data path. val dataPath = deltaLog.dataPath // Clear the cache to ensure that a fresh DeltaLog is created with the provided clock. DeltaLog.clearCache() // Create a new DeltaLog with the clock and the log path. DeltaLog.forTable(spark, dataPath, clock) } test("Enable ICT on commit 0") { withTempTable(createTable = false) { tableName => spark.range(10).write.format("delta").saveAsTable(tableName) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val ver0Snapshot = deltaLog.snapshot assert(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(ver0Snapshot.metadata)) assert(ver0Snapshot.timestamp == getInCommitTimestamp(deltaLog, 0)) } } // Catalog Owned will also automatically enable ICT. testWithDefaultCommitCoordinatorUnset( "Create a non-inCommitTimestamp table and then enable timestamp") { withSQLConf( DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey -> false.toString ) { withTempTable(createTable = false) { tableName => spark.range(10).write.format("delta").saveAsTable(tableName) spark.sql(s"INSERT INTO $tableName VALUES 10") val ver1Snapshot = DeltaLog.forTable(spark, TableIdentifier(tableName)).snapshot // File timestamp should be the same as snapshot.getTimestamp when inCommitTimestamp is not // enabled assert( ver1Snapshot.logSegment.lastCommitFileModificationTimestamp == ver1Snapshot.timestamp) spark.sql(s"ALTER TABLE $tableName " + s"SET TBLPROPERTIES ('${DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key}' = 'true')") val ver2Snapshot = DeltaLog.forTable(spark, TableIdentifier(tableName)).snapshot // File timestamp should be different from snapshot.getTimestamp when inCommitTimestamp is // enabled assert(ver2Snapshot.timestamp == getInCommitTimestamp(ver2Snapshot.deltaLog, version = 2)) assert(ver2Snapshot.timestamp > ver1Snapshot.timestamp) spark.sql(s"INSERT INTO $tableName VALUES 11") val ver3Snapshot = DeltaLog.forTable(spark, TableIdentifier(tableName)).snapshot assert(ver3Snapshot.timestamp > ver2Snapshot.timestamp) } } } test("InCommitTimestamps are monotonic even when the clock is skewed") { withTempTable(createTable = false) { tableName => spark.range(10).write.format("delta").saveAsTable(tableName) val startTime = System.currentTimeMillis() val clock = new ManualClock(startTime) val deltaLog = getDeltaLogWithClock(tableName, clock) // Move backwards in time. deltaLog.startTransaction().commit(Seq(createTestAddFile("1")), ManualUpdate) val ver1Timestamp = deltaLog.snapshot.timestamp clock.setTime(startTime - 10000) deltaLog.startTransaction().commit(Seq(createTestAddFile("2")), ManualUpdate) val ver2Timestamp = deltaLog.snapshot.timestamp assert(ver2Timestamp > ver1Timestamp) } } test("Conflict resolution of timestamps") { withTempTable(createTable = false) { tableName => spark.range(10).write.format("delta").saveAsTable(tableName) val startTime = System.currentTimeMillis() val clock = new ManualClock(startTime) // Clear the cache to ensure that a new DeltaLog is created with the new clock. val deltaLog = getDeltaLogWithClock(tableName, clock) val txn1 = deltaLog.startTransaction() clock.setTime(startTime) deltaLog.startTransaction().commit(Seq(createTestAddFile("1")), ManualUpdate) // Move time backwards for the conflicting commit. clock.setTime(startTime - 10000) val usageRecords = Log4jUsageLogger.track { txn1.commit(Seq(createTestAddFile("2")), ManualUpdate) } // Make sure that this transaction resulted in a conflict. assert(filterUsageRecords(usageRecords, "delta.commit.retry").length == 1) assert(getInCommitTimestamp(deltaLog, 2) > getInCommitTimestamp(deltaLog, 1)) } } for (useCommitLarge <- BOOLEAN_DOMAIN) test("txn.commit should use clock.currentTimeMillis() for ICT" + s" [useCommitLarge: $useCommitLarge]") { withTempTable(createTable = false) { tableName => spark.range(2).write.format("delta").saveAsTable(tableName) val expectedCommit1Time = System.currentTimeMillis() val clock = new ManualClock(expectedCommit1Time) val deltaLog = getDeltaLogWithClock(tableName, clock) val ver0Snapshot = deltaLog.snapshot assert(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(ver0Snapshot.metadata)) val usageRecords = Log4jUsageLogger.track { if (useCommitLarge) { deltaLog.startTransaction().commitLarge( spark, Seq(createTestAddFile("1")).toIterator, newProtocolOpt = None, DeltaOperations.ManualUpdate, context = Map.empty, metrics = Map.empty) } else { deltaLog.startTransaction().commit( Seq(createTestAddFile("1")), DeltaOperations.ManualUpdate, tags = Map.empty ) } } val ver1Snapshot = deltaLog.snapshot val retrievedTimestamp = getInCommitTimestamp(deltaLog, version = 1) assert(ver1Snapshot.timestamp == retrievedTimestamp) assert(ver1Snapshot.timestamp == expectedCommit1Time) val expectedOpType = if (useCommitLarge) "delta.commit.large" else "delta.commit" assert(filterUsageRecords(usageRecords, expectedOpType).length == 1) } } test("Missing CommitInfo should result in a DELTA_MISSING_COMMIT_INFO exception") { // Make sure that we don't retrieve the time from the CRC. withSQLConf(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> "false") { withTempTable(createTable = false) { tableName => val deltaLog = createInitialTable(tableName) // Remove CommitInfo from the commit. val commit1Path = DeltaCommitFileProvider(deltaLog.unsafeVolatileSnapshot).deltaFile(1) val actions = deltaLog.store.readAsIterator(commit1Path, deltaLog.newDeltaHadoopConf()) val actionsWithoutCommitInfo = actions.filterNot(Action.fromJson(_).isInstanceOf[CommitInfo]) deltaLog.store.write( commit1Path, actionsWithoutCommitInfo, overwrite = true, deltaLog.newDeltaHadoopConf()) DeltaLog.clearCache() val latestSnapshot = DeltaLog.forTable(spark, TableIdentifier(tableName)).snapshot val e = intercept[DeltaIllegalStateException] { latestSnapshot.timestamp } checkError( e, "DELTA_MISSING_COMMIT_INFO", parameters = Map( "featureName" -> InCommitTimestampTableFeature.name, "version" -> "1")) } } } test("Missing CommitInfo.commitTimestamp should result in a " + "DELTA_MISSING_COMMIT_TIMESTAMP exception") { // Make sure that we don't retrieve the time from the CRC. withSQLConf(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> "false") { withTempTable(createTable = false) { tableName => val deltaLog = createInitialTable(tableName) // Remove CommitInfo.commitTimestamp from the commit. val commit1Path = DeltaCommitFileProvider(deltaLog.unsafeVolatileSnapshot).deltaFile(1) val actions = deltaLog.store.readAsIterator( commit1Path, deltaLog.newDeltaHadoopConf()).toList val actionsWithoutCommitInfoCommitTimestamp = actions.map(Action.fromJson).map { case ci: CommitInfo => ci.copy(inCommitTimestamp = None).json case other => other.json }.toIterator deltaLog.store.write( commit1Path, actionsWithoutCommitInfoCommitTimestamp, overwrite = true, deltaLog.newDeltaHadoopConf()) DeltaLog.clearCache() val latestSnapshot = DeltaLog.forTable(spark, TableIdentifier(tableName)).snapshot val e = intercept[DeltaIllegalStateException] { latestSnapshot.timestamp } checkError( e, "DELTA_MISSING_COMMIT_TIMESTAMP", parameters = Map("featureName" -> InCommitTimestampTableFeature.name, "version" -> "1")) } } } test("InCommitTimestamp is equal to snapshot.timestamp") { withTempTable(createTable = false) { tableName => spark.range(10).write.format("delta").saveAsTable(tableName) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val ver0Snapshot = deltaLog.snapshot assert(ver0Snapshot.timestamp == getInCommitTimestamp(deltaLog, 0)) } } test("CREATE OR REPLACE should not disable ICT") { withoutDefaultCCTableFeature { withSQLConf( DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey -> false.toString ) { withTempDir { tempDir => spark.range(10).write.format("delta").save(tempDir.getAbsolutePath) spark.sql( s"ALTER TABLE delta.`${tempDir.getAbsolutePath}` " + s"SET TBLPROPERTIES ('${DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key}' = 'true')") spark.sql( s"CREATE OR REPLACE TABLE delta.`${tempDir.getAbsolutePath}` (id long) USING delta") val deltaLogAfterCreateOrReplace = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) val snapshot = deltaLogAfterCreateOrReplace.snapshot assert(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(snapshot.metadata)) assert(snapshot.timestamp == getInCommitTimestamp(deltaLogAfterCreateOrReplace, snapshot.version)) } } } } test("Enablement tracking properties should not be added if ICT is enabled on commit 0") { withTempTable(createTable = false) { tableName => spark.range(10).write.format("delta").saveAsTable(tableName) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val ver0Snapshot = deltaLog.snapshot val observedEnablementTimestamp = DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.fromMetaData(ver0Snapshot.metadata) val observedEnablementVersion = DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.fromMetaData(ver0Snapshot.metadata) assert(observedEnablementTimestamp.isEmpty) assert(observedEnablementVersion.isEmpty) } } // Catalog Owned will also automatically enable ICT. testWithDefaultCommitCoordinatorUnset( "Enablement tracking works when ICT is enabled post commit 0") { withSQLConf( DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey -> false.toString ) { withTempTable(createTable = false) { tableName => spark.range(10).write.format("delta").saveAsTable(tableName) spark.sql(s"INSERT INTO $tableName VALUES 10") spark.sql( s"ALTER TABLE $tableName " + s"SET TBLPROPERTIES ('${DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key}' = 'true')") val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val ver2Snapshot = deltaLog.snapshot val observedEnablementTimestamp = DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.fromMetaData(ver2Snapshot.metadata) val observedEnablementVersion = DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.fromMetaData(ver2Snapshot.metadata) assert(observedEnablementTimestamp.isDefined) assert(observedEnablementVersion.isDefined) assert(observedEnablementTimestamp.get == getInCommitTimestamp(deltaLog, version = 2)) assert(observedEnablementVersion.get == 2) } } } // Catalog Owned will also automatically enable ICT. testWithDefaultCommitCoordinatorUnset("Conflict resolution of enablement version") { withSQLConf( DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey -> false.toString ) { withTempTable(createTable = false) { tableName => spark.range(10).write.format("delta").saveAsTable(tableName) spark.sql(s"INSERT INTO $tableName VALUES 10") val startTime = System.currentTimeMillis() val clock = new ManualClock(startTime) val deltaLog = getDeltaLogWithClock(tableName, clock) val snapshot = deltaLog.snapshot val txn1 = deltaLog.startTransaction() clock.setTime(startTime) deltaLog.startTransaction().commit(Seq(createTestAddFile("1")), ManualUpdate) val ictEnablementMetadataConfig = snapshot.metadata.configuration ++ Map( DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key -> "true") val ictEnablementMetadata = snapshot.metadata.copy(configuration = ictEnablementMetadataConfig) val usageRecords = Log4jUsageLogger.track { txn1.commit(Seq(ictEnablementMetadata), ManualUpdate) } val ver3Snapshot = deltaLog.update() val observedEnablementTimestamp = DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.fromMetaData(ver3Snapshot.metadata) val observedEnablementVersion = DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.fromMetaData(ver3Snapshot.metadata) // Make sure that this transaction resulted in a conflict. assert(filterUsageRecords(usageRecords, "delta.commit.retry").length == 1) assert(observedEnablementTimestamp.get == getInCommitTimestamp(deltaLog, version = 3)) assert(observedEnablementVersion.get == 3) } } } // Catalog Owned will also automatically enable ICT. testWithDefaultCommitCoordinatorUnset( "commitLarge should correctly set the enablement tracking properties") { withTempTable(createTable = false) { tableName => spark.range(2).write.format("delta").saveAsTable(tableName) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val ver0Snapshot = deltaLog.snapshot assert(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(ver0Snapshot.metadata)) // Disable ICT in version 1. spark.sql( s"ALTER TABLE $tableName " + s"SET TBLPROPERTIES ('${DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key}' = 'false')") assert(!DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(deltaLog.update().metadata)) // Use a restore command to return the table state to version 0. // This should internally invoke commitLarge and the enablement tracking properties should be // updated correctly. val usageRecords = Log4jUsageLogger.track { spark.sql(s"RESTORE TABLE $tableName TO VERSION AS OF 0") } val ver2Snapshot = deltaLog.update() assert(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(ver2Snapshot.metadata)) val observedEnablementTimestamp = DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.fromMetaData(ver2Snapshot.metadata) val observedEnablementVersion = DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.fromMetaData(ver2Snapshot.metadata) assert(filterUsageRecords(usageRecords, "delta.commit.large").length == 1) assert(observedEnablementTimestamp.isDefined) assert(observedEnablementVersion.isDefined) assert(observedEnablementTimestamp.get == getInCommitTimestamp(deltaLog, version = 2)) assert(observedEnablementVersion.get == 2) } } test("snapshot.timestamp should be read from the CRC") { withTempTable(createTable = false) { tableName => var deltaLog: DeltaLog = null var timestamp = -1L val usageRecords = Log4jUsageLogger.track { spark.range(1).write.format("delta").saveAsTable(tableName) DeltaLog.clearCache() // Clear the post-commit snapshot from the cache. deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val fs = deltaLog.logPath.getFileSystem(deltaLog.newDeltaHadoopConf()) assert(fs.exists(FileNames.checksumFile(deltaLog.logPath, 0))) timestamp = deltaLog.snapshot.timestamp } assert(timestamp == getInCommitTimestamp(deltaLog, 0)) // No explicit read. assert(filterUsageRecords(usageRecords, "delta.inCommitTimestamp.read").isEmpty) } } test("postCommitSnapshot.timestamp should be populated by protocolMetadataAndICTReconstruction " + "when the table has no checkpoints") { // Make sure that we don't retrieve the time from the CRC. withSQLConf(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> "false") { withTempTable(createTable = false) { tableName => var deltaLog: DeltaLog = null var timestamp = -1L spark.range(1).write.format("delta").saveAsTable(tableName) DeltaLog.clearCache() val usageRecords = Log4jUsageLogger.track { deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) timestamp = deltaLog.snapshot.timestamp } assert(timestamp == getInCommitTimestamp(deltaLog, 0)) // No explicit read. assert(filterUsageRecords(usageRecords, "delta.inCommitTimestamp.read").isEmpty) } } } test("snapshot.timestamp should be populated by protocolMetadataAndICTReconstruction " + "during cold reads of checkpoints + deltas") { // Make sure that we don't retrieve the time from the CRC. withSQLConf(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> "false") { withTempTable(createTable = false) { tableName => var deltaLog: DeltaLog = null var timestamp = -1L spark.range(1).write.format("delta").saveAsTable(tableName) deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) deltaLog.createCheckpointAtVersion(0) deltaLog.startTransaction().commit(Seq(createTestAddFile("c1")), ManualUpdate) val usageRecords = Log4jUsageLogger.track { DeltaLog.clearCache() // Clear the post-commit snapshot from the cache. deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) timestamp = deltaLog.snapshot.timestamp } assert(deltaLog.snapshot.checkpointProvider.version == 0) assert(deltaLog.snapshot.version == 1) assert(timestamp == getInCommitTimestamp(deltaLog, 1)) // No explicit read. assert(filterUsageRecords(usageRecords, "delta.inCommitTimestamp.read").isEmpty) } } } test("snapshot.timestamp cannot be populated by protocolMetadataAndICTReconstruction " + "during cold reads of checkpoints") { // Make sure that we don't retrieve the time from the CRC. withSQLConf(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> "false") { withTempTable(createTable = false) { tableName => var deltaLog: DeltaLog = null var timestamp = -1L spark.range(1).write.format("delta").saveAsTable(tableName) DeltaLog.forTable(spark, TableIdentifier(tableName)).createCheckpointAtVersion(0) val usageRecords = Log4jUsageLogger.track { DeltaLog.clearCache() // Clear the post-commit snapshot from the cache. deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) timestamp = deltaLog.snapshot.timestamp } assert(deltaLog.snapshot.checkpointProvider.version == 0) assert(timestamp == getInCommitTimestamp(deltaLog, 0)) assert(filterUsageRecords(usageRecords, "delta.inCommitTimestamp.read").length == 1) } } } test("snapshot.timestamp is read from file when CRC doesn't have ICT and " + "the latest version has a checkpoint") { withTempTable(createTable = false) { tableName => var deltaLog: DeltaLog = null var timestamp = -1L spark.range(1).write.format("delta").saveAsTable(tableName) deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) deltaLog.createCheckpointAtVersion(0) // Remove the ICT from the CRC. InCommitTimestampTestUtils.overwriteICTInCrc(deltaLog, 0, None) val usageRecords = Log4jUsageLogger.track { DeltaLog.clearCache() // Clear the post-commit snapshot from the cache. deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) timestamp = deltaLog.snapshot.timestamp } assert(deltaLog.snapshot.checkpointProvider.version == 0) assert(timestamp == getInCommitTimestamp(deltaLog, 0)) val ictReadLog = filterUsageRecords(usageRecords, "delta.inCommitTimestamp.read").head val blob = JsonUtils.fromJson[Map[String, String]](ictReadLog.blob) assert(blob("version") == "0") assert(blob("checkpointVersion") == "0") assert(blob("isCRCPresent") == "true") } } test("Exceptions during ICT reads from file should be logged") { // Make sure that we don't retrieve the time from the CRC. withSQLConf(DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> "false") { withTempTable(createTable = false) { tableName => val deltaLog = createInitialTable(tableName) // Remove CommitInfo from the commit. val commit1Path = DeltaCommitFileProvider(deltaLog.unsafeVolatileSnapshot).deltaFile(1) val actions = deltaLog.store.readAsIterator(commit1Path, deltaLog.newDeltaHadoopConf()) val actionsWithoutCommitInfo = actions.filterNot(Action.fromJson(_).isInstanceOf[CommitInfo]) deltaLog.store.write( commit1Path, actionsWithoutCommitInfo, overwrite = true, deltaLog.newDeltaHadoopConf()) DeltaLog.clearCache() val latestSnapshot = DeltaLog.forTable(spark, TableIdentifier(tableName)).snapshot val usageRecords = Log4jUsageLogger.track { try { latestSnapshot.timestamp } catch { case _ : DeltaIllegalStateException => () } } val ictReadLog = filterUsageRecords(usageRecords, "delta.inCommitTimestamp.read").head val blob = JsonUtils.fromJson[Map[String, String]](ictReadLog.blob) assert(blob("version") == "1") assert(blob("checkpointVersion") == "-1") assert(blob("exceptionMessage").startsWith("[DELTA_MISSING_COMMIT_INFO]")) assert(blob("exceptionStackTrace").contains(Snapshot.getClass.getName.stripSuffix("$"))) } } } test("DeltaHistoryManager.getActiveCommitAtTimeFromICTRange") { withTempTable(createTable = false) { tableName => spark.range(10).write.format("delta").saveAsTable(tableName) val startTime = System.currentTimeMillis() val clock = new ManualClock(startTime) val deltaLog = getDeltaLogWithClock(tableName, clock) val commitTimeDelta = 10 val numberAdditionalCommits = 25 assert(clock eq deltaLog.clock) for (i <- 1 to numberAdditionalCommits) { clock.setTime(startTime + i*commitTimeDelta) deltaLog.startTransaction().commit(Seq(createTestAddFile(i.toString)), ManualUpdate) } val deltaCommitFileProvider = DeltaCommitFileProvider(deltaLog.update()) val commit0 = DeltaHistoryManager.Commit(0, getInCommitTimestamp(deltaLog, 0)) var commit = DeltaHistoryManager.getActiveCommitAtTimeFromICTRange( startTime + commitTimeDelta*11, startCommit = commit0, numberAdditionalCommits + 1, deltaLog.newDeltaHadoopConf(), deltaLog.logPath, deltaLog.store, numChunks = 3, spark, deltaCommitFileProvider).get assert(commit.version == 11) assert(commit.version == deltaLog.history.getActiveCommitAtTime( startTime + commitTimeDelta*11, true).version) // Search for commit 11 when the timestamp is not an exact match. commit = DeltaHistoryManager.getActiveCommitAtTimeFromICTRange( startTime + commitTimeDelta * 11 + 5, startCommit = commit0, numberAdditionalCommits + 1, deltaLog.newDeltaHadoopConf(), deltaLog.logPath, deltaLog.store, numChunks = 3, spark, deltaCommitFileProvider).get assert(commit.version == 11) // Search for the last commit. commit = DeltaHistoryManager.getActiveCommitAtTimeFromICTRange( startTime + commitTimeDelta*25, startCommit = commit0, numberAdditionalCommits + 1, deltaLog.newDeltaHadoopConf(), deltaLog.logPath, deltaLog.store, numChunks = 3, spark, deltaCommitFileProvider).get assert(commit.version == 25) // Search for the first commit. commit = DeltaHistoryManager.getActiveCommitAtTimeFromICTRange( commit0.timestamp, startCommit = commit0, numberAdditionalCommits + 1, deltaLog.newDeltaHadoopConf(), deltaLog.logPath, deltaLog.store, numChunks = 3, spark, deltaCommitFileProvider).get assert(commit.version == 0) } } test("DeltaHistoryManager.getActiveCommitAtTimeFromICTRange --- " + "search for a timestamp after the last commit") { withTempTable(createTable = false) { tableName => spark.range(10).write.format("delta").saveAsTable(tableName) val startTime = System.currentTimeMillis() val clock = new ManualClock(startTime) val deltaLog = getDeltaLogWithClock(tableName, clock) val commitTimeDelta = 10 val numberAdditionalCommits = 2 assert(clock eq deltaLog.clock) for (i <- 1 to numberAdditionalCommits) { clock.setTime(startTime + i * commitTimeDelta) deltaLog.startTransaction().commit(Seq(createTestAddFile(i.toString)), ManualUpdate) } val commit = deltaLog.history.getActiveCommitAtTime( new Timestamp(startTime + commitTimeDelta * (numberAdditionalCommits + 1)), catalogTableOpt = None, canReturnLastCommit = true) assert(commit.version == numberAdditionalCommits) // Searching beyond the last commit should throw an error // when canReturnLastCommit is false. val e = intercept[DeltaErrors.TemporallyUnstableInputException] { deltaLog.history.getActiveCommitAtTime( new Timestamp(startTime + commitTimeDelta * (numberAdditionalCommits + 1)), catalogTableOpt = None, canReturnLastCommit = false) } assert(e.getMessage.contains("The provided timestamp") && e.getMessage.contains("is after")) } } // Catalog Owned will also automatically enable ICT. testWithDefaultCommitCoordinatorUnset("DeltaHistoryManager.getActiveCommitAtTime: " + "works correctly when the history has both ICT and non-ICT commits") { withSQLConf( DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey -> false.toString) { withTempTable(createTable = false) { tableName => spark.range(10).write.format("delta").saveAsTable(tableName) val numNonICTCommits = 6 val numICTCommits = 5 val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) for (i <- 1 to (numNonICTCommits-1)) { deltaLog.startTransaction().commit(Seq(createTestAddFile(i.toString)), ManualUpdate) } // Enable ICT. spark.sql( s"ALTER TABLE $tableName " + s"SET TBLPROPERTIES ('${DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key}' = 'true')") for (i <- 1 to (numICTCommits-1)) { deltaLog.startTransaction().commit(Seq(createTestAddFile(i.toString)), ManualUpdate) } val currentVersion = deltaLog.update().version for (version <- 0L to currentVersion) { val ts = deltaLog.getSnapshotAt(version).timestamp // Search for the exact timestamp. var commit = deltaLog.history.getActiveCommitAtTime(ts, true) assert(commit.version == version) // Search using a timestamp just before the current timestamp. commit = deltaLog.history.getActiveCommitAtTime( new Timestamp(ts-1), catalogTableOpt = None, canReturnLastCommit = true, canReturnEarliestCommit = true) val expectedVersion = if (version == 0) 0 else version - 1 assert(commit.version == expectedVersion) // Search using a timestamp just after the current timestamp. commit = deltaLog.history.getActiveCommitAtTime(ts + 1, true) assert(commit.version == version) } val enablementCommit = InCommitTimestampUtils.getValidatedICTEnablementInfo(deltaLog.snapshot.metadata).get // Create a checkpoint before deleting commits. deltaLog.createCheckpointAtVersion(enablementCommit.version + 2) // Search for an ICT commit when all the ICT commits leading up to and including it are // absent. val fs = deltaLog.logPath.getFileSystem(deltaLog.newDeltaHadoopConf()) // Search for the commit immediately after the enablement commit. val searchTimestamp = getInCommitTimestamp(deltaLog, enablementCommit.version + 1) val minTimestamp = getInCommitTimestamp(deltaLog, enablementCommit.version + 2) val timestampFormatter = TimestampFormatter( DateTimeUtils.getTimeZone(SQLConf.get.sessionLocalTimeZone)) val minTimestampString = DateTimeUtils.timestampToString( timestampFormatter, DateTimeUtils.fromJavaTimestamp(new Timestamp(minTimestamp))) // Delete the first two ICT commits before performing the search. (enablementCommit.version to enablementCommit.version + 1).foreach { version => fs.delete(FileNames.unsafeDeltaFile(deltaLog.logPath, version), false) } val e = intercept[DeltaErrors.TimestampEarlierThanCommitRetentionException] { deltaLog.history.getActiveCommitAtTime(searchTimestamp, false) } checkError( e, "DELTA_TIMESTAMP_EARLIER_THAN_COMMIT_RETENTION", sqlState = "42816", parameters = Map( "userTimestamp" -> new Timestamp(searchTimestamp).toString, "commitTs" -> new Timestamp(minTimestamp).toString, "timestampString" -> minTimestampString) ) assert( e.getMessage.contains("The provided timestamp") && e.getMessage.contains("is before")) // Search for a non-ICT commit when all the non-ICT commits are missing. // Delete all the non-ICT commits. (0L until numNonICTCommits).foreach { version => fs.delete(FileNames.unsafeDeltaFile(deltaLog.logPath, version), false) } // The earliest available commit is at enablementCommit.version + 2 because we deleted // the first two ICT commits earlier. val earliestAvailableCommitTs = getInCommitTimestamp( deltaLog, enablementCommit.version + 2) val earliestAvailableCommitTimestampString = DateTimeUtils.timestampToString( timestampFormatter, DateTimeUtils.fromJavaTimestamp( new Timestamp(earliestAvailableCommitTs))) val e2 = intercept[DeltaErrors.TimestampEarlierThanCommitRetentionException] { deltaLog.history.getActiveCommitAtTime(enablementCommit.timestamp-1, false) } checkError( e2, "DELTA_TIMESTAMP_EARLIER_THAN_COMMIT_RETENTION", sqlState = "42816", parameters = Map( "userTimestamp" -> new Timestamp(enablementCommit.timestamp-1).toString, "commitTs" -> new Timestamp(earliestAvailableCommitTs).toString, "timestampString" -> earliestAvailableCommitTimestampString) ) // The same query should work when the earliest commit is allowed to be returned. // The returned commit will be the earliest available ICT commit. val commit = deltaLog.history.getActiveCommitAtTime( new Timestamp(enablementCommit.timestamp-1), catalogTableOpt = None, canReturnLastCommit = false, canReturnEarliestCommit = true) // Note that we have already deleted the first two ICT commits. assert(commit.version == enablementCommit.version + 2) val earliestAvailableICTCommitTs = getInCommitTimestamp( deltaLog, enablementCommit.version + 2) assert(commit.timestamp == earliestAvailableICTCommitTs) } } } // Catalog Owned will also automatically enable ICT. testWithDefaultCommitCoordinatorUnset("DeltaHistoryManager.getHistory --- " + "works correctly when the history has both ICT and non-ICT commits") { withSQLConf( DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey -> false.toString) { withTempTable(createTable = false) { tableName => spark.range(1).write.format("delta").saveAsTable(tableName) val numNonICTCommits = 6 val numICTCommits = 5 val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) for (i <- 1 to (numNonICTCommits - 1)) { deltaLog.startTransaction().commit(Seq(createTestAddFile(i.toString)), ManualUpdate) } // Enable ICT. spark.sql( s"ALTER TABLE $tableName " + s"SET TBLPROPERTIES ('${DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key}' = 'true')") for (i <- 1 to (numICTCommits - 1)) { deltaLog.startTransaction().commit(Seq(createTestAddFile(i.toString)), ManualUpdate) } val currentVersion = deltaLog.update().version val ictEnablementVersion = numNonICTCommits // Fetch the entire history. val history = deltaLog.history.getHistory(None) assert(history.length == currentVersion + 1) val fs = deltaLog.logPath.getFileSystem(deltaLog.newDeltaHadoopConf()) history.reverse.zipWithIndex.foreach { case (hist, version) => assert(hist.getVersion == version) val expectedTimestamp = if (version < ictEnablementVersion) { fs.getFileStatus(FileNames.unsafeDeltaFile(deltaLog.logPath, version)) .getModificationTime } else { getInCommitTimestamp(deltaLog, version) } assert(hist.timestamp.getTime == expectedTimestamp) } // Try fetching only the non-ICT commits. val nonICTHistory = deltaLog.history.getHistory(start = 0, end = Some(ictEnablementVersion - 1)) assert(nonICTHistory.length == ictEnablementVersion) nonICTHistory.reverse.zipWithIndex.foreach { case (hist, version) => assert(hist.getVersion == version) val expectedTimestamp = fs.getFileStatus(FileNames.unsafeDeltaFile(deltaLog.logPath, version)) .getModificationTime assert(hist.timestamp.getTime == expectedTimestamp) } // Try fetching only the ICT commits. val ictHistory = deltaLog.history.getHistory(start = ictEnablementVersion, end = None) assert(ictHistory.length == currentVersion - ictEnablementVersion + 1) ictHistory .reverse .zip(ictEnablementVersion to currentVersion.toInt) .foreach { case (hist, version) => assert(hist.getVersion == version) assert(hist.timestamp.getTime == getInCommitTimestamp(deltaLog, version)) } // Try fetching some non-ICT + some ICT commits. val mixedHistory = deltaLog.history.getHistory(start = 2, end = Some(6)) assert(mixedHistory.length == 5) mixedHistory .reverse .zip(2 to 6) .foreach { case (hist, version) => assert(hist.getVersion == version) val expectedTimestamp = if (version < ictEnablementVersion) { fs.getFileStatus(FileNames.unsafeDeltaFile(deltaLog.logPath, version)) .getModificationTime } else { getInCommitTimestamp(deltaLog, version) } assert(hist.timestamp.getTime == expectedTimestamp) } } } } test("DeltaHistoryManager.getActiveCommitAtTimeFromICTRange -- boundary cases" ) { withTempTable(createTable = false) { tableName => spark.range(10).write.format("delta").saveAsTable(tableName) val startTime = System.currentTimeMillis() val clock = new ManualClock(startTime) val deltaLog = getDeltaLogWithClock(tableName, clock) val commit0 = DeltaHistoryManager.Commit(0, deltaLog.snapshot.timestamp) val commitTimeDelta = 10 val numberAdditionalCommits = 10 assert(clock eq deltaLog.clock) for (i <- 1 to numberAdditionalCommits) { clock.setTime(startTime + i * commitTimeDelta) deltaLog.startTransaction().commit(Seq(createTestAddFile(i.toString)), ManualUpdate) } def getICTCommit(version: Long): DeltaHistoryManager.Commit = DeltaHistoryManager.Commit(version, startTime + commitTimeDelta * version) val deltaCommitFileProvider = DeltaCommitFileProvider(deltaLog.update()) // Degenerate case: start + 1 == end. var commit = DeltaHistoryManager.getActiveCommitAtTimeFromICTRange( getICTCommit(2).timestamp, getICTCommit(2), end = 2 + 1, deltaLog.newDeltaHadoopConf(), deltaLog.logPath, deltaLog.store, numChunks = 3, spark, deltaCommitFileProvider).get assert(commit == getICTCommit(2)) // start + 1 == end, search for a timestamp that is after the window. commit = DeltaHistoryManager.getActiveCommitAtTimeFromICTRange( getICTCommit(5).timestamp, getICTCommit(2), end = 2 + 1, deltaLog.newDeltaHadoopConf(), deltaLog.logPath, deltaLog.store, numChunks = 3, spark, deltaCommitFileProvider).get assert(commit == getICTCommit(2)) // start + 1 == end, search for a timestamp that is before the window. val commitOpt = DeltaHistoryManager.getActiveCommitAtTimeFromICTRange( getICTCommit(1).timestamp, getICTCommit(2), end = 2 + 1, deltaLog.newDeltaHadoopConf(), deltaLog.logPath, deltaLog.store, numChunks = 3, spark, deltaCommitFileProvider) assert(commitOpt.isEmpty) // window size is exactly equal to `numChunks`. // Search for an intermediate commit. commit = DeltaHistoryManager.getActiveCommitAtTimeFromICTRange( getICTCommit(7).timestamp, getICTCommit(5), end = 9 + 1, deltaLog.newDeltaHadoopConf(), deltaLog.logPath, deltaLog.store, numChunks = 5, spark, deltaCommitFileProvider).get assert(commit == getICTCommit(7)) // Search for the last commit. commit = DeltaHistoryManager.getActiveCommitAtTimeFromICTRange( getICTCommit(9).timestamp, getICTCommit(5), end = 9 + 1, deltaLog.newDeltaHadoopConf(), deltaLog.logPath, deltaLog.store, numChunks = 5, spark, deltaCommitFileProvider).get assert(commit == getICTCommit(9)) // Delete the last few commits in the window. val fs = deltaLog.logPath.getFileSystem(deltaLog.newDeltaHadoopConf()) fs.delete(FileNames.unsafeDeltaFile(deltaLog.logPath, 5), false) fs.delete(FileNames.unsafeDeltaFile(deltaLog.logPath, 6), false) // Search for the commit just before the deleted commits. commit = DeltaHistoryManager.getActiveCommitAtTimeFromICTRange( getICTCommit(4).timestamp, getICTCommit(2), end = 6 + 1, deltaLog.newDeltaHadoopConf(), deltaLog.logPath, deltaLog.store, numChunks = 3, spark, deltaCommitFileProvider).get assert(commit == getICTCommit(4)) // Search with the first couple of commits in the window deleted. commit = DeltaHistoryManager.getActiveCommitAtTimeFromICTRange( getICTCommit(8).timestamp, // Commits 5 and 6 have been deleted. We start from commit 5, // which does not exist anymore. getICTCommit(5), end = 10 + 1, deltaLog.newDeltaHadoopConf(), deltaLog.logPath, deltaLog.store, numChunks = 3, spark, deltaCommitFileProvider).get assert(commit == getICTCommit(8)) // Make one chunk in the first iteration completely empty. // Window -> [0, 11) // numChunks = 5, chunkSize = (11-0)/5 = 2 // chunks -> [0, 2), [2, 4), [4, 6), [6, 8), [8, 10), [10, 11) fs.delete(FileNames.unsafeDeltaFile(deltaLog.logPath, 4), false) // 4, 5, 6 have been deleted, so window [4, 6) is completely empty. // Search for the commit 6. commit = DeltaHistoryManager.getActiveCommitAtTimeFromICTRange( getICTCommit(6).timestamp, commit0, end = 11, deltaLog.newDeltaHadoopConf(), deltaLog.logPath, deltaLog.store, numChunks = 5, spark, deltaCommitFileProvider).get // [4,6] have been deleted, so we should get the commit at version 3. assert(commit == getICTCommit(3)) // Search for a commit just after the deleted chunk (7). commit = DeltaHistoryManager.getActiveCommitAtTimeFromICTRange( getICTCommit(7).timestamp, commit0, end = 11, deltaLog.newDeltaHadoopConf(), deltaLog.logPath, deltaLog.store, numChunks = 5, spark, deltaCommitFileProvider).get assert(commit == getICTCommit(7)) // Scenario with many empty chunks. fs.delete(FileNames.unsafeDeltaFile(deltaLog.logPath, 8), false) fs.delete(FileNames.unsafeDeltaFile(deltaLog.logPath, 9), false) // Window -> [0, 11) // numChunks = 11, chunkSize = (11-0)/11 = 1 // chunks -> [0, 1), [1, 2), [2, 3), ... [9, 10), [10, 11) // 4, 5, 6, 8, 9 have been deleted. // [4, 6), [5, 6) and [8, 9) are completely empty. // Search for a commit in between empty chunks (7). commit = DeltaHistoryManager.getActiveCommitAtTimeFromICTRange( getICTCommit(7).timestamp, commit0, end = 11, deltaLog.newDeltaHadoopConf(), deltaLog.logPath, deltaLog.store, numChunks = 11, spark, deltaCommitFileProvider).get assert(commit == getICTCommit(7)) // Search for a commit just after the last deleted chunk (10). commit = DeltaHistoryManager.getActiveCommitAtTimeFromICTRange( getICTCommit(10).timestamp, commit0, end = 11, deltaLog.newDeltaHadoopConf(), deltaLog.logPath, deltaLog.store, numChunks = 11, spark, deltaCommitFileProvider).get assert(commit == getICTCommit(10)) fs.delete(FileNames.unsafeDeltaFile(deltaLog.logPath, 10), false) // Everything after and including `end` does not exist. commit = DeltaHistoryManager.getActiveCommitAtTimeFromICTRange( getICTCommit(7).timestamp, commit0, // The last commit in the table is at version 9. But our // search window here is [7, 11). end = 11, deltaLog.newDeltaHadoopConf(), deltaLog.logPath, deltaLog.store, numChunks = 3, spark, deltaCommitFileProvider).get assert(commit == getICTCommit(7)) } } test("DeltaHistoryManager.getHistory --- all ICT commits") { withTempTable(createTable = false) { tableName => spark.range(1).write.format("delta").saveAsTable(tableName) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val numberAdditionalCommits = 4 for (i <- 1 to numberAdditionalCommits) { deltaLog.startTransaction().commit(Seq(createTestAddFile(i.toString)), ManualUpdate) } val history = deltaLog.history.getHistory(None) assert(history.length == numberAdditionalCommits + 1) history.reverse.zipWithIndex.foreach { case (hist, version) => assert(hist.timestamp.getTime == getInCommitTimestamp(deltaLog, version)) } // Try fetching a limited subset of the history. val historySubset = deltaLog.history.getHistory(start = 2, end = Some(2)) assert(historySubset.length == 1) assert(historySubset.head.timestamp.getTime == getInCommitTimestamp(deltaLog, 2)) } } for (ictEnablementVersion <- Seq(1, 4, 7)) testWithDefaultCommitCoordinatorUnset(s"CDC read with all commits being ICT " + s"[ictEnablementVersion = $ictEnablementVersion]") { withSQLConf( DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> "true", DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey -> "false" ) { withTempTable(createTable = false) { tableName => for (i <- 0 to 7) { if (i == ictEnablementVersion) { spark.sql( s"ALTER TABLE $tableName " + s"SET TBLPROPERTIES ('${DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key}' = 'true')") } else { spark.range(i, i + 1).write.format("delta").mode("append").saveAsTable(tableName) } } val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val result = spark.read .format("delta") .option("startingVersion", "1") .option("endingVersion", "7") .option("readChangeFeed", "true") .table(tableName) .select("_commit_timestamp", "_commit_version") .collect() val fileTimestampsMap = getFileModificationTimesMap(deltaLog, 0, 7) result.foreach { row => val v = row.getAs[Long]("_commit_version") val expectedTimestamp = if (v >= ictEnablementVersion) { getInCommitTimestamp(deltaLog, v) } else { fileTimestampsMap(v) } assert(row.getAs[Timestamp]("_commit_timestamp").getTime == expectedTimestamp) } } } } for (ictEnablementVersion <- Seq(1, 4, 7)) testWithDefaultCommitCoordinatorUnset(s"Streaming query + CDC " + s"[ictEnablementVersion = $ictEnablementVersion]") { withSQLConf( DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> "true", DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey -> "false" ) { withTempTable(createTable = false) { sourceTableName => withTempDir { checkpointDir => withTempTable(createTable = false) { sinkTableName => spark.range(0).write.format("delta").mode("append").saveAsTable(sourceTableName) val sourceDeltaLog = DeltaLog.forTable(spark, TableIdentifier(sourceTableName)) val streamingQuery = spark.readStream .format("delta") .option("readChangeFeed", "true") .table(sourceTableName) .select( col("_commit_timestamp").alias("source_commit_timestamp"), col("_commit_version").alias("source_commit_version")) .writeStream .format("delta") .option("checkpointLocation", checkpointDir.getCanonicalPath) .toTable(sinkTableName) for (i <- 1 to 7) { if (i == ictEnablementVersion) { spark.sql(s"ALTER TABLE $sourceTableName " + s"SET TBLPROPERTIES ('${DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key}' = 'true')") } else { spark.range(i, i + 1).write.format("delta").mode("append").saveAsTable(sourceTableName) } } streamingQuery.processAllAvailable() val fileTimestampsMap = getFileModificationTimesMap(sourceDeltaLog, 0, 7) val result = spark.read.format("delta") .table(sinkTableName) .collect() result.foreach { row => val v = row.getAs[Long]("source_commit_version") val expectedTimestamp = if (v >= ictEnablementVersion) { getInCommitTimestamp(sourceDeltaLog, v) } else { fileTimestampsMap(v) } assert( row.getAs[Timestamp]("source_commit_timestamp").getTime == expectedTimestamp) } }}} } } private def testICTEnablementPropertyRetention( expectRetention: Boolean, expectICTEnabled: Option[Boolean] = None)(runCommand: (String) => Unit): Unit = { val ictConfOpt = spark.conf.getOption(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey) try { spark.conf.unset(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey) withTempTable(createTable = false) { tableName => spark.range(1).write.format("delta").saveAsTable(tableName) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) // Enable ICT at version 1 instead of 0 so that we can test the retention of // enablement provenance properties as well. spark.sql( s"ALTER TABLE $tableName " + s"SET TBLPROPERTIES ('${DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key}' = 'true')") val enablementVersion = DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.fromMetaData( deltaLog.snapshot.metadata) val enablementTimestamp = DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.fromMetaData( deltaLog.snapshot.metadata) assert(enablementVersion.contains(1)) assert(enablementTimestamp.isDefined) spark.range(2, 3).write.format("delta").mode("overwrite").saveAsTable(tableName) // Run the REPLACE/CLONE command. runCommand(tableName) val metadataAfterReplace = deltaLog.update().metadata assert( DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData( metadataAfterReplace) == expectICTEnabled.getOrElse(expectRetention)) if (expectRetention) { assert( DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.fromMetaData( metadataAfterReplace) == enablementTimestamp) assert( DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.fromMetaData( metadataAfterReplace) == enablementVersion) } else { Seq( DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP.key, DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION.key ).foreach { key => assert(!metadataAfterReplace.configuration.contains(key)) } } } } finally { ictConfOpt.foreach { ictConf => spark.conf.set(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey, ictConf) } } } testWithDefaultCommitCoordinatorUnset( "ICT enablement properties remain unchanged after a REPLACE with explicit enablement") { testICTEnablementPropertyRetention(expectRetention = true) { tableName => sql(s"REPLACE TABLE $tableName USING delta " + s"TBLPROPERTIES ('${DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key}' = 'true') " + "AS SELECT * FROM range(3, 4)") } } testWithDefaultCommitCoordinatorUnset( "ICT enablement properties are dropped after a REPLACE with explicit enablement " + s"when the ${DeltaSQLConf.IN_COMMIT_TIMESTAMP_RETAIN_ENABLEMENT_INFO_FIX_ENABLED.key} " + s"is disabled") { withSQLConf( DeltaSQLConf.IN_COMMIT_TIMESTAMP_RETAIN_ENABLEMENT_INFO_FIX_ENABLED.key -> "false" ) { testICTEnablementPropertyRetention( expectRetention = false, expectICTEnabled = Some(true)) { tableName => sql( s"REPLACE TABLE $tableName USING delta " + s"TBLPROPERTIES ('${DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key}' = 'true') " + "AS SELECT * FROM range(3, 4)") } } } testWithDefaultCommitCoordinatorUnset( "ICT enablement properties are dropped after a REPLACE with explicit disablement") { testICTEnablementPropertyRetention(expectRetention = false) { tableName => sql(s"REPLACE TABLE $tableName USING delta " + s"TBLPROPERTIES ('${DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key}' = 'false') " + "AS SELECT * FROM range(3, 4)") } } testWithDefaultCommitCoordinatorUnset( "ICT is completely dropped after a REPLACE with no explicit disablement") { testICTEnablementPropertyRetention(expectRetention = false) { tableName => sql(s"REPLACE TABLE $tableName USING delta AS SELECT * FROM range(3, 4)") } } } class InCommitTimestampWithCatalogOwnedBatch1Suite extends InCommitTimestampSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class InCommitTimestampWithCatalogOwnedBatch2Suite extends InCommitTimestampSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class InCommitTimestampWithCatalogOwnedBatch5Suite extends InCommitTimestampSuite { override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey, "true") } override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(5) test("getActiveCommitAtTime works correctly within catalog owned range") { CatalogOwnedCommitCoordinatorProvider.clearBuilders() CatalogOwnedCommitCoordinatorProvider.registerBuilder( catalogName = CatalogOwnedTableUtils.DEFAULT_CATALOG_NAME_FOR_TESTING, commitCoordinatorBuilder = TrackingInMemoryCommitCoordinatorBuilder(batchSize = 10) ) withTempTable(createTable = false) { tableName => spark.range(10).write.format("delta").saveAsTable(tableName) val (deltaLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName)) val commit0 = DeltaHistoryManager.Commit(0, snapshot.timestamp) val tableCommitCoordinatorClient = CatalogOwnedTableUtils.populateTableCommitCoordinatorFromCatalog( spark, catalogTableOpt = deltaLog.initialCatalogTable, snapshot ).get val numberAdditionalCommits = 4 // Create 4 unbackfilled commits. for (i <- 1 to numberAdditionalCommits) { deltaLog.startTransaction().commit(Seq(createTestAddFile(i.toString)), ManualUpdate) } val commitFileProvider = DeltaCommitFileProvider(deltaLog.update()) val unbackfilledCommits = tableCommitCoordinatorClient .getCommits(Some(1L)) .getCommits.asScala .map { commit => DeltaHistoryManager.Commit(commit.getVersion, commit.getCommitTimestamp)} val commits = (Seq(commit0) ++ unbackfilledCommits).toList // Search for the exact timestamp. for (commit <- commits) { val resCommit = deltaLog.history.getActiveCommitAtTime( new Timestamp(commit.timestamp), catalogTableOpt = None, canReturnLastCommit = false) assert(resCommit.version == commit.version) assert(resCommit.timestamp == commit.timestamp) } // getActiveCommitAtTimeFromICTRange should throw an IllegalStateException // if it does not manage to find an unbackfilled commit. // Delete the target unbackfilled commit: val fs = deltaLog.logPath.getFileSystem(deltaLog.newDeltaHadoopConf()) val commit3Path = commitFileProvider.deltaFile(3) fs.delete(commit3Path, false) val commit3Timestamp = unbackfilledCommits(2).timestamp var errorOpt = Option.empty[org.apache.spark.SparkException] try { DeltaHistoryManager.getActiveCommitAtTimeFromICTRange( commit3Timestamp, commit0, numberAdditionalCommits + 1, deltaLog.newDeltaHadoopConf(), deltaLog.logPath, deltaLog.store, numChunks = 5, spark, commitFileProvider) } catch { case e: org.apache.spark.SparkException => errorOpt = Some(e) e.getStackTrace.exists(_.toString.contains( s"Could not find commit 3 which was expected to be at " + s"path ${commit3Path.toString}.")) } assert(errorOpt.isDefined) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/InCommitTimestampTestUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.actions.{Action, CommitInfo} import org.apache.spark.sql.delta.util.{DeltaCommitFileProvider, FileNames, JsonUtils} import org.apache.hadoop.fs.Path object InCommitTimestampTestUtils { /** * Overwrites the in-commit-timestamp in the delta file with the given timestamp. * It will also set operationParameters to an empty map because operationParameters * serialization/deserialization is broken. */ def overwriteICTInDeltaFile(deltaLog: DeltaLog, filePath: Path, ts: Option[Long]): Unit = { val updatedActionsList = deltaLog.store .readAsIterator(filePath, deltaLog.newDeltaHadoopConf()) .map(Action.fromJson) .map { case ci: CommitInfo => // operationParameters serialization/deserialization is broken as it uses a custom // serializer but a default deserializer. ci.copy(inCommitTimestamp = ts, operationParameters = Map.empty).json case other => other.json }.toList deltaLog.store.write( filePath, updatedActionsList.toIterator, overwrite = true, deltaLog.newDeltaHadoopConf()) } /** * Overwrites the in-commit-timestamp in the given CRC file with the given timestamp. */ def overwriteICTInCrc(deltaLog: DeltaLog, version: Long, ts: Option[Long]): Unit = { val crcPath = FileNames.checksumFile(deltaLog.logPath, version) val latestCrc = JsonUtils.fromJson[VersionChecksum]( deltaLog.store.read(crcPath, deltaLog.newDeltaHadoopConf()).mkString("")) val checksumWithNoICT = latestCrc.copy(inCommitTimestampOpt = ts) deltaLog.store.write( crcPath, Iterator(JsonUtils.toJson(checksumWithNoICT)), overwrite = true, deltaLog.newDeltaHadoopConf()) } /** * Retrieves the in-commit timestamp for a specific version of the Delta Log. */ def getInCommitTimestamp(deltaLog: DeltaLog, version: Long): Long = { val deltaFile = DeltaCommitFileProvider(deltaLog.unsafeVolatileSnapshot).deltaFile(version) val commitInfo = DeltaHistoryManager.getCommitInfoOpt( deltaLog.store, deltaFile, deltaLog.newDeltaHadoopConf()) assert(commitInfo.isDefined, s"CommitInfo should exist for version $version") assert(commitInfo.get.inCommitTimestamp.isDefined, s"InCommitTimestamp should exist for CommitInfo's version $version") commitInfo.get.inCommitTimestamp.get } /** * Retrieves a map of file modification times for Delta Log versions within a specified version * range. */ def getFileModificationTimesMap( deltaLog: DeltaLog, start: Long, end: Long): Map[Long, Long] = { deltaLog.store.listFrom( FileNames.listingPrefix(deltaLog.logPath, start), deltaLog.newDeltaHadoopConf()) .collect { case FileNames.DeltaFile(fs, v) => v -> fs.getModificationTime } .takeWhile(_._1 <= end) .toMap } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/LastCheckpointInfoSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.JsonUtils import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.exc.MismatchedInputException import com.google.common.io.{ByteStreams, Closeables} import org.apache.commons.codec.digest.DigestUtils import org.apache.commons.io.IOUtils import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{IntegerType, StructType} class LastCheckpointInfoSuite extends SharedSparkSession with DeltaSQLCommandTest { // same checkpoint schema for tests private val checkpointSchema = Some(new StructType().add("c1", IntegerType, nullable = false)) private def jsonStringToChecksum(jsonStr: String): String = { val rootNode = JsonUtils.mapper.readValue(jsonStr, classOf[JsonNode]) LastCheckpointInfo.treeNodeToChecksum(rootNode) } test("test json to checksum conversion with maps") { // test with different ordering and spaces, with different value data types val s1 = """{"k1":"v1","k4":"v4","k3":23.45,"k2":123}""" val s2 = """{"k1":"v1","k3":23.45,"k2":123, "k4":"v4"}""" assert(jsonStringToChecksum(s1) === jsonStringToChecksum(s2)) // test json with nested maps val s3 = """{"k1":"v1","k4":{"k41":"v41","k40":{"k401":401,"k402":"402"}},"k3":23.45,"k2":123}""" val s4 = """{"k1":"v1","k4":{"k40":{"k401":401,"k402":"402"}, "k41":"v41"},"k3":23.45,"k2":123}""" assert(jsonStringToChecksum(s3) === jsonStringToChecksum(s4)) // test empty json val s5 = """{ }""" val s6 = """{}""" assert(jsonStringToChecksum(s5) === jsonStringToChecksum(s6)) // negative test: value for a specific key k4 is not same. val s7 = """{"k1":"v1","k4":"v4","k3":23.45,"k2":123}""" val s8 = """{"k1":"v1","k4":"v1","k3":23.45,"k2":123}""" assert(jsonStringToChecksum(s7) != jsonStringToChecksum(s8)) } test("test json to checksum conversion with array") { // has top level array and array values are json objects val s1 = """[{"id":"j1","stuff":"things"},{"stuff":"t2","id":"j2"}]""" val s2 = """[{"id" : "j1", "stuff" : "things"}, {"id" : "j2", "stuff" : "t2"}]""" assert(jsonStringToChecksum(s1) === jsonStringToChecksum(s2)) // array as part of value for a json key and array value has single json object val s3 = """{"id":"j1","stuff":[{"hello": "world", "hello1": "world1"}]}""" val s4 = """{"id": "j1","stuff":[{"hello1": "world1", "hello": "world"}]}""" assert(jsonStringToChecksum(s3) === jsonStringToChecksum(s4)) // array as part of value for a json key and array values are multiple json objects val s5 = """{"id":"j1","stuff":[{"hello": "world"}, {"hello1": "world1"}]}""" val s6 = """{"id": "j1","stuff":[{"hello":"world"},{"hello1":"world1"}]}""" assert(jsonStringToChecksum(s5) === jsonStringToChecksum(s6)) // Negative case: array as part of value for a json key and array values are multiple json // objects with different order. val s7 = """{"id":"j1","stuff":[{"hello1": "world1"}, {"hello": "world"}]}""" val s8 = """{"id": "j1","stuff":[{"hello":"world"},{"hello1":"world1"}]}""" assert(jsonStringToChecksum(s7) != jsonStringToChecksum(s8)) // array has scalar string values val s9 = """{"id":"j1","stuff":["a", "b"]}""" val s10 = """{"stuff":["a","b"], "id": "j1"}""" assert(jsonStringToChecksum(s9) === jsonStringToChecksum(s10)) // array has scalar int values val s11 = """{"id":"j1","stuff":[1, 2]}""" val s12 = """{"stuff":[1,2], "id": "j1"}""" assert(jsonStringToChecksum(s11) === jsonStringToChecksum(s12)) // Negative case: array has scalar values in different order val s13 = """{"id":"j1","stuff":["a", "b", "c"]}""" val s14 = """{"id":"j1","stuff":["c", "a", "b"]}""" assert(jsonStringToChecksum(s13) != jsonStringToChecksum(s14)) } // scalastyle:off line.size.limit test("test json normalization") { // test with different data types val s1 = """{"k1":"v1","k4":"v4","k3":23.45,"k2":123,"k6":null,"k5":true}""" val normalizedS1 = """"k1"="v1","k2"=123,"k3"=23.45,"k4"="v4","k5"=true,"k6"=null""" assert(jsonStringToChecksum(s1) === DigestUtils.md5Hex(normalizedS1)) // test json with nested maps val s2 = """{"k1":"v1","k4":{"k41":"v41","k40":{"k401":401,"k402":"402"}},"k3":23.45,"k2":123}""" val normalizedS2 = """"k1"="v1","k2"=123,"k3"=23.45,"k4"+"k40"+"k401"=401,"k4"+"k40"+"k402"="402","k4"+"k41"="v41"""" assert(jsonStringToChecksum(s2) === DigestUtils.md5Hex(normalizedS2)) // test with arrays val s3 = """{"stuff":[{"hx": "wx","h1":"w1"}, {"h2": "w2"}],"id":1}""" val normalizedS3 = """"id"=1,"stuff"+0+"h1"="w1","stuff"+0+"hx"="wx","stuff"+1+"h2"="w2"""" assert(jsonStringToChecksum(s3) === DigestUtils.md5Hex(normalizedS3)) // test top level `checksum` key is ignored in canonicalization val s4 = """{"k1":"v1","checksum":"daswefdssfd","k3":23.45,"k2":123}""" val normalizedS4 = """"k1"="v1","k2"=123,"k3"=23.45""" assert(jsonStringToChecksum(s4) === DigestUtils.md5Hex(normalizedS4)) // test empty json val s5 = """{ }""" val normalizedS5 = """""" assert(jsonStringToChecksum(s5) === DigestUtils.md5Hex(normalizedS5)) // test with complex strings val s6 = """{"k0":"normal","k1":"'v1'","k4":"'v4","k3":":hello","k2":"\"double quote str\""}""" val normalizedS6 = """"k0"="normal","k1"="%27v1%27","k2"="%22double%20quote%20str%22","k3"="%3Ahello","k4"="%27v4"""" assert(jsonStringToChecksum(s6) === DigestUtils.md5Hex(normalizedS6)) // test covering different ASCII characters val s7 = """{"k0":"normal","k1":"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789%'`~!@#$%^&*()_+-={[}]|\\;:'\"\/?.>,<"}""" val normalizedS7 = """"k0"="normal","k1"="abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789%25%27%60~%21%40%23%24%25%5E%26%2A%28%29_%2B-%3D%7B%5B%7D%5D%7C%5C%3B%3A%27%22%2F%3F.%3E%2C%3C"""" assert(jsonStringToChecksum(s7) === DigestUtils.md5Hex(normalizedS7)) // test with nested maps and arrays // This example is also part of Delta's PROTOCOL.md. We should keep these two in sync. val s8 = """{"k0":"'v 0'", "checksum": "adsaskfljadfkjadfkj", "k1":{"k2": 2, "k3": ["v3", [1, 2], {"k4": "v4", "k5": ["v5", "v6", "v7"]}]}}""" val normalizedS8 = """"k0"="%27v%200%27","k1"+"k2"=2,"k1"+"k3"+0="v3","k1"+"k3"+1+0=1,"k1"+"k3"+1+1=2,"k1"+"k3"+2+"k4"="v4","k1"+"k3"+2+"k5"+0="v5","k1"+"k3"+2+"k5"+1="v6","k1"+"k3"+2+"k5"+2="v7"""" assert(jsonStringToChecksum(s8) === DigestUtils.md5Hex(normalizedS8)) assert(jsonStringToChecksum(s8) === "6a92d155a59bf2eecbd4b4ec7fd1f875") // test non-ASCII character // scalastyle:off nonascii val s9 = s"""{"k0":"normal","k1":"a€+"}""" val normalizedS9 = """"k0"="normal","k1"="a%E2%82%AC%2B"""" assert(jsonStringToChecksum(s9) === DigestUtils.md5Hex(normalizedS9)) // scalastyle:on nonascii } // scalastyle:on line.size.limit test("test LastCheckpointInfo checksum") { val ci1 = LastCheckpointInfo(version = 1, size = 2, parts = Some(3), sizeInBytes = Some(20L), numOfAddFiles = Some(2L), checkpointSchema = checkpointSchema) val (stored1, actual1) = LastCheckpointInfo.getChecksums(LastCheckpointInfo.serializeToJson(ci1, addChecksum = true)) assert(stored1 === Some(actual1)) // checksum mismatch when version changes. val ci2 = LastCheckpointInfo(version = 2, size = 2, parts = Some(3), sizeInBytes = Some(20L), numOfAddFiles = Some(2L), checkpointSchema = checkpointSchema) val (stored2, actual2) = LastCheckpointInfo.getChecksums(LastCheckpointInfo.serializeToJson(ci2, addChecksum = true)) assert(stored2 === Some(actual2)) assert(stored2 != stored1) // `checksum` doesn't participate in `actualChecksum` calculation. val ci3 = LastCheckpointInfo(version = 1, size = 2, parts = Some(3), checksum = Some("XYZ"), sizeInBytes = Some(20L), numOfAddFiles = Some(2L), checkpointSchema = checkpointSchema) val (stored3, actual3) = LastCheckpointInfo.getChecksums(LastCheckpointInfo.serializeToJson(ci3, addChecksum = true)) assert(stored3 === Some(actual3)) assert(stored3 === stored1) // checksum doesn't depend on spaces and order of field val json1 = """{"version":1,"size":2,"parts":3}""" val json2 = """{"version":1 ,"parts":3,"size":2}""" assert(jsonStringToChecksum(json1) === jsonStringToChecksum(json2)) // `checksum` is ignored while calculating json val json3 = """{"version":1 ,"parts":3,"size":2,"checksum":"xyz"}""" assert(jsonStringToChecksum(json1) === jsonStringToChecksum(json3)) // Change in any value changes the checksum val json4 = """{"version":4,"size":2,"parts":3}""" assert(jsonStringToChecksum(json1) != jsonStringToChecksum(json4)) } test("test backward compatibility - json without checksum is deserialized properly") { val jsonStr = """{"version":1,"size":2,"parts":3,"sizeInBytes":20,"numOfAddFiles":2,""" + """"checkpointSchema":{"type":"struct","fields":[{"name":"c1","type":"integer"""" + ""","nullable":false,"metadata":{}}]}}""" val expectedLastCheckpointInfo = LastCheckpointInfo( version = 1, size = 2, parts = Some(3), sizeInBytes = Some(20), numOfAddFiles = Some(2), checkpointSchema = Some(new StructType().add("c1", IntegerType, nullable = false))) assert(LastCheckpointInfo.deserializeFromJson(jsonStr, validate = true) === expectedLastCheckpointInfo) } test("LastCheckpointInfo - serialize/deserialize") { val ci1 = LastCheckpointInfo(version = 1, size = 2, parts = Some(3), checksum = Some("XYZ"), sizeInBytes = Some(20L), numOfAddFiles = Some(2L), checkpointSchema = checkpointSchema) val ci2 = LastCheckpointInfo(version = 1, size = 2, parts = Some(3), checksum = None, sizeInBytes = Some(20L), numOfAddFiles = Some(2L), checkpointSchema = checkpointSchema) val actualChecksum = LastCheckpointInfo.getChecksums( LastCheckpointInfo.serializeToJson(ci1, addChecksum = true))._2 val ciWithCorrectChecksum = ci1.copy(checksum = Some(actualChecksum)) for(ci <- Seq(ci1, ci2)) { val json = LastCheckpointInfo.serializeToJson(ci, addChecksum = true) assert(LastCheckpointInfo.deserializeFromJson(json, validate = true) === ciWithCorrectChecksum) // The below assertion also validates that fields version/size/parts are in the beginning of // the json. assert(LastCheckpointInfo.serializeToJson(ci, addChecksum = true) === """{"version":1,"size":2,"parts":3,"sizeInBytes":20,"numOfAddFiles":2,""" + s""""checkpointSchema":${JsonUtils.toJson(checkpointSchema)},""" + """"checksum":"524d4e2226f3c3f923df4ee42dae347e"}""") } assert(LastCheckpointInfo.serializeToJson(ci1, addChecksum = true) === LastCheckpointInfo.serializeToJson(ci2, addChecksum = true)) } test("LastCheckpointInfo - json with duplicate keys should fail") { val jsonString = """{"version":1,"size":3,"parts":3,"checksum":"d84a0aa11c93304d57feca6acaceb7fb","size":2}""" intercept[MismatchedInputException] { LastCheckpointInfo.deserializeFromJson(jsonString, validate = true) } // Deserialization shouldn't fail when validate is false and the last `size` overrides the // previous size. assert(LastCheckpointInfo.deserializeFromJson(jsonString, validate = false).size === 2) } test("LastCheckpointInfo - test checksum is written only when config is enabled") { withTempDir { dir => spark.range(10).write.format("delta").save(dir.getAbsolutePath) val log = DeltaLog.forTable(spark, dir) def readLastCheckpointFile(): String = { val fs = log.LAST_CHECKPOINT.getFileSystem(log.newDeltaHadoopConf()) val is = fs.open(log.LAST_CHECKPOINT) try { IOUtils.toString(is, "UTF-8") } finally { is.close() } } withSQLConf(DeltaSQLConf.LAST_CHECKPOINT_CHECKSUM_ENABLED.key -> "true") { DeltaLog.forTable(spark, dir).checkpoint() assert(readLastCheckpointFile().contains("checksum")) } spark.range(10).write.mode("append").format("delta").save(dir.getAbsolutePath) withSQLConf(DeltaSQLConf.LAST_CHECKPOINT_CHECKSUM_ENABLED.key -> "false") { DeltaLog.forTable(spark, dir).checkpoint() assert(!readLastCheckpointFile().contains("checksum")) } } } test("Suppress optional fields in _last_checkpoint") { val expectedStr = """{"version":1,"size":2,"parts":3}""" val info = LastCheckpointInfo( version = 1, size = 2, parts = Some(3), sizeInBytes = Some(20), numOfAddFiles = Some(2), checkpointSchema = Some(new StructType().add("c1", IntegerType, nullable = false))) val serializedJson = LastCheckpointInfo.serializeToJson( info, addChecksum = true, suppressOptionalFields = true) assert(serializedJson === expectedStr) val expectedStrNoPart = """{"version":1,"size":2}""" val serializedJsonNoPart = LastCheckpointInfo.serializeToJson( info.copy(parts = None), addChecksum = true, suppressOptionalFields = true) assert(serializedJsonNoPart === expectedStrNoPart) } test("read and write _last_checkpoint with optional fields suppressed") { withTempDir { dir => withSQLConf(DeltaSQLConf.SUPPRESS_OPTIONAL_LAST_CHECKPOINT_FIELDS.key -> "true") { // Create a Delta table with a checkpoint. spark.range(10).write.format("delta").save(dir.getAbsolutePath) DeltaLog.forTable(spark, dir).checkpoint() DeltaLog.clearCache() val log = DeltaLog.forTable(spark, dir) val metadata = log.readLastCheckpointFile().get val trimmed = metadata.productIterator.drop(3).forall { case o: Option[_] => o.isEmpty } assert(trimmed, s"Unexpected fields in _last_checkpoint: $metadata") } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/LogStoreProviderSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.storage.{DelegatingLogStore, LogStore, LogStoreAdaptor} import org.apache.spark.{SparkConf, SparkContext, SparkFunSuite} import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.spark.sql.LocalSparkSession._ class LogStoreProviderSuite extends SparkFunSuite { private val customLogStoreClassName = classOf[CustomPublicLogStore].getName private def fakeSchemeWithNoDefault = "fake" private def withoutSparkPrefix(key: String) = key.stripPrefix("spark.") private def constructSparkConf(confs: Seq[(String, String)]): SparkConf = { val sparkConf = new SparkConf(loadDefaults = false).setMaster("local") confs.foreach { case (key, value) => sparkConf.set(key, value) } sparkConf } /** * Test with class conf set and scheme conf unset using `scheme`. Test using class conf key both * with and without 'spark.' prefix. */ private def testLogStoreClassConfNoSchemeConf(scheme: String) { for (classKeys <- Seq( // set only prefixed key Seq(LogStore.logStoreClassConfKey), // set only non-prefixed key Seq(withoutSparkPrefix(LogStore.logStoreClassConfKey)), // set both spark-prefixed key and non-spark prefixed key Seq(LogStore.logStoreClassConfKey, withoutSparkPrefix(LogStore.logStoreClassConfKey)) )) { val sparkConf = constructSparkConf(classKeys.map((_, customLogStoreClassName))) withSparkSession(SparkSession.builder.config(sparkConf).getOrCreate()) { spark => assert(LogStore(spark).isInstanceOf[LogStoreAdaptor]) assert(LogStore(spark).asInstanceOf[LogStoreAdaptor] .logStoreImpl.getClass.getName == customLogStoreClassName) } } } /** * Test with class conf set and scheme conf set using `scheme`. This tests * checkLogStoreConfConflicts. Test conf keys both with and without 'spark.' prefix. */ private def testLogStoreClassConfAndSchemeConf(scheme: String, classConf: String, schemeConf: String) { val schemeKey = LogStore.logStoreSchemeConfKey(scheme) // we test with both the spark-prefixed and non-prefixed keys val schemeConfKeys = Seq(schemeKey, withoutSparkPrefix(schemeKey)) val classConfKeys = Seq(LogStore.logStoreClassConfKey, withoutSparkPrefix(LogStore.logStoreClassConfKey)) schemeConfKeys.foreach { schemeKey => classConfKeys.foreach { classKey => val sparkConf = constructSparkConf(Seq((schemeKey, schemeConf), (classKey, classConf))) val e = intercept[AnalysisException]( withSparkSession(SparkSession.builder.config(sparkConf).getOrCreate()) { spark => LogStore(spark) } ) assert(e.getMessage.contains( s"(`$classKey`) and (`$schemeKey`) cannot be set at the same time")) } } } test("class-conf = set, scheme has no default, scheme-conf = not set") { testLogStoreClassConfNoSchemeConf(fakeSchemeWithNoDefault) } test("class-conf = set, scheme has no default, scheme-conf = set") { testLogStoreClassConfAndSchemeConf(fakeSchemeWithNoDefault, customLogStoreClassName, DelegatingLogStore.defaultAzureLogStoreClassName) } test("class-conf = set, scheme has default, scheme-conf = not set") { testLogStoreClassConfNoSchemeConf("s3a") } test("class-conf = set, scheme has default, scheme-conf = set") { testLogStoreClassConfAndSchemeConf("s3a", customLogStoreClassName, DelegatingLogStore.defaultAzureLogStoreClassName) } test("verifyLogStoreConfs - scheme conf keys ") { Seq( fakeSchemeWithNoDefault, // scheme with no default "s3a" // scheme with default ).foreach { scheme => val schemeConfKey = LogStore.logStoreSchemeConfKey(scheme) for (confs <- Seq( // set only non-prefixed key Seq((withoutSparkPrefix(schemeConfKey), customLogStoreClassName)), // set only prefixed key Seq((schemeConfKey, customLogStoreClassName)), // set both spark-prefixed key and non-spark prefixed key to same value Seq((withoutSparkPrefix(schemeConfKey), customLogStoreClassName), (schemeConfKey, customLogStoreClassName)) )) { val sparkConf = constructSparkConf(confs) withSparkSession(SparkSession.builder.config(sparkConf).getOrCreate()) { spark => // no error is thrown LogStore(spark) } } // set both spark-prefixed key and non-spark-prefixed key to inconsistent values val sparkConf = constructSparkConf( Seq((withoutSparkPrefix(schemeConfKey), customLogStoreClassName), (schemeConfKey, DelegatingLogStore.defaultAzureLogStoreClassName))) val e = intercept[IllegalArgumentException]( withSparkSession(SparkSession.builder.config(sparkConf).getOrCreate()) { spark => LogStore(spark) } ) assert(e.getMessage.contains( s"(${withoutSparkPrefix(schemeConfKey)} = $customLogStoreClassName, " + s"$schemeConfKey = ${DelegatingLogStore.defaultAzureLogStoreClassName}) cannot be set " + s"to different values. Please only set one of them, or set them to the same value." )) } } test("verifyLogStoreConfs - class conf keys") { val classConfKey = LogStore.logStoreClassConfKey for (confs <- Seq( // set only non-prefixed key Seq((withoutSparkPrefix(classConfKey), customLogStoreClassName)), // set only prefixed key Seq((classConfKey, customLogStoreClassName)), // set both spark-prefixed key and non-spark prefixed key to same value Seq((withoutSparkPrefix(classConfKey), customLogStoreClassName), (classConfKey, customLogStoreClassName)) )) { val sparkConf = constructSparkConf(confs) withSparkSession(SparkSession.builder.config(sparkConf).getOrCreate()) { spark => // no error is thrown LogStore(spark) } } // set both spark-prefixed key and non-spark-prefixed key to inconsistent values val sparkConf = constructSparkConf( Seq((withoutSparkPrefix(classConfKey), customLogStoreClassName), (classConfKey, DelegatingLogStore.defaultAzureLogStoreClassName))) val e = intercept[IllegalArgumentException]( withSparkSession(SparkSession.builder.config(sparkConf).getOrCreate()) { spark => LogStore(spark) } ) assert(e.getMessage.contains( s"(${withoutSparkPrefix(classConfKey)} = $customLogStoreClassName, " + s"$classConfKey = ${DelegatingLogStore.defaultAzureLogStoreClassName})" + s" cannot be set to different values. Please only set one of them, or set them to the " + s"same value." )) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/LogStoreSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.{File, IOException} import java.net.URI import java.util.concurrent.atomic.AtomicInteger import scala.collection.mutable.ArrayBuffer // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.storage._ import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, FileSystem, FSDataOutputStream, Path, RawLocalFileSystem} import org.apache.spark.{SparkConf, SparkFunSuite} import org.apache.spark.sql.{LocalSparkSession, QueryTest, SparkSession} import org.apache.spark.sql.LocalSparkSession.withSparkSession import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.Utils /////////////////////////// // Child-specific traits // /////////////////////////// trait AzureLogStoreSuiteBase extends LogStoreSuiteBase { testHadoopConf( expectedErrMsg = ".*No FileSystem for scheme.*fake.*", "fs.fake.impl" -> classOf[FakeFileSystem].getName, "fs.fake.impl.disable.cache" -> "true") protected def shouldUseRenameToWriteCheckpoint: Boolean = true } trait HDFSLogStoreSuiteBase extends LogStoreSuiteBase { // HDFSLogStore is based on FileContext APIs and hence requires AbstractFileSystem-based // implementations. testHadoopConf( expectedErrMsg = ".*No FileSystem for scheme.*fake.*", "fs.fake.impl" -> classOf[FakeFileSystem].getName, "fs.fake.impl.disable.cache" -> "true") import testImplicits._ test("writes on systems without AbstractFileSystem implemented") { withSQLConf("fs.fake.impl" -> classOf[FakeFileSystem].getName, "fs.fake.impl.disable.cache" -> "true") { val tempDir = Utils.createTempDir() // scalastyle:off pathfromuri val path = new Path(new URI(s"fake://${tempDir.toURI.getRawPath}/1.json")) // scalastyle:on pathfromuri val e = intercept[IOException] { createLogStore(spark) .write(path, Iterator("zero", "none"), overwrite = false, sessionHadoopConf) } assert(e.getMessage .contains("The error typically occurs when the default LogStore implementation")) } } test("reads should work on systems without AbstractFileSystem implemented") { withTempDir { tempDir => val writtenFile = new File(tempDir, "1") val store = createLogStore(spark) store.write( new Path(writtenFile.getCanonicalPath), Iterator("zero", "none"), overwrite = false, sessionHadoopConf) withSQLConf("fs.fake.impl" -> classOf[FakeFileSystem].getName, "fs.fake.impl.disable.cache" -> "true") { val read = createLogStore(spark) .read(new Path("fake://" + writtenFile.getCanonicalPath), sessionHadoopConf) assert(read === ArrayBuffer("zero", "none")) } } } test( "No AbstractFileSystem - end to end test using data frame") { // Writes to the fake file system will fail withTempDir { tempDir => val fakeFSLocation = s"fake://${tempDir.getCanonicalFile}" withSQLConf("fs.fake.impl" -> classOf[FakeFileSystem].getName, "fs.fake.impl.disable.cache" -> "true") { val e = intercept[IOException] { Seq(1, 2, 4).toDF().write.format("delta").save(fakeFSLocation) } assert(e.getMessage .contains("The error typically occurs when the default LogStore implementation")) } } // Reading files written by other systems will work. withTempDir { tempDir => Seq(1, 2, 4).toDF().write.format("delta").save(tempDir.getAbsolutePath) withSQLConf("fs.fake.impl" -> classOf[FakeFileSystem].getName, "fs.fake.impl.disable.cache" -> "true") { val fakeFSLocation = s"fake://${tempDir.getCanonicalFile}" checkAnswer(spark.read.format("delta").load(fakeFSLocation), Seq(1, 2, 4).toDF()) } } } test("if fc.rename() fails, it should throw java.nio.file.FileAlreadyExistsException") { withTempDir { tempDir => withSQLConf( "fs.AbstractFileSystem.fake.impl" -> classOf[FailingRenameAbstractFileSystem].getName, "fs.fake.impl" -> classOf[FakeFileSystem].getName, "fs.fake.impl.disable.cache" -> "true") { val store = createLogStore(spark) val commit0 = new Path(s"fake://${tempDir.getCanonicalPath}/00000.json") intercept[java.nio.file.FileAlreadyExistsException] { store.write(commit0, Iterator("zero"), overwrite = false, sessionHadoopConf) } } } } test("Read after write consistency with msync") { withTempDir { tempDir => val tsFSLocation = s"ts://${tempDir.getCanonicalFile}" // Use the file scheme so that it uses a different FileSystem cached object withSQLConf( ("fs.ts.impl", classOf[TimestampLocalFileSystem].getCanonicalName), ("fs.AbstractFileSystem.ts.impl", classOf[TimestampAbstractFileSystem].getCanonicalName)) { val store = createLogStore(spark) val path = new Path(tsFSLocation, "1.json") // Initialize the TimestampLocalFileSystem object which will be reused later due to the // FileSystem cache assert(store.listFrom(path, sessionHadoopConf).length == 0) store.write(path, Iterator("zero", "none"), overwrite = false, sessionHadoopConf) // Verify `msync` is called by checking whether `listFrom` returns the latest result. // Without the `msync` call, the TimestampLocalFileSystem would not see this file. assert(store.listFrom(path, sessionHadoopConf).length == 1) } } } protected def shouldUseRenameToWriteCheckpoint: Boolean = true } trait LocalLogStoreSuiteBase extends LogStoreSuiteBase { testHadoopConf( expectedErrMsg = ".*No FileSystem for scheme.*fake.*", "fs.fake.impl" -> classOf[FakeFileSystem].getName, "fs.fake.impl.disable.cache" -> "true") protected def shouldUseRenameToWriteCheckpoint: Boolean = true } trait GCSLogStoreSuiteBase extends LogStoreSuiteBase { testHadoopConf( expectedErrMsg = ".*No FileSystem for scheme.*fake.*", "fs.fake.impl" -> classOf[FakeFileSystem].getName, "fs.fake.impl.disable.cache" -> "true") protected def shouldUseRenameToWriteCheckpoint: Boolean = false test("gcs write should happen in a new thread") { withTempDir { tempDir => // Use `FakeGCSFileSystemValidatingCommits` to verify we write in the correct thread. withSQLConf( "fs.gs.impl" -> classOf[FakeGCSFileSystemValidatingCommits].getName, "fs.gs.impl.disable.cache" -> "true") { val store = createLogStore(spark) store.write( new Path(s"gs://${tempDir.getCanonicalPath}", "1.json"), Iterator("foo"), overwrite = false, sessionHadoopConf) } } } test("handles precondition failure") { withTempDir { tempDir => withSQLConf( "fs.gs.impl" -> classOf[FailingGCSFileSystem].getName, "fs.gs.impl.disable.cache" -> "true") { val store = createLogStore(spark) assertThrows[java.nio.file.FileAlreadyExistsException] { store.write( new Path(s"gs://${tempDir.getCanonicalPath}", "1.json"), Iterator("foo"), overwrite = false, sessionHadoopConf) } store.write( new Path(s"gs://${tempDir.getCanonicalPath}", "1.json"), Iterator("foo"), overwrite = true, sessionHadoopConf) } } } } //////////////////////////////// // Concrete child test suites // //////////////////////////////// class HDFSLogStoreSuite extends HDFSLogStoreSuiteBase { override val logStoreClassName: String = classOf[HDFSLogStore].getName } class AzureLogStoreSuite extends AzureLogStoreSuiteBase { override val logStoreClassName: String = classOf[AzureLogStore].getName } class LocalLogStoreSuite extends LocalLogStoreSuiteBase { override val logStoreClassName: String = classOf[LocalLogStore].getName } //////////////////////////////// // File System Helper Classes // //////////////////////////////// /** A fake file system to test whether GCSLogStore properly handles precondition failures. */ class FailingGCSFileSystem extends RawLocalFileSystem { override def getScheme: String = "gs" override def getUri: URI = URI.create("gs:/") override def create(f: Path, overwrite: Boolean): FSDataOutputStream = { throw new IOException("412 Precondition Failed"); } } /** * A fake AbstractFileSystem to test whether session Hadoop configuration will be picked up. * This is a wrapper around [[FakeFileSystem]]. */ class FakeAbstractFileSystem(uri: URI, conf: org.apache.hadoop.conf.Configuration) extends org.apache.hadoop.fs.DelegateToFileSystem( uri, new FakeFileSystem, conf, FakeFileSystem.scheme, false) { // Implementation copied from RawLocalFs import org.apache.hadoop.fs.local.LocalConfigKeys import org.apache.hadoop.fs._ override def getUriDefaultPort(): Int = -1 override def getServerDefaults(): FsServerDefaults = LocalConfigKeys.getServerDefaults override def isValidName(src: String): Boolean = true } /** * A file system allowing to track how many times `rename` is called. * `TrackingRenameFileSystem.numOfRename` should be reset to 0 before starting to trace. */ class TrackingRenameFileSystem extends RawLocalFileSystem { override def rename(src: Path, dst: Path): Boolean = { TrackingRenameFileSystem.renameCounter.incrementAndGet() super.rename(src, dst) } } object TrackingRenameFileSystem { val renameCounter = new AtomicInteger(0) def resetCounter(): Unit = renameCounter.set(0) } /** * A fake AbstractFileSystem to ensure FileSystem.renameInternal(), and thus FileContext.rename(), * fails. This will be used to test HDFSLogStore.writeInternal corner case. */ class FailingRenameAbstractFileSystem(uri: URI, conf: org.apache.hadoop.conf.Configuration) extends FakeAbstractFileSystem(uri, conf) { override def renameInternal(src: Path, dst: Path, overwrite: Boolean): Unit = { throw new org.apache.hadoop.fs.FileAlreadyExistsException(s"$dst path already exists") } } //////////////////////////////////////////////////////////////////// // Public LogStore (Java) suite tests from delta-storage artifact // //////////////////////////////////////////////////////////////////// abstract class PublicLogStoreSuite extends LogStoreSuiteBase { protected val publicLogStoreClassName: String // The actual type of LogStore created will be LogStoreAdaptor. override val logStoreClassName: String = classOf[LogStoreAdaptor].getName protected override def sparkConf = { super.sparkConf.set(logStoreClassConfKey, publicLogStoreClassName) } protected override def testInitFromSparkConf(): Unit = { test("instantiation through SparkConf") { assert(spark.sparkContext.getConf.get(logStoreClassConfKey) == publicLogStoreClassName) assert(LogStore(spark).getClass.getName == logStoreClassName) assert(LogStore(spark).asInstanceOf[LogStoreAdaptor] .logStoreImpl.getClass.getName == publicLogStoreClassName) } } } class PublicHDFSLogStoreSuite extends PublicLogStoreSuite with HDFSLogStoreSuiteBase { override protected val publicLogStoreClassName: String = classOf[io.delta.storage.HDFSLogStore].getName } class PublicS3SingleDriverLogStoreSuite extends PublicLogStoreSuite with S3SingleDriverLogStoreSuiteBase { override protected val publicLogStoreClassName: String = classOf[io.delta.storage.S3SingleDriverLogStore].getName override protected def canInvalidateCache: Boolean = false } class PublicAzureLogStoreSuite extends PublicLogStoreSuite with AzureLogStoreSuiteBase { override protected val publicLogStoreClassName: String = classOf[io.delta.storage.AzureLogStore].getName } class PublicLocalLogStoreSuite extends PublicLogStoreSuite with LocalLogStoreSuiteBase { override protected val publicLogStoreClassName: String = classOf[io.delta.storage.LocalLogStore].getName } class PublicGCSLogStoreSuite extends PublicLogStoreSuite with GCSLogStoreSuiteBase { override protected val publicLogStoreClassName: String = classOf[io.delta.storage.GCSLogStore].getName } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/LogStoreSuiteBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.{File, IOException} import java.net.URI import java.util.concurrent.atomic.AtomicInteger import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.storage._ import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, FileSystem, FSDataOutputStream, Path, RawLocalFileSystem} import org.apache.spark.{SparkConf, SparkFunSuite} import org.apache.spark.sql.{LocalSparkSession, QueryTest, SparkSession} import org.apache.spark.sql.LocalSparkSession.withSparkSession import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.Utils ///////////////////// // Base Test Suite // ///////////////////// abstract class LogStoreSuiteBase extends QueryTest with LogStoreProvider with SharedSparkSession with DeltaSQLCommandTest { def logStoreClassName: String protected override def sparkConf = { super.sparkConf.set(logStoreClassConfKey, logStoreClassName) } // scalastyle:off deltahadoopconfiguration def sessionHadoopConf: Configuration = spark.sessionState.newHadoopConf // scalastyle:on deltahadoopconfiguration protected def testInitFromSparkConf(): Unit = { test("instantiation through SparkConf") { assert(spark.sparkContext.getConf.get(logStoreClassConfKey) == logStoreClassName) assert(LogStore(spark).getClass.getName == logStoreClassName) } } testInitFromSparkConf() protected def withTempLogDir(f: File => Unit): Unit = { val dir = Utils.createTempDir() val deltaLogDir = new File(dir, "_delta_log") deltaLogDir.mkdir() try f(deltaLogDir) finally { Utils.deleteRecursively(dir) } } test("read / write") { def assertNoLeakedCrcFiles(dir: File): Unit = { // crc file should not be leaked when origin file doesn't exist. // The implementation of Hadoop filesystem may filter out checksum file, so // listing files from local filesystem. val fileNames = dir.listFiles().toSeq.filter(p => p.isFile).map(p => p.getName) val crcFiles = fileNames.filter(n => n.startsWith(".") && n.endsWith(".crc")) val originFileNamesForExistingCrcFiles = crcFiles.map { name => // remove first "." and last ".crc" name.substring(1, name.length - 4) } // Check all origin files exist for all crc files. assert(originFileNamesForExistingCrcFiles.toSet.subsetOf(fileNames.toSet), s"Some of origin files for crc files don't exist - crc files: $crcFiles / " + s"expected origin files: $originFileNamesForExistingCrcFiles / actual files: $fileNames") } def pathToFileStatus(path: Path): FileStatus = path.getFileSystem(sessionHadoopConf).getFileStatus(path) withTempLogDir { tempLogDir => val store = createLogStore(spark) val deltas = Seq(0, 1) .map(i => new File(tempLogDir, i.toString)).map(_.toURI).map(new Path(_)) store.write(deltas.head, Iterator("zero", "none"), overwrite = false, sessionHadoopConf) store.write(deltas(1), Iterator("one"), overwrite = false, sessionHadoopConf) // Test Path based read APIs assert(store.read(deltas.head, sessionHadoopConf) == Seq("zero", "none")) assert(store.readAsIterator(deltas.head, sessionHadoopConf).toSeq == Seq("zero", "none")) assert(store.read(deltas(1), sessionHadoopConf) == Seq("one")) assert(store.readAsIterator(deltas(1), sessionHadoopConf).toSeq == Seq("one")) // Test FileStatus based read APIs assert(store.read(pathToFileStatus(deltas.head), sessionHadoopConf) == Seq("zero", "none")) assert(store.readAsIterator(pathToFileStatus(deltas.head), sessionHadoopConf).toSeq == Seq("zero", "none")) assert(store.read(pathToFileStatus(deltas(1)), sessionHadoopConf) == Seq("one")) assert(store.readAsIterator(pathToFileStatus(deltas(1)), sessionHadoopConf).toSeq == Seq("one")) assertNoLeakedCrcFiles(tempLogDir) } } test("detects conflict") { withTempLogDir { tempLogDir => val store = createLogStore(spark) val deltas = Seq(0, 1) .map(i => new File(tempLogDir, i.toString)).map(_.toURI).map(new Path(_)) store.write(deltas.head, Iterator("zero"), overwrite = false, sessionHadoopConf) store.write(deltas(1), Iterator("one"), overwrite = false, sessionHadoopConf) intercept[java.nio.file.FileAlreadyExistsException] { store.write(deltas(1), Iterator("uno"), overwrite = false, sessionHadoopConf) } } } test("listFrom") { withTempLogDir { tempLogDir => val store = createLogStore(spark) val deltas = Seq(0, 1, 2, 3, 4).map(i => new File(tempLogDir, i.toString)).map(_.toURI).map(new Path(_)) store.write(deltas(1), Iterator("zero"), overwrite = false, sessionHadoopConf) store.write(deltas(2), Iterator("one"), overwrite = false, sessionHadoopConf) store.write(deltas(3), Iterator("two"), overwrite = false, sessionHadoopConf) assert( store.listFrom(deltas.head, sessionHadoopConf) .map(_.getPath.getName).toArray === Seq(1, 2, 3).map(_.toString)) assert( store.listFrom(deltas(1), sessionHadoopConf) .map(_.getPath.getName).toArray === Seq(1, 2, 3).map(_.toString)) assert(store.listFrom(deltas(2), sessionHadoopConf) .map(_.getPath.getName).toArray === Seq(2, 3).map(_.toString)) assert(store.listFrom(deltas(3), sessionHadoopConf) .map(_.getPath.getName).toArray === Seq(3).map(_.toString)) assert(store.listFrom(deltas(4), sessionHadoopConf).map(_.getPath.getName).toArray === Nil) } } test("simple log store test") { val tempDir = Utils.createTempDir() val log1 = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) assert(log1.store.getClass.getName == logStoreClassName) val txn = log1.startTransaction() txn.commitManually(createTestAddFile()) log1.checkpoint() DeltaLog.clearCache() val log2 = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) assert(log2.store.getClass.getName == logStoreClassName) assert(log2.readLastCheckpointFile().map(_.version) === Some(0L)) assert(log2.snapshot.allFiles.count == 1) } protected def testHadoopConf(expectedErrMsg: String, fsImplConfs: (String, String)*): Unit = { test("should pick up fs impl conf from session Hadoop configuration") { withTempDir { tempDir => // scalastyle:off pathfromuri val path = new Path(new URI(s"fake://${tempDir.toURI.getRawPath}/1.json")) // scalastyle:on pathfromuri // Make sure it will fail without FakeFileSystem val e = intercept[IOException] { createLogStore(spark).listFrom(path, sessionHadoopConf) } assert(e.getMessage.matches(expectedErrMsg)) withSQLConf(fsImplConfs: _*) { createLogStore(spark).listFrom(path, sessionHadoopConf) } } } } /** * Whether the log store being tested should use rename to write checkpoint or not. The following * test is using this method to verify the behavior of `checkpoint`. */ protected def shouldUseRenameToWriteCheckpoint: Boolean test( "use isPartialWriteVisible to decide whether use rename") { withTempDir { tempDir => import testImplicits._ // Write 5 files to delta table (1 to 100).toDF().repartition(5).write.format("delta").save(tempDir.getCanonicalPath) withSQLConf( "fs.file.impl" -> classOf[TrackingRenameFileSystem].getName, "fs.file.impl.disable.cache" -> "true") { val deltaLog = DeltaLog.forTable(spark, tempDir.getCanonicalPath) TrackingRenameFileSystem.renameCounter.set(0) deltaLog.checkpoint() val expectedNumOfRename = if (shouldUseRenameToWriteCheckpoint) 1 else 0 assert(TrackingRenameFileSystem.renameCounter.get() === expectedNumOfRename) withSQLConf(DeltaSQLConf.DELTA_CHECKPOINT_PART_SIZE.key -> "9") { // Write 5 more files to the delta table (1 to 100).toDF().repartition(5).write .format("delta").mode("append").save(tempDir.getCanonicalPath) // At this point table has total 10 files, which won't fit in 1 checkpoint part file (as // DELTA_CHECKPOINT_PART_SIZE is set to 9 in this test). So this will end up generating // 2 PART files. TrackingRenameFileSystem.renameCounter.set(0) deltaLog.checkpoint() val expectedNumOfRename = if (shouldUseRenameToWriteCheckpoint) 2 else 0 assert(TrackingRenameFileSystem.renameCounter.get() === expectedNumOfRename) } } } } test("readAsIterator should be lazy") { withTempLogDir { tempLogDir => val store = createLogStore(spark) val testFile = new File(tempLogDir, "readAsIterator").getCanonicalPath store.write(new Path(testFile), Iterator("foo", "bar"), overwrite = false, sessionHadoopConf) withSQLConf( "fs.fake.impl" -> classOf[FakeFileSystem].getName, "fs.fake.impl.disable.cache" -> "true") { val fsStats = FileSystem.getStatistics("fake", classOf[FakeFileSystem]) fsStats.reset() val iter = store.readAsIterator(new Path(s"fake:///$testFile"), sessionHadoopConf) try { // We should not read any date when creating the iterator. assert(fsStats.getBytesRead == 0) assert(iter.toList == "foo" :: "bar" :: Nil) // Verify we are using the correct Statistics instance. assert(fsStats.getBytesRead == 8) } finally { iter.close() } } } } test("LogStoreInverseAdaptor is equivalent to base LogStore") { withTempLogDir { tempLogDir => val scalaStore = createLogStore(spark) val javaStore = new LogStoreInverseAdaptor(scalaStore, sessionHadoopConf) // Write with scala, read as java. val testFile = new File(tempLogDir, "readAsIteratorScala").getCanonicalPath scalaStore.write( new Path(testFile), Iterator("foo", "bar"), overwrite = false, sessionHadoopConf) val contents = javaStore.read(new Path(testFile), sessionHadoopConf) assert(contents.next() == "foo") assert(contents.next() == "bar") assert(!contents.hasNext) contents.close() // Write with java, read as scala. val testFile2 = new File(tempLogDir, "readAsIteratorJava").getCanonicalPath javaStore.write( new Path(testFile2), Iterator("foo", "bar").asJava, overwrite = false, sessionHadoopConf) val contents2 = scalaStore.readAsIterator(new Path(testFile), sessionHadoopConf) assert(contents2.toList == "foo" :: "bar" :: Nil) contents2.close() } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/MaterializePartitionColumnsFeatureSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.jdk.CollectionConverters._ import org.apache.spark.sql.delta.actions.{AddFile, Protocol} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.util.DeltaFileOperations import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.parquet.hadoop.ParquetFileReader import org.apache.parquet.hadoop.util.HadoopInputFile import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.test.SharedSparkSession class MaterializePartitionColumnsFeatureSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { private def validateWriterFeatureEnabled(deltaLog: DeltaLog, isEnabled: Boolean): Unit = { val protocol = deltaLog.update().protocol assert(protocol.isFeatureSupported(MaterializePartitionColumnsTableFeature) == isEnabled) assert(protocol.writerFeatures.getOrElse(Set.empty).contains("materializePartitionColumns") == isEnabled) assert(!protocol.readerFeatures.getOrElse(Set.empty).contains("materializePartitionColumns")) } private def validateAddedFilesFromLastOperationMaterializedPartitionColumn( deltaLog: DeltaLog, expectedMaterialized: Boolean, partCol: Seq[String]): Unit = { val snapshot = deltaLog.update() val currentVersion = snapshot.version val addedFiles = deltaLog.getChanges(currentVersion).flatMap(_._2).collect { case a: AddFile => a } assert(addedFiles.nonEmpty) val logicalToPhysicalNameMap = DeltaColumnMapping.getLogicalNameToPhysicalNameMap( snapshot.schema) val physicalPartCol = logicalToPhysicalNameMap(partCol).head addedFiles.foreach { file => val filePath = DeltaFileOperations.absolutePath(deltaLog.dataPath.toString, file.path) val path = new Path(filePath.toString) val fileReader = ParquetFileReader.open(HadoopInputFile.fromPath(path, new Configuration())) val parquetSchema = try { val metaData = fileReader.getFooter metaData.getFileMetaData.getSchema } finally { fileReader.close() } val fieldNames = parquetSchema.getFields.asScala.map(_.getName).toSet assert(fieldNames.contains(physicalPartCol) == expectedMaterialized) } } Seq(true, false).foreach { enable => test("MaterializePartitionColumnsTableFeature is auto-enabled when table property is set - " + s"enable=$enable") { val tbl = "tbl" withTable(tbl) { sql( s"""CREATE TABLE $tbl (id LONG, partCol INT) |USING DELTA |PARTITIONED BY (partCol) |""".stripMargin) sql(s"ALTER TABLE $tbl SET TBLPROPERTIES ('${DeltaConfigs .ENABLE_MATERIALIZE_PARTITION_COLUMNS_FEATURE.key}' = '$enable')") val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tbl)) validateWriterFeatureEnabled(deltaLog, isEnabled = enable) } } } test("DROP / Add back feature for materializePartitionColumns removes / adds feature and " + "stops / starts materializing columns") { val tbl = "tbl" withTable(tbl) { // Create table with materializePartitionColumns feature enabled sql( s"""CREATE TABLE $tbl (id LONG, partCol INT) |USING DELTA |PARTITIONED BY (partCol) |""".stripMargin) sql(s"ALTER TABLE $tbl SET TBLPROPERTIES ('${DeltaConfigs .ENABLE_MATERIALIZE_PARTITION_COLUMNS_FEATURE.key}' = 'true')") val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tbl)) // Verify feature is enabled validateWriterFeatureEnabled(deltaLog, isEnabled = true) // Insert data - partition columns should be materialized sql(s"INSERT INTO $tbl VALUES (1, 100), (2, 200)") validateAddedFilesFromLastOperationMaterializedPartitionColumn( deltaLog, expectedMaterialized = true, partCol = Seq("partCol")) // Drop the feature sql(s"ALTER TABLE $tbl DROP FEATURE materializePartitionColumns") // Verify feature is removed from protocol validateWriterFeatureEnabled(deltaLog, isEnabled = false) // Insert more data - new files should NOT have materialized partition columns sql(s"INSERT INTO $tbl VALUES (3, 300), (4, 400)") validateAddedFilesFromLastOperationMaterializedPartitionColumn( deltaLog, expectedMaterialized = false, partCol = Seq("partCol")) // Add table feature back and verify partition columns are materialized again sql(s"ALTER TABLE $tbl SET TBLPROPERTIES ('${DeltaConfigs .ENABLE_MATERIALIZE_PARTITION_COLUMNS_FEATURE.key}' = 'true')") validateWriterFeatureEnabled(deltaLog, isEnabled = true) // Insert data - partition columns should be materialized again, all the files // including old and new should have partition columns sql(s"INSERT INTO $tbl VALUES (5, 500), (6, 600)") validateAddedFilesFromLastOperationMaterializedPartitionColumn( deltaLog, expectedMaterialized = true, partCol = Seq("partCol")) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoAccumulatorSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.concurrent.atomic.AtomicReference import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.commands.MergeIntoCommandBase import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent, SparkListenerNodeExcluded, SparkListenerTaskEnd} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.status.TaskDataWrapper import org.apache.spark.util.JsonProtocol /** * Tests how the accumulator used by the MERGE command reacts with other Spark components such as * Spark UI. These tests stay in a separated file so that we can use the package name * `org.apache.spark.sql.delta` to access `private[spark]` APIs. */ class MergeIntoAccumulatorSuite extends SharedSparkSession with DeltaSQLCommandTest { import testImplicits._ private def runTestMergeCommand(): Unit = { // Run a simple merge command withTempView("source") { withTempDir { tempDir => val tempPath = tempDir.getCanonicalPath Seq((1, 1), (0, 3)).toDF("key", "value").createOrReplaceTempView("source") Seq((2, 2), (1, 4)).toDF("key", "value").write.format("delta").save(tempPath) spark.sql(s""" |MERGE INTO delta.`$tempPath` target |USING source src |ON src.key = target.key |WHEN MATCHED THEN UPDATE SET * |WHEN NOT MATCHED THEN INSERT * |""".stripMargin) } } } test("accumulators used by MERGE should not be tracked by Spark UI") { runTestMergeCommand() // Make sure all Spark events generated by the above command have been processed spark.sparkContext.listenerBus.waitUntilEmpty(30000) val store = spark.sparkContext.statusStore.store val iter = store.view(classOf[TaskDataWrapper]).closeableIterator() try { // Collect all accumulator names tracked by Spark UI. val accumNames = iter.asScala.toVector.flatMap { task => task.accumulatorUpdates.map(_.name) }.toSet // Verify accumulators used by MergeIntoCommand are not tracked. assert(!accumNames.contains(MergeIntoCommandBase.TOUCHED_FILES_ACCUM_NAME)) } finally { iter.close() } } test("accumulators used by MERGE should not fail Spark event log generation") { // Register a listener to convert `SparkListenerTaskEnd` to json and catch failures. val failure = new AtomicReference[Throwable]() val listener = new SparkListener { override def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit = { try JsonProtocol.sparkEventToJsonString(taskEnd) catch { case t: Throwable => failure.compareAndSet(null, t) } } } spark.sparkContext.listenerBus.addToSharedQueue(listener) try { runTestMergeCommand() // Make sure all Spark events generated by the above command have been processed spark.sparkContext.listenerBus.waitUntilEmpty(30000) // Converting `SparkListenerEvent` to json should not fail assert(failure.get == null) } finally { spark.sparkContext.listenerBus.removeListener(listener) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoDVsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.cdc.MergeCDCMixin import org.apache.spark.sql.delta.commands.{DeletionVectorBitmapGenerator, DMLWithDeletionVectorsHelper} import org.apache.spark.sql.delta.files.TahoeBatchFileIndex import org.apache.spark.SparkException import org.apache.spark.sql.QueryTest import org.apache.spark.sql.functions.col trait MergeIntoDVsMixin extends MergeIntoSQLMixin with DeletionVectorsTestUtils { override def beforeAll(): Unit = { super.beforeAll() enableDeletionVectors(spark, merge = true) } override def excluded: Seq[String] = { val miscFailures = Seq( "basic case - merge to view on a Delta table, " + "partitioned: true skippingEnabled: false useSqlView: true", "basic case - merge to view on a Delta table, " + "partitioned: true skippingEnabled: false useSqlView: false", "basic case - merge to view on a Delta table, " + "partitioned: false skippingEnabled: false useSqlView: true", "basic case - merge to view on a Delta table, " + "partitioned: false skippingEnabled: false useSqlView: false", "basic case - merge to Delta table, isPartitioned: false skippingEnabled: false", "basic case - merge to Delta table, isPartitioned: true skippingEnabled: false", "not matched by source - all 3 clauses - no changes - " + "isPartitioned: true - cdcEnabled: true", "not matched by source - all 3 clauses - no changes - " + "isPartitioned: false - cdcEnabled: true", "test merge on temp view - view with too many internal aliases - Dataset TempView" ) super.excluded ++ miscFailures } protected override lazy val expectedOpTypes: Set[String] = Set( "delta.dml.merge.materializeSource", "delta.dml.merge.findTouchedFiles", "delta.dml.merge.writeModifiedRowsOnly", "delta.dml.merge.writeDeletionVectors", "delta.dml.merge") } trait MergeIntoDVsTests extends MergeIntoDVsMixin { import testImplicits._ private def assertOperationalDVMetrics( tablePath: String, numDeletedRows: Long, numUpdatedRows: Long, numCopiedRows: Long, numTargetFilesRemoved: Long, numDeletionVectorsAdded: Long, numDeletionVectorsRemoved: Long, numDeletionVectorsUpdated: Long): Unit = { val table = io.delta.tables.DeltaTable.forPath(tablePath) val mergeMetrics = DeltaMetricsUtils.getLastOperationMetrics(table) assert(mergeMetrics.getOrElse("numTargetRowsDeleted", -1) === numDeletedRows) assert(mergeMetrics.getOrElse("numTargetRowsUpdated", -1) === numUpdatedRows) assert(mergeMetrics.getOrElse("numTargetRowsCopied", -1) === numCopiedRows) assert(mergeMetrics.getOrElse("numTargetFilesRemoved", -1) === numTargetFilesRemoved) assert(mergeMetrics.getOrElse("numTargetDeletionVectorsAdded", -1) === numDeletionVectorsAdded) assert( mergeMetrics.getOrElse("numTargetDeletionVectorsRemoved", -1) === numDeletionVectorsRemoved) assert( mergeMetrics.getOrElse("numTargetDeletionVectorsUpdated", -1) === numDeletionVectorsUpdated) } test(s"Merge with DVs metrics - Incremental Updates") { withTempDir { dir => val sourcePath = s"$dir/source" spark.range(0, 10, 2).write.format("delta").save(sourcePath) append(spark.range(10).toDF()) executeMerge( tgt = s"$tableSQLIdentifier t", src = s"delta.`$sourcePath` s", cond = "t.id = s.id", clauses = updateNotMatched(set = "id = t.id * 10")) checkAnswer(readDeltaTableByIdentifier(), Seq(0, 10, 2, 30, 4, 50, 6, 70, 8, 90).toDF("id")) assertOperationalDVMetrics( deltaLog.dataPath.toString, numDeletedRows = 0, numUpdatedRows = 5, numCopiedRows = 0, numTargetFilesRemoved = 0, // No files were fully deleted. numDeletionVectorsAdded = 2, numDeletionVectorsRemoved = 0, numDeletionVectorsUpdated = 0) executeMerge( tgt = s"$tableSQLIdentifier t", src = s"delta.`$sourcePath` s", cond = "t.id = s.id", clauses = delete(condition = "t.id = 2")) checkAnswer(readDeltaTableByIdentifier(), Seq(0, 10, 30, 4, 50, 6, 70, 8, 90).toDF("id")) assertOperationalDVMetrics( deltaLog.dataPath.toString, numDeletedRows = 1, numUpdatedRows = 0, numCopiedRows = 0, numTargetFilesRemoved = 0, numDeletionVectorsAdded = 1, // Updating a DV equals removing and adding. numDeletionVectorsRemoved = 1, // Updating a DV equals removing and adding. numDeletionVectorsUpdated = 1) // Delete all rows from a file. executeMerge( tgt = s"$tableSQLIdentifier t", src = s"delta.`$sourcePath` s", cond = "t.id = s.id", clauses = delete(condition = "t.id < 5")) checkAnswer(readDeltaTableByIdentifier(), Seq(10, 30, 50, 6, 70, 8, 90).toDF("id")) assertOperationalDVMetrics( deltaLog.dataPath.toString, numDeletedRows = 2, numUpdatedRows = 0, numCopiedRows = 0, numTargetFilesRemoved = 1, numDeletionVectorsAdded = 0, numDeletionVectorsRemoved = 1, numDeletionVectorsUpdated = 0) } } test(s"Merge with DVs metrics - delete entire file") { withTempDir { dir => val sourcePath = s"$dir/source" spark.range(0, 7).write.format("delta").save(sourcePath) append(spark.range(10).toDF()) executeMerge( tgt = s"$tableSQLIdentifier t", src = s"delta.`$sourcePath` s", cond = "t.id = s.id", clauses = update(set = "id = t.id * 10")) checkAnswer(readDeltaTableByIdentifier(), Seq(0, 10, 20, 30, 40, 50, 60, 7, 8, 9).toDF("id")) assertOperationalDVMetrics( deltaLog.dataPath.toString, numDeletedRows = 0, numUpdatedRows = 7, numCopiedRows = 0, // No rows were copied. numTargetFilesRemoved = 1, // 1 file was removed entirely. numDeletionVectorsAdded = 1, // 1 file was deleted partially. numDeletionVectorsRemoved = 0, numDeletionVectorsUpdated = 0) } } test(s"Verify error is produced when paths are not joined correctly") { withTempDir { dir => val sourcePath = s"$dir/source" val targetPath = s"$dir/target" spark.range(0, 10, 2).write.format("delta").save(sourcePath) spark.range(10).write.format("delta").save(targetPath) // Execute buildRowIndexSetsForFilesMatchingCondition with a corrupted touched files list. val sourceDF = io.delta.tables.DeltaTable.forPath(sourcePath).toDF val targetDF = io.delta.tables.DeltaTable.forPath(targetPath).toDF val targetLog = DeltaLog.forTable(spark, targetPath) val condition = col("s.id") === col("t.id") val allFiles = targetLog.update().allFiles.collect().toSeq assert(allFiles.size === 2) val corruptedFiles = Seq( allFiles.head, allFiles.last.copy(path = "corruptedPath")) val txn = targetLog.startTransaction(catalogTableOpt = None) val fileIndex = new TahoeBatchFileIndex( spark, actionType = "merge", addFiles = allFiles, deltaLog = targetLog, path = targetLog.dataPath, snapshot = txn.snapshot) val targetDFWithMetadata = DMLWithDeletionVectorsHelper.createTargetDfForScanningForMatches( spark, targetDF.queryExecution.logical, fileIndex) val e = intercept[SparkException] { DeletionVectorBitmapGenerator.buildRowIndexSetsForFilesMatchingCondition( spark, txn, tableHasDVs = true, targetDf = sourceDF.as("s").join(targetDFWithMetadata.as("t"), condition), candidateFiles = corruptedFiles, condition = condition.expr, fileNameColumnOpt = Option(col("s._metadata.file_name")), rowIndexColumnOpt = Option(col("s._metadata.row_index")) ) } assert(e.getCause.getMessage.contains("Encountered a non matched file path.")) } } } trait MergeCDCWithDVsMixin extends QueryTest with MergeCDCMixin with DeletionVectorsTestUtils { override def beforeAll(): Unit = { super.beforeAll() enableDeletionVectors(spark, merge = true) } override def excluded: Seq[String] = { /** * Merge commands that result to no actions do not generate a new commit when DVs are enabled. * We correct affected tests by changing the expected CDC result (Create table CDC). */ val miscFailures = "merge CDC - all conditions failed for all rows" super.excluded :+ miscFailures } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoMaterializeSourceSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.collection.mutable import scala.concurrent.duration._ import scala.reflect.ClassTag import scala.util.control.NonFatal import com.databricks.spark.util.{Log4jUsageLogger, MetricDefinitions, UsageRecord} import org.apache.spark.sql.delta.DeltaTestUtils._ import org.apache.spark.sql.delta.commands.merge.{MergeIntoMaterializeSourceError, MergeIntoMaterializeSourceErrorType, MergeIntoMaterializeSourceReason, MergeStats} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.util.JsonUtils import org.scalactic.source.Position import org.scalatest.Tag import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.sql.{DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Expression, Literal} import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.execution.{FilterExec, LogicalRDD, RDDScanExec, SQLExecution} import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ import org.apache.spark.storage.StorageLevel import org.apache.spark.util.Utils trait MergeIntoMaterializeSourceMixin extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with DeltaSQLTestUtils with DeltaTestUtilsBase { override def beforeAll(): Unit = { super.beforeAll() // trigger source materialization in all tests spark.conf.set(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE.key, "all") } // Runs a merge query with source materialization, while a killer thread tries to unpersist it. protected def testMergeMaterializedSourceUnpersist( tblName: String, numKills: Int): Seq[UsageRecord] = { val maxAttempts = spark.conf.get(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE_MAX_ATTEMPTS) // when we ask to join the killer thread, it should exit in the next iteration. val killerThreadJoinTimeoutMs = 10000 // sleep between attempts to unpersist val killerIntervalMs = 1 // Data does not need to be big; there is enough latency to unpersist even with small data. val targetDF = spark.range(100).toDF("id") targetDF.write.format("delta").saveAsTable(tblName) spark.range(90, 120).toDF("id").createOrReplaceTempView("s") val mergeQuery = s"MERGE INTO $tblName t USING s ON t.id = s.id " + "WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT *" // Killer thread tries to unpersist any persisted mergeMaterializedSource RDDs, // until it has seen more than numKills distinct ones (from distinct Merge retries) @volatile var finished = false @volatile var invalidStorageLevel: Option[String] = None val killerThread = new Thread() { override def run(): Unit = { val seenSources = mutable.Set[Int]() while (!finished) { sparkContext.getPersistentRDDs.foreach { case (rddId, rdd) => if (rdd.name == "mergeMaterializedSource") { if (!seenSources.contains(rddId)) { logInfo(s"First time seeing mergeMaterializedSource with id=$rddId") seenSources.add(rddId) } if (seenSources.size > numKills) { // already unpersisted numKills different source materialization attempts, // the killer can retire logInfo(s"seenSources.size=${seenSources.size}. Proceeding to finish.") finished = true } else { // Need to wait until it is actually checkpointed, otherwise if we try to unpersist // before it starts to actually persist it fails with // java.lang.AssertionError: assumption failed: // Storage level StorageLevel(1 replicas) is not appropriate for local checkpointing // (this wouldn't happen in real world scenario of losing the block because executor // was lost; there nobody manipulates with StorageLevel; if failure happens during // computation of the materialized rdd, the task would be reattempted using the // regular task retry mechanism) if (rdd.isCheckpointed) { // Use this opportunity to test if the source has the correct StorageLevel. val expectedStorageLevel = StorageLevel.fromString( if (seenSources.size == 1) { spark.conf.get(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE_RDD_STORAGE_LEVEL) } else if (seenSources.size == 2) { spark.conf.get( DeltaSQLConf.MERGE_MATERIALIZE_SOURCE_RDD_STORAGE_LEVEL_FIRST_RETRY) } else { spark.conf.get(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE_RDD_STORAGE_LEVEL_RETRY) } ) val rddStorageLevel = rdd.getStorageLevel if (rddStorageLevel != expectedStorageLevel) { invalidStorageLevel = Some(s"For attempt ${seenSources.size} of materialized source expected " + s"$expectedStorageLevel but got ${rddStorageLevel}") finished = true } logInfo(s"Unpersisting mergeMaterializedSource with id=$rddId") // don't make it blocking, so that the killer turns around quickly and is ready // for the next kill when Merge retries rdd.unpersist(blocking = false) } } } } Thread.sleep(killerIntervalMs) } logInfo(s"seenSources.size=${seenSources.size}. Proceeding to finish.") } } killerThread.start() val events = Log4jUsageLogger.track { try { sql(mergeQuery) } catch { case NonFatal(ex) => if (numKills < maxAttempts) { // The merge should succeed with retries throw ex } } finally { finished = true // put the killer to rest, if it didn't retire already killerThread.join(killerThreadJoinTimeoutMs) assert(!killerThread.isAlive) } }.filter(_.metric == MetricDefinitions.EVENT_TAHOE.name) // If killer thread recorded an invalid StorageLevel, throw it here assert(invalidStorageLevel.isEmpty, invalidStorageLevel.toString) events } } trait MergeIntoMaterializeSourceErrorTests extends MergeIntoMaterializeSourceMixin { import testImplicits._ // Test error message that we check if blocks of materialized source RDD were evicted. test("missing RDD blocks error message") { val checkpointedDf = sql("select * from range(10)") .localCheckpoint(eager = false) val rdd = checkpointedDf.queryExecution.analyzed.asInstanceOf[LogicalRDD].rdd checkpointedDf.collect() // trigger lazy materialization rdd.unpersist() val ex = intercept[Exception] { checkpointedDf.collect() } assert(ex.isInstanceOf[SparkException], ex) val sparkEx = ex.asInstanceOf[SparkException] assert( sparkEx.getErrorClass == "CHECKPOINT_RDD_BLOCK_ID_NOT_FOUND" && sparkEx.getMessageParameters.get("rddBlockId").contains(s"rdd_${rdd.id}")) } for { materialized <- BOOLEAN_DOMAIN } test(s"merge logs out of disk errors - materialized=$materialized") { import DeltaSQLConf.MergeMaterializeSource withSQLConf( DeltaSQLConf.MERGE_MATERIALIZE_SOURCE.key -> (if (materialized) MergeMaterializeSource.AUTO else MergeMaterializeSource.NONE)) { val injectEx = new java.io.IOException("No space left on device") testWithCustomErrorInjected[SparkException](injectEx) { (thrownEx, errorOpt) => // Compare messages instead of instances, since the equals method for these exceptions // takes more into account. assert(thrownEx.getCause.getMessage === injectEx.getMessage) if (materialized) { assert(errorOpt.isDefined) val error = errorOpt.get assert(error.errorType == MergeIntoMaterializeSourceErrorType.OUT_OF_DISK.toString) assert(error.attempt == 1) val storageLevel = StorageLevel.fromString( spark.conf.get(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE_RDD_STORAGE_LEVEL)) assert(error.materializedSourceRDDStorageLevel == storageLevel.toString) } else { assert(errorOpt.isEmpty) } } } } test("merge rethrows arbitrary errors") { val injectEx = new RuntimeException("test") testWithCustomErrorInjected[SparkException](injectEx) { (thrownEx, error) => // Compare messages instead of instances, since the equals method for these exceptions // takes more into account. assert(thrownEx.getCause.getMessage === injectEx.getMessage) assert(error.isEmpty) } } private def testWithCustomErrorInjected[Intercept >: Null <: Exception with AnyRef : ClassTag]( inject: Exception)( handle: (Intercept, Option[MergeIntoMaterializeSourceError]) => Unit): Unit = { { val tblName = "target" withTable(tblName) { val targetDF = spark.range(10).toDF("id").withColumn("value", rand()) targetDF.write.format("delta").saveAsTable(tblName) spark .range(10) .mapPartitions { x => throw inject x } .toDF("id") .withColumn("value", rand()) .createOrReplaceTempView("s") var thrownException: Intercept = null val events = Log4jUsageLogger .track { thrownException = intercept[Intercept] { sql(s"MERGE INTO $tblName t USING s ON t.id = s.id " + s"WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT *") } } .filter { e => e.metric == MetricDefinitions.EVENT_TAHOE.name && e.tags.get("opType").contains(MergeIntoMaterializeSourceError.OP_TYPE) } val error = events.headOption .map(e => JsonUtils.fromJson[MergeIntoMaterializeSourceError](e.blob)) handle(thrownException, error) } } } private def testMergeMaterializeSourceUnpersistRetries = { val maxAttempts = DeltaSQLConf.MERGE_MATERIALIZE_SOURCE_MAX_ATTEMPTS.defaultValue.get val tblName = "target" // For 1 to maxAttempts - 1 RDD block lost failures, merge should retry and succeed. for { kills <- 1 to maxAttempts - 1 } { test(s"materialize source unpersist with $kills kill attempts succeeds") { withTable(tblName) { val allDeltaEvents = testMergeMaterializedSourceUnpersist(tblName, kills) val events = allDeltaEvents.filter(_.tags.get("opType").contains("delta.dml.merge.stats")) assert(events.length == 1, s"allDeltaEvents:\n$allDeltaEvents") val mergeStats = JsonUtils.fromJson[MergeStats](events(0).blob) assert(mergeStats.materializeSourceAttempts.isDefined, s"MergeStats:\n$mergeStats") assert( mergeStats.materializeSourceAttempts.get == kills + 1, s"MergeStats:\n$mergeStats") // Check query result after merge val tab = sql(s"select * from $tblName order by id") .collect() .map(row => row.getLong(0)) .toSeq assert(tab == (0L until 90L) ++ (100L until 120L)) } } } // Eventually it should fail after exceeding maximum number of attempts. test(s"materialize source unpersist with $maxAttempts kill attempts fails") { withTable(tblName) { val allDeltaEvents = testMergeMaterializedSourceUnpersist(tblName, maxAttempts) val events = allDeltaEvents .filter(_.tags.get("opType").contains(MergeIntoMaterializeSourceError.OP_TYPE)) assert(events.length == 1, s"allDeltaEvents:\n$allDeltaEvents") val error = JsonUtils.fromJson[MergeIntoMaterializeSourceError](events(0).blob) assert(error.errorType == MergeIntoMaterializeSourceErrorType.RDD_BLOCK_LOST.toString) assert(error.attempt == maxAttempts) } } } testMergeMaterializeSourceUnpersistRetries } trait MergeIntoMaterializeSourceTests extends MergeIntoMaterializeSourceMixin { import testImplicits._ private def getHints(df: => DataFrame): Seq[(Seq[ResolvedHint], JoinHint)] = { val plans = withAllPlansCaptured(spark) { df } var plansWithMaterializedSource = 0 val hints = plans.flatMap { p => val materializedSourceExists = p.analyzed.exists { case l: LogicalRDD if l.rdd.name == "mergeMaterializedSource" => true case _ => false } if (materializedSourceExists) { // If it is a plan with materialized source, there should be exactly one join // of target and source. We collect resolved hints from analyzed plans, and the hint // applied to the join from optimized plan. plansWithMaterializedSource += 1 val hints = p.analyzed.collect { case h: ResolvedHint => h } val joinHints = p.optimized.collect { case j: Join => j.hint } assert(joinHints.length == 1, s"Got $joinHints") val joinHint = joinHints.head // Only preserve join strategy hints, because we are testing with these. // Other hints may be added by MERGE internally, e.g. hints to force DFP/DPP, that // we don't want to be considering here. val retHints = hints .filter(_.hints.strategy.nonEmpty) def retJoinHintInfo(hintInfo: Option[HintInfo]): Option[HintInfo] = hintInfo match { case Some(h) if h.strategy.nonEmpty => Some(HintInfo(strategy = h.strategy)) case _ => None } val retJoinHint = joinHint.copy( leftHint = retJoinHintInfo(joinHint.leftHint), rightHint = retJoinHintInfo(joinHint.rightHint) ) Some((retHints, retJoinHint)) } else { None } } assert(plansWithMaterializedSource == 2, s"2 plans should have materialized source, but got: $plans") hints } test(s"materialize source preserves dataframe hints") { withTable("A", "B", "T") { sql("select id, id as v from range(50000)").write.format("delta").saveAsTable("T") sql("select id, id+2 as v from range(10000)").write.format("csv").saveAsTable("A") sql("select id, id*2 as v from range(1000)").write.format("csv").saveAsTable("B") // Manually added broadcast hint will mess up the expected hints hence disable it withSQLConf( SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") { // Simple BROADCAST hint val hSimple = getHints( sql("MERGE INTO T USING (SELECT /*+ BROADCAST */ * FROM A) s ON T.id = s.id" + " WHEN MATCHED THEN UPDATE SET * WHEN NOT MATCHED THEN INSERT *") ) hSimple.foreach { case (hints, joinHint) => assert(hints.length == 1) assert(hints.head.hints == HintInfo(strategy = Some(BROADCAST))) assert(joinHint == JoinHint(Some(HintInfo(strategy = Some(BROADCAST))), None)) } // Simple MERGE hint val hSimpleMerge = getHints( sql("MERGE INTO T USING (SELECT /*+ MERGE */ * FROM A) s ON T.id = s.id" + " WHEN MATCHED THEN UPDATE SET * WHEN NOT MATCHED THEN INSERT *") ) hSimpleMerge.foreach { case (hints, joinHint) => assert(hints.length == 1) assert(hints.head.hints == HintInfo(strategy = Some(SHUFFLE_MERGE))) assert(joinHint == JoinHint(Some(HintInfo(strategy = Some(SHUFFLE_MERGE))), None)) } // Aliased hint val hAliased = getHints( sql("MERGE INTO T USING " + "(SELECT /*+ BROADCAST(FOO) */ * FROM (SELECT * FROM A) FOO) s ON T.id = s.id" + " WHEN MATCHED THEN UPDATE SET * WHEN NOT MATCHED THEN INSERT *") ) hAliased.foreach { case (hints, joinHint) => assert(hints.length == 1) assert(hints.head.hints == HintInfo(strategy = Some(BROADCAST))) assert(joinHint == JoinHint(Some(HintInfo(strategy = Some(BROADCAST))), None)) } // Aliased hint - hint propagation does not work from under an alias // (remove if this ever gets implemented in the hint framework) val hAliasedInner = getHints( sql("MERGE INTO T USING " + "(SELECT /*+ BROADCAST(A) */ * FROM (SELECT * FROM A) FOO) s ON T.id = s.id" + " WHEN MATCHED THEN UPDATE SET * WHEN NOT MATCHED THEN INSERT *") ) hAliasedInner.foreach { case (hints, joinHint) => assert(hints.length == 0) assert(joinHint == JoinHint(None, None)) } // This hint applies to the join inside the source, not to the source as a whole val hJoinInner = getHints( sql("MERGE INTO T USING " + "(SELECT /*+ BROADCAST(A) */ A.* FROM A JOIN B WHERE A.id = B.id) s ON T.id = s.id" + " WHEN MATCHED THEN UPDATE SET * WHEN NOT MATCHED THEN INSERT *") ) hJoinInner.foreach { case (hints, joinHint) => assert(hints.length == 0) assert(joinHint == JoinHint(None, None)) } // Two hints - top one takes effect val hTwo = getHints( sql("MERGE INTO T USING (SELECT /*+ BROADCAST, MERGE */ * FROM A) s ON T.id = s.id" + " WHEN MATCHED THEN UPDATE SET * WHEN NOT MATCHED THEN INSERT *") ) hTwo.foreach { case (hints, joinHint) => assert(hints.length == 2) assert(hints(0).hints == HintInfo(strategy = Some(BROADCAST))) assert(hints(1).hints == HintInfo(strategy = Some(SHUFFLE_MERGE))) // top one takes effect assert(joinHint == JoinHint(Some(HintInfo(strategy = Some(BROADCAST))), None)) } } } } test("materialize source for non-deterministic source formats") { val targetSchema = StructType(Array( StructField("id", IntegerType, nullable = false), StructField("value", StringType, nullable = true))) val targetData = Seq( Row(1, "update"), Row(2, "skip"), Row(3, "delete")) val sourceData = Seq(1, 3, 4).toDF("id") val expectedResult = Seq( Row(1, "new"), // Updated Row(2, "skip"), // Copied // 3 is deleted Row(4, "new")) // Inserted // There are more, but these are easiest to test for. val nonDeterministicFormats = List("parquet", "json") // Return MergeIntoMaterializeSourceReason string def executeMerge(sourceDf: DataFrame): String = { val sourceDfWithAction = sourceDf.withColumn("value", lit("new")) var materializedSource: String = "" withTable("target") { val targetRdd = spark.sparkContext.parallelize(targetData) val targetDf = spark.createDataFrame(targetRdd, targetSchema) targetDf.write.format("delta").mode("overwrite").saveAsTable("target") val targetTable = io.delta.tables.DeltaTable.forName("target") val events: Seq[UsageRecord] = Log4jUsageLogger.track { targetTable.merge(sourceDfWithAction, col("target.id") === sourceDfWithAction("id")) .whenMatched(col("target.value") === lit("update")).updateAll() .whenMatched(col("target.value") === lit("delete")).delete() .whenNotMatched().insertAll() .execute() } // Can't return values out of withTable. materializedSource = mergeSourceMaterializeReason(events) checkAnswer( spark.read.format("delta").table("target"), expectedResult) } materializedSource } def checkSourceMaterialization( format: String, reason: String): Unit = { // Test once by name and once using path, as they produce different plans. withTable("source") { sourceData.write.format(format).saveAsTable("source") val sourceDf = spark.read.format(format).table("source") assert(executeMerge(sourceDf) == reason, s"Wrong materialization reason for $format") } withTempPath { sourcePath => sourceData.write.format(format).save(sourcePath.toString) val sourceDf = spark.read.format(format).load(sourcePath.toString) assert(executeMerge(sourceDf) == reason, s"Wrong materialization reason for $format") } } withSQLConf(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE.key -> "auto") { for (format <- nonDeterministicFormats) { checkSourceMaterialization( format, reason = MergeIntoMaterializeSourceReason.NON_DETERMINISTIC_SOURCE_NON_DELTA.toString) } // Delta should not materialize source. checkSourceMaterialization( "delta", reason = MergeIntoMaterializeSourceReason.NOT_MATERIALIZED_AUTO.toString) } // Test with non-Delta sources in subqueries. def checkSourceMaterializationSubquery( delta: String, filterSub: String, projectSub: String, nestedFilterSub: String, nestedProjectSub: String, testType: String, reason: String): Unit = { val df = spark.sql( s""" |SELECT | CASE WHEN id IN | (SELECT id kk FROM $projectSub WHERE id IN (SELECT * FROM $nestedFilterSub)) | THEN id ELSE -1 END AS id, | 0.5 AS value |FROM $delta |WHERE id IN | (SELECT CASE WHEN id IN (SELECT * FROM $nestedProjectSub) THEN id ELSE -1 END kk | FROM $filterSub) |""".stripMargin) assert(executeMerge(df) == reason, s"Wrong materialization reason with $testType subquery") } def checkSourceMaterializationSubqueries(deltaSource: String, nonDeltaSource: String): Unit = { checkSourceMaterializationSubquery( delta = deltaSource, filterSub = deltaSource, projectSub = deltaSource, nestedFilterSub = deltaSource, nestedProjectSub = deltaSource, testType = "all Delta", reason = MergeIntoMaterializeSourceReason.NOT_MATERIALIZED_AUTO.toString) checkSourceMaterializationSubquery( delta = deltaSource, filterSub = nonDeltaSource, projectSub = deltaSource, nestedFilterSub = deltaSource, nestedProjectSub = deltaSource, testType = "non-Delta filter", reason = MergeIntoMaterializeSourceReason.NON_DETERMINISTIC_SOURCE_NON_DELTA.toString) checkSourceMaterializationSubquery( delta = deltaSource, filterSub = deltaSource, projectSub = nonDeltaSource, nestedFilterSub = deltaSource, nestedProjectSub = deltaSource, testType = "non-Delta project", reason = MergeIntoMaterializeSourceReason.NON_DETERMINISTIC_SOURCE_NON_DELTA.toString) checkSourceMaterializationSubquery( delta = deltaSource, filterSub = deltaSource, projectSub = deltaSource, nestedFilterSub = nonDeltaSource, nestedProjectSub = deltaSource, testType = "non-Delta nested filter", reason = MergeIntoMaterializeSourceReason.NON_DETERMINISTIC_SOURCE_NON_DELTA.toString) checkSourceMaterializationSubquery( delta = deltaSource, filterSub = deltaSource, projectSub = deltaSource, nestedFilterSub = deltaSource, nestedProjectSub = nonDeltaSource, testType = "non-Delta nested project", reason = MergeIntoMaterializeSourceReason.NON_DETERMINISTIC_SOURCE_NON_DELTA.toString) } withSQLConf(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE.key -> "auto") { // Test once by name and once using path, as they produce different plans. withTable("deltaSource", "nonDeltaSource") { sourceData.write.format("delta").saveAsTable("deltaSource") sourceData.write.format("parquet").saveAsTable("nonDeltaSource") checkSourceMaterializationSubqueries("deltaSource", "nonDeltaSource") } withTempPath { deltaSourcePath => sourceData.write.format("delta").save(deltaSourcePath.toString) withTempPath { nonDeltaSourcePath => sourceData.write.format("parquet").save(nonDeltaSourcePath.toString) checkSourceMaterializationSubqueries( s"delta.`$deltaSourcePath`", s"parquet.`$nonDeltaSourcePath`") } } } // Mixed safe/unsafe queries should materialize source. def checkSourceMaterializationForMixedSources( format1: String, format2: String, shouldMaterializeSource: Boolean): Unit = { def checkWithSources(source1Df: DataFrame, source2Df: DataFrame): Unit = { val sourceDf = source1Df.union(source2Df) val materializeReason = executeMerge(sourceDf) if (shouldMaterializeSource) { assert(materializeReason == MergeIntoMaterializeSourceReason.NON_DETERMINISTIC_SOURCE_NON_DELTA.toString, s"$format1 union $format2 are not deterministic as a source and should materialize.") } else { assert(materializeReason == MergeIntoMaterializeSourceReason.NOT_MATERIALIZED_AUTO.toString, s"$format1 union $format2 is deterministic as a source and should not materialize.") } } // Test once by name and once using path, as they produce different plans. withTable("source1", "source2") { sourceData.filter(col("id") < 2).write.format(format1).saveAsTable("source1") val source1Df = spark.read.format(format1).table("source1") sourceData.filter(col("id") >= 2).write.format(format2).saveAsTable("source2") val source2Df = spark.read.format(format2).table("source2") checkWithSources(source1Df, source2Df) } withTempPaths(2) { case Seq(source1, source2) => sourceData.filter(col("id") < 2).write .mode("overwrite").format(format1).save(source1.toString) val source1Df = spark.read.format(format1).load(source1.toString) sourceData.filter(col("id") >= 2).write .mode("overwrite").format(format2).save(source2.toString) val source2Df = spark.read.format(format2).load(source2.toString) checkWithSources(source1Df, source2Df) } } withSQLConf(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE.key -> "auto") { val allFormats = "delta" :: nonDeterministicFormats // Try all combinations for { format1 <- allFormats format2 <- allFormats } checkSourceMaterializationForMixedSources( format1 = format1, format2 = format2, shouldMaterializeSource = !(format1 == "delta" && format2 == "delta")) } withSQLConf(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE.key -> "none") { // With "none", it should not materialize, even though parquet is non-deterministic. checkSourceMaterialization( "parquet", reason = MergeIntoMaterializeSourceReason.NOT_MATERIALIZED_NONE.toString) } withSQLConf(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE.key -> "all") { // With "all"", it should materialize, even though Delta is deterministic. checkSourceMaterialization( "delta", reason = MergeIntoMaterializeSourceReason.MATERIALIZE_ALL.toString) } } test("materialize source for non-deterministic source queries - udf") { { val targetSchema = StructType(Array( StructField("id", IntegerType, nullable = false), StructField("value", IntegerType, nullable = true))) val targetData = Seq( Row(1, 0), Row(2, 0), Row(3, 0)) val sourceData = Seq(1, 3).toDF("id") withSQLConf(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE.key -> "auto") { withTable("target", "source") { val targetRdd = spark.sparkContext.parallelize(targetData) val targetDf = spark.createDataFrame(targetRdd, targetSchema) targetDf.write.format("delta").mode("overwrite").saveAsTable("target") val targetTable = io.delta.tables.DeltaTable.forName("target") sourceData.write.format("delta").mode("overwrite").saveAsTable("source") val f = udf { () => 1L } val sourceDf = spark.table("source").withColumn("value", f()) val events: Seq[UsageRecord] = Log4jUsageLogger.track { targetTable .merge(sourceDf, col("target.id") === sourceDf("id")) .whenMatched(col("target.value") > sourceDf("value")).delete() .whenMatched().updateAll() .whenNotMatched().insertAll() .execute() } val materializeReason = mergeSourceMaterializeReason(events) assert(materializeReason == MergeIntoMaterializeSourceReason. NON_DETERMINISTIC_SOURCE_WITH_DETERMINISTIC_UDF.toString, "Source has a udf and merge should have materialized the source.") } } } } test("materialize source for non-deterministic source queries - rand expr") { val targetSchema = StructType(Array( StructField("id", IntegerType, nullable = false), StructField("value", FloatType, nullable = true))) val targetData = Seq( Row(1, 0.5f), Row(2, 0.3f), Row(3, 0.8f)) val sourceData = Seq(1, 3).toDF("id") withSQLConf(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE.key -> "auto") { def executeMerge(sourceDf: DataFrame): Unit = { val nonDeterministicSourceDf = sourceDf.withColumn("value", rand()) withTable("target") { val targetRdd = spark.sparkContext.parallelize(targetData) val targetDf = spark.createDataFrame(targetRdd, targetSchema) targetDf.write.format("delta").mode("overwrite").saveAsTable("target") val targetTable = io.delta.tables.DeltaTable.forName("target") val events: Seq[UsageRecord] = Log4jUsageLogger.track { targetTable .merge(nonDeterministicSourceDf, col("target.id") === nonDeterministicSourceDf("id")) .whenMatched(col("target.value") > nonDeterministicSourceDf("value")).delete() .whenMatched().updateAll() .whenNotMatched().insertAll() .execute() } val materializeReason = mergeSourceMaterializeReason(events) assert(materializeReason == MergeIntoMaterializeSourceReason.NON_DETERMINISTIC_SOURCE_OPERATORS.toString, "Source has non deterministic operations and should have materialized source.") } } // Test once by name and once using path, as they produce different plans. withTable("source") { sourceData.write.format("delta").saveAsTable("source") val sourceDf = spark.read.format("delta").table("source") executeMerge(sourceDf) } withTempPath { sourcePath => sourceData.write.format("delta").save(sourcePath.toString) val sourceDf = spark.read.format("delta").load(sourcePath.toString) executeMerge(sourceDf) } } } test("don't materialize source for deterministic source queries with current_date") { val targetSchema = StructType(Array( StructField("id", IntegerType, nullable = false), StructField("date", DateType, nullable = true))) val targetData = Seq( Row(1, java.sql.Date.valueOf("2022-01-01")), Row(2, java.sql.Date.valueOf("2022-02-01")), Row(3, java.sql.Date.valueOf("2022-03-01"))) val sourceData = Seq(1, 3).toDF("id") withSQLConf(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE.key -> "auto") { def executeMerge(sourceDf: DataFrame): Unit = { val nonDeterministicSourceDf = sourceDf.withColumn("date", current_date()) withTable("target") { val targetRdd = spark.sparkContext.parallelize(targetData) val targetDf = spark.createDataFrame(targetRdd, targetSchema) targetDf.write.format("delta").mode("overwrite").saveAsTable("target") val targetTable = io.delta.tables.DeltaTable.forName("target") val events: Seq[UsageRecord] = Log4jUsageLogger.track { targetTable .merge(nonDeterministicSourceDf, col("target.id") === nonDeterministicSourceDf("id")) .whenMatched(col("target.date") < nonDeterministicSourceDf("date")).delete() .whenMatched().updateAll() .whenNotMatched().insertAll() .execute() } val materializeReason = mergeSourceMaterializeReason(events) assert(materializeReason == MergeIntoMaterializeSourceReason.NOT_MATERIALIZED_AUTO.toString, "Source query is deterministic and should not be materialized.") } } // Test once by name and once using path, as they produce different plans. withTable("source") { sourceData.write.format("delta").saveAsTable("source") val sourceDf = spark.read.format("delta").table("source") executeMerge(sourceDf) } withTempPath { sourcePath => sourceData.write.format("delta").save(sourcePath.toString) val sourceDf = spark.read.format("delta").load(sourcePath.toString) executeMerge(sourceDf) } } } test("materialize source for non-deterministic source queries - subquery") { val sourceDataFrame = spark.range(0, 10) .toDF("id") .withColumn("value", rand()) val targetDataFrame = spark.range(0, 5) .toDF("id") .withColumn("value", rand()) withSQLConf(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE.key -> "auto") { // Return MergeIntoMaterializeSourceReason def executeMerge(sourceDf: DataFrame, clue: String): Unit = { withTable("target") { targetDataFrame.write .format("delta") .saveAsTable("target") val targetTable = io.delta.tables.DeltaTable.forName("target") val events: Seq[UsageRecord] = Log4jUsageLogger.track { targetTable.merge(sourceDf, col("target.id") === sourceDf("id")) .whenMatched(col("target.value") > sourceDf("value")).delete() .whenMatched().updateAll() .whenNotMatched().insertAll() .execute() } val materializeReason = mergeSourceMaterializeReason(events) assert(materializeReason == MergeIntoMaterializeSourceReason.NON_DETERMINISTIC_SOURCE_OPERATORS.toString, s"Source query has non deterministic subqueries and should materialize ($clue).") } } def checkSubquery(from: String, subquery: String): Unit = { // check subquery in filter val sourceDfFilterSubquery = spark.sql( s""" |SELECT id, 0.5 AS value |FROM $from WHERE id IN ($subquery) |""".stripMargin) executeMerge(sourceDfFilterSubquery, s"reading from `$from`, subquery `$subquery` in filter") // check subquery in project val sourceDfProjectSubquery = spark.sql( s""" |SELECT CASE WHEN id IN ($subquery) THEN id ELSE -1 END AS id, 0.5 AS value |FROM $from |""".stripMargin) executeMerge(sourceDfProjectSubquery, s"reading from `$from`, subquery `$subquery` in project") } def checkSubqueries(from: String): Unit = { // check non-deterministic plan checkSubquery(from, s"SELECT id FROM $from WHERE id < rand() * 10") // check too complex plan in subquery, even though plan.deterministic is true val subqueryComplex = s"SELECT A.id kk FROM $from A JOIN $from B ON A.id = B.id" assert(spark.sql(subqueryComplex).queryExecution.analyzed.deterministic, "We want the subquery plan to be deterministic for this test.") checkSubquery(from, subqueryComplex) // check nested subquery val subqueryNestedFilter = s"SELECT id AS kk FROM $from WHERE id IN ($subqueryComplex)" checkSubquery(from, subqueryNestedFilter) val subqueryNestedProject = s"SELECT CASE WHEN id IN ($subqueryComplex) THEN id ELSE -1 END AS kk FROM $from" checkSubquery(from, subqueryNestedProject) // check correlated subquery val subqueryCorrelated = s"SELECT kk FROM (SELECT id AS kk from $from) WHERE kk = id" checkSubquery(from, subqueryCorrelated) } // Test once by name and once using path, as they produce different plans. withTable("source") { sourceDataFrame.write.format("delta").saveAsTable("source") checkSubqueries("source") } withTempPath { sourcePath => sourceDataFrame.write.format("delta").save(sourcePath.toString) checkSubqueries(s"delta.`${sourcePath.toString}`") } } } test("don't materialize insert only merge") { val tblName = "mergeTarget" withTable(tblName) { val targetDF = spark.range(100).toDF("id") targetDF.write.format("delta").saveAsTable(tblName) spark.range(90, 120).toDF("id").createOrReplaceTempView("s") val mergeQuery = s"MERGE INTO $tblName t USING s ON t.id = s.id WHEN NOT MATCHED THEN INSERT *" val events: Seq[UsageRecord] = Log4jUsageLogger.track { withSQLConf(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE.key -> "auto") { sql(mergeQuery) } } assert(mergeSourceMaterializeReason(events) == MergeIntoMaterializeSourceReason.NOT_MATERIALIZED_AUTO_INSERT_ONLY.toString) checkAnswer( spark.read.format("delta").table(tblName), (0 until 120).map(i => Row(i.toLong))) } } test("don't unpersist locally checkpointed RDDs") { val tblName = "mergeTarget" withTable(tblName) { val targetDF = Seq( ("2023-01-01", "trade1", 100.0, "buy", "user1", "2023-01-01 10:00:00"), ("2023-01-02", "trade2", 200.0, "sell", "user2", "2023-01-02 11:00:00") ).toDF("block_date", "unique_trade_id", "transaction_amount", "transaction_type", "user_id", "timestamp") targetDF.write.format("delta").saveAsTable(tblName) Seq( ("2023-01-01", "trade1", 150.0, "buy", "user1_updated", "2023-01-01 12:00:00"), ("2023-01-03", "trade3", 300.0, "buy", "user3", "2023-01-03 10:00:00") ).toDF("block_date", "unique_trade_id", "transaction_amount", "transaction_type", "user_id", "timestamp").createOrReplaceTempView("s") val mergeQuery = s"""MERGE INTO $tblName t USING s |ON t.block_date = s.block_date AND t.unique_trade_id = s.unique_trade_id |WHEN MATCHED THEN UPDATE SET * |WHEN NOT MATCHED THEN INSERT *""".stripMargin Log4jUsageLogger.track { withSQLConf(DeltaSQLConf.MERGE_MATERIALIZE_SOURCE.key -> "auto") { sql(mergeQuery) } } // Check if the source RDDs have been locally checkpointed and not unpersisted assert(sparkContext.getPersistentRDDs.values.nonEmpty, "Source RDDs" + " should be locally checkpointed") checkAnswer( spark.read.format("delta").table(tblName), Seq( Row("2023-01-01", "trade1", 150.0, "buy", "user1_updated", "2023-01-01 12:00:00"), Row("2023-01-02", "trade2", 200.0, "sell", "user2", "2023-01-02 11:00:00"), Row("2023-01-03", "trade3", 300.0, "buy", "user3", "2023-01-03 10:00:00")) ) } } private def mergeStats(events: Seq[UsageRecord]): MergeStats = { val mergeStats = events.filter { e => e.metric == MetricDefinitions.EVENT_TAHOE.name && e.tags.get("opType").contains("delta.dml.merge.stats") } assert(mergeStats.size == 1) JsonUtils.fromJson[MergeStats](mergeStats.head.blob) } private def mergeSourceMaterializeReason(events: Seq[UsageRecord]): String = { val stats = mergeStats(events) assert(stats.materializeSourceReason.isDefined) stats.materializeSourceReason.get } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoMetricsBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.{DataFrame, QueryTest, Row} import org.apache.spark.sql.functions._ import org.apache.spark.sql.functions.expr import org.apache.spark.sql.test.SharedSparkSession /** * Tests for the metrics of MERGE INTO command in Delta log. * * This test suite checks the values of metrics that are emitted in Delta log by MERGE INTO command, * with Changed Data Feed (CDF) enabled/disabled. * * Metrics related with number of affected rows are deterministic and so the expected values are * explicitly checked. Metrics related with number of affected files and execution times are not * deterministic, and so we check only their presence and some invariants. * */ trait MergeIntoMetricsBase extends QueryTest with SharedSparkSession { self: DescribeDeltaHistorySuiteBase => import MergeIntoMetricsBase._ import testImplicits._ /////////////////////// // container classes // /////////////////////// private case class MergeTestConfiguration(partitioned: Boolean, cdfEnabled: Boolean) { /** Return a [[MetricValue]] for this config with the provided default value. */ def metricValue(defaultValue: Int): MetricValue = { new MetricValue(this, defaultValue) } } /** * Helper class to compute values of metrics that depend on the configuration. * * Objects are initialized with a test configuration and a default value. The value can then be * overwritten with helper methods that check the test config and value() can be called to * retrieve the final expected value for a test. */ private class MetricValue(testConfig: MergeTestConfiguration, defaultValue: Int) { private var currentValue: Int = defaultValue def value: Int = currentValue // e.g. ifCDF } //////////////// // test utils // //////////////// val testsToIgnore = Seq( // The below tests fail due to incorrect numTargetRowsCopied metric. "delete-only with condition", "delete-only with update with unsatisfied condition", "delete-only with unsatisfied condition", "delete-only with target-only condition", "delete-only with source-only condition", "match-only with unsatisfied condition" ) // Helper to generate tests with different configurations. private def testMergeMetrics(name: String)(testFn: MergeTestConfiguration => Unit): Unit = { for { partitioned <- Seq(true, false) cdfEnabled <- Seq(true, false) } { val testConfig = MergeTestConfiguration(partitioned = partitioned, cdfEnabled = cdfEnabled) val testName = s"merge-metrics: $name - Partitioned = $partitioned, CDF = $cdfEnabled" if (testsToIgnore.contains(name)) { // Currently multiple metrics are wrong for Merge. We have added tests for these scenarios // but we need to ignore the failing tests until the metrics are fixed. ignore(testName) { testFn(testConfig) } } else { test(testName) { testFn(testConfig) } } } } /** * Check invariants for row metrics of MERGE INTO command. * * @param metrics The merge operation metrics from the Delta history. */ private def checkMergeOperationRowMetricsInvariants(metrics: Map[String, String]): Unit = { assert( metrics("numTargetRowsUpdated").toLong === metrics("numTargetRowsMatchedUpdated").toLong + metrics("numTargetRowsNotMatchedBySourceUpdated").toLong) assert( metrics("numTargetRowsDeleted").toLong === metrics("numTargetRowsMatchedDeleted").toLong + metrics("numTargetRowsNotMatchedBySourceDeleted").toLong) } /** * Check invariants for file metrics of MERGE INTO command. * * @param metrics The merge operation metrics from the Delta history. */ private def checkMergeOperationFileMetricsInvariants(metrics: Map[String, String]): Unit = { // numTargetFilesAdded should have a positive value if rows were added and be zero // otherwise. { val numFilesAdded = metrics("numTargetFilesAdded").toLong val numBytesAdded = metrics("numTargetBytesAdded").toLong val numRowsWritten = metrics("numTargetRowsInserted").toLong + metrics("numTargetRowsUpdated").toLong + metrics("numTargetRowsCopied").toLong lazy val assertMsgNumFiles = { val expectedNumFilesAdded = if (numRowsWritten == 0) "0" else s"between 1 and $numRowsWritten" s"""Unexpected value for numTargetFilesAdded metric. | Expected: $expectedNumFilesAdded | Actual: $numFilesAdded | numRowsWritten: $numRowsWritten | Metrics: ${metrics.toString} |""".stripMargin } lazy val assertMsgBytes = { val expected = if (numRowsWritten == 0) "0" else "greater than 0" s"""Unexpected value for numTargetBytesAdded metric. | Expected: $expected | Actual: $numBytesAdded | numRowsWritten: $numRowsWritten | numFilesAdded: $numFilesAdded | Metrics: ${metrics.toString} |""".stripMargin } if (numRowsWritten == 0) { assert(numFilesAdded === 0, assertMsgNumFiles) assert(numBytesAdded === 0, assertMsgBytes) } else { assert(numFilesAdded > 0 && numFilesAdded <= numRowsWritten, assertMsgNumFiles) assert(numBytesAdded > 0, assertMsgBytes) } } // numTargetFilesRemoved should have a positive value if rows were updated or deleted and be // zero otherwise. In case of classic merge we also count copied rows as changed, because if // match clauses have conditions we may end up copying rows even if no other rows are // updated/deleted. { val numFilesRemoved = metrics("numTargetFilesRemoved").toLong val numBytesRemoved = metrics("numTargetBytesRemoved").toLong val numRowsTouched = metrics("numTargetRowsDeleted").toLong + metrics("numTargetRowsUpdated").toLong + metrics("numTargetRowsCopied").toLong lazy val assertMsgNumFiles = { val expectedNumFilesRemoved = if (numRowsTouched == 0) "0" else s"between 1 and $numRowsTouched" s"""Unexpected value for numTargetFilesRemoved metric. | Expected: $expectedNumFilesRemoved | Actual: $numFilesRemoved | numRowsTouched: $numRowsTouched | Metrics: ${metrics.toString} |""".stripMargin } lazy val assertMsgBytes = { val expectedNumBytesRemoved = if (numRowsTouched == 0) "0" else "greater than 0" s"""Unexpected value for numTargetBytesRemoved metric. | Expected: $expectedNumBytesRemoved | Actual: $numBytesRemoved | numRowsTouched: $numRowsTouched | Metrics: ${metrics.toString} |""".stripMargin } if (numRowsTouched == 0) { assert(numFilesRemoved === 0, assertMsgNumFiles) assert(numBytesRemoved === 0, assertMsgBytes) } else { assert(numFilesRemoved > 0 && numFilesRemoved <= numRowsTouched, assertMsgNumFiles) assert(numBytesRemoved > 0, assertMsgBytes) } } } /** * Helper method to create a target table with the desired options, run a merge command and check * the operation metrics in the Delta history. * * For operation metrics the following checks are performed: * a) The operation metrics in Delta history must match [[DeltaOperationMetrics.MERGE]] schema, * i.e. no metrics can be missing or unknown metrics can exist. * b) All operation metrics must have a non-negative values. * c) The values of metrics that are specified in 'expectedOpMetrics' argument must match the * operation metrics. Metrics with a value of -1 are ignored, to allow callers always specify * metrics that don't exist under some configurations. * d) Row-related operation metrics that are not specified in 'expectedOpMetrics' must be zero. * e) File/Time-related operation metrics that are not specified in 'expectedOpMetrics' can have * non-zero values. These metrics are not deterministic and so this method only checks that * some invariants hold. * * @param targetDf The DataFrame to generate the target table for the merge command. * @param sourceDf The DataFrame to generate the source table for the merge command. * @param mergeCmdFn The function that actually runs the merge command. * @param expectedOpMetrics A map with values for expected operation metrics. * @param testConfig The configuration options for this test * @param overrideExpectedOpMetrics Sequence of expected operation metric values to override from * those provided in expectedOpMetrics for specific * configurations of partitioned and cdfEnabled. Elements * provided as: * ((partitioned, cdfEnabled), (metric_name, metric_value)) */ private def runMergeCmdAndTestMetrics( targetDf: DataFrame, sourceDf: DataFrame, mergeCmdFn: MergeCmd, expectedOpMetrics: Map[String, Int], testConfig: MergeTestConfiguration, overrideExpectedOpMetrics: Seq[((Boolean, Boolean), (String, Int))] = Seq.empty ): Unit = { withSQLConf( DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true", DeltaSQLConf.DELTA_SKIP_RECORDING_EMPTY_COMMITS.key -> "false", DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> testConfig.cdfEnabled.toString ) { withTempDir { tempDir => def addExtraColumns(tableDf: DataFrame): DataFrame = { // Add a column to be used for data partitioning and one extra column for filters in // queries. val numRows = tableDf.count() val numPartitions = tableDf.rdd.getNumPartitions val numRowsPerPart = if (numRows > 0) numRows / numPartitions else 1 tableDf.withColumn("partCol", expr(s"floor(id / $numRowsPerPart)")) .withColumn("extraCol", expr(s"$numRows - id")) } // Add extra columns and create target table. val tempPath = tempDir.getAbsolutePath val partitionBy = if (testConfig.partitioned) Seq("partCol") else Seq() val targetDfWithExtraCols = addExtraColumns(targetDf) targetDfWithExtraCols .write .partitionBy(partitionBy: _*) .format("delta") .save(tempPath) val targetTable = io.delta.tables.DeltaTable.forPath(tempPath) // Also add extra columns in source to be able to call updateAll()/insertAll(). val sourceDfWithExtraCols = addExtraColumns(sourceDf) // Run MERGE INTO command val mergeResultDf = mergeCmdFn(targetTable, sourceDfWithExtraCols) checkMergeResultMetrics(mergeResultDf, expectedOpMetrics) // Query the operation metrics from the Delta log history. val operationMetrics: Map[String, String] = getOperationMetrics(targetTable.history(1)) // Get the default row operation metrics and override them with the provided ones. val metricsWithDefaultZeroValue = mergeRowMetrics.map(_ -> "0").toMap var expectedOpMetricsWithDefaults = metricsWithDefaultZeroValue ++ expectedOpMetrics.filter(m => m._2 >= 0).mapValues(_.toString) overrideExpectedOpMetrics.foreach { case ((partitioned, cdfEnabled), (metric, value)) => if (partitioned == testConfig.partitioned && cdfEnabled == testConfig.cdfEnabled) { expectedOpMetricsWithDefaults = expectedOpMetricsWithDefaults + (metric -> value.toString) } } // Check that all operation metrics are positive numbers. for ((metricName, metricValue) <- operationMetrics) { assert(metricValue.toLong >= 0, s"Invalid negative value for metric $metricName = $metricValue") } // Check that operation metrics match the schema and that values match the expected ones. checkOperationMetrics( expectedOpMetricsWithDefaults, operationMetrics, DeltaOperationMetrics.MERGE ) // Check row metrics invariants. checkMergeOperationRowMetricsInvariants(operationMetrics) // Check file metrics invariants. checkMergeOperationFileMetricsInvariants(operationMetrics) // Check time metrics invariants. checkOperationTimeMetricsInvariant(mergeTimeMetrics, operationMetrics) // Check CDF metrics invariants. checkMergeOperationCdfMetricsInvariants(operationMetrics, testConfig.cdfEnabled) } } } private def checkMergeResultMetrics( mergeResultDf: DataFrame, metrics: Map[String, Int]): Unit = { val numRowsUpdated = metrics.get("numTargetRowsUpdated").map(_.toLong).getOrElse(0L) val numRowsDeleted = metrics.get("numTargetRowsDeleted").map(_.toLong).getOrElse(0L) val numRowsInserted = metrics.get("numTargetRowsInserted").map(_.toLong).getOrElse(0L) val numRowsTouched = numRowsUpdated + numRowsDeleted + numRowsInserted assert(mergeResultDf.collect() === Array(Row(numRowsTouched, numRowsUpdated, numRowsDeleted, numRowsInserted))) } ///////////////////////////// // insert-only merge tests // ///////////////////////////// testMergeMetrics("insert-only") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 50, end = 150, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable.as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenNotMatched() .insertAll() .execute() } val expectedOpMetrics = Map( "numSourceRows" -> 100, "numOutputRows" -> 50, "numTargetRowsInserted" -> 50 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("insert-only with skipping") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 100, end = 200, step = 1, numPartitions = 5).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable.as("t") .merge(sourceDf.as("s"), "s.id = t.id and t.partCol >= 2") .whenNotMatched() .insertAll() .execute() } val expectedOpMetrics = Map( "numSourceRows" -> 100, "numOutputRows" -> 100, "numTargetRowsInserted" -> 100 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("insert-only with condition") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 50, end = 150, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenNotMatched("s.id >= 125") .insertAll() .execute() } val expectedOpMetrics = Map( "numSourceRows" -> 100, "numOutputRows" -> 25, "numTargetRowsInserted" -> 25 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("insert-only when all rows match") { testConfig => { val targetDf = spark.range(start = 0, end = 200, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 50, end = 150, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenNotMatched() .insertAll() .execute() } val expectedOpMetrics = Map( "numSourceRows" -> 100 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("insert-only with unsatisfied condition") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 50, end = 150, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenNotMatched("s.id > 150") .insertAll() .execute() } val expectedOpMetrics = Map( "numSourceRows" -> 100 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("insert-only with empty source") { testConfig => { val targetDf = spark.range(start = 0, end = 200, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(0).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenNotMatched() .insertAll() .execute() } val expectedOpMetrics = Map( "numSourceRows" -> 0 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("insert-only with empty target") { testConfig => { val targetDf = spark.range(0).toDF() val sourceDf = spark.range(start = 50, end = 150, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenNotMatched() .insertAll() .execute() } val expectedOpMetrics = Map( "numSourceRows" -> 100, "numOutputRows" -> 100, "numTargetRowsInserted" -> 100 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("insert-only with disjoint tables") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 100, end = 200, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenMatched() .updateAll() .whenNotMatched() .insertAll() .execute() } val expectedOpMetrics = Map( "numSourceRows" -> 100, "numOutputRows" -> 100, "numTargetRowsInserted" -> 100 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("insert-only with update/delete with unsatisfied conditions") { testConfig => { val targetDf = spark.range(start = 0, end = 50, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 0, end = 150, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenMatched("s.id + t.id > 200") .updateAll() .whenMatched("s.id + t.id < 0") .delete() .whenNotMatched() .insertAll() .execute() } // In classic merge we are copying all rows from job1. val expectedOpMetrics = Map( "numSourceRows" -> 150, "numOutputRows" -> 150, "numTargetRowsInserted" -> 100, "numTargetRowsCopied" -> 50, "numTargetFilesRemoved" -> 5 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} ///////////////////////////// // delete-only merge tests // ///////////////////////////// testMergeMetrics("delete-only") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 50, end = 150, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenMatched() .delete() .execute() } val expectedOpMetrics = Map[String, Int]( "numSourceRows" -> 100, "numTargetRowsDeleted" -> 50, "numTargetRowsMatchedDeleted" -> 50, "numTargetRowsRemoved" -> -1, "numOutputRows" -> 10, "numTargetRowsCopied" -> 10, "numTargetFilesAdded" -> 1, "numTargetFilesRemoved" -> -1 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("delete-only with skipping") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 50, end = 150, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id and t.partCol >= 2") .whenMatched() .delete() .execute() } val expectedOpMetrics = Map[String, Int]( "numSourceRows" -> 100, "numOutputRows" -> 10, "numTargetRowsCopied" -> 10, "numTargetRowsDeleted" -> 50, "numTargetRowsMatchedDeleted" -> 50, "numTargetRowsRemoved" -> -1, "numTargetFilesAdded" -> 1, "numTargetFilesRemoved" -> 3 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("delete-only with disjoint tables") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 100, end = 200, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenMatched() .delete() .execute() } val expectedOpMetrics = Map( "numSourceRows" -> 100, "numTargetFilesAdded" -> 0, "numTargetFilesRemoved" -> 0 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("delete-only delete all rows") { testConfig => { val targetDf = spark.range(start = 100, end = 200, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 0, end = 300, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenMatched() .delete() .execute() } val expectedOpMetrics = Map[String, Int]( "numSourceRows" -> 300, "numOutputRows" -> 0, "numTargetRowsCopied" -> 0, "numTargetRowsDeleted" -> 100, "numTargetRowsMatchedDeleted" -> 100, "numTargetRowsRemoved" -> -1, "numTargetFilesAdded" -> 0, "numTargetFilesRemoved" -> 5 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("delete-only with condition") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 0, end = 150, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenMatched("s.id + t.id < 50") .delete() .execute() } val expectedOpMetrics = Map[String, Int]( "numSourceRows" -> 150, "numOutputRows" -> 15, "numTargetRowsCopied" -> 15, "numTargetRowsDeleted" -> 25, "numTargetRowsMatchedDeleted" -> 25, "numTargetRowsRemoved" -> -1, "numTargetFilesAdded" -> 1, "numTargetFilesRemoved" -> 2 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("delete-only with update with unsatisfied condition") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenMatched("s.id + t.id > 1000") .updateAll() .whenMatched("s.id + t.id < 50") .delete() .execute() } // In case of partitioned tables, files are mixed-in even though finally there are no matches. val expectedOpMetrics = Map[String, Int]( "numSourceRows" -> 100, "numOutputRows" -> 15, "numTargetRowsCopied" -> 15, "numTargetRowsDeleted" -> 25, "numTargetRowsMatchedDeleted" -> 25, "numTargetRowsRemoved" -> -1, "numTargetFilesAdded" -> 1, "numTargetFilesRemoved" -> 2 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("delete-only with condition on delete and insert with no matching rows") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenMatched("s.id + t.id < 50") .delete() .whenNotMatched() .insertAll() .execute() } // In classic merge we are copying all rows from job1. // In case of partitioned tables, files are mixed-in even though finally there are no matches. val expectedOpMetrics = Map[String, Int]( "numSourceRows" -> 100, "numOutputRows" -> 75, "numTargetRowsCopied" -> 75, "numTargetRowsDeleted" -> 25, "numTargetRowsMatchedDeleted" -> 25, "numTargetRowsRemoved" -> -1, "numTargetFilesRemoved" -> 5 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) } } testMergeMetrics("delete-only with unsatisfied condition") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 0, end = 150, step = 1, numPartitions = 15).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenMatched("s.id + t.id > 1000") .delete() .execute() } val expectedOpMetrics = Map[String, Int]( "numSourceRows" -> 150, "numTargetFilesAdded" -> 0, "numTargetFilesRemoved" -> 0 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("delete-only with target-only condition") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 0, end = 150, step = 1, numPartitions = 15).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenMatched("t.id >= 45") .delete() .execute() } val expectedOpMetrics = Map[String, Int]( "numSourceRows" -> 150, "numOutputRows" -> 5, "numTargetRowsCopied" -> 5, "numTargetRowsDeleted" -> 55, "numTargetRowsMatchedDeleted" -> 55, "numTargetRowsRemoved" -> -1, "numTargetFilesAdded" -> 1, "numTargetFilesRemoved" -> 3 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("delete-only with source-only condition") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 50, end = 150, step = 1, numPartitions = 100).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenMatched("s.id >= 70") .delete() .execute() } val expectedOpMetrics = Map[String, Int]( "numSourceRows" -> 100, "numOutputRows" -> 10, "numTargetRowsCopied" -> 10, "numTargetRowsDeleted" -> 30, "numTargetRowsMatchedDeleted" -> 30, "numTargetRowsRemoved" -> -1, "numTargetFilesAdded" -> 1, "numTargetFilesRemoved" -> 2 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("delete-only with empty source") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 4).toDF() val sourceDf = spark.range(0).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenMatched("t.id > 25") .delete() .execute() } val expectedOpMetrics = Map( "numSourceRows" -> 0, "numTargetFilesAdded" -> 0, "numTargetFilesRemoved" -> 0 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("delete-only with empty target") { testConfig => { val targetDf = spark.range(0).toDF() val sourceDf = spark.range(start = 50, end = 150, step = 1, numPartitions = 3).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenMatched() .delete() .execute() } val expectedOpMetrics = Map( // This actually goes through a special code path in MERGE because the optimizer optimizes // away the join to the source table entirely if the target table is empty. "numSourceRows" -> 100, "numTargetFilesAdded" -> 0, "numTargetFilesRemoved" -> 0 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("delete-only without join empty source") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(0).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "t.id >= 50") .whenMatched() .delete() .execute() } val expectedOpMetrics = Map[String, Int]( "numSourceRows" -> 0, "numTargetFilesAdded" -> 0, "numTargetFilesRemoved" -> 0 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("delete-only without join with source with 1 row") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 0, end = 1, step = 1, numPartitions = 1).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "t.id >= 50") .whenMatched() .delete() .execute() } val expectedOpMetrics = Map[String, Int]( "numSourceRows" -> 1, "numOutputRows" -> 10, "numTargetRowsCopied" -> 10, "numTargetRowsDeleted" -> 50, "numTargetRowsMatchedDeleted" -> 50, "numTargetRowsRemoved" -> -1, "numTargetFilesAdded" -> 1, "numTargetFilesRemoved" -> 3 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("delete-only without join") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 0, end = 200, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "t.id >= 50") .whenMatched() .delete() .execute() } val expectedOpMetrics = Map[String, Int]( "numSourceRows" -> 200, "numOutputRows" -> 10, "numTargetRowsCopied" -> 10, "numTargetRowsDeleted" -> 50, "numTargetRowsMatchedDeleted" -> 50, "numTargetRowsRemoved" -> -1, "numTargetFilesAdded" -> 1, "numTargetFilesRemoved" -> 3 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("delete-only with duplicates") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() // This will cause duplicates due to rounding. val sourceDf = spark .range(start = 50, end = 150, step = 1, numPartitions = 2) .toDF() .select(floor($"id" / 2).as("id")) val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenMatched() .delete() .execute() } val expectedOpMetrics = Map[String, Int]( "numSourceRows" -> 100, "numOutputRows" -> 10, "numTargetRowsDeleted" -> 50, "numTargetRowsMatchedDeleted" -> 50, "numTargetRowsRemoved" -> -1, "numTargetRowsCopied" -> 10, "numTargetFilesAdded" -> 2, "numTargetFilesRemoved" -> 3 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig, // When cdf=true in this test we hit the corner case where there are duplicate matches with a // delete clause and we generate duplicate cdc data. This is further detailed in // MergeIntoCommand at the definition of isDeleteWithDuplicateMatchesAndCdc. Our fix for this // scenario includes deduplicating the output data which reshuffles the output data. // Thus when the table is not partitioned, the data is rewritten into 1 new file rather than 2 overrideExpectedOpMetrics = Seq( ((false, true), ("numTargetFilesAdded", 1)), ((false, false), ( "numTargetFilesAdded", 1) ) ) ) }} ///////////////////////////// // match-only merge tests // ///////////////////////////// testMergeMetrics("match-only") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 50, end = 150, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenMatched() .updateAll() .execute() } val expectedOpMetrics = Map( "numSourceRows" -> 100, "numOutputRows" -> 60, "numTargetRowsUpdated" -> 50, "numTargetRowsMatchedUpdated" -> 50, "numTargetRowsCopied" -> 10, "numTargetFilesRemoved" -> 3 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("match-only with skipping") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 50, end = 150, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id and t.partCol >= 2") .whenMatched() .updateAll() .execute() } val expectedOpMetrics = Map( "numSourceRows" -> 100, "numOutputRows" -> 60, "numTargetRowsUpdated" -> 50, "numTargetRowsMatchedUpdated" -> 50, "numTargetRowsCopied" -> 10, "numTargetFilesRemoved" -> 3 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("match-only with update/delete with unsatisfied conditions") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 50, end = 150, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenMatched("s.id + t.id > 1000") .delete() .whenMatched("s.id + t.id < 1000") .updateAll() .whenNotMatched("s.id > 1000") .insertAll() .execute() } val expectedOpMetrics = Map( "numSourceRows" -> 100, "numOutputRows" -> 60, "numTargetRowsUpdated" -> 50, "numTargetRowsMatchedUpdated" -> 50, "numTargetRowsCopied" -> 10, "numTargetFilesRemoved" -> 3 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("match-only with unsatisfied condition") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 50, end = 150, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenMatched("s.id + t.id > 1000") .updateAll() .execute() } val expectedOpMetrics = Map( "numSourceRows" -> 100 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} ///////////////////////////////////////////// // not matched by source only merge tests // ///////////////////////////////////////////// testMergeMetrics("not matched by source update only") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 50, end = 150, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenNotMatchedBySource("t.id < 20") .updateExpr(Map("t.extraCol" -> "t.extraCol + 1")) .execute() } val expectedOpMetrics = Map[String, Int]( "numSourceRows" -> 100, "numOutputRows" -> 100, "numTargetRowsUpdated" -> 20, "numTargetRowsNotMatchedBySourceUpdated" -> 20, "numTargetRowsCopied" -> 80, "numTargetFilesRemoved" -> 5 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} ///////////////////////////// // full merge tests // ///////////////////////////// testMergeMetrics("upsert") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 50, end = 150, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenMatched() .updateAll() .whenNotMatched() .insertAll() .execute() } val expectedOpMetrics = Map( "numSourceRows" -> 100, "numOutputRows" -> 110, "numTargetRowsInserted" -> 50, "numTargetRowsUpdated" -> 50, "numTargetRowsMatchedUpdated" -> 50, "numTargetRowsCopied" -> 10, "numTargetFilesRemoved" -> 3 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("replace target with source") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 50, end = 150, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenMatched() .updateAll() .whenNotMatched() .insertAll() .whenNotMatchedBySource() .delete() .execute() } val expectedOpMetrics = Map( "numSourceRows" -> 100, "numOutputRows" -> 100, "numTargetRowsInserted" -> 50, "numTargetRowsUpdated" -> 50, "numTargetRowsMatchedUpdated" -> 50, "numTargetRowsDeleted" -> 50, "numTargetRowsNotMatchedBySourceDeleted" -> 50, "numTargetRowsCopied" -> 0, "numTargetFilesRemoved" -> 5 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics("upsert and delete with conditions") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 10).toDF() val sourceDf = spark.range(start = 50, end = 150, step = 1, numPartitions = 3).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenMatched("t.id >= 55 and t.id < 60") .updateAll() .whenMatched("t.id < 70") .delete() .whenNotMatched() .insertAll() .whenNotMatchedBySource("t.id < 10") .updateExpr(Map("t.extraCol" -> "t.extraCol + 1")) .whenNotMatchedBySource("t.id >= 45") .delete() .execute() } val expectedOpMetrics = Map( "numSourceRows" -> 100, "numOutputRows" -> 130, "numTargetRowsInserted" -> 50, "numTargetRowsUpdated" -> 15, "numTargetRowsMatchedUpdated" -> 5, "numTargetRowsNotMatchedBySourceUpdated" -> 10, "numTargetRowsDeleted" -> 20, "numTargetRowsMatchedDeleted" -> 15, "numTargetRowsNotMatchedBySourceDeleted" -> 5, "numTargetRowsCopied" -> 65, "numTargetFilesRemoved" -> 10 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} testMergeMetrics( "update/delete/insert with some unsatisfied conditions") { testConfig => { val targetDf = spark.range(start = 0, end = 100, step = 1, numPartitions = 5).toDF() val sourceDf = spark.range(start = 50, end = 150, step = 1, numPartitions = 10).toDF() val mergeCmdFn: MergeCmd = (targetTable, sourceDf) => { targetTable .as("t") .merge(sourceDf.as("s"), "s.id = t.id") .whenMatched("s.id + t.id > 1000") .delete() .whenNotMatchedBySource("t.id > 1000") .delete() .whenNotMatchedBySource("t.id < 1000") .updateExpr(Map("t.extraCol" -> "t.extraCol + 1")) .whenNotMatched("s.id > 1000") .insertAll() .execute() } val expectedOpMetrics = Map[String, Int]( "numSourceRows" -> 100, "numOutputRows" -> 100, "numTargetRowsUpdated" -> 50, "numTargetRowsNotMatchedBySourceUpdated" -> 50, "numTargetRowsCopied" -> 50, "numTargetFilesRemoved" -> 5 ) runMergeCmdAndTestMetrics( targetDf = targetDf, sourceDf = sourceDf, mergeCmdFn = mergeCmdFn, expectedOpMetrics = expectedOpMetrics, testConfig = testConfig ) }} } object MergeIntoMetricsBase extends QueryTest with SharedSparkSession { /////////////////////// // helpful constants // /////////////////////// // Metrics related with affected number of rows. Values should always be deterministic. val mergeRowMetrics = Set( "numSourceRows", "numTargetRowsInserted", "numTargetRowsUpdated", "numTargetRowsMatchedUpdated", "numTargetRowsNotMatchedBySourceUpdated", "numTargetRowsDeleted", "numTargetRowsMatchedDeleted", "numTargetRowsNotMatchedBySourceDeleted", "numTargetRowsCopied", "numOutputRows" ) // Metrics related with affected number of files. Values depend on the file layout. val mergeFileMetrics = Set( "numTargetFilesAdded", "numTargetFilesRemoved", "numTargetBytesAdded", "numTargetBytesRemoved") // Metrics related with execution times. val mergeTimeMetrics = Set( "executionTimeMs", "materializeSourceTimeMs", "scanTimeMs", "rewriteTimeMs") // Metrics related with CDF. Available only when CDF is available. val mergeCdfMetrics = Set("numTargetChangeFilesAdded") // DV Metrics. val mergeDVMetrics = Set( "numTargetDeletionVectorsAdded", "numTargetDeletionVectorsUpdated", "numTargetDeletionVectorsRemoved") // Ensure that all metrics are properly copied here. assert( DeltaOperationMetrics.MERGE.size == mergeRowMetrics.size + mergeFileMetrics.size + mergeTimeMetrics.size + mergeCdfMetrics.size + mergeDVMetrics.size ) /////////////////// // helpful types // /////////////////// type MergeCmd = (io.delta.tables.DeltaTable, DataFrame) => DataFrame ///////////////////// // helpful methods // ///////////////////// /** * Check invariants for the CDF metrics of MERGE INTO command. Checking the actual values * is avoided since they depend on the file layout and the type of merge. * * @param metrics The merge operation metrics from the Delta history. * @param cdfEnabled Whether CDF was enabled or not. */ def checkMergeOperationCdfMetricsInvariants( metrics: Map[String, String], cdfEnabled: Boolean): Unit = { val numRowsUpdated = metrics("numTargetRowsUpdated").toLong val numRowsDeleted = metrics("numTargetRowsDeleted").toLong val numRowsInserted = metrics("numTargetRowsInserted").toLong val numRowsChanged = numRowsUpdated + numRowsDeleted + numRowsInserted val numTargetChangeFilesAdded = metrics("numTargetChangeFilesAdded").toLong lazy val assertMsg = s"""Unexpected value for numTargetChangeFilesAdded metric: | Expected : ${if (numRowsChanged == 0) 0 else "Positive integer value"} | Actual : $numTargetChangeFilesAdded | cdfEnabled: $cdfEnabled | numRowsChanged: $numRowsChanged | Metrics: ${metrics.toString} |""".stripMargin if (!cdfEnabled || numRowsChanged == 0) { assert(numTargetChangeFilesAdded === 0, assertMsg) } else { // In case of insert-only merges where only new files are added, CDF data are not required // since the CDF reader can read the corresponding added files. However, there are cases // where we produce CDF data even in insert-only merges (see 'insert-only-dynamic-predicate' // testcase for an example). Here we skip the assertion, since both behaviours can be // considered valid. val isInsertOnly = numRowsInserted > 0 && numRowsChanged == numRowsInserted if (!isInsertOnly) { assert(numTargetChangeFilesAdded > 0, assertMsg) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoNotMatchedBySourceSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.Row trait MergeIntoNotMatchedBySourceWithCDCMixin extends MergeIntoSuiteBaseMixin { import testImplicits._ /** * Variant of `testExtendedMerge` that runs a MERGE INTO command, checks the expected result and * additionally validate that the CDC produced is correct. */ protected def testExtendedMergeWithCDC( name: String, namePrefix: String = "not matched by source")( source: Seq[(Int, Int)], target: Seq[(Int, Int)], mergeOn: String, mergeClauses: MergeClause*)( result: Seq[(Int, Int)], cdc: Seq[(Int, Int, String)]): Unit = { for { isPartitioned <- BOOLEAN_DOMAIN cdcEnabled <- BOOLEAN_DOMAIN } { test(s"$namePrefix - $name - isPartitioned: $isPartitioned - cdcEnabled: $cdcEnabled") { withSQLConf( DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> cdcEnabled.toString) { withKeyValueData(source, target, isPartitioned) { case (sourceName, targetName) => withSQLConf(DeltaSQLConf.MERGE_INSERT_ONLY_ENABLED.key -> "true") { executeMerge(s"$targetName t", s"$sourceName s", mergeOn, mergeClauses: _*) } checkAnswer(readDeltaTableByIdentifier(targetName), result.map { case (k, v) => Row(k, v) }) } if (cdcEnabled) { checkAnswer(getCDCForLatestOperation(deltaLog, DeltaOperations.OP_MERGE), cdc.toDF()) } } } } } } trait MergeIntoNotMatchedBySourceCDCPart1Tests extends MergeIntoNotMatchedBySourceWithCDCMixin { // Test correctness with NOT MATCHED BY SOURCE clauses. testExtendedMergeWithCDC("all 3 types of match clauses without conditions")( source = (0, 0) :: (1, 1) :: (5, 5) :: Nil, target = (2, 20) :: (1, 10) :: (5, 50) :: Nil, mergeOn = "s.key = t.key", update(set = "*"), insert(values = "*"), deleteNotMatched())( result = Seq( (0, 0), // No matched by target, inserted (1, 1), // Matched, updated // (2, 20) Not matched by source, deleted (5, 5) // Matched, updated ), cdc = Seq( (0, 0, "insert"), (1, 10, "update_preimage"), (1, 1, "update_postimage"), (2, 20, "delete"), (5, 50, "update_preimage"), (5, 5, "update_postimage"))) testExtendedMergeWithCDC("all 3 types of match clauses with conditions")( source = (0, 0) :: (1, 1) :: (5, 5) :: (6, 6) :: Nil, target = (1, 10) :: (2, 20) :: (5, 50) :: (7, 70) :: Nil, mergeOn = "s.key = t.key", update(set = "*", condition = "t.value < 30"), insert(values = "*", condition = "s.value < 4"), deleteNotMatched(condition = "t.value > 40"))( result = Seq( (0, 0), // Not matched by target, inserted (1, 1), // Matched, updated (2, 20), // Not matched by source, no change (5, 50) // Matched, not updated // (6, 6) Not matched by target, no change // (7, 7) Not matched by source, deleted ), cdc = Seq( (0, 0, "insert"), (1, 10, "update_preimage"), (1, 1, "update_postimage"), (7, 70, "delete"))) testExtendedMergeWithCDC("unconditional delete only when not matched by source")( source = (0, 0) :: (1, 1) :: (5, 5) :: Nil, target = (2, 20) :: (1, 10) :: (5, 50) :: (6, 60) :: Nil, mergeOn = "s.key = t.key", deleteNotMatched())( result = Seq( (1, 10), // Matched, no change // (2, 20) Not matched by source, deleted (5, 50) // Matched, no change // (6, 60) Not matched by source, deleted ), cdc = Seq((2, 20, "delete"), (6, 60, "delete"))) testExtendedMergeWithCDC("conditional delete only when not matched by source")( source = (0, 0) :: (1, 1) :: (5, 5) :: Nil, target = (1, 10) :: (2, 20) :: (5, 50) :: (6, 60) :: Nil, mergeOn = "s.key = t.key", deleteNotMatched(condition = "t.value > 40"))( result = Seq( (1, 10), // Matched, no change (2, 20), // Not matched by source, no change (5, 50) // Matched, no change // (6, 60) Not matched by source, deleted ), cdc = Seq((6, 60, "delete"))) testExtendedMergeWithCDC("delete only matched and not matched by source")( source = (1, 1) :: (2, 2) :: (5, 5) :: (6, 6) :: Nil, target = (1, 10) :: (2, 20) :: (3, 30) :: (4, 40) :: Nil, mergeOn = "s.key = t.key", delete("s.value % 2 = 0"), deleteNotMatched("t.value % 20 = 0"))( result = Seq( (1, 10), // Matched, no change // (2, 20) Matched, deleted (3, 30) // Not matched by source, no change // (4, 40) Not matched by source, deleted ), cdc = Seq((2, 20, "delete"), (4, 40, "delete"))) testExtendedMergeWithCDC("unconditionally delete matched and not matched by source")( source = (0, 0) :: (1, 1) :: (5, 5) :: (6, 6) :: Nil, target = (1, 10) :: (2, 20) :: (5, 50) :: Nil, mergeOn = "s.key = t.key", delete(), deleteNotMatched())( result = Seq.empty, cdc = Seq((1, 10, "delete"), (2, 20, "delete"), (5, 50, "delete"))) testExtendedMergeWithCDC("unconditional not matched by source update")( source = (0, 0) :: (1, 1) :: (5, 5) :: Nil, target = (1, 10) :: (2, 20) :: (4, 40) :: (5, 50) :: Nil, mergeOn = "s.key = t.key", updateNotMatched(set = "t.value = t.value + 1"))( result = Seq( (1, 10), // Matched, no change (2, 21), // Not matched by source, updated (4, 41), // Not matched by source, updated (5, 50) // Matched, no change ), cdc = Seq( (2, 20, "update_preimage"), (2, 21, "update_postimage"), (4, 40, "update_preimage"), (4, 41, "update_postimage"))) testExtendedMergeWithCDC("conditional not matched by source update")( source = (0, 0) :: (1, 1) :: (5, 5) :: Nil, target = (1, 10) :: (2, 20) :: (4, 40) :: (5, 50) :: Nil, mergeOn = "s.key = t.key", updateNotMatched(condition = "t.value = 20", set = "t.value = t.value + 1"))( result = Seq( (1, 10), // Matched, no change (2, 21), // Not matched by source, updated (4, 40), // Not matched by source, no change (5, 50) // Matched, no change ), cdc = Seq((2, 20, "update_preimage"), (2, 21, "update_postimage"))) testExtendedMergeWithCDC("not matched by source update and delete with skipping")( source = (0, 0) :: (1, 1) :: (2, 2) :: (5, 5) :: Nil, target = (1, 10) :: (2, 20) :: (4, 40) :: (5, 50) :: Nil, mergeOn = "s.key = t.key and t.key > 4", updateNotMatched(condition = "t.key = 1", set = "t.value = t.value + 1"), deleteNotMatched(condition = "t.key = 4"))( result = Seq( (1, 11), // Not matched by source based on merge condition, updated (2, 20), // Not matched by source based on merge condition, no change // (4, 40), Not matched by source, deleted (5, 50) // Matched, no change ), cdc = Seq( (1, 10, "update_preimage"), (1, 11, "update_postimage"), (4, 40, "delete"))) testExtendedMergeWithCDC( "matched delete and not matched by source update with skipping")( source = (0, 0) :: (1, 1) :: (2, 2) :: (5, 5) :: (6, 6) :: Nil, target = (1, 10) :: (2, 20) :: (4, 40) :: (5, 50) :: (6, 60) :: Nil, mergeOn = "s.key = t.key and t.key > 4", delete(condition = "t.key = 5"), updateNotMatched(condition = "t.key = 1", set = "t.value = t.value + 1"))( result = Seq( (1, 11), // Not matched by source based on merge condition, updated (2, 20), // Not matched by source based on merge condition, no change (4, 40), // Not matched by source, no change // (5, 50), Matched, deleted (6, 60) // Matched, no change ), cdc = Seq( (1, 10, "update_preimage"), (1, 11, "update_postimage"), (5, 50, "delete"))) } trait MergeIntoNotMatchedBySourceCDCPart2Tests extends MergeIntoNotMatchedBySourceWithCDCMixin { testExtendedMergeWithCDC("not matched by source update + delete clauses")( source = (0, 0) :: (1, 1) :: (5, 5) :: Nil, target = (1, 10) :: (2, 20) :: (7, 70) :: Nil, mergeOn = "s.key = t.key", deleteNotMatched("t.value % 20 = 0"), updateNotMatched(set = "t.value = t.value + 1"))( result = Seq( (1, 10), // Matched, no change // (2, 20) Not matched by source, deleted (7, 71) // Not matched by source, updated ), cdc = Seq((2, 20, "delete"), (7, 70, "update_preimage"), (7, 71, "update_postimage"))) testExtendedMergeWithCDC("unconditional not matched by source update + not matched insert")( source = (0, 0) :: (1, 1) :: (4, 4) :: (5, 5) :: Nil, target = (1, 10) :: (2, 20) :: (4, 40) :: (7, 70) :: Nil, mergeOn = "s.key = t.key", insert("*"), updateNotMatched(set = "t.value = t.value + 1"))( result = Seq( (0, 0), // Not matched by target, inserted (1, 10), // Matched, no change (2, 21), // Not matched by source, updated (4, 40), // Matched, no change (5, 5), // Not matched by target, inserted (7, 71) // Not matched by source, updated ), cdc = Seq( (0, 0, "insert"), (2, 20, "update_preimage"), (2, 21, "update_postimage"), (5, 5, "insert"), (7, 70, "update_preimage"), (7, 71, "update_postimage"))) testExtendedMergeWithCDC("not matched by source delete + not matched insert")( source = (0, 0) :: (1, 1) :: (5, 5) :: Nil, target = (1, 10) :: (2, 20) :: (7, 70) :: Nil, mergeOn = "s.key = t.key", insert("*"), deleteNotMatched("t.value % 20 = 0"))( result = Seq( (0, 0), // Not matched by target, inserted (1, 10), // Matched, no change // (2, 20), Not matched by source, deleted (5, 5), // Not matched by target, inserted (7, 70) // Not matched by source, no change ), cdc = Seq((0, 0, "insert"), (2, 20, "delete"), (5, 5, "insert"))) testExtendedMergeWithCDC("multiple not matched by source clauses")( source = (0, 0) :: (1, 1) :: (5, 5) :: Nil, target = (6, 6) :: (7, 7) :: (8, 8) :: (9, 9) :: (10, 10) :: (11, 11) :: Nil, mergeOn = "s.key = t.key", updateNotMatched(condition = "t.key % 6 = 0", set = "t.value = t.value + 5"), updateNotMatched(condition = "t.key % 6 = 1", set = "t.value = t.value + 4"), updateNotMatched(condition = "t.key % 6 = 2", set = "t.value = t.value + 3"), updateNotMatched(condition = "t.key % 6 = 3", set = "t.value = t.value + 2"), updateNotMatched(condition = "t.key % 6 = 4", set = "t.value = t.value + 1"), deleteNotMatched())( result = Seq( (6, 11), // Not matched by source, updated (7, 11), // Not matched by source, updated (8, 11), // Not matched by source, updated (9, 11), // Not matched by source, updated (10, 11) // Not matched by source, updated // (11, 11) Not matched by source, deleted ), cdc = Seq( (6, 6, "update_preimage"), (6, 11, "update_postimage"), (7, 7, "update_preimage"), (7, 11, "update_postimage"), (8, 8, "update_preimage"), (8, 11, "update_postimage"), (9, 9, "update_preimage"), (9, 11, "update_postimage"), (10, 10, "update_preimage"), (10, 11, "update_postimage"), (11, 11, "delete"))) testExtendedMergeWithCDC("not matched by source update + conditional insert")( source = (1, 1) :: (0, 2) :: (5, 5) :: Nil, target = (2, 2) :: (1, 4) :: (7, 3) :: Nil, mergeOn = "s.key = t.key", insert(condition = "s.value % 2 = 0", values = "*"), updateNotMatched(set = "t.value = t.value + 1"))( result = Seq( (0, 2), // Not matched (by target), inserted (2, 3), // Not matched by source, updated (1, 4), // Matched, no change // (5, 5) // Not matched (by target), not inserted (7, 4) // Not matched by source, updated ), cdc = Seq( (0, 2, "insert"), (2, 2, "update_preimage"), (2, 3, "update_postimage"), (7, 3, "update_preimage"), (7, 4, "update_postimage"))) testExtendedMergeWithCDC("not matched by source delete + conditional insert")( source = (1, 1) :: (0, 2) :: (5, 5) :: Nil, target = (2, 2) :: (1, 4) :: (7, 3) :: Nil, mergeOn = "s.key = t.key", insert(condition = "s.value % 2 = 0", values = "*"), deleteNotMatched(condition = "t.value > 2"))( result = Seq( (0, 2), // Not matched (by target), inserted (2, 2), // Not matched by source, no change (1, 4) // Matched, no change // (5, 5) // Not matched (by target), not inserted // (7, 3) Not matched by source, deleted ), cdc = Seq((0, 2, "insert"), (7, 3, "delete"))) testExtendedMergeWithCDC("when not matched by source updates all rows")( source = (1, 1) :: (0, 2) :: (5, 5) :: Nil, target = (3, 3) :: (4, 4) :: (6, 6) :: (7, 7) :: (8, 8) :: (9, 9) :: Nil, mergeOn = "s.key = t.key", updateNotMatched(set = "t.value = t.value + 1"))( result = Seq( (3, 4), // Not matched by source, updated (4, 5), // Not matched by source, updated (6, 7), // Not matched by source, updated (7, 8), // Not matched by source, updated (8, 9), // Not matched by source, updated (9, 10) // Not matched by source, updated ), cdc = Seq( (3, 3, "update_preimage"), (3, 4, "update_postimage"), (4, 4, "update_preimage"), (4, 5, "update_postimage"), (6, 6, "update_preimage"), (6, 7, "update_postimage"), (7, 7, "update_preimage"), (7, 8, "update_postimage"), (8, 8, "update_preimage"), (8, 9, "update_postimage"), (9, 9, "update_preimage"), (9, 10, "update_postimage"))) testExtendedMergeWithCDC("insert only with dummy not matched by source")( source = (1, 1) :: (0, 2) :: (5, 5) :: Nil, target = (2, 2) :: (1, 4) :: (7, 3) :: Nil, mergeOn = "s.key = t.key", insert(condition = "s.value % 2 = 0", values = "*"), deleteNotMatched(condition = "t.value > 10"))( result = Seq( (0, 2), // Not matched (by target), inserted (2, 2), // Not matched by source, no change (1, 4), // Matched, no change // (5, 5) // Not matched (by target), not inserted (7, 3) // Not matched by source, no change ), cdc = Seq((0, 2, "insert"))) testExtendedMergeWithCDC("empty source")( source = Nil, target = (2, 2) :: (1, 4) :: (7, 3) :: Nil, mergeOn = "s.key = t.key", updateNotMatched(condition = "t.key = 2", set = "value = t.value + 1"), deleteNotMatched(condition = "t.key = 7"))( result = Seq( (2, 3), // Not matched by source, updated (1, 4) // Not matched by source, no change // (7, 3) Not matched by source, deleted ), cdc = Seq( (2, 2, "update_preimage"), (2, 3, "update_postimage"), (7, 3, "delete"))) testExtendedMergeWithCDC("empty source delete only")( source = Nil, target = (2, 2) :: (1, 4) :: (7, 3) :: Nil, mergeOn = "s.key = t.key", deleteNotMatched(condition = "t.key = 7"))( result = Seq( (2, 2), // Not matched by source, no change (1, 4) // Not matched by source, no change // (7, 3) Not matched by source, deleted ), cdc = Seq((7, 3, "delete"))) testExtendedMergeWithCDC("all 3 clauses - no changes")( source = (1, 1) :: (0, 2) :: (5, 5) :: Nil, target = (2, 2) :: (1, 4) :: (7, 3) :: Nil, mergeOn = "s.key = t.key", update(condition = "t.value > 10", set = "*"), insert(condition = "s.value > 10", values = "*"), deleteNotMatched(condition = "t.value > 10"))( result = Seq( (2, 2), // Not matched by source, no change (1, 4), // Matched, no change (7, 3) // Not matched by source, no change ), cdc = Seq.empty) } trait MergeIntoNotMatchedBySourceSuite extends MergeIntoSuiteBaseMixin { import testImplicits._ // Test analysis errors with NOT MATCHED BY SOURCE clauses. testMergeAnalysisException( "error on multiple not matched by source update clauses without condition")( mergeOn = "s.key = t.key", updateNotMatched(condition = "t.key == 3", set = "value = 2 * value"), updateNotMatched(set = "value = 3 * value"), updateNotMatched(set = "value = 4 * value"))( expectedErrorClass = "NON_LAST_NOT_MATCHED_BY_SOURCE_CLAUSE_OMIT_CONDITION", expectedMessageParameters = Map.empty) testMergeAnalysisException( "error on multiple not matched by source update/delete clauses without condition")( mergeOn = "s.key = t.key", updateNotMatched(condition = "t.key == 3", set = "value = 2 * value"), deleteNotMatched(), updateNotMatched(set = "value = 4 * value"))( expectedErrorClass = "NON_LAST_NOT_MATCHED_BY_SOURCE_CLAUSE_OMIT_CONDITION", expectedMessageParameters = Map.empty) testMergeAnalysisException( "error on non-empty condition following empty condition in not matched by source " + "update clauses")( mergeOn = "s.key = t.key", updateNotMatched(set = "value = 2 * value"), updateNotMatched(condition = "t.key < 3", set = "value = value"))( expectedErrorClass = "NON_LAST_NOT_MATCHED_BY_SOURCE_CLAUSE_OMIT_CONDITION", expectedMessageParameters = Map.empty) testMergeAnalysisException( "error on non-empty condition following empty condition in not matched by source " + "delete clauses")( mergeOn = "s.key = t.key", deleteNotMatched(), deleteNotMatched(condition = "t.key < 3"))( expectedErrorClass = "NON_LAST_NOT_MATCHED_BY_SOURCE_CLAUSE_OMIT_CONDITION", expectedMessageParameters = Map.empty) testMergeAnalysisException("update not matched condition - unknown reference")( mergeOn = "s.key = t.key", updateNotMatched(condition = "unknownAttrib > 1", set = "tgtValue = tgtValue + 1"))( expectedErrorClass = "DELTA_MERGE_UNRESOLVED_EXPRESSION", expectedMessageParameters = Map( "sqlExpr" -> "unknownAttrib", "clause" -> "UPDATE condition", "cols" -> "t.key, t.tgtValue")) testMergeAnalysisException("update not matched condition - aggregation function")( mergeOn = "s.key = t.key", updateNotMatched(condition = "max(0) > 0", set = "tgtValue = tgtValue + 1"))( expectedErrorClass = "DELTA_AGGREGATION_NOT_SUPPORTED", expectedMessageParameters = Map( "operation" -> "UPDATE condition of MERGE operation", "predicate" -> "(condition = (max(0) > 0))")) testMergeAnalysisException("update not matched condition - subquery")( mergeOn = "s.key = t.key", updateNotMatched(condition = "tgtValue in (select value from t)", set = "tgtValue = 1"))( expectedErrorClass = "TABLE_OR_VIEW_NOT_FOUND", expectedMessageParameters = Map("relationName" -> "`t`")) testMergeAnalysisException("delete not matched condition - unknown reference")( mergeOn = "s.key = t.key", deleteNotMatched(condition = "unknownAttrib > 1"))( expectedErrorClass = "DELTA_MERGE_UNRESOLVED_EXPRESSION", expectedMessageParameters = Map( "sqlExpr" -> "unknownAttrib", "clause" -> "DELETE condition", "cols" -> "t.key, t.tgtValue")) testMergeAnalysisException("delete not matched condition - aggregation function")( mergeOn = "s.key = t.key", deleteNotMatched(condition = "max(0) > 0"))( expectedErrorClass = "DELTA_AGGREGATION_NOT_SUPPORTED", expectedMessageParameters = Map( "operation" -> "DELETE condition of MERGE operation", "predicate" -> "(condition = (max(0) > 0))")) testMergeAnalysisException("delete not matched condition - subquery")( mergeOn = "s.key = t.key", deleteNotMatched(condition = "tgtValue in (select tgtValue from t)"))( expectedErrorClass = "TABLE_OR_VIEW_NOT_FOUND", expectedMessageParameters = Map("relationName" -> "`t`")) test("special character in path - not matched by source delete", NameBasedAccessIncompatible) { withTempDir { tempDir => val source = s"$tempDir/sou rce^" val target = s"$tempDir/tar get=" spark.range(0, 10, 2).write.format("delta").save(source) spark.range(10).write.format("delta").save(target) executeMerge( tgt = s"delta.`$target` t", src = s"delta.`$source` s", cond = "t.id = s.id", clauses = deleteNotMatched()) checkAnswer(readDeltaTableByIdentifier(s"delta.`$target`"), Seq(0, 2, 4, 6, 8).toDF("id")) } } test("special character in path - not matched by source update", NameBasedAccessIncompatible) { withTempDir { tempDir => val source = s"$tempDir/sou rce@" val target = s"$tempDir/tar get#" spark.range(0, 10, 2).write.format("delta").save(source) spark.range(10).write.format("delta").save(target) executeMerge( tgt = s"delta.`$target` t", src = s"delta.`$source` s", cond = "t.id = s.id", clauses = updateNotMatched(set = "id = t.id * 10")) checkAnswer( readDeltaTableByIdentifier(s"delta.`$target`"), Seq(0, 10, 2, 30, 4, 50, 6, 70, 8, 90).toDF("id")) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoSQLSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaColumnMappingSelectedTestMixin, DeltaSQLCommandTest} import org.scalatest.matchers.must.Matchers.be import org.scalatest.matchers.should.Matchers.noException import org.apache.spark.sql.{AnalysisException, QueryTest, Row} import org.apache.spark.sql.catalyst.analysis.{Analyzer, ResolveSessionCatalog} import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.catalyst.plans.logical.{DeltaMergeInto, LogicalPlan} import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution.FileSourceScanExec import org.apache.spark.sql.functions.udf import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{IntegerType, StructField, StructType} trait MergeIntoSQLMixin extends MergeIntoSuiteBaseMixin with MergeIntoSQLTestUtils with DeltaSQLCommandTest with DeltaTestUtilsForTempViews { override def excluded: Seq[String] = super.excluded ++ Seq( // Schema evolution SQL syntax is not yet supported "schema evolution enabled for the current command" ) } trait MergeIntoSQLNondeterministicOrderTests extends MergeIntoSQLMixin { private def testNondeterministicOrder(insertOnly: Boolean): Unit = { withTable("target") { // For the spark sql random() function the seed is fixed for both invocations val trueRandom = () => Math.random() val trueRandomUdf = udf(trueRandom) spark.udf.register("trueRandom", trueRandomUdf.asNondeterministic()) sql("CREATE TABLE target(`trgKey` INT, `trgValue` INT) using delta") sql("INSERT INTO target VALUES (1,2), (3,4)") // This generates different data sets on every execution val sourceSql = s""" |(SELECT r.id AS srcKey, r.id AS srcValue | FROM range(1, 100000) as r | JOIN (SELECT trueRandom() * 100000 AS bound) ON r.id < bound |) AS source |""".stripMargin if (insertOnly) { sql(s""" |MERGE INTO target |USING ${sourceSql} |ON srcKey = trgKey |WHEN NOT MATCHED THEN | INSERT (trgValue, trgKey) VALUES (srcValue, srcKey) |""".stripMargin) } else { sql(s""" |MERGE INTO target |USING ${sourceSql} |ON srcKey = trgKey |WHEN MATCHED THEN | UPDATE SET trgValue = srcValue |WHEN NOT MATCHED THEN | INSERT (trgValue, trgKey) VALUES (srcValue, srcKey) |""".stripMargin) } } } test(s"detect nondeterministic source - flag on") { withSQLConf( // materializing source would fix determinism DeltaSQLConf.MERGE_MATERIALIZE_SOURCE.key -> DeltaSQLConf.MergeMaterializeSource.NONE, DeltaSQLConf.MERGE_FAIL_IF_SOURCE_CHANGED.key -> "true" ) { val e = intercept[UnsupportedOperationException]( testNondeterministicOrder(insertOnly = false) ) assert(e.getMessage.contains("source dataset is not deterministic")) } } test(s"detect nondeterministic source - flag on - insertOnly") { withSQLConf( // materializing source would fix determinism DeltaSQLConf.MERGE_MATERIALIZE_SOURCE.key -> DeltaSQLConf.MergeMaterializeSource.NONE, DeltaSQLConf.MERGE_FAIL_IF_SOURCE_CHANGED.key -> "true") { testNondeterministicOrder(insertOnly = true) } } test("detect nondeterministic source - flag off") { withSQLConf( // materializing source would fix determinism DeltaSQLConf.MERGE_MATERIALIZE_SOURCE.key -> DeltaSQLConf.MergeMaterializeSource.NONE, DeltaSQLConf.MERGE_FAIL_IF_SOURCE_CHANGED.key -> "false" ) { testNondeterministicOrder(insertOnly = false) } } test("detect nondeterministic source - flag on, materialized") { withSQLConf( // materializing source fixes determinism, so the source is no longer nondeterministic DeltaSQLConf.MERGE_MATERIALIZE_SOURCE.key -> DeltaSQLConf.MergeMaterializeSource.ALL, DeltaSQLConf.MERGE_FAIL_IF_SOURCE_CHANGED.key -> "true" ) { testNondeterministicOrder(insertOnly = false) } } } trait MergeIntoSQLTests extends MergeIntoSQLMixin { import testImplicits._ test("CTE as a source in MERGE") { withTable("source") { Seq((1, 1), (0, 3)).toDF("key1", "value").write.format("delta").saveAsTable("source") append(Seq((2, 2), (1, 4)).toDF("key2", "value")) val merge = basicMergeStmt( cte = Some("WITH cte1 AS (SELECT key1 + 2 AS key3, value FROM source)"), target = s"$tableSQLIdentifier as target", source = "cte1 src", condition = "src.key3 = target.key2", update = "key2 = 20 + src.key3, value = 20 + src.value", insert = "(key2, value) VALUES (src.key3 - 10, src.value + 10)") QueryTest.checkAnswer(sql(merge), Seq(Row(2, 1, 0, 1))) checkAnswer(readDeltaTableByIdentifier(), Row(1, 4) :: // No change Row(22, 23) :: // Update Row(-7, 11) :: // Insert Nil) } } test("inline tables with set operations in source query") { withTable("source") { append(Seq((2, 2), (1, 4)).toDF("key2", "value")) executeMerge( target = s"$tableSQLIdentifier as trg", source = """ |( SELECT * FROM VALUES (1, 6, "a") as t1(key1, value, others) | UNION | SELECT * FROM VALUES (0, 3, "b") as t2(key1, value, others) |) src """.stripMargin, condition = "src.key1 = trg.key2", update = "trg.key2 = 20 + key1, value = 20 + src.value", insert = "(trg.key2, value) VALUES (key1 - 10, src.value + 10)") checkAnswer(readDeltaTableByIdentifier(), Row(2, 2) :: // No change Row(21, 26) :: // Update Row(-10, 13) :: // Insert Nil) } } testNestedDataSupport("conflicting assignments between two nested fields")( source = """{ "key": "A", "value": { "a": { "x": 0 } } }""", target = """{ "key": "A", "value": { "a": { "x": 1 } } }""", update = "value.a.x = 2" :: "value.a.x = 3" :: Nil, errorStrs = "There is a conflict from these SET columns" :: Nil) test("Negative case - basic syntax analysis SQL") { withTable("source") { Seq((1, 1), (0, 3), (1, 5)).toDF("key1", "value").createOrReplaceTempView("source") append(Seq((2, 2), (1, 4)).toDF("key2", "value")) // duplicate column names in update clause var e = intercept[AnalysisException] { executeMerge( target = s"$tableSQLIdentifier as target", source = "source src", condition = "src.key1 = target.key2", update = "key2 = 1, key2 = 2", insert = "(key2, value) VALUES (3, 4)") }.getMessage errorContains(e, "There is a conflict from these SET columns") // duplicate column names in insert clause e = intercept[AnalysisException] { executeMerge( target = s"$tableSQLIdentifier as target", source = "source src", condition = "src.key1 = target.key2", update = "key2 = 1, value = 2", insert = "(key2, key2) VALUES (3, 4)") }.getMessage errorContains(e, "Duplicate column names in INSERT clause") } } Seq(true, false).foreach { isPartitioned => test(s"no column is used from source table - column pruning, isPartitioned: $isPartitioned") { withTable("source") { val partitions = if (isPartitioned) "key2" :: Nil else Nil append(Seq((2, 2), (1, 4)).toDF("key2", "value"), partitions) Seq((1, 1, "a"), (0, 3, "b")).toDF("key1", "value", "col1") .createOrReplaceTempView("source") // filter pushdown can cause empty join conditions and cross-join being used withCrossJoinEnabled { val merge = basicMergeStmt( target = tableSQLIdentifier, source = "source src", condition = "key2 < 0", // no row match update = "key2 = 20, value = 20", insert = "(key2, value) VALUES (10, 10)") val df = sql(merge) val readSchema: Seq[StructType] = df.queryExecution.executedPlan.collect { case f: FileSourceScanExec => f.requiredSchema } assert(readSchema.flatten.isEmpty, "column pruning does not work") } checkAnswer(readDeltaTableByIdentifier(), Row(2, 2) :: // No change Row(1, 4) :: // No change Row(10, 10) :: // Insert Row(10, 10) :: // Insert Nil) } } } test("negative case - omit multiple insert conditions") { withTable("source") { Seq((1, 1), (0, 3)).toDF("srcKey", "srcValue").write.saveAsTable("source") append(Seq((2, 2), (1, 4)).toDF("trgKey", "trgValue")) // only the last NOT MATCHED clause can omit the condition val e = intercept[ParseException]( sql(s""" |MERGE INTO $tableSQLIdentifier |USING source |ON srcKey = trgKey |WHEN NOT MATCHED THEN | INSERT (trgValue, trgKey) VALUES (srcValue, srcKey + 1) |WHEN NOT MATCHED THEN | INSERT (trgValue, trgKey) VALUES (srcValue, srcKey) """.stripMargin)) assert(e.getMessage.contains( "only the last NOT MATCHED [BY TARGET] clause can omit the condition")) } } test("detect nondeterministic merge condition") { withKeyValueData( source = (0, 0) :: (1, 10) :: (2, 20) :: Nil, target = (-1, -10) :: (1, 1) :: (2, 2) :: Nil ) { case (sourceName, targetName) => val nonDeterministicCondition = "rand() > 0.5" val e = intercept[AnalysisException]( executeMerge( tgt = s"$targetName t", src = s"$sourceName s", cond = s"s.key = t.key AND $nonDeterministicCondition", update(condition = "s.key < 2", set = "key = s.key, value = s.value"))) assert(e.getMessage.contains("DELTA_NON_DETERMINISTIC_FUNCTION_NOT_SUPPORTED")) } } test("detect nondeterministic update condition in merge") { withKeyValueData( source = (0, 0) :: (1, 10) :: (2, 20) :: Nil, target = (-1, -10) :: (1, 1) :: (2, 2) :: Nil ) { case (sourceName, targetName) => val nonDeterministicCondition = "rand() > 0.5" val e = intercept[AnalysisException]( executeMerge( tgt = s"$targetName t", src = s"$sourceName s", cond = s"s.key = t.key", update( condition = nonDeterministicCondition, set = "key = s.key, value = s.value"))) assert(e.getMessage.contains("DELTA_NON_DETERMINISTIC_FUNCTION_NOT_SUPPORTED")) } } test("detect nondeterministic delete condition in merge") { withKeyValueData( source = (0, 0) :: (1, 10) :: (2, 20) :: Nil, target = (-1, -10) :: (1, 1) :: (2, 2) :: Nil ) { case (sourceName, targetName) => val nonDeterministicCondition = "rand() > 0.5" val e = intercept[AnalysisException]( executeMerge( tgt = s"$targetName t", src = s"$sourceName s", cond = s"s.key = t.key", delete(condition = nonDeterministicCondition))) assert(e.getMessage.contains("DELTA_NON_DETERMINISTIC_FUNCTION_NOT_SUPPORTED")) } } test("detect nondeterministic insert condition in merge") { withKeyValueData( source = (0, 0) :: (1, 10) :: (2, 20) :: Nil, target = (-1, -10) :: (1, 1) :: (2, 2) :: Nil ) { case (sourceName, targetName) => val nonDeterministicCondition = "rand() > 0.5" val e = intercept[AnalysisException]( executeMerge( tgt = s"$targetName t", src = s"$sourceName s", cond = s"s.key = t.key", insert( condition = nonDeterministicCondition, values = "(key, value) VALUES (s.key, s.value)"))) assert(e.getMessage.contains("DELTA_NON_DETERMINISTIC_FUNCTION_NOT_SUPPORTED")) } } test("detect nondeterministic updateNotMatched condition in merge") { withKeyValueData( source = (0, 0) :: (1, 10) :: (2, 20) :: Nil, target = (-1, -10) :: (1, 1) :: (2, 2) :: Nil ) { case (sourceName, targetName) => val nonDeterministicCondition = "rand() > 0.5" val e = intercept[AnalysisException]( executeMerge( tgt = s"$targetName t", src = s"$sourceName s", cond = s"s.key = t.key", updateNotMatched( condition = nonDeterministicCondition, set = "key = t.key, value = t.value + 1"))) assert(e.getMessage.contains("DELTA_NON_DETERMINISTIC_FUNCTION_NOT_SUPPORTED")) } } test("detect nondeterministic deleteNotMatched condition in merge") { withKeyValueData( source = (0, 0) :: (1, 10) :: (2, 20) :: Nil, target = (-1, -10) :: (1, 1) :: (2, 2) :: Nil ) { case (sourceName, targetName) => val nonDeterministicCondition = "rand() > 0.5" val e = intercept[AnalysisException]( executeMerge( tgt = s"$targetName t", src = s"$sourceName s", cond = s"s.key = t.key", deleteNotMatched(condition = nonDeterministicCondition))) assert(e.getMessage.contains("DELTA_NON_DETERMINISTIC_FUNCTION_NOT_SUPPORTED")) } } test("allow nondeterministic update action in merge") { withKeyValueData( source = (0, 0) :: (1, 10) :: (2, 20) :: Nil, target = (-1, -10) :: (1, 1) :: (2, 2) :: Nil ) { case (sourceName, targetName) => val nonDeterministicAction = "rand()" noException should be thrownBy { executeMerge( tgt = s"$targetName t", src = s"$sourceName s", cond = s"s.key = t.key", update( condition = "s.key < 2", set = s"key = s.key, value = $nonDeterministicAction")) } } } test("allow nondeterministic insert action in merge") { withKeyValueData( source = (0, 0) :: (1, 10) :: (2, 20) :: Nil, target = (-1, -10) :: (1, 1) :: (2, 2) :: Nil ) { case (sourceName, targetName) => val nonDeterministicAction = "rand()" noException should be thrownBy { executeMerge( tgt = s"$targetName t", src = s"$sourceName s", cond = s"s.key = t.key", insert( condition = "s.key < 2", values = s"(key, value) VALUES (s.key, $nonDeterministicAction)")) } } } test("allow nondeterministic updateNotMatched action in merge") { withKeyValueData( source = (0, 0) :: (1, 10) :: (2, 20) :: Nil, target = (-1, -10) :: (1, 1) :: (2, 2) :: Nil ) { case (sourceName, targetName) => val nonDeterministicAction = "rand()" noException should be thrownBy { executeMerge( tgt = s"$targetName t", src = s"$sourceName s", cond = s"s.key = t.key", updateNotMatched( condition = "t.key < 2", set = s"key = t.key, value = t.value + $nonDeterministicAction")) } } } test("merge into a dataset temp views with star") { withTempView("v") { def testMergeWithView(testClue: String): Unit = { withClue(testClue) { withTempView("src") { sql("CREATE TEMP VIEW src AS SELECT * FROM VALUES (10, 1), (20, 2) AS t(value, key)") sql( s""" |MERGE INTO v |USING src |ON src.key = v.key |WHEN MATCHED THEN | UPDATE SET * |WHEN NOT MATCHED THEN | INSERT * |""".stripMargin) checkAnswer(spark.sql(s"select * from v"), Seq(Row(0, 0), Row(1, 10), Row(2, 20))) } } } // View on path-based table append(Seq((0, 0), (1, 1)).toDF("key", "value")) readDeltaTableByIdentifier().createOrReplaceTempView("v") testMergeWithView("with path-based table") // View on catalog table withTable("tab") { Seq((0, 0), (1, 1)).toDF("key", "value").write.format("delta").saveAsTable("tab") spark.table("tab").as("name").createOrReplaceTempView("v") testMergeWithView(tableSQLIdentifier) } } } testWithTempView("Update specific column does not work in temp views") { isSQLTempView => withJsonData( """{ "key": "A", "value": { "a": { "x": 1 } } }""", """{ "key": "A", "value": { "a": { "x": 2 } } }""" ) { (sourceName, targetName) => createTempViewFromTable(targetName, isSQLTempView) val fieldNames = spark.table(targetName).schema.fieldNames val fieldNamesStr = fieldNames.mkString("`", "`, `", "`") val e = intercept[DeltaAnalysisException] { executeMerge( target = "v t", source = s"$sourceName s", condition = "s.key = t.key", update = "value.a.x = s.value.a.x", insert = s"($fieldNamesStr) VALUES ($fieldNamesStr)") } assert(e.getMessage.contains("Unexpected assignment key")) } } test("Complex Data Type - Array of Struct") { withTable("source") { withTable("target") { // scalastyle:off line.size.limit sql("CREATE TABLE source(`smtUidNr` STRING,`evt` ARRAY>, `evtShu`: ARRAY>>>, `evtTypCd`: STRING, `evtUsrNr`: STRING, `evtUtcTcfQy`: STRING, `evtUtcTs`: STRING, `evtWstNa`: STRING, `loc`: ARRAY>, `mltDelOdrNr`: STRING, `mltPrfOfDelNa`: STRING, `mltSmtConNr`: STRING, `mnfOidNr`: STRING, `rpnEntLinNr`: STRING, `rpnEntLvlStsCd`: STRING, `rpnGovAcoTe`: STRING, `rpnInfSrcCrtLclTmZnNa`: STRING, `rpnInfSrcCrtLclTs`: STRING, `rpnInfSrcCrtUtcTcfQy`: STRING, `rpnInfSrcCrtUtcTs`: STRING, `rpnLinLvlStsCd`: STRING, `rpnPgaLinNr`: STRING, `smtDcvDt`: STRING, `smtNr`: STRING, `smtUidNr`: STRING, `xcpCtmDspCd`: STRING, `xcpGovAcoTe`: STRING, `xcpPgmCd`: STRING, `xcpRlvCd`: STRING, `xcpRlvDscTe`: STRING, `xcpRlvLclTmZnNa`: STRING, `xcpRlvLclTs`: STRING, `xcpRlvUtcTcfQy`: STRING, `xcpRlvUtcTs`: STRING, `xcpRsnCd`: STRING, `xcpRsnDscTe`: STRING, `xcpStsCd`: STRING, `xcpStsDscTe`: STRING>>,`msgTs` TIMESTAMP) using delta") sql("CREATE TABLE target(`smtUidNr` STRING,`evt` ARRAY>, `evtShu`: ARRAY>>>, `evtTypCd`: STRING, `evtUsrNr`: STRING, `evtUtcTcfQy`: STRING, `evtUtcTs`: STRING, `evtWstNa`: STRING, `loc`: ARRAY>, `mltDelOdrNr`: STRING, `mltPrfOfDelNa`: STRING, `mltSmtConNr`: STRING, `mnfOidNr`: STRING, `rpnEntLinNr`: STRING, `rpnEntLvlStsCd`: STRING, `rpnGovAcoTe`: STRING, `rpnInfSrcCrtLclTmZnNa`: STRING, `rpnInfSrcCrtLclTs`: STRING, `rpnInfSrcCrtUtcTcfQy`: STRING, `rpnInfSrcCrtUtcTs`: STRING, `rpnLinLvlStsCd`: STRING, `smtDcvDt`: STRING, `smtNr`: STRING, `smtUidNr`: STRING, `xcpCtmDspCd`: STRING, `xcpRlvCd`: STRING, `xcpRlvDscTe`: STRING, `xcpRlvLclTmZnNa`: STRING, `xcpRlvLclTs`: STRING, `xcpRlvUtcTcfQy`: STRING, `xcpRlvUtcTs`: STRING, `xcpRsnCd`: STRING, `xcpRsnDscTe`: STRING, `xcpStsCd`: STRING, `xcpStsDscTe`: STRING, `cmyHdrOidNr`: STRING, `cmyLinNr`: STRING, `coeOidNr`: STRING, `rpnPgaLinNr`: STRING, `xcpGovAcoTe`: STRING, `xcpPgmCd`: STRING>>,`msgTs` TIMESTAMP) using delta") // scalastyle:on line.size.limit sql( s""" |MERGE INTO target as r |USING source as u |ON u.smtUidNr = r.smtUidNr |WHEN MATCHED and u.msgTs > r.msgTs THEN | UPDATE SET * |WHEN NOT MATCHED THEN | INSERT * """.stripMargin) } } } Seq(true, false).foreach { partitioned => test(s"User defined _change_type column doesn't get dropped - partitioned=$partitioned") { withTable("target") { sql( s"""CREATE TABLE target USING DELTA |${if (partitioned) "PARTITIONED BY (part) " else ""} |TBLPROPERTIES (delta.enableChangeDataFeed = false) |AS SELECT id, int(id / 10) AS part, 'foo' as _change_type |FROM RANGE(1000) |""".stripMargin) executeMerge( target = "target as t", source = """( | SELECT id * 42 AS id, int(id / 10) AS part, 'bar' as _change_type FROM RANGE(33) |) s""".stripMargin, condition = "t.id = s.id", update = "*", insert = "*") sql("SELECT id, _change_type FROM target").collect().foreach { row => val _change_type = row.getString(1) assert(_change_type === "foo" || _change_type === "bar", s"Invalid _change_type for id=${row.get(0)}") } } } } test("SET * with schema evolution") { withTable("tgt", "src") { withSQLConf("spark.databricks.delta.schema.autoMerge.enabled" -> "true") { sql("create table tgt(id int, delicious string, dummy_col string) using delta") sql("create table src(id int, delicious string) using parquet") // Make sure this MERGE command can resolve sql( """ |merge into tgt as target |using (select * from src) as source on target.id=source.id |when matched then update set * |when not matched then insert *; |""".stripMargin) } } } } trait MergeIntoSQLColumnMappingOverrides extends DeltaColumnMappingSelectedTestMixin { override protected def runOnlyTests: Seq[String] = Seq("schema evolution - new nested column with update non-* and insert * - " + "array of struct - longer target") } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoScalaSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.Locale import org.apache.spark.sql.delta.actions.SetTransaction import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaExcludedTestMixin, DeltaSQLCommandTest} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.plans.Inner import org.apache.spark.sql.catalyst.plans.logical.{Assignment, DeltaMergeIntoClause, Join} import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.StructType trait MergeIntoScalaMixin extends MergeIntoSuiteBaseMixin with MergeIntoScalaTestUtils with DeltaSQLCommandTest with DeltaDMLTestUtilsPathBased with DeltaTestUtilsForTempViews with DeltaExcludedTestMixin { // Maps expected error classes to actual error classes. Used to handle error classes that are // different when running using SQL vs. Scala. override protected val mappedErrorClasses: Map[String, String] = Map( "NON_LAST_MATCHED_CLAUSE_OMIT_CONDITION" -> "DELTA_NON_LAST_MATCHED_CLAUSE_OMIT_CONDITION", "NON_LAST_NOT_MATCHED_BY_TARGET_CLAUSE_OMIT_CONDITION" -> "DELTA_NON_LAST_NOT_MATCHED_CLAUSE_OMIT_CONDITION", "NON_LAST_NOT_MATCHED_BY_SOURCE_CLAUSE_OMIT_CONDITION" -> "DELTA_NON_LAST_NOT_MATCHED_BY_SOURCE_CLAUSE_OMIT_CONDITION" ) // scalastyle:off argcount override def testNestedDataSupport(name: String, namePrefix: String = "nested data support")( source: String, target: String, update: Seq[String], insert: String = null, targetSchema: StructType = null, sourceSchema: StructType = null, result: String = null, errorStrs: Seq[String] = null, confs: Seq[(String, String)] = Seq.empty): Unit = { // scalastyle:on argcount require(result == null ^ errorStrs == null, "either set the result or the error strings") val testName = if (result != null) s"$namePrefix - $name" else s"$namePrefix - analysis error - $name" test(testName) { withSQLConf(confs: _*) { withJsonData(source, target, targetSchema, sourceSchema) { case (sourceName, targetName) => val pathOrName = parsePath(targetName) val fieldNames = readDeltaTable(pathOrName).schema.fieldNames val keyName = s"`${fieldNames.head}`" def execMerge() = { val t = DeltaTestUtils.getDeltaTableForIdentifierOrPath( spark, DeltaTestUtils.getTableIdentifierOrPath(targetName)) val m = t.as("t") .merge( spark.table(sourceName).as("s"), s"s.$keyName = t.$keyName") val withUpdate = if (update == Seq("*")) { m.whenMatched().updateAll() } else { val updateColExprMap = parseUpdate(update) m.whenMatched().updateExpr(updateColExprMap) } if (insert == "*") { withUpdate.whenNotMatched().insertAll().execute() } else { val insertExprMaps = if (insert != null) { parseInsert(insert, None) } else { fieldNames.map { f => s"t.`$f`" -> s"s.`$f`" }.toMap } withUpdate.whenNotMatched().insertExpr(insertExprMaps).execute() } } if (result != null) { execMerge() val expectedDf = readFromJSON(strToJsonSeq(result), targetSchema) checkAnswer(readDeltaTable(pathOrName), expectedDf) } else { val e = intercept[AnalysisException] { execMerge() } errorStrs.foreach { s => errorContains(e.getMessage, s) } } } } } } } trait MergeIntoScalaTests extends MergeIntoScalaMixin { import testImplicits._ test("basic scala API") { withTable("source") { append(Seq((1, 10), (2, 20)).toDF("key1", "value1"), Nil) // target val source = Seq((1, 100), (3, 30)).toDF("key2", "value2") // source io.delta.tables.DeltaTable.forPath(spark, tempPath) .merge(source, "key1 = key2") .whenMatched().updateExpr(Map("key1" -> "key2", "value1" -> "value2")) .whenNotMatched().insertExpr(Map("key1" -> "key2", "value1" -> "value2")) .execute() checkAnswer( readDeltaTable(tempPath), Row(1, 100) :: // Update Row(2, 20) :: // No change Row(3, 30) :: // Insert Nil) } } // test created to validate a fix for a bug where merge command was // resulting in a empty target table when statistics collection is disabled test("basic scala API - without stats") { withSQLConf((DeltaSQLConf.DELTA_COLLECT_STATS.key, "false")) { withTable("source") { append(Seq((1, 10), (2, 20)).toDF("key1", "value1"), Nil) // target val source = Seq((1, 100), (3, 30)).toDF("key2", "value2") // source io.delta.tables.DeltaTable.forPath(spark, tempPath) .merge(source, "key1 = key2") .whenMatched().updateExpr(Map("key1" -> "key2", "value1" -> "value2")) .whenNotMatched().insertExpr(Map("key1" -> "key2", "value1" -> "value2")) .execute() checkAnswer( readDeltaTable(tempPath), Row(1, 100) :: // Update Row(2, 20) :: // No change Row(3, 30) :: // Insert Nil) } } } test("extended scala API") { withTable("source") { append(Seq((1, 10), (2, 20), (4, 40)).toDF("key1", "value1"), Nil) // target val source = Seq((1, 100), (3, 30), (4, 41)).toDF("key2", "value2") // source io.delta.tables.DeltaTable.forPath(spark, tempPath) .merge(source, "key1 = key2") .whenMatched("key1 = 4").delete() .whenMatched("key2 = 1").updateExpr(Map("key1" -> "key2", "value1" -> "value2")) .whenNotMatched("key2 = 3").insertExpr(Map("key1" -> "key2", "value1" -> "value2")) .execute() checkAnswer( readDeltaTable(tempPath), Row(1, 100) :: // Update Row(2, 20) :: // No change Row(3, 30) :: // Insert Nil) } } test("extended scala API with Column") { withTable("source") { append(Seq((1, 10), (2, 20), (4, 40)).toDF("key1", "value1"), Nil) // target val source = Seq((1, 100), (3, 30), (4, 41)).toDF("key2", "value2") // source io.delta.tables.DeltaTable.forPath(spark, tempPath) .merge(source, functions.expr("key1 = key2")) .whenMatched(functions.expr("key1 = 4")).delete() .whenMatched(functions.expr("key2 = 1")) .update(Map("key1" -> functions.col("key2"), "value1" -> functions.col("value2"))) .whenNotMatched(functions.expr("key2 = 3")) .insert(Map("key1" -> functions.col("key2"), "value1" -> functions.col("value2"))) .execute() checkAnswer( readDeltaTable(tempPath), Row(1, 100) :: // Update Row(2, 20) :: // No change Row(3, 30) :: // Insert Nil) } } test("updateAll and insertAll") { withTable("source") { append(Seq((1, 10), (2, 20), (4, 40), (5, 50)).toDF("key", "value"), Nil) val source = Seq((1, 100), (3, 30), (4, 41), (5, 51), (6, 60)) .toDF("key", "value").createOrReplaceTempView("source") executeMerge( target = s"delta.`$tempPath` as t", source = "source s", condition = "s.key = t.key", update = "*", insert = "*") checkAnswer( readDeltaTable(tempPath), Row(1, 100) :: // Update Row(2, 20) :: // No change Row(3, 30) :: // Insert Row(4, 41) :: // Update Row(5, 51) :: // Update Row(6, 60) :: // Insert Nil) } } test("updateAll and insertAll with columns containing dot") { withTable("source") { append(Seq((1, 10), (2, 20), (4, 40)).toDF("key", "the.value"), Nil) // target val source = Seq((1, 100), (3, 30), (4, 41)).toDF("key", "the.value") // source io.delta.tables.DeltaTable.forPath(spark, tempPath).as("t") .merge(source.as("s"), "t.key = s.key") .whenMatched() .updateAll() .whenNotMatched() .insertAll() .execute() checkAnswer( readDeltaTable(tempPath), Row(1, 100) :: // Update Row(2, 20) :: // No change Row(4, 41) :: // Update Row(3, 30) :: // Insert Nil) } } test("update with empty map should do nothing") { append(Seq((1, 10), (2, 20)).toDF("trgKey", "trgValue"), Nil) // target val source = Seq((1, 100), (3, 30)).toDF("srcKey", "srcValue") // source io.delta.tables.DeltaTable.forPath(spark, tempPath) .merge(source, "srcKey = trgKey") .whenMatched().updateExpr(Map[String, String]()) .whenNotMatched().insertExpr(Map("trgKey" -> "srcKey", "trgValue" -> "srcValue")) .execute() checkAnswer( readDeltaTable(tempPath), Row(1, 10) :: // Not updated since no update clause Row(2, 20) :: // No change due to merge condition Row(3, 30) :: // Not updated since no update clause Nil) // match condition should not be ignored when map is empty io.delta.tables.DeltaTable.forPath(spark, tempPath) .merge(source, "srcKey = trgKey") .whenMatched("trgKey = 1").updateExpr(Map[String, String]()) .whenMatched().delete() .whenNotMatched().insertExpr(Map("trgKey" -> "srcKey", "trgValue" -> "srcValue")) .execute() checkAnswer( readDeltaTable(tempPath), Row(1, 10) :: // Neither updated, nor deleted (condition is not ignored) Row(2, 20) :: // No change due to merge condition Nil) // Deleted (3, 30) } // Checks specific to the APIs that are automatically handled by parser for SQL test("check invalid merge API calls") { withTable("source") { append(Seq((1, 10), (2, 20)).toDF("trgKey", "trgValue"), Nil) // target val source = Seq((1, 100), (3, 30)).toDF("srcKey", "srcValue") // source // There must be at least one WHEN clause in a MERGE statement var e = intercept[AnalysisException] { io.delta.tables.DeltaTable.forPath(spark, tempPath) .merge(source, "srcKey = trgKey") .execute() } errorContains(e.getMessage, "There must be at least one WHEN clause in a MERGE statement") // When there are multiple MATCHED clauses in a MERGE statement, // the first MATCHED clause must have a condition e = intercept[AnalysisException] { io.delta.tables.DeltaTable.forPath(spark, tempPath) .merge(source, "srcKey = trgKey") .whenMatched().delete() .whenMatched("trgKey = 1").updateExpr(Map("trgKey" -> "srcKey", "trgValue" -> "srcValue")) .whenNotMatched().insertExpr(Map("trgKey" -> "srcKey", "trgValue" -> "srcValue")) .execute() } errorContains(e.getMessage, "When there are more than one MATCHED clauses in a MERGE " + "statement, only the last MATCHED clause can omit the condition.") e = intercept[AnalysisException] { io.delta.tables.DeltaTable.forPath(spark, tempPath) .merge(source, "srcKey = trgKey") .whenMatched().updateExpr(Map("trgKey" -> "srcKey", "*" -> "*")) .whenNotMatched().insertExpr(Map("trgKey" -> "srcKey", "trgValue" -> "srcValue")) .execute() } errorContains(e.getMessage, "cannot resolve `*` in UPDATE clause") e = intercept[AnalysisException] { io.delta.tables.DeltaTable.forPath(spark, tempPath) .merge(source, "srcKey = trgKey") .whenMatched().updateExpr(Map("trgKey" -> "srcKey", "trgValue" -> "srcValue")) .whenNotMatched().insertExpr(Map("*" -> "*")) .execute() } errorContains(e.getMessage, "cannot resolve `*` in INSERT clause") e = intercept[AnalysisException] { io.delta.tables.DeltaTable.forPath(spark, tempPath) .merge(source, "srcKey = trgKey") .whenNotMatchedBySource().updateExpr(Map("*" -> "*")) .execute() } errorContains(e.getMessage, "cannot resolve `*` in UPDATE clause") } } test("merge after schema change") { withSQLConf((DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key, "true")) { withTempPath { targetDir => val targetPath = targetDir.getCanonicalPath spark.range(10).write.format("delta").save(targetPath) val t = io.delta.tables.DeltaTable.forPath(spark, targetPath).as("t") assert(t.toDF.schema == StructType.fromDDL("id LONG")) // Do one merge to change the schema. t.merge(Seq((11L, "newVal11")).toDF("id", "newCol1").as("s"), "t.id = s.id") .whenMatched().updateAll() .whenNotMatched().insertAll() .execute() // assert(t.toDF.schema == StructType.fromDDL("id LONG, newCol1 STRING")) // SC-35564 - ideally this shouldn't throw an error, but right now we can't fix it without // causing a regression. val ex = intercept[Exception] { t.merge(Seq((12L, "newVal12")).toDF("id", "newCol2").as("s"), "t.id = s.id") .whenMatched().updateAll() .whenNotMatched().insertAll() .execute() } ex.getMessage.contains("schema of your Delta table has changed in an incompatible way") } } } test("merge without table alias") { withTempDir { dir => val location = dir.getAbsolutePath Seq((1, 1, 1), (2, 2, 2)).toDF("part", "id", "n").write .format("delta") .partitionBy("part") .save(location) val table = io.delta.tables.DeltaTable.forPath(spark, location) val data1 = Seq((2, 2, 4, 2), (9, 3, 6, 9), (3, 3, 9, 3)).toDF("part", "id", "n", "part2") table.alias("t").merge( data1, "t.part = part2") .whenMatched().updateAll() .whenNotMatched().insertAll() .execute() } } test("pre-resolved exprs: should work in all expressions in absence of duplicate refs") { withTempDir { dir => val location = dir.getAbsolutePath Seq((1, 1), (2, 2)).toDF("key", "value").write .format("delta") .save(location) val table = io.delta.tables.DeltaTable.forPath(spark, location) val target = table.toDF val source = Seq((1, 10), (2, 20), (3, 30), (4, 40)).toDF("key", "value") table.merge(source, target("key") === source("key")) .whenMatched(target("key") === lit(1) && source("value") === lit(10)) .update(Map("value" -> (target("value") + source("value")))) .whenMatched(target("key") === lit(2) && source("value") === lit(20)) .delete() .whenNotMatched(source("key") === lit(3) && source("value") === lit(30)) .insert(Map("key" -> source("key"), "value" -> source("value"))) .execute() checkAnswer(table.toDF, Seq((1, 11), (3, 30)).toDF("key", "value")) } } test("pre-resolved exprs: negative cases with refs resolved to wrong Dataframes") { withTempDir { dir => val location = dir.getAbsolutePath Seq((1, 1), (2, 2)).toDF("key", "value").write .format("delta") .save(location) val table = io.delta.tables.DeltaTable.forPath(spark, location) val target = table.toDF val source = Seq((1, 10), (2, 20), (3, 30), (4, 40)).toDF("key", "value") val dummyDF = Seq((0, 0)).toDF("key", "value") def checkError(f: => Unit): Unit = { val e = intercept[AnalysisException] { f } Seq("Resolved attribute", "missing from").foreach { m => assert(e.getMessage.toLowerCase(Locale.ROOT).contains(m.toLowerCase(Locale.ROOT))) } } // Merge condition checkError { table.merge(source, target("key") === dummyDF("key")) .whenMatched().delete().execute() } // Matched clauses checkError { table.merge(source, target("key") === source("key")) .whenMatched(dummyDF("key") === lit(1)).updateAll().execute() } checkError { table.merge(source, target("key") === source("key")) .whenMatched().update(Map("key" -> dummyDF("key"))).execute() } // Not matched clauses checkError { table.merge(source, target("key") === source("key")) .whenNotMatched(dummyDF("key") === lit(1)).insertAll().execute() } checkError { table.merge(source, target("key") === source("key")) .whenNotMatched().insert(Map("key" -> dummyDF("key"))).execute() } } } /** Make sure the joins generated by merge do not have the duplicate AttributeReferences */ private def verifyNoDuplicateRefsAcrossSourceAndTarget(f: => Unit): Unit = { val executedPlans = DeltaTestUtils.withLogicalPlansCaptured(spark, optimizedPlan = true) { f } val plansWithInnerJoin = executedPlans.filter { p => p.collect { case b: Join if b.joinType == Inner => b }.nonEmpty } assert(plansWithInnerJoin.size == 1, "multiple plans found with inner join\n" + plansWithInnerJoin.mkString("\n")) val join = plansWithInnerJoin.head.collect { case j: Join => j }.head assert(join.left.outputSet.intersect(join.right.outputSet).isEmpty) } test("self-merge: duplicate attrib refs should be removed") { withTempDir { tempDir => val df = spark.range(5).selectExpr("id as key", "id as value") df.write.format("delta").save(tempDir.toString) val deltaTable = io.delta.tables.DeltaTable.forPath(tempDir.toString) val target = deltaTable.toDF val source = target.filter("key = 4") val duplicateRefs = target.queryExecution.analyzed.outputSet.intersect(source.queryExecution.analyzed.outputSet) require(duplicateRefs.nonEmpty, "source and target were expected to have duplicate refs") verifyNoDuplicateRefsAcrossSourceAndTarget { deltaTable.as("t") .merge(source.as("s"), "t.key = s.key") .whenMatched() .delete() .execute() } checkAnswer(deltaTable.toDF, spark.range(4).selectExpr("id as key", "id as value")) } } test( "self-merge + pre-resolved exprs: merge condition fails with pre-resolved, duplicate refs") { withTempDir { tempDir => val df = spark.range(5).selectExpr("id as key", "id as value") df.write.format("delta").save(tempDir.toString) val deltaTable = io.delta.tables.DeltaTable.forPath(tempDir.toString) val target = deltaTable.toDF val source = target.filter("key = 4") val e = intercept[AnalysisException] { deltaTable.merge(source, target("key") === source("key")) // this is ambiguous .whenMatched() .delete() .execute() } assert(e.getMessage.toLowerCase(Locale.ROOT).contains("ambiguous")) } } test( "self-merge + pre-resolved exprs: duplicate refs should resolve in not-matched clauses") { withTempDir { tempDir => val df = spark.range(5).selectExpr("id as key", "id as value") df.write.format("delta").save(tempDir.toString) val deltaTable = io.delta.tables.DeltaTable.forPath(tempDir.toString) val target = deltaTable.toDF val source = target.filter("key = 4") // Insert clause can refer to only source attributes, so pre-resolved references, // even when written as`target("column")`, are actually unambiguous verifyNoDuplicateRefsAcrossSourceAndTarget { deltaTable.as("t") .merge(source.as("s"), "t.key = s.key") .whenNotMatched(source("value") > 0 && target("key") > 0) .insert(Map("key" -> source("key"), "value" -> target("value"))) .whenMatched().update(Map("key" -> $"s.key")) // no-op .execute() } // nothing should be inserted as source matches completely with target checkAnswer(deltaTable.toDF, spark.range(5).selectExpr("id as key", "id as value")) } } test( "self-merge + pre-resolved exprs: non-duplicate but pre-resolved refs should still resolve") { withTempDir { tempDir => val df = spark.range(5).selectExpr("id as key", "id as value") df.write.format("delta").save(tempDir.toString) val deltaTable = io.delta.tables.DeltaTable.forPath(tempDir.toString) val target = deltaTable.toDF val source = target.filter("key = 0").drop("value") .withColumn("value", col("key") + lit(0)) .withColumn("other", lit(0)) // source is just one row (key, value, other) = (4, 4, 0) // `value` should not be duplicate ref as its recreated in the source and have different // exprIds than the target value. val duplicateRefs = target.queryExecution.analyzed.outputSet.intersect(source.queryExecution.analyzed.outputSet) require(duplicateRefs.map(_.name).toSet == Set("key"), "unexpected duplicate refs, should be only 'key': " + duplicateRefs) // So both `source("value")` and `target("value")` are not ambiguous. // `source("other")` is obviously not ambiguous. verifyNoDuplicateRefsAcrossSourceAndTarget { deltaTable.as("t") .merge( source.as("s"), expr("t.key = s.key") && source("other") === 0 && target("value") === 4) .whenMatched(source("value") > 0 && target("value") > 0 && source("other") === 0) .update(Map( "key" -> expr("s.key"), "value" -> (target("value") + source("value") + source("other")))) .whenNotMatched(source("value") > 0 && source("other") === 0) .insert(Map( "key" -> expr("s.key"), "value" -> (source("value") + source("other")))) .execute() } // key = 4 should be updated to same values, and nothing should be inserted checkAnswer(deltaTable.toDF, spark.range(5).selectExpr("id as key", "id as value")) } } test("self-merge + pre-resolved exprs: negative cases in matched clauses with duplicate refs") { // Only matched clauses can have attribute references from both source and target, hence // pre-resolved expression can be ambiguous in presence of duplicate references from self-merge withTempDir { tempDir => val df = spark.range(5).selectExpr("id as key", "id as value") df.write.format("delta").save(tempDir.toString) val deltaTable = io.delta.tables.DeltaTable.forPath(tempDir.toString) val target = deltaTable.toDF val source = target.filter("key = 4") def checkError(f: => Unit): Unit = { val e = intercept[AnalysisException] { f } assert(e.getMessage.toLowerCase(Locale.ROOT).contains("ambiguous")) } checkError { deltaTable .merge(source, target("key") === source("key")) // this is ambiguous .whenMatched() .delete() .execute() } // Update checkError { deltaTable.as("t").merge(source.as("s"), "t.key = s.key") .whenMatched(target("key") === functions.lit(4)) // can map to either key column .updateAll() .execute() } checkError { deltaTable.as("t").merge(source.as("s"), "t.key = s.key") .whenMatched() .update(Map("value" -> target("value").plus(1))) // can map to either value column .execute() } // Delete checkError { deltaTable.as("t").merge(source.as("s"), "t.key = s.key") .whenMatched(target("key") === functions.lit(4)) // can map to either key column .delete() .execute() } } } test("merge clause matched and not matched can interleave") { append(Seq((1, 10), (2, 20)).toDF("trgKey", "trgValue"), Nil) // target val source = Seq((1, 100), (2, 200), (3, 300), (4, 400)).toDF("srcKey", "srcValue") // source io.delta.tables.DeltaTable.forPath(spark, tempPath) .merge(source, "srcKey = trgKey") .whenMatched("trgKey = 1").updateExpr(Map("trgKey" -> "srcKey", "trgValue" -> "srcValue")) .whenNotMatched("srcKey = 3").insertExpr(Map("trgKey" -> "srcKey", "trgValue" -> "srcValue")) .whenMatched().delete() .whenNotMatched().insertExpr(Map("trgKey" -> "srcKey", "trgValue" -> "srcValue")) .execute() checkAnswer( readDeltaTable(tempPath), Row(1, 100) :: // Update (1, 10) // Delete (2, 20) Row(3, 300) :: // Insert (3, 300) Row(4, 400) :: // Insert (4, 400) Nil) } test("schema evolution with multiple update clauses") { withSQLConf(("spark.databricks.delta.schema.autoMerge.enabled", "true")) { withTable("target", "src") { Seq((1, "a"), (2, "b"), (3, "c")).toDF("id", "targetValue") .write.format("delta").saveAsTable("target") val source = Seq((1, "x"), (2, "y"), (4, "z")).toDF("id", "srcValue") io.delta.tables.DeltaTable.forName("target") .merge(source, col("target.id") === source.col("id")) .whenMatched("target.id = 1").updateExpr(Map("targetValue" -> "srcValue")) .whenMatched("target.id = 2").updateAll() .whenNotMatched().insertAll() .execute() checkAnswer( sql("select * from target"), Row(1, "x", null) +: Row(2, "b", "y") +: Row(3, "c", null) +: Row(4, null, "z") +: Nil) } } } /* Exclude tempViews, because DeltaTable.forName does not resolve them correctly, so no one can * use them anyway with the Scala API. // Scala API won't hit the resolution exception. testWithTempView("Update specific column works fine in temp views") { isSQLTempView => withJsonData( """{ "key": "A", "value": { "a": { "x": 1, "y": 2 } } }""", """{ "key": "A", "value": { "a": { "x": 2, "y": 1 } } }""" ) { (sourceName, targetName) => createTempViewFromTable(targetName, isSQLTempView) val fieldNames = spark.table(targetName).schema.fieldNames val fieldNamesStr = fieldNames.mkString("`", "`, `", "`") executeMerge( target = "v t", source = s"$sourceName s", condition = "s.key = t.key", update = "value.a.x = s.value.a.x", insert = s"($fieldNamesStr) VALUES ($fieldNamesStr)") checkAnswer( spark.read.format("delta").table("v"), spark.read.json( strToJsonSeq("""{ "key": "A", "value": { "a": { "x": 1, "y": 1 } } }""").toDS) ) } } */ test("delta merge into clause with invalid data type.") { import org.apache.spark.sql.catalyst.dsl.expressions._ intercept[DeltaAnalysisException] { DeltaMergeIntoClause.toActions(Seq(Assignment("1".expr, "1".expr))) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoSchemaEvolutionSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.language.implicitConversions import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.functions.{array, lit, struct} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{ArrayType, DateType, IntegerType, LongType, MapType, NullType, StringType, StructField, StructType} import org.apache.spark.util.Utils /** * Trait collecting schema evolution test runner methods and other helpers. */ trait MergeIntoSchemaEvolutionMixin extends QueryTest { self: SharedSparkSession with MergeIntoTestUtils => protected implicit def strToJsonSeq(str: String): Seq[String] = { str.split("\n").filter(_.trim.length > 0) } /** * Helper method similar to [[testEvolution()]] but without aliasing the target and source tables * as 't' and 's'. Used to check that attribute resolution works correctly with schema evolution * when using column name qualified with the actual table name: `table_name.column`. */ def testEvolutionWithoutTableAliases(name: String)( targetData: => DataFrame, sourceData: => DataFrame, clauses: MergeClause*)( expected: => Seq[Row] = Seq.empty, expectErrorContains: String = null, expectErrorWithoutEvolutionContains: String = null): Unit = for (schemaEvolutionEnabled <- BOOLEAN_DOMAIN) test(s"schema evolution - $name - schemaEvolutionEnabled= $schemaEvolutionEnabled") { withTable("target", "source") { targetData.write.format("delta").saveAsTable("target") sourceData.write.format("delta").saveAsTable("source") withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> schemaEvolutionEnabled.toString) { if (!schemaEvolutionEnabled && expectErrorWithoutEvolutionContains != null) { val ex = intercept[AnalysisException] { executeMerge(tgt = "target", src = "source", cond = "1 = 1", clauses: _*) } errorContains(ex.getMessage, expectErrorWithoutEvolutionContains) } else if (schemaEvolutionEnabled && expectErrorContains != null) { val ex = intercept[AnalysisException] { executeMerge(tgt = "target", src = "source", cond = "1 = 1", clauses: _*) } errorContains(ex.getMessage, expectErrorContains) } else { executeMerge(tgt = "target", src = "source", cond = "1 = 1", clauses: _*) checkAnswer(spark.read.table("target"), expected) } } } } /** * Test runner used by most non-nested schema evolution tests. Runs the MERGE operation once with * schema evolution disabled then with schema evolution enabled. Tests must provide for each case * either the expected result or the expected error message but not both. */ // scalastyle:off argcount protected def testEvolution(name: String)( targetData: => DataFrame, sourceData: => DataFrame, cond: String = "t.key = s.key", clauses: Seq[MergeClause] = Seq.empty, expected: => DataFrame = null, expectedWithoutEvolution: => DataFrame = null, expectedSchema: StructType = null, expectedSchemaWithoutEvolution: StructType = null, expectErrorContains: String = null, expectErrorWithoutEvolutionContains: String = null, confs: Seq[(String, String)] = Seq(), partitionCols: Seq[String] = Seq.empty): Unit = { def executeMergeAndAssert(df: DataFrame, schema: StructType, error: String): Unit = { append(targetData, partitionCols) withTempView("source") { sourceData.createOrReplaceTempView("source") if (error != null) { val ex = intercept[AnalysisException] { executeMerge(s"$tableSQLIdentifier t", "source s", cond, clauses: _*) } errorContains(Utils.exceptionString(ex), error) } else { executeMerge(s"$tableSQLIdentifier t", "source s", cond, clauses: _*) checkAnswer(readDeltaTableByIdentifier(), df.collect()) if (schema != null) { assert(readDeltaTableByIdentifier().schema === schema) } else { // Check against the schema of the expected result df if no explicit schema was // provided. Nullability of fields will vary depending on the actual data in the df so // we ignore it. assert(readDeltaTableByIdentifier().schema.asNullable === df.schema.asNullable) } } } } test(s"schema evolution - $name - with evolution disabled") { withSQLConf(confs :+ (DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key, "false"): _*) { executeMergeAndAssert(expectedWithoutEvolution, expectedSchemaWithoutEvolution, expectErrorWithoutEvolutionContains) } } test(s"schema evolution - $name") { withSQLConf((confs :+ (DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key, "true")): _*) { executeMergeAndAssert(expected, expectedSchema, expectErrorContains) } } } // scalastyle:on argcount /** * Test runner used by most nested schema evolution tests. Similar to `testEvolution()` except * that the target & source data and expected results are parsed as JSON strings for convenience. */ // scalastyle:off argcount protected def testNestedStructsEvolution(name: String)( target: Seq[String], source: Seq[String], targetSchema: StructType, sourceSchema: StructType, cond: String = "t.key = s.key", clauses: Seq[MergeClause] = Seq.empty, result: Seq[String] = null, resultSchema: StructType = null, resultWithoutEvolution: Seq[String] = null, expectErrorContains: String = null, expectErrorWithoutEvolutionContains: String = null, confs: Seq[(String, String)] = Seq()): Unit = { testEvolution(name) ( targetData = readFromJSON(target, targetSchema), sourceData = readFromJSON(source, sourceSchema), cond, clauses = clauses, expected = if (result != null ) { val schema = if (resultSchema != null) resultSchema else targetSchema readFromJSON(result, schema) } else { null }, expectedSchema = resultSchema, expectErrorContains = expectErrorContains, expectedWithoutEvolution = if (resultWithoutEvolution != null) { readFromJSON(resultWithoutEvolution, targetSchema) } else { null }, expectedSchemaWithoutEvolution = targetSchema, expectErrorWithoutEvolutionContains = expectErrorWithoutEvolutionContains, confs = confs ) } // scalastyle:on argcount } /** * Trait collecting a subset of tests providing core coverage for schema evolution. Mix this trait * in other suites to get basic test coverage for schema evolution in combination with other * features, e.g. CDF, DVs. */ trait MergeIntoSchemaEvolutionCoreTests extends MergeIntoSchemaEvolutionMixin { self: MergeIntoTestUtils with SharedSparkSession => import testImplicits._ testEvolution("new column with only insert *")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, "extra1"), (2, 2, "extra2")).toDF("key", "value", "extra"), clauses = insert("*") :: Nil, expected = ((0, 0, null) +: (3, 30, null) +: // unchanged (1, 10, null) +: // not updated (2, 2, "extra2") +: Nil // newly inserted ).toDF("key", "value", "extra"), expectedWithoutEvolution = ((0, 0) +: (3, 30) +: (1, 10) +: (2, 2) +: Nil).toDF("key", "value") ) testEvolution("new column with only update *")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, "extra1"), (2, 2, "extra2")).toDF("key", "value", "extra"), clauses = update("*") :: Nil, expected = ((0, 0, null) +: (3, 30, null) +: (1, 1, "extra1") +: // updated Nil // row 2 not inserted ).toDF("key", "value", "extra"), expectedWithoutEvolution = ((0, 0) +: (3, 30) +: (1, 1) +: Nil).toDF("key", "value") ) testEvolution("new column with insert * and delete not matched by source")( sourceData = Seq((1, 1, "extra1"), (2, 2, "extra2")).toDF("key", "value", "extra"), targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), clauses = insert("*") :: deleteNotMatched() :: Nil, expected = Seq( // (0, 0) Not matched by source, deleted (1, 10, null), // Matched, updated (2, 2, "extra2") // Not matched by target, inserted // (3, 30) Not matched by source, deleted ).toDF("key", "value", "extra"), expectedWithoutEvolution = Seq((1, 10), (2, 2)).toDF("key", "value")) testNestedStructsEvolution("new nested source field added when updating top-level column")( target = Seq("""{ "key": "A", "value": { "a": 1 } }"""), source = Seq("""{ "key": "A", "value": { "a": 2, "b": 3 } }"""), targetSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("a", IntegerType)), sourceSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("a", IntegerType) .add("b", IntegerType)), clauses = update("value = s.value") :: Nil, result = Seq("""{ "key": "A", "value": { "a": 2, "b": 3 } }"""), resultSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("a", IntegerType) .add("b", IntegerType)), expectErrorWithoutEvolutionContains = "Cannot cast") } /** * Trait collecting all base and new column tests for schema evolution. */ trait MergeIntoSchemaEvolutionBaseNewColumnTests extends MergeIntoSchemaEvolutionMixin { self: MergeIntoTestUtils with SharedSparkSession => import testImplicits._ // Schema evolution with UPDATE SET alone testEvolution("new column with update set")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, "extra1"), (2, 2, "extra2")).toDF("key", "value", "extra"), clauses = update(set = "key = s.key, value = s.value, extra = s.extra") :: Nil, expected = ((0, 0, null) +: (3, 30, null) +: (1, 1, "extra1") +: Nil) .toDF("key", "value", "extra"), expectErrorWithoutEvolutionContains = "cannot resolve extra in UPDATE clause") testEvolution("new column updated with value from existing column")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, -1), (2, 2, -2)) .toDF("key", "value", "extra"), clauses = update(set = "extra = s.value") :: Nil, expected = ((0, 0, null) +: (1, 10, 1) +: (3, 30, null) +: Nil) .asInstanceOf[List[(Integer, Integer, Integer)]] .toDF("key", "value", "extra"), expectErrorWithoutEvolutionContains = "cannot resolve extra in UPDATE clause") // Schema evolution with INSERT alone testEvolution("new column with insert values")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, "extra1"), (2, 2, "extra2")).toDF("key", "value", "extra"), clauses = insert(values = "(key, value, extra) VALUES (s.key, s.value, s.extra)") :: Nil, expected = ((0, 0, null) +: (1, 10, null) +: (3, 30, null) +: (2, 2, "extra2") +: Nil) .toDF("key", "value", "extra"), expectErrorWithoutEvolutionContains = "cannot resolve extra in INSERT clause") testEvolution("new column inserted with value from existing column")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, -1), (2, 2, -2)) .toDF("key", "value", "extra"), clauses = insert(values = "(key, extra) VALUES (s.key, s.value)") :: Nil, expected = ((0, 0, null) +: (1, 10, null) +: (3, 30, null) +: (2, null, 2) +: Nil) .asInstanceOf[List[(Integer, Integer, Integer)]] .toDF("key", "value", "extra"), expectErrorWithoutEvolutionContains = "cannot resolve extra in INSERT clause") // Schema evolution (UPDATE) with two new columns in the source but only one added to the target. testEvolution("new column with update set and column not updated")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, "extra1", "unused1"), (2, 2, "extra2", "unused2")) .toDF("key", "value", "extra", "unused"), clauses = update(set = "extra = s.extra") :: Nil, expected = ((0, 0, null) +: (1, 10, "extra1") +: (3, 30, null) +: Nil) .asInstanceOf[List[(Integer, Integer, String)]] .toDF("key", "value", "extra"), expectErrorWithoutEvolutionContains = "cannot resolve extra in UPDATE clause") testEvolution("new column updated from other new column")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, "extra1", "unused1"), (2, 2, "extra2", "unused2")) .toDF("key", "value", "extra", "unused"), clauses = update(set = "extra = s.unused") :: Nil, expected = ((0, 0, null) +: (1, 10, "unused1") +: (3, 30, null) +: Nil) .asInstanceOf[List[(Integer, Integer, String)]] .toDF("key", "value", "extra"), expectErrorWithoutEvolutionContains = "cannot resolve extra in UPDATE clause") // Schema evolution (INSERT) with two new columns in the source but only one added to the target. testEvolution("new column with insert values and column not inserted")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, "extra1", "unused1"), (2, 2, "extra2", "unused2")) .toDF("key", "value", "extra", "unused"), clauses = insert(values = "(key, extra) VALUES (s.key, s.extra)") :: Nil, expected = ((0, 0, null) +: (1, 10, null) +: (3, 30, null) +: (2, null, "extra2") +: Nil) .asInstanceOf[List[(Integer, Integer, String)]] .toDF("key", "value", "extra"), expectErrorWithoutEvolutionContains = "cannot resolve extra in INSERT clause") testEvolution("new column inserted from other new column")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, "extra1", "unused1"), (2, 2, "extra2", "unused2")) .toDF("key", "value", "extra", "unused"), clauses = insert(values = "(key, extra) VALUES (s.key, s.unused)") :: Nil, expected = ((0, 0, null) +: (1, 10, null) +: (3, 30, null) +: (2, null, "unused2") +: Nil) .asInstanceOf[List[(Integer, Integer, String)]] .toDF("key", "value", "extra"), expectErrorWithoutEvolutionContains = "cannot resolve extra in INSERT clause") // Schema evolution with two new columns added by UPDATE and INSERT resp. testEvolution("new column added by insert and other new column added by update")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, "extra1", "other1"), (2, 2, "extra2", "other2")) .toDF("key", "value", "extra", "other"), clauses = update(set = "extra = s.extra") :: insert(values = "(key, other) VALUES (s.key, s.other)") :: Nil, expected = ((0, 0, null, null) +: (1, 10, "extra1", null) +: (3, 30, null, null) +: (2, null, null, "other2") +: Nil) .asInstanceOf[List[(Integer, Integer, String, String)]] .toDF("key", "value", "extra", "other"), expectErrorWithoutEvolutionContains = "cannot resolve extra in UPDATE clause") testEvolution("new column with insert existing column")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, "extra1"), (2, 2, "extra2")).toDF("key", "value", "extra"), clauses = insert(values = "(key) VALUES (s.key)") :: Nil, expected = ((0, 0) +: (1, 10) +: (2, null) +: (3, 30) +: Nil) .asInstanceOf[List[(Integer, Integer)]] .toDF("key", "value"), expectedWithoutEvolution = ((0, 0) +: (1, 10) +: (2, null) +: (3, 30) +: Nil) .asInstanceOf[List[(Integer, Integer)]] .toDF("key", "value")) testEvolution("new column with update set and update *")( targetData = Seq((0, 0), (1, 10), (2, 20)).toDF("key", "value"), sourceData = Seq((1, 1, "extra1"), (2, 2, "extra2")).toDF("key", "value", "extra"), clauses = update(condition = "s.key < 2", set = "value = s.value") :: update("*") :: Nil, expected = ((0, 0, null) +: (1, 1, null) +: // updated by first clause (2, 2, "extra2") +: // updated by second clause Nil ).toDF("key", "value", "extra"), expectedWithoutEvolution = ((0, 0) +: (1, 1) +: (2, 2) +: Nil).toDF("key", "value") ) testEvolution("new column with update non-* and insert *")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, 1), (2, 2, 2)).toDF("key", "value", "extra"), clauses = update("key = s.key, value = s.value") :: insert("*") :: Nil, expected = ((0, 0, null) +: (2, 2, 2) +: (3, 30, null) +: // null because `extra` isn't an update action, even though it's 1 in the source data (1, 1, null) +: Nil) .asInstanceOf[List[(Integer, Integer, Integer)]].toDF("key", "value", "extra"), expectedWithoutEvolution = ((0, 0) +: (2, 2) +: (3, 30) +: (1, 1) +: Nil).toDF("key", "value") ) testEvolution("new column with update * and insert non-*")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, 1), (2, 2, 2)).toDF("key", "value", "extra"), clauses = update("*") :: insert("(key, value) VALUES (s.key, s.value)") :: Nil, expected = ((0, 0, null) +: (1, 1, 1) +: (3, 30, null) +: // null because `extra` isn't an insert action, even though it's 2 in the source data (2, 2, null) +: Nil) .asInstanceOf[List[(Integer, Integer, Integer)]].toDF("key", "value", "extra"), expectedWithoutEvolution = ((0, 0) +: (2, 2) +: (3, 30) +: (1, 1) +: Nil).toDF("key", "value") ) testEvolution("evolve partitioned table")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, "extra1"), (2, 2, "extra2")).toDF("key", "value", "extra"), clauses = update("*") :: insert("*") :: Nil, expected = ((0, 0, null) +: (1, 1, "extra1") +: (2, 2, "extra2") +: (3, 30, null) +: Nil) .toDF("key", "value", "extra"), expectedWithoutEvolution = ((0, 0) +: (2, 2) +: (3, 30) +: (1, 1) +: Nil).toDF("key", "value") ) testEvolution("star expansion with names including dots")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value.with.dotted.name"), sourceData = Seq((1, 1, "extra1"), (2, 2, "extra2")).toDF( "key", "value.with.dotted.name", "extra.dotted"), clauses = update("*") :: insert("*") :: Nil, expected = ((0, 0, null) +: (1, 1, "extra1") +: (2, 2, "extra2") +: (3, 30, null) +: Nil) .toDF("key", "value.with.dotted.name", "extra.dotted"), expectedWithoutEvolution = ((0, 0) +: (2, 2) +: (3, 30) +: (1, 1) +: Nil) .toDF("key", "value.with.dotted.name") ) testEvolution("extra nested column in source - insert")( targetData = Seq((1, (1, 10))).toDF("key", "x"), sourceData = Seq((2, (2, 20, 30))).toDF("key", "x"), clauses = insert("*") :: Nil, expected = ((1, (1, 10, null)) +: (2, (2, 20, 30)) +: Nil) .asInstanceOf[List[(Integer, (Integer, Integer, Integer))]].toDF("key", "x"), expectErrorWithoutEvolutionContains = "Cannot cast" ) testEvolution("add non-nullable column to target schema")( targetData = Seq(1, 2).toDF("key"), sourceData = Seq((1, 10), (3, 30)).toDF("key", "value"), clauses = update("*") :: insert("*") :: Nil, expected = ((1, 10) :: (2, null) :: (3, 30) :: Nil) .asInstanceOf[List[(Integer, Integer)]].toDF("key", "value"), expectedSchema = new StructType() .add("key", IntegerType) .add("value", IntegerType, nullable = true), expectedWithoutEvolution = Seq(1, 2, 3).toDF("key") ) testEvolution("extra nested column in source - update - single target partition")( targetData = Seq((1, (1, 10)), (2, (2, 2000))).toDF("key", "x") .selectExpr("key", "named_struct('a', x._1, 'c', x._2) as x").repartition(1), sourceData = Seq((1, (10, 100, 1000))).toDF("key", "x") .selectExpr("key", "named_struct('a', x._1, 'b', x._2, 'c', x._3) as x"), clauses = update("*") :: Nil, expected = ((1, (10, 100, 1000)) +: (2, (2, null, 2000)) +: Nil) .asInstanceOf[List[(Integer, (Integer, Integer, Integer))]].toDF("key", "x") .selectExpr("key", "named_struct('a', x._1, 'c', x._3, 'b', x._2) as x"), expectErrorWithoutEvolutionContains = "Cannot cast" ) testEvolution("multiple clauses")( // 1 and 2 should be updated from the source, 3 and 4 should be deleted. Only 5 is unchanged targetData = Seq((1, "a"), (2, "b"), (3, "c"), (4, "d"), (5, "e")).toDF("key", "targetVal"), // 1 and 2 should be updated into the target, 6 and 7 should be inserted. 8 should be ignored sourceData = Seq((1, "t"), (2, "u"), (3, "v"), (4, "w"), (6, "x"), (7, "y"), (8, "z")) .toDF("key", "srcVal"), clauses = update("targetVal = srcVal", "s.key = 1") :: update("*", "s.key = 2") :: delete("s.key = 3") :: delete("s.key = 4") :: insert("(key) VALUES (s.key)", "s.key = 6") :: insert("*", "s.key = 7") :: Nil, expected = ((1, "t", null) :: (2, "b", "u") :: (5, "e", null) :: (6, null, null) :: (7, null, "y") :: Nil) .asInstanceOf[List[(Integer, String, String)]].toDF("key", "targetVal", "srcVal"), // The UPDATE * clause won't resolve without evolution because the source and target columns // don't match. expectErrorWithoutEvolutionContains = "cannot resolve targetVal" ) testEvolution("multiple INSERT * clauses with UPDATE")( // 1 and 2 should be updated from the source, 3 and 4 should be deleted. Only 5 is unchanged targetData = Seq((1, "a"), (2, "b"), (3, "c"), (4, "d"), (5, "e")).toDF("key", "targetVal"), // 1 and 2 should be updated into the target, 6 and 7 should be inserted. 8 should be ignored sourceData = Seq((1, "t"), (2, "u"), (3, "v"), (4, "w"), (6, "x"), (7, "y"), (8, "z")) .toDF("key", "srcVal"), clauses = update("targetVal = srcVal", "s.key = 1") :: update("*", "s.key = 2") :: delete("s.key = 3") :: delete("s.key = 4") :: insert("*", "s.key = 6") :: insert("*", "s.key = 7") :: Nil, expected = ((1, "t", null) :: (2, "b", "u") :: (5, "e", null) :: (6, null, "x") :: (7, null, "y") :: Nil) .asInstanceOf[List[(Integer, String, String)]].toDF("key", "targetVal", "srcVal"), // The UPDATE * clause won't resolve without evolution because the source and target columns // don't match. expectErrorWithoutEvolutionContains = "cannot resolve targetVal" ) testEvolution("array of struct should work with containsNull as false")( targetData = Seq(500000).toDF("key"), sourceData = Seq(500000, 100000).toDF("key") .withColumn("generalDeduction", struct( lit("2024-11-08").cast(DateType).as("date"), array(struct(lit(0d).as("data"))))), clauses = update("*") :: insert("*") :: Nil, expected = Seq(500000, 100000).toDF("key") .withColumn("generalDeduction", struct( lit("2024-11-08").cast(DateType).as("date"), array(struct(lit(0d).as("data"))))), expectedWithoutEvolution = Seq(500000, 100000).toDF("key") ) testEvolution("test array_union with schema evolution")( targetData = Seq(1).toDF("key") .withColumn("openings", array( (2010 to 2019).map { i => struct( lit(s"$i-01-19T09:29:00.000+0000").as("opened_at"), lit(null).cast(StringType).as("opened_with"), lit(s"$i").as("location") ) }: _*)), sourceData = Seq(1).toDF("key") .withColumn("openings", array( (2020 to 8020).map { i => struct( lit(null).cast(StringType).as("opened_with"), lit(s"$i-01-19T09:29:00.000+0000").as("opened_at") ) }: _*)), clauses = update(set = "openings = array_union(s.openings, s.openings)") :: insert("*") :: Nil, expected = Seq(1).toDF("key") .withColumn("openings", array( (2020 to 8020).map { i => struct( lit(s"$i-01-19T09:29:00.000+0000").as("opened_at"), lit(null).cast(StringType).as("opened_with"), lit(null).cast(StringType).as("location") ) }: _*)), expectErrorWithoutEvolutionContains = "All nested columns must match" ) testEvolution("test array_intersect with schema evolution")( targetData = Seq(1).toDF("key") .withColumn("openings", array( (2010 to 2019).map { i => struct( lit(s"$i-01-19T09:29:00.000+0000").as("opened_at"), lit(null).cast(StringType).as("opened_with"), lit(s"$i").as("location") ) }: _*)), sourceData = Seq(1).toDF("key") .withColumn("openings", array( (2020 to 8020).map { i => struct( lit(null).cast(StringType).as("opened_with"), lit(s"$i-01-19T09:29:00.000+0000").as("opened_at") ) }: _*)), clauses = update(set = "openings = array_intersect(s.openings, s.openings)") :: insert("*") :: Nil, expected = Seq(1).toDF("key") .withColumn("openings", array( (2020 to 8020).map { i => struct( lit(s"$i-01-19T09:29:00.000+0000").as("opened_at"), lit(null).cast(StringType).as("opened_with"), lit(null).cast(StringType).as("location") ) }: _*)), expectErrorWithoutEvolutionContains = "All nested columns must match" ) testEvolution("test array_except with schema evolution")( targetData = Seq(1).toDF("key") .withColumn("openings", array( (2010 to 2020).map { i => struct( lit(s"$i-01-19T09:29:00.000+0000").as("opened_at"), lit(null).cast(StringType).as("opened_with"), lit(s"$i").as("location") ) }: _*)), sourceData = Seq(1).toDF("key") .withColumn("openings", array( (2020 to 8020).map { i => struct( lit(null).cast(StringType).as("opened_with"), lit(s"$i-01-19T09:29:00.000+0000").as("opened_at") ) }: _*)), clauses = update(set = "openings = array_except(s.openings, s.openings)") :: insert("*") :: Nil, expected = Seq(1).toDF("key") .withColumn( "openings", array().cast( new ArrayType( new StructType() .add("opened_at", StringType) .add("opened_with", StringType) .add("location", StringType), true ) ) ), expectErrorWithoutEvolutionContains = "All nested columns must match" ) testEvolution("test array_remove with schema evolution")( targetData = Seq(1).toDF("key") .withColumn("openings", array( (2010 to 2019).map { i => struct( lit(s"$i-01-19T09:29:00.000+0000").as("opened_at"), lit(null).cast(StringType).as("opened_with"), lit(s"$i").as("location") ) }: _*)), sourceData = Seq(1).toDF("key") .withColumn("openings", array( (2020 to 8020).map { i => struct( lit(null).cast(StringType).as("opened_with"), lit(s"$i-01-19T09:29:00.000+0000").as("opened_at") ) }: _*)), clauses = update( set = "openings = array_remove(s.openings," + "named_struct('opened_with', cast(null as string)," + "'opened_at', '2020-01-19T09:29:00.000+0000'))") :: insert("*") :: Nil, expected = Seq(1).toDF("key") .withColumn( "openings", array((2021 to 8020).map { i => struct( lit(s"$i-01-19T09:29:00.000+0000").as("opened_at"), lit(null).cast(StringType).as("opened_with"), lit(null).cast(StringType).as("location") ) }: _*)), expectErrorWithoutEvolutionContains = "All nested columns must match" ) testEvolution("test array_distinct with schema evolution")( targetData = Seq(1).toDF("key") .withColumn("openings", array( (2010 to 2019).map { i => struct( lit(s"$i-01-19T09:29:00.000+0000").as("opened_at"), lit(null).cast(StringType).as("opened_with"), lit(s"$i").as("location") ) }: _* )), sourceData = Seq(1).toDF("key") .withColumn("openings", array( ((2020 to 8020) ++ (2020 to 8020)).map { i => struct( lit(null).cast(StringType).as("opened_with"), lit(s"$i-01-19T09:29:00.000+0000").as("opened_at") ) }: _* )), clauses = update(set = "openings = array_distinct(s.openings)") :: insert("*") :: Nil, expected = Seq(1).toDF("key") .withColumn( "openings", array((2020 to 8020).map { i => struct( lit(s"$i-01-19T09:29:00.000+0000").as("opened_at"), lit(null).cast(StringType).as("opened_with"), lit(null).cast(StringType).as("location") ) }: _*)), expectErrorWithoutEvolutionContains = "All nested columns must match" ) testEvolution("void columns are not allowed")( targetData = Seq((1, 1)).toDF("key", "value"), sourceData = Seq((1, 100, null), (2, 200, null)).toDF("key", "value", "extra"), clauses = update("*") :: insert("*") :: Nil, expectErrorContains = "Cannot add column `extra` with type VOID", expectedWithoutEvolution = Seq((1, 100), (2, 200)).toDF("key", "value") ) testEvolution("top-level column assignment qualified with source alias")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, "extra1"), (2, 2, "extra2")).toDF("key", "value", "extra"), clauses = update(set = "s.value = s.value") :: Nil, // Assigning to the source is just wrong and should fail. expected = ((0, 0) +: (3, 30) +: (1, 1) +: Nil) .toDF("key", "value"), expectErrorWithoutEvolutionContains = "cannot resolve s.value in UPDATE clause") test("schema evolution enabled for the current command") { withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "false") { withTable("target", "source") { Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value") .write.format("delta").saveAsTable("target") Seq((1, 1, 1), (2, 2, 2)).toDF("key", "value", "extra") .write.format("delta").saveAsTable("source") // Should fail without schema evolution val e = intercept[org.apache.spark.sql.AnalysisException] { executeMerge( "target", "source", "target.key = source.key", update("extra = -1"), insert("*")) } assert(e.getErrorClass === "DELTA_MERGE_UNRESOLVED_EXPRESSION") assert(e.getMessage.contains("resolve extra in UPDATE clause")) // Should succeed with schema evolution executeMergeWithSchemaEvolution( "target", "source", "target.key = source.key", update("extra = -1"), insert("*")) checkAnswer( spark.table("target"), Seq[(Integer, Integer, Integer)]((0, 0, null), (1, 10, -1), (2, 2, 2), (3, 30, null)) .toDF("key", "value", "extra")) } } } testEvolutionWithoutTableAliases("new top-level column assignment qualified with target name")( targetData = Seq((0, 1)).toDF("a", "nested_a") .selectExpr("a", "named_struct('a', nested_a) as target"), sourceData = Seq((2, 3, 4, 5)).toDF("a", "b", "nested_a", "nested_b") .selectExpr("a", "b", "named_struct('a', nested_a, 'b', nested_b) as target"), clauses = update("target.b = source.b"))( expected = Seq(Row(0, Row(1, 3))), expectErrorWithoutEvolutionContains = "No such struct field `b` in `a") testEvolutionWithoutTableAliases("new nested field assignment qualified with target name")( targetData = Seq((0, 1)).toDF("a", "nested_a") .selectExpr("a", "named_struct('a', nested_a) as target"), sourceData = Seq((2, 3, 4, 5)).toDF("a", "b", "nested_a", "nested_b") .selectExpr("a", "b", "named_struct('a', nested_a, 'b', nested_b) as target"), clauses = update("target.target.b = source.target.b"))( // target.target.b gets resolved to source struct target, accessing nested field target.target.b // which doesn't exist. expectErrorContains = "No such struct field `target` in `a`, `b`", // target.target.b: target.target gets resolved to target table 'target' column with nested // field b which doesn't exist. expectErrorWithoutEvolutionContains = "No such struct field `b` in `a`") } /** * Trait collecting all base and existing column tests for schema evolution. */ trait MergeIntoSchemaEvolutionBaseExistingColumnTests extends MergeIntoSchemaEvolutionMixin { self: MergeIntoTestUtils with SharedSparkSession => import testImplicits._ // No schema evolution testEvolution("old column updated from new column")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, -1), (2, 2, -2)) .toDF("key", "value", "extra"), clauses = update(set = "value = s.extra") :: Nil, expected = ((0, 0) +: (1, -1) +: (3, 30) +: Nil).toDF("key", "value"), expectedWithoutEvolution = ((0, 0) +: (1, -1) +: (3, 30) +: Nil).toDF("key", "value")) testEvolution("old column inserted from new column")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, -1), (2, 2, -2)) .toDF("key", "value", "extra"), clauses = insert(values = "(key) VALUES (s.extra)") :: Nil, expected = ((0, 0) +: (1, 10) +: (3, 30) +: (-2, null) +: Nil) .asInstanceOf[List[(Integer, Integer)]] .toDF("key", "value"), expectedWithoutEvolution = ((0, 0) +: (1, 10) +: (3, 30) +: (-2, null) +: Nil) .asInstanceOf[List[(Integer, Integer)]] .toDF("key", "value")) // Column doesn't exist with UPDATE/INSERT alone. testEvolution("update set nonexistent column")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, "extra1"), (2, 2, "extra2")).toDF("key", "value", "extra"), clauses = update(set = "nonexistent = s.extra") :: Nil, expectErrorContains = "cannot resolve nonexistent in UPDATE clause", expectErrorWithoutEvolutionContains = "cannot resolve nonexistent in UPDATE clause") testEvolution("insert values nonexistent column")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, "extra1"), (2, 2, "extra2")).toDF("key", "value", "extra"), clauses = insert(values = "(nonexistent) VALUES (s.extra)") :: Nil, expectErrorContains = "cannot resolve nonexistent in INSERT clause", expectErrorWithoutEvolutionContains = "cannot resolve nonexistent in INSERT clause") testEvolution("update * with column not in source")( targetData = Seq((0, 0, 0), (1, 10, 10), (3, 30, 30)).toDF("key", "value", "extra"), sourceData = Seq((1, 1), (2, 2)).toDF("key", "value"), clauses = update("*") :: Nil, // update went through even though `extra` wasn't there expected = ((0, 0, 0) +: (1, 1, 10) +: (3, 30, 30) +: Nil).toDF("key", "value", "extra"), expectErrorWithoutEvolutionContains = "cannot resolve extra in UPDATE clause" ) testEvolution("insert * with column not in source")( targetData = Seq((0, 0, 0), (1, 10, 10), (3, 30, 30)).toDF("key", "value", "extra"), sourceData = Seq((1, 1), (2, 2)).toDF("key", "value"), clauses = insert("*") :: Nil, // insert went through even though `extra` wasn't there expected = ((0, 0, 0) +: (1, 10, 10) +: (2, 2, null) +: (3, 30, 30) +: Nil) .asInstanceOf[List[(Integer, Integer, Integer)]] .toDF("key", "value", "extra"), expectErrorWithoutEvolutionContains = "cannot resolve extra in INSERT clause" ) testEvolution("explicitly insert subset of columns")( targetData = Seq((0, 0, 0), (1, 10, 10), (3, 30, 30)).toDF("key", "value", "extra"), sourceData = Seq((1, 1, 1), (2, 2, 2)).toDF("key", "value", "extra"), clauses = insert("(key, value) VALUES (s.key, s.value)") :: Nil, // 2 should have extra = null, since extra wasn't in the insert spec. expected = ((0, 0, 0) +: (1, 10, 10) +: (2, 2, null) +: (3, 30, 30) +: Nil) .asInstanceOf[List[(Integer, Integer, Integer)]] .toDF("key", "value", "extra"), expectedWithoutEvolution = ((0, 0, 0) +: (1, 10, 10) +: (2, 2, null) +: (3, 30, 30) +: Nil) .asInstanceOf[List[(Integer, Integer, Integer)]] .toDF("key", "value", "extra") ) testEvolution("explicitly update one column")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, 1), (2, 2, 2)).toDF("key", "value", "extra"), clauses = update("value = s.value") :: Nil, // Both results should be the same - we're checking that no evolution logic triggers // even though there's an extra source column. expected = ((0, 0) +: (1, 1) +: (3, 30) +: Nil).toDF("key", "value"), expectedWithoutEvolution = ((0, 0) +: (1, 1) +: (3, 30) +: Nil).toDF("key", "value") ) testEvolution(s"case-insensitive insert")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1), (2, 2)).toDF("key", "VALUE"), clauses = insert("(key, value, VALUE) VALUES (s.key, s.value, s.VALUE)") :: Nil, expected = ((0, 0) +: (1, 10) +: (3, 30) +: (2, 2) +: Nil).toDF("key", "value"), expectedWithoutEvolution = ((0, 0) +: (1, 10) +: (3, 30) +: (2, 2) +: Nil).toDF("key", "value"), confs = Seq(SQLConf.CASE_SENSITIVE.key -> "false") ) // TODO: Add a test for case-sensitive insert and column not in target testEvolution("case-sensitive insert, column not in source")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1), (2, 2)).toDF("key", "VALUE"), clauses = insert("(key, value) VALUES (s.key, s.value)") :: Nil, expectErrorContains = "Cannot resolve s.value in INSERT clause", expectErrorWithoutEvolutionContains = "Cannot resolve s.value in INSERT clause", confs = Seq(SQLConf.CASE_SENSITIVE.key -> "true") ) // Note that incompatible types are those where a cast to the target type can't resolve - any // valid cast will be permitted. testEvolution("incompatible types in update *")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, Array[Byte](1)), (2, Array[Byte](2))).toDF("key", "value"), clauses = update("*") :: Nil, expectErrorContains = "Failed to merge incompatible data types IntegerType and BinaryType", expectErrorWithoutEvolutionContains = "cannot cast" ) testEvolution("incompatible types in insert *")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, Array[Byte](1)), (2, Array[Byte](2))).toDF("key", "value"), clauses = insert("*") :: Nil, expectErrorContains = "Failed to merge incompatible data types IntegerType and BinaryType", expectErrorWithoutEvolutionContains = "cannot cast" ) // All integral types other than long can be upcasted to integer. testEvolution("upcast numeric source types into integer target")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1.toByte, 1.toShort), (2.toByte, 2.toShort)).toDF("key", "value"), clauses = update("*") :: insert("*") :: Nil, expected = Seq((0, 0), (1, 1), (2, 2), (3, 30)).toDF("key", "value"), expectedWithoutEvolution = Seq((0, 0), (1, 1), (2, 2), (3, 30)).toDF("key", "value") ) // Delta's automatic schema evolution allows converting table columns with a numeric type narrower // than integer to integer, because in the underlying Parquet they're all stored as ints. testEvolution("upcast numeric target types from integer source")( targetData = Seq((0.toByte, 0.toShort), (1.toByte, 10.toShort)).toDF("key", "value"), sourceData = Seq((1, 1), (2, 2)).toDF("key", "value"), clauses = update("*") :: insert("*") :: Nil, expected = ((0.toByte, 0.toShort) +: (1.toByte, 1.toShort) +: (2.toByte, 2.toShort) +: Nil ).toDF("key", "value"), expectedWithoutEvolution = ((0.toByte, 0.toShort) +: (1.toByte, 1.toShort) +: (2.toByte, 2.toShort) +: Nil ).toDF("key", "value") ) testEvolution("upcast int source type into long target")( targetData = Seq((0, 0L), (1, 10L), (3, 30L)).toDF("key", "value"), sourceData = Seq((1, 1), (2, 2)).toDF("key", "value"), clauses = update("*") :: insert("*") :: Nil, expected = ((0, 0L) +: (1, 1L) +: (2, 2L) +: (3, 30L) +: Nil).toDF("key", "value"), expectedWithoutEvolution = ((0, 0L) +: (1, 1L) +: (2, 2L) +: (3, 30L) +: Nil).toDF("key", "value") ) testEvolution("write string into int column")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, "1"), (2, "2"), (5, "notANumber")).toDF("key", "value"), clauses = insert("*") :: Nil, expected = ((0, 0) +: (1, 10) +: (2, 2) +: (3, 30) +: (5, null) +: Nil) .asInstanceOf[List[(Integer, Integer)]].toDF("key", "value"), expectedWithoutEvolution = ((0, 0) +: (1, 10) +: (2, 2) +: (3, 30) +: (5, null) +: Nil) .asInstanceOf[List[(Integer, Integer)]].toDF("key", "value"), // Disable ANSI as this test needs to cast string "notANumber" to int confs = Seq(SQLConf.STORE_ASSIGNMENT_POLICY.key -> "LEGACY") ) // This is kinda bug-for-bug compatibility. It doesn't really make sense that infinity is casted // to int as Int.MaxValue, but that's the behavior. testEvolution("write double into int column")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1.1), (2, 2.2), (5, Double.PositiveInfinity)).toDF("key", "value"), clauses = insert("*") :: Nil, expected = ((0, 0) +: (1, 10) +: (2, 2) +: (3, 30) +: (5, Int.MaxValue) +: Nil) .asInstanceOf[List[(Integer, Integer)]].toDF("key", "value"), expectedWithoutEvolution = ((0, 0) +: (1, 10) +: (2, 2) +: (3, 30) +: (5, Int.MaxValue) +: Nil) .asInstanceOf[List[(Integer, Integer)]].toDF("key", "value"), // Disable ANSI as this test needs to cast Double.PositiveInfinity to int confs = Seq(SQLConf.STORE_ASSIGNMENT_POLICY.key -> "LEGACY") ) testEvolution("missing nested column in source - insert")( targetData = Seq((1, (1, 2, 3))).toDF("key", "x"), sourceData = Seq((2, (2, 3))).toDF("key", "x"), clauses = insert("*") :: Nil, expected = ((1, (1, 2, 3)) +: (2, (2, 3, null)) +: Nil) .asInstanceOf[List[(Integer, (Integer, Integer, Integer))]].toDF("key", "x"), expectErrorWithoutEvolutionContains = "Cannot cast" ) testEvolution("missing nested column resolved by name - insert")( targetData = Seq((1, 1, 2, 3)).toDF("key", "a", "b", "c") .selectExpr("key", "named_struct('a', a, 'b', b, 'c', c) as x"), sourceData = Seq((2, 2, 4)).toDF("key", "a", "c") .selectExpr("key", "named_struct('a', a, 'c', c) as x"), clauses = insert("*") :: Nil, expected = ((1, (1, 2, 3)) +: (2, (2, null, 4)) +: Nil) .asInstanceOf[List[(Integer, (Integer, Integer, Integer))]].toDF("key", "x") .selectExpr("key", "named_struct('a', x._1, 'b', x._2, 'c', x._3) as x"), expectErrorWithoutEvolutionContains = "Cannot cast" ) testEvolutionWithoutTableAliases( "existing top-level column assignment qualified with target name")( targetData = Seq((0, 1)).toDF("a", "nested_a") .selectExpr("a", "named_struct('a', nested_a) as target"), sourceData = Seq((2, 3)).toDF("a", "nested_a") .selectExpr("a", "named_struct('a', nested_a) as target"), clauses = update("target.a = source.a"))( expected = Seq(Row(2, Row(1)))) testEvolutionWithoutTableAliases("existing nested field assignment qualified with target name")( targetData = Seq((0, 1)).toDF("a", "nested_a") .selectExpr("a", "named_struct('a', nested_a) as target"), sourceData = Seq((2, 3)).toDF("a", "nested_a") .selectExpr("a", "named_struct('a', nested_a) as target"), clauses = update("target.target.a = source.target.a"))( expected = Seq(Row(0, Row(3)))) } trait MergeIntoSchemaEvoStoreAssignmentPolicyTests extends MergeIntoSchemaEvolutionMixin { self: MergeIntoTestUtils with SharedSparkSession => import testImplicits._ // Upcasting is always allowed. for (storeAssignmentPolicy <- StoreAssignmentPolicy.values) testEvolution("upcast int source type into long target, storeAssignmentPolicy = " + s"$storeAssignmentPolicy")( targetData = Seq((0, 0L), (1, 1L), (3, 3L)).toDF("key", "value"), sourceData = Seq((1, 1), (2, 2)).toDF("key", "value"), clauses = update("*") :: insert("*") :: Nil, expected = ((0, 0L) +: (1, 1L) +: (2, 2L) +: (3, 3L) +: Nil).toDF("key", "value"), expectedWithoutEvolution = ((0, 0L) +: (1, 1L) +: (2, 2L) +: (3, 3L) +: Nil).toDF("key", "value"), confs = Seq( SQLConf.STORE_ASSIGNMENT_POLICY.key -> storeAssignmentPolicy.toString, DeltaSQLConf.UPDATE_AND_MERGE_CASTING_FOLLOWS_ANSI_ENABLED_FLAG.key -> "false") ) // Casts that are not valid implicit casts (e.g. string -> boolean) are never allowed with // schema evolution enabled and allowed only when storeAssignmentPolicy is LEGACY or ANSI when // schema evolution is disabled. for (storeAssignmentPolicy <- StoreAssignmentPolicy.values - StoreAssignmentPolicy.STRICT) testEvolution("invalid implicit cast string source type into boolean target, " + s"storeAssignmentPolicy = $storeAssignmentPolicy")( targetData = Seq((0, true), (1, false), (3, true)).toDF("key", "value"), sourceData = Seq((1, "true"), (2, "false")).toDF("key", "value"), clauses = update("*") :: insert("*") :: Nil, expectErrorContains = "Failed to merge incompatible data types BooleanType and StringType", expectedWithoutEvolution = ((0, true) +: (1, true) +: (2, false) +: (3, true) +: Nil) .toDF("key", "value"), confs = Seq( SQLConf.STORE_ASSIGNMENT_POLICY.key -> storeAssignmentPolicy.toString, DeltaSQLConf.UPDATE_AND_MERGE_CASTING_FOLLOWS_ANSI_ENABLED_FLAG.key -> "false") ) // Casts that are not valid implicit casts (e.g. string -> boolean) are not allowed with // storeAssignmentPolicy = STRICT. testEvolution("invalid implicit cast string source type into boolean target, " + s"storeAssignmentPolicy = ${StoreAssignmentPolicy.STRICT}")( targetData = Seq((0, true), (1, false), (3, true)).toDF("key", "value"), sourceData = Seq((1, "true"), (2, "false")).toDF("key", "value"), clauses = update("*") :: insert("*") :: Nil, expectErrorContains = "Failed to merge incompatible data types BooleanType and StringType", expectErrorWithoutEvolutionContains = "cannot up cast s.value from \"string\" to \"boolean\"", confs = Seq( SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.STRICT.toString, DeltaSQLConf.UPDATE_AND_MERGE_CASTING_FOLLOWS_ANSI_ENABLED_FLAG.key -> "false") ) // Valid implicit casts that are not upcasts (e.g. string -> int) are allowed with // storeAssignmentPolicy = LEGACY or ANSI. for (storeAssignmentPolicy <- StoreAssignmentPolicy.values - StoreAssignmentPolicy.STRICT) testEvolution("valid implicit cast string source type into int target, " + s"storeAssignmentPolicy = ${storeAssignmentPolicy}")( targetData = Seq((0, 0), (1, 1), (3, 3)).toDF("key", "value"), sourceData = Seq((1, "1"), (2, "2")).toDF("key", "value"), clauses = update("*") :: insert("*") :: Nil, expected = ((0, 0)+: (1, 1) +: (2, 2) +: (3, 3) +: Nil).toDF("key", "value"), expectedWithoutEvolution = ((0, 0) +: (1, 1) +: (2, 2) +: (3, 3) +: Nil).toDF("key", "value"), confs = Seq( SQLConf.STORE_ASSIGNMENT_POLICY.key -> storeAssignmentPolicy.toString, DeltaSQLConf.UPDATE_AND_MERGE_CASTING_FOLLOWS_ANSI_ENABLED_FLAG.key -> "false") ) for (storeAssignmentPolicy <- StoreAssignmentPolicy.values - StoreAssignmentPolicy.STRICT) testEvolution("valid implicit cast long source type into int target, " + s"storeAssignmentPolicy = $storeAssignmentPolicy")( targetData = Seq((0, 0), (1, 1), (3, 3)).toDF("key", "value"), sourceData = Seq((1, 1L), (2, 2L)).toDF("key", "value"), clauses = update("*") :: insert("*") :: Nil, expected = ((0, 0)+: (1, 1) +: (2, 2) +: (3, 3) +: Nil).toDF("key", "value"), expectedWithoutEvolution = ((0, 0) +: (1, 1) +: (2, 2) +: (3, 3) +: Nil).toDF("key", "value"), confs = Seq( SQLConf.STORE_ASSIGNMENT_POLICY.key -> storeAssignmentPolicy.toString, DeltaSQLConf.UPDATE_AND_MERGE_CASTING_FOLLOWS_ANSI_ENABLED_FLAG.key -> "false") ) // Valid implicit casts that are not upcasts (e.g. string -> int) are rejected with // storeAssignmentPolicy = STRICT. testEvolution("valid implicit cast string source type into int target, " + s"storeAssignmentPolicy = ${StoreAssignmentPolicy.STRICT}")( targetData = Seq((0, 0), (1, 1), (3, 3)).toDF("key", "value"), sourceData = Seq((1, "1"), (2, "2")).toDF("key", "value"), clauses = update("*") :: insert("*") :: Nil, expectErrorContains = "cannot up cast s.value from \"string\" to \"int\"", expectErrorWithoutEvolutionContains = "cannot up cast s.value from \"string\" to \"int\"", confs = Seq( SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.STRICT.toString, DeltaSQLConf.UPDATE_AND_MERGE_CASTING_FOLLOWS_ANSI_ENABLED_FLAG.key -> "false") ) testEvolution("multiple casts with storeAssignmentPolicy = STRICT")( targetData = Seq((0L, "0"), (1L, "10"), (3L, "30")).toDF("key", "value"), sourceData = Seq((1, 1L), (2, 2L)).toDF("key", "value"), clauses = update("*") :: insert("*") :: Nil, expected = ((0L, "0") +: (1L, "1") +: (2L, "2") +: (3L, "30") +: Nil).toDF("key", "value"), expectedWithoutEvolution = ((0L, "0") +: (1L, "1") +: (2L, "2") +: (3L, "30") +: Nil).toDF("key", "value"), confs = Seq( SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.STRICT.toString, DeltaSQLConf.UPDATE_AND_MERGE_CASTING_FOLLOWS_ANSI_ENABLED_FLAG.key -> "false")) testEvolution("new column with storeAssignmentPolicy = STRICT")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, "one"), (2, 2, "two")).toDF("key", "value", "extra"), clauses = update("value = CAST(s.value AS short)") :: insert("*") :: Nil, expected = ((0, 0, null) +: (1, 1, null) +: (2, 2, "two") +: (3, 30, null) +: Nil) .toDF("key", "value", "extra"), expectedWithoutEvolution = ((0, 0) +: (1, 1) +: (2, 2) +: (3, 30) +: Nil).toDF("key", "value"), confs = Seq( SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.STRICT.toString, DeltaSQLConf.UPDATE_AND_MERGE_CASTING_FOLLOWS_ANSI_ENABLED_FLAG.key -> "false")) } /** * Trait collecting tests for schema evolution with a NOT MATCHED BY SOURCE clause. */ trait MergeIntoSchemaEvolutionNotMatchedBySourceTests extends MergeIntoSchemaEvolutionMixin { self: MergeIntoTestUtils with SharedSparkSession => import testImplicits._ // Test schema evolution with NOT MATCHED BY SOURCE clauses. testEvolution("new column with insert * and conditional update not matched by source")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, "extra1"), (2, 2, "extra2")).toDF("key", "value", "extra"), clauses = insert("*") :: updateNotMatched(condition = "key > 0", set = "value = value + 1") :: Nil, expected = Seq( (0, 0, null), // Not matched by source, no change (1, 10, null), // Matched, no change (2, 2, "extra2"), // Not matched by target, inserted (3, 31, null) // Not matched by source, updated ).toDF("key", "value", "extra"), expectedWithoutEvolution = Seq((0, 0), (1, 10), (2, 2), (3, 31)).toDF("key", "value")) testEvolution("new column not inserted and conditional update not matched by source")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, "extra1"), (2, 2, "extra2")).toDF("key", "value", "extra"), clauses = updateNotMatched(condition = "key > 0", set = "value = value + 1") :: Nil, expected = Seq( (0, 0), // Not matched by source, no change (1, 10), // Matched, no change (3, 31) // Not matched by source, updated ).toDF("key", "value"), expectedWithoutEvolution = Seq((0, 0), (1, 10), (3, 31)).toDF("key", "value")) testEvolution("new column referenced in matched condition but not inserted")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, "extra1"), (2, 2, "extra2")).toDF("key", "value", "extra"), clauses = delete(condition = "extra = 'extra1'") :: updateNotMatched(condition = "key > 0", set = "value = value + 1") :: Nil, expected = Seq( (0, 0), // Not matched by source, no change // (1, 10), Matched, deleted (3, 31) // Not matched by source, updated ).toDF("key", "value"), expectedWithoutEvolution = Seq((0, 0), (3, 31)).toDF("key", "value")) testEvolution("matched update * and conditional update not matched by source")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, "extra1"), (2, 2, "extra2")).toDF("key", "value", "extra"), clauses = update("*") :: updateNotMatched(condition = "key > 0", set = "value = value + 1") :: Nil, expected = Seq( (0, 0, null), // Not matched by source, no change (1, 1, "extra1"), // Matched, updated (3, 31, null) // Not matched by source, updated ).toDF("key", "value", "extra"), expectedWithoutEvolution = Seq((0, 0), (1, 1), (3, 31)).toDF("key", "value")) // Migrating new column via WHEN NOT MATCHED BY SOURCE is not allowed. testEvolution("update new column with not matched by source fails")( targetData = Seq((0, 0), (1, 10), (3, 30)).toDF("key", "value"), sourceData = Seq((1, 1, "extra3"), (2, 2, "extra2")).toDF("key", "value", "extra"), clauses = updateNotMatched("extra = s.extra") :: Nil, expectErrorContains = "cannot resolve extra in UPDATE clause", expectErrorWithoutEvolutionContains = "cannot resolve extra in UPDATE clause") } /** * Trait collecting all tests for nested struct evolution. */ trait MergeIntoNestedStructInMapEvolutionTests extends MergeIntoSchemaEvolutionMixin { self: MergeIntoTestUtils with SharedSparkSession => import testImplicits._ // scalastyle:off line.size.limit // Struct evolution inside of map values. testNestedStructsEvolution("new source column in map struct value")( target = """{ "key": "A", "map": { "key": { "a": 1 } } } { "key": "C", "map": { "key": { "a": 3 } } }""", source = """{ "key": "A", "map": { "key": { "a": 2, "b": 2 } } } { "key": "B", "map": { "key": { "a": 1, "b": 2 } } }""", targetSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, new StructType().add("a", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, new StructType().add("a", IntegerType).add("b", IntegerType))), resultSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, new StructType().add("a", IntegerType).add("b", IntegerType))), clauses = update("*") :: insert("*") :: Nil, result = """{ "key": "A", "map": { "key": { "a": 2, "b": 2 } } } { "key": "B", "map": { "key": { "a": 1, "b": 2 } } } { "key": "C", "map": { "key": { "a": 3, "b": null } } }""", expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("new source column in nested map struct value")( target = """{"key": "A", "map": { "key": { "innerKey": { "a": 1 } } } } {"key": "C", "map": { "key": { "innerKey": { "a": 3 } } } }""", source = """{"key": "A", "map": { "key": { "innerKey": { "a": 2, "b": 3 } } } } {"key": "B", "map": { "key": { "innerKey": { "a": 2, "b": 3 } } } }""", targetSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, MapType(StringType, new StructType().add("a", IntegerType)))), sourceSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, MapType(StringType, new StructType().add("a", IntegerType).add("b", IntegerType)))), resultSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, MapType(StringType, new StructType().add("a", IntegerType).add("b", IntegerType)))), clauses = update("*") :: insert("*") :: Nil, result = """{"key": "A", "map": { "key": { "innerKey": { "a": 2, "b": 3 } } } } {"key": "B", "map": { "key": { "innerKey": { "a": 2, "b": 3 } } } } {"key": "C", "map": { "key": { "innerKey": { "a": 3, "b": null } } } }""", expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("source map struct value contains less columns than target")( target = """{ "key": "A", "map": { "key": { "a": 1, "b": 1 } } } { "key": "C", "map": { "key": { "a": 3, "b": 1 } } }""", source = """{ "key": "A", "map": { "key": { "a": 2 } } } { "key": "B", "map": { "key": { "a": 1 } } }""", targetSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, new StructType().add("a", IntegerType).add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, new StructType().add("a", IntegerType))), resultSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, new StructType().add("a", IntegerType).add("b", IntegerType))), clauses = update("*") :: insert("*") :: Nil, result = """{ "key": "A", "map": { "key": { "a": 2, "b": null } } } { "key": "B", "map": { "key": { "a": 1, "b": null } } } { "key": "C", "map": { "key": { "a": 3, "b": 1 } } }""", expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("source nested map struct value contains less columns than target")( target = """{"key": "A", "map": { "key": { "innerKey": { "a": 1, "b": 1 } } } } {"key": "C", "map": { "key": { "innerKey": { "a": 3, "b": 1 } } } }""", source = """{"key": "A", "map": { "key": { "innerKey": { "a": 2 } } } } {"key": "B", "map": { "key": { "innerKey": { "a": 2 } } } }""", targetSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, MapType(StringType, new StructType().add("a", IntegerType).add("b", IntegerType)))), sourceSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, MapType(StringType, new StructType().add("a", IntegerType)))), resultSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, MapType(StringType, new StructType().add("a", IntegerType).add("b", IntegerType)))), clauses = update("*") :: insert("*") :: Nil, result = """{"key": "A", "map": { "key": { "innerKey": { "a": 2, "b": null } } } } {"key": "B", "map": { "key": { "innerKey": { "a": 2, "b": null } } } } {"key": "C", "map": { "key": { "innerKey": { "a": 3, "b": 1 } } } }""", expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("source nested map struct value contains different type than target")( target = """{"key": "A", "map": { "key": { "a": 1, "b" : 1 } } } {"key": "C", "map": { "key": { "a": 3, "b" : 1 } } }""", source = """{"key": "A", "map": { "key": { "a": 1, "b" : "2" } } } {"key": "B", "map": { "key": { "a": 2, "b" : "2" } } }""", targetSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, new StructType().add("a", IntegerType).add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, new StructType().add("a", IntegerType).add("b", StringType))), resultSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, new StructType().add("a", IntegerType).add("b", IntegerType))), clauses = update("*") :: insert("*") :: Nil, result = """{"key": "A", "map": { "key": { "a": 1, "b" : 2 } } } {"key": "B", "map": { "key": { "a": 2, "b" : 2 } } } {"key": "C", "map": { "key": { "a": 3, "b" : 1 } } }""", resultWithoutEvolution = """{"key": "A", "map": { "key": { "a": 1, "b" : 2 } } } {"key": "B", "map": { "key": { "a": 2, "b" : 2 } } } {"key": "C", "map": { "key": { "a": 3, "b" : 1 } } }""") testNestedStructsEvolution("source nested map struct value in different order")( target = """{"key": "A", "map": { "key": { "a" : 1, "b" : 1 } } } {"key": "C", "map": { "key": { "a" : 3, "b" : 1 } } }""", source = """{"key": "A", "map": { "key": { "b" : 2, "a" : 1, "c" : 3 } } } {"key": "B", "map": { "key": { "b" : 2, "a" : 2, "c" : 4 } } }""", targetSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, new StructType().add("a", IntegerType).add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, new StructType().add("b", IntegerType).add("a", IntegerType).add("c", IntegerType))), resultSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, new StructType().add("a", IntegerType).add("b", IntegerType).add("c", IntegerType))), clauses = update("*") :: insert("*") :: Nil, result = """{"key": "A", "map": { "key": { "a": 1, "b" : 2, "c" : 3 } } } {"key": "B", "map": { "key": { "a": 2, "b" : 2, "c" : 4 } } } {"key": "C", "map": { "key": { "a": 3, "b" : 1, "c" : null } } }""", expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("source map struct value to map array value")( target = """{ "key": "A", "map": { "key": [ 1, 2 ] } } { "key": "C", "map": { "key": [ 3, 4 ] } }""", source = """{ "key": "A", "map": { "key": { "a": 2 } } } { "key": "B", "map": { "key": { "a": 1 } } }""", targetSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, ArrayType(IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, new StructType().add("a", IntegerType))), clauses = update("*") :: insert("*") :: Nil, expectErrorContains = "Failed to merge incompatible data types", expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("source struct nested in map array values contains more columns in different order")( target = """{ "key": "A", "map": { "key": [ { "a": 1, "b": 2 } ] } } { "key": "C", "map": { "key": [ { "a": 3, "b": 4 } ] } }""", source = """{ "key": "A", "map": { "key": [ { "b": 6, "c": 7, "a": 5 } ] } } { "key": "B", "map": { "key": [ { "b": 9, "c": 10, "a": 8 } ] } }""", targetSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, ArrayType( new StructType().add("a", IntegerType).add("b", IntegerType)))), sourceSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, ArrayType( new StructType().add("b", IntegerType).add("c", IntegerType).add("a", IntegerType)))), resultSchema = new StructType() .add("key", StringType) .add("map", MapType( StringType, ArrayType( new StructType().add("a", IntegerType).add("b", IntegerType).add("c", IntegerType)))), clauses = update("*") :: insert("*") :: Nil, result = """{ "key": "A", "map": { "key": [ { "a": 5, "b": 6, "c": 7 } ] } } { "key": "B", "map": { "key": [ { "a": 8, "b": 9, "c": 10 } ] } } { "key": "C", "map": { "key": [ { "a": 3, "b": 4, "c": null } ] } }""", expectErrorWithoutEvolutionContains = "Cannot cast") // Struct evolution inside of map keys. testEvolution("new source column in map struct key")( targetData = Seq((1, 2, 3, 4), (3, 5, 6, 7)).toDF("key", "a", "b", "value") .selectExpr("key", "map(named_struct('a', a, 'b', b), value) as x"), sourceData = Seq((1, 10, 30, 50, 1), (2, 20, 40, 60, 2)).toDF("key", "a", "b", "c", "value") .selectExpr("key", "map(named_struct('a', a, 'b', b, 'c', c), value) as x"), clauses = update("*") :: insert("*") :: Nil, expected = Seq((1, 10, 30, 50, 1), (2, 20, 40, 60, 2), (3, 5, 6, null, 7)) .asInstanceOf[List[(Integer, Integer, Integer, Integer, Integer)]] .toDF("key", "a", "b", "c", "value") .selectExpr("key", "map(named_struct('a', a, 'b', b, 'c', c), value) as x"), expectErrorWithoutEvolutionContains = "Cannot cast" ) testEvolution("source nested map struct key contains less columns than target")( targetData = Seq((1, 2, 3, 4, 5), (3, 6, 7, 8, 9)).toDF("key", "a", "b", "c", "value") .selectExpr("key", "map(named_struct('a', a, 'b', b, 'c', c), value) as x"), sourceData = Seq((1, 10, 50, 1), (2, 20, 60, 2)).toDF("key", "a", "c", "value") .selectExpr("key", "map(named_struct('a', a, 'c', c), value) as x"), clauses = update("*") :: insert("*") :: Nil, expected = Seq((1, 10, null, 50, 1), (2, 20, null, 60, 2), (3, 6, 7, 8, 9)) .asInstanceOf[List[(Integer, Integer, Integer, Integer, Integer)]] .toDF("key", "a", "b", "c", "value") .selectExpr("key", "map(named_struct('a', a, 'b', b, 'c', c), value) as x"), expectErrorWithoutEvolutionContains = "Cannot cast" ) testEvolution("source nested map struct key contains different type than target")( targetData = Seq((1, 2, 3, 4), (3, 5, 6, 7)).toDF("key", "a", "b", "value") .selectExpr("key", "map(named_struct('a', a, 'b', b), value) as x"), sourceData = Seq((1, 10, "30", 1), (2, 20, "40", 2)).toDF("key", "a", "b", "value") .selectExpr("key", "map(named_struct('a', a, 'b', b), value) as x"), clauses = update("*") :: insert("*") :: Nil, expected = Seq((1, 10, 30, 1), (2, 20, 40, 2), (3, 5, 6, 7)) .asInstanceOf[List[(Integer, Integer, Integer, Integer)]] .toDF("key", "a", "b", "value") .selectExpr("key", "map(named_struct('a', a, 'b', b), value) as x"), expectedWithoutEvolution = Seq((1, 10, 30, 1), (2, 20, 40, 2), (3, 5, 6, 7)) .asInstanceOf[List[(Integer, Integer, Integer, Integer)]] .toDF("key", "a", "b", "value") .selectExpr("key", "map(named_struct('a', a, 'b', b), value) as x") ) testEvolution("source nested map struct key in different order")( targetData = Seq((1, 2, 3, 4), (3, 5, 6, 7)).toDF("key", "a", "b", "value") .selectExpr("key", "map(named_struct('a', a, 'b', b), value) as x"), sourceData = Seq((1, 10, 30, 1), (2, 20, 40, 2)).toDF("key", "a", "b", "value") .selectExpr("key", "map(named_struct('b', b, 'a', a), value) as x"), clauses = update("*") :: insert("*") :: Nil, expected = Seq((1, 10, 30, 1), (2, 20, 40, 2), (3, 5, 6, 7)) .asInstanceOf[List[(Integer, Integer, Integer, Integer)]] .toDF("key", "a", "b", "value") .selectExpr("key", "map(named_struct('a', a, 'b', b), value) as x"), expectedWithoutEvolution = Seq((1, 10, 30, 1), (2, 20, 40, 2), (3, 5, 6, 7)) .asInstanceOf[List[(Integer, Integer, Integer, Integer)]] .toDF("key", "a", "b", "value") .selectExpr("key", "map(named_struct('a', a, 'b', b), value) as x") ) testEvolution("struct nested in map array keys contains more columns")( targetData = Seq((1, 2, 3, 4), (3, 5, 6, 7)).toDF("key", "a", "b", "value") .selectExpr("key", "map(array(named_struct('a', a, 'b', b)), value) as x"), sourceData = Seq((1, 10, 30, 50, 1), (2, 20, 40, 60, 2)).toDF("key", "a", "b", "c", "value") .selectExpr("key", "map(array(named_struct('a', a, 'b', b, 'c', c)), value) as x"), clauses = update("*") :: insert("*") :: Nil, expected = Seq((1, 10, 30, 50, 1), (2, 20, 40, 60, 2), (3, 5, 6, null, 7)) .asInstanceOf[List[(Integer, Integer, Integer, Integer, Integer)]] .toDF("key", "a", "b", "c", "value") .selectExpr("key", "map(array(named_struct('a', a, 'b', b, 'c', c)), value) as x"), expectErrorWithoutEvolutionContains = "cannot cast" ) testEvolution("update-only struct nested in map array keys contains more columns")( targetData = Seq((1, 2, 3, 4), (3, 5, 6, 7)).toDF("key", "a", "b", "value") .selectExpr("key", "map(array(named_struct('a', a, 'b', b)), value) as x"), sourceData = Seq((1, 10, 30, 50, 1), (2, 20, 40, 60, 2)).toDF("key", "a", "b", "c", "value") .selectExpr("key", "map(array(named_struct('a', a, 'b', b, 'c', c)), value) as x"), clauses = update("*") :: Nil, expected = Seq((1, 10, 30, 50, 1), (3, 5, 6, null, 7)) .asInstanceOf[List[(Integer, Integer, Integer, Integer, Integer)]] .toDF("key", "a", "b", "c", "value") .selectExpr("key", "map(array(named_struct('a', a, 'b', b, 'c', c)), value) as x"), expectErrorWithoutEvolutionContains = "cannot cast" ) testEvolution("struct evolution in both map keys and values")( targetData = Seq((1, 2, 3, 4, 5), (3, 6, 7, 8, 9)).toDF("key", "a", "b", "d", "e") .selectExpr("key", "map(named_struct('a', a, 'b', b), named_struct('d', d, 'e', e)) as x"), sourceData = Seq((1, 10, 30, 50, 70, 90, 110), (2, 20, 40, 60, 80, 100, 120)) .toDF("key", "a", "b", "c", "d", "e", "f") .selectExpr("key", "map(named_struct('a', a, 'b', b, 'c', c), named_struct('d', d, 'e', e, 'f', f)) as x"), clauses = update("*") :: insert("*") :: Nil, expected = Seq((1, 10, 30, 50, 70, 90, 110), (2, 20, 40, 60, 80, 100, 120), (3, 6, 7, null, 8, 9, null)) .asInstanceOf[List[(Integer, Integer, Integer, Integer, Integer, Integer, Integer)]] .toDF("key", "a", "b", "c", "d", "e", "f") .selectExpr("key", "map(named_struct('a', a, 'b', b, 'c', c), named_struct('d', d, 'e', e, 'f', f)) as x"), expectErrorWithoutEvolutionContains = "cannot cast" ) testEvolution("update only struct evolution in both map keys and values")( targetData = Seq((1, 2, 3, 4, 5), (3, 6, 7, 8, 9)).toDF("key", "a", "b", "d", "e") .selectExpr("key", "map(named_struct('a', a, 'b', b), named_struct('d', d, 'e', e)) as x"), sourceData = Seq((1, 10, 30, 50, 70, 90, 110), (2, 20, 40, 60, 80, 100, 120)) .toDF("key", "a", "b", "c", "d", "e", "f") .selectExpr("key", "map(named_struct('a', a, 'b', b, 'c', c), named_struct('d', d, 'e', e, 'f', f)) as x"), clauses = update("*") :: Nil, expected = Seq((1, 10, 30, 50, 70, 90, 110), (3, 6, 7, null, 8, 9, null)) .asInstanceOf[List[(Integer, Integer, Integer, Integer, Integer, Integer, Integer)]] .toDF("key", "a", "b", "c", "d", "e", "f") .selectExpr("key", "map(named_struct('a', a, 'b', b, 'c', c), named_struct('d', d, 'e', e, 'f', f)) as x"), expectErrorWithoutEvolutionContains = "cannot cast" ) // scalastyle:on line.size.limit } trait MergeIntoNestedStructEvolutionInsertTests extends MergeIntoSchemaEvolutionMixin { self: MergeIntoTestUtils with SharedSparkSession => import testImplicits._ // Nested Schema evolution with INSERT alone testNestedStructsEvolution("new nested source field added when inserting top-level column")( target = """{ "key": "A", "value": { "a": 1 } }""", source = """{ "key": "B", "value": { "a": 2, "b": 3 } }""", targetSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("a", IntegerType)), sourceSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("a", IntegerType) .add("b", IntegerType)), clauses = insert("(value) VALUES (s.value)") :: Nil, result = """{ "key": "A", "value": { "a": 1, "b": null } } { "key": null, "value": { "a": 2, "b": 3 } }""".stripMargin, resultSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("a", IntegerType) .add("b", IntegerType)), expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("insert new nested source field not supported")( target = """{ "key": "A", "value": { "a": 1 } }""", source = """{ "key": "A", "value": { "a": 2, "b": 3, "c": 4 } }""", targetSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("a", IntegerType)), sourceSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("a", IntegerType) .add("b", IntegerType) .add("c", IntegerType)), clauses = insert("(value.b) VALUES (s.value.b)") :: Nil, expectErrorContains = "Nested field is not supported in the INSERT clause of MERGE operation", expectErrorWithoutEvolutionContains = "No such struct field") // scalastyle:off line.size.limit testNestedStructsEvolution("new nested column with update non-* and insert * - array of struct - longer source")( target = """{ "key": "A", "value": [ { "a": { "x": 1, "y": 2 }, "b": 1, "c": 2 } ] }""", source = """{ "key": "A", "value": [ { "b": "2", "a": { "y": 20, "x": 10 } }, { "b": "3", "a": { "y": 30, "x": 20 } }, { "b": "4", "a": { "y": 30, "x": 20 } } ] } { "key": "B", "value": [ { "b": "3", "a": { "y": 30, "x": 40 } }, { "b": "4", "a": { "y": 30, "x": 40 } } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("y", IntegerType)) .add("b", IntegerType) .add("c", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType().add("y", IntegerType).add("x", IntegerType)))), clauses = update("value = s.value") :: insert("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "x": 10, "y": 20 }, "b": 2, "c": null}, { "a": { "x": 20, "y": 30}, "b": 3, "c": null }, { "a": { "x": 20, "y": 30}, "b": 4, "c": null } ] } { "key": "B", "value": [ { "a": { "x": 40, "y": 30 }, "b": 3, "c": null }, { "a": { "x": 40, "y": 30}, "b": 4, "c": null } ] }""".stripMargin, expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("new nested column with update non-* and insert * - array of struct - longer target")( target = """{ "key": "A", "value": [ { "a": { "x": 1, "y": 2 }, "b": 1, "c": 2 }, { "a": { "x": 3, "y": 2 }, "b": 2, "c": 2 } ] }""", source = """{ "key": "A", "value": [ { "b": "2", "a": { "y": 20, "x": 10 } } ] } { "key": "B", "value": [ { "b": "3", "a": { "y": 30, "x": 40 } } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("y", IntegerType)) .add("b", IntegerType) .add("c", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType().add("y", IntegerType).add("x", IntegerType)))), clauses = update("value = s.value") :: insert("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "x": 10, "y": 20}, "b": 2, "c": null } ] } { "key": "B", "value": [ { "a": { "x": 40, "y": 30}, "b": 3, "c": null } ] }""".stripMargin, expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("new nested column with update non-* and insert * - nested array of struct - longer source")( target = """{ "key": "A", "value": [ { "a": { "y": 2, "x": [ { "c": 1, "d": 3 } ] }, "b": 1, "c": 4 } ] }""", source = """{ "key": "A", "value": [ { "b": "2", "a": {"x": [ { "d": "30", "c": 10 }, { "d": "20", "c": 10 }, { "d": "20", "c": 10 } ], "y": 20 } } ] } { "key": "B", "value": [ { "b": "3", "a": {"x": [ { "d": "50", "c": 20 }, { "d": "20", "c": 10 } ], "y": 60 } } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType) ))) .add("b", IntegerType) .add("c", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType() .add("x", ArrayType( new StructType() .add("d", StringType) .add("c", IntegerType) )) .add("y", IntegerType)))), clauses = update("value = s.value") :: insert("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": 30 }, { "c": 10, "d": 20 }, { "c": 10, "d": 20 } ] }, "b": 2, "c": null}] } { "key": "B", "value": [ { "a": { "y": 60, "x": [ { "c": 20, "d": 50 }, { "c": 10, "d": 20 } ] }, "b": 3, "c": null } ] }""", expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("new nested column with update non-* and insert * - nested array of struct - longer target")( target = """{ "key": "A", "value": [ { "a": { "y": 2, "x": [ { "c": 1, "d": 3}, { "c": 2, "d": 3 } ] }, "b": 1, "c": 4 } ] }""", source = """{ "key": "A", "value": [ { "b": "2", "a": {"x": [ { "d": "30", "c": 10 } ], "y": 20 } } ] } { "key": "B", "value": [ { "b": "3", "a": {"x": [ { "d": "50", "c": 20 } ], "y": 60 } } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType) ))) .add("b", IntegerType) .add("c", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType() .add("x", ArrayType( new StructType() .add("d", StringType) .add("c", IntegerType) )) .add("y", IntegerType)))), clauses = update("value = s.value") :: insert("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": 30} ] }, "b": 2, "c": null}] } { "key": "B", "value": [ { "a": { "y": 60, "x": [ { "c": 20, "d": 50 } ] }, "b": 3, "c": null } ] }""", expectErrorWithoutEvolutionContains = "Cannot cast") // scalastyle:on line.size.limit testEvolution("new nested-nested column with update non-* and insert *")( targetData = Seq((1, 1, 2, 3)).toDF("key", "a", "b", "c") .selectExpr("key", "named_struct('y', named_struct('a', a, 'b', b, 'c', c)) as x"), sourceData = Seq((1, 10, 30), (2, 20, 40)).toDF("key", "a", "c") .selectExpr("key", "named_struct('y', named_struct('a', a, 'c', c)) as x"), clauses = update("x.y.a = s.x.y.a") :: insert("*") :: Nil, expected = Seq((1, 10, 2, 3), (2, 20, null, 40)) .asInstanceOf[List[(Integer, Integer, Integer, Integer)]] .toDF("key", "a", "b", "c") .selectExpr("key", "named_struct('y', named_struct('a', a, 'b', b, 'c', c)) as x"), expectErrorWithoutEvolutionContains = "Cannot cast" ) // scalastyle:off line.size.limit testNestedStructsEvolution("new nested-nested column with update non-* and insert * - array of struct - longer source")( target = """{ "key": "A", "value": [ { "a": { "x": 1, "y": 2, "z": 3 }, "b": 1 } ] }""", source = """{ "key": "A", "value": [ { "b": "2", "a": { "y": 20, "x": 10 } }, { "b": "3", "a": { "y": 20, "x": 30 } }, { "b": "4", "a": { "y": 20, "x": 30 } } ] } { "key": "B", "value": [ { "b": "3", "a": { "y": 30, "x": 40 } }, { "b": "4", "a": { "y": 30, "x": 40 } } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("y", IntegerType) .add("z", IntegerType)) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType().add("y", IntegerType).add("x", IntegerType)))), clauses = update("value = s.value") :: insert("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "x": 10, "y": 20, "z": null }, "b": 2 }, { "a": { "x": 30, "y": 20, "z": null }, "b": 3}, { "a": { "x": 30, "y": 20, "z": null }, "b": 4 } ] } { "key": "B", "value": [ { "a": { "x": 40, "y": 30, "z": null }, "b": 3 }, { "a": { "x": 40, "y": 30, "z": null }, "b": 4 } ] }""".stripMargin, expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("new nested-nested column with update non-* and insert * - array of struct - longer target")( target = """{ "key": "A", "value": [ { "a": { "x": 1, "y": 2, "z": 3 }, "b": 1 }, { "a": { "x": 2, "y": 3, "z": 4 }, "b": 1 } ] }""", source = """{ "key": "A", "value": [ { "b": "2", "a": { "y": 20, "x": 10 } } ] } { "key": "B", "value": [ { "b": "3", "a": { "y": 30, "x": 40 } } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("y", IntegerType) .add("z", IntegerType)) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType().add("y", IntegerType).add("x", IntegerType)))), clauses = update("value = s.value") :: insert("*") :: Nil, result = """{ "key": "A", "value": [{ "a": { "x": 10, "y": 20, "z": null }, "b": 2 }] } { "key": "B", "value": [{ "a": { "x": 40, "y": 30, "z": null }, "b": 3 }] }""".stripMargin, expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("new nested-nested column with update non-* and insert * - nested array of struct - longer source")( target = """{ "key": "A", "value": [ { "a": { "y": 2, "x": [ { "c": 1, "d": 3, "e": 1 } ] }, "b": 1 } ] }""", source = """{ "key": "A", "value": [ { "b": "2", "a": {"x": [ { "d": "30", "c": 10 }, { "d": "30", "c": 40 }, { "d": "30", "c": 50 } ], "y": 20 } } ] } { "key": "B", "value": [ { "b": "3", "a": {"x": [ { "d": "50", "c": 20 }, { "d": "50", "c": 30 } ], "y": 60 } } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType) .add("e", IntegerType) ))) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType() .add("x", ArrayType( new StructType() .add("d", StringType) .add("c", IntegerType) )) .add("y", IntegerType)))), clauses = update("value = s.value") :: insert("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": 30, "e": null }, { "c": 40, "d": 30, "e": null }, { "c": 50, "d": 30, "e": null } ] }, "b": 2 } ] } { "key": "B", "value": [ { "a": { "y": 60, "x": [ { "c": 20, "d": 50, "e": null }, { "c": 30, "d": 50, "e": null } ] }, "b": 3 } ] }""", expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("new nested-nested column with update non-* and insert * - nested array of struct - longer target")( target = """{ "key": "A", "value": [ { "a": { "y": 2, "x": [ { "c": 1, "d": 3, "e": 1 }, { "c": 2, "d": 3, "e": 4 } ] }, "b": 1 } ] }""", source = """{ "key": "A", "value": [ { "b": "2", "a": { "x": [ { "d": "30", "c": 10 } ], "y": 20 } } ] } { "key": "B", "value": [ { "b": "3", "a": { "x": [ { "d": "50", "c": 20 } ], "y": 60 } } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType) .add("e", IntegerType) ))) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType() .add("x", ArrayType( new StructType() .add("d", StringType) .add("c", IntegerType) )) .add("y", IntegerType)))), clauses = update("value = s.value") :: insert("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": 30, "e": null } ] }, "b": 2 } ] } { "key": "B", "value": [ { "a": { "y": 60, "x": [ { "c": 20, "d": 50, "e": null } ] }, "b": 3 } ] }""", expectErrorWithoutEvolutionContains = "Cannot cast") // scalastyle:on line.size.limit // Note that the obvious dual of this test, "update * and insert non-*", doesn't exist // because nested columns can't be explicitly INSERTed to. testEvolution("new nested column with update non-* and insert *")( targetData = Seq((1, 1, 2, 3)).toDF("key", "a", "b", "c") .selectExpr("key", "named_struct('a', a, 'b', b, 'c', c) as x"), sourceData = Seq((1, 10, 30), (2, 20, 40)).toDF("key", "a", "c") .selectExpr("key", "named_struct('a', a, 'c', c) as x"), clauses = update("x.a = s.x.a") :: insert("*") :: Nil, expected = Seq((1, 10, 2, 3), (2, 20, null, 40)) .asInstanceOf[List[(Integer, Integer, Integer, Integer)]] .toDF("key", "a", "b", "c") .selectExpr("key", "named_struct('a', a, 'b', b, 'c', c) as x"), expectErrorWithoutEvolutionContains = "Cannot cast" ) // scalastyle:off line.size.limit testNestedStructsEvolution("missing nested column resolved by name - insert - array of struct")( target = """{ "key": "A", "value": [ { "a": { "x": 1, "y": 2, "z": 1 }, "b": 1 } ] }""", source = """{ "key": "A", "value": [ { "a": { "x": 10, "z": 20 }, "b": "2" } ] } { "key": "B", "value": [ { "a": { "x": 40, "z": 30 }, "b": "3" }, { "a": { "x": 40, "z": 30 }, "b": "4" }, { "a": { "x": 40, "z": 30 }, "b": "5" } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("y", IntegerType) .add("z", IntegerType)) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("z", IntegerType)) .add("b", StringType))), clauses = insert("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "x": 1, "y": 2, "z": 1 }, "b": 1 } ] } { "key": "B", "value": [ { "a": { "x": 40, "y": null, "z": 30 }, "b": 3 }, { "a": { "x": 40, "y": null, "z": 30 }, "b": 4 }, { "a": { "x": 40, "y": null, "z": 30 }, "b": 5 } ] }""", expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("missing nested column resolved by name - insert - nested array of struct")( target = """{ "key": "A", "value": [ { "a": { "y": 2, "x": [ { "c": 1, "d": 3, "e": 1 } ] }, "b": 1 } ] }""", source = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "e": "30" } ] }, "b": "2" } ] } { "key": "B", "value": [ {"a": {"y": 60, "x": [ { "c": 20, "e": "50" }, { "c": 20, "e": "60" }, { "c": 20, "e": "80" } ] }, "b": "3" } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType) .add("e", IntegerType) ))) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("e", StringType) ))) .add("b", StringType))), clauses = insert("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "y": 2, "x": [ { "c": 1, "d": 3, "e": 1 } ] }, "b": 1 } ] } { "key": "B", "value": [ { "a": { "y": 60, "x": [ { "c": 20, "d": null, "e": 50 }, { "c": 20, "d": null, "e": 60 }, { "c": 20, "d": null, "e": 80 } ] }, "b": 3 } ] }""", expectErrorWithoutEvolutionContains = "Cannot cast") // scalastyle:on line.size.limit testEvolution("additional nested column in source resolved by name - insert")( targetData = Seq((1, 10, 30)).toDF("key", "a", "c") .selectExpr("key", "named_struct('a', a, 'c', c) as x"), sourceData = Seq((2, 20, 30, 40)).toDF("key", "a", "b", "c") .selectExpr("key", "named_struct('a', a, 'b', b, 'c', c) as x"), clauses = insert("*") :: Nil, expected = ((1, (10, null, 30)) +: ((2, (20, 30, 40)) +: Nil)) .asInstanceOf[List[(Integer, (Integer, Integer, Integer))]].toDF("key", "x") .selectExpr("key", "named_struct('a', x._1, 'c', x._3, 'b', x._2) as x"), expectErrorWithoutEvolutionContains = "Cannot cast" ) // scalastyle:off line.size.limit testNestedStructsEvolution("additional nested column in source resolved by name - insert - array of struct")( target = """{ "key": "A", "value": [ { "a": { "x": 1, "z": 2 }, "b": 1 } ] }""", source = """{ "key": "A", "value": [ { "a": { "x": 10, "y": 20, "z": 2 }, "b": "2 "} ] } { "key": "B", "value": [ {"a": { "x": 40, "y": 30, "z": 3 }, "b": "3" }, {"a": { "x": 40, "y": 30, "z": 3 }, "b": "4" } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("z", IntegerType)) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType).add("y", IntegerType).add("z", IntegerType)) .add("b", StringType))), clauses = insert("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "x": 1, "z": 2, "y": null }, "b": 1 } ] } { "key": "B", "value": [ { "a": { "x": 40, "z": 3, "y": 30 }, "b": 3 }, { "a": { "x": 40, "z": 3, "y": 30 }, "b": 4 } ] }""", resultSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("z", IntegerType) .add("y", IntegerType)) .add("b", IntegerType))), expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("additional nested column in source resolved by name - insert - nested array of struct")( target = """{ "key": "A", "value": [ { "a": { "y": 2, "x": [ { "c": 1, "e": 3 } ] }, "b": 1 } ] }""", source = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": "30", "e": 1 } ] }, "b": "2" } ] } { "key": "B", "value": [ { "a": { "y": 60, "x": [ { "c": 20, "d": "50", "e": 2 }, { "c": 20, "d": "50", "e": 3 } ] }, "b": "3" } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("e", IntegerType) ))) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", StringType) .add("e", IntegerType) ))) .add("b", StringType))), resultSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("e", IntegerType) .add("d", StringType) ))) .add("b", IntegerType))), clauses = insert("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "y": 2, "x": [ { "c": 1, "e": 3, "d": null } ] }, "b": 1 } ] } { "key": "B", "value": [ { "a": { "y": 60, "x": [ { "c": 20, "e": 2, "d": "50" }, { "c": 20, "e": 3, "d": "50" } ] }, "b": 3 } ] }""", expectErrorWithoutEvolutionContains = "Cannot cast") // scalastyle:on line.size.limit testEvolution("nested columns resolved by name with same column count but different names")( targetData = Seq((1, 1, 2, 3)).toDF("key", "a", "b", "c") .selectExpr("key", "struct(a, b, c) as x"), sourceData = Seq((1, 10, 20, 30), (2, 20, 30, 40)).toDF("key", "a", "b", "d") .selectExpr("key", "struct(a, b, d) as x"), clauses = update("*") :: insert("*") :: Nil, // We evolve to the schema (key, x.{a, b, c, d}). expected = ((1, (10, 20, 3, 30)) +: (2, (20, 30, null, 40)) +: Nil) .asInstanceOf[List[(Integer, (Integer, Integer, Integer, Integer))]] .toDF("key", "x") .selectExpr("key", "named_struct('a', x._1, 'b', x._2, 'c', x._3, 'd', x._4) as x"), expectErrorWithoutEvolutionContains = "All nested columns must match." ) // scalastyle:off line.size.limit testNestedStructsEvolution("nested columns resolved by name with same column count but different names - array of struct - longer source")( target = """{ "key": "A", "value": [ { "a": { "x": 1, "y": 2, "o": 4 }, "b": 1 } ] }""", source = """{ "key": "A", "value": [ { "a": { "x": 10, "y": 20, "z": 2 }, "b": "2" }, { "a": { "x": 10, "y": 20, "z": 2 }, "b": "3" }, { "a": { "x": 10, "y": 20, "z": 2 }, "b": "4" } ] } { "key": "B", "value": [ {"a": { "x": 40, "y": 30, "z": 3 }, "b": "3" }, {"a": { "x": 40, "y": 30, "z": 3 }, "b": "4" } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("y", IntegerType) .add("o", IntegerType)) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType).add("y", IntegerType).add("z", IntegerType)) .add("b", StringType))), clauses = update("*") :: insert("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "x": 10, "y": 20, "o": null, "z": 2 }, "b": 2 }, { "a": { "x": 10, "y": 20, "o": null, "z": 2 }, "b": 3 }, { "a": { "x": 10, "y": 20, "o": null, "z": 2 }, "b": 4 } ] } { "key": "B", "value": [ {"a": { "x": 40, "y": 30, "o": null, "z": 3 }, "b": 3 }, {"a": { "x": 40, "y": 30, "o": null, "z": 3 }, "b": 4 } ] }""", resultSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("y", IntegerType) .add("o", IntegerType) .add("z", IntegerType)) .add("b", IntegerType))), expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("nested columns resolved by name with same column count but different names - array of struct - longer target")( target = """{ "key": "A", "value": [ { "a": { "x": 1, "y": 2, "o": 4 }, "b": 1 }, { "a": { "x": 1, "y": 2, "o": 4 }, "b": 2 } ] }""", source = """{ "key": "A", "value": [ { "a": { "x": 10, "y": 20, "z": 2 }, "b": "2" } ] } { "key": "B", "value": [ {"a": { "x": 40, "y": 30, "z": 3 }, "b": "3" } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("y", IntegerType) .add("o", IntegerType)) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType).add("y", IntegerType).add("z", IntegerType)) .add("b", StringType))), clauses = update("*") :: insert("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "x": 10, "y": 20, "o": null, "z": 2 }, "b": 2 } ] } { "key": "B", "value": [ {"a": { "x": 40, "y": 30, "o": null, "z": 3 }, "b": 3 } ] }""", resultSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("y", IntegerType) .add("o", IntegerType) .add("z", IntegerType)) .add("b", IntegerType))), expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("nested columns resolved by name with same column count but different names - nested array of struct - longer source")( target = """{ "key": "A", "value": [ { "a": { "y": 2, "x": [ { "c": 1, "d": 3, "f": 4 } ] }, "b": 1 } ] }""", source = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": "30", "e": 1 }, { "c": 10, "d": "30", "e": 2 }, { "c": 10, "d": "30", "e": 3 } ] }, "b": "2" } ] } { "key": "B", "value": [ { "a": { "y": 60, "x": [ { "c": 20, "d": "50", "e": 2 }, { "c": 20, "d": "50", "e": 3 } ] }, "b": "3" } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType) .add("f", IntegerType) ))) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", StringType) .add("e", IntegerType) ))) .add("b", StringType))), resultSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType) .add("f", IntegerType) .add("e", IntegerType) ))) .add("b", IntegerType))), clauses = update("*") :: insert("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": 30, "f": null, "e": 1 }, { "c": 10, "d": 30, "f": null, "e": 2 }, { "c": 10, "d": 30, "f": null, "e": 3 } ] }, "b": 2 } ] } { "key": "B", "value": [ { "a": { "y": 60, "x": [ { "c": 20, "d": 50, "f": null, "e": 2 }, { "c": 20, "d": 50, "f": null, "e": 3 } ] }, "b": 3} ] }""", expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("nested columns resolved by name with same column count but different names - nested array of struct - longer target")( target = """{ "key": "A", "value": [ { "a": { "y": 2, "x": [ { "c": 1, "d": 3, "f": 4 }, { "c": 1, "d": 3, "f": 4 } ] }, "b": 1 } ] }""", source = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": "30", "e": 1 } ] }, "b": "2" } ] } { "key": "B", "value": [ { "a": { "y": 60, "x": [ { "c": 20, "d": "50", "e": 2 } ] }, "b": "3" } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType) .add("f", IntegerType) ))) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", StringType) .add("e", IntegerType) ))) .add("b", StringType))), resultSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType) .add("f", IntegerType) .add("e", IntegerType) ))) .add("b", IntegerType))), clauses = update("*") :: insert("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": 30, "f": null, "e": 1 } ] }, "b": 2 } ] } { "key": "B", "value": [ { "a": { "y": 60, "x": [ { "c": 20, "d": 50, "f": null, "e": 2 } ] }, "b": 3} ] }""", expectErrorWithoutEvolutionContains = "Cannot cast") // scalastyle:on line.size.limit testEvolution("nested columns resolved by position with same column count but different names")( targetData = Seq((1, 1, 2, 3)).toDF("key", "a", "b", "c") .selectExpr("key", "struct(a, b, c) as x"), sourceData = Seq((1, 10, 20, 30), (2, 20, 30, 40)).toDF("key", "a", "b", "d") .selectExpr("key", "struct(a, b, d) as x"), clauses = update("*") :: insert("*") :: Nil, expectErrorContains = "cannot cast", expectedWithoutEvolution = ((1, (10, 20, 30)) +: (2, (20, 30, 40)) +: Nil) .asInstanceOf[List[(Integer, (Integer, Integer, Integer))]] .toDF("key", "x") .selectExpr("key", "named_struct('a', x._1, 'b', x._2, 'c', x._3) as x"), confs = (DeltaSQLConf.DELTA_RESOLVE_MERGE_UPDATE_STRUCTS_BY_NAME.key, "false") +: Nil ) // scalastyle:off line.size.limit testNestedStructsEvolution("nested columns resolved by position with same column count but different names - array of struct - longer source")( target = """{ "key": "A", "value": [{ "a": { "x": 1, "y": 2, "o": 4 }, "b": 1 } ] }""", source = """{ "key": "A", "value": [ { "a": { "x": 10, "y": 20, "z": 2 }, "b": "2" }, { "a": { "x": 10, "y": 20, "z": 3 }, "b": "2" }, { "a": { "x": 10, "y": 20, "z": 3 }, "b": "3" } ] } { "key": "B", "value": [ { "a": { "x": 40, "y": 30, "z": 3 }, "b": "3" }, { "a": { "x": 40, "y": 30, "z": 3 }, "b": "4" } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("y", IntegerType) .add("o", IntegerType)) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType).add("y", IntegerType).add("z", IntegerType)) .add("b", StringType))), clauses = update("*") :: insert("*") :: Nil, resultWithoutEvolution = """{ "key": "A", "value": [ { "a": { "x": 10, "y": 20, "o": 2 }, "b": 2 }, { "a": { "x": 10, "y": 20, "o": 3 }, "b": 2 }, { "a": { "x": 10, "y": 20, "o": 3 }, "b": 3 } ] } { "key": "B", "value": [ {"a": { "x": 40, "y": 30, "o": 3 }, "b": 3 }, {"a": { "x": 40, "y": 30, "o": 3 }, "b": 4 } ] }""", expectErrorContains = "cannot cast", confs = (DeltaSQLConf.DELTA_RESOLVE_MERGE_UPDATE_STRUCTS_BY_NAME.key, "false") +: Nil) testNestedStructsEvolution("nested columns resolved by position with same column count but different names - array of struct - longer target")( target = """{ "key": "A", "value": [{ "a": { "x": 1, "y": 2, "o": 4 }, "b": 1}, { "a": { "x": 1, "y": 2, "o": 4 }, "b": 2}] }""", source = """{ "key": "A", "value": [{ "a": { "x": 10, "y": 20, "z": 2 }, "b": "2" } ] } { "key": "B", "value": [{"a": { "x": 40, "y": 30, "z": 3 }, "b": "3" } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("y", IntegerType) .add("o", IntegerType)) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType).add("y", IntegerType).add("z", IntegerType)) .add("b", StringType))), clauses = update("*") :: insert("*") :: Nil, resultWithoutEvolution = """{ "key": "A", "value": [{ "a": { "x": 10, "y": 20, "o": 2}, "b": 2}] } { "key": "B", "value": [{"a": { "x": 40, "y": 30, "o": 3}, "b": 3}] }""", expectErrorContains = "cannot cast", confs = (DeltaSQLConf.DELTA_RESOLVE_MERGE_UPDATE_STRUCTS_BY_NAME.key, "false") +: Nil) testNestedStructsEvolution("nested columns resolved by position with same column count but different names - nested array of struct - longer source")( target = """{ "key": "A", "value": [ { "a": { "y": 2, "x": [ { "c": 1, "d": 3, "f": 4 } ] }, "b": 1 } ] }""", source = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": "30", "e": 1 }, { "c": 10, "d": "30", "e": 2 }, { "c": 10, "d": "30", "e": 3} ] }, "b": "2" } ] } { "key": "B", "value": [ { "a": { "y": 60, "x": [ { "c": 20, "d": "50", "e": 2 }, { "c": 20, "d": "50", "e": 3 } ] }, "b": "3" } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType) .add("f", IntegerType) ))) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", StringType) .add("e", IntegerType) ))) .add("b", StringType))), clauses = update("*") :: insert("*") :: Nil, resultWithoutEvolution = """{ "key": "A", "value": [ { "a": {"y": 20, "x": [ { "c": 10, "d": 30, "f": 1 }, { "c": 10, "d": 30, "f": 2 }, { "c": 10, "d": 30, "f": 3 } ] }, "b": 2 } ] } { "key": "B", "value": [ { "a": {"y": 60, "x": [ { "c": 20, "d": 50, "f": 2 }, { "c": 20, "d": 50, "f": 3 } ] }, "b": 3}]}""", expectErrorContains = "cannot cast", confs = (DeltaSQLConf.DELTA_RESOLVE_MERGE_UPDATE_STRUCTS_BY_NAME.key, "false") +: Nil) testNestedStructsEvolution("nested columns resolved by position with same column count but different names - nested array of struct - longer target")( target = """{ "key": "A", "value": [{ "a": { "y": 2, "x": [ { "c": 1, "d": 3, "f": 5 }, { "c": 1, "d": 3, "f": 6 } ] }, "b": 1}] }""", source = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": "30", "e": 1 } ] }, "b": "2" } ] } { "key": "B", "value": [ { "a": { "y": 60, "x": [ { "c": 20, "d": "50", "e": 2 } ] }, "b": "3" } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType) .add("f", IntegerType) ))) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", StringType) .add("e", IntegerType) ))) .add("b", StringType))), clauses = update("*") :: insert("*") :: Nil, resultWithoutEvolution = """{ "key": "A", "value": [ { "a": {"y": 20, "x": [ { "c": 10, "d": 30, "f": 1 } ] }, "b": 2 } ] } { "key": "B", "value": [ { "a": {"y": 60, "x": [ { "c": 20, "d": 50, "f": 2 } ] }, "b": 3 } ] }""", expectErrorContains = "cannot cast", confs = (DeltaSQLConf.DELTA_RESOLVE_MERGE_UPDATE_STRUCTS_BY_NAME.key, "false") +: Nil) // scalastyle:on line.size.limit testEvolution("struct in different order")( targetData = Seq((1, (1, 10, 100)), (2, (2, 20, 200))).toDF("key", "x") .selectExpr("key", "named_struct('a', x._1, 'b', x._2, 'c', x._3) as x"), sourceData = Seq((1, (1111, 111, 11)), (3, (3333, 333, 33))).toDF("key", "x") .selectExpr("key", "named_struct('c', x._1, 'b', x._2, 'a', x._3) as x"), clauses = update("*") :: insert("*") :: Nil, expected = ((1, (11, 111, 1111)) :: (2, (2, 20, 200)) :: (3, (33, 333, 3333)) :: Nil) .toDF("key", "x") .selectExpr("key", "named_struct('a', x._1, 'b', x._2, 'c', x._3) as x"), expectedWithoutEvolution = ((1, (11, 111, 1111)) :: (2, (2, 20, 200)) :: (3, (33, 333, 3333)) :: Nil) .toDF("key", "x") .selectExpr("key", "named_struct('a', x._1, 'b', x._2, 'c', x._3) as x") ) // scalastyle:off line.size.limit testNestedStructsEvolution("struct in different order - array of struct")( target = """{ "key": "A", "value": [{ "a": { "x": 1, "y": 2 }, "b": 1 }] }""", source = """{ "key": "A", "value": [{ "b": "2", "a": { "y": 20, "x": 10}}, { "b": "3", "a": { "y": 30, "x": 40}}] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType().add("x", IntegerType).add("y", IntegerType)) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType().add("y", IntegerType).add("x", IntegerType)))), clauses = update("*") :: insert("*") :: Nil, result = """{ "key": "A", "value": [{ "a": { "x": 10, "y": 20 }, "b": 2 }, { "a": { "y": 30, "x": 40}, "b": 3 }] }""", resultWithoutEvolution = """{ "key": "A", "value": [{ "a": { "x": 10, "y": 20 }, "b": 2 }, { "a": { "y": 30, "x": 40}, "b": 3 }] }""") testNestedStructsEvolution("struct in different order - nested array of struct")( target = """{ "key": "A", "value": [{ "a": { "y": 2, "x": [{ "c": 1, "d": 3}]}, "b": 1 }] }""", source = """{ "key": "A", "value": [{ "b": "2", "a": {"x": [{ "d": "30", "c": 10}, { "d": "40", "c": 3}], "y": 20}}]}""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType)))) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType() .add("x", ArrayType( new StructType() .add("d", StringType) .add("c", IntegerType) )) .add("y", IntegerType)))), clauses = update("*") :: insert("*") :: Nil, result = """{ "key": "A", "value": [{ "a": { "y": 20, "x": [{ "c": 10, "d": 30}, { "c": 3, "d": 40}]}, "b": 2 }]}""", resultWithoutEvolution = """{ "key": "A", "value": [{ "a": { "y": 20, "x": [{ "c": 10, "d": 30}, { "c": 3, "d": 40}]}, "b": 2 }]}""") // scalastyle:on line.size.limit testNestedStructsEvolution("array of struct with same columns but in different order" + " which can be casted implicitly - by name")( target = """{ "key": "A", "value": [ { "a": 1, "b": 2 } ] }""", source = """{ "key": "A", "value": [ { "b": 4, "a": 3 } ] } { "key": "B", "value": [ { "b": 2, "a": 5 } ] }""".stripMargin, targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", IntegerType) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", IntegerType) .add("a", IntegerType))), clauses = update("*") :: insert("*") :: Nil, result = """{ "key": "A", "value": [ { "a": 3, "b": 4 } ] } { "key": "B", "value": [ { "a": 5, "b": 2 } ] }""".stripMargin, resultWithoutEvolution = """{ "key": "A", "value": [ { "a": 3, "b": 4 } ] } { "key": "B", "value": [ { "a": 5, "b": 2 } ] }""".stripMargin) testNestedStructsEvolution("array of struct with same columns but in different order" + " which can be casted implicitly - by position")( target = """{ "key": "A", "value": [ { "a": 1, "b": 2 } ] }""", source = """{ "key": "A", "value": [ { "b": 4, "a": 3 } ] } { "key": "B", "value": [ { "b": 2, "a": 5 } ] }""".stripMargin, targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", IntegerType) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", IntegerType) .add("a", IntegerType))), clauses = update("*") :: insert("*") :: Nil, result = """{ "key": "A", "value": [ { "a": 4, "b": 3 } ] } { "key": "B", "value": [ { "a": 2, "b": 5 } ] }""".stripMargin, resultWithoutEvolution = """{ "key": "A", "value": [ { "a": 4, "b": 3 } ] } { "key": "B", "value": [ { "a": 2, "b": 5 } ] }""".stripMargin, confs = (DeltaSQLConf.DELTA_RESOLVE_MERGE_UPDATE_STRUCTS_BY_NAME.key, "false") +: Nil) testNestedStructsEvolution("array of struct with same column count but all different names" + " - by name")( target = """{ "key": "A", "value": [ { "a": 1, "b": 2 } ] }""", source = """{ "key": "A", "value": [ { "c": 4, "d": 3 } ] } { "key": "B", "value": [ { "c": 2, "d": 5 } ] }""".stripMargin, targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", IntegerType) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType))), clauses = update("*") :: insert("*") :: Nil, resultSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", IntegerType) .add("b", IntegerType) .add("c", IntegerType) .add("d", IntegerType))), result = """{ "key": "A", "value": [ { "a": null, "b": null, "c": 4, "d": 3 } ] } { "key": "B", "value": [ { "a": null, "b": null, "c": 2, "d": 5 } ] }""".stripMargin, expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("array of struct with same column count but all different names" + " - by position")( target = """{ "key": "A", "value": [ { "a": 1, "b": 2 } ] }""", source = """{ "key": "A", "value": [ { "c": 4, "d": 3 } ] } { "key": "B", "value": [ { "c": 2, "d": 5 } ] }""".stripMargin, targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", IntegerType) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType))), clauses = update("*") :: insert("*") :: Nil, resultWithoutEvolution = """{ "key": "A", "value": [ { "a": 4, "b": 3 } ] } { "key": "B", "value": [ { "a": 2, "b": 5 } ] }""".stripMargin, expectErrorContains = " cannot cast", confs = (DeltaSQLConf.DELTA_RESOLVE_MERGE_UPDATE_STRUCTS_BY_NAME.key, "false") +: Nil) testNestedStructsEvolution("array of struct with same columns but in different order" + " which cannot be casted implicitly - by name")( target = """{ "key": "A", "value": [ { "a": {"c" : 1}, "b": 2 } ] }""", source = """{ "key": "A", "value": [ { "b": 4, "a": {"c" : 3 } } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType().add("c", IntegerType)) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", IntegerType) .add("a", new StructType().add("c", IntegerType)))), clauses = update("*") :: insert("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "c" : 3 }, "b": 4 } ] }""", resultWithoutEvolution = """{ "key": "A", "value": [ { "a": { "c" : 3 }, "b": 4 } ] }""") testNestedStructsEvolution("array of struct with same columns but in different order" + " which cannot be casted implicitly - by position")( target = """{ "key": "A", "value": [ { "a": {"c" : 1}, "b": 2 } ] }""", source = """{ "key": "A", "value": [ { "b": 4, "a": {"c" : 3 } } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType().add("c", IntegerType)) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", IntegerType) .add("a", new StructType().add("c", IntegerType)))), clauses = update("*") :: insert("*") :: Nil, expectErrorContains = " cannot cast", expectErrorWithoutEvolutionContains = " cannot cast", confs = (DeltaSQLConf.DELTA_RESOLVE_MERGE_UPDATE_STRUCTS_BY_NAME.key, "false") +: Nil) testNestedStructsEvolution("array of struct with additional column in target - by name")( target = """{ "key": "A", "value": [ { "a": 1, "b": 2, "c": 3 } ] }""", source = """{ "key": "A", "value": [ { "b": 4, "a": 3 } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", IntegerType) .add("b", IntegerType) .add("c", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", IntegerType) .add("a", IntegerType))), clauses = update("*") :: insert("*") :: Nil, result = """{ "key": "A", "value": [ { "a": 3, "b": 4, "c": null } ] }""", expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("array of struct with additional column in target - by position")( target = """{ "key": "A", "value": [ { "a": 1, "b": 2, "c": 3 } ] }""", source = """{ "key": "A", "value": [ { "b": 4, "a": 3 } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", IntegerType) .add("b", IntegerType) .add("c", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", IntegerType) .add("a", IntegerType))), clauses = update("*") :: insert("*") :: Nil, expectErrorContains = " cannot cast", expectErrorWithoutEvolutionContains = "cannot cast", confs = (DeltaSQLConf.DELTA_RESOLVE_MERGE_UPDATE_STRUCTS_BY_NAME.key, "false") +: Nil) testNestedStructsEvolution("struct with extra source column not used in update, without fix")( target = """{ "key": 1, "value": { "a": 10 } }""", source = """{ "key": 1, "value": { "a": 11, "b": 21 } } { "key": 2, "value": { "a": 12, "b": 22 } }""".stripMargin, targetSchema = new StructType() .add("key", IntegerType) .add("value", new StructType() .add("a", IntegerType)), sourceSchema = new StructType() .add("key", IntegerType) .add("value", new StructType() .add("a", IntegerType) .add("b", IntegerType)), cond = "t.key = s.key", clauses = update(set = "key = 0") :: insert("*") :: Nil, expectErrorContains = "data type mismatch", expectErrorWithoutEvolutionContains = "cannot cast", confs = Seq( DeltaSQLConf.DELTA_MERGE_SCHEMA_EVOLUTION_FIX_NESTED_STRUCT_ALIGNMENT.key -> "false") ) testNestedStructsEvolution("struct with extra source column not used in update")( target = """{ "key": 1, "value": { "a": 10 } }""", source = """{ "key": 1, "value": { "a": 11, "b": 21 } } { "key": 2, "value": { "a": 12, "b": 22 } }""".stripMargin, targetSchema = new StructType() .add("key", IntegerType) .add("value", new StructType() .add("a", IntegerType)), sourceSchema = new StructType() .add("key", IntegerType) .add("value", new StructType() .add("a", IntegerType) .add("b", IntegerType)), cond = "t.key = s.key", clauses = update(set = "key = 0") :: insert("*") :: Nil, result = """{ "key": 0, "value": { "a": 10, "b": null } } { "key": 2, "value": { "a": 12, "b": 22 } }""".stripMargin, resultSchema = new StructType() .add("key", IntegerType) .add("value", new StructType() .add("a", IntegerType) .add("b", IntegerType)), expectErrorWithoutEvolutionContains = "cannot cast", confs = Seq( DeltaSQLConf.DELTA_MERGE_SCHEMA_EVOLUTION_FIX_NESTED_STRUCT_ALIGNMENT.key -> "true") ) testNestedStructsEvolution("array struct with extra source column not used in update")( target = """{ "key": 1, "value": [ { "a": 10 } ] }""", source = """{ "key": 1, "value": [ { "a": 11, "b": 21 } ] } { "key": 2, "value": [ { "a": 12, "b": 22 } ] }""".stripMargin, targetSchema = new StructType() .add("key", IntegerType) .add("value", ArrayType( new StructType() .add("a", IntegerType))), sourceSchema = new StructType() .add("key", IntegerType) .add("value", ArrayType( new StructType() .add("a", IntegerType) .add("b", IntegerType))), cond = "t.key = s.key", clauses = update(set = "key = 0") :: insert("*") :: Nil, result = """{ "key": 0, "value": [ { "a": 10, "b": null } ] } { "key": 2, "value": [ { "a": 12, "b": 22 } ] }""".stripMargin, resultSchema = new StructType() .add("key", IntegerType) .add("value", ArrayType( new StructType() .add("a", IntegerType) .add("b", IntegerType))), expectErrorWithoutEvolutionContains = "cannot cast", confs = Seq( DeltaSQLConf.DELTA_MERGE_SCHEMA_EVOLUTION_FIX_NESTED_STRUCT_ALIGNMENT.key -> "true") ) testNestedStructsEvolution("nested struct with extra source column not used in update")( target = """{ "key": 1, "value": { "a": { "aa": 1 } } }""", source = """{ "key": 1, "value": { "a": { "aa": 11, "bb": 31 }, "b": 21 } } { "key": 2, "value": { "a": { "aa": 12, "bb": 32 }, "b": 22 } }""".stripMargin, targetSchema = new StructType() .add("key", IntegerType) .add("value", new StructType() .add("a", new StructType() .add("aa", IntegerType))), sourceSchema = new StructType() .add("key", IntegerType) .add("value", new StructType() .add("a", new StructType() .add("aa", IntegerType) .add("bb", IntegerType)) .add("b", IntegerType)), cond = "t.key = s.key", clauses = update(set = "key = 0") :: insert("*") :: Nil, result = """{ "key": 0, "value": { "a": { "aa": 1, "bb": null }, "b": null } } { "key": 2, "value": { "a": { "aa": 12, "bb": 32 }, "b": 22 } }""".stripMargin, resultSchema = new StructType() .add("key", IntegerType) .add("value", new StructType() .add("a", new StructType() .add("aa", IntegerType) .add("bb", IntegerType)) .add("b", IntegerType)), expectErrorWithoutEvolutionContains = "cannot cast", confs = Seq( DeltaSQLConf.DELTA_MERGE_SCHEMA_EVOLUTION_FIX_NESTED_STRUCT_ALIGNMENT.key -> "true") ) } trait MergeIntoNestedStructEvolutionUpdateOnlyTests extends MergeIntoSchemaEvolutionMixin { self: MergeIntoTestUtils with SharedSparkSession => import testImplicits._ // Nested Schema evolution with UPDATE alone testNestedStructsEvolution("new nested source field not in update is ignored")( target = """{ "key": "A", "value": { "a": 1 } }""", source = """{ "key": "A", "value": { "a": 2, "b": 3 } }""", targetSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("a", IntegerType)), sourceSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("a", IntegerType) .add("b", IntegerType)), clauses = update("value.a = s.value.a") :: Nil, result = """{ "key": "A", "value": { "a": 2 } }""", resultWithoutEvolution = """{ "key": "A", "value": { "a": 2 } }""") testNestedStructsEvolution("two new nested source fields with update: one added, one ignored")( target = """{ "key": "A", "value": { "a": 1 } }""", source = """{ "key": "A", "value": { "a": 2, "b": 3, "c": 4 } }""", targetSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("a", IntegerType)), sourceSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("a", IntegerType) .add("b", IntegerType) .add("c", IntegerType)), clauses = update("value.b = s.value.b") :: Nil, result = """{ "key": "A", "value": { "a": 1, "b": 3 } }""", resultSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("a", IntegerType) .add("b", IntegerType)), expectErrorWithoutEvolutionContains = "No such struct field") testNestedStructsEvolution("nested void columns are not allowed")( target = """{ "key": "A", "value": { "a": { "x": 1 }, "b": 1 } }""", source = """{ "key": "A", "value": { "a": { "x": 2, "z": null } }""", targetSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("a", new StructType().add("x", IntegerType)) .add("b", IntegerType)), sourceSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("a", new StructType().add("x", IntegerType).add("z", NullType))), clauses = update("*") :: Nil, expectErrorContains = "Cannot add column `value`.`a`.`z` with type VOID", expectErrorWithoutEvolutionContains = "All nested columns must match") for (isPartitioned <- BOOLEAN_DOMAIN) testEvolution(s"extra nested column in source - update, isPartitioned=$isPartitioned")( targetData = Seq((1, (1, 10)), (2, (2, 2000))).toDF("key", "x") .selectExpr("key", "named_struct('a', x._1, 'c', x._2) as x"), sourceData = Seq((1, (10, 100, 1000))).toDF("key", "x") .selectExpr("key", "named_struct('a', x._1, 'b', x._2, 'c', x._3) as x"), clauses = update("*") :: Nil, expected = ((1, (10, 100, 1000)) +: (2, (2, null, 2000)) +: Nil) .asInstanceOf[List[(Integer, (Integer, Integer, Integer))]].toDF("key", "x") .selectExpr("key", "named_struct('a', x._1, 'c', x._3, 'b', x._2) as x"), expectErrorWithoutEvolutionContains = "Cannot cast", partitionCols = if (isPartitioned) Seq("key") else Seq.empty ) testEvolution("extra nested column in source - update, partition on unused column")( targetData = Seq((1, 2, (1, 10)), (2, 2, (2, 2000))).toDF("key", "part", "x") .selectExpr("part", "key", "named_struct('a', x._1, 'c', x._2) as x"), sourceData = Seq((1, 2, (10, 100, 1000))).toDF("key", "part", "x") .selectExpr("key", "part", "named_struct('a', x._1, 'b', x._2, 'c', x._3) as x"), clauses = update("*") :: Nil, expected = ((1, 2, (10, 100, 1000)) +: (2, 2, (2, null, 2000)) +: Nil) .asInstanceOf[List[(Integer, Integer, (Integer, Integer, Integer))]].toDF("key", "part", "x") .selectExpr("part", "key", "named_struct('a', x._1, 'c', x._3, 'b', x._2) as x"), expectErrorWithoutEvolutionContains = "Cannot cast", partitionCols = Seq("part") ) // scalastyle:off line.size.limit testNestedStructsEvolution("extra nested column in source - update - array of struct - longer source")( target = """{ "key": "A", "value": [ { "a": { "x": 1, "y": 2 }, "b": 1 } ] } { "key": "B", "value": [ { "a": { "x": 40, "y": 30 }, "b": 3 } ] }""", source = """{ "key": "A", "value": [ { "a": { "x": 10, "y": 20, "z": 2 }, "b": "2" }, { "a": { "x": 10, "y": 20, "z": 2 }, "b": "3" }, { "a": { "x": 10, "y": 20, "z": 2 }, "b": "4" } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("y", IntegerType)) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType).add("y", IntegerType).add("z", IntegerType)) .add("b", StringType))), clauses = update("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "x": 10, "y": 20, "z": 2 }, "b": 2 }, { "a": { "x": 10, "y": 20, "z": 2 }, "b": 3 }, { "a": { "x": 10, "y": 20, "z": 2 }, "b": 4 } ] } { "key": "B", "value": [ { "a": { "x": 40, "y": 30, "z": null }, "b": 3 } ] }""", resultSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("y", IntegerType) .add("z", IntegerType)) .add("b", IntegerType))), expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("extra nested column in source - update - array of struct - longer target")( target = """{ "key": "A", "value": [ { "a": { "x": 1, "y": 2 }, "b": 1 }, { "a": { "x": 1, "y": 2 }, "b": 2 } ] } { "key": "B", "value": [ { "a": { "x": 40, "y": 30 }, "b": 3 }, { "a": { "x": 40, "y": 30 }, "b": 4 }, { "a": { "x": 40, "y": 30 }, "b": 5 } ] }""", source = """{ "key": "A", "value": [ { "a": { "x": 10, "y": 20, "z": 2 }, "b": "2" } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("y", IntegerType)) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType).add("y", IntegerType).add("z", IntegerType)) .add("b", StringType))), clauses = update("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "x": 10, "y": 20, "z": 2 }, "b": 2 } ] } { "key": "B", "value": [ { "a": { "x": 40, "y": 30, "z": null }, "b": 3 }, { "a": { "x": 40, "y": 30, "z": null }, "b": 4 }, { "a": { "x": 40, "y": 30, "z": null }, "b": 5 } ] }""".stripMargin, resultSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("y", IntegerType) .add("z", IntegerType)) .add("b", IntegerType))), expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("extra nested column in source - update - nested array of struct - longer source")( target = """{ "key": "A", "value": [ { "a": { "y": 2, "x": [ { "c": 1, "d": 3 } ] }, "b": 1 } ] } { "key": "B", "value": [ { "a": { "y": 60, "x": [ { "c": 20, "d": 50 } ] }, "b": 3 } ] }""", source = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": "30", "e": 1 }, { "c": 10, "d": "30", "e": 2 }, { "c": 10, "d": "30", "e": 3 } ] }, "b": 2 } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType) ))) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", StringType) .add("e", IntegerType) ))) .add("b", StringType))), resultSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType) .add("e", IntegerType) ))) .add("b", IntegerType))), clauses = update("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": 30, "e": 1 }, { "c": 10, "d": 30, "e": 2 }, { "c": 10, "d": 30, "e": 3 } ] }, "b": 2 } ] } { "key": "B", "value": [ { "a": { "y": 60, "x": [ { "c": 20, "d": 50, "e": null } ] }, "b": 3 } ] }""", expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("extra nested column in source - update - nested array of struct - longer target")( target = """{ "key": "A", "value": [ { "a": { "y": 2, "x": [ { "c": 1, "d": 3 }, { "c": 1, "d": 2 } ] }, "b": 1 } ] } { "key": "B", "value": [ { "a": { "y": 60, "x": [ { "c": 20, "d": 50 }, { "c": 20, "d": 40 }, { "c": 20, "d": 60 } ] }, "b": 3 } ] }""", source = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": "30", "e": 1 } ] }, "b": "2" } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType) ))) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", StringType) .add("e", IntegerType) ))) .add("b", StringType))), resultSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType) .add("e", IntegerType) ))) .add("b", IntegerType))), clauses = update("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": 30, "e": 1 } ] }, "b": 2 } ] } { "key": "B", "value": [ { "a": { "y": 60, "x": [ { "c": 20, "d": 50, "e": null }, { "c": 20, "d": 40, "e": null }, { "c": 20, "d": 60, "e": null } ] }, "b": 3 } ] }""", expectErrorWithoutEvolutionContains = "Cannot cast") // scalastyle:on line.size.limit testEvolution("missing nested column in source - update")( targetData = Seq((1, (1, 10, 100)), (2, (2, 20, 200))).toDF("key", "x") .selectExpr("key", "named_struct('a', x._1, 'b', x._2, 'c', x._3) as x"), sourceData = Seq((1, (0, 0))).toDF("key", "x") .selectExpr("key", "named_struct('a', x._1, 'c', x._2) as x"), clauses = update("*") :: Nil, expected = ((1, (0, 10, 0)) +: (2, (2, 20, 200)) +: Nil).toDF("key", "x") .selectExpr("key", "named_struct('a', x._1, 'b', x._2, 'c', x._3) as x"), expectErrorWithoutEvolutionContains = "Cannot cast" ) // scalastyle:off line.size.limit testNestedStructsEvolution("missing nested column in source - update - array of struct - longer source")( target = """{ "key": "A", "value": [ { "a": { "x": 1, "y": 2, "z": 3 }, "b": 1 } ] }""", source = """{ "key": "A", "value": [ { "a": { "x": 10, "z": 2 }, "b": "2" }, { "a": { "x": 10, "z": 2 }, "b": "3" }, { "a": { "x": 10, "z": 2 }, "b": "4" } ] } { "key": "B", "value": [ { "a": { "x": 40, "z": 3 }, "b": "3" } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("y", IntegerType) .add("z", IntegerType)) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType).add("z", IntegerType)) .add("b", StringType))), clauses = update("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "x": 10, "y": null, "z": 2 }, "b": 2 }, { "a": { "x": 10, "y": null, "z": 2 }, "b": 3 }, { "a": { "x": 10, "y": null, "z": 2 }, "b": 4 } ] }""", resultSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("y", IntegerType) .add("z", IntegerType)) .add("b", IntegerType))), expectErrorWithoutEvolutionContains = "Cannot cast") // scalastyle:off line.size.limit testNestedStructsEvolution("missing nested column in source - update - array of struct - longer target")( target = """{ "key": "A", "value": [ { "a": { "x": 1, "y": 2, "z": 3 }, "b": 1 }, { "a": { "x": 1, "y": 2, "z": 3 }, "b": 2 } ] }""", source = """{ "key": "A", "value": [ { "a": { "x": 10, "z": 2 }, "b": "2" } ] } { "key": "B", "value": [ { "a": { "x": 40, "z": 3 }, "b": "3" } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("y", IntegerType) .add("z", IntegerType)) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType).add("z", IntegerType)) .add("b", StringType))), clauses = update("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "x": 10, "y": null, "z": 2 }, "b": 2 } ] }""", resultSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("x", IntegerType) .add("y", IntegerType) .add("z", IntegerType)) .add("b", IntegerType))), expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("missing nested column in source - update - nested array of struct - longer source")( target = """{ "key": "A", "value": [ { "a": { "y": 2, "x": [ { "c": 1, "d": 3, "e": 4 } ] }, "b": 1 } ] }""", source = """{ "key": "A", "value": [ { "a": {"y": 20, "x": [ { "c": 10, "e": 1 }, { "c": 10, "e": 2 }, { "c": 10, "e": 3 } ] }, "b": "2" } ] } { "key": "B", "value": [ { "a": {"y": 60, "x": [{ "c": 20, "e": 2 } ] }, "b": "3" } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType) .add("e", IntegerType) ))) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("e", IntegerType) ))) .add("b", StringType))), resultSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType) .add("e", IntegerType) ))) .add("b", IntegerType))), clauses = update("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": null, "e": 1 }, { "c": 10, "d": null, "e": 2 }, { "c": 10, "d": null, "e": 3} ] }, "b": 2 } ] }""", expectErrorWithoutEvolutionContains = "Cannot cast") testNestedStructsEvolution("missing nested column in source - update - nested array of struct - longer target")( target = """{ "key": "A", "value": [ { "a": { "y": 2, "x": [ { "c": 1, "d": 3, "e": 4 }, { "c": 1, "d": 3, "e": 5 } ] }, "b": 1 } ] }""", source = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "e": 1 } ] }, "b": "2" } ] } { "key": "B", "value": [ { "a": { "y": 60, "x": [ { "c": 20, "e": 2 } ] }, "b": "3" } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType) .add("e", IntegerType) ))) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("e", IntegerType) ))) .add("b", StringType))), resultSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType) .add("e", IntegerType) ))) .add("b", IntegerType))), clauses = update("*") :: Nil, result = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": null, "e": 1 } ] }, "b": 2 } ] }""", expectErrorWithoutEvolutionContains = "Cannot cast") // scalastyle:on line.size.limit testNestedStructsEvolution("add non-nullable struct field to target schema")( target = """{ "key": "A" }""", source = """{ "key": "B", "value": 4}""", targetSchema = new StructType() .add("key", StringType), sourceSchema = new StructType() .add("key", StringType) .add("value", IntegerType, nullable = false), clauses = update("*") :: Nil, result = """{ "key": "A", "value": null }""".stripMargin, // Even though `value` is non-nullable in the source, it must be nullable in the target as // existing rows will contain null values. resultSchema = new StructType() .add("key", StringType) .add("value", IntegerType, nullable = true), resultWithoutEvolution = """{ "key": "A" }""") testNestedStructsEvolution("struct in array with storeAssignmentPolicy = STRICT")( target = """{ "key": "A", "value": [ { "a": 1 } ] }""", source = """{ "key": "A", "value": [ { "a": 2 } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType(new StructType() .add("a", LongType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType(new StructType() .add("a", IntegerType))), clauses = update("*") :: Nil, result = """{ "key": "A", "value": [ { "a": 2 } ] }""", resultWithoutEvolution = """{ "key": "A", "value": [ { "a": 2 } ] }""", confs = Seq( SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.STRICT.toString, DeltaSQLConf.UPDATE_AND_MERGE_CASTING_FOLLOWS_ANSI_ENABLED_FLAG.key -> "false")) testNestedStructsEvolution("nested field assignment qualified with source alias")( target = Seq("""{ "a": 1, "t": { "a": 2 } }"""), source = Seq("""{ "a": 3, "t": { "a": 5 } }"""), targetSchema = new StructType() .add("a", IntegerType) .add("t", new StructType() .add("a", IntegerType)), sourceSchema = new StructType() .add("a", IntegerType) .add("t", new StructType() .add("a", IntegerType)), cond = "1 = 1", clauses = update("s.t.a = s.t.a") :: Nil, // Assigning to the source is just wrong and should fail. result = Seq("""{ "a": 1, "t": { "a": 5 } }"""), resultSchema = new StructType() .add("a", IntegerType) .add("t", new StructType() .add("a", IntegerType)), expectErrorWithoutEvolutionContains = "cannot resolve s.t.a in UPDATE") testNestedStructsEvolution("existing top-level column assignment qualified with target alias")( target = Seq("""{ "a": 1, "t": { "a": 2 } }"""), source = Seq("""{ "a": 3, "t": { "a": 5 } }"""), targetSchema = new StructType() .add("a", IntegerType) .add("t", new StructType() .add("a", IntegerType)), sourceSchema = new StructType() .add("a", IntegerType) .add("t", new StructType() .add("a", IntegerType)), cond = "1 = 1", // This succeeds and updates 'a': the fully qualified column name 't.a' gets precedence over // the unqualified struct field name '(t.)t.a' to resolve the ambiguity. clauses = update("t.a = s.a") :: Nil, result = Seq("""{ "a": 3, "t": { "a": 2 } }"""), resultSchema = new StructType() .add("a", IntegerType) .add("t", new StructType() .add("a", IntegerType)), resultWithoutEvolution = Seq("""{ "a": 3, "t": { "a": 2 } }""")) testNestedStructsEvolution("existing nested field assignment qualified with target alias")( target = Seq("""{ "a": 1, "t": { "a": 2 } }"""), source = Seq("""{ "a": 3, "t": { "a": 5 } }"""), targetSchema = new StructType() .add("a", IntegerType) .add("t", new StructType() .add("a", IntegerType)), sourceSchema = new StructType() .add("a", IntegerType) .add("t", new StructType() .add("a", IntegerType)), cond = "1 = 1", // This is unambiguous: and resolves to the struct field 't.t.a' during resolution clauses = update("t.t.a = s.t.a") :: Nil, result = Seq("""{ "a": 1, "t": { "a": 5 } }"""), resultSchema = new StructType() .add("a", IntegerType) .add("t", new StructType() .add("a", IntegerType)), resultWithoutEvolution = Seq("""{ "a": 1, "t": { "a": 5 } }""")) testNestedStructsEvolution("new top-level column assignment qualified with target alias")( target = Seq("""{ "a": 1, "t": { "a": 2 } }"""), source = Seq("""{ "a": 3, "b": 4, "t": { "a": 5, "b": 6 } }"""), targetSchema = new StructType() .add("a", IntegerType) .add("t", new StructType() .add("a", IntegerType)), sourceSchema = new StructType() .add("a", IntegerType) .add("b", IntegerType) .add("t", new StructType() .add("a", IntegerType) .add("b", IntegerType)), cond = "1 = 1", clauses = update("t.b = s.b") :: Nil, result = Seq("""{ "a": 1, "t": { "a": 2, "b": 4 } }"""), resultSchema = new StructType() .add("a", IntegerType) .add("t", new StructType() .add("a", IntegerType) .add("b", IntegerType)), expectErrorWithoutEvolutionContains = "No such struct field `b` in `a`") testNestedStructsEvolution("new nested field assignment qualified with target alias")( target = Seq("""{ "a": 1, "t": { "a": 2 } }"""), source = Seq("""{ "a": 3, "b": 4, "t": { "a": 5, "b": 6 } }"""), targetSchema = new StructType() .add("a", IntegerType) .add("t", new StructType() .add("a", IntegerType)), sourceSchema = new StructType() .add("a", IntegerType) .add("b", IntegerType) .add("t", new StructType() .add("a", IntegerType) .add("b", IntegerType)), cond = "1 = 1", clauses = update("t.t.b = s.t.b") :: Nil, // t.t.b gets resolved to source struct t, accessing nested field t.t.b which doesn't exist. expectErrorContains = "No such struct field `t` in `a`, `b`", resultSchema = new StructType() .add("a", IntegerType) .add("t", new StructType() .add("a", IntegerType) .add("b", IntegerType)), // t.t.b: t.t gets resolved to target t with nested field b which doesn't exist in fields (a) expectErrorWithoutEvolutionContains = "No such struct field `b` in `a`") } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoStructEvolutionNullnessSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.language.implicitConversions import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.sources.DeltaSQLConf import com.fasterxml.jackson.annotation.JsonInclude.Include import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} import com.fasterxml.jackson.module.scala.{ClassTagExtensions, DefaultScalaModule} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{ArrayType, IntegerType, StructType} /** Trait containing common utility methods for struct evolution nullness tests. */ trait MergeIntoStructEvolutionNullnessTestUtils extends MergeHelpers { /** Whether to preserve null source structs for struct evolution tests. */ protected def preserveNullSourceStructs: Boolean = true /** Whether to preserve null source structs for UPDATE * specifically. */ protected def preserveNullSourceStructsUpdateStar: Boolean = true /** Configurations for preserving null source structs. */ protected val preserveNullStructsConfs: Seq[(String, String)] = Seq( DeltaSQLConf.DELTA_MERGE_PRESERVE_NULL_SOURCE_STRUCTS.key -> preserveNullSourceStructs.toString, DeltaSQLConf.DELTA_MERGE_PRESERVE_NULL_SOURCE_STRUCTS_UPDATE_STAR.key -> preserveNullSourceStructsUpdateStar.toString) // `SourceType`, `TargetType` and `ActionType` assume that the source and target tables // have a single top-level column `col` of struct type. protected object SourceType extends Enumeration { case class Val(displayName: String) extends super.Val val NonNullLeaves: Val = Val("non-null source leaves") val NullLeaves: Val = Val("null source leaves") val NullNestedStruct: Val = Val("null source nested struct") val NullNestedArray: Val = Val("null source nested array") val NullNestedMap: Val = Val("null source nested map") val NullCol: Val = Val("null source col") def getName(sourceType: Value): String = sourceType.asInstanceOf[Val].displayName } protected object TargetType extends Enumeration { case class Val(displayName: String) extends super.Val val NonNullLeaves: Val = Val("non-null target leaves") val NullLeaves: Val = Val("null target leaves") val NullNestedStruct: Val = Val("null target nested struct") val NullNestedArray: Val = Val("null source nested array") val NullNestedMap: Val = Val("null source nested map") val NullCol: Val = Val("null target col") val Empty: Val = Val("empty target") def getName(targetType: Value): String = targetType.asInstanceOf[Val].displayName } protected object ActionType extends Enumeration { case class Val(displayName: String, clause: MergeClause) extends super.Val val UpdateStar: Val = Val("UPDATE *", update("*")) val UpdateCol: Val = Val("UPDATE t.col = s.col", update("t.col = s.col")) val UpdateColY: Val = Val("UPDATE t.col.y = s.col.y", update("t.col.y = s.col.y")) val InsertStar: Val = Val("INSERT *", insert("*")) val InsertCol: Val = Val("INSERT col", insert("(key, col) VALUES (s.key, s.col)")) implicit def toVal(v: Value): Val = v.asInstanceOf[Val] def getName(actionType: Value): String = actionType.asInstanceOf[Val].displayName def getClause(actionType: Value): MergeClause = actionType.asInstanceOf[Val].clause def isWholeStructAssignment(actionType: Value): Boolean = Seq(ActionType.UpdateCol, ActionType.InsertStar, ActionType.InsertCol).contains(actionType) } /** Casts Any to Map[String, Any]. Returns null if `value` is null. */ protected def castToMap(value: Any): Map[String, Any] = if (value == null) null else value.asInstanceOf[Map[String, Any]] /** Gets a value from a map. Returns null if the map is null. */ protected def getNestedValue(map: Map[String, Any], key: String): Any = { if (map == null) null else map(key) } /** * JSON mapper that preserves null values during serialization. * This is necessary because the default JsonUtils uses Include.NON_ABSENT which filters out * null values, which is not what we want for the nullness tests. * Uses ClassTagExtensions instead of deprecated ScalaObjectMapper for Scala 3 compatibility. */ private val jsonMapper = { val mapper = new ObjectMapper() with ClassTagExtensions mapper.setSerializationInclusion(Include.ALWAYS) // Preserve null values mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) mapper.registerModule(DefaultScalaModule) mapper } /** * Converts a Scala object to JSON string, preserving null values. * Unlike org.apache.spark.sql.util.JsonUtils.toJson(), this method uses Include.ALWAYS * to ensure null values are preserved in the JSON output. * Uses ClassTag instead of deprecated Manifest for Scala 3 compatibility. */ protected def toJsonWithNulls[T: scala.reflect.ClassTag](obj: T): String = { jsonMapper.writeValueAsString(obj) } /** Parses a JSON string to a Map[String, Any]. */ protected def fromJsonToMap(jsonStr: String): Map[String, Any] = { jsonMapper.readValue[Map[String, Any]](jsonStr) } /** * Determines whether the target struct should be overwritten with null. * * Conditions to set the target struct to NULL: * - For UPDATE *, the source struct is null AND the target struct is null. * - For whole-struct assignment (UPDATE col = s.col, INSERT), the source struct is null. * * @param sourceCol The source column value (can be null) * @param targetColOpt Optional target column value corresponding to sourceCol * @param actionType The action type * @return true if target should be overwritten with null, false otherwise */ protected def shouldOverwriteWithNull( sourceCol: Map[String, Any], targetColOpt: Option[Map[String, Any]], actionType: ActionType.Value): Boolean = { sourceCol == null && preserveNullSourceStructs && ( // `targetColOpt` being None means it's an INSERT targetColOpt.isEmpty || ActionType.isWholeStructAssignment(actionType) || (actionType == ActionType.UpdateStar && preserveNullSourceStructsUpdateStar && targetColOpt.get == null) ) } /** * Checks if null source struct preservation is enabled for UPDATE SET * operations. * @return true if both preserveNullSourceStructs and preserveNullSourceStructsUpdateStar are true */ protected def shouldPreserveNullSourceStructsForUpdateStar: Boolean = { shouldPreserveNullSourceStructsForWholeStructAssignment && preserveNullSourceStructsUpdateStar } /** * Checks if null source struct preservation is enabled for whole-struct assignments. * @return true if preserveNullSourceStructs is true */ protected def shouldPreserveNullSourceStructsForWholeStructAssignment: Boolean = { preserveNullSourceStructs } /** Represents a struct evolution nullness test case. */ protected case class StructEvolutionNullnessTestCase( testName: String, sourceSchema: StructType, targetSchema: StructType, sourceData: String, targetData: Seq[String], actionClause: MergeClause, resultSchema: StructType, expectedResult: String, confs: Seq[(String, String)]) /** * Generates test cases for struct evolution nullness tests. * * @param testNamePrefix Prefix for test names * @param sourceSchema Source table schema * @param targetSchema Target table schema * @param sourceTypes Source types to test * @param updateTargetTypes Target types to use for UPDATE operations * @param actionTypes Action types to test * @param generateResultSchemaFn Function to determine result schema based on action type * @param generateSourceRowFn Function to generate source row * @param generateTargetRowFn Function to generate target row * @param generateExpectedResultFn Function to generate expected result */ protected def generateStructEvolutionNullnessTests( testNamePrefix: String, sourceSchema: StructType, targetSchema: StructType, sourceTypes: Seq[SourceType.Value], updateTargetTypes: Seq[TargetType.Value], actionTypes: Seq[ActionType.Value], generateResultSchemaFn: ActionType.Value => StructType, generateSourceRowFn: SourceType.Value => String, generateTargetRowFn: TargetType.Value => Option[String], generateExpectedResultFn: (String, Option[String], ActionType.Value) => String) : Seq[StructEvolutionNullnessTestCase] = { for { actionType <- actionTypes sourceType <- sourceTypes // For INSERT, only use Empty target; for UPDATE, use specified target types. targetType <- { if (actionType == ActionType.InsertStar || actionType == ActionType.InsertCol) { Seq(TargetType.Empty) } else { updateTargetTypes } } } yield { val sourceRowJson = generateSourceRowFn(sourceType) val targetRowJsonOpt = generateTargetRowFn(targetType) val expectedResultJson = generateExpectedResultFn(sourceRowJson, targetRowJsonOpt, actionType) StructEvolutionNullnessTestCase( testName = s"$testNamePrefix${SourceType.getName(sourceType)}, " + s"${TargetType.getName(targetType)}, " + s"${ActionType.getName(actionType)}", sourceSchema = sourceSchema, targetSchema = targetSchema, sourceData = sourceRowJson, targetData = targetRowJsonOpt.toSeq, actionClause = ActionType.getClause(actionType), resultSchema = generateResultSchemaFn(actionType), expectedResult = expectedResultJson, confs = preserveNullStructsConfs ) } } } /** * Trait collecting tests verifying the nullness of the results for top-level struct evolution. */ trait MergeIntoTopLevelStructEvolutionNullnessTests extends MergeIntoSchemaEvolutionMixin with MergeIntoStructEvolutionNullnessTestUtils { self: MergeIntoTestUtils with SharedSparkSession => private val testNamePrefix = "top-level struct - " private val topLevelStructTargetSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", IntegerType) .add("z", IntegerType)) private val topLevelStructSourceSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("x", IntegerType) .add("y", IntegerType)) private val topLevelStructResultSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", IntegerType) .add("z", IntegerType) .add("x", IntegerType)) private def generateTopLevelStructSourceRow(sourceType: SourceType.Value): String = { sourceType match { case SourceType.NonNullLeaves => """{"key":1,"col":{"x":1,"y":1}}""" case SourceType.NullLeaves => """{"key":1,"col":{"x":null,"y":null}}""" case SourceType.NullCol => """{"key":1,"col":null}""" } } private def generateTopLevelStructTargetRow( targetType: TargetType.Value): Option[String] = { targetType match { case TargetType.NonNullLeaves => Some("""{"key":1,"col":{"y":2,"z":2}}""") case TargetType.NullLeaves => Some("""{"key":1,"col":{"y":null,"z":null}}""") case TargetType.NullCol => Some("""{"key":1,"col":null}""") case TargetType.Empty => None } } /** * Generates the expected result based on `sourceRowJson`, `targetRowJsonOpt`, and `actionType`. * Semantics: * - UPDATE *: field-level merge, preserves target-only fields (col.z). * - UPDATE t.col = s.col, INSERT: whole-struct assignment, nulls target-only fields. */ private def generateTopLevelStructExpectedResult( sourceRowJson: String, targetRowJsonOpt: Option[String], actionType: ActionType.Value): String = { val sourceRow = fromJsonToMap(sourceRowJson) val targetRowOpt = targetRowJsonOpt.map(fromJsonToMap) val sourceCol = castToMap(sourceRow("col")) val targetColOpt = targetRowOpt.map(row => castToMap(row("col"))) if (shouldOverwriteWithNull(sourceCol, targetColOpt, actionType)) { return """{"key":1,"col":null}""" } val sourceX = getNestedValue(sourceCol, "x") val sourceY = getNestedValue(sourceCol, "y") val (resultX, resultY, resultZ) = actionType match { case ActionType.UpdateStar => // UPDATE SET * preserves target-only field (col.z). val targetZ = getNestedValue(targetColOpt.get, "z") (sourceX, sourceY, targetZ) case ActionType.UpdateCol | ActionType.InsertStar | ActionType.InsertCol => // Whole-struct assignments null out target-only field (col.z). (sourceX, sourceY, null) } val resultMap = Map( "key" -> 1, "col" -> Map("y" -> resultY, "z" -> resultZ, "x" -> resultX)) toJsonWithNulls(resultMap) } private def generateTopLevelStructNullnessTests(): Seq[StructEvolutionNullnessTestCase] = { generateStructEvolutionNullnessTests( testNamePrefix = testNamePrefix, sourceSchema = topLevelStructSourceSchema, targetSchema = topLevelStructTargetSchema, sourceTypes = Seq(SourceType.NonNullLeaves, SourceType.NullLeaves, SourceType.NullCol), updateTargetTypes = Seq(TargetType.NonNullLeaves, TargetType.NullLeaves, TargetType.NullCol), actionTypes = Seq( ActionType.UpdateStar, ActionType.UpdateCol, ActionType.InsertStar, ActionType.InsertCol), generateResultSchemaFn = _ => topLevelStructResultSchema, generateSourceRowFn = generateTopLevelStructSourceRow, generateTargetRowFn = generateTopLevelStructTargetRow, generateExpectedResultFn = generateTopLevelStructExpectedResult ) } generateTopLevelStructNullnessTests().foreach { testCase => testNestedStructsEvolution(testCase.testName)( target = testCase.targetData, source = Seq(testCase.sourceData), targetSchema = testCase.targetSchema, sourceSchema = testCase.sourceSchema, clauses = testCase.actionClause :: Nil, result = Seq(testCase.expectedResult), resultSchema = testCase.resultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = testCase.confs) } testNestedStructsEvolution( s"${testNamePrefix}null expansion - UPDATE * with null target col")( target = Seq("""{"key":1,"col":null}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = topLevelStructTargetSchema, sourceSchema = topLevelStructSourceSchema, clauses = update("*") :: Nil, result = Seq( if (shouldPreserveNullSourceStructsForUpdateStar) { """{"key":1,"col":null}""" } else { """{"key":1,"col":{"y":null,"z":null,"x":null}}""" } ), resultSchema = topLevelStructResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution( s"${testNamePrefix}null expansion - UPDATE * with non-null target col")( target = Seq("""{"key":1,"col":{"y":null,"z":null}}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = topLevelStructTargetSchema, sourceSchema = topLevelStructSourceSchema, clauses = update("*") :: Nil, result = Seq("""{"key":1,"col":{"y":null,"z":null,"x":null}}"""), resultSchema = topLevelStructResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) // The following tests verify that we overwrite the target struct with NULL if there // are no target-only fields. private val topLevelStructTargetSchemaWithoutTargetOnlyField = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("Y", IntegerType)) // Use uppercase to verify case-insensitive comparison. testNestedStructsEvolution( s"${testNamePrefix}null expansion - " + s"UPDATE * with non-null target col without target-only field")( target = Seq("""{"key":1,"col":{"Y":null}}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = topLevelStructTargetSchemaWithoutTargetOnlyField, sourceSchema = topLevelStructSourceSchema, clauses = update("*") :: Nil, result = Seq( if (shouldPreserveNullSourceStructsForUpdateStar) { """{"key":1,"col":null}""" } else { """{"key":1,"col":{"y":null,"x":null}}""" } ), resultSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("Y", IntegerType) .add("x", IntegerType)), expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) private val expectedResult = if (shouldPreserveNullSourceStructsForWholeStructAssignment) { """{"key":1,"col":null}""" } else { """{"key":1,"col":{"y":null,"z":null,"x":null}}""" } testNestedStructsEvolution(s"${testNamePrefix}null expansion - UPDATE t.col = s.col")( target = Seq("""{"key":1,"col":{"y":2,"z":2}}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = topLevelStructTargetSchema, sourceSchema = topLevelStructSourceSchema, clauses = update("t.col = s.col") :: Nil, result = Seq(expectedResult), resultSchema = topLevelStructResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution(s"${testNamePrefix}null expansion - INSERT *")( target = Seq.empty, source = Seq("""{"key":1,"col":null}"""), targetSchema = topLevelStructTargetSchema, sourceSchema = topLevelStructSourceSchema, clauses = insert("*") :: Nil, result = Seq(expectedResult), resultSchema = topLevelStructResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution(s"${testNamePrefix}null expansion - INSERT (key, col)")( target = Seq.empty, source = Seq("""{"key":1,"col":null}"""), targetSchema = topLevelStructTargetSchema, sourceSchema = topLevelStructSourceSchema, clauses = insert("(key, col) VALUES (s.key, s.col)") :: Nil, result = Seq(expectedResult), resultSchema = topLevelStructResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution( s"${testNamePrefix}non-nullable target struct becomes nullable")( target = Seq("""{"key":1,"col":{"y":2,"z":2}}"""), source = Seq("""{"key":1,"col":null}"""), // Target schema has non-nullable struct targetSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", IntegerType) .add("z", IntegerType), nullable = false), sourceSchema = topLevelStructSourceSchema, clauses = update("t.col = s.col") :: Nil, result = Seq(if (shouldPreserveNullSourceStructsForWholeStructAssignment) { """{"key":1,"col":null}""" } else { """{"key":1,"col":{"y":null,"z":null,"x":null}}""" }), // Result schema has nullable struct resultSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", IntegerType) .add("z", IntegerType) .add("x", IntegerType), nullable = true), expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) // Tests for multiple target-only fields to verify the original values of all target-only // fields are preserved. private val multiTargetOnlyFieldTargetSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", IntegerType) .add("z", IntegerType) // target-only .add("w", IntegerType) // target-only ) private val multiTargetOnlyFieldResultSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", IntegerType) .add("z", IntegerType) .add("w", IntegerType) .add("x", IntegerType)) testNestedStructsEvolution( s"${testNamePrefix}multiple target-only fields - UPDATE * with all target-only fields null")( target = Seq("""{"key":1,"col":{"y":2,"z":null,"w":null}}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = multiTargetOnlyFieldTargetSchema, sourceSchema = topLevelStructSourceSchema, clauses = update("*") :: Nil, result = Seq("""{"key":1,"col":{"y":null,"z":null,"w":null,"x":null}}"""), resultSchema = multiTargetOnlyFieldResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution( s"${testNamePrefix}multiple target-only fields - UPDATE * with a non-null target-only field")( target = Seq("""{"key":1,"col":{"y":2,"z":5,"w":null}}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = multiTargetOnlyFieldTargetSchema, sourceSchema = topLevelStructSourceSchema, clauses = update("*") :: Nil, result = Seq("""{"key":1,"col":{"y":null,"z":5,"w":null,"x":null}}"""), resultSchema = multiTargetOnlyFieldResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution( s"${testNamePrefix}multiple target-only fields - UPDATE * with non-null target-only fields")( target = Seq("""{"key":1,"col":{"y":2,"z":5,"w":6}}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = multiTargetOnlyFieldTargetSchema, sourceSchema = topLevelStructSourceSchema, clauses = update("*") :: Nil, result = Seq("""{"key":1,"col":{"y":null,"z":5,"w":6,"x":null}}"""), resultSchema = multiTargetOnlyFieldResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) } /** * Trait collecting tests verifying the nullness of the results for nested struct evolution. */ trait MergeIntoNestedStructEvolutionNullnessTests extends MergeIntoSchemaEvolutionMixin with MergeIntoStructEvolutionNullnessTestUtils { self: MergeIntoTestUtils with SharedSparkSession => private val testNamePrefix = "nested struct - " private val nestedStructTargetSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", new StructType() .add("d", IntegerType) .add("e", IntegerType)) .add("z", new StructType() .add("f", IntegerType) .add("g", IntegerType))) private val nestedStructSourceSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("x", new StructType() .add("a", IntegerType) .add("b", IntegerType)) .add("y", new StructType() .add("c", IntegerType) .add("d", IntegerType))) // Result schema for UPDATE * and UPDATE t.col = s.col: adds both col.x and col.y.c private val nestedStructColEvolutionResultSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", new StructType() .add("d", IntegerType) .add("e", IntegerType) .add("c", IntegerType)) .add("z", new StructType() .add("f", IntegerType) .add("g", IntegerType)) .add("x", new StructType() .add("a", IntegerType) .add("b", IntegerType))) // Result schema for UPDATE t.col.y = s.col.y: only adds col.y.c (no col.x) private val nestedStructColYEvolutionResultSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", new StructType() .add("d", IntegerType) .add("e", IntegerType) .add("c", IntegerType)) .add("z", new StructType() .add("f", IntegerType) .add("g", IntegerType))) private def generateNestedStructSourceRow(sourceType: SourceType.Value): String = { sourceType match { case SourceType.NonNullLeaves => """{"key":1,"col":{"x":{"a":10,"b":20},"y":{"c":30,"d":40}}}""" case SourceType.NullLeaves => """{"key":1,"col":{"x":{"a":null,"b":null},"y":{"c":null,"d":null}}}""" case SourceType.NullNestedStruct => """{"key":1,"col":{"x":null,"y":null}}""" case SourceType.NullCol => """{"key":1,"col":null}""" } } private def generateNestedStructTargetRow( targetType: TargetType.Value): Option[String] = { targetType match { case TargetType.NonNullLeaves => Some("""{"key":1,"col":{"y":{"d":4,"e":5},"z":{"f":6,"g":7}}}""") case TargetType.NullLeaves => Some("""{"key":1,"col":{"y":{"d":null,"e":null},"z":{"f":null,"g":null}}}""") case TargetType.NullNestedStruct => Some("""{"key":1,"col":{"y":null,"z":null}}""") case TargetType.NullCol => Some("""{"key":1,"col":null}""") case TargetType.Empty => None } } /** * Generates the expected result based on `sourceRowJson`, `targetRowJsonOpt`, and `actionType`. * Semantics: * - UPDATE *: field-level merge, preserves target-only fields. * - UPDATE t.col = s.col, INSERT: whole-struct assignment, nulls target-only fields. * - UPDATE t.col.y = s.col.y: whole-struct assignment, nulls target-only fields. */ private def generateNestedStructExpectedResult( sourceRowJson: String, targetRowJsonOpt: Option[String], actionType: ActionType.Value): String = { val sourceRow = fromJsonToMap(sourceRowJson) val targetRowOpt = targetRowJsonOpt.map(fromJsonToMap) val sourceCol = castToMap(sourceRow("col")) val targetColOpt = targetRowOpt.map(row => castToMap(row("col"))) if (shouldOverwriteWithNull(sourceCol, targetColOpt, actionType)) { return """{"key":1,"col":null}""" } val sourceX = castToMap(getNestedValue(sourceCol, "x")) // col.x is source-only. val resultXOpt: Option[Any] = actionType match { case ActionType.UpdateStar => if (sourceX == null) { if (shouldPreserveNullSourceStructsForUpdateStar) { Some(null) } else { Some(Map("a" -> null, "b" -> null)) } } else { // Keep struct as is. Some(sourceX) } case ActionType.UpdateCol | ActionType.InsertStar | ActionType.InsertCol => // Whole-struct assignment: null stays null, struct stays struct. Some(sourceX) case ActionType.UpdateColY => // col.x is not added in result. None } // col.y exists in both the source and target. val sourceY = castToMap(getNestedValue(sourceCol, "y")) val targetY = targetColOpt.map(col => castToMap(getNestedValue(col, "y"))) val resultY: Any = { if (shouldOverwriteWithNull(sourceY, targetY, actionType)) { null } else { val sourceD = getNestedValue(sourceY, "d") val sourceC = getNestedValue(sourceY, "c") actionType match { case ActionType.UpdateStar => // Update * preserve target-only field (col.y.e) val targetE = getNestedValue(targetY.get, "e") Map("d" -> sourceD, "e" -> targetE, "c" -> sourceC) case ActionType.UpdateCol | ActionType.UpdateColY | ActionType.InsertStar | ActionType.InsertCol => // Whole-struct assignment nulls out target-only field (col.y.e). if (sourceY == null && shouldPreserveNullSourceStructsForWholeStructAssignment) { null } else { Map("d" -> sourceD, "e" -> null, "c" -> sourceC) } } } } // col.z is target-only. val resultZ: Any = actionType match { case ActionType.UpdateStar | ActionType.UpdateColY => val targetCol = targetColOpt.get val targetZ = castToMap(getNestedValue(targetCol, "z")) // UPDATE * preserves target-only field (col.z); // UPDATE col.y = s.col.y does not change t.col.z. targetZ case ActionType.UpdateCol | ActionType.InsertStar | ActionType.InsertCol => // Whole-struct assignment nulls out target-only field (col.z). null } val colMap = resultXOpt match { case Some(resultX) => Map("y" -> resultY, "z" -> resultZ, "x" -> resultX) case None => Map("y" -> resultY, "z" -> resultZ) } val resultMap = Map("key" -> 1, "col" -> colMap) toJsonWithNulls(resultMap) } private def getNestedStructResultSchema(actionType: ActionType.Value): StructType = { actionType match { case ActionType.UpdateStar | ActionType.UpdateCol | ActionType.InsertStar | ActionType.InsertCol => nestedStructColEvolutionResultSchema case ActionType.UpdateColY => nestedStructColYEvolutionResultSchema } } private def generateNestedStructNullnessTests(): Seq[StructEvolutionNullnessTestCase] = { generateStructEvolutionNullnessTests( testNamePrefix = testNamePrefix, sourceSchema = nestedStructSourceSchema, targetSchema = nestedStructTargetSchema, sourceTypes = Seq( SourceType.NonNullLeaves, SourceType.NullLeaves, SourceType.NullNestedStruct, SourceType.NullCol), updateTargetTypes = Seq( TargetType.NonNullLeaves, TargetType.NullLeaves, TargetType.NullNestedStruct, TargetType.NullCol), actionTypes = Seq( ActionType.UpdateStar, ActionType.UpdateCol, ActionType.UpdateColY, ActionType.InsertStar, ActionType.InsertCol), generateResultSchemaFn = getNestedStructResultSchema, generateSourceRowFn = generateNestedStructSourceRow, generateTargetRowFn = generateNestedStructTargetRow, generateExpectedResultFn = generateNestedStructExpectedResult ) } generateNestedStructNullnessTests().foreach { testCase => testNestedStructsEvolution(testCase.testName)( target = testCase.targetData, source = Seq(testCase.sourceData), targetSchema = testCase.targetSchema, sourceSchema = testCase.sourceSchema, clauses = testCase.actionClause :: Nil, result = Seq(testCase.expectedResult), resultSchema = testCase.resultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = testCase.confs) } testNestedStructsEvolution( s"${testNamePrefix}null expansion - UPDATE * with null target col")( target = Seq("""{"key":1,"col":null}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = nestedStructTargetSchema, sourceSchema = nestedStructSourceSchema, clauses = update("*") :: Nil, result = Seq( if (shouldPreserveNullSourceStructsForUpdateStar) { """{"key":1,"col":null}""" } else { """{"key":1,"col":{"y":{"d":null,"e":null,"c":null},"z":null,"x":{"a":null,"b":null}}}""" } ), resultSchema = nestedStructColEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) // scalastyle:off line.size.limit testNestedStructsEvolution( s"${testNamePrefix}null expansion - UPDATE * with null target nested structs")( target = Seq("""{"key":1,"col":{"y":null,"z":null}}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = nestedStructTargetSchema, sourceSchema = nestedStructSourceSchema, clauses = update("*") :: Nil, result = Seq( if (shouldPreserveNullSourceStructsForUpdateStar) { """{"key":1,"col":{"y":null,"z":null,"x":null}}""" } else { """{"key":1,"col":{"y":{"d":null,"e":null,"c":null},"z":null,"x":{"a":null,"b":null}}}""" } ), resultSchema = nestedStructColEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution( s"${testNamePrefix}null expansion - UPDATE * with null target leaves")( target = Seq("""{"key":1,"col":{"y":{"d":null,"e":null},"z":{"f":null,"g":null}}}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = nestedStructTargetSchema, sourceSchema = nestedStructSourceSchema, clauses = update("*") :: Nil, result = Seq( if (shouldPreserveNullSourceStructsForUpdateStar) { """{"key":1,"col":{"y":{"d":null,"e":null,"c":null},"z":{"f":null,"g":null},"x":null}}""" } else { """{"key":1,"col":{"y":{"d":null,"e":null,"c":null},"z":{"f":null,"g":null},"x":{"a":null,"b":null}}}""" } ), resultSchema = nestedStructColEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) // The following tests verify that we don't overwrite the target if the target struct // has extra nested fields (t.col.y.e) and the target is not NULL. private val nestedStructTargetSchemaWithoutZ = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", new StructType() .add("d", IntegerType) .add("e", IntegerType))) // col.y.e is target-only private val nestedStructResultSchemaWithoutZ = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", new StructType() .add("d", IntegerType) .add("e", IntegerType) .add("c", IntegerType)) .add("x", new StructType() .add("a", IntegerType) .add("b", IntegerType))) testNestedStructsEvolution( s"${testNamePrefix}null expansion - UPDATE * with null target nested structs without col.z")( target = Seq("""{"key":1,"col":{"y":null}}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = nestedStructTargetSchemaWithoutZ, sourceSchema = nestedStructSourceSchema, clauses = update("*") :: Nil, result = Seq( if (shouldPreserveNullSourceStructsForUpdateStar) { """{"key":1,"col":{"y":null,"x":null}}""" } else { """{"key":1,"col":{"y":{"d":null,"e":null,"c":null},"x":{"a":null,"b":null}}}""" } ), resultSchema = nestedStructResultSchemaWithoutZ, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution( s"${testNamePrefix}null expansion - UPDATE * with null target leaves without col.z")( target = Seq("""{"key":1,"col":{"y":{"d":null,"e":null}}}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = nestedStructTargetSchemaWithoutZ, sourceSchema = nestedStructSourceSchema, clauses = update("*") :: Nil, result = Seq( if (shouldPreserveNullSourceStructsForUpdateStar) { """{"key":1,"col":{"y":{"d":null,"e":null,"c":null},"x":null}}""" } else { """{"key":1,"col":{"y":{"d":null,"e":null,"c":null},"x":{"a":null,"b":null}}}""" } ), resultSchema = nestedStructResultSchemaWithoutZ, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) // The following tests verify that we overwrite the target struct with NULL if the // target has no extra fields. private val nestedStructTargetSchemaWithoutTargetOnlyFields = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", new StructType() .add("d", IntegerType))) private val nestedStructResultSchemaWithoutTargetOnlyFields = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", new StructType() .add("d", IntegerType) .add("c", IntegerType)) .add("x", new StructType() .add("a", IntegerType) .add("b", IntegerType))) testNestedStructsEvolution( s"${testNamePrefix}null expansion - UPDATE * with null target without target-only fields")( target = Seq("""{"key":1,"col":{"y":null}}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = nestedStructTargetSchemaWithoutTargetOnlyFields, sourceSchema = nestedStructSourceSchema, clauses = update("*") :: Nil, result = Seq( if (shouldPreserveNullSourceStructsForUpdateStar) { """{"key":1,"col":null}""" } else { """{"key":1,"col":{"y":{"d":null,"c":null},"x":{"a":null,"b":null}}}""" } ), resultSchema = nestedStructResultSchemaWithoutTargetOnlyFields, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution( s"${testNamePrefix}null expansion - UPDATE * with null target nested structs without target-only fields")( target = Seq("""{"key":1,"col":{"y":null}}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = nestedStructTargetSchemaWithoutTargetOnlyFields, sourceSchema = nestedStructSourceSchema, clauses = update("*") :: Nil, result = Seq( if (shouldPreserveNullSourceStructsForUpdateStar) { """{"key":1,"col":null}""" } else { """{"key":1,"col":{"y":{"d":null,"c":null},"x":{"a":null,"b":null}}}""" } ), resultSchema = nestedStructResultSchemaWithoutTargetOnlyFields, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution( s"${testNamePrefix}null expansion - UPDATE * with null target leaves without target-only fields")( target = Seq("""{"key":1,"col":{"y":{"d":null}}}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = nestedStructTargetSchemaWithoutTargetOnlyFields, sourceSchema = nestedStructSourceSchema, clauses = update("*") :: Nil, result = Seq( if (shouldPreserveNullSourceStructsForUpdateStar) { """{"key":1,"col":null}""" } else { """{"key":1,"col":{"y":{"d":null,"c":null},"x":{"a":null,"b":null}}}""" } ), resultSchema = nestedStructResultSchemaWithoutTargetOnlyFields, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) // scalastyle:on line.size.limit private val expectedResult = if (shouldPreserveNullSourceStructsForWholeStructAssignment) { """{"key":1,"col":null}""" } else { """{"key":1,"col":{"y":{"d":null,"e":null,"c":null},"z":null,"x":null}}""" } testNestedStructsEvolution(s"${testNamePrefix}null expansion - UPDATE t.col = s.col")( target = Seq("""{"key":1,"col":{"y":{"d":2,"e":2},"z":{"f":2,"g":2}}}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = nestedStructTargetSchema, sourceSchema = nestedStructSourceSchema, clauses = update("t.col = s.col") :: Nil, result = Seq(expectedResult), resultSchema = nestedStructColEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution(s"${testNamePrefix}null expansion - INSERT *")( target = Seq.empty, source = Seq("""{"key":1,"col":null}"""), targetSchema = nestedStructTargetSchema, sourceSchema = nestedStructSourceSchema, clauses = insert("*") :: Nil, result = Seq(expectedResult), resultSchema = nestedStructColEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution(s"${testNamePrefix}null expansion - INSERT (key, col)")( target = Seq.empty, source = Seq("""{"key":1,"col":null}"""), targetSchema = nestedStructTargetSchema, sourceSchema = nestedStructSourceSchema, clauses = insert("(key, col) VALUES (s.key, s.col)") :: Nil, result = Seq(expectedResult), resultSchema = nestedStructColEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) } /** * Trait collecting tests verifying the nullness of the results for array-of-struct evolution. */ trait MergeIntoTopLevelArrayStructEvolutionNullnessTests extends MergeIntoSchemaEvolutionMixin with MergeIntoStructEvolutionNullnessTestUtils { self: MergeIntoTestUtils with SharedSparkSession => import org.apache.spark.sql.types.ArrayType private val testNamePrefix = "top-level array-of-struct - " private val topLevelArrayStructTargetSchema = new StructType() .add("key", IntegerType) .add("col", ArrayType(new StructType() .add("y", IntegerType) .add("z", IntegerType))) private val topLevelArrayStructSourceSchema = new StructType() .add("key", IntegerType) .add("col", ArrayType(new StructType() .add("x", IntegerType) .add("y", IntegerType))) // Result schema: adds col[].x private val topLevelArrayStructEvolutionResultSchema = new StructType() .add("key", IntegerType) .add("col", ArrayType(new StructType() .add("y", IntegerType) .add("z", IntegerType) .add("x", IntegerType))) private def generateTopLevelArrayStructSourceRow( sourceType: SourceType.Value): String = { sourceType match { case SourceType.NonNullLeaves => """{"key":1,"col":[{"x":10,"y":20},{"x":30,"y":40}]}""" case SourceType.NullLeaves => """{"key":1,"col":[{"x":null,"y":null},{"x":null,"y":null}]}""" case SourceType.NullNestedStruct => """{"key":1,"col":[null,null]}""" case SourceType.NullCol => """{"key":1,"col":null}""" } } private def generateTopLevelArrayStructTargetRow( targetType: TargetType.Value): Option[String] = { targetType match { case TargetType.NonNullLeaves => Some("""{"key":1,"col":[{"y":2,"z":3},{"y":4,"z":5}]}""") case TargetType.Empty => None } } /** * Generates the expected result based on `sourceRowJson`, `targetRowJsonOpt`, and `actionType`. * Semantics: Entire arrays are overwritten, and structs within the array evolve. * Note: `targetRowJsonOpt` and `actionType` are not used since arrays are always overwritten. * They are added to match the data type of * `generateStructEvolutionNullnessTests.generateExpectedResultFn`. * */ private def generateTopLevelArrayStructExpectedResult( sourceRowJson: String, targetRowJsonOpt: Option[String], actionType: ActionType.Value): String = { val sourceRow = fromJsonToMap(sourceRowJson) val sourceCol = sourceRow("col") val resultCol = if (sourceCol == null) { null } else { val sourceArray = sourceCol.asInstanceOf[Seq[Any]] sourceArray.map { elem => if (elem == null) { if (shouldPreserveNullSourceStructsForWholeStructAssignment) { null } else { Map("y" -> null, "z" -> null, "x" -> null) } } else { val sourceStruct = elem.asInstanceOf[Map[String, Any]] Map( "y" -> sourceStruct("y"), // Target-only field `z` in array element structs is added as null. "z" -> null, "x" -> sourceStruct("x") ) } } } val resultMap = Map("key" -> 1, "col" -> resultCol) toJsonWithNulls(resultMap) } /** * Generates test cases for combinations of source type, target type, and action type. */ private def generateTopLevelArrayStructNullnessTests(): Seq[StructEvolutionNullnessTestCase] = { generateStructEvolutionNullnessTests( testNamePrefix = testNamePrefix, sourceSchema = topLevelArrayStructSourceSchema, targetSchema = topLevelArrayStructTargetSchema, sourceTypes = Seq( SourceType.NonNullLeaves, SourceType.NullLeaves, SourceType.NullNestedStruct, SourceType.NullCol), updateTargetTypes = Seq(TargetType.NonNullLeaves), actionTypes = Seq( ActionType.UpdateStar, ActionType.UpdateCol, ActionType.InsertStar, ActionType.InsertCol), generateResultSchemaFn = _ => topLevelArrayStructEvolutionResultSchema, generateSourceRowFn = generateTopLevelArrayStructSourceRow, generateTargetRowFn = generateTopLevelArrayStructTargetRow, generateExpectedResultFn = generateTopLevelArrayStructExpectedResult ) } generateTopLevelArrayStructNullnessTests().foreach { testCase => testNestedStructsEvolution(testCase.testName)( target = testCase.targetData, source = Seq(testCase.sourceData), targetSchema = testCase.targetSchema, sourceSchema = testCase.sourceSchema, clauses = testCase.actionClause :: Nil, result = Seq(testCase.expectedResult), resultSchema = testCase.resultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = testCase.confs) } testNestedStructsEvolution( s"${testNamePrefix}null expansion - UPDATE *")( target = Seq("""{"key":1,"col":[{"y":2,"z":2}]}"""), source = Seq("""{"key":1,"col":[null]}"""), targetSchema = topLevelArrayStructTargetSchema, sourceSchema = topLevelArrayStructSourceSchema, clauses = update("*") :: Nil, result = Seq( if (shouldPreserveNullSourceStructsForWholeStructAssignment) { """{"key":1,"col":[null]}""" } else { """{"key":1,"col":[{"y":null,"z":null,"x":null}]}""" } ), resultSchema = topLevelArrayStructEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) private val expectedResult = if (shouldPreserveNullSourceStructsForWholeStructAssignment) { """{"key":1,"col":[null]}""" } else { """{"key":1,"col":[{"y":null,"z":null,"x":null}]}""" } testNestedStructsEvolution(s"${testNamePrefix}null expansion - UPDATE t.col = s.col")( target = Seq("""{"key":1,"col":[{"y":2,"z":2}]}"""), source = Seq("""{"key":1,"col":[null]}"""), targetSchema = topLevelArrayStructTargetSchema, sourceSchema = topLevelArrayStructSourceSchema, clauses = update("t.col = s.col") :: Nil, result = Seq(expectedResult), resultSchema = topLevelArrayStructEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution(s"${testNamePrefix}null expansion - INSERT *")( target = Seq.empty, source = Seq("""{"key":1,"col":[null]}"""), targetSchema = topLevelArrayStructTargetSchema, sourceSchema = topLevelArrayStructSourceSchema, clauses = insert("*") :: Nil, result = Seq(expectedResult), resultSchema = topLevelArrayStructEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution(s"${testNamePrefix}null expansion - INSERT (key, col)")( target = Seq.empty, source = Seq("""{"key":1,"col":[null]}"""), targetSchema = topLevelArrayStructTargetSchema, sourceSchema = topLevelArrayStructSourceSchema, clauses = insert("(key, col) VALUES (s.key, s.col)") :: Nil, result = Seq(expectedResult), resultSchema = topLevelArrayStructEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) } /** * Trait collecting tests verifying the nullness of the results for nested array-of-struct * evolution. */ trait MergeIntoNestedArrayStructEvolutionNullnessTests extends MergeIntoSchemaEvolutionMixin with MergeIntoStructEvolutionNullnessTestUtils { self: MergeIntoTestUtils with SharedSparkSession => import org.apache.spark.sql.types.ArrayType private val testNamePrefix = "nested array-of-struct - " // Nested arrays: col is a struct containing multiple array-of-struct fields. private val nestedArrayStructTargetSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", ArrayType(new StructType() .add("d", IntegerType) .add("e", IntegerType))) .add("z", ArrayType(new StructType() .add("f", IntegerType) .add("g", IntegerType)))) private val nestedArrayStructSourceSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("x", ArrayType(new StructType() .add("a", IntegerType) .add("b", IntegerType))) .add("y", ArrayType(new StructType() .add("c", IntegerType) .add("d", IntegerType)))) // Result schema for UPDATE * and UPDATE t.col = s.col: adds both col.x and col.y[].c. private val nestedArrayColEvolutionResultSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", ArrayType(new StructType() .add("d", IntegerType) .add("e", IntegerType) .add("c", IntegerType))) .add("z", ArrayType(new StructType() .add("f", IntegerType) .add("g", IntegerType))) .add("x", ArrayType(new StructType() .add("a", IntegerType) .add("b", IntegerType)))) // Result schema for UPDATE t.col.y = s.col.y: only adds col.y[].c (no col.x). private val nestedArrayColYEvolutionResultSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", ArrayType(new StructType() .add("d", IntegerType) .add("e", IntegerType) .add("c", IntegerType))) .add("z", ArrayType(new StructType() .add("f", IntegerType) .add("g", IntegerType)))) private def generateNestedArraySourceRow(sourceType: SourceType.Value): String = { sourceType match { case SourceType.NonNullLeaves => """{"key":1,"col":{"x":[{"a":10,"b":20}],"y":[{"c":30,"d":40}]}}""" case SourceType.NullLeaves => """{"key":1,"col":{"x":[{"a":null,"b":null}],"y":[{"c":null,"d":null}]}}""" case SourceType.NullNestedStruct => """{"key":1,"col":{"x":[null],"y":[null]}}""" case SourceType.NullNestedArray => """{"key":1,"col":{"x":null,"y":null}}""" case SourceType.NullCol => """{"key":1,"col":null}""" } } private def generateNestedArrayTargetRow( targetType: TargetType.Value): Option[String] = { targetType match { case TargetType.NonNullLeaves => Some("""{"key":1,"col":{"y":[{"d":4,"e":5}],"z":[{"f":6,"g":7}]}}""") case TargetType.NullLeaves => Some("""{"key":1,"col":{"y":[{"d":null,"e":null}],"z":[{"f":null,"g":null}]}}""") case TargetType.NullNestedStruct => Some("""{"key":1,"col":{"y":[null],"z":[null]}}""") case TargetType.NullNestedArray => Some("""{"key":1,"col":{"y":null,"z":null}}""") case TargetType.NullCol => Some("""{"key":1,"col":null}""") case TargetType.Empty => None } } /** * Generates the expected result based on `sourceRowJson`, `targetRowJsonOpt`, and `actionType`. * Semantics: col struct evolves, nested arrays are overwritten, and structs within the * array evolve. */ private def generateNestedArrayExpectedResult( sourceRowJson: String, targetRowJsonOpt: Option[String], actionType: ActionType.Value): String = { val sourceRow = fromJsonToMap(sourceRowJson) val targetRowOpt = targetRowJsonOpt.map(fromJsonToMap) val sourceCol = castToMap(sourceRow("col")) val targetColOpt = targetRowOpt.map(row => castToMap(row("col"))) if (shouldOverwriteWithNull(sourceCol, targetColOpt, actionType)) { return """{"key":1,"col":null}""" } // col.x is source-only. val sourceX = getNestedValue(sourceCol, "x") val resultXOpt: Option[Any] = actionType match { case ActionType.UpdateStar | ActionType.UpdateCol | ActionType.InsertStar | ActionType.InsertCol => // col.x is kept as is. Some(sourceX) case ActionType.UpdateColY => // col.x is not added in the result schema. None } val sourceY = getNestedValue(sourceCol, "y") // col.y exists in both the source and target. // Semantics: replace entire array and evolve struct within the array. val resultY: Any = if (sourceY == null) { null } else { val sourceYArray = sourceY.asInstanceOf[Seq[Any]] sourceYArray.map { elem => if (elem == null) { if (shouldPreserveNullSourceStructsForWholeStructAssignment) { null } else { Map("d" -> null, "e" -> null, "c" -> null) } } else { val sourceYStruct = elem.asInstanceOf[Map[String, Any]] Map( "d" -> sourceYStruct("d"), // Target-only field `e` in array element structs is added as null. "e" -> null, "c" -> sourceYStruct("c") ) } } } // col.z is target-only. val resultZ: Any = actionType match { case ActionType.UpdateStar | ActionType.UpdateColY => val targetCol = targetColOpt.get val targetZ = getNestedValue(targetCol, "z") // UPDATE * preserves target-only field (col.z). // UPDATE col.y = s.col.y preserves fields not in assignment (col.z). targetZ case ActionType.UpdateCol | ActionType.InsertStar | ActionType.InsertCol => // Whole-struct assignment nulls target-only field (col.z). null } val colMap = resultXOpt match { case Some(resultX) => Map("y" -> resultY, "z" -> resultZ, "x" -> resultX) case None => Map("y" -> resultY, "z" -> resultZ) } val resultMap = Map("key" -> 1, "col" -> colMap) toJsonWithNulls(resultMap) } private def getNestedArrayResultSchema(actionType: ActionType.Value): StructType = { actionType match { case ActionType.UpdateStar | ActionType.UpdateCol | ActionType.InsertStar | ActionType.InsertCol => nestedArrayColEvolutionResultSchema case ActionType.UpdateColY => nestedArrayColYEvolutionResultSchema } } /** * Generates test cases for combinations of source type, target type, and action type. */ private def generateNestedArrayStructNullnessTests(): Seq[StructEvolutionNullnessTestCase] = { generateStructEvolutionNullnessTests( testNamePrefix = testNamePrefix, sourceSchema = nestedArrayStructSourceSchema, targetSchema = nestedArrayStructTargetSchema, sourceTypes = Seq( SourceType.NonNullLeaves, SourceType.NullLeaves, SourceType.NullNestedStruct, SourceType.NullNestedArray, SourceType.NullCol), updateTargetTypes = Seq( TargetType.NonNullLeaves, TargetType.NullLeaves, TargetType.NullNestedStruct, TargetType.NullNestedArray, TargetType.NullCol), actionTypes = Seq( ActionType.UpdateStar, ActionType.UpdateCol, ActionType.UpdateColY, ActionType.InsertStar, ActionType.InsertCol), generateResultSchemaFn = getNestedArrayResultSchema, generateSourceRowFn = generateNestedArraySourceRow, generateTargetRowFn = generateNestedArrayTargetRow, generateExpectedResultFn = generateNestedArrayExpectedResult ) } generateNestedArrayStructNullnessTests().foreach { testCase => testNestedStructsEvolution(testCase.testName)( target = testCase.targetData, source = Seq(testCase.sourceData), targetSchema = testCase.targetSchema, sourceSchema = testCase.sourceSchema, clauses = testCase.actionClause :: Nil, result = Seq(testCase.expectedResult), resultSchema = testCase.resultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = testCase.confs) } testNestedStructsEvolution( s"${testNamePrefix}null expansion - UPDATE * with null target col struct")( target = Seq("""{"key":1,"col":null}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = nestedArrayStructTargetSchema, sourceSchema = nestedArrayStructSourceSchema, clauses = update("*") :: Nil, result = Seq( if (shouldPreserveNullSourceStructsForUpdateStar) { """{"key":1,"col":null}""" } else { """{"key":1,"col":{"y":null,"z":null,"x":null}}""" } ), resultSchema = nestedArrayColEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution( s"${testNamePrefix}null expansion - UPDATE * with non-null target col struct")( target = Seq("""{"key":1,"col":{"y":null,"z":null}}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = nestedArrayStructTargetSchema, sourceSchema = nestedArrayStructSourceSchema, clauses = update("*") :: Nil, result = Seq("""{"key":1,"col":{"y":null,"z":null,"x":null}}"""), resultSchema = nestedArrayColEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) // scalastyle:off line.size.limit testNestedStructsEvolution( s"${testNamePrefix}null expansion - UPDATE * with null structs nested in source array")( target = Seq("""{"key":1,"col":{"y":[{"d":1,"e":2},{"d":null,"e":null}],"z":[null]}}"""), source = Seq("""{"key":1,"col":{"x":[null],"y":[null]}}"""), targetSchema = nestedArrayStructTargetSchema, sourceSchema = nestedArrayStructSourceSchema, clauses = update("*") :: Nil, result = Seq( // The original array value should be overwritten by source. if (preserveNullSourceStructs) { """{"key":1,"col":{"y":[null],"z":[null],"x":[null]}}""" } else { """{"key":1,"col":{"y":[{"d":null,"e":null,"c":null}],"z":[null],"x":[null]}}""" } ), resultSchema = nestedArrayColEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) // scalastyle:on line.size.limit private val expectedResult = if (shouldPreserveNullSourceStructsForWholeStructAssignment) { """{"key":1,"col":null}""" } else { """{"key":1,"col":{"y":null,"z":null,"x":null}}""" } testNestedStructsEvolution(s"${testNamePrefix}null expansion - UPDATE t.col = s.col")( target = Seq("""{"key":1,"col":{"y":[{"d":2,"e":2}],"z":[{"f":2,"g":2}]}}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = nestedArrayStructTargetSchema, sourceSchema = nestedArrayStructSourceSchema, clauses = update("t.col = s.col") :: Nil, result = Seq(expectedResult), resultSchema = nestedArrayColEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution(s"${testNamePrefix}null expansion - INSERT *")( target = Seq.empty, source = Seq("""{"key":1,"col":null}"""), targetSchema = nestedArrayStructTargetSchema, sourceSchema = nestedArrayStructSourceSchema, clauses = insert("*") :: Nil, result = Seq(expectedResult), resultSchema = nestedArrayColEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution(s"${testNamePrefix}null expansion - INSERT (key, col)")( target = Seq.empty, source = Seq("""{"key":1,"col":null}"""), targetSchema = nestedArrayStructTargetSchema, sourceSchema = nestedArrayStructSourceSchema, clauses = insert("(key, col) VALUES (s.key, s.col)") :: Nil, result = Seq(expectedResult), resultSchema = nestedArrayColEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) } /** * Trait collecting tests verifying the nullness of the results for map-of-struct evolution. */ trait MergeIntoTopLevelMapStructEvolutionNullnessTests extends MergeIntoSchemaEvolutionMixin with MergeIntoStructEvolutionNullnessTestUtils { self: MergeIntoTestUtils with SharedSparkSession => import org.apache.spark.sql.types.{MapType, StringType} private val testNamePrefix = "top-level map-of-struct - " private val topLevelMapStructTargetSchema = new StructType() .add("key", IntegerType) .add("col", MapType(StringType, new StructType() .add("y", IntegerType) .add("z", IntegerType))) private val topLevelMapStructSourceSchema = new StructType() .add("key", IntegerType) .add("col", MapType(StringType, new StructType() .add("x", IntegerType) .add("y", IntegerType))) // Result schema: adds col{}.x private val topLevelMapStructEvolutionResultSchema = new StructType() .add("key", IntegerType) .add("col", MapType(StringType, new StructType() .add("y", IntegerType) .add("z", IntegerType) .add("x", IntegerType))) private def generateTopLevelMapStructSourceRow( sourceType: SourceType.Value): String = { sourceType match { case SourceType.NonNullLeaves => """{"key":1,"col":{"k1":{"x":10,"y":20},"k2":{"x":30,"y":40}}}""" case SourceType.NullLeaves => """{"key":1,"col":{"k1":{"x":null,"y":null},"k2":{"x":null,"y":null}}}""" case SourceType.NullNestedStruct => """{"key":1,"col":{"k1":null,"k2":null}}""" case SourceType.NullCol => """{"key":1,"col":null}""" } } private def generateTopLevelMapStructTargetRow( targetType: TargetType.Value): Option[String] = { targetType match { case TargetType.NonNullLeaves => Some("""{"key":1,"col":{"k2":{"y":2,"z":3},"k3":{"y":4,"z":5}}}""") case TargetType.Empty => None } } /** * Generates the expected result based on `sourceRowJson`, `targetRowJsonOpt`, and `actionType`. * Semantics: Entire maps are overwritten, and structs within the map evolve. * Note: `targetRowJsonOpt` and `actionType` are not used since maps are always overwritten. * They are added to match the data type of * `generateStructEvolutionNullnessTests.generateExpectedResultFn`. * */ private def generateTopLevelMapStructExpectedResult( sourceRowJson: String, targetRowJsonOpt: Option[String], actionType: ActionType.Value): String = { val sourceRow = fromJsonToMap(sourceRowJson) val sourceCol = sourceRow("col") val resultCol = if (sourceCol == null) { null } else { val sourceMap = sourceCol.asInstanceOf[Map[String, Any]] sourceMap.map { case (key, value) => val resultValue = if (value == null) { if (shouldPreserveNullSourceStructsForWholeStructAssignment) { null } else { Map("y" -> null, "z" -> null, "x" -> null) } } else { val sourceStruct = value.asInstanceOf[Map[String, Any]] Map( "y" -> sourceStruct("y"), // Target-only field `z` in map value structs is added as null. "z" -> null, "x" -> sourceStruct("x") ) } key -> resultValue } } val resultMap = Map("key" -> 1, "col" -> resultCol) toJsonWithNulls(resultMap) } /** * Generates test cases for combinations of source type, target type, and action type. */ private def generateTopLevelMapStructNullnessTests(): Seq[StructEvolutionNullnessTestCase] = { generateStructEvolutionNullnessTests( testNamePrefix = testNamePrefix, sourceSchema = topLevelMapStructSourceSchema, targetSchema = topLevelMapStructTargetSchema, sourceTypes = Seq( SourceType.NonNullLeaves, SourceType.NullLeaves, SourceType.NullNestedStruct, SourceType.NullCol), updateTargetTypes = Seq(TargetType.NonNullLeaves), actionTypes = Seq( ActionType.UpdateStar, ActionType.UpdateCol, ActionType.InsertStar, ActionType.InsertCol), generateResultSchemaFn = _ => topLevelMapStructEvolutionResultSchema, generateSourceRowFn = generateTopLevelMapStructSourceRow, generateTargetRowFn = generateTopLevelMapStructTargetRow, generateExpectedResultFn = generateTopLevelMapStructExpectedResult ) } generateTopLevelMapStructNullnessTests().foreach { testCase => testNestedStructsEvolution(testCase.testName)( target = testCase.targetData, source = Seq(testCase.sourceData), targetSchema = testCase.targetSchema, sourceSchema = testCase.sourceSchema, clauses = testCase.actionClause :: Nil, result = Seq(testCase.expectedResult), resultSchema = testCase.resultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = testCase.confs) } testNestedStructsEvolution( s"${testNamePrefix}null expansion - UPDATE *")( target = Seq("""{"key":1,"col":{"k1":{"y":2,"z":2}}}"""), source = Seq("""{"key":1,"col":{"k1":null}}"""), targetSchema = topLevelMapStructTargetSchema, sourceSchema = topLevelMapStructSourceSchema, clauses = update("*") :: Nil, result = Seq( if (shouldPreserveNullSourceStructsForWholeStructAssignment) { """{"key":1,"col":{"k1":null}}""" } else { """{"key":1,"col":{"k1":{"y":null,"z":null,"x":null}}}""" } ), resultSchema = topLevelMapStructEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) private val expectedResult = if (shouldPreserveNullSourceStructsForWholeStructAssignment) { """{"key":1,"col":{"k1":null}}""" } else { """{"key":1,"col":{"k1":{"y":null,"z":null,"x":null}}}""" } testNestedStructsEvolution(s"${testNamePrefix}null expansion - UPDATE t.col = s.col")( target = Seq("""{"key":1,"col":{"k1":{"y":2,"z":2}}}"""), source = Seq("""{"key":1,"col":{"k1":null}}"""), targetSchema = topLevelMapStructTargetSchema, sourceSchema = topLevelMapStructSourceSchema, clauses = update("t.col = s.col") :: Nil, result = Seq(expectedResult), resultSchema = topLevelMapStructEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution(s"${testNamePrefix}null expansion - INSERT *")( target = Seq.empty, source = Seq("""{"key":1,"col":{"k1":null}}"""), targetSchema = topLevelMapStructTargetSchema, sourceSchema = topLevelMapStructSourceSchema, clauses = insert("*") :: Nil, result = Seq(expectedResult), resultSchema = topLevelMapStructEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution(s"${testNamePrefix}null expansion - INSERT (key, col)")( target = Seq.empty, source = Seq("""{"key":1,"col":{"k1":null}}"""), targetSchema = topLevelMapStructTargetSchema, sourceSchema = topLevelMapStructSourceSchema, clauses = insert("(key, col) VALUES (s.key, s.col)") :: Nil, result = Seq(expectedResult), resultSchema = topLevelMapStructEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) } /** * Trait collecting tests verifying the nullness of the results for nested map-of-struct * evolution. */ trait MergeIntoNestedMapStructEvolutionNullnessTests extends MergeIntoSchemaEvolutionMixin with MergeIntoStructEvolutionNullnessTestUtils { self: MergeIntoTestUtils with SharedSparkSession => import org.apache.spark.sql.types.{MapType, StringType} private val testNamePrefix = "nested map-of-struct - " // Nested maps: col is a struct containing multiple map-of-struct fields. private val nestedMapStructTargetSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", MapType(StringType, new StructType() .add("d", IntegerType) .add("e", IntegerType))) .add("z", MapType(StringType, new StructType() .add("f", IntegerType) .add("g", IntegerType)))) private val nestedMapStructSourceSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("x", MapType(StringType, new StructType() .add("a", IntegerType) .add("b", IntegerType))) .add("y", MapType(StringType, new StructType() .add("c", IntegerType) .add("d", IntegerType)))) // Result schema for UPDATE * and UPDATE t.col = s.col: adds both col.x and col.y{}.c. private val nestedMapColEvolutionResultSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", MapType(StringType, new StructType() .add("d", IntegerType) .add("e", IntegerType) .add("c", IntegerType))) .add("z", MapType(StringType, new StructType() .add("f", IntegerType) .add("g", IntegerType))) .add("x", MapType(StringType, new StructType() .add("a", IntegerType) .add("b", IntegerType)))) // Result schema for UPDATE t.col.y = s.col.y: only adds col.y{}.c (no col.x). private val nestedMapColYEvolutionResultSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", MapType(StringType, new StructType() .add("d", IntegerType) .add("e", IntegerType) .add("c", IntegerType))) .add("z", MapType(StringType, new StructType() .add("f", IntegerType) .add("g", IntegerType)))) private def generateNestedMapSourceRow(sourceType: SourceType.Value): String = { sourceType match { case SourceType.NonNullLeaves => """{"key":1,"col":{"x":{"k1":{"a":10,"b":20}},"y":{"k1":{"c":30,"d":40}}}}""" case SourceType.NullLeaves => """{"key":1,"col":{"x":{"k1":{"a":null,"b":null}},"y":{"k1":{"c":null,"d":null}}}}""" case SourceType.NullNestedStruct => """{"key":1,"col":{"x":{"k1":null},"y":{"k1":null}}}""" case SourceType.NullNestedMap => """{"key":1,"col":{"x":null,"y":null}}""" case SourceType.NullCol => """{"key":1,"col":null}""" } } private def generateNestedMapTargetRow( targetType: TargetType.Value): Option[String] = { targetType match { case TargetType.NonNullLeaves => Some("""{"key":1,"col":{"y":{"k2":{"d":4,"e":5}},"z":{"k2":{"f":6,"g":7}}}}""") case TargetType.NullLeaves => Some("""{"key":1,"col":{"y":{"k2":{"d":null,"e":null}},"z":{"k2":{"f":null,"g":null}}}}""") case TargetType.NullNestedStruct => Some("""{"key":1,"col":{"y":{"k2":null},"z":{"k2":null}}}""") case TargetType.NullNestedMap => Some("""{"key":1,"col":{"y":null,"z":null}}""") case TargetType.NullCol => Some("""{"key":1,"col":null}""") case TargetType.Empty => None } } /** * Generates the expected result based on `sourceRowJson`, `targetRowJsonOpt`, and `actionType`. * Semantics: col struct evolves, nested maps are overwritten, and structs within the * map evolve. */ private def generateNestedMapExpectedResult( sourceRowJson: String, targetRowJsonOpt: Option[String], actionType: ActionType.Value): String = { val sourceRow = fromJsonToMap(sourceRowJson) val targetRowOpt = targetRowJsonOpt.map(fromJsonToMap) val sourceCol = castToMap(sourceRow("col")) val targetColOpt = targetRowOpt.map(row => castToMap(row("col"))) if (shouldOverwriteWithNull(sourceCol, targetColOpt, actionType)) { return """{"key":1,"col":null}""" } // col.x is source-only. val sourceX = getNestedValue(sourceCol, "x") val resultXOpt: Option[Any] = actionType match { case ActionType.UpdateStar | ActionType.UpdateCol | ActionType.InsertStar | ActionType.InsertCol => // col.x is kept as is. Some(sourceX) case ActionType.UpdateColY => // col.x is not added in the result schema. None } val sourceY = getNestedValue(sourceCol, "y") // col.y exists in both the source and target. // Semantics: replace entire map and evolve struct within the map. val resultY: Any = if (sourceY == null) { null } else { val sourceYMap = sourceY.asInstanceOf[Map[String, Any]] sourceYMap.map { case (key, value) => val resultValue = if (value == null) { if (shouldPreserveNullSourceStructsForWholeStructAssignment) { null } else { Map("d" -> null, "e" -> null, "c" -> null) } } else { val sourceYStruct = value.asInstanceOf[Map[String, Any]] Map( "d" -> sourceYStruct("d"), // Target-only field `e` in map value structs is added as null. "e" -> null, "c" -> sourceYStruct("c") ) } key -> resultValue } } // col.z is target-only. val resultZ: Any = actionType match { case ActionType.UpdateStar | ActionType.UpdateColY => val targetCol = targetColOpt.get val targetZ = getNestedValue(targetCol, "z") // UPDATE * preserves target-only field (col.z). // UPDATE col.y = s.col.y preserves fields not in assignment (col.z). targetZ case ActionType.UpdateCol | ActionType.InsertStar | ActionType.InsertCol => // Whole-struct assignment nulls target-only field (col.z). null } val colMap = resultXOpt match { case Some(resultX) => Map("y" -> resultY, "z" -> resultZ, "x" -> resultX) case None => Map("y" -> resultY, "z" -> resultZ) } val resultMap = Map("key" -> 1, "col" -> colMap) toJsonWithNulls(resultMap) } private def getNestedMapResultSchema(actionType: ActionType.Value): StructType = { actionType match { case ActionType.UpdateStar | ActionType.UpdateCol | ActionType.InsertStar | ActionType.InsertCol => nestedMapColEvolutionResultSchema case ActionType.UpdateColY => nestedMapColYEvolutionResultSchema } } /** * Generates test cases for combinations of source type, target type, and action type. */ private def generateNestedMapStructNullnessTests(): Seq[StructEvolutionNullnessTestCase] = { generateStructEvolutionNullnessTests( testNamePrefix = testNamePrefix, sourceSchema = nestedMapStructSourceSchema, targetSchema = nestedMapStructTargetSchema, sourceTypes = Seq( SourceType.NonNullLeaves, SourceType.NullLeaves, SourceType.NullNestedStruct, SourceType.NullNestedMap, SourceType.NullCol), updateTargetTypes = Seq( TargetType.NonNullLeaves, TargetType.NullLeaves, TargetType.NullNestedStruct, TargetType.NullNestedMap, TargetType.NullCol), actionTypes = Seq( ActionType.UpdateStar, ActionType.UpdateCol, ActionType.UpdateColY, ActionType.InsertStar, ActionType.InsertCol), generateResultSchemaFn = getNestedMapResultSchema, generateSourceRowFn = generateNestedMapSourceRow, generateTargetRowFn = generateNestedMapTargetRow, generateExpectedResultFn = generateNestedMapExpectedResult ) } generateNestedMapStructNullnessTests().foreach { testCase => testNestedStructsEvolution(testCase.testName)( target = testCase.targetData, source = Seq(testCase.sourceData), targetSchema = testCase.targetSchema, sourceSchema = testCase.sourceSchema, clauses = testCase.actionClause :: Nil, result = Seq(testCase.expectedResult), resultSchema = testCase.resultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = testCase.confs) } testNestedStructsEvolution( s"${testNamePrefix}null expansion - UPDATE * with null target col struct")( target = Seq("""{"key":1,"col":null}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = nestedMapStructTargetSchema, sourceSchema = nestedMapStructSourceSchema, clauses = update("*") :: Nil, result = Seq( if (shouldPreserveNullSourceStructsForUpdateStar) { """{"key":1,"col":null}""" } else { """{"key":1,"col":{"y": null,"z": null, "x": null}}""" } ), resultSchema = nestedMapColEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution( s"${testNamePrefix}null expansion - UPDATE * with non-null target col struct")( target = Seq("""{"key":1,"col":{"y":null,"z":null}}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = nestedMapStructTargetSchema, sourceSchema = nestedMapStructSourceSchema, clauses = update("*") :: Nil, result = Seq("""{"key":1,"col":{"y":null,"z":null,"x":null}}"""), resultSchema = nestedMapColEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) // scalastyle:off line.size.limit testNestedStructsEvolution( s"${testNamePrefix}null expansion - UPDATE * with null structs nested in source map")( target = Seq("""{"key":1,"col":{"y":{"k2":{"d":1,"e":2},"k3":{"d":null,"e":null}},"z":{"k2":null}}}"""), source = Seq("""{"key":1,"col":{"x":{"k1":null},"y":{"k1":null}}}"""), targetSchema = nestedMapStructTargetSchema, sourceSchema = nestedMapStructSourceSchema, clauses = update("*") :: Nil, result = Seq( // The original map value should be overwritten by source. if (preserveNullSourceStructs) { """{"key":1,"col":{"y":{"k1":null},"z":{"k2":null},"x":{"k1":null}}}""" } else { """{"key":1,"col":{"y":{"k1":{"d":null,"e":null,"c":null}},"z":{"k2":null},"x":{"k1":null}}}""" } ), resultSchema = nestedMapColEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) // scalastyle:on line.size.limit private val expectedResult = if (shouldPreserveNullSourceStructsForWholeStructAssignment) { """{"key":1,"col":null}""" } else { """{"key":1,"col":{"y": null,"z": null, "x": null}}""" } testNestedStructsEvolution(s"${testNamePrefix}null expansion - UPDATE t.col = s.col")( target = Seq("""{"key":1,"col":{"y":{"k1":{"d":2,"e":2}},"z":{"k1":{"f":2,"g":2}}}}"""), source = Seq("""{"key":1,"col":null}"""), targetSchema = nestedMapStructTargetSchema, sourceSchema = nestedMapStructSourceSchema, clauses = update("t.col = s.col") :: Nil, result = Seq(expectedResult), resultSchema = nestedMapColEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution(s"${testNamePrefix}null expansion - INSERT *")( target = Seq.empty, source = Seq("""{"key":1,"col":null}"""), targetSchema = nestedMapStructTargetSchema, sourceSchema = nestedMapStructSourceSchema, clauses = insert("*") :: Nil, result = Seq(expectedResult), resultSchema = nestedMapColEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution(s"${testNamePrefix}null expansion - INSERT (key, col)")( target = Seq.empty, source = Seq("""{"key":1,"col":null}"""), targetSchema = nestedMapStructTargetSchema, sourceSchema = nestedMapStructSourceSchema, clauses = insert("(key, col) VALUES (s.key, s.col)") :: Nil, result = Seq(expectedResult), resultSchema = nestedMapColEvolutionResultSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) } /** * Trait collecting tests with multiple MERGE clauses for struct evolution nullness behavior. * * When multiple clauses have different actions, fields excluded in one clause may still be * added to the final evolved schema by another clause. The tests verify the nullness of the * results in these scenarios. */ trait MergeIntoStructEvolutionNullnessMultiClauseTests extends MergeIntoSchemaEvolutionMixin with MergeIntoStructEvolutionNullnessTestUtils { self: MergeIntoTestUtils with SharedSparkSession => private val testNamePrefix = s"multiple clauses - " + s"preserveNullSourceStructs=$preserveNullSourceStructs - " + s"preserveNullSourceStructsUpdateStar=$preserveNullSourceStructsUpdateStar - " private val targetSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", IntegerType) .add("z", IntegerType)) private val sourceSchemaWithTopLevelExtra = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("x", IntegerType) .add("y", IntegerType)) .add("extra", new StructType() .add("val", IntegerType) .add("val2", IntegerType)) private val fullyEvolvedTargetSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", IntegerType) .add("z", IntegerType) .add("x", IntegerType)) .add("extra", new StructType() .add("val", IntegerType) .add("val2", IntegerType)) // The following tests cover UPDATE SET col = s.col combined with the other action // which adds new column `extra` to the target schema: // - UPDATE SET col = s.col, UPDATE SET extra = s.extra // - UPDATE SET col = s.col, UPDATE SET extra.val = s.extra.val // - UPDATE SET col = s.col, UPDATE SET * // - UPDATE SET col = s.col, INSERT (key, col, extra) VALUES (s.key, s.col, s.extra) // - UPDATE SET col = s.col, INSERT * testNestedStructsEvolution( s"${testNamePrefix}UPDATE SET col = s.col, UPDATE SET extra = s.extra")( target = Seq("""{"key":1,"col":null}""", """{"key":2,"col":null}"""), source = Seq( """{"key":1,"col":null,"extra":null}""", """{"key":2,"col":null,"extra":null}""" ), targetSchema = targetSchema, sourceSchema = sourceSchemaWithTopLevelExtra, clauses = update(condition = "s.key = 1", set = "col = s.col") :: update(condition = "s.key = 2", set = "extra = s.extra") :: Nil, result = if (shouldPreserveNullSourceStructsForWholeStructAssignment) { Seq( """{"key":1,"col":null,"extra":null}""", """{"key":2,"col":null,"extra":null}""" ) } else { Seq( """{"key":1,"col":{"y":null,"z":null,"x":null},"extra":{"val":null,"val2":null}}""", """{"key":2,"col":{"y":null,"z":null,"x":null},"extra":null}""" ) }, resultSchema = fullyEvolvedTargetSchema, expectErrorWithoutEvolutionContains = "Cannot resolve", confs = preserveNullStructsConfs) testNestedStructsEvolution( s"${testNamePrefix}UPDATE SET col = s.col, UPDATE SET extra.val = s.extra.val")( target = Seq("""{"key":1,"col":null}""", """{"key":2,"col":null}"""), source = Seq( """{"key":1,"col":null,"extra":null}""", """{"key":2,"col":null,"extra":null}""" ), targetSchema = targetSchema, sourceSchema = sourceSchemaWithTopLevelExtra, clauses = update(condition = "s.key = 1", set = "col = s.col") :: update(condition = "s.key = 2", set = "extra.val = s.extra.val") :: Nil, result = if (shouldPreserveNullSourceStructsForWholeStructAssignment) { Seq( """{"key":1,"col":null,"extra":null}""", """{"key":2,"col":null,"extra":{"val":null}}""" ) } else { Seq( """{"key":1,"col":{"y":null,"z":null,"x":null},"extra":{"val":null}}""", """{"key":2,"col":{"y":null,"z":null,"x":null},"extra":{"val":null}}""" ) }, resultSchema = new StructType() .add("key", IntegerType) .add("col", new StructType() .add("y", IntegerType) .add("z", IntegerType) .add("x", IntegerType)) .add("extra", new StructType() .add("val", IntegerType)), expectErrorWithoutEvolutionContains = "Cannot resolve", confs = preserveNullStructsConfs) testNestedStructsEvolution(s"${testNamePrefix}UPDATE SET col = s.col, UPDATE SET *")( target = Seq("""{"key":1,"col":null}""", """{"key":2,"col":null}"""), source = Seq( """{"key":1,"col":null,"extra":null}""", """{"key":2,"col":null,"extra":null}""" ), targetSchema = targetSchema, sourceSchema = sourceSchemaWithTopLevelExtra, clauses = update(condition = "s.key = 1", set = "col = s.col") :: update(condition = "s.key = 2", set = "*") :: Nil, result = if (shouldPreserveNullSourceStructsForUpdateStar) { Seq( """{"key":1,"col":null,"extra":null}""", """{"key":2,"col":null,"extra":null}""" ) } else if (shouldPreserveNullSourceStructsForWholeStructAssignment) { Seq( """{"key":1,"col":null,"extra":null}""", """{"key":2,"col":{"y":null,"z":null,"x":null},"extra":{"val":null,"val2":null}}""" ) } else { Seq( """{"key":1,"col":{"y":null,"z":null,"x":null},"extra":{"val":null,"val2":null}}""", """{"key":2,"col":{"y":null,"z":null,"x":null},"extra":{"val":null,"val2":null}}""" ) }, resultSchema = fullyEvolvedTargetSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) testNestedStructsEvolution( s"${testNamePrefix}UPDATE SET col = s.col, INSERT (key, col, extra)")( target = Seq("""{"key":1,"col":null}"""), source = Seq( """{"key":1,"col":null,"extra":null}""", """{"key":2,"col":null,"extra":null}""" ), targetSchema = targetSchema, sourceSchema = sourceSchemaWithTopLevelExtra, clauses = update(condition = "s.key = 1", set = "col = s.col") :: insert(values = "(key, col, extra) VALUES (s.key, s.col, s.extra)") :: Nil, result = if (shouldPreserveNullSourceStructsForWholeStructAssignment) { Seq( """{"key":1,"col":null,"extra":null}""", """{"key":2,"col":null,"extra":null}""" ) } else { Seq( """{"key":1,"col":{"y":null,"z":null,"x":null},"extra":{"val":null,"val2":null}}""", """{"key":2,"col":{"y":null,"z":null,"x":null},"extra":null}""" ) }, resultSchema = fullyEvolvedTargetSchema, expectErrorWithoutEvolutionContains = "Cannot resolve", confs = preserveNullStructsConfs) testNestedStructsEvolution(s"${testNamePrefix}UPDATE SET col = s.col, INSERT *")( target = Seq("""{"key":1,"col":null}"""), source = Seq( """{"key":1,"col":null,"extra":null}""", """{"key":2,"col":null,"extra":null}""" ), targetSchema = targetSchema, sourceSchema = sourceSchemaWithTopLevelExtra, clauses = update(condition = "s.key = 1", set = "col = s.col") :: insert(values = "*") :: Nil, result = if (shouldPreserveNullSourceStructsForWholeStructAssignment) { Seq( """{"key":1,"col":null,"extra":null}""", """{"key":2,"col":null,"extra":null}""" ) } else { Seq( """{"key":1,"col":{"y":null,"z":null,"x":null},"extra":{"val":null,"val2":null}}""", """{"key":2,"col":{"y":null,"z":null,"x":null},"extra":null}""" ) }, resultSchema = fullyEvolvedTargetSchema, expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoSuiteBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.lang.{Integer => JInt} import scala.language.implicitConversions import com.databricks.spark.util.{Log4jUsageLogger, MetricDefinitions, UsageRecord} import org.apache.spark.sql.delta.commands.MergeIntoCommand import org.apache.spark.sql.delta.commands.merge.MergeStats import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.test.ScanReportHelper import org.apache.spark.sql.delta.util.JsonUtils import org.apache.hadoop.fs.Path import org.apache.spark.QueryContext import org.apache.spark.sql.{functions, AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, UnsafeArrayData} import org.apache.spark.sql.catalyst.plans.logical.{SubqueryAlias, View} import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecution import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ trait MergeIntoSuiteBaseMixin extends QueryTest with SharedSparkSession with DeltaSQLTestUtils with ScanReportHelper with MergeIntoTestUtils with MergeIntoSchemaEvolutionMixin { import testImplicits._ // Maps expected error classes to actual error classes. Used to handle error classes that are // different when running using SQL vs. Scala. protected val mappedErrorClasses: Map[String, String] = Map.empty // scalastyle:off argcount def testNestedDataSupport(name: String, namePrefix: String = "nested data support")( source: String, target: String, update: Seq[String], insert: String = null, targetSchema: StructType = null, sourceSchema: StructType = null, result: String = null, errorStrs: Seq[String] = null, confs: Seq[(String, String)] = Seq.empty): Unit = { // scalastyle:on argcount require(result == null ^ errorStrs == null, "either set the result or the error strings") val testName = if (result != null) s"$namePrefix - $name" else s"$namePrefix - analysis error - $name" test(testName) { withSQLConf(confs: _*) { withJsonData(source, target, targetSchema, sourceSchema) { case (sourceName, targetName) => val fieldNames = spark.table(targetName).schema.fieldNames val fieldNamesStr = fieldNames.mkString("`", "`, `", "`") val keyName = s"`${fieldNames.head}`" def execMerge() = executeMerge( target = s"$targetName t", source = s"$sourceName s", condition = s"s.$keyName = t.$keyName", update = update.mkString(", "), insert = Option(insert).getOrElse(s"($fieldNamesStr) VALUES ($fieldNamesStr)")) if (result != null) { execMerge() val expectedDf = readFromJSON(strToJsonSeq(result), targetSchema) checkAnswer(spark.table(targetName), expectedDf) } else { val e = intercept[AnalysisException] { execMerge() } errorStrs.foreach { s => errorContains(e.getMessage, s) } } } } } } /** * Test runner to cover analysis exception in MERGE INTO. */ protected def testMergeAnalysisException( name: String)( mergeOn: String, mergeClauses: MergeClause*)( expectedErrorClass: String, expectedMessageParameters: Map[String, String]): Unit = { test(s"analysis errors - $name") { withKeyValueData( source = Seq.empty, target = Seq.empty, sourceKeyValueNames = ("key", "srcValue"), targetKeyValueNames = ("key", "tgtValue")) { case (sourceName, targetName) => val ex = intercept[AnalysisException] { executeMerge(s"$targetName t", s"$sourceName s", mergeOn, mergeClauses: _*) } // Spark 3.5 and below uses QueryContext, Spark 4.0 and above uses ExpectedContext. // Implicitly convert to ExpectedContext when needed for compatibility. implicit def toExpectedContext(ctxs: Array[QueryContext]): Array[ExpectedContext] = ctxs.map { ctx => ExpectedContext(ctx.fragment(), ctx.startIndex(), ctx.stopIndex()) } checkError( exception = ex, mappedErrorClasses.getOrElse(expectedErrorClass, expectedErrorClass), parameters = expectedMessageParameters, queryContext = ex.getQueryContext ) } } } protected def withJsonData( source: Seq[String], target: Seq[String], schema: StructType = null, sourceSchema: StructType = null)( thunk: (String, String) => Unit): Unit = { def toDF(strs: Seq[String]) = { if (sourceSchema != null && strs == source) { spark.read.schema(sourceSchema).json(strs.toDS) } else if (schema != null) { spark.read.schema(schema).json(strs.toDS) } else { spark.read.json(strs.toDS) } } append(toDF(target), Nil) withTempView("source") { toDF(source).createOrReplaceTempView("source") thunk("source", tableSQLIdentifier) } } protected def insertOnlyMergeFeatureFlagOff(sourceName: String, targetName: String): Unit = { executeMerge( tgt = s"$targetName t", src = s"$sourceName s", cond = "s.key = t.key", insert(values = "(key, value) VALUES (s.key, s.value)")) checkAnswer(sql(s"SELECT key, value FROM $targetName"), Row(1, 1) :: Row(3, 30) :: Nil) val metrics = spark.sql(s"DESCRIBE HISTORY $targetName LIMIT 1") .select("operationMetrics") .collect().head.getMap(0).asInstanceOf[Map[String, String]] assert(metrics.contains("numTargetFilesRemoved")) // If insert-only code path is not used, then the general code path will rewrite existing // target files when DVs are not enabled. if (!spark.conf.get(DeltaSQLConf.MERGE_USE_PERSISTENT_DELETION_VECTORS)) { assert(metrics("numTargetFilesRemoved").toInt > 0) } } def testMergeWithRepartition( name: String, partitionColumns: Seq[String], srcRange: Range, expectLessFilesWithRepartition: Boolean, clauses: MergeClause*): Unit = { test(s"merge with repartition - $name", DisableAdaptiveExecution("AQE coalese would partition number")) { withTempView("source") { withTempDir { basePath => val tgt1 = basePath + "target" val tgt2 = basePath + "targetRepartitioned" val df = spark.range(100).withColumn("part1", 'id % 5).withColumn("part2", 'id % 3) df.write.format("delta").partitionBy(partitionColumns: _*).save(tgt1) df.write.format("delta").partitionBy(partitionColumns: _*).save(tgt2) val cond = "src.id = t.id" val src = srcRange.toDF("id") .withColumn("part1", 'id % 5) .withColumn("part2", 'id % 3) .createOrReplaceTempView("source") // execute merge without repartition withSQLConf(DeltaSQLConf.MERGE_REPARTITION_BEFORE_WRITE.key -> "false") { executeMerge( tgt = s"delta.`$tgt1` as t", src = "source src", cond = cond, clauses = clauses: _*) } // execute merge with repartition - default behavior executeMerge( tgt = s"delta.`$tgt2` as t", src = "source src", cond = cond, clauses = clauses: _*) checkAnswer( io.delta.tables.DeltaTable.forPath(tgt2).toDF, io.delta.tables.DeltaTable.forPath(tgt1).toDF ) val filesAfterNoRepartition = DeltaLog.forTable(spark, tgt1).snapshot.numOfFiles val filesAfterRepartition = DeltaLog.forTable(spark, tgt2).snapshot.numOfFiles // check if there are fewer are number of files for merge with repartition if (expectLessFilesWithRepartition) { assert(filesAfterNoRepartition > filesAfterRepartition) } else { assert(filesAfterNoRepartition === filesAfterRepartition) } } } } } /** * Test whether data skipping on matched predicates of a merge command is performed. * @param name The name of the test case. * @param source The source for merge. * @param target The target for merge. * @param dataSkippingOnTargetOnly The boolean variable indicates whether * when matched clauses are on target fields only. * Data Skipping should be performed before inner join if * this variable is true. * @param isMatchedOnly The boolean variable indicates whether the merge command only * contains when matched clauses. * @param mergeClauses Merge Clauses. */ protected def testMergeDataSkippingOnMatchPredicates( name: String)( source: Seq[(Int, Int)], target: Seq[(Int, Int)], dataSkippingOnTargetOnly: Boolean, isMatchedOnly: Boolean, mergeClauses: MergeClause*)( result: Seq[(Int, Int)]): Unit = { test(s"data skipping with matched predicates - $name") { withKeyValueData(source, target) { case (sourceName, targetName) => val stats = performMergeAndCollectStatsForDataSkippingOnMatchPredicates( sourceName, targetName, result, mergeClauses) // Data skipping on match predicates should only be performed when it's a // matched only merge. if (isMatchedOnly) { // The number of files removed/added should be 0 because of the additional predicates. assert(stats.targetFilesRemoved == 0) assert(stats.targetFilesAdded == 0) // Verify that the additional predicates on data skipping // before inner join filters file out for match predicates only // on target. if (dataSkippingOnTargetOnly) { assert(stats.targetBeforeSkipping.files.get > stats.targetAfterSkipping.files.get) } } else { if (!spark.conf.get(DeltaSQLConf.MERGE_USE_PERSISTENT_DELETION_VECTORS)) { assert(stats.targetFilesRemoved > 0) } // If there is no insert clause and the flag is enabled, data skipping should be // performed on targetOnly predicates. // However, with insert clauses, it's expected that no additional data skipping // is performed on matched clauses. assert(stats.targetBeforeSkipping.files.get == stats.targetAfterSkipping.files.get) assert(stats.targetRowsUpdated == 0) } } } } protected def performMergeAndCollectStatsForDataSkippingOnMatchPredicates( sourceName: String, targetName: String, result: Seq[(Int, Int)], mergeClauses: Seq[MergeClause]): MergeStats = { var events: Seq[UsageRecord] = Seq.empty // Perform merge on merge condition with matched clauses. events = Log4jUsageLogger.track { executeMerge(s"$targetName t", s"$sourceName s", "s.key = t.key", mergeClauses: _*) } checkAnswer( readDeltaTableByIdentifier(targetName), result.map { case (k, v) => Row(k, v) }) // Verify merge stats from usage events val mergeStats = events.filter { e => e.metric == MetricDefinitions.EVENT_TAHOE.name && e.tags.get("opType").contains("delta.dml.merge.stats") } assert(mergeStats.size == 1) JsonUtils.fromJson[MergeStats](mergeStats.head.blob) } /** * @param function the unsupported function. * @param functionType The type of the unsupported expression to be tested. * @param sourceData the data in the source table. * @param targetData the data in the target table. * @param mergeCondition the merge condition containing the unsupported expression. * @param clauseCondition the clause condition containing the unsupported expression. * @param clauseAction the clause action containing the unsupported expression. * @param expectExceptionInAction whether expect exception thrown in action. * @param customConditionErrorRegex the customized error regex for condition. * @param customActionErrorRegex the customized error regex for action. */ def testUnsupportedExpression( function: String, functionType: String, sourceData: => DataFrame, targetData: => DataFrame, mergeCondition: String, clauseCondition: String, clauseAction: String, expectExceptionInAction: Option[Boolean] = None, customConditionErrorRegex: Option[String] = None, customActionErrorRegex: Option[String] = None) { test(s"$functionType functions in merge" + s" - expect exception in action: ${expectExceptionInAction.getOrElse(true)}") { withTable("source", "target") { sourceData.write.format("delta").saveAsTable("source") targetData.write.format("delta").saveAsTable("target") val expectedErrorRegex = "(?s).*(?i)unsupported.*(?i).*Invalid expressions.*" def checkExpression( expectException: Boolean, condition: Option[String] = None, clause: Option[MergeClause] = None, expectedRegex: Option[String] = None) { if (expectException) { val dataBeforeException = spark.read.format("delta").table("target").collect() val e = intercept[Exception] { executeMerge( tgt = "target as t", src = "source as s", cond = condition.getOrElse("s.a = t.a"), clause.getOrElse(update(set = "b = s.b")) ) } def extractErrorClass(e: Throwable): String = e match { case dt: DeltaThrowable => s"\\[${dt.getErrorClass}\\] " case _ => "" } val (message, errorClass) = if (e.getCause != null) { (e.getCause.getMessage, extractErrorClass(e.getCause)) } else (e.getMessage, extractErrorClass(e)) assert(message.matches(errorClass + expectedRegex.getOrElse(expectedErrorRegex))) checkAnswer(spark.read.format("delta").table("target"), dataBeforeException) } else { executeMerge( tgt = "target as t", src = "source as s", cond = condition.getOrElse("s.a = t.a"), clause.getOrElse(update(set = "b = s.b")) ) } } // on merge condition checkExpression( expectException = true, condition = Option(mergeCondition), expectedRegex = customConditionErrorRegex ) // on update condition checkExpression( expectException = true, clause = Option(update(condition = clauseCondition, set = "b = s.b")), expectedRegex = customConditionErrorRegex ) // on update action checkExpression( expectException = expectExceptionInAction.getOrElse(true), clause = Option(update(set = s"b = $clauseAction")), expectedRegex = customActionErrorRegex ) // on insert condition checkExpression( expectException = true, clause = Option( insert(values = "(a, b, c) VALUES (s.a, s.b, s.c)", condition = clauseCondition)), expectedRegex = customConditionErrorRegex ) sql("update source set a = 2") // on insert action checkExpression( expectException = expectExceptionInAction.getOrElse(true), clause = Option(insert(values = s"(a, b, c) VALUES ($clauseAction, s.b, s.c)")), expectedRegex = customActionErrorRegex ) } } } protected def testExtendedMerge( name: String, namePrefix: String = "extended syntax")( source: Seq[(Int, Int)], target: Seq[(Int, Int)], mergeOn: String, mergeClauses: MergeClause*)( result: Seq[(Int, Int)]): Unit = { Seq(true, false).foreach { isPartitioned => test(s"$namePrefix - $name - isPartitioned: $isPartitioned ") { withKeyValueData(source, target, isPartitioned) { case (sourceName, targetName) => withSQLConf(DeltaSQLConf.MERGE_INSERT_ONLY_ENABLED.key -> "true") { executeMerge(s"$targetName t", s"$sourceName s", mergeOn, mergeClauses: _*) } checkAnswer( readDeltaTableByIdentifier(targetName), result.map { case (k, v) => Row(k, v) }) } } } } protected lazy val expectedOpTypes: Set[String] = Set( "delta.dml.merge.materializeSource", "delta.dml.merge.findTouchedFiles", "delta.dml.merge.writeAllChanges", "delta.dml.merge") protected lazy val expectedOpTypesInsertOnly: Set[String] = Set( "delta.dml.merge.materializeSource", "delta.dml.merge.findTouchedFiles", "delta.dml.merge.writeInsertsOnlyWhenNoMatches", "delta.dml.merge") } trait MergeIntoBasicTests extends MergeIntoSuiteBaseMixin { import testImplicits._ Seq(true, false).foreach { isPartitioned => test(s"basic case - merge to Delta table by path, isPartitioned: $isPartitioned", NameBasedAccessIncompatible) { withTable("source") { val partitions = if (isPartitioned) "key2" :: Nil else Nil append(Seq((2, 2), (1, 4)).toDF("key2", "value"), partitions) Seq((1, 1), (0, 3)).toDF("key1", "value").createOrReplaceTempView("source") executeMerge( target = tableSQLIdentifier, source = "source src", condition = "src.key1 = key2", update = "key2 = 20 + key1, value = 20 + src.value", insert = "(key2, value) VALUES (key1 - 10, src.value + 10)") checkAnswer(readDeltaTableByIdentifier(), Row(2, 2) :: // No change Row(21, 21) :: // Update Row(-10, 13) :: // Insert Nil) } } } Seq(true, false).foreach { skippingEnabled => Seq(true, false).foreach { isPartitioned => test("basic case - merge to Delta table, " + s"isPartitioned: $isPartitioned skippingEnabled: $skippingEnabled") { withTable("source") { withSQLConf(DeltaSQLConf.DELTA_STATS_SKIPPING.key -> skippingEnabled.toString) { Seq((1, 1), (0, 3), (1, 6)).toDF("key1", "value").createOrReplaceTempView("source") val partitions = if (isPartitioned) "key2" :: Nil else Nil append(Seq((2, 2), (1, 4)).toDF("key2", "value"), partitions) executeMerge( target = s"$tableSQLIdentifier tgt", source = "source src", condition = "src.key1 = key2 AND src.value < tgt.value", update = "key2 = 20 + key1, value = 20 + src.value", insert = "(key2, value) VALUES (key1 - 10, src.value + 10)") checkAnswer(readDeltaTableByIdentifier(), Row(2, 2) :: // No change Row(21, 21) :: // Update Row(-10, 13) :: // Insert Row(-9, 16) :: // Insert Nil) } } } } } test("basic case - update value from both source and target table") { withTable("source") { append(Seq((2, 2), (1, 4)).toDF("key2", "value")) Seq((1, 1), (0, 3)).toDF("key1", "value").createOrReplaceTempView("source") executeMerge( target = s"$tableSQLIdentifier as trgNew", source = "source src", condition = "src.key1 = key2", update = "key2 = 20 + key2, value = trgNew.value + src.value", insert = "(key2, value) VALUES (key1 - 10, src.value + 10)") checkAnswer(readDeltaTableByIdentifier(), Row(2, 2) :: // No change Row(21, 5) :: // Update Row(-10, 13) :: // Insert Nil) } } test("basic case - columns are specified in wrong order") { withTable("source") { append(Seq((2, 2), (1, 4)).toDF("key2", "value")) Seq((1, 1), (0, 3)).toDF("key1", "value").createOrReplaceTempView("source") executeMerge( target = s"$tableSQLIdentifier as trgNew", source = "source src", condition = "src.key1 = key2", update = "value = trgNew.value + src.value, key2 = 20 + key2", insert = "(value, key2) VALUES (src.value + 10, key1 - 10)") checkAnswer(readDeltaTableByIdentifier(), Row(2, 2) :: // No change Row(21, 5) :: // Update Row(-10, 13) :: // Insert Nil) } } test("basic case - not all columns are specified in update") { withTable("source") { append(Seq((2, 2), (1, 4)).toDF("key2", "value")) Seq((1, 1), (0, 3)).toDF("key1", "value").createOrReplaceTempView("source") executeMerge( target = s"$tableSQLIdentifier as trgNew", source = "source src", condition = "src.key1 = key2", update = "value = trgNew.value + 3", insert = "(key2, value) VALUES (key1 - 10, src.value + 10)") checkAnswer(readDeltaTableByIdentifier(), Row(2, 2) :: // No change Row(1, 7) :: // Update Row(-10, 13) :: // Insert Nil) } } test("basic case - multiple inserts") { withTable("source") { append(Seq((2, 2), (1, 4)).toDF("key2", "value")) Seq((1, 1), (0, 3), (3, 5)).toDF("key1", "value").createOrReplaceTempView("source") executeMerge( tgt = s"$tableSQLIdentifier as trgNew", src = "source src", cond = "src.key1 = key2", insert(condition = "key1 = 0", values = "(key2, value) VALUES (src.key1, src.value + 3)"), insert(values = "(key2, value) VALUES (src.key1 - 10, src.value + 10)")) checkAnswer(readDeltaTableByIdentifier(), Row(2, 2) :: // No change Row(1, 4) :: // No change Row(0, 6) :: // Insert Row(-7, 15) :: // Insert Nil) } } test("basic case - upsert with only rows inserted") { withTable("source") { append(Seq((2, 2), (1, 4)).toDF("key2", "value")) Seq((1, 1), (0, 3)).toDF("key1", "value").createOrReplaceTempView("source") executeMerge( tgt = s"$tableSQLIdentifier as trgNew", src = "source src", cond = "src.key1 = key2", update(condition = "key2 = 5", set = "value = src.value + 3"), insert(values = "(key2, value) VALUES (src.key1 - 10, src.value + 10)")) checkAnswer(readDeltaTableByIdentifier(), Row(2, 2) :: // No change Row(1, 4) :: // No change Row(-10, 13) :: // Insert Nil) } } private def testNullCase(name: String)( target: Seq[(JInt, JInt)], source: Seq[(JInt, JInt)], condition: String, expectedResults: Seq[(JInt, JInt)]) = { Seq(true, false).foreach { isPartitioned => test(s"basic case - null handling - $name, isPartitioned: $isPartitioned") { withView("sourceView") { val partitions = if (isPartitioned) "key" :: Nil else Nil append(target.toDF("key", "value"), partitions) source.toDF("key", "value").createOrReplaceTempView("sourceView") executeMerge( target = s"$tableSQLIdentifier as t", source = "sourceView s", condition = condition, update = "t.value = s.value", insert = "(t.key, t.value) VALUES (s.key, s.value)") checkAnswer( readDeltaTableByIdentifier(), expectedResults.map { r => Row(r._1, r._2) } ) } } } } testNullCase("null value in target")( target = Seq((null, null), (1, 1)), source = Seq((1, 10), (2, 20)), condition = "s.key = t.key", expectedResults = Seq( (null, null), // No change (1, 10), // Update (2, 20) // Insert )) testNullCase("null value in source")( target = Seq((1, 1)), source = Seq((1, 10), (2, 20), (null, null)), condition = "s.key = t.key", expectedResults = Seq( (1, 10), // Update (2, 20), // Insert (null, null) // Insert )) testNullCase("null value in both source and target")( target = Seq((1, 1), (null, null)), source = Seq((1, 10), (2, 20), (null, 0)), condition = "s.key = t.key", expectedResults = Seq( (null, null), // No change as null in source does not match null in target (1, 10), // Update (2, 20), // Insert (null, 0) // Insert )) testNullCase("null value in both source and target + IS NULL in condition")( target = Seq((1, 1), (null, null)), source = Seq((1, 10), (2, 20), (null, 0)), condition = "s.key = t.key AND s.key IS NULL", expectedResults = Seq( (null, null), // No change as s.key != t.key (1, 1), // No change as s.key is not null (null, 0), // Insert (1, 10), // Insert (2, 20) // Insert )) testNullCase("null value in both source and target + IS NOT NULL in condition")( target = Seq((1, 1), (null, null)), source = Seq((1, null), (2, 20), (null, 0)), condition = "s.key = t.key AND t.value IS NOT NULL", expectedResults = Seq( (null, null), // No change as t.value is null (1, null), // Update as t.value is not null (null, 0), // Insert (2, 20) // Insert )) testNullCase("null value in both source and target + <=> in condition")( target = Seq((1, 1), (null, null)), source = Seq((1, 10), (2, 20), (null, 0)), condition = "s.key <=> t.key", expectedResults = Seq( (null, 0), // Update (1, 10), // Update (2, 20) // Insert )) testNullCase("NULL in condition")( target = Seq((1, 1), (null, null)), source = Seq((1, 10), (2, 20), (null, 0)), condition = "s.key = t.key AND NULL", expectedResults = Seq( (null, null), // No change as NULL condition did not match anything (1, 1), // No change as NULL condition did not match anything (null, 0), // Insert (1, 10), // Insert (2, 20) // Insert )) test("basic case - only insert") { withTable("source") { Seq((5, 5)).toDF("key1", "value").createOrReplaceTempView("source") append(Seq.empty[(Int, Int)].toDF("key2", "value")) executeMerge( target = s"$tableSQLIdentifier as target", source = "source src", condition = "src.key1 = target.key2", update = "key2 = 20 + key1, value = 20 + src.value", insert = "(key2, value) VALUES (key1 - 10, src.value + 10)") checkAnswer(readDeltaTableByIdentifier(), Row(-5, 15) :: // Insert Nil) } } test("basic case - both source and target are empty") { withTable("source") { Seq.empty[(Int, Int)].toDF("key1", "value").createOrReplaceTempView("source") append(Seq.empty[(Int, Int)].toDF("key2", "value")) executeMerge( target = s"$tableSQLIdentifier as target", source = "source src", condition = "src.key1 = target.key2", update = "key2 = 20 + key1, value = 20 + src.value", insert = "(key2, value) VALUES (key1 - 10, src.value + 10)") checkAnswer(readDeltaTableByIdentifier(), Nil) } } test("basic case - only update") { withTable("source") { Seq((1, 5), (2, 9)).toDF("key1", "value").createOrReplaceTempView("source") append(Seq((2, 2), (1, 4)).toDF("key2", "value")) executeMerge( target = s"$tableSQLIdentifier as target", source = "source src", condition = "src.key1 = target.key2", update = "key2 = 20 + key1, value = 20 + src.value", insert = "(key2, value) VALUES (key1 - 10, src.value + 10)") checkAnswer(readDeltaTableByIdentifier(), Row(21, 25) :: // Update Row(22, 29) :: // Update Nil) } } private def testLocalPredicates(name: String)( target: Seq[(String, String, String)], source: Seq[(String, String)], condition: String, expectedResults: Seq[(String, String, String)], numFilesPerPartition: Int = 2) = { Seq(true, false).foreach { isPartitioned => test(s"$name, isPartitioned: $isPartitioned") { withTable("source") { val partitions = if (isPartitioned) "key2" :: Nil else Nil append(target.toDF("key2", "value", "op").repartition(numFilesPerPartition), partitions) source.toDF("key1", "value").createOrReplaceTempView("source") // Local predicates are likely to be pushed down leading empty join conditions // and cross-join being used withCrossJoinEnabled { executeMerge( target = s"$tableSQLIdentifier trg", source = "source src", condition = condition, update = "key2 = src.key1, value = src.value, op = 'update'", insert = "(key2, value, op) VALUES (src.key1, src.value, 'insert')") } checkAnswer( readDeltaTableByIdentifier(), expectedResults.map { r => Row(r._1, r._2, r._3) } ) } }} } testLocalPredicates("basic case - local predicates - predicate has no matches, only inserts")( target = Seq(("2", "2", "noop"), ("1", "4", "noop"), ("3", "2", "noop"), ("4", "4", "noop")), source = Seq(("1", "8"), ("0", "3")), condition = "src.key1 = key2 and key2 != '1'", expectedResults = ("2", "2", "noop") :: ("1", "4", "noop") :: ("3", "2", "noop") :: ("4", "4", "noop") :: ("1", "8", "insert") :: ("0", "3", "insert") :: Nil) testLocalPredicates("basic case - local predicates - predicate has matches, updates and inserts")( target = Seq(("1", "2", "noop"), ("1", "4", "noop"), ("3", "2", "noop"), ("4", "4", "noop")), source = Seq(("1", "8"), ("0", "3")), condition = "src.key1 = key2 and key2 < '3'", expectedResults = ("3", "2", "noop") :: ("4", "4", "noop") :: ("1", "8", "update") :: ("1", "8", "update") :: ("0", "3", "insert") :: Nil) testLocalPredicates("basic case - local predicates - predicate has matches, only updates")( target = Seq(("1", "2", "noop"), ("1", "4", "noop"), ("3", "2", "noop"), ("4", "4", "noop")), source = Seq(("1", "8")), condition = "key2 < '3'", expectedResults = ("3", "2", "noop") :: ("4", "4", "noop") :: ("1", "8", "update") :: ("1", "8", "update") :: Nil) testLocalPredicates("basic case - local predicates - always false predicate, only inserts")( target = Seq(("1", "2", "noop"), ("1", "4", "noop"), ("3", "2", "noop"), ("4", "4", "noop")), source = Seq(("1", "8"), ("0", "3")), condition = "1 != 1", expectedResults = ("1", "2", "noop") :: ("1", "4", "noop") :: ("3", "2", "noop") :: ("4", "4", "noop") :: ("1", "8", "insert") :: ("0", "3", "insert") :: Nil) testLocalPredicates("basic case - local predicates - always true predicate, all updated")( target = Seq(("1", "2", "noop"), ("1", "4", "noop"), ("3", "2", "noop"), ("4", "4", "noop")), source = Seq(("1", "8")), condition = "1 = 1", expectedResults = ("1", "8", "update") :: ("1", "8", "update") :: ("1", "8", "update") :: ("1", "8", "update") :: Nil) testLocalPredicates("basic case - local predicates - single file, updates and inserts")( target = Seq(("1", "2", "noop"), ("1", "4", "noop"), ("3", "2", "noop"), ("4", "4", "noop")), source = Seq(("1", "8"), ("3", "10"), ("0", "3")), condition = "src.key1 = key2 and key2 < '3'", expectedResults = ("3", "2", "noop") :: ("4", "4", "noop") :: ("1", "8", "update") :: ("1", "8", "update") :: ("0", "3", "insert") :: ("3", "10", "insert") :: Nil, numFilesPerPartition = 1 ) Seq(true, false).foreach { isPartitioned => test(s"basic case - column pruning, isPartitioned: $isPartitioned") { withTable("source") { val partitions = if (isPartitioned) "key2" :: Nil else Nil append(Seq((2, 2), (1, 4)).toDF("key2", "value"), partitions) Seq((1, 1, "a"), (0, 3, "b")).toDF("key1", "value", "col1") .createOrReplaceTempView("source") executeMerge( target = s"$tableSQLIdentifier", source = "source src", condition = "src.key1 = key2", update = "key2 = 20 + key1, value = 20 + src.value", insert = "(key2, value) VALUES (key1 - 10, src.value + 10)") checkAnswer(readDeltaTableByIdentifier(), Row(2, 2) :: // No change Row(21, 21) :: // Update Row(-10, 13) :: // Insert Nil) } } } private def testNullCaseInsertOnly(name: String)( target: Seq[(JInt, JInt)], source: Seq[(JInt, JInt)], condition: String, expectedResults: Seq[(JInt, JInt)], insertCondition: Option[String] = None) = { Seq(true, false).foreach { isPartitioned => test(s"basic case - null handling - $name, isPartitioned: $isPartitioned") { withView("sourceView") { val partitions = if (isPartitioned) "key" :: Nil else Nil append(target.toDF("key", "value"), partitions) source.toDF("key", "value").createOrReplaceTempView("sourceView") withSQLConf(DeltaSQLConf.MERGE_INSERT_ONLY_ENABLED.key -> "true") { if (insertCondition.isDefined) { executeMerge( s"$tableSQLIdentifier as t", "sourceView s", condition, insert("(t.key, t.value) VALUES (s.key, s.value)", condition = insertCondition.get)) } else { executeMerge( s"$tableSQLIdentifier as t", "sourceView s", condition, insert("(t.key, t.value) VALUES (s.key, s.value)")) } } checkAnswer( readDeltaTableByIdentifier(), expectedResults.map { r => Row(r._1, r._2) } ) } } } } testNullCaseInsertOnly("insert only merge - null in source") ( target = Seq((1, 1)), source = Seq((1, 10), (2, 20), (null, null)), condition = "s.key = t.key", expectedResults = Seq( (1, 1), // Existing value (2, 20), // Insert (null, null) // Insert )) testNullCaseInsertOnly("insert only merge - null value in both source and target")( target = Seq((1, 1), (null, null)), source = Seq((1, 10), (2, 20), (null, 0)), condition = "s.key = t.key", expectedResults = Seq( (null, null), // No change as null in source does not match null in target (1, 1), // Existing value (2, 20), // Insert (null, 0) // Insert )) testNullCaseInsertOnly("insert only merge - null in insert clause")( target = Seq((1, 1), (2, 20)), source = Seq((1, 10), (3, 30), (null, 0)), condition = "s.key = t.key", expectedResults = Seq( (1, 1), // Existing value (2, 20), // Existing value (null, 0) // Insert ), insertCondition = Some("s.key IS NULL") ) } trait MergeIntoTempViewsTests extends MergeIntoSuiteBaseMixin with DeltaTestUtilsForTempViews { import testImplicits._ Seq(true, false).foreach { skippingEnabled => Seq(true, false).foreach { partitioned => Seq(true, false).foreach { useSQLView => test("basic case - merge to view on a Delta table, " + s"partitioned: $partitioned skippingEnabled: $skippingEnabled useSqlView: $useSQLView") { withTable("delta_target", "source") { withSQLConf(DeltaSQLConf.DELTA_STATS_SKIPPING.key -> skippingEnabled.toString) { Seq((1, 1), (0, 3), (1, 6)).toDF("key1", "value").createOrReplaceTempView("source") val partitions = if (partitioned) "key2" :: Nil else Nil append(Seq((2, 2), (1, 4)).toDF("key2", "value"), partitions) if (useSQLView) { sql(s"CREATE OR REPLACE TEMP VIEW delta_target AS " + s"SELECT * FROM $tableSQLIdentifier t") } else { readDeltaTableByIdentifier() .createOrReplaceTempView("delta_target") } executeMerge( target = "delta_target", source = "source src", condition = "src.key1 = key2 AND src.value < delta_target.value", update = "key2 = 20 + key1, value = 20 + src.value", insert = "(key2, value) VALUES (key1 - 10, src.value + 10)") checkAnswer(sql("SELECT key2, value FROM delta_target"), Row(2, 2) :: // No change Row(21, 21) :: // Update Row(-10, 13) :: // Insert Row(-9, 16) :: // Insert Nil) } } } } } } test("Negative case - more operations between merge and delta target") { withTempView("source", "target") { Seq((1, 1), (0, 3), (1, 5)).toDF("key1", "value").createOrReplaceTempView("source") append(Seq((2, 2), (1, 4)).toDF("key2", "value")) readDeltaTableByIdentifier().filter("value <> 0").createTempView("target") val e = intercept[AnalysisException] { executeMerge( target = "target", source = "source src", condition = "src.key1 = target.key2", update = "key2 = 20 + key1, value = 20 + src.value", insert = "(key2, value) VALUES (key1 - 10, src.value + 10)") }.getMessage errorContains(e, "Expect a full scan of Delta sources, but found a partial scan") } } /** * Ensure we can successfully remove the temp view from the target plan during MERGE analysis. * Failing to do so can cause MERGE execution to fail later on as it is assumed the target plan is * a simple logical relation by then without projections. */ private def checkStripViewFromTarget(target: String): Unit = { val targetViewPlan = sql(s"SELECT * FROM $target").queryExecution.analyzed.collect { case v: View => v } assert(targetViewPlan.size === 1, s"Expected 1 view in target plan, got ${targetViewPlan.size}") DeltaViewHelper.stripTempViewForMerge(targetViewPlan.head, conf) match { case SubqueryAlias(_, _: LogicalRelation) => case _ => fail(s"DeltaViewHelper.stripTempViewForMerge doesn't correctly handle" + s"removing the view from plan:\n${targetViewPlan.head}") } } private def testTempViews(name: String)( text: String, mergeCondition: String, expectedResult: ExpectedResult[Seq[Row]], checkViewStripped: Boolean = true): Unit = { testWithTempView(s"test merge on temp view - $name") { isSQLTempView => withTable("tab") { withTempView("src") { Seq((0, 3), (1, 2)).toDF("key", "value").write.format("delta").saveAsTable("tab") createTempViewFromSelect(text, isSQLTempView) sql("CREATE TEMP VIEW src AS SELECT * FROM VALUES (1, 2), (3, 4) AS t(a, b)") def runMerge(): Unit = executeMerge( target = "v", source = "src", condition = mergeCondition, update = "v.value = src.b + 1", insert = "(v.key, v.value) VALUES (src.a, src.b)") expectedResult match { case ExpectedResult.Failure(checkError) => val ex = intercept[AnalysisException] { runMerge() } checkError(ex) case ExpectedResult.Success(expectedRows: Seq[Row]) => if (checkViewStripped) { checkStripViewFromTarget(target = "v") } runMerge() checkAnswer(spark.table("v"), expectedRows) } } } } } testTempViews("basic")( text = "SELECT * FROM tab", mergeCondition = "src.a = v.key AND src.b = v.value", expectedResult = ExpectedResult.Success(Seq(Row(0, 3), Row(1, 3), Row(3, 4))) ) testTempViews("basic - merge condition references subset of target cols")( text = "SELECT * FROM tab", mergeCondition = "src.a = v.key", expectedResult = ExpectedResult.Success(Seq(Row(0, 3), Row(1, 3), Row(3, 4))) ) testTempViews("subset cols")( text = "SELECT key FROM tab", mergeCondition = "src.a = v.key AND src.b = v.value", expectedResult = ExpectedResult.Failure { ex => assert(ex.getErrorClass === "UNRESOLVED_COLUMN.WITH_SUGGESTION") } ) testTempViews("superset cols")( text = "SELECT key, value, 1 FROM tab", mergeCondition = "src.a = v.key AND src.b = v.value", // The analyzer can't tell whether the table originally had the extra column or not. expectedResult = ExpectedResult.Failure { ex => checkErrorMatchPVals( ex, "DELTA_SCHEMA_CHANGE_SINCE_ANALYSIS", parameters = Map( "schemaDiff" -> "(?s)Latest schema is missing field.*", "legacyFlagMessage" -> "" )) } ) testTempViews("nontrivial projection")( text = "SELECT value as key, key as value FROM tab", mergeCondition = "src.a = v.key AND src.b = v.value", expectedResult = ExpectedResult.Success(Seq(Row(2, 1), Row(2, 1), Row(3, 0), Row(4, 3))), // The view doesn't get stripped by DeltaViewHelper.stripTempViewForMerge during analysis in // this case, doing it would be incorrect since the view is not a simple projection. // We really shouldn't support this use case altogether but due to historical reasons we have to // keep supporting it. checkViewStripped = false ) testTempViews("view with too many internal aliases")( text = "SELECT * FROM (SELECT * FROM tab AS t1) AS t2", mergeCondition = "src.a = v.key AND src.b = v.value", expectedResult = ExpectedResult.Success(Seq(Row(0, 3), Row(1, 3), Row(3, 4))) ) testTempViews("view with too many internal aliases - merge condition references subset of " + s"target cols")( text = "SELECT * FROM (SELECT * FROM tab AS t1) AS t2", mergeCondition = "src.a = v.key", expectedResult = ExpectedResult.Success(Seq(Row(0, 3), Row(1, 3), Row(3, 4))) ) } trait MergeIntoNestedDataTests extends MergeIntoSuiteBaseMixin { testNestedDataSupport("no update when not matched, only insert")( source = """ { "key": { "x": "X3", "y": 3}, "value": { "a": 300, "b": "B300" } }""", target = """ { "key": { "x": "X1", "y": 1}, "value": { "a": 1, "b": "B1" } } { "key": { "x": "X2", "y": 2}, "value": { "a": 2, "b": "B2" } }""", update = "value.b = 'UPDATED'" :: Nil, result = """ { "key": { "x": "X1", "y": 1}, "value": { "a": 1, "b": "B1" } } { "key": { "x": "X2", "y": 2}, "value": { "a": 2, "b": "B2" } } { "key": { "x": "X3", "y": 3}, "value": { "a": 300, "b": "B300" } }""") testNestedDataSupport("update entire nested column")( source = """ { "key": { "x": "X1", "y": 1}, "value": { "a": 100, "b": "B100" } }""", target = """ { "key": { "x": "X1", "y": 1}, "value": { "a": 1, "b": "B1" } } { "key": { "x": "X2", "y": 2}, "value": { "a": 2, "b": "B2" } }""", update = "value = s.value" :: Nil, result = """ { "key": { "x": "X1", "y": 1}, "value": { "a": 100, "b": "B100" } } { "key": { "x": "X2", "y": 2}, "value": { "a": 2, "b": "B2" } }""") testNestedDataSupport("update one nested field")( source = """ { "key": { "x": "X1", "y": 1}, "value": { "a": 100, "b": "B100" } }""", target = """ { "key": { "x": "X1", "y": 1}, "value": { "a": 1, "b": "B1" } } { "key": { "x": "X2", "y": 2}, "value": { "a": 2, "b": "B2" } }""", update = "value.b = s.value.b" :: Nil, result = """ { "key": { "x": "X1", "y": 1}, "value": { "a": 1, "b": "B100" } } { "key": { "x": "X2", "y": 2}, "value": { "a": 2, "b": "B2" } }""") testNestedDataSupport("update multiple fields at different levels")( source = """ { "key": { "x": "X1", "y": { "i": 1.0 } }, "value": { "a": 100, "b": "B100" } }""", target = """ { "key": { "x": "X1", "y": { "i": 1.0 } }, "value": { "a": 1, "b": "B1" } } { "key": { "x": "X2", "y": { "i": 2.0 } }, "value": { "a": 2, "b": "B2" } }""", update = "key.x = 'XXX'" :: "key.y.i = 9000" :: "value = named_struct('a', 9000, 'b', s.value.b)" :: Nil, result = """ { "key": { "x": "XXX", "y": { "i": 9000 } }, "value": { "a": 9000, "b": "B100" } } { "key": { "x": "X2" , "y": { "i": 2.0 } }, "value": { "a": 2, "b": "B2" } }""") testNestedDataSupport("update multiple fields at different levels to NULL")( source = """ { "key": { "x": "X1", "y": { "i": 1.0 } }, "value": { "a": 100, "b": "B100" } }""", target = """ { "key": { "x": "X1", "y": { "i": 1.0 } }, "value": { "a": 1, "b": "B1" } } { "key": { "x": "X2", "y": { "i": 2.0 } }, "value": { "a": 2, "b": "B2" } }""", update = "value = NULL" :: "key.x = NULL" :: "key.y.i = NULL" :: Nil, result = """ { "key": { "x": null, "y": { "i" : null } }, "value": null } { "key": { "x": "X2" , "y": { "i" : 2.0 } }, "value": { "a": 2, "b": "B2" } }""") testNestedDataSupport("update multiple fields at different levels with implicit casting")( source = """ { "key": { "x": "X1", "y": { "i": 1.0 } }, "value": { "a": 100, "b": "B100" } }""", target = """ { "key": { "x": "X1", "y": { "i": 1.0 } }, "value": { "a": 1, "b": "B1" } } { "key": { "x": "X2", "y": { "i": 2.0 } }, "value": { "a": 2, "b": "B2" } }""", update = "key.x = 'XXX' " :: "key.y.i = '9000'" :: "value = named_struct('a', '9000', 'b', s.value.b)" :: Nil, result = """ { "key": { "x": "XXX", "y": { "i": 9000 } }, "value": { "a": 9000, "b": "B100" } } { "key": { "x": "X2" , "y": { "i": 2.0 } }, "value": { "a": 2, "b": "B2" } }""") testNestedDataSupport("update array fields at different levels")( source = """ { "key": { "x": "X1", "y": [ 1, 11 ] }, "value": [ -1, -10 , -100 ] }""", target = """ { "key": { "x": "X1", "y": [ 1, 11 ] }, "value": [ 1, 10 , 100 ]} } { "key": { "x": "X2", "y": [ 2, 22 ] }, "value": [ 2, 20 , 200 ]} }""", update = "value = array(-9000)" :: "key.y = array(-1, -11)" :: Nil, result = """ { "key": { "x": "X1", "y": [ -1, -11 ] }, "value": [ -9000 ]} } { "key": { "x": "X2", "y": [ 2, 22 ] }, "value": [ 2, 20 , 200 ]} }""") testNestedDataSupport("update using quoted names at different levels", "dotted name support")( source = """ { "key": { "x": "X1", "y.i": 1.0 }, "value.a": "A" }""", target = """ { "key": { "x": "X1", "y.i": 1.0 }, "value.a": "A1" } { "key": { "x": "X2", "y.i": 2.0 }, "value.a": "A2" }""", update = "`t`.key.`y.i` = 9000" :: "t.`value.a` = 'UPDATED'" :: Nil, result = """ { "key": { "x": "X1", "y.i": 9000 }, "value.a": "UPDATED" } { "key": { "x": "X2", "y.i" : 2.0 }, "value.a": "A2" }""") testNestedDataSupport("unknown nested field")( source = """{ "key": "A", "value": { "a": 0 } }""", target = """{ "key": "A", "value": { "a": 1 } }""", update = "value.c = 'UPDATED'" :: Nil, errorStrs = "No such struct field" :: Nil) testNestedDataSupport("assigning simple type to struct field")( source = """{ "key": "A", "value": { "a": { "x": 1 } } }""", target = """{ "key": "A", "value": { "a": { "x": 1 } } }""", update = "value.a = 'UPDATED'" :: Nil, errorStrs = "data type mismatch" :: Nil) testNestedDataSupport("conflicting assignments between two nested fields at different levels")( source = """{ "key": "A", "value": { "a": { "x": 0 } } }""", target = """{ "key": "A", "value": { "a": { "x": 1 } } }""", update = "value.a.x = 2" :: "value.a = named_struct('x', 3)" :: Nil, errorStrs = "There is a conflict from these SET columns" :: Nil) testNestedDataSupport("conflicting assignments between nested field and top-level column")( source = """{ "key": "A", "value": { "a": 0 } }""", target = """{ "key": "A", "value": { "a": 1 } }""", update = "value.a = 2" :: "value = named_struct('a', 3)" :: Nil, errorStrs = "There is a conflict from these SET columns" :: Nil) testNestedDataSupport("nested field not supported in INSERT")( source = """{ "key": "A", "value": { "a": 0 } }""", target = """{ "key": "B", "value": { "a": 1 } }""", update = "value.a = 2" :: Nil, insert = """(key, value.a) VALUES (s.key, s.value.a)""", errorStrs = "Nested field is not supported in the INSERT clause" :: Nil) testNestedDataSupport("updating map type")( source = """{ "key": "A", "value": { "a": 0 } }""", target = """{ "key": "A", "value": { "a": 1 } }""", update = "value.a = 2" :: Nil, targetSchema = new StructType().add("key", StringType).add("value", MapType(StringType, IntegerType)), errorStrs = "Updating nested fields is only supported for StructType" :: Nil) testNestedDataSupport("updating array type")( source = """{ "key": "A", "value": [ { "a": 0 } ] }""", target = """{ "key": "A", "value": [ { "a": 1 } ] }""", update = "value.a = 2" :: Nil, targetSchema = new StructType().add("key", StringType).add("value", MapType(StringType, IntegerType)), errorStrs = "Updating nested fields is only supported for StructType" :: Nil) testNestedDataSupport("resolution by name - update specific column")( source = """{ "key": "A", "value": { "b": 2, "a": { "y": 20, "x": 10} } }""", target = """{ "key": "A", "value": { "a": { "x": 1, "y": 2 }, "b": 1 }}""", targetSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("a", new StructType().add("x", IntegerType).add("y", IntegerType)) .add("b", IntegerType)), sourceSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("b", IntegerType) .add("a", new StructType().add("y", IntegerType).add("x", IntegerType))), update = "value.a = s.value.a", result = """{ "key": "A", "value": { "a": { "x": 10, "y": 20 }, "b": 1 } }""") // scalastyle:off line.size.limit testNestedDataSupport("resolution by name - update specific column - array of struct - longer source")( source = """{ "key": "A", "value": [ { "b": "2", "a": { "y": 20, "x": 10 } }, { "b": "3", "a": { "y": 30, "x": 40 } } ] }""", target = """{ "key": "A", "value": [ { "a": { "x": 1, "y": 2 }, "b": 1 } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType().add("x", IntegerType).add("y", IntegerType)) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType().add("y", IntegerType).add("x", IntegerType)))), update = "value = s.value", result = """{ "key": "A", "value": [ { "a": { "x": 10, "y": 20 }, "b": 2 }, { "a": { "y": 30, "x": 40}, "b": 3 } ] }""") testNestedDataSupport("resolution by name - update specific column - array of struct - longer target")( source = """{ "key": "A", "value": [ { "b": "2", "a": { "y": 20, "x": 10 } } ] }""", target = """{ "key": "A", "value": [{ "a": { "x": 1, "y": 2 }, "b": 1 }, { "a": { "x": 2, "y": 3 }, "b": 2 }] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType().add("x", IntegerType).add("y", IntegerType)) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType().add("y", IntegerType).add("x", IntegerType)))), update = "value = s.value", result = """{ "key": "A", "value": [ { "a": { "x": 10, "y": 20 }, "b": 2 } ] }""") testNestedDataSupport("resolution by name - update specific column - nested array of struct - longer source")( source = """{ "key": "A", "value": [ { "b": "2", "a": { "y": 20, "x": [ { "c": 10, "d": "30" }, { "c": 3, "d": "40" } ] } } ] }""", target = """{ "key": "A", "value": [ { "a": { "y": 2, "x": [ { "c": 1, "d": 3} ] }, "b": 1 } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType)))) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", StringType) ))))), update = "value = s.value", result = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": 30 }, { "c": 3, "d": 40 } ] }, "b": 2 } ] }""") testNestedDataSupport("resolution by name - update specific column - nested array of struct - longer target")( source = """{ "key": "A", "value": [ { "b": "2", "a": { "y": 20, "x": [ { "c": 10, "d": "30" } ] } } ] }""", target = """{ "key": "A", "value": [ { "a": { "y": 2, "x": [ { "c": 1, "d": 3 }, { "c": 2, "d": 4 } ] }, "b": 1 } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType)))) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", StringType) ))))), update = "value = s.value", result = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": 30}]}, "b": 2 } ] }""") // scalastyle:on line.size.limit testNestedDataSupport("resolution by name - update *")( source = """{ "key": "A", "value": { "b": 2, "a": { "y": 20, "x": 10} } }""", target = """{ "key": "A", "value": { "a": { "x": 1, "y": 2 }, "b": 1 }}""", targetSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("a", new StructType().add("x", IntegerType).add("y", IntegerType)) .add("b", IntegerType)), sourceSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("b", IntegerType) .add("a", new StructType().add("y", IntegerType).add("x", IntegerType))), update = "*", result = """{ "key": "A", "value": { "a": { "x": 10, "y": 20 } , "b": 2} }""") // scalastyle:off line.size.limit testNestedDataSupport("resolution by name - update * - array of struct - longer source")( source = """{ "key": "A", "value": [ { "b": "2", "a": { "y": 20, "x": 10 } }, { "b": "3", "a": { "y": 30, "x": 40 } } ] }""", target = """{ "key": "A", "value": [ { "a": { "x": 1, "y": 2 }, "b": 1 } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType().add("x", IntegerType).add("y", IntegerType)) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType().add("y", IntegerType).add("x", IntegerType)))), update = "*", result = """{ "key": "A", "value": [ { "a": { "x": 10, "y": 20 }, "b": 2 }, { "a": { "y": 30, "x": 40}, "b": 3 } ] }""".stripMargin) testNestedDataSupport("resolution by name - update * - array of struct - longer target")( source = """{ "key": "A", "value": [ { "b": "2", "a": { "y": 20, "x": 10 } } ] }""", target = """{ "key": "A", "value": [ { "a": { "x": 1, "y": 2 }, "b": 1 }, { "a": { "x": 2, "y": 3 }, "b": 4 }] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType().add("x", IntegerType).add("y", IntegerType)) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType().add("y", IntegerType).add("x", IntegerType)))), update = "*", result = """{ "key": "A", "value": [ { "a": { "x": 10, "y": 20 }, "b": 2 } ] }""".stripMargin) testNestedDataSupport("resolution by name - update * - nested array of struct - longer source")( source = """{ "key": "A", "value": [ { "b": "2", "a": { "y": 20, "x": [{ "c": 10, "d": "30"}, { "c": 3, "d": "40" } ] } } ] }""", target = """{ "key": "A", "value": [ { "a": { "y": 2, "x": [ { "c": 1, "d": 3 } ] }, "b": 1 } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType)))) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", StringType) ))))), update = "*", result = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": 30}, { "c": 3, "d": 40 } ] }, "b": 2 } ] }""") testNestedDataSupport("resolution by name - update * - nested array of struct - longer target")( source = """{ "key": "A", "value": [ { "b": "2", "a": { "y": 20, "x": [ { "c": 10, "d": "30" } ] } } ] }""", target = """{ "key": "A", "value": [ { "a": { "y": 2, "x": [ { "c": 1, "d": 3}, { "c": 2, "d": 4} ] }, "b": 1 } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType)))) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", StringType) ))))), update = "*", result = """{ "key": "A", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": 30 } ] }, "b": 2 } ] }""") // scalastyle:on line.size.limit testNestedDataSupport("resolution by name - insert specific column")( source = """{ "key": "B", "value": { "b": 2, "a": { "y": 20, "x": 10 } } }""", target = """{ "key": "A", "value": { "a": { "x": 1, "y": 2 }, "b": 1 } }""", targetSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("a", new StructType().add("x", IntegerType).add("y", IntegerType)) .add("b", IntegerType)), sourceSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("b", IntegerType) .add("a", new StructType().add("y", IntegerType).add("x", IntegerType))), update = "*", insert = "(key, value) VALUES (s.key, s.value)", result = """ |{ "key": "A", "value": { "a": { "x": 1, "y": 2 }, "b": 1 } }, |{ "key": "B", "value": { "a": { "x": 10, "y": 20 }, "b": 2 } }""".stripMargin) // scalastyle:off line.size.limit testNestedDataSupport("resolution by name - insert specific column - array of struct")( source = """{ "key": "B", "value": [ { "b": "2", "a": { "y": 20, "x": 10 } }, { "b": "3", "a": { "y": 30, "x": 40 } } ] }""", target = """{ "key": "A", "value": [ { "a": { "x": 1, "y": 2 }, "b": 1 } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType().add("x", IntegerType).add("y", IntegerType)) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType().add("y", IntegerType).add("x", IntegerType)))), update = "*", insert = "(key, value) VALUES (s.key, s.value)", result = """ { "key": "A", "value": [ { "a": { "x": 1, "y": 2 }, "b": 1 } ] }, { "key": "B", "value": [ { "a": { "x": 10, "y": 20 }, "b": 2 }, { "a": { "y": 30, "x": 40}, "b": 3 } ] }""") testNestedDataSupport("resolution by name - insert specific column - nested array of struct")( source = """{ "key": "B", "value": [ { "b": "2", "a": { "y": 20, "x": [ { "c": 10, "d": "30" }, { "c": 3, "d": "40" } ] } } ] }""", target = """{ "key": "A", "value": [ { "a": { "y": 2, "x": [ { "c": 1, "d": 3 } ] }, "b": 1 } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType)))) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", StringType) ))))), update = "*", insert = "(key, value) VALUES (s.key, s.value)", result = """ { "key": "A", "value": [ { "a": { "y": 2, "x": [ { "c": 1, "d": 3 } ] }, "b": 1 } ] }, { "key": "B", "value": [ { "a": { "y": 20, "x": [ { "c": 10, "d": 30 }, { "c": 3, "d": 40 } ] }, "b": 2 } ] }""") // scalastyle:on line.size.limit testNestedDataSupport("resolution by name - insert *")( source = """{ "key": "B", "value": { "b": 2, "a": { "y": 20, "x": 10} } }""", target = """{ "key": "A", "value": { "a": { "x": 1, "y": 2 }, "b": 1 } }""", targetSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("a", new StructType().add("x", IntegerType).add("y", IntegerType)) .add("b", IntegerType)), sourceSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("b", IntegerType) .add("a", new StructType().add("y", IntegerType).add("x", IntegerType))), update = "*", insert = "*", result = """ |{ "key": "A", "value": { "a": { "x": 1, "y": 2 }, "b": 1 } }, |{ "key": "B", "value": { "a": { "x": 10, "y": 20 }, "b": 2 } }""".stripMargin) // scalastyle:off line.size.limit testNestedDataSupport("resolution by name - insert * - array of struct")( source = """{ "key": "B", "value": [ { "b": "2", "a": { "y": 20, "x": 10} }, { "b": "3", "a": { "y": 30, "x": 40 } } ] }""", target = """{ "key": "A", "value": [ { "a": { "x": 1, "y": 2 }, "b": 1 } ] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType().add("x", IntegerType).add("y", IntegerType)) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType().add("y", IntegerType).add("x", IntegerType)))), update = "*", insert = "*", result = """ |{ "key": "A", "value": [ { "a": { "x": 1, "y": 2 }, "b": 1 } ] }, |{ "key": "B", "value": [ { "a": { "x": 10, "y": 20 }, "b": 2 }, { "a": { "y": 30, "x": 40}, "b": 3 } ] }""".stripMargin) testNestedDataSupport("resolution by name - insert * - nested array of struct")( source = """{ "key": "B", "value": [{ "b": "2", "a": { "y": 20, "x": [ { "c": 10, "d": "30"}, { "c": 3, "d": "40"} ] } } ] }""", target = """{ "key": "A", "value": [{ "a": { "y": 2, "x": [ { "c": 1, "d": 3} ] }, "b": 1 }] }""", targetSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", IntegerType)))) .add("b", IntegerType))), sourceSchema = new StructType() .add("key", StringType) .add("value", ArrayType( new StructType() .add("b", StringType) .add("a", new StructType() .add("y", IntegerType) .add("x", ArrayType( new StructType() .add("c", IntegerType) .add("d", StringType) ))))), update = "*", insert = "*", result = """ |{ "key": "A", "value": [{ "a": { "y": 2, "x": [{ "c": 1, "d": 3}]}, "b": 1 }] }, |{ "key": "B", "value": [{ "a": { "y": 20, "x": [{ "c": 10, "d": 30}, { "c": 3, "d": 40}]}, "b": 2 }]}""".stripMargin) // scalastyle:on line.size.limit // Note that value.b has to be in the right position for this test to avoid throwing an error // trying to write its integer value into the value.a struct. testNestedDataSupport("update resolution by position with conf")( source = """{ "key": "A", "value": { "a": { "y": 20, "x": 10}, "b": 2 }}""", target = """{ "key": "A", "value": { "a": { "x": 1, "y": 2 }, "b": 1 } }""", targetSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("a", new StructType().add("x", IntegerType).add("y", IntegerType)) .add("b", IntegerType)), sourceSchema = new StructType() .add("key", StringType) .add("value", new StructType() .add("a", new StructType().add("y", IntegerType).add("x", IntegerType)) .add("b", IntegerType)), update = "*", insert = "(key, value) VALUES (s.key, s.value)", result = """{ "key": "A", "value": { "a": { "x": 20, "y": 10 }, "b": 2 } }""", confs = (DeltaSQLConf.DELTA_RESOLVE_MERGE_UPDATE_STRUCTS_BY_NAME.key, "false") +: Nil) } trait MergeIntoUnlimitedMergeClausesTests extends MergeIntoSuiteBaseMixin { private def testUnlimitedClauses( name: String)( source: Seq[(Int, Int)], target: Seq[(Int, Int)], mergeOn: String, mergeClauses: MergeClause*)( result: Seq[(Int, Int)]): Unit = testExtendedMerge(name, "unlimited clauses")(source, target, mergeOn, mergeClauses : _*)(result) testUnlimitedClauses("two conditional update + two conditional delete + insert")( source = (0, 0) :: (1, 100) :: (3, 300) :: (4, 400) :: (5, 500) :: Nil, target = (1, 10) :: (2, 20) :: (3, 30) :: (4, 40) :: Nil, mergeOn = "s.key = t.key", delete(condition = "s.key < 2"), delete(condition = "s.key > 4"), update(condition = "s.key == 3", set = "key = s.key, value = s.value"), update(condition = "s.key == 4", set = "key = s.key, value = 2 * s.value"), insert(condition = null, values = "(key, value) VALUES (s.key, s.value)"))( result = Seq( (0, 0), // insert (0, 0) // delete (1, 10) (2, 20), // neither updated nor deleted as it didn't match (3, 300), // update (3, 30) (4, 800), // update (4, 40) (5, 500) // insert (5, 500) )) testUnlimitedClauses("two conditional delete + conditional update + update + insert")( source = (0, 0) :: (1, 100) :: (2, 200) :: (3, 300) :: (4, 400) :: Nil, target = (1, 10) :: (2, 20) :: (3, 30) :: (4, 40) :: Nil, mergeOn = "s.key = t.key", delete(condition = "s.key < 2"), delete(condition = "s.key > 3"), update(condition = "s.key == 2", set = "key = s.key, value = s.value"), update(condition = null, set = "key = s.key, value = 2 * s.value"), insert(condition = null, values = "(key, value) VALUES (s.key, s.value)"))( result = Seq( (0, 0), // insert (0, 0) // delete (1, 10) (2, 200), // update (2, 20) (3, 600) // update (3, 30) // delete (4, 40) )) testUnlimitedClauses("conditional delete + two conditional update + two conditional insert")( source = (1, 100) :: (2, 200) :: (3, 300) :: (4, 400) :: (6, 600) :: Nil, target = (1, 10) :: (2, 20) :: (3, 30) :: Nil, mergeOn = "s.key = t.key", delete(condition = "s.key < 2"), update(condition = "s.key == 2", set = "key = s.key, value = s.value"), update(condition = "s.key == 3", set = "key = s.key, value = 2 * s.value"), insert(condition = "s.key < 5", values = "(key, value) VALUES (s.key, s.value)"), insert(condition = "s.key > 5", values = "(key, value) VALUES (s.key, 1 + s.value)"))( result = Seq( // delete (1, 10) (2, 200), // update (2, 20) (3, 600), // update (3, 30) (4, 400), // insert (4, 400) (6, 601) // insert (6, 600) )) testUnlimitedClauses("conditional update + update + conditional delete + conditional insert")( source = (1, 100) :: (2, 200) :: (3, 300) :: (4, 400) :: (5, 500) :: Nil, target = (0, 0) :: (1, 10) :: (2, 20) :: (3, 30) :: Nil, mergeOn = "s.key = t.key", update(condition = "s.key < 2", set = "key = s.key, value = s.value"), update(condition = "s.key < 3", set = "key = s.key, value = 2 * s.value"), delete(condition = "s.key < 4"), insert(condition = "s.key > 4", values = "(key, value) VALUES (s.key, s.value)"))( result = Seq( (0, 0), // no change (1, 100), // (1, 10) updated by matched_0 (2, 400), // (2, 20) updated by matched_1 // (3, 30) deleted by matched_2 (5, 500) // (5, 500) inserted )) testUnlimitedClauses("conditional insert + insert")( source = (1, 100) :: (2, 200) :: (3, 300) :: (4, 400) :: (5, 500) :: Nil, target = (0, 0) :: (1, 10) :: (2, 20) :: (3, 30) :: Nil, mergeOn = "s.key = t.key", insert(condition = "s.key < 5", values = "(key, value) VALUES (s.key, s.value)"), insert(condition = null, values = "(key, value) VALUES (s.key, s.value + 1)"))( result = Seq( (0, 0), // no change (1, 10), // no change (2, 20), // no change (3, 30), // no change (4, 400), // (4, 400) inserted by notMatched_0 (5, 501) // (5, 501) inserted by notMatched_1 )) testUnlimitedClauses("2 conditional inserts")( source = (1, 100) :: (2, 200) :: (3, 300) :: (4, 400) :: (5, 500) :: (6, 600) :: Nil, target = (0, 0) :: (1, 10) :: (2, 20) :: (3, 30) :: Nil, mergeOn = "s.key = t.key", insert(condition = "s.key < 5", values = "(key, value) VALUES (s.key, s.value)"), insert(condition = "s.key = 5", values = "(key, value) VALUES (s.key, s.value + 1)"))( result = Seq( (0, 0), // no change (1, 10), // no change (2, 20), // no change (3, 30), // no change (4, 400), // (4, 400) inserted by notMatched_0 (5, 501) // (5, 501) inserted by notMatched_1 // (6, 600) not inserted as not insert condition matched )) testUnlimitedClauses("update/delete (no matches) + conditional insert + insert")( source = (4, 400) :: (5, 500) :: Nil, target = (0, 0) :: (1, 10) :: (2, 20) :: (3, 30) :: Nil, mergeOn = "s.key = t.key", update(condition = "t.key = 0", set = "key = s.key, value = s.value"), delete(condition = null), insert(condition = "s.key < 5", values = "(key, value) VALUES (s.key, s.value)"), insert(condition = null, values = "(key, value) VALUES (s.key, s.value + 1)"))( result = Seq( (0, 0), // no change (1, 10), // no change (2, 20), // no change (3, 30), // no change (4, 400), // (4, 400) inserted by notMatched_0 (5, 501) // (5, 501) inserted by notMatched_1 )) testUnlimitedClauses("update/delete (no matches) + 2 conditional inserts")( source = (4, 400) :: (5, 500) :: (6, 600) :: Nil, target = (0, 0) :: (1, 10) :: (2, 20) :: (3, 30) :: Nil, mergeOn = "s.key = t.key", update(condition = "t.key = 0", set = "key = s.key, value = s.value"), delete(condition = null), insert(condition = "s.key < 5", values = "(key, value) VALUES (s.key, s.value)"), insert(condition = "s.key = 5", values = "(key, value) VALUES (s.key, s.value + 1)"))( result = Seq( (0, 0), // no change (1, 10), // no change (2, 20), // no change (3, 30), // no change (4, 400), // (4, 400) inserted by notMatched_0 (5, 501) // (5, 501) inserted by notMatched_1 // (6, 600) not inserted as not insert condition matched )) testUnlimitedClauses("2 update + 2 delete + 4 insert")( source = (1, 100) :: (2, 200) :: (3, 300) :: (4, 400) :: (5, 500) :: (6, 600) :: (7, 700) :: (8, 800) :: (9, 900) :: Nil, target = (0, 0) :: (1, 10) :: (2, 20) :: (3, 30) :: (4, 40) :: Nil, mergeOn = "s.key = t.key", update(condition = "s.key == 1", set = "key = s.key, value = s.value"), delete(condition = "s.key == 2"), update(condition = "s.key == 3", set = "key = s.key, value = 2 * s.value"), delete(condition = null), insert(condition = "s.key == 5", values = "(key, value) VALUES (s.key, s.value)"), insert(condition = "s.key == 6", values = "(key, value) VALUES (s.key, 1 + s.value)"), insert(condition = "s.key == 7", values = "(key, value) VALUES (s.key, 2 + s.value)"), insert(condition = null, values = "(key, value) VALUES (s.key, 3 + s.value)"))( result = Seq( (0, 0), // no change (1, 100), // (1, 10) updated by matched_0 // (2, 20) deleted by matched_1 (3, 600), // (3, 30) updated by matched_2 // (4, 40) deleted by matched_3 (5, 500), // (5, 500) inserted by notMatched_0 (6, 601), // (6, 600) inserted by notMatched_1 (7, 702), // (7, 700) inserted by notMatched_2 (8, 803), // (8, 800) inserted by notMatched_3 (9, 903) // (9, 900) inserted by notMatched_3 )) testMergeAnalysisException("error on multiple insert clauses without condition")( mergeOn = "s.key = t.key", update(condition = "s.key == 3", set = "key = s.key, value = 2 * srcValue"), insert(condition = null, values = "(key, value) VALUES (s.key, srcValue)"), insert(condition = null, values = "(key, value) VALUES (s.key, 1 + srcValue)"))( expectedErrorClass = "NON_LAST_NOT_MATCHED_BY_TARGET_CLAUSE_OMIT_CONDITION", expectedMessageParameters = Map.empty) testMergeAnalysisException("error on multiple update clauses without condition")( mergeOn = "s.key = t.key", update(condition = "s.key == 3", set = "key = s.key, value = 2 * srcValue"), update(condition = null, set = "key = s.key, value = 3 * srcValue"), update(condition = null, set = "key = s.key, value = 4 * srcValue"), insert(condition = null, values = "(key, value) VALUES (s.key, srcValue)"))( expectedErrorClass = "NON_LAST_MATCHED_CLAUSE_OMIT_CONDITION", expectedMessageParameters = Map.empty) testMergeAnalysisException("error on multiple update/delete clauses without condition")( mergeOn = "s.key = t.key", update(condition = "s.key == 3", set = "key = s.key, value = 2 * srcValue"), delete(condition = null), update(condition = null, set = "key = s.key, value = 4 * srcValue"), insert(condition = null, values = "(key, value) VALUES (s.key, srcValue)"))( expectedErrorClass = "NON_LAST_MATCHED_CLAUSE_OMIT_CONDITION", expectedMessageParameters = Map.empty) testMergeAnalysisException( "error on non-empty condition following empty condition for update clauses")( mergeOn = "s.key = t.key", update(condition = null, set = "key = s.key, value = 2 * srcValue"), update(condition = "s.key < 3", set = "key = s.key, value = srcValue"), insert(condition = null, values = "(key, value) VALUES (s.key, srcValue)"))( expectedErrorClass = "NON_LAST_MATCHED_CLAUSE_OMIT_CONDITION", expectedMessageParameters = Map.empty) testMergeAnalysisException( "error on non-empty condition following empty condition for insert clauses")( mergeOn = "s.key = t.key", update(condition = null, set = "key = s.key, value = srcValue"), insert(condition = null, values = "(key, value) VALUES (s.key, srcValue)"), insert(condition = "s.key < 3", values = "(key, value) VALUES (s.key, srcValue)"))( expectedErrorClass = "NON_LAST_NOT_MATCHED_BY_TARGET_CLAUSE_OMIT_CONDITION", expectedMessageParameters = Map.empty) } trait MergeIntoAnalysisExceptionTests extends MergeIntoSuiteBaseMixin { testMergeAnalysisException("update condition - ambiguous reference")( mergeOn = "s.key = t.key", update(condition = "key > 1", set = "tgtValue = srcValue"))( expectedErrorClass = "AMBIGUOUS_REFERENCE", expectedMessageParameters = Map( "name" -> "`key`", "referenceNames" -> "[`s`.`key`, `t`.`key`]")) testMergeAnalysisException("update condition - unknown reference")( mergeOn = "s.key = t.key", update(condition = "unknownAttrib > 1", set = "tgtValue = srcValue"))( // Should show unknownAttrib as invalid ref and (key, tgtValue, srcValue) as valid column names. expectedErrorClass = "DELTA_MERGE_UNRESOLVED_EXPRESSION", expectedMessageParameters = Map( "sqlExpr" -> "unknownAttrib", "clause" -> "UPDATE condition", "cols" -> "t.key, t.tgtValue, s.key, s.srcValue")) testMergeAnalysisException("update condition - aggregation function")( mergeOn = "s.key = t.key", update(condition = "max(0) > 0", set = "tgtValue = srcValue"))( expectedErrorClass = "DELTA_AGGREGATION_NOT_SUPPORTED", expectedMessageParameters = Map( "operation" -> "UPDATE condition of MERGE operation", "predicate" -> "(condition = (max(0) > 0))")) testMergeAnalysisException("update condition - subquery")( mergeOn = "s.key = t.key", update(condition = "s.srcValue in (select value from t)", set = "tgtValue = srcValue"))( expectedErrorClass = "TABLE_OR_VIEW_NOT_FOUND", expectedMessageParameters = Map("relationName" -> "`t`")) testMergeAnalysisException("delete condition - ambiguous reference")( mergeOn = "s.key = t.key", delete(condition = "key > 1"))( expectedErrorClass = "AMBIGUOUS_REFERENCE", expectedMessageParameters = Map( "name" -> "`key`", "referenceNames" -> "[`s`.`key`, `t`.`key`]")) testMergeAnalysisException("delete condition - unknown reference")( mergeOn = "s.key = t.key", delete(condition = "unknownAttrib > 1"))( // Should show unknownAttrib as invalid ref and (key, tgtValue, srcValue) as valid column names. expectedErrorClass = "DELTA_MERGE_UNRESOLVED_EXPRESSION", expectedMessageParameters = Map( "sqlExpr" -> "unknownAttrib", "clause" -> "DELETE condition", "cols" -> "t.key, t.tgtValue, s.key, s.srcValue")) testMergeAnalysisException("delete condition - aggregation function")( mergeOn = "s.key = t.key", delete(condition = "max(0) > 0"))( expectedErrorClass = "DELTA_AGGREGATION_NOT_SUPPORTED", expectedMessageParameters = Map( "operation" -> "DELETE condition of MERGE operation", "predicate" -> "(condition = (max(0) > 0))")) testMergeAnalysisException("delete condition - subquery")( mergeOn = "s.key = t.key", delete(condition = "s.srcValue in (select tgtValue from t)"))( expectedErrorClass = "TABLE_OR_VIEW_NOT_FOUND", expectedMessageParameters = Map("relationName" -> "`t`")) testMergeAnalysisException("insert condition - unknown reference")( mergeOn = "s.key = t.key", insert(condition = "unknownAttrib > 1", values = "(key, tgtValue) VALUES (s.key, s.srcValue)"))( // Should show unknownAttrib as invalid ref and (key, srcValue) as valid column names, // but not show tgtValue as a valid name as target columns cannot be present in insert clause. expectedErrorClass = "DELTA_MERGE_UNRESOLVED_EXPRESSION", expectedMessageParameters = Map( "sqlExpr" -> "unknownAttrib", "clause" -> "INSERT condition", "cols" -> "s.key, s.srcValue")) testMergeAnalysisException("insert condition - reference to target table column")( mergeOn = "s.key = t.key", insert(condition = "tgtValue > 1", values = "(key, tgtValue) VALUES (s.key, s.srcValue)"))( // Should show tgtValue as invalid ref and (key, srcValue) as valid column names expectedErrorClass = "DELTA_MERGE_UNRESOLVED_EXPRESSION", expectedMessageParameters = Map( "sqlExpr" -> "tgtValue", "clause" -> "INSERT condition", "cols" -> "s.key, s.srcValue")) testMergeAnalysisException("insert condition - aggregation function")( mergeOn = "s.key = t.key", insert(condition = "max(0) > 0", values = "(key, tgtValue) VALUES (s.key, s.srcValue)"))( expectedErrorClass = "DELTA_AGGREGATION_NOT_SUPPORTED", expectedMessageParameters = Map( "operation" -> "INSERT condition of MERGE operation", "predicate" -> "(condition = (max(0) > 0))")) testMergeAnalysisException("insert condition - subquery")( mergeOn = "s.key = t.key", insert( condition = "s.srcValue in (select srcValue from s)", values = "(key, tgtValue) VALUES (s.key, s.srcValue)"))( expectedErrorClass = "TABLE_OR_VIEW_NOT_FOUND", expectedMessageParameters = Map("relationName" -> "`s`")) } trait MergeIntoExtendedSyntaxTests extends MergeIntoSuiteBaseMixin { import testImplicits._ private def testMergeErrorOnMultipleMatches( name: String, confs: Seq[(String, String)] = Seq.empty)( source: Seq[(Int, Int)], target: Seq[(Int, Int)], mergeOn: String, mergeClauses: MergeClause*): Unit = { test(s"extended syntax - $name") { withSQLConf(confs: _*) { withKeyValueData(source, target) { case (sourceName, targetName) => val docURL = "/delta-update.html#upsert-into-a-table-using-merge" checkError( exception = intercept[DeltaUnsupportedOperationException] { executeMerge(s"$targetName t", s"$sourceName s", mergeOn, mergeClauses: _*) }, "DELTA_MULTIPLE_SOURCE_ROW_MATCHING_TARGET_ROW_IN_MERGE", parameters = Map( "usageReference" -> DeltaErrors.generateDocsLink( spark.sparkContext.getConf, docURL, skipValidation = true)) ) } } } } testExtendedMerge("only update")( source = (0, 0) :: (1, 10) :: (3, 30) :: Nil, target = (1, 1) :: (2, 2) :: Nil, mergeOn = "s.key = t.key", update(set = "key = s.key, value = s.value"))( result = Seq( (1, 10), // (1, 1) updated (2, 2) )) testMergeErrorOnMultipleMatches("only update with multiple matches")( source = (0, 0) :: (1, 10) :: (1, 11) :: (2, 20) :: Nil, target = (1, 1) :: (2, 2) :: Nil, mergeOn = "s.key = t.key", update(set = "key = s.key, value = s.value")) testExtendedMerge("only conditional update")( source = (0, 0) :: (1, 10) :: (2, 20) :: (3, 30) :: Nil, target = (1, 1) :: (2, 2) :: (3, 3) :: Nil, mergeOn = "s.key = t.key", update(condition = "s.value <> 20 AND t.value <> 3", set = "key = s.key, value = s.value"))( result = Seq( (1, 10), // updated (2, 2), // not updated due to source-only condition `s.value <> 20` (3, 3) // not updated due to target-only condition `t.value <> 3` )) testMergeErrorOnMultipleMatches("only conditional update with multiple matches")( source = (0, 0) :: (1, 10) :: (1, 11) :: (2, 20) :: Nil, target = (1, 1) :: (2, 2) :: Nil, mergeOn = "s.key = t.key", update(condition = "s.value = 10", set = "key = s.key, value = s.value")) testExtendedMerge("only delete")( source = (0, 0) :: (1, 10) :: (3, 30) :: Nil, target = (1, 1) :: (2, 2) :: Nil, mergeOn = "s.key = t.key", delete())( result = Seq( (2, 2) // (1, 1) deleted )) // (3, 30) not inserted as not insert clause // This is not ambiguous even when there are multiple matches testExtendedMerge(s"only delete with multiple matches")( source = (0, 0) :: (1, 10) :: (1, 100) :: (3, 30) :: Nil, target = (1, 1) :: (2, 2) :: Nil, mergeOn = "s.key = t.key", delete())( result = Seq( (2, 2) // (1, 1) matches multiple source rows but unambiguously deleted ) ) testExtendedMerge("only conditional delete")( source = (0, 0) :: (1, 10) :: (2, 20) :: (3, 30) :: Nil, target = (1, 1) :: (2, 2) :: (3, 3) :: Nil, mergeOn = "s.key = t.key", delete(condition = "s.value <> 20 AND t.value <> 3"))( result = Seq( (2, 2), // not deleted due to source-only condition `s.value <> 20` (3, 3) // not deleted due to target-only condition `t.value <> 3` )) // (1, 1) deleted testMergeErrorOnMultipleMatches("only conditional delete with multiple matches")( source = (0, 0) :: (1, 10) :: (1, 100) :: (2, 20) :: Nil, target = (1, 1) :: (2, 2) :: Nil, mergeOn = "s.key = t.key", delete(condition = "s.value = 10")) testExtendedMerge("conditional update + delete")( source = (0, 0) :: (1, 10) :: (2, 20) :: Nil, target = (1, 1) :: (2, 2) :: (3, 3) :: Nil, mergeOn = "s.key = t.key", update(condition = "s.key <> 1", set = "key = s.key, value = s.value"), delete())( result = Seq( (2, 20), // (2, 2) updated, (1, 1) deleted as it did not match update condition (3, 3) )) testMergeErrorOnMultipleMatches("conditional update + delete with multiple matches")( source = (0, 0) :: (1, 10) :: (2, 20) :: (2, 200) :: Nil, target = (1, 1) :: (2, 2) :: Nil, mergeOn = "s.key = t.key", update(condition = "s.value = 20", set = "key = s.key, value = s.value"), delete()) testExtendedMerge("conditional update + conditional delete")( source = (0, 0) :: (1, 10) :: (2, 20) :: (3, 30) :: Nil, target = (1, 1) :: (2, 2) :: (3, 3) :: (4, 4) :: Nil, mergeOn = "s.key = t.key", update(condition = "s.key <> 1", set = "key = s.key, value = s.value"), delete(condition = "s.key <> 2"))( result = Seq( (2, 20), // (2, 2) updated as it matched update condition (3, 30), // (3, 3) updated even though it matched update and delete conditions, as update 1st (4, 4) )) // (1, 1) deleted as it matched delete condition testMergeErrorOnMultipleMatches( "conditional update + conditional delete with multiple matches")( source = (0, 0) :: (1, 10) :: (1, 100) :: (2, 20) :: (2, 200) :: Nil, target = (1, 1) :: (2, 2) :: Nil, mergeOn = "s.key = t.key", update(condition = "s.value = 20", set = "key = s.key, value = s.value"), delete(condition = "s.value = 10")) testExtendedMerge("conditional delete + conditional update (order matters)")( source = (0, 0) :: (1, 10) :: (2, 20) :: (3, 30) :: Nil, target = (1, 1) :: (2, 2) :: (3, 3) :: (4, 4) :: Nil, mergeOn = "s.key = t.key", delete(condition = "s.key <> 2"), update(condition = "s.key <> 1", set = "key = s.key, value = s.value"))( result = Seq( (2, 20), // (2, 2) updated as it matched update condition (4, 4) // (4, 4) unchanged )) // (1, 1) and (3, 3) deleted as they matched delete condition (before update cond) testExtendedMerge("only insert")( source = (0, 0) :: (1, 10) :: (3, 30) :: Nil, target = (1, 1) :: (2, 2) :: Nil, mergeOn = "s.key = t.key", insert(values = "(key, value) VALUES (s.key, s.value)"))( result = Seq( (0, 0), // (0, 0) inserted (1, 1), // (1, 1) not updated as no update clause (2, 2), // (2, 2) not updated as no update clause (3, 30) // (3, 30) inserted )) testExtendedMerge("only conditional insert")( source = (0, 0) :: (1, 10) :: (3, 30) :: Nil, target = (1, 1) :: (2, 2) :: Nil, mergeOn = "s.key = t.key", insert(condition = "s.value <> 30", values = "(key, value) VALUES (s.key, s.value)"))( result = Seq( (0, 0), // (0, 0) inserted by condition but not (3, 30) (1, 1), (2, 2) )) testExtendedMerge("update + conditional insert")( source = (0, 0) :: (1, 10) :: (3, 30) :: Nil, target = (1, 1) :: (2, 2) :: Nil, mergeOn = "s.key = t.key", update("key = s.key, value = s.value"), insert(condition = "s.value <> 30", values = "(key, value) VALUES (s.key, s.value)"))( result = Seq( (0, 0), // (0, 0) inserted by condition but not (3, 30) (1, 10), // (1, 1) updated (2, 2) )) testExtendedMerge("conditional update + conditional insert")( source = (0, 0) :: (1, 10) :: (2, 20) :: (3, 30) :: Nil, target = (1, 1) :: (2, 2) :: Nil, mergeOn = "s.key = t.key", update(condition = "s.key > 1", set = "key = s.key, value = s.value"), insert(condition = "s.key > 1", values = "(key, value) VALUES (s.key, s.value)"))( result = Seq( (1, 1), // (1, 1) not updated by condition (2, 20), // (2, 2) updated by condition (3, 30) // (3, 30) inserted by condition but not (0, 0) )) // This is specifically to test the MergeIntoDeltaCommand.writeOnlyInserts code paths testExtendedMerge("update + conditional insert clause with data to only insert, no updates")( source = (0, 0) :: (3, 30) :: Nil, target = (1, 1) :: (2, 2) :: Nil, mergeOn = "s.key = t.key", update("key = s.key, value = s.value"), insert(condition = "s.value <> 30", values = "(key, value) VALUES (s.key, s.value)"))( result = Seq( (0, 0), // (0, 0) inserted by condition but not (3, 30) (1, 1), (2, 2) )) testExtendedMerge(s"delete + insert with multiple matches for both") ( source = (1, 10) :: (1, 100) :: (3, 30) :: (3, 300) :: Nil, target = (1, 1) :: (2, 2) :: Nil, mergeOn = "s.key = t.key", delete(), insert(values = "(key, value) VALUES (s.key, s.value)")) ( result = Seq( // (1, 1) matches multiple source rows but unambiguously deleted (2, 2), // existed previously (3, 30), // inserted (3, 300) // inserted ) ) testExtendedMerge("conditional update + conditional delete + conditional insert")( source = (0, 0) :: (1, 10) :: (2, 20) :: (3, 30) :: (4, 40) :: Nil, target = (1, 1) :: (2, 2) :: (3, 3) :: Nil, mergeOn = "s.key = t.key", update(condition = "s.key < 2", set = "key = s.key, value = s.value"), delete(condition = "s.key < 3"), insert(condition = "s.key > 1", values = "(key, value) VALUES (s.key, s.value)"))( result = Seq( (1, 10), // (1, 1) updated by condition, but not (2, 2) or (3, 3) (3, 3), // neither updated nor deleted as it matched neither condition (4, 40) // (4, 40) inserted by condition, but not (0, 0) )) // (2, 2) deleted by condition but not (1, 1) or (3, 3) testMergeErrorOnMultipleMatches( "conditional update + conditional delete + conditional insert with multiple matches")( source = (0, 0) :: (1, 10) :: (1, 100) :: (2, 20) :: (2, 200) :: Nil, target = (1, 1) :: (2, 2) :: Nil, mergeOn = "s.key = t.key", update(condition = "s.value = 20", set = "key = s.key, value = s.value"), delete(condition = "s.value = 10"), insert(condition = "s.value = 0", values = "(key, value) VALUES (s.key, s.value)")) // complex merge condition = has target-only and source-only predicates testExtendedMerge( "conditional update + conditional delete + conditional insert + complex merge condition ")( source = (-1, -10) :: (0, 0) :: (1, 10) :: (2, 20) :: (3, 30) :: (4, 40) :: (5, 50) :: Nil, target = (-1, -1) :: (1, 1) :: (2, 2) :: (3, 3) :: (5, 5) :: Nil, mergeOn = "s.key = t.key AND t.value > 0 AND s.key < 5", update(condition = "s.key < 2", set = "key = s.key, value = s.value"), delete(condition = "s.key < 3"), insert(condition = "s.key > 1", values = "(key, value) VALUES (s.key, s.value)"))( result = Seq( (-1, -1), // (-1, -1) not matched with (-1, -10) by target-only condition 't.value > 0', so // not updated, But (-1, -10) not inserted as insert condition is 's.key > 1' // (0, 0) not matched any target but not inserted as insert condition is 's.key > 1' (1, 10), // (1, 1) matched with (1, 10) and updated as update condition is 's.key < 2' // (2, 2) matched with (2, 20) and deleted as delete condition is 's.key < 3' (3, 3), // (3, 3) matched with (3, 30) but neither updated nor deleted as it did not // satisfy update or delete condition (4, 40), // (4, 40) not matched any target, so inserted as insert condition is 's.key > 1' (5, 5), // (5, 5) not matched with (5, 50) by source-only condition 's.key < 5', no update (5, 50) // (5, 50) inserted as inserted as insert condition is 's.key > 1' )) test("extended syntax - different # cols in source than target") { val sourceData = (0, 0, 0) :: (1, 10, 100) :: (2, 20, 200) :: (3, 30, 300) :: (4, 40, 400) :: Nil val targetData = (1, 1) :: (2, 2) :: (3, 3) :: Nil withTempView("source") { append(targetData.toDF("key", "value"), Nil) sourceData.toDF("key", "value", "extra").createOrReplaceTempView("source") executeMerge( s"$tableSQLIdentifier t", "source s", cond = "s.key = t.key", update(condition = "s.key < 2", set = "key = s.key, value = s.value + s.extra"), delete(condition = "s.key < 3"), insert(condition = "s.key > 1", values = "(key, value) VALUES (s.key, s.value + s.extra)")) checkAnswer( readDeltaTableByIdentifier(), Seq( Row(1, 110), // (1, 1) updated by condition, but not (2, 2) or (3, 3) Row(3, 3), // neither updated nor deleted as it matched neither condition Row(4, 440) // (4, 40) inserted by condition, but not (0, 0) )) // (2, 2) deleted by condition but not (1, 1) or (3, 3) } } test("extended syntax - nested data - conditions and actions") { withJsonData( source = """{ "key": { "x": "X1", "y": 1}, "value" : { "a": 100, "b": "B100" } } { "key": { "x": "X2", "y": 2}, "value" : { "a": 200, "b": "B200" } } { "key": { "x": "X3", "y": 3}, "value" : { "a": 300, "b": "B300" } } { "key": { "x": "X4", "y": 4}, "value" : { "a": 400, "b": "B400" } }""", target = """{ "key": { "x": "X1", "y": 1}, "value" : { "a": 1, "b": "B1" } } { "key": { "x": "X2", "y": 2}, "value" : { "a": 2, "b": "B2" } }""" ) { case (sourceName, targetName) => executeMerge( s"$targetName t", s"$sourceName s", cond = "s.key = t.key", update(condition = "s.key.y < 2", set = "key = s.key, value = s.value"), insert(condition = "s.key.x < 'X4'", values = "(key, value) VALUES (s.key, s.value)")) checkAnswer( readDeltaTableByIdentifier(), spark.read.json(Seq( """{ "key": { "x": "X1", "y": 1}, "value" : { "a": 100, "b": "B100" } }""", // updated """{ "key": { "x": "X2", "y": 2}, "value" : { "a": 2, "b": "B2" } }""", // not updated """{ "key": { "x": "X3", "y": 3}, "value" : { "a": 300, "b": "B300" } }""" // inserted ).toDS)) } } testExtendedMerge("insert only merge")( source = (0, 0) :: (1, 10) :: (3, 30) :: Nil, target = (1, 1) :: (2, 2) :: Nil, mergeOn = "s.key = t.key", insert(values = "*"))( result = Seq( (0, 0), // inserted (1, 1), // existed previously (2, 2), // existed previously (3, 30) // inserted )) testExtendedMerge("insert only merge with insert condition on source")( source = (0, 0) :: (1, 10) :: (3, 30) :: Nil, target = (1, 1) :: (2, 2) :: Nil, mergeOn = "s.key = t.key", insert(values = "*", condition = "s.key = s.value"))( result = Seq( (0, 0), // inserted (1, 1), // existed previously (2, 2) // existed previously )) testExtendedMerge("insert only merge with predicate insert")( source = (0, 0) :: (1, 10) :: (3, 30) :: Nil, target = (1, 1) :: (2, 2) :: Nil, mergeOn = "s.key = t.key", insert(values = "(t.key, t.value) VALUES (s.key + 10, s.value + 10)"))( result = Seq( (10, 10), // inserted (1, 1), // existed previously (2, 2), // existed previously (13, 40) // inserted )) testExtendedMerge(s"insert only merge with multiple matches") ( source = (0, 0) :: (1, 10) :: (1, 100) :: (3, 30) :: (3, 300) :: Nil, target = (1, 1) :: (2, 2) :: Nil, mergeOn = "s.key = t.key", insert(values = "(key, value) VALUES (s.key, s.value)")) ( result = Seq( (0, 0), // inserted (1, 1), // existed previously (2, 2), // existed previously (3, 30), // inserted (3, 300) // key exists but still inserted ) ) testMergeErrorOnMultipleMatches( "unconditional insert only merge - multiple matches when feature flag off", confs = Seq(DeltaSQLConf.MERGE_INSERT_ONLY_ENABLED.key -> "false"))( source = (1, 10) :: (1, 100) :: (2, 20) :: Nil, target = (1, 1) :: Nil, mergeOn = "s.key = t.key", insert(values = "(key, value) VALUES (s.key, s.value)")) testMergeErrorOnMultipleMatches( "conditional insert only merge - multiple matches when feature flag off", confs = Seq(DeltaSQLConf.MERGE_INSERT_ONLY_ENABLED.key -> "false"))( source = (1, 10) :: (1, 100) :: (2, 20) :: (2, 200) :: Nil, target = (1, 1) :: Nil, mergeOn = "s.key = t.key", insert(condition = "s.value = 20", values = "(key, value) VALUES (s.key, s.value)")) } trait MergeIntoSuiteBaseMiscTests extends MergeIntoSuiteBaseMixin { import testImplicits._ test("same column names in source and target") { withTable("source") { Seq((1, 5), (2, 9)).toDF("key", "value").createOrReplaceTempView("source") append(Seq((2, 2), (1, 4)).toDF("key", "value")) executeMerge( target = s"$tableSQLIdentifier as target", source = "source src", condition = "src.key = target.key", update = "target.key = 20 + src.key, target.value = 20 + src.value", insert = "(key, value) VALUES (src.key - 10, src.value + 10)") checkAnswer( readDeltaTableByIdentifier(), Row(21, 25) :: // Update Row(22, 29) :: // Update Nil) } } test("Source is a query") { withTable("source") { Seq((1, 6, "a"), (0, 3, "b")).toDF("key1", "value", "others") .createOrReplaceTempView("source") append(Seq((2, 2), (1, 4)).toDF("key2", "value")) executeMerge( target = s"$tableSQLIdentifier as trg", source = "(SELECT key1, value, others FROM source) src", condition = "src.key1 = trg.key2", update = "trg.key2 = 20 + key1, value = 20 + src.value", insert = "(trg.key2, value) VALUES (key1 - 10, src.value + 10)") checkAnswer( readDeltaTableByIdentifier(), Row(2, 2) :: // No change Row(21, 26) :: // Update Row(-10, 13) :: // Insert Nil) withCrossJoinEnabled { executeMerge( target = s"$tableSQLIdentifier as trg", source = "(SELECT 5 as key1, 5 as value) src", condition = "src.key1 = trg.key2", update = "trg.key2 = 20 + key1, value = 20 + src.value", insert = "(trg.key2, value) VALUES (key1 - 10, src.value + 10)") } checkAnswer(readDeltaTableByIdentifier(), Row(2, 2) :: Row(21, 26) :: Row(-10, 13) :: Row(-5, 15) :: // new row Nil) } } test("self merge") { append(Seq((2, 2), (1, 4)).toDF("key2", "value")) executeMerge( target = s"$tableSQLIdentifier as target", source = s"$tableSQLIdentifier as src", condition = "src.key2 = target.key2", update = "key2 = 20 + src.key2, value = 20 + src.value", insert = "(key2, value) VALUES (src.key2 - 10, src.value + 10)") checkAnswer(readDeltaTableByIdentifier(), Row(22, 22) :: // UPDATE Row(21, 24) :: // UPDATE Nil) } test("order by + limit in source query #1") { withTable("source") { Seq((1, 6, "a"), (0, 3, "b")).toDF("key1", "value", "others") .createOrReplaceTempView("source") append(Seq((2, 2), (1, 4)).toDF("key2", "value")) executeMerge( target = s"$tableSQLIdentifier as trg", source = "(SELECT key1, value, others FROM source order by key1 limit 1) src", condition = "src.key1 = trg.key2", update = "trg.key2 = 20 + key1, value = 20 + src.value", insert = "(trg.key2, value) VALUES (key1 - 10, src.value + 10)") checkAnswer(readDeltaTableByIdentifier(), Row(1, 4) :: // No change Row(2, 2) :: // No change Row(-10, 13) :: // Insert Nil) } } test("order by + limit in source query #2") { withTable("source") { Seq((1, 6, "a"), (0, 3, "b")).toDF("key1", "value", "others") .createOrReplaceTempView("source") append(Seq((2, 2), (1, 4)).toDF("key2", "value")) executeMerge( target = s"$tableSQLIdentifier as trg", source = "(SELECT key1, value, others FROM source order by value DESC limit 1) src", condition = "src.key1 = trg.key2", update = "trg.key2 = 20 + key1, value = 20 + src.value", insert = "(trg.key2, value) VALUES (key1 - 10, src.value + 10)") checkAnswer(readDeltaTableByIdentifier(), Row(2, 2) :: // No change Row(21, 26) :: // UPDATE Nil) } } testQuietly("Negative case - more than one source rows match the same target row") { withTable("source") { Seq((1, 1), (0, 3), (1, 5)).toDF("key1", "value").createOrReplaceTempView("source") append(Seq((2, 2), (1, 4)).toDF("key2", "value")) val e = intercept[Exception] { executeMerge( target = s"$tableSQLIdentifier as target", source = "source src", condition = "src.key1 = target.key2", update = "key2 = 20 + key1, value = 20 + src.value", insert = "(key2, value) VALUES (key1 - 10, src.value + 10)") }.toString val expectedEx = DeltaErrors.multipleSourceRowMatchingTargetRowInMergeException(spark) assert(e.contains(expectedEx.getMessage)) } } test("More than one target rows match the same source row") { withTable("source") { Seq((1, 5), (2, 9)).toDF("key1", "value").createOrReplaceTempView("source") append(Seq((2, 2), (1, 4)).toDF("key2", "value"), Seq("key2")) withCrossJoinEnabled { executeMerge( target = s"$tableSQLIdentifier as target", source = "source src", condition = "key1 = 1", update = "key2 = 20 + key1, value = 20 + src.value", insert = "(key2, value) VALUES (key1 - 10, src.value + 10)") } checkAnswer(readDeltaTableByIdentifier(), Row(-8, 19) :: // Insert Row(21, 25) :: // Update Row(21, 25) :: // Update Nil) } } Seq(true, false).foreach { isPartitioned => test(s"Merge table using different data types - implicit casting, parts: $isPartitioned") { withTable("source") { Seq((1, "5"), (3, "9"), (3, "a")).toDF("key1", "value").createOrReplaceTempView("source") val partitions = if (isPartitioned) "key2" :: Nil else Nil append(Seq((2, 2), (1, 4)).toDF("key2", "value"), partitions) executeMerge( target = s"$tableSQLIdentifier as target", source = "source src", condition = "key1 = key2", update = "key2 = 33 + cast(key2 as double), value = '20'", insert = "(key2, value) VALUES ('44', try_cast(src.value as double) + 10)") checkAnswer(readDeltaTableByIdentifier(), Row(44, 19) :: // Insert // NULL is generated when the type casting does not work for some values) Row(44, null) :: // Insert Row(34, 20) :: // Update Row(2, 2) :: // No change Nil) } } } test("Negative case - basic syntax analysis") { withTable("source") { Seq((1, 1), (0, 3)).toDF("key1", "value").createOrReplaceTempView("source") append(Seq((2, 2), (1, 4)).toDF("key2", "value")) // insert expressions have target table reference var e = intercept[AnalysisException] { executeMerge( target = s"$tableSQLIdentifier as target", source = "source src", condition = "src.key1 = target.key2", update = "key2 = key1, value = src.value", insert = "(key2, value) VALUES (3, src.value + key2)") }.getMessage errorContains(e, "cannot resolve key2") // to-update columns have source table reference e = intercept[AnalysisException] { executeMerge( target = s"$tableSQLIdentifier as target", source = "source src", condition = "src.key1 = target.key2", update = "key1 = 1, value = 2", insert = "(key2, value) VALUES (3, 4)") }.getMessage errorContains(e, "Cannot resolve key1 in UPDATE clause") errorContains(e, "key2") // should show key2 as a valid name in target columns // to-insert columns have source table reference e = intercept[AnalysisException] { executeMerge( target = s"$tableSQLIdentifier as target", source = "source src", condition = "src.key1 = target.key2", update = "key2 = 1, value = 2", insert = "(key1, value) VALUES (3, 4)") }.getMessage errorContains(e, "Cannot resolve key1 in INSERT clause") errorContains(e, "key2") // should contain key2 as a valid name in target columns // ambiguous reference e = intercept[AnalysisException] { executeMerge( target = s"$tableSQLIdentifier as target", source = "source src", condition = "src.key1 = target.key2", update = "key2 = 1, value = value", insert = "(key2, value) VALUES (3, 4)") }.getMessage Seq("value", "is ambiguous", "could be").foreach(x => errorContains(e, x)) // non-deterministic search condition e = intercept[AnalysisException] { executeMerge( target = s"$tableSQLIdentifier as target", source = "source src", condition = "src.key1 = target.key2 and rand() > 0.5", update = "key2 = 1, value = 2", insert = "(key2, value) VALUES (3, 4)") }.getMessage errorContains(e, "Non-deterministic functions are not supported in the search condition") // aggregate function e = intercept[AnalysisException] { executeMerge( target = s"$tableSQLIdentifier as target", source = "source src", condition = "src.key1 = target.key2 and max(target.key2) > 20", update = "key2 = 1, value = 2", insert = "(key2, value) VALUES (3, 4)") }.getMessage errorContains(e, "Aggregate functions are not supported in the search condition") } } test("Merge should use the same SparkSession consistently", NameBasedAccessIncompatible) { withTempDir { dir => withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "false") { val r = dir.getCanonicalPath val sourcePath = s"$r/source" val targetPath = s"$r/target" val numSourceRecords = 20 spark.range(numSourceRecords) .withColumn("x", $"id") .withColumn("y", $"id") .write.mode("overwrite").format("delta").save(sourcePath) spark.range(1) .withColumn("x", $"id") .write.mode("overwrite").format("delta").save(targetPath) val spark2 = spark.newSession spark2.conf.set(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key, "true") val target = io.delta.tables.DeltaTable.forPath(spark2, targetPath) val source = spark.read.format("delta").load(sourcePath).alias("s") val merge = target.alias("t") .merge(source, "t.id = s.id") .whenMatched.updateExpr(Map("t.x" -> "t.x + 1")) .whenNotMatched.insertAll() .execute() // The target table should have the same number of rows as the source after the merge assert(spark.read.format("delta").load(targetPath).count() == numSourceRecords) } } } test("Variant type") { withTable("source") { // Insert ("0", 0), ("1", 1) val dstDf = sql( """SELECT parse_json(cast(id as string)) v, id i FROM range(2)""") append(dstDf) // Insert ("1", 2), ("2", 3) // The first row will update, the second will insert. sql( s"""SELECT parse_json(cast(id as string)) v, id + 1 i FROM range(1, 3)""").createOrReplaceTempView("source") executeMerge( target = s"$tableSQLIdentifier as trgNew", source = "source src", condition = "to_json(src.v) = to_json(trgNew.v)", update = "i = 10 + src.i + trgNew.i, v = trgNew.v", insert = """(i, v) VALUES (i + 100, parse_json('"inserted"'))""") checkAnswer(readDeltaTableByIdentifier().selectExpr("i", "to_json(v)"), Row(0, "0") :: // No change Row(13, "1") :: // Update Row(103, "\"inserted\"") :: // Insert Nil) } } // Enable this test in OSS when Spark has the change to report better errors // when MERGE is not supported. ignore("Negative case - non-delta target") { withTable("source", "target") { Seq((1, 1), (0, 3), (1, 5)).toDF("key1", "value").createOrReplaceTempView("source") Seq((1, 1), (0, 3), (1, 5)).toDF("key2", "value").write.saveAsTable("target") val e = intercept[AnalysisException] { executeMerge( target = "target", source = "source src", condition = "src.key1 = target.key2", update = "key2 = 20 + key1, value = 20 + src.value", insert = "(key2, value) VALUES (key1 - 10, src.value + 10)") }.getMessage assert(e.contains("does not support MERGE") || // The MERGE Scala API is for Delta only and reports error differently. e.contains("is not a Delta table") || e.contains("MERGE destination only supports Delta sources")) } } test("Negative case - update assignments conflict because " + "same column with different references") { withTable("source") { Seq((1, 1), (0, 3), (1, 5)).toDF("key1", "value").createOrReplaceTempView("source") append(Seq((2, 2), (1, 4)).toDF("key2", "value")) val e = intercept[AnalysisException] { executeMerge( target = s"$tableSQLIdentifier as t", source = "source s", condition = "s.key1 = t.key2", update = "key2 = key1, t.key2 = key1", insert = "(key2, value) VALUES (3, 4)") }.getMessage errorContains(e, "there is a conflict from these set columns") } } test("Negative case - MERGE to the child directory", NameBasedAccessIncompatible) { withTempDir { tempDir => val tempPath = tempDir.getCanonicalPath val df = Seq((1, 1), (0, 3), (1, 5)).toDF("key2", "value") val partitions = "key2" :: Nil df.write.format("delta").mode("append").partitionBy(partitions: _*).save(tempPath) val e = intercept[AnalysisException] { executeMerge( target = s"delta.`$tempPath/key2=1` target", source = "(SELECT 5 as key1, 5 as value) src", condition = "src.key1 = target.key2", update = "key2 = 20 + key1, value = 20 + src.value", insert = "(key2, value) VALUES (key1 - 10, src.value + 10)") }.getMessage errorContains(e, "Expect a full scan of Delta sources, but found a partial scan") } } test(s"special character in path - matched delete", NameBasedAccessIncompatible) { withTempDir { tempDir => val source = s"$tempDir/sou rce~" val target = s"$tempDir/tar get>" spark.range(0, 10, 2).write.format("delta").save(source) spark.range(10).write.format("delta").save(target) executeMerge( tgt = s"delta.`$target` t", src = s"delta.`$source` s", cond = "t.id = s.id", clauses = delete()) checkAnswer(readDeltaTableByIdentifier(s"delta.`$target`"), Seq(1, 3, 5, 7, 9).toDF("id")) } } test(s"special character in path - matched update", NameBasedAccessIncompatible) { withTempDir { tempDir => val source = s"$tempDir/sou rce(" val target = s"$tempDir/tar get*" spark.range(0, 10, 2).write.format("delta").save(source) spark.range(10).write.format("delta").save(target) executeMerge( tgt = s"delta.`$target` t", src = s"delta.`$source` s", cond = "t.id = s.id", clauses = update(set = "id = t.id * 10")) checkAnswer(readDeltaTableByIdentifier(s"delta.`$target`"), Seq(0, 1, 20, 3, 40, 5, 60, 7, 80, 9).toDF("id")) } } Seq(true, false).foreach { isPartitioned => test(s"single file, isPartitioned: $isPartitioned") { withTable("source") { val df = spark.range(5).selectExpr("id as key1", "id as key2", "id as col1").repartition(1) val partitions = if (isPartitioned) "key1" :: "key2" :: Nil else Nil append(df, partitions) df.createOrReplaceTempView("source") executeMerge( target = s"$tableSQLIdentifier target", source = "(SELECT key1 as srcKey, key2, col1 FROM source where key1 < 3) AS source", condition = "srcKey = target.key1", update = "target.key1 = srcKey - 1000, target.key2 = source.key2 + 1000, " + "target.col1 = source.col1", insert = "(key1, key2, col1) VALUES (srcKey, source.key2, source.col1)") checkAnswer(readDeltaTableByIdentifier(), Row(-998, 1002, 2) :: // Update Row(-999, 1001, 1) :: // Update Row(-1000, 1000, 0) :: // Update Row(4, 4, 4) :: // No change Row(3, 3, 3) :: // No change Nil) } } } test("merge into cached table") { // Merge with a cached target only works in the join-based implementation right now withTable("source") { append(Seq((2, 2), (1, 4)).toDF("key2", "value")) Seq((1, 1), (0, 3), (3, 3)).toDF("key1", "value").createOrReplaceTempView("source") spark.table(tableSQLIdentifier).cache() spark.table(tableSQLIdentifier).collect() append(Seq((100, 100), (3, 5)).toDF("key2", "value")) // cache is in effect, as the above change is not reflected checkAnswer(spark.table(tableSQLIdentifier), Row(2, 2) :: Row(1, 4) :: Row(100, 100) :: Row(3, 5) :: Nil) executeMerge( target = s"$tableSQLIdentifier as trgNew", source = "source src", condition = "src.key1 = key2", update = "value = trgNew.value + 3", insert = "(key2, value) VALUES (key1, src.value + 10)") checkAnswer(spark.table(tableSQLIdentifier), Row(100, 100) :: // No change (newly inserted record) Row(2, 2) :: // No change Row(1, 7) :: // Update Row(3, 8) :: // Update (on newly inserted record) Row(0, 13) :: // Insert Nil) } } def testStar( name: String)( source: Seq[String], target: Seq[String], mergeClauses: MergeClause*)( result: Seq[String] = null, errorStrs: Seq[String] = null) { require(result == null ^ errorStrs == null, "either set the result or the error strings") val testName = if (result != null) s"star syntax - $name" else s"star syntax - analysis error - $name" test(testName) { withJsonData(source, target) { case (sourceName, targetName) => def execMerge() = executeMerge(s"$targetName t", s"$sourceName s", "s.key = t.key", mergeClauses: _*) if (result != null) { execMerge() checkAnswer( readDeltaTableByIdentifier(targetName), readFromJSON(result)) } else { val e = intercept[AnalysisException] { execMerge() } errorStrs.foreach { s => errorContains(e.getMessage, s) } } } } } testStar("basic star expansion")( source = """{ "key": "a", "value" : 10 } { "key": "c", "value" : 30 }""", target = """{ "key": "a", "value" : 1 } { "key": "b", "value" : 2 }""", update(set = "*"), insert(values = "*"))( result = """{ "key": "a", "value" : 10 } { "key": "b", "value" : 2 } { "key": "c", "value" : 30 }""") testStar("multiples columns and extra columns in source")( source = """{ "key": "a", "value" : 10, "value2" : 100, "value3" : 1000 } { "key": "c", "value" : 30, "value2" : 300, "value3" : 3000 }""", target = """{ "key": "a", "value" : 1, "value2" : 1 } { "key": "b", "value" : 2, "value2" : 2 }""", update(set = "*"), insert(values = "*"))( result = """{ "key": "a", "value" : 10, "value2" : 100 } { "key": "b", "value" : 2, "value2" : 2 } { "key": "c", "value" : 30, "value2" : 300 }""") test("insert only merge - turn off feature flag") { withSQLConf(DeltaSQLConf.MERGE_INSERT_ONLY_ENABLED.key -> "false") { withKeyValueData( source = (1, 10) :: (3, 30) :: Nil, target = (1, 1) :: Nil ) { case (sourceName, targetName) => insertOnlyMergeFeatureFlagOff(sourceName, targetName) } } } testMergeWithRepartition( name = "partition on multiple columns", partitionColumns = Seq("part1", "part2"), srcRange = Range(80, 110), expectLessFilesWithRepartition = true, update("t.part2 = 1"), insert("(id, part1, part2) VALUES (id, part1, part2)") ) testMergeWithRepartition( name = "insert only merge", partitionColumns = Seq("part1"), srcRange = Range(110, 150), expectLessFilesWithRepartition = true, insert("(id, part1, part2) VALUES (id, part1, part2)") ) testMergeWithRepartition( name = "non partitioned table", partitionColumns = Seq(), srcRange = Range(80, 180), expectLessFilesWithRepartition = false, update("t.part2 = 1"), insert("(id, part1, part2) VALUES (id, part1, part2)") ) protected def testMatchedOnlyOptimization( name: String)( source: Seq[(Int, Int)], target: Seq[(Int, Int)], mergeOn: String, mergeClauses: MergeClause*) ( result: Seq[(Int, Int)]): Unit = { Seq(true, false).foreach { matchedOnlyEnabled => Seq(true, false).foreach { isPartitioned => val s = if (matchedOnlyEnabled) "enabled" else "disabled" test(s"matched only merge - $s - $name - isPartitioned: $isPartitioned ") { withKeyValueData(source, target, isPartitioned) { case (sourceName, targetName) => withSQLConf(DeltaSQLConf.MERGE_MATCHED_ONLY_ENABLED.key -> s"$matchedOnlyEnabled") { executeMerge(s"$targetName t", s"$sourceName s", mergeOn, mergeClauses: _*) } checkAnswer( readDeltaTableByIdentifier(targetName), result.map { case (k, v) => Row(k, v) }) } } } } } testMatchedOnlyOptimization("with update") ( source = Seq((1, 100), (3, 300), (5, 500)), target = Seq((1, 10), (2, 20), (3, 30)), mergeOn = "s.key = t.key", update("t.key = s.key, t.value = s.value")) ( result = Seq( (1, 100), // updated (2, 20), // existed previously (3, 300) // updated ) ) testMatchedOnlyOptimization("with delete") ( source = Seq((1, 100), (3, 300), (5, 500)), target = Seq((1, 10), (2, 20), (3, 30)), mergeOn = "s.key = t.key", delete()) ( result = Seq( (2, 20) // existed previously ) ) testMatchedOnlyOptimization("with update and delete")( source = Seq((1, 100), (3, 300), (5, 500)), target = Seq((1, 10), (3, 30), (5, 30)), mergeOn = "s.key = t.key", update("t.value = s.value", "t.key < 3"), delete("t.key > 3")) ( result = Seq( (1, 100), // updated (3, 30) // existed previously ) ) protected def testNullCaseMatchedOnly(name: String) ( source: Seq[(JInt, JInt)], target: Seq[(JInt, JInt)], mergeOn: String, result: Seq[(JInt, JInt)]) = { Seq(true, false).foreach { isPartitioned => withSQLConf(DeltaSQLConf.MERGE_MATCHED_ONLY_ENABLED.key -> "true") { test(s"matched only merge - null handling - $name, isPartitioned: $isPartitioned") { withView("sourceView") { val partitions = if (isPartitioned) "key" :: Nil else Nil append(target.toDF("key", "value"), partitions) source.toDF("key", "value").createOrReplaceTempView("sourceView") executeMerge( tgt = s"$tableSQLIdentifier as t", src = "sourceView s", cond = mergeOn, update("t.value = s.value")) checkAnswer( readDeltaTableByIdentifier(), result.map { r => Row(r._1, r._2) } ) } } } } } testNullCaseMatchedOnly("null in source") ( source = Seq((1, 10), (2, 20), (null, null)), target = Seq((1, 1)), mergeOn = "s.key = t.key", result = Seq( (1, 10) // update ) ) testNullCaseMatchedOnly("null value in both source and target") ( source = Seq((1, 10), (2, 20), (null, 0)), target = Seq((1, 1), (null, null)), mergeOn = "s.key = t.key", result = Seq( (null, null), // No change as null in source does not match null in target (1, 10) // update ) ) test("data skipping - target-only condition") { withKeyValueData( source = (1, 10) :: Nil, target = (1, 1) :: (2, 2) :: Nil, isKeyPartitioned = true) { case (sourceName, targetName) => val report = getScanReport { executeMerge( target = s"$targetName t", source = s"$sourceName s", condition = "s.key = t.key AND t.key <= 1", update = "t.key = s.key, t.value = s.value", insert = "(key, value) VALUES (s.key, s.value)") }.head checkAnswer(readDeltaTableByIdentifier(), Row(1, 10) :: // Updated Row(2, 2) :: // File should be skipped Nil) assert(report.size("scanned").bytesCompressed != report.size("total").bytesCompressed) } } test("insert only merge - target data skipping") { val tblName = "merge_target" withTable(tblName) { spark.range(10).withColumn("part", 'id % 5).withColumn("value", 'id + 'id) .write.format("delta").partitionBy("part").mode("append").saveAsTable(tblName) val source = "source" withTable(source) { spark.range(20).withColumn("part", functions.lit(1)).withColumn("value", 'id + 'id) .write.format("delta").saveAsTable(source) val scans = getScanReport { withSQLConf(DeltaSQLConf.MERGE_INSERT_ONLY_ENABLED.key -> "true") { executeMerge( s"$tblName t", s"$source s", "s.id = t.id AND t.part = 1", insert(condition = "s.id % 5 = s.part", values = "*")) } } checkAnswer( spark.table(tblName).where("part = 1"), Row(1, 1, 2) :: Row(6, 1, 12) :: Row(11, 1, 22) :: Row(16, 1, 32) :: Nil ) assert(scans.length === 2, "We should scan the source and target " + "data once in an insert only optimization") // check if the source and target tables are scanned just once val sourceRoot = DeltaTableUtils.findDeltaTableRoot( spark, new Path(spark.table(source).inputFiles.head)).get.toString val targetRoot = DeltaTableUtils.findDeltaTableRoot( spark, new Path(spark.table(tblName).inputFiles.head)).get.toString assert(scans.map(_.path).toSet == Set(sourceRoot, targetRoot)) // check scanned files val targetScans = scans.find(_.path == targetRoot) val deltaLog = DeltaLog.forTable(spark, targetScans.get.path) val numTargetFiles = deltaLog.snapshot.numOfFiles assert(targetScans.get.metrics("numFiles") < numTargetFiles) // check scanned sizes val scanSizes = targetScans.head.size assert(scanSizes("total").bytesCompressed.get > scanSizes("scanned").bytesCompressed.get, "Should have partition pruned target table") } } } testMergeDataSkippingOnMatchPredicates("match conditions on target fields only")( source = Seq((1, 100), (3, 300), (5, 500)), target = Seq((1, 10), (2, 20), (3, 30)), dataSkippingOnTargetOnly = true, isMatchedOnly = true, update(condition = "t.key == 10", set = "*"), update(condition = "t.value == 100", set = "*"))( result = Seq((1, 10), (2, 20), (3, 30)) ) testMergeDataSkippingOnMatchPredicates("match conditions on source fields only")( source = Seq((1, 100), (3, 300), (5, 500)), target = Seq((1, 10), (2, 20), (3, 30)), dataSkippingOnTargetOnly = false, isMatchedOnly = true, update(condition = "s.key == 10", set = "*"), update(condition = "s.value == 10", set = "*"))( result = Seq((1, 10), (2, 20), (3, 30)) ) testMergeDataSkippingOnMatchPredicates("match on source and target fields")( source = Seq((1, 100), (3, 300), (5, 500)), target = Seq((1, 10), (2, 20), (3, 30)), dataSkippingOnTargetOnly = false, isMatchedOnly = true, update(condition = "s.key == 10", set = "*"), update(condition = "s.value == 10", set = "*"), delete(condition = "t.key == 4"))( result = Seq((1, 10), (2, 20), (3, 30)) ) testMergeDataSkippingOnMatchPredicates("with insert clause")( source = Seq((1, 100), (3, 300), (5, 500)), target = Seq((1, 10), (2, 20), (3, 30)), dataSkippingOnTargetOnly = false, isMatchedOnly = false, update(condition = "t.key == 10", set = "*"), insert(condition = null, values = "(key, value) VALUES (s.key, s.value)"))( result = Seq((1, 10), (2, 20), (3, 30), (5, 500)) ) testMergeDataSkippingOnMatchPredicates("when matched and conjunction")( source = Seq((1, 100), (3, 300), (5, 500)), target = Seq((1, 10), (2, 20), (3, 30)), dataSkippingOnTargetOnly = true, isMatchedOnly = true, update(condition = "t.key == 1 AND t.value == 5", set = "*"))( result = Seq((1, 10), (2, 20), (3, 30))) test("SC-70829 - prevent re-resolution with star and schema evolution") { val source = "source" val target = "target" withTable(source, target) { sql(s"""CREATE TABLE $source (id string, new string, old string, date DATE) USING delta""") sql(s"""CREATE TABLE $target (id string, old string, date DATE) USING delta""") withSQLConf("spark.databricks.delta.schema.autoMerge.enabled" -> "true") { executeMerge( tgt = s"$target t", src = s"$source s", // functions like date_sub requires additional work to resolve cond = "s.id = t.id AND t.date >= date_sub(current_date(), 3)", update(set = "*"), insert(values = "*")) } } } testUnsupportedExpression( function = "row_number", functionType = "Window", sourceData = Seq((1, 2, 3)).toDF("a", "b", "c"), targetData = Seq((1, 5, 6)).toDF("a", "b", "c"), mergeCondition = "(row_number() over (order by s.c)) = (row_number() over (order by t.c))", clauseCondition = "row_number() over (order by s.c) > 1", clauseAction = "row_number() over (order by s.c)" ) testUnsupportedExpression( function = "max", functionType = "Aggregate", sourceData = Seq((1, 2, 3)).toDF("a", "b", "c"), targetData = Seq((1, 5, 6)).toDF("a", "b", "c"), mergeCondition = "t.a = max(s.a)", clauseCondition = "max(s.b) > 1", clauseAction = "max(s.c)", customConditionErrorRegex = Option("Aggregate functions are not supported in the .* condition of MERGE operation.*") ) test("merge correctly handle field metadata") { withTable("source", "target") { // Create a target table with user metadata (comments) and internal metadata (column mapping // information) on both a top-level column and a nested field. sql( """ |CREATE TABLE target( | key int not null COMMENT 'data column', | value int not null, | cstruct struct) |USING DELTA |TBLPROPERTIES ( | 'delta.minReaderVersion' = '2', | 'delta.minWriterVersion' = '5', | 'delta.columnMapping.mode' = 'name') """.stripMargin ) sql(s"INSERT INTO target VALUES (0, 0, null)") sql("CREATE TABLE source (key int not null, value int not null) USING DELTA") sql(s"INSERT INTO source VALUES (1, 1)") executeMerge( tgt = "target", src = "source", cond = "source.key = target.key", update(condition = "target.key = 1", set = "target.value = 42"), updateNotMatched(condition = "target.key = 100", set = "target.value = 22")) } } test("UDT Data Types - simple and nested") { withTable("source") { withTable("target") { // scalastyle:off line.size.limit val targetData = Seq( Row(SimpleTest(0), ComplexTest(10, Array(1, 2, 3))), Row(SimpleTest(1), ComplexTest(20, Array(4, 5))), Row(SimpleTest(2), ComplexTest(30, Array(6, 7, 8)))) val sourceData = Seq( Row(SimpleTest(0), ComplexTest(40, Array(9, 10))), Row(SimpleTest(3), ComplexTest(50, Array(11)))) val resultData = Seq( Row(SimpleTest(0), ComplexTest(40, Array(9, 10))), Row(SimpleTest(1), ComplexTest(20, Array(4, 5))), Row(SimpleTest(2), ComplexTest(30, Array(6, 7, 8))), Row(SimpleTest(3), ComplexTest(50, Array(11)))) val schema = StructType(Array( StructField("id", new SimpleTestUDT), StructField("complex", new ComplexTestUDT))) val df = spark.createDataFrame(sparkContext.parallelize(targetData), schema) df.collect() spark.createDataFrame(sparkContext.parallelize(targetData), schema) .write.format("delta").saveAsTable("target") spark.createDataFrame(sparkContext.parallelize(sourceData), schema) .write.format("delta").saveAsTable("source") // scalastyle:on line.size.limit sql( s""" |MERGE INTO target as t |USING source as s |ON t.id = s.id |WHEN MATCHED THEN | UPDATE SET * |WHEN NOT MATCHED THEN | INSERT * """.stripMargin) checkAnswer(sql("select * from target"), resultData) } } } test("recorded operations - write all changes") { var events: Seq[UsageRecord] = Seq.empty withKeyValueData( source = (0, 0) :: (1, 10) :: (2, 20) :: (3, 30) :: (4, 40) :: Nil, target = (1, 1) :: (2, 2) :: (3, 3) :: (5, 5) :: (6, 6) :: Nil, isKeyPartitioned = true) { case (sourceName, targetName) => events = Log4jUsageLogger.track { executeMerge( tgt = s"$targetName t", src = s"$sourceName s", cond = "s.key = t.key", update(condition = "s.key > 1", set = "key = s.key, value = s.value"), insert(condition = "s.key < 1", values = "(key, value) VALUES (s.key, s.value)"), deleteNotMatched(condition = "t.key > 5")) } checkAnswer(readDeltaTableByIdentifier(targetName), Seq( Row(0, 0), // inserted Row(1, 1), // existed previously Row(2, 20), // updated Row(3, 30), // updated Row(5, 5) // existed previously // Row(6, 6) deleted )) } // Get recorded operations from usage events val opTypes = events.filter { e => e.metric == "sparkOperationDuration" && e.opType.get.typeName.contains("delta.dml.merge") }.map(_.opType.get.typeName).toSet assert(opTypes == expectedOpTypes) } test("insert only merge - recorded operation") { var events: Seq[UsageRecord] = Seq.empty withKeyValueData( source = (0, 0) :: (1, 10) :: (2, 20) :: (3, 30) :: (4, 40) :: Nil, target = (1, 1) :: (2, 2) :: (3, 3) :: Nil, isKeyPartitioned = true) { case (sourceName, targetName) => withSQLConf(DeltaSQLConf.MERGE_INSERT_ONLY_ENABLED.key -> "true") { events = Log4jUsageLogger.track { executeMerge( tgt = s"$targetName t", src = s"$sourceName s", cond = "s.key = t.key AND t.key > 1", insert(condition = "s.key = 4", values = "(key, value) VALUES (s.key, s.value)")) } } checkAnswer(readDeltaTableByIdentifier(targetName), Seq( Row(1, 1), // existed previously Row(2, 2), // existed previously Row(3, 3), // existed previously Row(4, 40) // inserted )) } // Get recorded operations from usage events val opTypes = events.filter { e => e.metric == "sparkOperationDuration" && e.opType.get.typeName.contains("delta.dml.merge") }.map(_.opType.get.typeName).toSet assert(opTypes == Set( "delta.dml.merge", "delta.dml.merge.materializeSource", "delta.dml.merge.writeInsertsOnlyWhenNoMatchedClauses")) } test("recorded operations - write inserts only") { var events: Seq[UsageRecord] = Seq.empty withKeyValueData( source = (0, 0) :: (1, 10) :: (2, 20) :: (3, 30) :: (4, 40) :: Nil, target = (1, 1) :: (2, 2) :: (3, 3) :: Nil, isKeyPartitioned = true) { case (sourceName, targetName) => events = Log4jUsageLogger.track { executeMerge( tgt = s"$targetName t", src = s"$sourceName s", cond = "s.key = t.key AND s.key > 5", update(condition = "s.key > 10", set = "key = s.key, value = s.value"), insert(condition = "s.key < 1", values = "(key, value) VALUES (s.key, s.value)")) } checkAnswer(readDeltaTableByIdentifier(targetName), Seq( Row(0, 0), // inserted Row(1, 1), // existed previously Row(2, 2), // existed previously Row(3, 3) // existed previously )) } // Get recorded operations from usage events val opTypes = events.filter { e => e.metric == "sparkOperationDuration" && e.opType.get.typeName.contains("delta.dml.merge") }.map(_.opType.get.typeName).toSet assert(opTypes == expectedOpTypesInsertOnly) } test("merge execution is recorded with QueryExecutionListener") { withKeyValueData( source = (0, 0) :: (1, 10) :: Nil, target = (1, 1) :: (2, 2) :: Nil) { case (sourceName, targetName) => val plans = withLogicalPlansCaptured(spark, optimizedPlan = false) { executeMerge( tgt = s"$targetName t", src = s"$sourceName s", cond = "s.key = t.key", update(set = "*")) } val mergeCommands = plans.collect { case m: MergeIntoCommand => m } assert(mergeCommands.size === 1, "Merge command wasn't properly recorded by QueryExecutionListener") } } test("merge on partitioned table with special chars") { withTable("source") { val part1 = "part%1" val part2 = "part%2" val part3 = "part%3" val part4 = "part%4" for (part <- Seq(part1, part2, part3)) { writeTable( spark.range(0, 3, 1, 1) .toDF("key") .withColumn("value", functions.lit(part)) .write.format("delta") .partitionBy("value") .mode("append"), tableSQLIdentifier) } Seq( (0, part1), (0, part2), (1, part2), (0, part3), (1, part3), (2, part3), (0, part4) ).toDF("key", "value").createOrReplaceTempView("source") executeMerge( tgt = s"$tableSQLIdentifier t", src = "source s", cond = "t.key = s.key AND t.value = s.value", delete(condition = s"s.value = '$part2'"), update(set = s"t.key = -1"), insert("*") ) checkAnswer( readDeltaTableByIdentifier(), Row(-1, part1) :: Row(1, part1) :: Row(2, part1) :: Row(2, part2) :: Row(-1, part3) :: Row(-1, part3) :: Row(-1, part3) :: Row(0, part4) :: Nil) } } } @SQLUserDefinedType(udt = classOf[SimpleTestUDT]) case class SimpleTest(value: Int) class SimpleTestUDT extends UserDefinedType[SimpleTest] { override def sqlType: DataType = IntegerType override def serialize(input: SimpleTest): Any = input.value override def deserialize(datum: Any): SimpleTest = datum match { case a: Int => SimpleTest(a) } override def userClass: Class[SimpleTest] = classOf[SimpleTest] } @SQLUserDefinedType(udt = classOf[ComplexTestUDT]) case class ComplexTest(key: Int, values: Array[Int]) class ComplexTestUDT extends UserDefinedType[ComplexTest] { override def sqlType: DataType = StructType(Seq( StructField("key", IntegerType), StructField("values", ArrayType(IntegerType, containsNull = false)))) override def serialize(input: ComplexTest): Any = { val row = new GenericInternalRow(2) row.setInt(0, input.key) row.update(1, UnsafeArrayData.fromPrimitiveArray(input.values)) row } override def deserialize(datum: Any): ComplexTest = datum match { case row: InternalRow => ComplexTest(row.getInt(0), row.getArray(1).toIntArray()) } override def userClass: Class[ComplexTest] = classOf[ComplexTest] } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoTestUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import io.delta.tables._ import org.apache.spark.sql.DataFrame import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession /** * Base trait collecting helper methods to run MERGE tests. Merge test suite will want to mix in * either [[MergeIntoSQLTestUtils]] or [[MergeIntoScalaTestUtils]] to run merge tests using the SQL * or Scala API resp. */ trait MergeIntoTestUtils extends DeltaDMLTestUtils with MergeHelpers { self: SharedSparkSession => protected def executeMerge( target: String, source: String, condition: String, update: String, insert: String): Unit protected def executeMerge( tgt: String, src: String, cond: String, clauses: MergeClause*): Unit protected def executeMergeWithSchemaEvolution( tgt: String, src: String, cond: String, clauses: MergeClause*): Unit protected def withCrossJoinEnabled(body: => Unit): Unit = { withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "true") { body } } } trait MergeIntoSQLTestUtils extends DeltaSQLTestUtils with MergeIntoTestUtils { self: SharedSparkSession => protected def basicMergeStmt( cte: Option[String] = None, target: String, source: String, condition: String, update: String, insert: String): String = { basicMergeStmt( cte = cte, target = target, source = source, condition = condition, withSchemaEvolution = false, super.update(set = update), super.insert(values = insert)) } protected def basicMergeStmt( cte: Option[String], target: String, source: String, condition: String, withSchemaEvolution: Boolean, clauses: MergeClause*): String = { val clausesStr = clauses.map(_.sql).mkString("\n") val schemaEvolutionStr = if (withSchemaEvolution) "WITH SCHEMA EVOLUTION" else "" s""" |${cte.getOrElse("")} |MERGE $schemaEvolutionStr INTO $target |USING $source |ON $condition |$clausesStr """.stripMargin } override protected def executeMerge( target: String, source: String, condition: String, update: String, insert: String): Unit = spark.sql(basicMergeStmt(cte = None, target, source, condition, update, insert)) override protected def executeMerge( tgt: String, src: String, cond: String, clauses: MergeClause*): Unit = { spark.sql(basicMergeStmt(cte = None, tgt, src, cond, withSchemaEvolution = false, clauses: _*)) } override protected def executeMergeWithSchemaEvolution( tgt: String, src: String, cond: String, clauses: MergeClause*): Unit = { throw new UnsupportedOperationException( "The SQL syntax [WITH SCHEMA EVOLUTION] is not yet supported.") } } trait MergeIntoScalaTestUtils extends MergeIntoTestUtils { self: SharedSparkSession => override protected def executeMerge( target: String, source: String, condition: String, update: String, insert: String): Unit = { executeMerge( tgt = target, src = source, cond = condition, this.update(set = update), this.insert(values = insert)) } override protected def executeMerge( tgt: String, src: String, cond: String, clauses: MergeClause*): Unit = getMergeBuilder(tgt, src, cond, clauses: _*).execute() override protected def executeMergeWithSchemaEvolution( tgt: String, src: String, cond: String, clauses: MergeClause*): Unit = getMergeBuilder(tgt, src, cond, clauses: _*).withSchemaEvolution().execute() private def getMergeBuilder( tgt: String, src: String, cond: String, clauses: MergeClause*): DeltaMergeBuilder = { def buildClause(clause: MergeClause, mergeBuilder: DeltaMergeBuilder) : DeltaMergeBuilder = clause match { case _: MatchedClause => val actionBuilder: DeltaMergeMatchedActionBuilder = if (clause.condition != null) mergeBuilder.whenMatched(clause.condition) else mergeBuilder.whenMatched() if (clause.action.startsWith("DELETE")) { // DELETE clause actionBuilder.delete() } else { // UPDATE clause val setColExprStr = clause.action.trim.stripPrefix("UPDATE SET") if (setColExprStr.trim == "*") { // UPDATE SET * actionBuilder.updateAll() } else if (setColExprStr.contains("array_")) { // UPDATE SET x = array_union(..) val setColExprPairs = parseUpdate(Seq(setColExprStr)) actionBuilder.updateExpr(setColExprPairs) } else { // UPDATE SET x = a, y = b, z = c val setColExprPairs = parseUpdate(setColExprStr.split(",")) actionBuilder.updateExpr(setColExprPairs) } } case _: NotMatchedClause => // INSERT clause val actionBuilder: DeltaMergeNotMatchedActionBuilder = if (clause.condition != null) mergeBuilder.whenNotMatched(clause.condition) else mergeBuilder.whenNotMatched() val valueStr = clause.action.trim.stripPrefix("INSERT") if (valueStr.trim == "*") { // INSERT * actionBuilder.insertAll() } else { // INSERT (x, y, z) VALUES (a, b, c) val valueColExprsPairs = parseInsert(valueStr, Some(clause)) actionBuilder.insertExpr(valueColExprsPairs) } case _: NotMatchedBySourceClause => val actionBuilder: DeltaMergeNotMatchedBySourceActionBuilder = if (clause.condition != null) mergeBuilder.whenNotMatchedBySource(clause.condition) else mergeBuilder.whenNotMatchedBySource() if (clause.action.startsWith("DELETE")) { // DELETE clause actionBuilder.delete() } else { // UPDATE clause val setColExprStr = clause.action.trim.stripPrefix("UPDATE SET") if (setColExprStr.contains("array_")) { // UPDATE SET x = array_union(..) val setColExprPairs = parseUpdate(Seq(setColExprStr)) actionBuilder.updateExpr(setColExprPairs) } else { // UPDATE SET x = a, y = b, z = c val setColExprPairs = parseUpdate(setColExprStr.split(",")) actionBuilder.updateExpr(setColExprPairs) } } } val deltaTable = DeltaTestUtils.getDeltaTableForIdentifierOrPath( spark, DeltaTestUtils.getTableIdentifierOrPath(tgt)) val sourceDataFrame: DataFrame = { val (tableOrQuery, optionalAlias) = DeltaTestUtils.parseTableAndAlias(src) var df = if (tableOrQuery.startsWith("(")) spark.sql(tableOrQuery) else spark.table(tableOrQuery) optionalAlias.foreach { alias => df = df.as(alias) } df } var mergeBuilder = deltaTable.merge(sourceDataFrame, cond) clauses.foreach { clause => mergeBuilder = buildClause(clause, mergeBuilder) } mergeBuilder } protected def parseUpdate(update: Seq[String]): Map[String, String] = { update.map { _.split("=").toList }.map { case setCol :: setExpr :: Nil => setCol.trim -> setExpr.trim case _ => fail("error parsing update actions " + update) }.toMap } protected def parseInsert(valueStr: String, clause: Option[MergeClause]): Map[String, String] = { valueStr.split("VALUES").toList match { case colsStr :: exprsStr :: Nil => def parse(str: String): Seq[String] = { str.trim.stripPrefix("(").stripSuffix(")").split(",").map(_.trim) } val cols = parse(colsStr) val exprs = parse(exprsStr) require(cols.size == exprs.size, s"Invalid insert action ${clause.get.action}: cols = $cols, exprs = $exprs") cols.zip(exprs).toMap case list => fail(s"Invalid insert action ${clause.get.action} split into $list") } } protected def parsePath(nameOrPath: String): String = { if (nameOrPath.startsWith("delta.`")) { nameOrPath.stripPrefix("delta.`").stripSuffix("`") } else nameOrPath } } trait MergeHelpers { /** A simple representative of a any WHEN clause in a MERGE statement */ protected sealed trait MergeClause { def condition: String def action: String def clause: String def sql: String = { assert(action != null, "action not specified yet") val cond = if (condition != null) s"AND $condition" else "" s"WHEN $clause $cond THEN $action" } } protected case class MatchedClause(condition: String, action: String) extends MergeClause { override def clause: String = "MATCHED" } protected case class NotMatchedClause(condition: String, action: String) extends MergeClause { override def clause: String = "NOT MATCHED" } protected case class NotMatchedBySourceClause(condition: String, action: String) extends MergeClause { override def clause: String = "NOT MATCHED BY SOURCE" } protected def update(set: String = null, condition: String = null): MergeClause = { MatchedClause(condition, s"UPDATE SET $set") } protected def delete(condition: String = null): MergeClause = { MatchedClause(condition, s"DELETE") } protected def insert(values: String = null, condition: String = null): MergeClause = { NotMatchedClause(condition, s"INSERT $values") } protected def updateNotMatched(set: String = null, condition: String = null): MergeClause = { NotMatchedBySourceClause(condition, s"UPDATE SET $set") } protected def deleteNotMatched(condition: String = null): MergeClause = { NotMatchedBySourceClause(condition, s"DELETE") } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoTimestampConsistencySuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.sql.Timestamp import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.expressions.{CurrentTimestamp, Now} import org.apache.spark.sql.functions.{current_timestamp, lit, timestamp_seconds} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.Utils class MergeIntoTimestampConsistencySuite extends MergeIntoTimestampConsistencySuiteBase { override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaSQLConf.MERGE_USE_PERSISTENT_DELETION_VECTORS.key, "false") } } abstract class MergeIntoTimestampConsistencySuiteBase extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { private def withTestTables(block: => Unit): Unit = { def setupTablesAndRun(): Unit = { spark.range(0, 5) .toDF("id") .withColumn("updated", lit(false)) .withColumn("timestampOne", timestamp_seconds(lit(1))) .withColumn("timestampTwo", timestamp_seconds(lit(1337))) .write .format("delta") .saveAsTable("target") spark.range(0, 10) .toDF("id") .withColumn("updated", lit(true)) .withColumn("timestampOne", current_timestamp()) .withColumn("timestampTwo", current_timestamp()) .createOrReplaceTempView("source") block } Utils.tryWithSafeFinally(setupTablesAndRun) { sql("DROP VIEW IF EXISTS source") sql("DROP TABLE IF EXISTS target") } } test("Consistent timestamps between source and ON condition") { withTestTables { sql(s"""MERGE INTO target t | USING source s | ON s.id = t.id AND s.timestampOne = now() | WHEN MATCHED THEN UPDATE SET *""".stripMargin) assertAllRowsAreUpdated() } } test("Consistent timestamps between source and WHEN MATCHED condition") { withTestTables { sql(s"""MERGE INTO target t | USING source s | ON s.id = t.id | WHEN MATCHED AND s.timestampOne = now() AND s.timestampTwo = now() | THEN UPDATE SET *""".stripMargin) assertAllRowsAreUpdated() } } test("Consistent timestamps between source and UPDATE SET") { withTestTables { sql( s"""MERGE INTO target t | USING source s | ON s.id = t.id | WHEN MATCHED THEN UPDATE | SET updated = s.updated, t.timestampOne = s.timestampOne, t.timestampTwo = now() |""".stripMargin) assertUpdatedTimestampsInTargetAreAllEqual() } } test("Consistent timestamps between source and WHEN NOT MATCHED condition") { withTestTables { sql(s"""MERGE INTO target t | USING source s | ON s.id = t.id | WHEN NOT MATCHED AND s.timestampOne = now() AND s.timestampTwo = now() | THEN INSERT * |""".stripMargin) assertNewSourceRowsInserted() } } test("Consistent timestamps between source and INSERT VALUES") { withTestTables { sql( s"""MERGE INTO target t | USING source s | ON s.id = t.id | WHEN NOT MATCHED THEN INSERT (id, updated, timestampOne, timestampTwo) | VALUES (s.id, s.updated, s.timestampOne, now()) |""".stripMargin) assertUpdatedTimestampsInTargetAreAllEqual() } } test("Consistent timestamps with subquery in source") { withTestTables { val sourceWithSubqueryTable = "source_with_subquery" withTempView(s"$sourceWithSubqueryTable") { sql( s"""CREATE OR REPLACE TEMPORARY VIEW $sourceWithSubqueryTable | AS SELECT * FROM source WHERE timestampOne IN (SELECT now()) |""".stripMargin).collect() sql(s"""MERGE INTO target t | USING $sourceWithSubqueryTable s | ON s.id = t.id | WHEN MATCHED THEN UPDATE SET *""".stripMargin) assertAllRowsAreUpdated() } } } private def assertAllRowsAreUpdated(): Unit = { val nonUpdatedRowsCount = sql("SELECT * FROM target WHERE updated = FALSE").count() assert(0 === nonUpdatedRowsCount, "Un-updated rows in target table") } private def assertNewSourceRowsInserted(): Unit = { val numNotInsertedSourceRows = sql("SELECT * FROM source s LEFT ANTI JOIN target t ON s.id = t.id").count() assert(0 === numNotInsertedSourceRows, "Un-inserted rows in source table") } private def assertUpdatedTimestampsInTargetAreAllEqual(): Unit = { import testImplicits._ val timestampCombinations = sql(s"""SELECT timestampOne, timestampTwo | FROM target WHERE updated = TRUE GROUP BY timestampOne, timestampTwo |""".stripMargin) val rows = timestampCombinations.as[(Timestamp, Timestamp)].collect() assert(1 === rows.length, "Multiple combinations of timestamp values in target table") assert(rows(0)._1 === rows(0)._2, "timestampOne and timestampTwo are not equal in target table") } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/NonFateSharingFutureSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.concurrent.atomic.AtomicInteger import scala.concurrent.duration._ import scala.util.control.ControlThrowable import org.apache.spark.sql.delta.util.threads.DeltaThreadPool import org.apache.spark.{SparkException, SparkFunSuite} import org.apache.spark.sql.test.SharedSparkSession class NonFateSharingFutureSuite extends SparkFunSuite with SharedSparkSession { test("function only runs once on success") { val count = new AtomicInteger val future = DeltaThreadPool("test", 1).submitNonFateSharing { _ => count.incrementAndGet } assert(future.get(10.seconds) === 1) assert(future.get(10.seconds) === 1) spark.cloneSession().withActive { assert(future.get(10.seconds) === 1) } } test("non-fatal exception in future is ignored") { val count = new AtomicInteger val future = DeltaThreadPool("test", 1).submitNonFateSharing { _ => count.incrementAndGet match { case 1 => throw new Exception case i => i } } // Make sure the future already failed before waiting on it. This should happen ~immediately // unless the test runner is horribly overloaded/slow/etc, and stabilizes the assertions below. eventually(timeout(100.seconds)) { assert(count.get == 1) } spark.cloneSession().withActive { assert(future.get(1.seconds) === 2) } assert(future.get(1.seconds) === 3) spark.cloneSession().withActive { assert(future.get(1.seconds) === 4) } assert(future.get(1.seconds) === 5) } test("fatal exception in future only propagates once, and only to owning session") { val count = new AtomicInteger val future = DeltaThreadPool("test", 1).submitNonFateSharing { _ => count.incrementAndGet match { case 1 => throw new InternalError case i => i } } // Make sure the future already failed before waiting on it. This should happen ~immediately // unless the test runner is horribly overloaded/slow/etc, and stabilizes the assertions below. eventually(timeout(100.seconds)) { assert(count.get == 1) } spark.cloneSession().withActive { assert(future.get(1.seconds) === 2) } intercept[InternalError] { future.get(1.seconds) } spark.cloneSession().withActive { assert(future.get(1.seconds) === 3) } assert(future.get(1.seconds) === 4) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/OptimisticTransactionLegacyTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.DeltaOperations.ManualUpdate import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile import org.apache.spark.sql.delta.actions.{Action, AddCDCFile, AddFile, FileAction, Metadata, RemoveFile, SetTransaction} import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.hadoop.fs.Path import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{StringType, StructField, StructType} // These tests are potentially a subset of the tests already in OptimisticTransactionSuite. // These tests can potentially be removed but only after confirming that these tests are // truly a subset of the tests in OptimisticTransactionSuite. trait OptimisticTransactionLegacyTests extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { private val addA = createTestAddFile(encodedPath = "a") private val addB = createTestAddFile(encodedPath = "b") private val addC = createTestAddFile(encodedPath = "c") import testImplicits._ test("block append against metadata change") { withTempDir { tempDir => val log = DeltaLog.forTable(spark, tempDir) // Initialize the log. log.startTransaction().commitManually() val txn = log.startTransaction() val winningTxn = log.startTransaction() winningTxn.commit(Metadata() :: Nil, ManualUpdate) intercept[MetadataChangedException] { txn.commit(addA :: Nil, ManualUpdate) } } } test("block read+append against append") { withTempDir { tempDir => val log = DeltaLog.forTable(spark, tempDir) // Initialize the log. log.startTransaction().commitManually() val txn = log.startTransaction() // reads the table txn.filterFiles() val winningTxn = log.startTransaction() winningTxn.commit(addA :: Nil, ManualUpdate) intercept[ConcurrentAppendException] { txn.commit(addB :: Nil, ManualUpdate) } } } test("allow blind-append against any data change") { withTempDir { tempDir => val log = DeltaLog.forTable(spark, tempDir) // Initialize the log and add data. log.startTransaction().commitManually(addA) val txn = log.startTransaction() val winningTxn = log.startTransaction() winningTxn.commit(addA.remove :: addB :: Nil, ManualUpdate) txn.commit(addC :: Nil, ManualUpdate) checkAnswer(log.update().allFiles.select("path"), Row("b") :: Row("c") :: Nil) } } test("allow read+append+delete against no data change") { withTempDir { tempDir => val log = DeltaLog.forTable(spark, tempDir) // Initialize the log and add data. ManualUpdate is just a no-op placeholder. log.startTransaction().commitManually(addA) val txn = log.startTransaction() txn.filterFiles() val winningTxn = log.startTransaction() winningTxn.commit(Nil, ManualUpdate) txn.commit(addA.remove :: addB :: Nil, ManualUpdate) checkAnswer(log.update().allFiles.select("path"), Row("b") :: Nil) } } val A_P1 = "part=1/a" val B_P1 = "part=1/b" val C_P1 = "part=1/c" val C_P2 = "part=2/c" val D_P2 = "part=2/d" val E_P3 = "part=3/e" val F_P3 = "part=3/f" val G_P4 = "part=4/g" private val addA_P1 = AddFile(A_P1, Map("part" -> "1"), 1, 1, dataChange = true) private val addB_P1 = AddFile(B_P1, Map("part" -> "1"), 1, 1, dataChange = true) private val addC_P1 = AddFile(C_P1, Map("part" -> "1"), 1, 1, dataChange = true) private val addC_P2 = AddFile(C_P2, Map("part" -> "2"), 1, 1, dataChange = true) private val addD_P2 = AddFile(D_P2, Map("part" -> "2"), 1, 1, dataChange = true) private val addE_P3 = AddFile(E_P3, Map("part" -> "3"), 1, 1, dataChange = true) private val addF_P3 = AddFile(F_P3, Map("part" -> "3"), 1, 1, dataChange = true) private val addG_P4 = AddFile(G_P4, Map("part" -> "4"), 1, 1, dataChange = true) test("allow concurrent commit on disjoint partitions") { withLog(addA_P1 :: addE_P3 :: Nil) { log => val tx1 = log.startTransaction() // TX1 reads P3 (but not P1) val tx1Read = tx1.filterFiles(('part === 3).expr :: Nil) assert(tx1Read.map(_.path) == E_P3 :: Nil) val tx2 = log.startTransaction() tx2.filterFiles() // TX2 modifies only P1 tx2.commit(addB_P1 :: Nil, ManualUpdate) // free to commit because P1 modified by TX2 was not read tx1.commit(addC_P2 :: addE_P3.remove :: Nil, ManualUpdate) checkAnswer( log.update().allFiles.select("path"), Row(A_P1) :: // start (E_P3 was removed by TX1) Row(B_P1) :: // TX2 Row(C_P2) :: Nil) // TX1 } } test("allow concurrent commit on disjoint partitions reading all partitions") { withLog(addA_P1 :: addD_P2 :: Nil) { log => val tx1 = log.startTransaction() // TX1 read P1 tx1.filterFiles(('part isin 1).expr :: Nil) val tx2 = log.startTransaction() tx2.filterFiles() tx2.commit(addC_P2 :: addD_P2.remove :: Nil, ManualUpdate) tx1.commit(addE_P3 :: addF_P3 :: Nil, ManualUpdate) checkAnswer( log.update().allFiles.select("path"), Row(A_P1) :: // start Row(C_P2) :: // TX2 Row(E_P3) :: Row(F_P3) :: Nil) // TX1 } } test("block concurrent commit when read partition was appended to by concurrent write") { withLog(addA_P1 :: addD_P2 :: addE_P3 :: Nil) { log => val tx1 = log.startTransaction() // TX1 reads only P1 val tx1Read = tx1.filterFiles(('part === 1).expr :: Nil) assert(tx1Read.map(_.path) == A_P1 :: Nil) val tx2 = log.startTransaction() tx2.filterFiles() // TX2 modifies only P1 tx2.commit(addB_P1 :: Nil, ManualUpdate) intercept[ConcurrentAppendException] { // P1 was modified tx1.commit(addC_P2 :: addE_P3 :: Nil, ManualUpdate) } } } test("block concurrent commit on full table scan") { withLog(addA_P1 :: addD_P2 :: Nil) { log => val tx1 = log.startTransaction() // TX1 full table scan tx1.filterFiles() tx1.filterFiles(('part === 1).expr :: Nil) val tx2 = log.startTransaction() tx2.filterFiles() tx2.commit(addC_P2 :: addD_P2.remove :: Nil, ManualUpdate) intercept[ConcurrentAppendException] { tx1.commit(addE_P3 :: addF_P3 :: Nil, ManualUpdate) } } } val A_1_1 = "a=1/b=1/a" val B_1_2 = "a=1/b=2/b" val C_2_1 = "a=2/b=1/c" val D_3_1 = "a=3/b=1/d" val addA_1_1_nested = AddFile( A_1_1, Map("a" -> "1", "b" -> "1"), 1, 1, dataChange = true) val addB_1_2_nested = AddFile( B_1_2, Map("a" -> "1", "b" -> "2"), 1, 1, dataChange = true) val addC_2_1_nested = AddFile( C_2_1, Map("a" -> "2", "b" -> "1"), 1, 1, dataChange = true) val addD_3_1_nested = AddFile( D_3_1, Map("a" -> "3", "b" -> "1"), 1, 1, dataChange = true) test("allow concurrent adds to disjoint nested partitions when read is disjoint from write") { withLog(addA_1_1_nested :: Nil, partitionCols = "a" :: "b" :: Nil) { log => val tx1 = log.startTransaction() // TX1 reads a=1/b=1 val tx1Read = tx1.filterFiles(('a === 1 and 'b === 1).expr :: Nil) assert(tx1Read.map(_.path) == A_1_1 :: Nil) val tx2 = log.startTransaction() tx2.filterFiles() // TX2 reads all partitions and modifies only a=1/b=2 tx2.commit(addB_1_2_nested :: Nil, ManualUpdate) // TX1 reads a=1/b=1 which was not modified by TX2, hence TX1 can write to a=2/b=1 tx1.commit(addC_2_1_nested :: Nil, ManualUpdate) checkAnswer( log.update().allFiles.select("path"), Row(A_1_1) :: // start Row(B_1_2) :: // TX2 Row(C_2_1) :: Nil) // TX1 } } test("allow concurrent adds to same nested partitions when read is disjoint from write") { withLog(addA_1_1_nested :: Nil, partitionCols = "a" :: "b" :: Nil) { log => val tx1 = log.startTransaction() // TX1 reads a=1/b=1 val tx1Read = tx1.filterFiles(('a === 1 and 'b === 1).expr :: Nil) assert(tx1Read.map(_.path) == A_1_1 :: Nil) val tx2 = log.startTransaction() tx2.filterFiles() // TX2 modifies a=1/b=2 tx2.commit(addB_1_2_nested :: Nil, ManualUpdate) // TX1 reads a=1/b=1 which was not modified by TX2, hence TX1 can write to a=2/b=1 val add = AddFile( "a=1/b=2/x", Map("a" -> "1", "b" -> "2"), 1, 1, dataChange = true) tx1.commit(add :: Nil, ManualUpdate) checkAnswer( log.update().allFiles.select("path"), Row(A_1_1) :: // start Row(B_1_2) :: // TX2 Row("a=1/b=2/x") :: Nil) // TX1 } } test("allow concurrent add when read at lvl1 partition is disjoint from concur. write at lvl2") { withLog( addA_1_1_nested :: addB_1_2_nested :: Nil, partitionCols = "a" :: "b" :: Nil) { log => val tx1 = log.startTransaction() // TX1 reads a=1 val tx1Read = tx1.filterFiles(('a === 1).expr :: Nil) assert(tx1Read.map(_.path).toSet == Set(A_1_1, B_1_2)) val tx2 = log.startTransaction() tx2.filterFiles() // TX2 modifies only a=2/b=1 tx2.commit(addC_2_1_nested :: Nil, ManualUpdate) // free to commit a=2/b=1 tx1.commit(addD_3_1_nested :: Nil, ManualUpdate) checkAnswer( log.update().allFiles.select("path"), Row(A_1_1) :: Row(B_1_2) :: // start Row(C_2_1) :: // TX2 Row(D_3_1) :: Nil) // TX1 } } test("block commit when read at lvl1 partition reads lvl2 file concur. deleted") { withLog( addA_1_1_nested :: addB_1_2_nested :: Nil, partitionCols = "a" :: "b" :: Nil) { log => val tx1 = log.startTransaction() // TX1 reads a=1 val tx1Read = tx1.filterFiles(('a === 1).expr :: Nil) assert(tx1Read.map(_.path).toSet == Set(A_1_1, B_1_2)) val tx2 = log.startTransaction() tx2.filterFiles() // TX2 modifies a=1/b=1 tx2.commit(addA_1_1_nested.remove :: Nil, ManualUpdate) intercept[ConcurrentDeleteReadException] { // TX2 modified a=1, which was read by TX1 tx1.commit(addD_3_1_nested :: Nil, ManualUpdate) } } } test("block commit when full table read conflicts with concur. write in lvl2 nested partition") { withLog(addA_1_1_nested :: Nil, partitionCols = "a" :: "b" :: Nil) { log => val tx1 = log.startTransaction() // TX1 full table scan tx1.filterFiles() val tx2 = log.startTransaction() tx2.filterFiles() // TX2 modifies only a=1/b=2 tx2.commit(addB_1_2_nested :: Nil, ManualUpdate) intercept[ConcurrentAppendException] { // TX2 modified table all of which was read by TX1 tx1.commit(addC_2_1_nested :: Nil, ManualUpdate) } } } test("block commit when part. range read conflicts with concur. write in lvl2 nested partition") { withLog( addA_1_1_nested :: Nil, partitionCols = "a" :: "b" :: Nil) { log => val tx1 = log.startTransaction() // TX1 reads multiple nested partitions a >= 1 or b > 1 val tx1Read = tx1.filterFiles(('a >= 1 or 'b > 1).expr :: Nil) assert(tx1Read.map(_.path).toSet == Set(A_1_1)) val tx2 = log.startTransaction() tx2.filterFiles() // TX2 modifies a=1/b=2 tx2.commit(addB_1_2_nested :: Nil, ManualUpdate) intercept[ConcurrentAppendException] { // partition a=1/b=2 conflicts with our read a >= 1 or 'b > 1 tx1.commit(addD_3_1_nested :: Nil, ManualUpdate) } } } test("block commit with concurrent removes on same file") { withLog(addB_1_2_nested :: Nil, partitionCols = "a" :: "b" :: Nil) { log => val tx1 = log.startTransaction() // TX1 reads a=2 so that read is disjoint with write partition. tx1.filterFiles(('a === 2).expr :: Nil) val tx2 = log.startTransaction() tx2.filterFiles() // TX2 modifies a=1/b=2 tx2.commit(addB_1_2_nested.remove :: Nil, ManualUpdate) intercept[ConcurrentDeleteDeleteException] { // TX1 read does not conflict with TX2 as disjoint partitions // But TX2 removed the same file that TX1 is trying to remove tx1.commit(addB_1_2_nested.remove:: Nil, ManualUpdate) } } } test("block commit when full table read conflicts with add in any partition") { withLog(addA_P1 :: addC_P2 :: Nil) { log => val tx1 = log.startTransaction() tx1.filterFiles() val tx2 = log.startTransaction() tx2.filterFiles() tx2.commit(addC_P2.remove :: addB_P1 :: Nil, ManualUpdate) intercept[ConcurrentAppendException] { // TX1 read whole table but TX2 concurrently modified partition P2 tx1.commit(addD_P2 :: Nil, ManualUpdate) } } } test("block commit when full table read conflicts with delete in any partition") { withLog(addA_P1 :: addC_P2 :: Nil) { log => val tx1 = log.startTransaction() tx1.filterFiles() val tx2 = log.startTransaction() tx2.filterFiles() tx2.commit(addA_P1.remove :: Nil, ManualUpdate) intercept[ConcurrentDeleteReadException] { // TX1 read whole table but TX2 concurrently modified partition P1 tx1.commit(addB_P1.remove :: Nil, ManualUpdate) } } } test("block concurrent replaceWhere initial empty") { withLog(addA_P1 :: Nil) { log => val tx1 = log.startTransaction() // replaceWhere (part >= 2) -> empty read val tx1Read = tx1.filterFiles(('part >= 2).expr :: Nil) assert(tx1Read.isEmpty) val tx2 = log.startTransaction() // replaceWhere (part >= 2) -> empty read val tx2Read = tx2.filterFiles(('part >= 2).expr :: Nil) assert(tx2Read.isEmpty) tx2.commit(addE_P3 :: Nil, ManualUpdate) intercept[ConcurrentAppendException] { // Tx2 have modified P2 which conflicts with our read (part >= 2) tx1.commit(addC_P2 :: Nil, ManualUpdate) } } } test("allow concurrent replaceWhere disjoint partitions initial empty") { withLog(addA_P1 :: Nil) { log => val tx1 = log.startTransaction() // replaceWhere (part > 2 and part <= 3) -> empty read val tx1Read = tx1.filterFiles(('part > 1 and 'part <= 3).expr :: Nil) assert(tx1Read.isEmpty) val tx2 = log.startTransaction() // replaceWhere (part > 3) -> empty read val tx2Read = tx2.filterFiles(('part > 3).expr :: Nil) assert(tx2Read.isEmpty) tx1.commit(addC_P2 :: Nil, ManualUpdate) // P2 doesn't conflict with read predicate (part > 3) tx2.commit(addG_P4 :: Nil, ManualUpdate) checkAnswer( log.update().allFiles.select("path"), Row(A_P1) :: // start Row(C_P2) :: // TX1 Row(G_P4) :: Nil) // TX2 } } test("block concurrent replaceWhere NOT empty but conflicting predicate") { withLog(addA_P1 :: addG_P4 :: Nil) { log => val tx1 = log.startTransaction() // replaceWhere (part <= 3) -> read P1 val tx1Read = tx1.filterFiles(('part <= 3).expr :: Nil) assert(tx1Read.map(_.path) == A_P1 :: Nil) val tx2 = log.startTransaction() // replaceWhere (part >= 2) -> read P4 val tx2Read = tx2.filterFiles(('part >= 2).expr :: Nil) assert(tx2Read.map(_.path) == G_P4 :: Nil) tx1.commit(addA_P1.remove :: addC_P2 :: Nil, ManualUpdate) intercept[ConcurrentAppendException] { // Tx1 have modified P2 which conflicts with our read (part >= 2) tx2.commit(addG_P4.remove :: addE_P3 :: Nil, ManualUpdate) } } } test("block concurrent commit on read & add conflicting partitions") { withLog(addA_P1 :: Nil) { log => val tx1 = log.startTransaction() // read P1 val tx1Read = tx1.filterFiles(('part === 1).expr :: Nil) assert(tx1Read.map(_.path) == A_P1 :: Nil) // tx2 commits before tx1 val tx2 = log.startTransaction() tx2.filterFiles() tx2.commit(addB_P1 :: Nil, ManualUpdate) intercept[ConcurrentAppendException] { // P1 read by TX1 was modified by TX2 tx1.commit(addE_P3 :: Nil, ManualUpdate) } } } test("block concurrent commit on read & delete conflicting partitions") { withLog(addA_P1 :: addB_P1 :: Nil) { log => val tx1 = log.startTransaction() // read P1 tx1.filterFiles(('part === 1).expr :: Nil) // tx2 commits before tx1 val tx2 = log.startTransaction() tx2.filterFiles() tx2.commit(addA_P1.remove :: Nil, ManualUpdate) intercept[ConcurrentDeleteReadException] { // P1 read by TX1 was removed by TX2 tx1.commit(addE_P3 :: Nil, ManualUpdate) } } } test("block 2 concurrent replaceWhere transactions") { withLog(addA_P1 :: Nil) { log => val tx1 = log.startTransaction() // read P1 tx1.filterFiles(('part === 1).expr :: Nil) val tx2 = log.startTransaction() // read P1 tx2.filterFiles(('part === 1).expr :: Nil) // tx1 commits before tx2 tx1.commit(addA_P1.remove :: addB_P1 :: Nil, ManualUpdate) intercept[ConcurrentAppendException] { // P1 read & deleted by TX1 is being modified by TX2 tx2.commit(addA_P1.remove :: addC_P1 :: Nil, ManualUpdate) } } } test("block 2 concurrent replaceWhere transactions changing partitions") { withLog(addA_P1 :: addC_P2 :: addE_P3 :: Nil) { log => val tx1 = log.startTransaction() // read P3 tx1.filterFiles(('part === 3 or 'part === 1).expr :: Nil) val tx2 = log.startTransaction() // read P3 tx2.filterFiles(('part === 3 or 'part === 2).expr :: Nil) // tx1 commits before tx2 tx1.commit(addA_P1.remove :: addE_P3.remove :: addB_P1 :: Nil, ManualUpdate) intercept[ConcurrentDeleteReadException] { // P3 read & deleted by TX1 is being modified by TX2 tx2.commit(addC_P2.remove :: addE_P3.remove :: addD_P2 :: Nil, ManualUpdate) } } } test("block concurrent full table scan after concurrent write completes") { withLog(addA_P1 :: addE_P3 :: Nil) { log => val tx1 = log.startTransaction() val tx2 = log.startTransaction() tx2.filterFiles() tx2.commit(addC_P2 :: Nil, ManualUpdate) tx1.filterFiles(('part === 1).expr :: Nil) // full table scan tx1.filterFiles() intercept[ConcurrentAppendException] { tx1.commit(addA_P1.remove :: Nil, ManualUpdate) } } } test("block concurrent commit mixed metadata and data predicate") { withLog(addA_P1 :: addE_P3 :: Nil) { log => val tx1 = log.startTransaction() val tx2 = log.startTransaction() tx2.filterFiles() tx2.commit(addC_P2 :: Nil, ManualUpdate) // actually a full table scan tx1.filterFiles(('part === 1 or 'year > 2019).expr :: Nil) intercept[ConcurrentAppendException] { tx1.commit(addA_P1.remove :: Nil, ManualUpdate) } } } test("block concurrent read (2 scans) and add when read partition was changed by concur. write") { withLog(addA_P1 :: addE_P3 :: Nil) { log => val tx1 = log.startTransaction() tx1.filterFiles(('part === 1).expr :: Nil) val tx2 = log.startTransaction() tx2.filterFiles() tx2.commit(addC_P2 :: Nil, ManualUpdate) tx1.filterFiles(('part > 1 and 'part < 3).expr :: Nil) intercept[ConcurrentAppendException] { // P2 added by TX2 conflicts with our read condition 'part > 1 and 'part < 3 tx1.commit(addA_P1.remove :: Nil, ManualUpdate) } } } def setDataChangeFalse(fileActions: Seq[FileAction]): Seq[FileAction] = { fileActions.map { case a: AddFile => a.copy(dataChange = false) case r: RemoveFile => r.copy(dataChange = false) case cdc: AddCDCFile => cdc // change files are always dataChange = false } } test("no data change: allow data rearrange when new files concurrently added") { withLog(addA_P1 :: addB_P1 :: Nil) { log => val tx1 = log.startTransaction() tx1.filterFiles() val tx2 = log.startTransaction() tx2.filterFiles() tx2.commit( addE_P3 :: Nil, ManualUpdate) // tx1 rearranges files tx1.commit( setDataChangeFalse(addA_P1.remove :: addB_P1.remove :: addC_P1 :: Nil), ManualUpdate) checkAnswer( log.update().allFiles.select("path"), Row(C_P1) :: Row(E_P3) :: Nil) } } test("no data change: block data rearrange when concurrently delete removes same file") { withLog(addA_P1 :: addB_P1 :: Nil) { log => val tx1 = log.startTransaction() tx1.filterFiles() // tx2 removes file val tx2 = log.startTransaction() tx2.filterFiles() tx2.commit(addA_P1.remove :: Nil, ManualUpdate) intercept[ConcurrentDeleteReadException] { // tx1 reads to rearrange the same file that tx2 deleted tx1.commit( setDataChangeFalse(addA_P1.remove :: addB_P1.remove :: addC_P1 :: Nil), ManualUpdate) } } } test("readWholeTable should block concurrent delete") { withLog(addA_P1 :: Nil) { log => val tx1 = log.startTransaction() tx1.readWholeTable() // tx2 removes file val tx2 = log.startTransaction() tx2.commit(addA_P1.remove :: Nil, ManualUpdate) intercept[ConcurrentDeleteReadException] { // tx1 reads the whole table but tx2 removes files before tx1 commits tx1.commit(addB_P1 :: Nil, ManualUpdate) } } } def withLog( actions: Seq[Action], partitionCols: Seq[String] = "part" :: Nil)( test: DeltaLog => Unit): Unit = { val schema = StructType(partitionCols.map(p => StructField(p, StringType)).toArray) val metadata = Metadata(partitionColumns = partitionCols, schemaString = schema.json) val actionWithMetaData = actions :+ metadata withTempDir { tempDir => val log = DeltaLog.forTable(spark, tempDir) // Initialize the log and add data. ManualUpdate is just a no-op placeholder. log.startTransaction().commit(Seq(metadata), ManualUpdate) log.startTransaction().commitManually(actionWithMetaData: _*) test(log) } } test("allow concurrent set-txns with different app ids") { withTempDir { tempDir => val log = DeltaLog.forTable(spark, tempDir) // Initialize the log. log.startTransaction().commitManually() val txn = log.startTransaction() txn.txnVersion("t1") val winningTxn = log.startTransaction() winningTxn.commit(SetTransaction("t2", 1, Some(1234L)) :: Nil, ManualUpdate) txn.commit(Nil, ManualUpdate) assert(log.update().transactions === Map("t2" -> 1)) } } test("block concurrent set-txns with the same app id") { withTempDir { tempDir => val log = DeltaLog.forTable(spark, tempDir) // Initialize the log. log.startTransaction().commitManually() val txn = log.startTransaction() txn.txnVersion("t1") val winningTxn = log.startTransaction() winningTxn.commit(SetTransaction("t1", 1, Some(1234L)) :: Nil, ManualUpdate) intercept[ConcurrentTransactionException] { txn.commit(Nil, ManualUpdate) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/OptimisticTransactionSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.io.File import java.nio.file.FileAlreadyExistsException import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta.DeltaOperations.{ManualUpdate, Truncate} import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile import org.apache.spark.sql.delta.actions.{Action, AddFile, CommitInfo, Metadata, Protocol, RemoveFile, SetTransaction} import org.apache.spark.sql.delta.coordinatedcommits.{CommitCoordinatorBuilder, CommitCoordinatorProvider, InMemoryCommitCoordinator, InMemoryCommitCoordinatorBuilder, TableCommitCoordinatorClient} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import io.delta.storage.LogStore import io.delta.storage.commit.{CommitCoordinatorClient, CommitFailedException, CommitResponse, TableDescriptor, UpdatedActions} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.sql.{Row, SaveMode, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.expressions.{EqualTo, Literal} import org.apache.spark.sql.functions.{col, lit} import org.apache.spark.sql.types.{IntegerType, StructType} import org.apache.spark.util.ManualClock class OptimisticTransactionSuite extends OptimisticTransactionLegacyTests with OptimisticTransactionSuiteBase { import testImplicits._ // scalastyle:off: removeFile private val addA = createTestAddFile(encodedPath = "a") private val addB = createTestAddFile(encodedPath = "b") /* ************************** * * Allowed concurrent actions * * ************************** */ check( "append / append", conflicts = false, reads = Seq( t => t.metadata ), concurrentWrites = Seq( addA), actions = Seq( addB)) check( "disjoint txns", conflicts = false, reads = Seq( t => t.txnVersion("t1") ), concurrentWrites = Seq( SetTransaction("t2", 0, Some(1234L))), actions = Nil) check( "disjoint delete / read", conflicts = false, setup = Seq( Metadata( schemaString = new StructType().add("x", IntegerType).json, partitionColumns = Seq("x")), AddFile("a", Map("x" -> "2"), 1, 1, dataChange = true) ), reads = Seq( t => t.filterFiles(EqualTo('x, Literal(1)) :: Nil) ), concurrentWrites = Seq( RemoveFile("a", Some(4))), actions = Seq()) check( "disjoint add / read", conflicts = false, setup = Seq( Metadata( schemaString = new StructType().add("x", IntegerType).json, partitionColumns = Seq("x")) ), reads = Seq( t => t.filterFiles(EqualTo('x, Literal(1)) :: Nil) ), concurrentWrites = Seq( AddFile("a", Map("x" -> "2"), 1, 1, dataChange = true)), actions = Seq()) /* ***************************** * * Disallowed concurrent actions * * ***************************** */ check( "delete / delete", conflicts = true, reads = Nil, concurrentWrites = Seq( RemoveFile("a", Some(4))), actions = Seq( RemoveFile("a", Some(5)))) check( "add / read + write", conflicts = true, setup = Seq( Metadata( schemaString = new StructType().add("x", IntegerType).json, partitionColumns = Seq("x")) ), reads = Seq( t => t.filterFiles(EqualTo('x, Literal(1)) :: Nil) ), concurrentWrites = Seq( AddFile("a", Map("x" -> "1"), 1, 1, dataChange = true)), actions = Seq(AddFile("b", Map("x" -> "1"), 1, 1, dataChange = true)), // commit info should show operation as truncate, because that's the operation used by the // harness expectedErrorClass = Some("DELTA_CONCURRENT_APPEND.WITH_PARTITION_HINT"), expectedErrorMessageParameters = Some(Map( "operation" -> "TRUNCATE", "version" -> "1", "partitionValues" -> "\\[x=1\\]", "docLink" -> ".*" ))) check( "add / read + no write", // no write = no real conflicting change even though data was added conflicts = false, // so this should not conflict setup = Seq( Metadata( schemaString = new StructType().add("x", IntegerType).json, partitionColumns = Seq("x")) ), reads = Seq( t => t.filterFiles(EqualTo('x, Literal(1)) :: Nil) ), concurrentWrites = Seq( AddFile("a", Map("x" -> "1"), 1, 1, dataChange = true)), actions = Seq()) check( "add in part=2 / read from part=1,2 and write to part=1", conflicts = true, setup = Seq( Metadata( schemaString = new StructType().add("x", IntegerType).json, partitionColumns = Seq("x")) ), reads = Seq( t => { // Filter files twice - once for x=1 and again for x=2 t.filterFiles(Seq(EqualTo('x, Literal(1)))) t.filterFiles(Seq(EqualTo('x, Literal(2)))) } ), concurrentWrites = Seq( AddFile( path = "a", partitionValues = Map("x" -> "1"), size = 1, modificationTime = 1, dataChange = true) ), actions = Seq( AddFile( path = "b", partitionValues = Map("x" -> "2"), size = 1, modificationTime = 1, dataChange = true) )) check( "delete / read", conflicts = true, setup = Seq( Metadata( schemaString = new StructType().add("x", IntegerType).json, partitionColumns = Seq("x")), AddFile("a", Map("x" -> "1"), 1, 1, dataChange = true) ), reads = Seq( t => t.filterFiles(EqualTo('x, Literal(1)) :: Nil) ), concurrentWrites = Seq( RemoveFile("a", Some(4))), actions = Seq(), expectedErrorClass = Some("DELTA_CONCURRENT_DELETE_READ.WITH_PARTITION_HINT"), expectedErrorMessageParameters = Some(Map( "operation" -> "TRUNCATE", "version" -> "2", "partitionValues" -> "\\[x=1\\]", "docLink" -> ".*" ))) check( "schema change", conflicts = true, reads = Seq( t => t.metadata ), concurrentWrites = Seq( Metadata()), actions = Nil) check( "conflicting txns", conflicts = true, reads = Seq( t => t.txnVersion("t1") ), concurrentWrites = Seq( SetTransaction("t1", 0, Some(1234L))), actions = Nil) check( "upgrade / upgrade", conflicts = true, reads = Seq( t => t.metadata ), concurrentWrites = Seq( Action.supportedProtocolVersion(featuresToExclude = Seq(CatalogOwnedTableFeature))), actions = Seq( Action.supportedProtocolVersion(featuresToExclude = Seq(CatalogOwnedTableFeature)))) check( "taint whole table", conflicts = true, setup = Seq( Metadata( schemaString = new StructType().add("x", IntegerType).json, partitionColumns = Seq("x")), AddFile("a", Map("x" -> "2"), 1, 1, dataChange = true) ), reads = Seq( t => t.filterFiles(EqualTo('x, Literal(1)) :: Nil), // `readWholeTable` should disallow any concurrent change, even if the change // is disjoint with the earlier filter t => t.readWholeTable() ), concurrentWrites = Seq( AddFile("b", Map("x" -> "3"), 1, 1, dataChange = true)), actions = Seq( AddFile("c", Map("x" -> "4"), 1, 1, dataChange = true))) check( "taint whole table + concurrent remove", conflicts = true, setup = Seq( Metadata(schemaString = new StructType().add("x", IntegerType).json), AddFile("a", Map.empty, 1, 1, dataChange = true) ), reads = Seq( // `readWholeTable` should disallow any concurrent `RemoveFile`s. t => t.readWholeTable() ), concurrentWrites = Seq( RemoveFile("a", Some(4L))), actions = Seq( AddFile("b", Map.empty, 1, 1, dataChange = true))) override def beforeEach(): Unit = { super.beforeEach() CommitCoordinatorProvider.clearNonDefaultBuilders() } test("initial commit without metadata should fail") { withTempDir { tempDir => val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) val txn = log.startTransaction() withSQLConf(DeltaSQLConf.DELTA_COMMIT_VALIDATION_ENABLED.key -> "true") { val e = intercept[DeltaIllegalStateException] { txn.commit(Nil, ManualUpdate) } assert(e.getMessage == DeltaErrors.metadataAbsentException().getMessage) } } } test("initial commit with multiple metadata actions should fail") { withTempDir { tempDir => val log = DeltaLog.forTable(spark, new Path(tempDir.getAbsolutePath)) val txn = log.startTransaction() val e = intercept[AssertionError] { txn.commit(Seq(Metadata(), Metadata()), ManualUpdate) } assert(e.getMessage.contains("Cannot change the metadata more than once in a transaction.")) } } test("enabling Coordinated Commits on an existing table should create commit dir") { withTempDir { tempDir => val log = DeltaLog.forTable(spark, new Path(tempDir.getAbsolutePath)) val metadata = Metadata() log.startTransaction().commit(Seq(metadata), ManualUpdate) val fs = log.logPath.getFileSystem(log.newDeltaHadoopConf()) val commitDir = FileNames.commitDirPath(log.logPath) // Delete commit directory. fs.delete(commitDir) assert(!fs.exists(commitDir)) // With no Coordinated Commits conf, commit directory should not be created. log.startTransaction().commit(Seq(metadata), ManualUpdate) assert(!fs.exists(commitDir)) // Enabling Coordinated Commits on an existing table should create the commit dir. CommitCoordinatorProvider.registerBuilder(InMemoryCommitCoordinatorBuilder(3)) val newMetadata = metadata.copy(configuration = (metadata.configuration ++ Map(DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.key -> "in-memory")).toMap) log.startTransaction().commit(Seq(newMetadata), ManualUpdate) assert(fs.exists(commitDir)) log.update().ensureCommitFilesBackfilled() // With no new Coordinated Commits conf, commit directory should not be created and so the // transaction should fail because of corrupted dir. fs.delete(commitDir) assert(!fs.exists(commitDir)) intercept[java.io.FileNotFoundException] { log.startTransaction().commit(Seq(newMetadata), ManualUpdate) } } } test("concurrent feature enablement with failConcurrentTransactionsAtUpgrade should conflict") { withSQLConf( DeltaSQLConf.DELTA_CONFLICT_CHECKER_ENFORCE_FEATURE_ENABLEMENT_VALIDATION.key -> "true") { withTempDir { tempDir => val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) val metadata = Metadata( schemaString = new StructType().add("x", IntegerType).json, partitionColumns = Seq("x")) val protocol = Protocol(3, 7) log.startTransaction().commit(Seq(metadata, protocol), ManualUpdate) // Start a transaction that will write to the table concurrently with feature enablement. val txn = log.startTransaction() // Concurrently, enable the MaterializePartitionColumns feature // This feature has failConcurrentTransactionsAtUpgrade = true val newProtocol = txn.snapshot.protocol.withFeatures( Set(MaterializePartitionColumnsTableFeature)) log.startTransaction().commit(Seq(newProtocol), ManualUpdate) // The original transaction should fail when trying to commit // because a feature with failConcurrentTransactionsAtUpgrade=true was added val e = intercept[io.delta.exceptions.ProtocolChangedException] { txn.commit( Seq(AddFile("test", Map("x" -> "1"), 1, 1, dataChange = true)), ManualUpdate) } // Verify the error message assert(e.getMessage.contains("The protocol version of the Delta table has been changed")) } } } test("AddFile with different partition schema compared to metadata should fail") { withTempDir { tempDir => val log = DeltaLog.forTable(spark, new Path(tempDir.getAbsolutePath)) val initTxn = log.startTransaction() initTxn.updateMetadataForNewTable(Metadata( schemaString = StructType.fromDDL("col2 string, a int").json, partitionColumns = Seq("col2"))) initTxn.commit(Seq(), ManualUpdate) withSQLConf(DeltaSQLConf.DELTA_COMMIT_VALIDATION_ENABLED.key -> "true") { val e = intercept[IllegalStateException] { log.startTransaction().commit(Seq(AddFile( log.dataPath.toString, Map("col3" -> "1"), 12322, 0L, true, null, null)), ManualUpdate) } assert(e.getMessage == DeltaErrors.addFilePartitioningMismatchException( Seq("col3"), Seq("col2")).getMessage) } } } test("isolation level shouldn't be null") { withTempDir { tempDir => val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) log.startTransaction().commit(Seq(Metadata()), ManualUpdate) val txn = log.startTransaction() txn.commit(addA :: Nil, ManualUpdate) val isolationLevels = log.history.getHistory(Some(10)).map(_.isolationLevel) assert(isolationLevels.size == 2) assert(isolationLevels(0).exists(_.contains("Serializable"))) assert(isolationLevels(0).exists(_.contains("Serializable"))) } } test("every transaction should use a unique identifier in the commit") { withTempDir { tempDir => // Initialize delta table. val clock = new ManualClock() val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath), clock) log.startTransaction().commit(Seq(Metadata()), ManualUpdate) clock.advance(100) // Start two transactions which commits at same time with same content. val txn1 = log.startTransaction() val txn2 = log.startTransaction() clock.advance(100) val version1 = txn1.commit(Seq(), ManualUpdate) val version2 = txn2.commit(Seq(), ManualUpdate) // Validate that actions in both transactions are not exactly same. def readActions(version: Long): Seq[Action] = { log.store.read(FileNames.unsafeDeltaFile(log.logPath, version), log.newDeltaHadoopConf()) .map(Action.fromJson) } def removeTxnIdAndMetricsFromActions(actions: Seq[Action]): Seq[Action] = actions.map { case c: CommitInfo => c.copy(txnId = None, operationMetrics = None) case other => other } val actions1 = readActions(version1) val actions2 = readActions(version2) val actionsWithoutTxnId1 = removeTxnIdAndMetricsFromActions(actions1) val actionsWithoutTxnId2 = removeTxnIdAndMetricsFromActions(actions2) assert(actions1 !== actions2) // Without the txn id, the actions are same as of today but they need not be in future. In // future we might have other fields which may make these actions from two different // transactions different. In that case, the below assertion can be removed. assert(actionsWithoutTxnId1 === actionsWithoutTxnId2) } } test("pre-command actions committed") { withTempDir { tempDir => // Initialize delta table. val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) log.startTransaction().commit(Seq(Metadata()), ManualUpdate) val txn = log.startTransaction() txn.updateSetTransaction("TestAppId", 1L, None) val version = txn.commit(Seq(), ManualUpdate) def readActions(version: Long): Seq[Action] = { log.store.read(FileNames.unsafeDeltaFile(log.logPath, version), log.newDeltaHadoopConf()) .map(Action.fromJson) } val actions = readActions(version) assert(actions.collectFirst { case SetTransaction("TestAppId", 1L, _) => }.isDefined) } } test("has SetTransaction version conflicts") { withTempDir { tempDir => // Initialize delta table. val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) log.startTransaction().commit(Seq(Metadata()), ManualUpdate) val txn = log.startTransaction() txn.updateSetTransaction("TestAppId", 1L, None) val e = intercept[IllegalArgumentException] { txn.commit(Seq(SetTransaction("TestAppId", 2L, None)), ManualUpdate) } assert(e.getMessage == DeltaErrors.setTransactionVersionConflict("TestAppId", 2L, 1L) .getMessage) } } test("conflict event logs winningTxnOperation for observability") { withTempDir { tempDir => val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) // Initialize delta table with partitioned schema. val metadata = Metadata( schemaString = new StructType().add("x", IntegerType).json, partitionColumns = Seq("x"), // Set isolation level to SERIALIZABLE to ensure conflict detection. configuration = Map(DeltaConfigs.ISOLATION_LEVEL.key -> Serializable.toString)) log.startTransaction().commit(Seq(metadata), ManualUpdate) // Start a transaction that reads partition x=1. val txn = log.startTransaction() txn.filterFiles(EqualTo('x, Literal(1)) :: Nil) // Commit a concurrent write to the same partition that will cause conflict. log.startTransaction().commit( Seq(AddFile("a", Map("x" -> "1"), 1, 1, dataChange = true)), Truncate()) // Attempt to write to the same partition - should conflict. val conflictLogs = Log4jUsageLogger.track { intercept[DeltaConcurrentModificationException] { txn.commit( Seq(AddFile("b", Map("x" -> "1"), 1, 1, dataChange = true)), Truncate()) } }.filter(usageLog => usageLog.metric == "tahoeEvent" && usageLog.tags.getOrElse("opType", "").startsWith("delta.commit.conflict")) // Verify the conflict event is logged with winningTxnOperation. assert(conflictLogs.size == 1, "Expected exactly one conflict event to be logged") val conflictBlob = JsonUtils.fromJson[Map[String, Any]](conflictLogs.head.blob) assert(conflictBlob.get("winningTxnOperation") .exists(_.toString == "TRUNCATE")) } } test("removes duplicate SetTransactions") { withTempDir { tempDir => // Initialize delta table. val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) log.startTransaction().commit(Seq(Metadata()), ManualUpdate) val txn = log.startTransaction() txn.updateSetTransaction("TestAppId", 1L, None) val version = txn.commit(Seq(SetTransaction("TestAppId", 1L, None)), ManualUpdate) def readActions(version: Long): Seq[Action] = { log.store.read(FileNames.unsafeDeltaFile(log.logPath, version), log.newDeltaHadoopConf()) .map(Action.fromJson) } assert(readActions(version).collectFirst { case SetTransaction("TestAppId", 1L, _) => }.isDefined) } } test("preCommitLogSegment is updated during conflict checking") { withTempDir { tempDir => val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) log.startTransaction().commit(Seq(Metadata()), ManualUpdate) sql(s"ALTER TABLE delta.`${tempDir.getAbsolutePath}` " + s"SET TBLPROPERTIES (${DeltaConfigs.CHECKPOINT_INTERVAL.key} = 10)") val testTxn = log.startTransaction() val testTxnStartTs = System.currentTimeMillis() for (_ <- 1 to 11) { log.startTransaction().commit(Seq.empty, ManualUpdate) } val testTxnEndTs = System.currentTimeMillis() // preCommitLogSegment should not get updated until a commit is triggered assert(testTxn.preCommitLogSegment.version == 1) assert(testTxn.preCommitLogSegment.lastCommitFileModificationTimestamp < testTxnStartTs) assert(testTxn.preCommitLogSegment.deltas.size == 2) assert(testTxn.preCommitLogSegment.checkpointProvider.isEmpty) testTxn.commit(Seq.empty, ManualUpdate) // preCommitLogSegment should get updated to the version right before the txn commits assert(testTxn.preCommitLogSegment.version == 12) assert(testTxn.preCommitLogSegment.lastCommitFileModificationTimestamp < testTxnEndTs) assert(testTxn.preCommitLogSegment.deltas.size == 2) assert(testTxn.preCommitLogSegment.checkpointProvider.version == 10) } } test("Limited retries for non-conflict retryable CommitFailedExceptions") { val commitCoordinatorName = "retryable-non-conflict-commit-coordinator" var commitAttempts = 0 val numRetries = "100" val numNonConflictRetries = "10" val initialNonConflictErrors = 5 val initialConflictErrors = 5 object RetryableNonConflictCommitCoordinatorBuilder$ extends CommitCoordinatorBuilder { override def getName: String = commitCoordinatorName val commitCoordinatorClient: InMemoryCommitCoordinator = { new InMemoryCommitCoordinator(batchSize = 1000L) { override def commit( logStore: LogStore, hadoopConf: Configuration, tableDesc: TableDescriptor, commitVersion: Long, actions: java.util.Iterator[String], updatedActions: UpdatedActions): CommitResponse = { // Fail all commits except first one if (commitVersion == 0) { return super.commit( logStore, hadoopConf, tableDesc, commitVersion, actions, updatedActions) } commitAttempts += 1 throw new CommitFailedException( true, commitAttempts > initialNonConflictErrors && commitAttempts <= (initialNonConflictErrors + initialConflictErrors), "") } } } override def build( spark: SparkSession, conf: Map[String, String]): CommitCoordinatorClient = commitCoordinatorClient } CommitCoordinatorProvider.registerBuilder(RetryableNonConflictCommitCoordinatorBuilder$) withSQLConf( DeltaSQLConf.DELTA_MAX_RETRY_COMMIT_ATTEMPTS.key -> numRetries, DeltaSQLConf.DELTA_MAX_NON_CONFLICT_RETRY_COMMIT_ATTEMPTS.key -> numNonConflictRetries) { withTempDir { tempDir => val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) val conf = Map(DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.key -> commitCoordinatorName) log.startTransaction().commit(Seq(Metadata(configuration = conf)), ManualUpdate) val testTxn = log.startTransaction() intercept[CommitFailedException] { testTxn.commit(Seq.empty, ManualUpdate) } // num-attempts = 1 + num-retries assert(commitAttempts == (initialNonConflictErrors + initialConflictErrors + numNonConflictRetries.toInt + 1)) } } } test("No retries for FileAlreadyExistsException with commit-coordinator") { val commitCoordinatorName = "file-already-exists-commit-coordinator" var commitAttempts = 0 object FileAlreadyExistsCommitCoordinatorBuilder extends CommitCoordinatorBuilder { override def getName: String = commitCoordinatorName lazy val commitCoordinatorClient: CommitCoordinatorClient = { new InMemoryCommitCoordinator(batchSize = 1000L) { override def commit( logStore: LogStore, hadoopConf: Configuration, tableDesc: TableDescriptor, commitVersion: Long, actions: java.util.Iterator[String], updatedActions: UpdatedActions): CommitResponse = { // Fail all commits except first one if (commitVersion == 0) { return super.commit( logStore, hadoopConf, tableDesc, commitVersion, actions, updatedActions) } commitAttempts += 1 throw new FileAlreadyExistsException("Commit-File Already Exists") } } } override def build( spark: SparkSession, conf: Map[String, String]): CommitCoordinatorClient = commitCoordinatorClient } CommitCoordinatorProvider.registerBuilder(FileAlreadyExistsCommitCoordinatorBuilder) withSQLConf( DeltaSQLConf.DELTA_MAX_RETRY_COMMIT_ATTEMPTS.key -> "100", DeltaSQLConf.DELTA_MAX_NON_CONFLICT_RETRY_COMMIT_ATTEMPTS.key -> "10") { withTempDir { tempDir => val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) val conf = Map(DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.key -> commitCoordinatorName) log.startTransaction().commit(Seq(Metadata(configuration = conf)), ManualUpdate) val testTxn = log.startTransaction() intercept[FileAlreadyExistsException] { testTxn.commit(Seq.empty, ManualUpdate) } // Test that there are no retries for the FileAlreadyExistsException in // CommitCoordinatorClient.commit() // num-attempts(1) = 1 + num-retries(0) assert(commitAttempts == 1) } } } /** * Here we test whether ConflictChecker correctly resolves conflicts when using * OptimisticTransaction.filterFiles(partitions) to perform dynamic partition overwrites. * */ private def testDynamicPartitionOverwrite( caseName: String, concurrentActions: String => Seq[Action], expectedErrorClass: Option[String] = None, expectedErrorMessageParameters: String => Map[String, String] = _ => Map.empty) = { // We test with a partition column named "partitionValues" to make sure we correctly skip // rewriting the filters for (partCol <- Seq("part", "partitionValues")) { test("filterFiles(partitions) correctly updates readPredicates and ConflictChecker " + s"correctly detects conflicts for $caseName with partition column [$partCol]") { withTempDir { tempDir => val tablePath = tempDir.getCanonicalPath val log = DeltaLog.forTable(spark, tablePath) // set up log.startTransaction.commit(Seq( Metadata( schemaString = new StructType() .add(partCol, IntegerType) .add("value", IntegerType).json, partitionColumns = Seq(partCol)) ), ManualUpdate) log.startTransaction.commit( Seq(AddFile("a", Map(partCol -> "0"), 1, 1, dataChange = true), AddFile("b", Map(partCol -> "1"), 1, 1, dataChange = true)), ManualUpdate) // new data we want to overwrite dynamically to the table val newData = Seq(AddFile("x", Map(partCol -> "0"), 1, 1, dataChange = true)) // txn1: read files in partitions of our new data (part=0) val txn = log.startTransaction() val addFiles = txn.filterFiles(newData.map(_.partitionValues).toSet) // txn2 log.startTransaction().commit(concurrentActions(partCol), ManualUpdate) // txn1: remove files read in the partition and commit newData def commitTxn1 = { txn.commit(addFiles.map(_.remove) ++ newData, ManualUpdate) } if (expectedErrorClass.isDefined) { val e = intercept[DeltaConcurrentModificationException] { commitTxn1 } checkError( e.asInstanceOf[DeltaThrowable], expectedErrorClass.get, Some("2D521"), expectedErrorMessageParameters(partCol) ++ Map("tableName" -> s"delta.`${log.dataPath}`"), matchPVals = true ) } else { commitTxn1 } } } } } testDynamicPartitionOverwrite( caseName = "concurrent append in same partition", concurrentActions = partCol => Seq(AddFile("y", Map(partCol -> "0"), 1, 1, dataChange = true)), expectedErrorClass = Some("DELTA_CONCURRENT_APPEND.WITH_PARTITION_HINT"), expectedErrorMessageParameters = partCol => Map( "operation" -> "Manual Update", "partitionValues" -> s"\\[$partCol=0\\]", "version" -> "2", "docLink" -> ".*" ) ) testDynamicPartitionOverwrite( caseName = "concurrent append in different partition", concurrentActions = partCol => Seq(AddFile("y", Map(partCol -> "1"), 1, 1, dataChange = true)) ) testDynamicPartitionOverwrite( caseName = "concurrent delete in same partition", concurrentActions = partCol => Seq( RemoveFile("a", None, partitionValues = Map(partCol -> "0"))), expectedErrorClass = Some("DELTA_CONCURRENT_DELETE_DELETE.WITH_PARTITION_HINT"), expectedErrorMessageParameters = partCol => Map( "operation" -> "Manual Update", "partitionValues" -> s"\\[$partCol=0\\]", "version" -> "2", "docLink" -> ".*" ) ) testDynamicPartitionOverwrite( caseName = "concurrent delete in different partition", concurrentActions = partCol => Seq( RemoveFile("b", None, partitionValues = Map(partCol -> "1"))) ) test("can set partition columns in first commit") { withTempDir { tableDir => val partitionColumns = Array("part") val exampleAddFile = AddFile( path = "test-path", partitionValues = Map("part" -> "one"), size = 1234, modificationTime = 5678, dataChange = true, stats = """{"numRecords": 1}""", tags = Map.empty) val deltaLog = DeltaLog.forTable(spark, tableDir) val schema = new StructType() .add("id", "long") .add("part", "string") deltaLog.withNewTransaction { txn => val protocol = Action.supportedProtocolVersion( featuresToExclude = Seq(CatalogOwnedTableFeature)) val metadata = Metadata( schemaString = schema.json, partitionColumns = partitionColumns) txn.commit(Seq(protocol, metadata, exampleAddFile), DeltaOperations.ManualUpdate) } val snapshot = deltaLog.update() assert(snapshot.metadata.partitionColumns.sameElements(partitionColumns)) } } test("only single Protocol action per commit - implicit") { withTempDir { tableDir => val deltaLog = DeltaLog.forTable(spark, tableDir) val schema = new StructType() .add("id", "long") .add("col", "string") val e = intercept[java.lang.AssertionError] { deltaLog.withNewTransaction { txn => val protocol = Protocol(2, 3) val metadata = Metadata( schemaString = schema.json, configuration = Map("delta.enableChangeDataFeed" -> "true")) txn.commit(Seq(protocol, metadata), DeltaOperations.ManualUpdate) } } assert(e.getMessage.contains( "assertion failed: Cannot change the protocol more than once in a transaction.")) } } test("only single Protocol action per commit - explicit") { withTempDir { tableDir => val deltaLog = DeltaLog.forTable(spark, tableDir) val e = intercept[java.lang.AssertionError] { deltaLog.withNewTransaction { txn => val protocol1 = Protocol(2, 3) val protocol2 = Protocol(1, 4) txn.commit(Seq(protocol1, protocol2), DeltaOperations.ManualUpdate) } } assert(e.getMessage.contains( "assertion failed: Cannot change the protocol more than once in a transaction.")) } } test("DVs cannot be added to files without numRecords stat") { withTempPath { tempPath => val path = tempPath.getPath val deltaLog = DeltaLog.forTable(spark, path) val firstFile = writeDuplicateActionsData(path).head enableDeletionVectorsInTable(deltaLog) val (addFileWithDV, removeFile) = addDVToFileInTable(path, firstFile) val addFileWithDVWithoutStats = addFileWithDV.copy(stats = null) testRuntimeErrorOnCommit(Seq(addFileWithDVWithoutStats, removeFile), deltaLog) { e => val expErrorClass = "DELTA_DELETION_VECTOR_MISSING_NUM_RECORDS" assert(e.getErrorClass == expErrorClass) assert(e.getSqlState == "2D521") } } } test("commitInfo tags") { withTempDir { tableDir => val deltaLog = DeltaLog.forTable(spark, tableDir) val schema = new StructType().add("id", "long") def checkLastCommitTags(expectedTags: Option[Map[String, String]]): Unit = { val ci = deltaLog.getChanges(deltaLog.update().version).map(_._2).flatten.collectFirst { case ci: CommitInfo => ci }.head assert(ci.tags === expectedTags) } val metadata = Metadata(schemaString = schema.json) // Check empty tags deltaLog.withNewTransaction { txn => txn.commit(metadata :: Nil, DeltaOperations.ManualUpdate, tags = Map.empty) } checkLastCommitTags(expectedTags = None) deltaLog.withNewTransaction { txn => txn.commit(addA :: Nil, DeltaOperations.Write(SaveMode.Append), tags = Map.empty) } checkLastCommitTags(expectedTags = None) // Check non-empty tags val tags1 = Map("testTag1" -> "testValue1") deltaLog.withNewTransaction { txn => txn.commit(metadata :: Nil, DeltaOperations.ManualUpdate, tags = tags1) } checkLastCommitTags(expectedTags = Some(tags1)) val tags2 = Map("testTag1" -> "testValue1", "testTag2" -> "testValue2") deltaLog.withNewTransaction { txn => txn.commit(addB :: Nil, DeltaOperations.Write(SaveMode.Append), tags = tags2) } checkLastCommitTags(expectedTags = Some(tags2)) } } test("empty commits are elided on write by default") { withTempDir { tableDir => val df = Seq((1, 0), (2, 1)).toDF("key", "value") df.write.format("delta").mode("append").save(tableDir.getCanonicalPath) val deltaLog = DeltaLog.forTable(spark, tableDir) val expectedSnapshot = deltaLog.update() val expectedDeltaVersion = expectedSnapshot.version val emptyDf = Seq.empty[(Integer, Integer)].toDF("key", "value") emptyDf.write.format("delta").mode("append").save(tableDir.getCanonicalPath) val actualSnapshot = deltaLog.update() val actualDeltaVersion = actualSnapshot.version checkAnswer(spark.read.format("delta").load(tableDir.getCanonicalPath), Row(1, 0) :: Row(2, 1) :: Nil) assert(expectedDeltaVersion === actualDeltaVersion) } } Seq(true, false).foreach { skip => test(s"Elide empty commits when requested - skipRecordingEmptyCommits=$skip") { withSQLConf(DeltaSQLConf.DELTA_SKIP_RECORDING_EMPTY_COMMITS.key -> skip.toString) { withTempDir { tableDir => val df = Seq((1, 0), (2, 1)).toDF("key", "value") df.write.format("delta").mode("append").save(tableDir.getCanonicalPath) val deltaLog = DeltaLog.forTable(spark, tableDir) val expectedSnapshot = deltaLog.update() val expectedDeltaVersion = if (skip) { expectedSnapshot.version } else { expectedSnapshot.version + 1 } val emptyDf = Seq.empty[(Integer, Integer)].toDF("key", "value") emptyDf.write.format("delta").mode("append").save(tableDir.getCanonicalPath) val actualSnapshot = deltaLog.update() val actualDeltaVersion = actualSnapshot.version checkAnswer(spark.read.format("delta").load(tableDir.getCanonicalPath), Row(1, 0) :: Row(2, 1) :: Nil) assert(expectedDeltaVersion === actualDeltaVersion) } } } } BOOLEAN_DOMAIN.foreach { conflict => test(s"commitLarge should handle Commit Failed Exception with conflict: $conflict") { withTempDir { tempDir => val deltaLog = DeltaLog.forTable(spark, tempDir.getAbsolutePath) val commitCoordinatorName = "retryable-conflict-commit-coordinator" class RetryableConflictCommitCoordinatorClient extends InMemoryCommitCoordinator(batchSize = 5) { override def commit( logStore: LogStore, hadoopConf: Configuration, tableDesc: TableDescriptor, commitVersion: Long, actions: java.util.Iterator[String], updatedActions: UpdatedActions): CommitResponse = { if (updatedActions.getCommitInfo.asInstanceOf[CommitInfo].operation == DeltaOperations.OP_RESTORE) { deltaLog.startTransaction().commit(addB :: Nil, ManualUpdate) throw new CommitFailedException(true, conflict, "") } super.commit(logStore, hadoopConf, tableDesc, commitVersion, actions, updatedActions) } } object RetryableConflictCommitCoordinatorBuilder$ extends CommitCoordinatorBuilder { lazy val commitCoordinatorClient = new RetryableConflictCommitCoordinatorClient() override def getName: String = commitCoordinatorName override def build( spark: SparkSession, conf: Map[String, String]): CommitCoordinatorClient = commitCoordinatorClient } CommitCoordinatorProvider.registerBuilder(RetryableConflictCommitCoordinatorBuilder$) val conf = Map(DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.key -> commitCoordinatorName) deltaLog.startTransaction().commit(Seq(Metadata(configuration = conf)), ManualUpdate) deltaLog.startTransaction().commit(addA :: Nil, ManualUpdate) val records = Log4jUsageLogger.track { // commitLarge must fail because of a conflicting commit at version-2. val e = intercept[Exception] { deltaLog.startTransaction().commitLarge( spark, nonProtocolMetadataActions = (addB :: Nil).iterator, newProtocolOpt = None, op = DeltaOperations.Restore(Some(0), None), context = Map.empty, metrics = Map.empty) } if (conflict) { assert(e.isInstanceOf[ConcurrentWriteException]) assert( e.getMessage.contains( "A concurrent transaction has written new data since the current transaction " + s"read the table. Please try the operation again")) } else { assert(e.isInstanceOf[CommitFailedException]) } assert(deltaLog.update().version == 2) } val failureRecord = filterUsageRecords(records, "delta.commitLarge.failure") assert(failureRecord.size == 1) val data = JsonUtils.fromJson[Map[String, Any]](failureRecord.head.blob) assert(data("fromCoordinatedCommits") == true) assert(data("fromCoordinatedCommitsConflict") == conflict) assert(data("fromCoordinatedCommitsRetryable") == true) } } } test("Append does not trigger snapshot state computation") { withSQLConf( DeltaSQLConf.DELTA_WRITE_CHECKSUM_ENABLED.key -> "false", DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED.key -> "true", DeltaSQLConf.INCREMENTAL_COMMIT_FORCE_VERIFY_IN_TESTS.key -> "false", DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_VERIFICATION_MODE_ENABLED.key -> "false", DeltaSQLConf.DELTA_ALL_FILES_IN_CRC_FORCE_VERIFICATION_MODE_FOR_NON_UTC_ENABLED.key -> "false", DeltaSQLConf.INCREMENTAL_COMMIT_FORCE_VERIFY_IN_TESTS.key -> "false" ) { withTempDir { tableDir => val df = Seq((1, 0), (2, 1)).toDF("key", "value") df.write.format("delta").mode("append").save(tableDir.getCanonicalPath) val deltaLog = DeltaLog.forTable(spark, tableDir) val preCommitSnapshot = deltaLog.update() assert(!preCommitSnapshot.stateReconstructionTriggered) df.write.format("delta").mode("append").save(tableDir.getCanonicalPath) val postCommitSnapshot = deltaLog.update() assert(!preCommitSnapshot.stateReconstructionTriggered) assert(!postCommitSnapshot.stateReconstructionTriggered) } } } test("partition column changes not thrown for valid CREATE/REPLACE operations") { withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath sql(s"""CREATE TABLE delta.`$tablePath` USING delta PARTITIONED BY (part) AS SELECT id, id % 3 as part FROM range(10)""") sql(s"""CREATE OR REPLACE TABLE delta.`$tablePath` USING delta PARTITIONED BY (newpart) AS SELECT id, id % 5 as newpart FROM range(10)""") sql(s"""REPLACE TABLE delta.`$tablePath` USING delta PARTITIONED BY (newpart) AS SELECT id, id % 5 as newpart FROM range(10)""") } } Seq(("path", true), ("catalog", false)).foreach { case (tableType, isPath) => Seq("dfv1", "dfv2", "sql").foreach { method => Seq("error", "overwrite") .filterNot(mode => method == "dfv2" && mode == "overwrite") .foreach { mode => test(s"partition column changes not thrown for $method $mode on new $tableType table") { withTempDir { tempDir => val pathOrTable = getPathOrTable(tempDir, isPath, s"${method}_${mode}_${tableType}") try { writePartitionedTable( pathOrTable, isPath, "part", 10, mode = mode, method = method) assertPartitionColumnsForTest(pathOrTable, isPath, Seq("part")) } finally { if (!isPath) sql(s"DROP TABLE IF EXISTS $pathOrTable") } } } } } } Seq(("path", true), ("catalog", false)).foreach { case (tableType, isPath) => Seq("sql", "dfv1", "dfv2").foreach { method => test(s"partition column changes not thrown for $method append on $tableType") { withTempDir { tempDir => val pathOrTable = getPathOrTable(tempDir, isPath, s"${method}_append_${tableType}") try { writePartitionedTable( pathOrTable, isPath, "part", 10, mode = "error", method = method) writePartitionedTable( pathOrTable, isPath, "part", 10, mode = "append", rangeStart = 10, rangeEnd = 20, method = method) assertPartitionColumnsForTest(pathOrTable, isPath, Seq("part")) } finally { if (!isPath) sql(s"DROP TABLE IF EXISTS $pathOrTable") } } } } } // Among others, this includes a test containing an overwrite using .saveAsTable() which // creates a ReplaceTable operation under the hood. Seq( ("path", true), ("catalog", false) ).foreach { case (tableType, isPath) => Seq("dfv1", "dfv2", "sql").foreach { method => Seq("overwrite", "createOrReplace").foreach { mode => // Skip unsupported combinations if (!(method == "dfv1" && mode == "createOrReplace")) { test(s"partition col changes allowed for $method $tableType $mode") { withTempDir { tempDir => val pathOrTable = if (isPath) { tempDir.getAbsolutePath } else { s"test_table_${method}_${tableType}_${mode}" } // Create initial table writeThreeColumnTable(pathOrTable, "col1") if (isPath) { assertPartitionColumns(pathOrTable, Seq("col1")) } else { assertPartitionColumns(new TableIdentifier(pathOrTable), Seq("col1")) } // Overwrite with different partition column writeThreeColumnTable(pathOrTable, "col2", method = method, mode = mode) val expectedAnswer = spark.range(10) .withColumn("col1", col("id") % 3).withColumn("col2", col("id") % 5) if (isPath) { assertPartitionColumns(pathOrTable, Seq("col2")) checkAnswer(spark.read.format("delta").load(pathOrTable), expectedAnswer) } else { assertPartitionColumns(new TableIdentifier(pathOrTable), Seq("col2")) checkAnswer(spark.table(pathOrTable), expectedAnswer) sql(s"DROP TABLE IF EXISTS $pathOrTable") } } } } } } } test("partition column changes validation modes for UPDATE operations") { val testCases = Seq( ("true", Some(classOf[DeltaAnalysisException]), true), ("log-only", None, true), ("false", None, false) ) testCases.foreach { case (mode, exceptionClassOpt, expectLogEvent) => withSQLConf(DeltaSQLConf.DELTA_PARTITION_COLUMN_CHANGE_CHECK.key -> mode) { withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath writeThreeColumnTable(tablePath, "col1") val deltaLog = DeltaLog.forTable(spark, tablePath) val txn = deltaLog.startTransaction() val newMetadata = txn.metadata.copy(partitionColumns = Seq("col2")) val logRecords = Log4jUsageLogger.track { exceptionClassOpt match { case Some(_) => checkError( intercept[DeltaAnalysisException] { txn.commit( Seq(newMetadata), DeltaOperations.Update(predicate = Some(EqualTo(Literal(1), Literal(1)))) ) }, condition = "DELTA_UNSUPPORTED_PARTITION_COLUMN_CHANGE", sqlState = "42P10", parameters = Map( "operation" -> "UPDATE", "oldPartitionColumns" -> "col1", "newPartitionColumns" -> "col2" ) ) case None => // Should succeed without throwing txn.commit( Seq(newMetadata), DeltaOperations.Update(predicate = Some(EqualTo(Literal(1), Literal(1)))) ) } } // Check for log event if expected if (expectLogEvent) { val matchingRecords = filterUsageRecords(logRecords, "delta.metadataCheck.illegalPartitionColumnChange") assert(matchingRecords.nonEmpty, "Expected to find log event 'delta.metadataCheck.illegalPartitionColumnChange' " + "but it was not logged") } // Verify final state only if commit succeeded if (exceptionClassOpt.isEmpty) { assertPartitionColumns(tablePath, Seq("col2")) } } } } } test("partition column changes validation default mode blocks UPDATE operations") { // Test that the default behavior (without explicit config) is to block partition column changes withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath writeThreeColumnTable(tablePath, "col1") val deltaLog = DeltaLog.forTable(spark, tablePath) val txn = deltaLog.startTransaction() val newMetadata = txn.metadata.copy(partitionColumns = Seq("col2")) checkError( intercept[DeltaAnalysisException] { txn.commit( Seq(newMetadata), DeltaOperations.Update(predicate = Some(EqualTo(Literal(1), Literal(1)))) ) }, condition = "DELTA_UNSUPPORTED_PARTITION_COLUMN_CHANGE", sqlState = "42P10", parameters = Map( "operation" -> "UPDATE", "oldPartitionColumns" -> "col1", "newPartitionColumns" -> "col2" ) ) assertPartitionColumns(tablePath, Seq("col1")) } } Seq( // Recreation of running DFv1 .save() overwrite changing partition cols. ("blocked for Write(Overwrite, partitionBy)", "WRITE", (_: Metadata) => DeltaOperations.Write(SaveMode.Overwrite, partitionBy = Some(Seq("col2")))), // Recreation of running DFv1 .save() replaceWhere changing partition cols. ("blocked for Write(Overwrite, partitionBy, predicate)", "WRITE", (_: Metadata) => DeltaOperations.Write(SaveMode.Overwrite, partitionBy = Some(Seq("col2")), predicate = Some("col1=0"))), // Recreation of running DFv1 .save() DPO changing partition cols. ("blocked for Write(Overwrite, partitionBy, DPO)", "WRITE", (_: Metadata) => DeltaOperations.Write(SaveMode.Overwrite, partitionBy = Some(Seq("col2")), isDynamicPartitionOverwrite = Some(true))), // Recreation of running DFv1 .saveAsTable() overwrite changing partition cols. ("blocked for ReplaceTable(isV1SaveAsTableOverwrite=true)", "CREATE OR REPLACE TABLE AS SELECT", (newMeta: Metadata) => DeltaOperations.ReplaceTable( metadata = newMeta, isManaged = true, orCreate = true, asSelect = true, isV1SaveAsTableOverwrite = Some(true))) ).foreach { case (testSuffix, expectedOpName, mkOp) => test(s"partition column changes $testSuffix") { withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath writeThreeColumnTable(tablePath, "col1") val deltaLog = DeltaLog.forTable(spark, tablePath) val txn = deltaLog.startTransaction() val newMetadata = txn.metadata.copy(partitionColumns = Seq("col2")) checkError( intercept[DeltaAnalysisException] { txn.commit(Seq(newMetadata), mkOp(newMetadata)) }, condition = "DELTA_UNSUPPORTED_PARTITION_COLUMN_CHANGE", sqlState = "42P10", parameters = Map( "operation" -> expectedOpName, "oldPartitionColumns" -> "col1", "newPartitionColumns" -> "col2" ) ) assertPartitionColumns(tablePath, Seq("col1")) } } } test("partition column changes allowed for CLONE operations") { withTempDir { sourceDir => withTempDir { targetDir => val sourcePath = sourceDir.getAbsolutePath val targetPath = targetDir.getAbsolutePath writePartitionedTable(sourcePath, isPath = true, "part", 10) // Test SHALLOW CLONE sql(s"CREATE TABLE delta.`$targetPath` SHALLOW CLONE delta.`$sourcePath`") assertPartitionColumns(targetPath, Seq("part")) } } } test("partition column changes allowed for RenameColumn when partition column renamed") { withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath writePartitionedTable(tablePath, isPath = true, "part", 10) sql(s"ALTER TABLE delta.`$tablePath` SET TBLPROPERTIES ('delta.columnMapping.mode' = 'name')") sql(s"ALTER TABLE delta.`$tablePath` RENAME COLUMN part TO renamed_part") assertPartitionColumns(tablePath, Seq("renamed_part")) } } Seq(("path", true), ("catalog", false)).foreach { case (tableType, isPath) => test(s"dfv1 overwrite without overwriteSchema blocks partition column changes - $tableType") { withTempDir { tempDir => val pathOrTable = getPathOrTable(tempDir, isPath, s"dfv1_overwrite_${tableType}") try { writeThreeColumnTable(pathOrTable, "col1") assertPartitionColumnsForTest(pathOrTable, isPath, Seq("col1")) val df = spark.range(10) .withColumn("col1", col("id") % 3) .withColumn("col2", col("id") % 5) val ex = intercept[DeltaAnalysisException] { val writer = df.write.format("delta").mode("overwrite").partitionBy("col2") if (isPath) writer.save(pathOrTable) else writer.saveAsTable(pathOrTable) } assert(ex.getMessage.contains("overwriteSchema") || ex.getMessage.contains("incompatible")) assertPartitionColumnsForTest(pathOrTable, isPath, Seq("col1")) } finally { if (!isPath) sql(s"DROP TABLE IF EXISTS $pathOrTable") } } } } // Helper methods private def writePartitionedTable( pathOrTable: String, isPath: Boolean, partitionCol: String, range: Int, mode: String = "error", rangeStart: Int = 0, rangeEnd: Int = -1, method: String = "dfv1"): Unit = { val end = if (rangeEnd == -1) range else rangeEnd val df = spark.range(rangeStart, end) .withColumn(partitionCol, col("id") % 3) val tableRef = if (isPath) s"delta.`$pathOrTable`" else pathOrTable val v2WriteDf = df.writeTo(tableRef).using("delta").partitionedBy(col(partitionCol)) method match { case "dfv1" => val writer = df.write.format("delta").mode(mode).partitionBy(partitionCol) if (isPath) { writer.save(pathOrTable) } else { writer.saveAsTable(pathOrTable) } case "dfv2" => mode match { case "error" | "errorifexists" => v2WriteDf.create() case "overwrite" => v2WriteDf.replace() case "append" => df.writeTo(tableRef).append() case "createOrReplace" => v2WriteDf.createOrReplace() } case "sql" => val tempView = s"temp_view_${System.nanoTime()}" df.createOrReplaceTempView(tempView) val stmt = mode match { case "append" => s"INSERT INTO $tableRef SELECT * FROM $tempView" case _ => s"""CREATE OR REPLACE TABLE $tableRef |USING delta PARTITIONED BY ($partitionCol) |AS SELECT * FROM $tempView""".stripMargin } sql(stmt) } } private def writeThreeColumnTable( pathOrTable: String, partitionCol: String, method: String = "dfv1", mode: String = "error"): Unit = { val df = spark.range(10) .withColumn("col1", col("id") % 3) .withColumn("col2", col("id") % 5) val isPath = pathOrTable.contains("/") method match { case "dfv1" => val writer = df.write.format("delta").partitionBy(partitionCol) if (mode == "overwrite") { writer.option("overwriteSchema", "true") } if (isPath) { writer.mode(mode).save(pathOrTable) } else { writer.mode(mode).saveAsTable(pathOrTable) } case "dfv2" => val tableRef = if (isPath) s"delta.`$pathOrTable`" else pathOrTable mode match { case "error" => df.writeTo(tableRef).using("delta").partitionedBy(col(partitionCol)).create() case "overwrite" => df.writeTo(tableRef).using("delta").partitionedBy(col(partitionCol)).replace() case "createOrReplace" => df.writeTo(tableRef).using("delta").partitionedBy(col(partitionCol)) .createOrReplace() } case "sql" => val tempView = s"temp_view_${System.nanoTime()}" df.createOrReplaceTempView(tempView) val tableRef = if (isPath) s"delta.`$pathOrTable`" else pathOrTable val stmt = mode match { case "append" => s"INSERT INTO $tableRef SELECT * FROM $tempView" case _ => s"""CREATE OR REPLACE TABLE $tableRef |USING delta PARTITIONED BY ($partitionCol) |AS SELECT * FROM $tempView""".stripMargin } sql(stmt) } } private def assertPartitionColumns(pathOrTableId: Any, expected: Seq[String]): Unit = { val deltaLog = pathOrTableId match { case path: String => DeltaLog.forTable(spark, path) case tableId: TableIdentifier => DeltaLog.forTable(spark, tableId) } assert(deltaLog.update().metadata.partitionColumns === expected) } // Helper to generate path or table name based on test context private def getPathOrTable( tempDir: File, isPath: Boolean, testSuffix: String): String = { if (isPath) { tempDir.getAbsolutePath } else { s"test_table_$testSuffix" } } // Helper to assert partition columns for either path or table private def assertPartitionColumnsForTest( pathOrTable: String, isPath: Boolean, expected: Seq[String]): Unit = { if (isPath) { assertPartitionColumns(pathOrTable, expected) } else { assertPartitionColumns(new TableIdentifier(pathOrTable), expected) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/OptimisticTransactionSuiteBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.ConcurrentModificationException import org.apache.spark.sql.delta.DeltaOperations.{ManualUpdate, Truncate} import org.apache.spark.sql.delta.actions.{Action, AddFile, FileAction, Metadata, RemoveFile} import org.apache.spark.sql.delta.deletionvectors.RoaringBitmapArray import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.hadoop.fs.Path import org.apache.spark.sql.QueryTest import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.Utils trait OptimisticTransactionSuiteBase extends QueryTest with SharedSparkSession with DeltaTestUtilsBase with DeletionVectorsTestUtils { /** * Check whether the test transaction conflict with the concurrent writes by executing the * given params in the following order: * - setup (including setting table isolation level * - reads * - concurrentWrites * - actions * * When `conflicts` == true, this function checks to make sure the commit of `actions` fails with * [[ConcurrentModificationException]], otherwise checks that the commit is successful. * * @param name test name * @param conflicts should test transaction is expected to conflict or not * @param setup sets up the initial delta log state (set schema, partitioning, etc.) * @param reads reads made in the test transaction * @param concurrentWrites writes made by concurrent transactions after the test txn reads * @param actions actions to be committed by the test transaction * @param expectedErrorClass Expected error class for the exception * @param expectedErrorMessageParameters Expected parameter map for error message validation * @param exceptionClass A substring to expect in the exception class name */ protected def check( name: String, conflicts: Boolean, setup: Seq[Action] = Seq(Metadata(), Action.supportedProtocolVersion( featuresToExclude = Seq(CatalogOwnedTableFeature))), reads: Seq[OptimisticTransaction => Unit], concurrentWrites: Seq[Action], actions: Seq[Action], expectedErrorClass: Option[String] = None, expectedErrorMessageParameters: Option[Map[String, String]] = None, exceptionClass: Option[String] = None): Unit = { val concurrentTxn: OptimisticTransaction => Unit = (opt: OptimisticTransaction) => opt.commit(concurrentWrites, Truncate()) def initialSetup(log: DeltaLog): Unit = { // Setup the log setup.foreach { action => log.startTransaction().commit(Seq(action), ManualUpdate) } } check( name, conflicts, initialSetup _, reads, Seq(concurrentTxn), actions, operation = Truncate(), // a data-changing operation expectedErrorClass = expectedErrorClass, expectedErrorMessageParameters = expectedErrorMessageParameters, exceptionClass = exceptionClass, additionalSQLConfs = Seq.empty ) } /** * Check whether the test transaction conflict with the concurrent writes by executing the * given params in the following order: * - sets up the initial delta log state using `initialSetup` (set schema, partitioning, etc.) * - reads * - concurrentWrites * - actions * * When `conflicts` == true, this function checks to make sure the commit of `actions` fails with * [[ConcurrentModificationException]], otherwise checks that the commit is successful. * * @param name test name * @param conflicts should test transaction is expected to conflict or not * @param initialSetup sets up the initial delta log state (set schema, partitioning, etc.) * @param reads reads made in the test transaction * @param concurrentTxns concurrent txns that may write data after the test txn reads * @param actions actions to be committed by the test transaction * @param expectedErrorClass Expected error class for the exception * @param expectedErrorMessageParameters Expected parameter map for error message validation * @param exceptionClass A substring to expect in the exception class name */ // scalastyle:off argcount protected def check( name: String, conflicts: Boolean, initialSetup: DeltaLog => Unit, reads: Seq[OptimisticTransaction => Unit], concurrentTxns: Seq[OptimisticTransaction => Unit], actions: Seq[Action], operation: DeltaOperations.Operation, expectedErrorClass: Option[String], expectedErrorMessageParameters: Option[Map[String, String]], exceptionClass: Option[String], additionalSQLConfs: Seq[(String, String)]): Unit = { // scalastyle:on argcount val conflict = if (conflicts) "should conflict" else "should not conflict" test(s"$name - $conflict") { withSQLConf(additionalSQLConfs: _*) { val tempDir = Utils.createTempDir() val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) // Setup the log initialSetup(log) // Perform reads val txn = log.startTransaction() reads.foreach(_ (txn)) // Execute concurrent txn while current transaction is active concurrentTxns.foreach(txn => txn(log.startTransaction())) // Try commit and check expected conflict behavior if (conflicts) { val e = intercept[ConcurrentModificationException] { txn.commit(actions, operation) } if (expectedErrorClass.isDefined) { checkError( e.asInstanceOf[DeltaThrowable], expectedErrorClass.get, parameters = expectedErrorMessageParameters.get ++ Map("tableName" -> s"delta.`${log.dataPath}`"), matchPVals = true ) } if (exceptionClass.nonEmpty) { assert(e.getClass.getName.contains(exceptionClass.get)) } } else { txn.commit(actions, operation) } } } } /** * Write 3 files at target path and return AddFiles. */ protected def writeDuplicateActionsData(path: String): Seq[AddFile] = { val deltaLog = DeltaLog.forTable(spark, path) spark.range(start = 0, end = 6, step = 1, numPartitions = 3) .write.format("delta").save(path) val files = deltaLog.update().allFiles.collect().sortBy(_.insertionTime) for (file <- files) { assert(file.numPhysicalRecords.isDefined) } files } protected def addDVToFileInTable(path: String, file: AddFile): (AddFile, RemoveFile) = { val deltaLog = DeltaLog.forTable(spark, path) val dv = writeDV(deltaLog, RoaringBitmapArray(0L)) updateFileDV(file, dv) } protected def testRuntimeErrorOnCommit( actions: Seq[FileAction], deltaLog: DeltaLog)( checkErrorFun: DeltaRuntimeException => Unit): Unit = { val operation = DeltaOperations.Optimize(Seq.empty, zOrderBy = Seq.empty) val txn = deltaLog.startTransaction() val e = intercept[DeltaRuntimeException] { withSQLConf(DeltaSQLConf.DELTA_DUPLICATE_ACTION_CHECK_ENABLED.key -> "true") { txn.commit(actions, operation) } } checkErrorFun(e) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/ProtocolMetadataAdapterSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.actions.{Metadata, Protocol} import org.apache.spark.SparkFunSuite import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{IntegerType, LongType, MetadataBuilder, StringType, StructField, StructType} /** * Abstract base class for testing ProtocolMetadataAdapter implementations. */ abstract class ProtocolMetadataAdapterSuiteBase extends SparkFunSuite with SharedSparkSession { /** * Creates a wrapper instance based on different parameters. * * @param minReaderVersion Protocol reader version * @param minWriterVersion Protocol writer version * @param readerFeatures Optional set of reader features * @param writerFeatures Optional set of writer features * @param schema Table schema * @param configuration Table properties/configuration */ protected def createWrapper( minReaderVersion: Int = 1, minWriterVersion: Int = 2, readerFeatures: Option[Set[String]] = None, writerFeatures: Option[Set[String]] = None, schema: StructType = new StructType().add("id", IntegerType), configuration: Map[String, String] = Map.empty): ProtocolMetadataAdapter Seq[(DeltaColumnMappingMode, Map[String, String])]( (NoMapping, Map.empty), (NameMapping, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> "name")), (IdMapping, Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> "id")) ).foreach { case (expectedMode, config) => test(s"columnMappingMode with $expectedMode") { val wrapper = createWrapper(configuration = config) assert(wrapper.columnMappingMode === expectedMode) } } Seq[(String, StructType)]( // Empty schema: table with no columns ("empty schema", new StructType()), // Simple schema: flat structure with primitive types ("simple schema", new StructType().add("id", IntegerType).add("name", StringType)), // Nested schema: struct within struct ("nested schema", new StructType() .add("user", new StructType() .add("id", IntegerType) .add("name", StringType)) .add("timestamp", LongType)) ).foreach { case (testCaseName, schema) => test(s"getReferenceSchema with $testCaseName") { val wrapper = createWrapper(schema = schema) assert(wrapper.getReferenceSchema === schema) } } Seq[(String, Boolean, Map[String, String], Option[Set[String]], Option[Set[String]])]( // Row tracking enabled by setting table features ("enabled", true, Map(DeltaConfigs.ROW_TRACKING_ENABLED.key -> "true"), Some(Set(RowTrackingFeature.name)), Some(Set(RowTrackingFeature.name))), // Row tracking disabled by default ("disabled", false, Map.empty, None, None), // Row tracking explicitly disabled via config ("explicitly disabled", false, Map(DeltaConfigs.ROW_TRACKING_ENABLED.key -> "false"), None, None) ).foreach { case (testCaseName, expectedRowIdEnabled, config, readerFeatures, writerFeatures) => test(s"isRowIdEnabled when $testCaseName") { val wrapper = createWrapper( minReaderVersion = if (readerFeatures.isDefined) 3 else 1, minWriterVersion = if (writerFeatures.isDefined) 7 else 2, readerFeatures = readerFeatures, writerFeatures = writerFeatures, configuration = config) assert(wrapper.isRowIdEnabled === expectedRowIdEnabled) } } Seq[(String, Boolean, Option[Set[String]], Option[Set[String]])]( // Deletion vectors enabled via table features ("enabled", true, Some(Set(DeletionVectorsTableFeature.name)), Some(Set(DeletionVectorsTableFeature.name))), // Deletion vectors disabled by default ("disabled", false, None, None) ).foreach { case (testCaseName, expectedDeletionVectorReadable, readerFeatures, writerFeatures) => test(s"isDeletionVectorReadable when $testCaseName") { val wrapper = createWrapper( minReaderVersion = if (readerFeatures.isDefined) 3 else 1, minWriterVersion = if (writerFeatures.isDefined) 7 else 2, readerFeatures = readerFeatures, writerFeatures = writerFeatures) assert(wrapper.isDeletionVectorReadable === expectedDeletionVectorReadable) } } Seq[(String, Boolean, Map[String, String])]( // IcebergCompat V1 enabled ("v1 enabled", true, Map(DeltaConfigs.ICEBERG_COMPAT_V1_ENABLED.key -> "true")), // IcebergCompat V2 enabled ("v2 enabled", true, Map(DeltaConfigs.ICEBERG_COMPAT_V2_ENABLED.key -> "true")), // No IcebergCompat enabled ("disabled", false, Map.empty) ).foreach { case (testCaseName, expectedIcebergCompatEnabled, config) => test(s"isIcebergCompatAnyEnabled when $testCaseName") { val wrapper = createWrapper( configuration = config) assert(wrapper.isIcebergCompatAnyEnabled === expectedIcebergCompatEnabled) } } Seq[(String, Map[String, String], Seq[(Int, Boolean)])]( // IcebergCompat V1 enabled: only version 1 should return true ("v1 enabled", Map(DeltaConfigs.ICEBERG_COMPAT_V1_ENABLED.key -> "true"), Seq[(Int, Boolean)]((1, true), (2, false), (3, false))), // V2 enabled: version 1 and 2 should return true ("v2 enabled", Map(DeltaConfigs.ICEBERG_COMPAT_V2_ENABLED.key -> "true"), Seq[(Int, Boolean)]((1, true), (2, true), (3, false))), // No version enabled: all versions should return false ("disabled", Map.empty, Seq[(Int, Boolean)]((1, false), (2, false), (3, false))) ).foreach { case (testCaseName, config, versionChecks) => test(s"isIcebergCompatGeqEnabled when $testCaseName") { val wrapper = createWrapper( configuration = config) versionChecks.foreach { case (version, expectedEnabled) => assert(wrapper.isIcebergCompatGeqEnabled(version) === expectedEnabled, s"version $version check failed") } } } Seq[(String, Option[org.apache.spark.sql.types.Metadata], Boolean, Map[String, String], Option[Set[String]], Option[Set[String]])]( // Table with no special features should be readable ("readable table", None, true, Map.empty, None, None), // Table with unsupported type widening (string -> integer), should not be readable ("table with unsupported type widening", Some(new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( new MetadataBuilder() .putLong("tableVersion", 1L) .putString("fromType", "string") .putString("toType", "integer") .build() )) .build()), false, Map(DeltaConfigs.ENABLE_TYPE_WIDENING.key -> "true"), Some(Set(TypeWideningTableFeature.name)), Some(Set(TypeWideningTableFeature.name))) ).foreach { case (testCaseName, typeChangeMetadata, tableReadable, config, readerFeatures, writerFeatures) => test(s"assertTableReadable with $testCaseName") { val schema = typeChangeMetadata match { case Some(metadata) => new StructType().add("col1", IntegerType, nullable = true, metadata = metadata) case None => new StructType().add("id", IntegerType) } val wrapper = createWrapper( minReaderVersion = if (readerFeatures.isDefined) 3 else 1, minWriterVersion = if (writerFeatures.isDefined) 7 else 2, readerFeatures = readerFeatures, writerFeatures = writerFeatures, schema = schema, configuration = config) if (tableReadable) { // Should not throw wrapper.assertTableReadable(spark) } else { // Should throw exception intercept[Exception] { wrapper.assertTableReadable(spark) } } } } Seq[(String, Map[String, String], Option[Set[String]], Option[Set[String]], Boolean, Boolean)]( // Row tracking disabled: should return no fields ("row tracking disabled", Map.empty, None, None, false, false), ("row tracking enabled with constant or generated metadata col non nullable", Map( DeltaConfigs.ROW_TRACKING_ENABLED.key -> "true", "delta.rowTracking.materializedRowIdColumnName" -> "_row_id_col", "delta.rowTracking.materializedRowCommitVersionColumnName" -> "_row_commit_version_col"), Some(Set(RowTrackingFeature.name)), Some(Set(RowTrackingFeature.name)), false, false), ("row tracking enabled with constant or generated metadata col nullable", Map( DeltaConfigs.ROW_TRACKING_ENABLED.key -> "true", "delta.rowTracking.materializedRowIdColumnName" -> "_row_id_col", "delta.rowTracking.materializedRowCommitVersionColumnName" -> "_row_commit_version_col"), Some(Set(RowTrackingFeature.name)), Some(Set(RowTrackingFeature.name)), true, true) ).foreach { case (testCaseName, config, readerFeatures, writerFeatures, nullableConstant, nullableGenerated) => test(s"createRowTrackingMetadataFields when $testCaseName") { val wrapper = createWrapper( minReaderVersion = if (readerFeatures.isDefined) 3 else 1, minWriterVersion = if (writerFeatures.isDefined) 7 else 2, readerFeatures = readerFeatures, writerFeatures = writerFeatures, configuration = config) val fields = wrapper.createRowTrackingMetadataFields( nullableConstant, nullableGenerated).toSeq val expectedFields = if (!config.get(DeltaConfigs.ROW_TRACKING_ENABLED.key).contains("true")) { // Row tracking disabled: no fields Seq.empty[StructField] } else { Seq[StructField]( StructField("row_id", LongType, nullableGenerated), StructField("base_row_id", LongType, nullableConstant), StructField("default_row_commit_version", LongType, nullableConstant), StructField("row_commit_version", LongType, nullableGenerated)) } val actualSimplified = fields.map(f => StructField(f.name, f.dataType, f.nullable)) assert(actualSimplified === expectedFields) } } } /** * Unit tests for ProtocolMetadataAdapterV1. * * This suite tests the V1 wrapper implementation that adapts delta-spark's Protocol and Metadata * to the ProtocolMetadataAdapter interface. */ class ProtocolMetadataAdapterV1Suite extends ProtocolMetadataAdapterSuiteBase { override protected def createWrapper( minReaderVersion: Int = 1, minWriterVersion: Int = 2, readerFeatures: Option[Set[String]] = None, writerFeatures: Option[Set[String]] = None, schema: StructType = new StructType().add("id", IntegerType), configuration: Map[String, String] = Map.empty): ProtocolMetadataAdapter = { val protocol = Protocol( minReaderVersion = minReaderVersion, minWriterVersion = minWriterVersion, readerFeatures = readerFeatures, writerFeatures = writerFeatures) val metadata = Metadata( schemaString = schema.json, configuration = configuration) ProtocolMetadataAdapterV1(protocol, metadata) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/RestoreTableSQLSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.{AnalysisException, DataFrame} /** Restore tests using the SQL. */ class RestoreTableSQLSuite extends RestoreTableSuiteBase { override def restoreTableToVersion( tblId: String, version: Int, isTable: Boolean, expectNoOp: Boolean = false): DataFrame = { val identifier = if (isTable) { tblId } else { s"delta.`$tblId`" } spark.sql(s"RESTORE TABLE $identifier VERSION AS OF ${version}") } override def restoreTableToTimestamp( tblId: String, timestamp: String, isTable: Boolean, expectNoOp: Boolean = false): DataFrame = { val identifier = if (isTable) { tblId } else { s"delta.`$tblId`" } spark.sql(s"RESTORE $identifier TO TIMESTAMP AS OF '${timestamp}'") } test("restoring a table that doesn't exist") { val ex = intercept[AnalysisException] { sql(s"RESTORE TABLE not_exists VERSION AS OF 0") } assert(ex.getMessage.contains("Table not found") || ex.getMessage.contains("TABLE_OR_VIEW_NOT_FOUND")) } test("restoring a view") { withTempView("tmp") { sql("CREATE OR REPLACE TEMP VIEW tmp AS SELECT * FROM range(10)") val ex = intercept[AnalysisException] { sql(s"RESTORE tmp TO VERSION AS OF 0") } assert(ex.getMessage.contains("only supported for Delta tables")) } } test("restoring a view over a Delta table") { withTable("delta_table") { withView("tmp") { sql("CREATE TABLE delta_table USING delta AS SELECT * FROM range(10)") sql("CREATE VIEW tmp AS SELECT * FROM delta_table") val ex = intercept[AnalysisException] { sql(s"RESTORE TABLE tmp VERSION AS OF 0") } assert(ex.getMessage.contains("only supported for Delta tables")) } } } } class RestoreTableSQLWithCatalogOwnedBatch1Suite extends RestoreTableSQLSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class RestoreTableSQLWithCatalogOwnedBatch2Suite extends RestoreTableSQLSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class RestoreTableSQLWithCatalogOwnedBatch100Suite extends RestoreTableSQLSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } class RestoreTableSQLNameColumnMappingSuite extends RestoreTableSQLSuite with DeltaColumnMappingEnableNameMode { import testImplicits._ override protected def runOnlyTests = Seq( "path based table", "metastore based table" ) test("restore prior to column mapping upgrade should fail") { withTempDir { tempDir => val df1 = Seq(1, 2, 3).toDF("id") val df2 = Seq(4, 5, 6).toDF("id") def deltaLog: DeltaLog = DeltaLog.forTable(spark, tempDir.getAbsolutePath) withColumnMappingConf("none") { df1.write.format("delta").save(tempDir.getAbsolutePath) require(deltaLog.update().version == 0) df2.write.format("delta").mode("append").save(tempDir.getAbsolutePath) assert(deltaLog.update().version == 1) } // upgrade to column mapping mode sql( s""" |ALTER TABLE delta.`$tempDir` |SET TBLPROPERTIES ( | ${DeltaConfigs.COLUMN_MAPPING_MODE.key} = '$columnMappingModeString', | ${DeltaConfigs.MIN_READER_VERSION.key} = '2', | ${DeltaConfigs.MIN_WRITER_VERSION.key} = '5' |) |""".stripMargin) assert(deltaLog.update().version == 2) // try restore back to version 1 before column mapping should fail intercept[ColumnMappingUnsupportedException] { restoreTableToVersion(tempDir.getAbsolutePath, version = 1, isTable = false) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/RestoreTableScalaSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.sql.Date import scala.concurrent.duration._ // scalastyle:off import.ordering.noEmptyLine import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.test.DeltaExcludedTestMixin import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.{DataFrame, Row} import org.apache.spark.util.Utils /** Restore tests using the Scala APIs. */ class RestoreTableScalaSuite extends RestoreTableSuiteBase { override def restoreTableToVersion( tblId: String, version: Int, isTable: Boolean, expectNoOp: Boolean = false): DataFrame = { val deltaTable = if (isTable) { io.delta.tables.DeltaTable.forName(spark, tblId) } else { io.delta.tables.DeltaTable.forPath(spark, tblId) } deltaTable.restoreToVersion(version) } override def restoreTableToTimestamp( tblId: String, timestamp: String, isTable: Boolean, expectNoOp: Boolean = false): DataFrame = { val deltaTable = if (isTable) { io.delta.tables.DeltaTable.forName(spark, tblId) } else { io.delta.tables.DeltaTable.forPath(spark, tblId) } deltaTable.restoreToTimestamp(timestamp) } } class RestoreTableScalaWithCatalogOwnedBatch1Suite extends RestoreTableScalaSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class RestoreTableScalaWithCatalogOwnedBatch2Suite extends RestoreTableScalaSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class RestoreTableScalaWithCatalogOwnedBatch100Suite extends RestoreTableScalaSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } class RestoreTableScalaDeletionVectorSuite extends RestoreTableScalaSuite with DeletionVectorsTestUtils with DeltaExcludedTestMixin { import testImplicits._ override def beforeAll(): Unit = { super.beforeAll() enableDeletionVectors(spark.conf) } override def excluded: Seq[String] = super.excluded ++ Seq( // These tests perform a delete to produce a file to vacuum, but with persistent DVs enabled, // we actually just add a DV to the file instead, so there's no unreferenced file for vacuum. "restore after vacuum", "restore after vacuum - cloned table", // These rely on the new-table protocol version to be lower than the latest, // but this isn't true for DVs. "restore downgrade protocol (allowed=true)", "restore downgrade protocol (allowed=false)", "restore downgrade protocol with table features (allowed=true)", "restore downgrade protocol with table features (allowed=false)", "cdf + RESTORE with write amplification reduction", "RESTORE doesn't account for session defaults" ) case class RestoreAndCheckArgs(versionToRestore: Int, expectedResult: DataFrame) type RestoreAndCheckFunction = RestoreAndCheckArgs => Unit /** * Tests `testFun` once by restoring to version and once to timestamp. * * `testFun` is expected to perform setup before executing the `RestoreAndTestFunction` and * cleanup afterwards. */ protected def testRestoreByTimestampAndVersion (testName: String) (testFun: (String, RestoreAndCheckFunction) => Unit): Unit = { for (restoreToVersion <- BOOLEAN_DOMAIN) { val restoringTo = if (restoreToVersion) "version" else "timestamp" test(testName + s" - restoring to $restoringTo") { withTempDir{ dir => val path = dir.toString val restoreAndCheck: RestoreAndCheckFunction = (args: RestoreAndCheckArgs) => { val deltaLog = DeltaLog.forTable(spark, path) if (restoreToVersion) { restoreTableToVersion(path, args.versionToRestore, isTable = false) } else { // Set a custom timestamp for the commit val desiredDateS = new Date(System.currentTimeMillis() - 3.days.toMillis).toString setTimestampToCommitFileAtVersion( deltaLog, version = args.versionToRestore, date = desiredDateS) // Set all previous versions to something lower, so we don't error out. for (version <- 0 until args.versionToRestore) { val previousDateS = new Date(System.currentTimeMillis() - 5.days.toMillis).toString setTimestampToCommitFileAtVersion( deltaLog, version = version, date = previousDateS) } restoreTableToTimestamp(path, desiredDateS, isTable = false) } checkAnswer(spark.read.format("delta").load(path), args.expectedResult) } testFun(path, restoreAndCheck) } } } } testRestoreByTimestampAndVersion( "Restoring table with persistent DVs to version without DVs") { (path, restoreAndCheck) => val deltaLog = DeltaLog.forTable(spark, path) val df1 = Seq(1, 2, 3, 4, 5).toDF("id") val values2 = Seq(6, 7, 8, 9, 10) val df2 = values2.toDF("id") // Write all values into version 0. df1.union(df2).coalesce(1).write.format("delta").save(path) // version 0 checkAnswer(spark.read.format("delta").load(path), expectedAnswer = df1.union(df2)) val snapshotV0 = deltaLog.update() assert(snapshotV0.version === 0) // Delete values 2 so that version 1 is `df1`. spark.sql(s"DELETE FROM delta.`$path` WHERE id IN (${values2.mkString(", ")})") // version 1 assert(getFilesWithDeletionVectors(deltaLog).size > 0) checkAnswer(spark.read.format("delta").load(path), expectedAnswer = df1) val snapshotV1 = deltaLog.snapshot assert(snapshotV1.version === 1) restoreAndCheck(RestoreAndCheckArgs(versionToRestore = 0, expectedResult = df1.union(df2))) assert(getFilesWithDeletionVectors(deltaLog).size === 0) } testRestoreByTimestampAndVersion( "Restoring table with persistent DVs to version with DVs") { (path, restoreAndCheck) => val deltaLog = DeltaLog.forTable(spark, path) val df1 = Seq(1, 2, 3, 4, 5).toDF("id") val values2 = Seq(6, 7) val df2 = values2.toDF("id") val values3 = Seq(8, 9, 10) val df3 = values3.toDF("id") // Write all values into version 0. df1.union(df2).union(df3).coalesce(1).write.format("delta").save(path) // version 0 // Delete values 2 and 3 in reverse order, so that version 1 is `df1.union(df2)`. spark.sql(s"DELETE FROM delta.`$path` WHERE id IN (${values3.mkString(", ")})") // version 1 assert(getFilesWithDeletionVectors(deltaLog).size > 0) checkAnswer(spark.read.format("delta").load(path), expectedAnswer = df1.union(df2)) spark.sql(s"DELETE FROM delta.`$path` WHERE id IN (${values2.mkString(", ")})") // version 2 assert(getFilesWithDeletionVectors(deltaLog).size > 0) restoreAndCheck(RestoreAndCheckArgs(versionToRestore = 1, expectedResult = df1.union(df2))) assert(getFilesWithDeletionVectors(deltaLog).size > 0) } testRestoreByTimestampAndVersion("Restoring table with persistent DVs to version " + "without persistent DVs enabled") { (path, restoreAndCheck) => withSQLConf( DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> "false", // Disable the log clean up. Tests sets the timestamp on commit files to long back // in time that triggers the commit file clean up as part of the [[MetadataCleanup]] DeltaConfigs.ENABLE_EXPIRED_LOG_CLEANUP.defaultTablePropertyKey -> "false") { val deltaLog = DeltaLog.forTable(spark, path) val df1 = Seq(1, 2, 3, 4, 5).toDF("id") val values2 = Seq(6, 7, 8, 9, 10) val df2 = values2.toDF("id") // Write all values into version 0. df1.union(df2).coalesce(1).write.format("delta").save(path) // version 0 checkAnswer(spark.read.format("delta").load(path), expectedAnswer = df1.union(df2)) val snapshotV0 = deltaLog.update() assert(snapshotV0.version === 0) assert(!DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.fromMetaData(snapshotV0.metadata)) // Upgrade to us DVs spark.sql(s"ALTER TABLE delta.`$path` SET TBLPROPERTIES " + s"(${DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key} = true)") val snapshotV1 = deltaLog.update() assert(snapshotV1.version === 1) assert(DeletionVectorUtils.deletionVectorsReadable(snapshotV1)) assert(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.fromMetaData(snapshotV1.metadata)) // Delete values 2 so that version 1 is `df1`. spark.sql(s"DELETE FROM delta.`$path` WHERE id IN (${values2.mkString(", ")})") // version 2 assert(getFilesWithDeletionVectors(deltaLog).size > 0) checkAnswer(spark.read.format("delta").load(path), expectedAnswer = df1) val snapshotV2 = deltaLog.update() assert(snapshotV2.version === 2) // Restore to before the version upgrade. Protocol version should be retained (to make the // history readable), but DV creation should be disabled again. restoreAndCheck(RestoreAndCheckArgs(versionToRestore = 0, expectedResult = df1.union(df2))) val snapshotV3 = deltaLog.update() assert(getFilesWithDeletionVectors(deltaLog).size === 0) assert(DeletionVectorUtils.deletionVectorsReadable(snapshotV3)) assert(!DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.fromMetaData(snapshotV3.metadata)) // Check that we can still read versions that did have DVs. checkAnswer( spark.read.format("delta").option("versionAsOf", "2").load(path), expectedAnswer = df1) } } test("CDF + DV + RESTORE") { withSQLConf(DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> "true") { withTempDir { tempDir => val df0 = Seq(0, 1).toDF("id") // version 0 = [0, 1] df0.write.format("delta").save(tempDir.getAbsolutePath) val df1 = Seq(2).toDF("id") // version 1: append to df0 = [0, 1, 2] df1.write.mode("append").format("delta").save(tempDir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tempDir.getAbsolutePath) deltaTable.delete("id < 1") // version 2: delete (0) = [1, 2] deltaTable.updateExpr( "id > 1", Map("id" -> "4") ) // version 3: update 2 --> 4 = [1, 4] // version 4: restore to version 2 (delete 4, insert 2) = [1, 2] restoreTableToVersion(tempDir.getAbsolutePath, 2, false) checkAnswer( CDCReader.changesToBatchDF(DeltaLog.forTable(spark, tempDir), 4, 4, spark) .drop(CDCReader.CDC_COMMIT_TIMESTAMP), Row(4, "delete", 4) :: Row(2, "insert", 4) :: Nil ) // version 5: restore to version 1 (insert 0) = [0, 1, 2] restoreTableToVersion(tempDir.getAbsolutePath, 1, false) checkAnswer( CDCReader.changesToBatchDF(DeltaLog.forTable(spark, tempDir), 5, 5, spark) .drop(CDCReader.CDC_COMMIT_TIMESTAMP), Row(0, "insert", 5) :: Nil ) // version 6: restore to version 0 (delete 2) = [0, 1] restoreTableToVersion(tempDir.getAbsolutePath, 0, false) checkAnswer( CDCReader.changesToBatchDF(DeltaLog.forTable(spark, tempDir), 6, 6, spark) .drop(CDCReader.CDC_COMMIT_TIMESTAMP), Row(2, "delete", 6) :: Nil ) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/RestoreTableSuiteBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import java.sql.Date import java.sql.Timestamp import java.text.SimpleDateFormat import scala.concurrent.duration._ import org.apache.spark.sql.delta.actions.{Protocol, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils.{TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION} import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.FileNames import org.apache.spark.sql.{DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ /** Base suite containing the restore tests. */ trait RestoreTableSuiteBase extends QueryTest with SharedSparkSession with DeltaSQLTestUtils with DeltaSQLCommandTest with CatalogOwnedTestBaseSuite { import testImplicits._ // Will be overridden in sub-class /** * @param tblId - the table identifier either table name or path * @param version - version to restore to * @param isMetastoreTable - whether its a path based table or metastore table * @param expectNoOp - whether the restore is no-op or not */ protected def restoreTableToVersion( tblId: String, version: Int, isMetastoreTable: Boolean, expectNoOp: Boolean = false): DataFrame /** * @param tblId - the table identifier either table name or path * @param timestamp - timestamp to restore to * @param isMetastoreTable - whether its a path based table or a metastore table. * @param expectNoOp - whether the restore is no-op or not */ protected def restoreTableToTimestamp( tblId: String, timestamp: String, isMetastoreTable: Boolean, expectNoOp: Boolean = false): DataFrame /** * Override the timestamp of the commit file at the given version. * If CC is enabled, then the timestamp generated by InCommitTimestamp will be * overridden since ICT is a dependent feature of CC. * Otherwise, the file modification time will be overridden. * * NOTE: When CC is enabled, this method will only override the commit timestamp * for the backfilled/published commit files. * * @param deltaLog The DeltaLog for the table. * @param version The specific version of the commit file to override. * @param timestamp The timestamp to set as the commit timestamp. */ protected def overrideTimestampOfCommitFile( deltaLog: DeltaLog, version: Int, timestamp: Long): Unit = { if (catalogOwnedDefaultCreationEnabledInTests) { // If CC is enabled, then timestamp generated by ICT will need to be overridden. InCommitTimestampTestUtils.overwriteICTInDeltaFile( deltaLog, filePath = FileNames.unsafeDeltaFile(path = deltaLog.logPath, version), ts = Some(timestamp) ) } else { // Otherwise we manually modify the file modification time. setTimestampToCommitFileAtVersion(deltaLog, version, timestamp) } } test("path based table") { withTempDir { tempDir => val path = tempDir.getAbsolutePath val df1 = Seq(1, 2, 3, 4, 5).toDF("id") val df2 = Seq(6, 7).toDF("id") val df3 = Seq(8, 9, 10).toDF("id") // write version 0 of the table df1.write.format("delta").save(path) // version 0 val deltaLog = DeltaLog.forTable(spark, path) require(deltaLog.snapshot.version == 0) // append df2 to the table df2.write.format("delta").mode("append").save(path) // version 1 // append df3 to the table df3.write.format("delta").mode("append").save(path) // version 2 // check if the table has all the three dataframes written checkAnswer(spark.read.format("delta").load(path), df1.union(df2).union(df3)) // restore by version to version 1 restoreTableToVersion(path, 1, false) checkAnswer(spark.read.format("delta").load(path), df1.union(df2)) // Set a custom timestamp for the commit val desiredDate = new Date(System.currentTimeMillis() - 5.days.toMillis) overrideTimestampOfCommitFile( deltaLog, version = 0, timestamp = dateStringToTimestamp(date = desiredDate.toString) ) // restore by timestamp to version 0 restoreTableToTimestamp(path, desiredDate.toString, false) checkAnswer(spark.read.format("delta").load(path), df1) } } protected def dateStringToTimestamp(date: String): Long = { val format = new java.text.SimpleDateFormat("yyyy-MM-dd") format.parse(date).getTime } protected def timeStringToTimestamp(time: String): Long = { val format = new java.text.SimpleDateFormat("yyyy-MM-dd hh:mm:ss Z") format.parse(time).getTime } protected def setTimestampToCommitFileAtVersion( deltaLog: DeltaLog, version: Int, date: String): Unit = { val timestamp = dateStringToTimestamp(date) setTimestampToCommitFileAtVersion(deltaLog, version, timestamp) } protected def setTimestampToCommitFileAtVersion( deltaLog: DeltaLog, version: Int, timestamp: Long): Unit = { val file = new File(FileNames.unsafeDeltaFile(deltaLog.logPath, version).toUri) file.setLastModified(timestamp) } test("metastore based table") { val identifier = "tbl" withTable(identifier) { val df1 = Seq(1, 2, 3, 4, 5).toDF("id") val df2 = Seq(6, 7).toDF("id") // write first version of the table df1.write.format("delta").saveAsTable(identifier) // version 0 val deltaLog = DeltaLog.forTable(spark, new TableIdentifier(identifier)) require(deltaLog.snapshot.version == 0) // append df2 to the table df2.write.format("delta").mode("append").saveAsTable(identifier) // version 1 // check if the table has all the three dataframes written checkAnswer(spark.read.format("delta").table(identifier), df1.union(df2)) // restore by version to version 0 restoreTableToVersion(identifier, 0, true) checkAnswer(spark.read.format("delta").table(identifier), df1) } } test("restore a restore back to pre-restore version") { withTempDir { tempDir => val df1 = Seq(1, 2, 3).toDF("id") val df2 = Seq(4, 5, 6).toDF("id") val df3 = Seq(7, 8, 9).toDF("id") df1.write.format("delta").save(tempDir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, tempDir.getAbsolutePath) require(deltaLog.snapshot.version == 0) df2.write.format("delta").mode("append").save(tempDir.getAbsolutePath) assert(deltaLog.update().version == 1) df3.write.format("delta").mode("append").save(tempDir.getAbsolutePath) assert(deltaLog.update().version == 2) // we have three versions now, let's restore to version 1 first restoreTableToVersion(tempDir.getAbsolutePath, 1, false) checkAnswer(spark.read.format("delta").load(tempDir.getAbsolutePath), df1.union(df2)) assert(deltaLog.update().version == 3) restoreTableToVersion(tempDir.getAbsolutePath, 2, false) checkAnswer( spark.read.format("delta").load(tempDir.getAbsolutePath), df1.union(df2).union(df3)) assert(deltaLog.update().version == 4) } } test("restore to a restored version") { withTempDir { tempDir => val df1 = Seq(1, 2, 3).toDF("id") val df2 = Seq(4, 5, 6).toDF("id") val df3 = Seq(7, 8, 9).toDF("id") df1.write.format("delta").save(tempDir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, tempDir.getAbsolutePath) require(deltaLog.update().version == 0) df2.write.format("delta").mode("append").save(tempDir.getAbsolutePath) assert(deltaLog.update().version == 1) // we have two versions now, let's restore to version 0 first restoreTableToVersion(tempDir.getAbsolutePath, 0, false) checkAnswer(spark.read.format("delta").load(tempDir.getAbsolutePath), df1) assert(deltaLog.update().version == 2) df3.write.format("delta").mode("append").save(tempDir.getAbsolutePath) assert(deltaLog.update().version == 3) // now we restore a restored version restoreTableToVersion(tempDir.getAbsolutePath, 2, false) checkAnswer(spark.read.format("delta").load(tempDir.getAbsolutePath), df1) assert(deltaLog.update().version == 4) } } for (downgradeAllowed <- DeltaTestUtils.BOOLEAN_DOMAIN) test(s"restore downgrade protocol (allowed=$downgradeAllowed)") { if (catalogOwnedDefaultCreationEnabledInTests) { cancel("CatalogOwned is not compatible w/ the test since protocol version would be " + "set to (3, 7) by default for CC tables.") } withTempDir { tempDir => val path = tempDir.getAbsolutePath spark.range(5).write.format("delta").save(path) val deltaLog = DeltaLog.forTable(spark, path) val oldProtocolVersion = deltaLog.snapshot.protocol // Update table to latest version. deltaLog.upgradeProtocol( oldProtocolVersion.merge(Protocol().withFeature(TestReaderWriterFeature))) val newProtocolVersion = deltaLog.snapshot.protocol assert(newProtocolVersion.minReaderVersion > oldProtocolVersion.minReaderVersion && newProtocolVersion.minWriterVersion > oldProtocolVersion.minWriterVersion, s"newProtocolVersion=$newProtocolVersion is not strictly greater than" + s" oldProtocolVersion=$oldProtocolVersion") withSQLConf(DeltaSQLConf.RESTORE_TABLE_PROTOCOL_DOWNGRADE_ALLOWED.key -> downgradeAllowed.toString) { // Restore to before the upgrade. restoreTableToVersion(path, version = 0, isMetastoreTable = false) } val restoredProtocolVersion = deltaLog.snapshot.protocol if (downgradeAllowed) { assert(restoredProtocolVersion === oldProtocolVersion) } else { assert(restoredProtocolVersion === newProtocolVersion.merge(oldProtocolVersion)) } } } for (downgradeAllowed <- DeltaTestUtils.BOOLEAN_DOMAIN) test( s"restore downgrade protocol with table features (allowed=$downgradeAllowed)") { if (catalogOwnedDefaultCreationEnabledInTests) { cancel("CatalogOwned is not compatible w/ the test since protocol version would be " + "set to (3, 7) by default for CC tables.") } withTempDir { tempDir => val path = tempDir.getAbsolutePath spark.range(5).write.format("delta").save(path) val deltaLog = DeltaLog.forTable(spark, path) val oldProtocolVersion = deltaLog.snapshot.protocol // Update table to latest version. deltaLog.upgradeProtocol( Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeatures(Seq(TestLegacyReaderWriterFeature)) .withFeatures(oldProtocolVersion.implicitlyAndExplicitlySupportedFeatures)) val newProtocolVersion = deltaLog.snapshot.protocol assert( newProtocolVersion.minReaderVersion > oldProtocolVersion.minReaderVersion && newProtocolVersion.minWriterVersion >= oldProtocolVersion.minWriterVersion, s"newProtocolVersion=$newProtocolVersion is not strictly greater than" + s" oldProtocolVersion=$oldProtocolVersion") withSQLConf( DeltaSQLConf.RESTORE_TABLE_PROTOCOL_DOWNGRADE_ALLOWED.key -> downgradeAllowed.toString) { // Restore to before the upgrade. restoreTableToVersion(path, version = 0, isMetastoreTable = false) } val restoredProtocolVersion = deltaLog.snapshot.protocol if (downgradeAllowed) { assert(restoredProtocolVersion === oldProtocolVersion) } else { assert(restoredProtocolVersion === newProtocolVersion.merge(oldProtocolVersion)) } } } test("RESTORE doesn't account for session defaults") { withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> "1", DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "1") { if (catalogOwnedDefaultCreationEnabledInTests) { cancel("CatalogOwned is not compatible w/ the test since protocol version would be " + "set to (3, 7) by default for CC tables.") } withTempDir { dir => spark.range(10).write.format("delta").save(dir.getAbsolutePath) spark .range(start = 10, end = 20) .write .format("delta") .mode("append") .save(dir.getAbsolutePath) val log = DeltaLog.forTable(spark, dir.getAbsolutePath) val oldProtocol = log.update().protocol assert(oldProtocol === Protocol(1, 1)) withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> "2", DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "2", TableFeatureProtocolUtils.defaultPropertyKey(TestWriterFeature) -> "enabled") { restoreTableToVersion(dir.getAbsolutePath, 0, isMetastoreTable = false) } val newProtocol = log.update().protocol assert(newProtocol === oldProtocol) } } } test("restore operation metrics in Delta table history") { withSQLConf( DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true") { withTempDir { tempDir => val df1 = Seq(1, 2, 3).toDF("id") val df2 = Seq(4, 5, 6).toDF("id") df1.write.format("delta").save(tempDir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, tempDir.getAbsolutePath) df2.write.format("delta").mode("append").save(tempDir.getAbsolutePath) assert(deltaLog.update().version == 1) // we have two versions now, let's restore to version 0 first restoreTableToVersion(tempDir.getAbsolutePath, 0, false) val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tempDir.getAbsolutePath) val actualOperationMetrics = deltaTable.history(1).select("operationMetrics") .take(1) .head .getMap(0) .asInstanceOf[Map[String, String]] // File sizes are flaky due to differences in order of data (=> encoding size differences) assert(actualOperationMetrics.get("tableSizeAfterRestore").isDefined) assert(actualOperationMetrics.get("numOfFilesAfterRestore").get == "2") assert(actualOperationMetrics.get("numRemovedFiles").get == "2") assert(actualOperationMetrics.get("numRestoredFiles").get == "0") // File sizes are flaky due to differences in order of data (=> encoding size differences) assert(actualOperationMetrics.get("removedFilesSize").isDefined) assert(actualOperationMetrics.get("restoredFilesSize").get == "0") } } } test("restore command output metrics") { withSQLConf( DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true") { withTempDir { tempDir => val df1 = Seq(1, 2, 3).toDF("id") val df2 = Seq(4, 5, 6).toDF("id") df1.write.format("delta").save(tempDir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, tempDir.getAbsolutePath) df2.write.format("delta").mode("append").save(tempDir.getAbsolutePath) assert(deltaLog.update().version == 1) // we have two versions now, let's restore to version 0 first val actualOutputMetrics = restoreTableToVersion(tempDir.getAbsolutePath, 0, false) // verify the schema val expectedRestoreOutputSchema = StructType(Seq( StructField("table_size_after_restore", LongType), StructField("num_of_files_after_restore", LongType), StructField("num_removed_files", LongType), StructField("num_restored_files", LongType), StructField("removed_files_size", LongType), StructField("restored_files_size", LongType) )) assert(actualOutputMetrics.schema == expectedRestoreOutputSchema) val outputRow = actualOutputMetrics.take(1).head // File sizes are flaky due to differences in order of data (=> encoding size differences) assert(outputRow.getLong(0) > 0L) // table_size_after_restore assert(outputRow.getLong(1) == 2L) // num_of_files_after_restore assert(outputRow.getLong(2) == 2L) // num_removed_files assert(outputRow.getLong(3) == 0L) // num_restored_files // File sizes are flaky due to differences in order of data (=> encoding size differences) assert(outputRow.getLong(4) > 0L) // removed_files_size assert(outputRow.getLong(5) == 0L) // restored_files_size } } } test("cdf + RESTORE") { withSQLConf( DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> "true") { withTempDir { tempDir => val df0 = Seq(0, 1).toDF("id") // version 0 = [0, 1] df0.write.format("delta").save(tempDir.getAbsolutePath) val df1 = Seq(2).toDF("id") // version 1: append to df0 = [0, 1, 2] df1.write.mode("append").format("delta").save(tempDir.getAbsolutePath) val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tempDir.getAbsolutePath) deltaTable.delete("id < 1") // version 2: delete (0) = [1, 2] deltaTable.updateExpr( "id > 1", Map("id" -> "4") ) // version 3: update 2 --> 4 = [1, 4] // version 4: restore to version 2 (delete 4, insert 2) = [1, 2] restoreTableToVersion(tempDir.getAbsolutePath, 2, false) checkAnswer( CDCReader.changesToBatchDF(DeltaLog.forTable(spark, tempDir), 4, 4, spark) .drop(CDCReader.CDC_COMMIT_TIMESTAMP), Row(4, "delete", 4) :: Row(2, "insert", 4) :: Nil ) // version 5: restore to version 1 (insert 0) = [0, 1, 2] restoreTableToVersion(tempDir.getAbsolutePath, 1, false) checkAnswer( CDCReader.changesToBatchDF(DeltaLog.forTable(spark, tempDir), 5, 5, spark) .drop(CDCReader.CDC_COMMIT_TIMESTAMP), Row(0, "insert", 5) :: Nil ) // version 6: restore to version 0 (delete 2) = [0, 1] restoreTableToVersion(tempDir.getAbsolutePath, 0, false) checkAnswer( CDCReader.changesToBatchDF(DeltaLog.forTable(spark, tempDir), 6, 6, spark) .drop(CDCReader.CDC_COMMIT_TIMESTAMP), Row(2, "delete", 6) :: Nil ) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/S3LikeLocalFileSystem.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.net.URI import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.RawLocalFileSystem /** * A local filesystem on scheme s3. Useful for testing paths on non-default schemes. */ class S3LikeLocalFileSystem extends RawLocalFileSystem { private var uri: URI = _ override def getScheme: String = "s3" override def initialize(name: URI, conf: Configuration): Unit = { uri = URI.create(name.getScheme + ":///") super.initialize(name, conf) } override def getUri(): URI = if (uri == null) { // RawLocalFileSystem's constructor will call this one before `initialize` is called. // Just return the super's URI to avoid NPE. super.getUri } else { uri } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/S3SingleDriverLogStoreSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import org.apache.spark.sql.delta.storage.{HDFSLogStore, LogStore, S3SingleDriverLogStore} import org.apache.spark.sql.delta.util.FileNames import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} trait S3SingleDriverLogStoreSuiteBase extends LogStoreSuiteBase { private def checkLogStoreList( store: LogStore, path: Path, expectedVersions: Seq[Int], hadoopConf: Configuration): Unit = { assert(store.listFrom(path, hadoopConf).map(FileNames.deltaVersion).toSeq === expectedVersions) } private def checkFileSystemList(fs: FileSystem, path: Path, expectedVersions: Seq[Int]): Unit = { val fsList = fs.listStatus(path.getParent).filter(_.getPath.getName >= path.getName) assert(fsList.map(FileNames.deltaVersion).sorted === expectedVersions) } testHadoopConf( ".*No FileSystem for scheme.*fake.*", "fs.fake.impl" -> classOf[FakeFileSystem].getName, "fs.fake.impl.disable.cache" -> "true") test("file system has priority over cache") { withTempDir { dir => val store = createLogStore(spark) val deltas = Seq(0, 1, 2).map(i => FileNames.unsafeDeltaFile(new Path(dir.toURI), i)) store.write(deltas(0), Iterator("zero"), overwrite = false, sessionHadoopConf) store.write(deltas(1), Iterator("one"), overwrite = false, sessionHadoopConf) store.write(deltas(2), Iterator("two"), overwrite = false, sessionHadoopConf) // delete delta file 2 and its checksum from file system val fs = new Path(dir.getCanonicalPath).getFileSystem(sessionHadoopConf) val delta2CRC = FileNames.checksumFile(new Path(dir.toURI), 2) fs.delete(deltas(2), true) fs.delete(delta2CRC, true) // magically create a different version of file 2 in the FileSystem only val hackyStore = new HDFSLogStore(sparkConf, sessionHadoopConf) hackyStore.write(deltas(2), Iterator("foo"), overwrite = true, sessionHadoopConf) // we should see "foo" (FileSystem value) instead of "two" (cache value) assert(store.read(deltas(2), sessionHadoopConf).head == "foo") } } protected def shouldUseRenameToWriteCheckpoint: Boolean = false /** * S3SingleDriverLogStore.scala can invalidate cache * S3SingleDriverLogStore.java cannot invalidate cache */ protected def canInvalidateCache: Boolean } class S3SingleDriverLogStoreSuite extends S3SingleDriverLogStoreSuiteBase { override val logStoreClassName: String = classOf[S3SingleDriverLogStore].getName override protected def canInvalidateCache: Boolean = true } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/SchemaValidationSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.util.concurrent.CountDownLatch import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.{AnalysisException, QueryTest, Row, SparkSession} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.functions._ import org.apache.spark.sql.test.SharedSparkSession /** * This Suite tests the behavior of Delta commands when a schema altering commit is run after the * command completes analysis but before the command starts the transaction. We want to make sure * That we do not corrupt tables. */ class SchemaValidationSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { class BlockingRule( blockActionLatch: CountDownLatch, startConcurrentUpdateLatch: CountDownLatch) extends Rule[LogicalPlan] { override def apply(plan: LogicalPlan): LogicalPlan = { startConcurrentUpdateLatch.countDown() blockActionLatch.await() plan } } /** * Blocks the thread with the help of an optimizer rule until end of scope. * We need two latches to ensure that the thread executing the query is blocked until * the other thread concurrently updates the metadata. `blockActionLatch` blocks the action * until it is counted down by the thread updating the metadata. `startConcurrentUpdateLatch` * will block the concurrent update to happen until it is counted down by the action reaches the * optimizer rule. */ private def withBlockedExecution( t: Thread, blockActionLatch: CountDownLatch, startConcurrentUpdateLatch: CountDownLatch)(f: => Unit): Unit = { t.start() startConcurrentUpdateLatch.await() try { f } finally { blockActionLatch.countDown() t.join() } } def cloneSession(spark: SparkSession): SparkSession = spark.cloneSession() /** * Common base method for both the path based and table name based tests. */ private def testConcurrentChangeBase(identifier: String)( createTable: (SparkSession, String) => Unit, actionToTest: (SparkSession, String) => Unit, concurrentChange: (SparkSession, String) => Unit): Unit = { createTable(spark, identifier) // Clone the session to run the query in a separate thread. val newSession = cloneSession(spark) val blockActionLatch = new CountDownLatch(1) val startConcurrentUpdateLatch = new CountDownLatch(1) val rule = new BlockingRule(blockActionLatch, startConcurrentUpdateLatch) newSession.experimental.extraOptimizations :+= rule var actionException: Exception = null val actionToTestThread = new Thread() { override def run(): Unit = { try { actionToTest(newSession, identifier) } catch { case e: Exception => actionException = e } } } withBlockedExecution(actionToTestThread, blockActionLatch, startConcurrentUpdateLatch) { concurrentChange(spark, identifier) } if (actionException != null) { throw actionException } } /** * tests the behavior of concurrent changes to schema on a blocked command. * @param testName - name of the test * @param createTable - method that creates a table given an identifier and spark session. * @param actionToTest - the method we want to test. * @param concurrentChange - the concurrent query that updates the schema of the table * * All the above methods take SparkSession and the table path as parameters */ def testConcurrentChange(testName: String, testTags: org.scalatest.Tag*)( createTable: (SparkSession, String) => Unit, actionToTest: (SparkSession, String) => Unit, concurrentChange: (SparkSession, String) => Unit): Unit = { test(testName, testTags: _*) { withTempDir { tempDir => testConcurrentChangeBase(tempDir.getCanonicalPath)( createTable, actionToTest, concurrentChange ) } } } /** * tests the behavior of concurrent changes pf schema on a blocked command with metastore tables. * @param testName - name of the test * @param createTable - method that creates a table given an identifier and spark session. * @param actionToTest - the method we want to test. * @param concurrentChange - the concurrent query that updates the schema of the table * * All the above methods take SparkSession and the table name as parameters */ def testConcurrentChangeWithTable(testName: String)( createTable: (SparkSession, String) => Unit, actionToTest: (SparkSession, String) => Unit, concurrentChange: (SparkSession, String) => Unit): Unit = { val tblName = "metastoreTable" test(testName) { withTable(tblName) { testConcurrentChangeBase(tblName)( createTable, actionToTest, concurrentChange ) } } } /** * Creates a method to remove a column from the table by taking column as an argument. */ def dropColFromSampleTable(col: String): (SparkSession, String) => Unit = { (spark: SparkSession, tblPath: String) => { spark.read.format("delta").load(tblPath) .drop(col) .write .format("delta") .mode("overwrite") .option("overwriteSchema", "true") .save(tblPath) } } /** * Adding a column to the schema will result in the blocked thread appending to the table * with null values for the new column. */ testConcurrentChange("write - add a column concurrently")( createTable = (spark: SparkSession, tblPath: String) => { spark.range(10).write.format("delta").save(tblPath) }, actionToTest = (spark: SparkSession, tblPath: String) => { spark.range(11, 20).write.format("delta") .mode("append") .save(tblPath) val appendedCol2Values = spark.read.format("delta") .load(tblPath) .filter(col("id") <= 20) .select("col2") .distinct() .collect() .toList assert(appendedCol2Values == List(Row(null))) }, concurrentChange = (spark: SparkSession, tblPath: String) => { spark.range(21, 30).withColumn("col2", lit(2)).write .format("delta") .mode("append") .option("mergeSchema", "true") .save(tblPath) } ) /** * Removing a column while a query is in running should throw an analysis * exception */ testConcurrentChange("write - remove a column concurrently")( createTable = (spark: SparkSession, tblPath: String) => { spark.range(10).withColumn("col2", lit(1)) .write .format("delta") .save(tblPath) }, actionToTest = (spark: SparkSession, tblPath: String) => { val e = intercept[AnalysisException] { spark.range(11, 20) .withColumn("col2", lit(1)).write.format("delta") .mode("append") .save(tblPath) } assert(e.getMessage.contains( "A schema mismatch detected when writing to the Delta table")) }, concurrentChange = dropColFromSampleTable("col2") ) /** * Removing a column while performing a delete should be caught while * writing the deleted files(i.e files with rows that were not deleted). */ testConcurrentChange("delete - remove a column concurrently")( createTable = (spark: SparkSession, tblPath: String) => { spark.range(10).withColumn("col2", lit(1)) .write .format("delta") .save(tblPath) }, actionToTest = (spark: SparkSession, tblPath: String) => { val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tblPath) val e = intercept[Exception] { deltaTable.delete(col("id") === 1) } assert(e.getMessage.contains(s"Can't resolve column col2")) }, concurrentChange = dropColFromSampleTable("col2") ) /** * Removing a column(referenced in condition) while performing a delete will * result in a no-op. */ testConcurrentChange("test delete query against a concurrent query which removes the" + " delete condition column" )( createTable = (spark: SparkSession, tblPath: String) => { spark.range(10).withColumn("col2", lit(1)) .repartition(2) .write .format("delta") .save(tblPath) }, actionToTest = (spark: SparkSession, tblPath: String) => { val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tblPath) deltaTable.delete(col("id") === 1) // check if delete is no-op checkAnswer( sql(s"SELECT * FROM delta.`$tblPath`"), Seq(Row(1), Row(1), Row(1), Row(1), Row(1), Row(1), Row(1), Row(1), Row(1), Row(1))) }, concurrentChange = dropColFromSampleTable("id") ) /** * An update command that has to rewrite files will have the old schema, * we catch the outdated schema during the write. */ testConcurrentChange("update - remove a column concurrently")( createTable = (spark: SparkSession, tblPath: String) => { spark.range(10).withColumn("col2", lit(1)) .write .format("delta") .save(tblPath) }, actionToTest = (spark: SparkSession, tblPath: String) => { val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tblPath) val e = intercept[AnalysisException] { deltaTable.update(col("id") =!= 1, Map("col2" -> lit(-1))) } assert(e.getMessage.contains(s"Can't resolve column col2")) }, concurrentChange = dropColFromSampleTable("col2") ) /** * Removing a column(referenced in condition) while performing a update will * result in a no-op. */ testConcurrentChange("test update query against a concurrent query which removes the" + " update condition column" )( createTable = (spark: SparkSession, tblPath: String) => { spark.range(10).withColumn("col2", lit(1)) .repartition(2) .write .format("delta") .save(tblPath) }, actionToTest = (spark: SparkSession, tblPath: String) => { val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tblPath) deltaTable.update(col("id") === 1, Map("id" -> lit("2"))) // check if update is no-op checkAnswer( sql(s"SELECT * FROM delta.`$tblPath`"), Seq(Row(1), Row(1), Row(1), Row(1), Row(1), Row(1), Row(1), Row(1), Row(1), Row(1))) }, concurrentChange = dropColFromSampleTable("id") ) /** * Concurrently drop column in merge condition. Merge command detects the schema change while * resolving the target and throws a DeltaAnalysisException */ testConcurrentChange("merge - remove a column in merge condition concurrently")( createTable = (spark: SparkSession, tblPath: String) => { spark.range(10).withColumn("col2", lit(1)) .write .format("delta") .save(tblPath) }, actionToTest = (spark: SparkSession, tblPath: String) => { val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tblPath) val sourceDf = spark.range(10).withColumn("col2", lit(2)) val e = intercept[DeltaAnalysisException] { deltaTable.as("t1") .merge(sourceDf.as("t2"), "t1.id == t2.id") .whenNotMatched() .insertAll() .whenMatched() .updateAll() .execute() } checkErrorMatchPVals( e, "DELTA_SCHEMA_CHANGE_SINCE_ANALYSIS", parameters = Map( "schemaDiff" -> ".*id.*", "legacyFlagMessage" -> "" ) ) }, concurrentChange = dropColFromSampleTable("id") ) /** * Concurrently drop column not in merge condition but in target. Merge command detects the schema * change while resolving the target and throws a DeltaAnalysisException */ testConcurrentChange("merge - remove a column not in merge condition concurrently")( createTable = (spark: SparkSession, tblPath: String) => { spark.range(10).withColumn("col2", lit(1)) .write .format("delta") .save(tblPath) }, actionToTest = (spark: SparkSession, tblPath: String) => { val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tblPath) val sourceDf = spark.range(10).withColumn("col2", lit(2)) val e = intercept[DeltaAnalysisException] { deltaTable.as("t1") .merge(sourceDf.as("t2"), "t1.id == t2.id") .whenNotMatched() .insertAll() .whenMatched() .updateAll() .execute() } checkErrorMatchPVals( e, "DELTA_SCHEMA_CHANGE_SINCE_ANALYSIS", parameters = Map( "schemaDiff" -> ".*col2.*", "legacyFlagMessage" -> "" ) ) }, concurrentChange = dropColFromSampleTable("col2") ) /** * Alter table to add a column and at the same time add a column concurrently. */ testConcurrentChangeWithTable("alter table add column - remove column and add same column")( createTable = (spark: SparkSession, tblName: String) => { spark.range(10).write.format("delta").saveAsTable(tblName) }, actionToTest = (spark: SparkSession, tblName: String) => { val e = intercept[AnalysisException] { spark.sql(s"ALTER TABLE `$tblName` ADD COLUMNS (col2 string)") } assert(e.getMessage.contains("Found duplicate column(s) in adding columns: col2")) }, concurrentChange = (spark: SparkSession, tblName: String) => { spark.read.format("delta").table(tblName) .withColumn("col2", lit(1)) .write .format("delta") .option("overwriteSchema", "true") .mode("overwrite") .saveAsTable(tblName) } ) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/ShowDeltaTableColumnsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.functions.struct import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.Utils class ShowDeltaTableColumnsSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with DeltaTestUtilsForTempViews { import testImplicits._ private val outputColumnNames = Seq("col_name") private val outputColumnValues = Seq(Seq("column1"), Seq("column2")) protected def checkResult( result: DataFrame, expected: Seq[Seq[Any]], columns: Seq[String]): Unit = { checkAnswer( result.select(columns.head, columns.tail: _*), expected.map { x => Row(x: _*)}) assert(result.columns.toSeq == outputColumnNames) } private def showDeltaColumnsTest( fileToTableNameMapper: File => String, schemaName: Option[String] = None): Unit = { withDatabase("delta") { val tempDir = Utils.createTempDir() Seq(1 -> 1) .toDF("column1", "column2") .write .format("delta") .mode("overwrite") .save(tempDir.toString) val finalSchema = if (schemaName.nonEmpty) s"FROM ${schemaName.get}" else "" checkResult(sql(s"SHOW COLUMNS IN ${fileToTableNameMapper(tempDir)} $finalSchema"), outputColumnValues, outputColumnNames) } } test("delta table: table identifier") { showDeltaColumnsTest(f => s"delta.`${f.toString}`") } test("delta table: table name with separated schema name") { showDeltaColumnsTest(f => s"`${f.toString}`", schemaName = Some("delta")) } test("non-delta table: table identifier with catalog table") { // Non-Delta table represent by catalog identifier (e.g.: sales.line_ite) is supported in // SHOW COLUMNS command. withTable("show_columns") { sql(s""" |CREATE TABLE show_columns(column1 INT, column2 INT) |USING parquet |COMMENT "describe a non delta table" """.stripMargin) checkResult(sql("SHOW COLUMNS IN show_columns"), outputColumnValues, outputColumnNames) } } test("delta table: table name not found") { val fakeTableName = s"test_table" val schemaName = s"delta" showDeltaColumnsTest(f => s"$schemaName.`${f.toString}`") val e = intercept[AnalysisException] { sql(s"SHOW COLUMNS IN `$fakeTableName` IN $schemaName") } assert(e.getMessage().contains(s"Table or view not found: $schemaName.$fakeTableName") || e.getMessage().contains(s"table or view `$schemaName`.`$fakeTableName` cannot be found")) } test("delta table: check duplicated schema name") { // When `schemaName` and `tableIdentity.database` both exists, we will throw error if they are // not the same. val schemaName = s"default" val tableName = s"test_table" val fakeSchemaName = s"epsilon" withTable(tableName) { sql(s""" |CREATE TABLE $tableName(column1 INT, column2 INT) |USING delta """.stripMargin) // when no schema name provided, default schema name is `default`. checkResult( sql(s"SHOW COLUMNS IN $tableName"), outputColumnValues, outputColumnNames) checkResult( sql(s"SHOW COLUMNS IN $schemaName.$tableName"), outputColumnValues, outputColumnNames) var e = intercept[AnalysisException] { sql(s"SHOW COLUMNS IN $tableName IN $fakeSchemaName") } assert(e .getMessage() .contains(s"Table or view not found: $fakeSchemaName.$tableName") || e.getMessage() .contains(s"table or view `$fakeSchemaName`.`$tableName` cannot be found")) e = intercept[AnalysisException] { sql(s"SHOW COLUMNS IN $fakeSchemaName.$tableName IN $schemaName") } assert(e .getMessage() .contains(s"Table or view not found: $fakeSchemaName.$tableName") || e.getMessage() .contains(s"table or view `$fakeSchemaName`.`$tableName` cannot be found")) checkShowColumns(fakeSchemaName, schemaName, intercept[AnalysisException] { sql(s"SHOW COLUMNS IN $schemaName.$tableName IN $fakeSchemaName") }) } } testWithTempView(s"show columns on temp view should fallback to Spark") { isSQLTempView => val tableName = "test_table_2" withTable(tableName) { Seq(1 -> 1) .toDF("column1", "column2") .write .format("delta") .saveAsTable(tableName) val viewName = "v" createTempViewFromTable(tableName, isSQLTempView) checkResult(sql(s"SHOW COLUMNS IN $viewName"), outputColumnValues, outputColumnNames) } } test(s"delta table: show columns on a nested column") { withTempDir { tempDir => (70.to(79).seq ++ 75.to(79).seq) .toDF("id") .withColumn("nested", struct(struct('id + 2 as "b", 'id + 3 as "c") as "sub")) .write .format("delta") .save(tempDir.toString) checkResult( sql(s"SHOW COLUMNS IN delta.`${tempDir.toString}`"), Seq(Seq("id"), Seq("nested")), outputColumnNames) } } test("delta table: respect the Spark configuration on whether schema name is case sensitive") { withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { checkShowColumns("DELTA", "delta", intercept[AnalysisException] { showDeltaColumnsTest(f => s"delta.`${f.toString}`", schemaName = Some("DELTA")) }) checkShowColumns("delta", "DELTA", intercept[AnalysisException] { showDeltaColumnsTest(f => s"DELTA.`${f.toString}`", schemaName = Some("delta")) }) } withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") { showDeltaColumnsTest(f => s"delta.`${f.toString}`", schemaName = Some("DELTA")) showDeltaColumnsTest(f => s"DELTA.`${f.toString}`", schemaName = Some("delta")) } } private def checkShowColumns(schema1: String, schema2: String, e: AnalysisException): Unit = { val expectedMessage = Seq( s"SHOW COLUMNS with conflicting databases: '$schema1' != '$schema2'", // SPARK-3.5 s"SHOW COLUMNS with conflicting namespaces: `$schema1` != `$schema2`") // SPARK-4.0 assert(expectedMessage.exists(e.getMessage().contains)) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/SnapshotManagementSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.{File, RandomAccessFile} import java.util.concurrent.CountDownLatch import scala.collection.mutable import com.databricks.spark.util.{Log4jUsageLogger, UsageRecord} import org.apache.spark.sql.delta.DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME import org.apache.spark.sql.delta.DeltaTestUtils.{verifyBackfilled, verifyUnbackfilled, BOOLEAN_DOMAIN} import org.apache.spark.sql.delta.coordinatedcommits.{CommitCoordinatorBuilder, CommitCoordinatorProvider, CoordinatedCommitsBaseSuite, CoordinatedCommitsUsageLogs, InMemoryCommitCoordinator} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.storage.LocalLogStore import org.apache.spark.sql.delta.storage.LogStore.logStoreClassConfKey import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.{DeltaCommitFileProvider, FileNames, JsonUtils} import io.delta.storage.LogStore import io.delta.storage.commit.{Commit, CommitCoordinatorClient, GetCommitsResponse, TableDescriptor} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.FileStatus import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.SparkException import org.apache.spark.sql.QueryTest import org.apache.spark.sql.SparkSession import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.storage.StorageLevel class SnapshotManagementSuite extends QueryTest with DeltaSQLTestUtils with SharedSparkSession with DeltaSQLCommandTest with CoordinatedCommitsBaseSuite { protected override def sparkConf = { // Disable loading protocol and metadata from checksum file. Otherwise, creating a Snapshot // won't touch the checkpoint file and we won't be able to retry. super.sparkConf .set(DeltaSQLConf.USE_PROTOCOL_AND_METADATA_FROM_CHECKSUM_ENABLED.key, "false") } /** * Truncate an existing checkpoint file to create a corrupt file. * * @param path the Delta table path * @param checkpointVersion the checkpoint version to be updated * @param shouldBeEmpty whether to create an empty checkpoint file */ private def makeCorruptCheckpointFile( path: String, checkpointVersion: Long, shouldBeEmpty: Boolean, multipart: Option[(Int, Int)] = None): Unit = { if (multipart.isDefined) { val (part, totalParts) = multipart.get val checkpointFile = FileNames.checkpointFileWithParts(new Path(path, "_delta_log"), checkpointVersion, totalParts)(part - 1).toString assert(new File(checkpointFile).exists) val cp = new RandomAccessFile(checkpointFile, "rw") cp.setLength(if (shouldBeEmpty) 0 else 10) cp.close() } else { val checkpointFile = FileNames.checkpointFileSingular(new Path(path, "_delta_log"), checkpointVersion).toString assert(new File(checkpointFile).exists) val cp = new RandomAccessFile(checkpointFile, "rw") cp.setLength(if (shouldBeEmpty) 0 else 10) cp.close() } } private def deleteLogVersion(path: String, version: Long): Unit = { val deltaFile = new File( FileNames.unsafeDeltaFile(new Path(path, "_delta_log"), version).toString) assert(deltaFile.exists(), s"Could not find $deltaFile") assert(deltaFile.delete(), s"Failed to delete $deltaFile") } private def deleteCheckpointVersion(path: String, version: Long): Unit = { val deltaFile = new File( FileNames.checkpointFileSingular(new Path(path, "_delta_log"), version).toString) assert(deltaFile.exists(), s"Could not find $deltaFile") assert(deltaFile.delete(), s"Failed to delete $deltaFile") } private def testWithAndWithoutMultipartCheckpoint(name: String)(f: (Option[Int]) => Unit) = { testQuietly(name) { withSQLConf(DeltaSQLConf.DELTA_CHECKPOINT_PART_SIZE.key -> "1") { f(Some(1)) f(Some(2)) } f(None) } } testWithAndWithoutMultipartCheckpoint("recover from a corrupt checkpoint: previous checkpoint " + "doesn't exist") { partToCorrupt => withTempDir { tempDir => val path = tempDir.getCanonicalPath spark.range(10).write.format("delta").save(path) var deltaLog = DeltaLog.forTable(spark, path) deltaLog.checkpoint() DeltaLog.clearCache() deltaLog = DeltaLog.forTable(spark, path) val checkpointParts = deltaLog.snapshot.logSegment.checkpointProvider.topLevelFiles.size val multipart = partToCorrupt.map((_, checkpointParts)) // We have different code paths for empty and non-empty checkpoints for (testEmptyCheckpoint <- Seq(true, false)) { makeCorruptCheckpointFile(path, checkpointVersion = 0, shouldBeEmpty = testEmptyCheckpoint, multipart = multipart) DeltaLog.clearCache() // Checkpoint 0 is corrupted. Verify that we can still create the snapshot using // existing json files. DeltaLog.forTable(spark, path).snapshot } } } testWithAndWithoutMultipartCheckpoint("recover from a corrupt checkpoint: previous checkpoint " + "exists") { partToCorrupt => withTempDir { tempDir => // Create checkpoint 0 and 1 val path = tempDir.getCanonicalPath spark.range(10).write.format("delta").save(path) var deltaLog = DeltaLog.forTable(spark, path) deltaLog.checkpoint() spark.range(10).write.format("delta").mode("append").save(path) deltaLog.update() deltaLog.checkpoint() DeltaLog.clearCache() deltaLog = DeltaLog.forTable(spark, path) val checkpointParts = deltaLog.snapshot.logSegment.checkpointProvider.topLevelFiles.size val multipart = partToCorrupt.map((_, checkpointParts)) // We have different code paths for empty and non-empty checkpoints for (testEmptyCheckpoint <- Seq(true, false)) { makeCorruptCheckpointFile(path, checkpointVersion = 1, shouldBeEmpty = testEmptyCheckpoint, multipart = multipart) // Checkpoint 1 is corrupted. Verify that we can still create the snapshot using // checkpoint 0. DeltaLog.clearCache() DeltaLog.forTable(spark, path).snapshot } } } testWithAndWithoutMultipartCheckpoint("should not recover when the current checkpoint is " + "broken but we don't have the entire history") { partToCorrupt => withTempDir { tempDir => val path = tempDir.getCanonicalPath spark.range(10).write.format("delta").save(path) spark.range(10).write.format("delta").mode("append").save(path) DeltaLog.forTable(spark, path).checkpoint() deleteLogVersion(path, version = 0) DeltaLog.clearCache() val deltaLog = DeltaLog.forTable(spark, path) val checkpointParts = deltaLog.snapshot.logSegment.checkpointProvider.topLevelFiles.size val multipart = partToCorrupt.map((_, checkpointParts)) DeltaLog.clearCache() // We have different code paths for empty and non-empty checkpoints, and also different // code paths when listing with or without a checkpoint hint. for (testEmptyCheckpoint <- Seq(true, false)) { makeCorruptCheckpointFile(path, checkpointVersion = 1, shouldBeEmpty = testEmptyCheckpoint, multipart = multipart) // When finding a Delta log for the first time, we rely on _last_checkpoint hint val e = intercept[Exception] { DeltaLog.forTable(spark, path).snapshot } if (testEmptyCheckpoint) { // - checkpoint 1 is NOT in the list result // - try to get an alternative LogSegment in `getLogSegmentForVersion` // - fail to get an alternative LogSegment // - throw the below exception assert(e.isInstanceOf[IllegalStateException] && e.getMessage.contains( "Couldn't find all part files of the checkpoint version: 1")) } else { // - checkpoint 1 is in the list result // - Snapshot creation triggers state reconstruction // - fail to read protocol+metadata from checkpoint 1 // - throw FileReadException // - fail to get an alternative LogSegment // - cannot find log file 0 so throw the above checkpoint 1 read failure // Guava cache wraps the root cause assert(e.isInstanceOf[SparkException] && e.getMessage.contains("0001.checkpoint") && e.getMessage.contains("Encountered error while reading file")) } } } } testWithAndWithoutMultipartCheckpoint("should not recover when both the current and previous " + "checkpoints are broken") { partToCorrupt => withTempDir { tempDir => val path = tempDir.getCanonicalPath val staleLog = DeltaLog.forTable(spark, path) DeltaLog.clearCache() spark.range(10).write.format("delta").save(path) val deltaLog = DeltaLog.forTable(spark, path) deltaLog.checkpoint() DeltaLog.clearCache() val checkpointParts0 = DeltaLog.forTable(spark, path).snapshot.logSegment.checkpointProvider.topLevelFiles.size spark.range(10).write.format("delta").mode("append").save(path) deltaLog.update() deltaLog.checkpoint() deleteLogVersion(path, version = 0) DeltaLog.clearCache() val checkpointParts1 = DeltaLog.forTable(spark, path).snapshot.logSegment.checkpointProvider.topLevelFiles.size makeCorruptCheckpointFile(path, checkpointVersion = 0, shouldBeEmpty = false, multipart = partToCorrupt.map((_, checkpointParts0))) val multipart = partToCorrupt.map((_, checkpointParts1)) // We have different code paths for empty and non-empty checkpoints for (testEmptyCheckpoint <- Seq(true, false)) { makeCorruptCheckpointFile(path, checkpointVersion = 1, shouldBeEmpty = testEmptyCheckpoint, multipart = multipart) // The code paths are different, but the error and message end up being the same: // // testEmptyCheckpoint = true: // - checkpoint 1 is NOT in the list result. // - fallback to load version 0 using checkpoint 0 // - fail to read checkpoint 0 // - cannot find log file 0 so throw the above checkpoint 0 read failure // // testEmptyCheckpoint = false: // - checkpoint 1 is in the list result. // - Snapshot creation triggers state reconstruction // - fail to read protocol+metadata from checkpoint 1 // - fallback to load version 0 using checkpoint 0 // - fail to read checkpoint 0 // - cannot find log file 0 so throw the original checkpoint 1 read failure val e = intercept[SparkException] { staleLog.update() } val version = if (testEmptyCheckpoint) 0 else 1 assert(e.getMessage.contains(f"$version%020d.checkpoint") && e.getMessage.contains("Encountered error while reading file")) } } } test("should throw a clear exception when checkpoint exists but its corresponding delta file " + "doesn't exist") { withTempDir { tempDir => val path = tempDir.getCanonicalPath val staleLog = DeltaLog.forTable(spark, path) DeltaLog.clearCache() spark.range(10).write.format("delta").save(path) DeltaLog.forTable(spark, path).checkpoint() // Delete delta files new File(tempDir, "_delta_log").listFiles().filter(_.getName.endsWith(".json")) .foreach(_.delete()) val e = intercept[IllegalStateException] { staleLog.update() } assert(e.getMessage.contains("Could not find any delta files for version 0")) } } test("should throw an exception when trying to load a non-existent version") { withTempDir { tempDir => val path = tempDir.getCanonicalPath val staleLog = DeltaLog.forTable(spark, path) DeltaLog.clearCache() spark.range(10).write.format("delta").save(path) DeltaLog.forTable(spark, path).checkpoint() val e = intercept[IllegalStateException] { staleLog.getSnapshotAt(2) } assert(e.getMessage.contains("Trying to load a non-existent version 2")) } } test("should throw a clear exception when the checkpoint is corrupt " + "but could not find any delta files") { withTempDir { tempDir => val path = tempDir.getCanonicalPath val staleLog = DeltaLog.forTable(spark, path) DeltaLog.clearCache() spark.range(10).write.format("delta").save(path) DeltaLog.forTable(spark, path).checkpoint() // Delete delta files new File(tempDir, "_delta_log").listFiles().filter(_.getName.endsWith(".json")) .foreach(_.delete()) if (coordinatedCommitsEnabledInTests) { new File(new File(tempDir, "_delta_log"), "_staged_commits") .listFiles() .filter(_.getName.endsWith(".json")) .foreach(_.delete()) } makeCorruptCheckpointFile(path, checkpointVersion = 0, shouldBeEmpty = false) val e = intercept[IllegalStateException] { staleLog.update() } assert(e.getMessage.contains("Could not find any delta files for version 0")) } } test("verifyDeltaVersions") { import SnapshotManagement.verifyDeltaVersions // empty array verifyDeltaVersions( spark, versions = Array.empty, expectedStartVersion = None, expectedEndVersion = None, cachedSnapshot = None) // contiguous versions verifyDeltaVersions( spark, versions = Array(1, 2, 3), expectedStartVersion = None, expectedEndVersion = None, cachedSnapshot = None) // contiguous versions with correct `expectedStartVersion` and `expectedStartVersion` verifyDeltaVersions( spark, versions = Array(1, 2, 3), expectedStartVersion = None, expectedEndVersion = Some(3), cachedSnapshot = None) verifyDeltaVersions( spark, versions = Array(1, 2, 3), expectedStartVersion = Some(1), expectedEndVersion = None, cachedSnapshot = None) verifyDeltaVersions( spark, versions = Array(1, 2, 3), expectedStartVersion = Some(1), expectedEndVersion = Some(3), cachedSnapshot = None) // `expectedStartVersion` or `expectedEndVersion` doesn't match intercept[IllegalArgumentException] { verifyDeltaVersions( spark, versions = Array(1, 2), expectedStartVersion = Some(0), expectedEndVersion = None, cachedSnapshot = None) } intercept[IllegalArgumentException] { verifyDeltaVersions( spark, versions = Array(1, 2), expectedStartVersion = None, expectedEndVersion = Some(3), cachedSnapshot = None) } intercept[IllegalArgumentException] { verifyDeltaVersions( spark, versions = Array.empty, expectedStartVersion = Some(0), expectedEndVersion = None, cachedSnapshot = None) } intercept[IllegalArgumentException] { verifyDeltaVersions( spark, versions = Array.empty, expectedStartVersion = None, expectedEndVersion = Some(3), cachedSnapshot = None) } // non contiguous versions intercept[IllegalStateException] { verifyDeltaVersions( spark, versions = Array(1, 3), expectedStartVersion = None, expectedEndVersion = None, cachedSnapshot = None) } // duplicates in versions intercept[IllegalStateException] { verifyDeltaVersions( spark, versions = Array(1, 2, 2, 3), expectedStartVersion = None, expectedEndVersion = None, cachedSnapshot = None) } // unsorted versions intercept[IllegalStateException] { verifyDeltaVersions( spark, versions = Array(3, 2, 1), expectedStartVersion = None, expectedEndVersion = None, cachedSnapshot = None) } // ----------------------------------------------------- // | Usage logs validation for non-contiguous versions | // ----------------------------------------------------- /** * Helper function to validate the usage log properties for the * given `usageLogs` and `expected*` values. */ def validateUsageLogProperties( usageLogs: Seq[UsageRecord], expectedStartVersion: Long, expectedEndVersion: Long, expectedVersionToLoad: Long, expectedLatestSnapshotVersion: Long, expectedLatestCheckpointVersion: Long, shouldChecksumOptPresent: Boolean): Unit = { assert(usageLogs.size == 1) val usageLog = usageLogs.head // `tags.opType` should be "delta.exceptions.deltaVersionsNotContiguous" assert(usageLog.tags.getOrElse("opType", "null") == "delta.exceptions.deltaVersionsNotContiguous") val blob = JsonUtils.fromJson[Map[String, Any]](usageLog.blob) // `blob` validation assert(blob.get("startVersion").exists(_.toString.toLong == expectedStartVersion)) assert(blob.get("endVersion").exists(_.toString.toLong == expectedEndVersion)) assert(blob.get("versionToLoad").exists(_.toString.toLong == expectedVersionToLoad)) assert( blob.get("unsafeVolatileSnapshot.latestSnapshotVersion").exists(_.toString.toLong == expectedLatestSnapshotVersion)) assert( blob.get("unsafeVolatileSnapshot.latestCheckpointVersion").exists(_.toString.toLong == expectedLatestCheckpointVersion)) // `stackTrace` should contain the entire stack trace, // here we verify the starting of the stack trace. assert(blob.get("stackTrace").exists(_.toString.startsWith( "org.apache.spark.sql.delta.SnapshotManagement$.verifyDeltaVersions"))) // Check whether `unsafeVolatileSnapshot.checksumOpt` is present or not assert(blob.contains("unsafeVolatileSnapshot.checksumOpt") == shouldChecksumOptPresent) } // 1. Basic usage log validation. val usageLogs = Log4jUsageLogger.track { intercept[IllegalStateException] { verifyDeltaVersions( spark, versions = Array(1, 3), expectedStartVersion = None, expectedEndVersion = None, cachedSnapshot = None) } }.filter(_.metric == "tahoeEvent") validateUsageLogProperties( usageLogs, expectedStartVersion = 1, expectedEndVersion = 3, expectedVersionToLoad = -1, expectedLatestSnapshotVersion = -1, expectedLatestCheckpointVersion = -1, shouldChecksumOptPresent = false) // 2. Usage log validation with `expectedStartVersion`, `expectedEndVersion` // and `cachedSnapshot`. withTempDir { dir => val path = dir.getCanonicalPath import testImplicits._ // Commit 0 - to trigger the initial snapshot construction for version 0 Seq(1).toDF().write.format("delta").mode("overwrite").save(path) val snapshot = DeltaLog.forTable(spark, path).update() val usageLogs = Log4jUsageLogger.track { intercept[IllegalStateException] { verifyDeltaVersions( spark, versions = Array(1, 3), expectedStartVersion = Some(1), expectedEndVersion = Some(4), cachedSnapshot = Some(snapshot)) } }.filter(_.metric == "tahoeEvent") validateUsageLogProperties( usageLogs, expectedStartVersion = 1, expectedEndVersion = 3, expectedVersionToLoad = 4, expectedLatestSnapshotVersion = 0, expectedLatestCheckpointVersion = -1, shouldChecksumOptPresent = true) } } test("configurable snapshot cache storage level") { withTempDir { tempDir => val path = tempDir.getCanonicalPath spark.range(10).write.format("delta").save(path) DeltaLog.clearCache() // Corrupted snapshot tests leave a cached snapshot not tracked by the DeltaLog cache sparkContext.getPersistentRDDs.foreach(_._2.unpersist()) assert(sparkContext.getPersistentRDDs.isEmpty) withSQLConf(DeltaSQLConf.DELTA_SNAPSHOT_CACHE_STORAGE_LEVEL.key -> "DISK_ONLY") { DeltaLog.forTable(spark, path).snapshot.stateDS.collect() val persistedRDDs = sparkContext.getPersistentRDDs assert(persistedRDDs.size == 1) assert(persistedRDDs.values.head.getStorageLevel == StorageLevel.DISK_ONLY) } DeltaLog.clearCache() assert(sparkContext.getPersistentRDDs.isEmpty) withSQLConf(DeltaSQLConf.DELTA_SNAPSHOT_CACHE_STORAGE_LEVEL.key -> "NONE") { DeltaLog.forTable(spark, path).snapshot.stateDS.collect() val persistedRDDs = sparkContext.getPersistentRDDs assert(persistedRDDs.size == 1) assert(persistedRDDs.values.head.getStorageLevel == StorageLevel.NONE) } DeltaLog.clearCache() assert(sparkContext.getPersistentRDDs.isEmpty) withSQLConf(DeltaSQLConf.DELTA_SNAPSHOT_CACHE_STORAGE_LEVEL.key -> "invalid") { intercept[IllegalArgumentException] { spark.read.format("delta").load(path).collect() } } } } test("SerializableFileStatus json serialization/deserialization") { val testCases = Seq( SerializableFileStatus(path = "xyz", length = -1, isDir = true, modificationTime = 0) -> """{"path":"xyz","length":-1,"isDir":true,"modificationTime":0}""", SerializableFileStatus( path = "s3://a.b/pq", length = 123L, isDir = false, modificationTime = 246L) -> """{"path":"s3://a.b/pq","length":123,"isDir":false,"modificationTime":246}""" ) for ((obj, json) <- testCases) { assert(JsonUtils.toJson(obj) == json) val status = JsonUtils.fromJson[SerializableFileStatus](json) assert(status.modificationTime === obj.modificationTime) assert(status.isDir === obj.isDir) assert(status.length === obj.length) assert(status.path === obj.path) } } test("getLogSegmentAfterCommit can find specified commit") { withTempDir { tempDir => val path = tempDir.getCanonicalPath val log = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) val oldLogSegment = log.snapshot.logSegment spark.range(10).write.format("delta").save(path) val newLogSegment = log.snapshot.logSegment assert(log.getLogSegmentAfterCommit( log.snapshot.tableCommitCoordinatorClientOpt, catalogTableOpt = None, oldLogSegment.checkpointProvider) === newLogSegment) spark.range(10).write.format("delta").mode("append").save(path) val fs = log.logPath.getFileSystem(log.newDeltaHadoopConf()) val commitFileProvider = DeltaCommitFileProvider(log.snapshot) intercept[IllegalArgumentException] { val commitFile = fs.getFileStatus(commitFileProvider.deltaFile(1)) val commit = new Commit(1, commitFile, 0) // Version exists, but not contiguous with old logSegment log.getLogSegmentAfterCommit( 1, None, oldLogSegment, commit, None, None, EmptyCheckpointProvider) } intercept[IllegalArgumentException] { val commitFile = fs.getFileStatus(commitFileProvider.deltaFile(0)) val commit = new Commit(0, commitFile, 0) // Version exists, but newLogSegment already contains it log.getLogSegmentAfterCommit( 0, None, newLogSegment, commit, None, None, EmptyCheckpointProvider) } assert(log.getLogSegmentAfterCommit( log.snapshot.tableCommitCoordinatorClientOpt, catalogTableOpt = None, oldLogSegment.checkpointProvider) === log.snapshot.logSegment) } } testQuietly("checkpoint/json not found when executor restart " + "after expired checkpoints in the snapshot cache are cleaned up") { withTempDir { tempDir => // Create checkpoint 1 and 3 val path = tempDir.getCanonicalPath spark.range(10).write.format("delta").save(path) spark.range(10).write.format("delta").mode("append").save(path) val deltaLog = DeltaLog.forTable(spark, path) deltaLog.checkpoint() spark.range(10).write.format("delta").mode("append").save(path) spark.range(10).write.format("delta").mode("append").save(path) deltaLog.checkpoint() // simulate checkpoint 1 expires and is cleaned up deleteCheckpointVersion(path, 1) // simulate executor hangs and restart, cache invalidation deltaLog.snapshot.uncache() spark.read.format("delta").load(path).collect() } } test("getUpdatedLogSegment without new files returns the original log segment") { withTempDir { tempDir => // Create checkpoint 1 and 3 val path = tempDir.getCanonicalPath spark.range(10).write.format("delta").save(path) spark.range(10).write.format("delta").mode("append").save(path) val deltaLog = DeltaLog.forTable(spark, path) deltaLog.checkpoint() val snapshot = deltaLog.update() val (updatedLogSegment, _) = deltaLog.getUpdatedLogSegment( snapshot.logSegment, tableCommitCoordinatorClientOpt = None, catalogTableOpt = None ) assert(updatedLogSegment === snapshot.logSegment) } } } class SnapshotManagementWithCoordinatedCommitsBatch1Suite extends SnapshotManagementSuite { override def coordinatedCommitsBackfillBatchSize: Option[Int] = Some(1) } class SnapshotManagementWithCoordinatedCommitsBatch2Suite extends SnapshotManagementSuite { override def coordinatedCommitsBackfillBatchSize: Option[Int] = Some(2) } class SnapshotManagementWithCoordinatedCommitsBatch100Suite extends SnapshotManagementSuite { override def coordinatedCommitsBackfillBatchSize: Option[Int] = Some(100) } class CountDownLatchLogStore(sparkConf: SparkConf, hadoopConf: Configuration) extends LocalLogStore(sparkConf, hadoopConf) { override def listFrom(path: Path, hadoopConf: Configuration): Iterator[FileStatus] = { val files = super.listFrom(path, hadoopConf).toSeq if (ConcurrentBackfillCommitCoordinatorClient.beginConcurrentBackfills) { CountDownLatchLogStore.listFromCalled.countDown() } files.iterator } } object CountDownLatchLogStore { val listFromCalled = new CountDownLatch(1) } case class ConcurrentBackfillCommitCoordinatorClient( synchronousBackfillThreshold: Long, override val batchSize: Long ) extends InMemoryCommitCoordinator(batchSize) { private val deferredBackfills: mutable.Map[Long, () => Unit] = mutable.Map.empty override def getCommits( tableDesc: TableDescriptor, startVersion: java.lang.Long, endVersion: java.lang.Long): GetCommitsResponse = { if (ConcurrentBackfillCommitCoordinatorClient.beginConcurrentBackfills) { CountDownLatchLogStore.listFromCalled.await() logInfo(s"Finishing pending backfills concurrently: ${deferredBackfills.keySet}") deferredBackfills.keys.toSeq.sorted.foreach((version: Long) => deferredBackfills(version)()) deferredBackfills.clear() } super.getCommits(tableDesc, startVersion, endVersion) } override def backfill( logStore: LogStore, hadoopConf: Configuration, logPath: Path, version: Long, fileStatus: FileStatus): Unit = { if (version > synchronousBackfillThreshold && ConcurrentBackfillCommitCoordinatorClient.deferBackfills) { deferredBackfills(version) = () => super.backfill(logStore, hadoopConf, logPath, version, fileStatus) } else { super.backfill(logStore, hadoopConf, logPath, version, fileStatus) } } } object ConcurrentBackfillCommitCoordinatorClient { var deferBackfills = false var beginConcurrentBackfills = false } object ConcurrentBackfillCommitCoordinatorBuilder extends CommitCoordinatorBuilder { val batchSize = 5 private lazy val concurrentBackfillCommitCoordinatorClient = ConcurrentBackfillCommitCoordinatorClient(synchronousBackfillThreshold = 2, batchSize) override def getName: String = "awaiting-commit-coordinator" override def build(spark: SparkSession, conf: Map[String, String]): CommitCoordinatorClient = { concurrentBackfillCommitCoordinatorClient } } /** * Setup (Assuming batch size = 5 & synchronousBackfillThreshold = 2): * - LogStore contains backfilled commits [0, 2] * - CommitCoordinatorClient contains unbackfilled commits [3, ...] * - Backfills are pending for versions [3, 5] * * Goal: Create a gap for versions [3, 5] in the LogStore and CommitCoordinatorClient listings. * * Step 1: LogStore retrieves delta files for versions [0, 2] from the file system. * Step 2: Wait on the latch to ensure step (1) is completed before step (3) begins. * Step 3: Backfill commits [3, 5] from CommitCoordinatorClient to LogStore using deferredBackfills * map. * Step 4: CommitCoordinatorClient returns commits [6, ...] (if valid). * * Test that the code correctly handles the gap in the LogStore and CommitCoordinatorClient listings * by making an additional call to LogStore to fetch versions [3, 5]. */ class SnapshotManagementParallelListingSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { override protected def sparkConf: SparkConf = super.sparkConf.set(logStoreClassConfKey, classOf[CountDownLatchLogStore].getName) override protected def beforeEach(): Unit = { super.beforeEach() CommitCoordinatorProvider.clearNonDefaultBuilders() CommitCoordinatorProvider.registerBuilder(ConcurrentBackfillCommitCoordinatorBuilder) ConcurrentBackfillCommitCoordinatorClient.beginConcurrentBackfills = false ConcurrentBackfillCommitCoordinatorClient.deferBackfills = false } private def writeDeltaData(path: String, endVersion: Long): Unit = { spark.range(10).write.format("delta").save(path) (1L to endVersion).foreach( _ => spark.range(10).write.format("delta").mode("append").save(path)) } private def captureUsageRecordsAndGetSnapshot(dataPath: Path): (Snapshot, Seq[UsageRecord]) = { var snapshot: Snapshot = null val records = Log4jUsageLogger.track { snapshot = DeltaLog.forTable(spark, dataPath).update() } (snapshot, records) } private def verifyUsageRecords( records: Seq[UsageRecord], expectedNeedAdditionalFsListingCount: Int): Unit = { val filteredLogs = DeltaTestUtils.filterUsageRecords( records, CoordinatedCommitsUsageLogs.COMMIT_COORDINATOR_ADDITIONAL_LISTING_REQUIRED) assert(filteredLogs.size === expectedNeedAdditionalFsListingCount) } private def verifySnapshotBackfills(snapshot: Snapshot, backfillUntilInclusive: Long): Unit = { snapshot.logSegment.deltas.zipWithIndex.foreach { case (delta, index) => if (index <= backfillUntilInclusive) { verifyBackfilled(delta) } else { verifyUnbackfilled(delta) } } } /** * concurrentBackfills: Whether to defer backfills for versions > synchronousBackfillThreshold to * simulate concurrent backfills to test addition file-system listing. * tryIncludeGapAtTheEnd: Whether to include a gap in listing at end of the version range or * somewhere in the middle. */ BOOLEAN_DOMAIN.foreach { concurrentBackfills => BOOLEAN_DOMAIN.foreach { tryIncludeGapAtTheEnd => test( s"Backfills are properly reconciled with concurrentBackfills: $concurrentBackfills, " + s"tryIncludeGapAtTheEnd: $tryIncludeGapAtTheEnd") { ConcurrentBackfillCommitCoordinatorClient.deferBackfills = concurrentBackfills val batchSize = ConcurrentBackfillCommitCoordinatorBuilder.batchSize val endVersion = if (tryIncludeGapAtTheEnd) { batchSize } else { batchSize + 3 } withSQLConf( COORDINATED_COMMITS_COORDINATOR_NAME.defaultTablePropertyKey -> ConcurrentBackfillCommitCoordinatorBuilder.getName, DeltaSQLConf.DELTALOG_MINOR_COMPACTION_USE_FOR_READS.key -> "false") { withTempDir { tempDir => val path = tempDir.getCanonicalPath val dataPath = new Path(path) writeDeltaData(path, endVersion) // Invalidate cache to ensure re-listing. DeltaLog.invalidateCache(spark, dataPath) ConcurrentBackfillCommitCoordinatorClient.beginConcurrentBackfills = true val (snapshot, records) = captureUsageRecordsAndGetSnapshot(dataPath) val expectedNeedAdditionalFsListingCount = if (concurrentBackfills) { 1 } else { 0 } verifyUsageRecords(records, expectedNeedAdditionalFsListingCount) verifySnapshotBackfills(snapshot, backfillUntilInclusive = batchSize) } } } } } test("throws exception when additional listing also can't reconcile") { val batchSize = ConcurrentBackfillCommitCoordinatorBuilder.batchSize val endVersion = batchSize + 3 withSQLConf( COORDINATED_COMMITS_COORDINATOR_NAME.defaultTablePropertyKey -> ConcurrentBackfillCommitCoordinatorBuilder.getName, DeltaSQLConf.DELTALOG_MINOR_COMPACTION_USE_FOR_READS.key -> "false") { withTempDir { tempDir => val path = tempDir.getCanonicalPath val dataPath = new Path(path) writeDeltaData(path, endVersion) // Delete 5.json to create a permanent gap between file-system i.e. [0, 4] and // commit-store [6, 8] which would even an additional listing won't be able to reconcile. val deltaLog = DeltaLog.forTable(spark, dataPath) deltaLog.logPath.getFileSystem(deltaLog.newDeltaHadoopConf()).delete( FileNames.unsafeDeltaFile(deltaLog.logPath, batchSize), true) // Invalidate cache to ensure re-listing. DeltaLog.invalidateCache(spark, dataPath) val e = intercept[IllegalStateException] { DeltaLog.forTable(spark, dataPath).update() } assert(e.getMessage.contains("unexpectedly still requires additional file-system listing")) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/TableRedirectSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.io.File // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.coordinatedcommits.CoordinatedCommitsBaseSuite import org.apache.spark.sql.delta.redirect.{ DropRedirectInProgress, EnableRedirectInProgress, NoRedirectRule, PathBasedRedirectSpec, RedirectFeature, RedirectReaderWriter, RedirectReady, RedirectSpec, RedirectState, RedirectWriterOnly, TableRedirect, TableRedirectConfiguration } import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.util.JsonUtils import org.apache.commons.text.StringEscapeUtils import org.apache.hadoop.fs.Path import org.apache.spark.sql.{QueryTest, SaveMode, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.test.SharedSparkSession class TableRedirectSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with CoordinatedCommitsBaseSuite with DeltaCheckpointTestUtils with DeltaSQLTestUtils { private def validateState( deltaLog: DeltaLog, redirectState: RedirectState, sourceTablePath: File, destTablePath: File, feature: TableRedirect ): Unit = { val snapshot = deltaLog.update() assert(feature.isFeatureSet(snapshot.metadata)) val redirectConfig = feature.getRedirectConfiguration(snapshot.metadata).get val protocol = snapshot.protocol if (feature != RedirectWriterOnly) { assert(protocol.readerFeatureNames.contains(RedirectReaderWriterFeature.name)) assert(protocol.writerFeatureNames.contains(RedirectReaderWriterFeature.name)) } else { assert(!protocol.readerFeatureNames.contains(RedirectWriterOnlyFeature.name)) assert(protocol.writerFeatureNames.contains(RedirectWriterOnlyFeature.name)) } assert(redirectConfig.redirectState == redirectState) assert(redirectConfig.`type` == PathBasedRedirectSpec.REDIRECT_TYPE) val srcPath = sourceTablePath.getCanonicalPath val dstPath = destTablePath.getCanonicalPath val expectedSpecValue = s"""{"sourcePath":"$srcPath","destPath":"$dstPath"}""" assert(redirectConfig.specValue == expectedSpecValue) val redirectSpec = redirectConfig.spec.asInstanceOf[PathBasedRedirectSpec] assert(redirectSpec.sourcePath == srcPath) assert(redirectSpec.destPath == dstPath) } private def validateRemovedState(deltaLog: DeltaLog, feature: TableRedirect): Unit = { val snapshot = deltaLog.update() val protocol = snapshot.protocol assert(!feature.isFeatureSet(snapshot.metadata)) if (feature != RedirectWriterOnly) { assert(protocol.readerFeatureNames.contains(RedirectReaderWriterFeature.name)) assert(protocol.writerFeatureNames.contains(RedirectReaderWriterFeature.name)) } else { assert(!protocol.readerFeatureNames.contains(RedirectWriterOnlyFeature.name)) assert(protocol.writerFeatureNames.contains(RedirectWriterOnlyFeature.name)) } } def redirectTest( label: String, enableRedirect: Boolean )(f: (DeltaLog, File, File, CatalogTable) => Unit): Unit = { test(s"basic table redirect: $label") { withTempDir { sourceTablePath => withTempDir { destTablePath => withSQLConf(DeltaSQLConf.ENABLE_TABLE_REDIRECT_FEATURE.key -> enableRedirect.toString) { withTable("t1", "t2") { sql(s"CREATE external TABLE t1(c0 long) USING delta LOCATION '$sourceTablePath';") val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t1")) val deltaLog = DeltaLog.forTable(spark, new Path(sourceTablePath.getCanonicalPath)) f(deltaLog, sourceTablePath, destTablePath, catalogTable) } } } } } } Seq(RedirectReaderWriter, RedirectWriterOnly).foreach { feature => val featureName = feature.config.key Seq(true, false).foreach { hasCatalogTable => redirectTest(s"basic redirect: $featureName - " + s"hasCatalogTable: $hasCatalogTable", enableRedirect = false) { case (deltaLog, source, dest, catalogTable) => val snapshot = deltaLog.update() assert(!feature.isFeatureSet(snapshot.metadata)) val redirectSpec = new PathBasedRedirectSpec( source.getCanonicalPath, dest.getCanonicalPath ) val catalogTableOpt = if (hasCatalogTable) Some(catalogTable) else None val redirectType = PathBasedRedirectSpec.REDIRECT_TYPE // Step-1: Initiate table redirection and set to EnableRedirectInProgress state. feature.add(deltaLog, catalogTableOpt, redirectType, redirectSpec) validateState(deltaLog, EnableRedirectInProgress, source, dest, feature) // Step-2: Complete table redirection and set to RedirectReady state. feature.update(deltaLog, catalogTableOpt, RedirectReady, redirectSpec) validateState(deltaLog, RedirectReady, source, dest, feature) // Step-3: Start dropping table redirection and set to DropRedirectInProgress state. feature.update(deltaLog, catalogTableOpt, DropRedirectInProgress, redirectSpec) validateState(deltaLog, DropRedirectInProgress, source, dest, feature) // Step-4: Finish dropping table redirection and remove the property completely. feature.remove(deltaLog, catalogTableOpt) validateRemovedState(deltaLog, feature) // Step-5: Initiate table redirection and set to EnableRedirectInProgress state one // more time. withTempDir { destTablePath2 => val redirectSpec = new PathBasedRedirectSpec( source.getCanonicalPath, destTablePath2.getCanonicalPath ) feature.add(deltaLog, catalogTableOpt, redirectType, redirectSpec) validateState(deltaLog, EnableRedirectInProgress, source, destTablePath2, feature) // Step-6: Finish dropping table redirection and remove the property completely. feature.remove(deltaLog, catalogTableOpt) validateRemovedState(deltaLog, feature) } } redirectTest(s"Redirect $featureName: empty no redirect rules - " + s"hasCatalogTable: $hasCatalogTable", enableRedirect = false) { case (deltaLog, source, dest, catalogTable) => val snapshot = deltaLog.update() assert(!feature.isFeatureSet(snapshot.metadata)) val redirectSpec = new PathBasedRedirectSpec( source.getCanonicalPath, dest.getCanonicalPath ) val catalogTableOpt = if (hasCatalogTable) Some(catalogTable) else None val redirectType = PathBasedRedirectSpec.REDIRECT_TYPE // 0. Initialize table redirection by setting table to EnableRedirectInProgress state. feature.add(deltaLog, catalogTableOpt, redirectType, redirectSpec) validateState(deltaLog, EnableRedirectInProgress, source, dest, feature) // 1. INSERT should hit DELTA_COMMIT_INTERMEDIATE_REDIRECT_STATE because table is in // EnableRedirectInProgress, which doesn't allow any DML and DDL. val exception1 = intercept[DeltaIllegalStateException] { sql(s"insert into delta.`$source` values(1),(2),(3),(4),(5),(6)") } assert(exception1.getErrorClass == "DELTA_COMMIT_INTERMEDIATE_REDIRECT_STATE") // 2. DDL should hit DELTA_COMMIT_INTERMEDIATE_REDIRECT_STATE because table is in // EnableRedirectInProgress, which doesn't allow any DML and DDL. val exception2 = intercept[DeltaIllegalStateException] { sql(s"alter table delta.`$source` add column c3 long") } assert(exception2.getErrorClass == "DELTA_COMMIT_INTERMEDIATE_REDIRECT_STATE") // 3. Move to RedirectReady state. feature.update(deltaLog, catalogTableOpt, RedirectReady, redirectSpec) // 4. INSERT should hit DELTA_NO_REDIRECT_RULES_VIOLATED since the // no-redirect-rules is empty. validateState(deltaLog, RedirectReady, source, dest, feature) val exception3 = intercept[DeltaIllegalStateException] { sql(s"insert into delta.`$source` values(1),(2),(3),(4),(5),(6)") } assert(exception3.getErrorClass == "DELTA_NO_REDIRECT_RULES_VIOLATED") // 5. DDL should hit DELTA_NO_REDIRECT_RULES_VIOLATED since the // no-redirect-rules is empty. val exception4 = intercept[DeltaIllegalStateException] { sql(s"alter table delta.`$source` add column c3 long") } assert(exception4.getErrorClass == "DELTA_NO_REDIRECT_RULES_VIOLATED") // 6. Move to DropRedirectInProgress state. feature.update(deltaLog, catalogTableOpt, DropRedirectInProgress, redirectSpec) // 7. INSERT should hit DELTA_COMMIT_INTERMEDIATE_REDIRECT_STATE because table is in // DropRedirectInProgress, which doesn't allow any DML and DDL. validateState(deltaLog, DropRedirectInProgress, source, dest, feature) val exception5 = intercept[DeltaIllegalStateException] { sql(s"insert into delta.`$source` values(1),(2),(3),(4),(5),(6)") } assert(exception5.getErrorClass == "DELTA_COMMIT_INTERMEDIATE_REDIRECT_STATE") // 8. DDL should hit DELTA_COMMIT_INTERMEDIATE_REDIRECT_STATE because table is in // DropRedirectInProgress, which doesn't allow any DML and DDL. val exception6 = intercept[DeltaIllegalStateException] { sql(s"alter table delta.`$source` add column c3 long") } assert(exception6.getErrorClass == "DELTA_COMMIT_INTERMEDIATE_REDIRECT_STATE") } redirectTest(s"Redirect $featureName: no redirect rules - " + s"hasCatalogTable: $hasCatalogTable", enableRedirect = false) { case (deltaLog, source, dest, catalogTable) => val snapshot = deltaLog.update() assert(!feature.isFeatureSet(snapshot.metadata)) val redirectSpec = new PathBasedRedirectSpec( source.getCanonicalPath, dest.getCanonicalPath ) val catalogTableOpt = if (hasCatalogTable) Some(catalogTable) else None val redirectType = PathBasedRedirectSpec.REDIRECT_TYPE sql(s"insert into delta.`$source` values(1),(2),(3),(4),(5),(6)") feature.add(deltaLog, catalogTableOpt, redirectType, redirectSpec) validateState(deltaLog, EnableRedirectInProgress, source, dest, feature) // 1. Move table redirect to RedirectReady state with no redirect rules that // allows WRITE, DELETE, UPDATE. var noRedirectRules = Set( NoRedirectRule( appName = None, allowedOperations = Set( DeltaOperations.OP_WRITE, DeltaOperations.OP_DELETE, DeltaOperations.OP_UPDATE ) ) ) feature.update(deltaLog, catalogTableOpt, RedirectReady, redirectSpec, noRedirectRules) validateState(deltaLog, RedirectReady, source, dest, feature) sql(s"insert into delta.`$source` values(1),(2),(3),(4),(5),(6)") sql(s"update delta.`$source` set c0 = 100") sql(s"delete from delta.`$source` where c0 = 1") // 2. Move table redirect to RedirectReady state with no-redirect-rules that // allows UPDATE. noRedirectRules = Set( NoRedirectRule( appName = None, allowedOperations = Set(DeltaOperations.Update(None).name) ) ) feature.update(deltaLog, catalogTableOpt, RedirectReady, redirectSpec, noRedirectRules) validateState(deltaLog, RedirectReady, source, dest, feature) // 2.1. WRITE should be aborted because no-redirect-rules only allow UPDATE. val exception1 = intercept[DeltaIllegalStateException] { sql(s"insert into delta.`$source` values(1),(2),(3),(4),(5),(6)") } assert(exception1.getErrorClass == "DELTA_NO_REDIRECT_RULES_VIOLATED") // 2.2. UPDATE should pass because no-redirect-rules is fulfilled. sql(s"update delta.`$source` set c0 = 100") // 2.3. DELETE should be aborted because no-redirect-rules only allow UPDATE. val exception3 = intercept[DeltaIllegalStateException] { sql(s"delete from delta.`$source` where c0 = 1") } assert(exception3.getErrorClass == "DELTA_NO_REDIRECT_RULES_VIOLATED") // 2.4. Disabling SKIP_REDIRECT_FEATURE should allow all DMLs to pass. withSQLConf(DeltaSQLConf.SKIP_REDIRECT_FEATURE.key -> "true") { sql(s"insert into delta.`$source` values(1),(2),(3),(4),(5),(6)") sql(s"delete from delta.`$source` where c0 = 1") } // 3. Move table redirect to RedirectReady state with no-redirect-rules that // allows Write on appName "etl" . noRedirectRules = Set( NoRedirectRule( appName = Some("etl"), allowedOperations = Set(DeltaOperations.Write(SaveMode.Append).name) ) ) feature.update(deltaLog, catalogTableOpt, RedirectReady, redirectSpec, noRedirectRules) validateState(deltaLog, RedirectReady, source, dest, feature) // 3.1. The WRITE of appName "dummy" would be aborted because no-redirect-rules // only allow WRITE on application "etl". val exception4 = intercept[DeltaIllegalStateException] { spark.conf.set("spark.app.name", "dummy") sql(s"insert into delta.`$source` values(1),(2),(3),(4),(5),(6)") } assert(exception4.getErrorClass == "DELTA_NO_REDIRECT_RULES_VIOLATED") // 3.1. WRITE should pass spark.conf.set("spark.app.name", "etl") sql(s"insert into delta.`$source` values(1),(2),(3),(4),(5),(6)") // 3.2. UPDATE should be aborted because no-redirect-rules only allow WRITE. val exception5 = intercept[DeltaIllegalStateException] { sql(s"update delta.`$source` set c0 = 100") } assert(exception5.getErrorClass == "DELTA_NO_REDIRECT_RULES_VIOLATED") // 3.3. DELETE should be aborted because no-redirect-rules only allow WRITE. val exception6 = intercept[DeltaIllegalStateException] { sql(s"delete from delta.`$source` where c0 = 1") } assert(exception6.getErrorClass == "DELTA_NO_REDIRECT_RULES_VIOLATED") // 3.4. Disabling SKIP_REDIRECT_FEATURE should allow all DMLs to pass. withSQLConf(DeltaSQLConf.SKIP_REDIRECT_FEATURE.key -> "true") { sql(s"insert into delta.`$source` values(1),(2),(3),(4),(5),(6)") sql(s"update delta.`$source` set c0 = 100") sql(s"delete from delta.`$source` where c0 = 1") } } } def alterRedirect( table: String, redirectType: String, redirectState: RedirectState, spec: RedirectSpec, noRedirectRules: Set[NoRedirectRule] ): Unit = { val enableConfig = TableRedirectConfiguration( redirectType, redirectState.name, JsonUtils.toJson(spec), noRedirectRules ) val enableConfigJson = StringEscapeUtils.escapeJson(JsonUtils.toJson(enableConfig)) sql(s"alter table $table set TBLPROPERTIES('$featureName' = '$enableConfigJson')") } redirectTest(s"Redirect $featureName: modify table property", enableRedirect = true) { case (deltaLog, source, dest, catalogTable) => val redirectSpec = new PathBasedRedirectSpec( source.getCanonicalPath, dest.getCanonicalPath ) val redirectType = PathBasedRedirectSpec.REDIRECT_TYPE val destPath = dest.toString val srcPath = source.toString sql(s"CREATE external TABLE t2(c0 long) USING delta LOCATION '$dest';") sql(s"insert into t2 values(1),(2),(3),(4),(5)") val destTable = s"delta.`$destPath`" val srcTable = s"delta.`$srcPath`" // Initialize the redirection by moving table into EnableRedirectInProgress state. alterRedirect(srcTable, redirectType, EnableRedirectInProgress, redirectSpec, Set.empty) alterRedirect(destTable, redirectType, EnableRedirectInProgress, redirectSpec, Set.empty) // Delta log is cloned, then moves both redirect destination table and redirect source // table to RedirectReady state. alterRedirect(srcTable, redirectType, RedirectReady, redirectSpec, Set.empty) alterRedirect(destTable, redirectType, RedirectReady, redirectSpec, Set.empty) sql(s"insert into $srcTable values(1), (2), (3)") sql(s"insert into $destTable values(1), (2), (3)") sql(s"insert into t1 values(1), (2), (3)") sql(s"insert into t2 values(1), (2), (3)") var result = sql("select * from t1").collect() assert(result.length == 17) result = sql("select * from t2").collect() assert(result.length == 17) result = sql(s"select * from $srcTable ").collect() assert(result.length == 17) result = sql(s"select * from $destTable ").collect() assert(result.length == 17) val root = new Path(catalogTable.location) val fs = root.getFileSystem(deltaLog.newDeltaHadoopConf) var files = fs.listStatus(new Path(srcPath + "/_delta_log")) .filter(_.getPath.toString.endsWith(".json")) assert(files.length == 3) files = fs.listStatus(new Path(destPath + "/_delta_log")) .filter(_.getPath.toString.endsWith(".json")) assert(files.length == 8) // Drop redirection by moving both redirect destination table and redirect source table to // DropRedirectInProgress. alterRedirect(destTable, redirectType, DropRedirectInProgress, redirectSpec, Set.empty) alterRedirect(srcTable, redirectType, DropRedirectInProgress, redirectSpec, Set.empty) // Remove table redirect feature from redirect source table and verify table content. sql(s"alter table $srcTable unset TBLPROPERTIES('$featureName')") result = sql("select * from t1").collect() assert(result.length == 0) sql("insert into t1 values(1), (2), (3), (4)") result = sql("select * from t1").collect() assert(result.length == 4) } } test("test getRedirectConfiguration") { val redirectSpec = new PathBasedRedirectSpec("sourcePath", "targetPath") val properties1 = RedirectReaderWriter.generateRedirectMetadata( PathBasedRedirectSpec.REDIRECT_TYPE, EnableRedirectInProgress, redirectSpec, noRedirectRules = Set.empty) val properties2 = RedirectWriterOnly.generateRedirectMetadata( PathBasedRedirectSpec.REDIRECT_TYPE, RedirectReady, redirectSpec, noRedirectRules = Set.empty) val configuration = RedirectFeature.getRedirectConfiguration(properties1 ++ properties2) assert(configuration.isDefined) // redirect-reader-writer should be preferred over redirect-writer-only. assert(JsonUtils.toJson(configuration.get) == properties1(DeltaConfigs.REDIRECT_READER_WRITER.key)) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/TightBoundsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import scala.collection.mutable.ArrayBuffer // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.DeltaStatistics.{MIN, NULL_COUNT, NUM_RECORDS, TIGHT_BOUNDS} import org.apache.spark.sql.delta.stats.StatisticsCollection import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.JsonUtils import com.fasterxml.jackson.databind.node.ObjectNode import org.apache.spark.sql.{DataFrame, QueryTest, Row} import org.apache.spark.sql.functions.{col, lit, map_values, when} import org.apache.spark.sql.test.SharedSparkSession class TightBoundsSuite extends QueryTest with SharedSparkSession with DeletionVectorsTestUtils with DeltaSQLCommandTest { import testImplicits._ override def beforeAll(): Unit = { super.beforeAll() enableDeletionVectors(spark.conf) } test("Validate TIGHT_BOUND column") { val targetDF = createTestDF(0, 100, 2) val sourceDF = targetDF def runDelete(target: io.delta.tables.DeltaTable): Int = { target.delete("id >= 75") 2 // Expected number of files. } val operations = ArrayBuffer[io.delta.tables.DeltaTable => Int](runDelete) for { // Make sure it works for all operations that add DVs runOperation <- operations // Make sure tightBounds update is backwards compatible tightBoundDisabled <- BOOLEAN_DOMAIN } { val conf = Seq( DeltaSQLConf.TIGHT_BOUND_COLUMN_ON_FILE_INIT_DISABLED.key -> tightBoundDisabled.toString) withSQLConf(conf: _*) { withTempDeltaTable(targetDF) { (targetTable, targetLog) => val snapshotBeforeOperation = targetLog.update() val statsColumnName = snapshotBeforeOperation.getBaseStatsColumnName val tightBoundsValuesBeforeOperation = snapshotBeforeOperation.withStatsDeduplicated .select(col(s"${statsColumnName}.$TIGHT_BOUNDS")) .collect() assert(tightBoundsValuesBeforeOperation.length === 2) val expectedTightBoundsValue = if (tightBoundDisabled) "[null]" else "[true]" tightBoundsValuesBeforeOperation .foreach(r => assert(r.toString == expectedTightBoundsValue)) val expectedNumberOfFiles = runOperation(targetTable()) // All operations only touch the second file. assert(getFilesWithDeletionVectors(targetLog).size == 1) val snapshotAfterOperation = targetLog.update() val tightBoundsValuesAfterOperation = snapshotAfterOperation.withStatsDeduplicated // Order by returns non-null DVs last. Thus, the file with the wide bounds // should be the last one. .orderBy(col("deletionVector").asc_nulls_first) .select(col(s"${statsColumnName}.$TIGHT_BOUNDS")) .collect() // Make sure tightsBounds is generated even for files that initially // did not contain the column. Note, we expect 2 files each from merge and delete // operations and three from update. This is because update creates a new file for the // updated rows. assert(tightBoundsValuesAfterOperation.length === expectedNumberOfFiles) assert(tightBoundsValuesAfterOperation.head.toString === expectedTightBoundsValue) assert(tightBoundsValuesAfterOperation.last.toString === "[false]") } } } } test("Verify exception is thrown if we commit files with DVs and tight bounds") { val targetDF = createTestDF(0, 100, 2) withTempDeltaTable(targetDF, enableDVs = true) { (targetTable, targetLog) => // Remove one record from each file. targetTable().delete("id in (0, 50)") verifyDVsExist(targetLog, 2) // Commit actions with DVs and tight bounds. val txn = targetLog.startTransaction() val addFiles = txn.snapshot.allFiles.collect().toSeq.map { action => action.copy(stats = s"""{"${NUM_RECORDS}":${action.numPhysicalRecords.get}, | "${TIGHT_BOUNDS}":true}""".stripMargin) } val exception = intercept[DeltaIllegalStateException] { txn.commitActions(DeltaOperations.TestOperation(), addFiles: _*) } assert(exception.getErrorClass === "DELTA_ADDING_DELETION_VECTORS_WITH_TIGHT_BOUNDS_DISALLOWED") } } protected def getStats(snapshot: Snapshot, statName: String): Array[Row] = { val statsColumnName = snapshot.getBaseStatsColumnName snapshot .withStatsDeduplicated .select(s"$statsColumnName.$statName") .collect() } protected def getStatFromLastFile(snapshot: Snapshot, statName: String): Row = { val statsColumnName = snapshot.getBaseStatsColumnName snapshot .withStatsDeduplicated .select(s"$statsColumnName.$statName") .orderBy(s"$statsColumnName.$MIN") .collect() .last } protected def getStatFromLastFileWithDVs(snapshot: Snapshot, statName: String): Row = { val statsColumnName = snapshot.getBaseStatsColumnName snapshot .withStatsDeduplicated .filter("isNotNull(deletionVector)") .select(s"$statsColumnName.$statName") .collect() .last } /** * Helper method that returns stats for every file in the snapshot as row objects. * * Return value schema is { * numRecords: Int, * RminValues: Row(Int, Int, ...), // Min value for each column * maxValues: Row(Int, Int, ...), // Max value for each column * nullCount: Row(Int, Int, ...), // Null count for each column * tightBounds: boolean * } */ protected def getStatsInPartitionOrder(snapshot: Snapshot): Array[Row] = { val statsColumnName = snapshot.getBaseStatsColumnName snapshot .withStatsDeduplicated .orderBy(map_values(col("partitionValues"))) .select(s"$statsColumnName.*") .collect() } protected def getNullCountFromFirstFileWithDVs(snapshot: Snapshot): Row = { // Note, struct columns in Spark are returned with datatype Row. getStatFromLastFile(snapshot, NULL_COUNT) .getAs[Row](NULL_COUNT) } test("NULL COUNT is updated correctly when all values are nulls" ) { val targetDF = spark.range(0, 100, 1, 2) .withColumn("value", when(col("id") < 25, col("id")) .otherwise(null)) withTempDeltaTable(targetDF, enableDVs = true) { (targetTable, targetLog) => targetTable().delete("id >= 80") assert(getNullCountFromFirstFileWithDVs(targetLog.update()) === Row(0, 50)) targetTable().delete("id >= 70") assert(getNullCountFromFirstFileWithDVs(targetLog.update()) === Row(0, 50)) } } test("NULL COUNT is updated correctly where there are no nulls" ) { val targetDF = spark.range(0, 100, 1, 2) .withColumn("value", col("id")) withTempDeltaTable(targetDF, enableDVs = true) { (targetTable, targetLog) => val expectedResult = Row(0, 0) targetTable().delete("id >= 80") assert(getNullCountFromFirstFileWithDVs(targetLog.update()) === expectedResult) targetTable().delete("id >= 70") assert(getNullCountFromFirstFileWithDVs(targetLog.update()) === expectedResult) } } test("NULL COUNT is updated correctly when some values are nulls" ) { val targetDF = spark.range(0, 100, 1, 2) .withColumn("value", when(col("id") < 75, col("id")) .otherwise(null)) withTempDeltaTable(targetDF, enableDVs = true) { (targetTable, targetLog) => targetTable().delete("id >= 80") assert(getNullCountFromFirstFileWithDVs(targetLog.update()) === Row(0, 25)) targetTable().delete("id >= 70") assert(getNullCountFromFirstFileWithDVs(targetLog.update()) === Row(0, 25)) } } test("DML operations fetch stats on tables with partial stats") { val targetDF = createTestDF(0, 200, 4) .withColumn("v", col("id")) .withColumn("partCol", (col("id") / lit(50)).cast("Int")) val conf = Seq(DeltaSQLConf.DELTA_COLLECT_STATS.key -> false.toString) withTempDeltaTable(targetDF, Seq("partCol"), conf = conf) { (targetTable, targetLog) => val statsBeforeFirstDelete = getStatsInPartitionOrder(targetLog.update()) val expectedStatsBeforeFirstDelete = Seq( Row(null, null, null, null, null), // File 1. Row(null, null, null, null, null), // File 2. Row(null, null, null, null, null), // File 3. Row(null, null, null, null, null) // File 4. ) assert(statsBeforeFirstDelete === expectedStatsBeforeFirstDelete) // This operation touches files 2 and 3. Files 1 and 4 should still have not stats. targetTable().delete("id in (50, 100)") // Expect the stats for every file that got a DV added to it with tightBounds = false val statsAfterFirstDelete = getStatsInPartitionOrder(targetLog.update()) val expectedStatsAfterFirstDelete = Seq( Row(null, null, null, null, null), // File 1. Row(50, Row(50, 50), Row(99, 99), Row(0, 0), false), // File 2. Row(50, Row(100, 100), Row(149, 149), Row(0, 0), false), // File 3. Row(null, null, null, null, null) // File 4. ) assert(statsAfterFirstDelete === expectedStatsAfterFirstDelete) } } test("Update file without minValue and maxValue stats to wide bounds") { // The table has only binary columns, for which Delta does not collect minValue or maxValue // stats. The file stats should still include numRecords, nullCount, and tightBounds. withTempDeltaTable( dataDF = spark.range(0, 10, 1, 1).toDF("id") .select(col("id").cast("string").cast("binary").as("b")), enableDVs = true ) { (targetTable, targetLog) => val statsBeforeDelete = getStatsInPartitionOrder(targetLog.update()) val expectedStatsBeforeDelete = Seq(Row(10, Row(0), true)) assert(statsBeforeDelete === expectedStatsBeforeDelete) // The DELETE command updates file stats to wide bounds. targetTable().delete(col("b") === lit("1").cast("string").cast("binary")) val statsAfterDelete = getStatsInPartitionOrder(targetLog.update()) val expectedStatsAfterDelete = Seq(Row(10, Row(0), false)) assert(statsAfterDelete === expectedStatsAfterDelete) } } test("Update file without column stats to wide bounds") { // We disable gathering stats for any of the columns in this table. // In this case, the file stats should include numRecords and tightBounds only, // but not minValue, maxValue or nullCount. withTempDeltaTable( dataDF = spark.range(0, 10, 1, 1).toDF("id"), conf = Map(DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.defaultTablePropertyKey -> "0").toSeq, enableDVs = true ) { (targetTable, targetLog) => val statsBeforeDelete = getStatsInPartitionOrder(targetLog.update()) val expectedStatsBeforeDelete = Seq(Row(10, true)) assert(statsBeforeDelete === expectedStatsBeforeDelete) // The DELETE command updates file stats to wide bounds. targetTable().delete("id = 1") val statsAfterDelete = getStatsInPartitionOrder(targetLog.update()) val expectedStatsAfterDelete = Seq(Row(10, false)) assert(statsAfterDelete === expectedStatsAfterDelete) } } def tableAddDVAndTightStats( targetTable: () => io.delta.tables.DeltaTable, targetLog: DeltaLog, deleteCond: String): Unit = { // Add DVs. Stats should have tightBounds = false afterwards. targetTable().delete(deleteCond) val initialStats = getStats(targetLog.update(), "*") assert(initialStats.forall(_.get(4) === false)) // tightBounds // Other systems may support Compute Stats that recomputes tightBounds stats on tables with DVs. // Simulate this with a manual update commit that introduces tight stats. val txn = targetLog.startTransaction() val addFiles = txn.snapshot.allFiles.collect().toSeq.map { action => val node = JsonUtils.mapper.readTree(action.stats).asInstanceOf[ObjectNode] assert(node.has("numRecords")) val numRecords = node.get("numRecords").asInt() action.copy(stats = s"""{ "numRecords" : $numRecords, "tightBounds" : true }""") } txn.commitActions(DeltaOperations.ManualUpdate, addFiles: _*) } test("CLONE on table with DVs and tightBound stats") { val targetDF = spark.range(0, 100, 1, 1).toDF() withTempDeltaTable(targetDF) { (targetTable, targetLog) => val targetPath = targetLog.dataPath.toString tableAddDVAndTightStats(targetTable, targetLog, "id >= 80") // CLONE shouldn't throw // DELTA_ADDING_DELETION_VECTORS_WITH_TIGHT_BOUNDS_DISALLOWED withTempPath("cloned") { clonedPath => sql(s"CREATE TABLE delta.`$clonedPath` SHALLOW CLONE delta.`$targetPath`") } } } test("RESTORE TABLE on table with DVs and tightBound stats") { val targetDF = spark.range(0, 100, 1, 1).toDF() withTempDeltaTable(targetDF) { (targetTable, targetLog) => val targetPath = targetLog.dataPath.toString // adds version 1 (delete) and 2 (compute stats) tableAddDVAndTightStats(targetTable, targetLog, "id >= 80") // adds version 3 (delete more) targetTable().delete("id < 20") // Restore back to version 2 (after compute stats) // After 2nd delete, new DVs are added to the file, so the restore will // have to recommit the file with old DVs. targetTable().restoreToVersion(2) // Verify that the restored table has DVs and tight bounds. val stats = getStatFromLastFileWithDVs(targetLog.update(), "*") assert(stats.get(4) === true) // tightBounds } } test("Row Tracking backfill on table with DVs and tightBound stats") { // Enabling Row Tracking and backfill shouldn't throw // DELTA_ADDING_DELETION_VECTORS_WITH_TIGHT_BOUNDS_DISALLOWED withSQLConf(DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey -> "false") { val targetDF = spark.range(0, 100, 1, 1).toDF() withTempDeltaTable(targetDF) { (targetTable, targetLog) => val targetPath = targetLog.dataPath.toString tableAddDVAndTightStats(targetTable, targetLog, "id >= 80") // Make sure that we start with no RowTracking feature. assert(!RowTracking.isSupported(targetLog.unsafeVolatileSnapshot.protocol)) assert(!RowId.isEnabled(targetLog.unsafeVolatileSnapshot.protocol, targetLog.unsafeVolatileSnapshot.metadata)) sql(s"ALTER TABLE delta.`$targetPath` SET TBLPROPERTIES " + "('delta.enableRowTracking' = 'true')") assert(targetLog.history.getHistory(None) .count(_.operation == DeltaOperations.ROW_TRACKING_BACKFILL_OPERATION_NAME) == 1) } } } } class TightBoundsColumnMappingSuite extends TightBoundsSuite with DeltaColumnMappingEnableIdMode ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/TimestampLocalFileSystem.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import java.net.URI import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{DelegateToFileSystem, Path, RawLocalFileSystem} import org.apache.hadoop.fs.FileStatus /** * This custom fs implementation is used for testing the msync calling in HDFSLogStore writes. * If `msync` is not called, `listStatus` will return stale results. */ class TimestampLocalFileSystem extends RawLocalFileSystem { private var uri: URI = _ private var latestTimestamp: Long = 0 override def getScheme: String = TimestampLocalFileSystem.scheme override def initialize(name: URI, conf: Configuration): Unit = { uri = URI.create(name.getScheme + ":///") super.initialize(name, conf) } override def getUri(): URI = if (uri == null) { // RawLocalFileSystem's constructor will call this one before `initialize` is called. // Just return the super's URI to avoid NPE. super.getUri } else { uri } override def listStatus(path: Path): Array[FileStatus] = { super.listStatus(path).filter(_.getModificationTime <= latestTimestamp) } override def msync(): Unit = { latestTimestamp = System.currentTimeMillis() } } class TimestampAbstractFileSystem(uri: URI, conf: Configuration) extends DelegateToFileSystem( uri, new TimestampLocalFileSystem, conf, TimestampLocalFileSystem.scheme, false) /** * Singleton for BlockWritesLocalFileSystem used to initialize the file system countdown latch. */ object TimestampLocalFileSystem { val scheme = "ts" } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/UCManagedTableKillSwitchSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.actions.DomainMetadata import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.skipping.clustering.{ClusteredTableUtils, ClusteringColumn} import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.catalyst.TableIdentifier /** * Unit tests for the kill switch that blocks clustering column changes on UC-managed * CatalogOwned tables. These tests bypass UCSingleCatalog (only available in sparkUnityCatalog) * by overriding [[OptimisticTransaction.isUCManagedTable]] in a test-local subclass. * * The kill switch fires in [[OptimisticTransaction.commitLarge]], which is used by RESTORE TABLE * and bypasses prepareCommit. */ class UCManagedTableKillSwitchSuite extends CatalogOwnedTestBaseSuite with DeltaSQLTestUtils with DeltaSQLCommandTest { // Enable CatalogOwned by default so every CREATE TABLE produces a CatalogOwned table. override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) /** * Test subclass that pretends it targets a UC-managed table, bypassing the real * [[CatalogOwnedTableUtils.getCatalogName]] check that requires UCSingleCatalog. */ private class UCManagedTxn(log: DeltaLog, snap: Snapshot) extends OptimisticTransaction(log, None, snap) { override protected[delta] lazy val isUCManagedTable: Boolean = true } private val clusteringOnId: Seq[DomainMetadata] = Seq(ClusteredTableUtils.createDomainMetadata(Seq(ClusteringColumn(Seq("id"))))) private val clusteringOnName: Seq[DomainMetadata] = Seq(ClusteredTableUtils.createDomainMetadata(Seq(ClusteringColumn(Seq("name"))))) /** Creates a CatalogOwned clustered-by-id table and returns its (log, snapshot). */ private def createClusteredTable(tableName: String): (DeltaLog, Snapshot) = { sql(s"CREATE TABLE $tableName (id INT, name STRING) USING delta " + s"CLUSTER BY (id) TBLPROPERTIES ('delta.feature.catalogManaged' = 'supported')") sql(s"INSERT INTO $tableName VALUES (1, 'a')") val (log, snap) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName)) assert(snap.isCatalogOwned, "table should be CatalogOwned") (log, snap) } test("commitLarge blocks clustering change on UC-managed CatalogOwned table") { withTable("tbl") { val (log, snap) = createClusteredTable("tbl") val ex = intercept[DeltaAnalysisException] { new UCManagedTxn(log, snap).commitLarge( spark, clusteringOnName.iterator, newProtocolOpt = None, op = DeltaOperations.Restore(Some(0L), None), context = Map.empty, metrics = Map.empty) } assert(ex.getMessage.contains("Clustering column changes on Unity Catalog managed tables")) } } test("commit with unchanged clustering is allowed on UC-managed CatalogOwned table") { withTable("tbl") { val (log, snap) = createClusteredTable("tbl") // Commit the same clustering DomainMetadata that the snapshot already has - must not throw. new UCManagedTxn(log, snap).commit(clusteringOnId, DeltaOperations.ManualUpdate) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/UniversalFormatSuiteBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.Utils.try_element_at import org.apache.spark.sql.{DataFrameWriter, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.col import org.apache.spark.sql.test.SQLTestUtils trait UniversalFormatTestHelper { val allCompatObjects: Seq[IcebergCompatBase] = Seq( IcebergCompatV1, IcebergCompatV2 ) def compatObjectFromVersion(version: Int): IcebergCompatBase = allCompatObjects(version - 1) def getCompatVersionOtherThan(version: Int): Int = { val targetVersion = getCompatVersionsOtherThan(version).head assert(targetVersion != version) targetVersion } def getCompatVersionsOtherThan(version: Int): Seq[Int] = { allCompatObjects .filter(_.version != version) .map(_.version.toInt) } } trait UniversalFormatSuiteBase extends IcebergCompatUtilsBase with UniversalFormatTestHelper { protected def assertUniFormIcebergProtocolAndProperties( tableId: String, compatVersion: Int = compatVersion): Unit = { assertIcebergCompatProtocolAndProperties(tableId, compatObjectFromVersion(compatVersion)) val snapshot = DeltaLog.forTable(spark, TableIdentifier(tableId)).update() assert(UniversalFormat.icebergEnabled(snapshot.metadata)) } protected def getDfWriter( colName: String, mode: String, enableUniform: Boolean = true): DataFrameWriter[Row] = { var df = spark.range(10) .toDF(colName) .write .mode(mode) .format("delta") df = if (mode == "overwrite") df.option("overwriteSchema", "true") else df if (enableUniform) { df.option(s"delta.enableIcebergCompatV$compatVersion", "true") df.option("delta.universalFormat.enabledFormats", "iceberg") } else { df } } protected def assertAddFileIcebergCompatVersion( snapshot: Snapshot, icebergCompatVersion: Int, count: Int): Unit = { val addFilesWithTagCount = snapshot.allFiles .select("tags") .where(try_element_at(col("tags"), AddFile.Tags.ICEBERG_COMPAT_VERSION.name) === s"$icebergCompatVersion") .count() assert(addFilesWithTagCount == count) } protected def runReorgTableForUpgradeUniform( tableId: String, icebergCompatVersion: Int = compatVersion): Unit = { executeSql(s""" | REORG TABLE $tableId APPLY | (UPGRADE UNIFORM (ICEBERG_COMPAT_VERSION = $icebergCompatVersion)) |""".stripMargin) } protected def checkFileNotRewritten( prevSnapshot: Snapshot, currSnapshot: Snapshot): Unit = { val prevFiles = prevSnapshot.allFiles.collect().map(f => (f.path, f.modificationTime)) val currFiles = currSnapshot.allFiles.collect().map(f => (f.path, f.modificationTime)) val unchangedFiles = currFiles.filter { case (path, time) => prevFiles.find(_._1 == path).exists(_._2 == time) } assert(unchangedFiles.length == currFiles.length) } test("create new UniForm table while manually enabling IcebergCompat") { allReaderWriterVersions.foreach { case (r, w) => withTempTableAndDir { case (id, _) => executeSql(s""" |CREATE TABLE $id (ID INT) USING DELTA TBLPROPERTIES ( | 'delta.universalFormat.enabledFormats' = 'iceberg', | 'delta.enableIcebergCompatV$compatVersion' = 'true', | 'delta.minReaderVersion' = $r, | 'delta.minWriterVersion' = $w |)""".stripMargin) assertUniFormIcebergProtocolAndProperties(id) } } } test("create new UniForm table while manually enabling IcebergCompat with no rw version") { withTempTableAndDir { case (id, _) => executeSql(s""" |CREATE TABLE $id (ID INT) USING DELTA TBLPROPERTIES ( | 'delta.universalFormat.enabledFormats' = 'iceberg', | 'delta.enableIcebergCompatV$compatVersion' = 'true' |)""".stripMargin) assertUniFormIcebergProtocolAndProperties(id) } } test("create new UniForm table via clone") { withTempTableAndDir { case (id, loc) => executeSql(s""" |CREATE TABLE $id (ID INT) USING DELTA TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name') | """.stripMargin) executeSql(s""" |INSERT INTO $id values (1) """.stripMargin) withTempTableAndDir { case (cloneId, _) => executeSql(s""" |CREATE TABLE $cloneId SHALLOW CLONE $id TBLPROPERTIES ( | 'delta.universalFormat.enabledFormats' = 'iceberg', | 'delta.enableIcebergCompatV$compatVersion' = 'true', | 'delta.columnMapping.mode' = 'name' |) """.stripMargin) assertUniFormIcebergProtocolAndProperties(cloneId) } } } test("enable UniForm on existing table with IcebergCompat enabled") { allReaderWriterVersions.foreach { case (r, w) => withTempTableAndDir { case (id, _) => executeSql(s""" |CREATE TABLE $id (ID INT) USING DELTA TBLPROPERTIES ( | 'delta.minReaderVersion' = $r, | 'delta.minWriterVersion' = $w, | 'delta.enableIcebergCompatV$compatVersion' = true |)""".stripMargin) executeSql(s"ALTER TABLE $id SET TBLPROPERTIES " + s"('delta.universalFormat.enabledFormats' = 'iceberg')") assertUniFormIcebergProtocolAndProperties(id) } } } test("enable UniForm on existing table without IcebergCompat") { allReaderWriterVersions.foreach { case (r, w) => withTempTableAndDir { case (id, _) => executeSql(s""" |CREATE TABLE $id (ID INT) USING DELTA TBLPROPERTIES ( | 'delta.minReaderVersion' = $r, | 'delta.minWriterVersion' = $w |)""".stripMargin) executeSql(s"ALTER TABLE $id SET TBLPROPERTIES " + s"('delta.universalFormat.enabledFormats' = 'iceberg'," + s" 'delta.columnMapping.mode' = 'name', " + s" 'delta.enableIcebergCompatV$compatVersion' = true) ") assertUniFormIcebergProtocolAndProperties(id) } } } test("enable UniForm on existing table with ColumnMapping") { allReaderWriterVersions.foreach { case (r, w) => withTempTableAndDir { case (id, _) => executeSql(s""" |CREATE TABLE $id (ID INT) USING DELTA TBLPROPERTIES ( | 'delta.minReaderVersion' = $r, | 'delta.minWriterVersion' = $w, | 'delta.columnMapping.mode' = 'name' |)""".stripMargin) executeSql(s"ALTER TABLE $id SET TBLPROPERTIES " + s"('delta.universalFormat.enabledFormats' = 'iceberg'," + s" 'delta.enableIcebergCompatV$compatVersion' = true) ") assertUniFormIcebergProtocolAndProperties(id) } } } test("enable UniForm on existing table but IcebergCompat isn't enabled - fail") { allReaderWriterVersions.foreach { case (r, w) => withTempTableAndDir { case (id, _) => executeSql(s""" |CREATE TABLE $id (ID INT) USING DELTA TBLPROPERTIES ( | 'delta.minReaderVersion' = $r, | 'delta.minWriterVersion' = $w, | 'delta.enableIcebergCompatV$compatVersion' = false, | 'delta.feature.icebergCompatV$compatVersion' = 'supported' |)""".stripMargin) val e = intercept[DeltaUnsupportedOperationException] { executeSql(s"ALTER TABLE $id SET TBLPROPERTIES " + s"('delta.universalFormat.enabledFormats' = 'iceberg')") } assert(e.getErrorClass === "DELTA_UNIVERSAL_FORMAT_VIOLATION") } } } test("disabling UniForm will not disable IcebergCompat") { withTempTableAndDir { case (id, _) => executeSql( s""" |CREATE TABLE $id (ID INT) USING DELTA TBLPROPERTIES ( | 'delta.universalFormat.enabledFormats' = 'iceberg', | 'delta.enableIcebergCompatV$compatVersion' = 'true' |)""".stripMargin) assertUniFormIcebergProtocolAndProperties(id) executeSql(s"ALTER TABLE $id UNSET TBLPROPERTIES ('delta.universalFormat.enabledFormats')") assert(getProperties(id)(s"delta.enableIcebergCompatV$compatVersion").toBoolean) } } test("disabling IcebergCompat will disable UniForm if enabled") { allReaderWriterVersions.foreach { case (r, w) => withTempTableAndDir { case (id, _) => executeSql(s""" |CREATE TABLE $id (ID INT) USING DELTA TBLPROPERTIES ( | 'delta.minReaderVersion' = $r, | 'delta.minWriterVersion' = $w, | 'delta.universalFormat.enabledFormats' = 'iceberg', | 'delta.enableIcebergCompatV$compatVersion' = true |)""".stripMargin) var tableprops = getProperties(id) assert(tableprops("delta.universalFormat.enabledFormats") === "iceberg") assert(tableprops(s"delta.enableIcebergCompatV$compatVersion").toBoolean) executeSql(s""" |ALTER TABLE $id SET TBLPROPERTIES ( |'delta.enableIcebergCompatV$compatVersion' = false) |""".stripMargin) tableprops = getProperties(id) assert(!tableprops.contains("delta.universalFormat.enabledFormats")) assert(!tableprops(s"delta.enableIcebergCompatV$compatVersion").toBoolean) } } } } trait UniFormWithIcebergCompatV1SuiteBase extends UniversalFormatSuiteBase { protected override val compatObject: IcebergCompatBase = IcebergCompatV1 test("enable UniForm and V1 on existing table") { withTempTableAndDir { case (id, loc) => executeSql(s"CREATE TABLE $id (ID INT) USING DELTA LOCATION $loc") executeSql(s""" |ALTER TABLE $id SET TBLPROPERTIES ( | 'delta.minReaderVersion' = 2, | 'delta.minWriterVersion' = 5, | 'delta.universalFormat.enabledFormats' = 'iceberg', | 'delta.enableIcebergCompatV1' = true, | 'delta.columnMapping.mode' = 'name' |)""".stripMargin) assertUniFormIcebergProtocolAndProperties(id) } } test("REORG TABLE for table from icebergCompatVx to icebergCompatV1, should skip rewrite") { getCompatVersionsOtherThan(1).foreach(originalVersion => { withTempTableAndDir { case (id, _) => executeSql(s""" | CREATE TABLE $id (ID INT) USING DELTA TBLPROPERTIES ( | 'delta.universalFormat.enabledFormats' = 'iceberg', | 'delta.enableIcebergCompatV$originalVersion' = 'true' |) | """.stripMargin) executeSql(s""" | INSERT INTO TABLE $id (ID) | VALUES (1),(2),(3),(4),(5),(6),(7)""".stripMargin) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(id)) val snapshot = deltaLog.update() assertAddFileIcebergCompatVersion( snapshot, icebergCompatVersion = originalVersion, count = 1) runReorgTableForUpgradeUniform(id, icebergCompatVersion = 1) val updatedSnapshot = deltaLog.update() assert(updatedSnapshot.getProperties("delta.enableIcebergCompatV1") === "true") assertAddFileIcebergCompatVersion( deltaLog.update(), icebergCompatVersion = originalVersion, count = 1) checkFileNotRewritten(snapshot, updatedSnapshot) } }) } } trait UniFormWithIcebergCompatV2SuiteBase extends UniversalFormatSuiteBase { override val compatObject: IcebergCompatBase = IcebergCompatV2 test("can downgrade from V2 to V1 with ALTER with UniForm enabled") { withTempTableAndDir { case (id, loc) => executeSql(s""" |CREATE TABLE $id (ID INT) USING DELTA LOCATION $loc TBLPROPERTIES ( | 'delta.enableIcebergCompatV2' = 'true', | 'delta.universalFormat.enabledFormats' = 'iceberg' |)""".stripMargin) executeSql(s""" |ALTER TABLE $id SET TBLPROPERTIES ( | 'delta.enableIcebergCompatV1' = true, | 'delta.enableIcebergCompatV2' = false |)""".stripMargin) assertUniFormIcebergProtocolAndProperties(id, 1) } } test("REORG TABLE for table from icebergCompatVx to icebergCompatV2") { val originalVersion = 1 withTempTableAndDir { case (id, loc) => executeSql(s""" | CREATE TABLE $id (ID INT) USING DELTA LOCATION $loc TBLPROPERTIES ( | 'delta.universalFormat.enabledFormats' = 'iceberg', | 'delta.enableIcebergCompatV$originalVersion' = 'true' |)""".stripMargin) executeSql(s""" | INSERT INTO TABLE $id (ID) | VALUES (1),(2),(3),(4),(5),(6),(7)""".stripMargin) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(id)) val snapshot1 = deltaLog.update() assert(snapshot1.allFiles.collect().nonEmpty) assertAddFileIcebergCompatVersion(snapshot1, icebergCompatVersion = 2, count = 0) runReorgTableForUpgradeUniform(id, icebergCompatVersion = 2) val snapshot2 = deltaLog.update() assert(snapshot2.getProperties("delta.enableIcebergCompatV2") === "true") assert(snapshot2.getProperties("delta.enableDeletionVectors") === "false") assertAddFileIcebergCompatVersion(snapshot2, icebergCompatVersion = 2, count = 1) } } test( "REORG TABLE: new files would have ICEBERG_COMPAT_VERSION tag if enableIcebergCompat is on") { withTempTableAndDir { case (id, loc) => executeSql( s""" | CREATE TABLE $id (ID INT) USING DELTA LOCATION $loc TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name' |) | """.stripMargin) executeSql( s""" | INSERT INTO TABLE $id (ID) | VALUES (1),(2),(3),(4),(5),(6),(7)""".stripMargin) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(id)) val txn = deltaLog.startTransaction() val metadata = txn.metadata val enableIcebergCompatConf = Map( DeltaConfigs.ICEBERG_COMPAT_V1_ENABLED.key -> "false", DeltaConfigs.ICEBERG_COMPAT_V2_ENABLED.key -> "true") val newMetadata = metadata.copy( configuration = metadata.configuration ++ enableIcebergCompatConf) txn.updateMetadata(newMetadata) txn.commit( Nil, DeltaOperations.UpgradeUniformProperties(enableIcebergCompatConf) ) assertAddFileIcebergCompatVersion( deltaLog.update(), icebergCompatVersion = 2, count = 0) // The new file would have the ICEBERG_COMPAT_VERSION tag while the exist files would not executeSql(s""" | INSERT INTO TABLE $id (ID) | VALUES (8),(9),(10)""".stripMargin) assertAddFileIcebergCompatVersion( deltaLog.update(), icebergCompatVersion = 2, count = 1) // After REORG TABLE command, all the exist files would have ICEBERG_COMPAT_VERSION tag runReorgTableForUpgradeUniform(id, 2) val finalSnapshot = deltaLog.update() assert(finalSnapshot.getProperties("delta.enableIcebergCompatV2") === "true") assertAddFileIcebergCompatVersion(finalSnapshot, icebergCompatVersion = 2, count = 2) } } } trait UniversalFormatMiscSuiteBase extends IcebergCompatUtilsBase with UniversalFormatTestHelper { test("enforceInvariantsAndDependenciesForCTAS") { withTempTableAndDir { case (id, _) => executeSql(s"CREATE TABLE $id (id INT) USING DELTA") val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(id)) val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(id)) var configurationUnderTest = Map("dummykey1" -> "dummyval1", "dummykey2" -> "dummyval2") // The enforce is not lossy. It will do nothing if there is no Universal related key. def getUpdatedConfiguration(conf: Map[String, String]): Map[String, String] = UniversalFormat.enforceDependenciesInConfiguration(spark, catalogTable = catalogTable, conf, snapshot) var updatedConfiguration = getUpdatedConfiguration(configurationUnderTest) assert(configurationUnderTest == configurationUnderTest) configurationUnderTest = Map( "delta.universalFormat.enabledFormats" -> "iceberg", "dummykey" -> "dummyvalue" ) val e = intercept[DeltaUnsupportedOperationException] { updatedConfiguration = getUpdatedConfiguration(configurationUnderTest) } assert(e.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION") for (icv <- allCompatObjects.map(_.version)) { configurationUnderTest = Map( s"delta.enableIcebergCompatV$icv" -> "true", "delta.universalFormat.enabledFormats" -> "iceberg", "dummykey" -> "dummyvalue" ) updatedConfiguration = getUpdatedConfiguration(configurationUnderTest) assert(updatedConfiguration.size == 5) assert(updatedConfiguration("dummykey") == "dummyvalue") assert(updatedConfiguration("delta.universalFormat.enabledFormats") == "iceberg") assert(updatedConfiguration("delta.columnMapping.mode") == "name") assert(updatedConfiguration(s"delta.enableIcebergCompatV$icv") == "true") assert(updatedConfiguration("delta.columnMapping.maxColumnId") == "1") configurationUnderTest = Map( s"delta.enableIcebergCompatV$icv" -> "true", "delta.universalFormat.enabledFormats" -> "iceberg", "dummykey" -> "dummyvalue", "delta.columnMapping.mode" -> "id" ) updatedConfiguration = getUpdatedConfiguration(configurationUnderTest) assert(updatedConfiguration.size == 4) assert(updatedConfiguration("dummykey") == "dummyvalue") assert(updatedConfiguration("delta.columnMapping.mode") == "id") assert(updatedConfiguration("delta.universalFormat.enabledFormats") == "iceberg") assert(updatedConfiguration(s"delta.enableIcebergCompatV$icv") == "true") } } } test("UniForm config validation") { Seq("ICEBERG", "iceberg,iceberg", "iceber", "paimon").foreach { invalidConf => withTempTableAndDir { case (id, loc) => val errMsg = intercept[IllegalArgumentException] { executeSql(s""" |CREATE TABLE $id (ID INT) USING DELTA LOCATION $loc TBLPROPERTIES ( | 'delta.universalFormat.enabledFormats' = '$invalidConf', | 'delta.enableIcebergCompatV1' = 'true', | 'delta.columnMapping.mode' = 'name' |)""".stripMargin) }.getMessage assert( errMsg.contains("Must be a comma-separated list of formats from the list"), errMsg ) } } } test("create new UniForm table without manually enabling IcebergCompat - fail") { allReaderWriterVersions.foreach { case (r, w) => withTempTableAndDir { case (id, loc) => val e = intercept[DeltaUnsupportedOperationException] { executeSql(s""" |CREATE TABLE $id (ID INT) USING DELTA LOCATION $loc TBLPROPERTIES ( | 'delta.universalFormat.enabledFormats' = 'iceberg', | 'delta.minReaderVersion' = $r, | 'delta.minWriterVersion' = $w |)""".stripMargin) } assert(e.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION") val e1 = intercept[DeltaUnsupportedOperationException] { executeSql(s""" |CREATE TABLE $id USING DELTA LOCATION $loc TBLPROPERTIES ( | 'delta.universalFormat.enabledFormats' = 'iceberg', | 'delta.minReaderVersion' = $r, | 'delta.minWriterVersion' = $w |) AS SELECT 1""".stripMargin) } assert(e1.getErrorClass == "DELTA_UNIVERSAL_FORMAT_VIOLATION") } } } test("enable UniForm on existing table but IcebergCompat isn't enabled - fail") { allReaderWriterVersions.foreach { case (r, w) => withTempTableAndDir { case (id, loc) => executeSql(s""" |CREATE TABLE $id (ID INT) USING DELTA LOCATION $loc TBLPROPERTIES ( | 'delta.minReaderVersion' = $r, | 'delta.minWriterVersion' = $w |)""".stripMargin) val e = intercept[DeltaUnsupportedOperationException] { executeSql(s"ALTER TABLE $id SET TBLPROPERTIES " + s"('delta.universalFormat.enabledFormats' = 'iceberg')") } assert(e.getErrorClass === "DELTA_UNIVERSAL_FORMAT_VIOLATION") } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/UpdateMetricsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import com.databricks.spark.util.DatabricksLogging import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.{Dataset, QueryTest} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.expr import org.apache.spark.sql.test.SharedSparkSession /** * Tests for metrics of Delta UPDATE command. */ class UpdateMetricsSuite extends QueryTest with SharedSparkSession with DatabricksLogging with DeltaSQLCommandTest { /** * Case class to parameterize tests. */ case class TestConfiguration( partitioned: Boolean, cdfEnabled: Boolean ) /** * Case class to parameterize metric results. */ case class TestMetricResults( operationMetrics: Map[String, Long] ) /** * Helper to generate tests for all configuration parameters. */ protected def testUpdateMetrics(name: String)(testFn: TestConfiguration => Unit): Unit = { for { partitioned <- BOOLEAN_DOMAIN cdfEnabled <- Seq(false) } { val testConfig = TestConfiguration(partitioned = partitioned, cdfEnabled = cdfEnabled ) var testName = s"update-metrics: $name - Partitioned = $partitioned, cdfEnabled = $cdfEnabled" test(testName) { testFn(testConfig) } } } /** * Create a table from the provided dataset. * * If an partitioned table is needed, then we create one data partition per Spark partition, * i.e. every data partition will contain one file. * * Also an extra column is added to be used in non-partition filters. */ protected def createTempTable( table: Dataset[_], tableName: String, testConfig: TestConfiguration): Unit = { val numRows = table.count() val numPartitions = table.rdd.getNumPartitions val numRowsPerPart = if (numRows > 0 && numPartitions < numRows) { numRows / numPartitions } else { 1 } val partitionBy = if (testConfig.partitioned) { Seq("partCol") } else { Seq() } table.withColumn("partCol", expr(s"floor(id / $numRowsPerPart)")) .withColumn("extraCol", expr(s"$numRows - id")) .write .partitionBy(partitionBy: _*) .format("delta") .saveAsTable(tableName) } /** * Run an update command and capture operation metrics from Delta log. * */ private def runUpdateAndCaptureMetrics( table: Dataset[_], where: String, testConfig: TestConfiguration): TestMetricResults = { val tableName = "target" val whereClause = if (where.nonEmpty) { s"WHERE $where" } else { "" } var operationMetrics: Map[String, Long] = null import testImplicits._ withSQLConf( DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true", DeltaSQLConf.DELTA_SKIP_RECORDING_EMPTY_COMMITS.key -> "false", DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> testConfig.cdfEnabled.toString) { withTable(tableName) { createTempTable(table, tableName, testConfig) val resultDf = spark.sql(s"UPDATE $tableName SET id = -1 $whereClause") operationMetrics = DeltaMetricsUtils.getLastOperationMetrics(tableName) // Check operation metrics against commit actions. val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) DeltaMetricsUtils.checkOperationMetricsAgainstCommitActions( deltaLog, deltaLog.update().version, operationMetrics) } } TestMetricResults( operationMetrics ) } /** * Run a update command and check all available metrics. * We allow some metrics to be missing, by setting their value to -1. */ private def runUpdateAndCheckMetrics( table: Dataset[_], where: String, expectedOperationMetrics: Map[String, Long], testConfig: TestConfiguration): Unit = { // Run the update capture and get all metrics. val results = runUpdateAndCaptureMetrics(table, where, testConfig) // Check operation metrics schema. val unknownKeys = results.operationMetrics.keySet -- DeltaOperationMetrics.UPDATE -- DeltaOperationMetrics.WRITE assert(unknownKeys.isEmpty, s"Unknown operation metrics for UPDATE command: ${unknownKeys.mkString(", ")}") // Check values of expected operation metrics. For all unspecified deterministic metrics, // we implicitly expect a zero value. val requiredMetrics = Set( "numCopiedRows", "numUpdatedRows", "numAddedFiles", "numRemovedFiles", "numAddedChangeFiles") val expectedMetricsWithDefaults = requiredMetrics.map(k => k -> 0L).toMap ++ expectedOperationMetrics val expectedMetricsFiltered = expectedMetricsWithDefaults.filter(_._2 >= 0) DeltaMetricsUtils.checkOperationMetrics( expectedMetrics = expectedMetricsFiltered, operationMetrics = results.operationMetrics) // Check time operation metrics. val expectedTimeMetrics = Set("scanTimeMs", "rewriteTimeMs", "executionTimeMs").filter( k => expectedOperationMetrics.get(k).forall(_ >= 0) ) DeltaMetricsUtils.checkOperationTimeMetrics( operationMetrics = results.operationMetrics, expectedMetrics = expectedTimeMetrics) } for (whereClause <- Seq("", "1 = 1")) { testUpdateMetrics(s"update all with where = '$whereClause'") { testConfig => val numFiles = 5 val numRows = 100 val numAddedChangeFiles = if (testConfig.partitioned && testConfig.cdfEnabled) { 5 } else if (testConfig.cdfEnabled) { 2 } else { 0 } runUpdateAndCheckMetrics( table = spark.range(start = 0, end = numRows, step = 1, numPartitions = numFiles), where = whereClause, expectedOperationMetrics = Map( "numCopiedRows" -> 0, "numUpdatedRows" -> -1, "numOutputRows" -> -1, "numFiles" -> -1, "numAddedFiles" -> -1, "numRemovedFiles" -> numFiles, "numAddedChangeFiles" -> numAddedChangeFiles ), testConfig = testConfig ) } } testUpdateMetrics("update with false predicate") { testConfig => runUpdateAndCheckMetrics( table = spark.range(start = 0, end = 100, step = 1, numPartitions = 5), where = "1 != 1", expectedOperationMetrics = Map( "numCopiedRows" -> 0, "numUpdatedRows" -> 0, "numAddedFiles" -> 0, "numRemovedFiles" -> 0, "numAddedChangeFiles" -> 0, "scanTimeMs" -> -1, "rewriteTimeMs" -> -1, "executionTimeMs" -> -1 ), testConfig = testConfig ) } testUpdateMetrics("update with unsatisfied static predicate") { testConfig => runUpdateAndCheckMetrics( table = spark.range(start = 0, end = 100, step = 1, numPartitions = 5), where = "id < 0 or id > 100", expectedOperationMetrics = Map( "numCopiedRows" -> 0, "numUpdatedRows" -> 0, "numAddedFiles" -> 0, "numRemovedFiles" -> 0, "numAddedChangeFiles" -> 0, "scanTimeMs" -> -1, "rewriteTimeMs" -> -1, "executionTimeMs" -> -1 ), testConfig = testConfig ) } testUpdateMetrics("update with unsatisfied dynamic predicate") { testConfig => runUpdateAndCheckMetrics( table = spark.range(start = 0, end = 100, step = 1, numPartitions = 5), where = "id / 200 > 1 ", expectedOperationMetrics = Map( "numCopiedRows" -> 0, "numUpdatedRows" -> 0, "numAddedFiles" -> 0, "numRemovedFiles" -> 0, "numAddedChangeFiles" -> 0, "scanTimeMs" -> -1, "rewriteTimeMs" -> -1, "executionTimeMs" -> -1 ), testConfig = testConfig ) } for (whereClause <- Seq("id = 0", "id >= 49 and id < 50")) { testUpdateMetrics(s"update one row with where = `$whereClause`") { testConfig => var numCopiedRows = 19 val numAddedFiles = 1 var numRemovedFiles = 1 runUpdateAndCheckMetrics( table = spark.range(start = 0, end = 100, step = 1, numPartitions = 5), where = whereClause, expectedOperationMetrics = Map( "numCopiedRows" -> numCopiedRows, "numUpdatedRows" -> 1, "numAddedFiles" -> numAddedFiles, "numRemovedFiles" -> numRemovedFiles, "numAddedChangeFiles" -> { if (testConfig.cdfEnabled) { 1 } else { 0 } } ), testConfig = testConfig ) } } testUpdateMetrics("update one file") { testConfig => runUpdateAndCheckMetrics( table = spark.range(start = 0, end = 100, step = 1, numPartitions = 5), where = "id < 20", expectedOperationMetrics = Map( "numCopiedRows" -> 0, "numUpdatedRows" -> 20, "numAddedFiles" -> 1, "numRemovedFiles" -> 1, "numAddedChangeFiles" -> { if (testConfig.cdfEnabled) { 1 } else { 0 } } ), testConfig = testConfig ) } testUpdateMetrics("update one row per file") { testConfig => val numPartitions = 5 var numCopiedRows = 95 val numAddedFiles = if (testConfig.partitioned) 5 else 2 var numRemovedFiles = 5 var unpartitionedNumAddFiles = 2 runUpdateAndCheckMetrics( table = spark.range(start = 0, end = 100, step = 1, numPartitions = numPartitions), where = "id in (5, 25, 45, 65, 85)", expectedOperationMetrics = Map( "numCopiedRows" -> numCopiedRows, "numUpdatedRows" -> 5, "numAddedFiles" -> { if (testConfig.partitioned) { 5 } else { unpartitionedNumAddFiles } }, "numRemovedFiles" -> numRemovedFiles, "numAddedChangeFiles" -> { if (testConfig.cdfEnabled) { if (testConfig.partitioned) { 5 } else { unpartitionedNumAddFiles } } else { 0 } } ), testConfig = testConfig ) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/UpdateSQLSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.actions.{AddFile, FileAction, RemoveFile} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaExcludedTestMixin, DeltaSQLCommandTest} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.{AnalysisException, QueryTest, Row} import org.apache.spark.sql.errors.QueryExecutionErrors.toSQLType import org.apache.spark.sql.functions.col import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy trait UpdateSQLMixin extends UpdateBaseMixin with DeltaSQLCommandTest with DeltaDMLTestUtils { override protected def executeUpdate( target: String, set: String, where: String = null): Unit = { val whereClause = Option(where).map(c => s"WHERE $c").getOrElse("") sql(s"UPDATE $target SET $set $whereClause") } } trait UpdateSQLTests extends UpdateSQLMixin { import testImplicits._ test("explain") { append(Seq((2, 2)).toDF("key", "value")) val df = sql(s"EXPLAIN UPDATE $tableSQLIdentifier SET key = 1, value = 2 WHERE key = 2") val outputs = df.collect().map(_.mkString).mkString assert(outputs.contains("Delta")) assert(!outputs.contains("index") && !outputs.contains("ActionLog")) // no change should be made by explain checkAnswer(readDeltaTableByIdentifier(), Row(2, 2)) } test("SC-11376: Update command should check target columns during analysis, same key") { val targetDF = spark.read.json( """ {"a": {"c": {"d": 'random', "e": 'str'}, "g": 1}, "z": 10} {"a": {"c": {"d": 'random2', "e": 'str2'}, "g": 2}, "z": 20}""" .split("\n").toSeq.toDS()) testAnalysisException( targetDF, set = "z = 30" :: "z = 40" :: Nil, errMsgs = "There is a conflict from these SET columns" :: Nil) testAnalysisException( targetDF, set = "a.c.d = 'rand'" :: "a.c.d = 'RANDOM2'" :: Nil, errMsgs = "There is a conflict from these SET columns" :: Nil) } test("update a dataset temp view") { withTable("tab") { withTempView("v") { Seq((0, 3)).toDF("key", "value").write.format("delta").saveAsTable("tab") spark.table("tab").as("name").createTempView("v") sql("UPDATE v SET key = 1 WHERE key = 0 AND value = 3") checkAnswer(spark.table("tab"), Row(1, 3)) } } } test("update a SQL temp view") { withTable("tab") { withTempView("v") { Seq((0, 3)).toDF("key", "value").write.format("delta").saveAsTable("tab") sql("CREATE TEMP VIEW v AS SELECT * FROM tab") QueryTest.checkAnswer(sql("UPDATE v SET key = 1 WHERE key = 0 AND value = 3"), Seq(Row(1))) checkAnswer(spark.table("tab"), Row(1, 3)) } } } Seq(true, false).foreach { partitioned => test(s"User defined _change_type column doesn't get dropped - partitioned=$partitioned") { withTable("tab") { sql( s"""CREATE TABLE tab USING DELTA |${if (partitioned) "PARTITIONED BY (part) " else ""} |TBLPROPERTIES (delta.enableChangeDataFeed = false) |AS SELECT id, int(id / 10) AS part, 'foo' as _change_type |FROM RANGE(1000) |""".stripMargin) val rowsToUpdate = (1 to 1000 by 42).mkString("(", ", ", ")") executeUpdate("tab", "_change_type = 'bar'", s"id in $rowsToUpdate") sql("SELECT id, _change_type FROM tab").collect().foreach { row => val _change_type = row.getString(1) assert(_change_type === "foo" || _change_type === "bar", s"Invalid _change_type for id=${row.get(0)}") } } } } // The following two tests are run only against the SQL API because using the Scala API // incorrectly triggers the analyzer rule [[ResolveRowLevelCommandAssignments]] which allows // the casts without respecting the value of `storeAssignmentPolicy`. // Casts that are not valid upcasts (e.g. string -> boolean) are not allowed with // storeAssignmentPolicy = STRICT. test("invalid implicit cast string source type into boolean target, " + s"storeAssignmentPolicy = ${StoreAssignmentPolicy.STRICT}") { append(Seq((99, true), (100, false), (101, true)).toDF("key", "value")) withSQLConf( SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.STRICT.toString, DeltaSQLConf.UPDATE_AND_MERGE_CASTING_FOLLOWS_ANSI_ENABLED_FLAG.key -> "false") { checkError( intercept[AnalysisException] { executeUpdate(target = tableSQLIdentifier, set = "value = 'false'") }, "CANNOT_UP_CAST_DATATYPE", parameters = Map( "expression" -> "'false'", "sourceType" -> toSQLType("STRING"), "targetType" -> toSQLType("BOOLEAN"), "details" -> ("The type path of the target object is:\n\nYou can either add an explicit " + "cast to the input data or choose a higher precision type of the field in the target " + "object"))) } } // Implicit casts that are not upcasts are not allowed with storeAssignmentPolicy = STRICT. test("valid implicit cast string source type into int target, " + s"storeAssignmentPolicy = ${StoreAssignmentPolicy.STRICT}") { append(Seq((99, 2), (100, 4), (101, 3)).toDF("key", "value")) withSQLConf( SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.STRICT.toString, DeltaSQLConf.UPDATE_AND_MERGE_CASTING_FOLLOWS_ANSI_ENABLED_FLAG.key -> "false") { checkError( intercept[AnalysisException] { executeUpdate(target = tableSQLIdentifier, set = "value = '5'") }, "CANNOT_UP_CAST_DATATYPE", parameters = Map( "expression" -> "'5'", "sourceType" -> toSQLType("STRING"), "targetType" -> toSQLType("INT"), "details" -> ("The type path of the target object is:\n\nYou can either add an explicit " + "cast to the input data or choose a higher precision type of the field in the target " + "object"))) } } } trait UpdateSQLWithDeletionVectorsMixin extends UpdateSQLMixin with DeltaExcludedTestMixin with DeletionVectorsTestUtils { override def beforeAll(): Unit = { super.beforeAll() enableDeletionVectors(spark, update = true) } override def excluded: Seq[String] = super.excluded ++ Seq( // The following two tests must fail when DV is used. Covered by another test case: // "throw error when non-pinned TahoeFileIndex snapshot is used". "data and partition predicates - Partition=true Skipping=false", "data and partition predicates - Partition=false Skipping=false", // The scan schema contains additional row index filter columns. "schema pruning on finding files to update", "nested schema pruning on finding files to update" ) } trait UpdateSQLWithDeletionVectorsTests extends UpdateSQLWithDeletionVectorsMixin { test("repeated UPDATE produces deletion vectors") { withTempDir { dir => val path = dir.getCanonicalPath val log = DeltaLog.forTable(spark, path) spark.range(0, 10, 1, numPartitions = 2).write.format("delta").save(path) // scalastyle:off argcount def updateAndCheckLog( where: String, expectedAnswer: Seq[Row], numAddFilesWithDVs: Int, sumNumRowsInAddFileWithDV: Int, sumNumRowsInAddFileWithoutDV: Int, sumDvCardinalityInAddFile: Long, numRemoveFilesWithDVs: Int, sumNumRowsInRemoveFileWithDV: Int, sumNumRowsInRemoveFileWithoutDV: Int, sumDvCardinalityInRemoveFile: Long): Unit = { executeUpdate(s"delta.`$path`", "id = -1", where) checkAnswer(sql(s"SELECT * FROM delta.`$path`"), expectedAnswer) val fileActions = log.getChanges(log.update().version).flatMap(_._2) .collect { case f: FileAction => f } .toSeq val addFiles = fileActions.collect { case f: AddFile => f } val removeFiles = fileActions.collect { case f: RemoveFile => f } val (addFilesWithDV, addFilesWithoutDV) = addFiles.partition(_.deletionVector != null) assert(addFilesWithDV.size === numAddFilesWithDVs) assert( addFilesWithDV.map(_.numPhysicalRecords.getOrElse(0L)).sum === sumNumRowsInAddFileWithDV) assert( addFilesWithDV.map(_.deletionVector.cardinality).sum === sumDvCardinalityInAddFile) assert( addFilesWithoutDV.map(_.numPhysicalRecords.getOrElse(0L)).sum === sumNumRowsInAddFileWithoutDV) val (removeFilesWithDV, removeFilesWithoutDV) = removeFiles.partition(_.deletionVector != null) assert(removeFilesWithDV.size === numRemoveFilesWithDVs) assert( removeFilesWithDV.map(_.numPhysicalRecords.getOrElse(0L)).sum === sumNumRowsInRemoveFileWithDV) assert( removeFilesWithDV.map(_.deletionVector.cardinality).sum === sumDvCardinalityInRemoveFile) assert( removeFilesWithoutDV.map(_.numPhysicalRecords.getOrElse(0L)).sum === sumNumRowsInRemoveFileWithoutDV) } // scalastyle:on argcount def assertDVMetrics( numUpdatedRows: Long = 0, numCopiedRows: Long = 0, numDeletionVectorsAdded: Long = 0, numDeletionVectorsRemoved: Long = 0, numDeletionVectorsUpdated: Long = 0): Unit = { val table = io.delta.tables.DeltaTable.forPath(path) val updateMetrics = DeltaMetricsUtils.getLastOperationMetrics(table) assert(updateMetrics.getOrElse("numUpdatedRows", -1) === numUpdatedRows) assert(updateMetrics.getOrElse("numCopiedRows", -1) === numCopiedRows) assert(updateMetrics.getOrElse("numDeletionVectorsAdded", -1) === numDeletionVectorsAdded) assert( updateMetrics.getOrElse("numDeletionVectorsRemoved", -1) === numDeletionVectorsRemoved) assert( updateMetrics.getOrElse("numDeletionVectorsUpdated", -1) === numDeletionVectorsUpdated) } // DV created. 4 rows updated. updateAndCheckLog( "id % 3 = 0", Seq(-1, 1, 2, -1, 4, 5, -1, 7, 8, -1).map(Row(_)), numAddFilesWithDVs = 2, sumNumRowsInAddFileWithDV = 10, sumNumRowsInAddFileWithoutDV = 4, sumDvCardinalityInAddFile = 4, numRemoveFilesWithDVs = 0, sumNumRowsInRemoveFileWithDV = 0, sumNumRowsInRemoveFileWithoutDV = 10, sumDvCardinalityInRemoveFile = 0) assertDVMetrics(numUpdatedRows = 4, numDeletionVectorsAdded = 2) // DV updated. 2 rows from the original file updated. updateAndCheckLog( "id % 4 = 0", Seq(-1, 1, 2, -1, -1, 5, -1, 7, -1, -1).map(Row(_)), numAddFilesWithDVs = 2, sumNumRowsInAddFileWithDV = 10, sumNumRowsInAddFileWithoutDV = 2, sumDvCardinalityInAddFile = 6, numRemoveFilesWithDVs = 2, sumNumRowsInRemoveFileWithDV = 10, sumNumRowsInRemoveFileWithoutDV = 0, sumDvCardinalityInRemoveFile = 4) assertDVMetrics( numUpdatedRows = 2, numDeletionVectorsAdded = 2, numDeletionVectorsRemoved = 2, numDeletionVectorsUpdated = 2) // Original files DV removed, because all rows in the SECOND FILE are deleted. updateAndCheckLog( "id IN (5, 7)", Seq(-1, 1, 2, -1, -1, -1, -1, -1, -1, -1).map(Row(_)), numAddFilesWithDVs = 0, sumNumRowsInAddFileWithDV = 0, sumNumRowsInAddFileWithoutDV = 2, sumDvCardinalityInAddFile = 0, numRemoveFilesWithDVs = 1, sumNumRowsInRemoveFileWithDV = 5, sumNumRowsInRemoveFileWithoutDV = 0, sumDvCardinalityInRemoveFile = 3) assertDVMetrics(numUpdatedRows = 2, numDeletionVectorsRemoved = 1) } } test("UPDATE a whole partition do not produce DVs") { withTempDir { dir => val path = dir.getCanonicalPath val log = DeltaLog.forTable(spark, path) spark.range(10).withColumn("part", col("id") % 2) .write .format("delta") .partitionBy("part") .save(path) executeUpdate(s"delta.`$path`", "id = -1", where = "part = 0") checkAnswer( sql(s"SELECT * FROM delta.`$path`"), Row(-1, 0) :: Row(1, 1) :: Row(-1, 0) :: Row(3, 1) :: Row(-1, 0) :: Row(5, 1) :: Row(-1, 0) :: Row(7, 1) :: Row(-1, 0) :: Row(9, 1) :: Nil) val fileActions = log.getChanges(log.update().version).flatMap(_._2) .collect { case f: FileAction => f } .toSeq val addFiles = fileActions.collect { case f: AddFile => f } val removeFiles = fileActions.collect { case f: RemoveFile => f } assert(addFiles.map(_.numPhysicalRecords.getOrElse(0L)).sum === 5) assert(removeFiles.map(_.numPhysicalRecords.getOrElse(0L)).sum === 5) for (a <- addFiles) assert(a.deletionVector === null) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/UpdateScalaSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.test.{DeltaExcludedTestMixin, DeltaSQLCommandTest} import org.apache.spark.sql.{functions, Row} trait UpdateScalaMixin extends UpdateBaseMixin with DeltaSQLCommandTest with DeltaExcludedTestMixin with DeltaDMLTestUtilsPathBased { override protected def executeUpdate( target: String, set: String, where: String = null): Unit = { executeUpdate(target, set.split(","), where) } override protected def executeUpdate( target: String, set: Seq[String], where: String): Unit = { val deltaTable = DeltaTestUtils.getDeltaTableForIdentifierOrPath( spark, DeltaTestUtils.getTableIdentifierOrPath(target)) val setColumns = set.map { assign => val kv = assign.split("=") require(kv.size == 2) kv(0).trim -> kv(1).trim }.toMap if (where == null) { deltaTable.updateExpr(setColumns) } else { deltaTable.updateExpr(where, setColumns) } } } trait UpdateScalaTests extends UpdateScalaMixin { import testImplicits._ test("update usage test - without condition") { append(Seq((1, 10), (2, 20), (3, 30), (4, 40)).toDF("key", "value")) val table = io.delta.tables.DeltaTable.forPath(tempPath) table.updateExpr(Map("key" -> "100")) checkAnswer(readDeltaTable(tempPath), Row(100, 10) :: Row(100, 20) :: Row(100, 30) :: Row(100, 40) :: Nil) } test("update usage test - without condition, using Column") { append(Seq((1, 10), (2, 20), (3, 30), (4, 40)).toDF("key", "value")) val table = io.delta.tables.DeltaTable.forPath(tempPath) table.update(Map("key" -> functions.expr("100"))) checkAnswer(readDeltaTable(tempPath), Row(100, 10) :: Row(100, 20) :: Row(100, 30) :: Row(100, 40) :: Nil) } test("update usage test - with condition") { append(Seq((1, 10), (2, 20), (3, 30), (4, 40)).toDF("key", "value")) val table = io.delta.tables.DeltaTable.forPath(tempPath) table.updateExpr("key = 1 or key = 2", Map("key" -> "100")) checkAnswer(readDeltaTable(tempPath), Row(100, 10) :: Row(100, 20) :: Row(3, 30) :: Row(4, 40) :: Nil) } test("update usage test - with condition, using Column") { append(Seq((1, 10), (2, 20), (3, 30), (4, 40)).toDF("key", "value")) val table = io.delta.tables.DeltaTable.forPath(tempPath) table.update(functions.expr("key = 1 or key = 2"), Map("key" -> functions.expr("100"), "value" -> functions.expr("101"))) checkAnswer(readDeltaTable(tempPath), Row(100, 101) :: Row(100, 101) :: Row(3, 30) :: Row(4, 40) :: Nil) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/UpdateSuiteBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine import java.util.Locale import scala.language.implicitConversions import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.test.shims.UnsupportedTableOperationErrorShims import org.apache.spark.{SparkThrowable, SparkUnsupportedOperationException} import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.execution.FileSourceScanExec import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.functions.{lit, struct} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ trait UpdateBaseMixin extends QueryTest with SharedSparkSession with DeltaDMLTestUtils with DeltaSQLTestUtils with DeltaTestUtilsForTempViews { import testImplicits._ protected def executeUpdate(target: String, set: Seq[String], where: String): Unit = { executeUpdate(target, set.mkString(", "), where) } protected def executeUpdate(target: String, set: String, where: String = null): Unit implicit def jsonStringToSeq(json: String): Seq[String] = json.split("\n") protected def checkUpdate( condition: Option[String], setClauses: String, expectedResults: Seq[Row], tableName: Option[String] = None, prefix: String = ""): Unit = { val target = tableName.getOrElse(tableSQLIdentifier) executeUpdate(target, setClauses, where = condition.orNull) checkAnswer( readDeltaTableByIdentifier(target).select(s"${prefix}key", s"${prefix}value"), expectedResults) } protected def checkUpdateJson( target: Seq[String], source: Seq[String] = Nil, updateWhere: String, set: Seq[String], expected: Seq[String]): Unit = { withTempView("source") { def toDF(jsonStrs: Seq[String]) = spark.read.json(jsonStrs.toDS()) append(toDF(target)) if (source.nonEmpty) { toDF(source).createOrReplaceTempView("source") } executeUpdate(tableSQLIdentifier, set, updateWhere) checkAnswer(readDeltaTableByIdentifier(), toDF(expected)) dropTable() } } protected def testAnalysisException( targetDF: DataFrame, set: Seq[String], where: String = null, errMsgs: Seq[String] = Nil): Unit = { dropTable() append(targetDF) val e = intercept[AnalysisException] { executeUpdate(target = tableSQLIdentifier, set, where) } errMsgs.foreach { msg => assert(e.getMessage.toLowerCase(Locale.ROOT).contains(msg.toLowerCase(Locale.ROOT))) } } } trait UpdateBaseTempViewTests extends UpdateBaseMixin { import testImplicits._ test("different variations of column references - TempView") { append(Seq((99, 2), (100, 4), (101, 3), (102, 5)).toDF("key", "value")) readDeltaTableByIdentifier().createOrReplaceTempView("tblName") checkUpdate( condition = Some("tblName.key = 101"), setClauses = "tblName.value = -1", expectedResults = Row(99, 2) :: Row(100, 4) :: Row(101, -1) :: Row(102, 5) :: Nil, tableName = Some("tblName")) checkUpdate( condition = Some("`tblName`.`key` = 102"), setClauses = "`tblName`.`value` = -1", expectedResults = Row(99, 2) :: Row(100, 4) :: Row(101, -1) :: Row(102, -1) :: Nil, tableName = Some("tblName")) } Seq(true, false).foreach { isPartitioned => val testName = s"test update on temp view - basic - Partition=$isPartitioned" testWithTempView(testName) { isSQLTempView => val partitions = if (isPartitioned) "key" :: Nil else Nil append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value"), partitions) createTempViewFromTable(tableSQLIdentifier, isSQLTempView) checkUpdate( condition = Some("key >= 1"), setClauses = "value = key + value, key = key + 1", expectedResults = Row(0, 3) :: Row(2, 5) :: Row(2, 2) :: Row(3, 4) :: Nil, tableName = Some("v")) } } private def testInvalidTempViews(name: String)( text: String, expectedErrorMsgForSQLTempView: String = null, expectedErrorMsgForDataSetTempView: String = null, expectedErrorClassForSQLTempView: String = null, expectedErrorClassForDataSetTempView: String = null): Unit = { testWithTempView(s"test update on temp view - $name") { isSQLTempView => withTable("tab") { Seq((0, 3), (1, 2)).toDF("key", "value").write.format("delta").saveAsTable("tab") createTempViewFromSelect(text, isSQLTempView) val ex = intercept[AnalysisException] { executeUpdate( "v", where = "key >= 1 and value < 3", set = "value = key + value, key = key + 1" ) } testErrorMessageAndClass( isSQLTempView, ex, expectedErrorMsgForSQLTempView, expectedErrorMsgForDataSetTempView, expectedErrorClassForSQLTempView, expectedErrorClassForDataSetTempView) } } } testInvalidTempViews("subset cols")( text = "SELECT key FROM tab", expectedErrorClassForSQLTempView = "UNRESOLVED_COLUMN.WITH_SUGGESTION", expectedErrorClassForDataSetTempView = "UNRESOLVED_COLUMN.WITH_SUGGESTION" ) testInvalidTempViews("superset cols")( text = "SELECT key, value, 1 FROM tab", // The analyzer can't tell whether the table originally had the extra column or not. expectedErrorMsgForSQLTempView = "Can't resolve column 1 in root", expectedErrorMsgForDataSetTempView = "Can't resolve column 1 in root" ) private def testComplexTempViews(name: String)(text: String, expectedResult: Seq[Row]) = { testWithTempView(s"test update on temp view - $name") { isSQLTempView => withTable("tab") { Seq((0, 3), (1, 2)).toDF("key", "value").write.format("delta").saveAsTable("tab") createTempViewFromSelect(text, isSQLTempView) executeUpdate( "v", where = "key >= 1 and value < 3", set = "value = key + value, key = key + 1" ) checkAnswer(spark.read.format("delta").table("v"), expectedResult) } } } testComplexTempViews("nontrivial projection")( text = "SELECT value as key, key as value FROM tab", expectedResult = Seq(Row(3, 0), Row(3, 3)) ) testComplexTempViews("view with too many internal aliases")( text = "SELECT * FROM (SELECT * FROM tab AS t1) AS t2", expectedResult = Seq(Row(0, 3), Row(2, 3)) ) } trait UpdateBaseMiscTests extends UpdateBaseMixin { import testImplicits._ val fileFormat: String = "parquet" test("basic case") { append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value")) checkUpdate(condition = None, setClauses = "key = 1, value = 2", expectedResults = Row(1, 2) :: Row(1, 2) :: Row(1, 2) :: Row(1, 2) :: Nil) } Seq(true, false).foreach { isPartitioned => test(s"basic update - Partition=$isPartitioned") { val partitions = if (isPartitioned) "key" :: Nil else Nil append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value"), partitions) checkUpdate( condition = Some("key >= 1"), setClauses = "value = key + value, key = key + 1", expectedResults = Row(0, 3) :: Row(2, 5) :: Row(2, 2) :: Row(3, 4) :: Nil) } } Seq(true, false).foreach { isPartitioned => test(s"basic update - Delta table by name - Partition=$isPartitioned") { withTable("delta_table") { val partitionByClause = if (isPartitioned) "PARTITIONED BY (key)" else "" sql(s""" |CREATE TABLE delta_table(key INT, value INT) USING delta |$partitionByClause """.stripMargin) Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value") .write .format("delta") .mode("append") .saveAsTable("delta_table") checkUpdate( condition = Some("key >= 1"), setClauses = "value = key + value, key = key + 1", expectedResults = Row(0, 3) :: Row(2, 5) :: Row(2, 2) :: Row(3, 4) :: Nil, tableName = Some("delta_table")) } } } Seq(true, false).foreach { skippingEnabled => Seq(true, false).foreach { isPartitioned => test(s"data and partition predicates - Partition=$isPartitioned Skipping=$skippingEnabled") { withSQLConf(DeltaSQLConf.DELTA_STATS_SKIPPING.key -> skippingEnabled.toString) { val partitions = if (isPartitioned) "key" :: Nil else Nil append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value"), partitions) checkUpdate(condition = Some("key >= 1 and value != 4"), setClauses = "value = key + value, key = key + 5", expectedResults = Row(0, 3) :: Row(7, 4) :: Row(1, 4) :: Row(6, 2) :: Nil) } } } } Seq(true, false).foreach { isPartitioned => test(s"SC-12276: table has null values - partitioned=$isPartitioned") { val partitions = if (isPartitioned) "key" :: Nil else Nil append(Seq(("a", 1), (null, 2), (null, 3), ("d", 4)).toDF("key", "value"), partitions) // predicate evaluates to null; no-op checkUpdate(condition = Some("key = null"), setClauses = "value = -1", expectedResults = Row("a", 1) :: Row(null, 2) :: Row(null, 3) :: Row("d", 4) :: Nil) checkUpdate(condition = Some("key = 'a'"), setClauses = "value = -1", expectedResults = Row("a", -1) :: Row(null, 2) :: Row(null, 3) :: Row("d", 4) :: Nil) checkUpdate(condition = Some("key is null"), setClauses = "value = -2", expectedResults = Row("a", -1) :: Row(null, -2) :: Row(null, -2) :: Row("d", 4) :: Nil) checkUpdate(condition = Some("key is not null"), setClauses = "value = -3", expectedResults = Row("a", -3) :: Row(null, -2) :: Row(null, -2) :: Row("d", -3) :: Nil) checkUpdate(condition = Some("key <=> null"), setClauses = "value = -4", expectedResults = Row("a", -3) :: Row(null, -4) :: Row(null, -4) :: Row("d", -3) :: Nil) } } test("basic case - condition is false") { append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value")) checkUpdate(condition = Some("1 != 1"), setClauses = "key = 1, value = 2", expectedResults = Row(2, 2) :: Row(1, 4) :: Row(1, 1) :: Row(0, 3) :: Nil) } test("basic case - condition is true") { append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value")) checkUpdate(condition = Some("1 = 1"), setClauses = "key = 1, value = 2", expectedResults = Row(1, 2) :: Row(1, 2) :: Row(1, 2) :: Row(1, 2) :: Nil) } Seq(true, false).foreach { isPartitioned => test(s"basic update - without where - Partition=$isPartitioned") { val partitions = if (isPartitioned) "key" :: Nil else Nil append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value"), partitions) checkUpdate(condition = None, setClauses = "key = 1, value = 2", expectedResults = Row(1, 2) :: Row(1, 2) :: Row(1, 2) :: Row(1, 2) :: Nil) } } Seq(true, false).foreach { isPartitioned => test(s"basic update - without where and partial columns - Partition=$isPartitioned") { val partitions = if (isPartitioned) "key" :: Nil else Nil append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value"), partitions) checkUpdate(condition = None, setClauses = "key = 1", expectedResults = Row(1, 1) :: Row(1, 2) :: Row(1, 3) :: Row(1, 4) :: Nil) } } Seq(true, false).foreach { isPartitioned => test(s"basic update - without where and out-of-order columns - Partition=$isPartitioned") { val partitions = if (isPartitioned) "key" :: Nil else Nil append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value"), partitions) checkUpdate(condition = None, setClauses = "value = 3, key = 1", expectedResults = Row(1, 3) :: Row(1, 3) :: Row(1, 3) :: Row(1, 3) :: Nil) } } Seq(true, false).foreach { isPartitioned => test(s"basic update - without where and complex input - Partition=$isPartitioned") { val partitions = if (isPartitioned) "key" :: Nil else Nil append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value"), partitions) checkUpdate(condition = None, setClauses = "value = key + 3, key = key + 1", expectedResults = Row(1, 3) :: Row(2, 4) :: Row(2, 4) :: Row(3, 5) :: Nil) } } Seq(true, false).foreach { isPartitioned => test(s"basic update - with where - Partition=$isPartitioned") { val partitions = if (isPartitioned) "key" :: Nil else Nil append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value"), partitions) checkUpdate(condition = Some("key = 1"), setClauses = "value = 3, key = 1", expectedResults = Row(1, 3) :: Row(2, 2) :: Row(0, 3) :: Row(1, 3) :: Nil) } } Seq(true, false).foreach { isPartitioned => test(s"basic update - with where and complex input - Partition=$isPartitioned") { val partitions = if (isPartitioned) "key" :: Nil else Nil append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value"), partitions) checkUpdate(condition = Some("key >= 1"), setClauses = "value = key + value, key = key + 1", expectedResults = Row(0, 3) :: Row(2, 5) :: Row(2, 2) :: Row(3, 4) :: Nil) } } Seq(true, false).foreach { isPartitioned => test(s"basic update - with where and no row matched - Partition=$isPartitioned") { val partitions = if (isPartitioned) "key" :: Nil else Nil append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value"), partitions) checkUpdate(condition = Some("key >= 10"), setClauses = "value = key + value, key = key + 1", expectedResults = Row(0, 3) :: Row(1, 1) :: Row(1, 4) :: Row(2, 2) :: Nil) } } Seq(true, false).foreach { isPartitioned => test(s"type mismatch - Partition=$isPartitioned") { val partitions = if (isPartitioned) "key" :: Nil else Nil append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value"), partitions) checkUpdate(condition = Some("key >= 1"), setClauses = "value = key + cast(value as double), key = cast(key as double) + 1", expectedResults = Row(0, 3) :: Row(2, 5) :: Row(3, 4) :: Row(2, 2) :: Nil) } } Seq(true, false).foreach { isPartitioned => test(s"set to null - Partition=$isPartitioned") { val partitions = if (isPartitioned) "key" :: Nil else Nil append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value"), partitions) checkUpdate(condition = Some("key >= 1"), setClauses = "value = key, key = null + 1D", expectedResults = Row(0, 3) :: Row(null, 1) :: Row(null, 1) :: Row(null, 2) :: Nil) } } Seq(true, false).foreach { isPartitioned => test(s"basic update - TypeCoercion twice - Partition=$isPartitioned") { val partitions = if (isPartitioned) "key" :: Nil else Nil append(Seq((99, 2), (100, 4), (101, 3)).toDF("key", "value"), partitions) checkUpdate( condition = Some("cast(key as long) * cast('1.0' as decimal(38, 18)) > 100"), setClauses = "value = -3", expectedResults = Row(100, 4) :: Row(101, -3) :: Row(99, 2) :: Nil) } } for (storeAssignmentPolicy <- StoreAssignmentPolicy.values) test("upcast int source type into long target, storeAssignmentPolicy = " + s"$storeAssignmentPolicy") { append(Seq((99, 2L), (100, 4L), (101, 3L)).toDF("key", "value")) withSQLConf( SQLConf.STORE_ASSIGNMENT_POLICY.key -> storeAssignmentPolicy.toString, DeltaSQLConf.UPDATE_AND_MERGE_CASTING_FOLLOWS_ANSI_ENABLED_FLAG.key -> "false") { checkUpdate( condition = None, setClauses = "value = 4", expectedResults = Row(100, 4) :: Row(101, 4) :: Row(99, 4) :: Nil) } } // Casts that are not valid implicit casts (e.g. string -> boolean) are allowed only when // storeAssignmentPolicy is LEGACY or ANSI. STRICT is tested in [[UpdateSQLTests]] only due to // limitations when using the Scala API. for (storeAssignmentPolicy <- StoreAssignmentPolicy.values - StoreAssignmentPolicy.STRICT) test("invalid implicit cast string source type into boolean target, " + s"storeAssignmentPolicy = $storeAssignmentPolicy") { append(Seq((99, true), (100, false), (101, true)).toDF("key", "value")) withSQLConf( SQLConf.STORE_ASSIGNMENT_POLICY.key -> storeAssignmentPolicy.toString, DeltaSQLConf.UPDATE_AND_MERGE_CASTING_FOLLOWS_ANSI_ENABLED_FLAG.key -> "false") { checkUpdate( condition = None, setClauses = "value = 'false'", expectedResults = Row(100, false) :: Row(101, false) :: Row(99, false) :: Nil) } } // Valid implicit casts that are not upcasts (e.g. string -> int) are allowed only when // storeAssignmentPolicy is LEGACY or ANSI. STRICT is tested in [[UpdateSQLTests]] only due to // limitations when using the Scala API. for (storeAssignmentPolicy <- StoreAssignmentPolicy.values - StoreAssignmentPolicy.STRICT) test("valid implicit cast string source type into int target, " + s"storeAssignmentPolicy = ${storeAssignmentPolicy}") { append(Seq((99, 2), (100, 4), (101, 3)).toDF("key", "value")) withSQLConf( SQLConf.STORE_ASSIGNMENT_POLICY.key -> storeAssignmentPolicy.toString, DeltaSQLConf.UPDATE_AND_MERGE_CASTING_FOLLOWS_ANSI_ENABLED_FLAG.key -> "false") { checkUpdate( condition = None, setClauses = "value = '5'", expectedResults = Row(100, 5) :: Row(101, 5) :: Row(99, 5) :: Nil) } } test("update cached table") { append(Seq((2, 2), (1, 4)).toDF("key", "value")) readDeltaTableByIdentifier().cache() readDeltaTableByIdentifier().collect() executeUpdate(tableSQLIdentifier, set = "key = 3") checkAnswer(readDeltaTableByIdentifier(), Row(3, 2) :: Row(3, 4) :: Nil) } test("different variations of column references") { append(Seq((99, 2), (100, 4), (101, 3), (102, 5)).toDF("key", "value")) checkUpdate( condition = Some("key = 99"), setClauses = "value = -1", expectedResults = Row(99, -1) :: Row(100, 4) :: Row(101, 3) :: Row(102, 5) :: Nil) checkUpdate( condition = Some("`key` = 100"), setClauses = "`value` = -1", expectedResults = Row(99, -1) :: Row(100, -1) :: Row(101, 3) :: Row(102, 5) :: Nil) } test("target columns can have db and table qualifiers") { withTable("target") { spark.read.json(""" {"a": {"b.1": 1, "c.e": 'random'}, "d": 1} {"a": {"b.1": 3, "c.e": 'string'}, "d": 2}""" .split("\n").toSeq.toDS()).write.format("delta").saveAsTable("`target`") executeUpdate( target = "target", set = "`default`.`target`.a.`b.1` = -1, target.a.`c.e` = 'RANDOM'", where = "d = 1") checkAnswer(spark.table("target"), spark.read.json(""" {"a": {"b.1": -1, "c.e": 'RANDOM'}, "d": 1} {"a": {"b.1": 3, "c.e": 'string'}, "d": 2}""" .split("\n").toSeq.toDS())) } } test("Negative case - non-delta target") { writeTable( Seq((1, 1), (0, 3), (1, 5)).toDF("key1", "value").write.mode("overwrite").format("parquet"), tableSQLIdentifier) intercept[SparkThrowable] { executeUpdate(target = tableSQLIdentifier, set = "key1 = 3") } match { // Thrown when running with path-based SQL case e: DeltaAnalysisException if e.getCondition == "DELTA_TABLE_NOT_FOUND" => checkError(e, "DELTA_TABLE_NOT_FOUND", parameters = Map("tableName" -> tableSQLIdentifier.stripPrefix("delta."))) case e: DeltaAnalysisException if e.getCondition == "DELTA_MISSING_TRANSACTION_LOG" => checkErrorMatchPVals(e, "DELTA_MISSING_TRANSACTION_LOG", parameters = Map("operation" -> "read from", "path" -> ".*", "docLink" -> "https://.*")) // Thrown when running with path-based Scala API case e: DeltaAnalysisException if e.getCondition == "DELTA_MISSING_DELTA_TABLE" => checkError(e, "DELTA_MISSING_DELTA_TABLE", parameters = Map("tableName" -> tableSQLIdentifier.stripPrefix("delta."))) // Thrown when running with name-based SQL case e: SparkUnsupportedOperationException => checkError(e, UnsupportedTableOperationErrorShims.UNSUPPORTED_TABLE_OPERATION_ERROR_CODE, parameters = UnsupportedTableOperationErrorShims.updateTableErrorParameters( tableSQLIdentifier)) } } test("Negative case - check target columns during analysis") { withTable("table") { sql("CREATE TABLE table (s int, t string) USING delta PARTITIONED BY (s)") var ae = intercept[AnalysisException] { executeUpdate("table", set = "column_doesnt_exist = 'San Francisco'", where = "t = 'a'") } // The error class is renamed from MISSING_COLUMN to UNRESOLVED_COLUMN in Spark 3.4 assert(ae.getErrorClass == "UNRESOLVED_COLUMN.WITH_SUGGESTION" || ae.getErrorClass == "MISSING_COLUMN" ) withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") { executeUpdate(target = "table", set = "S = 1, T = 'b'", where = "T = 'a'") ae = intercept[AnalysisException] { executeUpdate(target = "table", set = "S = 1, s = 'b'", where = "s = 1") } assert(ae.message.contains("There is a conflict from these SET columns")) } withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { ae = intercept[AnalysisException] { executeUpdate(target = "table", set = "S = 1", where = "t = 'a'") } // The error class is renamed from MISSING_COLUMN to UNRESOLVED_COLUMN in Spark 3.4 assert(ae.getErrorClass == "UNRESOLVED_COLUMN.WITH_SUGGESTION" || ae.getErrorClass == "MISSING_COLUMN" ) ae = intercept[AnalysisException] { executeUpdate(target = "table", set = "S = 1, s = 'b'", where = "s = 1") } // The error class is renamed from MISSING_COLUMN to UNRESOLVED_COLUMN in Spark 3.4 assert(ae.getErrorClass == "UNRESOLVED_COLUMN.WITH_SUGGESTION" || ae.getErrorClass == "MISSING_COLUMN" ) // unresolved column in condition ae = intercept[AnalysisException] { executeUpdate(target = "table", set = "s = 1", where = "T = 'a'") } // The error class is renamed from MISSING_COLUMN to UNRESOLVED_COLUMN in Spark 3.4 assert(ae.getErrorClass == "UNRESOLVED_COLUMN.WITH_SUGGESTION" || ae.getErrorClass == "MISSING_COLUMN" ) } } } test("Negative case - UPDATE the child directory", NameBasedAccessIncompatible) { withTempDir { dir => val tempPath = dir.getCanonicalPath val df = Seq((2, 2), (3, 2)).toDF("key", "value") df.write.format("delta").partitionBy("key").save(tempPath) val e = intercept[AnalysisException] { executeUpdate( target = s"delta.`$tempPath/key=2`", set = "key = 1, value = 2", where = "value = 2") }.getMessage assert(e.contains("Expect a full scan of Delta sources, but found a partial scan")) } } test("Negative case - do not support subquery test") { append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value")) Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("c", "d").createOrReplaceTempView("source") // basic subquery val e0 = intercept[AnalysisException] { executeUpdate(target = tableSQLIdentifier, set = "key = 1", where = "key < (SELECT max(c) FROM source)") }.getMessage assert(e0.contains("Subqueries are not supported")) // subquery with EXISTS val e1 = intercept[AnalysisException] { executeUpdate(target = tableSQLIdentifier, set = "key = 1", where = "EXISTS (SELECT max(c) FROM source)") }.getMessage assert(e1.contains("Subqueries are not supported")) // subquery with NOT EXISTS val e2 = intercept[AnalysisException] { executeUpdate(target = tableSQLIdentifier, set = "key = 1", where = "NOT EXISTS (SELECT max(c) FROM source)") }.getMessage assert(e2.contains("Subqueries are not supported")) // subquery with IN val e3 = intercept[AnalysisException] { executeUpdate(target = tableSQLIdentifier, set = "key = 1", where = "key IN (SELECT max(c) FROM source)") }.getMessage assert(e3.contains("Subqueries are not supported")) // subquery with NOT IN val e4 = intercept[AnalysisException] { executeUpdate(target = tableSQLIdentifier, set = "key = 1", where = "key NOT IN (SELECT max(c) FROM source)") }.getMessage assert(e4.contains("Subqueries are not supported")) } test("nested data support") { // set a nested field checkUpdateJson(target = """ {"a": {"c": {"d": 'random', "e": 'str'}, "g": 1}, "z": 10} {"a": {"c": {"d": 'random2', "e": 'str2'}, "g": 2}, "z": 20}""", updateWhere = "z = 10", set = "a.c.d = 'RANDOM'" :: Nil, expected = """ {"a": {"c": {"d": 'RANDOM', "e": 'str'}, "g": 1}, "z": 10} {"a": {"c": {"d": 'random2', "e": 'str2'}, "g": 2}, "z": 20}""") // do nothing as condition has no match val unchanged = """ {"a": {"c": {"d": 'RANDOM', "e": 'str'}, "g": 1}, "z": 10} {"a": {"c": {"d": 'random2', "e": 'str2'}, "g": 2}, "z": 20}""" checkUpdateJson(target = unchanged, updateWhere = "z = 30", set = "a.c.d = 'RANDOMMMMM'" :: Nil, expected = unchanged) // set multiple nested fields at different levels checkUpdateJson( target = """ {"a": {"c": {"d": 'RANDOM', "e": 'str'}, "g": 1}, "z": 10} {"a": {"c": {"d": 'random2', "e": 'str2'}, "g": 2}, "z": 20}""", updateWhere = "z = 20", set = "a.c.d = 'RANDOM2'" :: "a.c.e = 'STR2'" :: "a.g = -2" :: "z = -20" :: Nil, expected = """ {"a": {"c": {"d": 'RANDOM', "e": 'str'}, "g": 1}, "z": 10} {"a": {"c": {"d": 'RANDOM2', "e": 'STR2'}, "g": -2}, "z": -20}""") // set nested fields to null checkUpdateJson( target = """ {"a": {"c": {"d": 'RANDOM', "e": 'str'}, "g": 1}, "z": 10} {"a": {"c": {"d": 'random2', "e": 'str2'}, "g": 2}, "z": 20}""", updateWhere = "a.c.d = 'random2'", set = "a.c = null" :: "a.g = null" :: Nil, expected = """ {"a": {"c": {"d": 'RANDOM', "e": 'str'}, "g": 1}, "z": 10} {"a": {"c": null, "g": null}, "z": 20}""") // set a top struct type column to null checkUpdateJson( target = """ {"a": {"c": {"d": 'RANDOM', "e": 'str'}, "g": 1}, "z": 10} {"a": {"c": {"d": 'random2', "e": 'str2'}, "g": 2}, "z": 20}""", updateWhere = "a.c.d = 'random2'", set = "a = null" :: Nil, expected = """ {"a": {"c": {"d": 'RANDOM', "e": 'str'}, "g": 1}, "z": 10} {"a": null, "z": 20}""") // set a nested field using named_struct checkUpdateJson( target = """ {"a": {"c": {"d": 'RANDOM', "e": 'str'}, "g": 1}, "z": 10} {"a": {"c": {"d": 'random2', "e": 'str2'}, "g": 2}, "z": 20}""", updateWhere = "a.g = 2", set = "a.c = named_struct('d', 'RANDOM2', 'e', 'STR2')" :: Nil, expected = """ {"a": {"c": {"d": 'RANDOM', "e": 'str'}, "g": 1}, "z": 10} {"a": {"c": {"d": 'RANDOM2', "e": 'STR2'}, "g": 2}, "z": 20}""") // set an integer nested field with a string that can be casted into an integer checkUpdateJson( target = """ {"a": {"c": {"d": 'random', "e": 'str'}, "g": 1}, "z": 10} {"a": {"c": {"d": 'random2', "e": 'str2'}, "g": 2}, "z": 20}""", updateWhere = "z = 10", set = "a.g = '-1'" :: "z = '30'" :: Nil, expected = """ {"a": {"c": {"d": 'random', "e": 'str'}, "g": -1}, "z": 30} {"a": {"c": {"d": 'random2', "e": 'str2'}, "g": 2}, "z": 20}""") // set the nested data that has an Array field checkUpdateJson( target = """ {"a": {"c": {"d": 'random', "e": [1, 11]}, "g": 1}, "z": 10} {"a": {"c": {"d": 'RANDOM2', "e": [2, 22]}, "g": 2}, "z": 20}""", updateWhere = "z = 20", set = "a.c.d = 'RANDOM22'" :: "a.g = -2" :: Nil, expected = """ {"a": {"c": {"d": 'random', "e": [1, 11]}, "g": 1}, "z": 10} {"a": {"c": {"d": 'RANDOM22', "e": [2, 22]}, "g": -2}, "z": 20}""") // set an array field checkUpdateJson( target = """ {"a": {"c": {"d": 'random', "e": [1, 11]}, "g": 1}, "z": 10} {"a": {"c": {"d": 'RANDOM22', "e": [2, 22]}, "g": -2}, "z": 20}""", updateWhere = "z = 10", set = "a.c.e = array(-1, -11)" :: "a.g = -1" :: Nil, expected = """ {"a": {"c": {"d": 'random', "e": [-1, -11]}, "g": -1}, "z": 10} {"a": {"c": {"d": 'RANDOM22', "e": [2, 22]}, "g": -2}, "z": 20}""") // set an array field as a top-level attribute checkUpdateJson( target = """ {"a": [1, 11], "b": 'Z'} {"a": [2, 22], "b": 'Y'}""", updateWhere = "b = 'Z'", set = "a = array(-1, -11, -111)" :: Nil, expected = """ {"a": [-1, -11, -111], "b": 'Z'} {"a": [2, 22], "b": 'Y'}""") } test("nested data resolution order") { // By default, resolve by name. checkUpdateJson( target = """ {"a": {"c": {"d": 'RANDOM', "e": 'str'}, "g": 1}, "z": 10} {"a": {"c": {"d": 'random2', "e": 'str2'}, "g": 2}, "z": 20}""", updateWhere = "a.g = 2", set = "a = named_struct('g', 20, 'c', named_struct('e', 'str0', 'd', 'randomNew'))" :: Nil, expected = """ {"a": {"c": {"d": 'RANDOM', "e": 'str'}, "g": 1}, "z": 10} {"a": {"c": {"d": 'randomNew', "e": 'str0'}, "g": 20}, "z": 20}""") checkUpdateJson( target = """ {"a": {"c": {"d": 'RANDOM', "e": 'str'}, "g": 1}, "z": 10} {"a": {"c": {"d": 'random2', "e": 'str2'}, "g": 2}, "z": 20}""", updateWhere = "a.g = 2", set = "a.c = named_struct('e', 'str0', 'd', 'randomNew')" :: Nil, expected = """ {"a": {"c": {"d": 'RANDOM', "e": 'str'}, "g": 1}, "z": 10} {"a": {"c": {"d": 'randomNew', "e": 'str0'}, "g": 2}, "z": 20}""") // With the legacy conf, resolve by position. withSQLConf((DeltaSQLConf.DELTA_RESOLVE_MERGE_UPDATE_STRUCTS_BY_NAME.key, "false")) { checkUpdateJson( target = """ {"a": {"c": {"d": 'RANDOM', "e": 'str'}, "g": 1}, "z": 10} {"a": {"c": {"d": 'random2', "e": 'str2'}, "g": 2}, "z": 20}""", updateWhere = "a.g = 2", set = "a.c = named_struct('e', 'str0', 'd', 'randomNew')" :: Nil, expected = """ {"a": {"c": {"d": 'RANDOM', "e": 'str'}, "g": 1}, "z": 10} {"a": {"c": {"d": 'str0', "e": 'randomNew'}, "g": 2}, "z": 20}""") val e = intercept[AnalysisException] { checkUpdateJson( target = """ {"a": {"c": {"d": 'RANDOM', "e": 'str'}, "g": 1}, "z": 10} {"a": {"c": {"d": 'random2', "e": 'str2'}, "g": 2}, "z": 20}""", updateWhere = "a.g = 2", set = "a = named_struct('g', 20, 'c', named_struct('e', 'str0', 'd', 'randomNew'))" :: Nil, expected = """ {"a": {"c": {"d": 'RANDOM', "e": 'str'}, "g": 1}, "z": 10} {"a": {"c": {"d": 'randomNew', "e": 'str0'}, "g": 20}, "z": 20}""") } assert(e.getMessage.contains("cannot cast")) } } testQuietly("nested data - negative case") { val targetDF = spark.read.json(""" {"a": {"c": {"d": 'random', "e": 'str'}, "g": 1}, "z": 10} {"a": {"c": {"d": 'random2', "e": 'str2'}, "g": 2}, "z": 20}""" .split("\n").toSeq.toDS()) testAnalysisException( targetDF, set = "a.c = 'RANDOM2'" :: Nil, where = "z = 10", errMsgs = "data type mismatch" :: Nil) testAnalysisException( targetDF, set = "a.c.z = 'RANDOM2'" :: Nil, errMsgs = "No such struct field" :: Nil) testAnalysisException( targetDF, set = "a.c = named_struct('d', 'rand', 'e', 'str')" :: "a.c.d = 'RANDOM2'" :: Nil, errMsgs = "There is a conflict from these SET columns" :: Nil) testAnalysisException( targetDF, set = Seq("a = named_struct('c', named_struct('d', 'rand', 'e', 'str'), 'g', 3)", "a.c.d = 'RANDOM2'"), errMsgs = "There is a conflict from these SET columns" :: Nil) val schema = new StructType().add("a", MapType(StringType, IntegerType)) val mapData = spark.read.schema(schema).json(Seq("""{"a": {"b": 1}}""").toDS()) testAnalysisException( mapData, set = "a.b = -1" :: Nil, errMsgs = "Updating nested fields is only supported for StructType" :: Nil) // Updating an ArrayStruct is not supported val arrayStructData = spark.read.json(Seq("""{"a": [{"b": 1}, {"b": 2}]}""").toDS()) testAnalysisException( arrayStructData, set = "a.b = array(-1)" :: Nil, errMsgs = "Updating nested fields is only supported for StructType" :: Nil) } test("schema pruning on finding files to update") { append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value")) // Start from a cached snapshot state deltaLog.update().stateDF val executedPlans = DeltaTestUtils.withPhysicalPlansCaptured(spark) { checkUpdate(condition = Some("key = 2"), setClauses = "key = 1, value = 3", expectedResults = Row(1, 3) :: Row(1, 4) :: Row(1, 1) :: Row(0, 3) :: Nil) } val scans = executedPlans.flatMap(_.collect { case f: FileSourceScanExec => f }) // The first scan is for finding files to update. We only are matching against the key // so that should be the only field in the schema. assert(scans.head.schema == StructType( Seq( StructField("key", IntegerType) ) )) } test("nested schema pruning on finding files to update") { append(Seq((2, 2), (1, 4), (1, 1), (0, 3)).toDF("key", "value") .select(struct("key", "value").alias("nested"))) // Start from a cached snapshot state deltaLog.update().stateDF val executedPlans = DeltaTestUtils.withPhysicalPlansCaptured(spark) { checkUpdate(condition = Some("nested.key = 2"), setClauses = "nested.key = 1, nested.value = 3", expectedResults = Row(1, 3) :: Row(1, 4) :: Row(1, 1) :: Row(0, 3) :: Nil, prefix = "nested.") } val scans = executedPlans.flatMap(_.collect { case f: FileSourceScanExec => f }) assert(scans.head.schema == StructType.fromDDL("nested STRUCT")) } /** * @param function the unsupported function. * @param functionType The type of the unsupported expression to be tested. * @param data the data in the table. * @param set the set action containing the unsupported expression. * @param where the where clause containing the unsupported expression. * @param expectException whether an exception is expected to be thrown * @param customErrorRegex customized error regex. */ private def testUnsupportedExpression( function: String, functionType: String, data: => DataFrame, set: String, where: String, expectException: Boolean, customErrorRegex: Option[String] = None) { test(s"$functionType functions in update - expect exception: $expectException") { withTable("deltaTable") { data.write.format("delta").saveAsTable("deltaTable") val expectedErrorRegex = "(?s).*(?i)unsupported.*(?i).*Invalid expressions.*" def checkExpression( setOption: Option[String] = None, whereOption: Option[String] = None) { var catchException = if (functionType.equals("Generate") && setOption.nonEmpty) { expectException } else true var errorRegex = if (functionType.equals("Generate") && whereOption.nonEmpty) { ".*Subqueries are not supported in the UPDATE.*" } else customErrorRegex.getOrElse(expectedErrorRegex) if (catchException) { val dataBeforeException = spark.read.format("delta").table("deltaTable").collect() val e = intercept[Exception] { executeUpdate( "deltaTable", setOption.getOrElse("b = 4"), whereOption.getOrElse("a = 1")) } val message = if (e.getCause != null) { e.getCause.getMessage } else e.getMessage assert(message.matches(errorRegex)) checkAnswer(spark.read.format("delta").table("deltaTable"), dataBeforeException) } else { executeUpdate( "deltaTable", setOption.getOrElse("b = 4"), whereOption.getOrElse("a = 1")) } } // on set checkExpression(setOption = Option(set)) // on condition checkExpression(whereOption = Option(where)) } } } testUnsupportedExpression( function = "row_number", functionType = "Window", data = Seq((1, 2, 3)).toDF("a", "b", "c"), set = "b = row_number() over (order by c)", where = "row_number() over (order by c) > 1", expectException = true ) testUnsupportedExpression( function = "max", functionType = "Aggregate", data = Seq((1, 2, 3)).toDF("a", "b", "c"), set = "b = max(c)", where = "b > max(c)", expectException = true ) // Explode functions are supported in set and where if there's only one row generated. testUnsupportedExpression( function = "explode", functionType = "Generate", data = Seq((1, 2, List(3))).toDF("a", "b", "c"), set = "b = (select explode(c) from deltaTable)", where = "b = (select explode(c) from deltaTable)", expectException = false // only one row generated, no exception. ) // Explode functions are supported in set and where but if there's more than one row generated, // it will throw an exception. testUnsupportedExpression( function = "explode", functionType = "Generate", data = Seq((1, 2, List(3, 4))).toDF("a", "b", "c"), set = "b = (select explode(c) from deltaTable)", where = "b = (select explode(c) from deltaTable)", expectException = true, // more than one generated, expect exception. customErrorRegex = Some(".*ore than one row returned by a subquery used as an expression(?s).*") ) test("Variant type") { val df = sql( """SELECT parse_json(cast(id as string)) v, id i FROM range(2)""") append(df) executeUpdate(target = tableSQLIdentifier, where = "to_json(v) = '1'", set = "i = 10, v = parse_json('123')") checkAnswer(readDeltaTableByIdentifier().selectExpr("i", "to_json(v)"), Seq(Row(0, "0"), Row(10, "123"))) } test("update on partitioned table with special chars") { val partA = "part%one" val partB = "part%two" append(spark.range(0, 3, 1, 1).toDF("key").withColumn("value", lit(partA)), "value") checkUpdate( condition = Some(s"value = '$partA' AND key = 1"), setClauses = s"value = '$partB'", expectedResults = Row(0, partA) :: Row(1, partB) :: Row(2, partA) :: Nil ) checkUpdate( condition = Some(s"value = '$partA' AND key = 2"), setClauses = s"value = '$partB'", expectedResults = Row(0, partA) :: Row(1, partB) :: Row(2, partB) :: Nil ) checkUpdate( condition = Some(s"value = '$partA'"), setClauses = s"value = '$partB'", expectedResults = Row(0, partB) :: Row(1, partB) :: Row(2, partB) :: Nil ) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/VersionChecksumHistogramCompatSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.FileSizeHistogram import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import org.apache.spark.sql.QueryTest import org.apache.spark.sql.test.SharedSparkSession /** * Tests for backward-compatible deserialization of the VersionChecksum histogram field. * * Delta spec and Kernel (Java/Rust) write CRC files using "fileSizeHistogram" as the JSON field * name, while Delta-Spark historically used "histogramOpt". The `@JsonAlias` on * [[VersionChecksum.histogramOpt]] allows reading both field names so that CRC files written by * either Kernel or Delta-Spark are compatible. */ class VersionChecksumHistogramCompatSuite extends QueryTest with DeltaSQLCommandTest with SharedSparkSession { import testImplicits._ test("CRC with spec-compliant fileSizeHistogram field (Kernel format) is readable") { // Delta spec and Kernel (Java/Rust) use "fileSizeHistogram" as the JSON field name. // Delta-Spark historically used "histogramOpt". This test verifies that Delta-Spark // can read CRC files written by Kernel (i.e., JSON with "fileSizeHistogram" key). // Part 1: hardcoded JSON (unit-level deserialization check) val kernelWrittenJson = """{ | "txnId": "kernel-txn-id", | "tableSizeBytes": 2000, | "numFiles": 5, | "numDeletedRecordsOpt": null, | "numDeletionVectorsOpt": null, | "numMetadata": 1, | "numProtocol": 1, | "inCommitTimestampOpt": null, | "setTransactions": null, | "domainMetadata": null, | "metadata": {"id": "kernel-test-table-id", "format": {"provider": "parquet"}, | "partitionColumns": [], "configuration": {}}, | "protocol": {"minReaderVersion": 1, "minWriterVersion": 2}, | "fileSizeHistogram": { | "sortedBinBoundaries": [0, 1024, 10240, 102400, 1048576, 10485760], | "fileCounts": [2, 1, 0, 1, 1, 0], | "totalBytes": [1000, 5000, 0, 200000, 2000000, 0] | }, | "deletedRecordCountsHistogramOpt": null, | "allFiles": null |}""".stripMargin val parsedChecksum = JsonUtils.mapper.readValue[VersionChecksum](kernelWrittenJson) assert(parsedChecksum.histogramOpt.isDefined, "histogramOpt should be populated from the fileSizeHistogram JSON field") val parsedHistogram = parsedChecksum.histogramOpt.get assert(parsedHistogram.sortedBinBoundaries === IndexedSeq(0L, 1024L, 10240L, 102400L, 1048576L, 10485760L)) assert(parsedHistogram.fileCounts.toSeq === Seq(2L, 1L, 0L, 1L, 1L, 0L)) assert(parsedHistogram.totalBytes.toSeq === Seq(1000L, 5000L, 0L, 200000L, 2000000L, 0L)) // Part 2: integration check via real Delta table and DeltaLog withTempDir { dir => spark.range(10).write.format("delta").save(dir.getAbsolutePath) val log = DeltaLog.forTable(spark, dir.getAbsolutePath) val version = log.snapshot.version // Scenario A: persist the hardcoded test JSON as the CRC and read it back via DeltaLog. // Re-serialize as compact JSON (single line) because store.read() splits by newline and // readChecksum takes only the first line. log.store.write( FileNames.checksumFile(log.logPath, version), Iterator(JsonUtils.toJson(parsedChecksum)), overwrite = true) DeltaLog.clearCache() val snapshotA = DeltaLog.forTable(spark, dir.getAbsolutePath).snapshot assert(snapshotA.checksumOpt.isDefined) assert(snapshotA.checksumOpt.get.histogramOpt.isDefined, "Scenario A: histogramOpt should be populated from the fileSizeHistogram JSON field") assert(snapshotA.checksumOpt.get.histogramOpt.get === parsedHistogram) // Scenario B: read the real CRC produced by Delta-Spark, replace "histogramOpt" key // with "fileSizeHistogram" (simulating a CRC rewritten by Kernel), and read it back val realChecksum = log.readChecksum(version).get assert(realChecksum.histogramOpt.isDefined, "expected histogram in real CRC") val realHistogram = realChecksum.histogramOpt.get val kernelFormatJson = JsonUtils.toJson(realChecksum) .replace("\"histogramOpt\":", "\"fileSizeHistogram\":") log.store.write( FileNames.checksumFile(log.logPath, version), Iterator(kernelFormatJson), overwrite = true) DeltaLog.clearCache() val snapshotB = DeltaLog.forTable(spark, dir.getAbsolutePath).snapshot assert(snapshotB.checksumOpt.isDefined) assert(snapshotB.checksumOpt.get.histogramOpt.isDefined, "Scenario B: histogramOpt should be populated from the fileSizeHistogram JSON field") assert(snapshotB.checksumOpt.get.histogramOpt.get === realHistogram) } } test("CRC missing both histogramOpt and fileSizeHistogram fields deserializes without error") { // CRC files written before histogram support was added have neither field. // Readers must gracefully return None for histogramOpt. val noHistogramJson = """{ | "txnId": "old-txn-id", | "tableSizeBytes": 500, | "numFiles": 2, | "numDeletedRecordsOpt": null, | "numDeletionVectorsOpt": null, | "numMetadata": 1, | "numProtocol": 1, | "inCommitTimestampOpt": null, | "setTransactions": null, | "domainMetadata": null, | "metadata": null, | "protocol": null, | "deletedRecordCountsHistogramOpt": null, | "allFiles": null |}""".stripMargin val checksum = JsonUtils.mapper.readValue[VersionChecksum](noHistogramJson) assert(checksum.histogramOpt.isEmpty, "histogramOpt should be None when neither histogramOpt nor fileSizeHistogram is present") } test("CRC with both histogramOpt and fileSizeHistogram - last field in JSON takes priority") { // In practice a CRC will only contain one of these fields (Delta-Spark writes // histogramOpt, Kernel writes fileSizeHistogram). However, if both are present, // Jackson maps both to the same // VersionChecksum.histogramOpt field (via @JsonAlias) and processes them sequentially, // so the LAST occurrence in the JSON wins. This test documents that behavior. // Part 1: hardcoded JSON (unit-level deserialization check) // histogramOpt fileCounts = [10, 20, 30] - distinguishable "Delta-Spark value" // fileSizeHistogram fileCounts = [1, 2, 3] - used as a distinguishable "Kernel value" // Case 1: fileSizeHistogram appears last -> fileSizeHistogram value wins val fileSizeHistogramLast = """{ | "txnId": "txn-1", | "tableSizeBytes": 1000, | "numFiles": 3, | "numDeletedRecordsOpt": null, | "numDeletionVectorsOpt": null, | "numMetadata": 1, | "numProtocol": 1, | "inCommitTimestampOpt": null, | "setTransactions": null, | "domainMetadata": null, | "metadata": {"id": "kernel-test-table-id", "format": {"provider": "parquet"}, | "partitionColumns": [], "configuration": {}}, | "protocol": {"minReaderVersion": 1, "minWriterVersion": 2}, | "histogramOpt": { | "sortedBinBoundaries": [0, 1024, 10240], | "fileCounts": [10, 20, 30], | "totalBytes": [100, 200, 300] | }, | "fileSizeHistogram": { | "sortedBinBoundaries": [0, 1024, 10240], | "fileCounts": [1, 2, 3], | "totalBytes": [10, 20, 30] | }, | "deletedRecordCountsHistogramOpt": null, | "allFiles": null |}""".stripMargin val checksumCase1 = JsonUtils.mapper.readValue[VersionChecksum](fileSizeHistogramLast) assert(checksumCase1.histogramOpt.get.fileCounts.toSeq === Seq(1L, 2L, 3L), "fileSizeHistogram (last in JSON) should win over histogramOpt") // Case 2: histogramOpt appears last -> histogramOpt value wins val histogramOptLast = """{ | "txnId": "txn-2", | "tableSizeBytes": 1000, | "numFiles": 3, | "numDeletedRecordsOpt": null, | "numDeletionVectorsOpt": null, | "numMetadata": 1, | "numProtocol": 1, | "inCommitTimestampOpt": null, | "setTransactions": null, | "domainMetadata": null, | "metadata": {"id": "kernel-test-table-id", "format": {"provider": "parquet"}, | "partitionColumns": [], "configuration": {}}, | "protocol": {"minReaderVersion": 1, "minWriterVersion": 2}, | "fileSizeHistogram": { | "sortedBinBoundaries": [0, 1024, 10240], | "fileCounts": [1, 2, 3], | "totalBytes": [10, 20, 30] | }, | "histogramOpt": { | "sortedBinBoundaries": [0, 1024, 10240], | "fileCounts": [10, 20, 30], | "totalBytes": [100, 200, 300] | }, | "deletedRecordCountsHistogramOpt": null, | "allFiles": null |}""".stripMargin val checksumCase2 = JsonUtils.mapper.readValue[VersionChecksum](histogramOptLast) assert(checksumCase2.histogramOpt.get.fileCounts.toSeq === Seq(10L, 20L, 30L), "histogramOpt (last in JSON) should win over fileSizeHistogram") // Part 2: integration check via real Delta table and DeltaLog withTempDir { dir => spark.range(10).write.format("delta").save(dir.getAbsolutePath) val log = DeltaLog.forTable(spark, dir.getAbsolutePath) val version = log.snapshot.version // Scenario A: persist the hardcoded test JSON (both fields, fileSizeHistogram last) // and verify fileSizeHistogram wins when read back via DeltaLog. // Use readTree->writeValueAsString to compact to a single line (store.read() splits // by newline and readChecksum takes only .head), while preserving both JSON fields. val compactBothFields = JsonUtils.mapper.writeValueAsString( JsonUtils.mapper.readTree(fileSizeHistogramLast)) log.store.write( FileNames.checksumFile(log.logPath, version), Iterator(compactBothFields), overwrite = true) DeltaLog.clearCache() val snapshotA = DeltaLog.forTable(spark, dir.getAbsolutePath).snapshot assert(snapshotA.checksumOpt.isDefined) assert(snapshotA.checksumOpt.get.histogramOpt.get.fileCounts.toSeq === Seq(1L, 2L, 3L), "Scenario A: fileSizeHistogram (last in JSON) should win over histogramOpt") // Scenario B: read the real CRC produced by Delta-Spark, inject both fields by // appending "fileSizeHistogram" (with bumped fileCounts) after the existing // "histogramOpt", and verify fileSizeHistogram wins when read back via DeltaLog val realChecksum = log.readChecksum(version).get assert(realChecksum.histogramOpt.isDefined, "expected histogram in real CRC") val realHistogramJson = JsonUtils.toJson(realChecksum.histogramOpt.get) val altHistogram = realChecksum.histogramOpt.get.copy( fileCounts = realChecksum.histogramOpt.get.fileCounts.map(_ + 1)) val altHistogramJson = JsonUtils.toJson(altHistogram) // Insert fileSizeHistogram after histogramOpt so it appears last and wins val bothFieldsFSHLast = JsonUtils.toJson(realChecksum).replace( s""""histogramOpt":$realHistogramJson""", s""""histogramOpt":$realHistogramJson,"fileSizeHistogram":$altHistogramJson""") log.store.write( FileNames.checksumFile(log.logPath, version), Iterator(bothFieldsFSHLast), overwrite = true) DeltaLog.clearCache() val snapshotB = DeltaLog.forTable(spark, dir.getAbsolutePath).snapshot assert(snapshotB.checksumOpt.isDefined) assert(snapshotB.checksumOpt.get.histogramOpt.get === altHistogram, "Scenario B: fileSizeHistogram (last in JSON) should win over histogramOpt") } } test("writeChecksumFile writes correct field name based on conf") { withTempDir { dir => spark.range(10).write.format("delta").save(dir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) val testHistogram = FileSizeHistogram( sortedBinBoundaries = Vector(0L, 1024L, 10240L), fileCounts = Array(5L, 10L, 15L), totalBytes = Array(100L, 200L, 300L)) val checksum = deltaLog.snapshot.checksumOpt.get.copy(histogramOpt = Some(testHistogram)) val currentSpark = spark val currentLog = deltaLog val writer = new RecordChecksum { override val deltaLog: DeltaLog = currentLog override protected def spark: org.apache.spark.sql.SparkSession = currentSpark def writeChecksum(version: Long, cs: VersionChecksum): Unit = writeChecksumFile(version, cs) } // Write with flag OFF (default) -- should use histogramOpt val versionOff = deltaLog.snapshot.version + 1 withSQLConf(DeltaSQLConf.DELTA_CHECKSUM_HISTOGRAM_FIELD_FOLLOWS_PROTOCOL.key -> "false") { writer.writeChecksum(versionOff, checksum) } val crcJsonOff = deltaLog.store.read(FileNames.checksumFile(deltaLog.logPath, versionOff)).head assert(crcJsonOff.contains("\"histogramOpt\":"), "Flag OFF: CRC should contain histogramOpt") assert(!crcJsonOff.contains("\"fileSizeHistogram\":"), "Flag OFF: CRC should not contain fileSizeHistogram") // Write with flag ON -- should use fileSizeHistogram val versionOn = versionOff + 1 withSQLConf(DeltaSQLConf.DELTA_CHECKSUM_HISTOGRAM_FIELD_FOLLOWS_PROTOCOL.key -> "true") { writer.writeChecksum(versionOn, checksum) } val crcJsonOn = deltaLog.store.read(FileNames.checksumFile(deltaLog.logPath, versionOn)).head assert(crcJsonOn.contains("\"fileSizeHistogram\":"), "Flag ON: CRC should contain fileSizeHistogram") assert(!crcJsonOn.contains("\"histogramOpt\":"), "Flag ON: CRC should not contain histogramOpt") // Both CRCs should be readable and produce the same histogram val checksumOff = JsonUtils.mapper.readValue[VersionChecksum](crcJsonOff) val checksumOn = JsonUtils.mapper.readValue[VersionChecksum](crcJsonOn) assert(checksumOff.histogramOpt.get === testHistogram, "Flag OFF: read-back histogram should match the test histogram") assert(checksumOn.histogramOpt.get === testHistogram, "Flag ON: read-back histogram should match the test histogram") } } test("VersionChecksumProtocolCompliant fields match VersionChecksum") { // Use reflection to ensure the two classes stay in sync. If someone adds a field to // VersionChecksum but forgets VersionChecksumProtocolCompliant, this test will catch it. val checksumFields = classOf[VersionChecksum].getDeclaredFields .map(f => (f.getName, f.getType)).toSet val protocolCompliantFields = classOf[VersionChecksumProtocolCompliant].getDeclaredFields .map(f => (f.getName, f.getType)).toSet // The only difference should be histogramOpt vs fileSizeHistogram (same type) val expectedOnlyInChecksum = Set(("histogramOpt", classOf[Option[_]])) val expectedOnlyInProtocolCompliant = Set(("fileSizeHistogram", classOf[Option[_]])) val onlyInChecksum = checksumFields -- protocolCompliantFields val onlyInProtocolCompliant = protocolCompliantFields -- checksumFields assert(onlyInChecksum === expectedOnlyInChecksum, s"Unexpected fields only in VersionChecksum: $onlyInChecksum. " + "Did you add a new field to VersionChecksum without updating " + "VersionChecksumProtocolCompliant?") assert(onlyInProtocolCompliant === expectedOnlyInProtocolCompliant, s"Unexpected fields only in VersionChecksumProtocolCompliant: $onlyInProtocolCompliant. " + "Did you add a new field to VersionChecksumProtocolCompliant without updating " + "VersionChecksum?") } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/actions/AddFileSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.actions import org.apache.spark.sql.delta.{DeltaConfigs, DeltaLog, DeltaRuntimeException} import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.expressions.{Cast, Literal} import org.apache.spark.sql.errors.QueryErrorsBase import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ class AddFileSuite extends SparkFunSuite with SharedSparkSession with DeltaSQLCommandTest with QueryErrorsBase { private def withJvmTimeZone[T](tzId: String)(block: => T): T = { val originalTz = java.util.TimeZone.getDefault try { java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone(tzId)) block } finally { java.util.TimeZone.setDefault(originalTz) } } private def createAddFileWithPartitionValue(partitionValues: Map[String, String]): AddFile = { AddFile( path = "test.parquet", partitionValues = partitionValues, size = 100, modificationTime = 0, dataChange = true) } private def timestampLiteral(value: String, tz: String = "UTC"): Literal = { Literal.create( Cast(Literal(value), TimestampType, Some(tz), ansiEnabled = false).eval(), TimestampType) } private def dateLiteral(value: String): Literal = { Literal.create( Cast(Literal(value), DateType, None, ansiEnabled = false).eval(), DateType) } private def timestampNTZLiteral(value: String): Literal = { Literal.create( Cast(Literal(value), TimestampNTZType, None, ansiEnabled = false).eval(), TimestampNTZType) } test("normalizedPartitionValues for non-timestamp partitions returns typed literals") { withSQLConf(DeltaSQLConf.DELTA_NORMALIZE_PARTITION_VALUES_ON_READ.key -> "true") { withTempDir { tempDir => spark.createDataFrame( spark.sparkContext.emptyRDD[org.apache.spark.sql.Row], StructType(Seq( StructField("data", StringType), StructField("strCol", StringType), StructField("intCol", IntegerType) )) ).write.format("delta").partitionBy("strCol", "intCol").save(tempDir.getCanonicalPath) val deltaTxn = DeltaLog.forTable(spark, tempDir.getCanonicalPath).startTransaction() val file = createAddFileWithPartitionValue(Map("strCol" -> "value1", "intCol" -> "42")) val normalized = file.normalizedPartitionValues(spark, deltaTxn) assert(normalized("strCol") == Literal("value1")) assert(normalized("intCol") == Literal.create(42, IntegerType)) } } } test("normalizedPartitionValues for timestamp partitions with well formatted values") { withSQLConf(DeltaSQLConf.DELTA_NORMALIZE_PARTITION_VALUES_ON_READ.key -> "true") { withTempDir { tempDir => spark.createDataFrame( spark.sparkContext.emptyRDD[org.apache.spark.sql.Row], StructType(Seq( StructField("data", StringType), StructField("tsCol", TimestampType) )) ).write.format("delta").partitionBy("tsCol").save(tempDir.getCanonicalPath) val deltaTxn = DeltaLog.forTable(spark, tempDir.getCanonicalPath).startTransaction() val file = createAddFileWithPartitionValue(Map("tsCol" -> "2000-01-01T20:00:00.000000Z")) val normalized = file.normalizedPartitionValues(spark, deltaTxn) assert(normalized("tsCol") == timestampLiteral("2000-01-01T20:00:00.000000Z")) } } } for (enableNormalization <- BOOLEAN_DOMAIN) { test("normalizedPartitionValues for UTC timestamps partitions with different string formats, " + s"enableNormalization=$enableNormalization") { withJvmTimeZone("UTC") { withSQLConf( DeltaSQLConf.DELTA_NORMALIZE_PARTITION_VALUES_ON_READ.key -> enableNormalization.toString, "spark.sql.session.timeZone" -> "UTC") { withTempDir { tempDir => // Create empty Delta table with tsCol as partition column spark.createDataFrame( spark.sparkContext.emptyRDD[org.apache.spark.sql.Row], StructType(Seq( StructField("data", StringType), StructField("tsCol", TimestampType) )) ).write.format("delta").partitionBy("tsCol").save(tempDir.getCanonicalPath) val deltaTxn = DeltaLog.forTable(spark, tempDir.getCanonicalPath).startTransaction() val fileNonUtc = createAddFileWithPartitionValue(Map("tsCol" -> "2000-01-01 12:00:00")) val fileUtc = createAddFileWithPartitionValue(Map("tsCol" -> "2000-01-01T12:00:00.000000Z")) val normalizedNonUtc = fileNonUtc.normalizedPartitionValues(spark, deltaTxn) val normalizedUtc = fileUtc.normalizedPartitionValues(spark, deltaTxn) if (enableNormalization) { assert(normalizedNonUtc == normalizedUtc) } else { assert(normalizedNonUtc != normalizedUtc) } } } } } } for (enableNormalization <- BOOLEAN_DOMAIN) { test("normalizedPartitionValues for TimestampNTZ partitions returns the correct literal, " + s"enableNormalization=$enableNormalization") { // Per Delta protocol, TimestampNTZ values should be stored as // "{year}-{month}-{day} {hour}:{minute}:{second}" without any time zone conversion withSQLConf( DeltaSQLConf.DELTA_NORMALIZE_PARTITION_VALUES_ON_READ.key -> enableNormalization.toString ) { withTempDir { tempDir => spark.createDataFrame( spark.sparkContext.emptyRDD[org.apache.spark.sql.Row], StructType(Seq( StructField("data", StringType), StructField("tsCol", TimestampNTZType) )) ).write.format("delta").partitionBy("tsCol").save(tempDir.getCanonicalPath) val deltaTxn = DeltaLog.forTable(spark, tempDir.getCanonicalPath).startTransaction() val file = createAddFileWithPartitionValue(Map("tsCol" -> "2000-01-01 12:00:00")) val normalized = file.normalizedPartitionValues(spark, deltaTxn) if (enableNormalization) { assert(normalized("tsCol") == timestampNTZLiteral("2000-01-01 12:00:00")) } else { assert(normalized("tsCol") == Literal("2000-01-01 12:00:00")) } } } } } test("normalizedPartitionValues preserves null partition values") { withSQLConf(DeltaSQLConf.DELTA_NORMALIZE_PARTITION_VALUES_ON_READ.key -> "true") { withTempDir { tempDir => spark.createDataFrame( spark.sparkContext.emptyRDD[org.apache.spark.sql.Row], StructType(Seq( StructField("data", StringType), StructField("tsCol", TimestampType), StructField("strCol", StringType) )) ).write.format("delta").partitionBy("tsCol", "strCol").save(tempDir.getCanonicalPath) val deltaTxn = DeltaLog.forTable(spark, tempDir.getCanonicalPath).startTransaction() val file = createAddFileWithPartitionValue(Map("tsCol" -> null, "strCol" -> "value")) val normalized = file.normalizedPartitionValues(spark, deltaTxn) assert(normalized("tsCol") == timestampLiteral(null)) assert(normalized("strCol") == Literal("value")) } } } for (enableNormalization <- BOOLEAN_DOMAIN) { test("normalizedPartitionValues with mixed timestamp and non-timestamp partitions, " + s"enableNormalization=$enableNormalization") { withJvmTimeZone("UTC") { withSQLConf( DeltaSQLConf.DELTA_NORMALIZE_PARTITION_VALUES_ON_READ.key -> enableNormalization.toString, "spark.sql.session.timeZone" -> "UTC" ) { withTempDir { tempDir => spark.createDataFrame( spark.sparkContext.emptyRDD[org.apache.spark.sql.Row], StructType(Seq( StructField("data", StringType), StructField("tsCol", TimestampType), StructField("strCol", StringType), StructField("intCol", IntegerType) )) ).write.format("delta") .partitionBy("tsCol", "strCol", "intCol") .save(tempDir.getCanonicalPath) val deltaTxn = DeltaLog.forTable(spark, tempDir.getCanonicalPath).startTransaction() val file1 = createAddFileWithPartitionValue( Map("tsCol" -> "2000-01-01 12:00:00", "strCol" -> "value", "intCol" -> "42")) val file2 = createAddFileWithPartitionValue( Map("tsCol" -> "2000-01-01T12:00:00.000000Z", "strCol" -> "value", "intCol" -> "42")) val normalized1 = file1.normalizedPartitionValues(spark, deltaTxn) val normalized2 = file2.normalizedPartitionValues(spark, deltaTxn) if (enableNormalization) { // Timestamp columns should normalize to same value (same microseconds) assert(normalized1("tsCol") == normalized2("tsCol")) // Non-timestamp columns should be typed literals assert(normalized1("strCol") == Literal("value")) assert(normalized1("intCol") == Literal.create(42, IntegerType)) } else { // Without normalization the partition values are different string literals assert(normalized1 != normalized2) // Normalized partition values should be string literals of original values assert(normalized1("tsCol") == Literal("2000-01-01 12:00:00")) assert(normalized2("tsCol") == Literal("2000-01-01T12:00:00.000000Z")) } } } } } } for (enableNormalization <- BOOLEAN_DOMAIN) { test("normalizedPartitionValues for equal timestamps with different time zone offsets, " + s"enableNormalization=$enableNormalization") { withSQLConf( DeltaSQLConf.DELTA_NORMALIZE_PARTITION_VALUES_ON_READ.key -> enableNormalization.toString, "spark.sql.session.timeZone" -> "UTC" ) { withTempDir { tempDir => spark.createDataFrame( spark.sparkContext.emptyRDD[org.apache.spark.sql.Row], StructType(Seq( StructField("data", StringType), StructField("tsCol", TimestampType) )) ).write.format("delta").partitionBy("tsCol").save(tempDir.getCanonicalPath) val deltaTxn = DeltaLog.forTable(spark, tempDir.getCanonicalPath).startTransaction() // All three represent the same instant: 2000-01-01 12:00:00 UTC val fileUtc = createAddFileWithPartitionValue(Map("tsCol" -> "2000-01-01T12:00:00.000+0000")) // EST is UTC-5, so 07:00 EST = 12:00 UTC val fileEst = createAddFileWithPartitionValue(Map("tsCol" -> "2000-01-01T07:00:00.000-0500")) // PST is UTC-8, so 04:00 PST = 12:00 UTC val filePst = createAddFileWithPartitionValue(Map("tsCol" -> "2000-01-01T04:00:00.000-0800")) val normalizedUtc = fileUtc.normalizedPartitionValues(spark, deltaTxn) val normalizedEst = fileEst.normalizedPartitionValues(spark, deltaTxn) val normalizedPst = filePst.normalizedPartitionValues(spark, deltaTxn) if (enableNormalization) { // All should normalize to the same value since they represent the same moment assert(normalizedUtc == normalizedEst) assert(normalizedUtc == normalizedPst) assert(normalizedEst == normalizedPst) } else { // Without normalization, returns string literals of original values assert(normalizedUtc("tsCol") == Literal("2000-01-01T12:00:00.000+0000")) assert(normalizedEst("tsCol") == Literal("2000-01-01T07:00:00.000-0500")) assert(normalizedPst("tsCol") == Literal("2000-01-01T04:00:00.000-0800")) // The normalized values should be different since they're just string literals assert(normalizedUtc != normalizedEst) assert(normalizedUtc != normalizedPst) assert(normalizedEst != normalizedPst) } } } } } for (enableNormalization <- BOOLEAN_DOMAIN) { test("normalizedPartitionValues for same timestamp with different time zone notations, " + s"enableNormalization=$enableNormalization") { withSQLConf( DeltaSQLConf.DELTA_NORMALIZE_PARTITION_VALUES_ON_READ.key -> enableNormalization.toString, "spark.sql.session.timeZone" -> "UTC" ) { withTempDir { tempDir => spark.createDataFrame( spark.sparkContext.emptyRDD[org.apache.spark.sql.Row], StructType(Seq( StructField("data", StringType), StructField("tsCol", TimestampType) )) ).write.format("delta").partitionBy("tsCol").save(tempDir.getCanonicalPath) val deltaTxn = DeltaLog.forTable(spark, tempDir.getCanonicalPath).startTransaction() // All three represent the same instant: 2000-01-15 12:00:00 UTC // CET time zone abbreviation (UTC+1) val fileCet = createAddFileWithPartitionValue(Map("tsCol" -> "2000-01-15 13:00:00 CET")) // Europe/Berlin time zone name (UTC+1 in winter) val fileEuropeBerlin = createAddFileWithPartitionValue(Map("tsCol" -> "2000-01-15 13:00:00 Europe/Berlin")) // Numeric offset notation: +0100 val fileNumericOffset = createAddFileWithPartitionValue(Map("tsCol" -> "2000-01-15T13:00:00.000+0100")) val normalizedCet = fileCet.normalizedPartitionValues(spark, deltaTxn) val normalizedEuropeBerlin = fileEuropeBerlin.normalizedPartitionValues(spark, deltaTxn) val normalizedNumeric = fileNumericOffset.normalizedPartitionValues(spark, deltaTxn) if (enableNormalization) { // All should normalize to the same value since they represent the same moment assert(normalizedCet == normalizedEuropeBerlin, s"CET and Europe/Berlin should match: $normalizedCet vs $normalizedEuropeBerlin") assert(normalizedCet == normalizedNumeric, s"CET and numeric offset should match: $normalizedCet vs $normalizedNumeric") } else { // Without normalization, returns string literals of the original values assert(normalizedCet("tsCol") == Literal("2000-01-15 13:00:00 CET")) assert(normalizedEuropeBerlin("tsCol") == Literal("2000-01-15 13:00:00 Europe/Berlin")) assert(normalizedNumeric("tsCol") == Literal("2000-01-15T13:00:00.000+0100")) // The normalized values should be different since they're just string literals assert(normalizedCet != normalizedEuropeBerlin) assert(normalizedCet != normalizedNumeric) assert(normalizedNumeric != normalizedEuropeBerlin) } } } } } test("normalizedPartitionValues for DateType should return the original date string") { withSQLConf( DeltaSQLConf.DELTA_NORMALIZE_PARTITION_VALUES_ON_READ.key -> "true" ) { withTempDir { tempDir => spark.createDataFrame( spark.sparkContext.emptyRDD[org.apache.spark.sql.Row], StructType(Seq( StructField("data", StringType), StructField("dateCol", DateType) )) ).write.format("delta").partitionBy("dateCol").save(tempDir.getCanonicalPath) val deltaTxn = DeltaLog.forTable(spark, tempDir.getCanonicalPath).startTransaction() val originalDateString = "2000-01-01" val file = createAddFileWithPartitionValue(Map("dateCol" -> originalDateString)) val normalized = file.normalizedPartitionValues(spark, deltaTxn) val normalizedDateValue = normalized("dateCol") assert(normalizedDateValue == dateLiteral(originalDateString)) } } } test("normalizedPartitionValues should handle __HIVE_DEFAULT_PARTITION__") { withSQLConf( DeltaSQLConf.DELTA_NORMALIZE_PARTITION_VALUES_ON_READ.key -> "true" ) { withTempDir { tempDir => spark.createDataFrame( spark.sparkContext.emptyRDD[org.apache.spark.sql.Row], StructType(Seq( StructField("data", StringType), StructField("foo", IntegerType) )) ).write.format("delta").partitionBy("data").save(tempDir.getCanonicalPath) val deltaTxn = DeltaLog.forTable(spark, tempDir.getCanonicalPath).startTransaction() // Tombstone value __HIVE_DEFAULT_PARTITION__ should be preserved as a string for AddFiles val file = createAddFileWithPartitionValue(Map("data" -> "__HIVE_DEFAULT_PARTITION__")) val normalized = file.normalizedPartitionValues(spark, deltaTxn) assert(normalized("data") == Literal.create("__HIVE_DEFAULT_PARTITION__", StringType)) } } } test("normalizedPartitionValues preserves escaped characters in AddFile partition values") { withSQLConf( DeltaSQLConf.DELTA_NORMALIZE_PARTITION_VALUES_ON_READ.key -> "true" ) { withTempDir { tempDir => spark.createDataFrame( spark.sparkContext.emptyRDD[org.apache.spark.sql.Row], StructType(Seq( StructField("data", StringType), StructField("foo", IntegerType) )) ).write.format("delta").partitionBy("data").save(tempDir.getCanonicalPath) val deltaTxn = DeltaLog.forTable(spark, tempDir.getCanonicalPath).startTransaction() // Escaped characters like %aa should be preserved as-is in AddFile partition values // since they are not unescaped in AddFile partition values. val escapedValue = "test%aa%20value" val file = createAddFileWithPartitionValue(Map("data" -> escapedValue)) val normalized = file.normalizedPartitionValues(spark, deltaTxn) assert(normalized("data") == Literal.create(escapedValue, StringType)) } } } test("normalizedPartitionValues with a non UTC session time zone gets converted to UTC") { withJvmTimeZone("Europe/Berlin") { withSQLConf( DeltaSQLConf.DELTA_NORMALIZE_PARTITION_VALUES_ON_READ.key -> "true", "spark.sql.session.timeZone" -> "Europe/Berlin" // UTC + 1 in winter time ) { withTempDir { tempDir => spark.createDataFrame( spark.sparkContext.emptyRDD[org.apache.spark.sql.Row], StructType(Seq( StructField("data", StringType), StructField("tsCol", TimestampType) )) ).write.format("delta").partitionBy("tsCol").save(tempDir.getCanonicalPath) val deltaTxn = DeltaLog.forTable(spark, tempDir.getCanonicalPath).startTransaction() val file = createAddFileWithPartitionValue(Map("tsCol" -> "2000-01-01 12:00:00")) // The normalized timestamp should be 11:00 UTC // Parsed in Europe/Berlin (UTC+1), so 12:00 Berlin = 11:00 UTC val normalizedTimestamp = file.normalizedPartitionValues(spark, deltaTxn)("tsCol") assert(normalizedTimestamp == timestampLiteral("2000-01-01 12:00:00", "Europe/Berlin")) assert(normalizedTimestamp == timestampLiteral("2000-01-01 11:00:00", "UTC")) } } } } test("normalizedPartitionValues of timestamps strings with time zone offsets " + "and a non UTC session time zone gets converted to UTC.") { withSQLConf( DeltaSQLConf.DELTA_NORMALIZE_PARTITION_VALUES_ON_READ.key -> "true", "spark.sql.session.timeZone" -> "America/Los_Angeles" // UTC - 8 in winter time ) { withTempDir { tempDir => spark.createDataFrame( spark.sparkContext.emptyRDD[org.apache.spark.sql.Row], StructType(Seq( StructField("data", StringType), StructField("tsCol", TimestampType) )) ).write.format("delta").partitionBy("tsCol").save(tempDir.getCanonicalPath) val deltaTxn = DeltaLog.forTable(spark, tempDir.getCanonicalPath).startTransaction() // Timestamp at 17:30 with a +05:30 offset (India Standard Time) is 12:00 UTC val fileWithIstOffset = createAddFileWithPartitionValue(Map("tsCol" -> "2000-01-01T17:30:00.000+0530")) // Timestamp at 13:00 in a Europe/Berlin time zone (UTC+1 in winter) is 12:00 UTC val fileWithCetOffset = createAddFileWithPartitionValue(Map("tsCol" -> "2000-01-01 13:00:00 Europe/Berlin")) val normalizedIstTimestamp = fileWithIstOffset.normalizedPartitionValues(spark, deltaTxn)("tsCol") val normalizedCetTimestamp = fileWithCetOffset.normalizedPartitionValues(spark, deltaTxn)("tsCol") // Both should represent 2000-01-01 12:00:00 UTC val expectedTimestamp = timestampLiteral("2000-01-01 12:00:00", "UTC") assert(normalizedIstTimestamp == expectedTimestamp) assert(normalizedCetTimestamp == expectedTimestamp) } } } test("normalizedPartitionValues with missing leading zeroes in timestamp are accepted") { withJvmTimeZone("UTC") { withSQLConf( DeltaSQLConf.DELTA_NORMALIZE_PARTITION_VALUES_ON_READ.key -> "true", "spark.sql.session.timeZone" -> "UTC" ) { withTempDir { tempDir => spark.createDataFrame( spark.sparkContext.emptyRDD[org.apache.spark.sql.Row], StructType(Seq( StructField("data", StringType), StructField("tsCol", TimestampType) )) ).write.format("delta").partitionBy("tsCol").save(tempDir.getCanonicalPath) val deltaTxn = DeltaLog.forTable(spark, tempDir.getCanonicalPath).startTransaction() def getNormalizedTimestamp(tsValue: String): Literal = createAddFileWithPartitionValue(Map("tsCol" -> tsValue)) .normalizedPartitionValues(spark, deltaTxn)("tsCol") // Missing leading zero in hours: "1:00:00" vs "01:00:00" val hoursWithout = getNormalizedTimestamp("2000-01-01 1:00:00") val hoursWith = getNormalizedTimestamp("2000-01-01 01:00:00") val expectedHours = timestampLiteral("2000-01-01 01:00:00", "UTC") assert(hoursWithout == hoursWith) assert(hoursWith == expectedHours) // Missing leading zero in minutes: "01:2:00" vs "01:02:00" val minutesWithout = getNormalizedTimestamp("2000-01-01 01:2:00") val minutesWith = getNormalizedTimestamp("2000-01-01 01:02:00") val expectedMinutes = timestampLiteral("2000-01-01 01:02:00", "UTC") assert(minutesWithout == minutesWith) assert(minutesWith == expectedMinutes) // Missing leading zero in seconds: "01:02:3" vs "01:02:03" val secondsWithout = getNormalizedTimestamp("2000-01-01 01:02:3") val secondsWith = getNormalizedTimestamp("2000-01-01 01:02:03") val expectedSeconds = timestampLiteral("2000-01-01 01:02:03", "UTC") assert(secondsWithout == secondsWith) assert(secondsWith == expectedSeconds) // All missing leading zeroes: "1:2:3" vs "01:02:03" val allWithout = getNormalizedTimestamp("2000-01-01 1:2:3") val allWith = getNormalizedTimestamp("2000-01-01 01:02:03") val expectedAll = timestampLiteral("2000-01-01 01:02:03", "UTC") assert(allWithout == allWith) assert(allWith == expectedAll) } } } } test("normalizedPartitionValues with ISO 8601 format with T separator but no time zone") { withJvmTimeZone("Europe/Berlin") { withSQLConf( DeltaSQLConf.DELTA_NORMALIZE_PARTITION_VALUES_ON_READ.key -> "true", "spark.sql.session.timeZone" -> "Europe/Berlin" // UTC + 1 in winter time ) { withTempDir { tempDir => spark.createDataFrame( spark.sparkContext.emptyRDD[org.apache.spark.sql.Row], StructType(Seq( StructField("data", StringType), StructField("tsCol", TimestampType) )) ).write.format("delta").partitionBy("tsCol").save(tempDir.getCanonicalPath) val deltaTxn = DeltaLog.forTable(spark, tempDir.getCanonicalPath).startTransaction() // ISO 8601 format with 'T' separator but no time zone should use the JVM time zone. val file = createAddFileWithPartitionValue(Map("tsCol" -> "2000-01-01T12:00:00")) // The normalized timestamp should be 11:00 UTC (12:00 Berlin = 11:00 UTC) val normalized = file.normalizedPartitionValues(spark, deltaTxn) assert(normalized("tsCol") == timestampLiteral("2000-01-01 12:00:00", "Europe/Berlin")) } } } } test("normalizedPartitionValues should also use the JVM timezone on read") { withJvmTimeZone("America/Los_Angeles") { withSQLConf( DeltaSQLConf.DELTA_NORMALIZE_PARTITION_VALUES_ON_READ.key -> "true", "spark.sql.session.timeZone" -> "UTC" ) { withTempDir { tempDir => spark.createDataFrame( spark.sparkContext.emptyRDD[org.apache.spark.sql.Row], StructType(Seq( StructField("data", StringType), StructField("tsCol", TimestampType) )) ).write.format("delta").partitionBy("tsCol").save(tempDir.getCanonicalPath) val deltaTxn = DeltaLog.forTable(spark, tempDir.getCanonicalPath).startTransaction() // ON WRITE we use the JVM timezone, parsing this as an America/Los_Angeles timestamp. val file = createAddFileWithPartitionValue(Map("tsCol" -> "2000-01-01 12:00:00")) val normalized = file.normalizedPartitionValues(spark, deltaTxn) // ON READ we also need to use the JVM timezone again, reading it again as an // America/Los_Angeles timestamp. assert( normalized("tsCol") == timestampLiteral("2000-01-01 12:00:00", "America/Los_Angeles")) assert(normalized("tsCol") != timestampLiteral("2000-01-01 12:00:00", "UTC")) } } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptorSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.actions import java.util.UUID // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor._ import org.apache.hadoop.fs.Path import org.apache.spark.SparkFunSuite import org.apache.spark.paths.SparkPath // scalastyle:on import.ordering.noEmptyLine /** * Test: DV descriptor creation, created DV descriptor properties and utility methods are * working as expected. */ class DeletionVectorDescriptorSuite extends SparkFunSuite { test("Inline DV") { val dv = inlineInLog(testDVData, cardinality = 3) // Make sure the metadata (type, size etc.) in the DV is as expected assert(!dv.isOnDisk && dv.isInline, s"Incorrect DV storage type: $dv") assertCardinality(dv, 3) val encodedDVData = "0rJua" assert(dv.pathOrInlineDv === encodedDVData) assert(dv.sizeInBytes === testDVData.size) assert(dv.inlineData === testDVData) assert(dv.estimatedSerializedSize === 18) assert(dv.offset.isEmpty) // There shouldn't be an offset for inline DV // Unique id to identify the DV assert(dv.uniqueId === s"i$encodedDVData") assert(dv.uniqueFileId === s"i$encodedDVData") // There is no on-disk file name for an inline DV intercept[IllegalArgumentException] { dv.absolutePath(testTablePath) } // Copy as on-disk DV with absolute path and relative path - // expect the returned DV is same as input, since this is inline // so paths are irrelevant. assert(dv.copyWithAbsolutePath(testTablePath) === dv) assert(dv.copyWithNewRelativePath(UUID.randomUUID(), "predix2") === dv) } for (offset <- Seq(None, Some(25))) { test(s"On disk DV with absolute path with offset=$offset") { val dv = onDiskWithAbsolutePath(testDVAbsPath, sizeInBytes = 15, cardinality = 10, offset) // Make sure the metadata (type, size etc.) in the DV is as expected assert(dv.isOnDisk && !dv.isInline, s"Incorrect DV storage type: $dv") assertCardinality(dv, 10) assert(dv.pathOrInlineDv === testDVAbsPath) assert(dv.sizeInBytes === 15) intercept[Exception] { dv.inlineData } assert(dv.estimatedSerializedSize === (if (offset.isDefined) 4 else 0) + 37) assert(dv.offset === offset) // Unique id to identify the DV val offsetSuffix = offset.map(o => s"@$o").getOrElse("") assert(dv.uniqueId === s"p$testDVAbsPath$offsetSuffix") assert(dv.uniqueFileId === s"p$testDVAbsPath") // Given the input already has an absolute path, it should return the path in DV assert(dv.absolutePath(testTablePath) === new Path(testDVAbsPath)) // Given the input already has an absolute path, expect the output to be same as input assert(dv.copyWithAbsolutePath(testTablePath) === dv) // Copy DV as a relative path DV val uuid = UUID.randomUUID() val dvCopyWithRelativePath = dv.copyWithNewRelativePath(uuid, "prefix") assert(dvCopyWithRelativePath.isRelative) assert(dvCopyWithRelativePath.isOnDisk) assert(dvCopyWithRelativePath.pathOrInlineDv === encodeUUID(uuid, "prefix")) } } for (offset <- Seq(None, Some(25))) { test(s"On-disk DV with relative path with offset=$offset") { val uuid = UUID.randomUUID() val dv = onDiskWithRelativePath( uuid, randomPrefix = "prefix", sizeInBytes = 15, cardinality = 25, offset) // Make sure the metadata (type, size etc.) in the DV is as expected assert(dv.isOnDisk && !dv.isInline, s"Incorrect DV storage type: $dv") assertCardinality(dv, 25) assert(dv.pathOrInlineDv === encodeUUID(uuid, "prefix")) assert(dv.sizeInBytes === 15) intercept[Exception] { dv.inlineData } assert(dv.estimatedSerializedSize === (if (offset.isDefined) 4 else 0) + 39) assert(dv.offset === offset) // Unique id to identify the DV val offsetSuffix = offset.map(o => s"@$o").getOrElse("") val encodedUUID = encodeUUID(uuid, "prefix") assert(dv.uniqueId === s"u$encodedUUID$offsetSuffix") assert(dv.uniqueFileId === s"u$encodedUUID") // Expect the DV final path to be under the given table path assert(dv.absolutePath(testTablePath) === new Path(s"$testTablePath/prefix/${DELETION_VECTOR_FILE_NAME_CORE}_$uuid.bin")) // Copy DV with an absolute path location val dvCopyWithAbsPath = dv.copyWithAbsolutePath(testTablePath) assert(dvCopyWithAbsPath.isAbsolute) assert(dvCopyWithAbsPath.isOnDisk) // pathOrInlineDV is URL-encoded. assert( SparkPath.fromUrlString(dvCopyWithAbsPath.pathOrInlineDv).toPath.toString === s"$testTablePath/prefix/${DELETION_VECTOR_FILE_NAME_CORE}_$uuid.bin") // Copy DV as a relative path DV - expect to return the same DV as the current // DV already contains relative path. assert(dv.copyWithNewRelativePath(UUID.randomUUID(), "predix2") === dv) } } private def assertCardinality(dv: DeletionVectorDescriptor, expSize: Int): Unit = { if (expSize == 0) { assert(dv.isEmpty, s"Expected DV to be empty: $dv") } else { assert(!dv.isEmpty && dv.cardinality == expSize, s"Invalid size expected: $expSize, $dv") } } private val testTablePath = new Path("s3a://table/test") private val testDVAbsPath = "s3a://table/test/dv1.bin" private val testDVData: Array[Byte] = Array(1, 2, 3, 4) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/cdc/CDCReaderSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.cdc // scalastyle:off import.ordering.noEmptyLine import java.io.File import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.DeltaOperations.Delete import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.actions.{Action, AddCDCFile, AddFile} import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.commands.cdc.CDCReader._ import org.apache.spark.sql.delta.files.DelayedCommitProtocol import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.sql.{DataFrame, QueryTest} import org.apache.spark.sql.{Row, SaveMode} import org.apache.spark.sql.execution.{LogicalRDD, SQLExecution} import org.apache.spark.sql.execution.datasources.FileFormatWriter import org.apache.spark.sql.functions._ import org.apache.spark.sql.test.SharedSparkSession class CDCReaderSuite extends QueryTest with CheckCDCAnswer with SharedSparkSession with DeltaSQLCommandTest with DeltaColumnMappingTestUtils { override protected def sparkConf: SparkConf = super.sparkConf .set(DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey, "true") /** * Write a commit with just CDC data. Returns the committed version. */ private def writeCdcData( log: DeltaLog, data: DataFrame, extraActions: Seq[Action] = Seq.empty): Long = { log.withNewTransaction { txn => val qe = data.queryExecution val basePath = log.dataPath.toString // column mapped mode forces to use random file prefix val randomPrefixes = if (columnMappingEnabled) { Some(DeltaConfigs.RANDOM_PREFIX_LENGTH.fromMetaData(log.snapshot.metadata)) } else { None } // we need to convert to physical name in column mapping mode val mappedOutput = if (columnMappingEnabled) { val metadata = log.snapshot.metadata DeltaColumnMapping.createPhysicalAttributes( qe.analyzed.output, metadata.schema, metadata.columnMappingMode ) } else { qe.analyzed.output } SQLExecution.withNewExecutionId(qe) { var committer = new DelayedCommitProtocol("delta", basePath, randomPrefixes, None) FileFormatWriter.write( sparkSession = spark, plan = qe.executedPlan, fileFormat = log.fileFormat(log.snapshot.protocol, log.unsafeVolatileMetadata), committer = committer, outputSpec = FileFormatWriter.OutputSpec(basePath, Map.empty, mappedOutput), hadoopConf = log.newDeltaHadoopConf(), partitionColumns = Seq.empty, bucketSpec = None, statsTrackers = Seq.empty, options = Map.empty) val cdc = committer.addedStatuses.map { a => AddCDCFile(a.path, Map.empty, a.size) } txn.commit(extraActions ++ cdc, DeltaOperations.ManualUpdate) } } } def createCDFDF(start: Long, end: Long, commitVersion: Long, changeType: String): DataFrame = { spark.range(start, end) .withColumn(CDC_TYPE_COLUMN_NAME, lit(changeType)) .withColumn(CDC_COMMIT_VERSION, lit(commitVersion)) } test("simple CDC scan") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir.getAbsolutePath) val data = spark.range(10) val cdcData = spark.range(20, 25).withColumn(CDC_TYPE_COLUMN_NAME, lit("insert")) data.write.format("delta").save(dir.getAbsolutePath) sql(s"DELETE FROM delta.`${dir.getAbsolutePath}`") writeCdcData(log, cdcData) // For this basic test, we check each of the versions individually in addition to the full // range to try and catch weird corner cases. checkCDCAnswer( log, CDCReader.changesToBatchDF(log, 0, 0, spark), data.withColumn(CDC_TYPE_COLUMN_NAME, lit("insert")) .withColumn(CDC_COMMIT_VERSION, lit(0)) ) checkCDCAnswer( log, CDCReader.changesToBatchDF(log, 1, 1, spark), data.withColumn(CDC_TYPE_COLUMN_NAME, lit("delete")) .withColumn(CDC_COMMIT_VERSION, lit(1)) ) checkCDCAnswer( log, CDCReader.changesToBatchDF(log, 2, 2, spark), cdcData.withColumn(CDC_COMMIT_VERSION, lit(2)) ) checkCDCAnswer( log, CDCReader.changesToBatchDF(log, 0, 2, spark), data.withColumn(CDC_TYPE_COLUMN_NAME, lit("insert")) .withColumn(CDC_COMMIT_VERSION, lit(0)) .unionAll(data .withColumn(CDC_TYPE_COLUMN_NAME, lit("delete")) .withColumn(CDC_COMMIT_VERSION, lit(1))) .unionAll(cdcData.withColumn(CDC_COMMIT_VERSION, lit(2))) ) } } test("CDC has correct stats") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir.getAbsolutePath) val data = spark.range(10) val cdcData = spark.range(20, 25).withColumn(CDC_TYPE_COLUMN_NAME, lit("insert")) data.write.format("delta").save(dir.getAbsolutePath) sql(s"DELETE FROM delta.`${dir.getAbsolutePath}`") writeCdcData(log, cdcData) assert( CDCReader .changesToBatchDF(log, 0, 2, spark) .queryExecution .optimizedPlan .collectLeaves() .exists { case l: LogicalRDD => l.stats.sizeInBytes == 0 && !l.isStreaming case _ => false } ) } } test("cdc update ops") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir.getAbsolutePath) val data = spark.range(10) data.write.format("delta").save(dir.getAbsolutePath) writeCdcData( log, spark.range(20, 25).toDF().withColumn(CDC_TYPE_COLUMN_NAME, lit("update_pre"))) writeCdcData( log, spark.range(30, 35).toDF().withColumn(CDC_TYPE_COLUMN_NAME, lit("update_post"))) checkCDCAnswer( log, CDCReader.changesToBatchDF(log, 0, 2, spark), data.withColumn(CDC_TYPE_COLUMN_NAME, lit("insert")) .withColumn(CDC_COMMIT_VERSION, lit(0)) .unionAll(spark.range(20, 25).withColumn(CDC_TYPE_COLUMN_NAME, lit("update_pre")) .withColumn(CDC_COMMIT_VERSION, lit(1)) ) .unionAll(spark.range(30, 35).withColumn(CDC_TYPE_COLUMN_NAME, lit("update_post")) .withColumn(CDC_COMMIT_VERSION, lit(2)) ) ) } } test("dataChange = false operations ignored") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir.getAbsolutePath) val data = spark.range(10) data.write.format("delta").save(dir.getAbsolutePath) sql(s"OPTIMIZE delta.`${dir.getAbsolutePath}`") checkCDCAnswer( log, CDCReader.changesToBatchDF(log, 0, 1, spark), data.withColumn(CDC_TYPE_COLUMN_NAME, lit("insert")) .withColumn(CDC_COMMIT_VERSION, lit(0)) ) } } test("range with start and end equal") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir.getAbsolutePath) val data = spark.range(10) val cdcData = spark.range(0, 5).withColumn(CDC_TYPE_COLUMN_NAME, lit("delete")) .withColumn(CDC_COMMIT_VERSION, lit(1)) data.write.format("delta").save(dir.getAbsolutePath) writeCdcData(log, cdcData) checkCDCAnswer( log, CDCReader.changesToBatchDF(log, 0, 0, spark), data.withColumn(CDC_TYPE_COLUMN_NAME, lit("insert")) .withColumn(CDC_COMMIT_VERSION, lit(0)) ) checkCDCAnswer( log, CDCReader.changesToBatchDF(log, 1, 1, spark), cdcData) } } test("range past the end of the log") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir.getAbsolutePath) spark.range(10).write.format("delta").save(dir.getAbsolutePath) checkCDCAnswer( log, CDCReader.changesToBatchDF(log, 0, 1, spark), spark.range(10).withColumn(CDC_TYPE_COLUMN_NAME, lit("insert")) .withColumn(CDC_COMMIT_VERSION, lit(0)) ) } } test("invalid range - end before start") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir.getAbsolutePath) spark.range(10).write.format("delta").save(dir.getAbsolutePath) spark.range(20).write.format("delta").mode("append").save(dir.getAbsolutePath) intercept[IllegalArgumentException] { CDCReader.changesToBatchDF(log, 1, 0, spark) } } } testQuietly("invalid range - start after last version of CDF") { withTempDir { dir => spark.range(10).write.format("delta").save(dir.getAbsolutePath) spark.range(20).write.format("delta").mode("append").save(dir.getAbsolutePath) val e = intercept[DeltaIllegalArgumentException] { spark.read.format("delta") .option("readChangeFeed", "true") .option("startingVersion", Long.MaxValue) .option("endingVersion", Long.MaxValue) .load(dir.toString) .count() } checkError( e, condition = "DELTA_CDC_START_VERSION_AFTER_LATEST", sqlState = "22003", parameters = Map("start" -> Long.MaxValue.toString, "latest" -> "1") ) } } test("partition filtering of removes and cdc files") { withTempDir { dir => withSQLConf((DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey, "true")) { val path = dir.getAbsolutePath val log = DeltaLog.forTable(spark, path) spark.range(6).selectExpr("id", "'old' as text", "id % 2 as part") .write.format("delta").partitionBy("part").save(path) // Generate some CDC files. withTempView("source") { spark.range(4).createOrReplaceTempView("source") sql( s"""MERGE INTO delta.`$path` t USING source s ON s.id = t.id |WHEN MATCHED AND s.id = 1 THEN UPDATE SET text = 'new' |WHEN MATCHED AND s.id = 3 THEN DELETE""".stripMargin) } // This will generate just remove files due to the partition delete optimization. sql(s"DELETE FROM delta.`$path` WHERE part = 0") checkCDCAnswer( log, CDCReader.changesToBatchDF(log, 0, 2, spark).filter("_change_type = 'insert'"), Range(0, 6).map { i => Row(i, "old", i % 2, "insert", 0) }) checkCDCAnswer( log, CDCReader.changesToBatchDF(log, 0, 2, spark).filter("_change_type = 'delete'"), Seq(0, 2, 3, 4).map { i => Row(i, "old", i % 2, "delete", if (i % 2 == 0) 2 else 1) }) checkCDCAnswer( log, CDCReader.changesToBatchDF(log, 0, 2, spark).filter("_change_type = 'update_preimage'"), Row(1, "old", 1, "update_preimage", 1) :: Nil) checkCDCAnswer( log, CDCReader.changesToBatchDF(log, 0, 2, spark).filter("_change_type = 'update_postimage'"), Row(1, "new", 1, "update_postimage", 1) :: Nil) } } } test("file layout - unpartitioned") { withTempDir { dir => withSQLConf((DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey, "true")) { val path = dir.getAbsolutePath spark.range(10).repartition(1).write.format("delta").save(path) sql(s"DELETE FROM delta.`$path` WHERE id < 5") val log = DeltaLog.forTable(spark, path) // The data path should contain four files: the delta log, the CDC folder `__is_cdc=true`, // and two data files with randomized names from before and after the DELETE command. The // commit protocol should have stripped out __is_cdc=false. val baseDirFiles = log.logPath.getFileSystem(log.newDeltaHadoopConf()).listStatus(log.dataPath) assert(baseDirFiles.length == 4) assert(baseDirFiles.exists { f => f.isDirectory && f.getPath.getName == "_delta_log"}) assert(baseDirFiles.exists { f => f.isDirectory && f.getPath.getName == CDC_LOCATION}) assert(!baseDirFiles.exists { f => f.getPath.getName.contains(CDC_PARTITION_COL) }) } } } test("file layout - partitioned") { withTempDir { dir => withSQLConf((DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey, "true")) { val path = dir.getAbsolutePath spark.range(10).withColumn("part", col("id") % 2) .repartition(1).write.format("delta").partitionBy("part").save(path) sql(s"DELETE FROM delta.`$path` WHERE id < 5") val log = DeltaLog.forTable(spark, path) // The data path should contain four directories: the delta log, the CDC folder // `__is_cdc=true`, and the two partition folders. The commit protocol // should have stripped out __is_cdc=false. val fs = log.logPath.getFileSystem(log.newDeltaHadoopConf()) val baseDirFiles = fs.listStatus(log.dataPath) baseDirFiles.foreach { f => assert(f.isDirectory) } assert(baseDirFiles.map(_.getPath.getName).toSet == Set("_delta_log", CDC_LOCATION, "part=0", "part=1")) // Each partition folder should contain only two data files from before and after the read. // In particular, they should not contain any __is_cdc folder - that should always be the // top level partition. for (partitionFolder <- Seq("part=0", "part=1")) { val files = fs.listStatus(new Path(log.dataPath, partitionFolder)) assert(files.length === 2) files.foreach { f => assert(!f.isDirectory) assert(!f.getPath.getName.startsWith(CDC_LOCATION)) } } // The CDC folder should also contain the two partitions. val cdcPartitions = fs.listStatus(new Path(log.dataPath, CDC_LOCATION)) cdcPartitions.foreach { f => assert(f.isDirectory, s"$f was not a directory") } assert(cdcPartitions.map(_.getPath.getName).toSet == Set("part=0", "part=1")) } } } test("for CDC add backtick in column name with dot [.] ") { import testImplicits._ withTempDir { dir => withSQLConf((DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey, "true")) { val path = dir.getAbsolutePath // 0th commit Seq(2, 4).toDF("id.num") .withColumn("id.num`s", lit(10)) .withColumn("struct_col", struct(lit(1).as("field"), lit(2).as("field.one"))) .write.format("delta").save(path) // 1st commit Seq(1, 3, 5).toDF("id.num") .withColumn("id.num`s", lit(10)) .withColumn("struct_col", struct(lit(1).as("field"), lit(2).as("field.one"))) .write.format("delta").mode(SaveMode.Append).save(path) // Reading from 0th version val actual = spark.read.format("delta") .option("readChangeFeed", "true").option("startingVersion", 0) .load(path).drop(CDCReader.CDC_COMMIT_TIMESTAMP) val expected = spark.range(1, 6).toDF("id.num").withColumn("id.num`s", lit(10)) .withColumn("struct_col", struct(lit(1).as("field"), lit(2).as("field.one"))) .withColumn(CDCReader.CDC_TYPE_COLUMN_NAME, lit("insert")) .withColumn(CDCReader.CDC_COMMIT_VERSION, col("`id.num`") % 2) checkAnswer(actual, expected) } } } for (cdfEnabled <- BOOLEAN_DOMAIN) test(s"Coarse-grained CDF, cdfEnabled=$cdfEnabled") { withSQLConf(DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey -> cdfEnabled.toString) { withTempDir { dir => val log = DeltaLog.forTable(spark, dir.getAbsolutePath) // commit 0: 2 inserts spark.range(start = 0, end = 2, step = 1, numPartitions = 1) .write.format("delta").save(dir.getAbsolutePath) var df = CDCReader.changesToBatchDF( log, 0, 1, spark, catalogTableOpt = None, useCoarseGrainedCDC = true) checkAnswer(df.drop(CDC_COMMIT_TIMESTAMP), createCDFDF(start = 0, end = 2, commitVersion = 0, changeType = "insert")) // commit 1: 2 inserts spark.range(start = 2, end = 4) .write.mode("append").format("delta").save(dir.getAbsolutePath) df = CDCReader.changesToBatchDF( log, 1, 2, spark, catalogTableOpt = None, useCoarseGrainedCDC = true) checkAnswer(df.drop(CDC_COMMIT_TIMESTAMP), createCDFDF(start = 2, end = 4, commitVersion = 1, changeType = "insert")) // commit 2 sql(s"DELETE FROM delta.`$dir` WHERE id = 0") df = CDCReader.changesToBatchDF( log, 2, 3, spark, catalogTableOpt = None, useCoarseGrainedCDC = true) .drop(CDC_COMMIT_TIMESTAMP) // Using only Add and RemoveFiles should generate 2 deletes and 1 insert. Even when CDF // is enabled, we want to use only Add and RemoveFiles. val dfWithDeletesFirst = df.sort(CDC_TYPE_COLUMN_NAME) val expectedAnswer = createCDFDF(start = 0, end = 2, commitVersion = 2, changeType = "delete") .union( createCDFDF(start = 1, end = 2, commitVersion = 2, changeType = "insert")) checkAnswer(dfWithDeletesFirst, expectedAnswer) } } } test("Logs are generated for changesToDF") { withTempDir { dir => val events = Log4jUsageLogger.track { val log = DeltaLog.forTable(spark, dir.getAbsolutePath) val data = spark.range(10) data.write.format("delta").save(dir.getAbsolutePath) sql(s"DELETE FROM delta.`${dir.getAbsolutePath}`") CDCReader.changesToBatchDF(log, 0, 1, spark) } assert(events.exists(event => event.metric == "tahoeEvent" && event.tags.get("opType") == Option("delta.changeDataFeed.changesToDF"))) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/cdc/CDCWorkloadSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.cdc import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.QueryTest import org.apache.spark.sql.test.SharedSparkSession /** * Small end to end tests of workloads using CDC from Delta. */ class CDCWorkloadSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { test("replication workload") { withSQLConf((DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey, "true")) { withTempPaths(2) { paths => // Create an empty table at `path` we're going to replicate from, and a replication // destination at `replicatedPath`. The destination contains a subset of the final keys, // but with out-of-date enrichment data. val path = paths.head.getAbsolutePath val replicatedPath = paths(1).getAbsolutePath spark.range(0).selectExpr("id", "'none' as text").write.format("delta").save(path) spark.range(50) .selectExpr("id", "'oldEnrichment' as text") .filter("id % 4 = 0") .write.format("delta").save(replicatedPath) // Add data to the replication source in overlapping batches, so we produce both insert and // update events. for (i <- 0 to 8) { withTempView("source") { spark.range(i * 5, i * 5 + 10) .selectExpr("id", "'newEnrichment' as text") .createOrReplaceTempView("source") sql( s"""MERGE INTO delta.`$path` t USING source s ON s.id = t.id |WHEN MATCHED THEN UPDATE SET * |WHEN NOT MATCHED THEN INSERT *""".stripMargin) } } // Delete some data too. sql(s"DELETE FROM delta.`$path` WHERE id < 5") for (v <- 0 to 10) { withTempView("cdcSource") { val changes = spark.read.format("delta") .option("readChangeFeed", "true") .option("startingVersion", v) .option("endingVersion", v) .load(path) // Filter out the preimage so the update events only have the final row, as required by // our merge API. changes.filter("_change_type != 'update_preimage'").createOrReplaceTempView("cdcSource") sql( s"""MERGE INTO delta.`$replicatedPath` t USING cdcSource s ON s.id = t.id |WHEN MATCHED AND s._change_type = 'update_postimage' OR s._change_type = 'insert' | THEN UPDATE SET * |WHEN MATCHED AND s._change_type = 'delete' THEN DELETE |WHEN NOT MATCHED THEN INSERT *""".stripMargin) } } // We should have all the rows, all with the new enrichment data from the replication // source, except for 0 to 5 which were deleted. val expected = spark.range(5, 50).selectExpr("id", "'newEnrichment' as text") checkAnswer(spark.read.format("delta").load(replicatedPath), expected) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/cdc/DeleteCDCSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.cdc // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.commands.cdc.CDCReader._ import org.apache.spark.sql.delta.sources.DeltaSQLConf._ import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.Dataset import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.lit trait DeleteCDCMixin extends DeleteSQLMixin with CDCEnabled { protected def testCDCDelete( name: String)( initialData: => Dataset[_], partitionColumns: Seq[String] = Seq.empty, deleteCondition: String, expectedData: => Dataset[_], expectedChangeDataWithoutVersion: => Dataset[_] ): Unit = { test(s"CDC - $name") { withSQLConf( (DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey, "true")) { append(initialData.toDF(), partitionColumns) executeDelete(tableSQLIdentifier, deleteCondition) checkAnswer( readDeltaTableByIdentifier(), expectedData.toDF()) checkAnswer( getCDCForLatestOperation(deltaLog, operation = "DELETE"), expectedChangeDataWithoutVersion.toDF()) } } } } trait DeleteCDCTests extends DeleteCDCMixin with CDCTestMixin { import testImplicits._ testCDCDelete("unconditional")( initialData = spark.range(0, 10, step = 1, numPartitions = 3), deleteCondition = "", expectedData = spark.range(0), expectedChangeDataWithoutVersion = spark.range(10) .withColumn(CDC_TYPE_COLUMN_NAME, lit("delete")) ) testCDCDelete("conditional covering all rows")( initialData = spark.range(0, 10, step = 1, numPartitions = 3), deleteCondition = "id < 100", expectedData = spark.range(0), expectedChangeDataWithoutVersion = spark.range(10) .withColumn(CDC_TYPE_COLUMN_NAME, lit("delete")) ) testCDCDelete("two random rows")( initialData = spark.range(0, 10, step = 1, numPartitions = 3), deleteCondition = "id = 2 OR id = 8", expectedData = Seq(0, 1, 3, 4, 5, 6, 7, 9).toDF(), expectedChangeDataWithoutVersion = Seq(2, 8).toDF() .withColumn(CDC_TYPE_COLUMN_NAME, lit("delete")) ) testCDCDelete("delete unconditionally - partitioned table")( initialData = spark.range(0, 100, step = 1, numPartitions = 10) .selectExpr("id % 10 as part", "id"), partitionColumns = Seq("part"), deleteCondition = "", expectedData = Seq.empty[(Long, Long)].toDF("part", "id"), expectedChangeDataWithoutVersion = spark.range(100) .selectExpr("id % 10 as part", "id", "'delete' as _change_type") ) testCDCDelete("delete all rows by condition - partitioned table")( initialData = spark.range(0, 100, step = 1, numPartitions = 10) .selectExpr("id % 10 as part", "id"), partitionColumns = Seq("part"), deleteCondition = "id < 1000", expectedData = Seq.empty[(Long, Long)].toDF("part", "id"), expectedChangeDataWithoutVersion = spark.range(100) .selectExpr("id % 10 as part", "id", "'delete' as _change_type") ) testCDCDelete("partition-optimized delete")( initialData = spark.range(0, 100, step = 1, numPartitions = 10) .selectExpr("id % 10 as part", "id"), partitionColumns = Seq("part"), deleteCondition = "part = 3", expectedData = spark.range(100).selectExpr("id % 10 as part", "id").where("part != 3"), expectedChangeDataWithoutVersion = Range(0, 10).map(x => x * 10 + 3).toDF("id") .selectExpr("3 as part", "id", "'delete' as _change_type")) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/cdc/MergeCDCSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.cdc // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.SparkConf import org.apache.spark.sql.{DataFrame, QueryTest} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.lit import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{IntegerType, StructField, StructType} trait CDCEnabled extends SharedSparkSession { override protected def sparkConf: SparkConf = super.sparkConf .set(DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey, "true") } trait MergeCDCMixin extends SharedSparkSession with MergeIntoSQLTestUtils with DeltaColumnMappingTestUtils with DeltaSQLCommandTest with MergePersistentDVDisabled /** * Tests for MERGE INTO in CDC output mode. * */ trait MergeCDCTests extends QueryTest with CDCEnabled with MergeCDCMixin with CDCTestMixin { import testImplicits._ // scalastyle:off argcount /** * Utility method for simpler test writing when there's at most clause of each type. */ private def testMergeCdc(name: String)( target: => DataFrame, source: => DataFrame, deleteWhen: String = null, update: String = null, insert: String = null, expectedTableData: => DataFrame = null, expectedCdcDataWithoutVersion: => DataFrame = null, expectErrorContains: String = null, confs: Seq[(String, String)] = Seq()): Unit = { val updateClauses = Option(update).map(u => this.update(set = u)).toSeq val insertClauses = Option(insert).map(i => this.insert(values = i)).toSeq val deleteClauses = Option(deleteWhen).map(d => this.delete(condition = d)).toSeq testMergeCdcUnlimitedClauses(name)( target = target, source = source, clauses = deleteClauses ++ updateClauses ++ insertClauses, expectedTableData = expectedTableData, expectedCdcDataWithoutVersion = expectedCdcDataWithoutVersion, expectErrorContains = expectErrorContains, confs = confs) } // scalastyle:on argcount private def testMergeCdcUnlimitedClauses(name: String)( target: => DataFrame, source: => DataFrame, mergeCondition: String = "s.key = t.key", clauses: Seq[MergeClause], expectedTableData: => DataFrame = null, expectedCdcDataWithoutVersion: => DataFrame = null, expectErrorContains: String = null, confs: Seq[(String, String)] = Seq(), targetTableSchema: Option[StructType] = None): Unit = { test(s"merge CDC - $name") { withSQLConf(confs: _*) { targetTableSchema.foreach { schema => io.delta.tables.DeltaTable.create(spark) .tableName(tableSQLIdentifier) .location(deltaLog.dataPath.toUri.getPath) .addColumns(schema) .execute() } append(target) withTempView("source") { source.createOrReplaceTempView("source") if (expectErrorContains != null) { val ex = intercept[Exception] { executeMerge(s"$tableSQLIdentifier t", "source s", mergeCondition, clauses.toSeq: _*) } assert(ex.getMessage.contains(expectErrorContains)) } else { executeMerge(s"$tableSQLIdentifier t", "source s", mergeCondition, clauses.toSeq: _*) checkAnswer( readDeltaTableByIdentifier(), expectedTableData) // Craft expected CDC data val latestVersion = deltaLog.snapshot.version val expectedCdcData = expectedCdcDataWithoutVersion .withColumn(CDCReader.CDC_COMMIT_VERSION, lit(latestVersion)) // The timestamp is nondeterministic so we drop it when comparing results. checkAnswer( computeCDC(spark, deltaLog, latestVersion, latestVersion) .drop(CDCReader.CDC_COMMIT_TIMESTAMP), expectedCdcData) } } } } } testMergeCdc("insert only")( target = ((0, 0) :: (1, 10) :: (3, 30) :: Nil).toDF("key", "n"), source = ((1, 1) :: (2, 2) :: Nil).toDF("key", "n"), insert = "*", expectedTableData = ((0, 0) :: (1, 10) :: (2, 2) :: (3, 30) :: Nil).toDF(), expectedCdcDataWithoutVersion = ((2, 2, "insert") :: Nil).toDF() ) testMergeCdc("update only")( target = ((0, 0) :: (1, 10) :: (3, 30) :: Nil).toDF("key", "n"), source = ((1, 1) :: (2, 2) :: Nil).toDF("key", "n"), update = "*", expectedTableData = ((0, 0) :: (1, 1) :: (3, 30) :: Nil).toDF(), expectedCdcDataWithoutVersion = ( (1, 10, "update_preimage") :: (1, 1, "update_postimage") :: Nil).toDF() ) testMergeCdc("delete only")( target = ((0, 0) :: (1, 10) :: (3, 30) :: Nil).toDF("key", "n"), source = ((1, 1) :: (2, 2) :: Nil).toDF("key", "n"), deleteWhen = "true", expectedTableData = ((0, 0) :: (3, 30) :: Nil).toDF(), expectedCdcDataWithoutVersion = ((1, 10, "delete") :: Nil).toDF() ) testMergeCdc("delete only with duplicate matches")( target = ((0, 0) :: (1, 10) :: (3, 30) :: Nil).toDF("key", "n"), source = ((1, 1) :: (1, 2) :: (2, 3) :: Nil).toDF("key", "n"), deleteWhen = "true", expectErrorContains = "attempted to modify the same\ntarget row" ) testMergeCdc("update + delete + insert together")( target = ((0, 0) :: (1, 10) :: (3, 30) :: Nil).toDF("key", "n"), source = ((1, 1) :: (2, 2) :: (3, -1) :: Nil).toDF("key", "n"), insert = "*", update = "*", deleteWhen = "s.key = 3", expectedTableData = ((0, 0) :: (1, 1) :: (2, 2) :: Nil).toDF(), expectedCdcDataWithoutVersion = ( (2, 2, "insert") :: (1, 10, "update_preimage") :: (1, 1, "update_postimage") :: (3, 30, "delete") :: Nil).toDF() ) testMergeCdcUnlimitedClauses("unlimited clauses - conditional final branch")( target = ((0, 0) :: (1, 10) :: (3, 30) :: (4, 40) :: (6, 60) :: Nil).toDF("key", "n"), source = ((1, 1) :: (2, 2) :: (3, -1) :: (4, 4) :: (5, 0) :: (6, 0) :: Nil).toDF("key", "n"), clauses = update("*", "s.key = 1") :: update("n = 400", "s.key = 4") :: delete("s.key = 3") :: delete("s.key = 6") :: insert("*", "s.key = 2") :: insert("(key, n) VALUES (50, 50)", "s.key = 5") :: Nil, expectedTableData = ((0, 0) :: (1, 1) :: (2, 2) :: (4, 400) :: (50, 50) :: Nil).toDF(), expectedCdcDataWithoutVersion = ( (2, 2, "insert") :: (50, 50, "insert") :: (1, 10, "update_preimage") :: (1, 1, "update_postimage") :: (4, 40, "update_preimage") :: (4, 400, "update_postimage") :: (3, 30, "delete") :: (6, 60, "delete") :: Nil).toDF() ) testMergeCdcUnlimitedClauses("unlimited clauses - unconditional final branch")( target = ((0, 0) :: (1, 10) :: (3, 30) :: (4, 40) :: (6, 60) :: Nil).toDF("key", "n"), source = ((1, 1) :: (2, 2) :: (3, -1) :: (4, 4) :: (5, 0) :: (6, 0) :: Nil).toDF("key", "n"), clauses = update("*", "s.key = 1") :: update("n = 400", "s.key = 4") :: delete("s.key = 3") :: delete(condition = null) :: insert("*", "s.key = 2") :: insert("(key, n) VALUES (50, 50)", condition = null) :: Nil, expectedTableData = ((0, 0) :: (1, 1) :: (2, 2) :: (4, 400) :: (50, 50) :: Nil).toDF(), expectedCdcDataWithoutVersion = ( (2, 2, "insert") :: (50, 50, "insert") :: (1, 10, "update_preimage") :: (1, 1, "update_postimage") :: (4, 40, "update_preimage") :: (4, 400, "update_postimage") :: (3, 30, "delete") :: (6, 60, "delete") :: Nil).toDF() ) testMergeCdc("basic schema evolution")( target = ((0, 0) :: (1, 10) :: (3, 30) :: Nil).toDF("key", "n"), source = ((1, 1, "a") :: (2, 2, "b") :: (3, -1, "c") :: Nil).toDF("key", "n", "text"), insert = "*", update = "*", deleteWhen = "s.key = 3", expectedTableData = ((0, 0, null) :: (1, 1, "a") :: (2, 2, "b") :: Nil) .asInstanceOf[Seq[(Int, Int, String)]].toDF(), expectedCdcDataWithoutVersion = ( (1, 10, null, "update_preimage") :: (1, 1, "a", "update_postimage") :: (2, 2, "b", "insert") :: (3, 30, null, "delete") :: Nil) .asInstanceOf[List[(Integer, Integer, String, String)]] .toDF("key", "targetVal", "srcVal", "_change_type"), confs = (DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key, "true") :: Nil ) testMergeCdcUnlimitedClauses("schema evolution with non-nullable schema")( target = ((0, 0) :: (1, 10) :: (3, 30) :: Nil).toDF("key", "n"), source = ((1, 1, "a") :: (2, 2, "b") :: (3, -1, "c") :: Nil).toDF("key", "n", "text"), mergeCondition = "t.key = s.key", clauses = delete(condition = "s.key = 3") :: update("*") :: insert("*") :: Nil, expectedTableData = ((0, 0, null) :: (1, 1, "a") :: (2, 2, "b") :: Nil) .asInstanceOf[Seq[(Int, Int, String)]].toDF(), expectedCdcDataWithoutVersion = ( (1, 10, null, "update_preimage") :: (1, 1, "a", "update_postimage") :: (2, 2, "b", "insert") :: (3, 30, null, "delete") :: Nil) .asInstanceOf[List[(Integer, Integer, String, String)]] .toDF("key", "targetVal", "srcVal", "_change_type"), confs = (DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key, "true") :: Nil, targetTableSchema = Some(StructType(Seq( StructField("key", IntegerType, nullable = false), StructField("n", IntegerType, nullable = false)))) ) testMergeCdcUnlimitedClauses("schema evolution with non-nullable schema - matched only")( target = ((0, 0) :: (1, 10) :: (3, 30) :: Nil).toDF("key", "n"), source = ((1, 1, "a") :: (2, 2, "b") :: (3, -1, "c") :: Nil).toDF("key", "n", "text"), mergeCondition = "t.key = s.key", clauses = delete(condition = "s.key = 3") :: update("*") :: Nil, expectedTableData = ((0, 0, null) :: (1, 1, "a") :: Nil) .asInstanceOf[Seq[(Int, Int, String)]].toDF(), expectedCdcDataWithoutVersion = ( (1, 10, null, "update_preimage") :: (1, 1, "a", "update_postimage") :: (3, 30, null, "delete") :: Nil) .asInstanceOf[List[(Integer, Integer, String, String)]] .toDF("key", "targetVal", "srcVal", "_change_type"), confs = (DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key, "true") :: Nil, targetTableSchema = Some(StructType(Seq( StructField("key", IntegerType, nullable = false), StructField("n", IntegerType, nullable = false)))) ) testMergeCdcUnlimitedClauses("unconditional delete only with duplicate matches")( target = Seq(0, 1).toDF("value"), source = Seq(1, 1).toDF("value"), mergeCondition = "t.value = s.value", clauses = delete() :: Nil, expectedTableData = Seq(0).toDF(), expectedCdcDataWithoutVersion = ((1, "delete") :: Nil).toDF() ) testMergeCdcUnlimitedClauses( "unconditional delete only with duplicate matches without duplicates rows in the source")( target = Seq(0).toDF("value"), source = ((0, 0) :: (0, 1) :: Nil).toDF("col1", "col2"), mergeCondition = "t.value = s.col1", clauses = delete() :: Nil, expectedTableData = Nil.asInstanceOf[List[Integer]] .toDF("value"), expectedCdcDataWithoutVersion = ((0, "delete") :: Nil).toDF() ) testMergeCdcUnlimitedClauses( "unconditional delete only with duplicate matches with duplicates in the target")( target = Seq(0, 1, 1).toDF("value"), source = Seq(1, 1).toDF("value"), mergeCondition = "t.value = s.value", clauses = delete() :: Nil, expectedTableData = Seq(0).toDF(), expectedCdcDataWithoutVersion = ((1, "delete") :: (1, "delete") :: Nil).toDF() ) testMergeCdcUnlimitedClauses("unconditional delete only with target-only merge condition")( target = Seq(0, 1).toDF("value"), source = Seq(0, 1).toDF("value"), mergeCondition = "t.value > 0", clauses = delete() :: Nil, expectedTableData = Seq(0).toDF(), expectedCdcDataWithoutVersion = ((1, "delete") :: Nil).toDF() ) testMergeCdcUnlimitedClauses( "unconditional delete only with target-only merge condition with duplicates in the target")( target = Seq(0, 1, 1).toDF("value"), source = Seq(0, 1).toDF("value"), mergeCondition = "t.value > 0", clauses = delete() :: Nil, expectedTableData = Seq(0).toDF(), expectedCdcDataWithoutVersion = ((1, "delete") :: (1, "delete") :: Nil).toDF() ) testMergeCdcUnlimitedClauses("unconditional delete only with source-only merge condition")( target = Seq(0, 1).toDF("value"), source = Seq(0, 1).toDF("value"), mergeCondition = "s.value < 2", clauses = delete() :: Nil, expectedTableData = Nil.asInstanceOf[List[Integer]] .toDF("value"), expectedCdcDataWithoutVersion = ((0, "delete") :: (1, "delete") :: Nil).toDF() ) testMergeCdcUnlimitedClauses( "unconditional delete only with source-only merge condition with duplicates in the target")( target = Seq(0, 1, 1).toDF("value"), source = Seq(0, 1).toDF("value"), mergeCondition = "s.value < 2", clauses = delete() :: Nil, expectedTableData = Nil.asInstanceOf[List[Integer]] .toDF("value"), expectedCdcDataWithoutVersion = ((0, "delete") :: (1, "delete") :: (1, "delete") :: Nil).toDF() ) testMergeCdcUnlimitedClauses("unconditional delete with duplicate matches + insert")( target = ((1, 1) :: (2, 2) :: Nil).toDF("key", "value"), source = ((1, 10) :: (1, 100) :: (3, 30) :: (3, 300) :: Nil).toDF("key", "value"), mergeCondition = "s.key = t.key", clauses = delete() :: insert(values = "(key, value) VALUES (s.key, s.value)") :: Nil, expectedTableData = ((2, 2) :: (3, 30) :: (3, 300) :: Nil).toDF("key", "value"), expectedCdcDataWithoutVersion = ((1, 1, "delete") :: (3, 30, "insert") :: (3, 300, "insert") :: Nil).toDF() ) testMergeCdcUnlimitedClauses( "unconditional delete with duplicate matches + insert with duplicate rows")( target = ((1, 1) :: (2, 2) :: Nil).toDF("key", "value"), source = ((1, 10) :: (1, 100) :: (3, 30) :: (3, 300) :: (3, 300) :: Nil).toDF("key", "value"), mergeCondition = "s.key = t.key", clauses = delete() :: insert(values = "(key, value) VALUES (s.key, s.value)") :: Nil, expectedTableData = ((2, 2) :: (3, 30) :: (3, 300) :: (3, 300) :: Nil).toDF("key", "value"), expectedCdcDataWithoutVersion = ((1, 1, "delete") :: (3, 30, "insert") :: (3, 300, "insert") :: (3, 300, "insert") :: Nil).toDF() ) testMergeCdcUnlimitedClauses("unconditional delete with duplicate matches " + "+ insert a duplicate of the unmatched target rows")( target = Seq(1, 2).toDF("value"), source = ((1, 10) :: (1, 100) :: (3, 2) :: Nil).toDF("col1", "col2"), mergeCondition = "s.col1 = t.value", clauses = delete() :: insert(values = "(value) VALUES (col2)") :: Nil, expectedTableData = Seq(2, 2).toDF(), expectedCdcDataWithoutVersion = ((1, "delete") :: (2, "insert") :: Nil).toDF() ) testMergeCdcUnlimitedClauses("all conditions failed for all rows")( target = Seq((1, "a"), (2, "b")).toDF("key", "val"), source = Seq((1, "t"), (2, "u")).toDF("key", "val"), clauses = update("t.val = s.val", "s.key = 10") :: insert("*", "s.key = 11") :: Nil, expectedTableData = Seq((1, "a"), (2, "b")).asInstanceOf[List[(Integer, String)]].toDF("key", "targetVal"), expectedCdcDataWithoutVersion = Nil.asInstanceOf[List[(Integer, String, String)]] .toDF("key", "targetVal", "_change_type") ) testMergeCdcUnlimitedClauses("unlimited clauses schema evolution")( // 1 and 2 should be updated from the source, 3 and 4 should be deleted. Only 5 is unchanged target = Seq((1, "a"), (2, "b"), (3, "c"), (4, "d"), (5, "e")).toDF("key", "targetVal"), // 1 and 2 should be updated into the target, 6 and 7 should be inserted. 8 should be ignored source = Seq((1, "t"), (2, "u"), (3, "v"), (4, "w"), (6, "x"), (7, "y"), (8, "z")) .toDF("key", "srcVal"), clauses = update("targetVal = srcVal", "s.key = 1") :: update("*", "s.key = 2") :: delete("s.key = 3") :: delete("s.key = 4") :: insert("(key) VALUES (s.key)", "s.key = 6") :: insert("*", "s.key = 7") :: Nil, expectedTableData = ((1, "t", null) :: (2, "b", "u") :: (5, "e", null) :: (6, null, null) :: (7, null, "y") :: Nil) .asInstanceOf[List[(Integer, String, String)]].toDF("key", "targetVal", "srcVal"), expectedCdcDataWithoutVersion = ( (1, "a", null, "update_preimage") :: (1, "t", null, "update_postimage") :: (2, "b", null, "update_preimage") :: (2, "b", "u", "update_postimage") :: (3, "c", null, "delete") :: (4, "d", null, "delete") :: (6, null, null, "insert") :: (7, null, "y", "insert") :: Nil) .asInstanceOf[List[(Integer, String, String, String)]] .toDF("key", "targetVal", "srcVal", "_change_type"), confs = (DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key, "true") :: Nil ) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/cdc/UpdateCDCSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.cdc // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.{AddCDCFile, AddFile, RemoveFile} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.TableIdentifier trait UpdateCDCTests extends UpdateSQLMixin with DeltaColumnMappingTestUtils with DeltaDMLTestUtilsPathBased with CDCTestMixin { import testImplicits._ test("CDC for unconditional update") { append(Seq((1, 1), (2, 2), (3, 3), (4, 4)).toDF("key", "value")) checkUpdate( condition = None, setClauses = "value = -1", expectedResults = Row(1, -1) :: Row(2, -1) :: Row(3, -1) :: Row(4, -1) :: Nil) val latestVersion = deltaLog.update().version checkAnswer( computeCDC(spark, deltaLog, latestVersion, latestVersion) .drop(CDCReader.CDC_COMMIT_TIMESTAMP), Row(1, 1, "update_preimage", latestVersion) :: Row(1, -1, "update_postimage", latestVersion) :: Row(2, 2, "update_preimage", latestVersion) :: Row(2, -1, "update_postimage", latestVersion) :: Row(3, 3, "update_preimage", latestVersion) :: Row(3, -1, "update_postimage", latestVersion) :: Row(4, 4, "update_preimage", latestVersion) :: Row(4, -1, "update_postimage", latestVersion) :: Nil) } test("CDC for conditional update on all rows") { append(Seq((1, 1), (2, 2), (3, 3), (4, 4)).toDF("key", "value")) checkUpdate( condition = Some("key < 10"), setClauses = "value = -1", expectedResults = Row(1, -1) :: Row(2, -1) :: Row(3, -1) :: Row(4, -1) :: Nil) val latestVersion = deltaLog.update().version checkAnswer( computeCDC(spark, deltaLog, latestVersion, latestVersion) .drop(CDCReader.CDC_COMMIT_TIMESTAMP), Row(1, 1, "update_preimage", latestVersion) :: Row(1, -1, "update_postimage", latestVersion) :: Row(2, 2, "update_preimage", latestVersion) :: Row(2, -1, "update_postimage", latestVersion) :: Row(3, 3, "update_preimage", latestVersion) :: Row(3, -1, "update_postimage", latestVersion) :: Row(4, 4, "update_preimage", latestVersion) :: Row(4, -1, "update_postimage", latestVersion) :: Nil) } test("CDC for point update") { append(Seq((1, 1), (2, 2), (3, 3), (4, 4)).toDF("key", "value")) checkUpdate( condition = Some("key = 1"), setClauses = "value = -1", expectedResults = Row(1, -1) :: Row(2, 2) :: Row(3, 3) :: Row(4, 4) :: Nil) val latestVersion = deltaLog.update().version checkAnswer( computeCDC(spark, deltaLog, latestVersion, latestVersion) .drop(CDCReader.CDC_COMMIT_TIMESTAMP), Row(1, 1, "update_preimage", latestVersion) :: Row(1, -1, "update_postimage", latestVersion) :: Nil) } test("CDC for repeated point update") { append(Seq((1, 1), (2, 2), (3, 3), (4, 4)).toDF("key", "value")) checkUpdate( condition = Some("key = 1"), setClauses = "value = -1", expectedResults = Row(1, -1) :: Row(2, 2) :: Row(3, 3) :: Row(4, 4) :: Nil) val latestVersion1 = deltaLog.update().version checkAnswer( computeCDC(spark, deltaLog, latestVersion1, latestVersion1) .drop(CDCReader.CDC_COMMIT_TIMESTAMP), Row(1, 1, "update_preimage", latestVersion1) :: Row(1, -1, "update_postimage", latestVersion1) :: Nil) checkUpdate( condition = Some("key = 3"), setClauses = "value = -3", expectedResults = Row(1, -1) :: Row(2, 2) :: Row(3, -3) :: Row(4, 4) :: Nil) val latestVersion2 = deltaLog.update().version checkAnswer( computeCDC(spark, deltaLog, latestVersion1, latestVersion2) .drop(CDCReader.CDC_COMMIT_TIMESTAMP), Row(1, 1, "update_preimage", latestVersion1) :: Row(1, -1, "update_postimage", latestVersion1) :: Row(3, 3, "update_preimage", latestVersion2) :: Row(3, -3, "update_postimage", latestVersion2) :: Nil) } test("CDC for partition-optimized update") { append( Seq((1, 1, 1), (2, 2, 0), (3, 3, 1), (4, 4, 0)).toDF("key", "value", "part"), partitionBy = Seq("part")) checkUpdate( condition = Some("part = 1"), setClauses = "value = -1", expectedResults = Row(1, -1) :: Row(2, 2) :: Row(3, -1) :: Row(4, 4) :: Nil) val latestVersion = deltaLog.update().version checkAnswer( computeCDC(spark, deltaLog, latestVersion, latestVersion) .drop(CDCReader.CDC_COMMIT_TIMESTAMP), Row(1, 1, 1, "update_preimage", latestVersion) :: Row(1, -1, 1, "update_postimage", latestVersion) :: Row(3, 3, 1, "update_preimage", latestVersion) :: Row(3, -1, 1, "update_postimage", latestVersion) :: Nil) } test("update a partitioned CDC enabled table to set the partition column to null") { val tableName = "part_table_test" withTable(tableName) { Seq((0, 0, 0), (1, 1, 1), (2, 2, 2)) .toDF("key", "partition_column", "value") .write .partitionBy("partition_column") .format("delta") .saveAsTable(tableName) sql(s"INSERT INTO $tableName VALUES (4, 4, 4)") sql(s"UPDATE $tableName SET partition_column = null WHERE partition_column = 4") checkAnswer( computeCDC(spark, DeltaLog.forTable( spark, spark.sessionState.sqlParser.parseTableIdentifier(tableName) ), 1, 2) .drop(CDCReader.CDC_COMMIT_TIMESTAMP), Row(4, 4, 4, "insert", 1) :: Row(4, 4, 4, "update_preimage", 2) :: Row(4, null, 4, "update_postimage", 2) :: Nil) } } } trait UpdateCDCWithDeletionVectorsTests extends UpdateSQLWithDeletionVectorsMixin with CDCTestMixin { test("UPDATE with DV write CDC files explicitly") { append(spark.range(0, 10, 1, numPartitions = 2).toDF()) executeUpdate(tableSQLIdentifier, "id = -1", "id % 4 = 0") val latestVersion = deltaLog.update().version checkAnswer( computeCDC(spark, deltaLog, latestVersion, latestVersion) .drop(CDCReader.CDC_COMMIT_TIMESTAMP), Row(0, "update_preimage", latestVersion) :: Row(-1, "update_postimage", latestVersion) :: Row(4, "update_preimage", latestVersion) :: Row(-1, "update_postimage", latestVersion) :: Row(8, "update_preimage", latestVersion) :: Row(-1, "update_postimage", latestVersion) :: Nil) val allActions = deltaLog.getChanges(latestVersion).flatMap(_._2).toSeq val addActions = allActions.collect { case f: AddFile => f } val removeActions = allActions.collect { case f: RemoveFile => f } val cdcActions = allActions.collect { case f: AddCDCFile => f } assert(addActions.count(_.deletionVector != null) === 2) assert(removeActions.size === 2) assert(cdcActions.nonEmpty) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/clustering/ClusteredTableClusteringSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.clustering import org.apache.spark.sql.delta.skipping.ClusteredTableTestUtils import org.apache.spark.sql.delta.skipping.clustering.ClusteredTableUtils import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession class ClusteredTableClusteringSuite extends SparkFunSuite with SharedSparkSession with ClusteredTableTestUtils with DeltaSQLCommandTest { import testImplicits._ private val table: String = "test_table" // Ingest data to create numFiles files with one row in each file. private def addFiles(table: String, numFiles: Int): Unit = { val df = (1 to numFiles).map(i => (i, i)).toDF("col1", "col2") withSQLConf(SQLConf.MAX_RECORDS_PER_FILE.key -> "1") { df.write.format("delta").mode("append").saveAsTable(table) } } private def getFiles(table: String): Set[AddFile] = { val deltaLog = DeltaLog.forTable(spark, TableIdentifier(table)) deltaLog.update().allFiles.collect().toSet } private def assertClustered(files: Set[AddFile]): Unit = { assert(files.forall(_.clusteringProvider.contains(ClusteredTableUtils.clusteringProvider))) } private def assertNotClustered(files: Set[AddFile]): Unit = { assert(files.forall(_.clusteringProvider.isEmpty)) } test("optimize clustered table") { withSQLConf(SQLConf.MAX_RECORDS_PER_FILE.key -> "2") { withClusteredTable( table = table, schema = "col1 int, col2 int", clusterBy = "col1, col2") { addFiles(table, numFiles = 4) val files0 = getFiles(table) assert(files0.size === 4) assertNotClustered(files0) // Optimize should cluster the data into two 2 files since MAX_RECORDS_PER_FILE is 2. runOptimize(table) { metrics => assert(metrics.numFilesRemoved == 4) assert(metrics.numFilesAdded == 2) } val files1 = getFiles(table) assert(files1.size == 2) assertClustered(files1) } } } test("cluster by 1 column") { withSQLConf(SQLConf.MAX_RECORDS_PER_FILE.key -> "2") { withClusteredTable( table = table, schema = "col1 int, col2 int", clusterBy = "col1") { addFiles(table, numFiles = 4) val files0 = getFiles(table) assert(files0.size === 4) assertNotClustered(files0) // Optimize should cluster the data into two 2 files since MAX_RECORDS_PER_FILE is 2. runOptimize(table) { metrics => assert(metrics.numFilesRemoved == 4) assert(metrics.numFilesAdded == 2) } val files1 = getFiles(table) assert(files1.size == 2) assertClustered(files1) } } } test("optimize clustered table with batching") { Seq(("1", 2), ("1g", 1)).foreach { case (batchSize, optimizeCommits) => withClusteredTable( table = table, schema = "col1 int, col2 int", clusterBy = "col1, col2") { addFiles(table, numFiles = 4) val files0 = getFiles(table) assert(files0.size === 4) assertNotClustered(files0) val totalSize = files0.toSeq.map(_.size).sum val halfSize = totalSize / 2 withSQLConf( DeltaSQLConf.DELTA_OPTIMIZE_BATCH_SIZE.key -> batchSize, DeltaSQLConf.DELTA_OPTIMIZE_CLUSTERING_MIN_CUBE_SIZE.key -> halfSize.toString, DeltaSQLConf.DELTA_OPTIMIZE_CLUSTERING_TARGET_CUBE_SIZE.key -> halfSize.toString) { // Optimize should create 2 cubes, which will be in separate batches if the batch size // is small enough runOptimize(table) { metrics => assert(metrics.numFilesRemoved == 4) assert(metrics.numFilesAdded == 2) } val files1 = getFiles(table) assert(files1.size == 2) assertClustered(files1) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(table)) val commits = deltaLog.history.getHistory(None) assert(commits.filter(_.operation == "OPTIMIZE").length == optimizeCommits) } } } } test("optimize clustered table with batching on an empty table") { withClusteredTable( table = table, schema = "col1 int, col2 int", clusterBy = "col1, col2") { withSQLConf(DeltaSQLConf.DELTA_OPTIMIZE_BATCH_SIZE.key -> "1g") { runOptimize(table) { metrics => assert(metrics.numFilesRemoved == 0) assert(metrics.numFilesAdded == 0) } } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/clustering/ClusteringMetadataDomainSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.clustering import org.apache.spark.sql.delta.skipping.clustering.ClusteringColumn import org.apache.spark.SparkFunSuite class ClusteringMetadataDomainSuite extends SparkFunSuite { test("serialized string follows the spec") { val clusteringColumns = Seq(ClusteringColumn(Seq("col1", "`col2,col3`", "`col4.col5`,col6"))) val clusteringMetadataDomain = ClusteringMetadataDomain.fromClusteringColumns(clusteringColumns) val serializedString = clusteringMetadataDomain.toDomainMetadata.json assert(serializedString === """|{"domainMetadata":{"domain":"delta.clustering","configuration": |"{\"clusteringColumns\":[[\"col1\",\"`col2,col3`\",\"`col4.col5`,col6\"]], |\"domainName\":\"delta.clustering\"}","removed":false}}""".stripMargin.replace("\n", "")) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/clustering/ClusteringTableFeatureSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.clustering import com.databricks.spark.util.{Log4jUsageLogger, MetricDefinitions} import org.apache.spark.sql.delta.skipping.ClusteredTableTestUtils import org.apache.spark.sql.delta.skipping.clustering.ClusteredTableUtils import org.apache.spark.sql.delta.{ClusteringTableFeature, DeltaAnalysisException, DeltaLog, TableFeature} import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession class ClusteringTableFeatureSuite extends SparkFunSuite with SharedSparkSession with ClusteredTableTestUtils with DeltaSQLCommandTest { import testImplicits._ test("create table without cluster by clause cannot set clustering table properties") { withTable("tbl") { val e = intercept[DeltaAnalysisException] { sql("CREATE TABLE tbl(a INT, b STRING) USING DELTA " + "TBLPROPERTIES('delta.feature.clustering' = 'supported')") } checkError( e, "DELTA_CREATE_TABLE_SET_CLUSTERING_TABLE_FEATURE_NOT_ALLOWED", parameters = Map("tableFeature" -> "clustering")) } } test("use alter table set table properties to enable clustering is not allowed.") { withTable("tbl") { sql("CREATE TABLE tbl(a INT, b STRING) USING DELTA") val e = intercept[DeltaAnalysisException] { sql("ALTER TABLE tbl SET TBLPROPERTIES ('delta.feature.clustering' = 'supported')") } checkError( e, "DELTA_ALTER_TABLE_SET_CLUSTERING_TABLE_FEATURE_NOT_ALLOWED", parameters = Map("tableFeature" -> "clustering")) } } test("alter table cluster by partitioned tables is not allowed.") { withTable("tbl") { sql("CREATE TABLE tbl(a INT, b STRING) USING DELTA PARTITIONED BY (a)") val e1 = intercept[DeltaAnalysisException] { sql("ALTER TABLE tbl CLUSTER BY (a)") } checkError( e1, "DELTA_ALTER_TABLE_CLUSTER_BY_ON_PARTITIONED_TABLE_NOT_ALLOWED", parameters = Map.empty) val e2 = intercept[DeltaAnalysisException] { sql("ALTER TABLE tbl CLUSTER BY NONE") } checkError( e2, "DELTA_ALTER_TABLE_CLUSTER_BY_ON_PARTITIONED_TABLE_NOT_ALLOWED", parameters = Map.empty) } } test("alter table cluster by unpartitioned tables is supported.") { val table = "tbl" withTable(table) { sql(s"CREATE TABLE $table (a INT, b STRING) USING DELTA") val (_, startingSnapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(table)) assert(!ClusteredTableUtils.isSupported(startingSnapshot.protocol)) val clusterByLogs = Log4jUsageLogger.track { sql(s"ALTER TABLE $table CLUSTER BY (a)") }.filter { e => e.metric == MetricDefinitions.EVENT_TAHOE.name && e.tags.get("opType").contains("delta.ddl.alter.clusterBy") } assert(clusterByLogs.nonEmpty) val clusterByLogJson = JsonUtils.fromJson[Map[String, Any]](clusterByLogs.head.blob) assert(!clusterByLogJson("isClusterByNoneSkipped").asInstanceOf[Boolean]) val (_, finalSnapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(table)) assert(ClusteredTableUtils.isSupported(finalSnapshot.protocol)) val dependentFeatures = TableFeature.getDependentFeatures(ClusteringTableFeature) dependentFeatures.foreach { feature => assert(finalSnapshot.protocol.isFeatureSupported(feature)) } withSQLConf(SQLConf.MAX_RECORDS_PER_FILE.key -> "2") { val df = (1 to 4).map(i => (i, i.toString)).toDF("a", "b") withSQLConf(SQLConf.MAX_RECORDS_PER_FILE.key -> "1") { df.write.format("delta").mode("append").saveAsTable(table) } // Optimize should cluster the data into two 2 files since MAX_RECORDS_PER_FILE is 2. runOptimize(table) { metrics => assert(metrics.numFilesRemoved == 4) assert(metrics.numFilesAdded == 2) } val deltaLog = DeltaLog.forTable(spark, TableIdentifier(table)) val files1 = deltaLog.update().allFiles.collect().toSet assert(files1.size == 2) assert(files1.forall(_.clusteringProvider.contains(ClusteredTableUtils.clusteringProvider))) // Check if min-max intervals of 'a' are sorted val minMaxIntervals = files1.map { file => val stats = JsonUtils.mapper.readTree(file.stats) (stats.get("minValues").get("a").asInt, stats.get("maxValues").get("a").asInt) } val sortedAsc = minMaxIntervals.sliding(2).forall { case Seq((_, maxA1), (minA2, _)) => maxA1.asInstanceOf[Int] < minA2.asInstanceOf[Int] case _ => true } val sortedDesc = minMaxIntervals.sliding(2).forall { case Seq((minA1, _), (_, maxA2)) => minA1.asInstanceOf[Int] > maxA2.asInstanceOf[Int] case _ => true } assert(sortedAsc || sortedDesc, "Min-max intervals for column 'a' are not sorted.") } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/DropColumnMappingFeatureSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.columnmapping import java.util.concurrent.TimeUnit import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.DeltaConfigs._ import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.sources.DeltaSQLConf._ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.util.ManualClock /** * Test dropping column mapping feature from a table. */ class DropColumnMappingFeatureSuite extends RemoveColumnMappingSuiteUtils { override def beforeAll(): Unit = { super.beforeAll() // All the drop feature tests below are based on the drop feature with history truncation // implementation. The fast drop feature implementation does not require any waiting time. // The fast drop feature implementation is tested extensively in the DeltaFastDropFeatureSuite. spark.conf.set(DeltaSQLConf.FAST_DROP_FEATURE_ENABLED.key, false.toString) } val clock = new ManualClock(System.currentTimeMillis()) test("column mapping cannot be dropped without the feature flag") { withSQLConf(ALLOW_COLUMN_MAPPING_REMOVAL.key -> "false") { sql(s"""CREATE TABLE $testTableName |USING delta |TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'name', | 'delta.minReaderVersion' = '3', | 'delta.minWriterVersion' = '7') |AS SELECT 1 as a |""".stripMargin) intercept[DeltaColumnMappingUnsupportedException] { dropColumnMappingTableFeature() } } } test("table without column mapping enabled") { sql(s"""CREATE TABLE $testTableName |USING delta |TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'none') |AS SELECT 1 as a |""".stripMargin) val e = intercept[DeltaTableFeatureException] { dropColumnMappingTableFeature() } checkError(e, DeltaErrors.dropTableFeatureFeatureNotSupportedByProtocol(".").getErrorClass, parameters = Map("feature" -> "columnMapping")) } test("invalid column names") { val invalidColName1 = colName("col1") val invalidColName2 = colName("col2") sql( s"""CREATE TABLE $testTableName (a INT, `$invalidColName1` INT, `$invalidColName2` INT) |USING delta |TBLPROPERTIES ('delta.columnMapping.mode' = 'name') |""".stripMargin) val e = intercept[DeltaAnalysisException] { // Try to drop column mapping. dropColumnMappingTableFeature() } checkError(e, "DELTA_INVALID_COLUMN_NAMES_WHEN_REMOVING_COLUMN_MAPPING", parameters = Map("invalidColumnNames" -> "col1 with special chars ,;{}()\n\t=, col2 with special chars ,;{}()\n\t=")) } test("drop column mapping from a table without table feature") { sql( s"""CREATE TABLE $testTableName |USING delta |TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'name', | '${DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key}' = 'false', | 'delta.minReaderVersion' = '3', | 'delta.minWriterVersion' = '7') |AS SELECT id as $logicalColumnName, id + 1 as $secondColumn | FROM RANGE(0, $totalRows, 1, $numFiles) |""".stripMargin) testDroppingColumnMapping() } test("drop column mapping from a table with table feature") { sql( s"""CREATE TABLE $testTableName |USING delta |TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'name', | '${DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key}' = 'false', | 'delta.minReaderVersion' = '3', | 'delta.minWriterVersion' = '7') |AS SELECT id as $logicalColumnName, id + 1 as $secondColumn | FROM RANGE(0, $totalRows, 1, $numFiles) |""".stripMargin) testDroppingColumnMapping() } test("drop column mapping from a table without column mapping table property") { sql( s"""CREATE TABLE $testTableName |USING delta |TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'name', | '${DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key}' = 'false', | 'delta.minReaderVersion' = '3', | 'delta.minWriterVersion' = '7') |AS SELECT id as $logicalColumnName, id + 1 as $secondColumn | FROM RANGE(0, $totalRows, 1, $numFiles) |""".stripMargin) unsetColumnMappingProperty(useUnset = true) val e = intercept[DeltaTableFeatureException] { dropColumnMappingTableFeature() } checkError( e, "DELTA_FEATURE_DROP_HISTORICAL_VERSIONS_EXIST", parameters = Map( "feature" -> "columnMapping", "logRetentionPeriodKey" -> "delta.logRetentionDuration", "logRetentionPeriod" -> "30 days", "truncateHistoryLogRetentionPeriod" -> "24 hours") ) } test("drop column mapping in id mode") { sql( s"""CREATE TABLE $testTableName |USING delta |TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'id', | '${DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key}' = 'false', | 'delta.minReaderVersion' = '3', | 'delta.minWriterVersion' = '7') |AS SELECT id as $logicalColumnName, id + 1 as $secondColumn | FROM RANGE(0, $totalRows, 1, $numFiles) |""".stripMargin) testDroppingColumnMapping() } def testDroppingColumnMapping(): Unit = { withSQLConf( "spark.databricks.delta.vacuum.enforceDeletedFileAndLogRetentionDurationCompatibility" -> "false") { // Verify the input data is as expected. val originalData = spark.table(tableName = testTableName).select(logicalColumnName).collect() // Add a schema comment and verify it is preserved after the rewrite. val comment = "test comment" sql(s"ALTER TABLE $testTableName ALTER COLUMN $logicalColumnName COMMENT '$comment'") val table = DeltaTableV2(spark, TableIdentifier(tableName = testTableName), "") val originalSnapshot = table.initialSnapshot assert(originalSnapshot.schema.head.getComment().get == comment, "Renamed column should preserve comment.") val originalFiles = getFiles(originalSnapshot) val startingVersion = originalSnapshot.version val e = intercept[DeltaTableFeatureException] { dropColumnMappingTableFeature() } checkError( e, "DELTA_FEATURE_DROP_WAIT_FOR_RETENTION_PERIOD", parameters = Map( "feature" -> "columnMapping", "logRetentionPeriodKey" -> "delta.logRetentionDuration", "logRetentionPeriod" -> "30 days", "truncateHistoryLogRetentionPeriod" -> "24 hours") ) verifyRewrite( unsetTableProperty = true, table, originalFiles, startingVersion, originalData = originalData, droppedFeature = true) // Verify the schema comment is preserved after the rewrite. assert(deltaLog.update().schema.head.getComment().get == comment, "Should preserve the schema comment.") verifyDropFeatureTruncateHistory() } } protected def verifyDropFeatureTruncateHistory() = { val deltaLog1 = DeltaLog.forTable(spark, TableIdentifier(tableName = testTableName), clock) // Populate the delta cache with the delta log with the right data path so it stores the clock. // This is currently the only way to make sure the drop feature command uses the clock. DeltaLog.clearCache() DeltaLog.forTable(spark, deltaLog1.dataPath, clock) // Set the log retention to 0 so that we can test truncate history. sql( s""" |ALTER TABLE $testTableName SET TBLPROPERTIES ( | '${TABLE_FEATURE_DROP_TRUNCATE_HISTORY_LOG_RETENTION.key}' = '0 hours', | '${LOG_RETENTION.key}' = '0 hours') |""".stripMargin) // Pretend enough time has passed for the history to be truncated. clock.advance(TimeUnit.MINUTES.toMillis(5)) sql( s""" |ALTER TABLE $testTableName DROP FEATURE ${ColumnMappingTableFeature.name} TRUNCATE HISTORY |""".stripMargin) val newSnapshot = deltaLog.update() assert(newSnapshot.protocol.readerAndWriterFeatures.isEmpty, "Should drop the feature.") assert(newSnapshot.protocol.minWriterVersion == 1) assert(newSnapshot.protocol.minReaderVersion == 1) } protected def dropColumnMappingTableFeature(): Unit = { sql( s""" |ALTER TABLE $testTableName DROP FEATURE ${ColumnMappingTableFeature.name} |""".stripMargin) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/RemoveColumnMappingCDCSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.columnmapping import org.apache.spark.sql.delta.DeltaConfigs import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.DeltaUnsupportedOperationException import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.TableIdentifier /** * Test suite for removing column mapping(CM) from a table with CDC enabled. Test different * scenarios with respect to table schema at different points of time. There are a few events we * are interested in: UP: enable column mapping DOWN: disable column mapping RE, DROP: rename, * drop a column * * And there are two parameters for reading CDC: START: starting version END: ending version * * We test all the possible combinations of these events and parameters. */ class RemoveColumnMappingCDCSuite extends RemoveColumnMappingSuiteUtils { // These two cases will fail because latest schema will be used to read CDC. // Table doesn't have column mapping enabled in any of the start, end or latest versions. // So it defaults to the non-column mapping behavior. runScenario(Start, End, Upgrade, Rename, Downgrade)(ReadCDCIncompatibleDataSchema) runScenario(Start, End, Upgrade, Drop, Downgrade)(ReadCDCIncompatibleDataSchema) runScenario(Start, End, Upgrade, Rename, Downgrade, Upgrade)(ReadCDCSuccess) runScenario(Start, End, Upgrade, Drop, Downgrade, Upgrade)(ReadCDCSuccess) // This will use the endVersion schema to read CDC because endVersion has columnampping enabled. runScenario(Start, Upgrade, End, Rename, Downgrade)(ReadCDCSuccess) runScenario(Start, Upgrade, End, Drop, Downgrade)(ReadCDCSuccess) runScenario(Start, Upgrade, End, Rename, Downgrade, Upgrade)(ReadCDCSuccess) runScenario(Start, Upgrade, End, Drop, Downgrade, Upgrade)(ReadCDCSuccess) // Reading across non-additive schema change. runScenario(Start, Upgrade, Rename, End, Downgrade)(ReadCDCIncompatibleSchemaChange) runScenario(Start, Upgrade, Drop, End, Downgrade)(ReadCDCIncompatibleSchemaChange) runScenario(Start, Upgrade, Rename, End, Downgrade, Upgrade)(ReadCDCIncompatibleSchemaChange) runScenario(Start, Upgrade, Drop, End, Downgrade, Upgrade)(ReadCDCIncompatibleSchemaChange) runScenario(Start, Upgrade, Rename, Downgrade, End)(ReadCDCIncompatibleSchemaChange) runScenario(Start, Upgrade, Drop, Downgrade, End)(ReadCDCIncompatibleSchemaChange) runScenario(Start, Upgrade, Rename, Downgrade, End, Upgrade)(ReadCDCIncompatibleSchemaChange) runScenario(Start, Upgrade, Drop, Downgrade, End, Upgrade)(ReadCDCIncompatibleSchemaChange) runScenario(Start, Upgrade, Rename, Downgrade, Upgrade, End)(ReadCDCIncompatibleSchemaChange) runScenario(Start, Upgrade, Drop, Downgrade, Upgrade, End)(ReadCDCIncompatibleSchemaChange) runScenario(Upgrade, Start, End, Rename, Downgrade)(ReadCDCSuccess) runScenario(Upgrade, Start, End, Drop, Downgrade)(ReadCDCSuccess) runScenario(Upgrade, Start, End, Rename, Downgrade, Upgrade)(ReadCDCSuccess) runScenario(Upgrade, Start, End, Drop, Downgrade, Upgrade)(ReadCDCSuccess) // Reading across non-additive schema change. runScenario(Upgrade, Start, Rename, End, Downgrade)(ReadCDCIncompatibleDataSchema) runScenario(Upgrade, Start, Drop, End, Downgrade)(ReadCDCIncompatibleDataSchema) runScenario(Upgrade, Start, Rename, End, Downgrade, Upgrade)(ReadCDCIncompatibleDataSchema) runScenario(Upgrade, Start, Drop, End, Downgrade, Upgrade)(ReadCDCIncompatibleDataSchema) // Reading across non-additive schema change. runScenario(Upgrade, Start, Rename, Downgrade, End)(ReadCDCIncompatibleSchemaChange) runScenario(Upgrade, Start, Drop, Downgrade, End)(ReadCDCIncompatibleSchemaChange) runScenario(Upgrade, Start, Rename, Downgrade, End, Upgrade)(ReadCDCIncompatibleSchemaChange) runScenario(Upgrade, Start, Drop, Downgrade, End, Upgrade)(ReadCDCIncompatibleSchemaChange) runScenario(Upgrade, Start, Rename, Downgrade, Upgrade, End)(ReadCDCIncompatibleSchemaChange) runScenario(Upgrade, Start, Drop, Downgrade, Upgrade, End)(ReadCDCIncompatibleDataSchema) runScenario(Upgrade, Rename, Start, End, Downgrade)(ReadCDCSuccess) runScenario(Upgrade, Drop, Start, End, Downgrade)(ReadCDCSuccess) runScenario(Upgrade, Rename, Start, End, Downgrade, Upgrade)(ReadCDCSuccess) runScenario(Upgrade, Drop, Start, End, Downgrade, Upgrade)(ReadCDCSuccess) // Reading across downgrade. runScenario(Upgrade, Rename, Start, Downgrade, End)(ReadCDCIncompatibleDataSchema) runScenario(Upgrade, Drop, Start, Downgrade, End)(ReadCDCIncompatibleDataSchema) runScenario(Upgrade, Rename, Start, Downgrade, End, Upgrade)(ReadCDCIncompatibleDataSchema) runScenario(Upgrade, Drop, Start, Downgrade, End, Upgrade)(ReadCDCIncompatibleDataSchema) runScenario(Upgrade, Rename, Start, Downgrade, Upgrade, End)(ReadCDCIncompatibleDataSchema) // Schema is readable in this range. runScenario(Upgrade, Drop, Start, Downgrade, Upgrade, End)(ReadCDCSuccess) runScenario(Upgrade, Rename, Downgrade, Start, End)(ReadCDCSuccess) runScenario(Upgrade, Drop, Downgrade, Start, End)(ReadCDCSuccess) runScenario(Upgrade, Rename, Downgrade, Start, End, Upgrade)(ReadCDCSuccess) runScenario(Upgrade, Drop, Downgrade, Start, End, Upgrade)(ReadCDCSuccess) runScenario(Upgrade, Rename, Downgrade, Start, Upgrade, End)(ReadCDCSuccess) runScenario(Upgrade, Drop, Downgrade, Start, Upgrade, End)(ReadCDCSuccess) runScenario(Upgrade, Rename, Downgrade, Upgrade, Start, End)(ReadCDCSuccess) runScenario(Upgrade, Drop, Downgrade, Upgrade, Start, End)(ReadCDCSuccess) private def runScenario(operations: Operation*)(readCDC: ReadCDC): Unit = { val testName = operations.map { _.toString }.mkString(", ") + " - " + readCDC.toString var startVersion = 0L var endVersion: Option[Long] = None test(testName) { createTable() operations.foreach { case op @ Start => startVersion = deltaLog.update().version op.runOperation() case op @ End => op.runOperation() endVersion = Some(deltaLog.update().version) case op => op.runOperation() } readCDC.runReadCDC(startVersion, endVersion) } } private abstract class Operation { def runOperation(): Unit } private case object Start extends Operation { override def runOperation(): Unit = { insertMoreRows() } } private case object End extends Operation { override def runOperation(): Unit = { insertMoreRows() } } private case object Upgrade extends Operation { override def runOperation(): Unit = { enableColumnMapping() insertMoreRows() } } private case object Downgrade extends Operation { override def runOperation(): Unit = { unsetColumnMappingProperty(useUnset = false) insertMoreRows() } } private case object Rename extends Operation { override def runOperation(): Unit = { renameColumn() insertMoreRows() } } private case object Drop extends Operation { override def runOperation(): Unit = { dropColumn() insertMoreRows() } } private abstract class ReadCDC { def runReadCDC(start: Long, end: Option[Long]): Unit } private case object ReadCDCSuccess extends ReadCDC { override def runReadCDC(start: Long, end: Option[Long]): Unit = { val changes = getChanges(start, end) assert(changes.length > 0, "should have read some changes") changes.foreach { row => assert(!row.anyNull, "None of the values should be null") } } } private case object ReadCDCIncompatibleSchemaChange extends ReadCDC { override def runReadCDC(start: Long, end: Option[Long]): Unit = { getCDCAndFailIncompatibleSchemaChange(start, end) } } private case object ReadCDCIncompatibleDataSchema extends ReadCDC { override def runReadCDC(start: Long, end: Option[Long]): Unit = { getCDCAndFailIncompatibleDataSchema(start, end) } } private def createTable(): Unit = { val columnMappingMode = "none" sql(s"""CREATE TABLE $testTableName |USING delta |TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = '$columnMappingMode', | '${DeltaConfigs.CHANGE_DATA_FEED.key}' = 'true' |) |AS SELECT id as $firstColumn, id + 1 as $secondColumn, id + 2 as $thirdColumn | FROM RANGE(0, $totalRows, 1, $numFiles) |""".stripMargin) } private def insertMoreRows(): Unit = { sql(s"INSERT INTO $testTableName SELECT * FROM $testTableName LIMIT $totalRows") } private def getCDCAndFailIncompatibleSchemaChange( startVersion: Long, endVersion: Option[Long]) = { val e = intercept[DeltaUnsupportedOperationException] { getChanges(startVersion, endVersion) } assert(e.getErrorClass === "DELTA_CHANGE_DATA_FEED_INCOMPATIBLE_SCHEMA_CHANGE") } private def getCDCAndFailIncompatibleDataSchema( startVersion: Long, endVersion: Option[Long]) = { val e = intercept[DeltaUnsupportedOperationException] { getChanges(startVersion, endVersion) } assert(e.getErrorClass === "DELTA_CHANGE_DATA_FEED_INCOMPATIBLE_DATA_SCHEMA") } private def getChanges(startVersion: Long, endVersion: Option[Long]): Array[Row] = { val endVersionStr = if (endVersion.isDefined) s", ${endVersion.get}" else "" sql(s"SELECT * FROM table_changes('$testTableName', $startVersion$endVersionStr)") .collect() } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/RemoveColumnMappingRowTrackingSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.columnmapping import org.apache.spark.sql.delta.DeltaConfigs import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.RowId import org.apache.spark.sql.delta.RowId.RowTrackingMetadataDomain import org.apache.spark.sql.delta.Snapshot import org.apache.spark.sql.delta.sources.DeltaSQLConf._ import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.TableIdentifier class RemoveColumnMappingRowTrackingSuite extends RemoveColumnMappingSuiteUtils { test("row ids are preserved") { sql( s"""CREATE TABLE $testTableName |USING delta |TBLPROPERTIES ( |'${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'name', |'${DeltaConfigs.ROW_TRACKING_ENABLED.key}' = 'true' |) |AS SELECT id as $logicalColumnName, id + 1 as $secondColumn | FROM RANGE(0, $totalRows, 1, $numFiles) |""".stripMargin) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName = testTableName)) val snapshot = deltaLog.update() val originalDf = spark.read.table(testTableName) val originalRowIds = originalDf.select(logicalColumnName, RowId.QUALIFIED_COLUMN_NAME) .collect() val originalDomainMetadata = RowTrackingMetadataDomain.fromSnapshot(snapshot).get testRemovingColumnMapping() verifyRowIdsStayTheSame(originalRowIds) verifyDomainMetadata(deltaLog.update(), originalDomainMetadata, diffRows = totalRows) // Add back column mapping and remove it again. Row ids should stay the same sql( s"""ALTER TABLE $testTableName |SET TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'name')""".stripMargin) // Update a row from each file to force materialize the row ids and metadata. val predicate = s"$logicalColumnName % $rowsPerFile == 0" withSQLConf(UPDATE_USE_PERSISTENT_DELETION_VECTORS.key -> "false") { sql(s"UPDATE $testTableName SET $secondColumn = -1 WHERE $predicate ") } testRemovingColumnMapping() verifyRowIdsStayTheSame(originalRowIds) // High watermark increased 3 times from the original value. Rewrite, UPDATE, Rewrite. verifyDomainMetadata(deltaLog.update(), originalDomainMetadata, diffRows = totalRows * 3) } private def verifyRowIdsStayTheSame(originalRowIds: Array[Row]) = { val newRowIds = spark.read.table(testTableName) .select(logicalColumnName, RowId.QUALIFIED_COLUMN_NAME) checkAnswer(newRowIds, originalRowIds) } private def verifyDomainMetadata( snapshot: Snapshot, originalDomainMetadata: RowTrackingMetadataDomain, diffRows: Int) = { val newDomainMetadata = RowTrackingMetadataDomain.fromSnapshot(snapshot).get assert(newDomainMetadata.rowIdHighWaterMark === originalDomainMetadata.rowIdHighWaterMark + diffRows, "Should increase the high watermark by the number of rewritten rows.") } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/RemoveColumnMappingStreamingReadSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.columnmapping import org.apache.spark.sql.delta.ColumnMappingStreamingTestUtils import org.apache.spark.sql.delta.DeltaConfigs import org.apache.spark.sql.delta.DeltaOptions import org.apache.spark.sql.delta.DeltaRuntimeException import org.apache.spark.sql.Row import org.apache.spark.sql.streaming.StreamTest /** * Test suite for removing column mapping(CM) from a streaming table. Test different * scenarios with respect to table schema at different points of time. There are a few events we * are interested in: * Upgrade: enable column mapping * Downgrade: disable column mapping * Rename, Drop: rename, drop a column * * And we can decide when we start the streaming read with StartStreamRead. * * We test all the possible combinations of these events. * * Additionally, we test each scenario with schema tracking enabled which in general results in * a failure as schema tracking prohibits reading across an Upgrade. */ class RemoveColumnMappingStreamingReadSuite extends RemoveColumnMappingSuiteUtils with StreamTest with ColumnMappingStreamingTestUtils { // Here the physical/logical names don't change between start and the end // so it succeeds without schema tracking. Schema tracking prohibits reading across an upgrade. runScenario(StartStreamRead, Upgrade, Downgrade, SuccessAndFailSchemaTracking) // Here the physical names do change. We start with existing physical names but end without them. runScenario(Upgrade, StartStreamRead, Downgrade, FailNonAdditiveChange) // This is just reading from a normal table. runScenario(Upgrade, Downgrade, StartStreamRead, Success) // In all of this cases there is a non-additive change between the start of the stream and // the end. runScenario(StartStreamRead, Upgrade, Rename, Downgrade, FailNonAdditiveChange) runScenario(StartStreamRead, Upgrade, Drop, Downgrade, FailNonAdditiveChange) runScenario(StartStreamRead, Upgrade, Rename, Downgrade, Upgrade, FailNonAdditiveChange) runScenario(StartStreamRead, Upgrade, Drop, Downgrade, Upgrade, FailNonAdditiveChange) runScenario(Upgrade, StartStreamRead, Rename, Downgrade, FailNonAdditiveChange) runScenario(Upgrade, StartStreamRead, Drop, Downgrade, FailNonAdditiveChange) runScenario(Upgrade, StartStreamRead, Rename, Downgrade, Upgrade, FailNonAdditiveChange) runScenario(Upgrade, StartStreamRead, Drop, Downgrade, Upgrade, FailNonAdditiveChange) // In these cases schema pinned at the start of the stream is different from the end schema on // the physical level. // Essentially, prohibit reading across the downgrade. runScenario(Upgrade, Rename, StartStreamRead, Downgrade, FailNonAdditiveChange) runScenario(Upgrade, Rename, StartStreamRead, Downgrade, Upgrade, FailNonAdditiveChange) runScenario(Upgrade, Drop, StartStreamRead, Downgrade, FailNonAdditiveChange) // Here the schema at the end version has different physical names than at the start version. runScenario(Upgrade, Drop, StartStreamRead, Downgrade, Upgrade, FailNonAdditiveChange) // This is just reading from a table without column mapping. runScenario(Upgrade, Rename, Downgrade, StartStreamRead, Success) runScenario(Upgrade, Drop, Downgrade, StartStreamRead, Success) // Reading across the upgrade is fine without schema tracking. runScenario(Upgrade, Rename, Downgrade, StartStreamRead, Upgrade, SuccessAndFailSchemaTracking) runScenario(Upgrade, Drop, Downgrade, StartStreamRead, Upgrade, SuccessAndFailSchemaTracking) private def runScenario(operations: Operation*): Unit = { // Run each scenario with and without schema tracking. for (shouldTrackSchema <- Seq(true, false)) { withTempPath { tempPath => val metadataLocation = tempPath.getCanonicalPath val testName = generateTestName(operations, shouldTrackSchema) val schemaTrackingLocation = if (shouldTrackSchema) { Some(metadataLocation) } else { None } test(testName) { createTable() // Run all actions before the stream starts val streamStartIndex = operations.indexWhere(_ == StartStreamRead) operations.take(streamStartIndex).foreach(_.runOperation()) // Run the rest as stream actions val remainingActions = operations.takeRight(operations.size - streamStartIndex) testStream(testTableStreamDf(schemaTrackingLocation))( remainingActions.flatMap { // Add an explicit StartStream so we can pass the checkpoint location. case StartStreamRead => Seq(StartStream(checkpointLocation = metadataLocation)) // Fail scenarios when schema tracking is enabled. case FailNonAdditiveChange | SuccessAndFailSchemaTracking if shouldTrackSchema => FailSchemaEvolution.toStreamActions case op: StreamActionLike => op.toStreamActions case op => Seq( Execute { _ => op.runOperation() }) }: _* ) } } } } private def generateTestName(operations: Seq[Operation], shouldTrackSchema: Boolean) = { val testNameSuffix = if (shouldTrackSchema) { " with schema tracking" } else { "" } val testName = operations.map(_.toString).mkString(", ") + testNameSuffix testName } private abstract class Operation { def runOperation(): Unit = {} } private case object StartStreamRead extends Operation private case object Upgrade extends Operation { override def runOperation(): Unit = { enableColumnMapping() } } private case object Downgrade extends Operation { override def runOperation(): Unit = { unsetColumnMappingProperty(useUnset = false) insertMoreRows() } } private case object Rename extends Operation { override def runOperation(): Unit = { renameColumn() } } private case object Drop extends Operation { override def runOperation(): Unit = { dropColumn() } } private case object FailNonAdditiveChange extends Operation with StreamActionLike { override def toStreamActions: Seq[StreamAction] = Seq( ProcessAllAvailableIgnoreError, ExpectInStreamSchemaChangeFailure ) } private case object FailSchemaEvolution extends Operation with StreamActionLike { override def toStreamActions: Seq[StreamAction] = Seq( ProcessAllAvailableIgnoreError, ExpectMetadataEvolutionException ) } private trait CheckAnswerStreamActionLike extends Operation with StreamActionLike { override def toStreamActions: Seq[StreamAction] = Seq( ProcessAllAvailable(), // The end state should be the original consecutive rows and then -1s. CheckAnswer( (0 until totalRows) .map( i => Row((0 until currentNumCols) .map(colInd => i + colInd): _*)) ++ (totalRows until totalRows * 2).map(_ => Row(List.fill(currentNumCols)(-1): _*)) : _* ) ) } // Expected to succeed and check the rows in the sink. private case object Success extends CheckAnswerStreamActionLike // Expected to fail with schema tracking enabled. Schema tracking in general puts more limitations // on which operations are permitted during a streaming read. private case object SuccessAndFailSchemaTracking extends CheckAnswerStreamActionLike protected val ExpectMetadataEvolutionException = ExpectFailure[DeltaRuntimeException](e => assert( e.asInstanceOf[DeltaRuntimeException].getErrorClass == "DELTA_STREAMING_METADATA_EVOLUTION" && e.getStackTrace.exists( _.toString.contains("updateMetadataTrackingLogAndFailTheStreamIfNeeded")) ) ) trait StreamActionLike { def toStreamActions: Seq[StreamAction] } private def testTableStreamDf(schemaTrackingLocation: Option[String]) = { var streamReader = spark.readStream.format("delta") schemaTrackingLocation.foreach { loc => streamReader = streamReader .option(DeltaOptions.SCHEMA_TRACKING_LOCATION, loc) } streamReader.table(testTableName) } private def createTable(columnMappingMode: String = "none"): Unit = { sql(s"""CREATE TABLE $testTableName |USING delta |TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = '$columnMappingMode', | '${DeltaConfigs.CHANGE_DATA_FEED.key}' = 'true' |) |AS SELECT id as $firstColumn, id + 1 as $secondColumn, id + 2 as $thirdColumn | FROM RANGE(0, $totalRows, 1, $numFiles) |""".stripMargin) } private def insertMoreRows(v: Int = -1): Unit = { val values = List.fill(currentNumCols)(v.toString).mkString(", ") sql(s"INSERT INTO $testTableName SELECT $values FROM $testTableName LIMIT $totalRows") } private def currentNumCols = deltaLog.update().schema.length override protected def isCdcTest: Boolean = false } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/RemoveColumnMappingSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.columnmapping import io.delta.tables.DeltaTable import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.schema.DeltaInvariantViolationException import org.apache.spark.sql.delta.sources.DeltaSQLConf._ import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.TableIdentifier /** * Test removing column mapping from a table. */ class RemoveColumnMappingSuite extends RemoveColumnMappingSuiteUtils { test("column mapping cannot be removed without the feature flag") { withSQLConf(ALLOW_COLUMN_MAPPING_REMOVAL.key -> "false") { sql(s"""CREATE TABLE $testTableName |USING delta |TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'name') |AS SELECT 1 as a |""".stripMargin) intercept[DeltaColumnMappingUnsupportedException] { sql(s""" |ALTER TABLE $testTableName |SET TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'none') |""".stripMargin) } } } test("table without column mapping enabled") { sql(s"""CREATE TABLE $testTableName |USING delta |TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'none') |AS SELECT 1 as a |""".stripMargin) unsetColumnMappingProperty(useUnset = true) } test("invalid column names") { val invalidColName1 = colName("col1") val invalidColName2 = colName("col2") sql( s"""CREATE TABLE $testTableName (a INT, `$invalidColName1` INT, `$invalidColName2` INT) |USING delta |TBLPROPERTIES ('delta.columnMapping.mode' = 'name') |""".stripMargin) val e = intercept[DeltaAnalysisException] { // Try to remove column mapping. unsetColumnMappingProperty(useUnset = true) } checkError(e, "DELTA_INVALID_COLUMN_NAMES_WHEN_REMOVING_COLUMN_MAPPING", "42K05", Map("invalidColumnNames" -> s"$invalidColName1, $invalidColName2")) } test("ALTER TABLE with multiple table properties") { sql( s"""CREATE TABLE $testTableName (a INT, b INT, c INT) |USING delta |TBLPROPERTIES ('delta.columnMapping.mode' = 'name') |""".stripMargin) // Remove column mapping and set another property. val myProperty = ("acme", "1234") sql(s"ALTER TABLE $testTableName SET TBLPROPERTIES " + s"('delta.columnMapping.mode' = 'none', '${myProperty._1}' = '${myProperty._2}')") val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName = testTableName)) assert(deltaLog.update().metadata.configuration.get(myProperty._1).contains(myProperty._2)) } test("ALTER TABLE UNSET column mapping") { val propertyToKeep = "acme" val propertyToUnset = "acme2" sql( s"""CREATE TABLE $testTableName (a INT, b INT, c INT) |USING delta |TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'name', |'$propertyToKeep' = '1234', '$propertyToUnset' = '1234') |""".stripMargin) sql(s"ALTER TABLE $testTableName UNSET TBLPROPERTIES " + s"('delta.columnMapping.mode', '$propertyToKeep')") val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName = testTableName)) assert(!deltaLog.update() .metadata.configuration.contains(DeltaConfigs.COLUMN_MAPPING_MODE.key)) assert(!deltaLog.update().metadata.configuration.contains(propertyToKeep)) assert(deltaLog.update().metadata.configuration.contains(propertyToUnset)) } test("ALTER TABLE UNSET column mapping with invalid column names") { val invalidColName1 = colName("col1") val invalidColName2 = colName("col2") val propertyToKeep = "acme" val propertyToUnset = "acme2" sql( s"""CREATE TABLE $testTableName (a INT, `$invalidColName1` INT, `$invalidColName2` INT) |USING delta |TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'name', |'$propertyToKeep' = '1234', '$propertyToUnset' = '1234') |""".stripMargin) val e = intercept[DeltaAnalysisException] { // Try to remove column mapping. sql(s"ALTER TABLE $testTableName UNSET TBLPROPERTIES " + s"('delta.columnMapping.mode', '$propertyToKeep')") } checkError(e, "DELTA_INVALID_COLUMN_NAMES_WHEN_REMOVING_COLUMN_MAPPING", "42K05", Map("invalidColumnNames" -> s"$invalidColName1, $invalidColName2")) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName = testTableName)) // Column mapping property should stay the same. assert(deltaLog.update() .metadata.configuration.contains(DeltaConfigs.COLUMN_MAPPING_MODE.key)) // Both other properties should stay the same. assert(deltaLog.update().metadata.configuration.contains(propertyToKeep)) assert(deltaLog.update().metadata.configuration.contains(propertyToUnset)) } test("remove column mapping from a table") { sql( s"""CREATE TABLE $testTableName |USING delta |TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'name') |AS SELECT id as $logicalColumnName, id + 1 as $secondColumn | FROM RANGE(0, $totalRows, 1, $numFiles) |""".stripMargin) testRemovingColumnMapping() } test("remove column mapping using unset") { sql( s"""CREATE TABLE $testTableName |USING delta |TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'name') |AS SELECT id as $logicalColumnName, id + 1 as $secondColumn | FROM RANGE(0, $totalRows, 1, $numFiles) |""".stripMargin) testRemovingColumnMapping(unsetTableProperty = true) } test("remove column mapping from a partitioned table") { sql( s"""CREATE TABLE $testTableName |USING delta |PARTITIONED BY (part) |TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'name') |AS SELECT id as $logicalColumnName, id + 1 as $secondColumn, id % 2 as part | FROM RANGE(0, $totalRows, 1, $numFiles) |""".stripMargin) testRemovingColumnMapping() } test("remove column mapping from a partitioned table with two part columns") { sql( s"""CREATE TABLE $testTableName |USING delta |PARTITIONED BY (part1, part2) |TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'name') |AS SELECT id as $logicalColumnName, id + 1 as $secondColumn, id % 2 as part1, |id % 3 as part2 | FROM RANGE(0, $totalRows, 1, $numFiles) |""".stripMargin) testRemovingColumnMapping() } test("remove column mapping from a table with only logical names") { sql( s"""CREATE TABLE $testTableName |USING delta |TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'none') |AS SELECT id as $logicalColumnName, id + 1 as $secondColumn | FROM RANGE(0, $totalRows, 1, $numFiles) |""".stripMargin) // Add column mapping without renaming any columns. // That is, the column names in the table should be the same as the logical column names. sql( s"""ALTER TABLE $testTableName |SET TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'name', 'delta.minReaderVersion' = '2', 'delta.minWriterVersion' = '5' |)""".stripMargin) testRemovingColumnMapping() } test("dropped column is added back") { sql( s"""CREATE TABLE $testTableName |USING delta |TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'none') |AS SELECT id as $logicalColumnName, id + 1 as $secondColumn | FROM RANGE(0, $totalRows, 1, $numFiles) |""".stripMargin) // Add column mapping without renaming any columns. // That is, the column names in the table should be the same as the logical column names. sql( s"""ALTER TABLE $testTableName |SET TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'name', 'delta.minReaderVersion' = '2', 'delta.minWriterVersion' = '5' |)""".stripMargin) // Drop the second column. sql(s"ALTER TABLE $testTableName DROP COLUMN $secondColumn") // Remove column mapping, this should rewrite the table to physically remove the dropped column. testRemovingColumnMapping() // Add the same column back. sql(s"ALTER TABLE $testTableName ADD COLUMN $secondColumn BIGINT") // Read from the table, ensure none of the original values of secondColumn are present. assert(sql(s"SELECT $secondColumn FROM $testTableName WHERE $secondColumn IS NOT NULL").count() == 0) } test("remove column mapping from a table with deletion vectors") { sql( s"""CREATE TABLE $testTableName |USING delta |TBLPROPERTIES ( | '${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'name', | '${DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key}' = true) |AS SELECT id as $logicalColumnName, id + 1 as $secondColumn | FROM RANGE(0, $totalRows, 1, $numFiles) |""".stripMargin) sql(s"DELETE FROM $testTableName WHERE $logicalColumnName % 2 = 0") testRemovingColumnMapping() } test("remove column mapping from a table with a generated column") { // Note: generate expressions are using logical column names and renaming referenced columns // is forbidden. DeltaTable.create(spark) .tableName(testTableName) .addColumn(logicalColumnName, "LONG") .addColumn( DeltaTable.columnBuilder(secondColumn) .dataType("LONG") .generatedAlwaysAs(s"$logicalColumnName + 1") .build()) .property(DeltaConfigs.COLUMN_MAPPING_MODE.key, "name") .execute() // Insert data into the table. spark.range(totalRows) .selectExpr(s"id as $logicalColumnName") .writeTo(testTableName) .append() val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName = testTableName)) assert(GeneratedColumn.getGeneratedColumns(deltaLog.update()).head.name == secondColumn) testRemovingColumnMapping() // Verify the generated column is still there. assert(GeneratedColumn.getGeneratedColumns(deltaLog.update()).head.name == secondColumn) // Insert more rows. spark.range(totalRows) .selectExpr(s"id + $totalRows as $logicalColumnName") .writeTo(testTableName) .append() // Verify the generated column values are correct. checkAnswer(sql(s"SELECT $logicalColumnName, $secondColumn FROM $testTableName"), (0 until totalRows * 2).map(i => Row(i, i + 1))) } test("column constraints are preserved") { // Note: constraints are using logical column names and renaming is forbidden until // constraint is dropped. sql( s"""CREATE TABLE $testTableName |USING delta |TBLPROPERTIES ( | '${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'name') |AS SELECT id as $logicalColumnName, id + 1 as $secondColumn | FROM RANGE(0, $totalRows, 1, $numFiles) |""".stripMargin) val constraintName = "secondcolumnaddone" val constraintExpr = s"$secondColumn = $logicalColumnName + 1" sql(s"ALTER TABLE $testTableName ADD CONSTRAINT " + s"$constraintName CHECK ($constraintExpr)") val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName = testTableName)) assert(deltaLog.update().metadata.configuration(s"delta.constraints.$constraintName") == constraintExpr) testRemovingColumnMapping() // Verify the constraint is still there. assert(deltaLog.update().metadata.configuration(s"delta.constraints.$constraintName") == constraintExpr) // Verify the constraint is still enforced. intercept[DeltaInvariantViolationException] { sql(s"INSERT INTO $testTableName VALUES (0, 0)") } } test("remove column mapping in id mode") { sql( s"""CREATE TABLE $testTableName |USING delta |TBLPROPERTIES ( | '${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'id') |AS SELECT id as $logicalColumnName, id + 1 as $secondColumn | FROM RANGE(0, $totalRows, 1, $numFiles) |""".stripMargin) testRemovingColumnMapping() } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/columnmapping/RemoveColumnMappingSuiteUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.columnmapping import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.DeltaOperations.RemoveColumnMapping import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.schema.SchemaMergingUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf._ import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import com.fasterxml.jackson.databind.ObjectMapper import org.apache.spark.sql.DataFrame import org.apache.spark.sql.QueryTest import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.col /** * A base trait for testing removing column mapping. * Takes care of setting basic SQL configs and dropping the [[testTableName]] after each test. */ trait RemoveColumnMappingSuiteUtils extends QueryTest with DeltaColumnMappingSuiteUtils { override protected def beforeAll(): Unit = { super.beforeAll() spark.conf.set(ALLOW_COLUMN_MAPPING_REMOVAL.key, "true") } override protected def afterEach(): Unit = { sql(s"DROP TABLE IF EXISTS $testTableName") super.afterEach() } protected val numFiles = 10 protected val totalRows = 100 protected val rowsPerFile = totalRows / numFiles protected val logicalColumnName = "logical_column_name" protected val secondColumn = "second_column_name" protected val firstColumn = "first_column_name" protected val thirdColumn = "third_column_name" protected val renamedThirdColumn = "renamed_third_column_name" protected val testTableName: String = "test_table_" + this.getClass.getSimpleName protected def deltaLog = DeltaLog.forTable(spark, TableIdentifier(testTableName)) import testImplicits._ protected def testRemovingColumnMapping(unsetTableProperty: Boolean = false): Any = { // Verify the input data is as expected. val originalData = spark.table(tableName = testTableName).select(logicalColumnName).collect() // Add a schema comment and verify it is preserved after the rewrite. val comment = "test comment" sql(s"ALTER TABLE $testTableName ALTER COLUMN $logicalColumnName COMMENT '$comment'") val table = DeltaTableV2(spark, TableIdentifier(tableName = testTableName)) val originalSnapshot = table.update() assert(originalSnapshot.schema.head.getComment().get == comment, "Renamed column should preserve comment.") val originalFiles = getFiles(originalSnapshot) val startingVersion = table.update().version unsetColumnMappingProperty(useUnset = unsetTableProperty) verifyRewrite( unsetTableProperty = unsetTableProperty, table, originalFiles, startingVersion, originalData = originalData) // Verify the schema comment is preserved after the rewrite. assert(deltaLog.update().schema.head.getComment().get == comment, "Should preserve the schema comment.") } /** * Verify the table still contains the same data after the rewrite, column mapping is removed * from table properties and the operation recorded properly. */ protected def verifyRewrite( unsetTableProperty: Boolean, table: DeltaTableV2, originalFiles: Array[AddFile], startingVersion: Long, originalData: Array[Row], droppedFeature: Boolean = false): Unit = { checkAnswer( spark.table(tableName = testTableName).select(logicalColumnName), originalData) val newSnapshot = table.update() val versionsAddedByRewrite = if (droppedFeature) { 2 } else { 1 } assert(newSnapshot.version - startingVersion == versionsAddedByRewrite, s"Should rewrite the table in $versionsAddedByRewrite commits.") val rewriteVersion = newSnapshot.version - versionsAddedByRewrite + 1 val history = table.deltaLog.history.getHistory(rewriteVersion, Some(rewriteVersion), table.catalogTable) verifyColumnMappingOperationIsRecordedInHistory(history) assert(newSnapshot.schema.head.name == logicalColumnName, "Should rename the first column.") verifyColumnMappingSchemaMetadataIsRemoved(newSnapshot) verifyColumnMappingTablePropertiesAbsent(newSnapshot, unsetTableProperty || droppedFeature) assert(originalFiles.map(_.numLogicalRecords.get).sum == newSnapshot.allFiles.map(_.numLogicalRecords.get).collect().sum, "Should have the same number of records.") } protected def unsetColumnMappingProperty(useUnset: Boolean): Unit = { val unsetStr = if (useUnset) { s"UNSET TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}')" } else { s"SET TBLPROPERTIES ('${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'none')" } sql( s""" |ALTER TABLE $testTableName $unsetStr |""".stripMargin) } protected def enableColumnMapping(): Unit = { sql( s"""ALTER TABLE $testTableName SET TBLPROPERTIES ( '${DeltaConfigs.COLUMN_MAPPING_MODE.key}' = 'name', 'delta.minReaderVersion' = '2', 'delta.minWriterVersion' = '5')""") } protected def renameColumn(): Unit = { sql(s"ALTER TABLE $testTableName RENAME COLUMN $thirdColumn TO $renamedThirdColumn") } protected def dropColumn(): Unit = { sql(s"ALTER TABLE $testTableName DROP COLUMN $thirdColumn") } /** * Get all files in snapshot. */ protected def getFiles(snapshot: Snapshot): Array[AddFile] = snapshot.allFiles.collect() protected def verifyColumnMappingOperationIsRecordedInHistory(history: Seq[DeltaHistory]) = { val op = RemoveColumnMapping() assert(history.head.operation === op.name) assert(history.head.operationParameters === op.parameters.mapValues(_.toString).toMap) } protected def verifyColumnMappingSchemaMetadataIsRemoved(newSnapshot: Snapshot) = { SchemaMergingUtils.explode(newSnapshot.schema).foreach { case(_, col) => assert(!DeltaColumnMapping.hasPhysicalName(col)) assert(!DeltaColumnMapping.hasColumnId(col)) } } protected def verifyColumnMappingTablePropertiesAbsent( newSnapshot: Snapshot, unsetTablePropertyUsed: Boolean) = { val columnMappingPropertyKey = DeltaConfigs.COLUMN_MAPPING_MODE.key val columnMappingMaxIdPropertyKey = DeltaConfigs.COLUMN_MAPPING_MAX_ID.key val newColumnMappingModeOpt = newSnapshot.metadata.configuration.get(columnMappingPropertyKey) if (unsetTablePropertyUsed) { assert(newColumnMappingModeOpt.isEmpty) } else { assert(newColumnMappingModeOpt.contains("none")) } assert(!newSnapshot.metadata.configuration.contains(columnMappingMaxIdPropertyKey)) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/commands/DeltaCommandInvariantsSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands import scala.util.{Failure, Success, Try} import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.{SparkFunSuite, SparkThrowable} import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.DeltaOperations.EmptyCommit import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.util.JsonUtils class DeltaCommandInvariantsSuite extends SparkFunSuite with DeltaSQLCommandTest { for { shouldSucceed <- BOOLEAN_DOMAIN shouldThrow <- BOOLEAN_DOMAIN } test("command invariant check - " + s"shouldSucceed=$shouldSucceed, shouldThrow=$shouldThrow") { withTempDir { dir => val path = dir.toString spark.range(10).write.format("delta").save(path) val deltaLog = DeltaLog.forTable(spark, path) val opType = "delta.assertions.unreliable.commandInvariantViolated" val events = Log4jUsageLogger.track { val result = Try { withSQLConf( DeltaSQLConf.COMMAND_INVARIANT_CHECKS_THROW.key -> shouldThrow.toString) { // Create an anonymous class, since checkCommandInvariant is protected here. new DeltaCommand { checkCommandInvariant( invariant = () => shouldSucceed, label = "shouldSucceed", op = EmptyCommit, deltaLog = deltaLog, parameters = Map("unused" -> 123), additionalInfo = Map("shouldSucceed" -> shouldSucceed.toString)) } } } if (!shouldSucceed && shouldThrow) { result match { case Failure(e: SparkThrowable) => checkErrorMatchPVals( e, "DELTA_COMMAND_INVARIANT_VIOLATION", parameters = Map( "operation" -> "Empty Commit", "uuid" -> ".*" // Doesn't matter ) ) case Failure(e) => throw e case Success(_) => fail("Expected Failure but got Success") } } else { assert(result.isSuccess) } } val violationEvents = events.filter(_.tags.get("opType").contains(opType)) if (shouldSucceed) { assert(violationEvents.isEmpty) } else { assert(violationEvents.size === 1) val violationEvent = violationEvents.head val violationEventInfo = JsonUtils.fromJson[CommandInvariantCheckInfo](violationEvent.blob) assert(violationEventInfo === CommandInvariantCheckInfo( exceptionThrown = shouldThrow, id = violationEventInfo.id, // Don't check this, it's random. invariantExpression = "shouldSucceed", invariantParameters = Map("unused" -> 123), operation = "Empty Commit", operationParameters = Map.empty, additionalInfo = Map("shouldSucceed" -> shouldSucceed.toString))) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/commands/backfill/RowTrackingBackfillBackfillConflictsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.backfill import java.util.concurrent.ExecutionException import org.apache.spark.sql.delta.{DeltaOperations, RowId} import org.apache.spark.sql.delta.sources.DeltaSQLConf class RowTrackingBackfillBackfillConflictsSuite extends RowTrackingBackfillConflictsTestBase { /** * Concurrent backfill starts after the main backfill enabled the table feature and tries to * commit its only batch and the metadata update after the main backfill is finished. */ test("Two Concurrent backfills") { withTestTable { withTrackedBackfillCommits { // Start main backfill and set table feature. val mainBackfillFuture = launchBackFillAndBlockAfterFeatureIsCommitted() prepareSingleBackfillBatchCommit() // Start concurrent backfill. Table feature is already enabled. val concurrentBackfillFuture = launchBackFillAndBlockAfterFeatureIsCommitted() prepareSingleBackfillBatchCommit() // Finish the main backfill which only requires one commit. commitPreparedBackfillBatchCommit() mainBackfillFuture.get() // Commit the batch of the concurrent backfill. commitPreparedBackfillBatchCommit() // Finish the concurrent backfill. It will commit 0 file. concurrentBackfillFuture.get() // Concurrent backfill does not upgrade the protocol again. assert(deltaLog.history.getHistory(None).count(_.operation == "UPGRADE PROTOCOL") === 1) validateResult(() => tableCreationDF) } } } test("A second backfill after a failed backfill") { withTestTable { withTrackedBackfillCommits { // Start a first backfill and set table feature. val firstBackfillFuture = launchBackFillAndBlockAfterFeatureIsCommitted() prepareSingleBackfillBatchCommit() // Concurrenty update the metadata. val updatedMetadata = latestSnapshot.metadata .copy(configuration = Map("foo" -> "bar")) deltaLog.startTransaction().commit(Seq(updatedMetadata), DeltaOperations.ManualUpdate) // Committing the batch and completing the command will fail because of the metadata update. commitPreparedBackfillBatchCommit() val e = intercept[ExecutionException] { firstBackfillFuture.get() } assertAbortedBecauseOfMetadataChange(e) // We need to remove that expected error or we'll fail below when checking the sink. BackgroundErrorSink.clear() assert(!RowId.isEnabled(latestSnapshot.protocol, latestSnapshot.metadata)) // Launch a second backfill to finish the aborted backfill. val secondBackfillFuture = launchBackFillAndBlockAfterFeatureIsCommitted() commitSingleBackfillBatch() secondBackfillFuture.get() validateResult(() => tableCreationDF) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/commands/backfill/RowTrackingBackfillCloneConflictsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.backfill import org.apache.spark.sql.delta.{MetadataChangedException, RowTracking} import org.apache.spark.sql.delta.fuzzer.{OptimisticTransactionPhases, PhaseLockingTransactionExecutionObserver => TransactionObserver} import org.apache.spark.SparkException import org.apache.spark.util.ThreadUtils /** * Clone uses commitLarge which does not do conflict checking nor retry. So when there is * a concurrent conflict, either Clone will fail from concurrent modification or * Backfill will fail from the metadata change. */ /* * Tests for conflict detection with Backfill and Clone. Note that each backfill in this case has * 2 stages: * - Upgrade the protocol to support the Row Tracking Table Feature * - Mark Row Tracking active by updating the table metadata * * -------------------------------------------> TIME --------------------------------------------> * * Backfill Row Tracking Row Tracking Row Tracking * Command Protocol upgrade -------- metadata update ---------- metadata update * Thread prepare + commit prepare commit * * * Clone * * Scenario 1 prepare ----------------- commit * Scenario 2 prepare -------------------------------------------- commit * * -------------------------------------------> TIME --------------------------------------------> */ class RowTrackingBackfillCloneConflictsSuite extends RowTrackingBackfillConflictsTestBase { override val testTableName = "BackfillCloneTarget" val sourceTableName = "CloneSource" private def withSourceTable(testBlock: => Unit): Unit = { withTable(sourceTableName) { withRowTrackingEnabled(enabled = false) { tableCreationAfterInsert().write.format("delta").saveAsTable(sourceTableName) testBlock } } } private def createUpdateMetadataBackfillObserver(): TransactionObserver = { // This observes the transaction in [[alterDeltaTableCommands]] after the backfill // that tries to update the table's metadata. new TransactionObserver( OptimisticTransactionPhases.forName("update-metadata-backfill-observer")) } test("Backfill fails from conflict with CLONE") { val cloneTransaction = () => sql(s"CREATE OR REPLACE TABLE " + s"$testTableName SHALLOW CLONE $sourceTableName").collect() withTrackedBackfillCommits { withSourceTable { withEmptyTestTable { // Row tracking will be enabled by the backfill. validateSuccessfulBackfillMetrics(expectedNumSuccessfulBatches = 0) { val Seq(backfillFuture) = runFunctionsWithOrderingFromObserver(Seq(backfillTransaction)) { // The upgradeProtocolBackfillObserver observes the transaction in // [[RowTrackingBackfillCommands]] that adds the Row tracking table feature support. case (upgradeProtocolBackfillObserver :: Nil) => val updateMetadataBackfillObserver = createUpdateMetadataBackfillObserver() upgradeProtocolBackfillObserver.setNextObserver( updateMetadataBackfillObserver, autoAdvance = true) // Prepare and commit Row Tracking table feature support transaction. prepareAndCommitWithNextObserverSet(upgradeProtocolBackfillObserver) val Seq(cloneFuture) = runFunctionsWithOrderingFromObserver(Seq(cloneTransaction)) { case (cloneTransactionObserver :: Nil) => // Prepare Clone commit. unblockUntilPreCommit(cloneTransactionObserver) waitForPrecommit(cloneTransactionObserver) // Prepare Row Tracking metadata update commit. unblockUntilPreCommit(updateMetadataBackfillObserver) waitForPrecommit(updateMetadataBackfillObserver) // Commit Clone. unblockCommit(cloneTransactionObserver) waitForCommit(cloneTransactionObserver) // Commit Row Tracking metadata update. unblockCommit(updateMetadataBackfillObserver) waitForCommit(updateMetadataBackfillObserver) } ThreadUtils.awaitResult(cloneFuture, timeout) checkAnswer(spark.table(testTableName), spark.table(sourceTableName)) } val ex = intercept[SparkException] { ThreadUtils.awaitResult(backfillFuture, timeout) } assertAbortedBecauseOfMetadataChange(ex) assert(!RowTracking.isEnabled(latestSnapshot.protocol, latestSnapshot.metadata)) } } } } } test("Clone fails from conflict with Backfill") { val cloneTransaction = () => sql(s"CREATE OR REPLACE TABLE " + s"$testTableName SHALLOW CLONE $sourceTableName").collect() withTrackedBackfillCommits { withSourceTable { withEmptyTestTable { // Row tracking will be enabled by the backfill. validateSuccessfulBackfillMetrics(expectedNumSuccessfulBatches = 0) { val Seq(backfillFuture) = runFunctionsWithOrderingFromObserver(Seq(backfillTransaction)) { // The upgradeProtocolBackfillObserver observes the transaction in // [[RowTrackingBackfillCommands]] that adds the Row tracking table feature support. case (upgradeProtocolBackfillObserver :: Nil) => val updateMetadataBackfillObserver = createUpdateMetadataBackfillObserver() upgradeProtocolBackfillObserver.setNextObserver( updateMetadataBackfillObserver, autoAdvance = true) // Prepare and commit Row Tracking table feature support transaction. prepareAndCommitWithNextObserverSet(upgradeProtocolBackfillObserver) val Seq(cloneFuture) = runFunctionsWithOrderingFromObserver(Seq(cloneTransaction)) { case (cloneTransactionObserver :: Nil) => // Prepare Clone commit. unblockUntilPreCommit(cloneTransactionObserver) waitForPrecommit(cloneTransactionObserver) // Prepare and commit Row Tracking metadata update commit. prepareAndCommit(updateMetadataBackfillObserver) // Commit Clone. unblockCommit(cloneTransactionObserver) waitForCommit(cloneTransactionObserver) } val ex = intercept[SparkException] { ThreadUtils.awaitResult(cloneFuture, timeout) } assertAbortedBecauseOfConcurrentWrite(ex) } ThreadUtils.awaitResult(backfillFuture, timeout) assert(RowTracking.isEnabled(latestSnapshot.protocol, latestSnapshot.metadata)) checkAnswer(spark.table(testTableName), Seq.empty) } } } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/commands/backfill/RowTrackingBackfillConflictsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.commands.backfill import java.util.concurrent.{ConcurrentLinkedDeque, ExecutionException, Future, TimeUnit} import scala.annotation.tailrec import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.DeltaOperations.ManualUpdate import org.apache.spark.sql.delta.actions.{Action, AddFile} import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.concurrency.{PhaseLockingTestMixin, TransactionExecutionTestMixin} import org.apache.spark.sql.delta.fuzzer.{OptimisticTransactionPhases, PhaseLockingTransactionExecutionObserver} import org.apache.spark.sql.delta.fuzzer.AtomicBarrier.State import org.apache.spark.sql.delta.rowid.RowIdTestUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import io.delta.exceptions.MetadataChangedException import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.sql.{DataFrame, Dataset, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.col import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.ThreadUtils /* * Tests for conflict detection with Backfill. Test suites have to override 'concurrentTransaction' * before testing the following scenarios. All scenarios include the execution one transaction * concurrently to a backfill command. Note that each backfill has 3 stages * - Upgrade the protocol to support the Row Tracking Table Feature * - Add baseRowId (done in multiple batches in parallel) * - Mark Row Tracking active by updating the table metadata * * -------------------------------------------> TIME --------------------------------------------> * RT RT RT * Backfill protocol protocol metadata * Command upgrade ---- upgrade -+--------------------------------+- update * Thread prepare commit \ / prepare + commit * \ / * Backfill \ BaseRowId BaseRowId / * Threads - Backfill ---- Backfill-/ * prepare commit * * * Concurrent transaction * * Scenario 1 prepare --- commit * Scenario 2 prepare ----------------- commit * Scenario 3 prepare ------------------------------------------------ commit * Scenario 4 prepare --------------------------------------------------------------------- commit * Scenario 5 prepare ------- commit * Scenario 6 prepare ---------------------- commit * Scenario 7 prepare ------------------------------------------- commit * * -------------------------------------------> TIME --------------------------------------------> */ trait RowTrackingBackfillConflictsTestBase extends RowIdTestUtils with TransactionExecutionTestMixin with PhaseLockingTestMixin with SharedSparkSession { override def sparkConf: SparkConf = super.sparkConf .set(DeltaSQLConf.DELTA_ROW_TRACKING_BACKFILL_ENABLED.key, "true") .set(DeltaSQLConf.FEATURE_ENABLEMENT_CONFLICT_RESOLUTION_ENABLED.key, "true") protected val usePersistentDeletionVectors = false protected val testTableName = "target" protected val colName = "id" protected val partitionColumnName = "partition" protected def tableCreationDF: DataFrame = withPartitionColumn(spark.range(end = numRows).toDF(colName)) protected def insertedRowDF: DataFrame = { val insertedRow = Seq((1337, 1)) spark.createDataFrame(insertedRow) } protected def tableCreationAfterInsert(): Dataset[Row] = { tableCreationDF.union(insertedRowDF) } protected def withPartitionColumn(dataFrame: DataFrame): DataFrame = { dataFrame.withColumn(partitionColumnName, (col(colName) % numPartitions).cast("int")) } protected val numPartitions: Int = 4 private val numFilesPerPartition: Int = 2 private val numRowsPerFile: Int = 10 protected val numFiles: Int = numPartitions * numFilesPerPartition protected val numRows: Int = numFiles * numRowsPerFile protected def deltaLog: DeltaLog = DeltaLog.forTable(spark, TableIdentifier(testTableName)) protected def latestSnapshot: Snapshot = deltaLog.update() protected def backfillTransaction(): Array[Row] = { sql(s"""ALTER TABLE $testTableName |SET TBLPROPERTIES('${DeltaConfigs.ROW_TRACKING_ENABLED.key}' = 'true')""".stripMargin) .collect() } // All observers for backfill threads will be added to the Deque. protected val backfillObservers = new ConcurrentLinkedDeque[PhaseLockingTransactionExecutionObserver] // Collects errors from background threads so busyWaitFor can fail fast with useful messages. object BackgroundErrorSink { private val errors = new ConcurrentLinkedDeque[Throwable]() def recordError(t: Throwable): Unit = errors.addLast(t) def isEmpty: Boolean = errors.isEmpty() def hasErrors: Boolean = !errors.isEmpty() def clear(): Unit = errors.clear() def checkAndThrow(): Unit = { if (hasErrors) { import scala.jdk.CollectionConverters._ val errorList = errors.iterator().asScala.toList val summaries = errorList.map { t => val sw = new java.io.StringWriter() t.printStackTrace(new java.io.PrintWriter(sw)) sw.toString } fail(s"Background thread(s) failed:\n${summaries.mkString("\n---\n")}") } } } // An observer that adds transaction observers to `backfillObservers` for all batches. object TrackingBackfillExecutionObserver extends BackfillExecutionObserver { override def executeBatch[T](f: => T): T = { val observer = new PhaseLockingTransactionExecutionObserver( OptimisticTransactionPhases.forName("backfill-observer")) backfillObservers.addLast(observer) TransactionExecutionObserver.withObserver(observer)(f) } } // Wait for one backfill thread to be ready and unblock it. The thread is chosen at random. protected def commitSingleBackfillBatch(): Unit = { prepareSingleBackfillBatchCommit() commitPreparedBackfillBatchCommit() } // Wait for one backfill thread to be ready and unblock it up until pre commit. The thread will // be kept at the front of 'backfillObservers'. protected def prepareSingleBackfillBatchCommit(): Unit = { var nextBlockedObserver: Option[PhaseLockingTransactionExecutionObserver] = None def foundNextBlockedObserver(): Boolean = { // Fail fast if background thread died. BackgroundErrorSink.checkAndThrow() backfillObservers.forEach { observer => val prepareEntryState = observer.phases.preparePhase.entryBarrier.load() if (nextBlockedObserver.isEmpty && Seq(State.Blocked, State.Requested).contains(prepareEntryState)) { nextBlockedObserver = Some(observer) } } nextBlockedObserver.isDefined } busyWaitFor(foundNextBlockedObserver, timeout) val observerToWaitOn = nextBlockedObserver.get unblockUntilPreCommit(observerToWaitOn) busyWaitForPreparePhase(observerToWaitOn) } // Commit the backfill batch that has been prepared by 'prepareSingleBackfillBatchCommit'. protected def commitPreparedBackfillBatchCommit(): Unit = { require(!backfillObservers.isEmpty) val observer = backfillObservers.removeFirst() require(observer.phases.preparePhase.hasEntered) unblockCommit(observer) waitForCommit(observer) } // Wait for an observer to enter the prepare phase, with fail-fast on background errors. protected def busyWaitForPreparePhase( observer: PhaseLockingTransactionExecutionObserver): Unit = { def hasPrepared(): Boolean = { BackgroundErrorSink.checkAndThrow() observer.phases.preparePhase.hasEntered } busyWaitFor(hasPrepared, timeout) } // Launch the backfill command and wait until the table feature has been committed. protected def launchBackFillAndBlockAfterFeatureIsCommitted(): Future[_] = { val backfillFuture = launchBackfillInBackgroundThread() busyWaitFor(RowTracking.isSupported(latestSnapshot.protocol), timeout) backfillFuture } // Launch the backfill in a separate thread to run in parallel to `concurrentTransaction`. private def launchBackfillInBackgroundThread(): Future[_] = { val threadPool = ThreadUtils.newDaemonSingleThreadExecutor("backfill-thread-pool") val backfillRunnable: Runnable = () => { try { backfillTransaction() } catch { case t: Throwable => BackgroundErrorSink.recordError(t) throw t } } threadPool.submit(backfillRunnable) } protected def withTrackedBackfillCommits(testBlock: => Unit): Unit = { assert(backfillObservers.isEmpty) assert(BackgroundErrorSink.isEmpty) try { BackfillExecutionObserver.withObserver(TrackingBackfillExecutionObserver) { testBlock } } finally { backfillObservers.clear() BackgroundErrorSink.clear() } } protected def withEmptyTestTable(testBlock: => Unit): Unit = { withTable(testTableName) { // Row tracking will be enabled by the backfill. withRowTrackingEnabled(enabled = false) { spark.range(0) .write.format("delta").saveAsTable(testTableName) testBlock } } } /** * Sets explicit insertion times on files to guarantee deterministic ordering for backfill. * Files in partitions 0-1 get early insertion time (batch 1). * Files in partitions 2-3 get later insertion time (batch 2). */ private def setDeterministicInsertionTimes(): Unit = { val log = deltaLog val snapshot = log.update() val allFiles = snapshot.allFiles.collect() // Partitions 0-1 get insertion time 1000, partitions 2-3 get insertion time 2000 val earlyTime = "1000" val lateTime = "2000" val updatedFiles = allFiles.map { file => val partition = file.partitionValues(partitionColumnName).toInt val insertionTime = if (partition < 2) earlyTime else lateTime val existingTags = Option(file.tags).getOrElse(Map.empty[String, String]) val newTags = existingTags + (AddFile.Tags.INSERTION_TIME.name -> insertionTime) file.copy(tags = newTags) } // Replace files with updated versions using ManualUpdate log.startTransaction(None).commit(updatedFiles, ManualUpdate) } protected def withTestTable(testBlock: => Unit): Unit = { withTable(testTableName) { // Row tracking will be enabled by the backfill. withRowTrackingEnabled(enabled = false) { withSQLConf(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> usePersistentDeletionVectors.toString) { tableCreationDF .repartitionByRange(numFilesPerPartition, col(colName)) .write.format("delta").partitionBy(partitionColumnName).saveAsTable(testTableName) // Set explicit insertion times to guarantee deterministic ordering: // - Partitions 0-1 get early insertion time (will be batch 1) // - Partitions 2-3 get later insertion time (will be batch 2) setDeterministicInsertionTimes() } val tableDF = spark.table(testTableName) val tableFiles: Array[AddFile] = latestSnapshot.allFiles.collect() assert(numPartitions === tableDF.select(partitionColumnName).distinct().count()) tableFiles.groupBy(_.partitionValues.get(partitionColumnName)).foreach { case (_, filesInPartition) => assert(filesInPartition.length === numFilesPerPartition) } assert(tableFiles.forall(_.numLogicalRecords.get == numRowsPerFile)) assert(numFiles === tableFiles.length) assert(numRows === tableDF.count()) testBlock } } } protected def validateResult(expectedResult: () => DataFrame): Unit = { assert(RowId.isEnabled(latestSnapshot.protocol, latestSnapshot.metadata)) assertRowIdsAreValid(deltaLog) checkAnswer(spark.table(testTableName), expectedResult()) } private def causedByMetadataUpdate(exception: Throwable): Boolean = { exception match { case _: MetadataChangedException => true case null => false case same if same.getCause == same => false case other => causedByMetadataUpdate(other.getCause) } } protected def assertAbortedBecauseOfMetadataChange(exception: ExecutionException): Unit = assert(causedByMetadataUpdate(exception), s"Unexpected abort: ${exception.getMessage}") protected def assertAbortedBecauseOfMetadataChange(exception: SparkException): Unit = assert(causedByMetadataUpdate(exception.getCause), s"Unexpected abort: ${exception.getMessage}") protected def assertAbortedBecauseOfConcurrentWrite( exception: SparkException): Unit = { @tailrec def causedByConcurrentModification(exception: Throwable): Boolean = { exception match { case _: ConcurrentWriteException => true case null => false case same if same.getCause == same => false case other => causedByConcurrentModification(other.getCause) } } assert(causedByConcurrentModification(exception.getCause), s"Unexpected abort: ${exception.getMessage}") } /** * This is a modification of scenario 5 in [[RowTrackingBackfillConflictsSuite]] with an * extra insert and failure expectations for commitLarge test. */ protected def testScenario5WithCommitLarge( concurrentTransaction: () => Array[Row], expectedResult: DataFrame): Unit = { withTestTable { withTrackedBackfillCommits { // Commit Row Tracking feature. val backfillFuture = launchBackFillAndBlockAfterFeatureIsCommitted() // Add some data to bump the table version. A RESTORE to the current table version is a NOOP // and will not even create the RestoreTableCommand object. We need a dummy commit in order // for a RESTORE to take place. insertedRowDF.write.insertInto(testTableName) assert(latestSnapshot.version > 1) val Seq(concurrentTransactionFuture) = runFunctionsWithOrderingFromObserver(Seq(concurrentTransaction)) { case (concurrentTransactionObserver :: Nil) => // Prepare concurrent transaction. unblockUntilPreCommit(concurrentTransactionObserver) busyWaitForPreparePhase(concurrentTransactionObserver) // Prepare the commit of one backfill batch. prepareSingleBackfillBatchCommit() // Commit concurrent transaction. unblockCommit(concurrentTransactionObserver) waitForCommit(concurrentTransactionObserver) // Try to finish the backfill, which only has one batch. This will fail because the // concurrent txn does a metadata update. commitPreparedBackfillBatchCommit() val e = intercept[ExecutionException] { backfillFuture.get(timeout.toSeconds, TimeUnit.SECONDS) } assertAbortedBecauseOfMetadataChange(e) } ThreadUtils.awaitResult(concurrentTransactionFuture, timeout) checkAnswer(spark.table(testTableName), expectedResult) } } } /** * This is a modification of scenario 6 in [[RowTrackingBackfillConflictsSuite]] with an extra * insert and failure expectations for commitLarge test. */ protected def testScenario6WithCommitLarge(concurrentTransaction: () => Array[Row]): Unit = { withTrackedBackfillCommits { withTestTable { // We enforce the backfill to use two commits such that we can stop the process before // the metadata update commit. withSQLConf(DeltaSQLConf.DELTA_BACKFILL_MAX_NUM_FILES_PER_COMMIT.key -> math.ceil(numFiles / 2.0).toInt.toString) { validateSuccessfulBackfillMetrics(expectedNumSuccessfulBatches = 2) { val backfillFuture = launchBackFillAndBlockAfterFeatureIsCommitted() // Add some data to bump the table version. A RESTORE to the current table version is a // NOOP and will not even create the RestoreTableCommand object. We need a dummy commit // in order for a RESTORE to take place. insertedRowDF.write.insertInto(testTableName) assert(latestSnapshot.version > 1) val Seq(concurrentTransactionFuture) = runFunctionsWithOrderingFromObserver(Seq(concurrentTransaction)) { case (concurrentTransactionObserver :: Nil) => // Prepare concurrent commit. unblockUntilPreCommit(concurrentTransactionObserver) busyWaitForPreparePhase(concurrentTransactionObserver) // Commit one backfill. commitSingleBackfillBatch() // Try to commit concurrent transaction. unblockCommit(concurrentTransactionObserver) waitForCommit(concurrentTransactionObserver) // Finish the backfill command, which has 2 batches. commitSingleBackfillBatch() backfillFuture.get(timeout.toSeconds, TimeUnit.SECONDS) } val e = intercept[SparkException] { ThreadUtils.awaitResult(concurrentTransactionFuture, timeout) } assertAbortedBecauseOfConcurrentWrite(e) validateResult(tableCreationAfterInsert) } } } } } } class RowTrackingBackfillConflictsSuite extends RowTrackingBackfillConflictsTestBase { private def testAllScenarios( concurrentTransactionName: String)( concurrentTransaction: () => Array[Row])( expectedResult: () => DataFrame): Unit = { /** * Scenario 1: Commit concurrent transaction in parallel to the protocol upgrade. */ test(s"$concurrentTransactionName - Scenario 1") { withTestTable { validateSuccessfulBackfillMetrics(expectedNumSuccessfulBatches = 1) { val Seq(concurrentTransactionFuture, backfillFuture) = runFunctionsWithOrderingFromObserver(Seq(concurrentTransaction, backfillTransaction)) { case (concurrentTransactionObserver :: backfillObserver :: Nil) => // Prepare concurrent transaction commit. unblockUntilPreCommit(concurrentTransactionObserver) busyWaitForPreparePhase(concurrentTransactionObserver) // Prepare table feature commit. unblockUntilPreCommit(backfillObserver) busyWaitForPreparePhase(backfillObserver) // Commit concurrent transaction. unblockCommit(concurrentTransactionObserver) waitForCommit(concurrentTransactionObserver) // Commit table feature and unblock further backfill commits. We replace the // txnObserver on the main thread with a NoOp txnObserver, which means the remaining // transactions on the main thread (the parent txn object of backfill and the // metadata update) will not be observed. backfillObserver.setNextObserver( NoOpTransactionExecutionObserver, autoAdvance = true) unblockCommit(backfillObserver) backfillObserver.phases.postCommitPhase.exitBarrier.unblock() waitForCommit(backfillObserver) } ThreadUtils.awaitResult(concurrentTransactionFuture, timeout) ThreadUtils.awaitResult(backfillFuture, timeout) validateResult(expectedResult) } } } /** * Scenario 2: Commit concurrent transaction after enabling Table feature requiring * conflict resolution. */ test(s"$concurrentTransactionName - Scenario 2") { withTrackedBackfillCommits { withTestTable { validateSuccessfulBackfillMetrics(expectedNumSuccessfulBatches = 1) { val Seq(concurrentTransactionFuture) = runFunctionsWithOrderingFromObserver(Seq(concurrentTransaction)) { case (concurrentTransactionObserver :: Nil) => // Prepare concurrent commit. unblockUntilPreCommit(concurrentTransactionObserver) busyWaitForPreparePhase(concurrentTransactionObserver) val backfillFuture = launchBackFillAndBlockAfterFeatureIsCommitted() busyWaitFor(backfillObservers.size() > 0, timeout) // Commit concurrent transaction. unblockCommit(concurrentTransactionObserver) waitForCommit(concurrentTransactionObserver) // Finish the backfill, which only has one batch. commitSingleBackfillBatch() backfillFuture.get(timeout.toSeconds, TimeUnit.SECONDS) } ThreadUtils.awaitResult(concurrentTransactionFuture, timeout) validateResult(expectedResult) } } } } /** * Scenario 3: Prepare the concurrent commit before the table feature is enabled * and commit after at least one backfill committed and before the metadata update commit. * * The concurrent transaction touches partitions 1 and 2 (one from batch 1, one from batch 2). * Partition 3 is not touched, guaranteeing batch 2 always has work to do. */ test(s"$concurrentTransactionName - Scenario 3") { withTrackedBackfillCommits { withTestTable { // Enforce the backfill to use two commits to stop the process after one commit. withSQLConf(DeltaSQLConf.DELTA_BACKFILL_MAX_NUM_FILES_PER_COMMIT.key -> math.ceil(numFiles / 2.0).toInt.toString) { validateSuccessfulBackfillMetrics(expectedNumSuccessfulBatches = 2) { val Seq(concurrentTransactionFuture) = runFunctionsWithOrderingFromObserver(Seq(concurrentTransaction)) { case (concurrentTransactionObserver :: Nil) => // Prepare concurrent commit. unblockUntilPreCommit(concurrentTransactionObserver) busyWaitForPreparePhase(concurrentTransactionObserver) val backfillFuture = launchBackFillAndBlockAfterFeatureIsCommitted() // Commit one backfill batch (processes partitions 0-1). commitSingleBackfillBatch() // Commit concurrent transaction (touches partitions 1 and 2). unblockCommit(concurrentTransactionObserver) waitForCommit(concurrentTransactionObserver) // Commit second backfill batch (processes partitions 2-3). // Partition 3 is untouched, so there's always work for batch 2. commitSingleBackfillBatch() backfillFuture.get(timeout.toSeconds, TimeUnit.SECONDS) } ThreadUtils.awaitResult(concurrentTransactionFuture, timeout) validateResult(expectedResult) } } } } } /** * Scenario 4: The concurrent operations starts before the backfill and ends after it. */ test(s"$concurrentTransactionName - Scenario 4") { withTestTable { validateSuccessfulBackfillMetrics(expectedNumSuccessfulBatches = 1) { val concurrentTransactionFuture = runTxnsWithOrder__A_Start__B__A_end_without_observer_on_B( concurrentTransaction, backfillTransaction) ThreadUtils.awaitResult(concurrentTransactionFuture, timeout) validateResult(expectedResult) } } } /** * Scenario 5: The concurrent transaction commits after the table feature has been enabled * and concurrently to one backfill batch, requiring conflict resolution on the backfill. */ test(s"$concurrentTransactionName - Scenario 5") { withTestTable { withTrackedBackfillCommits { validateSuccessfulBackfillMetrics(expectedNumSuccessfulBatches = 1) { val Seq(concurrentTransactionFuture) = runFunctionsWithOrderingFromObserver(Seq(concurrentTransaction)) { case (concurrentTransactionObserver :: Nil) => // Commit Row Tracking feature. val backfillFuture = launchBackFillAndBlockAfterFeatureIsCommitted() // Prepare concurrent transaction. unblockUntilPreCommit(concurrentTransactionObserver) busyWaitForPreparePhase(concurrentTransactionObserver) // Prepare the commit of one backfill batch. prepareSingleBackfillBatchCommit() // Commit concurrent transaction. unblockCommit(concurrentTransactionObserver) waitForCommit(concurrentTransactionObserver) // Finish the backfill, which only has one batch. commitPreparedBackfillBatchCommit() ThreadUtils.awaitResult(backfillFuture, timeout) } ThreadUtils.awaitResult(concurrentTransactionFuture, timeout) validateResult(expectedResult) } } } } /** * Scenario 6: The concurrent transaction starts after the table feature is enabled and commits * after one backfill has been committed concurrently, requiring conflict resolution on the * concurrent transaction. * * The concurrent transaction touches partitions 1 and 2 (one from batch 1, one from batch 2). * Partition 3 is not touched, guaranteeing batch 2 always has work to do. */ test(s"$concurrentTransactionName - Scenario 6") { withTrackedBackfillCommits { withTestTable { // We enforce the backfill to use two commits such that we can stop the process before // the metadata update commit. withSQLConf(DeltaSQLConf.DELTA_BACKFILL_MAX_NUM_FILES_PER_COMMIT.key -> math.ceil(numFiles / 2.0).toInt.toString) { validateSuccessfulBackfillMetrics(expectedNumSuccessfulBatches = 2) { val Seq(concurrentTransactionFuture) = runFunctionsWithOrderingFromObserver(Seq(concurrentTransaction)) { case (concurrentTransactionObserver :: Nil) => val backfillFuture = launchBackFillAndBlockAfterFeatureIsCommitted() // Prepare concurrent commit. unblockUntilPreCommit(concurrentTransactionObserver) busyWaitForPreparePhase(concurrentTransactionObserver) // Commit one backfill batch (processes partitions 0-1). commitSingleBackfillBatch() // Commit concurrent transaction (touches partitions 1 and 2). unblockCommit(concurrentTransactionObserver) waitForCommit(concurrentTransactionObserver) // Commit second backfill batch (processes partitions 2-3). // Partition 3 is untouched, so there's always work for batch 2. commitSingleBackfillBatch() ThreadUtils.awaitResult(backfillFuture, timeout) } ThreadUtils.awaitResult(concurrentTransactionFuture, timeout) validateResult(expectedResult) } } } } } /** * Scenario 7: The concurrent transaction starts after the table feature is enabled and * commits after the metadata update. */ test(s"$concurrentTransactionName - Scenario 7") { withTrackedBackfillCommits { withTestTable { validateSuccessfulBackfillMetrics(expectedNumSuccessfulBatches = 1) { val Seq(concurrentTransactionFuture) = runFunctionsWithOrderingFromObserver(Seq(concurrentTransaction)) { case (concurrentTransactionObserver :: Nil) => val backfillFuture = launchBackFillAndBlockAfterFeatureIsCommitted() // Prepare concurrent commit. unblockUntilPreCommit(concurrentTransactionObserver) busyWaitForPreparePhase(concurrentTransactionObserver) // Finish the backfill, which only has one batch. commitSingleBackfillBatch() backfillFuture.get(timeout.toSeconds, TimeUnit.SECONDS) // Unlock commit of concurrent transaction. unblockCommit(concurrentTransactionObserver) waitForCommit(concurrentTransactionObserver) } ThreadUtils.awaitResult(concurrentTransactionFuture, timeout) validateResult(expectedResult) } } } } } testAllScenarios("INSERT") { () => sql(s"INSERT INTO $testTableName($colName, $partitionColumnName) VALUES(1337, 1)").collect() } { () => val insertedRow = Seq((1337, 1)) tableCreationDF.union(spark.createDataFrame(insertedRow)) } // DELETE touches partitions 1 and 2 (id % 4 == 1 or 2) testAllScenarios("DELETE") { () => sql(s"DELETE FROM $testTableName WHERE $colName IN (1, 2)").collect() } { () => assert(!usePersistentDeletionVectors || !DeletionVectorUtils.isTableDVFree(latestSnapshot)) tableCreationDF.where(s"$colName NOT IN (1, 2)") } // UPDATE touches partitions 1 and 2 (id % 4 == 1 or 2) testAllScenarios("UPDATE") { () => sql(s"UPDATE $testTableName SET $colName = $colName + 1000 WHERE $colName IN (1, 2)").collect() } { () => assert( !usePersistentDeletionVectors || !DeletionVectorUtils.isTableDVFree(latestSnapshot) ) // id=1 becomes 1001 (partition 1), id=2 becomes 1002 (partition 2) val updatedRows = Seq((1001, 1), (1002, 2)) tableCreationDF.where("id NOT IN (1, 2)").union(spark.createDataFrame(updatedRows)) } // DF to create the view used as a source for MERGEs. When joining on 'colName', the lower half // of the rows in 'testTableName' is unmatched, the upper half of 'testTableName' is matched // by the lower half of 'mergeSourceDF', and the upper half of 'mergeSourceDF' is unmatched. private lazy val mergeSourceDF: DataFrame = withPartitionColumn(tableCreationDF.select(col(colName) + (numRows / 2) as colName)) // Create a temporary view used as the source for MERGEs based on 'mergeSourceDF'. private def withMergeSource(testBlock: String => Array[Row]): Array[Row] = { val sourceViewName = "source" mergeSourceDF.createTempView(sourceViewName) try { testBlock(sourceViewName) } finally { sql(s"DROP VIEW $sourceViewName") } } // MERGE touches partitions 1 and 2 only via partition conditions on WHEN clauses testAllScenarios("MERGE with not matched and not matched by source") { () => withMergeSource { sourceViewName => val mergeStatement = s"""MERGE INTO $testTableName t |USING $sourceViewName s |ON s.$colName = t.$colName |WHEN NOT MATCHED AND s.$partitionColumnName IN (1, 2) THEN INSERT * |WHEN NOT MATCHED BY SOURCE AND t.$partitionColumnName IN (1, 2) THEN DELETE |""".stripMargin sql(mergeStatement).collect() } } { () => // Target has ids [0, numRows), source has ids [numRows/2, numRows + numRows/2) // per mergeSourceDF definition. // MATCHED: [numRows/2, numRows) // NOT MATCHED BY SOURCE: [0, numRows/2) // NOT MATCHED: [numRows, ...) // With partition filter, we DELETE [0, numRows/2) in partitions 1,2 and // INSERT [numRows, ...) in partitions 1,2. // targetRowsToKeep: outside partitions 1,2 (untouched) OR id >= numRows/2 // (matched, not deleted). val targetRowsToKeep = tableCreationDF.where( s"$partitionColumnName NOT IN (1, 2) OR $colName >= ${numRows / 2}") val sourceRowsToInsert = mergeSourceDF.where( s"$partitionColumnName IN (1, 2) AND $colName >= $numRows") targetRowsToKeep.union(sourceRowsToInsert) } // MERGE touches partitions 1 and 2 only via partition conditions on WHEN clauses testAllScenarios("MERGE with matched and not matched") { () => withMergeSource { sourceViewName => val mergeStatement = s"""MERGE INTO $testTableName t |USING $sourceViewName s |ON s.$colName = t.$colName |WHEN MATCHED AND t.$partitionColumnName IN (1, 2) THEN UPDATE SET * |WHEN NOT MATCHED AND s.$partitionColumnName IN (1, 2) THEN INSERT * |""".stripMargin sql(mergeStatement).collect() } } { () => // Target has ids [0, numRows), source has ids [numRows/2, numRows + numRows/2) // per mergeSourceDF definition. // MATCHED: [numRows/2, numRows) // NOT MATCHED: [numRows, ...) // With partition filter, we UPDATE [numRows/2, numRows) in partitions 1,2 and // INSERT [numRows, ...) in partitions 1,2. // targetRowsNotUpdated: outside partitions 1,2 (untouched) OR id < numRows/2 (unmatched). val targetRowsNotUpdated = tableCreationDF.where( s"$partitionColumnName NOT IN (1, 2) OR $colName < ${numRows / 2}") val updatedRows = mergeSourceDF.where( s"$partitionColumnName IN (1, 2) AND $colName < $numRows") val insertedRows = mergeSourceDF.where( s"$partitionColumnName IN (1, 2) AND $colName >= $numRows") targetRowsNotUpdated.union(updatedRows).union(insertedRows) } // OPTIMIZE touches partitions 1 and 2 only testAllScenarios("OPTIMIZE") { () => sql(s"OPTIMIZE $testTableName WHERE $partitionColumnName IN (1, 2)").collect() } { () => tableCreationDF } /** * RESTORE uses commitLarge which does not do conflict checking nor retry. So when there is * a concurrent conflict, either RESTORE will fail from concurrent modification or * Backfill will fail from the metadata change. */ test("Backfill fails from conflict with RESTORE") { // This tries to undo the insert. val concurrentTransaction = () => { sql(s"RESTORE TABLE $testTableName TO VERSION AS OF 1").collect() } testScenario5WithCommitLarge(concurrentTransaction, tableCreationDF) } test("RESTORE fails from conflict with Backfill") { // This tries to undo the insert. val concurrentTransaction = () => { sql(s"RESTORE TABLE $testTableName TO VERSION AS OF 1").collect() } testScenario6WithCommitLarge(concurrentTransaction) } } class RowTrackingBackfillConflictsDVSuite extends RowTrackingBackfillConflictsSuite { override val usePersistentDeletionVectors = true } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/concurrency/PhaseLockingTestMixin.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.concurrency import scala.concurrent.duration._ import org.apache.spark.sql.delta.ConcurrencyHelpers import org.apache.spark.sql.delta.fuzzer.AtomicBarrier import org.apache.spark.SparkFunSuite trait PhaseLockingTestMixin { self: SparkFunSuite => /** Keep checking if `barrier` in `state` until it's the case or `waitTime` expires. */ def busyWaitForState( barrier: AtomicBarrier, state: AtomicBarrier.State, waitTime: FiniteDuration): Unit = busyWaitFor( barrier.load() == state, waitTime, s"Exceeded deadline waiting for $barrier to transition to state $state") /** * Keep checking if `check` return `true` until it's the case or `waitTime` expires. * * Optionally provide a custom error `message`. */ def busyWaitFor( check: => Boolean, timeout: FiniteDuration, // lazy evaluate so closed over states are evaluated at time of failure not invocation message: => String = "Exceeded deadline waiting for check to become true."): Unit = { if (!ConcurrencyHelpers.busyWaitFor(check, timeout)) { fail(message) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/concurrency/TransactionExecutionObserverSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.concurrency import scala.concurrent.duration._ import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.fuzzer.{AtomicBarrier, IllegalStateTransitionException, OptimisticTransactionPhases, PhaseLockingTransactionExecutionObserver} import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import io.delta.tables.{DeltaTable => IODeltaTable} import org.apache.spark.sql.QueryTest import org.apache.spark.sql.test.SharedSparkSession /** * Check that [[TransactionExecutionObserver]] is invoked correctly by transactions * and commands. * * Also check the testing tools that use this API. */ class TransactionExecutionObserverSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with PhaseLockingTestMixin with TransactionExecutionTestMixin { override val timeout: FiniteDuration = 10000.millis test("Phase Locking - sequential") { withTempDir { tempFile => val tempPath = tempFile.toString spark.range(100).write.format("delta").save(tempPath) val observer = new PhaseLockingTransactionExecutionObserver( OptimisticTransactionPhases.forName("test-txn")) val deltaLog = DeltaLog.forTable(spark, tempPath) // get things started observer.phases.initialPhase.entryBarrier.unblock() assert(!observer.phases.initialPhase.hasEntered) TransactionExecutionObserver.withObserver(observer) { deltaLog.withNewTransaction { txn => assert(observer.phases.initialPhase.hasEntered) assert(observer.phases.initialPhase.hasLeft) assert(!observer.phases.preparePhase.hasEntered) assert(!observer.phases.commitPhase.hasEntered) assert(!observer.phases.backfillPhase.hasEntered) assert(!observer.phases.postCommitPhase.hasEntered) // allow things to progress observer.phases.preparePhase.entryBarrier.unblock() observer.phases.commitPhase.entryBarrier.unblock() observer.phases.backfillPhase.entryBarrier.unblock() observer.phases.postCommitPhase.entryBarrier.unblock() val removedFiles = txn.snapshot.allFiles.collect().map(_.remove).toSeq txn.commit(removedFiles, DeltaOperations.ManualUpdate) assert(observer.phases.preparePhase.hasEntered) assert(observer.phases.preparePhase.hasLeft) assert(observer.phases.commitPhase.hasEntered) assert(observer.phases.commitPhase.hasLeft) assert(observer.phases.backfillPhase.hasEntered) assert(observer.phases.backfillPhase.hasLeft) assert(observer.phases.postCommitPhase.hasEntered) assert(observer.phases.postCommitPhase.hasLeft) } } val res = spark.read.format("delta").load(tempPath).collect() assert(res.isEmpty) } } test("Phase Locking - parallel") { withTempDir { tempFile => val tempPath = tempFile.toString spark.range(100).write.format("delta").save(tempPath) val observer = new PhaseLockingTransactionExecutionObserver( OptimisticTransactionPhases.forName("test-txn")) val deltaLog = DeltaLog.forTable(spark, tempPath) val testThread = new Thread(() => { // make sure the transaction will use our observer TransactionExecutionObserver.withObserver(observer) { deltaLog.withNewTransaction { txn => val removedFiles = txn.snapshot.allFiles.collect().map(_.remove).toSeq txn.commit(removedFiles, DeltaOperations.ManualUpdate) } } }) testThread.start() busyWaitForState( observer.phases.initialPhase.entryBarrier, AtomicBarrier.State.Requested, timeout) // get things started observer.phases.initialPhase.entryBarrier.unblock() busyWaitFor(observer.phases.initialPhase.hasEntered, timeout) busyWaitFor(observer.phases.initialPhase.hasLeft, timeout) assert(!observer.phases.preparePhase.hasEntered) observer.phases.preparePhase.entryBarrier.unblock() busyWaitFor(observer.phases.preparePhase.hasEntered, timeout) busyWaitFor(observer.phases.preparePhase.hasLeft, timeout) assert(!observer.phases.commitPhase.hasEntered) observer.phases.commitPhase.entryBarrier.unblock() busyWaitFor(observer.phases.commitPhase.hasEntered, timeout) busyWaitFor(observer.phases.commitPhase.hasLeft, timeout) observer.phases.backfillPhase.entryBarrier.unblock() busyWaitFor(observer.phases.backfillPhase.hasEntered, timeout) busyWaitFor(observer.phases.backfillPhase.hasLeft, timeout) observer.phases.postCommitPhase.entryBarrier.unblock() busyWaitFor(observer.phases.postCommitPhase.hasEntered, timeout) busyWaitFor(observer.phases.postCommitPhase.hasLeft, timeout) testThread.join(timeout.toMillis) assert(!testThread.isAlive) // should have passed the barrier and completed val res = spark.read.format("delta").load(tempPath).collect() assert(res.isEmpty) } } test("Phase Locking - no reusing observer") { withTempDir { tempFile => val tempPath = tempFile.toString spark.range(100).write.format("delta").save(tempPath) val observer = new PhaseLockingTransactionExecutionObserver( OptimisticTransactionPhases.forName("test-txn")) val deltaLog = DeltaLog.forTable(spark, tempPath) // get things started observer.phases.initialPhase.entryBarrier.unblock() assert(!observer.phases.initialPhase.hasEntered) TransactionExecutionObserver.withObserver(observer) { deltaLog.withNewTransaction { txn => // allow things to progress observer.phases.preparePhase.entryBarrier.unblock() observer.phases.commitPhase.entryBarrier.unblock() observer.phases.backfillPhase.entryBarrier.unblock() observer.phases.postCommitPhase.entryBarrier.unblock() val removedFiles = txn.snapshot.allFiles.collect().map(_.remove).toSeq txn.commit(removedFiles, DeltaOperations.ManualUpdate) } // Check that we fail trying to re-unblock the barrier assertThrows[IllegalStateTransitionException] { deltaLog.withNewTransaction { txn => // allow things to progress observer.phases.preparePhase.entryBarrier.unblock() observer.phases.commitPhase.entryBarrier.unblock() observer.phases.backfillPhase.entryBarrier.unblock() observer.phases.postCommitPhase.entryBarrier.unblock() val removedFiles = txn.snapshot.allFiles.collect().map(_.remove).toSeq txn.commit(removedFiles, DeltaOperations.ManualUpdate) } } // Check that we fail just waiting on the passed barrier assertThrows[IllegalStateTransitionException] { deltaLog.withNewTransaction { txn => val removedFiles = txn.snapshot.allFiles.collect().map(_.remove).toSeq txn.commit(removedFiles, DeltaOperations.ManualUpdate) } } } val res = spark.read.format("delta").load(tempPath).collect() assert(res.isEmpty) } } test("Phase Locking - delete command") { withTempDir { tempFile => val tempPath = tempFile.toString spark.range(100).write.format("delta").save(tempPath) val observer = new PhaseLockingTransactionExecutionObserver( OptimisticTransactionPhases.forName("test-txn")) val deltaLog = DeltaLog.forTable(spark, tempPath) val deltaTable = IODeltaTable.forPath(spark, tempPath) def assertOperationNotVisible(): Unit = assert(deltaTable.toDF.count() === 100) val testThread = new Thread(() => { // make sure the transaction will use our observer TransactionExecutionObserver.withObserver(observer) { deltaTable.delete() } }) testThread.start() busyWaitForState( observer.phases.initialPhase.entryBarrier, AtomicBarrier.State.Requested, timeout) assertOperationNotVisible() // get things started observer.phases.initialPhase.entryBarrier.unblock() busyWaitFor(observer.phases.initialPhase.hasLeft, timeout) assertOperationNotVisible() observer.phases.preparePhase.entryBarrier.unblock() busyWaitFor(observer.phases.preparePhase.hasLeft, timeout) assert(!observer.phases.commitPhase.hasEntered) assert(!observer.phases.backfillPhase.hasEntered) assert(!observer.phases.postCommitPhase.hasEntered) assertOperationNotVisible() observer.phases.commitPhase.entryBarrier.unblock() busyWaitFor(observer.phases.commitPhase.hasLeft, timeout) observer.phases.backfillPhase.entryBarrier.unblock() busyWaitFor(observer.phases.backfillPhase.hasLeft, timeout) observer.phases.postCommitPhase.entryBarrier.unblock() busyWaitFor(observer.phases.postCommitPhase.hasLeft, timeout) testThread.join(timeout.toMillis) assert(!testThread.isAlive) // should have passed the barrier and completed val res = spark.read.format("delta").load(tempPath).collect() assert(res.isEmpty) } } test("Phase Locking - set next observer after commit") { withTempDir { tempFile => val tempPath = tempFile.toString spark.range(end = 1).write.format("delta").save(tempPath) val observer = new PhaseLockingTransactionExecutionObserver( OptimisticTransactionPhases.forName("test-txn")) val deltaLog = DeltaLog.forTable(spark, tempPath) val initialTableVersion = deltaLog.update().version // get things started val replacementObserver = new PhaseLockingTransactionExecutionObserver( OptimisticTransactionPhases.forName("test-replacement-txn")) observer.setNextObserver(replacementObserver, autoAdvance = true) unblockAllPhases(observer) TransactionExecutionObserver.withObserver(observer) { deltaLog.withNewTransaction { txn => observer.phases.postCommitPhase.exitBarrier.unblock() val removedFiles = txn.snapshot.allFiles.collect().map(_.remove).toSeq txn.commit(removedFiles, DeltaOperations.ManualUpdate) } val tableVersionAfterFirstTxn = deltaLog.update().version assert(tableVersionAfterFirstTxn === initialTableVersion + 1, "expected a successful commit") // Check that we cannot re-use the old observer, with unblocks. assertThrows[IllegalStateTransitionException] { observer.phases.preparePhase.entryBarrier.unblock() } // Check that we can use the replaced observer to control a subsequent commit on the same // thread. val oldMetadata = deltaLog.update().metadata val newMetadata = oldMetadata.copy(configuration = Map("foo" -> "bar")) unblockAllPhases(replacementObserver) deltaLog.withNewTransaction { txn => txn.commit(Seq(newMetadata), DeltaOperations.ManualUpdate) } assert(deltaLog.update().version === tableVersionAfterFirstTxn + 1, "expected a successful commit") assert(replacementObserver.allPhasesHavePassed) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/concurrency/TransactionExecutionTestMixin.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.concurrency import java.util.concurrent.{ExecutorService, TimeUnit} import scala.concurrent.{ExecutionContext, Future} import scala.concurrent.duration._ import com.databricks.spark.util.{Log4jUsageLogger, UsageRecord} import org.apache.spark.sql.delta.{ConcurrencyHelpers, OptimisticTransaction, TransactionExecutionObserver} import org.apache.spark.sql.delta.fuzzer.{OptimisticTransactionPhases, PhaseLockingTransactionExecutionObserver => TransactionObserver} import org.apache.spark.SparkException import org.apache.spark.internal.Logging import org.apache.spark.sql.Row import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.ThreadUtils trait TransactionExecutionTestMixin { self: PhaseLockingTestMixin with SharedSparkSession with Logging => /** * Timeout used when waiting for individual phases of instrumented operations to complete. */ val timeout: FiniteDuration = 120.seconds /** Run a given function `fn` inside the given `executor` within a [[TransactionObserver]] */ private[delta] def runFunctionWithObserver( name: String, executorService: ExecutorService, fn: () => Array[Row]): (TransactionObserver, Future[Array[Row]]) = { val observer = new TransactionObserver(OptimisticTransactionPhases.forName(s"observer-txn-$name")) implicit val ec = ExecutionContext.fromExecutorService(executorService) val txn = OptimisticTransaction.getActive() val future = Future { ConcurrencyHelpers.withOptimisticTransaction(txn) { spark.withActive( try { TransactionExecutionObserver.withObserver(observer) { fn() } } catch { case ex: Exception => logError(s"Error on test thread", ex) throw ex } ) } } (observer, future) } /** Run a given `queryString` inside the given `executor` */ def runQueryWithObserver(name: String, executor: ExecutorService, queryString: String) : (TransactionObserver, Future[Array[Row]]) = { def fn(): Array[Row] = spark.sql(queryString).collect() runFunctionWithObserver(name, executor, fn) } /** * Run `functions` with the ordering defined by `observerOrdering` function. * This function returns futures for each of the query results. */ private[delta] def runFunctionsWithOrderingFromObserver (functions: Seq[() => Array[Row]]) (observerOrdering: (Seq[TransactionObserver]) => Unit) : Seq[Future[Array[Row]]] = { val executors = functions.zipWithIndex.map { case (_, index) => ThreadUtils.newDaemonSingleThreadExecutor(threadName = s"executor-txn-$index") } try { val (observers, futures) = functions.zipWithIndex.map { case (fn, index) => runFunctionWithObserver(name = s"query-$index", executors(index), fn) }.unzip // Run the observer ordering function. observerOrdering(observers) // wait for futures to succeed or fail for (future <- futures) { try { ThreadUtils.awaitResult(future, timeout) } catch { case _: SparkException => // pass true } } futures } finally { for (executor <- executors) { executor.shutdownNow() executor.awaitTermination(timeout.toMillis, TimeUnit.MILLISECONDS) } } } /** Unblocks all phases before the `commitPhase` for [[TransactionObserver]] */ def unblockUntilPreCommit(observer: TransactionObserver): Unit = { observer.phases.initialPhase.entryBarrier.unblock() observer.phases.preparePhase.entryBarrier.unblock() } /** * Unblocks the `commitPhase` and `backfillPhase` for [[TransactionObserver]]. */ def unblockCommit(observer: TransactionObserver): Unit = { observer.phases.commitPhase.entryBarrier.unblock() observer.phases.backfillPhase.entryBarrier.unblock() observer.phases.postCommitPhase.entryBarrier.unblock() } /** Unblocks all phases for [[TransactionObserver]] so that corresponding query can finish. */ def unblockAllPhases(observer: TransactionObserver): Unit = { observer.phases.initialPhase.entryBarrier.unblock() observer.phases.preparePhase.entryBarrier.unblock() observer.phases.commitPhase.entryBarrier.unblock() observer.phases.backfillPhase.entryBarrier.unblock() observer.phases.postCommitPhase.entryBarrier.unblock() } def waitForPrecommit(observer: TransactionObserver): Unit = busyWaitFor(observer.phases.preparePhase.hasEntered, timeout) def waitForCommit(observer: TransactionObserver): Unit = { busyWaitFor(observer.phases.commitPhase.hasLeft, timeout) busyWaitFor(observer.phases.backfillPhase.hasLeft, timeout) busyWaitFor(observer.phases.postCommitPhase.hasLeft, timeout) } /** * Prepare and commit the transaction managed by the given observer. * If nextObserver is set, we need to manually call backfillPhase.leave() to advance to the * nextObserver. Details in [[TransactionObserver.waitForCommitPhaseAndAdvanceToNextObserver]]. */ private def prepareAndCommitBase( observer: TransactionObserver, hasNextObserver: Boolean): Unit = { unblockUntilPreCommit(observer) waitForPrecommit(observer) unblockCommit(observer) if (hasNextObserver) { observer.phases.postCommitPhase.leave() } waitForCommit(observer) } /** * Prepare and commit the transaction managed by the given observer. */ def prepareAndCommit(observer: TransactionObserver): Unit = { prepareAndCommitBase(observer, hasNextObserver = false) } /** * Prepare and commit the transaction managed by the given observer which has nextObserver set. */ def prepareAndCommitWithNextObserverSet(observer: TransactionObserver): Unit = { prepareAndCommitBase(observer, hasNextObserver = true) } /** * Run 2 transactions A, B with following order: * * t1 -------------------------------------- TxnA starts * t2 --------- TxnB starts and commits (no transaction observer) * t6 -------------------------------------- TxnA commits * * This function returns futures for each of the query runs. */ def runTxnsWithOrder__A_Start__B__A_end_without_observer_on_B( txnA: () => Array[Row], txnB: () => Array[Row]): Future[Array[Row]] = { val Seq(futureA) = runFunctionsWithOrderingFromObserver(Seq(txnA)) { case (observerA :: Nil) => // A starts unblockUntilPreCommit(observerA) busyWaitFor(observerA.phases.preparePhase.hasEntered, timeout) // B starts and finishes txnB() // A commits unblockCommit(observerA) waitForCommit(observerA) } futureA } /** * Run 2 transactions A, B with following order: * * t1 -------------------------------------- TxnA starts * t2 --------- TxnB starts * t3 --------- TxnB commits * t6 -------------------------------------- TxnA commits * * This function returns futures for each of the query runs. */ def runTxnsWithOrder__A_Start__B__A_End(txnA: () => Array[Row], txnB: () => Array[Row]) : (Future[Array[Row]], Future[Array[Row]]) = { val Seq(futureA, futureB) = runFunctionsWithOrderingFromObserver(Seq(txnA, txnB)) { case (observerA :: observerB :: Nil) => // A starts unblockUntilPreCommit(observerA) busyWaitFor(observerA.phases.preparePhase.hasEntered, timeout) // B starts and commits unblockAllPhases(observerB) busyWaitFor(observerB.phases.postCommitPhase.hasLeft, timeout) // A commits observerA.phases.commitPhase.entryBarrier.unblock() busyWaitFor(observerA.phases.commitPhase.hasLeft, timeout) observerA.phases.backfillPhase.entryBarrier.unblock() busyWaitFor(observerA.phases.backfillPhase.hasLeft, timeout) observerA.phases.postCommitPhase.entryBarrier.unblock() busyWaitFor(observerA.phases.postCommitPhase.hasLeft, timeout) } (futureA, futureB) } /** * Run 3 queries A, B, C with following order: * * t1 -------------------------------------- TxnA starts * t2 --------- TxnB starts * t3 --------- TxnB commits * t4 ----------------- TxnC starts * t5 ----------------- TxnC commits * t6 -------------------------------------- TxnA commits * * This function returns futures for each of the query runs. */ def runTxnsWithOrder__A_Start__B__C__A_End( txnA: () => Array[Row], txnB: () => Array[Row], txnC: () => Array[Row]) : (Future[Array[Row]], Future[Array[Row]], Future[Array[Row]]) = { val Seq(futureA, futureB, futureC) = runFunctionsWithOrderingFromObserver(Seq(txnA, txnB, txnC)) { case (observerA :: observerB :: observerC :: Nil) => // A starts unblockUntilPreCommit(observerA) busyWaitFor(observerA.phases.preparePhase.hasEntered, timeout) // B starts and commits unblockAllPhases(observerB) busyWaitFor(observerB.phases.postCommitPhase.hasLeft, timeout) // C starts and commits unblockAllPhases(observerC) busyWaitFor(observerC.phases.postCommitPhase.hasLeft, timeout) // A commits observerA.phases.commitPhase.entryBarrier.unblock() busyWaitFor(observerA.phases.commitPhase.hasLeft, timeout) observerA.phases.backfillPhase.entryBarrier.unblock() busyWaitFor(observerA.phases.backfillPhase.hasLeft, timeout) observerA.phases.postCommitPhase.entryBarrier.unblock() busyWaitFor(observerA.phases.postCommitPhase.hasLeft, timeout) } (futureA, futureB, futureC) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CatalogManagedStreamingSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.test.shims.StreamingTestShims.MemoryStream import org.apache.spark.SparkConf import org.apache.spark.sql.streaming.StreamTest /** * General note on streaming tests: by default, a streaming query uses the ProcessingTime trigger * with a 0-second interval, so the query will be executed as fast as possible, and the read path * will be triggered periodically. Because of this, things like asserting the number of getCommits * on the tracking client will not be deterministic. */ trait CatalogManagedStreamingSuiteBase extends StreamTest with DeltaSQLTestUtils with DeltaSQLCommandTest with CatalogOwnedTestBaseSuite { import testImplicits._ import org.apache.spark.sql.delta.test.DeltaTestImplicits._ protected def assertNumCommitsCalled(expectedNumCommits: Int): Unit = { getTrackingClient.foreach { trackingClient => assert(trackingClient.numCommitsCalled.get === expectedNumCommits) } } protected def assertNumGetCommitsCalled(expectedNumGetCommits: Int): Unit = { getTrackingClient.foreach { trackingClient => assert(trackingClient.numGetCommitsCalled.get >= expectedNumGetCommits) } } protected def getTrackingClient: Option[TrackingCommitCoordinatorClient] = if (catalogOwnedDefaultCreationEnabledInTests) { Some(getCatalogOwnedCommitCoordinatorClient( CatalogOwnedTableUtils.DEFAULT_CATALOG_NAME_FOR_TESTING) .asInstanceOf[TrackingCommitCoordinatorClient]) } else { None } protected def resetTrackingClient(): Unit = { getTrackingClient.foreach(_.reset()) } protected override def beforeEach(): Unit = { super.beforeEach() resetTrackingClient() } test("stream from delta source") { withTempTable(createTable = false) { sourceTableName => sql(s"CREATE TABLE $sourceTableName (value INT) USING delta") val df = spark.readStream .format("delta") .table(sourceTableName) resetTrackingClient() testStream(df)( Execute{ _ => Seq(1, 2).toDF().write.format("delta").mode("append").saveAsTable(sourceTableName) }, ProcessAllAvailable(), CheckAnswer(1, 2), Execute { _ => assertNumCommitsCalled(1) }, // At least one read from the commit and one from checking the result Execute { _ => assertNumGetCommitsCalled(2) } ) } } test("stream to delta sink") { // The dir is only used as the checkpoint location and doesn't imply a path-based access. withTempDir { tempDir => withTempTable(createTable = false) { sinkTableName => var expectedNumCommits = 0 var expectedNumGetCommits = 0 val inputData = MemoryStream[Int] val query = inputData .toDF() .writeStream .format("delta") .option("checkpointLocation", tempDir.getAbsolutePath) .toTable(sinkTableName) query.processAllAvailable() try { inputData.addData(1) query.processAllAvailable() expectedNumCommits += 1 expectedNumGetCommits += 1 // Loading after the first write to ensure the delta log is created. val outputDf = spark.read.format("delta").table(sinkTableName) checkDatasetUnorderly(outputDf.as[Int], 1) expectedNumGetCommits += 1 assertNumCommitsCalled(expectedNumCommits) assertNumGetCommitsCalled(expectedNumGetCommits) inputData.addData(2) query.processAllAvailable() expectedNumCommits += 1 expectedNumGetCommits += 1 checkDatasetUnorderly(outputDf.as[Int], 1, 2) expectedNumGetCommits += 1 assertNumCommitsCalled(expectedNumCommits) assertNumGetCommitsCalled(expectedNumGetCommits) } finally { query.stop() } } } } test("stream from delta source to delta sink with shared commit coordinator") { withTempDir { tempDir => withTempTable(createTable = false) { sourceTableName => withTempTable(createTable = false) { sinkTableName => var expectedNumCommits = 0 var expectedNumGetCommits = 0 sql(s"CREATE TABLE $sourceTableName (value INT) USING delta") resetTrackingClient() val query = spark.readStream .format("delta") .table(sourceTableName) .toDF() .writeStream .format("delta") .option("checkpointLocation", tempDir.getAbsolutePath) .toTable(sinkTableName) query.processAllAvailable() expectedNumCommits += 1 expectedNumGetCommits += 1 assertNumCommitsCalled(expectedNumCommits) assertNumGetCommitsCalled(expectedNumGetCommits) try { Seq(1).toDF().write.format("delta").mode("append").saveAsTable(sourceTableName) query.processAllAvailable() // One commit to the source table and one commit to the sink table expectedNumCommits += 2 expectedNumGetCommits += 2 // Loading after the first write to ensure the delta log is created. val outputDf = spark.read.format("delta").table(sinkTableName) checkDatasetUnorderly(outputDf.as[Int], 1) expectedNumGetCommits += 1 assertNumCommitsCalled(expectedNumCommits) assertNumGetCommitsCalled(expectedNumGetCommits) Seq(2).toDF().write.format("delta").mode("append").saveAsTable(sourceTableName) query.processAllAvailable() // One commit to the source table and one commit to the sink table expectedNumCommits += 2 expectedNumGetCommits += 2 checkDatasetUnorderly(outputDf.as[Int], 1, 2) expectedNumGetCommits += 1 assertNumCommitsCalled(expectedNumCommits) assertNumGetCommitsCalled(expectedNumGetCommits) } finally { query.stop() } } } } } } class CatalogManagedStreamingSuite extends CatalogManagedStreamingSuiteBase { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(10) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CatalogOwnedEnablementSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import java.io.File import java.util.UUID import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.test.{DeltaExceptionTestUtils, DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.JsonUtils import org.apache.commons.lang3.NotImplementedException import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier class CatalogOwnedEnablementSuite extends QueryTest with CatalogOwnedTestBaseSuite with DeltaSQLTestUtils with DeltaSQLCommandTest with DeltaTestUtilsBase with DeltaExceptionTestUtils { override protected def beforeEach(): Unit = { super.beforeEach() CatalogOwnedCommitCoordinatorProvider.clearBuilders() CatalogOwnedCommitCoordinatorProvider.registerBuilder( catalogName = CatalogOwnedTableUtils.DEFAULT_CATALOG_NAME_FOR_TESTING, commitCoordinatorBuilder = TrackingInMemoryCommitCoordinatorBuilder(batchSize = 3) ) } private val ICT_ENABLED_KEY = DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key /** * Validate that the table has the expected enablement of Catalog-Owned table feature. * Specifically, [[CatalogOwnedTableFeature]] and its dependent features should be present * in the table protocol. * * @param tableName The name of the table to validate. * @param expectEnabled Whether the Catalog-Owned table feature should be enabled or not. */ private def validateCatalogOwnedCompleteEnablement( tableName: String, expectEnabled: Boolean): Unit = { val (_, snapshot) = getDeltaLogWithSnapshot(TableIdentifier(tableName)) assert(snapshot.isCatalogOwned == expectEnabled) Seq( CatalogOwnedTableFeature, VacuumProtocolCheckTableFeature, InCommitTimestampTableFeature ).foreach { feature => assert(snapshot.protocol.writerFeatures.exists(_.contains(feature.name)) == expectEnabled) } Seq( CatalogOwnedTableFeature, VacuumProtocolCheckTableFeature ).foreach { feature => assert(snapshot.protocol.readerFeatures.exists(_.contains(feature.name)) == expectEnabled) } } protected def commitCoordinatorName: String = CatalogOwnedTableUtils.DEFAULT_CATALOG_NAME_FOR_TESTING protected def createTable( tableName: String, clusterBy: Seq[String] = Seq.empty, properties: Map[String, String] = Map.empty): Unit = { var stmt = s"CREATE TABLE $tableName (id INT) USING delta " if (clusterBy.nonEmpty) { stmt += s"CLUSTER BY (${clusterBy.mkString(", ")}) " } if (properties.nonEmpty) { val kv = properties.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ") stmt += s"TBLPROPERTIES ($kv)" } sql(stmt) } /** * Helper function to set up and validate the initial table to be tested * for Catalog Owned enablement. * * @param tableName The name of the table to be created for the test. * @param createCatalogOwnedTableAtInit Whether to enable Catalog-Owned table feature * at the time of table creation. */ private def testCatalogOwnedEnablementSetup( tableName: String, createCatalogOwnedTableAtInit: Boolean): Unit = { if (createCatalogOwnedTableAtInit) { createTable(tableName, properties = Map( s"delta.feature.${CatalogOwnedTableFeature.name}" -> "supported")) } else { createTable(tableName) } // Insert initial data to the table spark.sql(s"INSERT INTO $tableName VALUES 1") // commit 1 spark.sql(s"INSERT INTO $tableName VALUES 2") // commit 2 validateCatalogOwnedCompleteEnablement( tableName, expectEnabled = createCatalogOwnedTableAtInit) } /** * Helper function to create a table and run the test. * * @param f The test function to run with the random-generated table name. */ private def withRandomTable( createCatalogOwnedTableAtInit: Boolean)(f: String => Unit): Unit = { val randomTableName = s"testTable_${UUID.randomUUID().toString.replace("-", "")}" withTable(randomTableName) { testCatalogOwnedEnablementSetup(randomTableName, createCatalogOwnedTableAtInit) f(randomTableName) } } /** * Validate the usage log blob. * * @param usageLogBlob The usage log blob to validate. * @param expectedPresentFields The fields that should be present in the usage log blob. * @param expectedAbsentFields The fields that should not be present in the usage log blob. * @param expectedValues The expected values for the fields. */ private def validateUsageLogBlob( usageLogBlob: Map[String, Any], expectedPresentFields: Seq[String] = Seq.empty, expectedAbsentFields: Seq[String] = Seq.empty, expectedValues: Map[String, Any] = Map.empty): Unit = { expectedPresentFields.foreach { field => assert(usageLogBlob.contains(field), s"Field '$field' should be present in usage log blob") } expectedAbsentFields.foreach { field => assert(!usageLogBlob.contains(field), s"Field '$field' should not be present in usage log blob") } assert(expectedValues.size === expectedPresentFields.size, "The size of `expectedValues` should match the size of `expectedPresentFields`.") expectedValues.foreach { case (field, expectedValue) => if (field == "stackTrace") { // Only validate the start of the stack trace. assert( usageLogBlob(field).asInstanceOf[String].startsWith(expectedValue.asInstanceOf[String]), s"Field '$field' should start with '$expectedValue' but was '${usageLogBlob(field)}'" ) } else if (field == "checksumOpt" || field == "properties") { // Validate a portion of the entire `checksumOpt` or `properties` map. if (expectedValue == null) { assert(usageLogBlob(field) == null, s"Field '$field' should be null but was '${usageLogBlob(field)}'") return } val properties = expectedValue.asInstanceOf[Map[String, Any]] properties.foreach { case (key, value) => assert( usageLogBlob(field).asInstanceOf[Map[String, Any]].get(key).contains(value), s"Field '$field' should contain '$key' with value '$value' " + s"but was '${usageLogBlob(field)}'" ) } } else { assert(usageLogBlob(field) === expectedValue, s"Field '$field' should have value '$expectedValue' but was '${usageLogBlob(field)}'") } } } /** * Helper function to validate usage log for commit coordinator population when path-based access * is blocked. */ private def validateInvalidPathBasedAccessUsageLog( tempDir: File, expectedVersion: String, expectedChecksumOpt: Any, sqlConf: Map[String, String] = Map.empty): Unit = { val log = DeltaLog.forTable(spark, tempDir.getCanonicalPath) val usageLog = Log4jUsageLogger.track { val error = intercept[DeltaIllegalStateException] { withSQLConf(sqlConf.toSeq: _*) { sql(s"INSERT INTO TABLE delta.`$tempDir` VALUES (1), (2), (3)") } } checkError( error, "DELTA_PATH_BASED_ACCESS_TO_CATALOG_MANAGED_TABLE_BLOCKED", sqlState = "KD00G", parameters = Map("path" -> log.logPath.toString)) }.filter { log => log.metric == "tahoeEvent" && log.tags.getOrElse("opType", null) == CatalogOwnedUsageLogs.COMMIT_COORDINATOR_POPULATION_INVALID_PATH_BASED_ACCESS } assert(usageLog.nonEmpty, "Should have usage log for INVALID_PATH_BASED_ACCESS scenario") val usageLogBlob = JsonUtils.fromJson[Map[String, Any]](usageLog.head.blob) val logStore = "org.apache.spark.sql.delta.storage.DelegatingLogStore" validateUsageLogBlob( usageLogBlob, expectedPresentFields = Seq( "path", "version", "stackTrace", "latestCheckpointVersion", "checksumOpt", "properties", "logStore" ), expectedAbsentFields = Seq( "catalogTable.identifier", "catalogTable.tableType", "commitCoordinator.getClass" ), expectedValues = Map( "path" -> log.logPath.toString, "version" -> expectedVersion, "stackTrace" -> ("org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTableUtils$" + ".recordCommitCoordinatorPopulationUsageLog"), "latestCheckpointVersion" -> -1, // Only check for certain fields of `checksumOpt` since the entire map // is too large to validate. "checksumOpt" -> expectedChecksumOpt, "properties" -> Map( "delta.minReaderVersion" -> "3", "delta.minWriterVersion" -> "7", "delta.feature.appendOnly" -> "supported", "delta.feature.invariants" -> "supported", // To avoid potential naming change in the future. s"delta.feature.${CatalogOwnedTableFeature.name}" -> "supported", "delta.feature.inCommitTimestamp" -> "supported", "delta.feature.vacuumProtocolCheck" -> "supported", "delta.enableInCommitTimestamps" -> "true" ), "logStore" -> logStore ) ) } test("Quality of life table feature should be enabled by CREATE CLONE for target table") { withRandomTable(createCatalogOwnedTableAtInit = false) { tableName => withTable("t1", "t2") { val qolTableFeatures = CatalogOwnedTableUtils.QOL_TABLE_FEATURES_AND_PROPERTIES.map(_._1) .toSet withSQLConf(defaultCatalogOwnedFeatureEnabledKey -> "supported") { sql(s"CREATE TABLE t1 SHALLOW CLONE $tableName") } sql(s"CREATE TABLE t2 SHALLOW CLONE $tableName TBLPROPERTIES " + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported')") Seq("t1", "t2").foreach { t => validateOnlySpecifiedQoLTableFeaturesAndMetadataPresent( tableName = t, supportedTableFeatures = qolTableFeatures ) validateCatalogOwnedCompleteEnablement(tableName = t, expectEnabled = true) sql(s"INSERT INTO $t VALUES (3), (4), (5), (6)") checkAnswer(sql(s"SELECT * FROM $t"), Seq(Row(1), Row(2), Row(3), Row(4), Row(5), Row(6))) } } } } test("Quality of life table features and corresponding metadata should be " + "automatically enabled for CatalogOwned table") { withRandomTable(createCatalogOwnedTableAtInit = true) { tableName => // [[RowTrackingFeature]], [[DeletionVectorsTableFeature]], // and [[V2CheckpointTableFeature]] should be enabled by default. validateQoLFeaturesEnablement(tableName, expected = true) } // QoL features should also be enabled w/ other additional features enablement // during CatalogOwned creation. withTable("t1") { // Enable [[ClusteringTableFeature]] and [[ChangeDataFeedTableFeature]] during CO creation. createTable("t1", clusterBy = Seq("id"), properties = Map( s"delta.feature.${CatalogOwnedTableFeature.name}" -> "supported", DeltaConfigs.CHANGE_DATA_FEED.key -> "true")) sql("INSERT INTO t1 VALUES (1), (2)") validateQoLFeaturesEnablement(tableName = "t1", expected = true) val (_, snapshot) = getDeltaLogWithSnapshot(TableIdentifier("t1")) val readerAndWriterFeatureNames = snapshot.protocol.readerAndWriterFeatureNames assert(readerAndWriterFeatureNames.contains(ChangeDataFeedTableFeature.name) && readerAndWriterFeatureNames.contains(ClusteringTableFeature.name)) validateCatalogOwnedCompleteEnablement(tableName = "t1", expectEnabled = true) } } test("ALTER TABLE should be blocked if attempts to disable ICT") { withRandomTable(createCatalogOwnedTableAtInit = true) { tableName => val error = interceptWithUnwrapping[DeltaIllegalArgumentException] { spark.sql(s"ALTER TABLE $tableName SET TBLPROPERTIES ('$ICT_ENABLED_KEY' = 'false')") } checkError( error, "DELTA_CANNOT_MODIFY_CATALOG_MANAGED_DEPENDENCIES", sqlState = "42616", parameters = Map[String, String]()) } } test("ALTER TABLE should be blocked if attempts to unset ICT") { withRandomTable(createCatalogOwnedTableAtInit = true) { tableName => val error = interceptWithUnwrapping[DeltaIllegalArgumentException] { spark.sql(s"ALTER TABLE $tableName UNSET TBLPROPERTIES ('$ICT_ENABLED_KEY')") } checkError( error, "DELTA_CANNOT_MODIFY_CATALOG_MANAGED_DEPENDENCIES", sqlState = "42616", parameters = Map[String, String]()) } } test("ALTER TABLE should be blocked if attempts to downgrade Catalog-Owned") { withRandomTable(createCatalogOwnedTableAtInit = true) { tableName => val error = intercept[DeltaTableFeatureException] { spark.sql(s"ALTER TABLE $tableName DROP FEATURE '${CatalogOwnedTableFeature.name}'") } checkError( error, "DELTA_FEATURE_DROP_UNSUPPORTED_CLIENT_FEATURE", sqlState = "0AKDC", parameters = Map("feature" -> CatalogOwnedTableFeature.name)) } } test("Upgrade should be blocked since it is not supported yet") { withRandomTable( // Do not enable Catalog-Owned at the beginning when creating table createCatalogOwnedTableAtInit = false ) { tableName => val error = intercept[NotImplementedException] { spark.sql(s"ALTER TABLE $tableName SET TBLPROPERTIES " + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported')") } assert(error.getMessage.contains("Upgrading to CatalogOwned table is not yet supported.")) } } test("Dropping CatalogOwned dependent features should be blocked") { withRandomTable(createCatalogOwnedTableAtInit = true) { tableName => val error1 = intercept[DeltaTableFeatureException] { spark.sql(s"ALTER TABLE $tableName DROP FEATURE '${InCommitTimestampTableFeature.name}'") } checkError( error1, "DELTA_FEATURE_DROP_DEPENDENT_FEATURE", sqlState = "55000", parameters = Map( "feature" -> InCommitTimestampTableFeature.name, "dependentFeatures" -> CatalogOwnedTableFeature.name)) val error2 = intercept[DeltaTableFeatureException] { spark.sql(s"ALTER TABLE $tableName DROP FEATURE '${VacuumProtocolCheckTableFeature.name}'") } checkError( error2, "DELTA_FEATURE_DROP_DEPENDENT_FEATURE", sqlState = "55000", parameters = Map( "feature" -> VacuumProtocolCheckTableFeature.name, "dependentFeatures" -> CatalogOwnedTableFeature.name)) } } test("CO_COMMIT should be recorded in usage_log for normal CO commit") { withRandomTable(createCatalogOwnedTableAtInit = true) { tableName => val usageLog = Log4jUsageLogger.track { sql(s"INSERT INTO $tableName VALUES 3") } val commitStatsUsageLog = filterUsageRecords(usageLog, "delta.commit.stats") val commitStats = JsonUtils.fromJson[CommitStats](commitStatsUsageLog.head.blob) assert(commitStats.coordinatedCommitsInfo === CoordinatedCommitsStats( coordinatedCommitsType = CoordinatedCommitType.CO_COMMIT.toString, commitCoordinatorName = commitCoordinatorName, commitCoordinatorConf = Map.empty)) } } test("FS_TO_CO_UPGRADE_COMMIT should be recorded in usage_log when creating " + "CatalogOwned table") { withTable("t1") { val usageLog = Log4jUsageLogger.track { createTable("t1", properties = Map( s"delta.feature.${CatalogOwnedTableFeature.name}" -> "supported")) } val commitStatsUsageLog = filterUsageRecords(usageLog, "delta.commit.stats") val commitStats = JsonUtils.fromJson[CommitStats](commitStatsUsageLog.head.blob) assert(commitStats.coordinatedCommitsInfo === CoordinatedCommitsStats( coordinatedCommitsType = CoordinatedCommitType.FS_TO_CO_UPGRADE_COMMIT.toString, // catalogTable is not available for FS_TO_CO_UPGRADE_COMMIT commitCoordinatorName = "CATALOG_MISSING", commitCoordinatorConf = Map.empty)) } } test("Testing usage log for commit coordinator population for invalid path-based access") { // Clear all potential CC builders so that we are not entering the UT-only path when // populating the commit coordinator. clearBuilders() withTempDir { tempDir => // Create a path-based table so that we can simulate the scenario // where catalog table is not available. createTable(s"delta.`$tempDir`", properties = Map( s"delta.feature.${CatalogOwnedTableFeature.name}" -> "supported")) validateInvalidPathBasedAccessUsageLog( tempDir, expectedVersion = "0", // Only check for certain fields of `checksumOpt` since the entire map // is too large to validate. expectedChecksumOpt = Map( "tableSizeBytes" -> 0, "numFiles" -> 0, "numMetadata" -> 1, "allFiles" -> List.empty ) ) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CatalogOwnedPropertySuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits // scalastyle:off import.ordering.noEmptyLine import java.util.UUID import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier // scalastyle:on import.ordering.noEmptyLine class CatalogOwnedPropertySuite extends QueryTest with DeltaSQLCommandTest with DeltaSQLTestUtils with CatalogOwnedTestBaseSuite { // Register the mock commit coordinator builder for testing, but don't enable // CatalogOwned by default (tests explicitly enable it when needed). override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(3) override def catalogOwnedDefaultCreationEnabledInTests: Boolean = false /** * Validate if the table is catalog owned or not by checking the following: * 1) [[Snapshot.isCatalogOwned]] === expected * 2) [[Metadata.configuration]] contains [[UCCommitCoordinatorClient.UC_TABLE_ID_KEY]] * 3) [[Protocol.readerAndWriterFeatureNames]] contains [[CatalogOwnedTableFeature.name]] * and its dependent features. * * @param tableName The name of the table to validate. * @param expected The expected value of whether the table is catalog owned or not. */ private def validateCatalogOwnedAndUCTableId(tableName: String, expected: Boolean): Unit = { val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, new TableIdentifier(tableName)) assert(snapshot.isCatalogOwned === expected) assert(snapshot.metadata.configuration.contains(UCCommitCoordinatorClient.UC_TABLE_ID_KEY) === expected) // CatalogOwned enabled table should have the protocol version of (3, 7), // since the table can't have a [[ReaderWriterTableFeature]] w/o this version. if (expected) { // Only verify protocol version for expected CatalogOwned table. validateProtocolMinReaderWriterVersion( tableName, expectedMinReaderVersion = 3, expectedMinWriterVersion = 7) // Check dependent features as well. CatalogOwnedTableFeature.requiredFeatures.foreach { feature => assert(snapshot.protocol.readerAndWriterFeatureNames.contains(feature.name), s"Table $tableName should have ${feature.name} in protocol.") } } } private def createTableAndValidateCatalogOwned( tableName: String, withCatalogOwned: Boolean): Unit = { val createTableSQLStr = if (withCatalogOwned) { s"CREATE TABLE $tableName (id LONG) USING delta TBLPROPERTIES " + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported')" } else { s"CREATE TABLE $tableName (id LONG) USING delta" } sql(createTableSQLStr) // Manually insert UC_TABLE_ID to mock UC integration behavior. if (withCatalogOwned) { val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName)) val deltaLog = DeltaLog.forTable(spark, catalogTable) val snapshot = deltaLog.update(catalogTableOpt = Some(catalogTable)) val newMetadata = snapshot.metadata.copy( configuration = snapshot.metadata.configuration + (UCCommitCoordinatorClient.UC_TABLE_ID_KEY -> java.util.UUID.randomUUID().toString) ) deltaLog.startTransaction(Some(catalogTable)) .commit(Seq(newMetadata), DeltaOperations.ManualUpdate) } validateCatalogOwnedAndUCTableId(tableName, expected = withCatalogOwned) } // Helper to manually insert UC_TABLE_ID to mock UC integration behavior. // This is needed in OSS tests since there's no real UC integration. private def mockUCTableIdInsertion(tableName: String): Unit = { val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName)) val deltaLog = DeltaLog.forTable(spark, catalogTable) val snapshot = deltaLog.update(catalogTableOpt = Some(catalogTable)) val newMetadata = snapshot.metadata.copy( configuration = snapshot.metadata.configuration + (UCCommitCoordinatorClient.UC_TABLE_ID_KEY -> java.util.UUID.randomUUID().toString) ) deltaLog.startTransaction(Some(catalogTable)) .commit(Seq(newMetadata), DeltaOperations.ManualUpdate) } private def getUCTableIdFromTable(tableName: String): String = { val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, new TableIdentifier(tableName)) val ucTableId = snapshot.metadata.configuration.get(UCCommitCoordinatorClient.UC_TABLE_ID_KEY) assert(ucTableId.isDefined, s"Table $tableName should have `ucTableId` in metadata.") ucTableId.get } private def getSnapshotVersion(tableName: String): Long = { val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, new TableIdentifier(tableName)) snapshot.version } private def validateInCommitTimestampTableFeature(tableName: String, expected: Boolean): Unit = { val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, new TableIdentifier(tableName)) val writerFeatureNames = snapshot.protocol.writerFeatureNames assert(writerFeatureNames.contains(InCommitTimestampTableFeature.name) === expected) } private def validateInCommitTimestampTableProperties( tableName: String, expectedEnabled: Boolean, expectedEnablementInfo: Boolean): Unit = { val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, new TableIdentifier(tableName)) val conf = snapshot.metadata.configuration assert(conf.contains(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key) === expectedEnabled) // In certain cases, we can't verify all three ICT table properties here. // This is because we only need to persist the ICT enablement info // if there are non-ICT commits in the Delta log. // I.e., [[DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_VERSION]] and // [[DeltaConfigs.IN_COMMIT_TIMESTAMP_ENABLEMENT_TIMESTAMP]]. // E.g., If the 0th commit enables ICT, then we will not need the above // two properties for the table in subsequent commits. // See [[InCommitTimestampUtils.getUpdatedMetadataWithICTEnablementInfo]] for details. CatalogOwnedTableUtils.ICT_TABLE_PROPERTY_KEYS.filter { k => k != DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key }.foreach { key => assert(conf.contains(key) === expectedEnablementInfo) } } private def validateInCommitTimestampPresent( tableName: String, expected: Boolean, expectedEnablementInfo: Boolean): Unit = { // Validate ICT table feature is present in [[Protocol]] first. validateInCommitTimestampTableFeature(tableName, expected) // Then validate ICT table properties are present in [[Metadata.configuration]]. validateInCommitTimestampTableProperties(tableName, expected, expectedEnablementInfo) } private def validateProtocolMinReaderWriterVersion( tableName: String, expectedMinReaderVersion: Int, expectedMinWriterVersion: Int): Unit = { val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, new TableIdentifier(tableName)) assert(snapshot.protocol.minReaderVersion === expectedMinReaderVersion) assert(snapshot.protocol.minWriterVersion === expectedMinWriterVersion) } test("[REPLACE] table UUID & table feature should retain for target " + "catalog-owned table during REPLACE TABLE") { withTable("t1") { createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) val ucTableId1 = getUCTableIdFromTable(tableName = "t1") // Target table should remain a catalog-owned table with same `ucTableId` sql("REPLACE TABLE t1 (id LONG) USING delta") val ucTableId2 = getUCTableIdFromTable(tableName = "t1") validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) // [[UCCommitCoordinatorClient.UC_TABLE_ID_KEY]] should be the same before and after REPLACE assert(ucTableId1 === ucTableId2) } } test("[REPLACE] Specifying table UUID for target table should be blocked " + "during REPLACE TABLE") { withTable("t1") { createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) val error = intercept[DeltaUnsupportedOperationException] { sql("REPLACE TABLE t1 (id LONG) USING delta TBLPROPERTIES " + s"('${UCCommitCoordinatorClient.UC_TABLE_ID_KEY}' = '${UUID.randomUUID().toString}')") } checkError(error, "DELTA_CANNOT_MODIFY_TABLE_PROPERTY", "42939", Map("prop" -> "io.unitycatalog.tableId")) } } test("[RTAS] Specifying table UUID when creating catalog-owned " + " table via RTAS should be blocked") { withTable("t1", "t2") { createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = false) sql("INSERT INTO t1 VALUES (1)") sql("INSERT INTO t1 VALUES (2)") createTableAndValidateCatalogOwned(tableName = "t2", withCatalogOwned = true) val ucTableIdOriginal = getUCTableIdFromTable(tableName = "t2") val error = intercept[DeltaUnsupportedOperationException] { sql(s"REPLACE TABLE t2 USING delta TBLPROPERTIES " + s"('${UCCommitCoordinatorClient.UC_TABLE_ID_KEY}' = '${UUID.randomUUID().toString}') " + s"AS SELECT * FROM t1") } checkError(error, "DELTA_CANNOT_MODIFY_TABLE_PROPERTY", "42939", Map("prop" -> "io.unitycatalog.tableId")) // Normal RTAS should not be blocked sql(s"REPLACE TABLE t2 USING delta AS SELECT * FROM t1") validateCatalogOwnedAndUCTableId(tableName = "t2", expected = true) val ucTableIdAfterRTAS = getUCTableIdFromTable(tableName = "t2") assert(ucTableIdOriginal === ucTableIdAfterRTAS) checkAnswer(sql("SELECT * FROM t2"), Seq(Row(1), Row(2))) } } test("[REPLACE] Specifying CatalogManaged for non-CatalogManaged table should " + "be blocked during REPLACE TABLE") { withTable("t1") { // Normal delta table. createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = false) val error = intercept[IllegalStateException] { sql("REPLACE TABLE t1 (id LONG) USING delta TBLPROPERTIES " + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported')") } assert(error.getMessage.contains( "Specifying CatalogManaged in REPLACE TABLE command is not supported")) } } test("[REPLACE] Specifying CatalogManaged for existing CatalogManaged table should " + "succeed as a no-op during REPLACE TABLE") { withTable("t1") { // CatalogManaged enabled table. createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") val versionBefore = getSnapshotVersion(tableName = "t1") // Specifying CatalogManaged on an already CatalogManaged table should succeed. // The CatalogManaged properties are treated as a no-op. sql("REPLACE TABLE t1 (id LONG) USING delta TBLPROPERTIES " + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported')") // Validate the table is still CatalogManaged with the same ucTableId. validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) val ucTableIdAfter = getUCTableIdFromTable(tableName = "t1") assert(ucTableIdBefore === ucTableIdAfter) assert(getSnapshotVersion(tableName = "t1") === versionBefore + 1) } } test("[RTAS] Specifying CatalogManaged for non-CatalogManaged table should " + "be blocked during RTAS") { withTable("t1", "t2") { // Normal delta table. createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = false) // Source RTAS table. createTableAndValidateCatalogOwned(tableName = "t2", withCatalogOwned = false) sql("INSERT INTO t2 VALUES (1), (2)") val error = intercept[IllegalStateException] { sql("REPLACE TABLE t1 USING delta TBLPROPERTIES " + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported') " + s"AS SELECT * FROM t2") } assert(error.getMessage.contains( "Specifying CatalogManaged in REPLACE TABLE command is not supported")) } } test("[RTAS] Specifying CatalogManaged for existing CatalogManaged table should " + "succeed as a no-op during RTAS") { withTable("t1", "t2") { // CatalogManaged enabled table. createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") val versionBefore = getSnapshotVersion(tableName = "t1") // Source RTAS table. createTableAndValidateCatalogOwned(tableName = "t2", withCatalogOwned = false) sql("INSERT INTO t2 VALUES (1), (2)") // Specifying CatalogManaged on an already CatalogManaged table should succeed. sql("REPLACE TABLE t1 USING delta TBLPROPERTIES " + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported') " + s"AS SELECT * FROM t2") // Validate the table is still CatalogManaged with the same ucTableId. validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) val ucTableIdAfter = getUCTableIdFromTable(tableName = "t1") assert(ucTableIdBefore === ucTableIdAfter) assert(getSnapshotVersion(tableName = "t1") === versionBefore + 1) checkAnswer(sql("SELECT * FROM t1"), Seq(Row(1), Row(2))) } } test("[RTAS] failed replace preserves existing catalog-managed table data and version") { withTable("t1") { createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") sql("INSERT INTO t1 VALUES (1), (2)") val versionBefore = getSnapshotVersion(tableName = "t1") intercept[Exception] { sql( """REPLACE TABLE t1 USING delta AS |SELECT IF(id = 2L, CAST(raise_error('boom') AS BIGINT), id) AS id |FROM VALUES (1L), (2L) AS src(id) |""".stripMargin) } validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) assert(getUCTableIdFromTable(tableName = "t1") === ucTableIdBefore) assert(getSnapshotVersion(tableName = "t1") === versionBefore) checkAnswer(sql("SELECT * FROM t1"), Seq(Row(1), Row(2))) } } test("[RTAS] replacing an existing catalog-managed table preserves UC identity") { withTable("t1", "t2") { createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") sql("INSERT INTO t1 VALUES (10)") val versionBefore = getSnapshotVersion(tableName = "t1") createTableAndValidateCatalogOwned(tableName = "t2", withCatalogOwned = false) sql("INSERT INTO t2 VALUES (1), (2)") sql("REPLACE TABLE t1 USING delta AS SELECT * FROM t2") validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) assert(getUCTableIdFromTable(tableName = "t1") === ucTableIdBefore) assert(getSnapshotVersion(tableName = "t1") === versionBefore + 1) checkAnswer(sql("SELECT * FROM t1"), Seq(Row(1), Row(2))) } } test("[CREATE OR REPLACE] with CatalogManaged on non-existing table should succeed") { withTable("t1") { // CREATE OR REPLACE on non-existing table with CatalogManaged should create a CC table. sql("CREATE OR REPLACE TABLE t1 (id LONG) USING delta TBLPROPERTIES " + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported')") // Mock UC integration behavior by inserting UC_TABLE_ID. mockUCTableIdInsertion("t1") validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) sql("INSERT INTO t1 VALUES (1), (2)") checkAnswer(sql("SELECT * FROM t1"), Seq(Row(1), Row(2))) } } test("[CREATE OR REPLACE] with CatalogManaged on existing CatalogManaged table " + "should succeed as a no-op") { withTable("t1") { // Create a CatalogManaged table first. createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") sql("INSERT INTO t1 VALUES (1)") val versionBefore = getSnapshotVersion(tableName = "t1") // CREATE OR REPLACE with CatalogManaged on existing CatalogManaged table should succeed. sql("CREATE OR REPLACE TABLE t1 (id LONG) USING delta TBLPROPERTIES " + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported')") // Validate the table is still CatalogManaged with the same ucTableId. validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) val ucTableIdAfter = getUCTableIdFromTable(tableName = "t1") assert(ucTableIdBefore === ucTableIdAfter) assert(getSnapshotVersion(tableName = "t1") === versionBefore + 1) } } test("[CREATE OR REPLACE] replacing an existing catalog-managed table is atomic") { withTable("t1") { createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") sql("INSERT INTO t1 VALUES (1)") val versionBefore = getSnapshotVersion(tableName = "t1") sql("CREATE OR REPLACE TABLE t1 USING delta TBLPROPERTIES " + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported') AS " + "SELECT 2 AS id") validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) assert(getUCTableIdFromTable(tableName = "t1") === ucTableIdBefore) assert(getSnapshotVersion(tableName = "t1") === versionBefore + 1) checkAnswer(sql("SELECT * FROM t1"), Seq(Row(2))) } } test("[REPLACE] replacing an existing catalog-managed table preserves UC identity") { withTable("t1") { createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") sql("INSERT INTO t1 VALUES (1)") val versionBefore = getSnapshotVersion(tableName = "t1") sql("REPLACE TABLE t1 (id LONG) USING delta") validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) assert(getUCTableIdFromTable(tableName = "t1") === ucTableIdBefore) assert(getSnapshotVersion(tableName = "t1") === versionBefore + 1) checkAnswer(sql("SELECT * FROM t1"), Seq.empty) } } test("[CREATE OR REPLACE] specifying table UUID for existing catalog-managed table " + "should be blocked") { withTable("t1") { createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") sql("INSERT INTO t1 VALUES (1)") val versionBefore = getSnapshotVersion(tableName = "t1") val error = intercept[DeltaUnsupportedOperationException] { sql("CREATE OR REPLACE TABLE t1 USING delta TBLPROPERTIES " + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported', " + s"'${UCCommitCoordinatorClient.UC_TABLE_ID_KEY}' = '${UUID.randomUUID().toString}') " + "AS SELECT 2 AS id") } checkError(error, "DELTA_CANNOT_MODIFY_TABLE_PROPERTY", "42939", Map("prop" -> "io.unitycatalog.tableId")) validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) assert(getUCTableIdFromTable(tableName = "t1") === ucTableIdBefore) assert(getSnapshotVersion(tableName = "t1") === versionBefore) checkAnswer(sql("SELECT * FROM t1"), Seq(Row(1))) } } test("[CREATE OR REPLACE] with CatalogManaged on existing non-CatalogManaged table " + "should be blocked") { withTable("t1") { // Create a non-CatalogManaged table first. createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = false) sql("INSERT INTO t1 VALUES (1)") // CREATE OR REPLACE with CatalogManaged on existing non-CatalogManaged table should fail. val error = intercept[IllegalStateException] { sql("CREATE OR REPLACE TABLE t1 (id LONG) USING delta TBLPROPERTIES " + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported')") } assert(error.getMessage.contains( "Specifying CatalogManaged in REPLACE TABLE command is not supported")) // Original table should remain unchanged. validateCatalogOwnedAndUCTableId(tableName = "t1", expected = false) checkAnswer(sql("SELECT * FROM t1"), Seq(Row(1))) } } test("[CREATE OR REPLACE] repeated same-schema CREATE OR REPLACE with CatalogManaged " + "should succeed repeatedly") { withTable("t1") { // First CREATE OR REPLACE creates the table. sql("CREATE OR REPLACE TABLE t1 (id LONG) USING delta TBLPROPERTIES " + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported')") // Mock UC integration behavior by inserting UC_TABLE_ID. mockUCTableIdInsertion("t1") validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) val ucTableIdFirst = getUCTableIdFromTable(tableName = "t1") sql("INSERT INTO t1 VALUES (1)") checkAnswer(sql("SELECT * FROM t1"), Seq(Row(1))) // Second CREATE OR REPLACE should succeed as a no-op for CC properties. sql("CREATE OR REPLACE TABLE t1 (id LONG) USING delta TBLPROPERTIES " + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported')") validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) val ucTableIdSecond = getUCTableIdFromTable(tableName = "t1") assert(ucTableIdFirst === ucTableIdSecond) // Third CREATE OR REPLACE should also succeed when the metadata is unchanged. sql("CREATE OR REPLACE TABLE t1 (id LONG) USING delta TBLPROPERTIES " + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported')") validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) val ucTableIdThird = getUCTableIdFromTable(tableName = "t1") assert(ucTableIdFirst === ucTableIdThird) // Verify table is functional. sql("INSERT INTO t1 VALUES (2)") checkAnswer(sql("SELECT * FROM t1"), Seq(Row(2))) } } test("[CREATE LIKE] Specifying table UUID should be blocked") { withTable("t1", "t2", "t3") { // Source catalog-owned table createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) val error1 = intercept[DeltaUnsupportedOperationException] { sql(s"CREATE TABLE t2 LIKE t1 TBLPROPERTIES " + s"('${UCCommitCoordinatorClient.UC_TABLE_ID_KEY}' = '${UUID.randomUUID().toString}')") } val error2 = intercept[DeltaUnsupportedOperationException] { sql(s"CREATE TABLE t3 LIKE t1 TBLPROPERTIES " + s"('${UCCommitCoordinatorClient.UC_TABLE_ID_KEY}' = '${UUID.randomUUID().toString}')") } Seq(error1, error2).foreach { error => checkError(error, "DELTA_CANNOT_MODIFY_TABLE_PROPERTY", "42939", Map("prop" -> "io.unitycatalog.tableId")) } } } test("[REPLACE] Protocol should not be downgraded when adding new table feature " + "to existing CatalogOwned enabled table") { withTable("t1", "t2") { createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) val ucTableIdBeforeReplace = getUCTableIdFromTable(tableName = "t1") sql("CREATE TABLE t2 (col1 LONG) USING delta") sql("INSERT INTO t2 VALUES (1)") sql("INSERT INTO t2 VALUES (2)") // Adding [[ClusteringTableFeature]] when replacing `t1` sql("REPLACE TABLE t1 USING delta CLUSTER BY (col1) " + "TBLPROPERTIES ('delta.dataSkippingNumIndexedCols' = '1') " + "AS SELECT * FROM t2") validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) val ucTableIdAfterReplace = getUCTableIdFromTable(tableName = "t1") assert(ucTableIdBeforeReplace === ucTableIdAfterReplace) checkAnswer(sql("SELECT * FROM t1"), Seq(Row(1), Row(2))) val log = DeltaLog.forTable(spark, new TableIdentifier("t1")) val readerWriterFeatureNames = log.unsafeVolatileSnapshot.protocol.readerAndWriterFeatureNames assert(readerWriterFeatureNames.contains(ClusteringTableFeature.name) && readerWriterFeatureNames.contains(CatalogOwnedTableFeature.name)) } } test("[REPLACE] ICT should not be present after replacing existing normal delta " + "table w/o ICT w/ default spark configuration") { withSQLConf( TableFeatureProtocolUtils.defaultPropertyKey(CatalogOwnedTableFeature) -> "supported") { withTable("t1", "t2") { withoutDefaultCCTableFeature { sql("CREATE TABLE t1 (id LONG) USING delta") sql("INSERT INTO t1 VALUES (1), (2)") sql("CREATE TABLE t2 (id LONG) USING delta") validateInCommitTimestampPresent( tableName = "t2", expected = false, expectedEnablementInfo = false) } sql("REPLACE TABLE t2 USING delta AS SELECT * FROM t1") checkAnswer(sql("SELECT * FROM t2"), Seq(Row(1), Row(2))) validateInCommitTimestampPresent( tableName = "t2", expected = false, expectedEnablementInfo = false) validateCatalogOwnedAndUCTableId(tableName = "t2", expected = false) sql("INSERT INTO t2 VALUES (3), (4)") checkAnswer(sql("SELECT * FROM t2"), Seq(Row(1), Row(2), Row(3), Row(4))) } } } test("[REPLACE] ICT-related properties should not be present after replacing existing " + "normal delta table w/ ICT w/ default spark configuration") { withSQLConf( TableFeatureProtocolUtils.defaultPropertyKey(CatalogOwnedTableFeature) -> "supported") { withTable("t1", "t2", "t3") { withoutDefaultCCTableFeature { sql("CREATE TABLE t1 (id LONG) USING delta") sql("INSERT INTO t1 VALUES (1), (2)") sql("CREATE TABLE t2 (id LONG) USING delta TBLPROPERTIES " + s"('${DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key}' = 'true')") validateInCommitTimestampPresent( tableName = "t2", expected = true, expectedEnablementInfo = false) sql("CREATE TABLE t3 (id LONG) USING delta") sql("INSERT INTO t3 VALUES (1), (2)") sql("ALTER TABLE t3 SET TBLPROPERTIES " + s"('${DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key}' = 'true')") validateInCommitTimestampPresent( tableName = "t3", expected = true, expectedEnablementInfo = true) } sql("REPLACE TABLE t2 USING delta AS SELECT * FROM t1") sql("REPLACE TABLE t3 USING delta AS SELECT * FROM t1") sql("INSERT INTO t2 VALUES (3)") sql("INSERT INTO t3 VALUES (3), (4)") checkAnswer(sql("SELECT * FROM t2"), Seq(Row(1), Row(2), Row(3))) checkAnswer(sql("SELECT * FROM t3"), Seq(Row(1), Row(2), Row(3), Row(4))) Seq("t2", "t3").foreach { tableName => // ICT-related properties should *NOT* be present after replacing existing normal // delta table w/o explicit overrides or default spark configuration. validateInCommitTimestampTableProperties( tableName = tableName, expectedEnabled = false, // All three ICT properties should not be present, // though currently there is only one for `t2`. expectedEnablementInfo = false) // ICT table feature will be preserved after REPLACE. validateInCommitTimestampTableFeature(tableName = tableName, expected = true) validateCatalogOwnedAndUCTableId(tableName = tableName, expected = false) } sql("INSERT INTO t2 VALUES (4), (5)") sql("INSERT INTO t3 VALUES (5)") checkAnswer(sql("SELECT * FROM t2"), Seq(Row(1), Row(2), Row(3), Row(4), Row(5))) checkAnswer(sql("SELECT * FROM t3"), Seq(Row(1), Row(2), Row(3), Row(4), Row(5))) } } } test("[REPLACE] ICT should be present after replacing existing normal delta " + "table w/o ICT w/ default spark configuration w/ explicitly overrides") { withSQLConf( TableFeatureProtocolUtils.defaultPropertyKey(CatalogOwnedTableFeature) -> "supported") { withTable("t1", "t2", "t3") { withoutDefaultCCTableFeature { sql("CREATE TABLE t1 (id LONG) USING delta") sql("INSERT INTO t1 VALUES (1), (2)") sql("CREATE TABLE t2 (id LONG) USING delta") sql("CREATE TABLE t3 (id LONG) USING delta") sql("INSERT INTO t3 VALUES (1), (2)") Seq("t2", "t3").foreach { tableName => validateInCommitTimestampPresent( tableName = tableName, expected = false, expectedEnablementInfo = false) } } Seq("t2", "t3").foreach { tableName => sql(s""" | REPLACE TABLE $tableName USING delta TBLPROPERTIES | ('${DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key}' = 'true') | AS SELECT * FROM t1 """.stripMargin) checkAnswer(sql(s"SELECT * FROM $tableName"), Seq(Row(1), Row(2))) validateCatalogOwnedAndUCTableId(tableName = tableName, expected = false) sql(s"INSERT INTO $tableName VALUES (3), (4)") checkAnswer(sql(s"SELECT * FROM $tableName"), Seq(Row(1), Row(2), Row(3), Row(4))) // ICT enablement info is present in both `t2` and `t3`. validateInCommitTimestampPresent( tableName = "t2", expected = true, expectedEnablementInfo = true) } } } } test("[REPLACE] ICT should be present after replacing existing normal delta " + "table w/ ICT enabled via default spark configuration") { withTable("t1", "t2", "t3") { sql("CREATE TABLE t1 (id LONG) USING delta") sql("INSERT INTO t1 VALUES (1), (2)") sql("CREATE TABLE t2 (id LONG) USING delta") sql("INSERT INTO t2 VALUES (1), (2)") sql("CREATE TABLE t3 (id LONG) USING delta") Seq("t2", "t3").foreach { tableName => validateInCommitTimestampPresent( tableName = tableName, expected = false, expectedEnablementInfo = false) validateCatalogOwnedAndUCTableId(tableName, expected = false) } // w/ default CO enabled withSQLConf( TableFeatureProtocolUtils.defaultPropertyKey(CatalogOwnedTableFeature) -> "supported", DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey -> "true") { sql("REPLACE TABLE t2 USING delta AS SELECT * FROM t1") validateInCommitTimestampPresent( tableName = "t2", expected = true, expectedEnablementInfo = true) validateCatalogOwnedAndUCTableId(tableName = "t2", expected = false) } // w/o default CO enabled withSQLConf(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey -> "true") { sql("REPLACE TABLE t3 USING delta AS SELECT * FROM t1") validateInCommitTimestampPresent( tableName = "t3", expected = true, expectedEnablementInfo = true) validateCatalogOwnedAndUCTableId(tableName = "t3", expected = false) } Seq("t1", "t2", "t3").foreach { tableName => sql(s"INSERT INTO $tableName VALUES (3), (4)") checkAnswer(sql(s"SELECT * FROM $tableName"), Seq(Row(1), Row(2), Row(3), Row(4))) } } } test("Table protocol should be kept intact before and after REPLACE " + "regardless of default CC enablement") { def getReaderAndWriterFeatureNamesFromTable(tableName: String): Set[String] = { val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName)) snapshot.protocol.readerAndWriterFeatureNames } withTable("t1", "t2", "source") { // Source table. sql("CREATE TABLE source (id LONG) USING delta") sql("INSERT INTO source VALUES (3), (4), (5)") // Create a normal delta table w/ protocol (1, 2). sql("CREATE TABLE t1 (id LONG) USING delta") val featuresBeforeReplaceT1 = getReaderAndWriterFeatureNamesFromTable(tableName = "t1") validateProtocolMinReaderWriterVersion( tableName = "t1", expectedMinReaderVersion = 1, expectedMinWriterVersion = 2) validateCatalogOwnedAndUCTableId(tableName = "t1", expected = false) CatalogOwnedTableFeature.requiredFeatures.foreach { feature => assert(!featuresBeforeReplaceT1.contains(feature.name)) } // Create a CC table w/ protocol (3, 7). createTableAndValidateCatalogOwned(tableName = "t2", withCatalogOwned = true) val featuresBeforeReplaceT2 = getReaderAndWriterFeatureNamesFromTable(tableName = "t2") // Insert initial data. Seq("t1", "t2").foreach { tableName => sql(s"INSERT INTO $tableName VALUES (1), (2)") } // Enable CC by default withDefaultCCTableFeature { Seq("t1", "t2").foreach { tableName => sql(s"REPLACE TABLE $tableName USING delta AS SELECT * FROM source") checkAnswer(sql(s"SELECT * FROM $tableName"), Seq(Row(3), Row(4), Row(5))) } // REPLACE TABLE should not change the protocol. val featuresAfterReplaceT1 = getReaderAndWriterFeatureNamesFromTable(tableName = "t1") validateProtocolMinReaderWriterVersion( tableName = "t1", expectedMinReaderVersion = 1, expectedMinWriterVersion = 2) assert(featuresBeforeReplaceT1 === featuresAfterReplaceT1) validateCatalogOwnedAndUCTableId(tableName = "t1", expected = false) val featuresAfterReplaceT2 = getReaderAndWriterFeatureNamesFromTable(tableName = "t2") assert(featuresBeforeReplaceT2 === featuresAfterReplaceT2) validateCatalogOwnedAndUCTableId(tableName = "t2", expected = true) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CommitCoordinatorClientImplSuiteBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import java.io.File import java.util.concurrent.{Executors, TimeUnit} import java.util.concurrent.atomic.AtomicInteger import scala.collection.JavaConverters._ import scala.concurrent.duration._ import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.actions.{Action, CommitInfo, Metadata, Protocol} import org.apache.spark.sql.delta.storage.{LogStore, LogStoreProvider} import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.FileNames import org.apache.spark.sql.delta.util.threads.DeltaThreadPool import io.delta.dynamodbcommitcoordinator.DynamoDBCommitCoordinatorClient import io.delta.storage.commit.{Commit => JCommit, CommitFailedException => JCommitFailedException, GetCommitsResponse => JGetCommitsResponse} import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.{ThreadUtils, Utils} trait CommitCoordinatorClientImplSuiteBase extends QueryTest with SharedSparkSession with LogStoreProvider with CoordinatedCommitsTestUtils with DeltaSQLTestUtils with DeltaSQLCommandTest { /** * Needs to be overwritten by implementing classes to provide a [[TableCommitCoordinatorClient]] * wrapping the commit coordinator client that should be tested. */ protected def createTableCommitCoordinatorClient(deltaLog: DeltaLog): TableCommitCoordinatorClient /** * Needs to be overwritten by implementing classes to provide an implementation * of backfill registration. */ protected def registerBackfillOp( tableCommitCoordinatorClient: TableCommitCoordinatorClient, deltaLog: DeltaLog, version: Long): Unit /** * Needs to be overwritten by implementing classes to provide a way of validating * that the commit coordinator client under test performs backfilling as expected at * the specified version. */ protected def validateBackfillStrategy( tableCommitCoordinatorClient: TableCommitCoordinatorClient, logPath: Path, version: Long): Unit /** * Needs to be overwritten by implementing classes to provide a way of validating * the results of a getCommits call with the specified start and end versions, * where maxVersion is the current latest version of the table. */ protected def validateGetCommitsResult( response: JGetCommitsResponse, startVersion: Option[Long], endVersion: Option[Long], maxVersion: Long): Unit /** * Checks that the commit coordinator state is correct in terms of * - The latest table version in the commit coordinator is correct * - All supposedly backfilled commits are indeed backfilled * - The contents of the backfilled commits are correct (verified * if commitTimestampOpt is provided) * * This can be overridden by implementing classes to implement * more specific invariants. */ protected def assertInvariants( logPath: Path, tableCommitCoordinatorClient: TableCommitCoordinatorClient, commitTimestampsOpt: Option[Array[Long]] = None): Unit = { val maxUntrackedVersion: Int = { val commitResponse = tableCommitCoordinatorClient.getCommits() if (commitResponse.getCommits.isEmpty) { commitResponse.getLatestTableVersion.toInt } else { assert(commitResponse.getCommits.asScala.last.getVersion == commitResponse.getLatestTableVersion, s"Max commit tracked by the commit coordinator " + s"${commitResponse.getCommits.asScala.last} must " + s"match latestTableVersion tracked by the commit coordinator " + s"${commitResponse.getLatestTableVersion}." ) val minVersion = commitResponse.getCommits.asScala.head.getVersion assert( commitResponse.getLatestTableVersion - minVersion + 1 == commitResponse.getCommits.size, "Commit map should have a contiguous range of unbackfilled commits." ) minVersion.toInt - 1 } } (0 to maxUntrackedVersion).foreach { version => assertBackfilled(version, logPath, commitTimestampsOpt.map(_(version))) } } protected def writeCommitZero(logPath: Path): Unit = { val commitInfo = CommitInfo.empty(version = Some(0)).withTimestamp(0) .copy(inCommitTimestamp = Some(0)) val actions = Iterator(commitInfo.json, Metadata().json, Protocol().json) store.write(FileNames.unsafeDeltaFile(logPath, 0), actions, overwrite = false) } /** * The metadata that should be passed to the registerTable call. By default, this * is empty but implementing classes can overwrite this method to provide custom * metadata. */ protected def initMetadata(): Metadata = Metadata() // scalastyle:off deltahadoopconfiguration protected def sessionHadoopConf: Configuration = spark.sessionState.newHadoopConf() // scalastyle:on deltahadoopconfiguration protected def store: LogStore = createLogStore(spark) protected def withTempTableDir(f: File => Unit): Unit = { val dir = Utils.createTempDir() val deltaLogDir = new File(dir, DeltaLog.LOG_DIR_NAME) deltaLogDir.mkdir() val commitLogDir = new File(deltaLogDir, FileNames.COMMIT_SUBDIR) commitLogDir.mkdir() try f(dir) finally { Utils.deleteRecursively(dir) } } protected def commit( version: Long, timestamp: Long, tableCommitCoordinatorClient: TableCommitCoordinatorClient, tableIdentifier: Option[TableIdentifier] = None): JCommit = { val commitInfo = CommitInfo.empty(version = Some(version)).withTimestamp(timestamp) .copy(inCommitTimestamp = Some(timestamp)) val updatedActions = if (version == 0) { getUpdatedActionsForZerothCommit(commitInfo) } else { getUpdatedActionsForNonZerothCommit(commitInfo) } tableCommitCoordinatorClient.commit( version, Iterator(commitInfo.json), updatedActions, tableIdentifier).getCommit } protected def assertBackfilled( version: Long, logPath: Path, timestampOpt: Option[Long] = None): Unit = { val delta = FileNames.unsafeDeltaFile(logPath, version) if (timestampOpt.isDefined) { val commitInfo = CommitInfo.empty(version = Some(version)) .withTimestamp(timestampOpt.get) .copy(inCommitTimestamp = timestampOpt) assert(store.read(delta, sessionHadoopConf).head == commitInfo.json) } else { assert(Action.fromJson(store.read(delta, sessionHadoopConf).head) .isInstanceOf[CommitInfo]) } } protected def assertCommitFail( currentVersion: Long, expectedVersion: Long, retryable: Boolean, commitFunc: => JCommit): Unit = { val e = intercept[JCommitFailedException] { commitFunc } assert(e.getRetryable == retryable) assert(e.getConflict == retryable) val expectedMessage = if (currentVersion == 0) { "Commit version 0 must go via filesystem." } else { s"Commit version $currentVersion is not valid. Expected version: $expectedVersion." } assert(e.getMessage === expectedMessage) } protected def assertResponseEquals( resp1: JGetCommitsResponse, resp2: JGetCommitsResponse): Unit = { assert(resp1.getLatestTableVersion == resp2.getLatestTableVersion) assert(resp1.getCommits == resp2.getCommits) } test("test basic commit and backfill functionality") { withTempTableDir { tempDir => val log = DeltaLog.forTable(spark, tempDir.toString) val logPath = log.logPath val tableCommitCoordinatorClient = createTableCommitCoordinatorClient(log) val e = intercept[JCommitFailedException] { commit(version = 0, timestamp = 0, tableCommitCoordinatorClient) } assert(e.getMessage === "Commit version 0 must go via filesystem.") writeCommitZero(logPath) assertResponseEquals(tableCommitCoordinatorClient.getCommits(), new JGetCommitsResponse(Seq.empty.asJava, -1)) assertBackfilled(version = 0, logPath, Some(0L)) // Test backfilling functionality for commits 1 - 8 (1 to 8).foreach { version => commit(version, version, tableCommitCoordinatorClient) validateBackfillStrategy(tableCommitCoordinatorClient, logPath, version) assert(tableCommitCoordinatorClient.getCommits().getLatestTableVersion == version) } // Test that out-of-order backfill is rejected intercept[IllegalArgumentException] { registerBackfillOp(tableCommitCoordinatorClient, log, 10) } assertInvariants(logPath, tableCommitCoordinatorClient) } } test("startVersion and endVersion are respected in getCommits") { def runGetCommitsAndValidate( client: TableCommitCoordinatorClient, startVersion: Option[Long], endVersion: Option[Long], maxVersion: Long): Unit = { val result = client.getCommits(startVersion, endVersion) validateGetCommitsResult(result, startVersion, endVersion, maxVersion) } withTempTableDir { tempDir => // prepare a table with 15 commits val log = DeltaLog.forTable(spark, tempDir.toString) val logPath = log.logPath val tableCommitCoordinatorClient = createTableCommitCoordinatorClient(log) writeCommitZero(logPath) val maxVersion = 15 (1 to maxVersion).foreach { version => commit(version, version, tableCommitCoordinatorClient) } runGetCommitsAndValidate(tableCommitCoordinatorClient, None, None, maxVersion) runGetCommitsAndValidate(tableCommitCoordinatorClient, Some(9), None, maxVersion) runGetCommitsAndValidate(tableCommitCoordinatorClient, Some(11), Some(14), maxVersion) runGetCommitsAndValidate(tableCommitCoordinatorClient, Some(12), Some(12), maxVersion) runGetCommitsAndValidate(tableCommitCoordinatorClient, None, Some(14), maxVersion) } } test("test out-of-order backfills are rejected") { withTempTableDir { tempDir => val log = DeltaLog.forTable(spark, tempDir.getPath) val logPath = log.logPath val tableCommitCoordinatorClient = createTableCommitCoordinatorClient(log) // commit-0 must be file system based writeCommitZero(logPath) (1 to 3).foreach(i => commit(i, i, tableCommitCoordinatorClient)) // Test that backfilling is idempotent for already-backfilled commits. registerBackfillOp(tableCommitCoordinatorClient, log, 2) registerBackfillOp(tableCommitCoordinatorClient, log, 2) // Test that backfilling uncommited commits fail. intercept[IllegalArgumentException] { registerBackfillOp(tableCommitCoordinatorClient, log, 4) } } } test("test out-of-order commits are rejected") { withTempTableDir { tempDir => val log = DeltaLog.forTable(spark, tempDir.toString) val logPath = log.logPath val tableCommitCoordinatorClient = createTableCommitCoordinatorClient(log) // commit-0 must be file system based writeCommitZero(logPath) // Verify that conflict-checker rejects out-of-order commits. (1 to 4).foreach(i => commit(i, i, tableCommitCoordinatorClient)) // A retry of commit 0 fails from commit coordinator client with a conflict and it can't be // retried as commit 0 is upgrading the commit coordinator client. assertCommitFail(0, 5, retryable = false, commit(0, 5, tableCommitCoordinatorClient)) assertCommitFail(4, 5, retryable = true, commit(4, 6, tableCommitCoordinatorClient)) commit(5, 5, tableCommitCoordinatorClient) validateGetCommitsResult(tableCommitCoordinatorClient.getCommits(), None, None, 5) assertCommitFail(5, 6, retryable = true, commit(5, 5, tableCommitCoordinatorClient)) assertCommitFail(7, 6, retryable = false, commit(7, 7, tableCommitCoordinatorClient)) assertInvariants(logPath, tableCommitCoordinatorClient) } } test("should handle concurrent readers and writers") { withTempTableDir { tempDir => val tablePath = new Path(tempDir.getCanonicalPath) val logPath = new Path(tablePath, DeltaLog.LOG_DIR_NAME) val tcs = createTableCommitCoordinatorClient(DeltaLog.forTable(spark, tablePath)) val numberOfWriters = 11 val numberOfCommitsPerWriter = 11 // scalastyle:off sparkThreadPools val executor = DeltaThreadPool("commitCoordinatorSuite", numberOfWriters) // scalastyle:on sparkThreadPools val runningTimestamp = new AtomicInteger(0) val commitFailedExceptions = new AtomicInteger(0) // commit-0 must be file system based writeCommitZero(logPath) try { val tasks = (0 until numberOfWriters).map { i => executor.submit(spark) { var currentWriterCommits = 0 while (currentWriterCommits < numberOfCommitsPerWriter) { val nextVersion = math.max(tcs.getCommits().getLatestTableVersion + 1, 1) try { val currentTimestamp = runningTimestamp.getAndIncrement() val commitResponse = commit(nextVersion, currentTimestamp, tcs) currentWriterCommits += 1 assert(commitResponse.getCommitTimestamp == currentTimestamp) assert(commitResponse.getVersion == nextVersion) } catch { case e: JCommitFailedException => assert(e.getConflict) assert(e.getRetryable) commitFailedExceptions.getAndIncrement() } finally { assertInvariants(logPath, tcs) } } } } tasks.foreach(ThreadUtils.awaitResult(_, 150.seconds)) } catch { case e: InterruptedException => fail("Test interrupted: " + e.getMessage) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CommitCoordinatorClientSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import java.util.Optional import scala.collection.JavaConverters._ import scala.reflect.runtime.universe._ import org.apache.spark.sql.delta.{CoordinatedCommitsTableFeature, DeltaConfigs, DeltaLog, DeltaOperations} import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import io.delta.storage.LogStore import io.delta.storage.commit.{CommitCoordinatorClient, CommitResponse, GetCommitsResponse => JGetCommitsResponse, TableDescriptor, TableIdentifier, UpdatedActions} import io.delta.storage.commit.actions.{AbstractMetadata, AbstractProtocol} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.sql.{QueryTest, SparkSession} import org.apache.spark.sql.test.SharedSparkSession class CommitCoordinatorClientSuite extends QueryTest with DeltaSQLTestUtils with SharedSparkSession with DeltaSQLCommandTest { private trait TestCommitCoordinatorClientBase extends CommitCoordinatorClient { override def commit( logStore: LogStore, hadoopConf: Configuration, tableDesc: TableDescriptor, commitVersion: Long, actions: java.util.Iterator[String], updatedActions: UpdatedActions): CommitResponse = { throw new UnsupportedOperationException("Not implemented") } override def getCommits( tableDesc: TableDescriptor, startVersion: java.lang.Long, endVersion: java.lang.Long): JGetCommitsResponse = new JGetCommitsResponse(Seq.empty.asJava, -1) override def backfillToVersion( logStore: LogStore, hadoopConf: Configuration, tableDesc: TableDescriptor, version: Long, lastKnownBackfilledVersion: java.lang.Long): Unit = {} override def registerTable( logPath: Path, tableIdentifier: Optional[TableIdentifier], currentVersion: Long, currentMetadata: AbstractMetadata, currentProtocol: AbstractProtocol): java.util.Map[String, String] = Map.empty[String, String].asJava override def semanticEquals(other: CommitCoordinatorClient): Boolean = this == other } private class TestCommitCoordinatorClient1 extends TestCommitCoordinatorClientBase private class TestCommitCoordinatorClient2 extends TestCommitCoordinatorClientBase override def beforeEach(): Unit = { super.beforeEach() CommitCoordinatorProvider.clearNonDefaultBuilders() CommitCoordinatorProvider.registerBuilder(InMemoryCommitCoordinatorBuilder(batchSize = 1)) } test("registering multiple commit-coordinator builders with same name") { object Builder1 extends CommitCoordinatorBuilder { override def build( spark: SparkSession, conf: Map[String, String]): CommitCoordinatorClient = null override def getName: String = "builder-1" } object BuilderWithSameName extends CommitCoordinatorBuilder { override def build( spark: SparkSession, conf: Map[String, String]): CommitCoordinatorClient = null override def getName: String = "builder-1" } object Builder3 extends CommitCoordinatorBuilder { override def build( spark: SparkSession, conf: Map[String, String]): CommitCoordinatorClient = null override def getName: String = "builder-3" } CommitCoordinatorProvider.registerBuilder(Builder1) intercept[Exception] { CommitCoordinatorProvider.registerBuilder(BuilderWithSameName) } CommitCoordinatorProvider.registerBuilder(Builder3) } test("getCommitCoordinator - builder returns same object") { object Builder1 extends CommitCoordinatorBuilder { val cs1 = new TestCommitCoordinatorClient1() val cs2 = new TestCommitCoordinatorClient2() override def build( spark: SparkSession, conf: Map[String, String]): CommitCoordinatorClient = { conf.getOrElse("url", "") match { case "url1" => cs1 case "url2" => cs2 case _ => throw new IllegalArgumentException("Invalid url") } } override def getName: String = "cs-x" } CommitCoordinatorProvider.registerBuilder(Builder1) val cs1 = CommitCoordinatorProvider.getCommitCoordinatorClient("cs-x", Map("url" -> "url1"), spark) assert(cs1.isInstanceOf[TestCommitCoordinatorClient1]) val cs1_again = CommitCoordinatorProvider.getCommitCoordinatorClient("cs-x", Map("url" -> "url1"), spark) assert(cs1 eq cs1_again) val cs2 = CommitCoordinatorProvider .getCommitCoordinatorClient("cs-x", Map("url" -> "url2", "a" -> "b"), spark) assert(cs2.isInstanceOf[TestCommitCoordinatorClient2]) // If builder receives a config which doesn't have expected params, then it can throw exception. intercept[IllegalArgumentException] { CommitCoordinatorProvider.getCommitCoordinatorClient("cs-x", Map("url" -> "url3"), spark) } } test("getCommitCoordinatorClient - builder returns new object each time") { object Builder1 extends CommitCoordinatorBuilder { override def build( spark: SparkSession, conf: Map[String, String]): CommitCoordinatorClient = { conf.getOrElse("url", "") match { case "url1" => new TestCommitCoordinatorClient1() case _ => throw new IllegalArgumentException("Invalid url") } } override def getName: String = "cs-name" } CommitCoordinatorProvider.registerBuilder(Builder1) val cs1 = CommitCoordinatorProvider.getCommitCoordinatorClient("cs-name", Map("url" -> "url1"), spark) assert(cs1.isInstanceOf[TestCommitCoordinatorClient1]) val cs1_again = CommitCoordinatorProvider.getCommitCoordinatorClient("cs-name", Map("url" -> "url1"), spark) assert(cs1 ne cs1_again) } test("COORDINATED_COMMITS_PROVIDER_CONF") { val m1 = Metadata( configuration = Map( DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF.key -> """{"key1": "string_value", "key2Int": 2, "key3ComplexStr": "\"hello\""}""") ) assert(DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF.fromMetaData(m1) === Map("key1" -> "string_value", "key2Int" -> "2", "key3ComplexStr" -> "\"hello\"")) val m2 = Metadata( configuration = Map( DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF.key -> """{"key1": "string_value", "key2Int": "2""") ) intercept[com.fasterxml.jackson.core.JsonParseException] { DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF.fromMetaData(m2) } } test("Commit fails if we try to put bad value for COORDINATED_COMMITS_PROVIDER_CONF") { withTempDir { dir => val path = dir.getCanonicalPath spark.range(10).write.format("delta").mode("append").save(path) val deltaLog = DeltaLog.forTable(spark, path) val metadataWithCorrectConf = Metadata( configuration = Map( DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF.key -> """{"key1": "string_value", "key2Int": 2, "key3ComplexStr": "\"hello\""}""") ) val metadataWithIncorrectConf = Metadata( configuration = Map( DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF.key -> """{"key1": "string_value", "key2Int": "2""") ) intercept[com.fasterxml.jackson.core.JsonParseException] { deltaLog.startTransaction().commit( Seq(metadataWithIncorrectConf), DeltaOperations.ManualUpdate) } DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF.fromMetaData(metadataWithCorrectConf) } } test( "Adding COORDINATED_COMMITS_PROVIDER_NAME table property automatically upgrades the Protocol") { withTempDir { dir => val path = dir.getCanonicalPath spark.range(10).write.format("delta").mode("append").save(path) val metadata = Metadata( configuration = Map(DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.key -> "in-memory")) val deltaLog = DeltaLog.forTable(spark, path) def getWriterFeatures(log: DeltaLog): Set[String] = { log.update().protocol.writerFeatures.getOrElse(Set.empty) } assert(!getWriterFeatures(deltaLog).contains(CoordinatedCommitsTableFeature.name)) deltaLog.startTransaction().commit(Seq(metadata), DeltaOperations.ManualUpdate) assert(getWriterFeatures(deltaLog).contains(CoordinatedCommitsTableFeature.name)) } } test("Semantic Equality works as expected on CommitCoordinatorClients") { class TestCommitCoordinatorClient(val key: String) extends TestCommitCoordinatorClientBase { override def semanticEquals(other: CommitCoordinatorClient): Boolean = other.isInstanceOf[TestCommitCoordinatorClient] && other.asInstanceOf[TestCommitCoordinatorClient].key == key } object Builder1 extends CommitCoordinatorBuilder { override def build( spark: SparkSession, conf: Map[String, String]): CommitCoordinatorClient = { new TestCommitCoordinatorClient(conf("key")) } override def getName: String = "cs-name" } CommitCoordinatorProvider.registerBuilder(Builder1) // Different CommitCoordinator with same keys should be semantically equal. val obj1 = CommitCoordinatorProvider.getCommitCoordinatorClient("cs-name", Map("key" -> "url1"), spark) val obj2 = CommitCoordinatorProvider.getCommitCoordinatorClient("cs-name", Map("key" -> "url1"), spark) assert(obj1 != obj2) assert(obj1.semanticEquals(obj2)) // Different CommitCoordinator with different keys should be semantically unequal. val obj3 = CommitCoordinatorProvider.getCommitCoordinatorClient("cs-name", Map("key" -> "url2"), spark) assert(obj1 != obj3) assert(!obj1.semanticEquals(obj3)) } private def checkMissing[Interface: TypeTag, Class: TypeTag](): Set[String] = { val fields = typeOf[Class].decls.collect { case m: MethodSymbol if m.isCaseAccessor => m.name.toString } val getters = typeOf[Interface].decls.collect { case m: MethodSymbol if m.isAbstract => m.name.toString }.toSet fields.filterNot { field => getters.contains(s"get${field.capitalize}") }.toSet } /** * We expect the Protocol action to have the same fields as AbstractProtocol (part of the * CommitCoordinatorClient interface). With this if any change has happened in the Protocol of the * table, the same change is propagated to the CommitCoordinatorClient as AbstractProtocol. The * CommitCoordinatorClient can access the changes using getters and decide to act on the changes * based on the spec of the commit coordinator. * * This test case ensures that any new field added in the Protocol action is also accessible in * the CommitCoordinatorClient via the getter. If the new field is something which we do not * expect to be passed to the CommitCoordinatorClient, the test needs to be modified accordingly. */ test("AbstractProtocol should have getter methods for all fields in Protocol") { val missingFields = checkMissing[AbstractProtocol, Protocol]() val expectedMissingFields = Set.empty[String] assert(missingFields == expectedMissingFields, s"Missing getter methods in AbstractProtocol") } /** * We expect the Metadata action to have the same fields as AbstractMetadata (part of the * CommitCoordinatorClient interface). With this if any change has happened in the Metadata of the * table, the same change is propagated to the CommitCoordinatorClient as AbstractMetadata. The * CommitCoordinatorClient can access the changes using getters and decide to act on the changes * based on the spec of the commit coordinator. * * This test case ensures that any new field added in the Metadata action is also accessible in * the CommitCoordinatorClient via the getter. If the new field is something which we do not * expect to be passed to the CommitCoordinatorClient, the test needs to be modified accordingly. */ test("BaseMetadata should have getter methods for all fields in Metadata") { val missingFields = checkMissing[AbstractMetadata, Metadata]() val expectedMissingFields = Set("format") assert(missingFields == expectedMissingFields, s"Missing getter methods in AbstractMetadata") } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsEnablementSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} class CoordinatedCommitsEnablementSuite extends CoordinatedCommitsBaseSuite with DeltaSQLTestUtils with DeltaSQLCommandTest with CoordinatedCommitsTestUtils { override def coordinatedCommitsBackfillBatchSize: Option[Int] = Some(3) import testImplicits._ private def validateCoordinatedCommitsCompleteEnablement( snapshot: Snapshot, expectEnabled: Boolean): Unit = { assert( DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.fromMetaData(snapshot.metadata).isDefined == expectEnabled) Seq( CoordinatedCommitsTableFeature, VacuumProtocolCheckTableFeature, InCommitTimestampTableFeature) .foreach { feature => assert(snapshot.protocol.writerFeatures.exists(_.contains(feature.name)) == expectEnabled) } assert( snapshot.protocol.readerFeatures.exists(_.contains(VacuumProtocolCheckTableFeature.name)) == expectEnabled) assert(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(snapshot.metadata) == expectEnabled) } // ---- Tests START: Enablement at commit 0 ---- test("enablement at commit 0: CC should enable ICT and VacuumProtocolCheck" + " --- writeintodelta api") { withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath Seq(1).toDF().write.format("delta").mode("overwrite").save(tablePath) val log = DeltaLog.forTable(spark, tablePath) validateCoordinatedCommitsCompleteEnablement(log.snapshot, expectEnabled = true) } } test("enablement at commit 0: CC should enable ICT and VacuumProtocolCheck" + " --- simple create table") { withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath sql(s"CREATE TABLE delta.`$tablePath` (id LONG) USING delta") val log = DeltaLog.forTable(spark, tablePath) validateCoordinatedCommitsCompleteEnablement(log.snapshot, expectEnabled = true) } } test("enablement at commit 0: CC should enable ICT and VacuumProtocolCheck" + " --- create or replace") { withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath Seq(1).toDF().write.format("delta").mode("overwrite").save(tablePath) Seq(1).toDF().write.format("delta").mode("overwrite").save(tablePath) val log = DeltaLog.forTable(spark, tablePath) validateCoordinatedCommitsCompleteEnablement(log.snapshot, expectEnabled = true) } } // ---- Tests END: Enablement at commit 0 ---- // ---- Tests START: Enablement after commit 0 ---- testWithDefaultCommitCoordinatorUnset( "enablement after commit 0: CC should enable ICT and VacuumProtocolCheck" + " --- update tblproperty") { withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath Seq(1).toDF().write.format("delta").mode("overwrite").save(tablePath) // commit 0 Seq(1).toDF().write.format("delta").mode("append").save(tablePath) // commit 1 val log = DeltaLog.forTable(spark, tablePath) validateCoordinatedCommitsCompleteEnablement(log.snapshot, expectEnabled = false) sql(s"ALTER TABLE delta.`$tablePath` SET TBLPROPERTIES " + // Enable CC s"('${DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.key}' = 'tracking-in-memory', " + s"'${DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF.key}' = '{}')") Seq(1).toDF().write.format("delta").mode("overwrite").save(tablePath) // commit 3 validateCoordinatedCommitsCompleteEnablement(log.update(), expectEnabled = true) } } // ---- Tests END: Enablement after commit 0 ---- } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsPropertySuiteBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import org.apache.spark.sql.delta.{DeltaIllegalArgumentException, DeltaLog} import org.apache.spark.sql.delta.DeltaConfigs.{COORDINATED_COMMITS_COORDINATOR_CONF, COORDINATED_COMMITS_COORDINATOR_NAME, COORDINATED_COMMITS_TABLE_CONF} import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.util.JsonUtils import io.delta.storage.commit.CommitCoordinatorClient import org.apache.spark.sql.{QueryTest, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.test.SharedSparkSession trait CoordinatedCommitsPropertySuiteBase extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with CoordinatedCommitsTestUtils { private def getRandomTableName: String = scala.util.Random.alphanumeric.take(10).mkString("") override def beforeEach(): Unit = { super.beforeEach() target = getRandomTableName source = getRandomTableName CommitCoordinatorProvider.clearNonDefaultBuilders() CommitCoordinatorProvider.registerBuilder(CommitCoordinatorBuilder1()) CommitCoordinatorProvider.registerBuilder(CommitCoordinatorBuilder2()) } protected val command: String protected val cc1: String = "commit-coordinator-1" private case class CommitCoordinatorBuilder1() extends CommitCoordinatorBuilder { private val commitCoordinator = new InMemoryCommitCoordinator(batchSize = 1000L) override def getName: String = cc1 override def build(spark: SparkSession, conf: Map[String, String]): CommitCoordinatorClient = commitCoordinator } protected val cc2: String = "commit-coordinator-2" private case class CommitCoordinatorBuilder2() extends CommitCoordinatorBuilder { private val commitCoordinator = new InMemoryCommitCoordinator(batchSize = 1000L) override def getName: String = cc2 override def build(spark: SparkSession, conf: Map[String, String]): CommitCoordinatorClient = commitCoordinator } protected var target: String = getRandomTableName protected var source: String = getRandomTableName protected val coordinatorNameKey: String = COORDINATED_COMMITS_COORDINATOR_NAME.key protected val coordinatorConfKey: String = COORDINATED_COMMITS_COORDINATOR_CONF.key protected val tableConfKey: String = COORDINATED_COMMITS_TABLE_CONF.key protected val coordinatorNameDefaultKey: String = COORDINATED_COMMITS_COORDINATOR_NAME.defaultTablePropertyKey protected val coordinatorConfDefaultKey: String = COORDINATED_COMMITS_COORDINATOR_CONF.defaultTablePropertyKey protected val tableConfDefaultKey: String = COORDINATED_COMMITS_TABLE_CONF.defaultTablePropertyKey protected val randomCoordinatorConf: String = JsonUtils.toJson(Map("randomCoordinatorConf" -> "randomCoordinatorConfValue")) protected val randomTableConf: String = JsonUtils.toJson(Map("randomTableConf" -> "randomTableConfValue")) def getCCPropertiesClause(properties: Seq[(String, String)]): String = { if (properties.nonEmpty) { " TBLPROPERTIES (" + properties.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ") + ")" } else { "" } } def verifyCommitCoordinator(table: String, expectedCoordinator: Option[String]): Unit = { assert(DeltaLog.forTable(spark, TableIdentifier(table)) .update().metadata.coordinatedCommitsCoordinatorName == expectedCoordinator) } def testImpl( commandConfs: Seq[(String, String)] = Seq(), defaultConfs: Seq[(String, String)] = Seq(), targetConfs: Seq[(String, String)] = Seq(), sourceConfs: Seq[(String, String)] = Seq(), expectedCoordinator: Option[String] = None): Unit } trait CoordinatedCommitsPropertyCreateTableSuiteBase extends CoordinatedCommitsPropertySuiteBase { test("Commit coordinators are picked from command specification.") { testImpl( commandConfs = Seq( coordinatorNameKey -> cc1, coordinatorConfKey -> randomCoordinatorConf), expectedCoordinator = Some(cc1)) } test("Commit coordinators are picked from default configurations if not specified in command.") { testImpl( defaultConfs = Seq( coordinatorNameDefaultKey -> cc1, coordinatorConfDefaultKey -> randomCoordinatorConf), expectedCoordinator = Some(cc1)) } test("Command-specified commit coordinators take precedence over default configurations.") { testImpl( commandConfs = Seq( coordinatorNameKey -> cc1, coordinatorConfKey -> randomCoordinatorConf), defaultConfs = Seq( coordinatorNameDefaultKey -> cc2, coordinatorConfDefaultKey -> randomCoordinatorConf), expectedCoordinator = Some(cc1)) } test("Illegal command-specified property combinations throw an exception.") { var e = intercept[DeltaIllegalArgumentException] { testImpl( commandConfs = Seq(coordinatorNameKey -> cc1)) } checkError( exception = e, "DELTA_MUST_SET_ALL_COORDINATED_COMMITS_CONFS_IN_COMMAND", sqlState = "42616", parameters = Map("command" -> command, "configuration" -> coordinatorConfKey)) e = intercept[DeltaIllegalArgumentException] { testImpl( commandConfs = Seq(coordinatorConfKey -> randomCoordinatorConf)) } checkError( exception = e, "DELTA_MUST_SET_ALL_COORDINATED_COMMITS_CONFS_IN_COMMAND", sqlState = "42616", parameters = Map("command" -> command, "configuration" -> coordinatorNameKey)) e = intercept[DeltaIllegalArgumentException] { testImpl( commandConfs = Seq( coordinatorNameKey -> cc1, coordinatorConfKey -> randomCoordinatorConf, tableConfKey -> randomTableConf)) } checkError( exception = e, "DELTA_CONF_OVERRIDE_NOT_SUPPORTED_IN_COMMAND", sqlState = "42616", parameters = Map("command" -> command, "configuration" -> tableConfKey)) } test("Illegal default property combinations throw an exception if none specified in command.") { var e = intercept[DeltaIllegalArgumentException] { testImpl( defaultConfs = Seq(coordinatorNameDefaultKey -> cc1)) } checkError( exception = e, "DELTA_MUST_SET_ALL_COORDINATED_COMMITS_CONFS_IN_SESSION", sqlState = "42616", parameters = Map("command" -> command, "configuration" -> coordinatorConfDefaultKey)) e = intercept[DeltaIllegalArgumentException] { testImpl( defaultConfs = Seq(coordinatorConfDefaultKey -> randomCoordinatorConf)) } checkError( exception = e, "DELTA_MUST_SET_ALL_COORDINATED_COMMITS_CONFS_IN_SESSION", sqlState = "42616", parameters = Map("command" -> command, "configuration" -> coordinatorNameDefaultKey)) e = intercept[DeltaIllegalArgumentException] { testImpl( defaultConfs = Seq( coordinatorNameDefaultKey -> cc1, coordinatorConfDefaultKey -> randomCoordinatorConf, tableConfDefaultKey -> randomTableConf)) } checkError( exception = e, "DELTA_CONF_OVERRIDE_NOT_SUPPORTED_IN_SESSION", sqlState = "42616", parameters = Map("command" -> command, "configuration" -> tableConfDefaultKey)) } test("Illegal default property combinations are ignored if command specifications are valid.") { testImpl( commandConfs = Seq( coordinatorNameKey -> cc1, coordinatorConfKey -> randomCoordinatorConf), defaultConfs = Seq( coordinatorNameDefaultKey -> cc2, coordinatorConfDefaultKey -> randomCoordinatorConf, tableConfDefaultKey -> randomTableConf), expectedCoordinator = Some(cc1)) } test("Illegal command-specified property combinations throw an exception even if default " + "configurations are valid.") { val e = intercept[DeltaIllegalArgumentException] { testImpl( commandConfs = Seq( coordinatorNameKey -> cc1, coordinatorConfKey -> randomCoordinatorConf, tableConfKey -> randomTableConf), defaultConfs = Seq( coordinatorNameDefaultKey -> cc2, coordinatorConfDefaultKey -> randomCoordinatorConf)) } checkError( exception = e, "DELTA_CONF_OVERRIDE_NOT_SUPPORTED_IN_COMMAND", sqlState = "42616", parameters = Map("command" -> command, "configuration" -> tableConfKey)) } } class CoordinatedCommitsPropertyCreateTableSuite extends CoordinatedCommitsPropertyCreateTableSuiteBase { override protected val command: String = "CREATE" override def testImpl( commandConfs: Seq[(String, String)], defaultConfs: Seq[(String, String)], targetConfs: Seq[(String, String)], sourceConfs: Seq[(String, String)], expectedCoordinator: Option[String]): Unit = { withTable(target) { withSQLConf(defaultConfs: _*) { sql(s"CREATE TABLE $target (id LONG) USING delta" + getCCPropertiesClause(commandConfs)) } verifyCommitCoordinator(target, expectedCoordinator) } } } class CoordinatedCommitsPropertyCreateTableAsSelectSuite extends CoordinatedCommitsPropertyCreateTableSuiteBase { override protected val command: String = "CREATE" override def testImpl( commandConfs: Seq[(String, String)], defaultConfs: Seq[(String, String)], targetConfs: Seq[(String, String)], sourceConfs: Seq[(String, String)], expectedCoordinator: Option[String]): Unit = { withTable(target, source) { sql(s"CREATE TABLE $source (id LONG) USING delta") sql(s"INSERT INTO $source VALUES (1)") withSQLConf(defaultConfs: _*) { sql(s"CREATE TABLE $target USING delta" + getCCPropertiesClause(commandConfs) + s" AS SELECT * FROM $source") } verifyCommitCoordinator(target, expectedCoordinator) } } } class CoordinatedCommitsPropertyCreateTableWithShallowCloneSuite extends CoordinatedCommitsPropertyCreateTableSuiteBase { override protected val command: String = "CREATE with CLONE" override def testImpl( commandConfs: Seq[(String, String)] = Seq(), defaultConfs: Seq[(String, String)] = Seq(), targetConfs: Seq[(String, String)] = Seq(), sourceConfs: Seq[(String, String)] = Seq(), expectedCoordinator: Option[String] = None): Unit = { withTable(target, source) { sql(s"CREATE TABLE $source (id LONG) USING delta" + getCCPropertiesClause(sourceConfs)) withSQLConf(defaultConfs: _*) { sql(s"CREATE TABLE $target SHALLOW CLONE $source" + getCCPropertiesClause(commandConfs)) } verifyCommitCoordinator(target, expectedCoordinator) } } test("Source table's commit coordinator should never be copied to the target table: no commit " + "coordinators are specified") { testImpl( sourceConfs = Seq( coordinatorNameKey -> cc1, coordinatorConfKey -> randomCoordinatorConf), expectedCoordinator = None) } test("Source table's commit coordinator should never be copied to the target table: command " + "specifies a commit coordinator") { testImpl( commandConfs = Seq( coordinatorNameKey -> cc1, coordinatorConfKey -> randomCoordinatorConf), sourceConfs = Seq( coordinatorNameKey -> cc2, coordinatorConfKey -> randomCoordinatorConf), expectedCoordinator = Some(cc1)) } test("Source table's commit coordinator should never be copied to the target table: default " + "configurations specify a commit coordinator") { testImpl( defaultConfs = Seq( coordinatorNameDefaultKey -> cc1, coordinatorConfDefaultKey -> randomCoordinatorConf), sourceConfs = Seq( coordinatorNameKey -> cc2, coordinatorConfKey -> randomCoordinatorConf), expectedCoordinator = Some(cc1)) } } trait CoordinatedCommitsPropertyReplaceTableSuiteBase extends CoordinatedCommitsPropertySuiteBase { test("Any command-specified Coordinated Commits overrides throw an exception") { var e = intercept[DeltaIllegalArgumentException] { testImpl( commandConfs = Seq( coordinatorNameKey -> cc1, coordinatorConfKey -> randomCoordinatorConf)) } checkError( exception = e, "DELTA_CANNOT_OVERRIDE_COORDINATED_COMMITS_CONFS", sqlState = "42616", parameters = Map("Command" -> command)) e = intercept[DeltaIllegalArgumentException] { testImpl( commandConfs = Seq( coordinatorNameKey -> cc1, coordinatorConfKey -> randomCoordinatorConf, tableConfKey -> randomTableConf)) } checkError( exception = e, "DELTA_CANNOT_OVERRIDE_COORDINATED_COMMITS_CONFS", sqlState = "42616", parameters = Map("Command" -> command)) } test("Default Coordinated Commits configurations from SparkSession are ignored") { testImpl( defaultConfs = Seq( coordinatorNameDefaultKey -> cc1, coordinatorConfDefaultKey -> randomCoordinatorConf), expectedCoordinator = None) testImpl( defaultConfs = Seq( coordinatorNameDefaultKey -> cc1, coordinatorConfDefaultKey -> randomCoordinatorConf, tableConfDefaultKey -> randomTableConf), expectedCoordinator = None) } test("Existing Coordinated Commits configurations from the target table are retained.") { testImpl( targetConfs = Seq( coordinatorNameKey -> cc1, coordinatorConfKey -> randomCoordinatorConf), expectedCoordinator = Some(cc1)) } } class CoordinatedCommitsPropertyReplaceTableSuite extends CoordinatedCommitsPropertyReplaceTableSuiteBase { override protected val command: String = "REPLACE" override def testImpl( commandConfs: Seq[(String, String)], defaultConfs: Seq[(String, String)], targetConfs: Seq[(String, String)], sourceConfs: Seq[(String, String)], expectedCoordinator: Option[String]): Unit = { withTable(target) { sql(s"CREATE TABLE $target (id LONG) USING delta" + getCCPropertiesClause(targetConfs)) withSQLConf(defaultConfs: _*) { sql(s"REPLACE TABLE $target (id STRING) USING delta" + getCCPropertiesClause(commandConfs)) } verifyCommitCoordinator(target, expectedCoordinator) } } } class CoordinatedCommitsPropertyReplaceTableAsSelectSuite extends CoordinatedCommitsPropertyReplaceTableSuiteBase { override protected val command: String = "REPLACE" override def testImpl( commandConfs: Seq[(String, String)], defaultConfs: Seq[(String, String)], targetConfs: Seq[(String, String)], sourceConfs: Seq[(String, String)], expectedCoordinator: Option[String]): Unit = { withTable(target, source) { sql(s"CREATE TABLE $source (id LONG) USING delta") sql(s"INSERT INTO $source VALUES (1)") sql(s"CREATE TABLE $target (id LONG) USING delta" + getCCPropertiesClause(targetConfs)) withSQLConf(defaultConfs: _*) { sql(s"REPLACE TABLE $target USING delta" + getCCPropertiesClause(commandConfs) + s" AS SELECT * FROM $source") } verifyCommitCoordinator(target, expectedCoordinator) } } } class CoordinatedCommitsPropertyReplaceTableWithShallowCloneSuite extends CoordinatedCommitsPropertyReplaceTableSuiteBase { override protected val command: String = "REPLACE with CLONE" override def testImpl( commandConfs: Seq[(String, String)] = Seq(), defaultConfs: Seq[(String, String)] = Seq(), targetConfs: Seq[(String, String)] = Seq(), sourceConfs: Seq[(String, String)] = Seq(), expectedCoordinator: Option[String] = None): Unit = { withTable(target, source) { sql(s"CREATE TABLE $target (id LONG) USING delta" + getCCPropertiesClause(targetConfs)) sql(s"CREATE TABLE $source (id LONG) USING delta" + getCCPropertiesClause(sourceConfs)) withSQLConf(defaultConfs: _*) { sql(s"REPLACE TABLE $target SHALLOW CLONE $source" + getCCPropertiesClause(commandConfs)) } verifyCommitCoordinator(target, expectedCoordinator) } } test("Source table's commit coordinator should never be copied to the target table: target " + "table does not have any coordinator") { testImpl( sourceConfs = Seq( coordinatorNameKey -> cc1, coordinatorConfKey -> randomCoordinatorConf), expectedCoordinator = None) } test("Source table's commit coordinator should never be copied to the target table: target " + "table has a coordinator") { testImpl( targetConfs = Seq( coordinatorNameKey -> cc1, coordinatorConfKey -> randomCoordinatorConf), sourceConfs = Seq( coordinatorNameKey -> cc2, coordinatorConfKey -> randomCoordinatorConf), expectedCoordinator = Some(cc1)) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import java.io.File import java.lang.{Long => JLong} import java.util.{Iterator => JIterator, Optional} import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer import com.databricks.spark.util.Log4jUsageLogger import com.databricks.spark.util.UsageRecord import org.apache.spark.sql.delta.{CatalogOwnedTableFeature, CheckpointPolicy, CommitCoordinatorGetCommitsFailedException, CommitStats, CoordinatedCommitsStats, CoordinatedCommitsTableFeature, DeltaIllegalArgumentException, DeltaOperations, DeltaTestUtilsBase, DeltaUnsupportedOperationException, V2CheckpointTableFeature} import org.apache.spark.sql.delta.CoordinatedCommitType._ import org.apache.spark.sql.delta.DeltaConfigs import org.apache.spark.sql.delta.DeltaConfigs.{CHECKPOINT_INTERVAL, CHECKPOINT_POLICY, COORDINATED_COMMITS_COORDINATOR_CONF, COORDINATED_COMMITS_COORDINATOR_NAME, COORDINATED_COMMITS_TABLE_CONF, IN_COMMIT_TIMESTAMPS_ENABLED} import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile import org.apache.spark.sql.delta.DummySnapshot import org.apache.spark.sql.delta.LogSegment import org.apache.spark.sql.delta.Snapshot import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaExceptionTestUtils import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import org.apache.spark.sql.delta.util.FileNames.{CompactedDeltaFile, DeltaFile, UnbackfilledDeltaFile} import io.delta.storage.LogStore import io.delta.storage.commit.{CommitCoordinatorClient, CommitResponse, CoordinatedCommitsUtils => JCoordinatedCommitsUtils, GetCommitsResponse => JGetCommitsResponse, TableDescriptor, TableIdentifier, UpdatedActions} import io.delta.storage.commit.actions.{AbstractMetadata, AbstractProtocol} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.SparkConf import org.apache.spark.sql.{QueryTest, Row, SparkSession} import org.apache.spark.sql.catalyst.{TableIdentifier => CatalystTableIdentifier} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.ManualClock class CoordinatedCommitsSuite extends CommitCoordinatorSuiteBase with CoordinatedCommitsBaseSuite { import testImplicits._ override def sparkConf: SparkConf = { // Make sure all new tables in tests use tracking-in-memory commit-coordinator by default. super.sparkConf .set(COORDINATED_COMMITS_COORDINATOR_NAME.defaultTablePropertyKey, "tracking-in-memory") .set(COORDINATED_COMMITS_COORDINATOR_CONF.defaultTablePropertyKey, JsonUtils.toJson(Map())) } test("helper method that recovers config from abstract metadata works properly") { val m1 = Metadata( configuration = Map(COORDINATED_COMMITS_COORDINATOR_NAME.key -> "string_value") ) assert(JCoordinatedCommitsUtils.getCoordinatorName(m1) === Optional.of("string_value")) val m2 = Metadata( configuration = Map(COORDINATED_COMMITS_COORDINATOR_NAME.key -> "") ) assert(JCoordinatedCommitsUtils.getCoordinatorName(m2)=== Optional.of("")) val m3 = Metadata( configuration = Map( COORDINATED_COMMITS_COORDINATOR_CONF.key -> """{"key1": "string_value", "key2Int": 2, "key3ComplexStr": "\"hello\""}""") ) assert(JCoordinatedCommitsUtils.getCoordinatorConf(m3) === Map("key1" -> "string_value", "key2Int" -> "2", "key3ComplexStr" -> "\"hello\"").asJava) val m4 = Metadata() assert(JCoordinatedCommitsUtils.getCoordinatorConf(m4) === Map.empty.asJava) } test("During ALTER, overriding Coordinated Commits configurations throws an exception.") { registerBuilder(TrackingInMemoryCommitCoordinatorBuilder(1)) registerBuilder(InMemoryCommitCoordinatorBuilder(1)) withTempDir { tempDir => sql(s"CREATE TABLE delta.`${tempDir.getAbsolutePath}` (id LONG) USING delta TBLPROPERTIES " + s"('${COORDINATED_COMMITS_COORDINATOR_NAME.key}' = 'tracking-in-memory', " + s"'${COORDINATED_COMMITS_COORDINATOR_CONF.key}' = '${JsonUtils.toJson(Map())}')") val e = interceptWithUnwrapping[DeltaIllegalArgumentException] { sql(s"ALTER TABLE delta.`${tempDir.getAbsolutePath}` SET TBLPROPERTIES " + s"('${COORDINATED_COMMITS_COORDINATOR_NAME.key}' = 'in-memory', " + s"'${COORDINATED_COMMITS_COORDINATOR_CONF.key}' = '${JsonUtils.toJson(Map())}')") } checkError( e, "DELTA_CANNOT_OVERRIDE_COORDINATED_COMMITS_CONFS", sqlState = "42616", parameters = Map("Command" -> "ALTER")) } } test("During ALTER, unsetting Coordinated Commits configurations throws an exception.") { registerBuilder(TrackingInMemoryCommitCoordinatorBuilder(1)) withTempDir { tempDir => sql(s"CREATE TABLE delta.`${tempDir.getAbsolutePath}` (id LONG) USING delta TBLPROPERTIES " + s"('${COORDINATED_COMMITS_COORDINATOR_NAME.key}' = 'tracking-in-memory', " + s"'${COORDINATED_COMMITS_COORDINATOR_CONF.key}' = '${JsonUtils.toJson(Map())}')") val e = interceptWithUnwrapping[DeltaIllegalArgumentException] { sql(s"ALTER TABLE delta.`${tempDir.getAbsolutePath}` UNSET TBLPROPERTIES " + s"('${COORDINATED_COMMITS_COORDINATOR_NAME.key}', " + s"'${COORDINATED_COMMITS_COORDINATOR_CONF.key}')") } checkError( e, "DELTA_CANNOT_UNSET_COORDINATED_COMMITS_CONFS", sqlState = "42616", parameters = Map[String, String]()) } } test("tableConf returned from registration API is recorded in deltaLog and passed " + "to CommitCoordinatorClient in future for all the APIs") { val tableConf = Map("tableID" -> "random-u-u-i-d", "1" -> "2").asJava val trackingCommitCoordinatorClient = new TrackingCommitCoordinatorClient( new InMemoryCommitCoordinator(batchSize = 10) { override def registerTable( logPath: Path, tableIdentifier: Optional[TableIdentifier], currentVersion: Long, currentMetadata: AbstractMetadata, currentProtocol: AbstractProtocol): java.util.Map[String, String] = { super.registerTable( logPath, tableIdentifier, currentVersion, currentMetadata, currentProtocol) tableConf } override def getCommits( tableDesc: TableDescriptor, startVersion: java.lang.Long, endVersion: java.lang.Long): JGetCommitsResponse = { assert(tableDesc.getTableConf === tableConf) super.getCommits(tableDesc, startVersion, endVersion) } override def commit( logStore: LogStore, hadoopConf: Configuration, tableDesc: TableDescriptor, commitVersion: Long, actions: java.util.Iterator[String], updatedActions: UpdatedActions): CommitResponse = { assert(tableDesc.getTableConf === tableConf) super.commit(logStore, hadoopConf, tableDesc, commitVersion, actions, updatedActions) } override def backfillToVersion( logStore: LogStore, hadoopConf: Configuration, tableDesc: TableDescriptor, version: Long, lastKnownBackfilledVersionOpt: java.lang.Long): Unit = { assert(tableDesc.getTableConf === tableConf) super.backfillToVersion( logStore, hadoopConf, tableDesc, version, lastKnownBackfilledVersionOpt) } } ) val builder = TrackingInMemoryCommitCoordinatorBuilder( batchSize = 10, Some(trackingCommitCoordinatorClient)) registerBuilder(builder) withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath val log = DeltaLog.forTable(spark, tablePath) val commitCoordinatorConf = Map(COORDINATED_COMMITS_COORDINATOR_NAME.key -> builder.getName) val newMetadata = Metadata().copy(configuration = commitCoordinatorConf) log.startTransaction().commitManually(newMetadata) assert(log.unsafeVolatileSnapshot.version === 0) assert(log.unsafeVolatileSnapshot.metadata.coordinatedCommitsTableConf.asJava === tableConf) log.startTransaction().commitManually(createTestAddFile("f1")) log.startTransaction().commitManually(createTestAddFile("f2")) log.checkpoint() log.startTransaction().commitManually(createTestAddFile("f2")) assert(trackingCommitCoordinatorClient.numCommitsCalled.get > 0) assert(trackingCommitCoordinatorClient.numGetCommitsCalled.get > 0) assert(trackingCommitCoordinatorClient.numBackfillToVersionCalled.get > 0) } } test("transfer from one commit-coordinator to another commit-coordinator fails " + "[CC-1 -> CC-2 fails]") { clearBuilders() val builder1 = TrackingInMemoryCommitCoordinatorBuilder(batchSize = 10) val builder2 = new TrackingInMemoryCommitCoordinatorBuilder(batchSize = 10) { override def getName: String = "tracking-in-memory-2" } Seq(builder1, builder2).foreach(registerBuilder(_)) withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath val log = DeltaLog.forTable(spark, tablePath) // A new table will automatically get `tracking-in-memory` as the whole suite is configured to // use it as default commit-coordinator via // [[COORDINATED_COMMITS_COORDINATOR_NAME.defaultTablePropertyKey]]. log.startTransaction().commitManually(Metadata()) assert(log.unsafeVolatileSnapshot.version === 0L) assert(log.unsafeVolatileSnapshot.tableCommitCoordinatorClientOpt.nonEmpty) // Change commit-coordinator val newCommitCoordinatorConf = Map(COORDINATED_COMMITS_COORDINATOR_NAME.key -> builder2.getName) val oldMetadata = log.unsafeVolatileSnapshot.metadata val newMetadata = oldMetadata.copy( configuration = oldMetadata.configuration ++ newCommitCoordinatorConf) val ex = intercept[IllegalStateException] { log.startTransaction().commitManually(newMetadata) } assert(ex.getMessage.contains( "from one commit-coordinator to another commit-coordinator is not allowed")) } } // This test has the following setup: // Setup: // 1. Make 2 commits on the table with CS1 as owner. // 2. Make 2 new commits to change the owner back to FS and then from FS to CS2. // 3. Do cold read from table and confirm we can construct snapshot v3 automatically. This will // need multiple snapshot update internally and both CS1 and CS2 will be contacted one // after the other. // 4. Write commit 4/5 using new commit-coordinator. // 5. Read the table again and make sure right APIs are called: // a) If read query is run in scala, we do listing 2 times. So CS2.getCommits will be called // twice. We should not be contacting CS1 anymore. // b) If read query is run on SQL, we do listing only once. So CS2.getCommits will be called // only once. test("snapshot is updated properly when owner changes multiple times") { val batchSize = 10 val cs1 = new TrackingCommitCoordinatorClient(new InMemoryCommitCoordinator(batchSize)) val cs2 = new TrackingCommitCoordinatorClient(new InMemoryCommitCoordinator(batchSize)) case class TrackingInMemoryCommitCoordinatorBuilder( name: String, commitCoordinatorClient: CommitCoordinatorClient) extends CommitCoordinatorBuilder { var numBuildCalled = 0 override def build( spark: SparkSession, conf: Map[String, String]): CommitCoordinatorClient = { numBuildCalled += 1 commitCoordinatorClient } override def getName: String = name } val builder1 = TrackingInMemoryCommitCoordinatorBuilder(name = "tracking-in-memory-1", cs1) val builder2 = TrackingInMemoryCommitCoordinatorBuilder(name = "tracking-in-memory-2", cs2) Seq(builder1, builder2).foreach(CommitCoordinatorProvider.registerBuilder) def resetMetrics(): Unit = { Seq(builder1, builder2).foreach { b => b.numBuildCalled = 0 } Seq(cs1, cs2).foreach(_.reset()) } withSQLConf( COORDINATED_COMMITS_COORDINATOR_NAME.defaultTablePropertyKey -> builder1.name) { withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath // Step-1: Make 2 commits on the table with CS1 as owner. Seq(0).toDF.write.format("delta").mode("append").save(tablePath) // version 0 Seq(1).toDF.write.format("delta").mode("append").save(tablePath) // version 1 DeltaLog.clearCache() checkAnswer(sql(s"SELECT * FROM delta.`$tablePath`"), Seq(Row(0), Row(1))) // Step-2: Add commit 2: change the table owner from "tracking-in-memory-1" to FS. // Add commit 3: change the table owner from FS to "tracking-in-memory-2". // Both of these commits should be FS based as the spec mandates an atomic backfill when // the commit-coordinator changes. { val log = DeltaLog.forTable(spark, tablePath) val conf = log.newDeltaHadoopConf() val segment = log.unsafeVolatileSnapshot.logSegment (0 to 1).foreach { version => assert(FileNames.deltaVersion(segment.deltas(version).getPath) === version) } val oldMetadata = log.unsafeVolatileMetadata val oldMetadataConf = oldMetadata.configuration val newMetadata1 = oldMetadata.copy( configuration = oldMetadataConf - COORDINATED_COMMITS_COORDINATOR_NAME.key) val newMetadata2 = oldMetadata.copy( configuration = oldMetadataConf + ( COORDINATED_COMMITS_COORDINATOR_NAME.key -> builder2.name)) log.startTransaction().commitManually(newMetadata1) log.startTransaction().commitManually(newMetadata2) // Also backfill commit 0, 1 -- which the spec mandates when the commit-coordinator // changes. // commit 0 should already be backfilled assert(segment.deltas(0).getPath.getName === "00000000000000000000.json") log.store.write( path = FileNames.unsafeDeltaFile(log.logPath, 1), actions = log.store.read(segment.deltas(1).getPath, conf).toIterator, overwrite = true, conf) } // Step-3: Do cold read from table and confirm we can construct snapshot v3 automatically. // This will update snapshot twice and both CS1 and CS2 will be contacted one after the // other. Do cold read from the table and confirm things works as expected. DeltaLog.clearCache() resetMetrics() checkAnswer(sql(s"SELECT * FROM delta.`$tablePath`"), Seq(Row(0), Row(1))) assert(builder1.numBuildCalled == 0) assert(builder2.numBuildCalled == 1) val snapshotV3 = DeltaLog.forTable(spark, tablePath).unsafeVolatileSnapshot assert( snapshotV3.tableCommitCoordinatorClientOpt.map(_.commitCoordinatorClient) === Some(cs2)) assert(snapshotV3.version === 3) // Step-4: Write more commits using new owner resetMetrics() Seq(2).toDF.write.format("delta").mode("append").save(tablePath) // version 4 Seq(3).toDF.write.format("delta").mode("append").save(tablePath) // version 5 assert((cs1.numCommitsCalled.get, cs2.numCommitsCalled.get) === (0, 2)) assert((cs1.numGetCommitsCalled.get, cs2.numGetCommitsCalled.get) === (0, 2)) // Step-5: Read the table again and assert that the right APIs are used resetMetrics() assert( sql(s"SELECT * FROM delta.`$tablePath`").collect().toSet === (0 to 3).map(Row(_)).toSet) // since this was hot query, so no new snapshot was created as part of this // deltaLog.update() and so commit-coordinator is not initialized again. assert((builder1.numBuildCalled, builder2.numBuildCalled) === (0, 0)) // Since this is dataframe read, so we invoke deltaLog.update() twice and so GetCommits API // is called twice. assert((cs1.numGetCommitsCalled.get, cs2.numGetCommitsCalled.get) === (0, 2)) // Step-6: Clear cache and simulate cold read again. // We will firstly create snapshot from listing: 0.json, 1.json, 2.json. // We create Snapshot v2 and find about owner CS2. // Then we contact CS2 to update snapshot and find that v3, v4 exist. // We create snapshot v4 and find that owner doesn't change. DeltaLog.clearCache() resetMetrics() assert( sql(s"SELECT * FROM delta.`$tablePath`").collect().toSet === (0 to 3).map(Row(_)).toSet) assert((cs1.numGetCommitsCalled.get, cs2.numGetCommitsCalled.get) === (0, 2)) assert((builder1.numBuildCalled, builder2.numBuildCalled) === (0, 2)) } } } } class CatalogOwnedSuite extends CommitCoordinatorSuiteBase with CatalogOwnedTestBaseSuite { override def sparkConf: SparkConf = { // Make sure all new tables in tests use CatalogOwned table feature by default. super.sparkConf.set(defaultCatalogOwnedFeatureEnabledKey, "supported") } } abstract class CommitCoordinatorSuiteBase extends QueryTest with CommitCoordinatorUtilBase with DeltaSQLTestUtils with DeltaTestUtilsBase with SharedSparkSession with DeltaSQLCommandTest with DeltaExceptionTestUtils { import testImplicits._ test("0th commit happens via filesystem") { val commitCoordinatorName = "tracking-in-memory" object NoBackfillingCommitCoordinatorBuilder$ extends CatalogOwnedCommitCoordinatorBuilder { override def getName: String = commitCoordinatorName override def build(spark: SparkSession, conf: Map[String, String]): CommitCoordinatorClient = new InMemoryCommitCoordinator(batchSize = 5) { override def commit( logStore: LogStore, hadoopConf: Configuration, tableDesc: TableDescriptor, commitVersion: Long, actions: JIterator[String], updatedActions: UpdatedActions): CommitResponse = { throw new IllegalStateException("Fail commit request") } } override def buildForCatalog(spark: SparkSession, name: String): CommitCoordinatorClient = new InMemoryCommitCoordinator(batchSize = 5) { override def commit( logStore: LogStore, hadoopConf: Configuration, tableDesc: TableDescriptor, commitVersion: Long, actions: JIterator[String], updatedActions: UpdatedActions): CommitResponse = { throw new IllegalStateException("Fail commit request") } } } registerBuilder(NoBackfillingCommitCoordinatorBuilder$) withDefaultCCTableFeature { withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath Seq(1).toDF.write.format("delta").save(tablePath) val log = DeltaLog.forTable(spark, tablePath) assert(log.store.listFrom(FileNames.listingPrefix(log.logPath, 0L)).exists { f => f.getPath.getName === "00000000000000000000.json" }) } } } test("basic write") { registerBuilder(TrackingInMemoryCommitCoordinatorBuilder(batchSize = 2)) withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath Seq(1).toDF.write.format("delta").mode("overwrite").save(tablePath) // version 0 Seq(2).toDF.write.format("delta").mode("overwrite").save(tablePath) // version 1 Seq(3).toDF.write.format("delta").mode("append").save(tablePath) // version 2 val log = DeltaLog.forTable(spark, tablePath) val commitsDir = new File(FileNames.commitDirPath(log.logPath).toUri) val unbackfilledCommitVersions = commitsDir .listFiles() .filterNot(f => f.getName.startsWith(".") && f.getName.endsWith(".crc")) .map(_.getAbsolutePath) .sortBy(path => path).map { commitPath => assert(FileNames.isDeltaFile(new Path(commitPath))) FileNames.deltaVersion(new Path(commitPath)) } assert(unbackfilledCommitVersions === Array(1, 2)) checkAnswer(sql(s"SELECT * FROM delta.`$tablePath`"), Seq(Row(2), Row(3))) } } test("cold snapshot initialization") { val builder = TrackingInMemoryCommitCoordinatorBuilder(batchSize = 10) val commitCoordinatorClient = builder.build(spark, Map.empty).asInstanceOf[TrackingCommitCoordinatorClient] registerBuilder(builder) withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath Seq(1).toDF.write.format("delta").mode("overwrite").save(tablePath) // version 0 DeltaLog.clearCache() val usageLogs1 = Log4jUsageLogger.track { checkAnswer(sql(s"SELECT * FROM delta.`$tablePath`"), Seq(Row(1))) } val getCommitsUsageLogs1 = filterUsageRecords( usageLogs1, CoordinatedCommitsUsageLogs.COMMIT_COORDINATOR_CLIENT_GET_COMMITS) val getCommitsEventData1 = JsonUtils.fromJson[Map[String, Any]](getCommitsUsageLogs1(0).blob) assert(getCommitsEventData1("startVersion") === 0) assert(getCommitsEventData1("versionToLoad") === -1) assert(getCommitsEventData1("async") === "true") assert(getCommitsEventData1("responseCommitsSize") === 0) assert(getCommitsEventData1("responseLatestTableVersion") === -1) Seq(2).toDF.write.format("delta").mode("overwrite").save(tablePath) // version 1 Seq(3).toDF.write.format("delta").mode("append").save(tablePath) // version 2 DeltaLog.clearCache() commitCoordinatorClient.numGetCommitsCalled.set(0) import testImplicits._ val result1 = sql(s"SELECT * FROM delta.`$tablePath`").collect() assert(result1.length === 2 && result1.toSet === Set(Row(2), Row(3))) assert(commitCoordinatorClient.numGetCommitsCalled.get === 2) } } // Test commit-coordinator changed on concurrent cluster testWithDefaultCommitCoordinatorUnset("snapshot is updated recursively when FS table" + " is converted to commit-coordinator table on a concurrent cluster") { if (isCatalogOwnedTest) { // TODO: CatalogOwned table cannot change its catalog, hence modify below to // test race upgrade from normal table after implementing upgrade. cancel("Upgrade is not yet supported for catalog owned tables") } val commitCoordinatorClient = new TrackingCommitCoordinatorClient(new InMemoryCommitCoordinator(batchSize = 10)) val builder = TrackingInMemoryCommitCoordinatorBuilder(batchSize = 10, Some(commitCoordinatorClient)) registerBuilder(builder) withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath val deltaLog1 = DeltaLog.forTable(spark, tablePath) deltaLog1.startTransaction().commitManually(Metadata()) deltaLog1.startTransaction().commitManually(createTestAddFile("f1")) deltaLog1.startTransaction().commitManually() val snapshotV2 = deltaLog1.update() assert(snapshotV2.version === 2) assert(snapshotV2.tableCommitCoordinatorClientOpt.isEmpty) DeltaLog.clearCache() // Add new commit to convert FS table to coordinated-commits table val deltaLog2 = DeltaLog.forTable(spark, tablePath) upgradeLogWithCCTableFeature(deltaLog2, commitCoordinator = "tracking-in-memory") deltaLog2.startTransaction().commitManually(createTestAddFile("f2")) deltaLog2.startTransaction().commitManually() val snapshotV5 = deltaLog2.unsafeVolatileSnapshot assert(snapshotV5.version === 5) assert(snapshotV5.tableCommitCoordinatorClientOpt.nonEmpty) // only delta 4/5 will be un-backfilled and should have two dots in filename (x.uuid.json) assert(snapshotV5.logSegment.deltas.count(_.getPath.getName.count(_ == '.') == 2) === 2) val usageRecords = Log4jUsageLogger.track { val newSnapshotV5 = deltaLog1.update() assert(newSnapshotV5.version === 5) assert(newSnapshotV5.logSegment.deltas === snapshotV5.logSegment.deltas) } assert(filterUsageRecords(usageRecords, "delta.readChecksum").size === 2) } } test("update works correctly with InitialSnapshot") { registerBuilder( TrackingInMemoryCommitCoordinatorBuilder(batchSize = 2)) withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath val clock = new ManualClock(System.currentTimeMillis()) val log = DeltaLog.forTable(spark, new Path(tablePath), clock) assert(log.unsafeVolatileSnapshot.isInstanceOf[DummySnapshot]) assert(log.unsafeVolatileSnapshot.tableCommitCoordinatorClientOpt.isEmpty) assert(log.getCapturedSnapshot().updateTimestamp == clock.getTimeMillis()) clock.advance(500) log.update() assert(log.unsafeVolatileSnapshot.isInstanceOf[DummySnapshot]) assert(log.getCapturedSnapshot().updateTimestamp == clock.getTimeMillis()) } } // This test has the following setup: // 1. Table is created with CS1 as commit-coordinator. // 2. Do another commit (v1) on table. // 3. Take a reference to current DeltaLog and clear the cache. This deltaLog object currently // points to the latest table snapshot i.e. v1. // 4. Do commit v2 on the table. // 5. Do commit v3 on table. As part of this, change commit-coordinator to FS. Do v4 on table and // change owner to CS2. // 6. Do commit v5 on table. This will happen via CS2. // 7. Invoke deltaLog.update() on the old deltaLog object which is still pointing to v1. // - While doing this, we will inject failure in CS2 so that it fails twice when cs2.getCommits // is called. // - Because of this old delta log will firstly contact cs1 and get newer commits i.e. v2/v3 // and create a snapshot out of it. Then it will contact cs2 and fail. So deltaLog.update() // won't succeed and throw exception. It also won't install the intermediate snapshot. The // older delta log will still be pointing to v1. // 8. Invoke deltaLog.update() two more times. 3rd attempt will succeed. // - the recorded timestamp for this should be clock timestamp. test("failures inside getCommits, correct timestamp is added in CapturedSnapshot") { if (isCatalogOwnedTest) { // TODO: This test is important to test the robustness of the ability to resolve // stale snapshot status interaction with upgrade/downgrade. Implement this suite // for catalog owned tables once we enable upgrade. cancel("Upgrade is not yet supported for catalog owned tables") } val batchSize = 10 val cs1 = new TrackingCommitCoordinatorClient(new InMemoryCommitCoordinator(batchSize)) val cs2 = new TrackingCommitCoordinatorClient(new InMemoryCommitCoordinator(batchSize)) { var failAttempts = Set[Int]() override def getCommits( tableDesc: TableDescriptor, startVersion: java.lang.Long, endVersion: java.lang.Long): JGetCommitsResponse = { if (failAttempts.contains(numGetCommitsCalled.get + 1)) { numGetCommitsCalled.incrementAndGet() throw new IllegalStateException("Injected failure") } super.getCommits(tableDesc, startVersion, endVersion) } } case class TrackingInMemoryCommitCoordinatorClientBuilder( name: String, commitCoordinatorClient: CommitCoordinatorClient) extends CommitCoordinatorBuilder { override def build( spark: SparkSession, conf: Map[String, String]): CommitCoordinatorClient = commitCoordinatorClient override def getName: String = name } val builder1 = TrackingInMemoryCommitCoordinatorClientBuilder(name = "in-memory-1", cs1) val builder2 = TrackingInMemoryCommitCoordinatorClientBuilder(name = "in-memory-2", cs2) Seq(builder1, builder2).foreach(CommitCoordinatorProvider.registerBuilder) def resetMetrics(): Unit = { cs1.reset() cs2.reset() cs2.failAttempts = Set() } withSQLConf(COORDINATED_COMMITS_COORDINATOR_NAME.defaultTablePropertyKey -> "in-memory-1") { withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath // Step-1 Seq(0).toDF.write.format("delta").mode("append").save(tablePath) // version 0 DeltaLog.clearCache() // Step-2 Seq(1).toDF.write.format("delta").mode("append").save(tablePath) // version 1 // Step-3 DeltaLog.clearCache() val clock = new ManualClock(System.currentTimeMillis()) val oldDeltaLog = DeltaLog.forTable(spark, new Path(tablePath), clock) DeltaLog.clearCache() // Step-4 Seq(2).toDF.write.format("delta").mode("append").save(tablePath) // version 2 // Step-5 val log = DeltaLog.forTable(spark, new Path(tablePath), clock) val oldMetadata = log.update().metadata assert(log.unsafeVolatileSnapshot.version === 2) val oldMetadataConf = log.update().metadata.configuration val newMetadata1 = oldMetadata.copy( configuration = oldMetadataConf - COORDINATED_COMMITS_COORDINATOR_NAME.key) val newMetadata2 = oldMetadata.copy( configuration = oldMetadataConf + ( COORDINATED_COMMITS_COORDINATOR_NAME.key -> "in-memory-2")) assert(log.update().tableCommitCoordinatorClientOpt.get.commitCoordinatorClient === cs1) log.startTransaction().commitManually(newMetadata1) // version 3 (1 to 3).foreach { v => // backfill commit 1 and 2 also as 3/4 are written directly to FS. val segment = log.unsafeVolatileSnapshot.logSegment log.store.write( path = FileNames.unsafeDeltaFile(log.logPath, v), actions = log.store.read(segment.deltas(v).getPath).toIterator, overwrite = true) } assert(log.update().tableCommitCoordinatorClientOpt === None) log.startTransaction().commitManually(newMetadata2) // version 4 assert(log.update().tableCommitCoordinatorClientOpt.get.commitCoordinatorClient === cs2) // Step-6 Seq(4).toDF.write.format("delta").mode("append").save(tablePath) // version 5 assert(log.unsafeVolatileSnapshot.version === 5L) assert(FileNames.deltaVersion(log.unsafeVolatileSnapshot.logSegment.deltas(5)) === 5) // Step-7 // Invoke deltaLog.update() on older copy of deltaLog which is still pointing to version 1 // Attempt-1 assert(oldDeltaLog.unsafeVolatileSnapshot.version === 1) clock.setTime(System.currentTimeMillis()) resetMetrics() cs2.failAttempts = Set(1, 2) // fail 0th and 1st attempt, 2nd attempt will succeed. val ex1 = intercept[CommitCoordinatorGetCommitsFailedException] { oldDeltaLog.update() } assert((cs1.numGetCommitsCalled.get, cs2.numGetCommitsCalled.get) === (1, 1)) assert(ex1.getMessage.contains("Injected failure")) assert(oldDeltaLog.unsafeVolatileSnapshot.version == 1) assert(oldDeltaLog.getCapturedSnapshot().updateTimestamp != clock.getTimeMillis()) // Attempt-2 // 2nd update also fails val ex2 = intercept[CommitCoordinatorGetCommitsFailedException] { oldDeltaLog.update() } assert((cs1.numGetCommitsCalled.get, cs2.numGetCommitsCalled.get) === (2, 2)) assert(ex2.getMessage.contains("Injected failure")) assert(oldDeltaLog.unsafeVolatileSnapshot.version == 1) assert(oldDeltaLog.getCapturedSnapshot().updateTimestamp != clock.getTimeMillis()) // Attempt-3: 3rd update succeeds clock.advance(500) assert(oldDeltaLog.update().version === 5) assert((cs1.numGetCommitsCalled.get, cs2.numGetCommitsCalled.get) === (3, 3)) assert(oldDeltaLog.getCapturedSnapshot().updateTimestamp == clock.getTimeMillis()) } } } testWithDifferentBackfillInterval("post commit snapshot creation") { backfillInterval => withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath def getDeltasInPostCommitSnapshot(log: DeltaLog): Seq[String] = { log .unsafeVolatileSnapshot .logSegment.deltas .map(_.getPath.getName.replace("0000000000000000000", "")) } // Commit 0 Seq(1).toDF.write.format("delta").mode("overwrite").save(tablePath) val log = DeltaLog.forTable(spark, tablePath) assert(getDeltasInPostCommitSnapshot(log) === Seq("0.json")) assert(log.unsafeVolatileSnapshot.getLastKnownBackfilledVersion == 0) var snapshot = log.update() assert(getDeltasInPostCommitSnapshot(log) === Seq("0.json")) assert(snapshot.getLastKnownBackfilledVersion == 0) // Commit 1 Seq(2).toDF.write.format("delta").mode("append").save(tablePath) // version 1 // Note: The assert for backfillInterval = 1 only works because with a batch // size of 1, the AbstractBatchBackfillingCommitCoordinator synchronously // backfills a commit and then directly returns the backfilled commit information // from the commit() call so the post commit snapshot creation directly appends // the backfilled commit to the LogSegment. // // The expected behavior before update() is: // Backfill interval 1 : post commit snapshot contains 0.json, 1.json, lkbv = 1 // Backfill interval 2 : post commit snapshot contains 0.json, 1.uuid.json, lkbv = 0 // Backfill interval 10: post commit snapshot contains 0.json, 1.uuid.json, lkbv = 0 val commit1 = if (backfillInterval < 2) "1.json" else "1.uuid-1.json" var backfillVersion = if (backfillInterval < 2) 1 else 0 assert(getDeltasInPostCommitSnapshot(log) === Seq("0.json", commit1)) assert(log.unsafeVolatileSnapshot.getLastKnownBackfilledVersion == backfillVersion) snapshot = log.update() // The expected behavior after update() is: // Backfill interval 1 : post commit snapshot contains 0.json, 1.json, lkbv = 1 // Backfill interval 2 : post commit snapshot contains 0.json, 1.uuid.json, lkbv = 0 // Backfill interval 10: post commit snapshot contains 0.json, 1.uuid.json, lkbv = 0 assert(getDeltasInPostCommitSnapshot(log) === Seq("0.json", commit1)) assert(snapshot.getLastKnownBackfilledVersion == backfillVersion) // Commit 2 Seq(3).toDF.write.format("delta").mode("append").save(tablePath) // version 2 // The expected behavior before update() is: // Backfill interval 1 : post commit snapshot contains // 0.json, 1.json, 2.json lkbv = 2 // Backfill interval 2 : post commit snapshot contains // 0.json, 1.uuid.json, 2.uuid.json lkbv = 0 // Backfill interval 10: post commit snapshot contains // 0.json, 1.uuid.json, 2.uuid.json, lkbv = 0 val commit2 = if (backfillInterval < 2) "2.json" else "2.uuid-2.json" backfillVersion = if (backfillInterval < 2) 2 else 0 assert(getDeltasInPostCommitSnapshot(log) === Seq("0.json", commit1, commit2)) assert(log.unsafeVolatileSnapshot.getLastKnownBackfilledVersion == backfillVersion) snapshot = log.update() // backfill would have happened at commit 2 for batchSize = 2 but we do not swap // the snapshot that contains the unbackfilled commits with the updated snapshot // (which contains the backfilled commits) during update because they are identical // and swapping would lead to losing the cached state. However, we update the // effective last known backfilled version on the snapshot. // // The expected behavior after update is // Backfill interval 1 : post commit snapshot contains // 0.json, 1.json, 2.json lkbv = 2 // Backfill interval 2 : post commit snapshot contains // 0.json, 1.uuid.json, 2.uuid.json lkbv = 0 lkbv = 2 // Backfill interval 10: post commit snapshot contains // 0.json, 1.uuid.json, 2.uuid.json lkbv = 0 backfillVersion = if (backfillInterval <= 2) 2 else 0 assert(getDeltasInPostCommitSnapshot(log) === Seq("0.json", commit1, commit2)) assert(snapshot.getLastKnownBackfilledVersion == backfillVersion) // Commit 3 Seq(4).toDF.write.format("delta").mode("append").save(tablePath) val commit3 = if (backfillInterval < 2) "3.json" else "3.uuid-3.json" // The post commit snapshot is a new snapshot and so its lastKnownBackfilledVersion // member is calculated from the LogSegment. Given that the LogSegment for // batchInterval > 1 only contains 0 as the only backfilled commit, we need // to set the expected backfillVersion to 0 here for backfillIntervals > 1. // // The expected behavior before update() is: // Backfill interval 1 : post commit snapshot contains // 0.json, 1.json, 2.json, 3.json lkbv = 3 // Backfill interval 2 : post commit snapshot contains // 0.json, 1.uuid.json, 2.uuid.json, 3.uuid.json lkbv = 0 // Backfill interval 10: post commit snapshot contains // 0.json, 1.uuid.json, 2.uuid.json, 3.uuid.json lkbv = 0 backfillVersion = if (backfillInterval < 2) 3 else 0 assert(getDeltasInPostCommitSnapshot(log) === Seq("0.json", commit1, commit2, commit3)) assert(log.unsafeVolatileSnapshot.getLastKnownBackfilledVersion == backfillVersion) snapshot = log.update() // The expected behavior after update() is: // Backfill interval 1 : post commit snapshot contains // 0.json, 1.json, 2.json, 3.json lkbv = 3 // Backfill interval 2 : post commit snapshot contains // 0.json, 1.uuid.json, 2.uuid.json, 3.uuid.json lkbv = 2 // Backfill interval 10: post commit snapshot contains // 0.json, 1.uuid.json, 2.uuid.json, 3.uuid.json lkbv = 0 backfillVersion = if (backfillInterval < 2) 3 else if (backfillInterval == 2) 2 else 0 assert(getDeltasInPostCommitSnapshot(log) === Seq("0.json", commit1, commit2, commit3)) assert(snapshot.getLastKnownBackfilledVersion == backfillVersion) checkAnswer(sql(s"SELECT * FROM delta.`$tablePath`"), Seq(Row(1), Row(2), Row(3), Row(4))) } } testWithDifferentBackfillInterval("Snapshot.ensureCommitFilesBackfilled") { _ => withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath // Add 10 commits to the table Seq(1).toDF().write.format("delta").mode("overwrite").save(tablePath) 2 to 10 foreach { i => Seq(i).toDF().write.format("delta").mode("append").save(tablePath) } val log = DeltaLog.forTable(spark, tablePath) val snapshot = log.update() snapshot.ensureCommitFilesBackfilled() val commitFiles = log.listFrom(0).filter(FileNames.isDeltaFile).map(_.getPath) val backfilledCommitFiles = (0 to 9).map( version => FileNames.unsafeDeltaFile(log.logPath, version)) assert(commitFiles.toSeq == backfilledCommitFiles) } } testWithDefaultCommitCoordinatorUnset("DeltaLog.getSnapshotAt") { val commitCoordinatorClient = new TrackingCommitCoordinatorClient(new InMemoryCommitCoordinator(batchSize = 10)) val builder = TrackingInMemoryCommitCoordinatorBuilder(batchSize = 10, Some(commitCoordinatorClient)) registerBuilder(builder) def checkGetSnapshotAt( deltaLog: DeltaLog, version: Long, expectedUpdateCount: Int, expectedListingCount: Int): Snapshot = { var snapshot: Snapshot = null val usageRecords = Log4jUsageLogger.track { snapshot = deltaLog.getSnapshotAt(version) assert(snapshot.version === version) } assert(filterUsageRecords(usageRecords, "deltaLog.update").size === expectedUpdateCount) // deltaLog.update() will internally do listing assert(filterUsageRecords(usageRecords, "delta.deltaLog.listDeltaAndCheckpointFiles").size === expectedListingCount) val versionsInLogSegment = if (version < 6) { snapshot.logSegment.deltas.map(FileNames.deltaVersion(_)) } else { snapshot.logSegment.deltas.flatMap { case DeltaFile(_, deltaVersion) => Seq(deltaVersion) case CompactedDeltaFile(_, startVersion, endVersion) => (startVersion to endVersion) } } assert(versionsInLogSegment === (0L to version)) snapshot } withTempDir { dir => val tablePath = dir.getAbsolutePath // Part-1: Validate getSnapshotAt API works as expected for non-coordinated commits tables // commit 0, 1, 2 on FS table Seq(1).toDF.write.format("delta").mode("overwrite").save(tablePath) // v0 Seq(1).toDF.write.format("delta").mode("overwrite").save(tablePath) // v1 val deltaLog1 = DeltaLog.forTable(spark, tablePath) DeltaLog.clearCache() Seq(1).toDF.write.format("delta").mode("overwrite").save(tablePath) // v2 assert(deltaLog1.unsafeVolatileSnapshot.version === 1) checkGetSnapshotAt(deltaLog1, version = 1, expectedUpdateCount = 0, expectedListingCount = 0) // deltaLog1 still points to version 1. So, we will do listing to get v0. checkGetSnapshotAt(deltaLog1, version = 0, expectedUpdateCount = 0, expectedListingCount = 1) // deltaLog1 still points to version 1 although we are asking for v2 So we do a // deltaLog.update - the update will internally do listing.Since the updated snapshot is same // as what we want, so we won't create another snapshot and do another listing. checkGetSnapshotAt(deltaLog1, version = 2, expectedUpdateCount = 1, expectedListingCount = 1) var deltaLog2 = DeltaLog.forTable(spark, tablePath) Seq(deltaLog1, deltaLog2).foreach { log => assert(log.unsafeVolatileSnapshot.version === 2) } DeltaLog.clearCache() // Part-2: Validate getSnapshotAt API works as expected for coordinated commits tables when // the switch is made // commit 3 upgradeLogWithCCTableFeature(DeltaLog.forTable(spark, tablePath), "tracking-in-memory") // commit 4 Seq(1).toDF.write.format("delta").mode("overwrite").save(tablePath) // the old deltaLog objects still points to version 2 Seq(deltaLog1, deltaLog2).foreach { log => assert(log.unsafeVolatileSnapshot.version === 2) } // deltaLog1 points to version 2. So, we will do listing to get v1. Snapshot update not // needed as what we are looking for is less than what deltaLog1 points to. checkGetSnapshotAt(deltaLog1, version = 1, expectedUpdateCount = 0, expectedListingCount = 1) // deltaLog1.unsafeVolatileSnapshot.version points to v2 - return it directly. checkGetSnapshotAt(deltaLog1, version = 2, expectedUpdateCount = 0, expectedListingCount = 0) // We are asking for v3 although the deltaLog1.unsafeVolatileSnapshot is for v2. So this will // need deltaLog.update() to get the latest snapshot first - this update itself internally // will do 2 round of listing as we are discovering a commit-coordinator after first round of // listing. Once the update finishes, deltaLog1 will point to v4. So we need another round of // listing to get just v3. checkGetSnapshotAt(deltaLog1, version = 3, expectedUpdateCount = 1, expectedListingCount = 3) // Ask for v3 again - this time deltaLog1.unsafeVolatileSnapshot points to v4. // So we don't need deltaLog.update as version which we are asking is less than pinned // version. Just do listing and get the snapshot. checkGetSnapshotAt(deltaLog1, version = 3, expectedUpdateCount = 0, expectedListingCount = 1) // deltaLog1.unsafeVolatileSnapshot.version points to v4 - return it directly. checkGetSnapshotAt(deltaLog1, version = 4, expectedUpdateCount = 0, expectedListingCount = 0) // We are asking for v3 although the deltaLog2.unsafeVolatileSnapshot is for v2. So this will // need deltaLog.update() to get the latest snapshot first - this update itself internally // will do 2 round of listing as we are discovering a commit-coordinator after first round of // listing. Once the update finishes, deltaLog2 will point to v4. It can be returned directly. checkGetSnapshotAt(deltaLog2, version = 4, expectedUpdateCount = 1, expectedListingCount = 2) // Part-2: Validate getSnapshotAt API works as expected for coordinated commits tables Seq(1).toDF.write.format("delta").mode("overwrite").save(tablePath) // v5 deltaLog2 = DeltaLog.forTable(spark, tablePath) DeltaLog.clearCache() Seq(1).toDF.write.format("delta").mode("overwrite").save(tablePath) // v6 Seq(1).toDF.write.format("delta").mode("overwrite").save(tablePath) // v7 assert(deltaLog2.unsafeVolatileSnapshot.version === 5) checkGetSnapshotAt(deltaLog2, version = 1, expectedUpdateCount = 0, expectedListingCount = 1) checkGetSnapshotAt(deltaLog2, version = 2, expectedUpdateCount = 0, expectedListingCount = 1) checkGetSnapshotAt(deltaLog2, version = 4, expectedUpdateCount = 0, expectedListingCount = 1) checkGetSnapshotAt(deltaLog2, version = 5, expectedUpdateCount = 0, expectedListingCount = 0) checkGetSnapshotAt(deltaLog2, version = 6, expectedUpdateCount = 1, expectedListingCount = 2) } } private def upgradeLogWithCCTableFeature(deltaLog: DeltaLog, commitCoordinator: String): Unit = { if (isCatalogOwnedTest) { cancel("Upgrade is not yet supported for catalog owned tables") } val oldMetadata = deltaLog.update().metadata val commitCoordinatorConf = (COORDINATED_COMMITS_COORDINATOR_NAME.key -> commitCoordinator) val newMetadata = oldMetadata.copy(configuration = oldMetadata.configuration + commitCoordinatorConf) deltaLog.startTransaction().commitManually(newMetadata) } for (upgradeExistingTable <- BOOLEAN_DOMAIN) testWithDifferentBackfillInterval("upgrade + downgrade [FS -> CC1 -> FS -> CC2]," + s" upgradeExistingTable = $upgradeExistingTable") { backfillInterval => if (isCatalogOwnedTest) { // TODO: Once upgrade is supported, this unit test can only test // first upgrade part (FS -> CC1) because there is no CC1 -> FS -> CC2 transition // in CatalogOwned table feature. Note that only one Catalog can exist for // each table identifier. cancel("Upgrade is not yet supported for catalog owned tables") } withoutDefaultCCTableFeature { clearBuilders() val builder1 = TrackingInMemoryCommitCoordinatorBuilder(batchSize = backfillInterval) val builder2 = new TrackingInMemoryCommitCoordinatorBuilder(batchSize = backfillInterval) { override def getName: String = "tracking-in-memory-2" } Seq(builder1, builder2).foreach(registerBuilder(_)) val cs1 = builder1 .trackingInMemoryCommitCoordinatorClient .asInstanceOf[TrackingCommitCoordinatorClient] val cs2 = builder2 .trackingInMemoryCommitCoordinatorClient .asInstanceOf[TrackingCommitCoordinatorClient] withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath val log = DeltaLog.forTable(spark, tablePath) val fs = log.logPath.getFileSystem(log.newDeltaHadoopConf()) var upgradeStartVersion = 0L // Create a non-coordinated commits table if we are testing upgrade for existing tables if (upgradeExistingTable) { log.startTransaction().commitManually(Metadata()) assert(log.unsafeVolatileSnapshot.version === 0) log.startTransaction().commitManually(createTestAddFile("1")) assert(log.unsafeVolatileSnapshot.version === 1) assert(log.unsafeVolatileSnapshot.tableCommitCoordinatorClientOpt.isEmpty) upgradeStartVersion = 2L } // Upgrade the table // [upgradeExistingTable = false] Commit-0 // [upgradeExistingTable = true] Commit-2 val commitCoordinatorConf = Map(COORDINATED_COMMITS_COORDINATOR_NAME.key -> builder1.getName) val newMetadata = Metadata().copy(configuration = commitCoordinatorConf) val usageLogs1 = Log4jUsageLogger.track { log.startTransaction().commitManually(newMetadata) } assert(log.unsafeVolatileSnapshot.version === upgradeStartVersion) assert(log.unsafeVolatileSnapshot.metadata.coordinatedCommitsCoordinatorName === Some(builder1.getName)) assert(log.unsafeVolatileSnapshot.tableCommitCoordinatorClientOpt.nonEmpty) assert(log.unsafeVolatileSnapshot.metadata.coordinatedCommitsTableConf === Map.empty) // upgrade commit always filesystem based assert(fs.exists(FileNames.unsafeDeltaFile(log.logPath, upgradeStartVersion))) assert(Seq(cs1, cs2).map(_.numCommitsCalled.get) == Seq(0, 0)) assert(Seq(cs1, cs2).map(_.numRegisterTableCalled.get) == Seq(1, 0)) // Check usage logs for upgrade commit val commitStatsUsageLogs1 = filterUsageRecords(usageLogs1, "delta.commit.stats") val commitStats1 = JsonUtils.fromJson[CommitStats](commitStatsUsageLogs1.head.blob) assert(commitStats1.coordinatedCommitsInfo === CoordinatedCommitsStats(FS_TO_CC_UPGRADE_COMMIT.toString, builder1.getName, Map.empty)) // Do couple of commits on the coordinated-commits table // [upgradeExistingTable = false] Commit-1/2 // [upgradeExistingTable = true] Commit-3/4 (1 to 2).foreach { versionOffset => val version = upgradeStartVersion + versionOffset val usageLogs2 = Log4jUsageLogger.track { log.startTransaction().commitManually(createTestAddFile(s"$versionOffset")) } assert(log.unsafeVolatileSnapshot.version === version) assert(log.unsafeVolatileSnapshot.tableCommitCoordinatorClientOpt.nonEmpty) assert(log.unsafeVolatileSnapshot.metadata.coordinatedCommitsCoordinatorName.nonEmpty) assert( log.unsafeVolatileSnapshot.metadata.coordinatedCommitsCoordinatorConf === Map.empty) assert(log.unsafeVolatileSnapshot.metadata.coordinatedCommitsTableConf === Map.empty) assert(cs1.numCommitsCalled.get === versionOffset) val backfillExpected = if (version % backfillInterval == 0) true else false assert(fs.exists(FileNames.unsafeDeltaFile(log.logPath, version)) == backfillExpected) // Check usage logs for INSERT commits on this coordinated-commits table. val commitStatsUsageLogs2 = filterUsageRecords(usageLogs2, "delta.commit.stats") val commitStats2 = JsonUtils.fromJson[CommitStats](commitStatsUsageLogs2.head.blob) assert(commitStats2.coordinatedCommitsInfo === CoordinatedCommitsStats(CC_COMMIT.toString, builder1.getName, Map.empty)) } // Downgrade the table // [upgradeExistingTable = false] Commit-3 // [upgradeExistingTable = true] Commit-5 val commitCoordinatorConfKeys = Seq( COORDINATED_COMMITS_COORDINATOR_NAME.key, COORDINATED_COMMITS_COORDINATOR_CONF.key, COORDINATED_COMMITS_TABLE_CONF.key ) val newConfig = log.snapshot.metadata.configuration .filterKeys(!commitCoordinatorConfKeys.contains(_)) ++ Map("downgraded_at" -> "v2") val newMetadata2 = log.snapshot.metadata.copy(configuration = newConfig.toMap) val usageLogs3 = Log4jUsageLogger.track { log.startTransaction().commitManually(newMetadata2) } assert(log.unsafeVolatileSnapshot.version === upgradeStartVersion + 3) assert(log.unsafeVolatileSnapshot.tableCommitCoordinatorClientOpt.isEmpty) assert(log.unsafeVolatileSnapshot.metadata.coordinatedCommitsCoordinatorName.isEmpty) assert(log.unsafeVolatileSnapshot.metadata.coordinatedCommitsCoordinatorConf === Map.empty) assert(log.unsafeVolatileSnapshot.metadata.coordinatedCommitsTableConf === Map.empty) assert(log.unsafeVolatileSnapshot.metadata === newMetadata2) // This must have increased by 1 as downgrade commit happens via CommitCoordinatorClient. assert(Seq(cs1, cs2).map(_.numCommitsCalled.get) == Seq(3, 0)) assert(Seq(cs1, cs2).map(_.numRegisterTableCalled.get) == Seq(1, 0)) (0 to 3).foreach { version => assert(fs.exists(FileNames.unsafeDeltaFile(log.logPath, version))) } // Check usage logs for downgrade commit val commitStatsUsageLogs3 = filterUsageRecords(usageLogs3, "delta.commit.stats") val commitStats3 = JsonUtils.fromJson[CommitStats](commitStatsUsageLogs3.head.blob) assert(commitStats3.coordinatedCommitsInfo === CoordinatedCommitsStats(CC_TO_FS_DOWNGRADE_COMMIT.toString, builder1.getName, Map.empty)) // Do commit after downgrade is over // [upgradeExistingTable = false] Commit-4 // [upgradeExistingTable = true] Commit-6 val usageLogs4 = Log4jUsageLogger.track { log.startTransaction().commitManually(createTestAddFile("post-upgrade-file")) } assert(log.unsafeVolatileSnapshot.version === upgradeStartVersion + 4) // no commit-coordinator after downgrade assert(log.unsafeVolatileSnapshot.tableCommitCoordinatorClientOpt.isEmpty) assert(log.unsafeVolatileSnapshot.metadata.coordinatedCommitsTableConf === Map.empty) // Metadata is same as what we added at time of downgrade assert(log.unsafeVolatileSnapshot.metadata === newMetadata2) // State reconstruction should give correct results var expectedFileNames = Set("1", "2", "post-upgrade-file") assert(log.unsafeVolatileSnapshot.allFiles.collect().toSet === expectedFileNames.map(name => createTestAddFile(name, dataChange = false))) // commit-coordinator should not be invoked for commit API. // Register table API should not be called until the end assert(Seq(cs1, cs2).map(_.numCommitsCalled.get) == Seq(3, 0)) assert(Seq(cs1, cs2).map(_.numRegisterTableCalled.get) == Seq(1, 0)) // 4th file is directly written to FS in backfilled way. assert(fs.exists(FileNames.unsafeDeltaFile(log.logPath, upgradeStartVersion + 4))) // Check usage logs for normal FS commit val commitStatsUsageLogs4 = filterUsageRecords(usageLogs4, "delta.commit.stats") val commitStats4 = JsonUtils.fromJson[CommitStats](commitStatsUsageLogs4.head.blob) assert(commitStats4.coordinatedCommitsInfo === CoordinatedCommitsStats(FS_COMMIT.toString, "NONE", Map.empty)) // Now transfer the table to another commit-coordinator // [upgradeExistingTable = false] Commit-5 // [upgradeExistingTable = true] Commit-7 val commitCoordinatorConf2 = Map(COORDINATED_COMMITS_COORDINATOR_NAME.key -> builder2.getName) val oldMetadata3 = log.unsafeVolatileSnapshot.metadata val newMetadata3 = oldMetadata3.copy( configuration = oldMetadata3.configuration ++ commitCoordinatorConf2) val usageLogs5 = Log4jUsageLogger.track { log.startTransaction().commitManually(newMetadata3, createTestAddFile("upgrade-2-file")) } assert(log.unsafeVolatileSnapshot.version === upgradeStartVersion + 5) assert(log.unsafeVolatileSnapshot.tableCommitCoordinatorClientOpt.nonEmpty) assert(log.unsafeVolatileSnapshot.metadata.coordinatedCommitsCoordinatorName === Some(builder2.getName)) assert(log.unsafeVolatileSnapshot.metadata.coordinatedCommitsCoordinatorConf === Map.empty) assert(log.unsafeVolatileSnapshot.metadata.coordinatedCommitsTableConf === Map.empty) expectedFileNames = Set("1", "2", "post-upgrade-file", "upgrade-2-file") assert(log.unsafeVolatileSnapshot.allFiles.collect().toSet === expectedFileNames.map(name => createTestAddFile(name, dataChange = false))) assert(Seq(cs1, cs2).map(_.numCommitsCalled.get) == Seq(3, 0)) assert(Seq(cs1, cs2).map(_.numRegisterTableCalled.get) == Seq(1, 1)) // Check usage logs for 2nd upgrade commit val commitStatsUsageLogs5 = filterUsageRecords(usageLogs5, "delta.commit.stats") val commitStats5 = JsonUtils.fromJson[CommitStats](commitStatsUsageLogs5.head.blob) assert(commitStats5.coordinatedCommitsInfo === CoordinatedCommitsStats(FS_TO_CC_UPGRADE_COMMIT.toString, builder2.getName, Map.empty)) // Make 1 more commit, this should go to new owner log.startTransaction().commitManually(newMetadata3, createTestAddFile("4")) expectedFileNames = Set("1", "2", "post-upgrade-file", "upgrade-2-file", "4") assert(log.unsafeVolatileSnapshot.allFiles.collect().toSet === expectedFileNames.map(name => createTestAddFile(name, dataChange = false))) assert(Seq(cs1, cs2).map(_.numCommitsCalled.get) == Seq(3, 1)) assert(Seq(cs1, cs2).map(_.numRegisterTableCalled.get) == Seq(1, 1)) assert(log.unsafeVolatileSnapshot.version === upgradeStartVersion + 6) } } } testWithDefaultCommitCoordinatorUnset("FS -> CC upgrade is not retried on a conflict") { if (isCatalogOwnedTest) { cancel("Upgrade is not yet supported for catalog owned tables") } val builder = TrackingInMemoryCommitCoordinatorBuilder(batchSize = 10) registerBuilder(builder) withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath val log = DeltaLog.forTable(spark, tablePath) val commitCoordinatorConf = Map(COORDINATED_COMMITS_COORDINATOR_NAME.key -> builder.getName) val newMetadata = Metadata().copy(configuration = commitCoordinatorConf) val txn = log.startTransaction() // upgrade txn started log.startTransaction().commitManually(createTestAddFile("f1")) intercept[io.delta.exceptions.ConcurrentWriteException] { txn.commitManually(newMetadata) // upgrade txn committed } } } testWithDefaultCommitCoordinatorUnset("FS -> CC upgrade with commitLarge API") { if (isCatalogOwnedTest) { cancel("Upgrade is not yet supported for catalog owned tables") } val builder = TrackingInMemoryCommitCoordinatorBuilder(batchSize = 10) val cs = builder.trackingInMemoryCommitCoordinatorClient.asInstanceOf[TrackingCommitCoordinatorClient] registerBuilder(builder) withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath Seq(1).toDF.write.format("delta").save(tablePath) Seq(1).toDF.write.mode("overwrite").format("delta").save(tablePath) var log = DeltaLog.forTable(spark, tablePath) assert(log.unsafeVolatileSnapshot.version === 1L) assert(log.unsafeVolatileSnapshot.tableCommitCoordinatorClientOpt.isEmpty) val commitCoordinatorConf = Map(COORDINATED_COMMITS_COORDINATOR_NAME.key -> builder.getName) val oldMetadata = log.unsafeVolatileSnapshot.metadata val newMetadata = oldMetadata.copy( configuration = oldMetadata.configuration ++ commitCoordinatorConf) val oldProtocol = log.unsafeVolatileSnapshot.protocol assert(!oldProtocol.readerAndWriterFeatures.contains(V2CheckpointTableFeature)) val newProtocol = oldProtocol.copy( minReaderVersion = TableFeatureProtocolUtils.TABLE_FEATURES_MIN_READER_VERSION, minWriterVersion = TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION, readerFeatures = Some(oldProtocol.readerFeatures.getOrElse(Set.empty) + V2CheckpointTableFeature.name), writerFeatures = Some( oldProtocol.writerFeatures.getOrElse(Set.empty) + CoordinatedCommitsTableFeature.name) ) assert(cs.numRegisterTableCalled.get === 0) assert(cs.numCommitsCalled.get === 0) val txn = log.startTransaction() txn.updateMetadataForNewTable(newMetadata) txn.commitLarge( spark, Seq(SetTransaction("app-1", 1, None)).toIterator, Some(newProtocol), DeltaOperations.TestOperation("TEST"), Map.empty, Map.empty) log = DeltaLog.forTable(spark, tablePath) assert(cs.numRegisterTableCalled.get === 1) assert(cs.numCommitsCalled.get === 0) assert(log.unsafeVolatileSnapshot.version === 2L) Seq(V2CheckpointTableFeature, CoordinatedCommitsTableFeature).foreach { feature => assert(log.unsafeVolatileSnapshot.protocol.isFeatureSupported(feature)) } assert(log.unsafeVolatileSnapshot.tableCommitCoordinatorClientOpt.nonEmpty) assert(log.unsafeVolatileSnapshot.metadata.coordinatedCommitsCoordinatorName === Some(builder.getName)) assert(log.unsafeVolatileSnapshot.metadata.coordinatedCommitsCoordinatorConf === Map.empty) assert(log.unsafeVolatileSnapshot.metadata.coordinatedCommitsTableConf === Map.empty) Seq(3).toDF.write.mode("append").format("delta").save(tablePath) assert(cs.numRegisterTableCalled.get === 1) assert(cs.numCommitsCalled.get === 1) assert(log.unsafeVolatileSnapshot.version === 3L) assert(log.unsafeVolatileSnapshot.tableCommitCoordinatorClientOpt.nonEmpty) } } test("Incomplete backfills are handled properly by next commit after CC to FS conversion") { if (isCatalogOwnedTest) { cancel("Downgrade is not yet supported for catalog owned tables") } val batchSize = 10 val neverBackfillingCommitCoordinator = new TrackingCommitCoordinatorClient(new InMemoryCommitCoordinator(batchSize) { override def backfillToVersion( logStore: LogStore, hadoopConf: Configuration, tableDesc: TableDescriptor, version: Long, lastKnownBackfilledVersionOpt: JLong): Unit = { } }) clearBuilders() val builder = TrackingInMemoryCommitCoordinatorBuilder(batchSize, Some(neverBackfillingCommitCoordinator)) registerBuilder(builder) withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath Seq(1).toDF.write.format("delta").mode("overwrite").save(tablePath) // v0 Seq(1).toDF.write.format("delta").mode("overwrite").save(tablePath) // v1 Seq(1).toDF.write.format("delta").mode("overwrite").save(tablePath) // v2 val log = DeltaLog.forTable(spark, tablePath) assert(log.unsafeVolatileSnapshot.tableCommitCoordinatorClientOpt.nonEmpty) assert(log.unsafeVolatileSnapshot.version === 2) assert( log.unsafeVolatileSnapshot.logSegment.deltas.count(FileNames.isUnbackfilledDeltaFile) == 2) val oldMetadata = log.unsafeVolatileSnapshot.metadata val downgradeMetadata = oldMetadata.copy( configuration = oldMetadata.configuration - COORDINATED_COMMITS_COORDINATOR_NAME.key) log.startTransaction().commitManually(downgradeMetadata) log.update() val snapshotAfterDowngrade = log.unsafeVolatileSnapshot assert(snapshotAfterDowngrade.version === 3) assert(snapshotAfterDowngrade.tableCommitCoordinatorClientOpt.isEmpty) assert(snapshotAfterDowngrade.logSegment.deltas.count(FileNames.isUnbackfilledDeltaFile) == 3) val records = Log4jUsageLogger.track { // commit 4 Seq(1).toDF.write.format("delta").mode("overwrite").save(tablePath) } val filteredUsageLogs = filterUsageRecords( records, "delta.coordinatedCommits.backfillWhenCoordinatedCommitsSupportedAndDisabled") assert(filteredUsageLogs.size === 1) val usageObj = JsonUtils.fromJson[Map[String, Any]](filteredUsageLogs.head.blob) assert(usageObj("numUnbackfilledFiles").asInstanceOf[Int] === 3) assert(usageObj("numAlreadyBackfilledFiles").asInstanceOf[Int] === 0) } } test("LogSegment comparison does not swap snapshots that only differ in " + "backfilled/unbackfilled commits") { // Use a batch size of two so we don't immediately backfill in // the AbstractBatchBackfillingCommitCoordinatorClient and so the // CommitResponse contains the UUID-based commit. registerBuilder( TrackingInMemoryCommitCoordinatorBuilder(batchSize = 2)) withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath val log = DeltaLog.forTable(spark, tablePath) withSQLConf( CHECKPOINT_POLICY.defaultTablePropertyKey -> CheckpointPolicy.Classic.name) { // Version 0 -- backfilled by default makeCommitAndAssertSnapshotState( data = Seq(0), expectedLastKnownBackfilledVersion = 0, expectedNumUnbackfilledCommits = 0, expectedLastKnownBackfilledFile = FileNames.unsafeDeltaFile(log.logPath, 0), log, tablePath) // Version 1 -- not backfilled immediately because of batchSize = 2 makeCommitAndAssertSnapshotState( data = Seq(1), expectedLastKnownBackfilledVersion = 0, expectedNumUnbackfilledCommits = 1, expectedLastKnownBackfilledFile = FileNames.unsafeDeltaFile(log.logPath, 0), log, tablePath) // Version 2 -- backfills versions 1 and 2 makeCommitAndAssertSnapshotState( data = Seq(2), expectedLastKnownBackfilledVersion = 2, expectedNumUnbackfilledCommits = 2, expectedLastKnownBackfilledFile = FileNames.unsafeDeltaFile(log.logPath, 0), log, tablePath) // Version 3 -- not backfilled immediately because of batchSize = 2 makeCommitAndAssertSnapshotState( data = Seq(3), expectedLastKnownBackfilledVersion = 2, expectedNumUnbackfilledCommits = 3, expectedLastKnownBackfilledFile = FileNames.unsafeDeltaFile(log.logPath, 0), log, tablePath) // Version 4 -- backfills versions 3 and 4 makeCommitAndAssertSnapshotState( data = Seq(4), expectedLastKnownBackfilledVersion = 4, expectedNumUnbackfilledCommits = 4, expectedLastKnownBackfilledFile = FileNames.unsafeDeltaFile(log.logPath, 0), log, tablePath) // Trigger a checkpoint log.checkpoint(log.update()) // Version 5 -- not backfilled immediately because of batchSize 2 makeCommitAndAssertSnapshotState( data = Seq(5), expectedLastKnownBackfilledVersion = 4, expectedNumUnbackfilledCommits = 1, expectedLastKnownBackfilledFile = FileNames.checkpointFileSingular(log.logPath, 4), log, tablePath) // Version 6 -- backfills versions 5 and 6 makeCommitAndAssertSnapshotState( data = Seq(6), expectedLastKnownBackfilledVersion = 6, expectedNumUnbackfilledCommits = 2, expectedLastKnownBackfilledFile = FileNames.checkpointFileSingular(log.logPath, 4), log, tablePath) } } } private def makeCommitAndAssertSnapshotState( data: Seq[Long], expectedLastKnownBackfilledVersion: Long, expectedNumUnbackfilledCommits: Long, expectedLastKnownBackfilledFile: Path, log: DeltaLog, tablePath: String): Unit = { data.toDF().write.format("delta").mode("overwrite").save(tablePath) val snapshot = log.update() val segment = snapshot.logSegment var numUnbackfilledCommits = 0 segment.deltas.foreach { case UnbackfilledDeltaFile(_, _, _) => numUnbackfilledCommits += 1 case _ => // do nothing } assert(snapshot.getLastKnownBackfilledVersion == expectedLastKnownBackfilledVersion) assert(numUnbackfilledCommits == expectedNumUnbackfilledCommits) val lastKnownBackfilledFile = CoordinatedCommitsUtils .getLastBackfilledFile(segment.deltas).getOrElse( segment.checkpointProvider.topLevelFiles.head ) assert(lastKnownBackfilledFile.getPath == expectedLastKnownBackfilledFile, s"$lastKnownBackfilledFile did not equal $expectedLastKnownBackfilledFile") } for (ignoreMissingCCImpl <- BOOLEAN_DOMAIN) test(s"missing coordinator implementation [ignoreMissingCCImpl = $ignoreMissingCCImpl]") { if (isCatalogOwnedTest) { cancel("Error message is not yet customized for CatalogOwned table.") } clearBuilders() registerBuilder(TrackingInMemoryCommitCoordinatorBuilder(batchSize = 2)) withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath Seq(0).toDF.write.format("delta").save(tablePath) (1 to 3).foreach { v => Seq(v).toDF.write.mode("append").format("delta").save(tablePath) } // The table has 3 backfilled commits [0, 1, 2] and 1 unbackfilled commit [3] clearBuilders() def getUsageLogsAndEnsurePresenceOfMissingCCImplLog( expectedFailIfImplUnavailable: Boolean)(f: => Unit): Seq[UsageRecord] = { val usageLogs = Log4jUsageLogger.track { f } val filteredLogs = filterUsageRecords( usageLogs, CoordinatedCommitsUsageLogs.COMMIT_COORDINATOR_MISSING_IMPLEMENTATION) assert(filteredLogs.nonEmpty) val usageObj = JsonUtils.fromJson[Map[String, Any]](filteredLogs.head.blob) assert(usageObj("commitCoordinatorName") === "tracking-in-memory") assert(usageObj("registeredCommitCoordinators") === CommitCoordinatorProvider.getRegisteredCoordinatorNames.mkString(", ")) assert(usageObj("failIfImplUnavailable") === expectedFailIfImplUnavailable.toString) usageLogs } withSQLConf( DeltaSQLConf.COORDINATED_COMMITS_IGNORE_MISSING_COORDINATOR_IMPLEMENTATION.key -> ignoreMissingCCImpl.toString) { DeltaLog.clearCache() if (!ignoreMissingCCImpl) { getUsageLogsAndEnsurePresenceOfMissingCCImplLog(expectedFailIfImplUnavailable = true) { val e = intercept[IllegalArgumentException] { DeltaLog.forTable(spark, tablePath) } assert(e.getMessage.contains("Unknown commit-coordinator")) } } else { val deltaLog = DeltaLog.forTable(spark, tablePath) assert(deltaLog.snapshot.tableCommitCoordinatorClientOpt.isEmpty) // This will create a stale deltaLog as the commit-coordinator is missing. assert(deltaLog.snapshot.version === 2L) DeltaLog.clearCache() getUsageLogsAndEnsurePresenceOfMissingCCImplLog(expectedFailIfImplUnavailable = false) { checkAnswer(spark.read.format("delta").load(tablePath), Seq(0, 1, 2).toDF()) } // Writes and checkpoints should still fail. val createCheckpointFn = () => (deltaLog.checkpoint()) val writeDataFn = () => Seq(4).toDF.write.format("delta").mode("append").save(tablePath) for (tableMutationFn <- Seq(createCheckpointFn, writeDataFn)) { DeltaLog.clearCache() val usageLogs = Log4jUsageLogger.track { val e = intercept[DeltaUnsupportedOperationException] { tableMutationFn() } checkError(e, "DELTA_UNSUPPORTED_WRITES_WITHOUT_COORDINATOR", sqlState = "0AKDC", parameters = Map("coordinatorName" -> "tracking-in-memory") ) assert(e.getMessage.contains( "no implementation of this coordinator is available in the current environment")) } val filteredLogs = filterUsageRecords( usageLogs, CoordinatedCommitsUsageLogs.COMMIT_COORDINATOR_MISSING_IMPLEMENTATION_WRITE) val usageObj = JsonUtils.fromJson[Map[String, Any]](filteredLogs.head.blob) assert(usageObj("commitCoordinatorName") === "tracking-in-memory") assert(usageObj("readVersion") === "2") } } } } } ///////////////////////////////////////////////////////////////////////////////////////////// // Test coordinated-commits with DeltaLog.getChangeLogFile API starts // ///////////////////////////////////////////////////////////////////////////////////////////// /** * Helper method which generates a delta table with `totalCommits`. * The `upgradeToCoordinatedCommitsVersion`th commit version upgrades this table to coordinated * commits and it uses `backfillInterval` for backfilling. * This method returns a mapping of version to DeltaLog for the versions in * `requiredDeltaLogVersions`. Each of this deltaLog object has a Snapshot as per what is * mentioned in the `requiredDeltaLogVersions`. */ private def generateDataForGetChangeLogFilesTest( dir: File, totalCommits: Int, upgradeToCoordinatedCommitsVersion: Int, backfillInterval: Int, requiredDeltaLogVersions: Set[Int]): Map[Int, DeltaLog] = { if (isCatalogOwnedTest) { cancel("Upgrade is not yet supported for catalog owned tables") } val commitCoordinatorClient = new TrackingCommitCoordinatorClient(new InMemoryCommitCoordinator(backfillInterval)) val builder = TrackingInMemoryCommitCoordinatorBuilder(backfillInterval, Some(commitCoordinatorClient)) registerBuilder(builder) val versionToDeltaLogMapping = collection.mutable.Map.empty[Int, DeltaLog] withSQLConf( CHECKPOINT_INTERVAL.defaultTablePropertyKey -> "100") { val tablePath = dir.getAbsolutePath (0 to totalCommits).foreach { v => if (v === upgradeToCoordinatedCommitsVersion) { val deltaLog = DeltaLog.forTable(spark, tablePath) val oldMetadata = deltaLog.unsafeVolatileSnapshot.metadata val commitCoordinator = (COORDINATED_COMMITS_COORDINATOR_NAME.key -> "tracking-in-memory") val newMetadata = oldMetadata.copy(configuration = oldMetadata.configuration + commitCoordinator) deltaLog.startTransaction().commitManually(newMetadata) } else { Seq(v).toDF().write.format("delta").mode("append").save(tablePath) } if (requiredDeltaLogVersions.contains(v)) { versionToDeltaLogMapping.put(v, DeltaLog.forTable(spark, tablePath)) DeltaLog.clearCache() } } } versionToDeltaLogMapping.toMap } def runGetChangeLogFiles( deltaLog: DeltaLog, startVersion: Long, endVersionOpt: Option[Long] = None, totalCommitsOnTable: Long = 8, expectedLastBackfilledCommit: Long, updateExpected: Boolean): Unit = { val usageRecords = Log4jUsageLogger.track { val iter = endVersionOpt match { case Some(endVersion) => deltaLog.getChangeLogFiles( startVersion, endVersion, catalogTableOpt = None, failOnDataLoss = false) case None => deltaLog.getChangeLogFiles(startVersion) } val paths = iter.map(_._2.getPath).toIndexedSeq val (backfilled, unbackfilled) = paths.partition(_.getParent === deltaLog.logPath) val expectedBackfilledCommits = startVersion to expectedLastBackfilledCommit val expectedUnbackfilledCommits = { val firstUnbackfilledVersion = (expectedLastBackfilledCommit + 1).max(startVersion) val expectedEndVersion = endVersionOpt.getOrElse(totalCommitsOnTable) firstUnbackfilledVersion to expectedEndVersion } assert(backfilled.map(FileNames.deltaVersion) === expectedBackfilledCommits) assert(unbackfilled.map(FileNames.deltaVersion) === expectedUnbackfilledCommits) } val updateCountEvents = if (updateExpected) 1 else 0 assert(filterUsageRecords(usageRecords, "deltaLog.update").size === updateCountEvents) } testWithDefaultCommitCoordinatorUnset("DeltaLog.getChangeLogFile with and" + " without endVersion [No Coordinated Commits]") { withTempDir { dir => val versionsToDeltaLogMapping = generateDataForGetChangeLogFilesTest( dir, totalCommits = 4, upgradeToCoordinatedCommitsVersion = -1, backfillInterval = -1, requiredDeltaLogVersions = Set(2, 4)) // We are asking for changes between 0 and 0 to a DeltaLog(unsafeVolatileSnapshot = 2). // So we should not need an update() as all the required files are on filesystem. runGetChangeLogFiles( versionsToDeltaLogMapping(2), totalCommitsOnTable = 4, startVersion = 0, endVersionOpt = Some(0), expectedLastBackfilledCommit = 0, updateExpected = false) // We are asking for changes between 0 to `end` to a DeltaLog(unsafeVolatileSnapshot = 2). // Since the commits in filesystem are more than what unsafeVolatileSnapshot has, we should // need an update() to get the latest snapshot and see if coordinated commits was enabled on // the table concurrently. runGetChangeLogFiles( versionsToDeltaLogMapping(2), totalCommitsOnTable = 4, startVersion = 0, expectedLastBackfilledCommit = 4, updateExpected = true) // We are asking for changes between 0 to `end` to a DeltaLog(unsafeVolatileSnapshot = 4). // The latest commit from filesystem listing is 4 -- same as unsafeVolatileSnapshot and this // unsafeVolatileSnapshot doesn't have coordinated commits enabled. So we should not need an // update(). runGetChangeLogFiles( versionsToDeltaLogMapping(4), totalCommitsOnTable = 4, startVersion = 0, expectedLastBackfilledCommit = 4, updateExpected = false) } } testWithDefaultCommitCoordinatorUnset("DeltaLog.getChangeLogFile with and" + " without endVersion [Coordinated Commits backfill size 1]") { withTempDir { dir => val versionsToDeltaLogMapping = generateDataForGetChangeLogFilesTest( dir, totalCommits = 4, upgradeToCoordinatedCommitsVersion = 2, backfillInterval = 1, requiredDeltaLogVersions = Set(0, 2, 4)) // We are asking for changes between 0 and 0 to a DeltaLog(unsafeVolatileSnapshot = 2). // So we should not need an update() as all the required files are on filesystem. runGetChangeLogFiles( versionsToDeltaLogMapping(2), totalCommitsOnTable = 4, startVersion = 0, endVersionOpt = Some(0), expectedLastBackfilledCommit = 0, updateExpected = false) // We are asking for changes between 0 to `end` to a DeltaLog(unsafeVolatileSnapshot = 2). // Since the commits in filesystem are more than what unsafeVolatileSnapshot has, we should // need an update() to get the latest snapshot and see if coordinated commits was enabled on // the table concurrently. runGetChangeLogFiles( versionsToDeltaLogMapping(2), totalCommitsOnTable = 4, startVersion = 0, expectedLastBackfilledCommit = 4, updateExpected = true) // We are asking for changes between 0 to 4 to a DeltaLog(unsafeVolatileSnapshot = 4). // Since the commits in filesystem are between 0 to 4, so we don't need to update() to get // the latest snapshot and see if coordinated commits was enabled on the table concurrently. runGetChangeLogFiles( versionsToDeltaLogMapping(4), totalCommitsOnTable = 4, startVersion = 0, endVersionOpt = Some(4), expectedLastBackfilledCommit = 4, updateExpected = false) // We are asking for changes between 0 to `end` to a DeltaLog(unsafeVolatileSnapshot = 4). // The latest commit from filesystem listing is 4 -- same as unsafeVolatileSnapshot and this // unsafeVolatileSnapshot has coordinated commits enabled. So we should need an update() to // find out latest commits from Commit Coordinator. runGetChangeLogFiles( versionsToDeltaLogMapping(4), totalCommitsOnTable = 4, startVersion = 0, expectedLastBackfilledCommit = 4, updateExpected = true) } } testWithDefaultCommitCoordinatorUnset("DeltaLog.getChangeLogFile with and" + " without endVersion [Coordinated Commits backfill size 10]") { withTempDir { dir => val versionsToDeltaLogMapping = generateDataForGetChangeLogFilesTest( dir, totalCommits = 8, upgradeToCoordinatedCommitsVersion = 2, backfillInterval = 10, requiredDeltaLogVersions = Set(2, 3, 4, 8)) // We are asking for changes between 0 and 1 to a DeltaLog(unsafeVolatileSnapshot = 2/4). // So we should not need an update() as all the required files are on filesystem. Seq(2, 3, 4).foreach { version => runGetChangeLogFiles( versionsToDeltaLogMapping(version), totalCommitsOnTable = 8, startVersion = 0, endVersionOpt = Some(1), expectedLastBackfilledCommit = 1, updateExpected = false) } // We are asking for changes between 0 to `end` to a DeltaLog(unsafeVolatileSnapshot = 2/4). // Since the unsafeVolatileSnapshot has coordinated-commits enabled, so we need to trigger an // update to find the latest commits from Commit Coordinator. Seq(2, 3, 4).foreach { version => runGetChangeLogFiles( versionsToDeltaLogMapping(version), totalCommitsOnTable = 8, startVersion = 0, expectedLastBackfilledCommit = 2, updateExpected = true) } // We are asking for changes between 0 to `4` to a DeltaLog(unsafeVolatileSnapshot = 8). // The filesystem has only commit 0/1/2. // After that we need to rely on deltaLog.update() to get the latest snapshot and return the // files until 8. // Ideally the unsafeVolatileSnapshot should have the info to generate files from 0 to 4 and // an update() should not be needed. This is an optimization that can be done in future. runGetChangeLogFiles( versionsToDeltaLogMapping(8), totalCommitsOnTable = 8, startVersion = 0, endVersionOpt = Some(4), expectedLastBackfilledCommit = 2, updateExpected = true) } } ///////////////////////////////////////////////////////////////////////////////////////////// // Test coordinated-commits with DeltaLog.getChangeLogFile API ENDS // ///////////////////////////////////////////////////////////////////////////////////////////// test("During ALTER, overriding ICT configurations on (potential) Coordinated Commits " + "or Catalog Owned tables throws an exception.") { registerBuilder(TrackingInMemoryCommitCoordinatorBuilder(1)) // For a table that had Coordinated Commits enabled before the ALTER command. withTempDir { tempDir => sql(s"CREATE TABLE delta.`${tempDir.getAbsolutePath}` (id LONG) USING delta TBLPROPERTIES " + propertiesString) val e = interceptWithUnwrapping[DeltaIllegalArgumentException] { sql(s"ALTER TABLE delta.`${tempDir.getAbsolutePath}` SET TBLPROPERTIES " + s"('${IN_COMMIT_TIMESTAMPS_ENABLED.key}' = 'false')") } if (isCatalogOwnedTest) { checkError( e, "DELTA_CANNOT_MODIFY_CATALOG_MANAGED_DEPENDENCIES", sqlState = "42616", parameters = Map[String, String]()) } else { checkError( e, "DELTA_CANNOT_MODIFY_COORDINATED_COMMITS_DEPENDENCIES", sqlState = "42616", parameters = Map("Command" -> "ALTER")) } } if (isCatalogOwnedTest) { cancel("Upgrade is not yet supported for catalog owned tables") } // For a table that is about to enable Coordinated Commits during the same ALTER command. withoutDefaultCCTableFeature { withTempDir { tempDir => sql(s"CREATE TABLE delta.`${tempDir.getAbsolutePath}` (id LONG) USING delta") val e = interceptWithUnwrapping[DeltaIllegalArgumentException] { sql(s"ALTER TABLE delta.`${tempDir.getAbsolutePath}` SET TBLPROPERTIES " + s"('${COORDINATED_COMMITS_COORDINATOR_NAME.key}' = 'tracking-in-memory', " + s"'${COORDINATED_COMMITS_COORDINATOR_CONF.key}' = '${JsonUtils.toJson(Map())}', " + s"'${IN_COMMIT_TIMESTAMPS_ENABLED.key}' = 'false')") } checkError( e, "DELTA_CANNOT_SET_COORDINATED_COMMITS_DEPENDENCIES", sqlState = "42616", parameters = Map("Command" -> "ALTER")) } } } test("During ALTER, unsetting ICT configurations on Coordinated Commits tables throws an " + "exception.") { registerBuilder(TrackingInMemoryCommitCoordinatorBuilder(1)) withTempDir { tempDir => sql(s"CREATE TABLE delta.`${tempDir.getAbsolutePath}` (id LONG) USING delta TBLPROPERTIES " + propertiesString) val e = interceptWithUnwrapping[DeltaIllegalArgumentException] { sql(s"ALTER TABLE delta.`${tempDir.getAbsolutePath}` UNSET TBLPROPERTIES " + s"('${IN_COMMIT_TIMESTAMPS_ENABLED.key}')") } if (isCatalogOwnedTest) { checkError( e, "DELTA_CANNOT_MODIFY_CATALOG_MANAGED_DEPENDENCIES", sqlState = "42616", parameters = Map[String, String]()) } else { checkError( e, "DELTA_CANNOT_MODIFY_COORDINATED_COMMITS_DEPENDENCIES", sqlState = "42616", parameters = Map("Command" -> "ALTER")) } } } test("During REPLACE, for non-CC tables, default CC configurations are ignored, but default " + "ICT confs are retained, and existing ICT confs are discarded") { // Non-CC table, REPLACE with default CC and ICT confs => Non-CC, but with ICT confs. withTempDir { tempDir => withoutDefaultCCTableFeature { sql(s"CREATE TABLE delta.`${tempDir.getAbsolutePath}` (id LONG) USING delta") } withSQLConf(IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey -> "true") { sql(s"REPLACE TABLE delta.`${tempDir.getAbsolutePath}` (id STRING) USING delta") } assert(DeltaLog.forTable(spark, tempDir).snapshot.tableCommitCoordinatorClientOpt.isEmpty) assert(!DeltaLog.forTable(spark, tempDir).snapshot.isCatalogOwned) assert(DeltaLog.forTable(spark, tempDir).snapshot.metadata.configuration.contains( IN_COMMIT_TIMESTAMPS_ENABLED.key)) } // Non-CC table with ICT confs, REPLACE with only default CC confs => Non-CC, also no ICT confs. withTempDir { tempDir => withoutDefaultCCTableFeature { withSQLConf(IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey -> "true") { sql(s"CREATE TABLE delta.`${tempDir.getAbsolutePath}` (id LONG) USING delta") } } sql(s"REPLACE TABLE delta.`${tempDir.getAbsolutePath}` (id STRING) USING delta") val snapshot = DeltaLog.forTable(spark, tempDir).unsafeVolatileSnapshot assert(snapshot.tableCommitCoordinatorClientOpt.isEmpty) assert(!snapshot.isCatalogOwned) assert(!snapshot.metadata.configuration.contains(IN_COMMIT_TIMESTAMPS_ENABLED.key)) } } test("During REPLACE, for CC tables, existing CC and ICT configurations are both retained.") { registerBuilder(TrackingInMemoryCommitCoordinatorBuilder(1)) withTable("t1") { if (isCatalogOwnedTest) { sql(s"CREATE TABLE t1 (id LONG) USING delta") sql(s"INSERT INTO t1 VALUES (0)") } else { withoutDefaultCCTableFeature { sql(s"CREATE TABLE t1 (id LONG) USING delta") sql(s"INSERT INTO t1 VALUES (0)") sql(s"ALTER TABLE t1 SET TBLPROPERTIES " + propertiesString) } } withoutDefaultCCTableFeature { // All three ICT configurations should be set because CC feature is enabled later. // REPLACE w/o default CC confs => CC, and all ICT confs. sql(s"REPLACE TABLE t1 (id STRING) USING delta") val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, CatalystTableIdentifier("t1")) if (isCatalogOwnedTest) { assert(snapshot.isCatalogOwned) // Only [[IN_COMMIT_TIMESTAMPS_ENABLED]] should be set for CatalogOwned // since we don't support upgrade yet. assert(snapshot.metadata.configuration.contains(IN_COMMIT_TIMESTAMPS_ENABLED.key)) } else { assert(snapshot.tableCommitCoordinatorClientOpt.nonEmpty) CoordinatedCommitsUtils.ICT_TABLE_PROPERTY_KEYS.foreach { key => assert(snapshot.metadata.configuration.contains(key)) } } } } } test("CREATE LIKE does not copy commit coordinated related feature config " + "from the source table.") { registerBuilder(TrackingInMemoryCommitCoordinatorBuilder(1)) val source = "sourcetable" val target = "targettable" sql(s"CREATE TABLE $source (id LONG) USING delta TBLPROPERTIES" + propertiesString) sql(s"CREATE TABLE $target LIKE $source") val snapshot = DeltaLog.forTable(spark, target).unsafeVolatileSnapshot assert(snapshot.tableCommitCoordinatorClientOpt.isEmpty) assert(!snapshot.isCatalogOwned) } test("CREATE an external table in a location with an existing table works correctly.") { if (isCatalogOwnedTest) { cancel("Creating an external table is not yet supported for CatalogOwned.") } registerBuilder(TrackingInMemoryCommitCoordinatorBuilder(1)) // When the existing table has a commit coordinator, omitting CC configurations in the command // should not throw an exception, and the commit coordinator should be retained, so should ICT. withTempDir { dir => val tableName = "testtable" val tablePath = dir.getAbsolutePath sql(s"CREATE TABLE delta.`${dir.getAbsolutePath}` (id LONG) USING delta TBLPROPERTIES " + s"('foo' = 'bar', " + propertiesString.substring(1)) sql(s"CREATE TABLE $tableName (id LONG) USING delta TBLPROPERTIES " + s"('foo' = 'bar') LOCATION '${dir.getAbsolutePath}'") val snapshot = DeltaLog.forTable(spark, tablePath).snapshot assert(snapshot.tableCommitCoordinatorClientOpt.nonEmpty || snapshot.isCatalogOwned) assert(snapshot.metadata.configuration.contains(IN_COMMIT_TIMESTAMPS_ENABLED.key)) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsTestUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import java.util.Optional import java.util.concurrent.atomic.AtomicInteger import scala.collection.mutable import scala.util.control.NonFatal import org.apache.spark.sql.delta.{CatalogOwnedTableFeature, CheckpointPolicy, DeltaColumnMappingMode, DeltaConfig, DeltaConfigs, DeltaLog, DeltaTestUtilsBase, DomainMetadataTableFeature, MaterializedRowCommitVersion, MaterializedRowId, RowTrackingFeature, Snapshot, TableFeature} import org.apache.spark.sql.delta.actions.{CommitInfo, Metadata, Protocol, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.util.{DeltaCommitFileProvider, JsonUtils} import io.delta.storage.LogStore import io.delta.storage.commit.{CommitCoordinatorClient, CommitResponse, TableDescriptor, TableIdentifier, UpdatedActions, GetCommitsResponse => JGetCommitsResponse} import io.delta.storage.commit.actions.{AbstractMetadata, AbstractProtocol} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.{SparkConf, SparkFunSuite} import org.apache.spark.sql.{QueryTest, Row, SparkSession} import org.apache.spark.sql.catalyst.{TableIdentifier => CatalystTableIdentifier} import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.functions.col import org.apache.spark.sql.test.SharedSparkSession // This trait is built to serve as a base trait for tests built for both CatalogOwned // and commit-coordinators table feature. trait CommitCoordinatorUtilBase { /** * Runs a specific test with commit coordinator feature unset. */ def testWithDefaultCommitCoordinatorUnset(testName: String)(f: => Unit) /** * Runs the function `f` with commit coordinator table feature unset. * Any table created in function `f` have CatalogOwned/CoordinatedCommits disabled by default. */ def withoutDefaultCCTableFeature(f: => Unit): Unit /** * Runs the function `f` with commit coordinator table feature set. * Any table created in function `f` have CatalogOwned/CoordinatedCommits enabled by default.` */ def withDefaultCCTableFeature(f: => Unit): Unit /** Run the test with different backfill batch sizes: 1, 2, 10 */ def testWithDifferentBackfillInterval(testName: String)(f: Int => Unit): Unit /** Register a builder to the appropriate builder provider. */ def registerBuilder(builder: CommitCoordinatorBuilder): Unit /** Clear relevant table feature commit coordinator builders that are registered. */ def clearBuilders(): Unit /** Returns the properties string to be used in the table creation for test. */ def propertiesString: String /** * Returns true if this test is about CatalogOwned table feature. * Returns false if this test is about CoordinatedCommits tabel feature. */ def isCatalogOwnedTest: Boolean /** Keeps track of the number of table names pointing to the location. */ protected val locRefCount: mutable.Map[String, Int] = mutable.Map.empty } trait CatalogOwnedTestBaseSuite extends SparkFunSuite with DeltaTestUtilsBase with CommitCoordinatorUtilBase with SharedSparkSession { val defaultCatalogOwnedFeatureEnabledKey: String = TableFeatureProtocolUtils.defaultPropertyKey(CatalogOwnedTableFeature) // If this config is not overridden, newly created table is not CatalogOwned by default. def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = None def catalogOwnedDefaultCreationEnabledInTests: Boolean = catalogOwnedCoordinatorBackfillBatchSize.nonEmpty /** * Returns the commit coordinator client for the specified catalog. * * @param catalogName The name of the catalog to get the commit coordinator client for. * @return The commit coordinator client for the specified catalog. */ protected def getCatalogOwnedCommitCoordinatorClient( catalogName: String): CommitCoordinatorClient = { CatalogOwnedCommitCoordinatorProvider.getBuilder(catalogName).getOrElse { throw new IllegalStateException( s"Commit coordinator builder is not available for the specified catalog: $catalogName") }.buildForCatalog(spark, catalogName) } override protected def sparkConf: SparkConf = { if (catalogOwnedDefaultCreationEnabledInTests) { super.sparkConf.set(defaultCatalogOwnedFeatureEnabledKey, "supported") } else { super.sparkConf } } override def clearBuilders(): Unit = { CatalogOwnedCommitCoordinatorProvider.clearBuilders() } override def propertiesString: String = s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported')" override protected def beforeEach(): Unit = { super.beforeEach() CatalogOwnedCommitCoordinatorProvider.clearBuilders() catalogOwnedCoordinatorBackfillBatchSize.foreach { batchSize => CatalogOwnedCommitCoordinatorProvider.registerBuilder( catalogName = CatalogOwnedTableUtils.DEFAULT_CATALOG_NAME_FOR_TESTING, commitCoordinatorBuilder = TrackingInMemoryCommitCoordinatorBuilder(batchSize) ) } DeltaLog.clearCache() } override def testWithDefaultCommitCoordinatorUnset(testName: String)(f: => Unit): Unit = { test(testName) { withoutDefaultCCTableFeature { f } } } override def withDefaultCCTableFeature(f: => Unit): Unit = { val oldConfig = spark.conf.getOption(defaultCatalogOwnedFeatureEnabledKey) spark.conf.set(defaultCatalogOwnedFeatureEnabledKey, "supported") try { f } finally { if (oldConfig.isDefined) { spark.conf.set(defaultCatalogOwnedFeatureEnabledKey, oldConfig.get) } else { spark.conf.unset(defaultCatalogOwnedFeatureEnabledKey) } } } override def withoutDefaultCCTableFeature(f: => Unit): Unit = { val oldConfig = spark.conf.getOption(defaultCatalogOwnedFeatureEnabledKey) spark.conf.unset(defaultCatalogOwnedFeatureEnabledKey) try { f } finally { if (oldConfig.isDefined) { spark.conf.set(defaultCatalogOwnedFeatureEnabledKey, oldConfig.get) } } } override def testWithDifferentBackfillInterval(testName: String)(f: Int => Unit): Unit = { Seq(1, 2, 10).foreach { backfillBatchSize => test(s"$testName [Backfill batch size: $backfillBatchSize]") { CatalogOwnedCommitCoordinatorProvider.clearBuilders() CatalogOwnedCommitCoordinatorProvider.registerBuilder( "spark_catalog", TrackingInMemoryCommitCoordinatorBuilder(batchSize = backfillBatchSize)) f(backfillBatchSize) } } } /** * Run the test against a [[TrackingCommitCoordinatorClient]] with backfill batch size = * `batchBackfillSize` */ def testWithCatalogOwned(backfillBatchSize: Int)(testName: String)(f: => Unit): Unit = { test(s"$testName [Backfill batch size: $backfillBatchSize]") { CatalogOwnedCommitCoordinatorProvider.clearBuilders() CatalogOwnedCommitCoordinatorProvider.registerBuilder( CatalogOwnedTableUtils.DEFAULT_CATALOG_NAME_FOR_TESTING, TrackingInMemoryCommitCoordinatorBuilder(batchSize = backfillBatchSize)) withDefaultCCTableFeature { f } } } override def registerBuilder(builder: CommitCoordinatorBuilder): Unit = { assert(builder.isInstanceOf[CatalogOwnedCommitCoordinatorBuilder], s"builder $builder(${builder.getName}) must be CatalogOwnedCommitCoordinatorBuilder") CatalogOwnedCommitCoordinatorProvider.registerBuilder( "spark_catalog", builder.asInstanceOf[CatalogOwnedCommitCoordinatorBuilder]) } override def isCatalogOwnedTest: Boolean = true def deleteCatalogOwnedTableFromCommitCoordinator(tableName: String): Unit = { val location = try { spark.sql(s"describe detail $tableName") .select("location") .first() .getAs[String](0) } catch { case NonFatal(_) => // Ignore if the table does not exist/broken. return } deleteCatalogOwnedTableFromCommitCoordinator(path = new Path(location)) } def deleteCatalogOwnedTableFromCommitCoordinator(path: Path): Unit = { val catalogName = "spark_catalog" val cc = CatalogOwnedCommitCoordinatorProvider.getBuilder(catalogName).getOrElse { throw new IllegalStateException( s"Unable to get CatalogOwnedCommitCoordinatorBuilder for table at path: ${path.toString}") }.buildForCatalog(spark, catalogName) assert( cc.isInstanceOf[TrackingCommitCoordinatorClient], s"Please implement delete/drop method for coordinator: ${cc.getClass.getName}") val locKey = path.toString.stripPrefix("file:") if (locRefCount.contains(locKey)) { locRefCount(locKey) -= 1 } // When we create an external table in a location where some table already existed, two table // names could be pointing to the same location. We should only clean up the table data in the // commit coordinator when the last table name pointing to the location is dropped. if (locRefCount.getOrElse(locKey, 0) == 0) { val logPath = new Path(path, "_delta_log") cc.asInstanceOf[TrackingCommitCoordinatorClient] .delegatingCommitCoordinatorClient .asInstanceOf[InMemoryCommitCoordinator] .dropTable(logPath) } DeltaLog.clearCache() } /** * Constructs the specific table properties for Catalog Owned tables. * * @param spark The Spark session. * @param metadata The metadata of the CC table. * @return A map of CC specific table properties. */ def constructCatalogOwnedSpecificTableProperties( spark: SparkSession, metadata: Metadata): Map[String, String] = { if (catalogOwnedDefaultCreationEnabledInTests) { val qolConfs = CatalogOwnedTableUtils.QOL_TABLE_FEATURES_AND_PROPERTIES .collect { case (feature, config, value) => config.key -> value } .toMap // RowTracking specific properties. qolConfs ++ Map( MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP -> metadata.configuration.getOrElse( MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP, fail(s"Failed to get ${MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP}.") ), MaterializedRowCommitVersion.MATERIALIZED_COLUMN_NAME_PROP -> metadata.configuration.getOrElse( MaterializedRowCommitVersion.MATERIALIZED_COLUMN_NAME_PROP, fail(s"Failed to get ${MaterializedRowCommitVersion.MATERIALIZED_COLUMN_NAME_PROP}.") ) ) ++ Map(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key -> "true") } else { Map.empty } } /** * Returns the properties that are expected to show up in the table properties of a Delta table * when catalog owned is enabled in tests. */ def extractCatalogOwnedSpecificPropertiesIfEnabled( metadata: Metadata): Iterable[(String, String)] = { if (catalogOwnedDefaultCreationEnabledInTests) { val CATALOG_OWNED_TABLE_QOL_PROPERTY_KEYS = CatalogOwnedTableUtils.QOL_TABLE_FEATURES_AND_PROPERTIES .map { case (_, config, _) => config.key } .filterNot(Set( DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key )) ++ Seq( MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP, MaterializedRowCommitVersion.MATERIALIZED_COLUMN_NAME_PROP ) CATALOG_OWNED_TABLE_QOL_PROPERTY_KEYS.map { key => key -> metadata.configuration.getOrElse(key, fail( s"Expected $key to be defined in the table properties")) } ++ Option(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.key -> "true") } else { Seq.empty } } protected def withClassicCheckpointPolicyForCatalogOwned(f: => Unit): Unit = { if (catalogOwnedDefaultCreationEnabledInTests) { withSQLConf( DeltaConfigs.CHECKPOINT_POLICY.defaultTablePropertyKey -> CheckpointPolicy.Classic.name) { f } } else { f } } protected def getDeltaLogWithSnapshot( tableIdentifier: CatalystTableIdentifier): (DeltaLog, Snapshot) = { DeltaLog.forTableWithSnapshot(spark, tableIdentifier) } protected def isICTEnabledForNewTablesCatalogOwned: Boolean = { catalogOwnedCoordinatorBackfillBatchSize.nonEmpty || spark.conf.getOption( DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey).contains("true") } protected def validateTableFeatureAndMetadata( tableName: String, tableFeature: TableFeature, tableFeatureShouldPresent: Boolean, metadataShouldPresent: Boolean, config: DeltaConfig[_], targetValue: String): Unit = { val (_, snapshot) = getDeltaLogWithSnapshot(CatalystTableIdentifier(tableName)) assert(snapshot.protocol.readerAndWriterFeatureNames.contains(tableFeature.name) === tableFeatureShouldPresent, s"expected table feature " + s"${tableFeature.name} to be ${if (tableFeatureShouldPresent) "present" else "absent"}") val metadataValue: String = if (config.key == DeltaConfigs.COLUMN_MAPPING_MODE.key) { config.fromMetaData(snapshot.metadata).asInstanceOf[DeltaColumnMappingMode].name } else if (config.key == DeltaConfigs.CHECKPOINT_POLICY.key) { config.fromMetaData(snapshot.metadata).asInstanceOf[CheckpointPolicy.Policy].name } else { config.fromMetaData(snapshot.metadata).toString } assert((metadataValue == targetValue) === metadataShouldPresent, s"expected the metadata configuration of ${tableFeature.name} to be " + s"${if (metadataShouldPresent) "present" else "absent"}") } protected def validateOnlySpecifiedQoLTableFeaturesAndMetadataPresent( tableName: String, supportedTableFeatures: Set[TableFeature]): Unit = { CatalogOwnedTableUtils.qolTableFeatureAndProperties.foreach { case (t, config, value) => val isSpecifiedTableFeature = supportedTableFeatures.contains(t) validateTableFeatureAndMetadata( tableName, tableFeature = t, tableFeatureShouldPresent = isSpecifiedTableFeature, metadataShouldPresent = isSpecifiedTableFeature, config, targetValue = value) } validateRowTrackingEnablement( tableName, expected = supportedTableFeatures.contains(RowTrackingFeature)) } protected def validateRowTrackingEnablement(tableName: String, expected: Boolean): Unit = { // [[DomainMetadataTableFeature]] is a dependent feature of // [[RowTrackingFeature]] and would be enabled at the same time. // Note: [[DomainMetadataTableFeature]] does not have the corresponding metadata. val (_, snapshot) = getDeltaLogWithSnapshot(CatalystTableIdentifier(tableName)) if (expected) { // If row tracking is enabled, we expect domain metadata to be added. // But when row tracking is not enabled, other features // such as Iceberg V2 could still add domain metadata. assert( snapshot.protocol.readerAndWriterFeatureNames.contains(DomainMetadataTableFeature.name)) } // All [[AddFiles]] should have `baseRowId` properly propagated if RowTracking is enabled. if (expected) { assert(snapshot.allFiles.where(col("baseRowId").isNull).isEmpty) } else { assert(snapshot.allFiles.where(col("baseRowId").isNotNull).isEmpty) } } protected def validateQoLFeaturesEnablement(tableName: String, expected: Boolean): Unit = { CatalogOwnedTableUtils.QOL_TABLE_FEATURES_AND_PROPERTIES.foreach { case (t, config, value) => validateTableFeatureAndMetadata( tableName, tableFeature = t, tableFeatureShouldPresent = expected, metadataShouldPresent = expected, config, targetValue = value) } sql(s"INSERT INTO $tableName VALUES (3), (4), (5), (6)") QueryTest.checkAnswer(sql(s"SELECT * FROM $tableName"), Seq(Row(1), Row(2), Row(3), Row(4), Row(5), Row(6))) validateRowTrackingEnablement( tableName, expected) } } trait CoordinatedCommitsTestUtils extends DeltaTestUtilsBase with CommitCoordinatorUtilBase { self: SparkFunSuite with SharedSparkSession => protected val defaultCommitsCoordinatorName = "tracking-in-memory" protected val defaultCommitsCoordinatorConf = Map("randomConf" -> "randomConfValue") override def testWithDefaultCommitCoordinatorUnset(testName: String)(f: => Unit): Unit = { test(testName) { withoutDefaultCCTableFeature { f } } } override def withDefaultCCTableFeature(f: => Unit): Unit = { val confJson = JsonUtils.toJson(defaultCommitsCoordinatorConf) withSQLConf( DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.defaultTablePropertyKey -> defaultCommitsCoordinatorName, DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF.defaultTablePropertyKey -> confJson) { f } } override def withoutDefaultCCTableFeature(f: => Unit): Unit = { val defaultCoordinatedCommitsConfs = CoordinatedCommitsUtils .getDefaultCCConfigurations(spark, withDefaultKey = true) defaultCoordinatedCommitsConfs.foreach { case (defaultKey, _) => spark.conf.unset(defaultKey) } try { f } finally { defaultCoordinatedCommitsConfs.foreach { case (defaultKey, oldValue) => spark.conf.set(defaultKey, oldValue) } } } def withCustomCoordinatedCommitsTableProperties( commitCoordinatorName: String, conf: Map[String, String] = Map("randomConf" -> "randomConfValue"))(f: => Unit): Unit = { withSQLConf( DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.defaultTablePropertyKey -> commitCoordinatorName, DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF.defaultTablePropertyKey -> JsonUtils.toJson(conf)) { f } } override def testWithDifferentBackfillInterval(testName: String)(f: Int => Unit): Unit = { Seq(1, 2, 10).foreach { backfillBatchSize => test(s"$testName [Backfill batch size: $backfillBatchSize]") { CommitCoordinatorProvider.clearNonDefaultBuilders() CommitCoordinatorProvider.registerBuilder( TrackingInMemoryCommitCoordinatorBuilder(backfillBatchSize)) CommitCoordinatorProvider.registerBuilder( InMemoryCommitCoordinatorBuilder(backfillBatchSize)) f(backfillBatchSize) } } } override def registerBuilder(builder: CommitCoordinatorBuilder): Unit = { CommitCoordinatorProvider.registerBuilder(builder) } override def clearBuilders(): Unit = { CommitCoordinatorProvider.clearNonDefaultBuilders() } override def propertiesString: String = { val coordinatedCommitsConfJson = JsonUtils.toJson(defaultCommitsCoordinatorConf) s"('${DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.key}' =" + s"'$defaultCommitsCoordinatorName', " + s"'${DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF.key}' = '$coordinatedCommitsConfJson')" } override def isCatalogOwnedTest: Boolean = false /** Run the test with: * 1. Without coordinated-commits * 2. With coordinated-commits with different backfill batch sizes */ def testWithDifferentBackfillIntervalOptional(testName: String)(f: Option[Int] => Unit): Unit = { test(s"$testName [Backfill batch size: None]") { f(None) } testWithDifferentBackfillInterval(testName) { backfillBatchSize => val coordinatedCommitsCoordinatorJson = JsonUtils.toJson(defaultCommitsCoordinatorConf) withSQLConf( DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.defaultTablePropertyKey -> defaultCommitsCoordinatorName, DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF.defaultTablePropertyKey -> coordinatedCommitsCoordinatorJson) { f(Some(backfillBatchSize)) } } } def getUpdatedActionsForZerothCommit( commitInfo: CommitInfo, oldMetadata: Metadata = Metadata()): UpdatedActions = { val newMetadataConfiguration = oldMetadata.configuration + (DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.key -> defaultCommitsCoordinatorName) val newMetadata = oldMetadata.copy(configuration = newMetadataConfiguration) new UpdatedActions(commitInfo, newMetadata, Protocol(), oldMetadata, Protocol()) } def getUpdatedActionsForNonZerothCommit(commitInfo: CommitInfo): UpdatedActions = { val updatedActions = getUpdatedActionsForZerothCommit(commitInfo) new UpdatedActions( updatedActions.getCommitInfo, updatedActions.getNewMetadata, updatedActions.getNewProtocol, updatedActions.getNewMetadata, updatedActions.getOldProtocol ) } } case class TrackingInMemoryCommitCoordinatorBuilder( batchSize: Long, defaultCommitCoordinatorClientOpt: Option[CommitCoordinatorClient] = None, defaultCommitCoordinatorName: String = "tracking-in-memory") extends CatalogOwnedCommitCoordinatorBuilder { lazy val trackingInMemoryCommitCoordinatorClient = defaultCommitCoordinatorClientOpt.getOrElse { new TrackingCommitCoordinatorClient( new PredictableUuidInMemoryCommitCoordinatorClient(batchSize)) } override def getName: String = defaultCommitCoordinatorName override def build(spark: SparkSession, conf: Map[String, String]): CommitCoordinatorClient = { trackingInMemoryCommitCoordinatorClient } override def buildForCatalog( spark: SparkSession, catalogName: String): CommitCoordinatorClient = { trackingInMemoryCommitCoordinatorClient } } class PredictableUuidInMemoryCommitCoordinatorClient(batchSize: Long) extends InMemoryCommitCoordinator(batchSize) { var nextUuidSuffix = 1L override def generateUUID(): String = { nextUuidSuffix += 1 s"uuid-${nextUuidSuffix - 1}" } } object TrackingCommitCoordinatorClient { private val insideOperation = new ThreadLocal[Boolean] { override def initialValue(): Boolean = false } } class TrackingCommitCoordinatorClient( val delegatingCommitCoordinatorClient: CommitCoordinatorClient) extends CommitCoordinatorClient { val numCommitsCalled = new AtomicInteger(0) val numGetCommitsCalled = new AtomicInteger(0) val numBackfillToVersionCalled = new AtomicInteger(0) val numRegisterTableCalled = new AtomicInteger(0) def recordOperation[T](op: String)(f: => T): T = { val oldInsideOperation = TrackingCommitCoordinatorClient.insideOperation.get() try { if (!TrackingCommitCoordinatorClient.insideOperation.get()) { op match { case "commit" => numCommitsCalled.incrementAndGet() case "getCommits" => numGetCommitsCalled.incrementAndGet() case "backfillToVersion" => numBackfillToVersionCalled.incrementAndGet() case "registerTable" => numRegisterTableCalled.incrementAndGet() case _ => () } } TrackingCommitCoordinatorClient.insideOperation.set(true) f } finally { TrackingCommitCoordinatorClient.insideOperation.set(oldInsideOperation) } } override def commit( logStore: LogStore, hadoopConf: Configuration, tableDesc: TableDescriptor, commitVersion: Long, actions: java.util.Iterator[String], updatedActions: UpdatedActions): CommitResponse = recordOperation("commit") { delegatingCommitCoordinatorClient.commit( logStore, hadoopConf, tableDesc, commitVersion, actions, updatedActions) } override def getCommits( tableDesc: TableDescriptor, startVersion: java.lang.Long, endVersion: java.lang.Long): JGetCommitsResponse = recordOperation("getCommits") { delegatingCommitCoordinatorClient.getCommits(tableDesc, startVersion, endVersion) } override def backfillToVersion( logStore: LogStore, hadoopConf: Configuration, tableDesc: TableDescriptor, version: Long, lastKnownBackfilledVersion: java.lang.Long): Unit = recordOperation("backfillToVersion") { delegatingCommitCoordinatorClient.backfillToVersion( logStore, hadoopConf, tableDesc, version, lastKnownBackfilledVersion) } override def semanticEquals(other: CommitCoordinatorClient): Boolean = { other match { case otherTracking: TrackingCommitCoordinatorClient => delegatingCommitCoordinatorClient.semanticEquals( otherTracking.delegatingCommitCoordinatorClient) case _ => delegatingCommitCoordinatorClient.semanticEquals(other) } } def reset(): Unit = { numCommitsCalled.set(0) numGetCommitsCalled.set(0) numBackfillToVersionCalled.set(0) } override def registerTable( logPath: Path, tableIdentifier: Optional[TableIdentifier], currentVersion: Long, currentMetadata: AbstractMetadata, currentProtocol: AbstractProtocol): java.util.Map[String, String] = recordOperation("registerTable") { delegatingCommitCoordinatorClient.registerTable( logPath, tableIdentifier, currentVersion, currentMetadata, currentProtocol) } } /** * A helper class which enables coordinated-commits for the test suite based on the given * `coordinatedCommitsBackfillBatchSize` conf. */ trait CoordinatedCommitsBaseSuite extends SparkFunSuite with SharedSparkSession with CoordinatedCommitsTestUtils { // If this config is not overridden, coordinated commits are disabled. def coordinatedCommitsBackfillBatchSize: Option[Int] = None final def coordinatedCommitsEnabledInTests: Boolean = coordinatedCommitsBackfillBatchSize.nonEmpty // In case some tests reuse the table path/name with DROP table, this method can be used to // clean the table data in the commit coordinator. Note that we should call this before // the table actually gets DROP. def deleteTableFromCommitCoordinator(tableName: String): Unit = { val location = try { spark.sql(s"describe detail $tableName") .select("location") .first() .getAs[String](0) } catch { case NonFatal(_) => // Ignore if the table does not exist/broken. return } deleteTableFromCommitCoordinator(new Path(location)) } def deleteTableFromCommitCoordinator(path: Path): Unit = { val cc = CommitCoordinatorProvider.getCommitCoordinatorClient( defaultCommitsCoordinatorName, defaultCommitsCoordinatorConf, spark) assert( cc.isInstanceOf[TrackingCommitCoordinatorClient], s"Please implement delete/drop method for coordinator: ${cc.getClass.getName}") val locKey = path.toString.stripPrefix("file:") if (locRefCount.contains(locKey)) { locRefCount(locKey) -= 1 } // When we create an external table in a location where some table already existed, two table // names could be pointing to the same location. We should only clean up the table data in the // commit coordinator when the last table name pointing to the location is dropped. if (locRefCount.getOrElse(locKey, 0) == 0) { val logPath = new Path(path, "_delta_log") cc.asInstanceOf[TrackingCommitCoordinatorClient] .delegatingCommitCoordinatorClient .asInstanceOf[InMemoryCommitCoordinator] .dropTable(logPath) } DeltaLog.clearCache() } override protected def sparkConf: SparkConf = { if (coordinatedCommitsBackfillBatchSize.nonEmpty) { val coordinatedCommitsCoordinatorJson = JsonUtils.toJson(defaultCommitsCoordinatorConf) super.sparkConf .set( DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.defaultTablePropertyKey, defaultCommitsCoordinatorName) .set( DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF.defaultTablePropertyKey, coordinatedCommitsCoordinatorJson) } else { super.sparkConf } } override def beforeEach(): Unit = { super.beforeEach() CommitCoordinatorProvider.clearNonDefaultBuilders() coordinatedCommitsBackfillBatchSize.foreach { batchSize => CommitCoordinatorProvider.registerBuilder(TrackingInMemoryCommitCoordinatorBuilder(batchSize)) } DeltaLog.clearCache() } protected def isICTEnabledForNewTables: Boolean = { spark.conf.getOption( DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.defaultTablePropertyKey).nonEmpty || spark.conf.getOption( DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.defaultTablePropertyKey).contains("true") } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtilsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import scala.jdk.CollectionConverters._ import org.apache.spark.sql.delta.DeltaConfigs.{COORDINATED_COMMITS_COORDINATOR_CONF, COORDINATED_COMMITS_COORDINATOR_NAME, COORDINATED_COMMITS_TABLE_CONF} import org.apache.spark.sql.delta.DeltaIllegalArgumentException import org.apache.spark.sql.delta.test.shims.GridTestShim import org.apache.spark.sql.QueryTest import org.apache.spark.sql.test.SharedSparkSession class CoordinatedCommitsUtilsSuite extends QueryTest with GridTestShim with SharedSparkSession with CoordinatedCommitsTestUtils { ///////////////////////////////////////////////////////////////////////////////////////////// // Test CoordinatedCommitsUtils.validateCoordinatedCommitsConfigurationsImpl STARTS // ///////////////////////////////////////////////////////////////////////////////////////////// private val cNameKey = COORDINATED_COMMITS_COORDINATOR_NAME.key private val cConfKey = COORDINATED_COMMITS_COORDINATOR_CONF.key private val tableConfKey = COORDINATED_COMMITS_TABLE_CONF.key private val cName = cNameKey -> "some-cc-name" private val cConf = cConfKey -> "some-cc-conf" private val tableConf = tableConfKey -> "some-table-conf" private val cNameDefaultKey = COORDINATED_COMMITS_COORDINATOR_NAME.defaultTablePropertyKey private val cConfDefaultKey = COORDINATED_COMMITS_COORDINATOR_CONF.defaultTablePropertyKey private val tableConfDefaultKey = COORDINATED_COMMITS_TABLE_CONF.defaultTablePropertyKey private val cNameDefault = cNameDefaultKey -> "some-cc-name" private val cConfDefault = cConfDefaultKey -> "some-cc-conf" private val tableConfDefault = tableConfDefaultKey -> "some-table-conf" private val command = "CLONE" private def errCannotOverride = new DeltaIllegalArgumentException( "DELTA_CANNOT_OVERRIDE_COORDINATED_COMMITS_CONFS", Array(command)) private def errMissingConfInCommand(key: String) = new DeltaIllegalArgumentException( "DELTA_MUST_SET_ALL_COORDINATED_COMMITS_CONFS_IN_COMMAND", Array(command, key)) private def errMissingConfInSession(key: String) = new DeltaIllegalArgumentException( "DELTA_MUST_SET_ALL_COORDINATED_COMMITS_CONFS_IN_SESSION", Array(command, key)) private def errTableConfInCommand = new DeltaIllegalArgumentException( "DELTA_CONF_OVERRIDE_NOT_SUPPORTED_IN_COMMAND", Array(command, tableConfKey)) private def errTableConfInSession = new DeltaIllegalArgumentException( "DELTA_CONF_OVERRIDE_NOT_SUPPORTED_IN_SESSION", Array(command, tableConfDefaultKey, tableConfDefaultKey)) private def testValidationForCreateDeltaTableCommand( tableExists: Boolean, propertyOverrides: Map[String, String], defaultConfs: Seq[(String, String)], errorOpt: Option[DeltaIllegalArgumentException]): Unit = { withoutDefaultCCTableFeature { withSQLConf(defaultConfs: _*) { if (errorOpt.isDefined) { val e = intercept[DeltaIllegalArgumentException] { CoordinatedCommitsUtils.validateConfigurationsForCreateDeltaTableCommandImpl( spark, propertyOverrides, tableExists, command) } checkError( e, errorOpt.get.getErrorClass, sqlState = errorOpt.get.getSqlState, parameters = errorOpt.get.getMessageParameters.asScala.toMap) } else { CoordinatedCommitsUtils.validateConfigurationsForCreateDeltaTableCommandImpl( spark, propertyOverrides, tableExists, command) } } } } // tableExists: True // | False // // propertyOverrides: Map.empty // | Map(cName) // | Map(cName, cConf) // | Map(cName, cConf, tableConf) // | Map(tableConf) // // defaultConf: Seq.empty // | Seq(cNameDefault) // | Seq(cNameDefault, cConfDefault) // | Seq(cNameDefault, cConfDefault, tableConfDefault) // | Seq(tableConfDefault) // // errorOpt: None // | Some(errCannotOverride) // | Some(errMissingConfInCommand(cConfKey)) // | Some(errMissingConfInSession(cConfKey)) // | Some(errTableConfInCommand) // | Some(errTableConfInSession) gridTest("During CLONE, CoordinatedCommitsUtils.validateCoordinatedCommitsConfigurationsImpl " + "passes for existing target tables with no explicit Coordinated Commits Configurations.") ( Seq( Seq.empty, // Not having any explicit Coordinated Commits configurations, but having an illegal // combination of Coordinated Commits configurations in default: pass. // This is because we don't consider default configurations when the table exists. Seq(cNameDefault), Seq(cNameDefault, cConfDefault), Seq(cNameDefault, cConfDefault, tableConfDefault), Seq(tableConfDefault) ) ) { defaultConfs: Seq[(String, String)] => testValidationForCreateDeltaTableCommand( tableExists = true, propertyOverrides = Map.empty, defaultConfs, errorOpt = None) } gridTest("During CLONE, CoordinatedCommitsUtils.validateCoordinatedCommitsConfigurationsImpl " + "fails for existing target tables with any explicit Coordinated Commits Configurations.") ( Seq( (Map(cName), Seq.empty), (Map(cName), Seq(cNameDefault)), (Map(cName), Seq(cNameDefault, cConfDefault)), (Map(cName), Seq(cNameDefault, cConfDefault, tableConfDefault)), (Map(cName), Seq(tableConfDefault)), (Map(cName, cConf), Seq.empty), (Map(cName, cConf), Seq(cNameDefault)), (Map(cName, cConf), Seq(cNameDefault, cConfDefault)), (Map(cName, cConf), Seq(cNameDefault, cConfDefault, tableConfDefault)), (Map(cName, cConf), Seq(tableConfDefault)), (Map(cName, cConf, tableConf), Seq.empty), (Map(cName, cConf, tableConf), Seq(cNameDefault)), (Map(cName, cConf, tableConf), Seq(cNameDefault, cConfDefault)), (Map(cName, cConf, tableConf), Seq(cNameDefault, cConfDefault, tableConfDefault)), (Map(cName, cConf, tableConf), Seq(tableConfDefault)), (Map(tableConf), Seq.empty), (Map(tableConf), Seq(cNameDefault)), (Map(tableConf), Seq(cNameDefault, cConfDefault)), (Map(tableConf), Seq(cNameDefault, cConfDefault, tableConfDefault)), (Map(tableConf), Seq(tableConfDefault)) ) ) { case ( propertyOverrides: Map[String, String], defaultConfs: Seq[(String, String)]) => testValidationForCreateDeltaTableCommand( tableExists = true, propertyOverrides, defaultConfs, errorOpt = Some(errCannotOverride)) } gridTest("During CLONE, CoordinatedCommitsUtils.validateCoordinatedCommitsConfigurationsImpl " + "works correctly for new target tables with default Coordinated Commits Configurations.") ( Seq( (Seq.empty, None), (Seq(cNameDefault), Some(errMissingConfInSession(cConfDefaultKey))), (Seq(cNameDefault, cConfDefault), None), (Seq(cNameDefault, cConfDefault, tableConfDefault), Some(errTableConfInSession)), (Seq(tableConfDefault), Some(errTableConfInSession)) ) ) { case ( defaultConfs: Seq[(String, String)], errorOpt: Option[DeltaIllegalArgumentException]) => testValidationForCreateDeltaTableCommand( tableExists = false, propertyOverrides = Map.empty, defaultConfs, errorOpt) } gridTest("During CLONE, CoordinatedCommitsUtils.validateCoordinatedCommitsConfigurationsImpl " + "fails for new target tables with any illegal explicit Coordinated Commits Configurations.") ( Seq( (Map(cName), Seq.empty, Some(errMissingConfInCommand(cConfKey))), (Map(cName), Seq(cNameDefault), Some(errMissingConfInCommand(cConfKey))), (Map(cName), Seq(cNameDefault, cConfDefault), Some(errMissingConfInCommand(cConfKey))), (Map(cName), Seq(cNameDefault, cConfDefault, tableConfDefault), Some(errMissingConfInCommand(cConfKey))), (Map(cName), Seq(tableConfDefault), Some(errMissingConfInCommand(cConfKey))), (Map(cName, cConf, tableConf), Seq.empty, Some(errTableConfInCommand)), (Map(cName, cConf, tableConf), Seq(cNameDefault), Some(errTableConfInCommand)), (Map(cName, cConf, tableConf), Seq(cNameDefault, cConfDefault), Some(errTableConfInCommand)), (Map(cName, cConf, tableConf), Seq(cNameDefault, cConfDefault, tableConfDefault), Some(errTableConfInCommand)), (Map(cName, cConf, tableConf), Seq(tableConfDefault), Some(errTableConfInCommand)), (Map(tableConf), Seq.empty, Some(errTableConfInCommand)), (Map(tableConf), Seq(cNameDefault), Some(errTableConfInCommand)), (Map(tableConf), Seq(cNameDefault, cConfDefault), Some(errTableConfInCommand)), (Map(tableConf), Seq(cNameDefault, cConfDefault, tableConfDefault), Some(errTableConfInCommand)), (Map(tableConf), Seq(tableConfDefault), Some(errTableConfInCommand)) ) ) { case ( propertyOverrides: Map[String, String], defaultConfs: Seq[(String, String)], errorOpt: Option[DeltaIllegalArgumentException]) => testValidationForCreateDeltaTableCommand( tableExists = false, propertyOverrides, defaultConfs, errorOpt) } gridTest("During CLONE, CoordinatedCommitsUtils.validateCoordinatedCommitsConfigurationsImpl " + "passes for new target tables with legal explicit Coordinated Commits Configurations.") ( Seq( // Having exactly Coordinator Name and Coordinator Conf explicitly, but having an illegal // combination of Coordinated Commits configurations in default: pass. // This is because we don't consider default configurations when explicit ones are provided. Seq.empty, Seq(cNameDefault), Seq(cNameDefault, cConfDefault), Seq(cNameDefault, cConfDefault, tableConfDefault), Seq(tableConfDefault) ) ) { defaultConfs: Seq[(String, String)] => testValidationForCreateDeltaTableCommand( tableExists = false, propertyOverrides = Map(cName, cConf), defaultConfs, errorOpt = None) } private def testValidateConfigurationsForAlterTableSetPropertiesDeltaCommand( existingConfs: Map[String, String], propertyOverrides: Map[String, String], errorOpt: Option[DeltaIllegalArgumentException]): Unit = { if (errorOpt.isDefined) { val e = intercept[DeltaIllegalArgumentException] { CoordinatedCommitsUtils.validateConfigurationsForAlterTableSetPropertiesDeltaCommand( existingConfs, propertyOverrides) } checkError( e, errorOpt.get.getErrorClass, sqlState = errorOpt.get.getSqlState, parameters = errorOpt.get.getMessageParameters.asScala.toMap) } else { CoordinatedCommitsUtils.validateConfigurationsForAlterTableSetPropertiesDeltaCommand( existingConfs, propertyOverrides) } } gridTest("During ALTER, `validateConfigurationsForAlterTableSetPropertiesDeltaCommand` " + "works correctly for tables without Coordinated Commits configurations.") { Seq( (Map.empty, None), (Map(cName), Some(new DeltaIllegalArgumentException( "DELTA_MUST_SET_ALL_COORDINATED_COMMITS_CONFS_IN_COMMAND", Array("ALTER", cConfKey)))), (Map(cName, cConf), None), (Map(cName, cConf, tableConf), Some(new DeltaIllegalArgumentException( "DELTA_CONF_OVERRIDE_NOT_SUPPORTED_IN_COMMAND", Array("ALTER", tableConfKey)))), (Map(tableConf), Some(new DeltaIllegalArgumentException( "DELTA_CONF_OVERRIDE_NOT_SUPPORTED_IN_COMMAND", Array("ALTER", tableConfKey)))) ) } { case ( propertyOverrides: Map[String, String], errorOpt: Option[DeltaIllegalArgumentException]) => testValidateConfigurationsForAlterTableSetPropertiesDeltaCommand( existingConfs = Map.empty, propertyOverrides, errorOpt) } test("During ALTER, `validateConfigurationsForAlterTableSetPropertiesDeltaCommand` " + "passes with no overrides for tables with Coordinated Commits configurations.") { testValidateConfigurationsForAlterTableSetPropertiesDeltaCommand( existingConfs = Map(cName, cConf, tableConf), propertyOverrides = Map.empty, errorOpt = None) } gridTest("During ALTER, `validateConfigurationsForAlterTableSetPropertiesDeltaCommand` " + "fails with overrides for tables with Coordinated Commits configurations.") ( Seq( Map(cName), Map(cName, cConf), Map(cName, cConf, tableConf), Map(tableConf) ) ) { propertyOverrides: Map[String, String] => testValidateConfigurationsForAlterTableSetPropertiesDeltaCommand( existingConfs = Map(cName, cConf, tableConf), propertyOverrides, errorOpt = Some(new DeltaIllegalArgumentException( "DELTA_CANNOT_OVERRIDE_COORDINATED_COMMITS_CONFS", Array("ALTER")))) } private def errCannotUnset = new DeltaIllegalArgumentException( "DELTA_CANNOT_UNSET_COORDINATED_COMMITS_CONFS", Array.empty) private def testValidateConfigurationsForAlterTableUnsetPropertiesDeltaCommand( existingConfs: Map[String, String], propKeysToUnset: Seq[String], errorOpt: Option[DeltaIllegalArgumentException]): Unit = { if (errorOpt.isDefined) { val e = intercept[DeltaIllegalArgumentException] { CoordinatedCommitsUtils.validateConfigurationsForAlterTableUnsetPropertiesDeltaCommand( existingConfs, propKeysToUnset) } checkError( e, errorOpt.get.getErrorClass, sqlState = errorOpt.get.getSqlState, parameters = errorOpt.get.getMessageParameters.asScala.toMap) } else { CoordinatedCommitsUtils.validateConfigurationsForAlterTableUnsetPropertiesDeltaCommand( existingConfs, propKeysToUnset) } } gridTest("During ALTER, `validateConfigurationsForAlterTableUnsetPropertiesDeltaCommand` " + "fails with overrides for tables with Coordinated Commits configurations.") { Seq( Seq(cNameKey), Seq(cNameKey, cConfKey), Seq(cNameKey, cConfKey, tableConfKey), Seq(tableConfKey) ) } { propKeysToUnset: Seq[String] => testValidateConfigurationsForAlterTableUnsetPropertiesDeltaCommand( existingConfs = Map(cName, cConf, tableConf), propKeysToUnset, errorOpt = Some(errCannotUnset)) } gridTest("During ALTER, `validateConfigurationsForAlterTableUnsetPropertiesDeltaCommand` " + "passes with no overrides for tables with or without Coordinated Commits configurations.") { Seq( Map.empty, Map(cName, cConf, tableConf) ) } { case existingConfs: Map[String, String] => testValidateConfigurationsForAlterTableUnsetPropertiesDeltaCommand( existingConfs, propKeysToUnset = Seq.empty, errorOpt = None) } gridTest("During ALTER, `validateConfigurationsForAlterTableUnsetPropertiesDeltaCommand` " + "passes with overrides for tables without Coordinated Commits configurations.") { Seq( Seq(cNameKey), Seq(cNameKey, cConfKey), Seq(cNameKey, cConfKey, tableConfKey), Seq(tableConfKey) ) } { propKeysToUnset: Seq[String] => testValidateConfigurationsForAlterTableUnsetPropertiesDeltaCommand( existingConfs = Map.empty, propKeysToUnset, errorOpt = None) } ///////////////////////////////////////////////////////////////////////////////////////////// // Test CoordinatedCommitsUtils.validateCoordinatedCommitsConfigurationsImpl ENDS // ///////////////////////////////////////////////////////////////////////////////////////////// } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/DynamoDBCommitCoordinatorClientSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import java.util.Optional import java.util.concurrent.locks.ReentrantReadWriteLock import scala.collection.JavaConverters._ import scala.collection.mutable import com.amazonaws.services.dynamodbv2.{AbstractAmazonDynamoDB, AmazonDynamoDB, AmazonDynamoDBClient} import com.amazonaws.services.dynamodbv2.model.{AttributeValue, ConditionalCheckFailedException, CreateTableRequest, CreateTableResult, DescribeTableResult, GetItemRequest, GetItemResult, PutItemRequest, PutItemResult, ResourceInUseException, ResourceNotFoundException, TableDescription, UpdateItemRequest, UpdateItemResult} import org.apache.spark.sql.delta.{DeltaConfigs, DeltaLog} import org.apache.spark.sql.delta.actions.{Metadata, Protocol} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import io.delta.dynamodbcommitcoordinator.{DynamoDBCommitCoordinatorClient, DynamoDBCommitCoordinatorClientBuilder} import io.delta.storage.commit.{CommitCoordinatorClient, CommitFailedException => JCommitFailedException, GetCommitsResponse => JGetCommitsResponse} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.sql.SparkSession /** * An in-memory implementation of DynamoDB client for testing. Only the methods used by * `DynamoDBCommitCoordinatorClient` are implemented. */ class InMemoryDynamoDBClient extends AbstractAmazonDynamoDB { /** * The db has multiple tables (outer map). Each table has multiple entries (inner map). */ val db: mutable.Map[String, mutable.Map[String, PerEntryData]] = mutable.Map.empty case class PerEntryData( lock: ReentrantReadWriteLock, data: mutable.Map[String, AttributeValue]) private def getTableData(tableName: String): mutable.Map[String, PerEntryData] = { db.getOrElse(tableName, throw new ResourceNotFoundException("table does not exist")) } override def createTable(createTableRequest: CreateTableRequest): CreateTableResult = { val tableName = createTableRequest.getTableName if (db.contains(tableName)) { throw new ResourceInUseException("Table already exists") } db.getOrElseUpdate(tableName, mutable.Map.empty) new CreateTableResult().withTableDescription( new TableDescription().withTableName(tableName)); } override def describeTable(tableName: String): DescribeTableResult = { if (!db.contains(tableName)) { throw new ResourceNotFoundException("table does not exist") } val tableDesc = new TableDescription().withTableName(tableName).withTableStatus("ACTIVE") new DescribeTableResult().withTable(tableDesc) } override def getItem(getItemRequest: GetItemRequest): GetItemResult = { val table = getTableData(getItemRequest.getTableName) val tableId = getItemRequest.getKey.values().iterator().next(); val entry = table.getOrElse(tableId.getS, throw new ResourceNotFoundException("table does not exist")) val lock = entry.lock.readLock() try { lock.lock() val result = new GetItemResult() getItemRequest.getAttributesToGet.forEach(attr => { entry.data.get(attr).foreach(result.addItemEntry(attr, _)) }) result } finally { lock.unlock() } } override def putItem(putItemRequest: PutItemRequest): PutItemResult = { val table = getTableData(putItemRequest.getTableName) val item = putItemRequest.getItem val tableId = item.get("tableId").getS if (table.contains(tableId)) { throw new ResourceInUseException("table already exists") } val entry = PerEntryData(new ReentrantReadWriteLock(), item.asScala) // This is not really safe, but tableId is a UUID, so it should be fine. table.put(tableId, entry) new PutItemResult() } override def updateItem(request: UpdateItemRequest): UpdateItemResult = { val table = getTableData(request.getTableName) val tableId = request.getKey.values().iterator().next(); val entry = table.getOrElse(tableId.getS, throw new ResourceNotFoundException("table does not exist")) val lock = entry.lock.writeLock() try { lock.lock() request.getExpected.forEach((attr, expectedVal) => { val actualVal = entry.data.getOrElse(attr, throw new ConditionalCheckFailedException("Expected attr not found")) if (actualVal != expectedVal.getValue) { throw new ConditionalCheckFailedException("Value does not match") } }) request.getAttributeUpdates.forEach((attr, update) => { if (attr != "commits") { entry.data.put(attr, update.getValue) } else { val commits = update.getValue.getL.asScala if (update.getAction == "ADD") { val existingCommits = entry.data.get("commits").map(_.getL.asScala).getOrElse(List()) entry.data.put( "commits", new AttributeValue().withL((existingCommits ++ commits).asJava)) } else if (update.getAction == "PUT") { entry.data.put("commits", update.getValue) } else { throw new IllegalArgumentException("Unsupported action") } } }) new UpdateItemResult() } finally { lock.unlock() } } } case class TestDynamoDBCommitCoordinatorBuilder(batchSize: Long) extends CommitCoordinatorBuilder { override def getName: String = "test-dynamodb" override def build( spark: SparkSession, config: Map[String, String]): CommitCoordinatorClient = { new DynamoDBCommitCoordinatorClient( "testTable", "test-endpoint", new InMemoryDynamoDBClient(), batchSize) } } abstract class DynamoDBCommitCoordinatorClientSuite(batchSize: Long) extends CommitCoordinatorClientImplSuiteBase { override protected def createTableCommitCoordinatorClient( deltaLog: DeltaLog): TableCommitCoordinatorClient = { val cs = TestDynamoDBCommitCoordinatorBuilder(batchSize = batchSize).build(spark, Map.empty) val tableConf = cs.registerTable( deltaLog.logPath, Optional.empty(), -1L, Metadata(), Protocol(1, 1)) TableCommitCoordinatorClient(cs, deltaLog, tableConf.asScala.toMap) } override protected def registerBackfillOp( tableCommitCoordinatorClient: TableCommitCoordinatorClient, deltaLog: DeltaLog, version: Long): Unit = { tableCommitCoordinatorClient.backfillToVersion(version) } override protected def validateBackfillStrategy( tableCommitCoordinatorClient: TableCommitCoordinatorClient, logPath: Path, version: Long): Unit = { val lastExpectedBackfilledVersion = (version - (version % batchSize)).toInt val unbackfilledCommitVersionsAll = tableCommitCoordinatorClient .getCommits().getCommits.asScala.map(_.getVersion) val expectedVersions = lastExpectedBackfilledVersion + 1 to version.toInt assert(unbackfilledCommitVersionsAll == expectedVersions) (0 to lastExpectedBackfilledVersion).foreach { v => assertBackfilled(v, logPath, Some(v)) } } protected def validateGetCommitsResult( result: JGetCommitsResponse, startVersion: Option[Long], endVersion: Option[Long], maxVersion: Long): Unit = { val commitVersions = result.getCommits.asScala.map(_.getVersion) val lastExpectedBackfilledVersion = (maxVersion - (maxVersion % batchSize)).toInt val expectedVersions = lastExpectedBackfilledVersion + 1 to maxVersion.toInt assert(commitVersions == expectedVersions) assert(result.getLatestTableVersion == maxVersion) } for (skipPathCheck <- Seq(true, false)) test(s"skipPathCheck should work correctly [skipPathCheck = $skipPathCheck]") { withTempTableDir { tempDir => val log = DeltaLog.forTable(spark, tempDir.toString) val logPath = log.logPath writeCommitZero(logPath) val dynamoDB = new InMemoryDynamoDBClient(); val commitCoordinator = new DynamoDBCommitCoordinatorClient( "testTable", "test-endpoint", dynamoDB, batchSize, 1, // readCapacityUnits 1, // writeCapacityUnits skipPathCheck) val tableConf = commitCoordinator.registerTable( logPath, Optional.empty(), -1L, Metadata(), Protocol(1, 1)) val wrongTablePath = new Path(logPath.getParent, "wrongTable") val wrongLogPath = new Path(wrongTablePath, logPath.getName) val fs = wrongLogPath.getFileSystem(log.newDeltaHadoopConf()) fs.mkdirs(wrongTablePath) fs.mkdirs(FileNames.commitDirPath(wrongLogPath)) val wrongTablePathTableCommitCoordinator = new TableCommitCoordinatorClient( commitCoordinator, wrongLogPath, tableConf.asScala.toMap, log.newDeltaHadoopConf(), log.store ) if (skipPathCheck) { // This should succeed because we are skipping the path check. val resp = commit(1L, 1L, wrongTablePathTableCommitCoordinator) assert(resp.getVersion == 1L) } else { val e = intercept[JCommitFailedException] { commit(1L, 1L, wrongTablePathTableCommitCoordinator) } assert(e.getMessage.contains("while the table is registered at")) } } } test("builder should read dynamic configs from sparkSession") { class TestDynamoDBCommitCoordinatorBuilder extends DynamoDBCommitCoordinatorClientBuilder { override def getName: String = "dynamodb-test" override def createAmazonDDBClient( endpoint: String, credentialProviderName: String, hadoopConf: Configuration): AmazonDynamoDB = { assert(endpoint == "endpoint-1224") assert(credentialProviderName == "creds-1225") new InMemoryDynamoDBClient() } override def getDynamoDBCommitCoordinatorClient( coordinatedCommitsTableName: String, dynamoDBEndpoint: String, ddbClient: AmazonDynamoDB, backfillBatchSize: Long, readCapacityUnits: Int, writeCapacityUnits: Int, skipPathCheck: Boolean): DynamoDBCommitCoordinatorClient = { assert(coordinatedCommitsTableName == "tableName-1223") assert(dynamoDBEndpoint == "endpoint-1224") assert(backfillBatchSize == 1) assert(readCapacityUnits == 1226) assert(writeCapacityUnits == 1227) assert(skipPathCheck) new DynamoDBCommitCoordinatorClient( coordinatedCommitsTableName, dynamoDBEndpoint, ddbClient, backfillBatchSize, readCapacityUnits, writeCapacityUnits, skipPathCheck) } } val commitCoordinatorConf = JsonUtils.toJson(Map( "dynamoDBTableName" -> "tableName-1223", "dynamoDBEndpoint" -> "endpoint-1224" )) withSQLConf( DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_NAME.defaultTablePropertyKey -> "dynamodb-test", DeltaConfigs.COORDINATED_COMMITS_COORDINATOR_CONF.defaultTablePropertyKey -> commitCoordinatorConf, DeltaSQLConf.COORDINATED_COMMITS_DDB_AWS_CREDENTIALS_PROVIDER_NAME.key -> "creds-1225", DeltaSQLConf.COORDINATED_COMMITS_DDB_SKIP_PATH_CHECK.key -> "true", DeltaSQLConf.COORDINATED_COMMITS_DDB_READ_CAPACITY_UNITS.key -> "1226", DeltaSQLConf.COORDINATED_COMMITS_DDB_WRITE_CAPACITY_UNITS.key -> "1227") { // clear default builders CommitCoordinatorProvider.clearNonDefaultBuilders() CommitCoordinatorProvider.registerBuilder(new TestDynamoDBCommitCoordinatorBuilder()) withTempTableDir { tempDir => val tablePath = tempDir.getAbsolutePath spark.range(1).write.format("delta").mode("overwrite").save(tablePath) val log = DeltaLog.forTable(spark, tempDir.toString) val tableCommitCoordinatorClient = log.snapshot.tableCommitCoordinatorClientOpt.get assert(tableCommitCoordinatorClient .commitCoordinatorClient.isInstanceOf[DynamoDBCommitCoordinatorClient]) assert(tableCommitCoordinatorClient.tableConf.contains("tableId")) } } } } class DynamoDBCommitCoordinatorClient5BackfillSuite extends DynamoDBCommitCoordinatorClientSuite(5) ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/InMemoryCommitCoordinatorSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import java.util.Optional import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.actions.Protocol import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import io.delta.storage.commit.{GetCommitsResponse => JGetCommitsResponse} import org.apache.hadoop.fs.Path abstract class InMemoryCommitCoordinatorSuite(batchSize: Int) extends CommitCoordinatorClientImplSuiteBase { override protected def createTableCommitCoordinatorClient( deltaLog: DeltaLog): TableCommitCoordinatorClient = { val cs = InMemoryCommitCoordinatorBuilder(batchSize).build(spark, Map.empty) val conf = cs.registerTable( deltaLog.logPath, Optional.empty(), -1L, initMetadata, Protocol(1, 1)) TableCommitCoordinatorClient(cs, deltaLog, conf.asScala.toMap) } override protected def registerBackfillOp( tableCommitCoordinatorClient: TableCommitCoordinatorClient, deltaLog: DeltaLog, version: Long): Unit = { val commitCoordinatorClient = tableCommitCoordinatorClient.commitCoordinatorClient val inMemoryCS = commitCoordinatorClient.asInstanceOf[InMemoryCommitCoordinator] inMemoryCS.registerBackfill(deltaLog.logPath, version) } override protected def validateBackfillStrategy( tableCommitCoordinatorClient: TableCommitCoordinatorClient, logPath: Path, version: Long): Unit = { val lastExpectedBackfilledVersion = (version - (version % batchSize)).toInt val unbackfilledCommitVersionsAll = tableCommitCoordinatorClient .getCommits().getCommits.asScala.map(_.getVersion) val expectedVersions = lastExpectedBackfilledVersion + 1 to version.toInt assert(unbackfilledCommitVersionsAll == expectedVersions) (0 to lastExpectedBackfilledVersion).foreach { v => assertBackfilled(v, logPath, Some(v)) } } protected def validateGetCommitsResult( result: JGetCommitsResponse, startVersion: Option[Long], endVersion: Option[Long], maxVersion: Long): Unit = { val commitVersions = result.getCommits.asScala.map(_.getVersion) val lastExpectedBackfilledVersion = (maxVersion - (maxVersion % batchSize)).toInt val expectedVersions = lastExpectedBackfilledVersion + 1 to maxVersion.toInt assert(commitVersions == expectedVersions) assert(result.getLatestTableVersion == maxVersion) } test("InMemoryCommitCoordinatorBuilder works as expected") { val builder1 = InMemoryCommitCoordinatorBuilder(5) val cs1 = builder1.build(spark, Map.empty) assert(cs1.isInstanceOf[InMemoryCommitCoordinator]) assert(cs1.asInstanceOf[InMemoryCommitCoordinator].batchSize == 5) val cs1_again = builder1.build(spark, Map.empty) assert(cs1_again.isInstanceOf[InMemoryCommitCoordinator]) assert(cs1 == cs1_again) val builder2 = InMemoryCommitCoordinatorBuilder(10) val cs2 = builder2.build(spark, Map.empty) assert(cs2.isInstanceOf[InMemoryCommitCoordinator]) assert(cs2.asInstanceOf[InMemoryCommitCoordinator].batchSize == 10) assert(cs2 ne cs1) val builder3 = InMemoryCommitCoordinatorBuilder(10) val cs3 = builder3.build(spark, Map.empty) assert(cs3.isInstanceOf[InMemoryCommitCoordinator]) assert(cs3.asInstanceOf[InMemoryCommitCoordinator].batchSize == 10) assert(cs3 ne cs2) } test("test commit > 1 is rejected as first commit") { withTempTableDir { tempDir => val log = DeltaLog.forTable(spark, tempDir.toString) val logPath = log.logPath val tcs = createTableCommitCoordinatorClient(log) // Anything other than version-0 or version-1 should be rejected as the first commit // version-0 will be directly backfilled and won't be recorded in InMemoryCommitCoordinator. // version-1 is what commit-coordinator is accepting. assertCommitFail(2, 1, retryable = false, commit(2, 0, tcs)) } } } class InMemoryCommitCoordinator1Suite extends InMemoryCommitCoordinatorSuite(1) class InMemoryCommitCoordinator5Suite extends InMemoryCommitCoordinatorSuite(5) ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/UCCommitCoordinatorBuilderSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import io.delta.storage.commit.uccommitcoordinator.{UCClient, UCCommitCoordinatorClient} import org.mockito.{Mock, Mockito} import org.mockito.ArgumentMatchers.{any, eq => meq} import org.mockito.Mockito.{mock, never, times, verify, when} import org.apache.spark.SparkFunSuite import org.apache.spark.sql.test.SharedSparkSession class UCCommitCoordinatorBuilderSuite extends SparkFunSuite with SharedSparkSession { @Mock private val mockFactory: UCClientFactory = mock(classOf[UCClientFactory]) override def beforeEach(): Unit = { super.beforeEach() Mockito.reset(mockFactory) CommitCoordinatorProvider.clearAllBuilders() UCCommitCoordinatorBuilder.ucClientFactory = mockFactory UCCommitCoordinatorBuilder.clearCache() CommitCoordinatorProvider.registerBuilder(UCCommitCoordinatorBuilder) } case class CatalogTestConfig( name: String, uri: Option[String] = None, configMap: Map[String, String] = Map.empty, metastoreId: Option[String] = None, path: Option[String] = Some("io.unitycatalog.spark.UCSingleCatalog") ) def setupCatalogs(configs: CatalogTestConfig*)(testCode: => Unit): Unit = { val allConfigs = configs.flatMap { config => val baseConfigs = Seq( config.path.map(p => s"spark.sql.catalog.${config.name}" -> p), config.uri.map(uri => s"spark.sql.catalog.${config.name}.uri" -> uri) ).flatten // Add all additional configurations from configMap (without any prefix) val additionalConfigs = config.configMap.map { case (key, value) => s"spark.sql.catalog.${config.name}.$key" -> value } baseConfigs ++ additionalConfigs } withSQLConf(allConfigs: _*) { configs.foreach { config => (config.uri, config.configMap.isEmpty, config.metastoreId) match { case (Some(uri), false, Some(id)) => registerMetastoreId(uri, config.configMap, id) case (Some(uri), false, None) => registerMetastoreIdException( uri, config.configMap, new RuntimeException("Invalid metastore ID")) case _ => // Do nothing for incomplete configs } } testCode } } test("build with valid configuration") { val expectedMetastoreId = "test-metastore-id" val catalog1 = CatalogTestConfig( name = "catalog1", uri = Some("https://test-uri-1.com"), configMap = Map("type" -> "static", "token" -> "test-token-1"), metastoreId = Some(expectedMetastoreId) ) val catalog2 = CatalogTestConfig( name = "catalog2", uri = Some("https://test-uri-2.com"), configMap = Map("type" -> "static", "token" -> "test-token-2"), metastoreId = Some("different-metastore-id") ) setupCatalogs(catalog1, catalog2) { val result = getCommitCoordinatorClient(expectedMetastoreId) assert(result.isInstanceOf[UCCommitCoordinatorClient]) verify(mockFactory, times(2)).createUCClient(catalog1.uri.get, catalog1.configMap) verify(mockFactory).createUCClient(catalog2.uri.get, catalog2.configMap) verify(mockFactory.createUCClient(catalog1.uri.get, catalog1.configMap)) .getMetastoreId verify(mockFactory.createUCClient(catalog2.uri.get, catalog2.configMap)) .getMetastoreId verify(mockFactory.createUCClient(catalog2.uri.get, catalog2.configMap)).close() verify(mockFactory.createUCClient(catalog1.uri.get, catalog1.configMap)).close() } } test("token based rest client factory default app versions") { val defaults = UCTokenBasedRestClientFactory.defaultAppVersions assert(defaults("Delta") === io.delta.VERSION) assert(defaults("Spark") === org.apache.spark.SPARK_VERSION) assert(defaults("Scala") === scala.util.Properties.versionNumberString) assert(defaults("Java") === System.getProperty("java.version")) } test("createUCClientWithVersions passes custom app versions to UCClient") { val customVersions = Map( "Delta" -> io.delta.VERSION, "Kernel" -> "4.0.0", "Delta V2 connector" -> "true" ) val defaults = UCTokenBasedRestClientFactory.defaultAppVersions val merged = defaults ++ customVersions assert(merged("Kernel") === "4.0.0") assert(merged("Delta V2 connector") === "true") assert(merged("Delta") === io.delta.VERSION) assert(merged("Spark") === org.apache.spark.SPARK_VERSION) } test("build with missing metastore ID") { val exception = intercept[IllegalArgumentException] { CommitCoordinatorProvider.getCommitCoordinatorClient( UCCommitCoordinatorBuilder.getName, Map.empty, spark) } assert(exception.getMessage.contains("UC metastore ID not found")) } test("build with no matching catalog") { val metastoreId = "test-metastore-id" val catalog = CatalogTestConfig( name = "catalog", uri = Some("https://test-uri.com"), configMap = Map("type" -> "static", "token" -> "test-token"), metastoreId = Some("different-metastore-id") ) setupCatalogs(catalog) { val exception = intercept[IllegalStateException] { getCommitCoordinatorClient(metastoreId) } assert(exception.getMessage.contains("No matching catalog found")) verify(mockFactory).createUCClient(catalog.uri.get, catalog.configMap) verify(mockFactory.createUCClient(catalog.uri.get, catalog.configMap)).getMetastoreId verify(mockFactory.createUCClient(catalog.uri.get, catalog.configMap)).close() } } test("build with multiple matching catalogs") { val metastoreId = "test-metastore-id" val catalog1 = CatalogTestConfig( name = "catalog1", uri = Some("https://test-uri1.com"), configMap = Map("type" -> "static", "token" -> "test-token-1"), metastoreId = Some(metastoreId) ) val catalog2 = CatalogTestConfig( name = "catalog2", uri = Some("https://test-uri2.com"), configMap = Map("type" -> "static", "token" -> "test-token-2"), metastoreId = Some(metastoreId) ) setupCatalogs(catalog1, catalog2) { val exception = intercept[IllegalStateException] { getCommitCoordinatorClient(metastoreId) } assert(exception.getMessage.contains("Found multiple catalogs")) verify(mockFactory).createUCClient(catalog1.uri.get, catalog1.configMap) verify(mockFactory).createUCClient(catalog2.uri.get, catalog2.configMap) verify(mockFactory.createUCClient(catalog1.uri.get, catalog1.configMap)) .getMetastoreId verify(mockFactory.createUCClient(catalog2.uri.get, catalog2.configMap)) .getMetastoreId verify(mockFactory.createUCClient(catalog1.uri.get, catalog1.configMap)).close() verify(mockFactory.createUCClient(catalog2.uri.get, catalog2.configMap)).close() } } test("build with mixed valid and invalid catalog configurations") { val expectedMetastoreId = "test-metastore-id" val validCatalog = CatalogTestConfig( name = "valid-catalog", uri = Some("https://valid-uri.com"), configMap = Map("type" -> "static", "token" -> "valid-token"), metastoreId = Some(expectedMetastoreId) ) val invalidCatalog1 = CatalogTestConfig( name = "invalid-catalog-1", uri = Some("https://invalid-uri.com"), configMap = Map("type" -> "static", "token" -> "invalid-token"), metastoreId = None ) val invalidCatalog2 = CatalogTestConfig( name = "invalid-catalog-2", uri = Some("random-uri"), configMap = Map("type" -> "static", "token" -> "invalid-token") ) val incompleteCatalog = CatalogTestConfig( name = "incomplete-catalog", path = None ) setupCatalogs(validCatalog, invalidCatalog1, invalidCatalog2, incompleteCatalog) { val result = getCommitCoordinatorClient(expectedMetastoreId) assert(result.isInstanceOf[UCCommitCoordinatorClient]) verify(mockFactory, times(2)).createUCClient( validCatalog.uri.get, validCatalog.configMap ) verify(mockFactory.createUCClient(validCatalog.uri.get, validCatalog.configMap), times(1)).close() } } test("build caching behavior") { val metastoreId = "test-metastore-id" val catalog = CatalogTestConfig( name = "catalog", uri = Some("https://test-uri.com"), configMap = Map("type" -> "static", "token" -> "test-token"), metastoreId = Some(metastoreId) ) setupCatalogs(catalog) { val result1 = getCommitCoordinatorClient(metastoreId) val result2 = getCommitCoordinatorClient(metastoreId) assert(result1 eq result2) } } test("build with multiple catalogs pointing to the same URI, token, and metastore") { val metastoreId = "shared-metastore-id" val sharedUri = "https://shared-test-uri.com" val sharedConfigMap = Map("type" -> "static", "token" -> "shared-test-token") val catalog1 = CatalogTestConfig( name = "catalog1", uri = Some(sharedUri), configMap = sharedConfigMap, metastoreId = Some(metastoreId) ) val catalog2 = CatalogTestConfig( name = "catalog2", uri = Some(sharedUri), configMap = sharedConfigMap, metastoreId = Some(metastoreId) ) val catalog3 = CatalogTestConfig( name = "catalog3", uri = Some(sharedUri), configMap = sharedConfigMap, metastoreId = Some(metastoreId) ) setupCatalogs(catalog1, catalog2, catalog3) { val result = getCommitCoordinatorClient(metastoreId) assert(result.isInstanceOf[UCCommitCoordinatorClient]) verify(mockFactory, times(2)).createUCClient(sharedUri, sharedConfigMap) verify(mockFactory.createUCClient(sharedUri, sharedConfigMap)).getMetastoreId verify(mockFactory.createUCClient(sharedUri, sharedConfigMap)).close() } } test("build with a catalog having invalid path but valid URI and token") { val metastoreId = "test-metastore-id" val catalog = CatalogTestConfig( name = "invalid-path-catalog", uri = Some("https://test-uri.com"), configMap = Map("type" -> "static", "token" -> "test-token"), metastoreId = Some(metastoreId), path = Some("invalid-catalog-path") ) setupCatalogs(catalog) { assert(UCCommitCoordinatorBuilder.getCatalogConfigs(spark).isEmpty) val e = intercept[IllegalStateException] { getCommitCoordinatorClient(metastoreId) } assert(e.getMessage.contains("No matching catalog found")) verify(mockFactory, never()).createUCClient(catalog.uri.get, catalog.configMap) } } private def registerMetastoreId( uri: String, configMap: Map[String, String], metastoreId: String): Unit = { val mockClient = org.mockito.Mockito.mock(classOf[UCClient]) when(mockClient.getMetastoreId).thenReturn(metastoreId) when(mockFactory.createUCClient(meq(uri), meq(configMap))).thenReturn(mockClient) } private def registerMetastoreIdException( uri: String, configMap: Map[String, String], exception: Throwable): Unit = { val mockClient = org.mockito.Mockito.mock(classOf[UCClient]) when(mockClient.getMetastoreId).thenThrow(exception) when(mockFactory.createUCClient(meq(uri), meq(configMap))).thenReturn(mockClient) } test("getCatalogConfigs with legacy token format") { val catalogName = "legacy_catalog" val uri = "https://test-uri.com" val token = "test-token" withSQLConf( s"spark.sql.catalog.$catalogName" -> "io.unitycatalog.spark.UCSingleCatalog", s"spark.sql.catalog.$catalogName.uri" -> uri, s"spark.sql.catalog.$catalogName.token" -> token ) { val configs = UCCommitCoordinatorBuilder.getCatalogConfigs(spark) assert(configs.length == 1) val (name, catalogUri, authConfigMap) = configs.head assert(name == catalogName) assert(catalogUri == uri) // Legacy token should be converted to new format assert(authConfigMap.contains("type")) assert(authConfigMap("type") == "static") assert(authConfigMap.contains("token")) assert(authConfigMap("token") == token) } } test("getCatalogConfigs with new auth.* format") { val catalogName = "new_catalog" val uri = "https://test-uri.com" val token = "test-token" withSQLConf( s"spark.sql.catalog.$catalogName" -> "io.unitycatalog.spark.UCSingleCatalog", s"spark.sql.catalog.$catalogName.uri" -> uri, s"spark.sql.catalog.$catalogName.auth.type" -> "static", s"spark.sql.catalog.$catalogName.auth.token" -> token ) { val configs = UCCommitCoordinatorBuilder.getCatalogConfigs(spark) assert(configs.length == 1) val (name, catalogUri, authConfigMap) = configs.head assert(name == catalogName) assert(catalogUri == uri) assert(authConfigMap("type") == "static") assert(authConfigMap("token") == token) } } test("getCatalogConfigs with nested auth.* configurations") { val catalogName = "oauth_catalog" val uri = "https://test-uri.com" withSQLConf( s"spark.sql.catalog.$catalogName" -> "io.unitycatalog.spark.UCSingleCatalog", s"spark.sql.catalog.$catalogName.uri" -> uri, s"spark.sql.catalog.$catalogName.auth.type" -> "oauth", s"spark.sql.catalog.$catalogName.auth.oauth.uri" -> "https://oauth.example.com", s"spark.sql.catalog.$catalogName.auth.oauth.client_id" -> "client123", s"spark.sql.catalog.$catalogName.auth.oauth.client_secret" -> "secret456" ) { val configs = UCCommitCoordinatorBuilder.getCatalogConfigs(spark) assert(configs.length == 1) val (name, catalogUri, authConfigMap) = configs.head assert(name == catalogName) assert(catalogUri == uri) assert(authConfigMap("type") == "oauth") assert(authConfigMap("oauth.uri") == "https://oauth.example.com") assert(authConfigMap("oauth.client_id") == "client123") assert(authConfigMap("oauth.client_secret") == "secret456") } } test("getCatalogConfigs skips catalog with no auth configurations") { val catalogName = "no_auth_catalog" val uri = "https://test-uri.com" withSQLConf( s"spark.sql.catalog.$catalogName" -> "io.unitycatalog.spark.UCSingleCatalog", s"spark.sql.catalog.$catalogName.uri" -> uri ) { val configs = UCCommitCoordinatorBuilder.getCatalogConfigs(spark) assert(configs.isEmpty, "Catalog without auth config should be skipped") } } test("getCatalogConfigs prefers new auth.* format over legacy token") { val catalogName = "mixed_catalog" val uri = "https://test-uri.com" val legacyToken = "legacy-token" val newToken = "new-token" withSQLConf( s"spark.sql.catalog.$catalogName" -> "io.unitycatalog.spark.UCSingleCatalog", s"spark.sql.catalog.$catalogName.uri" -> uri, s"spark.sql.catalog.$catalogName.token" -> legacyToken, s"spark.sql.catalog.$catalogName.auth.type" -> "static", s"spark.sql.catalog.$catalogName.auth.token" -> newToken ) { val configs = UCCommitCoordinatorBuilder.getCatalogConfigs(spark) assert(configs.length == 1) val (name, catalogUri, authConfigMap) = configs.head assert(name == catalogName) assert(catalogUri == uri) // New format should take precedence assert(authConfigMap("type") == "static") assert(authConfigMap("token") == newToken) assert(!authConfigMap.contains(legacyToken)) } } test("getCatalogConfigs handles multiple catalogs with mixed formats") { withSQLConf( "spark.sql.catalog.catalog1" -> "io.unitycatalog.spark.UCSingleCatalog", "spark.sql.catalog.catalog1.uri" -> "https://uri1.com", "spark.sql.catalog.catalog1.token" -> "token1", "spark.sql.catalog.catalog2" -> "io.unitycatalog.spark.UCSingleCatalog", "spark.sql.catalog.catalog2.uri" -> "https://uri2.com", "spark.sql.catalog.catalog2.auth.type" -> "static", "spark.sql.catalog.catalog2.auth.token" -> "token2", "spark.sql.catalog.catalog3" -> "io.unitycatalog.spark.UCSingleCatalog", "spark.sql.catalog.catalog3.uri" -> "https://uri3.com" ) { val configs = UCCommitCoordinatorBuilder.getCatalogConfigs(spark) // Only catalog1 and catalog2 should be included (catalog3 has no auth) assert(configs.length == 2) val catalog1 = configs.find(_._1 == "catalog1") assert(catalog1.isDefined) assert(catalog1.get._3("type") == "static") assert(catalog1.get._3("token") == "token1") val catalog2 = configs.find(_._1 == "catalog2") assert(catalog2.isDefined) assert(catalog2.get._3("type") == "static") assert(catalog2.get._3("token") == "token2") } } test("buildForCatalog with legacy token format") { val catalogName = "test_catalog" val uri = "https://test-uri.com" val token = "test-token" withSQLConf( s"spark.sql.catalog.$catalogName" -> "io.unitycatalog.spark.UCSingleCatalog", s"spark.sql.catalog.$catalogName.uri" -> uri, s"spark.sql.catalog.$catalogName.token" -> token ) { val result = UCCommitCoordinatorBuilder.buildForCatalog(spark, catalogName) assert(result.isInstanceOf[UCCommitCoordinatorClient]) // Verify that createUCClient was called with the converted auth config verify(mockFactory).createUCClient( meq(uri), any[Map[String, String]]() ) } } test("buildForCatalog with new auth.* format") { val catalogName = "test_catalog" val uri = "https://test-uri.com" val token = "test-token" withSQLConf( s"spark.sql.catalog.$catalogName" -> "io.unitycatalog.spark.UCSingleCatalog", s"spark.sql.catalog.$catalogName.uri" -> uri, s"spark.sql.catalog.$catalogName.auth.type" -> "static", s"spark.sql.catalog.$catalogName.auth.token" -> token ) { val result = UCCommitCoordinatorBuilder.buildForCatalog(spark, catalogName) assert(result.isInstanceOf[UCCommitCoordinatorClient]) verify(mockFactory).createUCClient( meq(uri), any[Map[String, String]]() ) } } test("buildForCatalog with non-existent catalog") { val exception = intercept[IllegalArgumentException] { UCCommitCoordinatorBuilder.buildForCatalog(spark, "non_existent_catalog") } assert(exception.getMessage.contains("not found")) } private def getCommitCoordinatorClient(metastoreId: String) = { CommitCoordinatorProvider.getCommitCoordinatorClient( UCCommitCoordinatorBuilder.getName, Map(UCCommitCoordinatorClient.UC_METASTORE_ID_KEY -> metastoreId), spark) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/UCCommitCoordinatorClientSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import java.io.IOException import java.lang.{Long => JLong} import java.util.{List => JList, Optional} import scala.collection.JavaConverters._ import scala.reflect.ClassTag // scalastyle:off import.ordering.noEmptyLine import com.databricks.spark.util.{Log4jUsageLogger, UsageRecord} import org.apache.spark.sql.delta.{DeltaConfigs, DeltaIllegalArgumentException, DeltaLog, LogSegment, Snapshot} import org.apache.spark.sql.delta.CommitCoordinatorGetCommitsFailedException import org.apache.spark.sql.delta.DeltaConfigs.{COORDINATED_COMMITS_COORDINATOR_CONF, COORDINATED_COMMITS_COORDINATOR_NAME, COORDINATED_COMMITS_TABLE_CONF} import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile import org.apache.spark.sql.delta.actions.{CommitInfo, Metadata, Protocol} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.storage.LogStoreInverseAdaptor import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import io.delta.storage.LogStore import io.delta.storage.commit.{ Commit => JCommit, CommitFailedException => JCommitFailedException, CoordinatedCommitsUtils => JCoordinatedCommitsUtils, TableDescriptor, UpdatedActions } import io.delta.storage.commit.uccommitcoordinator.{ UCCommitCoordinatorClient, UCCoordinatedCommitsUsageLogs} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, FileSystem, LocalFileSystem, Path} import org.mockito.ArgumentMatchers.anyString import org.mockito.Mockito import org.mockito.Mockito.{mock, when} import org.scalatest.PrivateMethodTester import org.scalatest.time.SpanSugar._ import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.internal.SQLConf import org.apache.spark.util.SystemClock class UCCommitCoordinatorClientSuite extends UCCommitCoordinatorClientSuiteBase with PrivateMethodTester { protected override def sparkConf = super.sparkConf .set("spark.sql.catalog.main", "io.unitycatalog.spark.UCSingleCatalog") .set("spark.sql.catalog.main.uri", "https://test-uri.com") .set("spark.sql.catalog.main.token", "test-token") .set("spark.hadoop.fs.file.impl", classOf[LocalFileSystem].getCanonicalName) override protected def commit( version: Long, timestamp: Long, tableCommitCoordinatorClient: TableCommitCoordinatorClient, tableIdentifier: Option[TableIdentifier] = None): JCommit = { val commitResult = super.commit( version, timestamp, tableCommitCoordinatorClient, tableIdentifier) // As backfilling for UC happens after every commit asynchronously, we block here until // the current in-progress backfill has completed in order to make tests deterministic. waitForBackfill(version, tableCommitCoordinatorClient) commitResult } protected def assertUsageLogsContains(usageLogs: Seq[UsageRecord], opType: String): Unit = { assert(usageLogs.exists { record => record.tags.get("opType").contains(opType) }) } test("incorrect last known backfilled version") { withTempTableDir { tempDir => val log = DeltaLog.forTable(spark, tempDir.toString) val logPath = log.logPath val tableCommitCoordinatorClient = createTableCommitCoordinatorClient(log) tableCommitCoordinatorClient.commitCoordinatorClient.registerTable( logPath, Optional.empty(), -1L, initMetadata, Protocol(1, 1)) // Write 11 commits. writeCommitZero(logPath) (1 to 10).foreach(i => commit(i, i, tableCommitCoordinatorClient)) // Now delete some backfilled versions val fs = logPath.getFileSystem(log.newDeltaHadoopConf()) fs.delete(FileNames.unsafeDeltaFile(logPath, 8), false) fs.delete(FileNames.unsafeDeltaFile(logPath, 9), false) fs.delete(FileNames.unsafeDeltaFile(logPath, 10), false) // Backfill with the wrong specified last version val e = intercept[IllegalStateException] { tableCommitCoordinatorClient.backfillToVersion(10L, Some(9L)) } assert(e.getMessage.contains("Last known backfilled version 9 doesn't exist")) // Backfill with the correct version tableCommitCoordinatorClient.backfillToVersion(10L, Some(7L)) // Everything should be backfilled now validateBackfillStrategy(tableCommitCoordinatorClient, logPath, 10) } } test("test getLastKnownBackfilledVersion") { withTempTableDir { tempDir => val backfillListingOffset = 5 val log = DeltaLog.forTable(spark, tempDir.toString) val logPath = log.logPath UCCommitCoordinatorClient.BACKFILL_LISTING_OFFSET = backfillListingOffset val tableCommitCoordinatorClient = createTableCommitCoordinatorClient(log) tableCommitCoordinatorClient.commitCoordinatorClient.registerTable( logPath, Optional.empty(), -1L, initMetadata, Protocol(1, 1)) val hadoopConf = log.newDeltaHadoopConf() val fs = logPath.getFileSystem(hadoopConf) writeCommitZero(logPath) val backfillThreshold = 5 (1 to backfillThreshold + backfillListingOffset + 5).foreach { commitVersion => commit(commitVersion, commitVersion, tableCommitCoordinatorClient) if (commitVersion > backfillThreshold) { // After x = backfillThreshold commits, delete all backfilled files to simulate // backfill failing. This means UC should keep track of all commits starting // from x and nothing >= x should be backfilled. (backfillThreshold + 1 to commitVersion).foreach { deleteVersion => fs.delete(FileNames.unsafeDeltaFile(logPath, deleteVersion), false) } val tableDesc = new TableDescriptor( logPath, Optional.empty(), tableCommitCoordinatorClient.tableConf.asJava) val ucCommitCoordinatorClient = tableCommitCoordinatorClient.commitCoordinatorClient .asInstanceOf[UCCommitCoordinatorClient] assert( ucCommitCoordinatorClient.getLastKnownBackfilledVersion( commitVersion, hadoopConf, LogStoreInverseAdaptor(log.store, hadoopConf), tableDesc ) == backfillThreshold ) } } } } test("commit-limit-reached exception handling") { withTempTableDir { tempDir => val log = DeltaLog.forTable(spark, tempDir.toString) val logPath = log.logPath // Create a client that does not register backfills to keep accumulating // commits in the commit coordinator. val noBackfillRegistrationClient = new UCCommitCoordinatorClient(Map.empty[String, String].asJava, ucClient) with DeltaLogging { override def backfillToVersion( logStore: LogStore, hadoopConf: Configuration, tableDesc: TableDescriptor, version: Long, lastKnownBackfilledVersion: JLong): Unit = { throw new IOException("Simulated exception") } override protected def recordDeltaEvent(opType: String, data: Any, path: Path): Unit = { data match { case ref: AnyRef => recordDeltaEvent(null, opType = opType, data = ref, path = Some(path)) } } } // Client 1 performs backfills correctly. val tcc1 = createTableCommitCoordinatorClient(log) // Client 2 does not backfill. val tcc2 = tcc1.copy(commitCoordinatorClient = noBackfillRegistrationClient) // Write 10 commits to fill up the commit coordinator (MAX_NUM_COMMITS is set to 10 // in the InMemoryUCCommitCoordinator). writeCommitZero(logPath) // We use super.commit here because tco2 does not backfill so the local override of // commit would fail waiting for the commits to be backfilled. This also applies // to the retry of commit 11 with tco2 below. (1 to 10).foreach(i => super.commit(version = i, timestamp = i, tableCommitCoordinatorClient = tcc2) ) // Commit 11 should trigger an exception and a full backfill should be attempted. // With tcc2, this backfill attempt should again fail, leading to a user facing // CommitLimitReachedException, along with the usage logs. var usageLogs = Log4jUsageLogger.track { val e1 = intercept[JCommitFailedException] { super.commit(version = 11, timestamp = 11, tableCommitCoordinatorClient = tcc2) } val tableId = tcc2.tableConf(UCCommitCoordinatorClient.UC_TABLE_ID_KEY) assert(e1.getMessage.contains(s"Too many unbackfilled commits for $tableId.")) assert(e1.getMessage.contains(s"A full backfill attempt failed due to: " + "java.io.IOException: Simulated exception")) } assertUsageLogsContains( usageLogs, UCCoordinatedCommitsUsageLogs.UC_FULL_BACKFILL_ATTEMPT_FAILED) // Retry commit 11 with tcc1. This should again trigger an exception and a full // backfill should be attempted but the backfill should succeed this time. The // commit is then retried automatically and should succeed. We use the local // override of commit here to ensure that we only return once commit 11 has // been backfilled and the remaining asserts pass. usageLogs = Log4jUsageLogger.track { commit(version = 11, timestamp = 11, tableCommitCoordinatorClient = tcc1) } assertUsageLogsContains(usageLogs, UCCoordinatedCommitsUsageLogs.UC_ATTEMPT_FULL_BACKFILL) validateBackfillStrategy(tcc1, logPath, version = 11) } } test("usage logs in commit calls are emitted correctly") { withTempTableDir { tempDir => val log = DeltaLog.forTable(spark, tempDir.toString) val eventLoggerClient = new UCCommitCoordinatorClient(Map.empty[String, String].asJava, ucClient) with DeltaLogging { override protected def recordDeltaEvent(opType: String, data: Any, path: Path): Unit = { data match { case ref: AnyRef => recordDeltaEvent(null, opType = opType, data = ref, path = Some(path)) } } } val logPath = log.logPath val tableCommitCoordinatorClient = createTableCommitCoordinatorClient(log) .copy(commitCoordinatorClient = eventLoggerClient) writeCommitZero(logPath) // A normal commit should emit one usage log. val usageLogs = Log4jUsageLogger.track { commit(version = 1, timestamp = 1, tableCommitCoordinatorClient) } assertUsageLogsContains(usageLogs, UCCoordinatedCommitsUsageLogs.UC_COMMIT_STATS) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/UCCommitCoordinatorClientSuiteBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.coordinatedcommits import java.io.File import java.net.URI import java.util.{Collections, Optional, UUID} import scala.collection.JavaConverters._ // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.DeltaConfigs.{COORDINATED_COMMITS_COORDINATOR_CONF, COORDINATED_COMMITS_COORDINATOR_NAME, COORDINATED_COMMITS_TABLE_CONF} import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.actions.{Metadata, Protocol} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import io.delta.storage.commit.{ CoordinatedCommitsUtils => JCoordinatedCommitsUtils, GetCommitsResponse => JGetCommitsResponse } import io.delta.storage.commit.uccommitcoordinator.{UCClient, UCCommitCoordinatorClient} import org.apache.hadoop.fs.Path import org.mockito.ArgumentMatchers.{any, anyString} import org.mockito.Mock import org.mockito.Mockito import org.mockito.Mockito.{mock, when} import org.scalatest.time.SpanSugar._ import org.apache.spark.sql.types.{IntegerType, StringType, StructField} trait UCCommitCoordinatorClientSuiteBase extends CommitCoordinatorClientImplSuiteBase { /** * A unique table ID for each test. */ protected var tableUUID = UUID.randomUUID() /** * A unique metastore ID for each test. */ protected var metastoreId = UUID.randomUUID() protected var ucClient: UCClient = _ @Mock protected var mockFactory: UCClientFactory = _ protected var ucCommitCoordinator: InMemoryUCCommitCoordinator = _ protected override def beforeAll(): Unit = { val tmpDirName = System.getProperty("java.io.tmpdir") val tmpDir = new File(tmpDirName) if (!tmpDir.exists()) { tmpDir.mkdirs() } super.beforeAll() mockFactory = mock(classOf[UCClientFactory]) } override def beforeEach(): Unit = { super.beforeEach() tableUUID = UUID.randomUUID() UCCommitCoordinatorClient.BACKFILL_LISTING_OFFSET = 100 metastoreId = UUID.randomUUID() DeltaLog.clearCache() Mockito.reset(mockFactory) CommitCoordinatorProvider.clearAllBuilders() UCCommitCoordinatorBuilder.ucClientFactory = mockFactory UCCommitCoordinatorBuilder.clearCache() CommitCoordinatorProvider.registerBuilder(UCCommitCoordinatorBuilder) ucCommitCoordinator = new InMemoryUCCommitCoordinator() ucClient = new InMemoryUCClient(metastoreId.toString, ucCommitCoordinator) when(mockFactory.createUCClient(anyString(), any[Map[String, String]]())).thenReturn(ucClient) } override protected def createTableCommitCoordinatorClient( deltaLog: DeltaLog): TableCommitCoordinatorClient = { var commitCoordinatorClient = UCCommitCoordinatorBuilder .build(spark, Map(UCCommitCoordinatorClient.UC_METASTORE_ID_KEY -> metastoreId.toString)) .asInstanceOf[UCCommitCoordinatorClient] commitCoordinatorClient = new UCCommitCoordinatorClient( commitCoordinatorClient.conf, commitCoordinatorClient.ucClient) with DeltaLogging { override def recordDeltaEvent(opType: String, data: Any, path: Path): Unit = { data match { case ref: AnyRef => recordDeltaEvent(null, opType = opType, data = ref, path = Some(path)) case _ => super.recordDeltaEvent(opType, data, path) } } } // Initialize table ID for the calling test // tableUUID = UUID.randomUUID().toString commitCoordinatorClient.registerTable( deltaLog.logPath, Optional.empty(), -1L, initMetadata(), Protocol(1, 1)) TableCommitCoordinatorClient( commitCoordinatorClient, deltaLog, Map(UCCommitCoordinatorClient.UC_TABLE_ID_KEY -> tableUUID.toString) ) } override protected def registerBackfillOp( tableCommitCoordinatorClient: TableCommitCoordinatorClient, deltaLog: DeltaLog, version: Long): Unit = { ucClient.commit( tableUUID.toString, JCoordinatedCommitsUtils.getTablePath(deltaLog.logPath).toUri, Optional.empty(), Optional.of(version), false, Optional.empty(), Optional.empty(), Optional.empty() /* icebergMetadata */) } override protected def validateBackfillStrategy( tableCommitCoordinatorClient: TableCommitCoordinatorClient, logPath: Path, version: Long): Unit = { val response = tableCommitCoordinatorClient.getCommits() assert(response.getCommits.size == 1) assert(response.getCommits.asScala.head.getVersion == version) assert(response.getLatestTableVersion == version) } protected def validateGetCommitsResult( response: JGetCommitsResponse, startVersion: Option[Long], endVersion: Option[Long], maxVersion: Long): Unit = { val expectedVersions = endVersion.map { _ => Seq.empty }.getOrElse(Seq(maxVersion)) assert(response.getCommits.asScala.map(_.getVersion) == expectedVersions) assert(response.getLatestTableVersion == maxVersion) } override protected def initMetadata(): Metadata = { // Ensure that the metadata that is passed to registerTable has the // correct table conf set. Metadata(configuration = Map( COORDINATED_COMMITS_TABLE_CONF.key -> JsonUtils.toJson(Map(UCCommitCoordinatorClient.UC_TABLE_ID_KEY -> tableUUID.toString)), COORDINATED_COMMITS_COORDINATOR_NAME.key -> UCCommitCoordinatorBuilder.getName, COORDINATED_COMMITS_COORDINATOR_CONF.key -> JsonUtils.toJson( Map(UCCommitCoordinatorClient.UC_METASTORE_ID_KEY -> metastoreId.toString)))) } protected def waitForBackfill( version: Long, tableCommitCoordinatorClient: TableCommitCoordinatorClient): Unit = { eventually(timeout(10.seconds)) { val logPath = tableCommitCoordinatorClient.logPath val log = DeltaLog.forTable(spark, JCoordinatedCommitsUtils.getTablePath(logPath)) val fs = logPath.getFileSystem(log.newDeltaHadoopConf()) assert(fs.exists(FileNames.unsafeDeltaFile(logPath, version))) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/DeletionVectorsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.deletionvectors import java.io.{File, FileNotFoundException} import java.net.URISyntaxException import org.apache.spark.sql.delta.{DeletionVectorsTableFeature, DeletionVectorsTestUtils, DeltaChecksumException, DeltaConfigs, DeltaLog, DeltaMetricsUtils, DeltaTestUtilsForTempViews} import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile import org.apache.spark.sql.delta.actions.{AddFile, DeletionVectorDescriptor, RemoveFile} import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor.EMPTY import org.apache.spark.sql.delta.deletionvectors.DeletionVectorsSuite._ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaExceptionTestUtils, DeltaSQLCommandTest} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.JsonUtils import com.fasterxml.jackson.databind.node.ObjectNode import io.delta.tables.DeltaTable import org.apache.commons.io.FileUtils import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.parquet.format.converter.ParquetMetadataConverter import org.apache.parquet.hadoop.ParquetFileReader import org.apache.spark.SparkConf import org.apache.spark.SparkException import org.apache.spark.sql.{DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.plans.logical.{AppendData, Subquery} import org.apache.spark.sql.execution.FileSourceScanExec import org.apache.spark.sql.functions.col import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession class DeletionVectorsSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with DeletionVectorsTestUtils with DeltaTestUtilsForTempViews with DeltaExceptionTestUtils { import testImplicits._ override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX.key, "false") } protected def hadoopConf(): Configuration = { // scalastyle:off hadoopconfiguration // This is to generate a Parquet file with two row groups spark.sparkContext.hadoopConfiguration // scalastyle:on hadoopconfiguration } test(s"read Delta table with deletion vectors") { def verifyVersion(version: Int, expectedData: Seq[Int]): Unit = { checkAnswer( spark.read.format("delta").option("versionAsOf", version.toString).load(table1Path), expectedData.toDF()) } // Verify all versions of the table verifyVersion(0, expectedTable1DataV0) verifyVersion(1, expectedTable1DataV1) verifyVersion(2, expectedTable1DataV2) verifyVersion(3, expectedTable1DataV3) verifyVersion(4, expectedTable1DataV4) } test(s"read partitioned Delta table with deletion vectors") { def verify(version: Int, expectedData: Seq[Int], filterExp: String = "true"): Unit = { val query = spark.read.format("delta") .option("versionAsOf", version.toString) .load(table3Path) .filter(filterExp) val expected = expectedData.toDF("id") .withColumn("partCol", col("id") % 10) .filter(filterExp) checkAnswer(query, expected) } // Verify all versions of the table verify(0, expectedTable3DataV0) verify(1, expectedTable3DataV1) verify(2, expectedTable3DataV2) verify(3, expectedTable3DataV3) verify(4, expectedTable3DataV4) verify(4, expectedTable3DataV4, filterExp = "partCol = 3") verify(3, expectedTable3DataV3, filterExp = "partCol = 3 and id > 25") verify(1, expectedTable3DataV1, filterExp = "id > 25") } test("select metadata columns from a Delta table with deletion vectors") { assert(spark.read.format("delta").load(table1Path) .select("_metadata.file_path").distinct().count() == 22) } test("throw error when non-pinned TahoeFileIndex snapshot is used") { // Corner case where we still have non-pinned TahoeFileIndex when data skipping is disabled withSQLConf(DeltaSQLConf.DELTA_STATS_SKIPPING.key -> "false") { def assertError(dataFrame: DataFrame): Unit = { val ex = intercept[IllegalArgumentException] { dataFrame.collect() } assert(ex.getMessage contains "Cannot work with a non-pinned table snapshot of the TahoeFileIndex") } assertError(spark.read.format("delta").load(table1Path)) assertError(spark.read.format("delta").option("versionAsOf", "2").load(table1Path)) } } test("read Delta table with deletion vectors with a filter") { checkAnswer( spark.read.format("delta").load(table1Path).where("value in (300, 787, 239)"), // 300 is removed in the final table Seq(787, 239).toDF()) } test("read Delta table with DV for a select files") { val deltaLog = DeltaLog.forTable(spark, table1Path) val snapshot = deltaLog.unsafeVolatileSnapshot // Select a subset of files with DVs and specific value range, this is just to test // that reading these files will respect the DVs var rowCount = 0L var deletedRowCount = 0L val selectFiles = snapshot.allFiles.collect().filter( addFile => { val stats = JsonUtils.mapper.readTree(addFile.stats).asInstanceOf[ObjectNode] // rowCount += stats.get("rowCount") val min = stats.get("minValues").get("value").toString val max = stats.get("maxValues").get("value").toString val selected = (min == "18" && max == "1988") || (min == "33" && max == "1995") || (min == "13" && max == "1897") // TODO: these steps will be easier and also change (depending upon tightBounds value) once // we expose more methods on AddFile as part of the data skipping changes with DVs if (selected) { rowCount += stats.get("numRecords").asInt(0) deletedRowCount += Option(addFile.deletionVector).getOrElse(EMPTY).cardinality } selected } ).toSeq assert(selectFiles.filter(_.deletionVector != null).size > 1) // make at least one file has DV assert(deltaLog.createDataFrame(snapshot, selectFiles).count() == rowCount - deletedRowCount) } for (optimizeMetadataQuery <- BOOLEAN_DOMAIN) test("read Delta tables with DVs in subqueries: " + s"metadataQueryOptimizationEnabled=$optimizeMetadataQuery") { withSQLConf(DeltaSQLConf.DELTA_OPTIMIZE_METADATA_QUERY_ENABLED.key -> optimizeMetadataQuery.toString) { val table1 = s"delta.`${new File(table1Path).getAbsolutePath}`" val table2 = s"delta.`${new File(table2Path).getAbsolutePath}`" def assertQueryResult(query: String, expected1: Int, expected2: Int): Unit = { val df = spark.sql(query) assertPlanContains(df, Subquery.getClass.getSimpleName.stripSuffix("$")) val actual = df.collect()(0) // fetch only row in the result assert(actual === Row(expected1, expected2)) } // same table used twice in the query val query1 = s"SELECT (SELECT COUNT(*) FROM $table1), (SELECT COUNT(*) FROM $table1)" assertQueryResult(query1, expectedTable1DataV4.size, expectedTable1DataV4.size) // two tables used in the query val query2 = s"SELECT (SELECT COUNT(*) FROM $table1), (SELECT COUNT(*) FROM $table2)" assertQueryResult(query2, expectedTable1DataV4.size, expectedTable2DataV1.size) } } test("insert into Delta table with DVs") { withTempDir { tempDir => val source1 = new File(table1Path) val source2 = new File(table2Path) val target = new File(tempDir, "insertTest") // Copy the source2 DV table to a temporary directory FileUtils.copyDirectory(source1, target) // Insert data from source2 into source1 (copied to target) // This blind append generates a plan with `V2WriteCommand` which is a corner // case in `PrepareDeltaScan` rule val insertDf = spark.sql(s"INSERT INTO TABLE delta.`${target.getAbsolutePath}` " + s"SELECT * FROM delta.`${source2.getAbsolutePath}`") // [[AppendData]] is one of the [[V2WriteCommand]] subtypes assertPlanContains(insertDf, AppendData.getClass.getSimpleName.stripSuffix("$")) val dataInTarget = spark.sql(s"SELECT * FROM delta.`${target.getAbsolutePath}`") // Make sure the number of rows is correct. for (metadataQueryOptimization <- BOOLEAN_DOMAIN) { withSQLConf(DeltaSQLConf.DELTA_OPTIMIZE_METADATA_QUERY_ENABLED.key -> metadataQueryOptimization.toString) { assert(dataInTarget.count() == expectedTable2DataV1.size + expectedTable1DataV4.size) } } // Make sure the contents are the same checkAnswer( dataInTarget, spark.sql( s"SELECT * FROM delta.`${source1.getAbsolutePath}` UNION ALL " + s"SELECT * FROM delta.`${source2.getAbsolutePath}`") ) } } test("DELETE with DVs - on a table with no prior DVs") { withDeletionVectorsEnabled() { withTempDir { dirName => // Create table with 500 files of 2 rows each. val numFiles = 500 val path = dirName.getAbsolutePath spark.range(0, 1000, step = 1, numPartitions = numFiles).write.format("delta").save(path) val tableName = s"delta.`$path`" val log = DeltaLog.forTable(spark, path) val beforeDeleteFilesWithStats = log.update().allFiles.collect() val beforeDeleteFiles = beforeDeleteFilesWithStats.map(_.path) val numFilesWithDVs = 100 val numDeletedRows = numFilesWithDVs * 1 spark.sql(s"DELETE FROM $tableName WHERE id % 2 = 0 AND id < 200") val snapshotAfterDelete = log.update() val afterDeleteFilesWithStats = snapshotAfterDelete.allFiles.collect() val afterDeleteFilesWithDVs = afterDeleteFilesWithStats.filter(_.deletionVector != null) val afterDeleteFiles = afterDeleteFilesWithStats.map(_.path) // Verify the expected no. of deletion vectors and deleted rows according to DV cardinality assert(afterDeleteFiles.length === numFiles) assert(afterDeleteFilesWithDVs.length === numFilesWithDVs) assert(afterDeleteFilesWithDVs.map(_.deletionVector.cardinality).sum == numDeletedRows) // Expect all DVs are written in one file assert( afterDeleteFilesWithDVs .map(_.deletionVector.absolutePath(new Path(path))) .toSet .size === 1) // Verify "tightBounds" is false for files that have DVs for (f <- afterDeleteFilesWithDVs) { assert(f.tightBounds.get === false) } // Verify all stats are the same except "tightBounds". // Drop "tightBounds" and convert the rest to JSON. val dropTightBounds: (AddFile => String) = _.stats.replaceAll("\"tightBounds\":(false|true)", "") val beforeStats = beforeDeleteFilesWithStats.map(dropTightBounds).sorted val afterStats = afterDeleteFilesWithStats.map(dropTightBounds).sorted assert(beforeStats === afterStats) // make sure the data file list is the same assert(beforeDeleteFiles === afterDeleteFiles) // Contents after the DELETE are as expected checkAnswer( spark.sql(s"SELECT * FROM $tableName"), Seq.range(0, 1000).filterNot(Seq.range(start = 0, end = 200, step = 2).contains(_)).toDF() ) } } } Seq("name", "id").foreach { mode => test(s"DELETE with DVs with column mapping mode=$mode") { withSQLConf("spark.databricks.delta.properties.defaults.columnMapping.mode" -> mode) { withTempDir { dirName => val path = dirName.getAbsolutePath val data = (0 until 50).map(x => (x % 10, x, s"foo${x % 5}")) data.toDF("part", "col1", "col2").write.format("delta").partitionBy( "part").save(path) val tableLog = DeltaLog.forTable(spark, path) enableDeletionVectorsInTable(tableLog, true) spark.sql(s"DELETE FROM delta.`$path` WHERE col1 = 2") checkAnswer(spark.sql(s"select * from delta.`$path` WHERE col1 = 2"), Seq()) verifyDVsExist(tableLog, 1) } } } test(s"variant types DELETE with DVs with column mapping mode=$mode") { withSQLConf("spark.databricks.delta.properties.defaults.columnMapping.mode" -> mode) { withTempDir { dirName => val path = dirName.getAbsolutePath val df = spark.range(0, 50).selectExpr( "id % 10 as part", "id", "parse_json(cast(id as string)) as v" ) df.write.format("delta").partitionBy("part").save(path) val tableLog = DeltaLog.forTable(spark, path) enableDeletionVectorsInTable(tableLog, true) spark.sql(s"DELETE FROM delta.`$path` WHERE v::int = 2") checkAnswer(spark.sql(s"select * from delta.`$path` WHERE v::int = 2"), Seq()) verifyDVsExist(tableLog, 1) } } } } test("DELETE with DVs - existing table already has DVs") { withSQLConf(DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS.key -> "true") { withTempDir { tempDir => val source = new File(table1Path) val target = new File(tempDir, "deleteTest") // Copy the source DV table to a temporary directory FileUtils.copyDirectory(source, target) val targetPath = s"delta.`${target.getAbsolutePath}`" val dataToRemove = Seq(1999, 299, 7, 87, 867, 456) val existingDVs = getFilesWithDeletionVectors(DeltaLog.forTable(spark, target)) spark.sql(s"DELETE FROM $targetPath WHERE value in (${dataToRemove.mkString(",")})") // Check new DVs are created val newDVs = getFilesWithDeletionVectors(DeltaLog.forTable(spark, target)) // expect the new DVs contain extra entries for the deleted rows. assert( existingDVs.map(_.deletionVector.cardinality).sum + dataToRemove.size === newDVs.map(_.deletionVector.cardinality).sum ) for (f <- newDVs) { assert(f.tightBounds.get === false) } // Check the data is valid val expectedTable1DataV5 = expectedTable1DataV4.filterNot(e => dataToRemove.contains(e)) checkAnswer(spark.sql(s"SELECT * FROM $targetPath"), expectedTable1DataV5.toDF()) } } } test("Metrics when deleting with DV") { withDeletionVectorsEnabled() { val tableName = "tbl" withTable(tableName) { spark.range(0, 10, 1, numPartitions = 2) .write.format("delta").saveAsTable(tableName) { // Delete one row from the first file, and the whole second file. val result = sql(s"DELETE FROM $tableName WHERE id >= 4") assert(result.collect() === Array(Row(6))) val opMetrics = DeltaMetricsUtils.getLastOperationMetrics(tableName) assert(opMetrics.getOrElse("numDeletedRows", -1) === 6) assert(opMetrics.getOrElse("numRemovedFiles", -1) === 1) assert(opMetrics.getOrElse("numDeletionVectorsAdded", -1) === 1) assert(opMetrics.getOrElse("numDeletionVectorsRemoved", -1) === 0) assert(opMetrics.getOrElse("numDeletionVectorsUpdated", -1) === 0) } { // Delete one row again. sql(s"DELETE FROM $tableName WHERE id = 3") val opMetrics = DeltaMetricsUtils.getLastOperationMetrics(tableName) assert(opMetrics.getOrElse("numDeletedRows", -1) === 1) assert(opMetrics.getOrElse("numRemovedFiles", -1) === 0) val initialNumDVs = 0 val numDVUpdated = 1 // An "updated" DV is "deleted" then "added" again. // We increment the count for "updated", "added", and "deleted". assert( opMetrics.getOrElse("numDeletionVectorsAdded", -1) === initialNumDVs + numDVUpdated) assert( opMetrics.getOrElse("numDeletionVectorsRemoved", -1) === initialNumDVs + numDVUpdated) assert( opMetrics.getOrElse("numDeletionVectorsUpdated", -1) === numDVUpdated) } { // Delete all renaming rows. sql(s"DELETE FROM $tableName WHERE id IN (0, 1, 2)") val opMetrics = DeltaMetricsUtils.getLastOperationMetrics(tableName) assert(opMetrics.getOrElse("numDeletedRows", -1) === 3) assert(opMetrics.getOrElse("numRemovedFiles", -1) === 1) assert(opMetrics.getOrElse("numDeletionVectorsAdded", -1) === 0) assert(opMetrics.getOrElse("numDeletionVectorsRemoved", -1) === 1) assert(opMetrics.getOrElse("numDeletionVectorsUpdated", -1) === 0) } } } } for(targetDVFileSize <- Seq(2, 200, 2000000)) { test(s"DELETE with DVs - packing multiple DVs into one file: target max DV file " + s"size=$targetDVFileSize") { withSQLConf( DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> "true", DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS.key -> "true", DeltaSQLConf.DELETION_VECTOR_PACKING_TARGET_SIZE.key -> targetDVFileSize.toString) { withTempDir { dirName => // Create table with 100 files of 2 rows each. val numFiles = 100 val path = dirName.getAbsolutePath spark.range(0, 200, step = 1, numPartitions = numFiles) .write.format("delta").save(path) val tableName = s"delta.`$path`" val beforeDeleteFiles = DeltaLog.forTable(spark, path) .unsafeVolatileSnapshot.allFiles.collect().map(_.path) val numFilesWithDVs = 10 val numDeletedRows = numFilesWithDVs * 1 spark.sql(s"DELETE FROM $tableName WHERE id % 2 = 0 AND id < 20") // Verify the expected number of AddFiles with DVs val allFiles = DeltaLog.forTable(spark, path).unsafeVolatileSnapshot.allFiles.collect() assert(allFiles.size === numFiles) val addFilesWithDV = allFiles.filter(_.deletionVector != null) assert(addFilesWithDV.size === numFilesWithDVs) assert(addFilesWithDV.map(_.deletionVector.cardinality).sum == numDeletedRows) val expectedDVFileCount = targetDVFileSize match { // Each AddFile will have its own DV file case 2 => numFilesWithDVs // Each DV size is about 34bytes according the latest format, plus 4 bytes for // checksum and another 4 bytes for data length. case 200 => (numFilesWithDVs.toDouble / (200 / (34 + 4 + 4)).toDouble).ceil.toInt // Expect all DVs in one file case 2000000 => 1 case default => throw new IllegalStateException(s"Unknown target DV file size: $default") } // Expect all DVs are written in one file assert( addFilesWithDV.map(_.deletionVector.absolutePath(new Path(path))).toSet.size === expectedDVFileCount) val afterDeleteFiles = allFiles.map(_.path) // make sure the data file list is the same assert(beforeDeleteFiles === afterDeleteFiles) // Contents after the DELETE are as expected checkAnswer( spark.sql(s"SELECT * FROM $tableName"), Seq.range(0, 200).filterNot( Seq.range(start = 0, end = 20, step = 2).contains(_)).toDF()) } } } } test("JOIN with DVs - self-join a table with DVs") { val tableDf = spark.read.format("delta").load(table2Path) val leftDf = tableDf.withColumn("key", col("value") % 2) val rightDf = tableDf.withColumn("key", col("value") % 2 + 1) checkAnswer( leftDf.as("left").join(rightDf.as("right"), "key").drop("key"), Seq(1, 3, 5, 7).flatMap(l => Seq(2, 4, 6, 8).map(r => (l, r))).toDF() ) } test("JOIN with DVs - non-DV table joins DV table") { val tableDf = spark.read.format("delta").load(table2Path) val tableDfV0 = spark.read.format("delta").option("versionAsOf", "0").load(table2Path) val leftDf = tableDf.withColumn("key", col("value") % 2) val rightDf = tableDfV0.withColumn("key", col("value") % 2 + 1) // Right has two more rows 0 and 9. 0 will be left in the join result. checkAnswer( leftDf.as("left").join(rightDf.as("right"), "key").drop("key"), Seq(1, 3, 5, 7).flatMap(l => Seq(0, 2, 4, 6, 8).map(r => (l, r))).toDF() ) } test("MERGE with DVs - merge into DV table") { withTempDir { tempDir => val source = new File(table1Path) val target = new File(tempDir, "mergeTest") FileUtils.copyDirectory(new File(table2Path), target) DeltaTable.forPath(spark, target.getAbsolutePath).as("target") .merge( spark.read.format("delta").load(source.getAbsolutePath).as("source"), "source.value = target.value") .whenMatched() .updateExpr(Map("value" -> "source.value + 10000")) .whenNotMatched() .insertExpr(Map("value" -> "source.value")) .execute() val snapshot = DeltaLog.forTable(spark, target).update() val allFiles = snapshot.allFiles.collect() val tombstones = snapshot.tombstones.collect() // DVs are removed for (ts <- tombstones) { assert(ts.deletionVector != null) } // target log should not contain DVs for (f <- allFiles) { assert(f.deletionVector == null) assert(f.tightBounds.get) } // Target table should contain "table2 records + 10000" and "table1 records \ table2 records". checkAnswer( spark.read.format("delta").load(target.getAbsolutePath), (expectedTable2DataV1.map(_ + 10000) ++ expectedTable1DataV4.filterNot(expectedTable2DataV1.contains)).toDF() ) } } test("UPDATE with DVs - update rewrite files with DVs") { withTempDir { tempDir => FileUtils.copyDirectory(new File(table2Path), tempDir) val deltaLog = DeltaLog.forTable(spark, tempDir) DeltaTable.forPath(spark, tempDir.getAbsolutePath) .update(col("value") === 1, Map("value" -> (col("value") + 1))) val snapshot = deltaLog.update() val allFiles = snapshot.allFiles.collect() val tombstones = snapshot.tombstones.collect() // DVs are removed for (ts <- tombstones) { assert(ts.deletionVector != null) } // target log should contain two files, one with and one without DV assert(allFiles.count(_.deletionVector != null) === 1) assert(allFiles.count(_.deletionVector == null) === 1) } } test("UPDATE with DVs - update deleted rows updates nothing") { withTempDir { tempDir => FileUtils.copyDirectory(new File(table2Path), tempDir) val deltaLog = DeltaLog.forTable(spark, tempDir) val snapshotBeforeUpdate = deltaLog.update() val allFilesBeforeUpdate = snapshotBeforeUpdate.allFiles.collect() DeltaTable.forPath(spark, tempDir.getAbsolutePath) .update(col("value") === 0, Map("value" -> (col("value") + 1))) val snapshot = deltaLog.update() val allFiles = snapshot.allFiles.collect() val tombstones = snapshot.tombstones.collect() // nothing changed assert(tombstones.length === 0) assert(allFiles === allFilesBeforeUpdate) checkAnswer( spark.read.format("delta").load(tempDir.getAbsolutePath), expectedTable2DataV1.toDF() ) } } test("INSERT + DELETE + MERGE + UPDATE with DVs") { withTempDir { tempDir => val path = tempDir.getAbsolutePath val deltaLog = DeltaLog.forTable(spark, path) def checkTableContents(rows: DataFrame): Unit = checkAnswer(sql(s"SELECT * FROM delta.`$path`"), rows) // Version 0: DV is enabled on table { withSQLConf( DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> "true") { spark.range(0, 10, 1, numPartitions = 2).write.format("delta").save(path) } val snapshot = deltaLog.update() assert(snapshot.protocol.isFeatureSupported(DeletionVectorsTableFeature)) for (f <- snapshot.allFiles.collect()) { assert(f.tightBounds.get) } } // Version 1: DELETE one row from each file { withSQLConf(DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS.key -> "true") { sql(s"DELETE FROM delta.`$path` WHERE id IN (1, 8)") } val (add, _) = getFileActionsInLastVersion(deltaLog) for (a <- add) { assert(a.deletionVector !== null) assert(a.deletionVector.cardinality === 1) assert(a.numPhysicalRecords.get === a.numLogicalRecords.get + 1) assert(a.tightBounds.get === false) } checkTableContents(Seq(0, 2, 3, 4, 5, 6, 7, 9).toDF()) } // Version 2: UPDATE one row in the first file { sql(s"UPDATE delta.`$path` SET id = -1 WHERE id = 0") val (added, removed) = getFileActionsInLastVersion(deltaLog) assert(added.length === 2) assert(removed.length === 1) // Added files must be two, one containing DV and one not assert(added.count(_.deletionVector != null) === 1) assert(added.count(_.deletionVector == null) === 1) // Removed files must contain DV for (r <- removed) { assert(r.deletionVector !== null) } checkTableContents(Seq(-1, 2, 3, 4, 5, 6, 7, 9).toDF()) } // Version 3: MERGE into the table using table2 { DeltaTable.forPath(spark, path).as("target") .merge( spark.read.format("delta").load(table2Path).as("source"), "source.value = target.id") .whenMatched() .updateExpr(Map("id" -> "source.value")) .whenNotMatchedBySource().delete().execute() val (added, removed) = getFileActionsInLastVersion(deltaLog) assert(removed.length === 3) for (a <- added) { assert(a.deletionVector === null) assert(a.tightBounds.get) } // Two of three removed files have DV assert(removed.count(_.deletionVector != null) === 2) // -1 and 9 are deleted by "when not matched by source" checkTableContents(Seq(2, 3, 4, 5, 6, 7).toDF()) } // Version 4: DELETE one row again { withSQLConf(DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS.key -> "true") { sql(s"DELETE FROM delta.`$path` WHERE id IN (4)") } val (add, _) = getFileActionsInLastVersion(deltaLog) for (a <- add) { assert(a.deletionVector !== null) assert(a.deletionVector.cardinality === 1) assert(a.numPhysicalRecords.get === a.numLogicalRecords.get + 1) assert(a.tightBounds.get === false) } checkTableContents(Seq(2, 3, 5, 6, 7).toDF()) } } } test("huge table: read from tables of 2B rows with existing DV of many zeros") { val canonicalTable5Path = new File(table5Path).getCanonicalPath val predicatePushDownEnabled = spark.conf.get(DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX) try { checkCountAndSum("value", table5Count, table5Sum, canonicalTable5Path) } catch { // TODO(SPARK-47731): Known issue. To be fixed in Spark 3.5 and/or Spark 4.0. case e: SparkException if predicatePushDownEnabled && (e.getMessage.contains("More than Int.MaxValue elements") || e.getCause.getMessage.contains("More than Int.MaxValue elements")) => () // Ignore. } } test("sanity check for non-incremental DV update") { val addFile = createTestAddFile() def bitmapToDvDescriptor(bitmap: RoaringBitmapArray): DeletionVectorDescriptor = { DeletionVectorDescriptor.inlineInLog( bitmap.serializeAsByteArray(RoaringBitmapArrayFormat.Portable), bitmap.cardinality) } val dv0 = bitmapToDvDescriptor(RoaringBitmapArray()) val dv1 = bitmapToDvDescriptor(RoaringBitmapArray(0L, 1L)) val dv2 = bitmapToDvDescriptor(RoaringBitmapArray(0L, 2L)) val dv3 = bitmapToDvDescriptor(RoaringBitmapArray(3L)) def removeRows(a: AddFile, dv: DeletionVectorDescriptor): (AddFile, RemoveFile) = { a.removeRows( deletionVector = dv, updateStats = true ) } // Adding an empty DV to a file is allowed. removeRows(addFile, dv0) // Updating with the same DV is allowed. val (addFileWithDV1, _) = removeRows(addFile, dv1) removeRows(addFileWithDV1, dv1) // Updating with a different DV with the same cardinality and different rows should not be // allowed, but is expensive to detect it. removeRows(addFileWithDV1, dv2) // Updating with a DV with lower cardinality should throw. for (dv <- Seq(dv0, dv3)) { assertThrows[DeltaChecksumException] { removeRows(addFileWithDV1, dv) } } } test("Check no resource leak when DV files are missing (table corrupted)") { withTempDir { tempDir => val source = new File(table2Path) val target = new File(tempDir, "resourceLeakTest") val targetPath = target.getAbsolutePath // Copy the source DV table to a temporary directory FileUtils.copyDirectory(source, target) val filesWithDvs = getFilesWithDeletionVectors(DeltaLog.forTable(spark, target)) assert(filesWithDvs.size > 0) deleteDVFile(targetPath, filesWithDvs(0)) val se = intercept[SparkException] { spark.sql(s"SELECT * FROM delta.`$targetPath`").collect() } assert(findIfResponsible[FileNotFoundException](se).nonEmpty, s"Expected a file not found exception as the cause, but got: [${se}]") } } test("absolute DV path with encoded special characters") { // This test uses hand-crafted path with special characters. // Do not test with a prefix that needs URL standard escaping. withTempDir(prefix = "spark") { dir => writeTableHavingSpecialCharInDVPath(dir, pathIsEncoded = true) checkAnswer( spark.read.format("delta").load(dir.getCanonicalPath), Seq(1, 3, 5, 7, 9).toDF()) } } test("absolute DV path with not-encoded special characters") { // This test uses hand-crafted path with special characters. // Do not test with a prefix that needs URL standard escaping. withTempDir(prefix = "spark") { dir => writeTableHavingSpecialCharInDVPath(dir, pathIsEncoded = false) val e = interceptWithUnwrapping[URISyntaxException] { spark.read.format("delta").load(dir.getCanonicalPath).collect() } assert(e.getMessage.contains("Malformed escape pair")) } } private sealed case class DeleteUsingDVWithResults( scale: String, sqlRule: String, count: Long, sum: Long) private val deleteUsingDvSmallScale = DeleteUsingDVWithResults( "small", "value = 1", table5CountByValues.filterKeys(_ != 1).values.sum, table5SumByValues.filterKeys(_ != 1).values.sum) private val deleteUsingDvMediumScale = DeleteUsingDVWithResults( "medium", "value > 10", table5CountByValues.filterKeys(_ <= 10).values.sum, table5SumByValues.filterKeys(_ <= 10).values.sum) private val deleteUsingDvLargeScale = DeleteUsingDVWithResults( "large", "value != 21", table5CountByValues(21), table5SumByValues(21)) // deleteUsingDvMediumScale and deleteUsingDvLargeScale runs too slow thus disabled. for (deleteSpec <- Seq(deleteUsingDvSmallScale)) { test( s"huge table: delete a ${deleteSpec.scale} number of rows from tables of 2B rows with DVs") { val predicatePushDownEnabled = spark.conf.get(DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX) withTempDir { dir => try { FileUtils.copyDirectory(new File(table5Path), dir) val log = DeltaLog.forTable(spark, dir) withDeletionVectorsEnabled() { sql(s"DELETE FROM delta.`${dir.getCanonicalPath}` WHERE ${deleteSpec.sqlRule}") } val (added, _) = getFileActionsInLastVersion(log) assert(added.forall(_.deletionVector != null)) checkCountAndSum("value", deleteSpec.count, deleteSpec.sum, dir.getCanonicalPath) } catch { // TODO(SPARK-47731): Known issue. To be fixed in Spark 3.5 and/or Spark 4.0. case e: SparkException if predicatePushDownEnabled && (e.getMessage.contains("More than Int.MaxValue elements") || e.getCause.getMessage.contains("More than Int.MaxValue elements")) => () // Ignore. } } } } private def checkCountAndSum(column: String, count: Long, sum: Long, tableDir: String): Unit = { checkAnswer( sql(s"SELECT count($column), sum($column) FROM delta.`$tableDir`"), Seq((count, sum)).toDF()) } private def assertPlanContains(queryDf: DataFrame, expected: String): Unit = { val optimizedPlan = queryDf.queryExecution.analyzed.toString() assert(optimizedPlan.contains(expected), s"Plan is missing `$expected`: $optimizedPlan") } } object DeletionVectorsSuite { val table1Path = "src/test/resources/delta/table-with-dv-large" // Table at version 0: contains [0, 2000) val expectedTable1DataV0 = Seq.range(0, 2000) // Table at version 1: removes rows with id = 0, 180, 300, 700, 1800 val v1Removed = Set(0, 180, 300, 700, 1800) val expectedTable1DataV1 = expectedTable1DataV0.filterNot(e => v1Removed.contains(e)) // Table at version 2: inserts rows with id = 300, 700 val v2Added = Set(300, 700) val expectedTable1DataV2 = expectedTable1DataV1 ++ v2Added // Table at version 3: removes rows with id = 300, 250, 350, 900, 1353, 1567, 1800 val v3Removed = Set(300, 250, 350, 900, 1353, 1567, 1800) val expectedTable1DataV3 = expectedTable1DataV2.filterNot(e => v3Removed.contains(e)) // Table at version 4: inserts rows with id = 900, 1567 val v4Added = Set(900, 1567) val expectedTable1DataV4 = expectedTable1DataV3 ++ v4Added val table2Path = "src/test/resources/delta/table-with-dv-small" // Table at version 0: contains 0 - 9 val expectedTable2DataV0 = Seq(0, 1, 2, 3, 4, 5, 6, 7, 8, 9) // Table at version 1: removes rows 0 and 9 val expectedTable2DataV1 = Seq(1, 2, 3, 4, 5, 6, 7, 8) val table3Path = "src/test/resources/delta/partitioned-table-with-dv-large" // Table at version 0: contains [0, 2000) val expectedTable3DataV0 = Seq.range(0, 2000) // Table at version 1: removes rows with id = (0, 180, 308, 225, 756, 1007, 1503) val table3V1Removed = Set(0, 180, 308, 225, 756, 1007, 1503) val expectedTable3DataV1 = expectedTable3DataV0.filterNot(e => table3V1Removed.contains(e)) // Table at version 2: inserts rows with id = 308, 756 val table3V2Added = Set(308, 756) val expectedTable3DataV2 = expectedTable3DataV1 ++ table3V2Added // Table at version 3: removes rows with id = (300, 257, 399, 786, 1353, 1567, 1800) val table3V3Removed = Set(300, 257, 399, 786, 1353, 1567, 1800) val expectedTable3DataV3 = expectedTable3DataV2.filterNot(e => table3V3Removed.contains(e)) // Table at version 4: inserts rows with id = 1353, 1567 val table3V4Added = Set(1353, 1567) val expectedTable3DataV4 = expectedTable3DataV3 ++ table3V4Added // Table with DV table feature as supported but no DVs val table4Path = "src/test/resources/delta/table-with-dv-feature-enabled" val expectedTable4DataV0 = Seq(1L) // Table with DV, (1<<31)+10=2147483658 rows in total including 2147484 rows deleted. Parquet is // generated by: // spark.range(0, (1L << 31) + 10, 1, numPartitions = 1) // .withColumn( // "value", // when($"id" % 1000 === 0, 1).otherwise(($"id" / 100000000).cast(IntegerType))) // All "id % 1000 = 0" rows are marked as deleted. // Column "value" ranges from 0 to 21. // 99900000 rows with values 0 to 20 each, and 47436174 rows with value 21. val table5Path = "src/test/resources/delta/table-with-dv-gigantic" val table5Count = 2145336174L val table5Sum = 21975159654L val table5CountByValues = (0 to 20).map(_ -> 99900000L).toMap + (21 -> 47436174L) val table5SumByValues = (0 to 20).map(v => v -> v * 99900000L).toMap + (21 -> 21 * 47436174L) // Generate a table with special characters in DV path. // Content of this table is range(0, 10) with all even numbers deleted. def writeTableHavingSpecialCharInDVPath(path: File, pathIsEncoded: Boolean): Unit = { val tableHavingSpecialCharInDVTemplate = "src/test/resources/delta/table-with-dv-special-char" FileUtils.copyDirectory(new File(tableHavingSpecialCharInDVTemplate), path) val fullPath = new File( path, if (pathIsEncoded) "folder&with%25special%20char" else "folder&with%special char") .getCanonicalPath val logJson = new File(path, "_delta_log/00000000000000000000.json") val logJsonContent = FileUtils.readFileToString(logJson, "UTF-8") val newLogJsonContent = logJsonContent.replace( "{{FOLDER_WITH_SPECIAL_CHAR}}", fullPath) FileUtils.delete(logJson) FileUtils.write(logJson, newLogJsonContent, "UTF-8") } } class DeletionVectorsWithPredicatePushdownSuite extends DeletionVectorsSuite { // ~4MBs. Should contain 2 row groups. val multiRowgroupTable = "multiRowgroupTable" val multiRowgroupTableRowsNum = 1000000 def assertParquetHasMultipleRowGroups(filePath: Path): Unit = { val parquetMetadata = ParquetFileReader.readFooter( hadoopConf, filePath, ParquetMetadataConverter.NO_FILTER) assert(parquetMetadata.getBlocks.size() > 1) } override def beforeAll(): Unit = { super.beforeAll() // 2MB rowgroups. hadoopConf().set("parquet.block.size", (2 * 1024 * 1024).toString) spark.range(0, multiRowgroupTableRowsNum, 1, 1).toDF("id") .write .option(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key, true.toString) .format("delta") .saveAsTable(multiRowgroupTable) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(multiRowgroupTable)) val files = deltaLog.update().allFiles.collect() assert(files.length === 1) assertParquetHasMultipleRowGroups(files.head.absolutePath(deltaLog)) spark.conf.set(DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX.key, "true") } override def afterAll(): Unit = { super.afterAll() sql(s"DROP TABLE IF EXISTS $multiRowgroupTable") } private def testPredicatePushDown( deletePredicates: Seq[String], selectPredicate: Option[String], expectedNumRows: Long, validationPredicate: String, vectorizedReaderEnabled: Boolean, readColumnarBatchAsRows: Boolean): Unit = { withTempDir { dir => // This forces the code generator to not use codegen. As a result, Spark sets options to get // rows instead of columnar batches from the Parquet reader. This allows to test the relevant // code path in DeltaParquetFileFormat. val codeGenMaxFields = if (readColumnarBatchAsRows) "0" else "100" withSQLConf( SQLConf.WHOLESTAGE_MAX_NUM_FIELDS.key -> codeGenMaxFields, SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorizedReaderEnabled.toString, SQLConf.FILES_MAX_PARTITION_BYTES.key -> "2MB") { sql(s"CREATE TABLE delta.`${dir.getCanonicalPath}` SHALLOW CLONE $multiRowgroupTable") val targetTable = io.delta.tables.DeltaTable.forPath(dir.getCanonicalPath) val deltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) val files = deltaLog.update().allFiles.collect() assert(files.length === 1) // Execute multiple delete statements. These require to reconsile the metadata column // between DV writing and scanning operations. deletePredicates.foreach(targetTable.delete) val targetTableDF = selectPredicate.map(targetTable.toDF.filter).getOrElse(targetTable.toDF) assertPredicatesArePushedDown(targetTableDF) // Make sure there are multiple row groups. assertParquetHasMultipleRowGroups(files.head.toPath) // Make sure we have 2 splits. assert(targetTableDF.rdd.partitions.size === 2) assert(targetTableDF.count() === expectedNumRows) // The deleted/filtered rows should not exist. assert(targetTableDF.filter(validationPredicate).count() === 0) } } } for { vectorizedReaderEnabled <- BOOLEAN_DOMAIN readColumnarBatchAsRows <- if (vectorizedReaderEnabled) BOOLEAN_DOMAIN else Seq(false) } test("PredicatePushdown: Single deletion at the first row group. " + s"vectorizedReaderEnabled: $vectorizedReaderEnabled " + s"readColumnarBatchAsRows: $readColumnarBatchAsRows") { testPredicatePushDown( deletePredicates = Seq("id == 100"), selectPredicate = None, expectedNumRows = multiRowgroupTableRowsNum - 1, validationPredicate = "id == 100", vectorizedReaderEnabled = vectorizedReaderEnabled, readColumnarBatchAsRows = readColumnarBatchAsRows) } for { vectorizedReaderEnabled <- BOOLEAN_DOMAIN readColumnarBatchAsRows <- if (vectorizedReaderEnabled) BOOLEAN_DOMAIN else Seq(false) } test("PredicatePushdown: Single deletion at the second row group. " + s"vectorizedReaderEnabled: $vectorizedReaderEnabled " + s"readColumnarBatchAsRows: $readColumnarBatchAsRows") { testPredicatePushDown( deletePredicates = Seq("id == 900000"), selectPredicate = None, expectedNumRows = multiRowgroupTableRowsNum - 1, // (rowId, Expected value). validationPredicate = "id == 900000", vectorizedReaderEnabled = vectorizedReaderEnabled, readColumnarBatchAsRows = readColumnarBatchAsRows) } for { vectorizedReaderEnabled <- BOOLEAN_DOMAIN readColumnarBatchAsRows <- if (vectorizedReaderEnabled) BOOLEAN_DOMAIN else Seq(false) } test("PredicatePushdown: Single delete statement with multiple ids. " + s"vectorizedReaderEnabled: $vectorizedReaderEnabled " + s"readColumnarBatchAsRows: $readColumnarBatchAsRows") { testPredicatePushDown( deletePredicates = Seq("id in (20, 200, 2000, 900000)"), selectPredicate = None, expectedNumRows = multiRowgroupTableRowsNum - 4, validationPredicate = "id in (20, 200, 2000, 900000)", vectorizedReaderEnabled = vectorizedReaderEnabled, readColumnarBatchAsRows = readColumnarBatchAsRows) } for { vectorizedReaderEnabled <- BOOLEAN_DOMAIN readColumnarBatchAsRows <- if (vectorizedReaderEnabled) BOOLEAN_DOMAIN else Seq(false) } test("PredicatePushdown: Multiple delete statements. " + s"vectorizedReaderEnabled: $vectorizedReaderEnabled " + s"readColumnarBatchAsRows: $readColumnarBatchAsRows") { testPredicatePushDown( deletePredicates = Seq("id = 20", "id = 200", "id = 2000", "id = 900000"), selectPredicate = None, expectedNumRows = multiRowgroupTableRowsNum - 4, validationPredicate = "id in (20, 200, 2000, 900000)", vectorizedReaderEnabled = vectorizedReaderEnabled, readColumnarBatchAsRows = readColumnarBatchAsRows) } for { vectorizedReaderEnabled <- BOOLEAN_DOMAIN readColumnarBatchAsRows <- if (vectorizedReaderEnabled) BOOLEAN_DOMAIN else Seq(false) } test("PredicatePushdown: Scan with predicates. " + s"vectorizedReaderEnabled: $vectorizedReaderEnabled " + s"readColumnarBatchAsRows: $readColumnarBatchAsRows") { testPredicatePushDown( deletePredicates = Seq("id = 20", "id = 2000"), selectPredicate = Some("id not in (200, 900000)"), expectedNumRows = multiRowgroupTableRowsNum - 4, validationPredicate = "id in (20, 200, 2000, 900000)", vectorizedReaderEnabled = vectorizedReaderEnabled, readColumnarBatchAsRows = readColumnarBatchAsRows) } for { vectorizedReaderEnabled <- BOOLEAN_DOMAIN readColumnarBatchAsRows <- if (vectorizedReaderEnabled) BOOLEAN_DOMAIN else Seq(false) } test("PredicatePushdown: Scan with predicates - no deletes. " + s"vectorizedReaderEnabled: $vectorizedReaderEnabled " + s"readColumnarBatchAsRows: $readColumnarBatchAsRows") { testPredicatePushDown( deletePredicates = Seq.empty, selectPredicate = Some("id not in (20, 200, 2000, 900000)"), expectedNumRows = multiRowgroupTableRowsNum - 4, validationPredicate = "id in (20, 200, 2000, 900000)", vectorizedReaderEnabled = vectorizedReaderEnabled, readColumnarBatchAsRows = readColumnarBatchAsRows) } test("Predicate pushdown works on queries that select metadata fields") { withTempDir { dir => withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> true.toString) { sql(s"CREATE TABLE delta.`${dir.getCanonicalPath}` SHALLOW CLONE $multiRowgroupTable") val targetTable = io.delta.tables.DeltaTable.forPath(dir.getCanonicalPath) targetTable.delete("id == 900000") val r1 = targetTable.toDF.select("id", "_metadata.row_index").count() assert(r1 === multiRowgroupTableRowsNum - 1) val r2 = targetTable.toDF.select("id", "_metadata.row_index", "_metadata.file_path").count() assert(r2 === multiRowgroupTableRowsNum - 1) val r3 = targetTable .toDF .select("id", "_metadata.file_block_start", "_metadata.file_path").count() assert(r3 === multiRowgroupTableRowsNum - 1) } } } private def assertPredicatesArePushedDown(df: DataFrame): Unit = { val scan = df.queryExecution.executedPlan.collectFirst { case scan: FileSourceScanExec => scan } assert(scan.map(_.dataFilters.nonEmpty).getOrElse(true)) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/RoaringBitmapArraySuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.deletionvectors import java.nio.{ByteBuffer, ByteOrder} import scala.collection.immutable.TreeSet import com.google.common.primitives.Ints import org.apache.spark.SparkFunSuite class RoaringBitmapArraySuite extends SparkFunSuite { final val BITMAP2_NUMBER = Int.MaxValue.toLong * 3L /** RoaringBitmap containers mostly use `Char` constants internally, so this is consistent. */ final val CONTAINER_BOUNDARY = Char.MaxValue.toLong + 1L final val BITMAP_BOUNDARY = 0xFFFFFFFFL + 1L private def testEquality(referenceResult: Seq[Long])( testOps: (RoaringBitmapArray => Unit)*): Unit = { val referenceBitmap = RoaringBitmapArray(referenceResult: _*) val testBitmap = RoaringBitmapArray() testOps.foreach(op => op(testBitmap)) assert(testBitmap === referenceBitmap) assert(testBitmap.## === referenceBitmap.##) assert(testBitmap.toArray === referenceBitmap.toArray) } test("equality") { testEquality(Seq(1))(_.add(1)) testEquality(Nil)(_.add(1), _.remove(1)) testEquality(Seq(1))(_.add(1), _.add(1)) testEquality(Nil)(_.add(1), _.add(1), _.remove(1)) testEquality(Nil)(_.add(1), _.remove(1), _.remove(1)) testEquality(Nil)(_.add(1), _.add(1), _.remove(1), _.remove(1)) testEquality(Seq(1))(_.add(1), _.remove(1), _.add(1)) testEquality(Nil)(_.add(1), _.remove(1), _.add(1), _.remove(1)) testEquality(Seq(BITMAP2_NUMBER))(_.add(BITMAP2_NUMBER)) testEquality(Nil)(_.add(BITMAP2_NUMBER), _.remove(BITMAP2_NUMBER)) testEquality(Seq(BITMAP2_NUMBER))(_.add(BITMAP2_NUMBER), _.add(BITMAP2_NUMBER)) testEquality(Nil)(_.add(BITMAP2_NUMBER), _.add(BITMAP2_NUMBER), _.remove(BITMAP2_NUMBER)) testEquality(Nil)(_.add(BITMAP2_NUMBER), _.remove(BITMAP2_NUMBER), _.remove(BITMAP2_NUMBER)) testEquality(Nil)( _.add(BITMAP2_NUMBER), _.add(BITMAP2_NUMBER), _.remove(BITMAP2_NUMBER), _.remove(BITMAP2_NUMBER)) testEquality(Seq(BITMAP2_NUMBER))( _.add(BITMAP2_NUMBER), _.remove(BITMAP2_NUMBER), _.add(BITMAP2_NUMBER)) testEquality(Nil)( _.add(BITMAP2_NUMBER), _.remove(BITMAP2_NUMBER), _.add(BITMAP2_NUMBER), _.remove(BITMAP2_NUMBER)) testEquality(Seq(1, BITMAP2_NUMBER))(_.add(1), _.add(BITMAP2_NUMBER)) testEquality(Seq(BITMAP2_NUMBER))(_.add(1), _.add(BITMAP2_NUMBER), _.remove(1)) testEquality(Seq(1, BITMAP2_NUMBER))(_.add(BITMAP2_NUMBER), _.add(1)) testEquality(Seq(BITMAP2_NUMBER))(_.add(BITMAP2_NUMBER), _.add(1), _.remove(1)) testEquality(Seq(BITMAP2_NUMBER))(_.add(1), _.remove(1), _.add(BITMAP2_NUMBER)) testEquality(Nil)(_.add(1), _.remove(1), _.add(BITMAP2_NUMBER), _.remove(BITMAP2_NUMBER)) testEquality(Nil)(_.add(1), _.add(BITMAP2_NUMBER), _.remove(1), _.remove(BITMAP2_NUMBER)) testEquality(Nil)(_.add(BITMAP2_NUMBER), _.add(1), _.remove(1), _.remove(BITMAP2_NUMBER)) testEquality(Nil)(_.add(BITMAP2_NUMBER), _.add(1), _.remove(BITMAP2_NUMBER), _.remove(1)) val denseSequence = 1L to (3L * CONTAINER_BOUNDARY) def addAll(v: Long): RoaringBitmapArray => Unit = rb => rb.add(v) testEquality(denseSequence)(denseSequence.map(addAll): _*) testEquality(denseSequence)(denseSequence.reverse.map(addAll): _*) val sparseSequence = 1L to BITMAP2_NUMBER by CONTAINER_BOUNDARY testEquality(sparseSequence)(sparseSequence.map(addAll): _*) testEquality(sparseSequence)(sparseSequence.reverse.map(addAll): _*) } /** * A [[RoaringBitmapArray]] that contains all 3 container types * in two [[org.roaringbitmap.RoaringBitmap]] instances. */ lazy val allContainerTypesBitmap: RoaringBitmapArray = { val bitmap = RoaringBitmapArray() // RoaringBitmap 1 Container 1 (Array) bitmap.addAll(1L, 17L, 63000L, CONTAINER_BOUNDARY - 1) // RoaringBitmap 1 Container 2 (RLE) bitmap.addRange((CONTAINER_BOUNDARY + 500L) until (CONTAINER_BOUNDARY + 1200L)) // RoaringBitmap 1 Container 3 (Bitset) bitmap.addRange((2L * CONTAINER_BOUNDARY) until (3L * CONTAINER_BOUNDARY - 1L) by 3L) // RoaringBitmap 2 Container 1 (Array) bitmap.addAll( BITMAP_BOUNDARY, BITMAP_BOUNDARY + 17L, BITMAP_BOUNDARY + 63000L, BITMAP_BOUNDARY + CONTAINER_BOUNDARY - 1) // RoaringBitmap 2 Container 2 (RLE) bitmap.addRange((BITMAP_BOUNDARY + CONTAINER_BOUNDARY + 500L) until (BITMAP_BOUNDARY + CONTAINER_BOUNDARY + 1200L)) // RoaringBitmap 2 Container 3 (Bitset) bitmap.addRange((BITMAP_BOUNDARY + 2L * CONTAINER_BOUNDARY) until (BITMAP_BOUNDARY + 3L * CONTAINER_BOUNDARY - 1L) by 3L) // Check that RLE containers are actually created. assert(bitmap.runOptimize()) bitmap } for (serializationFormat <- RoaringBitmapArrayFormat.values) { test(s"serialization - $serializationFormat") { checkSerializeDeserialize(RoaringBitmapArray(), serializationFormat) checkSerializeDeserialize(RoaringBitmapArray(1L), serializationFormat) checkSerializeDeserialize(RoaringBitmapArray(BITMAP2_NUMBER), serializationFormat) checkSerializeDeserialize(RoaringBitmapArray(1L, BITMAP2_NUMBER), serializationFormat) checkSerializeDeserialize(allContainerTypesBitmap, serializationFormat) } } private def checkSerializeDeserialize( input: RoaringBitmapArray, format: RoaringBitmapArrayFormat.Value): Unit = { val serializedSize = Ints.checkedCast(input.serializedSizeInBytes(format)) val buffer = ByteBuffer.allocate(serializedSize).order(ByteOrder.LITTLE_ENDIAN) input.serialize(buffer, format) val output = RoaringBitmapArray() buffer.flip() output.deserialize(buffer) assert(input === output) } for (serializationFormat <- RoaringBitmapArrayFormat.values) { test( s"serialization and deserialization with big endian buffers throws - $serializationFormat") { val roaringBitmapArray = RoaringBitmapArray(1L) val bigEndianBuffer = ByteBuffer .allocate(roaringBitmapArray.serializedSizeInBytes(serializationFormat).toInt) .order(ByteOrder.BIG_ENDIAN) assertThrows[IllegalArgumentException] { roaringBitmapArray.serialize(bigEndianBuffer, serializationFormat) } assertThrows[IllegalArgumentException] { roaringBitmapArray.deserialize(bigEndianBuffer) } } } test("empty") { val bitmap = RoaringBitmapArray() assert(bitmap.isEmpty) assert(bitmap.cardinality === 0L) assert(!bitmap.contains(0L)) assert(bitmap.toArray === Array.empty[Long]) var hadValue = false bitmap.forEach(_ => hadValue = true) assert(!hadValue) } test("special values") { testSpecialValue(0L) testSpecialValue(Int.MaxValue.toLong) testSpecialValue(CONTAINER_BOUNDARY - 1L) testSpecialValue(CONTAINER_BOUNDARY) testSpecialValue(BITMAP_BOUNDARY - 1L) testSpecialValue(BITMAP_BOUNDARY) testSpecialValue(3L * BITMAP_BOUNDARY + 42L) } private def testSpecialValue(value: Long): Unit = { val bitmap = RoaringBitmapArray(value) assert(bitmap.cardinality === 1L) assert(bitmap.contains(value)) assert(bitmap.toArray === Array(value)) var valueCount = 0 bitmap.forEach { v => valueCount += 1 assert(v === value) } assert(valueCount === 1) bitmap.remove(value) assert(!bitmap.contains(value)) assert(bitmap.cardinality === 0L) } test("negative numbers") { assertThrows[IllegalArgumentException] { val bitmap = RoaringBitmapArray() bitmap.add(-1L) } assertThrows[IllegalArgumentException] { RoaringBitmapArray(-1L) } assertThrows[IllegalArgumentException] { val bitmap = RoaringBitmapArray(1L) bitmap.remove(-1L) } assertThrows[IllegalArgumentException] { val bitmap = RoaringBitmapArray() bitmap.add(Long.MaxValue) } assertThrows[IllegalArgumentException] { RoaringBitmapArray(Long.MaxValue) } assertThrows[IllegalArgumentException] { val bitmap = RoaringBitmapArray(1L) bitmap.remove(Long.MaxValue) } assertThrows[IllegalArgumentException] { val bitmap = RoaringBitmapArray() bitmap.addAll(-1L, 1L) } assertThrows[IllegalArgumentException] { val bitmap = RoaringBitmapArray() bitmap.addRange(-3 to 1) } assertThrows[IllegalArgumentException] { val bitmap = RoaringBitmapArray() bitmap.addRange(-3L to 1L) } } private def testContainsButNoSimilarValues(value: Long, bitmap: RoaringBitmapArray): Unit = { assert(bitmap.contains(value)) for (i <- 1 to 3) { assert(!bitmap.contains(value + i * CONTAINER_BOUNDARY)) assert(!bitmap.contains(value + i * BITMAP_BOUNDARY)) } } test("small integers") { val bitmap = RoaringBitmapArray( 3L, 4L, CONTAINER_BOUNDARY - 1L, CONTAINER_BOUNDARY, Int.MaxValue.toLong) assert(bitmap.cardinality === 5L) testContainsButNoSimilarValues(3L, bitmap) testContainsButNoSimilarValues(4L, bitmap) testContainsButNoSimilarValues(CONTAINER_BOUNDARY - 1L, bitmap) testContainsButNoSimilarValues(CONTAINER_BOUNDARY, bitmap) testContainsButNoSimilarValues(Int.MaxValue.toLong, bitmap) assert(bitmap.toArray === Array(3L, 4L, CONTAINER_BOUNDARY - 1L, CONTAINER_BOUNDARY, Int.MaxValue.toLong)) var values: List[Long] = Nil bitmap.forEach { value => values ::= value } assert(values.reverse === List(3L, 4L, CONTAINER_BOUNDARY - 1L, CONTAINER_BOUNDARY, Int.MaxValue.toLong)) bitmap.remove(CONTAINER_BOUNDARY) assert(!bitmap.contains(CONTAINER_BOUNDARY)) assert(bitmap.cardinality === 4L) testContainsButNoSimilarValues(3L, bitmap) testContainsButNoSimilarValues(4L, bitmap) testContainsButNoSimilarValues(CONTAINER_BOUNDARY - 1L, bitmap) testContainsButNoSimilarValues(Int.MaxValue.toLong, bitmap) } test("large integers") { val container1Number = Int.MaxValue.toLong + 1L val container3Number = 2 * BITMAP_BOUNDARY + 1L val bitmap = RoaringBitmapArray( 3L, 4L, container1Number, BITMAP_BOUNDARY, BITMAP2_NUMBER, container3Number) assert(bitmap.cardinality === 6L) testContainsButNoSimilarValues(3L, bitmap) testContainsButNoSimilarValues(4L, bitmap) testContainsButNoSimilarValues(container1Number, bitmap) testContainsButNoSimilarValues(BITMAP_BOUNDARY, bitmap) testContainsButNoSimilarValues(BITMAP2_NUMBER, bitmap) testContainsButNoSimilarValues(container3Number, bitmap) assert(bitmap.toArray === Array(3L, 4L, container1Number, BITMAP_BOUNDARY, BITMAP2_NUMBER, container3Number)) var values: List[Long] = Nil bitmap.forEach { value => values ::= value } assert(values.reverse === List(3L, 4L, container1Number, BITMAP_BOUNDARY, BITMAP2_NUMBER, container3Number)) bitmap.remove(BITMAP_BOUNDARY) assert(!bitmap.contains(BITMAP_BOUNDARY)) assert(bitmap.cardinality === 5L) testContainsButNoSimilarValues(3L, bitmap) testContainsButNoSimilarValues(4L, bitmap) testContainsButNoSimilarValues(container1Number, bitmap) testContainsButNoSimilarValues(BITMAP2_NUMBER, bitmap) testContainsButNoSimilarValues(container3Number, bitmap) } test("add/remove round-trip") { // Single value in the second bitmap val bitmap = RoaringBitmapArray(BITMAP2_NUMBER) assert(bitmap.contains(BITMAP2_NUMBER)) bitmap.remove(BITMAP2_NUMBER) assert(!bitmap.contains(BITMAP2_NUMBER)) bitmap.add(BITMAP2_NUMBER) assert(bitmap.contains(BITMAP2_NUMBER)) // Two values in two bitmaps bitmap.add(CONTAINER_BOUNDARY) assert(bitmap.contains(CONTAINER_BOUNDARY)) assert(bitmap.contains(BITMAP2_NUMBER)) bitmap.remove(CONTAINER_BOUNDARY) assert(!bitmap.contains(CONTAINER_BOUNDARY)) assert(bitmap.contains(BITMAP2_NUMBER)) bitmap.add(CONTAINER_BOUNDARY) assert(bitmap.contains(CONTAINER_BOUNDARY)) assert(bitmap.contains(BITMAP2_NUMBER)) } test("or") { testOr(left = TreeSet.empty, right = TreeSet.empty) testOr(left = TreeSet(1L), right = TreeSet.empty) testOr(left = TreeSet.empty, right = TreeSet(1L)) testOr(left = TreeSet(0L, CONTAINER_BOUNDARY), right = TreeSet(1L, BITMAP_BOUNDARY - 1L)) testOr( left = TreeSet(0L, CONTAINER_BOUNDARY, BITMAP2_NUMBER), right = TreeSet(1L, BITMAP_BOUNDARY - 1L)) testOr( left = TreeSet(0L, CONTAINER_BOUNDARY), right = TreeSet(1L, BITMAP_BOUNDARY - 1L, BITMAP2_NUMBER)) } private def testOr(left: TreeSet[Long], right: TreeSet[Long]): Unit = { val leftBitmap = RoaringBitmapArray(left.toSeq: _*) val rightBitmap = RoaringBitmapArray(right.toSeq: _*) val expected = left.union(right).toSeq leftBitmap.or(rightBitmap) assert(leftBitmap.toArray.toSeq === expected) } test("andNot") { testAndNot(left = TreeSet.empty, right = TreeSet.empty) testAndNot(left = TreeSet(1L), right = TreeSet.empty) testAndNot(left = TreeSet.empty, right = TreeSet(1L)) testAndNot(left = TreeSet(0L, CONTAINER_BOUNDARY), right = TreeSet(1L, BITMAP_BOUNDARY - 1L)) testAndNot( left = TreeSet(0L, CONTAINER_BOUNDARY, BITMAP2_NUMBER), right = TreeSet(1L, BITMAP_BOUNDARY - 1L)) testAndNot( left = TreeSet(0L, CONTAINER_BOUNDARY), right = TreeSet(1L, BITMAP_BOUNDARY - 1L, BITMAP2_NUMBER)) } private def testAndNot(left: TreeSet[Long], right: TreeSet[Long]): Unit = { val leftBitmap = RoaringBitmapArray() left.foreach(leftBitmap.add) val rightBitmap = RoaringBitmapArray() right.foreach(rightBitmap.add) val expected = left.diff(right).toArray leftBitmap.andNot(rightBitmap) assert(leftBitmap.toArray === expected) } test("and") { // Empty result testAnd(left = TreeSet.empty, right = TreeSet.empty) testAnd(left = TreeSet.empty, right = TreeSet(1L)) testAnd(left = TreeSet.empty, right = TreeSet(1L, BITMAP_BOUNDARY - 1L)) testAnd(left = TreeSet.empty, right = TreeSet(0L, CONTAINER_BOUNDARY, BITMAP2_NUMBER)) testAnd(left = TreeSet(1L), right = TreeSet.empty) testAnd(left = TreeSet(1L), right = TreeSet(BITMAP_BOUNDARY)) testAnd(left = TreeSet(1L), right = TreeSet(CONTAINER_BOUNDARY)) testAnd(left = TreeSet(1L, BITMAP_BOUNDARY - 1L), right = TreeSet.empty) testAnd(left = TreeSet(0L, CONTAINER_BOUNDARY, BITMAP2_NUMBER), right = TreeSet.empty) testAnd( left = TreeSet(0L, CONTAINER_BOUNDARY, BITMAP2_NUMBER), right = TreeSet(1L, BITMAP_BOUNDARY - 1L)) testAnd( left = TreeSet(0L, CONTAINER_BOUNDARY), right = TreeSet(1L, BITMAP_BOUNDARY - 1L, BITMAP2_NUMBER)) // Non empty result testAnd(left = TreeSet(0L, 5L, 10L), right = TreeSet(5L, 15L)) testAnd( left = TreeSet(0L, CONTAINER_BOUNDARY, BITMAP2_NUMBER), right = TreeSet(1L, BITMAP2_NUMBER)) testAnd( left = TreeSet(1L, BITMAP_BOUNDARY, CONTAINER_BOUNDARY), right = TreeSet(1L, BITMAP_BOUNDARY, CONTAINER_BOUNDARY)) } private def testAnd(left: TreeSet[Long], right: TreeSet[Long]): Unit = { val leftBitmap = RoaringBitmapArray() leftBitmap.addAll(left.toSeq: _*) val rightBitmap = RoaringBitmapArray() rightBitmap.addAll(right.toSeq: _*) leftBitmap.and(rightBitmap) val expected = left.intersect(right) assert(leftBitmap.toArray === expected.toArray) } test("clear") { testEquality(Nil)(_.add(1), _.clear()) testEquality(Nil)(_.add(1), _.add(1), _.clear()) testEquality(Nil)(_.add(1), _.clear(), _.clear()) testEquality(Nil)(_.add(1), _.add(1), _.clear(), _.clear()) testEquality(Seq(1))(_.add(1), _.clear(), _.add(1)) testEquality(Nil)(_.add(1), _.clear(), _.add(1), _.clear()) testEquality(Nil)(_.add(BITMAP2_NUMBER), _.clear()) testEquality(Nil)(_.add(BITMAP2_NUMBER), _.add(BITMAP2_NUMBER), _.clear()) testEquality(Nil)(_.add(BITMAP2_NUMBER), _.clear(), _.clear()) testEquality(Nil)(_.add(BITMAP2_NUMBER), _.add(BITMAP2_NUMBER), _.clear(), _.clear()) testEquality(Seq(BITMAP2_NUMBER))(_.add(BITMAP2_NUMBER), _.clear(), _.add(BITMAP2_NUMBER)) testEquality(Nil)(_.add(BITMAP2_NUMBER), _.clear(), _.add(BITMAP2_NUMBER), _.clear()) testEquality(Nil)(_.add(1), _.add(BITMAP2_NUMBER), _.clear()) testEquality(Nil)(_.add(BITMAP2_NUMBER), _.add(1), _.clear()) testEquality(Seq(BITMAP2_NUMBER))(_.add(1), _.clear(), _.add(BITMAP2_NUMBER)) testEquality(Nil)(_.add(1), _.clear(), _.add(BITMAP2_NUMBER), _.clear()) testEquality(Nil)(_.add(1), _.add(BITMAP2_NUMBER), _.clear(), _.clear()) val denseSequence = 1L to (3L * CONTAINER_BOUNDARY) testEquality(Nil)(_.addAll(denseSequence: _*), _.clear()) val sparseSequence = 1L to BITMAP2_NUMBER by CONTAINER_BOUNDARY testEquality(Nil)(_.addAll(sparseSequence: _*), _.clear()) } test("bulk adds") { def testArrayEquality(referenceResult: Seq[Long], command: RoaringBitmapArray => Unit): Unit = { val testBitmap = RoaringBitmapArray() command(testBitmap) assert(testBitmap.toArray.toSeq === referenceResult) } val bitmap = RoaringBitmapArray(1L, 5L, CONTAINER_BOUNDARY, BITMAP_BOUNDARY) assert(bitmap.toArray.toSeq === Seq(1L, 5L, CONTAINER_BOUNDARY, BITMAP_BOUNDARY)) testArrayEquality( referenceResult = Seq(1L, 5L, CONTAINER_BOUNDARY, BITMAP_BOUNDARY), command = _.addAll(1L, 5L, CONTAINER_BOUNDARY, BITMAP_BOUNDARY)) testArrayEquality( referenceResult = (CONTAINER_BOUNDARY - 5L) to (CONTAINER_BOUNDARY + 5L), command = _.addRange((CONTAINER_BOUNDARY - 5L) to (CONTAINER_BOUNDARY + 5L))) testArrayEquality( referenceResult = (CONTAINER_BOUNDARY - 5L) to (CONTAINER_BOUNDARY + 5L) by 3L, command = _.addRange((CONTAINER_BOUNDARY - 5L) to (CONTAINER_BOUNDARY + 5L) by 3L)) // Int ranges call a different method. testArrayEquality( referenceResult = (CONTAINER_BOUNDARY - 5L) to (CONTAINER_BOUNDARY + 5L), command = _.addRange((CONTAINER_BOUNDARY - 5L).toInt to (CONTAINER_BOUNDARY + 5L).toInt)) testArrayEquality( referenceResult = (CONTAINER_BOUNDARY - 5L) to (CONTAINER_BOUNDARY + 5L) by 3L, command = _.addRange((CONTAINER_BOUNDARY - 5L).toInt to (CONTAINER_BOUNDARY + 5L).toInt by 3)) testArrayEquality( referenceResult = (BITMAP_BOUNDARY - 5L) to BITMAP_BOUNDARY, command = _.addRange((BITMAP_BOUNDARY - 5L) to BITMAP_BOUNDARY)) testArrayEquality( referenceResult = (BITMAP_BOUNDARY - 5L) to (BITMAP_BOUNDARY + 5L), command = _.addRange((BITMAP_BOUNDARY - 5L) to (BITMAP_BOUNDARY + 5L))) testArrayEquality( referenceResult = BITMAP_BOUNDARY to (BITMAP_BOUNDARY + 5L), command = _.addRange(BITMAP_BOUNDARY to (BITMAP_BOUNDARY + 5L))) } test("large cardinality") { val bitmap = RoaringBitmapArray() // We can't produce ranges in Scala whose lengths would be greater than Int.MaxValue // so we add them in stages of Int.MaxValue / 2 instead. for (index <- 0 until 6) { val start = index.toLong * Int.MaxValue.toLong / 2L val end = (index.toLong + 1L) * Int.MaxValue.toLong / 2L bitmap.addRange(start until end) } assert(bitmap.cardinality === (3L * Int.MaxValue.toLong)) for (index <- 0 until 6) { val start = index.toLong * Int.MaxValue.toLong / 2L val end = (index.toLong + 1L) * Int.MaxValue.toLong / 2L val stride = 1023 for (pos <- start until end by stride) { assert(bitmap.contains(pos)) } } assert(!bitmap.contains(3L * Int.MaxValue.toLong)) assert(!bitmap.contains(3L * Int.MaxValue.toLong + 42L)) } test("first/last") { { val bitmap = RoaringBitmapArray() assert(bitmap.first.isEmpty) assert(bitmap.last.isEmpty) } // Single value bitmaps. val valuesOfInterest = Seq(0L, 1L, 64L, CONTAINER_BOUNDARY, BITMAP_BOUNDARY, BITMAP2_NUMBER) for (v <- valuesOfInterest) { val bitmap = RoaringBitmapArray(v) assert(bitmap.first === Some(v)) assert(bitmap.last === Some(v)) } // Two value bitmaps. for { start <- valuesOfInterest end <- valuesOfInterest if start < end } { val bitmap = RoaringBitmapArray(start, end) assert(bitmap.first === Some(start)) assert(bitmap.last === Some(end)) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/RowIndexMarkingFiltersSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.deletionvectors import org.apache.spark.sql.delta.RowIndexFilter import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor._ import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore._ import org.apache.spark.sql.delta.util.PathWithFileSystem import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.sql.QueryTest import org.apache.spark.sql.execution.vectorized.{OnHeapColumnVector, WritableColumnVector} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.ByteType import org.apache.spark.util.Utils class RowIndexMarkingFiltersSuite extends QueryTest with SharedSparkSession { test("empty deletion vector (drop filter)") { val rowIndexFilter = DropMarkedRowsFilter.createInstance( DeletionVectorDescriptor.EMPTY, newHadoopConf, tablePath = None) assert(getMarked(rowIndexFilter, start = 0, end = 20) === Seq.empty) assert(getMarked(rowIndexFilter, start = 20, end = 200) === Seq.empty) assert(getMarked(rowIndexFilter, start = 200, end = 2000) === Seq.empty) } test("empty deletion vector (keep filter)") { val rowIndexFilter = KeepMarkedRowsFilter.createInstance( DeletionVectorDescriptor.EMPTY, newHadoopConf, tablePath = None) assert(getMarked(rowIndexFilter, start = 0, end = 20) === 0.until(20)) assert(getMarked(rowIndexFilter, start = 20, end = 200) === 20.until(200)) assert(getMarked(rowIndexFilter, start = 200, end = 2000) === 200.until(2000)) } private val filtersToBeTested = Seq((DropMarkedRowsFilter, "drop"), (KeepMarkedRowsFilter, "keep")) for { (filterType, filterName) <- filtersToBeTested isInline <- BOOLEAN_DOMAIN } { test(s"deletion vector single row marked (isInline=$isInline) ($filterName filter)") { withTempDir { tableDir => val tablePath = unescapedStringToPath(tableDir.toString) val dv = createDV(isInline, tablePath, 25) val rowIndexFilter = filterType.createInstance(dv, newHadoopConf, Some(tablePath)) def correctValues(range: Seq[Long]): Seq[Long] = filterName match { case "drop" => range.filter(_ == 25) case "keep" => range.filterNot(_ == 25) case _ => throw new RuntimeException("unreachable code reached") } for ((start, end) <- Seq((0, 20), (20, 35), (35, 325))) { val actual = getMarked(rowIndexFilter, start, end) val correct = correctValues(start.toLong.until(end)) assert(actual === correct) } } } } for { (filterType, filterName) <- filtersToBeTested isInline <- BOOLEAN_DOMAIN } { test(s"deletion vector with multiple rows marked (isInline=$isInline) ($filterName filter)") { withTempDir { tableDir => val tablePath = unescapedStringToPath(tableDir.toString) val markedRows = Seq[Long](0, 25, 35, 2000, 50000) val dv = createDV(isInline, tablePath, markedRows: _*) val rowIndexFilter = filterType.createInstance(dv, newHadoopConf, Some(tablePath)) def correctValues(range: Seq[Long]): Seq[Long] = filterName match { case "drop" => range.filter(markedRows.contains(_)) case "keep" => range.filterNot(markedRows.contains(_)) case _ => throw new RuntimeException("unreachable code reached") } for ((start, end) <- Seq( (0, 20), (20, 35), (35, 325), (325, 1000), (1000, 60000), (60000, 800000))) { val actual = getMarked(rowIndexFilter, start, end) val correct = correctValues(start.toLong.until(end)) assert(actual === correct) } } } } private def newBatch(capacity: Int): WritableColumnVector = new OnHeapColumnVector(capacity, ByteType) protected def newHadoopConf: Configuration = { // scalastyle:off deltahadoopconfiguration spark.sessionState.newHadoopConf() // scalastyle:on deltahadoopconfiguration } /** * Helper method that creates DV with the given deleted row ids and returns * a [[DeletionVectorDescriptor]]. DV created can be an in-line or on disk */ protected def createDV( isInline: Boolean, tablePath: Path, markedRows: Long*): DeletionVectorDescriptor = { val bitmap = RoaringBitmapArray(markedRows: _*) val serializedBitmap = bitmap.serializeAsByteArray(RoaringBitmapArrayFormat.Portable) val cardinality = markedRows.size if (isInline) { inlineInLog(serializedBitmap, cardinality) } else { val tableWithFS = PathWithFileSystem.withConf(tablePath, newHadoopConf).makeQualified() val dvPath = dvStore.generateUniqueNameInTable(tableWithFS) val dvRange = Utils.tryWithResource(dvStore.createWriter(dvPath)) { writer => writer.write(serializedBitmap) } onDiskWithAbsolutePath( pathToEscapedString(dvPath.path), dvRange.length, cardinality, Some(dvRange.offset)) } } /** Evaluate the given row index filter instance and return sequence of marked rows indexes */ protected def getMarked(rowIndexFilter: RowIndexFilter, start: Long, end: Long): Seq[Long] = { val batchSize = (end - start + 1).toInt val batch = newBatch(batchSize) rowIndexFilter.materializeIntoVector(start, end, batch) batch.getBytes(0, batchSize).toSeq .zip(Seq.range(start, end)) .filter(_._1 == RowIndexFilter.DROP_ROW_VALUE) // filter out marked rows .map(_._2) // select only the row id .toSeq } lazy val dvStore: DeletionVectorStore = DeletionVectorStore.createInstance(newHadoopConf) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/expressions/DecodeNestedZ85EncodedVariantSuite.scala ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.expressions import java.util.Arrays import org.apache.spark.sql.{Column, QueryTest, Row} import org.apache.spark.sql.catalyst.expressions.variant.VariantExpressionEvalUtils import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.util.DeltaStatsJsonUtils import org.apache.spark.sql.functions.{col, from_json} import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructField, StructType, VariantType} import org.apache.spark.types.variant.Variant import org.apache.spark.unsafe.types.{UTF8String, VariantVal} class DecodeNestedZ85EncodedVariantSuite extends QueryTest with DeltaSQLCommandTest { test("RoundTrip alternateVariantEncoding Z85") { val jsonValues = Seq( "21", "1021", "-29183652", "[1, null, true, {\"a\": 1}]", "{\"key1\": \"value_1\", \"key_2\": [\"value2\", 1385731029.1236421], \"key3\": false}" ) jsonValues.foreach { json => val inputVariant = VariantExpressionEvalUtils.parseJson(UTF8String.fromString(json)) val variant = new Variant(inputVariant.getValue, inputVariant.getMetadata) // Encode as Z85 val z85 = DeltaStatsJsonUtils.encodeVariantAsZ85(variant) // Create a DataFrame with the Z85 string val df = spark.range(1).selectExpr(s"""'{"v":"$z85"}' as z85_string""") // Parse as JSON (this creates a VariantVal containing the Z85 string) val statsSchema = StructType(Seq(StructField("v", VariantType))) val parsedDf = df.withColumn("parsed", from_json(col("z85_string"), statsSchema)) // Apply DecodeNestedZ85EncodedVariant val decodedDf = parsedDf.withColumn( "decoded", Column(DecodeNestedZ85EncodedVariant(col("parsed").expr)) ) // Extract the decoded variant and verify val result = decodedDf.select("decoded.v").head().get(0) val decodedVariant = result.asInstanceOf[VariantVal] assert(Arrays.equals(inputVariant.getMetadata, decodedVariant.getMetadata), s"Metadata mismatch for JSON: $json") assert(Arrays.equals(inputVariant.getValue, decodedVariant.getValue), s"Value mismatch for JSON: $json") } } test("DecodeNestedZ85EncodedVariantSuite with nested struct and mixed types") { val json1 = "{\"id\": 100, \"name\": \"test\"}" val inputVariant1 = VariantExpressionEvalUtils.parseJson(UTF8String.fromString(json1)) val variant1 = new Variant(inputVariant1.getValue, inputVariant1.getMetadata) val z85_1 = DeltaStatsJsonUtils.encodeVariantAsZ85(variant1) val json2 = "{\"count\": 42}" val inputVariant2 = VariantExpressionEvalUtils.parseJson(UTF8String.fromString(json2)) val variant2 = new Variant(inputVariant2.getValue, inputVariant2.getMetadata) val z85_2 = DeltaStatsJsonUtils.encodeVariantAsZ85(variant2) // Create stats schema with nested variant, non-variant fields, and nullable variant val statsSchema = StructType(Seq( StructField("numRecords", LongType, nullable = true), StructField("minValues", StructType(Seq( StructField("intCol", IntegerType, nullable = true), StructField("stringCol", StringType, nullable = true), StructField("v", VariantType, nullable = true), StructField("v2", VariantType, nullable = true), StructField("missingField", StringType, nullable = true) )), nullable = true), StructField("maxValues", StructType(Seq( StructField("intCol", IntegerType, nullable = true), StructField("stringCol", StringType, nullable = true), StructField("v", VariantType, nullable = true), StructField("v2", VariantType, nullable = true) )), nullable = true) )) val statsJson = s"""{"numRecords": 1000,""" + s""""minValues": {"intCol": 1, "stringCol": "a", "v": "$z85_1"},""" + s""""maxValues": {"intCol": 100, "stringCol": "z", "v": "$z85_1", "v2": "$z85_2"}""" + s"""}""" val df = spark.range(1).selectExpr(s"""'${statsJson}' as stats""") val parsedDf = df.withColumn("parsed", from_json(col("stats"), statsSchema)) val decodedDf = parsedDf.withColumn( "decoded", Column(DecodeNestedZ85EncodedVariant(col("parsed").expr)) ) val result = decodedDf.select( "decoded.numRecords", "decoded.minValues.intCol", "decoded.minValues.stringCol", "decoded.minValues.v", "decoded.minValues.v2", "decoded.minValues.missingField", "decoded.maxValues.intCol", "decoded.maxValues.stringCol", "decoded.maxValues.v", "decoded.maxValues.v2" ).head() // Check non-variant fields pass through unchanged assert(result.getLong(0) == 1000L) assert(result.getInt(1) == 1) assert(result.getString(2) == "a") // Check decoded variant val decodedVariant1Min = result.get(3).asInstanceOf[VariantVal] assert(Arrays.equals(inputVariant1.getMetadata, decodedVariant1Min.getMetadata)) assert(Arrays.equals(inputVariant1.getValue, decodedVariant1Min.getValue)) // Check null variant (v2 in minValues) assert(result.isNullAt(4)) // Check missing field returns null assert(result.isNullAt(5)) // Check maxValues assert(result.getInt(6) == 100) assert(result.getString(7) == "z") val decodedVariant1Max = result.get(8).asInstanceOf[VariantVal] assert(Arrays.equals(inputVariant1.getMetadata, decodedVariant1Max.getMetadata)) assert(Arrays.equals(inputVariant1.getValue, decodedVariant1Max.getValue)) val decodedVariant2Max = result.get(9).asInstanceOf[VariantVal] assert(Arrays.equals(inputVariant2.getMetadata, decodedVariant2Max.getMetadata)) assert(Arrays.equals(inputVariant2.getValue, decodedVariant2Max.getValue)) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/expressions/HilbertIndexSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.expressions import java.util import org.apache.spark.SparkFunSuite import org.apache.spark.sql.delta.test.shims.GridTestShim class HilbertIndexSuite extends SparkFunSuite with GridTestShim { /** * Represents a test case. Each n-k pair will verify the continuity of the mapping, * and the reversibility of it. * @param n The number of dimensions * @param k The number of bits in each dimension */ case class TestCase(n: Int, k: Int) val testCases = Seq( TestCase(2, 10), TestCase(3, 6), TestCase(4, 5), TestCase(5, 4), TestCase(6, 3) ) gridTest("HilbertStates caches states")(2 to 9) { n => val start = System.nanoTime() HilbertStates.getStateList(n) val end = System.nanoTime() HilbertStates.getStateList(n) val end2 = System.nanoTime() assert(end2 - end < end - start) } gridTest("Hilbert Mapping is continuous (long keys)")(testCases) { case TestCase(n, k) => val generator = HilbertIndex.getStateGenerator(n) val stateList = generator.generateStateList() val states = stateList.getDKeyToNPointStateMap val maxDKeys = 1L << (k * n) var d = 0 var lastPoint = new Array[Int](n) while (d < maxDKeys) { val point = states.translateDKeyToNPoint(d, k) if (d != 0) { assert(HilbertUtils.manhattanDist(lastPoint, point) == 1) } lastPoint = point d += 1 } } gridTest("Hilbert Mapping is 1 to 1 (long keys)")(testCases) { case TestCase(n, k) => val generator = HilbertIndex.getStateGenerator(n) val stateList = generator.generateStateList() val d2p = stateList.getDKeyToNPointStateMap val p2d = stateList.getNPointToDKeyStateMap val maxDKeys = 1L << (k * n) var d = 0 while (d < maxDKeys) { val point = d2p.translateDKeyToNPoint(d, k) val d2 = p2d.translateNPointToDKey(point, k) assert(d == d2) d += 1 } } gridTest("Hilbert Mapping is continuous (array keys)")(testCases) { case TestCase(n, k) => val generator = HilbertIndex.getStateGenerator(n) val stateList = generator.generateStateList() val states = stateList.getDKeyToNPointStateMap val maxDKeys = 1L << (k * n) val d = new Array[Byte](((k * n) / 8) + 1) var lastPoint = new Array[Int](n) var i = 0 while (i < maxDKeys) { val point = states.translateDKeyArrayToNPoint(d, k) if (i != 0) { assert(HilbertUtils.manhattanDist(lastPoint, point) == 1, s"$i ${d.toSeq.map(_.toBinaryString.takeRight(8))} ${lastPoint.toSeq} to ${point.toSeq}") } lastPoint = point i += 1 HilbertUtils.addOne(d) } } gridTest("Hilbert Mapping is 1 to 1 (array keys)")(testCases) { case TestCase(n, k) => val generator = HilbertIndex.getStateGenerator(n) val stateList = generator.generateStateList() val d2p = stateList.getDKeyToNPointStateMap val p2d = stateList.getNPointToDKeyStateMap val maxDKeys = 1L << (k * n) val d = new Array[Byte](((k * n) / 8) + 1) var i = 0 while (i < maxDKeys) { val point = d2p.translateDKeyArrayToNPoint(d, k) val d2 = p2d.translateNPointToDKeyArray(point, k) assert(util.Arrays.equals(d, d2), s"$i ${d.toSeq}, ${d2.toSeq}") i += 1 HilbertUtils.addOne(d) } } gridTest("continuous and 1 to 1 for all spaces")((2 to 9).map(n => TestCase(n, 15 - n))) { case TestCase(n, k) => val generator = HilbertIndex.getStateGenerator(n) val stateList = generator.generateStateList() val d2p = stateList.getDKeyToNPointStateMap val p2d = stateList.getNPointToDKeyStateMap val numBits = k * n val numBytes = (numBits + 7) / 8 // test 1000 contiguous 1000 point blocks to make sure the mapping is continuous and one to one val maxDKeys = 1L << (k * n) val step = maxDKeys / 1000 var x = 0L for (_ <- 0 until 1000) { var dLong = x val bigIntArray = BigInt(dLong).toByteArray val dArray = new Array[Byte](numBytes) System.arraycopy( bigIntArray, math.max(0, bigIntArray.length - dArray.length), dArray, math.max(0, dArray.length - bigIntArray.length), math.min(bigIntArray.length, dArray.length) ) var lastPoint: Array[Int] = null for (_ <- 0 until 1000) { val pArray = d2p.translateDKeyArrayToNPoint(dArray, k) val pLong = d2p.translateDKeyToNPoint(dLong, k) assert(util.Arrays.equals(pArray, pLong), s"points should be the same at $dLong") if (lastPoint != null) { assert(HilbertUtils.manhattanDist(lastPoint, pLong) == 1, s"distance between point and last point should be the same at $dLong") } val dArray2 = p2d.translateNPointToDKeyArray(pArray, k) val dLong2 = p2d.translateNPointToDKey(pLong, k) assert(dLong == dLong2, s"reversing the points should map correctly at $dLong != $dLong2") assert(util.Arrays.equals(dArray, dArray2), s"reversing the points should map correctly at $dLong") lastPoint = pLong dLong += 1 HilbertUtils.addOne(dArray) } x += step } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/expressions/HilbertUtilsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.expressions import java.util import org.apache.spark.sql.delta.expressions.HilbertUtils.HilbertMatrix import org.apache.spark.SparkFunSuite class HilbertUtilsSuite extends SparkFunSuite { test("circularLeftShift") { assert( (0 until (1 << 10) by 7).forall(i => HilbertUtils.circularLeftShift(10, i, 0) == i), "Shift by 0 should be a no op" ) assert( (0 until (1 << 10) by 7).forall(i => HilbertUtils.circularLeftShift(10, i, 10) == i), "Shift by n should be a no op" ) // 0111 (<< 2) => 1101 assert( HilbertUtils.circularLeftShift(4, 7, 2) == 13, "handle wrapping" ) assert( (0 until (1 << 5)).forall(HilbertUtils.circularLeftShift(5, _, 5) <= (1 << 5)), "always mask values based on n" ) } test("circularRightShift") { assert( (0 until (1 << 10) by 7).forall(i => HilbertUtils.circularRightShift(10, i, 0) == i), "Shift by 0 should be a no op" ) assert( (0 until (1 << 10) by 7).forall(i => HilbertUtils.circularRightShift(10, i, 10) == i), "Shift by n should be a no op" ) // 0111 (>> 2) => 1101 assert( HilbertUtils.circularRightShift(4, 7, 2) == 13, "handle wrapping" ) assert( (0 until (1 << 5)).forall(HilbertUtils.circularRightShift(5, _, 5) <= (1 << 5)), "always mask values based on n" ) } test("getSetColumn should return the column that is set") { (0 until 16) foreach { i => assert(HilbertUtils.getSetColumn(16, 1 << i) == 16 - 1 - i) } } test("HilbertMatrix makes sense") { val identityMatrix = HilbertMatrix.identity(10) (0 until (1 << 10) by 7) foreach { i => assert(identityMatrix.transform(i) == i, s"$i transformed by the identity should be $i") } identityMatrix.multiply(HilbertMatrix.identity(10)) == identityMatrix val shift5 = HilbertMatrix(10, 0, 5) assert(shift5.multiply(shift5) == identityMatrix, "shift by 5 twice should equal identity") } test("HilbertUtils.getBits") { assert(HilbertUtils.getBits(Array(0, 0, 1), 22, 2) == 1) val array = Array[Byte](0, 0, -1, 0) assert(HilbertUtils.getBits(array, 16, 4) == 15) assert(HilbertUtils.getBits(array, 18, 3) == 7) assert(HilbertUtils.getBits(array, 23, 1) == 1) assert(HilbertUtils.getBits(array, 23, 2) == 2) assert(HilbertUtils.getBits(array, 23, 8) == 128) assert(HilbertUtils.getBits(array, 16, 3) == 7) assert(HilbertUtils.getBits(array, 16, 2) == 3) assert(HilbertUtils.getBits(array, 16, 1) == 1) assert(HilbertUtils.getBits(array, 15, 2) == 1) assert(HilbertUtils.getBits(array, 15, 1) == 0) assert(HilbertUtils.getBits(array, 12, 8) == 15) assert(HilbertUtils.getBits(array, 12, 12) == 255) assert(HilbertUtils.getBits(array, 12, 13) == (255 << 1)) assert(HilbertUtils.getBits(Array(0, 1, 0), 6, 6) == 0) assert(HilbertUtils.getBits(Array(0, 1, 0), 12, 6) == 4) assert(HilbertUtils.getBits(Array(0, 1, 0), 18, 6) == 0) } def check(received: Array[Byte], expected: Array[Byte]): Unit = { assert(util.Arrays.equals(expected, received), s"${expected.toSeq.map(_.toBinaryString.takeRight(8))} " + s"${received.toSeq.map(_.toBinaryString.takeRight(8))}") } test("HilbertUtils.setBits") { check(HilbertUtils.setBits(Array(0, 0, 0), 7, 8, 4), Array(1, 0, 0)) check(HilbertUtils.setBits(Array(0, 0, 0), 7, 12, 4), Array(1, (1.toByte << 7).toByte, 0)) check(HilbertUtils.setBits(Array(8, 0, 5), 7, 12, 4), Array(9, (1.toByte << 7).toByte, 5)) check(HilbertUtils.setBits(Array(8, 0, 2), 7, -1, 12), Array(9, -1, ((7.toByte << 5).toByte | 2).toByte)) check(HilbertUtils.setBits(Array(8, 14, 2), 15, 1, 1), Array(8, 15, 2)) } test("addOne") { check(HilbertUtils.addOne(Array(0, 0, 0)), Array(0, 0, 1)) check(HilbertUtils.addOne(Array(0, 0, -1)), Array(0, 1, 0)) check(HilbertUtils.addOne(Array(0, 0, -2)), Array(0, 0, -1)) check(HilbertUtils.addOne(Array(0, -1, -1)), Array(1, 0, 0)) check(HilbertUtils.addOne(Array(-1, -1, -1)), Array(0, 0, 0)) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/expressions/InterleaveBitsBenchmark.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.expressions import org.apache.spark.benchmark.{Benchmark, BenchmarkBase} import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow} import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.expressions.Expression /** * Benchmark to measure performance for interleave bits. * To run this benchmark: * {{{ * build/sbt "core/test:runMain org.apache.spark.sql.delta.expressions.InterleaveBitsBenchmark" * }}} */ object InterleaveBitsBenchmark extends BenchmarkBase { private val numRows = 1 * 1000 * 1000 private def seqInt(numColumns: Int): Seq[Array[Int]] = { (1 to numRows).map { l => val arr = new Array[Int](numColumns) (0 until numColumns).foreach(col => arr(col) = l) arr } } private def randomInt(numColumns: Int): Seq[Array[Int]] = { (1 to numRows).map { l => val arr = new Array[Int](numColumns) (0 until numColumns).foreach(col => arr(col) = scala.util.Random.nextInt()) arr } } private def createExpression(numColumns: Int): Expression = { val inputs = (0 until numColumns).map { i => $"c_$i".int.at(i) } InterleaveBits(inputs) } protected def create_row(values: Any*): InternalRow = { InternalRow.fromSeq(values.map(CatalystTypeConverters.convertToCatalyst)) } override def runBenchmarkSuite(mainArgs: Array[String]): Unit = { val benchmark = new Benchmark(s"$numRows rows interleave bits benchmark", numRows, output = output) benchmark.addCase("sequence - 1 int columns benchmark", 3) { _ => val interleaveBits = createExpression(1) seqInt(1).foreach { input => interleaveBits.eval(create_row(input: _*)) } } benchmark.addCase("sequence - 2 int columns benchmark", 3) { _ => val interleaveBits = createExpression(2) seqInt(2).foreach { input => interleaveBits.eval(create_row(input: _*)) } } benchmark.addCase("sequence - 3 int columns benchmark", 3) { _ => val interleaveBits = createExpression(3) seqInt(3).foreach { input => interleaveBits.eval(create_row(input: _*)) } } benchmark.addCase("sequence - 4 int columns benchmark", 3) { _ => val interleaveBits = createExpression(4) seqInt(4).foreach { input => interleaveBits.eval(create_row(input: _*)) } } benchmark.addCase("random - 1 int columns benchmark", 3) { _ => val interleaveBits = createExpression(1) randomInt(1).foreach { input => interleaveBits.eval(create_row(input: _*)) } } benchmark.addCase("random - 2 int columns benchmark", 3) { _ => val interleaveBits = createExpression(2) randomInt(2).foreach { input => interleaveBits.eval(create_row(input: _*)) } } benchmark.addCase("random - 3 int columns benchmark", 3) { _ => val interleaveBits = createExpression(3) randomInt(3).foreach { input => interleaveBits.eval(create_row(input: _*)) } } benchmark.addCase(" random - 4 int columns benchmark", 3) { _ => val interleaveBits = createExpression(4) randomInt(4).foreach { input => interleaveBits.eval(create_row(input: _*)) } } benchmark.run() } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/expressions/InterleaveBitsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.expressions import java.nio.ByteBuffer import scala.util.Random import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.TypeCheckSuccess import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionEvalHelper, Literal} import org.apache.spark.sql.types.IntegerType class InterleaveBitsSuite extends SparkFunSuite with ExpressionEvalHelper { def intToBinary(x: Int): Array[Byte] = { ByteBuffer.allocate(4).putInt(x).array() } def checkInterleaving(input: Seq[Expression], expectedOutput: Any): Unit = { Seq("true", "false").foreach { flag => withSQLConf(DeltaSQLConf.FAST_INTERLEAVE_BITS_ENABLED.key -> flag) { checkEvaluation(InterleaveBits(input), expectedOutput) } } } test("0 inputs") { checkInterleaving(Seq.empty[Expression], Array.empty[Byte]) } test("1 input") { for { i <- 1.to(10) } { val r = Random.nextInt() checkInterleaving(Seq(Literal(r)), intToBinary(r)) } } test("2 inputs") { checkInterleaving( input = Seq( 0x000ff0ff, 0xfff00f00 ).map(Literal(_)), expectedOutput = Array(0x55, 0x55, 0x55, 0xaa, 0xaa, 0x55, 0xaa, 0xaa) .map(_.toByte)) } test("3 inputs") { checkInterleaving( input = Seq( 0xff00, 0x00ff, 0x0000 ).map(Literal(_)), expectedOutput = Array(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0x49, 0x24, 0x49, 0x24, 0x92) .map(_.toByte)) } test("9 inputs") { val result = Array( 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0xffffff92, 0x00000049, 0x00000024, 0xffffff92, 0x00000049, 0x00000024, 0xffffff92, 0x00000049, 0x00000024, 0x00000049, 0x00000024, 0xffffff92, 0x00000049, 0x00000024, 0xffffff92, 0x00000049, 0x00000024, 0xffffff92 ) checkInterleaving( input = Seq( 0xff00, 0x00ff, 0x0000, 0xff00, 0x00ff, 0x0000, 0xff00, 0x00ff, 0x0000 ).map(Literal(_)), expectedOutput = result.map(_.toByte) ) } test("nulls") { val ones = 0xffffffff checkInterleaving( Seq(Literal(ones), Literal.create(null, IntegerType)), Array.fill(8)(0xaa.toByte)) checkInterleaving( Seq(Literal.create(null, IntegerType), Literal(ones)), Array.fill(8)(0x55.toByte)) for { i <- 0.to(6) } { checkInterleaving( Seq.fill(i)(Literal.create(null, IntegerType)), Array.fill(i * 4)(0x00.toByte)) } } test("consistency") { for { num_inputs <- 1 to 10 } { checkConsistencyBetweenInterpretedAndCodegen(InterleaveBits(_), IntegerType, num_inputs) } } test("supported types") { // only int for now InterleaveBits(Seq(Literal(0))).checkInputDataTypes() == TypeCheckSuccess // nothing else InterleaveBits(Seq(Literal(false))).checkInputDataTypes() != TypeCheckSuccess InterleaveBits(Seq(Literal(0.toLong))).checkInputDataTypes() != TypeCheckSuccess InterleaveBits(Seq(Literal(0.toDouble))).checkInputDataTypes() != TypeCheckSuccess InterleaveBits(Seq(Literal(0.toString))).checkInputDataTypes() != TypeCheckSuccess } test("randomization interleave bits") { val numIters = sys.env .get("NUMBER_OF_ITERATIONS_TO_INTERLEAVE_BITS") .map(_.toInt) .getOrElse(1000000) var i = 0 while (i < numIters) { // generate n columns where 1 <= n <= 8 val numCols = Random.nextInt(8) + 1 val input = new Array[Int](numCols) var j = 0 while (j < numCols) { input(j) = Random.nextInt() j += 1 } val r1 = InterleaveBits.interleaveBits(input, true) val r2 = InterleaveBits.interleaveBits(input, false) assert(java.util.Arrays.equals(r1, r2), s"input: ${input.mkString(",")}") i += 1 } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/expressions/RangePartitionIdSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.expressions import scala.reflect.ClassTag import org.apache.spark.{Partitioner, RangePartitioner, SparkFunSuite, SparkThrowable} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.test.SharedSparkSession class RangePartitionIdSuite extends SparkFunSuite with ExpressionEvalHelper with SharedSparkSession { def getPartitioner[T : Ordering : ClassTag](data: Seq[T], partitions: Int): Partitioner = { implicit val ordering = new Ordering[GenericInternalRow] { override def compare(x: GenericInternalRow, y: GenericInternalRow): Int = { def getValue0AsT(row: GenericInternalRow): T = row.values.head.asInstanceOf[T] val orderingT = implicitly[Ordering[T]] orderingT.compare(getValue0AsT(x), getValue0AsT(y)) } } val rdd = spark.sparkContext.parallelize(data).filter(_ != null) .map(key => (new GenericInternalRow(Array[Any](key)), null)) new RangePartitioner(partitions, rdd) } def testRangePartitionerExpr[T : Ordering : ClassTag]( data: Seq[T], partitions: Int, childExpr: Expression, expected: Any): Unit = { val rangePartitioner = getPartitioner(data, partitions) checkEvaluation(PartitionerExpr(childExpr, rangePartitioner), expected) } test("RangePartitionerExpr: test basic") { val data = 0.until(12) for { numPartitions <- Seq(2, 3, 4, 6) } { val rangePartitioner = getPartitioner(data, numPartitions) data.foreach { i => val expected = i / (data.size / numPartitions) checkEvaluation(PartitionerExpr(Literal(i), rangePartitioner), expected) } } } test("RangePartitionerExpr: null values") { testRangePartitionerExpr( data = 0.until(10), partitions = 2, childExpr = Literal(null), expected = 0) } test("RangePartitionerExpr: null data") { testRangePartitionerExpr( data = 0.until(10).map(_ => null), partitions = 2, childExpr = Literal("asd"), expected = 0) } test("RangePartitionId: unevaluable") { intercept[Exception with SparkThrowable] { evaluateWithoutCodegen(RangePartitionId(Literal(2), 10)) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/expressions/aggregation/BitmapAggregatorSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.expressions.aggregation import scala.collection.mutable import org.apache.spark.sql.catalyst.expressions.aggregation.BitmapAggregator import org.apache.spark.sql.delta.deletionvectors.{RoaringBitmapArray, RoaringBitmapArrayFormat} import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.BoundReference import org.apache.spark.sql.types.LongType class BitmapAggregatorSuite extends SparkFunSuite { import BitmapAggregatorSuite._ private val childExpression = BoundReference(0, LongType, nullable = true) /** Creates a bitmap aggregate expression, using the child expression defined above. */ private def newBitmapAgg(format: RoaringBitmapArrayFormat.Value): BitmapAggregator = new BitmapAggregator(childExpression, format) for (serializationFormat <- RoaringBitmapArrayFormat.values) test(s"Bitmap serialization - $serializationFormat") { val bitmapSet = fillSetWithAggregator(newBitmapAgg(serializationFormat), Array(1L, 2L, 3L, 4L)) val serialized = bitmapSet.serializeAsByteArray(serializationFormat) val deserialized = RoaringBitmapArray.readFrom(serialized) assert(bitmapSet === deserialized) assert(bitmapSet.## === deserialized.##) } for (serializationFormat <- RoaringBitmapArrayFormat.values) test(s"Aggregator serialization - $serializationFormat") { val aggregator = newBitmapAgg(serializationFormat) val bitmapSet = fillSetWithAggregator(aggregator, Array(1L, 2L, 3L, 4L)) val deserialized = aggregator.deserialize(aggregator.serialize(bitmapSet)) assert(bitmapSet === deserialized) assert(bitmapSet.## === deserialized.##) } for (serializationFormat <- RoaringBitmapArrayFormat.values) test(s"Bitmap Aggregator merge no duplicates - $serializationFormat") { val (dataset1, dataset2) = createDatasetsNoDuplicates val finalResult = fillSetWithAggregatorAndMerge( newBitmapAgg(serializationFormat), dataset1, dataset2) verifyContainsAll(finalResult, dataset1) verifyContainsAll(finalResult, dataset2) } for (serializationFormat <- RoaringBitmapArrayFormat.values) test(s"Bitmap Aggregator with duplicates - $serializationFormat") { val (dataset1, dataset2) = createDatasetsWithDuplicates val finalResult = fillSetWithAggregatorAndMerge( newBitmapAgg(serializationFormat), dataset1, dataset2) verifyContainsAll(finalResult, dataset1) verifyContainsAll(finalResult, dataset2) } private lazy val createDatasetsNoDuplicates: (List[Long], List[Long]) = { val primeSet = primes(DATASET_SIZE).toSet val notPrime = (0 until DATASET_SIZE).filterNot(primeSet.contains).toList (primeSet.map(_.toLong).toList, notPrime.map(_.toLong)) } private def createDatasetsWithDuplicates: (List[Long], List[Long]) = { var (primes, notPrimes) = createDatasetsNoDuplicates // duplicate all powers of 3 (powers of 2 might align with container boundaries) notPrimes ::= 3L var value = 3L while (value < DATASET_SIZE.toLong) { value *= 3L primes ::= value } (primes, notPrimes) } // List the first primes smaller than `end` private def primes(end: Int): List[Int] = { // scalastyle:off // Basically https://en.wikipedia.org/wiki/Sieve_of_Eratosthenes#Algorithm_and_variants // but concretely the implementation is adapted from: // https://medium.com/coding-with-clarity/functional-vs-iterative-prime-numbers-in-scala-7e22447146f0 // scalastyle:on val primeIndices = mutable.ArrayBuffer.fill((end + 1) / 2)(true) val intSqrt = Math.sqrt(end).toInt for { i <- 3 to end by 2 if i <= intSqrt nonPrime <- i * i to end by 2 * i } primeIndices.update(nonPrime / 2, false) (for (i <- primeIndices.indices if primeIndices(i)) yield 2 * i + 1).tail.toList } private def fillSetWithAggregatorAndMerge( aggregator: BitmapAggregator, dataset1: Seq[Long], dataset2: Seq[Long]): RoaringBitmapArray = { val buffer1 = fillSetWithAggregator(aggregator, dataset1) val buffer2 = fillSetWithAggregator(aggregator, dataset2) val merged = aggregator.merge(buffer1, buffer2) val fieldIndex = aggregator.dataType.fieldIndex("bitmap") val result = aggregator.eval(merged).getBinary(fieldIndex) RoaringBitmapArray.readFrom(result) } private def fillSetWithAggregator( aggregator: BitmapAggregator, dataset: Seq[Long]): RoaringBitmapArray = { val buffer = aggregator.createAggregationBuffer() for (entry <- dataset) { val row = InternalRow(entry) aggregator.update(buffer, row) } buffer } private def verifyContainsAll( aggregator: RoaringBitmapArray, dataset: Seq[Long]): Unit = { for (entry <- dataset) { assert(aggregator.contains(entry), s"Aggregator did not contain file $entry") } } } object BitmapAggregatorSuite { // Pick something over 64k to make sure we fill a few different bitmap containers val DATASET_SIZE: Int = 100000 } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/files/TransactionalWriteSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.files import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.functions.column import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{StringType, StructType} class TransactionalWriteSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { test("writing out an empty dataframe produces no AddFiles") { withTempDir { dir => spark.range(100).write.format("delta").save(dir.getCanonicalPath) val log = DeltaLog.forTable(spark, dir.getCanonicalPath) val schema = new StructType().add("id", StringType) val emptyDf = spark.createDataFrame(spark.sparkContext.emptyRDD[Row], schema) assert(log.startTransaction().writeFiles(emptyDf).isEmpty) } } test("write data files to the data subdir") { withSQLConf(DeltaSQLConf.WRITE_DATA_FILES_TO_SUBDIR.key -> "true") { def validateDataSubdir(tablePath: String): Unit = { val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, tablePath) snapshot.allFiles.collect().foreach { f => assert(f.path.startsWith("data/")) } } withTempDir { dir => spark.range(100).toDF("id").write.format("delta").save(dir.getCanonicalPath) validateDataSubdir(dir.getCanonicalPath) } withTempDir { dir => spark.range(100).toDF("id").withColumn("id1", column("id")).write.format("delta") .partitionBy("id").save(dir.getCanonicalPath) validateDataSubdir(dir.getCanonicalPath) } } withSQLConf(DeltaSQLConf.WRITE_DATA_FILES_TO_SUBDIR.key -> "false") { withTempDir { dir => spark.range(100).toDF("id").write.format("delta").save(dir.getCanonicalPath) val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, dir.getCanonicalPath) snapshot.allFiles.collect().foreach { f => assert(!f.path.startsWith("data/")) } } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/fuzzer/AtomicBarrierSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.fuzzer import scala.concurrent.duration._ import org.apache.spark.sql.delta.concurrency.PhaseLockingTestMixin import org.apache.spark.SparkFunSuite class AtomicBarrierSuite extends SparkFunSuite with PhaseLockingTestMixin { val timeout: FiniteDuration = 5000.millis test("Atomic Barrier - wait before unblock") { val barrier = new AtomicBarrier assert(AtomicBarrier.State.Blocked === barrier.load()) val thread = new Thread(() => { barrier.waitToPass() }) assert(AtomicBarrier.State.Blocked === barrier.load()) thread.start() busyWaitForState(barrier, AtomicBarrier.State.Requested, timeout) assert(thread.isAlive) // should be stuck waiting for unblock barrier.unblock() busyWaitForState(barrier, AtomicBarrier.State.Passed, timeout) thread.join(timeout.toMillis) // shouldn't take long assert(!thread.isAlive) // should have passed the barrier and completed } test("Atomic Barrier - unblock before wait") { val barrier = new AtomicBarrier assert(AtomicBarrier.State.Blocked === barrier.load()) val thread = new Thread(() => { barrier.waitToPass() }) assert(AtomicBarrier.State.Blocked === barrier.load()) barrier.unblock() assert(AtomicBarrier.State.Unblocked === barrier.load()) thread.start() busyWaitForState(barrier, AtomicBarrier.State.Passed, timeout) thread.join(timeout.toMillis) // shouldn't take long assert(!thread.isAlive) // should have passed the barrier and completed } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/DeleteSuitesDeleteBaseTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class DeleteBaseSQLNameBasedSuite extends DeleteBaseTests with DeleteSQLMixin with DeltaDMLTestUtilsNameBased class DeleteBaseSQLPathBasedCDCOnSuite extends DeleteBaseTests with DeleteSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with DeleteCDCMixin class DeleteBaseSQLPathBasedColMapIdModeSuite extends DeleteBaseTests with DeleteSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode class DeleteBaseSQLPathBasedColMapNameModeSuite extends DeleteBaseTests with DeleteSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with DeleteSQLNameColumnMappingMixin class DeleteBaseSQLPathBasedDVPredPushOffSuite extends DeleteBaseTests with DeleteSQLMixin with DeltaDMLTestUtilsPathBased with DeleteSQLWithDeletionVectorsMixin with PredicatePushdownDisabled class DeleteBaseSQLPathBasedDVPredPushOnSuite extends DeleteBaseTests with DeleteSQLMixin with DeltaDMLTestUtilsPathBased with DeleteSQLWithDeletionVectorsMixin with PredicatePushdownEnabled class DeleteBaseSQLPathBasedSuite extends DeleteBaseTests with DeleteSQLMixin with DeltaDMLTestUtilsPathBased class DeleteBaseScalaSuite extends DeleteBaseTests with DeleteScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/DeleteSuitesDeleteCDCTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class DeleteCDCSQLPathBasedCDCOnSuite extends DeleteCDCTests with DeleteSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with DeleteCDCMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/DeleteSuitesDeleteSQLTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class DeleteSQLSQLNameBasedSuite extends DeleteSQLTests with DeleteSQLMixin with DeltaDMLTestUtilsNameBased class DeleteSQLSQLPathBasedCDCOnSuite extends DeleteSQLTests with DeleteSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with DeleteCDCMixin class DeleteSQLSQLPathBasedColMapIdModeSuite extends DeleteSQLTests with DeleteSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode class DeleteSQLSQLPathBasedColMapNameModeSuite extends DeleteSQLTests with DeleteSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with DeleteSQLNameColumnMappingMixin class DeleteSQLSQLPathBasedDVPredPushOffSuite extends DeleteSQLTests with DeleteSQLMixin with DeltaDMLTestUtilsPathBased with DeleteSQLWithDeletionVectorsMixin with PredicatePushdownDisabled class DeleteSQLSQLPathBasedDVPredPushOnSuite extends DeleteSQLTests with DeleteSQLMixin with DeltaDMLTestUtilsPathBased with DeleteSQLWithDeletionVectorsMixin with PredicatePushdownEnabled class DeleteSQLSQLPathBasedSuite extends DeleteSQLTests with DeleteSQLMixin with DeltaDMLTestUtilsPathBased ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/DeleteSuitesDeleteScalaTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class DeleteScalaScalaSuite extends DeleteScalaTests with DeleteScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/DeleteSuitesDeleteTempViewTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class DeleteTempViewSQLNameBasedSuite extends DeleteTempViewTests with DeleteSQLMixin with DeltaDMLTestUtilsNameBased class DeleteTempViewSQLPathBasedCDCOnSuite extends DeleteTempViewTests with DeleteSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with DeleteCDCMixin class DeleteTempViewSQLPathBasedColMapIdModeSuite extends DeleteTempViewTests with DeleteSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode class DeleteTempViewSQLPathBasedColMapNameModeSuite extends DeleteTempViewTests with DeleteSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with DeleteSQLNameColumnMappingMixin class DeleteTempViewSQLPathBasedDVPredPushOffSuite extends DeleteTempViewTests with DeleteSQLMixin with DeltaDMLTestUtilsPathBased with DeleteSQLWithDeletionVectorsMixin with PredicatePushdownDisabled class DeleteTempViewSQLPathBasedDVPredPushOnSuite extends DeleteTempViewTests with DeleteSQLMixin with DeltaDMLTestUtilsPathBased with DeleteSQLWithDeletionVectorsMixin with PredicatePushdownEnabled class DeleteTempViewSQLPathBasedSuite extends DeleteTempViewTests with DeleteSQLMixin with DeltaDMLTestUtilsPathBased ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/DeleteSuitesRowTrackingDeleteDvBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class RowTrackingDeleteDvBaseCDCOnDVOnColMapIdModeSuite extends RowTrackingDeleteDvBase with CDCEnabled with PersistentDVEnabled with DeltaColumnMappingEnableIdMode class RowTrackingDeleteDvBaseCDCOnDVOnColMapNameModeSuite extends RowTrackingDeleteDvBase with CDCEnabled with PersistentDVEnabled with DeltaColumnMappingEnableNameMode class RowTrackingDeleteDvBaseCDCOnDVOnSuite extends RowTrackingDeleteDvBase with CDCEnabled with PersistentDVEnabled class RowTrackingDeleteDvBaseDVOnSuite extends RowTrackingDeleteDvBase with PersistentDVEnabled ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/DeleteSuitesRowTrackingDeleteSuiteBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class RowTrackingDeleteSuiteBaseCDCOnDVOffSuite extends RowTrackingDeleteSuiteBase with CDCEnabled with PersistentDVDisabled class RowTrackingDeleteSuiteBaseCDCOnDVOnColMapIdModeSuite extends RowTrackingDeleteSuiteBase with CDCEnabled with PersistentDVEnabled with DeltaColumnMappingEnableIdMode class RowTrackingDeleteSuiteBaseCDCOnDVOnColMapNameModeSuite extends RowTrackingDeleteSuiteBase with CDCEnabled with PersistentDVEnabled with DeltaColumnMappingEnableNameMode class RowTrackingDeleteSuiteBaseCDCOnDVOnSuite extends RowTrackingDeleteSuiteBase with CDCEnabled with PersistentDVEnabled class RowTrackingDeleteSuiteBaseDVOffColMapIdModeSuite extends RowTrackingDeleteSuiteBase with PersistentDVDisabled with DeltaColumnMappingEnableIdMode class RowTrackingDeleteSuiteBaseDVOffColMapNameModeSuite extends RowTrackingDeleteSuiteBase with PersistentDVDisabled with DeltaColumnMappingEnableNameMode class RowTrackingDeleteSuiteBaseDVOffSuite extends RowTrackingDeleteSuiteBase with PersistentDVDisabled class RowTrackingDeleteSuiteBaseDVOnSuite extends RowTrackingDeleteSuiteBase with PersistentDVEnabled ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/InsertSuitesDeltaInsertIntoImplicitCastStreamingWriteTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ class DeltaInsertIntoImplicitCastStreamingWriteSuite extends DeltaInsertIntoImplicitCastStreamingWriteTests ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/InsertSuitesDeltaInsertIntoImplicitCastTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ class DeltaInsertIntoImplicitCastSuite extends DeltaInsertIntoImplicitCastTests ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeCDCTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeCDCSQLPathBasedCDCOnDVsPredPushOffSuite extends MergeCDCTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeCDCSQLPathBasedCDCOnDVsPredPushOnSuite extends MergeCDCTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeCDCSQLPathBasedCDCOnSuite extends MergeCDCTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeCDCMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoAnalysisExceptionTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoAnalysisExceptionSQLNameBasedSuite extends MergeIntoAnalysisExceptionTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased class MergeIntoAnalysisExceptionSQLPathBasedCDCOnDVsPredPushOffSuite extends MergeIntoAnalysisExceptionTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoAnalysisExceptionSQLPathBasedCDCOnDVsPredPushOnSuite extends MergeIntoAnalysisExceptionTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoAnalysisExceptionSQLPathBasedCDCOnSuite extends MergeIntoAnalysisExceptionTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeCDCMixin class MergeIntoAnalysisExceptionSQLPathBasedColMapIdModeSuite extends MergeIntoAnalysisExceptionTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode with MergeIntoSQLColumnMappingOverrides class MergeIntoAnalysisExceptionSQLPathBasedColMapNameModeSuite extends MergeIntoAnalysisExceptionTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with MergeIntoSQLColumnMappingOverrides class MergeIntoAnalysisExceptionSQLPathBasedDVsPredPushOffSuite extends MergeIntoAnalysisExceptionTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownDisabled class MergeIntoAnalysisExceptionSQLPathBasedDVsPredPushOnSuite extends MergeIntoAnalysisExceptionTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownEnabled class MergeIntoAnalysisExceptionSQLPathBasedSuite extends MergeIntoAnalysisExceptionTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased class MergeIntoAnalysisExceptionScalaSuite extends MergeIntoAnalysisExceptionTests with MergeIntoScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoBasicTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoBasicSQLNameBasedSuite extends MergeIntoBasicTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased class MergeIntoBasicSQLPathBasedCDCOnDVsPredPushOffSuite extends MergeIntoBasicTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoBasicSQLPathBasedCDCOnDVsPredPushOnSuite extends MergeIntoBasicTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoBasicSQLPathBasedCDCOnSuite extends MergeIntoBasicTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeCDCMixin class MergeIntoBasicSQLPathBasedColMapIdModeSuite extends MergeIntoBasicTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode with MergeIntoSQLColumnMappingOverrides class MergeIntoBasicSQLPathBasedColMapNameModeSuite extends MergeIntoBasicTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with MergeIntoSQLColumnMappingOverrides class MergeIntoBasicSQLPathBasedDVsPredPushOffSuite extends MergeIntoBasicTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownDisabled class MergeIntoBasicSQLPathBasedDVsPredPushOnSuite extends MergeIntoBasicTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownEnabled class MergeIntoBasicSQLPathBasedSuite extends MergeIntoBasicTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased class MergeIntoBasicScalaSuite extends MergeIntoBasicTests with MergeIntoScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoDVsTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoDVsSQLPathBasedCDCOnDVsPredPushOffSuite extends MergeIntoDVsTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoDVsSQLPathBasedCDCOnDVsPredPushOnSuite extends MergeIntoDVsTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoDVsSQLPathBasedDVsPredPushOffSuite extends MergeIntoDVsTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownDisabled class MergeIntoDVsSQLPathBasedDVsPredPushOnSuite extends MergeIntoDVsTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownEnabled ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoExtendedSyntaxTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoExtendedSyntaxSQLNameBasedSuite extends MergeIntoExtendedSyntaxTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased class MergeIntoExtendedSyntaxSQLPathBasedCDCOnDVsPredPushOffSuite extends MergeIntoExtendedSyntaxTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoExtendedSyntaxSQLPathBasedCDCOnDVsPredPushOnSuite extends MergeIntoExtendedSyntaxTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoExtendedSyntaxSQLPathBasedCDCOnSuite extends MergeIntoExtendedSyntaxTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeCDCMixin class MergeIntoExtendedSyntaxSQLPathBasedColMapIdModeSuite extends MergeIntoExtendedSyntaxTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode with MergeIntoSQLColumnMappingOverrides class MergeIntoExtendedSyntaxSQLPathBasedColMapNameModeSuite extends MergeIntoExtendedSyntaxTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with MergeIntoSQLColumnMappingOverrides class MergeIntoExtendedSyntaxSQLPathBasedDVsPredPushOffSuite extends MergeIntoExtendedSyntaxTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownDisabled class MergeIntoExtendedSyntaxSQLPathBasedDVsPredPushOnSuite extends MergeIntoExtendedSyntaxTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownEnabled class MergeIntoExtendedSyntaxSQLPathBasedSuite extends MergeIntoExtendedSyntaxTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased class MergeIntoExtendedSyntaxScalaSuite extends MergeIntoExtendedSyntaxTests with MergeIntoScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoMaterializeSourceErrorTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoMaterializeSourceErrorMergePersistentDVOffSuite extends MergeIntoMaterializeSourceErrorTests with MergePersistentDVDisabled ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoMaterializeSourceTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoMaterializeSourceMergePersistentDVOffSuite extends MergeIntoMaterializeSourceTests with MergePersistentDVDisabled ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedArrayStructEvolutionNullnessTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedSuite extends MergeIntoNestedArrayStructEvolutionNullnessTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedDataTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoNestedDataSQLNameBasedSuite extends MergeIntoNestedDataTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased class MergeIntoNestedDataSQLPathBasedCDCOnDVsPredPushOffSuite extends MergeIntoNestedDataTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoNestedDataSQLPathBasedCDCOnDVsPredPushOnSuite extends MergeIntoNestedDataTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoNestedDataSQLPathBasedCDCOnSuite extends MergeIntoNestedDataTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeCDCMixin class MergeIntoNestedDataSQLPathBasedColMapIdModeSuite extends MergeIntoNestedDataTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode with MergeIntoSQLColumnMappingOverrides class MergeIntoNestedDataSQLPathBasedColMapNameModeSuite extends MergeIntoNestedDataTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with MergeIntoSQLColumnMappingOverrides class MergeIntoNestedDataSQLPathBasedDVsPredPushOffSuite extends MergeIntoNestedDataTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownDisabled class MergeIntoNestedDataSQLPathBasedDVsPredPushOnSuite extends MergeIntoNestedDataTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownEnabled class MergeIntoNestedDataSQLPathBasedSuite extends MergeIntoNestedDataTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased class MergeIntoNestedDataScalaSuite extends MergeIntoNestedDataTests with MergeIntoScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedMapStructEvolutionNullnessTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedSuite extends MergeIntoNestedMapStructEvolutionNullnessTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedStructEvolutionInsertTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoNestedStructEvolutionInsertSQLNameBasedSuite extends MergeIntoNestedStructEvolutionInsertTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased class MergeIntoNestedStructEvolutionInsertSQLPathBasedCDCOnDVsPredPushOffSuite extends MergeIntoNestedStructEvolutionInsertTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoNestedStructEvolutionInsertSQLPathBasedCDCOnDVsPredPushOnSuite extends MergeIntoNestedStructEvolutionInsertTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoNestedStructEvolutionInsertSQLPathBasedCDCOnSuite extends MergeIntoNestedStructEvolutionInsertTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeCDCMixin class MergeIntoNestedStructEvolutionInsertSQLPathBasedColMapIdModeSuite extends MergeIntoNestedStructEvolutionInsertTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode with MergeIntoSQLColumnMappingOverrides class MergeIntoNestedStructEvolutionInsertSQLPathBasedColMapNameModeSuite extends MergeIntoNestedStructEvolutionInsertTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with MergeIntoSQLColumnMappingOverrides class MergeIntoNestedStructEvolutionInsertSQLPathBasedDVsPredPushOffSuite extends MergeIntoNestedStructEvolutionInsertTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownDisabled class MergeIntoNestedStructEvolutionInsertSQLPathBasedDVsPredPushOnSuite extends MergeIntoNestedStructEvolutionInsertTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownEnabled class MergeIntoNestedStructEvolutionInsertSQLPathBasedSuite extends MergeIntoNestedStructEvolutionInsertTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased class MergeIntoNestedStructEvolutionInsertScalaSuite extends MergeIntoNestedStructEvolutionInsertTests with MergeIntoScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedStructEvolutionNullnessTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoNestedStructEvolutionNullnessSQLNameBasedSuite extends MergeIntoNestedStructEvolutionNullnessTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedStructEvolutionUpdateOnlyTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoNestedStructEvolutionUpdateOnlySQLNameBasedSuite extends MergeIntoNestedStructEvolutionUpdateOnlyTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased class MergeIntoNestedStructEvolutionUpdateOnlySQLPathBasedCDCOnDVsPredPushOffSuite extends MergeIntoNestedStructEvolutionUpdateOnlyTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoNestedStructEvolutionUpdateOnlySQLPathBasedCDCOnDVsPredPushOnSuite extends MergeIntoNestedStructEvolutionUpdateOnlyTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoNestedStructEvolutionUpdateOnlySQLPathBasedCDCOnSuite extends MergeIntoNestedStructEvolutionUpdateOnlyTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeCDCMixin class MergeIntoNestedStructEvolutionUpdateOnlySQLPathBasedColMapIdModeSuite extends MergeIntoNestedStructEvolutionUpdateOnlyTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode with MergeIntoSQLColumnMappingOverrides class MergeIntoNestedStructEvolutionUpdateOnlySQLPathBasedColMapNameModeSuite extends MergeIntoNestedStructEvolutionUpdateOnlyTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with MergeIntoSQLColumnMappingOverrides class MergeIntoNestedStructEvolutionUpdateOnlySQLPathBasedDVsPredPushOffSuite extends MergeIntoNestedStructEvolutionUpdateOnlyTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownDisabled class MergeIntoNestedStructEvolutionUpdateOnlySQLPathBasedDVsPredPushOnSuite extends MergeIntoNestedStructEvolutionUpdateOnlyTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownEnabled class MergeIntoNestedStructEvolutionUpdateOnlySQLPathBasedSuite extends MergeIntoNestedStructEvolutionUpdateOnlyTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased class MergeIntoNestedStructEvolutionUpdateOnlyScalaSuite extends MergeIntoNestedStructEvolutionUpdateOnlyTests with MergeIntoScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedStructInMapEvolutionTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoNestedStructInMapEvolutionSQLNameBasedSuite extends MergeIntoNestedStructInMapEvolutionTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased class MergeIntoNestedStructInMapEvolutionSQLPathBasedCDCOnDVsPredPushOffSuite extends MergeIntoNestedStructInMapEvolutionTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoNestedStructInMapEvolutionSQLPathBasedCDCOnDVsPredPushOnSuite extends MergeIntoNestedStructInMapEvolutionTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoNestedStructInMapEvolutionSQLPathBasedCDCOnSuite extends MergeIntoNestedStructInMapEvolutionTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeCDCMixin class MergeIntoNestedStructInMapEvolutionSQLPathBasedColMapIdModeSuite extends MergeIntoNestedStructInMapEvolutionTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode with MergeIntoSQLColumnMappingOverrides class MergeIntoNestedStructInMapEvolutionSQLPathBasedColMapNameModeSuite extends MergeIntoNestedStructInMapEvolutionTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with MergeIntoSQLColumnMappingOverrides class MergeIntoNestedStructInMapEvolutionSQLPathBasedDVsPredPushOffSuite extends MergeIntoNestedStructInMapEvolutionTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownDisabled class MergeIntoNestedStructInMapEvolutionSQLPathBasedDVsPredPushOnSuite extends MergeIntoNestedStructInMapEvolutionTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownEnabled class MergeIntoNestedStructInMapEvolutionSQLPathBasedSuite extends MergeIntoNestedStructInMapEvolutionTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased class MergeIntoNestedStructInMapEvolutionScalaSuite extends MergeIntoNestedStructInMapEvolutionTests with MergeIntoScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNotMatchedBySourceCDCPart1Tests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoNotMatchedBySourceCDCPart1SQLNameBasedSuite extends MergeIntoNotMatchedBySourceCDCPart1Tests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased class MergeIntoNotMatchedBySourceCDCPart1SQLPathBasedCDCOnDVsPredPushOffSuite extends MergeIntoNotMatchedBySourceCDCPart1Tests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoNotMatchedBySourceCDCPart1SQLPathBasedCDCOnDVsPredPushOnSuite extends MergeIntoNotMatchedBySourceCDCPart1Tests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoNotMatchedBySourceCDCPart1SQLPathBasedCDCOnSuite extends MergeIntoNotMatchedBySourceCDCPart1Tests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeCDCMixin class MergeIntoNotMatchedBySourceCDCPart1SQLPathBasedColMapIdModeSuite extends MergeIntoNotMatchedBySourceCDCPart1Tests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode with MergeIntoSQLColumnMappingOverrides class MergeIntoNotMatchedBySourceCDCPart1SQLPathBasedColMapNameModeSuite extends MergeIntoNotMatchedBySourceCDCPart1Tests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with MergeIntoSQLColumnMappingOverrides class MergeIntoNotMatchedBySourceCDCPart1SQLPathBasedDVsPredPushOffSuite extends MergeIntoNotMatchedBySourceCDCPart1Tests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownDisabled class MergeIntoNotMatchedBySourceCDCPart1SQLPathBasedDVsPredPushOnSuite extends MergeIntoNotMatchedBySourceCDCPart1Tests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownEnabled class MergeIntoNotMatchedBySourceCDCPart1SQLPathBasedSuite extends MergeIntoNotMatchedBySourceCDCPart1Tests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased class MergeIntoNotMatchedBySourceCDCPart1ScalaSuite extends MergeIntoNotMatchedBySourceCDCPart1Tests with MergeIntoScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNotMatchedBySourceCDCPart2Tests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoNotMatchedBySourceCDCPart2SQLNameBasedSuite extends MergeIntoNotMatchedBySourceCDCPart2Tests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased class MergeIntoNotMatchedBySourceCDCPart2SQLPathBasedCDCOnDVsPredPushOffSuite extends MergeIntoNotMatchedBySourceCDCPart2Tests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoNotMatchedBySourceCDCPart2SQLPathBasedCDCOnDVsPredPushOnSuite extends MergeIntoNotMatchedBySourceCDCPart2Tests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoNotMatchedBySourceCDCPart2SQLPathBasedCDCOnSuite extends MergeIntoNotMatchedBySourceCDCPart2Tests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeCDCMixin class MergeIntoNotMatchedBySourceCDCPart2SQLPathBasedColMapIdModeSuite extends MergeIntoNotMatchedBySourceCDCPart2Tests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode with MergeIntoSQLColumnMappingOverrides class MergeIntoNotMatchedBySourceCDCPart2SQLPathBasedColMapNameModeSuite extends MergeIntoNotMatchedBySourceCDCPart2Tests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with MergeIntoSQLColumnMappingOverrides class MergeIntoNotMatchedBySourceCDCPart2SQLPathBasedDVsPredPushOffSuite extends MergeIntoNotMatchedBySourceCDCPart2Tests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownDisabled class MergeIntoNotMatchedBySourceCDCPart2SQLPathBasedDVsPredPushOnSuite extends MergeIntoNotMatchedBySourceCDCPart2Tests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownEnabled class MergeIntoNotMatchedBySourceCDCPart2SQLPathBasedSuite extends MergeIntoNotMatchedBySourceCDCPart2Tests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased class MergeIntoNotMatchedBySourceCDCPart2ScalaSuite extends MergeIntoNotMatchedBySourceCDCPart2Tests with MergeIntoScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNotMatchedBySourceSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoNotMatchedBySourceSQLNameBasedSuite extends MergeIntoNotMatchedBySourceSuite with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased class MergeIntoNotMatchedBySourceSQLPathBasedCDCOnDVsPredPushOffSuite extends MergeIntoNotMatchedBySourceSuite with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoNotMatchedBySourceSQLPathBasedCDCOnDVsPredPushOnSuite extends MergeIntoNotMatchedBySourceSuite with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoNotMatchedBySourceSQLPathBasedCDCOnSuite extends MergeIntoNotMatchedBySourceSuite with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeCDCMixin class MergeIntoNotMatchedBySourceSQLPathBasedColMapIdModeSuite extends MergeIntoNotMatchedBySourceSuite with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode with MergeIntoSQLColumnMappingOverrides class MergeIntoNotMatchedBySourceSQLPathBasedColMapNameModeSuite extends MergeIntoNotMatchedBySourceSuite with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with MergeIntoSQLColumnMappingOverrides class MergeIntoNotMatchedBySourceSQLPathBasedDVsPredPushOffSuite extends MergeIntoNotMatchedBySourceSuite with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownDisabled class MergeIntoNotMatchedBySourceSQLPathBasedDVsPredPushOnSuite extends MergeIntoNotMatchedBySourceSuite with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownEnabled class MergeIntoNotMatchedBySourceSQLPathBasedSuite extends MergeIntoNotMatchedBySourceSuite with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased class MergeIntoNotMatchedBySourceScalaSuite extends MergeIntoNotMatchedBySourceSuite with MergeIntoScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoSQLNondeterministicOrderTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoSQLNondeterministicOrderSQLNameBasedSuite extends MergeIntoSQLNondeterministicOrderTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased class MergeIntoSQLNondeterministicOrderSQLPathBasedCDCOnDVsPredPushOffSuite extends MergeIntoSQLNondeterministicOrderTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoSQLNondeterministicOrderSQLPathBasedCDCOnDVsPredPushOnSuite extends MergeIntoSQLNondeterministicOrderTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoSQLNondeterministicOrderSQLPathBasedCDCOnSuite extends MergeIntoSQLNondeterministicOrderTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeCDCMixin class MergeIntoSQLNondeterministicOrderSQLPathBasedColMapIdModeSuite extends MergeIntoSQLNondeterministicOrderTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode with MergeIntoSQLColumnMappingOverrides class MergeIntoSQLNondeterministicOrderSQLPathBasedColMapNameModeSuite extends MergeIntoSQLNondeterministicOrderTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with MergeIntoSQLColumnMappingOverrides class MergeIntoSQLNondeterministicOrderSQLPathBasedDVsPredPushOffSuite extends MergeIntoSQLNondeterministicOrderTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownDisabled class MergeIntoSQLNondeterministicOrderSQLPathBasedDVsPredPushOnSuite extends MergeIntoSQLNondeterministicOrderTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownEnabled class MergeIntoSQLNondeterministicOrderSQLPathBasedSuite extends MergeIntoSQLNondeterministicOrderTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoSQLTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoSQLSQLNameBasedSuite extends MergeIntoSQLTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased class MergeIntoSQLSQLPathBasedCDCOnDVsPredPushOffSuite extends MergeIntoSQLTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoSQLSQLPathBasedCDCOnDVsPredPushOnSuite extends MergeIntoSQLTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoSQLSQLPathBasedCDCOnSuite extends MergeIntoSQLTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeCDCMixin class MergeIntoSQLSQLPathBasedColMapIdModeSuite extends MergeIntoSQLTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode with MergeIntoSQLColumnMappingOverrides class MergeIntoSQLSQLPathBasedColMapNameModeSuite extends MergeIntoSQLTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with MergeIntoSQLColumnMappingOverrides class MergeIntoSQLSQLPathBasedDVsPredPushOffSuite extends MergeIntoSQLTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownDisabled class MergeIntoSQLSQLPathBasedDVsPredPushOnSuite extends MergeIntoSQLTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownEnabled class MergeIntoSQLSQLPathBasedSuite extends MergeIntoSQLTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoScalaTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoScalaScalaSuite extends MergeIntoScalaTests with MergeIntoScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoSchemaEvoStoreAssignmentPolicyTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoSchemaEvoStoreAssignmentPolicySQLNameBasedSuite extends MergeIntoSchemaEvoStoreAssignmentPolicyTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased class MergeIntoSchemaEvoStoreAssignmentPolicySQLPathBasedCDCOnDVsPredPushOffSuite extends MergeIntoSchemaEvoStoreAssignmentPolicyTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoSchemaEvoStoreAssignmentPolicySQLPathBasedCDCOnDVsPredPushOnSuite extends MergeIntoSchemaEvoStoreAssignmentPolicyTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoSchemaEvoStoreAssignmentPolicySQLPathBasedCDCOnSuite extends MergeIntoSchemaEvoStoreAssignmentPolicyTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeCDCMixin class MergeIntoSchemaEvoStoreAssignmentPolicySQLPathBasedColMapIdModeSuite extends MergeIntoSchemaEvoStoreAssignmentPolicyTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode with MergeIntoSQLColumnMappingOverrides class MergeIntoSchemaEvoStoreAssignmentPolicySQLPathBasedColMapNameModeSuite extends MergeIntoSchemaEvoStoreAssignmentPolicyTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with MergeIntoSQLColumnMappingOverrides class MergeIntoSchemaEvoStoreAssignmentPolicySQLPathBasedDVsPredPushOffSuite extends MergeIntoSchemaEvoStoreAssignmentPolicyTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownDisabled class MergeIntoSchemaEvoStoreAssignmentPolicySQLPathBasedDVsPredPushOnSuite extends MergeIntoSchemaEvoStoreAssignmentPolicyTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownEnabled class MergeIntoSchemaEvoStoreAssignmentPolicySQLPathBasedSuite extends MergeIntoSchemaEvoStoreAssignmentPolicyTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased class MergeIntoSchemaEvoStoreAssignmentPolicyScalaSuite extends MergeIntoSchemaEvoStoreAssignmentPolicyTests with MergeIntoScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoSchemaEvolutionBaseExistingColumnTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoSchemaEvolutionBaseExistingColumnSQLNameBasedSuite extends MergeIntoSchemaEvolutionBaseExistingColumnTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased class MergeIntoSchemaEvolutionBaseExistingColumnSQLPathBasedCDCOnDVsPredPushOffSuite extends MergeIntoSchemaEvolutionBaseExistingColumnTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoSchemaEvolutionBaseExistingColumnSQLPathBasedCDCOnDVsPredPushOnSuite extends MergeIntoSchemaEvolutionBaseExistingColumnTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoSchemaEvolutionBaseExistingColumnSQLPathBasedCDCOnSuite extends MergeIntoSchemaEvolutionBaseExistingColumnTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeCDCMixin class MergeIntoSchemaEvolutionBaseExistingColumnSQLPathBasedColMapIdModeSuite extends MergeIntoSchemaEvolutionBaseExistingColumnTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode with MergeIntoSQLColumnMappingOverrides class MergeIntoSchemaEvolutionBaseExistingColumnSQLPathBasedColMapNameModeSuite extends MergeIntoSchemaEvolutionBaseExistingColumnTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with MergeIntoSQLColumnMappingOverrides class MergeIntoSchemaEvolutionBaseExistingColumnSQLPathBasedDVsPredPushOffSuite extends MergeIntoSchemaEvolutionBaseExistingColumnTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownDisabled class MergeIntoSchemaEvolutionBaseExistingColumnSQLPathBasedDVsPredPushOnSuite extends MergeIntoSchemaEvolutionBaseExistingColumnTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownEnabled class MergeIntoSchemaEvolutionBaseExistingColumnSQLPathBasedSuite extends MergeIntoSchemaEvolutionBaseExistingColumnTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased class MergeIntoSchemaEvolutionBaseExistingColumnScalaSuite extends MergeIntoSchemaEvolutionBaseExistingColumnTests with MergeIntoScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoSchemaEvolutionBaseNewColumnTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoSchemaEvolutionBaseNewColumnSQLNameBasedSuite extends MergeIntoSchemaEvolutionBaseNewColumnTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased class MergeIntoSchemaEvolutionBaseNewColumnSQLPathBasedCDCOnDVsPredPushOffSuite extends MergeIntoSchemaEvolutionBaseNewColumnTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoSchemaEvolutionBaseNewColumnSQLPathBasedCDCOnDVsPredPushOnSuite extends MergeIntoSchemaEvolutionBaseNewColumnTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoSchemaEvolutionBaseNewColumnSQLPathBasedCDCOnSuite extends MergeIntoSchemaEvolutionBaseNewColumnTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeCDCMixin class MergeIntoSchemaEvolutionBaseNewColumnSQLPathBasedColMapIdModeSuite extends MergeIntoSchemaEvolutionBaseNewColumnTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode with MergeIntoSQLColumnMappingOverrides class MergeIntoSchemaEvolutionBaseNewColumnSQLPathBasedColMapNameModeSuite extends MergeIntoSchemaEvolutionBaseNewColumnTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with MergeIntoSQLColumnMappingOverrides class MergeIntoSchemaEvolutionBaseNewColumnSQLPathBasedDVsPredPushOffSuite extends MergeIntoSchemaEvolutionBaseNewColumnTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownDisabled class MergeIntoSchemaEvolutionBaseNewColumnSQLPathBasedDVsPredPushOnSuite extends MergeIntoSchemaEvolutionBaseNewColumnTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownEnabled class MergeIntoSchemaEvolutionBaseNewColumnSQLPathBasedSuite extends MergeIntoSchemaEvolutionBaseNewColumnTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased class MergeIntoSchemaEvolutionBaseNewColumnScalaSuite extends MergeIntoSchemaEvolutionBaseNewColumnTests with MergeIntoScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoSchemaEvolutionCoreTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoSchemaEvolutionCoreSQLNameBasedSuite extends MergeIntoSchemaEvolutionCoreTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased class MergeIntoSchemaEvolutionCoreSQLPathBasedCDCOnDVsPredPushOffSuite extends MergeIntoSchemaEvolutionCoreTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoSchemaEvolutionCoreSQLPathBasedCDCOnDVsPredPushOnSuite extends MergeIntoSchemaEvolutionCoreTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoSchemaEvolutionCoreSQLPathBasedCDCOnSuite extends MergeIntoSchemaEvolutionCoreTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeCDCMixin class MergeIntoSchemaEvolutionCoreSQLPathBasedColMapIdModeSuite extends MergeIntoSchemaEvolutionCoreTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode with MergeIntoSQLColumnMappingOverrides class MergeIntoSchemaEvolutionCoreSQLPathBasedColMapNameModeSuite extends MergeIntoSchemaEvolutionCoreTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with MergeIntoSQLColumnMappingOverrides class MergeIntoSchemaEvolutionCoreSQLPathBasedDVsPredPushOffSuite extends MergeIntoSchemaEvolutionCoreTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownDisabled class MergeIntoSchemaEvolutionCoreSQLPathBasedDVsPredPushOnSuite extends MergeIntoSchemaEvolutionCoreTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownEnabled class MergeIntoSchemaEvolutionCoreSQLPathBasedSuite extends MergeIntoSchemaEvolutionCoreTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased class MergeIntoSchemaEvolutionCoreScalaSuite extends MergeIntoSchemaEvolutionCoreTests with MergeIntoScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoSchemaEvolutionNotMatchedBySourceTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoSchemaEvolutionNotMatchedBySourceSQLNameBasedSuite extends MergeIntoSchemaEvolutionNotMatchedBySourceTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased class MergeIntoSchemaEvolutionNotMatchedBySourceSQLPathBasedCDCOnDVsPredPushOffSuite extends MergeIntoSchemaEvolutionNotMatchedBySourceTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoSchemaEvolutionNotMatchedBySourceSQLPathBasedCDCOnDVsPredPushOnSuite extends MergeIntoSchemaEvolutionNotMatchedBySourceTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoSchemaEvolutionNotMatchedBySourceSQLPathBasedCDCOnSuite extends MergeIntoSchemaEvolutionNotMatchedBySourceTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeCDCMixin class MergeIntoSchemaEvolutionNotMatchedBySourceSQLPathBasedColMapIdModeSuite extends MergeIntoSchemaEvolutionNotMatchedBySourceTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode with MergeIntoSQLColumnMappingOverrides class MergeIntoSchemaEvolutionNotMatchedBySourceSQLPathBasedColMapNameModeSuite extends MergeIntoSchemaEvolutionNotMatchedBySourceTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with MergeIntoSQLColumnMappingOverrides class MergeIntoSchemaEvolutionNotMatchedBySourceSQLPathBasedDVsPredPushOffSuite extends MergeIntoSchemaEvolutionNotMatchedBySourceTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownDisabled class MergeIntoSchemaEvolutionNotMatchedBySourceSQLPathBasedDVsPredPushOnSuite extends MergeIntoSchemaEvolutionNotMatchedBySourceTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownEnabled class MergeIntoSchemaEvolutionNotMatchedBySourceSQLPathBasedSuite extends MergeIntoSchemaEvolutionNotMatchedBySourceTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased class MergeIntoSchemaEvolutionNotMatchedBySourceScalaSuite extends MergeIntoSchemaEvolutionNotMatchedBySourceTests with MergeIntoScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoStructEvolutionNullnessMultiClauseTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedSuite extends MergeIntoStructEvolutionNullnessMultiClauseTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoSuiteBaseMiscTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoSuiteBaseMiscSQLNameBasedSuite extends MergeIntoSuiteBaseMiscTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased class MergeIntoSuiteBaseMiscSQLPathBasedCDCOnDVsPredPushOffSuite extends MergeIntoSuiteBaseMiscTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoSuiteBaseMiscSQLPathBasedCDCOnDVsPredPushOnSuite extends MergeIntoSuiteBaseMiscTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoSuiteBaseMiscSQLPathBasedCDCOnSuite extends MergeIntoSuiteBaseMiscTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeCDCMixin class MergeIntoSuiteBaseMiscSQLPathBasedColMapIdModeSuite extends MergeIntoSuiteBaseMiscTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode with MergeIntoSQLColumnMappingOverrides class MergeIntoSuiteBaseMiscSQLPathBasedColMapNameModeSuite extends MergeIntoSuiteBaseMiscTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with MergeIntoSQLColumnMappingOverrides class MergeIntoSuiteBaseMiscSQLPathBasedDVsPredPushOffSuite extends MergeIntoSuiteBaseMiscTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownDisabled class MergeIntoSuiteBaseMiscSQLPathBasedDVsPredPushOnSuite extends MergeIntoSuiteBaseMiscTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownEnabled class MergeIntoSuiteBaseMiscSQLPathBasedSuite extends MergeIntoSuiteBaseMiscTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased class MergeIntoSuiteBaseMiscScalaSuite extends MergeIntoSuiteBaseMiscTests with MergeIntoScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTempViewsTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoTempViewsSQLNameBasedSuite extends MergeIntoTempViewsTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased class MergeIntoTempViewsSQLPathBasedCDCOnDVsPredPushOffSuite extends MergeIntoTempViewsTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoTempViewsSQLPathBasedCDCOnDVsPredPushOnSuite extends MergeIntoTempViewsTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoTempViewsSQLPathBasedCDCOnSuite extends MergeIntoTempViewsTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeCDCMixin class MergeIntoTempViewsSQLPathBasedColMapIdModeSuite extends MergeIntoTempViewsTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode with MergeIntoSQLColumnMappingOverrides class MergeIntoTempViewsSQLPathBasedColMapNameModeSuite extends MergeIntoTempViewsTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with MergeIntoSQLColumnMappingOverrides class MergeIntoTempViewsSQLPathBasedDVsPredPushOffSuite extends MergeIntoTempViewsTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownDisabled class MergeIntoTempViewsSQLPathBasedDVsPredPushOnSuite extends MergeIntoTempViewsTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownEnabled class MergeIntoTempViewsSQLPathBasedSuite extends MergeIntoTempViewsTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelArrayStructEvolutionNullnessTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedSuite extends MergeIntoTopLevelArrayStructEvolutionNullnessTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelMapStructEvolutionNullnessTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedSuite extends MergeIntoTopLevelMapStructEvolutionNullnessTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelStructEvolutionNullnessTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedSuite extends MergeIntoTopLevelStructEvolutionNullnessTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoUnlimitedMergeClausesTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class MergeIntoUnlimitedMergeClausesSQLNameBasedSuite extends MergeIntoUnlimitedMergeClausesTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased class MergeIntoUnlimitedMergeClausesSQLPathBasedCDCOnDVsPredPushOffSuite extends MergeIntoUnlimitedMergeClausesTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownDisabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoUnlimitedMergeClausesSQLPathBasedCDCOnDVsPredPushOnSuite extends MergeIntoUnlimitedMergeClausesTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeIntoDVsMixin with PredicatePushdownEnabled with MergeCDCMixin with MergeCDCWithDVsMixin class MergeIntoUnlimitedMergeClausesSQLPathBasedCDCOnSuite extends MergeIntoUnlimitedMergeClausesTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with MergeCDCMixin class MergeIntoUnlimitedMergeClausesSQLPathBasedColMapIdModeSuite extends MergeIntoUnlimitedMergeClausesTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableIdMode with MergeIntoSQLColumnMappingOverrides class MergeIntoUnlimitedMergeClausesSQLPathBasedColMapNameModeSuite extends MergeIntoUnlimitedMergeClausesTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with DeltaColumnMappingEnableNameMode with MergeIntoSQLColumnMappingOverrides class MergeIntoUnlimitedMergeClausesSQLPathBasedDVsPredPushOffSuite extends MergeIntoUnlimitedMergeClausesTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownDisabled class MergeIntoUnlimitedMergeClausesSQLPathBasedDVsPredPushOnSuite extends MergeIntoUnlimitedMergeClausesTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased with MergeIntoDVsMixin with PredicatePushdownEnabled class MergeIntoUnlimitedMergeClausesSQLPathBasedSuite extends MergeIntoUnlimitedMergeClausesTests with MergeIntoSQLMixin with DeltaDMLTestUtilsPathBased class MergeIntoUnlimitedMergeClausesScalaSuite extends MergeIntoUnlimitedMergeClausesTests with MergeIntoScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesRowTrackingMergeCommonTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ class RowTrackingMergeCommonNameBasedCDCOnDVOffMergePersistentDVOffSuite extends RowTrackingMergeCommonTests with DeltaDMLTestUtilsNameBased with CDCEnabled with PersistentDVDisabled with MergePersistentDVDisabled class RowTrackingMergeCommonNameBasedCDCOnRowTrackingMergeDVSuite extends RowTrackingMergeCommonTests with DeltaDMLTestUtilsNameBased with CDCEnabled with RowTrackingMergeDVMixin class RowTrackingMergeCommonNameBasedCDCOnSuite extends RowTrackingMergeCommonTests with DeltaDMLTestUtilsNameBased with CDCEnabled class RowTrackingMergeCommonNameBasedColMapIdModeCDCOnRowTrackingMergeDVSuite extends RowTrackingMergeCommonTests with DeltaDMLTestUtilsNameBased with DeltaColumnMappingEnableIdMode with CDCEnabled with RowTrackingMergeDVMixin class RowTrackingMergeCommonNameBasedColMapIdModeSuite extends RowTrackingMergeCommonTests with DeltaDMLTestUtilsNameBased with DeltaColumnMappingEnableIdMode class RowTrackingMergeCommonNameBasedColMapNameModeCDCOnRowTrackingMergeDVSuite extends RowTrackingMergeCommonTests with DeltaDMLTestUtilsNameBased with DeltaColumnMappingEnableNameMode with CDCEnabled with RowTrackingMergeDVMixin class RowTrackingMergeCommonNameBasedColMapNameModeSuite extends RowTrackingMergeCommonTests with DeltaDMLTestUtilsNameBased with DeltaColumnMappingEnableNameMode class RowTrackingMergeCommonNameBasedDVOffMergePersistentDVOffSuite extends RowTrackingMergeCommonTests with DeltaDMLTestUtilsNameBased with PersistentDVDisabled with MergePersistentDVDisabled class RowTrackingMergeCommonNameBasedRowTrackingMergeDVSuite extends RowTrackingMergeCommonTests with DeltaDMLTestUtilsNameBased with RowTrackingMergeDVMixin class RowTrackingMergeCommonNameBasedSuite extends RowTrackingMergeCommonTests with DeltaDMLTestUtilsNameBased ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/UpdateSuitesRowTrackingUpdateCommonTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ import org.apache.spark.sql.delta.rowtracking._ class RowTrackingUpdateCommonCDCOnColMapIdModeSuite extends RowTrackingUpdateCommonTests with CDCEnabled with DeltaColumnMappingEnableIdMode class RowTrackingUpdateCommonCDCOnColMapNameModeSuite extends RowTrackingUpdateCommonTests with CDCEnabled with DeltaColumnMappingEnableNameMode class RowTrackingUpdateCommonCDCOnSuite extends RowTrackingUpdateCommonTests with CDCEnabled class RowTrackingUpdateCommonColMapIdModeSuite extends RowTrackingUpdateCommonTests with DeltaColumnMappingEnableIdMode class RowTrackingUpdateCommonColMapNameModeSuite extends RowTrackingUpdateCommonTests with DeltaColumnMappingEnableNameMode class RowTrackingUpdateCommonRowTrackingUpdateDVCDCOnColMapIdModeSuite extends RowTrackingUpdateCommonTests with RowTrackingUpdateDVMixin with CDCEnabled with DeltaColumnMappingEnableIdMode class RowTrackingUpdateCommonRowTrackingUpdateDVCDCOnColMapNameModeSuite extends RowTrackingUpdateCommonTests with RowTrackingUpdateDVMixin with CDCEnabled with DeltaColumnMappingEnableNameMode class RowTrackingUpdateCommonRowTrackingUpdateDVCDCOnSuite extends RowTrackingUpdateCommonTests with RowTrackingUpdateDVMixin with CDCEnabled class RowTrackingUpdateCommonRowTrackingUpdateDVSuite extends RowTrackingUpdateCommonTests with RowTrackingUpdateDVMixin class RowTrackingUpdateCommonSuite extends RowTrackingUpdateCommonTests ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/UpdateSuitesUpdateBaseMiscTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ import org.apache.spark.sql.delta.rowtracking._ class UpdateBaseMiscSQLNameBasedSuite extends UpdateBaseMiscTests with UpdateSQLMixin with DeltaDMLTestUtilsNameBased class UpdateBaseMiscSQLPathBasedCDCOnDVSuite extends UpdateBaseMiscTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with UpdateSQLWithDeletionVectorsMixin class UpdateBaseMiscSQLPathBasedCDCOnRowTrackingOffSuite extends UpdateBaseMiscTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with RowTrackingDisabled class UpdateBaseMiscSQLPathBasedCDCOnRowTrackingOnSuite extends UpdateBaseMiscTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with RowTrackingEnabled with UpdateWithRowTrackingOverrides class UpdateBaseMiscSQLPathBasedCDCOnSuite extends UpdateBaseMiscTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled class UpdateBaseMiscSQLPathBasedDVPredPushOffSuite extends UpdateBaseMiscTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with UpdateSQLWithDeletionVectorsMixin with PredicatePushdownDisabled class UpdateBaseMiscSQLPathBasedDVPredPushOnSuite extends UpdateBaseMiscTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with UpdateSQLWithDeletionVectorsMixin with PredicatePushdownEnabled class UpdateBaseMiscSQLPathBasedRowTrackingOffSuite extends UpdateBaseMiscTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with RowTrackingDisabled class UpdateBaseMiscSQLPathBasedRowTrackingOnSuite extends UpdateBaseMiscTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with RowTrackingEnabled with UpdateWithRowTrackingOverrides class UpdateBaseMiscSQLPathBasedSuite extends UpdateBaseMiscTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased class UpdateBaseMiscScalaSuite extends UpdateBaseMiscTests with UpdateScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/UpdateSuitesUpdateBaseTempViewTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ import org.apache.spark.sql.delta.rowtracking._ class UpdateBaseTempViewSQLNameBasedSuite extends UpdateBaseTempViewTests with UpdateSQLMixin with DeltaDMLTestUtilsNameBased class UpdateBaseTempViewSQLPathBasedCDCOnDVSuite extends UpdateBaseTempViewTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with UpdateSQLWithDeletionVectorsMixin class UpdateBaseTempViewSQLPathBasedCDCOnRowTrackingOffSuite extends UpdateBaseTempViewTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with RowTrackingDisabled class UpdateBaseTempViewSQLPathBasedCDCOnRowTrackingOnSuite extends UpdateBaseTempViewTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with RowTrackingEnabled with UpdateWithRowTrackingOverrides class UpdateBaseTempViewSQLPathBasedCDCOnSuite extends UpdateBaseTempViewTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled class UpdateBaseTempViewSQLPathBasedDVPredPushOffSuite extends UpdateBaseTempViewTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with UpdateSQLWithDeletionVectorsMixin with PredicatePushdownDisabled class UpdateBaseTempViewSQLPathBasedDVPredPushOnSuite extends UpdateBaseTempViewTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with UpdateSQLWithDeletionVectorsMixin with PredicatePushdownEnabled class UpdateBaseTempViewSQLPathBasedRowTrackingOffSuite extends UpdateBaseTempViewTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with RowTrackingDisabled class UpdateBaseTempViewSQLPathBasedRowTrackingOnSuite extends UpdateBaseTempViewTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with RowTrackingEnabled with UpdateWithRowTrackingOverrides class UpdateBaseTempViewSQLPathBasedSuite extends UpdateBaseTempViewTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/UpdateSuitesUpdateCDCTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ import org.apache.spark.sql.delta.rowtracking._ class UpdateCDCSQLPathBasedCDCOnDVSuite extends UpdateCDCTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with UpdateSQLWithDeletionVectorsMixin class UpdateCDCSQLPathBasedCDCOnRowTrackingOffSuite extends UpdateCDCTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with RowTrackingDisabled class UpdateCDCSQLPathBasedCDCOnRowTrackingOnSuite extends UpdateCDCTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with RowTrackingEnabled with UpdateWithRowTrackingOverrides class UpdateCDCSQLPathBasedCDCOnSuite extends UpdateCDCTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/UpdateSuitesUpdateCDCWithDeletionVectorsTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ import org.apache.spark.sql.delta.rowtracking._ class UpdateCDCWithDeletionVectorsSQLPathBasedCDCOnDVSuite extends UpdateCDCWithDeletionVectorsTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with UpdateSQLWithDeletionVectorsMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/UpdateSuitesUpdateSQLTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ import org.apache.spark.sql.delta.rowtracking._ class UpdateSQLSQLNameBasedSuite extends UpdateSQLTests with UpdateSQLMixin with DeltaDMLTestUtilsNameBased class UpdateSQLSQLPathBasedCDCOnDVSuite extends UpdateSQLTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with UpdateSQLWithDeletionVectorsMixin class UpdateSQLSQLPathBasedCDCOnRowTrackingOffSuite extends UpdateSQLTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with RowTrackingDisabled class UpdateSQLSQLPathBasedCDCOnRowTrackingOnSuite extends UpdateSQLTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with RowTrackingEnabled with UpdateWithRowTrackingOverrides class UpdateSQLSQLPathBasedCDCOnSuite extends UpdateSQLTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled class UpdateSQLSQLPathBasedDVPredPushOffSuite extends UpdateSQLTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with UpdateSQLWithDeletionVectorsMixin with PredicatePushdownDisabled class UpdateSQLSQLPathBasedDVPredPushOnSuite extends UpdateSQLTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with UpdateSQLWithDeletionVectorsMixin with PredicatePushdownEnabled class UpdateSQLSQLPathBasedRowTrackingOffSuite extends UpdateSQLTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with RowTrackingDisabled class UpdateSQLSQLPathBasedRowTrackingOnSuite extends UpdateSQLTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with RowTrackingEnabled with UpdateWithRowTrackingOverrides class UpdateSQLSQLPathBasedSuite extends UpdateSQLTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/UpdateSuitesUpdateSQLWithDeletionVectorsTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ import org.apache.spark.sql.delta.rowtracking._ class UpdateSQLWithDeletionVectorsSQLPathBasedCDCOnDVSuite extends UpdateSQLWithDeletionVectorsTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with CDCEnabled with UpdateSQLWithDeletionVectorsMixin class UpdateSQLWithDeletionVectorsSQLPathBasedDVPredPushOffSuite extends UpdateSQLWithDeletionVectorsTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with UpdateSQLWithDeletionVectorsMixin with PredicatePushdownDisabled class UpdateSQLWithDeletionVectorsSQLPathBasedDVPredPushOnSuite extends UpdateSQLWithDeletionVectorsTests with UpdateSQLMixin with DeltaDMLTestUtilsPathBased with UpdateSQLWithDeletionVectorsMixin with PredicatePushdownEnabled ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/UpdateSuitesUpdateScalaTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // *********************************************************************************** // * This file is automatically generated. Manual modification is not allowed. * // * There is a unit test that should prevent merging a manual change. * // * * // * To make changes to the suites, modify the generator script config at * // * SuiteGeneratorConfig.scala and run it. The generator can be run via the * // * sbt command deltaSuiteGenerator / run. * // * * // * DO NOT TOUCH ANYTHING IN THIS FILE! * // *********************************************************************************** // scalastyle:off line.size.limit package org.apache.spark.sql.delta.generatedsuites import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ import org.apache.spark.sql.delta.rowtracking._ class UpdateScalaScalaSuite extends UpdateScalaTests with UpdateScalaMixin ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/logging/DeltaStructuredLoggingSuite.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.logging import java.io.File import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.regex.Pattern import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.scala.DefaultScalaModule import org.apache.logging.log4j.Level import org.apache.spark.SparkFunSuite import org.apache.spark.internal.{LogEntry, Logging, LogKey, MDC} class DeltaStructuredLoggingSuite extends SparkFunSuite with Logging { private def className: String = classOf[DeltaStructuredLoggingSuite].getSimpleName private def logFilePath: String = "target/structured.log" private lazy val logFile: File = { val pwd = new File(".").getCanonicalPath new File(pwd + "/" + logFilePath) } override def beforeAll(): Unit = { super.beforeAll() Logging.enableStructuredLogging() } override def afterAll(): Unit = { Logging.disableStructuredLogging() super.afterAll() } private val jsonMapper = new ObjectMapper().registerModule(DefaultScalaModule) private def compactAndToRegexPattern(json: String): String = { jsonMapper.readTree(json).toString. replace("", """[^"]+"""). replace("""""""", """.*"""). replace("{", """\{""") + "\n" } // Return the newly added log contents in the log file after executing the function `f` private def captureLogOutput(f: () => Unit): String = { val content = if (logFile.exists()) { new String(Files.readAllBytes(logFile.toPath), StandardCharsets.UTF_8) } else { "" } f() val newContent = new String(Files.readAllBytes(logFile.toPath), StandardCharsets.UTF_8) newContent.substring(content.length) } private def basicMsg: String = "This is a log message" private def msgWithMDC: LogEntry = log"Lost executor ${MDC(DeltaLogKeys.EXECUTOR_ID, "1")}." private def msgWithMDCValueIsNull: LogEntry = log"Lost executor ${MDC(DeltaLogKeys.EXECUTOR_ID, null)}." private def msgWithMDCAndException: LogEntry = log"Error in executor ${MDC(DeltaLogKeys.EXECUTOR_ID, "1")}." private def msgWithConcat: LogEntry = log"Min Size: ${MDC(DeltaLogKeys.MIN_SIZE, "2")}, " + log"Max Size: ${MDC(DeltaLogKeys.MAX_SIZE, "4")}. " + log"Please double check." private val customLog = log"${MDC(CustomLogKeys.CUSTOM_LOG_KEY, "Custom log message.")}" def expectedPatternForBasicMsg(level: Level): String = { compactAndToRegexPattern( s""" { "ts": "", "level": "$level", "msg": "This is a log message", "logger": "$className" }""") } def expectedPatternForBasicMsgWithException(level: Level): String = { compactAndToRegexPattern( s""" { "ts": "", "level": "$level", "msg": "This is a log message", "exception": { "class": "java.lang.RuntimeException", "msg": "OOM", "stacktrace": "" }, "logger": "$className" }""") } def expectedPatternForMsgWithMDC(level: Level): String = { compactAndToRegexPattern( s""" { "ts": "", "level": "$level", "msg": "Lost executor 1.", "context": { "executor_id": "1" }, "logger": "$className" }""") } def expectedPatternForMsgWithMDCValueIsNull(level: Level): String = { compactAndToRegexPattern( s""" { "ts": "", "level": "$level", "msg": "Lost executor null.", "context": { "executor_id": null }, "logger": "$className" }""") } def expectedPatternForMsgWithMDCAndException(level: Level): String = { compactAndToRegexPattern( s""" { "ts": "", "level": "$level", "msg": "Error in executor 1.", "context": { "executor_id": "1" }, "exception": { "class": "java.lang.RuntimeException", "msg": "OOM", "stacktrace": "" }, "logger": "$className" }""") } def expectedPatternForCustomLogKey(level: Level): String = { compactAndToRegexPattern( s""" { "ts": "", "level": "$level", "msg": "Custom log message.", "logger": "$className" }""" ) } def verifyMsgWithConcat(level: Level, logOutput: String): Unit = { val pattern1 = compactAndToRegexPattern( s""" { "ts": "", "level": "$level", "msg": "Min Size: 2, Max Size: 4. Please double check.", "context": { "min_size": "2", "max_size": "4" }, "logger": "$className" }""") val pattern2 = compactAndToRegexPattern( s""" { "ts": "", "level": "$level", "msg": "Min Size: 2, Max Size: 4. Please double check.", "context": { "max_size": "4", "min_size": "2" }, "logger": "$className" }""") assert(Pattern.compile(pattern1).matcher(logOutput).matches() || Pattern.compile(pattern2).matcher(logOutput).matches()) } test("Basic logging") { Seq( (Level.ERROR, () => logError(basicMsg)), (Level.WARN, () => logWarning(basicMsg)), (Level.INFO, () => logInfo(basicMsg))).foreach { case (level, logFunc) => val logOutput = captureLogOutput(logFunc) assert(Pattern.compile(expectedPatternForBasicMsg(level)).matcher(logOutput).matches()) } } test("Basic logging with Exception") { val exception = new RuntimeException("OOM") Seq( (Level.ERROR, () => logError(basicMsg, exception)), (Level.WARN, () => logWarning(basicMsg, exception)), (Level.INFO, () => logInfo(basicMsg, exception))).foreach { case (level, logFunc) => val logOutput = captureLogOutput(logFunc) assert( Pattern.compile(expectedPatternForBasicMsgWithException(level)).matcher(logOutput) .matches()) } } test("Logging with MDC") { Seq( (Level.ERROR, () => logError(msgWithMDC)), (Level.WARN, () => logWarning(msgWithMDC)), (Level.INFO, () => logInfo(msgWithMDC))).foreach { case (level, logFunc) => val logOutput = captureLogOutput(logFunc) assert( Pattern.compile(expectedPatternForMsgWithMDC(level)).matcher(logOutput).matches()) } } test("Logging with MDC(the value is null)") { Seq( (Level.ERROR, () => logError(msgWithMDCValueIsNull)), (Level.WARN, () => logWarning(msgWithMDCValueIsNull)), (Level.INFO, () => logInfo(msgWithMDCValueIsNull))).foreach { case (level, logFunc) => val logOutput = captureLogOutput(logFunc) assert( Pattern.compile(expectedPatternForMsgWithMDCValueIsNull(level)).matcher(logOutput) .matches()) } } test("Logging with MDC and Exception") { val exception = new RuntimeException("OOM") Seq( (Level.ERROR, () => logError(msgWithMDCAndException, exception)), (Level.WARN, () => logWarning(msgWithMDCAndException, exception)), (Level.INFO, () => logInfo(msgWithMDCAndException, exception))).foreach { case (level, logFunc) => val logOutput = captureLogOutput(logFunc) assert( Pattern.compile(expectedPatternForMsgWithMDCAndException(level)).matcher(logOutput) .matches()) } } test("Logging with custom LogKey") { Seq( (Level.ERROR, () => logError(customLog)), (Level.WARN, () => logWarning(customLog)), (Level.INFO, () => logInfo(customLog))).foreach { case (level, logFunc) => val logOutput = captureLogOutput(logFunc) assert(Pattern.compile(expectedPatternForCustomLogKey(level)).matcher(logOutput).matches()) } } test("Logging with concat") { Seq( (Level.ERROR, () => logError(msgWithConcat)), (Level.WARN, () => logWarning(msgWithConcat)), (Level.INFO, () => logInfo(msgWithConcat))).foreach { case (level, logFunc) => val logOutput = captureLogOutput(logFunc) verifyMsgWithConcat(level, logOutput) } } } object CustomLogKeys { // Custom `LogKey` must extend LogKey case object CUSTOM_LOG_KEY extends DeltaLogKey } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/logging/LogThrottlingSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.logging import scala.concurrent.duration._ import org.apache.spark.sql.delta.metering.{DeadlineWithTimeSource, LogThrottler, NanoTimeTimeSource} import org.apache.spark.SparkFunSuite class LogThrottlingSuite extends SparkFunSuite { // Make sure that the helper works right. test("time control") { val nanoTimeControl = new MockedNanoTime assert(nanoTimeControl.nanoTime() === 0L) assert(nanoTimeControl.nanoTime() === 0L) nanoTimeControl.advance(112L.nanos) assert(nanoTimeControl.nanoTime() === 112L) assert(nanoTimeControl.nanoTime() === 112L) } test("deadline with time control") { val nanoTimeControl = new MockedNanoTime assert(DeadlineWithTimeSource.now(nanoTimeControl).isOverdue()) val deadline = DeadlineWithTimeSource.now(nanoTimeControl) + 5.nanos assert(!deadline.isOverdue()) nanoTimeControl.advance(5.nanos) assert(deadline.isOverdue()) nanoTimeControl.advance(5.nanos) assert(deadline.isOverdue()) // Check addition. assert(deadline + 0.nanos === deadline) val increasedDeadline = deadline + 10.nanos assert(!increasedDeadline.isOverdue()) nanoTimeControl.advance(5.nanos) assert(increasedDeadline.isOverdue()) // Ensure that wrapping keeps throwing this exact exception, since we rely on it in // LogThrottler.tryRecoverTokens assertThrows[IllegalArgumentException] { deadline + Long.MaxValue.nanos } // Check difference and ordering. assert(deadline - deadline === 0.nanos) assert(increasedDeadline - deadline === 10.nanos) assert(increasedDeadline - deadline > 9.nanos) assert(increasedDeadline - deadline < 11.nanos) assert(deadline - increasedDeadline === -10.nanos) assert(deadline - increasedDeadline < -9.nanos) assert(deadline - increasedDeadline > -11.nanos) } test("unthrottled, no burst") { val nanotTimeControl = new MockedNanoTime val throttler = new LogThrottler( bucketSize = 1, tokenRecoveryInterval = 5.nanos, timeSource = nanotTimeControl) val numInvocations = 100 var timesExecuted = 0 for (i <- 0 until numInvocations) { throttler.throttled { skipped => assert(skipped === 0L) timesExecuted += 1 } nanotTimeControl.advance(5.nanos) } assert(timesExecuted === numInvocations) } test("unthrottled, burst") { val nanotTimeControl = new MockedNanoTime val throttler = new LogThrottler( bucketSize = 100, tokenRecoveryInterval = 1000000.nanos, // Just to make it obvious that it's a large number. timeSource = nanotTimeControl) val numInvocations = 100 var timesExecuted = 0 for (_ <- 0 until numInvocations) { throttler.throttled { skipped => assert(skipped === 0L) timesExecuted += 1 } nanotTimeControl.advance(5.nanos) } assert(timesExecuted === numInvocations) } test("throttled, no burst") { val nanoTimeControl = new MockedNanoTime val throttler = new LogThrottler( bucketSize = 1, tokenRecoveryInterval = 5.nanos, timeSource = nanoTimeControl) val numInvocations = 100 var timesExecuted = 0 for (i <- 0 until numInvocations) { throttler.throttled { skipped => if (timesExecuted == 0) { assert(skipped === 0L) } else { assert(skipped === 4L) } timesExecuted += 1 } nanoTimeControl.advance(1.nanos) } assert(timesExecuted === numInvocations / 5) } test("throttled, single burst") { val nanoTimeControl = new MockedNanoTime val throttler = new LogThrottler( bucketSize = 5, tokenRecoveryInterval = 10.nanos, timeSource = nanoTimeControl) val numInvocations = 100 var timesExecuted = 0 for (i <- 0 until numInvocations) { throttler.throttled { skipped => if (i < 5) { // First burst assert(skipped === 0L) } else if (i == 10) { // First token recovery assert(skipped === 5L) } else { // All other token recoveries assert(skipped === 9L) } timesExecuted += 1 } nanoTimeControl.advance(1.nano) } // A burst of 5 and then 1 every 10ns/invocations. assert(timesExecuted === 5 + (numInvocations - 10) / 10) } test("throttled, bursty") { val nanoTimeControl = new MockedNanoTime val throttler = new LogThrottler( bucketSize = 5, tokenRecoveryInterval = 10.nanos, timeSource = nanoTimeControl) val numBursts = 10 val numInvocationsPerBurst = 10 var timesExecuted = 0 for (burst <- 0 until numBursts) { for (i <- 0 until numInvocationsPerBurst) { throttler.throttled { skipped => if (i == 0 && burst != 0) { // first after recovery assert(skipped === 5L) } else { // either first burst, or post-recovery on every other burst. assert(skipped === 0L) } timesExecuted += 1 } nanoTimeControl.advance(1.nano) } nanoTimeControl.advance(100.nanos) } // Bursts of 5. assert(timesExecuted === 5 * numBursts) } test("wraparound") { val nanoTimeControl = new MockedNanoTime val throttler = new LogThrottler( bucketSize = 1, tokenRecoveryInterval = 100.nanos, timeSource = nanoTimeControl) def executeThrottled(expectedSkipped: Long = 0L): Boolean = { var executed = false throttler.throttled { skipped => assert(skipped === expectedSkipped) executed = true } executed } assert(executeThrottled()) assert(!executeThrottled()) // Move to 2 ns before wrapping. nanoTimeControl.advance((Long.MaxValue - 1L).nanos) assert(executeThrottled(expectedSkipped = 1L)) assert(!executeThrottled()) nanoTimeControl.advance(1.nano) assert(!executeThrottled()) // Wrapping nanoTimeControl.advance(1.nano) assert(!executeThrottled()) // Recover nanoTimeControl.advance(100.nanos) assert(executeThrottled(expectedSkipped = 3L)) } } /** * Use a mocked object to replace calls to `System.nanoTime()` with a custom value that can be * controlled by calling `advance(nanos)` on an instance of this class. */ class MockedNanoTime extends NanoTimeTimeSource { private var currentTimeNs: Long = 0L override def nanoTime(): Long = currentTimeNs def advance(time: FiniteDuration): Unit = { currentTimeNs += time.toNanos } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/metric/IncrementMetricSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.metric import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.SparkConf import org.apache.spark.sql.{AnalysisException, Column, DataFrame, QueryTest} import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.metric.SQLMetrics import org.apache.spark.sql.functions._ import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.BooleanType abstract class IncrementMetricSuiteBase extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { import testImplicits._ import SQLMetrics._ val ROWS_IN_DF = 1000 protected override def beforeAll(): Unit = { super.beforeAll() spark.range(ROWS_IN_DF).toDF("a") .withColumn("gb", rand(0).multiply(10).cast("integer")) .write .format("parquet") .mode("overwrite") .save("test-df") } def testDf: DataFrame = spark.read.format("parquet").load("test-df") test("Increment the same metric") { val metric = createMetric(sparkContext, "metric") val increment = IncrementMetric(Literal(true), metric) val groupByKey = IncrementMetric(UnresolvedAttribute("gb"), metric) val havingIncrement = IncrementMetric( GreaterThan(UnresolvedAttribute("s"), Literal(10)), metric) val df = testDf .filter(Column(increment)) .groupBy(Column(groupByKey).as("gby")) .agg(sum("a").as("s")) .filter(Column(havingIncrement)) val numGroups = df.collect().size validatePlan(df.queryExecution.executedPlan) assert(metric.value === 2 * ROWS_IN_DF + numGroups) } test("Increment with filter and conditional") { val trueBranchCount = createMetric(sparkContext, "true") val falseBranchCount = createMetric(sparkContext, "false") val incrementTrueBranch = IncrementMetric(Literal(true), trueBranchCount) val incrementFalseBranch = IncrementMetric(Literal(false), falseBranchCount) val incrementMetric = createMetric(sparkContext, "increment") val increment = IncrementMetric(Literal(true), incrementMetric) val incrementPreFilterMetric = createMetric(sparkContext, "incrementPreFilter") val incrementPreFilter = IncrementMetric(Literal(true), incrementPreFilterMetric) val ifCondition: Expression = ('a < Literal(20)).expr val conditional = If(ifCondition, incrementTrueBranch, incrementFalseBranch) val df = testDf .filter(Column(incrementPreFilter)) .filter('a < 25) .filter(Column(increment)) .filter(Column(conditional)) val numRows = df.collect().size validatePlan(df.queryExecution.executedPlan) assert(incrementPreFilterMetric.value === ROWS_IN_DF) assert(trueBranchCount.value === numRows) assert(falseBranchCount.value + numRows === incrementMetric.value) } test("ConditionalIncrementMetric with mixed conditions in filters") { val divisibleBy3Metric = createMetric(sparkContext, "divisibleBy3") val oneMod7Metric = createMetric(sparkContext, "divisibleBy7") val rangeMetric = createMetric(sparkContext, "inRange") val largeEvenMetric = createMetric(sparkContext, "largeEven") // Count rows divisible by 3 (metric condition) while filtering for divisible by 5 // (filter inside ConditionalIncrementMetric). val divisibleBy5Filter = ConditionalIncrementMetric( EqualTo(Pmod(UnresolvedAttribute("a"), Literal(5)), Literal(0)), EqualTo(Pmod(UnresolvedAttribute("a"), Literal(3)), Literal(0)), divisibleBy3Metric) // Count numbers where a % 7 == 1 while filtering for a > 10 // (filter outside ConditionalIncrementMetric). val divisibleBy7Condition = EqualTo(Pmod(UnresolvedAttribute("a"), Literal(7)), Literal(1)) val divisibleBy7Increment = ConditionalIncrementMetric( UnresolvedAttribute("a"), EqualTo(Pmod(UnresolvedAttribute("a"), Literal(7)), Literal(1)), oneMod7Metric) val greaterThan10Filter = GreaterThan(divisibleBy7Increment, Literal(10)) // Count rows in range 30-70 while filtering for < 50. val rangeCondition = And( GreaterThanOrEqual(UnresolvedAttribute("a"), Literal(30)), LessThanOrEqual(UnresolvedAttribute("a"), Literal(70)) ) val rangeIncrement = ConditionalIncrementMetric( UnresolvedAttribute("a"), rangeCondition, rangeMetric) val lessThan50Filter = LessThan(rangeIncrement, Literal(50)) // Count even numbers >= 20 while selecting column a. val largeEvenCondition = And( GreaterThanOrEqual(UnresolvedAttribute("a"), Literal(20)), EqualTo(Pmod(UnresolvedAttribute("a"), Literal(2)), Literal(0)) ) val largeEvenIncrement = ConditionalIncrementMetric(UnresolvedAttribute("a"), largeEvenCondition, largeEvenMetric) val df = testDf .filter(Column(divisibleBy5Filter)) // Filter: a % 5 == 0, Metric: counts a % 3 == 0 .filter(Column(greaterThan10Filter)) // Filter: a > 10, Metric: counts a % 7 == 1 .filter(Column(lessThan50Filter)) // Filter: a < 80, Metric: counts 30 <= a <= 70 .select( Column(largeEvenIncrement).as("result"), // Metric inside Project: counts even a >= 20 col("a") ) .filter(col("a") >= 30) // Additional filter after select. val results = df.collect() val numRows = results.size validatePlan(df.queryExecution.executedPlan) // divisibleBy3Metric counts values divisible by 3 among ALL rows (0-999) // The ConditionalIncrementMetric outputs (a % 5 == 0) as boolean for filtering, // but internally counts when (a % 3 == 0). // Divisible by 3: 0,3,6,9,12,15,...,999 // Count: 1000/3 = 333.33, so 334 values (includes 0) assert(divisibleBy3Metric.value === 334) // oneMod7Metric counts a % 7 == 1 among rows that pass divisible by 5 filter. // Divisible by 5: 0,5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,...,995 (200 values) // Among these, a % 7 == 1: 15,50,85,...,995 // Pattern: starts at 15, then every 35 (LCM of 5 and 7) // Count: (995-15)/35 + 1 = 29 values assert(oneMod7Metric.value === 29) // After divisible by 5 and > 10 filters: 15,20,25,30,35,40,45,50,55,60,65,70,75,... // Among these, in range [30,70]: 30,35,40,45,50,55,60,65,70 = 9 values assert(rangeMetric.value === 9) // After divisible by 5, > 10, < 50: 15,20,25,30,35,40,45 // Among these, even and >= 20: 20,30,40 = 3 values. assert(largeEvenMetric.value === 3) // Final result: divisible by 5, > 10, < 50, >= 30 // Values: 30,35,40,45 = 4 rows. assert(numRows === 4) } test("ConditionalIncrementMetric with mixed conditions in projections") { val trueMetric = createMetric(sparkContext, "conditionTrue") val falseMetric = createMetric(sparkContext, "conditionFalse") val equalMetric = createMetric(sparkContext, "conditionEqual") val trueCondition = LessThan(UnresolvedAttribute("a"), Literal(500)) val falseCondition = GreaterThan(UnresolvedAttribute("a"), Literal(ROWS_IN_DF)) val equalCondition = EqualTo(UnresolvedAttribute("a"), Literal(42)) val trueIncrement = ConditionalIncrementMetric(UnresolvedAttribute("a"), trueCondition, trueMetric) val falseIncrement = ConditionalIncrementMetric(UnresolvedAttribute("a"), falseCondition, falseMetric) val equalIncrement = ConditionalIncrementMetric(UnresolvedAttribute("a"), equalCondition, equalMetric) val df = testDf .select( Column(trueIncrement).as("true_result"), Column(falseIncrement).as("false_result"), Column(equalIncrement).as("equal_result") ) val numRows = df.collect().size validatePlan(df.queryExecution.executedPlan) assert(trueMetric.value === 500) // a < 500: rows 0-499 assert(falseMetric.value === 0) // a > 1000: no rows assert(equalMetric.value === 1) // a == 42: exactly 1 row assert(numRows === ROWS_IN_DF) } test("ConditionalIncrementMetric with nullable condition") { val metric = createMetric(sparkContext, "nullable_condition") // Create a DataFrame with nullable boolean values. val df = spark.range(10).selectExpr( "id", "CASE WHEN id % 3 = 0 THEN null WHEN id % 3 = 1 THEN true ELSE false END AS nullable_bool" ) // Create condition that can be null. val conditionExpr = UnresolvedAttribute("nullable_bool") val incrementExpr = ConditionalIncrementMetric( UnresolvedAttribute("id"), conditionExpr, metric ) val resultDf = df.select(Column(incrementExpr).as("result")) val numRows = resultDf.collect().size validatePlan(resultDf.queryExecution.executedPlan) // The metric should only count rows where condition is true (not null and not false). // id=1,4,7 have nullable_bool=true (3 rows) // id=0,3,6,9 have nullable_bool=null (4 rows) - should NOT be counted // id=2,5,8 have nullable_bool=false (3 rows) - should NOT be counted assert(metric.value === 3) assert(numRows === 10) } test("ConditionalIncrementMetric with invalid condition type") { val metric = createMetric(sparkContext, "invalidType") val nonBooleanCondition = UnresolvedAttribute("a") // Integer type val increment = ConditionalIncrementMetric(UnresolvedAttribute("a"), nonBooleanCondition, metric) // This should fail during analysis due to non-boolean condition type. val exception = intercept[AnalysisException] { val df = testDf.select(Column(increment).as("result")) df.collect() } assert(exception.getErrorClass == "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE") } for (enabled <- BOOLEAN_DOMAIN) { test(s"ConditionalIncrementMetric optimization - literal conditions - enabled=$enabled") { withSQLConf( DeltaSQLConf.DELTA_OPTIMIZE_CONDITIONAL_INCREMENT_METRIC_ENABLED.key -> enabled.toString) { val trueMetric = createMetric(sparkContext, s"literalTrue$enabled") val falseMetric = createMetric(sparkContext, s"literalFalse$enabled") val nullMetric = createMetric(sparkContext, s"literalNull$enabled") val nonConstMetric = createMetric(sparkContext, s"nonConst$enabled") val childExpr = UnresolvedAttribute("a") val trueCondition = Literal(true, BooleanType) val falseCondition = Literal(false, BooleanType) val nullCondition = Literal(null, BooleanType) val nonConstCondition = GreaterThan(UnresolvedAttribute("a"), Literal(100)) val trueExpr = ConditionalIncrementMetric(childExpr, trueCondition, trueMetric) val falseExpr = ConditionalIncrementMetric(childExpr, falseCondition, falseMetric) val nullExpr = ConditionalIncrementMetric(childExpr, nullCondition, nullMetric) val nonConstExpr = ConditionalIncrementMetric(childExpr, nonConstCondition, nonConstMetric) val df = testDf.select( Column(trueExpr).as("true_result"), Column(falseExpr).as("false_result"), Column(nullExpr).as("null_result"), Column(nonConstExpr).as("nonconst_result") ) // Check optimized plan transformations. val optimizedPlan = df.queryExecution.optimizedPlan var conditionalCount = 0 var nonConditionalCount = 0 optimizedPlan.foreach { _.transformExpressions { case e: ConditionalIncrementMetric => conditionalCount += 1 e case e: IncrementMetric => nonConditionalCount += 1 e } } if (enabled) { assert(conditionalCount === 1, s"ConditionalIncrementMetric with non-const cond should remain unchanged." + s"\n$optimizedPlan") assert(nonConditionalCount === 1, s"ConditionalIncrementMetric with cond true should become IncrementMetric" + s"\n$optimizedPlan") // ConditionalIncrementMetric with condition false or null have gotten removed. } else { assert(conditionalCount === 4, s"All ConditionalIncrementMetric expressions should remain unchanged when " + s"optimization is disabled.\n$optimizedPlan") assert(nonConditionalCount === 0, s"No ConditionalIncrementMetric should be converted to IncrementMetric when " + s"optimization is disabled.\n$optimizedPlan") } // Verify metrics work correctly regardless of optimization state. df.collect() validatePlan(df.queryExecution.executedPlan) assert(trueMetric.value === ROWS_IN_DF) // All rows counted (true condition) assert(falseMetric.value === 0) // No rows counted (false condition) assert(nullMetric.value === 0) // No rows counted (null condition) assert(nonConstMetric.value === 899) // Rows where a > 100 (101-999) } } } test("ConditionalIncrementMetric with all-null condition column") { val metric = createMetric(sparkContext, "allNullCondition") // Create a DataFrame where the condition column is always null. val df = spark.range(10).selectExpr("id", "try_divide(id, 0) < 0 as null_condition") // Use the null condition column (not a literal, but all values are null). val incrementExpr = ConditionalIncrementMetric( UnresolvedAttribute("id"), UnresolvedAttribute("null_condition"), metric) val resultDf = df.select(Column(incrementExpr).as("result")) // This should NOT be optimized away since it's not a literal condition. val optimizedPlan = resultDf.queryExecution.optimizedPlan var conditionalCount = 0 optimizedPlan.foreach { _.transformExpressions { case e: ConditionalIncrementMetric => conditionalCount += 1 e } } assert(conditionalCount === 1, s"Non-literal all-null condition should preserve ConditionalIncrementMetric\n$optimizedPlan") val numRows = resultDf.collect().size validatePlan(resultDf.queryExecution.executedPlan) // The metric should be 0 since all condition values are null. assert(metric.value === 0, "All-null condition should count 0 rows") assert(numRows === 10) } protected def validatePlan(plan: SparkPlan): Unit = {} } class IncrementMetricSuite extends IncrementMetricSuiteBase {} ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/optimize/CompactionTestHelper.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.optimize // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.commands.optimize.OptimizeMetrics import org.apache.spark.sql.delta.hooks.AutoCompact import org.apache.spark.sql.delta.sources.DeltaSQLConf._ import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.QueryTest import org.apache.spark.sql.SparkSession /** * A trait used by unit tests to trigger compaction over a table. */ private[delta] trait CompactionTestHelper extends QueryTest with DeltaSQLTestUtils { /** * Compact files under the given `tablePath` using AutoCompaction/OPTIMIZE and * returns the [[OptimizeMetrics]] */ def compactAndGetMetrics(tablePath: String, where: String = ""): OptimizeMetrics /** config controlling the min file size required for compaction */ val minFileSizeConf: String /** config controlling the target file size for compaction */ val maxFileSizeConf: String /** Create `numFilePartitions` partitions and each partition has `numFilesPerPartition` files. */ def createFilesToPartitions( numFilePartitions: Int, numFilesPerPartition: Int, dir: String) (implicit spark: SparkSession): Unit = { val totalNumFiles = numFilePartitions * numFilesPerPartition spark.range(start = 0, end = totalNumFiles, step = 1, numPartitions = totalNumFiles) .selectExpr(s"id % $numFilePartitions as c0", "id as c1") .write .format("delta") .partitionBy("c0") .mode("append") .save(dir) } /** Create `numFiles` files without any partition. */ def createFilesWithoutPartitions( numFiles: Int, dir: String)(implicit spark: SparkSession): Unit = { spark.range(start = 0, end = numFiles, step = 1, numPartitions = numFiles) .selectExpr("id as c0", "id as c1", "id as c2") .write .format("delta") .mode("append") .save(dir) } } private[delta] trait CompactionTestHelperForOptimize extends CompactionTestHelper { override def compactAndGetMetrics(tablePath: String, where: String = ""): OptimizeMetrics = { import testImplicits._ val whereClause = if (where != "") s"WHERE $where" else "" val res = spark.sql(s"OPTIMIZE tahoe.`$tablePath` $whereClause") val metrics: OptimizeMetrics = res.select($"metrics.*").as[OptimizeMetrics].head() metrics } override val minFileSizeConf: String = DELTA_OPTIMIZE_MIN_FILE_SIZE.key override val maxFileSizeConf: String = DELTA_OPTIMIZE_MAX_FILE_SIZE.key } private[delta] trait CompactionTestHelperForAutoCompaction extends CompactionTestHelper { override def compactAndGetMetrics(tablePath: String, where: String = ""): OptimizeMetrics = { // Set min num files to 2 - so that even if two small files are present in a partition, then // also they are compacted. var metrics: Option[OptimizeMetrics] = None withSQLConf(DELTA_AUTO_COMPACT_MIN_NUM_FILES.key -> "2") { metrics = Some( AutoCompact.compact( spark, DeltaLog.forTable(spark, tablePath) ).head ) } metrics.get } override val minFileSizeConf: String = DELTA_AUTO_COMPACT_MIN_FILE_SIZE.key override val maxFileSizeConf: String = DELTA_AUTO_COMPACT_MAX_FILE_SIZE.key } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/optimize/DeltaReorgSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.optimize import org.apache.spark.sql.delta.{DeletionVectorsTestUtils, DeltaColumnMapping, DeltaLog, DeltaUnsupportedOperationException} import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.commands.VacuumCommand.generateCandidateFileMap import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.util.DeltaFileOperations import io.delta.tables.DeltaTable import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.parquet.hadoop.Footer import org.apache.spark.sql.QueryTest import org.apache.spark.sql.functions.col import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.SerializableConfiguration class DeltaReorgSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with DeltaSQLTestUtils with DeletionVectorsTestUtils { import testImplicits._ def executePurge(table: String, condition: Option[String] = None): Unit = { condition match { case Some(cond) => sql(s"REORG TABLE delta.`$table` WHERE $cond APPLY (PURGE)") case None => sql(s"REORG TABLE delta.`$table` APPLY (PURGE)") } } test("Purge DVs will combine small files") { val targetDf = spark.range(0, 100, 1, numPartitions = 5).toDF withTempDeltaTable(targetDf) { (_, log) => val path = log.dataPath.toString sql(s"DELETE FROM delta.`$path` WHERE id IN (0, 99)") assert(log.update().allFiles.filter(_.deletionVector != null).count() === 2) withSQLConf(DeltaSQLConf.DELTA_OPTIMIZE_MAX_FILE_SIZE.key -> "1073741824") { // 1gb executePurge(path) } val (addFiles, _) = getFileActionsInLastVersion(log) assert(addFiles.size === 1, "files should be combined") assert(addFiles.forall(_.deletionVector === null)) checkAnswer( sql(s"SELECT * FROM delta.`$path`"), (1 to 98).toDF()) // Verify commit history and operation metrics checkOpHistory( tablePath = path, expOpParams = Map("applyPurge" -> "true", "predicate" -> "[]"), numFilesRemoved = 2, numFilesAdded = 1) } } test("Purge DVs") { val targetDf = spark.range(0, 100, 1, numPartitions = 5).toDF() withTempDeltaTable(targetDf) { (_, log) => val path = log.dataPath.toString sql(s"DELETE FROM delta.`$path` WHERE id IN (0, 99)") assert(log.update().allFiles.filter(_.deletionVector != null).count() === 2) // First purge executePurge(path) val (addFiles, _) = getFileActionsInLastVersion(log) assert(addFiles.size === 1) // two files are combined assert(addFiles.forall(_.deletionVector === null)) checkAnswer( sql(s"SELECT * FROM delta.`$path`"), (1 to 98).toDF()) // Verify commit history and operation metrics checkOpHistory( tablePath = path, expOpParams = Map("applyPurge" -> "true", "predicate" -> "[]"), numFilesRemoved = 2, numFilesAdded = 1) // Second purge is a noop val versionBefore = log.update().version executePurge(path) val versionAfter = log.update().version assert(versionBefore === versionAfter) } } test("Purge a non-DV table is a noop") { val targetDf = spark.range(0, 100, 1, numPartitions = 5).toDF() withTempDeltaTable(targetDf, enableDVs = false) { (_, log) => val versionBefore = log.update().version executePurge(log.dataPath.toString) val versionAfter = log.update().version assert(versionBefore === versionAfter) } } test("Purge some partitions of a table with DV") { val targetDf = spark.range(0, 100, 1, numPartitions = 1) .withColumn("part", col("id") % 4) .toDF() withTempDeltaTable(targetDf, partitionBy = Seq("part")) { (_, log) => val path = log.dataPath // Delete one row from each partition sql(s"DELETE FROM delta.`$path` WHERE id IN (48, 49, 50, 51)") val (addFiles1, _) = getFileActionsInLastVersion(log) assert(addFiles1.size === 4) assert(addFiles1.forall(_.deletionVector !== null)) // PURGE two partitions sql(s"REORG TABLE delta.`$path` WHERE part IN (0, 2) APPLY (PURGE)") val (addFiles2, _) = getFileActionsInLastVersion(log) assert(addFiles2.size === 2) assert(addFiles2.forall(_.deletionVector === null)) // Verify commit history and operation metrics checkOpHistory( tablePath = path.toString, expOpParams = Map("applyPurge" -> "true", "predicate" -> "[\"'part IN (0,2)\"]"), numFilesRemoved = 2, numFilesAdded = 2) } } private def checkOpHistory( tablePath: String, expOpParams: Map[String, String], numFilesRemoved: Long, numFilesAdded: Long): Unit = { val (opName, opParams, opMetrics) = DeltaTable.forPath(tablePath) .history(1) .select("operation", "operationParameters", "operationMetrics") .as[(String, Map[String, String], Map[String, String])] .head() assert(opName === "REORG") assert(opParams === expOpParams) assert(opMetrics("numAddedFiles").toLong === numFilesAdded) assert(opMetrics("numRemovedFiles").toLong === numFilesRemoved) // Because each deleted file has a DV associated it which gets rewritten as part of PURGE assert(opMetrics("numDeletionVectorsRemoved").toLong === numFilesRemoved) } /** * Get all parquet footers for the input `files`, used only for testing. * * @param files the sequence of `AddFile` used to read the parquet footers * by the data file path in each `AddFile`. * @param log the delta log used to get the configuration and data path. * @return the sequence of the corresponding parquet footers, corresponds to * the sequence of `AddFile`. */ private def getParquetFooters( files: Seq[AddFile], log: DeltaLog): Seq[Footer] = { val serializedConf = new SerializableConfiguration(log.newDeltaHadoopConf()) val dataPath = new Path(log.dataPath.toString) val nameToAddFileMap = generateCandidateFileMap(dataPath, files) val fileStatuses = nameToAddFileMap.map { case (absPath, addFile) => new FileStatus( /* length */ addFile.size, /* isDir */ false, /* blockReplication */ 0, /* blockSize */ 1, /* modificationTime */ addFile.modificationTime, new Path(absPath) ) } DeltaFileOperations.readParquetFootersInParallel( serializedConf.value, fileStatuses.toList, ignoreCorruptFiles = false ) } test("Purge dropped columns of a table without DV") { val targetDf = spark.range(0, 100, 1, numPartitions = 5) .withColumn("id_dropped", col("id") % 4) .toDF() withTempDeltaTable(targetDf) { (_, log) => val path = log.dataPath.toString val (addFiles1, _) = getFileActionsInLastVersion(log) assert(addFiles1.size === 5) val footers1 = getParquetFooters(addFiles1, log) footers1.foreach { footer => val fields = footer.getParquetMetadata.getFileMetaData.getSchema.getFields assert(fields.size == 2) assert(fields.toArray.map { _.toString }.contains("optional int64 id_dropped")) } // enable column-mapping first sql( s""" | ALTER TABLE delta.`$path` | SET TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name' | ) |""".stripMargin ) // drop the extra column by alter table and run REORG PURGE sql( s""" | ALTER TABLE delta.`$path` | DROP COLUMN id_dropped |""".stripMargin ) executePurge(path) val (addFiles2, _) = getFileActionsInLastVersion(log) assert(addFiles2.size === 1) val footers2 = getParquetFooters(addFiles2, log) footers2.foreach { footer => val fields = footer.getParquetMetadata.getFileMetaData.getSchema.getFields assert(fields.size == 1) assert(!fields.toArray.map { _.toString }.contains("optional int64 id_dropped")) } } } test("Columns being renamed should not be purged") { val targetDf = spark.range(0, 100, 1, numPartitions = 5) .withColumn("id_before_rename", col("id") % 4) .withColumn("id_dropped", col("id") % 5) .toDF() withTempDeltaTable(targetDf) { (_, log) => val path = log.dataPath.toString val (addFiles1, _) = getFileActionsInLastVersion(log) assert(addFiles1.size === 5) val footers1 = getParquetFooters(addFiles1, log) footers1.foreach { footer => val fields = footer.getParquetMetadata.getFileMetaData.getSchema.getFields assert(fields.size == 3) assert(fields.toArray.map { _.toString }.contains("optional int64 id_dropped")) assert(fields.toArray.map { _.toString }.contains("optional int64 id_before_rename")) } // enable column-mapping first sql( s""" | ALTER TABLE delta.`$path` | SET TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name' | ) |""".stripMargin ) // drop `id_dropped` and rename `id_before_rename` via alter table and run REORG PURGE, // this should remove `id_dropped` but keep `id_after_rename` in the parquet files. sql( s""" | ALTER TABLE delta.`$path` | DROP COLUMN id_dropped |""".stripMargin ) sql( s""" | ALTER TABLE delta.`$path` | RENAME COLUMN id_before_rename TO id_after_rename |""".stripMargin ) executePurge(path) val tableSchema = log.update().schema val tablePhysicalSchema = DeltaColumnMapping.renameColumns(tableSchema) val beforeRenameColStr = "StructField(id_before_rename,LongType,true)" val afterRenameColStr = "StructField(id_after_rename,LongType,true)" assert(tableSchema.fields.length == 2 && tableSchema.map { _.toString }.contains(afterRenameColStr)) assert(tablePhysicalSchema.fields.length == 2 && tablePhysicalSchema.map { _.toString }.contains(beforeRenameColStr)) val (addFiles2, _) = getFileActionsInLastVersion(log) assert(addFiles2.size === 1) val footers2 = getParquetFooters(addFiles2, log) footers2.foreach { footer => val fields = footer.getParquetMetadata.getFileMetaData.getSchema.getFields assert(fields.size == 2) assert(!fields.toArray.map { _.toString }.contains("optional int64 id_dropped = 3")) // do note that the actual name for the column will not be // changed in parquet file level assert(fields.toArray.map { _.toString }.contains("optional int64 id_before_rename = 2")) } } } test("reorg on a catalog managed table should fail") { withCatalogManagedTable() { tableName => checkError( intercept[DeltaUnsupportedOperationException] { spark.sql(s"REORG TABLE $tableName APPLY (PURGE)") }, "DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION", parameters = Map("operation" -> "OPTIMIZE") ) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/optimize/OptimizeCompactionSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.optimize import java.io.File import scala.collection.JavaConverters._ // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaColumnMappingSelectedTestMixin, DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import io.delta.tables.DeltaTable import org.scalatest.concurrent.TimeLimits.failAfter import org.scalatest.time.SpanSugar._ import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.functions.lit import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.LongType /** * Base class containing tests for Delta table Optimize (file compaction) */ trait OptimizeCompactionSuiteBase extends QueryTest with SharedSparkSession with DeltaSQLTestUtils with DeletionVectorsTestUtils with DeltaColumnMappingTestUtils { import testImplicits._ def executeOptimizeTable(table: String, condition: Option[String] = None) def executeOptimizePath(path: String, condition: Option[String] = None) test("optimize command: with database and table name") { withTempDir { tempDir => val dbName = "delta_db" val tableName = s"$dbName.delta_optimize" withDatabase(dbName) { spark.sql(s"create database $dbName") withTable(tableName) { appendToDeltaTable(Seq(1, 2, 3).toDF(), tempDir.toString, partitionColumns = None) appendToDeltaTable(Seq(4, 5, 6).toDF(), tempDir.toString, partitionColumns = None) spark.sql(s"create table $tableName using delta location '$tempDir'") val deltaLog = DeltaLog.forTable(spark, tempDir) val versionBeforeOptimize = deltaLog.snapshot.version executeOptimizeTable(tableName) deltaLog.update() assert(deltaLog.snapshot.version === versionBeforeOptimize + 1) checkDatasetUnorderly(spark.table(tableName).as[Int], 1, 2, 3, 4, 5, 6) } } } } test("optimize command") { withTempDir { tempDir => appendToDeltaTable(Seq(1, 2, 3).toDF(), tempDir.toString, partitionColumns = None) appendToDeltaTable(Seq(4, 5, 6).toDF(), tempDir.toString, partitionColumns = None) def data: DataFrame = spark.read.format("delta").load(tempDir.toString) val deltaLog = DeltaLog.forTable(spark, tempDir) val versionBeforeOptimize = deltaLog.snapshot.version executeOptimizePath(tempDir.getCanonicalPath) deltaLog.update() assert(deltaLog.snapshot.version === versionBeforeOptimize + 1) checkDatasetUnorderly(data.toDF().as[Int], 1, 2, 3, 4, 5, 6) // Make sure thread pool is shut down assert(Thread.getAllStackTraces.keySet.asScala .filter(_.getName.startsWith("OptimizeJob")).isEmpty) } } test("optimize command: predicate on non-partition column") { withTempDir { tempDir => val path = new File(tempDir, "testTable").getCanonicalPath val partitionColumns = Some(Seq("id")) appendToDeltaTable( Seq(1, 2, 3).toDF("value").withColumn("id", 'value % 2), path, partitionColumns) val e = intercept[AnalysisException] { // Should fail when predicate is on a non-partition column executeOptimizePath(path, Some("value < 4")) } assert(e.getMessage.contains("Predicate references non-partition column 'value'. " + "Only the partition columns may be referenced: [id]")) } } test("optimize command: on partitioned table - all partitions") { withTempDir { tempDir => val path = new File(tempDir, "testTable").getCanonicalPath val partitionColumns = Some(Seq("id")) appendToDeltaTable( Seq(1, 2, 3).toDF("value").withColumn("id", 'value % 2), path, partitionColumns) appendToDeltaTable( Seq(4, 5, 6).toDF("value").withColumn("id", 'value % 2), path, partitionColumns) val deltaLogBefore = DeltaLog.forTable(spark, path) val txnBefore = deltaLogBefore.startTransaction(); val fileListBefore = txnBefore.filterFiles(); val versionBefore = deltaLogBefore.snapshot.version val id = "id".phy(deltaLogBefore) // Expect each partition have more than one file (0 to 1).foreach(partId => assert(fileListBefore.count(_.partitionValues === Map(id -> partId.toString)) > 1)) executeOptimizePath(path) val deltaLogAfter = DeltaLog.forTable(spark, path) val txnAfter = deltaLogAfter.startTransaction(); val fileListAfter = txnAfter.filterFiles(); (0 to 1).foreach(partId => assert(fileListAfter.count(_.partitionValues === Map(id -> partId.toString)) === 1)) // version is incremented assert(deltaLogAfter.snapshot.version === versionBefore + 1) // data should remain the same after the OPTIMIZE checkDatasetUnorderly( spark.read.format("delta").load(path).select("value").as[Long], (1L to 6L): _*) } } test( s"optimize command with DVs") { withTempDir { tempDir => val path = tempDir.getAbsolutePath withSQLConf( DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> "true") { // Create 10 files each with 1000 records spark.range(start = 0, end = 10000, step = 1, numPartitions = 10) .toDF("id") .withColumn(colName = "extra", lit("just a random text to fill up the space.....")) .write.format("delta").mode("append").save(path) // v0 val deltaLog = DeltaLog.forTable(spark, path) val filesV0 = deltaLog.unsafeVolatileSnapshot.allFiles.collect() assert(filesV0.size == 10) // Default `optimize.maxDeletedRowsRatio` is 0.05. // Delete slightly more than threshold ration in two files, less in one of the file val file0 = filesV0(1) val file1 = filesV0(4) val file2 = filesV0(8) deleteRows(deltaLog, file0, approxPhyRows = 1000, ratioOfRowsToDelete = 0.06d) // v1 deleteRows(deltaLog, file1, approxPhyRows = 1000, ratioOfRowsToDelete = 0.06d) // v2 deleteRows(deltaLog, file2, approxPhyRows = 1000, ratioOfRowsToDelete = 0.01d) // v3 // Add a one small file, so that the file selection is based on both the file size and // deleted rows ratio spark.range(start = 1, end = 2, step = 1, numPartitions = 1) .toDF("id").withColumn(colName = "extra", lit("")) .write.format("delta").mode("append").save(path) // v4 val smallFiles = addedFiles( deltaLog.getChanges(startVersion = 4, catalogTableOpt = None).next()._2) assert(smallFiles.size == 1) // Save the data before optimize for comparing it later with optimize val data = spark.read.format("delta").load(path) // Set a low value for minFileSize so that the criteria for file selection is based on DVs // and not based on the file size. val targetSmallSize = smallFiles(0).size + 10 // A number just higher than the `smallFile` withSQLConf(DeltaSQLConf.DELTA_OPTIMIZE_MIN_FILE_SIZE.key -> targetSmallSize.toString) { executeOptimizePath(path) // v5 } val changes = deltaLog.getChanges(startVersion = 5, catalogTableOpt = None).next()._2 // We expect the two files containing more than the threshold rows to be compacted. var expectedRemoveFiles = Set(file0.path, file1.path) // Expect the small file also to be compacted always expectedRemoveFiles += smallFiles(0).path assert(removedFiles(changes).map(_.path).toSet === expectedRemoveFiles) assert(addedFiles(changes).size == 1) // Expect one new file added // Verify the final data after optimization hasn't changed. checkAnswer(spark.read.format("delta").load(path), data) } } } private def removedFiles(actions: Seq[Action]): Seq[RemoveFile] = { actions.filter(_.isInstanceOf[RemoveFile]).map(_.asInstanceOf[RemoveFile]) } private def addedFiles(actions: Seq[Action]): Seq[AddFile] = { actions.filter(_.isInstanceOf[AddFile]).map(_.asInstanceOf[AddFile]) } def appendRowsToDeltaTable( path: String, numFiles: Int, numRowsPerFiles: Int, partitionColumns: Option[Seq[String]], partitionValues: Seq[Int]): Unit = { partitionValues.foreach { partition => (0 until numFiles).foreach { _ => appendToDeltaTable( (0 until numRowsPerFiles).toDF("value").withColumn("id", lit(partition)), path, partitionColumns) } } } def testOptimizeCompactWithLargeFile( name: String, unCompactablePartitions: Seq[Int], compactablePartitions: Seq[Int]) { test(name) { withTempDir { tempDir => val path = new File(tempDir, "testTable").getCanonicalPath val partitionColumns = Some(Seq("id")) // Create un-compactable partitions. appendRowsToDeltaTable( path, numFiles = 1, numRowsPerFiles = 200, partitionColumns, unCompactablePartitions) // Create compactable partitions with 5 files appendRowsToDeltaTable( path, numFiles = 5, numRowsPerFiles = 10, partitionColumns, compactablePartitions) val deltaLogBefore = DeltaLog.forTable(spark, path) val txnBefore = deltaLogBefore.startTransaction() val fileListBefore = txnBefore.filterFiles() val versionBefore = deltaLogBefore.snapshot.version val id = "id".phy(deltaLogBefore) unCompactablePartitions.foreach(partId => assert(fileListBefore.count(_.partitionValues === Map(id -> partId.toString)) == 1)) compactablePartitions.foreach(partId => assert(fileListBefore.count(_.partitionValues === Map(id -> partId.toString)) == 5)) // Optimize compact all partitions spark.sql(s"OPTIMIZE '$path'") val deltaLogAfter = DeltaLog.forTable(spark, path) val txnAfter = deltaLogAfter.startTransaction(); val fileListAfter = txnAfter.filterFiles(); // All partitions should only contains single file. (unCompactablePartitions ++ compactablePartitions).foreach(partId => assert(fileListAfter.count(_.partitionValues === Map(id -> partId.toString)) === 1)) // version is incremented assert(deltaLogAfter.snapshot.version === versionBefore + 1) } } } testOptimizeCompactWithLargeFile( "optimize command: interleaves compactable/un-compactable partitions", unCompactablePartitions = Seq(1, 3, 5), compactablePartitions = Seq(2, 4, 6)) testOptimizeCompactWithLargeFile( "optimize command: first two and last two partitions are un-compactable", unCompactablePartitions = Seq(1, 2, 5, 6), compactablePartitions = Seq(3, 4)) testOptimizeCompactWithLargeFile( "optimize command: only first and last partition are compactable", unCompactablePartitions = Seq(2, 3, 4, 5), compactablePartitions = Seq(1, 6)) testOptimizeCompactWithLargeFile( "optimize command: only first partition is un-compactable", unCompactablePartitions = Seq(1), compactablePartitions = Seq(2, 3, 4, 5, 6)) testOptimizeCompactWithLargeFile( "optimize command: only first partition is compactable", unCompactablePartitions = Seq(2, 3, 4, 5, 6), compactablePartitions = Seq(1)) test("optimize command: on partitioned table - selected partitions") { withTempDir { tempDir => val path = new File(tempDir, "testTable").getCanonicalPath val partitionColumns = Some(Seq("id")) appendToDeltaTable( Seq(1, 2, 3).toDF("value").withColumn("id", 'value % 2), path, partitionColumns) appendToDeltaTable( Seq(4, 5, 6).toDF("value").withColumn("id", 'value % 2), path, partitionColumns) val deltaLogBefore = DeltaLog.forTable(spark, path) val txnBefore = deltaLogBefore.startTransaction(); val fileListBefore = txnBefore.filterFiles() val id = "id".phy(deltaLogBefore) assert(fileListBefore.length >= 3) assert(fileListBefore.count(_.partitionValues === Map(id -> "0")) > 1) val versionBefore = deltaLogBefore.snapshot.version executeOptimizePath(path, Some("id = 0")) val deltaLogAfter = DeltaLog.forTable(spark, path) val txnAfter = deltaLogBefore.startTransaction(); val fileListAfter = txnAfter.filterFiles() assert(fileListBefore.length > fileListAfter.length) // Optimized partition should contain only one file assert(fileListAfter.count(_.partitionValues === Map(id -> "0")) === 1) // File counts in partitions that are not part of the OPTIMIZE should remain the same assert(fileListAfter.count(_.partitionValues === Map(id -> "1")) === fileListAfter.count(_.partitionValues === Map(id -> "1"))) // version is incremented assert(deltaLogAfter.snapshot.version === versionBefore + 1) // data should remain the same after the OPTIMIZE checkDatasetUnorderly( spark.read.format("delta").load(path).select("value").as[Long], (1L to 6L): _*) } } test("optimize command: on null partition columns") { withTempDir { tempDir => val path = new File(tempDir, "testTable").getCanonicalPath val partitionColumn = "part" (1 to 5).foreach { _ => appendToDeltaTable( Seq(("a", 1), ("b", 2), (null.asInstanceOf[String], 3), ("", 4)) .toDF(partitionColumn, "value"), path, Some(Seq(partitionColumn))) } val deltaLogBefore = DeltaLog.forTable(spark, path) val txnBefore = deltaLogBefore.startTransaction(); val fileListBefore = txnBefore.filterFiles() val versionBefore = deltaLogBefore.snapshot.version val partitionColumnPhysicalName = partitionColumn.phy(deltaLogBefore) // we have only 1 partition here val filesInEachPartitionBefore = groupInputFilesByPartition( fileListBefore.map(_.toPath.toString).toArray, deltaLogBefore) // There exist at least one file in each partition assert(filesInEachPartitionBefore.forall(_._2.length > 1)) // And there is a partition for null values assert(filesInEachPartitionBefore.keys.exists( _ === (partitionColumnPhysicalName, nullPartitionValue))) executeOptimizePath(path) val deltaLogAfter = DeltaLog.forTable(spark, path) val txnAfter = deltaLogBefore.startTransaction(); val fileListAfter = txnAfter.filterFiles() // Number of files is less than before optimize assert(fileListBefore.length > fileListAfter.length) // Optimized partition should contain only one file in null partition assert(fileListAfter.count( _.partitionValues === Map[String, String](partitionColumnPhysicalName -> null)) === 1) // version is incremented assert(deltaLogAfter.snapshot.version === versionBefore + 1) // data should remain the same after the OPTIMIZE checkAnswer( spark.read.format("delta").load(path).groupBy(partitionColumn).count(), Seq(Row("a", 5), Row("b", 5), Row(null, 10))) } } test("optimize command: on table with multiple partition columns") { withTempDir { tempDir => val path = new File(tempDir, "testTable").getCanonicalPath val partitionColumns = Seq("date", "part") Seq(10, 100).foreach { count => appendToDeltaTable( spark.range(count) .select('id, lit("2017-10-10").cast("date") as "date", 'id % 5 as "part"), path, Some(partitionColumns)) } val deltaLogBefore = DeltaLog.forTable(spark, path) val txnBefore = deltaLogBefore.startTransaction(); val fileListBefore = txnBefore.filterFiles() val versionBefore = deltaLogBefore.snapshot.version val date = "date".phy(deltaLogBefore) val part = "part".phy(deltaLogBefore) val fileCountInTestPartitionBefore = fileListBefore .count(_.partitionValues === Map[String, String](date -> "2017-10-10", part -> "3")) executeOptimizePath(path, Some("date = '2017-10-10' and part = 3")) val deltaLogAfter = DeltaLog.forTable(spark, path) val txnAfter = deltaLogBefore.startTransaction(); val fileListAfter = txnAfter.filterFiles() // Number of files is less than before optimize assert(fileListBefore.length > fileListAfter.length) // Optimized partition should contain only one file in null partition and less number // of files than before optimize val fileCountInTestPartitionAfter = fileListAfter .count(_.partitionValues === Map[String, String](date -> "2017-10-10", part -> "3")) assert(fileCountInTestPartitionAfter === 1L) assert(fileCountInTestPartitionBefore > fileCountInTestPartitionAfter, "Expected the partition to count less number of files after optimzie.") // version is incremented assert(deltaLogAfter.snapshot.version === versionBefore + 1) } } test("optimize - multiple jobs start executing at once ") { // The idea here is to make sure multiple optimize jobs execute concurrently. We can // block the writes of one batch with a countdown latch that will unblock only // after the second batch also tries to write. val numPartitions = 2 withTempDir { tempDir => spark.range(100) .withColumn("pCol", 'id % numPartitions) .repartition(10) .write .format("delta") .partitionBy("pCol") .save(tempDir.getAbsolutePath) // We have two partitions so we would have two tasks. We can make sure we have two batches withSQLConf( ("fs.AbstractFileSystem.block.impl", classOf[BlockWritesAbstractFileSystem].getCanonicalName), ("fs.block.impl", classOf[BlockWritesLocalFileSystem].getCanonicalName)) { val path = s"block://${tempDir.getAbsolutePath}" val deltaLog = DeltaLog.forTable(spark, path) require(deltaLog.snapshot.numOfFiles === 20) // 10 files in each partition // block the first write until the second batch can attempt to write. BlockWritesLocalFileSystem.blockUntilConcurrentWrites(numPartitions) failAfter(60.seconds) { executeOptimizePath(path) } assert(deltaLog.snapshot.numOfFiles === numPartitions) // 1 file per partition } } } test("optimize command with multiple partition predicates") { withTempDir { tempDir => def writeData(count: Int): Unit = { spark.range(count).select('id, lit("2017-10-10").cast("date") as "date", 'id % 5 as "part") .write .partitionBy("date", "part") .format("delta") .mode("append") .save(tempDir.getAbsolutePath) } writeData(10) writeData(100) executeOptimizePath(tempDir.getAbsolutePath, Some("date = '2017-10-10' and part = 3")) val df = spark.read.format("delta").load(tempDir.getAbsolutePath) val deltaLog = loadDeltaLog(tempDir.getAbsolutePath) val part = "part".phy(deltaLog) val files = groupInputFilesByPartition(df.inputFiles, deltaLog) assert(files.filter(_._1._1 == part).minBy(_._2.length)._1 === (part, "3"), "part 3 should have been optimized and have least amount of files") } } test("optimize command with multiple partition predicates with multiple where") { withTempDir { tempDir => def writeData(count: Int): Unit = { spark.range(count).select('id, lit("2017-10-10").cast("date") as "date", 'id % 5 as "part") .write .partitionBy("date", "part") .format("delta") .mode("append") .save(tempDir.getAbsolutePath) } writeData(10) writeData(100) DeltaTable.forPath(tempDir.getAbsolutePath).optimize() .where("part = 3") .where("date = '2017-10-10'") .executeCompaction() val df = spark.read.format("delta").load(tempDir.getAbsolutePath) val deltaLog = loadDeltaLog(tempDir.getAbsolutePath) val part = "part".phy(deltaLog) val files = groupInputFilesByPartition(df.inputFiles, deltaLog) assert(files.filter(_._1._1 == part).minBy(_._2.length)._1 === (part, "3"), "part 3 should have been optimized and have least amount of files") } } def optimizeWithBatching( batchSize: String, expectedCommits: Int, condition: Option[String], partitionFileCount: Map[String, Int]): Unit = { withSQLConf(DeltaSQLConf.DELTA_OPTIMIZE_BATCH_SIZE.key -> batchSize) { withTempDir { tempDir => def writeData(count: Int): Unit = { spark.range(count).select('id, 'id % 5 as "part") .coalesce(1) .write .partitionBy("part") .format("delta") .mode("append") .save(tempDir.getAbsolutePath) } writeData(10) writeData(100) val data = spark.read.format("delta").load(tempDir.getAbsolutePath()).collect() executeOptimizePath(tempDir.getAbsolutePath, condition) val df = spark.read.format("delta").load(tempDir.getAbsolutePath) checkAnswer(df, data) val deltaLog = loadDeltaLog(tempDir.getAbsolutePath) val commits = deltaLog.history.getHistory(None) assert(commits.filter(_.operation == "OPTIMIZE").length == expectedCommits) val files = groupInputFilesByPartition(df.inputFiles, deltaLog) for ((part, fileCount) <- partitionFileCount) { assert(files(("part", part)).length == fileCount) } } } } test("optimize command with batching") { // Batch size of 1 byte means each bin will run in its own batch, and lead to 5 batches, // one for each partition. Seq(("1", 5), ("1g", 1)).foreach { case (batchSize, optimizeCommits) => // All partitions should be one file after optimizing val partitionFileCount = (0 to 4).map(_.toString -> 1).toMap optimizeWithBatching(batchSize, optimizeCommits, None, partitionFileCount) } } test("optimize command with where clause and batching") { // Batch size of 1 byte means each bin will run in its own batch, and lead to 2 batches // for the two partitions we are optimizing. Seq(("1", 2), ("1g", 1)).foreach { case (batchSize, optimizeCommits) => // First two partitions should have 1 file, last 3 should have two val partitionFileCount = Map( "0" -> 1, "1" -> 1, "2" -> 2, "3" -> 2, "4" -> 2 ) val files = optimizeWithBatching(batchSize, optimizeCommits, Some("part <= 1"), partitionFileCount) } } test("optimize an empty table with batching") { // Batch size of 1 byte means each bin will run in its own batch withSQLConf(DeltaSQLConf.DELTA_OPTIMIZE_BATCH_SIZE.key -> "1") { withTempDir { tempDir => DeltaTable.create(spark) .location(tempDir.getAbsolutePath()) .addColumn("id", LongType) .addColumn("part", LongType) .partitionedBy("part") .execute() // Just make sure it succeeds executeOptimizePath(tempDir.getAbsolutePath) assert(spark.read.format("delta").load(tempDir.getAbsolutePath()).count() == 0) } } } /** * Utility method to append the given data to the Delta table located at the given path. * Optionally partitions the data. */ protected def appendToDeltaTable[T]( data: Dataset[T], tablePath: String, partitionColumns: Option[Seq[String]] = None): Unit = { var df = data.repartition(1).write; partitionColumns.map(columns => { df = df.partitionBy(columns: _*) }) df.format("delta").mode("append").save(tablePath) } test("optimize on a catalog managed table should fail") { withCatalogManagedTable() { tableName => checkError( intercept[DeltaUnsupportedOperationException] { spark.sql(s"OPTIMIZE $tableName") }, "DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION", parameters = Map("operation" -> "OPTIMIZE") ) } } } /** * Runs optimize compaction tests using OPTIMIZE SQL */ class OptimizeCompactionSQLSuite extends OptimizeCompactionSuiteBase with DeltaSQLCommandTest { import testImplicits._ def executeOptimizeTable(table: String, condition: Option[String] = None): Unit = { val conditionClause = condition.map(c => s"WHERE $c").getOrElse("") spark.sql(s"OPTIMIZE $table $conditionClause") } def executeOptimizePath(path: String, condition: Option[String] = None): Unit = { executeOptimizeTable(s"'$path'", condition) } test("optimize command: missing path") { val e = intercept[ParseException] { spark.sql(s"OPTIMIZE") } assert(e.getMessage.contains("OPTIMIZE")) } test("optimize command: missing predicate on path") { val e = intercept[ParseException] { spark.sql(s"OPTIMIZE /doesnt/exist WHERE") } assert(e.getMessage.contains("OPTIMIZE")) } test("optimize command: non boolean expression") { val e = intercept[ParseException] { spark.sql(s"OPTIMIZE /doesnt/exist WHERE 1+1") } assert(e.getMessage.contains("OPTIMIZE")) } test("optimize with partition value containing space") { withTempDir { tempDir => val baseDf = Seq(("a space", 1), ("other", 2)).toDF("name", "value") def write(): Unit = { baseDf.write .format("delta") .partitionBy("name") .mode("append") .save(tempDir.getAbsolutePath) } write() write() sql(s"optimize '${tempDir.getAbsolutePath}'") val df = spark.read.format("delta").load(tempDir.getAbsolutePath) assert(df.inputFiles.length === 2, "2 files for 2 partitions") checkAnswer( df, baseDf.union(baseDf)) } } test("optimize command: subquery predicate") { val tableName = "myTable" withTable(tableName) { spark.sql(s"create table $tableName (p int, id int) using delta partitioned by(p)") val e = intercept[DeltaAnalysisException] { spark.sql(s"optimize $tableName where p >= (select p from $tableName where id > 5)") } checkError(e, "DELTA_UNSUPPORTED_SUBQUERY_IN_PARTITION_PREDICATES", "0AKDC", Map.empty[String, String]) } } } /** * Runs optimize compaction tests using OPTIMIZE Scala APIs */ class OptimizeCompactionScalaSuite extends OptimizeCompactionSuiteBase with DeltaSQLCommandTest { def executeOptimizeTable(table: String, condition: Option[String] = None): Unit = { if (condition.isDefined) { DeltaTable.forName(table).optimize().where(condition.get).executeCompaction() } else { DeltaTable.forName(table).optimize().executeCompaction() } } def executeOptimizePath(path: String, condition: Option[String] = None): Unit = { if (condition.isDefined) { DeltaTable.forPath(path).optimize().where(condition.get).executeCompaction() } else { DeltaTable.forPath(path).optimize().executeCompaction() } } } trait OptimizeCompactionColumnMappingSuiteBase extends DeltaColumnMappingSelectedTestMixin { override protected def runOnlyTests = Seq( "optimize command: on table with multiple partition columns", "optimize command: on null partition columns" ) } class OptimizeCompactionIdColumnMappingSuite extends OptimizeCompactionSQLSuite with DeltaColumnMappingEnableIdMode with OptimizeCompactionColumnMappingSuiteBase { } class OptimizeCompactionNameColumnMappingSuite extends OptimizeCompactionSQLSuite with DeltaColumnMappingEnableNameMode with OptimizeCompactionColumnMappingSuiteBase ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/optimize/OptimizeConflictSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.optimize import java.io.File import scala.concurrent.duration.Duration import org.apache.spark.SparkException import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.concurrency.PhaseLockingTestMixin import org.apache.spark.sql.delta.concurrency.TransactionExecutionTestMixin import org.apache.spark.sql.delta.fuzzer.{OptimisticTransactionPhases, PhaseLockingTransactionExecutionObserver} import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.ThreadUtils class OptimizeConflictSuite extends QueryTest with SharedSparkSession with PhaseLockingTestMixin with TransactionExecutionTestMixin with DeltaSQLCommandTest { protected def appendRows(dir: File, numRows: Int, numFiles: Int): Unit = { spark.range(start = 0, end = numRows, step = 1, numPartitions = numFiles) .write.format("delta").mode("append").save(dir.getAbsolutePath) } test("conflict handling between Optimize and Business Txn") { withTempDir { tempDir => // Create table with 100 rows. appendRows(tempDir, numRows = 100, numFiles = 10) // Enable DVs. sql(s"ALTER TABLE delta.`${tempDir.toString}` " + "SET TBLPROPERTIES ('delta.enableDeletionVectors' = true);") val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tempDir.getAbsolutePath) def optimizeTxn(): Array[Row] = { deltaTable.optimize().executeCompaction() Array.empty } def deleteTxn(): Array[Row] = { // Delete 50% of the rows. sql(s"DELETE FROM delta.`${tempDir}` WHERE id%2 = 0").collect() } val Seq(future) = runFunctionsWithOrderingFromObserver(Seq(optimizeTxn)) { case (optimizeObserver :: Nil) => // Create a replacement observer for the retry thread of Optimize. val retryObserver = new PhaseLockingTransactionExecutionObserver( OptimisticTransactionPhases.forName("test-replacement-txn")) // Block Optimize during the first commit attempt. optimizeObserver.setNextObserver(retryObserver, autoAdvance = true) unblockUntilPreCommit(optimizeObserver) busyWaitFor(optimizeObserver.phases.preparePhase.hasEntered, timeout) // Delete starts and finishes deleteTxn() // Allow Optimize to resume. unblockCommit(optimizeObserver) busyWaitFor(optimizeObserver.phases.commitPhase.hasLeft, timeout) optimizeObserver.phases.postCommitPhase.exitBarrier.unblock() // The first txn will not commit as there was a conflict commit // (deleteTxn). Optimize will attempt to auto resolve and retry // Wait for the retry txn to finish. // Resume the retry txn. unblockAllPhases(retryObserver) } val e = intercept[SparkException] { ThreadUtils.awaitResult(future, timeout) } // The retry txn should fail as the same files are modified(DVs added) by // the delete txn. assert(e.getCause.getMessage.contains("DELTA_CONCURRENT_DELETE_READ")) assert(sql(s"SELECT * FROM delta.`${tempDir}`").count() == 50) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/optimize/OptimizeMetricsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.optimize // scalastyle:off import.ordering.noEmptyLine import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.commands.optimize.{FileSizeStats, OptimizeMetrics, ZOrderStats} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.JsonUtils import io.delta.tables.DeltaTable import org.apache.spark.sql.QueryTest import org.apache.spark.sql.functions.floor import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ /** Tests that run optimize and verify the returned output (metrics) is expected. */ trait OptimizeMetricsSuiteBase extends QueryTest with SharedSparkSession with DeletionVectorsTestUtils { import testImplicits._ test("optimize metrics") { withTempDir { tempDir => val skewedRightSeq = 0.to(79).seq ++ 40.to(79).seq ++ 60.to(79).seq ++ 70.to(79).seq ++ 75.to(79).seq skewedRightSeq.toDF().withColumn("p", floor('value / 10)).repartition(4) .write.partitionBy("p").format("delta").save(tempDir.toString) val deltaLog = DeltaLog.forTable(spark, tempDir) val startCount = deltaLog.unsafeVolatileSnapshot.numOfFiles val startSizes = deltaLog.unsafeVolatileSnapshot.allFiles.select('size).as[Long].collect() val res = spark.sql(s"OPTIMIZE delta.`${tempDir.toString}`") val metrics: OptimizeMetrics = res.select($"metrics.*").as[OptimizeMetrics].head() val finalSizes = deltaLog.unsafeVolatileSnapshot.allFiles .select('size).collect().map(_.getLong(0)) val finalNumFiles = deltaLog.unsafeVolatileSnapshot.numOfFiles assert(metrics.numFilesAdded == finalNumFiles) assert(metrics.numFilesRemoved == startCount) assert(metrics.filesAdded.min.get == finalSizes.min) assert(metrics.filesAdded.max.get == finalSizes.max) assert(metrics.filesAdded.totalSize == finalSizes.sum) assert(metrics.filesAdded.totalFiles == finalSizes.length) assert(metrics.filesRemoved.max.get == startSizes.max) assert(metrics.filesRemoved.min.get == startSizes.min) assert(metrics.filesRemoved.totalSize == startSizes.sum) assert(metrics.filesRemoved.totalFiles == startSizes.length) assert(metrics.totalConsideredFiles == startCount) assert(metrics.totalFilesSkipped == 0) assert(metrics.numTableColumns == 2) assert(metrics.numTableColumnsWithStats == 2) } } /** * Ensure public API for metrics persists */ test("optimize command output schema") { val zOrderFileStatsSchema = StructType(Seq( StructField("num", LongType, nullable = false), StructField("size", LongType, nullable = false) )) val zOrderStatsSchema = StructType(Seq( StructField("strategyName", StringType, nullable = true), StructField("inputCubeFiles", zOrderFileStatsSchema, nullable = true), StructField("inputOtherFiles", zOrderFileStatsSchema, nullable = true), StructField("inputNumCubes", LongType, nullable = false), StructField("mergedFiles", zOrderFileStatsSchema, nullable = true), StructField("numOutputCubes", LongType, nullable = false), StructField("mergedNumCubes", LongType, nullable = true) )) val clusteringFileStatsSchema = StructType(Seq( StructField("numFiles", LongType, nullable = false), StructField("size", LongType, nullable = false))) val clusteringStatsSchema = StructType(Seq( StructField("inputZCubeFiles", clusteringFileStatsSchema, nullable = true), StructField("inputOtherFiles", clusteringFileStatsSchema, nullable = true), StructField("inputNumZCubes", LongType, nullable = false), StructField("mergedFiles", clusteringFileStatsSchema, nullable = true), StructField("numOutputZCubes", LongType, nullable = false))) val fileSizeMetricsSchema = StructType(Seq( StructField("min", LongType, nullable = true), StructField("max", LongType, nullable = true), StructField("avg", DoubleType, nullable = false), StructField("totalFiles", LongType, nullable = false), StructField("totalSize", LongType, nullable = false) )) val parallelismMetricsSchema = StructType(Seq( StructField("maxClusterActiveParallelism", LongType, nullable = true), StructField("minClusterActiveParallelism", LongType, nullable = true), StructField("maxSessionActiveParallelism", LongType, nullable = true), StructField("minSessionActiveParallelism", LongType, nullable = true) )) val dvMetricsSchema = StructType(Seq( StructField("numDeletionVectorsRemoved", LongType, nullable = false), StructField("numDeletionVectorRowsRemoved", LongType, nullable = false) )) val optimizeMetricsSchema = StructType(Seq( StructField("numFilesAdded", LongType, nullable = false), StructField("numFilesRemoved", LongType, nullable = false), StructField("filesAdded", fileSizeMetricsSchema, nullable = true), StructField("filesRemoved", fileSizeMetricsSchema, nullable = true), StructField("partitionsOptimized", LongType, nullable = false), StructField("zOrderStats", zOrderStatsSchema, nullable = true), StructField("clusteringStats", clusteringStatsSchema, nullable = true), StructField("numBins", LongType, nullable = false), StructField("numBatches", LongType, nullable = false), StructField("totalConsideredFiles", LongType, nullable = false), StructField("totalFilesSkipped", LongType, nullable = false), StructField("preserveInsertionOrder", BooleanType, nullable = false), StructField("numFilesSkippedToReduceWriteAmplification", LongType, nullable = false), StructField("numBytesSkippedToReduceWriteAmplification", LongType, nullable = false), StructField("startTimeMs", LongType, nullable = false), StructField("endTimeMs", LongType, nullable = false), StructField("totalClusterParallelism", LongType, nullable = false), StructField("totalScheduledTasks", LongType, nullable = false), StructField("autoCompactParallelismStats", parallelismMetricsSchema, nullable = true), StructField("deletionVectorStats", dvMetricsSchema, nullable = true), StructField("numTableColumns", LongType, nullable = false), StructField("numTableColumnsWithStats", LongType, nullable = false) )) val optimizeSchema = StructType(Seq( StructField("path", StringType, nullable = true), StructField("metrics", optimizeMetricsSchema, nullable = true) )) withTempDir { tempDir => spark.range(0, 10).write.format("delta").save(tempDir.toString) val res = sql(s"OPTIMIZE delta.`${tempDir.toString}`") assert(res.schema == optimizeSchema) } } test("optimize operation metrics in Delta table history") { withSQLConf(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true") { withTempDir { tempDir => val sampleData = 0.to(79).seq ++ 40.to(79).seq ++ 60.to(79).seq ++ 70.to(79).seq ++ 75.to(79).seq // partition the data and write to test table sampleData.toDF().withColumn("p", floor('value / 10)).repartition(4) .write.partitionBy("p").format("delta").save(tempDir.toString) spark.sql(s"OPTIMIZE delta.`${tempDir.toString}`") // run optimize on the table val actualOperationMetricsAndName = DeltaTable.forPath(spark, tempDir.getAbsolutePath) .history(1) .select("operationMetrics", "operation") .head val actualOperationMetrics = actualOperationMetricsAndName .getMap(0) .asInstanceOf[Map[String, String]] // File sizes depend on the order of how they are merged (=> compression). In order to avoid // flaky test, just test that the metric exists. Seq( "numAddedFiles", "numAddedBytes", "numRemovedBytes", "numRemovedFiles", "numRemovedBytes", "minFileSize", "maxFileSize", "p25FileSize", "p50FileSize", "p75FileSize", "numDeletionVectorsRemoved" ).foreach(metric => assert(actualOperationMetrics.get(metric).isDefined)) val operationName = actualOperationMetricsAndName(1).asInstanceOf[String] assert(operationName === DeltaOperations.OPTIMIZE_OPERATION_NAME) } } } test("optimize metrics on idempotent operations") { val tblName = "tblName" withTable(tblName) { // Create Delta table spark.range(10).write.format("delta").saveAsTable(tblName) // First Optimize spark.sql(s"OPTIMIZE $tblName") // Second Optimize val res = spark.sql(s"OPTIMIZE $tblName") val actMetrics: OptimizeMetrics = res.select($"metrics.*").as[OptimizeMetrics].head() var preserveInsertionOrder = false val expMetrics = OptimizeMetrics( numFilesAdded = 0, numFilesRemoved = 0, filesAdded = FileSizeStats().toFileSizeMetrics, filesRemoved = FileSizeStats().toFileSizeMetrics, partitionsOptimized = 0, zOrderStats = None, numBins = 0, numBatches = 1, totalConsideredFiles = 1, totalFilesSkipped = 1, preserveInsertionOrder = preserveInsertionOrder, startTimeMs = actMetrics.startTimeMs, endTimeMs = actMetrics.endTimeMs, totalClusterParallelism = 2, totalScheduledTasks = 0, numTableColumns = 1, numTableColumnsWithStats = 1) assert(actMetrics === expMetrics) } } test("optimize metrics when certain table columns have no stats") { val tblName = "tblName" withTable(tblName) { // Create Delta table with 5 columns spark.range(10) .withColumn("col2", 'id * 2) .withColumn("col3", 'id * 3) .withColumn("col4", 'id * 4) .withColumn("col5", 'id * 5) .write.format("delta").saveAsTable(tblName) // Set to only collect data skipping stats on 3 columns spark.sql(s""" |ALTER TABLE $tblName |SET TBLPROPERTIES ( | 'delta.dataSkippingNumIndexedCols' = '3' |)""".stripMargin) // Optimize val res = spark.sql(s"OPTIMIZE $tblName") val actMetrics: OptimizeMetrics = res.select($"metrics.*").as[OptimizeMetrics].head() // The table has 5 columns assert(actMetrics.numTableColumns == 5) // There are only 3 columns to collect stats because of the dataSkippingNumIndexedCols config assert(actMetrics.numTableColumnsWithStats == 3) } } test("optimize ZOrderBy operation metrics in Delta table history") { withSQLConf( DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED.key -> "true") { withTempDir { tempDir => // create a partitioned table with each partition containing multiple files 0.to(100).seq.toDF() .withColumn("col1", floor('value % 7)) .withColumn("col2", floor('value % 27)) .withColumn("p", floor('value % 10)) .repartition(4).write.partitionBy("p").format("delta").save(tempDir.toString) val startSizes = DeltaLog.forTable(spark, tempDir) .unsafeVolatileSnapshot.allFiles.select('size).as[Long].collect().sorted spark.sql(s"OPTIMIZE delta.`${tempDir.toString}` ZORDER BY (col1, col2)").show() val finalSizes = DeltaLog.forTable(spark, tempDir) .unsafeVolatileSnapshot.allFiles.select('size).collect().map(_.getLong(0)).sorted val actualOperation = DeltaTable.forPath(spark, tempDir.getAbsolutePath).history(1) .select( "operationParameters.zOrderBy", "operationMetrics", "operation") .head // Verify ZOrder operation parameters val actualOpParameters = actualOperation.getString(0) assert(actualOpParameters === "[\"col1\",\"col2\"]") // Verify metrics records in commit log. val actualMetrics = actualOperation .getMap(1) .asInstanceOf[Map[String, String]] val expMetricsJson = s"""{ | "numRemovedFiles" : "37", | "numAddedFiles" : "10", | "numAddedBytes" : "${finalSizes.sum}", | "numRemovedBytes" : "${startSizes.sum}", | "minFileSize" : "${finalSizes.min}", | "maxFileSize" : "${finalSizes.max}", | "p25FileSize" : "${finalSizes(finalSizes.length / 4)}", | "p50FileSize" : "${finalSizes(finalSizes.length / 2)}", | "p75FileSize" : "${finalSizes(3 * finalSizes.length / 4)}", | "numDeletionVectorsRemoved" : "0" |}""".stripMargin.trim val expMetrics = JsonUtils.fromJson[Map[String, String]](expMetricsJson) assert(actualMetrics === expMetrics) val operationName = actualOperation(2).asInstanceOf[String] assert(operationName === DeltaOperations.OPTIMIZE_OPERATION_NAME) } } } test("optimize ZOrderBy operation metrics in command output") { withSQLConf( DeltaSQLConf.DELTA_OPTIMIZE_MAX_FILE_SIZE.key -> "1000000") { withTempDir { tempDir => // create a partitioned table with each partition containing multiple files 0.to(100).seq.toDF() .withColumn("col1", floor('value % 7)) .withColumn("col2", floor('value % 27)) .withColumn("p", floor('value % 10)) .repartition(4).write.partitionBy("p").format("delta").save(tempDir.toString) val deltaLog = DeltaLog.forTable(spark, tempDir) val startCount = deltaLog.unsafeVolatileSnapshot.allFiles.count() val startSizes = deltaLog.unsafeVolatileSnapshot.allFiles.select('size).as[Long].collect() val result = spark.sql(s"OPTIMIZE delta.`${tempDir.toString}` ZORDER BY (col1, col2)") val metrics: OptimizeMetrics = result.select($"metrics.*").as[OptimizeMetrics].head() val finalSizes = deltaLog.unsafeVolatileSnapshot.allFiles .select('size).collect().map(_.getLong(0)) val finalNumFiles = deltaLog.unsafeVolatileSnapshot.allFiles.collect().length assert(metrics.filesAdded.totalFiles === finalNumFiles) assert(metrics.filesRemoved.totalFiles === startCount) assert(metrics.filesAdded.min.get === finalSizes.min) assert(metrics.filesAdded.max.get === finalSizes.max) assert(metrics.filesRemoved.max.get === startSizes.max) assert(metrics.filesRemoved.min.get === startSizes.min) assert(metrics.totalFilesSkipped === 0) assert(metrics.totalConsideredFiles === metrics.numFilesRemoved) val expZOrderMetrics = s"""{ | "strategyName" : "all", | "inputCubeFiles" : { | "num" : 0, | "size" : 0 | }, | "inputOtherFiles" : { | "num" : $startCount, | "size" : ${startSizes.sum} | }, | "inputNumCubes" : 0, | "mergedFiles" : { | "num" : $startCount, | "size" : ${startSizes.sum} | }, | "numOutputCubes" : 10 |}""".stripMargin assert(metrics.zOrderStats === Some(JsonUtils.fromJson[ZOrderStats](expZOrderMetrics))) } } } val optimizeCommands = Seq("optimize", "zorder", "purge") for (cmd <- optimizeCommands) { testWithDVs(s"deletion vector metrics - $cmd") { withTempDir { dirName => // Create table with 100 files of 10 rows each. val numFiles = 100 val path = dirName.getAbsolutePath spark.range(0, 1000, step = 1, numPartitions = numFiles) .write.format("delta").save(path) val tableName = s"delta.`$path`" val deltaTable = DeltaTable.forPath(spark, path) val deltaLog = DeltaLog.forTable(spark, path) var allFiles = deltaLog.unsafeVolatileSnapshot.allFiles.collect().toSeq // Delete two rows each from 5 files to create Deletion Vectors. val numFilesWithDVs = 5 val numDeletedRows = numFilesWithDVs * 2 allFiles.take(numFilesWithDVs).foreach( file => removeRowsFromFile(deltaLog, file, Seq(1, 5))) allFiles = deltaLog.unsafeVolatileSnapshot.allFiles.collect().toSeq assert(allFiles.size === numFiles) assert(allFiles.filter(_.deletionVector != null).size === numFilesWithDVs) var expOpName = DeltaOperations.OPTIMIZE_OPERATION_NAME val metrics: Seq[OptimizeMetrics] = cmd match { case "optimize" => spark.sql(s"OPTIMIZE $tableName") .select("metrics.*").as[OptimizeMetrics].collect().toSeq case "zorder" => spark.sql(s"OPTIMIZE $tableName ZORDER BY (id)") .select("metrics.*").as[OptimizeMetrics].collect().toSeq case "purge" => expOpName = DeltaOperations.REORG_OPERATION_NAME spark.sql(s"REORG TABLE $tableName APPLY (PURGE)") .select("metrics.*").as[OptimizeMetrics].collect().toSeq case unknown => throw new IllegalArgumentException(s"Unknown command: $unknown") } // Check DV metrics in the result. assert(metrics.length === 1) val dvStats = metrics.head.deletionVectorStats assert(dvStats.get.numDeletionVectorsRemoved === numFilesWithDVs) assert(dvStats.get.numDeletionVectorRowsRemoved === numDeletedRows) // Check DV metrics in the Delta history. val opMetricsAndName = deltaTable.history.select("operationMetrics", "operation") .head val opMetrics = opMetricsAndName .getMap(0) .asInstanceOf[Map[String, String]] val dvMetrics = opMetrics.keys.filter(_.contains("DeletionVector")) assert(dvMetrics === Set("numDeletionVectorsRemoved")) assert(opMetrics("numDeletionVectorsRemoved") === numFilesWithDVs.toString) val operationName = opMetricsAndName(1).asInstanceOf[String] assert(operationName === expOpName) } } } } class OptimizeMetricsSuite extends OptimizeMetricsSuiteBase with DeltaSQLCommandTest ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/optimize/OptimizeZOrderSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.optimize // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.commands.optimize.OptimizeMetrics import org.apache.spark.sql.delta.sources.DeltaSQLConf._ import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils, TestsStatistics} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import io.delta.tables.DeltaTable import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.functions.{col, floor, lit, max, struct} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.LongType trait OptimizePartitionTableHelper extends QueryTest { def testPartition(str: String)(testFun: => Any): Unit = { test("partitioned table - " + str) { testFun } } } /** Tests for Optimize Z-Order by */ trait OptimizeZOrderSuiteBase extends OptimizePartitionTableHelper with TestsStatistics with SharedSparkSession with DeltaSQLTestUtils with DeltaColumnMappingTestUtils { import testImplicits._ def executeOptimizeTable(table: String, zOrderBy: Seq[String], condition: Option[String] = None): DataFrame def executeOptimizePath(path: String, zOrderBy: Seq[String], condition: Option[String] = None): DataFrame test("optimize command: checks existence of interleaving columns") { withTempDir { tempDir => Seq(1, 2, 3).toDF("value") .select('value, 'value % 2 as "id", 'value % 3 as "id2") .write .format("delta") .save(tempDir.toString) val e = intercept[IllegalArgumentException] { executeOptimizePath(tempDir.getCanonicalPath, Seq("id", "id3")) } assert(Seq("id3", "data schema").forall(e.getMessage.contains)) } } test("optimize command: interleaving columns can't be partitioning columns") { withTempDir { tempDir => Seq(1, 2, 3).toDF("value") .select('value, 'value % 2 as "id", 'value % 3 as "id2") .write .format("delta") .partitionBy("id") .save(tempDir.toString) val e = intercept[IllegalArgumentException] { executeOptimizePath(tempDir.getCanonicalPath, Seq("id", "id2")) } assert(e.getMessage === DeltaErrors.zOrderingOnPartitionColumnException("id").getMessage) } } test("optimize command: interleaving with nested columns") { withTempDir { tempDir => val df = spark.read.json(Seq("""{"a":1,"b":{"c":2,"d":3}}""").toDS()) df.write.format("delta").save(tempDir.toString) executeOptimizePath(tempDir.getCanonicalPath, Seq("a", "b.c")) } } testPartition("optimize on null partition column") { withTempDir { tempDir => (1 to 5).foreach { _ => Seq(("a", 1), ("b", 2), (null.asInstanceOf[String], 3), ("", 4)).toDF("part", "value") .write .partitionBy("part") .format("delta") .mode("append") .save(tempDir.getAbsolutePath) } var df = spark.read.format("delta").load(tempDir.getAbsolutePath) val deltaLog = loadDeltaLog(tempDir.getAbsolutePath) val part = "part".phy(deltaLog) var preOptInputFiles = groupInputFilesByPartition(df.inputFiles, deltaLog) assert(preOptInputFiles.forall(_._2.length > 1)) assert(preOptInputFiles.keys.exists(_ == (part, nullPartitionValue))) executeOptimizePath(tempDir.getAbsolutePath, Seq("value")) df = spark.read.format("delta").load(tempDir.getAbsolutePath) preOptInputFiles = groupInputFilesByPartition(df.inputFiles, deltaLog) assert(preOptInputFiles.forall(_._2.length == 1)) assert(preOptInputFiles.keys.exists(_ == (part, nullPartitionValue))) checkAnswer( df.groupBy('part).count(), Seq(Row("a", 5), Row("b", 5), Row(null, 10)) ) } } test("optimize: Zorder on col name containing dot") { withTempDir { tempDir => (0.to(79).seq ++ 40.to(79).seq ++ 60.to(79).seq ++ 70.to(79).seq ++ 75.to(79).seq) .toDF("id") .withColumn("flat.a", $"id" + 1) .write .format("delta") .save(tempDir.toString) val deltaLog = DeltaLog.forTable(spark, tempDir) val numFilesBefore = deltaLog.snapshot.numOfFiles val res = executeOptimizePath(tempDir.getCanonicalPath, Seq("`flat.a`")) val metrics = res.select($"metrics.*").as[OptimizeMetrics].head() val numFilesAfter = deltaLog.snapshot.numOfFiles assert(metrics.numFilesAdded === numFilesAfter) assert(metrics.numFilesRemoved === numFilesBefore) } } test("optimize: Zorder on a nested column") { withTempDir { tempDir => (0.to(79).seq ++ 40.to(79).seq ++ 60.to(79).seq ++ 70.to(79).seq ++ 75.to(79).seq) .toDF("id") .withColumn("nested", struct(struct('id + 2 as "b", 'id + 3 as "c") as "sub")) .write .format("delta") .save(tempDir.toString) val deltaLog = DeltaLog.forTable(spark, tempDir) val numFilesBefore = deltaLog.snapshot.numOfFiles val res = executeOptimizePath(tempDir.getCanonicalPath, Seq("nested.sub.c")) val metrics = res.select($"metrics.*").as[OptimizeMetrics].head() val numFilesAfter = deltaLog.snapshot.numOfFiles assert(metrics.numFilesAdded === numFilesAfter) assert(metrics.numFilesRemoved === numFilesBefore) } } test("optimize: ZOrder on a column without stats") { withTempDir { tempDir => withSQLConf("spark.databricks.delta.properties.defaults.dataSkippingNumIndexedCols" -> "1", DELTA_OPTIMIZE_ZORDER_COL_STAT_CHECK.key -> "true") { val data = Seq(1, 2, 3).toDF("id") data.withColumn("nested", struct(struct('id + 1 as "p1", 'id + 2 as "p2") as "a", 'id + 3 as "b")) .write .format("delta") .save(tempDir.getAbsolutePath) val e1 = intercept[AnalysisException] { executeOptimizeTable(s"delta.`${tempDir.getPath}`", Seq("nested.b")) } assert(e1.getMessage == DeltaErrors .zOrderingOnColumnWithNoStatsException(Seq[String]("nested.b"), spark) .getMessage) val e2 = intercept[AnalysisException] { executeOptimizeTable(s"delta.`${tempDir.getPath}`", Seq("nested.a.p1")) } assert(e2.getMessage == DeltaErrors .zOrderingOnColumnWithNoStatsException(Seq[String]("nested.a.p1"), spark) .getMessage) val e3 = intercept[AnalysisException] { executeOptimizeTable(s"delta.`${tempDir.getPath}`", Seq("nested.a.p1", "nested.b")) } assert(e3.getMessage == DeltaErrors .zOrderingOnColumnWithNoStatsException( Seq[String]("nested.a.p1", "nested.b"), spark) .getMessage) } } } def optimizeWithBatching( batchSize: String, expectedCommits: Int, condition: Option[String], partitionFileCount: Map[String, Int]): Unit = { withSQLConf(DELTA_OPTIMIZE_BATCH_SIZE.key -> batchSize) { withTempDir { tempDir => def writeData(count: Int): Unit = { spark.range(count).select('id, 'id % 5 as "part") .coalesce(1) .write .partitionBy("part") .format("delta") .mode("append") .save(tempDir.getAbsolutePath) } writeData(10) writeData(100) val data = spark.read.format("delta").load(tempDir.getAbsolutePath()).collect() executeOptimizePath(tempDir.getAbsolutePath, Seq("id"), condition) val df = spark.read.format("delta").load(tempDir.getAbsolutePath) checkAnswer(df, data) val deltaLog = loadDeltaLog(tempDir.getAbsolutePath) val commits = deltaLog.history.getHistory(None) assert(commits.filter(_.operation == "OPTIMIZE").length == expectedCommits) val files = groupInputFilesByPartition(df.inputFiles, deltaLog) for ((part, fileCount) <- partitionFileCount) { assert(files(("part", part)).length == fileCount) } } } } test("optimize command with batching") { // Batch size of 1 byte means each bin will run in its own batch, and lead to 5 batches, // one for each partition. Seq(("1", 5), ("1g", 1)).foreach { case (batchSize, optimizeCommits) => // All partitions should be one file after optimizing val partitionFileCount = (0 to 4).map(_.toString -> 1).toMap optimizeWithBatching(batchSize, optimizeCommits, None, partitionFileCount) } } test("optimize command with where clause and batching") { // Batch size of 1 byte means each bin will run in its own batch, and lead to 2 batches // for the two partitions we are optimizing. Seq(("1", 2), ("1g", 1)).foreach { case (batchSize, optimizeCommits) => // First two partitions should have 1 file, last 3 should have two val partitionFileCount = Map( "0" -> 1, "1" -> 1, "2" -> 2, "3" -> 2, "4" -> 2 ) val files = optimizeWithBatching(batchSize, optimizeCommits, Some("part <= 1"), partitionFileCount) } } test("optimize an empty table with batching") { // Batch size of 1 byte means each bin will run in its own batch withSQLConf(DELTA_OPTIMIZE_BATCH_SIZE.key -> "1") { withTempDir { tempDir => DeltaTable.create(spark) .location(tempDir.getAbsolutePath()) .addColumn("id", LongType) .addColumn("part", LongType) .partitionedBy("part") .execute() // Just make sure it succeeds executeOptimizePath(tempDir.getAbsolutePath, Seq("id")) assert(spark.read.format("delta").load(tempDir.getAbsolutePath()).count() == 0) } } } statsTest("optimize command: interleaving") { def statsDF(deltaLog: DeltaLog): DataFrame = { val (c1, c2, c3) = ("c1".phy(deltaLog), "c2".phy(deltaLog), "c3".phy(deltaLog)) getStatsDf(deltaLog, Seq( $"numRecords", struct($"minValues.`$c1`", $"minValues.`$c2`", $"minValues.`$c3`"), struct($"maxValues.`$c1`", $"maxValues.`$c2`", $"maxValues.`$c3`"))) } withTempDir { tempDir => val deltaLog = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) { val df = spark.range(100) .map(i => (i, 99 - i, (i + 50) % 100)) .toDF("c1", "c2", "c3") df.repartitionByRange(4, $"c1", $"c2", $"c3") .write .format("delta") .save(tempDir.toString) } assert(deltaLog.snapshot.allFiles.count() == 4) checkAnswer(statsDF(deltaLog), Seq( Row(25, Row(0, 75, 50), Row(24, 99, 74)), Row(25, Row(25, 50, 75), Row(49, 74, 99)), Row(25, Row(50, 25, 0), Row(74, 49, 24)), Row(25, Row(75, 0, 25), Row(99, 24, 49)))) withSQLConf( DELTA_OPTIMIZE_MAX_FILE_SIZE.key -> "1000000" ) { val res = executeOptimizePath(tempDir.getCanonicalPath, Seq("c1", "c2", "c3")) val metrics = res.select($"metrics.*").as[OptimizeMetrics].head() assert(metrics.zOrderStats.get.mergedFiles.num == 4) assert(deltaLog.snapshot.allFiles.count() == 1) checkAnswer(statsDF(deltaLog), Row(100, Row(0, 0, 0), Row(99, 99, 99))) } // I want to get 4 files again, in order for this to be comparable to the initial scenario val maxFileSize = deltaLog.snapshot.allFiles.head().size / 4 withSQLConf( DELTA_OPTIMIZE_MAX_FILE_SIZE.key -> maxFileSize.toString ) { val res = executeOptimizePath(tempDir.getCanonicalPath, Seq("c1", "c2", "c3")) val metrics = res.select($"metrics.*").as[OptimizeMetrics].head() val expectedFileCount = 4 val expectedStats: Seq[Row] = Seq( Row(25, Row(0, 50, 50), Row(49, 99, 99)), Row(25, Row(16, 20, 18), Row(79, 83, 85)), Row(25, Row(36, 36, 0), Row(63, 63, 96)), Row(25, Row(64, 0, 14), Row(99, 35, 49))) assert(metrics.zOrderStats.get.mergedFiles.num == 1) assert(deltaLog.snapshot.allFiles.count() == expectedFileCount) checkAnswer(statsDF(deltaLog), expectedStats) } } } } /** * Runs optimize compaction tests using OPTIMIZE SQL */ class OptimizeZOrderSQLSuite extends OptimizeZOrderSuiteBase with DeltaSQLCommandTest { import testImplicits._ def executeOptimizeTable(table: String, zOrderBy: Seq[String], condition: Option[String] = None): DataFrame = { val conditionClause = condition.map(c => s"WHERE $c").getOrElse("") val zOrderClause = s"ZORDER BY (${zOrderBy.mkString(", ")})" spark.sql(s"OPTIMIZE $table $conditionClause $zOrderClause") } def executeOptimizePath(path: String, zOrderBy: Seq[String], condition: Option[String] = None): DataFrame = { executeOptimizeTable(s"'$path'", zOrderBy, condition) } test("optimize command: no need for parenthesis") { withTempDir { tempDir => val df = spark.read.json(Seq("""{"a":1,"b":{"c":2,"d":3}}""").toDS()) df.write.format("delta").save(tempDir.toString) spark.sql(s"OPTIMIZE '${tempDir.getCanonicalPath}' ZORDER BY a, b.c") } } } /** * Runs optimize compaction tests using OPTIMIZE Scala APIs */ class OptimizeZOrderScalaSuite extends OptimizeZOrderSuiteBase with DeltaSQLCommandTest { def executeOptimizeTable(table: String, zOrderBy: Seq[String], condition: Option[String] = None): DataFrame = { if (condition.isDefined) { DeltaTable.forName(table).optimize().where(condition.get).executeZOrderBy(zOrderBy: _*) } else { DeltaTable.forName(table).optimize().executeZOrderBy(zOrderBy: _*) } } def executeOptimizePath(path: String, zOrderBy: Seq[String], condition: Option[String] = None): DataFrame = { if (condition.isDefined) { DeltaTable.forPath(path).optimize().where(condition.get).executeZOrderBy(zOrderBy: _*) } else { DeltaTable.forPath(path).optimize().executeZOrderBy(zOrderBy: _*) } } } class OptimizeZOrderNameColumnMappingSuite extends OptimizeZOrderSQLSuite with DeltaColumnMappingEnableNameMode with DeltaColumnMappingTestUtils class OptimizeZOrderIdColumnMappingSuite extends OptimizeZOrderSQLSuite with DeltaColumnMappingEnableIdMode with DeltaColumnMappingTestUtils ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/perf/OptimizeGeneratedColumnSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.perf import java.sql.Timestamp import java.util.Locale import java.util.concurrent.{CountDownLatch, TimeUnit} import scala.util.matching.Regex // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.sources.DeltaSQLConf.{DELTA_COLLECT_STATS, GENERATED_COLUMN_PARTITION_FILTER_OPTIMIZATION_ENABLED} import org.apache.spark.sql.delta.stats.PrepareDeltaScanBase import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.{DataFrame, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.execution.{FileSourceScanExec, QueryExecution} import org.apache.spark.sql.types.TimestampType import org.apache.spark.util.ThreadUtils import org.apache.spark.util.Utils class OptimizeGeneratedColumnSuite extends GeneratedColumnTest { import testImplicits._ private val regex = new Regex(s"(\\S+)\\s(\\S+)\\sGENERATED\\sALWAYS\\sAS\\s\\((.*)\\s?\\)", "col_name", "data_type", "generated_as") private def getPushedPartitionFilters(queryExecution: QueryExecution): Seq[Expression] = { queryExecution.executedPlan.collectFirst { case scan: FileSourceScanExec => scan.partitionFilters }.getOrElse(Nil) } protected def insertInto(path: String, df: DataFrame) = { df.write.format("delta").mode("append").save(path) } /** * Verify we can recognize an `OptimizablePartitionExpression` and generate corresponding * partition filters correctly. * * @param dataSchema DDL of the data columns * @param partitionSchema DDL of the partition columns * @param generatedColumns a map of generated partition columns defined using the above data * columns * @param expectedPartitionExpr the expected `OptimizablePartitionExpression` to be recognized * @param auxiliaryTestName string to append to the generated test name * @param expressionKey key to check for the optmizable expression if not the default first * word in the data schema * @param skipNested Whether to skip the nested variant of the test * @param filterTestCases test cases for partition filters. The key is the data filter, and the * value is the partition filters we should generate. */ private def testOptimizablePartitionExpression( dataSchema: String, partitionSchema: String, generatedColumns: Map[String, String], expectedPartitionExpr: OptimizablePartitionExpression, auxiliaryTestName: Option[String] = None, expressionKey: Option[String] = None, skipNested: Boolean = false, filterTestCases: Seq[(String, Seq[String])]): Unit = { test(expectedPartitionExpr.toString + auxiliaryTestName.getOrElse("")) { val normalCol = dataSchema.split(" ")(0) withTableName("optimizable_partition_expression") { table => createTable( table, None, s"$dataSchema, $partitionSchema", generatedColumns, generatedColumns.keys.toSeq ) val metadata = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(table))._2.metadata assert(metadata.optimizablePartitionExpressions(expressionKey.getOrElse( normalCol).toLowerCase(Locale.ROOT)) == expectedPartitionExpr :: Nil) filterTestCases.foreach { filterTestCase => val partitionFilters = getPushedPartitionFilters( sql(s"SELECT * from $table where ${filterTestCase._1}").queryExecution) assert(partitionFilters.map(_.sql) == filterTestCase._2) } } } if (!skipNested) { test(expectedPartitionExpr.toString + auxiliaryTestName.getOrElse("") + " nested") { val normalCol = dataSchema.split(" ")(0) val nestedSchema = s"nested struct<${dataSchema.replace(" ", ": ")}>" val updatedGeneratedColumns = generatedColumns.mapValues(v => v.replaceAll(s"(?i)($normalCol)", "nested.$1")).toMap withTableName("optimizable_partition_expression") { table => createTable( table, None, s"$nestedSchema, $partitionSchema", updatedGeneratedColumns, updatedGeneratedColumns.keys.toSeq ) val metadata = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(table))._2.metadata val nestedColPath = s"nested.${expressionKey.getOrElse(normalCol).toLowerCase(Locale.ROOT)}" assert(metadata.optimizablePartitionExpressions(nestedColPath) == expectedPartitionExpr :: Nil) filterTestCases.foreach { filterTestCase => val updatedFilter = filterTestCase._1.replaceAll(s"(?i)($normalCol)", "nested.$1") val partitionFilters = getPushedPartitionFilters( sql(s"SELECT * from $table where $updatedFilter").queryExecution) assert(partitionFilters.map(_.sql) == filterTestCase._2) } } } } } /** Format a human readable SQL filter into Spark's compact SQL format */ private def compactFilter(filter: String): String = { filter.replaceAllLiterally("\n", "") .replaceAll("(?<=\\)) +(?=\\))", "") .replaceAll("(?<=\\() +(?=\\()", "") .replaceAll("\\) +OR +\\(", ") OR (") .replaceAll("\\) +AND +\\(", ") AND (") } testOptimizablePartitionExpression( "eventTime TIMESTAMP", "date DATE", Map("date" -> "CAST(eventTime AS DATE)"), expectedPartitionExpr = DatePartitionExpr("date"), auxiliaryTestName = Option(" from cast(timestamp)"), filterTestCases = Seq( "eventTime < '2021-01-01 18:00:00'" -> Seq("((date <= DATE '2021-01-01') " + "OR ((date <= DATE '2021-01-01') IS NULL))"), "eventTime <= '2021-01-01 18:00:00'" -> Seq("((date <= DATE '2021-01-01') " + "OR ((date <= DATE '2021-01-01') IS NULL))"), "eventTime = '2021-01-01 18:00:00'" -> Seq("((date = DATE '2021-01-01') " + "OR ((date = DATE '2021-01-01') IS NULL))"), "eventTime > '2021-01-01 18:00:00'" -> Seq("((date >= DATE '2021-01-01') " + "OR ((date >= DATE '2021-01-01') IS NULL))"), "eventTime >= '2021-01-01 18:00:00'" -> Seq("((date >= DATE '2021-01-01') " + "OR ((date >= DATE '2021-01-01') IS NULL))"), "eventTime is null" -> Seq("(date IS NULL)"), // Verify we can reverse the order "'2021-01-01 18:00:00' > eventTime" -> Seq("((date <= DATE '2021-01-01') " + "OR ((date <= DATE '2021-01-01') IS NULL))"), "'2021-01-01 18:00:00' >= eventTime" -> Seq("((date <= DATE '2021-01-01') " + "OR ((date <= DATE '2021-01-01') IS NULL))"), "'2021-01-01 18:00:00' = eventTime" -> Seq("((date = DATE '2021-01-01') " + "OR ((date = DATE '2021-01-01') IS NULL))"), "'2021-01-01 18:00:00' < eventTime" -> Seq("((date >= DATE '2021-01-01') " + "OR ((date >= DATE '2021-01-01') IS NULL))"), "'2021-01-01 18:00:00' <= eventTime" -> Seq("((date >= DATE '2021-01-01') " + "OR ((date >= DATE '2021-01-01') IS NULL))"), // Verify date type literal. In theory, the best filter should be date < DATE '2021-01-01'. // But Spark's analyzer converts eventTime < '2021-01-01' to // `eventTime` < TIMESTAMP '2021-01-01 00:00:00'. So it's the same as // eventTime < '2021-01-01 18:00:00' for `OptimizeGeneratedColumn`. "eventTime < '2021-01-01'" -> Seq("((date <= DATE '2021-01-01') " + "OR ((date <= DATE '2021-01-01') IS NULL))") ) ) testOptimizablePartitionExpression( "eventDate DATE", "date DATE", Map("date" -> "CAST(eventDate AS DATE)"), expectedPartitionExpr = DatePartitionExpr("date"), auxiliaryTestName = Option(" from cast(date)"), filterTestCases = Seq( "eventDate < '2021-01-01 18:00:00'" -> Seq("((date <= DATE '2021-01-01') " + "OR ((date <= DATE '2021-01-01') IS NULL))"), "eventDate <= '2021-01-01 18:00:00'" -> Seq("((date <= DATE '2021-01-01') " + "OR ((date <= DATE '2021-01-01') IS NULL))"), "eventDate = '2021-01-01 18:00:00'" -> Seq("((date = DATE '2021-01-01') " + "OR ((date = DATE '2021-01-01') IS NULL))"), "eventDate > '2021-01-01 18:00:00'" -> Seq("((date >= DATE '2021-01-01') " + "OR ((date >= DATE '2021-01-01') IS NULL))"), "eventDate >= '2021-01-01 18:00:00'" -> Seq("((date >= DATE '2021-01-01') " + "OR ((date >= DATE '2021-01-01') IS NULL))"), "eventDate is null" -> Seq("(date IS NULL)"), // Verify we can reverse the order "'2021-01-01 18:00:00' > eventDate" -> Seq("((date <= DATE '2021-01-01') " + "OR ((date <= DATE '2021-01-01') IS NULL))"), "'2021-01-01 18:00:00' >= eventDate" -> Seq("((date <= DATE '2021-01-01') " + "OR ((date <= DATE '2021-01-01') IS NULL))"), "'2021-01-01 18:00:00' = eventDate" -> Seq("((date = DATE '2021-01-01') " + "OR ((date = DATE '2021-01-01') IS NULL))"), "'2021-01-01 18:00:00' < eventDate" -> Seq("((date >= DATE '2021-01-01') " + "OR ((date >= DATE '2021-01-01') IS NULL))"), "'2021-01-01 18:00:00' <= eventDate" -> Seq("((date >= DATE '2021-01-01') " + "OR ((date >= DATE '2021-01-01') IS NULL))"), // Verify date type literal. In theory, the best filter should be date < DATE '2021-01-01'. // But Spark's analyzer converts eventTime < '2021-01-01' to // `eventTime` < TIMESTAMP '2021-01-01 00:00:00'. So it's the same as // eventTime < '2021-01-01 18:00:00' for `OptimizeGeneratedColumn`. "eventDate < '2021-01-01'" -> Seq("((date <= DATE '2021-01-01') " + "OR ((date <= DATE '2021-01-01') IS NULL))") ) ) testOptimizablePartitionExpression( "eventTime TIMESTAMP", "year INT, month INT, day INT, hour INT", Map( "year" -> "YEAR(eventTime)", "month" -> "MONTH(eventTime)", "day" -> "DAY(eventTime)", "hour" -> "HOUR(eventTime)" ), expectedPartitionExpr = YearMonthDayHourPartitionExpr("year", "month", "day", "hour"), filterTestCases = Seq( "eventTime < '2021-01-01 18:00:00'" -> Seq( compactFilter( """( | ( | ( | (year < 2021) | OR | ( | (year = 2021) | AND | (month < 1) | ) | ) | OR | ( | ( | (year = 2021) | AND | (month = 1) | ) | AND | (day < 1) | ) | ) | OR | ( | ( | ( | (year = 2021) | AND | (month = 1) | ) | AND | (day = 1) | ) | AND | (hour <= 18) | ) |) |""".stripMargin)), "eventTime <= '2021-01-01 18:00:00'" -> Seq( compactFilter( """( | ( | ( | (year < 2021) | OR | ( | (year = 2021) | AND | (month < 1) | ) | ) | OR | ( | ( | (year = 2021) | AND | (month = 1) | ) | AND | (day < 1) | ) | ) | OR | ( | ( | ( | (year = 2021) | AND | (month = 1) | ) | AND | (day = 1) | ) | AND | (hour <= 18) | ) |) |""".stripMargin)), "eventTime = '2021-01-01 18:00:00'" -> Seq( "(year = 2021)", "(month = 1)", "(day = 1)", "(hour = 18)" ), "eventTime > '2021-01-01 18:00:00'" -> Seq( compactFilter( """( | ( | ( | (year > 2021) | OR | ( | (year = 2021) | AND | (month > 1) | ) | ) | OR | ( | ( | (year = 2021) | AND | (month = 1) | ) | AND | (day > 1) | ) | ) | OR | ( | ( | ( | (year = 2021) | AND | (month = 1) | ) | AND | (day = 1) | ) | AND | (hour >= 18) | ) |) |""".stripMargin)), "eventTime >= '2021-01-01 18:00:00'" ->Seq( compactFilter( """( | ( | ( | (year > 2021) | OR | ( | (year = 2021) | AND | (month > 1) | ) | ) | OR | ( | ( | (year = 2021) | AND | (month = 1) | ) | AND | (day > 1) | ) | ) | OR | ( | ( | ( | (year = 2021) | AND | (month = 1) | ) | AND | (day = 1) | ) | AND | (hour >= 18) | ) |) |""".stripMargin)), "eventTime is null" -> Seq( "(year IS NULL)", "(month IS NULL)", "(day IS NULL)", "(hour IS NULL)" ) ) ) testOptimizablePartitionExpression( "eventTime TIMESTAMP", "year INT, month INT, day INT", Map( "year" -> "YEAR(eventTime)", "month" -> "MONTH(eventTime)", "day" -> "DAY(eventTime)" ), expectedPartitionExpr = YearMonthDayPartitionExpr("year", "month", "day"), filterTestCases = Seq( "eventTime < '2021-01-01 18:00:00'" -> Seq( compactFilter( """( | ( | (year < 2021) | OR | ( | (year = 2021) | AND | (month < 1) | ) | ) | OR | ( | ( | (year = 2021) | AND | (month = 1) | ) | AND | (day <= 1) | ) |) |""".stripMargin)), "eventTime <= '2021-01-01 18:00:00'" -> Seq( compactFilter( """( | ( | (year < 2021) | OR | ( | (year = 2021) | AND | (month < 1) | ) | ) | OR | ( | ( | (year = 2021) | AND | (month = 1) | ) | AND | (day <= 1) | ) |) |""".stripMargin)), "eventTime = '2021-01-01 18:00:00'" -> Seq( "(year = 2021)", "(month = 1)", "(day = 1)" ), "eventTime > '2021-01-01 18:00:00'" -> Seq( compactFilter( """( | ( | (year > 2021) | OR | ( | (year = 2021) | AND | (month > 1) | ) | ) | OR | ( | ( | (year = 2021) | AND | (month = 1) | ) | AND | (day >= 1) | ) |) |""".stripMargin)), "eventTime >= '2021-01-01 18:00:00'" -> Seq( compactFilter( """( | ( | (year > 2021) | OR | ( | (year = 2021) | AND | (month > 1) | ) | ) | OR | ( | ( | (year = 2021) | AND | (month = 1) | ) | AND | (day >= 1) | ) |) |""".stripMargin)), "eventTime is null" -> Seq( "(year IS NULL)", "(month IS NULL)", "(day IS NULL)" ) ) ) testOptimizablePartitionExpression( "eventTime TIMESTAMP", "year INT, month INT", // Use different cases to verify we can recognize the same column using different cases in // generation expressions. Map( "year" -> "YEAR(EVENTTIME)", "month" -> "MONTH(eventTime)" ), expectedPartitionExpr = YearMonthPartitionExpr("year", "month"), filterTestCases = Seq( "eventTime < '2021-01-01 18:00:00'" -> Seq( compactFilter( """( | (year < 2021) | OR | ( | (year = 2021) | AND | (month <= 1) | ) |) |""".stripMargin)), "eventTime <= '2021-01-01 18:00:00'" -> Seq( compactFilter( """( | (year < 2021) | OR | ( | (year = 2021) | AND | (month <= 1) | ) |) |""".stripMargin)), "eventTime = '2021-01-01 18:00:00'" -> Seq( "(year = 2021)", "(month = 1)" ), "eventTime > '2021-01-01 18:00:00'" -> Seq( compactFilter( """( | (year > 2021) | OR | ( | (year = 2021) | AND | (month >= 1) | ) |) |""".stripMargin)), "eventTime >= '2021-01-01 18:00:00'" -> Seq( compactFilter( """( | (year > 2021) | OR | ( | (year = 2021) | AND | (month >= 1) | ) |) |""".stripMargin)), "eventTime is null" -> Seq("(year IS NULL)", "(month IS NULL)") ) ) testOptimizablePartitionExpression( "eventTime TIMESTAMP", "year INT", Map("year" -> "YEAR(eventTime)"), expectedPartitionExpr = YearPartitionExpr("year"), filterTestCases = Seq( "eventTime < '2021-01-01 18:00:00'" -> Seq("((year <= 2021) " + "OR ((year <= 2021) IS NULL))"), "eventTime <= '2021-01-01 18:00:00'" -> Seq("((year <= 2021) " + "OR ((year <= 2021) IS NULL))"), "eventTime = '2021-01-01 18:00:00'" -> Seq("((year = 2021) " + "OR ((year = 2021) IS NULL))"), "eventTime > '2021-01-01 18:00:00'" -> Seq("((year >= 2021) " + "OR ((year >= 2021) IS NULL))"), "eventTime >= '2021-01-01 18:00:00'" -> Seq("((year >= 2021) " + "OR ((year >= 2021) IS NULL))"), "eventTime is null" -> Seq("(year IS NULL)") ) ) Seq(("YEAR(eventDate)", " from year(date)"), ("YEAR(CAST(eventDate AS DATE))", " from year(cast(date))")) .foreach { case (partCol, auxTestName) => testOptimizablePartitionExpression( "eventDate DATE", "year INT", Map("year" -> partCol), expectedPartitionExpr = YearPartitionExpr("year"), auxiliaryTestName = Option(auxTestName), filterTestCases = Seq( "eventDate < '2021-01-01'" -> Seq("((year <= 2021) " + "OR ((year <= 2021) IS NULL))"), "eventDate <= '2021-01-01'" -> Seq("((year <= 2021) " + "OR ((year <= 2021) IS NULL))"), "eventDate = '2021-01-01'" -> Seq("((year = 2021) " + "OR ((year = 2021) IS NULL))"), "eventDate > '2021-01-01'" -> Seq("((year >= 2021) " + "OR ((year >= 2021) IS NULL))"), "eventDate >= '2021-01-01'" -> Seq("((year >= 2021) " + "OR ((year >= 2021) IS NULL))"), "eventDate is null" -> Seq("(year IS NULL)") ) ) } testOptimizablePartitionExpression( "value STRING", "substr STRING", Map("substr" -> "SUBSTRING(value, 2, 3)"), expectedPartitionExpr = SubstringPartitionExpr("substr", 2, 3), filterTestCases = Seq( "value < 'foo'" -> Nil, "value <= 'foo'" -> Nil, "value = 'foo'" -> Seq("((substr IS NULL) OR (substr = 'oo'))"), "value > 'foo'" -> Nil, "value >= 'foo'" -> Nil, "value is null" -> Seq("(substr IS NULL)") ) ) testOptimizablePartitionExpression( "value STRING", "substr STRING", Map("substr" -> "SUBSTRING(value, 0, 3)"), expectedPartitionExpr = SubstringPartitionExpr("substr", 0, 3), filterTestCases = Seq( "value < 'foo'" -> Seq("((substr IS NULL) OR (substr <= 'foo'))"), "value <= 'foo'" -> Seq("((substr IS NULL) OR (substr <= 'foo'))"), "value = 'foo'" -> Seq("((substr IS NULL) OR (substr = 'foo'))"), "value > 'foo'" -> Seq("((substr IS NULL) OR (substr >= 'foo'))"), "value >= 'foo'" -> Seq("((substr IS NULL) OR (substr >= 'foo'))"), "value is null" -> Seq("(substr IS NULL)") ) ) testOptimizablePartitionExpression( "value STRING", "substr STRING", Map("substr" -> "SUBSTRING(value, 1, 3)"), expectedPartitionExpr = SubstringPartitionExpr("substr", 1, 3), filterTestCases = Seq( "value < 'foo'" -> Seq("((substr IS NULL) OR (substr <= 'foo'))"), "value <= 'foo'" -> Seq("((substr IS NULL) OR (substr <= 'foo'))"), "value = 'foo'" -> Seq("((substr IS NULL) OR (substr = 'foo'))"), "value > 'foo'" -> Seq("((substr IS NULL) OR (substr >= 'foo'))"), "value >= 'foo'" -> Seq("((substr IS NULL) OR (substr >= 'foo'))"), "value is null" -> Seq("(substr IS NULL)") ) ) testOptimizablePartitionExpression( "value STRING", "`my.substr` STRING", Map("my.substr" -> "SUBSTRING(value, 1, 3)"), expectedPartitionExpr = SubstringPartitionExpr("my.substr", 1, 3), filterTestCases = Seq( "value < 'foo'" -> Seq("((`my.substr` IS NULL) OR (`my.substr` <= 'foo'))"), "value <= 'foo'" -> Seq("((`my.substr` IS NULL) OR (`my.substr` <= 'foo'))"), "value = 'foo'" -> Seq("((`my.substr` IS NULL) OR (`my.substr` = 'foo'))"), "value > 'foo'" -> Seq("((`my.substr` IS NULL) OR (`my.substr` >= 'foo'))"), "value >= 'foo'" -> Seq("((`my.substr` IS NULL) OR (`my.substr` >= 'foo'))"), "value is null" -> Seq("(`my.substr` IS NULL)") ) ) testOptimizablePartitionExpression( "outer struct>>", "substr STRING", Map("substr" -> "SUBSTRING(outer.inner.nested.value, 1, 3)"), expectedPartitionExpr = SubstringPartitionExpr("substr", 1, 3), auxiliaryTestName = Some(" deeply nested"), expressionKey = Some("outer.inner.nested.value"), skipNested = true, filterTestCases = Seq( "outer.inner.nested.value < 'foo'" -> Seq("((substr IS NULL) OR (substr <= 'foo'))"), "outer.inner.nested.value <= 'foo'" -> Seq("((substr IS NULL) OR (substr <= 'foo'))"), "outer.inner.nested.value = 'foo'" -> Seq("((substr IS NULL) OR (substr = 'foo'))"), "outer.inner.nested.value > 'foo'" -> Seq("((substr IS NULL) OR (substr >= 'foo'))"), "outer.inner.nested.value >= 'foo'" -> Seq("((substr IS NULL) OR (substr >= 'foo'))"), "outer.inner.nested.value is null" -> Seq("(substr IS NULL)") ) ) testOptimizablePartitionExpression( "eventTime TIMESTAMP", "eventTimeTrunc TIMESTAMP", Map("eventTimeTrunc" -> "date_trunc('YEAR', eventTime)"), expectedPartitionExpr = TimestampTruncPartitionExpr("YEAR", "eventTimeTrunc"), auxiliaryTestName = Option(" from date_trunc(timestamp)"), filterTestCases = Seq( "eventTime < '2021-01-01 18:00:00'" -> Seq("((eventTimeTrunc <= TIMESTAMP '2021-01-01 00:00:00') " + "OR ((eventTimeTrunc <= TIMESTAMP '2021-01-01 00:00:00') IS NULL))"), "eventTime <= '2021-01-01 18:00:00'" -> Seq("((eventTimeTrunc <= TIMESTAMP '2021-01-01 00:00:00') " + "OR ((eventTimeTrunc <= TIMESTAMP '2021-01-01 00:00:00') IS NULL))"), "eventTime = '2021-01-01 18:00:00'" -> Seq("((eventTimeTrunc = TIMESTAMP '2021-01-01 00:00:00') " + "OR ((eventTimeTrunc = TIMESTAMP '2021-01-01 00:00:00') IS NULL))"), "eventTime > '2021-01-01 18:00:00'" -> Seq("((eventTimeTrunc >= TIMESTAMP '2021-01-01 00:00:00') " + "OR ((eventTimeTrunc >= TIMESTAMP '2021-01-01 00:00:00') IS NULL))"), "eventTime >= '2021-01-01 18:00:00'" -> Seq("((eventTimeTrunc >= TIMESTAMP '2021-01-01 00:00:00') " + "OR ((eventTimeTrunc >= TIMESTAMP '2021-01-01 00:00:00') IS NULL))"), "eventTime is null" -> Seq("(eventTimeTrunc IS NULL)"), // Verify we can reverse the order "'2021-01-01 18:00:00' > eventTime" -> Seq("((eventTimeTrunc <= TIMESTAMP '2021-01-01 00:00:00') " + "OR ((eventTimeTrunc <= TIMESTAMP '2021-01-01 00:00:00') IS NULL))"), "'2021-01-01 18:00:00' >= eventTime" -> Seq("((eventTimeTrunc <= TIMESTAMP '2021-01-01 00:00:00') " + "OR ((eventTimeTrunc <= TIMESTAMP '2021-01-01 00:00:00') IS NULL))"), "'2021-01-01 18:00:00' = eventTime" -> Seq("((eventTimeTrunc = TIMESTAMP '2021-01-01 00:00:00') " + "OR ((eventTimeTrunc = TIMESTAMP '2021-01-01 00:00:00') IS NULL))"), "'2021-01-01 18:00:00' < eventTime" -> Seq("((eventTimeTrunc >= TIMESTAMP '2021-01-01 00:00:00') " + "OR ((eventTimeTrunc >= TIMESTAMP '2021-01-01 00:00:00') IS NULL))"), "'2021-01-01 18:00:00' <= eventTime" -> Seq("((eventTimeTrunc >= TIMESTAMP '2021-01-01 00:00:00') " + "OR ((eventTimeTrunc >= TIMESTAMP '2021-01-01 00:00:00') IS NULL))") ) ) testOptimizablePartitionExpression( "eventDate DATE", "eventTimeTrunc TIMESTAMP", Map("eventTimeTrunc" -> "date_trunc('DD', eventDate)"), expectedPartitionExpr = TimestampTruncPartitionExpr("DD", "eventTimeTrunc"), auxiliaryTestName = Option(" from date_trunc(cast(date))"), filterTestCases = Seq( "eventDate < '2021-01-01'" -> Seq("((eventTimeTrunc <= TIMESTAMP '2021-01-01 00:00:00') " + "OR ((eventTimeTrunc <= TIMESTAMP '2021-01-01 00:00:00') IS NULL))"), "eventDate <= '2021-01-01'" -> Seq("((eventTimeTrunc <= TIMESTAMP '2021-01-01 00:00:00') " + "OR ((eventTimeTrunc <= TIMESTAMP '2021-01-01 00:00:00') IS NULL))"), "eventDate = '2021-01-01'" -> Seq("((eventTimeTrunc = TIMESTAMP '2021-01-01 00:00:00') " + "OR ((eventTimeTrunc = TIMESTAMP '2021-01-01 00:00:00') IS NULL))"), "eventDate > '2021-01-01'" -> Seq("((eventTimeTrunc >= TIMESTAMP '2021-01-01 00:00:00') " + "OR ((eventTimeTrunc >= TIMESTAMP '2021-01-01 00:00:00') IS NULL))"), "eventDate >= '2021-01-01'" -> Seq("((eventTimeTrunc >= TIMESTAMP '2021-01-01 00:00:00') " + "OR ((eventTimeTrunc >= TIMESTAMP '2021-01-01 00:00:00') IS NULL))"), "eventDate is null" -> Seq("(eventTimeTrunc IS NULL)"), // Verify we can reverse the order "'2021-01-01' > eventDate" -> Seq("((eventTimeTrunc <= TIMESTAMP '2021-01-01 00:00:00') " + "OR ((eventTimeTrunc <= TIMESTAMP '2021-01-01 00:00:00') IS NULL))"), "'2021-01-01' >= eventDate" -> Seq("((eventTimeTrunc <= TIMESTAMP '2021-01-01 00:00:00') " + "OR ((eventTimeTrunc <= TIMESTAMP '2021-01-01 00:00:00') IS NULL))"), "'2021-01-01' = eventDate" -> Seq("((eventTimeTrunc = TIMESTAMP '2021-01-01 00:00:00') " + "OR ((eventTimeTrunc = TIMESTAMP '2021-01-01 00:00:00') IS NULL))"), "'2021-01-01' < eventDate" -> Seq("((eventTimeTrunc >= TIMESTAMP '2021-01-01 00:00:00') " + "OR ((eventTimeTrunc >= TIMESTAMP '2021-01-01 00:00:00') IS NULL))"), "'2021-01-01' <= eventDate" -> Seq("((eventTimeTrunc >= TIMESTAMP '2021-01-01 00:00:00') " + "OR ((eventTimeTrunc >= TIMESTAMP '2021-01-01 00:00:00') IS NULL))") ) ) testOptimizablePartitionExpression( "value STRING", "part STRING", Map("part" -> "value"), expectedPartitionExpr = IdentityPartitionExpr("part"), expressionKey = Some("value"), filterTestCases = Seq( "value < 'foo'" -> Seq("((part IS NULL) OR (part < 'foo'))"), "value <= 'foo'" -> Seq("((part IS NULL) OR (part <= 'foo'))"), "value = 'foo'" -> Seq("((part IS NULL) OR (part = 'foo'))"), "value > 'foo'" -> Seq("((part IS NULL) OR (part > 'foo'))"), "value >= 'foo'" -> Seq("((part IS NULL) OR (part >= 'foo'))"), "value is null" -> Seq("(part IS NULL)") ) ) /** * In order to distinguish between field names with periods and nested field names, * fields with periods must be escaped. Otherwise in the example below, there's * no way to tell whether a filter on nested.value should be applied to part1 or part2. */ testOptimizablePartitionExpression( "`nested.value` STRING, nested struct", "part1 STRING, part2 STRING", Map("part1" -> "`nested.value`", "part2" -> "nested.value"), auxiliaryTestName = Some(" escaped field names"), expectedPartitionExpr = IdentityPartitionExpr("part1"), expressionKey = Some("`nested.value`"), skipNested = true, filterTestCases = Seq( "`nested.value` < 'foo'" -> Seq("((part1 IS NULL) OR (part1 < 'foo'))"), "`nested.value` <= 'foo'" -> Seq("((part1 IS NULL) OR (part1 <= 'foo'))"), "`nested.value` = 'foo'" -> Seq("((part1 IS NULL) OR (part1 = 'foo'))"), "`nested.value` > 'foo'" -> Seq("((part1 IS NULL) OR (part1 > 'foo'))"), "`nested.value` >= 'foo'" -> Seq("((part1 IS NULL) OR (part1 >= 'foo'))"), "`nested.value` is null" -> Seq("(part1 IS NULL)") ) ) test("end-to-end optimizable partition expression") { withTempDir { tempDir => withTableName("optimizable_partition_expression") { table => createTable( table, Some(tempDir.getCanonicalPath), "c1 INT, c2 TIMESTAMP, c3 DATE", Map("c3" -> "CAST(c2 AS DATE)"), Seq("c3") ) Seq( Tuple2(1, "2020-12-31 11:00:00"), Tuple2(2, "2021-01-01 12:00:00"), Tuple2(3, "2021-01-02 13:00:00") ).foreach { values => insertInto( tempDir.getCanonicalPath, Seq(values).toDF("c1", "c2") .withColumn("c2", $"c2".cast(TimestampType)) ) } assert(tempDir.listFiles().map(_.getName).toSet == Set("c3=2021-01-01", "c3=2021-01-02", "c3=2020-12-31", "_delta_log")) // Delete folders which should not be read if we generate the partition filters correctly tempDir.listFiles().foreach { f => if (f.getName != "c3=2021-01-01" && f.getName != "_delta_log") { Utils.deleteRecursively(f) } } assert(tempDir.listFiles().map(_.getName).toSet == Set("c3=2021-01-01", "_delta_log")) checkAnswer( sql(s"select * from $table where " + s"c2 >= '2021-01-01 12:00:00' AND c2 <= '2021-01-01 18:00:00'"), Row(2, sqlTimestamp("2021-01-01 12:00:00"), sqlDate("2021-01-01"))) // Verify `OptimizeGeneratedColumn` doesn't mess up Projects. checkAnswer( sql(s"select c1 from $table where " + s"c2 >= '2021-01-01 12:00:00' AND c2 <= '2021-01-01 18:00:00'"), Row(2)) // Check both projection orders to make sure projection orders are handled correctly checkAnswer( sql(s"select c1, c2 from $table where " + s"c2 >= '2021-01-01 12:00:00' AND c2 <= '2021-01-01 18:00:00'"), Row(2, Timestamp.valueOf("2021-01-01 12:00:00"))) checkAnswer( sql(s"select c2, c1 from $table where " + s"c2 >= '2021-01-01 12:00:00' AND c2 <= '2021-01-01 18:00:00'"), Row(Timestamp.valueOf("2021-01-01 12:00:00"), 2)) // Verify the optimization works for limit. val limitQuery = sql( s"""select * from $table |where c2 >= '2021-01-01 12:00:00' AND c2 <= '2021-01-01 18:00:00' |limit 10""".stripMargin) val expectedPartitionFilters = Seq( "((c3 >= DATE '2021-01-01') OR ((c3 >= DATE '2021-01-01') IS NULL))", "((c3 <= DATE '2021-01-01') OR ((c3 <= DATE '2021-01-01') IS NULL))" ) assert(expectedPartitionFilters == getPushedPartitionFilters(limitQuery.queryExecution).map(_.sql)) checkAnswer(limitQuery, Row(2, sqlTimestamp("2021-01-01 12:00:00"), sqlDate("2021-01-01"))) } } } test("empty string and null ambiguity in a partition column") { withTempDir { tempDir => withTableName("optimizable_partition_expression") { table => createTable( table, Some(tempDir.getCanonicalPath), "c1 STRING, c2 STRING", Map("c2" -> "SUBSTRING(c1, 1, 4)"), Seq("c2") ) insertInto( tempDir.getCanonicalPath, Seq(Tuple1("")).toDF("c1") ) checkAnswer( sql(s"select * from $table where c1 = ''"), Row("", null)) // The following check shows the weird behavior of SPARK-24438 and confirms the generated // partition filter doesn't impact the answer. withSQLConf(GENERATED_COLUMN_PARTITION_FILTER_OPTIMIZATION_ENABLED.key -> "false") { checkAnswer( sql(s"select * from $table where c1 = ''"), Row("", null)) } } } } test("substring on multibyte characters") { withTempDir { tempDir => withTableName("multibyte_characters") { table => createTable( table, Some(tempDir.getCanonicalPath), "c1 STRING, c2 STRING", Map("c2" -> "SUBSTRING(c1, 1, 2)"), Seq("c2") ) // scalastyle:off nonascii insertInto( tempDir.getCanonicalPath, Seq(Tuple1("一二三四")).toDF("c1") ) val testQuery = s"select * from $table where c1 > 'abcd'" assert("((c2 IS NULL) OR (c2 >= 'ab'))" :: Nil == getPushedPartitionFilters(sql(testQuery).queryExecution).map(_.sql)) checkAnswer( sql(testQuery), Row("一二三四", "一二")) // scalastyle:on nonascii } } } testOptimizablePartitionExpression( "eventTime TIMESTAMP", "month STRING", Map("month" -> "DATE_FORMAT(eventTime, 'yyyy-MM')"), expectedPartitionExpr = DateFormatPartitionExpr("month", "yyyy-MM"), auxiliaryTestName = Option(" from timestamp"), filterTestCases = Seq( "eventTime < '2021-06-28 18:00:00'" -> Seq("((unix_timestamp(month, 'yyyy-MM') <= 1622530800L) " + "OR ((unix_timestamp(month, 'yyyy-MM') <= 1622530800L) IS NULL))"), "eventTime <= '2021-06-28 18:00:00'" -> Seq("((unix_timestamp(month, 'yyyy-MM') <= 1622530800L) " + "OR ((unix_timestamp(month, 'yyyy-MM') <= 1622530800L) IS NULL))"), "eventTime = '2021-06-28 18:00:00'" -> Seq("((unix_timestamp(month, 'yyyy-MM') = 1622530800L) " + "OR ((unix_timestamp(month, 'yyyy-MM') = 1622530800L) IS NULL))"), "eventTime > '2021-06-28 18:00:00'" -> Seq("((unix_timestamp(month, 'yyyy-MM') >= 1622530800L) " + "OR ((unix_timestamp(month, 'yyyy-MM') >= 1622530800L) IS NULL))"), "eventTime >= '2021-06-28 18:00:00'" -> Seq("((unix_timestamp(month, 'yyyy-MM') >= 1622530800L) " + "OR ((unix_timestamp(month, 'yyyy-MM') >= 1622530800L) IS NULL))"), "eventTime is null" -> Seq("(month IS NULL)") ) ) testOptimizablePartitionExpression( "eventDate DATE", "month STRING", Map("month" -> "DATE_FORMAT(eventDate, 'yyyy-MM')"), expectedPartitionExpr = DateFormatPartitionExpr("month", "yyyy-MM"), auxiliaryTestName = Option(" from cast(date)"), filterTestCases = Seq( "eventDate < '2021-06-28 18:00:00'" -> Seq("((unix_timestamp(month, 'yyyy-MM') <= 1622530800L) " + "OR ((unix_timestamp(month, 'yyyy-MM') <= 1622530800L) IS NULL))"), "eventDate <= '2021-06-28 18:00:00'" -> Seq("((unix_timestamp(month, 'yyyy-MM') <= 1622530800L) " + "OR ((unix_timestamp(month, 'yyyy-MM') <= 1622530800L) IS NULL))"), "eventDate = '2021-06-28 18:00:00'" -> Seq("((unix_timestamp(month, 'yyyy-MM') = 1622530800L) " + "OR ((unix_timestamp(month, 'yyyy-MM') = 1622530800L) IS NULL))"), "eventDate > '2021-06-28 18:00:00'" -> Seq("((unix_timestamp(month, 'yyyy-MM') >= 1622530800L) " + "OR ((unix_timestamp(month, 'yyyy-MM') >= 1622530800L) IS NULL))"), "eventDate >= '2021-06-28 18:00:00'" -> Seq("((unix_timestamp(month, 'yyyy-MM') >= 1622530800L) " + "OR ((unix_timestamp(month, 'yyyy-MM') >= 1622530800L) IS NULL))"), "eventDate is null" -> Seq("(month IS NULL)") ) ) testOptimizablePartitionExpression( "eventTime TIMESTAMP", "day STRING", Map("day" -> "DATE_FORMAT(eventTime, 'yyyy-MM-dd')"), expectedPartitionExpr = DateFormatPartitionExpr("day", "yyyy-MM-dd"), auxiliaryTestName = Option(" from timestamp"), filterTestCases = Seq( "eventTime < '2021-06-28 18:00:00'" -> Seq("((unix_timestamp(day, 'yyyy-MM-dd') <= 1624863600L) " + "OR ((unix_timestamp(day, 'yyyy-MM-dd') <= 1624863600L) IS NULL))"), "eventTime <= '2021-06-28 18:00:00'" -> Seq("((unix_timestamp(day, 'yyyy-MM-dd') <= 1624863600L) " + "OR ((unix_timestamp(day, 'yyyy-MM-dd') <= 1624863600L) IS NULL))"), "eventTime = '2021-06-28 18:00:00'" -> Seq("((unix_timestamp(day, 'yyyy-MM-dd') = 1624863600L) " + "OR ((unix_timestamp(day, 'yyyy-MM-dd') = 1624863600L) IS NULL))"), "eventTime > '2021-06-28 18:00:00'" -> Seq("((unix_timestamp(day, 'yyyy-MM-dd') >= 1624863600L) " + "OR ((unix_timestamp(day, 'yyyy-MM-dd') >= 1624863600L) IS NULL))"), "eventTime >= '2021-06-28 18:00:00'" -> Seq("((unix_timestamp(day, 'yyyy-MM-dd') >= 1624863600L) " + "OR ((unix_timestamp(day, 'yyyy-MM-dd') >= 1624863600L) IS NULL))"), "eventTime is null" -> Seq("(day IS NULL)") ) ) testOptimizablePartitionExpression( "eventTime TIMESTAMP", "hour STRING", Map("hour" -> "(DATE_FORMAT(eventTime, 'yyyy-MM-dd-HH'))"), expectedPartitionExpr = DateFormatPartitionExpr("hour", "yyyy-MM-dd-HH"), filterTestCases = Seq( "eventTime < '2021-06-28 18:00:00'" -> Seq("((unix_timestamp(hour, 'yyyy-MM-dd-HH') <= 1624928400L) " + "OR ((unix_timestamp(hour, 'yyyy-MM-dd-HH') <= 1624928400L) IS NULL))"), "eventTime <= '2021-06-28 18:00:00'" -> Seq("((unix_timestamp(hour, 'yyyy-MM-dd-HH') <= 1624928400L) " + "OR ((unix_timestamp(hour, 'yyyy-MM-dd-HH') <= 1624928400L) IS NULL))"), "eventTime = '2021-06-28 18:00:00'" -> Seq("((unix_timestamp(hour, 'yyyy-MM-dd-HH') = 1624928400L) " + "OR ((unix_timestamp(hour, 'yyyy-MM-dd-HH') = 1624928400L) IS NULL))"), "eventTime > '2021-06-28 18:00:00'" -> Seq("((unix_timestamp(hour, 'yyyy-MM-dd-HH') >= 1624928400L) " + "OR ((unix_timestamp(hour, 'yyyy-MM-dd-HH') >= 1624928400L) IS NULL))"), "eventTime >= '2021-06-28 18:00:00'" -> Seq("((unix_timestamp(hour, 'yyyy-MM-dd-HH') >= 1624928400L) " + "OR ((unix_timestamp(hour, 'yyyy-MM-dd-HH') >= 1624928400L) IS NULL))"), "eventTime is null" -> Seq("(hour IS NULL)") ) ) testOptimizablePartitionExpression( "eventTime TIMESTAMP", "date DATE", Map("date" -> "(trunc(eventTime, 'year'))"), expectedPartitionExpr = TruncDatePartitionExpr("date", "year"), filterTestCases = Seq( "eventTime < '2021-01-01 18:00:00'" -> Seq("((date <= DATE '2021-01-01') " + "OR ((date <= DATE '2021-01-01') IS NULL))"), "eventTime <= '2021-01-01 18:00:00'" -> Seq("((date <= DATE '2021-01-01') " + "OR ((date <= DATE '2021-01-01') IS NULL))"), "eventTime = '2021-01-01 18:00:00'" -> Seq("((date = DATE '2021-01-01') " + "OR ((date = DATE '2021-01-01') IS NULL))"), "eventTime > '2021-01-01 18:00:00'" -> Seq("((date >= DATE '2021-01-01') " + "OR ((date >= DATE '2021-01-01') IS NULL))"), "eventTime >= '2021-01-01 18:00:00'" -> Seq("((date >= DATE '2021-01-01') " + "OR ((date >= DATE '2021-01-01') IS NULL))"), "eventTime is null" -> Seq("(date IS NULL)") ) ) testOptimizablePartitionExpression( "eventDate DATE", "date DATE", Map("date" -> "(trunc(eventDate, 'month'))"), expectedPartitionExpr = TruncDatePartitionExpr("date", "month"), filterTestCases = Seq( "eventDate < '2021-12-01'" -> Seq("((date <= DATE '2021-12-01') " + "OR ((date <= DATE '2021-12-01') IS NULL))"), "eventDate <= '2021-12-01'" -> Seq("((date <= DATE '2021-12-01') " + "OR ((date <= DATE '2021-12-01') IS NULL))"), "eventDate = '2021-12-01'" -> Seq("((date = DATE '2021-12-01') " + "OR ((date = DATE '2021-12-01') IS NULL))"), "eventDate > '2021-12-01'" -> Seq("((date >= DATE '2021-12-01') " + "OR ((date >= DATE '2021-12-01') IS NULL))"), "eventDate >= '2021-12-01'" -> Seq("((date >= DATE '2021-12-01') " + "OR ((date >= DATE '2021-12-01') IS NULL))"), "eventDate is null" -> Seq("(date IS NULL)") ) ) testOptimizablePartitionExpression( "eventDateStr STRING", "date DATE", Map("date" -> "(trunc(eventDateStr, 'quarter'))"), expectedPartitionExpr = TruncDatePartitionExpr("date", "quarter"), filterTestCases = Seq( "eventDateStr < '2022-04-01'" -> Seq("((date <= DATE '2022-04-01') " + "OR ((date <= DATE '2022-04-01') IS NULL))"), "eventDateStr <= '2022-04-01'" -> Seq("((date <= DATE '2022-04-01') " + "OR ((date <= DATE '2022-04-01') IS NULL))"), "eventDateStr = '2022-04-01'" -> Seq("((date = DATE '2022-04-01') " + "OR ((date = DATE '2022-04-01') IS NULL))"), "eventDateStr > '2022-04-01'" -> Seq("((date >= DATE '2022-04-01') " + "OR ((date >= DATE '2022-04-01') IS NULL))"), "eventDateStr >= '2022-04-01'" -> Seq("((date >= DATE '2022-04-01') " + "OR ((date >= DATE '2022-04-01') IS NULL))"), "eventDateStr is null" -> Seq("(date IS NULL)") ) ) test("five digits year in a year month day partition column") { withTempDir { tempDir => withTableName("optimizable_partition_expression") { table => createTable( table, Some(tempDir.getCanonicalPath), "c1 TIMESTAMP, c2 INT, c3 INT, c4 INT", Map( "c2" -> "YEAR(c1)", "c3" -> "MONTH(c1)", "c4" -> "DAY(c1)" ), Seq("c2", "c3", "c4") ) insertInto( tempDir.getCanonicalPath, Seq(Tuple1("12345-07-15 18:00:00")) .toDF("c1") .withColumn("c1", $"c1".cast(TimestampType)) ) checkAnswer( sql(s"select * from $table where c1 = CAST('12345-07-15 18:00:00' as timestamp)"), Row(new Timestamp(327420320400000L), 12345, 7, 15)) withSQLConf(GENERATED_COLUMN_PARTITION_FILTER_OPTIMIZATION_ENABLED.key -> "false") { checkAnswer( sql(s"select * from $table where c1 = CAST('12345-07-15 18:00:00' as timestamp)"), Row(new Timestamp(327420320400000L), 12345, 7, 15)) } } } } test("five digits year in a date_format yyyy-MM partition column") { withTempDir { tempDir => withTableName("optimizable_partition_expression") { table => createTable( table, Some(tempDir.getCanonicalPath), "c1 TIMESTAMP, c2 STRING", Map("c2" -> "DATE_FORMAT(c1, 'yyyy-MM')"), Seq("c2") ) insertInto( tempDir.getCanonicalPath, Seq(Tuple1("12345-07-15 18:00:00")) .toDF("c1") .withColumn("c1", $"c1".cast(TimestampType)) ) checkAnswer( sql(s"select * from $table where c1 = CAST('12345-07-15 18:00:00' as timestamp)"), Row(new Timestamp(327420320400000L), "+12345-07")) withSQLConf(GENERATED_COLUMN_PARTITION_FILTER_OPTIMIZATION_ENABLED.key -> "false") { checkAnswer( sql(s"select * from $table where c1 = CAST('12345-07-15 18:00:00' as timestamp)"), Row(new Timestamp(327420320400000L), "+12345-07")) } } } } test("five digits year in a date_format yyyy-MM-dd-HH partition column") { withTempDir { tempDir => withTableName("optimizable_partition_expression") { table => createTable( table, Some(tempDir.getCanonicalPath), "c1 TIMESTAMP, c2 STRING", Map("c2" -> "DATE_FORMAT(c1, 'yyyy-MM-dd-HH')"), Seq("c2") ) insertInto( tempDir.getCanonicalPath, Seq(Tuple1("12345-07-15 18:00:00")) .toDF("c1") .withColumn("c1", $"c1".cast(TimestampType)) ) checkAnswer( sql(s"select * from $table where c1 = CAST('12345-07-15 18:00:00' as timestamp)"), Row(new Timestamp(327420320400000L), "+12345-07-15-18")) withSQLConf(GENERATED_COLUMN_PARTITION_FILTER_OPTIMIZATION_ENABLED.key -> "false") { checkAnswer( sql(s"select * from $table where c1 = CAST('12345-07-15 18:00:00' as timestamp)"), Row(new Timestamp(327420320400000L), "+12345-07-15-18")) } } } } test("end-to-end test of behaviors of write/read null on partition column") { // unix_timestamp('12345-12', 'yyyy-MM') | unix_timestamp('+12345-12', 'yyyy-MM') // EXCEPTION fail | 327432240000 // CORRECTED null | 327432240000 // LEGACY 327432240000 | null withTempDir { tempDir => withTableName("optimizable_partition_expression") { table => createTable( table, Some(tempDir.getCanonicalPath), "c1 TIMESTAMP, c2 STRING", Map("c2" -> "DATE_FORMAT(c1, 'yyyy-MM')"), Seq("c2") ) // write in LEGACY withSQLConf( "spark.sql.legacy.timeParserPolicy" -> "CORRECTED" ) { insertInto( tempDir.getCanonicalPath, Seq(Tuple1("12345-07-01 00:00:00")) .toDF("c1") .withColumn("c1", $"c1".cast(TimestampType)) ) insertInto( tempDir.getCanonicalPath, Seq(Tuple1("+23456-07-20 18:30:00")) .toDF("c1") .withColumn("c1", $"c1".cast(TimestampType)) ) } // write in LEGACY withSQLConf( "spark.sql.legacy.timeParserPolicy" -> "LEGACY" ) { insertInto( tempDir.getCanonicalPath, Seq(Tuple1("+12349-07-01 00:00:00")) .toDF("c1") .withColumn("c1", $"c1".cast(TimestampType)) ) insertInto( tempDir.getCanonicalPath, Seq(Tuple1("+30000-12-30 20:00:00")) .toDF("c1") .withColumn("c1", $"c1".cast(TimestampType)) ) } // we have partitions based on CORRECTED + LEGACY parser (with +) assert(tempDir.listFiles().map(_.getName).toSet == Set("c2=+23456-07", "c2=12349-07", "c2=30000-12", "c2=+12345-07", "_delta_log")) // read behaviors in CORRECTED, we still can query correctly withSQLConf("spark.sql.legacy.timeParserPolicy" -> "CORRECTED") { checkAnswer( sql(s"select (unix_timestamp('+20000-01', 'yyyy-MM')) as value"), Row(568971849600L) ) withSQLConf("spark.sql.ansi.enabled" -> "false") { checkAnswer( sql(s"select (unix_timestamp('20000-01', 'yyyy-MM')) as value"), Row(null) ) checkAnswer( sql(s"select * from $table where " + s"c1 >= '20000-01-01 12:00:00'"), // 23456-07-20 18:30:00 Row(new Timestamp(678050098200000L), "+23456-07") :: // 30000-12-30 20:00:00 Row(new Timestamp(884572891200000L), "30000-12") :: Nil ) } } // read behaviors in LEGACY, we still can query correctly withSQLConf("spark.sql.legacy.timeParserPolicy" -> "LEGACY") { checkAnswer( sql(s"select (unix_timestamp('20000-01', 'yyyy-MM')) as value"), Row(568971849600L) ) withSQLConf("spark.sql.ansi.enabled" -> "false") { checkAnswer( sql(s"select (unix_timestamp('+20000-01', 'yyyy-MM')) as value"), Row(null) ) checkAnswer( sql(s"select * from $table where " + s"c1 >= '20000-01-01 12:00:00'"), // 23456-07-20 18:30:00 Row(new Timestamp(678050098200000L), "+23456-07") :: // 30000-12-30 20:00:00 Row(new Timestamp(884572891200000L), "30000-12") :: Nil ) } } } } } test("generated partition filters should avoid conflicts") { withTempDir { tempDir => val path = tempDir.getCanonicalPath withTableName("avoid_conflicts") { table => createTable( table, Some(path), "eventTime TIMESTAMP, date DATE", Map("date" -> "CAST(eventTime AS DATE)"), Seq("date") ) insertInto( path, Seq(Tuple1("2021-01-01 00:00:00"), Tuple1("2021-01-02 00:00:00")) .toDF("eventTime") .withColumn("eventTime", $"eventTime".cast(TimestampType)) ) val unblockQueries = new CountDownLatch(1) val waitForAllQueries = new CountDownLatch(2) PrepareDeltaScanBase.withCallbackOnGetDeltaScanGenerator(_ => { waitForAllQueries.countDown() assert( unblockQueries.await(30, TimeUnit.SECONDS), "the main thread didn't wake up queries") }) { val threadPool = ThreadUtils.newDaemonFixedThreadPool(2, "test") try { // Run two queries that should not conflict with each other if we generate the partition // filter correctly. val f1 = threadPool.submit(() => { spark.read.format("delta").load(path).where("eventTime = '2021-01-01 00:00:00'") .write.mode("append").format("delta").save(path) true }) val f2 = threadPool.submit(() => { spark.read.format("delta").load(path).where("eventTime = '2021-01-02 00:00:00'") .write.mode("append").format("delta").save(path) true }) assert( waitForAllQueries.await(30, TimeUnit.SECONDS), "queries didn't finish before timeout") unblockQueries.countDown() f1.get(30, TimeUnit.SECONDS) f2.get(30, TimeUnit.SECONDS) } finally { threadPool.shutdownNow() } } } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/perf/OptimizeMetadataOnlyDeltaQuerySuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.perf import scala.collection.mutable import org.apache.spark.sql.delta.{DeletionVectorsTestUtils, DeltaColumnMappingEnableIdMode, DeltaColumnMappingEnableNameMode, DeltaLog, DeltaTestUtils} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.PrepareDeltaScanBase import org.apache.spark.sql.delta.stats.StatisticsCollection import org.apache.spark.sql.delta.test.DeltaColumnMappingSelectedTestMixin import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import io.delta.tables.DeltaTable import org.apache.hadoop.fs.Path import org.scalatest.BeforeAndAfterAll import org.apache.spark.sql.{DataFrame, Dataset, QueryTest, Row, SaveMode} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.plans.logical.LocalRelation import org.apache.spark.sql.functions._ import org.apache.spark.sql.test.SharedSparkSession class OptimizeMetadataOnlyDeltaQuerySuite extends QueryTest with SharedSparkSession with BeforeAndAfterAll with DeltaSQLCommandTest with DeletionVectorsTestUtils { val testTableName = "table_basic" val noStatsTableName = " table_nostats" val mixedStatsTableName = " table_mixstats" var dfPart1: DataFrame = null var dfPart2: DataFrame = null var totalRows: Long = -1 var minId: Long = -1 var maxId: Long = -1 override def beforeAll(): Unit = { super.beforeAll() dfPart1 = generateRowsDataFrame(spark.range(1L, 6L)) dfPart2 = generateRowsDataFrame(spark.range(6L, 11L)) withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false") { dfPart1.write.format("delta").mode(SaveMode.Overwrite).saveAsTable(noStatsTableName) dfPart1.write.format("delta").mode(SaveMode.Overwrite).saveAsTable(mixedStatsTableName) spark.sql(s"DELETE FROM $noStatsTableName WHERE id = 1") spark.sql(s"DELETE FROM $mixedStatsTableName WHERE id = 1") dfPart2.write.format("delta").mode("append").saveAsTable(noStatsTableName) } withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "true") { dfPart1.write.format("delta").mode(SaveMode.Overwrite).saveAsTable(testTableName) spark.sql(s"DELETE FROM $testTableName WHERE id = 1") dfPart2.write.format("delta").mode(SaveMode.Append).saveAsTable(testTableName) dfPart2.write.format("delta").mode(SaveMode.Append).saveAsTable(mixedStatsTableName) // Run updates to generate more Delta Log and trigger a checkpoint // and make sure stats works after checkpoints for (a <- 1 to 10) { spark.sql(s"UPDATE $testTableName SET data='$a' WHERE id = 7") } spark.sql(s"UPDATE $testTableName SET data=NULL WHERE id = 7") // Creates an empty (numRecords == 0) AddFile record generateRowsDataFrame(spark.range(11L, 12L)) .write.format("delta").mode("append").saveAsTable(testTableName) spark.sql(s"DELETE FROM $testTableName WHERE id = 11") } withSQLConf(DeltaSQLConf.DELTA_OPTIMIZE_METADATA_QUERY_ENABLED.key -> "false") { val result = spark.sql(s"SELECT COUNT(*), MIN(id), MAX(id) FROM $testTableName").head totalRows = result.getLong(0) minId = result.getLong(1) maxId = result.getLong(2) } } /** Class to hold test parameters */ case class ScalaTestParams(name: String, queryScala: () => DataFrame, expectedPlan: String) Seq( new ScalaTestParams( name = "count - simple query", queryScala = () => spark.read.format("delta").table(testTableName) .agg(count(col("*"))), expectedPlan = "LocalRelation [none#0L]"), new ScalaTestParams( name = "min-max - simple query", queryScala = () => spark.read.format("delta").table(testTableName) .agg(min(col("id")), max(col("id")), min(col("TinyIntColumn")), max(col("TinyIntColumn")), min(col("SmallIntColumn")), max(col("SmallIntColumn")), min(col("IntColumn")), max(col("IntColumn")), min(col("BigIntColumn")), max(col("BigIntColumn")), min(col("FloatColumn")), max(col("FloatColumn")), min(col("DoubleColumn")), max(col("DoubleColumn")), min(col("DateColumn")), max(col("DateColumn"))), expectedPlan = "LocalRelation [none#0L, none#1L, none#2, none#3, none#4, none#5, none#6" + ", none#7, none#8L, none#9L, none#10, none#11, none#12, none#13, none#14, none#15]")) .foreach { testParams => test(s"optimization supported - Scala - ${testParams.name}") { checkResultsAndOptimizedPlan(testParams.queryScala, testParams.expectedPlan) } } /** Class to hold test parameters */ case class SqlTestParams( name: String, querySql: String, expectedPlan: String, querySetup: Option[Seq[String]] = None) Seq( new SqlTestParams( name = "count - simple query", querySql = s"SELECT COUNT(*) FROM $testTableName", expectedPlan = "LocalRelation [none#0L]"), new SqlTestParams( name = "min-max - simple query", querySql = s"SELECT MIN(id), MAX(id)" + s", MIN(TinyIntColumn), MAX(TinyIntColumn)" + s", MIN(SmallIntColumn), MAX(SmallIntColumn)" + s", MIN(IntColumn), MAX(IntColumn)" + s", MIN(BigIntColumn), MAX(BigIntColumn)" + s", MIN(FloatColumn), MAX(FloatColumn)" + s", MIN(DoubleColumn), MAX(DoubleColumn)" + s", MIN(DateColumn), MAX(DateColumn)" + s"FROM $testTableName", expectedPlan = "LocalRelation [none#0L, none#1L, none#2, none#3, none#4, none#5, none#6" + ", none#7, none#8L, none#9L, none#10, none#11, none#12, none#13, none#14, none#15]"), new SqlTestParams( name = "min-max - column name non-matching case", querySql = s"SELECT MIN(ID), MAX(iD)" + s", MIN(tINYINTCOLUMN), MAX(tinyintcolumN)" + s", MIN(sMALLINTCOLUMN), MAX(smallintcolumN)" + s", MIN(iNTCOLUMN), MAX(intcolumN)" + s", MIN(bIGINTCOLUMN), MAX(bigintcolumN)" + s", MIN(fLOATCOLUMN), MAX(floatcolumN)" + s", MIN(dOUBLECOLUMN), MAX(doublecolumN)" + s", MIN(dATECOLUMN), MAX(datecolumN)" + s"FROM $testTableName", expectedPlan = "LocalRelation [none#0L, none#1L, none#2, none#3, none#4, none#5, none#6" + ", none#7, none#8L, none#9L, none#10, none#11, none#12, none#13, none#14, none#15]"), new SqlTestParams( name = "count with column name alias", querySql = s"SELECT COUNT(*) as MyCount FROM $testTableName", expectedPlan = "LocalRelation [none#0L]"), new SqlTestParams( name = "count-min-max with column name alias", querySql = s"SELECT COUNT(*) as MyCount, MIN(id) as MyMinId, MAX(id) as MyMaxId" + s" FROM $testTableName", expectedPlan = "LocalRelation [none#0L, none#1L, none#2L]"), new SqlTestParams( name = "count-min-max - table name with alias", querySql = s"SELECT COUNT(*), MIN(id), MAX(id) FROM $testTableName MyTable", expectedPlan = "LocalRelation [none#0L, none#1L, none#2L]"), new SqlTestParams( name = "count-min-max - query using time travel - version 0", querySql = s"SELECT COUNT(*), MIN(id), MAX(id) " + s"FROM $testTableName VERSION AS OF 0", expectedPlan = "LocalRelation [none#0L, none#1L, none#2L]"), new SqlTestParams( name = "count-min-max - query using time travel - version 1", querySql = s"SELECT COUNT(*), MIN(id), MAX(id) " + s"FROM $testTableName VERSION AS OF 1", expectedPlan = "LocalRelation [none#0L, none#1L, none#2L]"), new SqlTestParams( name = "count-min-max - query using time travel - version 2", querySql = s"SELECT COUNT(*), MIN(id), MAX(id) " + s"FROM $testTableName VERSION AS OF 2", expectedPlan = "LocalRelation [none#0L, none#1L, none#2L]"), new SqlTestParams( name = "count - sub-query", querySql = s"SELECT (SELECT COUNT(*) FROM $testTableName)", expectedPlan = "Project [scalar-subquery#0 [] AS #0L]\n" + ": +- LocalRelation [none#0L]\n+- OneRowRelation"), new SqlTestParams( name = "min - sub-query", querySql = s"SELECT (SELECT MIN(id) FROM $testTableName)", expectedPlan = "Project [scalar-subquery#0 [] AS #0L]\n" + ": +- LocalRelation [none#0L]\n+- OneRowRelation"), new SqlTestParams( name = "max - sub-query", querySql = s"SELECT (SELECT MAX(id) FROM $testTableName)", expectedPlan = "Project [scalar-subquery#0 [] AS #0L]\n" + ": +- LocalRelation [none#0L]\n+- OneRowRelation"), new SqlTestParams( name = "count - sub-query filter", querySql = s"SELECT 'ABC' WHERE" + s" (SELECT COUNT(*) FROM $testTableName) = $totalRows", expectedPlan = "Project [ABC AS #0]\n+- Filter (scalar-subquery#0 [] = " + totalRows + ")\n : +- LocalRelation [none#0L]\n +- OneRowRelation"), new SqlTestParams( name = "min - sub-query filter", querySql = s"SELECT 'ABC' WHERE" + s" (SELECT MIN(id) FROM $testTableName) = $minId", expectedPlan = "Project [ABC AS #0]\n+- Filter (scalar-subquery#0 [] = " + minId + ")\n : +- LocalRelation [none#0L]\n +- OneRowRelation"), new SqlTestParams( name = "max - sub-query filter", querySql = s"SELECT 'ABC' WHERE" + s" (SELECT MAX(id) FROM $testTableName) = $maxId", expectedPlan = "Project [ABC AS #0]\n+- Filter (scalar-subquery#0 [] = " + maxId + ")\n : +- LocalRelation [none#0L]\n +- OneRowRelation"), // Limit doesn't affect aggregation results new SqlTestParams( name = "count-min-max - query with limit", querySql = s"SELECT COUNT(*), MIN(id), MAX(id) FROM $testTableName LIMIT 3", expectedPlan = "LocalRelation [none#0L, none#1L, none#2L]"), new SqlTestParams( name = "count-min-max - duplicated functions", querySql = s"SELECT COUNT(*), COUNT(*), MIN(id), MIN(id), MAX(id), MAX(id)" + s" FROM $testTableName", expectedPlan = "LocalRelation [none#0L, none#1L, none#2L, none#3L, none#4L, none#5L]"), new SqlTestParams( name = "count - empty table", querySetup = Some(Seq("CREATE TABLE TestEmpty (c1 int) USING DELTA")), querySql = "SELECT COUNT(*) FROM TestEmpty", expectedPlan = "LocalRelation [none#0L]"), /** Dates are stored as Int in literals. This test make sure Date columns works * and NULL are handled correctly */ new SqlTestParams( name = "min-max - date columns", querySetup = Some(Seq( "CREATE TABLE TestDateValues" + " (Column1 DATE, Column2 DATE, Column3 DATE) USING DELTA;", "INSERT INTO TestDateValues" + " (Column1, Column2, Column3) VALUES (NULL, current_date(), current_date());", "INSERT INTO TestDateValues" + " (Column1, Column2, Column3) VALUES (NULL, NULL, current_date());")), querySql = "SELECT COUNT(*), MIN(Column1), MAX(Column1), MIN(Column2)" + ", MAX(Column2), MIN(Column3), MAX(Column3) FROM TestDateValues", expectedPlan = "LocalRelation [none#0L, none#1, none#2, none#3, none#4, none#5, none#6]"), new SqlTestParams( name = "min-max - floating point infinity", querySetup = Some(Seq( "CREATE TABLE TestFloatInfinity (FloatColumn Float, DoubleColumn Double) USING DELTA", "INSERT INTO TestFloatInfinity (FloatColumn, DoubleColumn) VALUES (1, 1);", "INSERT INTO TestFloatInfinity (FloatColumn, DoubleColumn) VALUES (NULL, NULL);", "INSERT INTO TestFloatInfinity (FloatColumn, DoubleColumn) VALUES " + "(float('inf'), double('inf'))" + ", (float('+inf'), double('+inf'))" + ", (float('infinity'), double('infinity'))" + ", (float('+infinity'), double('+infinity'))" + ", (float('-inf'), double('-inf'))" + ", (float('-infinity'), double('-infinity'))" )), querySql = "SELECT COUNT(*), MIN(FloatColumn), MAX(FloatColumn), MIN(DoubleColumn)" + ", MAX(DoubleColumn) FROM TestFloatInfinity", expectedPlan = "LocalRelation [none#0L, none#1, none#2, none#3, none#4]"), // NaN is larger than any other value, including Infinity new SqlTestParams( name = "min-max - floating point NaN values", querySetup = Some(Seq( "CREATE TABLE TestFloatNaN (FloatColumn Float, DoubleColumn Double) USING DELTA", "INSERT INTO TestFloatNaN (FloatColumn, DoubleColumn) VALUES (1, 1);", "INSERT INTO TestFloatNaN (FloatColumn, DoubleColumn) VALUES (NULL, NULL);", "INSERT INTO TestFloatNaN (FloatColumn, DoubleColumn) VALUES " + "(float('inf'), double('inf'))" + ", (float('+inf'), double('+inf'))" + ", (float('infinity'), double('infinity'))" + ", (float('+infinity'), double('+infinity'))" + ", (float('-inf'), double('-inf'))" + ", (float('-infinity'), double('-infinity'))", "INSERT INTO TestFloatNaN (FloatColumn, DoubleColumn) VALUES " + "(float('NaN'), double('NaN'));" )), querySql = "SELECT COUNT(*), MIN(FloatColumn), MAX(FloatColumn), MIN(DoubleColumn)" + ", MAX(DoubleColumn) FROM TestFloatNaN", expectedPlan = "LocalRelation [none#0L, none#1, none#2, none#3, none#4]"), new SqlTestParams( name = "min-max - floating point min positive value", querySetup = Some(Seq( "CREATE TABLE TestFloatPrecision (FloatColumn Float, DoubleColumn Double) USING DELTA", "INSERT INTO TestFloatPrecision (FloatColumn, DoubleColumn) VALUES " + "(CAST('1.4E-45' as FLOAT), CAST('4.9E-324' as DOUBLE))" + ", (CAST('-1.4E-45' as FLOAT), CAST('-4.9E-324' as DOUBLE))" + ", (0, 0);" )), querySql = "SELECT COUNT(*), MIN(FloatColumn), MAX(FloatColumn), MIN(DoubleColumn)" + ", MAX(DoubleColumn) FROM TestFloatPrecision", expectedPlan = "LocalRelation [none#0L, none#1, none#2, none#3, none#4]"), new SqlTestParams( name = "min-max - NULL and non-NULL values", querySetup = Some(Seq( "CREATE TABLE TestNullValues (Column1 INT, Column2 INT, Column3 INT) USING DELTA", "INSERT INTO TestNullValues (Column1, Column2, Column3) VALUES (NULL, 1, 1);", "INSERT INTO TestNullValues (Column1, Column2, Column3) VALUES (NULL, NULL, 1);" )), querySql = "SELECT COUNT(*), MIN(Column1), MAX(Column1)," + "MIN(Column2), MAX(Column2) FROM TestNullValues", expectedPlan = "LocalRelation [none#0L, none#1, none#2, none#3, none#4]"), new SqlTestParams( name = "min-max - only NULL values", querySetup = Some(Seq( "CREATE TABLE TestOnlyNullValues (Column1 INT, Column2 INT, Column3 INT) USING DELTA", "INSERT INTO TestOnlyNullValues (Column1, Column2, Column3) VALUES (NULL, NULL, 1);", "INSERT INTO TestOnlyNullValues (Column1, Column2, Column3) VALUES (NULL, NULL, 2);" )), querySql = "SELECT COUNT(*), MIN(Column1), MAX(Column1), MIN(Column2), MAX(Column2), " + "MIN(Column3), MAX(Column3) FROM TestOnlyNullValues", expectedPlan = "LocalRelation [none#0L, none#1, none#2, none#3, none#4, none#5, none#6]"), new SqlTestParams( name = "min-max - all supported data types", querySetup = Some(Seq( "CREATE TABLE TestMinMaxValues (" + "TINYINTColumn TINYINT, SMALLINTColumn SMALLINT, INTColumn INT, BIGINTColumn BIGINT, " + "FLOATColumn FLOAT, DOUBLEColumn DOUBLE, DATEColumn DATE) USING DELTA", "INSERT INTO TestMinMaxValues (TINYINTColumn, SMALLINTColumn, INTColumn, BIGINTColumn," + " FLOATColumn, DOUBLEColumn, DATEColumn)" + " VALUES (-128, -32768, -2147483648, -9223372036854775808," + " -3.4028235E38, -1.7976931348623157E308, CAST('1582-10-15' AS DATE));", "INSERT INTO TestMinMaxValues (TINYINTColumn, SMALLINTColumn, INTColumn, BIGINTColumn," + " FLOATColumn, DOUBLEColumn, DATEColumn)" + " VALUES (127, 32767, 2147483647, 9223372036854775807," + " 3.4028235E38, 1.7976931348623157E308, CAST('9999-12-31' AS DATE));" )), querySql = "SELECT COUNT(*)," + "MIN(TINYINTColumn), MAX(TINYINTColumn)" + ", MIN(SMALLINTColumn), MAX(SMALLINTColumn)" + ", MIN(INTColumn), MAX(INTColumn)" + ", MIN(BIGINTColumn), MAX(BIGINTColumn)" + ", MIN(FLOATColumn), MAX(FLOATColumn)" + ", MIN(DOUBLEColumn), MAX(DOUBLEColumn)" + ", MIN(DATEColumn), MAX(DATEColumn)" + " FROM TestMinMaxValues", expectedPlan = "LocalRelation [none#0L, none#1, none#2, none#3, none#4, none#5, none#6" + ", none#7L, none#8L, none#9, none#10, none#11, none#12, none#13, none#14]"), new SqlTestParams( name = "count-min-max - partitioned table - simple query", querySetup = Some(Seq( "CREATE TABLE TestPartitionedTable (Column1 INT, Column2 INT, Column3 INT, Column4 INT)" + " USING DELTA PARTITIONED BY (Column2, Column3)", "INSERT INTO TestPartitionedTable" + " (Column1, Column2, Column3, Column4) VALUES (1, 2, 3, 4);", "INSERT INTO TestPartitionedTable" + " (Column1, Column2, Column3, Column4) VALUES (2, 2, 3, 5);", "INSERT INTO TestPartitionedTable" + " (Column1, Column2, Column3, Column4) VALUES (3, 3, 2, 6);", "INSERT INTO TestPartitionedTable" + " (Column1, Column2, Column3, Column4) VALUES (4, 3, 2, 7);" )), querySql = "SELECT COUNT(*)" + ", MIN(Column1), MAX(Column1)" + ", MIN(Column2), MAX(Column2)" + ", MIN(Column3), MAX(Column3)" + ", MIN(Column4), MAX(Column4)" + " FROM TestPartitionedTable", expectedPlan = "LocalRelation [none#0L, none#1, none#2, none#3," + " none#4, none#5, none#6, none#7, none#8]"), /** Partitioned columns should be able to return MIN and MAX data * even when there are no column stats */ new SqlTestParams( name = "count-min-max - partitioned table - no stats", querySetup = Some(Seq( "CREATE TABLE TestPartitionedTableNoStats" + " (Column1 INT, Column2 INT, Column3 INT, Column4 INT)" + " USING DELTA PARTITIONED BY (Column2, Column3)" + " TBLPROPERTIES('delta.dataSkippingNumIndexedCols' = 0)", "INSERT INTO TestPartitionedTableNoStats" + " (Column1, Column2, Column3, Column4) VALUES (1, 2, 3, 4);", "INSERT INTO TestPartitionedTableNoStats" + " (Column1, Column2, Column3, Column4) VALUES (2, 2, 3, 5);", "INSERT INTO TestPartitionedTableNoStats" + " (Column1, Column2, Column3, Column4) VALUES (3, 3, 2, 6);", "INSERT INTO TestPartitionedTableNoStats" + " (Column1, Column2, Column3, Column4) VALUES (4, 3, 2, 7);" )), querySql = "SELECT COUNT(*)" + ", MIN(Column2), MAX(Column2)" + ", MIN(Column3), MAX(Column3)" + " FROM TestPartitionedTableNoStats", expectedPlan = "LocalRelation [none#0L, none#1, none#2, none#3, none#4]"), new SqlTestParams( name = "min-max - partitioned table - all supported data types", querySetup = Some(Seq( "CREATE TABLE TestAllTypesPartitionedTable (" + "TINYINTColumn TINYINT, SMALLINTColumn SMALLINT, INTColumn INT, BIGINTColumn BIGINT, " + "FLOATColumn FLOAT, DOUBLEColumn DOUBLE, DATEColumn DATE, Data INT) USING DELTA" + " PARTITIONED BY (TINYINTColumn, SMALLINTColumn, INTColumn, BIGINTColumn," + " FLOATColumn, DOUBLEColumn, DATEColumn)", "INSERT INTO TestAllTypesPartitionedTable" + " (TINYINTColumn, SMALLINTColumn, INTColumn, BIGINTColumn," + " FLOATColumn, DOUBLEColumn, DATEColumn, Data)" + " VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);", "INSERT INTO TestAllTypesPartitionedTable" + " (TINYINTColumn, SMALLINTColumn, INTColumn, BIGINTColumn," + " FLOATColumn, DOUBLEColumn, DATEColumn, Data)" + " VALUES (-128, -32768, -2147483648, -9223372036854775808," + " -3.4028235E38, -1.7976931348623157E308, CAST('1582-10-15' AS DATE), 1);" )), querySql = "SELECT COUNT(*)," + "MIN(TINYINTColumn), MAX(TINYINTColumn)" + ", MIN(SMALLINTColumn), MAX(SMALLINTColumn)" + ", MIN(INTColumn), MAX(INTColumn)" + ", MIN(BIGINTColumn), MAX(BIGINTColumn)" + ", MIN(FLOATColumn), MAX(FLOATColumn)" + ", MIN(DOUBLEColumn), MAX(DOUBLEColumn)" + ", MIN(DATEColumn), MAX(DATEColumn)" + ", MIN(Data), MAX(Data)" + " FROM TestAllTypesPartitionedTable", expectedPlan = "LocalRelation [none#0L, none#1, none#2, none#3, none#4, none#5, none#6, " + "none#7L, none#8L, none#9, none#10, none#11, none#12, none#13, none#14, none#15, none#16]"), new SqlTestParams( name = "min-max - partitioned table - only NULL values", querySetup = Some(Seq( "CREATE TABLE TestOnlyNullValuesPartitioned (" + "TINYINTColumn TINYINT, SMALLINTColumn SMALLINT, INTColumn INT, BIGINTColumn BIGINT, " + "FLOATColumn FLOAT, DOUBLEColumn DOUBLE, DATEColumn DATE, Data INT) USING DELTA" + " PARTITIONED BY (TINYINTColumn, SMALLINTColumn, INTColumn, BIGINTColumn," + " FLOATColumn, DOUBLEColumn, DATEColumn)", "INSERT INTO TestOnlyNullValuesPartitioned" + " (TINYINTColumn, SMALLINTColumn, INTColumn, BIGINTColumn," + " FLOATColumn, DOUBLEColumn, DATEColumn, Data)" + " VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);" )), querySql = "SELECT COUNT(*)," + "MIN(TINYINTColumn), MAX(TINYINTColumn)" + ", MIN(SMALLINTColumn), MAX(SMALLINTColumn)" + ", MIN(INTColumn), MAX(INTColumn)" + ", MIN(BIGINTColumn), MAX(BIGINTColumn)" + ", MIN(FLOATColumn), MAX(FLOATColumn)" + ", MIN(DOUBLEColumn), MAX(DOUBLEColumn)" + ", MIN(DATEColumn), MAX(DATEColumn)" + ", MIN(Data), MAX(Data)" + " FROM TestOnlyNullValuesPartitioned", expectedPlan = "LocalRelation [none#0L, none#1, none#2, none#3, none#4, none#5, none#6, " + "none#7L, none#8L, none#9, none#10, none#11, none#12, none#13, none#14, none#15, none#16]"), new SqlTestParams( name = "min-max - partitioned table - NULL and NON-NULL values", querySetup = Some(Seq( "CREATE TABLE TestNullPartitioned (Column1 INT, Column2 INT, Column3 INT)" + " USING DELTA PARTITIONED BY (Column2, Column3)", "INSERT INTO TestNullPartitioned (Column1, Column2, Column3) VALUES (NULL, NULL, 1);", "INSERT INTO TestNullPartitioned (Column1, Column2, Column3) VALUES (NULL, NULL, NULL);", "INSERT INTO TestNullPartitioned (Column1, Column2, Column3) VALUES (NULL, NULL, 2);" )), querySql = "SELECT COUNT(*), MIN(Column1), MAX(Column1), MIN(Column2), MAX(Column2), " + "MIN(Column3), MAX(Column3) FROM TestNullPartitioned", expectedPlan = "LocalRelation [none#0L, none#1, none#2, none#3, none#4, none#5, none#6]"), new SqlTestParams( name = "min-max - column name containing punctuation", querySetup = Some(Seq( "CREATE TABLE TestPunctuationColumnName (`My.!?Column` INT) USING DELTA", "INSERT INTO TestPunctuationColumnName (`My.!?Column`) VALUES (1), (2), (3);" )), querySql = "SELECT COUNT(*), MIN(`My.!?Column`), MAX(`My.!?Column`)" + " FROM TestPunctuationColumnName", expectedPlan = "LocalRelation [none#0L, none#1, none#2]"), new SqlTestParams( name = "min-max - partitioned table - column name containing punctuation", querySetup = Some(Seq( "CREATE TABLE TestPartitionedPunctuationColumnName (`My.!?Column` INT, Data INT)" + " USING DELTA PARTITIONED BY (`My.!?Column`)", "INSERT INTO TestPartitionedPunctuationColumnName" + " (`My.!?Column`, Data) VALUES (1, 1), (2, 1), (3, 1);" )), querySql = "SELECT COUNT(*), MIN(`My.!?Column`), MAX(`My.!?Column`)" + " FROM TestPartitionedPunctuationColumnName", expectedPlan = "LocalRelation [none#0L, none#1, none#2]"), new SqlTestParams( name = "min-max - partitioned table - special characters in column name", querySetup = Some(Seq( "CREATE TABLE TestColumnMappingPartitioned" + " (Column1 INT, Column2 INT, `Column3 .,;{}()\n\t=` INT, Column4 INT)" + " USING DELTA PARTITIONED BY (Column2, `Column3 .,;{}()\n\t=`)" + " TBLPROPERTIES('delta.columnMapping.mode' = 'name')", "INSERT INTO TestColumnMappingPartitioned" + " (Column1, Column2, `Column3 .,;{}()\n\t=`, Column4)" + " VALUES (1, 2, 3, 4);", "INSERT INTO TestColumnMappingPartitioned" + " (Column1, Column2, `Column3 .,;{}()\n\t=`, Column4)" + " VALUES (2, 2, 3, 5);", "INSERT INTO TestColumnMappingPartitioned" + " (Column1, Column2, `Column3 .,;{}()\n\t=`, Column4)" + " VALUES (3, 3, 2, 6);", "INSERT INTO TestColumnMappingPartitioned" + " (Column1, Column2, `Column3 .,;{}()\n\t=`, Column4)" + " VALUES (4, 3, 2, 7);")), querySql = "SELECT COUNT(*)" + ", MIN(Column1), MAX(Column1)" + ", MIN(Column2), MAX(Column2)" + ", MIN(`Column3 .,;{}()\n\t=`), MAX(`Column3 .,;{}()\n\t=`)" + ", MIN(Column4), MAX(Column4)" + " FROM TestColumnMappingPartitioned", expectedPlan = "LocalRelation [none#0L, none#1, none#2, none#3," + " none#4, none#5, none#6, none#7, none#8]")) .foreach { testParams => test(s"optimization supported - SQL - ${testParams.name}") { if (testParams.querySetup.isDefined) { testParams.querySetup.get.foreach(spark.sql) } checkResultsAndOptimizedPlan(testParams.querySql, testParams.expectedPlan) } } test("count-min-max - external table") { withTempDir { dir => val testTablePath = dir.getCanonicalPath dfPart1.write.format("delta").mode("overwrite").save(testTablePath) DeltaTable.forPath(spark, testTablePath).delete("id = 1") dfPart2.write.format("delta").mode(SaveMode.Append).save(testTablePath) checkResultsAndOptimizedPlan( s"SELECT COUNT(*), MIN(id), MAX(id) FROM delta.`$testTablePath`", "LocalRelation [none#0L, none#1L, none#2L]") } } test("min-max - partitioned column stats disabled") { withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false") { val tableName = "TestPartitionedNoStats" spark.sql(s"CREATE TABLE $tableName (Column1 INT, Column2 INT)" + " USING DELTA PARTITIONED BY (Column2)") spark.sql(s"INSERT INTO $tableName (Column1, Column2) VALUES (1, 3);") spark.sql(s"INSERT INTO $tableName (Column1, Column2) VALUES (2, 4);") // Has no stats, including COUNT checkOptimizationIsNotTriggered( s"SELECT COUNT(*), MIN(Column2), MAX(Column2) FROM $tableName") // Should work for partitioned columns even without stats checkResultsAndOptimizedPlan( s"SELECT MIN(Column2), MAX(Column2) FROM $tableName", "LocalRelation [none#0, none#1]") } } test("min-max - recompute column missing stats") { val tableName = "TestRecomputeMissingStat" spark.sql(s"CREATE TABLE $tableName (Column1 INT, Column2 INT) USING DELTA" + s" TBLPROPERTIES('delta.dataSkippingNumIndexedCols' = 0)") spark.sql(s"INSERT INTO $tableName (Column1, Column2) VALUES (1, 4);") spark.sql(s"INSERT INTO $tableName (Column1, Column2) VALUES (2, 5);") spark.sql(s"INSERT INTO $tableName (Column1, Column2) VALUES (3, 6);") checkOptimizationIsNotTriggered(s"SELECT COUNT(*), MIN(Column1), MAX(Column1) FROM $tableName") spark.sql(s"ALTER TABLE $tableName SET TBLPROPERTIES('delta.dataSkippingNumIndexedCols' = 1);") StatisticsCollection.recompute( spark, DeltaLog.forTable(spark, TableIdentifier(tableName)), DeltaTableV2(spark, TableIdentifier(tableName)).catalogTable) checkResultsAndOptimizedPlan( s"SELECT COUNT(*), MIN(Column1), MAX(Column1) FROM $tableName", "LocalRelation [none#0L, none#1, none#2]") checkOptimizationIsNotTriggered(s"SELECT COUNT(*), MIN(Column2), MAX(Column2) FROM $tableName") spark.sql(s"ALTER TABLE $tableName SET TBLPROPERTIES('delta.dataSkippingNumIndexedCols' = 2);") StatisticsCollection.recompute( spark, DeltaLog.forTable(spark, TableIdentifier(tableName)), DeltaTableV2(spark, TableIdentifier(tableName)).catalogTable) checkResultsAndOptimizedPlan( s"SELECT COUNT(*), MIN(Column2), MAX(Column2) FROM $tableName", "LocalRelation [none#0L, none#1, none#2]") } test("min-max - recompute added column") { val tableName = "TestRecomputeAddedColumn" spark.sql(s"CREATE TABLE $tableName (Column1 INT) USING DELTA") spark.sql(s"INSERT INTO $tableName (Column1) VALUES (1);") spark.sql(s"ALTER TABLE $tableName ADD COLUMN (Column2 INT)") spark.sql(s"INSERT INTO $tableName (Column1, Column2) VALUES (2, 5);") checkResultsAndOptimizedPlan( s"SELECT COUNT(*), MIN(Column1), MAX(Column1) FROM $tableName", "LocalRelation [none#0L, none#1, none#2]") checkOptimizationIsNotTriggered(s"SELECT COUNT(*), " + s"MIN(Column1), MAX(Column1), MIN(Column2), MAX(Column2) FROM $tableName") StatisticsCollection.recompute( spark, DeltaLog.forTable(spark, TableIdentifier(tableName)), DeltaTableV2(spark, TableIdentifier(tableName)).catalogTable) checkResultsAndOptimizedPlan(s"SELECT COUNT(*), " + s"MIN(Column1), MAX(Column1), MIN(Column2), MAX(Column2) FROM $tableName", "LocalRelation [none#0L, none#1, none#2, none#3, none#4]") } test("Select Count: snapshot isolation") { sql(s"CREATE TABLE TestSnapshotIsolation (c1 int) USING DELTA") spark.sql("INSERT INTO TestSnapshotIsolation VALUES (1)") val scannedVersions = mutable.ArrayBuffer[Long]() val query = "SELECT (SELECT COUNT(*) FROM TestSnapshotIsolation), " + "(SELECT COUNT(*) FROM TestSnapshotIsolation)" checkResultsAndOptimizedPlan( query, "Project [scalar-subquery#0 [] AS #0L, scalar-subquery#0 [] AS #1L]\n" + ": :- LocalRelation [none#0L]\n" + ": +- LocalRelation [none#0L]\n" + "+- OneRowRelation") PrepareDeltaScanBase.withCallbackOnGetDeltaScanGenerator(scanGenerator => { // Record the scanned version and make changes to the table. We will verify changes in the // middle of the query are not visible to the query. scannedVersions += scanGenerator.snapshotToScan.version // Insert a row after each call to get scanGenerator // to test if the count doesn't change in the same query spark.sql("INSERT INTO TestSnapshotIsolation VALUES (1)") }) { val result = spark.sql(query).collect()(0) val c1 = result.getLong(0) val c2 = result.getLong(1) assertResult(c1, "Snapshot isolation should guarantee the results are always the same")(c2) assert( scannedVersions.toSet.size == 1, s"Scanned multiple versions of the same table in one query: ${scannedVersions.toSet}") } } test(".collect() and .show() both use this optimization") { var resultRow: Row = null withSQLConf(DeltaSQLConf.DELTA_OPTIMIZE_METADATA_QUERY_ENABLED.key -> "false") { resultRow = spark.sql(s"SELECT COUNT(*), MIN(id), MAX(id) FROM $testTableName").head } val totalRows = resultRow.getLong(0) val minId = resultRow.getLong(1) val maxId = resultRow.getLong(2) val collectPlans = DeltaTestUtils.withLogicalPlansCaptured(spark, optimizedPlan = true) { spark.sql(s"SELECT COUNT(*) FROM $testTableName").collect() } val collectResultData = collectPlans.collect { case x: LocalRelation => x.data } assert(collectResultData.size === 1) assert(collectResultData.head.head.getLong(0) === totalRows) val showPlans = DeltaTestUtils.withLogicalPlansCaptured(spark, optimizedPlan = true) { spark.sql(s"SELECT COUNT(*) FROM $testTableName").show() } val showResultData = showPlans.collect { case x: LocalRelation => x.data } assert(showResultData.size === 1) assert(showResultData.head.head.getString(0).toLong === totalRows) val showMultAggPlans = DeltaTestUtils.withLogicalPlansCaptured(spark, optimizedPlan = true) { spark.sql(s"SELECT COUNT(*), MIN(id), MAX(id) FROM $testTableName").show() } val showMultipleAggResultData = showMultAggPlans.collect { case x: LocalRelation => x.data } assert(showMultipleAggResultData.size === 1) val firstRow = showMultipleAggResultData.head.head assert(firstRow.getString(0).toLong === totalRows) assert(firstRow.getString(1).toLong === minId) assert(firstRow.getString(2).toLong === maxId) } test("min-max .show() - only NULL values") { val tableName = "TestOnlyNullValuesShow" spark.sql(s"CREATE TABLE $tableName (Column1 INT) USING DELTA") spark.sql(s"INSERT INTO $tableName (Column1) VALUES (NULL);") val showMultAggPlans = DeltaTestUtils.withLogicalPlansCaptured(spark, optimizedPlan = true) { spark.sql(s"SELECT MIN(Column1), MAX(Column1) FROM $tableName").show() } val showMultipleAggResultData = showMultAggPlans.collect { case x: LocalRelation => x.data } assert(showMultipleAggResultData.size === 1) val firstRow = showMultipleAggResultData.head.head assert(firstRow.getString(0) === "NULL") assert(firstRow.getString(1) === "NULL") } test("min-max .show() - Date Columns") { val tableName = "TestDateColumnsShow" spark.sql(s"CREATE TABLE $tableName (Column1 DATE, Column2 DATE) USING DELTA") spark.sql(s"INSERT INTO $tableName (Column1, Column2) VALUES " + s"(CAST('1582-10-15' AS DATE), NULL);") val showMultAggPlans = DeltaTestUtils.withLogicalPlansCaptured(spark, optimizedPlan = true) { spark.sql(s"SELECT MIN(Column1), MIN(Column2) FROM $tableName").show() } val showMultipleAggResultData = showMultAggPlans.collect { case x: LocalRelation => x.data } assert(showMultipleAggResultData.size === 1) val firstRow = showMultipleAggResultData.head.head assert(firstRow.getString(0) === "1582-10-15") assert(firstRow.getString(1) === "NULL") } test("count - dv-enabled") { withTempDir { dir => val tempPath = dir.getCanonicalPath spark.range(1, 10, 1, 1).write.format("delta").save(tempPath) enableDeletionVectorsInTable(new Path(tempPath), true) DeltaTable.forPath(spark, tempPath).delete("id = 1") assert(!getFilesWithDeletionVectors(DeltaLog.forTable(spark, new Path(tempPath))).isEmpty) checkResultsAndOptimizedPlan( s"SELECT COUNT(*) FROM delta.`$tempPath`", "LocalRelation [none#0L]") } } test("count - zero rows AddFile") { withTempDir { dir => val tempPath = dir.getCanonicalPath val df = spark.range(1, 10) val expectedResult = df.count() df.write.format("delta").save(tempPath) // Creates AddFile entries with non-existing files // The query should read only the delta log and not the parquet files val log = DeltaLog.forTable(spark, tempPath) val txn = log.startTransaction() txn.commitManually( DeltaTestUtils.createTestAddFile(encodedPath = "1.parquet", stats = "{\"numRecords\": 0}"), DeltaTestUtils.createTestAddFile(encodedPath = "2.parquet", stats = "{\"numRecords\": 0}"), DeltaTestUtils.createTestAddFile(encodedPath = "3.parquet", stats = "{\"numRecords\": 0}")) withSQLConf(DeltaSQLConf.DELTA_OPTIMIZE_METADATA_QUERY_ENABLED.key -> "true") { val queryDf = spark.sql(s"SELECT COUNT(*) FROM delta.`$tempPath`") val optimizedPlan = queryDf.queryExecution.optimizedPlan.canonicalized.toString() assert(queryDf.head().getLong(0) === expectedResult) assertResult("LocalRelation [none#0L]") { optimizedPlan.trim } } } } // Tests to validate the optimizer won't incorrectly change queries it can't correctly handle Seq((s"SELECT COUNT(*) FROM $mixedStatsTableName", "missing stats"), (s"SELECT COUNT(*) FROM $noStatsTableName", "missing stats"), (s"SELECT MIN(id), MAX(id) FROM $mixedStatsTableName", "missing stats"), (s"SELECT MIN(id), MAX(id) FROM $noStatsTableName", "missing stats"), (s"SELECT group, COUNT(*) FROM $testTableName GROUP BY group", "group by"), (s"SELECT group, MIN(id), MAX(id) FROM $testTableName GROUP BY group", "group by"), (s"SELECT COUNT(*) + 1 FROM $testTableName", "plus literal"), (s"SELECT MAX(id) + 1 FROM $testTableName", "plus literal"), (s"SELECT COUNT(DISTINCT data) FROM $testTableName", "distinct count"), (s"SELECT COUNT(*) FROM $testTableName WHERE id > 0", "filter"), (s"SELECT MAX(id) FROM $testTableName WHERE id > 0", "filter"), (s"SELECT (SELECT COUNT(*) FROM $testTableName WHERE id > 0)", "sub-query with filter"), (s"SELECT (SELECT MAX(id) FROM $testTableName WHERE id > 0)", "sub-query with filter"), (s"SELECT COUNT(ALL data) FROM $testTableName", "count non-null"), (s"SELECT COUNT(data) FROM $testTableName", "count non-null"), (s"SELECT COUNT(*) FROM $testTableName A, $testTableName B", "join"), (s"SELECT MAX(A.id) FROM $testTableName A, $testTableName B", "join"), (s"SELECT COUNT(*) OVER() FROM $testTableName LIMIT 1", "over"), ( s"SELECT MAX(id) OVER() FROM $testTableName LIMIT 1", "over") ) .foreach { case (query, desc) => test(s"optimization not supported - $desc - $query") { checkOptimizationIsNotTriggered(query) } } test("optimization not supported - min-max unsupported data types") { val tableName = "TestUnsupportedTypes" spark.sql(s"CREATE TABLE $tableName " + s"(STRINGColumn STRING, DECIMALColumn DECIMAL(38,0)" + s", TIMESTAMPColumn TIMESTAMP, BINARYColumn BINARY, " + s"BOOLEANColumn BOOLEAN, ARRAYColumn ARRAY, MAPColumn MAP, " + s"STRUCTColumn STRUCT) USING DELTA") spark.sql(s"INSERT INTO $tableName" + s" (STRINGColumn, DECIMALColumn, TIMESTAMPColumn, BINARYColumn" + s", BOOLEANColumn, ARRAYColumn, MAPColumn, STRUCTColumn) VALUES " + s"('A', -99999999999999999999999999999999999999, CAST('1900-01-01 00:00:00.0' AS TIMESTAMP)" + s", X'1ABF', TRUE, ARRAY(1, 2, 3), MAP(1, 10, 2, 20), STRUCT(1, 'Spark'));") val columnNames = List("STRINGColumn", "DECIMALColumn", "TIMESTAMPColumn", "BINARYColumn", "BOOLEANColumn", "ARRAYColumn", "STRUCTColumn") columnNames.foreach(colName => checkOptimizationIsNotTriggered(s"SELECT MAX($colName) FROM $tableName") ) } test("optimization not supported - min-max column without stats") { val tableName = "TestColumnWithoutStats" spark.sql(s"CREATE TABLE $tableName (Column1 INT, Column2 INT) USING DELTA" + s" TBLPROPERTIES('delta.dataSkippingNumIndexedCols' = 1)") spark.sql(s"INSERT INTO $tableName (Column1, Column2) VALUES (1, 2);") checkOptimizationIsNotTriggered( s"SELECT MAX(Column2) FROM $tableName") } // For empty tables the stats won't be found and the query should not be optimized test("optimization not supported - min-max empty table") { val tableName = "TestMinMaxEmptyTable" spark.sql(s"CREATE TABLE $tableName (Column1 INT) USING DELTA") checkOptimizationIsNotTriggered( s"SELECT MIN(Column1), MAX(Column1) FROM $tableName") } test("optimization not supported - min-max dv-enabled") { withTempDir { dir => val tempPath = dir.getCanonicalPath spark.range(1, 10, 1, 1).write.format("delta").save(tempPath) val querySql = s"SELECT MIN(id), MAX(id) FROM delta.`$tempPath`" checkResultsAndOptimizedPlan(querySql, "LocalRelation [none#0L, none#1L]") enableDeletionVectorsInTable(new Path(tempPath), true) DeltaTable.forPath(spark, tempPath).delete("id = 1") assert(!getFilesWithDeletionVectors(DeltaLog.forTable(spark, new Path(tempPath))).isEmpty) checkOptimizationIsNotTriggered(querySql) } } test("optimization not supported - filter on partitioned column") { val tableName = "TestPartitionedFilter" spark.sql(s"CREATE TABLE $tableName (Column1 INT, Column2 INT)" + " USING DELTA PARTITIONED BY (Column2)") spark.sql(s"INSERT INTO $tableName (Column1, Column2) VALUES (1, 2);") spark.sql(s"INSERT INTO $tableName (Column1, Column2) VALUES (2, 2);") spark.sql(s"INSERT INTO $tableName (Column1, Column2) VALUES (3, 3);") spark.sql(s"INSERT INTO $tableName (Column1, Column2) VALUES (4, 3);") // Filter by partition column checkOptimizationIsNotTriggered( "SELECT COUNT(*)" + ", MIN(Column1), MAX(Column1)" + ", MIN(Column2), MAX(Column2)" + s" FROM $tableName WHERE Column2 = 2") // Filter both partition and data columns checkOptimizationIsNotTriggered( "SELECT COUNT(*)" + ", MIN(Column1), MAX(Column1)" + ", MIN(Column2), MAX(Column2)" + s" FROM $tableName WHERE Column1 = 2 AND Column2 = 2") } test("optimization not supported - sub-query with column alias") { val tableName = "TestColumnAliasSubQuery" spark.sql(s"CREATE TABLE $tableName (Column1 INT, Column2 INT, Column3 INT) USING DELTA") spark.sql(s"INSERT INTO $tableName (Column1, Column2, Column3) VALUES (1, 2, 3);") checkOptimizationIsNotTriggered( s"SELECT MAX(Column2) FROM (SELECT Column1 AS Column2 FROM $tableName)") checkOptimizationIsNotTriggered( s"SELECT MAX(Column1), MAX(Column2), MAX(Column3) FROM " + s"(SELECT Column1 AS Column2, Column2 AS Column3, Column3 AS Column1 FROM $tableName)") } test("optimization not supported - nested columns") { val tableName = "TestNestedColumns" spark.sql(s"CREATE TABLE $tableName " + s"(Column1 STRUCT, " + s"`Column1.Id` INT) USING DELTA") spark.sql(s"INSERT INTO $tableName" + s" (Column1, `Column1.Id`) VALUES " + s"(STRUCT(1), 2);") // Nested Column checkOptimizationIsNotTriggered( s"SELECT MAX(Column1.Id) FROM $tableName") checkOptimizationIsNotTriggered( s"SELECT MAX(Column1.Id) AS XYZ FROM $tableName") // Validate the scenario where all the columns are read // since it creates a different query plan checkOptimizationIsNotTriggered( s"SELECT MAX(Column1.Id), " + s"MAX(`Column1.Id`) FROM $tableName") // The optimization for columns with dots should still work checkResultsAndOptimizedPlan(s"SELECT MAX(`Column1.Id`) FROM $tableName", "LocalRelation [none#0]") } private def generateRowsDataFrame(source: Dataset[java.lang.Long]): DataFrame = { import testImplicits._ source.select('id, 'id.cast("tinyint") as 'TinyIntColumn, 'id.cast("smallint") as 'SmallIntColumn, 'id.cast("int") as 'IntColumn, 'id.cast("bigint") as 'BigIntColumn, ('id / 3.3).cast("float") as 'FloatColumn, ('id / 3.3).cast("double") as 'DoubleColumn, date_add(lit("2022-08-31").cast("date"), col("id").cast("int")) as 'DateColumn, ('id % 2).cast("integer") as 'group, 'id.cast("string") as 'data) } /** Validate the results of the query is the same with the flag * DELTA_OPTIMIZE_METADATA_QUERY_ENABLED enabled and disabled. * And the expected Optimized Query Plan with the flag enabled */ private def checkResultsAndOptimizedPlan( query: String, expectedOptimizedPlan: String): Unit = { checkResultsAndOptimizedPlan(() => spark.sql(query), expectedOptimizedPlan) } /** Validate the results of the query is the same with the flag * DELTA_OPTIMIZE_METADATA_QUERY_ENABLED enabled and disabled. * And the expected Optimized Query Plan with the flag enabled. */ private def checkResultsAndOptimizedPlan( generateQueryDf: () => DataFrame, expectedOptimizedPlan: String): Unit = { var expectedAnswer: scala.Seq[Row] = null withSQLConf(DeltaSQLConf.DELTA_OPTIMIZE_METADATA_QUERY_ENABLED.key -> "false") { expectedAnswer = generateQueryDf().collect() } withSQLConf(DeltaSQLConf.DELTA_OPTIMIZE_METADATA_QUERY_ENABLED.key -> "true") { val queryDf = generateQueryDf() val optimizedPlan = queryDf.queryExecution.optimizedPlan.canonicalized.toString() assert(queryDf.collect().sameElements(expectedAnswer)) assertResult(expectedOptimizedPlan.trim) { optimizedPlan.trim } } } /** * Verify the query plans and results are the same with/without metadata query optimization. * This method can be used to verify cases that we shouldn't trigger optimization * or cases that we can potentially improve. * @param query */ private def checkOptimizationIsNotTriggered(query: String) { var expectedOptimizedPlan: String = null var expectedAnswer: scala.Seq[Row] = null withSQLConf(DeltaSQLConf.DELTA_OPTIMIZE_METADATA_QUERY_ENABLED.key -> "false") { val generateQueryDf = spark.sql(query) expectedOptimizedPlan = generateQueryDf.queryExecution.optimizedPlan .canonicalized.toString() expectedAnswer = generateQueryDf.collect() } withSQLConf(DeltaSQLConf.DELTA_OPTIMIZE_METADATA_QUERY_ENABLED.key -> "true") { val generateQueryDf = spark.sql(query) val optimizationEnabledQueryPlan = generateQueryDf.queryExecution.optimizedPlan .canonicalized.toString() assert(generateQueryDf.collect().sameElements(expectedAnswer)) assertResult(expectedOptimizedPlan) { optimizationEnabledQueryPlan } } } } trait OptimizeMetadataOnlyDeltaQueryColumnMappingSuiteBase extends DeltaColumnMappingSelectedTestMixin { override protected def runAllTests = true } class OptimizeMetadataOnlyDeltaQueryIdColumnMappingSuite extends OptimizeMetadataOnlyDeltaQuerySuite with DeltaColumnMappingEnableIdMode with OptimizeMetadataOnlyDeltaQueryColumnMappingSuiteBase class OptimizeMetadataOnlyDeltaQueryNameColumnMappingSuite extends OptimizeMetadataOnlyDeltaQuerySuite with DeltaColumnMappingEnableNameMode with OptimizeMetadataOnlyDeltaQueryColumnMappingSuiteBase ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/perf/OptimizedWritesSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.perf import java.io.File import scala.language.implicitConversions import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta.{DeltaConfigs, DeltaLog, DeltaOptions, DeltaTestUtils} import org.apache.spark.sql.delta.CommitStats import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.shims.StreamingTestShims.MemoryStream import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.sql.{DataFrame, QueryTest, Row} import org.apache.spark.sql.streaming.StreamingQuery import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{LongType, StructType} abstract class OptimizedWritesSuiteBase extends QueryTest with SharedSparkSession { import testImplicits._ protected def writeTest(testName: String)(f: String => Unit): Unit = { test(testName) { withTempDir { dir => withSQLConf(DeltaConfigs.OPTIMIZE_WRITE.defaultTablePropertyKey -> "true") { f(dir.getCanonicalPath) } } } } protected def checkResult(df: DataFrame, numFileCheck: Long => Boolean, dir: String): Unit = { val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, dir) val files = snapshot.numOfFiles assert(numFileCheck(files), s"file check failed: received $files") checkAnswer( spark.read.format("delta").load(dir), df ) } protected implicit def fileToPathString(dir: File): String = dir.getCanonicalPath writeTest("non-partitioned write - table config") { dir => val df = spark.range(0, 100, 1, 4).toDF() df.write.format("delta").save(dir) checkResult( df, numFileCheck = _ === 1, dir) } test("non-partitioned write - table config compatibility") { withTempDir { tempDir => val dir = tempDir.getCanonicalPath // When table property is not set, we use session conf value. // Writes 1 file instead of 4 when OW is enabled withSQLConf( DeltaSQLConf.DELTA_OPTIMIZE_WRITE_ENABLED.key -> "true") { val df = spark.range(0, 100, 1, 4).toDF() val commitStats = Log4jUsageLogger.track { df.write.format("delta").mode("append").save(dir) }.filter(_.tags.get("opType") === Some("delta.commit.stats")) assert(commitStats.length >= 1) checkResult( df, numFileCheck = _ === 1, dir) } } // Test order of precedence between table property "delta.autoOptimize.optimizeWrite" and // session conf. for { sqlConf <- DeltaTestUtils.BOOLEAN_DOMAIN tableProperty <- DeltaTestUtils.BOOLEAN_DOMAIN } { withTempDir { tempDir => withSQLConf( DeltaSQLConf.DELTA_OPTIMIZE_WRITE_ENABLED.key -> sqlConf.toString) { val dir = tempDir.getCanonicalPath // Write one file to be able to set tblproperties spark.range(10).coalesce(1).write.format("delta") .mode("append").save(dir) sql(s"ALTER TABLE delta.`$dir` SET TBLPROPERTIES" + s" (delta.autoOptimize.optimizeWrite = ${tableProperty.toString})") val df = spark.range(0, 100, 1, 4).toDF() // OW adds one file vs non-OW adds 4 files val expectedNumberOfFiles = if (sqlConf) 2 else 5 df.write.format("delta").mode("append").save(dir) checkResult( df.union(spark.range(10).toDF()), numFileCheck = _ === expectedNumberOfFiles, dir) } } } } test("non-partitioned write - data frame config") { withTempDir { dir => val df = spark.range(0, 100, 1, 4).toDF() df.write.format("delta") .option(DeltaOptions.OPTIMIZE_WRITE_OPTION, "true").save(dir) checkResult( df, numFileCheck = _ === 1, dir) } } writeTest("non-partitioned write - data frame config trumps table config") { dir => val df = spark.range(0, 100, 1, 4).toDF() df.write.format("delta").option(DeltaOptions.OPTIMIZE_WRITE_OPTION, "false").save(dir) checkResult( df, numFileCheck = _ === 4, dir) } writeTest("partitioned write - table config") { dir => val df = spark.range(0, 100, 1, 4) .withColumn("part", 'id % 5) df.write.partitionBy("part").format("delta").save(dir) checkResult( df, numFileCheck = _ <= 5, dir) } test("partitioned write - data frame config") { withTempDir { dir => val df = spark.range(0, 100, 1, 4) .withColumn("part", 'id % 5) df.write.partitionBy("part").option(DeltaOptions.OPTIMIZE_WRITE_OPTION, "true") .format("delta").save(dir) checkResult( df, numFileCheck = _ <= 5, dir) } } writeTest("partitioned write - data frame config trumps table config") { dir => val df = spark.range(0, 100, 1, 4) .withColumn("part", 'id % 5) df.write.partitionBy("part").format("delta") .option(DeltaOptions.OPTIMIZE_WRITE_OPTION, "false").save(dir) checkResult( df, numFileCheck = _ === 20, dir) } writeTest("multi-partitions - table config") { dir => val df = spark.range(0, 100, 1, 4) .withColumn("part", 'id % 5) .withColumn("part2", ('id / 20).cast("int")) df.write.partitionBy("part", "part2").format("delta").save(dir) checkResult( df, numFileCheck = _ <= 25, dir) } test("multi-partitions - data frame config") { withTempDir { dir => val df = spark.range(0, 100, 1, 4) .withColumn("part", 'id % 5) .withColumn("part2", ('id / 20).cast("int")) df.write.partitionBy("part", "part2") .option(DeltaOptions.OPTIMIZE_WRITE_OPTION, "true").format("delta").save(dir) checkResult( df, numFileCheck = _ <= 25, dir) } } test("optimized writes used if enabled when a stream starts") { withTempDir { f => // Write some data into the table so it already exists Seq(1).toDF().write.format("delta").save(f) // Use optimized writes just when starting the stream val inputData = MemoryStream[Int] val df = inputData.toDF().repartition(10) var stream: StreamingQuery = null // Start the stream with optimized writes enabled, and then reset the conf withSQLConf(DeltaSQLConf.DELTA_OPTIMIZE_WRITE_ENABLED.key -> "true") { val checkpoint = new File(f, "checkpoint").getCanonicalPath stream = df.writeStream.format("delta").option("checkpointLocation", checkpoint).start(f) } try { inputData.addData(1 to 100) stream.processAllAvailable() } finally { stream.stop() } val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, f) assert(snapshot.numOfFiles == 2, "Optimized writes were not used") } } writeTest("multi-partitions - data frame config trumps table config") { dir => val df = spark.range(0, 100, 1, 4) .withColumn("part", 'id % 5) .withColumn("part2", ('id / 20).cast("int")) df.write.partitionBy("part", "part2") .option(DeltaOptions.OPTIMIZE_WRITE_OPTION, "false").format("delta").save(dir) checkResult( df, numFileCheck = _ > 25, dir) } writeTest("optimize should not leverage optimized writes") { dir => val df = spark.range(0, 10, 1, 2) val logs1 = Log4jUsageLogger.track { df.write.format("delta").mode("append").save(dir) df.write.format("delta").mode("append").save(dir) }.filter(_.metric == "tahoeEvent") assert(logs1.count(_.tags.get("opType") === Some("delta.optimizeWrite.planned")) === 2) val logs2 = Log4jUsageLogger.track { sql(s"optimize delta.`$dir`") }.filter(_.metric == "tahoeEvent") assert(logs2.count(_.tags.get("opType") === Some("delta.optimizeWrite.planned")) === 0) } writeTest("map task with more partitions than target shuffle blocks - non-partitioned") { dir => val df = spark.range(0, 20, 1, 4) withSQLConf(DeltaSQLConf.DELTA_OPTIMIZE_WRITE_SHUFFLE_BLOCKS.key -> "2") { df.write.format("delta").mode("append").save(dir) } checkResult( df.toDF(), numFileCheck = _ === 1, dir) } writeTest("map task with more partitions than target shuffle blocks - partitioned") { dir => val df = spark.range(0, 20, 1, 4).withColumn("part", 'id % 5) withSQLConf(DeltaSQLConf.DELTA_OPTIMIZE_WRITE_SHUFFLE_BLOCKS.key -> "2") { df.write.format("delta").partitionBy("part").mode("append").save(dir) } checkResult( df, numFileCheck = _ === 5, dir) } writeTest("zero partition dataframe write") { dir => val df = spark.range(0, 20, 1, 4).withColumn("part", 'id % 5) df.write.format("delta").partitionBy("part").mode("append").save(dir) val schema = new StructType().add("id", LongType).add("part", LongType) spark.createDataFrame(sparkContext.emptyRDD[Row], schema).write.format("delta") .partitionBy("part").mode("append").save(dir) checkResult( df, numFileCheck = _ === 5, dir) } test("OptimizedWriterBlocks is not serializable") { assert(!new OptimizedWriterBlocks(Array.empty).isInstanceOf[Serializable], "The blocks should not be serializable so that they don't get shipped to executors.") } writeTest("single partition dataframe write") { dir => val df = spark.range(0, 20).repartition(1).withColumn("part", 'id % 5) val logs1 = Log4jUsageLogger.track { df.write.format("delta").partitionBy("part").mode("append").save(dir) }.filter(_.metric == "tahoeEvent") // doesn't use optimized writes assert(logs1.count(_.tags.get("opType") === Some("delta.optimizeWrite.planned")) === 0) checkResult( df, numFileCheck = _ === 5, dir) } writeTest("do not create tons of shuffle partitions during optimized writes") { dir => // 50M shuffle blocks would've led to 25M shuffle partitions withSQLConf(DeltaSQLConf.DELTA_OPTIMIZE_WRITE_SHUFFLE_BLOCKS.key -> "50000000") { val df = spark.range(0, 20).repartition(2).withColumn("part", 'id % 5) val logs1 = Log4jUsageLogger.track { df.write.format("delta").partitionBy("part").mode("append").save(dir) }.filter(_.metric == "tahoeEvent") .filter(_.tags.get("opType") === Some("delta.optimizeWrite.planned")) assert(logs1.length === 1) val blob = JsonUtils.fromJson[Map[String, Any]](logs1.head.blob) assert(blob("outputPartitions") === 5) assert(blob("originalPartitions") === 2) assert(blob("numShuffleBlocks") === 50000000) assert(blob("shufflePartitions") === spark.conf.get(DeltaSQLConf.DELTA_OPTIMIZE_WRITE_MAX_SHUFFLE_PARTITIONS)) checkResult( df, numFileCheck = _ === 5, dir) } } } class OptimizedWritesSuite extends OptimizedWritesSuiteBase with DeltaSQLCommandTest {} ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/rowid/ConflictCheckerRowIdSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.rowid import java.io.File import scala.concurrent.duration.Duration import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.DeltaOperations.ManualUpdate import org.apache.spark.sql.delta.DeltaTestUtils.createTestAddFile import org.apache.spark.sql.delta.concurrency.PhaseLockingTestMixin import org.apache.spark.sql.delta.concurrency.TransactionExecutionTestMixin import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.ThreadUtils class ConflictCheckerRowIdSuite extends QueryTest with SharedSparkSession with PhaseLockingTestMixin with TransactionExecutionTestMixin with RowIdTestUtils { protected def appendRows(dir: File, numRows: Int, numFiles: Int): Unit = { withRowTrackingEnabled(enabled = true) { spark.range(start = 0, end = numRows, step = 1, numPartitions = numFiles) .write.format("delta").mode("append").save(dir.getAbsolutePath) } } protected def setIsolationLevel(tableDir: File): Unit = { spark.sql( s"""ALTER TABLE delta.`${tableDir.getAbsolutePath}` |SET TBLPROPERTIES ('${DeltaConfigs.ISOLATION_LEVEL.key}' = '$Serializable') |""".stripMargin ) } test("concurrent transactions do not assign overlapping row IDs") { withTempDir { tempDir => appendRows(tempDir, numRows = 100, numFiles = 1) setIsolationLevel(tempDir) def txnA(): Array[Row] = { appendRows(tempDir, numRows = 1000, numFiles = 2) Array.empty } def txnB(): Array[Row] = { appendRows(tempDir, numRows = 1500, numFiles = 3) Array.empty } val (futureA, futureB) = runTxnsWithOrder__A_Start__B__A_End(txnA, txnB) ThreadUtils.awaitResult(futureA, Duration.Inf) ThreadUtils.awaitResult(futureB, Duration.Inf) val log = DeltaLog.forTable(spark, tempDir) assertRowIdsAreValid(log) } } test("re-added files keep their row ids") { withTempDir { tempDir => appendRows(tempDir, numRows = 100, numFiles = 3) setIsolationLevel(tempDir) val log = DeltaLog.forTable(spark, tempDir) val filesBefore = log.update().allFiles.collect() val baseRowIdsBefore = filesBefore.map(f => f.path -> f.baseRowId.get).toMap def txnA(): Array[Row] = { log.startTransaction().commit(filesBefore, ManualUpdate) Array.empty } def txnB(): Array[Row] = { appendRows(tempDir, numRows = 113, numFiles = 2) Array.empty } val (futureA, futureB) = runTxnsWithOrder__A_Start__B__A_End(txnA, txnB) ThreadUtils.awaitResult(futureA, Duration.Inf) ThreadUtils.awaitResult(futureB, Duration.Inf) assertRowIdsAreValid(log) val filesAfter = log.update().allFiles.collect() val baseRowIdsAfter = filesAfter.map(f => f.path -> f.baseRowId.get).toMap filesBefore.foreach { file => assert(baseRowIdsBefore(file.path) === baseRowIdsAfter(file.path)) } } } test("Re-added files keep their row IDs after conflict with txn not " + "updating high watermark") { withTempDir { dir => appendRows(dir, numRows = 10, numFiles = 2) setIsolationLevel(dir) val log = DeltaLog.forTable(spark, dir) val filesBefore = log.update().allFiles.collect() assert(filesBefore.length === 2) // Adds one file that will change the high water mark, and one file that was // in the table before. def txnA(): Array[Row] = { val file1 = createTestAddFile() val oldFile = filesBefore.last log.startTransaction().commit(Seq(oldFile, file1), ManualUpdate) Array.empty } // Adds another file that was in the table before, so we have a conflict // with a txn that does not change the high water mark. def txnB(): Array[Row] = { log.startTransaction().commit(Seq(filesBefore.head), ManualUpdate) Array.empty } // One more transaction to change the high water mark, which will lead to txnA reassigning // its row IDs. def txnC(): Array[Row] = { appendRows(dir, numRows = 30, numFiles = 3) Array.empty } val (futureA, futureB, futureC) = runTxnsWithOrder__A_Start__B__C__A_End(txnA, txnB, txnC) ThreadUtils.awaitResult(futureA, Duration.Inf) ThreadUtils.awaitResult(futureB, Duration.Inf) ThreadUtils.awaitResult(futureC, Duration.Inf) assertRowIdsAreValid(log) val baseRowIdsAfter = log.update().allFiles.collect().map(f => f.path -> f.baseRowId).toMap filesBefore.foreach { file => assert(file.baseRowId === baseRowIdsAfter(file.path)) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/rowid/GenerateRowIDsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.rowid import org.apache.spark.sql.delta.{RowCommitVersion, RowId} import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.{DataFrame, QueryTest} import org.apache.spark.sql.catalyst.expressions.{Add, Alias, AttributeReference, Coalesce, EqualTo, Expression, FileSourceMetadataAttribute, GetStructField, MetadataAttributeWithLogicalName} import org.apache.spark.sql.catalyst.plans.logical.{Filter, Join, LogicalPlan, Project} import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.types.StructType /** * This test suite checks the optimized logical plans produced after applying the [[GenerateRowIDs]] * rule. It ensures that the rule is correctly applied to all Delta scans in different scenarios and * that the optimizer is able to remove redundant expressions or nodes when possible. */ class GenerateRowIDsSuite extends QueryTest with RowIdTestUtils { protected val testTable: String = "generateRowIDsTestTable" override def beforeAll(): Unit = { super.beforeAll() withRowTrackingEnabled(enabled = true) { spark.range(start = 0, end = 20) .toDF("id") .write .format("delta") .saveAsTable(testTable) } } override def afterAll(): Unit = { sql(s"DROP TABLE IF EXISTS $testTable") super.afterAll() } /** * Test runner checking that the optimized plan for the given dataframe matches the expected plan. * The expected plan is defined as a partial function `check`, e.g.: * check = { * case Project(_, LogicalRelation) => // Do additional checks * } * * Note: Pass `df` by name to avoid evaluating anything before test setup. */ protected def testRowIdPlan( testName: String, df: => DataFrame, rowTrackingEnabled: Boolean = true)( check: PartialFunction[LogicalPlan, Unit]): Unit = { test(testName) { withRowTrackingEnabled(enabled = rowTrackingEnabled) { check.applyOrElse(df.queryExecution.optimizedPlan, { plan: LogicalPlan => fail(s"Unexpected optimized plan: $plan") }) } } } /** * Checks that the given expression corresponds to the expression used to generate Row IDs: * coalesce(_metadata.row_id, _metadata.base_row_id + _metadata.row_index). */ protected def checkRowIdExpr(expr: Expression): Unit = { expr match { case Coalesce( Seq( GetStructField(FileSourceMetadataAttribute(_), _, _), Add( GetStructField(FileSourceMetadataAttribute(_), _, _), GetStructField(FileSourceMetadataAttribute(_), _, _), _))) => () case Alias(aliasedExpr, RowId.ROW_ID) => checkRowIdExpr(aliasedExpr) case _ => fail(s"Expression didn't match expected Row ID expression: $expr") } } /** * Checks that the given expression corresponds to the an expression used to generate Row commit * versions: * coalesce(_metadata.row_commit_version, _metadata.default_row_commit_version). */ protected def checkRowCommitVersionExpr(expr: Expression): Unit = expr match { case Coalesce( Seq( GetStructField(FileSourceMetadataAttribute(_), _, _), GetStructField(FileSourceMetadataAttribute(_), _, _))) => () case Alias(aliasedExpr, RowCommitVersion.METADATA_STRUCT_FIELD_NAME) => checkRowCommitVersionExpr(aliasedExpr) case _ => fail(s"Expression didn't match expected Row commit version expression: $expr") } /** * Checks that a metadata column is present in `output` and that it contains the given fields and * only these. */ protected def checkMetadataFieldsPresent( output: Seq[AttributeReference], expectedFieldNames: Seq[String]) : Unit = { val metadataSchema = output.collect { case FileSourceMetadataAttribute( MetadataAttributeWithLogicalName( AttributeReference(_, schema: StructType, _, _), _)) => schema } assert(metadataSchema.nonEmpty, s"No metadata column present in output: $output") assert(metadataSchema.head.fieldNames === expectedFieldNames, "Unexpected metadata fields present in the metadata output.") } for (rowTrackingEnabled <- BOOLEAN_DOMAIN) testRowIdPlan(s"Regular column selected, rowTrackingEnabled: $rowTrackingEnabled", sql(s"SELECT id FROM $testTable"), rowTrackingEnabled) { // No projection is added when no metadata column is selected. case lr: LogicalRelation => assert(lr.output.map(_.name) === Seq("id"), "Scan list didn't match") } for (rowTrackingEnabled <- BOOLEAN_DOMAIN) testRowIdPlan(s"Metadata column selected, rowTrackingEnabled: $rowTrackingEnabled", sql(s"SELECT _metadata.file_path FROM $testTable"), rowTrackingEnabled) { // Selecting a metadata column adds a projection to unpack metadata fields (unrelated to Row // IDs). Row IDs don't introduce an extra projection. case Project(projectList, lr: LogicalRelation) => assert(projectList.map(_.name) === Seq("file_path"), "Project list didn't match") assert(lr.output.map(_.name) === Seq("id", "_metadata"), "Scan list didn't match") checkMetadataFieldsPresent(lr.output, Seq("file_path")) } testRowIdPlan("Row ID column selected", sql(s"SELECT _metadata.row_id FROM $testTable")) { // Selecting Row IDs injects an expression to generate default Row IDs. case Project(Seq(rowIdExpr), lr: LogicalRelation) => assert(rowIdExpr.name == RowId.ROW_ID) checkRowIdExpr(rowIdExpr) assert(lr.output.map(_.name) === Seq("id", "_metadata")) checkMetadataFieldsPresent(lr.output, Seq("row_index", "row_id", "base_row_id")) } testRowIdPlan("Filter on Row ID column", sql(s"SELECT * FROM $testTable WHERE _metadata.row_id = 5")) { // Filtering on Row IDs injects an expression to generate default Row IDs in the filter. case Project(projectList, Filter(EqualTo(rowIdExpr, _), lr: LogicalRelation)) => assert(projectList.map(_.name) === Seq("id"), "Project list didn't match") checkRowIdExpr(rowIdExpr) assert(lr.output.map(_.name) === Seq("id", "_metadata"), "Scan list didn't match") checkMetadataFieldsPresent(lr.output, Seq("row_index", "row_id", "base_row_id")) } testRowIdPlan("Filter on Row ID in subquery", sql(s"SELECT * FROM $testTable WHERE _metadata.row_id IN (SELECT id FROM $testTable)")) { // Filtering on Row IDs using a subquery injects an expression to generate default Row IDs in // the subquery. case Project( projectList, Join(right: LogicalRelation, left: LogicalPlan, _, joinCond, _)) => assert(projectList.map(_.name) === Seq("id"), "Project list didn't match") assert(right.output.map(_.name) === Seq("id", "_metadata"), "Outer scan output didn't match") checkMetadataFieldsPresent(right.output, Seq("row_index", "row_id", "base_row_id")) assert(left.output.map(_.name) === Seq("id"), "Subquery scan output didn't match") joinCond match { case Some(EqualTo(rowIdExpr, _)) => checkRowIdExpr(rowIdExpr) case _ => fail(s"Subquery was transformed into a join with an unexpected condition.") } } testRowIdPlan("Row commit version column selected", sql(s"SELECT _metadata.row_commit_version FROM $testTable")) { // Selecting Row commit versions injects an expression to generate default Row commit versions. case Project(Seq(rowIdExpr), lr: LogicalRelation) => assert(rowIdExpr.name == RowCommitVersion.METADATA_STRUCT_FIELD_NAME) checkRowCommitVersionExpr(rowIdExpr) assert(lr.output.map(_.name) === Seq("id", "_metadata")) checkMetadataFieldsPresent(lr.output, Seq("default_row_commit_version", "row_commit_version")) } testRowIdPlan("Filter on Row commit version column", sql(s"SELECT * FROM $testTable WHERE _metadata.row_commit_version = 5")) { // Filtering on Row commit version injects an expression to generate default Row commit version // in the filter. case Project(projectList, Filter(EqualTo(rowIdExpr, _), lr: LogicalRelation)) => assert(projectList.map(_.name) === Seq("id"), "Project list didn't match") checkRowCommitVersionExpr(rowIdExpr) assert(lr.output.map(_.name) === Seq("id", "_metadata"), "Scan list didn't match") checkMetadataFieldsPresent(lr.output, Seq("default_row_commit_version", "row_commit_version")) } testRowIdPlan("Filter on Row commit version in subquery", sql(s"SELECT * FROM $testTable WHERE _metadata.row_commit_version IN (SELECT id FROM " + s"$testTable)")) { // Filtering on Row commit versions using a subquery injects an expression to generate default // Row commit versions in the subquery. case Project( projectList, Join(right: LogicalRelation, left: LogicalPlan, _, joinCond, _)) => assert(projectList.map(_.name) === Seq("id"), "Project list didn't match") assert(right.output.map(_.name) === Seq("id", "_metadata"), "Outer scan output didn't match") checkMetadataFieldsPresent(right.output, Seq("default_row_commit_version", "row_commit_version")) assert(left.output.map(_.name) === Seq("id"), "Subquery scan output didn't match") joinCond match { case Some(EqualTo(rowIdExpr, _)) => checkRowCommitVersionExpr(rowIdExpr) case _ => fail(s"Subquery was transformed into a join with an unexpected condition.") } } testRowIdPlan("Rename metadata column", sql(s"SELECT renamed_metadata FROM (SELECT _metadata AS renamed_metadata FROM $testTable)" )) { case Project(projectList, lr: LogicalRelation) => assert(projectList.map(_.name) === Seq("renamed_metadata"), "Project list didn't match") assert(lr.output.map(_.name) === Seq("id", "_metadata")) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowIdCloneSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.rowid import org.apache.spark.sql.delta.{DeltaConfigs, DeltaIllegalStateException, DeltaLog, DeltaUnsupportedOperationException, RowId} import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.SparkFunSuite import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.test.SharedSparkSession class RowIdCloneSuite extends QueryTest with SharedSparkSession with RowIdTestUtils { val numRows = 10 test("clone assigns fresh row IDs when explicitly adding row IDs support") { withTables( TableSetupInfo(tableName = "source", rowIdsEnabled = false, tableState = TableState.NON_EMPTY), TableSetupInfo(tableName = "target", rowIdsEnabled = false, tableState = TableState.NON_EXISTING)) { cloneTable( targetTableName = "target", sourceTableName = "source", tblProperties = s"'$rowTrackingFeatureName' = 'supported'" :: Nil) val (targetLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier("target")) assertRowIdsAreValid(targetLog) assert(RowId.isSupported(snapshot.protocol)) assert(!RowId.isEnabled(snapshot.protocol, snapshot.metadata)) } } test("clone a table with row tracking enabled into non-existing target " + "enables row tracking even if disabled by default") { withTables( TableSetupInfo(tableName = "source", rowIdsEnabled = true, tableState = TableState.NON_EMPTY), TableSetupInfo(tableName = "target", rowIdsEnabled = false, tableState = TableState.NON_EXISTING)) { withRowTrackingEnabled(enabled = false) { cloneTable(targetTableName = "target", sourceTableName = "source") val (targetLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier("target")) assertRowIdsAreValid(targetLog) assert(RowId.isSupported(targetLog.update().protocol)) assert(RowId.isEnabled(snapshot.protocol, snapshot.metadata)) } } } for (enableRowIdsForSource <- BOOLEAN_DOMAIN) test("self-clone an empty table does not change the table's Row Tracking " + s"enablement and does not set Row IDs, enableRowIdsForSource=$enableRowIdsForSource") { withTables( TableSetupInfo(tableName = "source", rowIdsEnabled = enableRowIdsForSource, tableState = TableState.EMPTY)) { cloneTable(targetTableName = "source", sourceTableName = "source") val (targetLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier("source")) assertRowIdsAreNotSet(targetLog) assert(RowId.isSupported(targetLog.update().protocol) === enableRowIdsForSource) assert(RowId.isEnabled(snapshot.protocol, snapshot.metadata) === enableRowIdsForSource) } } for { rowIdsEnabledOnSource <- BOOLEAN_DOMAIN targetTableState <- Seq(TableState.EMPTY, TableState.NON_EXISTING) } { test("clone from empty source into an empty or non-existing target " + s"does not assign row IDs, rowIdsEnabledOnSource=$rowIdsEnabledOnSource, " + s"targetTableState=$targetTableState") { withTables( TableSetupInfo(tableName = "source", rowIdsEnabled = rowIdsEnabledOnSource, tableState = TableState.EMPTY), TableSetupInfo(tableName = "target", rowIdsEnabled = false, tableState = targetTableState)) { cloneTable(targetTableName = "target", sourceTableName = "source") val (targetLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier("target")) assertRowIdsAreNotSet(targetLog) assert(RowId.isSupported(snapshot.protocol) === rowIdsEnabledOnSource) assert(RowId.isEnabled(snapshot.protocol, snapshot.metadata) === rowIdsEnabledOnSource) } } } for (targetTableState <- Seq(TableState.EMPTY, TableState.NON_EXISTING)) test("clone from empty source into an empty or non-existing target " + s"using property override does not assign row IDs, targetTableState=$targetTableState") { withTables( TableSetupInfo(tableName = "source", rowIdsEnabled = false, tableState = TableState.EMPTY), TableSetupInfo(tableName = "target", rowIdsEnabled = false, tableState = targetTableState)) { cloneTable( targetTableName = "target", sourceTableName = "source", tblProperties = s"'${DeltaConfigs.ROW_TRACKING_ENABLED.key}' = true" :: Nil) val (targetLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier("target")) assertRowIdsAreNotSet(targetLog) assert(RowId.isSupported(snapshot.protocol)) assert(RowId.isEnabled(snapshot.protocol, snapshot.metadata)) } } test("clone that add row ID feature using table property override " + "doesn't enable row IDs on target") { withTables( TableSetupInfo(tableName = "source", rowIdsEnabled = false, tableState = TableState.NON_EMPTY), TableSetupInfo(tableName = "target", rowIdsEnabled = false, tableState = TableState.EMPTY)) { cloneTable( targetTableName = "target", sourceTableName = "source", tblProperties = s"'$rowTrackingFeatureName' = 'supported'" :: s"'delta.minWriterVersion' = $TABLE_FEATURES_MIN_WRITER_VERSION" :: Nil) val (targetLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier("target")) assertRowIdsAreValid(targetLog) assert(RowId.isSupported(snapshot.protocol)) assert(!RowId.isEnabled(snapshot.protocol, snapshot.metadata)) } } test("clone can disable row IDs using property override") { withTables( TableSetupInfo(tableName = "source", rowIdsEnabled = true, tableState = TableState.NON_EMPTY), TableSetupInfo(tableName = "target", rowIdsEnabled = true, tableState = TableState.EMPTY)) { cloneTable( targetTableName = "target", sourceTableName = "source", tblProperties = s"'${DeltaConfigs.ROW_TRACKING_ENABLED.key}' = false" :: Nil) val (targetLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier("target")) assertRowIdsAreValid(targetLog) assert(RowId.isSupported(snapshot.protocol)) assert(!RowId.isEnabled(snapshot.protocol, snapshot.metadata)) } } test("clone throws error when assigning row IDs without stats") { withSQLConf( DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false") { withTable("source", "target") { withRowTrackingEnabled(enabled = false) { spark.range(end = 10) .write.format("delta").saveAsTable("source") } withRowTrackingEnabled(enabled = true) { // enable stats to create table with row IDs withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "true") { spark.range(0).write.format("delta").saveAsTable("target") } val err = intercept[DeltaUnsupportedOperationException] { cloneTable(targetTableName = "target", sourceTableName = "source") } checkError(err, "DELTA_CLONE_WITH_ROW_TRACKING_WITHOUT_STATS") } } } } test("clone can enable row tracking on empty target using property override") { withTables( TableSetupInfo(tableName = "source", rowIdsEnabled = false, tableState = TableState.NON_EMPTY), TableSetupInfo(tableName = "target", rowIdsEnabled = false, tableState = TableState.EMPTY)) { cloneTable( targetTableName = "target", sourceTableName = "source", tblProperties = s"'${DeltaConfigs.ROW_TRACKING_ENABLED.key}' = true" :: Nil) val (targetLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier("target")) assertRowIdsAreValid(targetLog) assert(RowId.isSupported(snapshot.protocol)) assert(RowId.isEnabled(snapshot.protocol, snapshot.metadata)) } } test("clone assigns fresh row IDs for empty target") { withTables( TableSetupInfo(tableName = "source", rowIdsEnabled = false, tableState = TableState.NON_EMPTY), TableSetupInfo(tableName = "target", rowIdsEnabled = true, tableState = TableState.EMPTY)) { cloneTable(targetTableName = "target", sourceTableName = "source") val (targetLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier("target")) assertRowIdsAreValid(targetLog) assert(RowId.isSupported(snapshot.protocol)) assert(!RowId.isEnabled(snapshot.protocol, snapshot.metadata)) } } test("clone can't assign row IDs for non-empty target") { withTables( TableSetupInfo(tableName = "source", rowIdsEnabled = false, tableState = TableState.NON_EMPTY), TableSetupInfo(tableName = "target", rowIdsEnabled = true, tableState = TableState.NON_EMPTY)) { assert(intercept[DeltaIllegalStateException] { cloneTable(targetTableName = "target", sourceTableName = "source") }.getErrorClass === "DELTA_UNSUPPORTED_NON_EMPTY_CLONE") } } test("clone from source with row tracking enabled into existing empty target " + "without row tracking enables row tracking") { withTables( TableSetupInfo(tableName = "source", rowIdsEnabled = true, tableState = TableState.NON_EMPTY), TableSetupInfo(tableName = "target", rowIdsEnabled = false, tableState = TableState.EMPTY)) { cloneTable(targetTableName = "target", sourceTableName = "source") val (targetLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier("target")) assertRowIdsAreValid(targetLog) assert(RowId.isSupported(targetLog.update().protocol)) assert(RowId.isEnabled(snapshot.protocol, snapshot.metadata)) } } def cloneTable( targetTableName: String, sourceTableName: String, tblProperties: Seq[String] = Seq.empty): Unit = { val tblPropertiesStr = if (tblProperties.nonEmpty) { s"TBLPROPERTIES ${tblProperties.mkString("(", ",", ")")}" } else { "" } sql( s""" |CREATE OR REPLACE TABLE $targetTableName |SHALLOW CLONE $sourceTableName |$tblPropertiesStr |""".stripMargin) } final object TableState extends Enumeration { type TableState = Value val NON_EMPTY, EMPTY, NON_EXISTING = Value } case class TableSetupInfo( tableName: String, rowIdsEnabled: Boolean, tableState: TableState.TableState) private def withTables(tables: TableSetupInfo*)(f: => Unit): Unit = { if (tables.isEmpty) { f } else { val firstTable = tables.head withTable(firstTable.tableName) { firstTable.tableState match { case TableState.NON_EMPTY | TableState.EMPTY => val rows = if (firstTable.tableState == TableState.NON_EMPTY) { spark.range(start = 0, end = numRows, step = 1, numPartitions = 1) } else { spark.range(0) } withRowTrackingEnabled(enabled = firstTable.rowIdsEnabled) { rows.write.format("delta").saveAsTable(firstTable.tableName) } case TableState.NON_EXISTING => } withTables(tables.drop(1): _*)(f) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowIdCreateReplaceTableSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.rowid import org.apache.spark.sql.delta.{DeltaConfigs, DeltaLog} import org.apache.spark.sql.delta.RowId.extractHighWatermark import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION import org.apache.spark.sql.QueryTest import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.test.SharedSparkSession class RowIdCreateReplaceTableSuite extends QueryTest with SharedSparkSession with RowIdTestUtils { private val numSourceRows = 50 test("Create or replace table with values list") { withRowTrackingEnabled(enabled = true) { withTable("target") { writeTargetTestData(withRowIds = true) val (log, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier("target")) val highWaterMarkBefore = extractHighWatermark(snapshot).get createReplaceTargetTable( commandName = "CREATE OR REPLACE", query = "SELECT * FROM VALUES (0, 0), (1, 1)") assertHighWatermarkIsCorrectAfterUpdate( log, highWaterMarkBefore, expectedNumRecordsWritten = 2) assertRowIdsAreLargerThanValue(log, highWaterMarkBefore) } } } test("Create or replace table with other delta table") { withRowTrackingEnabled(enabled = true) { withTable("source", "target") { writeTargetTestData(withRowIds = true) writeSourceTestData(withRowIds = true) val (log, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier("target")) val highWaterMarkBefore = extractHighWatermark(snapshot).get createReplaceTargetTable(commandName = "CREATE OR REPLACE", query = "SELECT * FROM source") assertHighWatermarkIsCorrectAfterUpdate( log, highWaterMarkBefore, expectedNumRecordsWritten = numSourceRows) assertRowIdsAreLargerThanValue(log, highWaterMarkBefore) } } } test("Replace table with values list") { withRowTrackingEnabled(enabled = true) { withTable("target") { writeTargetTestData(withRowIds = true) val (log, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier("target")) val highWaterMarkBefore = extractHighWatermark(snapshot).get createReplaceTargetTable(commandName = "REPLACE", query = "SELECT * FROM VALUES (0), (1)") assertHighWatermarkIsCorrectAfterUpdate( log, highWaterMarkBefore, expectedNumRecordsWritten = 2) assertRowIdsAreLargerThanValue(log, highWaterMarkBefore) } } } test("Replace table with another delta table") { withRowTrackingEnabled(enabled = true) { withTable("source", "target") { writeTargetTestData(withRowIds = true) val log = DeltaLog.forTable(spark, TableIdentifier("target")) writeSourceTestData(withRowIds = true) val highWaterMarkBefore = extractHighWatermark(log.update()).get createReplaceTargetTable(commandName = "REPLACE", query = "SELECT * FROM source") assertHighWatermarkIsCorrectAfterUpdate( log, highWaterMarkBefore, expectedNumRecordsWritten = numSourceRows) assertRowIdsAreLargerThanValue(log, highWaterMarkBefore) } } } test("Replace table with row IDs with table without row IDs assigns new row IDs") { withTable("source", "target") { writeTargetTestData(withRowIds = true) val log = DeltaLog.forTable(spark, TableIdentifier("target")) writeSourceTestData(withRowIds = false) val highWaterMarkBefore = extractHighWatermark(log.update()).get withRowTrackingEnabled(enabled = false) { createReplaceTargetTable(commandName = "REPLACE", query = "SELECT * FROM source") } assertHighWatermarkIsCorrectAfterUpdate( log, highWaterMarkBefore, expectedNumRecordsWritten = numSourceRows) } } test("Replacing a table without row IDs with row IDs enabled assigns new row IDs") { withTable("source", "target") { writeTargetTestData(withRowIds = false) writeSourceTestData(withRowIds = true) val log = DeltaLog.forTable(spark, TableIdentifier("target")) assertRowIdsAreNotSet(log) withRowTrackingEnabled(enabled = true) { createReplaceTargetTable( commandName = "REPLACE", query = "SELECT * FROM source", tblProperties = s"'$rowTrackingFeatureName' = 'supported'" :: s"'delta.minWriterVersion' = $TABLE_FEATURES_MIN_WRITER_VERSION" :: Nil) } assertRowIdsAreValid(log) val df = spark.read.table("target").select("*", "_metadata.row_id") checkAnswer(df, (0 until 50).map(i => Row(i, i))) } } test("CREATE OR REPLACE on existing table without row IDs assigns new row IDs when enabling " + "row IDs") { withTable("target") { writeTargetTestData(withRowIds = false) val log = DeltaLog.forTable(spark, TableIdentifier("target")) assertRowIdsAreNotSet(log) withRowTrackingEnabled(enabled = true) { createReplaceTargetTable( commandName = "CREATE OR REPLACE", query = "SELECT * FROM VALUES (0), (1)", tblProperties = s"${DeltaConfigs.ROW_TRACKING_ENABLED.key} = 'true'" :: Nil) } assertRowIdsAreValid(log) val df = spark.read.table("target").select("*", "_metadata.row_id") checkAnswer(df, Seq(Row(0, 0), Row(1, 1))) } } test("CTAS assigns new row IDs when immediately enabling row IDs") { withTable("target") { createReplaceTargetTable( commandName = "CREATE", query = "SELECT * FROM VALUES (0), (1)", tblProperties = s"${DeltaConfigs.ROW_TRACKING_ENABLED.key} = 'true'" :: Nil) val log = DeltaLog.forTable(spark, TableIdentifier("target")) assertRowIdsAreValid(log) val df = spark.read.table("target").select("*", "_metadata.row_id") checkAnswer(df, Seq(Row(0, 0), Row(1, 1))) } } test("CTAS assigns new row IDs when row IDs are by default enabled") { withTable("target") { withSQLConf(DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey -> "true") { createReplaceTargetTable( commandName = "CREATE", query = "SELECT * FROM VALUES (0), (1)") val log = DeltaLog.forTable(spark, TableIdentifier("target")) assertRowIdsAreValid(log) val df = spark.read.table("target").select("*", "_metadata.row_id") checkAnswer(df, Seq(Row(0, 0), Row(1, 1))) } } } def createReplaceTargetTable( commandName: String, query: String, tblProperties: Seq[String] = Seq.empty): Unit = { val tblPropertiesStr = if (tblProperties.nonEmpty) { s"TBLPROPERTIES ${tblProperties.mkString("(", ",", ")")}" } else { "" } sql( s""" |$commandName TABLE target |USING delta |$tblPropertiesStr |AS $query |""".stripMargin) } def writeTargetTestData(withRowIds: Boolean): Unit = { withRowTrackingEnabled(enabled = withRowIds) { spark.range(start = 0, end = 100, step = 1, numPartitions = 1) .write.format("delta").saveAsTable("target") } } def writeSourceTestData(withRowIds: Boolean): Unit = { withRowTrackingEnabled(enabled = withRowIds) { spark.range(start = 0, end = numSourceRows, step = 1, numPartitions = 1) .write.format("delta").saveAsTable("source") } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowIdSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.rowid import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.{DataFrameUtils, DeltaConfigs, DeltaIllegalStateException, DeltaLog, DeltaOperations, DeltaTableUtils, MaterializedRowCommitVersion, MaterializedRowId, RowCommitVersion, RowId, RowTrackingFeature, Serializable, SnapshotIsolation} import org.apache.spark.sql.delta.DeltaOperations.ManualUpdate import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.RowId.RowTrackingMetadataDomain import org.apache.spark.sql.delta.actions.{CommitInfo, Protocol} import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.FileNames import org.apache.parquet.column.Encoding import org.apache.parquet.column.ParquetProperties import org.apache.parquet.hadoop.ParquetOutputFormat import org.apache.spark.SparkException import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.{FileSourceScanExec, SparkPlan} import org.apache.spark.sql.execution.datasources.parquet.ParquetTest import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{LongType, MetadataBuilder, StructField, StructType} class RowIdSuite extends QueryTest with SharedSparkSession with ParquetTest with RowIdTestUtils { test("Enabling row IDs on existing table does not set row IDs as readable") { withRowTrackingEnabled(enabled = false) { withTable("tbl") { spark.range(10).write.format("delta") .saveAsTable("tbl") sql( s""" |ALTER TABLE tbl |SET TBLPROPERTIES ( |'$rowTrackingFeatureName' = 'supported', |'delta.minWriterVersion' = $TABLE_FEATURES_MIN_WRITER_VERSION)""".stripMargin) val (log, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier("tbl")) assert(RowId.isSupported(snapshot.protocol)) assert(!RowId.isEnabled(snapshot.protocol, snapshot.metadata)) } } } test("row ids are assigned when they are enabled") { withRowTrackingEnabled(enabled = true) { withTempDir { dir => spark.range(start = 0, end = 1000, step = 1, numPartitions = 10) .write.format("delta").save(dir.getAbsolutePath) val log = DeltaLog.forTable(spark, dir) assertRowIdsAreValid(log) spark.range(start = 1000, end = 1500, step = 1, numPartitions = 3) .write.format("delta").mode("append").save(dir.getAbsolutePath) assertRowIdsAreValid(log) } } } test("row ids are not assigned when they are disabled") { withRowTrackingEnabled(enabled = false) { withTempDir { dir => spark.range(start = 0, end = 1000, step = 1, numPartitions = 10) .write.format("delta").save(dir.getAbsolutePath) val log = DeltaLog.forTable(spark, dir) assertRowIdsAreNotSet(log) spark.range(start = 1000, end = 1500, step = 1, numPartitions = 3) .write.format("delta").mode("append").save(dir.getAbsolutePath) assertRowIdsAreNotSet(log) } } } test("row ids can be disabled") { withRowTrackingEnabled(enabled = true) { withTempDir { dir => spark.range(start = 0, end = 1000, step = 1, numPartitions = 10) .write.format("delta").save(dir.getAbsolutePath) val log = DeltaLog.forTable(spark, dir) assertRowIdsAreValid(log) sql(s"ALTER TABLE delta.`${dir.getAbsolutePath}` " + s"SET TBLPROPERTIES ('${DeltaConfigs.ROW_TRACKING_ENABLED.key}' = false)") checkAnswer( spark.read.format("delta").load(dir.getAbsolutePath), (0 until 1000).map(Row(_))) } } } test("high watermark survives checkpointing") { withRowTrackingEnabled(enabled = true) { withTempDir { dir => spark.range(start = 0, end = 1000, step = 1, numPartitions = 10) .write.format("delta").save(dir.getAbsolutePath) val log1 = DeltaLog.forTable(spark, dir) assertRowIdsAreValid(log1) // Force a checkpoint and add an empty commit, so that we can delete the first commit log1.checkpoint(log1.update()) log1.startTransaction().commit(Nil, ManualUpdate) DeltaLog.clearCache() // Delete the first commit and all checksum files to force the next read to read the high // watermark from the checkpoint. val fs = log1.logPath.getFileSystem(log1.newDeltaHadoopConf()) fs.delete(FileNames.unsafeDeltaFile(log1.logPath, version = 0), true) fs.delete(FileNames.checksumFile(log1.logPath, version = 0), true) fs.delete(FileNames.checksumFile(log1.logPath, version = 1), true) spark.range(start = 1000, end = 1500, step = 1, numPartitions = 3) .write.format("delta").mode("append").save(dir.getAbsolutePath) val log2 = DeltaLog.forTable(spark, dir) assertRowIdsAreValid(log2) } } } test("re-added files keep their row ids") { withRowTrackingEnabled(enabled = true) { withTempDir { dir => spark.range(start = 0, end = 1000, step = 1, numPartitions = 10) .write.format("delta").save(dir.getAbsolutePath) val log = DeltaLog.forTable(spark, dir) assertRowIdsAreValid(log) val filesBefore = log.update().allFiles.collect() val baseRowIdsBefore = filesBefore.map(f => f.path -> f.baseRowId.get).toMap log.startTransaction().commit(filesBefore, ManualUpdate) assertRowIdsAreValid(log) val filesAfter = log.update().allFiles.collect() val baseRowIdsAfter = filesAfter.map(f => f.path -> f.baseRowId.get).toMap assert(baseRowIdsBefore == baseRowIdsAfter) } } } test("RESTORE retains high watermark") { withRowTrackingEnabled(enabled = true) { withTempDir { dir => // version 0: high watermark = 9 spark.range(start = 0, end = 10) .write.format("delta").save(dir.getAbsolutePath) val log = DeltaLog.forTable(spark, dir) val deltaTable = io.delta.tables.DeltaTable.forPath(spark, dir.getAbsolutePath) // version 1: high watermark = 19 spark.range(start = 10, end = 20) .write.mode("append").format("delta").save(dir.getAbsolutePath) val highWatermarkBeforeRestore = RowId.extractHighWatermark(log.update()) // back to version 0: high watermark should be still equal to before the restore. deltaTable.restoreToVersion(0) val highWatermarkAfterRestore = RowId.extractHighWatermark(log.update()) assert(highWatermarkBeforeRestore == highWatermarkAfterRestore) assertRowIdsDoNotOverlap(log) // version 1 (overridden): high watermark = 29 spark.range(start = 10, end = 20) .write.mode("append").format("delta").save(dir.getAbsolutePath) assertHighWatermarkIsCorrectAfterUpdate( log, highWatermarkBeforeUpdate = highWatermarkAfterRestore.get, expectedNumRecordsWritten = 10) assertRowIdsDoNotOverlap(log) val highWatermarkWithNewData = RowId.extractHighWatermark(log.update()) // back to version 0: high watermark should still be 29. deltaTable.restoreToVersion(0) val highWatermarkWithNewDataAfterRestore = RowId.extractHighWatermark(log.update()) assert(highWatermarkWithNewData == highWatermarkWithNewDataAfterRestore) assertRowIdsDoNotOverlap(log) } } } for (downgradeAllowed <- BOOLEAN_DOMAIN) { test(s"RESTORE with potential row tracking downgrade, downgradeAllowed=$downgradeAllowed") { withTempDir { dir => withRowTrackingEnabled(enabled = false) { spark.range(5).write.format("delta").save(dir.toString) } val log = DeltaLog.forTable(spark, dir) val oldProtocolVersion = log.update().protocol assert(!oldProtocolVersion.isFeatureSupported(RowTrackingFeature)) val protocolWithRowTracking = Protocol(minWriterVersion = TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(RowTrackingFeature) val newProtocolVersion = oldProtocolVersion.merge(protocolWithRowTracking) log.upgradeProtocol(newProtocolVersion) withSQLConf( DeltaSQLConf.RESTORE_TABLE_PROTOCOL_DOWNGRADE_ALLOWED.key -> downgradeAllowed.toString) { sql(s"RESTORE TABLE delta.`$dir` VERSION AS OF 0") } val restoredProtocolVersion = log.update().protocol if (downgradeAllowed) { assert(restoredProtocolVersion === oldProtocolVersion) } else { assert(restoredProtocolVersion === newProtocolVersion) } } } } test("Check missing High Watermark for newly created empty table") { withRowTrackingEnabled(enabled = true) { withTempDir { dir => spark.range(start = 0, end = 0) .write.format("delta").save(dir.getAbsolutePath) val log = DeltaLog.forTable(spark, dir) assert(RowId.extractHighWatermark(log.update()) === None) assertRowIdsAreNotSet(log) spark.range(start = 0, end = 10) .write.mode("append").format("delta").save(dir.getAbsolutePath) assert(RowId.extractHighWatermark(log.update()) === Some(9)) assertRowIdsAreValid(log) } } } test("row_id column with row ids disabled") { withRowTrackingEnabled(enabled = false) { withTempDir { dir => spark.range(start = 0, end = 1000, step = 1, numPartitions = 5) .select((col("id") + 10000L).as("row_id")) .write.format("delta").save(dir.getAbsolutePath) checkAnswer( spark.read.format("delta").load(dir.getAbsolutePath), (0 until 1000).map(i => Row(i + 10000L)) ) } } } test("Throws error when assigning row IDs without stats") { withSQLConf( DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey -> "true", DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false") { withTable("target") { val err = intercept[DeltaIllegalStateException] { spark.range(end = 10).write.format("delta").saveAsTable("target") } checkError(err, "DELTA_ROW_ID_ASSIGNMENT_WITHOUT_STATS") } } } test("manually setting row ID high watermark is not allowed") { withRowTrackingEnabled(enabled = true) { withTempDir { dir => spark.range(start = 0, end = 1000, step = 1, numPartitions = 10) .write.format("delta").save(dir.getAbsolutePath) val log = DeltaLog.forTable(spark, dir) val exception = intercept[IllegalStateException] { log.startTransaction().commit( Seq(RowTrackingMetadataDomain(rowIdHighWaterMark = 9001).toDomainMetadata), ManualUpdate) } assert(exception.getMessage.contains( "Manually setting the Row ID high water mark is not allowed")) } } } for (prevIsolationLevel <- Seq( Serializable)) test(s"Maintenance operations can downgrade to snapshot isolation, " + s"previousIsolationLevel = $prevIsolationLevel") { withTable("table") { withSQLConf( DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey -> "true", DeltaConfigs.ISOLATION_LEVEL.defaultTablePropertyKey -> prevIsolationLevel.toString) { // Create two files that will be picked up by OPTIMIZE spark.range(10).repartition(2).write.format("delta").saveAsTable("table") val (log, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier("table")) val versionBeforeOptimize = snapshot.version spark.sql("OPTIMIZE table").collect() val commitInfos = log.getChanges(versionBeforeOptimize + 1).flatMap(_._2).flatMap { case commitInfo: CommitInfo => Some(commitInfo) case _ => None }.toList assert(commitInfos.size == 1) assert(commitInfos.forall(_.isolationLevel.get == SnapshotIsolation.toString)) } } } test("ALTER TABLE cannot enable Row IDs on existing table") { withSQLConf(DeltaSQLConf.DELTA_ROW_TRACKING_BACKFILL_ENABLED.key -> "false") { withRowTrackingEnabled(enabled = false) { withTable("tbl") { spark.range(10).write.format("delta").saveAsTable("tbl") val (log, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier("tbl")) assert(!RowId.isEnabled(snapshot.protocol, snapshot.metadata)) val err = intercept[UnsupportedOperationException] { sql(s"ALTER TABLE tbl " + s"SET TBLPROPERTIES ('${DeltaConfigs.ROW_TRACKING_ENABLED.key}' = true)") } assert(err.getMessage === "Cannot enable Row IDs on an existing table.") } } } } test(s"CONVERT TO DELTA assigns row ids") { withRowTrackingEnabled(enabled = true) { withTempDir { dir => spark.range(10).repartition(1) .write.format("parquet").mode("overwrite").save(dir.getAbsolutePath) sql(s"CONVERT TO DELTA parquet.`$dir`") val log = DeltaLog.forTable(spark, dir) assertRowIdsAreValid(log) assert(extractMaterializedRowIdColumnName(log).isDefined) val df = spark.read.format("delta").load(dir.toString) checkAnswer( df.select("id", "_metadata.row_id"), (0 until 10).map(i => Row(i, i))) } } } test(s"CONVERT TO DELTA NO STATISTICS throws error") { withRowTrackingEnabled(enabled = true) { withTempDir { dir => spark.range(10) .write.format("parquet").mode("overwrite").save(dir.getAbsolutePath) val err = intercept[DeltaIllegalStateException] { sql(s"CONVERT TO DELTA parquet.`$dir` NO STATISTICS") } checkError(err, "DELTA_CONVERT_TO_DELTA_ROW_TRACKING_WITHOUT_STATS", parameters = Map( "statisticsCollectionPropertyKey" -> DeltaSQLConf.DELTA_COLLECT_STATS.key, "rowTrackingTableFeatureDefaultKey" -> defaultRowTrackingFeatureProperty, "rowTrackingDefaultPropertyKey" -> DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey)) } } } test(s"CONVERT TO DELTA without stats collection enabled throws error") { withRowTrackingEnabled(enabled = true) { withTempDir { dir => spark.range(10) .write.format("parquet").mode("overwrite").save(dir.getAbsolutePath) withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false") { val err = intercept[DeltaIllegalStateException] { sql(s"CONVERT TO DELTA parquet.`$dir`") } checkError(err, "DELTA_CONVERT_TO_DELTA_ROW_TRACKING_WITHOUT_STATS", parameters = Map( "statisticsCollectionPropertyKey" -> DeltaSQLConf.DELTA_COLLECT_STATS.key, "rowTrackingTableFeatureDefaultKey" -> defaultRowTrackingFeatureProperty, "rowTrackingDefaultPropertyKey" -> DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey)) } } } } test("Base Row ID metadata field has the expected type") { withRowTrackingEnabled(enabled = true) { withTempDir { tempDir => spark.range(start = 0, end = 20).toDF("id") .write.format("delta").save(tempDir.getAbsolutePath) val df = spark.read.format("delta").load(tempDir.getAbsolutePath) .select(QUALIFIED_BASE_ROW_ID_COLUMN_NAME) val expectedBaseRowIdMetadata = new MetadataBuilder() .putBoolean("__base_row_id_metadata_col", value = true) .build() val expectedBaseRowIdField = StructField( RowId.BASE_ROW_ID, LongType, nullable = false, metadata = expectedBaseRowIdMetadata) Seq(df.schema, df.queryExecution.analyzed.schema, df.queryExecution.optimizedPlan.schema) .foreach { schema => assert(schema === new StructType().add(expectedBaseRowIdField)) } } } } test("Base Row IDs can be read with conflicting metadata column name") { withRowTrackingEnabled(enabled = true) { withTempDir { tempDir => // Generate 2 files with base Row ID 0 and 20 resp. spark.range(start = 0, end = 20).toDF("_metadata").repartition(1) .write.format("delta").save(tempDir.getAbsolutePath) spark.range(start = 20, end = 30).toDF("_metadata").repartition(1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) val df = spark.read.format("delta").load(tempDir.getAbsolutePath).select("_metadata") val dfWithConflict = df .select( col("_metadata"), df.metadataColumn("_metadata") .getField({RowId.BASE_ROW_ID}) .as("real_base_row_id")) .where("real_base_row_id % 5 = 0") checkAnswer(dfWithConflict, (0 until 20).map(Row(_, 0)) ++ (20 until 30).map(Row(_, 20))) } } } test("Base Row IDs can be read through the Scala syntax") { withRowTrackingEnabled(enabled = true) { withTempDir { tempDir => // Generate 2 files with base Row ID 0 and 20 resp. spark.range(start = 0, end = 20).toDF("id").repartition(1) .write.format("delta").save(tempDir.getAbsolutePath) spark.range(start = 20, end = 30).toDF("id").repartition(1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) val df = spark.read.format("delta").load(tempDir.getAbsolutePath) .select("id", QUALIFIED_BASE_ROW_ID_COLUMN_NAME) checkAnswer(df, (0 until 20).map(Row(_, 0)) ++ (20 until 30).map(Row(_, 20))) } } } test("Base Row IDs can be read through the SQL syntax") { withRowTrackingEnabled(enabled = true) { withTempDir { tempDir => // Generate 2 files with base Row ID 0 and 20 resp. spark.range(start = 0, end = 20).toDF("id").repartition(1) .write.format("delta").save(tempDir.getAbsolutePath) spark.range(start = 20, end = 30).toDF("id").repartition(1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) val rows = sql( s""" |SELECT id, $QUALIFIED_BASE_ROW_ID_COLUMN_NAME FROM delta.`${tempDir.getAbsolutePath}` """.stripMargin ) checkAnswer(rows, (0 until 20).map(Row(_, 0)) ++ (20 until 30).map(Row(_, 20))) } } } test("Filter by base Row IDs") { withRowTrackingEnabled(enabled = true) { withTempDir { tempDir => // Generate 3 files with base Row ID 0, 10 and 20 resp. spark.range(start = 0, end = 10).toDF("id").repartition(1) .write.format("delta").save(tempDir.getAbsolutePath) spark.range(start = 10, end = 20).toDF("id").repartition(1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) spark.range(start = 20, end = 30).toDF("id").repartition(1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) val df = spark.read.format("delta").load(tempDir.getAbsolutePath) .where(col(QUALIFIED_BASE_ROW_ID_COLUMN_NAME) === 10) checkAnswer(df, (10 until 20).map(Row(_))) } } } test("Base Row IDs can be read in subquery") { withRowTrackingEnabled(enabled = true) { withTempDir { tempDir => // Generate 2 files with base Row ID 0 and 20 resp. spark.range(start = 0, end = 20).toDF("id").repartition(1) .write.format("delta").save(tempDir.getAbsolutePath) spark.range(start = 20, end = 30).toDF("id").repartition(1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) val rows = sql( s""" |SELECT * FROM delta.`${tempDir.getAbsolutePath}` |WHERE id IN ( | SELECT $QUALIFIED_BASE_ROW_ID_COLUMN_NAME | FROM delta.`${tempDir.getAbsolutePath}`) """.stripMargin) checkAnswer(rows, Seq(Row(0), Row(20))) } } } test("Filter by base Row IDs in subquery") { withRowTrackingEnabled(enabled = true) { withTempDir { tempDir => // Generate 2 files with base Row ID 0 and 20 resp. spark.range(start = 0, end = 20).toDF("id").repartition(1) .write.format("delta").save(tempDir.getAbsolutePath) spark.range(start = 20, end = 30).toDF("id").repartition(1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) val rows = sql( s""" |SELECT * FROM delta.`${tempDir.getAbsolutePath}` |WHERE id IN ( | SELECT id | FROM delta.`${tempDir.getAbsolutePath}` | WHERE $QUALIFIED_BASE_ROW_ID_COLUMN_NAME = 20) """.stripMargin) checkAnswer(rows, (20 until 30).map(Row(_))) } } } test("row ids cannot be read when they are disabled") { withRowTrackingEnabled(enabled = false) { withTempDir { dir => spark.range(start = 0, end = 1000, step = 1, numPartitions = 10) .write.format("delta").save(dir.getAbsolutePath) withAllParquetReaders { val err = intercept[AnalysisException] { spark.read.format("delta").load(dir.toString).select("_metadata.row_id").collect() } assert(err.getMessage.contains("No such struct field")) } } } } // Although readers don't have any row-id specific implementation, we still check that we // are able to read row IDs to check that we switch to a reader that supports row IDs if the // selected reader isn't able to. test("row ids can be read back") { withRowTrackingEnabled(enabled = true) { withAllParquetReaders { assertRowIdsCanBeReadWithRowGroupSkipping(start = 50) // Column mapping withSQLConf(DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey -> "name") { assertRowIdsCanBeRead(start = 100, numRows = 100) } } } } test("Can read both row id and row index") { withRowTrackingEnabled(enabled = true) { withAllParquetReaders { withTempDir { dir => val start = 10 val recordsPerFile = 5 spark.range(start = start, end = 20, step = 1, numPartitions = 2) .toDF("value") .write .format("delta") .save(dir.getAbsolutePath) val df1 = spark.read.format("delta").load(dir.getAbsolutePath) .select(RowId.QUALIFIED_COLUMN_NAME, "value", "_metadata.row_index") checkAnswer(df1, (0 until 10).map(i => Row(i, start + i, i % recordsPerFile))) } } } } test("Row ID metadata field has the expected type") { withRowTrackingEnabled(enabled = true) { withTempDir { tempDir => spark.range(start = 0, end = 20).toDF("id") .write.format("delta").save(tempDir.getAbsolutePath) val df = spark.read.format("delta").load(tempDir.getAbsolutePath) .select(RowId.QUALIFIED_COLUMN_NAME) val expectedRowIdMetadata = new MetadataBuilder() .putBoolean("__row_id_metadata_col", value = true) .build() val expectedRowIdField = StructField( RowId.ROW_ID, LongType, nullable = false, metadata = expectedRowIdMetadata) Seq(df.schema, df.queryExecution.analyzed.schema, df.queryExecution.optimizedPlan.schema) .foreach { schema => assert(schema === new StructType().add(expectedRowIdField)) } } } } test("Row IDs can be read in subquery") { withRowTrackingEnabled(enabled = true) { withTempDir { tempDir => // Generate 2 files with base Row ID 0 and 20 resp. spark.range(start = 0, end = 20).toDF("id").repartition(1) .write.format("delta").save(tempDir.getAbsolutePath) spark.range(start = 20, end = 30).toDF("id").repartition(1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) val rows = sql( s""" |SELECT * FROM delta.`${tempDir.getAbsolutePath}` |WHERE id IN ( | SELECT ${RowId.QUALIFIED_COLUMN_NAME} | FROM delta.`${tempDir.getAbsolutePath}`) """.stripMargin) checkAnswer(rows, (0 until 30).map(Row(_))) } } } test("Filter by Row IDs") { withRowTrackingEnabled(enabled = true) { withTempDir { tempDir => spark.range(start = 100, end = 110).toDF("id") .write.format("delta").save(tempDir.getAbsolutePath) val rows = spark.read.format("delta") .load(tempDir.getAbsolutePath).filter("_metadata.row_id % 2 = 0") checkAnswer(rows, (100.until(end = 110, step = 2)).map(Row(_))) } } } test("Filter by Row IDs in subquery") { withRowTrackingEnabled(enabled = true) { withTempDir { tempDir => // Generate 2 files with base Row ID 0 and 20 resp. spark.range(start = 0, end = 20).toDF("id").repartition(1) .write.format("delta").save(tempDir.getAbsolutePath) spark.range(start = 20, end = 30).toDF("id").repartition(1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) val rows = sql( s""" |SELECT * FROM delta.`${tempDir.getAbsolutePath}` |WHERE id IN ( | SELECT id | FROM delta.`${tempDir.getAbsolutePath}` | WHERE ${RowId.QUALIFIED_COLUMN_NAME} % 5 = 0) """.stripMargin) checkAnswer(rows, Seq(Row(0), Row(5), Row(10), Row(15), Row(20), Row(25))) } } } test("Row IDs cannot be read if the table property is not enabled") { withRowTrackingEnabled(enabled = true) { withAllParquetReaders { withTable("target") { spark.range(10).repartition(1).write.format("delta").saveAsTable("target") var df = spark.read.table("target") val expected = (0 until 10).map(i => Row(i, i)) // Check that row IDs can be read while table property is enabled checkAnswer(df.select("id", "_metadata.row_id"), expected) sql( s""" |ALTER TABLE target |SET TBLPROPERTIES ('${DeltaConfigs.ROW_TRACKING_ENABLED.key}' = false) |""".stripMargin) df = spark.read.format("delta").table("target") val err = intercept[AnalysisException] { checkAnswer(df.select("id", "_metadata.row_id"), expected) } assert(err.getMessage.contains("No such struct field")) // can still read other columns when table property disabled checkAnswer(df.select("id"), (0 until 10).map(Row(_))) } } } } test("No row-group skipping on _metadata.row_id") { withAllParquetReaders { withRowTrackingEnabled(enabled = true) { withTempPath { path => val numRows = ParquetProperties.DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK val materializedColName = "materialized_rowid_col" val df = spark.range(start = 0, end = numRows, step = 1, numPartitions = 1) .toDF("value") .withColumn(materializedColName, when(col("value") < (numRows / 2), col("value")) .otherwise(lit(null))) writeParquetWithMinimalRowGroupSize(df, path.toString) sql(s"CONVERT TO DELTA parquet.`$path`") setRowIdMaterializedColumnName( DeltaLog.forTable(spark, path), colName = materializedColName) checkFileLayout( path, numFiles = 1, numRowGroupsPerFile = 1, rowCountPerRowGroup = numRows) // Filter by row IDs that are not part of the materialized column. If we don't take fresh // row IDs into account, the row group will be skipped and the test will fail. val dfWithSkippingOnRowId = spark.read.format("delta").load(path.toString).select("value") .where(col(RowId.QUALIFIED_COLUMN_NAME) >= (numRows / 2)) checkAnswer(dfWithSkippingOnRowId, ((numRows / 2) until numRows).map(Row(_))) checkScanMetrics( dfWithSkippingOnRowId.queryExecution.executedPlan, expectedNumOfRows = numRows) } } } } test("No dictionary filtering on _metadata.row_id") { withAllParquetReaders { withRowTrackingEnabled(enabled = true) { withTempPath { path => val numRows = ParquetProperties.DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK val materializedColName = "materialized_rowid_col" val df = spark.range(start = 0, end = numRows, step = 1, numPartitions = 1) .toDF("value") .withColumn(materializedColName, // This will cause dictionary encoding to be used, as the column has few unique // values. Normally this shouldn't happen with row IDs, but we want to ensure that // we can still read row IDs correctly if dictionary encoding is used. when(col("value") > 0, lit(1L)) .otherwise(lit(null))) writeParquetWithMinimalRowGroupSize(df, path.toString) sql(s"CONVERT TO DELTA parquet.`$path`") setRowIdMaterializedColumnName( DeltaLog.forTable(spark, path), colName = materializedColName) checkFileLayout( path, numFiles = 1, numRowGroupsPerFile = 1, rowCountPerRowGroup = numRows) // We can't check directly whether dictionary filtering will take place, but we can ensure // that the row ID column is dictionary encoded, which should mean that the // optimization is applied. readRowGroupsPerFile(path).flatten.foreach { block => val rowIdColChunk = block.getColumns.asScala.find( _.getPath.asScala.exists(_ == materializedColName)).get assert(rowIdColChunk.getEncodings.contains(Encoding.PLAIN_DICTIONARY)) } // Filter by row IDs that are not part of the materialized column. If we don't take fresh // row IDs into account, the row group will be skipped and the test will fail. val dfWithSkippingOnRowId = spark.read.format("delta").load(path.toString).select("value") .where(col(RowId.QUALIFIED_COLUMN_NAME).equalTo(0)) checkAnswer(dfWithSkippingOnRowId, Row(0)) checkScanMetrics( dfWithSkippingOnRowId.queryExecution.executedPlan, expectedNumOfRows = numRows) } } } } test("Reading row IDs when file is split and splits are recombined") { withSQLConf( DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey -> "true", // 10 byte partition sizes SQLConf.FILES_MAX_PARTITION_BYTES.key -> "10B") { withTempDir { dir => spark.range(end = 10).repartition(1) // Add some more random columns, leads to multiple splits being recombined into a single // partition .selectExpr("id", "id as id2", "id as id3", "id as id4") .write.format("delta").save(dir.toString) val log = DeltaLog.forTable(spark, dir) // Make sure we would create at least two splits of a single file val necessarySplitSizeBytes = 20 assert(log.update().allFiles.collect().forall(_.size > necessarySplitSizeBytes)) checkAnswer( spark.read.format("delta").load(dir.toString).select("id", RowId.QUALIFIED_COLUMN_NAME), (0 until 10).map(i => Row(i, i))) } } } test("missing base row ids and default row commit versions") { val tableName = "my_table" withTable(tableName) { // Create a table with some rows without row tracking enabled. spark.range(start = 0, end = 10).repartition(1).sortWithinPartitions("id") .write.format("delta").mode("overwrite").saveAsTable(tableName) // Hack to enable row tracking without triggering a backfill. val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val snapshot = deltaLog.update() val actions = Seq( snapshot.metadata.copy( configuration = snapshot.metadata.configuration ++ Map( DeltaConfigs.ROW_TRACKING_ENABLED.key -> "true", MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP -> "x", MaterializedRowCommitVersion.MATERIALIZED_COLUMN_NAME_PROP -> "y" ) ) ) deltaLog.startTransaction().commit(actions, DeltaOperations.ManualUpdate) // Append some rows with base row ids and default row commit set on the files. spark.range(start = 10, end = 20).repartition(1).sortWithinPartitions("id") .write.format("delta").mode("append").saveAsTable(tableName) // Ensure that we cannot read the row id and row commit version by default. intercept[SparkException] { spark.read.table(tableName).select("id", RowId.QUALIFIED_COLUMN_NAME).collect() } intercept[SparkException] { spark.read.table(tableName).select("id", RowCommitVersion.QUALIFIED_COLUMN_NAME).collect() } // Create a dataframe that allows reading missing row ids and row commit versions. val originalPlan = spark.read.table(tableName).queryExecution.analyzed val transformedPlan = DeltaTableUtils.transformFileFormat(originalPlan) { case format => format.copy( nullableRowTrackingConstantFields = true, nullableRowTrackingGeneratedFields = true ) } val df = DataFrameUtils.ofRows(spark, transformedPlan) checkAnswer( df.select("id", RowId.QUALIFIED_COLUMN_NAME, RowCommitVersion.QUALIFIED_COLUMN_NAME), (0 until 10).map(i => Row(i, null, null)) ++ (0 until 10).map(i => Row(10 + i, i, 2)) ) } } protected def assertRowIdsCanBeRead(start: Int, numRows: Int): Unit = { withTempDir { dir => spark.range(start, end = start + numRows, step = 1, numPartitions = 3) .toDF("value") .write .format("delta") .save(dir.getAbsolutePath) val df1 = spark.read.format("delta").load(dir.getAbsolutePath) .select(RowId.QUALIFIED_COLUMN_NAME, "value") checkAnswer(df1, (0L until numRows).map(i => Row(i, start + i))) val df2 = spark.read.format("delta").load(dir.getAbsolutePath) .select("value", RowId.QUALIFIED_COLUMN_NAME) checkAnswer(df2, (0L until numRows).map(i => Row(start + i, i))) } } protected def writeParquetWithMinimalRowGroupSize(df: DataFrame, path: String): Unit = { df.write .format("parquet") // The minimum row count in a row group is // `ParquetProperties.DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK`, if we specify a // block size that can't accommodate the minimum row count, we'll write exactly // the minimum row count per row group. .option(ParquetOutputFormat.BLOCK_SIZE, 0) .save(path) } protected def setRowIdMaterializedColumnName(log: DeltaLog, colName: String): Unit = { val metadata = log.update().metadata val configWithUpdatedRowIdColName = metadata.configuration + ( MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP -> colName) // We need to remove the column from the schema as we are not allowed to have the // materialized row ID column be part of the schema. val schemaFieldsWithoutRowIdCol = metadata.schema.filterNot(_.name == colName) val updatedMetadata = metadata.copy( configuration = configWithUpdatedRowIdColName, schemaString = metadata.schema.copy(fields = schemaFieldsWithoutRowIdCol.toArray).json) log.startTransaction().commit(Seq(updatedMetadata), DeltaOperations.ManualUpdate) } protected def checkScanMetrics(plan: SparkPlan, expectedNumOfRows: Long): Unit = { var numOutputRows = 0L plan.foreach { case f: FileSourceScanExec => numOutputRows += f.metrics("numOutputRows").value case _ => // Not a scan node, do nothing. } assert(expectedNumOfRows === numOutputRows) } private def assertRowIdsCanBeReadWithRowGroupSkipping(start: Int): Unit = { val rowGroupRowCount = ParquetProperties.DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK // write at least two row groups val numRows = rowGroupRowCount * 2 withTempPath { path => val df = spark.range(start, end = start + numRows, step = 1, numPartitions = 1).toDF("value") writeParquetWithMinimalRowGroupSize(df, path.toString) sql(s"CONVERT TO DELTA parquet.`$path`") import testImplicits._ checkFileLayout( path, numFiles = 1, numRowGroupsPerFile = 2, rowCountPerRowGroup = rowGroupRowCount) val rowGroups = readRowGroupsPerFile(path).head val minValueSecondRowGroup = rowGroups(1).getColumns.get(0).getStatistics.genericGetMin() val df1 = spark.read.format("delta").load(path.getAbsolutePath) .filter($"value" >= minValueSecondRowGroup) .select(RowId.QUALIFIED_COLUMN_NAME, "value") checkAnswer(df1, (rowGroupRowCount until numRows).map(i => Row(i, start + i))) checkScanMetrics(df1.queryExecution.executedPlan, expectedNumOfRows = rowGroupRowCount) val df2 = spark.read.format("delta").load(path.getAbsolutePath) .filter($"value" >= minValueSecondRowGroup) .select("value", RowId.QUALIFIED_COLUMN_NAME) checkAnswer(df2, (rowGroupRowCount until numRows).map(i => Row(start + i, i))) checkScanMetrics(df2.queryExecution.executedPlan, expectedNumOfRows = rowGroupRowCount) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowIdTestUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.rowid import java.io.File import scala.collection.JavaConverters._ import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.{AddFile, CommitInfo, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.commands.backfill.{BackfillBatchStats, BackfillCommandStats} import org.apache.spark.sql.delta.rowtracking.RowTrackingTestUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.JsonUtils import org.apache.hadoop.fs.Path import org.apache.parquet.hadoop.metadata.BlockMetaData import org.apache.spark.sql.{Column, DataFrame} import org.apache.spark.sql.execution.datasources.FileFormat trait RowIdTestUtils extends RowTrackingTestUtils with DeltaSQLCommandTest { val QUALIFIED_BASE_ROW_ID_COLUMN_NAME = s"${FileFormat.METADATA_NAME}.${RowId.BASE_ROW_ID}" protected def getRowIdRangeInclusive(f: AddFile): (Long, Long) = { val min = f.baseRowId.get val max = min + f.numPhysicalRecords.get - 1L (min, max) } def assertRowIdsDoNotOverlap(log: DeltaLog): Unit = { val files = log.update().allFiles.collect() val sortedRanges = files .map(f => (f.path, getRowIdRangeInclusive(f))) .sortBy { case (_, (min, _)) => min } for (i <- sortedRanges.indices.dropRight(1)) { val (curPath, (_, curMax)) = sortedRanges(i) val (nextPath, (nextMin, _)) = sortedRanges(i + 1) assert(curMax < nextMin, s"$curPath and $nextPath have overlapping row IDs") } } def assertHighWatermarkIsCorrect(log: DeltaLog): Unit = { val snapshot = log.update() val files = snapshot.allFiles.collect() val highWatermarkOpt = RowId.extractHighWatermark(snapshot) if (files.isEmpty) { assert(highWatermarkOpt.isDefined) } else { val maxAssignedRowId = files .map(a => a.baseRowId.get + a.numPhysicalRecords.get - 1L) .max assert(highWatermarkOpt.get == maxAssignedRowId) } } def assertRowIdsAreValid(log: DeltaLog): Unit = { assertRowIdsDoNotOverlap(log) assertHighWatermarkIsCorrect(log) } def assertHighWatermarkIsCorrectAfterUpdate( log: DeltaLog, highWatermarkBeforeUpdate: Long, expectedNumRecordsWritten: Long): Unit = { val highWaterMarkAfterUpdate = RowId.extractHighWatermark(log.update()).get assert((highWatermarkBeforeUpdate + expectedNumRecordsWritten) === highWaterMarkAfterUpdate) assertRowIdsAreValid(log) } def assertRowIdsAreNotSet(log: DeltaLog): Unit = { val snapshot = log.update() val highWatermarks = RowId.extractHighWatermark(snapshot) assert(highWatermarks.isEmpty) val files = snapshot.allFiles.collect() assert(files.forall(_.baseRowId.isEmpty)) } def assertRowIdsAreLargerThanValue(log: DeltaLog, value: Long): Unit = { log.update().allFiles.collect().foreach { f => val minRowId = getRowIdRangeInclusive(f)._1 assert(minRowId > value, s"${f.toString} has a row id smaller or equal than $value") } } def extractMaterializedRowIdColumnName(log: DeltaLog): Option[String] = { log.update().metadata.configuration.get(MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP) } protected def readRowGroupsPerFile(dir: File): Seq[Seq[BlockMetaData]] = { assert(dir.isDirectory) readAllFootersWithoutSummaryFiles( // scalastyle:off deltahadoopconfiguration new Path(dir.getAbsolutePath), spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration .map(_.getParquetMetadata.getBlocks.asScala.toSeq) } protected def checkFileLayout( dir: File, numFiles: Int, numRowGroupsPerFile: Int, rowCountPerRowGroup: Int): Unit = { val rowGroupsPerFile = readRowGroupsPerFile(dir) assert(numFiles === rowGroupsPerFile.size) for (rowGroups <- rowGroupsPerFile) { assert(numRowGroupsPerFile === rowGroups.size) for (rowGroup <- rowGroups) { assert(rowCountPerRowGroup === rowGroup.getRowCount) } } } // easily add a rowid column to a dataframe by calling [[df.withMaterializedRowIdColumn]] implicit class DataFrameRowIdColumn(df: DataFrame) { def withMaterializedRowIdColumn( materializedColumnName: String, rowIdColumn: Column): DataFrame = RowId.preserveRowIdsUnsafe( df, materializedColumnName, rowIdColumn, shouldSetIcebergReservedFieldId = false) def withMaterializedRowCommitVersionColumn( materializedColumnName: String, rowCommitVersionColumn: Column): DataFrame = RowCommitVersion.preserveRowCommitVersionsUnsafe( df, materializedColumnName, rowCommitVersionColumn, shouldSetIcebergReservedFieldId = false) } def extractMaterializedRowCommitVersionColumnName(log: DeltaLog): Option[String] = { log.update().metadata.configuration .get(MaterializedRowCommitVersion.MATERIALIZED_COLUMN_NAME_PROP) } /** Returns a Map of file path to base row ID from the AddFiles in a Snapshot. */ private def getAddFilePathToBaseRowIdMap(snapshot: Snapshot): Map[String, Long] = { val allAddFiles = snapshot.allFiles.collect() allAddFiles.foreach(addFile => assert(addFile.baseRowId.isDefined, "Every AddFile should have a base row ID")) allAddFiles.map(a => a.path -> a.baseRowId.get).toMap } /** Returns a Map of file path to base row ID from the RemoveFiles in a Snapshot. */ private def getRemoveFilePathToBaseRowIdMap(snapshot: Snapshot): Map[String, Long] = { val removeFiles = snapshot.tombstones.collect() removeFiles.foreach(removeFile => assert(removeFile.baseRowId.isDefined, "Every RemoveFile should have a base row ID")) removeFiles.map(r => r.path -> r.baseRowId.get).toMap } /** Check that the high watermark does not get updated if there aren't any new files */ def checkHighWatermarkBeforeAndAfterOperation(log: DeltaLog)(operation: => Unit): Unit = { val prevSnapshot = log.update() val prevHighWatermark = RowId.extractHighWatermark(prevSnapshot) val prevAddFiles = getAddFilePathToBaseRowIdMap(prevSnapshot).keySet operation val newAddFiles = getAddFilePathToBaseRowIdMap(log.update()).keySet val newFilesAdded = newAddFiles.diff(prevAddFiles).nonEmpty val newHighWatermark = RowId.extractHighWatermark(log.update()) if (newFilesAdded) { assert(prevHighWatermark.get < newHighWatermark.get, "The high watermark should have been updated after creating new files") } else { assert(prevHighWatermark === newHighWatermark, "The high watermark should not be updated when there are no new file") } } /** * Check that file actions do not violate Row ID invariants after an operation. * More specifically: * - We do not reassign the base row ID to the same AddFile. * - RemoveFiles have the same base row ID as the corresponding AddFile * with the same file path. */ def checkFileActionInvariantBeforeAndAfterOperation(log: DeltaLog)(operation: => Unit): Unit = { val prevAddFilePathToBaseRowId = getAddFilePathToBaseRowIdMap(log.update()) operation val snapshot = log.update() val newAddFileBaseRowIdsMap = getAddFilePathToBaseRowIdMap(snapshot) val newRemoveFileBaseRowIds = getRemoveFilePathToBaseRowIdMap(snapshot) prevAddFilePathToBaseRowId.foreach { case (path, prevRowId) => if (newAddFileBaseRowIdsMap.contains(path)) { val currRowId = newAddFileBaseRowIdsMap(path) assert(currRowId === prevRowId, "We should not reassign base row IDs if it's the same AddFile") } else if (newRemoveFileBaseRowIds.contains(path)) { assert(newRemoveFileBaseRowIds(path) === prevRowId, "No new base row ID should be assigned to RemoveFiles") } } } /** * Checks whether Row tracking is marked as preserved on the [[CommitInfo]] action * committed during `operation`. */ def rowTrackingMarkedAsPreservedForCommit(log: DeltaLog)(operation: => Unit): Boolean = { val versionPriorToCommit = log.update().version operation val versionOfCommit = log.update().version assert(versionPriorToCommit < versionOfCommit) val commitInfos = log.getChanges(versionOfCommit).flatMap(_._2).flatMap { case commitInfo: CommitInfo => Some(commitInfo) case _ => None }.toList assert(commitInfos.size === 1) commitInfos.forall { commitInfo => commitInfo.tags .getOrElse(Map.empty) .getOrElse(DeltaCommitTag.PreservedRowTrackingTag.key, "false").toBoolean } } def checkRowTrackingMarkedAsPreservedForCommit(log: DeltaLog)(operation: => Unit): Unit = { assert(rowTrackingMarkedAsPreservedForCommit(log)(operation)) } /** * Capture backfill related metrics for basic validation. */ def validateSuccessfulBackfillMetrics( expectedNumSuccessfulBatches: Int, nameOfTriggeringOperation: String = DeltaOperations.OP_SET_TBLPROPERTIES) (testBlock: => Unit): Unit = { val backfillUsageRecords = Log4jUsageLogger.track { testBlock }.filter(_.metric == "tahoeEvent") val backfillRecords = backfillUsageRecords .filter(_.tags.get("opType").contains(DeltaUsageLogsOpTypes.BACKFILL_COMMAND)) assert(backfillRecords.size === 1, "Row Tracking Backfill should have " + "only been executed once.") val backfillStats = JsonUtils.fromJson[BackfillCommandStats](backfillRecords.head.blob) assert(backfillStats.wasSuccessful) assert(backfillStats.numFailedBatches === 0) assert(backfillStats.totalExecutionTimeMs > 0) assert(backfillStats.numSuccessfulBatches === expectedNumSuccessfulBatches) assert(backfillStats.nameOfTriggeringOperation === nameOfTriggeringOperation) val parentTxnId = backfillStats.transactionId val backfillBatchRecords = backfillUsageRecords .filter(_.tags.get("opType").contains(DeltaUsageLogsOpTypes.BACKFILL_BATCH)) val backfillBatchStats = backfillBatchRecords.map { backfillBatchRecord => JsonUtils.fromJson[BackfillBatchStats](backfillBatchRecord.blob) } // Sanity check that the individual child commits were successful. backfillBatchStats.foreach { backfillBatchStat => assert(backfillBatchStat.wasSuccessful) assert(backfillBatchStat.totalExecutionTimeInMs > 0) assert(backfillBatchStat.initialNumFiles > 0) assert(backfillBatchStat.parentTransactionId === parentTxnId) } } /** * This triggers backfill on the test table in this suite by calling the user-facing syntax * `ALTER TABLE t SET TBLPROPERTIES()`. We check for proper protocol upgrade (if any) and * that the table has valid row IDs afterwards. */ def triggerBackfillOnTestTableUsingAlterTable( targetTableName: String, numRowsInTable: Int, log: DeltaLog): Unit = { val prevMinReaderVersion = log.update().protocol.minReaderVersion val prevMinWriterVersion = log.update().protocol.minWriterVersion val rowIdPropertyKey = DeltaConfigs.ROW_TRACKING_ENABLED.key spark.sql(s"ALTER TABLE $targetTableName SET TBLPROPERTIES ('$rowIdPropertyKey'=true)") assert(lastCommitHasRowTrackingEnablementOnlyTag(log)) // Check the protocol upgrade is as expected. We should only bump the minWriterVersion if // necessary and add the table feature support for row IDs. val snapshot = log.update() val newProtocol = snapshot.protocol assert(newProtocol.isFeatureSupported(RowTrackingFeature)) assert(newProtocol.minReaderVersion === prevMinReaderVersion, "The reader version does not need to be upgraded") val expectedMinWriterVersion = Math.max( prevMinWriterVersion, TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION) assert(newProtocol.minWriterVersion === expectedMinWriterVersion) // Tables should have the table property enabled at the end of ALTER TABLE command. assert(RowId.isEnabled(newProtocol, snapshot.metadata)) val highWaterMarkBefore = -1L assertRowIdsAreValid(log) assertRowIdsAreLargerThanValue(log, highWaterMarkBefore) assertHighWatermarkIsCorrectAfterUpdate(log, highWaterMarkBefore, numRowsInTable) } /** * Returns a Boolean indicating whether the last commit on a Delta table has the tag * [[DeltaCommitTag.RowTrackingEnablementOnlyTag.key]]. */ def lastCommitHasRowTrackingEnablementOnlyTag(log: DeltaLog): Boolean = { val lastTableVersion = log.update().version val (_, lastCommitActions) = log.getChanges(lastTableVersion).toList.last val findRowTrackingEnablementOnlyTag = lastCommitActions.collectFirst { case commitInfo: CommitInfo => DeltaCommitTag.getTagValueFromCommitInfo( Some(commitInfo), DeltaCommitTag.RowTrackingEnablementOnlyTag.key) }.flatten findRowTrackingEnablementOnlyTag.exists(_.toBoolean) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowTrackingBackfillSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.rowid import java.util.concurrent.atomic.AtomicInteger import scala.concurrent.ExecutionException import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.actions.{AddFile, Protocol, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.AlterTableSetPropertiesDeltaCommand import org.apache.spark.sql.delta.commands.backfill.{BackfillBatch, BackfillBatchStats, BackfillCommandStats, RowTrackingBackfillBatch, RowTrackingBackfillCommand, RowTrackingBackfillExecutor} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.SparkConf import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.test.SharedSparkSession /** Test-only object that overrides a method to force a failure. */ case class FailingRowTrackingBackfillBatch(filesInBatch: Seq[AddFile]) extends BackfillBatch { override val backfillBatchStatsOpType = DeltaUsageLogsOpTypes.BACKFILL_BATCH override def prepareFilesAndCommit( spark: SparkSession, txn: OptimisticTransaction, batchId: Int): Long = { throw new IllegalStateException("mock exception for test") } } class RowTrackingBackfillSuite extends RowIdTestUtils with SharedSparkSession { protected val initialNumRows = 1000 protected val testTableName = "target" protected val numFilesInTable = 10 override def sparkConf: SparkConf = super.sparkConf .set(DeltaSQLConf.DELTA_ROW_TRACKING_BACKFILL_ENABLED.key, "true") protected def createTable(tableName: String): Unit = { // We disable Optimize Write to ensure the right number of files are created. withSQLConf( DeltaSQLConf.DELTA_OPTIMIZE_WRITE_ENABLED.key -> "false" ) { spark.range(start = 0, end = initialNumRows, step = 1, numPartitions = numFilesInTable) .write .format("delta") .saveAsTable(tableName) val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName)) assert(snapshot.allFiles.count() === numFilesInTable) } } /** Create the default test table used by this suite, which has row IDs disabled. */ protected def withTestTableWithNoRowTracking()(f: => Unit): Unit = { withRowTrackingEnabled(enabled = false) { withTable(testTableName) { createTable(testTableName) val (log, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(testTableName)) assertRowIdsAreNotSet(log) assert(!RowTracking.isSupported(snapshot.protocol)) assert(!RowId.isEnabled(snapshot.protocol, snapshot.metadata)) f } } } protected def withTestTableWithRowTrackingDisabled()(f: => Unit): Unit = { withTable(testTableName) { // Do not create baseRowIds. withRowTrackingEnabled(enabled = false) { createTable(testTableName) } val log = DeltaLog.forTable(spark, TableIdentifier(testTableName)) AlterTableSetPropertiesDeltaCommand( table = DeltaTableV2(spark, log.dataPath), configuration = Map( s"delta.feature.${RowTrackingFeature.name}" -> "supported", DeltaConfigs.ROW_TRACKING_ENABLED.key -> "false")) .run(spark) val snapshot = log.update() assert(RowTracking.isSupported(snapshot.protocol)) assert(!RowId.isEnabled(snapshot.protocol, snapshot.metadata)) f } } /** Check the number of backfill commits in the Delta history. */ protected def assertNumBackfillCommits(expectedNumCommits: Int): Unit = { val log = DeltaLog.forTable(spark, TableIdentifier(testTableName)) val actualNumBackfillCommits = log.history.getHistory(None) .filter(_.operation == DeltaOperations.ROW_TRACKING_BACKFILL_OPERATION_NAME) assert(actualNumBackfillCommits.size === expectedNumCommits) } /** Check the protocol, the number of backfill commits and the table property. */ protected def assertBackfillWasSuccessful(expectedNumCommits: Int): Unit = { val (log, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(testTableName)) assert(RowTracking.isSupported(snapshot.protocol)) assertNumBackfillCommits(expectedNumCommits) assert(RowId.isEnabled(snapshot.protocol, snapshot.metadata)) } test("getCandidateFilesToBackfill returns right files on tables with row IDs disabled") { withTestTableWithNoRowTracking() { val (log, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(testTableName)) val initialFilesInTable = snapshot.allFiles.collect().toSet assert(initialFilesInTable.size === numFilesInTable, "We expect the AddFiles to be unique in this table.") // ALl files in a table with row ID disabled should require backfill. val filesToBackfillBeforeTableFeatureSupport = RowTrackingBackfillExecutor.getCandidateFilesToBackfill(log.update()).collect() assert(filesToBackfillBeforeTableFeatureSupport.toSet === initialFilesInTable) // Let's add table feature support without enabling row ID, i.e., force new files to have // base row IDs. This is not the same as enabling the table property. This does not trigger // backfill commits. sql( s"""ALTER TABLE $testTableName |SET TBLPROPERTIES('$rowTrackingFeatureName' = 'supported')""".stripMargin) val snapshotAfterTableSupport = log.update() assert(RowTracking.isSupported(snapshotAfterTableSupport.protocol)) assert(!RowId.isEnabled( snapshotAfterTableSupport.protocol, snapshotAfterTableSupport.metadata)) spark.range(end = 1).write.mode("append").insertInto(testTableName) val snapshotAfterInsert = log.update() assert(snapshotAfterInsert.allFiles.count() === numFilesInTable + 1) // Only the files before the table feature support should need backfill. val filesToBackfillAfterTableFeatureSupport = RowTrackingBackfillExecutor.getCandidateFilesToBackfill(snapshotAfterInsert).collect() assert(filesToBackfillAfterTableFeatureSupport.toSet === initialFilesInTable) } } test("Trigger backfill by calling command directly") { // No one should be calling this directly. We just want to unit test outside of ALTER TABLE. withTestTableWithNoRowTracking() { val log = DeltaLog.forTable(spark, TableIdentifier(testTableName)) RowTrackingBackfillCommand( log, nameOfTriggeringOperation = DeltaOperations.OP_SET_TBLPROPERTIES, None).run(spark) val snapshot = log.update() assert(RowTracking.isSupported(snapshot.protocol)) assert(!RowId.isEnabled(snapshot.protocol, snapshot.metadata)) assertNumBackfillCommits(expectedNumCommits = 1) } } test("Calling the command directly should not trigger backfill when " + "Row Tracking Backfill is not enabled") { withSQLConf(DeltaSQLConf.DELTA_ROW_TRACKING_BACKFILL_ENABLED.key -> "false") { // No one should be calling this directly. We just want to unit test outside of ALTER TABLE. withTestTableWithNoRowTracking() { val log = DeltaLog.forTable(spark, TableIdentifier(testTableName)) val ex = intercept[UnsupportedOperationException] { RowTrackingBackfillCommand( log, nameOfTriggeringOperation = DeltaOperations.OP_SET_TBLPROPERTIES, None).run(spark) } assert(ex.getMessage === "Cannot enable Row IDs on an existing table.") } } } test("Trigger backfill using ALTER TABLE") { withTestTableWithNoRowTracking() { val log = DeltaLog.forTable(spark, TableIdentifier(testTableName)) validateSuccessfulBackfillMetrics(expectedNumSuccessfulBatches = 1) { triggerBackfillOnTestTableUsingAlterTable(testTableName, initialNumRows, log) } assertBackfillWasSuccessful(expectedNumCommits = 1) val snapshot = log.update() assertRowIdsAreValid(log) assert(RowTracking.isSupported(snapshot.protocol)) assert(RowId.isEnabled(snapshot.protocol, snapshot.metadata)) val prevHighWatermark = RowId.extractHighWatermark(log.update()).get // Commits after the backfill command should have row IDs. val numNewRows = 100 spark.range(end = numNewRows).write.insertInto(testTableName) assertRowIdsAreValid(log) assertHighWatermarkIsCorrectAfterUpdate(log, prevHighWatermark, numNewRows) } } test("Backfill respects the max file limit per commit") { val maxNumFilesPerCommit = 3 withTestTableWithNoRowTracking() { withSQLConf( DeltaSQLConf.DELTA_BACKFILL_MAX_NUM_FILES_PER_COMMIT.key -> maxNumFilesPerCommit.toString) { val expectedNumBackfillCommits = Math.ceil(numFilesInTable.toDouble / maxNumFilesPerCommit.toDouble).toInt validateSuccessfulBackfillMetrics(expectedNumBackfillCommits) { val log = DeltaLog.forTable(spark, TableIdentifier(testTableName)) triggerBackfillOnTestTableUsingAlterTable(testTableName, initialNumRows, log) } assertBackfillWasSuccessful(expectedNumBackfillCommits) } } } test("Backfill on table with row tracking already enabled") { withRowTrackingEnabled(enabled = true) { withTable(testTableName) { createTable(testTableName) val (log, snapshot1) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(testTableName)) var snapshot = snapshot1 assert(RowTracking.isSupported(snapshot.protocol)) assert(RowId.isEnabled(snapshot.protocol, snapshot.metadata)) // ALTER TABLE should do nothing other than set the table properties. validateSuccessfulBackfillMetrics(expectedNumSuccessfulBatches = 0) { triggerBackfillOnTestTableUsingAlterTable(testTableName, initialNumRows, log) assertNumBackfillCommits(expectedNumCommits = 0) } // Now, let's test UNSET TBLPROPERTIES val rowTrackingKey = DeltaConfigs.ROW_TRACKING_ENABLED.key sql(s"ALTER TABLE $testTableName UNSET TBLPROPERTIES('$rowTrackingKey')") // This should not downgrade the table and it should not trigger any backfill. // It will only change the table property. snapshot = log.update() assertNumBackfillCommits(expectedNumCommits = 0) assert(RowTracking.isSupported(snapshot.protocol)) assert(!RowId.isEnabled(snapshot.protocol, snapshot.metadata)) val prevMaterializedColumnName = extractMaterializedRowIdColumnName(log) // If we re-enable the table property again, we should expect the materialized column name // to be the same. validateSuccessfulBackfillMetrics(expectedNumSuccessfulBatches = 0) { triggerBackfillOnTestTableUsingAlterTable(testTableName, initialNumRows, log) } assert(prevMaterializedColumnName === extractMaterializedRowIdColumnName(log)) } } } test("Backfill on table that enabled the table feature separately") { withTestTableWithNoRowTracking() { val log = DeltaLog.forTable(spark, TableIdentifier(testTableName)) // User decides to enable the table feature separately first. sql( s"""ALTER TABLE $testTableName |SET TBLPROPERTIES('$rowTrackingFeatureName' = 'supported')""".stripMargin) val snapshotAfterTableSupport = log.update() assert(RowTracking.isSupported(snapshotAfterTableSupport.protocol)) assert(!RowId.isEnabled( snapshotAfterTableSupport.protocol, snapshotAfterTableSupport.metadata)) val deltaHistory = log.history.getHistory(None) // 1 commit to create table, 1 commit to upgrade protocol assert(deltaHistory.size === 2) // New data is inserted into the table. The new files should have base row ID. val numNewRowsInserted = 1 spark.range(end = numNewRowsInserted).write.mode("append").insertInto(testTableName) val snapshotAfterInsert = log.update() assert(snapshotAfterInsert.allFiles.count() === numFilesInTable + 1) val numRowsInTableAfterInsert = initialNumRows + numNewRowsInserted // Trigger backfill. Only the files before the table feature support should need backfill. validateSuccessfulBackfillMetrics(expectedNumSuccessfulBatches = 1) { triggerBackfillOnTestTableUsingAlterTable(testTableName, numRowsInTableAfterInsert, log) assertBackfillWasSuccessful(expectedNumCommits = 1) } // We should not try to upgrade the protocol again. val deltaHistory2 = log.history.getHistory(None) // We should have 3 more commits since the protocol upgrade: // 1 commit to insert, 1 commit to backfill, 1 commit to set tbl properties. assert(deltaHistory2.size === 5) assertRowIdsAreValid(log) } } test("Backfill should be idempotent") { withTestTableWithNoRowTracking() { val log = DeltaLog.forTable(spark, TableIdentifier(testTableName)) validateSuccessfulBackfillMetrics(expectedNumSuccessfulBatches = 1) { triggerBackfillOnTestTableUsingAlterTable(testTableName, initialNumRows, log) } val snapshot = log.update() assert(RowId.isEnabled(snapshot.protocol, snapshot.metadata)) val materializedColumnName = extractMaterializedRowIdColumnName(log) val deltaHistory = log.history.getHistory(None) // 1 commit to create table, 1 commit to upgrade protocol, 1 commit for Backfill, // 1 commit to set row tracking to enabled. assert(deltaHistory.size === 4) // We should not upgrade the protocol again and we should not backfill. validateSuccessfulBackfillMetrics(expectedNumSuccessfulBatches = 0) { triggerBackfillOnTestTableUsingAlterTable(testTableName, initialNumRows, log) } assert(extractMaterializedRowIdColumnName(log) === materializedColumnName, "the materialized column name should not change") val deltaHistory2 = log.history.getHistory(None) // 1 more commit to SET TBLPROPERTIES, nothing else. assert(deltaHistory2.size === 5) assert(deltaHistory2.head.operation === DeltaOperations.OP_SET_TBLPROPERTIES) } } test("ALTER TABLE that don't enable row tracking should not backfill") { withTestTableWithNoRowTracking() { val rowTrackingKey = DeltaConfigs.ROW_TRACKING_ENABLED.key sql(s"ALTER TABLE $testTableName SET TBLPROPERTIES('$rowTrackingKey' = false)") // This should not upgrade the table protocol and it should not trigger any backfill. // It will only set the table property. val (log, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(testTableName)) assertNumBackfillCommits(expectedNumCommits = 0) assert(!RowTracking.isSupported(snapshot.protocol)) assert(!RowId.isEnabled(snapshot.protocol, snapshot.metadata)) assert(!lastCommitHasRowTrackingEnablementOnlyTag(log), "RowTrackingEnablementOnly tag should not be set if the table property value is false") } } test("Trigger backfill using ALTER TABLE, but another property fails") { withTestTableWithNoRowTracking() { // Enable column mapping val columnMappingMode = DeltaConfigs.COLUMN_MAPPING_MODE.key val minReaderKey = DeltaConfigs.MIN_READER_VERSION.key val minWriterKey = DeltaConfigs.MIN_WRITER_VERSION.key val rowTrackingKey = DeltaConfigs.ROW_TRACKING_ENABLED.key sql(s"""ALTER TABLE $testTableName SET TBLPROPERTIES( |'$minReaderKey' = '2', |'$minWriterKey' = '5', |'$columnMappingMode'='name')""".stripMargin) val log = DeltaLog.forTable(spark, TableIdentifier(testTableName)) assert(!lastCommitHasRowTrackingEnablementOnlyTag(log), "RowTrackingEnablementOnly tag should not be set for other table properties") // Try to enable row IDs at the same time as we set column mapping mode to id. // This should fail due to illegal column mapping mode change. intercept[ColumnMappingUnsupportedException] { sql(s"ALTER TABLE $testTableName SET " + s"TBLPROPERTIES('$columnMappingMode'='id', '$rowTrackingKey'=true)") } // Despite the failure, there are side effects: the protocol is still upgraded and // backfill commits occurred. However, row IDs should still be disabled because we were unable // to set the property. val snapshot = log.update() assert(RowTracking.isSupported(snapshot.protocol)) assertNumBackfillCommits(expectedNumCommits = 1) assert(!RowId.isEnabled(snapshot.protocol, snapshot.metadata)) } } /** * Validate that if a user triggers backfill with other protocol changes within the ALTER TABLE * command, we end up with a correct final protocol object. * * @param tableBelowTableFeatureLevel : Boolean indicating whether the test table has a protocol * writer version below table feature support prior to calling * the ALTER TABLE command that triggers backfill. * @param isOtherTableFeatureLegacy : Boolean indicating whether the other table feature being * supported along row tracking enablement is legacy (i.e. it * requires minWriterVersion and minReaderVersion below table * feature level). */ private def checkProtocolUpgradeWithBackfill( tableBelowTableFeatureLevel: Boolean, isOtherTableFeatureLegacy: Boolean): Unit = { val initialMinReaderVersion = ColumnMappingTableFeature.minReaderVersion val initialMinWriterVersion = ColumnMappingTableFeature.minWriterVersion withSQLConf( DeltaConfigs.MIN_WRITER_VERSION.defaultTablePropertyKey -> initialMinWriterVersion.toString, DeltaConfigs.MIN_READER_VERSION.defaultTablePropertyKey -> initialMinReaderVersion.toString ) { withTestTableWithNoRowTracking() { val (log, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(testTableName)) val prevProtocol = snapshot.protocol val expectedInitialProtocol = Protocol(initialMinReaderVersion, initialMinWriterVersion) assert(prevProtocol === expectedInitialProtocol) // Build the TBLPROPERTIES String for the ALTER TABLE command. val rowTrackingKey = DeltaConfigs.ROW_TRACKING_ENABLED.key var propertiesMap: Map[String, String] = Map(rowTrackingKey -> "true") if (isOtherTableFeatureLegacy) { val columnMappingMode = DeltaConfigs.COLUMN_MAPPING_MODE.key propertiesMap = propertiesMap ++ Map(columnMappingMode -> "name") } else { val deletionVectorsKey = DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key propertiesMap = propertiesMap ++ Map(deletionVectorsKey -> "true") } val tblPropertiesString = propertiesMap.map { case (key, value) => s"'$key'='$value'" }.mkString(", ") sql(s"ALTER TABLE $testTableName SET TBLPROPERTIES($tblPropertiesString)") assert(!lastCommitHasRowTrackingEnablementOnlyTag(log), "RowTrackingEnablementOnly tag should not be set if ALTER TABLE is changing" + " multiple table properties") val expectedFinalProtocol = if (isOtherTableFeatureLegacy) { Protocol( minReaderVersion = ColumnMappingTableFeature.minReaderVersion, minWriterVersion = TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(RowTrackingFeature) .withFeature(ColumnMappingTableFeature) .merge(prevProtocol) } else { Protocol( minReaderVersion = TableFeatureProtocolUtils.TABLE_FEATURES_MIN_READER_VERSION, minWriterVersion = TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(RowTrackingFeature) .withFeature(DeletionVectorsTableFeature) .merge(prevProtocol) } assertBackfillWasSuccessful(expectedNumCommits = 1) val finalSnapshot = log.update() assert(finalSnapshot.protocol === expectedFinalProtocol) } } } for { tableBelowTableFeatureLevel <- BOOLEAN_DOMAIN isOtherTableFeatureLegacy <- BOOLEAN_DOMAIN } { test("ALTER TABLE does other protocol upgrade with backfill, " + s"tableBelowTableFeatureLevel=$tableBelowTableFeatureLevel, " + s"isOtherTableFeatureLegacy=$isOtherTableFeatureLegacy") { checkProtocolUpgradeWithBackfill(tableBelowTableFeatureLevel, isOtherTableFeatureLegacy) } } test("lower MIN_WRITER_VERSION along with Row ID prop can upgrade protocol") { withTestTableWithNoRowTracking() { val (log, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(testTableName)) val prevProtocol = snapshot.protocol assert(prevProtocol.minWriterVersion < TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION) // Try to enable row IDs at the same time as we enable column mapping mode. val columnMappingMode = DeltaConfigs.COLUMN_MAPPING_MODE.key val minReaderKey = DeltaConfigs.MIN_READER_VERSION.key val minWriterKey = DeltaConfigs.MIN_WRITER_VERSION.key val rowTrackingKey = DeltaConfigs.ROW_TRACKING_ENABLED.key // If we specify lower minWriterKey along with the row tracking prop, we will // do an implicit upgrade to minWriterKey = 7. sql( s"""ALTER TABLE $testTableName SET TBLPROPERTIES( |'$minReaderKey' = '2', |'$minWriterKey' = '5', |'$columnMappingMode'='name', |'$rowTrackingKey'=true |)""".stripMargin) val afterProtocol = log.update().protocol assert(afterProtocol.minReaderVersion === 2) assert( afterProtocol.minWriterVersion === TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION) assert(afterProtocol.readerFeatures === None) assert( afterProtocol.writerFeatures === Some(( prevProtocol.implicitlyAndExplicitlySupportedFeatures ++ Protocol(2, 5).implicitlySupportedFeatures ++ Set( ColumnMappingTableFeature, DomainMetadataTableFeature, // Required by Row Tracking RowTrackingFeature)) .map(_.name))) } } test("BackfillCommandStats metrics are correct in case of failure") { withTestTableWithRowTrackingDisabled() { val (log, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(testTableName)) val allFiles = snapshot.allFiles.collect() val numFilesInSuccessfulBackfillBatch = 3 val filesInSuccessfulBackfill = allFiles.take(numFilesInSuccessfulBackfillBatch) val numFilesInFailingBackfillBatch = 2 val filesInFailingBackfill = allFiles.takeRight(numFilesInFailingBackfillBatch) val txn = log.startTransaction() val backfillStats = BackfillCommandStats( transactionId = txn.txnId, nameOfTriggeringOperation = DeltaOperations.OP_SET_TBLPROPERTIES) val backfillExecutor = new RowTrackingBackfillExecutor( spark, log, catalogTableOpt = None, txn.txnId, backfillStats ) { var batchIdx = 0 override def constructBatch(files: Seq[AddFile]): BackfillBatch = { batchIdx += 1 if (batchIdx == 1) { RowTrackingBackfillBatch(filesInSuccessfulBackfill) } else { FailingRowTrackingBackfillBatch(filesInFailingBackfill) } } } val backfillUsageRecords = Log4jUsageLogger.track { intercept[IllegalStateException] { backfillExecutor.run(maxNumFilesPerCommit = 3) } }.filter(_.metric == "tahoeEvent") val backfillBatchRecords = backfillUsageRecords .filter(_.tags.get("opType").contains(DeltaUsageLogsOpTypes.BACKFILL_BATCH)) val backfillBatchStatsSeq = backfillBatchRecords.map { backfillBatchRecord => JsonUtils.fromJson[BackfillBatchStats](backfillBatchRecord.blob) } // Check parent backfill stats. The total execution time and wasSuccessful are manipulated in // RowTrackingBackfillCommand, not BackfillExecutor so it is still 0 and false respectively. assert(backfillStats.numFailedBatches === 1) assert(backfillStats.numSuccessfulBatches === 1) assert(backfillStats.transactionId === txn.txnId) assert(backfillStats.nameOfTriggeringOperation === DeltaOperations.OP_SET_TBLPROPERTIES) // Check the individual batch stats assert(backfillBatchStatsSeq.size === 2) backfillBatchStatsSeq.foreach { backfillBatchStat => backfillBatchStat.batchId match { case 0 => assert(backfillBatchStat.wasSuccessful) assert(backfillBatchStat.initialNumFiles === numFilesInSuccessfulBackfillBatch) assert(backfillBatchStat.totalExecutionTimeInMs > 0) case 1 => assert(!backfillBatchStat.wasSuccessful) assert(backfillBatchStat.initialNumFiles === numFilesInFailingBackfillBatch) // Failing batch can have totalExecutionTimeInMs be 0 because it ends faster. assert(backfillBatchStat.totalExecutionTimeInMs >= 0) case id => fail(s"Unexpected batch id $id for RowTrackingBackfillBatch.") } assert(backfillBatchStat.parentTransactionId === txn.txnId) } } } test("BackfillBatchStats failure leads to correct metrics") { withTestTableWithNoRowTracking() { val table = DeltaTableV2(spark, TableIdentifier(testTableName)) val filesInBackfillBatch = table.snapshot.allFiles.head(2) val batch = FailingRowTrackingBackfillBatch(filesInBackfillBatch) val batchId = 17 val numSuccessfulBatch = new AtomicInteger(0) val numFailedBatch = new AtomicInteger(0) val backfillTxnId = "backfill-txn-id" val txn = table.startTransactionWithInitialSnapshot() val backfillUsageRecords = Log4jUsageLogger.track { intercept[IllegalStateException] { batch.execute(spark, backfillTxnId, batchId, txn, numSuccessfulBatch, numFailedBatch) } }.filter(_.metric == "tahoeEvent") val backfillBatchRecords = backfillUsageRecords .filter(_.tags.get("opType").contains(DeltaUsageLogsOpTypes.BACKFILL_BATCH)) assert(backfillBatchRecords.size === 1, "Row Tracking Backfill should have " + "only been executed once.") val backfillBatchStats = JsonUtils.fromJson[BackfillBatchStats](backfillBatchRecords.head.blob) assert(numSuccessfulBatch.get() === 0) assert(numFailedBatch.get() === 1) assert(backfillBatchStats.batchId === batchId) assert(!backfillBatchStats.wasSuccessful) assert(backfillBatchStats.initialNumFiles === filesInBackfillBatch.length) // Failing batch can have totalExecutionTimeInMs be 0 because it ends faster. assert(backfillBatchStats.totalExecutionTimeInMs >= 0) assert(backfillBatchStats.parentTransactionId === backfillTxnId) assert(backfillBatchStats.transactionId != null && backfillBatchStats.transactionId.nonEmpty) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowTrackingCompactionSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.rowid import java.io.File import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.commands.optimize.OptimizeMetrics import org.apache.spark.sql.delta.hooks.AutoCompact import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.{DataFrame, QueryTest} import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions.{EqualTo, Literal} import org.apache.spark.sql.functions._ import org.apache.spark.sql.test.SharedSparkSession trait RowTrackingCompactionTestsBase extends QueryTest with SharedSparkSession with RowIdTestUtils { protected def commandName: String protected val numSoftDeletedRows: Int = 0 protected def createTable( dir: File, rowTrackingEnabled: Boolean, partitioned: Boolean, withMaterializedRowTrackingColumns: Boolean): Unit = { withRowTrackingEnabled(rowTrackingEnabled) { val partitionClause = if (partitioned) "PARTITIONED BY (key)" else "" spark.sql( s"""CREATE TABLE delta.`${dir.getAbsolutePath}` (key LONG, value LONG) |USING DELTA |$partitionClause""".stripMargin) def writeValues(start: Long, end: Long): Unit = { val df = spark.range(start, end, step = 1, numPartitions = 1) .select(col("id") % 10 as "key", col("id") as "value") writeDf(dir, partitioned, withMaterializedRowTrackingColumns, df) } // Write 3 times to create 3 commits with different versions writeValues(start = 0, end = 100) writeValues(start = 100, end = 200) writeValues(start = 200, end = 300) } } protected def writeDf( dir: File, partitioned: Boolean, withMaterializedRowTrackingColumns: Boolean, _df: DataFrame): Unit = { var df = _df if (withMaterializedRowTrackingColumns) { val deltaLog = DeltaLog.forTable(spark, dir) val snapshot = deltaLog.update() val materializedRowIdColName = MaterializedRowId.getMaterializedColumnNameOrThrow( snapshot.protocol, snapshot.metadata, deltaLog.unsafeVolatileTableId) df = df.withMaterializedRowIdColumn(materializedRowIdColName, col("value")) val materializedRowCommitVersionColName = MaterializedRowCommitVersion.getMaterializedColumnNameOrThrow( snapshot.protocol, snapshot.metadata, deltaLog.unsafeVolatileTableId) df = df.withMaterializedRowCommitVersionColumn( materializedRowCommitVersionColName, col("value")) } var writer = df.write.format("delta").mode("append") if (partitioned) { writer = writer.partitionBy("key") } writer.save(dir.getAbsolutePath) } protected def runStatement(statement: String): OptimizeMetrics = { import testImplicits._ spark.sql(statement) .select(col("metrics.*")) .as[OptimizeMetrics] .head() } protected def runCompaction(dir: File, applyFilter: Boolean): OptimizeMetrics protected def checkCompactionRowTrackingPreservation(dir: File, applyFilter: Boolean): Unit = { val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) def readWithRowTrackingColumns(): DataFrame = { spark.read.format("delta").load(dir.getAbsolutePath) .select("value", RowId.QUALIFIED_COLUMN_NAME, RowCommitVersion.QUALIFIED_COLUMN_NAME) } val rowsBeforeFirstCompaction = readWithRowTrackingColumns().collect() // Run compaction, check that at least one file in the table was rewritten, and check that // the fresh row IDs have been preserved. checkRowTrackingMarkedAsPreservedForCommit(deltaLog) { val metrics = runCompaction(dir, applyFilter) assert(metrics.numFilesRemoved > 0) } checkAnswer(readWithRowTrackingColumns(), rowsBeforeFirstCompaction) assertRowIdsAreValid(deltaLog) } protected def runTest( partitioned: Boolean, withMaterializedRowTrackingColumns: Boolean, applyFilter: Boolean): Unit = { withTempDir { dir => createTable(dir, rowTrackingEnabled = true, partitioned, withMaterializedRowTrackingColumns) checkCompactionRowTrackingPreservation(dir, applyFilter) } } } trait RowTrackingCompactionTests extends RowTrackingCompactionTestsBase { test(s"$commandName unpartitioned table with fresh row IDs") { runTest(partitioned = false, withMaterializedRowTrackingColumns = false, applyFilter = false) } test(s"$commandName unpartitioned table with stable row IDs") { runTest(partitioned = false, withMaterializedRowTrackingColumns = true, applyFilter = false) } test(s"$commandName partitioned table with fresh row IDs") { runTest(partitioned = true, withMaterializedRowTrackingColumns = false, applyFilter = false) } test(s"$commandName partitioned table with stable row IDs") { runTest(partitioned = true, withMaterializedRowTrackingColumns = true, applyFilter = false) } test(s"$commandName partitioned table with fresh row IDs and filter") { runTest(partitioned = true, withMaterializedRowTrackingColumns = false, applyFilter = true) } test(s"$commandName partitioned table with stable row IDs and filter") { runTest(partitioned = true, withMaterializedRowTrackingColumns = true, applyFilter = true) } test("Row tracking marked as not preserved when row tracking disabled") { withTempDir { dir => withRowTrackingEnabled(enabled = false) { createTable( dir, rowTrackingEnabled = false, partitioned = false, withMaterializedRowTrackingColumns = false) } val log = DeltaLog.forTable(spark, dir) assert(!rowTrackingMarkedAsPreservedForCommit(log)(runCompaction(dir, applyFilter = false))) } } test(s"$commandName preserves row tracking on backfill enabled tables") { withSQLConf(DeltaSQLConf.DELTA_ROW_TRACKING_BACKFILL_ENABLED.key -> "true") { withTempDir { dir => createTable( dir, rowTrackingEnabled = false, partitioned = false, withMaterializedRowTrackingColumns = false) val log = DeltaLog.forTable(spark, dir) val snapshot = log.update() assert(!RowTracking.isEnabled(snapshot.protocol, snapshot.metadata)) val numRows = spark.read.format("delta").load(dir.getAbsolutePath).count() validateSuccessfulBackfillMetrics(expectedNumSuccessfulBatches = 1) { triggerBackfillOnTestTableUsingAlterTable( targetTableName = s"delta.`${dir.getAbsolutePath}`", numRowsInTable = numRows.toInt + numSoftDeletedRows, log) } checkCompactionRowTrackingPreservation(dir, applyFilter = false) } } } } trait RowTrackingOptimizeTests extends RowTrackingCompactionTests { override protected def commandName: String = "optimize" override def runCompaction(dir: File, applyFilter: Boolean): OptimizeMetrics = { runStatement( s"""OPTIMIZE delta.`${dir.getAbsolutePath}` |${if (applyFilter) "WHERE key = 5" else ""}""".stripMargin) } } trait RowTrackingZorderTests extends RowTrackingCompactionTests { override protected def commandName: String = "z-order" override def runCompaction(dir: File, applyFilter: Boolean): OptimizeMetrics = { runStatement( s"""OPTIMIZE delta.`${dir.getAbsolutePath}` |${if (applyFilter) "WHERE key = 5" else ""} |ZORDER BY (value)""".stripMargin) } } trait RowTrackingAutoCompactionTests extends RowTrackingCompactionTests { override protected def commandName: String = "auto-compact" override def runCompaction(dir: File, applyFilter: Boolean): OptimizeMetrics = { val log = DeltaLog.forTable(spark, dir) val partitionPredicates = if (applyFilter) { Seq(EqualTo(UnresolvedAttribute("key"), Literal(5L))) } else { Nil } var metrics: OptimizeMetrics = null withSQLConf(DeltaSQLConf.DELTA_AUTO_COMPACT_MIN_NUM_FILES.key -> "0") { metrics = AutoCompact.compact( spark, log, partitionPredicates).head } metrics } } trait RowTrackingPurgeTests extends RowTrackingCompactionTests with PersistentDVEnabled { override protected val numSoftDeletedRows: Int = 3 override protected def commandName: String = "purge" override protected def createTable( dir: File, rowTrackingEnabled: Boolean, partitioned: Boolean, withMaterializedRowTrackingColumns: Boolean): Unit = { withRowTrackingEnabled(enabled = rowTrackingEnabled) { val partitionClause = if (partitioned) "PARTITIONED BY (key)" else "" spark.sql( s"""CREATE TABLE delta.`${dir.getAbsolutePath}` (key LONG, value LONG, value2 LONG) |USING DELTA |$partitionClause""".stripMargin) def writeValues(start: Long, end: Long): Unit = { val df = spark.range(start, end, step = 1, numPartitions = 1) .select(col("id") % 10 as "key", col("id") as "value", col("id") as "value2") writeDf(dir, partitioned, withMaterializedRowTrackingColumns, df) } // Write 3 times to create 3 commits with different versions writeValues(start = 0, end = 100) writeValues(start = 100, end = 200) writeValues(start = 200, end = 300) // Add Deletion Vectors to the table so that we can trigger purge. val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) removeRowsFromAllFilesInLog(deltaLog, numRowsToRemovePerFile = 1) } } override def runCompaction(dir: File, applyFilter: Boolean): OptimizeMetrics = { val statement = s"""REORG TABLE delta.`${dir.getAbsolutePath}` |${if (applyFilter) "WHERE key = 5" else ""} |APPLY (PURGE)""".stripMargin val metricsFirstRun = runStatement(statement) // Check that a second of run of PURGE does not modify the table. // This could be an issue with row IDs, as the materialized row ID is not part of the schema // of the table. val metricsSecondRun = runStatement(statement) assert(metricsSecondRun.numFilesRemoved === 0) assert(metricsSecondRun.numFilesAdded === 0) metricsFirstRun } } trait RowTrackingCompactionTestsWithNameColumnMapping extends RowTrackingCompactionTestsBase with DeltaColumnMappingEnableNameMode trait RowIdCompactionTestsWithIdColumnMapping extends RowTrackingCompactionTestsBase with DeltaColumnMappingEnableIdMode class RowTrackingOptimizeSuite extends RowTrackingOptimizeTests class RowTrackingOptimizeSuiteWithIdColumnMapping extends RowTrackingOptimizeTests with RowIdCompactionTestsWithIdColumnMapping class RowTrackingOptimizeSuiteWithNameColumnMapping extends RowTrackingOptimizeTests with RowTrackingCompactionTestsWithNameColumnMapping class RowTrackingZorderSuite extends RowTrackingZorderTests class RowTrackingZorderSuiteWithIdColumnMapping extends RowTrackingZorderTests with RowIdCompactionTestsWithIdColumnMapping class RowTrackingZorderSuiteWithNameColumnMapping extends RowTrackingZorderTests with RowTrackingCompactionTestsWithNameColumnMapping class RowTrackingAutoCompactionSuite extends RowTrackingAutoCompactionTests class RowTrackingAutoCompactionSuiteWithIdColumnMapping extends RowTrackingAutoCompactionTests with RowIdCompactionTestsWithIdColumnMapping class RowTrackingAutoCompactionSuiteWithNameColumnMapping extends RowTrackingAutoCompactionTests with RowTrackingCompactionTestsWithNameColumnMapping class RowTrackingPurgeSuite extends RowTrackingPurgeTests class RowTrackingPurgeSuiteWithIdColumnMapping extends RowTrackingPurgeTests with RowIdCompactionTestsWithIdColumnMapping class RowTrackingPurgeSuiteWithNameColumnMapping extends RowTrackingPurgeTests with RowTrackingCompactionTestsWithNameColumnMapping ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowTrackingDeleteSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.rowid import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.SparkConf import org.apache.spark.sql.{AnalysisException, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.col trait RowTrackingDeleteTestDimension extends QueryTest with RowIdTestUtils { val testTableName = "rowIdDeleteTable" val initialNumRows = 5000 /** * Create a table and validate that it has Row IDs and the expected number of files. */ def createTestTable( tableName: String, isPartitioned: Boolean, multipleFilesPerPartition: Boolean): Unit = { // We disable Optimize Write to ensure the right number of files are created. withSQLConf(DeltaSQLConf.DELTA_OPTIMIZE_WRITE_ENABLED.key -> "false") { val numFilesPerPartition = if (isPartitioned && multipleFilesPerPartition) 2 else 1 val numRowsPerPartition = 100 val expectedNumFiles = if (isPartitioned) { numFilesPerPartition * (initialNumRows / numRowsPerPartition) } else { 10 } val partitionColumnValue = (col("id") / numRowsPerPartition).cast("int") val df = spark.range(0, initialNumRows, 1, expectedNumFiles) .withColumn("part", partitionColumnValue) if (isPartitioned) { df.repartition(numFilesPerPartition) .write .format("delta") .partitionBy("part") .saveAsTable(tableName) } else { df.write .format("delta") .saveAsTable(tableName) } val (log, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName)) assert(snapshot.allFiles.count() === expectedNumFiles) } } def withRowIdTestTable(isPartitioned: Boolean)(f: => Unit): Unit = { withRowTrackingEnabled(enabled = true) { withTable(testTableName) { createTestTable(testTableName, isPartitioned, multipleFilesPerPartition = false) f } } } /** * Read the stable row IDs before and after the DELETE operation. * Validate the row IDs are the same. */ def deleteAndValidateStableRowId(whereCondition: Option[String]): Unit = { val expectedRows: Array[Row] = spark.table(testTableName) .select("id", RowId.QUALIFIED_COLUMN_NAME, RowCommitVersion.QUALIFIED_COLUMN_NAME) .where(s"NOT (${whereCondition.getOrElse("true")})") .collect() val log = DeltaLog.forTable(spark, TableIdentifier(testTableName)) checkRowTrackingMarkedAsPreservedForCommit(log) { checkFileActionInvariantBeforeAndAfterOperation(log) { checkHighWatermarkBeforeAndAfterOperation(log) { executeDelete(whereCondition) } } } val actualDF = spark.table(testTableName) .select("id", RowId.QUALIFIED_COLUMN_NAME, RowCommitVersion.QUALIFIED_COLUMN_NAME) checkAnswer(actualDF, expectedRows) } def executeDelete(whereCondition: Option[String]): Unit = { val whereClause = whereCondition.map(cond => s" WHERE $cond").getOrElse("") spark.sql(s"DELETE FROM $testTableName$whereClause") } override protected def sparkConf: SparkConf = { super.sparkConf .set(DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey, "false") } } trait RowTrackingDeleteSuiteBase extends RowTrackingDeleteTestDimension { val subqueryTableName = "subqueryTable" for { isPartitioned <- BOOLEAN_DOMAIN whereClause <- Seq( "id IN (5, 7, 11, 57, 66, 77, 79, 88, 91, 95)", // 0.2%, 10 rows match "part = 5", // 10%, 500 rows match "id % 20 = 0", // 20%, 1000 rows match "id >= 0" // 100%, 5000 rows match ) } { test(s"DELETE preserves Row IDs, isPartitioned=$isPartitioned, whereClause=`$whereClause`") { withRowIdTestTable(isPartitioned) { deleteAndValidateStableRowId(Some(whereClause)) } } } test("Preserving Row Tracking - Subqueries are not supported in DELETE") { withRowIdTestTable(isPartitioned = false) { withTable(subqueryTableName) { createTestTable(subqueryTableName, isPartitioned = false, multipleFilesPerPartition = false) val ex = intercept[AnalysisException] { deleteAndValidateStableRowId(Some( s"id in (SELECT id FROM $subqueryTableName WHERE id = 7 OR id = 11)")) }.getMessage assert(ex.contains("Subqueries are not supported in the DELETE")) } } } for (isPartitioned <- BOOLEAN_DOMAIN) { test(s"Multiple DELETEs preserve Row IDs, isPartitioned=$isPartitioned") { withRowIdTestTable(isPartitioned) { val whereClause1 = "id % 20 = 0" deleteAndValidateStableRowId(Some(whereClause1)) val whereClause2 = "id % 10 = 0" deleteAndValidateStableRowId(Some(whereClause2)) } } } for (isPartitioned <- BOOLEAN_DOMAIN) { test(s"Insert after DELETE on whole table, isPartitioned=$isPartitioned") { withRowIdTestTable(isPartitioned) { // Delete whole table. deleteAndValidateStableRowId(whereCondition = None) spark.sql(s"INSERT INTO $testTableName VALUES (1, 0), (2, 0), (3, 0), (4, 0)") // The new rows should have new row IDs. val actualDF = spark.table(testTableName) .select("id", RowId.QUALIFIED_COLUMN_NAME) assert(actualDF.filter(s"row_id < $initialNumRows").count() <= 0) } } } for { isPartitioned <- BOOLEAN_DOMAIN } { test(s"DELETE with optimized writes preserves Row ID, isPartitioned=$isPartitioned") { withRowTrackingEnabled(enabled = true) { withTable(testTableName) { createTestTable(testTableName, isPartitioned, multipleFilesPerPartition = true) val whereClause = "id % 20 = 0" withSQLConf( DeltaSQLConf.DELTA_OPTIMIZE_WRITE_ENABLED.key -> "true" ) { deleteAndValidateStableRowId(whereCondition = Some(whereClause)) val (log, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(testTableName)) val currentNumFiles = snapshot.allFiles.count() val deletionVectorEnabled = spark.conf .getOption(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey) .contains("true") val expectedNumFiles = if (deletionVectorEnabled) { if (isPartitioned) 100 else 10 } else { if (isPartitioned) 53 else 1 } assert(currentNumFiles === expectedNumFiles, s"The current num files $currentNumFiles is unexpected for optimized writes") } } } } } test("Row tracking marked as not preserved when row tracking disabled") { withRowTrackingEnabled(enabled = false) { withTable(testTableName) { createTestTable(testTableName, isPartitioned = false, multipleFilesPerPartition = false) val log = DeltaLog.forTable(spark, TableIdentifier(testTableName)) assert( !rowTrackingMarkedAsPreservedForCommit(log)( executeDelete(whereCondition = Some("id = 5")))) } } } for { isPartitioned <- BOOLEAN_DOMAIN } { test("DELETE preserves Row ID on tables with row IDs enabled using backfill," + s"isPartitioned=$isPartitioned") { withSQLConf(DeltaSQLConf.DELTA_ROW_TRACKING_BACKFILL_ENABLED.key -> "true") { withRowTrackingEnabled(enabled = false) { withTable(testTableName) { createTestTable(testTableName, isPartitioned, multipleFilesPerPartition = false) val (log, snapshot) = DeltaLog. forTableWithSnapshot(spark, TableIdentifier(testTableName)) assert(!RowTracking.isEnabled(snapshot.protocol, snapshot.metadata)) validateSuccessfulBackfillMetrics(expectedNumSuccessfulBatches = 1) { triggerBackfillOnTestTableUsingAlterTable(testTableName, initialNumRows, log) } deleteAndValidateStableRowId(whereCondition = Some("id % 10 = 4")) } } } } } } trait RowTrackingDeleteDvBase extends RowTrackingDeleteTestDimension with PersistentDVEnabled { for (isPartitioned <- BOOLEAN_DOMAIN) { test(s"DELETE with persistent DVs disabled, isPartitioned=$isPartitioned") { val whereClause = "id % 20 = 0" withDeletionVectorsEnabled(enabled = false) { withRowIdTestTable(isPartitioned) { deleteAndValidateStableRowId(whereCondition = Some(whereClause)) } } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowTrackingMergeSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.rowid import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.{Dataset, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.{col, lit} trait RowTrackingMergeSuiteBase extends RowIdTestUtils with DeltaDMLTestUtils with MergeHelpers { import testImplicits._ protected val SOURCE_TABLE_NAME = "source" protected val numRows = 4000 protected val numUnmatchedRows = 2000 // Source table with 4000 rows with 'key' 2000 until 6000. protected def createSourceTable(tableName: String, lastModifiedVersion: Long): Unit = { spark.range(start = numUnmatchedRows, end = numUnmatchedRows + numRows).toDF("key") .withColumn("stored_id", col("key")) .withColumn("last_modified_version", lit(lastModifiedVersion)) .write.format("delta").saveAsTable(tableName) } override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey, value = "true") createSourceTable(SOURCE_TABLE_NAME, lastModifiedVersion = 1L) } override protected def afterAll(): Unit = { sql(s"DROP TABLE IF EXISTS $SOURCE_TABLE_NAME") } protected def withTestTable( partitionedTarget: Boolean, lastModifiedVersion: Long = 0L)(f: => Unit): Unit = { val targetCreationDF = spark.range(end = numRows) .toDF("key") .withColumn("stored_id", col("key")) .withColumn("last_modified_version", lit(lastModifiedVersion)) if (partitionedTarget) { append(targetCreationDF .withColumn("partition", lit(0)) .repartition(numPartitions = 2), Seq("partition")) } else { append(targetCreationDF.repartition(numPartitions = 2)) } f } protected def executeMerge( targetReference: String, sourceReference: String, clauses: MergeClause*): Unit = { val mergeSQL = s"""MERGE INTO $targetReference AS t |USING $sourceReference AS s |ON s.key = t.key |${clauses.map(_.sql).mkString("\n")} |""".stripMargin sql(mergeSQL) } /** * Create a test validating stable Row IDs and Row Commit Versions in MERGE. The test uses a fixed * source table and a modifiable target table. By default the source and the target table have * rows not matched in a join on 'key'. * * source table target table * * | key | stored_id | last_modified_version | * | 0 | 0 | 0 | * | 1 | 1 | 0 | * | ... | ... | ... | * | key | stored_id | last_modified_version | | 1999 | 1999 | 0 | * | 2000 | 2000 | 1 | | 2000 | 2000 | 0 | * | 2001 | 2001 | 1 | | 2001 | 2001 | 0 | * | ... | ... | ... | | ... | ... | ... | * | 3999 | 3999 | 1 | | 3999 | 3999 | 0 | * | 4000 | 4000 | 1 | * | ... | ... | ... | * | 5999 | 5999 | 1 | * * Tests also include CDF validation, which only works if 'key' is not changed in update clauses. */ protected def rowTrackingMergeTests( name: String)( partitionedTarget: Boolean = false, targetAsView: Boolean = false, source: => Option[String] = None, targetTablePostSetupAction: Option[() => Unit] = None, sqlConfs: Seq[(String, String)] = Seq.empty)( clauses: MergeClause*)( expected: Seq[Row], numFilesAfterMerge: Option[Int] = None): Unit = { test(name) { withTestTable(partitionedTarget) { // Post setup actions can be used to modify the target table, for example by inserting // more rows into it. targetTablePostSetupAction.foreach(_.apply()) val preMergeRowIdMapping = getPreMergeRowIdMapping val sourceReference = source.getOrElse { if (partitionedTarget) { s"(SELECT *, 0 AS partition FROM $SOURCE_TABLE_NAME)" } else { SOURCE_TABLE_NAME } } val targetReference = if (targetAsView) { sql(s"CREATE TEMPORARY VIEW target_view AS SELECT * FROM $tableSQLIdentifier") "target_view" } else { tableSQLIdentifier } withSQLConf(sqlConfs: _*) { checkRowTrackingMarkedAsPreservedForCommit(deltaLog) { checkFileActionInvariantBeforeAndAfterOperation(deltaLog) { checkHighWatermarkBeforeAndAfterOperation(deltaLog) { executeMerge(targetReference, sourceReference, clauses: _*) } } } } checkAnswer(sql(s"SELECT key, last_modified_version FROM $tableSQLIdentifier"), expected) validateRowIdsPostMerge(preMergeRowIdMapping) validateRowCommitVersionsPostMerge() if (numFilesAfterMerge.isDefined) { val targetTableFiles = deltaLog.update().allFiles assert(targetTableFiles.count() === numFilesAfterMerge.get, s"Expected ${numFilesAfterMerge.get} but got ${targetTableFiles.collect().mkString}") } val cdfEnabled = spark.conf .getOption(DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey) .contains("true") if (cdfEnabled && targetTablePostSetupAction.isEmpty) { assert(deltaLog.update().version === 1, "Table has been modified more than once.") // The tableIdentifier will be overridden as a path identifier if a class/trait also mixes // in DeltaDMLTestUtilsPathBased. So we need a check to ensure it's a name identifier // before using it to get the catalog table. val catalogTableOpt = if (DeltaTableIdentifier.isDeltaPath(spark, tableIdentifier)) { None } else { Some(spark.sessionState.catalog.getTableMetadata(tableIdentifier)) } // Only read CDF from version 1 (the version after the MERGE) val cdfResult = CDCReader.changesToBatchDF( deltaLog, start = 1, end = 1, spark, catalogTableOpt, useCoarseGrainedCDC = true) .select("stored_id", "key", "last_modified_version", "_change_type") .collect() val initialTableDf = spark.read.format("delta") .option("versionAsOf", 0) .table(tableSQLIdentifier) .select("*", "_metadata.row_id") .alias("initial") val postMergeTableDf = spark.read.format("delta") .option("versionAsOf", 1) .table(tableSQLIdentifier) .select("*", "_metadata.row_id") .alias("postMerge") // Outer join the table at the state before the merge with the state after the MERGE on // 'key' under the assumption that 'key' is not altered by the MERGE. val joinedInitialAndPost = initialTableDf .join(postMergeTableDf, usingColumn = "key", joinType = "fullouter") .select( "initial.key", "initial.stored_id", "initial.last_modified_version", "initial.row_id", "postMerge.key", "postMerge.stored_id", "postMerge.last_modified_version", "postMerge.row_id") .collect() joinedInitialAndPost.foreach { case Row(_, storedIdInitial, lastModifiedVersionInitial, rowIdInitial, _, storedIdPostMerge, lastModifiedVersionPostMerge, rowIdPostMerge) => if (lastModifiedVersionPostMerge == null) { // Row has been deleted. val cdfEntries = cdfResult.filter(row => row.getAs("stored_id") == storedIdInitial) assert(cdfEntries.length === 1, s"Invalid number of CDF entries for deleted row with stored_id = " + s"$storedIdInitial. ${cdfEntries.mkString}") assert(cdfEntries.head.getAs[String]("_change_type") === "delete", s"Invalid _change_type (!= delete) for inserted row with stored_id = " + s" $storedIdInitial") assert(rowIdInitial.asInstanceOf[Long] < numRows, "Row ID for delete row not from initial range") } else if (lastModifiedVersionInitial == null) { // Row has been inserted. val cdfEntries = cdfResult.filter(row => row.getAs("stored_id") == storedIdPostMerge) assert(cdfEntries.length === 1, s"Invalid number of CDF entries for inserted row with stored_id = " + s" $storedIdPostMerge. ${cdfEntries.mkString}") assert(cdfEntries.head.getAs[String]("_change_type") === "insert", s"Invalid _change_type (!= insert) for row with stored_id = $storedIdPostMerge") assert(rowIdPostMerge.asInstanceOf[Long] >= numRows, "Row ID for inserted row from initial range") } else { // Row has been updated or is unchanged. val cdfEntries = cdfResult.filter(row => row.getAs("stored_id") == storedIdPostMerge) if (lastModifiedVersionInitial != lastModifiedVersionPostMerge) { // Row has been updated assert(cdfEntries.length === 2, s"Invalid number of CDF entries for updated/copied row with " + s"stored_id = $storedIdPostMerge. ${cdfEntries.mkString}") } else { // Row is untouched or copied. assert(Seq(0, 2).contains(cdfEntries.length), s"Invalid number of CDF entries for updated/copied row with " + s"stored_id = $storedIdPostMerge. ${cdfEntries.mkString}") } } } } } } } /** * This method retrieves the mapping of stored_id to row_id from the target table * before the merge operation. * It groups the collected data by stored_id and ensures that each stored_id * is associated with only row_id. * * @return A Map of stored_id to row_id before the merge operation. */ protected def getPreMergeRowIdMapping: Map[Long, Long] = { spark.table(tableSQLIdentifier) .select("stored_id", RowId.QUALIFIED_COLUMN_NAME) .as[(Long, Long)] .collect() .groupBy(_._1) .mapValues { values => assert(values.length === 1) values.head._2 }.toMap } /** * This method validates the row ids after the merge operation. * It ensures that the rows that existed before the merge * operation have kept their original row ids. * For the newly inserted rows, it checks that they have been assigned fresh row ids * that are greater than any row id before the merge operation. * * @param preMergeRowIdMapping The mapping of stored_id to row_id before the merge operation. */ def validateRowIdsPostMerge(preMergeRowIdMapping: Map[Long, Long]): Unit = { val highestRowIdPreMerge = preMergeRowIdMapping.values.max val rowsAfterMerge = spark.read.table(tableSQLIdentifier) .select("stored_id", RowId.QUALIFIED_COLUMN_NAME) .as[(Long, Long)] .collect() val (otherRows, insertedRows) = rowsAfterMerge.partition { case (storedId, _) => preMergeRowIdMapping.contains(storedId) } // Validate that rows kept their stable Row IDs. otherRows.foreach { case (storedId, rowId) => assert(preMergeRowIdMapping(storedId) === rowId, s"Row ID has change for row with stored_id = $storedId") } assert(insertedRows.length === insertedRows.map { case (_, rowId) => rowId }.distinct.length, s"Row IDs are not unique for inserted rows: ${insertedRows.mkString}") // Validate that inserted rows received a fresh Row ID. insertedRows.foreach { case (storedId, rowId) => assert(rowId > highestRowIdPreMerge, s"Row ID not fresh for inserted row with stored_id $storedId") } } /** * This method validates the row commit versions after the merge operation. * It ensures that the row commit version for each row in the target table * matches its last_modified_version. * This is to ensure that the row commit version is updated correctly during * the merge operation. */ def validateRowCommitVersionsPostMerge(): Unit = { val rowsAfterMerge = spark.read.table(tableSQLIdentifier) .select("stored_id", "last_modified_version", RowCommitVersion.QUALIFIED_COLUMN_NAME) .as[(Long, Long, Long)] .collect() rowsAfterMerge.foreach { case (storedId, lastModifiedVersion, rowCommitVersion) => assert(rowCommitVersion === lastModifiedVersion, s"row commit version does not match for row with stored_id $storedId") } } } trait RowTrackingMergeCommonTests extends RowTrackingMergeSuiteBase { rowTrackingMergeTests("INSERT NOT MATCHED only MERGE")()( clauses = insert("*"))( // The old rows that are in the target initially and the added rows. expected = (0 until numRows).map(Row(_, 0L)) ++ (0 until numUnmatchedRows).map(id => Row(numRows + id, 1L)) ) rowTrackingMergeTests("DELETE MATCHED only MERGE")()( clauses = delete())( // All unmatched rows. expected = (0 until numUnmatchedRows).map(Row(_, 0L)) ) rowTrackingMergeTests("UPDATE MATCHED only MERGE")()( clauses = update("*"))( // Matched rows updated, other rows untouched. expected = (0 until numUnmatchedRows).map(Row(_, 0L)) ++ (numUnmatchedRows until numRows).map(Row(_, 1L)) ) rowTrackingMergeTests("DELETE WHEN NOT MATCHED BY SOURCE only MERGE")()( clauses = deleteNotMatched())( // Unmatched rows only. expected = (numUnmatchedRows until numRows).map(Row(_, 0L)) ) rowTrackingMergeTests("UPDATE only WHEN NOT MATCHED BY SOURCE MERGE")()( clauses = updateNotMatched("last_modified_version = 1"))( // All rows not matched by source updated. expected = (0 until numUnmatchedRows).map(Row(_, 1L)) ++ (numUnmatchedRows until numRows).map(Row(_, 0L)) ) rowTrackingMergeTests("UPDATE + DELETE WHEN NOT MATCHED BY SOURCE MERGE")()( clauses = deleteNotMatched("t.stored_id % 2 = 0"), updateNotMatched("last_modified_version = 1"))( expected = (0 until numUnmatchedRows).filter(_ % 2 == 1).map(Row(_, 1L)) ++ (numUnmatchedRows until numRows).map(Row(_, 0L))) rowTrackingMergeTests("UPDATE only with source rows matching multiple target rows")( // Duplicate all target rows. targetTablePostSetupAction = Some(() => { append(spark.read.table(tableSQLIdentifier) .withColumn("stored_id", col("stored_id") + numRows) .withColumn("last_modified_version", lit(1L))) }))( clauses = update("t.last_modified_version = 2"))( // Updated 'key' and 'last_modified_version' for matched rows. expected = (0 until numUnmatchedRows).flatMap(id => Seq(Row(id, 0L), Row(id, 1L))) ++ (numUnmatchedRows until numRows).flatMap(id => Seq(Row(id, 2L), Row(id, 2L))) ) rowTrackingMergeTests("DELETE only with source rows matching multiple target rows")( // Duplicate all target rows. targetTablePostSetupAction = Some(() => { append(spark.read.table(tableSQLIdentifier) .withColumn("stored_id", col("stored_id") + numRows) .withColumn("last_modified_version", lit(1L))) }))( clauses = delete())( // Deleted all matches (2 target rows per source row). expected = (0 until numUnmatchedRows).flatMap(id => Seq(Row(id, 0L), Row(id, 1L))) ) rowTrackingMergeTests("Target is accessed through a view")(targetAsView = true)( clauses = update("*"))( expected = (0 until numUnmatchedRows).map(Row(_, 0L)) // Untouched. ++ (numUnmatchedRows until numRows).map(Row(_, 1L)) // Updated. ) rowTrackingMergeTests("Optimized writes on partitioned table")(partitionedTarget = true)( clauses = update("*", "s.key % 2 = 0"), delete(), insert("*"), deleteNotMatched())( expected = (numUnmatchedRows.until(numRows, step = 2)).map(Row(_, 1L)) // Updated. ++ (numRows until numRows + numUnmatchedRows).map(Row(_, 1L)), // Inserted. numFilesAfterMerge = Some(1) ) rowTrackingMergeTests("Optimized writes disabled on partitioned table")( partitionedTarget = true, sqlConfs = Seq(DeltaSQLConf.DELTA_OPTIMIZE_WRITE_ENABLED.key -> "false"))( clauses = update("*", "s.key % 2 = 0"), delete(), insert("*"), deleteNotMatched())( expected = (numUnmatchedRows.until(numRows, step = 2)).map(Row(_, 1L)) // Updated. ++ (numRows until numRows + numUnmatchedRows).map(Row(_, 1L)), // Inserted. numFilesAfterMerge = Some(1) ) rowTrackingMergeTests("Optimized writes on un-partitioned table")( sqlConfs = Seq(DeltaSQLConf.DELTA_OPTIMIZE_WRITE_ENABLED.key -> "true"))( clauses = update("*", "s.stored_id % 2 = 0"), delete(), insert("*"), deleteNotMatched())( expected = (numUnmatchedRows.until(numRows, step = 2)).map(Row(_, 1L)) // Updated. ++ (numRows until numRows + numUnmatchedRows).map(Row(_, 1L)), // Inserted. numFilesAfterMerge = Some(1) ) rowTrackingMergeTests("Source and target referencing to the same table")( source = Some( s"(SELECT key, stored_id, 1L as last_modified_version FROM $tableSQLIdentifier)"))( clauses = update("*"))( // All rows updated. expected = (0 until numRows).map(Row(_, 1L)) ) test("Multiple merges into the same table") { val numMerges = 5 require(numMerges <= numUnmatchedRows) // Create the target table using half the rows from the source table. append(spark.table(SOURCE_TABLE_NAME) .withColumn("last_modified_version", lit(0L)) .filter("key % 2 = 0") .repartition(numPartitions = 2)) val preMergeRowIdMapping = getPreMergeRowIdMapping // Give the target the same rows as the source table, one row at a time. for (i <- 0 until numMerges) { executeMerge( tableSQLIdentifier, sourceReference = s"(SELECT ${numUnmatchedRows + i} AS key)", clauses = update(s"last_modified_version = ${i + 1}"), insert(s"(key, stored_id, last_modified_version) VALUES (key, key, ${i + 1})")) } checkAnswer(sql(s"SELECT key, last_modified_version FROM $tableSQLIdentifier"), // Updated rows. (0 until numMerges).map(i => Row(numUnmatchedRows + i, i + 1)) // Untouched rows. ++ (numUnmatchedRows + numMerges + 1).until(numRows + numUnmatchedRows, step = 2) .map(Row(_, 0L)) ) validateRowIdsPostMerge(preMergeRowIdMapping) validateRowCommitVersionsPostMerge() } test("Row tracking marked as not preserved when row tracking disabled") { withRowTrackingEnabled(enabled = false) { withTestTable(partitionedTarget = false) { assert(!rowTrackingMarkedAsPreservedForCommit(deltaLog)( executeMerge( tableSQLIdentifier, SOURCE_TABLE_NAME, clauses = update("*"), insert("*")))) } } } test("schema evolution, extra nested column in source - update") { import testImplicits._ val targetData = Seq((0L, 0L, 0L, (1, 10)), (1L, 1L, 0L, (2, 2000))) .toDF("key", "stored_id", "last_modified_version", "x") .selectExpr( "key", "stored_id", "last_modified_version", "named_struct('a', x._1, 'c', x._2) as x") append(targetData.repartition(1)) val sourceData = Seq((0L, 0L, 1L, (10, 100, 10000))) .toDF("key", "stored_id", "last_modified_version", "x") .selectExpr( "key", "stored_id", "last_modified_version", "named_struct('a', x._1, 'b', x._2, 'c', x._3) as x") val preMergeRowIdMapping = getPreMergeRowIdMapping withTempView("src") { sourceData.createOrReplaceTempView("src") withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { executeMerge( tableSQLIdentifier, sourceReference = "src", clauses = update("*")) } } checkAnswer(sql(s"SELECT stored_id, last_modified_version FROM $tableSQLIdentifier"), Seq(Row(0L, 1L), Row(1L, 0L))) validateRowIdsPostMerge(preMergeRowIdMapping) validateRowCommitVersionsPostMerge() } test("MERGE preserves Row Tracking on tables enabled using backfill") { withSQLConf(DeltaSQLConf.DELTA_ROW_TRACKING_BACKFILL_ENABLED.key -> "true") { val SOURCE_TABLE_NAME_FOR_BACKFILL_TEST = "backfilled_source" withTable(SOURCE_TABLE_NAME_FOR_BACKFILL_TEST) { createSourceTable(SOURCE_TABLE_NAME_FOR_BACKFILL_TEST, lastModifiedVersion = 4L) withRowTrackingEnabled(enabled = false) { withTestTable(partitionedTarget = false, lastModifiedVersion = 2L) { val snapshot = deltaLog.update() assert(!RowTracking.isEnabled(snapshot.protocol, snapshot.metadata)) validateSuccessfulBackfillMetrics(expectedNumSuccessfulBatches = 1) { triggerBackfillOnTestTableUsingAlterTable(tableSQLIdentifier, numRows, deltaLog) } val preMergeRowIdMapping = getPreMergeRowIdMapping executeMerge( tableSQLIdentifier, SOURCE_TABLE_NAME_FOR_BACKFILL_TEST, clauses = update("*"), insert("*")) validateRowIdsPostMerge(preMergeRowIdMapping) validateRowCommitVersionsPostMerge() } } } } } } trait RowTrackingMergeDVMixin extends RowTrackingMergeSuiteBase with DeletionVectorsTestUtils { override def beforeAll(): Unit = { super.beforeAll() enableDeletionVectors(spark, delete = true, update = true, merge = true) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowTrackingRemovalConcurrencySuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.rowid import java.io.File import java.util.concurrent.atomic.AtomicInteger import org.apache.spark.sql.delta.{ConflictResolutionTestUtils, DeltaConfigs, DeltaErrors, DeltaIllegalStateException, DeltaLog, DeltaOperations, DeltaTableFeatureException, ProtocolChangedException, RemovableFeature, RowTrackingFeature, TableFeature} import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.actions.{AddFile, DropTableFeatureUtils} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.AlterTableSetPropertiesDeltaCommand import org.apache.spark.sql.delta.commands.backfill.{BackfillCommandStats, BackfillExecutionObserver, BackfillExecutor, RowTrackingBackfillCommand, RowTrackingBackfillExecutor, RowTrackingUnBackfillCommand, RowTrackingUnBackfillExecutor} import org.apache.spark.sql.delta.concurrency.{PhaseLockingTestMixin, TransactionExecutionTestMixin} import org.apache.spark.sql.delta.fuzzer.{AtomicBarrier, OptimisticTransactionPhases, PhaseLockingTransactionExecutionObserver => TransactionObserver} import org.apache.spark.sql.delta.implicits._ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.hadoop.fs.Path import org.apache.spark.{SparkConf, SparkThrowable} import org.apache.spark.sql.Row import org.apache.spark.util.ThreadUtils class RowTrackingRemovalConcurrencySuite extends RowTrackingRemovalSuiteBase with ConflictResolutionTestUtils with TransactionExecutionTestMixin with PhaseLockingTestMixin { protected val areDVsEnabled = true private val ignoreSuspensionConf = DeltaSQLConf.DELTA_ROW_TRACKING_IGNORE_SUSPENSION.key -> "true" protected override def sparkConf: SparkConf = super.sparkConf .set(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey, areDVsEnabled.toString) protected def dropRowTrackingTransaction(tableName: String): Array[Row] = { sql(s"""ALTER TABLE $tableName DROP FEATURE ${RowTrackingFeature.name}""").collect() } protected def enableRowTrackingTransaction(tableName: String): Array[Row] = { sql(s"""ALTER TABLE $tableName |SET TBLPROPERTIES('${DeltaConfigs.ROW_TRACKING_ENABLED.key}' = 'true') |""".stripMargin) .collect() } protected def backfillTransaction(deltaLog: DeltaLog): Seq[Row] = { RowTrackingBackfillCommand( deltaLog, nameOfTriggeringOperation = "TEST", catalogTableOpt = None).run(spark) } protected def unBackfillTransaction(deltaLog: DeltaLog): Seq[Row] = { RowTrackingUnBackfillCommand( deltaLog, nameOfTriggeringOperation = "TEST", catalogTableOpt = None).run(spark) } /** * Represents a delete transaction by a third party writer that does not respect * property `delta.rowTrackingSuspended`. */ case class ThirdPartyDelete( condition: String) extends TestTransaction(Map(ignoreSuspensionConf)) { override val name: String = s"THIRD PARTY DELETE($condition)($sqlConfStr)" override def dataChange: Boolean = true override def toSQL(tableName: String): String = s"DELETE FROM $tableName WHERE $condition" } /** * Represents an update transaction by a third party writer that does not respect * property `delta.rowTrackingSuspended`. */ case class ThirdPartyUpdate( set: String, condition: String) extends TestTransaction(Map(ignoreSuspensionConf)) { override val name: String = s"THIRD PARTY UPDATE($set)($condition)($sqlConfStr)" override def dataChange: Boolean = true override def toSQL(tableName: String): String = s"UPDATE $tableName SET $set WHERE $condition" } /** * Represents an insert transaction by a third party writer that does not respect * property `delta.rowTrackingSuspended`. */ case class ThirdPartyInsert( start: Long, end: Long, numPartitions: Int = 2, sqlConf: Map[String, String] = Map(ignoreSuspensionConf)) extends TestTransaction(sqlConf) { override val name: String = s"THIRD PARTY INSERT($start-$end)($sqlConfStr)" override def dataChange: Boolean = true override def toSQL(tableName: String): String = { throw new UnsupportedOperationException("No SQL implementation for ThirdPartyInsert") } override def executeImpl(ctx: TestContext): Unit = { withSQLConf(sqlConf.toSeq: _*) { // This should generate baseRowIds. spark.range(start, end, step = 1, numPartitions) .write .format("delta") .mode("append") .insertInto(s"delta.`${ctx.deltaLog.dataPath}`") } } } /** * Test transaction that performs a protocol downgrade for a given feature. * Note, it does not add the checkpointProtection feature. */ case class DowngradeProtocol( feature: TableFeature with RemovableFeature) extends TestTransaction(Map.empty) { override val name: String = s"Downgrade(${feature.name})($sqlConfStr)" override def dataChange: Boolean = false override def toSQL(tableName: String): String = { throw new UnsupportedOperationException("No SQL implementation for DowngradeProtocol") } override def executeImpl(ctx: TestContext): Unit = { val deltaLog = ctx.deltaLog val table = DeltaTableV2(spark, deltaLog.dataPath) val txn = deltaLog.startTransaction(catalogTableOpt = None) if (!feature.validateDropInvariants(table, txn.snapshot)) { throw DeltaErrors.dropTableFeatureConflictRevalidationFailed() } txn.updateProtocol(txn.protocol.removeFeature(feature)) val metadataWithNewConfiguration = DropTableFeatureUtils .getDowngradedProtocolMetadata(feature, txn.metadata) txn.updateMetadata(metadataWithNewConfiguration) val commitActions = feature.actionsToIncludeAtDowngradeCommit(txn.snapshot) txn.commit(commitActions, DeltaOperations.DropTableFeature(feature.name, false)) } } /** Test implementation of backfill batch. */ private def backfillBatchImplementation( deltaLog: DeltaLog, executor: BackfillExecutor): Unit = { BackfillExecutionObserver.getObserver.executeBatch { val txn = deltaLog.startTransaction(catalogTableOpt = None) val filesInBatch = executor .filesToBackfill(txn.snapshot) .collect() if (filesInBatch.isEmpty) { return } val batch = executor.constructBatch(filesInBatch) txn.trackFilesRead(filesInBatch) batch.execute( spark, backfillTxnId = executor.backfillTxnId, batchId = 0, txn = txn, numSuccessfulBatch = new AtomicInteger(0), numFailedBatch = new AtomicInteger(0)) } } /** * Test transaction that unbackfill baseRowIDs. It assumes all files can be * unbackfilled in a single commit. */ case class UnbackfillBatch() extends TestTransaction(Map.empty) { override val name: String = "UNBACKFILL" override def dataChange: Boolean = false override def toSQL(tableName: String): String = { throw new UnsupportedOperationException("No SQL implementation for Unbackfill") } override def executeImpl(ctx: TestContext): Unit = { val deltaLog = ctx.deltaLog val propertyKey = DeltaSQLConf.DELTA_ROW_TRACKING_IGNORE_SUSPENSION.key withSQLConf(propertyKey -> "false") { val backfillStats = BackfillCommandStats( transactionId = "test-backfill-batch", "TEST" ) backfillBatchImplementation( deltaLog, new RowTrackingUnBackfillExecutor( spark, deltaLog, catalogTableOpt = None, backfillTxnId = backfillStats.transactionId, backfillStats = backfillStats )) } } } /** * Test transaction that backfills baseRowIDs. It ignores * `rowTrackingSuspended` property. However,it assumes the third party * client uses ROW_TRACKING_BACKFILL_OPERATION_NAME. Finally, it assumes all files can be * backfilled in a single commit. */ case class ThirdPartyBackfillBatch() extends TestTransaction(Map.empty) { override val name: String = "BACKFILL" override def dataChange: Boolean = false override def toSQL(tableName: String): String = { throw new UnsupportedOperationException("No SQL implementation for Backfill") } override def executeImpl(ctx: TestContext): Unit = { val deltaLog = ctx.deltaLog withSQLConf(DeltaSQLConf.DELTA_ROW_TRACKING_IGNORE_SUSPENSION.key -> "true") { val backfillStats = BackfillCommandStats( transactionId = "test-backfill-batch", "TEST" ) backfillBatchImplementation( deltaLog, new RowTrackingBackfillExecutor( spark, deltaLog, catalogTableOpt = None, backfillTxnId = backfillStats.transactionId, backfillStats = backfillStats )) } } } private def createTestTable( dir: File, numPartitions: Int = 2, rowTrackingEnabled: Boolean = true): DeltaLog = { sql( s"""CREATE TABLE delta.`${dir.getCanonicalPath}` (id bigint) |USING delta |TBLPROPERTIES( |'${DeltaConfigs.ROW_TRACKING_ENABLED.key}' = '${rowTrackingEnabled.toString}' |)""".stripMargin) spark.range(start = 0, end = 100, step = 1, numPartitions) .write .format("delta") .mode("append") .save(dir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) if (rowTrackingEnabled) { validateRowTrackingState(deltaLog, isPresent = true) } val expectedFileCountWithRowIDs = if (rowTrackingEnabled) numPartitions else 0 validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs) deltaLog } private def validateRowTrackingMetadataInAddFiles( deltaLog: DeltaLog, expectedFileCountWithRowIDs: Int): Unit = { val snapshot = deltaLog.update() val filesWithRowIDsCount = snapshot .allFiles .filter("baseRowId IS NOT NULL or defaultRowCommitVersion IS NOT NULL") .count() assert(filesWithRowIDsCount === expectedFileCountWithRowIDs) } private def disableRowTracking(tablePath: String): Unit = { // Fist stage of row tracking removal: disable the feature. val propertiesToSet = Map( DeltaConfigs.ROW_TRACKING_ENABLED.key -> "false", DeltaConfigs.ROW_TRACKING_SUSPENDED.key -> "true") val table = DeltaTableV2(spark, new Path(tablePath)) AlterTableSetPropertiesDeltaCommand(table, propertiesToSet).run(spark) } private def waitForTransactionStart( observer: TransactionObserver): Unit = { def transactionStart: Boolean = { observer.phases.initialPhase.entryBarrier.load() == AtomicBarrier.State.Requested } busyWaitFor(transactionStart, timeout) } /* * -------------------------------------------> TIME -------------------------------------------> * * Drop Feature * Command Metadata upgrade -------- Unbackfill batches ---------------- Downgrade Commit * prepare + commit prepare + commit prepare + commit * * * Business Txn prepare + commit * * -------------------------------------------> TIME -------------------------------------------> */ for (op <- Seq("insert", "update", "delete")) test(s"$op interleaves between the last unbackfill and the protocol downgrade") { withTempDir { dir => val deltaLog = createTestTable(dir) val ctx = new TestContext(deltaLog) val table = s"delta.`${dir.getAbsolutePath}`" val dropRowTrackingFn = () => dropRowTrackingTransaction(table) val Seq(dropFuture) = runFunctionsWithOrderingFromObserver(Seq(dropRowTrackingFn)) { case (updateMetadataDropObserver :: Nil) => val unbackfillDropObserver = new TransactionObserver( OptimisticTransactionPhases.forName("unbackfill-drop-txn")) val downgradeDropObserver = new TransactionObserver( OptimisticTransactionPhases.forName("downgrade-drop-txn")) updateMetadataDropObserver.setNextObserver( unbackfillDropObserver, autoAdvance = true) unbackfillDropObserver.setNextObserver( downgradeDropObserver, autoAdvance = true) prepareAndCommitWithNextObserverSet(updateMetadataDropObserver) prepareAndCommitWithNextObserverSet(unbackfillDropObserver) waitForTransactionStart(downgradeDropObserver) val businessTxn = op match { case "insert" => ThirdPartyInsert(start = 100, end = 200) case "update" => ThirdPartyUpdate(set = "id = 200", condition = "id = 90") case "delete" => ThirdPartyDelete("id = 90") } businessTxn.execute(ctx) val expectedFileCountWithRowIDs = op match { case "insert" => 2 case "update" => if (areDVsEnabled) 2 else 1 case "delete" => 1 } validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs) prepareAndCommit(downgradeDropObserver) } ThreadUtils.awaitResult(dropFuture, timeout) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 0) validateRowTrackingState(deltaLog, isPresent = false) val targetTable = io.delta.tables.DeltaTable.forPath(dir.getAbsolutePath) val expectedValues = op match { case "insert" => (0 to 199) case "update" => (0 to 99).filterNot(_ == 90) :+ 200 case "delete" => (0 to 99).filterNot(_ == 90) } checkAnswer(targetTable.toDF, expectedValues.map(Row(_))) } } /* * -------------------------------------------> TIME -------------------------------------------> * * Drop Feature * Command Metadata upgrade -------- Unbackfill -------------- Unbackfill --- Downgrade * prepare + commit Batch 1 Batch 2 Commit * prep+commit prep+commit prep+commit * * * Business Txn prep+commit * * -------------------------------------------> TIME -------------------------------------------> */ test("Business txn interleaves between two unbackfill batches") { withTempDir { dir => val deltaLog = createTestTable(dir, numPartitions = 3) val ctx = new TestContext(deltaLog) val table = s"delta.`${dir.getAbsolutePath}`" val dropRowTrackingFn = () => dropRowTrackingTransaction(table) withSQLConf(DeltaSQLConf.DELTA_BACKFILL_MAX_NUM_FILES_PER_COMMIT.key -> "2") { val Seq(dropFuture) = runFunctionsWithOrderingFromObserver(Seq(dropRowTrackingFn)) { case (updateMetadataDropObserver :: Nil) => val unbackfillBatch1DropObserver = new TransactionObserver( OptimisticTransactionPhases.forName("unbackfill-batch-1-drop-txn")) val unbackfillBatch2DropObserver = new TransactionObserver( OptimisticTransactionPhases.forName("unbackfill-batch-2-drop-txn")) val downgradeDropObserver = new TransactionObserver( OptimisticTransactionPhases.forName("downgrade-drop-txn")) updateMetadataDropObserver.setNextObserver( unbackfillBatch1DropObserver, autoAdvance = true) unbackfillBatch1DropObserver.setNextObserver( unbackfillBatch2DropObserver, autoAdvance = true) unbackfillBatch2DropObserver.setNextObserver( downgradeDropObserver, autoAdvance = true) prepareAndCommitWithNextObserverSet(updateMetadataDropObserver) // Block unbackfill batch 1 right after commit. unblockUntilPreCommit(unbackfillBatch1DropObserver) waitForPrecommit(unbackfillBatch1DropObserver) unblockCommit(unbackfillBatch1DropObserver) busyWaitFor(unbackfillBatch1DropObserver.phases.commitPhase.hasLeft, timeout) busyWaitFor(unbackfillBatch1DropObserver.phases.backfillPhase.hasLeft, timeout) // We unbackfilled 2 of the 3 files. validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 1) ThirdPartyInsert(start = 100, end = 110, numPartitions = 1).execute(ctx) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 2) // Allow to proceed to batch 2. unbackfillBatch1DropObserver.phases.postCommitPhase.leave() // Batch 2 picked the single backfilled file by the interleaved txn. prepareAndCommitWithNextObserverSet(unbackfillBatch2DropObserver) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 0) prepareAndCommit(downgradeDropObserver) } ThreadUtils.awaitResult(dropFuture, timeout) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 0) validateRowTrackingState(deltaLog, isPresent = false) val targetTable = io.delta.tables.DeltaTable.forPath(dir.getAbsolutePath) val expectedValues = (0 to 109) checkAnswer(targetTable.toDF, expectedValues.map(Row(_))) } } } /* * -------------------------------------------> TIME -------------------------------------------> * * Drop Feature * Command Metadata upgrade -------- Unbackfill ---- Unbackfill --------------- Downgrade * prepare + commit Batch 1 Batch 2 Commit * prep+commit prepare commit prep+commit * * * Enable Row Tracking * Scenario 1: Metadata * upgrade * Scenario 2: Backfill * * -------------------------------------------> TIME -------------------------------------------> */ for (isScenario1 <- BOOLEAN_DOMAIN) test(s"Enable row tracking during unbackfill - isScenario1: $isScenario1") { withTempDir { dir => val deltaLog = createTestTable(dir, numPartitions = 3) val table = s"delta.`${dir.getAbsolutePath}`" val dropRowTrackingFn = () => dropRowTrackingTransaction(table) withSQLConf(DeltaSQLConf.DELTA_BACKFILL_MAX_NUM_FILES_PER_COMMIT.key -> "2") { val Seq(dropFuture) = runFunctionsWithOrderingFromObserver(Seq(dropRowTrackingFn)) { case (updateMetadataDropObserver :: Nil) => val unbackfillBatch1DropObserver = new TransactionObserver( OptimisticTransactionPhases.forName("unbackfill-batch-1-drop-txn")) val unbackfillBatch2DropObserver = new TransactionObserver( OptimisticTransactionPhases.forName("unbackfill-batch-2-drop-txn")) val downgradeDropObserver = new TransactionObserver( OptimisticTransactionPhases.forName("downgrade-drop-txn")) updateMetadataDropObserver.setNextObserver( unbackfillBatch1DropObserver, autoAdvance = true) unbackfillBatch1DropObserver.setNextObserver( unbackfillBatch2DropObserver, autoAdvance = true) unbackfillBatch2DropObserver.setNextObserver( downgradeDropObserver, autoAdvance = true) prepareAndCommitWithNextObserverSet(updateMetadataDropObserver) prepareAndCommitWithNextObserverSet(unbackfillBatch1DropObserver) unblockUntilPreCommit(unbackfillBatch2DropObserver) waitForPrecommit(unbackfillBatch2DropObserver) if (isScenario1) { // Trying to re-enable row tracking during removal causes the alter table command // to fail. val e = intercept[DeltaIllegalStateException] { enableRowTrackingTransaction(table) } assert(e.getErrorClass === "DELTA_ROW_TRACKING_ILLEGAL_PROPERTY_COMBINATION") } else { // Backfill fails if run together backfill. assertThrows[IllegalStateException] { backfillTransaction(deltaLog) } } // Commit batch 2. unblockCommit(unbackfillBatch2DropObserver) unbackfillBatch2DropObserver.phases.postCommitPhase.leave() waitForCommit(unbackfillBatch2DropObserver) prepareAndCommit(downgradeDropObserver) } ThreadUtils.awaitResult(dropFuture, timeout) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 0) validateRowTrackingState(deltaLog, isPresent = false) } } } /* * -------------------------------------------> TIME -------------------------------------------> * * Drop Feature * Command Metadata upgrade -------- Unbackfill ---------- Downgrade -------------------- * Batch 1 Protocol * prepare + commit prep+commit prep+commit * * * Business Txn prepare commit * * -------------------------------------------> TIME -------------------------------------------> */ for (op <- Seq("insert", "update", "delete")) test(s"Interleaved $op right after protocol downgrade should abort due to protocol change") { withTempDir { dir => val deltaLog = createTestTable(dir) val table = s"delta.`${dir.getAbsolutePath}`" val ctx = new TestContext(deltaLog) val dropRowTrackingFn = () => dropRowTrackingTransaction(table) val Seq(dropFuture) = runFunctionsWithOrderingFromObserver(Seq(dropRowTrackingFn)) { case (updateMetadataDropObserver :: Nil) => val unbackfillDropObserver = new TransactionObserver( OptimisticTransactionPhases.forName("unbackfill-drop-txn")) val downgradeDropObserver = new TransactionObserver( OptimisticTransactionPhases.forName("downgrade-drop-txn")) updateMetadataDropObserver.setNextObserver( unbackfillDropObserver, autoAdvance = true) unbackfillDropObserver.setNextObserver( downgradeDropObserver, autoAdvance = true) prepareAndCommitWithNextObserverSet(updateMetadataDropObserver) prepareAndCommitWithNextObserverSet(unbackfillDropObserver) val businessTxn = op match { case "insert" => ThirdPartyInsert(start = 100, end = 200) case "update" => ThirdPartyUpdate(set = "id = 200", condition = "id = 10") case "delete" => ThirdPartyDelete("id = 10") } val businessTxnFn = () => { businessTxn.execute(ctx) Array.empty[Row] } val Seq(businessTxnFuture) = runFunctionsWithOrderingFromObserver(Seq(businessTxnFn)) { case (businessTxnObserver :: Nil) => unblockUntilPreCommit(businessTxnObserver) waitForPrecommit(businessTxnObserver) prepareAndCommit(downgradeDropObserver) unblockCommit(businessTxnObserver) } val e = intercept[org.apache.spark.SparkException] { ThreadUtils.awaitResult(businessTxnFuture, timeout) } assert(e.getCause.isInstanceOf[ProtocolChangedException]) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 0) } ThreadUtils.awaitResult(dropFuture, timeout) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 0) validateRowTrackingState(deltaLog, isPresent = false) } } /* * -------------------------------------------> TIME -------------------------------------------> * * Unbackfill Batch: prepare commit * Business Txn: prepare + commit * * -------------------------------------------> TIME -------------------------------------------> */ for (op <- Seq("insert", "update", "delete")) test(s"Third party $op that interleaves with unbackfill is resolved") { withTempDir { dir => val deltaLog = createTestTable(dir) val ctx = new TestContext(deltaLog) // Fist stage of row tracking removal: disable the feature. disableRowTracking(dir.getAbsolutePath) val txnA = UnbackfillBatch() val txnB = op match { case "insert" => ThirdPartyInsert(start = 100, end = 200) case "update" => ThirdPartyUpdate("id = 200", "id = 90") case "delete" => ThirdPartyDelete("id = 90") } txnA.start(ctx) txnA.observer.foreach(o => busyWaitFor(o.phases.commitPhase.hasReached, timeout)) txnB.execute(ctx) val expectedFileCountWithRowIDs = op match { case "insert" => 4 case "update" => if (areDVsEnabled) 3 else 2 case "delete" => 2 } validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs) txnA.commit(ctx) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 0) // Unbackfill conflict should be resolved and the downgrade should proceed. DowngradeProtocol(RowTrackingFeature).execute(ctx) validateRowTrackingState(deltaLog, isPresent = false) } } /* * -------------------------------------------> TIME -------------------------------------------> * * Unbackfill Batch prepare + commit * Business Txn prepare commit * * -------------------------------------------> TIME -------------------------------------------> */ for (op <- Seq("insert", "update", "delete")) test(s"Single Unbackfill batch interleaves $op") { withTempDir { dir => val deltaLog = createTestTable(dir, numPartitions = 3) val ctx = new TestContext(deltaLog) disableRowTracking(dir.getAbsolutePath) val businessTxn = op match { case "insert" => ThirdPartyInsert(start = 100, end = 200) case "update" => ThirdPartyUpdate(set = "id = 200", condition = "id = 10") case "delete" => ThirdPartyDelete("id = 10") } businessTxn.start(ctx) UnbackfillBatch().execute(ctx) businessTxn.commit(ctx) val expectedFileCountWithRowIDs = op match { case "insert" => 2 case "update" => if (areDVsEnabled) 2 else 1 case "delete" => 1 } validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs) val targetTable = io.delta.tables.DeltaTable.forPath(dir.getAbsolutePath) val expectedValues = op match { case "insert" => (0 to 199) case "update" => (0 to 99).filterNot(_ == 10) :+ 200 case "delete" => (0 to 99).filterNot(_ == 10) } checkAnswer(targetTable.toDF, expectedValues.map(Row(_))) } } /* * -------------------------------------------> TIME -------------------------------------------> * * Drop Feature * Command Metadata upgrade -------- Unbackfill ----- Downgrade ------------------------- * Batch 1 Protocol * prepare + commit prep+commit Prepare Commit * * * Business Txn prep+commit * * -------------------------------------------> TIME -------------------------------------------> */ for (op <- Seq("insert", "update", "delete")) test(s"Drop feature conflict resolves unbackfills addFiles of interleaved commits ($op)") { withTempDir { dir => val deltaLog = createTestTable(dir) val table = s"delta.`${dir.getAbsolutePath}`" val ctx = new TestContext(deltaLog) val dropRowTrackingFn = () => dropRowTrackingTransaction(table) val Seq(dropFuture) = runFunctionsWithOrderingFromObserver(Seq(dropRowTrackingFn)) { case (updateMetadataDropObserver :: Nil) => val unbackfillDropObserver = new TransactionObserver( OptimisticTransactionPhases.forName("unbackfill-drop-txn")) val downgradeDropObserver = new TransactionObserver( OptimisticTransactionPhases.forName("downgrade-drop-txn")) updateMetadataDropObserver.setNextObserver( unbackfillDropObserver, autoAdvance = true) unbackfillDropObserver.setNextObserver( downgradeDropObserver, autoAdvance = true) prepareAndCommitWithNextObserverSet(updateMetadataDropObserver) prepareAndCommitWithNextObserverSet(unbackfillDropObserver) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 0) unblockUntilPreCommit(downgradeDropObserver) waitForPrecommit(downgradeDropObserver) val businessTxn = op match { case "insert" => ThirdPartyInsert(start = 100, end = 200) case "update" => ThirdPartyUpdate(set = "id = 200", condition = "id = 10") case "delete" => ThirdPartyDelete("id = 10") } businessTxn.execute(ctx) val expectedFileCountWithRowIDs = op match { case "insert" => 2 case "update" => if (areDVsEnabled) 2 else 1 case "delete" => 1 } validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs) unblockCommit(downgradeDropObserver) waitForCommit(downgradeDropObserver) } ThreadUtils.awaitResult(dropFuture, timeout) validateRowTrackingState(deltaLog, isPresent = false) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 0) } } /* * -------------------------------------------> TIME -------------------------------------------> * * Unbackfill Batch: prepare commit * Backfill Batch: prepare + commit * * -------------------------------------------> TIME -------------------------------------------> */ test("Backfill interleaves unbackfill") { withTempDir { dir => val deltaLog = createTestTable(dir) val ctx = new TestContext(deltaLog) // Fist stage of row tracking removal: disable the feature. disableRowTracking(dir.getAbsolutePath) // Add some more data without row IDs. addData(dir.getAbsolutePath, start = 100, end = 200) val txnA = UnbackfillBatch() val txnB = ThirdPartyBackfillBatch() txnA.start(ctx) txnB.execute(ctx) val e = intercept[org.apache.spark.SparkException] { txnA.commit(ctx) } checkError( e.getCause.asInstanceOf[SparkThrowable], "DELTA_ROW_TRACKING_BACKFILL_RUNNING_CONCURRENTLY_WITH_UNBACKFILL", parameters = Map.empty) } } /* * -------------------------------------------> TIME -------------------------------------------> * * Backfill Batch: prepare commit * Unackfill Batch: prepare + commit * * -------------------------------------------> TIME -------------------------------------------> */ test("Unbackfill interleaves backfill") { withTempDir { dir => val deltaLog = createTestTable(dir) val ctx = new TestContext(deltaLog) // Fist stage of row tracking removal: disable the feature. disableRowTracking(dir.getAbsolutePath) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 2) // Add some more data without row IDs. addData(dir.getAbsolutePath, start = 100, end = 200) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 2) val txnA = ThirdPartyBackfillBatch() val txnB = UnbackfillBatch() txnA.start(ctx) txnB.execute(ctx) txnA.commit(ctx) // Backfill backfilled the initial 2 files. The unbackfill unbackfilled the following 2 files. // The backfill at conflict resolution did not find any common files with the conflicting // unbackfill and proceeded. validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 2) // Downgrade commit detects the issue and aborts. val e = intercept[DeltaTableFeatureException] { DowngradeProtocol(RowTrackingFeature).execute(ctx) } assert(e.getErrorClass === "DELTA_FEATURE_DROP_CONFLICT_REVALIDATION_FAIL") } } /* * -------------------------------------------> TIME -------------------------------------------> * * Unbackfill Command: * Unbackfill ------------------------------ Unbackfill ------------------------ * Batch 1 Batch 2 * prepare + commit prepare + commit * * * Business Txn A prepare + commit * Business Txn B prepare + commit * * -------------------------------------------> TIME -------------------------------------------> */ test(s"Unbackfill terminates when small competing txns run concurrently ") { withTempDir { dir => val deltaLog = createTestTable(dir, numPartitions = 4) val ctx = new TestContext(deltaLog) disableRowTracking(dir.getAbsolutePath) val unbackillFn = () => { unBackfillTransaction(deltaLog) Array.empty[Row] } withSQLConf(DeltaSQLConf.DELTA_BACKFILL_MAX_NUM_FILES_PER_COMMIT.key -> "3") { val Seq(unbackfillFuture) = runFunctionsWithOrderingFromObserver(Seq(unbackillFn)) { case (unbackfillBatch1DropObserver :: Nil) => val unbackfillBatch2DropObserver = new TransactionObserver( OptimisticTransactionPhases.forName("unbackfill-batch-2-drop-txn")) unbackfillBatch1DropObserver.setNextObserver( unbackfillBatch2DropObserver, autoAdvance = true) unblockUntilPreCommit(unbackfillBatch1DropObserver) waitForPrecommit(unbackfillBatch1DropObserver) unblockCommit(unbackfillBatch1DropObserver) busyWaitFor(unbackfillBatch1DropObserver.phases.commitPhase.hasLeft, timeout) busyWaitFor(unbackfillBatch1DropObserver.phases.backfillPhase.hasLeft, timeout) // We started with 4 files and unbackfilled 3. validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 1) // Adds 1 backfilled file. ThirdPartyInsert(start = 100, end = 110, numPartitions = 1).execute(ctx) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 2) unbackfillBatch1DropObserver.phases.postCommitPhase.leave() // Finds both files and unbackfills them. It terminates since the number of found // files is less than the max batch size. prepareAndCommit(unbackfillBatch2DropObserver) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 0) // Adds 1 backfilled file. ThirdPartyInsert(start = 110, end = 120, numPartitions = 1).execute(ctx) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 1) } ThreadUtils.awaitResult(unbackfillFuture, timeout) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 1) } } } /* * -------------------------------------------> TIME -------------------------------------------> * * Unbackfill Command: * Unbackfill ------------- Unbackfill ------------ Unbackfill ----------------- * Batch 1 Batch 2 Batch 3 * prep+commit prep+commit prep+commit * * * Business Txn A prep+commit * Business Txn B prep+commit * Business Txn C prep+commit * * -------------------------------------------> TIME -------------------------------------------> */ test(s"Unbackfill terminates when large competing txns run concurrently") { withTempDir { dir => val deltaLog = createTestTable(dir, numPartitions = 4) val ctx = new TestContext(deltaLog) disableRowTracking(dir.getAbsolutePath) val unbackillFn = () => { unBackfillTransaction(deltaLog) Array.empty[Row] } withSQLConf( DeltaSQLConf.DELTA_BACKFILL_MAX_NUM_FILES_PER_COMMIT.key -> "3", DeltaSQLConf.DELTA_BACKFILL_MAX_NUM_FILES_FACTOR.key -> "2") { val Seq(unbackfillFuture) = runFunctionsWithOrderingFromObserver(Seq(unbackillFn)) { case (unbackfillBatch1DropObserver :: Nil) => val unbackfillBatch2DropObserver = new TransactionObserver( OptimisticTransactionPhases.forName("unbackfill-batch-2-drop-txn")) val unbackfillBatch3DropObserver = new TransactionObserver( OptimisticTransactionPhases.forName("unbackfill-batch-3-drop-txn")) unbackfillBatch1DropObserver.setNextObserver( unbackfillBatch2DropObserver, autoAdvance = true) unbackfillBatch2DropObserver.setNextObserver( unbackfillBatch3DropObserver, autoAdvance = true) unblockUntilPreCommit(unbackfillBatch1DropObserver) waitForPrecommit(unbackfillBatch1DropObserver) unblockCommit(unbackfillBatch1DropObserver) busyWaitFor(unbackfillBatch1DropObserver.phases.commitPhase.hasLeft, timeout) busyWaitFor(unbackfillBatch1DropObserver.phases.backfillPhase.hasLeft, timeout) // We started with 4 files and unbackfilled 3. validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 1) // Adds 4 backfilled files (5 in total left). ThirdPartyInsert(start = 100, end = 150, numPartitions = 4).execute(ctx) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 5) unbackfillBatch1DropObserver.phases.postCommitPhase.leave() // Start batch 2. unblockUntilPreCommit(unbackfillBatch2DropObserver) waitForPrecommit(unbackfillBatch2DropObserver) unblockCommit(unbackfillBatch2DropObserver) busyWaitFor(unbackfillBatch2DropObserver.phases.commitPhase.hasLeft, timeout) busyWaitFor(unbackfillBatch2DropObserver.phases.backfillPhase.hasLeft, timeout) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 2) // Adds 4 backfilled files (6 in total left). ThirdPartyInsert(start = 150, end = 200, numPartitions = 4).execute(ctx) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 6) unbackfillBatch2DropObserver.phases.postCommitPhase.leave() // Start batch 3. Although there more than 3 files left to unbackfill the // job terminates. This is because we reached the max number of files to unbackfill // (Number of initial files in the table * 2). prepareAndCommit(unbackfillBatch3DropObserver) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 3) // Adds 2 backfilled files (5 in total left). ThirdPartyInsert(start = 200, end = 220, numPartitions = 2).execute(ctx) } ThreadUtils.awaitResult(unbackfillFuture, timeout) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 5) } } } /* * -------------------------------------------> TIME -------------------------------------------> * * Row Tracking Enablement: * Protocol ------------- Backfill ---------------------------- Alter Table --- * Upgrade * prepare + commit prepare + commit prepare + commit * * * Business Txn A prepare + commit * * -------------------------------------------> TIME -------------------------------------------> */ test(s"Row tracking enablement fails when not all files are backfilled") { withTempDir { dir => val deltaLog = createTestTable(dir, rowTrackingEnabled = false) val enablementFn = () => { enableRowTrackingTransaction(s"delta.`${dir.getAbsolutePath}`") Array.empty[Row] } val Seq(enablementFuture) = runFunctionsWithOrderingFromObserver(Seq(enablementFn)) { case (protocolUpgradeObserver :: Nil) => val backfillObserver = new TransactionObserver( OptimisticTransactionPhases.forName("backfill-txn")) val alterTableObserver = new TransactionObserver( OptimisticTransactionPhases.forName("alter-table-txn")) protocolUpgradeObserver.setNextObserver(backfillObserver, autoAdvance = true) backfillObserver.setNextObserver(alterTableObserver, autoAdvance = true) prepareAndCommitWithNextObserverSet(protocolUpgradeObserver) unblockUntilPreCommit(backfillObserver) waitForPrecommit(backfillObserver) unblockCommit(backfillObserver) busyWaitFor(backfillObserver.phases.commitPhase.hasLeft, timeout) busyWaitFor(backfillObserver.phases.backfillPhase.hasLeft, timeout) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 2) // Add 2 non-backfilled files. val table = DeltaTableV2(spark, new Path(dir.getAbsolutePath)) val propertiesToSet = Map(DeltaConfigs.ROW_TRACKING_SUSPENDED.key -> "true") AlterTableSetPropertiesDeltaCommand(table, propertiesToSet).run(spark) addData(dir.getAbsolutePath, start = 100, end = 200) val propertiesToUnSet = Map(DeltaConfigs.ROW_TRACKING_SUSPENDED.key -> "false") AlterTableSetPropertiesDeltaCommand(table, propertiesToUnSet).run(spark) // No row IDs were generated. validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 2) backfillObserver.phases.postCommitPhase.leave() // Alter table should throw an exception since not all files were backfilled. unblockUntilPreCommit(alterTableObserver) } val e = intercept[org.apache.spark.SparkException] { ThreadUtils.awaitResult(enablementFuture, timeout) } assert(e.getCause.isInstanceOf[ProtocolChangedException]) validateRowTrackingMetadataInAddFiles(deltaLog, expectedFileCountWithRowIDs = 2) } } } class RowTrackingRemovalConcurrencyWithoutDVsSuite extends RowTrackingRemovalConcurrencySuite { override protected val areDVsEnabled = false } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowTrackingRemovalSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.rowid import java.util.concurrent.TimeUnit import org.apache.spark.sql.delta.{DeltaConfigs, DeltaIllegalStateException, DeltaLog, DeltaOperations, MaterializedRowCommitVersion, MaterializedRowId, RowId, RowTrackingFeature} import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.RowId.RowTrackingMetadataDomain import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.AlterTableSetPropertiesDeltaCommand import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import io.delta.tables.DeltaTable import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.sql.{DataFrame, QueryTest} import org.apache.spark.sql.functions.{expr, lit} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.ManualClock trait RowTrackingRemovalSuiteBase extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { protected override def sparkConf: SparkConf = super.sparkConf .set(DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey, "true") def validateRowTrackingState(deltaLog: DeltaLog, isPresent: Boolean): Unit = { val snapshot = deltaLog.update() val configuration = snapshot.metadata.configuration val allFiles = snapshot.allFiles.collect() assert(RowId.isSupported(snapshot.protocol) === isPresent) assert(!configuration.contains(DeltaConfigs.ROW_TRACKING_SUSPENDED.key)) if (isPresent) { assert(DeltaConfigs.ROW_TRACKING_ENABLED.fromMetaData(snapshot.metadata)) assert(configuration.contains(MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP)) assert(configuration.contains(MaterializedRowCommitVersion.MATERIALIZED_COLUMN_NAME_PROP)) assert(RowTrackingMetadataDomain .fromSnapshot(snapshot) .forall(_.rowIdHighWaterMark > RowId.MISSING_HIGH_WATER_MARK)) assert(allFiles.forall(a => a.baseRowId.isDefined && a.defaultRowCommitVersion.isDefined)) } else { assert(!configuration.contains(DeltaConfigs.ROW_TRACKING_ENABLED.key)) assert(!configuration.contains(DeltaConfigs.ROW_TRACKING_SUSPENDED.key)) assert(!configuration.contains(MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP)) assert(!configuration.contains(MaterializedRowCommitVersion.MATERIALIZED_COLUMN_NAME_PROP)) assert(RowTrackingMetadataDomain.fromSnapshot(snapshot).isEmpty) assert(allFiles.forall(a => a.baseRowId.isEmpty && a.defaultRowCommitVersion.isEmpty)) } } def dropRowTracking(deltaLog: DeltaLog, truncateHistory: Boolean = false): Unit = { val sqlText = s""" |ALTER TABLE delta.`${deltaLog.dataPath}` |DROP FEATURE ${RowTrackingFeature.name} |${if (truncateHistory) "TRUNCATE HISTORY" else ""} |""".stripMargin sql(sqlText) } def addData(path: String, start: Long, end: Long): Unit = { spark.range(start, end, step = 1, numPartitions = 2) .write .format("delta") .mode("append") .save(path) } } class RowTrackingRemovalSuite extends RowTrackingRemovalSuiteBase { test("Basic row tracking removal") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) addData(dir.getAbsolutePath, 0, 10) assert(RowId.isSupported(deltaLog.update().protocol)) dropRowTracking(deltaLog) validateRowTrackingState(deltaLog, isPresent = false) } } test("Remove row tracking and then re-enable it") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) addData(dir.getAbsolutePath, 0, 10) addData(dir.getAbsolutePath, 10, 20) val table = DeltaTable.forPath(spark, dir.getAbsolutePath) table.update(expr("id == 2"), Map("id" -> lit(200))) // Store the old materialized column names. val configuration = deltaLog.update().metadata.configuration val oldMaterializedRowIdName = configuration(MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP) val oldMaterializedRowCommitVersionName = configuration(MaterializedRowCommitVersion.MATERIALIZED_COLUMN_NAME_PROP) dropRowTracking(deltaLog) addData(dir.getAbsolutePath, 20, 30) validateRowTrackingState(deltaLog, isPresent = false) // Re-enable row tracking. sql( s"""ALTER TABLE delta.`${dir.getAbsolutePath}` |SET TBLPROPERTIES( |${DeltaConfigs.ROW_TRACKING_ENABLED.key} = 'true' |)""".stripMargin) addData(dir.getAbsolutePath, 30, 40) validateRowTrackingState(deltaLog, isPresent = true) // Make sure the materialized column names are different. val newConfiguration = deltaLog.update().metadata.configuration val newMaterializedRowIdName = newConfiguration(MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP) val newMaterializedRowCommitVersionName = newConfiguration(MaterializedRowCommitVersion.MATERIALIZED_COLUMN_NAME_PROP) assert(newMaterializedRowIdName != oldMaterializedRowIdName) assert(newMaterializedRowCommitVersionName != oldMaterializedRowCommitVersionName) } } // This test verifies we can recover a drop feature failure. this is for the scenario // the user decides to re-enable row tracking instead of retrying drop feature. test("Row tracking can recover from suspension") { import org.apache.spark.sql.delta.implicits._ withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) def getMaterializedRowId(df: DataFrame, id: Long): Long = { df .filter(s"id == $id") .select(RowId.QUALIFIED_COLUMN_NAME) .as[Long] .collect() .head } def getHighWatermark(): Long = { RowId.extractHighWatermark(deltaLog.update()).getOrElse { throw new IllegalStateException("High watermark is missing") } } addData(dir.getAbsolutePath, 0, 10) val table = DeltaTable.forPath(spark, dir.getAbsolutePath) // These operations should materialize row IDs. table.update(expr("id == 2"), Map("id" -> lit(200))) table.delete("id == 4") val watermarkPreDisablement = getHighWatermark() val materializedRowIdPreDisablement = getMaterializedRowId(table.toDF, 200) AlterTableSetPropertiesDeltaCommand( table = DeltaTableV2(spark, deltaLog.dataPath), configuration = Map( DeltaConfigs.ROW_TRACKING_ENABLED.key -> "false", DeltaConfigs.ROW_TRACKING_SUSPENDED.key -> "true")) .run(spark) // Should not generate row IDs. addData(dir.getAbsolutePath, 10, 15) table.update(expr("id == 200"), Map("id" -> lit(300))) assert(getHighWatermark() === watermarkPreDisablement) // Lift row identity generation suspension. AlterTableSetPropertiesDeltaCommand( table = DeltaTableV2(spark, deltaLog.dataPath), configuration = Map(DeltaConfigs.ROW_TRACKING_SUSPENDED.key -> "false")) .run(spark) // Backfill. AlterTableSetPropertiesDeltaCommand( table = DeltaTableV2(spark, deltaLog.dataPath), configuration = Map(DeltaConfigs.ROW_TRACKING_ENABLED.key -> "true")) .run(spark) // Row tracking continued from previous high watermark. assert(getHighWatermark() > watermarkPreDisablement) // Row tracking does not guarantee the materialized row ID will be the same after // re-enablement. assert(getMaterializedRowId(table.toDF, 300) != materializedRowIdPreDisablement) // All add files should have row IDs. assert(deltaLog.update().allFiles.where("baseRowId IS NULL").count() === 0) } } for (dvsEnabled <- BOOLEAN_DOMAIN) test(s"Property `delta.rowTrackingSuspended` is respected - dvsEnabled: $dvsEnabled") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) addData(dir.getAbsolutePath, 0, 10) sql( s"""ALTER TABLE delta.`${dir.getAbsolutePath}` |SET TBLPROPERTIES( |${DeltaConfigs.ROW_TRACKING_ENABLED.key} = 'false', |${DeltaConfigs.ROW_TRACKING_SUSPENDED.key} = 'true', |${DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key} = '$dvsEnabled' |)""".stripMargin) val filesWithRowIds = deltaLog.update().allFiles.collect().toSet addData(dir.getAbsolutePath, 10, 15) val targetTable = DeltaTable.forPath(dir.getAbsolutePath) targetTable.update(expr("id IN (2, 12)"), Map("id" -> lit(200))) targetTable.delete("id == 3") val allFiles = deltaLog.update() .allFiles .collect() .filterNot(filesWithRowIds.contains) assert(allFiles.forall(a => a.baseRowId.isEmpty && a.defaultRowCommitVersion.isEmpty)) } } test(s"Cannot enable both configurations at the same time") { withTempDir { dir => addData(dir.getAbsolutePath, 0, 10) sql( s"""ALTER TABLE delta.`${dir.getAbsolutePath}` |SET TBLPROPERTIES( |${DeltaConfigs.ROW_TRACKING_ENABLED.key} = 'true' |)""".stripMargin) assertThrows[IllegalStateException] { sql( s"""ALTER TABLE delta.`${dir.getAbsolutePath}` |SET TBLPROPERTIES( |${DeltaConfigs.ROW_TRACKING_SUSPENDED.key} = 'true' |)""".stripMargin) } sql( s"""ALTER TABLE delta.`${dir.getAbsolutePath}` |SET TBLPROPERTIES( |${DeltaConfigs.ROW_TRACKING_ENABLED.key} = 'false', |${DeltaConfigs.ROW_TRACKING_SUSPENDED.key} = 'true' |)""".stripMargin) assertThrows[IllegalStateException] { sql( s"""ALTER TABLE delta.`${dir.getAbsolutePath}` |SET TBLPROPERTIES( |${DeltaConfigs.ROW_TRACKING_ENABLED.key} = 'true' |)""".stripMargin) } } } test("Third party writer enables row tracking without disabling suspension property") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) addData(dir.getAbsolutePath, 0, 10) // Third party writer messes up the configs. val txn = deltaLog.startTransaction(catalogTableOpt = None) val newConfiguration = txn.metadata.configuration ++ Map( DeltaConfigs.ROW_TRACKING_ENABLED.key -> "true", DeltaConfigs.ROW_TRACKING_SUSPENDED.key -> "true" ) txn.updateMetadata(txn.metadata.copy(configuration = newConfiguration)) txn.commit(Seq.empty, DeltaOperations.ManualUpdate) val e = intercept[DeltaIllegalStateException] { addData(dir.getAbsolutePath, 10, 20) } checkError( e, "DELTA_ROW_TRACKING_ILLEGAL_PROPERTY_COMBINATION", parameters = Map( "property1" -> DeltaConfigs.ROW_TRACKING_ENABLED.key, "property2" -> DeltaConfigs.ROW_TRACKING_SUSPENDED.key)) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/rowid/RowTrackingUpdateSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.rowid import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.rowtracking.RowTrackingEnabled import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.SparkConf import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.{col, lit} trait RowTrackingUpdateSuiteBase extends RowIdTestUtils with RowTrackingEnabled { protected def dvsEnabled: Boolean = false protected val numRowsTarget = 3000 protected val numRowsPerFile = 250 protected val numFiles: Int = numRowsTarget / numRowsPerFile protected val targetTableName = "target" override protected def sparkConf: SparkConf = { super.sparkConf .set(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey, dvsEnabled.toString) .set(DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS.key, dvsEnabled.toString) .set(DeltaSQLConf.UPDATE_USE_PERSISTENT_DELETION_VECTORS.key, dvsEnabled.toString) .set(DeltaSQLConf.MERGE_USE_PERSISTENT_DELETION_VECTORS.key, dvsEnabled.toString) } protected def writeTestTable( tableName: String, isPartitioned: Boolean, lastModifiedVersion: Long = 0L): Unit = { // Disable optimized writes to write out the specified number of files. withSQLConf(DeltaSQLConf.DELTA_OPTIMIZE_WRITE_ENABLED.key -> "false") { val df = spark.range( start = 0, end = numRowsTarget, step = 1, numPartitions = numFiles) .withColumn("last_modified_version", lit(lastModifiedVersion)) .withColumn("partition", (col("id") / (numRowsTarget / 3)).cast("int")) .write.format("delta") if (isPartitioned) { df.partitionBy("partition").saveAsTable(tableName) } else { df.saveAsTable(tableName) } val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName)) assert(snapshot.allFiles.count() === numFiles) } } protected def withRowIdTestTable(isPartitioned: Boolean)(f: => Unit): Unit = { withTable(targetTableName) { writeTestTable(targetTableName, isPartitioned) f } } protected def checkAndExecuteUpdate( tableName: String, condition: Option[String], newVersion: Long = 1L): Unit = { val expectedRowIds = spark.read.table(tableName).select("id", RowId.QUALIFIED_COLUMN_NAME).collect() val log = DeltaLog.forTable(spark, TableIdentifier(targetTableName)) checkRowTrackingMarkedAsPreservedForCommit(log) { checkFileActionInvariantBeforeAndAfterOperation(log) { executeUpdate(tableName, condition, newVersion) } } val actualRowIds = spark.read.table(tableName).select("id", RowId.QUALIFIED_COLUMN_NAME) checkAnswer(actualRowIds, expectedRowIds) assertRowIdsAreValid(log) val actualRowCommitVersions = spark.read.table(tableName).select("id", RowCommitVersion.QUALIFIED_COLUMN_NAME) val expectedRowCommitVersions = spark.read.table(tableName).select("id", "last_modified_version").collect() checkAnswer(actualRowCommitVersions, expectedRowCommitVersions) } protected def executeUpdate(tableName: String, where: Option[String], newVersion: Long): Unit = { val whereClause = where.map(c => s"WHERE $c").getOrElse("") sql(s"""UPDATE $tableName as t |SET last_modified_version = $newVersion |$whereClause""".stripMargin) } } trait RowTrackingUpdateCommonTests extends RowTrackingUpdateSuiteBase { for { isPartitioned <- BOOLEAN_DOMAIN whereClause <- Seq( Some(s"id < ${(numFiles / 2) * numRowsPerFile}"), // 50% of files match Some(s"id < ${numRowsPerFile / 2}"), // One file matches None // No condition, 100% of files match ) } { test(s"Preserves row IDs, whereClause = $whereClause, isPartitioned = $isPartitioned") { withRowIdTestTable(isPartitioned = isPartitioned) { checkAndExecuteUpdate(tableName = targetTableName, condition = whereClause) } } } for (isPartitioned <- BOOLEAN_DOMAIN) test(s"Preserves row IDs across multiple updates, isPartitioned = $isPartitioned") { withRowIdTestTable(isPartitioned = false) { checkAndExecuteUpdate(targetTableName, condition = Some("id % 20 = 0")) checkAndExecuteUpdate(targetTableName, condition = Some("id % 10 = 0"), newVersion = 2L) } } test("Preserves row IDs in update on partition column, whole file update") { withRowIdTestTable(isPartitioned = true) { checkAndExecuteUpdate(tableName = targetTableName, condition = Some("partition = 0")) } } test(s"Preserves row IDs on unpartitioned table with optimized writes") { withRowIdTestTable(isPartitioned = false) { val whereClause = Some(s"id = 0 OR id = $numRowsTarget - 1") withSQLConf( DeltaSQLConf.DELTA_OPTIMIZE_WRITE_ENABLED.key -> "true") { checkAndExecuteUpdate(targetTableName, condition = whereClause) } val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(targetTableName)) val expectedNumFiles = if (dvsEnabled) numFiles + 1 else numFiles - 1 assert(snapshot.allFiles.count() === expectedNumFiles) } } test("Row tracking marked as not preserved when row tracking disabled") { withRowTrackingEnabled(enabled = false) { withRowIdTestTable(isPartitioned = false) { val log = DeltaLog.forTable(spark, TableIdentifier(targetTableName)) assert( !rowTrackingMarkedAsPreservedForCommit(log)(executeUpdate( targetTableName, where = None, newVersion = -1L))) } } } test("Preserving Row Tracking - Subqueries are not supported in UPDATE") { withRowTrackingEnabled(enabled = true) { withRowIdTestTable(isPartitioned = false) { val ex = intercept[AnalysisException] { checkAndExecuteUpdate( tableName = targetTableName, condition = Some( s"""id in (SELECT id FROM $targetTableName s WHERE s.id = 0 OR s.id = $numRowsPerFile)""")) }.getMessage assert(ex.contains("Subqueries are not supported in the UPDATE")) } } } for { isPartitioned <- BOOLEAN_DOMAIN } { test("UPDATE preserves Row Tracking on tables enabled using backfill, " + s"isPartitioned=$isPartitioned") { withSQLConf(DeltaSQLConf.DELTA_ROW_TRACKING_BACKFILL_ENABLED.key -> "true") { // This is the expected delta log history by the end of the test. // version 0: Table Creation // version 1: Protocol upgrade // version 2: Backfill commit // version 3: Metadata upgrade (tbl properties) // version 4: Update val backfillCommitVersion = 2L withRowTrackingEnabled(enabled = false) { withTable(targetTableName) { writeTestTable( targetTableName, isPartitioned, lastModifiedVersion = backfillCommitVersion) val (log, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(targetTableName)) assert(!RowTracking.isEnabled(snapshot.protocol, snapshot.metadata)) validateSuccessfulBackfillMetrics(expectedNumSuccessfulBatches = 1) { triggerBackfillOnTestTableUsingAlterTable(targetTableName, numRowsTarget, log) } val whereClause = s"id < ${numRowsPerFile / 2}" // The newVersion should be 4, the commit associated with the UPDATE. val newVersion = 4L checkAndExecuteUpdate( tableName = targetTableName, condition = Some(whereClause), newVersion) } } } } } } trait RowTrackingUpdateDVMixin extends RowTrackingUpdateSuiteBase with DeletionVectorsTestUtils { override protected def dvsEnabled: Boolean = true } // Base trait for UPDATE tests with row tracking. trait UpdateWithRowTrackingOverrides extends UpdateSQLMixin { override def excluded: Seq[String] = super.excluded ++ Seq( // TODO: UPDATE on views can't find metadata column "test update on temp view - view with too many internal aliases - Dataset TempView", "test update on temp view - view with too many internal aliases - SQL TempView", "test update on temp view - view with too many internal aliases " + "with write amplification reduction - Dataset TempView", "test update on temp view - view with too many internal aliases " + "with write amplification reduction - SQL TempView", "test update on temp view - basic - Partition=true - SQL TempView", "test update on temp view - basic - Partition=false - SQL TempView", "test update on temp view - superset cols - Dataset TempView", "test update on temp view - superset cols - SQL TempView", "test update on temp view - nontrivial projection - Dataset TempView", "test update on temp view - nontrivial projection - SQL TempView", "test update on temp view - nontrivial projection " + "with write amplification reduction - Dataset TempView", "test update on temp view - nontrivial projection " + "with write amplification reduction - SQL TempView", "update a SQL temp view", // Checks file size written out "usage metrics" ) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/rowtracking/DefaultRowCommitVersionSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.rowtracking import scala.collection.mutable import org.apache.spark.sql.delta.{DeltaConfigs, DeltaLog, RowTrackingFeature} import org.apache.spark.sql.delta.DeltaOperations.ManualUpdate import org.apache.spark.sql.delta.actions.{AddFile, Metadata, Protocol, RemoveFile} import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils.{TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION} import org.apache.spark.sql.delta.rowid.RowIdTestUtils import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.datasources.parquet.ParquetTest import org.apache.spark.sql.functions.col import org.apache.spark.sql.test.SharedSparkSession class DefaultRowCommitVersionSuite extends QueryTest with SharedSparkSession with ParquetTest with RowIdTestUtils { import testImplicits._ def expectedCommitVersionsForAllFiles(deltaLog: DeltaLog): Map[String, Long] = { val commitVersionForFiles = mutable.Map.empty[String, Long] deltaLog.getChanges( startVersion = 0, catalogTableOpt = None).foreach { case (commitVersion, actions) => actions.foreach { case a: AddFile if !commitVersionForFiles.contains(a.path) => commitVersionForFiles += a.path -> commitVersion case r: RemoveFile if commitVersionForFiles.contains(r.path) => assert(r.defaultRowCommitVersion.contains(commitVersionForFiles(r.path))) case _ => // Do nothing } } commitVersionForFiles.toMap } test("defaultRowCommitVersion is not set when feature is disabled") { withRowTrackingEnabled(enabled = false) { withTempDir { tempDir => spark.range(start = 0, end = 100, step = 1, numPartitions = 1) .write.format("delta").mode("overwrite").save(tempDir.getAbsolutePath) spark.range(start = 100, end = 200, step = 1, numPartitions = 1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, tempDir) deltaLog.update().allFiles.collect().foreach { f => assert(f.defaultRowCommitVersion.isEmpty) } } } } test("checkpoint preserves defaultRowCommitVersion") { withRowTrackingEnabled(enabled = true) { withTempDir { tempDir => spark.range(start = 0, end = 100, step = 1, numPartitions = 1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) spark.range(start = 100, end = 200, step = 1, numPartitions = 1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) spark.range(start = 200, end = 300, step = 1, numPartitions = 1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, tempDir) val commitVersionForFiles = expectedCommitVersionsForAllFiles(deltaLog) deltaLog.update().allFiles.collect().foreach { f => assert(f.defaultRowCommitVersion.contains(commitVersionForFiles(f.path))) } deltaLog.checkpoint(deltaLog.update()) deltaLog.update().allFiles.collect().foreach { f => assert(f.defaultRowCommitVersion.contains(commitVersionForFiles(f.path))) } } } } test("data skipping reads defaultRowCommitVersion") { withRowTrackingEnabled(enabled = true) { withTempDir { tempDir => spark.range(start = 0, end = 100, step = 1, numPartitions = 1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) spark.range(start = 100, end = 200, step = 1, numPartitions = 1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) spark.range(start = 200, end = 300, step = 1, numPartitions = 1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, tempDir) val commitVersionForFiles = expectedCommitVersionsForAllFiles(deltaLog) val filters = Seq(col("id = 150").expr) val scan = deltaLog.update().filesForScan(filters) scan.files.foreach { f => assert(f.defaultRowCommitVersion.contains(commitVersionForFiles(f.path))) } } } } test("clone does not preserve default row commit versions") { withRowTrackingEnabled(enabled = true) { withTempDir { sourceDir => spark.range(start = 0, end = 100, step = 1, numPartitions = 1) .write.format("delta").mode("append").save(sourceDir.getAbsolutePath) spark.range(start = 100, end = 200, step = 1, numPartitions = 1) .write.format("delta").mode("append").save(sourceDir.getAbsolutePath) spark.range(start = 200, end = 300, step = 1, numPartitions = 1) .write.format("delta").mode("append").save(sourceDir.getAbsolutePath) withTable("target") { spark.sql(s"CREATE TABLE target SHALLOW CLONE delta.`${sourceDir.getAbsolutePath}` " + s"TBLPROPERTIES ('${DeltaConfigs.ROW_TRACKING_ENABLED.key}' = 'true')") val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier("target")) snapshot.allFiles.collect().foreach { f => assert(f.defaultRowCommitVersion.contains(0L)) } } } } } test("restore does preserve default row commit versions") { withRowTrackingEnabled(enabled = true) { withTempDir { tempDir => spark.range(start = 0, end = 100, step = 1, numPartitions = 1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) spark.range(start = 100, end = 200, step = 1, numPartitions = 1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) spark.range(start = 200, end = 300, step = 1, numPartitions = 1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, tempDir) val commitVersionForFiles = expectedCommitVersionsForAllFiles(deltaLog) spark.sql(s"RESTORE delta.`${tempDir.getAbsolutePath}` TO VERSION AS OF 1") deltaLog.update().allFiles.collect().foreach { f => assert(f.defaultRowCommitVersion.contains(commitVersionForFiles(f.path))) } } } } test("default row commit versions are reassigned on conflict") { withTempDir { tempDir => val deltaLog = DeltaLog.forTable(spark, tempDir) // Initial setup - version 0 val protocol = Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) .withFeature(RowTrackingFeature) val metadata = Metadata() deltaLog.startTransaction().commit(Seq(protocol, metadata), ManualUpdate) // Start a transaction val txn = deltaLog.startTransaction() // Commit two concurrent transactions - version 1 and 2 deltaLog.startTransaction().commit(Nil, ManualUpdate) deltaLog.startTransaction().commit(Nil, ManualUpdate) // Commit the transaction - version 3 val addA = AddFile(path = "a", partitionValues = Map.empty, size = 1, modificationTime = 1, dataChange = true, stats = "{\"numRecords\": 1}") val addB = AddFile(path = "b", partitionValues = Map.empty, size = 1, modificationTime = 1, dataChange = true, stats = "{\"numRecords\": 1}") txn.commit(Seq(addA, addB), ManualUpdate) deltaLog.update().allFiles.collect().foreach { f => assert(f.defaultRowCommitVersion.contains(3)) } } } test("default row commit versions are assigned when concurrent txn enables row tracking") { withTempDir { tempDir => val deltaLog = DeltaLog.forTable(spark, tempDir) // Initial setup - version 0 val protocolWithoutRowTracking = Protocol(TABLE_FEATURES_MIN_READER_VERSION, TABLE_FEATURES_MIN_WRITER_VERSION) val metadata = Metadata() deltaLog.startTransaction().commit(Seq(protocolWithoutRowTracking, metadata), ManualUpdate) // Start a transaction val txn = deltaLog.startTransaction() // Commit concurrent transactions enabling row tracking - version 1 and 2 val protocolWithRowTracking = protocolWithoutRowTracking.withFeature(RowTrackingFeature) deltaLog.startTransaction().commit(Seq(protocolWithRowTracking), ManualUpdate) deltaLog.startTransaction().commit(Nil, ManualUpdate) // Commit the transaction - version 3 val addA = AddFile(path = "a", partitionValues = Map.empty, size = 1, modificationTime = 1, dataChange = true, stats = "{\"numRecords\": 1}") val addB = AddFile(path = "b", partitionValues = Map.empty, size = 1, modificationTime = 1, dataChange = true, stats = "{\"numRecords\": 1}") txn.commit(Seq(addA, addB), ManualUpdate) deltaLog.update().allFiles.collect().foreach { f => assert(f.defaultRowCommitVersion.contains(3)) } } } test("can read default row commit versions") { withRowTrackingEnabled(enabled = true) { withTempDir { tempDir => spark.range(start = 0, end = 100, step = 1, numPartitions = 1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) spark.range(start = 100, end = 200, step = 1, numPartitions = 1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) spark.range(start = 200, end = 300, step = 1, numPartitions = 1) .write.format("delta").mode("append").save(tempDir.getAbsolutePath) withAllParquetReaders { checkAnswer( spark.read.format("delta").load(tempDir.getAbsolutePath) .select("id", "_metadata.default_row_commit_version"), (0L until 100L).map(Row(_, 0L)) ++ (100L until 200L).map(Row(_, 1L)) ++ (200L until 300L).map(Row(_, 2L))) } } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/rowtracking/MaterializedColumnSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.rowtracking import org.apache.spark.sql.delta.{DeltaConfigs, DeltaIllegalStateException, DeltaLog, DeltaRuntimeException, MaterializedRowCommitVersion, MaterializedRowId, RowTracking} import org.apache.spark.sql.delta.rowid.RowIdTestUtils import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.datasources.parquet.ParquetTest class MaterializedColumnSuite extends RowIdTestUtils with ParquetTest { private val testTableName = "target" private val testDataColumnName = "test_data" private def withTestTable(testFunction: => Unit): Unit = { withTable(testTableName) { spark.range(end = 10).toDF(testDataColumnName) .write.format("delta").saveAsTable(testTableName) testFunction } } private def getMaterializedRowIdColumnName(tableName: String): Option[String] = { val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName)) snapshot.metadata.configuration.get(MaterializedRowId.MATERIALIZED_COLUMN_NAME_PROP) } private def getMaterializedRowCommitVersionColumnName(tableName: String): Option[String] = { val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName)) snapshot.metadata.configuration.get( MaterializedRowCommitVersion.MATERIALIZED_COLUMN_NAME_PROP) } for ((name, getMaterializedColumnName) <- Map( "row ids" -> getMaterializedRowIdColumnName _, "row commit versions" -> getMaterializedRowCommitVersionColumnName _ )) { test(s"materialized $name column name is stored when row tracking is enabled") { withRowTrackingEnabled(enabled = true) { withTestTable { assert(getMaterializedColumnName(testTableName).isDefined) } } } test(s"materialized $name column name is not stored when row tracking is disabled") { withRowTrackingEnabled(enabled = false) { withTestTable { assert(getMaterializedColumnName(testTableName).isEmpty) } } } test(s"adding a column with the same name as the materialized $name column name fails") { withRowTrackingEnabled(enabled = true) { withTestTable { val materializedColumnName = getMaterializedColumnName(testTableName).get val error = intercept[DeltaRuntimeException] { sql(s"ALTER TABLE $testTableName ADD COLUMN (`$materializedColumnName` BIGINT)") } checkError(error, "DELTA_ADDING_COLUMN_WITH_INTERNAL_NAME_FAILED", parameters = Map("colName" -> materializedColumnName)) } } } test(s"renaming a column to the materialized $name column name fails") { withRowTrackingEnabled(enabled = true) { withSQLConf(DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey -> "name") { withTestTable { val materializedColumnName = getMaterializedColumnName(testTableName).get val error = intercept[DeltaRuntimeException] { sql(s"ALTER TABLE $testTableName " + s"RENAME COLUMN $testDataColumnName TO `$materializedColumnName`") } checkError(error, "DELTA_ADDING_COLUMN_WITH_INTERNAL_NAME_FAILED", parameters = Map("colName" -> materializedColumnName)) } } } } test(s"cloning a table with a column equal to the materialized $name column name fails") { val targetName = "target" val sourceName = "source" withTable(targetName, sourceName) { withRowTrackingEnabled(enabled = true) { spark.range(0).toDF("val") .write.format("delta").saveAsTable(targetName) val materializedColumnName = getMaterializedColumnName(targetName).get spark.range(0).toDF(materializedColumnName) .write.format("delta").saveAsTable(sourceName) val error = intercept[DeltaRuntimeException] { sql(s"CREATE OR REPLACE TABLE $targetName SHALLOW CLONE $sourceName") } checkError(error, "DELTA_ADDING_COLUMN_WITH_INTERNAL_NAME_FAILED", parameters = Map("colName" -> materializedColumnName)) } } } test(s"replace keeps the materialized $name column name") { withRowTrackingEnabled(enabled = true) { withTestTable { val materializedColumnNameBefore = getMaterializedColumnName(testTableName) sql( s""" |CREATE OR REPLACE TABLE $testTableName |USING delta AS |SELECT * FROM VALUES (0), (1) |""".stripMargin) val materializedColumnNameAfter = getMaterializedColumnName(testTableName) assert(materializedColumnNameBefore == materializedColumnNameAfter) } } } test(s"restore keeps the materialized $name column name") { withRowTrackingEnabled(enabled = true) { withTestTable { spark.range(end = 100).toDF(testDataColumnName) .write.format("delta").mode("overwrite").saveAsTable(testTableName) val materializedColumnNameBefore = getMaterializedColumnName(testTableName) io.delta.tables.DeltaTable.forName(testTableName).restoreToVersion(0) val materializedColumnNameAfter = getMaterializedColumnName(testTableName) assert(materializedColumnNameBefore == materializedColumnNameAfter) } } } test(s"clone assigns a materialized $name column when table property is set") { val sourceTableName = "source" val targetTableName = "target" withTable(sourceTableName, targetTableName) { withRowTrackingEnabled(enabled = false) { spark.range(end = 1).write.format("delta").saveAsTable(sourceTableName) assert(getMaterializedColumnName(sourceTableName).isEmpty) sql(s"CREATE OR REPLACE TABLE $targetTableName SHALLOW CLONE $sourceTableName " + s"TBLPROPERTIES ('${DeltaConfigs.ROW_TRACKING_ENABLED.key}' = 'true')") assert(getMaterializedColumnName(targetTableName).isDefined) } } } test(s"clone assigns a materialized $name column when source enables row tracking") { val sourceTableName = "source" val targetTableName = "target" withTable(sourceTableName, targetTableName) { withRowTrackingEnabled(enabled = true) { spark.range(end = 1).toDF("col1").write.format("delta").saveAsTable(sourceTableName) sql(s"CREATE TABLE $targetTableName SHALLOW CLONE $sourceTableName") val sourceTableColumnName = getMaterializedColumnName(sourceTableName) val targetTableColumnName = getMaterializedColumnName(targetTableName) assert(sourceTableColumnName.isDefined) assert(targetTableColumnName.isDefined) assert(sourceTableColumnName !== targetTableColumnName) } } } test(s"clone gives new materialized $name column name for existing empty target table") { val sourceTableName = "source" val targetTableName = "target" withTable(sourceTableName, targetTableName) { withRowTrackingEnabled(enabled = true) { spark.range(end = 1).toDF("col1").write.format("delta").saveAsTable(sourceTableName) spark.range(end = 0).toDF("col2").write.format("delta").saveAsTable(targetTableName) val oldMaterializedColumnName = getMaterializedColumnName(targetTableName).get sql(s"CREATE OR REPLACE TABLE $targetTableName SHALLOW CLONE $sourceTableName") val newMaterializedColumnName = getMaterializedColumnName(targetTableName).get assert(oldMaterializedColumnName === newMaterializedColumnName) val sourceMaterializedColName = getMaterializedColumnName(sourceTableName).get assert(sourceMaterializedColName !== newMaterializedColumnName) } } } test("double clone from an empty source table maintains the same " + s"materialized $name column name") { val sourceTableName = "source" val targetTableName = "target" withTable(sourceTableName, targetTableName) { withRowTrackingEnabled(enabled = true) { spark.range(end = 0).toDF("col1").write.format("delta").saveAsTable(sourceTableName) sql(s"CREATE OR REPLACE TABLE $targetTableName SHALLOW CLONE $sourceTableName " + s"TBLPROPERTIES ('${DeltaConfigs.ROW_TRACKING_ENABLED.key}' = 'true')") val materializedColumnNameBefore = getMaterializedColumnName(targetTableName) val sourceMaterializedColName = getMaterializedColumnName(sourceTableName) assert(sourceMaterializedColName !== materializedColumnNameBefore) sql(s"CREATE OR REPLACE TABLE $targetTableName SHALLOW CLONE $sourceTableName") val materializedColumnNameAfter = getMaterializedColumnName(targetTableName) assert(materializedColumnNameBefore === materializedColumnNameAfter) } } } test(s"self clone of an empty table maintains the same materialized $name column name") { withRowTrackingEnabled(enabled = true) { withTestTable { spark.range(end = 0).toDF(testDataColumnName) .write.format("delta").mode("overwrite").saveAsTable(testTableName) val materializedColumnNameBefore = getMaterializedColumnName(testTableName) sql(s"CREATE OR REPLACE TABLE $testTableName SHALLOW CLONE $testTableName") val materializedColumnNameAfter = getMaterializedColumnName(testTableName) assert(materializedColumnNameBefore === materializedColumnNameAfter) val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(testTableName)) assert(RowTracking.isEnabled(snapshot.protocol, snapshot.metadata)) } } } test(s"can't clone materialized $name column name for existing non-empty target") { val sourceTableName = "source" val targetTableName = "target" withTable(sourceTableName, targetTableName) { withRowTrackingEnabled(enabled = true) { spark.range(end = 1).toDF("col1").write.format("delta").saveAsTable(sourceTableName) spark.range(end = 1).toDF("col2").write.format("delta").saveAsTable(targetTableName) assert(intercept[DeltaIllegalStateException] { sql(s"CREATE OR REPLACE TABLE $targetTableName SHALLOW CLONE $sourceTableName") }.getErrorClass === "DELTA_UNSUPPORTED_NON_EMPTY_CLONE") } } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/rowtracking/RowTrackingConflictResolutionSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.rowtracking import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.DeltaOperations.{ManualUpdate, Truncate} import org.apache.spark.sql.delta.actions.{Action, AddFile} import org.apache.spark.sql.delta.actions.{Metadata, Protocol, RemoveFile} import org.apache.spark.sql.delta.commands.backfill.{BackfillCommandStats, RowTrackingBackfillExecutor} import org.apache.spark.sql.delta.deletionvectors.RoaringBitmapArray import org.apache.spark.sql.delta.rowid.RowIdTestUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import io.delta.exceptions.MetadataChangedException import org.apache.spark.SparkConf import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.expressions.{EqualTo, Literal} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{IntegerType, StructType} class RowTrackingConflictResolutionSuite extends QueryTest with DeletionVectorsTestUtils with SharedSparkSession with RowIdTestUtils { override def sparkConf: SparkConf = super.sparkConf .set(DeltaSQLConf.DELTA_ROW_TRACKING_BACKFILL_ENABLED.key, "true") .set(DeltaSQLConf.FEATURE_ENABLEMENT_CONFLICT_RESOLUTION_ENABLED.key, "true") private val testTableName = "test_table" private def deltaLog = DeltaLog.forTable(spark, TableIdentifier(testTableName)) private def latestSnapshot = deltaLog.update() private def withTestTable(testBlock: => Unit): Unit = { withTable(testTableName) { withRowTrackingEnabled(enabled = false) { // Table is initially empty. spark.range(end = 0).toDF().write.format("delta").saveAsTable(testTableName) testBlock } } } /** Create an AddFile action for testing purposes. */ private def addFile(path: String): AddFile = { AddFile( path = path, partitionValues = Map.empty, size = 1337, modificationTime = 1, dataChange = true, stats = """{ "numRecords": 1 }""" ) } /** Add Row tracking table feature support. */ private def activateRowTracking(): Unit = { require(!latestSnapshot.protocol.isFeatureSupported(RowTrackingFeature)) val protocolWithRowTracking = Protocol(3, 7).withFeature(RowTrackingFeature) deltaLog.upgradeProtocol( None, latestSnapshot, latestSnapshot.protocol.merge(protocolWithRowTracking)) } // Add 'numRecords' records to the table. private def commitRecords(numRecords: Int): Unit = { spark.range(numRecords).write.format("delta").mode("append").saveAsTable(testTableName) } test("Set baseRowId if table feature was committed concurrently") { withTestTable { val txn = deltaLog.startTransaction() activateRowTracking() txn.commit(Seq(addFile(path = "file_path")), DeltaOperations.ManualUpdate) assertRowIdsAreValid(deltaLog) } } test("Set valid baseRowId if table feature and RowIdHighWaterMark are committed concurrently") { withTestTable { val filePath = "file_path" val numConcurrentRecords = 11 val txn = deltaLog.startTransaction() activateRowTracking() commitRecords(numConcurrentRecords) txn.commit(Seq(addFile(filePath)), DeltaOperations.ManualUpdate) assertRowIdsAreValid(deltaLog) val committedAddFile = latestSnapshot.allFiles.collect().filter(_.path == filePath) assert(committedAddFile.size === 1) assert(committedAddFile.head.baseRowId === Some(numConcurrentRecords)) } } test("Conflict resolution if table feature and initial AddFiles are in the same commit") { withTestTable { val filePath = "file_path" val txn = deltaLog.startTransaction() val protocolWithRowTracking = Protocol(3, 7).withFeature(RowTrackingFeature) deltaLog.startTransaction().commit( Seq( latestSnapshot.protocol.merge(protocolWithRowTracking), addFile("other_path") ), DeltaOperations.ManualUpdate) txn.commit(Seq(addFile(filePath)), DeltaOperations.ManualUpdate) assertRowIdsAreValid(deltaLog) val committedAddFile = latestSnapshot.allFiles.collect().filter(_.path == filePath) assert(committedAddFile.size === 1) assert(committedAddFile.head.baseRowId === Some(1)) } } test("Conflict resolution with concurrent INSERT") { withTestTable { val filePath = "file_path" val numInitialRecords = 7 val numConcurrentRecords = 11 activateRowTracking() commitRecords(numInitialRecords) val txn = deltaLog.startTransaction() commitRecords(numConcurrentRecords) txn.commit(Seq(addFile(filePath)), DeltaOperations.ManualUpdate) assertRowIdsAreValid(deltaLog) val committedAddFile = latestSnapshot.allFiles.collect().filter(_.path == filePath) assert(committedAddFile.size === 1) assert(committedAddFile.head.baseRowId === Some(numInitialRecords + numConcurrentRecords)) val currentHighWaterMark = RowId.extractHighWatermark(latestSnapshot).get assert(currentHighWaterMark === numInitialRecords + numConcurrentRecords) } } test("Handle commits that do not bump the high water mark") { withTestTable { val filePath = "file_path" val numInitialRecords = 7 activateRowTracking() commitRecords(numInitialRecords) val txn = deltaLog.startTransaction() val concurrentTxn = deltaLog.startTransaction() val updatedProtocol = latestSnapshot.protocol concurrentTxn.commit(Seq(updatedProtocol), DeltaOperations.ManualUpdate) txn.commit(Seq(addFile(filePath)), DeltaOperations.ManualUpdate) assertRowIdsAreValid(deltaLog) } } /** * Setup a test table with four files and return these files to the caller. */ private def setupTableAndGetAllFiles(log: DeltaLog): (AddFile, AddFile, AddFile, AddFile) = { val f1 = DeltaTestUtils.createTestAddFile(encodedPath = "a", partitionValues = Map("x" -> "1")) val f2 = DeltaTestUtils.createTestAddFile(encodedPath = "b", partitionValues = Map("x" -> "1")) val f3 = DeltaTestUtils.createTestAddFile(encodedPath = "c", partitionValues = Map("x" -> "2")) val f4 = DeltaTestUtils.createTestAddFile(encodedPath = "d", partitionValues = Map("x" -> "2")) val setupActions: Seq[Action] = Seq( Metadata( schemaString = new StructType().add("x", IntegerType).json, partitionColumns = Seq("x")), f1, f2, f3, f4, Action.supportedProtocolVersion( featuresToExclude = Seq(CatalogOwnedTableFeature)).withFeature(RowTrackingFeature) ) log.startTransaction().commit(setupActions, ManualUpdate) (f1, f2, f3, f4) } /** Add a dummy DV to a file in a table. */ private def addDVToFileInTable(deltaLog: DeltaLog, file: AddFile): (AddFile, RemoveFile) = { val dv = writeDV(deltaLog, RoaringBitmapArray(0L)) updateFileDV(file, dv) } /** Execute backfill on the table associated with the delta log passed in. */ private def executeBackfill(log: DeltaLog, backfillTxn: OptimisticTransaction): Unit = { val backfillStats = BackfillCommandStats( backfillTxn.txnId, nameOfTriggeringOperation = DeltaOperations.OP_SET_TBLPROPERTIES) val backfillExecutor = new RowTrackingBackfillExecutor( spark, log, catalogTableOpt = None, backfillTxn.txnId, backfillStats ) backfillExecutor.run(maxNumFilesPerCommit = 4) } /** Check if base row IDs and default row commit versions have been assigned. */ def assertBaseRowIDsAndDefaultRowCommitVersionsAssigned(finalFiles: Seq[AddFile]): Unit = { finalFiles.foreach(addedFile => assert(addedFile.baseRowId.nonEmpty)) finalFiles.foreach(addedFile => assert(addedFile.defaultRowCommitVersion.nonEmpty)) } test("Backfill conflict with a delete, Delete wins") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir.getCanonicalPath) // Setup val (file1, file2, file3, file4) = setupTableAndGetAllFiles(log) // Start Backfill. val backfillTxn = log.startTransaction() // A delete occurs in parallel. Delete wins. val deleteTxn = log.startTransaction() deleteTxn.filterFiles(EqualTo('x, Literal(1)) :: Nil) val deleteActions = Seq(file1.remove, file2.remove) // Truncate is a data-changing operation. deleteTxn.commit(deleteActions, Truncate()) // Finish backfill. executeBackfill(log, backfillTxn) val finalFiles = log.update().allFiles.collect() assertBaseRowIDsAndDefaultRowCommitVersionsAssigned(finalFiles) assertRowIdsAreValid(log) assert(finalFiles.map(_.path).toSet === Seq(file3, file4).map(_.path).toSet) } } test("Backfill conflicts with a delete, Backfill wins") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir.getCanonicalPath) // Setup val (file1, file2, file3, file4) = setupTableAndGetAllFiles(log) // Start delete val deleteTxn = log.startTransaction() deleteTxn.filterFiles(EqualTo('x, Literal(1)) :: Nil) // Backfill occurs in parallel and wins. val backfillTxn = log.startTransaction() executeBackfill(log, backfillTxn) val deleteActions = Seq(file1.remove, file2.remove) // Truncate is a data-changing operation. deleteTxn.commit(deleteActions, Truncate()) val finalFiles = log.update().allFiles.collect() assertBaseRowIDsAndDefaultRowCommitVersionsAssigned(finalFiles) assertRowIdsAreValid(log) assert(finalFiles.map(_.path).toSet === Seq(file3, file4).map(_.path).toSet) } } test("Backfill conflicts with a DV delete, Delete wins") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir.getCanonicalPath) // Setup val (file1, file2, file3, file4) = setupTableAndGetAllFiles(log) enableDeletionVectorsInTable(log) // Start Backfill val backfillTxn = log.startTransaction() // A delete occurs in parallel. Delete wins. val deleteTxn = log.startTransaction() deleteTxn.filterFiles(EqualTo('x, Literal(1)) :: Nil) val (addFile1WithDV, removeFile1) = addDVToFileInTable(log, file1) val (addFile2WithDV, removeFile2) = addDVToFileInTable(log, file2) val deleteActions = Seq(addFile1WithDV, removeFile1, addFile2WithDV, removeFile2) // Truncate is a data-changing operation. deleteTxn.commit(deleteActions, Truncate()) // Finish Backfill executeBackfill(log, backfillTxn) val finalFiles = log.update().allFiles.collect() assertBaseRowIDsAndDefaultRowCommitVersionsAssigned(finalFiles) assertRowIdsAreValid(log) val allFiles = Seq(file1, file2, file3, file4) assert(finalFiles.map(_.path).toSet === allFiles.map(_.path).toSet) } } test("Backfill conflicts with a DV delete, Backfill wins") { withTempDir { dir => val log = DeltaLog.forTable(spark, dir.getCanonicalPath) // Setup val (file1, file2, file3, file4) = setupTableAndGetAllFiles(log) enableDeletionVectorsInTable(log) // Start delete val deleteTxn = log.startTransaction() deleteTxn.filterFiles(EqualTo('x, Literal(1)) :: Nil) // Backfill occurs in parallel and wins. val backfillTxn = log.startTransaction() executeBackfill(log, backfillTxn) val (addFile1WithDV, removeFile1) = addDVToFileInTable(log, file1) val (addFile2WithDV, removeFile2) = addDVToFileInTable(log, file2) val deleteActions = Seq(addFile1WithDV, removeFile1, addFile2WithDV, removeFile2) // Truncate is a data-changing operation. deleteTxn.commit(deleteActions, Truncate()) val finalFiles = log.update().allFiles.collect() assertBaseRowIDsAndDefaultRowCommitVersionsAssigned(finalFiles) assertRowIdsAreValid(log) val allFiles = Seq(file1, file2, file3, file4) assert(finalFiles.map(_.path).toSet === allFiles.map(_.path).toSet) } } private def addRowTrackingEnabledConfigToMetadata(metadata: Metadata): Metadata = { val newConfigs = metadata.configuration updated (DeltaConfigs.ROW_TRACKING_ENABLED.key, "true") metadata.copy(configuration = newConfigs) } private def enableRowTrackingOnlyMetadataUpdate(): Unit = { val txn = deltaLog.startTransaction() val updatedMetadata = addRowTrackingEnabledConfigToMetadata(latestSnapshot.metadata) val tags = Map(DeltaCommitTag.RowTrackingEnablementOnlyTag.key -> "true") txn.updateMetadata(updatedMetadata) txn.commit(Nil, ManualUpdate, tags) } test("RowTrackingEnablementOnly metadata update does not fail txns that don't update metadata") { withTestTable { withSQLConf(DeltaSQLConf.FEATURE_ENABLEMENT_CONFLICT_RESOLUTION_ENABLED.key -> "false") { val txn = deltaLog.startTransaction() activateRowTracking() enableRowTrackingOnlyMetadataUpdate() val rowTrackingPreserved = rowTrackingMarkedAsPreservedForCommit(deltaLog) { txn.commit(Seq(addFile(path = "file_path")), DeltaOperations.ManualUpdate) } assert(!rowTrackingPreserved, "Commits conflicting with a metadata update " + "that enables row tracking only should have row tracking marked as not preserved.") assertRowIdsAreValid(deltaLog) assert(RowTracking.isEnabled(latestSnapshot.protocol, latestSnapshot.metadata)) } } } test("RowTrackingEnablementOnly metadata update fails transactions " + "that perform a metadata update") { withTestTable { activateRowTracking() val numInitialRecords = 7 commitRecords(numInitialRecords) val txn = deltaLog.startTransaction() val newConfigs = Map("key" -> "value") val newMetadata = latestSnapshot.metadata.copy(configuration = newConfigs) txn.updateMetadata(newMetadata) enableRowTrackingOnlyMetadataUpdate() val commitVersionBefore = latestSnapshot.version intercept[MetadataChangedException] { txn.commit(Nil, DeltaOperations.ManualUpdate) } assert(latestSnapshot.version === commitVersionBefore, "the commit should have failed") } } test("RowTrackingEnablementOnly metadata update fails another " + "RowTrackingEnablementOnly metadata update") { withTestTable { activateRowTracking() val txn = deltaLog.startTransaction() val newMetadata = addRowTrackingEnabledConfigToMetadata(latestSnapshot.metadata) txn.updateMetadata(newMetadata) enableRowTrackingOnlyMetadataUpdate() val commitVersionBefore = latestSnapshot.version intercept[MetadataChangedException] { val tags = Map(DeltaCommitTag.RowTrackingEnablementOnlyTag.key -> "true") txn.commit(Nil, DeltaOperations.ManualUpdate, tags) } assert(latestSnapshot.version === commitVersionBefore, "the commit should have failed") } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/rowtracking/RowTrackingReadWriteSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.rowtracking import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.{DeltaConfigs, DeltaLog, RowCommitVersion, RowId} import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.rowid.RowIdTestUtils import org.apache.spark.sql.{AnalysisException, Column, DataFrame, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.execution.datasources.parquet.ParquetTest import org.apache.spark.sql.functions.{col, lit, when} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{LongType, StructType} class RowTrackingReadWriteSuite extends RowIdTestUtils with ParquetTest { private val testTableName = "target" private val testDataColumnName = "test_data" test("select star does not read materialized columns") { withAllParquetReaders { withTable(testTableName) { writeWithMaterializedRowCommitVersionColumns( spark.range(100).toDF(testDataColumnName), rowIdColumn = lit(1L), rowCommitVersionColumn = lit(2L)) withAllParquetReaders { val df = sql(s"SELECT * FROM $testTableName") assert(df.schema.size === 1) assert(df.schema.head.name === testDataColumnName) } } } } test("write and read table without materialized columns") { withTable(testTableName) { withRowTrackingEnabled(enabled = true) { val numRows = 5L spark.range(numRows).toDF(testDataColumnName) .write.format("delta").saveAsTable(testTableName) try { // Confirm that the materialized columns are not present in the Parquet file(s). val deltaLog = DeltaLog.forTable(spark, TableIdentifier(testTableName)) val parquetDataFrame = spark.read.parquet(deltaLog.dataPath.toString) checkAnswer(parquetDataFrame, (0L until numRows).map(Row(_))) } catch { case e: Exception => Thread.sleep(1000 * 1000) } withAllParquetReaders { val df = spark.read.table(testTableName) .select( testDataColumnName, RowId.QUALIFIED_COLUMN_NAME, RowCommitVersion.QUALIFIED_COLUMN_NAME) checkAnswer(df, (0L until numRows).map(i => Row(i, i, 0))) } } } } test("write and read table with all-null materialized columns") { withTable(testTableName) { val numRows = 5L writeWithMaterializedRowCommitVersionColumns( spark.range(numRows).toDF(testDataColumnName), rowIdColumn = lit(null).cast("long"), rowCommitVersionColumn = lit(null).cast("long")) // Confirm that the materialized columns are present in the Parquet file(s). withSQLConf( // The function writeWithMaterializedRowCommitVersionColumns initially creates // an empty table with only the testDataColumnName column. After that, we write // some data to the Delta table, appending some Parquet files that include // both the testDataColumnName and the materialized Row Tracking Columns. // // PARQUET_SCHEMA_MERGING_ENABLED by default is disabled, so what then happens // is we pick a random data file to infer the schema of the underlying parquet // DataFrame of the Delta table in [[ParquetUtils]]. // // Thus, that inferred schema could either contains only the testDataColumnName // column or all three columns, the former leads to wrong result for the // checkAnswer below. // // This is the intended behavior as Delta doesn't give any guarantee // that the Parquet files will all have the same schema, and normally we should // not read raw Parquet files from a Delta table, we are only doing this for // testing purpose. SQLConf.PARQUET_SCHEMA_MERGING_ENABLED.key -> "true" ) { val deltaLog = DeltaLog.forTable(spark, TableIdentifier(testTableName)) val parquetDataFrame = spark.read.parquet(deltaLog.dataPath.toString) checkAnswer(parquetDataFrame, (0L until numRows).map(Row(_, null, null))) } withAllParquetReaders { val df = spark.read.table(testTableName) .select( testDataColumnName, RowId.QUALIFIED_COLUMN_NAME, RowCommitVersion.QUALIFIED_COLUMN_NAME) checkAnswer(df, (0L until 5L).map(i => Row(i, i, 1))) } } } test("write and read table with no-nulls materialized columns") { withTable(testTableName) { val numRows = 100 writeWithMaterializedRowCommitVersionColumns( spark.range(numRows).toDF(testDataColumnName), rowIdColumn = col(testDataColumnName) + 10, rowCommitVersionColumn = col(testDataColumnName) + 100) withAllParquetReaders { val df = spark.table(testTableName) .select( testDataColumnName, RowId.QUALIFIED_COLUMN_NAME, RowCommitVersion.QUALIFIED_COLUMN_NAME) val expectedAnswer = (0L until numRows).map(i => Row(i, i + 10, i + 100)) checkAnswer(df, expectedAnswer) } } } test("write and read table with mixed materialized columns") { withTable(testTableName) { val numRows = 100L writeWithMaterializedRowCommitVersionColumns( spark.range(numRows).toDF(testDataColumnName), rowIdColumn = when(col(testDataColumnName) % 2 === 0, col(testDataColumnName) + 10) .otherwise(null), rowCommitVersionColumn = when(col(testDataColumnName) % 3 === 0, col(testDataColumnName) + 100) .otherwise(null)) withAllParquetReaders { val df = spark.table(testTableName) .select( testDataColumnName, RowId.QUALIFIED_COLUMN_NAME, RowCommitVersion.QUALIFIED_COLUMN_NAME) val expectedAnswer = (0L until numRows).map { i => Row(i, if (i % 2 == 0) i + 10 else i, if (i % 3 == 0) i + 100 else 1) } checkAnswer(df, expectedAnswer) } } } test("read mixed materialized columns with filter") { withTable(testTableName) { val numRows = 100L writeWithMaterializedRowCommitVersionColumns( spark.range(numRows).toDF(testDataColumnName), rowIdColumn = when(col(testDataColumnName) % 2 === 0, lit(1000L)) .otherwise(null), rowCommitVersionColumn = when(col(testDataColumnName) % 3 === 0, lit(2000L)) .otherwise(null)) withAllParquetReaders { // Read the table with a filter that does not match any of the materialized row ids and // row commit versions, but that does match the default-generated ids and versions. val df = spark.table(testTableName) .select( testDataColumnName, RowId.QUALIFIED_COLUMN_NAME, RowCommitVersion.QUALIFIED_COLUMN_NAME) .filter(col(RowId.QUALIFIED_COLUMN_NAME) <= 100 && col(RowCommitVersion.QUALIFIED_COLUMN_NAME) <= 100) val expectedAnswer = (0L until numRows) .filter(i => i % 2 != 0 && i % 3 != 0) .map { i => Row(i, i, 1) } checkAnswer(df, expectedAnswer) } } } test("writing to materialized column requires correct metadata") { withTable(testTableName) { writeWithMaterializedRowCommitVersionColumns( spark.range(100).toDF("id"), rowIdColumn = lit(null), rowCommitVersionColumn = lit(null)) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(testTableName)) val materializedRowIdColumnName = extractMaterializedRowIdColumnName(deltaLog).get val materializedRowCommitVersionColumnName = extractMaterializedRowCommitVersionColumnName(deltaLog).get val insertStmt1 = s"INSERT INTO $testTableName (id, `$materializedRowIdColumnName`)" val errorRowIds = intercept[AnalysisException](sql(insertStmt1 + " VALUES(1, 2)")) checkError( errorRowIds, "UNRESOLVED_COLUMN.WITH_SUGGESTION", parameters = errorRowIds.messageParameters, queryContext = Array(ExpectedContext(insertStmt1, 0, insertStmt1.length - 1))) val insertStmt2 = s"INSERT INTO $testTableName (id, `$materializedRowCommitVersionColumnName`)" val errorRowCommitVersions = intercept[AnalysisException](sql(insertStmt2 + " VALUES(1, 2)")) checkError( errorRowCommitVersions, "UNRESOLVED_COLUMN.WITH_SUGGESTION", parameters = errorRowCommitVersions.messageParameters, queryContext = Array(ExpectedContext(insertStmt2, 0, insertStmt2.length - 1))) } } test("writing to materialized column requires correct name") { withRowTrackingEnabled(enabled = true) { withTable(testTableName) { writeWithMaterializedRowCommitVersionColumns( spark.range(1).toDF(testDataColumnName), rowIdColumn = lit(1L), rowCommitVersionColumn = lit(2L)) checkWritingToMaterializedColumnsRequiresCorrectName() } } } test("writing to materialized column requires row tracking to be enabled") { withTable(testTableName) { writeWithMaterializedRowCommitVersionColumns( spark.range(1).toDF(testDataColumnName), rowIdColumn = lit(1L), rowCommitVersionColumn = lit(2L)) spark.sql(s"ALTER TABLE $testTableName " + s"SET TBLPROPERTIES ('${DeltaConfigs.ROW_TRACKING_ENABLED.key}' = 'false')") checkWritingToMaterializedColumnsRequiresCorrectName() } } private def checkWritingToMaterializedColumnsRequiresCorrectName(): Unit = { val columnNames = Seq( RowId.QUALIFIED_COLUMN_NAME, s"`${FileFormat.METADATA_NAME}`.`${RowId.ROW_ID}`", s"`${RowId.QUALIFIED_COLUMN_NAME}`", RowId.QUALIFIED_COLUMN_NAME, s"`${FileFormat.METADATA_NAME}`.`${RowId.ROW_ID}`", s"`${RowId.QUALIFIED_COLUMN_NAME}`") for (columnName <- columnNames) { // Throw an error because only using the materialized column name is valid. val error = intercept[AnalysisException] { spark .range(end = 1) .toDF(testDataColumnName) .withMaterializedRowIdColumn(columnName, lit(1L)) .write .format("delta") .mode("append") .saveAsTable(testTableName) } checkError( error, "UNRESOLVED_COLUMN.WITH_SUGGESTION", parameters = error.messageParameters) } for (columnName <- columnNames) { // Throw an error because only using the materialized column name is valid. val error = intercept[AnalysisException] { spark .range(end = 1) .toDF(testDataColumnName) .withMaterializedRowCommitVersionColumn(columnName, lit(2L)) .write .format("delta") .mode("append") .saveAsTable(testTableName) } checkError( error, "UNRESOLVED_COLUMN.WITH_SUGGESTION", parameters = error.messageParameters) } } test("write and read with column names similar to row tracking columns in the table schema") { val columnNames = Seq( RowId.QUALIFIED_COLUMN_NAME, RowCommitVersion.QUALIFIED_COLUMN_NAME) for (columnName <- columnNames) { withTable(testTableName) { val numRows = 10L writeWithMaterializedRowCommitVersionColumns( spark.range(numRows).toDF(columnName), rowIdColumn = lit(1L), rowCommitVersionColumn = lit(2L)) withAllParquetReaders { val df = spark.read.table(testTableName).select( s"`$columnName`", RowId.QUALIFIED_COLUMN_NAME, RowCommitVersion.QUALIFIED_COLUMN_NAME) val expectedAnswer = (0L until numRows).map(Row(_, 1L, 2L)) checkAnswer(df, expectedAnswer) } } } } test("write and read with conflicting columns") { withTable(testTableName) { val tableSchema = new StructType() .add("id", LongType) .add(FileFormat.METADATA_NAME, new StructType() .add(RowId.ROW_ID, LongType) .add(RowCommitVersion.METADATA_STRUCT_FIELD_NAME, LongType)) writeWithMaterializedRowCommitVersionColumns( spark.createDataFrame( Seq(Row(1L, (11L, 111L)), Row(2L, (22L, 222L)), Row(3L, (33L, 333L))).asJava, tableSchema), rowIdColumn = lit(-1L), rowCommitVersionColumn = lit(-2L)) withAllParquetReaders { val table = spark.read.table(testTableName) val metadataCol = table.metadataColumn(FileFormat.METADATA_NAME) val userCol = table.col(FileFormat.METADATA_NAME) val df = table.select( col("id"), metadataCol.getField(RowId.ROW_ID), metadataCol.getField(RowCommitVersion.METADATA_STRUCT_FIELD_NAME), userCol.getField(RowId.ROW_ID), userCol.getField(RowCommitVersion.METADATA_STRUCT_FIELD_NAME)) val expectedAnswer = Seq(Row(1, -1, -2, 11, 111), Row(2, -1, -2, 22, 222), Row(3, -1, -2, 33, 333)) checkAnswer(df, expectedAnswer) } } } private def writeWithMaterializedRowCommitVersionColumns( df: DataFrame, rowIdColumn: Column, rowCommitVersionColumn: Column): Unit = { withRowTrackingEnabled(enabled = true) { // Create the table if it does not exist already. df.limit(n = 0) .write.format("delta").mode("append").saveAsTable(testTableName) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(testTableName)) val materializedRowIdColumnName = extractMaterializedRowIdColumnName(deltaLog).get val materializedRowCommitVersionName = extractMaterializedRowCommitVersionColumnName(deltaLog).get df.withMaterializedRowIdColumn( materializedRowIdColumnName, rowIdColumn) .withMaterializedRowCommitVersionColumn( materializedRowCommitVersionName, rowCommitVersionColumn) .write .mode("append") .format("delta") .saveAsTable(testTableName) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/rowtracking/RowTrackingTestUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.rowtracking import org.apache.spark.sql.delta.{DeltaConfigs, RowTrackingFeature} import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.SparkConf import org.apache.spark.sql.QueryTest import org.apache.spark.sql.execution.datasources.parquet.ParquetTest import org.apache.spark.sql.test.SharedSparkSession trait RowTrackingDisabled extends RowTrackingTestUtils { override protected def sparkConf: SparkConf = super.sparkConf .set(DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey, "false") .set(defaultRowTrackingFeatureProperty, "supported") } trait RowTrackingEnabled extends RowTrackingTestUtils { override protected def sparkConf: SparkConf = super.sparkConf .set(DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey, "true") } trait RowTrackingTestUtils extends QueryTest with SharedSparkSession with DeltaSQLTestUtils with ParquetTest { lazy val rowTrackingFeatureName: String = TableFeatureProtocolUtils.propertyKey(RowTrackingFeature) lazy val defaultRowTrackingFeatureProperty: String = TableFeatureProtocolUtils.defaultPropertyKey(RowTrackingFeature) def withRowTrackingEnabled(enabled: Boolean)(f: => Unit): Unit = { withSQLConf(DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey -> enabled.toString)(f) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/schema/CaseSensitivitySuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.schema import java.io.File import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.test.shims.StreamingTestShims.MemoryStream // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql._ import org.apache.spark.sql.functions.col import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.streaming.{StreamingQuery, StreamingQueryException} import org.apache.spark.sql.test.SharedSparkSession class CaseSensitivitySuite extends QueryTest with SharedSparkSession with DeltaSQLTestUtils with DeltaSQLCommandTest { import testImplicits._ private def testWithCaseSensitivity(name: String)(f: => Unit): Unit = { testQuietly(name) { withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { f } withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") { f } } } private def getPartitionValues(allFiles: Dataset[AddFile], colName: String): Array[String] = { allFiles.select(col(s"partitionValues.$colName")).where(col(colName).isNotNull) .distinct().as[String].collect() } testWithCaseSensitivity("case sensitivity of partition fields") { withTempDir { tempDir => val query = "SELECT id + 1 as Foo, id as Bar FROM RANGE(1)" sql(query).write.partitionBy("foo").format("delta").save(tempDir.getAbsolutePath) checkAnswer( sql(query), spark.read.format("delta").load(tempDir.getAbsolutePath) ) val allFiles = DeltaLog.forTable(spark, tempDir.getAbsolutePath).snapshot.allFiles assert(getPartitionValues(allFiles, "Foo") === Array("1")) checkAnswer( spark.read.format("delta").load(tempDir.getAbsolutePath), Row(1L, 0L) ) } } testQuietly("case sensitivity of partition fields (stream)") { // DataStreamWriter auto normalizes partition columns, therefore we don't need to check // case sensitive case withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") { withTempDir { tempDir => val memSource = MemoryStream[(Long, Long)] val stream1 = startStream(memSource.toDF().toDF("Foo", "Bar"), tempDir) try { memSource.addData((1L, 0L)) stream1.processAllAvailable() } finally { stream1.stop() } checkAnswer( spark.read.format("delta").load(tempDir.getAbsolutePath), Row(1L, 0L) ) val allFiles = DeltaLog.forTable(spark, tempDir.getAbsolutePath).snapshot.allFiles assert(getPartitionValues(allFiles, "Foo") === Array("1")) } } } testWithCaseSensitivity("two fields with same name") { withTempDir { tempDir => intercept[AnalysisException] { val query = "SELECT id as Foo, id as foo FROM RANGE(1)" sql(query).write.partitionBy("foo").format("delta").save(tempDir.getAbsolutePath) } } } testWithCaseSensitivity("two fields with same name (stream)") { withTempDir { tempDir => val memSource = MemoryStream[(Long, Long)] val stream1 = startStream(memSource.toDF().toDF("Foo", "foo"), tempDir) try { val e = intercept[StreamingQueryException] { memSource.addData((0L, 0L)) stream1.processAllAvailable() } assert(e.cause.isInstanceOf[AnalysisException]) } finally { stream1.stop() } } } testWithCaseSensitivity("schema merging is case insenstive but preserves original case") { withTempDir { tempDir => val query1 = "SELECT id as foo, id as bar FROM RANGE(1)" sql(query1).write.format("delta").save(tempDir.getAbsolutePath) val query2 = "SELECT id + 1 as Foo, id as bar FROM RANGE(1)" // notice how 'F' is capitalized sql(query2).write.format("delta").mode("append").save(tempDir.getAbsolutePath) val query3 = "SELECT id as bAr, id + 2 as Foo FROM RANGE(1)" // changed order as well sql(query3).write.format("delta").mode("append").save(tempDir.getAbsolutePath) val df = spark.read.format("delta").load(tempDir.getAbsolutePath) checkAnswer( df, Row(0, 0) :: Row(1, 0) :: Row(2, 0) :: Nil ) assert(df.schema.fieldNames === Seq("foo", "bar")) } } testWithCaseSensitivity("schema merging preserving column case (stream)") { withTempDir { tempDir => val memSource = MemoryStream[(Long, Long)] val stream1 = startStream(memSource.toDF().toDF("Foo", "Bar"), tempDir, None) try { memSource.addData((0L, 0L)) stream1.processAllAvailable() } finally { stream1.stop() } val stream2 = startStream(memSource.toDF().toDF("foo", "Bar"), tempDir, None) try { memSource.addData((1L, 2L)) stream2.processAllAvailable() } finally { stream2.stop() } val df = spark.read.format("delta").load(tempDir.getAbsolutePath) checkAnswer( df, Row(0L, 0L) :: Row(1L, 2L) :: Nil ) assert(df.schema.fieldNames === Seq("Foo", "Bar")) } } test("SC-12677: replaceWhere predicate should be case insensitive") { withTempDir { tempDir => val path = tempDir.getCanonicalPath Seq((1, "a"), (2, "b")).toDF("Key", "val").write .partitionBy("key").format("delta").mode("append").save(path) withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") { Seq((2, "c")).toDF("Key", "val").write .format("delta") .mode("overwrite") .option("replaceWhere", "key = 2") // note the different case .save(path) } checkAnswer( spark.read.format("delta").load(path), Row(1, "a") :: Row(2, "c") :: Nil ) withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { val e = intercept[AnalysisException] { Seq((2, "d")).toDF("Key", "val").write .format("delta") .mode("overwrite") .option("replaceWhere", "key = 2") // note the different case .save(path) } assert(e.getErrorClass == "UNRESOLVED_COLUMN.WITHOUT_SUGGESTION" || e.getErrorClass == "MISSING_COLUMN" || e.getErrorClass == "UNRESOLVED_COLUMN.WITH_SUGGESTION") } checkAnswer( spark.read.format("delta").load(path), Row(1, "a") :: Row(2, "c") :: Nil ) } } private def startStream( df: Dataset[_], tempDir: File, partitionBy: Option[String] = Some("foo")): StreamingQuery = { val writer = df.writeStream .option("checkpointLocation", new File(tempDir, "_checkpoint").getAbsolutePath) .format("delta") partitionBy.foreach(writer.partitionBy(_)) writer.start(tempDir.getAbsolutePath) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/schema/CheckConstraintsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.schema import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.{AllowedUserProvidedExpressions, DeltaConfigs, DeltaLog} import org.apache.spark.sql.delta.constraints.CharVarcharConstraint import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.sources.DeltaSQLConf.ValidateCheckConstraintsMode import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.SparkConf import org.apache.spark.sql.{AnalysisException, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{ArrayType, BooleanType, IntegerType, MapType, MetadataBuilder, StringType, StructField, StructType} class CheckConstraintsSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with DeltaSQLTestUtils { import testImplicits._ private def withTestTable(thunk: String => Unit) = { withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> "1", DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "3") { withTable("checkConstraintsTest") { Seq( (1, "a"), (2, "b"), (3, "c"), (4, "d"), (5, "e"), (6, "f") ).toDF("num", "text").write.format("delta").saveAsTable("checkConstraintsTest") thunk("checkConstraintsTest") } } } private def errorContains(errMsg: String, str: String): Unit = { errMsg.contains(str) } test("can't add unparseable constraint") { withTestTable { table => val e = intercept[ParseException] { sql(s"ALTER TABLE $table\nADD CONSTRAINT lessThan5 CHECK (id <)") } // Make sure we're still getting a useful parse error, even though we do some complicated // internal stuff to persist the constraint. Unfortunately this test may be a bit fragile. errorContains(e.getMessage, "Syntax error at or near end of input") errorContains(e.getMessage, """ |== SQL == |id < |----^^^ |""".stripMargin) } } test("Checking incorrect constraints added through table property in CREATE TABLE errors out") { val tableName = "test_tbl" withTable(tableName) { sql( s""" |CREATE TABLE $tableName ( |id INT, |event_date DATE |) USING DELTA |TBLPROPERTIES('delta.constraints.ch' = 'event_date < 2025-06-12');""".stripMargin) val e = intercept[AnalysisException] { sql(s"INSERT INTO $tableName VALUES(1, '2025-06-11')") } errorContains(e.getMessage, "Cannot resolve \"(event_date < ((2025 - 6) - 12))\" due to data type mismatch") } } test("CREATE TABLE with check constraint referencing non-existent column fails at create time") { withSQLConf(DeltaSQLConf.VALIDATE_CHECK_CONSTRAINTS.key -> ValidateCheckConstraintsMode.ASSERT.toString) { val tableName = "test_create_invalid_constraint" withTable(tableName) { checkError( exception = intercept[AnalysisException] { sql( s""" |CREATE TABLE $tableName ( |id INT, |value STRING |) USING DELTA |TBLPROPERTIES('delta.constraints.invalid' = 'non_existent_column > 0') |""".stripMargin) }, "DELTA_INVALID_CHECK_CONSTRAINT_REFERENCES", parameters = Map("colName" -> "`non_existent_column`") ) } } } test("CREATE TABLE with non-boolean check constraint fails at create time") { withSQLConf(DeltaSQLConf.VALIDATE_CHECK_CONSTRAINTS.key -> ValidateCheckConstraintsMode.ASSERT.toString) { val tableName = "test_create_non_boolean_constraint" withTable(tableName) { checkError( exception = intercept[AnalysisException] { sql( s""" |CREATE TABLE $tableName ( |id INT, |value STRING |) USING DELTA |TBLPROPERTIES('delta.constraints.nonbool' = 'id + 1') |""".stripMargin) }, "DELTA_NON_BOOLEAN_CHECK_CONSTRAINT", parameters = Map( "name" -> "nonbool", "expr" -> "(id + 1)" ) ) } } } test("CREATE TABLE with valid check constraint succeeds") { withSQLConf(DeltaSQLConf.VALIDATE_CHECK_CONSTRAINTS.key -> ValidateCheckConstraintsMode.ASSERT.toString) { val tableName = "test_create_valid_constraint" withTable(tableName) { sql( s""" |CREATE TABLE $tableName ( |id INT, |value STRING |) USING DELTA |TBLPROPERTIES('delta.constraints.positive_id' = 'id > 0') |""".stripMargin) } } } test("constraint must be boolean") { withTestTable { table => checkError( exception = intercept[AnalysisException] { sql(s"ALTER TABLE $table ADD CONSTRAINT integerVal CHECK (3)") }, "DELTA_NON_BOOLEAN_CHECK_CONSTRAINT", parameters = Map( "name" -> "integerVal", "expr" -> "3" ) ) } } test("can't add constraint referencing non-existent columns") { withTestTable { table => checkError( intercept[AnalysisException] { sql(s"ALTER TABLE $table ADD CONSTRAINT c CHECK (does_not_exist)") }, "UNRESOLVED_COLUMN.WITH_SUGGESTION", parameters = Map( "objectName" -> "`does_not_exist`", "proposal" -> "`text`, `num`" ) ) } } test("can't add constraint with duplicate name") { withTestTable { table => sql(s"ALTER TABLE $table ADD CONSTRAINT trivial CHECK (true)") val e = intercept[AnalysisException] { sql(s"ALTER TABLE $table ADD CONSTRAINT trivial CHECK (true)") } errorContains(e.getMessage, s"Constraint 'trivial' already exists as a CHECK constraint. Please delete the " + s"old constraint first.\nOld constraint:\ntrue") } } test("can't add constraint with names that are reserved for internal usage") { withTestTable { table => val reservedName = CharVarcharConstraint.INVARIANT_NAME val e = intercept[AnalysisException] { sql(s"ALTER TABLE $table ADD CONSTRAINT $reservedName CHECK (true)") } errorContains(e.getMessage, s"Cannot use '$reservedName' as the name of a CHECK constraint") } } test("duplicate constraint check is case insensitive") { withTestTable { table => sql(s"ALTER TABLE $table ADD CONSTRAINT trivial CHECK (true)") val e = intercept[AnalysisException] { sql(s"ALTER TABLE $table ADD CONSTRAINT TRIVIAL CHECK (true)") } errorContains(e.getMessage, s"Constraint 'TRIVIAL' already exists as a CHECK constraint. Please delete the " + s"old constraint first.\nOld constraint:\ntrue") } } testQuietly("can't add already violated constraint") { withTestTable { table => val e = intercept[AnalysisException] { sql(s"ALTER TABLE $table ADD CONSTRAINT lessThan5 CHECK (num < 5 and text < 'd')") } errorContains(e.getMessage, s"violate the new CHECK constraint (num < 5 and text < 'd')") } } testQuietly("can't add row violating constraint") { withTestTable { table => sql(s"ALTER TABLE $table ADD CONSTRAINT lessThan10 CHECK (num < 10 and text < 'g')") sql(s"INSERT INTO $table VALUES (5, 'a')") val e = intercept[InvariantViolationException] { sql(s"INSERT INTO $table VALUES (11, 'a')") } errorContains(e.getMessage, s"CHECK constraint lessthan10 ((num < 10) AND (text < 'g')) violated") } } test("drop constraint that doesn't exist throws an exception") { withTestTable { table => intercept[AnalysisException] { sql(s"ALTER TABLE $table DROP CONSTRAINT myConstraint") } } withSQLConf((DeltaSQLConf.DELTA_ASSUMES_DROP_CONSTRAINT_IF_EXISTS.key, "false")) { withTestTable { table => val e = intercept[AnalysisException] { sql(s"ALTER TABLE $table DROP CONSTRAINT myConstraint") } assert(e.getErrorClass == "DELTA_CONSTRAINT_DOES_NOT_EXIST") errorContains(e.getMessage, "nonexistent constraint myconstraint from table `default`.`checkconstraintstest`") errorContains(e.getMessage, "databricks.spark.delta.constraints.assumesDropIfExists.enabled to true") } } } test("can drop constraint that doesn't exist with IF EXISTS") { withTestTable { table => sql(s"ALTER TABLE $table DROP CONSTRAINT IF EXISTS myConstraint") } withSQLConf((DeltaSQLConf.DELTA_ASSUMES_DROP_CONSTRAINT_IF_EXISTS.key, "true")) { withTestTable { table => sql(s"ALTER TABLE $table DROP CONSTRAINT myConstraint") } } } test("drop constraint is case insensitive") { withTestTable { table => sql(s"ALTER TABLE $table ADD CONSTRAINT myConstraint CHECK (true)") sql(s"ALTER TABLE $table DROP CONSTRAINT MYCONSTRAINT") } } testQuietly("add row violating constraint after it's dropped") { withTestTable { table => sql(s"ALTER TABLE $table ADD CONSTRAINT lessThan10 CHECK (num < 10 and text < 'g')") intercept[InvariantViolationException] { sql(s"INSERT INTO $table VALUES (11, 'a')") } sql(s"ALTER TABLE $table DROP CONSTRAINT lessThan10") sql(s"INSERT INTO $table VALUES (11, 'a')") checkAnswer(sql(s"SELECT num FROM $table"), Seq(1, 2, 3, 4, 5, 6, 11).toDF()) } } test("see constraints in table properties") { withTestTable { table => sql(s"ALTER TABLE $table ADD CONSTRAINT toBeDropped CHECK (text < 'n')") sql(s"ALTER TABLE $table ADD CONSTRAINT trivial CHECK (true)") sql(s"ALTER TABLE $table ADD CONSTRAINT numLimit CHECK (num < 10)") sql(s"ALTER TABLE $table ADD CONSTRAINT combo CHECK (concat(num, text) != '9i')") sql(s"ALTER TABLE $table DROP CONSTRAINT toBeDropped") val props = sql(s"DESCRIBE DETAIL $table").selectExpr("properties").head().getMap[String, String](0) // We've round-tripped through the parser, so the text of the constraints stored won't exactly // match what was originally given. assert(props == Map( "delta.constraints.trivial" -> "true", "delta.constraints.numlimit" -> "num < 10", "delta.constraints.combo" -> "concat ( num , text ) != '9i'" )) } } test("delta history for constraints") { withTestTable { table => sql(s"ALTER TABLE $table ADD CONSTRAINT lessThan10 CHECK (num < 10)") checkAnswer( sql(s"DESCRIBE HISTORY $table") .where("operation = 'ADD CONSTRAINT'") .selectExpr("operation", "operationParameters"), Seq(("ADD CONSTRAINT", Map("name" -> "lessThan10", "expr" -> "num < 10"))).toDF()) sql(s"ALTER TABLE $table DROP CONSTRAINT IF EXISTS lessThan10") checkAnswer( sql(s"DESCRIBE HISTORY $table") .where("operation = 'DROP CONSTRAINT'") .selectExpr("operation", "operationParameters"), Seq(( "DROP CONSTRAINT", Map("name" -> "lessThan10", "expr" -> "num < 10", "existed" -> "true") )).toDF()) sql(s"ALTER TABLE $table DROP CONSTRAINT IF EXISTS lessThan10") checkAnswer( sql(s"DESCRIBE HISTORY $table") .where("operation = 'DROP CONSTRAINT'") .selectExpr("operation", "operationParameters"), Seq( ("DROP CONSTRAINT", Map("name" -> "lessThan10", "expr" -> "num < 10", "existed" -> "true")), ("DROP CONSTRAINT", Map("name" -> "lessThan10", "existed" -> "false")) ).toDF()) } } testQuietly("constraint on builtin methods") { withTestTable { table => sql(s"ALTER TABLE $table ADD CONSTRAINT textSize CHECK (LENGTH(text) < 10)") sql(s"INSERT INTO $table VALUES (11, 'abcdefg')") val e = intercept[InvariantViolationException] { sql(s"INSERT INTO $table VALUES (12, 'abcdefghijklmnop')") } errorContains(e.getMessage, "constraint textsize (LENGTH(text) < 10) violated by row") } } testQuietly("constraint with implicit casts") { withTestTable { table => sql(s"ALTER TABLE $table ADD CONSTRAINT maxWithImplicitCast CHECK (num < '10')") val e = intercept[InvariantViolationException] { sql(s"INSERT INTO $table VALUES (11, 'data')") } errorContains(e.getMessage, "constraint maxwithimplicitcast (num < '10') violated by row") } } testQuietly("constraint with nested parentheses") { withTestTable { table => sql(s"ALTER TABLE $table ADD CONSTRAINT maxWithParens " + s"CHECK (( (num < '10') AND ((LENGTH(text)) < 100) ))") val e = intercept[InvariantViolationException] { sql(s"INSERT INTO $table VALUES (11, 'data')") } errorContains(e.getMessage, "constraint maxwithparens ((num < '10') AND (LENGTH(text) < 100)) violated by row") } } for (expression <- Seq("year(current_date())", "unix_timestamp()")) testQuietly(s"constraint with analyzer-evaluated expressions. Expression: $expression") { // Explicitly block constraint validation since both functions are nondeterministic. val disabled = ValidateCheckConstraintsMode.OFF.toString withSQLConf(DeltaSQLConf.VALIDATE_CHECK_CONSTRAINTS.key -> disabled) { withTestTable { table => // We use current_timestamp()/current_date() as the most convenient // analyzer-evaluated expressions - of course in a realistic use case // it'd probably not be right to add a constraint on a // nondeterministic expression. sql(s"ALTER TABLE $table ADD CONSTRAINT maxWithAnalyzerEval " + s"CHECK (num < $expression)") val e = intercept[InvariantViolationException] { sql(s"INSERT INTO $table VALUES (${Int.MaxValue}, 'data')") } errorContains(e.getMessage, s"maxwithanalyzereval (num < $expression) violated by row") } } } testQuietly("constraints with nulls") { withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> "1", DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "3") { withTable("checkConstraintsTest") { val rows = Range(0, 10).map { i => Row( i, null, Row("constantWithinStruct", Map(i -> i), Array(i, null, i + 2))) } val schema = new StructType(Array( StructField("id", IntegerType), StructField("text", StringType), StructField("nested", new StructType(Array( StructField("constant", StringType), StructField("m", MapType(IntegerType, IntegerType, valueContainsNull = true)), StructField("arr", ArrayType(IntegerType, containsNull = true))))))) spark.createDataFrame(rows.toList.asJava, schema) .write.format("delta").saveAsTable("checkConstraintsTest") // Constraints checking for a null value should work. sql("ALTER TABLE checkConstraintsTest ADD CONSTRAINT textNull CHECK (text IS NULL)") sql("ALTER TABLE checkConstraintsTest ADD CONSTRAINT arr1Null " + "CHECK (nested.arr[1] IS NULL)") // Constraints incompatible with a null value will of course fail, but they should fail with // the same clear error as normal. var e: Exception = intercept[AnalysisException] { sql("ALTER TABLE checkConstraintsTest ADD CONSTRAINT arrLessThan5 " + "CHECK (nested.arr[1] < 5)") } errorContains(e.getMessage, s"10 rows in default.checkconstraintstest violate the new CHECK constraint " + s"(nested . arr [ 1 ] < 5)") // Adding a null value into a constraint should fail similarly, even if it's null // because a parent field is null. sql("ALTER TABLE checkConstraintsTest ADD CONSTRAINT arr0 " + "CHECK (nested.arr[0] < 100)") val newRows = Seq( Row(10, null, Row("c", Map(10 -> null), Array(null, null, 12))), Row(11, null, Row("c", Map(11 -> null), null)), Row(12, null, null)) newRows.foreach { r => e = intercept[InvariantViolationException] { spark.createDataFrame(List(r).asJava, schema) .write.format("delta").mode("append").saveAsTable("checkConstraintsTest") } errorContains(e.getMessage, "CHECK constraint arr0 (nested.arr[0] < 100) violated by row") } // On the other hand, existing constraints like arr1Null which do allow null values should // permit new rows even if the value's parent is null. sql("ALTER TABLE checkConstraintsTest DROP CONSTRAINT arr0") newRows.foreach { r => spark.createDataFrame(List(r).asJava, schema) .write.format("delta").mode("append").saveAsTable("checkConstraintsTest") } checkAnswer( spark.read.format("delta").table("checkConstraintsTest").select("id"), (0 to 12).toDF("id")) } } } testQuietly("complex constraints") { withSQLConf( DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_READER_VERSION.key -> "1", DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key -> "3") { withTable("checkConstraintsTest") { val rows = Range(0, 10).map { i => Row( i, ('a' + i).toString, Row("constantWithinStruct", Map(i -> i), Array(i, i + 1, i + 2))) } val schema = new StructType(Array( StructField("id", IntegerType), StructField("text", StringType), StructField("nested", new StructType(Array( StructField("constant", StringType), StructField("m", MapType(IntegerType, IntegerType, valueContainsNull = false)), StructField("arr", ArrayType(IntegerType, containsNull = false))))))) spark.createDataFrame(rows.toList.asJava, schema) .write.format("delta").saveAsTable("checkConstraintsTest") sql("ALTER TABLE checkConstraintsTest ADD CONSTRAINT arrLen CHECK (SIZE(nested.arr) = 3)") sql("ALTER TABLE checkConstraintsTest ADD CONSTRAINT mapIntegrity " + "CHECK (nested.m[id] = id)") val e = intercept[AnalysisException] { sql(s"ALTER TABLE checkConstraintsTest ADD CONSTRAINT violated " + s"CHECK (nested.arr[0] < id)") } errorContains(e.getMessage, s"violate the new CHECK constraint (nested . arr [ 0 ] < id)") } } } // TODO: https://github.com/delta-io/delta/issues/831 test("SET NOT NULL constraint fails") { withTable("my_table") { sql("CREATE TABLE my_table (id INT) USING DELTA;") sql("INSERT INTO my_table VALUES (1);") val e = intercept[AnalysisException] { sql("ALTER TABLE my_table CHANGE COLUMN id SET NOT NULL;") }.getMessage() assert(e.contains("Cannot change nullable column to non-nullable")) } } testQuietly("ending semi-colons no longer makes ADD, DROP constraint commands fail") { withTable("my_table") { sql("CREATE TABLE my_table (birthday DATE) USING DELTA;") sql("INSERT INTO my_table VALUES ('2021-11-11');") sql("ALTER TABLE my_table ADD CONSTRAINT aaa CHECK (birthday > '1900-01-01')") sql("ALTER TABLE my_table ADD CONSTRAINT bbb CHECK (birthday > '1900-02-02')") sql("ALTER TABLE my_table ADD CONSTRAINT ccc CHECK (birthday > '1900-03-03');") // semi-colon sql("ALTER TABLE my_table DROP CONSTRAINT aaa") sql("ALTER TABLE my_table DROP CONSTRAINT bbb;") // semi-colon } } test("validate check constraints on table with char/varchar columns") { withSQLConf(DeltaSQLConf.VALIDATE_CHECK_CONSTRAINTS.key -> ValidateCheckConstraintsMode.ASSERT.toString, SQLConf.READ_SIDE_CHAR_PADDING.key -> "true") { withTable("charVarcharConstraintTest") { sql( """CREATE TABLE charVarcharConstraintTest ( | id INT, | name VARCHAR(50), | code CHAR(10) |) USING DELTA |TBLPROPERTIES('delta.constraints.positive_id' = 'id > 0') |""".stripMargin) sql("INSERT INTO charVarcharConstraintTest VALUES (1, 'test', 'ABC')") checkAnswer( sql("SELECT id, name, code FROM charVarcharConstraintTest"), Seq(Row(1, "test", "ABC "))) } } } test("constraint induced by varchar") { withTable("table") { sql("CREATE TABLE table (id INT, value VARCHAR(12)) USING DELTA") sql("INSERT INTO table VALUES (1, 'short string')") val exception = intercept[DeltaInvariantViolationException] { sql("INSERT INTO table VALUES (2, 'a very long string')") } checkError( exception, "DELTA_EXCEED_CHAR_VARCHAR_LIMIT", parameters = Map( "value" -> "a very long string", "expr" -> "((value IS NULL) OR (length(value) <= 12))" ) ) } } test("drop table feature") { withSQLConf( DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> false.toString) { withTable("table") { sql("CREATE TABLE table (a INT, b INT) USING DELTA " + "TBLPROPERTIES ('delta.feature.checkConstraints' = 'supported')") sql("ALTER TABLE table ADD CONSTRAINT c1 CHECK (a > 0)") sql("ALTER TABLE table ADD CONSTRAINT c2 CHECK (b > 0)") val error1 = intercept[AnalysisException] { sql("ALTER TABLE table DROP FEATURE checkConstraints") } checkError( error1, "DELTA_CANNOT_DROP_CHECK_CONSTRAINT_FEATURE", parameters = Map("constraints" -> "`c1`, `c2`") ) val deltaLog = DeltaLog.forTable(spark, TableIdentifier("table")) val featureNames1 = deltaLog.update().protocol.implicitlyAndExplicitlySupportedFeatures.map(_.name) assert(featureNames1.contains("checkConstraints")) sql("ALTER TABLE table DROP CONSTRAINT c1") val error2 = intercept[AnalysisException] { sql("ALTER TABLE table DROP FEATURE checkConstraints") } checkError( error2, "DELTA_CANNOT_DROP_CHECK_CONSTRAINT_FEATURE", parameters = Map("constraints" -> "`c2`") ) val featureNames2 = deltaLog.update().protocol.implicitlyAndExplicitlySupportedFeatures.map(_.name) assert(featureNames2.contains("checkConstraints")) sql("ALTER TABLE table DROP CONSTRAINT c2") sql("ALTER TABLE table DROP FEATURE checkConstraints") val featureNames3 = deltaLog.update().protocol.implicitlyAndExplicitlySupportedFeatures.map(_.name) assert(!featureNames3.contains("checkConstraints")) } } } for (expression <- Seq("startsWith", "endsWith", "contains")) { test(s"Creating constraints with expressions in the allowList should work for: $expression") { withSQLConf(DeltaSQLConf.VALIDATE_CHECK_CONSTRAINTS.key -> ValidateCheckConstraintsMode.ASSERT.toString) { val testTable = "tbl" withTable(testTable) { sql(s"CREATE TABLE $testTable (id STRING, value BOOLEAN) USING DELTA " + "TBLPROPERTIES ('delta.feature.checkConstraints' = 'supported')") sql(s"ALTER TABLE $testTable ADD CONSTRAINT c1 CHECK (value == $expression(id, 'A'))") sql(s"INSERT INTO $testTable VALUES ('ABA', true), ('DEF', false)") } } } } test("check constraints with LIKE ANY/ALL and NOT LIKE ANY/ALL expressions") { withSQLConf(DeltaSQLConf.VALIDATE_CHECK_CONSTRAINTS.key -> ValidateCheckConstraintsMode.ASSERT.toString) { val testTable = "like_any_all_test" withTable(testTable) { sql(s"CREATE TABLE $testTable (id INT, name STRING, code STRING) USING DELTA " + "TBLPROPERTIES ('delta.feature.checkConstraints' = 'supported')") sql(s"ALTER TABLE $testTable ADD CONSTRAINT c_like_any " + "CHECK (name LIKE ANY ('%test%', '%prod%'))") sql(s"ALTER TABLE $testTable ADD CONSTRAINT c_not_like_any " + "CHECK (name NOT LIKE ANY ('%forbidden%', '%blocked%'))") sql(s"ALTER TABLE $testTable ADD CONSTRAINT c_like_all " + "CHECK (code LIKE ALL ('%A%', '%B%'))") sql(s"ALTER TABLE $testTable ADD CONSTRAINT c_not_like_all " + "CHECK (code NOT LIKE ALL ('%X%', '%Y%'))") sql(s"INSERT INTO $testTable VALUES (1, 'test_data', 'AB')") checkAnswer( sql(s"SELECT * FROM $testTable"), Seq(Row(1, "test_data", "AB"))) } } } test("check constraints with array_size and array_compact expressions") { withSQLConf(DeltaSQLConf.VALIDATE_CHECK_CONSTRAINTS.key -> ValidateCheckConstraintsMode.ASSERT.toString) { val testTable = "array_funcs_test" withTable(testTable) { sql(s"CREATE TABLE $testTable (id INT, tags ARRAY) USING DELTA " + "TBLPROPERTIES ('delta.feature.checkConstraints' = 'supported')") sql(s"ALTER TABLE $testTable ADD CONSTRAINT c_array_size " + "CHECK (array_size(tags) > 0)") sql(s"ALTER TABLE $testTable ADD CONSTRAINT c_array_compact " + "CHECK (array_size(array_compact(tags)) > 0)") sql(s"INSERT INTO $testTable VALUES (1, array('a', 'b'))") checkAnswer( sql(s"SELECT id FROM $testTable"), Seq(Row(1))) } } } test("check constraints with array_append, array_prepend, array_insert expressions") { withSQLConf(DeltaSQLConf.VALIDATE_CHECK_CONSTRAINTS.key -> ValidateCheckConstraintsMode.ASSERT.toString) { val testTable = "array_modify_funcs_test" withTable(testTable) { sql(s"CREATE TABLE $testTable (id INT, tags ARRAY) USING DELTA " + "TBLPROPERTIES ('delta.feature.checkConstraints' = 'supported')") sql(s"ALTER TABLE $testTable ADD CONSTRAINT c_array_append " + "CHECK (array_size(array_append(tags, 'x')) > 1)") sql(s"ALTER TABLE $testTable ADD CONSTRAINT c_array_prepend " + "CHECK (array_size(array_prepend(tags, 'y')) > 1)") sql(s"ALTER TABLE $testTable ADD CONSTRAINT c_array_insert " + "CHECK (array_size(array_insert(tags, 1, 'z')) > 1)") sql(s"INSERT INTO $testTable VALUES (1, array('a', 'b'))") checkAnswer( sql(s"SELECT id FROM $testTable"), Seq(Row(1))) } } } test("Creating constraints with expressions not in the allowList should throw an error") { withSQLConf(DeltaSQLConf.VALIDATE_CHECK_CONSTRAINTS.key -> ValidateCheckConstraintsMode.ASSERT.toString) { val testTable = "tbl" withTable(testTable) { sql(s"CREATE TABLE $testTable (id INT) USING DELTA " + "TBLPROPERTIES ('delta.feature.checkConstraints' = 'supported')") checkError( exception = intercept[AnalysisException] { sql(s"ALTER TABLE $testTable ADD CONSTRAINT c1 " + s"CHECK (id > (SELECT max(id) FROM $testTable))") }, "DELTA_UNSUPPORTED_EXPRESSION_CHECK_CONSTRAINT", parameters = Map("expression" -> "scalarsubquery()") ) } } } for (isEnabled <- Seq(ValidateCheckConstraintsMode.OFF, ValidateCheckConstraintsMode.ASSERT)) { test(s"Reject CHECK constraints with external UDF calls when validation is ${isEnabled}") { withSQLConf(DeltaSQLConf.VALIDATE_CHECK_CONSTRAINTS.key -> isEnabled.toString) { val testTable = "check_external_udf_test" withTable(testTable) { withUserDefinedFunction("external_udf" -> true) { sql(s"CREATE TABLE $testTable (id INT, value INT) USING DELTA " + "TBLPROPERTIES ('delta.feature.checkConstraints' = 'supported')") spark.udf.register("external_udf", (x: Int) => x > 0) val sqlText = s"ALTER TABLE $testTable ADD CONSTRAINT check_external_udf " + "CHECK (external_udf(value))" if (isEnabled == ValidateCheckConstraintsMode.ASSERT) { checkError( exception = intercept[AnalysisException] { sql(sqlText) }, "DELTA_UDF_IN_CHECK_CONSTRAINT", parameters = Map("expr" -> "external_udf(knownnotnull(value))") ) } else { sql(sqlText) } } } } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/schema/InvariantEnforcementSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.schema // scalastyle:off import.ordering.noEmptyLine import java.io.File import java.sql.Date import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.{CheckConstraintsTableFeature, DeltaLog, DeltaOperations} import org.apache.spark.sql.delta.actions.{Metadata, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.constraints.{Constraint, Constraints, Invariants} import org.apache.spark.sql.delta.constraints.Constraints.NotNull import org.apache.spark.sql.delta.constraints.Invariants.PersistedExpression import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.test.shims.StreamingTestShims.MemoryStream import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.streaming.StreamingQueryException import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ class InvariantEnforcementSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with DeltaSQLTestUtils { import testImplicits._ private def tableWithSchema(schema: StructType)(f: String => Unit): Unit = { withTempDir { tempDir => val deltaLog = DeltaLog.forTable(spark, tempDir) val txn = deltaLog.startTransaction() txn.commit(Metadata(schemaString = schema.json) :: Nil, DeltaOperations.ManualUpdate) spark.read.format("delta") .load(tempDir.getAbsolutePath) .write .format("delta") .mode("overwrite") .save(tempDir.getAbsolutePath) f(tempDir.getAbsolutePath) } } private def testBatchWriteRejection( invariant: Constraint, schema: StructType, df: Dataset[_], expectedErrors: String*): Unit = { tableWithSchema(schema) { path => val e = intercept[InvariantViolationException] { df.write.mode("append").format("delta").save(path) } checkConstraintException(e, (invariant.name +: expectedErrors): _*) } } private def checkConstraintException( e: InvariantViolationException, expectedErrors: String*): Unit = { val error = e.getMessage val allExpected = expectedErrors allExpected.foreach { expected => assert(error.contains(expected), s"$error didn't contain $expected") } } private def testStreamingWriteRejection[T: Encoder]( invariant: Constraint, schema: StructType, toDF: MemoryStream[T] => DataFrame, data: Seq[T], expectedErrors: String*): Unit = { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) val txn = deltaLog.startTransaction() txn.commit(Metadata(schemaString = schema.json) :: Nil, DeltaOperations.ManualUpdate) val memStream = MemoryStream[T] val stream = toDF(memStream).writeStream .outputMode("append") .format("delta") .option("checkpointLocation", new File(dir, "_checkpoint").getAbsolutePath) .start(dir.getAbsolutePath) try { val e = intercept[StreamingQueryException] { memStream.addData(data) stream.processAllAvailable() } // Produce a good error if the cause isn't the right type - just an assert makes it hard to // see what the wrong exception was. intercept[InvariantViolationException] { throw e.getCause } checkConstraintException( e.getCause.asInstanceOf[InvariantViolationException], (invariant.name +: expectedErrors): _*) } finally { stream.stop() } } } test("reject non-nullable top level column") { val schema = new StructType() .add("key", StringType, nullable = false) .add("value", IntegerType) testBatchWriteRejection( NotNull(Seq("key")), schema, Seq[(String, Int)](("a", 1), (null, 2)).toDF("key", "value"), "key" ) testStreamingWriteRejection[(String, Int)]( NotNull(Seq("key")), schema, _.toDF().toDF("key", "value"), Seq[(String, Int)](("a", 1), (null, 2)), "key" ) } test("reject non-nullable top level column - column doesn't exist") { val schema = new StructType() .add("key", StringType, nullable = false) .add("value", IntegerType) testBatchWriteRejection( NotNull(Seq("key")), schema, Seq[Int](1, 2).toDF("value"), "key" ) testStreamingWriteRejection[Int]( NotNull(Seq("key")), schema, _.toDF().toDF("value"), Seq[Int](1, 2), "key" ) } testQuietly("write empty DataFrame - zero rows") { val schema = new StructType() .add("key", StringType, nullable = false) .add("value", IntegerType) tableWithSchema(schema) { path => spark.createDataFrame(Seq.empty[Row].asJava, schema.asNullable).write .mode("append").format("delta").save(path) } } test("write empty DataFrame - zero columns") { val schema = new StructType() .add("key", StringType, nullable = false) .add("value", IntegerType) testBatchWriteRejection( NotNull(Seq("key")), schema, Seq[Int](1, 2).toDF("value").drop("value"), "key" ) testStreamingWriteRejection[Int]( NotNull(Seq("key")), schema, _.toDF().toDF("value").drop("value"), Seq[Int](1, 2), "key" ) } testQuietly("reject non-nullable nested column") { val schema = new StructType() .add("top", new StructType() .add("key", StringType, nullable = false) .add("value", IntegerType)) testBatchWriteRejection( NotNull(Seq("key")), schema, spark.createDataFrame(Seq(Row(Row("a", 1)), Row(Row(null, 2))).asJava, schema.asNullable), "top.key" ) testBatchWriteRejection( NotNull(Seq("key")), schema, spark.createDataFrame(Seq(Row(Row("a", 1)), Row(null)).asJava, schema.asNullable), "top.key" ) } testQuietly("reject non-nullable array column") { val schema = new StructType() .add("top", ArrayType(ArrayType(new StructType() .add("key", StringType) .add("value", IntegerType))), nullable = false) testBatchWriteRejection( NotNull(Seq("top", "value")), schema, spark.createDataFrame(Seq(Row(Seq(Seq(Row("a", 1)))), Row(null)).asJava, schema.asNullable), "top" ) } test("reject expression invariant on top level column") { val expr = "value < 3" val rule = Constraints.Check("", spark.sessionState.sqlParser.parseExpression(expr)) val metadata = new MetadataBuilder() .putString(Invariants.INVARIANTS_FIELD, PersistedExpression(expr).json) .build() val schema = new StructType() .add("key", StringType) .add("value", IntegerType, nullable = true, metadata) testBatchWriteRejection( rule, schema, Seq[(String, Int)](("a", 1), (null, 5)).toDF("key", "value"), "value", "5" ) testStreamingWriteRejection[(String, Int)]( rule, schema, _.toDF().toDF("key", "value"), Seq[(String, Int)](("a", 1), (null, 5)), "value" ) } testQuietly("reject expression invariant on nested column") { val expr = "top.value < 3" val rule = Constraints.Check("", spark.sessionState.sqlParser.parseExpression(expr)) val metadata = new MetadataBuilder() .putString(Invariants.INVARIANTS_FIELD, PersistedExpression(expr).json) .build() val schema = new StructType() .add("top", new StructType() .add("key", StringType) .add("value", IntegerType, nullable = true, metadata)) testBatchWriteRejection( rule, schema, spark.createDataFrame(Seq(Row(Row("a", 1)), Row(Row(null, 5))).asJava, schema.asNullable), "top.value", "5" ) } testQuietly("reject write on top level expression invariant when field is null") { val expr = "value < 3" val rule = Constraints.Check("", spark.sessionState.sqlParser.parseExpression(expr)) val metadata = new MetadataBuilder() .putString(Invariants.INVARIANTS_FIELD, PersistedExpression(expr).json) .build() val schema = new StructType() .add("key", StringType) .add("value", IntegerType, nullable = true, metadata) testBatchWriteRejection( rule, schema, Seq[String]("a", "b").toDF("key"), " - value : null" ) testBatchWriteRejection( rule, schema, Seq[(String, Integer)](("a", 1), ("b", null)).toDF("key", "value"), " - value : null" ) } testQuietly("reject write on nested expression invariant when field is null") { val expr = "top.value < 3" val metadata = new MetadataBuilder() .putString(Invariants.INVARIANTS_FIELD, PersistedExpression(expr).json) .build() val rule = Constraints.Check("", spark.sessionState.sqlParser.parseExpression(expr)) val schema = new StructType() .add("top", new StructType() .add("key", StringType) .add("value", IntegerType, nullable = true, metadata)) testBatchWriteRejection( rule, schema, spark.createDataFrame(Seq(Row(Row("a", 1)), Row(Row("b", null))).asJava, schema.asNullable), " - top.value : null" ) val schema2 = new StructType() .add("top", new StructType() .add("key", StringType)) testBatchWriteRejection( rule, schema, spark.createDataFrame(Seq(Row(Row("a")), Row(Row("b"))).asJava, schema2.asNullable), " - top.value : null" ) } testQuietly("is null on top level expression invariant when field is null") { val expr = "value is null or value < 3" val metadata = new MetadataBuilder() .putString(Invariants.INVARIANTS_FIELD, PersistedExpression(expr).json) .build() val schema = new StructType() .add("key", StringType) .add("value", IntegerType, nullable = true, metadata) tableWithSchema(schema) { path => Seq[String]("a", "b").toDF("key").write .mode("append").format("delta").save(path) Seq[(String, Integer)](("a", 1), ("b", null)).toDF("key", "value").write .mode("append").format("delta").save(path) } } testQuietly("is null on nested expression invariant when field is null") { val expr = "top.value is null or top.value < 3" val metadata = new MetadataBuilder() .putString(Invariants.INVARIANTS_FIELD, PersistedExpression(expr).json) .build() val schema = new StructType() .add("top", new StructType() .add("key", StringType) .add("value", IntegerType, nullable = true, metadata)) val schema2 = new StructType() .add("top", new StructType() .add("key", StringType)) tableWithSchema(schema) { path => spark.createDataFrame(Seq(Row(Row("a", 1)), Row(Row("b", null))).asJava, schema.asNullable) .write.mode("append").format("delta").save(path) spark.createDataFrame(Seq(Row(Row("a")), Row(Row("b"))).asJava, schema2.asNullable) .write.mode("append").format("delta").save(path) } } testQuietly("complex expressions - AND") { val expr = "value < 3 AND value > 0" val metadata = new MetadataBuilder() .putString(Invariants.INVARIANTS_FIELD, PersistedExpression(expr).json) .build() val schema = new StructType() .add("key", StringType) .add("value", IntegerType, nullable = true, metadata) tableWithSchema(schema) { path => Seq(1, 2).toDF("value").write.mode("append").format("delta").save(path) intercept[InvariantViolationException] { Seq(1, 4).toDF("value").write.mode("append").format("delta").save(path) } intercept[InvariantViolationException] { Seq(-1, 2).toDF("value").write.mode("append").format("delta").save(path) } } } testQuietly("complex expressions - IN SET") { val expr = "key in ('a', 'b', 'c')" val metadata = new MetadataBuilder() .putString(Invariants.INVARIANTS_FIELD, PersistedExpression(expr).json) .build() val schema = new StructType() .add("key", StringType, nullable = true, metadata) .add("value", IntegerType) tableWithSchema(schema) { tempDir => Seq("a", "b").toDF("key").write.mode("append").format("delta").save(tempDir) intercept[InvariantViolationException] { Seq("a", "d").toDF("key").write.mode("append").format("delta").save(tempDir) } intercept[InvariantViolationException] { Seq("e").toDF("key").write.mode("append").format("delta").save(tempDir) } } } test("CHECK constraint can't be created through SET TBLPROPERTIES") { withTable("noCheckConstraints") { spark.range(10).write.format("delta").saveAsTable("noCheckConstraints") val ex = intercept[AnalysisException] { spark.sql( "ALTER TABLE noCheckConstraints SET TBLPROPERTIES ('delta.constraints.mychk' = '1')") } assert(ex.getMessage.contains("ALTER TABLE ADD CONSTRAINT")) } } for (writerVersion <- Seq(2, TableFeatureProtocolUtils.TABLE_FEATURES_MIN_WRITER_VERSION)) testQuietly("CHECK constraint is enforced if somehow created (writerVersion = " + s"$writerVersion)") { withSQLConf((DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key, writerVersion.toString)) { withTable("constraint") { spark.range(10).selectExpr("id AS valueA", "id AS valueB", "id AS valueC") .write.format("delta").saveAsTable("constraint") val table = DeltaTableV2(spark, TableIdentifier("constraint", None)) val txn = table.startTransactionWithInitialSnapshot() val newMetadata = txn.metadata.copy( configuration = txn.metadata.configuration + ("delta.constraints.mychk" -> "valueA < valueB")) txn.commit(Seq(newMetadata), DeltaOperations.ManualUpdate) val protocol = table.deltaLog.update().protocol assert(protocol.implicitlyAndExplicitlySupportedFeatures .contains(CheckConstraintsTableFeature)) spark.sql("INSERT INTO constraint VALUES (50, 100, null)") val e = intercept[InvariantViolationException] { spark.sql("INSERT INTO constraint VALUES (100, 50, null)") } checkConstraintException( e, "CHECK constraint mychk (valueA < valueB) violated by row with values:", " - valueA : 100", " - valueB : 50") val e2 = intercept[InvariantViolationException] { spark.sql("INSERT INTO constraint VALUES (100, null, null)") } checkConstraintException( e2, "CHECK constraint mychk (valueA < valueB) violated by row with values:", " - valueA : 100", " - valueB : null") } } } test("table with CHECK constraint accepts other metadata changes") { withSQLConf((DeltaSQLConf.DELTA_PROTOCOL_DEFAULT_WRITER_VERSION.key, "3")) { withTable("constraint") { spark.range(10).selectExpr("id AS valueA", "id AS valueB") .write.format("delta").saveAsTable("constraint") val table = DeltaTableV2(spark, TableIdentifier("constraint", None)) val txn = table.startTransactionWithInitialSnapshot() val newMetadata = txn.metadata.copy( configuration = txn.metadata.configuration + ("delta.constraints.mychk" -> "valueA < valueB")) txn.commit(Seq(newMetadata), DeltaOperations.ManualUpdate) spark.sql("ALTER TABLE constraint ADD COLUMN valueC INT") } } } def testUnenforcedNestedConstraints( testName: String, schemaString: String, expectedError: String, data: Row): Unit = { testQuietly(testName) { val nullTable = "nullTbl" withTable(nullTable) { // Try creating the table with the check enabled first, which should fail, then create it // for real with the check off which should succeed. if (expectedError != null) { val ex = intercept[AnalysisException] { sql(s"CREATE TABLE $nullTable ($schemaString) USING delta") } assert(ex.getMessage.contains(expectedError)) } withSQLConf(("spark.databricks.delta.constraints.allowUnenforcedNotNull.enabled", "true")) { sql(s"CREATE TABLE $nullTable ($schemaString) USING delta") } // Once we've created the table, writes should succeed even if they violate the constraint. spark.createDataFrame( Seq(data).asJava, spark.table(nullTable).schema ).write.mode("append").format("delta").saveAsTable(nullTable) if (expectedError != null) { val ex = intercept[AnalysisException] { sql(s"REPLACE TABLE $nullTable ($schemaString) USING delta") } assert(ex.getMessage.contains(expectedError)) } withSQLConf(("spark.databricks.delta.constraints.allowUnenforcedNotNull.enabled", "true")) { sql(s"REPLACE TABLE $nullTable ($schemaString) USING delta") } } } } testUnenforcedNestedConstraints( "not null within array", schemaString = "arr array> NOT NULL", expectedError = "The element type of the field arr contains a NOT NULL constraint.", data = Row(Seq(Row("myName", null)))) testUnenforcedNestedConstraints( "not null within map key", schemaString = "m map, int> NOT NULL", expectedError = "The key type of the field m contains a NOT NULL constraint.", data = Row(Map(Row("myName", null) -> 1))) testUnenforcedNestedConstraints( "not null within map value", schemaString = "m map> NOT NULL", expectedError = "The value type of the field m contains a NOT NULL constraint.", data = Row(Map(1 -> Row("myName", null)))) testUnenforcedNestedConstraints( "not null within nested array", schemaString = "s struct> NOT NULL>", expectedError = "The element type of the field s.arr contains a NOT NULL constraint.", data = Row(Row(1, Seq(Row("myName", null))))) // Helper function to construct the full test name as "RuntimeRepalceable: func" private def testReplaceableExpr(targetFunc: String, testTags: org.scalatest.Tag*) (testFun: => Any) (implicit pos: org.scalactic.source.Position): Unit = { val fulLTestName = s"RuntimeReplaceable: ${targetFunc}" // Suppress exceptions output for invariant violations super.test(fulLTestName) { testFun } } private def testReplaceable[T: Encoder]( exprStr: String, colType: DataType, badValue: T) = { val rule = Constraints.Check("", spark.sessionState.sqlParser.parseExpression(exprStr)) val metadata = new MetadataBuilder() .putString(Invariants.INVARIANTS_FIELD, PersistedExpression(exprStr).json) .build() val schema = new StructType() .add("value", colType, nullable = true, metadata) val rows = Seq(Row(badValue)) testBatchWriteRejection( rule, schema, spark.createDataFrame(rows.toList.asJava, schema), "violated by row with values" ) testStreamingWriteRejection[T]( rule, schema, _.toDF().toDF("value"), Seq[T](badValue), "violated by row with values" ) } testReplaceableExpr("assert_true") { testReplaceable("assert_true(value < 2) is not null", IntegerType, 1) } testReplaceableExpr("date_part") { testReplaceable("date_part('YEAR', value) < 2000", DateType, Date.valueOf("2001-01-01")) } testReplaceableExpr("decode") { testReplaceable("decode(encode(value, 'utf-8'), 'utf-8') = 'abc'", StringType, "a") } testReplaceableExpr("extract") { testReplaceable("extract(YEAR FROM value) < 2000", DateType, Date.valueOf("2001-01-01")) } testReplaceableExpr("ifnull") { testReplaceable("ifnull(value, 1) = 1", IntegerType, 2) } testReplaceableExpr("left") { testReplaceable("left(value, 1) = 'a'", StringType, "b") } testReplaceableExpr("right") { testReplaceable("right(value, 1) = 'a'", StringType, "b") } testReplaceableExpr("nullif") { testReplaceable("nullif(value, 1) = 2", IntegerType, 1) } testReplaceableExpr("nvl") { testReplaceable("nvl(value, 1) = 1", IntegerType, 2) } testReplaceableExpr("nvl2") { testReplaceable("nvl2(value, 1, 2) = 3", IntegerType, 2) } testReplaceableExpr("to_date") { testReplaceable("to_date(value) = '2001-01-01'", StringType, "2002-01-01") } testReplaceableExpr("to_timestamp") { testReplaceable( "to_timestamp(value) = '2001-01-01'", StringType, "2002-01-01 00:12:00") } // Helper function to test with empty to null conf on and off. private def testEmptyToNull(name: String)(f: => Any): Unit = { // Suppress exceptions output for invariant violations testQuietly(name) { Seq(true, false).foreach { enabled => withSQLConf( DeltaSQLConf.CONVERT_EMPTY_TO_NULL_FOR_STRING_PARTITION_COL.key -> enabled.toString) { if (enabled) { f } else { intercept[Exception](f) } } } } } testEmptyToNull("reject empty string for NOT NULL string partition column - create") { val tblName = "empty_string_test" withTable(tblName) { sql( s""" |CREATE TABLE $tblName ( | c1 INT, | c2 STRING NOT NULL |) USING delta |PARTITIONED BY (c2) |""".stripMargin) val ex = intercept[InvariantViolationException] ( sql( s""" |INSERT INTO $tblName values (1, '') |""".stripMargin) ) assert(ex.getMessage.contains("violated")) } } testEmptyToNull("reject empty string for NOT NULL string partition column - multiple") { val tblName = "empty_string_test" withTable(tblName) { sql( s""" |CREATE TABLE $tblName ( | c1 INT, | c2 STRING NOT NULL, | c3 STRING |) USING delta |PARTITIONED BY (c2, c3) |""".stripMargin) val ex = intercept[InvariantViolationException] ( sql( s""" |INSERT INTO $tblName values (1, '', 'a') |""".stripMargin) ) assert(ex.getMessage.contains("violated")) sql( s""" |INSERT INTO $tblName values (1, 'a', '') |""".stripMargin) checkAnswer( sql(s"SELECT COUNT(*) from $tblName where c3 IS NULL"), Row(1L) ) } } testEmptyToNull("reject empty string for NOT NULL string partition column - multiple not null") { val tblName = "empty_string_test" withTable(tblName) { sql( s""" |CREATE TABLE $tblName ( | c1 INT, | c2 STRING NOT NULL, | c3 STRING NOT NULL |) USING delta |PARTITIONED BY (c2, c3) |""".stripMargin) val ex1 = intercept[InvariantViolationException] ( sql( s""" |INSERT INTO $tblName values (1, '', 'a') |""".stripMargin) ) assert(ex1.getMessage.contains("violated")) val ex2 = intercept[InvariantViolationException] ( sql( s""" |INSERT INTO $tblName values (1, 'a', '') |""".stripMargin) ) assert(ex2.getMessage.contains("violated")) val ex3 = intercept[InvariantViolationException] ( sql( s""" |INSERT INTO $tblName values (1, '', '') |""".stripMargin) ) assert(ex3.getMessage.contains("violated")) } } testEmptyToNull("reject empty string in check constraint") { val tblName = "empty_string_test" withTable(tblName) { sql( s""" |CREATE TABLE $tblName ( | c1 INT, | c2 STRING |) USING delta |PARTITIONED BY (c2); |""".stripMargin) sql( s""" |ALTER TABLE $tblName ADD CONSTRAINT test CHECK (c2 IS NOT NULL) |""".stripMargin) intercept[InvariantViolationException] ( sql( s""" |INSERT INTO ${tblName} VALUES (1, "") |""".stripMargin) ) } } test("streaming with additional project") { withSQLConf(DeltaSQLConf.CONVERT_EMPTY_TO_NULL_FOR_STRING_PARTITION_COL.key -> "true") { val tblName = "test" withTable(tblName) { withTempDir { checkpointDir => sql( s""" |CREATE TABLE $tblName ( | c1 INT, | c2 STRING |) USING delta |PARTITIONED BY (c2); |""".stripMargin) sql( s""" |ALTER TABLE $tblName ADD CONSTRAINT cons CHECK (c1 > 0) |""".stripMargin) val path = DeltaLog.forTable(spark, TableIdentifier(tblName)).dataPath.toString val stream = MemoryStream[Int] val q = stream.toDF() .map(_ => Tuple2(1, "a")) .toDF("c1", "c2") .writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .format("delta") .start(path) stream.addData(1) q.processAllAvailable() q.stop() } } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/schema/SchemaEnforcementSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.schema import java.io.File // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.{DeltaLog, DeltaOptions} import org.apache.spark.sql.delta.actions.SingleAction import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.test.shims.StreamingTestShims.MemoryStream import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import org.apache.spark.SparkConf import org.apache.spark.sql._ import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.streaming.StreamingQueryException import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ sealed trait SaveOperation { def apply(dfw: DataFrameWriter[_]): Unit } case class SaveWithPath(path: String = null) extends SaveOperation { override def apply(dfw: DataFrameWriter[_]): Unit = { if (path == null) dfw.save() else dfw.save(path) } } case class SaveAsTable(tableName: String) extends SaveOperation { override def apply(dfw: DataFrameWriter[_]): Unit = dfw.saveAsTable(tableName) } sealed trait SchemaEnforcementSuiteBase extends QueryTest with SharedSparkSession { protected def enableAutoMigration(f: => Unit): Unit = { withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { f } } protected def disableAutoMigration(f: => Unit): Unit = { withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "false") { f } } } sealed trait BatchWriterTest extends SchemaEnforcementSuiteBase with SharedSparkSession { def saveOperation: SaveOperation implicit class RichDataFrameWriter(dfw: DataFrameWriter[_]) { def append(path: File): Unit = { saveOperation(dfw.format("delta").mode("append").option("path", path.getAbsolutePath)) } def overwrite(path: File): Unit = { saveOperation(dfw.format("delta").mode("overwrite").option("path", path.getAbsolutePath)) } } def equivalenceTest(testName: String)(f: => Unit): Unit = { test(s"batch: $testName") { saveOperation match { case _: SaveWithPath => f case SaveAsTable(tbl) => withTable(tbl) { f } } } } } trait AppendSaveModeNullTests extends BatchWriterTest { import testImplicits._ equivalenceTest("JSON ETL workflow, NullType being only data column") { enableAutoMigration { val row1 = """{"key":"abc","id":null}""" withTempDir { dir => val schema1 = new StructType().add("key", StringType).add("id", NullType) val e = intercept[AnalysisException] { spark.read.schema(schema1).json(Seq(row1).toDS()).write.partitionBy("key").append(dir) } assert(e.getMessage.contains("NullType have been dropped")) } } } equivalenceTest("JSON ETL workflow, schema merging NullTypes") { enableAutoMigration { val row1 = """{"key":"abc","id":null,"extra":1}""" val row2 = """{"key":"def","id":2,"extra":null}""" val row3 = """{"key":"ghi","id":null,"extra":3}""" withTempDir { dir => val schema1 = new StructType() .add("key", StringType).add("id", NullType).add("extra", IntegerType) val schema2 = new StructType() .add("key", StringType).add("id", IntegerType).add("extra", NullType) spark.read.schema(schema1).json(Seq(row1).toDS()).write.append(dir) // NullType will be removed during the read checkAnswer( spark.read.format("delta").load(dir.getAbsolutePath), Row("abc", 1) :: Nil ) spark.read.schema(schema2).json(Seq(row2).toDS()).write.append(dir) spark.read.schema(schema1).json(Seq(row3).toDS()).write.append(dir) checkAnswer( spark.read.format("delta").load(dir.getAbsolutePath), Row("abc", null, 1) :: Row("def", 2, null) :: Row("ghi", null, 3) :: Nil ) } } } equivalenceTest("JSON ETL workflow, schema merging NullTypes - nested struct") { enableAutoMigration { val row1 = """{"key":"abc","top":{"id":null,"extra":1}}""" val row2 = """{"key":"def","top":{"id":2,"extra":null}}""" val row3 = """{"key":"ghi","top":{"id":null,"extra":3}}""" withTempDir { dir => val schema1 = new StructType().add("key", StringType) .add("top", new StructType().add("id", NullType).add("extra", IntegerType)) val schema2 = new StructType().add("key", StringType) .add("top", new StructType().add("id", IntegerType).add("extra", NullType)) val mergedSchema = new StructType().add("key", StringType) .add("top", new StructType().add("id", IntegerType).add("extra", IntegerType)) spark.read.schema(schema1).json(Seq(row1).toDS()).write.append(dir) // NullType will be removed during the read checkAnswer( spark.read.format("delta").load(dir.getAbsolutePath), Row("abc", Row(1)) :: Nil ) spark.read.schema(schema2).json(Seq(row2).toDS()).write.append(dir) assert(spark.read.format("delta").load(dir.getAbsolutePath).schema === mergedSchema) spark.read.schema(schema1).json(Seq(row3).toDS()).write.append(dir) assert(spark.read.format("delta").load(dir.getAbsolutePath).schema === mergedSchema) checkAnswer( spark.read.format("delta").load(dir.getAbsolutePath), Row("abc", Row(null, 1)) :: Row("def", Row(2, null)) :: Row("ghi", Row(null, 3)) :: Nil ) } } } equivalenceTest("JSON ETL workflow, schema merging NullTypes - throw error on complex types") { enableAutoMigration { val row1 = """{"key":"abc","top":[]}""" val row2 = """{"key":"abc","top":[{"id":null}]}""" withTempDir { dir => val schema1 = new StructType().add("key", StringType).add("top", ArrayType(NullType)) val schema2 = new StructType().add("key", StringType) .add("top", ArrayType(new StructType().add("id", NullType))) val e1 = intercept[AnalysisException] { spark.read.schema(schema1).json(Seq(row1).toDS()).write.append(dir) } assert(e1.getMessage.contains("NullType")) val e2 = intercept[AnalysisException] { spark.read.schema(schema2).json(Seq(row2).toDS()).write.append(dir) } assert(e2.getMessage.contains("NullType")) } } } } trait AppendSaveModeTests extends BatchWriterTest { import testImplicits._ equivalenceTest("reject schema changes by default") { disableAutoMigration { withTempDir { dir => spark.range(10).write.append(dir) val e = intercept[AnalysisException] { spark.range(10).withColumn("part", 'id + 1).write.append(dir) } assert(e.getMessage.contains(DeltaOptions.MERGE_SCHEMA_OPTION)) } } } equivalenceTest("allow schema changes when autoMigrate is enabled") { enableAutoMigration { withTempDir { dir => spark.range(10).write.append(dir) spark.range(10).withColumn("part", 'id + 1).write.append(dir) assert(spark.read.format("delta").load(dir.getAbsolutePath).schema.length == 2) } } } equivalenceTest("disallow schema changes when autoMigrate enabled but writer config disabled") { enableAutoMigration { withTempDir { dir => spark.range(10).write.append(dir) val e = intercept[AnalysisException] { spark.range(10).withColumn("part", 'id + 1).write .option(DeltaOptions.MERGE_SCHEMA_OPTION, "false").append(dir) } assert(e.getMessage.contains(DeltaOptions.MERGE_SCHEMA_OPTION)) } } } equivalenceTest("allow schema change with option") { disableAutoMigration { withTempDir { dir => spark.range(10).write.append(dir) spark.range(10).withColumn("part", 'id + 1).write .option(DeltaOptions.MERGE_SCHEMA_OPTION, "true").append(dir) assert(spark.read.format("delta").load(dir.getAbsolutePath).schema.length == 2) } } } equivalenceTest("JSON ETL workflow, NullType partition column should fail") { enableAutoMigration { val row1 = """{"key":"abc","id":null}""" withTempDir { dir => val schema1 = new StructType().add("key", StringType).add("id", NullType) intercept[AnalysisException] { spark.read.schema(schema1).json(Seq(row1).toDS()).write.partitionBy("id").append(dir) } intercept[AnalysisException] { // check case sensitivity with regards to column dropping spark.read.schema(schema1).json(Seq(row1).toDS()).write.partitionBy("iD").append(dir) } } } } equivalenceTest("reject columns that only differ by case - append") { withTempDir { dir => withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { intercept[AnalysisException] { spark.range(10).withColumn("ID", 'id + 1).write.append(dir) } intercept[AnalysisException] { spark.range(10).withColumn("ID", 'id + 1).write .option(DeltaOptions.MERGE_SCHEMA_OPTION, "true").append(dir) } intercept[AnalysisException] { spark.range(10).withColumn("a", 'id + 1).write .partitionBy("a", "A") .option(DeltaOptions.MERGE_SCHEMA_OPTION, "true").append(dir) } } } } equivalenceTest("ensure schema mismatch error message contains table ID") { disableAutoMigration { withTempDir { dir => spark.range(10).write.append(dir) val e = intercept[AnalysisException] { spark.range(10).withColumn("part", 'id + 1).write.append(dir) } assert(e.getMessage.contains("schema mismatch detected")) assert(e.getMessage.contains( s"Table ID: ${DeltaLog.forTable(spark, dir).unsafeVolatileTableId}")) } } } } trait AppendOutputModeTests extends SchemaEnforcementSuiteBase with SharedSparkSession with DeltaSQLTestUtils { import testImplicits._ testQuietly("reject schema changes by default - streaming") { withTempDir { dir => spark.range(10).write.format("delta").save(dir.getAbsolutePath) val memStream = MemoryStream[Long] val stream = memStream.toDS().toDF("value1234") // different column name .writeStream .option("checkpointLocation", new File(dir, "_checkpoint").getAbsolutePath) .format("delta") .start(dir.getAbsolutePath) try { disableAutoMigration { val e = intercept[StreamingQueryException] { memStream.addData(1L) stream.processAllAvailable() } assert(e.cause.isInstanceOf[AnalysisException]) assert(e.cause.getMessage.contains(DeltaOptions.MERGE_SCHEMA_OPTION)) } } finally { stream.stop() } } } testQuietly("reject schema changes when autoMigrate enabled but writer config disabled") { withTempDir { dir => spark.range(10).write.format("delta").save(dir.getAbsolutePath) val memStream = MemoryStream[Long] val stream = memStream.toDS().toDF("value1234") // different column name .writeStream .option("checkpointLocation", new File(dir, "_checkpoint").getAbsolutePath) .format("delta") .option(DeltaOptions.MERGE_SCHEMA_OPTION, "false") .start(dir.getAbsolutePath) try { enableAutoMigration { val e = intercept[StreamingQueryException] { memStream.addData(1L) stream.processAllAvailable() } assert(e.cause.isInstanceOf[AnalysisException]) assert(e.cause.getMessage.contains(DeltaOptions.MERGE_SCHEMA_OPTION)) } } finally { stream.stop() } } } test("allow schema changes when autoMigrate is enabled - streaming") { withTempDir { dir => spark.range(10).write.format("delta").save(dir.getAbsolutePath) enableAutoMigration { val memStream = MemoryStream[Long] val stream = memStream.toDS().toDF("value1234") // different column name .writeStream .option("checkpointLocation", new File(dir, "_checkpoint").getAbsolutePath) .format("delta") .start(dir.getAbsolutePath) try { memStream.addData(1L) stream.processAllAvailable() assert(spark.read.format("delta").load(dir.getAbsolutePath).schema.length == 2) } finally { stream.stop() } } } } test("allow schema change with option - streaming") { withTempDir { dir => spark.range(10).write.format("delta").save(dir.getAbsolutePath) val memStream = MemoryStream[Long] val stream = memStream.toDS().toDF("value1234") // different column name .writeStream .option("checkpointLocation", new File(dir, "_checkpoint").getAbsolutePath) .option(DeltaOptions.MERGE_SCHEMA_OPTION, "true") .format("delta") .start(dir.getAbsolutePath) try { disableAutoMigration { memStream.addData(1L) stream.processAllAvailable() assert(spark.read.format("delta").load(dir.getAbsolutePath).schema.length == 2) } } finally { stream.stop() } } } testQuietly("JSON ETL workflow, reject NullTypes") { enableAutoMigration { val row1 = """{"key":"abc","id":null}""" withTempDir { dir => val schema = new StructType().add("key", StringType).add("id", NullType) val memStream = MemoryStream[String] val stream = memStream.toDS().select(from_json('value, schema).as("value")) .select($"value.*") .writeStream .option("checkpointLocation", new File(dir, "_checkpoint").getAbsolutePath) .format("delta") .start(dir.getAbsolutePath) try { val e = intercept[StreamingQueryException] { memStream.addData(row1) stream.processAllAvailable() } assert(e.cause.isInstanceOf[AnalysisException]) assert(e.cause.getMessage.contains("NullType")) } finally { stream.stop() } } } } testQuietly("JSON ETL workflow, reject NullTypes on nested column") { enableAutoMigration { val row1 = """{"key":"abc","id":{"a":null}}""" withTempDir { dir => val schema = new StructType().add("key", StringType) .add("id", new StructType().add("a", NullType)) val memStream = MemoryStream[String] val stream = memStream.toDS().select(from_json('value, schema).as("value")) .select($"value.*") .writeStream .option("checkpointLocation", new File(dir, "_checkpoint").getAbsolutePath) .format("delta") .start(dir.getAbsolutePath) try { val e = intercept[StreamingQueryException] { memStream.addData(row1) stream.processAllAvailable() } assert(e.cause.isInstanceOf[AnalysisException]) assert(e.cause.getMessage.contains("NullType")) } finally { stream.stop() } } } } } trait OverwriteSaveModeTests extends BatchWriterTest { import testImplicits._ equivalenceTest("reject schema overwrites by default") { disableAutoMigration { withTempDir { dir => spark.range(10).write.overwrite(dir) val e = intercept[AnalysisException] { spark.range(10).withColumn("part", 'id + 1).write.overwrite(dir) } assert(e.getMessage.contains(DeltaOptions.OVERWRITE_SCHEMA_OPTION)) } } } equivalenceTest("can overwrite schema when using overwrite mode - option") { disableAutoMigration { withTempDir { dir => spark.range(5).toDF("id").write.overwrite(dir) spark.range(5).toDF("value").write.option(DeltaOptions.OVERWRITE_SCHEMA_OPTION, "true") .overwrite(dir) val df = spark.read.format("delta").load(dir.getAbsolutePath) assert(df.schema.fieldNames === Array("value")) } } } equivalenceTest("when autoMerge sqlConf is enabled, we merge schemas") { enableAutoMigration { withTempDir { dir => spark.range(5).toDF("id").write.overwrite(dir) spark.range(5).toDF("value").write.overwrite(dir) val df = spark.read.format("delta").load(dir.getAbsolutePath) assert(df.schema.fieldNames === Array("id", "value")) } } } equivalenceTest("reject migration when autoMerge sqlConf is enabled and writer config disabled") { enableAutoMigration { withTempDir { dir => spark.range(5).toDF("id").write.overwrite(dir) intercept[AnalysisException] { spark.range(5).toDF("value").write.option(DeltaOptions.MERGE_SCHEMA_OPTION, "false") .overwrite(dir) } val df = spark.read.format("delta").load(dir.getAbsolutePath) assert(df.schema.fieldNames === Array("id")) } } } equivalenceTest("schema merging with replaceWhere - sqlConf") { enableAutoMigration { withTempDir { dir => spark.range(5).toDF("id").withColumn("part", 'id % 2).write .partitionBy("part") .overwrite(dir) Seq((1L, 0L), (2L, 0L)).toDF("value", "part").write .option(DeltaOptions.REPLACE_WHERE_OPTION, "part = 0") .overwrite(dir) val df = spark.read.format("delta").load(dir.getAbsolutePath) assert(df.schema.fieldNames === Array("id", "part", "value")) } } } equivalenceTest("schema merging with replaceWhere - option") { disableAutoMigration { withTempDir { dir => spark.range(5).toDF("id").withColumn("part", 'id % 2).write .partitionBy("part") .overwrite(dir) Seq((1L, 0L), (2L, 0L)).toDF("value", "part").write .option(DeltaOptions.REPLACE_WHERE_OPTION, "part = 0") .option(DeltaOptions.MERGE_SCHEMA_OPTION, "true") .overwrite(dir) val df = spark.read.format("delta").load(dir.getAbsolutePath) assert(df.schema.fieldNames === Array("id", "part", "value")) } } } equivalenceTest("schema merging with replaceWhere - option case insensitive") { disableAutoMigration { withTempDir { dir => spark.range(5).toDF("id").withColumn("part", 'id % 2).write .partitionBy("part") .overwrite(dir) Seq((1L, 0L), (2L, 0L)).toDF("value", "part").write .option("RePlAcEwHeRe", "part = 0") .option("mErGeScHeMa", "true") .overwrite(dir) val df = spark.read.format("delta").load(dir.getAbsolutePath) assert(df.schema.fieldNames === Array("id", "part", "value")) } } } equivalenceTest("reject schema merging with replaceWhere - overwrite option") { disableAutoMigration { withTempDir { dir => spark.range(5).toDF("id").withColumn("part", 'id % 2).write .partitionBy("part") .overwrite(dir) val e = intercept[AnalysisException] { Seq((1L, 0L), (2L, 0L)).toDF("value", "part").write .option(DeltaOptions.REPLACE_WHERE_OPTION, "part = 0") .option(DeltaOptions.OVERWRITE_SCHEMA_OPTION, "true") .overwrite(dir) } assert(e.getMessage.contains(DeltaOptions.MERGE_SCHEMA_OPTION)) } } } equivalenceTest("reject schema merging with replaceWhere - no option") { disableAutoMigration { withTempDir { dir => spark.range(5).toDF("id").withColumn("part", 'id % 2).write .partitionBy("part") .overwrite(dir) val e = intercept[AnalysisException] { Seq((1L, 0L), (2L, 0L)).toDF("value", "part").write .partitionBy("part") .option(DeltaOptions.REPLACE_WHERE_OPTION, "part = 0") .overwrite(dir) } assert(e.getMessage.contains(DeltaOptions.MERGE_SCHEMA_OPTION)) } } } equivalenceTest("reject schema merging with replaceWhere - option set to false, config true") { enableAutoMigration { withTempDir { dir => spark.range(5).toDF("id").withColumn("part", 'id % 2).write .partitionBy("part") .overwrite(dir) val e = intercept[AnalysisException] { Seq((1L, 0L), (2L, 0L)).toDF("value", "part").write .partitionBy("part") .option(DeltaOptions.REPLACE_WHERE_OPTION, "part = 0") .option(DeltaOptions.MERGE_SCHEMA_OPTION, "false") .overwrite(dir) } assert(e.getMessage.contains(DeltaOptions.MERGE_SCHEMA_OPTION)) } } } equivalenceTest("reject change partitioning with overwrite - sqlConf") { enableAutoMigration { withTempDir { dir => spark.range(5).toDF("id").write .overwrite(dir) val e = intercept[AnalysisException] { spark.range(5).toDF("id").withColumn("part", 'id % 2).write .partitionBy("part") .overwrite(dir) } assert(e.getMessage.contains(DeltaOptions.OVERWRITE_SCHEMA_OPTION)) val deltaLog = DeltaLog.forTable(spark, dir) assert(deltaLog.snapshot.metadata.partitionColumns === Nil) assert(deltaLog.snapshot.metadata.schema.fieldNames === Array("id")) } } } equivalenceTest("can change partitioning with overwrite - option") { disableAutoMigration { withTempDir { dir => spark.range(5).toDF("id").write .overwrite(dir) spark.range(5).toDF("id").withColumn("part", 'id % 2).write .partitionBy("part") .option(DeltaOptions.OVERWRITE_SCHEMA_OPTION, "true") .overwrite(dir) val deltaLog = DeltaLog.forTable(spark, dir) assert(deltaLog.snapshot.metadata.partitionColumns === Seq("part")) assert(deltaLog.snapshot.metadata.schema.fieldNames === Array("id", "part")) } } } equivalenceTest("can't change partitioning with overwrite and replaceWhere - option") { disableAutoMigration { withTempDir { dir => spark.range(5).toDF("id").withColumn("part", 'id % 2).write .partitionBy("part") .overwrite(dir) intercept[AnalysisException] { spark.range(5).toDF("id").withColumn("part", lit(0L)).withColumn("test", 'id + 1).write .partitionBy("part", "test") .option(DeltaOptions.REPLACE_WHERE_OPTION, "part = 0") .option(DeltaOptions.OVERWRITE_SCHEMA_OPTION, "true") .overwrite(dir) } } } } equivalenceTest("can drop columns with overwriteSchema") { disableAutoMigration { withTempDir { dir => spark.range(5).toDF("id").withColumn("part", 'id % 2).write .overwrite(dir) spark.range(5).toDF("id").write .option(DeltaOptions.OVERWRITE_SCHEMA_OPTION, "true") .overwrite(dir) val deltaLog = DeltaLog.forTable(spark, dir) assert(deltaLog.snapshot.metadata.partitionColumns === Nil) assert(deltaLog.snapshot.metadata.schema.fieldNames === Array("id")) } } } equivalenceTest("can change column data type with overwriteSchema") { disableAutoMigration { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) spark.range(5).toDF("id").write .overwrite(dir) assert(deltaLog.snapshot.metadata.schema.head === StructField("id", LongType)) spark.range(5).toDF("id").selectExpr("cast(id as string) as id").write .option(DeltaOptions.OVERWRITE_SCHEMA_OPTION, "true") .overwrite(dir) assert(deltaLog.snapshot.metadata.schema.head === StructField("id", StringType)) } } } equivalenceTest("reject columns that only differ by case - overwrite") { withTempDir { dir => withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { intercept[AnalysisException] { spark.range(10).withColumn("ID", 'id + 1).write.overwrite(dir) } intercept[AnalysisException] { spark.range(10).withColumn("ID", 'id + 1).write .option(DeltaOptions.OVERWRITE_SCHEMA_OPTION, "true") .overwrite(dir) } intercept[AnalysisException] { spark.range(10).withColumn("a", 'id + 1).write .partitionBy("a", "A") .option(DeltaOptions.OVERWRITE_SCHEMA_OPTION, "true") .overwrite(dir) } } } } } trait CompleteOutputModeTests extends SchemaEnforcementSuiteBase with SharedSparkSession with DeltaSQLTestUtils { import testImplicits._ testQuietly("reject complete mode with new schema by default") { disableAutoMigration { withTempDir { dir => val memStream = MemoryStream[Long] val query = memStream.toDS().toDF("id") .withColumn("part", 'id % 3) .groupBy("part") .count() val stream1 = query.writeStream .option("checkpointLocation", new File(dir, "_checkpoint").getAbsolutePath) .outputMode("complete") .format("delta") .start(dir.getAbsolutePath) try { memStream.addData(1L) stream1.processAllAvailable() } finally { stream1.stop() } assert(spark.read.format("delta").load(dir.getAbsolutePath).schema.length == 2) val stream2 = query.withColumn("test", lit("abc")).writeStream .option("checkpointLocation", new File(dir, "_checkpoint").getAbsolutePath) .outputMode("complete") .format("delta") .start(dir.getAbsolutePath) try { val e = intercept[StreamingQueryException] { memStream.addData(2L) stream2.processAllAvailable() } assert(e.cause.isInstanceOf[AnalysisException]) assert(e.cause.getMessage.contains(DeltaOptions.OVERWRITE_SCHEMA_OPTION)) } finally { stream2.stop() } } } } test("complete mode can overwrite schema with option") { disableAutoMigration { withTempDir { dir => val memStream = MemoryStream[Long] val query = memStream.toDS().toDF("id") .withColumn("part", 'id % 3) .groupBy("part") .count() val stream1 = query.writeStream .option("checkpointLocation", new File(dir, "_checkpoint").getAbsolutePath) .option(DeltaOptions.OVERWRITE_SCHEMA_OPTION, "true") .outputMode("complete") .format("delta") .start(dir.getAbsolutePath) try { memStream.addData(1L) stream1.processAllAvailable() } finally { stream1.stop() } assert(spark.read.format("delta").load(dir.getAbsolutePath).schema.length == 2) val stream2 = query.withColumn("test", lit("abc")).writeStream .option("checkpointLocation", new File(dir, "_checkpoint").getAbsolutePath) .option(DeltaOptions.OVERWRITE_SCHEMA_OPTION, "true") .outputMode("complete") .format("delta") .start(dir.getAbsolutePath) try { memStream.addData(2L) stream2.processAllAvailable() memStream.addData(3L) stream2.processAllAvailable() } finally { stream2.stop() } val df = spark.read.format("delta").load(dir.getAbsolutePath) assert(df.schema.length == 3) val deltaLog = DeltaLog.forTable(spark, dir) val hadoopConf = deltaLog.newDeltaHadoopConf() val lastCommitFile = deltaLog.store .listFrom(FileNames.listingPrefix(deltaLog.logPath, 0L), hadoopConf) .map(_.getPath).filter(FileNames.isDeltaFile).toArray.last val lastCommitContainsMetadata = deltaLog.store.read(lastCommitFile, hadoopConf) .exists(JsonUtils.mapper.readValue[SingleAction](_).metaData != null) assert(!lastCommitContainsMetadata, "Metadata shouldn't be updated as long as schema doesn't change") checkAnswer( df, Row(0L, 1L, "abc") :: Row(1L, 1L, "abc") :: Row(2L, 1L, "abc") :: Nil) } } } test("complete mode behavior with autoMigrate enabled is to migrate schema") { enableAutoMigration { withTempDir { dir => val memStream = MemoryStream[Long] val query = memStream.toDS().toDF("id") .withColumn("part", 'id % 3) .groupBy("part") .count() val stream1 = query.writeStream .option("checkpointLocation", new File(dir, "_checkpoint").getAbsolutePath) .outputMode("complete") .format("delta") .start(dir.getAbsolutePath) try { memStream.addData(1L) stream1.processAllAvailable() } finally { stream1.stop() } assert(spark.read.format("delta").load(dir.getAbsolutePath).schema.length == 2) val stream2 = query.withColumn("test", lit("abc")).writeStream .option("checkpointLocation", new File(dir, "_checkpoint").getAbsolutePath) .outputMode("complete") .format("delta") .start(dir.getAbsolutePath) try { memStream.addData(2L) stream2.processAllAvailable() memStream.addData(3L) stream2.processAllAvailable() } finally { stream2.stop() } val df = spark.read.format("delta").load(dir.getAbsolutePath) assert(df.schema.length == 3) val deltaLog = DeltaLog.forTable(spark, dir) val hadoopConf = deltaLog.newDeltaHadoopConf() val lastCommitFile = deltaLog.store .listFrom(FileNames.listingPrefix(deltaLog.logPath, 0L), hadoopConf) .map(_.getPath).filter(FileNames.isDeltaFile).toArray.last val lastCommitContainsMetadata = deltaLog.store.read(lastCommitFile, hadoopConf) .exists(JsonUtils.mapper.readValue[SingleAction](_).metaData != null) assert(!lastCommitContainsMetadata, "Metadata shouldn't be updated as long as schema doesn't change") checkAnswer( df, Row(0L, 1L, "abc") :: Row(1L, 1L, "abc") :: Row(2L, 1L, "abc") :: Nil) } } } } class SchemaEnforcementWithPathSuite extends AppendSaveModeTests with AppendSaveModeNullTests with OverwriteSaveModeTests with DeltaSQLCommandTest { override val saveOperation = SaveWithPath() } class SchemaEnforcementWithTableSuite extends AppendSaveModeTests with OverwriteSaveModeTests with DeltaSQLCommandTest { override val saveOperation = SaveAsTable("delta_schema_test") } class SchemaEnforcementStreamingSuite extends AppendOutputModeTests with CompleteOutputModeTests with DeltaSQLCommandTest { } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/schema/SchemaUtilsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.schema // scalastyle:off import.ordering.noEmptyLine import java.util.Locale import java.util.regex.Pattern import scala.annotation.tailrec import org.apache.spark.sql.delta.{DeltaAnalysisException, DeltaLog, DeltaTestUtils, TypeWideningMode} import org.apache.spark.sql.delta.RowCommitVersion import org.apache.spark.sql.delta.RowId import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.schema.SchemaMergingUtils._ import org.apache.spark.sql.delta.sources.DeltaSourceUtils.GENERATION_EXPRESSION_METADATA_KEY import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.sources.DeltaSQLConf.AllowAutomaticWideningMode import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import io.delta.tables.DeltaTable import org.scalatest.GivenWhenThen import org.apache.spark.sql.{AnalysisException, Column, DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} import org.apache.spark.sql.catalyst.analysis.{caseInsensitiveResolution, caseSensitiveResolution, UnresolvedAttribute} import org.apache.spark.sql.catalyst.expressions.{Alias, Cast, Expression} import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project} import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ class SchemaUtilsSuite extends QueryTest with SharedSparkSession with GivenWhenThen with DeltaSQLTestUtils with DeltaSQLCommandTest { import SchemaUtils._ import TypeWideningMode._ import testImplicits._ private def expectFailure(shouldContain: String*)(f: => Unit): Unit = { val e = intercept[AnalysisException] { f } val msg = e.getMessage.toLowerCase(Locale.ROOT) assert(shouldContain.map(_.toLowerCase(Locale.ROOT)).forall(msg.contains), s"Error message '$msg' didn't contain: $shouldContain") } private def expectFailurePattern(shouldContainPatterns: String*)(f: => Unit): Unit = { val e = intercept[AnalysisException] { f } val patterns = shouldContainPatterns.map(regex => Pattern.compile(regex, Pattern.CASE_INSENSITIVE)) assert(patterns.forall(_.matcher(e.getMessage).find()), s"Error message '${e.getMessage}' didn't contain the patterns: $shouldContainPatterns") } private def expectAnalysisErrorClass( errorClass: String, params: Map[String, String], matchPVals: Boolean = true)( f: => Unit): Unit = { val e = intercept[AnalysisException] { f } @tailrec def getError(ex: Throwable): Option[DeltaAnalysisException] = ex match { case e: DeltaAnalysisException if e.getErrorClass() == errorClass => Some(e) case e: AnalysisException => getError(e.getCause) case _ => None } val err = getError(e) assert(err.isDefined, "exception with the error class not found") checkError( err.get, errorClass, parameters = params, matchPVals = matchPVals) } ///////////////////////////// // Duplicate Column Checks ///////////////////////////// test("duplicate column name in top level") { val schema = new StructType() .add("dupColName", IntegerType) .add("b", IntegerType) .add("dupColName", StringType) expectFailure("dupColName") { checkColumnNameDuplication(schema, "") } } test("duplicate column name in top level - case sensitivity") { val schema = new StructType() .add("dupColName", IntegerType) .add("b", IntegerType) .add("dupCOLNAME", StringType) expectFailure("dupColName") { checkColumnNameDuplication(schema, "") } } test("duplicate column name for nested column + non-nested column") { val schema = new StructType() .add("dupColName", new StructType() .add("a", IntegerType) .add("b", IntegerType)) .add("dupColName", IntegerType) expectFailure("dupColName") { checkColumnNameDuplication(schema, "") } } test("duplicate column name for nested column + non-nested column - case sensitivity") { val schema = new StructType() .add("dupColName", new StructType() .add("a", IntegerType) .add("b", IntegerType)) .add("dupCOLNAME", IntegerType) expectFailure("dupCOLNAME") { checkColumnNameDuplication(schema, "") } } test("duplicate column name in nested level") { val schema = new StructType() .add("top", new StructType() .add("dupColName", IntegerType) .add("b", IntegerType) .add("dupColName", StringType) ) expectFailure("top.dupColName") { checkColumnNameDuplication(schema, "") } } test("duplicate column name in nested level - case sensitivity") { val schema = new StructType() .add("top", new StructType() .add("dupColName", IntegerType) .add("b", IntegerType) .add("dupCOLNAME", StringType) ) expectFailure("top.dupColName") { checkColumnNameDuplication(schema, "") } } test("duplicate column name in double nested level") { val schema = new StructType() .add("top", new StructType() .add("b", new StructType() .add("dupColName", StringType) .add("c", IntegerType) .add("dupColName", StringType)) .add("d", IntegerType) ) expectFailure("top.b.dupColName") { checkColumnNameDuplication(schema, "") } } test("duplicate column name in double nested array") { val schema = new StructType() .add("top", new StructType() .add("b", ArrayType(ArrayType(new StructType() .add("dupColName", StringType) .add("c", IntegerType) .add("dupColName", StringType)))) .add("d", IntegerType) ) expectFailure("top.b.element.element.dupColName") { checkColumnNameDuplication(schema, "") } } test("duplicate column name in double nested map") { val keyType = new StructType() .add("dupColName", IntegerType) .add("d", StringType) expectFailure("top.b.key.dupColName") { val schema = new StructType() .add("top", new StructType() .add("b", MapType(keyType.add("dupColName", StringType), keyType)) ) checkColumnNameDuplication(schema, "") } expectFailure("top.b.value.dupColName") { val schema = new StructType() .add("top", new StructType() .add("b", MapType(keyType, keyType.add("dupColName", StringType))) ) checkColumnNameDuplication(schema, "") } // This is okay val schema = new StructType() .add("top", new StructType() .add("b", MapType(keyType, keyType)) ) checkColumnNameDuplication(schema, "") } test("duplicate column name in nested array") { val schema = new StructType() .add("top", ArrayType(new StructType() .add("dupColName", IntegerType) .add("b", IntegerType) .add("dupColName", StringType)) ) expectFailure("top.element.dupColName") { checkColumnNameDuplication(schema, "") } } test("duplicate column name in nested array - case sensitivity") { val schema = new StructType() .add("top", ArrayType(new StructType() .add("dupColName", IntegerType) .add("b", IntegerType) .add("dupCOLNAME", StringType)) ) expectFailure("top.element.dupColName") { checkColumnNameDuplication(schema, "") } } test("non duplicate column because of back tick") { val schema = new StructType() .add("top", new StructType() .add("a", IntegerType) .add("b", IntegerType)) .add("top.a", IntegerType) checkColumnNameDuplication(schema, "") } test("non duplicate column because of back tick - nested") { val schema = new StructType() .add("first", new StructType() .add("top", new StructType() .add("a", IntegerType) .add("b", IntegerType)) .add("top.a", IntegerType)) checkColumnNameDuplication(schema, "") } test("duplicate column with back ticks - nested") { val schema = new StructType() .add("first", new StructType() .add("top.a", StringType) .add("b", IntegerType) .add("top.a", IntegerType)) expectFailure("first.`top.a`") { checkColumnNameDuplication(schema, "") } } test("duplicate column with back ticks - nested and case sensitivity") { val schema = new StructType() .add("first", new StructType() .add("TOP.a", StringType) .add("b", IntegerType) .add("top.a", IntegerType)) expectFailure("first.`top.a`") { checkColumnNameDuplication(schema, "") } } ///////////////////////////// // Read Compatibility Checks ///////////////////////////// /** * Tests change of datatype within a schema. * - the make() function is a "factory" function to create schemas that vary only by the * given datatype in a specific position in the schema. * - other tests will call this method with different make() functions to test datatype * incompatibility in all the different places within a schema (in a top-level struct, * in a nested struct, as the element type of an array, etc.) */ def testDatatypeChange(scenario: String)(make: DataType => StructType): Unit = { val schemas = Map( ("int", make(IntegerType)), ("string", make(StringType)), ("struct", make(new StructType().add("a", StringType))), ("array", make(ArrayType(IntegerType))), ("map", make(MapType(StringType, FloatType))) ) test(s"change of datatype should fail read compatibility - $scenario") { for (a <- schemas.keys; b <- schemas.keys if a != b) { assert(!isReadCompatible(schemas(a), schemas(b)), s"isReadCompatible should have failed for: ${schemas(a)}, ${schemas(b)}") } } } /** * Tests change of nullability within a schema (making a field nullable is not allowed, * but making a nullable field non-nullable is ok). * - the make() function is a "factory" function to create schemas that vary only by the * nullability (of a field, array element, or map values) in a specific position in the schema. * - other tests will call this method with different make() functions to test nullability * incompatibility in all the different places within a schema (in a top-level struct, * in a nested struct, for the element type of an array, etc.) */ def testNullability(scenario: String)(make: Boolean => StructType): Unit = { val nullable = make(true) val nonNullable = make(false) Seq(true, false).foreach { forbidTightenNullability => val (blockedCase, blockedExisting, blockedRead) = if (forbidTightenNullability) { (s"tighten nullability should fail read compatibility " + s"(forbidTightenNullability=$forbidTightenNullability) - $scenario", nullable, nonNullable) } else { (s"relax nullability should fail read compatibility " + s"(forbidTightenNullability=$forbidTightenNullability) - $scenario", nonNullable, nullable) } val (allowedCase, allowedExisting, allowedRead) = if (forbidTightenNullability) { (s"relax nullability should not fail read compatibility " + s"(forbidTightenNullability=$forbidTightenNullability) - $scenario", nonNullable, nullable) } else { (s"tighten nullability should not fail read compatibility " + s"(forbidTightenNullability=$forbidTightenNullability) - $scenario", nullable, nonNullable) } test(blockedCase) { assert(!isReadCompatible(blockedExisting, blockedRead, forbidTightenNullability)) } test(allowedCase) { assert(isReadCompatible(allowedExisting, allowedRead, forbidTightenNullability)) } } } /** * Tests for fields of a struct: adding/dropping fields, changing nullability, case variation * - The make() function is a "factory" method to produce schemas. It takes a function that * mutates a struct (for example, but adding a column, or it could just not make any change). * - Following tests will call this method with different factory methods, to mutate the * various places where a struct can appear (at the top-level, nested in another struct, * within an array, etc.) * - This allows us to have one shared code to test compatibility of a struct field in all the * different places where it may occur. */ def testColumnVariations(scenario: String) (make: (StructType => StructType) => StructType): Unit = { // generate one schema without extra column, one with, one nullable, and one with mixed case val withoutExtra = make(struct => struct) // produce struct WITHOUT extra field val withExtraNullable = make(struct => struct.add("extra", StringType)) val withExtraMixedCase = make(struct => struct.add("eXtRa", StringType)) val withExtraNonNullable = make(struct => struct.add("extra", StringType, nullable = false)) test(s"dropping a field should fail read compatibility - $scenario") { assert(!isReadCompatible(withExtraNullable, withoutExtra)) } test(s"adding a nullable field should not fail read compatibility - $scenario") { assert(isReadCompatible(withoutExtra, withExtraNullable)) } test(s"adding a non-nullable field should not fail read compatibility - $scenario") { assert(isReadCompatible(withoutExtra, withExtraNonNullable)) } test(s"case variation of field name should fail read compatibility - $scenario") { assert(!isReadCompatible(withExtraNullable, withExtraMixedCase)) } testNullability(scenario)(b => make(struct => struct.add("extra", StringType, nullable = b))) testDatatypeChange(scenario)(datatype => make(struct => struct.add("extra", datatype))) } // -------------------------------------------------------------------- // tests for all kinds of places where a field can appear in a struct // -------------------------------------------------------------------- testColumnVariations("top level")( f => f(new StructType().add("a", IntegerType))) testColumnVariations("nested struct")( f => new StructType() .add("a", f(new StructType().add("b", IntegerType)))) testColumnVariations("nested in array")( f => new StructType() .add("array", ArrayType( f(new StructType().add("b", IntegerType))))) testColumnVariations("nested in map key")( f => new StructType() .add("map", MapType( f(new StructType().add("b", IntegerType)), StringType))) testColumnVariations("nested in map value")( f => new StructType() .add("map", MapType( StringType, f(new StructType().add("b", IntegerType))))) // -------------------------------------------------------------------- // tests for data type change in places other than struct // -------------------------------------------------------------------- testDatatypeChange("array element")( datatype => new StructType() .add("array", ArrayType(datatype))) testDatatypeChange("map key")( datatype => new StructType() .add("map", MapType(datatype, StringType))) testDatatypeChange("map value")( datatype => new StructType() .add("map", MapType(StringType, datatype))) // -------------------------------------------------------------------- // tests for nullability change in places other than struct // -------------------------------------------------------------------- testNullability("array contains null")( b => new StructType() .add("array", ArrayType(StringType, containsNull = b))) testNullability("map contains null values")( b => new StructType() .add("map", MapType(IntegerType, StringType, valueContainsNull = b))) testNullability("map nested in array")( b => new StructType() .add("map", ArrayType( MapType(IntegerType, StringType, valueContainsNull = b)))) testNullability("array nested in map")( b => new StructType() .add("map", MapType( IntegerType, ArrayType(StringType, containsNull = b)))) //////////////////////////// // reportDifference //////////////////////////// /** * @param existing the existing schema to compare to * @param specified the new specified schema * @param expected an expected list of messages, each describing a schema difference. * Every expected message is actually a regex patterns that is matched * against all diffs that are returned. This is necessary to tolerate * variance in ordering of field names, for example in a message such as * "Specified schema has additional field(s): x, y", we cannot predict * the order of x and y. */ def testReportDifferences(testName: String) (existing: StructType, specified: StructType, expected: String*): Unit = { test(testName) { val differences = SchemaUtils.reportDifferences(existing, specified) // make sure every expected difference is reported expected foreach ((exp: String) => assert(differences.exists(message => exp.r.findFirstMatchIn(message).isDefined), s"""Difference not reported. |Expected: |- $exp |Reported: ${differences.mkString("\n- ", "\n- ", "")} """.stripMargin)) // make sure there are no extra differences reported assert(expected.size == differences.size, s"""Too many differences reported. |Expected: ${expected.mkString("\n- ", "\n- ", "")} |Reported: ${differences.mkString("\n- ", "\n- ", "")} """.stripMargin) } } testReportDifferences("extra columns should be reported as a difference")( existing = new StructType() .add("a", IntegerType), specified = new StructType() .add("a", IntegerType) .add("b", StringType), expected = "additional field[(]s[)]: b" ) testReportDifferences("missing columns should be reported as a difference")( existing = new StructType() .add("a", IntegerType) .add("b", StringType), specified = new StructType() .add("a", IntegerType), expected = "missing field[(]s[)]: b" ) testReportDifferences("making a column nullable should be reported as a difference")( existing = new StructType() .add("a", IntegerType, nullable = false) .add("b", StringType, nullable = true), specified = new StructType() .add("a", IntegerType, nullable = true) .add("b", StringType, nullable = true), expected = "a is nullable in specified schema but non-nullable in existing schema" ) testReportDifferences("making a column non-nullable should be reported as a difference")( existing = new StructType() .add("a", IntegerType, nullable = false) .add("b", StringType, nullable = true), specified = new StructType() .add("a", IntegerType, nullable = false) .add("b", StringType, nullable = false), expected = "b is non-nullable in specified schema but nullable in existing schema" ) testReportDifferences("change in column metadata should be reported as a difference")( existing = new StructType() .add("a", IntegerType, nullable = true, new MetadataBuilder().putString("x", "1").build()) .add("b", StringType), specified = new StructType() .add("a", IntegerType, nullable = true, new MetadataBuilder().putString("x", "2").build()) .add("b", StringType), expected = "metadata for field a is different" ) testReportDifferences("change in generation expression for generated columns")( existing = new StructType() .add("a", IntegerType, nullable = true, new MetadataBuilder() .putString(GENERATION_EXPRESSION_METADATA_KEY, "b + 1") .putString("x", "1").build()) .add("b", StringType), specified = new StructType() .add("a", IntegerType, nullable = true, new MetadataBuilder() .putString(GENERATION_EXPRESSION_METADATA_KEY, "1 + b") .putString("x", "1").build()) .add("b", StringType), // Regex flags: DOTALL and MULTILINE expected = "(?sm)generation expression for field a is different" + // Not include "(?!.*metadata for field a is different)" ) testReportDifferences("change in column metadata for generated columns")( existing = new StructType() .add("a", IntegerType, nullable = true, new MetadataBuilder() .putString(GENERATION_EXPRESSION_METADATA_KEY, "b + 1") .putString("x", "1").build()) .add("b", StringType), specified = new StructType() .add("a", IntegerType, nullable = true, new MetadataBuilder() .putString(GENERATION_EXPRESSION_METADATA_KEY, "b + 1") .putString("x", "2").build()) .add("b", StringType), expected = "metadata for field a is different" ) testReportDifferences("change in generation expression and metadata for generated columns")( existing = new StructType() .add("a", IntegerType, nullable = true, new MetadataBuilder() .putString(GENERATION_EXPRESSION_METADATA_KEY, "b + 1") .putString("x", "1").build()) .add("b", StringType), specified = new StructType() .add("a", IntegerType, nullable = true, new MetadataBuilder() .putString(GENERATION_EXPRESSION_METADATA_KEY, "b + 2") .putString("x", "2").build()) .add("b", StringType), // Regex flags: DOTALL and MULTILINE expected = "(?sm)generation expression for field a is different" + ".*metadata for field a is different" ) testReportDifferences("change of column type should be reported as a difference")( existing = new StructType() .add("a", IntegerType) .add("b", StringType), specified = new StructType() .add("a", IntegerType) .add("b", new ArrayType( StringType, containsNull = false)), expected = "type for b is different" ) testReportDifferences("change of array nullability should be reported as a difference")( existing = new StructType() .add("a", IntegerType) .add("b", new ArrayType( new StructType().add("x", LongType), containsNull = true)), specified = new StructType() .add("a", IntegerType) .add("b", new ArrayType( new StructType().add("x", LongType), containsNull = false)), expected = "b\\[\\] can not contain null in specified schema but can in existing" ) testReportDifferences("change of element type should be reported as a difference")( existing = new StructType() .add("a", IntegerType) .add("b", new ArrayType(LongType, containsNull = true)), specified = new StructType() .add("a", IntegerType) .add("b", new ArrayType(StringType, containsNull = true)), expected = "type for b\\[\\] is different" ) testReportDifferences("change of element struct type should be reported as a difference")( existing = new StructType() .add("a", IntegerType) .add("b", new ArrayType( new StructType() .add("x", LongType), containsNull = true)), specified = new StructType() .add("a", IntegerType) .add("b", new ArrayType( new StructType() .add("x", StringType), containsNull = true)), expected = "type for b\\[\\].x is different" ) testReportDifferences("change of map value nullability should be reported as a difference")( existing = new StructType() .add("a", IntegerType) .add("b", new MapType( StringType, new StructType().add("x", LongType), valueContainsNull = true)), specified = new StructType() .add("a", IntegerType) .add("b", new MapType( StringType, new StructType().add("x", LongType), valueContainsNull = false)), expected = "b can not contain null values in specified schema but can in existing" ) testReportDifferences("change of map key type should be reported as a difference")( existing = new StructType() .add("a", IntegerType) .add("b", new MapType(LongType, StringType, valueContainsNull = true)), specified = new StructType() .add("a", IntegerType) .add("b", new MapType(StringType, StringType, valueContainsNull = true)), expected = "type for b\\[key\\] is different" ) testReportDifferences("change of value struct type should be reported as a difference")( existing = new StructType() .add("a", IntegerType) .add("b", new MapType( StringType, new StructType().add("x", LongType), valueContainsNull = true)), specified = new StructType() .add("a", IntegerType) .add("b", new MapType( StringType, new StructType().add("x", FloatType), valueContainsNull = true)), expected = "type for b\\[value\\].x is different" ) testReportDifferences("nested extra columns should be reported as a difference")( existing = new StructType() .add("x", new StructType() .add("a", IntegerType)), specified = new StructType() .add("x", new StructType() .add("a", IntegerType) .add("b", StringType) .add("c", LongType)), expected = "additional field[(]s[)]: (x.b, x.c|x.c, x.b)" ) testReportDifferences("nested missing columns should be reported as a difference")( existing = new StructType() .add("x", new StructType() .add("a", IntegerType) .add("b", StringType) .add("c", FloatType)), specified = new StructType() .add("x", new StructType() .add("a", IntegerType)), expected = "missing field[(]s[)]: (x.b, x.c|x.c, x.b)" ) testReportDifferences("making a nested column nullable should be reported as a difference")( existing = new StructType() .add("x", new StructType() .add("a", IntegerType, nullable = false) .add("b", StringType, nullable = true)), specified = new StructType() .add("x", new StructType() .add("a", IntegerType, nullable = true) .add("b", StringType, nullable = true)), expected = "x.a is nullable in specified schema but non-nullable in existing schema" ) testReportDifferences("making a nested column non-nullable should be reported as a difference")( existing = new StructType() .add("x", new StructType() .add("a", IntegerType, nullable = false) .add("b", StringType, nullable = true)), specified = new StructType() .add("x", new StructType() .add("a", IntegerType, nullable = false) .add("b", StringType, nullable = false)), expected = "x.b is non-nullable in specified schema but nullable in existing schema" ) testReportDifferences("change in nested column metadata should be reported as a difference")( existing = new StructType() .add("x", new StructType() .add("a", IntegerType, nullable = true, new MetadataBuilder().putString("x", "1").build()) .add("b", StringType)), specified = new StructType() .add("x", new StructType() .add("a", IntegerType, nullable = true, new MetadataBuilder().putString("x", "2").build()) .add("b", StringType)), expected = "metadata for field x.a is different" ) testReportDifferences("change of nested column type should be reported as a difference")( existing = new StructType() .add("x", new StructType() .add("a", IntegerType) .add("b", StringType)), specified = new StructType() .add("x", new StructType() .add("a", IntegerType) .add("b", new ArrayType( StringType, containsNull = false))), expected = "type for x.b is different" ) testReportDifferences("change of nested array nullability should be reported as a difference")( existing = new StructType() .add("x", new StructType() .add("a", IntegerType) .add("b", new ArrayType( new StructType() .add("x", LongType), containsNull = true))), specified = new StructType() .add("x", new StructType() .add("a", IntegerType) .add("b", new ArrayType( new StructType() .add("x", LongType), containsNull = false))), expected = "x.b\\[\\] can not contain null in specified schema but can in existing" ) testReportDifferences("change of nested element type should be reported as a difference")( existing = new StructType() .add("x", new StructType() .add("a", IntegerType) .add("b", new ArrayType(LongType, containsNull = true))), specified = new StructType() .add("x", new StructType() .add("a", IntegerType) .add("b", new ArrayType(StringType, containsNull = true))), expected = "type for x.b\\[\\] is different" ) testReportDifferences("change of nested element struct type should be reported as a difference")( existing = new StructType() .add("x", new StructType() .add("a", IntegerType) .add("b", new ArrayType( new StructType() .add("x", LongType), containsNull = true))), specified = new StructType() .add("x", new StructType() .add("a", IntegerType) .add("b", new ArrayType( new StructType() .add("x", StringType), containsNull = true))), expected = "type for x.b\\[\\].x is different" ) private val piiTrue = new MetadataBuilder().putBoolean("pii", value = true).build() private val piiFalse = new MetadataBuilder().putBoolean("pii", value = false).build() testReportDifferences("multiple differences should be reported")( existing = new StructType() .add("a", IntegerType) .add("b", StringType) .add("c", BinaryType) .add("f", LongType, nullable = true, piiTrue) .add("g", new MapType( IntegerType, new StructType() .add("a", IntegerType, nullable = false, piiFalse) .add("b", StringType) .add("d", new ArrayType( LongType, containsNull = false )), valueContainsNull = true)) .add("h", new MapType( LongType, StringType, valueContainsNull = true)), specified = new StructType() .add("a", FloatType) .add("d", StringType) .add("e", LongType) .add("f", LongType, nullable = false, piiFalse) .add("g", new MapType( StringType, new StructType() .add("a", LongType, nullable = true) .add("c", StringType) .add("d", new ArrayType( BooleanType, containsNull = true )), valueContainsNull = false)) .add("h", new MapType( LongType, new ArrayType(IntegerType, containsNull = false), valueContainsNull = true)), "type for a is different", "additional field[(]s[)]: (d, e|e, d)", "missing field[(]s[)]: (b, c|c, b)", "f is non-nullable in specified schema but nullable", "metadata for field f is different", "type for g\\[key\\] is different", "g can not contain null values in specified schema but can in existing", "additional field[(]s[)]: g\\[value\\].c", "missing field[(]s[)]: g\\[value\\].b", "type for g\\[value\\].a is different", "g\\[value\\].a is nullable in specified schema but non-nullable in existing", "metadata for field g\\[value\\].a is different", "field g\\[value\\].d\\[\\] can contain null in specified schema but can not in existing", "type for g\\[value\\].d\\[\\] is different", "type for h\\[value\\] is different" ) //////////////////////////// // findColumnPosition //////////////////////////// test("findColumnPosition") { val schema = new StructType() .add("struct", new StructType() .add("a", IntegerType) .add("b", IntegerType)) .add("array", ArrayType(new StructType() .add("c", IntegerType) .add("d", IntegerType))) .add("field", StringType) .add("map", MapType( new StructType() .add("e", IntegerType), new StructType() .add("f", IntegerType))) .add("mapStruct", MapType( IntegerType, new StructType() .add("g", new StructType() .add("h", IntegerType)))) .add("arrayMap", ArrayType( MapType( new StructType() .add("i", IntegerType), new StructType() .add("j", IntegerType)))) val List(structIdx, arrayIdx, fieldIdx, mapIdx, mapStructIdx, arrayMapIdx) = (0 to 5).toList val ARRAY_ELEMENT_INDEX = 0 val MAP_KEY_INDEX = 0 val MAP_VALUE_INDEX = 1 def checkPosition(column: Seq[String], position: Seq[Int]): Unit = assert(SchemaUtils.findColumnPosition(column, schema) === position) checkPosition(Seq("struct"), Seq(structIdx)) checkPosition(Seq("STRucT"), Seq(structIdx)) expectFailure("Couldn't find", schema.treeString) { SchemaUtils.findColumnPosition(Seq("struct", "array"), schema) } checkPosition(Seq("struct", "a"), Seq(structIdx, 0)) checkPosition(Seq("STRucT", "a"), Seq(structIdx, 0)) checkPosition(Seq("struct", "A"), Seq(structIdx, 0)) checkPosition(Seq("STRucT", "A"), Seq(structIdx, 0)) checkPosition(Seq("struct", "b"), Seq(structIdx, 1)) checkPosition(Seq("array"), Seq(arrayIdx)) checkPosition(Seq("array", "element", "C"), Seq(arrayIdx, ARRAY_ELEMENT_INDEX, 0)) checkPosition(Seq("array", "element", "d"), Seq(arrayIdx, ARRAY_ELEMENT_INDEX, 1)) checkPosition(Seq("field"), Seq(fieldIdx)) checkPosition(Seq("map"), Seq(mapIdx)) checkPosition(Seq("map", "key", "e"), Seq(mapIdx, MAP_KEY_INDEX, 0)) checkPosition(Seq("map", "value", "f"), Seq(mapIdx, MAP_VALUE_INDEX, 0)) checkPosition(Seq("map", "value", "F"), Seq(mapIdx, MAP_VALUE_INDEX, 0)) checkPosition(Seq("mapStruct", "key"), Seq(mapStructIdx, MAP_KEY_INDEX)) checkPosition(Seq("mapStruct", "value", "g"), Seq(mapStructIdx, MAP_VALUE_INDEX, 0)) checkPosition(Seq("mapStruct", "key"), Seq(mapStructIdx, MAP_KEY_INDEX)) checkPosition(Seq("mapStruct", "value"), Seq(mapStructIdx, MAP_VALUE_INDEX)) checkPosition(Seq("arrayMap"), Seq(arrayMapIdx)) checkPosition(Seq("arrayMap", "element"), Seq(arrayMapIdx, ARRAY_ELEMENT_INDEX)) checkPosition( Seq("arrayMap", "element", "key"), Seq(arrayMapIdx, ARRAY_ELEMENT_INDEX, MAP_KEY_INDEX)) checkPosition( Seq("arrayMap", "element", "value"), Seq(arrayMapIdx, ARRAY_ELEMENT_INDEX, MAP_VALUE_INDEX)) checkPosition( Seq("arrayMap", "element", "key", "i"), Seq(arrayMapIdx, ARRAY_ELEMENT_INDEX, MAP_KEY_INDEX, 0)) checkPosition( Seq("arrayMap", "element", "value", "j"), Seq(arrayMapIdx, ARRAY_ELEMENT_INDEX, MAP_VALUE_INDEX, 0)) val resolver = org.apache.spark.sql.catalyst.analysis.caseSensitiveResolution Seq(Seq("STRucT", "b"), Seq("struct", "B"), Seq("array", "element", "C"), Seq("map", "key", "E")).foreach { column => expectFailure("Couldn't find", schema.treeString) { SchemaUtils.findColumnPosition(column, schema, resolver) } } } test("findColumnPosition that doesn't exist") { val schema = new StructType() .add("a", IntegerType) .add("b", MapType(StringType, StringType)) .add("c", ArrayType(IntegerType)) expectFailure("Couldn't find", schema.treeString) { SchemaUtils.findColumnPosition(Seq("d"), schema) } expectFailure("A MapType was found", "mapType", schema.treeString) { SchemaUtils.findColumnPosition(Seq("b", "c"), schema) } expectFailure("An ArrayType was found", "arrayType", schema.treeString) { SchemaUtils.findColumnPosition(Seq("c", "b"), schema) } } //////////////////////////// // getNestedFieldFromPosition //////////////////////////// test("getNestedFieldFromPosition") { val a = StructField("a", IntegerType) val b = StructField("b", IntegerType) val c = StructField("c", IntegerType) val d = StructField("d", IntegerType) val e = StructField("e", IntegerType) val f = StructField("f", IntegerType) val g = StructField("g", IntegerType) val field = StructField("field", StringType) val struct = StructField("struct", new StructType().add(a).add(b)) val arrayElement = StructField("element", new StructType().add(c)) val array = StructField("array", ArrayType(arrayElement.dataType)) val mapKey = StructField("key", new StructType().add(d)) val mapValue = StructField("value", new StructType().add(e)) val map = StructField("map", MapType( keyType = mapKey.dataType, valueType = mapValue.dataType)) val arrayMapKey = StructField("key", new StructType().add(f)) val arrayMapValue = StructField("value", new StructType().add(g)) val arrayMapElement = StructField("element", MapType( keyType = arrayMapKey.dataType, valueType = arrayMapValue.dataType)) val arrayMap = StructField("arrayMap", ArrayType(arrayMapElement.dataType)) val root = StructField("root", StructType(Seq(field, struct, array, map, arrayMap))) val List(fieldIdx, structIdx, arrayIdx, mapIdx, arrayMapIdx) = (0 to 4).toList val ARRAY_ELEMENT_INDEX = 0 val MAP_KEY_INDEX = 0 val MAP_VALUE_INDEX = 1 def checkField(position: Seq[Int], expected: StructField): Unit = assert(getNestedFieldFromPosition(root, position) === expected) checkField(Seq.empty, root) checkField(Seq(fieldIdx), field) checkField(Seq(structIdx), struct) checkField(Seq(structIdx, 0), a) checkField(Seq(structIdx, 1), b) checkField(Seq(arrayIdx), array) checkField(Seq(arrayIdx, ARRAY_ELEMENT_INDEX), arrayElement) checkField(Seq(arrayIdx, ARRAY_ELEMENT_INDEX, 0), c) checkField(Seq(mapIdx), map) checkField(Seq(mapIdx, MAP_KEY_INDEX), mapKey) checkField(Seq(mapIdx, MAP_VALUE_INDEX), mapValue) checkField(Seq(mapIdx, MAP_KEY_INDEX, 0), d) checkField(Seq(mapIdx, MAP_VALUE_INDEX, 0), e) checkField(Seq(arrayMapIdx), arrayMap) checkField(Seq(arrayMapIdx, ARRAY_ELEMENT_INDEX), arrayMapElement) checkField(Seq(arrayMapIdx, ARRAY_ELEMENT_INDEX, MAP_KEY_INDEX), arrayMapKey) checkField(Seq(arrayMapIdx, ARRAY_ELEMENT_INDEX, MAP_VALUE_INDEX), arrayMapValue) checkField(Seq(arrayMapIdx, ARRAY_ELEMENT_INDEX, MAP_KEY_INDEX, 0), f) checkField(Seq(arrayMapIdx, ARRAY_ELEMENT_INDEX, MAP_VALUE_INDEX, 0), g) def checkError(position: Seq[Int]): Unit = assertThrows[IllegalArgumentException] { getNestedFieldFromPosition(root, position) } checkError(Seq(-1)) checkError(Seq(fieldIdx, 0)) checkError(Seq(structIdx, -1)) checkError(Seq(structIdx, 2)) checkError(Seq(arrayIdx, ARRAY_ELEMENT_INDEX - 1)) checkError(Seq(arrayIdx, ARRAY_ELEMENT_INDEX + 1)) checkError(Seq(mapIdx, MAP_KEY_INDEX - 1)) checkError(Seq(mapIdx, MAP_VALUE_INDEX + 1)) checkError(Seq(arrayMapIdx, ARRAY_ELEMENT_INDEX - 1)) checkError(Seq(arrayMapIdx, ARRAY_ELEMENT_INDEX + 1)) checkError(Seq(arrayMapIdx, ARRAY_ELEMENT_INDEX, MAP_KEY_INDEX - 1)) checkError(Seq(arrayMapIdx, ARRAY_ELEMENT_INDEX, MAP_VALUE_INDEX + 1)) checkError(Seq(arrayMapIdx + 1)) } test("getNestedTypeFromPosition") { val schema = new StructType().add("a", IntegerType) assert(getNestedTypeFromPosition(schema, Seq.empty) === schema) assert(getNestedTypeFromPosition(schema, Seq(0)) === IntegerType) assertThrows[IllegalArgumentException] { getNestedTypeFromPosition(schema, Seq(-1)) } assertThrows[IllegalArgumentException] { getNestedTypeFromPosition(schema, Seq(1)) } } //////////////////////////// // addColumn //////////////////////////// test("addColumn - simple") { val a = StructField("a", IntegerType) val b = StructField("b", StringType) val schema = new StructType().add(a).add(b) val x = StructField("x", LongType) assert(SchemaUtils.addColumn(schema, x, Seq(0)) === new StructType().add(x).add(a).add(b)) assert(SchemaUtils.addColumn(schema, x, Seq(1)) === new StructType().add(a).add(x).add(b)) assert(SchemaUtils.addColumn(schema, x, Seq(2)) === new StructType().add(a).add(b).add(x)) expectFailure("Index -1", "lower than 0") { SchemaUtils.addColumn(schema, x, Seq(-1)) } expectFailure("Index 3", "larger than struct length: 2") { SchemaUtils.addColumn(schema, x, Seq(3)) } expectFailure("parent is not a structtype") { SchemaUtils.addColumn(schema, x, Seq(0, 0)) } } test("addColumn - nested struct") { val a = StructField("a", IntegerType) val b = StructField("b", StringType) val first = StructField("first", new StructType().add(a).add(b)) val middle = StructField("middle", new StructType().add(a).add(b)) val last = StructField("last", new StructType().add(a).add(b)) val schema = new StructType().add(first).add(middle).add(last) val x = StructField("x", LongType) assert(SchemaUtils.addColumn(schema, x, Seq(0)) === new StructType().add(x).add(first).add(middle).add(last)) assert(SchemaUtils.addColumn(schema, x, Seq(1)) === new StructType().add(first).add(x).add(middle).add(last)) assert(SchemaUtils.addColumn(schema, x, Seq(2)) === new StructType().add(first).add(middle).add(x).add(last)) assert(SchemaUtils.addColumn(schema, x, Seq(3)) === new StructType().add(first).add(middle).add(last).add(x)) assert(SchemaUtils.addColumn(schema, x, Seq(0, 2)) === new StructType().add("first", new StructType().add(a).add(b).add(x)).add(middle).add(last)) assert(SchemaUtils.addColumn(schema, x, Seq(0, 1)) === new StructType().add("first", new StructType().add(a).add(x).add(b)).add(middle).add(last)) assert(SchemaUtils.addColumn(schema, x, Seq(0, 0)) === new StructType().add("first", new StructType().add(x).add(a).add(b)).add(middle).add(last)) assert(SchemaUtils.addColumn(schema, x, Seq(1, 0)) === new StructType().add(first).add("middle", new StructType().add(x).add(a).add(b)).add(last)) assert(SchemaUtils.addColumn(schema, x, Seq(2, 0)) === new StructType().add(first).add(middle).add("last", new StructType().add(x).add(a).add(b))) expectFailure("Index -1", "lower than 0") { SchemaUtils.addColumn(schema, x, Seq(0, -1)) } expectFailure("Index 3", "larger than struct length: 2") { SchemaUtils.addColumn(schema, x, Seq(0, 3)) } expectFailure("Struct not found at position 2") { SchemaUtils.addColumn(schema, x, Seq(0, 2, 0)) } expectFailure("parent is not a structtype") { SchemaUtils.addColumn(schema, x, Seq(0, 0, 0)) } } test("addColumn - nested map") { val k = StructField("k", IntegerType) val v = StructField("v", StringType) val schema = new StructType().add("m", MapType( keyType = new StructType().add(k), valueType = new StructType().add(v))) val MAP_KEY_INDEX = 0 val MAP_VALUE_INDEX = 1 val x = StructField("x", LongType) assert(SchemaUtils.addColumn(schema, x, Seq(0, MAP_KEY_INDEX, 0)) === new StructType().add("m", MapType( keyType = new StructType().add(x).add(k), valueType = new StructType().add(v)))) assert(SchemaUtils.addColumn(schema, x, Seq(0, MAP_KEY_INDEX, 1)) === new StructType().add("m", MapType( keyType = new StructType().add(k).add(x), valueType = new StructType().add(v)))) assert(SchemaUtils.addColumn(schema, x, Seq(0, MAP_VALUE_INDEX, 0)) === new StructType().add("m", MapType( keyType = new StructType().add(k), valueType = new StructType().add(x).add(v)))) assert(SchemaUtils.addColumn(schema, x, Seq(0, MAP_VALUE_INDEX, 1)) === new StructType().add("m", MapType( keyType = new StructType().add(k), valueType = new StructType().add(v).add(x)))) // Adding to map key/value. expectFailure("parent is not a structtype") { SchemaUtils.addColumn(schema, x, Seq(0, MAP_KEY_INDEX)) } expectFailure("parent is not a structtype") { SchemaUtils.addColumn(schema, x, Seq(0, MAP_VALUE_INDEX)) } // Invalid map access. expectFailure("parent is not a structtype") { SchemaUtils.addColumn(schema, x, Seq(0, MAP_KEY_INDEX - 1, 0)) } expectFailure("parent is not a structtype") { SchemaUtils.addColumn(schema, x, Seq(0, MAP_VALUE_INDEX + 1, 0)) } } test("addColumn - nested maps") { // Helper method to create a 2-level deep nested map of structs. The tests below each cover // adding a field to one of the leaf struct. def schema( kk: StructType = new StructType().add("kk", IntegerType), kv: StructType = new StructType().add("kv", IntegerType), vk: StructType = new StructType().add("vk", IntegerType), vv: StructType = new StructType().add("vv", IntegerType)) : StructType = new StructType().add("m", MapType( keyType = MapType( keyType = kk, valueType = kv), valueType = MapType( keyType = vk, valueType = vv))) val MAP_KEY_INDEX = 0 val MAP_VALUE_INDEX = 1 val x = StructField("x", LongType) // Add field `x` at the front of each leaf struct. assert(SchemaUtils.addColumn(schema(), x, Seq(0, MAP_KEY_INDEX, MAP_KEY_INDEX, 0)) === schema(kk = new StructType().add(x).add("kk", IntegerType))) assert(SchemaUtils.addColumn(schema(), x, Seq(0, MAP_VALUE_INDEX, MAP_KEY_INDEX, 0)) === schema(vk = new StructType().add(x).add("vk", IntegerType))) assert(SchemaUtils.addColumn(schema(), x, Seq(0, MAP_KEY_INDEX, MAP_VALUE_INDEX, 0)) === schema(kv = new StructType().add(x).add("kv", IntegerType))) assert(SchemaUtils.addColumn(schema(), x, Seq(0, MAP_VALUE_INDEX, MAP_VALUE_INDEX, 0)) === schema(vv = new StructType().add(x).add("vv", IntegerType))) // Add field `x` at the back of each leaf struct. assert(SchemaUtils.addColumn(schema(), x, Seq(0, MAP_KEY_INDEX, MAP_KEY_INDEX, 1)) === schema(kk = new StructType().add("kk", IntegerType).add(x))) assert(SchemaUtils.addColumn(schema(), x, Seq(0, MAP_VALUE_INDEX, MAP_KEY_INDEX, 1)) === schema(vk = new StructType().add("vk", IntegerType).add(x))) assert(SchemaUtils.addColumn(schema(), x, Seq(0, MAP_KEY_INDEX, MAP_VALUE_INDEX, 1)) === schema(kv = new StructType().add("kv", IntegerType).add(x))) assert(SchemaUtils.addColumn(schema(), x, Seq(0, MAP_VALUE_INDEX, MAP_VALUE_INDEX, 1)) === schema(vv = new StructType().add("vv", IntegerType).add(x))) // Adding to map key/value. expectFailure("parent is not a structtype") { SchemaUtils.addColumn(schema(), x, Seq(0, MAP_KEY_INDEX, MAP_KEY_INDEX)) } expectFailure("parent is not a structtype") { SchemaUtils.addColumn(schema(), x, Seq(0, MAP_KEY_INDEX, MAP_VALUE_INDEX)) } // Invalid map access. expectFailure("parent is not a structtype") { SchemaUtils.addColumn(schema(), x, Seq(0, MAP_KEY_INDEX, MAP_KEY_INDEX - 1, 0)) } expectFailure("parent is not a structtype") { SchemaUtils.addColumn(schema(), x, Seq(0, MAP_KEY_INDEX - 1, MAP_KEY_INDEX, 0)) } expectFailure("parent is not a structtype") { SchemaUtils.addColumn(schema(), x, Seq(0, MAP_KEY_INDEX, MAP_VALUE_INDEX + 1, 0)) } expectFailure("parent is not a structtype") { SchemaUtils.addColumn(schema(), x, Seq(0, MAP_VALUE_INDEX + 1, MAP_KEY_INDEX, 0)) } } test("addColumn - nested array") { val e = StructField("e", IntegerType) val schema = new StructType().add("a", ArrayType(new StructType().add(e))) val x = StructField("x", LongType) val ARRAY_ELEMENT_INDEX = 0 // Add field `x` at the front of the leaf struct. assert(SchemaUtils.addColumn(schema, x, Seq(0, ARRAY_ELEMENT_INDEX, 0)) === new StructType().add("a", ArrayType(new StructType().add(x).add(e)))) // Add field `x` at the back of the leaf struct. assert(SchemaUtils.addColumn(schema, x, Seq(0, ARRAY_ELEMENT_INDEX, 1)) === new StructType().add("a", ArrayType(new StructType().add(e).add(x)))) // Adding to array element. expectFailure("parent is not a structtype") { SchemaUtils.addColumn(schema, x, Seq(0, ARRAY_ELEMENT_INDEX)) } // Invalid array access. expectFailure("Incorrectly accessing an ArrayType") { SchemaUtils.addColumn(schema, x, Seq(0, ARRAY_ELEMENT_INDEX - 1, 0)) } expectFailure("Incorrectly accessing an ArrayType") { SchemaUtils.addColumn(schema, x, Seq(0, ARRAY_ELEMENT_INDEX + 1, 0)) } } test("addColumn - nested arrays") { val e = StructField("e", IntegerType) val schema = new StructType().add("a", ArrayType(ArrayType(new StructType().add(e)))) val x = StructField("x", LongType) val ARRAY_ELEMENT_INDEX = 0 // Add field `x` at the front of the leaf struct. assert(SchemaUtils.addColumn(schema, x, Seq(0, ARRAY_ELEMENT_INDEX, ARRAY_ELEMENT_INDEX, 0)) === new StructType().add("a", ArrayType(ArrayType(new StructType().add(x).add(e))))) // Add field `x` at the back of the leaf struct. assert(SchemaUtils.addColumn(schema, x, Seq(0, ARRAY_ELEMENT_INDEX, ARRAY_ELEMENT_INDEX, 1)) === new StructType().add("a", ArrayType(ArrayType(new StructType().add(e).add(x))))) // Adding to array element. expectFailure("parent is not a structtype") { SchemaUtils.addColumn(schema, x, Seq(0, ARRAY_ELEMENT_INDEX, ARRAY_ELEMENT_INDEX)) } // Invalid array access. expectFailure("Incorrectly accessing an ArrayType") { SchemaUtils.addColumn(schema, x, Seq(0, ARRAY_ELEMENT_INDEX, ARRAY_ELEMENT_INDEX - 1, 0)) } expectFailure("Incorrectly accessing an ArrayType") { SchemaUtils.addColumn(schema, x, Seq(0, ARRAY_ELEMENT_INDEX - 1, ARRAY_ELEMENT_INDEX, 0)) } expectFailure("Incorrectly accessing an ArrayType") { SchemaUtils.addColumn(schema, x, Seq(0, ARRAY_ELEMENT_INDEX, ARRAY_ELEMENT_INDEX + 1, 0)) } expectFailure("Incorrectly accessing an ArrayType") { SchemaUtils.addColumn(schema, x, Seq(0, ARRAY_ELEMENT_INDEX + 1, ARRAY_ELEMENT_INDEX, 0)) } } test("addColumn - top level array") { val a = StructField("a", IntegerType) val b = StructField("b", StringType) val schema = ArrayType(new StructType().add(a).add(b)) val x = StructField("x", LongType) assert(SchemaUtils.addColumn(schema, x, Seq(0, 1)) === ArrayType(new StructType().add(a).add(x).add(b))) } test("addColumn - top level map") { val k = StructField("k", IntegerType) val v = StructField("v", StringType) val schema = MapType( keyType = new StructType().add(k), valueType = new StructType().add(v)) val x = StructField("x", LongType) assert(SchemaUtils.addColumn(schema, x, Seq(0, 1)) === MapType( keyType = new StructType().add(k).add(x), valueType = new StructType().add(v))) assert(SchemaUtils.addColumn(schema, x, Seq(1, 1)) === MapType( keyType = new StructType().add(k), valueType = new StructType().add(v).add(x))) } //////////////////////////// // dropColumn //////////////////////////// test("dropColumn - simple") { val a = StructField("a", IntegerType) val b = StructField("b", StringType) val schema = new StructType().add(a).add(b) assert(SchemaUtils.dropColumn(schema, Seq(0)) === ((new StructType().add(b), a))) assert(SchemaUtils.dropColumn(schema, Seq(1)) === ((new StructType().add(a), b))) expectFailure("Index -1", "lower than 0") { SchemaUtils.dropColumn(schema, Seq(-1)) } expectFailure("Index 2", "equals to or is larger than struct length: 2") { SchemaUtils.dropColumn(schema, Seq(2)) } expectFailure("Can only drop nested columns from StructType") { SchemaUtils.dropColumn(schema, Seq(0, 0)) } } test("dropColumn - nested struct") { val a = StructField("a", IntegerType) val b = StructField("b", StringType) val c = StructField("c", StringType) val first = StructField("first", new StructType().add(a).add(b).add(c)) val middle = StructField("middle", new StructType().add(a).add(b).add(c)) val last = StructField("last", new StructType().add(a).add(b).add(c)) val schema = new StructType().add(first).add(middle).add(last) assert(SchemaUtils.dropColumn(schema, Seq(0)) === new StructType().add(middle).add(last) -> first) assert(SchemaUtils.dropColumn(schema, Seq(1)) === new StructType().add(first).add(last) -> middle) assert(SchemaUtils.dropColumn(schema, Seq(2)) === new StructType().add(first).add(middle) -> last) assert(SchemaUtils.dropColumn(schema, Seq(0, 2)) === new StructType().add("first", new StructType().add(a).add(b)).add(middle).add(last) -> c) assert(SchemaUtils.dropColumn(schema, Seq(0, 1)) === new StructType().add("first", new StructType().add(a).add(c)).add(middle).add(last) -> b) assert(SchemaUtils.dropColumn(schema, Seq(0, 0)) === new StructType().add("first", new StructType().add(b).add(c)).add(middle).add(last) -> a) assert(SchemaUtils.dropColumn(schema, Seq(1, 0)) === new StructType().add(first).add("middle", new StructType().add(b).add(c)).add(last) -> a) assert(SchemaUtils.dropColumn(schema, Seq(2, 0)) === new StructType().add(first).add(middle).add("last", new StructType().add(b).add(c)) -> a) expectFailure("Index -1", "lower than 0") { SchemaUtils.dropColumn(schema, Seq(0, -1)) } expectFailure("Index 3", "equals to or is larger than struct length: 3") { SchemaUtils.dropColumn(schema, Seq(0, 3)) } expectFailure("Can only drop nested columns from StructType") { SchemaUtils.dropColumn(schema, Seq(0, 0, 0)) } } test("dropColumn - nested map") { val a = StructField("a", IntegerType) val b = StructField("b", StringType) val c = StructField("c", LongType) val d = StructField("d", DateType) val schema = new StructType().add("m", MapType( keyType = new StructType().add(a).add(b), valueType = new StructType().add(c).add(d))) val MAP_KEY_INDEX = 0 val MAP_VALUE_INDEX = 1 assert(SchemaUtils.dropColumn(schema, Seq(0, MAP_KEY_INDEX, 0)) === (new StructType().add("m", MapType( keyType = new StructType().add(b), valueType = new StructType().add(c).add(d))), a)) assert(SchemaUtils.dropColumn(schema, Seq(0, MAP_KEY_INDEX, 1)) === (new StructType().add("m", MapType( keyType = new StructType().add(a), valueType = new StructType().add(c).add(d))), b)) assert(SchemaUtils.dropColumn(schema, Seq(0, MAP_VALUE_INDEX, 0)) === (new StructType().add("m", MapType( keyType = new StructType().add(a).add(b), valueType = new StructType().add(d))), c)) assert(SchemaUtils.dropColumn(schema, Seq(0, MAP_VALUE_INDEX, 1)) === (new StructType().add("m", MapType( keyType = new StructType().add(a).add(b), valueType = new StructType().add(c))), d)) // Dropping map key/value. expectFailure("can only drop nested columns from structtype") { SchemaUtils.dropColumn(schema, Seq(0, MAP_KEY_INDEX)) } expectFailure("can only drop nested columns from structtype") { SchemaUtils.dropColumn(schema, Seq(0, MAP_VALUE_INDEX)) } // Invalid map access. expectFailure("can only drop nested columns from structtype") { SchemaUtils.dropColumn(schema, Seq(0, MAP_KEY_INDEX - 1, 0)) } expectFailure("can only drop nested columns from structtype") { SchemaUtils.dropColumn(schema, Seq(0, MAP_VALUE_INDEX + 1, 0)) } } test("dropColumn - nested maps") { // Helper method to create a 2-level deep nested map of structs. The tests below each cover // dropping a field to one of the leaf struct. Each test adds an extra field `a` at a specific // position then drops it to end up with the default schema returned by `schema()` def schema( kk: StructType = new StructType().add("kk", IntegerType), kv: StructType = new StructType().add("kv", IntegerType), vk: StructType = new StructType().add("vk", IntegerType), vv: StructType = new StructType().add("vv", IntegerType)) : StructType = new StructType().add("m", MapType( keyType = MapType( keyType = kk, valueType = kv), valueType = MapType( keyType = vk, valueType = vv))) val a = StructField("a", LongType) val MAP_KEY_INDEX = 0 val MAP_VALUE_INDEX = 1 def checkDrop(initialSchema: StructType, position: Seq[Int]): Unit = assert(SchemaUtils.dropColumn(initialSchema, position) === (schema(), a)) // Drop field `a` from the front of each leaf struct. checkDrop( initialSchema = schema(kk = new StructType().add(a).add("kk", IntegerType)), position = Seq(0, MAP_KEY_INDEX, MAP_KEY_INDEX, 0)) checkDrop( initialSchema = schema(kv = new StructType().add(a).add("kv", IntegerType)), position = Seq(0, MAP_KEY_INDEX, MAP_VALUE_INDEX, 0)) checkDrop( initialSchema = schema(vk = new StructType().add(a).add("vk", IntegerType)), position = Seq(0, MAP_VALUE_INDEX, MAP_KEY_INDEX, 0)) checkDrop( initialSchema = schema(vv = new StructType().add(a).add("vv", IntegerType)), position = Seq(0, MAP_VALUE_INDEX, MAP_VALUE_INDEX, 0)) // Drop field `a` from the back of each leaf struct. checkDrop( initialSchema = schema(kk = new StructType().add("kk", IntegerType).add(a)), position = Seq(0, MAP_KEY_INDEX, MAP_KEY_INDEX, 1)) checkDrop( initialSchema = schema(kv = new StructType().add("kv", IntegerType).add(a)), position = Seq(0, MAP_KEY_INDEX, MAP_VALUE_INDEX, 1)) checkDrop( initialSchema = schema(vk = new StructType().add("vk", IntegerType).add(a)), position = Seq(0, MAP_VALUE_INDEX, MAP_KEY_INDEX, 1)) checkDrop( initialSchema = schema(vv = new StructType().add("vv", IntegerType).add(a)), position = Seq(0, MAP_VALUE_INDEX, MAP_VALUE_INDEX, 1)) // Dropping map key/value. expectFailure("can only drop nested columns from structtype") { SchemaUtils.dropColumn(schema(), Seq(0, MAP_KEY_INDEX, MAP_KEY_INDEX)) } expectFailure("can only drop nested columns from structtype") { SchemaUtils.dropColumn(schema(), Seq(0, MAP_KEY_INDEX, MAP_VALUE_INDEX)) } // Invalid map access. expectFailure("can only drop nested columns from structtype") { SchemaUtils.dropColumn(schema(), Seq(0, MAP_KEY_INDEX, MAP_KEY_INDEX - 1, 0)) } expectFailure("can only drop nested columns from structtype") { SchemaUtils.dropColumn(schema(), Seq(0, MAP_KEY_INDEX - 1, MAP_KEY_INDEX, 0)) } expectFailure("can only drop nested columns from structtype") { SchemaUtils.dropColumn(schema(), Seq(0, MAP_KEY_INDEX, MAP_VALUE_INDEX + 1, 0)) } expectFailure("can only drop nested columns from structtype") { SchemaUtils.dropColumn(schema(), Seq(0, MAP_VALUE_INDEX + 1, MAP_KEY_INDEX, 0)) } } test("dropColumn - nested array") { val e = StructField("e", IntegerType) val f = StructField("f", IntegerType) val schema = new StructType().add("a", ArrayType(new StructType().add(e).add(f))) val ARRAY_ELEMENT_INDEX = 0 // Drop field from the front of the leaf struct. assert(SchemaUtils.dropColumn(schema, Seq(0, ARRAY_ELEMENT_INDEX, 0)) === (new StructType().add("a", ArrayType(new StructType().add(f))), e)) // Drop field from the back of the leaf struct. assert(SchemaUtils.dropColumn(schema, Seq(0, ARRAY_ELEMENT_INDEX, 1)) === (new StructType().add("a", ArrayType(new StructType().add(e))), f)) // Dropping array element. expectFailure("can only drop nested columns from structtype") { SchemaUtils.dropColumn(schema, Seq(0, ARRAY_ELEMENT_INDEX)) } // Invalid array access. expectFailure("Incorrectly accessing an ArrayType") { SchemaUtils.dropColumn(schema, Seq(0, ARRAY_ELEMENT_INDEX - 1, 0)) } expectFailure("Incorrectly accessing an ArrayType") { SchemaUtils.dropColumn(schema, Seq(0, ARRAY_ELEMENT_INDEX + 1, 0)) } } test("dropColumn - nested arrays") { val e = StructField("e", IntegerType) val f = StructField("f", IntegerType) val schema = new StructType().add("a", ArrayType(ArrayType(new StructType().add(e).add(f)))) val ARRAY_ELEMENT_INDEX = 0 // Drop field `x` from the front of the leaf struct. assert(SchemaUtils.dropColumn(schema, Seq(0, ARRAY_ELEMENT_INDEX, ARRAY_ELEMENT_INDEX, 0)) === (new StructType().add("a", ArrayType(ArrayType(new StructType().add(f)))), e)) // Drop field `x` from the back of the leaf struct. assert(SchemaUtils.dropColumn(schema, Seq(0, ARRAY_ELEMENT_INDEX, ARRAY_ELEMENT_INDEX, 1)) === (new StructType().add("a", ArrayType(ArrayType(new StructType().add(e)))), f)) // Dropping array element. expectFailure("can only drop nested columns from structtype") { SchemaUtils.dropColumn(schema, Seq(0, ARRAY_ELEMENT_INDEX, ARRAY_ELEMENT_INDEX)) } // Invalid array access. expectFailure("Incorrectly accessing an ArrayType") { SchemaUtils.dropColumn(schema, Seq(0, ARRAY_ELEMENT_INDEX, ARRAY_ELEMENT_INDEX - 1, 0)) } expectFailure("Incorrectly accessing an ArrayType") { SchemaUtils.dropColumn(schema, Seq(0, ARRAY_ELEMENT_INDEX - 1, ARRAY_ELEMENT_INDEX, 0)) } expectFailure("Incorrectly accessing an ArrayType") { SchemaUtils.dropColumn(schema, Seq(0, ARRAY_ELEMENT_INDEX, ARRAY_ELEMENT_INDEX + 1, 0)) } expectFailure("Incorrectly accessing an ArrayType") { SchemaUtils.dropColumn(schema, Seq(0, ARRAY_ELEMENT_INDEX + 1, ARRAY_ELEMENT_INDEX, 0)) } } test("dropColumn - top level array") { val schema = ArrayType(new StructType().add("a", IntegerType).add("b", StringType)) assert(SchemaUtils.dropColumn(schema, Seq(0, 0))._1 === ArrayType(new StructType().add("b", StringType))) } test("dropColumn - top level map") { val schema = MapType( keyType = new StructType().add("k", IntegerType).add("k2", StringType), valueType = new StructType().add("v", StringType).add("v2", StringType)) assert(SchemaUtils.dropColumn(schema, Seq(0, 0))._1 === MapType( keyType = new StructType().add("k2", StringType), valueType = new StructType().add("v", StringType).add("v2", StringType))) assert(SchemaUtils.dropColumn(schema, Seq(1, 0))._1 === MapType( keyType = new StructType().add("k", IntegerType).add("k2", StringType), valueType = new StructType().add("v2", StringType))) } ///////////////////////////////// // normalizeColumnNamesInDataType ///////////////////////////////// private def runNormalizeColumnNamesInDataType( sourceDataType: DataType, tableDataType: DataType): DataType = { normalizeColumnNamesInDataType( deltaLog = null, sourceDataType, tableDataType, sourceParentFields = Seq.empty, tableSchema = new StructType()) } test("normalize column names in data type - top-level atomic types") { val source = new StructType() .add("a", IntegerType) .add("b", StringType) .add("c", LongType) .add("d", DateType) val table = new StructType() .add("B", StringType) .add("A", LongType) // LongType != IntegerType .add("D", DecimalType(10, 0)) // DecimalType != DateType .add("C", StringType) // StringType != LongType val expected = new StructType() .add("A", IntegerType) .add("B", StringType) .add("C", LongType) .add("D", DateType) assert(runNormalizeColumnNamesInDataType(source, table) == expected) } test("normalize column names in data type - incompatible top-level types") { val schema1a = new StructType() .add("a", IntegerType) .add("b", StringType) val schema1b = new StructType() .add("B", StringType) .add("A", new StructType()) // StructType != IntegerType intercept[AssertionError] { runNormalizeColumnNamesInDataType(schema1a, schema1b) } intercept[AssertionError] { runNormalizeColumnNamesInDataType(schema1b, schema1a) } val schema2a = new StructType() .add("x", StringType) .add("y", new StructType() .add("z", IntegerType) ) val schema2b = new StructType() .add("x", StringType) .add("Y", new StructType() .add("z", ArrayType(IntegerType)) // ArrayType != IntegerType ) intercept[AssertionError] { runNormalizeColumnNamesInDataType(schema2a, schema2b) } intercept[AssertionError] { runNormalizeColumnNamesInDataType(schema2b, schema2a) } } test("normalize column names in data type - nested structs") { val source = new StructType() .add("a1", IntegerType) .add("a2", new StructType() .add("b1", IntegerType) .add("b2", new StructType() .add("c1", IntegerType) .add("c2", LongType) ) .add("b3", LongType) ) .add("a3", new StructType() .add("d1", IntegerType) .add("d2", LongType) ) val table = new StructType() .add("A3", new StructType() .add("D2", LongType) .add("D3x", StringType) .add("D1", IntegerType) ) .add("A2", new StructType() .add("B3", LongType) .add("B4x", IntegerType) .add("B1", IntegerType) .add("B2", new StructType() .add("C3", LongType) .add("C2", LongType) .add("C1", IntegerType) ) ) .add("A4x", StringType) .add("A1", IntegerType) val expected = new StructType() .add("A1", IntegerType) .add("A2", new StructType() .add("B1", IntegerType) .add("B2", new StructType() .add("C1", IntegerType) .add("C2", LongType)) .add("B3", LongType) ) .add("A3", new StructType() .add("D1", IntegerType) .add("D2", LongType) ) assert(runNormalizeColumnNamesInDataType(source, table) == expected) } test("normalize column names in data type - different atomic types in a map") { val source = new StructType() .add("a", new StructType() .add("b", new StructType() .add("c", MapType(StringType, IntegerType)))) val table = new StructType() .add("A", new StructType() .add("B", new StructType() .add("C", MapType(IntegerType, StringType)))) val expected = new StructType() .add("A", new StructType() .add("B", new StructType() .add("C", MapType(StringType, IntegerType)))) assert(runNormalizeColumnNamesInDataType(source, table) == expected) } test("normalize column names in data type - incompatible nested types") { val schema1 = new StructType() .add("a", new StructType() .add("b", new StructType() .add("c", IntegerType))) val schema2 = new StructType() .add("A", new StructType() .add("B", new StructType() .add("C", ArrayType(IntegerType)))) val schema3 = new StructType() .add("A", new StructType() .add("b", new StructType() .add("C", new StructType()))) val schemas = Seq(schema1, schema2, schema3) for (left <- schemas; right <- schemas) { if (left == right) { // Make sure there's no error when the schemas are the same. assert(runNormalizeColumnNamesInDataType(left, right) == left) } else { intercept[AssertionError] { runNormalizeColumnNamesInDataType(left, right) } } } } test("normalize column names in data type - arrays, maps, structs") { val source = MapType( new StructType() .add("aa", IntegerType) .add("bb", StringType), ArrayType(new StructType() .add("aa", IntegerType) .add("bb", StringType))) val table = MapType( new StructType() .add("aA", IntegerType) .add("bB", StringType), ArrayType(new StructType() .add("Cc", IntegerType) .add("Aa", IntegerType) .add("Bb", StringType))) val expected = MapType( new StructType() .add("aA", IntegerType) .add("bB", StringType), ArrayType(new StructType() .add("Aa", IntegerType) .add("Bb", StringType))) assert(runNormalizeColumnNamesInDataType(source, table) == expected) } test("normalize column names in data type - missing column") { val source = ArrayType( new StructType() .add("aa", IntegerType) .add("bb", StringType) ) val target = ArrayType( new StructType() .add("AA", IntegerType) .add("CC", StringType) // "bb" != "CC" ) val exception = intercept[DeltaAnalysisException] { normalizeColumnNamesInDataType(deltaLog = null, source, target, Seq("x", "Y"), new StructType()) } checkError( exception, "DELTA_CANNOT_RESOLVE_COLUMN", sqlState = "42703", parameters = Map("columnName" -> "x.Y.bb", "schema" -> "root\n") ) } test("normalize column names in data type - preserve nullability and comments") { val source = new StructType() .add("a1", IntegerType, nullable = true) .add("a2", new StructType() .add("b1", IntegerType, nullable = false) .add("b2", ArrayType(IntegerType, containsNull = true), nullable = true, comment = "comment for b2") .add("b3", MapType(IntegerType, StringType, valueContainsNull = false), nullable = true, comment = "comment for b3"), nullable = false, comment = "comment for a2" ) val table = new StructType() .add("A1", IntegerType, nullable = false, "comment for A1") .add("A2", new StructType() .add("B1", IntegerType, nullable = true) .add("B2", ArrayType(IntegerType, containsNull = false), nullable = false, comment = "comment for B2") .add("B3", MapType(IntegerType, StringType, valueContainsNull = true), nullable = false, comment = "comment for B3"), nullable = true ) val expected = new StructType() .add("A1", IntegerType, nullable = true) .add("A2", new StructType() .add("B1", IntegerType, nullable = false) .add("B2", ArrayType(IntegerType, containsNull = true), nullable = true, comment = "comment for b2") .add("B3", MapType(IntegerType, StringType, valueContainsNull = false), nullable = true, comment = "comment for b3"), nullable = false, comment = "comment for a2" ) assert(runNormalizeColumnNamesInDataType(source, table) == expected) } test("normalize column names in data type - empty source struct") { val source = new StructType() val table = new StructType().add("a", IntegerType) val expected = new StructType() assert(runNormalizeColumnNamesInDataType(source, table) == expected) } //////////////////////////// // normalizeColumnNames //////////////////////////// /** * SchemaUtils.normalizeColumnNames() introduces a Project operator where for each of the * top-level columns: * - If a top-level field name differs from the table schema, we correct it using an Alias. * - If a nested field name differs from the table schema, we correct it using a Cast. * This function verifies that the Casts are only introduced for the correct subset of top-level * columns. */ private def verifyColumnsWithCasts(df: DataFrame, columnsWithCasts: Seq[String]): Unit = { @tailrec def isCast(expression: Expression): Boolean = expression match { case _: Cast => true case Alias(child, _) => isCast(child) case _ => false } val plan = df.queryExecution.analyzed val projections = plan.asInstanceOf[Project].projectList for (projection <- projections) { val expectedIsCast = columnsWithCasts.contains(projection.name) val actualIsCast = isCast(projection) assert(expectedIsCast === actualIsCast, s"Verifying cast for ${projection.name}") } } test("normalize column names - different top-level ordering") { val df = Seq((1, 2, 3)).toDF("def", "gHi", "abC") val tableSchema = new StructType() .add("abc", IntegerType) .add("Def", IntegerType) .add("ghi", IntegerType) // Add an extra column to the table schema to make sure it is not added, and does not cause // an error. .add("jkl", StringType) val expectedSchema = new StructType() .add("Def", IntegerType, false) .add("ghi", IntegerType, false) .add("abc", IntegerType, false) val normalized = normalizeColumnNames( deltaLog = null, tableSchema, df ) verifyColumnsWithCasts(normalized, Seq.empty) assert(normalized.schema == expectedSchema) } test("normalize column names - dots in the name") { val df = spark.read.json(Seq("""{"a.b":1,"c.d":{"x.y":2, "y.z":1}}""").toDS()) val tableSchema = new StructType() .add("c.D", new StructType() .add("y.z", LongType) .add("x.Y", LongType) ) .add("a.B", LongType) val expectedSchema = new StructType() .add("a.B", LongType, nullable = true) .add("c.D", new StructType() .add("x.Y", LongType, nullable = true) .add("y.z", LongType, nullable = true), nullable = true ) val normalized = normalizeColumnNames( deltaLog = null, tableSchema, df ) verifyColumnsWithCasts(normalized, Seq("c.D")) assert(normalized.schema === expectedSchema) } test("normalize column names - different case in struct") { // JSON schema inference does not preserve the order of columns, so we need an explicit schema. val jsonSchema = new StructType() .add("b", new StructType() .add("x", LongType) .add("y", new StructType() .add("T", LongType) .add("s", LongType) ) ) .add("a", LongType) val df = spark.read.schema(jsonSchema) .json(Seq("""{"b":{"x":1,"y":{"T":2, "s":1}}, "a":1}""").toDS()) val tableSchema = new StructType() .add("a", LongType) .add("b", new StructType() .add("x", LongType) .add("y", new StructType() .add("s", LongType) .add("t", LongType) ) ) val expectedSchema = new StructType() .add("b", new StructType() .add("x", LongType) .add("y", new StructType() .add("t", LongType) .add("s", LongType) ) ) .add("a", LongType) val normalized = normalizeColumnNames( deltaLog = null, tableSchema, df ) verifyColumnsWithCasts(normalized, Seq("b")) assert(normalized.schema === expectedSchema) } test("normalize column names - different case in array") { val df = spark.read.json(Seq("""{"X":1,"y":[{"Z": "alpha"},{"Z":"beta"}]}""").toDS()) val tableSchema = new StructType() .add("x", LongType) .add("y", ArrayType(new StructType().add("z", StringType))) val normalized = normalizeColumnNames( deltaLog = null, tableSchema, df ) verifyColumnsWithCasts(normalized, Seq("y")) assert(normalized.schema == tableSchema) } test("normalize column names - different case in map") { val sourceMapType = MapType(StringType, new StructType().add("Z", StringType)) val df = spark.range(1).toDF("X") .withColumn("y", lit(null).cast(sourceMapType)) .select(col("y"), col("X")) val tableSchema = new StructType() .add("x", LongType) .add("y", MapType(StringType, new StructType() .add("z", StringType) // Add an extra nested column to the table schema to make sure it is not added. .add("v", IntegerType))) val expectedSchema = new StructType() .add("y", MapType(StringType, new StructType().add("z", StringType))) .add("x", LongType, nullable = false) val normalized = normalizeColumnNames( deltaLog = null, tableSchema, df ) verifyColumnsWithCasts(normalized, Seq("y")) assert(normalized.schema === expectedSchema) } test("normalize column names - maintain nested column order") { val sourceStructColumnNames = Seq("the", "quick", "brown", "fox", "jumps", "over", "them", "lazy", "dog") val sourceStructType = new StructType( sourceStructColumnNames.map(StructField(_, IntegerType)).toArray) // Nested columns in the table are all name in upper case, and listed in reverse order. val tableStructColumnNames = Seq("LOOK") ++ sourceStructColumnNames.reverse.map(_.toUpperCase(Locale.ROOT)) val tableSchema = new StructType() .add("s", new StructType( tableStructColumnNames.map(StructField(_, IntegerType)).toArray)) // We expect the columns to maintain the same order as the source. val expectedStructColumnNames = sourceStructColumnNames.map(_.toUpperCase(Locale.ROOT)) val expectedSchema = new StructType() .add("s", new StructType( expectedStructColumnNames.map(StructField(_, IntegerType)).toArray)) val df = spark.range(1).toDF("id") .select(lit(null).cast(sourceStructType).as("s")) val normalized = normalizeColumnNames( deltaLog = null, tableSchema, df ) verifyColumnsWithCasts(normalized, Seq("s")) assert(normalized.schema === expectedSchema) } test("normalize column names - only top-level names of complex columns differ") { val structType = new StructType() .add("a", IntegerType) .add("b", IntegerType) val mapType = MapType(structType, structType) val arrayType = ArrayType(structType) val df = spark.range(1).toDF("id") .select(lit(null).cast(structType).as("x"), lit(null).cast(mapType).as("y"), lit(null).cast(arrayType).as("z")) val tableSchema = new StructType() .add("X", structType) .add("Y", mapType) .add("Z", arrayType) val normalized = normalizeColumnNames( deltaLog = null, tableSchema, df ) // If only top-level names differ, there is no need to cast complex types. verifyColumnsWithCasts(normalized, Seq.empty) assert(normalized.schema === tableSchema) } test("normalize column names - unmatched top-level column") { val df = spark.range(1).toDF("id") .select(lit(1L).as("one"), lit(2L).as("two")) val tableSchema = new StructType() .add("ONE", LongType) .add("THREE", LongType) val exception = intercept[DeltaAnalysisException] { normalizeColumnNames( deltaLog = null, tableSchema, df ) } checkError( exception, "DELTA_CANNOT_RESOLVE_COLUMN", sqlState = "42703", parameters = Map("columnName" -> "two", "schema" -> tableSchema.treeString) ) } test("normalize column names - unmatched nested column") { val sourceStructType = new StructType() .add("one", LongType) .add("two", LongType) val df = spark.range(1).toDF("id") .select(lit(null).cast(sourceStructType).as("s")) val tableSchema = new StructType() .add("S", new StructType() .add("ONE", LongType) .add("THREE", LongType) ) val exception = intercept[DeltaAnalysisException] { normalizeColumnNames( deltaLog = null, tableSchema, df ) } checkError( exception, "DELTA_CANNOT_RESOLVE_COLUMN", sqlState = "42703", parameters = Map("columnName" -> "s.two", "schema" -> tableSchema.treeString) ) } test("normalize column names - deeply nested schema") { // The only difference is the case of the most deeply nested column. val structTypes = Seq("z", "Z").map { finalColumnName => new StructType() .add("a", IntegerType) .add("b", MapType(StringType, new StructType() .add("c", IntegerType) .add("d", new StructType() .add("e", IntegerType) .add("f", IntegerType) .add("g", ArrayType(new StructType() .add("h", IntegerType) .add("i", IntegerType) .add("j", new StructType() .add("k", MapType(StringType, new StructType() .add("l", ArrayType(new StructType() .add("m", IntegerType) .add(finalColumnName, IntegerType) )))))))))) }.toArray val sourceStructType = structTypes(0) val df = spark.range(1).toDF("id") .select(lit(null).cast(sourceStructType).as("s")) val tableStructType = structTypes(1) val tableSchema = new StructType() .add("s", tableStructType) val normalized = normalizeColumnNames( deltaLog = null, tableSchema, df ) verifyColumnsWithCasts(normalized, Seq("s")) assert(normalized.schema === tableSchema) } test("normalize column names - can normalize row id column") { withTable("src") { spark.range(3).toDF("id").write .format("delta") .mode("overwrite") .option("delta.enableRowTracking", "true") .saveAsTable("src") val df = spark.read.format("delta").table("src") .select( col("*"), col("_metadata.row_id").as("row_id") ) .withMetadata( "row_id", RowId.RowIdMetadataStructField.metadata("name", shouldSetIcebergReservedFieldId = false) ) val tableSchema = new StructType().add("id", LongType) val normalized = normalizeColumnNames(deltaLog = null, tableSchema, df) assert(normalized.schema.fieldNames === Seq("id", "row_id")) } } test("normalize column names - can normalize both row id and commit version columns") { withTable("src") { spark.range(3).toDF("id").write .format("delta") .mode("overwrite") .option("delta.enableRowTracking", "true") .saveAsTable("src") val df = spark.read.format("delta").table("src") .select( col("*"), col("_metadata.row_id").as("row_id"), col("_metadata.row_commit_version").as("row_commit_version") ) .withMetadata( "row_id", RowId.RowIdMetadataStructField.metadata("name", shouldSetIcebergReservedFieldId = false)) .withMetadata( "row_commit_version", RowCommitVersion.MetadataStructField.metadata( "name", shouldSetIcebergReservedFieldId = false) ) val tableSchema = new StructType().add("id", LongType) val normalized = normalizeColumnNames(deltaLog = null, tableSchema, df) assert(normalized.schema.fieldNames === Seq("id", "row_id", "row_commit_version")) } } test("normalize column names - can normalize CDC type column") { val df = Seq((1, 2, 3, 4)).toDF("Abc", "def", "gHi", CDCReader.CDC_TYPE_COLUMN_NAME) val tableSchema = new StructType() .add("abc", IntegerType) .add("Def", IntegerType) .add("ghi", IntegerType) val normalized = normalizeColumnNames( deltaLog = null, tableSchema, df ) verifyColumnsWithCasts(normalized, Seq.empty) assert(normalized.schema.fieldNames === tableSchema.fieldNames :+ CDCReader.CDC_TYPE_COLUMN_NAME) } private def checkLatestStatsForOneRowFile( tableName: String, expectedStats: Map[String, Option[Any]]): Unit = { val snapshot = DeltaLog.forTable(spark, TableIdentifier(tableName)).update() val fileStats = snapshot.allFiles .orderBy(desc("modificationTime")) .limit(1) .withColumn("stats", from_json(col("stats"), snapshot.statsSchema)) .select("stats.*") val assertions = Seq(assert_true(col("numRecords") === lit(1L))) ++ expectedStats.flatMap { case (columnName, columnValue) => columnValue match { case Some(value) => Seq( assert_true(col("minValues." + columnName) === lit(value)), assert_true(col("maxValues." + columnName) === lit(value)), assert_true(col("nullCount." + columnName) === lit(0L))) case None => Seq( assert_true(col("minValues." + columnName).isNull), assert_true(col("maxValues." + columnName).isNull), assert_true(col("nullCount." + columnName) === lit(1L))) } } fileStats.select(assertions: _*).collect() } for (caseSensitive <- DeltaTestUtils.BOOLEAN_DOMAIN) { test(s"normalize column names - e2e nested struct (caseSensitive=$caseSensitive)") { withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) { val sourceData = Seq((105L, "foo", 205L, "bar", Struct1("James", 11, "Smith", 3000, "Correct"))) val sourceDf = sourceData.toDF("long1", "str1", "long2", "str2", "struct1") val sourceSchema = new StructType() .add("long1", LongType, nullable = false) .add("str1", StringType, nullable = true) .add("long2", LongType, nullable = false) .add("str2", StringType, nullable = true) .add("struct1", new StructType() .add("firstname", StringType, nullable = true) .add("numberone", LongType, nullable = false) .add("lastname", StringType, nullable = true) .add("numbertwo", LongType, nullable = false) .add("CorrectCase", StringType, nullable = true), nullable = true ) assert(sourceDf.schema === sourceSchema) val createTableCommand = """ CREATE TABLE t ( | Long2 LONG, Str2 STRING, Long1 LONG, Str1 STRING, Int1 INT, | Struct1 STRUCT | ) USING delta |""".stripMargin withTable("t") { sql(createTableCommand) sourceDf.write.format("delta").mode("append").saveAsTable("t") // Make sure all the values were inserted into the right columns, and columns missing in // the source were set to null. spark.table("t") .select( assert_true(col("Long2") === 205L), assert_true(col("Str2") === "bar"), assert_true(col("Long1") === 105L), assert_true(col("Str1") === "foo"), assert_true(col("Int1").isNull), assert_true(col("Struct1.LastName") === "Smith"), assert_true(col("Struct1.NumberTwo") === 3000L), assert_true(col("Struct1.FirstName") === "James"), assert_true(col("Struct1.NumberOne") === 11L), assert_true(col("Struct1.MissingNested").isNull), assert_true(col("Struct1.CorrectCase") === "Correct") ).collect() // Make sure each of the columns stats was computed correctly. checkLatestStatsForOneRowFile("t", Map( "Long2" -> Some(205L), "Str2" -> Some("bar"), "Long1" -> Some(105L), "Str1" -> Some("foo"), "Int1" -> None, "Struct1.LastName" -> Some("Smith"), "Struct1.NumberTwo" -> Some(3000L), "Struct1.FirstName" -> Some("James"), "Struct1.NumberOne" -> Some(11L), "Struct1.MissingNested" -> None, "Struct1.CorrectCase" -> Some("Correct") )) } } } } //////////////////////////// // mergeSchemas //////////////////////////// test("mergeSchemas: missing columns in df") { val base = new StructType().add("a", IntegerType).add("b", IntegerType) val write = new StructType().add("a", IntegerType) assert(mergeSchemas(base, write) === base) } test("mergeSchemas: missing columns in df - case sensitivity") { val base = new StructType().add("a", IntegerType).add("b", IntegerType) val write = new StructType().add("A", IntegerType) assert(mergeSchemas(base, write) === base) } test("new columns get added to the tail of the schema") { val base = new StructType().add("a", IntegerType) val write = new StructType().add("a", IntegerType).add("b", IntegerType) val write2 = new StructType().add("b", IntegerType).add("a", IntegerType) assert(mergeSchemas(base, write) === write) assert(mergeSchemas(base, write2) === write) } test("new columns get added to the tail of the schema - nested") { val base = new StructType() .add("regular", StringType) .add("struct", new StructType() .add("a", IntegerType)) val write = new StructType() .add("other", StringType) .add("struct", new StructType() .add("b", DateType) .add("a", IntegerType)) .add("this", StringType) val expected = new StructType() .add("regular", StringType) .add("struct", new StructType() .add("a", IntegerType) .add("b", DateType)) .add("other", StringType) .add("this", StringType) assert(mergeSchemas(base, write) === expected) } test("schema merging of incompatible types") { val base = new StructType() .add("top", StringType) .add("struct", new StructType() .add("a", IntegerType)) .add("array", ArrayType(new StructType() .add("b", DecimalType(18, 10)))) .add("map", MapType(StringType, StringType)) expectAnalysisErrorClass("DELTA_MERGE_INCOMPATIBLE_DATATYPE", Map("currentDataType" -> "StringType", "updateDataType" -> "IntegerType")) { mergeSchemas(base, new StructType().add("top", IntegerType)) } expectAnalysisErrorClass("DELTA_MERGE_INCOMPATIBLE_DATATYPE", Map("currentDataType" -> "IntegerType", "updateDataType" -> "DateType")) { mergeSchemas(base, new StructType() .add("struct", new StructType().add("a", DateType))) } // StructType's toString is different between Scala 2.12 and 2.13. // - In Scala 2.12, it extends `scala.collection.Seq` which returns // `StructType(StructField(a,IntegerType,true))`. // - In Scala 2.13, it extends `scala.collection.immutable.Seq` which returns // `Seq(StructField(a,IntegerType,true))`. expectAnalysisErrorClass("DELTA_MERGE_INCOMPATIBLE_DATATYPE", Map("currentDataType" -> "(StructType|Seq)\\(.*", "updateDataType" -> "MapType\\(.*")) { mergeSchemas(base, new StructType() .add("struct", MapType(StringType, IntegerType))) } expectAnalysisErrorClass("DELTA_MERGE_INCOMPATIBLE_DATATYPE", Map("currentDataType" -> "DecimalType\\(.*", "updateDataType" -> "DoubleType")) { mergeSchemas(base, new StructType() .add("array", ArrayType(new StructType().add("b", DoubleType)))) } expectAnalysisErrorClass("DELTA_MERGE_INCOMPATIBLE_DECIMAL_TYPE", Map("decimalRanges" -> "scale.*")) { mergeSchemas(base, new StructType() .add("array", ArrayType(new StructType().add("b", DecimalType(18, 12))))) } expectAnalysisErrorClass("DELTA_MERGE_INCOMPATIBLE_DECIMAL_TYPE", Map("decimalRanges" -> "precision.*")) { mergeSchemas(base, new StructType() .add("array", ArrayType(new StructType().add("b", DecimalType(16, 10))))) } // See the above comment about `StructType` expectAnalysisErrorClass("DELTA_MERGE_INCOMPATIBLE_DATATYPE", Map("currentDataType" -> "MapType\\(.*", "updateDataType" -> "(StructType|Seq)\\(.*")) { mergeSchemas(base, new StructType() .add("map", new StructType().add("b", StringType))) } expectAnalysisErrorClass("DELTA_MERGE_INCOMPATIBLE_DATATYPE", Map("currentDataType" -> "StringType", "updateDataType" -> "IntegerType")) { mergeSchemas(base, new StructType() .add("map", MapType(StringType, IntegerType))) } expectAnalysisErrorClass("DELTA_MERGE_INCOMPATIBLE_DATATYPE", Map("currentDataType" -> "StringType", "updateDataType" -> "IntegerType")) { mergeSchemas(base, new StructType() .add("map", MapType(IntegerType, StringType))) } } test("schema merging should pick current nullable and metadata") { val m = new MetadataBuilder().putDouble("a", 0.2).build() val base = new StructType() .add("top", StringType, nullable = false, m) .add("struct", new StructType() .add("a", IntegerType, nullable = false, m)) .add("array", ArrayType(new StructType() .add("b", DecimalType(18, 10))), nullable = false, m) .add("map", MapType(StringType, StringType), nullable = false, m) assert(mergeSchemas(base, new StructType().add("top", StringType)) === base) assert(mergeSchemas(base, new StructType().add("struct", new StructType() .add("a", IntegerType))) === base) assert(mergeSchemas(base, new StructType().add("array", ArrayType(new StructType() .add("b", DecimalType(18, 10))))) === base) assert(mergeSchemas(base, new StructType() .add("map", MapType(StringType, StringType))) === base) } test("schema merging null type") { val base = new StructType().add("top", NullType) val update = new StructType().add("top", StringType) assert(mergeSchemas(base, update) === update) assert(mergeSchemas(update, base) === update) } test("schema merging performs upcast between ByteType, ShortType, and IntegerType") { val byteType = new StructType().add("top", ByteType) val shortType = new StructType().add("top", ShortType) val intType = new StructType().add("top", IntegerType) assert(mergeSchemas(byteType, shortType) === shortType) assert(mergeSchemas(byteType, intType) === intType) assert(mergeSchemas(shortType, intType) === intType) assert(mergeSchemas(shortType, byteType) === shortType) assert(mergeSchemas(intType, shortType) === intType) assert(mergeSchemas(intType, byteType) === intType) val structInt = new StructType().add("top", new StructType().add("leaf", IntegerType)) val structShort = new StructType().add("top", new StructType().add("leaf", ShortType)) assert(mergeSchemas(structInt, structShort) === structInt) val map1 = new StructType().add("top", new MapType(IntegerType, ShortType, true)) val map2 = new StructType().add("top", new MapType(ShortType, IntegerType, true)) val mapMerged = new StructType().add("top", new MapType(IntegerType, IntegerType, true)) assert(mergeSchemas(map1, map2) === mapMerged) val arrInt = new StructType().add("top", new ArrayType(IntegerType, true)) val arrShort = new StructType().add("top", new ArrayType(ShortType, true)) assert(mergeSchemas(arrInt, arrShort) === arrInt) } test("schema merging allows upcasting to LongType with allowImplicitConversions") { val byteType = new StructType().add("top", ByteType) val shortType = new StructType().add("top", ShortType) val intType = new StructType().add("top", IntegerType) val longType = new StructType().add("top", LongType) Seq(byteType, shortType, intType).foreach { sourceType => assert( longType === mergeSchemas( longType, sourceType, allowImplicitConversions = true)) val e = intercept[DeltaAnalysisException] { mergeSchemas(longType, sourceType) } checkError( e.getCause.asInstanceOf[AnalysisException], "DELTA_MERGE_INCOMPATIBLE_DATATYPE", parameters = Map("currentDataType" -> "LongType", "updateDataType" -> sourceType.head.dataType.toString)) } } test("Upcast between ByteType, ShortType and IntegerType is OK for parquet") { import org.apache.spark.sql.functions._ def testParquetUpcast(): Unit = { withTempDir { dir => val tempDir = dir.getCanonicalPath spark.range(1.toByte).select(col("id") cast ByteType).write.save(tempDir + "/byte") spark.range(1.toShort).select(col("id") cast ShortType).write.save(tempDir + "/short") spark.range(1).select(col("id") cast IntegerType).write.save(tempDir + "/int") val shortSchema = new StructType().add("id", ShortType) val intSchema = new StructType().add("id", IntegerType) spark.read.schema(shortSchema).parquet(tempDir + "/byte").collect() === Seq(Row(1.toShort)) spark.read.schema(intSchema).parquet(tempDir + "/short").collect() === Seq(Row(1)) spark.read.schema(intSchema).parquet(tempDir + "/byte").collect() === Seq(Row(1)) } } testParquetUpcast() } test("schema merging non struct root type") { // Array root type val base1 = ArrayType(new StructType().add("a", IntegerType)) val update1 = ArrayType(new StructType().add("b", IntegerType)) val mergedType1 = mergeDataTypes( current = base1, update = update1, allowImplicitConversions = false, keepExistingType = false, typeWideningMode = TypeWideningMode.NoTypeWidening, caseSensitive = false, allowOverride = false, overrideMetadata = false) assert(mergedType1 === ArrayType(new StructType().add("a", IntegerType).add("b", IntegerType))) // Map root type val base2 = MapType( new StructType().add("a", IntegerType), new StructType().add("b", IntegerType) ) val update2 = MapType( new StructType().add("b", IntegerType), new StructType().add("c", IntegerType) ) val mergedType2 = mergeDataTypes( current = base2, update = update2, allowImplicitConversions = false, keepExistingType = false, typeWideningMode = TypeWideningMode.NoTypeWidening, caseSensitive = false, allowOverride = false, overrideMetadata = false) assert(mergedType2 === MapType( new StructType().add("a", IntegerType).add("b", IntegerType), new StructType().add("b", IntegerType).add("c", IntegerType) )) } test("schema merging allow override") { // override root type val base1 = new StructType().add("a", IntegerType) val update1 = ArrayType(LongType) val mergedSchema1 = mergeDataTypes( current = base1, update = update1, allowImplicitConversions = false, keepExistingType = false, typeWideningMode = TypeWideningMode.NoTypeWidening, caseSensitive = false, allowOverride = true, overrideMetadata = false) assert(mergedSchema1 === ArrayType(LongType)) // override nested type val base2 = ArrayType(new StructType().add("a", IntegerType).add("b", StringType)) val update2 = ArrayType(new StructType().add("a", MapType(StringType, StringType))) val mergedSchema2 = mergeDataTypes( current = base2, update = update2, allowImplicitConversions = false, keepExistingType = false, typeWideningMode = TypeWideningMode.NoTypeWidening, caseSensitive = false, allowOverride = true, overrideMetadata = false) assert(mergedSchema2 === ArrayType(new StructType().add("a", MapType(StringType, StringType)).add("b", StringType))) } test("keepExistingType and typeWideningMode both set allows both widening and " + "preserving non-widenable existing types") { val base = new StructType() .add("widened", ShortType) .add("struct", new StructType() .add("b", ByteType) .add("a", StringType)) .add("map", MapType(ShortType, IntegerType)) .add("array", ArrayType(StringType)) .add("nonwidened", IntegerType) val update = new StructType() .add("widened", IntegerType) .add("struct", new StructType() .add("b", IntegerType) .add("a", IntegerType)) .add("map", MapType(IntegerType, StringType)) .add("array", ArrayType(ByteType)) .add("nonwidened", StringType) val expected = new StructType() .add("widened", IntegerType) .add("struct", new StructType() .add("b", IntegerType) .add("a", StringType)) .add("map", MapType(IntegerType, IntegerType)) .add("array", ArrayType(StringType)) .add("nonwidened", IntegerType) val mergedSchema = mergeSchemas( base, update, typeWideningMode = TypeWideningMode.TypeEvolution( uniformIcebergCompatibleOnly = false, allowAutomaticWidening = AllowAutomaticWideningMode.default), keepExistingType = true ) assert(mergedSchema === expected) } private val allTypeWideningModes = Set( NoTypeWidening, AllTypeWidening, TypeEvolution( uniformIcebergCompatibleOnly = false, allowAutomaticWidening = AllowAutomaticWideningMode.default), TypeEvolution( uniformIcebergCompatibleOnly = true, allowAutomaticWidening = AllowAutomaticWideningMode.default), AllTypeWideningToCommonWiderType, TypeEvolutionToCommonWiderType(uniformIcebergCompatibleOnly = false), TypeEvolutionToCommonWiderType(uniformIcebergCompatibleOnly = true), AllTypeWideningWithDecimalCoercion, TypeEvolutionWithDecimalCoercion ) test("typeWideningMode - byte->short->int is always allowed") { val narrow = new StructType() .add("a", ByteType) .add("b", ByteType) .add("c", ShortType) .add("s", new StructType().add("x", ByteType)) .add("m", MapType(ByteType, ShortType)) .add("ar", ArrayType(ByteType)) val wide = new StructType() .add("a", ShortType) .add("b", IntegerType) .add("c", IntegerType) .add("s", new StructType().add("x", IntegerType)) .add("m", MapType(ShortType, IntegerType)) .add("ar", ArrayType(IntegerType)) for (typeWideningMode <- allTypeWideningModes) { // byte, short, int are all stored as INT64 in parquet, [[mergeSchemas]] always allows // widening between them. This was already the case before typeWideningMode was introduced. val merged1 = mergeSchemas(narrow, wide, typeWideningMode = typeWideningMode) assert(merged1 === wide) val merged2 = mergeSchemas(wide, narrow, typeWideningMode = typeWideningMode) assert(merged2 === wide) } } // These type changes will only be available once Delta uses Spark 4.0. for ((fromType, toType) <- Seq( IntegerType -> LongType, new StructType().add("x", IntegerType) -> new StructType().add("x", LongType), MapType(IntegerType, IntegerType) -> MapType(LongType, LongType), ArrayType(IntegerType) -> ArrayType(LongType) )) test(s"typeWideningMode ${fromType.sql} -> ${toType.sql}") { val narrow = new StructType().add("a", fromType) val wide = new StructType().add("a", toType) for (typeWideningMode <- Seq( NoTypeWidening, AllTypeWidening, TypeEvolution( uniformIcebergCompatibleOnly = false, allowAutomaticWidening = AllowAutomaticWideningMode.default), TypeEvolution( uniformIcebergCompatibleOnly = true, allowAutomaticWidening = AllowAutomaticWideningMode.default), AllTypeWideningWithDecimalCoercion, TypeEvolutionWithDecimalCoercion)) { // Narrowing is not allowed. expectAnalysisErrorClass("DELTA_MERGE_INCOMPATIBLE_DATATYPE", Map("currentDataType" -> "LongType", "updateDataType" -> "IntegerType")) { mergeSchemas(wide, narrow, typeWideningMode = typeWideningMode) } } for (typeWideningMode <- Seq( AllTypeWideningToCommonWiderType, TypeEvolutionToCommonWiderType(uniformIcebergCompatibleOnly = false), TypeEvolutionToCommonWiderType(uniformIcebergCompatibleOnly = true))) { // These modes don't enforce an order on the inputs, widening from second schema to first // is allowed. val merged = mergeSchemas(wide, narrow, typeWideningMode = typeWideningMode) assert(merged === wide) } for (typeWideningMode <- allTypeWideningModes -- Set(NoTypeWidening)) { // Widening is allowed, unless mode is NoTypeWidening. val merged = mergeSchemas(narrow, wide, typeWideningMode = typeWideningMode) assert(merged === wide) } expectAnalysisErrorClass("DELTA_MERGE_INCOMPATIBLE_DATATYPE", Map("currentDataType" -> "LongType", "updateDataType" -> "IntegerType")) { mergeSchemas(wide, narrow, typeWideningMode = NoTypeWidening) } } for ((fromType, toType) <- Seq( ShortType -> DoubleType, IntegerType -> DecimalType(10, 0) )) test( s"typeWideningMode - blocked type evolution ${fromType.sql} -> ${toType.sql}") { val narrow = new StructType().add("a", fromType) val wide = new StructType().add("a", toType) for (typeWideningMode <- Seq( TypeEvolution( uniformIcebergCompatibleOnly = false, allowAutomaticWidening = AllowAutomaticWideningMode.SAME_FAMILY_TYPE), TypeEvolutionToCommonWiderType(uniformIcebergCompatibleOnly = false), TypeEvolution( uniformIcebergCompatibleOnly = true, allowAutomaticWidening = AllowAutomaticWideningMode.SAME_FAMILY_TYPE), TypeEvolutionToCommonWiderType(uniformIcebergCompatibleOnly = true), TypeEvolutionWithDecimalCoercion)) { expectAnalysisErrorClass( "DELTA_MERGE_INCOMPATIBLE_DATATYPE", Map("currentDataType" -> fromType.toString, "updateDataType" -> toType.toString), matchPVals = false) { mergeSchemas(narrow, wide, typeWideningMode = typeWideningMode) } expectAnalysisErrorClass( "DELTA_MERGE_INCOMPATIBLE_DATATYPE", Map("currentDataType" -> toType.toString, "updateDataType" -> fromType.toString), matchPVals = false) { mergeSchemas(wide, narrow, typeWideningMode = typeWideningMode) } } } for ((fromType, toType) <- Seq( DateType -> TimestampNTZType, DecimalType(10, 2) -> DecimalType(12, 4) )) test( s"typeWideningMode - Uniform Iceberg compatibility ${fromType.sql} -> ${toType.sql}") { val narrow = new StructType().add("a", fromType) val wide = new StructType().add("a", toType) def checkAnalysisException(f: => Unit): Unit = { val ex = intercept[DeltaAnalysisException](f).getCause.asInstanceOf[AnalysisException] // Decimal scale increase return a slightly different error class. assert(ex.errorClass.contains("DELTA_MERGE_INCOMPATIBLE_DATATYPE") || ex.errorClass.contains("DELTA_MERGE_INCOMPATIBLE_DECIMAL_TYPE")) } for (typeWideningMode <- Seq( TypeEvolution( uniformIcebergCompatibleOnly = false, allowAutomaticWidening = AllowAutomaticWideningMode.ALWAYS), TypeEvolutionToCommonWiderType(uniformIcebergCompatibleOnly = false))) { // Unsupported type changes by Iceberg are allowed without Iceberg compatibility. val merged = mergeSchemas(narrow, wide, typeWideningMode = typeWideningMode) assert(merged === wide) } for (typeWideningMode <- Seq( TypeEvolution( uniformIcebergCompatibleOnly = true, allowAutomaticWidening = AllowAutomaticWideningMode.ALWAYS), TypeEvolutionToCommonWiderType(uniformIcebergCompatibleOnly = true))) { // Widening is blocked for unsupported type changes with Iceberg compatibility. checkAnalysisException { mergeSchemas(wide, narrow, typeWideningMode = typeWideningMode) } } // These modes don't enforce an order on the inputs, widening from second schema to first // is allowed without Iceberg compatibility. val merged = mergeSchemas(wide, narrow, typeWideningMode = TypeEvolutionToCommonWiderType(uniformIcebergCompatibleOnly = false)) assert(merged === wide) for (typeWideningMode <- Seq( TypeEvolution( uniformIcebergCompatibleOnly = true, allowAutomaticWidening = AllowAutomaticWideningMode.default), TypeEvolutionToCommonWiderType(uniformIcebergCompatibleOnly = true), TypeEvolution( uniformIcebergCompatibleOnly = true, allowAutomaticWidening = AllowAutomaticWideningMode.default))) { // Rejected either because this is a narrowing type change, or for the bidirectional mode, // because it is not supported by Iceberg. checkAnalysisException { mergeSchemas(wide, narrow, typeWideningMode = typeWideningMode) } } } test( s"typeWideningMode - widen to common wider decimal") { val left = new StructType().add("a", DecimalType(10, 2)) val right = new StructType().add("a", DecimalType(5, 4)) val wider = new StructType().add("a", DecimalType(12, 4)) val modesCanWidenToCommonWiderDecimal = Set( // Increasing decimal scale isn't supported by Iceberg, so only possible when we don't enforce // Iceberg compatibility. TypeEvolutionToCommonWiderType(uniformIcebergCompatibleOnly = false), AllTypeWideningToCommonWiderType, AllTypeWideningWithDecimalCoercion, TypeEvolutionWithDecimalCoercion ) for (typeWideningMode <- modesCanWidenToCommonWiderDecimal) { assert(mergeSchemas(left, right, typeWideningMode = typeWideningMode) === wider) assert(mergeSchemas(right, left, typeWideningMode = typeWideningMode) === wider) } for (typeWideningMode <- allTypeWideningModes -- modesCanWidenToCommonWiderDecimal) { expectAnalysisErrorClass( "DELTA_MERGE_INCOMPATIBLE_DECIMAL_TYPE", Map("decimalRanges" -> "precision 10 and 5 & scale 2 and 4"), matchPVals = false) { mergeSchemas(left, right, typeWideningMode = typeWideningMode) } expectAnalysisErrorClass( "DELTA_MERGE_INCOMPATIBLE_DECIMAL_TYPE", Map("decimalRanges" -> "precision 5 and 10 & scale 4 and 2"), matchPVals = false) { mergeSchemas(right, left, typeWideningMode = typeWideningMode) } } } test( s"typeWideningMode - widen to common wider decimal exceeds max decimal precision") { // We'd need a DecimalType(40, 19) to fit both types, which exceeds max decimal precision of 38. val left = new StructType().add("a", DecimalType(20, 19)) val right = new StructType().add("a", DecimalType(21, 0)) for (typeWideningMode <- allTypeWideningModes) { expectAnalysisErrorClass( "DELTA_MERGE_INCOMPATIBLE_DECIMAL_TYPE", Map("decimalRanges" -> "precision 20 and 21 & scale 19 and 0"), matchPVals = false) { mergeSchemas(left, right, typeWideningMode = typeWideningMode) } expectAnalysisErrorClass( "DELTA_MERGE_INCOMPATIBLE_DECIMAL_TYPE", Map("decimalRanges" -> "precision 21 and 20 & scale 0 and 19"), matchPVals = false) { mergeSchemas(right, left, typeWideningMode = typeWideningMode) } } } test(s"typeWideningMode - widen integral type to common wider decimal") { val left = new StructType() .add("a", ByteType) .add("b", ShortType) .add("c", IntegerType) .add("d", LongType) val right = new StructType() .add("a", DecimalType(2, 1)) .add("b", DecimalType(2, 1)) .add("c", DecimalType(2, 1)) .add("d", DecimalType(2, 1)) val wider = new StructType() .add("a", DecimalType(4, 1)) .add("b", DecimalType(6, 1)) .add("c", DecimalType(11, 1)) .add("d", DecimalType(21, 1)) assert(mergeSchemas(left, right, typeWideningMode = AllTypeWideningWithDecimalCoercion) == wider) assert(mergeSchemas(left, right, typeWideningMode = AllTypeWideningToCommonWiderType) == wider) // check that flipping conf to false prevents integral type decimal coercion // for `AllTypeWideningToCommonWiderType` withSQLConf(DeltaSQLConf.DELTA_TYPE_WIDENING_ALLOW_INTEGRAL_DECIMAL_COERCION.key -> "false") { val exception = intercept[DeltaAnalysisException] { mergeSchemas(left, right, typeWideningMode = AllTypeWideningToCommonWiderType) } checkError( exception, "DELTA_FAILED_TO_MERGE_FIELDS", sqlState = "22005", parameters = Map("currentField" -> "a", "updateField" -> "a") ) } } test("schema merging override field metadata") { val base1 = new StructType() .add("a", IntegerType) val update1 = new StructType() .add("a", IntegerType, nullable = true, new MetadataBuilder().putString("x", "1").build()) val mergedSchema1 = mergeDataTypes( current = base1, update = update1, allowImplicitConversions = false, keepExistingType = false, typeWideningMode = TypeWideningMode.NoTypeWidening, caseSensitive = false, allowOverride = false, overrideMetadata = true ) assert(mergedSchema1 === new StructType() .add("a", IntegerType, nullable = true, new MetadataBuilder().putString("x", "1").build())) // override nested metadata val base2 = ArrayType(new StructType() .add("a", new StructType() .add("b", IntegerType) .add("c", IntegerType))) val update2 = ArrayType(new StructType() .add("a", new StructType() .add("b", IntegerType) .add("c", IntegerType, nullable = true, new MetadataBuilder().putString("c_metadata", "2").build()), nullable = true, new MetadataBuilder().putString("a_metadata", "3").build())) val mergedSchema2 = mergeDataTypes( current = base2, update = update2, allowImplicitConversions = false, keepExistingType = false, typeWideningMode = TypeWideningMode.NoTypeWidening, caseSensitive = false, allowOverride = false, overrideMetadata = true ) assert(mergedSchema2 === ArrayType(new StructType() .add("a", new StructType() .add("b", IntegerType) .add("c", IntegerType, nullable = true, new MetadataBuilder().putString("c_metadata", "2").build()), nullable = true, new MetadataBuilder().putString("a_metadata", "3").build()))) } //////////////////////////// // transformColumns //////////////////////////// test("transform columns - simple") { val base = new StructType() .add("a", IntegerType) .add("b", StringType) val update = new StructType() .add("c", IntegerType) .add("b", StringType) // Identity. var visitedFields = 0 val res1 = SchemaMergingUtils.transformColumns(base) { case (Seq(), field, _) => visitedFields += 1 field } assert(visitedFields === 2) assert(base === res1) // Rename a -> c visitedFields = 0 val res2 = SchemaMergingUtils.transformColumns(base) { case (Seq(), field, _) => visitedFields += 1 val name = field.name field.copy(name = if (name == "a") "c" else name) } assert(visitedFields === 2) assert(update === res2) // Rename a -> c; using input map. visitedFields = 0 val res3 = transformColumns(base, (Seq("A"), "c") :: Nil) { case (Seq(), field, Seq((_, newName))) => visitedFields += 1 field.copy(name = newName) } assert(visitedFields === 1) assert(update === res3) } test("transform element field type") { val base = new StructType() .add("a", new StructType() .add("element", StringType)) val update = new StructType() .add("a", new StructType() .add("element", IntegerType)) // Update type var visitedFields = 0 val res = SchemaMergingUtils.transformColumns(base) { (path, field, _) => visitedFields += 1 val dataType = path :+ field.name match { case Seq("a", "element") => IntegerType case _ => field.dataType } field.copy(dataType = dataType) } assert(visitedFields === 2) assert(update === res) } test("transform array nested field type") { val nested = new StructType() .add("s1", IntegerType) .add("s2", LongType) val base = new StructType() .add("arr", ArrayType(nested)) val updatedNested = new StructType() .add("s1", StringType) .add("s2", LongType) val update = new StructType() .add("arr", ArrayType(updatedNested)) // Update type var visitedFields = 0 val res = SchemaMergingUtils.transformColumns(base) { (path, field, _) => visitedFields += 1 val dataType = path :+ field.name match { case Seq("arr", "element", "s1") => StringType case _ => field.dataType } field.copy(dataType = dataType) } assert(visitedFields === 3) assert(update === res) } test("transform map nested field type") { val nested = new StructType() .add("s1", IntegerType) .add("s2", LongType) val base = new StructType() .add("m", MapType(StringType, nested)) val updatedNested = new StructType() .add("s1", StringType) .add("s2", LongType) val update = new StructType() .add("m", MapType(StringType, updatedNested)) // Update type var visitedFields = 0 val res = SchemaMergingUtils.transformColumns(base) { (path, field, _) => visitedFields += 1 val dataType = path :+ field.name match { case Seq("m", "value", "s1") => StringType case _ => field.dataType } field.copy(dataType = dataType) } assert(visitedFields === 3) assert(update === res) } test("transform map type") { val base = new StructType() .add("m", MapType(StringType, IntegerType)) val update = new StructType() .add("m", MapType(StringType, StringType)) // Update type var visitedFields = 0 val res = SchemaMergingUtils.transformColumns(base) { (path, field, _) => visitedFields += 1 val dataType = path :+ field.name match { case Seq("m") => MapType(field.dataType.asInstanceOf[MapType].keyType, StringType) case _ => field.dataType } field.copy(dataType = dataType) } assert(visitedFields === 1) assert(update === res) } test("transform columns - nested") { val nested = new StructType() .add("s1", IntegerType) .add("s2", LongType) val base = new StructType() .add("nested", nested) .add("arr", ArrayType(nested)) .add("kvs", MapType(nested, nested)) val update = new StructType() .add("nested", new StructType() .add("t1", IntegerType) .add("s2", LongType)) .add("arr", ArrayType( new StructType() .add("s1", IntegerType) .add("a2", LongType))) .add("kvs", MapType( new StructType() .add("k1", IntegerType) .add("s2", LongType), new StructType() .add("s1", IntegerType) .add("v2", LongType))) // Identity. var visitedFields = 0 val res1 = SchemaMergingUtils.transformColumns(base) { case (_, field, _) => visitedFields += 1 field } assert(visitedFields === 11) assert(base === res1) // Rename visitedFields = 0 val res2 = SchemaMergingUtils.transformColumns(base) { (path, field, _) => visitedFields += 1 val name = path :+ field.name match { case Seq("nested", "s1") => "t1" case Seq("arr", "element", "s2") => "a2" case Seq("kvs", "key", "s1") => "k1" case Seq("kvs", "value", "s2") => "v2" case _ => field.name } field.copy(name = name) } assert(visitedFields === 11) assert(update === res2) // Rename; using map visitedFields = 0 val mapping = Seq( Seq("nested", "s1") -> "t1", Seq("arr", "element", "s2") -> "a2", Seq("kvs", "key", "S1") -> "k1", Seq("kvs", "value", "s2") -> "v2") val res3 = transformColumns(base, mapping) { case (_, field, Seq((_, name))) => visitedFields += 1 field.copy(name = name) } assert(visitedFields === 4) assert(update === res3) } test("transform top level array type") { val at = ArrayType( new StructType() .add("s1", IntegerType) ) var visitedFields = 0 val updated = SchemaMergingUtils.transformColumns(at) { case (_, field, _) => visitedFields += 1 field.copy(name = "s1_1", dataType = StringType) } assert(visitedFields === 1) assert(updated === ArrayType(new StructType().add("s1_1", StringType))) } test("transform top level map type") { val mt = MapType( new StructType() .add("k1", IntegerType), new StructType() .add("v1", IntegerType) ) var visitedFields = 0 val updated = SchemaMergingUtils.transformColumns(mt) { case (_, field, _) => visitedFields += 1 field.copy(name = field.name + "_1", dataType = StringType) } assert(visitedFields === 2) assert(updated === MapType( new StructType().add("k1_1", StringType), new StructType().add("v1_1", StringType) )) } //////////////////////////// // pruneEmptyStructs //////////////////////////// test("prune empty structs") { val emptySchema = new StructType() var schema = emptySchema assert(SchemaMergingUtils.pruneEmptyStructs(schema).isEmpty) val elementType = new StructType() .add("a", emptySchema) .add("b", new StructType().add("1", emptySchema).add("2", StringType)) val filteredElementType = new StructType().add("b", new StructType().add("2", StringType)) assert(SchemaMergingUtils.pruneEmptyStructs(elementType).get == filteredElementType) // filter out array element type with empty schema schema = new StructType().add("a", new ArrayType(emptySchema, false)) assert(SchemaMergingUtils.pruneEmptyStructs(schema).isEmpty) // nested empty schema schema = new StructType().add("a", new ArrayType(new StructType().add("a", emptySchema), false)) assert(SchemaMergingUtils.pruneEmptyStructs(schema).isEmpty) schema = new StructType().add("a", new ArrayType(elementType, false)) assert( SchemaMergingUtils.pruneEmptyStructs(schema).get == new StructType().add("a", new ArrayType(filteredElementType, false)) ) schema = new StructType().add("a", new MapType(emptySchema, StringType, false)) assert(SchemaMergingUtils.pruneEmptyStructs(schema).isEmpty) schema = new StructType().add("a", new MapType(StringType, emptySchema, false)) assert(SchemaMergingUtils.pruneEmptyStructs(schema).isEmpty) schema = new StructType().add("a", new MapType(StringType, elementType, false)) assert( SchemaMergingUtils.pruneEmptyStructs(schema).get == new StructType().add("a", new MapType(StringType, filteredElementType, false)) ) } //////////////////////////// // checkFieldNames //////////////////////////// test("check non alphanumeric column characters") { val badCharacters = " ,;{}()\n\t=" val goodCharacters = "#.`!@$%^&*~_<>?/:" badCharacters.foreach { char => Seq(s"a${char}b", s"${char}ab", s"ab${char}", char.toString).foreach { name => checkError( intercept[AnalysisException] { SchemaUtils.checkFieldNames(Seq(name)) }, "DELTA_INVALID_CHARACTERS_IN_COLUMN_NAME", parameters = Map("columnName" -> s"$name") ) } } goodCharacters.foreach { char => // no issues here SchemaUtils.checkFieldNames(Seq(s"a${char}b", s"${char}ab", s"ab${char}", char.toString)) } } test("fieldToColumn") { assert(SchemaUtils.fieldToColumn(StructField("a", IntegerType)).expr == new UnresolvedAttribute("a" :: Nil)) // Dot in the column name should be converted correctly assert(SchemaUtils.fieldToColumn(StructField("a.b", IntegerType)).expr == new UnresolvedAttribute("a.b" :: Nil)) } //////////////////////////// // findNestedFieldIgnoreCase //////////////////////////// test("complex schema access") { val st = StringType val it = IntegerType def m(a: DataType, b: DataType): MapType = MapType(a, b) def a(el: DataType): ArrayType = ArrayType(el) def struct(el: DataType): StructType = new StructType().add("f1", el) val schema = new StructType() .add("a", it) .add("b", struct(st)) .add("c", struct(struct(struct(st)))) .add("d", a(it)) .add("e", a(a(it))) .add("f", a(a(struct(st)))) .add("g", m(m(st, it), m(st, it))) .add("h", m(a(st), a(it))) .add("i", m(a(struct(st)), a(struct(st)))) .add("j", m(m(struct(st), struct(it)), m(struct(st), struct(it)))) .add("k", m(struct(a(a(struct(a(struct(st)))))), m(m(struct(st), struct(it)), m(struct(st), struct(it))))) def find(names: Seq[String]): Option[StructField] = SchemaUtils.findNestedFieldIgnoreCase(schema, names, true) val checks = Map( "a" -> it, "b" -> struct(st), "b.f1" -> st, "c.f1.f1.f1" -> st, "d.element" -> it, "e.element.element" -> it, "f.element.element.f1" -> st, "g.key.key" -> st, "g.key.value" -> it, "g.value.key" -> st, "g.value.value" -> it, "h.key.element" -> st, "h.value.element" -> it, "i.key.element.f1" -> st, "i.value.element.f1" -> st, "j.key.key.f1" -> st, "j.key.value.f1" -> it, "j.value.key.f1" -> st, "j.value.value.f1" -> it, "k.key.f1.element.element.f1.element.f1" -> st, "k.value.key.key.f1" -> st, "k.value.key.value.f1" -> it, "k.value.value.key.f1" -> st, "k.value.value.value.f1" -> it ) checks.foreach { pair => val (key, t) = pair val path = key.split('.') val f = find(path) assert(f.isDefined, s"cannot find $key") assert(f.get.name == path.last && f.get.dataType == t) } val negativeChecks = Seq( "x", "b.f2", "c.f1.f2", "c.f1.f1.f2", "d.f1", "d.element.f1", "e.element.element.f1", "f.element.key.f1", "g.key.element", "g.key.keyy", "g.key.valuee", "h.key.element.f1", "k.key.f1.element.element.f2.element.f1", "k.value.value.f1" ) negativeChecks.foreach { key => val path = key.split('.') val f = find(path) assert(f.isEmpty, s"$key should be empty") } } test("findUnsupportedDataTypes") { def assertUnsupportedDataType( dataType: DataType, expected: Seq[UnsupportedDataTypeInfo]): Unit = { val schema = StructType(Seq(StructField("col", dataType))) assert(findUnsupportedDataTypes(schema) == expected) } assertUnsupportedDataType(NullType, Nil) assertUnsupportedDataType(BooleanType, Nil) assertUnsupportedDataType(ByteType, Nil) assertUnsupportedDataType(ShortType, Nil) assertUnsupportedDataType(IntegerType, Nil) assertUnsupportedDataType(LongType, Nil) assertUnsupportedDataType( YearMonthIntervalType.DEFAULT, Seq(UnsupportedDataTypeInfo("col", YearMonthIntervalType.DEFAULT))) assertUnsupportedDataType( DayTimeIntervalType.DEFAULT, Seq(UnsupportedDataTypeInfo("col", DayTimeIntervalType.DEFAULT))) assertUnsupportedDataType(FloatType, Nil) assertUnsupportedDataType(DoubleType, Nil) assertUnsupportedDataType(StringType, Nil) assertUnsupportedDataType(DateType, Nil) assertUnsupportedDataType(TimestampType, Nil) assertUnsupportedDataType( CalendarIntervalType, Seq(UnsupportedDataTypeInfo("col", CalendarIntervalType))) assertUnsupportedDataType(BinaryType, Nil) assertUnsupportedDataType(DataTypes.createDecimalType(), Nil) assertUnsupportedDataType( UnsupportedDataType, Seq(UnsupportedDataTypeInfo("col", UnsupportedDataType))) // array assertUnsupportedDataType(ArrayType(IntegerType, true), Nil) assertUnsupportedDataType( ArrayType(UnsupportedDataType, true), Seq(UnsupportedDataTypeInfo("col[]", UnsupportedDataType))) // map assertUnsupportedDataType(MapType(IntegerType, IntegerType, true), Nil) assertUnsupportedDataType( MapType(UnsupportedDataType, IntegerType, true), Seq(UnsupportedDataTypeInfo("col[key]", UnsupportedDataType))) assertUnsupportedDataType( MapType(IntegerType, UnsupportedDataType, true), Seq(UnsupportedDataTypeInfo("col[value]", UnsupportedDataType))) assertUnsupportedDataType( MapType(UnsupportedDataType, UnsupportedDataType, true), Seq( UnsupportedDataTypeInfo("col[key]", UnsupportedDataType), UnsupportedDataTypeInfo("col[value]", UnsupportedDataType))) // struct assertUnsupportedDataType(StructType(StructField("f", LongType) :: Nil), Nil) assertUnsupportedDataType( StructType(StructField("a", LongType) :: StructField("dot.name", UnsupportedDataType) :: Nil), Seq(UnsupportedDataTypeInfo("col.`dot.name`", UnsupportedDataType))) val nestedStructType = StructType(Seq( StructField("a", LongType), StructField("b", StructType(Seq( StructField("c", LongType), StructField("d", UnsupportedDataType) ))), StructField("e", StructType(Seq( StructField("f", LongType), StructField("g", UnsupportedDataType) ))) )) assertUnsupportedDataType( nestedStructType, Seq( UnsupportedDataTypeInfo("col.b.d", UnsupportedDataType), UnsupportedDataTypeInfo("col.e.g", UnsupportedDataType))) // udt assertUnsupportedDataType(new PointUDT, Nil) assertUnsupportedDataType( new UnsupportedUDT, Seq(UnsupportedDataTypeInfo("col", UnsupportedDataType))) } test("findUndefinedTypes: basic types") { val schema = StructType(Seq( StructField("c1", NullType), StructField("c2", BooleanType), StructField("c3", ByteType), StructField("c4", ShortType), StructField("c5", IntegerType), StructField("c6", LongType), StructField("c7", FloatType), StructField("c8", DoubleType), StructField("c9", StringType), StructField("c10", DateType), StructField("c11", TimestampType), StructField("c12", BinaryType), StructField("c13", DataTypes.createDecimalType()), // undefined types StructField("c14", TimestampNTZType), StructField("c15", YearMonthIntervalType.DEFAULT), StructField("c16", DayTimeIntervalType.DEFAULT), StructField("c17", new PointUDT) // UserDefinedType )) val udts = findUndefinedTypes(schema) assert(udts.map(_.getClass.getName.stripSuffix("$")) == Seq( classOf[TimestampNTZType], classOf[YearMonthIntervalType], classOf[DayTimeIntervalType], classOf[PointUDT] ).map(_.getName.stripSuffix("$")) ) } test("findUndefinedTypes: complex types") { val schema = StructType(Seq( StructField("c1", new PointUDT), StructField("c2", ArrayType(new PointUDT, true)), StructField("c3", MapType(new PointUDT, new PointUDT, true)), StructField("c4", StructType(Seq( StructField("c1", new PointUDT), StructField("c2", ArrayType(new PointUDT, true)), StructField("c3", MapType(new PointUDT, new PointUDT, true)) ))) )) val udts = findUndefinedTypes(schema) assert(udts.size == 8) assert(udts.map(_.getClass.getName).toSet == Set(classOf[PointUDT].getName)) } test("check if column affects given dependent expressions") { val schema = StructType(Seq( StructField("cArray", ArrayType(StringType)), StructField("cStruct", StructType(Seq( StructField("cMap", MapType(IntegerType, ArrayType(BooleanType))), StructField("cMapWithComplexKey", MapType(StructType(Seq( StructField("a", ArrayType(StringType)), StructField("b", BooleanType) )), IntegerType)) ))) )) assert( SchemaUtils.containsDependentExpression( spark, columnToChange = Seq("cArray"), exprString = "cast(cStruct.cMap as string) == '{}'", schema, caseInsensitiveResolution) === false ) // Extracting value from map uses key type as well. assert( SchemaUtils.containsDependentExpression( spark, columnToChange = Seq("cStruct", "cMap", "key"), exprString = "cStruct.cMap['random_key'] == 'string'", schema, caseInsensitiveResolution) === true ) assert( SchemaUtils.containsDependentExpression( spark, columnToChange = Seq("cstruct"), exprString = "size(cStruct.cMap) == 0", schema, caseSensitiveResolution) === false ) assert( SchemaUtils.containsDependentExpression( spark, columnToChange = Seq("cStruct", "cMap", "key"), exprString = "size(cArray) == 1", schema, caseInsensitiveResolution) === false ) assert( SchemaUtils.containsDependentExpression( spark, columnToChange = Seq("cStruct", "cMap", "key"), exprString = "cStruct.cMapWithComplexKey[struct(cArray, false)] == 0", schema, caseInsensitiveResolution) === false ) assert( SchemaUtils.containsDependentExpression( spark, columnToChange = Seq("cArray", "element"), exprString = "cStruct.cMapWithComplexKey[struct(cArray, false)] == 0", schema, caseInsensitiveResolution) === true ) assert( SchemaUtils.containsDependentExpression( spark, columnToChange = Seq("cStruct", "cMapWithComplexKey", "key", "b"), exprString = "cStruct.cMapWithComplexKey[struct(cArray, false)] == 0", schema, caseInsensitiveResolution) === true ) assert( SchemaUtils.containsDependentExpression( spark, columnToChange = Seq("cArray", "element"), exprString = "concat_ws('', cArray) == 'string'", schema, caseInsensitiveResolution) === true ) assert( SchemaUtils.containsDependentExpression( spark, columnToChange = Seq("CARRAY"), exprString = "cArray[0] > 'a'", schema, caseInsensitiveResolution) === true ) assert( SchemaUtils.containsDependentExpression( spark, columnToChange = Seq("CARRAY", "element"), exprString = "cArray[0] > 'a'", schema, caseSensitiveResolution) === false ) } } object UnsupportedDataType extends DataType { override def defaultSize: Int = throw new UnsupportedOperationException("defaultSize") override def asNullable: DataType = throw new UnsupportedOperationException("asNullable") override def toString: String = "UnsupportedDataType" } @SQLUserDefinedType(udt = classOf[PointUDT]) case class Point(x: Int, y: Int) class PointUDT extends UserDefinedType[Point] { override def sqlType: DataType = StructType(Array( StructField("x", IntegerType, nullable = false), StructField("y", IntegerType, nullable = false))) override def serialize(obj: Point): Any = InternalRow(obj.x, obj.y) override def deserialize(datum: Any): Point = datum match { case row: InternalRow => Point(row.getInt(0), row.getInt(1)) } override def userClass: Class[Point] = classOf[Point] override def toString: String = "PointUDT" } class UnsupportedUDT extends PointUDT { override def sqlType: DataType = UnsupportedDataType } case class Struct1( firstname: String, numberone: Long, lastname: String, numbertwo: Long, CorrectCase: String) ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTableSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.serverSidePlanning import org.apache.spark.sql.{AnalysisException, QueryTest, Row} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.sources.{And, EqualTo, Filter, GreaterThan, LessThan} /** * Tests for server-side planning with a mock client. */ class ServerSidePlannedTableSuite extends QueryTest with DeltaSQLCommandTest { override def beforeAll(): Unit = { super.beforeAll() // Create test database and shared table once for all tests sql("CREATE DATABASE IF NOT EXISTS test_db") sql(""" CREATE TABLE test_db.shared_test ( id INT, name STRING, value INT, a STRUCT<`b.c`: STRING> ) USING parquet """) sql(""" INSERT INTO test_db.shared_test (id, name, value, a) VALUES (1, 'alpha', 10, struct('abc_1')), (2, 'beta', 20, struct('abc_2')), (3, 'gamma', 30, struct('abc_3')) """) } /** * Helper method to run tests with server-side planning enabled. * Automatically sets up the test factory and config, then cleans up afterwards. * This prevents test pollution from leaked configuration. */ private def withServerSidePlanningEnabled(f: => Unit): Unit = { withServerSidePlanningFactory(new TestServerSidePlanningClientFactory())(f) } /** * Helper method to run tests with pushdown capturing enabled. * TestServerSidePlanningClient captures pushdowns (filter, projection) passed to planScan(). */ private def withPushdownCapturingEnabled(f: => Unit): Unit = { withServerSidePlanningFactory(new TestServerSidePlanningClientFactory()) { try { f } finally { TestServerSidePlanningClient.clearCaptured() } } } /** * Common helper for setting up server-side planning with a specific factory. */ private def withServerSidePlanningFactory(factory: ServerSidePlanningClientFactory) (f: => Unit): Unit = { val originalConfig = spark.conf.getOption(DeltaSQLConf.ENABLE_SERVER_SIDE_PLANNING.key) ServerSidePlanningClientFactory.setFactory(factory) spark.conf.set(DeltaSQLConf.ENABLE_SERVER_SIDE_PLANNING.key, "true") try { f } finally { // Reset factory ServerSidePlanningClientFactory.clearFactory() // Restore original config originalConfig match { case Some(value) => spark.conf.set(DeltaSQLConf.ENABLE_SERVER_SIDE_PLANNING.key, value) case None => spark.conf.unset(DeltaSQLConf.ENABLE_SERVER_SIDE_PLANNING.key) } } } /** * Extract all leaf filters from a filter tree. * Spark may wrap filters with And and IsNotNull checks, so this flattens the tree. */ private def collectLeafFilters(filter: Filter): Seq[Filter] = filter match { case And(left, right) => collectLeafFilters(left) ++ collectLeafFilters(right) case other => Seq(other) } test("full query through DeltaCatalog with server-side planning") { // This test verifies server-side planning works end-to-end by checking: // (1) DeltaCatalog returns ServerSidePlannedTable (not normal table) // (2) Query execution returns correct results // If both are true, the server-side planning client worked correctly - that's the only way // ServerSidePlannedTable can read data. withServerSidePlanningEnabled { // (1) Verify that DeltaCatalog actually returns ServerSidePlannedTable val catalog = spark.sessionState.catalogManager.catalog("spark_catalog") .asInstanceOf[org.apache.spark.sql.connector.catalog.TableCatalog] val loadedTable = catalog.loadTable( org.apache.spark.sql.connector.catalog.Identifier.of( Array("test_db"), "shared_test")) assert(loadedTable.isInstanceOf[ServerSidePlannedTable], s"Expected ServerSidePlannedTable but got ${loadedTable.getClass.getName}") // (2) Execute query - should go through full server-side planning stack checkAnswer( sql("SELECT id, name, value FROM test_db.shared_test ORDER BY id"), Seq( Row(1, "alpha", 10), Row(2, "beta", 20), Row(3, "gamma", 30) ) ) } } test("verify normal path unchanged when feature disabled") { // Explicitly disable server-side planning spark.conf.set(DeltaSQLConf.ENABLE_SERVER_SIDE_PLANNING.key, "false") // Verify that DeltaCatalog returns normal table, not ServerSidePlannedTable val catalog = spark.sessionState.catalogManager.catalog("spark_catalog") .asInstanceOf[org.apache.spark.sql.connector.catalog.TableCatalog] val loadedTable = catalog.loadTable( org.apache.spark.sql.connector.catalog.Identifier.of( Array("test_db"), "shared_test")) assert(!loadedTable.isInstanceOf[ServerSidePlannedTable], s"Expected normal table but got ServerSidePlannedTable when config is disabled") } test("shouldUseServerSidePlanning() decision logic") { // ============================================================ // Production mode: skipUCRequirementForTests = false // Should return true ONLY when all three conditions are met: // 1. Unity Catalog table // 2. No credentials available // 3. Enable flag is set // ============================================================ assert(ServerSidePlannedTable.shouldUseServerSidePlanning( isUnityCatalog = true, hasCredentials = false, enableServerSidePlanning = true, skipUCRequirementForTests = false ) == true, "Production: UC without credentials + flag enabled should use SSP") // Group 1: flag disabled (even with valid UC setup) assert(ServerSidePlannedTable.shouldUseServerSidePlanning( isUnityCatalog = true, hasCredentials = false, enableServerSidePlanning = false, skipUCRequirementForTests = false ) == false, "Production: UC without credentials but flag disabled should NOT use SSP") // Group 2: Has credentials (SSP not needed) assert(ServerSidePlannedTable.shouldUseServerSidePlanning( isUnityCatalog = true, hasCredentials = true, enableServerSidePlanning = true, skipUCRequirementForTests = false ) == false, "Production: UC with credentials should NOT use SSP (has creds)") assert(ServerSidePlannedTable.shouldUseServerSidePlanning( isUnityCatalog = true, hasCredentials = true, enableServerSidePlanning = false, skipUCRequirementForTests = false ) == false, "Production: UC with credentials and no flag should NOT use SSP") // Group 3: Not Unity Catalog (always false, regardless of other params) for (hasCreds <- Seq(true, false)) { for (enableSSP <- Seq(true, false)) { assert(ServerSidePlannedTable.shouldUseServerSidePlanning( isUnityCatalog = false, hasCredentials = hasCreds, enableServerSidePlanning = enableSSP, skipUCRequirementForTests = false ) == false, s"Production: Non-UC should NOT use SSP (hasCreds=$hasCreds, enableSSP=$enableSSP)") } } // ============================================================ // Test mode: skipUCRequirementForTests = true // Should return true if flag is enabled (UC/creds checks bypassed) // Keep as loop since logic is simple and demonstrates bypass // ============================================================ for (isUC <- Seq(true, false)) { for (hasCreds <- Seq(true, false)) { for (enableSSP <- Seq(true, false)) { val description = s"Test mode: isUC=$isUC, hasCreds=$hasCreds, enableSSP=$enableSSP" val expected = enableSSP // In test mode, only the flag matters val result = ServerSidePlannedTable.shouldUseServerSidePlanning( isUnityCatalog = isUC, hasCredentials = hasCreds, enableServerSidePlanning = enableSSP, skipUCRequirementForTests = true ) assert(result == expected, s"$description -> expected $expected but got $result") } } } } test("ServerSidePlannedTable is read-only") { withTable("readonly_test") { sql(""" CREATE TABLE readonly_test ( id INT, data STRING ) USING parquet """) // First insert WITHOUT server-side planning should succeed sql("INSERT INTO readonly_test VALUES (1, 'initial')") checkAnswer( sql("SELECT * FROM readonly_test"), Seq(Row(1, "initial")) ) // Try to insert WITH server-side planning enabled - should fail withServerSidePlanningEnabled { val exception = intercept[AnalysisException] { sql("INSERT INTO readonly_test VALUES (2, 'should_fail')") } assert(exception.getMessage.contains("does not support append")) } // Verify data unchanged - second insert didn't happen checkAnswer( sql("SELECT * FROM readonly_test"), Seq(Row(1, "initial")) ) } } test("ServerSidePlanningMetadata.fromTable returns empty defaults for non-UC catalogs") { import org.apache.spark.sql.connector.catalog.Identifier // Create a simple identifier for testing val ident = Identifier.of(Array("my_catalog", "my_schema"), "my_table") // Call fromTable with a null table (we only use the identifier for catalog name extraction) val metadata = ServerSidePlanningMetadata.fromTable( table = null, spark = spark, ident = ident, isUnityCatalog = false ) // Verify the metadata has expected defaults assert(metadata.catalogName == "my_catalog") assert(metadata.planningEndpointUri == "") assert(metadata.authToken.isEmpty) assert(metadata.tableProperties.isEmpty) } test("UnityCatalogMetadata constructs base IRC endpoint from UC URI") { val ucUri = "https://unity-catalog-server.example.com" val metadata = UnityCatalogMetadata( catalogName = "test_catalog", ucUri = ucUri, ucToken = "test-token", tableProps = Map.empty ) // UnityCatalogMetadata returns the base Iceberg REST path up to /v1. // The IcebergRESTCatalogPlanningClient then calls config to get the prefix // and constructs the full endpoint URL per the Iceberg REST catalog spec. val expectedEndpoint = "https://unity-catalog-server.example.com/api/2.1/unity-catalog/" + "iceberg-rest/v1" assert(metadata.planningEndpointUri == expectedEndpoint) } test("simple EqualTo filter pushed to planning client") { withPushdownCapturingEnabled { sql("SELECT id, name, value FROM test_db.shared_test WHERE id = 2").collect() val capturedFilter = TestServerSidePlanningClient.getCapturedFilter assert(capturedFilter.isDefined, "Filter should be pushed down") // Extract leaf filters and find the EqualTo filter val leafFilters = collectLeafFilters(capturedFilter.get) val eqFilter = leafFilters.collectFirst { case eq: EqualTo if eq.attribute == "id" => eq } assert(eqFilter.isDefined, "Expected EqualTo filter on 'id'") assert(eqFilter.get.value == 2, s"Expected EqualTo value 2, got ${eqFilter.get.value}") } } test("compound And filter pushed to planning client") { withPushdownCapturingEnabled { sql("SELECT id, name, value FROM test_db.shared_test WHERE id > 1 AND value < 30").collect() val capturedFilter = TestServerSidePlanningClient.getCapturedFilter assert(capturedFilter.isDefined, "Filter should be pushed down") val filter = capturedFilter.get assert(filter.isInstanceOf[And], s"Expected And filter, got ${filter.getClass.getSimpleName}") // Extract all leaf filters from the And tree (Spark may add IsNotNull checks) val leafFilters = collectLeafFilters(filter) // Verify GreaterThan(id, 1) is present val gtFilter = leafFilters.collectFirst { case gt: GreaterThan if gt.attribute == "id" => gt } assert(gtFilter.isDefined, "Expected GreaterThan filter on 'id'") assert(gtFilter.get.value == 1, s"Expected GreaterThan value 1, got ${gtFilter.get.value}") // Verify LessThan(value, 30) is present val ltFilter = leafFilters.collectFirst { case lt: LessThan if lt.attribute == "value" => lt } assert(ltFilter.isDefined, "Expected LessThan filter on 'value'") assert(ltFilter.get.value == 30, s"Expected LessThan value 30, got ${ltFilter.get.value}") } } test("no filter pushed when no WHERE clause") { withPushdownCapturingEnabled { sql("SELECT id, name, value FROM test_db.shared_test").collect() val capturedFilter = TestServerSidePlanningClient.getCapturedFilter assert(capturedFilter.isEmpty, "No filter should be pushed when there's no WHERE clause") } } test("projection pushed when selecting specific columns") { withPushdownCapturingEnabled { sql("SELECT id, name FROM test_db.shared_test").collect() val capturedProjection = TestServerSidePlanningClient.getCapturedProjection assert(capturedProjection.isDefined, "Projection should be pushed down") assert(capturedProjection.get.toSet == Set("id", "name"), s"Expected {id, name}, got {${capturedProjection.get.mkString(", ")}}") } } test("no projection pushed when selecting all columns") { withPushdownCapturingEnabled { sql("SELECT * FROM test_db.shared_test").collect() val capturedProjection = TestServerSidePlanningClient.getCapturedProjection assert(capturedProjection.isEmpty, "No projection should be pushed when selecting all columns") } } test("projection escaping with dotted column names") { withPushdownCapturingEnabled { sql("SELECT a.`b.c` FROM test_db.shared_test").collect() val capturedProjection = TestServerSidePlanningClient.getCapturedProjection assert(capturedProjection.isDefined, "Projection should be pushed down") assert(capturedProjection.get == Seq("a.`b.c`"), s"Expected escaped [a.`b.c`], got ${capturedProjection.get}") } } test("projection and filter pushed together") { withPushdownCapturingEnabled { sql("SELECT id FROM test_db.shared_test WHERE value > 10").collect() val capturedProjection = TestServerSidePlanningClient.getCapturedProjection assert(capturedProjection.isDefined, "Projection should be pushed down") val projectedFields = capturedProjection.get.toSet assert(projectedFields == Set("id"), s"Expected projection with only SELECT columns {id}, " + s"got {${projectedFields.mkString(", ")}}") // Verify filter was also pushed val capturedFilter = TestServerSidePlanningClient.getCapturedFilter assert(capturedFilter.isDefined, "Filter should be pushed down") // Verify GreaterThan(value, 10) is in the filter val leafFilters = collectLeafFilters(capturedFilter.get) val gtFilter = leafFilters.collectFirst { case gt: GreaterThan if gt.attribute == "value" => gt } assert(gtFilter.isDefined, "Expected GreaterThan filter on 'value'") assert(gtFilter.get.value == 10, s"Expected GreaterThan value 10, got ${gtFilter.get.value}") } } test("projection and limit pushed together") { withPushdownCapturingEnabled { sql("SELECT id FROM test_db.shared_test LIMIT 5").collect() // Verify projection was pushed (only 'id' column) val capturedProjection = TestServerSidePlanningClient.getCapturedProjection assert(capturedProjection.isDefined, "Projection should be pushed down") val projectedFields = capturedProjection.get.toSet assert(projectedFields == Set("id"), s"Expected projection with just {id}, got {${projectedFields.mkString(", ")}}") // Verify limit was pushed val capturedLimit = TestServerSidePlanningClient.getCapturedLimit assert(capturedLimit.isDefined, "Limit should be pushed down") assert(capturedLimit.get == 5, s"Expected limit 5, got ${capturedLimit.get}") } } test("limit pushed to planning client") { withPushdownCapturingEnabled { sql("SELECT id, name, value FROM test_db.shared_test LIMIT 2").collect() val capturedLimit = TestServerSidePlanningClient.getCapturedLimit assert(capturedLimit.isDefined, "Limit should be pushed down") assert(capturedLimit.get == 2, s"Expected limit 2, got ${capturedLimit.get}") } } test("no limit pushed when no LIMIT clause") { withPushdownCapturingEnabled { sql("SELECT id, name, value FROM test_db.shared_test").collect() val capturedLimit = TestServerSidePlanningClient.getCapturedLimit assert(capturedLimit.isEmpty, "No limit should be pushed when there's no LIMIT clause") } } test("filter and limit pushed together when all filters are convertible") { withPushdownCapturingEnabled { // Query with convertible filter (GreaterThan) AND limit sql("SELECT id FROM test_db.shared_test WHERE value > 10 LIMIT 5").collect() // Verify filter was captured val capturedFilter = TestServerSidePlanningClient.getCapturedFilter assert(capturedFilter.isDefined, "Filter should be pushed to server") // Verify limit was captured (this is the key test - limit pushdown with filters) val capturedLimit = TestServerSidePlanningClient.getCapturedLimit assert(capturedLimit.isDefined, "Limit should be pushed to server when all filters are convertible") assert(capturedLimit.get == 5, s"Expected limit 5, got ${capturedLimit.get}") } } test("limit NOT pushed when any filter is unconvertible") { withPushdownCapturingEnabled { // Configure the test client to treat filters as unconvertible // This simulates a scenario where the filter cannot be converted to server's native format TestServerSidePlanningClient.setFiltersConvertible(false) try { // Query with filter AND limit sql("SELECT id FROM test_db.shared_test WHERE value > 10 LIMIT 5").collect() // Verify filter was still captured (server receives it) val capturedFilter = TestServerSidePlanningClient.getCapturedFilter assert(capturedFilter.isDefined, "Filter should still be sent to server") // Verify limit was NOT captured (because residual filters blocked pushdown) val capturedLimit = TestServerSidePlanningClient.getCapturedLimit assert(capturedLimit.isEmpty, "Limit should NOT be pushed when any filter is unconvertible") } finally { // Reset to default (convertible) for other tests TestServerSidePlanningClient.setFiltersConvertible(true) } } } test("avoid planInputPartitions call during Spark query planning") { withPushdownCapturingEnabled { sql("EXPLAIN EXTENDED SELECT id, name FROM test_db.shared_test").collect() val capturedProjection = TestServerSidePlanningClient.getCapturedProjection assert(capturedProjection.isEmpty, "Should not fire a planTable request for EXPLAIN") } } test("ServerSidePlannedTable closes planning client on close") { var clientClosed = false val trackingClient = new ServerSidePlanningClient { override def planScan( databaseName: String, table: String, filterOption: Option[Filter], projectionOption: Option[Seq[String]], limitOption: Option[Int]): ScanPlan = ScanPlan(Seq.empty) override def canConvertFilters(filters: Array[Filter]): Boolean = true override def close(): Unit = { clientClosed = true } } val table = new ServerSidePlannedTable( spark, "test_db", "test_table", new org.apache.spark.sql.types.StructType(), trackingClient) assert(!clientClosed, "Client should not be closed before table.close()") table.close() assert(clientClosed, "Client should be closed after table.close()") } test("planning client is closed after scan completes") { withPushdownCapturingEnabled { assert(!TestServerSidePlanningClient.isClientClosed, "Client should not be closed before query execution") sql("SELECT id FROM test_db.shared_test").collect() assert(TestServerSidePlanningClient.isClientClosed, "Planning client should be closed after scan plan is obtained") } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningClientFactorySuite.scala ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.serverSidePlanning import org.apache.spark.sql.QueryTest import org.apache.spark.sql.test.SharedSparkSession /** * Unit tests for ServerSidePlanningClientFactory core functionality. * Tests manual factory registration, state management, and lifecycle. */ class ServerSidePlanningClientFactorySuite extends QueryTest with SharedSparkSession { override def afterEach(): Unit = { try { ServerSidePlanningClientFactory.clearFactory() } finally { super.afterEach() } } // ========== Test Infrastructure ========== /** * Execute test block with clean factory state (setup + teardown). * Ensures factory is cleared before and after the test. */ private def withCleanFactory[T](testFn: => T): T = { try { ServerSidePlanningClientFactory.clearFactory() testFn } finally { ServerSidePlanningClientFactory.clearFactory() } } /** * Assert that getFactory() throws IllegalStateException indicating no factory is registered. */ private def assertNoFactory(context: String = ""): Unit = { val prefix = if (context.nonEmpty) s"[$context] " else "" val exception = intercept[IllegalStateException] { ServerSidePlanningClientFactory.getFactory() } assert( exception.getMessage.contains("No ServerSidePlanningClientFactory has been registered"), s"${prefix}Expected 'No ServerSidePlanningClientFactory' message, " + s"got: ${exception.getMessage}") } // ========== Tests ========== test("manual setFactory can replace existing factory") { withCleanFactory { // Set first factory val firstFactory = new TestServerSidePlanningClientFactory() ServerSidePlanningClientFactory.setFactory(firstFactory) assert(ServerSidePlanningClientFactory.getFactory() eq firstFactory, "Should return first factory") // Replace with second factory val secondFactory = new TestServerSidePlanningClientFactory() ServerSidePlanningClientFactory.setFactory(secondFactory) // Verify replacement val retrievedFactory = ServerSidePlanningClientFactory.getFactory() assert(retrievedFactory eq secondFactory, "getFactory() should return the second factory after replacement") assert(!(retrievedFactory eq firstFactory), "Should not return the first factory") } } test("getFactory returns same instance across multiple calls") { withCleanFactory { val testFactory = new TestServerSidePlanningClientFactory() ServerSidePlanningClientFactory.setFactory(testFactory) val factory1 = ServerSidePlanningClientFactory.getFactory() val factory2 = ServerSidePlanningClientFactory.getFactory() val factory3 = ServerSidePlanningClientFactory.getFactory() assert(factory1 eq factory2, "Second call should return same instance as first") assert(factory2 eq factory3, "Third call should return same instance as second") assert(factory1 eq testFactory, "Should return the originally set factory") } } test("clearFactory resets registration state") { withCleanFactory { val testFactory = new TestServerSidePlanningClientFactory() ServerSidePlanningClientFactory.setFactory(testFactory) // Verify factory is registered by successfully retrieving it assert(ServerSidePlanningClientFactory.getFactory() eq testFactory, "Factory should be registered and retrievable") // Clear factory ServerSidePlanningClientFactory.clearFactory() // Verify factory is no longer registered assertNoFactory("after clearFactory") } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/TestServerSidePlanningClient.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.serverSidePlanning import org.apache.hadoop.fs.Path import org.apache.spark.sql.SparkSession import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.functions.input_file_name import org.apache.spark.sql.sources.Filter /** * Implementation of ServerSidePlanningClient that uses Spark SQL with input_file_name() * to discover the list of files in a table. This allows end-to-end testing without * a real server that can do server-side planning. * * Also captures filter/projection parameters for test verification via companion object. */ class TestServerSidePlanningClient(spark: SparkSession) extends ServerSidePlanningClient { override def planScan( databaseName: String, table: String, filterOption: Option[Filter] = None, projectionOption: Option[Seq[String]] = None, limitOption: Option[Int] = None): ScanPlan = { // Capture filter, projection, and limit for test verification TestServerSidePlanningClient.capturedFilter = filterOption TestServerSidePlanningClient.capturedProjection = projectionOption TestServerSidePlanningClient.capturedLimit = limitOption val fullTableName = s"$databaseName.$table" // Temporarily disable server-side planning to avoid infinite recursion // when this test client internally loads the table val originalConfigValue = spark.conf.getOption(DeltaSQLConf.ENABLE_SERVER_SIDE_PLANNING.key) spark.conf.set(DeltaSQLConf.ENABLE_SERVER_SIDE_PLANNING.key, "false") try { // Use input_file_name() to get the list of files // Query: SELECT DISTINCT input_file_name() FROM table val filesDF = spark.table(fullTableName) .select(input_file_name().as("file_path")) .distinct() // Collect file paths val filePaths = filesDF.collect().map(_.getString(0)) // Get file metadata (size, format) from filesystem // scalastyle:off deltahadoopconfiguration // The rule prevents accessing Hadoop conf on executors where it could use wrong credentials // for multi-catalog scenarios. Safe here: test-only code simulating server filesystem access. val hadoopConf = spark.sessionState.newHadoopConf() // scalastyle:on deltahadoopconfiguration val files = filePaths.map { filePath => // input_file_name() returns URL-encoded paths, decode them val decodedPath = java.net.URLDecoder.decode(filePath, "UTF-8") val path = new Path(decodedPath) val fs = path.getFileSystem(hadoopConf) val fileStatus = fs.getFileStatus(path) ScanFile( filePath = decodedPath, fileSizeInBytes = fileStatus.getLen, fileFormat = getFileFormat(path) ) }.toSeq ScanPlan(files = files) } finally { // Restore original config value originalConfigValue match { case Some(value) => spark.conf.set(DeltaSQLConf.ENABLE_SERVER_SIDE_PLANNING.key, value) case None => spark.conf.unset(DeltaSQLConf.ENABLE_SERVER_SIDE_PLANNING.key) } } } override def canConvertFilters(filters: Array[Filter]): Boolean = { // For testing: check if filters should be treated as convertible // Tests can configure this via TestServerSidePlanningClient.setFiltersConvertible() TestServerSidePlanningClient.filtersConvertible } override def close(): Unit = { TestServerSidePlanningClient.clientClosed = true } private def getFileFormat(path: Path): String = "parquet" } /** * Companion object for TestServerSidePlanningClient. * Stores captured pushdown parameters (filter, projection, limit) for test verification. */ object TestServerSidePlanningClient { private var capturedFilter: Option[Filter] = None private var capturedProjection: Option[Seq[String]] = None private var capturedLimit: Option[Int] = None private var filtersConvertible: Boolean = true // Default: all filters convertible private[serverSidePlanning] var clientClosed: Boolean = false def getCapturedFilter: Option[Filter] = capturedFilter def getCapturedProjection: Option[Seq[String]] = capturedProjection def getCapturedLimit: Option[Int] = capturedLimit def isClientClosed: Boolean = clientClosed /** * Configure whether filters should be treated as convertible. * Used for testing filter conversion failure scenarios. */ def setFiltersConvertible(convertible: Boolean): Unit = { filtersConvertible = convertible } def clearCaptured(): Unit = { capturedFilter = None capturedProjection = None capturedLimit = None filtersConvertible = true // Reset to default clientClosed = false } } /** * Factory for creating TestServerSidePlanningClient instances. */ class TestServerSidePlanningClientFactory extends ServerSidePlanningClientFactory { override def buildClient( spark: SparkSession, metadata: ServerSidePlanningMetadata): ServerSidePlanningClient = { new TestServerSidePlanningClient(spark) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/skipping/ClusteredTableTestUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.skipping import org.apache.spark.sql.delta.skipping.clustering.{ClusteredTableUtils, ClusteringColumn, ClusteringColumnInfo} import org.apache.spark.sql.delta.skipping.clustering.temp.ClusterBySpec import org.apache.spark.sql.delta.{DeltaLog, Snapshot} import org.apache.spark.sql.delta.DeltaOperations import org.apache.spark.sql.delta.DeltaOperations.{CLUSTERING_PARAMETER_KEY, ZORDER_PARAMETER_KEY} import org.apache.spark.sql.delta.commands.optimize.OptimizeMetrics import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CatalogOwnedTestBaseSuite, CoordinatedCommitsBaseSuite} import org.apache.spark.sql.delta.hooks.UpdateCatalog import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.JsonUtils import org.junit.Assert.assertEquals import org.apache.spark.SparkFunSuite import org.apache.spark.sql.DataFrame import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.Utils trait ClusteredTableTestUtilsBase extends SparkFunSuite with SharedSparkSession with CatalogOwnedTestBaseSuite { import testImplicits._ /** * Helper for running optimize on the table with different APIs. * @param table the name of table */ def optimizeTable(table: String): DataFrame = { sql(s"OPTIMIZE $table") } /** * Runs optimize on the table and calls postHook on the metrics. * @param table the name of table * @param postHook callback triggered with OptimizeMetrics returned by the OPTIMIZE command */ def runOptimize(table: String)(postHook: OptimizeMetrics => Unit): Unit = { // Verify Delta history operation parameters' clusterBy val isPathBasedTable = table.startsWith("tahoe.") || table.startsWith("delta.") var (deltaLog, snapshot) = if (isPathBasedTable) { // Path based table e.g. delta.`path-to-directory` or tahoe.`path-to-directory`. Strip // 6 characters to extract table path. DeltaLog.forTableWithSnapshot(spark, table.drop(6).replace("`", "")) } else { DeltaLog.forTableWithSnapshot(spark, TableIdentifier(table)) } val beforeVersion = snapshot.version postHook(optimizeTable(table).select($"metrics.*").as[OptimizeMetrics].head()) snapshot = deltaLog.update() val afterVersion = snapshot.version val shouldCheckFullStatus = deltaLog.history.getHistory(Some(1)).headOption.exists { h => Seq(DeltaOperations.OPTIMIZE_OPERATION_NAME ).contains(h.operation) } // Note: Only expect isFull status when the table has non-empty clustering columns and // clustering table feature, otherwise the OPTIMIZE will fall back to compaction and // isFull status will not be relevant anymore. val expectedOperationParameters = ClusteredTableUtils .getClusteringColumnsOptional(snapshot) .filter { cols => cols.nonEmpty && shouldCheckFullStatus && ClusteredTableUtils.isSupported(snapshot.protocol) && afterVersion > beforeVersion } .map(_ => Map(DeltaOperations.CLUSTERING_IS_FULL_KEY -> false)) .getOrElse(Map.empty) verifyDescribeHistoryOperationParameters( table, expectedOperationParameters = expectedOperationParameters) } /** * Runs optimize full on the table and calls postHook on the metrics. * * @param table the name of table * @param postHook callback triggered with OptimizeMetrics returned by the OPTIMIZE command */ def runOptimizeFull(table: String)(postHook: OptimizeMetrics => Unit): Unit = { postHook(sql(s"OPTIMIZE $table FULL").select($"metrics.*").as[OptimizeMetrics].head()) // Verify Delta history operation parameters' clusterBy verifyDescribeHistoryOperationParameters(table, expectedOperationParameters = Map( DeltaOperations.CLUSTERING_IS_FULL_KEY -> true)) } def verifyClusteringColumnsInDomainMetadata( snapshot: Snapshot, logicalColumnNames: Seq[String]): Unit = { val expectedClusteringColumns = logicalColumnNames.map(ClusteringColumn(snapshot.schema, _)) val actualClusteringColumns = ClusteredTableUtils.getClusteringColumnsOptional(snapshot).orNull assert(expectedClusteringColumns == actualClusteringColumns) } // Verify the operation parameters of the last history event contains `clusterBy`. protected def verifyDescribeHistoryOperationParameters( table: String, expectedOperationParameters: Map[String, Any] = Map.empty): Unit = { val clusterBySupportedOperations = Set( "CREATE TABLE", "REPLACE TABLE", "CREATE OR REPLACE TABLE", "CREATE TABLE AS SELECT", "REPLACE TABLE AS SELECT", "CREATE OR REPLACE TABLE AS SELECT") val isPathBasedTable = table.startsWith("tahoe.") || table.startsWith("delta.") val (deltaLog, snapshot) = if (isPathBasedTable) { // Path based table. DeltaLog.forTableWithSnapshot(spark, table.drop(6).replace("`", "")) } else { DeltaLog.forTableWithSnapshot(spark, TableIdentifier(table)) } val isClusteredTable = ClusteredTableUtils.isSupported(snapshot.protocol) val clusteringColumns = ClusteringColumnInfo.extractLogicalNames(snapshot) val expectedClusterBy = JsonUtils.toJson(clusteringColumns) val expectClustering = isClusteredTable && clusteringColumns.nonEmpty val lastEvent = deltaLog.history.getHistory(Some(1)).head val lastOperationParameters = lastEvent.operationParameters def doAssert(assertion: => Boolean): Unit = { val debugMsg = "verifyDescribeHistoryOperationParameters DEBUG: " + "assert failed. Please check the expected behavior and " + "add the operation to the appropriate case in " + "verifyDescribeHistoryOperationParameters. " + s"table: $table, lastOperation: ${lastEvent.operation} " + s"lastOperationParameters: $lastOperationParameters " + s"expectedOperationParameters: $expectedOperationParameters" try { assert(assertion, debugMsg) } catch { case e: Throwable => throw new Throwable(debugMsg, e) } } // Check clusterBy exists and matches the expected clusterBy. def assertClusterByExists(): Unit = { doAssert(lastOperationParameters(CLUSTERING_PARAMETER_KEY) === expectedClusterBy) } // Check clusterBy does not exist or is empty. def assertClusterByEmptyOrNotExists(): Unit = { doAssert(!lastOperationParameters.contains(CLUSTERING_PARAMETER_KEY) || lastOperationParameters(CLUSTERING_PARAMETER_KEY) === "[]") } // Check clusterBy does not exist. def assertClusterByNotExist(): Unit = { doAssert(!lastOperationParameters.contains(CLUSTERING_PARAMETER_KEY)) } // Validate caller provided operator parameters from the last commit. for ((operationParameterKey, value) <- expectedOperationParameters) { // Convert value to string since value is stored as toString in operationParameters. doAssert(lastOperationParameters(operationParameterKey) === value.toString) } // Check clusterBy lastEvent.operation match { case "CLUSTER BY" => // Operation is [[DeltaOperations.ClusterBy]] - ALTER TABLE CLUSTER BY doAssert( lastOperationParameters("newClusteringColumns") === clusteringColumns.mkString(",") ) case "OPTIMIZE" => if (expectClustering) { doAssert(lastOperationParameters(CLUSTERING_PARAMETER_KEY) === expectedClusterBy) doAssert(lastOperationParameters(ZORDER_PARAMETER_KEY) === "[]") } else { // If the table clusters by NONE, OPTIMIZE will be a regular compaction. // In this case, both clustering and z-order parameters should be empty. doAssert(lastOperationParameters(CLUSTERING_PARAMETER_KEY) === "[]") doAssert(lastOperationParameters(ZORDER_PARAMETER_KEY) === "[]") } case "CLONE" => // CLUSTER BY not in operation parameters for CLONE - similar to PARTITION BY. doAssert(!lastOperationParameters.contains(CLUSTERING_PARAMETER_KEY)) case o if clusterBySupportedOperations.contains(o) => if (expectClustering) { assertClusterByExists() } else if (isClusteredTable && clusteringColumns.isEmpty) { assertClusterByEmptyOrNotExists() } else { assertClusterByNotExist() } case "WRITE" | "RESTORE" => // These are known operations from our tests that don't have clusterBy. doAssert(!lastOperationParameters.contains(CLUSTERING_PARAMETER_KEY)) case _ => // Other operations are not tested yet. If the test fails here, please check the expected // behavior and add the operation to the appropriate case. doAssert(false) } } protected def deleteTableFromCommitCoordinatorIfNeeded(table: String): Unit = { // Clean up the table data in commit coordinator because DROP/REPLACE TABLE does not bother // commit coordinator. if (CatalogOwnedTableUtils.defaultCatalogOwnedEnabled(spark) && catalogOwnedDefaultCreationEnabledInTests) { deleteCatalogOwnedTableFromCommitCoordinator(table) } } override def withTable(tableNames: String*)(f: => Unit): Unit = { Utils.tryWithSafeFinally(f) { tableNames.foreach { name => deleteTableFromCommitCoordinatorIfNeeded(name) spark.sql(s"DROP TABLE IF EXISTS $name") } } } def withClusteredTable[T]( table: String, schema: String, clusterBy: String, tableProperties: Map[String, String] = Map.empty, location: Option[String] = None)(f: => T): T = { createOrReplaceClusteredTable("CREATE", table, schema, clusterBy, tableProperties, location) Utils.tryWithSafeFinally(f) { deleteTableFromCommitCoordinatorIfNeeded(table) spark.sql(s"DROP TABLE IF EXISTS $table") } } /** * Helper for creating or replacing table with different APIs. * @param clause clause for SQL API ('CREATE', 'REPLACE', 'CREATE OR REPLACE') * @param table the name of table * @param schema comma separated list of "colName dataType" * @param clusterBy comma separated list of clustering columns */ def createOrReplaceClusteredTable( clause: String, table: String, schema: String, clusterBy: String, tableProperties: Map[String, String] = Map.empty, location: Option[String] = None): Unit = { val locationClause = if (location.isEmpty) "" else s"LOCATION '${location.get}'" val tablePropertiesClause = if (!tableProperties.isEmpty) { val tablePropertiesString = tableProperties.map { case (key, value) => s"'$key' = '$value'" }.mkString(",") s"TBLPROPERTIES($tablePropertiesString)" } else { "" } spark.sql(s"$clause TABLE $table ($schema) USING delta CLUSTER BY ($clusterBy) " + s"$tablePropertiesClause $locationClause") location.foreach { loc => locRefCount(loc) = locRefCount.getOrElse(loc, 0) + 1 } } protected def createOrReplaceAsSelectClusteredTable( clause: String, table: String, srcTable: String, clusterBy: String, location: Option[String] = None): Unit = { val locationClause = if (location.isEmpty) "" else s"LOCATION '${location.get}'" spark.sql(s"$clause TABLE $table USING delta CLUSTER BY ($clusterBy) " + s"$locationClause AS SELECT * FROM $srcTable") } def verifyClusteringColumns( tableIdentifier: TableIdentifier, expectedLogicalClusteringColumns: Seq[String], skipCatalogCheck: Boolean = false ): Unit = { val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, tableIdentifier) verifyClusteringColumnsInternal( snapshot, tableIdentifier.table, expectedLogicalClusteringColumns ) if (skipCatalogCheck) { return } val updateCatalogEnabled = spark.conf.get(DeltaSQLConf.DELTA_UPDATE_CATALOG_ENABLED) assert(updateCatalogEnabled, "need to enable [[DeltaSQLConf.DELTA_UPDATE_CATALOG_ENABLED]] to verify catalog updates.") UpdateCatalog.awaitCompletion(10000) val catalog = spark.sessionState.catalog catalog.refreshTable(tableIdentifier) val table = catalog.getTableMetadata(tableIdentifier) // Verify CatalogTable's clusterBySpec. assert(ClusteredTableUtils.getClusterBySpecOptional(table).isDefined) assertEquals(ClusterBySpec.fromColumnNames(expectedLogicalClusteringColumns), ClusteredTableUtils.getClusterBySpecOptional(table).get) } def verifyClusteringColumns( dataPath: String, expectedLogicalClusteringColumns: Seq[String]): Unit = { val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, dataPath) verifyClusteringColumnsInternal( snapshot, s"delta.`$dataPath`", expectedLogicalClusteringColumns ) } def verifyClusteringColumnsInternal( snapshot: Snapshot, tableNameOrPath: String, expectedLogicalClusteringColumns: Seq[String] ): Unit = { assert(ClusteredTableUtils.isSupported(snapshot.protocol) === true) verifyClusteringColumnsInDomainMetadata(snapshot, expectedLogicalClusteringColumns) // Verify Delta history operation parameters' clusterBy verifyDescribeHistoryOperationParameters( tableNameOrPath ) // Verify DESCRIBE DETAIL's properties doesn't contain the "clusteringColumns" key. val describeDetailProps = sql(s"describe detail $tableNameOrPath") .select("properties") .first .getAs[Map[String, String]](0) assert(!describeDetailProps.contains(ClusteredTableUtils.PROP_CLUSTERING_COLUMNS)) // Verify SHOW TBLPROPERTIES contains the correct clustering columns. val clusteringColumnsVal = sql(s"show tblproperties $tableNameOrPath") .filter($"key" === ClusteredTableUtils.PROP_CLUSTERING_COLUMNS) .select("value") .first .getString(0) val clusterBySpec = ClusterBySpec.fromProperties( Map(ClusteredTableUtils.PROP_CLUSTERING_COLUMNS -> clusteringColumnsVal)).get assert(expectedLogicalClusteringColumns === clusterBySpec.columnNames.map(_.toString)) } } trait ClusteredTableTestUtils extends ClusteredTableTestUtilsBase ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/skipping/MultiDimClusteringFunctionsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.skipping import java.nio.ByteBuffer import scala.util.Random import org.apache.spark.sql.delta.expressions.{HilbertByteArrayIndex, HilbertLongIndex} import org.apache.spark.sql.delta.skipping.MultiDimClusteringFunctions._ import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.SparkException import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.expressions.Cast import org.apache.spark.sql.functions.lit import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession /** Tests for [[MultiDimClusterFunctions]] */ class MultiDimClusteringFunctionsSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { import testImplicits._ test("range_partition_id(): simple") { val numTuples = 20 val data = 0.to(numTuples - 1) for { div <- Seq(1, 2, 4, 5, 10, 20) } { checkAnswer( Random.shuffle(data).toDF("col") .withColumn("rpi", range_partition_id($"col", data.size / div)), data.map(i => Row(i, i / div)) ) } } test("range_partition_id(): two columns") { val data = Seq("a" -> 10, "b" -> 20, "c" -> 30, "d" -> 40) checkAnswer( // randomize the order and expect the partition ids assigned correctly in sorted order Random.shuffle(data).toDF("c1", "c2") .withColumn("r1", range_partition_id($"c1", 2)) .withColumn("r2", range_partition_id($"c2", 4)), Seq( // Column c1 has values (a, b, c, d). Splitting this value range into two partitions // gets ranges [a, b] and [c, d]. Values in each range map to partition 0 and 1. // Similarly column c2 has values (10, 20, 30, 40). Splitting this into four partitions // gets ranges [10], [20], [30] and [40] which map to partition ids 0 to 3. Row("a", 10, 0, 0), Row("b", 20, 0, 1), Row("c", 30, 1, 2), Row("d", 40, 1, 3))) checkAnswer( Random.shuffle(data).toDF("c1", "c2") .withColumn("r1", range_partition_id($"c1", 2)) .distinct .withColumn("r2", range_partition_id($"c2", 4)), Seq( Row("a", 10, 0, 0), Row("b", 20, 0, 1), Row("c", 30, 1, 2), Row("d", 40, 1, 3))) checkAnswer( Random.shuffle(data).toDF("c1", "c2") .where(range_partition_id($"c1", 2) === 0) .sort(range_partition_id($"c2", 4)), Seq( Row("a", 10), Row("b", 20))) } testQuietly("range_partition_id(): corner cases") { // invalid number of partitions val ex1 = intercept[IllegalArgumentException] { spark.range(10).select(range_partition_id($"id", 0)).show } assert(ex1.getMessage contains "expected the number partitions to be greater than zero") val ex2 = intercept[IllegalArgumentException] { withSQLConf(SQLConf.RANGE_EXCHANGE_SAMPLE_SIZE_PER_PARTITION.key -> "0") { spark.range(10).withColumn("rpi", range_partition_id($"id", 10)).show } } assert(ex2.getMessage contains "Sample points per partition must be greater than 0 but found 0") // Number of partitions is way more than the cardinality of input column values checkAnswer( spark.range(1).withColumn("rpi", range_partition_id($"id", 1000)), Row(0, 0)) // compute range_partition_id on a dataframe with zero rows checkAnswer( spark.range(0).withColumn("rpi", range_partition_id($"id", 1000)), Seq.empty[Row]) // compute range_partition_id on column with null values checkAnswer( Seq("a", null, "b", null).toDF("id").withColumn("rpi", range_partition_id($"id", 10)), Seq( Row("a", 0), Row("b", 1), Row(null, 0), Row(null, 0))) // compute range_partition_id on column with one value which is null checkAnswer( spark.range(1).withColumn("id", lit(null)).withColumn("rpi", range_partition_id($"id", 10)), Row(null, 0)) // compute range_partition_id on array type column checkAnswer( spark.range(1).withColumn("id", lit(Array(1, 2))) .withColumn("rpi", range_partition_id($"id", 10)), Row(Array(1, 2), 0)) } test("interleave_bits(): 1 input = cast to binary") { val data = Seq.fill(100)(Random.nextInt()) checkAnswer( data.toDF("id").select(interleave_bits($"id")), data.map(i => Row(intToBinary(i))) ) } test(s"interleave_bits(): arbitrary num inputs") { val n = 1 + Random.nextInt(7) val zDF = spark.range(1).select() // Output is an array with number of elements equal to 4 * num_of_input_columns to interleave // Multiple columns each has value 0. Expect the final output an array of zeros checkAnswer( 1.to(n).foldLeft(zDF)((df, i) => df.withColumn(s"c$i", lit(0x00000000))) .select(interleave_bits(1.to(n).map(i => $"c$i"): _*)), Row(Array.fill(n * 4)(0x00.toByte)) ) // Multiple column each has value 1. As the bits are interleaved expect the following output // Inputs: c1=0x00000001, c2=0x00000001, c3=0x00000001, c4=0x00000001 // Output (divided into array of 4 bytes for readability) // [0x00, 0x00, 0x00, 0x00] [0x00, 0x00, 0x00, 0x00] // [0x00, 0x00, 0x00, 0x00] [0x00, 0x00, 0x00, 0x08] // (Inputs have last bit as 1 as we are interleaving bits across columns, all these // bits of value 1 they will end up as last 4 bits in the last byte of the output) checkAnswer( 1.to(n).foldLeft(zDF)((df, i) => df.withColumn(s"c$i", lit(0x00000001))) .select(interleave_bits(1.to(n).map(i => $"c$i"): _*)), Row(Array.fill(n * 4 - 1)(0x00.toByte) :+ ((1 << n) - 1).toByte) ) // Multiple columns each has value 0xFFFFFFFF. Expect the final output an array of 0xFF checkAnswer( 1.to(n).foldLeft(zDF)((df, i) => df.withColumn(s"c$i", lit(0xffffffff))) .select(interleave_bits(1.to(n).map(i => $"c$i"): _*)), Row(Array.fill(n * 4)(0xff.toByte)) ) } test("interleave_bits(): corner cases") { // null input checkAnswer( spark.range(1).select(interleave_bits(lit(null))), Row(Array.fill(4)(0x00.toByte)) ) // no inputs to interleave_bits -> expect an empty row checkAnswer( spark.range(1).select(interleave_bits()), Row(Array.empty[Byte]) ) // Non-integer type as input column val ex = intercept[AnalysisException] { Seq(false).toDF("col").select(interleave_bits($"col")).show } assert(ex.getMessage contains "") def invalidColumnTypeInput(df: DataFrame): Unit = { val ex = intercept[AnalysisException] { df.select(interleave_bits($"col")).show } assert(ex.getMessage contains "") } // Expect failure when a non-int type column is provided as input invalidColumnTypeInput(Seq(0L).toDF("col")) invalidColumnTypeInput(Seq(0.0).toDF("col")) invalidColumnTypeInput(Seq("asd").toDF("col")) invalidColumnTypeInput(Seq(Array(1, 2, 3)).toDF("col")) } test("interleave_bits(range_partition_ids)") { // test the combination of range_partition_id and interleave checkAnswer( spark.range(100).select(interleave_bits(range_partition_id($"id", 10))), 0.until(100).map(i => Row(intToBinary(i / 10))) ) // test the combination of range_partition_id and interleave on multiple columns checkAnswer( Seq( (false, 0, "0"), (true, 1, "1") ).toDF("c1", "c2", "c3") .select(interleave_bits( range_partition_id($"c1", 2), range_partition_id($"c2", 2), range_partition_id($"c3", 2) )), Seq( Row(Array.fill(3 * 4)(0x00.toByte)), Row(Array.fill(3 * 4 - 1)(0x00.toByte) :+ 0x07.toByte) ) ) } test("hilbert_index selects underlying expression correctly") { assert(hilbert_index(10, Seq($"c1", $"c2", $"c3", $"c4", $"c5", $"c6"): _*).expr .isInstanceOf[HilbertLongIndex]) assert( hilbert_index( 10, Seq($"c1", $"c2", $"c3", $"c4", $"c5", $"c6", $"c7", $"c8", $"c9"): _*) .expr.asInstanceOf[Cast].child.isInstanceOf[HilbertByteArrayIndex]) val e = intercept[SparkException]( hilbert_index( 11, Seq($"c1", $"c2", $"c3", $"c4", $"c5", $"c6", $"c7", $"c8", $"c9", $"c10"): _*) .expr.isInstanceOf[HilbertByteArrayIndex]) assert(e.getMessage.contains("Hilbert indexing can only be used on 9 or fewer columns.")) } private def intToBinary(x: Int): Array[Byte] = { ByteBuffer.allocate(4).putInt(x).array() } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/skipping/MultiDimClusteringSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.skipping import java.io.{File, FilenameFilter} import scala.util.Random // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.sources.DeltaSQLConf._ import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.QueryTest import org.apache.spark.sql.functions.expr import org.apache.spark.sql.test.SharedSparkSession class MultiDimClusteringSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { private lazy val sparkSession = spark // scalastyle:off sparkimplicits import sparkSession.implicits._ // scalastyle:on sparkimplicits test("Negative case - ZOrder clustering expression with zero columns") { val ex = intercept[AssertionError] { ZOrderClustering.getClusteringExpression(Seq.empty, 20) } assert(ex.getMessage contains "Cannot do Z-Order clustering by zero columns!") } test("ZOrder clustering expression with one column") { val cluster = ZOrderClustering.getClusteringExpression(Seq(expr("col1")), 20) assert(cluster.expr.toString === "cast(interleavebits(rangepartitionid('col1, 20)) as string)") } test("ZOrder clustering expression with two column") { val cluster = ZOrderClustering.getClusteringExpression(Seq(expr("col1"), expr("col2")), 20) assert(cluster.expr.toString === "cast(interleavebits(rangepartitionid('col1, 20), rangepartitionid('col2, 20)) as string)") } test("ensure records with close Z-order values are close in the output") { withTempDir { tempDir => withSQLConf( MDC_NUM_RANGE_IDS.key -> "4", MDC_ADD_NOISE.key -> "false") { val data = Seq( // "c1" -> "c2", // (rangeId_c1, rangeId_c2) -> ZOrder (decimal Z-Order) "a" -> 20, "a" -> 20, // (0, 0) -> 0b000000 (0) "b" -> 20, // (0, 0) -> 0b000000 (0) "c" -> 30, // (1, 1) -> 0b000011 (3) "d" -> 70, // (1, 2) -> 0b001011 (3) "e" -> 90, "e" -> 90, "e" -> 90, // (1, 2) -> 0b001001 (9) "f" -> 200, // (2, 3) -> 0b001110 (14) "g" -> 10, // (3, 0) -> 0b000101 (5) "h" -> 20) // (3, 0) -> 0b000101 (5) // Randomize the data. Use seed for deterministic input. val inputDf = new Random(seed = 101).shuffle(data) .toDF("c1", "c2") // Cluster the data and range partition into four partitions val outputDf = MultiDimClustering.cluster( inputDf, approxNumPartitions = 4, colNames = Seq("c1", "c2"), curve = "zorder") outputDf.write.parquet(new File(tempDir, "source").getCanonicalPath) // Load the partition 0 and verify that it contains (a, 20), (a, 20), (b, 20) checkAnswer( Seq("a" -> 20, "a" -> 20, "b" -> 20).toDF("c1", "c2"), sparkSession.read.parquet(new File(tempDir, "source/part-00000*").getCanonicalPath)) // partition 1 checkAnswer( Seq("c" -> 30, "d" -> 70, "e" -> 90, "e" -> 90, "e" -> 90).toDF("c1", "c2"), sparkSession.read.parquet(new File(tempDir, "source/part-00001*").getCanonicalPath)) // partition 2 checkAnswer( Seq("h" -> 20, "g" -> 10).toDF("c1", "c2"), sparkSession.read.parquet(new File(tempDir, "source/part-00002*").getCanonicalPath)) // partition 3 checkAnswer( Seq("f" -> 200).toDF("c1", "c2"), sparkSession.read.parquet(new File(tempDir, "source/part-00003*").getCanonicalPath)) } } } test("ensure records with close Hilbert curve values are close in the output") { withTempDir { tempDir => withSQLConf(MDC_NUM_RANGE_IDS.key -> "4", MDC_ADD_NOISE.key -> "false") { val data = Seq( // "c1" -> "c2", // (rangeId_c1, rangeId_c2) -> Decimal Hilbert index "a" -> 20, "a" -> 20, // (0, 0) -> 0 "b" -> 20, // (0, 0) -> 0 "c" -> 30, // (1, 1) -> 2 "d" -> 70, // (1, 2) -> 13 "e" -> 90, "e" -> 90, "e" -> 90, // (1, 2) -> 13 "f" -> 200, // (2, 3) -> 11 "g" -> 10, // (3, 0) -> 5 "h" -> 20) // (3, 0) -> 5 // Randomize the data. Use seed for deterministic input. val inputDf = new Random(seed = 101) .shuffle(data) .toDF("c1", "c2") // Cluster the data and range partition into four partitions val outputDf = MultiDimClustering.cluster( inputDf, approxNumPartitions = 2, colNames = Seq("c1", "c2"), curve = "hilbert") outputDf.write.parquet(new File(tempDir, "source").getCanonicalPath) // Load the partition 0 and verify its records. checkAnswer( Seq("a" -> 20, "a" -> 20, "b" -> 20, "c" -> 30, "g" -> 10, "h" -> 20).toDF("c1", "c2"), sparkSession.read.parquet(new File(tempDir, "source/part-00000*").getCanonicalPath) ) // partition 1 checkAnswer( Seq("d" -> 70, "e" -> 90, "e" -> 90, "e" -> 90, "f" -> 200).toDF("c1", "c2"), sparkSession.read.parquet(new File(tempDir, "source/part-00001*").getCanonicalPath) ) } } } test("ensure records in each partition are sorted according to Z-order values") { withSQLConf( MDC_SORT_WITHIN_FILES.key -> "true", MDC_ADD_NOISE.key -> "false") { val data = Seq( // "c1" -> "c2", // (rangeId_c1, rangeId_c2) -> ZOrder (decimal Z-Order) "a" -> 20, "a" -> 20, // (0, 1) -> 0x01 (1) "b" -> 20, // (1, 1) -> 0x03 (3) "c" -> 30, // (2, 2) -> 0x0C (12) "d" -> 70, // (3, 3) -> 0x0F (15) "e" -> 90, "e" -> 90, "e" -> 90, // (4, 4) -> 0x30 (48) "f" -> 200, // (5, 5) -> 0x33 (51) "g" -> 10, // (6, 0) -> 0x28 (40) "h" -> 20) // (7, 1) -> 0x2B (43) // Randomize the data. Use seed for deterministic input. val inputDf = new Random(seed = 101).shuffle(data) .toDF("c1", "c2") // Cluster the data, range partition into one partition, and sort. val outputDf = MultiDimClustering.cluster( inputDf, approxNumPartitions = 1, colNames = Seq("c1", "c2"), curve = "zorder") // Check that dataframe is sorted. checkAnswer( outputDf, Seq( "a" -> 20, "a" -> 20, "b" -> 20, "c" -> 30, "d" -> 70, "g" -> 10, "h" -> 20, "e" -> 90, "e" -> 90, "e" -> 90, "f" -> 200 ).toDF("c1", "c2").collect()) } } test("ensure records in each partition are sorted according to Hilbert curve values") { withSQLConf( MDC_SORT_WITHIN_FILES.key -> "true", MDC_ADD_NOISE.key -> "false") { val data = Seq( // "c1" -> "c2", // (rangeId_c1, rangeId_c2) -> Decimal Hilbert index "a" -> 20, "a" -> 20, // (0, 1) -> 3 "b" -> 20, // (1, 1) -> 2 "c" -> 30, // (2, 2) -> 8 "d" -> 70, // (3, 3) -> 10 "e" -> 90, "e" -> 90, "e" -> 90, // (4, 4) -> 32 "f" -> 200, // (5, 5) -> 34 "g" -> 10, // (6, 0) -> 20 "h" -> 20) // (7, 1) -> 22 // Randomize the data. Use seed for deterministic input. val inputDf = new Random(seed = 101).shuffle(data) .toDF("c1", "c2") // Cluster the data, range partition into one partition, and sort. val outputDf = MultiDimClustering.cluster( inputDf, approxNumPartitions = 1, colNames = Seq("c1", "c2"), curve = "hilbert") // Check that dataframe is sorted. checkAnswer( outputDf, Seq( "b" -> 20, "a" -> 20, "a" -> 20, "c" -> 30, "d" -> 70, "g" -> 10, "h" -> 20, "e" -> 90, "e" -> 90, "e" -> 90, "f" -> 200 ).toDF("c1", "c2").collect()) } } test("noise is helpful in skew handling") { Seq("zorder", "hilbert").foreach { curve => Seq("true", "false").foreach { addNoise => withTempDir { tempDir => withSQLConf( MDC_NUM_RANGE_IDS.key -> "4", MDC_ADD_NOISE.key -> addNoise) { val data = Array.fill(100)(20, 20) // all records have the same values val inputDf = data.toSeq.toDF("c1", "c2") // Cluster the data and range partition into four partitions val outputDf = MultiDimClustering.cluster( inputDf, approxNumPartitions = 4, colNames = Seq("c1", "c2"), curve) outputDf.write.parquet(new File(tempDir, "source").getCanonicalPath) // If there is no noise added, expect only one partition, otherwise four partition // as mentioned in the cluster command above. val partCount = new File(tempDir, "source").listFiles(new FilenameFilter { override def accept(dir: File, name: String): Boolean = { name.startsWith("part-0000") } }).length if ("true".equals(addNoise)) { assert(4 === partCount, s"Incorrect number of partitions when addNoise=$addNoise") } else { assert(1 === partCount, s"Incorrect number of partitions when addNoise=$addNoise") } } } } } } test(s"try clustering with different ranges and noise flag on/off") { Seq("zorder", "hilbert").foreach { curve => Seq("true", "false").foreach { addNoise => Seq("20", "100", "200", "1000").foreach { numRanges => withSQLConf(MDC_NUM_RANGE_IDS.key -> numRanges, MDC_ADD_NOISE.key -> addNoise) { val data = Seq.range(0, 100) val inputDf = Random.shuffle(data).map(x => (x, x * 113 % 101)).toDF("col1", "col2") val outputDf = MultiDimClustering.cluster( inputDf, approxNumPartitions = 10, colNames = Seq("col1", "col2"), curve) // Underlying data shouldn't change checkAnswer(outputDf, inputDf) } } } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/skipping/clustering/ClusteredTableDDLSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.skipping.clustering import java.io.File import com.databricks.spark.util.{Log4jUsageLogger, MetricDefinitions} import org.apache.spark.sql.delta.skipping.ClusteredTableTestUtils import org.apache.spark.sql.delta.{CatalogOwnedTableFeature, DeltaAnalysisException, DeltaColumnMappingEnableIdMode, DeltaColumnMappingEnableNameMode, DeltaConfigs, DeltaLog, DeltaUnsupportedOperationException, NoMapping} import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils import org.apache.spark.sql.delta.clustering.ClusteringMetadataDomain import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.hooks.UpdateCatalog import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.SkippingEligibleDataType import org.apache.spark.sql.delta.test.{DeltaColumnMappingSelectedTestMixin, DeltaSQLCommandTest} import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.sql.{AnalysisException, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.connector.expressions.FieldReference import org.apache.spark.sql.functions.col import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{ArrayType, IntegerType, StructField, StructType} import org.apache.spark.util.Utils trait ClusteredTableCreateOrReplaceDDLSuiteBase extends QueryTest with SharedSparkSession with ClusteredTableTestUtils { override def beforeAll(): Unit = { super.beforeAll() spark.conf.set(DeltaSQLConf.DELTA_UPDATE_CATALOG_ENABLED.key, "true") } override def afterAll(): Unit = { // Reset UpdateCatalog's thread pool to ensure it is re-initialized in the next test suite. // This is necessary because the [[SparkThreadLocalForwardingThreadPoolExecutor]] // retains a reference to the SparkContext. Without resetting, the new test suite would // reuse the same SparkContext from the previous suite, despite it being stopped. // // This will force the UpdateCatalog's background thread to use the new SparkContext. // // scalastyle:off line.size.limit // This is to avoid the following exception thrown from the UpdateCatalog's background thread: // java.lang.IllegalStateException: Cannot call methods on a stopped SparkContext. // This stopped SparkContext was created at: // // org.apache.spark.sql.delta.skipping.clustering.ClusteredTableDDLDataSourceV2NameColumnMappingSuite.beforeAll // // The currently active SparkContext was created at: // // org.apache.spark.sql.delta.skipping.clustering.ClusteredTableDDLDataSourceV2Suite.beforeAll // scalastyle:on line.size.limit UpdateCatalog.tp = null super.afterAll() } protected val testTable: String = "test_ddl_table" protected val sourceTable: String = "test_ddl_source" protected val targetTable: String = "test_ddl_target" protected def isPathBased: Boolean = false protected def supportedClauses: Seq[String] testCtasRtasHelper(supportedClauses) testClusteringColumnsPartOfStatsColumn(supportedClauses) testColTypeValidation("CREATE") def testCtasRtasHelper(clauses: Seq[String]): Unit = { Seq( ("", "a INT, b STRING, ts TIMESTAMP", Seq("a", "b")), (" multipart name", "a STRUCT, ts TIMESTAMP", Seq("a.b", "ts")) ).foreach { case (testSuffix, columns, clusteringColumns) => test(s"create/replace table$testSuffix") { withTable(testTable) { clauses.foreach { clause => createOrReplaceClusteredTable( clause, testTable, columns, clusteringColumns.mkString(",")) verifyClusteringColumns(TableIdentifier(testTable), clusteringColumns) } } } test(s"ctas/rtas$testSuffix") { withTable(sourceTable, targetTable) { sql(s"CREATE TABLE $sourceTable($columns) USING delta") withTempDirIfNecessary { location => clauses.foreach { clause => createOrReplaceAsSelectClusteredTable( clause, targetTable, sourceTable, clusteringColumns.mkString(","), location = location) verifyClusteringColumns(targetTable, clusteringColumns, location) } } } } if (clauses.contains("REPLACE")) { test(s"Replace from non clustered table$testSuffix") { withTable(targetTable) { sql(s"CREATE TABLE $targetTable($columns) USING delta") createOrReplaceClusteredTable( "REPLACE", targetTable, columns, clusteringColumns.mkString(",")) verifyClusteringColumns(TableIdentifier(targetTable), clusteringColumns) } } } } } protected def createTableWithStatsColumns( clause: String, table: String, clusterColumns: Seq[String], numIndexedColumns: Int, tableSchema: Option[String], statsColumns: Seq[String] = Seq.empty, location: Option[String] = None): Unit = { val clusterSpec = clusterColumns.mkString(",") val updatedTableProperties = collection.mutable.Map("delta.dataSkippingNumIndexedCols" -> s"$numIndexedColumns") if (statsColumns.nonEmpty) { updatedTableProperties(DeltaConfigs.DATA_SKIPPING_STATS_COLUMNS.key) = statsColumns.mkString(",") } val tablePropertiesString = updatedTableProperties.map { case (key, value) => s"'$key' = '$value'" }.mkString(",") val locationClause = if (location.isEmpty) "" else s"LOCATION '${location.get}'" if (clause == "REPLACE") { // Create the default before it can be replaced. sql(s"CREATE TABLE IF NOT EXISTS $table USING DELTA $locationClause") } if (tableSchema.isEmpty) { sql( s""" |$clause TABLE $table USING DELTA CLUSTER BY ($clusterSpec) |TBLPROPERTIES($tablePropertiesString) |$locationClause |AS SELECT * FROM $sourceTable |""".stripMargin) } else { createOrReplaceClusteredTable( clause, table, tableSchema.get, clusterSpec, updatedTableProperties.toMap, location) } } protected def testStatsCollectionHelper( tableSchema: String, numberOfIndexedCols: Int)(cb: => Unit): Unit = { withTable(sourceTable) { // Create a source table for CTAS. sql( s""" | CREATE TABLE $sourceTable($tableSchema) USING DELTA | TBLPROPERTIES('delta.dataSkippingNumIndexedCols' = '$numberOfIndexedCols') |""".stripMargin) // Run additional steps. cb } } protected def testColTypeValidation(clause: String): Unit = { test(s"validate column datatype checking on $clause table") { withTable("srcTbl", "dstTbl") { // Create reference table for CTAS/RTAS. val columnMappingMode = sparkConf .getOption(DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey) .getOrElse("none") val columnMappingEnabled = columnMappingMode != NoMapping.name val specialColName = "`f@q`" val commaColSql = if (columnMappingEnabled) { s",$specialColName INT" } else { "" } val schemaStr = s""" |a STRUCT |,d BOOLEAN |,e MAP |$commaColSql |""".stripMargin sql(s"CREATE table srcTbl ($schemaStr) USING delta") val data = (0 to 1000) .map(i => Row(Row(i + 1, i * 10), i % 2 == 0, Map(i -> i), i % 2 == 1)) val schema = StructType(List( StructField("a", StructType( Array( StructField("b", IntegerType), StructField("c", IntegerType) ) )))) spark.createDataFrame(spark.sparkContext.parallelize(data), StructType(schema)) .write.mode("append").format("delta").saveAsTable("srcTbl") val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, new TableIdentifier("srcTbl")) // Test multiple data types. // Columns "a", "d" and "e" are all unsupported data skipping types. // Columns "a.b" and "`f@q`" are eligible data skipping types. val commaClusterColOpt = if (columnMappingEnabled) { Some(specialColName) } else None (Seq("a", "d", "e", "a.b") ++ commaClusterColOpt).foreach { colName => withTempDir { tmpDir => // Since validation happens both on create and replace, validate for both cases to // ensure that datatype validation behaves consistently between the two. if (clause == "REPLACE") { sql("DROP TABLE IF EXISTS dstTbl") sql(s"CREATE TABLE dstTbl LIKE srcTbl LOCATION '${tmpDir.getAbsolutePath}'") } Seq( // Scenario 1: Standard CREATE/REPLACE TABLE. () => { val schema = s"a STRUCT, d BOOLEAN, e MAP, `f,q` INT" createOrReplaceClusteredTable( clause, "dstTbl", schemaStr, colName, location = Some(tmpDir.getAbsolutePath)) }, // Scenario 2: CTAS/RTAS. () => createOrReplaceAsSelectClusteredTable( clause, "dstTbl", "srcTbl", colName, location = Some(tmpDir.getAbsolutePath))) .foreach { f => if (colName == "a.b" || colName == specialColName) { if (clause == "CREATE") { // Drop the table and delete the _delta_log directory to allow // external delta table creation. deleteTableFromCommitCoordinatorIfNeeded("dstTbl") sql("DROP TABLE IF EXISTS dstTbl") Utils.deleteRecursively(new File(tmpDir, "_delta_log")) } // Qualified data types and no exception is expected. f() } else { val e = intercept[DeltaAnalysisException] { f() } val tableSchema = DeltaLog.forTable(spark, TableIdentifier("srcTbl")).update().metadata.schema val dataTypeOpt = tableSchema .findNestedField(FieldReference(colName).fieldNames()) .map(_._2.dataType) assert(dataTypeOpt.nonEmpty, s"Can't find column $colName " + s"in schema ${tableSchema.treeString}") checkError( e, "DELTA_CLUSTERING_COLUMNS_DATATYPE_NOT_SUPPORTED", parameters = Map("columnsWithDataTypes" -> s"$colName : ${dataTypeOpt.get.sql}") ) } } } } } } } test("cluster by with more than 4 columns - create table") { val testTable = "test_table" withTable(testTable) { val e = intercept[DeltaAnalysisException] { createOrReplaceClusteredTable( "CREATE", testTable, "a INT, b INT, c INT, d INT, e INT", "a, b, c, d, e") } checkError( e, "DELTA_CLUSTER_BY_INVALID_NUM_COLUMNS", parameters = Map("numColumnsLimit" -> "4", "actualNumColumns" -> "5") ) } } test("cluster by with more than 4 columns - ctas") { val testTable = "test_table" val schema = "a INT, b INT, c INT, d INT, e INT" withTempDirIfNecessary { location => withTable(sourceTable, testTable) { sql(s"CREATE TABLE $sourceTable($schema) USING delta") val e = intercept[DeltaAnalysisException] { createOrReplaceAsSelectClusteredTable( "CREATE", testTable, sourceTable, "a, b, c, d, e", location = location) } checkError( e, "DELTA_CLUSTER_BY_INVALID_NUM_COLUMNS", parameters = Map("numColumnsLimit" -> "4", "actualNumColumns" -> "5") ) } } } protected def verifyPartitionColumns( tableIdentifier: TableIdentifier, expectedPartitionColumns: Seq[String]): Unit = { val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, tableIdentifier) assert(snapshot.metadata.partitionColumns === expectedPartitionColumns) } protected def verifyClusteringColumns( table: String, expectedLogicalClusteringColumns: Seq[String], locationOpt: Option[String]): Unit = { locationOpt.map { location => verifyClusteringColumns( location, expectedLogicalClusteringColumns ) }.getOrElse { verifyClusteringColumns(TableIdentifier(table), expectedLogicalClusteringColumns) } } def testClusteringColumnsPartOfStatsColumn(clauses: Seq[String]): Unit = { clauses.foreach { clause => val mode = if (clause == "CREATE") "create table" else "replace table" test(s"Validate clustering columns part of stats columns - $mode") { val tableSchema = "col0 int, col1 STRUCT, col2 int" val indexedColumns = 2 testStatsCollectionHelper( tableSchema = tableSchema, numberOfIndexedCols = indexedColumns) { withTable(targetTable) { val deltaLogSrc = DeltaLog.forTable(spark, TableIdentifier(sourceTable)) // Validate the 3rd column `col1.col12` and 4th column `col2` can not be // clustering columns. val e = intercept[DeltaAnalysisException]( createTableWithStatsColumns( clause, targetTable, "col0" :: "col1.col11" :: "col1.col12" :: "col2" :: Nil, indexedColumns, Some(tableSchema))) checkError( e, "DELTA_CLUSTERING_COLUMN_MISSING_STATS", parameters = Map( "columns" -> "col1.col12, col2", "schema" -> """root | |-- col0: integer (nullable = true) | |-- col1: struct (nullable = true) | | |-- col11: integer (nullable = true) |""".stripMargin) ) // Validate the first two columns can be clustering columns. createTableWithStatsColumns( clause, targetTable, "col0" :: "col1.col11" :: Nil, indexedColumns, Some(tableSchema)) } } } } clauses.foreach { clause => val mode = if (clause == "CREATE") "ctas" else "rtas" test(s"Validate clustering columns part of stats columns - $mode") { // Add a suffix for the target table name to work around the issue that delta table's // location isn't removed by the DROP TABLE from ctas/rtas test cases. val table = targetTable + "_" + clause val tableSchema = "col0 int, col1 STRUCT, col2 int" val indexedColumns = 2 testStatsCollectionHelper( tableSchema = tableSchema, numberOfIndexedCols = indexedColumns) { withTable(table) { withTempDir { dir => val deltaLogSrc = DeltaLog.forTable(spark, TableIdentifier(sourceTable)) val targetLog = DeltaLog.forTable(spark, s"${dir.getPath}") val dataPath = new File(targetLog.dataPath.toString.replace("file:", "")) val initialNumFiles = if (dataPath.listFiles() != null) { // Returns null if directory doesn't exist -> 0 dataPath.listFiles().size } else { 0 } // Validate the 3rd column `col1.col12` and 4th column `col2` can not be // clustering columns. val e = intercept[DeltaAnalysisException]( createTableWithStatsColumns( clause, table, "col0" :: "col1.col11" :: "col1.col12" :: "col2" :: Nil, indexedColumns, None, location = Some(dir.getPath))) checkError( e, "DELTA_CLUSTERING_COLUMN_MISSING_STATS", parameters = Map( "columns" -> "col1.col12, col2", "schema" -> """root | |-- col0: integer (nullable = true) | |-- col1: struct (nullable = true) | | |-- col11: integer (nullable = true) |""".stripMargin) ) // Validate the first two columns can be clustering columns. createTableWithStatsColumns( clause, table, "col0" :: "col1.col11" :: Nil, indexedColumns, None) } } } } } } test("Validate clustering columns cannot be non-eligible data types") { val indexedColumns = 3 // Validate non-eligible column stat data type. val nonEligibleType = ArrayType(IntegerType) assert(!SkippingEligibleDataType(nonEligibleType)) val nonEligibleTableSchema = s"col0 int, col1 STRUCT, col12: string>" testStatsCollectionHelper( tableSchema = nonEligibleTableSchema, numberOfIndexedCols = indexedColumns) { withTable(targetTable) { val deltaLogSrc = DeltaLog.forTable(spark, TableIdentifier(sourceTable)) // Validate the 2nd column `col1.col11` cannot be clustering column. val e = intercept[DeltaAnalysisException]( createTableWithStatsColumns( "CREATE", targetTable, "col0" :: "col1.col11" :: Nil, indexedColumns, Some(nonEligibleTableSchema))) checkError( e, "DELTA_CLUSTERING_COLUMNS_DATATYPE_NOT_SUPPORTED", parameters = Map("columnsWithDataTypes" -> "col1.col11 : ARRAY") ) } } } test("Replace clustered table with non-clustered table") { import testImplicits._ withTable(sourceTable) { sql(s"CREATE TABLE $sourceTable(i int, s string) USING delta") spark.range(1000) .map(i => (i.intValue(), "string col")) .toDF("i", "s") .write .format("delta") .mode("append") .saveAsTable(sourceTable) // Validate REPLACE TABLE (AS SELECT). Seq("REPLACE", "CREATE OR REPLACE").foreach { clause => withClusteredTable(testTable, "a int", "a") { verifyClusteringColumns(TableIdentifier(testTable), Seq("a")) Seq(true, false).foreach { isRTAS => val testQuery = if (isRTAS) { s"$clause TABLE $testTable USING delta AS SELECT * FROM $sourceTable" } else { sql(s"$clause TABLE $testTable (i int, s string) USING delta") s"INSERT INTO $testTable SELECT * FROM $sourceTable" } sql(testQuery) // Note that clustering table feature are still retained after REPLACE TABLE. verifyClusteringColumns(TableIdentifier(testTable), Seq.empty) } } } } } test("Replace clustered table with non-clustered table - dataframe writer") { import testImplicits._ withTable(sourceTable) { sql(s"CREATE TABLE $sourceTable(i int, s string) USING delta") spark.range(1000) .map(i => (i.intValue(), "string col")) .toDF("i", "s") .write .format("delta") .mode("append") .saveAsTable(sourceTable) withClusteredTable(testTable, "a int", "a") { verifyClusteringColumns(TableIdentifier(testTable), Seq("a")) spark.table(sourceTable) .write .format("delta") .mode("overwrite") .option("overwriteSchema", "true") .saveAsTable(testTable) // Note that clustering table feature are still retained after REPLACE TABLE. verifyClusteringColumns(TableIdentifier(testTable), Seq.empty) } } } protected def withTempDirIfNecessary(f: Option[String] => Unit): Unit = { if (isPathBased) { withTempDir { dir => f(Some(dir.getAbsolutePath)) } } else { f(None) } } } trait ClusteredTableDDLWithColumnMapping extends ClusteredTableCreateOrReplaceDDLSuite with DeltaColumnMappingSelectedTestMixin { override protected def runOnlyTests: Seq[String] = Seq( "validate dropping clustering column is not allowed: single clustering column", "validate dropping clustering column is not allowed: multiple clustering columns", "validate dropping clustering column is not allowed: clustering column + " + "non-clustering column", "validate RESTORE on clustered table" ) test("validate dropping clustering column is not allowed: single clustering column") { withClusteredTable(testTable, "col1 INT, col2 STRING, col3 LONG", "col1") { val e = intercept[DeltaAnalysisException] { sql(s"ALTER TABLE $testTable DROP COLUMNS (col1)") } checkError( e, "DELTA_UNSUPPORTED_DROP_CLUSTERING_COLUMN", parameters = Map("columnList" -> "col1") ) // Drop non-clustering columns are allowed. sql(s"ALTER TABLE $testTable DROP COLUMNS (col2)") } } test("validate dropping clustering column is not allowed: multiple clustering columns") { withClusteredTable(testTable, "col1 INT, col2 STRING, col3 LONG", "col1, col2") { val e = intercept[DeltaAnalysisException] { sql(s"ALTER TABLE $testTable DROP COLUMNS (col1, col2)") } checkError( e, "DELTA_UNSUPPORTED_DROP_CLUSTERING_COLUMN", parameters = Map("columnList" -> "col1,col2") ) } } test("validate dropping clustering column is not allowed: clustering column + " + "non-clustering column") { withClusteredTable(testTable, "col1 INT, col2 STRING, col3 LONG", "col1, col2") { val e = intercept[DeltaAnalysisException] { sql(s"ALTER TABLE $testTable DROP COLUMNS (col1, col3)") } checkError( e, "DELTA_UNSUPPORTED_DROP_CLUSTERING_COLUMN", parameters = Map("columnList" -> "col1") ) } } } trait ClusteredTableDDLWithColumnMappingV2Base extends ClusteredTableDDLWithColumnMapping { test("test clustering column names (alter table + create table) with spaces") { withClusteredTable(testTable, "`col1 a` INT, col2 INT, col3 STRUCT, " + "`col6 c` STRUCT, `col9.f` INT", "`col1 a`") { val tableIdentifier = TableIdentifier(testTable) verifyClusteringColumns(tableIdentifier, Seq("`col1 a`")) // Test ALTER CLUSTER BY to change clustering columns away from names with spaces. sql(s"ALTER TABLE $testTable CLUSTER BY (col2)") verifyClusteringColumns(tableIdentifier, Seq("col2")) // Test ALTER CLUSTER BY to test with structs with spaces in varying places. sql(s"ALTER TABLE $testTable CLUSTER BY (col3.`col5 b`, `col6 c`.col7)") verifyClusteringColumns(tableIdentifier, Seq("col3.`col5 b`", "`col6 c`.col7")) // Test ALTER CLUSTER BY on structs with spaces on both entries and with no spaces in the same // clustering spec, including cases where there is a '.' in the name. sql(s"ALTER TABLE $testTable CLUSTER BY (col3.col4, `col6 c`.`col8 d.e`, `col1 a`)") verifyClusteringColumns(tableIdentifier, Seq("col3.col4", "`col6 c`.`col8 d.e`", "`col1 a`")) // Test ALTER TABLE CLUSTER BY after renaming a column to include spaces in the name. sql(s"ALTER TABLE $testTable RENAME COLUMN col2 to `col2 e`") sql(s"ALTER TABLE $testTable CLUSTER BY (`col2 e`)") verifyClusteringColumns(tableIdentifier, Seq("`col2 e`")) // Test ALTER TABLE with '.' in the name. sql(s"ALTER TABLE $testTable CLUSTER BY (`col9.f`)") verifyClusteringColumns(tableIdentifier, Seq("`col9.f`")) } } test("validate create table with commas in the column name") { withClusteredTable(testTable, "`col1,a` BIGINT", "`col1,a`") { verifyClusteringColumns(TableIdentifier(testTable), Seq("`col1,a`")) } withClusteredTable(testTable, "`,col1,a,` BIGINT", "`,col1,a,`") { verifyClusteringColumns(TableIdentifier(testTable), Seq("`,col1,a,`")) } withClusteredTable(testTable, "`,col1,a,` BIGINT, `col2` BIGINT", "`,col1,a,`, `col2`") { verifyClusteringColumns(TableIdentifier(testTable), Seq("`,col1,a,`", "col2")) } withClusteredTable(testTable, "`,col1,a,` BIGINT, col2 BIGINT", "col2") { sql(s"ALTER TABLE $testTable CLUSTER BY (`,col1,a,`)") verifyClusteringColumns(TableIdentifier(testTable), Seq("`,col1,a,`")) } } } trait ClusteredTableDDLWithColumnMappingV2 extends ClusteredTableDDLWithColumnMappingV2Base trait ClusteredTableCreateOrReplaceDDLSuite extends ClusteredTableCreateOrReplaceDDLSuiteBase trait ClusteredTableDDLSuiteBase extends ClusteredTableCreateOrReplaceDDLSuite with DeltaSQLCommandTest { import testImplicits._ test("cluster by with more than 4 columns - alter table") { val testTable = "test_table" withClusteredTable(testTable, "a INT, b INT, c INT, d INT, e INT", "a") { val e = intercept[DeltaAnalysisException] { sql(s"ALTER TABLE $testTable CLUSTER BY (a, b, c, d, e)") } checkError( e, "DELTA_CLUSTER_BY_INVALID_NUM_COLUMNS", parameters = Map( "numColumnsLimit" -> "4", "actualNumColumns" -> "5") ) } } test("alter table cluster by - valid scenarios") { withClusteredTable(testTable, "id INT, a STRUCT, name STRING", "id, name") { val tableIdentifier = TableIdentifier(testTable) verifyClusteringColumns(tableIdentifier, Seq("id", "name")) // Change the clustering columns and verify that they are changed in both // Delta logs and catalog. sql(s"ALTER TABLE $testTable CLUSTER BY (name)") verifyClusteringColumns(tableIdentifier, Seq("name")) // Nested column scenario. sql(s"ALTER TABLE $testTable CLUSTER BY (a.b, id)") verifyClusteringColumns(tableIdentifier, Seq("a.b", "id")) } } test("alter table cluster by - catalog reflects clustering columns when reordered") { withClusteredTable(testTable, "id INT, a STRUCT, name STRING", "id, name") { val tableIdentifier = TableIdentifier(testTable) verifyClusteringColumns(tableIdentifier, Seq("id", "name")) // Re-order the clustering keys and validate the catalog sees the correctly reordered keys. sql(s"ALTER TABLE $testTable CLUSTER BY (name, id)") verifyClusteringColumns(tableIdentifier, Seq("name", "id")) } } test("alter table cluster by - error scenarios") { withClusteredTable(testTable, "id INT, id2 INT, name STRING", "id, name") { // Specify non-existing columns. val e = intercept[AnalysisException] { sql(s"ALTER TABLE $testTable CLUSTER BY (invalid)") } assert(e.getMessage.contains("Couldn't find column")) // Specify duplicate clustering columns. val e2 = intercept[DeltaAnalysisException] { sql(s"ALTER TABLE $testTable CLUSTER BY (id, id)") } assert(e2.getErrorClass == "DELTA_DUPLICATE_COLUMNS_FOUND") assert(e2.getSqlState == "42711") assert(e2.getMessageParametersArray === Array("in CLUSTER BY", "`id`")) } } test("alter table cluster by none") { withClusteredTable(testTable, "id Int", "id") { val tableIdentifier = TableIdentifier(testTable) verifyClusteringColumns(tableIdentifier, Seq("id")) sql(s"ALTER TABLE $testTable CLUSTER BY NONE") verifyClusteringColumns(tableIdentifier, Seq.empty) } } test("optimize clustered table and trigger regular compaction") { assume(!catalogOwnedDefaultCreationEnabledInTests, "OPTIMIZE is blocked on catalog-managed tables") withClusteredTable(testTable, "a INT, b STRING", "a, b") { val tableIdentifier = TableIdentifier(testTable) verifyClusteringColumns(tableIdentifier, Seq("a", "b")) (1 to 1000).map(i => (i, i.toString)).toDF("a", "b") .write.mode("append").format("delta").saveAsTable(testTable) val (deltaLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(testTable)) val targetFileSize = (snapshot.sizeInBytes / 10).toString withSQLConf( DeltaSQLConf.DELTA_OPTIMIZE_MAX_FILE_SIZE.key -> targetFileSize, DeltaSQLConf.DELTA_OPTIMIZE_MIN_FILE_SIZE.key -> targetFileSize) { runOptimize(testTable) { metrics => assert(metrics.numFilesAdded > 0) assert(metrics.numFilesRemoved > 0) assert(metrics.clusteringStats.nonEmpty) assert(metrics.clusteringStats.get.numOutputZCubes == 1) } } // ALTER TABLE CLUSTER BY NONE and then OPTIMIZE to trigger regular compaction. sql(s"ALTER TABLE $testTable CLUSTER BY NONE") verifyClusteringColumns(tableIdentifier, Seq.empty) (1001 to 2000).map(i => (i, i.toString)).toDF("a", "b") .repartition(10).write.mode("append").format("delta").saveAsTable(testTable) val newSnapshot = deltaLog.update() val newTargetFileSize = (newSnapshot.sizeInBytes / 10).toString withSQLConf( DeltaSQLConf.DELTA_OPTIMIZE_MAX_FILE_SIZE.key -> newTargetFileSize, DeltaSQLConf.DELTA_OPTIMIZE_MIN_FILE_SIZE.key -> newTargetFileSize) { runOptimize(testTable) { metrics => assert(metrics.numFilesAdded > 0) assert(metrics.numFilesRemoved > 0) // No clustering or zorder stats indicates regular compaction. assert(metrics.clusteringStats.isEmpty) assert(metrics.zOrderStats.isEmpty) } } } } test("optimize clustered table - error scenarios") { assume(!catalogOwnedDefaultCreationEnabledInTests, "OPTIMIZE is blocked on catalog-managed tables") withClusteredTable(testTable, "a INT, b STRING", "a") { // Specify partition predicate. val e = intercept[DeltaUnsupportedOperationException] { sql(s"OPTIMIZE $testTable WHERE a > 0 and b = foo") } checkError( e, "DELTA_CLUSTERING_WITH_PARTITION_PREDICATE", parameters = Map("predicates" -> "a > 0 and b = foo") ) // Specify ZORDER BY. val e2 = intercept[DeltaAnalysisException] { sql(s"OPTIMIZE $testTable ZORDER BY (a)") } checkError( e2, "DELTA_CLUSTERING_WITH_ZORDER_BY", parameters = Map("zOrderBy" -> "a") ) } } test("Validate stats collected - alter table") { val tableSchema = "col0 int, col1 STRUCT" val indexedColumns = 2 // Validate ALTER TABLE can not change to a missing stats column. testStatsCollectionHelper( tableSchema = tableSchema, numberOfIndexedCols = indexedColumns) { withTable(testTable) { createTableWithStatsColumns( "CREATE", testTable, "col0" :: "col1.col11" :: Nil, indexedColumns, Some(tableSchema)) // Try to alter to col1.col12 which is missing stats. val e = intercept[DeltaAnalysisException] { sql( s""" |ALTER TABLE $testTable |CLUSTER BY (col0, col1.col12) |""".stripMargin) } val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(testTable)) checkError( e, "DELTA_CLUSTERING_COLUMN_MISSING_STATS", parameters = Map( "columns" -> "col1.col12", "schema" -> snapshot.statCollectionLogicalSchema.treeString) ) } } } Seq("true", "false").foreach { checkEnabled => test(s"Alter column after statement with stats schema update - checkEnabled=$checkEnabled") { withTable(testTable) { withSQLConf( DeltaSQLConf.DELTA_LIQUID_ALTER_COLUMN_AFTER_STATS_SCHEMA_CHECK.key -> checkEnabled) { val tableSchema = "c1 int, c2 int, c3 int, c4 int" val indexedColumns = 2 testStatsCollectionHelper( tableSchema = tableSchema, numberOfIndexedCols = indexedColumns) { createTableWithStatsColumns( "CREATE", testTable, Seq("c1", "c2"), indexedColumns, Some(tableSchema)) // Insert data to ensure stats are collected sql(s"INSERT INTO $testTable VALUES(1, 2, 3, 4), (5, 6, 7, 8)") // ALTER TABLE ALTER COLUMN should succeed when checkEnabled=false sql(s"ALTER TABLE $testTable ALTER COLUMN c1 AFTER c3") // Verify the column order changed val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(testTable)) assert(snapshot.schema.fieldNames.toSeq === Seq("c2", "c3", "c1", "c4")) // Try another ALTER - behavior depends on checkEnabled if (checkEnabled == "true") { // Should fail when validation is enabled val e = intercept[DeltaAnalysisException] { sql(s"ALTER TABLE $testTable ALTER COLUMN c2 AFTER c3") } assert(e.errorClass.contains("DELTA_CLUSTERING_COLUMN_MISSING_STATS")) } else { // Should succeed when validation is disabled sql(s"ALTER TABLE $testTable ALTER COLUMN c2 AFTER c3") val (_, snapshot2) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(testTable)) assert(snapshot2.schema.fieldNames.toSeq === Seq("c3", "c2", "c1", "c4")) } } } } } } test("validate CLONE on clustered table") { assume(!catalogOwnedDefaultCreationEnabledInTests, "OPTIMIZE is blocked on catalog-managed tables") import testImplicits._ val srcTable = "SrcTbl" val dstTable1 = "DestTbl1" val dstTable2 = "DestTbl2" val dstTable3 = "DestTbl3" withTable(srcTable, dstTable1, dstTable2, dstTable3) { // Create the source table. sql(s"CREATE TABLE $srcTable (col1 INT, col2 INT, col3 INT) " + s"USING delta CLUSTER BY (col1, col2)") val tableIdent = new TableIdentifier(srcTable) (1 to 100).map(i => (i, i + 1000, i + 100)).toDF("col1", "col2", "col3") .repartitionByRange(100, col("col1")) .write.format("delta").mode("append").saveAsTable(srcTable) // Force clustering on the source table. val (_, srcSnapshot) = DeltaLog.forTableWithSnapshot(spark, tableIdent) val ingestionSize = srcSnapshot.allFiles.collect().map(_.size).sum withSQLConf( DeltaSQLConf.DELTA_OPTIMIZE_MAX_FILE_SIZE.key -> (ingestionSize / 4).toString) { runOptimize(srcTable) { res => assert(res.numFilesAdded === 4) assert(res.numFilesRemoved === 100) } } // Create destination table as a clone of the source table. sql(s"CREATE TABLE $dstTable1 SHALLOW CLONE $srcTable") // Validate clustering columns and that clustering columns in stats schema. val (_, dstSnapshot1) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(dstTable1)) verifyClusteringColumns(TableIdentifier(dstTable1), Seq("col1", "col2")) ClusteredTableUtils.validateClusteringColumnsInStatsSchema(dstSnapshot1, Seq("col1", "col2")) // Change to CLUSTER BY NONE, then CLONE from earlier version to validate that the // clustering column information is maintainted. sql(s"ALTER TABLE $srcTable CLUSTER BY NONE") sql(s"CREATE TABLE $dstTable2 SHALLOW CLONE $srcTable VERSION AS OF 2") val (_, dstSnapshot2) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(dstTable2)) verifyClusteringColumns(TableIdentifier(dstTable2), Seq("col1", "col2")) ClusteredTableUtils.validateClusteringColumnsInStatsSchema(dstSnapshot2, Seq("col1", "col2")) // Validate CLONE after CLUSTER BY NONE sql(s"CREATE TABLE $dstTable3 SHALLOW CLONE $srcTable") val (_, dstSnapshot3) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(dstTable3)) verifyClusteringColumns(TableIdentifier(dstTable3), Seq.empty) ClusteredTableUtils.validateClusteringColumnsInStatsSchema(dstSnapshot3, Seq.empty) } } test("alter table cluster by none is a no-op on non-clustered tables") { withTable(testTable) { sql(s"CREATE TABLE $testTable (a INT, b STRING) USING delta") val tableIdentifier = TableIdentifier(testTable) val (_, initialSnapshot) = DeltaLog.forTableWithSnapshot(spark, tableIdentifier) // Verify that ALTER TABLE CLUSTER BY NONE does not enable clustering and is a no-op. val clusterByLogs = Log4jUsageLogger.track { sql(s"ALTER TABLE $testTable CLUSTER BY NONE") }.filter { e => e.metric == MetricDefinitions.EVENT_TAHOE.name && e.tags.get("opType").contains("delta.ddl.alter.clusterBy") } assert(clusterByLogs.nonEmpty) val clusterByLogJson = JsonUtils.fromJson[Map[String, Any]](clusterByLogs.head.blob) assert(clusterByLogJson("isClusterByNoneSkipped").asInstanceOf[Boolean]) val (_, finalSnapshot) = DeltaLog.forTableWithSnapshot(spark, tableIdentifier) assert(!ClusteredTableUtils.isSupported(finalSnapshot.protocol)) // Snapshot equality shows that no table features can be changed. assert(initialSnapshot.version == finalSnapshot.version) assert(initialSnapshot.protocol.readerAndWriterFeatureNames == finalSnapshot.protocol.readerAndWriterFeatureNames) } } test("alter table set tbl properties not allowed for clusteringColumns") { withClusteredTable(testTable, "a INT, b STRING", "a") { val e = intercept[DeltaUnsupportedOperationException] { sql(s""" |ALTER TABLE $testTable SET TBLPROPERTIES |('${ClusteredTableUtils.PROP_CLUSTERING_COLUMNS}' = '[[\"b\"]]') |""".stripMargin) } checkError( e, "DELTA_CANNOT_MODIFY_TABLE_PROPERTY", parameters = Map("prop" -> "clusteringColumns")) } } test("validate RESTORE on clustered table") { val tableIdentifier = TableIdentifier(testTable) // Scenario 1: restore clustered table to unclustered version. withTable(testTable) { sql(s"CREATE TABLE $testTable (a INT, b STRING) USING delta") val (_, startingSnapshot) = DeltaLog.forTableWithSnapshot(spark, tableIdentifier) assert(!ClusteredTableUtils.isSupported(startingSnapshot.protocol)) sql(s"ALTER TABLE $testTable CLUSTER BY (a)") verifyClusteringColumns(tableIdentifier, Seq("a")) sql(s"RESTORE TABLE $testTable TO VERSION AS OF 0") val (_, currentSnapshot) = DeltaLog.forTableWithSnapshot(spark, tableIdentifier) verifyClusteringColumns(tableIdentifier, Seq.empty, skipCatalogCheck = true) } // Scenario 2: restore clustered table to previous clustering columns. withClusteredTable(testTable, "a INT, b STRING", "a") { verifyClusteringColumns(tableIdentifier, Seq("a")) sql(s"ALTER TABLE $testTable CLUSTER BY (b)") verifyClusteringColumns(tableIdentifier, Seq("b")) sql(s"RESTORE TABLE $testTable TO VERSION AS OF 0") verifyClusteringColumns(tableIdentifier, Seq("a"), skipCatalogCheck = true) } // Scenario 3: restore from table with clustering columns to non-empty clustering columns withClusteredTable(testTable, "a int", "a") { verifyClusteringColumns(tableIdentifier, Seq("a")) sql(s"ALTER TABLE $testTable CLUSTER BY NONE") verifyClusteringColumns(tableIdentifier, Seq.empty) sql(s"RESTORE TABLE $testTable TO VERSION AS OF 0") verifyClusteringColumns(tableIdentifier, Seq("a"), skipCatalogCheck = true) } // Scenario 4: restore to start version. withClusteredTable(testTable, "a int", "a") { verifyClusteringColumns(tableIdentifier, Seq("a")) sql(s"INSERT INTO $testTable VALUES (1)") sql(s"RESTORE TABLE $testTable TO VERSION AS OF 0") verifyClusteringColumns(tableIdentifier, Seq("a"), skipCatalogCheck = true) } // Scenario 5: restore unclustered table to unclustered table. withTable(testTable) { sql(s"CREATE TABLE $testTable (a INT) USING delta") val (_, startingSnapshot) = DeltaLog.forTableWithSnapshot(spark, tableIdentifier) assert(!ClusteredTableUtils.isSupported(startingSnapshot.protocol)) assert(!startingSnapshot.domainMetadata.exists(_.domain == ClusteringMetadataDomain.domainName)) sql(s"INSERT INTO $testTable VALUES (1)") sql(s"RESTORE TABLE $testTable TO VERSION AS OF 0").collect val (_, currentSnapshot) = DeltaLog.forTableWithSnapshot(spark, tableIdentifier) assert(!ClusteredTableUtils.isSupported(currentSnapshot.protocol)) assert(!currentSnapshot.domainMetadata.exists(_.domain == ClusteringMetadataDomain.domainName)) } // Scenario 6: restore clustered table to unclustered table. withTable(testTable) { sql(s"CREATE TABLE $testTable (a INT) USING delta") val (_, startingSnapshot) = DeltaLog.forTableWithSnapshot(spark, tableIdentifier) assert(!ClusteredTableUtils.isSupported(startingSnapshot.protocol)) assert(!startingSnapshot.domainMetadata.exists(_.domain == ClusteringMetadataDomain.domainName)) sql(s"ALTER TABLE $testTable CLUSTER BY (a)") sql(s"RESTORE TABLE $testTable TO VERSION AS OF 0") val (_, currentSnapshot) = DeltaLog.forTableWithSnapshot(spark, tableIdentifier) assert(ClusteredTableUtils.isSupported(currentSnapshot.protocol)) verifyClusteringColumns(tableIdentifier, Seq.empty[String], skipCatalogCheck = true) } } test("Variant is not supported") { val e = intercept[DeltaAnalysisException] { createOrReplaceClusteredTable("CREATE", testTable, "id long, v variant", "v") } checkError( e, "DELTA_CLUSTERING_COLUMNS_DATATYPE_NOT_SUPPORTED", parameters = Map("columnsWithDataTypes" -> "v : VARIANT") ) } } trait ClusteredTableDDLSuite extends ClusteredTableDDLSuiteBase with CatalogOwnedTestBaseSuite trait ClusteredTableDDLWithNameColumnMapping extends ClusteredTableCreateOrReplaceDDLSuite with DeltaColumnMappingEnableNameMode trait ClusteredTableDDLWithIdColumnMapping extends ClusteredTableCreateOrReplaceDDLSuite with DeltaColumnMappingEnableIdMode trait ClusteredTableDDLWithV2Base extends ClusteredTableCreateOrReplaceDDLSuite with SharedSparkSession { override protected def supportedClauses: Seq[String] = Seq("CREATE", "REPLACE") testColTypeValidation("REPLACE") test("replace with different clustering columns") { withTable(sourceTable) { sql(s"CREATE TABLE $sourceTable(i int, s string) USING delta") // Validate REPLACE TABLE (AS SELECT). Seq("REPLACE", "CREATE OR REPLACE").foreach { clause => Seq(true, false).foreach { isRTAS => withTempDirIfNecessary { location => withClusteredTable(testTable, "a int", "a", location = location) { if (isRTAS) { createOrReplaceAsSelectClusteredTable( clause, testTable, sourceTable, "i", location = location) } else { createOrReplaceClusteredTable( clause, testTable, "i int, b string", "i", location = location) } verifyClusteringColumns(testTable, Seq("i"), location) } } } } } } test("Validate replacing clustered tables with partitioned tables is not allowed") { withTable(sourceTable) { sql(s"CREATE TABLE $sourceTable(i int, s string) USING delta") // Validate REPLACE TABLE (AS SELECT). Seq("REPLACE", "CREATE OR REPLACE").foreach { clause => withClusteredTable(testTable, "a int", "a") { verifyClusteringColumns(TableIdentifier(testTable), Seq("a")) Seq(true, false).foreach { isRTAS => val e = intercept[DeltaAnalysisException] { if (isRTAS) { sql(s"$clause TABLE $testTable USING delta PARTITIONED BY (i) " + s"AS SELECT * FROM $sourceTable") } else { sql(s"$clause TABLE $testTable (i int, b string) USING delta PARTITIONED BY (i)") } } checkError( e, "DELTA_CLUSTERING_REPLACE_TABLE_WITH_PARTITIONED_TABLE", parameters = Map.empty ) } } } } } test("Validate replacing partitioned tables with clustered tables is allowed") { withTable(sourceTable) { sql(s"CREATE TABLE $sourceTable(i int, s string) USING delta") // Validate REPLACE TABLE (AS SELECT). Seq("REPLACE", "CREATE OR REPLACE").foreach { clause => Seq(true, false).foreach { isRTAS => withTable(testTable) { withTempDirIfNecessary { location => val locationClause = if (location.isEmpty) "" else s"LOCATION '${location.get}'" sql(s"CREATE TABLE $testTable USING delta PARTITIONED BY (i) $locationClause" + s" SELECT 1 i, 'a' s") verifyPartitionColumns(TableIdentifier(testTable), Seq("i")) if (isRTAS) { createOrReplaceAsSelectClusteredTable( clause, testTable, sourceTable, "i", location = location) } else { createOrReplaceClusteredTable( clause, testTable, "i int, b string", "i", location = location) } verifyClusteringColumns(testTable, Seq("i"), location) verifyPartitionColumns(TableIdentifier(testTable), Seq()) } } } } } } Seq( ("", "a INT, b STRING, ts TIMESTAMP", Seq("a", "b")), (" multipart name", "a STRUCT, ts TIMESTAMP", Seq("a.b", "ts")) ).foreach { case (testSuffix, columns, clusteringColumns) => test(s"create/replace table createOrReplace$testSuffix") { withTable(testTable) { // Repeat two times to test both create and replace cases. (1 to 2).foreach { _ => createOrReplaceClusteredTable( "CREATE OR REPLACE", testTable, columns, clusteringColumns.mkString(",")) verifyClusteringColumns(TableIdentifier(testTable), clusteringColumns) } } } test(s"ctas/rtas createOrReplace$testSuffix") { withTable(sourceTable, targetTable) { sql(s"CREATE TABLE $sourceTable($columns) USING delta") withTempDirIfNecessary { location => // Repeat two times to test both create and replace cases. (1 to 2).foreach { _ => createOrReplaceAsSelectClusteredTable( "CREATE OR REPLACE", targetTable, sourceTable, clusteringColumns.mkString(","), location = location) verifyClusteringColumns(targetTable, clusteringColumns, location) } } } } } } trait ClusteredTableDDLWithV2 extends ClusteredTableDDLWithV2Base trait ClusteredTableDDLDataSourceV2SuiteBase extends ClusteredTableDDLWithV2 with ClusteredTableDDLSuite { test("Create clustered table from external location, " + "location has clustered table, schema not specified, cluster by not specified") { withTempDir { dir => // 1. Create a clustered table sql(s"create table delta.`${dir.getAbsolutePath}` (col1 int, col2 string) using delta " + "cluster by (col1)") // 2. Create a clustered table from the external location. withTable("clustered_table") { // When schema is not specified, the schema of the table is inferred from the external // table. sql(s"CREATE EXTERNAL TABLE clustered_table USING delta LOCATION '${dir.getAbsolutePath}'") verifyClusteringColumns(TableIdentifier("clustered_table"), Seq("col1")) } } } test("create external non-clustered table: location has clustered table, schema specified, " + "cluster by not specified") { val tableName = "clustered_table" withTempDir { dir => // 1. Create a clustered table in the external location. sql(s"create table delta.`${dir.getAbsolutePath}` (col1 int, col2 string) using delta " + "cluster by (col1)") // 2. Create a non-clustered table from the external location. withTable(tableName) { val e = intercept[DeltaAnalysisException] { // When schema is specified, the schema has to match the schema of the external table. sql(s"CREATE EXTERNAL TABLE $tableName (col1 INT, col2 STRING) USING delta " + s"LOCATION '${dir.getAbsolutePath}'") } checkError( e, "DELTA_CREATE_TABLE_WITH_DIFFERENT_CLUSTERING", parameters = Map( "path" -> dir.toURI.toString.stripSuffix("/"), "specifiedColumns" -> "", "existingColumns" -> "col1")) } } } test("create external clustered table: location has clustered table, schema specified, " + "cluster by specified with different clustering column") { val tableName = "clustered_table" withTempDir { dir => // 1. Create a clustered table in the external location. sql(s"create table delta.`${dir.getAbsolutePath}` (col1 int, col2 string) using delta " + "cluster by (col1)") // 2. Create a clustered table from the external location. withTable(tableName) { val e = intercept[DeltaAnalysisException] { sql(s"CREATE EXTERNAL TABLE $tableName (col1 INT, col2 STRING) USING delta " + s"CLUSTER BY (col2) LOCATION '${dir.getAbsolutePath}'") } checkError( e, "DELTA_CREATE_TABLE_WITH_DIFFERENT_CLUSTERING", parameters = Map( "path" -> dir.toURI.toString.stripSuffix("/"), "specifiedColumns" -> "col2", "existingColumns" -> "col1")) } } } test("create external clustered table: location has clustered table, schema specified, " + "cluster by specified with same clustering column") { if (catalogOwnedDefaultCreationEnabledInTests) { cancel("CatalogOwned does not support external table creation.") } val tableName = "clustered_table" withTempDir { dir => // 1. Create a clustered table in the external location. sql(s"create table delta.`${dir.getAbsolutePath}` (col1 int, col2 string) using delta " + "cluster by (col1)") // 2. Create a clustered table from the external location. withTable(tableName) { sql(s"CREATE EXTERNAL TABLE $tableName (col1 INT, col2 STRING) USING delta " + s"CLUSTER BY (col1) LOCATION '${dir.getAbsolutePath}'") verifyClusteringColumns(TableIdentifier(tableName), Seq("col1")) } } } test("create external clustered table: location has non-clustered table, schema specified, " + "cluster by specified") { if (catalogOwnedDefaultCreationEnabledInTests) { cancel("CatalogOwned does not support external table creation.") } val tableName = "clustered_table" withTempDir { dir => // 1. Create a non-clustered table in the external location. sql(s"create table delta.`${dir.getAbsolutePath}` (col1 int, col2 string) using delta") // 2. Create a clustered table from the external location. withTable(tableName) { val e = intercept[DeltaAnalysisException] { sql(s"CREATE EXTERNAL TABLE $tableName (col1 INT, col2 STRING) USING delta " + s"CLUSTER BY (col1) LOCATION '${dir.getAbsolutePath}'") } checkError( e, "DELTA_CREATE_TABLE_WITH_DIFFERENT_CLUSTERING", parameters = Map( "path" -> dir.toURI.toString.stripSuffix("/"), "specifiedColumns" -> "col1", "existingColumns" -> "")) } } } } class ClusteredTableDDLDataSourceV2Suite extends ClusteredTableDDLDataSourceV2SuiteBase class ClusteredTableDDLDataSourceV2IdColumnMappingSuite extends ClusteredTableDDLWithIdColumnMapping with ClusteredTableDDLWithV2 with ClusteredTableDDLWithColumnMappingV2 with ClusteredTableDDLSuite class ClusteredTableDDLDataSourceV2NameColumnMappingSuite extends ClusteredTableDDLWithNameColumnMapping with ClusteredTableDDLWithV2 with ClusteredTableDDLWithColumnMappingV2 with ClusteredTableDDLSuite class ClusteredTableDDLDataSourceV2WithCatalogOwnedBatch100Suite extends ClusteredTableDDLDataSourceV2Suite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/skipping/clustering/ClusteringColumnSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.skipping.clustering import org.apache.spark.sql.delta.skipping.ClusteredTableTestUtils import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.TableIdentifier class ClusteringColumnSuite extends ClusteredTableTestUtils with DeltaSQLCommandTest { test("ClusteringColumnInfo: validate logical column") { val table = "test_table" withTable(table) { // Create a table with nested dot name column. sql(s"CREATE TABLE $table(col0 int, col1 STRUCT<`x.y`: int, z: int>) USING DELTA") val col0 = "col0" val dotColumnName = "col1.`x.y`" val schema = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(table))._2.schema val columnInfo0 = ClusteringColumnInfo(schema, ClusteringColumn(schema, col0)) val columnInfoDot = ClusteringColumnInfo(schema, ClusteringColumn(schema, dotColumnName)) assert(columnInfo0.logicalName === col0) assert(columnInfoDot.logicalName === dotColumnName) } } test("ClusteringColumnInfo: extractLogicalNames") { val table = "test_table" // Create a table with nested dot name column. withClusteredTable( table, "col0 int, col1 STRUCT<`x.y`: int, z: int>", "col0, col1.`x.y`") { val col0 = "col0" val dotColumnName = "col1.`x.y`" val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(table)) assert(ClusteringColumnInfo.extractLogicalNames(snapshot) == Seq(col0, dotColumnName)) } } test("ClusteringColumn: throws correct error when column not found") { withTable("tbl") { sql("CREATE TABLE tbl (a INT) USING DELTA") val schema = spark.table("tbl").schema val e = intercept[AnalysisException] { ClusteringColumn(schema, "b") } checkError( e, "DELTA_COLUMN_NOT_FOUND_IN_SCHEMA", parameters = Map( "columnName" -> "b", "tableSchema" -> schema.treeString)) } } test("ClusteringColumn: throws correct error when nested column not found") { withTable("tbl") { sql("CREATE TABLE tbl (a INT, b STRING) USING DELTA") val schema = spark.table("tbl").schema val e = intercept[AnalysisException] { ClusteringColumn(schema, "b.c") } checkError( e, "DELTA_COLUMN_NOT_FOUND_IN_SCHEMA", parameters = Map( "columnName" -> "b.c", "tableSchema" -> schema.treeString)) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/skipping/clustering/ClusteringProviderSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.skipping.clustering // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.{DeltaConfigs, DeltaLog, DeltaOperations} import org.apache.spark.sql.delta.actions.{AddFile, Metadata} import org.apache.spark.sql.delta.actions.SingleAction._ import org.apache.spark.sql.delta.stats.DataSkippingReader import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.hadoop.fs.Path import org.apache.spark.sql.QueryTest import org.apache.spark.sql.test.SharedSparkSession class ClusteringProviderSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { private def testAddFileWithSnapshotReconstructionHelper( prefix: String)(collectFiles: DeltaLog => Seq[AddFile]): Unit = { for (checkpointPolicy <- Seq("none", "classic", "v2")) { test(s"$prefix - Validate clusteringProvider in snapshot reconstruction, " + s"checkpointPolicy = $checkpointPolicy") { val file = AddFile( path = "path", partitionValues = Map.empty, size = 1, modificationTime = 1, dataChange = true, clusteringProvider = Some("liquid")) withTempDir { dir => val log = DeltaLog.forTable(spark, new Path(dir.getCanonicalPath)) log.startTransaction(None).commit(Metadata() :: Nil, DeltaOperations.ManualUpdate) log.startTransaction(None).commit(file :: Nil, DeltaOperations.ManualUpdate) if (checkpointPolicy != "none") { spark.sql(s"ALTER TABLE delta.`${dir.getAbsolutePath}` SET TBLPROPERTIES " + s"('${DeltaConfigs.CHECKPOINT_POLICY.key}' = '$checkpointPolicy')") log.checkpoint(log.update()) // clear cache to force the snapshot reconstruction. DeltaLog.clearCache() } val files = collectFiles(log) assert(files.size === 1) assert(files.head.clusteringProvider === Some("liquid")) } } } } testAddFileWithSnapshotReconstructionHelper("Default snapshot reconstruction") { log => log.update().allFiles.collect() } testAddFileWithSnapshotReconstructionHelper("AddFile with stats") { log => val statsDF = log.update().withStats.withColumn("stats", DataSkippingReader.nullStringLiteral) statsDF.as[AddFile].collect() } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/skipping/clustering/IncrementalZCubeClusteringSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.skipping.clustering // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.skipping.ClusteredTableTestUtilsBase import org.apache.spark.sql.delta.skipping.clustering.{ClusteredTableUtils, ClusteringColumnInfo, ClusteringFileStats, ClusteringStats} import org.apache.spark.sql.delta.{DeltaLog, DeltaOperations, DeltaUnsupportedOperationException} import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.zorder.ZCubeInfo import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.internal.SQLConf class IncrementalZCubeClusteringSuite extends QueryTest with ClusteredTableTestUtilsBase with DeltaSQLCommandTest { import testImplicits._ private val table: String = "test_table" // Ingest data to create numFiles files with one row in each file. private def addFiles(table: String, numFiles: Int): Unit = { val df = (1 to numFiles).map(i => (i, i)).toDF("col1", "col2") withSQLConf(SQLConf.MAX_RECORDS_PER_FILE.key -> "1") { df.write.format("delta").mode("append").saveAsTable(table) } } private def getFiles(table: String): Set[AddFile] = { val deltaLog = DeltaLog.forTable(spark, TableIdentifier(table)) deltaLog.update().allFiles.collect().toSet } private def assertClustered(table: String, files: Set[AddFile]): Unit = { val deltaLog = DeltaLog.forTable(spark, TableIdentifier(table)) val clusteringColumns = ClusteringColumnInfo.extractLogicalNames(deltaLog.update()) assert(files.forall(_.clusteringProvider.contains(ClusteredTableUtils.clusteringProvider))) assert(files.forall { file => val zCubeInfo = ZCubeInfo.getForFile(file) if (zCubeInfo.isEmpty) { logError(s"File $file is missing ZCube info.") false } else { zCubeInfo.get.zOrderBy == clusteringColumns } }) } // The sentinel value to signal skipping size validation in ClusteringStats. This is used for the // cases where file size can not be predicated due to compression and encoding. private val SKIP_CHECK_SIZE_VALUE: Long = Long.MinValue private def validateClusteringMetrics( actualMetrics: ClusteringStats, expectedMetrics: ClusteringStats): Unit = { var finalActualMetrics = actualMetrics if (expectedMetrics.inputZCubeFiles.size == SKIP_CHECK_SIZE_VALUE) { val stats = finalActualMetrics.inputZCubeFiles finalActualMetrics = finalActualMetrics.copy(inputZCubeFiles = stats.copy(size = SKIP_CHECK_SIZE_VALUE)) } if (expectedMetrics.inputOtherFiles.size == SKIP_CHECK_SIZE_VALUE) { val stats = finalActualMetrics.inputOtherFiles finalActualMetrics = finalActualMetrics.copy(inputOtherFiles = stats.copy(size = SKIP_CHECK_SIZE_VALUE)) } if (expectedMetrics.mergedFiles.size == SKIP_CHECK_SIZE_VALUE) { val stats = finalActualMetrics.mergedFiles finalActualMetrics = finalActualMetrics.copy(mergedFiles = stats.copy(size = SKIP_CHECK_SIZE_VALUE)) } assert(expectedMetrics === finalActualMetrics) } private def getZCubeIds(table: String): Set[String] = { val files = getFiles(table) files.map(ZCubeInfo.getForFile).collect { case Some(ZCubeInfo(id, _)) => id } } test("test incremental clustering") { withSQLConf( SQLConf.MAX_RECORDS_PER_FILE.key -> "2") { withClusteredTable( table = table, schema = "col1 int, col2 int", clusterBy = "col1, col2") { addFiles(table, numFiles = 4) val files0 = getFiles(table) assert(files0.size === 4) // Optimize should cluster the data into two 2 files since MAX_RECORDS_PER_FILE is 2. runOptimize(table) { metrics => assert(metrics.clusteringStats.nonEmpty) validateClusteringMetrics( actualMetrics = metrics.clusteringStats.get, expectedMetrics = ClusteringStats( inputZCubeFiles = ClusteringFileStats(0, SKIP_CHECK_SIZE_VALUE), inputOtherFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), inputNumZCubes = 0, mergedFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), numOutputZCubes = 1)) assert(metrics.numFilesRemoved == 4) assert(metrics.numFilesAdded == 2) } val files1 = getFiles(table) assert(files1.size == 2) assertClustered(table, files1) assert(getZCubeIds(table).size === 1) // re-optimize is no-op if there is single ZCUBE in the whole table. withSQLConf( // Make the current ZCUBE big enough to include all input in a single ZCUBE. DeltaSQLConf.DELTA_OPTIMIZE_CLUSTERING_MIN_CUBE_SIZE.key -> Long.MaxValue.toString) { runOptimize(table) { metrics => assert(metrics.numFilesRemoved === 0) } } assert(files1 == getFiles(table)) // Append some new data and only cluster new files. addFiles(table, numFiles = 4) val files2 = getFiles(table) assert(files2.size === 6) withSQLConf( DeltaSQLConf.DELTA_OPTIMIZE_CLUSTERING_MIN_CUBE_SIZE.key -> 1.toString) { runOptimize(table) { metrics => assert(metrics.clusteringStats.nonEmpty) validateClusteringMetrics( actualMetrics = metrics.clusteringStats.get, expectedMetrics = ClusteringStats( inputZCubeFiles = ClusteringFileStats(2, SKIP_CHECK_SIZE_VALUE), inputOtherFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), inputNumZCubes = 1, mergedFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), numOutputZCubes = 1)) assert(metrics.numFilesRemoved === 4) assert(metrics.numFilesAdded === 2) } } val files3 = getFiles(table) assert(files3.intersect(files2) === files1) assert(getZCubeIds(table).size === 2) // Now there are 2 ZCUBEs, increase ZCUBE size and stable ZCUBEs should be re-clustered. withSQLConf( DeltaSQLConf.DELTA_OPTIMIZE_CLUSTERING_MIN_CUBE_SIZE.key -> Long.MaxValue.toString) { runOptimize(table) { metrics => assert(metrics.clusteringStats.nonEmpty) validateClusteringMetrics( actualMetrics = metrics.clusteringStats.get, expectedMetrics = ClusteringStats( inputZCubeFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), inputOtherFiles = ClusteringFileStats(0, SKIP_CHECK_SIZE_VALUE), inputNumZCubes = 2, mergedFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), numOutputZCubes = 1)) assert(metrics.numFilesRemoved === 4) // 2 records per file. assert(metrics.numFilesAdded === 4) } } val files4 = getFiles(table) assertClustered(table, files4) assert(getZCubeIds(table).size === 1) } } } test("test changing clustering columns") { withSQLConf( SQLConf.MAX_RECORDS_PER_FILE.key -> "2", // Enable update catalog for verifyClusteringColumns. DeltaSQLConf.DELTA_UPDATE_CATALOG_ENABLED.key -> "true") { withClusteredTable( table = table, schema = "col1 int, col2 int", clusterBy = "col1, col2") { addFiles(table, numFiles = 4) val files0 = getFiles(table) assert(files0.size === 4) // Cluster the table into two ZCUBEs. runOptimize(table) { metrics => assert(metrics.clusteringStats.nonEmpty) validateClusteringMetrics( actualMetrics = metrics.clusteringStats.get, expectedMetrics = ClusteringStats( inputZCubeFiles = ClusteringFileStats(0, SKIP_CHECK_SIZE_VALUE), inputOtherFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), inputNumZCubes = 0, mergedFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), numOutputZCubes = 1)) assert(metrics.numFilesRemoved == 4) assert(metrics.numFilesAdded == 2) } assert(getFiles(table).size == 2) addFiles(table, numFiles = 4) assert(getFiles(table).size == 6) withSQLConf( DeltaSQLConf.DELTA_OPTIMIZE_CLUSTERING_MIN_CUBE_SIZE.key -> 1.toString) { runOptimize(table) { metrics => assert(metrics.clusteringStats.nonEmpty) validateClusteringMetrics( actualMetrics = metrics.clusteringStats.get, expectedMetrics = ClusteringStats( inputZCubeFiles = ClusteringFileStats(2, SKIP_CHECK_SIZE_VALUE), inputOtherFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), inputNumZCubes = 1, mergedFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), numOutputZCubes = 1)) assert(metrics.numFilesRemoved == 4) assert(metrics.numFilesAdded == 2) } } val files1 = getFiles(table) assert(files1.size === 4) assertClustered(table, files1) assert(getZCubeIds(table).size == 2) sql(s"ALTER TABLE $table CLUSTER BY (col2, col1)") verifyClusteringColumns(TableIdentifier(table), Seq("col2", "col1")) // Incremental clustering won't touch those clustered files with different clustering // columns, so re-clustering should be a no-op. withSQLConf( DeltaSQLConf.DELTA_OPTIMIZE_CLUSTERING_MIN_CUBE_SIZE.key -> Long.MaxValue.toString) { runOptimize(table) { metrics => assert(metrics.clusteringStats.nonEmpty) assert(metrics.numFilesRemoved == 0) } } assert(getFiles(table) === files1) // Add more files and only new files are clustered. addFiles(table, numFiles = 4) val files2 = getFiles(table) assert(files2.size === 8) withSQLConf( DeltaSQLConf.DELTA_OPTIMIZE_CLUSTERING_MIN_CUBE_SIZE.key -> Long.MaxValue.toString) { runOptimize(table) { metrics => assert(metrics.clusteringStats.nonEmpty) validateClusteringMetrics( actualMetrics = metrics.clusteringStats.get, expectedMetrics = ClusteringStats( inputZCubeFiles = ClusteringFileStats(0, SKIP_CHECK_SIZE_VALUE), // 8 files: 4 files from previously clustered files with different cluster keys // and 4 files from newly added 4 un-clustered files. inputOtherFiles = ClusteringFileStats(8, SKIP_CHECK_SIZE_VALUE), inputNumZCubes = 0, mergedFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), numOutputZCubes = 1)) assert(metrics.numFilesRemoved == 4) assert(metrics.numFilesAdded == 2) } } val files3 = getFiles(table) assert(files3.size === 6) // files1 are files with old clustering columns 'col1'. assert(files3.intersect(files2) === files1) } } } test("OPTIMIZE FULL - change cluster keys") { withSQLConf( SQLConf.MAX_RECORDS_PER_FILE.key -> "2", // Enable update catalog for verifyClusteringColumns. DeltaSQLConf.DELTA_UPDATE_CATALOG_ENABLED.key -> "true") { withClusteredTable( table = table, schema = "col1 int, col2 int", clusterBy = "col1, col2") { addFiles(table, numFiles = 4) val files0 = getFiles(table) assert(files0.size === 4) // Cluster the table into two ZCUBEs. runOptimize(table) { metrics => assert(metrics.clusteringStats.nonEmpty) validateClusteringMetrics( actualMetrics = metrics.clusteringStats.get, expectedMetrics = ClusteringStats( inputZCubeFiles = ClusteringFileStats(0, SKIP_CHECK_SIZE_VALUE), inputOtherFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), inputNumZCubes = 0, mergedFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), numOutputZCubes = 1)) assert(metrics.numFilesRemoved == 4) assert(metrics.numFilesAdded == 2) } val files1 = getFiles(table) assert(files1.size === 2) addFiles(table, numFiles = 4) assert(getFiles(table).size == 6) // Change the clustering columns and verify files with previous clustering columns // are not clustered. sql(s"ALTER TABLE $table CLUSTER BY (col2, col1)") verifyClusteringColumns(TableIdentifier(table), Seq("col2", "col1")) withSQLConf( // Set an extreme value to make all zcubes unstable. DeltaSQLConf.DELTA_OPTIMIZE_CLUSTERING_MIN_CUBE_SIZE.key -> Long.MaxValue.toString) { runOptimize(table) { metrics => assert(metrics.clusteringStats.nonEmpty) assert(metrics.numFilesRemoved == 4) assert(metrics.numFilesAdded == 2) validateClusteringMetrics( actualMetrics = metrics.clusteringStats.get, expectedMetrics = ClusteringStats( inputZCubeFiles = ClusteringFileStats(0, SKIP_CHECK_SIZE_VALUE), inputOtherFiles = ClusteringFileStats(6, SKIP_CHECK_SIZE_VALUE), inputNumZCubes = 0, mergedFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), numOutputZCubes = 1)) } } val files2 = getFiles(table) assert(files2.size === 4) assert(files2.forall { file => val zCubeInfo = ZCubeInfo.getForFile(file) zCubeInfo.nonEmpty }) assert(getZCubeIds(table).size == 2) // validate files clustered to previous clustering columns are not re-clustered. assert(files2.intersect(files1) === files1) // OPTIMIZE FULL should re-cluster previously clustered files. withSQLConf( // Force all zcubes stable DeltaSQLConf.DELTA_OPTIMIZE_CLUSTERING_MIN_CUBE_SIZE.key -> 1.toString) { runOptimizeFull(table) { metrics => assert(metrics.clusteringStats.nonEmpty) // Only files with old cluster keys are rewritten. assert(metrics.numFilesRemoved == 2) assert(metrics.numFilesAdded == 2) validateClusteringMetrics( actualMetrics = metrics.clusteringStats.get, expectedMetrics = ClusteringStats( inputZCubeFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), inputOtherFiles = ClusteringFileStats(0, SKIP_CHECK_SIZE_VALUE), inputNumZCubes = 2, mergedFiles = ClusteringFileStats(2, SKIP_CHECK_SIZE_VALUE), numOutputZCubes = 1)) } } // all files have same clustering keys. assert(getFiles(table).forall { f => val zCubeInfo = ZCubeInfo.getForFile(f).get val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(table)) val clusteringColumns = ClusteringColumnInfo.extractLogicalNames(snapshot) zCubeInfo.zOrderBy == clusteringColumns }) // Incremental OPTIMIZE to validate no files should be clustered. withSQLConf( // Force all zcubes stable DeltaSQLConf.DELTA_OPTIMIZE_CLUSTERING_MIN_CUBE_SIZE.key -> 1.toString) { runOptimize(table) { metrics => assert(metrics.clusteringStats.nonEmpty) assert(metrics.numFilesRemoved == 0) validateClusteringMetrics( actualMetrics = metrics.clusteringStats.get, expectedMetrics = ClusteringStats( inputZCubeFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), inputOtherFiles = ClusteringFileStats(0, SKIP_CHECK_SIZE_VALUE), inputNumZCubes = 2, mergedFiles = ClusteringFileStats(0, SKIP_CHECK_SIZE_VALUE), numOutputZCubes = 0)) } } // OPTIMIZE FULL again and all clustered files have same clustering columns and // all ZCUBEs are stable. withSQLConf( // Force all zcubes stable DeltaSQLConf.DELTA_OPTIMIZE_CLUSTERING_MIN_CUBE_SIZE.key -> 1.toString) { runOptimizeFull(table) { metrics => assert(metrics.clusteringStats.nonEmpty) validateClusteringMetrics( actualMetrics = metrics.clusteringStats.get, expectedMetrics = ClusteringStats( inputZCubeFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), inputOtherFiles = ClusteringFileStats(0, SKIP_CHECK_SIZE_VALUE), inputNumZCubes = 2, mergedFiles = ClusteringFileStats(0, SKIP_CHECK_SIZE_VALUE), numOutputZCubes = 0)) assert(metrics.numFilesRemoved == 0) assert(metrics.numFilesAdded == 0) } } } } } test("OPTIMIZE FULL - change clustering provider") { withSQLConf( SQLConf.MAX_RECORDS_PER_FILE.key -> "2", // Enable update catalog for verifyClusteringColumns. DeltaSQLConf.DELTA_UPDATE_CATALOG_ENABLED.key -> "true") { withClusteredTable( table = table, schema = "col1 int, col2 int", clusterBy = "col1, col2") { addFiles(table, numFiles = 4) val files0 = getFiles(table) assert(files0.size === 4) // Cluster the table into two ZCUBEs. runOptimize(table) { metrics => assert(metrics.clusteringStats.nonEmpty) validateClusteringMetrics( actualMetrics = metrics.clusteringStats.get, expectedMetrics = ClusteringStats( inputZCubeFiles = ClusteringFileStats(0, SKIP_CHECK_SIZE_VALUE), inputOtherFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), inputNumZCubes = 0, mergedFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), numOutputZCubes = 1)) assert(metrics.numFilesRemoved == 4) assert(metrics.numFilesAdded == 2) } var files1 = getFiles(table) assert(files1.size === 2) for (f <- files1) { assert(f.clusteringProvider.contains(ClusteredTableUtils.clusteringProvider)) } // Change the clusteringProvider and verify files with different clusteringProvider // are not clustered. val (deltaLog, _) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(table)) val txn = deltaLog.startTransaction(None) files1 = files1.map(f => f.copy(clusteringProvider = Some("customProvider"))) txn.commit(files1.toIndexedSeq, DeltaOperations.ManualUpdate) files1 = getFiles(table) assert(files1.size === 2) for (f <- files1) { assert(f.clusteringProvider.contains("customProvider")) } addFiles(table, numFiles = 4) assert(getFiles(table).size == 6) withSQLConf( // Set an extreme value to make all zcubes unstable. DeltaSQLConf.DELTA_OPTIMIZE_CLUSTERING_MIN_CUBE_SIZE.key -> Long.MaxValue.toString) { runOptimize(table) { metrics => assert(metrics.clusteringStats.nonEmpty) assert(metrics.numFilesRemoved == 4) assert(metrics.numFilesAdded == 2) validateClusteringMetrics( actualMetrics = metrics.clusteringStats.get, expectedMetrics = ClusteringStats( inputZCubeFiles = ClusteringFileStats(2, SKIP_CHECK_SIZE_VALUE), inputOtherFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), inputNumZCubes = 1, mergedFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), numOutputZCubes = 1)) } } val files2 = getFiles(table) assert(files2.size === 4) assert(files2.forall { file => val zCubeInfo = ZCubeInfo.getForFile(file) zCubeInfo.nonEmpty }) assert(getZCubeIds(table).size == 2) // validate files with different clusteringProvider are not re-clustered. assert(files2.intersect(files1) === files1) // OPTIMIZE FULL should re-cluster previously clustered files. withSQLConf( // Force all zcubes stable DeltaSQLConf.DELTA_OPTIMIZE_CLUSTERING_MIN_CUBE_SIZE.key -> 1.toString) { runOptimizeFull(table) { metrics => assert(metrics.clusteringStats.nonEmpty) // Only files with old cluster keys are rewritten. assert(metrics.numFilesRemoved == 2) assert(metrics.numFilesAdded == 2) validateClusteringMetrics( actualMetrics = metrics.clusteringStats.get, expectedMetrics = ClusteringStats( inputZCubeFiles = ClusteringFileStats(4, SKIP_CHECK_SIZE_VALUE), inputOtherFiles = ClusteringFileStats(0, SKIP_CHECK_SIZE_VALUE), inputNumZCubes = 2, mergedFiles = ClusteringFileStats(2, SKIP_CHECK_SIZE_VALUE), numOutputZCubes = 1)) } } // all files have same clustering provider. assert(getFiles(table).forall { f => f.clusteringProvider.contains(ClusteredTableUtils.clusteringProvider) }) } } } // Test to validate OPTIMIZE FULL is only applied to a clustered table with non-empty clustering // columns. test("OPTIMIZE FULL - error cases") { withTable(table) { sql(s"CREATE TABLE $table(col1 INT, col2 INT, col3 INT) using delta") val e = intercept[DeltaUnsupportedOperationException] { sql(s"OPTIMIZE $table FULL") } checkError(e, "DELTA_OPTIMIZE_FULL_NOT_SUPPORTED") } withClusteredTable(table, "col1 INT, col2 INT, col3 INT", "col1") { sql(s"ALTER TABLE $table CLUSTER BY NONE") val e = intercept[DeltaUnsupportedOperationException] { sql(s"OPTIMIZE $table FULL") } checkError(e, "DELTA_OPTIMIZE_FULL_NOT_SUPPORTED") } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/sources/DeltaSourceMetadataEvolutionSupportSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.sources import org.apache.spark.sql.delta.{DeltaColumnMapping, DeltaOptions, DeltaTestUtilsBase, DeltaThrowable} import org.apache.spark.{SparkConf, SparkFunSuite} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.StructType /** * Unit tests covering `DeltaSourceMetadataEvolutionSupport`, which detects non-additive schema * changes when reading from a Delta source and checks user provided confs to decide whether to * allow resuming streaming processing. */ class DeltaSourceMetadataEvolutionSupportSuite extends SparkFunSuite with SharedSparkSession with DeltaTestUtilsBase { protected override def sparkConf: SparkConf = super.sparkConf .set(DeltaSQLConf.DELTA_ALLOW_TYPE_WIDENING_STREAMING_SOURCE.key, "true") .set(DeltaSQLConf.DELTA_TYPE_WIDENING_BYPASS_STREAMING_TYPE_CHANGE_CHECK.key, "false") private def persistedMetadata( schemaDDL: String, physicalNames: Map[Seq[String], String]): PersistedMetadata = { var schemaWithPhysicalNames = StructType.fromDDL(schemaDDL) schemaWithPhysicalNames = DeltaColumnMapping.setPhysicalNames( schema = schemaWithPhysicalNames, physicalNames ) schemaWithPhysicalNames = DeltaColumnMapping.assignPhysicalNames( schemaWithPhysicalNames, reuseLogicalName = true) PersistedMetadata( tableId = "tableId", deltaCommitVersion = 0, dataSchemaJson = schemaWithPhysicalNames.json, partitionSchemaJson = "", sourceMetadataPath = "sourceMetadataPath" ) } private def expectColumnMappingChangeBlocked(opType: String): ExpectedResult[Nothing] = ExpectedResult.Failure(ex => { assert( ex.getErrorClass === "DELTA_STREAMING_CANNOT_CONTINUE_PROCESSING_POST_SCHEMA_EVOLUTION") assert(ex.getMessageParameters.get("opType") === opType) }) private def expectTypeWideningBlocked(wideningTypeChanges: Seq[String]): ExpectedResult[Nothing] = ExpectedResult.Failure(ex => { assert( ex.getErrorClass === "DELTA_STREAMING_CANNOT_CONTINUE_PROCESSING_POST_SCHEMA_EVOLUTION") assert(ex.getMessageParameters.get("columnChangeDetails") .contains(wideningTypeChanges.mkString("\n"))) }) private def expectNonWideningTypeChangeError: ExpectedResult[Nothing] = ExpectedResult.Failure(ex => { assert( ex.getErrorClass === "DELTA_SCHEMA_CHANGED_WITH_VERSION") }) private def withSQLConfUnblockedChanges(unblock: Seq[String])(f: => Unit): Unit = { val confs = unblock.map( conf => s"spark.databricks.delta.streaming.$conf" -> "always") withSQLConf(confs: _*) { f } } /** * Unit test runner covering `validateIfSchemaChangeCanBeUnblocked()`. Takes as input * an initial schema (from) and an updated schema (to) and checks that: * 1. Non-additive schema changes are correctly detected: matches `expectedResult` * 2. Setting SQL confs to unblock the changes allows the check to succeeds. * @param name Name of the test. * @param fromDDL Initial schema, as a DDL string: 'a INT' * @param fromPhysicalNames Physical column/field names for the initial schema: assigning * physical names that are different than the logical names in * `fromDDL` allows simulating column mapping operations: DROP, RENAME. * @param toDDL Updated schema, as a DDL string. * @param toPhysicalNames Physical column/field names for the updated schema * @param expectedResult Expected result, either failure or success. In case of failure, a * check to apply on the returned expression can be passed. * @param unblock SQL confs to unblock the schema change. Each entry is a set of SQL * confs which together allow the schema change to be unblocked. There * can be multiple such sets, e.g. * [[allowSourceColumnDrop], [allowSourceColumnRenameAndDrop]] as both * allow dropping columns independently. * @param confs Additional SQL confs to set when running the test. */ private def testSchemaChange( name: String, fromDDL: String, fromPhysicalNames: Map[Seq[String], String] = Map.empty, toDDL: String, toPhysicalNames: Map[Seq[String], String] = Map.empty, expectedResult: ExpectedResult[Nothing], unblock: Seq[Seq[String]] = Seq.empty, confs: Seq[(String, String)] = Seq.empty): Unit = test(s"$name") { def validate(parameters: Map[String, String]): Unit = DeltaSourceMetadataEvolutionSupport.validateIfSchemaChangeCanBeUnblocked( spark, parameters, metadataPath = "sourceMetadataPath", currentSchema = persistedMetadata(toDDL, toPhysicalNames), previousSchema = persistedMetadata(fromDDL, fromPhysicalNames) ) withSQLConf(confs: _*) { expectedResult match { case ExpectedResult.Success(_) => validate(parameters = Map.empty) case ExpectedResult.Failure(checkError) => // Run first without setting any configuration to unblock and check that the validation // fails => column dropped, renamed or with changed type. val ex = intercept[DeltaThrowable] { validate(parameters = Map.empty) } checkError(ex) // Verify that we can unblock using SQL confs for (u <- unblock) { withSQLConfUnblockedChanges(u) { validate(parameters = Map.empty) } } // Verify that we can unblock using dataframe reader options. for (u <- unblock) { val parameters = u.flatMap { case "allowSourceColumnRenameAndDrop" => Seq(DeltaOptions.ALLOW_SOURCE_COLUMN_RENAME -> "always", DeltaOptions.ALLOW_SOURCE_COLUMN_DROP -> "always") case option => Seq(option -> "always") } validate(parameters.toMap) } } } } testSchemaChange( name = "no schema change, use logical names", fromDDL = "a int", toDDL = "a int", expectedResult = ExpectedResult.Success() ) testSchemaChange( name = "no schema change, use physical names", fromDDL = "a int", fromPhysicalNames = Map(Seq("a") -> "x"), toDDL = "a int", toPhysicalNames = Map(Seq("a") -> "x"), expectedResult = ExpectedResult.Success() ) testSchemaChange( name = "schema overwrite, different column name", fromDDL = "a int", toDDL = "b string", expectedResult = expectColumnMappingChangeBlocked("DROP COLUMN"), unblock = Seq( Seq("allowSourceColumnDrop"), Seq("allowSourceColumnRenameAndDrop") ) ) testSchemaChange( name = "schema overwrite, same column name, non-widening type change", fromDDL = "a int", toDDL = "a string", toPhysicalNames = Map(Seq("a") -> "b"), expectedResult = expectColumnMappingChangeBlocked("DROP COLUMN"), unblock = Seq( Seq("allowSourceColumnDrop"), Seq("allowSourceColumnRenameAndDrop") ) ) testSchemaChange( name = "schema overwrite, same column name, widening type change", fromDDL = "a byte", toDDL = "a int", toPhysicalNames = Map(Seq("a") -> "b"), expectedResult = expectColumnMappingChangeBlocked("DROP COLUMN"), unblock = Seq( Seq("allowSourceColumnDrop"), Seq("allowSourceColumnRenameAndDrop") ) ) ///////////////// // Rename column ///////////////// testSchemaChange( name = "column rename, use logical names", fromDDL = "a int", toDDL = "b int", toPhysicalNames = Map(Seq("b") -> "a"), expectedResult = expectColumnMappingChangeBlocked("RENAME COLUMN"), unblock = Seq( Seq("allowSourceColumnRename"), Seq("allowSourceColumnRenameAndDrop") ) ) testSchemaChange( name = "column rename, use physical names", fromDDL = "a int", fromPhysicalNames = Map(Seq("a") -> "x"), toDDL = "b int", toPhysicalNames = Map(Seq("b") -> "x"), expectedResult = expectColumnMappingChangeBlocked("RENAME COLUMN"), unblock = Seq( Seq("allowSourceColumnRename"), Seq("allowSourceColumnRenameAndDrop") ) ) testSchemaChange( name = "column rename with widening type change", fromDDL = "a byte", fromPhysicalNames = Map(Seq("a") -> "x"), toDDL = "b int", toPhysicalNames = Map(Seq("b") -> "x"), expectedResult = expectColumnMappingChangeBlocked("RENAME AND TYPE WIDENING"), unblock = Seq( Seq("allowSourceColumnRename", "allowSourceColumnTypeChange"), Seq("allowSourceColumnRenameAndDrop", "allowSourceColumnTypeChange") ) ) testSchemaChange( name = "column rename with non-widening type change", fromDDL = "a int", fromPhysicalNames = Map(Seq("a") -> "x"), toDDL = "b string", toPhysicalNames = Map(Seq("b") -> "x"), expectedResult = expectNonWideningTypeChangeError ) testSchemaChange( name = "swap columns", fromDDL = "a int, b int", toDDL = "b int, a int", toPhysicalNames = Map(Seq("b") -> "a", Seq("a") -> "b"), expectedResult = expectColumnMappingChangeBlocked("RENAME COLUMN"), unblock = Seq( Seq("allowSourceColumnRename"), Seq("allowSourceColumnRenameAndDrop") ) ) testSchemaChange( name = "swap columns with widening type change", fromDDL = "a byte, b byte", toDDL = "b byte, a int", toPhysicalNames = Map(Seq("b") -> "a", Seq("a") -> "b"), expectedResult = expectColumnMappingChangeBlocked("RENAME AND TYPE WIDENING"), unblock = Seq( Seq("allowSourceColumnRename", "allowSourceColumnTypeChange"), Seq("allowSourceColumnRenameAndDrop", "allowSourceColumnTypeChange") ) ) testSchemaChange( name = "swap columns with non-widening type change", fromDDL = "a int, b int", toDDL = "b int, a string", toPhysicalNames = Map(Seq("b") -> "a", Seq("a") -> "b"), expectedResult = expectNonWideningTypeChangeError ) testSchemaChange( name = "swap columns with widening and non-widening type change", fromDDL = "a byte, b int", toDDL = "b int, a string", toPhysicalNames = Map(Seq("b") -> "a", Seq("a") -> "b"), expectedResult = expectNonWideningTypeChangeError ) ///////////////// // Drop column ///////////////// testSchemaChange( name = "drop column, use logical names", fromDDL = "a int, b int", toDDL = "b int", expectedResult = expectColumnMappingChangeBlocked("DROP COLUMN"), unblock = Seq( Seq("allowSourceColumnDrop", "allowSourceColumnTypeChange"), Seq("allowSourceColumnRenameAndDrop", "allowSourceColumnTypeChange") ) ) testSchemaChange( name = "drop column, use physical names", fromDDL = "a int, b int", fromPhysicalNames = Map(Seq("a") -> "x", Seq("b") -> "y"), toDDL = "b int", toPhysicalNames = Map(Seq("b") -> "y"), expectedResult = expectColumnMappingChangeBlocked("DROP COLUMN"), unblock = Seq( Seq("allowSourceColumnDrop", "allowSourceColumnTypeChange"), Seq("allowSourceColumnRenameAndDrop", "allowSourceColumnTypeChange") ) ) testSchemaChange( name = "drop column with widening type change", fromDDL = "a byte, b byte", fromPhysicalNames = Map(Seq("a") -> "x", Seq("b") -> "y"), toDDL = "b int", toPhysicalNames = Map(Seq("b") -> "y"), expectedResult = expectColumnMappingChangeBlocked("DROP AND TYPE WIDENING"), unblock = Seq( Seq("allowSourceColumnDrop", "allowSourceColumnTypeChange"), Seq("allowSourceColumnRenameAndDrop", "allowSourceColumnTypeChange") ) ) testSchemaChange( name = "drop column with non-widening type change", fromDDL = "a int, b int", fromPhysicalNames = Map(Seq("a") -> "x", Seq("b") -> "y"), toDDL = "b string", toPhysicalNames = Map(Seq("b") -> "y"), expectedResult = expectNonWideningTypeChangeError ) testSchemaChange( name = "drop column, swapped physical names", fromDDL = "a int, b int", fromPhysicalNames = Map(Seq("a") -> "b", Seq("b") -> "a"), toDDL = "b int", toPhysicalNames = Map(Seq("b") -> "a"), expectedResult = expectColumnMappingChangeBlocked("DROP COLUMN"), unblock = Seq( Seq("allowSourceColumnDrop", "allowSourceColumnTypeChange"), Seq("allowSourceColumnRenameAndDrop", "allowSourceColumnTypeChange") ) ) testSchemaChange( name = "drop column, swapped physical names with widening type change", fromDDL = "a byte, b byte", fromPhysicalNames = Map(Seq("a") -> "b", Seq("b") -> "a"), toDDL = "b int", toPhysicalNames = Map(Seq("b") -> "a"), expectedResult = expectColumnMappingChangeBlocked("DROP AND TYPE WIDENING"), unblock = Seq( Seq("allowSourceColumnDrop", "allowSourceColumnTypeChange"), Seq("allowSourceColumnRenameAndDrop", "allowSourceColumnTypeChange") ) ) testSchemaChange( name = "drop column, swapped physical names with non-widening type change", fromDDL = "a int, b int", fromPhysicalNames = Map(Seq("a") -> "b", Seq("b") -> "a"), toDDL = "b float", toPhysicalNames = Map(Seq("b") -> "a"), expectedResult = expectNonWideningTypeChangeError ) ////////////////////////// // Drop and rename column ////////////////////////// testSchemaChange( name = "drop column, rename to other column name", fromDDL = "a int, b int", toDDL = "c int", toPhysicalNames = Map(Seq("c") -> "b"), expectedResult = expectColumnMappingChangeBlocked("RENAME AND DROP COLUMN"), unblock = Seq( Seq("allowSourceColumnRename", "allowSourceColumnDrop"), Seq("allowSourceColumnRenameAndDrop") ) ) testSchemaChange( name = "drop column, rename to other column name with widening type change", fromDDL = "a byte, b byte", toDDL = "c int", toPhysicalNames = Map(Seq("c") -> "b"), // We don't block the type change itself here as the column is also renamed expectedResult = expectColumnMappingChangeBlocked("RENAME, DROP AND TYPE WIDENING"), unblock = Seq( Seq("allowSourceColumnRename", "allowSourceColumnDrop", "allowSourceColumnTypeChange"), Seq("allowSourceColumnRenameAndDrop", "allowSourceColumnTypeChange") ) ) testSchemaChange( name = "drop column, rename to other column name with non-widening type change", fromDDL = "a int, b int", toDDL = "c string", toPhysicalNames = Map(Seq("c") -> "b"), expectedResult = expectNonWideningTypeChangeError ) testSchemaChange( name = "drop column, rename to same column name", fromDDL = "a int, b int", toDDL = "a int", toPhysicalNames = Map(Seq("a") -> "b"), expectedResult = expectColumnMappingChangeBlocked("RENAME AND DROP COLUMN"), unblock = Seq( Seq("allowSourceColumnRename", "allowSourceColumnDrop"), Seq("allowSourceColumnRenameAndDrop") ) ) testSchemaChange( name = "drop column, rename to same column name with widening type change", fromDDL = "a byte, b byte", toDDL = "a int", toPhysicalNames = Map(Seq("a") -> "b"), expectedResult = expectColumnMappingChangeBlocked("RENAME, DROP AND TYPE WIDENING"), unblock = Seq( Seq("allowSourceColumnRename", "allowSourceColumnDrop", "allowSourceColumnTypeChange"), Seq("allowSourceColumnRenameAndDrop", "allowSourceColumnTypeChange") ) ) testSchemaChange( name = "drop column, rename to same column name with non-widening type change", fromDDL = "a int, b int", toDDL = "a string", toPhysicalNames = Map(Seq("a") -> "b"), expectedResult = expectNonWideningTypeChangeError ) //////////////// // Type changes //////////////// testSchemaChange( name = "widen single column", fromDDL = "a byte", toDDL = "a int", expectedResult = expectTypeWideningBlocked(Seq("'a': TINYINT -> INT")), unblock = Seq( Seq("allowSourceColumnTypeChange") ) ) testSchemaChange( name = "widening and non-widening type changes", fromDDL = "a byte, b int", toDDL = "a int, b byte", expectedResult = expectNonWideningTypeChangeError ) testSchemaChange( name = "widen single column with type widening disabled in Delta source", fromDDL = "a byte", toDDL = "a int", expectedResult = expectNonWideningTypeChangeError, confs = Seq(DeltaSQLConf.DELTA_ALLOW_TYPE_WIDENING_STREAMING_SOURCE.key -> "false") ) testSchemaChange( name = "widen single column with type change check disabled", fromDDL = "a byte", toDDL = "a int", expectedResult = ExpectedResult.Success(), confs = Seq(DeltaSQLConf.DELTA_TYPE_WIDENING_BYPASS_STREAMING_TYPE_CHANGE_CHECK.key -> "true") ) testSchemaChange( name = "widen single column with both type widening and type change check disabled", fromDDL = "a byte", toDDL = "a int", expectedResult = ExpectedResult.Success(), confs = Seq( DeltaSQLConf.DELTA_ALLOW_TYPE_WIDENING_STREAMING_SOURCE.key -> "false", DeltaSQLConf.DELTA_TYPE_WIDENING_BYPASS_STREAMING_TYPE_CHANGE_CHECK.key -> "true" ) ) testSchemaChange( name = "narrow single column", fromDDL = "a long", toDDL = "a int", expectedResult = expectNonWideningTypeChangeError ) testSchemaChange( name = "narrow single column with type change check disabled", fromDDL = "a long", toDDL = "a int", expectedResult = ExpectedResult.Success(), confs = Seq(DeltaSQLConf.DELTA_TYPE_WIDENING_BYPASS_STREAMING_TYPE_CHANGE_CHECK.key -> "true") ) testSchemaChange( name = "change to nullable", fromDDL = "a int not null", toDDL = "a int", expectedResult = ExpectedResult.Success() ) testSchemaChange( name = "change to non-nullable", fromDDL = "a int", toDDL = "a int not null", expectedResult = expectNonWideningTypeChangeError ) testSchemaChange( name = "widen and change to nullable", fromDDL = "a byte not null", toDDL = "a int", expectedResult = expectTypeWideningBlocked(Seq("'a': TINYINT -> INT")), unblock = Seq( Seq("allowSourceColumnTypeChange") ) ) testSchemaChange( name = "widen change to non-nullable", fromDDL = "a byte", toDDL = "a int not null", expectedResult = expectNonWideningTypeChangeError ) testSchemaChange( name = "widen map", fromDDL = "a map", toDDL = "a map", expectedResult = expectTypeWideningBlocked(Seq( "'a.key': TINYINT -> SMALLINT", "'a.value': SMALLINT -> INT" )), unblock = Seq( Seq("allowSourceColumnTypeChange") ) ) testSchemaChange( name = "widen array", fromDDL = "a array", toDDL = "a array", expectedResult = expectTypeWideningBlocked(Seq("'a.element': TINYINT -> SMALLINT")), unblock = Seq( Seq("allowSourceColumnTypeChange") ) ) testSchemaChange( name = "widen struct", fromDDL = "a struct", toDDL = "a struct", expectedResult = expectTypeWideningBlocked(Seq("'a.x': TINYINT -> SMALLINT")), unblock = Seq( Seq("allowSourceColumnTypeChange") ) ) testSchemaChange( name = "widen struct and struct field rename", fromDDL = "a struct", toDDL = "a struct", toPhysicalNames = Map(Seq("a", "z") -> "y"), expectedResult = expectColumnMappingChangeBlocked("RENAME AND TYPE WIDENING"), unblock = Seq( Seq("allowSourceColumnRename", "allowSourceColumnTypeChange"), Seq("allowSourceColumnRenameAndDrop", "allowSourceColumnTypeChange") ) ) testSchemaChange( name = "widen struct and struct field drop", fromDDL = "a struct", toDDL = "a struct", expectedResult = expectColumnMappingChangeBlocked("DROP AND TYPE WIDENING"), unblock = Seq( Seq("allowSourceColumnDrop", "allowSourceColumnTypeChange"), Seq("allowSourceColumnRenameAndDrop", "allowSourceColumnTypeChange") ) ) testSchemaChange( name = "widen struct and struct field rename and drop", fromDDL = "a struct", toDDL = "a struct", toPhysicalNames = Map(Seq("a", "w") -> "y"), expectedResult = expectColumnMappingChangeBlocked("RENAME, DROP AND TYPE WIDENING"), unblock = Seq( Seq("allowSourceColumnRename", "allowSourceColumnDrop", "allowSourceColumnTypeChange"), Seq("allowSourceColumnRenameAndDrop", "allowSourceColumnTypeChange") ) ) test("combining individual SQL confs to unblock is supported") { withSQLConfUnblockedChanges(Seq("allowSourceColumnRename", "allowSourceColumnDrop")) { DeltaSourceMetadataEvolutionSupport.validateIfSchemaChangeCanBeUnblocked( spark, parameters = Map.empty, metadataPath = "sourceMetadataPath", currentSchema = persistedMetadata("a int", Map(Seq("a") -> "b")), previousSchema = persistedMetadata("a int, b int", Map.empty) ) } } test("combining SQL confs and reader options to unblock is supported") { withSQLConfUnblockedChanges(Seq("allowSourceColumnRename")) { DeltaSourceMetadataEvolutionSupport.validateIfSchemaChangeCanBeUnblocked( spark, parameters = Map("allowSourceColumnDrop" -> "always"), metadataPath = "sourceMetadataPath", currentSchema = persistedMetadata("a int", Map(Seq("a") -> "b")), previousSchema = persistedMetadata("a int, b int", Map.empty) ) } } test("unblocking column drop for specific version with reader option is supported") { DeltaSourceMetadataEvolutionSupport.validateIfSchemaChangeCanBeUnblocked( spark, parameters = Map("allowSourceColumnDrop" -> "0"), metadataPath = "sourceMetadataPath", currentSchema = persistedMetadata("a int", Map.empty), previousSchema = persistedMetadata("a int, b int", Map.empty) ) } test("unblocking column rename for specific version with reader option is supported") { DeltaSourceMetadataEvolutionSupport.validateIfSchemaChangeCanBeUnblocked( spark, parameters = Map("allowSourceColumnRename" -> "0"), metadataPath = "sourceMetadataPath", currentSchema = persistedMetadata("b int", Map(Seq("b") -> "a")), previousSchema = persistedMetadata("a int", Map.empty) ) } test("unblocking column type change for specific version with reader option is supported") { DeltaSourceMetadataEvolutionSupport.validateIfSchemaChangeCanBeUnblocked( spark, parameters = Map("allowSourceColumnTypeChange" -> "0"), metadataPath = "sourceMetadataPath", currentSchema = persistedMetadata("a int", Map.empty), previousSchema = persistedMetadata("a byte", Map.empty) ) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/stats/DataSkippingDeltaConstructDataFiltersSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.types.StringType class DataSkippingDeltaConstructDataFiltersSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { test("Verify constructDataFilters doesn't hang for expressions with Literal operands.") { val snapshot = DeltaLog.forTable(spark, "dummy_path").update() val dataFilterBuilder = new snapshot.DataFiltersBuilder( spark, DeltaDataSkippingType.dataSkippingOnlyV1) val literal = Literal.create("foo", StringType) Seq( EqualTo(literal, literal), Not(EqualTo(literal, literal)), EqualNullSafe(literal, literal), Not(EqualNullSafe(literal, literal)), LessThan(literal, literal), LessThanOrEqual(literal, literal), GreaterThan(literal, literal), GreaterThanOrEqual(literal, literal), Not(GreaterThanOrEqual(literal, literal)), In(literal, Seq(literal)), IsNull(literal), IsNotNull(literal), And(EqualTo(literal, literal), LessThan(literal, literal)) ).foreach { expression => assert(dataFilterBuilder.constructDataFilters(expression, isNullExpansionDepth = 0).isEmpty) } } test("Test when the query contains EqualTo(Literal, Literal) in the filter.") { setup { sql( """ |explain |select | * |from | view1 c | join view2 cv on c.type=cv.type and c.key=cv.key | join tbl3 b on cv.name=b.name |where | ( | (b.name="name1" and c.type="foo") | or | (b.name="name2" and c.type="bar") | ) |""".stripMargin) } } test("Verify areAllLeavesLiteral can't be recursively called b/c it can cause stack overflow") { val snapshot = DeltaLog.forTable(spark, "dummy_path").update() val dataFilterBuilder = new snapshot.DataFiltersBuilder( spark, DeltaDataSkippingType.dataSkippingOnlyV1) // Create a deeply nested Alias expression: Alias(Alias(Alias(...), "name3"), "name2"), "name1") val depth = 100000 // Deep enough to cause stack overflow with default JVM stack size var nestedExpr: Expression = Literal.create("foo", StringType) for (i <- 1 to depth) { nestedExpr = Alias(nestedExpr, s"name$i")() } assert(dataFilterBuilder.areAllLeavesLiteral(nestedExpr)) // This should cause a StackOverflowError when areAllLeavesLiteral is called recursively def areAllLeavesLiteral(e: Expression): Boolean = e match { case _: Literal => true case _ if e.children.nonEmpty => e.children.forall(areAllLeavesLiteral) case _ => false } intercept[StackOverflowError] { areAllLeavesLiteral(nestedExpr) } } private def setup(f: => Unit) { withTable("tbl1_foo", "tbl1_bar", "tbl2_foo", "tbl2_bar", "tbl3") { Seq("foo", "bar").foreach { tableType => sql(s"CREATE TABLE tbl1_$tableType (key STRING) USING delta") sql(s"CREATE TABLE tbl2_$tableType (key STRING, name STRING) USING delta") } sql("CREATE TABLE tbl3 (name STRING) USING delta") withView("view1", "view2") { sql( s""" |CREATE VIEW view1 (type, key) |AS ( | select 'foo' as type, * from tbl1_foo | union all | select 'bar' as type, * from tbl1_bar |) |""".stripMargin ) sql( s""" |CREATE VIEW view2 (type, key, name) |AS ( | select 'foo' as type, * from tbl2_foo | union all | select 'bar' as type, * from tbl2_bar |) |""".stripMargin ) f } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/stats/DataSkippingDeltaTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats import java.io.File import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite import org.apache.spark.sql.delta.metering.ScanReport import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.test.ScanReportHelper import org.apache.commons.io.FileUtils import org.apache.hadoop.fs.Path import org.scalatest.GivenWhenThen // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.SparkConf import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.QueryPlanningTracker import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.expressions.{Expression, Literal, PredicateHelper} import org.apache.spark.sql.catalyst.plans.logical.Filter import org.apache.spark.sql.functions.{col, lit} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ import org.apache.spark.util.Utils trait DataSkippingDeltaTestsBase extends QueryTest with SharedSparkSession with DeltaSQLCommandTest with DataSkippingDeltaTestsUtils with GivenWhenThen with ScanReportHelper with CatalogOwnedTestBaseSuite with DeltaSQLTestUtils { val defaultNumIndexedCols = DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.fromString( DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.defaultValue) import testImplicits._ protected def checkpointAndCreateNewLogIfNecessary(log: DeltaLog): DeltaLog = log protected val tableSchemaOnlyTag = org.scalatest.Tag("StatsCollectionWithTableSchemaOnly") /** * Test stats collection using both the table schema and DataFrame schema (if applicable) * TODO(lin): remove this after we remove the DELTA_COLLECT_STATS_USING_TABLE_SCHEMA flag */ protected override def test(testName: String, testTags: org.scalatest.Tag*) (testFun: => Any) (implicit pos: org.scalactic.source.Position): Unit = { super.test(testName, testTags : _*)(testFun)(pos) if (!testTags.contains(tableSchemaOnlyTag)) { super.test(testName + " - old behavior with DataFrame schema", testTags: _*) { withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS_USING_TABLE_SCHEMA.key -> "false") { testFun } } } } testSkipping( "top level, single 1", """{"a": 1}""", hits = Seq( "True", // trivial base case "a = 1", "a <=> 1", "a >= 1", "a <= 1", "a <= 2", "a >= 0", "1 = a", "1 <=> a", "1 <= a", "1 >= a", "2 >= a", "0 <= a", "NOT a <=> 2" ), misses = Seq( "NOT a = 1", "NOT a <=> 1", "a = 2", "a <=> 2", "a != 1", "2 = a", "2 <=> a", "1 != a", "a > 1", "a < 1", "a >= 2", "a <= 0", "1 < a", "1 > a", "2 <= a", "0 >= a" ) ) testSkipping( "nested, single 1", """{"a": {"b": 1}}""", hits = Seq( "a.b = 1", "a.b >= 1", "a.b <= 1", "a.b <= 2", "a.b >= 0" ), misses = Seq( "a.b = 2", "a.b > 1", "a.b < 1" ) ) testSkipping( "double nested, single 1", """{"a": {"b": {"c": 1}}}""", hits = Seq( "a.b.c = 1", "a.b.c >= 1", "a.b.c <= 1", "a.b.c <= 2", "a.b.c >= 0" ), misses = Seq( "a.b.c = 2", "a.b.c > 1", "a.b.c < 1" ) ) private def longString(str: String) = str * 1000 testSkipping( "long strings - long min", s""" {"a": '${longString("A")}'} {"a": 'B'} {"a": 'C'} """, hits = Seq( "a like 'A%'", s"a = '${longString("A")}'", "a > 'BA'", "a < 'AB'" ), misses = Seq( "a < 'AA'", "a > 'CD'" ) ) testSkipping( "long strings - long max", s""" {"a": 'A'} {"a": 'B'} {"a": '${longString("C")}'} """, hits = Seq( "a like 'A%'", "a like 'C%'", s"a = '${longString("C")}'", "a > 'BA'", "a < 'AB'", "a > 'CC'" ), misses = Seq( "a >= 'D'", "a > 'CD'" ) ) testSkipping( "starts with", """ {"a": 'apple'} {"a": 'microsoft'} """, hits = Seq( "a like 'a%'", "a like 'ap%'", "a like 'm%'", "a like 'mic%'", "a like '%'" ), misses = Seq( "a like 'xyz%'" ) ) testSkipping( "starts with, nested", """ {"a":{"b": 'apple'}} {"a":{"b": 'microsoft'}} """, hits = Seq( "a.b like 'a%'", "a.b like 'ap%'", "a.b like 'm%'", "a.b like 'mic%'", "a.b like '%'" ), misses = Seq( "a.b like 'xyz%'" ) ) testSkipping( "and statements - simple", """ {"a": 1} {"a": 2} """, hits = Seq( "a > 0 AND a < 3", "a <= 1 AND a > -1" ), misses = Seq( "a < 0 AND a > -2" ) ) testSkipping( "and statements - two fields", """ {"a": 1, "b": "2017-09-01"} {"a": 2, "b": "2017-08-31"} """, hits = Seq( "a > 0 AND b = '2017-09-01'", "a = 2 AND b >= '2017-08-30'", "a >= 2 AND b like '2017-08-%'" ), misses = Seq( "a > 0 AND b like '2016-%'" ) ) // One side of AND by itself still has pruning power. testSkipping( "and statements - one side unsupported", """ {"a": 10, "b": 10} {"a": 20: "b": 20} """, hits = Seq( "a % 100 < 10 AND b % 100 > 20" ), misses = Seq( "a < 10 AND b % 100 > 20", "a % 100 < 10 AND b > 20" ) ) testSkipping( "or statements - simple", """ {"a": 1} {"a": 2} """, hits = Seq( "a > 0 or a < -3", "a >= 2 or a < -1" ), misses = Seq( "a > 5 or a < -2" ) ) testSkipping( "or statements - two fields", """ {"a": 1, "b": "2017-09-01"} {"a": 2, "b": "2017-08-31"} """, hits = Seq( "a < 0 or b = '2017-09-01'", "a = 2 or b < '2017-08-30'", "a < 2 or b like '2017-08-%'", "a >= 2 or b like '2016-08-%'" ), misses = Seq( "a < 0 or b like '2016-%'" ) ) // One side of OR by itself isn't powerful enough to prune any files. testSkipping( "or statements - one side unsupported", """ {"a": 10, "b": 10} {"a": 20: "b": 20} """, hits = Seq( "a % 100 < 10 OR b > 20", "a < 10 OR b % 100 > 20" ), misses = Seq( "a < 10 OR b > 20" ) ) testSkipping( "not statements - simple", """ {"a": 1} {"a": 2} """, hits = Seq( "not a < 0" ), misses = Seq( "not a > 0" ) ) // NOT(AND(a, b)) === OR(NOT(a), NOT(b)) ==> One side by itself cannot prune. testSkipping( "not statements - and", """ {"a": 10, "b": 10} {"a": 20: "b": 20} """, hits = Seq( "NOT(a % 100 >= 10 AND b % 100 <= 20)", "NOT(a >= 10 AND b % 100 <= 20)", "NOT(a % 100 >= 10 AND b <= 20)" ), misses = Seq( "NOT(a >= 10 AND b <= 20)" ) ) // NOT(OR(a, b)) === AND(NOT(a), NOT(b)) => One side by itself is enough to prune. testSkipping( "not statements - or", """ {"a": 1, "b": 10} {"a": 2, "b": 20} """, hits = Seq( "NOT(a < 1 OR b > 20)", "NOT(a % 100 >= 1 OR b % 100 <= 20)" ), misses = Seq( "NOT(a >= 1 OR b <= 20)", "NOT(a % 100 >= 1 OR b <= 20)", "NOT(a >= 1 OR b % 100 <= 20)" ) ) // If a column does not have stats, it does not participate in data skipping, which disqualifies // that leg of whatever conjunct it was part of. testSkipping( "missing stats columns", """ {"a": 1, "b": 10} {"a": 2, "b": 20} """, hits = Seq( "b < 10", // disqualified "a < 1 OR b < 10", // a disqualified by b (same conjunct) "a < 1 OR (a >= 1 AND b < 10)" // ==> a < 1 OR a >=1 ==> TRUE ), misses = Seq( "a < 1 AND b < 10", // ==> a < 1 ==> FALSE "a < 1 OR (a > 10 AND b < 10)" // ==> a < 1 OR a > 10 ==> FALSE ), indexedCols = 1 ) private def generateJsonData(numCols: Int): String = { val fields = (0 until numCols).map(i => s""""col${"%02d".format(i)}":$i""".stripMargin) "{" + fields.mkString(",") + "}" } testSkipping( "more columns than indexed", generateJsonData(defaultNumIndexedCols + 1), hits = Seq( "col00 = 0", s"col$defaultNumIndexedCols = $defaultNumIndexedCols", s"col$defaultNumIndexedCols = -1" ), misses = Seq( "col00 = 1" ) ) testSkipping( "nested schema - # indexed column = 3", """{ "a": 1, "b": { "c": { "d": 2, "e": 3, "f": { "g": 4, "h": 5, "i": 6 }, "j": 7, "k": 8 }, "l": 9 }, "m": 10 }""".replace("\n", ""), hits = Seq( "a = 1", "b.c.d = 2", "b.c.e = 3", // below matches due to missing stats "b.c.f.g < 0", "b.c.f.i < 0", "b.l < 0"), misses = Seq( "a < 0", "b.c.d < 0", "b.c.e < 0"), indexedCols = 3 ) testSkipping( "nested schema - # indexed column = 6", """{ "a": 1, "b": { "c": { "d": 2, "e": 3, "f": { "g": 4, "h": 5, "i": 6 }, "j": 7, "k": 8 }, "l": 9 }, "m": 10 }""".replace("\n", ""), hits = Seq( "b.c.f.i = 6", // below matches are due to missing stats "b.c.j < 0", "b.c.k < 0", "b.l < 0"), misses = Seq( "a < 0", "b.c.f.i < 0" ), indexedCols = 6 ) testSkipping( "nested schema - # indexed column = 9", """{ "a": 1, "b": { "c": { "d": 2, "e": 3, "f": { "g": 4, "h": 5, "i": 6 }, "j": 7, "k": 8 }, "l": 9 }, "m": 10 }""".replace("\n", ""), hits = Seq( "b.c.d = 2", "b.c.f.i = 6", "b.l = 9", // below matches are due to missing stats "m < 0"), misses = Seq( "b.l < 0", "b.c.f.i < 0" ), indexedCols = 9 ) testSkipping( "nested schema - # indexed column = 0", """{ "a": 1, "b": { "c": { "d": 2, "e": 3, "f": { "g": 4, "h": 5, "i": 6 }, "j": 7, "k": 8 }, "l": 9 }, "m": 10 }""".replace("\n", ""), hits = Seq( // all included due to missing stats "a < 0", "b.c.d < 0", "b.c.f.i < 0", "b.l < 0", "m < 0"), misses = Seq(), indexedCols = 0 ) testSkipping( "indexed column names - empty list disables stats collection", """{ "a": 1, "b": 2, "c": 3, "d": 4 }""".replace("\n", ""), hits = Seq( "a < 0", "b < 0", "c < 0", "d < 0" ), misses = Seq(), indexedCols = 3, deltaStatsColNamesOpt = Some(" ") ) testSkipping( "indexed column names - naming a nested column indexes all leaf fields of that column", """{ "a": 1, "b": { "c": { "d": 2, "e": 3, "f": { "g": 4, "h": 5, "i": 6 }, "j": 7, "k": 8 }, "l": 9 }, "m": 10 }""".replace("\n", ""), hits = Seq( // these all have missing stats "a < 0", "b.l < 0", "m < 0" ), misses = Seq( "b.c.d < 0", "b.c.e < 0", "b.c.f.g < 0", "b.c.f.h < 0", "b.c.f.i < 0", "b.c.j < 0", "b.c.k < 0" ), indexedCols = 3, deltaStatsColNamesOpt = Some("b.c") ) testSkipping( "indexed column names - naming a nested column allows nested complex types", """{ "a": { "b": [1, 2, 3], "c": [4, 5, 6], "d": 7, "e": 8, "f": { "g": 9 } }, "i": 10 }""".replace("\n", ""), hits = Seq( "i < 0", "a.d > 6", "a.f.g < 10" ), misses = Seq( "a.d < 0", "a.e < 0", "a.f.g < 0" ), deltaStatsColNamesOpt = Some("a") ) testSkipping( "indexed column names - index only a subset of leaf columns", """{ "a": 1, "b": { "c": { "d": 2, "e": 3, "f": { "g": 4, "h": 5, "i": 6 }, "j": 7, "k": 8 }, "l": 9 }, "m": 10 }""".replace("\n", ""), hits = Seq( // these all have missing stats "a < 0", "b.c.d < 0", "b.c.f.g < 0", "b.c.f.i < 0", "b.c.j < 0", "m < 0" ), misses = Seq( "b.c.e < 0", "b.c.f.h < 0", "b.c.k < 0", "b.l < 0" ), indexedCols = 3, deltaStatsColNamesOpt = Some("b.c.e, b.c.f.h, b.c.k, b.l") ) testSkipping( "indexed column names - backtick escapes work as expected", """{ "a": 1, "b.c": 2, "b": { "c": 3, "d": 4 } }""".replace("\n", ""), hits = Seq( "b.c < 0" ), misses = Seq( "a < 0", "`b.c` < 0", "b.d < 0" ), indexedCols = 3, deltaStatsColNamesOpt = Some("`a`, `b.c`, `b`.`d`") ) testSkipping( "boolean comparisons", """{"a": false}""", hits = Seq( "!a", "NOT a", "a", // there is no skipping for BooleanValues "a = false", "NOT a = false", "a > true", "a <= false", "true = a", "true < a", "false = a or a" ), misses = Seq() ) // Data skipping by stats should still work even when the only data in file is null, in spite of // the NULL min/max stats that result -- this is different to having no stats at all. testSkipping( "nulls - only null in file", """ {"a": null } """, schema = new StructType().add(new StructField("a", IntegerType)), hits = Seq( "a IS NULL", "a = NULL", // Ideally this should not hit as it is always FALSE, but its correct to not skip "NOT a = NULL", // Same as previous case "a <=> NULL", // This is optimized to `IsNull(a)` by NullPropagation "TRUE", "FALSE", // Ideally this should not hit, but its correct to not skip "NULL AND a = 1", // This is optimized to FALSE by ReplaceNullWithFalse, so it's same as above "NOT a <=> 1", "(a > 1) IS NULL", // This pushes down the IS NULL to both sides of GreaterThan. "(a > 1 AND a > 0) IS NULL", // Pushdown of IS NULL on AND. "(a > 1 OR a < 0) IS NULL" // Pushdown of IS NULL on OR. ), misses = Seq( // stats tell us a is always NULL, so any predicate that requires non-NULL a should skip "a IS NOT NULL", "NOT a <=> NULL", // This is optimized to `IsNotNull(a)` "a = 1", "NOT a = 1", "a > 1", "a < 1", "a <> 1", "a <=> 1", "NOT ((a > 1) IS NULL)" ) ) testSkipping( "nulls - only non-null in file", """ {"a": 1, "b": 2} """, schema = new StructType() .add(new StructField("a", IntegerType)) .add(new StructField("b", IntegerType)), hits = Seq(), misses = Seq( "(a > 0 AND b > 1) IS NULL", "(a > 0 OR b > 1) IS NULL" ) ) testSkipping( "nulls - only non-null in file with enhanced pushdown disabled", """ {"a": 1, "b": 2} """, schema = new StructType() .add(new StructField("a", IntegerType)) .add(new StructField("b", IntegerType)), hits = Seq( "(a > 0 AND b > 1) IS NULL", "(a > 0 OR b > 1) IS NULL" ), misses = Seq.empty, sqlConfs = Seq((DeltaSQLConf.DELTA_DATASKIPPING_ISNULL_PUSHDOWN_EXPRS_ENABLED.key, "false")) ) testSkipping( "nulls - null + not-null in same file", """ {"a": null } {"a": 1 } """, schema = new StructType().add(new StructField("a", IntegerType)), hits = Seq( "a IS NULL", "a IS NOT NULL", "a = NULL", // Ideally this should not hit as it is always FALSE, but its correct to not skip "NOT a = NULL", // Same as previous case "a <=> NULL", // This is optimized to `IsNull(a)` by NullPropagation "NOT a <=> NULL", // This is optimized to `IsNotNull(a)` "a = 1", "a <=> 1", "TRUE", "FALSE", // Ideally this should not hit, but its correct to not skip "NULL AND a = 1", // This is optimized to FALSE by ReplaceNullWithFalse, so it's same as above "NOT a <=> 1", "(a > 0) IS NULL", "(a < 0) IS NULL", "(a > 1 AND a > 0) IS NULL", // Pushdown of IS NULL on AND. "(a > 1 OR a < 0) IS NULL", // Pushdown of IS NULL on OR. "NOT ((a > 0) IS NULL)" ), misses = Seq( "a <> 1", "a > 1", "a < 1", "NOT a = 1" ) ) testSkipping( "nulls - IsNull pushdown on complex expressions", """ {"a": 1, "b": 2} """, schema = new StructType() .add(new StructField("a", IntegerType)) .add(new StructField("b", IntegerType)), hits = Seq( "(a > 0 OR a == -1 OR a == -2 OR b == -1 OR b == -2 OR b > 10 OR b == 7) IS NULL" ), misses = Seq( "(a > 0 OR b > 1) IS NULL" ), sqlConfs = Seq((DeltaSQLConf.DELTA_DATASKIPPING_ISNULL_PUSHDOWN_EXPRS_MAX_DEPTH.key -> "1")) ) testSkipping( "nulls - non-nulls only in file", """ {"a": 1 } """, schema = new StructType().add(new StructField("a", IntegerType)), hits = Seq( "NOT ((a > 0) IS NULL)" ), misses = Seq( "(a > 0) IS NULL", "(a < 0) IS NULL", "(a > 1 AND a > 0) IS NULL", // Pushdown of IS NULL on AND. "(a > 1 OR a < 0) IS NULL" // Pushdown of IS NULL on OR. ) ) testSkipping( "nulls - non-nulls only in file with partial column stats", """ {"a": 1, "b": 2} """, hits = Seq( "NOT ((a > 0) IS NULL)", "(b > 0) IS NULL", "(b < 0) IS NULL", "(b > 1 AND a > 0) IS NULL", // Pushdown of IS NULL on AND. "(b > 1 OR a < 0) IS NULL" // Pushdown of IS NULL on OR. ), misses = Seq( "(a > 0) IS NULL", "(a < 0) IS NULL", "(a > 1 AND a > 0) IS NULL", // Pushdown of IS NULL on AND. "(a > 1 OR a < 0) IS NULL" // Pushdown of IS NULL on OR. ), indexedCols = 1 ) testSkipping( "nulls - non-strict null-intolerant predicate returns hits for IS NULL", """ {"a": [3, 4]} """, hits = Seq( "NOT (element_at(a, 3) IS NULL)", "element_at(a, 3) IS NULL" ), misses = Seq.empty ) test("data skipping with missing stats") { val tempDir = Utils.createTempDir() Seq(1, 2, 3).toDF().write.format("delta").save(tempDir.toString) val log = DeltaLog.forTable(spark, new Path(tempDir.toString)) val txn = log.startTransaction() val noStats = txn.filterFiles(Nil).map(_.copy(stats = null)) txn.commit(noStats, DeltaOperations.ComputeStats(Nil)) val df = spark.read.format("delta").load(tempDir.toString) checkAnswer(df.where("value > 0"), Seq(Row(1), Row(2), Row(3))) } test("data skipping stats before and after optimize") { assume(!catalogOwnedDefaultCreationEnabledInTests, "OPTIMIZE is blocked on catalog-managed tables") val tempDir = Utils.createTempDir() var r = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) val (numTuples, numFiles) = (10, 2) val data = spark.range(0, numTuples, 1, 2).repartition(numFiles) data.write.format("delta").save(r.dataPath.toString) r = checkpointAndCreateNewLogIfNecessary(r) def rStats: DataFrame = getStatsDf(r, $"numRecords", $"minValues.id".as("id_min"), $"maxValues.id".as("id_max")) checkAnswer(rStats, Seq(Row(4, 0, 8), Row(6, 1, 9))) val optimizeDf = sql(s"OPTIMIZE '$tempDir'") checkAnswer(rStats, Seq(Row(10, 0, 9))) } test("number of indexed columns") { val numTotalCols = defaultNumIndexedCols + 5 val path = Utils.createTempDir().getCanonicalPath var r = DeltaLog.forTable(spark, new Path(path)) val data = spark.range(10).select(Seq.tabulate(numTotalCols)(i => lit(i) as s"col$i"): _*) data.coalesce(1).write.format("delta").save(r.dataPath.toString) def checkNumIndexedCol(numIndexedCols: Int): Unit = { if (defaultNumIndexedCols != numTotalCols) { setNumIndexedColumns(r.dataPath.toString, numIndexedCols) } data.coalesce(1).write.format("delta").mode("overwrite").save(r.dataPath.toString) r = checkpointAndCreateNewLogIfNecessary(r) if (numIndexedCols == 0) { intercept[AnalysisException] { getStatsDf(r, $"numRecords", $"minValues.col0").first() } } else if (numIndexedCols < numTotalCols) { checkAnswer( getStatsDf(r, $"numRecords", $"minValues.col${numIndexedCols - 1}"), Seq(Row(10, numIndexedCols - 1))) intercept[AnalysisException] { getStatsDf(r, $"minValues.col$numIndexedCols").first() } } else { checkAnswer( getStatsDf(r, $"numRecords", $"minValues.col${numTotalCols - 1}"), Seq(Row(10, numTotalCols - 1))) intercept[AnalysisException] { getStatsDf(r, $"minValues.col$numTotalCols").first() } } } checkNumIndexedCol(defaultNumIndexedCols) checkNumIndexedCol(numTotalCols - 1) checkNumIndexedCol(numTotalCols) checkNumIndexedCol(numTotalCols + 1) checkNumIndexedCol(0) } test("remove redundant stats column references in data skipping expression") { withTable("table") { val colNames = (0 to 100).map(i => s"col_$i") sql(s"""CREATE TABLE `table` (${colNames.map(x => x + " INT").mkString(", ")}) using delta""") val conditions = colNames.map(i => s"$i != 1") val whereClause = conditions.mkString("WHERE ", " AND ", "") // This query reproduces the issue raised by running TPC-DS q41. Basically the breaking // condition is when the query involves a big boolean expression. As data skipping // generates many redundant null checks on the non-leaf stats columns, e.g., stats // and stats.minValues, the query complexity is amplified in the data skipping expression. // This fix was to simply apply a distinct() on stats column references before generating // the data skipping expression. sql(s"select col_0 from table $whereClause").collect } } test("data skipping shouldn't use expressions involving a subquery ") { withTable("t1", "t2") { sql(s"CREATE TABLE t1(i int, p string) USING delta partitioned by (i)") sql("INSERT INTO t1 SELECT 1, 'a1'") sql("INSERT INTO t1 SELECT 2, 'a2'") sql("INSERT INTO t1 SELECT 3, 'a3'") sql("INSERT INTO t1 SELECT 4, 'a4'") sql("CREATE TABLE t2(j int, q string) USING delta") sql("INSERT INTO t2 SELECT 1, 'b1'") sql("INSERT INTO t2 SELECT 2, 'b2'") // This query would fail before the fix, i.e., when skipping considers subquery filters. checkAnswer(sql("SELECT i FROM t1 join t2 on i + 2 = j + 1 where q = 'b2'"), Row(1)) // Partition filter with subquery should be ignored for skipping val r1 = getScanReport { checkAnswer( sql("SELECT p from t1 where i in (select j from t2 where q = 'b1')"), Seq(Row("a1"))) } assert(isFullScan(r1(0))) // Partition filter with subquery should be ignored for skipping val r3 = getScanReport { checkAnswer( sql("SELECT p from t1 where i in (select j from t2 where q = 'b1') and p = 'a2'"), Nil) } assert(r3(0).size("scanned").rows === Some(1)) } } test("support case insensitivity for partitioning filters") { withTable("table") { sql(s"CREATE TABLE table(Year int, P string, Y int) USING delta partitioned by (Year)") sql("INSERT INTO table SELECT 1999, 'a1', 1990") sql("INSERT INTO table SELECT 1989, 'a2', 1990") val Seq(r1) = getScanReport { checkAnswer(sql("SELECT * from table where year > 1990"), Row(1999, "a1", 1990)) } assert(!isFullScan(r1)) val Seq(r2) = getScanReport { checkAnswer( sql("SELECT * from table where year > 1990 and p = 'a1'"), Row(1999, "a1", 1990)) } assert(!isFullScan(r2)) val Seq(r3) = getScanReport { checkAnswer(sql("SELECT * from table where p = 'a1'"), Row(1999, "a1", 1990)) } assert(!isFullScan(r3)) checkAnswer(sql("SELECT * from table where year < y"), Row(1989, "a2", 1990)) withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { intercept[AnalysisException] { sql("SELECT * from table where year > 1990") } } } } test("Test file pruning metrics with data skipping") { withTempDir { tempDir => withTempView("t1", "t2") { val data = spark.range(10).toDF("col1") .withColumn("col2", 'col1./(3).cast(DataTypes.IntegerType)) data.write.format("delta").partitionBy("col1") .save(tempDir.getCanonicalPath) spark.read.format("delta").load(tempDir.getAbsolutePath).createTempView("t1") val deltaLog = DeltaLog.forTable(spark, tempDir.toString()) val query = "SELECT * from t1 where col1 > 5" val Seq(r1) = getScanReport { assert(sql(query).collect().length == 4) } val inputFiles = spark.sql(query).inputFiles assert(deltaLog.snapshot.numOfFiles - inputFiles.length == 6) val allQuery = "SELECT * from t1" val Seq(r2) = getScanReport { assert(sql(allQuery).collect().length == 10) } } } } test("loading data from Delta to parquet should skip data") { withTempDir { dir => val path = dir.getCanonicalPath spark.range(5).write.format("delta").save(path) spark.range(5, 10).write.format("delta").mode("append").save(path) withTempDir { dir2 => val path2 = dir2.getCanonicalPath val scans = getScanReport { spark.read.format("delta").load(path).where("id < 2") .write.format("parquet").mode("overwrite").save(path2) } assert(scans.size == 1) assert( scans.head.size("scanned").bytesCompressed != scans.head.size("total").bytesCompressed) } } } test("data skipping with a different DataFrame schema order", tableSchemaOnlyTag) { withTable("table") { sql("CREATE TABLE table (col1 Int, col2 Int, col3 Int) USING delta") val r = DeltaLog.forTable(spark, new TableIdentifier("table")) // Only index the first two columns setNumIndexedColumns(r.dataPath.toString, 2) val dataSeq = Seq((1, 2, 3)) // We should use the table schema to create stats and the DataFrame schema should be ignored dataSeq.toDF("col1", "col2", "col3") .select("col2", "col3", "col1") // DataFrame schema order .write.mode("append").format("delta") .save(r.dataPath.toString) var hits = Seq( "col3 = 10", "col1 = 1", "col2 = 2", "col3 = 3" ) var misses = Seq( "col1 = 5", "col1 = 5 AND col2 = 10", "col1 = 5 and col3 = 10", "col2 = 10", "col2 = 5 and col3 = 10", "col1 = 5 and col2 = 10 and col3 = 10" ) checkSkipping(r, hits, misses, dataSeq.toString(), false) // Change the statsSchema to 3 columns. But there are only two columns in the stats from // the file setNumIndexedColumns(r.dataPath.toString, 3) hits = Seq( "col3 = 3", // 3 is in col3, but no stats "col3 = 10", // No stats on col3 // The data skipping filters will be generated but verifyStatsForFilter will invalidate // the entire predicate "col1 = 5 and col3 = 10" ) misses = Seq( "col1 = 5", "col1 = 5 AND col2 = 10" ) checkSkipping(r, hits, misses, dataSeq.toString(), false) } } test("data skipping with a different DataFrame schema and column name case", tableSchemaOnlyTag) { withTable("table") { sql("CREATE TABLE table (col1 Int, col2 Int, col3 Int) USING delta") val r = DeltaLog.forTable(spark, new TableIdentifier("table")) // Only index the first two columns setNumIndexedColumns(r.dataPath.toString, 2) val dataSeq = Seq((1, 2, 3)) // We should use the table schema to create stats and the DataFrame schema should be ignored dataSeq.toDF("col1", "col2", "col3") .select("COL2", "Col3", "coL1") // DataFrame schema order .write.mode("append").format("delta") .save(r.dataPath.toString) val hits = Seq( "col3 = 10", // No stats for col3 // These values should be in the columns "col1 = 1", "col2 = 2", "col3 = 3" ) val misses = Seq( "col1 = 5", "col1 = 5 AND col2 = 10", "col1 = 5 and col3 = 10", "col2 = 10", "col2 = 5 and col3 = 10", "col1 = 5 and col2 = 10 and col3 = 10" ) checkSkipping(r, hits, misses, dataSeq.toString(), false) } } test("data skipping with a different DataFrame schema order and nested columns", tableSchemaOnlyTag) { withTempDir { dir => val structureData = Seq( Row(Row("James ", "", "Smith"), "36636", "M", 3100) ) val structureDataSchema = new StructType() .add("name", new StructType() .add("firstname", StringType) .add("middlename", StringType) .add("lastname", StringType)) .add("id", StringType) .add("gender", StringType) .add("salary", IntegerType) val data = spark.createDataFrame( spark.sparkContext.parallelize(structureData), structureDataSchema) data.write.partitionBy("id").format("delta").save(dir.getAbsolutePath) // Only index the first three columns (unnested), excluding partition column id val deltaLog = DeltaLog.forTable(spark, new Path(dir.getCanonicalPath)) setNumIndexedColumns(deltaLog.dataPath.toString, 3) val structureDfData = Seq( // The same content as previous row but different DataFrame schema order Row(3100, "M", Row("James ", "", "Smith"), "36636") ) val structureDfSchema = new StructType() .add("salary", IntegerType) .add("gender", StringType) .add("name", new StructType() .add("firstname", StringType) .add("middlename", StringType) .add("lastname", StringType)) .add("id", StringType) // middlename is missing, but we collect NULL_COUNT for it val df = spark.createDataFrame( spark.sparkContext.parallelize(structureDfData), structureDfSchema) df.write.mode("append").format("delta").save(dir.getAbsolutePath) val hits = Seq( // Can't skip them since stats schema only has three columns now "gender = 'M'", "salary = 3100" ) val misses = Seq( "name.firstname = 'Michael'", "name.middlename = 'L'", "name.lastname = 'Miller'", "id = '10000'", "name.firstname = 'Robert' and name.middlename = ''", "name.firstname = 'Robert' and salary = 3100" ) checkSkipping(deltaLog, hits, misses, structureDfData.toString(), false) } } test("compatibility with the old behavior that collect stats based on DataFrame schema", tableSchemaOnlyTag) { withTable("table") { sql("CREATE TABLE table (col2 Int, col3 Int, col1 Int) USING delta") val r = DeltaLog.forTable(spark, new TableIdentifier("table")) // Only index the first two columns setNumIndexedColumns(r.dataPath.toString, 2) val dataSeq = Seq((1, 2, 3)) // Only collect stats for col2 and col3 dataSeq.toDF("col1", "col2", "col3") .select("col2", "col3", "col1") // DataFrame schema order .write.mode("append").format("delta") .save(r.dataPath.toString) // Change the schema to (col1, col2, col3). The final result would be the same as using the // old approach to collect stats based on the DataFrame schema sql("ALTER TABLE table ALTER COLUMN col1 FIRST") // Since the stats schema is (col1, col2), and we only have stats on col2 and col3, only // the predicate on col2 can be used for filters val hits = Seq( "col1 = 1", "col2 = 2", "col3 = 3", "col1 = 5", "col3 = 10", "col1 = 5 AND col2 = 10", "col1 = 5 and col3 = 10", "col1 = 5 and col2 = 10 and col3 = 10" ) val misses = Seq( "col2 = 10", "col2 = 5 and col3 = 10" // This can pass because stats also exists on col3 ) checkSkipping(r, hits, misses, dataSeq.toString(), false) } } // TODO(lin): remove this after we remove the DELTA_COLLECT_STATS_USING_TABLE_SCHEMA flag test("old behavior with DELTA_COLLECT_STATS_USING_TABLE_SCHEMA set to false") { // This force the system restore the old stats collection behavior based on the DataFrame schema withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS_USING_TABLE_SCHEMA.key -> "false") { withTable("table") { sql("CREATE TABLE table (col1 Int, col2 Int, col3 Int) USING delta") val r = DeltaLog.forTable(spark, new TableIdentifier("table")) // Only index the first two columns setNumIndexedColumns(r.dataPath.toString, 2) val dataSeq = Seq((1, 2, 3)) // Only collect stats for col2 and col3 dataSeq.toDF("col1", "col2", "col3") .select("col2", "col3", "col1") // DataFrame schema order .write.mode("append").format("delta") .save(r.dataPath.toString) // Since the stats schema is (col1, col2), and we only have stats on col2 and col3, only // the predicate on col2 can be used for filters val hits = Seq( "col1 = 1", "col2 = 2", "col3 = 3", "col1 = 5", "col3 = 10", "col1 = 5 AND col2 = 10", "col1 = 5 and col3 = 10", "col1 = 5 and col2 = 10 and col3 = 10" ) val misses = Seq( "col2 = 10", "col2 = 5 and col3 = 10" // This can pass because stats also exists on col3 ) checkSkipping(r, hits, misses, dataSeq.toString(), false) } } } test("data skipping with missing columns in DataFrame", tableSchemaOnlyTag) { // case-1: dataframe schema has less columns than the dataSkippingNumIndexedCols withTempTable(createTable = false) { tableName => sql(s"CREATE TABLE $tableName (a Int, b Int, c Int, d Int, e Int) " + "USING delta PARTITIONED BY(b)") val r = DeltaLog.forTable(spark, new TableIdentifier(tableName)) // Only index the first three columns, excluding partition column b setNumIndexedColumns(r.dataPath.toString, 3) val dataSeq = Seq((1, 2, 3, 4, 5)) dataSeq.toDF("a", "b", "c", "d", "e") .select("a", "b") // DataFrame schema order .write.mode("append").format("delta") .save(r.dataPath.toString) val hits = Seq( // These values are in the table "a = 1", "b = 2", "c <=> null", "d is null", // No stats for e "e = 10" ) val misses = Seq( "a = 10", "b = 10", "c = 10", "c is not null", "d = 10", "isnotnull(d)" ) checkSkipping(r, hits, misses, dataSeq.toString(), false) } // case-2: dataframe schema lacks columns that are supposed to be part of the stats schema, // but has an additional column that should not collect stats on withTempTable(createTable = false) { tableName => sql(s"CREATE TABLE $tableName (a Int, b Int, c Int, d Int, e Int) " + "USING delta PARTITIONED BY(b)") val r = DeltaLog.forTable(spark, new TableIdentifier(tableName)) // Only index the first three columns, excluding partition column b setNumIndexedColumns(r.dataPath.toString, 3) val dataSeq = Seq((1, 2, 3, 4, 5)) dataSeq.toDF("a", "b", "c", "d", "e") .select("a", "b", "d", "e") // DataFrame schema order .write.mode("append").format("delta") .save(r.dataPath.toString) val hits = Seq( "a = 1", // In table "isnull(c)", // In table "e = 20" // No stats ) val misses = Seq( "a = 20", "b = 20", "c = 20", "d = 20", "a = 20 and c = 20", "a = 20 and e = 20" ) checkSkipping(r, hits, misses, dataSeq.toString(), false) } // case-3: Structured data with some columns missing and some additional columns withTempDir { dir => val structureData = Seq( Row(Row("James ", "", "Smith"), "36636", "M", 3100) ) val structureDataSchema = new StructType() .add("name", new StructType() .add("firstname", StringType) .add("middlename", StringType) .add("lastname", StringType)) .add("id", StringType) .add("gender", StringType) .add("salary", IntegerType) val data = spark.createDataFrame( spark.sparkContext.parallelize(structureData), structureDataSchema) data.write.partitionBy("id").format("delta").save(dir.getAbsolutePath) // Only index the first three columns (unnested), excluding partition column id val deltaLog = DeltaLog.forTable(spark, new Path(dir.getCanonicalPath)) setNumIndexedColumns(deltaLog.dataPath.toString, 3) val structureDfData = Seq( Row(2000, Row("Robert ", "Johnson"), "40000") ) val structureDfSchema = new StructType() .add("salary", IntegerType) .add("name", new StructType() .add("firstname", StringType) .add("lastname", StringType)) .add("id", StringType) // middlename is missing, but we collect NULL_COUNT for it val df = spark.createDataFrame( spark.sparkContext.parallelize(structureDfData), structureDfSchema) df.write.mode("append").format("delta").save(dir.getAbsolutePath) val hits = Seq( "gender = 'M'", // No stats "salary = 1000" // No stats ) val misses = Seq( "name.firstname = 'Michael'", "name.middlename = 'L'", "name.lastname = 'Miller'", "id = '10000'", "name.firstname = 'Robert' and name.middlename = 'L'" ) checkSkipping(deltaLog, hits, misses, structureDfData.toString(), false) } // case-4: dataframe schema does not have any columns within the first // dataSkippingNumIndexedCols columns of the table schema withTempTable(createTable = false) { tableName => sql(s"CREATE TABLE $tableName (a Int, b Int, c Int, d Int, e Int) USING delta") val r = DeltaLog.forTable(spark, new TableIdentifier(tableName)) // Only index the first three columns setNumIndexedColumns(r.dataPath.toString, 3) val dataSeq = Seq((1, 2, 3, 4, 5)) dataSeq.toDF("a", "b", "c", "d", "e") .select("d", "e") // DataFrame schema order .write.mode("append").format("delta") .save(r.dataPath.toString) val hits = Seq( "d = 40", // No stats "e = 40" // No stats ) // We can still collect NULL_COUNT for a, b, and c val misses = Seq( "a = 40", "b = 40", "c = 40" ) checkSkipping(r, hits, misses, dataSeq.toString(), false) } // case-5: The first dataSkippingNumIndexedCols columns of the table schema has map or array // types, which we only collect NULL_COUNT withTempTable(createTable = false) { tableName => sql(s"CREATE TABLE $tableName (a Int, b Map, c Array, d Int, e Int)" + " USING delta") val r = DeltaLog.forTable(spark, new TableIdentifier(tableName)) // Only index the first three columns setNumIndexedColumns(r.dataPath.toString, 3) val dataSeq = Seq((1, Map("key" -> 2), Seq(3, 3, 3), 4, 5)) dataSeq.toDF("a", "b", "c", "d", "e") .select("b", "c", "d") // DataFrame schema order .write.mode("append").format("delta") .save(r.dataPath.toString) val hits = Seq( "d = 50", // No stats "e = 50", // No stats // No min/max stats for c. We couldn't check = for b since EqualTo does not support // ordering on type maP "c = array(50, 50)", // b and c should have NULL_COUNT stats, but currently they're not SkippingEligibleColumn // (since they're not AtomicType), we couldn't skip for them "isnull(b)", "c is null", "ELEMENT_AT(c, 10) IS NULL" // Out-of-bounds access returns null. ) val misses = Seq( // a has NULL_COUNT stats since it's missing from DataFrame schema "a = 50" ) checkSkipping(r, hits, misses, dataSeq.toString(), false) } } test("data skipping with generated column") { withTable("table") { // OSS does not support the generated column syntax in SQL so we have to use table builder val tableBuilder = io.delta.tables.DeltaTable.create(spark).tableName("table") // add regular columns val col1 = io.delta.tables.DeltaTable.columnBuilder(spark, "col1") .dataType("int") .build() val col2 = io.delta.tables.DeltaTable.columnBuilder(spark, "col2") .dataType("string") .build() // add generated column val genCol3 = io.delta.tables.DeltaTable.columnBuilder(spark, "genCol3") .dataType("string") .generatedAlwaysAs("substring(col2, 3, 2)") .build() tableBuilder .addColumn(col1) .addColumn(col2) .addColumn(genCol3) .execute() // Only pass in two columns, and col3 will be generated as "st" val tableData = Seq((1, "test string")) tableData.toDF("col1", "col2") .write.format("delta").mode("append") .saveAsTable("table") val hits = Seq( "genCol3 = 'st'" ) val misses = Seq( "col1 = 10", "col2 = 'test'", "genCol3 = 'test'" ) val r = DeltaLog.forTable(spark, new TableIdentifier("table")) checkSkipping(r, hits, misses, tableData.toString(), false) } } test("data skipping by partitions and data values - nulls") { val tableDir = Utils.createTempDir().getAbsolutePath val dataSeqs = Seq( // each sequence produce a single file Seq((null, null)), Seq((null, "a")), Seq((null, "b")), Seq(("a", "a"), ("a", null)), Seq(("b", null)) ) dataSeqs.foreach { seq => seq.toDF("key", "value").coalesce(1) .write.format("delta").partitionBy("key").mode("append").save(tableDir) } val allData = dataSeqs.flatten def checkResults( predicate: String, expResults: Seq[(String, String)], expNumPartitions: Int, expNumFiles: Long): Unit = checkResultsWithPartitions(tableDir, predicate, expResults, expNumPartitions, expNumFiles) // Trivial base case checkResults( predicate = "True", expResults = allData, expNumPartitions = 3, expNumFiles = 5) // Conditions on partition key checkResults( predicate = "key IS NULL", expResults = allData.filter(_._1 == null), expNumPartitions = 1, expNumFiles = 3) // 3 files with key = null checkResults( predicate = "key IS NOT NULL", expResults = allData.filter(_._1 != null), expNumPartitions = 2, expNumFiles = 2) // 2 files with key = 'a', and 1 file with key = 'b' checkResults( predicate = "key <=> NULL", expResults = allData.filter(_._1 == null), expNumPartitions = 1, expNumFiles = 3) // 3 files with key = null checkResults( predicate = "key = 'a'", expResults = allData.filter(_._1 == "a"), expNumPartitions = 1, expNumFiles = 1) // 1 files with key = 'a' checkResults( predicate = "key <=> 'a'", expResults = allData.filter(_._1 == "a"), expNumPartitions = 1, expNumFiles = 1) // 1 files with key <=> 'a' checkResults( predicate = "key = 'b'", expResults = allData.filter(_._1 == "b"), expNumPartitions = 1, expNumFiles = 1) // 1 files with key = 'b' checkResults( predicate = "key <=> 'b'", expResults = allData.filter(_._1 == "b"), expNumPartitions = 1, expNumFiles = 1) // 1 files with key <=> 'b' // Conditions on partitions keys and values checkResults( predicate = "value IS NULL", expResults = allData.filter(_._2 == null), expNumPartitions = 3, expNumFiles = 3) // files with all non-NULL values get skipped checkResults( predicate = "value IS NOT NULL", expResults = allData.filter(_._2 != null), expNumPartitions = 2, // one of the partitions has no files left after data skipping expNumFiles = 3) // files with all NULL values get skipped checkResults( predicate = "value <=> NULL", expResults = allData.filter(_._2 == null), expNumPartitions = 3, expNumFiles = 3) // same as IS NULL case above checkResults( predicate = "value = 'a'", expResults = allData.filter(_._2 == "a"), expNumPartitions = 2, // one partition has no files left after data skipping expNumFiles = 2) // only two files contain "a" checkResults( predicate = "value <=> 'a'", expResults = allData.filter(_._2 == "a"), expNumPartitions = 2, // one partition has no files left after data skipping expNumFiles = 2) // only two files contain "a" checkResults( predicate = "value <> 'a'", expResults = allData.filter(x => x._2 != "a" && x._2 != null), // i.e., only (null, b) expNumPartitions = 1, expNumFiles = 1) // only one file contains 'b' checkResults( predicate = "value = 'b'", expResults = allData.filter(_._2 == "b"), expNumPartitions = 1, expNumFiles = 1) // same as previous case checkResults( predicate = "value <=> 'b'", expResults = allData.filter(_._2 == "b"), expNumPartitions = 1, expNumFiles = 1) // same as previous case // Conditions on both, partition keys and values checkResults( predicate = "key IS NULL AND value = 'a'", expResults = Seq((null, "a")), expNumPartitions = 1, expNumFiles = 1) // only one file in the partition has (*, "a") checkResults( predicate = "key IS NOT NULL AND value IS NOT NULL", expResults = Seq(("a", "a")), expNumPartitions = 1, expNumFiles = 1) // 1 file with (*, a) checkResults( predicate = "key <=> NULL AND value <=> NULL", expResults = Seq((null, null)), expNumPartitions = 1, expNumFiles = 1) // 3 files with key = null, but only 1 with val = null. checkResults( predicate = "key <=> NULL OR value <=> NULL", expResults = allData.filter(_ != (("a", "a"))), expNumPartitions = 3, expNumFiles = 5) // all 5 files } // Note that we cannot use testSkipping here, because the JSON parsing bug we're working around // prevents specifying a microsecond timestamp as input data. for (timestampType <- Seq("TIMESTAMP", "TIMESTAMP_NTZ")) { test(s"data skipping on $timestampType") { val data = "2019-09-09 01:02:03.456789" val df = Seq(data).toDF("strTs") .selectExpr( s"CAST(strTs AS $timestampType) AS ts", s"STRUCT(CAST(strTs AS $timestampType) AS ts) AS nested") val tempDir = Utils.createTempDir() val log = DeltaLog.forTable(spark, tempDir) df.coalesce(1).write.format("delta").save(log.dataPath.toString) checkSkipping( log, // Check to ensure that the value actually in the file is always in range queries. hits = Seq( s"""ts >= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", s"""ts <= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", s"""nested.ts >= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", s"""nested.ts <= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", s"""TS >= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", s"""nEstED.tS >= cast("2019-09-09 01:02:03.456789" AS $timestampType)"""), // Check the range of values that are far enough away to be data skipped. Note that the // values are aligned with millisecond boundaries because of the JSON serialization // truncation. misses = Seq( s"""ts >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", s"""ts <= cast("2019-09-04 01:02:03.455999" AS $timestampType)""", s"""nested.ts >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", s"""nested.ts <= cast("2019-09-09 01:02:03.455999" AS $timestampType)""", s"""TS >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", s"""nEstED.tS >= cast("2019-09-09 01:02:03.457001" AS $timestampType)"""), data = data, checkEmptyUnusedFiltersForHits = false) } } for (timestampType <- Seq("TIMESTAMP", "TIMESTAMP_NTZ")) { test(s"data skipping on $timestampType with Long.MaxValue") { val maxVal = "294247-01-10 04:00:54.775807Z" val tempDir = Utils.createTempDir() val log = DeltaLog.forTable(spark, tempDir) Seq(maxVal).toDF("strTs") .selectExpr(s"CAST(strTs AS $timestampType) AS ts") .coalesce(1) .write .format("delta") .save(log.dataPath.toString) checkSkipping( log, hits = Seq( s"""ts >= cast("$maxVal" AS $timestampType)""", s"""ts >= "$maxVal"""", s"""ts >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", // This still hits because of JSON truncation to milliseconds s"""ts < cast("$maxVal" AS $timestampType)""".stripMargin), misses = Seq( s"""ts <= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", s"""ts > cast("$maxVal" AS $timestampType)"""), data = maxVal, checkEmptyUnusedFiltersForHits = false) } } for (timestampType <- Seq("TIMESTAMP", "TIMESTAMP_NTZ")) { test(s"data skipping on $timestampType near Long.MaxValue") { val tempDir = Utils.createTempDir() val log = DeltaLog.forTable(spark, tempDir) val nearMaxMicros = Long.MaxValue - 999L // Create DataFrame with the near-max timestamp value Seq(nearMaxMicros).toDF("microsSinceEpoch") .selectExpr(s"TIMESTAMP_MICROS(microsSinceEpoch) AS ts") .selectExpr(s"CAST(ts AS $timestampType) AS ts") .coalesce(1) .write .format("delta") .save(log.dataPath.toString) checkSkipping( log, // maxValue of the stats on ts will be saturated to Long.MaxValue instead // of being added 1000 microseconds, which will cause a long overflow. hits = Seq( s"ts >= TIMESTAMP_MICROS($nearMaxMicros)", s"ts >= TIMESTAMP_MICROS(${nearMaxMicros - 100})", s"""ts >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", s"ts >= TIMESTAMP_MICROS(${Long.MaxValue - 1000})", s"ts < TIMESTAMP_MICROS($nearMaxMicros)"), misses = Seq( s"""ts <= cast("2019-09-09 01:02:03.457001" AS $timestampType)"""), data = nearMaxMicros.toString, checkEmptyUnusedFiltersForHits = false) } } test("Ensure that we don't reuse scans when tables are different") { withTempDir { dir => val table1 = new File(dir, "tbl1") val table1Dir = table1.getCanonicalPath val table2 = new File(dir, "tbl2") val table2Dir = table2.getCanonicalPath spark.range(100).withColumn("part", 'id % 5).withColumn("id2", 'id) .write.format("delta").partitionBy("part").save(table1Dir) FileUtils.copyDirectory(table1, table2) sql(s"DELETE FROM delta.`$table2Dir` WHERE part = 0 and id < 65") val query = sql(s"SELECT * FROM delta.`$table1Dir` WHERE part = 0 AND id2 < 85 AND " + s"id NOT IN (SELECT id FROM delta.`$table2Dir` WHERE part = 0 AND id2 < 85)") checkAnswer( query, sql(s"SELECT * FROM delta.`$table1Dir` WHERE part = 0 and id < 65")) } } test("Data skipping should always return files from latest commit version") { withTempDir { dir => // If this test is flacky it is broken Seq("aaa").toDF().write.format("delta").save(dir.getCanonicalPath) val (log, snapshot) = DeltaLog.forTableWithSnapshot(spark, dir.getPath) val addFile = snapshot.allFiles.collect().head val fileWithStat = snapshot.getSpecificFilesWithStats(Seq(addFile.path)).head // Ensure the stats has actual stats, not {} assert(fileWithStat.stats.size > 2) log.startTransaction().commitManually(addFile.copy(stats = "{}")) // Delta dedup should always keep AddFile from newer version so // getSpecificFilesWithStats should return the AddFile with empty stats log.update() val newfileWithStat = log.unsafeVolatileSnapshot.getSpecificFilesWithStats(Seq(addFile.path)).head assert(newfileWithStat.stats === "{}") } } Seq("create", "alter").foreach { label => test(s"Basic: Data skipping with delta statistic column $label") { withTable("table") { val tableProperty = if (label == "create") { "TBLPROPERTIES('delta.dataSkippingStatsColumns' = 'c1,c2,c3,c4,c5,c6,c7,c10')" } else { "" } sql( s"""CREATE TABLE table( |c1 long, c2 STRING, c3 FLOAT, c4 DOUBLE, c5 TIMESTAMP, c6 TIMESTAMP_NTZ, c7 DATE, |c8 BINARY, c9 BOOLEAN, c10 DECIMAL(3, 2) |) USING delta $tableProperty""".stripMargin) if (label == "alter") { sql( s"""ALTER TABLE table |SET TBLPROPERTIES ( | 'delta.dataSkippingStatsColumns' = 'c1,c2,c3,c4,c5,c6,c7,c10' |)""".stripMargin) } sql( """insert into table values |(1, '1', 1.0, 1.0, TIMESTAMP'2001-01-01 01:00', TIMESTAMP_NTZ'2001-01-01 01:00', |DATE'2001-01-01', '1111', true, 1.0), |(2, '2', 2.0, 2.0, TIMESTAMP'2002-02-02 02:00', TIMESTAMP_NTZ'2002-02-02 02:00', |DATE'2002-02-02', '2222', false, 2.0) |""".stripMargin).count() val hits = Seq( "c1 = 1", "c2 = \'2\'", "c3 < 1.5", "c4 > 1.0", "c5 >= \"2001-01-01 01:00:00\"", "c6 >= \"2001-01-01 01:00:00\"", "c7 = \"2002-02-02\"", "c8 = HEX(\"1111\")", // Binary Column doesn't support delta statistics. "c8 = HEX(\"3333\")", // Binary Column doesn't support delta statistics. "c9 = true", "c9 = false", "c10 > 1.5" ) val misses = Seq( "c1 = 10", "c2 = \'4\'", "c3 < 0.5", "c4 > 5.0", "c5 >= \"2003-01-01 01:00:00\"", "c6 >= \"2003-01-01 01:00:00\"", "c7 = \"2003-02-02\"", "c10 > 2.5" ) val dataSeq = Seq( (1L, "1", 1.0f, 1.0d, "2002-01-01 01:00", "2002-01-01 01:00", "2001-01-01", "1111", true, 1.0f), (2L, "2", 2.0f, 2.0d, "2002-02-02 02:00", "2002-02-02 02:00", "2002-02-02", "2222", false, 2.0f) ) val r = DeltaLog.forTable(spark, new TableIdentifier("table")) checkSkipping(r, hits, misses, dataSeq.toString(), false) } } } test("data skipping by stats - variant type") { Seq(false, true).foreach { pushVariantIntoScan => withSQLConf(SQLConf.PUSH_VARIANT_INTO_SCAN.key -> pushVariantIntoScan.toString) { val tableName = s"tbl_$pushVariantIntoScan" withTable(tableName) { sql(s"""CREATE TABLE $tableName(v VARIANT, v_struct STRUCT, null_v VARIANT, null_v_struct STRUCT) USING DELTA""") sql(s"""INSERT INTO $tableName (SELECT parse_json(cast(id as string)), named_struct('v', parse_json(cast(id as string))), cast(null as variant), named_struct('v', cast(null as variant)) FROM range(100))""") val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName, None, None)) val hits = Seq( "v IS NOT NULL", "v_struct.v IS NOT NULL", "null_v IS NULL", "null_v_struct.v IS NULL") val misses = Seq( "v IS NULL", "v_struct.v IS NULL", "null_v IS NOT NULL", "null_v_struct.v IS NOT NULL") val data = spark.sql(s"select * from $tableName").collect().toSeq.toString checkSkipping(deltaLog, hits, misses, data, false) } } } } test(s"Data skipping with delta statistic column rename column") { withTable("table") { sql( s"""CREATE TABLE table( |c1 long, c2 STRING, c3 FLOAT, c4 DOUBLE, c5 TIMESTAMP, c6 TIMESTAMP_NTZ, |c7 DATE, c8 BINARY, c9 BOOLEAN, c10 DECIMAL(3, 2) |) USING delta |TBLPROPERTIES( |'delta.dataSkippingStatsColumns' = 'c1,c2,c3,c4,c5,c6,c7,c10', |'delta.columnMapping.mode' = 'name', |'delta.minReaderVersion' = '2', |'delta.minWriterVersion' = '5' |) |""".stripMargin) (1 to 10).foreach { i => sql(s"alter table table RENAME COLUMN c$i to cc$i") } val newConfiguration = sql("SHOW TBLPROPERTIES table ") .collect() .map { row => row.getString(0) -> row.getString(1) } .filter(_._1 == "delta.dataSkippingStatsColumns") .toSeq assert( newConfiguration == Seq( ("delta.dataSkippingStatsColumns", "cc1,cc2,cc3,cc4,cc5,cc6,cc7,cc10")) ) sql( """insert into table values |(1, '1', 1.0, 1.0, TIMESTAMP'2001-01-01 01:00', TIMESTAMP_NTZ'2001-01-01 01:00', |DATE'2001-01-01', '1111', true, 1.0), |(2, '2', 2.0, 2.0, TIMESTAMP'2002-02-02 02:00', TIMESTAMP_NTZ'2002-02-02 02:00', |DATE'2002-02-02', '2222', false, 2.0) |""".stripMargin).count() val hits = Seq( "cc1 = 1", "cc2 = \'2\'", "cc3 < 1.5", "cc4 > 1.0", "cc5 >= \"2001-01-01 01:00:00\"", "cc6 >= \"2001-01-01 01:00:00\"", "cc7 = \"2002-02-02\"", "cc8 = HEX(\"1111\")", // Binary Column doesn't support delta statistics. "cc8 = HEX(\"3333\")", // Binary Column doesn't support delta statistics. "cc9 = true", "cc9 = false", "cc10 > 1.5" ) val misses = Seq( "cc1 = 10", "cc2 = \'4\'", "cc3 < 0.5", "cc4 > 5.0", "cc5 >= \"2003-01-01 01:00:00\"", "cc6 >= \"2003-01-01 01:00:00\"", "cc7 = \"2003-02-02\"", "cc10 > 2.5" ) val dataSeq = Seq( (1L, "1", 1.0f, 1.0d, "2002-01-01 01:00", "2002-01-01 01:00", "2001-01-01", "1111", true, 1.0f), (2L, "2", 2.0f, 2.0d, "2002-02-02 02:00", "2002-02-02 02:00", "2002-02-02", "2222", false, 2.0f) ) val r = DeltaLog.forTable(spark, new TableIdentifier("table")) checkSkipping(r, hits, misses, dataSeq.toString(), false) } } test(s"Data skipping with delta statistic column drop column") { withTable("table") { sql( s"""CREATE TABLE table( |c1 long, c2 STRING, c3 FLOAT, c4 DOUBLE, c5 TIMESTAMP, c6 TIMESTAMP_NTZ, |c7 DATE, c8 BINARY, c9 BOOLEAN, c10 DECIMAL(3, 2)) |USING delta |TBLPROPERTIES( |'delta.dataSkippingStatsColumns' = 'c1,c2,c3,c4,c5,c6,c7,c10', |'delta.columnMapping.mode' = 'name', |'delta.minReaderVersion' = '2', |'delta.minWriterVersion' = '5' |) |""".stripMargin) sql(s"alter table table drop COLUMN c2") sql(s"alter table table drop COLUMN c8") sql(s"alter table table drop COLUMN c9") val newConfiguration = sql("SHOW TBLPROPERTIES table ") .collect() .map { row => row.getString(0) -> row.getString(1) } .filter(_._1 == "delta.dataSkippingStatsColumns") .toSeq assert(newConfiguration == Seq(("delta.dataSkippingStatsColumns", "c1,c3,c4,c5,c6,c7,c10"))) sql( """insert into table values |(1, 1.0, 1.0, TIMESTAMP'2001-01-01 01:00', TIMESTAMP_NTZ'2001-01-01 01:00', |DATE'2001-01-01', 1.0), |(2, 2.0, 2.0, TIMESTAMP'2002-02-02 02:00', TIMESTAMP_NTZ'2002-02-02 02:00', |DATE'2002-02-02', 2.0) |""".stripMargin).count() val hits = Seq( "c1 = 1", "c3 < 1.5", "c4 > 1.0", "c5 >= \"2001-01-01 01:00:00\"", "c6 >= \"2001-01-01 01:00:00\"", "c7 = \"2002-02-02\"", "c10 > 1.5" ) val misses = Seq( "c1 = 10", "c3 < 0.5", "c4 > 5.0", "c5 >= \"2003-01-01 01:00:00\"", "c6 >= \"2003-01-01 01:00:00\"", "c7 = \"2003-02-02\"", "c10 > 2.5" ) val dataSeq = Seq( (1L, 1.0f, 1.0d, "2002-01-01 01:00", "2002-01-01 01:00", "2001-01-01", 1.0f), (2L, 2.0f, 2.0d, "2002-02-02 02:00", "2002-02-02 02:00", "2002-02-02", 2.0f) ) val r = DeltaLog.forTable(spark, new TableIdentifier("table")) checkSkipping(r, hits, misses, dataSeq.toString(), false) } } protected def expectedStatsForFile(index: Int, colName: String, deltaLog: DeltaLog): String = { if (deltaLog.unsafeVolatileSnapshot.protocol.isFeatureSupported(DeletionVectorsTableFeature)) { s"""{"numRecords":1,"minValues":{"$colName":$index},"maxValues":{"$colName":$index},""" + s""""nullCount":{"$colName":0},"tightBounds":true}""".stripMargin } else { s"""{"numRecords":1,"minValues":{"$colName":$index},"maxValues":{"$colName":$index},""" + s""""nullCount":{"$colName":0}}""".stripMargin } } test("data skipping get specific files with Stats API") { withTempDir { tempDir => val tableDirPath = tempDir.getCanonicalPath val fileCount = 5 // Create 5 files each having 1 row - x=1/x=2/x=3/x=4/x=5 val data = spark.range(1, fileCount).toDF("x").repartition(fileCount, col("x")) data.write.format("delta").save(tableDirPath) var deltaLog = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) // Get name of file corresponding to row x=1 val file1 = getFilesRead(deltaLog, "x = 1").head.path // Get name of file corresponding to row x=2 val file2 = getFilesRead(deltaLog, "x = 2").head.path // Get name of file corresponding to row x=3 val file3 = getFilesRead(deltaLog, "x = 3").head.path deltaLog = checkpointAndCreateNewLogIfNecessary(deltaLog) // Delete rows/files for x >= 3 from snapshot sql(s"DELETE FROM delta.`$tableDirPath` WHERE x >= 3") // Add another file with just one row x=6 in snapshot sql(s"INSERT INTO delta.`$tableDirPath` VALUES (6)") // We want the file from the INSERT VALUES (6) stmt. However, this `getFilesRead` call might // also return the AddFile (due to data file re-writes) from the DELETE stmt above. Since // they were committed in different commits, we can select the addFile with the higher // version val addPathToCommitVersion = deltaLog.getChanges(0).flatMap { case (version, actions) => actions .collect { case a: AddFile => a } .map(a => (a.path, version)) }.toMap val file6 = getFilesRead(deltaLog, "x = 6") .map(_.path) .maxBy(path => addPathToCommitVersion(path)) // At this point, our latest snapshot has only 3 rows: x=1, x=2, x=6 - all in // different files // Case-1: all passes files to the API exists in the snapshot val result1 = deltaLog.snapshot.getSpecificFilesWithStats(Seq(file1, file2)) .map(addFile => (addFile.path, addFile)).toMap assert(result1.size == 2) assert(result1.keySet == Set(file1, file2)) assert(result1(file1).stats === expectedStatsForFile(1, "x", deltaLog)) assert(result1(file2).stats === expectedStatsForFile(2, "x", deltaLog)) // Case-2: few passes files exists in the snapshot and few don't exists val result2 = deltaLog.snapshot.getSpecificFilesWithStats(Seq(file1, file2, file3)) .map(addFile => (addFile.path, addFile)).toMap assert(result1 == result2) // Case-3: all passed files don't exists in the snapshot val result3 = deltaLog.snapshot.getSpecificFilesWithStats(Seq(file3, "xyz")) assert(result3.isEmpty) // Case-4: file3 doesn't exist and file6 exists in the latest commit val result4 = deltaLog.snapshot.getSpecificFilesWithStats(Seq(file3, file6)) .map(addFile => (addFile.path, addFile)).toMap assert(result4.size == 1) assert(result4(file6).stats == expectedStatsForFile(6, "x", deltaLog)) } } test("File skipping with non-deterministic filters") { withTable("tbl") { // Create the table. val df = spark.range(100).toDF() df.write.mode("overwrite").format("delta").saveAsTable("tbl") // Append 9 times to the table. for (i <- 1 to 9) { val df = spark.range(i * 100, (i + 1) * 100).toDF() df.write.mode("append").format("delta").insertInto("tbl") } val query = "SELECT count(*) FROM tbl WHERE rand(0) < 0.25" val result = sql(query).collect().head.getLong(0) assert(result > 150, s"Expected around 250 rows (~0.25 * 1000), got: $result") val predicates = sql(query).queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition }.flatMap(splitConjunctivePredicates) val scanResult = DeltaLog.forTable(spark, TableIdentifier("tbl")) .update().filesForScan(predicates) assert(scanResult.unusedFilters.nonEmpty) } } test("File skipping with non-deterministic filters on partitioned tables") { withTable("tbl_partitioned") { import org.apache.spark.sql.functions.col // Create initial DataFrame and add a partition column. val df = spark.range(100).toDF().withColumn("p", col("id") % 10) df.write .mode("overwrite") .format("delta") .partitionBy("p") .saveAsTable("tbl_partitioned") // Append 9 more times to the table. for (i <- 1 to 9) { val newDF = spark.range(i * 100, (i + 1) * 100).toDF().withColumn("p", col("id") % 10) newDF.write.mode("append").format("delta").insertInto("tbl_partitioned") } // Run query with a nondeterministic filter. val query = "SELECT count(*) FROM tbl_partitioned WHERE rand(0) < 0.25" val result = sql(query).collect().head.getLong(0) // Assert that the row count is as expected (e.g., roughly 25% of rows). assert(result > 150, s"Expected a reasonable number of rows, got: $result") val predicates = sql(query).queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition }.flatMap(splitConjunctivePredicates) val scanResult = DeltaLog.forTable(spark, TableIdentifier("tbl_partitioned")) .update().filesForScan(predicates) assert(scanResult.unusedFilters.nonEmpty) // Assert that entries are fetched from all 10 partitions val distinctPartitions = sql("SELECT DISTINCT p FROM tbl_partitioned WHERE rand(0) < 0.25") .collect() .length assert(distinctPartitions == 10) } } test("Data skipping handles aliasing for _metadata fields") { withTable("t") { // Create table with BIGINT file_name column sql("create or replace table t(file_name BIGINT) using delta") sql("insert into t values (1), (2), (3)") sql("insert into t values (4), (5), (6)") val (fileName, fileCount) = { val dataFilesDF = sql("select distinct _metadata.file_name from t") (dataFilesDF.first().getString(0), dataFilesDF.count()) } // Filter rows by _metadata.file_name val df = sql(s"select * from t where _metadata.file_name = '$fileName'") // Verify the predicate is not used for data skipping val predicates = df.queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition }.flatMap(splitConjunctivePredicates) val scanResult = DeltaLog.forTable(spark, TableIdentifier("t")).update() .filesForScan(predicates) assert(scanResult.unusedFilters.nonEmpty, "Expected predicate to be ineligible for data skipping") } } protected def parse(deltaLog: DeltaLog, predicate: String): Seq[Expression] = super.parse(spark, deltaLog, predicate) protected def filesRead( deltaLog: DeltaLog, predicate: String, checkEmptyUnusedFilters: Boolean = false): Int = super.filesRead(spark, deltaLog, predicate, checkEmptyUnusedFilters) protected def getFilesRead( deltaLog: DeltaLog, predicate: String, checkEmptyUnusedFilters: Boolean = false): Seq[AddFile] = super.getFilesRead(spark, deltaLog, predicate, checkEmptyUnusedFilters) protected def checkResultsWithPartitions( tableDir: String, predicate: String, expResults: Seq[(String, String)], expNumPartitions: Int, expNumFiles: Long): Unit = { Given(predicate) val df = spark.read.format("delta").load(tableDir).where(predicate) checkAnswer(df, expResults.toDF()) val files = getFilesRead(DeltaLog.forTable(spark, tableDir), predicate) assert(files.size == expNumFiles, "# files incorrect:\n\t" + files.mkString("\n\t")) val partitionValues = files.map(_.partitionValues).distinct assert(partitionValues.size == expNumPartitions, "# partitions incorrect:\n\t" + partitionValues.mkString("\n\t")) } protected def getStatsDf(deltaLog: DeltaLog, columns: Column*): DataFrame = { deltaLog.snapshot.withStats.select("stats.*").select(columns: _*) } protected def failPretty(error: String, predicate: String, data: String) = { fail( s"""$error | |== Data == |$data """.stripMargin) } protected def setNumIndexedColumns(path: String, numIndexedCols: Int): Unit = { sql(s""" |ALTER TABLE delta.`$path` |SET TBLPROPERTIES ( | 'delta.dataSkippingNumIndexedCols' = '$numIndexedCols' |)""".stripMargin) } protected def setDeltaStatsColumns(path: String, deltaStatsColumns: String): Unit = { sql(s""" |ALTER TABLE delta.`$path` |SET TBLPROPERTIES ( | 'delta.dataSkippingStatsColumns' = '$deltaStatsColumns' |)""".stripMargin) } private def isFullScan(report: ScanReport): Boolean = { report.size("scanned").bytesCompressed === report.size("total").bytesCompressed } protected def checkSkipping( log: DeltaLog, hits: Seq[String], misses: Seq[String], data: String, checkEmptyUnusedFiltersForHits: Boolean): Unit = { hits.foreach { predicate => Given(predicate) if (filesRead(log, predicate, checkEmptyUnusedFiltersForHits) == 0) { failPretty(s"Expected hit but got miss for $predicate", predicate, data) } } misses.foreach { predicate => Given(predicate) if (filesRead(log, predicate) != 0) { failPretty(s"Expected miss but got hit for $predicate", predicate, data) } } val schemaDiff = SchemaUtils.reportDifferences( log.snapshot.statsSchema.asNullable, log.snapshot.statsSchema) if (schemaDiff.nonEmpty) { fail(s"The stats schema should be nullable. Differences:\n${schemaDiff.mkString("\n")}") } } protected def getDataSkippingConfs( indexedCols: Int, deltaStatsColNamesOpt: Option[String]): TraversableOnce[(String, String)] = { val numIndexedColsConfOpt = Option(indexedCols) .filter(_ != defaultNumIndexedCols) .map(DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.defaultTablePropertyKey -> _.toString) val indexedColNamesConfOpt = deltaStatsColNamesOpt .map(DeltaConfigs.DATA_SKIPPING_STATS_COLUMNS.defaultTablePropertyKey -> _) numIndexedColsConfOpt ++ indexedColNamesConfOpt } protected def testSkipping( name: String, data: String, schema: StructType = null, hits: Seq[String], misses: Seq[String], sqlConfs: Seq[(String, String)] = Nil, indexedCols: Int = defaultNumIndexedCols, deltaStatsColNamesOpt: Option[String] = None, checkEmptyUnusedFiltersForHits: Boolean = false, exceptionOpt: Option[Throwable] = None): Unit = { test(s"data skipping by stats - $name") { val allSQLConfs = sqlConfs ++ getDataSkippingConfs(indexedCols, deltaStatsColNamesOpt) withSQLConf(allSQLConfs: _*) { val jsonRecords = data.split("\n").toSeq val reader = spark.read if (schema != null) { reader.schema(schema) } val df = reader.json(jsonRecords.toDS()) val tempDir = Utils.createTempDir() val r = DeltaLog.forTable(spark, tempDir) df.coalesce(1).write.format("delta").save(r.dataPath.toString) exceptionOpt.map { exception => val except = intercept[Throwable] { deltaStatsColNamesOpt.foreach { deltaStatsColNames => setDeltaStatsColumns(r.dataPath.toString, deltaStatsColNames) df.coalesce(1).write.format("delta").mode("overwrite").save(r.dataPath.toString) if (indexedCols != defaultNumIndexedCols) { setNumIndexedColumns(r.dataPath.toString, indexedCols) df.coalesce(1).write.format("delta").mode("overwrite").save(r.dataPath.toString) } checkSkipping(r, hits, misses, data, checkEmptyUnusedFiltersForHits) } } assert(except.getClass == exception.getClass && except.getMessage.contains(exception.getMessage)) }.getOrElse { if (indexedCols != defaultNumIndexedCols) { setNumIndexedColumns(r.dataPath.toString, indexedCols) df.coalesce(1).write.format("delta").mode("overwrite").save(r.dataPath.toString) } deltaStatsColNamesOpt.foreach { deltaStatsColNames => setDeltaStatsColumns(r.dataPath.toString, deltaStatsColNames) df.coalesce(1).write.format("delta").mode("overwrite").save(r.dataPath.toString) } checkSkipping(r, hits, misses, data, checkEmptyUnusedFiltersForHits) } } } } } trait DataSkippingDeltaTestsUtils extends PredicateHelper { protected def parse( spark: SparkSession, deltaLog: DeltaLog, predicate: String): Seq[Expression] = { // We produce a wrong filter in this case otherwise if (predicate == "True") return Seq(Literal.TrueLiteral) val filtered = spark.read.format("delta").load(deltaLog.dataPath.toString).where(predicate) val optimizedPlan = filtered.queryExecution.optimizedPlan // When pushVariantIntoScan = true, the plan is transformed such that a projection is inserted // at the top of the plan. Therefore, the filter node is lower in the plan. val filterNode = optimizedPlan.collectFirst { case f: Filter => f }.getOrElse { optimizedPlan } filterNode .expressions .flatMap(splitConjunctivePredicates) } /** * Returns the number of files that should be included in a scan after applying the given * predicate on a snapshot of the Delta log. * * @param deltaLog Delta log for a table. * @param predicate Predicate to run on the Delta table. * @param checkEmptyUnusedFilters If true, check if there were no unused filters, meaning * the given predicate was used as data or partition filters. * @return The number of files that should be included in a scan after applying the predicate. */ protected def filesRead( spark: SparkSession, deltaLog: DeltaLog, predicate: String, checkEmptyUnusedFilters: Boolean): Int = getFilesRead(spark, deltaLog, predicate, checkEmptyUnusedFilters).size /** * Returns the files that should be included in a scan after applying the given predicate on * a snapshot of the Delta log. * @param deltaLog Delta log for a table. * @param predicate Predicate to run on the Delta table. * @param checkEmptyUnusedFilters If true, check if there were no unused filters, meaning * the given predicate was used as data or partition filters. * @return The files that should be included in a scan after applying the predicate. */ protected def getFilesRead( spark: SparkSession, deltaLog: DeltaLog, predicate: String, checkEmptyUnusedFilters: Boolean): Seq[AddFile] = { val parsed = parse(spark, deltaLog, predicate) val res = deltaLog.snapshot.filesForScan(parsed) assert(res.total.files.get == deltaLog.snapshot.numOfFiles) assert(res.total.bytesCompressed.get == deltaLog.snapshot.sizeInBytes) assert(res.scanned.files.get == res.files.size) assert(res.scanned.bytesCompressed.get == res.files.map(_.size).sum) assert(!checkEmptyUnusedFilters || res.unusedFilters.isEmpty) res.files } } trait DataSkippingDeltaTests extends DataSkippingDeltaTestsBase /** Tests code paths within DataSkippingReader.scala */ class DataSkippingDeltaV1Suite extends DataSkippingDeltaTests { import testImplicits._ test("data skipping flags") { val tempDir = Utils.createTempDir() val r = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) def rStats: DataFrame = getStatsDf(r, $"numRecords", $"minValues.id".as("id_min"), $"maxValues.id".as("id_max")) val data = spark.range(10).repartition(2) Given("appending data without collecting stats") withSQLConf( DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false", DeltaConfigs.ROW_TRACKING_ENABLED.defaultTablePropertyKey -> "false") { data.write.format("delta").save(r.dataPath.toString) checkAnswer(rStats, Seq(Row(null, null, null), Row(null, null, null))) } Given("appending data and collecting stats") withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "true") { data.write.format("delta").mode("append").save(r.dataPath.toString) checkAnswer(rStats, Seq(Row(null, null, null), Row(null, null, null), Row(4, 0, 8), Row(6, 1, 9))) } Given("querying reservoir without using stats") withSQLConf(DeltaSQLConf.DELTA_STATS_SKIPPING.key -> "false") { assert(filesRead(r, "id = 0") == 4) } Given("querying reservoir using stats") withSQLConf(DeltaSQLConf.DELTA_STATS_SKIPPING.key -> "true") { assert(filesRead(r, "id = 0") == 3) } } } /** * Used to disable the tests with the old stats collection behavior on long-running suites to * avoid time-out * TODO(lin): remove this after we remove the DELTA_COLLECT_STATS_USING_TABLE_SCHEMA flag */ trait DataSkippingDisableOldStatsSchemaTests extends DataSkippingDeltaTests { protected override def test(testName: String, testTags: org.scalatest.Tag*) (testFun: => Any) (implicit pos: org.scalactic.source.Position): Unit = { // Adding the null check in case tableSchemaOnlyTag has not been initialized in base traits val newTestTags = if (tableSchemaOnlyTag == null) testTags else tableSchemaOnlyTag +: testTags super.test(testName, newTestTags: _*)(testFun)(pos) } } /** DataSkipping tests under id column mapping */ trait DataSkippingDeltaIdColumnMappingTests extends DataSkippingDeltaTests with DeltaColumnMappingTestUtils { override def expectedStatsForFile(index: Int, colName: String, deltaLog: DeltaLog): String = { val x = colName.phy(deltaLog) if (deltaLog.unsafeVolatileSnapshot.protocol.isFeatureSupported(DeletionVectorsTableFeature)) { s"""{"numRecords":1,"minValues":{"$x":$index},"maxValues":{"$x":$index},""" + s""""nullCount":{"$x":0},"tightBounds":true}""".stripMargin } else { s"""{"numRecords":1,"minValues":{"$x":$index},"maxValues":{"$x":$index},""" + s""""nullCount":{"$x":0}}""".stripMargin } } } trait DataSkippingDeltaTestV1ColumnMappingMode extends DataSkippingDeltaIdColumnMappingTests { override protected def getStatsDf(deltaLog: DeltaLog, columns: Column*): DataFrame = { deltaLog.snapshot.withStats.select("stats.*") .select(convertToPhysicalColumns(columns, deltaLog): _*) } } class DataSkippingDeltaV1NameColumnMappingSuite extends DataSkippingDeltaV1Suite with DeltaColumnMappingEnableNameMode with DataSkippingDeltaTestV1ColumnMappingMode { override protected def runAllTests: Boolean = true } class DataSkippingDeltaV1JsonCheckpointV2Suite extends DataSkippingDeltaV1Suite { override def sparkConf: SparkConf = { super.sparkConf.setAll( Seq( DeltaConfigs.CHECKPOINT_POLICY.defaultTablePropertyKey -> CheckpointPolicy.V2.name, DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key -> V2Checkpoint.Format.JSON.name ) ) } } class DataSkippingDeltaV1ParquetCheckpointV2Suite extends DataSkippingDeltaV1Suite { override def sparkConf: SparkConf = { super.sparkConf.setAll( Seq( DeltaConfigs.CHECKPOINT_POLICY.defaultTablePropertyKey -> CheckpointPolicy.V2.name, DeltaSQLConf.CHECKPOINT_V2_TOP_LEVEL_FILE_FORMAT.key -> V2Checkpoint.Format.PARQUET.name ) ) } } class DataSkippingDeltaV1WithCatalogOwnedBatch1Suite extends DataSkippingDeltaV1Suite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) } class DataSkippingDeltaV1WithCatalogOwnedBatch2Suite extends DataSkippingDeltaV1Suite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(2) } class DataSkippingDeltaV1WithCatalogOwnedBatch100Suite extends DataSkippingDeltaV1Suite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/stats/PartitionLikeDataSkippingSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats import java.sql.{Date, Timestamp} import org.apache.spark.sql.delta.skipping.ClusteredTableTestUtils import org.apache.spark.sql.delta.{DeltaColumnMappingEnableIdMode, DeltaLog} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.scalatest.BeforeAndAfter import org.apache.spark.SparkConf import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.plans.logical.Filter import org.apache.spark.sql.functions.{array, col, concat, lit, struct} import org.apache.spark.sql.test.SharedSparkSession trait PartitionLikeDataSkippingSuiteBase extends QueryTest with SharedSparkSession with BeforeAndAfter with DeltaSQLCommandTest with ClusteredTableTestUtils { import testImplicits._ // Disable write optimization to ensure close control over the partitioning of ingested files. override protected def sparkConf: SparkConf = super.sparkConf .set(DeltaSQLConf.DELTA_OPTIMIZE_WRITE_ENABLED.key, "false") protected val testTableName = "test_table" private def longToTimestampMillis(i: Long): Long = { i + // Ensure that there are some millis that will be truncated. i * 1000L + // Add some seconds. i * 60 * 1000 + // Add some minutes. i * 60 * 60 * 1000 + // Add some hours. i * 24L * 60 * 60 * 1000 + // Add some days. i * 30L * 24 * 60 * 60 * 1000 + // Add some months. i * 365L * 24 * 60 * 60 * 1000 // Add some years. } // Helper to validate the expected scan metrics of a query. protected def validateExpectedScanMetrics( tableName: String, query: String, expectedNumFiles: Int, expectedNumPartitionLikeDataFilters: Int, allPredicatesUsed: Boolean, minNumFilesToApply: Long): Unit = { // Execute the query without partition-like filters. val baseResult = sql(query).collect() withSQLConf( DeltaSQLConf.DELTA_DATASKIPPING_PARTITION_LIKE_FILTERS_ENABLED.key -> "true", DeltaSQLConf.DELTA_DATASKIPPING_PARTITION_LIKE_FILTERS_THRESHOLD.key -> minNumFilesToApply.toString) { // Execute the query with partition-like filters and validate that the result matches. val res = sql(query).collect() assert(res.sortBy(_.toString).sameElements(baseResult.sortBy(_.toString))) val predicates = sql(query).queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition }.flatMap(splitConjunctivePredicates) val scanResult = DeltaLog.forTable(spark, TableIdentifier(tableName)) .update().filesForScan(predicates) assert(scanResult.files.length == expectedNumFiles) assert(allPredicatesUsed == scanResult.unusedFilters.isEmpty) assert(scanResult.partitionLikeDataFilters.size == expectedNumPartitionLikeDataFilters) } } protected override def beforeAll(): Unit = { super.beforeAll() // Create a shared test table to be used by many tests. sql(s"CREATE TABLE $testTableName(s STRUCT, c DATE, d TIMESTAMP, e INT) " + s"USING delta CLUSTER BY (s.a, s.b, c, d)") // Insert 10 files that each have the same value on all clustering columns. val srcDF = (0L until 10L).map{ i => val timestampMillis = longToTimestampMillis(i) val timestamp = new Timestamp(timestampMillis) (9 - i.toInt, timestamp.toString, new Date(timestampMillis), timestamp, i.toInt) }.toDF("a", "b", "c", "d", "e") .withColumn("s", struct(col("a"), col("b"))) .select("s", "c", "d", "e") srcDF.repartitionByRange(10, col("s.a")) .write .format("delta") .mode("append") .saveAsTable(testTableName) // Insert 10 files that each have the same value on all clustering columns, but that also // contain partial nulls. (0 until 10).foreach { i => srcDF.where(col("a") === i) .union( spark.range(1) .withColumn("a", lit(null).cast("int")) .withColumn("b", lit(null).cast("string")) .withColumn("s", struct(col("a"), col("b"))) .select("s") .withColumn("c", lit(null).cast("date")) .withColumn("d", lit(null).cast("timestamp")) .withColumn("e", lit(null).cast("int")) .drop("id")) .coalesce(1) .write.format("delta").mode("append").insertInto(testTableName) } // Insert 1 file that contains only nulls on each clustering column. spark.range(1) .withColumn("a", lit(null).cast("int")) .withColumn("b", lit(null).cast("string")) .withColumn("s", struct(col("a"), col("b"))) .select("s") .withColumn("c", lit(null).cast("date")) .withColumn("d", lit(null).cast("timestamp")) .withColumn("e", lit(null).cast("int")) .drop("id") .coalesce(1) .write.format("delta").mode("append").insertInto(testTableName) // Insert 1 file that does not have perfect clustering. srcDF .coalesce(1) .write.format("delta").mode("append").insertInto(testTableName) // Register a deterministic and nondeterministic UDF. spark.udf.register("isEven", (x: Int) => x % 2 == 0) spark.udf.register("randIsEven", (x: Int) => x % 2 == 0).asNondeterministic() } override protected def afterAll(): Unit = { // Cleanup shared table. sql(s"DROP TABLE IF EXISTS $testTableName") super.afterAll() } // Test cases for partition-like data skipping on the shared table. // Each test case has the format: // (name, predicate, num matching files (out of 10), number of partition-like predicates, // all predicates used for data skipping) private val partitionLikeTestCases = Seq( // Valid partition-like predicates. ("COALESCE", "COALESCE(null, s.a, 1) = 1", 2, 1, true), ("COALESCE", "COALESCE(null, s.a, 1) > 3", 6, 1, true), ("COALESCE", "COALESCE(null, s.a, 1) != 1", 9, 1, true), ("COALESCE", "COALESCE(s.A, 1) != 1", 9, 1, true), ("COALESCE", "COALESCE(TO_DATE(S.b), c) = '1976-07-03'", 1, 1, true), ("CAST", "CAST(s.A AS STRING) = '1'", 1, 1, true), ("CAST", "CAST(s.A AS STRING) < '3'", 3, 1, true), ("CAST", "CAST(s.A AS STRING) >= '3'", 7, 1, true), ("CAST", "TO_DATE(s.b) = '1976-07-03'", 1, 1, true), ("CAST", "CAST(s.a AS TIMESTAMP) = TIMESTAMP_SECONDS(1)", 1, 1, true), ("YEAR", "YEAR(TIMESTAMP(s.b)) = 1969", 1, 1, true), ("MONTH", "MONTH(TIMESTAMP(s.b)) = 1", 1, 1, true), ("DAYOFYEAR", "DAYOFYEAR(TIMESTAMP(s.b)) = 31", 1, 1, true), ("DAYOFMONTH", "DAYOFMONTH(TIMESTAMP(s.b)) = 2", 2, 1, true), ("MINUTE", "MINUTE(TIMESTAMP(s.b)) = 5", 1, 1, true), ("SECOND", "SECOND(TIMESTAMP(s.b)) = 5", 1, 1, true), ("LIKE", "s.b LIKE '%7.007'", 1, 1, true), ("DATE_FORMAT", "DATE_FORMAT(c, 'yyyy-MM') = '1976-07'", 1, 1, true), ("ENDSWITH", "ENDSWITH(s.b, '7.007')", 1, 1, true), ("LENGTH", "LENGTH(s.b) = 21", 1, 1, true), ("TRIM", "TRIM(CONCAT(' ', s.b, ' ')) = '1971-01-31 17:01:01.001'", 1, 1, true), ("LOWER", "LOWER(CONCAT('AAA', s.b)) = 'aaa1971-01-31 17:01:01.001'", 1, 1, true), ("CONCAT", "CONCAT(s.b, CAST(s.a AS STRING)) = '1971-01-31 17:01:01.0018'", 1, 1, true), ("FROM_UNIXTIME", "FROM_UNIXTIME(s.a) = '1969-12-31 16:00:00'", 1, 1, true), ("TO_UNIX_TIMESTAMP", "TO_UNIX_TIMESTAMP(CAST(s.b AS TIMESTAMP)) = 0", 1, 1, true), ("DATE_TRUNC", "DATE_TRUNC('MONTH', CAST(s.b AS DATE)) = '1976-07-01'", 1, 1, true), ("TRUNC", "TRUNC(CAST(s.b AS DATE), 'MONTH') = '1976-07-01'", 1, 1, true), ("DATE_FROM_UNIX_DATE", "DATE_FROM_UNIX_DATE(s.a) = '1970-01-05'", 1, 1, true), ("ISNULL", "CAST(s.a AS STRING) IS NULL", 1, 1, true), ("ISNOTNULL", "CAST(s.a AS STRING) IS NOT NULL", 10, 1, true), // Fully eligible compound predicates. ("NOT (valid)", "NOT CONTAINS(s.b, '7.007')", 9, 1, true), ( "AND (both valid)", "(CONTAINS(s.b, '-03') AND CONTAINS(s.b, '04')) OR ENDSWITH(s.b, '008')", 2, 1, true ), ("OR (both valid)", "ENDSWITH(s.b, '7.007') OR ENDSWITH(s.b, '8.008')", 2, 1, true), // Partially eligible compound predicates. ( "AND (one valid)", "(ISEVEN(s.a) AND ENDSWITH(s.b, '7.007')) OR ENDSWITH(s.b, '008')", 11, 0, false ), ( "OR (one valid)", "DATE_FROM_UNIX_DATE(e) = '1977-01-05' OR ENDSWITH(s.b, '7.007')", 11, 0, false ), ( "Inverted partially valid AND", "STRING((ENDSWITH(s.b, '7.007') OR ENDSWITH(s.b, '001')) AND e > 5) = 'false'", 11, 0, false ), // Fully ineligible compound predicates. ("NOT (invalid)", "NOT ISEVEN(s.a)", 11, 0, false), ( "AND (invalid)", "(LEN(STRING(d)) = 23 AND STRING(e) = '1') OR ENDSWITH(s.b, '008')", 11, 0, false ), ("OR (invalid)", "ISEVEN(s.a) OR STRING(e) = '1'", 11, 0, false), // Predicates on non-clustering columns. ("CAST", "CAST(e AS STRING) = '1'", 10, 0, false), ("DATE_FROM_UNIX_DATE", "DATE_FROM_UNIX_DATE(e) = '1977-01-05'", 10, 0, false), // Predicates on timestamp column. ("CAST", "CAST(d AS STRING) = '1970-01-01 00:00:00.000'", 10, 0, false), // Unsupported expressions. ("RAND", "RAND(0) * s.a > 0.5", 10, 0, false), ("UDF", "ISEVEN(s.a)", 11, 0, false), ("UDF", "RANDISEVEN(s.a)", 11, 0, false), ("SCALAR_SUBQUERY", s"DATE(s.b) = (SELECT(MAX(c)) FROM $testTableName)", 10, 0, false), ("REGEX", "REGEXP_EXTRACT(s.b, '([0-9][0-9][0-9][0-9]).*') = '1970'", 10, 0, false) ) // Test cases for combinations of partition-like data skipping with normal data skipping on the // shared table. // Each test case has the format: // (predicate, num matching files, number of partition-like predicates, // all predicates used for data skipping) // For these test cases, the number of matching files will be the sum across 3 groups of files: // 1. Files with the same min-max values and no nulls or all nulls (out of 11). // 2. Files with the same min-max values, but some nulls (out of 10). Only traditional data // skipping can affect these files. // 3. Files with different min-max values (out of 1). This file will always be read because it has // the full range of all values across the clustering columns. private val combinedDataSkippingTestCases = Seq( // All partition-like filters are eligible. ("s.a > 5 AND COALESCE(null, s.a, 1) < 7", 1 + 4 + 1, 1, true), ("e < 4 AND COALESCE(null, s.a, 1) < 7", 1 + 4 + 1, 1, true), ("d < '1975-01-01' AND DATE(s.b) > '1974-01-01'", 1 + 5 + 1, 1, true), // Some partition-like filters are eligible. ("ISEVEN(s.a) AND ENDSWITH(s.b, '007') AND s.a < 5", 1 + 5 + 1, 1, false), ("(ISEVEN(s.a) AND s.a > 5) OR ENDSWITH(s.b, '008')", 11 + 10 + 1, 0, false), ( "((ISEVEN(s.a) AND ENDSWITH(s.b, '007')) OR ENDSWITH(s.b, '008')) AND s.a < 5", 5 + 5 + 1, 0, false ), // No partition-like filters are eligible. ("ISEVEN(s.a) AND s.a < 5", 5 + 5 + 1, 0, false), ("STRING(e) = '1' AND s.a < 5", 5 + 5 + 1, 0, false) ) partitionLikeTestCases.foreach { case (name, predicate, expectedNumFiles, expectedNumPredicates, allPredicatesUsed) => test(s"partition-like data skipping for expression $name: $predicate") { validateExpectedScanMetrics( tableName = testTableName, query = s"SELECT * FROM $testTableName WHERE $predicate", expectedNumFiles = 11 + expectedNumFiles, expectedNumPartitionLikeDataFilters = expectedNumPredicates, allPredicatesUsed = allPredicatesUsed, minNumFilesToApply = 1L) } } combinedDataSkippingTestCases.foreach { case (predicate, expectedNumFiles, expectedNumPredicates, allPredicatesUsed) => test(s"combined data skipping test: $predicate") { validateExpectedScanMetrics( tableName = testTableName, query = s"SELECT * FROM $testTableName WHERE $predicate", expectedNumFiles = expectedNumFiles, expectedNumPartitionLikeDataFilters = expectedNumPredicates, allPredicatesUsed = allPredicatesUsed, minNumFilesToApply = 1L) } } test("partition-like data skipping not applied to truncated string column") { val tbl = "tbl" withClusteredTable(tbl, "a STRING, b BIGINT", "a, b") { // Insert 10 files with truncated string values. spark.range(10) .withColumnRenamed("id", "b") .withColumn("a", concat(lit("abcde" * 10), col("b"))) .select("a", "b") // Reorder columns to ensure the schema matches. .repartitionByRange(10, col("a")) .write.format("delta").mode("append").insertInto(tbl) // Insert 10 files with non-truncated string values. spark.range(10) .withColumnRenamed("id", "b") .withColumn("a", concat(lit("fghij" * 3), col("b"))) .select("a", "b") // Reorder columns to ensure the schema matches. .repartitionByRange(10, col("a")) .write.format("delta").mode("append").insertInto(tbl) // For a starts-with predicate (existing data skipping), we can skip files normally. validateExpectedScanMetrics( tbl, s"SELECT * FROM $tbl WHERE STARTSWITH(a, 'fghij')", 10, 0, true, 1L) // For an ends-with predicate, we can only skip files that don't have truncated stats. validateExpectedScanMetrics( tbl, s"SELECT * FROM $tbl WHERE ENDSWITH(a, '9')", 11, 1, true, 1L) } } test("partition-like data skipping evaluates file eligibility before skipping expression") { val tbl = "tbl" withClusteredTable(tbl, "a STRING, b BIGINT", "a") { spark.range(10) .withColumnRenamed("id", "b") .withColumn("a", concat(lit("abcde" * 10), lit("--"), col("b"))) .select("a", "b") // Reorder columns to ensure the schema matches. .repartitionByRange(10, col("a")) .write.format("delta").mode("append").insertInto(tbl) validateExpectedScanMetrics( tbl, s"SELECT * FROM $tbl WHERE SPLIT(a, '--')[1]=8", 10, 1, true, 1L) } } test("partition-like data skipping not applied to sufficiently small tables") { validateExpectedScanMetrics( tableName = testTableName, query = s"SELECT * FROM $testTableName WHERE COALESCE(null, s.a, 1) = 1", expectedNumFiles = 22, expectedNumPartitionLikeDataFilters = 0, allPredicatesUsed = false, minNumFilesToApply = 8000) } test("partition-like data skipping when predicate returns NULL") { // Predicate returns NULL both when the input attributes are NULL and when the input attributes // are non-null. val predicate = "GET(ARRAY(1, 2, 3), INT(SUBSTR(s.b, 4, 1))) IN (1, 2)" validateExpectedScanMetrics( tableName = testTableName, query = s"SELECT * FROM $testTableName WHERE $predicate", expectedNumFiles = 1 + 10 + 1, expectedNumPartitionLikeDataFilters = 1, allPredicatesUsed = true, minNumFilesToApply = 1) } test("partition-like data skipping expression references non-skipping eligible columns") { val tbl = "tbl" withClusteredTable( table = tbl, schema = "a BIGINT, b ARRAY, c STRUCT, e BIGINT>", clusterBy = "a") { spark.range(10) .withColumnRenamed("id", "a") .withColumn("b", array(col("a"), lit(0L))) .withColumn("c", struct(array(col("a"), lit(0L)), lit(0L))) .select("a", "b", "c") // Reorder columns to ensure the schema matches. .repartitionByRange(10, col("a")) .write.format("delta").mode("append").insertInto(tbl) // All files should be read because the filters are on columns that aren't skipping eligible. validateExpectedScanMetrics( tableName = tbl, query = s"SELECT * FROM $tbl WHERE GET(b, 1) = 0", expectedNumFiles = 10, expectedNumPartitionLikeDataFilters = 0, allPredicatesUsed = false, minNumFilesToApply = 1) validateExpectedScanMetrics( tableName = tbl, query = s"SELECT * FROM $tbl WHERE GET(c.d, 1) = 0", expectedNumFiles = 10, expectedNumPartitionLikeDataFilters = 0, allPredicatesUsed = false, minNumFilesToApply = 1) } } test("partition-like skipping can reference non-clustering columns via config") { withSQLConf( DeltaSQLConf.DELTA_DATASKIPPING_PARTITION_LIKE_FILTERS_CLUSTERING_COLUMNS_ONLY.key -> "false") { validateExpectedScanMetrics( tableName = testTableName, query = s"SELECT * FROM $testTableName WHERE CAST(e AS STRING) = '1'", expectedNumFiles = 12, expectedNumPartitionLikeDataFilters = 1, allPredicatesUsed = true, minNumFilesToApply = 1L) } } test("partition-like skipping whitelist can be expanded via config") { // Single additional supported expression. withSQLConf( DeltaSQLConf.DELTA_DATASKIPPING_PARTITION_LIKE_FILTERS_ADDITIONAL_SUPPORTED_EXPRESSIONS.key -> "org.apache.spark.sql.catalyst.expressions.RegExpExtract") { val query = s"SELECT * FROM $testTableName " + "WHERE REGEXP_EXTRACT(s.b, '([0-9][0-9][0-9][0-9]).*') = '1971'" validateExpectedScanMetrics( tableName = testTableName, query = query, expectedNumFiles = 12, expectedNumPartitionLikeDataFilters = 1, allPredicatesUsed = true, minNumFilesToApply = 1L) } // Multiple additional supported expressions. DeltaLog.clearCache() // Clear cache to avoid stale config reads. withSQLConf( DeltaSQLConf.DELTA_DATASKIPPING_PARTITION_LIKE_FILTERS_ADDITIONAL_SUPPORTED_EXPRESSIONS.key -> ("org.apache.spark.sql.catalyst.expressions.RegExpExtract," + "org.apache.spark.sql.catalyst.expressions.JsonToStructs")) { val query = s""" |SELECT * FROM $testTableName |WHERE (REGEXP_EXTRACT(s.b, '([0-9][0-9][0-9][0-9]).*') = '1971' OR |FROM_JSON(CONCAT('{"date":"', STRING(c), '"}'), 'date STRING')['date'] = '1972-03-02') |""".stripMargin validateExpectedScanMetrics( tableName = testTableName, query = query, expectedNumFiles = 13, expectedNumPartitionLikeDataFilters = 1, allPredicatesUsed = true, minNumFilesToApply = 1L) } } } class PartitionLikeDataSkippingSuite extends PartitionLikeDataSkippingSuiteBase class PartitionLikeDataSkippingColumnMappingSuite extends PartitionLikeDataSkippingSuiteBase with DeltaColumnMappingEnableIdMode { override def runAllTests: Boolean = true test("partition-like data skipping with special characters in column names") { val tbl = "tbl" withTable(tbl) { sql(s"CREATE TABLE $tbl SHALLOW CLONE $testTableName") sql(s"ALTER TABLE $tbl RENAME COLUMN c TO `a.b`") sql(s"ALTER TABLE $tbl ADD COLUMN `s.a` STRUCT") // Validate clustering columns are resolved with case-insensitive resolution. validateExpectedScanMetrics( tableName = tbl, query = s"SELECT * FROM $tbl WHERE DATE_FORMAT(`A.B`, 'yyyy-MM') = '1976-07'", expectedNumFiles = 12, expectedNumPartitionLikeDataFilters = 1, allPredicatesUsed = true, minNumFilesToApply = 1L) // Predicate not on a clustering column - should not be eligible for partition-like data // skipping. validateExpectedScanMetrics( tableName = tbl, query = s"SELECT * FROM $tbl WHERE COALESCE(null, `s.a`.b, 1) = 1", expectedNumFiles = 22, allPredicatesUsed = false, expectedNumPartitionLikeDataFilters = 0, minNumFilesToApply = 1L) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/stats/PreparedDeltaFileIndexRowCountSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats import org.apache.spark.sql.delta.{DeltaLog, DeltaTable} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.{DataFrame, QueryTest} import org.apache.spark.sql.functions._ import org.apache.spark.sql.test.SharedSparkSession /** * Test suite to verify when preparedScan.scanned.rows is populated in PreparedDeltaFileIndex, * and the behavior of the DELTA_ALWAYS_COLLECT_STATS flag. */ class PreparedDeltaFileIndexRowCountSuite extends QueryTest with DeltaSQLCommandTest { import testImplicits._ private def getDeltaScan(df: DataFrame): DeltaScan = { val scans = df.queryExecution.optimizedPlan.collect { case DeltaTable(prepared: PreparedDeltaFileIndex) => prepared.preparedScan } assert(scans.size == 1, s"Expected 1 DeltaScan, found ${scans.size}") scans.head } /** * Test utility that creates a partitioned Delta table and verifies scanned.rows and * scanned.logicalRows behavior. * * @param alwaysCollectStats value of the DELTA_ALWAYS_COLLECT_STATS flag * @param queryTransform function to transform the base DataFrame (apply filters) * @param expectedRowsDefined whether scanned.rows should be defined * @param expectedRowCount expected row count if defined (None to skip validation) * @param expectedLogicalRowsDefined whether scanned.logicalRows should be defined * (defaults to same as expectedRowsDefined) * @param expectedLogicalRowCount expected logical row count if defined (None to skip validation) */ private def testRowCountBehavior( alwaysCollectStats: Boolean, queryTransform: DataFrame => DataFrame, expectedRowsDefined: Boolean, expectedRowCount: Option[Long] = None, expectedLogicalRowsDefined: Option[Boolean] = None, expectedLogicalRowCount: Option[Long] = None): Unit = { withTempDir { dir => withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "true") { spark.range(100).toDF("id") .withColumn("part", $"id" % 4) .repartition(4) .write.format("delta").partitionBy("part").save(dir.getAbsolutePath) } DeltaLog.clearCache() withSQLConf(DeltaSQLConf.DELTA_ALWAYS_COLLECT_STATS.key -> alwaysCollectStats.toString) { val df = spark.read.format("delta").load(dir.getAbsolutePath) val scan = getDeltaScan(queryTransform(df)) if (expectedRowsDefined) { assert(scan.scanned.rows.isDefined, "scanned.rows should be defined") expectedRowCount.foreach { expected => assert(scan.scanned.rows.get == expected, s"Expected $expected rows, got ${scan.scanned.rows.get}") } } else { assert(scan.scanned.rows.isEmpty, "scanned.rows should be None") } // logicalRows should follow the same defined/undefined pattern as rows by default val logicalDefined = expectedLogicalRowsDefined.getOrElse(expectedRowsDefined) if (logicalDefined) { assert(scan.scanned.logicalRows.isDefined, "scanned.logicalRows should be defined") expectedLogicalRowCount.foreach { expected => assert(scan.scanned.logicalRows.get == expected, s"Expected $expected logical rows, got ${scan.scanned.logicalRows.get}") } } else { assert(scan.scanned.logicalRows.isEmpty, "scanned.logicalRows should be None") } } } } // Define query cases: (name, transform function, always collects rows) private val queryCases: Seq[(String, DataFrame => DataFrame, Boolean)] = Seq( ("no filter", identity[DataFrame], false), ("TrueLiteral filter", _.where(lit(true)), false), ("partition filter only", _.where($"part" === 1), false), ("data filter", _.where($"id" === 50), true), ("partition + data filter", _.where($"part" === 1).where($"id" === 49), true) ) // Grid test: all query cases x flag values for { (caseName, queryTransform, alwaysCollectsRows) <- queryCases alwaysCollectStats <- Seq(false, true) } { val flagDesc = s"alwaysCollectStats=$alwaysCollectStats" // If the query type always collects rows, rows is always defined; otherwise depends on flag val expectedRowsDefined = alwaysCollectsRows || alwaysCollectStats test(s"$caseName - $flagDesc") { testRowCountBehavior( alwaysCollectStats = alwaysCollectStats, queryTransform = queryTransform, expectedRowsDefined = expectedRowsDefined ) } } test("alwaysCollectStats with missing stats returns None") { withTempDir { dir => // Create table without stats withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false") { spark.range(100).toDF("id") .write.format("delta").save(dir.getAbsolutePath) } DeltaLog.clearCache() withSQLConf(DeltaSQLConf.DELTA_ALWAYS_COLLECT_STATS.key -> "true") { val df = spark.read.format("delta").load(dir.getAbsolutePath) val scan = getDeltaScan(df) assert(scan.scanned.rows.isEmpty, "scanned.rows should be None when stats are missing") assert(scan.scanned.logicalRows.isEmpty, "scanned.logicalRows should be None when stats are missing") } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/stats/StatsCollectionSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats import java.math.BigDecimal import java.sql.Date import java.time.LocalDateTime // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.DeltaColumnMapping import org.apache.spark.sql.delta.actions.Protocol import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.StatisticsCollection.{ASCII_MAX_CHARACTER, UTF8_MAX_CHARACTER} import org.apache.spark.sql.delta.test.{DeltaExceptionTestUtils, DeltaSQLCommandTest, DeltaSQLTestUtils, TestsStatistics} import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import org.apache.hadoop.fs.Path import org.scalatest.exceptions.TestFailedException import org.apache.spark.SparkException import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.expressions.{GenericRow, GenericRowWithSchema} import org.apache.spark.sql.functions._ import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ import org.apache.spark.sql.types.{IntegerType, MetadataBuilder, StringType, StructType} class StatsCollectionSuite extends QueryTest with SharedSparkSession with DeltaColumnMappingTestUtils with TestsStatistics with DeltaSQLCommandTest with DeltaSQLTestUtils with DeletionVectorsTestUtils with DeltaExceptionTestUtils { import testImplicits._ test("on write") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) val data = Seq(1, 2, 3).toDF().coalesce(1) data.write.format("delta").save(dir.getAbsolutePath) val snapshot = deltaLog.update() val statsJson = deltaLog.update().allFiles.head().stats // convert data schema to physical name if possible val dataRenamed = data.toDF( data.columns.map(name => getPhysicalName(name, deltaLog.snapshot.schema)): _*) val skipping = new StatisticsCollection { override val spark = StatsCollectionSuite.this.spark override def tableSchema: StructType = dataRenamed.schema override def outputTableStatsSchema: StructType = dataRenamed.schema override def outputAttributeSchema: StructType = dataRenamed.schema override val statsColumnSpec = DeltaStatsColumnSpec( None, Some( DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.fromString( DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.defaultValue) ) ) override def columnMappingMode: DeltaColumnMappingMode = deltaLog.snapshot.columnMappingMode override val protocol: Protocol = snapshot.protocol override def getDataSkippingStringPrefixLength: Int = StatsCollectionUtils.getDataSkippingStringPrefixLength(spark, snapshot.metadata) } val correctAnswer = dataRenamed .select(skipping.statsCollector) .select(to_json($"stats").as[String]) .collect() .head assert(statsJson === correctAnswer) } } test("gather stats") { withTempDir { dir => val deltaLog = DeltaLog.forTable(spark, dir) val data = spark.range(1, 10, 1, 10).withColumn("odd", $"id" % 2 === 1) data.write.partitionBy("odd").format("delta").save(dir.getAbsolutePath) val df = spark.read.format("delta").load(dir.getAbsolutePath) withSQLConf("spark.sql.parquet.filterPushdown" -> "false") { assert(recordsScanned(df) == 9) assert(recordsScanned(df.where("id = 1")) == 1) } } } test("statistics re-computation throws error on Delta tables with DVs") { withDeletionVectorsEnabled() { withTempDir { dir => val df = spark.range(start = 0, end = 20).toDF().repartition(numPartitions = 4) df.write.format("delta").save(dir.toString()) spark.sql(s"DELETE FROM delta.`${dir.toString}` WHERE id in (2, 15)") val e = intercept[DeltaCommandUnsupportedWithDeletionVectorsException] { val deltaLog = DeltaLog.forTable(spark, dir) StatisticsCollection.recompute(spark, deltaLog) } assert(e.getErrorClass == "DELTA_UNSUPPORTED_STATS_RECOMPUTE_WITH_DELETION_VECTORS") assert(e.getSqlState == "0AKDD") assert(e.getMessage == "[DELTA_UNSUPPORTED_STATS_RECOMPUTE_WITH_DELETION_VECTORS] " + "Statistics re-computation on a Delta table with deletion " + "vectors is not yet supported.") } } } statsTest("recompute stats basic") { withTempDir { tempDir => withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false") { val df = spark.range(2).coalesce(1).toDF() df.write.format("delta").save(tempDir.toString()) val deltaLog = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) assert(statsDF(deltaLog).where('numRecords.isNotNull).count() == 0) { StatisticsCollection.recompute(spark, deltaLog) } checkAnswer( spark.read.format("delta").load(tempDir.getCanonicalPath), df ) val statsDf = statsDF(deltaLog) assert(statsDf.where('numRecords.isNotNull).count() > 0) // Make sure stats indicate 2 rows, min [0], max [1] checkAnswer(statsDf, Row(2, Row(0), Row(1))) } } } statsTestSparkMasterOnly("recompute variant stats") { withTempDir { tempDir => withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false") { val df = spark.range(2) .selectExpr( "case when id % 2 = 0 then parse_json(cast(id as string)) else null end as v" ) .coalesce(1) .toDF() df.write.format("delta").save(tempDir.toString()) val deltaLog = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) assert(getStatsDf(deltaLog, Seq($"numRecords")).where('numRecords.isNotNull).count() == 0) { StatisticsCollection.recompute(spark, deltaLog) } checkAnswer( spark.read.format("delta").load(tempDir.getCanonicalPath), df ) val statsDf = getStatsDf(deltaLog, Seq($"numRecords", $"nullCount")) assert(statsDf.where('numRecords.isNotNull).count() > 0) // Make sure stats indicate 2 rows, nullCount [1] checkAnswer(statsDf, Row(2, Row(1))) } } } statsTest("recompute stats multiple columns and files") { withTempDir { tempDir => withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false") { val df = spark.range(10, 20).withColumn("x", 'id + 10).repartition(3) df.write.format("delta").save(tempDir.toString()) val deltaLog = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) assert(statsDF(deltaLog).where('numRecords.isNotNull).count() == 0) { StatisticsCollection.recompute(spark, deltaLog) } checkAnswer( spark.read.format("delta").load(tempDir.getCanonicalPath), df ) val statsDf = statsDF(deltaLog) assert(statsDf.where('numRecords.isNotNull).count() > 0) // scalastyle:off line.size.limit val expectedStats = Seq(Row(3, Row(10, 20), Row(19, 29)), Row(4, Row(12, 22), Row(17, 27)), Row(3, Row(11, 21), Row(18, 28))) // scalastyle:on line.size.limit checkAnswer(statsDf, expectedStats) } } } statsTest("recompute stats on partitioned table") { withTempDir { tempDir => withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false") { val df = spark.range(15).toDF("a") .withColumn("b", 'a % 3) .withColumn("c", 'a % 2) .repartition(3, 'b) df.write.format("delta").partitionBy("b").save(tempDir.toString()) val deltaLog = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) assert(statsDF(deltaLog).where('numRecords.isNotNull).count() == 0) { StatisticsCollection.recompute(spark, deltaLog) } checkAnswer( spark.read.format("delta").load(tempDir.getCanonicalPath), df ) val statsDf = statsDF(deltaLog) assert(statsDf.where('numRecords.isNotNull).count() > 0) checkAnswer(statsDf, Seq( Row(5, Row(1, 0), Row(13, 1)), Row(5, Row(0, 0), Row(12, 1)), Row(5, Row(2, 0), Row(14, 1)))) } } } statsTest("recompute stats with partition predicates") { withTempDir { tempDir => withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false") { val df = Seq( (1, 0, 10), (1, 2, 20), (1, 4, 30), (2, 6, 40), (2, 8, 50), (3, 10, 60), (4, 12, 70)) .toDF("a", "b", "c") df.write.format("delta").partitionBy("a").save(tempDir.toString()) val deltaLog = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) assert(statsDF(deltaLog).where('numRecords.isNotNull).count() == 0) { StatisticsCollection.recompute(spark, deltaLog, Seq(('a > 1).expr, ('a < 4).expr)) } checkAnswer( spark.read.format("delta").load(tempDir.getCanonicalPath), df ) val statsDf = statsDF(deltaLog) assert(statsDf.where('numRecords.isNotNull).count() == 2) checkAnswer(statsDf, Seq( Row(null, Row(null, null), Row(null, null)), Row(2, Row(6, 40), Row(8, 50)), Row(1, Row(10, 60), Row(10, 60)), Row(null, Row(null, null), Row(null, null)))) } } } statsTest("recompute stats with invalid partition predicates") { withTempDir { tempDir => withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false") { Seq((1, 0, 10), (1, 2, 20), (1, 4, 30), (2, 6, 40), (2, 8, 50), (3, 10, 60), (4, 12, 70)) .toDF("a", "b", "c") .write.format("delta").partitionBy("a").save(tempDir.toString()) val deltaLog = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) assert(statsDF(deltaLog).where('numRecords.isNotNull).count() == 0) { intercept[AnalysisException] { StatisticsCollection.recompute(spark, deltaLog, Seq(('b > 1).expr)) } intercept[AnalysisException] { StatisticsCollection.recompute(spark, deltaLog, Seq(('a > 1).expr, ('c > 1).expr)) } } assert(statsDF(deltaLog).where('numRecords.isNotNull).count() == 0) } } } statsTest("recompute stats on a table with corrupted stats") { withTempDir { tempDir => val df = Seq( (1, 0, 10), (1, 2, 20), (1, 4, 30), (2, 6, 40), (2, 8, 50), (3, 10, 60), (4, 12, 70)) .toDF("a", "b", "c") df.write.format("delta").partitionBy("a").save(tempDir.toString()) val deltaLog = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) val correctStats = statsDF(deltaLog) assert(correctStats.where('numRecords.isNotNull).count() == 4) // use physical names if possible val (a, b, c) = ( getPhysicalName("a", deltaLog.snapshot.schema), getPhysicalName("b", deltaLog.snapshot.schema), getPhysicalName("c", deltaLog.snapshot.schema) ) { // Corrupt stats on one of the files val txn = deltaLog.startTransaction() val f = deltaLog.snapshot.allFiles.filter(_.partitionValues(a) == "1").first() val corrupted = f.copy(stats = f.stats.replace( s"""maxValues":{"$b":4,"$c":30}""", s"""maxValues":{"$b":-100,"$c":100}""")) txn.commit(Seq(corrupted), DeltaOperations.ComputeStats(Nil)) intercept[TestFailedException] { checkAnswer(statsDF(deltaLog), correctStats) } // Recompute stats and verify they match the original ones StatisticsCollection.recompute(spark, deltaLog) checkAnswer( spark.read.format("delta").load(tempDir.getCanonicalPath), df ) checkAnswer(statsDF(deltaLog), correctStats) } } } statsTest("recompute stats with file filter") { withTempDir { tempDir => withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false") { val df = Seq( (1, 0, 10), (1, 2, 20), (1, 4, 30), (2, 6, 40), (2, 8, 50), (3, 10, 60), (4, 12, 70)) .toDF("a", "b", "c") df.write.format("delta").partitionBy("a").save(tempDir.toString()) val deltaLog = DeltaLog.forTable(spark, new Path(tempDir.getCanonicalPath)) assert(statsDF(deltaLog).where('numRecords.isNotNull).count() == 0) val biggest = deltaLog.snapshot.allFiles.agg(max('size)).first().getLong(0) { StatisticsCollection.recompute( spark, deltaLog, catalogTable = None, fileFilter = _.size == biggest) } checkAnswer( spark.read.format("delta").load(tempDir.getCanonicalPath), df ) val statsDf = statsDF(deltaLog) assert(statsDf.where('numRecords.isNotNull).count() == 1) checkAnswer(statsDf, Seq( Row(null, Row(null, null), Row(null, null)), Row(null, Row(null, null), Row(null, null)), Row(null, Row(null, null), Row(null, null)), Row(3, Row(0, 10), Row(4, 30)))) } } } test("Truncate min string") { // scalastyle:off nonascii val inputToExpected = Seq( (s"abcd", s"abc", 3), (s"abcdef", s"abcdef", 6), (s"abcde�", s"abcde�", 6), (s"$UTF8_MAX_CHARACTER$UTF8_MAX_CHARACTER$UTF8_MAX_CHARACTER$UTF8_MAX_CHARACTER", s"$UTF8_MAX_CHARACTER", 1), (s"$UTF8_MAX_CHARACTER$UTF8_MAX_CHARACTER", s"$UTF8_MAX_CHARACTER", 1), (s"abcd", null, 0) ) inputToExpected.foreach { case (input, expected, prefixLen) => val actual = StatisticsCollection.truncateMinStringAgg(prefixLen)(input) val debugMsg = s"input:$input, actual:$actual, expected:$expected" assert(actual == expected, debugMsg) if (actual != null) { assert(input.startsWith(actual), debugMsg) } } // scalastyle:on nonascii } test("Truncate max string") { // scalastyle:off nonascii val inputToExpected = Seq( (s"abcd", null, 0), (s"a${UTF8_MAX_CHARACTER}d", s"a$UTF8_MAX_CHARACTER$ASCII_MAX_CHARACTER", 2), (s"abcd", s"abcd", 6), (s"abcdef", s"abcdef", 6), (s"abcde�", s"abcde�", 6), (s"abcd�abcd", s"abcd�a$ASCII_MAX_CHARACTER", 6), (s"�abcd", s"�abcd", 6), (s"abcdef�", s"abcdef$UTF8_MAX_CHARACTER", 6), (s"abcdef��", s"abcdef$UTF8_MAX_CHARACTER", 6), (s"abcdef-abcdef�", s"abcdef$ASCII_MAX_CHARACTER", 6), (s"abcdef�abcdef", s"abcdef$UTF8_MAX_CHARACTER", 6), (s"abcde�abcdef�abcdef�abcdef", s"abcde�$ASCII_MAX_CHARACTER", 6), (s"漢字仮名한글தமி", s"漢字仮名한글$UTF8_MAX_CHARACTER", 6), (s"漢字仮名한글��", s"漢字仮名한글$UTF8_MAX_CHARACTER", 6), (s"漢字仮名한글", s"漢字仮名한글", 6), (s"abcdef🚀", s"abcdef$UTF8_MAX_CHARACTER", 6), (s"$UTF8_MAX_CHARACTER$UTF8_MAX_CHARACTER$UTF8_MAX_CHARACTER$UTF8_MAX_CHARACTER", null, 1), (s"$UTF8_MAX_CHARACTER$UTF8_MAX_CHARACTER$UTF8_MAX_CHARACTER$UTF8_MAX_CHARACTER", s"$UTF8_MAX_CHARACTER$UTF8_MAX_CHARACTER$UTF8_MAX_CHARACTER$UTF8_MAX_CHARACTER", 4), (s"����", s"��$UTF8_MAX_CHARACTER", 2), (s"���", s"�$UTF8_MAX_CHARACTER", 1), ("abcdefghijklm💞😉💕\n🥀🌹💐🌺🌷🌼🌻🌷🥀", s"abcdefghijklm💞😉💕\n🥀🌹💐🌺🌷🌼$UTF8_MAX_CHARACTER", 32) ) inputToExpected.foreach { case (input, expected, prefixLen) => val actual = StatisticsCollection.truncateMaxStringAgg(prefixLen)(input) // `Actual` should be higher or equal than `input` in UTF-8 encoded binary order. val debugMsg = s"input:$input, actual:$actual, expected:$expected" assert(actual == expected, debugMsg) } // scalastyle:off nonascii } test(s"Optimize Zorder for delta statistics column: table creation") { val tableName = "delta_table" withTable(tableName) { sql("create table delta_table (c1 long, c2 long) " + "using delta " + "TBLPROPERTIES('delta.dataSkippingStatsColumns' = 'c1,c2', " + "'delta.dataSkippingNumIndexedCols' = 0)") for (_ <- 1 to 10) { sql("insert into delta_table values(1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8)") } sql("optimize delta_table zorder by (c1)") sql("optimize delta_table zorder by (c2)") sql("optimize delta_table zorder by (c1,c2)") } } test(s"Optimize Zorder for delta statistics column: alter TBLPROPERTIES") { val tableName = "delta_table" withTable(tableName) { sql("create table delta_table (c1 long, c2 long) " + "using delta TBLPROPERTIES('delta.dataSkippingNumIndexedCols' = 0)") intercept[DeltaAnalysisException] { sql("optimize delta_table zorder by (c1)") } intercept[DeltaAnalysisException] { sql("optimize delta_table zorder by (c2)") } intercept[DeltaAnalysisException] { sql("optimize delta_table zorder by (c1,c2)") } for (_ <- 1 to 10) { sql("insert into delta_table values(1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8)") } sql("ALTER TABLE delta_table SET TBLPROPERTIES ('delta.dataSkippingStatsColumns' = 'c1,c2')") sql("optimize delta_table zorder by (c1)") sql("optimize delta_table zorder by (c2)") sql("optimize delta_table zorder by (c1,c2)") } } test(s"Delta statistic column: special characters") { val tableName = "delta_table_1" withTable(tableName) { sql( s"create table $tableName (`c1.` long, `c2*` long, `c3,` long, `c-4` long) using delta " + s"TBLPROPERTIES(" + s"'delta.dataSkippingStatsColumns'='`c1.`,`c2*`,`c3,`,`c-4`'," + s"'delta.columnMapping.mode' = 'name')" ) val dataSkippingStatsColumns = sql(s"SHOW TBLPROPERTIES $tableName") .collect() .map { row => row.getString(0) -> row.getString(1) } .filter(_._1 == "delta.dataSkippingStatsColumns") .toSeq val result1 = Seq(("delta.dataSkippingStatsColumns", "`c1.`,`c2*`,`c3,`,`c-4`")) assert(dataSkippingStatsColumns == result1) } } Seq("c1.", "c2*", "c3,", "c-4").foreach { col => test(s"Delta statistic column: invalid special characters $col") { val tableName = "delta_table_1" withTable(tableName) { val except = intercept[Exception] { sql( s"create table $tableName (`c1.` long, `c2*` long, `c3,` long, c4 long) using delta " + s"TBLPROPERTIES(" + s"'delta.dataSkippingStatsColumns'='$col'," + s"'delta.columnMapping.mode' = 'name')" ) } } } } Seq( ("BINARY", "BinaryType"), ("BOOLEAN", "BooleanType"), ("ARRAY", "ArrayType(ByteType,true)"), ("MAP", "MapType(DateType,IntegerType,true)") ).foreach { case (invalidType, typename) => val tableName1 = "delta_table_1" val tableName2 = "delta_table_2" test(s"Delta statistic column: invalid data type $invalidType") { withTable(tableName1, tableName2) { val columnName = "c2" val exceptOne = intercept[DeltaIllegalArgumentException] { sql( s"create table $tableName1 (c1 long, c2 $invalidType) using delta " + s"TBLPROPERTIES('delta.dataSkippingStatsColumns'='c2')" ) } assert( exceptOne.getErrorClass == "DELTA_COLUMN_DATA_SKIPPING_NOT_SUPPORTED_TYPE" && exceptOne.getMessageParametersArray.toSeq == Seq(columnName, typename) ) sql(s"create table $tableName2 (c1 long, c2 $invalidType) using delta") val exceptTwo = interceptWithUnwrapping[DeltaIllegalArgumentException] { sql(s"ALTER TABLE $tableName2 SET TBLPROPERTIES('delta.dataSkippingStatsColumns' = 'c2')") } assert( exceptTwo.getErrorClass == "DELTA_COLUMN_DATA_SKIPPING_NOT_SUPPORTED_TYPE" && exceptTwo.getMessageParametersArray.toSeq == Seq(columnName, typename) ) } } } test(s"Delta statistic column: mix case column name") { val tableName = "delta_table_1" withTable(tableName) { sql( s"create table $tableName (cOl1 LONG, COL2 struct, CoL3 LONG) " + s"using delta TBLPROPERTIES" + s"('delta.dataSkippingStatsColumns' = 'coL1, COL2.col20, COL2.col21, cOl3');" ) (1 to 10).foreach { _ => sql( s"""insert into $tableName values |(1, struct(1, 10), 1), (2, struct(2, 20), 2), (3, struct(3, 30), 3), |(4, struct(4, 40), 4), (5, struct(5, 50), 5), (6, struct(6, 60), 6), |(7, struct(7, 70), 7), (8, struct(8, 80), 8), (9, struct(9, 90), 9), |(10, struct(10, 100), 10), (null, struct(null, null), null), |(-1, struct(-1, -100), -1), (null, struct(null, null), null);""".stripMargin ) } sql(s"optimize $tableName") val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val df = deltaLog.update().withStatsDeduplicated val analyzedDfPlan = df.queryExecution.analyzed.toString val stats = if (analyzedDfPlan.indexOf("stats_parsed") > 0) "stats_parsed" else "stats" df.select(s"$stats.numRecords", s"$stats.nullCount", s"$stats.minValues", s"$stats.maxValues") .collect() .foreach { row => assert(row(0) == 130) assert(row(1).asInstanceOf[GenericRow] == Row(20, Row(20, 20), 20)) assert(row(2) == Row(-1, Row(-1, -100), -1)) assert(row(3) == Row(10, Row(10, 100), 10)) } } } Seq( "BIGINT", "DATE", "DECIMAL(3, 2)", "DOUBLE", "FLOAT", "INT", "SMALLINT", "STRING", "TIMESTAMP", "TIMESTAMP_NTZ", "TINYINT", "STRUCT", "STRUCT>" ).foreach { validType => val tableName1 = "delta_table_1" val tableName2 = "delta_table_2" test(s"Delta statistic column: valid data type $validType") { withTable(tableName1, tableName2) { sql( s"create table $tableName1 (c1 long, c2 $validType) using delta " + s"TBLPROPERTIES('delta.dataSkippingStatsColumns' = 'c2')" ) sql(s"create table $tableName2 (c1 long, c2 $validType) using delta") sql(s"ALTER TABLE $tableName2 SET TBLPROPERTIES('delta.dataSkippingStatsColumns'='c2')") } } test(s"Delta statistic column: valid data type $validType in nested column") { val tableName3 = "delta_table_3" val tableName4 = "delta_table_4" withTable(tableName1, tableName2, tableName3, tableName4) { sql( s"create table $tableName1 (c1 long, c2 STRUCT) " + s"using delta TBLPROPERTIES('delta.dataSkippingStatsColumns' = 'c2.c21')" ) sql( s"create table $tableName2 (c1 long, c2 STRUCT) " + s"using delta TBLPROPERTIES('delta.dataSkippingStatsColumns' = 'c2')" ) sql(s"create table $tableName3 (c1 long, c2 STRUCT) using delta") sql(s"ALTER TABLE $tableName3 SET TBLPROPERTIES('delta.dataSkippingStatsColumns'='c2.c21')") sql(s"create table $tableName4 (c1 long, c2 STRUCT) using delta") sql(s"ALTER TABLE $tableName4 SET TBLPROPERTIES('delta.dataSkippingStatsColumns'='c2')") } } } Seq("create", "alter").foreach { label => val tableName = "delta_table" val propertyName = "delta.dataSkippingStatsColumns" test(s"Delta statistics column with partition column: $label") { withTable(tableName) { if (label == "create") { val except = intercept[DeltaIllegalArgumentException] { sql( "create table delta_table(c0 int, c1 int) using delta partitioned by(c1) " + "TBLPROPERTIES('delta.dataSkippingStatsColumns' = 'c1')" ) } assert( except.getErrorClass == "DELTA_COLUMN_DATA_SKIPPING_NOT_SUPPORTED_PARTITIONED_COLUMN" && except.getMessageParametersArray.toSeq == Seq("c1") ) } else { sql("create table delta_table(c0 int, c1 int) using delta partitioned by(c1)") val except = interceptWithUnwrapping[DeltaIllegalArgumentException] { sql( "ALTER TABLE delta_table SET TBLPROPERTIES ('delta.dataSkippingStatsColumns' = 'c1')" ) } assert( except.getErrorClass == "DELTA_COLUMN_DATA_SKIPPING_NOT_SUPPORTED_PARTITIONED_COLUMN" && except.getMessageParametersArray.toSeq == Seq("c1") ) } } } test(s"Rename Nested Columns with delta statistics column: $label") { withTable(tableName) { if (label == "create") { sql( "create table delta_table (" + " id long," + " info STRUCT >, " + " prev_job STRUCT >)" + " using delta TBLPROPERTIES(" + s"'$propertyName' = 'info.title,info.depart.org,info.depart.perf'," + "'delta.columnMapping.mode' = 'name', " + "'delta.minReaderVersion' = '2', " + "'delta.minWriterVersion' = '5')" ) } else { sql( "create table delta_table (" + " id long," + " info STRUCT >, " + " prev_job STRUCT >)" + " using delta TBLPROPERTIES(" + "'delta.columnMapping.mode' = 'name', " + "'delta.minReaderVersion' = '2', " + "'delta.minWriterVersion' = '5')" ) } if (label == "alter") { sql(s"alter table delta_table set TBLPROPERTIES(" + s"'$propertyName' = 'info.title,info.depart.org,info.depart.perf')") } // Rename nested column leaf. sql("ALTER TABLE delta_table RENAME COLUMN info.title TO title_name;") var dataSkippingStatsColumns = sql("SHOW TBLPROPERTIES delta_table") .collect() .map { row => row.getString(0) -> row.getString(1) } .filter(_._1 == propertyName) .toSeq val result1 = Seq((propertyName, "info.title_name,info.depart.org,info.depart.perf")) assert(dataSkippingStatsColumns == result1) // Rename nested column root. sql("ALTER TABLE delta_table RENAME COLUMN info TO detail") dataSkippingStatsColumns = sql("SHOW TBLPROPERTIES delta_table") .collect() .map { row => row.getString(0) -> row.getString(1) } .filter(_._1 == propertyName) .toSeq val result2 = Seq( (propertyName, "detail.title_name,detail.depart.org,detail.depart.perf") ) assert(dataSkippingStatsColumns == result2) // Rename nested column intermediate node. sql("ALTER TABLE delta_table RENAME COLUMN detail.DEPART TO organization") dataSkippingStatsColumns = sql("SHOW TBLPROPERTIES delta_table") .collect() .map { row => row.getString(0) -> row.getString(1) } .filter(_._1 == propertyName) .toSeq val result3 = Seq( (propertyName, "detail.title_name,detail.organization.org,detail.organization.perf") ) assert(dataSkippingStatsColumns == result3) } } test(s"Drop Nested Columns with delta statistics column: $label") { withTable(tableName) { if (label == "create") { sql( "create table delta_table (" + " id long, " + " info STRUCT >, " + " prev_job STRUCT >)" + " using delta TBLPROPERTIES(" + s"'$propertyName' = " + "'info.title,info.depart.org,info.depart.perf,prev_job.title,prev_job.depart.perf', " + "'delta.columnMapping.mode' = 'name', " + "'delta.minReaderVersion' = '2', " + "'delta.minWriterVersion' = '5')" ) } else { sql( "create table delta_table (" + " id long," + " info STRUCT>, " + " prev_job STRUCT>)" + " using delta TBLPROPERTIES(" + "'delta.columnMapping.mode' = 'name', " + "'delta.minReaderVersion' = '2', " + "'delta.minWriterVersion' = '5')" ) } if (label == "alter") { sql( s"alter table delta_table set TBLPROPERTIES(" + s"'$propertyName' = " + s"'info.title,info.depart.org,info.depart.perf,prev_job.title,prev_job.depart.perf')" ) } // Drop nested column leaf. sql("ALTER TABLE delta_table DROP COLUMN info.title;") var dataSkippingStatsColumns = sql("SHOW TBLPROPERTIES delta_table") .collect() .map { row => row.getString(0) -> row.getString(1) } .filter(_._1 == propertyName) .toSeq val result1 = Seq( (propertyName, "info.depart.org,info.depart.perf,prev_job.title,prev_job.depart.perf") ) assert(dataSkippingStatsColumns == result1) // Drop nested column intermediate node. sql("ALTER TABLE delta_table DROP COLUMN info.depart;") dataSkippingStatsColumns = sql("SHOW TBLPROPERTIES delta_table") .collect() .map { row => row.getString(0) -> row.getString(1) } .filter(_._1 == propertyName) .toSeq val result3 = Seq((propertyName, "prev_job.title,prev_job.depart.perf")) assert(dataSkippingStatsColumns == result3) // Rename nested column root node. sql("ALTER TABLE delta_table DROP COLUMN prev_job;") dataSkippingStatsColumns = sql("SHOW TBLPROPERTIES delta_table") .collect() .map { row => row.getString(0) -> row.getString(1) } .filter(_._1 == propertyName) .toSeq val result2 = Seq((propertyName, "")) assert(dataSkippingStatsColumns == result2) } } } test("Change Columns with delta statistics column") { Seq( "BIGINT", "DATE", "DECIMAL(3, 2)", "DOUBLE", "FLOAT", "INT", "SMALLINT", "STRING", "TIMESTAMP", "TIMESTAMP_NTZ", "TINYINT" ).foreach { validType => Seq( "BINARY", "BOOLEAN", "ARRAY", "MAP", "STRUCT>" ).foreach { invalidType => withTable("delta_table") { sql( s"create table delta_table (c0 long, c1 long, c2 $validType) using delta " + s"TBLPROPERTIES('delta.dataSkippingStatsColumns' = 'c1,c2', " + "'delta.columnMapping.mode' = 'name', " + "'delta.minReaderVersion' = '2', " + "'delta.minWriterVersion' = '5')" ) intercept[AnalysisException] { sql(s"ALTER TABLE delta_table Change c2 TYPE $invalidType;") } } } } } test("Duplicated delta statistic columns: create") { Seq( ("'c0,c0'", "c0"), ("'c1,c1.c11'", "c1.c11"), ("'c1.c11,c1.c11'", "c1.c11"), ("'c1,c1'", "c1.c11,c1.c12") ).foreach { case (statsColumns, duplicatedColumns) => val exception = intercept[DeltaIllegalArgumentException] { sql( s"create table delta_table (c0 long, c1 struct) using delta " + s"TBLPROPERTIES('delta.dataSkippingStatsColumns' = $statsColumns, " + "'delta.columnMapping.mode' = 'name')" ) } assert( exception.getErrorClass == "DELTA_DUPLICATE_DATA_SKIPPING_COLUMNS" && exception.getMessageParametersArray.toSeq == Seq(duplicatedColumns) ) } } test("Duplicated delta statistic columns: alter") { sql( s"create table delta_table_t1 (c0 long, c1 struct) using delta " + s"TBLPROPERTIES('delta.columnMapping.mode' = 'name')" ) Seq( ("'c0,c0'", "c0"), ("'c1,c1.c11'", "c1.c11"), ("'c1.c11,c1.c11'", "c1.c11"), ("'c1,c1'", "c1.c11,c1.c12") ).foreach { case (statsColumns, duplicatedColumns) => val exception = interceptWithUnwrapping[DeltaIllegalArgumentException] { sql( s"ALTER TABLE delta_table_t1 " + s"SET TBLPROPERTIES('delta.dataSkippingStatsColumns'=$statsColumns)" ) } assert( exception.getErrorClass == "DELTA_DUPLICATE_DATA_SKIPPING_COLUMNS" && exception.getMessageParametersArray.toSeq == Seq(duplicatedColumns) ) } } test("handle special nested characters in column name") { withTable("t") { sql( s"create table t (`|` long, c struct, `s.a` int>) using delta " + s"TBLPROPERTIES('delta.columnMapping.mode' = 'name')") // Have this test to make sure there are no collisions with c.`s.a` and c.s.a. sql( s"ALTER TABLE t SET TBLPROPERTIES('delta.dataSkippingStatsColumns' = 'c')") val deltaLog = DeltaLog.forTable(spark, TableIdentifier("t")) val tblProperty = DeltaConfigs.DATA_SKIPPING_STATS_COLUMNS .fromMetaData(deltaLog.update().metadata) assert(tblProperty.get == "c") } } Seq("name", "id").foreach { mappingModeName => val mappingMode = if (mappingModeName == "name") NameMapping else IdMapping test(s"Throw ColumnMappingException when missing physical name" + s" - mappingModeName: $mappingModeName") { val fieldWithPhysicalName = StructField( name = "testColumn", dataType = StringType, nullable = true, metadata = new MetadataBuilder() .putString(DeltaColumnMapping.COLUMN_MAPPING_PHYSICAL_NAME_KEY, "col_12345_physical") .build()) val fieldWithoutPhysicalName = StructField( name = "testColumn", dataType = StringType, nullable = true, metadata = new MetadataBuilder().build()) // Use empty schemaNames so that schemaNames.contains(fullPath) returns false. // This forces the function to proceed to the physical name check. val emptySchemaNames = Seq.empty[String] val fullPath = "testColumn" val goodResult = StatisticsCollection.convertToPhysicalName( fullPath = fullPath, field = fieldWithPhysicalName, schemaNames = emptySchemaNames, mappingMode = mappingMode) assert(goodResult.name == "col_12345_physical") assert(goodResult.dataType == StringType) // Test with a field that does not have a physical name, expect ColumnMappingException. val exception = intercept[ColumnMappingException] { StatisticsCollection.convertToPhysicalName( fullPath = fullPath, field = fieldWithoutPhysicalName, schemaNames = emptySchemaNames, mappingMode = mappingMode) } // Verify the exception contains the expected message and correct mapping mode assert(exception.msg.contains(s"Missing physical name in column mapping mode " + s"`$mappingModeName`")) assert(exception.mode == mappingMode) } } private def recordsScanned(df: DataFrame): Long = { val scan = df.queryExecution.executedPlan.find { case FileScanExecNode(_) => true case _ => false }.get var executedScan = false if (!executedScan) { if (scan.supportsColumnar) { scan.executeColumnar().count() } else { scan.execute().count() } } scan.metrics.get("numOutputRows").get.value } private def statsDF(deltaLog: DeltaLog): DataFrame = { // use physical name if possible val dataColumns = deltaLog.snapshot.metadata.dataSchema.map(DeltaColumnMapping.getPhysicalName) val minValues = struct(dataColumns.map(c => $"minValues.$c"): _*) val maxValues = struct(dataColumns.map(c => $"maxValues.$c"): _*) val df = getStatsDf(deltaLog, Seq($"numRecords", minValues, maxValues)) val numRecordsCol = df.schema.head.name df.withColumnRenamed(numRecordsCol, "numRecords") } /** * Checks if the min/max values in the collected stats for the given string column are truncated * to the expected length. */ private def checkDataSkippingStringPrefixLength( tableName: String, columnName: String, expectedLength: Int, minValueRowContent: String, maxValueRowContent: String): Unit = { val (deltaLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName)) val physicalColumnName = DeltaColumnMapping.getPhysicalName( SchemaUtils.findNestedFieldIgnoreCase(snapshot.schema, Seq(columnName)).get) val statsDf = statsDF(deltaLog) val minValue = statsDf .select(min(s"`struct(minValues.$physicalColumnName)`.$physicalColumnName")) .collect().head .getString(0) val maxValue = statsDf .select(max(s"`struct(maxValues.$physicalColumnName)`.$physicalColumnName")) .collect().head .getString(0) assert(minValue == minValueRowContent.take(expectedLength)) assert(maxValue == maxValueRowContent.take(expectedLength) + ASCII_MAX_CHARACTER) } statsTest("Data-skipping-string-prefix-length delta table property override: basic") { val tableName = "delta_table" val strCol = "strCol" withTable(tableName) { val (a1000, b1000, c1000) = ("a" * 1000, "b" * 1000, "c" * 1000) // Create a table with table property override. sql( s""" | create table $tableName ($strCol string) using delta tblproperties | ('${DeltaConfigs.DATA_SKIPPING_STRING_PREFIX_LENGTH.key}' = '64') |""".stripMargin) sql(s"insert into $tableName values ('$a1000'), ('$b1000'), ('$c1000')") checkDataSkippingStringPrefixLength( tableName, columnName = strCol, expectedLength = 64, minValueRowContent = a1000, maxValueRowContent = c1000 ) } } statsTest("Data-skipping-string-prefix-length delta table property override: recompute stats") { val tableName = "delta_table" val strCol = "strCol" withTable(tableName) { val (a1000, b1000, c1000) = ("a" * 1000, "b" * 1000, "c" * 1000) // Create a table without table property override. sql(s"create table $tableName ($strCol string) using delta") sql(s"insert into $tableName values ('$a1000'), ('$b1000'), ('$c1000')") checkDataSkippingStringPrefixLength( tableName, columnName = strCol, expectedLength = DeltaSQLConf.DATA_SKIPPING_STRING_PREFIX_LENGTH.defaultValue.get, minValueRowContent = a1000, maxValueRowContent = c1000 ) // Set the table property override and recompute stats. sql( s""" | alter table $tableName set tblproperties | ('${DeltaConfigs.DATA_SKIPPING_STRING_PREFIX_LENGTH.key}' = '64') | """.stripMargin) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) StatisticsCollection.recompute(spark, deltaLog) checkDataSkippingStringPrefixLength( tableName, columnName = strCol, expectedLength = 64, minValueRowContent = a1000, maxValueRowContent = c1000 ) } } statsTest("Data-skipping-string-prefix-length delta table property override: RTAS") { val tableName = "delta_table" val sourceTableName = "source_table" val strCol = "strCol" withTable(tableName, sourceTableName) { val (a1000, b1000, c1000) = ("a" * 1000, "b" * 1000, "c" * 1000) val (x1000, y1000, z1000) = ("x" * 1000, "y" * 1000, "z" * 1000) // Create a source table. sql(s"create table $sourceTableName ($strCol string) using delta") sql(s"insert into $sourceTableName values ('$a1000'), ('$b1000'), ('$c1000')") // Create a table without table property override. sql(s"create table $tableName (strCol string) using delta") sql(s"insert into $tableName values ('$x1000'), ('$y1000'), ('$z1000')") checkDataSkippingStringPrefixLength( tableName, columnName = strCol, expectedLength = DeltaSQLConf.DATA_SKIPPING_STRING_PREFIX_LENGTH.defaultValue.get, minValueRowContent = x1000, maxValueRowContent = z1000 ) // Replace the table with table property override by selecting from the source table. sql( s""" | replace table $tableName using delta tblproperties | ('${DeltaConfigs.DATA_SKIPPING_STRING_PREFIX_LENGTH.key}' = '64') | as (select * from $sourceTableName) | """.stripMargin) checkDataSkippingStringPrefixLength( tableName, columnName = strCol, expectedLength = 64, minValueRowContent = a1000, maxValueRowContent = c1000 ) } } statsTest("Data-skipping-string-prefix-length delta table property override: Add/Remove Files") { /** * In the commit JSON file at the given version, checks if the min/max values for the given * string column stored in the Add/Remove File action are truncated to the expected length. */ def checkStringPrefixLengthInAction( tableName: String, version: Long, action: String, columnName: String, expectedLength: Int, minValueRowContent: String, maxValueRowContent: String): Unit = { val (deltaLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName)) val physicalColumnName = DeltaColumnMapping.getPhysicalName( SchemaUtils.findNestedFieldIgnoreCase(snapshot.schema, Seq(columnName)).get) val commit = spark.read.json(FileNames.unsafeDeltaFile(deltaLog.logPath, version).toString) val actionFile = commit.filter(col(action).isNotNull).select(s"$action.*") val minMaxStatsSchema = StructType(Seq( StructField("minValues", StructType(Seq(StructField(physicalColumnName, StringType)))), StructField("maxValues", StructType(Seq(StructField(physicalColumnName, StringType)))) )) val actionFileWithParsedStats = actionFile.withColumn( "parsed_stats", from_json(col("stats"), minMaxStatsSchema)) val minValue = actionFileWithParsedStats .select(min(col(s"parsed_stats.minValues.$physicalColumnName"))) .collect().head.getString(0) val maxValue = actionFileWithParsedStats .select(max(col(s"parsed_stats.maxValues.$physicalColumnName"))) .collect().head.getString(0) assert(minValue == minValueRowContent.take(expectedLength)) assert(maxValue == maxValueRowContent.take(expectedLength) + ASCII_MAX_CHARACTER) } val tableName = "delta_table" val strCol = "strCol" withTable(tableName) { // Create a table without table property override. sql(s"create table $tableName ($strCol string) using delta") val (a1000, b1000, c1000) = ("a" * 1000, "b" * 1000, "c" * 1000) sql(s"insert into $tableName values ('$a1000'), ('$b1000'), ('$c1000')") // [Add File] Min: "a" * 32, Max: "c" * 32 checkStringPrefixLengthInAction( tableName, version = 1, action = "add", columnName = strCol, expectedLength = DeltaSQLConf.DATA_SKIPPING_STRING_PREFIX_LENGTH.defaultValue.get, minValueRowContent = a1000, maxValueRowContent = c1000 ) // Set the table property override. sql( s""" | alter table $tableName set tblproperties | ('${DeltaConfigs.DATA_SKIPPING_STRING_PREFIX_LENGTH.key}' = '64') | """.stripMargin) val (x1000, y1000, z1000) = ("x" * 1000, "y" * 1000, "z" * 1000) sql(s"insert into $tableName values ('$x1000'), ('$y1000'), ('$z1000')") // [Add File] Min: "x" * 64, Max: "z" * 64 checkStringPrefixLengthInAction( tableName, version = 3, action = "add", columnName = strCol, expectedLength = 64, minValueRowContent = x1000, maxValueRowContent = z1000 ) } } } class StatsCollectionNameColumnMappingSuite extends StatsCollectionSuite with DeltaColumnMappingEnableNameMode { override protected def runOnlyTests = Seq( "on write", "recompute stats with partition predicates" ) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/stats/StatsUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.stats import org.apache.spark.sql.delta.DeltaTable import org.apache.spark.sql.DataFrame trait StatsUtils { protected def getStats(df: DataFrame): DeltaScan = { val stats = df.queryExecution.optimizedPlan.collect { case DeltaTable(prepared: PreparedDeltaFileIndex) => prepared.preparedScan } if (stats.size != 1) sys.error(s"Found ${stats.size} scans!") stats.head } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/storage/LineClosableIteratorSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.storage import java.io.{Reader, StringReader} import org.apache.spark.SparkFunSuite abstract class LineClosableIteratorSuiteBase extends SparkFunSuite { protected def createIter(_reader: Reader): ClosableIterator[String] test("empty") { var iter = createIter(new StringReader("")) assert(!iter.hasNext) intercept[NoSuchElementException] { iter.next() } iter = createIter(new StringReader("")) intercept[NoSuchElementException] { iter.next() } iter = createIter(new StringReader("")) iter.close() intercept[IllegalStateException] { iter.hasNext } intercept[IllegalStateException] { iter.next() } } test("one elem") { var iter = createIter(new StringReader("foo")) assert(iter.hasNext) assert(iter.next() == "foo") assert(!iter.hasNext) intercept[NoSuchElementException] { iter.next() } iter = createIter(new StringReader("foo")) assert(iter.next() == "foo") intercept[NoSuchElementException] { iter.next() } iter = createIter(new StringReader("foo")) iter.close() intercept[IllegalStateException] { iter.hasNext } intercept[IllegalStateException] { iter.next() } } test("two elems") { var iter = createIter(new StringReader("foo\nbar")) assert(iter.hasNext) assert(iter.next() == "foo") assert(iter.hasNext) assert(iter.next() == "bar") assert(!iter.hasNext) intercept[NoSuchElementException] { iter.next() } iter = createIter(new StringReader("foo\nbar")) assert(iter.next() == "foo") assert(iter.next() == "bar") intercept[NoSuchElementException] { iter.next() } iter = createIter(new StringReader("foo\nbar")) assert(iter.next() == "foo") iter.close() intercept[IllegalStateException] { iter.hasNext } intercept[IllegalStateException] { iter.next() } iter = createIter(new StringReader("foo\nbar")) assert(iter.hasNext) // Cache `nextValue` iter.close() // We should throw `IllegalStateException` even if there is a cached `nextValue`. intercept[IllegalStateException] { iter.hasNext } intercept[IllegalStateException] { iter.next() } } test("close should be called when the iterator reaches the end") { var closed = false val reader = new StringReader("foo") { override def close(): Unit = { super.close() closed = true } } val iter = createIter(reader) assert(iter.toList == "foo" :: Nil) assert(closed) } test("close should be called when the iterator is closed") { var closed = false val reader = new StringReader("foo") { override def close(): Unit = { super.close() closed = true } } val iter = createIter(reader) iter.close() assert(closed) } test("close should be called only once") { var closed = 0 val reader = new StringReader("foo") { override def close(): Unit = { super.close() closed += 1 } } val iter = createIter(reader) assert(iter.toList == "foo" :: Nil) iter.close() assert(closed == 1) } test("flatMapWithClose does not open any iterators on creation") { var opened = 0 var closed = 0 val outerReader = new StringReader("b\na\nr") createIter(outerReader).flatMapWithClose(_ => { val innerReader = new StringReader("f\no\no") { opened += 1 override def close(): Unit = { super.close() closed += 1 } } createIter(innerReader) }) assert(opened == 0) assert(closed == 0) } test("flatMapWithClose calls close only for opened iterators") { var opened = 0 var closed = 0 val outerReader = new StringReader("b\na\nr") val iter = createIter(outerReader).flatMapWithClose(_ => { val innerReader = new StringReader("f\no\no") { opened += 1 override def close(): Unit = { super.close() closed += 1 } } createIter(innerReader) }) assert(iter.take(5).toList == List("f", "o", "o", "f", "o")) iter.close() assert(opened == 2) assert(closed == 2) } test("flatMapWithClose calls close only for opened iterators - iter boundary") { var opened = 0 var closed = 0 val outerReader = new StringReader("b\na\nr") val iter = createIter(outerReader).flatMapWithClose(_ => { val innerReader = new StringReader("f\no\no") { opened += 1 override def close(): Unit = { super.close() closed += 1 } } createIter(innerReader) }) assert(iter.take(3).toList == List("f", "o", "o")) iter.close() assert(opened == 1) assert(closed == 1) } } class InternalLineClosableIteratorSuite extends LineClosableIteratorSuiteBase { override protected def createIter(_reader: Reader): ClosableIterator[String] = { new LineClosableIterator(_reader) } } class PublicLineClosableIteratorSuite extends LineClosableIteratorSuiteBase { override protected def createIter(_reader: Reader): ClosableIterator[String] = { val impl = new io.delta.storage.LineCloseableIterator(_reader) new LineClosableIteratorAdaptor(impl) } } private class LineClosableIteratorAdaptor( impl: io.delta.storage.LineCloseableIterator) extends ClosableIterator[String] { override def hasNext(): Boolean = impl.hasNext override def next(): String = impl.next() override def close(): Unit = impl.close() } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/storage/dv/DeletionVectorFileSizeSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.storage.dv import java.io.File import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.actions.{AddFile, DeletionVectorDescriptor, RemoveFile} import org.apache.spark.sql.delta.deletionvectors.DeletionVectorsSuite import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.apache.commons.io.FileUtils import org.apache.hadoop.fs.Path import org.apache.spark.sql.QueryTest import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession class DeletionVectorFileSizeSuite extends QueryTest with SharedSparkSession with DeltaSQLTestUtils with DeltaSQLCommandTest { private def getAddAndRemoveFilesFromCommitVersion( deltaLog: DeltaLog, commitVersion: Long): (Seq[AddFile], Seq[RemoveFile]) = { require(commitVersion <= deltaLog.update().version, "Commit version should be less than or equal to the current version") val changes = deltaLog.getChanges( commitVersion, commitVersion, catalogTableOpt = None, failOnDataLoss = true) val (changesItrForAddFiles, changesItrForRemoveFiles) = changes.duplicate val addFiles: Seq[AddFile] = changesItrForAddFiles.flatMap(_._2.collect { case a: AddFile => a }).toSeq val removeFiles: Seq[RemoveFile] = changesItrForRemoveFiles.flatMap(_._2.collect { case r: RemoveFile => r }).toSeq (addFiles, removeFiles) } private def getDeletionVectorFilePath( dvDescriptor: DeletionVectorDescriptor, tableDataPath: String ): Path = { val path = dvDescriptor.absolutePath(new Path(tableDataPath)) path } private def getFileSizeInBytes( absoluteFilePath: Path, hadoopConf: org.apache.hadoop.conf.Configuration): Long = { val file = new File(absoluteFilePath.toString) assert(file.exists()) val fs = absoluteFilePath.getFileSystem(hadoopConf) val fileStatus = fs.getFileStatus(absoluteFilePath) fileStatus.getLen } test("Bin Packing should take the size of existing DVs into account") { withTempDir { tempDir => val source = new File(DeletionVectorsSuite.table1Path) val target = new File(tempDir, "deleteTest") // Copy the source table with existing table layout to a temporary directory FileUtils.copyDirectory(source, target) val (deltaLog, snapshot) = DeltaLog.forTableWithSnapshot(spark, new Path(target.getAbsolutePath)) assert(snapshot.version === 4, "Table should exist") // All existing individual DVs either have 34 or 36 bytes, corresponding 1 or 2 deleted rows. val priorAddFiles = snapshot.allFiles.collect() priorAddFiles.forall(a => a.deletionVector == null || (a.deletionVector.sizeInBytes == 34 || a.deletionVector.sizeInBytes == 36)) assert(priorAddFiles.count(_.deletionVector != null) === 8, "8 AddFiles with DVs expected") val targetPackingFileSizeInBytes = 110 withSQLConf( DeltaSQLConf.DELETION_VECTOR_PACKING_TARGET_SIZE.key -> targetPackingFileSizeInBytes.toString) { // Delete some rows to trigger the creation of new DVs. sql(s"DELETE FROM delta.`${target.getAbsolutePath}` WHERE value IN (255, 303, 707, 1905)") val (addFiles, removeFiles) = getAddAndRemoveFilesFromCommitVersion(deltaLog, deltaLog.update().version) assert(addFiles.size === 3, "Deletion vector added to 3 different AddFiles") assert(addFiles.forall(_.deletionVector != null), "Deletion should have used DVs and not rewrite any files") val removeFilesPaths = removeFiles.map(_.path).toSet assert(priorAddFiles.filter(a => removeFilesPaths.contains(a.path)) .count(_.deletionVector != null) === 2, "2 of the AddFiles had existing DVs") // Get the file sizes of the DV files added by this commit. val dvFileSizes = addFiles.map(_.deletionVector) .map(dv => getDeletionVectorFilePath(dv, target.getAbsolutePath)) .toSet .map(path => getFileSizeInBytes(path, deltaLog.newDeltaHadoopConf())) assert(addFiles.forall(a => a.deletionVector.sizeInBytes <= targetPackingFileSizeInBytes), "the individual DVs can each fit in their own file if needed") assert(dvFileSizes.forall(_ <= targetPackingFileSizeInBytes), "The target file size should be respected when individual DVs fit under the file size") } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/storage/dv/DeletionVectorStoreSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.storage.dv import java.io.{DataInputStream, DataOutputStream, File} import org.apache.spark.sql.delta.{DeltaChecksumException, DeltaConfigs, DeltaLog} import org.apache.spark.sql.delta.deletionvectors.{RoaringBitmapArray, RoaringBitmapArrayFormat} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore.{getTotalSizeOfDVFieldsInFile, CHECKSUM_LEN} import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.util.PathWithFileSystem import com.google.common.primitives.Ints import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.sql.QueryTest import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.Utils trait DeletionVectorStoreSuiteBase extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { lazy val dvStore: DeletionVectorStore = DeletionVectorStore.createInstance(newHadoopConf) protected def newHadoopConf: Configuration = { // scalastyle:off deltahadoopconfiguration spark.sessionState.newHadoopConf() // scalastyle:on deltahadoopconfiguration } // Test bitmaps protected lazy val simpleBitmap = { val data = Seq(1L, 5L, 6L, 7L, 1000L, 8000000L, 8000001L) RoaringBitmapArray(data: _*) } protected lazy val simpleBitmap2 = { val data = Seq(78L, 256L, 998L, 1000002L, 22623423L) RoaringBitmapArray(data: _*) } def withTempHadoopFileSystemPath[T](f: Path => T): T = { val dir: File = Utils.createTempDir() dir.delete() val tempPath = DeletionVectorStore.unescapedStringToPath(dir.toString) try f(tempPath) finally Utils.deleteRecursively(dir) } testWithAllSerializationFormats("Write simple DV directly to disk") { serializationFormat => val readDV = withTempHadoopFileSystemPath { tableDir => val tableWithFS = PathWithFileSystem.withConf(tableDir, newHadoopConf) val dvPath = dvStore.generateUniqueNameInTable(tableWithFS) val serializedBitmap = simpleBitmap.serializeAsByteArray(serializationFormat) val dvRange = Utils.tryWithResource(dvStore.createWriter(dvPath)) { writer => writer.write(serializedBitmap) } assert(dvRange.offset === 1) // there's a version id at byte 0 assert(dvRange.length === serializedBitmap.length) dvStore.read(dvPath.path, dvRange.offset, dvRange.length) } assert(simpleBitmap === readDV) } testWithAllSerializationFormats("Detect corrupted DV checksum ") { serializationFormat => withTempHadoopFileSystemPath { tableDir => val tableWithFS = PathWithFileSystem.withConf(tableDir, newHadoopConf) val dvPath = dvStore.generateUniqueNameInTable(tableWithFS) val dvBytes = simpleBitmap.serializeAsByteArray(serializationFormat) val dvRange = Utils.tryWithResource(dvStore.createWriter(dvPath)) { writer => writer.write(dvBytes) } assert(dvRange.offset === 1) // there's a version id at byte 0 assert(dvRange.length === dvBytes.length) // corrupt 1 byte in the middle of the stored DV (after the checksum) corruptByte(dvPath, byteToCorrupt = DeletionVectorStore.CHECKSUM_LEN + dvRange.length / 2) val e = intercept[DeltaChecksumException] { dvStore.read(dvPath.path, dvRange.offset, dvRange.length) } // make sure this is our exception not ChecksumFileSystem's assert(e.getErrorClass == "DELTA_DELETION_VECTOR_CHECKSUM_MISMATCH") assert(e.getSqlState == "XXKDS") assert(e.getMessage == "[DELTA_DELETION_VECTOR_CHECKSUM_MISMATCH] " + "Could not verify deletion vector integrity, CRC checksum verification failed.") } } testWithAllSerializationFormats("Detect corrupted DV size") { serializationFormat => withTempHadoopFileSystemPath { tableDir => val tableWithFS = PathWithFileSystem.withConf(tableDir, newHadoopConf) val dvPath = dvStore.generateUniqueNameInTable(tableWithFS) val dvBytes = simpleBitmap.serializeAsByteArray(serializationFormat) val dvRange = Utils.tryWithResource(dvStore.createWriter(dvPath)) { writer => writer.write(dvBytes) } assert(dvRange.offset === 1) // there's a version id at byte 0 assert(dvRange.length === dvBytes.length) // Corrupt 1 byte in the part where the serialized DV size is stored. // Format: corruptByte(dvPath, byteToCorrupt = 2) val e = intercept[DeltaChecksumException] { dvStore.read(dvPath.path, dvRange.offset, dvRange.length) } assert(e.getErrorClass == "DELTA_DELETION_VECTOR_SIZE_MISMATCH") assert(e.getSqlState == "XXKDS") assert(e.getMessage == "[DELTA_DELETION_VECTOR_SIZE_MISMATCH] " + "Deletion vector integrity check failed. Encountered a size mismatch.") } } testWithAllSerializationFormats("Multiple DVs in one file") { serializationFormat => withTempHadoopFileSystemPath { tableDir => val tableWithFS = PathWithFileSystem.withConf(tableDir, newHadoopConf) val dvPath = dvStore.generateUniqueNameInTable(tableWithFS) val dvBytes1 = simpleBitmap.serializeAsByteArray(serializationFormat) val dvBytes2 = simpleBitmap2.serializeAsByteArray(serializationFormat) val (dvRange1, dvRange2) = Utils.tryWithResource(dvStore.createWriter(dvPath)) { writer => (writer.write(dvBytes1), writer.write(dvBytes2)) } assert(dvRange1.offset === 1) // there's a version id at byte 0 assert(dvRange1.length === dvBytes1.length) // DV2 should be written immediately after the DV1 val totalDV1Size = getTotalSizeOfDVFieldsInFile(dvBytes1.length) assert(dvRange2.offset === 1 + totalDV1Size) // 1byte for file format version assert(dvRange2.length === dvBytes2.length) // Read back DVs from the file and verify assert(dvStore.read(dvPath.path, dvRange1.offset, dvRange1.length) === simpleBitmap) assert(dvStore.read(dvPath.path, dvRange2.offset, dvRange2.length) === simpleBitmap2) } } test("Exception is thrown for DVDescriptors with invalid maxRowIndex") { withSQLConf( DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.defaultTablePropertyKey -> "true", DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS.key -> true.toString) { withTempDir { dir => val path = dir.toString spark.range(0, 50, 1, 1).write.format("delta").save(path) val targetTable = io.delta.tables.DeltaTable.forPath(path) val deltaLog = DeltaLog.forTable(spark, path) val tableName = s"delta.`$path`" spark.sql(s"DELETE FROM $tableName WHERE id = 3") val file = deltaLog.update().allFiles.first() val dvDescriptorWithInvalidRowIndex = file.deletionVector.copy(maxRowIndex = Some(50)) val e = intercept[DeltaChecksumException] { file.removeRows( dvDescriptorWithInvalidRowIndex, updateStats = false ) } assert(e.getErrorClass == "DELTA_DELETION_VECTOR_INVALID_ROW_INDEX") assert(e.getSqlState == "XXKDS") assert(e.getMessage == "[DELTA_DELETION_VECTOR_INVALID_ROW_INDEX] " + "Deletion vector integrity check failed. Encountered an invalid row index.") } } } /** Helper method to run the test using all DV serialization formats */ protected def testWithAllSerializationFormats(name: String) (func: RoaringBitmapArrayFormat.Value => Unit): Unit = { for (serializationFormat <- RoaringBitmapArrayFormat.values) { test(s"$name - $serializationFormat") { func(serializationFormat) } } } /** Helper to method to simulate data corruption in on-disk DV */ private def corruptByte(pathWithFS: PathWithFileSystem, byteToCorrupt: Int): Unit = { val fs = pathWithFS.fs val path = pathWithFS.path val status = fs.getFileStatus(path) val len = Ints.checkedCast(status.getLen) val bytes = Utils.tryWithResource(fs.open(path)) { stream => val reader = new DataInputStream(stream) // readAllBytes is not available in 1.8, yet val buffer = new Array[Byte](len) reader.readFully(buffer) buffer } bytes(byteToCorrupt) = (bytes(byteToCorrupt) + 1).toByte val overwrite = true Utils.tryWithResource(fs.create(path, overwrite)) { stream => val writer = new DataOutputStream(stream) writer.write(bytes) writer.flush() } } } class DeletionVectorStoreSuite extends DeletionVectorStoreSuiteBase ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/test/CustomCatalogs.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test import java.util import scala.collection.{immutable, mutable} import scala.collection.JavaConverters._ import org.apache.spark.sql.delta.catalog.{DeltaCatalog, DeltaTableV2} import org.apache.commons.io.FileUtils import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.{NamespaceAlreadyExistsException, NoSuchTableException} import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} import org.apache.spark.sql.connector.catalog.{DelegatingCatalogExtension, Identifier, NamespaceChange, SupportsNamespaces, Table, TableCatalog, TableChange, V1Table} import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.execution.datasources.v2.V2SessionCatalog import org.apache.spark.sql.types.StructType import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.util.Utils /** * A Utils class for custom catalog implementations that could be used for testing. */ class DummyCatalog extends TableCatalog { private val spark: SparkSession = SparkSession.active protected lazy val tempDir: Path = new Path(Utils.createTempDir().getAbsolutePath) // scalastyle:off deltahadoopconfiguration protected lazy val fs: FileSystem = tempDir.getFileSystem(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration override def name: String = "dummy" def getTablePath(tableName: String): Path = { new Path(tempDir.toString + "/" + tableName) } override def defaultNamespace(): Array[String] = Array("default") override def listTables(namespace: Array[String]): Array[Identifier] = { val status = fs.listStatus(tempDir) status.filter(_.isDirectory).map { dir => Identifier.of(namespace, dir.getPath.getName) } } override def tableExists(ident: Identifier): Boolean = { val tablePath = getTablePath(ident.name()) fs.exists(tablePath) } override def loadTable(ident: Identifier): Table = { if (!tableExists(ident)) { throw new NoSuchTableException(ident) } val tablePath = getTablePath(ident.name()) DeltaTableV2(spark = spark, path = tablePath, catalogTable = Some(createCatalogTable(ident))) } override def createTable( ident: Identifier, schema: StructType, partitions: Array[Transform], properties: java.util.Map[String, String]): Table = { val tablePath = getTablePath(ident.name()) // Create an empty Delta table on the tablePath val part = partitions.map(_.arguments().head.toString) spark.createDataFrame(List.empty[Row].asJava, schema) .write.format("delta").partitionBy(part: _*).save(tablePath.toString) DeltaTableV2(spark = spark, path = tablePath, catalogTable = Some(createCatalogTable(ident))) } override def alterTable(ident: Identifier, changes: TableChange*): Table = { // hack hack: no-op just for testing loadTable(ident) } override def dropTable(ident: Identifier): Boolean = { val tablePath = getTablePath(ident.name()) try { fs.delete(tablePath, true) true } catch { case _: Exception => false } } override def renameTable(oldIdent: Identifier, newIdent: Identifier): Unit = { throw new UnsupportedOperationException("Rename table operation is not supported.") } override def initialize(name: String, options: CaseInsensitiveStringMap): Unit = { // Initialize tempDir here if (!fs.exists(tempDir)) { fs.mkdirs(tempDir) } } private def createCatalogTable(ident: Identifier): CatalogTable = { val tablePath = getTablePath(ident.name()) CatalogTable( identifier = TableIdentifier(ident.name(), defaultNamespace.headOption, Some(name)), tableType = CatalogTableType.MANAGED, storage = CatalogStorageFormat(Some(tablePath.toUri), None, None, None, false, Map.empty), schema = spark.range(0).schema ) } } // A dummy catalog that adds additional table storage properties after the table is loaded. // It's only used inside `DummySessionCatalog`. class DummySessionCatalogInner extends DelegatingCatalogExtension { override def loadTable(ident: Identifier): Table = { val t = super.loadTable(ident).asInstanceOf[V1Table] V1Table(t.v1Table.copy( storage = t.v1Table.storage.copy( properties = t.v1Table.storage.properties ++ Map("fs.myKey" -> "val") ) )) } } // A dummy catalog that adds a layer between DeltaCatalog and the Spark SessionCatalog, // to attach additional table storage properties after the table is loaded, and generates location // for managed tables. class DummySessionCatalog extends TableCatalog { private var deltaCatalog: DeltaCatalog = null override def initialize(name: String, options: CaseInsensitiveStringMap): Unit = { val inner = new DummySessionCatalogInner() inner.setDelegateCatalog(new V2SessionCatalog( SparkSession.active.sessionState.catalogManager.v1SessionCatalog)) deltaCatalog = new DeltaCatalog() deltaCatalog.setDelegateCatalog(inner) } override def name(): String = deltaCatalog.name() override def listTables(namespace: Array[String]): Array[Identifier] = { deltaCatalog.listTables(namespace) } override def loadTable(ident: Identifier): Table = deltaCatalog.loadTable(ident) override def createTable( ident: Identifier, schema: StructType, partitions: Array[Transform], properties: java.util.Map[String, String]): Table = { if (!properties.containsKey(TableCatalog.PROP_EXTERNAL) && !properties.containsKey(TableCatalog.PROP_LOCATION)) { val newProps = new java.util.HashMap[String, String] newProps.putAll(properties) newProps.put(TableCatalog.PROP_LOCATION, properties.get("fakeLoc")) newProps.put(TableCatalog.PROP_IS_MANAGED_LOCATION, "true") deltaCatalog.createTable(ident, schema, partitions, newProps) } else { deltaCatalog.createTable(ident, schema, partitions, properties) } } override def alterTable(ident: Identifier, changes: TableChange*): Table = { deltaCatalog.alterTable(ident, changes: _*) } override def dropTable(ident: Identifier): Boolean = deltaCatalog.dropTable(ident) override def renameTable(oldIdent: Identifier, newIdent: Identifier): Unit = { deltaCatalog.renameTable(oldIdent, newIdent) } } // This catalog always does a CASCADE on DROP SCHEMA ... class DummyCatalogWithNamespace extends DummyCatalog with SupportsNamespaces { private val spark: SparkSession = SparkSession.active // To load a catalog into spark CatalogPlugin calls the Catalog's no-arg constructor and // then Catalog.initialize. To have a consistent state across different invocations // in the same test, this catalog impl uses a hard coded path. override lazy val tempDir: Path = DummyCatalogWithNamespace.catalogDir // scalastyle:off deltahadoopconfiguration override lazy val fs: FileSystem = tempDir.getFileSystem(spark.sessionState.newHadoopConf()) // scalastyle:on deltahadoopconfiguration // Map each namespace to its metadata protected val namespaces: util.Map[List[String], Map[String, String]] = new util.HashMap[List[String], Map[String, String]]() protected val tables: mutable.Map[Array[String], mutable.HashSet[Identifier]] = new mutable.HashMap[Array[String], mutable.HashSet[Identifier]]() override def name: String = "test_catalog" override def getTablePath(tableName: String): Path = { new Path(s"${tempDir.toString}/$name.$tableName") } override def tableExists(ident: Identifier): Boolean = { val tablePath = getTablePath(ident.toString) fs.exists(tablePath) } override def loadTable(ident: Identifier): Table = { if (!tableExists(ident)) { throw new NoSuchTableException(ident) } val tablePath = getTablePath(ident.toString) DeltaTableV2(spark = spark, path = tablePath, catalogTable = Some(createCatalogTable(ident))) } override def createTable( ident: Identifier, schema: StructType, partitions: Array[Transform], properties: java.util.Map[String, String]): Table = { val tablePath = getTablePath(ident.toString) // Create an empty Delta table on the tablePath val part = partitions.map(_.arguments().head.toString) spark.createDataFrame(List.empty[Row].asJava, schema) .write.format("delta").partitionBy(part: _*).save(tablePath.toString) val map = tables.getOrElseUpdate(ident.namespace(), new mutable.HashSet[Identifier]()) map.add(ident) tables.put(ident.namespace(), map) DeltaTableV2(spark = spark, path = tablePath, catalogTable = Some(createCatalogTable(ident))) } override def dropTable(ident: Identifier): Boolean = { val tablePath = getTablePath(ident.toString) try { fs.delete(tablePath, true) true } catch { case _: Exception => false } } override def initialize(name: String, options: CaseInsensitiveStringMap): Unit = { // Initialize tempDir here if (!fs.exists(tempDir)) { fs.mkdirs(tempDir) } fs.deleteOnExit(tempDir) } private def createCatalogTable(ident: Identifier): CatalogTable = { val tablePath = getTablePath(ident.toString) CatalogTable( identifier = TableIdentifier(ident.toString, defaultNamespace.headOption, Some(name)), tableType = CatalogTableType.MANAGED, storage = CatalogStorageFormat( Some(tablePath.toUri), None, None, None, false, immutable.Map.empty), schema = spark.range(0).schema ) } override def createNamespace( namespace: Array[String], metadata: util.Map[String, String]): Unit = { Option(namespaces.putIfAbsent(namespace.toList, metadata.asScala.toMap)) match { case Some(_) => throw new NamespaceAlreadyExistsException(namespace) case _ => // success } } override def alterNamespace(namespace: Array[String], changes: NamespaceChange*): Unit = { throw new UnsupportedOperationException("alter namespace metadata is not supported.") } override def dropNamespace(namespace: Array[String], cascade: Boolean): Boolean = { tables.getOrElse(namespace, mutable.HashSet.empty[Identifier]).foreach(dropTable) Option(namespaces.remove(namespace.toList)).isDefined } override def namespaceExists(namespace: Array[String]): Boolean = { namespaces.containsKey(namespace.toList) } override def listNamespaces(): Array[Array[String]] = { throw new UnsupportedOperationException("List namespaces operation is not supported.") } override def listNamespaces(namespace: Array[String]): Array[Array[String]] = { throw new UnsupportedOperationException("List namespaces operation is not supported.") } override def loadNamespaceMetadata(namespace: Array[String]): util.Map[String, String] = { new util.HashMap[String, String]() } } object DummyCatalogWithNamespace { val catalogDir: Path = new Path(Utils.createTempDir().getAbsolutePath) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/test/DeltaColumnMappingSelectedTestMixin.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test import scala.collection.mutable import org.apache.spark.sql.delta.{DeltaColumnMappingTestUtils, DeltaConfigs, NoMapping} import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.scalactic.source.Position import org.scalatest.Tag import org.scalatest.exceptions.TestFailedException import org.apache.spark.SparkFunSuite /** * A trait for selective enabling certain tests to run for column mapping modes */ trait DeltaColumnMappingSelectedTestMixin extends SparkFunSuite with DeltaSQLTestUtils with DeltaColumnMappingTestUtils { protected def skipTests: Seq[String] = Seq() protected def runOnlyTests: Seq[String] = Seq() /** * If true, will run all tests. * Requires that `runOnlyTests` is empty. */ protected def runAllTests: Boolean = false private val testsRun: mutable.Set[String] = mutable.Set.empty override protected def test( testName: String, testTags: Tag*)(testFun: => Any)(implicit pos: Position): Unit = { require(!runAllTests || runOnlyTests.isEmpty, "If `runAllTests` is true then `runOnlyTests` must be empty") if ((runAllTests || runOnlyTests.contains(testName)) && !skipTests.contains(testName)) { super.test(s"$testName - column mapping $columnMappingMode mode", testTags: _*) { testsRun.add(testName) withSQLConf( DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey -> columnMappingMode) { testFun } } } else { super.ignore(s"$testName - ignored by DeltaColumnMappingSelectedTestMixin")(testFun) } } override def afterAll(): Unit = { super.afterAll() val missingTests = runOnlyTests.toSet diff testsRun if (missingTests.nonEmpty) { throw new TestFailedException( Some("Not all selected column mapping tests were run. Missing: " + missingTests.mkString(", ")), None, 0) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/test/DeltaExceptionTestUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test import java.util.concurrent.ExecutionException import scala.annotation.tailrec import scala.reflect.ClassTag import org.scalactic.source.Position import org.scalatest.Assertions.intercept import org.apache.spark.SparkException trait DeltaExceptionTestUtils { /** * Handles a breaking change between Spark 3.5 and Spark Master (4.0) to improve error messaging * in Spark. Previously, in Spark 3.5, when an executor would throw an exception, the driver would * wrap it in a [[SparkException]]. Now, in Spark Master (4.0), the original executor exception is * thrown directly. * * This method, which is Spark-version agnostic, executes [[f]] and unwraps it as needed to return * the desired [[Throwable]] of type [[T]]. */ def interceptWithUnwrapping[T <: Throwable : ClassTag]( f: => Any)(implicit pos: Position): T = { @tailrec def unwrapIfNeeded(t: Throwable): T = { t match { case x: T => x case _: SparkException | _: ExecutionException if t.getCause != null => unwrapIfNeeded(t.getCause) case _ if t.getCause != null && t.getCause.isInstanceOf[T] => t.getCause.asInstanceOf[T] case _ => throw t // allow unrecognized exceptions to directly fail the test } } val ex = intercept[Throwable](f) unwrapIfNeeded(ex) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/test/DeltaExcludedTestMixin.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test import org.apache.spark.sql.QueryTest import org.scalactic.source.Position import org.scalatest.Tag trait DeltaExcludedTestMixin extends QueryTest { /** Tests to be ignored by the runner. */ override def excluded: Seq[String] = Seq.empty protected override def test(testName: String, testTags: Tag*) (testFun: => Any) (implicit pos: Position): Unit = { if (excluded.contains(testName)) { super.ignore(testName, testTags: _*)(testFun) } else { super.test(testName, testTags: _*)(testFun) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/test/DeltaHiveTest.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test import org.apache.spark.sql.delta.catalog.DeltaCatalog import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import io.delta.sql.DeltaSparkSessionExtension import org.scalatest.BeforeAndAfterAll import org.apache.spark.{SparkContext, SparkFunSuite} import org.apache.spark.sql.classic.SparkSession import org.apache.spark.sql.hive.test.{TestHive, TestHiveContext} import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf} /** * Test utility for initializing a SparkSession with a Hive Client and a Hive Catalog for testing * DDL operations. Typical tests leverage an in-memory catalog with a mock catalog client. Here we * use real Hive classes. */ trait DeltaHiveTest extends SparkFunSuite with BeforeAndAfterAll { self: DeltaSQLTestUtils => private var _session: SparkSession = _ private var _hiveContext: TestHiveContext = _ private var _sc: SparkContext = _ override def beforeAll(): Unit = { val conf = TestHive.sparkSession.sparkContext.getConf.clone() TestHive.sparkSession.stop() conf.set(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[DeltaCatalog].getName) conf.set(StaticSQLConf.SPARK_SESSION_EXTENSIONS.key, classOf[DeltaSparkSessionExtension].getName) _sc = new SparkContext("local", this.getClass.getName, conf) _hiveContext = new TestHiveContext(_sc) _session = _hiveContext.sparkSession SparkSession.setActiveSession(_session) super.beforeAll() } override protected def spark: SparkSession = _session override def afterAll(): Unit = { try { _hiveContext.reset() } finally { _sc.stop() } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/test/DeltaSQLCommandTest.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test import org.apache.spark.sql.delta.catalog.DeltaCatalog import io.delta.sql.DeltaSparkSessionExtension import org.apache.spark.SparkConf import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf} import org.apache.spark.sql.test.SharedSparkSession /** * A trait for tests that are testing a fully set up SparkSession with all of Delta's requirements, * such as the configuration of the DeltaCatalog and the addition of all Delta extensions. */ trait DeltaSQLCommandTest extends SharedSparkSession { override protected def sparkConf: SparkConf = { super.sparkConf .set(StaticSQLConf.SPARK_SESSION_EXTENSIONS.key, classOf[DeltaSparkSessionExtension].getName) .set(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[DeltaCatalog].getName) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/test/DeltaSQLTestUtils.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test import java.io.File import java.util.UUID import scala.util.Random import org.apache.spark.sql.delta.{CatalogOwnedTableFeature, DeltaColumnMappingTestUtilsBase, DeltaLog, DeltaTable, Snapshot, TableFeature} import org.apache.spark.sql.delta.actions.Protocol import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedCommitCoordinatorProvider, CatalogOwnedTableUtils, TrackingInMemoryCommitCoordinatorBuilder} import org.apache.spark.sql.delta.stats.{DeltaStatistics, PreparedDeltaFileIndex} import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} import org.apache.spark.sql.{AnalysisException, DataFrame} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.test.SQLTestUtils import org.apache.spark.sql.types._ import org.apache.spark.util.Utils trait DeltaSQLTestUtils extends SQLTestUtils { /** * Override the temp dir/path creation methods from [[SQLTestUtils]] to: * 1. Drop the call to `waitForTasksToFinish` which is a source of flakiness due to timeouts * without clear benefits. * 2. Allow creating paths with special characters for better test coverage. */ protected val defaultTempDirPrefix: String = "spark%dir%prefix" override protected def withTempDir(f: File => Unit): Unit = { withTempDir(prefix = defaultTempDirPrefix)(f) } override protected def withTempPaths(numPaths: Int)(f: Seq[File] => Unit): Unit = { withTempPaths(numPaths, prefix = defaultTempDirPrefix)(f) } override def withTempPath(f: File => Unit): Unit = { withTempPath(prefix = defaultTempDirPrefix)(f) } /** * Creates a temporary directory, which is then passed to `f` and will be deleted after `f` * returns. */ def withTempDir(prefix: String)(f: File => Unit): Unit = { val path = Utils.createTempDir(namePrefix = prefix) try f(path) finally Utils.deleteRecursively(path) } /** * Generates a temporary directory path without creating the actual directory, which is then * passed to `f` and will be deleted after `f` returns. */ def withTempPath(prefix: String)(f: File => Unit): Unit = { val path = Utils.createTempDir(namePrefix = prefix) path.delete() try f(path) finally Utils.deleteRecursively(path) } /** * Generates the specified number of temporary directory paths without creating the actual * directories, which are then passed to `f` and will be deleted after `f` returns. */ protected def withTempPaths(numPaths: Int, prefix: String)(f: Seq[File] => Unit): Unit = { val files = Seq.fill[File](numPaths)(Utils.createTempDir(namePrefix = prefix).getCanonicalFile) files.foreach(_.delete()) try f(files) finally { files.foreach(Utils.deleteRecursively) } } /** * Creates a temporary table with a unique name for testing and executes a function with it. * The table is automatically cleaned up after the function completes. * * @param createTable Whether to create an empty table. * @param f The function to execute with the generated table name. */ protected def withTempTable(createTable: Boolean)(f: String => Unit): Unit = { val tableName = s"test_table_${UUID.randomUUID().toString.filterNot(_ == '-')}" withTable(tableName) { if (createTable) { spark.sql(s"CREATE TABLE $tableName (id LONG) USING delta") } f(tableName) } } /** * Creates a Catalog-Managed Delta table for tests. * * @param createTable Whether to create the table with CatalogOwnedTableFeature enabled. * @param f The function to execute with the generated table name. */ protected def withCatalogManagedTable(createTable: Boolean = true)(f: String => Unit): Unit = { CatalogOwnedCommitCoordinatorProvider.clearBuilders() CatalogOwnedCommitCoordinatorProvider.registerBuilder( CatalogOwnedTableUtils.DEFAULT_CATALOG_NAME_FOR_TESTING, TrackingInMemoryCommitCoordinatorBuilder(batchSize = 3)) withTempTable(createTable = false) { tableName => if (createTable) { spark.sql(s"CREATE TABLE $tableName (id INT) USING delta TBLPROPERTIES " + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported')") } f(tableName) } } /** Returns random alphanumberic string to be used as a unique table name. */ def uniqueTableName: String = Random.alphanumeric.take(10).mkString /** Gets the latest snapshot of the table. */ def getSnapshot(tableName: String): Snapshot = { DeltaLog.forTable(spark, TableIdentifier(tableName)).update() } /** Gets the table protocol of the latest snapshot. */ def getProtocolForTable(tableName: String): Protocol = { getSnapshot(tableName).protocol } /** Gets the `StructField` of `columnPath`. */ final def getColumnField(schema: StructType, columnPath: Seq[String]): StructField = { schema.findNestedField(columnPath, includeCollections = true).get._2 } /** Gets the `StructField` of `columnName`. */ def getColumnField(tableName: String, columnName: String): StructField = { val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) getColumnField(deltaLog.update().schema, columnName.split("\\.")) } /** Gets the `DataType` of `columnPath`. */ def getColumnType(schema: StructType, columnPath: Seq[String]): DataType = { getColumnField(schema, columnPath).dataType } /** Gets the `DataType` of `columnName`. */ def getColumnType(tableName: String, columnName: String): DataType = { getColumnField(tableName, columnName).dataType } /** * Gets the stats fields from the AddFiles of `snapshot`. The stats are ordered by the * modification time of the files they are associated with. */ def getUnvalidatedStatsOrderByFileModTime(snapshot: Snapshot): Array[JsonNode] = { snapshot.allFiles .orderBy("modificationTime") .collect() .map(file => new ObjectMapper().readTree(file.stats)) } /** * Gets the stats fields from the AddFiles of `tableName`. The stats are ordered by the * modification time of the files they are associated with. */ def getUnvalidatedStatsOrderByFileModTime(tableName: String): Array[JsonNode] = getUnvalidatedStatsOrderByFileModTime(getSnapshot(tableName)) /** Gets the physical column path if there is column mapping metadata in the schema. */ def getPhysicalColumnPath(tableSchema: StructType, columnName: String): Seq[String] = { new DeltaColumnMappingTestUtilsBase {}.getPhysicalPathForStats( columnName.split("\\."), tableSchema ).get } /** Gets the value of a specified field from `stats` JSON node if it exists. */ def getStatFieldOpt(stats: JsonNode, path: Seq[String]): Option[JsonNode] = path.foldLeft(Option(stats)) { case (Some(node), key) if node.has(key) => Option(node.get(key)) case _ => None } /** Gets the min/max stats of `columnName` from `stats` if they exist. */ private def getMinMaxStatsOpt( tableName: String, stats: JsonNode, columnName: String): (Option[String], Option[String]) = { val schema = getSnapshot(tableName).schema val physicalColumnPath = getPhysicalColumnPath(schema, columnName) val minStatsPath = DeltaStatistics.MIN +: physicalColumnPath val maxStatsPath = DeltaStatistics.MAX +: physicalColumnPath ( getStatFieldOpt(stats, minStatsPath).map(_.asText()), getStatFieldOpt(stats, maxStatsPath).map(_.asText())) } /** Gets the min/max stats of `columnName` from `stats`. */ def getMinMaxStats( tableName: String, stats: JsonNode, columnName: String): (String, String) = { val (minOpt, maxOpt) = getMinMaxStatsOpt(tableName, stats, columnName) (minOpt.get, maxOpt.get) } /** Verifies whether there are min/max stats of `columnName` in `stats`. */ def assertMinMaxStatsPresence( tableName: String, stats: JsonNode, columnName: String, expectStats: Boolean): Unit = { val (minStats, maxStats) = getMinMaxStatsOpt(tableName, stats, columnName) assert(minStats.isDefined === expectStats) assert(maxStats.isDefined === expectStats) } /** Verifies min/max stats values of `columnName` in `stats`. */ def assertMinMaxStats( tableName: String, stats: JsonNode, columnName: String, expectedMin: String, expectedMax: String): Unit = { val (min, max) = getMinMaxStats(tableName, stats, columnName) assert(min === expectedMin, s"Expected $expectedMin, got $min") assert(max === expectedMax, s"Expected $expectedMax, got $max") } /** Verifies minReaderVersion and minWriterVersion of the protocol. */ def assertProtocolVersion( protocol: Protocol, minReaderVersion: Int, minWriterVersion: Int): Unit = { assert(protocol.minReaderVersion === minReaderVersion) assert(protocol.minWriterVersion === minWriterVersion) } /** Verifies column is of expected data type. */ def assertColumnDataType( tableName: String, columnName: String, expectedDataType: DataType): Unit = { assert(getColumnType(tableName, columnName) === expectedDataType) } /** Verifies `columnName` does not exist in `tableName`. */ def assertColumnNotExist(tableName: String, columnName: String): Unit = { val e = intercept[AnalysisException] { sql(s"SELECT $columnName FROM $tableName") } assert(e.getMessage.contains(s"`$columnName` cannot be resolved")) } /** * Runs `select` query on `tableName` with `predicate` and verifies the number of rows returned * and files read. */ def assertSelectQueryResults( tableName: String, predicate: String, numRows: Int, numFilesRead: Int): Unit = { val query = sql(s"SELECT * FROM $tableName WHERE $predicate") assertSelectQueryResults(query, numRows, numFilesRead) } /** * Runs `query` and verifies the number of rows returned * and files read. */ def assertSelectQueryResults( query: DataFrame, numRows: Int, numFilesRead: Int): Unit = { assert(query.count() === numRows, s"Expected $numRows rows, got ${query.count()}") val filesRead = getNumReadFiles(query) assert(filesRead === numFilesRead, s"Expected $numFilesRead files read, got $filesRead") } /** Returns the number of read files by the query with given query text. */ def getNumReadFiles(queryText: String): Int = { getNumReadFiles(sql(queryText)) } /** Returns the number of read files by the given data frame query. */ def getNumReadFiles(df: DataFrame): Int = { val deltaScans = df.queryExecution.optimizedPlan.collect { case DeltaTable(prepared: PreparedDeltaFileIndex) => prepared.preparedScan } assert(deltaScans.size == 1) deltaScans.head.files.length } /** Drops `columnName` from `tableName`. */ def dropColumn(tableName: String, columnName: String): Unit = { sql(s"ALTER TABLE $tableName DROP COLUMN $columnName") assertColumnNotExist(tableName, columnName) } /** Changes `columnName` to `newType` */ def alterColumnType(tableName: String, columnName: String, newType: String): Unit = { sql(s"ALTER TABLE $tableName ALTER COLUMN $columnName TYPE $newType") } /** Whether the table protocol supports the given table feature. */ def isFeatureSupported(tableName: String, tableFeature: TableFeature): Boolean = { val protocol = getProtocolForTable(tableName) protocol.isFeatureSupported(tableFeature) } /** Whether the table protocol supports the given table feature. */ def isFeatureSupported(tableName: String, featureName: String): Boolean = { val protocol = getProtocolForTable(tableName) protocol.readerFeatureNames.contains(featureName) || protocol.writerFeatureNames.contains(featureName) } /** Enables table feature for `tableName` and given `featureName`. */ def enableTableFeature(tableName: String, featureName: String): Unit = { sql(s""" |ALTER TABLE $tableName |SET TBLPROPERTIES('delta.feature.$featureName' = 'supported') |""".stripMargin) assert(isFeatureSupported(tableName, featureName)) } /** Drops table feature for `tableName` and `featureName`. */ def dropTableFeature(tableName: String, featureName: String): Unit = { sql(s"ALTER TABLE $tableName DROP FEATURE `$featureName`") assert(!isFeatureSupported(tableName, featureName)) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/test/DeltaTestImplicits.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test import java.io.File import java.sql.Timestamp import org.apache.spark.sql.delta.{CatalogOwnedTableFeature, DeltaHistoryManager, DeltaLog, OptimisticTransaction, Snapshot} import org.apache.spark.sql.delta.DeltaOperations.{ManualUpdate, Operation, Write} import org.apache.spark.sql.delta.SnapshotDescriptor import org.apache.spark.sql.delta.actions.{Action, AddFile, Metadata, Protocol, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.commands.optimize.OptimizeMetrics import org.apache.spark.sql.delta.coordinatedcommits.TableCommitCoordinatorClient import org.apache.spark.sql.delta.files.TahoeLogFileIndex import org.apache.spark.sql.delta.hooks.AutoCompact import org.apache.spark.sql.delta.stats.StatisticsCollection import io.delta.storage.commit.{CommitResponse, GetCommitsResponse, UpdatedActions} import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.sql.{SaveMode, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.expressions.{Expression, Literal} import org.apache.spark.util.Clock /** * Additional method definitions for Delta classes that are intended for use only in testing. */ object DeltaTestImplicits { implicit class OptimisticTxnTestHelper(txn: OptimisticTransaction) { /** Ensure that the initial commit of a Delta table always contains a Metadata action */ def commitActions(op: Operation, actions: Action*): Long = { if (txn.readVersion == -1) { val metadataOpt = actions.collectFirst { case m: Metadata => m } val protocolOpt = actions.collectFirst { case p: Protocol => p } val otherActions = actions.filterNot(a => a.isInstanceOf[Metadata] || a.isInstanceOf[Protocol]) (metadataOpt, protocolOpt) match { case (Some(metadata), Some(protocol)) => // When both metadata and protocol are explicitly passed, use them. txn.updateProtocol(protocol) // This will auto upgrade any required table features in the passed protocol as per // given metadata. txn.updateMetadataForNewTable(metadata) case (Some(metadata), None) => // When just metadata is passed, use it. // This will auto generate protocol as per metadata. txn.updateMetadataForNewTable(metadata) case (None, Some(protocol)) => txn.updateProtocol(protocol) txn.updateMetadataForNewTable(Metadata()) case (None, None) => // If neither metadata nor protocol is explicitly passed, then use default Metadata and // with the maximum protocol. txn.updateMetadataForNewTable(Metadata()) val enableCatalogOwnedByDefault = SparkSession.active.conf.getOption( TableFeatureProtocolUtils.defaultPropertyKey(CatalogOwnedTableFeature)) .contains("supported") if (enableCatalogOwnedByDefault) { txn.updateProtocol(Action.supportedProtocolVersion()) } else { txn.updateProtocol(Action.supportedProtocolVersion( // CatalogOwnedTableFeature is enabled by protocol only without metadata, and should // not be enabled by default. featuresToExclude = Seq(CatalogOwnedTableFeature))) } } txn.commit(otherActions, op) } else { txn.commit(actions, op) } } def commitManually(actions: Action*): Long = { commitActions(ManualUpdate, actions: _*) } def commitWriteAppend(actions: Action*): Long = { commitActions(Write(SaveMode.Append), actions: _*) } } /** Add test-only File overloads for DeltaTable.forPath */ implicit class DeltaLogObjectTestHelper(deltaLog: DeltaLog.type) { def forTable(spark: SparkSession, dataPath: File): DeltaLog = { DeltaLog.forTable(spark, new Path(dataPath.getCanonicalPath)) } def forTable(spark: SparkSession, dataPath: File, clock: Clock): DeltaLog = { DeltaLog.forTable(spark, new Path(dataPath.getCanonicalPath), clock) } } implicit class DeltaHistoryManagerTestHelper(history: DeltaHistoryManager) { def checkVersionExists(version: Long): Unit = { history.checkVersionExists(version, catalogTableOpt = None) } def getActiveCommitAtTime( timestamp: Long, canReturnLastCommit: Boolean): DeltaHistoryManager.Commit = { history.getActiveCommitAtTime( new Timestamp(timestamp), catalogTableOpt = None, canReturnLastCommit) } } /** Helper class for working with [[TableCommitCoordinatorClient]] */ implicit class TableCommitCoordinatorClientTestHelper( tableCommitCoordinatorClient: TableCommitCoordinatorClient) { def commit( commitVersion: Long, actions: Iterator[String], updatedActions: UpdatedActions): CommitResponse = { tableCommitCoordinatorClient.commit( commitVersion, actions, updatedActions, tableIdentifierOpt = None) } def getCommits( startVersion: Option[Long] = None, endVersion: Option[Long] = None): GetCommitsResponse = { tableCommitCoordinatorClient.getCommits(tableIdentifierOpt = None, startVersion, endVersion) } def backfillToVersion( version: Long, lastKnownBackfilledVersion: Option[Long] = None): Unit = { tableCommitCoordinatorClient.backfillToVersion( tableIdentifierOpt = None, version, lastKnownBackfilledVersion) } } /** Helper class for working with [[Snapshot]] */ implicit class SnapshotTestHelper(snapshot: Snapshot) { def ensureCommitFilesBackfilled(): Unit = { snapshot.ensureCommitFilesBackfilled(catalogTableOpt = None) } } /** * Helper class for working with the most recent snapshot in the deltaLog */ implicit class DeltaLogTestHelper(deltaLog: DeltaLog) { def snapshot: Snapshot = { deltaLog.unsafeVolatileSnapshot } def checkpoint(): Unit = { deltaLog.checkpoint(snapshot) } def checkpointInterval(): Int = { deltaLog.checkpointInterval(snapshot.metadata) } def deltaRetentionMillis(): Long = { deltaLog.deltaRetentionMillis(snapshot.metadata) } def enableExpiredLogCleanup(): Boolean = { deltaLog.enableExpiredLogCleanup(snapshot.metadata) } def upgradeProtocol(newVersion: Protocol): Unit = { upgradeProtocol(deltaLog.unsafeVolatileSnapshot, newVersion) } def upgradeProtocol(snapshot: Snapshot, newVersion: Protocol): Unit = { deltaLog.upgradeProtocol(None, snapshot, newVersion) } /** * Test helper method for getChangeLogFiles that provides catalogTableOpt = None * for backward compatibility with existing unit tests. */ def getChangeLogFiles(startVersion: Long): Iterator[(Long, FileStatus)] = { deltaLog.getChangeLogFiles(startVersion, catalogTableOpt = None) } /** * Test helper method for getChanges that provides catalogTableOpt = None for backward * compatibility with existing unit tests. */ def getChanges( startVersion: Long, failOnDataLoss: Boolean = false): Iterator[(Long, Seq[Action])] = { deltaLog.getChanges(startVersion, catalogTableOpt = None, failOnDataLoss) } } implicit class DeltaTableV2ObjectTestHelper(dt: DeltaTableV2.type) { /** Convenience overload that omits the cmd arg (which is not helpful in tests). */ def apply(spark: SparkSession, id: TableIdentifier): DeltaTableV2 = dt.apply(spark, id, "test") def apply(spark: SparkSession, tableDir: File): DeltaTableV2 = dt.apply(spark, new Path(tableDir.getAbsolutePath)) def apply(spark: SparkSession, tableDir: File, clock: Clock): DeltaTableV2 = { val tablePath = new Path(tableDir.getAbsolutePath) DeltaTableV2.testOnlyApplyWithCustomDeltaLog(spark, tablePath, clock) } } implicit class DeltaTableV2TestHelper(deltaTable: DeltaTableV2) { /** For backward compatibility with existing unit tests */ def snapshot: Snapshot = deltaTable.initialSnapshot } implicit class TahoeLogFileIndexObjectTestHelper(index: TahoeLogFileIndex.type) { def apply(spark: SparkSession, deltaLog: DeltaLog): TahoeLogFileIndex = { index.apply(spark, deltaLog, catalogTableOpt = None) } } implicit class AutoCompactObjectTestHelper(ac: AutoCompact.type) { private[delta] def compact( spark: SparkSession, deltaLog: DeltaLog, partitionPredicates: Seq[Expression] = Nil, opType: String = AutoCompact.OP_TYPE): Seq[OptimizeMetrics] = { AutoCompact.compact( spark, deltaLog, catalogTable = None, partitionPredicates, opType) } } implicit class StatisticsCollectionObjectTestHelper(sc: StatisticsCollection.type) { /** * This is an implicit helper required for backward compatibility with existing * unit tests. It allows to call [[StatisticsCollection.recompute]] without a * catalog table and in the actual call, sets it to [[None]]. */ def recompute( spark: SparkSession, deltaLog: DeltaLog, predicates: Seq[Expression] = Seq(Literal(true)), fileFilter: AddFile => Boolean = af => true): Unit = { StatisticsCollection.recompute( spark, deltaLog, catalogTable = None, predicates, fileFilter) } } implicit class CDCReaderObjectTestHelper(cdcReader: CDCReader.type) { /** * Test helper method for changesToBatchDF that provides catalogTableOpt = None * for backward compatibility with existing unit tests. */ def changesToBatchDF( deltaLog: DeltaLog, start: Long, end: Long, spark: SparkSession, readSchemaSnapshot: Option[Snapshot] = None, useCoarseGrainedCDC: Boolean = false, startVersionSnapshot: Option[SnapshotDescriptor] = None ): org.apache.spark.sql.DataFrame = { cdcReader.changesToBatchDF( deltaLog, start, end, spark, catalogTableOpt = None, readSchemaSnapshot, useCoarseGrainedCDC, startVersionSnapshot) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/test/ScanReportHelper.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test import scala.util.control.NonFatal import org.apache.spark.sql.delta.files.TahoeFileIndex import org.apache.spark.sql.delta.metering.ScanReport import org.apache.spark.sql.delta.stats.{DataSize, PreparedDeltaFileIndex} import org.apache.spark.sql.execution.{FileSourceScanExec, QueryExecution, SparkPlan} import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.util.QueryExecutionListener /** * A helper trait used by test classes that want to collect the scans (i.e. [[FileSourceScanExec]]) * generated by a given input query during query planning. * * This trait exposes a single public API [[getScanReport]]. */ trait ScanReportHelper extends SharedSparkSession with AdaptiveSparkPlanHelper { import ScanReportHelper._ /** * Collect the scan leaves in the given SparkPlan. */ private def collectScans(plan: SparkPlan): Seq[FileSourceScanExec] = { collectWithSubqueries(plan)({ case fs: FileSourceScanExec => Seq(fs) case cached: InMemoryTableScanExec => collectScans(cached.relation.cacheBuilder.cachedPlan) }).flatten } /** * Returns a new [[QueryExecutionListener]] that can be registered to the Spark listener bus * to analyse and collect metrics during query execution. * * Specifically, this listener will check for any [[FileSourceScanExec]] generated during query * planning, cast them into [[ScanReport]] (helper class to hold useful info about the scan), and * append to the singleton [[ScanReportHelper.scans]] */ private def getListener(): QueryExecutionListener = { new QueryExecutionListener { override def onSuccess(funcName: String, qe: QueryExecution, durationNs: Long): Unit = { try qe.assertAnalyzed() catch { case NonFatal(e) => logDebug("Not running Delta Metering because the query failed during analysis.", e) return } val fileScans = collectScans(qe.executedPlan) for (scanExec <- fileScans) { scanExec.relation.location match { case deltaTable: PreparedDeltaFileIndex => val preparedScan = deltaTable.preparedScan // The names of the partition columns that were used as filters in this scan. // Convert this to a set first to avoid double-counting partition columns that might // appear multiple times. val usedPartitionColumns = preparedScan.partitionFilters.map(_.references.map(_.name)).flatten.toSet.toSeq val report = ScanReport( tableId = deltaTable.metadata.id, path = deltaTable.path.toString, scanType = "delta-query", deltaDataSkippingType = preparedScan.dataSkippingType.toString, partitionFilters = preparedScan.partitionFilters.map(_.sql).toSeq, partitionLikeDataFilters = preparedScan.partitionLikeDataFilters.map(_.sql).toSeq, rewrittenPartitionLikeDataFilters = preparedScan.rewrittenPartitionLikeFilterSQL.toSeq, dataFilters = preparedScan.dataFilters.map(_.sql).toSeq, unusedFilters = preparedScan.unusedFilters.map(_.sql).toSeq, size = Map( "total" -> preparedScan.total, "partition" -> preparedScan.partition, "scanned" -> preparedScan.scanned), metrics = scanExec.metrics.mapValues(_.value).toMap + ("scanDurationMs" -> preparedScan.scanDurationMs), annotations = Map.empty, versionScanned = deltaTable.versionScanned, usedPartitionColumns = usedPartitionColumns, numUsedPartitionColumns = usedPartitionColumns.size, allPartitionColumns = deltaTable.metadata.partitionColumns, numAllPartitionColumns = deltaTable.metadata.partitionColumns.size, parentFilterOutputRows = None ) scans += report case deltaTable: TahoeFileIndex => val report = ScanReport( tableId = deltaTable.metadata.id, path = deltaTable.path.toString, scanType = "delta-unknown", partitionFilters = Nil, dataFilters = Nil, partitionLikeDataFilters = Nil, rewrittenPartitionLikeDataFilters = Nil, unusedFilters = Nil, size = Map( "total" -> DataSize( bytesCompressed = Some(deltaTable.deltaLog.unsafeVolatileSnapshot.sizeInBytes)), "scanned" -> DataSize(bytesCompressed = Some(deltaTable.sizeInBytes)) ), metrics = scanExec.metrics.mapValues(_.value).toMap, versionScanned = None, annotations = Map.empty ) scans += report case _ => // ignore } } } override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = { } } } /** * Execute function `f` and return the scans generated during query planning */ def getScanReport(f: => Unit): Seq[ScanReport] = { synchronized { assert(scans == null, "getScanReport does not support nested invocation.") scans = scala.collection.mutable.ArrayBuffer.empty[ScanReport] } val listener = getListener() spark.listenerManager.register(listener) var result: scala.collection.mutable.ArrayBuffer[ScanReport] = null try { f } finally { spark.sparkContext.listenerBus.waitUntilEmpty(15000) spark.listenerManager.unregister(listener) result = scans synchronized { scans = null } } result.toSeq } } object ScanReportHelper { @volatile var scans: scala.collection.mutable.ArrayBuffer[ScanReport] = null } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/test/TestsStatistics.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.execution.{ColumnarToRowExec, FileSourceScanExec, InputAdapter, SparkPlan} import org.apache.spark.sql.functions.from_json import org.apache.spark.sql.{Column, DataFrame} /** * Provides utilities for testing StatisticsCollection. */ trait TestsStatistics { self: DeltaSQLTestUtils => /** A function to get the reconciled statistics DataFrame from the DeltaLog */ protected var getStatsDf: (DeltaLog, Seq[Column]) => DataFrame = _ /** * Creates the correct `getStatsDf` to be used by the `testFun` and executes the `testFun`. */ protected def statsTest(testName: String, testTags: org.scalatest.Tag*)(testFun: => Any): Unit = { import testImplicits._ test(testName, testTags: _*) { getStatsDf = (deltaLog, columns) => { val snapshot = deltaLog.snapshot snapshot.allFiles .withColumn("stats", from_json($"stats", snapshot.statsSchema)) .select("stats.*") .select(columns: _*) } testFun } } /** * Creates the correct `getStatsDf` to be used by the `testFun` and executes the `testFun`. * Runs only against Spark master. */ protected def statsTestSparkMasterOnly( testName: String, testTags: org.scalatest.Tag*)(testFun: => Any): Unit = { import testImplicits._ test(testName, testTags: _*) { getStatsDf = (deltaLog, columns) => { val snapshot = deltaLog.snapshot snapshot.allFiles .withColumn("stats", from_json($"stats", snapshot.statsSchema)) .select("stats.*") .select(columns: _*) } testFun } } /** * A util to match a physical file scan node. */ object FileScanExecNode { def unapply(plan: SparkPlan): Option[FileSourceScanExec] = plan match { case f: FileSourceScanExec => Some(f) case InputAdapter(f: FileSourceScanExec) => Some(f) case ColumnarToRowExec(InputAdapter(f: FileSourceScanExec)) => Some(f) case _ => None } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningAlterTableNestedSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.typewidening import org.apache.spark.sql.{AnalysisException, QueryTest} import org.apache.spark.sql.execution.datasources.parquet.ParquetTest import org.apache.spark.sql.types._ /** * Suite providing additional coverage for widening nested fields using ALTER TABLE CHANGE COLUMN * TYPE. */ class TypeWideningAlterTableNestedSuite extends QueryTest with ParquetTest with TypeWideningTestMixin with TypeWideningAlterTableNestedTests trait TypeWideningAlterTableNestedTests { self: QueryTest with ParquetTest with TypeWideningTestMixin => import testImplicits._ /** Create a table with a struct, map and array for each test. */ protected def createNestedTable(): Unit = { sql(s"CREATE TABLE delta.`$tempPath` " + "(s struct, m map, a array) USING DELTA") append(Seq((1, 2, 3, 4)) .toDF("a", "b", "c", "d") .selectExpr( "named_struct('a', cast(a as byte)) as s", "map(cast(b as byte), cast(c as short)) as m", "array(cast(d as short)) as a")) assert(readDeltaTable(tempPath).schema === new StructType() .add("s", new StructType().add("a", ByteType)) .add("m", MapType(ByteType, ShortType)) .add("a", ArrayType(ShortType))) } test("unsupported ALTER TABLE CHANGE COLUMN on non-leaf fields") { createNestedTable() // Running ALTER TABLE CHANGE COLUMN on non-leaf fields is invalid. var alterTableSql = s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN s TYPE struct" checkError( intercept[AnalysisException] { sql(alterTableSql) }, "CANNOT_UPDATE_FIELD.STRUCT_TYPE", parameters = Map( "table" -> s"`spark_catalog`.`delta`.`$tempPath`", "fieldName" -> "`s`" ), context = ExpectedContext( fragment = alterTableSql, start = 0, stop = alterTableSql.length - 1) ) alterTableSql = s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN m TYPE map" checkError( intercept[AnalysisException] { sql(alterTableSql) }, "CANNOT_UPDATE_FIELD.MAP_TYPE", parameters = Map( "table" -> s"`spark_catalog`.`delta`.`$tempPath`", "fieldName" -> "`m`" ), context = ExpectedContext( fragment = alterTableSql, start = 0, stop = alterTableSql.length - 1) ) alterTableSql = s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE array" checkError( intercept[AnalysisException] { sql(alterTableSql) }, "CANNOT_UPDATE_FIELD.ARRAY_TYPE", parameters = Map( "table" -> s"`spark_catalog`.`delta`.`$tempPath`", "fieldName" -> "`a`" ), context = ExpectedContext( fragment = alterTableSql, start = 0, stop = alterTableSql.length - 1) ) } test("type widening with ALTER TABLE on nested fields") { createNestedTable() sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN s.a TYPE short") sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN m.key TYPE int") sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN m.value TYPE int") sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a.element TYPE int") assert(readDeltaTable(tempPath).schema === new StructType() .add("s", new StructType() .add("a", ShortType)) .add("m", MapType(IntegerType, IntegerType)) .add("a", ArrayType(IntegerType))) append(Seq((5, 6, 7, 8)) .toDF("a", "b", "c", "d") .selectExpr("named_struct('a', cast(a as short)) as s", "map(b, c) as m", "array(d) as a")) checkAnswer( readDeltaTable(tempPath), Seq((1, 2, 3, 4), (5, 6, 7, 8)) .toDF("a", "b", "c", "d") .selectExpr("named_struct('a', cast(a as short)) as s", "map(b, c) as m", "array(d) as a")) } test("type widening using ALTER TABLE REPLACE COLUMNS on nested fields") { createNestedTable() sql(s"ALTER TABLE delta.`$tempPath` REPLACE COLUMNS " + "(s struct, m map, a array)") assert(readDeltaTable(tempPath).schema === new StructType() .add("s", new StructType() .add("a", ShortType)) .add("m", MapType(IntegerType, IntegerType)) .add("a", ArrayType(IntegerType))) append(Seq((5, 6, 7, 8)) .toDF("a", "b", "c", "d") .selectExpr("named_struct('a', cast(a as short)) as s", "map(b, c) as m", "array(d) as a")) checkAnswer( readDeltaTable(tempPath), Seq((1, 2, 3, 4), (5, 6, 7, 8)) .toDF("a", "b", "c", "d") .selectExpr("named_struct('a', cast(a as short)) as s", "map(b, c) as m", "array(d) as a")) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningAlterTableSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.typewidening import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.expressions.Cast import org.apache.spark.sql.errors.QueryErrorsBase import org.apache.spark.sql.execution.datasources.parquet.ParquetTest import org.apache.spark.sql.functions.lit import org.apache.spark.sql.types._ /** * Suite providing core coverage for type widening using ALTER TABLE CHANGE COLUMN TYPE. */ class TypeWideningAlterTableSuite extends TypeWideningAlterTableTests with ParquetTest with TypeWideningTestMixin trait TypeWideningAlterTableTests extends QueryTest with QueryErrorsBase with TypeWideningTestCases { self: QueryTest with ParquetTest with TypeWideningTestMixin => import testImplicits._ for { testCase <- supportedTestCases ++ restrictedAutomaticWideningTestCases partitioned <- BOOLEAN_DOMAIN } { test(s"type widening ${testCase.fromType.sql} -> ${testCase.toType.sql}, " + s"partitioned=$partitioned") { def writeData(df: DataFrame): Unit = if (partitioned) { // The table needs to have at least 1 non-partition column, use a dummy one. append(df.withColumn("dummy", lit(1)), partitionBy = Seq("value")) } else { append(df) } writeData(testCase.initialValuesDF) sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN value TYPE ${testCase.toType.sql}") withAllParquetReaders { assert(readDeltaTable(tempPath).schema("value").dataType === testCase.toType) checkAnswerWithTolerance( actualDf = readDeltaTable(tempPath).select("value"), expectedDf = testCase.initialValuesDF.select($"value".cast(testCase.toType)), toType = testCase.toType ) } writeData(testCase.additionalValuesDF) withAllParquetReaders { checkAnswerWithTolerance( actualDf = readDeltaTable(tempPath).select("value"), expectedDf = testCase.expectedResult.select($"value".cast(testCase.toType)), toType = testCase.toType ) } } } for { testCase <- unsupportedTestCases partitioned <- BOOLEAN_DOMAIN } { test(s"unsupported type changes ${testCase.fromType.sql} -> ${testCase.toType.sql}, " + s"partitioned=$partitioned") { if (partitioned) { // The table needs to have at least 1 non-partition column, use a dummy one. append(testCase.initialValuesDF.withColumn("dummy", lit(1)), partitionBy = Seq("value")) } else { append(testCase.initialValuesDF) } val alterTableSql = s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN value TYPE ${testCase.toType.sql}" // Type changes that aren't upcast are rejected early during analysis by Spark, while upcasts // are rejected in Delta when the ALTER TABLE command is executed. if (Cast.canUpCast(testCase.fromType, testCase.toType)) { checkError( intercept[DeltaAnalysisException] { sql(alterTableSql) }, "DELTA_UNSUPPORTED_ALTER_TABLE_CHANGE_COL_OP", sqlState = None, parameters = Map( "fieldPath" -> "value", "oldField" -> testCase.fromType.sql, "newField" -> testCase.toType.sql) ) } else { checkError( intercept[AnalysisException] { sql(alterTableSql) }, "NOT_SUPPORTED_CHANGE_COLUMN", sqlState = None, parameters = Map( "table" -> s"`spark_catalog`.`delta`.`$tempPath`", "originName" -> toSQLId("value"), "originType" -> toSQLType(testCase.fromType), "newName" -> toSQLId("value"), "newType" -> toSQLType(testCase.toType)), context = ExpectedContext( fragment = alterTableSql, start = 0, stop = alterTableSql.length - 1) ) } } } test("type widening using ALTER TABLE REPLACE COLUMNS") { append(Seq(1, 2).toDF("value").select($"value".cast(ShortType))) assert(readDeltaTable(tempPath).schema === new StructType().add("value", ShortType)) sql(s"ALTER TABLE delta.`$tempPath` REPLACE COLUMNS (value INT)") assert(readDeltaTable(tempPath).schema === new StructType().add("value", IntegerType)) checkAnswer(readDeltaTable(tempPath), Seq(Row(1), Row(2))) append(Seq(3, 4).toDF("value")) checkAnswer(readDeltaTable(tempPath), Seq(Row(1), Row(2), Row(3), Row(4))) } def withTimestampNTZDisabled(f: => Unit): Unit = { val timestampNTZKey = TableFeatureProtocolUtils.defaultPropertyKey(TimestampNTZTableFeature) conf.unsetConf(timestampNTZKey) if (!conf.contains(timestampNTZKey)) return f val timestampNTZSupported = conf.getConfString(timestampNTZKey) conf.unsetConf(timestampNTZKey) try { f } finally { conf.setConfString(timestampNTZKey, timestampNTZSupported) } } test( "widening Date -> TimestampNTZ rejected when TimestampNTZ feature isn't supported") { withTimestampNTZDisabled { sql(s"CREATE TABLE delta.`$tempPath` (a date) USING DELTA") val currentProtocol = deltaLog.unsafeVolatileSnapshot.protocol val currentFeatures = currentProtocol.implicitlyAndExplicitlySupportedFeatures .map(_.name) .toSeq .sorted .mkString(", ") checkError( intercept[DeltaTableFeatureException] { sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE TIMESTAMP_NTZ") }, "DELTA_FEATURES_REQUIRE_MANUAL_ENABLEMENT", parameters = Map( "unsupportedFeatures" -> "timestampNtz", "supportedFeatures" -> currentFeatures ) ) } } test("type widening type change metrics") { sql(s"CREATE TABLE delta.`$tempDir` (a byte) USING DELTA") val usageLogs = Log4jUsageLogger.track { sql(s"ALTER TABLE delta.`$tempDir` CHANGE COLUMN a TYPE int") } val metrics = filterUsageRecords(usageLogs, "delta.typeWidening.typeChanges") .map(r => JsonUtils.fromJson[Map[String, Seq[Map[String, String]]]](r.blob)) .head assert(metrics("changes") === Seq( Map( "fromType" -> "TINYINT", "toType" -> "INT" )) ) } test("type widening with user-defined type in table") { val dataWithUDT = (1 to 10).map(x => Tuple2(x.toByte, new TestUDT.MyDenseVector(Array(x*0.5, x*2.0)))) append(dataWithUDT.toDF("a", "udt")) sql(s"ALTER TABLE delta.`$tempDir` CHANGE COLUMN a TYPE int") } test("type widening with null type in table") { sql(s"CREATE TABLE delta.`$tempDir` (a byte, n VOID) USING DELTA") sql(s"ALTER TABLE delta.`$tempDir` CHANGE COLUMN a TYPE int") } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningConstraintsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.typewidening import org.apache.spark.sql.delta.DeltaAnalysisException import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.functions.col import org.apache.spark.sql.types._ /** * Suite covering changing the type of columns referenced by constraints, e.g. CHECK constraints or * NOT NULL constraints. */ class TypeWideningConstraintsSuite extends QueryTest with TypeWideningTestMixin with TypeWideningConstraintsTests trait TypeWideningConstraintsTests { self: QueryTest with TypeWideningTestMixin => test("not null constraint with type change") { withTable("t") { sql("CREATE TABLE t (a byte NOT NULL) USING DELTA") sql("INSERT INTO t VALUES (1)") checkAnswer(sql("SELECT * FROM t"), Row(1)) // Changing the type of a column with a NOT NULL constraint is allowed. sql("ALTER TABLE t CHANGE COLUMN a TYPE SMALLINT") assert(sql("SELECT * FROM t").schema("a").dataType === ShortType) sql("INSERT INTO t VALUES (2)") checkAnswer(sql("SELECT * FROM t"), Seq(Row(1), Row(2))) } } test("check constraint with type change") { withTable("t") { sql("CREATE TABLE t (a byte, b byte) USING DELTA") sql("ALTER TABLE t ADD CONSTRAINT ck CHECK (hash(a) > 0)") sql("INSERT INTO t VALUES (2, 2)") checkAnswer(sql("SELECT hash(a) FROM t"), Row(1765031574)) // Changing the type of a column that a CHECK constraint depends on is not allowed. checkError( intercept[DeltaAnalysisException] { sql("ALTER TABLE t CHANGE COLUMN a TYPE SMALLINT") }, "DELTA_CONSTRAINT_DEPENDENT_COLUMN_CHANGE", parameters = Map( "columnName" -> "a", "constraints" -> "delta.constraints.ck -> hash ( a ) > 0" )) // Changing the type of `b` is allowed as it's not referenced by the constraint. sql("ALTER TABLE t CHANGE COLUMN b TYPE SMALLINT") assert(sql("SELECT * FROM t").schema("b").dataType === ShortType) checkAnswer(sql("SELECT * FROM t"), Row(2, 2)) } } test("check constraint on nested field with type change") { withTable("t") { sql("CREATE TABLE t (a struct) USING DELTA") sql("ALTER TABLE t ADD CONSTRAINT ck CHECK (hash(a.x) > 0)") sql("INSERT INTO t (a) VALUES (named_struct('x', 2, 'y', 3))") checkAnswer(sql("SELECT hash(a.x) FROM t"), Row(1765031574)) checkError( intercept[DeltaAnalysisException] { sql("ALTER TABLE t CHANGE COLUMN a.x TYPE SMALLINT") }, "DELTA_CONSTRAINT_DEPENDENT_COLUMN_CHANGE", parameters = Map( "columnName" -> "a.x", "constraints" -> "delta.constraints.ck -> hash ( a . x ) > 0" )) // Changing the type of a.y is allowed since it's not referenced by the CHECK constraint. sql("ALTER TABLE t CHANGE COLUMN a.y TYPE SMALLINT") checkAnswer(sql("SELECT * FROM t"), Row(Row(2, 3))) } } test("check constraint on arrays and maps with type change") { withTable("t") { sql("CREATE TABLE t (m map, a array) USING DELTA") sql("INSERT INTO t VALUES (map(1, 2, 7, -3), array(1, -2, 3))") sql("ALTER TABLE t CHANGE COLUMN a.element TYPE SMALLINT") sql("ALTER TABLE t ADD CONSTRAINT ch1 CHECK (hash(a[1]) = -1160545675)") checkError( intercept[DeltaAnalysisException] { sql("ALTER TABLE t CHANGE COLUMN a.element TYPE INTEGER") }, "DELTA_CONSTRAINT_DEPENDENT_COLUMN_CHANGE", parameters = Map( "columnName" -> "a.element", "constraints" -> "delta.constraints.ch1 -> hash ( a [ 1 ] ) = - 1160545675" ) ) sql("ALTER TABLE t CHANGE COLUMN m.value TYPE SMALLINT") sql("ALTER TABLE t ADD CONSTRAINT ch2 CHECK (sign(m[7]) < 0)") checkError( intercept[DeltaAnalysisException] { sql("ALTER TABLE t CHANGE COLUMN m.value TYPE INTEGER") }, "DELTA_CONSTRAINT_DEPENDENT_COLUMN_CHANGE", parameters = Map( "columnName" -> "m.value", "constraints" -> "delta.constraints.ch2 -> sign ( m [ 7 ] ) < 0" ) ) } } test(s"check constraint with type evolution") { withTable("t") { sql(s"CREATE TABLE t (a byte) USING DELTA") sql("ALTER TABLE t ADD CONSTRAINT ck CHECK (hash(a) > 0)") sql("INSERT INTO t VALUES (2)") checkAnswer(sql("SELECT hash(a) FROM t"), Row(1765031574)) withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { checkError( intercept[DeltaAnalysisException] { sql("INSERT INTO t VALUES (200)") }, "DELTA_CONSTRAINT_DATA_TYPE_MISMATCH", parameters = Map( "columnName" -> "a", "columnType" -> "TINYINT", "dataType" -> "INT", "constraints" -> "delta.constraints.ck -> hash ( a ) > 0" )) } } } test("check constraint on nested field with type evolution") { withTable("t") { sql("CREATE TABLE t (a struct) USING DELTA") sql("ALTER TABLE t ADD CONSTRAINT ck CHECK (hash(a.x) > 0)") sql("INSERT INTO t (a) VALUES (named_struct('x', 2, 'y', 3))") checkAnswer(sql("SELECT hash(a.x) FROM t"), Row(1765031574)) withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { checkError( intercept[DeltaAnalysisException] { sql("INSERT INTO t (a) VALUES (named_struct('x', 200, 'y', CAST(5 AS byte)))") }, "DELTA_CONSTRAINT_DATA_TYPE_MISMATCH", parameters = Map( "columnName" -> "a.x", "columnType" -> "TINYINT", "dataType" -> "INT", "constraints" -> "delta.constraints.ck -> hash ( a . x ) > 0" ) ) // changing the type of struct field `a.y` when it's not // the field referenced by the CHECK constraint is allowed. sql("INSERT INTO t (a) VALUES (named_struct('x', CAST(2 AS byte), 'y', 500))") checkAnswer(sql("SELECT hash(a.x) FROM t"), Seq(Row(1765031574), Row(1765031574))) } } } test("check constraint on nested field with complex type evolution") { withTable("t") { sql("CREATE TABLE t (a struct, y: byte>) USING DELTA") sql("ALTER TABLE t ADD CONSTRAINT ck CHECK (hash(a.x.z) > 0)") sql("INSERT INTO t (a) VALUES (named_struct('x', named_struct('z', 2, 'h', 3), 'y', 4))") checkAnswer(sql("SELECT hash(a.x.z) FROM t"), Row(1765031574)) withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { checkError( intercept[DeltaAnalysisException] { sql( s""" | INSERT INTO t (a) VALUES ( | named_struct('x', named_struct('z', 200, 'h', 3), 'y', 4) | ) |""".stripMargin ) }, "DELTA_CONSTRAINT_DATA_TYPE_MISMATCH", parameters = Map( "columnName" -> "a.x.z", "columnType" -> "TINYINT", "dataType" -> "INT", "constraints" -> "delta.constraints.ck -> hash ( a . x . z ) > 0" ) ) // changing the type of struct field `a.y` and `a.x.h` when it's not // the field referenced by the CHECK constraint is allowed. sql( """ | INSERT INTO t (a) VALUES ( | named_struct('x', named_struct('z', CAST(2 AS BYTE), 'h', 2002), 'y', 1030) | ) |""".stripMargin ) checkAnswer(sql("SELECT hash(a.x.z) FROM t"), Seq(Row(1765031574), Row(1765031574))) } } } test("check constraint on arrays and maps with type evolution") { withTable("t") { sql("CREATE TABLE t (s struct>>) USING DELTA") sql("ALTER TABLE t ADD CONSTRAINT ck CHECK (s.arr[0][3] = 3)") sql("INSERT INTO t(s) VALUES (struct(struct(array(map(1, 1, 3, 3)))))") checkAnswer(sql("SELECT s.arr[0][3] FROM t"), Row(3)) withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { // Insert by name is not supported by type evolution. checkError( intercept[DeltaAnalysisException] { // Migrate map's key to int type. spark.createDataFrame(Seq(Tuple1(Tuple1(Array(Map(999999 -> 1, 3 -> 3)))))) .toDF("s").withColumn("s", col("s").cast("struct>>")) .write.format("delta").mode("append").saveAsTable("t") }, "DELTA_CONSTRAINT_DATA_TYPE_MISMATCH", parameters = Map( "columnName" -> "s.arr.element.key", "columnType" -> "TINYINT", "dataType" -> "INT", "constraints" -> "delta.constraints.ck -> s . arr [ 0 ] [ 3 ] = 3" ) ) checkError( intercept[DeltaAnalysisException] { // Migrate map's value to int type. spark.createDataFrame(Seq(Tuple1(Tuple1(Array(Map(1 -> 999999, 3 -> 3)))))) .toDF("s").withColumn("s", col("s").cast("struct>>")) .write.format("delta").mode("append").saveAsTable("t") }, "DELTA_CONSTRAINT_DATA_TYPE_MISMATCH", parameters = Map( "columnName" -> "s.arr.element.value", "columnType" -> "TINYINT", "dataType" -> "INT", "constraints" -> "delta.constraints.ck -> s . arr [ 0 ] [ 3 ] = 3" ) ) } } } test("add constraint after type change then RESTORE") { withTable("t") { sql("CREATE TABLE t (a byte) USING DELTA") sql("INSERT INTO t VALUES (2)") sql("ALTER TABLE t CHANGE COLUMN a TYPE INT") sql("INSERT INTO t VALUES (5)") checkAnswer(sql("SELECT a, hash(a) FROM t"), Seq(Row(2, 1765031574), Row(5, 1023896466))) sql("ALTER TABLE t ADD CONSTRAINT ck CHECK (hash(a) > 0)") // Constraints are stored in the table metadata, RESTORE removes the constraint so the type // change can't get in the way. sql(s"RESTORE TABLE t VERSION AS OF 1") sql("INSERT INTO t VALUES (1)") checkAnswer(sql("SELECT a, hash(a) FROM t"), Seq(Row(2, 1765031574), Row(1, -559580957))) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningFeatureCompatibilitySuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.typewidening import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row} import org.apache.spark.sql.functions.col import org.apache.spark.sql.types._ class TypeWideningFeatureCompatibilitySuite extends QueryTest with DeltaDMLTestUtils with TypeWideningTestMixin with TypeWideningDropFeatureTestMixin with TypeWideningCompatibilityTests with TypeWideningColumnMappingTests /** Tests covering type widening compatibility with other delta features. */ trait TypeWideningCompatibilityTests { self: TypeWideningTestMixin with QueryTest with DeltaDMLTestUtils => import testImplicits._ test("reading CDF with a type change") { withSQLConf((DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey, "true")) { sql(s"CREATE TABLE delta.`$tempPath` (a smallint) USING DELTA") } append(Seq(1, 2).toDF("a").select($"a".cast(ShortType))) sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE int") append(Seq(3, 4).toDF("a")) def readCDF(start: Long, end: Long): DataFrame = CDCReader .changesToBatchDF(deltaLog, start, end, spark) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) .drop(CDCReader.CDC_COMMIT_VERSION) checkErrorMatchPVals( intercept[DeltaUnsupportedOperationException] { readCDF(start = 1, end = 1).collect() }, "DELTA_CHANGE_DATA_FEED_INCOMPATIBLE_DATA_SCHEMA", parameters = Map( "start" -> "1", "end" -> "1", "readSchema" -> ".*", "readVersion" -> "3", "incompatibleVersion" -> "1", "config" -> ".*defaultSchemaModeForColumnMappingTable" ) ) checkAnswer(readCDF(start = 3, end = 3), Seq(Row(3, "insert"), Row(4, "insert"))) } test("reading CDF with a type change using read schema from before the change") { withSQLConf((DeltaConfigs.CHANGE_DATA_FEED.defaultTablePropertyKey, "true")) { sql(s"CREATE TABLE delta.`$tempPath` (a smallint) USING DELTA") } append(Seq(1, 2).toDF("a").select($"a".cast(ShortType))) val readSchemaSnapshot = deltaLog.update() sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE int") append(Seq(3, 4).toDF("a")) def readCDF(start: Long, end: Long): DataFrame = CDCReader .changesToBatchDF( deltaLog, start, end, spark, catalogTableOpt = None, readSchemaSnapshot = Some(readSchemaSnapshot) ) .drop(CDCReader.CDC_COMMIT_TIMESTAMP) .drop(CDCReader.CDC_COMMIT_VERSION) checkAnswer(readCDF(start = 1, end = 1), Seq(Row(1, "insert"), Row(2, "insert"))) checkErrorMatchPVals( intercept[DeltaUnsupportedOperationException] { readCDF(start = 1, end = 3) }, "DELTA_CHANGE_DATA_FEED_INCOMPATIBLE_SCHEMA_CHANGE", parameters = Map( "start" -> "1", "end" -> "3", "readSchema" -> ".*", "readVersion" -> "1", "incompatibleVersion" -> "2" ) ) } test("time travel read before type change") { sql(s"CREATE TABLE delta.`$tempPath` (a byte) USING DELTA") append(Seq(1).toDF("a").select($"a".cast(ByteType))) sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE smallint") append(Seq(2).toDF("a").select($"a".cast(ShortType))) val previousVersion = sql(s"SELECT a FROM delta.`$tempPath` VERSION AS OF 1") assert(previousVersion.schema("a").dataType === ByteType) checkAnswer(previousVersion, Seq(Row(1))) val latestVersion = sql(s"SELECT a FROM delta.`$tempPath`") assert(latestVersion.schema("a").dataType === ShortType) checkAnswer(latestVersion, Seq(Row(1), Row(2))) } test("compatibility with char/varchar columns") { sql(s"CREATE TABLE delta.`$tempPath` (a byte, c char(3), v varchar(3)) USING DELTA") append(Seq((1.toByte, "abc", "def")).toDF("a", "c", "v")) checkAnswer(readDeltaTable(tempPath), Seq(Row(1, "abc", "def"))) sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE smallint") append(Seq((2.toShort, "ghi", "jkl")).toDF("a", "c", "v")) assert(readDeltaTable(tempPath).schema === new StructType() .add("a", ShortType) .add("c", StringType, nullable = true, metadata = new MetadataBuilder() .putString("__CHAR_VARCHAR_TYPE_STRING", "char(3)") .build() ) .add("v", StringType, nullable = true, metadata = new MetadataBuilder() .putString("__CHAR_VARCHAR_TYPE_STRING", "varchar(3)") .build())) checkAnswer(readDeltaTable(tempPath), Seq(Row(1, "abc", "def"), Row(2, "ghi", "jkl"))) sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN c TYPE string") sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN v TYPE string") append(Seq((3.toShort, "longer string 1", "longer string 2")).toDF("a", "c", "v")) assert(readDeltaTable(tempPath).schema === new StructType() .add("a", ShortType) .add("c", StringType) .add("v", StringType)) checkAnswer(readDeltaTable(tempPath), Seq(Row(1, "abc", "def"), Row(2, "ghi", "jkl"), Row(3, "longer string 1", "longer string 2"))) } test("type widening with row tracking") { // Start with row tracking disabled. sql(s"CREATE TABLE $tableSQLIdentifier (a TINYINT) USING DELTA " + s"TBLPROPERTIES('${DeltaConfigs.ROW_TRACKING_ENABLED.key}' = 'false')") append(Seq(1).toDF("a").select($"a".cast(ByteType))) sql(s"ALTER TABLE $tableSQLIdentifier SET TBLPROPERTIES " + s"('${DeltaConfigs.ROW_TRACKING_ENABLED.key}' = 'true')") def readWithRowTracking(): DataFrame = readDeltaTable(tempPath).select( "a", "_metadata.row_id", "_metadata.base_row_id", "_metadata.row_commit_version", "_metadata.default_row_commit_version" ) // [base_]row_id starting at 0, [default_]row_commit_version set to 3 when // Row Tracking got enabled. checkAnswer(readWithRowTracking(), Seq(Row(1, 0, 0, 3, 3))) sql(s"UPDATE $tableSQLIdentifier SET a = 2 WHERE a = 1") // Existing row moved to new file: base_row_id = 1. Version updated to 5 // (4 is internal row tracking backfill). checkAnswer(readWithRowTracking(), Seq(Row(2, 0, 1, 5, 5))) sql(s"ALTER TABLE $tableSQLIdentifier CHANGE COLUMN a TYPE INT") // No changes when enabling Type Widening. checkAnswer(readWithRowTracking(), Seq(Row(2, 0, 1, 5, 5))) append(Seq(Int.MaxValue).toDF("a")) // Adding new row in a new file [base_]row_id set to 2, [default_]row_commit_version set to 7. checkAnswer(readWithRowTracking(), Seq( Row(2, 0, 1, 5, 5), Row(Int.MaxValue, 2, 2, 7, 7) )) } } /** Trait collecting tests covering type widening + column mapping. */ trait TypeWideningColumnMappingTests { self: QueryTest with TypeWideningTestMixin with TypeWideningDropFeatureTestMixin => import testImplicits._ for (mappingMode <- Seq(IdMapping.name, NameMapping.name)) { test(s"change column type and rename it, mappingMode=$mappingMode") { withSQLConf((DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey, mappingMode)) { sql(s"CREATE TABLE delta.`$tempPath` (a byte) USING DELTA") } // Add some data and change type of column `a`. addSingleFile(Seq(1), ByteType) sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE smallint") addSingleFile(Seq(2), ShortType) assert(readDeltaTable(tempPath).schema("a").dataType === ShortType) checkAnswer(sql(s"SELECT a FROM delta.`$tempPath`"), Seq(Row(1), Row(2))) // Rename column `a` to `a (with reserved characters)`, add more data. val newColumnName = "a (with reserved characters)" sql(s"ALTER TABLE delta.`$tempPath` RENAME COLUMN a TO `$newColumnName`") assert(readDeltaTable(tempPath).schema(newColumnName).dataType === ShortType) checkAnswer( sql(s"SELECT `$newColumnName` FROM delta.`$tempPath`"), Seq(Row(1), Row(2)) ) append(Seq(3).toDF(newColumnName).select(col(newColumnName).cast(ShortType))) checkAnswer( sql(s"SELECT `$newColumnName` FROM delta.`$tempPath`"), Seq(Row(1), Row(2), Row(3)) ) // Change column type again, add more data. sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN `$newColumnName` TYPE int") assert( readDeltaTable(tempPath).schema(newColumnName).dataType === IntegerType) append(Seq(4).toDF(newColumnName).select(col(newColumnName).cast(IntegerType))) checkAnswer( sql(s"SELECT `$newColumnName` FROM delta.`$tempPath`"), Seq(Row(1), Row(2), Row(3), Row(4)) ) dropTableFeature( expectedOutcome = ExpectedOutcome.FAIL_CURRENT_VERSION_USES_FEATURE, // All files except the last one should be rewritten. expectedNumFilesRewritten = 3, expectedColumnTypes = Map(newColumnName -> IntegerType) ) } test(s"dropped column shouldn't cause files to be rewritten, mappingMode=$mappingMode") { withSQLConf((DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey, mappingMode)) { sql(s"CREATE TABLE delta.`$tempPath` (a byte, b byte) USING DELTA") } sql(s"INSERT INTO delta.`$tempPath` VALUES (1, 1)") sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN b TYPE int") sql(s"INSERT INTO delta.`$tempPath` VALUES (2, 2)") sql(s"ALTER TABLE delta.`$tempPath` DROP COLUMN b") dropTableFeature( expectedOutcome = ExpectedOutcome.FAIL_HISTORICAL_VERSION_USES_FEATURE, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> ByteType) ) } test(s"swap column names and change type, mappingMode=$mappingMode") { withSQLConf((DeltaConfigs.COLUMN_MAPPING_MODE.defaultTablePropertyKey, mappingMode)) { sql(s"CREATE TABLE delta.`$tempPath` (a byte, b byte) USING DELTA") } sql(s"INSERT INTO delta.`$tempPath` VALUES (1, 1)") sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN b TYPE int") sql(s"INSERT INTO delta.`$tempPath` VALUES (2, 2)") sql(s"ALTER TABLE delta.`$tempPath` RENAME COLUMN b TO c") sql(s"ALTER TABLE delta.`$tempPath` RENAME COLUMN a TO b") sql(s"ALTER TABLE delta.`$tempPath` RENAME COLUMN c TO a") sql(s"INSERT INTO delta.`$tempPath` VALUES (3, 3)") dropTableFeature( expectedOutcome = ExpectedOutcome.FAIL_CURRENT_VERSION_USES_FEATURE, expectedNumFilesRewritten = 1, expectedColumnTypes = Map( "a" -> IntegerType, "b" -> ByteType ) ) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningGeneratedColumnsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.typewidening import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.functions.col import org.apache.spark.sql.types._ /** * Suite covering changing the type of columns referenced by generated columns. */ class TypeWideningGeneratedColumnsSuite extends QueryTest with TypeWideningTestMixin with GeneratedColumnTest with TypeWideningGeneratedColumnTests trait TypeWideningGeneratedColumnTests extends GeneratedColumnTest { self: QueryTest with TypeWideningTestMixin => test("generated column with type change") { withTable("t") { createTable( tableName = "t", path = None, schemaString = "a byte, b byte, gen int", generatedColumns = Map("gen" -> "hash(a)"), partitionColumns = Seq.empty ) sql("INSERT INTO t (a, b) VALUES (2, 2)") checkAnswer(sql("SELECT hash(a) FROM t"), Row(1765031574)) // Changing the type of a column that a generated column depends on is not allowed. checkError( intercept[DeltaAnalysisException] { sql("ALTER TABLE t CHANGE COLUMN a TYPE SMALLINT") }, "DELTA_GENERATED_COLUMNS_DEPENDENT_COLUMN_CHANGE", parameters = Map( "columnName" -> "a", "generatedColumns" -> "gen -> hash(a)" )) // Changing the type of `b` is allowed as it's not referenced by the generated column. sql("ALTER TABLE t CHANGE COLUMN b TYPE SMALLINT") assert(sql("SELECT * FROM t").schema("b").dataType === ShortType) checkAnswer(sql("SELECT * FROM t"), Row(2, 2, 1765031574)) } } test("generated column on nested field with type change") { withTable("t") { createTable( tableName = "t", path = None, schemaString = "a struct, gen int", generatedColumns = Map("gen" -> "hash(a.x)"), partitionColumns = Seq.empty ) sql("INSERT INTO t (a) VALUES (named_struct('x', 2, 'y', 3))") checkAnswer(sql("SELECT hash(a.x) FROM t"), Row(1765031574)) checkError( intercept[DeltaAnalysisException] { sql("ALTER TABLE t CHANGE COLUMN a.x TYPE SMALLINT") }, "DELTA_GENERATED_COLUMNS_DEPENDENT_COLUMN_CHANGE", parameters = Map( "columnName" -> "a.x", "generatedColumns" -> "gen -> hash(a.x)" )) // Changing the type of a.y is allowed since it's not referenced by the CHECK constraint. sql("ALTER TABLE t CHANGE COLUMN a.y TYPE SMALLINT") checkAnswer(sql("SELECT * FROM t"), Row(Row(2, 3), 1765031574) :: Nil) } } test("generated column on arrays and maps with type change") { withTable("t") { createTable( tableName = "t", path = None, schemaString = "a array>, gen tinyint", generatedColumns = Map("gen" -> "a[0].f"), partitionColumns = Seq.empty ) sql("INSERT INTO t (a) VALUES (array(named_struct('f', 7, 'g', 8)))") checkAnswer(sql("SELECT gen FROM t"), Row(7)) sql("ALTER TABLE t CHANGE COLUMN a.element.g TYPE SMALLINT") checkError( intercept[DeltaAnalysisException] { sql("ALTER TABLE t CHANGE COLUMN a.element.f TYPE SMALLINT") }, "DELTA_GENERATED_COLUMNS_DEPENDENT_COLUMN_CHANGE", parameters = Map( "columnName" -> "a.element.f", "generatedColumns" -> "gen -> a[0].f" )) checkAnswer(sql("SELECT gen FROM t"), Row(7)) } } test("generated column with type evolution") { withTable("t") { createTable( tableName = "t", path = None, schemaString = "a byte, gen int", generatedColumns = Map("gen" -> "hash(a)"), partitionColumns = Seq.empty ) sql("INSERT INTO t (a) VALUES (2)") checkAnswer(sql("SELECT hash(a) FROM t"), Row(1765031574)) withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { checkError( intercept[DeltaAnalysisException] { sql("INSERT INTO t (a) VALUES (200)") }, "DELTA_GENERATED_COLUMNS_DATA_TYPE_MISMATCH", parameters = Map( "columnName" -> "a", "columnType" -> "TINYINT", "dataType" -> "INT", "generatedColumns" -> "gen -> hash(a)" )) } } } test("generated column on nested field with type evolution") { withTable("t") { createTable( tableName = "t", path = None, schemaString = "a struct, gen int", generatedColumns = Map("gen" -> "hash(a.x)"), partitionColumns = Seq.empty ) sql("INSERT INTO t (a) VALUES (named_struct('x', 2, 'y', 3))") checkAnswer(sql("SELECT gen FROM t"), Row(1765031574)) withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { checkError( intercept[DeltaAnalysisException] { sql("INSERT INTO t (a) VALUES (named_struct('x', 200, 'y', CAST(5 AS byte)))") }, "DELTA_GENERATED_COLUMNS_DATA_TYPE_MISMATCH", parameters = Map( "columnName" -> "a.x", "columnType" -> "TINYINT", "dataType" -> "INT", "generatedColumns" -> "gen -> hash(a.x)" ) ) // changing the type of struct field `a.y` when it's not // the field referenced by the generated column is allowed. sql("INSERT INTO t (a) VALUES (named_struct('x', CAST(2 AS byte), 'y', 200))") checkAnswer(sql("SELECT gen FROM t"), Seq(Row(1765031574), Row(1765031574))) } } } test("generated column on arrays and maps with type evolution") { withTable("t") { createTable( tableName = "t", path = None, schemaString = "a array, gen INT", generatedColumns = Map("gen" -> "hash(a[0])"), partitionColumns = Seq.empty ) sql("INSERT INTO t (a) VALUES (array(2, 3))") checkAnswer(sql("SELECT gen FROM t"), Row(1765031574)) withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { // Insert by name is not supported by type evolution. checkError( intercept[DeltaAnalysisException] { spark.createDataFrame(Seq(Tuple1(Array(200000, 12345)))) .toDF("a").withColumn("a", col("a").cast("array")) .write.format("delta").mode("append").saveAsTable("t") }, "DELTA_GENERATED_COLUMNS_DATA_TYPE_MISMATCH", parameters = Map( "columnName" -> "a.element", "columnType" -> "TINYINT", "dataType" -> "INT", "generatedColumns" -> "gen -> hash(a[0])" ) ) checkAnswer(sql("SELECT gen FROM t"), Row(1765031574)) } } } test("generated column on nested field with complex type evolution") { withTable("t") { createTable( tableName = "t", path = None, schemaString = "a struct, y: byte>, gen int", generatedColumns = Map("gen" -> "hash(a.x.z)"), partitionColumns = Seq.empty ) sql("INSERT INTO t (a) VALUES (named_struct('x', named_struct('z', 2, 'h', 3), 'y', 4))") checkAnswer(sql("SELECT gen FROM t"), Row(1765031574)) withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { checkError( intercept[DeltaAnalysisException] { sql( s""" | INSERT INTO t (a) VALUES ( | named_struct('x', named_struct('z', 200, 'h', 3), 'y', 4) | ) |""".stripMargin ) }, "DELTA_GENERATED_COLUMNS_DATA_TYPE_MISMATCH", parameters = Map( "columnName" -> "a.x.z", "columnType" -> "TINYINT", "dataType" -> "INT", "generatedColumns" -> "gen -> hash(a.x.z)" ) ) // changing the type of struct field `a.y` when it's not // the field referenced by the generated column is allowed. sql( """ | INSERT INTO t (a) VALUES ( | named_struct('x', named_struct('z', CAST(2 AS BYTE), 'h', 2002), 'y', 1030) | ) |""".stripMargin ) checkAnswer(sql("SELECT gen FROM t"), Seq(Row(1765031574), Row(1765031574))) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningInsertSchemaEvolutionBasicSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.typewidening import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.sql.{DataFrame, Dataset, QueryTest, Row, SaveMode} import org.apache.spark.sql.catalyst.plans.logical.{AppendData, LogicalPlan} import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy import org.apache.spark.sql.types._ import org.apache.spark.sql.util.CaseInsensitiveStringMap /** * Suite covering widening columns and fields type as part of automatic schema evolution in INSERT * when the type widening table feature is supported. */ class TypeWideningInsertSchemaEvolutionBasicSuite extends QueryTest with DeltaDMLTestUtils with TypeWideningTestMixin with TypeWideningInsertSchemaEvolutionBasicTests { protected override def sparkConf: SparkConf = { super.sparkConf .set(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key, "true") } } /** * Tests covering type widening during schema evolution in INSERT. */ trait TypeWideningInsertSchemaEvolutionBasicTests extends DeltaInsertIntoTest with TypeWideningTestCases { self: QueryTest with TypeWideningTestMixin with DeltaDMLTestUtils => import testImplicits._ import scala.collection.JavaConverters._ for { testCase <- restrictedAutomaticWideningTestCases ++ supportedTestCases } { test(s"INSERT - always automatic type widening " + s"${testCase.fromType.sql} -> ${testCase.toType.sql}") { append(testCase.initialValuesDF) withSQLConf(DeltaSQLConf.DELTA_ALLOW_AUTOMATIC_WIDENING.key -> "always") { testCase.additionalValuesDF .write .format("delta") .mode("append") .option("mergeSchema", "true") .insertInto(s"delta.`$tempPath`") } assert(readDeltaTable(tempPath).schema("value").dataType === testCase.toType) checkAnswerWithTolerance( actualDf = readDeltaTable(tempPath).select("value"), expectedDf = testCase.expectedResult.select($"value".cast(testCase.toType)), toType = testCase.toType ) } } for { testCase <- restrictedAutomaticWideningTestCases ++ supportedTestCases } { test(s"INSERT - never automatic type widening " + s"${testCase.fromType.sql} -> ${testCase.toType.sql}") { append(testCase.initialValuesDF) withSQLConf(SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.LEGACY.toString, DeltaSQLConf.DELTA_ALLOW_AUTOMATIC_WIDENING.key -> "never") { testCase.additionalValuesDF .write .format("delta") .mode("append") .option("mergeSchema", "true") .insertInto(s"delta.`$tempPath`") } assert(readDeltaTable(tempPath).schema("value").dataType === testCase.fromType) } } test(s"INSERT - logs for missed opportunity for conversion") { val testCase = restrictedAutomaticWideningTestCases.head append(testCase.initialValuesDF) val events = Log4jUsageLogger.track { withSQLConf( SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.LEGACY.toString, DeltaSQLConf.DELTA_ALLOW_AUTOMATIC_WIDENING.key -> "same_family_type") { testCase.additionalValuesDF .write .format("delta") .mode("append") .option("mergeSchema", "true") .insertInto(s"delta.`$tempPath`") } } assert(readDeltaTable(tempPath).schema("value").dataType === testCase.fromType) assert(events.exists(event => event.metric == "tahoeEvent" && event.tags.get("opType") == Option("delta.typeWidening.missedAutomaticWidening"))) } test(s"INSERT - no logs for lack of missed opportunity for conversion") { val testCase = supportedTestCases.head append(testCase.initialValuesDF) val events = Log4jUsageLogger.track { withSQLConf(SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.LEGACY.toString) { testCase.additionalValuesDF .write .format("delta") .mode("append") .option("mergeSchema", "true") .insertInto(s"delta.`$tempPath`") } } assert(readDeltaTable(tempPath).schema("value").dataType === testCase.toType) assert(!events.exists(event => event.metric == "tahoeEvent" && event.tags.get("opType") == Option("delta.typeWidening.missedAutomaticWidening"))) } for { testCase <- supportedTestCases ++ restrictedAutomaticWideningTestCases } { test(s"INSERT - automatic type widening ${testCase.fromType.sql} -> ${testCase.toType.sql}") { append(testCase.initialValuesDF) testCase.additionalValuesDF .write .mode("append") .insertInto(s"delta.`$tempPath`") assert(readDeltaTable(tempPath).schema("value").dataType === testCase.toType) checkAnswerWithTolerance( actualDf = readDeltaTable(tempPath).select("value"), expectedDf = testCase.expectedResult.select($"value".cast(testCase.toType)), toType = testCase.toType ) } } for { testCase <- unsupportedTestCases } { test(s"INSERT - unsupported automatic type widening " + s"${testCase.fromType.sql} -> ${testCase.toType.sql}") { append(testCase.initialValuesDF) // Test cases for some of the unsupported type changes may overflow while others only have // values that can be implicitly cast to the narrower type - e.g. double ->float. // We set storeAssignmentPolicy to LEGACY to ignore overflows, this test only ensures // that the table schema didn't evolve. withSQLConf(SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.LEGACY.toString) { testCase.additionalValuesDF.write.mode("append") .insertInto(s"delta.`$tempPath`") assert(readDeltaTable(tempPath).schema("value").dataType === testCase.fromType) } } } test("INSERT - type widening isn't applied when schema evolution is disabled") { sql(s"CREATE TABLE delta.`$tempPath` (a short) USING DELTA") withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "false") { // Insert integer values. This should succeed and downcast the values to short. sql(s"INSERT INTO delta.`$tempPath` VALUES (1), (2)") assert(readDeltaTable(tempPath).schema("a").dataType === ShortType) checkAnswer(readDeltaTable(tempPath), Seq(1, 2).toDF("a").select($"a".cast(ShortType))) } // Check that we would actually widen if schema evolution was enabled. withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { sql(s"INSERT INTO delta.`$tempPath` VALUES (3), (4)") assert(readDeltaTable(tempPath).schema("a").dataType === IntegerType) checkAnswer(readDeltaTable(tempPath), Seq(1, 2, 3, 4).toDF("a")) } } test("INSERT - type widening isn't applied when it's disabled") { sql(s"CREATE TABLE delta.`$tempPath` (a short) USING DELTA") enableTypeWidening(tempPath, enabled = false) sql(s"INSERT INTO delta.`$tempPath` VALUES (1), (2)") assert(readDeltaTable(tempPath).schema("a").dataType === ShortType) checkAnswer(readDeltaTable(tempPath), Seq(1, 2).toDF("a").select($"a".cast(ShortType))) } test("INSERT - type widening is triggered when schema evolution is enabled via option") { val tableName = "type_widening_insert_into_table" withTable(tableName) { sql(s"CREATE TABLE $tableName (a short) USING DELTA") Seq(1, 2).toDF("a") .write .format("delta") .mode("append") .option("mergeSchema", "true") .insertInto(tableName) val result = spark.read.format("delta").table(tableName) assert(result.schema("a").dataType === IntegerType) checkAnswer(result, Seq(1, 2).toDF("a")) } } /** * Short-hand to create a logical plan to insert into the table. This captures the state of the * table at the time the method is called, e.p. the type widening property value that will be used * during analysis. */ private def createInsertPlan(df: DataFrame): LogicalPlan = { val relation = DataSourceV2Relation.create( table = DeltaTableV2(spark, new Path(tempPath)), catalog = None, identifier = None, options = new CaseInsensitiveStringMap(Map.empty[String, String].asJava) ) AppendData.byPosition(relation, df.queryExecution.logical) } test(s"INSERT - fail if type widening gets enabled by a concurrent transaction") { sql(s"CREATE TABLE delta.`$tempPath` (a short) USING DELTA") enableTypeWidening(tempPath, enabled = false) val insert = createInsertPlan(Seq(1).toDF("a")) // Enabling type widening after analysis doesn't impact the insert operation: the data is // already cast to conform to the current schema. enableTypeWidening(tempPath, enabled = true) DataFrameUtils.ofRows(spark, insert).collect() assert(readDeltaTable(tempPath).schema == new StructType().add("a", ShortType)) checkAnswer(readDeltaTable(tempPath), Row(1)) } test(s"INSERT - fail if type widening gets disabled by a concurrent transaction") { sql(s"CREATE TABLE delta.`$tempPath` (a short) USING DELTA") val insert = createInsertPlan(Seq(1).toDF("a")) // Disabling type widening after analysis results in inserting data with a wider type into the // table while type widening is actually disabled during execution. We do actually widen the // table schema in that case because `short` and `int` are both stored as INT32 in parquet. enableTypeWidening(tempPath, enabled = false) DataFrameUtils.ofRows(spark, insert).collect() assert(readDeltaTable(tempPath).schema == new StructType().add("a", IntegerType)) checkAnswer(readDeltaTable(tempPath), Row(1)) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningInsertSchemaEvolutionExtendedSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.typewidening import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.SparkConf import org.apache.spark.sql.QueryTest import org.apache.spark.sql.types._ class TypeWideningInsertSchemaEvolutionExtendedSuite extends QueryTest with DeltaDMLTestUtils with TypeWideningTestMixin with TypeWideningInsertSchemaEvolutionExtendedTests { } trait TypeWideningInsertSchemaEvolutionExtendedTests extends DeltaInsertIntoTest with TypeWideningTestCases { self: QueryTest with TypeWideningTestMixin with DeltaDMLTestUtils => testInserts("top-level type evolution")( initialData = TestData("a int, b short", Seq("""{ "a": 1, "b": 2 }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, b int", Seq("""{ "a": 1, "b": 4 }""")), expectedResult = ExpectedResult.Success(new StructType() .add("a", IntegerType) .add("b", IntegerType)), withSchemaEvolution = true ) testInserts("top-level type evolution with column upcast")( initialData = TestData("a int, b short, c int", Seq("""{ "a": 1, "b": 2, "c": 3 }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, b int, c short", Seq("""{ "a": 1, "b": 5, "c": 6 }""")), expectedResult = ExpectedResult.Success(new StructType() .add("a", IntegerType) .add("b", IntegerType) .add("c", IntegerType)), withSchemaEvolution = true ) testInserts("top-level type evolution with schema evolution")( initialData = TestData("a int, b short", Seq("""{ "a": 1, "b": 2 }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, b int, c int", Seq("""{ "a": 1, "b": 4, "c": 5 }""")), expectedResult = ExpectedResult.Success(new StructType() .add("a", IntegerType) .add("b", IntegerType) .add("c", IntegerType)), // SQL INSERT by name doesn't support schema evolution. excludeInserts = insertsSQL.intersect(insertsByName), withSchemaEvolution = true ) testInserts("nested type evolution by position")( initialData = TestData( "key int, s struct, m map, a array", Seq("""{ "key": 1, "s": { "x": 1, "y": 2 }, "m": { "p": 3 }, "a": [4] }""")), partitionBy = Seq("key"), overwriteWhere = "key" -> 1, insertData = TestData( "key int, s struct, m map, a array", Seq("""{ "key": 1, "s": { "x": 4, "y": 5 }, "m": { "p": 6 }, "a": [7] }""")), expectedResult = ExpectedResult.Success(new StructType() .add("key", IntegerType) .add("s", new StructType() .add("x", ShortType) .add("y", IntegerType)) .add("m", MapType(StringType, IntegerType)) .add("a", ArrayType(IntegerType))), withSchemaEvolution = true ) testInserts("nested type evolution with struct evolution by position")( initialData = TestData( "key int, s struct, m map, a array", Seq("""{ "key": 1, "s": { "x": 1, "y": 2 }, "m": { "p": 3 }, "a": [4] }""")), partitionBy = Seq("key"), overwriteWhere = "key" -> 1, insertData = TestData( "key int, s struct, m map, a array", Seq("""{ "key": 1, "s": { "x": 4, "y": 5, "z": 8 }, "m": { "p": 6 }, "a": [7] }""")), expectedResult = ExpectedResult.Success(new StructType() .add("key", IntegerType) .add("s", new StructType() .add("x", ShortType) .add("y", IntegerType) .add("z", IntegerType)) .add("m", MapType(StringType, IntegerType)) .add("a", ArrayType(IntegerType))), withSchemaEvolution = true ) testInserts("nested struct type evolution with field upcast")( initialData = TestData( "key int, s struct", Seq("""{ "key": 1, "s": { "x": 1, "y": 2 } }""")), partitionBy = Seq("key"), overwriteWhere = "key" -> 1, insertData = TestData( "key int, s struct", Seq("""{ "key": 1, "s": { "x": 4, "y": 5 } }""")), expectedResult = ExpectedResult.Success(new StructType() .add("key", IntegerType) .add("s", new StructType() .add("x", IntegerType) .add("y", IntegerType))), withSchemaEvolution = true ) // Interestingly, we introduced a special case to handle schema evolution / casting for structs // directly nested into an array. This doesn't always work with maps or with elements that // aren't a struct (see other tests). testInserts("nested struct type evolution with field upcast in array")( initialData = TestData( "key int, a array>", Seq("""{ "key": 1, "a": [ { "x": 1, "y": 2 } ] }""")), partitionBy = Seq("key"), overwriteWhere = "key" -> 1, insertData = TestData( "key int, a array>", Seq("""{ "key": 1, "a": [ { "x": 3, "y": 4 } ] }""")), expectedResult = ExpectedResult.Success(new StructType() .add("key", IntegerType) .add("a", ArrayType(new StructType() .add("x", IntegerType) .add("y", IntegerType)))), withSchemaEvolution = true ) // maps now allow type evolution for INSERT by position and name in SQL and dataframe. testInserts("nested struct type evolution with field upcast in map")( initialData = TestData( "key int, m map>", Seq("""{ "key": 1, "m": { "a": { "x": 1, "y": 2 } } }""")), partitionBy = Seq("key"), overwriteWhere = "key" -> 1, insertData = TestData( "key int, m map>", Seq("""{ "key": 1, "m": { "a": { "x": 3, "y": 4 } } }""")), expectedResult = ExpectedResult.Success(new StructType() .add("key", IntegerType) // Type evolution was applied in the map. .add("m", MapType(StringType, new StructType() .add("x", IntegerType) .add("y", IntegerType)))), withSchemaEvolution = true ) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningMergeIntoSchemaEvolutionSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.typewidening import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.SparkConf import org.apache.spark.sql.QueryTest import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy import org.apache.spark.sql.types._ /** * Suite covering widening columns and fields type as part of automatic schema evolution in MERGE * INTO when the type widening table feature is supported. */ class TypeWideningMergeIntoSchemaEvolutionSuite extends TypeWideningMergeIntoSchemaEvolutionTests with DeltaDMLTestUtils with TypeWideningTestMixin { protected override def sparkConf: SparkConf = { super.sparkConf .set(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key, "true") } } /** * Tests covering type widening during schema evolution in MERGE INTO. */ trait TypeWideningMergeIntoSchemaEvolutionTests extends QueryTest with MergeIntoSQLTestUtils with MergeIntoSchemaEvolutionMixin with TypeWideningTestCases { self: QueryTest with TypeWideningTestMixin with DeltaDMLTestUtils => import testImplicits._ test(s"MERGE - always automatic type widening TINYINT -> DOUBLE") { withTable("source") { sql(s"CREATE TABLE delta.`$tempPath` (a short) USING DELTA") sql("CREATE TABLE source (a double) USING DELTA") sql("INSERT INTO source VALUES (3.0), (-10.5)") withSQLConf(DeltaSQLConf.DELTA_ALLOW_AUTOMATIC_WIDENING.key -> "always") { // Merge double values. This should succeed and widen the short column to double. executeMerge( tgt = s"delta.`$tempPath` t", src = "source", cond = "0 = 1", clauses = insert("*") ) assert(readDeltaTable(tempPath).schema("a").dataType === DoubleType) checkAnswer(readDeltaTable(tempPath), Seq(3.0, -10.5).toDF("a")) } } } test(s"MERGE - never automatic type widening TINYINT -> INT") { withTable("source") { sql(s"CREATE TABLE delta.`$tempPath` (a short) USING DELTA") sql("CREATE TABLE source (a int) USING DELTA") sql("INSERT INTO source VALUES (1), (2)") withSQLConf(DeltaSQLConf.DELTA_ALLOW_AUTOMATIC_WIDENING.key -> "never", SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.LEGACY.toString) { // Merge int values into short column. This should not widen the target schema. executeMerge( tgt = s"delta.`$tempPath` t", src = "source", cond = "0 = 1", clauses = insert("*") ) assert(readDeltaTable(tempPath).schema("a").dataType === ShortType) } } } for { testCase <- supportedTestCases ++ restrictedAutomaticWideningTestCases } { test(s"MERGE - automatic type widening ${testCase.fromType.sql} -> ${testCase.toType.sql}") { withTable("source") { testCase.additionalValuesDF.write.format("delta").saveAsTable("source") append(testCase.initialValuesDF) // We mainly want to ensure type widening is correctly applied to the schema. We use a // trivial insert only merge to make it easier to validate results. executeMerge( tgt = s"delta.`$tempPath` t", src = "source", cond = "0 = 1", clauses = insert("*")) assert(readDeltaTable(tempPath).schema("value").dataType === testCase.toType) checkAnswerWithTolerance( actualDf = readDeltaTable(tempPath).select("value"), expectedDf = testCase.expectedResult.select($"value".cast(testCase.toType)), toType = testCase.toType ) } } } for { testCase <- unsupportedTestCases } { test(s"MERGE - unsupported automatic type widening " + s"${testCase.fromType.sql} -> ${testCase.toType.sql}") { withTable("source") { testCase.additionalValuesDF.write.format("delta").saveAsTable("source") append(testCase.initialValuesDF) // Test cases for some of the unsupported type changes may overflow while others only have // values that can be implicitly cast to the narrower type - e.g. double ->float. // We set storeAssignmentPolicy to LEGACY to ignore overflows, this test only ensures // that the table schema didn't evolve. withSQLConf(SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.LEGACY.toString) { executeMerge( tgt = s"delta.`$tempPath` t", src = "source", cond = "0 = 1", clauses = insert("*")) assert(readDeltaTable(tempPath).schema("value").dataType === testCase.fromType) } } } } test("MERGE - type widening isn't applied when it's disabled") { withTable("source") { sql(s"CREATE TABLE delta.`$tempPath` (a short) USING DELTA") sql("CREATE TABLE source (a int) USING DELTA") sql("INSERT INTO source VALUES (1), (2)") enableTypeWidening(tempPath, enabled = false) withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { // Merge integer values. This should succeed and downcast the values to short. executeMerge( tgt = s"delta.`$tempPath` t", src = "source", cond = "0 = 1", clauses = insert("*") ) assert(readDeltaTable(tempPath).schema("a").dataType === ShortType) checkAnswer(readDeltaTable(tempPath), Seq(1, 2).toDF("a").select($"a".cast(ShortType))) } } } test("MERGE - type widening isn't applied when schema evolution is disabled") { withTable("source") { sql(s"CREATE TABLE delta.`$tempPath` (a short) USING DELTA") sql("CREATE TABLE source (a int) USING DELTA") sql("INSERT INTO source VALUES (1), (2)") withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "false") { // Merge integer values. This should succeed and downcast the values to short. executeMerge( tgt = s"delta.`$tempPath` t", src = "source", cond = "0 = 1", clauses = insert("*") ) assert(readDeltaTable(tempPath).schema("a").dataType === ShortType) checkAnswer(readDeltaTable(tempPath), Seq(1, 2).toDF("a").select($"a".cast(ShortType))) } // Check that we would actually widen if schema evolution was enabled. withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { executeMerge( tgt = s"delta.`$tempPath` t", src = "source", cond = "0 = 1", clauses = insert("*") ) assert(readDeltaTable(tempPath).schema("a").dataType === IntegerType) checkAnswer(readDeltaTable(tempPath), Seq(1, 2, 1, 2).toDF("a")) } } } /** * Wrapper around testNestedStructsEvolution that constrains the result with and without schema * evolution to be the same: the schema is different but the values should be the same. */ protected def testTypeEvolution(name: String)( target: Seq[String], source: Seq[String], targetSchema: StructType, sourceSchema: StructType, cond: String = "t.key = s.key", clauses: Seq[MergeClause] = Seq.empty, result: Seq[String], resultSchema: StructType): Unit = testNestedStructsEvolution(s"MERGE - $name")( target, source, targetSchema, sourceSchema, cond, clauses, result, resultWithoutEvolution = result, resultSchema = resultSchema) testTypeEvolution("change top-level column short -> int with update")( target = Seq("""{ "a": 0 }""", """{ "a": 10 }"""), source = Seq("""{ "a": 0 }""", """{ "a": 20 }"""), targetSchema = new StructType().add("a", ShortType), sourceSchema = new StructType().add("a", IntegerType), cond = "t.a = s.a", clauses = update("a = s.a + 1") :: Nil, result = Seq("""{ "a": 1 }""", """{ "a": 10 }"""), resultSchema = new StructType().add("a", IntegerType) ) testTypeEvolution("change top-level column short -> int with insert")( target = Seq("""{ "a": 0 }""", """{ "a": 10 }"""), source = Seq("""{ "a": 0 }""", """{ "a": 20 }"""), targetSchema = new StructType().add("a", ShortType), sourceSchema = new StructType().add("a", IntegerType), cond = "t.a = s.a", clauses = insert("(a) VALUES (s.a)") :: Nil, result = Seq("""{ "a": 0 }""", """{ "a": 10 }""", """{ "a": 20 }"""), resultSchema = new StructType().add("a", IntegerType) ) testTypeEvolution("updating using narrower value doesn't evolve schema")( target = Seq("""{ "a": 0 }""", """{ "a": 10 }"""), source = Seq("""{ "a": 0 }""", """{ "a": 20 }"""), targetSchema = new StructType().add("a", IntegerType), sourceSchema = new StructType().add("a", ShortType), cond = "t.a = s.a", clauses = update("a = s.a + 1") :: Nil, result = Seq("""{ "a": 1 }""", """{ "a": 10 }"""), resultSchema = new StructType().add("a", IntegerType) ) testTypeEvolution("only columns in assignments are widened")( target = Seq("""{ "a": 0, "b": 5 }""", """{ "a": 10, "b": 15 }"""), source = Seq("""{ "a": 0, "b": 6 }""", """{ "a": 20, "b": 16 }"""), targetSchema = new StructType() .add("a", ShortType) .add("b", ShortType), sourceSchema = new StructType() .add("a", IntegerType) .add("b", IntegerType), cond = "t.a = s.a", clauses = update("a = s.a + 1") :: Nil, result = Seq( """{ "a": 1, "b": 5 }""", """{ "a": 10, "b": 15 }"""), resultSchema = new StructType() .add("a", IntegerType) .add("b", ShortType) ) testTypeEvolution("automatic widening of struct field with struct assignment")( target = Seq("""{ "s": { "a": 1 } }""", """{ "s": { "a": 10 } }"""), source = Seq("""{ "s": { "a": 1 } }""", """{ "s": { "a": 20 } }"""), targetSchema = new StructType() .add("s", new StructType() .add("a", ShortType)), sourceSchema = new StructType() .add("s", new StructType() .add("a", IntegerType)), cond = "t.s.a = s.s.a", clauses = update("t.s.a = s.s.a + 1") :: Nil, result = Seq("""{ "s": { "a": 2 } }""", """{ "s": { "a": 10 } }"""), resultSchema = new StructType() .add("s", new StructType() .add("a", IntegerType)) ) testTypeEvolution("automatic widening of struct field with field assignment")( target = Seq("""{ "s": { "a": 1 } }""", """{ "s": { "a": 10 } }"""), source = Seq("""{ "s": { "a": 1 } }""", """{ "s": { "a": 20 } }"""), targetSchema = new StructType() .add("s", new StructType() .add("a", ShortType)), sourceSchema = new StructType() .add("s", new StructType() .add("a", IntegerType)), cond = "t.s.a = s.s.a", clauses = update("t.s.a = s.s.a + 1") :: Nil, result = Seq("""{ "s": { "a": 2 } }""", """{ "s": { "a": 10 } }"""), resultSchema = new StructType() .add("s", new StructType() .add("a", IntegerType)) ) testTypeEvolution("automatic widening of map value")( target = Seq("""{ "m": { "a": 1 } }"""), source = Seq("""{ "m": { "a": 2 } }"""), targetSchema = new StructType() .add("m", MapType(StringType, ShortType)), sourceSchema = new StructType() .add("m", MapType(StringType, IntegerType)), // Can't compare maps cond = "1 = 1", clauses = update("t.m = s.m") :: Nil, result = Seq("""{ "m": { "a": 2 } }"""), resultSchema = new StructType() .add("m", MapType(StringType, IntegerType)) ) testTypeEvolution("automatic widening of array element")( target = Seq("""{ "a": [1, 2] }"""), source = Seq("""{ "a": [3, 4] }"""), targetSchema = new StructType() .add("a", ArrayType(ShortType)), sourceSchema = new StructType() .add("a", ArrayType(IntegerType)), cond = "t.a != s.a", clauses = update("t.a = s.a") :: Nil, result = Seq("""{ "a": [3, 4] }"""), resultSchema = new StructType() .add("a", ArrayType(IntegerType)) ) testTypeEvolution("multiple automatic widening")( target = Seq("""{ "a": 1, "b": 2 }"""), source = Seq("""{ "a": 1, "b": 4 }""", """{ "a": 5, "b": 6 }"""), targetSchema = new StructType() .add("a", ByteType) .add("b", ShortType), sourceSchema = new StructType() .add("a", ShortType) .add("b", IntegerType), cond = "t.a = s.a", clauses = update("*") :: insert("*") :: Nil, result = Seq("""{ "a": 1, "b": 4 }""", """{ "a": 5, "b": 6 }"""), resultSchema = new StructType() .add("a", ShortType) .add("b", IntegerType) ) for (enabled <- BOOLEAN_DOMAIN) test(s"MERGE - fail if type widening gets ${if (enabled) "enabled" else "disabled"} by a " + "concurrent transaction") { sql(s"CREATE TABLE delta.`$tempPath` (a short) USING DELTA") enableTypeWidening(tempPath, !enabled) val target = io.delta.tables.DeltaTable.forPath(tempPath) import testImplicits._ val merge = target.as("target") .merge( source = Seq(1L).toDF("a").as("source"), condition = "target.a = source.a") .whenNotMatched().insertAll() // The MERGE operation was created with the previous type widening value, which will apply // during analysis. Toggle type widening so that the actual MERGE runs with a different setting. enableTypeWidening(tempPath, enabled) intercept[MetadataChangedException] { merge.execute() } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningMetadataSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.typewidening import java.io.File import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils.propertyKey import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.types._ /** * Suite that covers recording type change metadata in the table schema. */ class TypeWideningMetadataSuite extends QueryTest with TypeWideningTestMixin with TypeWideningMetadataTests with TypeWideningMetadataEndToEndTests with TypeWideningLeakingMetadataTests /** * Tests covering the [[TypeWideningMetadata]] and [[TypeChange]] classes used to handle the * metadata recorded by the Type Widening table feature in the table schema. */ trait TypeWideningMetadataTests extends QueryTest with DeltaSQLCommandTest { private val testTableName: String = "delta_type_widening_metadata_test" /** A dummy transaction to be used by tests covering `addTypeWideningMetadata`. */ private lazy val txn: OptimisticTransaction = { val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier(testTableName)) DeltaLog.forTable(spark, TableIdentifier(testTableName)) .startTransaction(catalogTableOpt = Some(table)) } override protected def beforeAll(): Unit = { super.beforeAll() sql(s"CREATE TABLE $testTableName (a int) USING delta TBLPROPERTIES (" + s"'${DeltaConfigs.ENABLE_TYPE_WIDENING.key}' = 'true', " + // Force the stable feature to be used by default in tests instead of the preview feature, // which is the one currently enabled by default. Tests that cover aspects specific to the // preview - e.p. populating the `tableVersion` field - explicitly enable the preview feature. s"'${propertyKey(TypeWideningTableFeature)}' = 'supported')") } override protected def afterAll(): Unit = { sql(s"DROP TABLE IF EXISTS $testTableName") super.afterAll() } /** * Short-hand to build the metadata for a type change to cut down on repetition. */ private def typeChangeMetadata( fromType: String, toType: String, path: String = ""): Metadata = { val builder = new MetadataBuilder() .putString("fromType", fromType) .putString("toType", toType) if (path.nonEmpty) { builder.putString("fieldPath", path) } builder.build() } test("toMetadata/fromMetadata with empty path") { val typeChange = TypeChange(version = None, IntegerType, LongType, Seq.empty) assert(typeChange.toMetadata === typeChangeMetadata("integer", "long")) assert(TypeChange.fromMetadata(typeChange.toMetadata) === typeChange) } test("toMetadata/fromMetadata with non-empty path") { val typeChange = TypeChange(version = None, DateType, TimestampNTZType, Seq("key", "element")) assert(typeChange.toMetadata === typeChangeMetadata("date", "timestamp_ntz", "key.element")) assert(TypeChange.fromMetadata(typeChange.toMetadata) === typeChange) } test("toMetadata/fromMetadata with tableVersion") { val typeChange = TypeChange(version = Some(1), ByteType, ShortType, Seq.empty) val expectedMetadata = new MetadataBuilder() .putLong("tableVersion", 1) .putString("fromType", "byte") .putString("toType", "short") .build() assert(typeChange.toMetadata === expectedMetadata) assert(TypeChange.fromMetadata(typeChange.toMetadata) === typeChange) } test("fromField with no type widening metadata") { val field = StructField("a", IntegerType) assert(TypeWideningMetadata.fromField(field) === None) } test("fromField with empty type widening metadata") { val field = StructField("a", IntegerType, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array.empty[Metadata]) .build() ) assert(TypeWideningMetadata.fromField(field) === Some(TypeWideningMetadata(Seq.empty))) val otherField = StructField("a", IntegerType) // Empty type widening metadata is discarded. assert(TypeWideningMetadata.fromField(field).get.appendToField(otherField) === StructField("a", IntegerType)) } test("fromField with single type change") { val field = StructField("a", IntegerType, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("integer", "long") )).build() ) assert(TypeWideningMetadata.fromField(field) === Some(TypeWideningMetadata(Seq( TypeChange(version = None, IntegerType, LongType, Seq.empty))))) val otherField = StructField("a", IntegerType) assert(TypeWideningMetadata.fromField(field).get.appendToField(otherField) === field) } test("fromField with multiple type changes") { val field = StructField("a", IntegerType, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("integer", "long"), typeChangeMetadata("decimal(5,0)", "decimal(10,2)", "element.element") )).build() ) assert(TypeWideningMetadata.fromField(field) === Some(TypeWideningMetadata(Seq( TypeChange(version = None, IntegerType, LongType, Seq.empty), TypeChange( version = None, DecimalType(5, 0), DecimalType(10, 2), Seq("element", "element")))))) val otherField = StructField("a", IntegerType) assert(TypeWideningMetadata.fromField(field).get.appendToField(otherField) === field) } test("fromField with tableVersion") { val typeChange = new MetadataBuilder() .putLong("tableVersion", 1) .putString("fromType", "integer") .putString("toType", "long") .build() val field = StructField("a", IntegerType, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array(typeChange)) .build() ) assert(TypeWideningMetadata.fromField(field) === Some(TypeWideningMetadata(Seq( TypeChange(version = Some(1), IntegerType, LongType, Seq.empty))))) val otherField = StructField("a", IntegerType) assert(TypeWideningMetadata.fromField(field).get.appendToField(otherField) === field) } test("appendToField on field with no type widening metadata") { val field = StructField("a", IntegerType) // Adding empty type widening metadata should not change the field. val emptyMetadata = TypeWideningMetadata(Seq.empty) assert(emptyMetadata.appendToField(field) === field) assert(TypeWideningMetadata.fromField(emptyMetadata.appendToField(field)).isEmpty) // Adding single type change should add the metadata to the field and not otherwise change it. val singleMetadata = TypeWideningMetadata(Seq( TypeChange(version = None, IntegerType, LongType, Seq.empty))) assert(singleMetadata.appendToField(field) === field.copy(metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("integer", "long") )).build() ) ) val singleMetadataFromField = TypeWideningMetadata.fromField(singleMetadata.appendToField(field)) assert(singleMetadataFromField.contains(singleMetadata)) // Adding multiple type changes should add the metadata to the field and not otherwise change // it. val multipleMetadata = TypeWideningMetadata(Seq( TypeChange(version = None, IntegerType, LongType, Seq.empty), TypeChange(version = None, FloatType, DoubleType, Seq("value")))) assert(multipleMetadata.appendToField(field) === field.copy(metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("integer", "long"), typeChangeMetadata("float", "double", "value") )).build() ) ) val multipleMetadataFromField = TypeWideningMetadata.fromField(multipleMetadata.appendToField(field)) assert(multipleMetadataFromField.contains(multipleMetadata)) } test("appendToField on field with existing type widening metadata") { val field = StructField("a", IntegerType, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("integer", "long") )).build() ) // Adding empty type widening metadata should not change the field. val emptyMetadata = TypeWideningMetadata(Seq.empty) assert(emptyMetadata.appendToField(field) === field) assert(TypeWideningMetadata.fromField(emptyMetadata.appendToField(field)).contains( TypeWideningMetadata(Seq( TypeChange(version = None, IntegerType, LongType, Seq.empty))) )) // Adding single type change should add the metadata to the field and not otherwise change it. val singleMetadata = TypeWideningMetadata(Seq( TypeChange(version = None, DecimalType(18, 0), DecimalType(19, 0), Seq.empty))) assert(singleMetadata.appendToField(field) === field.copy( metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("integer", "long"), typeChangeMetadata("decimal(18,0)", "decimal(19,0)") )).build() )) val singleMetadataFromField = TypeWideningMetadata.fromField(singleMetadata.appendToField(field)) assert(singleMetadataFromField.contains(TypeWideningMetadata(Seq( TypeChange(version = None, IntegerType, LongType, Seq.empty), TypeChange(version = None, DecimalType(18, 0), DecimalType(19, 0), Seq.empty))) )) // Adding multiple type changes should add the metadata to the field and not otherwise change // it. val multipleMetadata = TypeWideningMetadata(Seq( TypeChange(version = None, DecimalType(18, 0), DecimalType(19, 0), Seq.empty), TypeChange(version = None, FloatType, DoubleType, Seq("value")))) assert(multipleMetadata.appendToField(field) === field.copy( metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("integer", "long"), typeChangeMetadata("decimal(18,0)", "decimal(19,0)"), typeChangeMetadata("float", "double", "value") )).build() )) val multipleMetadataFromField = TypeWideningMetadata.fromField(multipleMetadata.appendToField(field)) assert(multipleMetadataFromField.contains(TypeWideningMetadata(Seq( TypeChange(version = None, IntegerType, LongType, Seq.empty), TypeChange(version = None, DecimalType(18, 0), DecimalType(19, 0), Seq.empty), TypeChange(version = None, FloatType, DoubleType, Seq("value")))) )) } test("addTypeWideningMetadata/removeTypeWideningMetadata with no type changes") { for { (oldSchema, newSchema) <- Seq( ("a short", "a short"), ("a short", "a short NOT NULL"), ("a short NOT NULL", "a short"), ("a short NOT NULL", "a short COMMENT 'a comment'"), ("a string, b int", "b int, a string"), ("a struct", "a struct"), ("a struct", "a struct"), ("a struct", "a struct"), ("a map", "m map"), ("a array", "a array"), ("a map, int>", "a map, int>"), ("a array>", "a array>") ).map { case (oldStr, newStr) => StructType.fromDDL(oldStr) -> StructType.fromDDL(newStr) } } { withClue(s"oldSchema = $oldSchema, newSchema = $newSchema") { val schema = TypeWideningMetadata.addTypeWideningMetadata(txn, newSchema, oldSchema) assert(schema === newSchema) assert(TypeWideningMetadata.removeTypeWideningMetadata(schema) === schema -> Seq.empty) } } } test("addTypeWideningMetadata/removeTypeWideningMetadata on top-level fields") { val schemaWithoutMetadata = StructType.fromDDL("i int, a array, m map") val firstOldSchema = StructType.fromDDL("i byte, a array, m map") val secondOldSchema = StructType.fromDDL("i short, a array, m map") var schema = TypeWideningMetadata.addTypeWideningMetadata(txn, schemaWithoutMetadata, firstOldSchema) assert(schema("i") === StructField("i", IntegerType, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("byte", "integer") )).build() )) assert(schema("a") === StructField("a", ArrayType(IntegerType), metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("byte", "integer", "element") )).build() )) assert(schema("m") === StructField("m", MapType(ShortType, IntegerType), metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("byte", "short", "key") )).build() )) assert(TypeWideningMetadata.removeTypeWideningMetadata(schema) === schemaWithoutMetadata -> Seq( Seq.empty -> schema("i"), Seq.empty -> schema("a"), Seq.empty -> schema("m") )) // Second type change on all fields. schema = TypeWideningMetadata.addTypeWideningMetadata(txn, schema, secondOldSchema) assert(schema("i") === StructField("i", IntegerType, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("byte", "integer"), typeChangeMetadata("short", "integer") )).build() )) assert(schema("a") === StructField("a", ArrayType(IntegerType), metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("byte", "integer", "element"), typeChangeMetadata("short", "integer", "element") )).build() )) assert(schema("m") === StructField("m", MapType(ShortType, IntegerType), metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("byte", "short", "key"), typeChangeMetadata("byte", "integer", "value") )).build() )) assert(TypeWideningMetadata.removeTypeWideningMetadata(schema) === schemaWithoutMetadata -> Seq( Seq.empty -> schema("i"), Seq.empty -> schema("a"), Seq.empty -> schema("m") )) } test("addTypeWideningMetadata/removeTypeWideningMetadata on nested fields") { val schemaWithoutMetadata = StructType.fromDDL( "s struct>, m: map, array>>") val firstOldSchema = StructType.fromDDL( "s struct>, m: map, array>>") val secondOldSchema = StructType.fromDDL( "s struct>, m: map, array>>") // First type change on all struct fields. var schema = TypeWideningMetadata.addTypeWideningMetadata(txn, schemaWithoutMetadata, firstOldSchema) var struct = schema("s").dataType.asInstanceOf[StructType] assert(struct("i") === StructField("i", IntegerType, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("byte", "integer") )).build() )) assert(struct("a") === StructField("a", ArrayType(MapType(IntegerType, IntegerType)), metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("byte", "integer", "element.key") )).build() )) assert(struct("m") === StructField("m", MapType(MapType(IntegerType, IntegerType), ArrayType(IntegerType)), metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("short", "integer", "key.key") )).build() )) assert(TypeWideningMetadata.removeTypeWideningMetadata(schema) === schemaWithoutMetadata -> Seq( Seq("s") -> struct("i"), Seq("s") -> struct("a"), Seq("s") -> struct("m") )) // Second type change on all struct fields. schema = TypeWideningMetadata.addTypeWideningMetadata(txn, schema, secondOldSchema) struct = schema("s").dataType.asInstanceOf[StructType] assert(struct("i") === StructField("i", IntegerType, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("byte", "integer"), typeChangeMetadata("short", "integer") )).build() )) assert(struct("a") === StructField("a", ArrayType(MapType(IntegerType, IntegerType)), metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("byte", "integer", "element.key"), typeChangeMetadata("short", "integer", "element.value") )).build() )) assert(struct("m") === StructField("m", MapType(MapType(IntegerType, IntegerType), ArrayType(IntegerType)), metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("short", "integer", "key.key"), typeChangeMetadata("short", "integer", "value.element") )).build() )) assert(TypeWideningMetadata.removeTypeWideningMetadata(schema) === schemaWithoutMetadata -> Seq( Seq("s") -> struct("i"), Seq("s") -> struct("a"), Seq("s") -> struct("m") )) } test("addTypeWideningMetadata/removeTypeWideningMetadata with added and removed fields") { val newSchema = StructType.fromDDL("a int, b int, d int") val oldSchema = StructType.fromDDL("a int, b short, c int") val schema = TypeWideningMetadata.addTypeWideningMetadata(txn, newSchema, oldSchema) assert(schema("a") === StructField("a", IntegerType)) assert(schema("d") === StructField("d", IntegerType)) assert(!schema.contains("c")) assert(schema("b") === StructField("b", IntegerType, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("short", "integer") )).build() )) assert(TypeWideningMetadata.removeTypeWideningMetadata(schema) === newSchema -> Seq(Seq.empty -> schema("b")) ) } test("addTypeWideningMetadata/removeTypeWideningMetadata with different field position") { val newSchema = StructType.fromDDL("a short, b int, s struct") val oldSchema = StructType.fromDDL("b int, a short, s struct") val schema = TypeWideningMetadata.addTypeWideningMetadata(txn, newSchema, oldSchema) // No type widening metadata is added. assert(schema("a") === StructField("a", ShortType)) assert(schema("b") === StructField("b", IntegerType)) assert(schema("s") === StructField("s", new StructType() .add("c", IntegerType) .add("d", LongType))) assert(TypeWideningMetadata.removeTypeWideningMetadata(schema) === newSchema -> Seq.empty) } test("addTypeWideningMetadata/removeTypeWideningMetadata with preview feature") { val newSchema = StructType.fromDDL("a short") val oldSchema = StructType.fromDDL("a byte") // Create a new transaction with the preview feature supported. val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier(testTableName)) val txn = DeltaLog.forTable(spark, TableIdentifier(testTableName)) .startTransaction(catalogTableOpt = Some(table)) txn.updateProtocol(txn.protocol.withFeature(TypeWideningPreviewTableFeature)) val schema = TypeWideningMetadata.addTypeWideningMetadata(txn, newSchema, oldSchema) // Type widening metadata is added with field `tableVersion` populated as this uses the preview // feature. That field is deprecated in the stable version of the feature. assert(schema("a") === StructField("a", ShortType, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( new MetadataBuilder() .putLong("tableVersion", 1) .putString("fromType", "byte") .putString("toType", "short") .build() )).build() )) assert(TypeWideningMetadata.removeTypeWideningMetadata(schema) === newSchema -> Seq(Seq.empty -> schema("a"))) } test("updateTypeChangeVersion with no type changes") { val schema = new StructType().add("a", IntegerType) assert(TypeWideningMetadata.updateTypeChangeVersion(schema, 1, 4) === schema) } test("updateTypeChangeVersion with field with single type change") { val schema = new StructType() .add("a", IntegerType, nullable = true, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("integer", "long") )) .build() ) assert(TypeWideningMetadata.updateTypeChangeVersion(schema, 1, 4) === new StructType() .add("a", IntegerType, nullable = true, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("integer", "long") )) .build() ) ) } test("updateTypeChangeVersion with field with multiple type changes") { val schema = new StructType() .add("a", IntegerType, nullable = true, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("integer", "long"), typeChangeMetadata("float", "double", "value") )) .build() ) // Update matching one of the type changes. assert(TypeWideningMetadata.updateTypeChangeVersion(schema, 1, 4) === new StructType() .add("a", IntegerType, nullable = true, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("integer", "long"), typeChangeMetadata("float", "double", "value") )) .build() ) ) // Update doesn't match any of the recorded type changes. assert( TypeWideningMetadata.updateTypeChangeVersion(schema, 3, 4) === schema ) } test("updateTypeChangeVersion with multiple fields with a type change") { val schema = new StructType() .add("a", IntegerType, nullable = true, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("integer", "long") )) .build()) .add("b", ArrayType(IntegerType), nullable = true, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("short", "integer", "element") )) .build()) // Update both type changes. assert(TypeWideningMetadata.updateTypeChangeVersion(schema, 1, 4) === new StructType() .add("a", IntegerType, nullable = true, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("integer", "long") )) .build()) .add("b", ArrayType(IntegerType), nullable = true, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( typeChangeMetadata("short", "integer", "element") )) .build()) ) // Update doesn't match any of the recorded type changes. assert( TypeWideningMetadata.updateTypeChangeVersion(schema, 3, 4) === schema ) } } /** * Tests that covers recording type change information as metadata in the table schema. For * lower-level tests, see [[TypeWideningMetadataTests]]. */ trait TypeWideningMetadataEndToEndTests { self: QueryTest with TypeWideningTestMixin => def testTypeWideningMetadata(name: String)( initialSchema: String, typeChanges: Seq[(String, String)], expectedJsonSchema: String): Unit = test(name) { sql(s"CREATE TABLE delta.`$tempPath` ($initialSchema) USING DELTA") typeChanges.foreach { case (fieldName, newType) => sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN $fieldName TYPE $newType") } // Parse the schemas as JSON to ignore whitespaces and field order. val actualSchema = JsonUtils.fromJson[Map[String, Any]](readDeltaTable(tempPath).schema.json) val expectedSchema = JsonUtils.fromJson[Map[String, Any]](expectedJsonSchema) assert(actualSchema === expectedSchema, s"${readDeltaTable(tempPath).schema.prettyJson} did not equal $expectedJsonSchema" ) } testTypeWideningMetadata("change top-level column type short->int")( initialSchema = "a short", typeChanges = Seq("a" -> "int"), expectedJsonSchema = """{ "type": "struct", "fields": [{ "name": "a", "type": "integer", "nullable": true, "metadata": {} }]}""".stripMargin) testTypeWideningMetadata("change top-level column type twice byte->short->int")( initialSchema = "a byte", typeChanges = Seq("a" -> "short", "a" -> "int"), expectedJsonSchema = """{ "type": "struct", "fields": [{ "name": "a", "type": "integer", "nullable": true, "metadata": {} }]}""".stripMargin) testTypeWideningMetadata("change type in map key and in struct in map value")( initialSchema = "a map>", typeChanges = Seq("a.key" -> "int", "a.value.b" -> "short"), expectedJsonSchema = """{ "type": "struct", "fields": [{ "name": "a", "type": { "type": "map", "keyType": "integer", "valueType": { "type": "struct", "fields": [{ "name": "b", "type": "short", "nullable": true, "metadata": {} }] }, "valueContainsNull": true }, "nullable": true, "metadata": {} } ]}""".stripMargin) testTypeWideningMetadata("change type in array and in struct in array")( initialSchema = "a array, b array>", typeChanges = Seq("a.element" -> "short", "b.element.c" -> "int"), expectedJsonSchema = """{ "type": "struct", "fields": [{ "name": "a", "type": { "type": "array", "elementType": "short", "containsNull": true }, "nullable": true, "metadata": {} }, { "name": "b", "type": { "type": "array", "elementType":{ "type": "struct", "fields": [{ "name": "c", "type": "integer", "nullable": true, "metadata": {} }] }, "containsNull": true }, "nullable": true, "metadata": { } } ]}""".stripMargin) } trait TypeWideningLeakingMetadataTests { self: QueryTest with TypeWideningTestMixin => test("stream read from type widening does not leak metadata") { val (t1, t2) = ("type_widening_table_1", "type_widening_table_2") withTable(t1, t2) { withTempDir { dir => sql(s"CREATE TABLE $t1 (part BYTE, value SHORT) USING DELTA PARTITIONED BY (part)") sql(s"INSERT INTO $t1 VALUES (1, 1), (2, 2)") // Change type of both partition and non-partition columns. sql(s"ALTER TABLE $t1 CHANGE COLUMN part TYPE INT") sql(s"ALTER TABLE $t1 CHANGE COLUMN value TYPE INT") // Stream read from source table val streamDf = spark.readStream.format("delta").table(t1) // Should not contain type widening metadata assert(streamDf.schema.forall(_.metadata.json == "{}")) // Create and write to another table val q = streamDf.writeStream .partitionBy("part") .trigger(org.apache.spark.sql.streaming.Trigger.AvailableNow()) .format("delta") .option("checkpointLocation", new File(dir, "_checkpoint1").getCanonicalPath) .toTable(t2) q.awaitTermination() // Check target table Delta log val deltaLog = DeltaLog.forTable(spark, TableIdentifier(t2)) assert(deltaLog.update().metadata.schema.forall(_.metadata.json == "{}")) // Check target table data checkAnswer(spark.table(t2), Seq(Row(1, 1), Row(2, 2))) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningStatsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.typewidening import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Expression, Literal} import org.apache.spark.sql.catalyst.plans.logical.LocalRelation import org.apache.spark.sql.types._ /** * Suite covering stats and data skipping with type changes. */ class TypeWideningStatsSuite extends QueryTest with TypeWideningTestMixin with TypeWideningStatsTests trait TypeWideningStatsTests { self: QueryTest with TypeWideningTestMixin => import testImplicits._ /** * Helper to create a table and run tests while enabling/disabling storing stats as JSON string or * strongly-typed structs in checkpoint files. Creates a */ def testStats( name: String, partitioned: Boolean, jsonStatsEnabled: Boolean, structStatsEnabled: Boolean)( body: => Unit): Unit = test(s"$name, partitioned=$partitioned, jsonStatsEnabled=$jsonStatsEnabled, " + s"structStatsEnabled=$structStatsEnabled") { withSQLConf( DeltaConfigs.CHECKPOINT_WRITE_STATS_AS_JSON.defaultTablePropertyKey -> jsonStatsEnabled.toString, DeltaConfigs.CHECKPOINT_WRITE_STATS_AS_STRUCT.defaultTablePropertyKey -> structStatsEnabled.toString ) { val partitionStr = if (partitioned) "PARTITIONED BY (a)" else "" sql(s""" |CREATE TABLE delta.`$tempPath` (a smallint, dummy int DEFAULT 1) |USING DELTA |$partitionStr |TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported') """.stripMargin) body } } /** Returns the latest checkpoint for the test table. */ def getLatestCheckpoint: LastCheckpointInfo = deltaLog.readLastCheckpointFile().getOrElse { fail("Expected the table to have a checkpoint but it didn't") } /** Returns the type used to store JSON stats in the checkpoint if JSON stats are present. */ def getJsonStatsType(checkpoint: LastCheckpointInfo): Option[DataType] = checkpoint.checkpointSchema.flatMap { _.findNestedField(Seq("add", "stats")) }.map(_._2.dataType) /** * Returns the type used to store parsed partition values for the given column in the checkpoint * if these are present. */ def getPartitionValuesType(checkpoint: LastCheckpointInfo, colName: String) : Option[DataType] = { checkpoint.checkpointSchema.flatMap { _.findNestedField(Seq("add", "partitionValues_parsed", colName)) }.map(_._2.dataType) } /** * Returns the type used to store parsed stat values for the given column in the checkpoint if * these are present. */ def getStructStatsType(checkpoint: LastCheckpointInfo, colName: String) : Option[DataType] = { checkpoint.checkpointSchema.flatMap { _.findNestedField(Seq("add", "stats_parsed", "minValues", colName)) }.map(_._2.dataType) } /** * Checks that stats and parsed partition values are stored in the checkpoint when enabled and * that their type matches the expected type. */ def checkCheckpointStats( checkpoint: LastCheckpointInfo, colName: String, colType: DataType, partitioned: Boolean, jsonStatsEnabled: Boolean, structStatsEnabled: Boolean): Unit = { val expectedJsonStatsType = if (jsonStatsEnabled) Some(StringType) else None assert(getJsonStatsType(checkpoint) === expectedJsonStatsType) val expectedPartitionStats = if (partitioned && structStatsEnabled) Some(colType) else None assert(getPartitionValuesType(checkpoint, colName) === expectedPartitionStats) val expectedStructStats = if (!partitioned && structStatsEnabled) Some(colType) else None assert(getStructStatsType(checkpoint, colName) === expectedStructStats) } /** * Reads the test table filtered by the given value and checks that files are skipped as expected. */ def checkFileSkipping(filterValue: Any, expectedFilesRead: Long): Unit = { val dataFilter: Expression = EqualTo(AttributeReference("a", IntegerType)(), Literal(filterValue)) val files = deltaLog.update().filesForScan(Seq(dataFilter), keepNumRecords = false).files assert(files.size === expectedFilesRead, s"Expected $expectedFilesRead files to be " + s"read but read ${files.size} files.") } for { partitioned <- BOOLEAN_DOMAIN jsonStatsEnabled <- BOOLEAN_DOMAIN structStatsEnabled <- BOOLEAN_DOMAIN } testStats(s"data skipping after type change", partitioned, jsonStatsEnabled, structStatsEnabled) { addSingleFile(Seq(1), ShortType) addSingleFile(Seq(2), ShortType) deltaLog.checkpoint() assert(readDeltaTable(tempPath).schema("a").dataType === ShortType) val initialCheckpoint = getLatestCheckpoint checkCheckpointStats( initialCheckpoint, "a", ShortType, partitioned, jsonStatsEnabled, structStatsEnabled) sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE INT") addSingleFile(Seq(Int.MinValue), IntegerType) var checkpoint = getLatestCheckpoint // Ensure there's no new checkpoint after the type change. assert(getLatestCheckpoint.semanticEquals(initialCheckpoint)) // Struct stats can be used as fallback for non-partition values when json stats are disabled. val canSkipFiles = jsonStatsEnabled || partitioned || structStatsEnabled // The last file added isn't part of the checkpoint, it always has stats that can be used for // skipping even when checkpoint stats can't be used for skipping. checkFileSkipping(filterValue = 1, expectedFilesRead = if (canSkipFiles) 1 else 2) checkAnswer(readDeltaTable(tempPath).filter("a = 1"), Row(1, 1)) checkFileSkipping(filterValue = Int.MinValue, expectedFilesRead = if (canSkipFiles) 1 else 3) checkAnswer(readDeltaTable(tempPath).filter(s"a = ${Int.MinValue}"), Row(Int.MinValue, 1)) // Trigger a new checkpoint after the type change and re-check data skipping. deltaLog.checkpoint() checkpoint = getLatestCheckpoint assert(!checkpoint.semanticEquals(initialCheckpoint)) checkCheckpointStats( checkpoint, "a", IntegerType, partitioned, jsonStatsEnabled, structStatsEnabled) // When checkpoint stats are completely disabled, the last file added can't be skipped anymore. checkFileSkipping(filterValue = 1, expectedFilesRead = if (canSkipFiles) 1 else 3) checkFileSkipping(filterValue = Int.MinValue, expectedFilesRead = if (canSkipFiles) 1 else 3) } for { partitioned <- BOOLEAN_DOMAIN jsonStatsEnabled <- BOOLEAN_DOMAIN structStatsEnabled <- BOOLEAN_DOMAIN } testStats(s"metadata-only query", partitioned, jsonStatsEnabled, structStatsEnabled) { addSingleFile(Seq(1), ShortType) addSingleFile(Seq(2), ShortType) sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE INT") addSingleFile(Seq(Int.MinValue), IntegerType) addSingleFile(Seq(Int.MaxValue), IntegerType) // Check that collecting aggregates using a metadata-only query works after the type change. val resultDf = sql(s"SELECT MIN(a), MAX(a), COUNT(*) FROM delta.`$tempPath`") val isMetadataOnly = resultDf.queryExecution.optimizedPlan.collectFirst { case l: LocalRelation => l }.nonEmpty assert(isMetadataOnly, "Expected the query to be metadata-only") checkAnswer(resultDf, Row(Int.MinValue, Int.MaxValue, 4)) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningStreamingSinkSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.typewidening import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.Relocated.StreamExecution import org.apache.spark.sql.delta.sources.{DeltaSink, DeltaSQLConf} import org.apache.spark.sql.Row import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.streaming.OutputMode import org.apache.spark.sql.types._ /** * Suite covering automatic type widening in the Delta streaming sink. */ class TypeWideningStreamingSinkSuite extends DeltaSinkImplicitCastSuiteBase with TypeWideningTestMixin { import testImplicits._ override def beforeAll(): Unit = { super.beforeAll() // Set by default confs to enable automatic type widening in all tests. Negative tests should // explicitly disable these. spark.conf.set(DeltaSQLConf.DELTA_STREAMING_SINK_ALLOW_IMPLICIT_CASTS.key, "true") spark.conf.set(DeltaConfigs.ENABLE_TYPE_WIDENING.defaultTablePropertyKey, "true") spark.conf.set(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key, "true") // Ensure we don't silently cast test inputs to null on overflow. spark.conf.set(SQLConf.ANSI_ENABLED.key, "true") } test("type is widened if automatic widening set to always") { withDeltaStream[Int] { stream => stream.write(17)("CAST(value AS SHORT)") assert(stream.currentSchema("value").dataType === ShortType) checkAnswer(stream.read(), Row(17)) withSQLConf(DeltaSQLConf.DELTA_ALLOW_AUTOMATIC_WIDENING.key -> "always") { stream.write(2)("CAST(value AS DOUBLE)") assert(stream.currentSchema("value").dataType === DoubleType) checkAnswer(stream.read(), Row(17.0) :: Row(2.0) :: Nil) } } } test("type isn't widened if automatic widening set to never") { withDeltaStream[Int] { stream => stream.write(17)("CAST(value AS SHORT)") assert(stream.currentSchema("value").dataType === ShortType) checkAnswer(stream.read(), Row(17)) withSQLConf(DeltaSQLConf.DELTA_ALLOW_AUTOMATIC_WIDENING.key -> "never") { stream.write(100)("CAST(value AS INT)") assert(stream.currentSchema("value").dataType === ShortType) checkAnswer(stream.read(), Row(17) :: Row(100) :: Nil) } } } test("type isn't widened if schema evolution is disabled") { withDeltaStream[Int] { stream => stream.write(17)("CAST(value AS SHORT)") assert(stream.currentSchema("value").dataType === ShortType) checkAnswer(stream.read(), Row(17)) withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "false") { stream.write(53)("CAST(value AS INT)") assert(stream.currentSchema("value").dataType === ShortType) checkAnswer(stream.read(), Row(17) :: Row(53) :: Nil) } } } test("type isn't widened if type widening is disabled") { withDeltaStream[Int] { stream => withSQLConf(DeltaConfigs.ENABLE_TYPE_WIDENING.defaultTablePropertyKey -> "false") { stream.write(17)("CAST(value AS SHORT)") assert(stream.currentSchema("value").dataType === ShortType) checkAnswer(stream.read(), Row(17)) stream.write(53)("CAST(value AS INT)") assert(stream.currentSchema("value").dataType === ShortType) checkAnswer(stream.read(), Row(17) :: Row(53) :: Nil) } } } test("type is widened if type widening and schema evolution are enabled") { withDeltaStream[Int] { stream => stream.write(17)("CAST(value AS SHORT)") assert(stream.currentSchema("value").dataType === ShortType) checkAnswer(stream.read(), Row(17)) stream.write(Int.MaxValue)("CAST(value AS INT)") assert(stream.currentSchema("value").dataType === IntegerType) checkAnswer(stream.read(), Row(17) :: Row(Int.MaxValue) :: Nil) } } test("type can be widened even if type casting is disabled in the sink") { withDeltaStream[Int] { stream => stream.write(17)("CAST(value AS SHORT)") assert(stream.currentSchema("value").dataType === ShortType) checkAnswer(stream.read(), Row(17)) withSQLConf(DeltaSQLConf.DELTA_STREAMING_SINK_ALLOW_IMPLICIT_CASTS.key -> "false") { stream.write(Int.MaxValue)("CAST(value AS INT)") assert(stream.currentSchema("value").dataType === IntegerType) checkAnswer(stream.read(), Row(17) :: Row(Int.MaxValue) :: Nil) } } } test("type isn't changed if it's not a wider type") { withDeltaStream[Int] { stream => stream.write(Int.MaxValue)("CAST(value AS INT)") assert(stream.currentSchema("value").dataType === IntegerType) checkAnswer(stream.read(), Row(Int.MaxValue)) stream.write(17)("CAST(value AS SHORT)") assert(stream.currentSchema("value").dataType === IntegerType) checkAnswer(stream.read(), Row(Int.MaxValue) :: Row(17) :: Nil) } } test("type isn't changed if it's not eligible for automatic widening: int -> decimal") { withSQLConf(DeltaSQLConf.DELTA_ALLOW_AUTOMATIC_WIDENING.key -> "same_family_type") { withDeltaStream[Int] { stream => stream.write(17)("CAST(value AS INT)") assert(stream.currentSchema("value").dataType === IntegerType) checkAnswer(stream.read(), Row(17)) stream.write(567)("CAST(value AS DECIMAL(20, 0))") assert(stream.currentSchema("value").dataType === IntegerType) checkAnswer(stream.read(), Row(17) :: Row(567) :: Nil) } } } test("type isn't changed if it's not eligible for automatic widening: int -> double") { withSQLConf(DeltaSQLConf.DELTA_ALLOW_AUTOMATIC_WIDENING.key -> "same_family_type") { withDeltaStream[Int] { stream => stream.write(17)("CAST(value AS INT)") assert(stream.currentSchema("value").dataType === IntegerType) checkAnswer(stream.read(), Row(17)) stream.write(567)("CAST(value AS DOUBLE)") assert(stream.currentSchema("value").dataType === IntegerType) checkAnswer(stream.read(), Row(17) :: Row(567) :: Nil) } } } test("widen type and add a new column with schema evolution") { withDeltaStream[(Int, Int)] { stream => stream.write((17, -1))("CAST(_1 AS SHORT) AS a") assert(stream.currentSchema === new StructType().add("a", ShortType)) checkAnswer(stream.read(), Row(17)) stream.write((12, 3456))("CAST(_1 AS INT) AS a", "CAST(_2 AS DECIMAL(10, 2)) AS b") assert(stream.currentSchema === new StructType() .add("a", IntegerType) .add("b", DecimalType(10, 2))) checkAnswer(stream.read(), Row(17, null) :: Row(12, 3456) :: Nil) } } test("widen type during write with missing column") { withDeltaStream[(Int, Int)] { stream => stream.write((17, 45))("CAST(_1 AS SHORT) AS a", "CAST(_2 AS LONG) AS b") assert(stream.currentSchema === new StructType() .add("a", ShortType) .add("b", LongType)) checkAnswer(stream.read(), Row(17, 45)) stream.write((12, -1))("CAST(_1 AS INT) AS a") assert(stream.currentSchema === new StructType() .add("a", IntegerType) .add("b", LongType)) checkAnswer(stream.read(), Row(17, 45) :: Row(12, null) :: Nil) } } test("widen type after column rename and drop") { withDeltaStream[(Int, Int)] { stream => stream.write((17, 45))("CAST(_1 AS SHORT) AS a", "CAST(_2 AS DECIMAL(10, 0)) AS b") assert(stream.currentSchema === new StructType() .add("a", ShortType) .add("b", DecimalType(10, 0))) checkAnswer(stream.read(), Row(17, 45)) sql( s""" |ALTER TABLE delta.`${stream.deltaLog.dataPath}` SET TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name', | 'delta.minReaderVersion' = '2', | 'delta.minWriterVersion' = '5' |) """.stripMargin) sql(s"ALTER TABLE delta.`${stream.deltaLog.dataPath}` DROP COLUMN b") sql(s"ALTER TABLE delta.`${stream.deltaLog.dataPath}` RENAME COLUMN a to c") assert(stream.currentSchema === new StructType().add("c", ShortType)) stream.write((12, -1))("CAST(_1 AS INT) AS c") assert(stream.currentSchema === new StructType().add("c", IntegerType)) checkAnswer(stream.read(), Row(17) :: Row(12) :: Nil) } } test("type widening in addBatch") { withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath val deltaLog = DeltaLog.forTable(spark, tablePath) sqlContext.sparkContext.setLocalProperty(StreamExecution.QUERY_ID_KEY, "streaming_query") val sink = DeltaSink( sqlContext, path = deltaLog.dataPath, partitionColumns = Seq.empty, outputMode = OutputMode.Append(), options = new DeltaOptions(options = Map.empty, conf = spark.sessionState.conf) ) val schema = new StructType().add("value", ShortType) { val data = Seq(0, 1).toDF("value").selectExpr("CAST(value AS SHORT)") sink.addBatch(0, data) val df = spark.read.format("delta").load(tablePath) assert(df.schema === schema) checkAnswer(df, Row(0) :: Row(1) :: Nil) } { val data = Seq(2, 3).toDF("value").selectExpr("CAST(value AS INT)") sink.addBatch(1, data) val df = spark.read.format("delta").load(tablePath) assert(df.schema === new StructType().add("value", IntegerType)) checkAnswer(df, Row(0) :: Row(1) :: Row(2) :: Row(3) :: Nil) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningStreamingSourceSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.typewidening import java.io.File import com.databricks.spark.util.{Log4jUsageLogger, MetricDefinitions} import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.sql.util.ScalaExtensions._ import org.scalatest.BeforeAndAfterAll import org.apache.spark.{SparkException, SparkThrowable} import org.apache.spark.SparkArithmeticException import org.apache.spark.sql.{DataFrame, Row, SaveMode} import org.apache.spark.sql.functions.{col, count, lit} import org.apache.spark.sql.streaming._ import org.apache.spark.sql.test.SQLTestUtils import org.apache.spark.sql.types._ /** * Suite covering streaming reads from a Delta table that had a column type widened, **without** * schema tracking, i.e. widening type changes don't require users to set a SQL conf to proceed. */ class TypeWideningStreamingSourceSuite extends TypeWideningStreamingSourceTests with TypeWideningStreamingSourceWithoutSchemaTrackingTests { override protected val schemaTrackingEnabled: Boolean = false // Changes are not blocked with schema tracking disabled, this is a no-op. override protected def withUnblockedTypeChanges(fn: => Unit): Unit = fn } /** * Suite covering streaming reads from a Delta table that had a column type widened, **with** * schema tracking, i.e. users must manually acknowledge type changes by setting a SQL conf for * the stream to resume processing. */ class TypeWideningStreamingSourceSchemaTrackingSuite extends TypeWideningStreamingSourceTests with TypeWideningStreamingSourceSchemaTrackingTests { override protected val schemaTrackingEnabled: Boolean = true override protected def withUnblockedTypeChanges(fn: => Unit): Unit = withSQLConf("spark.databricks.delta.streaming.allowSourceColumnTypeChange" -> "always")(fn) } trait TypeWideningStreamingSourceTestMixin extends TypeWideningTestMixin with BeforeAndAfterAll { self: StreamTest => /** Whether the suite uses schema tracking to handle widening type changes. */ protected val schemaTrackingEnabled: Boolean /** Unblocks the stream after a widening type change. */ protected def withUnblockedTypeChanges(fn: => Unit): Unit override def beforeAll(): Unit = { super.beforeAll() spark.sessionState.conf.setConf( DeltaSQLConf.DELTA_TYPE_WIDENING_ENABLE_STREAMING_SCHEMA_TRACKING, schemaTrackingEnabled) spark.udf.register("scala_udf", (x: Int) => x + 1) } override def afterAll(): Unit = { // The scala UDF is a temporary function, no need to drop it. super.afterAll() } /** Short-hand to read a data stream from the Delta table at the given location. */ protected def readStream( path: File, checkpointDir: File, options: Map[String, String] = Map.empty): DataFrame = { val allOptions = options ++ Option.when(schemaTrackingEnabled)( DeltaOptions.SCHEMA_TRACKING_LOCATION -> checkpointDir.toString ) spark.readStream.format("delta") .options(allOptions) .load(path.getCanonicalPath) } /** Test action checking that the stream fails due to a metadata change - typ. a schema change. */ object ExpectMetadataEvolutionException { def apply(): StreamAction = if (schemaTrackingEnabled) { ExpectFailure[DeltaRuntimeException] { ex => assert(ex.asInstanceOf[DeltaRuntimeException].getErrorClass === "DELTA_STREAMING_METADATA_EVOLUTION") } } else { ExpectFailure[DeltaIllegalStateException] { ex => assert(ex.asInstanceOf[SparkThrowable].getErrorClass === "DELTA_SCHEMA_CHANGED_WITH_VERSION") } } } /** Test action checking that the stream fails due to a type change being blocked. */ object ExpectTypeChangeBlockedException { def apply(): StreamAction = ExpectFailure[DeltaRuntimeException] { ex => assert(ex.asInstanceOf[DeltaRuntimeException].getErrorClass === "DELTA_STREAMING_CANNOT_CONTINUE_PROCESSING_POST_SCHEMA_EVOLUTION") assert(ex.asInstanceOf[DeltaRuntimeException].getMessage.contains("TYPE WIDENING")) } } /** Test action checking that the stream fails due to an unsupported type change. */ object ExpectIncompatibleSchemaChangeException { def apply(): StreamAction = ExpectFailure[DeltaIllegalStateException] { ex => assert(ex.asInstanceOf[DeltaIllegalStateException].getErrorClass === "DELTA_SCHEMA_CHANGED_WITH_VERSION") } } } /** * Common tests for type widening when streaming from a Delta source. * Can run both with and without schema tracking. */ trait TypeWideningStreamingSourceTests extends StreamTest with SQLTestUtils with TypeWideningStreamingSourceTestMixin { import testImplicits._ /** * Test a streaming query with a type widening operation. Creates a Delta source with two columns * `widened` and `other` of type `byte` and widens the `widened` column to `int`. The query under * test is used to read from the table and checked against the expected result. * @param name Test name. * @param query Streaming query to apply on the source. * @param expectedResult In case of success, checks the last batch of data written by the stream * matches the expected result. In case of failure, the caller provides a * check to perform on the exception. * @param outputMode Output mode of the streaming query. `Append` by default but can be * overriden to e.g. `Complete` for aggregations. */ private def testStreamTypeWidening( name: String, query: DataFrame => DataFrame, partitionBy: Option[String] = None, expectedResult: ExpectedResult[Seq[Row]], outputMode: OutputMode = OutputMode.Append()): Unit = { test(s"type change - $name") { withTempDir { dir => val partitionByStr = partitionBy.map(p => s"PARTITIONED BY ($p)").getOrElse("") sql(s"CREATE TABLE delta.`$dir` (widened byte, other byte) USING DELTA $partitionByStr") val checkpointDir = new File(dir, "sink_checkpoint") testStream(query(readStream(dir, checkpointDir)), outputMode)( StartStream(checkpointLocation = checkpointDir.toString), Execute { _ => sql(s"INSERT INTO delta.`$dir` VALUES (1, 1)") }, ProcessAllAvailable(), Execute { _ => sql(s"ALTER TABLE delta.`$dir`ALTER COLUMN widened TYPE int") }, ExpectMetadataEvolutionException() ) val streamActions = expectedResult match { case ExpectedResult.Success(rows: Seq[Row @unchecked]) => Seq( Execute { _ => sql(s"INSERT INTO delta.`$dir` VALUES (123456789, 2)") }, ProcessAllAvailable(), CheckLastBatch(rows: _*) ) case ExpectedResult.Failure(checkError) => Seq(AssertOnQuery { q => val ex = intercept[StreamingQueryException] { q.processAllAvailable() } val cause = if (ex.getCause.getMessage.contains( "Provided schema doesn't match to the schema for existing state!")) { // State store schema mismatches were non-spark exception until Spark 3.5. We wrap // them into a spark exception to be able to check them consistently across spark // versions. new SparkException( message = ex.getCause.getMessage, cause = ex, errorClass = Some("STATE_STORE_KEY_SCHEMA_NOT_COMPATIBLE"), messageParameters = Map.empty ) } else { assert(ex.getCause.isInstanceOf[SparkThrowable]) ex.getCause.asInstanceOf[SparkThrowable] } checkError(cause) true }) } // We need to unblock the type change to let the stream make progress. withUnblockedTypeChanges { testStream(query(readStream(dir, checkpointDir)), outputMode)( StartStream(checkpointLocation = checkpointDir.toString) +: streamActions: _* ) } } } } testStreamTypeWidening("filter", query = _.where(col("widened") > 10), expectedResult = ExpectedResult.Success(Seq(Row(123456789, 2))) ) testStreamTypeWidening("projection", query = _.withColumn("add", col("widened") + col("other")), expectedResult = ExpectedResult.Success(Seq(Row(123456789, 2, 123456791))) ) testStreamTypeWidening("projection partition column", query = _.withColumn("add", col("widened") + col("other")), partitionBy = Some("widened"), expectedResult = ExpectedResult.Success(Seq(Row(123456789, 2, 123456791))) ) testStreamTypeWidening("widen unused scala udf field", query = _.selectExpr("scala_udf(other)"), expectedResult = ExpectedResult.Success(Seq(Row(3))) ) testStreamTypeWidening("widen scala udf argument", query = _.selectExpr("scala_udf(widened)"), expectedResult = ExpectedResult.Success(Seq(Row(123456790))) ) testStreamTypeWidening("widen aggregation grouping key", query = _.groupBy("widened").agg(count(col("other"))), expectedResult = ExpectedResult.Failure { ex => assert(ex.getErrorClass === "STATE_STORE_KEY_SCHEMA_NOT_COMPATIBLE") }, outputMode = OutputMode.Complete() ) testStreamTypeWidening("widen aggregation expression", query = _.groupBy("other").agg(count(col("widened"))), expectedResult = ExpectedResult.Success(Seq(Row(1, 1), Row(2, 1))), outputMode = OutputMode.Complete() ) testStreamTypeWidening("widen aggregation expression partition column", query = _.groupBy("other").agg(count(col("widened"))), partitionBy = Some("widened"), expectedResult = ExpectedResult.Success(Seq(Row(1, 1), Row(2, 1))), outputMode = OutputMode.Complete() ) testStreamTypeWidening("widen aggregation expression after projection", query = _.groupBy(col("widened") + lit(1).cast(ByteType)).agg(count(col("other"))), expectedResult = ExpectedResult.Failure { ex => assert(ex.getErrorClass === "STATE_STORE_KEY_SCHEMA_NOT_COMPATIBLE") }, outputMode = OutputMode.Complete() ) testStreamTypeWidening("widen limit", query = _.select("widened").limit(1), expectedResult = ExpectedResult.Success(Seq.empty) ) testStreamTypeWidening("widen distinct", query = _.select("widened").distinct(), expectedResult = ExpectedResult.Failure { ex => assert(ex.getErrorClass === "STATE_STORE_KEY_SCHEMA_NOT_COMPATIBLE") } ) testStreamTypeWidening("widen drop duplicates", query = _.select("widened").dropDuplicates(), expectedResult = ExpectedResult.Failure { ex => assert(ex.getErrorClass === "STATE_STORE_KEY_SCHEMA_NOT_COMPATIBLE") }, outputMode = OutputMode.Update() ) testStreamTypeWidening("widen drop duplicates with watermark", query = _.select("widened") .withColumn("watermark", lit("2025-02-04").cast("timestamp")) .withWatermark("watermark", "0 seconds") .dropDuplicatesWithinWatermark(), expectedResult = ExpectedResult.Failure { ex => assert(ex.getErrorClass === "STATE_STORE_KEY_SCHEMA_NOT_COMPATIBLE") }, outputMode = OutputMode.Update() ) testStreamTypeWidening("widen flatMap groups with state", query = _.select("widened").as[Int] .groupByKey(x => x) .flatMapGroupsWithState( outputMode = OutputMode.Update, timeoutConf = GroupStateTimeout.NoTimeout )((key: Int, values: Iterator[Int], state: GroupState[Int]) => { Iterator(values.max) }) .toDF(), expectedResult = ExpectedResult.Success(Seq(Row(123456789))), outputMode = OutputMode.Update() ) test("widening type change then restore back") { withTempDir { dir => sql(s"CREATE TABLE delta.`$dir` (a byte) USING DELTA") val checkpointDir = new File(dir, "sink_checkpoint") testStream(readStream(dir, checkpointDir))( StartStream(checkpointLocation = checkpointDir.toString), Execute { _ => sql(s"INSERT INTO delta.`$dir` VALUES (1)") }, ProcessAllAvailable(), Execute { _ => sql(s"ALTER TABLE delta.`$dir`ALTER COLUMN a TYPE int") }, // Widening a column type requires restarting the stream so that the new, wider schema is // used to process the batch. ExpectMetadataEvolutionException() ) if (schemaTrackingEnabled) { testStream(readStream(dir, checkpointDir))( StartStream(checkpointLocation = checkpointDir.toString), // The type change is blocked until the user reviews it and unblocks the stream. ExpectTypeChangeBlockedException() ) } withUnblockedTypeChanges { testStream(readStream(dir, checkpointDir, options = Map("ignoreDeletes" -> "true")))( StartStream(checkpointLocation = checkpointDir.toString), Execute { _ => sql(s"INSERT INTO delta.`$dir` VALUES (123456789)") }, ProcessAllAvailable(), CheckLastBatch(123456789), // Restore will narrow the type back, the schema change fails the query. Execute { _ => sql(s"RESTORE delta.`$dir` VERSION AS OF 1") }, if (schemaTrackingEnabled) { // With schema tracking, the first try evolves the tracked schema. The unsupported // type change is surfaced on the next retry. ExpectMetadataEvolutionException() } else { ExpectIncompatibleSchemaChangeException() } ) } // Retrying doesn't allow the narrowing type change to go through. withUnblockedTypeChanges { testStream(readStream(dir, checkpointDir, options = Map("ignoreDeletes" -> "true")))( StartStream(checkpointLocation = checkpointDir.toString), ExpectIncompatibleSchemaChangeException() ) } } } for { (name: String, toType: DataType) <- Seq( ("narrowing", ByteType), ("arbitrary", StringType)) } { test(s"$name type changes are not supported") { withTempDir { dir => sql(s"CREATE TABLE delta.`$dir` (a int) USING DELTA") val checkpointDir = new File(dir, "sink_checkpoint") testStream(readStream(dir, checkpointDir, options = Map("ignoreDeletes" -> "true")))( StartStream(checkpointLocation = checkpointDir.toString), Execute { _ => sql(s"INSERT INTO delta.`$dir` VALUES (1)") }, ProcessAllAvailable(), Execute { _ => // Overwrite the table schema to apply an arbitrary type change. spark .createDataFrame( sparkContext.emptyRDD[Row], StructType.fromDDL(s"a ${toType.sql}")) .write .format("delta") .mode(SaveMode.Overwrite) .option("overwriteSchema", "true") .save(dir.getCanonicalPath) }, if (schemaTrackingEnabled) { // With schema tracking, the first try evolves the tracked schema. The unsupported // type change is surfaced on the next retry. ExpectMetadataEvolutionException() } else { ExpectIncompatibleSchemaChangeException() } ) // Try to restart the stream even though the error is not retryable and it will fail again. testStream(readStream(dir, checkpointDir, options = Map("ignoreDeletes" -> "true")))( StartStream(checkpointLocation = checkpointDir.toString), Execute { _ => sql(s"INSERT INTO delta.`$dir` VALUES (2)") }, ExpectIncompatibleSchemaChangeException() ) } } } test("type change in delta source writing to a delta sink") { // End-to-end test with a delta source and a delta sink. withTempDir { sourceDir => withTempDir { sinkDir => // The test mixin implicitly enables type widening on all tables, disable type widening on // the sink initially for this test. sql(s"CREATE TABLE delta.`$sourceDir` (a byte) USING DELTA") sql( s""" |CREATE TABLE delta.`$sinkDir` (a byte) USING DELTA |TBLPROPERTIES('delta.enableTypeWidening' = 'false') """.stripMargin) val checkpointDir = new File(sinkDir, "checkpoint_dir") def runStream(mergeSchema: Boolean): Unit = try { withUnblockedTypeChanges { val q = readStream(sourceDir, checkpointDir) .writeStream .format("delta") .option("checkpointLocation", checkpointDir.toString) .option("mergeSchema", mergeSchema.toString) .start(sinkDir.getCanonicalPath) q.processAllAvailable() q.stop() } } catch { case e: StreamingQueryException => // Unwrap the exception for convenience throw e.getCause } // Start with no type change. sql(s"INSERT INTO delta.`$sourceDir` VALUES (1)") runStream(mergeSchema = false) checkAnswer(readDeltaTable(sinkDir.toString), Seq(Row(1))) // Change type of column 'a' and introduce a new column 'b'. Schema evolution is enabled // so the new column 'b' is added to the sink, but type widening is disabled on the sink so // the type of column 'a' remains INT: values are downcasted from INT to BYTE on write. sql(s"ALTER TABLE delta.`$sourceDir`ALTER COLUMN a TYPE int") sql(s"ALTER TABLE delta.`$sourceDir`ADD COLUMN b int") sql(s"INSERT INTO delta.`$sourceDir` VALUES (2, 2)") if (schemaTrackingEnabled) { val evolutionException = intercept[DeltaRuntimeException] { runStream(mergeSchema = true) } assert(evolutionException.getErrorClass === "DELTA_STREAMING_METADATA_EVOLUTION") } runStream(mergeSchema = true) assert(readDeltaTable(sinkDir.toString).schema("a").dataType === ByteType) assert(readDeltaTable(sinkDir.toString).schema("b").dataType === IntegerType) checkAnswer(readDeltaTable(sinkDir.toString), Seq(Row(1, null), Row(2, 2))) // Enable type widening on the sink and insert a value in 'a' that won't fit, first with // schema evolution disabled: the type of column 'a' in the sink isn't automatically changed // to INT and values are downcast: the value overflows and fails. sql(s"ALTER TABLE delta.`$sinkDir` SET TBLPROPERTIES('delta.enableTypeWidening' = 'true')") sql(s"INSERT INTO delta.`$sourceDir` VALUES (${Int.MaxValue}, ${Int.MaxValue})") def getSparkArithmeticException(ex: Throwable): SparkArithmeticException = ex match { case e: SparkArithmeticException => e case e: Throwable if e.getCause != null => getSparkArithmeticException(e.getCause) case e => fail(s"Unexpected exception: $e") } val ex = intercept[Throwable] { runStream(mergeSchema = false) } assert(getSparkArithmeticException(ex).getErrorClass === "CAST_OVERFLOW_IN_TABLE_INSERT") // Retry with schema evolution enabled. Type widening is also enabled on the sink, the type // of column 'a' is widened to INT and the write succeeds. runStream(mergeSchema = true) assert(readDeltaTable(sinkDir.toString).schema("a").dataType === IntegerType) assert(readDeltaTable(sinkDir.toString).schema("b").dataType === IntegerType) checkAnswer( readDeltaTable(sinkDir.toString), Seq(Row(1, null), Row(2, 2), Row(Int.MaxValue, Int.MaxValue)) ) } } } } /** Tests that specifically cover type widening without schema tracking. */ trait TypeWideningStreamingSourceWithoutSchemaTrackingTests extends StreamTest with SQLTestUtils with TypeWideningStreamingSourceTestMixin { import testImplicits._ test("schema changed event is logged for type widening") { withTempDir { dir => sql(s"CREATE TABLE delta.`$dir` (widened byte) USING DELTA") val checkpointDir = new File(dir, "sink_checkpoint") val logs = Log4jUsageLogger.track { testStream(readStream(dir, checkpointDir))( StartStream(checkpointLocation = checkpointDir.toString), Execute { _ => sql(s"INSERT INTO delta.`$dir` VALUES (1)") }, ProcessAllAvailable(), Execute { _ => sql(s"ALTER TABLE delta.`$dir` ALTER COLUMN widened TYPE int") }, ExpectMetadataEvolutionException() ) testStream(readStream(dir, checkpointDir))( StartStream(checkpointLocation = checkpointDir.toString), Execute { _ => sql(s"INSERT INTO delta.`$dir` VALUES (123456789)") }, ProcessAllAvailable(), CheckLastBatch(Row(123456789)) ) } // Filter for the schema changed event val schemaChangedEvents = logs .filter(_.metric == MetricDefinitions.EVENT_TAHOE.name) .filter(_.tags.get("opType").contains("delta.streaming.source.schemaChanged")) assert(schemaChangedEvents.size === 1, "A single schema changed events should be logged") val eventData = JsonUtils.fromJson[Map[String, Any]](schemaChangedEvents.head.blob) assert(eventData("currentVersion") === 0) assert(eventData("newVersion") === 2) assert(eventData("retryable") === true) assert(eventData("backfilling") === false) assert(eventData("readChangeDataFeed") === false) assert(eventData("typeWideningEnabled") === true) assert(eventData("enableSchemaTrackingForTypeWidening") === false) assert(eventData("containsWideningTypeChanges") === true) } } test("schema changed event is not logged when there are no schema changes") { withTempDir { dir => sql(s"CREATE TABLE delta.`$dir` (widened byte) USING DELTA") val checkpointDir = new File(dir, "sink_checkpoint") val logs = Log4jUsageLogger.track { testStream(readStream(dir, checkpointDir))( StartStream(checkpointLocation = checkpointDir.toString), Execute { _ => sql(s"INSERT INTO delta.`$dir` VALUES (1)") }, // This doesn't do anything since the type is already `byte`. Execute { _ => sql(s"ALTER TABLE delta.`$dir` ALTER COLUMN widened TYPE byte") }, Execute { _ => sql(s"INSERT INTO delta.`$dir` VALUES (100)") }, ProcessAllAvailable(), CheckAnswer(1, 100) ) } // Filter for the schema changed event val schemaChangedEvents = logs .filter(_.metric == MetricDefinitions.EVENT_TAHOE.name) .filter(_.tags.get("opType").contains("delta.streaming.source.schemaChanged")) assert(schemaChangedEvents.isEmpty, "No schema changed events should be logged") } } } /** Tests that specifically cover schema tracking for type widening. */ trait TypeWideningStreamingSourceSchemaTrackingTests extends StreamTest with SQLTestUtils with TypeWideningStreamingSourceTestMixin { import testImplicits._ test( "type change first without schemaTrackingLocation and unblock using schemaTrackingLocation") { withTempDir { dir => sql(s"CREATE TABLE delta.`$dir` (widened byte) USING DELTA") val checkpointDir = new File(dir, "sink_checkpoint") def readWithoutSchemaTrackingLog(): DataFrame = spark.readStream.format("delta").load(dir.getCanonicalPath) testStream(readWithoutSchemaTrackingLog())( StartStream(checkpointLocation = checkpointDir.toString), Execute { _ => sql(s"INSERT INTO delta.`$dir` VALUES (1)") }, ProcessAllAvailable(), CheckAnswer(1) ) testStream(readWithoutSchemaTrackingLog())( StartStream(checkpointLocation = checkpointDir.toString), Execute { _ => sql(s"ALTER TABLE delta.`$dir`ALTER COLUMN widened TYPE int") }, Execute { _ => sql(s"INSERT INTO delta.`$dir` VALUES (123456789)") }, ExpectFailure[DeltaStreamingNonAdditiveSchemaIncompatibleException]() ) // First retry with schema log initializes it. testStream(readStream(dir, checkpointDir))( StartStream(checkpointLocation = checkpointDir.toString), ExpectMetadataEvolutionException() ) // Second retry updates the schema log after the type change, then fails. testStream(readStream(dir, checkpointDir))( StartStream(checkpointLocation = checkpointDir.toString), ExpectMetadataEvolutionException() ) // Third retry requests user action to unblock the stream. testStream(readStream(dir, checkpointDir))( StartStream(checkpointLocation = checkpointDir.toString), ExpectTypeChangeBlockedException() ) // Unblocking the stream goes through. withUnblockedTypeChanges { testStream(readStream(dir, checkpointDir))( StartStream(checkpointLocation = checkpointDir.toString), ProcessAllAvailable(), CheckAnswer(123456789) ) } } } for ((name, getSqlConf: (Int => String), value) <- Seq( ("unblock all", (_: Int) => "allowSourceColumnTypeChange", "always"), ("unblock stream", (hash: Int) => s"allowSourceColumnTypeChange.ckpt_$hash", "always"), ("unblock version", (hash: Int) => s"allowSourceColumnTypeChange.ckpt_$hash", "2") )) { test(s"unblocking stream with sql conf after type change - $name") { withTempDir { dir => sql(s"CREATE TABLE delta.`$dir` (widened byte, other byte) USING DELTA") // Getting the checkpoint dir through the delta log to ensure the format is consistent with // the path used internally to compute the hash of the checkpoint location to unblock the // stream. val deltaLog = DeltaLog.forTable(spark, dir.toString) val checkpointDir = new File(deltaLog.dataPath.toString, "sink_checkpoint") def readWithAgg(): DataFrame = readStream(dir, checkpointDir) .groupBy("other") .agg(count(col("widened"))) testStream(readWithAgg(), outputMode = OutputMode.Complete())( StartStream(checkpointLocation = checkpointDir.toString), Execute { _ => sql(s"INSERT INTO delta.`$dir` VALUES (1, 1)") }, Execute { _ => sql(s"ALTER TABLE delta.`$dir`ALTER COLUMN widened TYPE int") }, ExpectMetadataEvolutionException() ) testStream(readWithAgg(), outputMode = OutputMode.Complete())( StartStream(checkpointLocation = checkpointDir.toString), ExpectTypeChangeBlockedException() ) val checkpointHash = s"$checkpointDir/sources/0".hashCode withSQLConf(s"spark.databricks.delta.streaming.${getSqlConf(checkpointHash)}" -> value) { testStream(readWithAgg(), outputMode = OutputMode.Complete())( StartStream(checkpointLocation = checkpointDir.toString), Execute { _ => sql(s"INSERT INTO delta.`$dir` VALUES (123456789, 1)") }, ProcessAllAvailable(), CheckLastBatch(Row(1, 2)) ) } } } } for ((name, optionValue) <- Seq( ("unblock stream", "always"), ("unblock version", "2") )) { test(s"unblocking stream with reader option after type change - $name") { withTempDir { dir => sql(s"CREATE TABLE delta.`$dir` (widened byte, other byte) USING DELTA") val checkpointDir = new File(dir, "sink_checkpoint") def readWithAgg(options: Map[String, String] = Map.empty): DataFrame = readStream(dir, checkpointDir, options) .groupBy("other") .agg(count(col("widened"))) testStream(readWithAgg(), outputMode = OutputMode.Complete())( StartStream(checkpointLocation = checkpointDir.toString), Execute { _ => sql(s"INSERT INTO delta.`$dir` VALUES (1, 1)") }, Execute { _ => sql(s"ALTER TABLE delta.`$dir`ALTER COLUMN widened TYPE int") }, ExpectMetadataEvolutionException() ) testStream(readWithAgg(), outputMode = OutputMode.Complete())( StartStream(checkpointLocation = checkpointDir.toString), ExpectTypeChangeBlockedException() ) testStream( readWithAgg(Map("allowSourceColumnTypeChange" -> optionValue)), outputMode = OutputMode.Complete())( StartStream(checkpointLocation = checkpointDir.toString), Execute { _ => sql(s"INSERT INTO delta.`$dir` VALUES (123456789, 1)") }, ProcessAllAvailable(), CheckLastBatch(Row(1, 2)) ) } } } test(s"overwrite schema with type change and dropped column") { withTempDir { dir => sql(s"CREATE TABLE delta.`$dir` (a byte, b int) USING DELTA") val checkpointDir = new File(dir, "sink_checkpoint") testStream(readStream(dir, checkpointDir, options = Map("ignoreDeletes" -> "true")))( StartStream(checkpointLocation = checkpointDir.toString), Execute { _ => sql(s"INSERT INTO delta.`$dir` VALUES (1, 1)") }, ProcessAllAvailable(), Execute { _ => // Overwrite the table schema. spark .createDataFrame( sparkContext.emptyRDD[Row], StructType.fromDDL(s"a INT")) .write .format("delta") .mode(SaveMode.Overwrite) .option("overwriteSchema", "true") .save(dir.getCanonicalPath) }, ExpectMetadataEvolutionException() ) testStream(readStream(dir, checkpointDir, options = Map("ignoreDeletes" -> "true")))( StartStream(checkpointLocation = checkpointDir.toString), ExpectFailure[DeltaRuntimeException] { ex => checkErrorMatchPVals( exception = ex.asInstanceOf[DeltaRuntimeException], "DELTA_STREAMING_CANNOT_CONTINUE_PROCESSING_POST_SCHEMA_EVOLUTION", parameters = Map( "opType" -> "DROP AND TYPE WIDENING", "previousSchemaChangeVersion" -> "0", "currentSchemaChangeVersion" -> "2", "columnChangeDetails" -> s"""Columns dropped: |'b' |Columns with widened types: |'a': TINYINT -> INT |""".stripMargin, "unblockChangeOptions" -> ".*allowSourceColumnDrop(.|\\n)*allowSourceColumnTypeChange.*", "unblockStreamOptions" -> ".*allowSourceColumnDrop(.|\\n)*allowSourceColumnTypeChange.*", "unblockChangeConfs" -> ".*allowSourceColumnDrop(.|\\n)*allowSourceColumnTypeChange.*", "unblockStreamConfs" -> ".*allowSourceColumnDrop(.|\\n)*allowSourceColumnTypeChange.*", "unblockAllConfs" -> ".*allowSourceColumnDrop(.|\\n)*allowSourceColumnTypeChange.*" )) } ) // Allowing both source column drop and type widening allows the stream to proceed withSQLConf( "spark.databricks.delta.streaming.allowSourceColumnDrop" -> "always", "spark.databricks.delta.streaming.allowSourceColumnTypeChange" -> "always") { testStream(readStream(dir, checkpointDir, options = Map("ignoreDeletes" -> "true")))( StartStream(checkpointLocation = checkpointDir.toString), Execute { _ => sql(s"INSERT INTO delta.`$dir` VALUES (2)") }, ProcessAllAvailable() ) } } } test("disable schema tracking log using internal conf") { withTempDir { dir => sql(s"CREATE TABLE delta.`$dir` (a byte) USING DELTA") val checkpointDir = new File(dir, "sink_checkpoint") def readStream(): DataFrame = spark.readStream.format("delta").load(dir.getCanonicalPath) // When we disable schema tracking for widening type changes, the stream should succeed // without requiring the user to provide a schema tracking location or unblock the type // change. withSQLConf( DeltaSQLConf.DELTA_TYPE_WIDENING_ENABLE_STREAMING_SCHEMA_TRACKING.key -> "false") { testStream(readStream())( StartStream(checkpointLocation = checkpointDir.toString), Execute { _ => sql(s"INSERT INTO delta.`$dir` VALUES (1)") }, ProcessAllAvailable(), Execute { _ => sql(s"ALTER TABLE delta.`$dir`ALTER COLUMN a TYPE int") }, ExpectFailure[DeltaIllegalStateException] { ex => assert(ex.asInstanceOf[SparkThrowable].getErrorClass === "DELTA_SCHEMA_CHANGED_WITH_VERSION") } ) testStream(readStream())( StartStream(checkpointLocation = checkpointDir.toString), Execute { _ => sql(s"INSERT INTO delta.`$dir` VALUES (123456789)") }, ProcessAllAvailable(), CheckLastBatch(123456789) ) } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningTableFeatureSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.typewidening import java.io.PrintWriter import com.databricks.spark.util.Log4jUsageLogger import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.DeltaOperations.ManualUpdate import org.apache.spark.sql.delta.actions.TableFeatureProtocolUtils.propertyKey import org.apache.spark.sql.delta.rowtracking.RowTrackingTestUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.SparkException import org.apache.spark.sql.{AnalysisException, QueryTest, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ /** * Test suite covering feature enablement and configuration tests. */ class TypeWideningTableFeatureEnablementSuite extends TypeWideningTableFeatureEnablementTests with TypeWideningTestMixin with TypeWideningDropFeatureTestMixin trait TypeWideningTableFeatureEnablementTests extends QueryTest with TypeWideningTestCases { self: QueryTest with TypeWideningTestMixin with TypeWideningDropFeatureTestMixin => import testImplicits._ test("enable type widening at table creation then disable it") { sql(s"CREATE TABLE delta.`$tempPath` (a int) USING DELTA " + s"TBLPROPERTIES ('${DeltaConfigs.ENABLE_TYPE_WIDENING.key}' = 'true')") assert(isTypeWideningSupported) assert(isTypeWideningEnabled) enableTypeWidening(tempPath, enabled = false) assert(isTypeWideningSupported) assert(!isTypeWideningEnabled) } test("enable type widening after table creation then disable it") { sql(s"CREATE TABLE delta.`$tempPath` (a int) USING DELTA " + s"TBLPROPERTIES ('${DeltaConfigs.ENABLE_TYPE_WIDENING.key}' = 'false')") assert(!isTypeWideningSupported) assert(!isTypeWideningEnabled) // Setting the property to false shouldn't add the table feature if it's not present. enableTypeWidening(tempPath, enabled = false) assert(!isTypeWideningSupported) assert(!isTypeWideningEnabled) enableTypeWidening(tempPath) assert(isTypeWideningSupported) assert(isTypeWideningEnabled) enableTypeWidening(tempPath, enabled = false) assert(isTypeWideningSupported) assert(!isTypeWideningEnabled) } test("set table property to incorrect value") { val ex = intercept[IllegalArgumentException] { sql(s"CREATE TABLE delta.`$tempPath` (a int) USING DELTA " + s"TBLPROPERTIES ('${DeltaConfigs.ENABLE_TYPE_WIDENING.key}' = 'bla')") } assert(ex.getMessage.contains("For input string: \"bla\"")) sql(s"CREATE TABLE delta.`$tempPath` (a int) USING DELTA " + s"TBLPROPERTIES ('${DeltaConfigs.ENABLE_TYPE_WIDENING.key}' = 'false')") checkError( intercept[SparkException] { sql(s"ALTER TABLE delta.`$tempPath` " + s"SET TBLPROPERTIES ('${DeltaConfigs.ENABLE_TYPE_WIDENING.key}' = 'bla')") }, "_LEGACY_ERROR_TEMP_2045", parameters = Map( "message" -> "For input string: \"bla\"" ) ) assert(!isTypeWideningSupported) assert(!isTypeWideningEnabled) } test("change column type without table feature") { sql(s"CREATE TABLE delta.`$tempPath` (a TINYINT) USING DELTA " + s"TBLPROPERTIES ('${DeltaConfigs.ENABLE_TYPE_WIDENING.key}' = 'false')") checkError( intercept[AnalysisException] { sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE SMALLINT") }, "DELTA_UNSUPPORTED_ALTER_TABLE_CHANGE_COL_OP", parameters = Map( "fieldPath" -> "a", "oldField" -> "TINYINT", "newField" -> "SMALLINT" ) ) } test("change column type with type widening table feature supported but table property set to " + "false") { sql(s"CREATE TABLE delta.`$tempPath` (a SMALLINT) USING DELTA") sql(s"ALTER TABLE delta.`$tempPath` " + s"SET TBLPROPERTIES ('${DeltaConfigs.ENABLE_TYPE_WIDENING.key}' = 'false')") checkError( intercept[AnalysisException] { sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE INT") }, "DELTA_UNSUPPORTED_ALTER_TABLE_CHANGE_COL_OP", parameters = Map( "fieldPath" -> "a", "oldField" -> "SMALLINT", "newField" -> "INT" ) ) } test("no-op type changes are always allowed") { sql(s"CREATE TABLE delta.`$tempPath` (a int) USING DELTA " + s"TBLPROPERTIES ('${DeltaConfigs.ENABLE_TYPE_WIDENING.key}' = 'false')") sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE INT") enableTypeWidening(tempPath, enabled = true) sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE INT") enableTypeWidening(tempPath, enabled = false) sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE INT") } } /** * Test suite covering feature removal, rewriting data files with the old type and removing type * widening metadata. */ class TypeWideningTableFeatureDropSuite extends QueryTest with TypeWideningTestMixin with TypeWideningDropFeatureTestMixin with TypeWideningTableFeatureDropTests trait TypeWideningTableFeatureDropTests extends RowTrackingTestUtils with TypeWideningTestCases { self: QueryTest with TypeWideningTestMixin with TypeWideningDropFeatureTestMixin => import testImplicits._ test("drop unused table feature on empty table") { sql(s"CREATE TABLE delta.`$tempPath` (a byte) USING DELTA") dropTableFeature( expectedOutcome = ExpectedOutcome.SUCCESS, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> ByteType) ) checkAnswer(readDeltaTable(tempPath), Seq.empty) } test("drop feature using sql with multipart identifier") { withTempDatabase { databaseName => val tableName = "test_table" withTable(tableName) { sql(s"CREATE TABLE $databaseName.$tableName (a byte) USING DELTA " + s"TBLPROPERTIES ('${DeltaConfigs.ENABLE_TYPE_WIDENING.key}' = 'true')") sql(s"INSERT INTO $databaseName.$tableName VALUES (1)") sql(s"ALTER TABLE $databaseName.$tableName CHANGE COLUMN a TYPE INT") sql(s"INSERT INTO $databaseName.$tableName VALUES (2)") val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName, Some(databaseName))) checkError( intercept[DeltaTableFeatureException] { sql(s"ALTER TABLE $databaseName.$tableName " + s"DROP FEATURE '${TypeWideningTableFeature.name}'" ).collect() }, "DELTA_FEATURE_DROP_WAIT_FOR_RETENTION_PERIOD", parameters = Map( "feature" -> TypeWideningTableFeature.name, "logRetentionPeriodKey" -> DeltaConfigs.LOG_RETENTION.key, "logRetentionPeriod" -> DeltaConfigs.LOG_RETENTION .fromMetaData(deltaLog.unsafeVolatileMetadata).toString, "truncateHistoryLogRetentionPeriod" -> DeltaConfigs.TABLE_FEATURE_DROP_TRUNCATE_HISTORY_LOG_RETENTION .fromMetaData(deltaLog.unsafeVolatileMetadata).toString) ) } } } // Rewriting the data when dropping the table feature relies on the default row commit version // being set even when row tracking isn't enabled. for(rowTrackingEnabled <- BOOLEAN_DOMAIN) { test(s"drop unused table feature on table with data, rowTrackingEnabled=$rowTrackingEnabled") { sql(s"CREATE TABLE delta.`$tempPath` (a byte) USING DELTA") addSingleFile(Seq(1, 2, 3), ByteType) dropTableFeature( expectedOutcome = ExpectedOutcome.SUCCESS, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> ByteType) ) checkAnswer(readDeltaTable(tempPath), Seq(Row(1), Row(2), Row(3))) } test(s"drop unused table feature on table with data inserted before adding the table feature," + s"rowTrackingEnabled=$rowTrackingEnabled") { setupManualClock() sql(s"CREATE TABLE delta.`$tempPath` (a byte) USING DELTA " + s"TBLPROPERTIES ('${DeltaConfigs.ENABLE_TYPE_WIDENING.key}' = 'false')") addSingleFile(Seq(1, 2, 3), ByteType) enableTypeWidening(tempPath) sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE int") dropTableFeature( expectedOutcome = ExpectedOutcome.FAIL_CURRENT_VERSION_USES_FEATURE, expectedNumFilesRewritten = 1, expectedColumnTypes = Map("a" -> IntegerType) ) dropTableFeature( expectedOutcome = ExpectedOutcome.FAIL_HISTORICAL_VERSION_USES_FEATURE, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> IntegerType) ) advancePastRetentionPeriod() dropTableFeature( expectedOutcome = ExpectedOutcome.SUCCESS, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> IntegerType) ) } test(s"drop table feature on table with data added only after type change, " + s"rowTrackingEnabled=$rowTrackingEnabled") { setupManualClock() sql(s"CREATE TABLE delta.`$tempPath` (a byte) USING DELTA") sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE int") addSingleFile(Seq(1, 2, 3), IntegerType) // We could actually drop the table feature directly here instead of failing by checking that // there were no files added before the type change. This may be an expensive check for a rare // scenario so we don't do it. dropTableFeature( expectedOutcome = ExpectedOutcome.FAIL_CURRENT_VERSION_USES_FEATURE, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> IntegerType) ) dropTableFeature( expectedOutcome = ExpectedOutcome.FAIL_HISTORICAL_VERSION_USES_FEATURE, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> IntegerType) ) advancePastRetentionPeriod() dropTableFeature( expectedOutcome = ExpectedOutcome.SUCCESS, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> IntegerType) ) checkAnswer(readDeltaTable(tempPath), Seq(Row(1), Row(2), Row(3))) } test(s"drop table feature on table with data added before type change, " + s"rowTrackingEnabled=$rowTrackingEnabled") { setupManualClock() sql(s"CREATE TABLE delta.`$tempDir` (a byte) USING DELTA") addSingleFile(Seq(1, 2, 3), ByteType) sql(s"ALTER TABLE delta.`$tempDir` CHANGE COLUMN a TYPE int") dropTableFeature( expectedOutcome = ExpectedOutcome.FAIL_CURRENT_VERSION_USES_FEATURE, expectedNumFilesRewritten = 1, expectedColumnTypes = Map("a" -> IntegerType) ) dropTableFeature( expectedOutcome = ExpectedOutcome.FAIL_HISTORICAL_VERSION_USES_FEATURE, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> IntegerType) ) advancePastRetentionPeriod() dropTableFeature( expectedOutcome = ExpectedOutcome.SUCCESS, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> IntegerType) ) checkAnswer(readDeltaTable(tempPath), Seq(Row(1), Row(2), Row(3))) } test(s"drop table feature on table with data added before type change and fully rewritten " + s"after, rowTrackingEnabled=$rowTrackingEnabled") { setupManualClock() sql(s"CREATE TABLE delta.`$tempDir` (a byte) USING DELTA") addSingleFile(Seq(1, 2, 3), ByteType) sql(s"ALTER TABLE delta.`$tempDir` CHANGE COLUMN a TYPE int") sql(s"UPDATE delta.`$tempDir` SET a = a + 10") dropTableFeature( expectedOutcome = ExpectedOutcome.FAIL_CURRENT_VERSION_USES_FEATURE, // The file was already rewritten in UPDATE. expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> IntegerType) ) dropTableFeature( expectedOutcome = ExpectedOutcome.FAIL_HISTORICAL_VERSION_USES_FEATURE, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> IntegerType) ) advancePastRetentionPeriod() dropTableFeature( expectedOutcome = ExpectedOutcome.SUCCESS, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> IntegerType) ) checkAnswer(readDeltaTable(tempPath), Seq(Row(11), Row(12), Row(13))) } test(s"drop table feature on table with data added before type change and partially " + s"rewritten after, rowTrackingEnabled=$rowTrackingEnabled") { setupManualClock() withRowTrackingEnabled(rowTrackingEnabled) { sql(s"CREATE TABLE delta.`$tempDir` (a byte) USING DELTA") addSingleFile(Seq(1, 2, 3), ByteType) addSingleFile(Seq(4, 5, 6), ByteType) sql(s"ALTER TABLE delta.`$tempDir` CHANGE COLUMN a TYPE int") sql(s"UPDATE delta.`$tempDir` SET a = a + 10 WHERE a < 4") dropTableFeature( expectedOutcome = ExpectedOutcome.FAIL_CURRENT_VERSION_USES_FEATURE, // One file was already rewritten in UPDATE, leaving 1 file to rewrite. expectedNumFilesRewritten = 1, expectedColumnTypes = Map("a" -> IntegerType) ) dropTableFeature( expectedOutcome = ExpectedOutcome.FAIL_HISTORICAL_VERSION_USES_FEATURE, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> IntegerType) ) advancePastRetentionPeriod() dropTableFeature( expectedOutcome = ExpectedOutcome.SUCCESS, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> IntegerType) ) checkAnswer( readDeltaTable(tempPath), Seq(Row(11), Row(12), Row(13), Row(4), Row(5), Row(6))) } } } } /** * Additional tests covering e.g. unsupported type change check, CLONE, RESTORE. */ class TypeWideningTableFeatureAdvancedSuite extends TypeWideningTableFeatureAdvancedTests with TypeWideningTestMixin with TypeWideningDropFeatureTestMixin trait TypeWideningTableFeatureAdvancedTests extends QueryTest with TypeWideningTestCases { self: QueryTest with TypeWideningTestMixin with TypeWideningDropFeatureTestMixin => import testImplicits._ for { testCase <- supportedTestCases } test(s"drop feature after type change ${testCase.fromType.sql} -> ${testCase.toType.sql}") { append(testCase.initialValuesDF.repartition(2)) sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN value TYPE ${testCase.toType.sql}") append(testCase.additionalValuesDF.repartition(3)) dropTableFeature( expectedOutcome = ExpectedOutcome.FAIL_CURRENT_VERSION_USES_FEATURE, expectedNumFilesRewritten = 2, expectedColumnTypes = Map("value" -> testCase.toType) ) checkAnswer(readDeltaTable(tempPath), testCase.expectedResult) } test("drop feature after a type change with schema evolution") { setupManualClock() sql(s"CREATE TABLE delta.`$tempPath` (a byte) USING DELTA") addSingleFile(Seq(1), ByteType) withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { addSingleFile(Seq(1024), IntegerType) } assert(readDeltaTable(tempPath).schema("a").dataType === IntegerType) checkAnswer(readDeltaTable(tempPath), Seq(Row(1), Row(1024))) dropTableFeature( expectedOutcome = ExpectedOutcome.FAIL_CURRENT_VERSION_USES_FEATURE, expectedNumFilesRewritten = 1, expectedColumnTypes = Map("a" -> IntegerType) ) advancePastRetentionPeriod() dropTableFeature( expectedOutcome = ExpectedOutcome.SUCCESS, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> IntegerType) ) checkAnswer(readDeltaTable(tempPath), Seq(Row(1), Row(1024))) } test("unsupported type changes applied to the table") { sql(s"CREATE TABLE delta.`$tempDir` (a array) USING DELTA") val metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( new MetadataBuilder() .putString("toType", "string") .putString("fromType", "int") .putLong("tableVersion", 2) .putString("fieldPath", "element") .build() )).build() // Add an unsupported type change to the table schema. Only an implementation that isn't // compliant with the feature specification would allow this. deltaLog.withNewTransaction { txn => txn.commit( Seq(txn.snapshot.metadata.copy( schemaString = new StructType() .add("a", StringType, nullable = true, metadata).json )), ManualUpdate) } checkError( intercept[DeltaIllegalStateException] { readDeltaTable(tempPath).collect() }, "DELTA_UNSUPPORTED_TYPE_CHANGE_IN_SCHEMA", parameters = Map( "fieldName" -> "a.element", "fromType" -> "INT", "toType" -> "STRING" ) ) // Validate that the internal table property can be used to bypass the check if needed. withSQLConf( DeltaSQLConf.DELTA_TYPE_WIDENING_BYPASS_UNSUPPORTED_TYPE_CHANGE_CHECK.key -> "true") { readDeltaTable(tempPath).collect() } } test("unsupported type changes in nested structs") { sql(s"CREATE TABLE delta.`$tempDir` (s struct) USING DELTA") deltaLog.withNewTransaction { txn => txn.commit( Seq(txn.snapshot.metadata.copy( schemaString = new StructType() .add("s", new StructType() .add("a", BooleanType, nullable = true, metadata = typeWideningMetadata(IntegerType, BooleanType))) .json )), ManualUpdate) } checkError( intercept[DeltaIllegalStateException] { readDeltaTable(tempPath).collect() }, "DELTA_UNSUPPORTED_TYPE_CHANGE_IN_SCHEMA", parameters = Map( "fieldName" -> "s.a", "fromType" -> "INT", "toType" -> "BOOLEAN" ) ) } test("char/varchar/string type changes don't trigger the unsupported type change check") { sql( s""" |CREATE TABLE delta.`$tempDir` ( | a string, b string, c char(4), d char(4), e varchar(4), f varchar(4), s struct |) USING DELTA |""".stripMargin) // Add type change metadata for all string<->char<->varchar type changes and ensure the table // can still be read. // Note: compliant delta implementations shouldn't actually record these type changes in the // table schema metadata. This test ensures that if a non-compliant implementation still does, // we don't unnecessarily block reads. deltaLog.withNewTransaction { txn => txn.commit( Seq(txn.snapshot.metadata.copy( schemaString = new StructType() .add("a", StringType, nullable = true, metadata = typeWideningMetadata(StringType, CharType(4))) .add("b", StringType, nullable = true, metadata = typeWideningMetadata(StringType, VarcharType(4))) .add("c", StringType, nullable = true, metadata = typeWideningMetadata(CharType(4), StringType)) .add("d", StringType, nullable = true, metadata = typeWideningMetadata(CharType(4), VarcharType(4))) .add("e", StringType, nullable = true, metadata = typeWideningMetadata(VarcharType(4), StringType)) .add("f", StringType, nullable = true, metadata = typeWideningMetadata(VarcharType(4), CharType(4))) .add("s", new StructType() .add("x", StringType, nullable = true, metadata = typeWideningMetadata(StringType, CharType(4))) ) .json )), ManualUpdate) } readDeltaTable(tempPath).collect() } test("type widening rewrite metrics") { sql(s"CREATE TABLE delta.`$tempDir` (a byte) USING DELTA") addSingleFile(Seq(1, 2, 3), ByteType) addSingleFile(Seq(4, 5, 6), ByteType) sql(s"ALTER TABLE delta.`$tempDir` CHANGE COLUMN a TYPE int") // Update a row from the second file to rewrite it. Only the first file still contains the old // data type after this. sql(s"UPDATE delta.`$tempDir` SET a = a + 10 WHERE a < 4") val usageLogs = Log4jUsageLogger.track { dropTableFeature( expectedOutcome = ExpectedOutcome.FAIL_CURRENT_VERSION_USES_FEATURE, expectedNumFilesRewritten = 1, expectedColumnTypes = Map("a" -> IntegerType) ) } val metrics = filterUsageRecords(usageLogs, "delta.typeWidening.featureRemoval") .map(r => JsonUtils.fromJson[Map[String, String]](r.blob)) .head assert(metrics("downgradeTimeMs").toLong > 0L) // Only the first file should get rewritten here since the second file was already rewritten // during the UPDATE. assert(metrics("numFilesRewritten").toLong === 1L) assert(metrics("metadataRemoved").toBoolean) } test("dropping feature after CLONE correctly rewrite files with old type") { withTable("source") { sql("CREATE TABLE source (a byte) USING delta") sql("INSERT INTO source VALUES (1)") sql("INSERT INTO source VALUES (2)") sql(s"ALTER TABLE source CHANGE COLUMN a TYPE INT") sql("INSERT INTO source VALUES (200)") sql(s"CREATE OR REPLACE TABLE delta.`$tempPath` SHALLOW CLONE source") checkAnswer(readDeltaTable(tempPath), Seq(Row(1), Row(2), Row(200))) dropTableFeature( expectedOutcome = ExpectedOutcome.FAIL_CURRENT_VERSION_USES_FEATURE, expectedNumFilesRewritten = 2, expectedColumnTypes = Map("a" -> IntegerType) ) checkAnswer(readDeltaTable(tempPath), Seq(Row(1), Row(2), Row(200))) } } test("RESTORE to before type change") { addSingleFile(Seq(1), ShortType) sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE INT") sql(s"UPDATE delta.`$tempPath` SET a = ${Int.MinValue} WHERE a = 1") // RESTORE to version 0, before the type change was applied. sql(s"RESTORE TABLE delta.`$tempPath` VERSION AS OF 0") checkAnswer(readDeltaTable(tempPath), Seq(Row(1))) dropTableFeature( // There should be no files to rewrite but versions before RESTORE still use the feature. expectedOutcome = ExpectedOutcome.FAIL_HISTORICAL_VERSION_USES_FEATURE, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> ShortType) ) } test("dropping feature after RESTORE correctly rewrite files with old type") { addSingleFile(Seq(1), ShortType) addSingleFile(Seq(2), ShortType) sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE INT") // Delete the first file which will be added back again with RESTORE. sql(s"DELETE FROM delta.`$tempPath` WHERE a = 1") addSingleFile(Seq(Int.MinValue), IntegerType) // RESTORE to version 2 -> ALTER TABLE CHANGE COLUMN TYPE. // The type change is then still present and the first file initially added at version 0 is // added back during RESTORE (version 5). That file contains the old type and must be rewritten // when the feature is dropped. sql(s"RESTORE TABLE delta.`$tempPath` VERSION AS OF 2") checkAnswer(readDeltaTable(tempPath), Seq(Row(1), Row(2))) dropTableFeature( expectedOutcome = ExpectedOutcome.FAIL_CURRENT_VERSION_USES_FEATURE, // Both files added before the type change must be rewritten. expectedNumFilesRewritten = 2, expectedColumnTypes = Map("a" -> IntegerType) ) checkAnswer(readDeltaTable(tempPath), Seq(Row(1), Row(2))) } test("rewriting files fails if there are corrupted files") { sql(s"CREATE TABLE delta.`$tempPath` (a byte) USING DELTA") sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE INT") addSingleFile(Seq(2), IntegerType) addSingleFile(Seq(3), IntegerType) val filePath = deltaLog.update().allFiles.first().absolutePath(deltaLog).toUri.getPath val pw = new PrintWriter(filePath) pw.write("corrupted") pw.close() // Rewriting files when dropping type widening should ignore this config, if the corruption is // transient it will leave files behind that some clients can't read. withSQLConf(SQLConf.IGNORE_CORRUPT_FILES.key -> "true") { val ex = intercept[SparkException] { sql(s"ALTER TABLE delta.`$tempDir` DROP FEATURE '${TypeWideningTableFeature.name}'") } assert(ex.getMessage.contains("Cannot seek after EOF")) } } } /** * Test suite covering preview vs stable feature interactions. */ class TypeWideningTableFeaturePreviewSuite extends TypeWideningTableFeatureVersionTests with TypeWideningTestMixin with TypeWideningDropFeatureTestMixin trait TypeWideningTableFeatureVersionTests extends QueryTest with TypeWideningTestCases { self: QueryTest with TypeWideningTestMixin with TypeWideningDropFeatureTestMixin => import testImplicits._ /** * Directly add the preview/stable type widening table feature without using the type widening * table property. */ def addTableFeature(tablePath: String, feature: TypeWideningTableFeatureBase): Unit = sql(s"ALTER TABLE delta.`$tablePath` " + s"SET TBLPROPERTIES ('${propertyKey(feature)}' = 'supported')") /** Validate whether the preview stable type widening table feature are supported or not. */ def assertFeatureSupported(preview: Boolean, stable: Boolean): Unit = { val protocol = deltaLog.update().protocol def supported(supported: Boolean): String = if (supported) "supported" else "not supported" assert(protocol.isFeatureSupported(TypeWideningPreviewTableFeature) === preview, s"Expected the preview feature to be ${supported(preview)} but it is ${supported(!preview)}.") assert(protocol.isFeatureSupported(TypeWideningTableFeature) === stable, s"Expected the stable feature to be ${supported(stable)} but it is ${supported(!stable)}.") assert(TypeWidening.isSupported(protocol) === preview || stable, s"Expected type widening to be ${supported(preview || stable)} but it is " + s"${supported(!(preview || stable))}.") } test("automatically enabling the preview feature doesn't enable the stable feature") { setupManualClock() // Create a new table with type widening enabled. sql(s"CREATE TABLE delta.`$tempPath` (a byte) USING DELTA " + s"TBLPROPERTIES ('${DeltaConfigs.ENABLE_TYPE_WIDENING.key}' = 'true')") addSingleFile(Seq(1), ByteType) sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE int") // The preview feature isn't supported and can't be dropped. assertFeatureSupported(preview = false, stable = true) dropTableFeature( feature = TypeWideningPreviewTableFeature, expectedOutcome = ExpectedOutcome.FAIL_FEATURE_NOT_PRESENT, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> ByteType) ) // The stable feature is supported and can be dropped. dropTableFeature( feature = TypeWideningTableFeature, expectedOutcome = ExpectedOutcome.FAIL_CURRENT_VERSION_USES_FEATURE, expectedNumFilesRewritten = 1, expectedColumnTypes = Map("a" -> IntegerType) ) assertFeatureSupported(preview = false, stable = true) advancePastRetentionPeriod() dropTableFeature( feature = TypeWideningTableFeature, expectedOutcome = ExpectedOutcome.SUCCESS, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> IntegerType) ) assertFeatureSupported(preview = false, stable = false) } test("manually adding the stable and preview features and dropping them") { setupManualClock() sql(s"CREATE TABLE delta.`$tempPath` (a byte) USING DELTA " + s"TBLPROPERTIES ('${DeltaConfigs.ENABLE_TYPE_WIDENING.key}' = 'false')") assertFeatureSupported(preview = false, stable = false) // This is undocumented for type widening but users can manually add the preview/stable feature // to the table instead of using the table property. addTableFeature(tempPath, TypeWideningTableFeature) assertFeatureSupported(preview = false, stable = true) // Users can manually add both features to the table that way: this is allowed, the two // specifications are compatible and supported. addTableFeature(tempPath, TypeWideningPreviewTableFeature) assertFeatureSupported(preview = true, stable = true) enableTypeWidening(tempPath) addSingleFile(Seq(1), ByteType) sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE int") // Dropping the stable feature doesn't also drop the preview feature. dropTableFeature( feature = TypeWideningTableFeature, expectedOutcome = ExpectedOutcome.FAIL_CURRENT_VERSION_USES_FEATURE, expectedNumFilesRewritten = 1, expectedColumnTypes = Map("a" -> IntegerType) ) assertFeatureSupported(preview = true, stable = true) advancePastRetentionPeriod() dropTableFeature( feature = TypeWideningTableFeature, expectedOutcome = ExpectedOutcome.SUCCESS, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> IntegerType) ) assertFeatureSupported(preview = true, stable = false) // Dropping the preview feature is now immediate since all traces have already been removed from // the table history. dropTableFeature( feature = TypeWideningPreviewTableFeature, expectedOutcome = ExpectedOutcome.SUCCESS, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> IntegerType) ) assertFeatureSupported(preview = false, stable = false) } test("tables created with the preview feature aren't automatically enabling the stable feature") { setupManualClock() sql(s"CREATE TABLE delta.`$tempPath` (a byte) USING DELTA " + s"TBLPROPERTIES ('${DeltaConfigs.ENABLE_TYPE_WIDENING.key}' = 'false')") addTableFeature(tempPath, TypeWideningPreviewTableFeature) assertFeatureSupported(preview = true, stable = false) // Enable the table property, this should keep the preview feature but not add the stable one. enableTypeWidening(tempPath) assertFeatureSupported(preview = true, stable = false) addSingleFile(Seq(1), ByteType) sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE int") dropTableFeature( feature = TypeWideningTableFeature, expectedOutcome = ExpectedOutcome.FAIL_FEATURE_NOT_PRESENT, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> ByteType) ) // The preview table feature can be dropped. dropTableFeature( feature = TypeWideningPreviewTableFeature, expectedOutcome = ExpectedOutcome.FAIL_CURRENT_VERSION_USES_FEATURE, expectedNumFilesRewritten = 1, expectedColumnTypes = Map("a" -> IntegerType) ) assertFeatureSupported(preview = true, stable = false) advancePastRetentionPeriod() dropTableFeature( feature = TypeWideningPreviewTableFeature, expectedOutcome = ExpectedOutcome.SUCCESS, expectedNumFilesRewritten = 0, expectedColumnTypes = Map("a" -> IntegerType) ) assertFeatureSupported(preview = false, stable = false) } test("tableVersion metadata is correctly set and preserved when using the preview feature") { sql(s"CREATE TABLE delta.`$tempPath` (a byte) USING DELTA " + s"TBLPROPERTIES ('${DeltaConfigs.ENABLE_TYPE_WIDENING.key}' = 'false')") addTableFeature(tempPath, TypeWideningPreviewTableFeature) enableTypeWidening(tempPath) addSingleFile(Seq(1), ByteType) sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE short") assert(deltaLog.update().metadata.schema === new StructType() .add("a", ShortType, nullable = true, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( new MetadataBuilder() .putLong("tableVersion", 4) .putString("fromType", "byte") .putString("toType", "short") .build() )).build())) // It's allowed to manually add both the preview and stable feature to the same table - the // specs are compatible. In that case, we still populate the `tableVersion` field. addTableFeature(tempPath, TypeWideningTableFeature) sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE int") assert(deltaLog.update().metadata.schema === new StructType() .add("a", IntegerType, nullable = true, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( new MetadataBuilder() .putLong("tableVersion", 4) .putString("fromType", "byte") .putString("toType", "short") .build(), new MetadataBuilder() .putLong("tableVersion", 6) .putString("fromType", "short") .putString("toType", "integer") .build() )).build())) } test("tableVersion isn't set when using the stable feature") { sql(s"CREATE TABLE delta.`$tempPath` (a byte) USING DELTA " + s"TBLPROPERTIES ('${DeltaConfigs.ENABLE_TYPE_WIDENING.key}' = 'false')") addTableFeature(tempPath, TypeWideningTableFeature) enableTypeWidening(tempPath) addSingleFile(Seq(1), ByteType) sql(s"ALTER TABLE delta.`$tempPath` CHANGE COLUMN a TYPE short") assert(deltaLog.update().metadata.schema === new StructType() .add("a", ShortType, nullable = true, metadata = new MetadataBuilder() .putMetadataArray("delta.typeChanges", Array( new MetadataBuilder() .putString("fromType", "byte") .putString("toType", "short") .build() )).build())) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningTestCases.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.typewidening import org.apache.spark.sql.{DataFrame, Encoder, Row} import org.apache.spark.sql.test.{SharedSparkSession, SQLTestUtils} import org.apache.spark.sql.types._ /** * Trait collecting supported and unsupported type change test cases. */ trait TypeWideningTestCases extends SQLTestUtils { self: SharedSparkSession => import testImplicits._ /** * Represents the input of a type change test. * @param fromType The original type of the column 'value' in the test table. * @param toType The type to use when changing the type of column 'value'. */ abstract class TypeEvolutionTestCase( val fromType: DataType, val toType: DataType) { /** The initial values to insert with type `fromType` in column 'value' after table creation. */ def initialValuesDF: DataFrame /** Additional values to insert after changing the type of the column 'value' to `toType`. */ def additionalValuesDF: DataFrame /** Expected content of the table after inserting the additional values. */ def expectedResult: DataFrame } /** * Represents the input of a supported type change test. Handles converting the test values from * scala types to a dataframe. */ case class SupportedTypeEvolutionTestCase[ FromType <: DataType, ToType <: DataType, FromVal: Encoder, ToVal: Encoder ]( override val fromType: FromType, override val toType: ToType, initialValues: Seq[FromVal], additionalValues: Seq[ToVal] ) extends TypeEvolutionTestCase(fromType, toType) { override def initialValuesDF: DataFrame = initialValues.toDF("value").select($"value".cast(fromType)) override def additionalValuesDF: DataFrame = additionalValues.toDF("value").select($"value".cast(toType)) override def expectedResult: DataFrame = initialValuesDF.union(additionalValuesDF).select($"value".cast(toType)) } /** * Represents the input of an unsupported type change test. Handles converting the test values * from scala types to a dataframe. Additional values to insert are always empty since the type * change is expected to fail. */ case class UnsupportedTypeEvolutionTestCase[ FromType <: DataType, ToType <: DataType, FromVal : Encoder]( override val fromType: FromType, override val toType: ToType, initialValues: Seq[FromVal]) extends TypeEvolutionTestCase(fromType, toType) { override def initialValuesDF: DataFrame = initialValues.toDF("value").select($"value".cast(fromType)) override def additionalValuesDF: DataFrame = spark.createDataFrame( sparkContext.emptyRDD[Row], new StructType().add(StructField("value", toType))) override def expectedResult: DataFrame = initialValuesDF.select($"value".cast(toType)) } // Type changes that are supported by all Parquet readers. Byte, Short, Int are all stored as // INT32 in parquet so these changes are guaranteed to be supported. protected val supportedTestCases: Seq[TypeEvolutionTestCase] = Seq( SupportedTypeEvolutionTestCase(ByteType, ShortType, Seq(1, -1, Byte.MinValue, Byte.MaxValue, null.asInstanceOf[Byte]), Seq(4, -4, Short.MinValue, Short.MaxValue, null.asInstanceOf[Short])), SupportedTypeEvolutionTestCase(ByteType, IntegerType, Seq(1, -1, Byte.MinValue, Byte.MaxValue, null.asInstanceOf[Byte]), Seq(4, -4, Int.MinValue, Int.MaxValue, null.asInstanceOf[Int])), SupportedTypeEvolutionTestCase(ShortType, IntegerType, Seq(1, -1, Short.MinValue, Short.MaxValue, null.asInstanceOf[Short]), Seq(4, -4, Int.MinValue, Int.MaxValue, null.asInstanceOf[Int])), SupportedTypeEvolutionTestCase(ShortType, LongType, Seq(1, -1, Short.MinValue, Short.MaxValue, null.asInstanceOf[Short]), Seq(4L, -4L, Long.MinValue, Long.MaxValue, null.asInstanceOf[Long])), SupportedTypeEvolutionTestCase(IntegerType, LongType, Seq(1, -1, Int.MinValue, Int.MaxValue, null.asInstanceOf[Int]), Seq(4L, -4L, Long.MinValue, Long.MaxValue, null.asInstanceOf[Long])), SupportedTypeEvolutionTestCase(FloatType, DoubleType, Seq(1234.56789f, -0f, 0f, Float.NaN, Float.NegativeInfinity, Float.PositiveInfinity, Float.MinPositiveValue, Float.MinValue, Float.MaxValue, null.asInstanceOf[Float]), Seq(987654321.987654321d, -0d, 0d, Double.NaN, Double.NegativeInfinity, Double.PositiveInfinity, Double.MinPositiveValue, Double.MinValue, Double.MaxValue, null.asInstanceOf[Double])), SupportedTypeEvolutionTestCase(DateType, TimestampNTZType, Seq("2020-01-01", "2024-02-29", "1312-02-27"), Seq("2020-03-17 15:23:15.123456", "2058-12-31 23:59:59.999", "0001-01-01 00:00:00")), // Larger precision. SupportedTypeEvolutionTestCase(DecimalType(Decimal.MAX_INT_DIGITS, 2), DecimalType(Decimal.MAX_LONG_DIGITS, 2), Seq(BigDecimal("1.23"), BigDecimal("10.34"), null.asInstanceOf[BigDecimal]), Seq(BigDecimal("-67.89"), BigDecimal("9" * (Decimal.MAX_LONG_DIGITS - 2) + ".99"), null.asInstanceOf[BigDecimal])), // Larger precision and scale, same physical type. SupportedTypeEvolutionTestCase(DecimalType(Decimal.MAX_INT_DIGITS - 1, 2), DecimalType(Decimal.MAX_INT_DIGITS, 3), Seq(BigDecimal("1.23"), BigDecimal("10.34"), null.asInstanceOf[BigDecimal]), Seq(BigDecimal("-67.89"), BigDecimal("9" * (Decimal.MAX_INT_DIGITS - 3) + ".99"), null.asInstanceOf[BigDecimal])), // Larger precision and scale, different physical types. SupportedTypeEvolutionTestCase(DecimalType(Decimal.MAX_INT_DIGITS, 2), DecimalType(Decimal.MAX_LONG_DIGITS + 1, 3), Seq(BigDecimal("1.23"), BigDecimal("10.34"), null.asInstanceOf[BigDecimal]), Seq(BigDecimal("-67.89"), BigDecimal("9" * (Decimal.MAX_LONG_DIGITS - 2) + ".99"), null.asInstanceOf[BigDecimal])) ) // Type changes that are only eligible for automatic widening when // spark.databricks.delta.typeWidening.allowAutomaticWidening = ALWAYS. protected val restrictedAutomaticWideningTestCases: Seq[TypeEvolutionTestCase] = Seq( SupportedTypeEvolutionTestCase(IntegerType, DoubleType, Seq(1, -1, Int.MinValue, Int.MaxValue, null.asInstanceOf[Int]), Seq(987654321.987654321d, -0d, 0d, Double.NaN, Double.NegativeInfinity, Double.PositiveInfinity, Double.MinPositiveValue, Double.MinValue, Double.MaxValue, null.asInstanceOf[Double])), SupportedTypeEvolutionTestCase(ByteType, DecimalType(10, 0), Seq(1, -1, Byte.MinValue, Byte.MaxValue, null.asInstanceOf[Byte]), Seq(BigDecimal("1.23"), BigDecimal("9" * 10), null.asInstanceOf[BigDecimal])), SupportedTypeEvolutionTestCase(ShortType, DecimalType(10, 0), Seq(1, -1, Short.MinValue, Short.MaxValue, null.asInstanceOf[Short]), Seq(BigDecimal("1.23"), BigDecimal("9" * 10), null.asInstanceOf[BigDecimal])), SupportedTypeEvolutionTestCase(IntegerType, DecimalType(10, 0), Seq(1, -1, Int.MinValue, Int.MaxValue, null.asInstanceOf[Int]), Seq(BigDecimal("1.23"), BigDecimal("9" * 10), null.asInstanceOf[BigDecimal])), SupportedTypeEvolutionTestCase(LongType, DecimalType(20, 0), Seq(1L, -1L, Long.MinValue, Long.MaxValue, null.asInstanceOf[Int]), Seq(BigDecimal("1.23"), BigDecimal("9" * 20), null.asInstanceOf[BigDecimal])) ) // Test type changes that aren't supported. protected val unsupportedTestCases: Seq[TypeEvolutionTestCase] = Seq( UnsupportedTypeEvolutionTestCase(IntegerType, ByteType, Seq(1, 2, Int.MinValue)), UnsupportedTypeEvolutionTestCase(LongType, IntegerType, Seq(4, 5, Long.MaxValue)), UnsupportedTypeEvolutionTestCase(DoubleType, FloatType, Seq(987654321.987654321d, Double.NaN, Double.NegativeInfinity, Double.PositiveInfinity, Double.MinPositiveValue, Double.MinValue, Double.MaxValue)), UnsupportedTypeEvolutionTestCase(ByteType, DecimalType(2, 0), Seq(1, -1, Byte.MinValue)), UnsupportedTypeEvolutionTestCase(ShortType, DecimalType(4, 0), Seq(1, -1, Short.MinValue)), UnsupportedTypeEvolutionTestCase(IntegerType, DecimalType(9, 0), Seq(1, -1, Int.MinValue)), UnsupportedTypeEvolutionTestCase(LongType, DecimalType(19, 0), Seq(1, -1, Long.MinValue)), UnsupportedTypeEvolutionTestCase(TimestampNTZType, DateType, Seq("2020-03-17 15:23:15", "2023-12-31 23:59:59", "0001-01-01 00:00:00")), // Reduce scale UnsupportedTypeEvolutionTestCase(DecimalType(Decimal.MAX_INT_DIGITS, 2), DecimalType(Decimal.MAX_INT_DIGITS, 3), Seq(BigDecimal("-67.89"), BigDecimal("9" * (Decimal.MAX_INT_DIGITS - 2) + ".99"))), // Reduce precision UnsupportedTypeEvolutionTestCase(DecimalType(Decimal.MAX_INT_DIGITS, 2), DecimalType(Decimal.MAX_INT_DIGITS - 1, 2), Seq(BigDecimal("-67.89"), BigDecimal("9" * (Decimal.MAX_INT_DIGITS - 2) + ".99"))), // Reduce precision & scale UnsupportedTypeEvolutionTestCase(DecimalType(Decimal.MAX_LONG_DIGITS, 2), DecimalType(Decimal.MAX_INT_DIGITS - 1, 1), Seq(BigDecimal("-67.89"), BigDecimal("9" * (Decimal.MAX_LONG_DIGITS - 2) + ".99"))), // Increase scale more than precision UnsupportedTypeEvolutionTestCase(DecimalType(Decimal.MAX_INT_DIGITS, 2), DecimalType(Decimal.MAX_INT_DIGITS + 1, 4), Seq(BigDecimal("-67.89"), BigDecimal("9" * (Decimal.MAX_INT_DIGITS - 2) + ".99"))), // Smaller scale and larger precision. UnsupportedTypeEvolutionTestCase(DecimalType(Decimal.MAX_LONG_DIGITS, 2), DecimalType(Decimal.MAX_INT_DIGITS + 3, 1), Seq(BigDecimal("-67.89"), BigDecimal("9" * (Decimal.MAX_LONG_DIGITS - 2) + ".99"))) ) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningTestMixin.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.typewidening import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.{RemoveFile, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.AlterTableDropFeatureDeltaCommand import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.util.DeltaFileOperations import com.google.common.math.DoubleMath import org.apache.spark.SparkConf import org.apache.spark.sql.{DataFrame, Encoder, QueryTest} import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.functions.col import org.apache.spark.sql.internal.{LegacyBehaviorPolicy, SQLConf} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types._ /** * Test mixin that enables type widening by default for all tests in the suite. */ trait TypeWideningTestMixin extends DeltaSQLCommandTest with DeltaDMLTestUtilsPathBased { self: QueryTest => import testImplicits._ protected override def sparkConf: SparkConf = { super.sparkConf .set(DeltaConfigs.ENABLE_TYPE_WIDENING.defaultTablePropertyKey, "true") .set(DeltaSQLConf.DELTA_ALLOW_AUTOMATIC_WIDENING.key, "always") .set(TableFeatureProtocolUtils.defaultPropertyKey(TimestampNTZTableFeature), "supported") // Ensure we don't silently cast test inputs to null on overflow. .set(SQLConf.ANSI_ENABLED.key, "true") // Rebase mode must be set explicitly to allow writing dates before 1582-10-15. .set(SQLConf.PARQUET_REBASE_MODE_IN_WRITE.key, LegacyBehaviorPolicy.CORRECTED.toString) // All the drop feature tests below are based on the drop feature with history truncation // implementation. The fast drop feature implementation does not require any waiting time. // The fast drop feature implementation is tested extensively in the // DeltaFastDropFeatureSuite. .set(DeltaSQLConf.FAST_DROP_FEATURE_ENABLED.key, false.toString) } /** Enable (or disable) type widening for the table under the given path. */ protected def enableTypeWidening(tablePath: String, enabled: Boolean = true): Unit = sql(s"ALTER TABLE delta.`$tablePath` " + s"SET TBLPROPERTIES('${DeltaConfigs.ENABLE_TYPE_WIDENING.key}' = '${enabled.toString}')") /** Whether the test table supports the type widening table feature. */ def isTypeWideningSupported: Boolean = { TypeWidening.isSupported(deltaLog.update().protocol) } /** Whether the type widening table property is enabled on the test table. */ def isTypeWideningEnabled: Boolean = { val snapshot = deltaLog.update() TypeWidening.isEnabled(snapshot.protocol, snapshot.metadata) } /** Short-hand to create type widening metadata for struct fields. */ protected def typeWideningMetadata( from: AtomicType, to: AtomicType, path: Seq[String] = Seq.empty): Metadata = new MetadataBuilder() .putMetadataArray( "delta.typeChanges", Array(TypeChange(None, from, to, path).toMetadata)) .build() def addSingleFile[T: Encoder](values: Seq[T], dataType: DataType): Unit = append(values.toDF("a").select(col("a").cast(dataType)).repartition(1)) /** * Similar to `QueryTest.checkAnswer` but using fuzzy equality for double values. This is needed * because double partition values are serialized as string leading to loss of precision. Also * `checkAnswer` treats -0f and 0f as different values without tolerance. */ def checkAnswerWithTolerance( actualDf: DataFrame, expectedDf: DataFrame, toType: DataType, tolerance: Double = 0.001) : Unit = { // Widening to float isn't supported so only handle double here. if (toType == DoubleType) { val actual = actualDf.sort("value").collect() val expected = expectedDf.sort("value").collect() assert(actual.length === expected.length, s"Wrong result: $actual did not equal $expected") actual.zip(expected).foreach { case (a, e) => val expectedValue = e.getAs[Double]("value") val actualValue = a.getAs[Double]("value") val absTolerance = if (expectedValue.isNaN || expectedValue.isInfinity) { 0 } else { tolerance * Math.abs(expectedValue) } assert( DoubleMath.fuzzyEquals(actualValue, expectedValue, absTolerance), s"$actualValue did not equal $expectedValue" ) } } else { checkAnswer(actualDf, expectedDf) } } } /** * Mixin trait containing helpers to test dropping the type widening table feature. */ trait TypeWideningDropFeatureTestMixin extends QueryTest with SharedSparkSession with DeltaDMLTestUtils { /** Expected outcome of dropping the type widening table feature. */ object ExpectedOutcome extends Enumeration { val SUCCESS, FAIL_CURRENT_VERSION_USES_FEATURE, FAIL_HISTORICAL_VERSION_USES_FEATURE, FAIL_FEATURE_NOT_PRESENT = Value } def getCatalogTableOpt: Option[CatalogTable] = { if (DeltaTableIdentifier.isDeltaPath(spark, tableIdentifier)) { None } else { Some(spark.sessionState.catalog.getTableMetadata(tableIdentifier)) } } /** * Helper method to drop the type widening table feature and check for an expected outcome. * Validates in particular that the right number of files were rewritten and that the rewritten * files all contain the expected type for specified columns. */ def dropTableFeature( feature: TableFeature = TypeWideningTableFeature, expectedOutcome: ExpectedOutcome.Value, expectedNumFilesRewritten: Long, expectedColumnTypes: Map[String, DataType]): Unit = { val catalogTableOpt = getCatalogTableOpt val snapshot = deltaLog.update(catalogTableOpt = catalogTableOpt) // Need to directly call ALTER TABLE command to pass our deltaLog with manual clock. val dropFeature = AlterTableDropFeatureDeltaCommand(DeltaTableV2(spark, deltaLog.dataPath), feature.name) expectedOutcome match { case ExpectedOutcome.SUCCESS => dropFeature.run(spark) case ExpectedOutcome.FAIL_CURRENT_VERSION_USES_FEATURE => checkError( intercept[DeltaTableFeatureException] { dropFeature.run(spark) }, "DELTA_FEATURE_DROP_WAIT_FOR_RETENTION_PERIOD", parameters = Map( "feature" -> feature.name, "logRetentionPeriodKey" -> DeltaConfigs.LOG_RETENTION.key, "logRetentionPeriod" -> DeltaConfigs.LOG_RETENTION .fromMetaData(snapshot.metadata).toString, "truncateHistoryLogRetentionPeriod" -> DeltaConfigs.TABLE_FEATURE_DROP_TRUNCATE_HISTORY_LOG_RETENTION .fromMetaData(snapshot.metadata).toString) ) case ExpectedOutcome.FAIL_HISTORICAL_VERSION_USES_FEATURE => checkError( intercept[DeltaTableFeatureException] { dropFeature.run(spark) }, "DELTA_FEATURE_DROP_HISTORICAL_VERSIONS_EXIST", parameters = Map( "feature" -> feature.name, "logRetentionPeriodKey" -> DeltaConfigs.LOG_RETENTION.key, "logRetentionPeriod" -> DeltaConfigs.LOG_RETENTION .fromMetaData(snapshot.metadata).toString, "truncateHistoryLogRetentionPeriod" -> DeltaConfigs.TABLE_FEATURE_DROP_TRUNCATE_HISTORY_LOG_RETENTION .fromMetaData(snapshot.metadata).toString) ) case ExpectedOutcome.FAIL_FEATURE_NOT_PRESENT => checkError( intercept[DeltaTableFeatureException] { dropFeature.run(spark) }, "DELTA_FEATURE_DROP_FEATURE_NOT_PRESENT", parameters = Map("feature" -> feature.name) ) } if (expectedOutcome != ExpectedOutcome.FAIL_FEATURE_NOT_PRESENT) { assert(!TypeWideningMetadata.containsTypeWideningMetadata( deltaLog.update(catalogTableOpt = catalogTableOpt).schema)) } // Check the number of files rewritten. assert(getNumRemoveFilesSinceVersion(snapshot.version + 1) === expectedNumFilesRewritten, s"Expected $expectedNumFilesRewritten file(s) to be rewritten but found " + s"${getNumRemoveFilesSinceVersion(snapshot.version + 1)} rewritten file(s).") // Check that all files now contain the expected data types. expectedColumnTypes.foreach { case (colName, expectedType) => withSQLConf("spark.databricks.delta.formatCheck.enabled" -> "false") { deltaLog.update(catalogTableOpt = catalogTableOpt).filesForScan( Seq.empty, keepNumRecords = false).files.foreach { file => val filePath = DeltaFileOperations.absolutePath(deltaLog.dataPath.toString, file.path) val data = spark.read.parquet(filePath.toString) val physicalColName = DeltaColumnMapping.getPhysicalName(snapshot.schema(colName)) assert(data.schema(physicalColName).dataType === expectedType, s"File with values ${data.collect().mkString(", ")} wasn't rewritten.") } } } } /** Get the number of remove actions committed since the given table version (included). */ def getNumRemoveFilesSinceVersion(version: Long): Long = deltaLog .getChanges(startVersion = version, catalogTableOpt = getCatalogTableOpt) .flatMap { case (_, actions) => actions } .collect { case r: RemoveFile => r } .size } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/typewidening/TypeWideningUniformTests.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.typewidening import org.apache.spark.sql.delta.{DeltaInsertIntoTest, DeltaUnsupportedOperationException, IcebergCompatBase, IcebergCompatV1, IcebergCompatV2} import org.apache.spark.sql.delta.DeltaErrors.toSQLType import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.scalatest.GivenWhenThen import org.apache.spark.sql.{QueryTest, SaveMode} import org.apache.spark.sql.functions.col import org.apache.spark.sql.types.{DataType, DateType, DecimalType} /** Trait collecting tests covering type widening + Uniform Iceberg compatibility. */ trait TypeWideningUniformTests extends QueryTest with TypeWideningTestMixin with TypeWideningTestCases with DeltaInsertIntoTest with GivenWhenThen { // Iceberg supports all base type changes eligible for widening during schema evolution except // for date -> timestampNtz and decimal scale changes. private val icebergSupportedTestCases = supportedTestCases.filter { case SupportedTypeEvolutionTestCase(_ : DateType, _, _, _) => false case SupportedTypeEvolutionTestCase(from: DecimalType, to: DecimalType, _, _) => from.scale == to.scale case _ => true } // Unsupported type changes are all base changes that aren't supported above and all changes that // are not eligible for schema evolution: int -> double, int -> decimal private val icebergUnsupportedTestCases = supportedTestCases.diff(icebergSupportedTestCases) ++ restrictedAutomaticWideningTestCases /** Helper to enable Uniform with Iceberg compatibility on the given table. */ private def enableIcebergUniform(tableName: String, compat: IcebergCompatBase): Unit = sql( s""" |ALTER TABLE $tableName SET TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name', | '${compat.config.key}' = 'true', | 'delta.universalFormat.enabledFormats' = 'iceberg' |) """.stripMargin) /** Helper to check that the given function violates Uniform compatibility with type widening. */ private def checkIcebergCompatViolation( compat: IcebergCompatBase, fromType: DataType, toType: DataType)(f: => Unit): Unit = { Given(s"iceberg compat ${compat.getClass.getSimpleName}") checkError( exception = intercept[DeltaUnsupportedOperationException] { f }, "DELTA_ICEBERG_COMPAT_VIOLATION.UNSUPPORTED_TYPE_WIDENING", parameters = Map( "version" -> compat.version.toString, "prevType" -> toSQLType(fromType), "newType" -> toSQLType(toType), "fieldPath" -> "a" ) ) } test("apply supported type change then enable uniform") { for (testCase <- icebergSupportedTestCases) { Given(s"changing ${testCase.fromType.sql} -> ${testCase.toType.sql}") val tableName = "type_widening_uniform_supported_table" withTable(tableName) { sql(s"CREATE TABLE $tableName (a ${testCase.fromType.sql}) USING DELTA") sql(s"ALTER TABLE $tableName CHANGE COLUMN a TYPE ${testCase.toType.sql}") enableIcebergUniform(tableName, IcebergCompatV2) } } } test("apply unsupported type change then enable uniform") { for (testCase <- icebergUnsupportedTestCases) { val tableName = "type_widening_uniform_unsupported_table" withTable(tableName) { Given(s"changing ${testCase.fromType.sql} -> ${testCase.toType.sql}") sql(s"CREATE TABLE $tableName (a ${testCase.fromType.sql}) USING DELTA") sql(s"ALTER TABLE $tableName CHANGE COLUMN a TYPE ${testCase.toType.sql}") checkIcebergCompatViolation(IcebergCompatV1, testCase.fromType, testCase.toType) { enableIcebergUniform(tableName, IcebergCompatV1) } checkIcebergCompatViolation(IcebergCompatV2, testCase.fromType, testCase.toType) { enableIcebergUniform(tableName, IcebergCompatV2) } } } } test("enable uniform then apply supported type change - ALTER TABLE") { for (testCase <- icebergSupportedTestCases) { Given(s"changing ${testCase.fromType.sql} -> ${testCase.toType.sql}") val tableName = "type_widening_uniform_manual_supported_table" withTable(tableName) { sql(s"CREATE TABLE $tableName (a ${testCase.fromType.sql}) USING DELTA") enableIcebergUniform(tableName, IcebergCompatV2) sql(s"ALTER TABLE $tableName CHANGE COLUMN a TYPE ${testCase.toType.sql}") } } } test("enable uniform then apply unsupported type change - ALTER TABLE") { for (testCase <- icebergUnsupportedTestCases) { val tableName = "type_widening_uniform_manual_unsupported_table" withTable(tableName) { Given(s"changing ${testCase.fromType.sql} -> ${testCase.toType.sql}") sql(s"CREATE TABLE $tableName (a ${testCase.fromType.sql}) USING DELTA") enableIcebergUniform(tableName, IcebergCompatV2) checkIcebergCompatViolation(IcebergCompatV2, testCase.fromType, testCase.toType) { sql(s"ALTER TABLE $tableName CHANGE COLUMN a TYPE ${testCase.toType.sql}") } } } } test("enable uniform then apply supported type change - MERGE") { for (testCase <- icebergSupportedTestCases) { Given(s"changing ${testCase.fromType.sql} -> ${testCase.toType.sql}") withTable("source", "target") { testCase.initialValuesDF.write.format("delta").saveAsTable("target") testCase.additionalValuesDF.write.format("delta").saveAsTable("source") enableIcebergUniform("target", IcebergCompatV2) withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { sql( s""" |MERGE INTO target |USING source |ON 0 = 1 |WHEN NOT MATCHED THEN INSERT * """.stripMargin) } val result = sql(s"SELECT * FROM target") assert(result.schema("value").dataType === testCase.toType) checkAnswer(result, testCase.expectedResult) } } } test("enable uniform then apply unsupported type change - MERGE") { for (testCase <- icebergUnsupportedTestCases) { Given(s"changing ${testCase.fromType.sql} -> ${testCase.toType.sql}") withTable("source", "target") { testCase.initialValuesDF.write.format("delta").saveAsTable("target") // Here we use a source for MERGE that contains the same data that is already present in the // target, except that it uses a wider type. Since Uniform is enabled and Iceberg doesn't // support the given type change, we will keep the existing narrower type and downcast // values. `testCase.additionalValueDF` contains values that would overflow, which would // just fail, hence why we use `testCase.initialValueDF` instead. testCase.initialValuesDF .select(col("value").cast(testCase.toType)) .write .format("delta") .saveAsTable("source") enableIcebergUniform("target", IcebergCompatV2) withSQLConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true") { sql( s""" |MERGE INTO target |USING source |ON 0 = 1 |WHEN NOT MATCHED THEN INSERT * """.stripMargin) } val result = sql(s"SELECT * FROM target") val expected = testCase.initialValuesDF.union(testCase.initialValuesDF) assert(result.schema("value").dataType === testCase.fromType) checkAnswer(result, expected) } } } for (insert <- Set( // Cover only a subset of all INSERTs. There's little value in testing all of them and it // quickly gets expensive. SQLInsertByPosition(SaveMode.Append), SQLInsertByName(SaveMode.Append), DFv1InsertInto(SaveMode.Append), StreamingInsert)) { test(s"enable uniform then apply supported type change - ${insert.name}") { for (testCase <- icebergSupportedTestCases) { Given(s"changing ${testCase.fromType.sql} -> ${testCase.toType.sql}") withTable("source", "target") { testCase.initialValuesDF.write.format("delta").saveAsTable("target") testCase.additionalValuesDF.write.format("delta").saveAsTable("source") enableIcebergUniform("target", IcebergCompatV2) insert.runInsert( columns = Seq("value"), whereCol = "value", whereValue = 1, withSchemaEvolution = true) val result = sql(s"SELECT * FROM target") assert(result.schema("value").dataType === testCase.toType) checkAnswer(result, testCase.expectedResult) } } } test(s"enable uniform then apply unsupported type change - ${insert.name}") { for (testCase <- icebergUnsupportedTestCases) { Given(s"changing ${testCase.fromType.sql} -> ${testCase.toType.sql}") withTable("source", "target") { testCase.initialValuesDF.write.format("delta").saveAsTable("target") // Here we use a source for INSERT that contains the same data that is already present in // the target, except that it uses a wider type. Since Uniform is enabled and Iceberg // doesn't support the given type change, we will keep the existing narrower type and // downcast values. `testCase.additionalValueDF` contains values that would overflow, // which would just fail, hence why we use `testCase.initialValueDF` instead. testCase.initialValuesDF .select(col("value").cast(testCase.toType)) .write .format("delta") .saveAsTable("source") enableIcebergUniform("target", IcebergCompatV2) withSQLConf( DeltaSQLConf.DELTA_STREAMING_SINK_ALLOW_IMPLICIT_CASTS.key -> "true" ) { insert.runInsert( columns = Seq("value"), whereCol = "value", whereValue = 1, withSchemaEvolution = true) } val result = sql(s"SELECT * FROM target") val expected = testCase.initialValuesDF.union(testCase.initialValuesDF) assert(result.schema("value").dataType === testCase.fromType) checkAnswer(result, expected) } } } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/uniform/IcebergCompatV2EnableUniformByAlterTableSuiteBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.uniform import org.apache.spark.sql.delta.{DeltaConfigs, DeltaLog, DeltaUnsupportedOperationException, Snapshot} import org.apache.spark.sql.delta.actions.AddFile import org.apache.parquet.hadoop.metadata.ParquetMetadata import org.apache.spark.sql.{DataFrame, QueryTest, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier trait IcebergCompatV2EnableUniformByAlterTableSuiteBase extends QueryTest { override protected def spark: SparkSession /** * Assert the invariance between old and new parquet footers, * this will check if the number of overlapped parquet footers * is the same as expected and extract the newer portion of footers, * i.e., the portion of parquet files present in `newParquetFooters` * but not in `oldParquetFooters`; or the overlapped portion of footers, * as specified by the flag `newerOrOverlapped`. * * This function is useful when, e.g., * - checking the invariance of parquet footers before and after * ALTER TABLE to enable UniForm, a portion of parquet footers * will stay the same, and the new portion of parquet footers * should be `IcebergCompatV2`. * - after running REORG UPGRADE UNIFORM, there may be a portion of * parquet files that do not need to be rewritten, and the number * should be the same as expected. * * @param oldParquetFooters the old version of parquet footers. * @param newParquetFooters the new version of parquet footers. * @param expectedNumOfOverlappedParquetFiles the expected number of overlapped parquet footers. * @param expectedNumOfAddedParquetFiles the expected number of added portion of parquet footers. * @return a pair consists of (overlapped parquet footers, added parquet footers). */ protected def assertInvarianceAndExtractParquetFooters( oldParquetFooters: Seq[ParquetMetadata], newParquetFooters: Seq[ParquetMetadata], expectedNumOfOverlappedParquetFiles: Int, expectedNumOfAddedParquetFiles: Int): (Seq[ParquetMetadata], Seq[ParquetMetadata]) = { val oldParquetFootersInStr = oldParquetFooters.map { _.toString } val newParquetFootersInStr = newParquetFooters.map { _.toString } val overlappedParquetFootersInStr = oldParquetFootersInStr.filter { footer => newParquetFootersInStr.contains(footer) } assert( overlappedParquetFootersInStr.length == expectedNumOfOverlappedParquetFiles, s"expect number of overlapped parquet footers to be $expectedNumOfOverlappedParquetFiles, " + s"but get ${overlappedParquetFootersInStr.length}" ) val addedParquetFootersInStr = newParquetFootersInStr.filter { footer => !oldParquetFootersInStr.contains(footer) } assert( addedParquetFootersInStr.length == expectedNumOfAddedParquetFiles, s"expect number of newer parquet footers to be $expectedNumOfAddedParquetFiles, " + s"but get ${addedParquetFootersInStr.length}" ) val overlappedParquetFooters = oldParquetFooters.filter { footer => overlappedParquetFootersInStr.contains(footer.toString) } val addedParquetFooters = newParquetFooters.filter { footer => addedParquetFootersInStr.contains(footer.toString) } (overlappedParquetFooters, addedParquetFooters) } /** * Assert the properties for old and new parquet footers. * Specifically, first check the number of overlapped and added parquet footers * to match with the expected numbers; * then extract and assert whether each should be considered `IcebergCompatV2` * by the expected values. * * @param oldParquetFooters the old version of parquet footers. * @param newParquetFooters the new version of parquet footers. * @param expectedNumOfOverlappedParquetFiles the expected number of overlapped parquet footers. * @param expectedNumOfAddedParquetFiles the expected number of added parquet footers. * @param isOverlappedIcebergCompatV2 whether the overlapped parquet footers is expected * to be `IcebergCompatV2`. * @param isAddedIcebergCompatV2 whether the added parquet footers is expected to be * `IcebergCompatV2`. */ protected def assertParquetFootersProperties( oldParquetFooters: Seq[ParquetMetadata], newParquetFooters: Seq[ParquetMetadata], expectedNumOfOverlappedParquetFiles: Int, expectedNumOfAddedParquetFiles: Int, isOverlappedIcebergCompatV2: Boolean, isAddedIcebergCompatV2: Boolean): Unit = { val (overlapped, added) = assertInvarianceAndExtractParquetFooters( oldParquetFooters = oldParquetFooters, newParquetFooters = newParquetFooters, expectedNumOfOverlappedParquetFiles = expectedNumOfOverlappedParquetFiles, expectedNumOfAddedParquetFiles = expectedNumOfAddedParquetFiles ) assert(isParquetFootersIcebergCompatV2(overlapped) == isOverlappedIcebergCompatV2) assert(isParquetFootersIcebergCompatV2(added) == isAddedIcebergCompatV2) } /** Check if `IcebergCompatV1` is enabled based on the provided snapshot */ protected def isIcebergCompatV1Enabled(snapshot: Snapshot): Boolean = { snapshot .getProperties(DeltaConfigs.ICEBERG_COMPAT_V1_ENABLED.key) .contains("true") } /** Check if `IcebergCompatV2` is enabled based on the provided snapshot */ protected def isIcebergCompatV2Enabled(snapshot: Snapshot): Boolean = { snapshot .getProperties(DeltaConfigs.ICEBERG_COMPAT_V2_ENABLED.key) .contains("true") } /** * Insert three initial rows to the specified table. * * @param id the table id used for insertion. */ protected def insertInitialRowsIntoTable(id: String): Unit = { executeSql( s""" | INSERT INTO TABLE $id | VALUES | (1, 'Alex', '2000-01-01'), | (1, 'Cat', '2001-01-01'), | (2, 'Michael', '2002-10-30') |""".stripMargin ) } /** * Insert two additional rows to the specified table. * * @param id the table id used for insertion. */ protected def insertAdditionalRowsIntoTable(id: String): Unit = { executeSql( s""" | INSERT INTO TABLE $id | VALUES | (3, 'Cat', '2003-01-01'), | (4, 'Cat', '2004-01-02') |""".stripMargin ) } /** * Create a vanilla delta table with a single partition column. * * @param id the table id used for creation. * @param loc the table location. */ protected def createVanillaDeltaTableWithDV(id: String, loc: String): Unit = { executeSql( s""" | CREATE TABLE $id (id INT, name STRING, date TIMESTAMP) | USING DELTA | PARTITIONED BY (id) | LOCATION $loc |""".stripMargin ) } /** * Create a vanilla delta table with DV disabled and a single partition column. * * @param id the table id used for creation. * @param loc the table location. */ protected def createVanillaDeltaTableWithoutDV(id: String, loc: String): Unit = { executeSql( s""" | CREATE TABLE $id (id INT, name STRING, date TIMESTAMP) | USING DELTA | PARTITIONED BY (id) | LOCATION $loc | TBLPROPERTIES ( | 'delta.enableDeletionVectors' = 'false', | 'delta.minReaderVersion' = '2', | 'delta.minWriterVersion' = '7' | ) |""".stripMargin ) } /** * Create an `IcebergCompatV1` uniform table with a single partition column. * * @param id the table id used for creation. * @param loc the table location. */ protected def createIcebergCompatV1Table(id: String, loc: String): Unit = { executeSql( s""" | CREATE TABLE $id (id INT, name STRING, date TIMESTAMP) | USING DELTA | PARTITIONED BY (id) | LOCATION $loc | TBLPROPERTIES ( | 'delta.enableIcebergCompatV1' = 'true', | 'delta.universalFormat.enabledFormats' = 'iceberg' | ) |""".stripMargin ) } /** * Create a delta table with liquid clustering enabled for a single column. * * @param id the table id used for creation. * @param loc the table location. */ protected def createLiquidDeltaTable(id: String, loc: String): Unit = { executeSql( s""" | CREATE TABLE $id (id INT, name STRING, date TIMESTAMP) | USING DELTA | CLUSTER BY (id) | LOCATION $loc |""".stripMargin ) } /** * Create a delta table with nested types and column-mapping enabled. * * @param id the table id used for creation. * @param loc the table location. */ protected def createDeltaTableWithNestedTypesAndColumnMapping(id: String, loc: String): Unit = { executeSql( s""" | CREATE TABLE $id ( | id INT, | listOfInt ARRAY, | listOfList ARRAY>, | listOfMap ARRAY>, | map MAP, | mapOfMap MAP>, | mapOfList MAP>, | struct STRUCT, | structOfStruct STRUCT>, | structOfListAndMap STRUCT, col3: MAP>) | USING DELTA | LOCATION $loc | TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name' | ) |""".stripMargin ) } /** * Insert a single row to the delta table with nested types and column-mapping enabled. * * @param id the table used for insertion. */ protected def insertRowToDeltaTableWithNestedTypesAndColumnMapping(id: String): Unit = { executeSql( s""" | INSERT INTO TABLE $id VALUES | (1, | ARRAY(1, 2, 3), | ARRAY(ARRAY(1, 2), ARRAY(3, 4)), | ARRAY(MAP(1, 'Alex'), MAP(2, 'Michael')), | MAP(1, 'Alex'), | MAP(1, MAP(2, 'Michael')), | MAP(3, ARRAY('Cat', 'Cat')), | STRUCT(2, 'Michael'), | STRUCT(1, STRUCT(2, 'Cat')), | STRUCT(1, ARRAY(4, 5, 6), MAP(7, 'Alex')) | ) |""".stripMargin ) } /** * Get all parquet footers of data files for the specified table. * * @param spark the spark session used to get the footers. * @param id the table id from which to get all the parquet footers. * @return all parquet metadata/footers of the parquet (data) files for this table. */ protected def getParquetFooters(spark: SparkSession, id: String): Seq[ParquetMetadata] = { val snapshot = DeltaLog.forTable(spark, new TableIdentifier(id)).update() val basePath = snapshot.path.getParent.toString + "/" val addFiles: Array[AddFile] = snapshot.allFiles.collect() val parquetPaths: Array[String] = addFiles.map { basePath + _.toPath.toString } parquetPaths.map { ParquetIcebergCompatV2Utils.getParquetFooter } } /** * Check whether the current parquet footers are all `IcebergCompatV2`. * This will check two properties for each parquet footer, * see [[ParquetIcebergCompatV2Utils.isParquetIcebergCompatV2]] for details. * * @param parquetFooters the parquet footers to be checked. * @return whether the footers are considered `IcebergCompatV2` */ protected def isParquetFootersIcebergCompatV2(parquetFooters: Seq[ParquetMetadata]): Boolean = { parquetFooters.forall { parquetFooter => ParquetIcebergCompatV2Utils.isParquetIcebergCompatV2(parquetFooter) } } /** The wrapper function to execute sql */ protected def executeSql(sqlStr: String): DataFrame /** The wrapper function to assert the protocol and properties for UniForm Iceberg */ protected def assertUniFormIcebergProtocolAndProperties(id: String): Unit /** The wrapper function to generate a temporary table and directory */ protected def withTempTableAndDir(f: (String, String) => Unit): Unit /** * Helper function to enforce the properties that an `IcebergCompatV2` * Delta Uniform requires. * e.g., disable DV, ensure reader/writer versions, enable column-mapping. * * @param id the table to be altered. */ protected def enforceDeltaUniformRequireProperties(id: String): Unit = { executeSql( s""" | ALTER TABLE $id | SET TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name', | 'delta.enableDeletionVectors' = 'false', | 'delta.minReaderVersion' = '2', | 'delta.minWriterVersion' = '7' | ) |""".stripMargin ) } /** * Enable `IcebergCompatV2` by ALTER TABLE command for the table. * * @param id the table id used for ALTER TABLE to enable `IcebergCompatV2`. */ protected def enableIcebergCompatV2ByAlterTable(id: String): Unit = { executeSql( s""" | ALTER TABLE $id | SET TBLPROPERTIES ( | 'delta.enableIcebergCompatV2' = 'true', | 'delta.universalFormat.enabledFormats' = 'iceberg' | ) |""".stripMargin ) } /** * The basic test case for enable uniform by ALTER TABLE, * this could be used as a prior setup for subsequent tests. * * @param id the table id. * @param loc the table location. */ protected def alterTableToEnableIcebergCompatV2BaseCase(id: String, loc: String): Unit = { createVanillaDeltaTableWithDV(id, loc) insertInitialRowsIntoTable(id) val parquetFooters1 = getParquetFooters(spark, id) enforceDeltaUniformRequireProperties(id) enableIcebergCompatV2ByAlterTable(id) insertAdditionalRowsIntoTable(id) val parquetFooters2 = getParquetFooters(spark, id) assertParquetFootersProperties( oldParquetFooters = parquetFooters1, newParquetFooters = parquetFooters2, expectedNumOfOverlappedParquetFiles = 2, expectedNumOfAddedParquetFiles = 2, isOverlappedIcebergCompatV2 = false, isAddedIcebergCompatV2 = true ) assertUniFormIcebergProtocolAndProperties(id) } test("Enable IcebergCompatV2 By ALTER TABLE Base Case") { withTempTableAndDir { case (id, loc) => alterTableToEnableIcebergCompatV2BaseCase(id, loc) } } test("Enable IcebergCompatV2 For Vanilla Table By ALTER TABLE With Deletion Vectors Disabled") { withTempTableAndDir { case (id, loc) => createVanillaDeltaTableWithoutDV(id, loc) insertInitialRowsIntoTable(id) executeSql( s""" | ALTER TABLE $id | SET TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name' | ) |""".stripMargin ) val parquetFooters1 = getParquetFooters(spark, id) // no need to manually disable DV enableIcebergCompatV2ByAlterTable(id) insertAdditionalRowsIntoTable(id) val parquetFooters2 = getParquetFooters(spark, id) assertParquetFootersProperties( oldParquetFooters = parquetFooters1, newParquetFooters = parquetFooters2, expectedNumOfOverlappedParquetFiles = 2, expectedNumOfAddedParquetFiles = 2, isOverlappedIcebergCompatV2 = false, isAddedIcebergCompatV2 = true ) assertUniFormIcebergProtocolAndProperties(id) } } test("Enable IcebergCompatV2 By ALTER TABLE With Purging Deletion Vectors") { withTempTableAndDir { case (id, loc) => createVanillaDeltaTableWithDV(id, loc) insertInitialRowsIntoTable(id) executeSql( s""" | DELETE FROM $id | WHERE name = 'Alex' |""".stripMargin ) // purge the current table before running ALTER TABLE // note: this may be different if `autoOptimize` has been enabled, // which will automatically rewrite the parquet file with DV when // the above DELETE FROM command is triggered. executeSql( s""" | REORG TABLE $id APPLY (PURGE) |""".stripMargin ) val parquetFooters1 = getParquetFooters(spark, id) enforceDeltaUniformRequireProperties(id) enableIcebergCompatV2ByAlterTable(id) insertAdditionalRowsIntoTable(id) val parquetFooters2 = getParquetFooters(spark, id) assertParquetFootersProperties( oldParquetFooters = parquetFooters1, newParquetFooters = parquetFooters2, expectedNumOfOverlappedParquetFiles = 2, expectedNumOfAddedParquetFiles = 2, isOverlappedIcebergCompatV2 = false, isAddedIcebergCompatV2 = true ) assertUniFormIcebergProtocolAndProperties(id) } } test("REORG UPGRADE UNIFORM Should Rewrite All Parquet Files To Be IcebergCompatV2") { withTempTableAndDir { case (id, loc) => alterTableToEnableIcebergCompatV2BaseCase(id, loc) // run the REORG UPGRADE UNIFORM command to rewrite the portion of // parquet files that are not `IcebergCompatV2`. executeSql( s""" | REORG TABLE $id APPLY | (UPGRADE UNIFORM (ICEBERG_COMPAT_VERSION = 2)) |""".stripMargin ) // now all the parquet files must be `IcebergCompatV2` val parquetFooters3 = getParquetFooters(spark, id) assert(isParquetFootersIcebergCompatV2(parquetFooters3)) assertUniFormIcebergProtocolAndProperties(id) } } // TODO: update this test when automatically disable V1 is supported when upgrading // from an existing `IcebergCompatV1` table. test("Manually Enable V2 and Disable V1 When Upgrading From an IcebergCompatV1 Table") { withTempTableAndDir { case (id, loc) => createIcebergCompatV1Table(id, loc) insertInitialRowsIntoTable(id) val parquetFooters1 = getParquetFooters(spark, id) val snapshot1 = DeltaLog.forTable(spark, new TableIdentifier(id)).update() assert( isIcebergCompatV1Enabled(snapshot1), "`IcebergCompatV1` should be enabled for the current table" ) enforceDeltaUniformRequireProperties(id) // enable `IcebergCompatV2` for the `IcebergCompatV1` table. // note that `IcebergCompatV1` needs to be disabled *manually* as for now. executeSql( s""" | ALTER TABLE $id | SET TBLPROPERTIES ( | 'delta.enableIcebergCompatV1' = 'false', | 'delta.enableIcebergCompatV2' = 'true', | 'delta.universalFormat.enabledFormats' = 'iceberg' | ) |""".stripMargin ) insertAdditionalRowsIntoTable(id) val parquetFooters2 = getParquetFooters(spark, id) assertParquetFootersProperties( oldParquetFooters = parquetFooters1, newParquetFooters = parquetFooters2, expectedNumOfOverlappedParquetFiles = 2, expectedNumOfAddedParquetFiles = 2, isOverlappedIcebergCompatV2 = true, isAddedIcebergCompatV2 = true ) val snapshot2 = DeltaLog.forTable(spark, new TableIdentifier(id)).update() assert( !isIcebergCompatV1Enabled(snapshot2), "`IcebergCompatV1` should be disabled after enabling `IcebergCompatV2` by ALTER TABLE" ) assert( isIcebergCompatV2Enabled(snapshot2), "`IcebergCompatV2` should be enabled after being enabled by ALTER TABLE" ) assertUniFormIcebergProtocolAndProperties(id) } } test("Enabling V1 and V2 At The Same Time For Vanilla Delta Table Should Fail") { withTempTableAndDir { case (id, loc) => createVanillaDeltaTableWithDV(id, loc) insertInitialRowsIntoTable(id) enforceDeltaUniformRequireProperties(id) val ex = intercept[DeltaUnsupportedOperationException]( executeSql( s""" | ALTER TABLE $id | SET TBLPROPERTIES ( | 'delta.enableIcebergCompatV1' = 'true', | 'delta.enableIcebergCompatV2' = 'true', | 'delta.universalFormat.enabledFormats' = 'iceberg' | ) |""".stripMargin ) ) assertResult( "DELTA_ICEBERG_COMPAT_VIOLATION.VERSION_MUTUAL_EXCLUSIVE" )(ex.getErrorClass) } } test("Enabling V1 and V2 At The Same Time For IcebergCompatV1 Delta Uniform Table Should Fail") { withTempTableAndDir { case (id, loc) => createIcebergCompatV1Table(id, loc) insertInitialRowsIntoTable(id) enforceDeltaUniformRequireProperties(id) val ex = intercept[DeltaUnsupportedOperationException]( executeSql( s""" | ALTER TABLE $id | SET TBLPROPERTIES ( | 'delta.enableIcebergCompatV1' = 'true', | 'delta.enableIcebergCompatV2' = 'true', | 'delta.universalFormat.enabledFormats' = 'iceberg' | ) |""".stripMargin ) ) assertResult( "DELTA_ICEBERG_COMPAT_VIOLATION.VERSION_MUTUAL_EXCLUSIVE" )(ex.getErrorClass) } } test("Disable Column-Mapping When Enabling IcebergCompatV2 By ALTER TABLE Should Fail") { withTempTableAndDir { case (id, loc) => createVanillaDeltaTableWithDV(id, loc) insertInitialRowsIntoTable(id) enforceDeltaUniformRequireProperties(id) // disable column-mapping when enabling `IcebergCompatV2` // delta uniform by ALTER TABLE should fail val ex = intercept[DeltaUnsupportedOperationException]( executeSql( s""" | ALTER TABLE $id | SET TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'none', | 'delta.enableIcebergCompatV2' = 'true', | 'delta.universalFormat.enabledFormats' = 'iceberg' | ) |""".stripMargin ) ) assertResult( "DELTA_ICEBERG_COMPAT_VIOLATION.WRONG_REQUIRED_TABLE_PROPERTY" )(ex.getErrorClass) } } test("Enable UniForm With LIST, MAP, and Column-Mapping Enabled By ALTER TABLE") { withTempTableAndDir { case (id, loc) => createDeltaTableWithNestedTypesAndColumnMapping(id, loc) insertRowToDeltaTableWithNestedTypesAndColumnMapping(id) val parquetFooters1 = getParquetFooters(spark, id) // only DV needs to be disabled here executeSql( s""" | ALTER TABLE $id | SET TBLPROPERTIES ( | 'delta.enableDeletionVectors' = 'false' | ) |""".stripMargin ) enableIcebergCompatV2ByAlterTable(id) insertRowToDeltaTableWithNestedTypesAndColumnMapping(id) val parquetFooters2 = getParquetFooters(spark, id) assertParquetFootersProperties( oldParquetFooters = parquetFooters1, newParquetFooters = parquetFooters2, expectedNumOfAddedParquetFiles = 1, expectedNumOfOverlappedParquetFiles = 1, isOverlappedIcebergCompatV2 = false, isAddedIcebergCompatV2 = true ) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/uniform/SparkSessionSwitch.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark import org.apache.spark.sql.SparkSession /** * Helper for easily switch between multiple sessions in test */ trait SparkSessionSwitch { private val knownSessions = collection.mutable.HashMap[SparkSession, (Option[SparkContext], SparkEnv)]() /** * Create a SparkSession and save its context. Calling this will not change * the current active SparkSession. Use [[withSession]] when you want to use * the newly created session. * * @param factory used to create the session * @return the newly created session */ def newSession(factory: => SparkSession): SparkSession = { registerActiveSession() val old = SparkSession.getActiveSession clear() val created = factory registerActiveSession() old.foreach(restore) created } /** * Execute code with the given session. * @param session session to use * @param thunk code to execute within the specified session */ def withSession[T](session: SparkSession)(thunk: SparkSession => T): T = { val oldSession = SparkSession.getActiveSession restore(session) val result = thunk(session) oldSession.foreach(restore) result } /** * Record the SparkContext/SparkEnv for current active session */ private def registerActiveSession(): Unit = { SparkSession.getActiveSession .foreach(knownSessions.put(_, (SparkContext.getActive, SparkEnv.get))) } /** * Restore the snapshot made for the given session * @param session the session to be restore */ private def restore(session: SparkSession): Unit = { val (restoreContext, restoreEnv) = knownSessions.getOrElse( session, throw new IllegalArgumentException("Unknown Session to restore")) SparkSession.setActiveSession(session) SparkSession.setDefaultSession(session) val oldContext = SparkContext.getActive SparkContext.clearActiveContext() restoreContext.foreach(SparkContext.setActiveContext) // Synchronize the context (oldContext, restoreContext) match { case (Some(off), Some(on)) => syncContext(off, on) case _ => } SparkEnv.set(restoreEnv) } /** * Clear the session related context. Necessary before creating new sessions */ private def clear(): Unit = { SparkSession.clearActiveSession() SparkSession.clearDefaultSession() SparkContext.clearActiveContext() SparkEnv.set(null) } /** * Synchronize local properties when switch SparkContext by merging * and overwriting from off to on * @param off the context to be deactivated * @param on the context to be activated */ private def syncContext(off: SparkContext, on: SparkContext): Unit = { // NOTE: cannot use putAll due to a problem of Scala2 + JDK9+ // See https://github.com/scala/bug/issues/10418 for detail val onProperties = on.localProperties.get() off.localProperties.get().forEach((k, v) => onProperties.put(k, v)) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/uniform/UniFormE2EIcebergSuiteBase.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.uniform import scala.collection.mutable import org.apache.spark.sql.delta._ import org.apache.spark.sql.Row import org.apache.spark.sql.types._ /** * This test suite base aims at testing the end-to-end behavior of UniForm. * It writes Delta tables, and reads the generated Iceberg tables to * perform verification. */ abstract class UniFormE2EIcebergSuiteBase extends UniFormE2ETest { val testTableName = "delta_table" var compatVersions: Seq[Int] = Seq(1, 2) def extraTableProperties(compatVersion: Int): String = { val extraProps = mutable.HashMap[String, String]() val compat = IcebergCompat.getForVersion(compatVersion) if (compat.incompatibleTableFeatures.contains(DeletionVectorsTableFeature)) { extraProps.put(DeltaConfigs.ENABLE_DELETION_VECTORS_CREATION.key, "false") } extraProps.map(pair => s", '${pair._1}' = '${pair._2}'").mkString(" ") } compatVersions.foreach { compatVersion => test(s"Basic Insert - compatV$compatVersion") { withTable(testTableName) { write( s"""CREATE TABLE $testTableName (col1 INT) USING DELTA |TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name', | 'delta.enableIcebergCompatV$compatVersion' = 'true', | 'delta.universalFormat.enabledFormats' = 'iceberg' | ${extraTableProperties(compatVersion)} |)""".stripMargin) write(s"INSERT INTO $testTableName VALUES (123)") readAndVerify(testTableName, "col1", "col1", Seq(Row(123))) } } } compatVersions.foreach { compatVersion => test(s"CIUD - compatV$compatVersion") { withTable(testTableName) { write( s"""CREATE TABLE `$testTableName` (col1 INT) USING DELTA |TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name', | 'delta.enableIcebergCompatV$compatVersion' = 'true', | 'delta.universalFormat.enabledFormats' = 'iceberg' | ${extraTableProperties(compatVersion)} |)""".stripMargin) write(s"INSERT INTO `$testTableName` VALUES (123),(456),(567),(331)") readAndVerify(testTableName, "col1", "col1", Seq(Row(123), Row(331), Row(456), Row(567))) write(s"UPDATE `$testTableName` SET col1 = 191 WHERE col1 = 567") readAndVerify(testTableName, "col1", "col1", Seq(Row(123), Row(191), Row(331), Row(456))) write(s"DELETE FROM `$testTableName` WHERE col1 = 456") readAndVerify(testTableName, "col1", "col1", Seq(Row(123), Row(191), Row(331))) } } } compatVersions.foreach { compatVersion => test(s"CTAS - compatV$compatVersion") { withTable(testTableName, "source") { write("CREATE TABLE source (col1 INT) USING DELTA") write("INSERT INTO source VALUES (1), (2), (3)") write( s"""CREATE TABLE `$testTableName` USING DELTA |TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name', | 'delta.enableIcebergCompatV$compatVersion' = 'true', | 'delta.universalFormat.enabledFormats' = 'iceberg' | ${extraTableProperties(compatVersion)} |) AS SELECT col1 FROM source""".stripMargin) readAndVerify(testTableName, "col1", "col1", Seq(Row(1), Row(2), Row(3))) write(s"UPDATE `$testTableName` SET col1 = 100 WHERE col1 = 1") readAndVerify(testTableName, "col1", "col1", Seq(Row(2), Row(3), Row(100))) write(s"DELETE FROM `$testTableName` WHERE col1 = 3") readAndVerify(testTableName, "col1", "col1", Seq(Row(2), Row(100))) } } } compatVersions.foreach { compatVersion => test(s"Table with partition - compatV$compatVersion") { withTable(testTableName) { write( s"""CREATE TABLE $testTableName (id INT, part STRING) USING delta |PARTITIONED BY (part) |TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name', | 'delta.enableIcebergCompatV$compatVersion' = 'true', | 'delta.universalFormat.enabledFormats' = 'iceberg' | ${extraTableProperties(compatVersion)} |)""".stripMargin) write(s"INSERT INTO `$testTableName` VALUES (123, 'p1'), (456, 'p2'), (789, 'p1')") readAndVerify(testTableName, "id, part", "id", Seq(Row(123, "p1"), Row(456, "p2"), Row(789, "p1"))) } } } compatVersions.foreach { compatVersion => test(s"Nested struct schema test - compatV$compatVersion") { withTable(testTableName) { write( s"""CREATE TABLE $testTableName | (col1 INT, col2 STRUCT | , f6: INT>, f7: INT>) USING DELTA |TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name', | 'delta.enableIcebergCompatV$compatVersion' = 'true', | 'delta.universalFormat.enabledFormats' = 'iceberg' | ${extraTableProperties(compatVersion)} |)""".stripMargin) val data = Seq( Row(1, Row(Row(2, Row(3, 4), 5), 6)) ) val innerStruct3 = StructType( StructField("f4", IntegerType) :: StructField("f5", IntegerType) :: Nil) val innerStruct2 = StructType( StructField("f2", IntegerType) :: StructField("f3", innerStruct3) :: StructField("f6", IntegerType) :: Nil) val innerStruct = StructType( StructField("f1", innerStruct2) :: StructField("f7", IntegerType) :: Nil) val schema = StructType( StructField("col1", IntegerType) :: StructField("col2", innerStruct) :: Nil) val tableFullName = tableNameForRead(testTableName) spark.createDataFrame(spark.sparkContext.parallelize(data), schema) .write.format("delta").mode("append") .saveAsTable(testTableName) readAndVerify(tableFullName, "col1, col2", "col1", data) } } } test("reorg from v1 to v2") { withTable(testTableName) { write( s"""CREATE TABLE $testTableName (col1 INT) USING DELTA |TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name', | 'delta.enableIcebergCompatV1' = 'true', | 'delta.universalFormat.enabledFormats' = 'iceberg', | 'delta.enableDeletionVectors' = 'false' |)""".stripMargin) write(s"INSERT INTO $testTableName VALUES (1)") readAndVerify(testTableName, "col1", "col1", Seq(Row(1))) write(s"ALTER TABLE `$testTableName` UNSET TBLPROPERTIES " + s"('delta.universalFormat.enabledFormats')") write(s""" | REORG TABLE $testTableName APPLY | (UPGRADE UNIFORM (ICEBERG_COMPAT_VERSION = 2)) |""".stripMargin) write(s"INSERT INTO $testTableName VALUES (2)") readAndVerify(testTableName, "col1", "col1", Seq(Row(1), Row(2))) } } // TODO createReaderSparkSession is no longer supported. // Please use readAndVerify and re-enable the cases /* test("Insert Partitioned Table") { val partitionColumns = Array( "str STRING", "i INTEGER", "l LONG", "s SHORT", "b BYTE", "dt DATE", "bin BINARY", "bool BOOLEAN", "ts_ntz TIMESTAMP_NTZ", "ts TIMESTAMP") val partitionValues: Array[Any] = Array( "'some_value'", 1, 1234567L, 1000, 119, "to_date('2016-12-31', 'yyyy-MM-dd')", "'asdf'", true, "TIMESTAMP_NTZ'2021-12-06 00:00:00'", "TIMESTAMP'2023-08-18 05:00:00UTC-7'" ) partitionColumns zip partitionValues map { partitionColumnsAndValues => val partitionColumnName = partitionColumnsAndValues._1.split(" ")(0) val tableName = testTableName + "_" + partitionColumnName withTable(tableName) { write( s"""CREATE TABLE $tableName (${partitionColumnsAndValues._1}, col1 INT) | USING DELTA | PARTITIONED BY ($partitionColumnName) | TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name', | 'delta.enableIcebergCompatV2' = 'true', | 'delta.universalFormat.enabledFormats' = 'iceberg' |)""".stripMargin) write(s"INSERT INTO $tableName VALUES (${partitionColumnsAndValues._2}, 123)") val verificationQuery = s"SELECT col1 FROM $tableName " + s"where ${partitionColumnName}=${partitionColumnsAndValues._2}" // Verify against Delta read and Iceberg read checkAnswer(spark.sql(verificationQuery), Seq(Row(123))) checkAnswer(createReaderSparkSession.sql(verificationQuery), Seq(Row(123))) } } } test("Insert Partitioned Table - Multiple Partitions") { withTable(testTableName) { write( s"""CREATE TABLE $testTableName (id int, ts timestamp, col1 INT) | USING DELTA | PARTITIONED BY (id, ts) | TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name', | 'delta.enableIcebergCompatV2' = 'true', | 'delta.universalFormat.enabledFormats' = 'iceberg' |)""".stripMargin) write(s"INSERT INTO $testTableName VALUES (1, TIMESTAMP'2023-08-18 05:00:00UTC-7', 123)") val verificationQuery = s"SELECT col1 FROM $testTableName " + s"where id=1 and ts=TIMESTAMP'2023-08-18 05:00:00UTC-7'" // Verify against Delta read and Iceberg read checkAnswer(spark.sql(verificationQuery), Seq(Row(123))) checkAnswer(createReaderSparkSession.sql(verificationQuery), Seq(Row(123))) } } test("Insert Partitioned Table - UTC Adjustment for Non-ISO Timestamp Partition values") { withTable(testTableName) { withTimeZone("GMT-8") { withSQLConf(UTC_TIMESTAMP_PARTITION_VALUES.key -> "false") { write( s"""CREATE TABLE $testTableName (id int, ts timestamp) | USING DELTA | PARTITIONED BY (ts) | TBLPROPERTIES ( | 'delta.columnMapping.mode' = 'name', | 'delta.enableIcebergCompatV2' = 'true', | 'delta.universalFormat.enabledFormats' = 'iceberg' |)""".stripMargin) write(s"INSERT INTO $testTableName" + s" VALUES (1, timestamp'2021-06-30 00:00:00.123456')") // Verify partition values in Delta Log val deltaLog = DeltaLog.forTable(spark, TableIdentifier(testTableName)) val partitionColName = deltaLog.unsafeVolatileMetadata.physicalPartitionColumns.head val partitionValues = deltaLog.update().allFiles.head.partitionValues assert(partitionValues === Map(partitionColName -> "2021-06-30 00:00:00.123456")) // Verify against Delta read and Iceberg read val verificationQuery = s"SELECT id FROM $testTableName " + s"where ts=TIMESTAMP'2021-06-30 08:00:00.123456UTC'" checkAnswer(spark.sql(verificationQuery), Seq(Row(1))) checkAnswer(createReaderSparkSession.sql(verificationQuery), Seq(Row(1))) } } } } */ } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/uniform/UniFormE2ETest.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.uniform import org.apache.spark.sql.{DataFrame, QueryTest, Row} import org.apache.spark.sql.test.SharedSparkSession /** * Base classes for all UniForm end-to-end test cases. Provides support to * write data with Delta SparkSession and read data for verification. * * People who need to write a new test suite should extend this class and * implement their test cases with [[write]] and [[readAndVerify]], which execute * with the writer and reader respectively. * * Implementing classes need to correctly set up the reader and writer environments. * See [[UniFormE2EIcebergSuiteBase]] for existing examples. */ trait UniFormE2ETest extends QueryTest with SharedSparkSession { /** * Execute write operations through the writer SparkSession * * @param sqlText write query to the UniForm table */ protected def write(sqlText: String): DataFrame = sql(sqlText) /** * Verify the result by reading from the reader session and compare the result to the expected. * * @param table write table name * @param fields fields to verify, separated by comma. E.g., "col1, col2" * @param orderBy fields to order the results, separated by comma. * @param expect expected result */ protected def readAndVerify( table: String, fields: String, orderBy: String, expect: Seq[Row]): Unit = throw new UnsupportedOperationException /** * Subclasses should override this method when the table name for reading * is different from the table name used for writing. For example, when we * write a table using the name `table1`, and then read it from another catalog * `catalog_read`, this method should return `catalog_read.default.table1` * for the input `table1`. * * @param tableName table name for writing (name only) * @return table name for reading, default is no translation */ protected def tableNameForRead(tableName: String): String = tableName } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/uniform/hms/EmbeddedHMS.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.uniform.hms import java.util.UUID import java.io.{BufferedReader, File, IOException, InputStreamReader} import java.net.ServerSocket import java.nio.file.Files import java.sql.{Connection, DriverManager} import org.apache.commons.io.FileUtils import org.apache.hadoop.conf.Configuration import org.apache.hadoop.hive.conf.HiveConf import org.apache.hadoop.hive.conf.HiveConf.ConfVars /** * EmbeddedHMS is an embedded Hive MetaStore for testing purposes. * Multiple EmbeddedHMS instances can be started in parallel on the same host * (see [[HMSTest]] for how to use it in the code). */ class EmbeddedHMS { private var server: HMSServer = _ private var whFolder: String = _ private var dbName: String = _ private var started = false private var port: Int = 0 /** * Generate a random suffix for HMS warehouse/metastore to keep * the directory unique for each suite if running concurrently. */ def randomSuffix: String = { UUID.randomUUID().toString } /** * Start an EmbeddedHMS instance */ def start(): Unit = { if (started) return port = EmbeddedHMS.firstAvailablePort() val dbFolder = Files.createTempDirectory("ehms_metastore_" + randomSuffix) Files.delete(dbFolder) // Derby needs the folder to be non-existent dbName = dbFolder.toString whFolder = Files.createTempDirectory("ehms_warehouse_" + randomSuffix).toString initDatabase(dbName) val innerConf = new HiveConf() innerConf.set(ConfVars.HIVE_IN_TEST.varname, "false") innerConf.set(ConfVars.METASTOREWAREHOUSE.varname, whFolder) innerConf.set(ConfVars.METASTORECONNECTURLKEY.varname, s"jdbc:derby:$dbName;create=true") server = new HMSServer(innerConf, port) server.start() started = true } /** * Stop the instance and cleanup its resources */ def stop(): Unit = { if (!started) return server.stop() // Cleanup on exit FileUtils.deleteDirectory(new File(dbName)) FileUtils.deleteDirectory(new File(whFolder)) started = false } /** * Fetch the configuration used for clients to connect to the MetaStore * @return conf containing thrift uri and warehouse location */ def conf(): Configuration = { if (!started) throw new IllegalStateException("Not started") val conf = new Configuration() conf.set(ConfVars.METASTOREWAREHOUSE.varname, whFolder) conf.set(ConfVars.METASTOREURIS.varname, s"thrift://localhost:$port") conf } /** * Load SQL scripts into Apache Derby instance to initialize the metastore * schema. The script used here is copied from HMS official repo. * @param dbFolder the folder to create the database, also the database name */ private def initDatabase(dbFolder: String): Unit = { // scalastyle:off classforname // Register the Derby JDBC Driver Class.forName("org.apache.derby.jdbc.EmbeddedDriver").getConstructor().newInstance() // scalastyle:on classforname val con = DriverManager.getConnection(s"jdbc:derby:$dbFolder;create=true") // May need to use another version when upgrading Hive dependencies executeScript(con, "hms/hive-schema-3.1.0.derby.sql") con.close() // Shutdown the Derby instance properly, allowing it to clean up. try { DriverManager.getConnection(s"jdbc:derby:$dbFolder;shutdown=true") } catch { // From Derby doc: // "A successful shutdown always results in an SQLException to indicate // that Derby has shut down and that there is no other exception." // We thus ignore the exception here. case _: java.sql.SQLException => } } /** * Execute sql scripts in the given resource file * @param con database connection * @param scriptFile the name of the resource location of the sql script */ private def executeScript(con: Connection, scriptFile: String): Unit = { val scriptIs = Thread.currentThread().getContextClassLoader.getResourceAsStream(scriptFile) if (scriptIs == null) { throw new RuntimeException("Make sure derby init script is in the classpath") } val reader = new BufferedReader(new InputStreamReader(scriptIs)) var line: String = reader.readLine val buffer: StringBuilder = new StringBuilder() val stmt = con.createStatement() while (line != null) { line match { case comment if comment.startsWith("--") => case eos if eos.endsWith(";") => if (buffer.nonEmpty) buffer.append("\n") buffer.append(eos) buffer.deleteCharAt(buffer.length - 1) // Remove semicolon stmt.addBatch(buffer.toString) buffer.clear case piece => if (buffer.nonEmpty) buffer.append("\n") buffer.append(piece) } line = reader.readLine() } reader.close() stmt.executeBatch() stmt.close() } } object EmbeddedHMS { var start = 9084 def firstAvailablePort(): Integer = this.synchronized { for (port <- start until 65536) { var ss: ServerSocket = null try { ss = new ServerSocket(port) ss.setReuseAddress(true) start = port + 1 return port } catch { case e: IOException => } finally { if (ss != null) { try ss.close() catch { case e: IOException => } } } } throw new RuntimeException("No port is available") } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/uniform/hms/HMSServer.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.uniform.hms import java.net.InetSocketAddress import org.apache.hadoop.hive.conf.HiveConf import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler import org.apache.hadoop.hive.metastore.RetryingHMSHandler import org.apache.hadoop.hive.metastore.TSetIpAddressProcessor import org.apache.thrift.protocol.{TBinaryProtocol, TProtocol, TProtocolFactory} import org.apache.thrift.server.{ServerContext, TServer, TServerEventHandler, TThreadPoolServer} import org.apache.thrift.transport.{TServerSocket, TTransport, TTransportFactory} /** * Start a Thrift Server that accepts standard HMS thrift client. * * @param conf including database connection and warehouse location * @param port the port this thrift server listens */ class HMSServer(val conf: HiveConf, val port: Int) { private var tServer: TServer = _ private var serverThread: MetastoreThread = _ def start(): Unit = { val maxMessageSize = 100L * 1024 * 1024 val protocolFactory: TProtocolFactory = new TBinaryProtocol.Factory val inputProtoFactory: TProtocolFactory = new TBinaryProtocol.Factory( true, true, maxMessageSize, maxMessageSize) val hmsHandler = new HMSHandler("default", conf) val handler = RetryingHMSHandler.getProxy(conf, hmsHandler, false) val transFactory = new TTransportFactory val processor = new TSetIpAddressProcessor(handler) val serverSocket = new TServerSocket(new InetSocketAddress(port)) val args = new TThreadPoolServer.Args(serverSocket) .processor(processor) .transportFactory(transFactory) .protocolFactory(protocolFactory) .inputProtocolFactory(inputProtoFactory) .minWorkerThreads(5) .maxWorkerThreads(5); tServer = new TThreadPoolServer(args); val tServerEventHandler = new TServerEventHandler() { override def preServe(): Unit = () override def createContext(tProtocol: TProtocol, tProtocol1: TProtocol): ServerContext = null override def deleteContext( serverContext: ServerContext, tProtocol: TProtocol, tProtocol1: TProtocol): Unit = { // If the IMetaStoreClient#close was called, HMSHandler#shutdown would have already // cleaned up thread local RawStore. Otherwise, do it now. HMSServer.cleanupRawStore() } override def processContext( serverContext: ServerContext, tTransport: TTransport, tTransport1: TTransport): Unit = () } tServer.setServerEventHandler(tServerEventHandler) serverThread = new MetastoreThread serverThread.start() // Wait till the server is up while (!tServer.isServing) { Thread.sleep(100) } } def stop(): Unit = { HMSServer.cleanupRawStore() tServer.stop() } /** * The metastore thrift server will run in this thread */ private class MetastoreThread extends Thread { super.setDaemon(true) super.setName("EmbeddedHMS Metastore Thread") override def run(): Unit = { tServer.serve() } } } object HMSServer { private val localConfField = classOf[HMSHandler].getDeclaredField("threadLocalConf") localConfField.setAccessible(true) private val localConf = localConfField.get().asInstanceOf[ThreadLocal[HiveConf]] private def cleanupRawStore(): Unit = { try { val rs = HMSHandler.getRawStore if (rs != null) { rs.shutdown() } } finally { HMSHandler.removeRawStore() localConf.remove() } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/uniform/hms/HMSTest.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.uniform.hms import java.util.concurrent.ConcurrentLinkedQueue import org.apache.hadoop.conf.Configuration import org.apache.hadoop.hive.conf.HiveConf.ConfVars._ import org.scalatest.{BeforeAndAfterAll, Suite} import org.apache.spark.SparkConf import org.apache.spark.sql.SparkSession object HMSPool { private val pool = new ConcurrentLinkedQueue[EmbeddedHMS]() private val maxInstances = 5 def acquire(): EmbeddedHMS = synchronized { while (pool.isEmpty && pool.size >= maxInstances) { wait() } if (pool.isEmpty) { new EmbeddedHMS() } else { pool.poll() } } def release(hms: EmbeddedHMS): Unit = synchronized { pool.offer(hms) notify() } } /** * Provide support to testcases that need to use HMS. */ trait HMSTest extends Suite with BeforeAndAfterAll { private var sharedHMS: EmbeddedHMS = _ def withMetaStore(thunk: (Configuration) => Unit): Unit = { val conf = sharedHMS.conf() thunk(conf) } protected override def beforeAll(): Unit = { sharedHMS = HMSPool.acquire() sharedHMS.start() super.beforeAll() } protected override def afterAll(): Unit = { super.afterAll() releaseHMS() } protected def releaseHMS(): Unit = { if (sharedHMS != null) { HMSPool.release(sharedHMS) sharedHMS = null } } protected def setupSparkConfWithHMS(in: SparkConf): SparkConf = { val conf = sharedHMS.conf() in.set("spark.sql.warehouse.dir", conf.get(METASTOREWAREHOUSE.varname)) .set("hive.metastore.uris", conf.get(METASTOREURIS.varname)) .set("spark.sql.catalogImplementation", "hive") } protected def createDeltaSparkSession: SparkSession = { val conf = sharedHMS.conf() val sparkSession = SparkSession.builder() .master("local[*]") .appName("DeltaSession") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") .config("spark.sql.warehouse.dir", conf.get(METASTOREWAREHOUSE.varname)) .config("hive.metastore.uris", conf.get(METASTOREURIS.varname)) .config("spark.sql.catalogImplementation", "hive") .getOrCreate() sparkSession } protected def createIcebergSparkSession: SparkSession = { val conf = sharedHMS.conf() val sparkSession = SparkSession.builder() .master("local[*]") .appName("IcebergSession") .config("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions") .config("spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.cache-enabled", "false") .config("spark.sql.warehouse.dir", conf.get(METASTOREWAREHOUSE.varname)) .config("hive.metastore.uris", conf.get(METASTOREURIS.varname)) .config("spark.sql.catalogImplementation", "hive") .getOrCreate() sparkSession } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/util/AnalysisHelperSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import org.apache.spark.sql.QueryTest import org.apache.spark.sql.test.SharedSparkSession class AnalysisHelperSuite extends QueryTest with SharedSparkSession { test("should not throw NullPointerException when Exception has null description") { class FakeAnalysisHelper extends AnalysisHelper { def throwInterruptedException(): Unit = super.improveUnsupportedOpError { throw new InterruptedException() } } // Should throw original exception assertThrows[InterruptedException] { new FakeAnalysisHelper {}.throwInterruptedException() } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/util/BinPackingIteratorSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import scala.collection.generic.Sizing import org.apache.spark.sql.QueryTest import org.apache.spark.sql.test.SharedSparkSession case class IntArrayImplementingSizing(array: Seq[Int]) extends Sizing { override def size: Int = array.size } case class TestArbitrarySizing(size: Int) extends Sizing class BinPackingIteratorSuite extends QueryTest with SharedSparkSession { test("Bin packing works") { val targetSize = 4 val testInput = Seq( IntArrayImplementingSizing(Seq(1, 2, 3)), IntArrayImplementingSizing(Seq(1, 2, 3))) val binPackingIterator = new BinPackingIterator(testInput.iterator, targetSize) assert(binPackingIterator.hasNext) var count = 0 for (bin <- binPackingIterator) { assert(bin.size === 1) assert(bin.toSeq === Seq(IntArrayImplementingSizing(Seq(1, 2, 3)))) count += 1 } assert(count === 2) } test("Bin packing can handle overflows to internal size tracking") { val targetSize = Int.MaxValue / 2 val testInput = Seq( // 1st bin TestArbitrarySizing(1), TestArbitrarySizing(1), TestArbitrarySizing(2), // 2nd bin TestArbitrarySizing(Int.MaxValue - 14), // 3rd bin TestArbitrarySizing(Int.MaxValue / 2), // 4th bin TestArbitrarySizing(Int.MaxValue / 2) ) val binPackingIterator = new BinPackingIterator(testInput.iterator, targetSize) assert(binPackingIterator.hasNext) val firstBin = binPackingIterator.next() assert(firstBin.size === 3) assert(firstBin.toSeq === Seq( TestArbitrarySizing(1), TestArbitrarySizing(1), TestArbitrarySizing(2))) assert(binPackingIterator.hasNext) val secondBin = binPackingIterator.next() assert(secondBin.size === 1) assert(secondBin.toSeq === Seq(TestArbitrarySizing(Int.MaxValue - 14))) assert(binPackingIterator.hasNext) val thirdBin = binPackingIterator.next() assert(thirdBin.size === 1) assert(thirdBin.toSeq === Seq(TestArbitrarySizing(Int.MaxValue / 2))) assert(binPackingIterator.hasNext) val fourthBin = binPackingIterator.next() assert(fourthBin.size === 1) assert(fourthBin.toSeq === Seq(TestArbitrarySizing(Int.MaxValue / 2))) assert(!binPackingIterator.hasNext) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/util/BinPackingUtilsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import org.apache.spark.SparkFunSuite class BinPackingUtilsSuite extends SparkFunSuite { test("test bin-packing") { val binSize = 5 val cases = Seq[(Seq[Int], Seq[Seq[Int]])]( (Seq(1, 2, 3, 4, 5), Seq(Seq(1, 2), Seq(3), Seq(4), Seq(5))), (Seq(5, 4, 3, 2, 1), Seq(Seq(1, 2), Seq(3), Seq(4), Seq(5))), // Naive coalescing returns 5 bins where sort-then-coalesce gets 4. (Seq(4, 2, 4, 2, 5), Seq(Seq(2, 2), Seq(4), Seq(4), Seq(5))), // The last element exceeds binSize and it's in its own bin. (Seq(1, 2, 4, 5, 6), Seq(Seq(1, 2), Seq(4), Seq(5), Seq(6)))) for ((input, expect) <- cases) { assert(BinPackingUtils.binPackBySize(input, (x: Int) => x, (x: Int) => x, binSize) == expect) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/util/BitmapAggregatorE2ESuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import java.io.{File, IOException} import java.net.URI import java.nio.{ByteBuffer, ByteOrder} import java.nio.file.Files import org.apache.spark.sql.catalyst.expressions.aggregation.BitmapAggregator import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.deletionvectors.{PortableRoaringBitmapArraySerializationFormat, RoaringBitmapArray, RoaringBitmapArrayFormat} import org.apache.spark.sql.delta.test.DeltaSQLTestUtils import org.apache.spark.sql.{Column, QueryTest} import org.apache.spark.sql.test.SharedSparkSession class BitmapAggregatorE2ESuite extends QueryTest with SharedSparkSession with DeltaSQLTestUtils { import BitmapAggregatorE2ESuite._ import testImplicits._ import org.apache.spark.sql.functions._ for (serializationFormat <- RoaringBitmapArrayFormat.values) { test(s"DataFrame bitmap groupBy aggregate no duplicates - $serializationFormat") { dataFrameBitmapGroupByAggregateWithoutDuplicates(format = serializationFormat) } } for (serializationFormat <- RoaringBitmapArrayFormat.values) { test("DataFrame bitmap groupBy aggregate no duplicates - invalid Int ids" + s" - $serializationFormat") { dataFrameBitmapGroupByAggregateWithoutDuplicates( offset = INVALID_INT_OFFSET, format = serializationFormat) } } for (serializationFormat <- RoaringBitmapArrayFormat.values) { test("DataFrame bitmap groupBy aggregate no duplicates - invalid unsigned Int ids" + s" - $serializationFormat") { dataFrameBitmapGroupByAggregateWithoutDuplicates( offset = UNSIGNED_INT_OFFSET, format = serializationFormat) } } private def dataFrameBitmapGroupByAggregateWithoutDuplicates( offset: Long = 0L, format: RoaringBitmapArrayFormat.Value): Unit = { val baseDF = spark .range(DATASET_SIZE) .map { id => val newId = id + offset // put 2 adjacent and one with gap (newId % 6) match { case 0 | 1 | 4 => ("file1" -> newId) case 2 | 3 | 5 => ("file2" -> newId) } } .toDF("file", "id") .cache() val bitmapAgg = bitmapAggColumn(baseDF("id"), format) val aggregationOutput = baseDF .groupBy("file") .agg(bitmapAgg) .as[(String, (Long, Long, Array[Byte]))] .collect() .toMap .mapValues(v => RoaringBitmapArray.readFrom(v._3)) val dfFile1 = baseDF .select("id") .where("file = 'file1'") .as[Long] .collect() val dfFile2 = baseDF .select("id") .where("file = 'file2'") .as[Long] .collect() assertEqualContents(aggregationOutput("file1"), dfFile1) assertEqualContents(aggregationOutput("file2"), dfFile2) baseDF.unpersist() } for (serializationFormat <- RoaringBitmapArrayFormat.values) { test("DataFrame bitmap groupBy aggregate with duplicates" + s" - $serializationFormat") { dataFrameBitmapGroupAggregateWithDuplicates(format = serializationFormat) } } for (serializationFormat <- RoaringBitmapArrayFormat.values) { test("DataFrame bitmap groupBy aggregate with duplicates - invalid Int ids" + s" - $serializationFormat") { dataFrameBitmapGroupAggregateWithDuplicates( offset = INVALID_INT_OFFSET, format = serializationFormat) } } for (serializationFormat <- RoaringBitmapArrayFormat.values) { test("DataFrame bitmap groupBy aggregate with duplicates - invalid unsigned Int ids" + s" - $serializationFormat") { dataFrameBitmapGroupAggregateWithDuplicates( offset = UNSIGNED_INT_OFFSET, format = serializationFormat) } } def dataFrameBitmapGroupAggregateWithDuplicates( offset: Long = 0L, format: RoaringBitmapArrayFormat.Value) { val baseDF = spark .range(DATASET_SIZE) .flatMap { id => val newId = id + offset // put two adjacent and duplicate the one after a gap (newId % 6) match { case 0 | 1 => Seq("file1" -> newId) case 2 | 3 => Seq("file2" -> newId) case 4 => Seq("file1" -> newId, "file1" -> newId) // duplicate in file1 case 5 => Seq("file2" -> newId, "file2" -> newId) // duplicate in file2 } } .toDF("file", "id") .cache() val bitmapAgg = bitmapAggColumn(baseDF("id"), format) // scalastyle:off countstring val aggregationOutput = baseDF .groupBy("file") .agg(bitmapAgg, count("id")) .as[(String, (Long, Long, Array[Byte]), Long)] .collect() .map(t => (t._1 -> (RoaringBitmapArray.readFrom(t._2._3), t._3))) .toMap // scalastyle:on countstring val dfFile1 = baseDF .select("id") .where("file = 'file1'") .distinct() .as[Long] .collect() val dfFile2 = baseDF .select("id") .where("file = 'file2'") .distinct() .as[Long] .collect() val file1Value = aggregationOutput("file1") assert(file1Value._2 > file1Value._1.cardinality) val file2Value = aggregationOutput("file2") assert(file2Value._2 > file2Value._1.cardinality) assertEqualContents(file1Value._1, dfFile1) assertEqualContents(file2Value._1, dfFile2) } // modulo ordering private def assertEqualContents(aggregator: RoaringBitmapArray, dataset: Array[Long]): Unit = { // make sure they are in the same order val aggregatorArray = aggregator.values.sorted assert(aggregatorArray === dataset.sorted) } } object BitmapAggregatorE2ESuite { // Pick something large enough hat 2 files have at least 64k entries each final val DATASET_SIZE: Long = 1000000L // Cross the `isValidInt` threshold final val INVALID_INT_OFFSET: Long = Int.MaxValue.toLong - DATASET_SIZE / 2 // Cross the 32bit threshold final val UNSIGNED_INT_OFFSET: Long = (1L << 32) - DATASET_SIZE / 2 private[delta] def bitmapAggColumn( column: Column, format: RoaringBitmapArrayFormat.Value): Column = { val func = new BitmapAggregator(expression(column), format); Column(func.toAggregateExpression(isDistinct = false)) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/util/CatalogTableTestUtils.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} import org.apache.spark.sql.types.StructType import scala.jdk.CollectionConverters._ /** * Helpers for constructing [[CatalogTable]] instances inside Java tests. * * Spark's [[CatalogTable]] is defined in Scala and its constructor signature shifts between Spark * releases. Centralising the construction in Scala keeps the kernel tests insulated from those * binary changes and saves Java tests from manually wiring the many optional parameters. */ object CatalogTableTestUtils { /** * Creates a [[CatalogTable]] with configurable options. * * @param tableName table name (default: "tbl") * @param catalogName optional catalog name for the identifier * @param properties table properties (default: empty) * @param storageProperties storage properties (default: empty) * @param locationUri optional storage location URI * @param nullStorage if true, sets storage to null (for edge case testing) * @param nullStorageProperties if true, sets storage properties to null */ def createCatalogTable( tableName: String = "tbl", catalogName: Option[String] = None, properties: java.util.Map[String, String] = new java.util.HashMap[String, String](), storageProperties: java.util.Map[String, String] = new java.util.HashMap[String, String](), locationUri: Option[java.net.URI] = None, nullStorage: Boolean = false, nullStorageProperties: Boolean = false): CatalogTable = { val scalaProps = properties.asScala.toMap val scalaStorageProps = if (nullStorageProperties) null else storageProperties.asScala.toMap val identifier = catalogName match { case Some(catalog) => TableIdentifier(tableName, Some("default"), Some(catalog)) case None => TableIdentifier(tableName) } val storage = if (nullStorage) { null } else { CatalogStorageFormat( locationUri = locationUri, inputFormat = None, outputFormat = None, serde = None, compressed = false, properties = scalaStorageProps) } CatalogTable( identifier = identifier, tableType = CatalogTableType.MANAGED, storage = storage, schema = new StructType(), provider = None, partitionColumnNames = Seq.empty, bucketSpec = None, properties = scalaProps) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/util/CodecSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import java.nio.charset.StandardCharsets.US_ASCII import java.util.UUID import scala.util.Random import org.apache.spark.SparkFunSuite class CodecSuite extends SparkFunSuite { import CodecSuite._ // Z85 reference strings are generated by https://cryptii.com/pipes/z85-encoder val testUuids = Seq[(UUID, String)]( new UUID(0L, 0L) -> "00000000000000000000", new UUID(Long.MinValue, Long.MinValue) -> "Fb/MH00000Fb/MH00000", new UUID(-1L, -1L) -> "%nSc0%nSc0%nSc0%nSc0", new UUID(0L, Long.MinValue) -> "0000000000Fb/MH00000", new UUID(0L, -1L) -> "0000000000%nSc0%nSc0", new UUID(0L, Long.MaxValue) -> "0000000000Fb/MG%nSc0", new UUID(Long.MinValue, 0L) -> "Fb/MH000000000000000", new UUID(-1L, 0L) -> "%nSc0%nSc00000000000", new UUID(Long.MaxValue, 0L) -> "Fb/MG%nSc00000000000", new UUID(0L, 1L) -> "00000000000000000001", // Just a few random ones, using literals for test determinism new UUID(-4124158004264678669L, -6032951921472435211L) -> "-(5oirYA.yTvx6v@H:L>", new UUID(6453181356142382984L, 8208554093199893996L) -> "s=Mlx-0Pp@AQ6uw@k6=D", new UUID(6453181356142382984L, -8208554093199893996L) -> "s=Mlx-0Pp@JUL=R13LuL", new UUID(-4124158004264678669L, 8208554093199893996L) -> "-(5oirYA.yAQ6uw@k6=D") // From https://rfc.zeromq.org/spec/32/ - Test Case test("Z85 spec reference value") { val inputBytes: Array[Byte] = Array(0x86, 0x4F, 0xD2, 0x6F, 0xB5, 0x59, 0xF7, 0x5B).map(_.toByte) val expectedEncodedString = "HelloWorld" val actualEncodedString = Codec.Base85Codec.encodeBytes(inputBytes) assert(actualEncodedString === expectedEncodedString) val outputBytes = Codec.Base85Codec.decodeAlignedBytes(actualEncodedString) assert(outputBytes sameElements inputBytes) } test("Z85 reference implementation values") { for ((id, expectedEncodedString) <- testUuids) { val actualEncodedString = Codec.Base85Codec.encodeUUID(id) assert(actualEncodedString === expectedEncodedString) } } test("Z85 spec character map") { assert(Codec.Base85Codec.ENCODE_MAP.length === 85) val referenceBytes = Seq( 0x00, 0x09, 0x98, 0x62, 0x0f, 0xc7, 0x99, 0x43, 0x1f, 0x85, 0x9a, 0x24, 0x2f, 0x43, 0x9b, 0x05, 0x3f, 0x01, 0x9b, 0xe6, 0x4e, 0xbf, 0x9c, 0xc7, 0x5e, 0x7d, 0x9d, 0xa8, 0x6e, 0x3b, 0x9e, 0x89, 0x7d, 0xf9, 0x9f, 0x6a, 0x8d, 0xb7, 0xa0, 0x4b, 0x9d, 0x75, 0xa1, 0x2c, 0xad, 0x33, 0xa2, 0x0d, 0xbc, 0xf1, 0xa2, 0xee, 0xcc, 0xaf, 0xa3, 0xcf, 0xdc, 0x6d, 0xa4, 0xb0, 0xec, 0x2b, 0xa5, 0x91, 0xfb, 0xe9, 0xa6, 0x72) .map(_.toByte).toArray val referenceString = new String(Codec.Base85Codec.ENCODE_MAP, US_ASCII) val encodedString = Codec.Base85Codec.encodeBytes(referenceBytes) assert(encodedString === referenceString) val decodedBytes = Codec.Base85Codec.decodeAlignedBytes(encodedString) assert(decodedBytes sameElements referenceBytes) } test("Reject illegal Z85 input - unaligned string") { // Minimum string should 5 characters val illegalEncodedString = "abc" assertThrows[IllegalArgumentException] { Codec.Base85Codec.decodeBytes( illegalEncodedString, // This value is irrelevant, any value should cause the failure. outputLength = 3) } } // scalastyle:off nonascii test(s"Reject illegal Z85 input - illegal character") { for (char <- Seq[Char]('î', 'π', '"', 0x7F)) { val illegalEncodedString = String.valueOf(Array[Char]('a', 'b', char, 'd', 'e')) val ex = intercept[IllegalArgumentException] { Codec.Base85Codec.decodeAlignedBytes(illegalEncodedString) } assert(ex.getMessage.contains("Input is not valid Z85")) } } // scalastyle:on nonascii test("base85 codec uuid roundtrips") { for ((id, _) <- testUuids) { val encodedString = Codec.Base85Codec.encodeUUID(id) // 16 bytes always get encoded into 20 bytes with Base85. assert(encodedString.length === Codec.Base85Codec.ENCODED_UUID_LENGTH) val decodedId = Codec.Base85Codec.decodeUUID(encodedString) assert(id === decodedId, s"encodedString = $encodedString") } } test("base85 codec empty byte array") { val empty = Array.empty[Byte] val encodedString = Codec.Base85Codec.encodeBytes(empty) assert(encodedString === "") val decodedArray = Codec.Base85Codec.decodeAlignedBytes(encodedString) assert(decodedArray.isEmpty) val decodedArray2 = Codec.Base85Codec.decodeBytes(encodedString, 0) assert(decodedArray2.isEmpty) } test("base85 codec byte array random roundtrips") { val rand = new Random(1L) // Fixed seed for determinism val arrayLengths = (1 to 20) ++ Seq(32, 56, 64, 128, 1022, 11 * 1024 * 1024) for (len <- arrayLengths) { val inputArray: Array[Byte] = Array.ofDim(len) rand.nextBytes(inputArray) val encodedString = Codec.Base85Codec.encodeBytes(inputArray) val decodedArray = Codec.Base85Codec.decodeBytes(encodedString, len) assert(decodedArray === inputArray, s"encodedString = $encodedString") } } /** * Execute `thunk` works for strings containing any of the possible base85 characters at either * beginning, middle, or end positions. */ private def forAllEncodedStrings(thunk: String => Unit): Unit = { // Basically test that every possible character can occur at any // position with a 20 character string. val characterString = new String(Codec.Base85Codec.ENCODE_MAP, US_ASCII) // Use this to fill in the remaining 17 characters. val fillerChar = "x" var count = 0 for { firstChar <- characterString middleChar <- characterString finalChar <- characterString } { val sb = new StringBuilder sb += firstChar sb ++= fillerChar * 9 sb += middleChar sb ++= fillerChar * 8 sb += finalChar val encodedString = sb.toString() assert(encodedString.length === 20) thunk(encodedString) count += 1 } assert(count === 85 * 85 * 85) } test("base85 character set is JSON-safe") { forAllEncodedStrings { inputString => val inputObject = JsonRoundTripContainer(inputString) val jsonString = JsonUtils.toJson(inputObject) assert(jsonString.contains(inputString), "Some character from the input had to be escaped to be JSON-safe:" + s"input = '$inputString' vs JSON = '$jsonString'") val outputObject = JsonUtils.fromJson[JsonRoundTripContainer](jsonString) val outputString = outputObject.data assert(inputString === outputString) } } } object CodecSuite { final case class JsonRoundTripContainer(data: String) } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/util/DatasetRefCacheSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import org.apache.spark.sql.{QueryTest, SparkSession} import org.apache.spark.sql.test.SharedSparkSession class DatasetRefCacheSuite extends QueryTest with SharedSparkSession { test("should create a new Dataset when the active session is changed") { val cache = new DatasetRefCache(() => spark.range(1, 10) ) val ref = cache.get // Should reuse `Dataset` when the active session is the same assert(ref eq cache.get) SparkSession.setActiveSession(spark.newSession()) // Should create a new `Dataset` when the active session is changed assert(ref ne cache.get) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/util/DeltaLogGroupingIteratorSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import org.apache.spark.sql.delta.SerializableFileStatus import org.apache.spark.SparkFunSuite class DeltaLogGroupingIteratorSuite extends SparkFunSuite { test("DeltaLogGroupingIterator") { val paths = Seq( // both checkpoint and commit file present for v1 "file://a/b/_delta_log/1.checkpoint.parquet", "file://a/b/_delta_log/1.json", // only json file present for v2 "file://a/b/_delta_log/2.json", // v3 missing // multiple types of checkpoint present for v4 "file://a/b/_delta_log/4.checkpoint.parquet", "file://a/b/_delta_log/4.checkpoint.uuid.parquet", "file://a/b/_delta_log/4.checkpoint.json.parquet", "file://a/b/_delta_log/4.checkpoint.0.1.parquet", "file://a/b/_delta_log/4.checkpoint.1.1.parquet", "file://a/b/_delta_log/4.json", // v5, v6 with single checkpoint file "file://a/b/_delta_log/5.checkpoint.parquet", "file://a/b/_delta_log/5.json", "file://a/b/_delta_log/6.checkpoint.parquet", // no checkpoint files in the end "file://a/b/_delta_log/6.json", "file://a/b/_delta_log/7.json", "file://a/b/_delta_log/8.json", "file://a/b/_delta_log/9.json", "file://a/b/_delta_log/11.checkpoint.0.1.parquet", "file://a/b/_delta_log/11.checkpoint.1.1.parquet", "file://a/b/_delta_log/11.checkpoint.uuid.parquet", "file://a/b/_delta_log/12.checkpoint.parquet", "file://a/b/_delta_log/14.json" ) val fileStatuses = paths.map { path => SerializableFileStatus(path, length = 10, isDir = false, modificationTime = 1).toFileStatus }.toIterator val groupedFileStatuses = new DeltaLogGroupingIterator(fileStatuses) val groupedPaths = groupedFileStatuses.toIndexedSeq.map { case (version, files) => (version, files.map(_.getPath.toString).toList) } assert(groupedPaths === Seq( 1 -> List("file://a/b/_delta_log/1.checkpoint.parquet", "file://a/b/_delta_log/1.json"), 2 -> List("file://a/b/_delta_log/2.json"), 4 -> List( "file://a/b/_delta_log/4.checkpoint.parquet", "file://a/b/_delta_log/4.checkpoint.uuid.parquet", "file://a/b/_delta_log/4.checkpoint.json.parquet", "file://a/b/_delta_log/4.checkpoint.0.1.parquet", "file://a/b/_delta_log/4.checkpoint.1.1.parquet", "file://a/b/_delta_log/4.json"), 5 -> List("file://a/b/_delta_log/5.checkpoint.parquet", "file://a/b/_delta_log/5.json"), 6 -> List("file://a/b/_delta_log/6.checkpoint.parquet", "file://a/b/_delta_log/6.json"), 7 -> List("file://a/b/_delta_log/7.json"), 8 -> List("file://a/b/_delta_log/8.json"), 9 -> List("file://a/b/_delta_log/9.json"), 11 -> List( "file://a/b/_delta_log/11.checkpoint.0.1.parquet", "file://a/b/_delta_log/11.checkpoint.1.1.parquet", "file://a/b/_delta_log/11.checkpoint.uuid.parquet"), 12 -> List("file://a/b/_delta_log/12.checkpoint.parquet"), 14 -> List("file://a/b/_delta_log/14.json") )) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/util/JsonUtilsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util import scala.util.Random import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.actions.CommitInfo import org.apache.spark.sql.delta.stats.DataSize import com.fasterxml.jackson.core.StreamReadConstraints import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.test.SharedSparkSession class JsonUtilsSuite extends QueryTest with SharedSparkSession { test("DataSize json serialization") { val testCases = Seq( DataSize() -> """{}""", DataSize(bytesCompressed = Some(816L)) -> """{"bytesCompressed":816}""", DataSize(rows = Some(111L)) -> """{"rows":111}""", DataSize(rows = Some(0)) -> """{"rows":0}""", DataSize(logicalRows = Some(111L)) -> """{"logicalRows":111}""", DataSize(logicalRows = Some(-1L)) -> """{"logicalRows":-1}""", DataSize(bytesCompressed = Some(816L), rows = Some(111L), logicalRows = Some(111L)) -> """{"bytesCompressed":816,"rows":111,"logicalRows":111}""" ) for ((obj, json) <- testCases) { assert(JsonUtils.toJson(obj) == json) assert(JsonUtils.fromJson[DataSize](json) == obj) } } test("Serialize and de-serialize commit info with large message") { val operationStringSize = StreamReadConstraints.DEFAULT_MAX_STRING_LEN * 10 assert(operationStringSize > StreamReadConstraints.DEFAULT_MAX_STRING_LEN) val operation = Random.alphanumeric.take(operationStringSize).toString() val commitInfo = CommitInfo( time = System.currentTimeMillis(), operation, operationParameters = Map.empty, commandContext = Map.empty, readVersion = Some(1), isolationLevel = None, isBlindAppend = Some(false), operationMetrics = None, userMetadata = Some("I am a test and not a user"), tags = None, txnId = Some("Transaction with a veryyyyyyy large commit info") ) val serialized = JsonUtils.toJson(commitInfo) val deserialized = JsonUtils.fromJson[CommitInfo](serialized) assert(commitInfo === deserialized) } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/util/threads/DeltaThreadPoolSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util.threads import java.util.Properties import org.apache.spark.{SparkFunSuite, TaskContext, TaskContextImpl} import org.apache.spark.sql.test.SharedSparkSession class DeltaThreadPoolSuite extends SparkFunSuite with SharedSparkSession { val threadPool: DeltaThreadPool = DeltaThreadPool("test", 1) def makeTaskContext(id: Int): TaskContext = { new TaskContextImpl(id, 0, 0, 0, attemptNumber = 45613, 0, null, new Properties(), null) } def testForwarding(testName: String, id: Int)(f: => Unit): Unit = { test(testName) { val prevTaskContext = TaskContext.get() TaskContext.setTaskContext(makeTaskContext(id)) sparkContext.setLocalProperty("test", id.toString) try { f } finally { TaskContext.setTaskContext(prevTaskContext) } } } def assertTaskAndProperties(id: Int): Unit = { assert(TaskContext.get() !== null) assert(TaskContext.get().stageId() === id) assert(sparkContext.getLocalProperty("test") === id.toString) } testForwarding("parallelMap captures TaskContext", id = 0) { threadPool.parallelMap(spark, 0 until 1) { _ => assertTaskAndProperties(id = 0) } } testForwarding("submit captures TaskContext and local properties", id = 1) { threadPool.submit(spark) { assertTaskAndProperties(id = 1) } } testForwarding("submitNonFateSharing captures TaskContext and local properties", id = 2) { threadPool.submitNonFateSharing { _ => assertTaskAndProperties(id = 2) } } } ================================================ FILE: spark/src/test/scala/org/apache/spark/sql/delta/util/threads/SparkThreadLocalForwardingSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.util.threads import java.util.Properties import java.util.concurrent.{LinkedBlockingQueue, ThreadPoolExecutor, TimeUnit} import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future} import scala.concurrent.duration._ import org.apache.spark._ import org.apache.spark.util.ThreadUtils import org.apache.spark.util.ThreadUtils.namedThreadFactory class SparkThreadLocalForwardingSuite extends SparkFunSuite { private def createThreadPool(nThreads: Int, prefix: String): ThreadPoolExecutor = { val threadFactory = namedThreadFactory(prefix) val keepAliveTimeSeconds = 60 val threadPool = new SparkThreadLocalForwardingThreadPoolExecutor( nThreads, nThreads, keepAliveTimeSeconds, TimeUnit.MILLISECONDS, new LinkedBlockingQueue[Runnable], threadFactory) threadPool.allowCoreThreadTimeOut(true) threadPool } test("SparkThreadLocalForwardingThreadPoolExecutor properly propagates" + " TaskContext and Spark Local Properties") { val sc = SparkContext.getOrCreate(new SparkConf().setAppName("test").setMaster("local")) val executor = createThreadPool(1, "test-threads") implicit val executionContext: ExecutionContextExecutor = ExecutionContext.fromExecutor(executor) val prevTaskContext = TaskContext.get() try { // assert that each instance of submitting a task to the execution context captures the // current task context val futures = (1 to 10) map { i => setTaskAndProperties(i, sc) Future { checkTaskAndProperties(i, sc) }(executionContext) } assert(ThreadUtils.awaitResult(Future.sequence(futures), 10.seconds).forall(identity)) } finally { ThreadUtils.shutdown(executor) TaskContext.setTaskContext(prevTaskContext) sc.stop() } } def makeTaskContext(id: Int): TaskContext = { new TaskContextImpl(id, 0, 0, 0, attemptNumber = 45613, 0, null, new Properties(), null) } def setTaskAndProperties(i: Int, sc: SparkContext = SparkContext.getActive.get): Unit = { val tc = makeTaskContext(i) TaskContext.setTaskContext(tc) sc.setLocalProperty("test", i.toString) } def checkTaskAndProperties(i: Int, sc: SparkContext = SparkContext.getActive.get): Boolean = { TaskContext.get() != null && TaskContext.get().stageId() == i && sc.getLocalProperty("test") == i.toString } test("That CapturedSparkThreadLocals properly restores the existing state") { val sc = SparkContext.getOrCreate(new SparkConf().setAppName("test").setMaster("local")) val prevTaskContext = TaskContext.get() try { setTaskAndProperties(10) val capturedSparkThreadLocals = CapturedSparkThreadLocals() setTaskAndProperties(11) assert(!checkTaskAndProperties(10, sc)) assert(checkTaskAndProperties(11, sc)) capturedSparkThreadLocals.runWithCaptured { assert(checkTaskAndProperties(10, sc)) } assert(checkTaskAndProperties(11, sc)) } finally { TaskContext.setTaskContext(prevTaskContext) sc.stop() } } test("That CapturedSparkThreadLocals properly restores the existing spark properties." + " Changes to local properties inside a task do not affect the original properties") { val sc = SparkContext.getOrCreate(new SparkConf().setAppName("test").setMaster("local")) try { sc.setLocalProperty("TestProp", "1") val capturedSparkThreadLocals = CapturedSparkThreadLocals() assert(sc.getLocalProperty("TestProp") == "1") capturedSparkThreadLocals.runWithCaptured { sc.setLocalProperty("TestProp", "2") assert(sc.getLocalProperty("TestProp") == "2") } assert(sc.getLocalProperty("TestProp") == "1") } finally { sc.stop() } } test("captured spark thread locals are immutable") { val sc = SparkContext.getOrCreate(new SparkConf().setAppName("test").setMaster("local")) try { sc.setLocalProperty("test1", "good") sc.setLocalProperty("test2", "good") val threadLocals = CapturedSparkThreadLocals() sc.setLocalProperty("test2", "bad") assert(sc.getLocalProperty("test1") == "good") assert(sc.getLocalProperty("test2") == "bad") threadLocals.runWithCaptured { assert(sc.getLocalProperty("test1") == "good") assert(sc.getLocalProperty("test2") == "good") sc.setLocalProperty("test1", "bad") sc.setLocalProperty("test2", "maybe") assert(sc.getLocalProperty("test1") == "bad") assert(sc.getLocalProperty("test2") == "maybe") } assert(sc.getLocalProperty("test1") == "good") assert(sc.getLocalProperty("test2") == "bad") threadLocals.runWithCaptured { assert(sc.getLocalProperty("test1") == "good") assert(sc.getLocalProperty("test2") == "good") } } finally { sc.stop() } } } ================================================ FILE: spark/src/test/scala-shims/spark-4.0/GridTestShim.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test.shims import org.scalatest.Tag import org.apache.spark.SparkFunSuite /** * Shim for SparkFunSuite as gridTest doesn't exist in Spark 4.0 but we rely on it * in tests. */ trait GridTestShim { self: SparkFunSuite => def gridTest[A](testNamePrefix: String, testTags: Tag*)(params: Seq[A])( testFun: A => Unit): Unit = { for (param <- params) { test(testNamePrefix + s" ($param)", testTags: _*)(testFun(param)) } } } ================================================ FILE: spark/src/test/scala-shims/spark-4.0/InvalidDefaultValueErrorShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test.shims /** * Test shim for INVALID_DEFAULT_VALUE error codes that changed between Spark versions. * In Spark 4.0, the error code is INVALID_DEFAULT_VALUE.NOT_CONSTANT */ object InvalidDefaultValueErrorShims { val INVALID_DEFAULT_VALUE_ERROR_CODE: String = "INVALID_DEFAULT_VALUE.NOT_CONSTANT" } ================================================ FILE: spark/src/test/scala-shims/spark-4.0/StreamingTestShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test.shims /** * Test shims for streaming classes that were relocated in Spark 4.1. * In Spark 4.0, these classes are in their original locations. */ object StreamingTestShims { // MemoryStream type MemoryStream[T] = org.apache.spark.sql.execution.streaming.MemoryStream[T] val MemoryStream: org.apache.spark.sql.execution.streaming.MemoryStream.type = org.apache.spark.sql.execution.streaming.MemoryStream // MicroBatchExecution (class only, no companion object) type MicroBatchExecution = org.apache.spark.sql.execution.streaming.MicroBatchExecution // StreamingQueryWrapper (class only, no companion object) type StreamingQueryWrapper = org.apache.spark.sql.execution.streaming.StreamingQueryWrapper // StreamingExecutionRelation (class only, no companion object) type StreamingExecutionRelation = org.apache.spark.sql.execution.streaming.StreamingExecutionRelation // OffsetSeqLog (class only, no companion object) type OffsetSeqLog = org.apache.spark.sql.execution.streaming.OffsetSeqLog } ================================================ FILE: spark/src/test/scala-shims/spark-4.0/UnsupportedTableOperationErrorShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test.shims /** * Test shim for UNSUPPORTED_FEATURE.TABLE_OPERATION error codes that changed between Spark * versions. In Spark 4.0, the error code is _LEGACY_ERROR_TEMP_2096 */ object UnsupportedTableOperationErrorShims { val UNSUPPORTED_TABLE_OPERATION_ERROR_CODE: String = "_LEGACY_ERROR_TEMP_2096" /** * Returns the parameters map for UPDATE TABLE error in Spark 4.0 * @param tableSQLIdentifier Ignored in Spark 4.0, kept for API compatibility */ def updateTableErrorParameters(tableSQLIdentifier: String = ""): Map[String, String] = { Map("ddl" -> "UPDATE TABLE") } } ================================================ FILE: spark/src/test/scala-shims/spark-4.0/VariantShreddingTestShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test.shims /** * Test shim for variant shredding to handle differences between Spark versions. * In Spark 4.0, VARIANT_INFER_SHREDDING_SCHEMA does not exist. */ object VariantShreddingTestShims { /** * Returns true if VARIANT_INFER_SHREDDING_SCHEMA config is supported in this Spark version. * In Spark 4.0, this returns false. */ val variantInferShreddingSchemaSupported: Boolean = false /** * Returns a dummy config key for VARIANT_INFER_SHREDDING_SCHEMA. * In Spark 4.0, since this config doesn't exist, we return a dummy key that won't affect tests. * This allows tests to compile but the config will have no effect. */ val variantInferShreddingSchemaKey: String = "spark.sql.dummy.variantInferShreddingSchema" } ================================================ FILE: spark/src/test/scala-shims/spark-4.1/GridTestShim.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test.shims import org.apache.spark.SparkFunSuite /** * Shim for SparkFunSuite as gridTest doesn't exist in Spark 4.0 but we rely on it * in tests. In Spark 4.1 it exists so we don't need to do anything. */ trait GridTestShim { self: SparkFunSuite => } ================================================ FILE: spark/src/test/scala-shims/spark-4.1/InvalidDefaultValueErrorShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test.shims /** * Test shim for INVALID_DEFAULT_VALUE error codes that changed between Spark versions. * In Spark 4.1, the error code is INVALID_DEFAULT_VALUE.UNRESOLVED_EXPRESSION */ object InvalidDefaultValueErrorShims { val INVALID_DEFAULT_VALUE_ERROR_CODE: String = "INVALID_DEFAULT_VALUE.UNRESOLVED_EXPRESSION" } ================================================ FILE: spark/src/test/scala-shims/spark-4.1/StreamingTestShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test.shims /** * Test shims for streaming classes that were relocated in Spark 4.1. * In Spark 4.1, these classes moved to new package locations. */ object StreamingTestShims { // MemoryStream - moved to runtime package type MemoryStream[T] = org.apache.spark.sql.execution.streaming.runtime.MemoryStream[T] val MemoryStream: org.apache.spark.sql.execution.streaming.runtime.MemoryStream.type = org.apache.spark.sql.execution.streaming.runtime.MemoryStream // MicroBatchExecution - moved to runtime package (class only, no companion object) type MicroBatchExecution = org.apache.spark.sql.execution.streaming.runtime.MicroBatchExecution // StreamingQueryWrapper - moved to runtime package (class only, no companion object) type StreamingQueryWrapper = org.apache.spark.sql.execution.streaming.runtime.StreamingQueryWrapper // StreamingExecutionRelation - moved to runtime package (class only, no companion object) type StreamingExecutionRelation = org.apache.spark.sql.execution.streaming.runtime.StreamingExecutionRelation // OffsetSeqLog - moved to checkpointing package (class only, no companion object) type OffsetSeqLog = org.apache.spark.sql.execution.streaming.checkpointing.OffsetSeqLog } ================================================ FILE: spark/src/test/scala-shims/spark-4.1/UnsupportedTableOperationErrorShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test.shims /** * Test shim for UNSUPPORTED_FEATURE.TABLE_OPERATION error codes that changed between * Spark versions. In Spark 4.1, the error code is UNSUPPORTED_FEATURE.TABLE_OPERATION */ object UnsupportedTableOperationErrorShims { val UNSUPPORTED_TABLE_OPERATION_ERROR_CODE: String = "UNSUPPORTED_FEATURE.TABLE_OPERATION" /** * Returns the parameters map for UPDATE TABLE error in Spark 4.1 * @param tableSQLIdentifier The table identifier (e.g., "test_delta_table") */ def updateTableErrorParameters(tableSQLIdentifier: String): Map[String, String] = { // Construct the full table name with catalog prefix val fullTableName = s"`spark_catalog`.`default`.`$tableSQLIdentifier`" Map( "tableName" -> fullTableName, "operation" -> "UPDATE TABLE") } } ================================================ FILE: spark/src/test/scala-shims/spark-4.1/VariantShreddingTestShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test.shims import org.apache.spark.sql.internal.SQLConf /** * Test shim for variant shredding to handle differences between Spark versions. * In Spark 4.1, VARIANT_INFER_SHREDDING_SCHEMA exists. */ object VariantShreddingTestShims { /** * Returns true if VARIANT_INFER_SHREDDING_SCHEMA config is supported in this Spark version. * In Spark 4.1, this returns true. */ val variantInferShreddingSchemaSupported: Boolean = true /** * Returns the config key for VARIANT_INFER_SHREDDING_SCHEMA. * In Spark 4.1, this returns the actual SQLConf key. */ val variantInferShreddingSchemaKey: String = SQLConf.VARIANT_INFER_SHREDDING_SCHEMA.key } ================================================ FILE: spark/src/test/scala-shims/spark-4.2/GridTestShim.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test.shims import org.apache.spark.SparkFunSuite /** * Shim for SparkFunSuite as gridTest doesn't exist in Spark 4.0 but we rely on it * in tests. In Spark 4.2 it exists (same as 4.1) so we don't need to do anything. */ trait GridTestShim { self: SparkFunSuite => } ================================================ FILE: spark/src/test/scala-shims/spark-4.2/InvalidDefaultValueErrorShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test.shims /** * Test shim for INVALID_DEFAULT_VALUE error codes that changed between Spark versions. * In Spark 4.2, the error code is INVALID_DEFAULT_VALUE.NOT_CONSTANT */ object InvalidDefaultValueErrorShims { val INVALID_DEFAULT_VALUE_ERROR_CODE: String = "INVALID_DEFAULT_VALUE.NOT_CONSTANT" } ================================================ FILE: spark/src/test/scala-shims/spark-4.2/StreamingTestShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test.shims /** * Test shims for streaming classes that were relocated in Spark 4.1+. * In Spark 4.2, these classes remain in the same package locations as Spark 4.1. */ object StreamingTestShims { // MemoryStream - moved to runtime package type MemoryStream[T] = org.apache.spark.sql.execution.streaming.runtime.MemoryStream[T] val MemoryStream: org.apache.spark.sql.execution.streaming.runtime.MemoryStream.type = org.apache.spark.sql.execution.streaming.runtime.MemoryStream // MicroBatchExecution - moved to runtime package (class only, no companion object) type MicroBatchExecution = org.apache.spark.sql.execution.streaming.runtime.MicroBatchExecution // StreamingQueryWrapper - moved to runtime package (class only, no companion object) type StreamingQueryWrapper = org.apache.spark.sql.execution.streaming.runtime.StreamingQueryWrapper // StreamingExecutionRelation - moved to runtime package (class only, no companion object) type StreamingExecutionRelation = org.apache.spark.sql.execution.streaming.runtime.StreamingExecutionRelation // OffsetSeqLog - moved to checkpointing package (class only, no companion object) type OffsetSeqLog = org.apache.spark.sql.execution.streaming.checkpointing.OffsetSeqLog } ================================================ FILE: spark/src/test/scala-shims/spark-4.2/UnsupportedTableOperationErrorShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test.shims /** * Test shim for UNSUPPORTED_FEATURE.TABLE_OPERATION error codes that changed between * Spark versions. In Spark 4.2, the error code is UNSUPPORTED_FEATURE.TABLE_OPERATION * (same as Spark 4.1) */ object UnsupportedTableOperationErrorShims { val UNSUPPORTED_TABLE_OPERATION_ERROR_CODE: String = "UNSUPPORTED_FEATURE.TABLE_OPERATION" /** * Returns the parameters map for UPDATE TABLE error in Spark 4.2 (same as Spark 4.1) * @param tableSQLIdentifier The table identifier (e.g., "test_delta_table") */ def updateTableErrorParameters(tableSQLIdentifier: String): Map[String, String] = { // Construct the full table name with catalog prefix val fullTableName = s"`spark_catalog`.`default`.`$tableSQLIdentifier`" Map( "tableName" -> fullTableName, "operation" -> "UPDATE TABLE") } } ================================================ FILE: spark/src/test/scala-shims/spark-4.2/VariantShreddingTestShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test.shims import org.apache.spark.sql.internal.SQLConf /** * Test shim for variant shredding to handle differences between Spark versions. * In Spark 4.2, VARIANT_INFER_SHREDDING_SCHEMA exists. */ object VariantShreddingTestShims { /** * Returns true if VARIANT_INFER_SHREDDING_SCHEMA config is supported in this Spark version. * In Spark 4.2, this returns true. */ val variantInferShreddingSchemaSupported: Boolean = true /** * Returns the config key for VARIANT_INFER_SHREDDING_SCHEMA. * In Spark 4.2, this returns the actual SQLConf key. */ val variantInferShreddingSchemaKey: String = SQLConf.VARIANT_INFER_SHREDDING_SCHEMA.key } ================================================ FILE: spark/unitycatalog/src/test/java/io/sparkuctest/S3CredentialFileSystem.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.sparkuctest; import static org.assertj.core.api.Assertions.assertThat; import java.io.FileNotFoundException; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RawLocalFileSystem; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Progressable; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; /** * Fake S3 filesystem backed by local disk for integration testing. Maps {@code s3://bucket/path} to * local paths while verifying UC-vended credentials are correctly propagated. */ public class S3CredentialFileSystem extends RawLocalFileSystem { private static final String SCHEME = "s3:"; // Same as org.apache.hadoop.fs.s3a.Constants#AWS_CREDENTIALS_PROVIDER. private static final String S3A_CREDENTIALS_PROVIDER = "fs.s3a.aws.credentials.provider"; /** Set to {@code false} to skip credential assertions (useful for debugging). */ public static boolean credentialCheckEnabled = true; private AwsCredentialsProvider provider; @Override protected void checkPath(Path path) { // Accept any path without validation. } @Override public FSDataOutputStream create( Path f, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { return super.create(toLocalPath(f), overwrite, bufferSize, replication, blockSize, progress); } @Override public FileStatus getFileStatus(Path f) throws IOException { if (!f.toString().startsWith(SCHEME)) return super.getFileStatus(f); return restoreS3Path(f, super.getFileStatus(toLocalPath(f))); } @Override public FSDataInputStream open(Path f) throws IOException { return super.open(toLocalPath(f)); } @Override public FileStatus[] listStatus(Path f) throws IOException { FileStatus[] files; try { files = super.listStatus(toLocalPath(f)); } catch (FileNotFoundException e) { return new FileStatus[0]; // S3 returns empty for non-existent prefixes } FileStatus[] result = new FileStatus[files.length]; for (int i = 0; i < files.length; i++) { result[i] = restoreS3Path(f, files[i]); } return result; } @Override public boolean mkdirs(Path f, FsPermission permission) throws IOException { return super.mkdirs(toLocalPath(f), permission); } @Override public boolean rename(Path src, Path dst) throws IOException { return super.rename(toLocalPath(src), toLocalPath(dst)); } @Override public boolean delete(Path f, boolean recursive) throws IOException { return super.delete(toLocalPath(f), recursive); } /** Converts {@code s3://bucket/path} to local path, verifying credentials on the way. */ private Path toLocalPath(Path f) { checkCredentials(f); return new Path(f.toString().replaceAll(SCHEME + "//.*?/", "file:///")); } /** Replaces the file: scheme in a FileStatus with the original S3 prefix. */ private FileStatus restoreS3Path(Path originalS3Path, FileStatus status) { String s3Prefix = SCHEME + "//" + originalS3Path.toUri().getHost(); String restored = status.getPath().toString().replace("file:", s3Prefix); return new FileStatus( status.getLen(), status.isDirectory(), status.getReplication(), status.getBlockSize(), status.getModificationTime(), new Path(restored)); } private void checkCredentials(Path f) { if (!credentialCheckEnabled) return; String bucket = f.toUri().getHost(); assertThat(bucket).isEqualTo(UnityCatalogSupport.FAKE_S3_BUCKET); assertCredentials(); } /** Verifies UC-vended credentials via AwsCredentialsProvider or static Hadoop properties. */ private void assertCredentials() { Configuration conf = getConf(); AwsCredentialsProvider p = resolveProvider(conf); if (p != null) { AwsSessionCredentials creds = (AwsSessionCredentials) p.resolveCredentials(); assertThat(creds.accessKeyId()).isEqualTo("fakeAccessKey"); assertThat(creds.secretAccessKey()).isEqualTo("fakeSecretKey"); assertThat(creds.sessionToken()).isEqualTo("fakeSessionToken"); } else { assertThat(conf.get("fs.s3a.access.key")).isEqualTo("fakeAccessKey"); assertThat(conf.get("fs.s3a.secret.key")).isEqualTo("fakeSecretKey"); assertThat(conf.get("fs.s3a.session.token")).isEqualTo("fakeSessionToken"); } } private synchronized AwsCredentialsProvider resolveProvider(Configuration conf) { if (provider != null) return provider; String clazz = conf.get(S3A_CREDENTIALS_PROVIDER); if (clazz == null) return null; try { provider = (AwsCredentialsProvider) Class.forName(clazz).getConstructor(Configuration.class).newInstance(conf); } catch (Exception e) { throw new RuntimeException("Failed to instantiate credential provider: " + clazz, e); } return provider; } } ================================================ FILE: spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaStreamingTest.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.sparkuctest; import static org.junit.jupiter.api.Assertions.*; import io.unitycatalog.client.ApiClient; import io.unitycatalog.client.ApiException; import io.unitycatalog.client.api.DeltaCommitsApi; import io.unitycatalog.client.api.TablesApi; import io.unitycatalog.client.model.DeltaGetCommits; import io.unitycatalog.client.model.DeltaGetCommitsResponse; import io.unitycatalog.client.model.TableInfo; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.LongStream; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.delta.test.shims.StreamingTestShims; import org.apache.spark.sql.streaming.StreamingQuery; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.Metadata; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import scala.collection.JavaConverters; import scala.collection.immutable.Seq; /** * Streaming test suite for Delta Lake operations through Unity Catalog. * *

Tests structured streaming write and read operations with Delta format tables managed by Unity * Catalog. */ public class UCDeltaStreamingTest extends UCDeltaTableIntegrationBaseTest { /** * Creates a local temporary directory for checkpoint location. Checkpoint must be on local * filesystem since Spark doesn't have direct cloud storage credentials (credentials are managed * by UC server for catalog-managed tables only). */ public String createTempCheckpointDir() { try { return Files.createTempDirectory("spark-checkpoint-").toFile().getAbsolutePath(); } catch (IOException e) { throw new UncheckedIOException(e); } } @TestAllTableTypes public void testStreamingWriteToManagedTable(TableType tableType) throws Exception { withNewTable( "streaming_write_test", "id BIGINT, value STRING", tableType, (tableName) -> { // Define schema for the stream StructType schema = new StructType( new StructField[] { new StructField("id", DataTypes.LongType, false, Metadata.empty()), new StructField("value", DataTypes.StringType, false, Metadata.empty()) }); // Create MemoryStream - using Scala companion object with proper encoder via shims var memoryStream = StreamingTestShims.MemoryStream().apply(Encoders.row(schema), spark().sqlContext()); // Start streaming query writing to the Unity Catalog managed table StreamingQuery query = memoryStream .toDF() .writeStream() .format("delta") .outputMode("append") .option("checkpointLocation", createTempCheckpointDir()) .toTable(tableName); // Assert that the query is active assertTrue(query.isActive(), "Streaming query should be active"); // Let's do 3 rounds testing, and for every round, adding 1 row and waiting to be // available, and finally verify the results and unity catalog latest version are // expected. ApiClient client = unityCatalogInfo().createApiClient(); for (long i = 1; i <= 3; i += 1) { Seq batchRow = createRowsAsSeq(RowFactory.create(i, String.valueOf(i))); memoryStream.addData(batchRow); // Process all available data query.processAllAvailable(); // Verify the content check( tableName, LongStream.range(1, i + 1) .mapToObj(idx -> List.of(String.valueOf(idx), String.valueOf(idx))) .collect(Collectors.toUnmodifiableList())); // The UC server should have the latest version, for managed table. if (TableType.MANAGED == tableType) { assertUCManagedTableVersion(i, tableName, client); } } // Stop the stream. query.stop(); query.awaitTermination(); // Assert that the query has stopped assertFalse(query.isActive(), "Streaming query should have stopped"); }); } @TestAllTableTypes public void testStreamingReadFromTable(TableType tableType) throws Exception { String uniqueTableName = "streaming_read_test_" + UUID.randomUUID().toString().replace("-", ""); withNewTable( uniqueTableName, "id BIGINT, value STRING", tableType, (tableName) -> { SparkSession spark = spark(); String queryName = "uc_streaming_read_" + tableType.name().toLowerCase() + "_" + UUID.randomUUID().toString().replace("-", ""); StreamingQuery query = null; try { List> expected = new ArrayList<>(); // Seed an initial commit (required for managed tables, harmless for external). spark.sql(String.format("INSERT INTO %s VALUES (0, 'seed')", tableName)).collect(); expected.add(List.of("0", "seed")); Dataset input = spark.readStream().table(tableName); // Start the streaming query into a memory sink query = input .writeStream() .format("memory") .queryName(queryName) .option("checkpointLocation", createTempCheckpointDir()) .outputMode("append") .start(); assertTrue(query.isActive(), "Streaming query should be active"); // Write a few batches and verify the stream consumes them. for (long i = 1; i <= 3; i += 1) { String value = "value_" + i; spark .sql(String.format("INSERT INTO %s VALUES (%d, '%s')", tableName, i, value)) .collect(); query.processAllAvailable(); // Validate by checking if query and expected match. expected.add(List.of(String.valueOf(i), value)); check(queryName, expected); } } finally { if (query != null) { // TODO: remove additional processAllAvailable once interrupt is handled gracefully query.processAllAvailable(); query.awaitTermination(10000); query.stop(); assertFalse(query.isActive(), "Streaming query should have stopped"); } spark.sql("DROP VIEW IF EXISTS " + queryName); } }); } private void assertUCManagedTableVersion(long expectedVersion, String tableName, ApiClient client) throws ApiException { // Get the table info. TablesApi tablesApi = new TablesApi(client); TableInfo tableInfo = tablesApi.getTable(tableName, false, false); // Get the latest UC commit version. DeltaCommitsApi deltaCommitsApi = new DeltaCommitsApi(client); DeltaGetCommitsResponse resp = deltaCommitsApi.getCommits( new DeltaGetCommits().tableId(tableInfo.getTableId()).startVersion(0L)); assertNotNull(resp, "DeltaGetCommits response should not be null"); assertNotNull(resp.getLatestTableVersion(), "Latest table version should not be null"); // The UC server should have the latest version. assertEquals(expectedVersion, resp.getLatestTableVersion()); } private static Seq createRowsAsSeq(Row... rows) { return JavaConverters.asScalaIteratorConverter(Arrays.asList(rows).iterator()) .asScala() .toSeq(); } } ================================================ FILE: spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableBlockMetadataUpdateTest.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.sparkuctest; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.Test; /** * Tests that schema-changing and property-changing operations are blocked on Unity Catalog managed * (CatalogOwned) tables — regardless of which layer does the blocking. * *

There are two distinct layers of protection: * *

    *
  1. Kill switch in {@code OptimisticTransaction.updateMetadata()}: blocks any * commit that would change schema, partitions, description, or configuration on an existing * CatalogOwned table. A second guard in {@code prepareCommit()} and {@code commitLarge()} * blocks {@code delta.clustering} {@code DomainMetadata} changes (e.g. via RESTORE TABLE to a * version written by an older client that had different clustering columns). *
  2. UC catalog layer in {@code UCSingleCatalog}: {@code alterTable()} throws * {@code UnsupportedOperationException} for all ALTER TABLE variants. INSERT OVERWRITE with * {@code overwriteSchema=true} and {@code CREATE OR REPLACE TABLE} both route through REPLACE * TABLE AS SELECT (RTAS) because {@code UCSingleCatalog} does not implement {@code * StagingTableCatalog}; RTAS is not supported in OSS Delta. *
* *

EXTERNAL tables are not CatalogOwned and are NOT affected by the kill switch; they continue to * allow schema evolution as before. */ public class UCDeltaTableBlockMetadataUpdateTest extends UCDeltaTableIntegrationBaseTest { // Error produced by the kill switch in OptimisticTransaction.updateMetadata(). private static final String KILL_SWITCH_ERROR = "Metadata changes on Unity Catalog managed tables"; // Error produced by the clustering kill switch in commitLarge(). private static final String CLUSTERING_KILL_SWITCH_ERROR = "Clustering column changes on Unity Catalog managed tables"; // Error produced by UCSingleCatalog.alterTable() for all ALTER TABLE variants. private static final String ALTER_TABLE_ERROR = "Altering a table is not supported yet"; // Error produced by OSS Delta when REPLACE TABLE AS SELECT (RTAS) is attempted. // Triggered by CREATE OR REPLACE TABLE and DataFrame saveAsTable(overwrite+overwriteSchema) // when the target catalog does not implement StagingTableCatalog. private static final String RTAS_ERROR = "REPLACE TABLE AS SELECT (RTAS) is not supported"; // --------------------------------------------------------------------------- // Kill-switch tests: operations blocked by OptimisticTransaction.updateMetadata() // --------------------------------------------------------------------------- /** * INSERT with {@code autoMerge=true} and MERGE INTO with schema evolution must be blocked by the * kill switch in {@code updateMetadata()} on CatalogOwned tables. The kill switch covers all * metadata fields (schema, partitions, description, properties); schema evolution via autoMerge * is the primary user-facing path that reaches it without going through ALTER TABLE. */ @Test public void testMetadataChangesViaWritesAreBlocked() throws Exception { withNewTable( "block_schema_evolution_target", "id INT, name STRING", TableType.MANAGED, targetTable -> { sql("INSERT INTO %s VALUES (1, 'initial')", targetTable); withNewTable( "block_schema_evolution_source", "id INT, name STRING, extra STRING", TableType.EXTERNAL, sourceTable -> { sql("INSERT INTO %s VALUES (2, 'new', 'extra_value')", sourceTable); sql("SET spark.databricks.delta.schema.autoMerge.enabled = true"); try { // INSERT with autoMerge introduces a new column. assertThrowsWithCauseContaining( KILL_SWITCH_ERROR, () -> sql("INSERT INTO %s SELECT * FROM %s", targetTable, sourceTable)); // MERGE INTO with autoMerge introduces a new column from the source. assertThrowsWithCauseContaining( KILL_SWITCH_ERROR, () -> sql( "MERGE INTO %s AS target " + "USING %s AS source " + "ON target.id = source.id " + "WHEN NOT MATCHED THEN INSERT *", targetTable, sourceTable)); } finally { sql("SET spark.databricks.delta.schema.autoMerge.enabled = false"); } }); }); } // --------------------------------------------------------------------------- // UC catalog layer tests: operations blocked by UCSingleCatalog before reaching Delta // --------------------------------------------------------------------------- /** * All ALTER TABLE variants on a CatalogOwned table must be blocked by {@code * UCSingleCatalog.alterTable()}, which throws {@code UnsupportedOperationException} for every * table change regardless of the specific operation. * *

Covered operations: SET TBLPROPERTIES (configuration change), ADD COLUMNS (schema change), * and CLUSTER BY (clustering change). All share the same managed table and each throws before * modifying anything. */ @Test public void testAlterTableOperationsAreBlocked() throws Exception { withNewTable( "block_alter_table_test", "id INT, name STRING", TableType.MANAGED, tableName -> { sql("INSERT INTO %s VALUES (1, 'initial')", tableName); // ALTER TABLE SET TBLPROPERTIES would change configuration. assertThrowsWithCauseContaining( ALTER_TABLE_ERROR, () -> sql("ALTER TABLE %s SET TBLPROPERTIES ('custom.key' = 'value')", tableName)); // ALTER TABLE ADD COLUMNS would change the schema. assertThrowsWithCauseContaining( ALTER_TABLE_ERROR, () -> sql("ALTER TABLE %s ADD COLUMNS (extra STRING)", tableName)); // ALTER TABLE CLUSTER BY would change clustering columns. assertThrowsWithCauseContaining( ALTER_TABLE_ERROR, () -> sql("ALTER TABLE %s CLUSTER BY (id)", tableName)); }); } /** * RESTORE TABLE to a version with unchanged clustering must succeed. The clustering kill switch * only fires when clustering actually changes. */ @Test public void testRestoreTableWithUnchangedClusteringSucceeds() throws Exception { String tableName = fullTableName("restore_unchanged_clustering_test"); try { sql( "CREATE TABLE %s (id INT, name STRING) USING DELTA CLUSTER BY (id)" + " TBLPROPERTIES ('delta.feature.catalogManaged'='supported')", tableName); sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); long versionAfterInsert = currentVersion(tableName); // Restore to version 0 (before the insert): clustering is unchanged, must succeed. sql("RESTORE TABLE %s TO VERSION AS OF %d", tableName, versionAfterInsert - 1); check(tableName, List.of()); } finally { sql("DROP TABLE IF EXISTS %s", tableName); } } /** * RESTORE TABLE to a version whose clustering differs from the current version must be blocked by * the kill switch in {@code commitLarge()}. * *

A table can have different clustering at different versions if an older client or another * connector wrote a version before this guard was in place. This test simulates that by writing a * Delta log entry with different clustering directly to the table's underlying storage (bypassing * the kill switch), then attempting a RESTORE that would change clustering back. * *

This test is local-only: it needs direct filesystem access to write the fake commit, which * is only available when backed by {@code S3CredentialFileSystem} (local UC). */ @Test public void testRestoreTableWithClusteringChangeIsBlocked() throws Exception { Assumptions.assumeFalse( isUCRemoteConfigured(), "Requires local filesystem access (local UC only)"); String tableName = fullTableName("restore_clustering_change_test"); try { sql( "CREATE TABLE %s (id INT, name STRING) USING DELTA CLUSTER BY (id)" + " TBLPROPERTIES ('delta.feature.catalogManaged'='supported')", tableName); sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); long insertVersion = currentVersion(tableName); // Simulate an older client writing a version with different clustering directly to the // table's storage, bypassing the kill switch. This is how a real-world scenario could arise. String s3Location = sql("DESCRIBE FORMATTED %s", tableName).stream() .filter(r -> r.size() >= 2 && "Location".equalsIgnoreCase(r.get(0).trim())) .map(r -> r.get(1).trim()) .findFirst() .orElseThrow(); // Convert URI to a local filesystem path: // - S3CredentialFileSystem maps s3://fakeS3Bucket/abs/path → /abs/path // - Local UC may return a file: URI directly String localTablePath; if (s3Location.startsWith("s3://")) { localTablePath = s3Location.replaceAll("s3://[^/]+", ""); } else if (s3Location.startsWith("file:")) { localTablePath = s3Location.replaceAll("^file:/+", "/"); } else { localTablePath = s3Location; } long hackedVersion = insertVersion + 1; Path hackedCommitFile = Paths.get(localTablePath, "_delta_log", String.format("%020d.json", hackedVersion)); // Write a minimal Delta commit with delta.clustering on 'name' instead of 'id'. String clusteringOnName = "{\\\"clusteringColumns\\\":[[\\\"name\\\"]]}"; Files.writeString( hackedCommitFile, "{\"commitInfo\":{\"timestamp\":1000000000000,\"inCommitTimestamp\":1000000000000," + "\"operation\":\"MANUAL UPDATE\",\"operationParameters\":{},\"isBlindAppend\":false}}\n" + "{\"domainMetadata\":{\"domain\":\"delta.clustering\"," + "\"configuration\":\"" + clusteringOnName + "\",\"removed\":false}}\n"); // RESTORE to a version before the hacked commit: the current snapshot now shows 'name' // clustering, so restoring to 'id' clustering fires the kill switch. assertThrowsWithCauseContaining( CLUSTERING_KILL_SWITCH_ERROR, () -> sql("RESTORE TABLE %s TO VERSION AS OF %d", tableName, insertVersion - 1)); } finally { sql("DROP TABLE IF EXISTS %s", tableName); } } /** * INSERT OVERWRITE with {@code overwriteSchema=true} that would replace the schema of an existing * CatalogOwned table must be blocked. * *

Because {@code UCSingleCatalog} does not implement {@code StagingTableCatalog}, Spark routes * the overwrite-with-schema-change through REPLACE TABLE AS SELECT (RTAS), which OSS Delta does * not support. */ @Test public void testInsertOverwriteWithOverwriteSchemaIsBlocked() throws Exception { withNewTable( "block_overwrite_schema_target", "id INT, name STRING", TableType.MANAGED, targetTable -> { sql("INSERT INTO %s VALUES (1, 'initial')", targetTable); withNewTable( "block_overwrite_schema_source", "id INT, name STRING, extra STRING", TableType.EXTERNAL, sourceTable -> { sql("INSERT INTO %s VALUES (2, 'new', 'extra_val')", sourceTable); assertThrowsWithCauseContaining( RTAS_ERROR, () -> spark() .read() .table(sourceTable) .write() .format("delta") .mode("overwrite") .option("overwriteSchema", "true") .saveAsTable(targetTable)); }); }); } /** * {@code CREATE OR REPLACE TABLE} with a different schema on an existing CatalogOwned table must * be blocked. * *

Because {@code UCSingleCatalog} does not implement {@code StagingTableCatalog}, Spark routes * {@code CREATE OR REPLACE TABLE} through REPLACE TABLE AS SELECT (RTAS), which OSS Delta does * not support. */ @Test public void testReplaceTableWithNewSchemaIsBlocked() throws Exception { withNewTable( "block_replace_schema_test", "id INT, name STRING", TableType.MANAGED, tableName -> { sql("INSERT INTO %s VALUES (1, 'initial')", tableName); assertThrowsWithCauseContaining( RTAS_ERROR, () -> sql( "CREATE OR REPLACE TABLE %s (id INT, name STRING, extra STRING) " + "USING DELTA " + "TBLPROPERTIES ('delta.feature.catalogManaged'='supported')", tableName)); }); } // --------------------------------------------------------------------------- // Positive tests: operations that must still succeed // --------------------------------------------------------------------------- /** Normal INSERT with no metadata change must still succeed on CatalogOwned tables. */ @Test public void testNormalInsertSucceedsForManagedTable() throws Exception { withNewTable( "normal_insert_managed_test", "id INT, name STRING", TableType.MANAGED, tableName -> { sql("INSERT INTO %s VALUES (1, 'foo'), (2, 'bar')", tableName); check(tableName, List.of(List.of("1", "foo"), List.of("2", "bar"))); }); } /** * INSERT with {@code autoMerge=true} but no new columns must succeed -- {@code autoMerge} only * triggers a schema update when the incoming data actually introduces extra columns. */ @Test public void testInsertWithAutoMergeAndNoSchemaChangeSucceeds() throws Exception { withNewTable( "auto_merge_no_change_managed_test", "id INT, name STRING", TableType.MANAGED, tableName -> { sql("INSERT INTO %s VALUES (1, 'initial')", tableName); sql("SET spark.databricks.delta.schema.autoMerge.enabled = true"); try { sql("INSERT INTO %s VALUES (2, 'second')", tableName); check(tableName, List.of(List.of("1", "initial"), List.of("2", "second"))); } finally { sql("SET spark.databricks.delta.schema.autoMerge.enabled = false"); } }); } /** * Schema evolution via INSERT with {@code autoMerge=true} must still work on EXTERNAL (non- * CatalogOwned) tables. The kill switch must not affect tables that are not CatalogOwned. */ @Test public void testInsertWithMergeSchemaStillWorksForExternalTable() throws Exception { withNewTable( "merge_schema_external_target", "id INT, name STRING", TableType.EXTERNAL, targetTable -> { sql("INSERT INTO %s VALUES (1, 'initial')", targetTable); withNewTable( "merge_schema_external_source", "id INT, name STRING, extra STRING", TableType.EXTERNAL, sourceTable -> { sql("INSERT INTO %s VALUES (2, 'new', 'extra_value')", sourceTable); sql("SET spark.databricks.delta.schema.autoMerge.enabled = true"); try { // Should succeed: EXTERNAL tables are not CatalogOwned. sql("INSERT INTO %s SELECT * FROM %s", targetTable, sourceTable); // The target now has 3 columns; row 1 has null for 'extra'. check( targetTable, List.of(List.of("1", "initial", "null"), List.of("2", "new", "extra_value"))); } finally { sql("SET spark.databricks.delta.schema.autoMerge.enabled = false"); } }); }); } } ================================================ FILE: spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableCreationTest.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.sparkuctest; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import io.unitycatalog.client.ApiException; import io.unitycatalog.client.api.TablesApi; import io.unitycatalog.client.model.ColumnInfo; import io.unitycatalog.client.model.DataSourceFormat; import io.unitycatalog.client.model.TableInfo; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.Getter; import lombok.Setter; import lombok.ToString; import lombok.experimental.Accessors; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.fs.Path; import org.apache.log4j.Logger; import org.apache.spark.sql.connector.catalog.TableCatalog; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestFactory; /** Test suite for creating UC Delta Tables. */ public class UCDeltaTableCreationTest extends UCDeltaTableIntegrationBaseTest { private static final Logger LOG = Logger.getLogger(UCDeltaTableCreationTest.class); // Property constants related to managed table creation private static final String UC_TABLE_ID_KEY = "io.unitycatalog.tableId"; private static final String UC_TABLE_ID_KEY_OLD = "ucTableId"; private static final String DELTA_CATALOG_MANAGED_KEY = "delta.feature.catalogManaged"; private static final String SUPPORTED = "supported"; private static final String MANAGED_TBLPROPERTIES_CLAUSE = String.format("TBLPROPERTIES ('%s'='%s', 'Foo'='Bar')", DELTA_CATALOG_MANAGED_KEY, SUPPORTED); // In the table REPLACE test, a slightly different table property clause will be used to create // the first table. Then the REPLACE command would use TBLPROPERTIES_CLAUSE. This is to make sure // that the table properties are properly updated in the REPLACE command. private static final String MANAGED_TBLPROPERTIES_CLAUSE_OTHER = String.format( "TBLPROPERTIES ('%s'='%s', 'Foo2'='Bar2')", DELTA_CATALOG_MANAGED_KEY, SUPPORTED); // Expected table features to be enabled for managed tables private static final List EXPECTED_MANAGED_TABLE_FEATURES = List.of( "delta.feature.appendOnly", DELTA_CATALOG_MANAGED_KEY, "delta.feature.deletionVectors", "delta.feature.domainMetadata", "delta.feature.inCommitTimestamp", "delta.feature.invariants", "delta.feature.rowTracking", "delta.feature.v2Checkpoint", "delta.feature.vacuumProtocolCheck"); private static final Map EXPECTED_MANAGED_TABLE_FEATURES_PROPERTIES = EXPECTED_MANAGED_TABLE_FEATURES.stream() .collect(Collectors.toMap(Function.identity(), k -> SUPPORTED)); private static final String EXTERNAL_TBLPROPERTIES_CLAUSE = "TBLPROPERTIES ('Foo'='Bar')"; /** * Returns true if the Unity Catalog Spark version >0.4.0 so that it supports complex data types * in columns and partition index. */ private static boolean isUcSparkNewerThan040() { final int[] VER_0_4_0 = {0, 4, 0}; int[] ucSparkVersion = getUnityCatalogSparkVersion(); return Arrays.compare(ucSparkVersion, VER_0_4_0) > 0; } String tempDir; private Set tablesToCleanUp = new HashSet<>(); @BeforeEach public void setUp() { tempDir = unityCatalogInfo().baseTableLocation() + "/temp-" + UUID.randomUUID(); } @AfterEach public void cleanUpTables() { for (String fullTableName : tablesToCleanUp) { try { sql("DROP TABLE IF EXISTS %s", fullTableName); } catch (Exception e) { // Ignore during clean up. } } tablesToCleanUp.clear(); } /** Helper class for controlling table creation options during tests. */ @Accessors(chain = true) @Getter @Setter @ToString private class TableSetupOptions { private TableType tableType; private String catalogName; private String schemaName; private String tableName; private Optional partitionColumn = Optional.empty(); private Optional clusterColumn = Optional.empty(); private Optional> asSelect = Optional.empty(); private Optional comment = Optional.empty(); private boolean replaceTable = false; public TableSetupOptions() {} public TableSetupOptions setPartitionColumn(String column) { Preconditions.checkArgument(List.of("i", "s").contains(column)); Preconditions.checkState( clusterColumn.isEmpty(), "Can not have both PARTITIONED BY and CLUSTER BY."); partitionColumn = Optional.of(column); return this; } public TableSetupOptions setClusterColumn(String column) { Preconditions.checkArgument(List.of("i", "s").contains(column)); Preconditions.checkState( partitionColumn.isEmpty(), "Can not have both PARTITIONED BY and CLUSTER BY."); clusterColumn = Optional.of(column); return this; } public TableSetupOptions setAsSelect(int i, String s) { asSelect = Optional.of(Pair.of(i, s)); return this; } public TableSetupOptions setComment(String c) { comment = Optional.of(c); return this; } public String partitionClause() { return partitionColumn.map(c -> String.format("PARTITIONED BY (%s)", c)).orElse(""); } public String clusterClause() { return clusterColumn.map(c -> String.format("CLUSTER BY (%s)", c)).orElse(""); } public String columnsClause() { if (asSelect.isEmpty()) { return "(i INT, s STRING)"; } else { // "AS SELECT" can't specify columns return ""; } } public String asSelectClause() { return asSelect .map(x -> String.format("AS SELECT %d AS i, '%s' AS s", x.getLeft(), x.getRight())) .orElse(""); } public String commentClause() { return comment.map(c -> String.format("COMMENT '%s'", c)).orElse(""); } public String ddlCommand() { return replaceTable ? "REPLACE" : "CREATE"; } private String createManagedTableSql() { return String.format( "%s TABLE %s.%s.%s %s USING DELTA %s %s %s %s %s", ddlCommand(), catalogName, schemaName, tableName, columnsClause(), partitionClause(), clusterClause(), MANAGED_TBLPROPERTIES_CLAUSE, commentClause(), asSelectClause()); } public String getExternalTableLocation() { return tempDir + "/" + tableName; } private String createExternalTableSql() { return String.format( "%s TABLE %s.%s.%s %s USING DELTA %s %s %s %s LOCATION '%s' %s", ddlCommand(), catalogName, schemaName, tableName, columnsClause(), partitionClause(), clusterClause(), EXTERNAL_TBLPROPERTIES_CLAUSE, commentClause(), getExternalTableLocation(), asSelectClause()); } public String createTableSql() { if (tableType == TableType.MANAGED) { return createManagedTableSql(); } else { return createExternalTableSql(); } } public String fullTableName() { return String.join(".", catalogName, schemaName, tableName); } } @TestFactory public Stream testCreateTable() { int counter = 0; List tests = new ArrayList<>(); for (TableType tableType : TableType.values()) { for (boolean withPartition : List.of(true, false)) { for (boolean withCluster : List.of(true, false)) { if (withCluster && withPartition) { // Can not have CLUSTER BY and PARTITIONED BY on the same table continue; } for (boolean withAsSelect : List.of(true, false)) { for (boolean replaceTable : List.of(true, false)) { String displayName = String.format( "tableType=%s, withPartition=%s, withCluster=%s, withAsSelect=%s, replaceTable=%s", tableType, withPartition, withCluster, withAsSelect, replaceTable); counter++; int finalCounter = counter; tests.add( DynamicTest.dynamicTest( displayName, () -> runTableCreationTest( finalCounter, tableType, withPartition, withCluster, withAsSelect, replaceTable))); } } } } } return tests.stream(); } private void runTableCreationTest( int count, TableType tableType, boolean withPartition, boolean withCluster, boolean withAsSelect, boolean replaceTable) throws Exception { UnityCatalogInfo uc = unityCatalogInfo(); final String comment = "This is comment."; // Test with unity catalog only (spark_catalog is not configured as UC catalog) final String catalogName = uc.catalogName(); final String schemaName = uc.schemaName(); String tableName = "test_delta_table_" + count; TableSetupOptions options = new TableSetupOptions() .setCatalogName(catalogName) .setSchemaName(schemaName) .setTableName(tableName) .setTableType(tableType) .setReplaceTable(replaceTable) .setComment(comment); if (withPartition) { options.setPartitionColumn("i"); } if (withCluster) { options.setClusterColumn("s"); } if (withAsSelect) { options.setAsSelect(1, "a"); } LOG.info("Running table creation test: " + options); String fullTableName = options.fullTableName(); if (replaceTable) { // First, create a different table to replace. sql( "CREATE TABLE %s USING DELTA %s AS SELECT %s AS col1", fullTableName, MANAGED_TBLPROPERTIES_CLAUSE_OTHER, // Older version UC Spark client can't support Decimal type isUcSparkNewerThan040() ? "0.1" : "1"); tablesToCleanUp.add(fullTableName); } // TODO: Remove the block if UC and delta support the atomic RT and RTAS. if (replaceTable) { assertThatThrownBy(() -> sql(options.createTableSql())); return; } // Create table sql(options.createTableSql()); tablesToCleanUp.add(fullTableName); // Basic read/write test sql("INSERT INTO %s SELECT 2, 'b'", fullTableName); if (withAsSelect) { check(fullTableName, List.of(List.of("1", "a"), List.of("2", "b"))); } else { check(fullTableName, List.of(List.of("2", "b"))); } // Verify that table information maintained at the uc server side are expected. // TODO: Remove the block when delta supports the CTAS in the correct way. Currently CTAS // is missing AbstractDeltaCatalog.translateUCTableIdProperty if (!withAsSelect || replaceTable) { assertUCTableInfo( tableType, fullTableName, List.of("i", "s"), Map.of("Foo", "Bar"), comment, options.getExternalTableLocation(), withCluster, options.getClusterColumn(), options.getPartitionColumn()); } } @Test public void testCreateManagedTableErrors() { String tableName = "test_delta_errors"; UnityCatalogInfo uc = unityCatalogInfo(); String fullTableName = uc.catalogName() + "." + uc.schemaName() + "." + tableName; // Test 1: Non-Delta managed tables are not supported assertThatThrownBy( () -> sql( "CREATE TABLE %s(name STRING) USING parquet %s", fullTableName, MANAGED_TBLPROPERTIES_CLAUSE)) .hasMessageContaining("not support non-Delta managed table"); // Test 2: Invalid property value 'disabled' for catalogManaged feature assertThatThrownBy( () -> sql( "CREATE TABLE %s(name STRING) USING delta TBLPROPERTIES ('%s' = 'disabled')", fullTableName, DELTA_CATALOG_MANAGED_KEY)) .hasMessageContaining( String.format("Invalid property value 'disabled' for '%s'", DELTA_CATALOG_MANAGED_KEY)); // Test 3: Cannot set UC table ID manually for (String ucTableIdProperty : List.of(UC_TABLE_ID_KEY, UC_TABLE_ID_KEY_OLD)) { assertThatThrownBy( () -> sql( "CREATE TABLE %s(name STRING) USING delta TBLPROPERTIES ('%s' = 'some_id')", fullTableName, ucTableIdProperty)) .hasMessageContaining(ucTableIdProperty); } // Test 4: Cannot set is_managed_location to false for managed tables assertThatThrownBy( () -> sql( "CREATE TABLE %s(name STRING) USING delta TBLPROPERTIES ('%s' = 'false')", fullTableName, TableCatalog.PROP_IS_MANAGED_LOCATION)) .hasMessageContaining("is_managed_location"); // Test 5: Managed table creation requires catalogManaged property assertThatThrownBy(() -> sql("CREATE TABLE %s(name STRING) USING delta", fullTableName)) .hasMessageContaining( String.format( "Managed table creation requires table property '%s'='%s' to be set", DELTA_CATALOG_MANAGED_KEY, SUPPORTED)); } @TestAllTableTypes public void testCreateOrReplaceTable(TableType tableType) throws Exception { UnityCatalogInfo uc = unityCatalogInfo(); String tableName = String.format("%s.%s.create_or_replace", uc.catalogName(), uc.schemaName()); withTempDir( (Path dir) -> { try { // TODO: Once the UC and delta support the stageCreateOrReplace, then we should remove // the failure assertion. Please see https://github.com/delta-io/delta/issues/6013. // CREATE OR REPLACE with new schema if (tableType == TableType.MANAGED) { assertThatThrownBy( () -> sql( "CREATE OR REPLACE TABLE %s (id INT, name STRING) USING DELTA %s ", tableName, MANAGED_TBLPROPERTIES_CLAUSE)); } else { assertThatThrownBy( () -> sql( "CREATE OR REPLACE TABLE %s (id INT, name STRING) USING DELTA LOCATION '%s'", tableName, dir.toString())); } // TODO: Uncommon those code once support the stageCreateOrReplace, as said above. // Assert the unity catalog table information. // assertUCTableInfo( // tableType, tableName, List.of("id", "name"), Map.of("Foo", "Bar"), null, null); // Insert data to verify new schema // sql("INSERT INTO %s VALUES (1, 'Alice')", tableName); // check(tableName, List.of(List.of("1", "Alice"))); } finally { sql("DROP TABLE IF EXISTS %s", tableName); } }); } @TestAllTableTypes public void testTableWithSupportedDataTypes(TableType tableType) throws Exception { Assumptions.assumeTrue( isUcSparkNewerThan040() || tableType != TableType.MANAGED, "Older UC Spark package can't support uploading complex types to UC server for managed table"); String schema = // Numeric types "col_tinyint TINYINT, col_smallint SMALLINT, col_int INT, col_bigint BIGINT, " + "col_float FLOAT, col_double DOUBLE, col_decimal DECIMAL(10,2), " // String and binary types + "col_string STRING, col_char CHAR(10), col_varchar VARCHAR(20), col_binary BINARY, " // Boolean type + "col_boolean BOOLEAN, " // Date and time types + "col_date DATE, col_timestamp TIMESTAMP, col_timestamp_ntz TIMESTAMP_NTZ"; withNewTable( "supported_types_table", schema, tableType, tableName -> { // Insert sample data sql( "INSERT INTO %s VALUES (" // Numeric values + "CAST(1 AS TINYINT), CAST(100 AS SMALLINT), 1000, 100000, " + "2.5, 1.5, 123.45, " // String and binary values + "'test', 'char_test', 'varchar_test', X'CAFEBABE', " // Boolean value + "true, " // Date and time values + "DATE'2025-01-01', TIMESTAMP'2025-01-01 12:00:00', " + "TIMESTAMP_NTZ'2025-01-01 12:00:00')", tableName); // Assert the unity catalog table information. assertUCTableInfo( tableType, tableName, List.of( "col_tinyint", "col_smallint", "col_int", "col_bigint", "col_float", "col_double", "col_decimal", "col_string", "col_char", "col_varchar", "col_binary", "col_boolean", "col_date", "col_timestamp", "col_timestamp_ntz"), // This feature is automatically enabled due to use of TIMESTAMP_NTZ Map.of("delta.feature.timestampNtz", "supported"), null, null); // Verify data can be queried - checking that each column type is correctly // stored/retrieved List> results = sql("SELECT * FROM %s", tableName); assertThat(results).hasSize(1); List row = results.get(0); // Verify each column value assertThat(row.get(0)).isEqualTo("1"); // TINYINT assertThat(row.get(1)).isEqualTo("100"); // SMALLINT assertThat(row.get(2)).isEqualTo("1000"); // INT assertThat(row.get(3)).isEqualTo("100000"); // BIGINT assertThat(row.get(4)).isEqualTo("2.5"); // FLOAT assertThat(row.get(5)).isEqualTo("1.5"); // DOUBLE assertThat(row.get(6)).isEqualTo("123.45"); // DECIMAL assertThat(row.get(7)).isEqualTo("test"); // STRING assertThat(row.get(8)).isEqualTo("char_test "); // CHAR (padded with space) assertThat(row.get(9)).isEqualTo("varchar_test"); // VARCHAR assertThat(row.get(10)).startsWith("[B@"); // BINARY (Java byte array object reference) assertThat(row.get(11)).isEqualTo("true"); // BOOLEAN assertThat(row.get(12)).isEqualTo("2025-01-01"); // DATE assertThat(row.get(13)).isEqualTo("2025-01-01 12:00:00.0"); // TIMESTAMP assertThat(row.get(14)).isEqualTo("2025-01-01T12:00"); // TIMESTAMP_NTZ }); } @TestAllTableTypes public void testTableWithComplexTypes(TableType tableType) throws Exception { Assumptions.assumeTrue( isUcSparkNewerThan040() || tableType != TableType.MANAGED, "Older UC Spark package can't support uploading complex types to UC server for managed table"); String schema = "id INT, arr ARRAY, " + "map_col MAP, " + "struct_col STRUCT"; withNewTable( "complex_types_table", schema, tableType, tableName -> { // Insert sample data sql( "INSERT INTO %s VALUES (1, array(1, 2, 3), " + "map('key1', 10, 'key2', 20), " + "struct(42, 'test'))", tableName); // Assert the unity catalog table information. assertUCTableInfo( tableType, tableName, List.of("id", "arr", "map_col", "struct_col"), Map.of(), null, null); // Verify data can be queried check( tableName, List.of( List.of("1", "ArraySeq(1, 2, 3)", "Map(key1 -> 10, key2 -> 20)", "[42,test]"))); }); } @TestAllTableTypes public void testTableWithNotNullConstraints(TableType tableType) throws Exception { withNewTable( "not_null_table", "id INT NOT NULL, name STRING NOT NULL, optional STRING", tableType, tableName -> { // Insert valid data sql("INSERT INTO %s VALUES (1, 'Alice', 'extra')", tableName); sql("INSERT INTO %s VALUES (2, 'Bob', NULL)", tableName); check(tableName, List.of(List.of("1", "Alice", "extra"), List.of("2", "Bob", "null"))); // Assert the unity catalog table information. assertUCTableInfo( tableType, tableName, List.of("id", "name", "optional"), Map.of(), null, null); // Attempting to insert NULL into NOT NULL column should fail Assertions.assertThatThrownBy( () -> sql("INSERT INTO %s VALUES (NULL, 'Charlie', 'data')", tableName)) .isInstanceOf(Exception.class); }); } private void assertUCTableInfo( TableType tableType, String fullTableName, List expectedColumns, Map customizedProps, String comment, String externalTableLocation) throws ApiException { assertUCTableInfo( tableType, fullTableName, expectedColumns, customizedProps, comment, externalTableLocation, false, Optional.empty(), Optional.empty()); } private void assertUCTableInfo( TableType tableType, String fullTableName, List expectedColumns, Map customizedProps, String comment, String externalTableLocation, boolean withCluster, Optional clusterColumn, Optional partitionColumn) throws ApiException { UnityCatalogInfo uc = unityCatalogInfo(); String catalogName = uc.catalogName(); String schemaName = uc.schemaName(); // Verify that properties are set on server. This can not be done by DESC EXTENDED. TablesApi tablesApi = new TablesApi(uc.createApiClient()); TableInfo tableInfo = tablesApi.getTable(fullTableName, false, false); assertThat(tableInfo.getCatalogName()).isEqualTo(catalogName); assertThat(tableInfo.getName()).isEqualTo(parseTableName(fullTableName)); assertThat(tableInfo.getSchemaName()).isEqualTo(schemaName); assertThat(tableInfo.getTableType().name()).isEqualTo(tableType.name()); assertThat(tableInfo.getDataSourceFormat().name()).isEqualTo(DataSourceFormat.DELTA.name()); assertThat(tableInfo.getComment()).isEqualTo(comment); if (tableType == TableType.EXTERNAL && externalTableLocation != null) { assertThat(tableInfo.getStorageLocation()).isEqualTo(externalTableLocation); } // At this point table schema can not be sent to server yet because it won't be // updated later and that would cause problem. List columns = tableInfo.getColumns(); assertThat(columns).isNotNull(); if (tableType == TableType.MANAGED) { assertThat(columns).isNotEmpty(); List columnNamesFromServer = columns.stream().map(ColumnInfo::getName).collect(Collectors.toList()); assertThat(columnNamesFromServer).containsExactlyInAnyOrderElementsOf(expectedColumns); // Partition index is only set after UC-Spark 0.4.0 if (isUcSparkNewerThan040() && partitionColumn.isPresent()) { List matchingColumns = columns.stream() .filter(c -> c.getName().equals(partitionColumn.get())) .collect(Collectors.toList()); assertThat(matchingColumns).hasSize(1); assertThat(matchingColumns.get(0).getPartitionIndex()).isEqualTo(0); } else { assertThat(columns.stream().anyMatch(c -> c.getPartitionIndex() != null)).isFalse(); } // Delta sent properties of managed tables to server Map tablePropertiesFromServer = tableInfo.getProperties(); tablePropertiesFromServer.remove("table_type", "MANAGED"); // New property by Spark 4.1 // CLUSTER BY has two extra properties final Map expectedClusteringProperties = withCluster ? ImmutableMap.builder() .put("clusteringColumns", "[[\"" + clusterColumn.get() + "\"]]") .put("delta.feature.clustering", SUPPORTED) .build() : ImmutableMap.of(); final Map expectedOtherProperties = ImmutableMap.builder() .put("delta.checkpointPolicy", "v2") .put("delta.enableDeletionVectors", "true") .put("delta.enableInCommitTimestamps", "true") .put("delta.enableRowTracking", "true") .put("delta.lastUpdateVersion", "0") .put("delta.minReaderVersion", "3") .put("delta.minWriterVersion", "7") .put(UC_TABLE_ID_KEY, tableInfo.getTableId()) // User specified custom table property is also sent. .putAll(customizedProps) .putAll(expectedClusteringProperties) .build(); // The value of these properties aren't predictable. But at least we confirm their existence. final Set expectedPropertiesWithVariableValue = Set.of( "delta.lastCommitTimestamp", "delta.rowTracking.materializedRowCommitVersionColumnName", "delta.rowTracking.materializedRowIdColumnName"); // This is combination of expectedOtherProperties and // EXPECTED_MANAGED_TABLE_FEATURES_PROPERTIES. Map expectedProperties = Stream.concat( EXPECTED_MANAGED_TABLE_FEATURES_PROPERTIES.entrySet().stream(), expectedOtherProperties.entrySet().stream()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); // Server has all the expected table properties expectedProperties.forEach( (key, value) -> assertThat(tablePropertiesFromServer).containsEntry(key, value)); expectedPropertiesWithVariableValue.forEach( key -> assertThat(tablePropertiesFromServer).containsKey(key)); // Server doesn't have any unexpected table properties. If anyone introduces a new table // property and this fails, update the list of expected properties. Map unexpectedTablePropertiesFromServer = tablePropertiesFromServer.entrySet().stream() .filter( entry -> !expectedProperties.containsKey(entry.getKey()) && !expectedPropertiesWithVariableValue.contains(entry.getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); assertThat(unexpectedTablePropertiesFromServer).isEmpty(); } else { assertThat(columns).isEmpty(); } // Also verify table using DESC EXTENDED List> rows = sql("DESC EXTENDED %s", fullTableName); Map describeResult = new HashMap<>(); for (List row : rows) { String key = row.get(0); // Skip duplicate column names that appear in partition info if (!expectedColumns.contains(key)) { describeResult.put(key, row.get(1)); } } // Verify basic table properties assertThat(describeResult.get("Name")).isEqualTo(fullTableName); assertThat(describeResult.get("Type")).isEqualTo(tableType.name()); assertThat(describeResult.get("Provider")).isEqualToIgnoringCase("delta"); assertThat(describeResult.get("Is_managed_location")) .isEqualTo(tableType == TableType.MANAGED ? "true" : null); assertThat(describeResult).containsKey("Table Properties"); String tableProperties = describeResult.get("Table Properties"); if (tableType == TableType.MANAGED) { // Check for UC table ID assertThat(tableProperties).contains(UC_TABLE_ID_KEY); // Check for catalogManaged feature assertThat(tableProperties) .contains(String.format("%s=%s", DELTA_CATALOG_MANAGED_KEY, SUPPORTED)); } else { // Check for UC table ID assertThat(tableProperties).doesNotContain(UC_TABLE_ID_KEY); // Check for catalogManaged feature assertThat(tableProperties).doesNotContain(DELTA_CATALOG_MANAGED_KEY); } } private static String parseTableName(String fullTableName) { String[] splits = fullTableName.split("\\."); assertThat(splits.length).isEqualTo(3); return splits[splits.length - 1]; } } ================================================ FILE: spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDMLTest.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.sparkuctest; import java.util.List; /** * DML test suite for Delta Table operations through Unity Catalog. * *

Covers INSERT, UPDATE, DELETE, and MERGE operations with various conditions and scenarios. * Tests are parameterized to support different table types (currently EXTERNAL only, as Delta does * not support MANAGED catalog-owned tables). */ public class UCDeltaTableDMLTest extends UCDeltaTableIntegrationBaseTest { @TestAllTableTypes public void testBasicInsertOperations(TableType tableType) throws Exception { withNewTable( "insert_basic_test", "id INT, name STRING, active BOOLEAN", tableType, tableName -> { // Single row INSERT sql("INSERT INTO %s VALUES (1, 'initial', true)", tableName); // Verify single row check(tableName, List.of(List.of("1", "initial", "true"))); // Multiple rows in single INSERT sql("INSERT INTO %s VALUES (2, 'User2', false), (3, 'User3', true)", tableName); // Multiple separate INSERT operations sql("INSERT INTO %s VALUES (4, 'User4', false)", tableName); // Verify all inserts (appended data) check( tableName, List.of( List.of("1", "initial", "true"), List.of("2", "User2", "false"), List.of("3", "User3", "true"), List.of("4", "User4", "false"))); }); } @TestAllTableTypes public void testAdvancedInsertOperations(TableType tableType) throws Exception { // Test INSERT ... SELECT withNewTable( "insert_select_target", "id INT, category STRING", tableType, targetTable -> { withNewTable( "insert_select_source", "id INT, name STRING", tableType, sourceTable -> { sql("INSERT INTO %s VALUES (1, 'TypeA'), (2, 'TypeB'), (3, 'TypeA')", sourceTable); sql( "INSERT INTO %s SELECT id, name FROM %s WHERE name = 'TypeA'", targetTable, sourceTable); check(targetTable, List.of(List.of("1", "TypeA"), List.of("3", "TypeA"))); }); }); // Test INSERT OVERWRITE withNewTable( "insert_overwrite_test", "id INT, status STRING", tableType, tableName -> { sql("INSERT INTO %s VALUES (1, 'old'), (2, 'old'), (3, 'old')", tableName); sql("INSERT OVERWRITE %s VALUES (4, 'new'), (5, 'new')", tableName); check(tableName, List.of(List.of("4", "new"), List.of("5", "new"))); }); // Test INSERT ... REPLACE WHERE withNewTable( "insert_replace_test", "id INT, status STRING", tableType, tableName -> { sql("INSERT INTO %s VALUES (1, 'pending'), (2, 'pending'), (3, 'completed')", tableName); sql( "INSERT INTO %s REPLACE WHERE id <= 2 VALUES (1, 'replaced'), (2, 'replaced')", tableName); check( tableName, List.of( List.of("1", "replaced"), List.of("2", "replaced"), List.of("3", "completed"))); }); } @TestAllTableTypes public void testInsertWithDynamicPartitionOverwrite(TableType tableType) throws Exception { withNewTable( "insert_dynamic_partition_overwrite_test", "id INT, name STRING, date STRING", "date", tableType, tableName -> { // Setup initial data sql("INSERT INTO %s PARTITION (date='2025-11-01') VALUES (1, 'AAA')", tableName); sql("INSERT INTO %s PARTITION (date='2025-11-01') VALUES (2, 'BBB')", tableName); // Verify the result before dynamic partition overwrite. check( tableName, List.of(List.of("1", "AAA", "2025-11-01"), List.of("2", "BBB", "2025-11-01"))); // Enable dynamic partition overwrite sql("SET spark.databricks.delta.dynamicPartitionOverwrite.enabled = true"); try { // Insert with dynamic partition overwrite sql("INSERT OVERWRITE %s VALUES (3, 'CCC', '2025-11-01')", tableName); // Verify the result - should have replaced with the new value check(tableName, List.of(List.of("3", "CCC", "2025-11-01"))); } finally { // Disable dynamic partition overwrite sql("SET spark.databricks.delta.dynamicPartitionOverwrite.enabled = false"); } }); } @TestAllTableTypes public void testUpdateOperations(TableType tableType) throws Exception { withNewTable( "update_test", "id INT, priority INT, status STRING", tableType, tableName -> { // Setup data sql( "INSERT INTO %s VALUES " + "(1, 1, 'pending'), (2, 5, 'pending'), (3, 10, 'pending'), (4, 2, 'completed')", tableName); // Simple update: update specific row by id sql("UPDATE %s SET status = 'processed' WHERE id = 1", tableName); // Verify simple update check( tableName, List.of( List.of("1", "1", "processed"), List.of("2", "5", "pending"), List.of("3", "10", "pending"), List.of("4", "2", "completed"))); // Complex update: update based on priority condition sql("UPDATE %s SET status = 'urgent' WHERE priority >= 5", tableName); // Verify complex update check( tableName, List.of( List.of("1", "1", "processed"), List.of("2", "5", "urgent"), List.of("3", "10", "urgent"), List.of("4", "2", "completed"))); }); } @TestAllTableTypes public void testDeleteOperations(TableType tableType) throws Exception { withNewTable( "delete_test", "id INT, category STRING, value INT, active BOOLEAN", tableType, tableName -> { // Setup data sql( "INSERT INTO %s VALUES " + "(1, 'A', 10, true), (2, 'B', 20, false), (3, 'A', 30, true), " + "(4, 'C', 5, false), (5, 'B', 15, true)", tableName); // Simple delete: single condition sql("DELETE FROM %s WHERE active = false", tableName); // Verify simple delete check( tableName, List.of( List.of("1", "A", "10", "true"), List.of("3", "A", "30", "true"), List.of("5", "B", "15", "true"))); // Complex delete: multiple conditions with OR sql("DELETE FROM %s WHERE category = 'A' OR value < 10", tableName); // Verify complex delete check(tableName, List.of(List.of("5", "B", "15", "true"))); }); } @TestAllTableTypes public void testMergeInsertOnly(TableType tableType) throws Exception { withNewTable( "merge_insert_test", "id INT, value STRING", tableType, tableName -> { // Setup target table with initial data sql("INSERT INTO %s VALUES (1, 'existing1'), (2, 'existing2')", tableName); // Create source data and perform merge withNewTable( "merge_source", "id INT, value STRING", tableType, sourceTable -> { sql("INSERT INTO %s VALUES (3, 'new3'), (4, 'new4')", sourceTable); sql( "MERGE INTO %s AS target " + "USING %s AS source " + "ON target.id = source.id " + "WHEN NOT MATCHED THEN INSERT (id, value) VALUES (source.id, source.value)", tableName, sourceTable); }); // Verify merge result check( tableName, List.of( List.of("1", "existing1"), List.of("2", "existing2"), List.of("3", "new3"), List.of("4", "new4"))); }); } @TestAllTableTypes public void testMergeUpdateOnly(TableType tableType) throws Exception { withNewTable( "merge_update_test", "id INT, value STRING", tableType, tableName -> { // Setup target table sql("INSERT INTO %s VALUES (1, 'old1'), (2, 'old2'), (3, 'old3')", tableName); // Perform merge to update existing records withNewTable( "merge_update_source", "id INT, value STRING", tableType, sourceTable -> { sql("INSERT INTO %s VALUES (2, 'updated2'), (3, 'updated3')", sourceTable); sql( "MERGE INTO %s AS target " + "USING %s AS source " + "ON target.id = source.id " + "WHEN MATCHED THEN UPDATE SET value = source.value", tableName, sourceTable); }); // Verify merge result check( tableName, List.of(List.of("1", "old1"), List.of("2", "updated2"), List.of("3", "updated3"))); }); } @TestAllTableTypes public void testMergeCombinedInsertAndUpdate(TableType tableType) throws Exception { withNewTable( "merge_combined_test", "id INT, name STRING, status STRING", tableType, tableName -> { // Setup target table sql("INSERT INTO %s VALUES (1, 'Alice', 'active'), (2, 'Bob', 'inactive')", tableName); // Perform merge with both insert and update withNewTable( "merge_combined_source", "id INT, name STRING, status STRING", tableType, sourceTable -> { sql( "INSERT INTO %s VALUES " + "(2, 'Bob', 'active'), (3, 'Charlie', 'active'), (4, 'Diana', 'pending')", sourceTable); sql( "MERGE INTO %s AS target " + "USING %s AS source " + "ON target.id = source.id " + "WHEN MATCHED THEN UPDATE SET status = source.status " + "WHEN NOT MATCHED THEN INSERT (id, name, status) VALUES (source.id, source.name, source.status)", tableName, sourceTable); }); // Verify merge result check( tableName, List.of( List.of("1", "Alice", "active"), List.of("2", "Bob", "active"), List.of("3", "Charlie", "active"), List.of("4", "Diana", "pending"))); }); } @TestAllTableTypes public void testMergeWithDeleteAction(TableType tableType) throws Exception { withNewTable( "merge_delete_test", "id INT, active BOOLEAN", tableType, tableName -> { // Setup target table sql("INSERT INTO %s VALUES (1, true), (2, true), (3, false), (4, true)", tableName); // Perform merge with delete action withNewTable( "merge_delete_source", "id INT, active BOOLEAN", tableType, sourceTable -> { sql("INSERT INTO %s VALUES (2, false), (3, true), (5, true)", sourceTable); sql( "MERGE INTO %s AS target " + "USING %s AS source " + "ON target.id = source.id " + "WHEN MATCHED AND source.active = false THEN DELETE " + "WHEN MATCHED THEN UPDATE SET active = source.active " + "WHEN NOT MATCHED THEN INSERT (id, active) VALUES (source.id, source.active)", tableName, sourceTable); }); // Verify merge result - record 2 should be deleted, record 3 should be updated check( tableName, List.of( List.of("1", "true"), // not in source, no change List.of("3", "true"), // matched and updated from false to true List.of("4", "true"), // not in source, no change List.of("5", "true") // not matched in target, inserted )); }); } } ================================================ FILE: spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameReadTest.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.sparkuctest; import static org.apache.spark.sql.functions.col; import static org.assertj.core.api.Assertions.assertThat; import java.util.List; import java.util.stream.Collectors; import org.apache.spark.sql.DataFrameReader; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; /** * DataFrame read test suite for Delta Table operations through Unity Catalog. * *

Covers spark.table(), DataFrameReader, time travel, column pruning, and filter. Most tests run * against both EXTERNAL and MANAGED table types. */ public class UCDeltaTableDataFrameReadTest extends UCDeltaTableIntegrationBaseTest { @TestAllTableTypes public void testReadViaSparkTable(TableType tableType) throws Exception { withNewTable( "df_read_spark_table", "id INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); assertThat(ids(spark().table(tableName).orderBy("id"))).containsExactly(1, 2, 3); }); } @TestAllTableTypes public void testReadViaDataFrameReader(TableType tableType) throws Exception { withNewTable( "df_read_reader", "id INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); assertThat(ids(deltaDFReader().table(tableName).orderBy("id"))).containsExactly(1, 2, 3); }); } @TestAllTableTypes public void testTimeTravelByVersion(TableType tableType) throws Exception { withNewTable( "df_time_travel_version", "id INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); long v1 = currentVersion(tableName); sql("INSERT INTO %s VALUES (4), (5)", tableName); assertThat(ids(deltaDFReader().option("versionAsOf", v1).table(tableName).orderBy("id"))) .containsExactly(1, 2, 3); }); } @TestAllTableTypes public void testTimeTravelByTimestamp(TableType tableType) throws Exception { withNewTable( "df_time_travel_ts", "id INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); String ts = currentTimestamp(tableName); sql("INSERT INTO %s VALUES (4), (5)", tableName); assertThat( ids(deltaDFReader().option("timestampAsOf", ts).table(tableName).orderBy("id"))) .containsExactly(1, 2, 3); }); } @TestAllTableTypes public void testColumnPruning(TableType tableType) throws Exception { withNewTable( "df_column_pruning", "id INT, name STRING, value INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1, 'Alice', 100), (2, 'Bob', 200)", tableName); List rows = spark().table(tableName).select("id", "name").orderBy("id").collectAsList(); assertThat(rows.get(0).schema().fieldNames()).containsExactly("id", "name"); assertThat(rows.stream().map(r -> r.getInt(0)).collect(Collectors.toList())) .containsExactly(1, 2); assertThat(rows.stream().map(r -> r.getString(1)).collect(Collectors.toList())) .containsExactly("Alice", "Bob"); }); } @Test public void testReadViaPath() throws Exception { withNewTable( "df_read_via_path", "id INT", TableType.MANAGED, tableName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); String tablePath = sql("DESCRIBE EXTENDED %s", tableName).stream() .filter(row -> row.size() >= 2 && "Location".equals(row.get(0))) .map(row -> row.get(1)) .findFirst() .orElseThrow(() -> new AssertionError("Could not retrieve table location")); Assertions.assertThrows( Exception.class, () -> spark().read().format("delta").load(tablePath).collect(), "Path-based access should fail for managed tables"); }); } @TestAllTableTypes public void testChangeDataFeedViaDataFrameAPI(TableType tableType) throws Exception { withNewTable( "df_cdf_reader", "id INT", null, tableType, "'delta.enableChangeDataFeed'='true'", tableName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); sql("INSERT INTO %s VALUES (4), (5)", tableName); long currentVersion = currentVersion(tableName); List rows = deltaDFReader() .option("readChangeFeed", "true") .option("startingVersion", currentVersion) .table(tableName) .orderBy("id") .collectAsList(); assertThat(rows.stream().map(r -> r.getInt(0)).collect(Collectors.toList())) .containsExactly(4, 5); }); } @TestAllTableTypes public void testEmptyTableRead(TableType tableType) throws Exception { withNewTable( "df_empty_read", "id INT", tableType, tableName -> assertThat(spark().table(tableName).collectAsList()).isEmpty()); } @TestAllTableTypes public void testFilter(TableType tableType) throws Exception { withNewTable( "df_filter", "id INT, category STRING", tableType, tableName -> { sql("INSERT INTO %s VALUES (1, 'A'), (2, 'B'), (3, 'A'), (4, 'B')", tableName); assertThat( ids(spark().table(tableName).filter(col("category").equalTo("A")).orderBy("id"))) .containsExactly(1, 3); }); } private DataFrameReader deltaDFReader() { return spark().read().format("delta"); } private List ids(Dataset df) { return df.collectAsList().stream().map(r -> r.getInt(0)).collect(Collectors.toList()); } } ================================================ FILE: spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameStreamingTest.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.sparkuctest; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; import org.apache.spark.api.java.function.VoidFunction2; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.streaming.DataStreamReader; import org.apache.spark.sql.streaming.StreamingQuery; import org.apache.spark.sql.streaming.Trigger; import org.assertj.core.api.ThrowableAssert.ThrowingCallable; import org.junit.jupiter.api.io.TempDir; /** * DataFrame streaming test suite for Delta Table operations through Unity Catalog. * *

Covers streaming read and write via Structured Streaming. Uses {@link Trigger#AvailableNow()} * for deterministic, testable streaming without manual termination. Tests run against both EXTERNAL * and MANAGED table types. */ public class UCDeltaTableDataFrameStreamingTest extends UCDeltaTableIntegrationBaseTest { /** No-op foreachBatch sink used by negative tests that only need the stream to start. */ private static final VoidFunction2, Long> NOOP_BATCH = (df, id) -> {}; @TempDir private Path tempDir; private int checkpointCount; @TestAllTableTypes public void testStreamingReadWrite(TableType tableType) throws Exception { withNewTable( "streaming_rw_src", "id INT", tableType, srcName -> withNewTable( "streaming_rw_sink", "id INT", tableType, sinkName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", srcName); String ck = checkpoint(); // AvailableNow: process all existing data and terminate. spark() .readStream() .format("delta") .table(srcName) .writeStream() .format("delta") .outputMode("append") .trigger(Trigger.AvailableNow()) .option("checkpointLocation", ck) .toTable(sinkName) .awaitTermination(); check(sinkName, List.of(row("1"), row("2"), row("3"))); // Continuous: reuse same checkpoint so the query resumes from where // AvailableNow left off and only picks up newly inserted rows. StreamingQuery query = spark() .readStream() .format("delta") .table(srcName) .writeStream() .format("delta") .outputMode("append") .option("checkpointLocation", ck) .toTable(sinkName); try { sql("INSERT INTO %s VALUES (4), (5)", srcName); query.processAllAvailable(); check(sinkName, List.of(row("1"), row("2"), row("3"), row("4"), row("5"))); } finally { query.stop(); } })); } @TestAllTableTypes public void testStreamingReadFromVersion(TableType tableType) throws Exception { withNewTable( "streaming_version_test", "id INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); long v1 = currentVersion(tableName); sql("INSERT INTO %s VALUES (4), (5)", tableName); List result = new ArrayList<>(); spark() .readStream() .format("delta") .option("startingVersion", v1 + 1) .table(tableName) .writeStream() .trigger(Trigger.AvailableNow()) .option("checkpointLocation", checkpoint()) .foreachBatch((VoidFunction2, Long>) (df, id) -> result.addAll(ids(df))) .start() .awaitTermination(); assertThat(result).containsExactlyInAnyOrder(4, 5); }); } /** * Verifies incremental streaming via a foreachBatch sink (in-memory accumulator). Complements * {@link #testStreamingReadWrite}, which covers the same data-arrival scenario but writes to a * Delta table sink and validates via SQL. */ @TestAllTableTypes public void testStreamingContinuous(TableType tableType) throws Exception { withNewTable( "streaming_continuous_test", "id INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); List result = new ArrayList<>(); StreamingQuery query = spark() .readStream() .format("delta") .table(tableName) .writeStream() .option("checkpointLocation", checkpoint()) .foreachBatch( (VoidFunction2, Long>) (df, id) -> result.addAll(ids(df))) .start(); try { query.processAllAvailable(); assertThat(result).containsExactlyInAnyOrder(1, 2, 3); sql("INSERT INTO %s VALUES (4), (5)", tableName); query.processAllAvailable(); assertThat(result).containsExactlyInAnyOrder(1, 2, 3, 4, 5); } finally { query.stop(); } }); } /** * Verifies that {@code maxFilesPerTrigger=1} causes the stream to process each of the 3 separate * commits as its own micro-batch, producing exactly 3 batches total under {@link * Trigger#AvailableNow()}. */ @TestAllTableTypes public void testStreamingMaxFilesPerTrigger(TableType tableType) throws Exception { withNewTable( "streaming_max_files_test", "id INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1)", tableName); sql("INSERT INTO %s VALUES (2)", tableName); sql("INSERT INTO %s VALUES (3)", tableName); List result = new ArrayList<>(); List batchIds = new ArrayList<>(); spark() .readStream() .format("delta") .option("maxFilesPerTrigger", 1) .table(tableName) .writeStream() .trigger(Trigger.AvailableNow()) .option("checkpointLocation", checkpoint()) .foreachBatch( (VoidFunction2, Long>) (df, batchId) -> { result.addAll(ids(df)); batchIds.add(batchId); }) .start() .awaitTermination(); assertThat(result).containsExactlyInAnyOrder(1, 2, 3); assertThat(batchIds).hasSize(3); }); } /** * Verifies that streaming from a table that has rows deleted (RemoveFile with dataChange=true) * fails with a clear error directing the user to the {@code ignoreDeletes} option. */ @TestAllTableTypes public void testStreamingDeleteFailsWithHelpfulError(TableType tableType) throws Exception { withNewTable( "streaming_delete_error_test", "id INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); List result = new ArrayList<>(); StreamingQuery query = spark() .readStream() .format("delta") .table(tableName) .writeStream() .option("checkpointLocation", checkpoint()) .foreachBatch( (VoidFunction2, Long>) (df, id) -> result.addAll(ids(df))) .start(); try { query.processAllAvailable(); assertThat(result).containsExactlyInAnyOrder(1, 2, 3); sql("DELETE FROM %s WHERE id = 1", tableName); assertStreamingThrowsContaining(query::processAllAvailable, "ignoreDeletes"); } finally { query.stop(); } }); } /** * CDF streaming reads work for EXTERNAL tables but fail for MANAGED tables. * *

For EXTERNAL: verifies that inserts and a delete produce the expected typed change events. * For MANAGED: verifies the stream fails with an error containing "not supported" and "CDC". */ @TestAllTableTypes public void testStreamingCDFRead(TableType tableType) throws Exception { withNewTable( "streaming_cdf_read_test", "id INT", null, tableType, "'delta.enableChangeDataFeed'='true'", tableName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); long insertVersion = currentVersion(tableName); sql("DELETE FROM %s WHERE id = 1", tableName); if (tableType == TableType.EXTERNAL) { List changes = new ArrayList<>(); spark() .readStream() .format("delta") .option("readChangeFeed", "true") .option("startingVersion", insertVersion) .table(tableName) .writeStream() .trigger(Trigger.AvailableNow()) .option("checkpointLocation", checkpoint()) .foreachBatch( (VoidFunction2, Long>) (df, id) -> changes.addAll(df.select("id", "_change_type").collectAsList())) .start() .awaitTermination(); assertThat(changes) .extracting(r -> r.getString(1)) .containsExactlyInAnyOrder("insert", "insert", "insert", "delete"); assertThat(changes) .filteredOn(r -> "delete".equals(r.getString(1))) .extracting(r -> r.getInt(0)) .containsExactly(1); } else { assertInvalidStreamOption( tableName, r -> r.option("readChangeFeed", "true").option("startingVersion", insertVersion), "not supported", "CDC"); } }); } /** * Delta streaming sink does not support {@code complete} output mode. Verifies the stream fails * with a clear error mentioning "complete". */ @TestAllTableTypes public void testStreamingWriteCompleteModeNotSupported(TableType tableType) throws Exception { withNewTable( "streaming_complete_mode_src", "id INT", tableType, srcName -> withNewTable( "streaming_complete_mode_sink", "id INT", tableType, sinkName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", srcName); assertStreamingThrowsContaining( () -> spark() .readStream() .format("delta") .table(srcName) .writeStream() .format("delta") .outputMode("complete") .trigger(Trigger.AvailableNow()) .option("checkpointLocation", checkpoint()) .toTable(sinkName) .awaitTermination(), "complete"); })); } /** * Verifies that invalid or unsupported streaming read options are rejected with clear errors. * *

    *
  • Negative {@code startingVersion} is rejected (both table types). *
  • {@code startingVersion} beyond the table history is rejected (both table types). *
  • {@code ignoreChanges} and {@code ignoreDeletes} fail with a "not supported" error for * MANAGED tables; EXTERNAL tables accept these options silently. *
*/ @TestAllTableTypes public void testStreamingInvalidOptions(TableType tableType) throws Exception { withNewTable( "streaming_invalid_options_test", "id INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1)", tableName); assertInvalidStreamOption( tableName, r -> r.option("startingVersion", -1), "startingVersion"); assertInvalidStreamOption(tableName, r -> r.option("startingVersion", 999999), "999999"); if (tableType == TableType.MANAGED) { assertInvalidStreamOption( tableName, r -> r.option("ignoreChanges", "true"), "not supported", "ignoreChanges"); } }); } /** * Starts a streaming read with options applied by {@code configure}, then asserts the stream * fails with a message containing all {@code fragments} (case-insensitive). */ private void assertInvalidStreamOption( String tableName, Function configure, String... fragments) { assertStreamingThrowsContaining( () -> configure .apply(spark().readStream().format("delta")) .table(tableName) .writeStream() .trigger(Trigger.AvailableNow()) .option("checkpointLocation", checkpoint()) .foreachBatch(NOOP_BATCH) .start() .awaitTermination(), fragments); } /** * Asserts that {@code action} throws an exception whose cause chain contains all the given {@code * fragments} (case-insensitive). */ private static void assertStreamingThrowsContaining( ThrowingCallable action, String... fragments) { assertThatThrownBy(action) .satisfies( e -> { StringBuilder full = new StringBuilder(); for (Throwable t = e; t != null; t = t.getCause()) { if (t.getMessage() != null) full.append(t.getMessage()).append(' '); } String msg = full.toString(); for (String fragment : fragments) { assertThat(msg).containsIgnoringCase(fragment); } }); } private String checkpoint() throws IOException { Path ckDir = tempDir.resolve("ck-" + checkpointCount++); Files.createDirectory(ckDir); return ckDir.toString(); } private List ids(Dataset df) { return df.collectAsList().stream().map(r -> r.getInt(0)).collect(Collectors.toList()); } } ================================================ FILE: spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameWriteTest.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.sparkuctest; import static org.apache.spark.sql.functions.col; import static org.apache.spark.sql.functions.lit; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.Test; /** * DataFrame write test suite for Delta Table operations through Unity Catalog. * *

Covers DataFrame Writer V1 (insertInto, save) and Writer V2 (writeTo) operations. Tests run * against both EXTERNAL and MANAGED table types. */ public class UCDeltaTableDataFrameWriteTest extends UCDeltaTableIntegrationBaseTest { // Writer V1: insertInto @TestAllTableTypes public void testInsertIntoAppend(TableType tableType) throws Exception { withNewTable( "insert_into_append_test", "id INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); intDf(4, 5).write().mode("append").insertInto(tableName); check(tableName, List.of(row("1"), row("2"), row("3"), row("4"), row("5"))); }); } @TestAllTableTypes public void testInsertIntoOverwrite(TableType tableType) throws Exception { withNewTable( "insert_into_overwrite_test", "id INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); intDf(9).write().mode("overwrite").insertInto(tableName); check(tableName, List.of(row("9"))); }); } @TestAllTableTypes public void testInsertIntoReplaceWhere(TableType tableType) throws Exception { withNewTable( "insert_into_replace_where_test", "id INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); intDf(9).write().mode("overwrite").option("replaceWhere", "id > 1").insertInto(tableName); check(tableName, List.of(row("1"), row("9"))); }); } @TestAllTableTypes public void testSaveAsTableAppend(TableType tableType) throws Exception { withNewTable( "save_as_table_append_test", "id INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); intDf(4, 5).write().format("delta").mode("append").saveAsTable(tableName); check(tableName, List.of(row("1"), row("2"), row("3"), row("4"), row("5"))); }); } // TODO: Add saveAsTable overwrite/replaceWhere coverage once UCSingleCatalog supports REPLACE // TABLE AS SELECT (RTAS). Currently, saveAsTable with mode("overwrite") routes through Spark's // V2 catalog path as RTAS, which throws UnsupportedOperationException in UCSingleCatalog. @Test public void testSaveByPathBlockedForManagedTable() throws Exception { withNewTable( "save_path_blocked_test", "id INT", TableType.MANAGED, tableName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); String tablePath = sql("DESCRIBE FORMATTED %s", tableName).stream() .filter(r -> r.size() >= 2 && "Location".equalsIgnoreCase(r.get(0).trim())) .map(r -> r.get(1).trim()) .findFirst() .orElseThrow(); assertThatThrownBy(() -> intDf(4).write().format("delta").mode("append").save(tablePath)) .satisfies( e -> assertThat(e.getMessage()) .containsAnyOf( "Unable to load credentials", "DELTA_PATH_BASED_ACCESS_TO_CATALOG_MANAGED_TABLE_BLOCKED", "Path-based access is not allowed")); check(tableName, List.of(row("1"), row("2"), row("3"))); }); } // Writer V2: writeTo @TestAllTableTypes public void testWriteToAppend(TableType tableType) throws Exception { withNewTable( "write_to_append_test", "id INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); intDf(4, 5).writeTo(tableName).append(); check(tableName, List.of(row("1"), row("2"), row("3"), row("4"), row("5"))); }); } @TestAllTableTypes public void testWriteToOverwrite(TableType tableType) throws Exception { withNewTable( "write_to_overwrite_test", "id INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); intDf(9).writeTo(tableName).overwrite(lit(true)); check(tableName, List.of(row("9"))); }); } @TestAllTableTypes public void testWriteToOverwriteWithCondition(TableType tableType) throws Exception { withNewTable( "write_to_overwrite_condition_test", "id INT, category STRING", tableType, tableName -> { sql("INSERT INTO %s VALUES (1, 'A'), (2, 'B'), (3, 'A')", tableName); spark() .createDataFrame( List.of(RowFactory.create(9, "A")), new StructType() .add("id", DataTypes.IntegerType) .add("category", DataTypes.StringType)) .writeTo(tableName) .overwrite(col("category").equalTo("A")); check(tableName, List.of(row("2", "B"), row("9", "A"))); }); } @TestAllTableTypes public void testWriteToOverwritePartitions(TableType tableType) throws Exception { withNewTable( "write_to_overwrite_partitions_test", "id INT, category STRING", "category", tableType, tableName -> { sql("INSERT INTO %s VALUES (1, 'A'), (2, 'A'), (3, 'B')", tableName); spark() .createDataFrame( List.of(RowFactory.create(9, "A")), new StructType() .add("id", DataTypes.IntegerType) .add("category", DataTypes.StringType)) .writeTo(tableName) .overwritePartitions(); // Only partition 'A' is replaced; partition 'B' remains untouched. check(tableName, List.of(row("3", "B"), row("9", "A"))); }); } @Test public void testWriteToCreateNewManagedTable() throws Exception { String tableName = fullTableName("write_to_create_test"); try { intDf(1, 2) .writeTo(tableName) .using("delta") .tableProperty("delta.feature.catalogManaged", "supported") .create(); check(tableName, List.of(row("1"), row("2"))); } finally { sql("DROP TABLE IF EXISTS %s", tableName); } } @TestAllTableTypes public void testMergeSchema(TableType tableType) throws Exception { Assumptions.assumeFalse( isUCRemoteConfigured(), "mergeSchema not yet supported for UC managed tables remotely"); if (tableType == TableType.MANAGED) { // mergeSchema triggers updateMetadata() with a new schema, which the kill switch blocks // on CatalogOwned tables. Assert the failure rather than skipping. withNewTable( "merge_schema_blocked_test", "id INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1), (2)", tableName); assertThrowsWithCauseContaining( "Metadata changes on Unity Catalog", () -> spark() .createDataFrame( List.of(RowFactory.create(3, "extra")), new StructType() .add("id", DataTypes.IntegerType) .add("name", DataTypes.StringType)) .write() .format("delta") .mode("append") .option("mergeSchema", "true") .saveAsTable(tableName)); }); return; } withNewTable( "merge_schema_test", "id INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1), (2)", tableName); spark() .createDataFrame( List.of(RowFactory.create(3, "extra")), new StructType() .add("id", DataTypes.IntegerType) .add("name", DataTypes.StringType)) .write() .format("delta") .mode("append") .option("mergeSchema", "true") .saveAsTable(tableName); check(tableName, List.of(row("1", "null"), row("2", "null"), row("3", "extra"))); }); } @TestAllTableTypes public void testWriteToPartitionedTable(TableType tableType) throws Exception { withNewTable( "df_partitioned_write_test", "id INT, category STRING", "category", tableType, tableName -> { sql("INSERT INTO %s VALUES (1, 'A'), (2, 'B')", tableName); spark() .createDataFrame( List.of(RowFactory.create(3, "A"), RowFactory.create(4, "B")), new StructType() .add("id", DataTypes.IntegerType) .add("category", DataTypes.StringType)) .write() .mode("append") .insertInto(tableName); check(tableName, List.of(row("1", "A"), row("2", "B"), row("3", "A"), row("4", "B"))); }); } private Dataset intDf(Integer... ids) { return spark() .createDataFrame( Arrays.stream(ids).map(RowFactory::create).collect(Collectors.toList()), new StructType().add("id", DataTypes.IntegerType)); } } ================================================ FILE: spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDeltaAPITest.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.sparkuctest; import static org.apache.spark.sql.functions.col; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import io.delta.tables.DeltaTable; import java.util.List; import java.util.Map; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; /** * Programmatic DeltaTable API test suite for Delta Table operations through Unity Catalog. * *

Covers DeltaTable.forName(), update(), delete(), merge(), history(), optimize(), and restore() * via the io.delta.tables.DeltaTable API. These go through different code paths than SQL * equivalents tested in UCDeltaTableDMLTest and UCDeltaUtilityTest. Tests run against both EXTERNAL * and MANAGED table types. */ public class UCDeltaTableDeltaAPITest extends UCDeltaTableIntegrationBaseTest { @TestAllTableTypes public void testForNameAndToDF(TableType tableType) throws Exception { withNewTable( "dt_api_read", "id INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); assertThat(forName(tableName).toDF().orderBy("id").collectAsList()) .extracting(r -> r.getInt(0)) .containsExactly(1, 2, 3); }); } @TestAllTableTypes public void testUpdate(TableType tableType) throws Exception { withNewTable( "dt_api_update", "id INT, status STRING", tableType, tableName -> { sql("INSERT INTO %s VALUES (1, 'pending'), (2, 'pending'), (3, 'done')", tableName); forName(tableName).updateExpr("id = 1", Map.of("status", "'processed'")); check(tableName, List.of(row("1", "processed"), row("2", "pending"), row("3", "done"))); }); } @TestAllTableTypes public void testDelete(TableType tableType) throws Exception { withNewTable( "dt_api_delete", "id INT, active BOOLEAN", tableType, tableName -> { sql("INSERT INTO %s VALUES (1, true), (2, false), (3, true)", tableName); forName(tableName).delete(col("active").equalTo(false)); check(tableName, List.of(row("1", "true"), row("3", "true"))); }); } @TestAllTableTypes public void testMerge(TableType tableType) throws Exception { withNewTable( "dt_api_merge", "id INT, value STRING", tableType, tableName -> { sql("INSERT INTO %s VALUES (1, 'old1'), (2, 'old2')", tableName); Dataset source = spark() .createDataFrame( List.of(RowFactory.create(2, "updated2"), RowFactory.create(3, "new3")), new StructType() .add("id", DataTypes.IntegerType) .add("value", DataTypes.StringType)); forName(tableName) .as("target") .merge(source.as("source"), "target.id = source.id") .whenMatched() .updateExpr(Map.of("value", "source.value")) .whenNotMatched() .insertExpr(Map.of("id", "source.id", "value", "source.value")) .execute(); check(tableName, List.of(row("1", "old1"), row("2", "updated2"), row("3", "new3"))); }); } @TestAllTableTypes public void testHistory(TableType tableType) throws Exception { withNewTable( "dt_api_history", "id INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1)", tableName); sql("INSERT INTO %s VALUES (2)", tableName); // CREATE TABLE (v0) + 2 INSERTs (v1, v2) assertThat(forName(tableName).history().collectAsList()).hasSize(3); }); } @TestAllTableTypes public void testOptimize(TableType tableType) throws Exception { withNewTable( "dt_api_optimize", "id INT, category STRING", tableType, tableName -> { sql("INSERT INTO %s VALUES (1, 'A'), (2, 'B'), (3, 'A')", tableName); if (tableType == TableType.MANAGED) { assertThatThrownBy(() -> forName(tableName).optimize().executeCompaction()) .hasMessageContaining("DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION"); } else { assertThat(forName(tableName).optimize().executeCompaction().collectAsList()) .isNotEmpty(); assertThat(forName(tableName).optimize().executeZOrderBy("category").collectAsList()) .isNotEmpty(); } }); } @TestAllTableTypes public void testRestoreToVersion(TableType tableType) throws Exception { withNewTable( "dt_api_restore_version", "id INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); long v1 = currentVersion(tableName); sql("INSERT INTO %s VALUES (4), (5)", tableName); forName(tableName).restoreToVersion(v1); check(tableName, List.of(row("1"), row("2"), row("3"))); }); } @TestAllTableTypes public void testRestoreToTimestamp(TableType tableType) throws Exception { withNewTable( "dt_api_restore_ts", "id INT", tableType, tableName -> { sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); String ts = currentTimestamp(tableName); sql("INSERT INTO %s VALUES (4), (5)", tableName); forName(tableName).restoreToTimestamp(ts); check(tableName, List.of(row("1"), row("2"), row("3"))); }); } private DeltaTable forName(String tableName) { return DeltaTable.forName(spark(), tableName); } } ================================================ FILE: spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableIntegrationBaseTest.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.sparkuctest; import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.hadoop.fs.Path; import org.apache.spark.SparkConf; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.assertj.core.api.ThrowableAssert.ThrowingCallable; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.TestFactory; import org.opentest4j.TestAbortedException; /** * Abstract base class for Unity Catalog + Delta Table integration tests. * *

This class provides a pluggable SQL execution framework via the SQLExecutor interface, * allowing tests to be written once and executed via different execution engines (e.g., Spark SQL, * JDBC, REST API, etc.). * *

Subclasses must provide an executor by implementing the getSqlExecutor method. */ public abstract class UCDeltaTableIntegrationBaseTest extends UnityCatalogSupport { public static final List ALL_TABLE_TYPES = List.of(TableType.EXTERNAL, TableType.MANAGED); /** * Tests with this annotation will test against ALL_TABLE_TYPES. Example: * * 

{@code
   * @TestAllTableTypes
   * public void testAdvancedInsertOperations(TableType tableType)
   * }
*/ @Target(ElementType.METHOD) @Retention(RetentionPolicy.RUNTIME) public @interface TestAllTableTypes {} /** Generate dynamic tests for all methods with @TestAllTableTypes to run with ALL_TABLE_TYPES. */ @TestFactory Stream allTableTypesTestsFactory() { List methods = Stream.of(this.getClass().getDeclaredMethods()) .filter(m -> m.isAnnotationPresent(TestAllTableTypes.class)) .collect(Collectors.toList()); List containers = new ArrayList<>(); for (Method method : methods) { List tests = new ArrayList<>(); for (TableType tableType : ALL_TABLE_TYPES) { String testName = String.format("%s(%s)", method.getName(), tableType); tests.add( DynamicTest.dynamicTest( testName, () -> { try { method.invoke(this, tableType); } catch (InvocationTargetException e) { // Unwrap so JUnit sees the original exception type. Without this, // TestAbortedException (thrown by Assumptions) gets wrapped and JUnit // treats the test as failed instead of skipped. Also unwrap // RuntimeException/Error so assertThrows() in individual tests still // matches the expected exception class rather than InvocationTargetException. Throwable cause = e.getCause(); if (cause instanceof TestAbortedException) throw (TestAbortedException) cause; if (cause instanceof RuntimeException) throw (RuntimeException) cause; if (cause instanceof Error) throw (Error) cause; throw e; } })); } containers.add(DynamicContainer.dynamicContainer(method.getName(), tests)); } return containers.stream(); } private SparkSession sparkSession; /** Create the SparkSession before all tests. */ @BeforeAll public void setUpSpark() { // UC server is started by UnityCatalogSupport.setupServer() // And the BeforeAll of parent class UnityCatalogSupport will be called before this method. SparkConf conf = new SparkConf() .setAppName("UnityCatalog Integration Tests") .setMaster("local[2]") .set("spark.ui.enabled", "false") // Delta Lake required configurations .set("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .set( "spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog"); // Configure with Unity Catalog conf = configureSparkWithUnityCatalog(conf); sparkSession = SparkSession.builder().config(conf).getOrCreate(); } private SparkConf configureSparkWithUnityCatalog(SparkConf conf) { // Use fake S3 filesystem for local testing; real S3A for remote UC. if (isUCRemoteConfigured()) { conf.set("spark.hadoop.fs.s3.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem"); } else { conf.set("spark.hadoop.fs.s3.impl", S3CredentialFileSystem.class.getName()); } // Set the catalog specific configs. UnityCatalogInfo uc = unityCatalogInfo(); String catalogName = uc.catalogName(); return conf.set("spark.sql.catalog." + catalogName, "io.unitycatalog.spark.UCSingleCatalog") .set("spark.sql.catalog." + catalogName + ".uri", uc.serverUri()) .set("spark.sql.catalog." + catalogName + ".token", uc.serverToken()); } /** Stop the SparkSession after all tests. */ @AfterAll public void tearDownSpark() { if (sparkSession != null) { sparkSession.stop(); sparkSession = null; } // UC server is stopped by UnityCatalogSupport.tearDownServer() } /** Get the SparkSession for direct access (e.g., for streaming operations). */ protected SparkSession spark() { return sparkSession; } /** Get the SQL executor. Private to force subclasses to use sql() and check() methods. */ private SQLExecutor getSqlExecutor() { return new SparkSQLExecutor(sparkSession); } /** * Execute SQL through the SQL executor and return results. * *

When called with arguments, formats the SQL query using String.format: * * 

   * sql("INSERT INTO %s VALUES (%d, '%s')", tableName, 1, "value")
   *
* *

When called without arguments, executes the SQL as-is: * * 

   * sql("CREATE TABLE test (id INT)")
   *
* * @param sqlQuery SQL query with optional format specifiers (e.g., "SELECT * FROM %s WHERE id = * %d") * @param args Arguments to be formatted into the SQL query * @return List of result rows, each row is a list of string values */ protected List> sql(String sqlQuery, Object... args) { String formattedQuery = args.length > 0 ? String.format(sqlQuery, args) : sqlQuery; return getSqlExecutor().runSQL(formattedQuery); } /** * Verify table contents by selecting all rows ordered by the first column. * * @param tableName The fully qualified table name * @param expected The expected results as a list of rows */ protected void check(String tableName, List> expected) { getSqlExecutor().checkWithSQL("SELECT * FROM " + tableName + " ORDER BY 1", expected); } /** Helper method to run code with a temporary directory that gets cleaned up. */ protected void withTempDir(TempDirCode code) throws Exception { UnityCatalogInfo uc = unityCatalogInfo(); Path tempDir = new Path(uc.baseTableLocation(), "temp-" + UUID.randomUUID()); code.run(tempDir); } /** Table types for parameterized testing. */ public enum TableType { EXTERNAL, // Requires LOCATION clause MANAGED // No LOCATION clause (Spark manages the data) } /** * Helper method to create a new Delta table, run test code, and clean up. * * @param tableName The simple table name (without catalog/schema prefix) * @param tableSchema The table schema (e.g., "id INT, name STRING") * @param partitionFields The partition fields (e.g., "id, name") * @param tableType The type of table (EXTERNAL or MANAGED) * @param tableProperties Additional table properties (e.g., "delta.enableChangeDataFeed"="true") * @param testCode The test function that receives the full table name */ protected void withNewTable( String tableName, String tableSchema, String partitionFields, TableType tableType, String tableProperties, TestCode testCode) throws Exception { String fullTableName = fullTableName(tableName); // Create th partition cause. StringBuilder partitionCause = new StringBuilder(); if (partitionFields != null && !partitionFields.trim().isEmpty()) { partitionCause.append(String.format("PARTITIONED BY (%s)", partitionFields)); } // Build table properties clause StringBuilder tblPropertiesClause = new StringBuilder(); if (tableType == TableType.MANAGED) { tblPropertiesClause.append("'delta.feature.catalogManaged'='supported'"); } if (tableProperties != null && !tableProperties.trim().isEmpty()) { if (tblPropertiesClause.length() > 0) { tblPropertiesClause.append(", "); } tblPropertiesClause.append(tableProperties); } final String tblPropertiesSql; if (tblPropertiesClause.length() > 0) { tblPropertiesSql = "TBLPROPERTIES (" + tblPropertiesClause + ")"; } else { tblPropertiesSql = ""; } if (tableType == TableType.EXTERNAL) { // External table requires a location withTempDir( (Path dir) -> { Path tablePath = new Path(dir, tableName); sql( "CREATE TABLE %s (%s) USING DELTA %s %s LOCATION '%s'", fullTableName, tableSchema, partitionCause.toString(), tblPropertiesSql, tablePath.toString()); try { testCode.run(fullTableName); } finally { sql("DROP TABLE IF EXISTS %s", fullTableName); } }); } else { // Managed table - Spark manages the location // Unity Catalog requires 'delta.feature.catalogManaged'='supported' for managed tables sql( "CREATE TABLE %s (%s) USING DELTA %s %s", fullTableName, tableSchema, partitionCause.toString(), tblPropertiesSql); try { testCode.run(fullTableName); } finally { sql("DROP TABLE IF EXISTS %s", fullTableName); } } } /** * Helper method to create a new Delta table, run test code, and clean up. * * @param tableName The simple table name (without catalog/schema prefix) * @param tableSchema The table schema (e.g., "id INT, name STRING") * @param partitionFields The partition fields (e.g., "id, name") * @param tableType The type of table (EXTERNAL or MANAGED) * @param testCode The test function that receives the full table name */ protected void withNewTable( String tableName, String tableSchema, String partitionFields, TableType tableType, TestCode testCode) throws Exception { withNewTable(tableName, tableSchema, partitionFields, tableType, null, testCode); } /** * Helper method to create a new Delta table, run test code, and clean up. * * @param tableName The simple table name (without catalog/schema prefix) * @param tableSchema The table schema (e.g., "id INT, name STRING") * @param tableType The type of table (EXTERNAL or MANAGED) * @param testCode The test function that receives the full table name */ protected void withNewTable( String tableName, String tableSchema, TableType tableType, TestCode testCode) throws Exception { withNewTable(tableName, tableSchema, null, tableType, testCode); } /** Returns the fully qualified table name for a given simple table name. */ protected String fullTableName(String simpleName) { UnityCatalogInfo uc = unityCatalogInfo(); return uc.catalogName() + "." + uc.schemaName() + "." + simpleName; } /** Returns the current (latest) version of the table. */ protected long currentVersion(String tableName) { return Long.parseLong(sql("DESCRIBE HISTORY %s LIMIT 1", tableName).get(0).get(0)); } /** Returns the timestamp of the current (latest) version. */ protected String currentTimestamp(String tableName) { return sql("DESCRIBE HISTORY %s LIMIT 1", tableName).get(0).get(1); } /** * Asserts that the given operation throws an exception whose cause chain contains {@code * expectedMessage}. */ protected void assertThrowsWithCauseContaining( String expectedMessage, ThrowingCallable operation) { assertThatThrownBy(operation) .satisfies( e -> { Throwable t = e; while (t != null) { if (t.getMessage() != null && t.getMessage().contains(expectedMessage)) { return; } t = t.getCause(); } throw new AssertionError( "Expected exception containing '" + expectedMessage + "' in cause chain, but none found. Top-level: " + e, e); }); } /** Helper to build an expected row as a list of string values. */ protected static List row(String... values) { return List.of(values); } /** Functional interface for test code that takes a temporary directory. */ @FunctionalInterface protected interface TempDirCode { void run(Path dir) throws Exception; } /** Functional interface for test code that takes a table name parameter. */ @FunctionalInterface protected interface TestCode { void run(String tableName) throws Exception; } /** * Interface defining the interface for executing SQL and verifying results. * *

This abstraction allows tests to be independent of the execution engine, making it easy to * test the same logic via different interfaces (Spark SQL, JDBC, etc.). */ public interface SQLExecutor { /** * Execute a SQL statement and return the results. * * @param sql The SQL statement to execute * @return The query results as a list of rows, where each row is a list of strings */ List> runSQL(String sql); /** * Execute a SQL query and verify the results match the expected output. * * @param sql The SQL query to execute * @param expected The expected results as a list of rows */ void checkWithSQL(String sql, List> expected); } /** * Default SQL executor implementation using SparkSession. * *

This executor runs all SQL queries through Spark SQL and converts results to string lists * for easy comparison. */ public static class SparkSQLExecutor implements SQLExecutor { private final SparkSession spark; public SparkSQLExecutor(SparkSession spark) { this.spark = spark; } @Override public List> runSQL(String sql) { Dataset df = spark.sql(sql); Row[] rows = (Row[]) df.collect(); return Arrays.stream(rows) .map( row -> { List cells = new java.util.ArrayList<>(); for (int i = 0; i < row.length(); i++) { cells.add(row.isNullAt(i) ? "null" : row.get(i).toString()); } return cells; }) .collect(Collectors.toList()); } @Override public void checkWithSQL(String sql, List> expected) { List> actual = runSQL(sql); if (!actual.equals(expected)) { throw new AssertionError( String.format( "Query results do not match.\nSQL: %s\n Expected: %s\nActual: %s", sql, expected, actual)); } } } } ================================================ FILE: spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableReadTest.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.sparkuctest; import java.util.List; import org.junit.jupiter.api.Assertions; /** * Read operation test suite for Delta Table operations through Unity Catalog. * *

Covers time travel, change data feed, and path-based access scenarios. Tests are parameterized * to support different table types (EXTERNAL and MANAGED). */ public class UCDeltaTableReadTest extends UCDeltaTableIntegrationBaseTest { @TestAllTableTypes public void testTimeTravelRead(TableType tableType) throws Exception { withNewTable( "time_travel_test", "id INT", tableType, tableName -> { // Setup initial data sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); // Get current version and timestamp. long currentVersion = currentVersion(tableName); String currentTimestamp = currentTimestamp(tableName); // Add more data sql("INSERT INTO %s VALUES (4), (5)", tableName); // Test VERSION AS OF with SQL syntax List> versionResult = sql("SELECT * FROM %s VERSION AS OF %d ORDER BY id", tableName, currentVersion); check(versionResult, List.of(List.of("1"), List.of("2"), List.of("3"))); // Test TIMESTAMP AS OF with SQL syntax List> timestampResult = sql("SELECT * FROM %s TIMESTAMP AS OF '%s' ORDER BY id", tableName, currentTimestamp); check(timestampResult, List.of(List.of("1"), List.of("2"), List.of("3"))); }); } @TestAllTableTypes public void testChangeDataFeed(TableType tableType) throws Exception { withNewTable( "cdf_timestamp_test", "id INT", null, tableType, "'delta.enableChangeDataFeed'='true'", tableName -> { // Setup initial data (creates version 0) sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); // Add more data (creates version 1) sql("INSERT INTO %s VALUES (4), (5)", tableName); // Get current version and timestamp (both for version 1) long currentVersion = currentVersion(tableName); String currentTimestamp = currentTimestamp(tableName); // Query changes from version 1 (the second insert) check( sql( "SELECT id, _change_type FROM table_changes('%s', %d) ORDER BY id", tableName, currentVersion), List.of(List.of("4", "insert"), List.of("5", "insert"))); // Query changes from the timestamp of version 1 check( sql( "SELECT id, _change_type FROM table_changes('%s', '%s') ORDER BY id", tableName, currentTimestamp), List.of(List.of("4", "insert"), List.of("5", "insert"))); }); } @TestAllTableTypes public void testDeltaTableForPath(TableType tableType) throws Exception { withNewTable( "delta_table_for_path_test", "id INT", tableType, tableName -> { // Setup initial data sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); // Get table path List> describeResult = sql("DESCRIBE EXTENDED %s", tableName); // Find the Location row in the describe output String tablePath = describeResult.stream() .filter(row -> row.size() >= 2 && "Location".equals(row.get(0))) .map(row -> row.get(1)) .findFirst() .orElse(null); Assertions.assertTrue( tablePath != null && !tablePath.isEmpty(), "Could not retrieve table location from DESCRIBE EXTENDED"); // Path-based access isn't supported for catalog-owned (MANAGED) tables. if (tableType == TableType.MANAGED) { Assertions.assertThrows( Exception.class, () -> sql("SELECT * FROM delta.`%s`", tablePath), "For managed tables, path-based access should fail"); } else { // For EXTERNAL tables, path-based access should work S3CredentialFileSystem.credentialCheckEnabled = false; try { check( sql("SELECT * FROM delta.`%s` ORDER BY id", tablePath), List.of(List.of("1"), List.of("2"), List.of("3"))); } finally { S3CredentialFileSystem.credentialCheckEnabled = true; } } }); } private void check(List> actual, List> expected) { if (!actual.equals(expected)) { throw new AssertionError( String.format("Query results do not match.\nExpected: %s\nActual: %s", expected, actual)); } } } ================================================ FILE: spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaUtilityTest.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.sparkuctest; import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.util.ArrayList; import java.util.List; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; public class UCDeltaUtilityTest extends UCDeltaTableIntegrationBaseTest { @TestAllTableTypes public void testDescribeHistory(TableType tableType) throws Exception { withNewTable( "describe_history", "id INT, name STRING", tableType, tableName -> { // Assert the initial history. assertDescribeHistory(tableName, List.of(List.of("0", "CREATE TABLE", "Serializable"))); // The 1st operation. sql("INSERT INTO %s VALUES (1, 'AAA')", tableName); check(tableName, List.of(List.of("1", "AAA"))); // Assert the history. assertDescribeHistory( tableName, List.of( List.of("1", "WRITE", "Serializable"), List.of("0", "CREATE TABLE", "Serializable"))); // The 2nd operation. sql("UPDATE %s SET name='BBB' WHERE id = 1", tableName, tableName); check(tableName, List.of(List.of("1", "BBB"))); // Assert the history assertDescribeHistory( tableName, List.of( List.of("2", "UPDATE", "Serializable"), List.of("1", "WRITE", "Serializable"), List.of("0", "CREATE TABLE", "Serializable"))); }); } private void assertDescribeHistory(String tableName, List> expected) { List> results = sql("DESCRIBE HISTORY %s", tableName); // Only assert below columns, since other columns are null or undetermined (such as timestamp). // index 0: version // index 4: operation // index 10: isolationLevel List> prunedResults = new ArrayList<>(); for (List row : results) { prunedResults.add(List.of(row.get(0), row.get(4), row.get(10))); } Assertions.assertThat(prunedResults).isEqualTo(expected); } @TestAllTableTypes public void testFsPropertiesHiddenFromTableProperties(TableType tableType) throws Exception { withNewTable( "fs_props_hidden", "id INT, name STRING", null, // no partition tableType, "'myCustomProp'='myCustomValue'", tableName -> { // SHOW TBLPROPERTIES returns one row per property (key, value). List> propRows = sql("SHOW TBLPROPERTIES %s", tableName); List propKeys = new ArrayList<>(); for (List row : propRows) { propKeys.add(row.get(0)); } // Verify no key starts with option.fs. — these are internal catalog-vended // credentials/metadata that should not be user-visible. for (String key : propKeys) { Assertions.assertThat(key) .as("SHOW TBLPROPERTIES should not expose option.fs.* keys") .doesNotStartWith("option.fs."); } // Verify that non-fs storage properties and user-set table properties ARE // still present — confirming the filter is selective, not a blanket removal. Assertions.assertThat(propKeys) .as("User-set table properties should still be visible") .contains("myCustomProp"); Assertions.assertThat(propKeys) .as("Delta table properties should still be visible") .contains("delta.minReaderVersion"); // DESCRIBE EXTENDED returns a "Table Properties" row with all properties // in a single string like "[key1=val1,key2=val2,...]". boolean foundTableProperties = false; List> descRows = sql("DESCRIBE EXTENDED %s", tableName); for (List row : descRows) { if (row.size() >= 2 && "Table Properties".equals(row.get(0))) { foundTableProperties = true; Assertions.assertThat(row.get(1)) .as("DESCRIBE EXTENDED should not expose option.fs.* storage properties") .doesNotContain("option.fs."); Assertions.assertThat(row.get(1)) .as("DESCRIBE EXTENDED should not expose fs.* storage properties either") .doesNotContain("fs."); Assertions.assertThat(row.get(1)) .as("DESCRIBE EXTENDED should still show user-set properties") .contains("myCustomProp=myCustomValue"); } } Assertions.assertThat(foundTableProperties) .as("DESCRIBE EXTENDED must include a 'Table Properties' row") .isTrue(); // Verify the data path still works — credentials still flow to the filesystem // via CatalogTable.storage.properties even though they are hidden from properties(). sql("INSERT INTO %s VALUES (1, 'hello'), (2, 'world')", tableName); check(tableName, List.of(List.of("1", "hello"), List.of("2", "world"))); sql("INSERT INTO %s VALUES (3, 'foo')", tableName); check( tableName, List.of(List.of("1", "hello"), List.of("2", "world"), List.of("3", "foo"))); }); } @Test public void testMaintenanceOpsBlockedOnManagedTable() throws Exception { withNewTable( "maintenance_blocked", "id INT", TableType.MANAGED, tableName -> { sql("INSERT INTO %s VALUES (1)", tableName); assertThatThrownBy(() -> sql("OPTIMIZE %s", tableName)) .hasMessageContaining("DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION") .hasMessageContaining("OPTIMIZE"); assertThatThrownBy(() -> sql("VACUUM %s", tableName)) .hasMessageContaining("DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION") .hasMessageContaining("VACUUM"); assertThatThrownBy(() -> sql("REORG TABLE %s APPLY (PURGE)", tableName)) .hasMessageContaining("DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION") .hasMessageContaining("OPTIMIZE"); }); } } ================================================ FILE: spark/unitycatalog/src/test/java/io/sparkuctest/UnityCatalogSupport.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.sparkuctest; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import io.unitycatalog.client.ApiClient; import io.unitycatalog.client.ApiClientBuilder; import io.unitycatalog.client.VersionUtils; import io.unitycatalog.client.api.CatalogsApi; import io.unitycatalog.client.api.SchemasApi; import io.unitycatalog.client.auth.TokenProvider; import io.unitycatalog.client.model.CreateCatalog; import io.unitycatalog.client.model.CreateSchema; import io.unitycatalog.server.UnityCatalogServer; import io.unitycatalog.server.utils.ServerProperties; import java.io.File; import java.io.IOException; import java.net.ServerSocket; import java.nio.file.Files; import java.util.Properties; import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.TestInstance; /** * Abstract base class that provides Unity Catalog server integration for Delta tests. * *

Automatically starts a local Unity Catalog server before tests and stops it after. To use a * remote server instead, set {@code UC_REMOTE=true} and configure {@code UC_URI}, {@code UC_TOKEN}, * {@code UC_CATALOG_NAME}, {@code UC_SCHEMA_NAME}, and {@code UC_BASE_TABLE_LOCATION}. * *

{@code unityCatalogInfo()} is the only API for subclasses, All other methods are internal * implementation details. * * 

 * public class MyUCTest extends UnityCatalogSupport {
 *   {@literal @}Test
 *   public void myTest() {
 *     UnityCatalogInfo ucInfo = unityCatalogInfo();
 *     String tableName = ucInfo.catalogName() + "." + ucInfo.schemaName() + ".my_table";
 *     spark.sql("CREATE TABLE " + tableName + " (id INT) USING DELTA");
 *   }
 * }
 *
*/ @TestInstance(TestInstance.Lifecycle.PER_CLASS) public abstract class UnityCatalogSupport { private static final Logger LOG = Logger.getLogger(UnityCatalogSupport.class); protected static class UnityCatalogInfo { private final String serverUri; private final String catalogName; private final String serverToken; private final String schemaName; private final String baseTableLocation; public UnityCatalogInfo( String serverUri, String catalogName, String serverToken, String schemaName, String baseTableLocation) { this.serverUri = serverUri; this.catalogName = catalogName; this.serverToken = serverToken; this.schemaName = schemaName; this.baseTableLocation = baseTableLocation; } public String serverUri() { return serverUri; } public String catalogName() { return catalogName; } public String serverToken() { return serverToken; } public String schemaName() { return schemaName; } public String baseTableLocation() { return baseTableLocation; } /** Creates a configured Unity Catalog API client. */ public ApiClient createApiClient() { return ApiClientBuilder.create() .uri(serverUri) .tokenProvider( TokenProvider.create(ImmutableMap.of("type", "static", "token", serverToken))) .build(); } } public static final String UC_STATIC_TOKEN = "static-token"; /** The fake S3 bucket name used for local integration tests. */ static final String FAKE_S3_BUCKET = "fakeS3Bucket"; // Environment variables for configuring access to remote unity catalog server. public static final String UC_REMOTE = "UC_REMOTE"; public static final String UC_URI = "UC_URI"; public static final String UC_TOKEN = "UC_TOKEN"; public static final String UC_CATALOG_NAME = "UC_CATALOG_NAME"; public static final String UC_SCHEMA_NAME = "UC_SCHEMA_NAME"; public static final String UC_BASE_TABLE_LOCATION = "UC_BASE_TABLE_LOCATION"; protected static boolean isUCRemoteConfigured() { String ucRemote = System.getenv(UC_REMOTE); return ucRemote != null && ucRemote.equalsIgnoreCase("true"); } /** The Unity Catalog info instance for subclasses access */ private UnityCatalogInfo ucInfo = null; /** The Unity Catalog server instance. */ private UnityCatalogServer ucServer; /** The port on which the UC server is running. */ private int ucServerPort; /** The temporary directory for UC server data. */ private File ucServerDir; /** The temporary directory for external table location */ private File ucBaseTableLocation = null; /** * Returns the Unity Catalog configuration for use in tests. * *

This is the primary method subclasses should use to access Unity Catalog connection details, * authentication tokens, and storage locations. * *

Note: This is the only public API intended for subclasses. All other * methods are internal implementation details. * * @return the Unity Catalog configuration * @see UnityCatalogInfo */ protected synchronized UnityCatalogInfo unityCatalogInfo() { Preconditions.checkNotNull( ucInfo, "No UnityCatalogInfo available, please make sure the unity catalog server is available"); return ucInfo; } private UnityCatalogInfo remoteUnityCatalogInfo() { String serverUri = System.getenv(UC_URI); String catalogName = System.getenv(UC_CATALOG_NAME); String serverToken = System.getenv(UC_TOKEN); String schemaName = System.getenv(UC_SCHEMA_NAME); String baseTableLocation = System.getenv(UC_BASE_TABLE_LOCATION); Preconditions.checkNotNull(serverUri, "%s must be set when UC_REMOTE=true", UC_URI); Preconditions.checkNotNull(catalogName, "%s must be set when UC_REMOTE=true", UC_CATALOG_NAME); Preconditions.checkNotNull(serverToken, "%s must be set when UC_REMOTE=true", UC_TOKEN); Preconditions.checkNotNull(schemaName, "%s must be set when UC_REMOTE=true", UC_SCHEMA_NAME); Preconditions.checkNotNull( baseTableLocation, "%s must be set when UC_REMOTE=true", UC_BASE_TABLE_LOCATION); return new UnityCatalogInfo(serverUri, catalogName, serverToken, schemaName, baseTableLocation); } private UnityCatalogInfo localUnityCatalogInfo() { Preconditions.checkNotNull(ucServer, "Local Unity Catalog Server is not configured"); Preconditions.checkNotNull( ucBaseTableLocation, "Local Unity Catalog Temp Directory is not configured"); // Use fake S3 bucket (backed by local filesystem via S3CredentialFileSystem). return new UnityCatalogInfo( String.format("http://localhost:%s/", ucServerPort), "unity", UC_STATIC_TOKEN, "default", "s3://" + FAKE_S3_BUCKET + ucBaseTableLocation.getAbsolutePath()); } /** Finds an available port for the UC server. */ private int findAvailablePort() throws IOException { try (ServerSocket socket = new ServerSocket(0)) { return socket.getLocalPort(); } } /** * Starts the Unity Catalog server before all tests. IMPORTANT: Starts the server BEFORE calling * other setup to ensure the server is running when SharedSparkSession creates the SparkSession. */ @BeforeAll public void setupServer() throws Exception { if (isUCRemoteConfigured()) { setUpRemoteServer(); } else { setUpLocalServer(); } } private void setUpRemoteServer() { // For remote UC, log the configuration ucInfo = remoteUnityCatalogInfo(); LOG.info("Using remote Unity Catalog server at " + ucInfo.serverUri()); LOG.info("Catalog: " + ucInfo.catalogName() + ", Schema: " + ucInfo.schemaName()); LOG.info("Base location: " + ucInfo.baseTableLocation()); LOG.info( "Note: Schema '" + ucInfo.catalogName() + "." + ucInfo.schemaName() + "' must already exist in the remote UC server"); } private void setUpLocalServer() throws Exception { // Create temporary directory for UC server data ucServerDir = Files.createTempDirectory("unity-catalog-test-").toFile(); ucServerDir.deleteOnExit(); // Create temporary directory for external tables testing. ucBaseTableLocation = Files.createTempDirectory("base-table-location-").toFile(); ucBaseTableLocation.deleteOnExit(); // Find an available port ucServerPort = findAvailablePort(); // Set up server properties Properties serverProps = new Properties(); serverProps.setProperty("server.env", "test"); // Enable managed tables (experimental feature in Unity Catalog) serverProps.setProperty("server.managed-table.enabled", "true"); serverProps.setProperty( "storage-root.tables", new File(ucServerDir, "ucroot").getAbsolutePath()); // Configure S3 credentials for the fake bucket (mirrors UC OSS BaseSparkIntegrationTest). serverProps.setProperty("s3.bucketPath.0", "s3://" + FAKE_S3_BUCKET); serverProps.setProperty("s3.accessKey.0", "fakeAccessKey"); serverProps.setProperty("s3.secretKey.0", "fakeSecretKey"); serverProps.setProperty("s3.sessionToken.0", "fakeSessionToken"); // Start UC server with configuration ServerProperties initServerProperties = new ServerProperties(serverProps); UnityCatalogServer server = UnityCatalogServer.builder() .port(ucServerPort) .serverProperties(initServerProperties) .build(); server.start(); ucServer = server; // Poll for server readiness by checking if we can create an API client and query catalogs int maxRetries = 30; int retryDelayMs = 500; boolean serverReady = false; int retries = 0; ucInfo = localUnityCatalogInfo(); while (!serverReady && retries < maxRetries) { try { CatalogsApi catalogsApi = new CatalogsApi(ucInfo.createApiClient()); catalogsApi.listCatalogs(null, null); // This will throw if server is not ready serverReady = true; } catch (Exception e) { Thread.sleep(retryDelayMs); retries++; } } if (!serverReady) { throw new RuntimeException( "Unity Catalog server did not become ready after " + (maxRetries * retryDelayMs) + "ms"); } // Create the catalog and default schema in the UC server ApiClient client = ucInfo.createApiClient(); CatalogsApi catalogsApi = new CatalogsApi(client); SchemasApi schemasApi = new SchemasApi(client); // Create catalog catalogsApi.createCatalog( new CreateCatalog() .name(ucInfo.catalogName()) .comment("Test catalog for Delta Lake integration")); // Create default schema schemasApi.createSchema(new CreateSchema().name("default").catalogName(ucInfo.catalogName())); LOG.info("Unity Catalog server started and ready at " + ucInfo.serverUri()); LOG.info("Created catalog '" + ucInfo.catalogName() + "' with schema 'default'"); } /** Stops the Unity Catalog server after all tests. */ @AfterAll public void tearDownServer() { if (isUCRemoteConfigured()) { return; } if (ucServer != null) { ucServer.stop(); LOG.info("Unity Catalog server stopped"); ucServer = null; } // Clean up uc server temporary directory if (ucServerDir != null && ucServerDir.exists()) { deleteRecursively(ucServerDir); } // Clear up base table locations. if (ucBaseTableLocation != null && ucBaseTableLocation.exists()) { deleteRecursively(ucBaseTableLocation); } } /** Recursively deletes a directory and all its contents. */ private void deleteRecursively(File file) { FileUtils.deleteQuietly(file); } /** Returns Unity Catalog Spark version, like [0, 4, 0]. */ protected static int[] getUnityCatalogSparkVersion() { String version = Preconditions.checkNotNull(VersionUtils.VERSION); String[] parts = version.split("[.\\-]", 4); int major = Integer.parseInt(parts[0]); int minor = Integer.parseInt(parts[1]); int patch = Integer.parseInt(parts[2]); return new int[] {major, minor, patch}; } } ================================================ FILE: spark/unitycatalog/src/test/java/io/sparkuctest/UnityCatalogSupportTest.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.sparkuctest; import static io.sparkuctest.UnityCatalogSupport.UC_BASE_TABLE_LOCATION; import static io.sparkuctest.UnityCatalogSupport.UC_CATALOG_NAME; import static io.sparkuctest.UnityCatalogSupport.UC_REMOTE; import static io.sparkuctest.UnityCatalogSupport.UC_SCHEMA_NAME; import static io.sparkuctest.UnityCatalogSupport.UC_TOKEN; import static io.sparkuctest.UnityCatalogSupport.UC_URI; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.sparkuctest.UnityCatalogSupport.UnityCatalogInfo; import java.lang.reflect.Field; import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; public class UnityCatalogSupportTest { private static final List ALL_ENVS = ImmutableList.of( UC_REMOTE, UC_URI, UC_TOKEN, UC_CATALOG_NAME, UC_SCHEMA_NAME, UC_BASE_TABLE_LOCATION); @Test public void testUnityCatalogInfo() throws Exception { withEnvTesting( ImmutableMap.of( UC_REMOTE, "true", UC_URI, "http://localhost:8080", UC_TOKEN, "TestRemoteToken", UC_CATALOG_NAME, "TestRemoteCatalog", UC_SCHEMA_NAME, "TestRemoteSchema", UC_BASE_TABLE_LOCATION, "s3://test-bucket/key"), () -> { TestingUCSupport ucSupport = new TestingUCSupport(); UnityCatalogInfo uc = ucSupport.accessUnityCatalogInfo(); assertThat(uc.catalogName()).isEqualTo("TestRemoteCatalog"); assertThat(uc.serverUri()).isEqualTo("http://localhost:8080"); assertThat(uc.serverToken()).isEqualTo("TestRemoteToken"); assertThat(uc.schemaName()).isEqualTo("TestRemoteSchema"); assertThat(uc.baseTableLocation()).isEqualTo("s3://test-bucket/key"); }); } @Test public void testNoUri() throws Exception { withEnvTesting( ImmutableMap.of( UC_REMOTE, "true", UC_TOKEN, "TestRemoteToken", UC_CATALOG_NAME, "TestRemoteCatalog", UC_SCHEMA_NAME, "TestRemoteSchema", UC_BASE_TABLE_LOCATION, "s3://test-bucket/key"), () -> { TestingUCSupport uc = new TestingUCSupport(); assertThatThrownBy(uc::accessUnityCatalogInfo) .isInstanceOf(NullPointerException.class) .hasMessageContaining("UC_URI must be set when UC_REMOTE=true"); }); } @Test public void testNoCatalogName() throws Exception { withEnvTesting( ImmutableMap.of( UC_REMOTE, "true", UC_URI, "http://localhost:8080", UC_TOKEN, "TestRemoteToken", UC_SCHEMA_NAME, "TestRemoteSchema", UC_BASE_TABLE_LOCATION, "s3://test-bucket/key"), () -> { TestingUCSupport uc = new TestingUCSupport(); assertThatThrownBy(uc::accessUnityCatalogInfo) .isInstanceOf(NullPointerException.class) .hasMessageContaining("UC_CATALOG_NAME must be set when UC_REMOTE=true"); }); } @Test public void testNoToken() throws Exception { withEnvTesting( ImmutableMap.of( UC_REMOTE, "true", UC_URI, "http://localhost:8080", UC_CATALOG_NAME, "TestRemoteCatalog", UC_SCHEMA_NAME, "TestRemoteSchema", UC_BASE_TABLE_LOCATION, "s3://test-bucket/key"), () -> { TestingUCSupport uc = new TestingUCSupport(); assertThatThrownBy(uc::accessUnityCatalogInfo) .isInstanceOf(NullPointerException.class) .hasMessageContaining("UC_TOKEN must be set when UC_REMOTE=true"); }); } @Test public void testNoSchemaName() throws Exception { withEnvTesting( ImmutableMap.of( UC_REMOTE, "true", UC_URI, "http://localhost:8080", UC_TOKEN, "TestRemoteToken", UC_CATALOG_NAME, "TestRemoteCatalog", UC_BASE_TABLE_LOCATION, "s3://test-bucket/key"), () -> { TestingUCSupport uc = new TestingUCSupport(); assertThatThrownBy(uc::accessUnityCatalogInfo) .isInstanceOf(NullPointerException.class) .hasMessageContaining("UC_SCHEMA_NAME must be set when UC_REMOTE=true"); }); } @Test public void testNoBaseTableLocation() throws Exception { withEnvTesting( ImmutableMap.of( UC_REMOTE, "true", UC_URI, "http://localhost:8080", UC_TOKEN, "TestRemoteToken", UC_CATALOG_NAME, "TestRemoteCatalog", UC_SCHEMA_NAME, "TestRemoteSchema"), () -> { TestingUCSupport uc = new TestingUCSupport(); assertThatThrownBy(uc::accessUnityCatalogInfo) .isInstanceOf(NullPointerException.class) .hasMessageContaining("UC_BASE_TABLE_LOCATION must be set when UC_REMOTE=true"); }); } public interface TestCall { void call() throws Exception; } public void withEnvTesting(Map envs, TestCall testCall) throws Exception { // Clear all UC-related environment variables first to ensure clean state ALL_ENVS.forEach(UnityCatalogSupportTest::removeEnv); envs.forEach(UnityCatalogSupportTest::setEnv); try { testCall.call(); } finally { // Clean up all UC-related environment variables after test ALL_ENVS.forEach(UnityCatalogSupportTest::removeEnv); } } @SuppressWarnings("unchecked") private static void setEnv(String key, String value) { try { Map env = System.getenv(); Field f = env.getClass().getDeclaredField("m"); f.setAccessible(true); ((Map) f.get(env)).put(key, value); } catch (Exception e) { throw new RuntimeException(e); } } @SuppressWarnings("unchecked") private static void removeEnv(String key) { try { Map env = System.getenv(); Field field = env.getClass().getDeclaredField("m"); field.setAccessible(true); ((Map) field.get(env)).remove(key); } catch (NoSuchFieldException e) { // Ignore if field doesn't exist (different JVM implementation) } catch (Exception e) { throw new RuntimeException(e); } } private static class TestingUCSupport extends UnityCatalogSupport { public UnityCatalogInfo accessUnityCatalogInfo() throws Exception { setupServer(); return unityCatalogInfo(); } } } ================================================ FILE: spark/v2/README.md ================================================ # Delta Spark V2 Connector The **Delta Spark V2 Connector** enables Apache Spark to read Delta tables using the Delta Kernel. It leverages Spark's DataSource V2 (DSV2) APIs to integrate Delta Kernel into Spark's query execution pipeline. --- ## High-Level Design The **Delta Spark V2 Connector** sits between Spark and Delta Kernel, bridging the two: 1. **Spark Driver** requests the table schema and pushes down both static and dynamic filters through the connector. 2. **Delta Spark V2 Connector** translates these requests to Delta Kernel APIs: - Requests table schema from Delta Kernel. - Pushes down filters and fetches the list of files to scan. 3. **Delta Kernel** resolves table state from Catalog and Delta logs, applies file skipping, and returns the necessary files. 4. **Spark Engine** partitions the files (default 128MB splits) and executes the actual Parquet scans using Spark’s existing `ParquetFileFormat`.

Delta-kernel-connector

================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/SparkTable.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.catalog; import static io.delta.spark.internal.v2.utils.ScalaUtils.toJavaOptional; import static io.delta.spark.internal.v2.utils.ScalaUtils.toScalaMap; import static io.delta.spark.internal.v2.utils.StatsUtils.toV2Statistics; import static java.util.Objects.requireNonNull; import io.delta.kernel.Snapshot; import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.engine.Engine; import io.delta.spark.internal.v2.read.SparkScanBuilder; import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; import io.delta.spark.internal.v2.snapshot.SnapshotManagerFactory; import io.delta.spark.internal.v2.utils.SchemaUtils; import java.util.*; import java.util.function.Supplier; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.catalyst.catalog.CatalogTable; import org.apache.spark.sql.connector.catalog.*; import org.apache.spark.sql.connector.expressions.Expressions; import org.apache.spark.sql.connector.expressions.Transform; import org.apache.spark.sql.connector.read.ScanBuilder; import org.apache.spark.sql.connector.read.Statistics; import org.apache.spark.sql.connector.write.LogicalWriteInfo; import org.apache.spark.sql.connector.write.WriteBuilder; import org.apache.spark.sql.delta.DeltaTableUtils; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; /** DataSource V2 Table implementation for Delta Lake using the Delta Kernel API. */ public class SparkTable implements Table, SupportsRead, SupportsWrite { private static final Set CAPABILITIES = Collections.unmodifiableSet( EnumSet.of( TableCapability.BATCH_READ, TableCapability.MICRO_BATCH_READ, TableCapability.BATCH_WRITE)); private final Identifier identifier; private final String tablePath; private final Map options; private final DeltaSnapshotManager snapshotManager; /** Snapshot created during connector setup */ private final Snapshot initialSnapshot; private final Configuration hadoopConf; private final SchemaProvider schemaProvider; private final Optional catalogTable; /** * Creates a SparkTable from a filesystem path without a catalog table. * * @param identifier logical table identifier used by Spark's catalog * @param tablePath filesystem path to the Delta table root * @throws NullPointerException if identifier or tablePath is null */ public SparkTable(Identifier identifier, String tablePath) { this(identifier, tablePath, Collections.emptyMap(), Optional.empty()); } /** * Creates a SparkTable from a filesystem path with options. * * @param identifier logical table identifier used by Spark's catalog * @param tablePath filesystem path to the Delta table root * @param options table options used to configure the Hadoop conf, table reads and writes * @throws NullPointerException if identifier or tablePath is null */ public SparkTable(Identifier identifier, String tablePath, Map options) { this(identifier, tablePath, options, Optional.empty()); } /** * Constructor that accepts a Spark CatalogTable and user-provided options. Extracts the table * location and storage properties from the catalog table, then merges with user options. User * options take precedence over catalog properties in case of conflicts. * * @param identifier logical table identifier used by Spark's catalog * @param catalogTable the Spark CatalogTable containing table metadata including location * @param options user-provided options to override catalog properties */ public SparkTable(Identifier identifier, CatalogTable catalogTable, Map options) { this( identifier, getDecodedPath(requireNonNull(catalogTable, "catalogTable is null").location()), options, Optional.of(catalogTable)); } /** * Creates a SparkTable backed by a Delta Kernel snapshot manager and initializes Spark-facing * metadata (schemas, partitioning, capabilities). * *

Side effects: - Initializes a SnapshotManager for the given tablePath. - Loads the latest * snapshot via the manager. - Builds Hadoop configuration from options for subsequent I/O. - * Derives data schema, partition schema, and full table schema from the snapshot. * *

Notes: - Partition column order from the snapshot is preserved for partitioning and appended * after data columns in the public Spark schema, per Spark conventions. - Read-time scan options * are later merged with these options. */ private SparkTable( Identifier identifier, String tablePath, Map userOptions, Optional catalogTable) { this.identifier = requireNonNull(identifier, "identifier is null"); this.tablePath = requireNonNull(tablePath, "tablePath is null"); this.catalogTable = catalogTable; // Merge options: file system options from catalog + user options (user takes precedence) // This follows the same pattern as DeltaTableV2 in delta-spark Map merged = new HashMap<>(); // Only extract file system options from table storage properties catalogTable.ifPresent( table -> scala.collection.JavaConverters.mapAsJavaMap(table.storage().properties()) .forEach( (key, value) -> { if (DeltaTableUtils.validDeltaTableHadoopPrefixes() .exists(prefix -> key.startsWith(prefix))) { merged.put(key, value); } })); // User options override catalog properties merged.putAll(userOptions); this.options = Collections.unmodifiableMap(merged); this.hadoopConf = SparkSession.active().sessionState().newHadoopConfWithOptions(toScalaMap(options)); Engine kernelEngine = DefaultEngine.create(this.hadoopConf); this.snapshotManager = SnapshotManagerFactory.create(tablePath, kernelEngine, catalogTable); // Load the initial snapshot through the manager this.initialSnapshot = snapshotManager.loadLatestSnapshot(); // Schema-related metadata is lazily computed on first access within SchemaProvider this.schemaProvider = new SchemaProvider(SparkSession.active(), initialSnapshot); } /** * Helper method to decode URI path handling URL-encoded characters correctly. E.g., converts * "spark%25dir%25prefix" to "spark%dir%prefix" * *

Uses Hadoop's Path class to properly handle all URI schemes (file, s3, abfss, gs, hdfs, * etc.), not just file:// URIs. */ private static String getDecodedPath(java.net.URI location) { Path hadoopPath = new Path(location); // For local file system paths, return just the path component without the scheme // to maintain consistency with path-based table construction where tablePath is a // plain filesystem path string. if (location.getScheme() == null || "file".equals(location.getScheme())) { return hadoopPath.toUri().getPath(); } return hadoopPath.toString(); } /** * Returns the CatalogTable if this SparkTable was created from a catalog table. * * @return Optional containing the CatalogTable, or empty if this table was created from a path */ public Optional getCatalogTable() { return catalogTable; } /** * Returns the Path to the Delta table root. * * @return Path created from the table path */ public Path getTablePath() { return new Path(tablePath); } /** * Returns the table name in a format compatible with DeltaTableV2. * *

For catalog-based tables, returns the fully qualified table name (e.g., * "spark_catalog.default.table_name"). For path-based tables, returns the path-based identifier * (e.g., "delta.`/path/to/table`"). * * @return the table name string */ @Override public String name() { return catalogTable .map(ct -> ct.identifier().unquotedString()) .orElse("delta.`" + tablePath + "`"); } @Override public StructType schema() { return schemaProvider.getPublicSchema(); } @Override public Column[] columns() { return schemaProvider.getColumns(); } @Override public Transform[] partitioning() { return schemaProvider.getPartitionTransforms(); } @Override public Map properties() { Map props = new HashMap<>(initialSnapshot.getTableProperties()); return Collections.unmodifiableMap(props); } @Override public Set capabilities() { return CAPABILITIES; } @Override public ScanBuilder newScanBuilder(CaseInsensitiveStringMap scanOptions) { Map combined = new HashMap<>(this.options); combined.putAll(scanOptions.asCaseSensitiveMap()); CaseInsensitiveStringMap merged = new CaseInsensitiveStringMap(combined); Optional catalogStats = catalogTable .flatMap(ct -> toJavaOptional(ct.stats())) .map( stats -> toV2Statistics( stats, schemaProvider.getDataSchema(), schemaProvider.getPartitionSchema())); return new SparkScanBuilder( name(), initialSnapshot, snapshotManager, schemaProvider.getDataSchema(), schemaProvider.getPartitionSchema(), schemaProvider.getRawSchema(), catalogStats, merged); } /** * Batch write for Delta tables via the DSv2 connector is not yet supported. * *

The write entrypoint is intentionally present to advertise DSv2 write capability while * follow-up changes land the full write implementation. */ @Override public WriteBuilder newWriteBuilder(LogicalWriteInfo info) { requireNonNull(info, "write info is null"); throw new UnsupportedOperationException( "Batch write for Delta tables via the DSv2 connector is not yet supported."); } @Override public String toString() { return "SparkTable{identifier=" + identifier + '}'; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } SparkTable that = (SparkTable) o; return Objects.equals(identifier, that.identifier) && Objects.equals(tablePath, that.tablePath) && Objects.equals(options, that.options) && Objects.equals(catalogTable, that.catalogTable) && Objects.equals(initialSnapshot.getPath(), that.initialSnapshot.getPath()) && initialSnapshot.getVersion() == that.initialSnapshot.getVersion(); } @Override public int hashCode() { return Objects.hash( identifier, tablePath, options, catalogTable, initialSnapshot.getPath(), initialSnapshot.getVersion()); } /** * Private helper class that lazily computes and caches schema-related metadata. * *

This class encapsulates all schema computation logic including: * *

    *
  • Raw schema conversion from Kernel to Spark *
  • Public schema with internal metadata removed *
  • Data and partition schema derivation *
  • Column and partition transform creation *
* *

All schema computations are deferred until first access. */ private static class SchemaProvider { private final SparkSession sparkSession; private final Snapshot snapshot; // Lazily computed fields private boolean initialized = false; private StructType rawSchema; private StructType publicSchema; private List partColNames; private StructType dataSchema; private StructType partitionSchema; private Column[] columns; private Transform[] partitionTransforms; SchemaProvider(SparkSession sparkSession, Snapshot snapshot) { this.sparkSession = sparkSession; this.snapshot = snapshot; } private synchronized void ensureInitialized() { if (initialized) { return; } // Convert Kernel schema to Spark schema - keep all metadata for internal use this.rawSchema = SchemaUtils.convertKernelSchemaToSparkSchema(snapshot.getSchema()); // Create public schema by removing internal metadata (for schema() method) this.publicSchema = DeltaTableUtils.removeInternalDeltaMetadata( sparkSession, DeltaTableUtils.removeInternalWriterMetadata(sparkSession, rawSchema)); this.partColNames = Collections.unmodifiableList(new ArrayList<>(snapshot.getPartitionColumnNames())); final List dataFields = new ArrayList<>(); final List partitionFields = new ArrayList<>(); // Build a map for O(1) field lookups to improve performance // Use rawSchema (with metadata) for deriving data and partition schemas Map fieldMap = new HashMap<>(); for (StructField field : rawSchema.fields()) { fieldMap.put(field.name(), field); } // IMPORTANT: Add partition fields in the exact order specified by partColNames // This is crucial because the order in partColNames may differ from the order // in snapshotSchema, and we need to preserve the partColNames order for // proper partitioning behavior for (String partColName : partColNames) { StructField field = fieldMap.get(partColName); if (field != null) { partitionFields.add(field); } } // Add remaining fields as data fields (non-partition columns) // These are fields that exist in the schema but are not partition columns for (StructField field : rawSchema.fields()) { if (!partColNames.contains(field.name())) { dataFields.add(field); } } this.dataSchema = new StructType(dataFields.toArray(new StructField[0])); this.partitionSchema = new StructType(partitionFields.toArray(new StructField[0])); // Use publicSchema (cleaned) for external API this.columns = CatalogV2Util.structTypeToV2Columns(publicSchema); this.partitionTransforms = partColNames.stream().map(Expressions::identity).toArray(Transform[]::new); this.initialized = true; } private T withInit(Supplier supplier) { ensureInitialized(); return supplier.get(); } StructType getPublicSchema() { return withInit(() -> publicSchema); } StructType getDataSchema() { return withInit(() -> dataSchema); } StructType getPartitionSchema() { return withInit(() -> partitionSchema); } StructType getRawSchema() { return withInit(() -> rawSchema); } Column[] getColumns() { return withInit(() -> columns); } Transform[] getPartitionTransforms() { return withInit(() -> partitionTransforms); } } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/exception/VersionNotFoundException.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.exception; /** Exception thrown when a requested version is not available in the Delta log. */ public class VersionNotFoundException extends RuntimeException { private final long userVersion; private final long earliest; private final long latest; public VersionNotFoundException(long userVersion, long earliest, long latest) { super( String.format( "Cannot time travel Delta table to version %d. Available versions: [%d, %d].", userVersion, earliest, latest)); this.userVersion = userVersion; this.earliest = earliest; this.latest = latest; } public long getUserVersion() { return userVersion; } public long getEarliest() { return earliest; } public long getLatest() { return latest; } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaInputPartition.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.read; import java.util.Objects; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.connector.read.HasPartitionKey; import org.apache.spark.sql.connector.read.InputPartition; import org.apache.spark.sql.execution.datasources.FilePartition; /** * A Delta-specific InputPartition that wraps a FilePartition and implements HasPartitionKey. This * enables Spark to leverage partition information for optimizations like shuffle elimination in * joins and aggregations when using KeyGroupedPartitioning. * *

Each DeltaInputPartition represents files from a single logical partition (i.e., all files * share the same partition values). The partition key is the InternalRow containing the partition * column values. */ public class DeltaInputPartition implements InputPartition, HasPartitionKey { private final FilePartition filePartition; private final InternalRow partitionKey; /** * Creates a new DeltaInputPartition. * * @param filePartition The underlying FilePartition containing the files to read. * @param partitionKey The partition key (partition column values) for all files in this * partition. Must not be null. */ public DeltaInputPartition(FilePartition filePartition, InternalRow partitionKey) { this.filePartition = Objects.requireNonNull(filePartition, "filePartition is null"); this.partitionKey = Objects.requireNonNull(partitionKey, "partitionKey is null"); } /** * Returns the partition key (partition column values) associated with this partition. All files * in this partition have the same partition key. * * @return The partition key as an InternalRow. */ @Override public InternalRow partitionKey() { return partitionKey; } /** * Returns the underlying FilePartition. * * @return The FilePartition containing the files to read. */ public FilePartition getFilePartition() { return filePartition; } @Override public String[] preferredLocations() { return filePartition.preferredLocations(); } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; DeltaInputPartition that = (DeltaInputPartition) o; return Objects.equals(filePartition, that.filePartition) && Objects.equals(partitionKey, that.partitionKey); } @Override public int hashCode() { return Objects.hash(filePartition, partitionKey); } @Override public String toString() { return String.format( "DeltaInputPartition(partitionKey=%s, files=%d)", partitionKey, filePartition.files().length); } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaParquetFileFormatV2.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.read; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import org.apache.spark.sql.delta.DeltaParquetFileFormatBase; import scala.Option; /** * V2 implementation of DeltaParquetFileFormat using Kernel's Protocol and Metadata. * *

This class enables the V2 connector to reuse delta-spark-v1's DeltaParquetFileFormatBase for * reading Parquet files with Delta-specific features like column mapping. */ public class DeltaParquetFileFormatV2 extends DeltaParquetFileFormatBase { private static final long serialVersionUID = 1L; /** * Creates a DeltaParquetFileFormatV2. * * @param protocol Kernel's Protocol * @param metadata Kernel's Metadata * @param nullableRowTrackingConstantFields if true, row tracking constant fields (e.g., base row * ID, default row commit version) will be created as nullable in the schema * @param nullableRowTrackingGeneratedFields if true, row tracking generated fields will be * created as nullable in the schema * @param optimizationsEnabled whether to enable optimizations (splits, predicate pushdown) * @param tablePath table path for deletion vector support * @param isCDCRead whether this is a CDC read * @param useMetadataRowIndex V2: explicit control over _metadata.row_index usage for DV filtering */ public DeltaParquetFileFormatV2( Protocol protocol, Metadata metadata, boolean nullableRowTrackingConstantFields, boolean nullableRowTrackingGeneratedFields, boolean optimizationsEnabled, Option tablePath, boolean isCDCRead, Option useMetadataRowIndex) { super( new ProtocolMetadataAdapterV2(protocol, metadata), nullableRowTrackingConstantFields, nullableRowTrackingGeneratedFields, optimizationsEnabled, tablePath, isCDCRead, // Java's Option can't directly pass to Scala's Option[Boolean] parameter, // because Scala compiles Option[Boolean] to Option in bytecode for primitive // handling. useMetadataRowIndex.map(x -> x)); } @Override public boolean equals(Object other) { if (this == other) return true; if (!(other instanceof DeltaParquetFileFormatV2)) return false; DeltaParquetFileFormatV2 that = (DeltaParquetFileFormatV2) other; return this.columnMappingMode().equals(that.columnMappingMode()) && this.referenceSchema().equals(that.referenceSchema()) && this.optimizationsEnabled() == that.optimizationsEnabled() && this.tablePath().equals(that.tablePath()) && this.isCDCRead() == that.isCDCRead(); } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/read/IndexedFile.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.read; import static io.delta.kernel.internal.util.Preconditions.checkState; import io.delta.kernel.internal.actions.AddFile; import org.apache.spark.sql.delta.sources.AdmittableFile; /** * Java version of IndexedFile.scala that uses Kernel's action classes. * *

File: represents a data file in Delta. * *

Indexed: refers to the index in DeltaSourceOffset, assigned by the streaming engine. */ public class IndexedFile implements AdmittableFile { private final long version; private final long index; private final AddFile addFile; public IndexedFile(long version, long index, AddFile addFile) { this.version = version; this.index = index; this.addFile = addFile; } public long getVersion() { return version; } public long getIndex() { return index; } public AddFile getAddFile() { return addFile; } @Override public boolean hasFileAction() { return addFile != null; } @Override public long getFileSize() { checkState(addFile != null, "check hasFileAction() before calling getFileSize()"); return addFile.getSize(); } @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("IndexedFile{"); sb.append("version=").append(version); sb.append(", index=").append(index); sb.append(", addFile=").append(addFile); sb.append('}'); return sb.toString(); } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/read/ProtocolMetadataAdapterV2.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.read; import io.delta.kernel.internal.TableConfig; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.rowtracking.RowTracking; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.internal.util.ColumnMapping; import io.delta.spark.internal.v2.utils.RowTrackingUtils; import io.delta.spark.internal.v2.utils.SchemaUtils; import java.io.Serializable; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.delta.DeltaColumnMappingMode; import org.apache.spark.sql.delta.IdMapping$; import org.apache.spark.sql.delta.NameMapping$; import org.apache.spark.sql.delta.NoMapping$; import org.apache.spark.sql.delta.ProtocolMetadataAdapter; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import scala.jdk.javaapi.CollectionConverters; /** * Implementation of ProtocolMetadataAdapter for Delta Kernel's Protocol and Metadata. * *

This class adapts Kernel's Protocol and Metadata to the ProtocolMetadataAdapter interface, * enabling the V2 connector to reuse delta-spark-v1's DeltaParquetFileFormat for reading Parquet * files. * *

Key responsibilities: * *

    *
  • Bridge Kernel's Protocol/Metadata to delta-spark's ProtocolMetadataAdapter interface *
  • Convert column mapping modes between Kernel and delta-spark representations *
  • Provide Delta-aware feature checks (deletion vectors, row tracking, Iceberg compatibility) *
  • Convert schemas between Kernel and Spark formats *
*/ public class ProtocolMetadataAdapterV2 implements ProtocolMetadataAdapter, Serializable { private static final long serialVersionUID = 1L; private final Protocol protocol; private final Metadata metadata; public ProtocolMetadataAdapterV2(Protocol protocol, Metadata metadata) { this.protocol = protocol; this.metadata = metadata; } @Override public DeltaColumnMappingMode columnMappingMode() { ColumnMapping.ColumnMappingMode kernelMode = ColumnMapping.getColumnMappingMode(metadata.getConfiguration()); switch (kernelMode) { case NONE: return NoMapping$.MODULE$; case ID: return IdMapping$.MODULE$; case NAME: return NameMapping$.MODULE$; default: throw new UnsupportedOperationException("Unsupported column mapping mode: " + kernelMode); } } @Override public StructType getReferenceSchema() { return SchemaUtils.convertKernelSchemaToSparkSchema(metadata.getSchema()); } @Override public boolean isRowIdEnabled() { return RowTracking.isEnabled(protocol, metadata); } @Override public boolean isDeletionVectorReadable() { return protocol.supportsFeature(TableFeatures.DELETION_VECTORS_RW_FEATURE) && "parquet".equalsIgnoreCase(metadata.getFormat().getProvider()); } @Override public boolean isIcebergCompatAnyEnabled() { return TableConfig.ICEBERG_COMPAT_V2_ENABLED.fromMetadata(metadata) || TableConfig.ICEBERG_COMPAT_V3_ENABLED.fromMetadata(metadata); } @Override public boolean isIcebergCompatGeqEnabled(int version) { boolean v2Enabled = TableConfig.ICEBERG_COMPAT_V2_ENABLED.fromMetadata(metadata); boolean v3Enabled = TableConfig.ICEBERG_COMPAT_V3_ENABLED.fromMetadata(metadata); // IcebergCompatV1 is not supported in Kernel, so V2 is the minimum version for v2 connector // until kernel supports IcebergCompatV1. // For version 1 or 2, we return true if V2 or V3 is enabled. switch (version) { case 1: case 2: return v2Enabled || v3Enabled; case 3: return v3Enabled; default: return false; } } @Override public void assertTableReadable(SparkSession sparkSession) { // TODO(delta-io/delta#5649): Add type widening validation. } @Override public scala.collection.Iterable createRowTrackingMetadataFields( boolean nullableRowTrackingConstantFields, boolean nullableRowTrackingGeneratedFields) { // Use RowTrackingUtils.createMetadataStructFields which handles: // - Checking if row tracking is enabled // - Creating fields with proper Spark metadata attributes // - Handling materialized column names return CollectionConverters.asScala( RowTrackingUtils.createMetadataStructFields( protocol, metadata, nullableRowTrackingConstantFields, nullableRowTrackingGeneratedFields)) .toSeq(); } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkBatch.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.read; import io.delta.kernel.Snapshot; import io.delta.kernel.expressions.Predicate; import io.delta.spark.internal.v2.utils.PartitionUtils; import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; import org.apache.hadoop.conf.Configuration; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.connector.read.Batch; import org.apache.spark.sql.connector.read.InputPartition; import org.apache.spark.sql.connector.read.PartitionReaderFactory; import org.apache.spark.sql.execution.datasources.FilePartition; import org.apache.spark.sql.execution.datasources.FilePartition$; import org.apache.spark.sql.execution.datasources.PartitionedFile; import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.sources.Filter; import org.apache.spark.sql.types.StructType; import scala.collection.JavaConverters; public class SparkBatch implements Batch { private final Snapshot snapshot; private final StructType readDataSchema; private final StructType dataSchema; private final StructType partitionSchema; private final Predicate[] pushedToKernelFilters; private final Filter[] dataFilters; private final Configuration hadoopConf; private final SQLConf sqlConf; private final long totalBytes; private scala.collection.immutable.Map scalaOptions; private final List partitionedFiles; public SparkBatch( Snapshot snapshot, StructType dataSchema, StructType partitionSchema, StructType readDataSchema, List partitionedFiles, Predicate[] pushedToKernelFilters, Filter[] dataFilters, long totalBytes, scala.collection.immutable.Map scalaOptions, Configuration hadoopConf) { this.snapshot = Objects.requireNonNull(snapshot, "snapshot is null"); this.dataSchema = Objects.requireNonNull(dataSchema, "dataSchema is null"); this.partitionSchema = Objects.requireNonNull(partitionSchema, "partitionSchema is null"); this.readDataSchema = Objects.requireNonNull(readDataSchema, "readDataSchema is null"); this.partitionedFiles = java.util.Collections.unmodifiableList( new ArrayList<>(Objects.requireNonNull(partitionedFiles, "partitionedFiles is null"))); this.pushedToKernelFilters = pushedToKernelFilters != null ? Arrays.copyOf(pushedToKernelFilters, pushedToKernelFilters.length) : new Predicate[0]; this.dataFilters = dataFilters != null ? Arrays.copyOf(dataFilters, dataFilters.length) : new Filter[0]; this.totalBytes = totalBytes; this.scalaOptions = Objects.requireNonNull(scalaOptions, "scalaOptions is null"); this.hadoopConf = Objects.requireNonNull(hadoopConf, "hadoopConf is null"); this.sqlConf = SQLConf.get(); } @Override public InputPartition[] planInputPartitions() { SparkSession sparkSession = SparkSession.active(); long maxSplitBytes = PartitionUtils.calculateMaxSplitBytes( sparkSession, totalBytes, partitionedFiles.size(), sqlConf); // For non-partitioned tables, use simple file partitioning if (partitionSchema.fields().length == 0) { scala.collection.Seq filePartitions = FilePartition$.MODULE$.getFilePartitions( sparkSession, JavaConverters.asScalaBuffer(partitionedFiles).toSeq(), maxSplitBytes); return JavaConverters.seqAsJavaList(filePartitions).toArray(new InputPartition[0]); } // For partitioned tables, group files by partition values and wrap in DeltaInputPartition // to support HasPartitionKey for KeyGroupedPartitioning optimizations return planPartitionedInputPartitions(sparkSession, maxSplitBytes); } /** * Plans input partitions for partitioned tables by grouping files by their partition values. Each * resulting DeltaInputPartition implements HasPartitionKey, enabling Spark to leverage partition * information for optimizations like shuffle elimination. */ private InputPartition[] planPartitionedInputPartitions( SparkSession sparkSession, long maxSplitBytes) { // Note: Using InternalRow as map key relies on GenericInternalRow's value-based // equals()/hashCode(), which is what PartitionUtils.getPartitionRow() returns. Map> filesByPartition = new LinkedHashMap<>(); for (PartitionedFile file : partitionedFiles) { InternalRow partitionKey = file.partitionValues(); filesByPartition.computeIfAbsent(partitionKey, k -> new ArrayList<>()).add(file); } // Create DeltaInputPartitions for each partition group List result = new ArrayList<>(); int partitionIndex = 0; for (Map.Entry> entry : filesByPartition.entrySet()) { InternalRow partitionKey = entry.getKey(); List filesInPartition = entry.getValue(); // Split files within this partition based on maxSplitBytes scala.collection.Seq filePartitions = FilePartition$.MODULE$.getFilePartitions( sparkSession, JavaConverters.asScalaBuffer(filesInPartition).toSeq(), maxSplitBytes); // Wrap each FilePartition in a DeltaInputPartition with the partition key. // Re-index partitions with a global counter because getFilePartitions returns 0-based // indices within each partition group, but we need unique indices across all groups. for (FilePartition fp : JavaConverters.seqAsJavaList(filePartitions)) { FilePartition reindexedPartition = new FilePartition(partitionIndex++, fp.files()); result.add(new DeltaInputPartition(reindexedPartition, partitionKey)); } } return result.toArray(new InputPartition[0]); } @Override public PartitionReaderFactory createReaderFactory() { return PartitionUtils.createDeltaParquetReaderFactory( snapshot, dataSchema, partitionSchema, readDataSchema, dataFilters, scalaOptions, hadoopConf, sqlConf); } @Override public boolean equals(Object obj) { if (this == obj) return true; if (!(obj instanceof SparkBatch)) return false; SparkBatch that = (SparkBatch) obj; return Objects.equals(this.snapshot, that.snapshot) && Objects.equals(this.readDataSchema, that.readDataSchema) && Objects.equals(this.dataSchema, that.dataSchema) && Objects.equals(this.partitionSchema, that.partitionSchema) && Arrays.equals(this.pushedToKernelFilters, that.pushedToKernelFilters) && Arrays.equals(this.dataFilters, that.dataFilters) && partitionedFiles.size() == that.partitionedFiles.size(); } @Override public int hashCode() { int result = snapshot.hashCode(); result = 31 * result + readDataSchema.hashCode(); result = 31 * result + dataSchema.hashCode(); result = 31 * result + partitionSchema.hashCode(); result = 31 * result + Arrays.hashCode(pushedToKernelFilters); result = 31 * result + Arrays.hashCode(dataFilters); result = 31 * result + Integer.hashCode(partitionedFiles.size()); return result; } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.read; import static io.delta.kernel.internal.tablefeatures.TableFeatures.TYPE_WIDENING_RW_FEATURE; import static io.delta.kernel.internal.tablefeatures.TableFeatures.TYPE_WIDENING_RW_PREVIEW_FEATURE; import io.delta.kernel.CommitActions; import io.delta.kernel.CommitRange; import io.delta.kernel.Scan; import io.delta.kernel.Snapshot; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.UnsupportedTableFeatureException; import io.delta.kernel.internal.DeltaHistoryManager; import io.delta.kernel.internal.DeltaLogActionUtils.DeltaAction; import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.actions.AddFile; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.RemoveFile; import io.delta.kernel.internal.util.ColumnMapping; import io.delta.kernel.internal.util.ColumnMapping.ColumnMappingMode; import io.delta.kernel.internal.util.Preconditions; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.internal.util.VectorUtils; import io.delta.kernel.utils.CloseableIterator; import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; import io.delta.spark.internal.v2.utils.PartitionUtils; import io.delta.spark.internal.v2.utils.ScalaUtils; import io.delta.spark.internal.v2.utils.SchemaUtils; import io.delta.spark.internal.v2.utils.StreamingHelper; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.channels.ClosedByInterruptException; import java.sql.Timestamp; import java.time.ZoneId; import java.util.*; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.catalyst.expressions.Literal$; import org.apache.spark.sql.connector.read.InputPartition; import org.apache.spark.sql.connector.read.PartitionReaderFactory; import org.apache.spark.sql.connector.read.streaming.*; import org.apache.spark.sql.delta.DeltaColumnMapping; import org.apache.spark.sql.delta.DeltaErrors; import org.apache.spark.sql.delta.DeltaOptions; import org.apache.spark.sql.delta.DeltaStartingVersion; import org.apache.spark.sql.delta.DeltaTimeTravelSpec; import org.apache.spark.sql.delta.StartingVersion; import org.apache.spark.sql.delta.StartingVersionLatest$; import org.apache.spark.sql.delta.TypeWidening; import org.apache.spark.sql.delta.sources.DeltaSQLConf; import org.apache.spark.sql.delta.sources.DeltaSource; import org.apache.spark.sql.delta.sources.DeltaSourceOffset; import org.apache.spark.sql.delta.sources.DeltaSourceOffset$; import org.apache.spark.sql.delta.sources.DeltaStreamUtils; import org.apache.spark.sql.execution.datasources.FilePartition; import org.apache.spark.sql.execution.datasources.FilePartition$; import org.apache.spark.sql.execution.datasources.PartitionedFile; import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.sources.Filter; import org.apache.spark.sql.types.DataType; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import scala.Option; import scala.Some; import scala.collection.JavaConverters; import scala.collection.immutable.Seq; import scala.collection.immutable.Seq$; import scala.jdk.javaapi.CollectionConverters; import scala.util.matching.Regex; // TODO(#5318): Use DeltaErrors error framework for consistent error handling. public class SparkMicroBatchStream implements MicroBatchStream, SupportsAdmissionControl, SupportsTriggerAvailableNow { private static final Logger logger = LoggerFactory.getLogger(SparkMicroBatchStream.class); private static final Set ACTION_SET = Collections.unmodifiableSet( new HashSet<>(Arrays.asList(DeltaAction.ADD, DeltaAction.REMOVE, DeltaAction.METADATA))); private final Engine engine; private final DeltaSnapshotManager snapshotManager; private final DeltaOptions options; private final boolean skipChangeCommits; private final SnapshotImpl snapshotAtSourceInit; private final String tableId; private final StructType readSchemaAtSourceInit; private final boolean shouldValidateOffsets; private final Optional excludeRegex; private final SparkSession spark; private final String tablePath; private final StructType readDataSchema; private final StructType dataSchema; private final StructType partitionSchema; private final Filter[] dataFilters; private final Configuration hadoopConf; private final SQLConf sqlConf; private final scala.collection.immutable.Map scalaOptions; /** * Tracks whether this is the first batch for this stream (no checkpointed offset). * *

- First batch: initialOffset() -> latestOffset(Offset, ReadLimit) - Set `isFirstBatch` to * true in initialOffset() - in latestOffset(Offset, ReadLimit), use `isFirstBatch` to determine * whether to return null vs previousOffset (when no data is available) - set `isFirstBatch` to * false - Subsequent batches: latestOffset(Offset, ReadLimit) */ private boolean isFirstBatch = false; /** * Configuration options for handling schema changes behavior. Controls unsafe operations like * column mapping changes, partition column changes, nullability changes, and type widening. */ private DeltaStreamUtils.SchemaReadOptions schemaReadOptions; /** * A global flag to mark whether we have done a per-stream start check for column mapping schema * changes (rename / drop). */ private volatile boolean hasCheckedReadIncompatibleSchemaChangesOnStreamStart = false; /** * When AvailableNow is used, this offset will be the upper bound where this run of the query will * process up. We may run multiple micro batches, but the query will stop itself when it reaches * this offset. */ private Optional lastOffsetForTriggerAvailableNow = Optional.empty(); private boolean isLastOffsetForTriggerAvailableNowInitialized = false; private boolean isTriggerAvailableNow = false; // Cached starting version to ensure idempotent behavior for "latest" starting version. // getStartingVersion() must return the same value across multiple calls. private volatile Optional cachedStartingVersion = null; // Cache for the initial snapshot files to avoid re-sorting on repeated access. private static class InitialSnapshotCache { final Long version; final List files; InitialSnapshotCache(Long version, List files) { this.version = version; this.files = files; } } private final AtomicReference cachedInitialSnapshot = new AtomicReference<>(null); private final int maxInitialSnapshotFiles; public SparkMicroBatchStream( DeltaSnapshotManager snapshotManager, Snapshot snapshotAtSourceInit, Configuration hadoopConf, SparkSession spark, DeltaOptions options, String tablePath, StructType dataSchema, StructType partitionSchema, StructType readDataSchema, Filter[] dataFilters, scala.collection.immutable.Map scalaOptions) { this.snapshotManager = Objects.requireNonNull(snapshotManager, "snapshotManager is null"); this.hadoopConf = Objects.requireNonNull(hadoopConf, "hadoopConf is null"); this.spark = Objects.requireNonNull(spark, "spark is null"); this.engine = DefaultEngine.create(hadoopConf); this.options = Objects.requireNonNull(options, "options is null"); this.skipChangeCommits = this.options.skipChangeCommits(); // Normalize tablePath to ensure it ends with "/" for consistent path construction String normalizedTablePath = Objects.requireNonNull(tablePath, "tablePath is null"); this.tablePath = normalizedTablePath.endsWith("/") ? normalizedTablePath : normalizedTablePath + "/"; this.dataSchema = Objects.requireNonNull(dataSchema, "dataSchema is null"); this.partitionSchema = Objects.requireNonNull(partitionSchema, "partitionSchema is null"); this.readDataSchema = Objects.requireNonNull(readDataSchema, "readDataSchema is null"); this.dataFilters = Arrays.copyOf( Objects.requireNonNull(dataFilters, "dataFilters is null"), dataFilters.length); this.sqlConf = SQLConf.get(); this.scalaOptions = Objects.requireNonNull(scalaOptions, "scalaOptions is null"); this.snapshotAtSourceInit = (SnapshotImpl) snapshotAtSourceInit; this.tableId = this.snapshotAtSourceInit.getMetadata().getId(); // TODO(#5319): schema tracking for non-additive schema changes this.readSchemaAtSourceInit = Objects.requireNonNull( SchemaUtils.convertKernelSchemaToSparkSchema(snapshotAtSourceInit.getSchema()), "readSchemaAtSourceInit is null"); this.shouldValidateOffsets = Objects.requireNonNull( (Boolean) spark.sessionState().conf().getConf(DeltaSQLConf.STREAMING_OFFSET_VALIDATION()), "shouldValidateOffsets is null"); this.excludeRegex = ScalaUtils.toJavaOptional(options.excludeRegex()); this.maxInitialSnapshotFiles = (Integer) spark .sessionState() .conf() .getConf(DeltaSQLConf.DELTA_STREAMING_INITIAL_SNAPSHOT_MAX_FILES()); boolean isStreamingFromColumnMappingTable = ColumnMapping.getColumnMappingMode( this.snapshotAtSourceInit.getMetadata().getConfiguration()) != ColumnMappingMode.NONE; boolean isTypeWideningSupportedInProtocol = this.snapshotAtSourceInit.getProtocol().supportsFeature(TYPE_WIDENING_RW_PREVIEW_FEATURE) || this.snapshotAtSourceInit.getProtocol().supportsFeature(TYPE_WIDENING_RW_FEATURE); this.schemaReadOptions = Objects.requireNonNull( DeltaStreamUtils.SchemaReadOptions$.MODULE$.fromSparkSession( spark, isStreamingFromColumnMappingTable, isTypeWideningSupportedInProtocol), "schemaReadOptions is null"); validateSchemaCompatibilityOnStartup(dataSchema, partitionSchema, readSchemaAtSourceInit); } @Override public void prepareForTriggerAvailableNow() { logger.info("The streaming query reports to use Trigger.AvailableNow."); isTriggerAvailableNow = true; } /** * initialize the internal states for AvailableNow if this method is called first time after * prepareForTriggerAvailableNow. */ private void initForTriggerAvailableNowIfNeeded(DeltaSourceOffset startOffsetOpt) { if (isTriggerAvailableNow && !isLastOffsetForTriggerAvailableNowInitialized) { isLastOffsetForTriggerAvailableNowInitialized = true; initLastOffsetForTriggerAvailableNow(startOffsetOpt); } } private void initLastOffsetForTriggerAvailableNow(DeltaSourceOffset startOffsetOpt) { lastOffsetForTriggerAvailableNow = latestOffsetInternal(startOffsetOpt, ReadLimit.allAvailable()); lastOffsetForTriggerAvailableNow.ifPresent( lastOffset -> logger.info("lastOffset for Trigger.AvailableNow has set to " + lastOffset.json())); } //////////// // offset // //////////// /** * Returns the initial offset for a streaming query to start reading from (if there's no * checkpointed offset). */ @Override public Offset initialOffset() { Optional startingVersionOpt = getStartingVersion(); long version; boolean isInitialSnapshot; isFirstBatch = true; if (startingVersionOpt.isPresent()) { version = startingVersionOpt.get(); isInitialSnapshot = false; } else { // No starting version specified in the options, use snapshot captured // at source initialization. version = snapshotAtSourceInit.getVersion(); isInitialSnapshot = true; } return DeltaSourceOffset.apply( tableId, version, DeltaSourceOffset.BASE_INDEX(), isInitialSnapshot); } @Override public Offset latestOffset() { throw new IllegalStateException( "latestOffset() should not be called - use latestOffset(Offset, ReadLimit) instead"); } /** * Get the latest offset with rate limiting (SupportsAdmissionControl). * * @param startOffset The starting offset * @param limit The read limit for rate limiting * @return The latest offset, or null if no data is available to read. */ @Override public Offset latestOffset(Offset startOffset, ReadLimit limit) { Objects.requireNonNull(startOffset, "startOffset should not be null for MicroBatchStream"); Objects.requireNonNull(limit, "limit should not be null for MicroBatchStream"); try { DeltaSourceOffset deltaStartOffset = DeltaSourceOffset.apply(tableId, startOffset); initForTriggerAvailableNowIfNeeded(deltaStartOffset); // Return null when no data is available for this batch. DeltaSourceOffset endOffset = latestOffsetInternal(deltaStartOffset, limit).orElse(null); isFirstBatch = false; return endOffset; } catch (Exception e) { // Kernel's DefaultJsonHandler wraps ClosedByInterruptException (thrown by NIO // channels on thread interrupt) inside KernelEngineException (a RuntimeException). // Spark's StreamExecution.isInterruptionException recognizes // ClosedByInterruptException and UncheckedIOException but not // KernelEngineException. Re-wrap so Spark's isInterruptedByStop — which also // verifies state == TERMINATED — handles it as a clean stream shutdown. Optional interruptCause = findClosedByInterruptCause(e); if (interruptCause.isPresent()) { throw new UncheckedIOException(interruptCause.get()); } throw e; } } /** * Internal implementation of latestOffset using DeltaSourceOffset directly, without null checks * and state management. */ private Optional latestOffsetInternal( DeltaSourceOffset deltaStartOffset, ReadLimit limit) { Optional limits = ScalaUtils.toJavaOptional(DeltaSource.AdmissionLimits$.MODULE$.apply(options, limit)); Optional endOffset = getNextOffsetFromPreviousOffset(deltaStartOffset, limits, isFirstBatch); if (shouldValidateOffsets && endOffset.isPresent()) { DeltaSourceOffset.validateOffsets(deltaStartOffset, endOffset.get()); } return endOffset; } @Override public Offset deserializeOffset(String json) { return DeltaSourceOffset$.MODULE$.apply(tableId, json); } @Override public ReadLimit getDefaultReadLimit() { return DeltaSource.AdmissionLimits$.MODULE$.toReadLimit(options); } /** * Return the next offset when previous offset exists. Mimics * DeltaSource.getNextOffsetFromPreviousOffset. * * @param previousOffset The previous offset * @param limits Rate limits for this batch (Optional.empty() for no limits) * @param isFirstBatch Whether this is the first batch for this stream * @return The next offset, or the previous offset if no new data is available (except on the * initial batch where we return empty to match DSv1's * getStartingOffsetFromSpecificDeltaVersion behavior) */ private Optional getNextOffsetFromPreviousOffset( DeltaSourceOffset previousOffset, Optional limits, boolean isFirstBatch) { // TODO(#5319): Special handling for schema tracking. CloseableIterator changes = getFileChangesWithRateLimit( previousOffset.reservoirVersion(), previousOffset.index(), previousOffset.isInitialSnapshot(), limits); Optional lastFileChange = Utils.iteratorLast(changes); if (!lastFileChange.isPresent()) { // For the first batch, return empty to match DSv1's // getStartingOffsetFromSpecificDeltaVersion if (isFirstBatch) { return Optional.empty(); } return Optional.of(previousOffset); } // Block latestOffset() from generating an invalid offset by proactively // verifying incompatible schema changes under column mapping. See more details in the // method java doc. checkReadIncompatibleSchemaChangeOnStreamStartOnce( previousOffset.reservoirVersion(), /* batchEndVersion= */ null); IndexedFile lastFile = lastFileChange.get(); return Optional.of( DeltaSource.buildOffsetFromIndexedFile( tableId, lastFile.getVersion(), lastFile.getIndex(), previousOffset.reservoirVersion(), previousOffset.isInitialSnapshot())); } //////////// /// data /// //////////// @Override public InputPartition[] planInputPartitions(Offset start, Offset end) { DeltaSourceOffset startOffset = (DeltaSourceOffset) start; DeltaSourceOffset endOffset = (DeltaSourceOffset) end; long fromVersion = startOffset.reservoirVersion(); long fromIndex = startOffset.index(); boolean isInitialSnapshot = startOffset.isInitialSnapshot(); List partitionedFiles = new ArrayList<>(); long totalBytesToRead = 0; try (CloseableIterator fileChanges = getFileChanges(fromVersion, fromIndex, isInitialSnapshot, Optional.of(endOffset))) { while (fileChanges.hasNext()) { IndexedFile indexedFile = fileChanges.next(); if (!indexedFile.hasFileAction() || indexedFile.getAddFile() == null) { continue; } AddFile addFile = indexedFile.getAddFile(); // TODO(#5319): Apply excludeRegex to RemoveFile/AddCDCFile when CDC is supported if (excludeRegex.isPresent() && excludeRegex.get().findFirstIn(addFile.getPath()).isDefined()) { continue; } PartitionedFile partitionedFile = PartitionUtils.buildPartitionedFile( addFile, partitionSchema, tablePath, ZoneId.of(sqlConf.sessionLocalTimeZone())); totalBytesToRead += addFile.getSize(); partitionedFiles.add(partitionedFile); } } catch (IOException e) { throw new RuntimeException( String.format( "Failed to get file changes for table %s from version %d index %d to offset %s", tablePath, fromVersion, fromIndex, endOffset), e); } catch (RuntimeException e) { // Same interrupt handling as latestOffset(): Kernel wraps ClosedByInterruptException // in KernelEngineException (a RuntimeException). Re-wrap as UncheckedIOException so // Spark's isInterruptedByStop recognizes it as a clean shutdown. Optional interruptCause = findClosedByInterruptCause(e); if (interruptCause.isPresent()) { throw new UncheckedIOException(interruptCause.get()); } throw e; } long maxSplitBytes = PartitionUtils.calculateMaxSplitBytes( spark, totalBytesToRead, partitionedFiles.size(), sqlConf); // Partitions files into Spark FilePartitions. Seq filePartitions = FilePartition$.MODULE$.getFilePartitions( spark, JavaConverters.asScalaBuffer(partitionedFiles).toSeq(), maxSplitBytes); return JavaConverters.seqAsJavaList(filePartitions).toArray(new InputPartition[0]); } @Override public PartitionReaderFactory createReaderFactory() { return PartitionUtils.createDeltaParquetReaderFactory( snapshotAtSourceInit, dataSchema, partitionSchema, readDataSchema, dataFilters, scalaOptions, hadoopConf, sqlConf); } /////////////// // lifecycle // /////////////// @Override public void commit(Offset end) { // TODO(#5319): update metadata tracking log. } @Override public void stop() { cachedInitialSnapshot.set(null); } /** * If the given exception wraps a {@link ClosedByInterruptException} as its direct cause, returns * it. This occurs when Spark interrupts the micro-batch thread during stream shutdown and the * thread is blocked inside Kernel's {@code DefaultJsonHandler} reading delta log files via NIO * channels. */ static Optional findClosedByInterruptCause(Throwable t) { Throwable cause = t.getCause(); if (cause instanceof ClosedByInterruptException) { return Optional.of((ClosedByInterruptException) cause); } return Optional.empty(); } /////////////////////// // getStartingVersion // /////////////////////// /** * Extracts whether users provided the option to time travel a relation. If a query restarts from * a checkpoint and the checkpoint has recorded the offset, this method should never be called. * *

Returns Optional.empty() if no starting version is provided. * *

This is the DSv2 Kernel-based implementation of DeltaSource.getStartingVersion. */ synchronized Optional getStartingVersion() { if (cachedStartingVersion != null) { return cachedStartingVersion; } // Note: returning a version beyond latest snapshot version won't be a problem as callers // of this function won't use the version to retrieve snapshot(refer to // [[getStartingOffset]]). // TODO(#5319): fetch spark config if CDF is supported. boolean allowOutOfRange = false; if (options.startingVersion().isDefined()) { DeltaStartingVersion startingVersion = options.startingVersion().get(); if (startingVersion instanceof StartingVersionLatest$) { Snapshot latestSnapshot = snapshotManager.loadLatestSnapshot(); // "latest": start reading from the next commit cachedStartingVersion = Optional.of(latestSnapshot.getVersion() + 1); return cachedStartingVersion; } else if (startingVersion instanceof StartingVersion) { long version = ((StartingVersion) startingVersion).version(); if (!validateProtocolAt(spark, snapshotManager, engine, version)) { // When starting from a given version, we don't require that the snapshot of this // version can be reconstructed, even though the input table is technically in an // inconsistent state. If the snapshot cannot be reconstructed, then the protocol // check is skipped, so this is technically not safe, but we keep it this way for // historical reasons. snapshotManager.checkVersionExists( version, /* mustBeRecreatable= */ false, /* allowOutOfRange= */ false); } cachedStartingVersion = Optional.of(version); return cachedStartingVersion; } } else if (options.startingTimestamp().isDefined()) { // Set enforceRetention to true to align with V1 connector Timestamp timestamp = new DeltaTimeTravelSpec( /* timestamp= */ options.startingTimestamp().map(Literal$.MODULE$::apply), /* version= */ Option.empty(), /* creationSource= */ Some.apply("sparkMicroBatchStream"), /* enforceRetention= */ true) .getTimestamp(spark.sessionState().conf()); long startingVersion = getStartingVersionFromTimestamp( spark, snapshotManager, engine, timestamp, allowOutOfRange); cachedStartingVersion = Optional.of(startingVersion); return cachedStartingVersion; } cachedStartingVersion = Optional.empty(); return cachedStartingVersion; } /** * Returns the earliest commit version whose timestamp is >= the provided timestamp. * *

This method fetches the commit at the given timestamp via * [[DeltaSnapshotManager.getActiveCommitAtTime]], computes the starting version using * [[DeltaStreamUtils.getStartingVersionFromCommitAtTimestamp]], and validates the protocol at the * returned version. */ private static long getStartingVersionFromTimestamp( SparkSession spark, DeltaSnapshotManager snapshotManager, Engine engine, Timestamp timestamp, boolean canExceedLatest) { // TODO(#5999): optimize duplicate loadLatestSnapshot calls DeltaHistoryManager.Commit commit = snapshotManager.getActiveCommitAtTime( timestamp.getTime(), /* canReturnLastCommit= */ true, /* mustBeRecreatable= */ false, /* canReturnEarliestCommit= */ true); long latestVersion = snapshotManager.loadLatestSnapshot().getVersion(); long startingVersion = DeltaStreamUtils.getStartingVersionFromCommitAtTimestamp( /* timeZone= */ spark.sessionState().conf().sessionLocalTimeZone(), /* commitTimestamp= */ commit.getTimestamp(), /* commitVersion= */ commit.getVersion(), /* latestVersion= */ latestVersion, /* timestamp= */ timestamp, /* canExceedLatest= */ canExceedLatest); if (startingVersion <= latestVersion) { validateProtocolAt(spark, snapshotManager, engine, startingVersion); } return startingVersion; } /** * Validate the protocol at a given version. If the snapshot reconstruction fails for any other * reason than unsupported feature exception, we suppress it. This allows fallback to previous * behavior where the starting version/timestamp was not mandatory to point to reconstructable * snapshot. * *

This is the DSv2 Kernel-based implementation of DeltaSource.validateProtocolAt. * *

Returns true when the validation was performed and succeeded. */ private static boolean validateProtocolAt( SparkSession spark, DeltaSnapshotManager snapshotManager, Engine engine, long version) { boolean alwaysValidateProtocol = (Boolean) spark .sessionState() .conf() .getConf(DeltaSQLConf.FAST_DROP_FEATURE_STREAMING_ALWAYS_VALIDATE_PROTOCOL()); if (!alwaysValidateProtocol) { return false; } try { // Attempt to construct a snapshot at the startingVersion to validate the protocol // If snapshot reconstruction fails, fall back to old behavior where the only // requirement was for the commit to exist snapshotManager.loadSnapshotAt(version); return true; } catch (UnsupportedTableFeatureException e) { // Re-throw fatal unsupported table feature exceptions throw e; } catch (Exception e) { // Suppress non-fatal exceptions logger.warn("Protocol validation failed at version {} with: {}", version, e.getMessage()); return false; } } //////////////////// // getFileChanges // //////////////////// /** * Get file changes with rate limiting applied. Mimics DeltaSource.getFileChangesWithRateLimit. * * @param fromVersion The starting version (exclusive with fromIndex) * @param fromIndex The starting index within fromVersion (exclusive) * @param isInitialSnapshot Whether this is the initial snapshot * @param limits Rate limits to apply (Optional.empty() for no limits) * @return An iterator of IndexedFile with rate limiting applied */ CloseableIterator getFileChangesWithRateLimit( long fromVersion, long fromIndex, boolean isInitialSnapshot, Optional limits) { // TODO(#5319): getFileChangesForCDC if CDC is enabled. CloseableIterator changes = getFileChanges( fromVersion, fromIndex, isInitialSnapshot, /* endOffset= */ Optional.empty()); // Take each change until we've seen the configured number of addFiles. Some changes don't // represent file additions; we retain them for offset tracking, but they don't count toward // the maxFilesPerTrigger conf. if (limits.isPresent()) { DeltaSource.AdmissionLimits admissionLimits = limits.get(); changes = changes.takeWhile(admissionLimits::admit); } // TODO(#5318): Stop at schema change barriers return changes; } /** * Get file changes between fromVersion/fromIndex and endOffset. This is the Kernel-based * implementation of DeltaSource.getFileChanges. * *

Package-private for testing. * * @param fromVersion The starting version (exclusive with fromIndex) * @param fromIndex The starting index within fromVersion (exclusive) * @param isInitialSnapshot Whether this is the initial snapshot * @param endOffset The end offset (inclusive), or empty to read all available commits * @return An iterator of IndexedFile representing the file changes */ CloseableIterator getFileChanges( long fromVersion, long fromIndex, boolean isInitialSnapshot, Optional endOffset) { CloseableIterator result; if (isInitialSnapshot) { // Lazily combine snapshot files with delta logs starting from fromVersion + 1. // filterDeltaLogs handles the case when no commits exist after fromVersion. CloseableIterator snapshotFiles = getSnapshotFiles(fromVersion); CloseableIterator deltaChanges = filterDeltaLogs(fromVersion + 1, endOffset); result = snapshotFiles.combine(deltaChanges); } else { result = filterDeltaLogs(fromVersion, endOffset); } // Check start boundary (exclusive) result = result.filter( file -> file.getVersion() > fromVersion || (file.getVersion() == fromVersion && file.getIndex() > fromIndex)); // If endOffset is provided, we are getting a batch on a constructed range so we should use // the endOffset as the limit. // Otherwise, we are looking for a new offset, so we try to use the latestOffset we found for // Trigger.availableNow() as limit. We know endOffset <= lastOffsetForTriggerAvailableNow. Optional lastOffsetForThisScan = endOffset.or(() -> lastOffsetForTriggerAvailableNow); // Check end boundary (inclusive) if (lastOffsetForThisScan.isPresent()) { DeltaSourceOffset bound = lastOffsetForThisScan.get(); result = result.takeWhile( file -> file.getVersion() < bound.reservoirVersion() || (file.getVersion() == bound.reservoirVersion() && file.getIndex() <= bound.index())); } return result; } private CloseableIterator filterDeltaLogs( long startVersion, Optional endOffset) { Optional endVersionOpt = endOffset.isPresent() ? Optional.of(endOffset.get().reservoirVersion()) : Optional.empty(); if (endVersionOpt.isPresent()) { // Cap endVersion to the latest available version. The Kernel's getTableChanges requires // endVersion to be an actual existing version or empty. long latestVersion = snapshotAtSourceInit.getVersion(); if (endVersionOpt.get() > latestVersion) { // This could happen because: // 1. data could be added after snapshotAtSourceInit was captured. // 2. buildOffsetFromIndexedFile bumps the version up by one when we hit the END_INDEX. // TODO(#5318): consider caching the latest version to avoid loading a new snapshot. // TODO(#5318): kernel should ideally relax this constraint. endVersionOpt = Optional.of(snapshotManager.loadLatestSnapshot().getVersion()); } // After capping, check if startVersion is beyond the endVersion. // This can happen when all files in the batch come from the initial snapshot // (e.g., offset was bumped to next version due to END_INDEX, but no new commits exist). if (startVersion > endVersionOpt.get()) { return Utils.toCloseableIterator(Collections.emptyIterator()); } } else { // When endOffset is empty (offset discovery), check if startVersion exceeds the current // latest version. We must load the current latest (not snapshotAtSourceInit) because new // commits may have arrived since stream initialization. long currentLatestVersion = snapshotManager.loadLatestSnapshot().getVersion(); if (startVersion > currentLatestVersion) { return Utils.toCloseableIterator(Collections.emptyIterator()); } } CommitRange commitRange; try { commitRange = snapshotManager.getTableChanges(engine, startVersion, endVersionOpt); } catch (io.delta.kernel.exceptions.CommitRangeNotFoundException e) { // If the requested version range doesn't exist (e.g., we're asking for version 6 when // the table only has versions 0-5). return Utils.toCloseableIterator(Collections.emptyIterator()); } // Use getCommitActionsFromRangeUnsafe instead of CommitRange.getCommitActions() because: // 1. CommitRange.getCommitActions() requires a snapshot at exactly the startVersion, but when // startingVersion option is used, we may not be able to recreate that exact snapshot // (e.g., if log files have been cleaned up after checkpointing). // 2. This matches DSv1 behavior which uses snapshotAtSourceInit's P&M to interpret all // AddFile actions and performs per-commit protocol validation. CloseableIterator commitsIterator = StreamingHelper.getCommitActionsFromRangeUnsafe( engine, (io.delta.kernel.internal.commitrange.CommitRangeImpl) commitRange, snapshotAtSourceInit.getPath(), ACTION_SET); return commitsIterator.flatMap( commit -> processCommitToIndexedFiles(commit, startVersion, endOffset)); } /** * Processes a single commit and returns an iterator of IndexedFiles wrapped with BEGIN/END * sentinels. */ private CloseableIterator processCommitToIndexedFiles( CommitActions commit, long startVersion, Optional endOffsetOpt) { try { long version = commit.getVersion(); // First pass: Validate the commit and decide whether to skip it. // // We must validate the ENTIRE commit before emitting ANY files. This is a correctness // requirement: commits could contain both AddFiles and RemoveFiles. // If we emitted AddFiles before discovering a RemoveFile(dataChange=true) later in the // commit, downstream would produce incorrect results. // // TODO(#5318): consider caching the commit actions to avoid reading the same commit twice. // TODO(#5319): don't verify metadata action when schema tracking is enabled boolean shouldSkipCommit = validateCommitAndDecideSkipping( commit, version, startVersion, snapshotAtSourceInit.getPath(), endOffsetOpt, /* verifyMetadataAction= */ true); // Second pass: Build a lazy iterator of IndexedFiles. // // BEGIN (BASE_INDEX) + actual file actions + END (END_INDEX) // // These sentinel IndexedFiles have null file actions and are used for proper offset // tracking: // - BASE_INDEX: marks "before any files in this version", allowing the offset to // reference the start of a version. // - END_INDEX: marks end of version, triggers version advancement in // buildOffsetFromIndexedFile to skip re-reading completed versions. // // See DeltaSource.addBeginAndEndIndexOffsetsForVersion for the Scala equivalent. CloseableIterator fileActions = shouldSkipCommit ? Utils.toCloseableIterator(Collections.emptyIterator()) : getFilesFromCommit(commit, version); CloseableIterator inner = Utils.singletonCloseableIterator( new IndexedFile(version, DeltaSourceOffset.BASE_INDEX(), /* addFile= */ null)) .combine(fileActions) .combine( Utils.singletonCloseableIterator( new IndexedFile( version, DeltaSourceOffset.END_INDEX(), /* addFile= */ null))); // Wrap the iterator so that closing it also closes the CommitActions, releasing its // internal ActionsIterator and any associated file handles / parsed data. return wrapIteratorWithCommitClose(inner, commit); } catch (Exception e) { // commit is not a CloseableIterator, we need to close it manually. Utils.closeCloseables(commit); throw (e instanceof RuntimeException) ? (RuntimeException) e : new RuntimeException(e); } } /** * Wraps an iterator so that closing it also closes the given {@link CommitActions}, releasing its * internal resources. This is necessary because {@link CommitActions} is an {@link AutoCloseable} * but not a {@link CloseableIterator}, so closing the iterator chain alone does not close the * {@link CommitActions} that produced it. Package-private for testing. */ static CloseableIterator wrapIteratorWithCommitClose( CloseableIterator inner, CommitActions commit) { return new CloseableIterator() { @Override public boolean hasNext() { return inner.hasNext(); } @Override public IndexedFile next() { return inner.next(); } @Override public void close() throws IOException { Utils.closeCloseables(inner, commit); } }; } private CloseableIterator getFilesFromCommit(CommitActions commit, long version) { // Assign each IndexedFile a unique index within the commit. We use a mutable array // because variables captured by a lambda must be effectively final (never reassigned). long[] fileIndex = {0}; return commit .getActions() .flatMap( batch -> { // Processing each batch eagerly because they are already loaded into memory. List files = new ArrayList<>(); fileIndex[0] = addIndexedFilesAndReturnNextIndex(batch, version, fileIndex[0], files); return Utils.toCloseableIterator(files.iterator()); }); } /** * Validates a commit, fail the stream if it's invalid and decides whether to skip it. Mimics * DeltaSource.validateCommitAndDecideSkipping in Scala. * * @param commit the CommitActions representing a single commit * @param version the commit version * @param batchStartVersion Starting version of the batch being processed * @param tablePath the path to the Delta table * @param endOffsetOpt optional end offset for boundary checking * @param verifyMetadataAction Whether to verify metadata action compatibility * @return true if the commit should be skipped (no AddFiles emitted), false otherwise * @throws RuntimeException if the commit is invalid. */ private boolean validateCommitAndDecideSkipping( CommitActions commit, long version, long batchStartVersion, String tablePath, Optional endOffsetOpt, boolean verifyMetadataAction) { // If endOffset is at the beginning of this version, exit early. if (endOffsetOpt.isPresent()) { DeltaSourceOffset endOffset = endOffsetOpt.get(); if (endOffset.reservoirVersion() == version && endOffset.index() == DeltaSourceOffset.BASE_INDEX()) { return false; } } // TODO(#5319): Implement ignoreChanges & ignoreFileDeletion (deprecated) // A check on the source table that disallows changes on the source data. boolean shouldAllowChanges = skipChangeCommits; // A check on the source table that disallows commits that only include deletes to the data. boolean shouldAllowDeletes = shouldAllowChanges || options.ignoreDeletes(); boolean hasFileAdd = false; boolean shouldSkipCommit = false; Metadata metadataAction = null; String removeFileActionPath = null; try (CloseableIterator actionsIter = commit.getActions()) { while (actionsIter.hasNext()) { ColumnarBatch batch = actionsIter.next(); int numRows = batch.getSize(); for (int rowId = 0; rowId < numRows; rowId++) { // Track AddFile(dataChange=true) Optional addOpt = StreamingHelper.getAddFileWithDataChange(batch, rowId); if (addOpt.isPresent()) { hasFileAdd = true; } // Track RemoveFile(dataChange=true) Optional removeOpt = StreamingHelper.getDataChangeRemove(batch, rowId); if (removeOpt.isPresent()) { // skip change commits include delete-only commits shouldSkipCommit = skipChangeCommits; if (removeFileActionPath == null) { removeFileActionPath = removeOpt.get().getPath(); } } // Track Metadata for read-incompatible schema changes. Optional metadataOpt = StreamingHelper.getMetadata(batch, rowId); if (metadataOpt.isPresent()) { Metadata metadata = metadataOpt.get(); Preconditions.checkArgument( metadataAction == null, "Should not encounter two metadata actions in the same commit of version %d", version); metadataAction = metadata; Long batchEndVersion = endOffsetOpt.map(DeltaSourceOffset::reservoirVersion).orElse(null); if (verifyMetadataAction) { checkReadIncompatibleSchemaChanges( metadata, version, batchStartVersion, batchEndVersion, /* validatedDuringStreamStart */ false); } } } } } catch (IOException e) { throw new RuntimeException("Failed to process commit at version " + version, e); } if (removeFileActionPath != null) { if (hasFileAdd && !shouldAllowChanges) { // Commit contains data changes (adds + removes) and changes are disallowed. // TODO(#5319): log CommitInfo action's operation instead of path throw (RuntimeException) DeltaErrors.deltaSourceIgnoreChangesError(version, removeFileActionPath, tablePath); } else if (!hasFileAdd && !shouldAllowDeletes) { // Commit contains only removes (deletes) and deletes are disallowed. throw (RuntimeException) DeltaErrors.deltaSourceIgnoreDeleteError(version, removeFileActionPath, tablePath); } } return shouldSkipCommit; } /** * Narrow waist to verify a metadata action for read-incompatible schema changes, specifically: 1. * Any column mapping related schema changes (rename / drop) columns 2. Standard * read-compatibility changes including: a) No missing columns b) No data type changes c) No * read-incompatible nullability changes If the check fails, we throw an exception to exit the * stream. If lazy log initialization is required, we also run a one time scan to safely * initialize the metadata tracking log upon any non-additive schema change failures. * * @param metadata Metadata that contains a potential schema change * @param version Version for the metadata action * @param batchStartVersion Starting version of the batch being processed * @param batchEndVersion Ending version of the batch being processed, or null for the latest * @param validatedDuringStreamStart Whether this check is being done during stream start. */ private void checkReadIncompatibleSchemaChanges( Metadata metadata, long version, long batchStartVersion, Long batchEndVersion, boolean validatedDuringStreamStart) { logger.info( "checking read incompatibility with schema at version {}, inside batch[{}, {}].", version, batchStartVersion, batchEndVersion != null ? batchEndVersion : "latest"); Metadata newMetadata, oldMetadata; if (version < snapshotAtSourceInit.getVersion()) { newMetadata = snapshotAtSourceInit.getMetadata(); oldMetadata = metadata; } else { newMetadata = metadata; oldMetadata = snapshotAtSourceInit.getMetadata(); } // Table ID has changed during streaming if (!Objects.equals(newMetadata.getId(), oldMetadata.getId())) { throw (RuntimeException) DeltaErrors.differentDeltaTableReadByStreamingSource( newMetadata.getId(), oldMetadata.getId()); } // Partition column change will be ignored if user enable the unsafe flag Seq newPartitionColumns, oldPartitionColumns; if (schemaReadOptions.allowUnsafeStreamingReadOnPartitionColumnChanges()) { newPartitionColumns = (Seq) Seq$.MODULE$.empty(); oldPartitionColumns = (Seq) Seq$.MODULE$.empty(); } else { newPartitionColumns = CollectionConverters.asScala( VectorUtils.toJavaList(newMetadata.getPartitionColumns()).stream() .map(Object::toString) .collect(Collectors.toList())) .toSeq(); oldPartitionColumns = CollectionConverters.asScala( VectorUtils.toJavaList(oldMetadata.getPartitionColumns()).stream() .map(Object::toString) .collect(Collectors.toList())) .toSeq(); } checkNonAdditiveSchemaChanges( oldMetadata, newMetadata, oldPartitionColumns, newPartitionColumns, validatedDuringStreamStart); // Other standard read compatibility changes if (!validatedDuringStreamStart || !schemaReadOptions .forceEnableStreamingReadOnReadIncompatibleSchemaChangesDuringStreamStart()) { StructType schemaChange = SchemaUtils.convertKernelSchemaToSparkSchema(metadata.getSchema()); // There is a schema change. All files after this commit will use `schemaChange`. Hence, we // check whether we can use `schema` (the fixed source schema we use in the same run of the // query) to read these new files safely. boolean backfilling = version < snapshotAtSourceInit.getVersion(); DeltaStreamUtils.SchemaCompatibilityResult checkResult = DeltaStreamUtils.checkSchemaChangesWhenNoSchemaTracking( schemaChange, readSchemaAtSourceInit, newPartitionColumns, oldPartitionColumns, backfilling, schemaReadOptions); if (!DeltaStreamUtils.SchemaCompatibilityResult$.MODULE$.isCompatible(checkResult)) { boolean isRetryable = DeltaStreamUtils.SchemaCompatibilityResult$.MODULE$.isRetryableIncompatible( checkResult); throw (RuntimeException) DeltaErrors.schemaChangedException( readSchemaAtSourceInit, schemaChange, isRetryable, Some.apply(version), options.containsStartingVersionOrTimestamp()); } } } // TODO(#5319): schema tracking for non-additive schema changes // TODO(#5319): Extract the entire non-additive schema check into a static utility and share it // with v1 by refactoring DeltaColumnMapping.hasNoColumnMappingSchemaChanges so it can be reused // by both v1 and v2. // Non-additive schema changes include rename column, drop column and change column type private void checkNonAdditiveSchemaChanges( Metadata oldMetadata, Metadata newMetadata, Seq oldPartitionColumns, Seq newPartitionColumns, boolean validatedDuringStreamStart) { StructType sparkNewSchema = SchemaUtils.convertKernelSchemaToSparkSchema(newMetadata.getSchema()); StructType sparkOldSchema = SchemaUtils.convertKernelSchemaToSparkSchema(oldMetadata.getSchema()); boolean shouldTrackSchema; if (schemaReadOptions.typeWideningEnabled() && schemaReadOptions.enableSchemaTrackingForTypeWidening() && TypeWidening.containsWideningTypeChanges(sparkOldSchema, sparkNewSchema)) { // If schema tracking is enabled for type widening, we will detect widening type changes and // block the stream until the user sets `allowSourceColumnTypeChange` - similar to handling // DROP/RENAME for column mapping. shouldTrackSchema = true; } else if (schemaReadOptions.allowUnsafeStreamingReadOnColumnMappingSchemaChanges()) { shouldTrackSchema = false; } else { ColumnMappingMode NONE = ColumnMappingMode.NONE; ColumnMappingMode oldMode = ColumnMapping.getColumnMappingMode(oldMetadata.getConfiguration()); ColumnMappingMode newMode = ColumnMapping.getColumnMappingMode(newMetadata.getConfiguration()); if (oldMode != NONE && newMode != NONE) { Preconditions.checkArgument(oldMode == newMode, "changing mode is not supported"); shouldTrackSchema = DeltaColumnMapping.hasColMappingOrPartitionSchemaChange( sparkNewSchema, sparkOldSchema, newPartitionColumns, oldPartitionColumns, /* isBothColumnMappingEnabled */ true); } else if (oldMode == NONE && newMode != NONE) { // TODO(#5319): We should disallow user to upgrade column mapping mode for now since we // don't support schema tracking shouldTrackSchema = true; } else { // Prohibit reading across a downgrade. shouldTrackSchema = oldMode != NONE && newMode == NONE; } } if (shouldTrackSchema) { throw (RuntimeException) DeltaErrors.blockStreamingReadsWithIncompatibleNonAdditiveSchemaChanges( spark, sparkOldSchema, sparkNewSchema, !validatedDuringStreamStart, /* isV2DataSource= */ true); } } /** * Check read-incompatible schema changes during stream (re)start so we could fail fast. * *

This is called ONCE during the first latestOffset call to catch edge cases that normal * per-commit validation (checkReadIncompatibleSchemaChanges) misses. * *

Why needed? Normal validation only checks commits with metadata actions. If a stream * starts at version 1 with the latest version is version 3 and there is a schema change at * version 2, validateCommits only validates version 2's schema against version 3 (SAME, passes). * But version 1 files may have an incompatible older schema that was never checked since version * 1 has no metadata action. * *

This method explicitly loads and validates the snapshot in the scan range, regardless of * whether it has a metadata action, catching such incompatibilities before planInputPartitions * *

Skipped if schema tracking log is already initialized. * * @param batchStartVersion Start version we want to verify read compatibility against * @param batchEndVersion Optionally, if we are checking against an existing constructed batch * during streaming initialization, we would also like to verify all schema changes in between * as well before we can lazily initialize the schema log if needed. */ private void checkReadIncompatibleSchemaChangeOnStreamStartOnce( long batchStartVersion, Long batchEndVersion) { // TODO(#5319): skip if enable schema tracking log if (hasCheckedReadIncompatibleSchemaChangesOnStreamStart) return; SnapshotImpl startVersionSnapshot = null; Exception err = null; try { startVersionSnapshot = (SnapshotImpl) snapshotManager.loadSnapshotAt(batchStartVersion); } catch (Exception e) { err = e; } // Cannot perfectly verify column mapping schema changes if we cannot compute a start snapshot. if (!schemaReadOptions.allowUnsafeStreamingReadOnColumnMappingSchemaChanges() && schemaReadOptions.isStreamingFromColumnMappingTable() && (err != null)) { throw (RuntimeException) DeltaErrors.failedToGetSnapshotDuringColumnMappingStreamingReadCheck(err); } // Perform schema check if we need to, considering all escape flags. if (!schemaReadOptions.allowUnsafeStreamingReadOnColumnMappingSchemaChanges() || schemaReadOptions.typeWideningEnabled() || !schemaReadOptions .forceEnableStreamingReadOnReadIncompatibleSchemaChangesDuringStreamStart()) { if (startVersionSnapshot != null) { checkReadIncompatibleSchemaChanges( startVersionSnapshot.getMetadata(), startVersionSnapshot.getVersion(), batchStartVersion, batchEndVersion, /* validatedDuringStreamStart= */ true); // If end version is defined (i.e. we have a pending batch), let's also eagerly check all // intermediate schema changes against the stream read schema to capture corners cases such // as rename and rename back. if (batchEndVersion != null) { for (Map.Entry entry : StreamingHelper.collectMetadataActionsFromRangeUnsafe( batchStartVersion, Optional.of(batchEndVersion), snapshotManager, engine, snapshotAtSourceInit.getPath()) .entrySet()) { long version = entry.getKey(); Metadata metadata = entry.getValue(); checkReadIncompatibleSchemaChanges( metadata, version, batchStartVersion, batchEndVersion, /* validatedDuringStreamStart= */ true); } } } } // Mark as checked hasCheckedReadIncompatibleSchemaChangesOnStreamStart = true; } /** * Validates that the analysis-time schema matches the latest snapshot schema. This catches cases * where the table schema changed after the streaming query was analyzed, and the user restarts * the stream with a stale DataFrame. * * @param dataSchema data columns from analysis time (excludes partition columns) * @param partitionSchema partition columns from analysis time * @param snapshotSchema full table schema from the latest snapshot at stream start */ private void validateSchemaCompatibilityOnStartup( StructType dataSchema, StructType partitionSchema, StructType snapshotSchema) { // Reconstruct the full analysis-time table schema from dataSchema + partitionSchema. // StructType is immutable — add() returns a new instance without modifying the original. StructType querySchema = dataSchema; for (StructField field : partitionSchema.fields()) { querySchema = querySchema.add(field); } // Compare the structural schema of the analysis-time schema and snapshot schema. if (!DataType.equalsStructurally(querySchema, snapshotSchema, /* ignoreNullability */ false)) { throw DeltaErrors.streamingSchemaMismatchOnRestart(querySchema, snapshotSchema); } } /** * Extracts IndexedFiles from a batch of actions for a given version and adds them to the output * list. Assigns an index to each IndexedFile. * * @return The next available index after processing this batch */ private long addIndexedFilesAndReturnNextIndex( ColumnarBatch batch, long version, long startIndex, List output) { long index = startIndex; for (int rowId = 0; rowId < batch.getSize(); rowId++) { // Only include AddFiles with dataChange=true. Skip changes that optimize or reorganize // data without changing the logical content. Optional addOpt = StreamingHelper.getAddFileWithDataChange(batch, rowId); if (addOpt.isPresent()) { AddFile addFile = addOpt.get(); output.add(new IndexedFile(version, index++, addFile)); } } return index; } /** * Get all files from a snapshot at the specified version, sorted by modificationTime and path, * with indices assigned sequentially, and wrapped with BEGIN/END sentinels. * *

Mimics DeltaSourceSnapshot in DSv1. * * @param version The snapshot version to read * @return An iterator of IndexedFile representing the snapshot files */ private CloseableIterator getSnapshotFiles(long version) { InitialSnapshotCache cache = cachedInitialSnapshot.get(); if (cache != null && cache.version != null && cache.version == version) { return Utils.toCloseableIterator(cache.files.iterator()); } List indexedFiles = loadAndValidateSnapshot(version); cachedInitialSnapshot.set( new InitialSnapshotCache(version, Collections.unmodifiableList(indexedFiles))); return Utils.toCloseableIterator(indexedFiles.iterator()); } /** Loads snapshot files at the specified version. */ private List loadAndValidateSnapshot(long version) { Snapshot snapshot = snapshotManager.loadSnapshotAt(version); Scan scan = snapshot.getScanBuilder().build(); List addFiles = new ArrayList<>(); try (CloseableIterator filesIter = scan.getScanFiles(engine)) { while (filesIter.hasNext()) { FilteredColumnarBatch filteredBatch = filesIter.next(); // Get all AddFiles from the batch. Include both dataChange=true and dataChange=false // (checkpoint files) files. StreamingHelper.getAddFile respects the selection vector // to filter out duplicate files (e.g., stats re-collection re-adds files with updated // stats). for (int rowId = 0; rowId < filteredBatch.getData().getSize(); rowId++) { Optional addOpt = StreamingHelper.getAddFile(filteredBatch, rowId); if (addOpt.isPresent()) { addFiles.add(addOpt.get()); // Basic memory protection: each IndexedFile is ~1-2KB (path, stats, partition values, // etc.). // This limit aims to prevent OOM for large tables. // TODO(#5318): support large tables and remove this limit. if (addFiles.size() > maxInitialSnapshotFiles) { throw (RuntimeException) DeltaErrors.initialSnapshotTooLargeForStreaming( version, addFiles.size(), maxInitialSnapshotFiles, tablePath); } } } } } catch (IOException e) { throw new RuntimeException( String.format("Failed to read snapshot files at version %d", version), e); } // TODO(#5318): For large snapshots, consider external sorting. // CRITICAL: Sort by modificationTime, then path for deterministic ordering addFiles.sort( Comparator.comparing(AddFile::getModificationTime).thenComparing(AddFile::getPath)); // Build IndexedFile list with sentinels List indexedFiles = new ArrayList<>(); // Add BEGIN sentinel indexedFiles.add(new IndexedFile(version, DeltaSourceOffset.BASE_INDEX(), null)); // Add data files with sequential indices starting from 0 for (int i = 0; i < addFiles.size(); i++) { indexedFiles.add(new IndexedFile(version, i, addFiles.get(i))); } // Add END sentinel indexedFiles.add(new IndexedFile(version, DeltaSourceOffset.END_INDEX(), null)); return indexedFiles; } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkPartitionReader.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.read; import java.io.Closeable; import java.io.IOException; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.connector.read.PartitionReader; import org.apache.spark.sql.execution.datasources.FilePartition; import org.apache.spark.sql.execution.datasources.PartitionedFile; import scala.Function1; import scala.collection.Iterator; public class SparkPartitionReader implements PartitionReader { // Function that produces an Iterator for a given file. private final Function1> readFunc; private final FilePartition partition; // Index of the next file to read within the partition. private int currentFileIndex = 0; // Current iterator for the file being read. private Iterator currentIterator = null; public SparkPartitionReader( Function1> readFunc, FilePartition partition) { this.readFunc = java.util.Objects.requireNonNull(readFunc, "readFunc"); this.partition = java.util.Objects.requireNonNull(partition, "partition"); } @Override public boolean next() throws IOException { // Advance to the next available record, opening readers as needed and closing exhausted ones. while (true) { if (currentIterator != null && currentIterator.hasNext()) { return true; } closeCurrentIterator(); if (currentFileIndex >= partition.files().length) { return false; } final PartitionedFile file = partition.files()[currentFileIndex++]; @SuppressWarnings("unchecked") Iterator it = (Iterator) readFunc.apply(file); currentIterator = it; } } @Override public T get() { if (currentIterator == null) { throw new IllegalStateException("No current record. Call next() before get()."); } return currentIterator.next(); } @Override public void close() throws IOException { closeCurrentIterator(); } private void closeCurrentIterator() throws IOException { if (currentIterator != null) { if (currentIterator instanceof Closeable) { ((Closeable) currentIterator).close(); } currentIterator = null; } } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkReaderFactory.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.read; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.connector.read.InputPartition; import org.apache.spark.sql.connector.read.PartitionReader; import org.apache.spark.sql.connector.read.PartitionReaderFactory; import org.apache.spark.sql.execution.datasources.FilePartition; import org.apache.spark.sql.execution.datasources.PartitionedFile; import org.apache.spark.sql.vectorized.ColumnarBatch; import scala.Function1; import scala.collection.Iterator; public class SparkReaderFactory implements PartitionReaderFactory { private Function1> readFunc; private boolean supportsColumnar; public SparkReaderFactory( Function1> readFunc, boolean supportsColumnar) { this.readFunc = readFunc; this.supportsColumnar = supportsColumnar; } @Override public PartitionReader createColumnarReader(InputPartition partition) { return new SparkPartitionReader(readFunc, extractFilePartition(partition)); } @Override public boolean supportColumnarReads(InputPartition partition) { return supportsColumnar; } @Override public PartitionReader createReader(InputPartition partition) { return new SparkPartitionReader(readFunc, extractFilePartition(partition)); } /** * Extracts the FilePartition from the given InputPartition. Handles both DeltaInputPartition (for * partitioned tables) and FilePartition (for non-partitioned tables). */ private FilePartition extractFilePartition(InputPartition partition) { if (partition instanceof DeltaInputPartition) { return ((DeltaInputPartition) partition).getFilePartition(); } else if (partition instanceof FilePartition) { return (FilePartition) partition; } else { throw new IllegalArgumentException( "Unexpected partition type: " + partition.getClass().getName() + ". Expected DeltaInputPartition or FilePartition."); } } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.read; import static io.delta.spark.internal.v2.utils.ExpressionUtils.dsv2PredicateToCatalystExpression; import io.delta.kernel.Snapshot; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.engine.Engine; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.internal.actions.AddFile; import io.delta.kernel.internal.data.ScanStateRow; import io.delta.kernel.utils.CloseableIterator; import io.delta.spark.internal.v2.read.deletionvector.DeletionVectorSchemaContext; import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; import io.delta.spark.internal.v2.utils.PartitionUtils; import io.delta.spark.internal.v2.utils.ScalaUtils; import java.io.IOException; import java.time.ZoneId; import java.util.*; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.expressions.Expression; import org.apache.spark.sql.catalyst.expressions.InterpretedPredicate; import org.apache.spark.sql.connector.expressions.FieldReference; import org.apache.spark.sql.connector.expressions.NamedReference; import org.apache.spark.sql.connector.read.*; import org.apache.spark.sql.connector.read.colstats.ColumnStatistics; import org.apache.spark.sql.connector.read.partitioning.KeyGroupedPartitioning; import org.apache.spark.sql.connector.read.partitioning.Partitioning; import org.apache.spark.sql.connector.read.partitioning.UnknownPartitioning; import org.apache.spark.sql.connector.read.streaming.MicroBatchStream; import org.apache.spark.sql.delta.DeltaOptions; import org.apache.spark.sql.execution.datasources.*; import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils; import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.sources.Filter; import org.apache.spark.sql.types.StringType; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; /** Spark DSV2 Scan implementation backed by Delta Kernel. */ public class SparkScan implements Scan, SupportsReportStatistics, SupportsRuntimeV2Filtering, SupportsReportPartitioning { /** Supported streaming options for the V2 connector. */ private static final List SUPPORTED_STREAMING_OPTIONS = Collections.unmodifiableList( Arrays.asList( DeltaOptions.STARTING_VERSION_OPTION(), DeltaOptions.STARTING_TIMESTAMP_OPTION(), DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION(), DeltaOptions.MAX_BYTES_PER_TRIGGER_OPTION(), DeltaOptions.IGNORE_DELETES_OPTION(), DeltaOptions.SKIP_CHANGE_COMMITS_OPTION(), DeltaOptions.EXCLUDE_REGEX_OPTION())); /** * Block list of DeltaOptions that are not supported for streaming in V2 connector. Only * startingVersion, startingTimestamp, maxFilesPerTrigger, maxBytesPerTrigger, ignoreDeletes, * skipChangeCommits, and excludeRegex are supported. User-defined custom options (not in * DeltaOptions) are allowed to pass through. */ private static final Set UNSUPPORTED_STREAMING_OPTIONS = Collections.unmodifiableSet( new HashSet<>( Arrays.asList( DeltaOptions.IGNORE_FILE_DELETION_OPTION().toLowerCase(), DeltaOptions.IGNORE_CHANGES_OPTION().toLowerCase(), DeltaOptions.FAIL_ON_DATA_LOSS_OPTION().toLowerCase(), DeltaOptions.CDC_READ_OPTION().toLowerCase(), DeltaOptions.CDC_READ_OPTION_LEGACY().toLowerCase(), DeltaOptions.CDC_END_VERSION().toLowerCase(), DeltaOptions.CDC_END_TIMESTAMP().toLowerCase(), DeltaOptions.SCHEMA_TRACKING_LOCATION().toLowerCase(), DeltaOptions.SCHEMA_TRACKING_LOCATION_ALIAS().toLowerCase(), DeltaOptions.STREAMING_SOURCE_TRACKING_ID().toLowerCase(), DeltaOptions.ALLOW_SOURCE_COLUMN_DROP().toLowerCase(), DeltaOptions.ALLOW_SOURCE_COLUMN_RENAME().toLowerCase(), DeltaOptions.ALLOW_SOURCE_COLUMN_TYPE_CHANGE().toLowerCase()))); private final DeltaSnapshotManager snapshotManager; private final Snapshot initialSnapshot; private final StructType readDataSchema; private final StructType dataSchema; private final StructType partitionSchema; private final Predicate[] pushedToKernelFilters; private final Filter[] dataFilters; private final io.delta.kernel.Scan kernelScan; private final Optional catalogStats; private final Configuration hadoopConf; private final CaseInsensitiveStringMap options; private final scala.collection.immutable.Map scalaOptions; private final SQLConf sqlConf; private final ZoneId zoneId; // Planned input files and stats private List partitionedFiles = new ArrayList<>(); private long totalBytes = 0L; // Estimated size in bytes accounting for column projection, used for query optimizer cost // estimation private long estimatedSizeInBytes = 0L; private volatile boolean planned = false; // Runtime predicates applied after planning (using Set for order-independent comparison) private final Set appliedRuntimePredicates = new HashSet<>(); public SparkScan( DeltaSnapshotManager snapshotManager, Snapshot initialSnapshot, StructType dataSchema, StructType partitionSchema, StructType readDataSchema, Predicate[] pushedToKernelFilters, Filter[] dataFilters, io.delta.kernel.Scan kernelScan, Optional catalogStats, CaseInsensitiveStringMap options) { this.snapshotManager = Objects.requireNonNull(snapshotManager, "snapshotManager is null"); this.initialSnapshot = Objects.requireNonNull(initialSnapshot, "initialSnapshot is null"); this.dataSchema = Objects.requireNonNull(dataSchema, "dataSchema is null"); this.partitionSchema = Objects.requireNonNull(partitionSchema, "partitionSchema is null"); this.readDataSchema = Objects.requireNonNull(readDataSchema, "readDataSchema is null"); this.pushedToKernelFilters = pushedToKernelFilters == null ? new Predicate[0] : pushedToKernelFilters.clone(); this.dataFilters = dataFilters == null ? new Filter[0] : dataFilters.clone(); this.kernelScan = Objects.requireNonNull(kernelScan, "kernelScan is null"); this.catalogStats = Objects.requireNonNull(catalogStats, "catalogStats is null"); this.options = Objects.requireNonNull(options, "options is null"); this.scalaOptions = ScalaUtils.toScalaMap(options); this.hadoopConf = SparkSession.active().sessionState().newHadoopConfWithOptions(scalaOptions); this.sqlConf = SQLConf.get(); this.zoneId = ZoneId.of(sqlConf.sessionLocalTimeZone()); } /** * Read schema for the scan, which is the projection of data columns followed by partition * columns. */ @Override public StructType readSchema() { final List fields = new ArrayList<>(readDataSchema.fields().length + partitionSchema.fields().length); Collections.addAll(fields, readDataSchema.fields()); Collections.addAll(fields, partitionSchema.fields()); return new StructType(fields.toArray(new StructField[0])); } /** * Override columnarSupportMode to explicitly declare whether this scan supports columnar * (vectorized) reading. Without this override, the default {@code PARTITION_DEFINED} mode causes * Spark to eagerly call {@code planInputPartitions()} during query planning to check * per-partition columnar support, triggering unnecessary early file enumeration. * *

Since columnar support is uniform across all partitions (determined by schema compatibility * and table features, not by individual files), we can declare it at the scan level to avoid this * overhead. * *

This must stay consistent with the vectorized reader decision in {@link * PartitionUtils#createDeltaParquetReaderFactory}. In particular, deletion-vector-enabled tables * augment the read schema with internal columns (e.g., {@code __delta_internal_is_row_deleted}), * which changes the schema passed to the vectorized reader check. We replicate that logic here to * ensure the scan-level declaration matches the per-partition reader behavior. */ @Override public Scan.ColumnarSupportMode columnarSupportMode() { // When the table supports deletion vectors, the reader factory augments the read schema // with internal columns via DeletionVectorSchemaContext. Reuse the same class here so the // batch-read check stays consistent — if DeletionVectorSchemaContext adds new fields in // the future, this code path picks them up automatically. StructType schemaForBatchCheck = PartitionUtils.tableSupportsDeletionVectors(initialSnapshot) ? new DeletionVectorSchemaContext(readDataSchema, partitionSchema) .getSchemaWithDvColumn() : readDataSchema; return ParquetUtils.isBatchReadSupportedForSchema(sqlConf, schemaForBatchCheck) ? Scan.ColumnarSupportMode.SUPPORTED : Scan.ColumnarSupportMode.UNSUPPORTED; } @Override public Batch toBatch() { ensurePlanned(); return new SparkBatch( initialSnapshot, dataSchema, partitionSchema, readDataSchema, partitionedFiles, pushedToKernelFilters, dataFilters, totalBytes, scalaOptions, hadoopConf); } @Override public MicroBatchStream toMicroBatchStream(String checkpointLocation) { DeltaOptions deltaOptions = new DeltaOptions(scalaOptions, sqlConf); // Validate streaming options immediately after constructing DeltaOptions validateStreamingOptions(deltaOptions); return new SparkMicroBatchStream( snapshotManager, // Loads a fresh snapshot as the baseline for schema change detection and table identity // checks. SparkScan's initialSnapshot is from analysis time and may be stale by stream // start/restart. // Matches V1's DeltaDataSource.createSource() behavior. snapshotManager.loadLatestSnapshot(), hadoopConf, SparkSession.active(), deltaOptions, getTablePath(), dataSchema, partitionSchema, readDataSchema, dataFilters != null ? dataFilters : new Filter[0], scalaOptions != null ? scalaOptions : scala.collection.immutable.Map$.MODULE$.empty()); } @Override public String description() { final String pushed = Arrays.stream(pushedToKernelFilters) .map(Object::toString) .collect(Collectors.joining(", ")); final String data = Arrays.stream(dataFilters).map(Object::toString).collect(Collectors.joining(", ")); return String.format(Locale.ROOT, "PushedFilters: [%s], DataFilters: [%s]", pushed, data); } @Override public Statistics estimateStatistics() { ensurePlanned(); final long plannedBytes = estimatedSizeInBytes; // When catalog stats are available and CBO is enabled, combine table-level stats // (for numRows/columnStats) with planned file stats (for sizeInBytes). // This mirrors V1's LogicalRelation.computeStats() which gates column stats on // conf.cboEnabled || conf.planStatsEnabled. boolean useCatalogStats = sqlConf.cboEnabled() || sqlConf.planStatsEnabled(); if (useCatalogStats && catalogStats.isPresent()) { final Statistics stats = catalogStats.get(); return new Statistics() { @Override public OptionalLong sizeInBytes() { // Planned file size is authoritative (even if 0 for an empty table) return OptionalLong.of(plannedBytes); } @Override public OptionalLong numRows() { // TODO: Use accurate row count from planned files (sum of AddFile.numRecords) // instead of catalog stats, which are stale (point-in-time from ANALYZE) and // not adjusted for partition pruning. return stats.numRows(); } @Override public Map columnStats() { // TODO: After partition pruning, column stats (e.g. min, max, nullCount, // distinctCount) could be tightened based on the pruned file-level stats. return stats.columnStats(); } }; } // No catalog stats available or CBO disabled — return stats from planned files only return new Statistics() { @Override public OptionalLong sizeInBytes() { return OptionalLong.of(plannedBytes); } @Override public OptionalLong numRows() { // Row count is unknown without catalog stats return OptionalLong.empty(); } }; } /** * Computes the estimated size in bytes accounting for column projection. * *

This mirrors what {@code SizeInBytesOnlyStatsPlanVisitor.visitUnaryNode} (from Spark code) * would compute for a {@code Project} over a {@code LogicalRelation}: {@code sizeInBytes = * childSizeInBytes * outputRowSize / childRowSize} * *

Where: * *

    *
  • childRowSize = {@code ROW_OVERHEAD + dataSchema + partitionSchema} (equivalent to * LogicalRelation output) *
  • outputRowSize = {@code ROW_OVERHEAD + readDataSchema + partitionSchema} * (equivalent to Project output) *
* *

When catalog column stats are available, uses per-column {@code avgLen} instead of {@code * defaultSize()} for more accurate size estimation, mirroring {@code * EstimationUtils.getSizePerRow()} behavior. * *

This provides consistent statistics with the v1 code path (LogicalRelation + visitUnaryNode * from Spark code directory). * * @param totalBytes the total size in bytes of the planned files (raw physical size) * @return the estimated size in bytes after accounting for column projection */ private long computeEstimatedSizeWithColumnProjection(long totalBytes) { if (totalBytes <= 0) { return totalBytes; } // Row overhead constant, matching EstimationUtils.getSizePerRow (from Spark) final int ROW_OVERHEAD = 8; // Use avgLen from catalog column stats when available for more accurate estimation Map avgLenByColumn = getAvgLenByColumn(); final long fullSchemaRowSize = ROW_OVERHEAD + getSchemaSize(dataSchema, avgLenByColumn) + getSchemaSize(partitionSchema, avgLenByColumn); final long outputRowSize = ROW_OVERHEAD + getSchemaSize(readSchema(), avgLenByColumn); long estimatedBytes = (totalBytes * outputRowSize) / fullSchemaRowSize; return Math.max(1L, estimatedBytes); } /** * Returns a map of column name to avgLen from catalog column stats, used to improve size * estimation accuracy when catalog stats are available. */ private Map getAvgLenByColumn() { if (!catalogStats.isPresent()) { return Collections.emptyMap(); } Map colStats = catalogStats.get().columnStats(); if (colStats.isEmpty()) { return Collections.emptyMap(); } Map result = new HashMap<>(); for (Map.Entry entry : colStats.entrySet()) { result.put(entry.getKey().fieldNames()[0], entry.getValue().avgLen()); } return result; } /** * Computes the estimated in-memory size for a schema, using avgLen from catalog stats when * available, falling back to defaultSize(). Mirrors EstimationUtils.getSizePerRow(). For * StringType columns with avgLen, adds UTF8String overhead (base + offset + numBytes = 12 bytes). */ private static long getSchemaSize(StructType schema, Map avgLenByColumn) { long size = 0; for (StructField field : schema.fields()) { OptionalLong avgLen = avgLenByColumn.getOrDefault(field.name(), OptionalLong.empty()); if (avgLen.isPresent()) { if (field.dataType() instanceof StringType) { // UTF8String: base + offset + numBytes (matching EstimationUtils.getSizePerRow) size += avgLen.getAsLong() + 8 + 4; } else { size += avgLen.getAsLong(); } } else { size += field.dataType().defaultSize(); } } return size; } /** * Get the table path from the scan state. * * @return the table path with trailing slash */ public String getTablePath() { final Engine tableEngine = DefaultEngine.create(hadoopConf); final Row scanState = kernelScan.getScanState(tableEngine); final String tableRoot = ScanStateRow.getTableRoot(scanState).toUri().toString(); return tableRoot.endsWith("/") ? tableRoot : tableRoot + "/"; } /** * Plan the files to scan by materializing {@link PartitionedFile}s and aggregating size stats. * Ensures all iterators are closed to avoid resource leaks. */ private void planScanFiles() { final Engine tableEngine = DefaultEngine.create(hadoopConf); final String tablePath = getTablePath(); final Iterator scanFileBatches = kernelScan.getScanFiles(tableEngine); final String[] locations = new String[0]; final scala.collection.immutable.Map otherConstantMetadataColumnValues = scala.collection.immutable.Map$.MODULE$.empty(); while (scanFileBatches.hasNext()) { final FilteredColumnarBatch batch = scanFileBatches.next(); try (CloseableIterator addFileRowIter = batch.getRows()) { while (addFileRowIter.hasNext()) { final Row row = addFileRowIter.next(); final AddFile addFile = new AddFile(row.getStruct(0)); final PartitionedFile partitionedFile = PartitionUtils.buildPartitionedFile(addFile, partitionSchema, tablePath, zoneId); totalBytes += addFile.getSize(); partitionedFiles.add(partitionedFile); } } catch (IOException e) { throw new RuntimeException(e); } } // Pre-compute estimated size accounting for column projection estimatedSizeInBytes = computeEstimatedSizeWithColumnProjection(totalBytes); } /** * Ensure the scan is planned exactly once in a thread-safe manner, optionally applying runtime * filters. */ private synchronized void ensurePlanned(List runtimePredicates) { // First, ensure planning is done if (!planned) { planScanFiles(); planned = true; } // Then apply runtime predicates if provided if (runtimePredicates != null && !runtimePredicates.isEmpty()) { // Record the applied predicates for equals/hashCode comparison for (RuntimePredicate filter : runtimePredicates) { appliedRuntimePredicates.add(filter.predicate); } List runtimeFilteredPartitionedFiles = new ArrayList<>(); for (PartitionedFile pf : this.partitionedFiles) { InternalRow partitionValues = pf.partitionValues(); boolean allMatch = runtimePredicates.stream() .allMatch(predicate -> predicate.evaluator.eval(partitionValues)); if (allMatch) { runtimeFilteredPartitionedFiles.add(pf); } } // Update partitionedFiles, totalBytes, and estimatedSizeInBytes if any partition is // filtered out if (runtimeFilteredPartitionedFiles.size() < this.partitionedFiles.size()) { this.partitionedFiles = runtimeFilteredPartitionedFiles; this.totalBytes = runtimeFilteredPartitionedFiles.stream().mapToLong(PartitionedFile::fileSize).sum(); this.estimatedSizeInBytes = computeEstimatedSizeWithColumnProjection(this.totalBytes); } } } /** Ensure the scan is planned exactly once in a thread-safe manner. */ private void ensurePlanned() { // Pass null to indicate no runtime predicate should be applied - just perform the scan planning ensurePlanned(null); } public StructType getDataSchema() { return dataSchema; } public StructType getPartitionSchema() { return partitionSchema; } public StructType getReadDataSchema() { return readDataSchema; } public CaseInsensitiveStringMap getOptions() { return options; } public Configuration getConfiguration() { return hadoopConf; } @Override public NamedReference[] filterAttributes() { return Arrays.stream(partitionSchema.fields()) .map(field -> FieldReference.column(field.name())) .toArray(NamedReference[]::new); } @Override public void filter(org.apache.spark.sql.connector.expressions.filter.Predicate[] predicates) { // Try to convert runtime predicates to catalyst expressions, then create predicate evaluators // Only track predicates that successfully convert to evaluators List runtimePredicates = new ArrayList<>(); for (org.apache.spark.sql.connector.expressions.filter.Predicate predicate : predicates) { // only the predicates on partition columns will be converted Optional catalystExpr = dsv2PredicateToCatalystExpression(predicate, partitionSchema); if (catalystExpr.isPresent()) { InterpretedPredicate predicateEvaluator = org.apache.spark.sql.catalyst.expressions.Predicate.createInterpreted( catalystExpr.get()); runtimePredicates.add(new RuntimePredicate(predicate, predicateEvaluator)); } } if (!runtimePredicates.isEmpty()) { // Apply runtime predicates within the synchronized ensurePlanned method ensurePlanned(runtimePredicates); } } /** * Validates that unsupported streaming options are not used. Uses a block list approach - only * blocks known DeltaOptions that are unsupported, allowing user-defined custom options to pass * through. * *

Note: DeltaOptions internally uses CaseInsensitiveMap, which preserves the original key * casing but performs case-insensitive lookups. * * @param deltaOptions the DeltaOptions to validate * @throws UnsupportedOperationException if unsupported options are found */ static void validateStreamingOptions(DeltaOptions deltaOptions) { List unsupportedOptions = new ArrayList<>(); scala.collection.Iterator keysIterator = deltaOptions.options().keysIterator(); while (keysIterator.hasNext()) { String key = keysIterator.next(); // DeltaOptions uses CaseInsensitiveMap with keys already lowercased. if (UNSUPPORTED_STREAMING_OPTIONS.contains(key)) { unsupportedOptions.add(key); } } if (!unsupportedOptions.isEmpty()) { throw new UnsupportedOperationException( String.format( "The following streaming options are not supported: [%s]. " + "Supported options are: [%s].", String.join(", ", unsupportedOptions), String.join(", ", SUPPORTED_STREAMING_OPTIONS))); } } /** * Reports partition key expressions to Spark so it can recognize partition-aligned data layout. * Called by V2ScanPartitioningAndOrdering during logical optimization to extract partition keys. * Together with HasPartitionKey on DeltaInputPartition, this enables Spark to eliminate shuffles * for joins and aggregations on partition columns. * *

Note: This method triggers scan file materialization via {@link #ensurePlanned()} because * {@code numPartitions} is derived from the planned file count. Since Spark calls this during * logical optimization (before {@link #toBatch()}), this changes when planning occurs compared to * the non-partitioned path. This is functionally correct as {@code ensurePlanned} is idempotent. */ @Override public Partitioning outputPartitioning() { // If no partition columns, return unknown partitioning if (partitionSchema.fields().length == 0) { return new UnknownPartitioning(0); } ensurePlanned(); // Create partition key expressions from partition schema org.apache.spark.sql.connector.expressions.Expression[] keys = Arrays.stream(partitionSchema.fields()) .map( field -> (org.apache.spark.sql.connector.expressions.Expression) FieldReference.column(field.name())) .toArray(org.apache.spark.sql.connector.expressions.Expression[]::new); // numPartitions is not used by Spark's KeyGroupedPartitioning handling (Spark derives // partition count from the actual InputPartition[] with HasPartitionKey), so we use the // file count as a reasonable upper-bound estimate. return new KeyGroupedPartitioning(keys, partitionedFiles.size()); } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } SparkScan that = (SparkScan) o; return Objects.equals(initialSnapshot.getPath(), that.initialSnapshot.getPath()) && initialSnapshot.getVersion() == that.initialSnapshot.getVersion() && Objects.equals(dataSchema, that.dataSchema) && Objects.equals(partitionSchema, that.partitionSchema) && Objects.equals(readDataSchema, that.readDataSchema) && Arrays.equals(pushedToKernelFilters, that.pushedToKernelFilters) && Arrays.equals(dataFilters, that.dataFilters) // ignoring kernelScan because it is derived from Snapshot which is created from tablePath, // with pushed down filters that are also recorded in `pushedToKernelFilters` && Objects.equals(options, that.options) && Objects.equals(appliedRuntimePredicates, that.appliedRuntimePredicates) && Objects.equals(catalogStats, that.catalogStats); } @Override public int hashCode() { int result = Objects.hash( catalogStats, initialSnapshot.getPath(), initialSnapshot.getVersion(), dataSchema, partitionSchema, readDataSchema, options, appliedRuntimePredicates); result = 31 * result + Arrays.hashCode(pushedToKernelFilters); result = 31 * result + Arrays.hashCode(dataFilters); return result; } /** * Holds a runtime predicate from {@link #filter(Predicate[])} along with its compiled evaluator. * *

Only created for predicates that can be successfully converted to Catalyst expressions * (typically predicates on partition columns) and compiled into InterpretedPredicate evaluators. * Predicates that cannot be converted are not instantiated as RuntimePredicate objects. */ private static class RuntimePredicate { final org.apache.spark.sql.connector.expressions.filter.Predicate predicate; final InterpretedPredicate evaluator; RuntimePredicate( org.apache.spark.sql.connector.expressions.filter.Predicate predicate, InterpretedPredicate evaluator) { this.predicate = predicate; this.evaluator = evaluator; } } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScanBuilder.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.read; import static java.util.Objects.requireNonNull; import io.delta.kernel.Snapshot; import io.delta.kernel.expressions.And; import io.delta.kernel.expressions.Predicate; import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; import io.delta.spark.internal.v2.utils.ExpressionUtils; import java.util.*; import java.util.stream.Collectors; import org.apache.spark.sql.connector.read.ScanBuilder; import org.apache.spark.sql.connector.read.Statistics; import org.apache.spark.sql.connector.read.SupportsPushDownFilters; import org.apache.spark.sql.connector.read.SupportsPushDownRequiredColumns; import org.apache.spark.sql.sources.Filter; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; /** * A Spark ScanBuilder implementation that wraps Delta Kernel's ScanBuilder. This allows Spark to * use Delta Kernel for reading Delta tables. */ public class SparkScanBuilder implements ScanBuilder, SupportsPushDownRequiredColumns, SupportsPushDownFilters { private io.delta.kernel.ScanBuilder kernelScanBuilder; private final Snapshot initialSnapshot; private final DeltaSnapshotManager snapshotManager; private final StructType dataSchema; private final StructType partitionSchema; private final StructType tableSchema; private final Optional catalogStats; private final CaseInsensitiveStringMap options; private final Set partitionColumnSet; private StructType requiredDataSchema; // pushedKernelPredicates: Predicates that have been pushed down to the Delta Kernel for // evaluation. // pushedSparkFilters: The same pushed predicates, but represented using Spark’s {@link Filter} // API (needed because Spark operates on Filter objects while the Kernel uses Predicate) private Predicate[] pushedKernelPredicates; private Filter[] pushedSparkFilters; private Filter[] dataFilters; /** * Creates a SparkScanBuilder with the given snapshot and configuration. * * @param tableName the name of the table * @param initialSnapshot Snapshot created during connector setup * @param snapshotManager the snapshot manager for this table * @param dataSchema the data schema (non-partition columns) * @param partitionSchema the partition schema * @param tableSchema the full table schema (all columns) for filter type alignment * @param catalogStats optional V2 Statistics converted from catalog stats * @param options scan options */ public SparkScanBuilder( String tableName, io.delta.kernel.Snapshot initialSnapshot, DeltaSnapshotManager snapshotManager, StructType dataSchema, StructType partitionSchema, StructType tableSchema, Optional catalogStats, CaseInsensitiveStringMap options) { this.initialSnapshot = requireNonNull(initialSnapshot, "initialSnapshot is null"); this.kernelScanBuilder = initialSnapshot.getScanBuilder(); this.snapshotManager = requireNonNull(snapshotManager, "snapshotManager is null"); this.dataSchema = requireNonNull(dataSchema, "dataSchema is null"); this.partitionSchema = requireNonNull(partitionSchema, "partitionSchema is null"); this.tableSchema = requireNonNull(tableSchema, "tableSchema is null"); this.catalogStats = requireNonNull(catalogStats, "catalogStats is null"); this.options = requireNonNull(options, "options is null"); this.requiredDataSchema = this.dataSchema; this.partitionColumnSet = Arrays.stream(this.partitionSchema.fields()) .map(f -> f.name().toLowerCase(Locale.ROOT)) .collect(Collectors.toSet()); this.pushedKernelPredicates = new Predicate[0]; this.dataFilters = new Filter[0]; } @Override public void pruneColumns(StructType requiredSchema) { requireNonNull(requiredSchema, "requiredSchema is null"); this.requiredDataSchema = new StructType( Arrays.stream(requiredSchema.fields()) .filter(f -> !partitionColumnSet.contains(f.name().toLowerCase(Locale.ROOT))) .toArray(StructField[]::new)); } @Override public Filter[] pushFilters(Filter[] filters) { List kernelSupportedFilters = new ArrayList<>(); List convertedKernelPredicates = new ArrayList<>(); List dataFilterList = new ArrayList<>(); List postScanFilters = new ArrayList<>(); for (Filter filter : filters) { ExpressionUtils.FilterClassificationResult classification = ExpressionUtils.classifyFilter(filter, partitionColumnSet, tableSchema); // Collect kernel predicates if supported if (classification.isKernelSupported) { convertedKernelPredicates.add(classification.kernelPredicate.get()); if (!classification.isPartialConversion) { // Add filter to kernelSupportedFilters if it is fully converted // TODO: add partially converted Spark filter as well // right now we only have the partially converted kernel predicate kernelSupportedFilters.add(filter); } } // Collect data filters if (classification.isDataFilter) { dataFilterList.add(filter); } // Collect post-scan filters // Filters with the following characteristics need to be evaluated after delta kernel scan: // 1. filters that are not supported by delta kernel, thus kernel cannot apply them during // scan // 2. filters that are not fully converted to kernel predicates, thus the unconverted part // needs to be evaluated after scan // 3. filters that are data filters, as kernel only evaluate data filter based on min/max // stats, thus need to be evaluated with actual data after scan // // Fully converted partition filters are used to prune partitions during scan. Only the // partitions that satisfy the filters will be scanned, so no need for post-scan evaluation. if (!classification.isKernelSupported || classification.isPartialConversion || classification.isDataFilter) { postScanFilters.add(filter); } } this.pushedSparkFilters = kernelSupportedFilters.toArray(new Filter[0]); this.pushedKernelPredicates = convertedKernelPredicates.toArray(new Predicate[0]); if (this.pushedKernelPredicates.length > 0) { Optional kernelAnd = Arrays.stream(this.pushedKernelPredicates).reduce(And::new); this.kernelScanBuilder = this.kernelScanBuilder.withFilter(kernelAnd.get()); } this.dataFilters = dataFilterList.toArray(new Filter[0]); return postScanFilters.toArray(new Filter[0]); } @Override public Filter[] pushedFilters() { return this.pushedSparkFilters; } @Override public org.apache.spark.sql.connector.read.Scan build() { return new SparkScan( snapshotManager, initialSnapshot, dataSchema, partitionSchema, requiredDataSchema, pushedKernelPredicates, dataFilters, kernelScanBuilder.build(), catalogStats, options); } CaseInsensitiveStringMap getOptions() { return options; } StructType getDataSchema() { return dataSchema; } StructType getPartitionSchema() { return partitionSchema; } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/read/deletionvector/ColumnVectorWithFilter.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.read.deletionvector; import org.apache.spark.sql.types.Decimal; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.vectorized.ColumnVector; import org.apache.spark.sql.vectorized.ColumnarArray; import org.apache.spark.sql.vectorized.ColumnarMap; import org.apache.spark.unsafe.types.UTF8String; /** * A column vector that applies row-level filtering using a row ID mapping. * *

Wraps an existing column vector and remaps row indices during data access, effectively * filtering the original data to only expose the live subset of rows without copying data. * *

Follows Apache Iceberg's ColumnVectorWithFilter pattern. */ public class ColumnVectorWithFilter extends ColumnVector { private final ColumnVector delegate; private final int[] rowIdMapping; private volatile ColumnVectorWithFilter[] children = null; public ColumnVectorWithFilter(ColumnVector delegate, int[] rowIdMapping) { super(delegate.dataType()); this.delegate = delegate; this.rowIdMapping = rowIdMapping; } @Override public void close() { delegate.close(); } @Override public boolean hasNull() { return delegate.hasNull(); } @Override public int numNulls() { // Computing the actual number of nulls with rowIdMapping is expensive. // It is OK to overestimate and return the number of nulls in the original vector. return delegate.numNulls(); } @Override public boolean isNullAt(int rowId) { return delegate.isNullAt(rowIdMapping[rowId]); } @Override public boolean getBoolean(int rowId) { return delegate.getBoolean(rowIdMapping[rowId]); } @Override public byte getByte(int rowId) { return delegate.getByte(rowIdMapping[rowId]); } @Override public short getShort(int rowId) { return delegate.getShort(rowIdMapping[rowId]); } @Override public int getInt(int rowId) { return delegate.getInt(rowIdMapping[rowId]); } @Override public long getLong(int rowId) { return delegate.getLong(rowIdMapping[rowId]); } @Override public float getFloat(int rowId) { return delegate.getFloat(rowIdMapping[rowId]); } @Override public double getDouble(int rowId) { return delegate.getDouble(rowIdMapping[rowId]); } @Override public ColumnarArray getArray(int rowId) { return delegate.getArray(rowIdMapping[rowId]); } @Override public ColumnarMap getMap(int rowId) { return delegate.getMap(rowIdMapping[rowId]); } @Override public Decimal getDecimal(int rowId, int precision, int scale) { return delegate.getDecimal(rowIdMapping[rowId], precision, scale); } @Override public UTF8String getUTF8String(int rowId) { return delegate.getUTF8String(rowIdMapping[rowId]); } @Override public byte[] getBinary(int rowId) { return delegate.getBinary(rowIdMapping[rowId]); } /** * Returns the child vector at the given ordinal, wrapped with the same row ID mapping. * *

Uses double-checked locking for thread-safe lazy initialization: the volatile {@code * children} field allows lock-free reads after initialization, while the synchronized block * ensures at-most-once creation. All children are created eagerly within the lock to avoid * per-element races. */ @Override public ColumnVector getChild(int ordinal) { if (children == null) { synchronized (this) { if (children == null) { // Eagerly create all children to avoid race condition on children[ordinal] access StructType structType = (StructType) dataType(); ColumnVectorWithFilter[] newChildren = new ColumnVectorWithFilter[structType.fields().length]; for (int i = 0; i < newChildren.length; i++) { newChildren[i] = new ColumnVectorWithFilter(delegate.getChild(i), rowIdMapping); } children = newChildren; } } } return children[ordinal]; } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/read/deletionvector/DeletionVectorReadFunction.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.read.deletionvector; import io.delta.spark.internal.v2.utils.CloseableIterator; import java.io.Serializable; import java.util.Arrays; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.ProjectingInternalRow; import org.apache.spark.sql.execution.datasources.PartitionedFile; import org.apache.spark.sql.vectorized.ColumnVector; import org.apache.spark.sql.vectorized.ColumnarBatch; import scala.Function1; import scala.collection.Iterator; import scala.runtime.AbstractFunction1; /** * Wraps a Parquet reader function to apply deletion vector filtering. * *

This function: * *

    *
  1. Reads rows from the base Parquet reader (which includes the is_row_deleted column) *
  2. Filters out deleted rows (where is_row_deleted != 0) *
  3. Projects out the is_row_deleted column from the output *
* *

The returned iterator implements {@link java.io.Closeable} to ensure proper resource cleanup * of the underlying Parquet reader, even when the iterator is not fully consumed. * *

The reader mode (row-based vs vectorized) is determined once at scan planning time and does * not change mid-stream. See {@link #wrap}. */ public class DeletionVectorReadFunction extends AbstractFunction1> implements Serializable { private static final long serialVersionUID = 1L; /** Byte value in the DV column indicating the row is NOT deleted (row should be kept). */ private static final byte ROW_NOT_DELETED = 0; private final Function1> baseReadFunc; private final DeletionVectorSchemaContext dvSchemaContext; private final boolean isVectorizedReader; private DeletionVectorReadFunction( Function1> baseReadFunc, DeletionVectorSchemaContext dvSchemaContext, boolean isVectorizedReader) { this.baseReadFunc = baseReadFunc; this.dvSchemaContext = dvSchemaContext; this.isVectorizedReader = isVectorizedReader; } @Override public Iterator apply(PartitionedFile file) { if (isVectorizedReader) { return applyBatch(file); } else { return applyRow(file); } } /** Row-based: filter deleted rows and project out the DV column. */ private Iterator applyRow(PartitionedFile file) { int dvColumnIndex = dvSchemaContext.getDvColumnIndex(); ProjectingInternalRow projection = ProjectingInternalRow.apply( dvSchemaContext.getOutputSchema(), dvSchemaContext.getOutputColumnOrdinals()); return CloseableIterator.wrap(baseReadFunc.apply(file)) .filterCloseable(row -> row.getByte(dvColumnIndex) == ROW_NOT_DELETED) .mapCloseable( row -> { projection.project(row); return (InternalRow) projection; }); } /** Vectorized: filter active rows and project out the DV column from each batch. */ @SuppressWarnings("unchecked") private Iterator applyBatch(PartitionedFile file) { int dvColumnIndex = dvSchemaContext.getDvColumnIndex(); // In vectorized mode Spark passes ColumnarBatch via type erasure as InternalRow. Iterator baseIterator = (Iterator) (Iterator) baseReadFunc.apply(file); return (Iterator) (Iterator) CloseableIterator.wrap(baseIterator) .mapCloseable( item -> { if (item instanceof ColumnarBatch) { return filterAndProjectBatch((ColumnarBatch) item, dvColumnIndex); } throw new IllegalStateException( "Expected ColumnarBatch when vectorized reader is enabled, but got: " + item.getClass()); }); } /** Filter active rows and project out the DV column from a ColumnarBatch. */ private static ColumnarBatch filterAndProjectBatch(ColumnarBatch batch, int dvColumnIndex) { int[] activeRows = findActiveRows(batch, dvColumnIndex); ColumnVector[] filteredColumns = buildFilteredColumns(batch, dvColumnIndex, activeRows); return new ColumnarBatch(filteredColumns, activeRows.length); } /** * Build projected output columns by dropping the DV column and applying active row mapping. The * excluded DV column's lifecycle is managed by the base Spark vectorized reader, not by us. */ private static ColumnVector[] buildFilteredColumns( ColumnarBatch batch, int dvColumnIndex, int[] activeRows) { ColumnVector[] filteredColumns = new ColumnVector[batch.numCols() - 1]; int outputIndex = 0; for (int inputIndex = 0; inputIndex < batch.numCols(); inputIndex++) { if (inputIndex == dvColumnIndex) { continue; } filteredColumns[outputIndex++] = new ColumnVectorWithFilter(batch.column(inputIndex), activeRows); } return filteredColumns; } /** Find indices of rows where DV column is 0 (not deleted). */ private static int[] findActiveRows(ColumnarBatch batch, int dvColumnIndex) { ColumnVector dvColumn = batch.column(dvColumnIndex); int[] temp = new int[batch.numRows()]; int count = 0; for (int i = 0; i < batch.numRows(); i++) { if (dvColumn.getByte(i) == ROW_NOT_DELETED) { temp[count++] = i; } } return Arrays.copyOf(temp, count); } /** Factory method to wrap a reader function with DV filtering. */ public static DeletionVectorReadFunction wrap( Function1> baseReadFunc, DeletionVectorSchemaContext dvSchemaContext, boolean isVectorizedReader) { return new DeletionVectorReadFunction(baseReadFunc, dvSchemaContext, isVectorizedReader); } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/read/deletionvector/DeletionVectorSchemaContext.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.read.deletionvector; import java.io.Serializable; import java.util.Arrays; import org.apache.spark.sql.delta.DeltaParquetFileFormat; import org.apache.spark.sql.types.StructType; import scala.collection.immutable.Seq; /** * Schema context for deletion vector processing in the V2 connector. * *

Encapsulates schema with DV column and pre-computed indices needed for DV filtering. */ public class DeletionVectorSchemaContext implements Serializable { private static final long serialVersionUID = 1L; private final StructType schemaWithDvColumn; private final int dvColumnIndex; private final int inputColumnCount; private final StructType outputSchema; private final Seq outputColumnOrdinals; /** * Create a DV schema context for encapsulating schema info and indices needed for DV filtering. * * @param readDataSchema original data schema without DV column * @param partitionSchema partition columns schema * @throws IllegalArgumentException if readDataSchema already contains the DV column */ public DeletionVectorSchemaContext(StructType readDataSchema, StructType partitionSchema) { // Validate that readDataSchema doesn't already contain the DV column to ensure the DV column // is added only once. While Delta uses the "__delta_internal_" prefix as a naming convention // for internal columns (listed in DeltaColumnMapping.DELTA_INTERNAL_COLUMNS), there's no // enforced schema validation that prevents users from creating such columns. This check // provides a safety guard in the V2 connector. String dvColumnName = DeltaParquetFileFormat.IS_ROW_DELETED_COLUMN_NAME(); if (Arrays.asList(readDataSchema.fieldNames()).contains(dvColumnName)) { throw new IllegalArgumentException( "readDataSchema already contains the deletion vector column: " + dvColumnName); } this.schemaWithDvColumn = readDataSchema.add(DeltaParquetFileFormat.IS_ROW_DELETED_STRUCT_FIELD()); this.dvColumnIndex = schemaWithDvColumn.fieldIndex(DeltaParquetFileFormat.IS_ROW_DELETED_COLUMN_NAME()); this.inputColumnCount = schemaWithDvColumn.fields().length + partitionSchema.fields().length; this.outputSchema = readDataSchema.merge(partitionSchema, /* handleDuplicateColumns= */ false); // Pre-compute output column ordinals: all indices except dvColumnIndex. int[] ordinals = new int[inputColumnCount - 1]; int idx = 0; for (int i = 0; i < inputColumnCount; i++) { if (i != dvColumnIndex) { ordinals[idx++] = i; } } this.outputColumnOrdinals = scala.Predef.wrapIntArray(ordinals).toSeq(); } /** Returns schema with the __delta_internal_is_row_deleted column added. */ public StructType getSchemaWithDvColumn() { return schemaWithDvColumn; } public int getDvColumnIndex() { return dvColumnIndex; } public int getInputColumnCount() { return inputColumnCount; } public StructType getOutputSchema() { return outputSchema; } /** Returns pre-computed output column ordinals for ProjectingInternalRow. */ public Seq getOutputColumnOrdinals() { return outputColumnOrdinals; } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/DeltaSnapshotManager.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.snapshot; import io.delta.kernel.CommitRange; import io.delta.kernel.Snapshot; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.DeltaHistoryManager; import io.delta.spark.internal.v2.exception.VersionNotFoundException; import java.util.Optional; import org.apache.spark.annotation.Experimental; /** * Interface for managing Delta table snapshots. * *

This interface provides methods for loading, caching, and querying Delta table snapshots. It * supports both current snapshot access and historical snapshot queries based on version or * timestamp. * *

Implementations of this interface are responsible for managing snapshot lifecycle, including * loading snapshots from storage and maintaining any necessary caching. */ @Experimental public interface DeltaSnapshotManager { /** * Loads and returns the latest snapshot of the Delta table. * * @return the latest snapshot of the Delta table */ Snapshot loadLatestSnapshot(); /** * Loads and returns a snapshot at a specific version of the Delta table. * * @param version the version number to load (must be >= 0) * @return the snapshot at the specified version * @throws io.delta.kernel.exceptions.KernelException if the version cannot be loaded */ Snapshot loadSnapshotAt(long version); /** * Finds and returns the commit that was active at a specific timestamp. * * @param timestampMillis the timestamp in milliseconds since epoch (UTC) * @param canReturnLastCommit if true, returns the last commit if the timestamp is after all * commits; if false, throws an exception * @param mustBeRecreatable if true, only considers commits that can be fully recreated from * available log files; if false, considers all commits * @param canReturnEarliestCommit if true, returns the earliest commit if the timestamp is before * all commits; if false, throws an exception * @return the commit that was active at the specified timestamp * @throws io.delta.kernel.exceptions.KernelException if no suitable commit is found based on the * provided flags */ DeltaHistoryManager.Commit getActiveCommitAtTime( long timestampMillis, boolean canReturnLastCommit, boolean mustBeRecreatable, boolean canReturnEarliestCommit); /** * Checks if a specific version of the Delta table exists and is accessible. * * @param version the version to check * @param mustBeRecreatable if true, requires that the version can be fully recreated from * available log files; if false, only requires that the version's log file exists * @param allowOutOfRange if true, allows versions greater than the latest version without * throwing an exception; if false, throws exception for out-of-range versions * @throws VersionNotFoundException if the version is not available or does not meet the specified * criteria */ void checkVersionExists(long version, boolean mustBeRecreatable, boolean allowOutOfRange) throws VersionNotFoundException; /** * Gets a range of table changes (commits) between start and end versions. * *

Expected Behavior: * *

    *
  • Returns a {@link io.delta.kernel.CommitRange} representing all commits from the start * version (inclusive) to the end version (inclusive if provided) *
  • If endVersion is not provided, the range extends to the latest available version *
  • The returned CommitRange can be used to iterate through actions in the version range *
  • This is typically used for streaming and incremental processing scenarios *
* *

Use Case: Use this method for streaming queries, incremental processing, or CDC * scenarios where you need to process changes between versions. * * @param engine the engine implementation for executing operations * @param startVersion the starting version (inclusive) * @param endVersion optional ending version (inclusive); if not provided, extends to latest * @return a CommitRange representing the specified range of commits */ CommitRange getTableChanges(Engine engine, long startVersion, Optional endVersion); } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/PathBasedSnapshotManager.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.snapshot; import static java.util.Objects.requireNonNull; import io.delta.kernel.CommitRange; import io.delta.kernel.CommitRangeBuilder; import io.delta.kernel.Snapshot; import io.delta.kernel.TableManager; import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.DeltaHistoryManager; import io.delta.kernel.internal.SnapshotImpl; import io.delta.spark.internal.v2.exception.VersionNotFoundException; import java.util.ArrayList; import java.util.Optional; import org.apache.hadoop.conf.Configuration; import org.apache.spark.annotation.Experimental; /** Implementation of DeltaSnapshotManager for managing Delta snapshots for Path-based Table. */ @Experimental public class PathBasedSnapshotManager implements DeltaSnapshotManager { private final String tablePath; private final Engine kernelEngine; public PathBasedSnapshotManager(String tablePath, Configuration hadoopConf) { this(tablePath, DefaultEngine.create(requireNonNull(hadoopConf, "hadoopConf is null"))); } public PathBasedSnapshotManager(String tablePath, Engine kernelEngine) { this.tablePath = requireNonNull(tablePath, "tablePath is null"); this.kernelEngine = requireNonNull(kernelEngine, "kernelEngine is null"); } /** * Loads the latest snapshot of the Delta table. * * @return the newly loaded snapshot */ @Override public Snapshot loadLatestSnapshot() { return TableManager.loadSnapshot(tablePath).build(kernelEngine); } /** * Loads a specific version of the Delta table. * * @param version the version to load * @return the snapshot at the specified version */ @Override public Snapshot loadSnapshotAt(long version) { return TableManager.loadSnapshot(tablePath).atVersion(version).build(kernelEngine); } /** * Finds the active commit at a specific timestamp. * *

This method searches the Delta table's commit history to find the commit that was active at * the specified timestamp. * * @param timestampMillis the timestamp in milliseconds since epoch (UTC) * @param canReturnLastCommit if true, returns the last commit if the timestamp is after all * commits * @param mustBeRecreatable if true, only considers commits that can be recreated (i.e., all * necessary log files are available) * @param canReturnEarliestCommit if true, returns the earliest commit if the timestamp is before * all commits * @return the commit that was active at the specified timestamp */ @Override public DeltaHistoryManager.Commit getActiveCommitAtTime( long timestampMillis, boolean canReturnLastCommit, boolean mustBeRecreatable, boolean canReturnEarliestCommit) { SnapshotImpl snapshot = (SnapshotImpl) loadLatestSnapshot(); return DeltaHistoryManager.getActiveCommitAtTimestamp( kernelEngine, snapshot, snapshot.getLogPath(), timestampMillis, mustBeRecreatable, canReturnLastCommit, canReturnEarliestCommit, new ArrayList<>()); } /** * Checks if a specific version of the Delta table exists and is accessible. * * @param version the version to check * @param mustBeRecreatable if true, requires that the version can be fully recreated from * available log files * @param allowOutOfRange if true, allows versions greater than the latest version without * throwing an exception * @throws VersionNotFoundException if the version is not available */ @Override public void checkVersionExists(long version, boolean mustBeRecreatable, boolean allowOutOfRange) throws VersionNotFoundException { SnapshotImpl snapshot = (SnapshotImpl) loadLatestSnapshot(); long earliest = mustBeRecreatable ? DeltaHistoryManager.getEarliestRecreatableCommit( kernelEngine, snapshot.getLogPath(), Optional.empty() /*earliestRatifiedCommitVersion*/) : DeltaHistoryManager.getEarliestDeltaFile( kernelEngine, snapshot.getLogPath(), Optional.empty() /*earliestRatifiedCommitVersion*/); long latest = snapshot.getVersion(); if (version < earliest || ((version > latest) && !allowOutOfRange)) { throw new VersionNotFoundException(version, earliest, latest); } } @Override public CommitRange getTableChanges(Engine engine, long startVersion, Optional endVersion) { CommitRangeBuilder builder = TableManager.loadCommitRange( tablePath, CommitRangeBuilder.CommitBoundary.atVersion(startVersion)); if (endVersion.isPresent()) { builder = builder.withEndBoundary(CommitRangeBuilder.CommitBoundary.atVersion(endVersion.get())); } return builder.build(engine); } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/SnapshotManagerFactory.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.snapshot; import io.delta.kernel.Meta; import io.delta.kernel.engine.Engine; import io.delta.kernel.unitycatalog.UCCatalogManagedClient; import io.delta.spark.internal.v2.snapshot.unitycatalog.UCManagedTableSnapshotManager; import io.delta.spark.internal.v2.snapshot.unitycatalog.UCTableInfo; import io.delta.spark.internal.v2.snapshot.unitycatalog.UCUtils; import io.delta.storage.commit.uccommitcoordinator.UCClient; import java.util.Map; import java.util.Optional; import org.apache.spark.annotation.Experimental; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.catalyst.catalog.CatalogTable; import org.apache.spark.sql.delta.coordinatedcommits.UCTokenBasedRestClientFactory$; /** * Factory for creating {@link DeltaSnapshotManager} instances. * *

This factory determines the appropriate snapshot manager based on the table configuration: * *

    *
  • For Unity Catalog managed tables: creates {@link UCManagedTableSnapshotManager} *
  • For path-based tables: creates {@link PathBasedSnapshotManager} *
*/ @Experimental public final class SnapshotManagerFactory { // Utility class - no instances private SnapshotManagerFactory() {} /** * Creates a snapshot manager for the given table. * * @param tablePath the filesystem path to the Delta table * @param kernelEngine the pre-configured Kernel {@link Engine} to use for table operations * @param catalogTable optional Spark catalog table metadata * @return a {@link DeltaSnapshotManager} appropriate for the table type */ public static DeltaSnapshotManager create( String tablePath, Engine kernelEngine, Optional catalogTable) { if (catalogTable.isPresent()) { Optional ucTableInfo = UCUtils.extractTableInfo(catalogTable.get(), SparkSession.active()); if (ucTableInfo.isPresent()) { return createUCManagedSnapshotManager(ucTableInfo.get(), kernelEngine); } // Catalog table without UC metadata falls back to path-based handling. } // Default: path-based snapshot manager for non-UC tables return new PathBasedSnapshotManager(tablePath, kernelEngine); } private static UCManagedTableSnapshotManager createUCManagedSnapshotManager( UCTableInfo tableInfo, Engine kernelEngine) { // Start from defaults (Delta, Spark, Scala, Java) and add connector-specific entries Map appVersions = UCTokenBasedRestClientFactory$.MODULE$.defaultAppVersionsAsJava(); appVersions.put("Kernel", Meta.KERNEL_VERSION); appVersions.put("Delta V2 connector", "true"); UCClient ucClient = UCTokenBasedRestClientFactory$.MODULE$.createUCClientWithVersions( tableInfo.getUcUri(), tableInfo.getAuthConfig(), appVersions); UCCatalogManagedClient ucCatalogClient = new UCCatalogManagedClient(ucClient); return new UCManagedTableSnapshotManager(ucCatalogClient, tableInfo, kernelEngine); } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/unitycatalog/UCManagedTableSnapshotManager.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.snapshot.unitycatalog; import static java.util.Objects.requireNonNull; import io.delta.kernel.CommitRange; import io.delta.kernel.Snapshot; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.DeltaHistoryManager; import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.files.ParsedCatalogCommitData; import io.delta.kernel.unitycatalog.UCCatalogManagedClient; import io.delta.spark.internal.v2.exception.VersionNotFoundException; import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; import java.util.List; import java.util.Optional; /** * Snapshot manager for Unity Catalog managed tables. * *

Used for tables with the catalog-managed commit feature enabled. Unity Catalog serves as the * source of truth for the table's commit history. */ public class UCManagedTableSnapshotManager implements DeltaSnapshotManager { private final UCCatalogManagedClient ucCatalogManagedClient; private final String tableId; private final String tablePath; private final Engine engine; /** * Creates a new UCManagedTableSnapshotManager. * * @param ucCatalogManagedClient the UC client for catalog-managed operations * @param tableInfo the UC table information (tableId, tablePath, etc.) * @param engine the Kernel engine for table operations */ public UCManagedTableSnapshotManager( UCCatalogManagedClient ucCatalogManagedClient, UCTableInfo tableInfo, Engine engine) { this.ucCatalogManagedClient = requireNonNull(ucCatalogManagedClient, "ucCatalogManagedClient is null"); requireNonNull(tableInfo, "tableInfo is null"); this.tableId = tableInfo.getTableId(); this.tablePath = tableInfo.getTablePath(); this.engine = requireNonNull(engine, "engine is null"); } /** * Loads and returns the latest snapshot of the UC-managed Delta table. * * @return the latest snapshot of the table */ @Override public Snapshot loadLatestSnapshot() { return ucCatalogManagedClient.loadSnapshot( engine, tableId, tablePath, Optional.empty() /* versionOpt */, Optional.empty() /* timestampOpt */); } @Override public Snapshot loadSnapshotAt(long version) { return ucCatalogManagedClient.loadSnapshot( engine, tableId, tablePath, Optional.of(version), Optional.empty() /* timestampOpt */); } /** * Finds the active commit at a specific timestamp. * *

For UC-managed tables, this loads the latest snapshot and uses {@link * DeltaHistoryManager#getActiveCommitAtTimestamp} to resolve the timestamp to a commit. * * @param timestampMillis the timestamp to find the version for in milliseconds since the unix * epoch * @param canReturnLastCommit whether we can return the latest version of the table if the * provided timestamp is after the latest commit * @param mustBeRecreatable whether the state at the returned commit should be recreatable * @param canReturnEarliestCommit whether we can return the earliest version of the table if the * provided timestamp is before the earliest commit * @return the commit that was active at the specified timestamp */ @Override public DeltaHistoryManager.Commit getActiveCommitAtTime( long timestampMillis, boolean canReturnLastCommit, boolean mustBeRecreatable, boolean canReturnEarliestCommit) { SnapshotImpl snapshot = (SnapshotImpl) loadLatestSnapshot(); List catalogCommits = snapshot.getLogSegment().getAllCatalogCommits(); return DeltaHistoryManager.getActiveCommitAtTimestamp( engine, snapshot, snapshot.getLogPath(), timestampMillis, mustBeRecreatable, canReturnLastCommit, canReturnEarliestCommit, catalogCommits); } /** * Checks if a specific version exists and is accessible. * *

For UC-managed tables with catalogManaged, log files may be cleaned up, so we need to use * DeltaHistoryManager to find the earliest available version based on filesystem state. * * @param version the version to check * @param mustBeRecreatable whether the state at this version should be recreatable * @param allowOutOfRange whether versions greater than the latest version are allowed without * throwing an exception * @throws VersionNotFoundException if the version is not available or does not meet the specified * criteria */ @Override public void checkVersionExists(long version, boolean mustBeRecreatable, boolean allowOutOfRange) throws VersionNotFoundException { // Load latest to get the current version bounds SnapshotImpl snapshot = (SnapshotImpl) loadLatestSnapshot(); // Latest version visible in this UC-managed snapshot. long latestSnapshotVersion = snapshot.getVersion(); // Fast path: check upper bound before expensive filesystem operations if ((version > latestSnapshotVersion) && !allowOutOfRange) { throw new VersionNotFoundException(version, 0 /* earliest */, latestSnapshotVersion); } // Get the earliest version among catalog commits. This bounds the Kernel's filesystem search // for the earliest available version (e.g., if catalog has v0, no filesystem search is needed). List catalogCommits = snapshot.getLogSegment().getAllCatalogCommits(); Optional earliestCatalogCommitVersion = catalogCommits.stream().map(ParsedCatalogCommitData::getVersion).min(Long::compare); long earliestVersion = mustBeRecreatable ? DeltaHistoryManager.getEarliestRecreatableCommit( engine, snapshot.getLogPath(), earliestCatalogCommitVersion) : DeltaHistoryManager.getEarliestDeltaFile( engine, snapshot.getLogPath(), earliestCatalogCommitVersion); if (version < earliestVersion) { throw new VersionNotFoundException(version, earliestVersion, latestSnapshotVersion); } } /** * Gets a range of table changes (commits) between start and end versions. * * @param engine the engine implementation for executing operations * @param startVersion the starting version (inclusive) * @param endVersion optional ending version (inclusive); if not provided, extends to latest * @return a CommitRange representing the specified range of commits */ @Override public CommitRange getTableChanges(Engine engine, long startVersion, Optional endVersion) { return ucCatalogManagedClient.loadCommitRange( engine, tableId, tablePath, Optional.of(startVersion) /* startVersionOpt */, Optional.empty() /* startTimestampOpt */, endVersion /* endVersionOpt */, Optional.empty() /* endTimestampOpt */); } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/unitycatalog/UCTableInfo.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.snapshot.unitycatalog; import static java.util.Objects.requireNonNull; import java.util.Collections; import java.util.Map; /** * Table information for Unity Catalog managed tables. * *

This POJO encapsulates all the information needed to interact with a Unity Catalog table * without requiring Spark dependencies. */ public final class UCTableInfo { private final String tableId; private final String tablePath; private final String ucUri; private final Map authConfig; public UCTableInfo( String tableId, String tablePath, String ucUri, Map authConfig) { this.tableId = requireNonNull(tableId, "tableId is null"); this.tablePath = requireNonNull(tablePath, "tablePath is null"); this.ucUri = requireNonNull(ucUri, "ucUri is null"); this.authConfig = Collections.unmodifiableMap(requireNonNull(authConfig, "authConfig is null")); } public String getTableId() { return tableId; } public String getTablePath() { return tablePath; } public String getUcUri() { return ucUri; } public Map getAuthConfig() { return authConfig; } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/snapshot/unitycatalog/UCUtils.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.snapshot.unitycatalog; import static java.util.Objects.requireNonNull; import static scala.jdk.javaapi.CollectionConverters.asJava; import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient; import java.util.Map; import java.util.Optional; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.catalyst.catalog.CatalogTable; import org.apache.spark.sql.delta.coordinatedcommits.UCCatalogConfig; import org.apache.spark.sql.delta.coordinatedcommits.UCCommitCoordinatorBuilder$; import org.apache.spark.sql.delta.util.CatalogTableUtils; /** * Utility class for extracting Unity Catalog table information from Spark catalog metadata. * *

This class isolates Spark dependencies, allowing {@link UCManagedTableSnapshotManager} to be * created without Spark if table info is provided directly via {@link UCTableInfo}. */ public final class UCUtils { // Utility class - no instances private UCUtils() {} /** * Extracts Unity Catalog table information from Spark catalog table metadata. * * @param catalogTable Spark catalog table metadata * @param spark SparkSession for resolving Unity Catalog configurations * @return table info if table is UC-managed, empty otherwise * @throws IllegalArgumentException if table is UC-managed but configuration is invalid */ public static Optional extractTableInfo( CatalogTable catalogTable, SparkSession spark) { requireNonNull(catalogTable, "catalogTable is null"); requireNonNull(spark, "spark is null"); if (!CatalogTableUtils.isUnityCatalogManagedTable(catalogTable)) { return Optional.empty(); } String tableId = extractUCTableId(catalogTable); String tablePath = extractTablePath(catalogTable); // Get catalog name - require explicit catalog in identifier scala.Option catalogOption = catalogTable.identifier().catalog(); if (catalogOption.isEmpty()) { throw new IllegalArgumentException( "Unable to determine Unity Catalog for table " + catalogTable.identifier() + ": catalog name is missing. Use a fully-qualified table name with an explicit " + "catalog (e.g., catalog.schema.table)."); } String catalogName = catalogOption.get(); // Get UC endpoint and token from Spark configs scala.collection.immutable.Map ucConfigs = UCCommitCoordinatorBuilder$.MODULE$.getCatalogConfigMap(spark); scala.Option configOpt = ucConfigs.get(catalogName); if (configOpt.isEmpty()) { throw new IllegalArgumentException( "Cannot create UC client for table " + catalogTable.identifier() + ": Unity Catalog configuration not found for catalog '" + catalogName + "'."); } UCCatalogConfig config = configOpt.get(); String ucUri = config.uri(); return Optional.of(new UCTableInfo(tableId, tablePath, ucUri, asJava(config.authConfig()))); } private static String extractUCTableId(CatalogTable catalogTable) { Map storageProperties = scala.jdk.javaapi.CollectionConverters.asJava(catalogTable.storage().properties()); // TODO: UC constants should be consolidated in a shared location (future PR) String ucTableId = storageProperties.get(UCCommitCoordinatorClient.UC_TABLE_ID_KEY); if (ucTableId == null || ucTableId.isEmpty()) { throw new IllegalArgumentException( "Cannot extract ucTableId from table " + catalogTable.identifier()); } return ucTableId; } private static String extractTablePath(CatalogTable catalogTable) { if (catalogTable.location() == null) { throw new IllegalArgumentException( "Cannot extract table path: location is null for table " + catalogTable.identifier()); } return catalogTable.location().toString(); } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/utils/CloseableIterator.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.utils; import java.io.Closeable; import java.io.IOException; import java.util.NoSuchElementException; import java.util.function.Function; import java.util.function.Predicate; import scala.collection.AbstractIterator; import scala.collection.Iterator; /** * A Scala iterator that implements {@link Closeable}. Unlike standard Scala iterators, the {@link * #filterCloseable} and {@link #mapCloseable} methods return a {@link CloseableIterator} that * properly delegates {@link #close()} to the underlying iterator. * *

This is important for iterators that hold resources (e.g., file handles from Parquet readers) * that must be released when iteration completes or is interrupted. * *

Inspired by Spark's {@code org.apache.spark.sql.util.CloseableIterator}, which is * package-private ({@code private[sql]}) and cannot be used directly. * * @param the type of elements returned by this iterator */ public abstract class CloseableIterator extends AbstractIterator implements Closeable { /** * Wraps a Scala iterator as a {@link CloseableIterator}. If the iterator already implements * {@link Closeable}, {@link #close()} will delegate to it; otherwise close is a no-op. */ public static CloseableIterator wrap(Iterator iterator) { if (iterator instanceof CloseableIterator) { return (CloseableIterator) iterator; } return new WrappedIterator<>(iterator); } /** * Returns a new {@link CloseableIterator} that applies the given function to each element. * Closing the returned iterator will close this iterator. */ public CloseableIterator mapCloseable(Function mapper) { CloseableIterator self = this; return new CloseableIterator() { @Override public boolean hasNext() { return self.hasNext(); } @Override public U next() { return mapper.apply(self.next()); } @Override public void close() throws IOException { self.close(); } }; } /** * Returns a new {@link CloseableIterator} that includes only elements matching the predicate. * Closing the returned iterator will close this iterator. */ public CloseableIterator filterCloseable(Predicate predicate) { CloseableIterator self = this; return new CloseableIterator() { private T nextElement; private boolean hasNextElement; @Override public boolean hasNext() { if (hasNextElement) { return true; } while (self.hasNext()) { T element = self.next(); if (predicate.test(element)) { nextElement = element; hasNextElement = true; return true; } } return false; } @Override public T next() { if (!hasNext()) { throw new NoSuchElementException(); } hasNextElement = false; return nextElement; } @Override public void close() throws IOException { self.close(); } }; } /** A wrapper that makes any Scala iterator closeable. */ private static class WrappedIterator extends CloseableIterator { private final Iterator delegate; WrappedIterator(Iterator delegate) { this.delegate = delegate; } @Override public boolean hasNext() { return delegate.hasNext(); } @Override public T next() { return delegate.next(); } @Override public void close() throws IOException { if (delegate instanceof Closeable) { ((Closeable) delegate).close(); } } } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.utils; import static org.apache.spark.sql.connector.catalog.CatalogV2Implicits.parseColumnPath; import com.google.common.annotations.VisibleForTesting; import io.delta.kernel.expressions.AlwaysFalse; import io.delta.kernel.expressions.And; import io.delta.kernel.expressions.Column; import io.delta.kernel.expressions.In; import io.delta.kernel.expressions.Literal; import io.delta.kernel.expressions.Or; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.internal.util.InternalUtils; import java.math.BigDecimal; import java.sql.Date; import java.sql.Timestamp; import java.util.*; import org.apache.spark.sql.catalyst.expressions.BoundReference; import org.apache.spark.sql.catalyst.expressions.Expression; import org.apache.spark.sql.connector.expressions.LiteralValue; import org.apache.spark.sql.connector.expressions.NamedReference; import org.apache.spark.sql.sources.*; import org.apache.spark.sql.types.DataType; import org.apache.spark.sql.types.DecimalType; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.apache.spark.unsafe.types.UTF8String; import scala.collection.JavaConverters; /** * Utility class for converting Spark SQL filter expressions to Delta Kernel predicates. * *

This class provides methods to convert Spark's {@link Filter} objects into Delta Kernel's * {@link Predicate} objects for push-down query optimization. * *

Note: Only expressions that can be safely converted are processed */ public final class ExpressionUtils { /** * Converts a Spark SQL filter to a Delta Kernel predicate. * *

Supported filter types: * *

    *
  • Comparison: EqualTo, GreaterThan, LessThan, etc. *
  • Null tests: IsNull, IsNotNull *
  • Null-safe comparison: EqualNullSafe *
  • String prefix: StringStartsWith *
  • Set membership: In *
  • Logical operators: And, Or, Not *
* * @param filter the Spark SQL filter to convert * @return ConvertedPredicate containing the converted Kernel predicate, or empty if conversion is * not supported, along with a boolean indicating whether the conversion was partial */ public static ConvertedPredicate convertSparkFilterToKernelPredicate(Filter filter) { return convertSparkFilterToKernelPredicate(filter, true /*canPartialPushDown*/, null); } /** * Converts a Spark SQL filter to a Delta Kernel predicate with table schema for type alignment. * *

When a table schema is provided, decimal literal types in comparison filters are widened to * match the column's declared decimal type. This prevents type mismatch errors in the Kernel's * expression evaluator during data skipping, where stats column types (from the table schema) * must match the filter literal types exactly. * * @param filter the Spark SQL filter to convert * @param tableSchema the table schema for looking up column types, may be null * @return ConvertedPredicate containing the converted Kernel predicate, or empty if conversion is * not supported, along with a boolean indicating whether the conversion was partial */ public static ConvertedPredicate convertSparkFilterToKernelPredicate( Filter filter, StructType tableSchema) { return convertSparkFilterToKernelPredicate(filter, true /*canPartialPushDown*/, tableSchema); } /** * Converts a Spark SQL filter to a Delta Kernel predicate with partial pushdown control. When * canPartialPushDown is true, AND filters can be partially converted if at least one operand can * be converted. OR filters always require both operands to be convertible. NOT filters disable * partial pushdown for their child to preserve semantic correctness. * *

Return a ConvertedPredicate object, which contains: - Optional: the converted * Kernel predicate, or empty if conversion is not supported - boolean isPartial: indicates * whether the conversion was partial */ @VisibleForTesting static ConvertedPredicate convertSparkFilterToKernelPredicate( Filter filter, boolean canPartialPushDown) { return convertSparkFilterToKernelPredicate(filter, canPartialPushDown, null); } /** * Core implementation of filter-to-predicate conversion with partial pushdown control and * optional table schema for decimal type alignment. * * @param filter the Spark SQL filter to convert * @param canPartialPushDown whether partial pushdown is allowed for AND filters * @param tableSchema the table schema for looking up column types, may be null */ private static ConvertedPredicate convertSparkFilterToKernelPredicate( Filter filter, boolean canPartialPushDown, StructType tableSchema) { if (filter instanceof EqualTo) { EqualTo f = (EqualTo) filter; return new ConvertedPredicate( convertComparisonLiteral(f.value(), f.attribute(), tableSchema) .map(l -> new Predicate("=", kernelColumn(f.attribute()), l))); } if (filter instanceof EqualNullSafe) { EqualNullSafe f = (EqualNullSafe) filter; // EqualNullSafe with null value should be translated to IS_NULL // For non-null values, we use "=" operator. return new ConvertedPredicate( f.value() == null ? Optional.of(new Predicate("IS_NULL", kernelColumn(f.attribute()))) : convertComparisonLiteral(f.value(), f.attribute(), tableSchema) .map(l -> new Predicate("=", kernelColumn(f.attribute()), l))); } if (filter instanceof GreaterThan) { GreaterThan f = (GreaterThan) filter; return new ConvertedPredicate( convertComparisonLiteral(f.value(), f.attribute(), tableSchema) .map(l -> new Predicate(">", kernelColumn(f.attribute()), l))); } if (filter instanceof GreaterThanOrEqual) { GreaterThanOrEqual f = (GreaterThanOrEqual) filter; return new ConvertedPredicate( convertComparisonLiteral(f.value(), f.attribute(), tableSchema) .map(l -> new Predicate(">=", kernelColumn(f.attribute()), l))); } if (filter instanceof LessThan) { LessThan f = (LessThan) filter; return new ConvertedPredicate( convertComparisonLiteral(f.value(), f.attribute(), tableSchema) .map(l -> new Predicate("<", kernelColumn(f.attribute()), l))); } if (filter instanceof LessThanOrEqual) { LessThanOrEqual f = (LessThanOrEqual) filter; return new ConvertedPredicate( convertComparisonLiteral(f.value(), f.attribute(), tableSchema) .map(l -> new Predicate("<=", kernelColumn(f.attribute()), l))); } if (filter instanceof IsNull) { IsNull f = (IsNull) filter; return new ConvertedPredicate( Optional.of(new Predicate("IS_NULL", kernelColumn(f.attribute())))); } if (filter instanceof IsNotNull) { IsNotNull f = (IsNotNull) filter; return new ConvertedPredicate( Optional.of(new Predicate("IS_NOT_NULL", kernelColumn(f.attribute())))); } if (filter instanceof StringStartsWith) { StringStartsWith f = (StringStartsWith) filter; return new ConvertedPredicate( convertValueToKernelLiteral(f.value()) .map(l -> new Predicate("STARTS_WITH", kernelColumn(f.attribute()), l))); } if (filter instanceof org.apache.spark.sql.sources.In) { org.apache.spark.sql.sources.In f = (org.apache.spark.sql.sources.In) filter; // An empty IN list can never match any row. Push ALWAYS_FALSE so the kernel skips // all files entirely, rather than scanning every file only to discard every row. if (f.values().length == 0) { return new ConvertedPredicate(Optional.of(AlwaysFalse.ALWAYS_FALSE)); } List literals = new ArrayList<>(); for (Object value : f.values()) { Optional lit = convertValueToKernelLiteral(value); if (!lit.isPresent()) { // A value that can't be converted (e.g. null, unsupported type) makes the whole // IN expression unsafe to push down; return empty to keep it for post-scan evaluation. return new ConvertedPredicate(Optional.empty()); } literals.add(lit.get()); } return new ConvertedPredicate(Optional.of(new In(kernelColumn(f.attribute()), literals))); } if (filter instanceof org.apache.spark.sql.sources.And) { org.apache.spark.sql.sources.And f = (org.apache.spark.sql.sources.And) filter; ConvertedPredicate left = convertSparkFilterToKernelPredicate(f.left(), canPartialPushDown, tableSchema); ConvertedPredicate right = convertSparkFilterToKernelPredicate(f.right(), canPartialPushDown, tableSchema); boolean isPartial = left.isPartial() || right.isPartial(); if (left.isPresent() && right.isPresent()) { return new ConvertedPredicate(Optional.of(new And(left.get(), right.get())), isPartial); } if (canPartialPushDown && left.isPresent()) { return new ConvertedPredicate(left.getConvertedPredicate(), true); } if (canPartialPushDown && right.isPresent()) { return new ConvertedPredicate(right.getConvertedPredicate(), true); } return new ConvertedPredicate(Optional.empty(), isPartial); } if (filter instanceof org.apache.spark.sql.sources.Or) { org.apache.spark.sql.sources.Or f = (org.apache.spark.sql.sources.Or) filter; ConvertedPredicate left = convertSparkFilterToKernelPredicate(f.left(), canPartialPushDown, tableSchema); ConvertedPredicate right = convertSparkFilterToKernelPredicate(f.right(), canPartialPushDown, tableSchema); // OR requires both operands to be convertible for correctness boolean isPartial = left.isPartial() || right.isPartial(); if (!left.isPresent() || !right.isPresent()) { return new ConvertedPredicate(Optional.empty(), isPartial); } return new ConvertedPredicate(Optional.of(new Or(left.get(), right.get())), isPartial); } if (filter instanceof Not) { Not f = (Not) filter; // NOT disables partial pushdown for semantic correctness. // Example: Pushing down NOT(A AND B) requires both A and B to be convertible. // We cannot convert it to just return NOT A if only A is convertible when B is not, // because: // // Original: NOT(age < 30 AND name = "John") // // Row 1: age=25, name="John" // Row 2: age=25, name="Mike" // (age < 30 AND name = "John") = (true AND true) = true // (age < 30 AND name = "Mike") = (true AND false) = false // NOT(true) = false → row 1 should be EXCLUDED // NOT(false) = true → row 2 should be INCLUDED // But if we naively push down just NOT(age < 30): // // NOT(age < 30) = NOT(true) = false → system excludes both row // We will return incorrect result, then. ConvertedPredicate child = convertSparkFilterToKernelPredicate(f.child(), false /*canPartialPushDown*/, tableSchema); return new ConvertedPredicate( child.getConvertedPredicate().map(c -> new Predicate("NOT", c)), child.isPartial()); } return new ConvertedPredicate(Optional.empty()); } /** * Converts a filter literal value to a Kernel Literal, aligning decimal types with the column's * declared type from the table schema when available. * *

When the value is a {@link java.math.BigDecimal} and the table schema is provided, this * method looks up the column's declared {@link org.apache.spark.sql.types.DecimalType} and widens * the literal to match. This prevents type mismatch errors in the Kernel's expression evaluator * during data skipping, where stats column types must match the filter literal types exactly. * *

If the literal's scale exceeds the column's scale, or the widened value exceeds the column's * precision, the filter cannot be safely pushed down and an empty Optional is returned. * * @param value the filter literal value * @param attribute the column name referenced by the filter * @param tableSchema the table schema for type lookup, may be null * @return Optional containing the Kernel Literal with aligned type, or empty if conversion fails */ private static Optional convertComparisonLiteral( Object value, String attribute, StructType tableSchema) { if (value instanceof BigDecimal && tableSchema != null) { BigDecimal bd = (BigDecimal) value; Optional columnType = lookupColumnType(attribute, tableSchema); if (columnType.isPresent() && columnType.get() instanceof DecimalType) { DecimalType colDecimalType = (DecimalType) columnType.get(); return widenDecimalLiteral(bd, colDecimalType.precision(), colDecimalType.scale()); } } return convertValueToKernelLiteral(value); } /** * Widens a BigDecimal literal to match the target decimal precision and scale. Returns empty if * the literal cannot be safely represented in the target type (e.g., the literal has more decimal * digits than the target scale, or the widened value exceeds the target precision). */ private static Optional widenDecimalLiteral( BigDecimal bd, int targetPrecision, int targetScale) { if (bd.scale() <= targetScale) { BigDecimal widened = bd.setScale(targetScale); if (widened.precision() <= targetPrecision) { return Optional.of(Literal.ofDecimal(widened, targetPrecision, targetScale)); } } // Literal doesn't fit in column type or has higher scale - skip pushdown return Optional.empty(); } /** * Looks up the data type of a top-level column in the table schema using case-insensitive name * matching. Returns empty for nested columns or if the column is not found. */ private static Optional lookupColumnType(String attribute, StructType tableSchema) { for (StructField field : tableSchema.fields()) { if (field.name().equalsIgnoreCase(attribute)) { return Optional.of(field.dataType()); } } return Optional.empty(); } /** * Creates a Delta Kernel Column from a Spark SQL column attribute name. * *

This method handles nested column references (e.g., "user.profile.name") by parsing the * dot-separated path into an array of field names using Spark's column path parser. * *

If a column name contains literal dots that should not be treated as field separators, it * must be properly quoted/escaped in the original Spark SQL. For example: * *

    *
  • {@code `my.column.with.dots`} - treats the entire string as a single column name *
  • {@code my.nested.field} - treats this as nested field access: my -> nested -> field *
* * @param attribute the column attribute name, potentially dot-separated for nested fields * @return Delta Kernel Column object representing the parsed column path */ private static Column kernelColumn(String attribute) { scala.collection.Seq seq = parseColumnPath(attribute); String[] parts = JavaConverters.seqAsJavaList(seq).toArray(new String[0]); return new Column(parts); } /** * Converts a Java object to a Delta Kernel Literal with appropriate type inference. * *

This method handles the most common Java types and converts them to their corresponding * Delta Kernel Literal representations. The type mapping follows standard SQL data type * conventions. * *

Supported types: * *

    *
  • Primitives: Boolean, Byte, Short, Integer, Long, Float, Double *
  • BigDecimal (with precision and scale preservation) *
  • String (for string literals from Spark V1 filters) *
  • byte[] (binary data) *
  • java.sql.Date (converted to days since epoch) *
  • java.sql.Timestamp (converted to microseconds since epoch) *
* *

Note: null values return empty Optional, which is correct SQL behavior for most operations. * Only EqualNullSafe should handle null values explicitly. * * @param value the Java object to convert * @return Optional containing the Delta Kernel Literal, or empty if the value is null or of an * unsupported type */ @VisibleForTesting static Optional convertValueToKernelLiteral(Object value) { // TODO: convert null to NULL literal. if (value == null) return Optional.empty(); if (value instanceof Boolean) { Boolean b = (Boolean) value; return Optional.of(Literal.ofBoolean(b)); } if (value instanceof Byte) { Byte b = (Byte) value; return Optional.of(Literal.ofByte(b)); } if (value instanceof Short) { Short s = (Short) value; return Optional.of(Literal.ofShort(s)); } if (value instanceof Integer) { Integer i = (Integer) value; return Optional.of(Literal.ofInt(i)); } if (value instanceof Long) { Long l = (Long) value; return Optional.of(Literal.ofLong(l)); } if (value instanceof Float) { Float f = (Float) value; return Optional.of(Literal.ofFloat(f)); } if (value instanceof Double) { Double d = (Double) value; return Optional.of(Literal.ofDouble(d)); } if (value instanceof BigDecimal) { // Preserve precision and scale from the original BigDecimal BigDecimal bd = (BigDecimal) value; return Optional.of(Literal.ofDecimal(bd, bd.precision(), bd.scale())); } if (value instanceof UTF8String) { UTF8String s = (UTF8String) value; return Optional.of(Literal.ofString(s.toString())); } if (value instanceof String) { String s = (String) value; return Optional.of(Literal.ofString(s)); } if (value instanceof byte[]) { byte[] arr = (byte[]) value; return Optional.of(Literal.ofBinary(arr)); } if (value instanceof Date) { // Convert java.sql.Date to days since epoch Date date = (Date) value; return Optional.of(Literal.ofDate(InternalUtils.daysSinceEpoch(date))); } if (value instanceof Timestamp) { // Convert java.sql.Timestamp to microseconds since epoch Timestamp timestamp = (Timestamp) value; return Optional.of(Literal.ofTimestamp(InternalUtils.microsSinceEpoch(timestamp))); } // Unsupported type - return empty Optional to skip the conversion. return Optional.empty(); } /* * Wrapper class to hold the result of converting a Spark Filter to a Kernel Predicate, * including a boolean indicator for whether the conversion was partial. */ public static final class ConvertedPredicate { private final Optional convertedPredicate; private final boolean isPartial; public ConvertedPredicate(Optional convertedPredicate) { this.convertedPredicate = convertedPredicate; this.isPartial = false; } public ConvertedPredicate(Optional convertedPredicate, boolean isPartial) { this.convertedPredicate = convertedPredicate; this.isPartial = isPartial; } public Optional getConvertedPredicate() { return convertedPredicate; } public boolean isPartial() { return isPartial; } public boolean isPresent() { return convertedPredicate.isPresent(); } public Predicate get() { assert convertedPredicate.isPresent(); return convertedPredicate.get(); } } /* * Helper class to hold the classification result of a Filter */ public static class FilterClassificationResult { public final Boolean isKernelSupported; public final Boolean isPartialConversion; public final Boolean isDataFilter; public final Optional kernelPredicate; public FilterClassificationResult( Boolean isKernelSupported, Boolean isPartialConversion, Boolean isDataFilter, Optional kernelPredicate) { this.isKernelSupported = isKernelSupported; this.isPartialConversion = isPartialConversion; this.isDataFilter = isDataFilter; this.kernelPredicate = kernelPredicate; } } /** * Classifies a Spark Filter based on its convertibility to a Kernel Predicate and whether it is a * data filter (i.e., references non-partition columns). * * @param filter the Spark Filter to classify * @param partitionColumnSet a set of partition column names (in lower case) for identifying data * filters * @return FilterClassificationResult containing: *

    *
  • isKernelSupported: true if the filter can be converted to a Kernel Predicate *
  • isPartialConversion: true if the conversion was partial (for AND filters) *
  • isDataFilter: true if the filter references at least one non-partition column *
  • kernelPredicate: Optional containing the converted Kernel Predicate, if any *
*/ public static FilterClassificationResult classifyFilter( Filter filter, Set partitionColumnSet) { return classifyFilter(filter, partitionColumnSet, null); } /** * Classifies a Spark Filter with table schema for decimal type alignment. * * @param filter the Spark Filter to classify * @param partitionColumnSet a set of partition column names (in lower case) for identifying data * filters * @param tableSchema the table schema for aligning decimal literal types, may be null * @return FilterClassificationResult containing classification details */ public static FilterClassificationResult classifyFilter( Filter filter, Set partitionColumnSet, StructType tableSchema) { // try to convert Spark filter to Kernel Predicate ConvertedPredicate convertedPredicate = ExpressionUtils.convertSparkFilterToKernelPredicate(filter, tableSchema); boolean isKernelSupported = convertedPredicate.isPresent(); boolean isPartialConversion = convertedPredicate.isPartial(); Optional kernelPredicate = convertedPredicate.getConvertedPredicate(); // check if the filter is a data filter // A data filter is a filter that references at least one non-partition column. String[] refs = filter.references(); boolean isDataFilter = refs != null && refs.length > 0 && Arrays.stream(refs) .anyMatch((col -> !partitionColumnSet.contains(col.toLowerCase(Locale.ROOT)))); return new FilterClassificationResult( isKernelSupported, isPartialConversion, isDataFilter, kernelPredicate); } /** * Converts a Spark DataSourceV2 Predicate to a Catalyst Expression. * *

This method translates supported DSV2 predicates into their equivalent Catalyst expressions, * using the provided schema for column resolution. Unsupported predicates, or those referencing * unknown columns, will result in an empty Optional. * *

Supported predicates include: * *

    *
  • Null tests: IS_NULL, IS_NOT_NULL *
  • String functions: STARTS_WITH, ENDS_WITH, CONTAINS *
  • IN operator *
  • Comparison: =, >, >=, <, <= *
  • Null-safe comparison: <=> *
  • Logical operators: AND, OR, NOT *
  • Constant predicates: ALWAYS_TRUE, ALWAYS_FALSE *
* * @param predicate the DSV2 Predicate to convert * @param schema the schema used for resolving column references * @return Catalyst Expression representing the converted predicate, or empty if the predicate is * unsupported or references unknown columns */ public static Optional dsv2PredicateToCatalystExpression( org.apache.spark.sql.connector.expressions.filter.Predicate predicate, StructType schema) { String predicateName = predicate.name(); org.apache.spark.sql.connector.expressions.Expression[] children = predicate.children(); switch (predicateName) { case "IS_NULL": if (children.length == 1) { Optional expressionOpt = dsv2ExpressionToCatalystExpression(children[0], schema); if (expressionOpt.isPresent()) { return Optional.of( new org.apache.spark.sql.catalyst.expressions.IsNull(expressionOpt.get())); } } break; case "IS_NOT_NULL": if (children.length == 1) { Optional expressionOpt = dsv2ExpressionToCatalystExpression(children[0], schema); if (expressionOpt.isPresent()) { return Optional.of( new org.apache.spark.sql.catalyst.expressions.IsNotNull(expressionOpt.get())); } } break; case "STARTS_WITH": if (children.length == 2) { Optional leftOpt = dsv2ExpressionToCatalystExpression(children[0], schema); Optional rightOpt = dsv2ExpressionToCatalystExpression(children[1], schema); if (leftOpt.isPresent() && rightOpt.isPresent()) { return Optional.of( new org.apache.spark.sql.catalyst.expressions.StartsWith( leftOpt.get(), rightOpt.get())); } } break; case "ENDS_WITH": if (children.length == 2) { Optional leftOpt = dsv2ExpressionToCatalystExpression(children[0], schema); Optional rightOpt = dsv2ExpressionToCatalystExpression(children[1], schema); if (leftOpt.isPresent() && rightOpt.isPresent()) { return Optional.of( new org.apache.spark.sql.catalyst.expressions.EndsWith( leftOpt.get(), rightOpt.get())); } } break; case "CONTAINS": if (children.length == 2) { Optional leftOpt = dsv2ExpressionToCatalystExpression(children[0], schema); Optional rightOpt = dsv2ExpressionToCatalystExpression(children[1], schema); if (leftOpt.isPresent() && rightOpt.isPresent()) { return Optional.of( new org.apache.spark.sql.catalyst.expressions.Contains( leftOpt.get(), rightOpt.get())); } } break; case "IN": if (children.length >= 2) { Optional firstOpt = dsv2ExpressionToCatalystExpression(children[0], schema); if (firstOpt.isPresent()) { List values = new ArrayList<>(); for (int i = 1; i < children.length; i++) { Optional valueOpt = dsv2ExpressionToCatalystExpression(children[i], schema); if (valueOpt.isPresent()) { values.add(valueOpt.get()); } else { // if any value in the IN list cannot be converted, return empty return Optional.empty(); } } return Optional.of( new org.apache.spark.sql.catalyst.expressions.In( firstOpt.get(), JavaConverters.asScalaBuffer(values).toSeq())); } } break; case "=": if (children.length == 2) { Optional leftOpt = dsv2ExpressionToCatalystExpression(children[0], schema); Optional rightOpt = dsv2ExpressionToCatalystExpression(children[1], schema); if (leftOpt.isPresent() && rightOpt.isPresent()) { return Optional.of( new org.apache.spark.sql.catalyst.expressions.EqualTo( leftOpt.get(), rightOpt.get())); } } break; case "<>": if (children.length == 2) { Optional leftOpt = dsv2ExpressionToCatalystExpression(children[0], schema); Optional rightOpt = dsv2ExpressionToCatalystExpression(children[1], schema); if (leftOpt.isPresent() && rightOpt.isPresent()) { return Optional.of( new org.apache.spark.sql.catalyst.expressions.Not( new org.apache.spark.sql.catalyst.expressions.EqualTo( leftOpt.get(), rightOpt.get()))); } } break; case "<=>": if (children.length == 2) { Optional leftOpt = dsv2ExpressionToCatalystExpression(children[0], schema); Optional rightOpt = dsv2ExpressionToCatalystExpression(children[1], schema); if (leftOpt.isPresent() && rightOpt.isPresent()) { return Optional.of( new org.apache.spark.sql.catalyst.expressions.EqualNullSafe( leftOpt.get(), rightOpt.get())); } } break; case "<": if (children.length == 2) { Optional leftOpt = dsv2ExpressionToCatalystExpression(children[0], schema); Optional rightOpt = dsv2ExpressionToCatalystExpression(children[1], schema); if (leftOpt.isPresent() && rightOpt.isPresent()) { return Optional.of( new org.apache.spark.sql.catalyst.expressions.LessThan( leftOpt.get(), rightOpt.get())); } } break; case "<=": if (children.length == 2) { Optional leftOpt = dsv2ExpressionToCatalystExpression(children[0], schema); Optional rightOpt = dsv2ExpressionToCatalystExpression(children[1], schema); if (leftOpt.isPresent() && rightOpt.isPresent()) { return Optional.of( new org.apache.spark.sql.catalyst.expressions.LessThanOrEqual( leftOpt.get(), rightOpt.get())); } } break; case ">": if (children.length == 2) { Optional leftOpt = dsv2ExpressionToCatalystExpression(children[0], schema); Optional rightOpt = dsv2ExpressionToCatalystExpression(children[1], schema); if (leftOpt.isPresent() && rightOpt.isPresent()) { return Optional.of( new org.apache.spark.sql.catalyst.expressions.GreaterThan( leftOpt.get(), rightOpt.get())); } } break; case ">=": if (children.length == 2) { Optional leftOpt = dsv2ExpressionToCatalystExpression(children[0], schema); Optional rightOpt = dsv2ExpressionToCatalystExpression(children[1], schema); if (leftOpt.isPresent() && rightOpt.isPresent()) { return Optional.of( new org.apache.spark.sql.catalyst.expressions.GreaterThanOrEqual( leftOpt.get(), rightOpt.get())); } } break; case "AND": if (children.length == 2) { Optional leftOpt = dsv2PredicateToCatalystExpression( (org.apache.spark.sql.connector.expressions.filter.Predicate) predicate.children()[0], schema); Optional rightOpt = dsv2PredicateToCatalystExpression( (org.apache.spark.sql.connector.expressions.filter.Predicate) predicate.children()[1], schema); if (leftOpt.isPresent() && rightOpt.isPresent()) { return Optional.of( new org.apache.spark.sql.catalyst.expressions.And(leftOpt.get(), rightOpt.get())); } } break; case "OR": if (children.length == 2) { Optional leftOpt = dsv2PredicateToCatalystExpression( (org.apache.spark.sql.connector.expressions.filter.Predicate) predicate.children()[0], schema); Optional rightOpt = dsv2PredicateToCatalystExpression( (org.apache.spark.sql.connector.expressions.filter.Predicate) predicate.children()[1], schema); if (leftOpt.isPresent() && rightOpt.isPresent()) { return Optional.of( new org.apache.spark.sql.catalyst.expressions.Or(leftOpt.get(), rightOpt.get())); } } break; case "NOT": if (children.length == 1) { Optional childOpt = dsv2PredicateToCatalystExpression( (org.apache.spark.sql.connector.expressions.filter.Predicate) predicate.children()[0], schema); if (childOpt.isPresent()) { return Optional.of(new org.apache.spark.sql.catalyst.expressions.Not(childOpt.get())); } } break; case "ALWAYS_TRUE": if (children.length == 0) { return Optional.of( org.apache.spark.sql.catalyst.expressions.Literal.create( true, org.apache.spark.sql.types.DataTypes.BooleanType)); } break; case "ALWAYS_FALSE": if (children.length == 0) { return Optional.of( org.apache.spark.sql.catalyst.expressions.Literal.create( false, org.apache.spark.sql.types.DataTypes.BooleanType)); } break; } return Optional.empty(); } /** * Resolves a DSV2 Expression to a Catalyst Expression using the provided schema. * *

This method handles NamedReference and LiteralValue expressions. NamedReferences are * resolved to BoundReferences based on the schema, while LiteralValues are converted to Catalyst * Literals. Unsupported expression types or references to unknown columns will result in an empty * Optional. * * @param expr the DSV2 Expression to resolve * @param schema the schema used for resolving column references * @return Catalyst Expression representing the resolved expression, or empty if the expression is * unsupported or references unknown columns */ private static Optional dsv2ExpressionToCatalystExpression( org.apache.spark.sql.connector.expressions.Expression expr, StructType schema) { if (expr instanceof NamedReference) { NamedReference ref = (NamedReference) expr; String columnName = ref.fieldNames()[0]; try { int index = schema.fieldIndex(columnName); StructField field = schema.fields()[index]; return Optional.of(new BoundReference(index, field.dataType(), field.nullable())); } catch (IllegalArgumentException e) { // schema.fieldIndex(columnName) throws IllegalArgumentException if a field with the given // name does not exist return Optional.empty(); } } else if (expr instanceof LiteralValue) { LiteralValue literal = (LiteralValue) expr; return Optional.of( org.apache.spark.sql.catalyst.expressions.Literal.create( literal.value(), literal.dataType())); } else { return Optional.empty(); } } private ExpressionUtils() {} } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/utils/PartitionUtils.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.utils; import io.delta.kernel.Snapshot; import io.delta.kernel.data.MapValue; import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.actions.AddFile; import io.delta.kernel.internal.actions.DeletionVectorDescriptor; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.spark.internal.v2.read.DeltaParquetFileFormatV2; import io.delta.spark.internal.v2.read.SparkReaderFactory; import io.delta.spark.internal.v2.read.deletionvector.DeletionVectorReadFunction; import io.delta.spark.internal.v2.read.deletionvector.DeletionVectorSchemaContext; import java.time.ZoneId; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Optional; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.spark.paths.SparkPath; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.connector.read.PartitionReaderFactory; import org.apache.spark.sql.delta.DeltaColumnMapping; import org.apache.spark.sql.delta.DeltaParquetFileFormat; import org.apache.spark.sql.delta.RowIndexFilterType; import org.apache.spark.sql.execution.datasources.FileFormat$; import org.apache.spark.sql.execution.datasources.PartitionedFile; import org.apache.spark.sql.execution.datasources.PartitioningUtils; import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils; import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.sources.Filter; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import scala.Function1; import scala.Option; import scala.Tuple2; import scala.collection.Iterator; import scala.jdk.javaapi.CollectionConverters; /** Utility class for partition-related operations shared across Delta Kernel Spark components. */ public class PartitionUtils { private PartitionUtils() {} /** * Returns whether the given snapshot's table supports deletion vectors. A table supports DVs when * its protocol includes the {@link TableFeatures#DELETION_VECTORS_RW_FEATURE} and the table * format is Parquet. */ public static boolean tableSupportsDeletionVectors(Snapshot snapshot) { SnapshotImpl snapshotImpl = (SnapshotImpl) snapshot; Protocol protocol = snapshotImpl.getProtocol(); Metadata metadata = snapshotImpl.getMetadata(); return protocol.supportsFeature(TableFeatures.DELETION_VECTORS_RW_FEATURE) && "parquet".equalsIgnoreCase(metadata.getFormat().getProvider()); } /** * Calculate the maximum split bytes for file partitioning, considering total bytes and file * count. This is used for optimal file splitting in both batch and streaming read. */ public static long calculateMaxSplitBytes( SparkSession sparkSession, long totalBytes, int fileCount, SQLConf sqlConf) { long defaultMaxSplitBytes = sqlConf.filesMaxPartitionBytes(); long openCostInBytes = sqlConf.filesOpenCostInBytes(); Option minPartitionNumOption = sqlConf.filesMinPartitionNum(); int minPartitionNum = minPartitionNumOption.isDefined() ? ((Number) minPartitionNumOption.get()).intValue() : sqlConf .getConf(SQLConf.LEAF_NODE_DEFAULT_PARALLELISM()) .getOrElse(() -> sparkSession.sparkContext().defaultParallelism()); if (minPartitionNum <= 0) { minPartitionNum = 1; } long calculatedTotalBytes = totalBytes + (long) fileCount * openCostInBytes; long bytesPerCore = calculatedTotalBytes / minPartitionNum; return Math.min(defaultMaxSplitBytes, Math.max(openCostInBytes, bytesPerCore)); } /** * Build the partition {@link InternalRow} from kernel partition values by casting them to the * desired Spark types using the session time zone for temporal types. * *

Note: Partition values in AddFile use physical column names as keys when column mapping is * enabled. This method uses DeltaColumnMapping.getPhysicalName to map from logical schema fields * to physical partition value keys. * * @implNote The returned {@link InternalRow} is a {@code GenericInternalRow} (via {@code * InternalRow.fromSeq}), which has value-based {@code equals()}/{@code hashCode()}. Callers * such as {@code SparkBatch.planPartitionedInputPartitions} rely on this for grouping files * by partition key. Changing the return type to a different InternalRow subtype (e.g. {@code * UnsafeRow}) may break that contract. */ public static InternalRow getPartitionRow( MapValue partitionValues, StructType partitionSchema, ZoneId zoneId) { final int numPartCols = partitionSchema.fields().length; assert partitionValues.getSize() == numPartCols : String.format( java.util.Locale.ROOT, "Partition values size from add file %d != partition columns size %d", partitionValues.getSize(), numPartCols); final Object[] values = new Object[numPartCols]; // Build physical name -> index map once // Partition values use physical names as keys when column mapping is enabled final Map physicalNameToIndex = new HashMap<>(numPartCols); for (int i = 0; i < numPartCols; i++) { StructField field = partitionSchema.fields()[i]; String physicalName = DeltaColumnMapping.getPhysicalName(field); physicalNameToIndex.put(physicalName, i); values[i] = null; } // Fill values in a single pass over partitionValues for (int idx = 0; idx < partitionValues.getSize(); idx++) { final String key = partitionValues.getKeys().getString(idx); final String strVal = partitionValues.getValues().getString(idx); final Integer pos = physicalNameToIndex.get(key); if (pos != null) { final StructField field = partitionSchema.fields()[pos]; values[pos] = (strVal == null) ? null : PartitioningUtils.castPartValueToDesiredType(field.dataType(), strVal, zoneId); } } return InternalRow.fromSeq( CollectionConverters.asScala(Arrays.asList(values).iterator()).toSeq()); } /** * Build a PartitionedFile from an AddFile with the given partition schema and table path. * * @param addFile The AddFile to convert * @param partitionSchema The partition schema for parsing partition values * @param tablePath The table path * @param zoneId The timezone for temporal partition values * @return A PartitionedFile ready for Spark execution */ public static PartitionedFile buildPartitionedFile( AddFile addFile, StructType partitionSchema, String tablePath, ZoneId zoneId) { InternalRow partitionRow = getPartitionRow(addFile.getPartitionValues(), partitionSchema, zoneId); // Preferred node locations are not used. String[] preferredLocations = new String[0]; // Build metadata map with DV info if present scala.collection.immutable.Map otherConstantMetadataColumnValues = buildDvMetadata(addFile.getDeletionVector()); return new PartitionedFile( partitionRow, SparkPath.fromUrlString(new Path(tablePath, addFile.getPath()).toString()), /* start= */ 0L, /* length= */ addFile.getSize(), preferredLocations, addFile.getModificationTime(), /* fileSize= */ addFile.getSize(), otherConstantMetadataColumnValues); } /** * Create a PartitionReaderFactory for reading Parquet files with Delta-specific features. * *

Uses DeltaParquetFileFormatV2 which supports column mapping, deletion vectors, and other * Delta features through the ProtocolMetadataAdapterV2. * *

For tables with deletion vectors enabled, this method: * *

    *
  1. Adds __delta_internal_is_row_deleted column to read schema *
  2. Creates a reader that generates the is_row_deleted column using DV bitmap *
  3. Wraps the reader to filter out deleted rows and remove internal columns *
* * @param snapshot The Delta table snapshot containing protocol, metadata, and table path */ public static PartitionReaderFactory createDeltaParquetReaderFactory( Snapshot snapshot, StructType dataSchema, StructType partitionSchema, StructType readDataSchema, Filter[] dataFilters, scala.collection.immutable.Map scalaOptions, Configuration hadoopConf, SQLConf sqlConf) { SnapshotImpl snapshotImpl = (SnapshotImpl) snapshot; Protocol protocol = snapshotImpl.getProtocol(); Metadata metadata = snapshotImpl.getMetadata(); // Use Path.toString() instead of toUri().toString() to avoid URL encoding issues. // toUri().toString() encodes special characters (e.g., space -> %20), which causes // DV file path resolution failures. String tablePath = snapshotImpl.getDataPath().toString(); // Create DV schema context if table supports deletion vectors Optional dvSchemaContext = tableSupportsDeletionVectors(snapshot) ? Optional.of(new DeletionVectorSchemaContext(readDataSchema, partitionSchema)) : Optional.empty(); StructType finalReadDataSchema = dvSchemaContext .map(DeletionVectorSchemaContext::getSchemaWithDvColumn) .orElse(readDataSchema); boolean enableVectorizedReader = ParquetUtils.isBatchReadSupportedForSchema(sqlConf, finalReadDataSchema); scala.collection.immutable.Map optionsWithVectorizedReading = scalaOptions.$plus( new Tuple2<>( FileFormat$.MODULE$.OPTION_RETURNING_BATCH(), String.valueOf(enableVectorizedReader))); // TODO(https://github.com/delta-io/delta/issues/5859): Enable file splitting for DV tables boolean optimizationsEnabled = !dvSchemaContext.isPresent(); // TODO(https://github.com/delta-io/delta/issues/5859): Support _metadata.row_index for DV Option useMetadataRowIndex = dvSchemaContext.isPresent() ? Option.apply(Boolean.FALSE) : Option.empty(); DeltaParquetFileFormatV2 deltaFormat = new DeltaParquetFileFormatV2( protocol, metadata, /* nullableRowTrackingConstantFields */ false, /* nullableRowTrackingGeneratedFields */ false, optimizationsEnabled, Option.apply(tablePath), /* isCDCRead */ false, /* useMetadataRowIndexOpt */ useMetadataRowIndex); Function1> readFunc = deltaFormat.buildReaderWithPartitionValues( SparkSession.active(), dataSchema, partitionSchema, finalReadDataSchema, CollectionConverters.asScala(Arrays.asList(dataFilters)).toSeq(), optionsWithVectorizedReading, hadoopConf); // Wrap reader to filter deleted rows and remove internal columns if DV is enabled. if (dvSchemaContext.isPresent()) { readFunc = DeletionVectorReadFunction.wrap(readFunc, dvSchemaContext.get(), enableVectorizedReader); } return new SparkReaderFactory(readFunc, enableVectorizedReader); } /** * Build metadata map for PartitionedFile containing DV descriptor if present. * *

The metadata is used by DeltaParquetFileFormat to generate the is_row_deleted column. */ private static scala.collection.immutable.Map buildDvMetadata( Optional dvOpt) { Map metadata = new HashMap<>(); if (dvOpt.isPresent()) { metadata.put( DeltaParquetFileFormat.FILE_ROW_INDEX_FILTER_ID_ENCODED(), dvOpt.get().serializeToBase64()); metadata.put( DeltaParquetFileFormat.FILE_ROW_INDEX_FILTER_TYPE(), RowIndexFilterType.IF_CONTAINED); } return scala.collection.immutable.Map$.MODULE$.from(CollectionConverters.asScala(metadata)); } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/utils/RowTrackingUtils.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.utils; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.rowtracking.MaterializedRowTrackingColumn; import io.delta.kernel.internal.rowtracking.RowTracking; import java.util.ArrayList; import java.util.List; import org.apache.spark.sql.catalyst.expressions.FileSourceConstantMetadataStructField; import org.apache.spark.sql.catalyst.expressions.FileSourceGeneratedMetadataStructField; import org.apache.spark.sql.delta.DeltaIllegalStateException; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.MetadataBuilder; import org.apache.spark.sql.types.StructField; /** * Utility methods for row tracking in Kernel based connector. This class provides row tracking * functionality with Spark-specific metadata attributes for marking metadata columns as constant or * generated fields. */ public class RowTrackingUtils { // Field names for row tracking metadata columns (matching Spark V1 definitions) private static final String BASE_ROW_ID = "base_row_id"; private static final String ROW_ID = "row_id"; private static final String DEFAULT_ROW_COMMIT_VERSION = "default_row_commit_version"; private static final String ROW_COMMIT_VERSION = "row_commit_version"; // Metadata keys for row tracking metadata fields private static final String BASE_ROW_ID_METADATA_COL_ATTR_KEY = "__base_row_id_metadata_col"; private static final String DEFAULT_ROW_COMMIT_VERSION_METADATA_COL_ATTR_KEY = "__default_row_version_metadata_col"; private static final String ROW_ID_METADATA_COL_ATTR_KEY = "__row_id_metadata_col"; private static final String ROW_COMMIT_VERSION_METADATA_COL_ATTR_KEY = "__row_commit_version_metadata_col"; private RowTrackingUtils() {} /** * Create the row tracking metadata struct fields for reading. * *

The order and presence of fields matches Spark V1 implementation: * *

    *
  • row_id (generated field, always present when row tracking is enabled) *
  • base_row_id (constant field, always present when row tracking is enabled) *
  • default_row_commit_version (constant field, always present when row tracking is enabled) *
  • row_commit_version (generated field, always present when row tracking is enabled) *
* * @param protocol the protocol * @param metadata the metadata * @param nullableConstantFields whether constant fields should be nullable * @param nullableGeneratedFields whether generated fields should be nullable * @return list of struct fields for row tracking metadata, or empty list if row tracking is not * enabled * @throws DeltaIllegalStateException if row tracking is enabled but materialized column names are * missing */ public static List createMetadataStructFields( Protocol protocol, Metadata metadata, boolean nullableConstantFields, boolean nullableGeneratedFields) { if (!RowTracking.isEnabled(protocol, metadata)) { return new ArrayList<>(); } List fields = new ArrayList<>(); // Add row_id (generated field) - will throw if materialized column name is not configured String rowIdPhysicalName = getPhysicalColumnNameOrThrow(MaterializedRowTrackingColumn.MATERIALIZED_ROW_ID, metadata); fields.add( new StructField( ROW_ID, DataTypes.LongType, nullableGeneratedFields, createGeneratedFieldMetadata(ROW_ID, rowIdPhysicalName, ROW_ID_METADATA_COL_ATTR_KEY))); // Add base_row_id (constant field) fields.add( new StructField( BASE_ROW_ID, DataTypes.LongType, nullableConstantFields, createConstantFieldMetadata(BASE_ROW_ID, BASE_ROW_ID_METADATA_COL_ATTR_KEY))); // Add default_row_commit_version (constant field) fields.add( new StructField( DEFAULT_ROW_COMMIT_VERSION, DataTypes.LongType, nullableConstantFields, createConstantFieldMetadata( DEFAULT_ROW_COMMIT_VERSION, DEFAULT_ROW_COMMIT_VERSION_METADATA_COL_ATTR_KEY))); // Add row_commit_version (generated field) - will throw if materialized column name is not // configured String rowCommitVersionPhysicalName = getPhysicalColumnNameOrThrow( MaterializedRowTrackingColumn.MATERIALIZED_ROW_COMMIT_VERSION, metadata); fields.add( new StructField( ROW_COMMIT_VERSION, DataTypes.LongType, nullableGeneratedFields, createGeneratedFieldMetadata( ROW_COMMIT_VERSION, rowCommitVersionPhysicalName, ROW_COMMIT_VERSION_METADATA_COL_ATTR_KEY))); return fields; } /** * Helper method to get physical column name from MaterializedRowTrackingColumn, converting kernel * IllegalArgumentException to Spark DeltaIllegalStateException. This matches the exception thrown * by Spark V1's MaterializedRowTrackingColumn. * * @param column the MaterializedRowTrackingColumn instance * @param metadata the table metadata * @return the physical column name * @throws DeltaIllegalStateException if the materialized column name is missing */ private static String getPhysicalColumnNameOrThrow( MaterializedRowTrackingColumn column, Metadata metadata) { try { return column.getPhysicalColumnName(metadata.getConfiguration()); } catch (IllegalArgumentException e) { // Convert kernel exception to Spark V1-compatible DeltaIllegalStateException // Use the same error class as Spark V1: DELTA_MATERIALIZED_ROW_TRACKING_COLUMN_NAME_MISSING String rowTrackingColumnType = column == MaterializedRowTrackingColumn.MATERIALIZED_ROW_ID ? "Row ID" : "Row Commit Version"; throw new DeltaIllegalStateException( "DELTA_MATERIALIZED_ROW_TRACKING_COLUMN_NAME_MISSING", new String[] {rowTrackingColumnType, metadata.getId()}, e); } } private static org.apache.spark.sql.types.Metadata createConstantFieldMetadata( String columnName, String attrKey) { return new MetadataBuilder() .withMetadata(FileSourceConstantMetadataStructField.metadata(columnName)) .putBoolean(attrKey, true) .build(); } private static org.apache.spark.sql.types.Metadata createGeneratedFieldMetadata( String readColumnName, String writeColumnName, String attrKey) { return new MetadataBuilder() .withMetadata( FileSourceGeneratedMetadataStructField.metadata(readColumnName, writeColumnName)) .putBoolean(attrKey, true) .build(); } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ScalaUtils.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.utils; import java.util.Collections; import java.util.Map; import java.util.Optional; import scala.Option; import scala.Tuple2; import scala.collection.immutable.Map$; import scala.collection.mutable.Builder; import scala.jdk.javaapi.CollectionConverters; public final class ScalaUtils { public static scala.collection.immutable.Map toScalaMap( Map javaMap) { if (javaMap == null) throw new NullPointerException("options"); // Works on Scala 2.12 and 2.13 @SuppressWarnings({"rawtypes", "unchecked"}) Builder, scala.collection.immutable.Map> b = (Builder) Map$.MODULE$.newBuilder(); for (Map.Entry e : javaMap.entrySet()) { b.$plus$eq(new Tuple2<>(e.getKey(), e.getValue())); } return b.result(); } public static Map toJavaMap( scala.collection.immutable.Map scalaMap) { if (scalaMap == null) { return null; } if (scalaMap.isEmpty()) { return Collections.emptyMap(); } return CollectionConverters.asJava(scalaMap); } /** * Converts a Java {@link Optional} to a Scala {@link Option}. * * @param optional the Java Optional to convert * @param the type of the value * @return the corresponding Scala Option */ public static Option toScalaOption(Optional optional) { return optional.map(Option::apply).orElse(Option.empty()); } /** * Converts a Scala {@link Option} to a Java {@link Optional}. * * @param option the Scala Option to convert * @param the type of the value * @return the corresponding Java Optional */ public static Optional toJavaOptional(Option option) { return option.isDefined() ? Optional.of(option.get()) : Optional.empty(); } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/utils/SchemaUtils.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.utils; import static java.util.Objects.requireNonNull; import io.delta.kernel.types.ArrayType; import io.delta.kernel.types.BinaryType; import io.delta.kernel.types.BooleanType; import io.delta.kernel.types.ByteType; import io.delta.kernel.types.DataType; import io.delta.kernel.types.DateType; import io.delta.kernel.types.DecimalType; import io.delta.kernel.types.DoubleType; import io.delta.kernel.types.FloatType; import io.delta.kernel.types.IntegerType; import io.delta.kernel.types.LongType; import io.delta.kernel.types.MapType; import io.delta.kernel.types.ShortType; import io.delta.kernel.types.StringType; import io.delta.kernel.types.StructField; import io.delta.kernel.types.StructType; import io.delta.kernel.types.TimestampNTZType; import io.delta.kernel.types.TimestampType; import io.delta.kernel.types.VariantType; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.Metadata; import org.apache.spark.sql.types.MetadataBuilder; import scala.jdk.CollectionConverters; /** A utility class for converting between Delta Kernel and Spark schemas and data types. */ public class SchemaUtils { ////////////////////// // Kernel --> Spark // ////////////////////// /** Converts a Delta Kernel schema to a Spark schema. */ public static org.apache.spark.sql.types.StructType convertKernelSchemaToSparkSchema( StructType kernelSchema) { requireNonNull(kernelSchema); List fields = new ArrayList<>(); for (StructField field : kernelSchema.fields()) { fields.add( new org.apache.spark.sql.types.StructField( field.getName(), convertKernelDataTypeToSparkDataType(field.getDataType()), field.isNullable(), convertKernelFieldMetadataToSparkMetadata(field.getMetadata()))); } return new org.apache.spark.sql.types.StructType( fields.toArray(new org.apache.spark.sql.types.StructField[0])); } /** Converts a Delta Kernel data type to a Spark data type. */ public static org.apache.spark.sql.types.DataType convertKernelDataTypeToSparkDataType( DataType kernelDataType) { requireNonNull(kernelDataType); if (kernelDataType instanceof StringType) { return DataTypes.StringType; } else if (kernelDataType instanceof BooleanType) { return DataTypes.BooleanType; } else if (kernelDataType instanceof IntegerType) { return DataTypes.IntegerType; } else if (kernelDataType instanceof LongType) { return DataTypes.LongType; } else if (kernelDataType instanceof BinaryType) { return DataTypes.BinaryType; } else if (kernelDataType instanceof ByteType) { return DataTypes.ByteType; } else if (kernelDataType instanceof DateType) { return DataTypes.DateType; } else if (kernelDataType instanceof DecimalType) { DecimalType kernelDecimal = (DecimalType) kernelDataType; return DataTypes.createDecimalType(kernelDecimal.getPrecision(), kernelDecimal.getScale()); } else if (kernelDataType instanceof DoubleType) { return DataTypes.DoubleType; } else if (kernelDataType instanceof FloatType) { return DataTypes.FloatType; } else if (kernelDataType instanceof ShortType) { return DataTypes.ShortType; } else if (kernelDataType instanceof TimestampType) { return DataTypes.TimestampType; } else if (kernelDataType instanceof TimestampNTZType) { return DataTypes.TimestampNTZType; } else if (kernelDataType instanceof ArrayType) { ArrayType kernelArray = (ArrayType) kernelDataType; return DataTypes.createArrayType( convertKernelDataTypeToSparkDataType(kernelArray.getElementType()), kernelArray.containsNull()); } else if (kernelDataType instanceof MapType) { MapType kernelMap = (MapType) kernelDataType; return DataTypes.createMapType( convertKernelDataTypeToSparkDataType(kernelMap.getKeyType()), convertKernelDataTypeToSparkDataType(kernelMap.getValueType()), kernelMap.isValueContainsNull()); } else if (kernelDataType instanceof StructType) { return convertKernelSchemaToSparkSchema((StructType) kernelDataType); } else if (kernelDataType instanceof VariantType) { return DataTypes.VariantType; } else { throw new IllegalArgumentException("unsupported data type " + kernelDataType); } } ////////////////////// // Spark --> Kernel // ////////////////////// /** Converts a Spark schema to a Delta Kernel schema. */ public static StructType convertSparkSchemaToKernelSchema( org.apache.spark.sql.types.StructType sparkSchema) { requireNonNull(sparkSchema); List kernelFields = new ArrayList<>(); for (org.apache.spark.sql.types.StructField field : sparkSchema.fields()) { kernelFields.add( new StructField( field.name(), convertSparkDataTypeToKernelDataType(field.dataType()), field.nullable(), convertSparkMetadataToKernelFieldMetadata(field.metadata()))); } return new StructType(kernelFields); } /** Converts a Spark data type to a Delta Kernel data type. */ public static DataType convertSparkDataTypeToKernelDataType( org.apache.spark.sql.types.DataType sparkDataType) { requireNonNull(sparkDataType); if (sparkDataType instanceof org.apache.spark.sql.types.StringType) { return StringType.STRING; } else if (sparkDataType instanceof org.apache.spark.sql.types.BooleanType) { return BooleanType.BOOLEAN; } else if (sparkDataType instanceof org.apache.spark.sql.types.IntegerType) { return IntegerType.INTEGER; } else if (sparkDataType instanceof org.apache.spark.sql.types.LongType) { return LongType.LONG; } else if (sparkDataType instanceof org.apache.spark.sql.types.BinaryType) { return BinaryType.BINARY; } else if (sparkDataType instanceof org.apache.spark.sql.types.ByteType) { return ByteType.BYTE; } else if (sparkDataType instanceof org.apache.spark.sql.types.DateType) { return DateType.DATE; } else if (sparkDataType instanceof org.apache.spark.sql.types.DecimalType) { org.apache.spark.sql.types.DecimalType sparkDecimal = (org.apache.spark.sql.types.DecimalType) sparkDataType; return new DecimalType(sparkDecimal.precision(), sparkDecimal.scale()); } else if (sparkDataType instanceof org.apache.spark.sql.types.DoubleType) { return DoubleType.DOUBLE; } else if (sparkDataType instanceof org.apache.spark.sql.types.FloatType) { return FloatType.FLOAT; } else if (sparkDataType instanceof org.apache.spark.sql.types.ShortType) { return ShortType.SHORT; } else if (sparkDataType instanceof org.apache.spark.sql.types.TimestampType) { return TimestampType.TIMESTAMP; } else if (sparkDataType instanceof org.apache.spark.sql.types.TimestampNTZType) { return TimestampNTZType.TIMESTAMP_NTZ; } else if (sparkDataType instanceof org.apache.spark.sql.types.ArrayType) { org.apache.spark.sql.types.ArrayType sparkArray = (org.apache.spark.sql.types.ArrayType) sparkDataType; return new ArrayType( convertSparkDataTypeToKernelDataType(sparkArray.elementType()), sparkArray.containsNull()); } else if (sparkDataType instanceof org.apache.spark.sql.types.MapType) { org.apache.spark.sql.types.MapType sparkMap = (org.apache.spark.sql.types.MapType) sparkDataType; return new MapType( convertSparkDataTypeToKernelDataType(sparkMap.keyType()), convertSparkDataTypeToKernelDataType(sparkMap.valueType()), sparkMap.valueContainsNull()); } else if (sparkDataType instanceof org.apache.spark.sql.types.StructType) { return convertSparkSchemaToKernelSchema( (org.apache.spark.sql.types.StructType) sparkDataType); } else if (sparkDataType instanceof org.apache.spark.sql.types.VariantType) { return VariantType.VARIANT; } else { throw new IllegalArgumentException("unsupported data type " + sparkDataType); } } /////////////////////////////// // Field Metadata Conversion // /////////////////////////////// /** * Converts Kernel FieldMetadata to Spark Metadata. * * @param kernelMetadata the Kernel FieldMetadata to convert * @return the equivalent Spark Metadata */ public static Metadata convertKernelFieldMetadataToSparkMetadata( io.delta.kernel.types.FieldMetadata kernelMetadata) { requireNonNull(kernelMetadata); if (kernelMetadata.getEntries().isEmpty()) { return Metadata.empty(); } MetadataBuilder builder = new MetadataBuilder(); kernelMetadata .getEntries() .forEach( (key, value) -> { if (value instanceof Long) { builder.putLong(key, (Long) value); } else if (value instanceof Double) { builder.putDouble(key, (Double) value); } else if (value instanceof Boolean) { builder.putBoolean(key, (Boolean) value); } else if (value instanceof String) { builder.putString(key, (String) value); } else if (value instanceof io.delta.kernel.types.FieldMetadata) { builder.putMetadata( key, convertKernelFieldMetadataToSparkMetadata( (io.delta.kernel.types.FieldMetadata) value)); } else if (value instanceof Long[]) { builder.putLongArray(key, unboxLongArray((Long[]) value, key)); } else if (value instanceof Double[]) { builder.putDoubleArray(key, unboxDoubleArray((Double[]) value, key)); } else if (value instanceof Boolean[]) { builder.putBooleanArray(key, unboxBooleanArray((Boolean[]) value, key)); } else if (value instanceof String[]) { builder.putStringArray(key, (String[]) value); } else if (value instanceof io.delta.kernel.types.FieldMetadata[]) { io.delta.kernel.types.FieldMetadata[] kernelMetadatas = (io.delta.kernel.types.FieldMetadata[]) value; Metadata[] sparkMetadatas = Arrays.stream(kernelMetadatas) .map(SchemaUtils::convertKernelFieldMetadataToSparkMetadata) .toArray(Metadata[]::new); builder.putMetadataArray(key, sparkMetadatas); } else if (value == null) { builder.putNull(key); } else { throw new UnsupportedOperationException( "Unsupported metadata value type: " + value.getClass().getName()); } }); return builder.build(); } /** * Converts Spark Metadata to Kernel FieldMetadata. * * @param sparkMetadata the Spark Metadata to convert * @return the equivalent Kernel FieldMetadata */ public static io.delta.kernel.types.FieldMetadata convertSparkMetadataToKernelFieldMetadata( Metadata sparkMetadata) { requireNonNull(sparkMetadata); if (sparkMetadata.map().isEmpty()) { return io.delta.kernel.types.FieldMetadata.empty(); } io.delta.kernel.types.FieldMetadata.Builder builder = io.delta.kernel.types.FieldMetadata.builder(); CollectionConverters.MapHasAsJava(sparkMetadata.map()) .asJava() .forEach( (key, value) -> { if (value instanceof Long) { builder.putLong(key, (Long) value); } else if (value instanceof Double) { builder.putDouble(key, (Double) value); } else if (value instanceof Boolean) { builder.putBoolean(key, (Boolean) value); } else if (value instanceof String) { builder.putString(key, (String) value); } else if (value instanceof Metadata) { builder.putFieldMetadata( key, convertSparkMetadataToKernelFieldMetadata((Metadata) value)); } else if (value instanceof long[]) { builder.putLongArray( key, Arrays.stream((long[]) value).boxed().toArray(Long[]::new)); } else if (value instanceof double[]) { builder.putDoubleArray( key, Arrays.stream((double[]) value).boxed().toArray(Double[]::new)); } else if (value instanceof boolean[]) { boolean[] valArray = (boolean[]) value; Boolean[] booleanArray = new Boolean[valArray.length]; for (int i = 0; i < valArray.length; i++) { booleanArray[i] = valArray[i]; } builder.putBooleanArray(key, booleanArray); } else if (value instanceof String[]) { builder.putStringArray(key, (String[]) value); } else if (value instanceof Metadata[]) { Metadata[] sparkMetadatas = (Metadata[]) value; io.delta.kernel.types.FieldMetadata[] kernelMetadatas = Arrays.stream(sparkMetadatas) .map(SchemaUtils::convertSparkMetadataToKernelFieldMetadata) .toArray(io.delta.kernel.types.FieldMetadata[]::new); builder.putFieldMetadataArray(key, kernelMetadatas); } else if (value == null) { builder.putNull(key); } else { throw new UnsupportedOperationException( "Unsupported metadata value type: " + value.getClass().getName()); } }); return builder.build(); } /** * Unboxes a Long[] to long[], checking for nulls. * * @throws NullPointerException if any element is null */ private static long[] unboxLongArray(Long[] boxedArray, String key) { long[] primitiveArray = new long[boxedArray.length]; for (int i = 0; i < boxedArray.length; i++) { requireNonNull( boxedArray[i], String.format("Null element at index %s in Long array for key '%s'", i, key)); primitiveArray[i] = boxedArray[i]; } return primitiveArray; } /** * Unboxes a Double[] to double[], checking for nulls. * * @throws NullPointerException if any element is null */ private static double[] unboxDoubleArray(Double[] boxedArray, String key) { double[] primitiveArray = new double[boxedArray.length]; for (int i = 0; i < boxedArray.length; i++) { requireNonNull( boxedArray[i], String.format("Null element at index %s in Double array for key '%s'", i, key)); primitiveArray[i] = boxedArray[i]; } return primitiveArray; } /** * Unboxes a Boolean[] to boolean[], checking for nulls. * * @throws NullPointerException if any element is null */ private static boolean[] unboxBooleanArray(Boolean[] boxedArray, String key) { boolean[] primitiveArray = new boolean[boxedArray.length]; for (int i = 0; i < boxedArray.length; i++) { requireNonNull( boxedArray[i], String.format("Null element at index %s in Boolean array for key '%s'", i, key)); primitiveArray[i] = boxedArray[i]; } return primitiveArray; } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/utils/SerializableKernelRowWrapper.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.utils; import io.delta.kernel.data.Row; import io.delta.kernel.defaults.internal.json.JsonUtils; import io.delta.kernel.internal.types.DataTypeJsonSerDe; import io.delta.kernel.types.StructType; import java.io.Serializable; public class SerializableKernelRowWrapper implements Serializable { private final String rowJson; private final String schemaJson; private transient Row row; public SerializableKernelRowWrapper(Row row) { this.rowJson = JsonUtils.rowToJson(row); this.schemaJson = DataTypeJsonSerDe.serializeDataType(row.getSchema()); this.row = row; } public Row getRow() { if (row == null) { StructType schema = DataTypeJsonSerDe.deserializeStructType(schemaJson); row = JsonUtils.rowFromJson(rowJson, schema); } return row; } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; SerializableKernelRowWrapper that = (SerializableKernelRowWrapper) o; return rowJson.equals(that.rowJson) && schemaJson.equals(that.schemaJson); } @Override public int hashCode() { int result = rowJson.hashCode(); result = 31 * result + schemaJson.hashCode(); return result; } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/utils/StatsUtils.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.utils; import static io.delta.spark.internal.v2.utils.ScalaUtils.toJavaOptional; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.OptionalLong; import org.apache.spark.sql.catalyst.catalog.CatalogColumnStat; import org.apache.spark.sql.catalyst.catalog.CatalogColumnStat$; import org.apache.spark.sql.catalyst.catalog.CatalogStatistics; import org.apache.spark.sql.connector.expressions.FieldReference; import org.apache.spark.sql.connector.expressions.NamedReference; import org.apache.spark.sql.connector.read.Statistics; import org.apache.spark.sql.connector.read.colstats.ColumnStatistics; import org.apache.spark.sql.types.DataType; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; /** Utilities for converting catalog statistics to V2 connector statistics. */ public final class StatsUtils { private StatsUtils() {} /** * Convert {@link CatalogStatistics} to V2 connector {@link Statistics}. * * @param catalogStats the catalog statistics to convert * @param dataSchema the data schema (non-partition columns) * @param partitionSchema the partition schema * @return V2 Statistics representation */ public static Statistics toV2Statistics( CatalogStatistics catalogStats, StructType dataSchema, StructType partitionSchema) { // Build a map of column name -> DataType from both schemas Map columnTypes = new HashMap<>(); for (StructField field : dataSchema.fields()) { columnTypes.put(field.name(), field.dataType()); } for (StructField field : partitionSchema.fields()) { columnTypes.put(field.name(), field.dataType()); } Map colStatsMap = buildColumnStats(catalogStats, columnTypes); return new Statistics() { @Override public OptionalLong sizeInBytes() { return OptionalLong.of(catalogStats.sizeInBytes().longValue()); } @Override public OptionalLong numRows() { return toJavaOptional(catalogStats.rowCount()) .map(r -> OptionalLong.of(r.longValue())) .orElse(OptionalLong.empty()); } @Override public Map columnStats() { return colStatsMap; } }; } private static Map buildColumnStats( CatalogStatistics catalogStats, Map columnTypes) { Map colStats = scala.collection.JavaConverters.mapAsJavaMapConverter(catalogStats.colStats()).asJava(); if (colStats.isEmpty()) { return Collections.emptyMap(); } Map result = new HashMap<>(); for (Map.Entry entry : colStats.entrySet()) { String colName = entry.getKey(); CatalogColumnStat stat = entry.getValue(); DataType dataType = columnTypes.get(colName); if (dataType == null) { continue; } NamedReference ref = FieldReference.apply(colName); int version = stat.version(); // Eagerly parse min/max to avoid repeated fromExternalString calls Optional minValue = toJavaOptional(stat.min()) .map( minStr -> CatalogColumnStat$.MODULE$.fromExternalString( minStr, colName, dataType, version)); Optional maxValue = toJavaOptional(stat.max()) .map( maxStr -> CatalogColumnStat$.MODULE$.fromExternalString( maxStr, colName, dataType, version)); OptionalLong distinctCount = toJavaOptional(stat.distinctCount()) .map(d -> OptionalLong.of(d.longValue())) .orElse(OptionalLong.empty()); OptionalLong nullCount = toJavaOptional(stat.nullCount()) .map(n -> OptionalLong.of(n.longValue())) .orElse(OptionalLong.empty()); OptionalLong avgLen = toJavaOptional(stat.avgLen()) .map(v -> OptionalLong.of(((Number) v).longValue())) .orElse(OptionalLong.empty()); OptionalLong maxLen = toJavaOptional(stat.maxLen()) .map(v -> OptionalLong.of(((Number) v).longValue())) .orElse(OptionalLong.empty()); ColumnStatistics v2ColStats = new ColumnStatistics() { @Override public OptionalLong distinctCount() { return distinctCount; } @Override public Optional min() { return minValue; } @Override public Optional max() { return maxValue; } @Override public OptionalLong nullCount() { return nullCount; } @Override public OptionalLong avgLen() { return avgLen; } @Override public OptionalLong maxLen() { return maxLen; } }; result.put(ref, v2ColStats); } return Collections.unmodifiableMap(result); } } ================================================ FILE: spark/v2/src/main/java/io/delta/spark/internal/v2/utils/StreamingHelper.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.utils; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static io.delta.kernel.internal.util.Preconditions.checkState; import io.delta.kernel.CommitActions; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.data.Row; import io.delta.kernel.engine.Engine; import io.delta.kernel.internal.DeltaLogActionUtils; import io.delta.kernel.internal.actions.AddFile; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.RemoveFile; import io.delta.kernel.internal.commitrange.CommitRangeImpl; import io.delta.kernel.internal.data.StructRow; import io.delta.kernel.internal.util.Preconditions; import io.delta.kernel.utils.CloseableIterator; import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; import java.io.IOException; import java.util.*; import org.apache.spark.annotation.Experimental; /** * Helper class providing utilities for working with Delta table data in streaming scenarios. * *

This class provides static utility methods for extracting information from Delta table * batches, such as version numbers and data change actions. */ @Experimental public class StreamingHelper { /** * Returns the index of the field with the given name in the schema of the batch. Throws an {@link * IllegalArgumentException} if the field is not found. */ private static int getFieldIndex(ColumnarBatch batch, String fieldName) { int index = batch.getSchema().indexOf(fieldName); checkArgument(index >= 0, "Field '%s' not found in schema: %s", fieldName, batch.getSchema()); return index; } /** * Get the version from a {@link ColumnarBatch} of Delta log actions. Assumes all rows in the * batch belong to the same commit version, so it reads the version from the first row (rowId=0). */ public static long getVersion(ColumnarBatch batch) { int versionColIdx = getFieldIndex(batch, "version"); return batch.getColumnVector(versionColIdx).getLong(0); } /** * Get AddFile action from a FilteredColumnarBatch at the specified row, if present. * *

This method respects the selection vector to filter out duplicate files that may appear when * stats re-collection (e.g., ANALYZE TABLE COMPUTE STATISTICS) re-adds files with updated stats. * The Kernel uses selection vectors to mark which rows (AddFiles) are logically valid. * * @param batch the FilteredColumnarBatch containing AddFile actions * @param rowId the row index to check * @return Optional containing the AddFile if present and selected, empty otherwise */ public static Optional getAddFile(FilteredColumnarBatch batch, int rowId) { // Check selection vector first - rows may be filtered out when stats re-collection // re-adds files with updated stats Optional selectionVector = batch.getSelectionVector(); boolean isFiltered = selectionVector.map(sv -> sv.isNullAt(rowId) || !sv.getBoolean(rowId)).orElse(false); if (isFiltered) { return Optional.empty(); } return getAddFile(batch.getData(), rowId); } /** * Get AddFile action from a ColumnarBatch at the specified row, if present. * *

Caller should ensure all rows are valid (e.g., not filtered by selection vector). For * FilteredColumnarBatch with selection vectors, use {@link #getAddFile(FilteredColumnarBatch, * int)} instead. */ private static Optional getAddFile(ColumnarBatch batch, int rowId) { int addIdx = getFieldIndex(batch, DeltaLogActionUtils.DeltaAction.ADD.colName); ColumnVector addVector = batch.getColumnVector(addIdx); if (addVector.isNullAt(rowId)) { return Optional.empty(); } Row addFileRow = StructRow.fromStructVector(addVector, rowId); checkState( addFileRow != null, String.format("Failed to extract AddFile struct from batch at rowId=%d.", rowId)); return Optional.of(new AddFile(addFileRow)); } /** Get AddFile action from a batch at the specified row, if present and has dataChange=true. */ public static Optional getAddFileWithDataChange(ColumnarBatch batch, int rowId) { return getAddFile(batch, rowId).filter(AddFile::getDataChange); } /** * Get RemoveFile action from a batch at the specified row, if present and has dataChange=true. */ public static Optional getDataChangeRemove(ColumnarBatch batch, int rowId) { int removeIdx = getFieldIndex(batch, DeltaLogActionUtils.DeltaAction.REMOVE.colName); ColumnVector removeVector = batch.getColumnVector(removeIdx); if (removeVector.isNullAt(rowId)) { return Optional.empty(); } Row removeFileRow = StructRow.fromStructVector(removeVector, rowId); checkState( removeFileRow != null, String.format("Failed to extract RemoveFile struct from batch at rowId=%d.", rowId)); RemoveFile removeFile = new RemoveFile(removeFileRow); return removeFile.getDataChange() ? Optional.of(removeFile) : Optional.empty(); } /** Get Metadata action from a batch at the specified row, if present. */ public static Optional getMetadata(ColumnarBatch batch, int rowId) { int metadataIdx = getFieldIndex(batch, DeltaLogActionUtils.DeltaAction.METADATA.colName); ColumnVector metadataVector = batch.getColumnVector(metadataIdx); Metadata metadata = Metadata.fromColumnVector(metadataVector, rowId); return Optional.ofNullable(metadata); } /** * Gets commit-level actions from a commit range without requiring a snapshot at the exact start * version. * *

Returns an iterator over {@link CommitActions}, where each CommitActions represents a single * commit. * *

This method is "unsafe" because it bypasses the standard {@code * CommitRange.getCommitActions()} API which requires a snapshot at the exact start version for * protocol validation. * * @param engine the Delta engine * @param commitRange the commit range to read actions from * @param tablePath the path to the Delta table * @param actionSet the set of actions to read (e.g., ADD, REMOVE) * @return an iterator over {@link CommitActions}, one per commit version */ public static CloseableIterator getCommitActionsFromRangeUnsafe( Engine engine, CommitRangeImpl commitRange, String tablePath, Set actionSet) { return DeltaLogActionUtils.getActionsFromCommitFilesWithProtocolValidation( engine, tablePath, commitRange.getDeltaFiles(), actionSet); } /** * Collects metadata actions from a commit range, mapping each version to its metadata. * *

This method is "unsafe" because it uses {@code getActionsFromRangeUnsafe()} which bypasses * the standard snapshot requirement for protocol validation. * *

Returns a map preserving version order (via LinkedHashMap) where each version maps to its * metadata action. Throws an exception if multiple metadata actions are found in the same commit. * * @param startVersion the starting version (inclusive) of the commit range * @param endVersionOpt optional ending version (exclusive) of the commit range * @param snapshotManager the Delta snapshot manager * @param engine the Delta engine * @param tablePath the path to the Delta table * @return a map from version number to metadata action, in version order */ public static Map collectMetadataActionsFromRangeUnsafe( long startVersion, Optional endVersionOpt, DeltaSnapshotManager snapshotManager, Engine engine, String tablePath) { CommitRangeImpl commitRange = (CommitRangeImpl) snapshotManager.getTableChanges(engine, startVersion, endVersionOpt); // LinkedHashMap to preserve insertion order Map versionToMetadata = new LinkedHashMap<>(); try (CloseableIterator commitsIter = getCommitActionsFromRangeUnsafe( engine, commitRange, tablePath, Set.of(DeltaLogActionUtils.DeltaAction.METADATA))) { while (commitsIter.hasNext()) { try (CommitActions commit = commitsIter.next()) { long version = commit.getVersion(); try (CloseableIterator actionsIter = commit.getActions()) { while (actionsIter.hasNext()) { ColumnarBatch batch = actionsIter.next(); int numRows = batch.getSize(); for (int rowId = 0; rowId < numRows; rowId++) { Optional metadataOpt = StreamingHelper.getMetadata(batch, rowId); if (metadataOpt.isPresent()) { Metadata existing = versionToMetadata.putIfAbsent(version, metadataOpt.get()); Preconditions.checkArgument( existing == null, "Should not encounter two metadata actions in the same commit of version %d", version); } } } } catch (IOException e) { throw new RuntimeException("Failed to process commit at version " + version, e); } } } } catch (RuntimeException e) { throw e; // Rethrow runtime exceptions directly } catch (Exception e) { // CommitActions.close() throws Exception throw new RuntimeException("Failed to process commits", e); } return versionToMetadata; } /** Private constructor to prevent instantiation of this utility class. */ private StreamingHelper() {} } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2; import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.engine.Engine; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; public abstract class DeltaV2TestBase { protected static SparkSession spark; protected static Engine defaultEngine; @BeforeAll public static void setUpSparkAndEngine() { spark = SparkSession.builder() .master("local[*]") .appName("SparkKernelDsv2Tests") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtensionV1") .config( "spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalogV1") .getOrCreate(); defaultEngine = DefaultEngine.create(spark.sessionState().newHadoopConf()); } @AfterAll public static void tearDownSpark() { if (spark != null) { spark.stop(); spark = null; } } protected void createTestTableWithData(String path, String tableName) { spark.sql( String.format( "CREATE TABLE %s (id INT, name STRING, value DOUBLE) USING delta LOCATION '%s'", tableName, path)); spark.sql( String.format( "INSERT INTO %s VALUES (1, 'Alice', 10.5), (2, 'Bob', 20.5), (3, 'Charlie', 30.5)", tableName)); } protected void createEmptyTestTable(String path, String tableName) { spark.sql( String.format( "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tableName, path)); } protected void createEmptyPartitionedTestTable(String path, String tableName) { spark.sql( String.format( "CREATE TABLE %s (id INT, name STRING) USING delta PARTITIONED BY (name) LOCATION '%s'", tableName, path)); } protected void createSchemaEvolutionTestTable(String path, String tableName) { spark.sql( String.format( "CREATE TABLE %s (id INT NOT NULL, " + "name String, value FLOAT, " + "info STRUCT) USING delta LOCATION '%s'" + "TBLPROPERTIES (" + "'delta.columnMapping.mode' = 'name', " + "'delta.enableTypeWidening' = 'true')", tableName, path)); spark.sql( String.format( "INSERT INTO %s VALUES " + "(1, 'Alice', 10.5, named_struct('col1', 27, 'col2', 'LA')), " + "(2,'Bob', NULL, named_struct('col1', 30, 'col2', 'NYC'))", tableName)); } /** A runnable that can throw checked exceptions, for use with {@link #withSQLConf}. */ @FunctionalInterface protected interface ThrowingRunnable { void run() throws Exception; } /** * Runs the given action with a Spark SQL configuration temporarily set, then restores the * original value afterwards (similar to Scala's {@code withSQLConf}). */ protected void withSQLConf(String key, String value, ThrowingRunnable action) throws Exception { scala.Option original = spark.conf().getOption(key); spark.conf().set(key, value); try { action.run(); } finally { if (original.isDefined()) { spark.conf().set(key, original.get()); } else { spark.conf().unset(key); } } } /** * Runs the given action and drops the specified tables afterwards, similar to Scala's {@code * withTable}. */ protected void withTable(String[] tableNames, ThrowingRunnable action) throws Exception { try { action.run(); } finally { for (String tableName : tableNames) { spark.sql(String.format("DROP TABLE IF EXISTS %s", tableName)); } } } protected static void createPartitionedTable(String tableName, String path) { spark.sql( String.format( "CREATE TABLE `%s` (part INT, date STRING, city STRING, name STRING, cnt INT) USING delta LOCATION '%s' PARTITIONED BY (date, city, part)", tableName, path)); spark.sql( String.format( "INSERT INTO %s VALUES " + "('1', '20180520', 'hz', 'Alice', '10')," + "('1', '20180718', 'hz', 'Bob', '20')," + "('1', '20180512', 'sh', 'Charlie', '30')," + "('2', '20180520', 'bj', 'David', '40')," + "('2', '20181212', 'sz', 'Eve', '50')", tableName)); } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/InternalRowTestUtils.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2; import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.expressions.GenericInternalRow; import org.apache.spark.sql.execution.datasources.PartitionedFile; import org.apache.spark.sql.vectorized.ColumnarBatch; import org.apache.spark.unsafe.types.UTF8String; import scala.Function1; import scala.collection.Iterator; import scala.jdk.javaapi.CollectionConverters; import scala.runtime.AbstractFunction1; /** Test helper utilities for InternalRow operations. */ public class InternalRowTestUtils { private InternalRowTestUtils() {} /** Create a mock base reader that returns the given rows. */ public static Function1> mockReader( List rows) { return new AbstractFunction1>() { @Override public Iterator apply(PartitionedFile file) { return CollectionConverters.asScala(rows.iterator()); } }; } /** * Create a mock reader that returns ColumnarBatch objects through Iterator<InternalRow>. * *

Mimics Spark vectorized mode where ColumnarBatch is passed via type erasure. */ @SuppressWarnings("unchecked") public static Function1> mockBatchReader( List batches) { return new AbstractFunction1>() { @Override public Iterator apply(PartitionedFile file) { return (Iterator) (Iterator) CollectionConverters.asScala(batches.iterator()); } }; } /** Collect all rows from iterator into a list, copying each row. */ public static List collectRows(Iterator iter) { List result = new ArrayList<>(); while (iter.hasNext()) { result.add(iter.next().copy()); } return result; } /** Collect all ColumnarBatch objects from an iterator (for vectorized mode testing). */ @SuppressWarnings("unchecked") public static List collectBatches(Iterator iter) { Iterator objectIter = (Iterator) (Iterator) iter; List result = new ArrayList<>(); while (objectIter.hasNext()) { result.add((ColumnarBatch) objectIter.next()); } return result; } /** Create an InternalRow from values. Strings are converted to UTF8String. */ public static InternalRow row(Object... values) { Object[] converted = new Object[values.length]; for (int i = 0; i < values.length; i++) { converted[i] = values[i] instanceof String ? UTF8String.fromString((String) values[i]) : values[i]; } return new GenericInternalRow(converted); } /** Assert that actual InternalRows match expected rows. Converts to Row for comparison. */ public static void assertRowsEquals(List actual, List expected) { assertEquals(toRows(expected), toRows(actual)); } /** Convert InternalRows to Rows, converting UTF8String back to String. */ private static List toRows(List rows) { return rows.stream() .map( r -> RowFactory.create( IntStream.range(0, r.numFields()) .mapToObj( i -> r.get(i, null) instanceof UTF8String ? r.get(i, null).toString() : r.get(i, null)) .toArray())) .collect(Collectors.toList()); } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/V2DDLTest.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2; import static org.junit.jupiter.api.Assertions.*; import java.io.File; import java.util.Arrays; import java.util.List; import org.apache.spark.sql.*; import org.apache.spark.sql.types.DataTypes; import org.junit.jupiter.api.*; import org.junit.jupiter.api.io.TempDir; /** Tests for V2 DDL operations. */ public class V2DDLTest extends V2TestBase { @Test public void testCreateTable() { spark.sql( str( "CREATE TABLE dsv2.%s.create_table_test (id INT, name STRING, value DOUBLE)", nameSpace)); Dataset actual = spark.sql(str("DESCRIBE TABLE dsv2.%s.create_table_test", nameSpace)); List expectedRows = Arrays.asList( RowFactory.create("id", "int", null), RowFactory.create("name", "string", null), RowFactory.create("value", "double", null)); assertDatasetEquals(actual, expectedRows); } @Test public void testQueryTableNotExist() { AnalysisException e = org.junit.jupiter.api.Assertions.assertThrows( AnalysisException.class, () -> spark.sql(str("SELECT * FROM dsv2.%s.not_found_test", nameSpace))); assertEquals( "TABLE_OR_VIEW_NOT_FOUND", e.getErrorClass(), "Missing table should raise TABLE_OR_VIEW_NOT_FOUND"); } @Test public void testPathBasedTable(@TempDir File deltaTablePath) { String tablePath = deltaTablePath.getAbsolutePath(); // Create test data and write as Delta table Dataset testData = spark.createDataFrame( Arrays.asList( RowFactory.create(1, "Alice", 100.0), RowFactory.create(2, "Bob", 200.0), RowFactory.create(3, "Charlie", 300.0)), DataTypes.createStructType( Arrays.asList( DataTypes.createStructField("id", DataTypes.IntegerType, false), DataTypes.createStructField("name", DataTypes.StringType, false), DataTypes.createStructField("value", DataTypes.DoubleType, false)))); testData.write().format("delta").save(tablePath); // TODO: [delta-io/delta#5001] change to select query after batch read is supported for dsv2 // path. Dataset actual = spark.sql(str("DESCRIBE TABLE dsv2.delta.`%s`", tablePath)); List expectedRows = Arrays.asList( RowFactory.create("id", "int", null), RowFactory.create("name", "string", null), RowFactory.create("value", "double", null)); assertDatasetEquals(actual, expectedRows); } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/V2ReadTest.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; import java.nio.file.Files; import java.util.List; import org.apache.spark.sql.delta.DeltaLog; import org.junit.jupiter.api.*; import org.junit.jupiter.api.io.TempDir; import scala.Option; /** Tests for V2 batch read operations. */ public class V2ReadTest extends V2TestBase { @Test public void testBatchRead() { spark.sql( str("CREATE TABLE dsv2.%s.batch_read_test (id INT, name STRING, value DOUBLE)", nameSpace)); check(str("SELECT * FROM dsv2.%s.batch_read_test", nameSpace), List.of()); } @Test public void testColumnMappingRead(@TempDir File deltaTablePath) { String tablePath = deltaTablePath.getAbsolutePath(); // Create a Delta table with column mapping enabled using name mode spark.sql( str( "CREATE TABLE delta.`%s` (id INT, user_name STRING, amount DOUBLE) " + "USING delta " + "TBLPROPERTIES ('delta.columnMapping.mode' = 'name')", tablePath)); // Insert test data spark.sql( str("INSERT INTO delta.`%s` VALUES (1, 'Alice', 100.0), (2, 'Bob', 200.0)", tablePath)); // Read through V2 and verify check( str("SELECT * FROM dsv2.delta.`%s` ORDER BY id", tablePath), List.of(row(1, "Alice", 100.0), row(2, "Bob", 200.0))); } @Test public void testDeletionVectorRead(@TempDir File tempDir) throws Exception { // Create a directory with space in the name to test URL encoding handling File dirWithSpace = new File(tempDir, "my table"); Files.createDirectories(dirWithSpace.toPath()); String tablePath = dirWithSpace.getAbsolutePath(); // Create a Delta table with deletion vectors enabled. spark.sql( str( "CREATE TABLE delta.`%s` (id LONG, value STRING) " + "USING delta " + "TBLPROPERTIES ('delta.enableDeletionVectors' = 'true')", tablePath)); // Insert enough data so that DELETE creates DVs instead of rewriting the file. // Use spark.range() to generate more rows. spark .range(1000) .selectExpr("id", "cast(id as string) as value") .write() .format("delta") .mode("append") .save(tablePath); // Delete some rows to create deletion vectors (not whole file deletions). spark.sql(str("DELETE FROM delta.`%s` WHERE id %% 2 = 0", tablePath)); // Verify that deletion vectors were actually created. DeltaLog deltaLog = DeltaLog.forTable(spark, tablePath); long numDVs = (long) deltaLog .update(false, Option.empty(), Option.empty()) .numDeletionVectorsOpt() .getOrElse(() -> 0L); assertTrue(numDVs > 0, "Expected deletion vectors to be created, but none were found"); // Read through V2 and verify deleted rows are filtered out (only odd ids remain). long count = spark.sql(str("SELECT * FROM dsv2.delta.`%s`", tablePath)).count(); // 500 odd numbers from 0-999: 1, 3, 5, ..., 999 assertTrue(count == 500, "Expected 500 rows after DV filtering, got " + count); } @Test public void testPartitionedJoinEliminatesShuffle(@TempDir File tempDir) { String tablePath = tempDir.getAbsolutePath(); // Create a partitioned Delta table via V1 spark.sql( str( "CREATE TABLE delta.`%s` (id INT, data STRING, part INT) " + "USING delta PARTITIONED BY (part)", tablePath)); spark.sql( str( "INSERT INTO delta.`%s` VALUES (1, 'a', 1), (2, 'b', 1), (3, 'c', 2), (4, 'd', 3)", tablePath)); // Disable broadcast join so Spark must use shuffle or partition-aware join. // Enable V2 bucketing so Spark recognizes KeyGroupedPartitioning from // SupportsReportPartitioning (default is false in Spark 4.0, true in 4.1). withSQLConf( "spark.sql.autoBroadcastJoinThreshold", "-1", () -> withSQLConf( "spark.sql.sources.v2.bucketing.enabled", "true", () -> { // Self-join on partition column via DSv2 catalog String joinQuery = str( "SELECT a.id, b.data FROM dsv2.delta.`%s` a " + "JOIN dsv2.delta.`%s` b ON a.part = b.part", tablePath, tablePath); String explainOutput = spark.sql(joinQuery).queryExecution().executedPlan().toString(); // With SupportsReportPartitioning + HasPartitionKey, Spark should recognize // both sides are KeyGroupedPartitioned on 'part' and skip the shuffle // (no Exchange node) assertFalse( explainOutput.contains("Exchange"), "Expected no Exchange (shuffle) in plan for partition-aligned join, " + "but found one.\nPlan:\n" + explainOutput); })); } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/V2StreamingReadTest.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2; import static org.junit.jupiter.api.Assertions.*; import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.LongStream; import org.apache.spark.sql.*; import org.apache.spark.sql.catalyst.expressions.Expression; import org.apache.spark.sql.catalyst.expressions.Literal$; import org.apache.spark.sql.delta.DeltaIllegalStateException; import org.apache.spark.sql.delta.DeltaLog; import org.apache.spark.sql.delta.stats.StatisticsCollection; import org.apache.spark.sql.streaming.StreamingQuery; import org.apache.spark.sql.streaming.StreamingQueryException; import org.junit.jupiter.api.*; import org.junit.jupiter.api.io.TempDir; import scala.Option; import scala.collection.JavaConverters; /** Tests for V2 streaming read operations. */ public class V2StreamingReadTest extends V2TestBase { @Test public void testStreamingReadWithStartingVersion(@TempDir File deltaTablePath) throws Exception { String tablePath = deltaTablePath.getAbsolutePath(); // Write version 0 spark .createDataFrame(Arrays.asList(RowFactory.create(1, "Alice", 100.0)), TEST_SCHEMA) .write() .format("delta") .save(tablePath); // Write version 1 spark .createDataFrame(Arrays.asList(RowFactory.create(2, "Bob", 200.0)), TEST_SCHEMA) .write() .format("delta") .mode("append") .save(tablePath); // Write version 2 spark .createDataFrame(Arrays.asList(RowFactory.create(3, "Charlie", 300.0)), TEST_SCHEMA) .write() .format("delta") .mode("append") .save(tablePath); // Start streaming from version 0 - should read all three versions String dsv2TableRef = str("dsv2.delta.`%s`", tablePath); Dataset streamingDF = spark.readStream().option("startingVersion", "1").table(dsv2TableRef); assertTrue(streamingDF.isStreaming(), "Dataset should be streaming"); // Process all batches - should have all data from versions 0, 1, and 2 List actualRows = processStreamingQuery(streamingDF, "test_with_starting_version"); List expectedRows = Arrays.asList(RowFactory.create(2, "Bob", 200.0), RowFactory.create(3, "Charlie", 300.0)); assertDataEquals(actualRows, expectedRows); } @Test public void testStreamingRead(@TempDir File deltaTablePath) throws Exception { String tablePath = deltaTablePath.getAbsolutePath(); // Write version 0 spark .createDataFrame(Arrays.asList(RowFactory.create(1, "Alice", 100.0)), TEST_SCHEMA) .write() .format("delta") .save(tablePath); // Write version 1 spark .createDataFrame(Arrays.asList(RowFactory.create(2, "Bob", 200.0)), TEST_SCHEMA) .write() .format("delta") .mode("append") .save(tablePath); String dsv2TableRef = str("dsv2.delta.`%s`", tablePath); Dataset streamingDF = spark.readStream().table(dsv2TableRef); assertTrue(streamingDF.isStreaming(), "Dataset should be streaming"); List actualRows = processStreamingQuery(streamingDF, "test_streaming_read"); List expectedRows = Arrays.asList(RowFactory.create(1, "Alice", 100.0), RowFactory.create(2, "Bob", 200.0)); assertDataEquals(actualRows, expectedRows); } /** * Tests that streaming read after stats recompute does not produce duplicate rows. * *

StatisticsCollection.recompute re-adds files with updated stats (dataChange=false), creating * duplicate AddFile entries in the log. The initial snapshot scan must use the selection vector * to filter out stale entries. */ @Test public void testStreamingReadAfterStatsRecompute(@TempDir File deltaTablePath) throws Exception { String tablePath = deltaTablePath.getAbsolutePath(); // Write data with stats collection disabled - files will have no stats withSQLConf( "spark.databricks.delta.stats.collect", "false", () -> spark .range(10) .selectExpr("id", "cast(id as string) as value") .write() .format("delta") .save(tablePath)); // Recompute statistics - this re-adds files with updated stats (dataChange=false), // creating duplicate AddFile entries in the log that must be filtered by selection vector DeltaLog deltaLog = DeltaLog.forTable(spark, tablePath); StatisticsCollection.recompute( spark, deltaLog, Option.empty(), JavaConverters.asScalaBuffer( new ArrayList<>(List.of((Expression) Literal$.MODULE$.apply(true)))) .toList(), af -> (Object) Boolean.TRUE); // Stream via V2 - should see each row exactly once, not duplicated String dsv2TableRef = str("dsv2.delta.`%s`", tablePath); Dataset streamingDF = spark.readStream().table(dsv2TableRef); List actualRows = processStreamingQuery(streamingDF, "test_stats_recompute"); List expectedRows = LongStream.range(0, 10) .mapToObj(i -> RowFactory.create(i, String.valueOf(i))) .collect(Collectors.toList()); assertDataEquals(actualRows, expectedRows); } /** * Tests that streaming read correctly handles multiple deletion vectors on the same file. * *

When multiple DELETE operations are applied to the same file, each creates/updates a * deletion vector. The streaming initial snapshot should correctly apply the cumulative DV and * only return non-deleted rows. */ @Test public void testStreamingReadWithMultipleDeletionVectors(@TempDir File deltaTablePath) throws Exception { String tablePath = deltaTablePath.getAbsolutePath(); // Create table with deletion vectors enabled spark.sql( str( "CREATE TABLE delta.`%s` (value INT) USING delta " + "TBLPROPERTIES ('delta.enableDeletionVectors' = 'true')", tablePath)); // Write 10 rows (0-9) in a single file spark .range(10) .selectExpr("cast(id as int) as value") .coalesce(1) .write() .format("delta") .mode("append") .save(tablePath); // Delete rows 0, 1, 2 - each creates/updates a DV on the same file spark.sql(str("DELETE FROM delta.`%s` WHERE value = 0", tablePath)); spark.sql(str("DELETE FROM delta.`%s` WHERE value = 1", tablePath)); spark.sql(str("DELETE FROM delta.`%s` WHERE value = 2", tablePath)); // Verify DVs were created DeltaLog deltaLog = DeltaLog.forTable(spark, tablePath); long numDVs = (long) deltaLog .update(false, Option.empty(), Option.empty()) .numDeletionVectorsOpt() .getOrElse(() -> 0L); assertTrue(numDVs > 0, "Expected deletion vectors to be created"); // Stream via V2 - should see rows 3-9 only (rows 0, 1, 2 deleted) String dsv2TableRef = str("dsv2.delta.`%s`", tablePath); Dataset streamingDF = spark.readStream().table(dsv2TableRef); List actualRows = processStreamingQuery(streamingDF, "test_multiple_dvs"); List expectedRows = Arrays.asList( RowFactory.create(3), RowFactory.create(4), RowFactory.create(5), RowFactory.create(6), RowFactory.create(7), RowFactory.create(8), RowFactory.create(9)); assertDataEquals(actualRows, expectedRows); } /** * Verifies that stopping a V2 streaming query does not surface an exception. * *

When Spark stops a streaming query it calls {@link Thread#interrupt()} on the micro-batch * thread. If that thread is blocked inside Kernel's {@code DefaultJsonHandler} reading delta log * JSON files via NIO channels, the interrupt causes a {@link * java.nio.channels.ClosedByInterruptException} wrapped in a {@code KernelEngineException}. The * fix in {@code SparkMicroBatchStream.latestOffset()} and {@code * SparkMicroBatchStream.planInputPartitions()} re-wraps this as an {@link * java.io.UncheckedIOException} so Spark's {@code isInterruptedByStop} recognizes it as a clean * shutdown. */ @Test public void testStreamingQueryStopDoesNotSurfaceException(@TempDir File deltaTablePath) throws Exception { String tablePath = deltaTablePath.getAbsolutePath(); // Write data spark .createDataFrame(Arrays.asList(RowFactory.create(1, "Alice", 100.0)), TEST_SCHEMA) .write() .format("delta") .save(tablePath); String dsv2TableRef = str("dsv2.delta.`%s`", tablePath); Dataset streamingDF = spark.readStream().table(dsv2TableRef); StreamingQuery query = streamingDF .writeStream() .format("memory") .queryName("test_stop_no_exception") .outputMode("append") .start(); // Continuously write new commits so latestOffset() keeps reading fresh delta log JSON files // via NIO. This ensures Thread.interrupt() from stop() is likely to arrive while a channel // is open inside DefaultJsonHandler.hasNext(), directly exercising the fix. ExecutorService writer = Executors.newSingleThreadExecutor(); try { writer.submit( () -> { for (int i = 0; i < 100; i++) { try { spark .createDataFrame( Arrays.asList(RowFactory.create(i + 2, "User" + i, (double) i * 10)), TEST_SCHEMA) .write() .format("delta") .mode("append") .save(tablePath); Thread.sleep(20); } catch (Exception ignored) { return; } } }); // Let the query process a few batches with active NIO reads before stopping. Thread.sleep(300); query.stop(); } finally { writer.shutdownNow(); writer.awaitTermination(5, TimeUnit.SECONDS); } // Release cached DeltaLog references so @TempDir cleanup can delete the directory. DeltaLog.clearCache(); // The stop should be clean — no exception should have been captured assertTrue( query.exception().isEmpty(), () -> "Expected no exception after query.stop(), but got: " + query.exception().get().toString()); } // TODO(#5319): The V1 source does not detect nested field additions on restart. Port this // validation to V1 for parity. @Test public void testNestedColumnAdditionDetectedOnRestart(@TempDir File deltaTablePath) throws Exception { String tablePath = deltaTablePath.getAbsolutePath(); String dsv2TableRef = str("dsv2.delta.`%s`", tablePath); // Create table with struct column: data STRUCT spark.sql( str("CREATE TABLE delta.`%s` (id STRING, data STRUCT) USING delta", tablePath)); spark.sql(str("INSERT INTO delta.`%s` VALUES ('0', named_struct('x', 1))", tablePath)); // Start streaming and process initial data File checkpointDir = new File(deltaTablePath, "_checkpoint"); Dataset streamingDF = spark.readStream().table(dsv2TableRef); StreamingQuery query = streamingDF .writeStream() .format("noop") .option("checkpointLocation", checkpointDir.getAbsolutePath()) .start(); query.processAllAvailable(); query.stop(); // Evolve struct via ALTER TABLE (metadata-only, no file deletion): // add nested field y -> data STRUCT spark.sql(str("ALTER TABLE delta.`%s` ADD COLUMNS (data.y INT)", tablePath)); // Restart with stale DataFrame — should fail with schema mismatch StreamingQueryException ex = assertThrows( StreamingQueryException.class, () -> { StreamingQuery q = streamingDF .writeStream() .format("noop") .option("checkpointLocation", checkpointDir.getAbsolutePath()) .start(); try { q.processAllAvailable(); } finally { q.stop(); } }); assertInstanceOf(DeltaIllegalStateException.class, ex.cause()); assertTrue( ex.getMessage().contains("DELTA_STREAMING_SCHEMA_MISMATCH_ON_RESTART"), "Expected DELTA_STREAMING_SCHEMA_MISMATCH_ON_RESTART but got: " + ex.getMessage()); } // TODO(#6232): v2 source cannot adopt type widening schema change without refreshing the // dataframe due to the lack of support in spark stream engine. Throw an error at stream // start time to instruct user. @Test public void testNestedTypeWideningDetectedOnRestart(@TempDir File deltaTablePath) throws Exception { String tablePath = deltaTablePath.getAbsolutePath(); String dsv2TableRef = str("dsv2.delta.`%s`", tablePath); // Create table with struct column: data STRUCT spark.sql( str("CREATE TABLE delta.`%s` (id STRING, data STRUCT) USING delta", tablePath)); spark.sql(str("INSERT INTO delta.`%s` VALUES ('0', named_struct('x', 1))", tablePath)); // Start streaming and process initial data File checkpointDir = new File(deltaTablePath, "_checkpoint"); Dataset streamingDF = spark.readStream().table(dsv2TableRef); StreamingQuery query = streamingDF .writeStream() .format("noop") .option("checkpointLocation", checkpointDir.getAbsolutePath()) .start(); query.processAllAvailable(); query.stop(); // Widen nested field type via ALTER TABLE (metadata-only, no file deletion): // data.x INT -> data.x BIGINT spark.sql( str( "ALTER TABLE delta.`%s` SET TBLPROPERTIES ('delta.enableTypeWidening' = 'true')", tablePath)); spark.sql(str("ALTER TABLE delta.`%s` ALTER COLUMN data.x TYPE BIGINT", tablePath)); // Restart with stale DataFrame — should fail with schema mismatch StreamingQueryException ex = assertThrows( StreamingQueryException.class, () -> { StreamingQuery q = streamingDF .writeStream() .format("noop") .option("checkpointLocation", checkpointDir.getAbsolutePath()) .start(); try { q.processAllAvailable(); } finally { q.stop(); } }); assertInstanceOf(DeltaIllegalStateException.class, ex.cause()); assertTrue( ex.getMessage().contains("DELTA_STREAMING_SCHEMA_MISMATCH_ON_RESTART"), "Expected DELTA_STREAMING_SCHEMA_MISMATCH_ON_RESTART but got: " + ex.getMessage()); } // TODO(#6232): v2 source cannot adopt type widening schema change without refreshing the // dataframe due to the lack of support in spark stream engine. Throw an error at stream // start time to instruct user. @Test public void testNestedNullabilityRelaxDetectedOnRestart(@TempDir File deltaTablePath) throws Exception { String tablePath = deltaTablePath.getAbsolutePath(); String dsv2TableRef = str("dsv2.delta.`%s`", tablePath); // Create table via SQL DDL to preserve the NOT NULL constraint on the nested field. // DataFrame writes go through ImplicitMetadataOperation which calls schema.asNullable, // forcing all fields (including nested ones) to nullable — losing the NOT NULL. spark.sql( str( "CREATE TABLE delta.`%s` (id STRING, data STRUCT) USING delta", tablePath)); spark.sql(str("INSERT INTO delta.`%s` VALUES ('0', named_struct('x', 1))", tablePath)); // Start streaming and process initial data File checkpointDir = new File(deltaTablePath, "_checkpoint"); Dataset streamingDF = spark.readStream().table(dsv2TableRef); StreamingQuery query = streamingDF .writeStream() .format("noop") .option("checkpointLocation", checkpointDir.getAbsolutePath()) .start(); query.processAllAvailable(); query.stop(); // Relax nullability via ALTER TABLE (metadata-only, no file deletion): // data.x NOT NULL -> data.x nullable spark.sql(str("ALTER TABLE delta.`%s` ALTER COLUMN data.x DROP NOT NULL", tablePath)); // Restart with stale DataFrame — should fail with schema mismatch StreamingQueryException ex = assertThrows( StreamingQueryException.class, () -> { StreamingQuery q = streamingDF .writeStream() .format("noop") .option("checkpointLocation", checkpointDir.getAbsolutePath()) .start(); try { q.processAllAvailable(); } finally { q.stop(); } }); assertInstanceOf(DeltaIllegalStateException.class, ex.cause()); assertTrue( ex.getMessage().contains("DELTA_STREAMING_SCHEMA_MISMATCH_ON_RESTART"), "Expected DELTA_STREAMING_SCHEMA_MISMATCH_ON_RESTART but got: " + ex.getMessage()); } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/V2TestBase.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2; import static org.junit.jupiter.api.Assertions.*; import java.io.File; import java.util.Arrays; import java.util.List; import java.util.UUID; import org.apache.spark.SparkConf; import org.apache.spark.sql.*; import org.apache.spark.sql.streaming.StreamingQuery; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; import org.junit.jupiter.api.*; import org.junit.jupiter.api.io.TempDir; /** * Base class for V2 tests with common SparkSession setup and helper methods. * *

This base class configures the Spark session with the V2 catalog and provides utility methods * for assertions. The V1 catalog is still used for write operations because this is currently * necessary until V2 supports write operations. */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) public abstract class V2TestBase { protected SparkSession spark; protected String nameSpace; protected static final StructType TEST_SCHEMA = DataTypes.createStructType( Arrays.asList( DataTypes.createStructField("id", DataTypes.IntegerType, false), DataTypes.createStructField("name", DataTypes.StringType, false), DataTypes.createStructField("value", DataTypes.DoubleType, false))); @BeforeAll public void setUp(@TempDir File tempDir) { // Spark doesn't allow '-' nameSpace = "ns_" + UUID.randomUUID().toString().replace('-', '_'); SparkConf conf = new SparkConf() .set("spark.sql.catalog.dsv2", "io.delta.spark.internal.v2.catalog.TestCatalog") .set("spark.sql.catalog.dsv2.base_path", tempDir.getAbsolutePath()) .set("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtensionV1") .set( "spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalogV1") .setMaster("local[*]") .setAppName(getClass().getSimpleName()); spark = SparkSession.builder().config(conf).getOrCreate(); } @AfterAll public void tearDown() { if (spark != null) { spark.stop(); } } /** * Builds a formatted string by substituting placeholders with the provided arguments. Useful for * constructing SQL queries and table identifiers. */ protected static String str(String template, Object... args) { return String.format(template, args); } /** * Executes a SQL query and verifies the result matches the expected rows. * * @param sql the SQL query to execute * @param expectedRows the expected rows as a list of lists (each inner list is a row) */ protected void check(String sql, List> expectedRows) { Dataset result = spark.sql(sql); List actualRows = result.collectAsList(); List expected = expectedRows.stream() .map(row -> RowFactory.create(row.toArray())) .collect(java.util.stream.Collectors.toList()); assertEquals( expected, actualRows, () -> "Query: " + sql + "\nExpected: " + expected + "\nActual: " + actualRows); } /** Creates a row representation as a list of values. */ protected static List row(Object... values) { return Arrays.asList(values); } /** Asserts that a dataset equals the expected rows. */ protected void assertDatasetEquals(Dataset actual, List expectedRows) { List actualRows = actual.collectAsList(); assertEquals( expectedRows, actualRows, () -> "Datasets differ: expected=" + expectedRows + "\nactual=" + actualRows); } /** * Processes a streaming query and returns the collected rows. * * @param streamingDF the streaming DataFrame to process * @param queryName the name for the memory sink query * @return the list of rows collected from the stream * @throws Exception if the streaming query fails */ protected List processStreamingQuery(Dataset streamingDF, String queryName) throws Exception { StreamingQuery query = null; try { query = streamingDF .writeStream() .format("memory") .queryName(queryName) .outputMode("append") .start(); query.processAllAvailable(); // Query the memory sink to get results Dataset results = spark.sql("SELECT * FROM " + queryName); return results.collectAsList(); } finally { if (query != null) { query.stop(); } } } /** * Runs the given action with a Spark SQL configuration temporarily set, then restores the * original value afterwards (similar to Scala's {@code withSQLConf}). */ protected void withSQLConf(String key, String value, Runnable action) { scala.Option original = spark.conf().getOption(key); spark.conf().set(key, value); try { action.run(); } finally { if (original.isDefined()) { spark.conf().set(key, original.get()); } else { spark.conf().unset(key); } } } /** * Asserts that rows equal the expected rows (order-independent). * * @param actualRows the actual rows * @param expectedRows the expected rows */ protected void assertDataEquals(List actualRows, List expectedRows) { assertEquals( expectedRows.size(), actualRows.size(), () -> "Row count differs: expected=" + expectedRows.size() + " actual=" + actualRows.size() + "\nExpected rows: " + expectedRows + "\nActual rows: " + actualRows); // Compare rows (order-independent for robustness) assertTrue( actualRows.containsAll(expectedRows) && expectedRows.containsAll(actualRows), () -> "Data differs:\nExpected: " + expectedRows + "\nActual: " + actualRows); } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/catalog/SparkTableTest.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.catalog; import static org.apache.spark.sql.connector.catalog.TableCapability.BATCH_READ; import static org.apache.spark.sql.connector.catalog.TableCapability.BATCH_WRITE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import io.delta.spark.internal.v2.DeltaV2TestBase; import java.io.File; import java.lang.reflect.Method; import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.BiFunction; import java.util.stream.Stream; import org.apache.hadoop.fs.Path; import org.apache.spark.sql.catalyst.TableIdentifier; import org.apache.spark.sql.catalyst.catalog.CatalogTable; import org.apache.spark.sql.connector.catalog.Column; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.SupportsWrite; import org.apache.spark.sql.connector.expressions.Transform; import org.apache.spark.sql.connector.write.LogicalWriteInfo; import org.apache.spark.sql.delta.catalog.DeltaTableV2; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import scala.Option; public class SparkTableTest extends DeltaV2TestBase { @ParameterizedTest(name = "{0} - {1}") @MethodSource("tableTestCases") public void testDeltaKernelTable( TableTestCase testCase, ConstructionMethod method, @TempDir File tempDir) throws Exception { String path = tempDir.getAbsolutePath(); String tableName = "test_" + testCase.name.toLowerCase().replace(" ", "_") + "_" + method.name().toLowerCase(); testCase.createTableSql.apply(tableName, path); Identifier identifier = Identifier.of(new String[] {"default"}, tableName); // Create SparkTable based on construction method SparkTable kernelTable; CatalogTable catalogTable = null; switch (method) { case FROM_PATH: kernelTable = new SparkTable(identifier, path); break; case FROM_CATALOG_TABLE: catalogTable = spark.sessionState().catalog().getTableMetadata(new TableIdentifier(tableName)); kernelTable = new SparkTable(identifier, catalogTable, Collections.emptyMap()); break; default: throw new IllegalArgumentException("Unknown construction method: " + method); } // ===== Test table name ===== String expectedName; switch (method) { case FROM_PATH: expectedName = "delta.`" + path + "`"; break; case FROM_CATALOG_TABLE: // Catalog table should return fully qualified name: spark_catalog.default.tableName expectedName = "spark_catalog.default." + tableName; break; default: throw new IllegalArgumentException("Unknown method: " + method); } assertEquals(expectedName, kernelTable.name()); // ===== Test schema ===== StructType sparkSchema = kernelTable.schema(); Column[] actualColumns = kernelTable.columns(); assertEquals(testCase.expectedColumns.size(), sparkSchema.fields().length); for (int i = 0; i < testCase.expectedColumns.size(); i++) { Column expectedCol = testCase.expectedColumns.get(i); assertEquals( expectedCol.name(), sparkSchema.fields()[i].name(), "Column name mismatch at position " + i); assertEquals( expectedCol.dataType(), sparkSchema.fields()[i].dataType(), "Data type mismatch for column: " + expectedCol.name()); // Check column object from table.columns() assertEquals(expectedCol, actualColumns[i], "Column mismatch at position " + i); } // ===== Verify schema consistency with DeltaTableV2 ===== // This ensures SparkTable (Kernel-based) returns the same schema as DeltaTableV2 (V1-based) // Both should properly remove internal Delta metadata (e.g., column mapping physical names) DeltaTableV2 deltaTableV2; switch (method) { case FROM_PATH: deltaTableV2 = DeltaTableV2.apply( spark, new Path(path), Option.empty(), Option.empty(), scala.collection.immutable.Map$.MODULE$.empty(), Option.empty()); break; case FROM_CATALOG_TABLE: deltaTableV2 = DeltaTableV2.apply( spark, new Path(path), Option.apply(catalogTable), Option.apply(tableName), scala.collection.immutable.Map$.MODULE$.empty(), Option.empty()); break; default: throw new IllegalArgumentException("Unknown method: " + method); } // Verify schemas are equal (including field names, types, and metadata) assertEquals( deltaTableV2.schema(), sparkSchema, "SparkTable schema should match DeltaTableV2 schema for test case: " + testCase.name); // ===== Test partitioning ===== Transform[] partitioning = kernelTable.partitioning(); assertEquals(testCase.expectedPartitionColumns.length, partitioning.length); for (int i = 0; i < testCase.expectedPartitionColumns.length; i++) { assertEquals( testCase.expectedPartitionColumns[i], partitioning[i].references()[0].describe(), "Partition column mismatch at position " + i); } // ===== Test properties ===== Map properties = kernelTable.properties(); testCase.expectedProperties.forEach( (key, value) -> { assertTrue(properties.containsKey(key), "Property not found: " + key); assertEquals(value, properties.get(key), "Property value mismatch for: " + key); }); // ===== Test capabilities ===== assertTrue(kernelTable.capabilities().contains(BATCH_READ)); assertTrue(kernelTable.capabilities().contains(BATCH_WRITE)); assertTrue(kernelTable instanceof SupportsWrite); // ===== Test getCatalogTable based on construction method ===== Optional retrievedCatalogTable = kernelTable.getCatalogTable(); switch (method) { case FROM_PATH: assertFalse( retrievedCatalogTable.isPresent(), "Path-based SparkTable should not have catalog table"); break; case FROM_CATALOG_TABLE: assertTrue( retrievedCatalogTable.isPresent(), "CatalogTable-based SparkTable should have catalog table"); assertEquals( catalogTable, retrievedCatalogTable.get(), "Retrieved catalog table should match the original"); break; } // ===== Test getTablePath returns Path from tablePath ===== Path retrievedPath = kernelTable.getTablePath(); assertEquals(new Path(path), retrievedPath, "getTablePath should return Path from tablePath"); } /** Enum to represent different construction methods for SparkTable */ enum ConstructionMethod { FROM_PATH("Path"), FROM_CATALOG_TABLE("CatalogTable"); private final String displayName; ConstructionMethod(String displayName) { this.displayName = displayName; } @Override public String toString() { return displayName; } } /** Represents a test case configuration for Delta tables */ private static class TableTestCase { final String name; final BiFunction createTableSql; final List expectedColumns; final String[] expectedPartitionColumns; final Map expectedProperties; public TableTestCase( String name, BiFunction createTableSql, List expectedColumns, String[] expectedPartitionColumns, Map expectedProperties) { this.name = name; this.createTableSql = createTableSql; this.expectedColumns = expectedColumns; this.expectedPartitionColumns = expectedPartitionColumns; this.expectedProperties = expectedProperties; } @Override public String toString() { return name; } } /** Provides different test cases for Delta tables combined with construction methods */ static Stream tableTestCases() { // ===== Partitioned Table ===== List partitionedTableColumns = new ArrayList<>(); partitionedTableColumns.add(Column.create("id", DataTypes.IntegerType)); partitionedTableColumns.add(Column.create("data", DataTypes.StringType)); partitionedTableColumns.add(Column.create("part", DataTypes.IntegerType)); // ===== Unpartitioned Table ===== List unPartitionedTableColumns = new ArrayList<>(); unPartitionedTableColumns.add(Column.create("id", DataTypes.IntegerType)); unPartitionedTableColumns.add(Column.create("data", DataTypes.StringType)); // ===== Setup Single Properties ===== Map basicProps = new HashMap<>(); basicProps.put("foo", "bar"); // ===== Setup Multiple Properties ===== Map multiProps = new HashMap<>(); multiProps.put("prop1", "value1"); multiProps.put("prop2", "value2"); multiProps.put("delta.enableChangeDataFeed", "true"); List singleColumn = new ArrayList<>(); singleColumn.add(Column.create("id", DataTypes.IntegerType)); // ===== Name Mapping Table ===== List nameMappingTableColumns = new ArrayList<>(); nameMappingTableColumns.add(Column.create("id", DataTypes.IntegerType)); nameMappingTableColumns.add(Column.create("name", DataTypes.StringType)); nameMappingTableColumns.add(Column.create("value", DataTypes.DoubleType)); Map nameMappingProps = new HashMap<>(); nameMappingProps.put("delta.columnMapping.mode", "name"); List testCases = Arrays.asList( new TableTestCase( "Partitioned Table", (tableName, path) -> { spark.sql( String.format( "CREATE TABLE %s (id INT, data STRING, part INT) USING delta " + "PARTITIONED BY (part) TBLPROPERTIES ('foo'='bar') LOCATION '%s'", tableName, path)); return null; }, partitionedTableColumns, new String[] {"part"}, basicProps), new TableTestCase( "UnPartitioned Table", (tableName, path) -> { spark.sql( String.format( "CREATE TABLE %s (id INT, data STRING) USING delta LOCATION '%s'", tableName, path)); return null; }, unPartitionedTableColumns, new String[] {}, new HashMap<>()), new TableTestCase( "Multiple Properties", (tableName, path) -> { spark.sql( String.format( "CREATE TABLE %s (id INT) USING delta " + "TBLPROPERTIES ('prop1'='value1', 'prop2'='value2', 'delta.enableChangeDataFeed'='true') " + "LOCATION '%s'", tableName, path)); return null; }, singleColumn, new String[] {}, multiProps), new TableTestCase( "Name Mapping Table", (tableName, path) -> { spark.sql( String.format( "CREATE TABLE %s (id INT, name STRING, value DOUBLE) USING delta " + "TBLPROPERTIES ('delta.columnMapping.mode'='name') " + "LOCATION '%s'", tableName, path)); spark.sql(String.format("INSERT INTO %s VALUES (1, 'test', 100.0)", tableName)); return null; }, nameMappingTableColumns, new String[] {}, nameMappingProps)); // Create cartesian product of test cases and construction methods return testCases.stream() .flatMap( testCase -> Stream.of(ConstructionMethod.FROM_PATH, ConstructionMethod.FROM_CATALOG_TABLE) .map(method -> Arguments.of(testCase, method))); } /** * Test that getDecodedPath handles various URI schemes correctly, not just file:// URIs. This * verifies the fix for supporting cloud storage paths (s3, abfss, gs) and HDFS. */ @ParameterizedTest(name = "URI scheme: {0}") @MethodSource("uriSchemeTestCases") public void testGetDecodedPathSupportsVariousUriSchemes(String scheme, String uriString) throws Exception { // Access the private static method via reflection Method getDecodedPath = SparkTable.class.getDeclaredMethod("getDecodedPath", java.net.URI.class); getDecodedPath.setAccessible(true); URI uri = new URI(uriString); String result = (String) getDecodedPath.invoke(null, uri); // Verify the path is decoded correctly // The result should contain the path portion without URL encoding issues assertTrue( result.contains("/path/to/table"), "Decoded path should contain the expected path. Got: " + result); } /** Test that URL-encoded characters are properly decoded */ @Test public void testGetDecodedPathDecodesUrlEncodedCharacters() throws Exception { // Access the private static method via reflection Method getDecodedPath = SparkTable.class.getDeclaredMethod("getDecodedPath", java.net.URI.class); getDecodedPath.setAccessible(true); // Test URL-encoded path: "spark%25dir%25prefix" should decode to "spark%dir%prefix" // %25 is the URL encoding for % URI uri = new URI("file:///data/spark%25dir%25prefix/table"); String result = (String) getDecodedPath.invoke(null, uri); // For file URIs, getDecodedPath returns just the path without the scheme assertEquals( "/data/spark%dir%prefix/table", result, "URL-encoded characters should be properly decoded"); } /** Provides test cases for different URI schemes */ static Stream uriSchemeTestCases() { return Stream.of( Arguments.of("file", "file:///path/to/table"), Arguments.of("s3", "s3://bucket/path/to/table"), Arguments.of("s3a", "s3a://bucket/path/to/table"), Arguments.of("abfss", "abfss://container@account.dfs.core.windows.net/path/to/table"), Arguments.of("gs", "gs://bucket/path/to/table"), Arguments.of("hdfs", "hdfs://namenode:8020/path/to/table")); } @Test public void testEqualsAndHashCode(@TempDir File tempDir) { String path = tempDir.getAbsolutePath(); spark.sql(String.format("CREATE TABLE test_equals (id INT) USING delta LOCATION '%s'", path)); Identifier identifier = Identifier.of(new String[] {"default"}, "test_equals"); Map options = Collections.singletonMap("key", "value"); SparkTable table1 = new SparkTable(identifier, path, options); SparkTable table2 = new SparkTable(identifier, path, options); SparkTable table3 = new SparkTable(identifier, path, Collections.emptyMap()); // Same identifier, path, and options should be equal assertEquals(table1, table2); assertEquals(table1.hashCode(), table2.hashCode()); // Different options should not be equal and hashCodes should differ assertNotEquals(table1, table3); assertNotEquals(table1.hashCode(), table3.hashCode()); } @Test public void testEqualsAndHashCodeWithCatalogTable(@TempDir File tempDir) throws Exception { String path1 = new File(tempDir, "table1").getAbsolutePath(); String path2 = new File(tempDir, "table2").getAbsolutePath(); spark.sql( String.format("CREATE TABLE test_catalog1 (id INT) USING delta LOCATION '%s'", path1)); spark.sql( String.format("CREATE TABLE test_catalog2 (id INT) USING delta LOCATION '%s'", path2)); Identifier identifier = Identifier.of(new String[] {"default"}, "test_catalog"); // Create table1 and table2 with separately fetched CatalogTable objects (not same instance) SparkTable table1 = new SparkTable( identifier, spark.sessionState().catalog().getTableMetadata(new TableIdentifier("test_catalog1")), Collections.emptyMap()); SparkTable table2 = new SparkTable( identifier, spark.sessionState().catalog().getTableMetadata(new TableIdentifier("test_catalog1")), Collections.emptyMap()); // Same identifier, catalogTable, and options should be equal assertEquals(table1, table2); assertEquals(table1.hashCode(), table2.hashCode()); // Different catalogTable should not be equal SparkTable table3 = new SparkTable( identifier, spark.sessionState().catalog().getTableMetadata(new TableIdentifier("test_catalog2")), Collections.emptyMap()); assertNotEquals(table1, table3); assertNotEquals(table1.hashCode(), table3.hashCode()); // Path-based table (no catalogTable) should not equal catalog-based table SparkTable table4 = new SparkTable(identifier, path1, Collections.emptyMap()); assertNotEquals(table1, table4); assertNotEquals(table1.hashCode(), table4.hashCode()); } @Test public void testEqualsAndHashCodeWithDifferentSnapshotVersions(@TempDir File tempDir) { String path = tempDir.getAbsolutePath(); spark.sql(String.format("CREATE TABLE test_snapshot (id INT) USING delta LOCATION '%s'", path)); Identifier identifier = Identifier.of(new String[] {"default"}, "test_snapshot"); // Create first SparkTable instance at version 0 SparkTable table1 = new SparkTable(identifier, path); // Modify the table to create a new version spark.sql("INSERT INTO test_snapshot VALUES (1)"); // Create second SparkTable instance at version 1 SparkTable table2 = new SparkTable(identifier, path); // Same identifier and path but different snapshot versions should not be equal assertNotEquals( table1, table2, "SparkTable instances with different snapshot versions should not be equal"); assertNotEquals( table1.hashCode(), table2.hashCode(), "Hash codes should differ for different snapshot versions"); } @Test public void testNewWriteBuilderThrowsUnsupported(@TempDir File tempDir) throws Exception { String path = tempDir.getAbsolutePath(); spark.sql( String.format( "CREATE TABLE test_write_builder_unsupported (id INT) USING delta LOCATION '%s'", path)); SparkTable table = new SparkTable( Identifier.of(new String[] {"default"}, "test_write_builder_unsupported"), path); LogicalWriteInfo writeInfo = new LogicalWriteInfo() { @Override public String queryId() { return "test-query-id"; } @Override public StructType schema() { return new StructType().add("id", DataTypes.IntegerType); } @Override public CaseInsensitiveStringMap options() { return new CaseInsensitiveStringMap(Collections.emptyMap()); } }; UnsupportedOperationException ex = assertThrows(UnsupportedOperationException.class, () -> table.newWriteBuilder(writeInfo)); assertEquals( "Batch write for Delta tables via the DSv2 connector is not yet supported.", ex.getMessage()); } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/catalog/TestCatalog.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.catalog; import io.delta.kernel.Operation; import io.delta.kernel.defaults.engine.DefaultEngine; import io.delta.kernel.engine.Engine; import io.delta.kernel.utils.CloseableIterable; import io.delta.spark.internal.v2.utils.SchemaUtils; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.conf.Configuration; import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.Table; import org.apache.spark.sql.connector.catalog.TableCatalog; import org.apache.spark.sql.connector.catalog.TableChange; import org.apache.spark.sql.connector.expressions.Transform; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; /** * A {@link TableCatalog} implementation that uses Delta Kernel for table operations. This catalog * is used for facilitating testing for spark-dsv2 code path. * *

This catalog is initialized with a base path where all tables will be created. The catalog * maintains a mapping of table identifiers to their physical paths on the filesystem. When a table * is created, it gets a unique subdirectory under the base path to store its data. */ public class TestCatalog implements TableCatalog { /** The name of this catalog instance, set during initialization. */ private String catalogName; /** * The base directory path where all tables created by this catalog will be stored. Each table * gets a unique subdirectory under this path. */ private String basePath; // TODO: Support catalog owned commit. private final Map tablePaths = new ConcurrentHashMap<>(); private final Configuration hadoopConf = new Configuration(); private final Engine engine = DefaultEngine.create(hadoopConf); @Override public Identifier[] listTables(String[] namespace) { throw new UnsupportedOperationException("listTables method is not implemented"); } @Override public Table loadTable(Identifier ident) throws NoSuchTableException { // Check if this is a path-based table identifier String tablePath; if (isPathIdentifier(ident)) { tablePath = ident.name(); } else { // Handle catalog-managed tables String tableKey = getTableKey(ident); tablePath = tablePaths.get(tableKey); } if (tablePath == null) { throw new NoSuchTableException(ident); } try { return new SparkTable(ident, tablePath); } catch (Exception e) { throw new RuntimeException("Failed to load table: " + ident, e); } } @Override public Table createTable( Identifier ident, StructType schema, Transform[] partitions, Map properties) { String tableKey = getTableKey(ident); String tablePath = basePath + UUID.randomUUID() + "/"; tablePaths.put(tableKey, tablePath); try { // TODO: migrate to use CCv2 table io.delta.kernel.Table kernelTable = io.delta.kernel.Table.forPath(engine, tablePath); List partitionColumns = new ArrayList<>(); for (Transform partition : partitions) { // Extract column name from partition transform String columnName = partition.references()[0].describe(); partitionColumns.add(columnName); } // TODO: migrate to use CCv2's committer API io.delta.kernel.Table.forPath(engine, tablePath) .createTransactionBuilder( engine, "kernel-spark-dsv2-test-catalog", Operation.CREATE_TABLE) .withSchema(engine, SchemaUtils.convertSparkSchemaToKernelSchema(schema)) .withPartitionColumns(engine, partitionColumns) .withTableProperties(engine, properties) .build(engine) .commit(engine, CloseableIterable.emptyIterable()); // Load the created table and return SparkTable return new SparkTable(ident, tablePath); } catch (Exception e) { // Remove the table entry if creation fails tablePaths.remove(tableKey); throw new RuntimeException("Failed to create table: " + ident, e); } } @Override public Table alterTable(Identifier ident, TableChange... changes) { throw new UnsupportedOperationException("alterTable method is not implemented"); } @Override public boolean dropTable(Identifier ident) { String tableKey = getTableKey(ident); return tablePaths.remove(tableKey) != null; } @Override public void renameTable(Identifier oldIdent, Identifier newIdent) { throw new UnsupportedOperationException("renameTable method is not implemented"); } @Override public void initialize(String name, CaseInsensitiveStringMap options) { this.catalogName = name; // Use a default path if base_path is not provided this.basePath = options.getOrDefault("base_path", "/tmp/dsv2_test/"); } @Override public String name() { return catalogName; } /** * Check if the given identifier represents a path-based table. Path-based tables are identified * by having a delta namespace. This follows the same logic as Delta Spark's * SupportsPathIdentifier. */ private boolean isPathIdentifier(Identifier ident) { // For testing, simply check if it has a delta namespace return ident.namespace().length == 1 && ident.namespace()[0].toLowerCase().equals("delta"); } /** Helper method to get the table key from identifier. */ private String getTableKey(Identifier ident) { if (ident.namespace().length == 0) { return ident.name(); } else { return String.join(".", ident.namespace()) + "." + ident.name(); } } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkGoldenTableTest.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.read; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import io.delta.golden.GoldenTableUtils$; import io.delta.kernel.expressions.Column; import io.delta.kernel.expressions.Literal; import io.delta.kernel.expressions.Predicate; import io.delta.spark.internal.v2.catalog.SparkTable; import java.io.File; import java.lang.reflect.Field; import java.math.BigDecimal; import java.util.*; import org.apache.hadoop.conf.Configuration; import org.apache.spark.SparkConf; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.QueryTest$; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.catalyst.expressions.GenericRow; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.expressions.Expression; import org.apache.spark.sql.connector.read.Scan; import org.apache.spark.sql.connector.read.ScanBuilder; import org.apache.spark.sql.sources.*; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.io.TempDir; @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class SparkGoldenTableTest { private SparkSession spark; @BeforeAll public void setUp(@TempDir File tempDir) { SparkConf conf = new SparkConf() .set("spark.sql.catalog.dsv2", "io.delta.spark.internal.v2.catalog.TestCatalog") .set("spark.sql.catalog.dsv2.base_path", tempDir.getAbsolutePath()) .set("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtensionV1") .set( "spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalogV1") .setMaster("local[*]") .setAppName("SparkGoldenTableTest"); spark = SparkSession.builder().config(conf).getOrCreate(); } @AfterAll public void tearDown() { if (spark != null) { spark.stop(); spark = null; } } /** Helper method to check DataFrame results against expected rows. */ private void checkAnswer(Dataset df, List expected) { QueryTest$.MODULE$.checkAnswer(df, expected); } @Test public void testDsv2Internal() throws Exception { String tableName = "deltatbl-partition-prune"; String tablePath = goldenTablePath("hive/" + tableName); CaseInsensitiveStringMap options = new CaseInsensitiveStringMap( new java.util.HashMap() { { put("key1", "value1"); put("key2", "value2"); } }); SparkTable table = new SparkTable( Identifier.of(new String[] {"spark_catalog", "default"}, tableName), tablePath, options); StructType expectedDataSchema = DataTypes.createStructType( new StructField[] { DataTypes.createStructField("name", DataTypes.StringType, true), DataTypes.createStructField("cnt", DataTypes.IntegerType, true), }); StructType expectedPartitionSchema = DataTypes.createStructType( new StructField[] { DataTypes.createStructField("date", DataTypes.StringType, true), DataTypes.createStructField("city", DataTypes.StringType, true), }); StructType expectedSchema = DataTypes.createStructType( new StructField[] { expectedPartitionSchema.fields()[1], expectedPartitionSchema.fields()[0], expectedDataSchema.fields()[0], expectedDataSchema.fields()[1] }); assertEquals(expectedSchema, table.schema()); assertEquals(String.format("delta.`%s`", tablePath), table.name()); // Check table columns assertEquals(4, table.columns().length); assertEquals("city", table.columns()[0].name()); assertEquals("date", table.columns()[1].name()); assertEquals("name", table.columns()[2].name()); assertEquals("cnt", table.columns()[3].name()); // Check table partitioning assertEquals(2, table.partitioning().length); assertEquals("identity(date)", table.partitioning()[0].toString()); assertEquals("identity(city)", table.partitioning()[1].toString()); // Check table properties assertEquals(Map.of(), table.properties()); CaseInsensitiveStringMap scanOptions = new CaseInsensitiveStringMap( new java.util.HashMap() { { put("key3", "value3"); put("key2", "new_value2"); } }); ScanBuilder builder = table.newScanBuilder(scanOptions); assertTrue((builder instanceof SparkScanBuilder)); SparkScanBuilder scanBuilder = (SparkScanBuilder) builder; assertEquals(expectedDataSchema, scanBuilder.getDataSchema()); assertEquals(expectedPartitionSchema, scanBuilder.getPartitionSchema()); CaseInsensitiveStringMap combinedOptions = new CaseInsensitiveStringMap( new java.util.HashMap() { { put("key1", "value1"); put("key2", "new_value2"); put("key3", "value3"); } }); assertEquals(combinedOptions, scanBuilder.getOptions()); Scan scan1 = scanBuilder.build(); assertTrue(scan1 instanceof SparkScan); SparkScan sparkScan1 = (SparkScan) scan1; assertEquals(expectedDataSchema, sparkScan1.getDataSchema()); assertEquals(expectedDataSchema, sparkScan1.getReadDataSchema()); assertEquals(expectedPartitionSchema, sparkScan1.getPartitionSchema()); assertEquals(combinedOptions, sparkScan1.getOptions()); verifyHadoopConf(sparkScan1.getConfiguration()); // check SupportsPushDownRequiredColumns StructType prunedSchema = DataTypes.createStructType( new StructField[] { expectedDataSchema.fields()[0], expectedPartitionSchema.fields()[0], }); scanBuilder.pruneColumns(prunedSchema); Scan scan2 = scanBuilder.build(); assertTrue(scan2 instanceof SparkScan); SparkScan sparkScan2 = (SparkScan) scan2; assertEquals(expectedDataSchema, sparkScan2.getDataSchema()); StructType expectedReadDataSchemaAfterPrune = DataTypes.createStructType(new StructField[] {expectedDataSchema.fields()[0]}); assertEquals(expectedReadDataSchemaAfterPrune, sparkScan2.getReadDataSchema()); assertEquals(combinedOptions, sparkScan2.getOptions()); verifyHadoopConf(sparkScan2.getConfiguration()); // check SupportsPushDownFilters // case 1: mix of supported and unsupported, data and partition filters checkSupportsPushDownFilters( table, scanOptions, // input filters new Filter[] { new GreaterThan("cnt", 10), // supported data filter new StringStartsWith("name", "foo"), // supported data filter new EqualTo("date", "2025-09-01"), // supported partition filter new StringEndsWith("city", "York"), // unsupported partition filter }, // expected post-scan filters new Filter[] { new GreaterThan("cnt", 10), new StringStartsWith("name", "foo"), new StringEndsWith("city", "York"), }, // expected pushed filters new Filter[] { new GreaterThan("cnt", 10), new StringStartsWith("name", "foo"), new EqualTo("date", "2025-09-01") }, // expected pushed kernel predicates new Predicate[] { new Predicate(">", new Column("cnt"), Literal.ofInt(10)), new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("foo")), new Predicate("=", new Column("date"), Literal.ofString("2025-09-01")) }, // expected data filters new Filter[] {new GreaterThan("cnt", 10), new StringStartsWith("name", "foo")}, // expected kernel scan builder predicate Optional.of( new Predicate( "AND", new Predicate( "AND", new Predicate(">", new Column("cnt"), Literal.ofInt(10)), new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("foo"))), new Predicate("=", new Column("date"), Literal.ofString("2025-09-01"))))); // case 2: OR and NOT filters checkSupportsPushDownFilters( table, scanOptions, // input filters new Filter[] { new Or(new GreaterThan("cnt", 10), new StringStartsWith("name", "foo")), new Or(new EqualTo("cnt", 50), new EqualTo("date", "2025-10-01")), new Not(new And(new GreaterThan("cnt", 100), new EqualTo("date", "2025-09-01"))), new Not(new Or(new EqualTo("name", "foo"), new StringStartsWith("city", "New"))) }, // expected post-scan filters new Filter[] { new Or(new GreaterThan("cnt", 10), new StringStartsWith("name", "foo")), new Or(new EqualTo("cnt", 50), new EqualTo("date", "2025-10-01")), new Not(new And(new GreaterThan("cnt", 100), new EqualTo("date", "2025-09-01"))), new Not(new Or(new EqualTo("name", "foo"), new StringStartsWith("city", "New"))) }, // expected pushed filters new Filter[] { new Or(new GreaterThan("cnt", 10), new StringStartsWith("name", "foo")), new Or(new EqualTo("cnt", 50), new EqualTo("date", "2025-10-01")), new Not(new And(new GreaterThan("cnt", 100), new EqualTo("date", "2025-09-01"))), new Not(new Or(new EqualTo("name", "foo"), new StringStartsWith("city", "New"))) }, // expected pushed kernel predicates new Predicate[] { new Predicate( "OR", new Predicate(">", new Column("cnt"), Literal.ofInt(10)), new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("foo"))), new Predicate( "OR", new Predicate("=", new Column("cnt"), Literal.ofInt(50)), new Predicate("=", new Column("date"), Literal.ofString("2025-10-01"))), new Predicate( "NOT", new Predicate( "AND", new Predicate(">", new Column("cnt"), Literal.ofInt(100)), new Predicate("=", new Column("date"), Literal.ofString("2025-09-01")))), new Predicate( "NOT", new Predicate( "OR", new Predicate("=", new Column("name"), Literal.ofString("foo")), new Predicate("STARTS_WITH", new Column("city"), Literal.ofString("New")))) }, // expected data filters new Filter[] { new Or(new GreaterThan("cnt", 10), new StringStartsWith("name", "foo")), new Or(new EqualTo("cnt", 50), new EqualTo("date", "2025-10-01")), new Not(new And(new GreaterThan("cnt", 100), new EqualTo("date", "2025-09-01"))), new Not(new Or(new EqualTo("name", "foo"), new StringStartsWith("city", "New"))) }, // expected kernel scan builder predicate // reduce(And::new) over 4 predicates gives left-associative nesting: // AND(AND(AND(pred1, pred2), pred3), pred4) Optional.of( new Predicate( "AND", new Predicate( "AND", new Predicate( "AND", new Predicate( "OR", new Predicate(">", new Column("cnt"), Literal.ofInt(10)), new Predicate( "STARTS_WITH", new Column("name"), Literal.ofString("foo"))), new Predicate( "OR", new Predicate("=", new Column("cnt"), Literal.ofInt(50)), new Predicate( "=", new Column("date"), Literal.ofString("2025-10-01")))), new Predicate( "NOT", new Predicate( "AND", new Predicate(">", new Column("cnt"), Literal.ofInt(100)), new Predicate( "=", new Column("date"), Literal.ofString("2025-09-01"))))), new Predicate( "NOT", new Predicate( "OR", new Predicate("=", new Column("name"), Literal.ofString("foo")), new Predicate( "STARTS_WITH", new Column("city"), Literal.ofString("New"))))))); // check SupportsRuntimeV2Filtering // city = 'hz' AND date = '20180520' org.apache.spark.sql.connector.expressions.filter.Predicate andPredicate = new org.apache.spark.sql.connector.expressions.filter.Predicate( "AND", new Expression[] {SparkScanTest.cityPredicate, SparkScanTest.datePredicate}); SparkScanTest.checkSupportsRuntimeFilters( table, options, new org.apache.spark.sql.connector.expressions.filter.Predicate[] {andPredicate}, Arrays.asList("date=20180520/city=hz")); // city = 'hz' OR date = '20180520' org.apache.spark.sql.connector.expressions.filter.Predicate orPredicate = new org.apache.spark.sql.connector.expressions.filter.Predicate( "OR", new Expression[] {SparkScanTest.cityPredicate, SparkScanTest.datePredicate}); SparkScanTest.checkSupportsRuntimeFilters( table, scanOptions, new org.apache.spark.sql.connector.expressions.filter.Predicate[] {orPredicate}, Arrays.asList("city=hz", "date=20180520")); // city = 'hz', cnt > 10 SparkScanTest.checkSupportsRuntimeFilters( table, options, new org.apache.spark.sql.connector.expressions.filter.Predicate[] { SparkScanTest.cityPredicate, SparkScanTest.dataPredicate }, Arrays.asList("city=hz")); // city = 'hz' OR cnt > 10 org.apache.spark.sql.connector.expressions.filter.Predicate orDataPredicate = new org.apache.spark.sql.connector.expressions.filter.Predicate( "OR", new Expression[] {SparkScanTest.cityPredicate, SparkScanTest.dataPredicate}); SparkScanTest.checkSupportsRuntimeFilters( table, options, new org.apache.spark.sql.connector.expressions.filter.Predicate[] {orDataPredicate}, SparkScanTest.allCities); // city = date SparkScanTest.checkSupportsRuntimeFilters( table, options, new org.apache.spark.sql.connector.expressions.filter.Predicate[] { SparkScanTest.negativeInterColPredicate }, Arrays.asList()); // city <> date SparkScanTest.checkSupportsRuntimeFilters( table, options, new org.apache.spark.sql.connector.expressions.filter.Predicate[] { SparkScanTest.interColPredicate }, SparkScanTest.allCities); } private void checkSupportsPushDownFilters( SparkTable table, CaseInsensitiveStringMap scanOptions, Filter[] inputFilters, Filter[] expectedPostScanFilters, Filter[] expectedPushedFilters, Predicate[] expectedPushedKernelPredicates, Filter[] expectedDataFilters, Optional expectedKernelScanBuilderPredicate) throws Exception { ScanBuilder newBuilder = table.newScanBuilder(scanOptions); SparkScanBuilder builder = (SparkScanBuilder) newBuilder; Filter[] postScanFilters = builder.pushFilters(inputFilters); assertEquals( new HashSet<>(Arrays.asList(expectedPostScanFilters)), new HashSet<>(Arrays.asList(postScanFilters))); assertEquals( new HashSet<>(Arrays.asList(expectedPushedFilters)), new HashSet<>(Arrays.asList(builder.pushedFilters()))); Predicate[] pushedPredicates = getPushedKernelPredicates(builder); assertEquals( new HashSet<>(Arrays.asList(expectedPushedKernelPredicates)), new HashSet<>(Arrays.asList(pushedPredicates))); Filter[] dataFilters = getDataFilters(builder); assertEquals( new HashSet<>(Arrays.asList(expectedDataFilters)), new HashSet<>(Arrays.asList(dataFilters))); Optional predicateOpt = getKernelScanBuilderPredicate(builder); assertEquals(expectedKernelScanBuilderPredicate, predicateOpt); } private Predicate[] getPushedKernelPredicates(SparkScanBuilder builder) throws Exception { Field field = SparkScanBuilder.class.getDeclaredField("pushedKernelPredicates"); field.setAccessible(true); return (Predicate[]) field.get(builder); } private Filter[] getDataFilters(SparkScanBuilder builder) throws Exception { Field field = SparkScanBuilder.class.getDeclaredField("dataFilters"); field.setAccessible(true); return (Filter[]) field.get(builder); } private Optional getKernelScanBuilderPredicate(SparkScanBuilder builder) throws Exception { Field field = SparkScanBuilder.class.getDeclaredField("kernelScanBuilder"); field.setAccessible(true); Object kernelScanBuilder = field.get(builder); Field predicateField = kernelScanBuilder.getClass().getDeclaredField("predicate"); predicateField.setAccessible(true); Object raw = predicateField.get(kernelScanBuilder); if (raw == null) { return Optional.empty(); } Optional opt = (Optional) raw; return opt.map(Predicate.class::cast); } @Test public void testDsv2InteralWithNestedStruct() { String tableName = "data-reader-nested-struct"; String tablePath = goldenTablePath(tableName); SparkTable table = new SparkTable( Identifier.of(new String[] {"spark_catalog", "default"}, tableName), tablePath); StructType expectedSchema = StructType.fromDDL( "a STRUCT>,b INT"); assertEquals(expectedSchema, table.schema()); assertEquals(String.format("delta.`%s`", tablePath), table.name()); assertEquals(0, table.partitioning().length); CaseInsensitiveStringMap options = new CaseInsensitiveStringMap( java.util.Collections.singletonMap("another_option_key", "another_option_value")); ScanBuilder builder = table.newScanBuilder(options); assertTrue((builder instanceof SparkScanBuilder)); SparkScanBuilder scanBuilder = (SparkScanBuilder) builder; assertEquals(expectedSchema, scanBuilder.getDataSchema()); assertTrue(scanBuilder.getPartitionSchema().isEmpty()); assertEquals(options, scanBuilder.getOptions()); // Initial scan (no pruning) Scan scan1 = scanBuilder.build(); assertTrue(scan1 instanceof SparkScan); SparkScan sparkScan1 = (SparkScan) scan1; assertEquals(expectedSchema, sparkScan1.getDataSchema()); assertEquals(expectedSchema, sparkScan1.getReadDataSchema()); assertTrue(sparkScan1.getPartitionSchema().isEmpty()); assertEquals(options, sparkScan1.getOptions()); StructType prunedSchema = StructType.fromDDL("a STRUCT"); scanBuilder.pruneColumns(prunedSchema); Scan scan2 = scanBuilder.build(); assertTrue(scan2 instanceof SparkScan); SparkScan sparkScan2 = (SparkScan) scan2; assertEquals(expectedSchema, sparkScan2.getDataSchema()); assertEquals(prunedSchema, sparkScan2.getReadDataSchema()); assertTrue(sparkScan2.getPartitionSchema().isEmpty()); assertEquals(options, sparkScan2.getOptions()); } @Test public void testTablePrimitives() throws Exception { List expected = new ArrayList<>(); for (int i = 0; i <= 10; i++) { if (i == 10) { expected.add( new GenericRow( new Object[] {null, null, null, null, null, null, null, null, null, null})); } else { expected.add( new GenericRow( new Object[] { i, (long) i, (byte) i, (short) i, i % 2 == 0, (float) i, (double) i, Integer.toString(i), new byte[] {(byte) i, (byte) i}, new BigDecimal(i) })); } } checkTable("data-reader-primitives", expected); } @Test public void testTableWithNestedStruct() { List expected = new ArrayList<>(); for (int i = 0; i < 10; i++) { Row innerMost = new GenericRow(new Object[] {i, (long) i}); Row middle = new GenericRow(new Object[] {Integer.toString(i), Integer.toString(i), innerMost}); expected.add(new GenericRow(new Object[] {middle, i})); } // Assuming `checkTable` is made accessible (e.g., protected in base class) checkTable("data-reader-nested-struct", expected); } @Test public void testPartitionedTable() { // Build expected rows (excluding unsupported partition column `as_timestamp`) List expected = new ArrayList<>(); java.sql.Date fixedDate = java.sql.Date.valueOf("2021-09-08"); for (int i = 0; i < 2; i++) { // Array field: Seq(TestRow(i), TestRow(i), TestRow(i)) where TestRow(i) => struct(i) List arrElems = Arrays.asList( new GenericRow(new Object[] {i}), new GenericRow(new Object[] {i}), new GenericRow(new Object[] {i})); Object arrSeq = scala.collection.JavaConverters.asScalaBuffer(arrElems).toList(); // Nested struct: TestRow(i.toString, i.toString, TestRow(i, i.toLong)) Row innerMost = new GenericRow(new Object[] {i, (long) i}); Row middle = new GenericRow(new Object[] {Integer.toString(i), Integer.toString(i), innerMost}); expected.add( new GenericRow( new Object[] { i, // int (long) i, // long (byte) i, // byte (short) i, // short i % 2 == 0, // boolean (float) i, // float (double) i, // double Integer.toString(i), // string "null", // literal string fixedDate, // date (was daysSinceEpoch int) new BigDecimal(i), // decimal arrSeq, // array middle, // nested struct Integer.toString(i) // final string })); } // Null row variant with specific non-null complex fields (matches Scala test) List nullArrElems = Arrays.asList( new GenericRow(new Object[] {2}), new GenericRow(new Object[] {2}), new GenericRow(new Object[] {2})); Object nullArrSeq = scala.collection.JavaConverters.asScalaBuffer(nullArrElems).toList(); Row nullInnerMost = new GenericRow(new Object[] {2, 2L}); Row nullMiddle = new GenericRow(new Object[] {"2", "2", nullInnerMost}); expected.add( new GenericRow( new Object[] { null, null, null, null, null, null, null, null, null, null, null, nullArrSeq, nullMiddle, "2" })); // Read table, drop unsupported column `as_timestamp` String tablePath = goldenTablePath("data-reader-partition-values"); Dataset full = spark.sql("SELECT * FROM `dsv2`.`delta`.`" + tablePath + "`"); List projectedCols = new ArrayList<>(); for (String f : full.schema().fieldNames()) { if (!f.equals("as_timestamp")) { projectedCols.add(f); } } Dataset df = full.selectExpr(projectedCols.toArray(new String[0])); checkAnswer(df, expected); } @Test public void testVariantTypeTable() { String tablePath = goldenTablePath("spark-variant-checkpoint"); Dataset df = spark.sql("SELECT * FROM `dsv2`.`delta`.`" + tablePath + "`"); // Verify schema: id (long) + 6 variant/nested-variant columns StructType schema = df.schema(); assertEquals(7, schema.fields().length); assertEquals(DataTypes.LongType, schema.apply("id").dataType()); assertEquals(DataTypes.VariantType, schema.apply("v").dataType()); assertEquals( DataTypes.createArrayType(DataTypes.VariantType, true), schema.apply("array_of_variants").dataType()); assertEquals( DataTypes.createStructType( new StructField[] { new StructField( "v", DataTypes.VariantType, true, org.apache.spark.sql.types.Metadata.empty()) }), schema.apply("struct_of_variants").dataType()); assertEquals( DataTypes.createMapType(DataTypes.StringType, DataTypes.VariantType, true), schema.apply("map_of_variants").dataType()); // Verify row count: 100 base rows + 2 appended rows assertEquals(102, df.count()); // Verify id values are readable (non-variant column) List ids = df.select("id").orderBy("id").limit(3).collectAsList(); assertEquals(0L, ids.get(0).getLong(0)); assertEquals(0L, ids.get(1).getLong(0)); assertEquals(1L, ids.get(2).getLong(0)); // Verify all variant column values. Each variant value is parse_json('{"key": id}'), // so variant_get(..., '$.key', 'long') must equal id for all rows. // - v: direct variant // - array_of_variants[0]: first element (indices 1, 3 are null) // - struct_of_variants.v: struct field // - map_of_variants[CAST(id AS STRING)]: map value by string key // - array_of_struct_of_variants[0].v: first struct element's variant field // - struct_of_array_of_variants.v[1]: struct's array field at index 1 (index 0 is null) long matchingRows = df.where( "variant_get(v, '$.key', 'long') = id" + " AND variant_get(array_of_variants[0], '$.key', 'long') = id" + " AND variant_get(struct_of_variants.v, '$.key', 'long') = id" + " AND variant_get(map_of_variants[CAST(id AS STRING)], '$.key', 'long') = id" + " AND variant_get(array_of_struct_of_variants[0].v, '$.key', 'long') = id" + " AND variant_get(struct_of_array_of_variants.v[1], '$.key', 'long') = id") .count(); assertEquals(102, matchingRows); // Verify known null values within variant columns: // - array_of_variants[1] and [3]: null array elements // - map_of_variants['nullKey']: null map value // - array_of_struct_of_variants[1].v: non-null struct but null variant field // - array_of_struct_of_variants[2]: null struct element // - struct_of_array_of_variants.v[0]: null first element of struct's array field long nullMatchingRows = df.where( "array_of_variants[1] IS NULL" + " AND array_of_variants[3] IS NULL" + " AND map_of_variants['nullKey'] IS NULL" + " AND array_of_struct_of_variants[1].v IS NULL" + " AND array_of_struct_of_variants[2] IS NULL" + " AND struct_of_array_of_variants.v[0] IS NULL") .count(); assertEquals(102, nullMatchingRows); } @Test public void testAllGoldenTables() { List tableNames = getAllGoldenTableNames(); List unsupportedTables = Arrays.asList( "canonicalized-paths-normal-a", "canonicalized-paths-normal-b", "canonicalized-paths-special-a", "canonicalized-paths-special-b", "checkpoint", "corrupted-last-checkpoint", "data-reader-absolute-paths-escaped-chars", "data-reader-escaped-chars", // File delete-re-add-same-file-different-transactions/bar does not exist "delete-re-add-same-file-different-transactions", // Root node at key schemaString is null but field isn't nullable "deltalog-commit-info", // [DELTA_INVALID_PROTOCOL_VERSION] Unsupported Delta protocol version "deltalog-invalid-protocol-version", // [DELTA_STATE_RECOVER_ERROR] The metadata of your Delta table could not be recovered // while Reconstructing "deltalog-state-reconstruction-from-checkpoint-missing-metadata", // [DELTA_STATE_RECOVER_ERROR] The protocol of your Delta table could not be recovered // while Reconstructing "deltalog-state-reconstruction-from-checkpoint-missing-protocol"); for (String tableName : tableNames) { if (unsupportedTables.contains(tableName)) { continue; } // For simplicity, just check that we can read the table and it has at least one row String tablePath = goldenTablePath(tableName); // Many golden tables only have corrupted _delta_log subdir. The new kernel table reader will // fail on some of those. // TODO: fix the read result of those tables. if (hasOnlyDeltaLogSubdir(tablePath)) { continue; } Dataset df = spark.sql("SELECT * FROM `spark_catalog`.`delta`.`" + tablePath + "`"); Dataset df2 = spark.sql("SELECT * FROM `dsv2`.`delta`.`" + tablePath + "`"); assertEquals(df.schema(), df2.schema(), "Schema mismatch for table: " + tableName); checkAnswer(df2, df.collectAsList()); } } private void verifyHadoopConf(Configuration conf) { assertEquals("value1", conf.get("key1")); assertEquals("new_value2", conf.get("key2")); assertEquals("value3", conf.get("key3")); } private boolean hasOnlyDeltaLogSubdir(String path) { File dir = new File(path); if (!dir.exists() || !dir.isDirectory()) { return false; } File[] subFiles = dir.listFiles(File::isDirectory); if (subFiles == null) { return false; } // Check: only one subdirectory, and it is "_delta_log" return subFiles.length == 1 && "_delta_log".equals(subFiles[0].getName()); } private void checkTable(String path, List expected) { String tablePath = goldenTablePath(path); Dataset df = spark.sql("SELECT * FROM `dsv2`.`delta`.`" + tablePath + "`"); checkAnswer(df, expected); } private String goldenTablePath(String name) { return GoldenTableUtils$.MODULE$.goldenTablePath(name); } private List getAllGoldenTableNames() { return scala.collection.JavaConverters.seqAsJavaList(GoldenTableUtils$.MODULE$.allTableNames()); } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.read; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.*; import io.delta.kernel.CommitActions; import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.utils.CloseableIterator; import io.delta.spark.internal.v2.DeltaV2TestBase; import io.delta.spark.internal.v2.snapshot.PathBasedSnapshotManager; import io.delta.spark.internal.v2.utils.ScalaUtils; import java.io.File; import java.nio.channels.ClosedByInterruptException; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.catalog.CatalogTable; import org.apache.spark.sql.catalyst.expressions.Expression; import org.apache.spark.sql.connector.read.InputPartition; import org.apache.spark.sql.connector.read.PartitionReader; import org.apache.spark.sql.connector.read.PartitionReaderFactory; import org.apache.spark.sql.connector.read.streaming.Offset; import org.apache.spark.sql.connector.read.streaming.ReadLimit; import org.apache.spark.sql.delta.*; import org.apache.spark.sql.delta.sources.DeltaSQLConf; import org.apache.spark.sql.delta.sources.DeltaSource; import org.apache.spark.sql.delta.sources.DeltaSourceOffset; import org.apache.spark.sql.delta.sources.ReadMaxBytes; import org.apache.spark.sql.delta.storage.ClosableIterator; import org.apache.spark.sql.delta.util.JsonUtils; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import scala.Option; import scala.collection.JavaConverters; import scala.collection.immutable.Map$; import scala.collection.immutable.Seq; public class SparkMicroBatchStreamTest extends DeltaV2TestBase { /** * Helper method to create a minimal SparkMicroBatchStream instance for tests that only check for * UnsupportedOperationException. */ private SparkMicroBatchStream createTestStream(File tempDir) { String tablePath = tempDir.getAbsolutePath(); String tableName = "test_unsupported_" + System.nanoTime(); createEmptyTestTable(tablePath, tableName); Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(tablePath, hadoopConf); return createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); } private DeltaOptions emptyDeltaOptions() { return new DeltaOptions(Map$.MODULE$.empty(), spark.sessionState().conf()); } @Test public void testLatestOffset_throwsUnsupportedOperationException(@TempDir File tempDir) { SparkMicroBatchStream microBatchStream = createTestStream(tempDir); IllegalStateException exception = assertThrows(IllegalStateException.class, () -> microBatchStream.latestOffset()); } @Test public void testInitialOffset_withInitialSnapshot(@TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_initial_snapshot_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); // Insert some data to create versions insertVersions( testTableName, /* numVersions= */ 3, /* rowsPerVersion= */ 10, /* includeEmptyVersion= */ false); // Create stream without startingVersion (emptyDeltaOptions) Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); // Get the latest version DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); long latestVersion = deltaLog.update(false, Option.empty(), Option.empty()).version(); // Call initialOffset - should create initial snapshot at latest version Offset initialOffset = stream.initialOffset(); assertNotNull(initialOffset, "Initial offset should not be null"); DeltaSourceOffset deltaOffset = (DeltaSourceOffset) initialOffset; assertEquals( latestVersion, deltaOffset.reservoirVersion(), "Initial offset should be at latest version"); assertEquals( DeltaSourceOffset.BASE_INDEX(), deltaOffset.index(), "Initial offset should start at BASE_INDEX"); assertTrue( deltaOffset.isInitialSnapshot(), "Initial offset should be marked as initial snapshot"); } @Test public void testDeserializeOffset_ValidJson(@TempDir File tempDir) throws Exception { String tablePath = tempDir.getAbsolutePath(); SparkMicroBatchStream stream = createTestStream(tempDir); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(tablePath)); String tableId = deltaLog.tableId(); DeltaSourceOffset expected = new DeltaSourceOffset(tableId, 5L, 10L, false); String json = org.apache.spark.sql.delta.util.JsonUtils.mapper().writeValueAsString(expected); Offset result = stream.deserializeOffset(json); DeltaSourceOffset actual = (DeltaSourceOffset) result; assertEquals(expected.reservoirId(), actual.reservoirId()); assertEquals(expected.reservoirVersion(), actual.reservoirVersion()); assertEquals(expected.index(), actual.index()); assertEquals(expected.isInitialSnapshot(), actual.isInitialSnapshot()); } @Test public void testDeserializeOffset_MismatchedTableId(@TempDir File tempDir) throws Exception { SparkMicroBatchStream stream = createTestStream(tempDir); // Create offset with wrong tableId String wrongTableId = "wrong-table-id"; DeltaSourceOffset offset = new DeltaSourceOffset( wrongTableId, /* reservoirVersion= */ 1L, /* index= */ 0L, /* isInitialSnapshot= */ false); String json = JsonUtils.mapper().writeValueAsString(offset); RuntimeException exception = assertThrows(RuntimeException.class, () -> stream.deserializeOffset(json)); assertTrue( exception .getMessage() .contains("streaming query was reading from an unexpected Delta table")); } @Test public void testDeserializeOffset_InvalidJson(@TempDir File tempDir) { SparkMicroBatchStream stream = createTestStream(tempDir); String invalidJson = "{this is not valid json}"; assertThrows(RuntimeException.class, () -> stream.deserializeOffset(invalidJson)); } @Test public void testDeserializeOffset_WithInitialSnapshot(@TempDir File tempDir) throws Exception { String tablePath = tempDir.getAbsolutePath(); SparkMicroBatchStream stream = createTestStream(tempDir); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(tablePath)); String tableId = deltaLog.tableId(); long baseIndex = DeltaSourceOffset.BASE_INDEX(); DeltaSourceOffset expected = new DeltaSourceOffset( tableId, /* reservoirVersion= */ 0L, baseIndex, /* isInitialSnapshot= */ true); String json = org.apache.spark.sql.delta.util.JsonUtils.mapper().writeValueAsString(expected); Offset result = stream.deserializeOffset(json); DeltaSourceOffset actual = (DeltaSourceOffset) result; assertTrue(actual.isInitialSnapshot()); assertEquals(0L, actual.reservoirVersion()); assertEquals(baseIndex, actual.index()); } @Test public void testCommit_NoOp(@TempDir File tempDir) throws Exception { String tablePath = tempDir.getAbsolutePath(); SparkMicroBatchStream stream = createTestStream(tempDir); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(tablePath)); String tableId = deltaLog.tableId(); DeltaSourceOffset offset = new DeltaSourceOffset(tableId, 1L, 0L, false); assertDoesNotThrow(() -> stream.commit(offset)); } @Test public void testStop_NoOp(@TempDir File tempDir) { SparkMicroBatchStream stream = createTestStream(tempDir); assertDoesNotThrow(() -> stream.stop()); } // ================================================================================================ // Tests for initialOffset parity between DSv1 and DSv2 // ================================================================================================ @ParameterizedTest @MethodSource("initialOffsetParameters") public void testInitialOffset_firstBatchParity( String startingVersion, ReadLimitConfig limitConfig, String testDescription, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_initial_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); insertVersions( testTableName, /* numVersions= */ 5, /* rowsPerVersion= */ 10, /* includeEmptyVersion= */ false); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); ReadLimit readLimit = limitConfig.toReadLimit(); DeltaOptions options; if (startingVersion == null) { options = emptyDeltaOptions(); } else { scala.collection.immutable.Map scalaMap = Map$.MODULE$.empty().updated("startingVersion", startingVersion); options = new DeltaOptions(scalaMap, spark.sessionState().conf()); } // DSv1 DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath, options); // DSv1 sources don't have an initialOffset() method. // Batch 0 is called with startOffset=null. Offset dsv1Offset = deltaSource.latestOffset(/* startOffset= */ null, readLimit); // DSv2 Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, options); Offset initialOffset = stream.initialOffset(); Offset dsv2Offset = stream.latestOffset(initialOffset, readLimit); compareOffsets(dsv1Offset, dsv2Offset, testDescription); } /** Provides test parameters for the initialOffset parity test. */ private static Stream initialOffsetParameters() { return Stream.of( // Initial snapshot cases (no startingVersion) Arguments.of(null, ReadLimitConfig.noLimit(), "InitialSnapshot_NoLimit"), Arguments.of(null, ReadLimitConfig.maxFiles(5), "InitialSnapshot_MaxFiles"), Arguments.of(null, ReadLimitConfig.maxBytes(1000), "InitialSnapshot_MaxBytes"), // Specific version cases Arguments.of("0", ReadLimitConfig.noLimit(), "NoLimit1"), Arguments.of("1", ReadLimitConfig.noLimit(), "NoLimit2"), Arguments.of("3", ReadLimitConfig.noLimit(), "NoLimit3"), Arguments.of("latest", ReadLimitConfig.noLimit(), "LatestNoLimit"), Arguments.of("latest", ReadLimitConfig.maxFiles(1000), "LatestMaxFiles"), Arguments.of("latest", ReadLimitConfig.maxBytes(1000), "LatestMaxBytes"), Arguments.of("0", ReadLimitConfig.maxFiles(5), "MaxFiles1"), Arguments.of("1", ReadLimitConfig.maxFiles(10), "MaxFiles2"), Arguments.of("0", ReadLimitConfig.maxBytes(1000), "MaxBytes1"), Arguments.of("1", ReadLimitConfig.maxBytes(2000), "MaxBytes2")); } // ================================================================================================ // Tests for getFileChanges parity between DSv1 and DSv2 // ================================================================================================ /** * Parameterized test that verifies parity between DSv1 DeltaSource.getFileChanges and DSv2 * SparkMicroBatchStream.getFileChanges using Delta Kernel APIs. * *

Tests both regular delta log streaming and initial snapshot scenarios. * *

TODO(#5319): consider adding a test similar to SparkGoldenTableTest.java. * *

TODO(#5318): add tests for ccv2 tables once we fully support them. */ @ParameterizedTest @MethodSource("getFileChangesParameters") public void testGetFileChanges( long fromVersion, long fromIndex, boolean isInitialSnapshot, Optional endVersion, Optional endIndex, String testDescription, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); // Use unique table name per test instance to avoid conflicts String testTableName = "test_file_changes_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); // Create 5 versions of data (versions 1-5, version 0 is the CREATE TABLE) // Insert 100 rows per commit to potentially trigger multiple batches insertVersions( testTableName, /* numVersions= */ 5, /* rowsPerVersion= */ 100, /* includeEmptyVersion= */ false); // dsv1 DeltaSource DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath); Option scalaEndOffset = Option.empty(); if (endVersion.isPresent()) { long offsetIndex = endIndex.orElse(DeltaSourceOffset.END_INDEX()); scalaEndOffset = Option.apply( new DeltaSourceOffset( deltaLog.tableId(), endVersion.get(), offsetIndex, isInitialSnapshot)); } ClosableIterator deltaChanges = deltaSource.getFileChanges( fromVersion, fromIndex, isInitialSnapshot, scalaEndOffset, /* verifyMetadataAction= */ true); List deltaFilesList = new ArrayList<>(); while (deltaChanges.hasNext()) { deltaFilesList.add(deltaChanges.next()); } deltaChanges.close(); // dsv2 SparkMicroBatchStream Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); Optional endOffsetOption = ScalaUtils.toJavaOptional(scalaEndOffset); try (CloseableIterator kernelChanges = stream.getFileChanges(fromVersion, fromIndex, isInitialSnapshot, endOffsetOption)) { List kernelFilesList = new ArrayList<>(); while (kernelChanges.hasNext()) { kernelFilesList.add(kernelChanges.next()); } compareFileChanges(deltaFilesList, kernelFilesList); } } /** Provides test parameters for the parameterized getFileChanges test. */ private static Stream getFileChangesParameters() { boolean notInitialSnapshot = false; boolean isInitialSnapshot = true; long BASE_INDEX = DeltaSourceOffset.BASE_INDEX(); long END_INDEX = DeltaSourceOffset.END_INDEX(); Optional noEndVersion = Optional.empty(); Optional noEndIndex = Optional.empty(); // Arguments: (fromVersion, fromIndex, isInitialSnapshot, endVersion, endIndex, testDescription) return Stream.of( // With FromVersion: start with BASE_INDEX, no endVersion Arguments.of( 0L, BASE_INDEX, notInitialSnapshot, noEndVersion, noEndIndex, "With FromVersion 1"), Arguments.of( 3L, BASE_INDEX, notInitialSnapshot, noEndVersion, noEndIndex, "With FromVersion 2"), // With FromIndex: start with specific fromIndex, no endVersion Arguments.of(0L, 0L, notInitialSnapshot, noEndVersion, noEndIndex, "With FromIndex 1"), Arguments.of(1L, 5L, notInitialSnapshot, noEndVersion, noEndIndex, "With FromIndex 2"), // With EndVersion Arguments.of( 1L, BASE_INDEX, notInitialSnapshot, Optional.of(3L), noEndIndex, "With EndVersion 1"), Arguments.of( 1L, BASE_INDEX, notInitialSnapshot, Optional.of(2L), Optional.of(5L), "With EndVersion 2"), Arguments.of( 1L, 5L, notInitialSnapshot, Optional.of(3L), Optional.of(END_INDEX), "With EndVersion 3"), Arguments.of( 1L, END_INDEX, notInitialSnapshot, Optional.of(2L), Optional.of(END_INDEX), "With EndVersion 4"), // Empty Range Arguments.of(2L, 50L, notInitialSnapshot, Optional.of(2L), Optional.of(40L), "Empty Range"), // Initial Snapshot: snapshot only (no subsequent delta changes to combine) Arguments.of( 0L, BASE_INDEX, isInitialSnapshot, noEndVersion, noEndIndex, "InitialSnapshot_Version0_NoDelta"), Arguments.of( 2L, BASE_INDEX, isInitialSnapshot, noEndVersion, noEndIndex, "InitialSnapshot_Version2_NoDelta"), // Initial Snapshot: snapshot + delta changes (tests combine logic) // Note: These assume the table has 5 versions (latest=5), so snapshot at version 0-2 will // have delta changes to combine Arguments.of( 0L, BASE_INDEX, isInitialSnapshot, Optional.of(5L), Optional.of(END_INDEX), "InitialSnapshot_Version0_WithDelta_ToEnd"), Arguments.of( 1L, BASE_INDEX, isInitialSnapshot, Optional.of(3L), Optional.of(END_INDEX), "InitialSnapshot_Version1_WithDelta_ToVersion3"), Arguments.of( 2L, BASE_INDEX, isInitialSnapshot, Optional.of(4L), noEndIndex, "InitialSnapshot_Version2_WithDelta_ToVersion4"), // Initial Snapshot: with specific endIndex Arguments.of( 0L, BASE_INDEX, isInitialSnapshot, Optional.of(2L), Optional.of(5L), "InitialSnapshot_Version0_WithEndIndex"), // Initial Snapshot: at latest version (latestVersion == fromVersion, no delta to combine) Arguments.of( 5L, BASE_INDEX, isInitialSnapshot, noEndVersion, noEndIndex, "InitialSnapshot_LatestVersion_NoDelta")); } // ================================================================================================ // Tests for getFileChangesWithRateLimit parity between DSv1 and DSv2 // ================================================================================================ /** * Test that verifies parity between DSv1 DeltaSource.getFileChangesWithRateLimit and DSv2 * SparkMicroBatchStream.getFileChangesWithRateLimit. * *

Tests both regular delta log streaming and initial snapshot scenarios with rate limiting. */ @ParameterizedTest @MethodSource("getFileChangesWithRateLimitParameters") public void testGetFileChangesWithRateLimit( long fromVersion, boolean isInitialSnapshot, Optional maxFiles, Optional maxBytes, String testDescription, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_rate_limit_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); // Create 5 versions with 10 rows each (versions 1-5) insertVersions( testTableName, /* numVersions= */ 5, /* rowsPerVersion= */ 10, /* includeEmptyVersion= */ false); // dsv1 DeltaSource DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath); DeltaOptions options = emptyDeltaOptions(); Optional dsv1Limits = createAdmissionLimits(deltaSource, maxFiles, maxBytes); ClosableIterator deltaChanges = deltaSource.getFileChangesWithRateLimit( fromVersion, DeltaSourceOffset.BASE_INDEX(), isInitialSnapshot, ScalaUtils.toScalaOption(dsv1Limits)); List deltaFilesList = new ArrayList<>(); while (deltaChanges.hasNext()) { deltaFilesList.add(deltaChanges.next()); } deltaChanges.close(); // dsv2 SparkMicroBatchStream Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); // We need a separate AdmissionLimits object for DSv2 because the method is stateful. Optional dsv2Limits = createAdmissionLimits(deltaSource, maxFiles, maxBytes); try (CloseableIterator kernelChanges = stream.getFileChangesWithRateLimit( fromVersion, DeltaSourceOffset.BASE_INDEX(), isInitialSnapshot, dsv2Limits)) { List kernelFilesList = new ArrayList<>(); while (kernelChanges.hasNext()) { kernelFilesList.add(kernelChanges.next()); } compareFileChanges(deltaFilesList, kernelFilesList); } } /** Provides test parameters for the parameterized getFileChangesWithRateLimit test. */ private static Stream getFileChangesWithRateLimitParameters() { boolean notInitialSnapshot = false; boolean isInitialSnapshot = true; Optional noMaxFiles = Optional.empty(); Optional noMaxBytes = Optional.empty(); // Arguments: (fromVersion, isInitialSnapshot, maxFiles, maxBytes, testDescription) return Stream.of( // Regular delta log streaming (not initial snapshot) Arguments.of(0L, notInitialSnapshot, noMaxFiles, noMaxBytes, "DeltaLog_NoLimits"), Arguments.of(0L, notInitialSnapshot, Optional.of(5), noMaxBytes, "DeltaLog_MaxFiles"), Arguments.of(0L, notInitialSnapshot, noMaxFiles, Optional.of(5000L), "DeltaLog_MaxBytes"), Arguments.of( 0L, notInitialSnapshot, Optional.of(10), Optional.of(10000L), "DeltaLog_MaxFilesAndMaxBytes"), // Initial snapshot with rate limiting Arguments.of( 0L, isInitialSnapshot, noMaxFiles, noMaxBytes, "InitialSnapshot_Version0_NoLimits"), Arguments.of( 0L, isInitialSnapshot, Optional.of(5), noMaxBytes, "InitialSnapshot_Version0_MaxFiles"), Arguments.of( 0L, isInitialSnapshot, noMaxFiles, Optional.of(5000L), "InitialSnapshot_Version0_MaxBytes"), Arguments.of( 0L, isInitialSnapshot, Optional.of(10), Optional.of(10000L), "InitialSnapshot_Version0_MaxFilesAndMaxBytes"), Arguments.of( 2L, isInitialSnapshot, Optional.of(5), noMaxFiles, "InitialSnapshot_Version2_MaxFiles"), Arguments.of( 2L, isInitialSnapshot, noMaxFiles, Optional.of(3000L), "InitialSnapshot_Version2_MaxBytes")); } private void compareFileChanges( List deltaSourceFiles, List kernelFiles) { assertEquals( deltaSourceFiles.size(), kernelFiles.size(), String.format( "Number of file changes should match between dsv1 (%d) and dsv2 (%d)", deltaSourceFiles.size(), kernelFiles.size())); for (int i = 0; i < deltaSourceFiles.size(); i++) { org.apache.spark.sql.delta.sources.IndexedFile deltaFile = deltaSourceFiles.get(i); IndexedFile kernelFile = kernelFiles.get(i); assertEquals( deltaFile.version(), kernelFile.getVersion(), String.format( "Version mismatch at index %d: dsv1=%d, dsv2=%d", i, deltaFile.version(), kernelFile.getVersion())); assertEquals( deltaFile.index(), kernelFile.getIndex(), String.format( "Index mismatch at index %d: dsv1=%d, dsv2=%d", i, deltaFile.index(), kernelFile.getIndex())); // Sentinel files have null AddFile and null RemoveFile. String deltaPath = deltaFile.add() != null ? deltaFile.add().path() : null; String kernelPath = kernelFile.getAddFile() != null ? kernelFile.getAddFile().getPath() : null; if (deltaPath != null || kernelPath != null) { assertEquals( deltaPath, kernelPath, String.format( "AddFile path mismatch at index %d: dsv1=%s, dsv2=%s", i, deltaPath, kernelPath)); } } } // ================================================================================================ // Tests for commits with no data file changes // ================================================================================================ /** * Parameterized test that verifies both DSv1 and DSv2 handle commits with no ADD or REMOVE * actions correctly. Such commits only contain METADATA, PROTOCOL, or other non-data changes. */ @ParameterizedTest @MethodSource("emptyVersionScenarios") public void testGetFileChanges_emptyVersions( ScenarioSetup scenarioSetup, List expectedEmptyVersions, String testDescription, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_empty_versions_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); // Execute the scenario-specific setup scenarioSetup.setup(testTableName, tempDir); // Read from version 0 (start of the table) to capture all changes long fromVersion = 0L; long fromIndex = DeltaSourceOffset.BASE_INDEX(); boolean isInitialSnapshot = false; Option endOffset = Option.empty(); // Test DSv1 DeltaSource DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath); ClosableIterator deltaChanges = deltaSource.getFileChanges( fromVersion, fromIndex, isInitialSnapshot, endOffset, /* verifyMetadataAction= */ true); List deltaFilesList = new ArrayList<>(); while (deltaChanges.hasNext()) { deltaFilesList.add(deltaChanges.next()); } deltaChanges.close(); // Test DSv2 SparkMicroBatchStream Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); try (CloseableIterator kernelChanges = stream.getFileChanges( fromVersion, fromIndex, isInitialSnapshot, ScalaUtils.toJavaOptional(endOffset))) { List kernelFilesList = new ArrayList<>(); while (kernelChanges.hasNext()) { kernelFilesList.add(kernelChanges.next()); } // Compare results compareFileChanges(deltaFilesList, kernelFilesList); } } /** Provides test scenarios with various types of empty versions (no ADD/REMOVE actions). */ private static Stream emptyVersionScenarios() { return Stream.of( Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2')", tableName); sql("ALTER TABLE %s SET TBLPROPERTIES ('test.property' = 'value1')", tableName); sql("INSERT INTO %s VALUES (3, 'User3')", tableName); }, Arrays.asList(2L), "Single metadata-only version"), Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("INSERT INTO %s VALUES (1, 'User1')", tableName); sql("ALTER TABLE %s SET TBLPROPERTIES ('p1' = 'v1')", tableName); sql("ALTER TABLE %s SET TBLPROPERTIES ('p2' = 'v2')", tableName); sql("ALTER TABLE %s SET TBLPROPERTIES ('p3' = 'v3')", tableName); }, Arrays.asList(2L), "Multiple consecutive metadata-only versions")); } @Test public void testGetFileChanges_lazyLoading(@TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_lazy_loading_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); // Create 3 INSERT versions (versions 1-3 with ADD files only) sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2')", testTableName); sql("INSERT INTO %s VALUES (3, 'User3'), (4, 'User4')", testTableName); sql("INSERT INTO %s VALUES (5, 'User5'), (6, 'User6')", testTableName); // Version 4: DELETE operation that will create a REMOVE file sql("DELETE FROM %s WHERE id = 1", testTableName); Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); // Get file changes from version 0 (which will include versions 1-4) CloseableIterator kernelChanges = stream.getFileChanges( /* fromVersion= */ 0L, /* fromIndex= */ DeltaSourceOffset.BASE_INDEX(), /* isInitialSnapshot= */ false, /* endOffset= */ Optional.empty()); try { // Partially consume the iterator: only read files from versions 1-2 // With lazy loading, this should succeed without hitting the REMOVE file error in version 4 List partialFiles = new ArrayList<>(); int filesRead = 0; int targetVersion = 2; // Only read up to version 2 while (kernelChanges.hasNext()) { IndexedFile file = kernelChanges.next(); partialFiles.add(file); filesRead++; // Stop after we've passed version 2's END sentinel // Each version has: BEGIN sentinel + actual files + END sentinel if (file.getVersion() == targetVersion && file.getIndex() == DeltaSourceOffset.END_INDEX()) { break; } } // If we got here, lazy loading worked - we successfully read versions 1-2 without // encountering the REMOVE file error in version 4 // Version 0 (CREATE TABLE): BEGIN + 1 metadata/protocol action + END = 3 files // Version 1 (INSERT): BEGIN + 1 data file + END = 3 files // Version 2 (INSERT): BEGIN + 1 data file + END = 3 files // Total = 9 files assertEquals(9, filesRead, "Should have read exactly 9 IndexedFiles from versions 0-2"); // Now consume the rest of the iterator - this should hit the REMOVE file in version 4 assertThrows( UnsupportedOperationException.class, () -> { while (kernelChanges.hasNext()) { kernelChanges.next(); // This should throw when it reaches version 4's REMOVE } }, "Should throw UnsupportedOperationException when reaching version 4 with REMOVE file"); } finally { try { kernelChanges.close(); } catch (Exception ignored) { } } } // ================================================================================================ // Tests for REMOVE file handling // ================================================================================================ /** * Parameterized test that verifies both DSv1 and DSv2 throw UnsupportedOperationException when * encountering REMOVE actions (from DELETE, UPDATE, MERGE operations). */ @ParameterizedTest @MethodSource("removeFileScenarios") public void testGetFileChanges_onRemoveFile_throwError( ScenarioSetup scenarioSetup, String testDescription, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_remove_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); // Execute the scenario-specific setup (which will generate REMOVE actions) scenarioSetup.setup(testTableName, tempDir); // Try to read from version 0, which should include commits with REMOVE actions long fromVersion = 0L; long fromIndex = DeltaSourceOffset.BASE_INDEX(); boolean isInitialSnapshot = false; Option endOffset = Option.empty(); // Test DSv1 DeltaSource DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath); AtomicInteger dsv1SuccessfulCalls = new AtomicInteger(0); assertThrows( UnsupportedOperationException.class, () -> { ClosableIterator deltaChanges = deltaSource.getFileChanges( fromVersion, fromIndex, isInitialSnapshot, endOffset, /* verifyMetadataAction= */ true); try { while (deltaChanges.hasNext()) { deltaChanges.next(); // Should throw when hitting REMOVE file dsv1SuccessfulCalls.incrementAndGet(); } } finally { deltaChanges.close(); } }, String.format("DSv1 should throw on REMOVE for scenario: %s", testDescription)); // Test DSv2 SparkMicroBatchStream Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); AtomicInteger dsv2SuccessfulCalls = new AtomicInteger(0); assertThrows( UnsupportedOperationException.class, () -> { CloseableIterator kernelChanges = stream.getFileChanges( fromVersion, fromIndex, isInitialSnapshot, ScalaUtils.toJavaOptional(endOffset)); try { while (kernelChanges.hasNext()) { kernelChanges.next(); // Should throw when hitting REMOVE file dsv2SuccessfulCalls.incrementAndGet(); } } finally { kernelChanges.close(); } }, String.format("DSv2 should throw on REMOVE for scenario: %s", testDescription)); // Verify both threw at the exact same point assertEquals( dsv1SuccessfulCalls.get(), dsv2SuccessfulCalls.get(), String.format( "DSv1 and DSv2 should throw after the same number of next() calls for scenario: %s. " + "DSv1=%d, DSv2=%d", testDescription, dsv1SuccessfulCalls.get(), dsv2SuccessfulCalls.get())); } /** Provides test scenarios that generate REMOVE actions through various DML operations. */ private static Stream removeFileScenarios() { return Stream.of( // Simple DELETE scenario Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2')", tableName); sql("INSERT INTO %s VALUES (3, 'User3'), (4, 'User4')", tableName); sql("DELETE FROM %s WHERE id = 1", tableName); }, "DELETE: Simple delete"), // Many ADDs followed by REMOVE Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { // Create 10 versions with ADDs (50 rows each) for (int i = 0; i < 10; i++) { StringBuilder values = new StringBuilder(); for (int j = 0; j < 50; j++) { if (j > 0) values.append(", "); int id = i * 50 + j; values.append(String.format("(%d, 'User%d')", id, id)); } sql("INSERT INTO %s VALUES %s", tableName, values); } sql("DELETE FROM %s WHERE id < 100", tableName); }, "DELETE: Many ADDs (10 versions) followed by REMOVE"), // UPDATE scenario (generates REMOVE + ADD pairs) Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql( "INSERT INTO %s VALUES (1, 'User1'), (2, 'User2'), (3, 'User3'), (4," + " 'User4'), (5, 'User5')", tableName); sql("INSERT INTO %s VALUES (6, 'User6'), (7, 'User7'), (8, 'User8')", tableName); sql("UPDATE %s SET name = 'UpdatedUser' WHERE id <= 3", tableName); }, "UPDATE: Update multiple rows (generates REMOVE + ADD)"), // MERGE scenario (generates REMOVE + ADD for matched, ADD for not matched) Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2'), (3, 'User3')", tableName); // Create a source table for MERGE String sourceTableName = "merge_source_" + System.nanoTime(); sql( "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", sourceTableName, new File(tempDir, "source").getAbsolutePath()); sql("INSERT INTO %s VALUES (2, 'UpdatedUser2'), (4, 'User4')", sourceTableName); // Perform MERGE operation sql( "MERGE INTO %s AS target USING %s AS source ON target.id = source.id WHEN" + " MATCHED THEN UPDATE SET target.name = source.name WHEN NOT MATCHED" + " THEN INSERT (id, name) VALUES (source.id, source.name)", tableName, sourceTableName); sql("DROP TABLE IF EXISTS %s", sourceTableName); }, "MERGE: Matched (REMOVE+ADD) and not matched (ADD)"), // Full table delete (only RemoveFiles, no AddFiles) Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2')", tableName); sql("DELETE FROM %s", tableName); }, "DELETE: Full table delete (removes only, no adds)")); } // ================================================================================================ // Tests for ignoreDeletes parity between DSv1 and DSv2 // ================================================================================================ /** * Verifies that with ignoreDeletes=true, both DSv1 and DSv2 produce the same file changes for * delete-only commits (commits with only RemoveFile actions, no AddFile actions). The delete-only * commit should be silently skipped (only sentinels emitted, no data files). */ @ParameterizedTest @MethodSource("deleteOnlyScenarios") public void testGetFileChanges_withIgnoreDeletes_deleteOnlyParity( ScenarioSetup scenarioSetup, boolean isInitialSnapshot, String testDescription, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_ignore_deletes_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); createEmptyPartitionedTestTable(testTablePath, testTableName); scenarioSetup.setup(testTableName, tempDir); long fromVersion = 0L; long fromIndex = DeltaSourceOffset.BASE_INDEX(); DeltaOptions options = createDeltaOptions("ignoreDeletes", "true"); // DSv1 DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath, options); Option endOffset = Option.empty(); ClosableIterator deltaChanges = deltaSource.getFileChanges( fromVersion, fromIndex, isInitialSnapshot, endOffset, /* verifyMetadataAction= */ true); List deltaFilesList = new ArrayList<>(); while (deltaChanges.hasNext()) { deltaFilesList.add(deltaChanges.next()); } deltaChanges.close(); // DSv2 Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, options); try (CloseableIterator kernelChanges = stream.getFileChanges(fromVersion, fromIndex, isInitialSnapshot, Optional.empty())) { List kernelFilesList = new ArrayList<>(); while (kernelChanges.hasNext()) { kernelFilesList.add(kernelChanges.next()); } compareFileChanges(deltaFilesList, kernelFilesList); } } /** * Verifies that with ignoreDeletes=true, both DSv1 and DSv2 still throw on commits containing * both adds and removes (e.g., UPDATE, MERGE), since ignoreDeletes only suppresses delete-only * commits. */ @ParameterizedTest @MethodSource("changeCommitScenarios") public void testGetFileChanges_withIgnoreDeletes_changeCommitStillThrows( ScenarioSetup scenarioSetup, boolean isInitialSnapshot, String testDescription, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_ignore_deletes_change_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); scenarioSetup.setup(testTableName, tempDir); long fromVersion = 0L; long fromIndex = DeltaSourceOffset.BASE_INDEX(); Option endOffset = Option.empty(); DeltaOptions options = createDeltaOptions("ignoreDeletes", "true"); // DSv1 DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath, options); AtomicInteger dsv1SuccessfulCalls = new AtomicInteger(0); assertThrows( UnsupportedOperationException.class, () -> { ClosableIterator deltaChanges = deltaSource.getFileChanges( fromVersion, fromIndex, isInitialSnapshot, endOffset, /* verifyMetadataAction= */ true); try { while (deltaChanges.hasNext()) { deltaChanges.next(); dsv1SuccessfulCalls.incrementAndGet(); } } finally { deltaChanges.close(); } }, String.format( "DSv1 should throw on change commit with ignoreDeletes for: %s", testDescription)); // DSv2 Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, options); AtomicInteger dsv2SuccessfulCalls = new AtomicInteger(0); assertThrows( UnsupportedOperationException.class, () -> { CloseableIterator kernelChanges = stream.getFileChanges(fromVersion, fromIndex, isInitialSnapshot, Optional.empty()); try { while (kernelChanges.hasNext()) { kernelChanges.next(); dsv2SuccessfulCalls.incrementAndGet(); } } finally { kernelChanges.close(); } }, String.format( "DSv2 should throw on change commit with ignoreDeletes for: %s", testDescription)); assertEquals( dsv1SuccessfulCalls.get(), dsv2SuccessfulCalls.get(), String.format( "DSv1 and DSv2 should throw after the same number of next() calls for: %s. " + "DSv1=%d, DSv2=%d", testDescription, dsv1SuccessfulCalls.get(), dsv2SuccessfulCalls.get())); } // ================================================================================================ // Tests for skipChangeCommits parity between DSv1 and DSv2 // ================================================================================================ /** * Verifies that with skipChangeCommits=true, both DSv1 and DSv2 produce the same file changes for * delete-only commits. Since skipChangeCommits suppresses all commits containing RemoveFile * actions, these commits should be silently skipped (only sentinels emitted, no data files). */ @ParameterizedTest @MethodSource("deleteOnlyScenarios") public void testGetFileChanges_withSkipChangeCommits_deleteOnlyParity( ScenarioSetup scenarioSetup, boolean isInitialSnapshot, String testDescription, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_skip_change_del_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); createEmptyPartitionedTestTable(testTablePath, testTableName); scenarioSetup.setup(testTableName, tempDir); long fromVersion = 0L; long fromIndex = DeltaSourceOffset.BASE_INDEX(); DeltaOptions options = createDeltaOptions("skipChangeCommits", "true"); // DSv1 DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath, options); Option endOffset = Option.empty(); ClosableIterator deltaChanges = deltaSource.getFileChanges( fromVersion, fromIndex, isInitialSnapshot, endOffset, /* verifyMetadataAction= */ true); List deltaFilesList = new ArrayList<>(); while (deltaChanges.hasNext()) { deltaFilesList.add(deltaChanges.next()); } deltaChanges.close(); // DSv2 Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, options); try (CloseableIterator kernelChanges = stream.getFileChanges(fromVersion, fromIndex, isInitialSnapshot, Optional.empty())) { List kernelFilesList = new ArrayList<>(); while (kernelChanges.hasNext()) { kernelFilesList.add(kernelChanges.next()); } compareFileChanges(deltaFilesList, kernelFilesList); } } /** * Verifies that with skipChangeCommits=true, both DSv1 and DSv2 silently skip commits containing * both adds and removes (e.g., UPDATE, MERGE), instead of throwing. This is the key behavioral * difference from ignoreDeletes, which throws on such commits. */ @ParameterizedTest @MethodSource("changeCommitScenarios") public void testGetFileChanges_withSkipChangeCommits_changeCommitParity( ScenarioSetup scenarioSetup, boolean isInitialSnapshot, String testDescription, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_skip_change_chg_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); scenarioSetup.setup(testTableName, tempDir); long fromVersion = 0L; long fromIndex = DeltaSourceOffset.BASE_INDEX(); DeltaOptions options = createDeltaOptions("skipChangeCommits", "true"); // DSv1 DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath, options); Option endOffset = Option.empty(); ClosableIterator deltaChanges = deltaSource.getFileChanges( fromVersion, fromIndex, isInitialSnapshot, endOffset, /* verifyMetadataAction= */ true); List deltaFilesList = new ArrayList<>(); while (deltaChanges.hasNext()) { deltaFilesList.add(deltaChanges.next()); } deltaChanges.close(); // DSv2 Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, options); try (CloseableIterator kernelChanges = stream.getFileChanges(fromVersion, fromIndex, isInitialSnapshot, Optional.empty())) { List kernelFilesList = new ArrayList<>(); while (kernelChanges.hasNext()) { kernelFilesList.add(kernelChanges.next()); } compareFileChanges(deltaFilesList, kernelFilesList); } } // ================================================================================================ // Shared scenario providers for ignoreDeletes and skipChangeCommits tests // ================================================================================================ /** * Provides delete-only scenarios: commits with only RemoveFile actions and no AddFile actions. * Used by both ignoreDeletes and skipChangeCommits tests. * *

Arguments: (ScenarioSetup, isInitialSnapshot, testDescription) */ private static Stream deleteOnlyScenarios() { return Stream.of( Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2')", tableName); sql("INSERT INTO %s VALUES (3, 'User3'), (4, 'User4')", tableName); sql("DELETE FROM %s", tableName); }, false, "Full table delete"), Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2')", tableName); sql("DELETE FROM %s", tableName); sql("INSERT INTO %s VALUES (3, 'User3'), (4, 'User4')", tableName); }, false, "Insert-Delete-Insert: data resumes after delete"), Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("INSERT INTO %s VALUES (1, 'PartA'), (2, 'PartA'), (3, 'PartB')", tableName); sql("DELETE FROM %s WHERE name = 'PartA'", tableName); }, false, "Partitioned table: delete entire partition"), Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql( "ALTER TABLE %s SET TBLPROPERTIES ('delta.enableDeletionVectors' = true)", tableName); sql( "INSERT INTO %s SELECT /*+ COALESCE(1) */ * FROM VALUES " + "(1, 'User1'), (2, 'User2'), (3, 'User3') AS t(id, name)", tableName); sql("DELETE FROM %s WHERE id >= 1", tableName); }, false, "Full DELETE with DV: full file delete via WHERE clause"), Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2')", tableName); sql("INSERT INTO %s VALUES (3, 'User3'), (4, 'User4')", tableName); sql("DELETE FROM %s", tableName); }, true, "Full table delete with initial snapshot")); } /** * Provides change-commit scenarios: commits containing both AddFile and RemoveFile actions (e.g., * UPDATE, MERGE). Used by both ignoreDeletes and skipChangeCommits tests — ignoreDeletes expects * these to throw, while skipChangeCommits expects them to be silently skipped. * *

Arguments: (ScenarioSetup, isInitialSnapshot, testDescription) */ private static Stream changeCommitScenarios() { return Stream.of( Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2'), (3, 'User3')", tableName); sql("UPDATE %s SET name = 'Updated' WHERE id = 1", tableName); }, false, "UPDATE: AddFile + RemoveFile"), Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2'), (3, 'User3')", tableName); String sourceTableName = "merge_src_" + System.nanoTime(); sql( "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", sourceTableName, new File(tempDir, "source").getAbsolutePath()); sql("INSERT INTO %s VALUES (2, 'UpdatedUser2'), (4, 'User4')", sourceTableName); sql( "MERGE INTO %s AS target USING %s AS source ON target.id = source.id " + "WHEN MATCHED THEN UPDATE SET target.name = source.name " + "WHEN NOT MATCHED THEN INSERT (id, name) " + "VALUES (source.id, source.name)", tableName, sourceTableName); sql("DROP TABLE IF EXISTS %s", sourceTableName); }, false, "MERGE: AddFile + RemoveFile"), Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql( "ALTER TABLE %s SET TBLPROPERTIES ('delta.enableDeletionVectors' = true)", tableName); // Coalesce to to ensure DV is partial delete sql( "INSERT INTO %s SELECT /*+ COALESCE(1) */ * FROM VALUES " + "(1, 'User1'), (2, 'User2'), (3, 'User3') AS t(id, name)", tableName); sql("DELETE FROM %s WHERE id = 1", tableName); }, false, "Partial DELETE with DV: AddFile(with DV) + RemoveFile"), Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2'), (3, 'User3')", tableName); sql("UPDATE %s SET name = 'Updated' WHERE id = 1", tableName); }, true, "UPDATE with initial snapshot: AddFile + RemoveFile")); } @Test public void testGetFileChanges_startingVersionAfterCheckpointAndLogCleanup(@TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_checkpoint_cleanup_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); // Insert 5 versions for (int i = 1; i <= 5; i++) { sql("INSERT INTO %s VALUES (%d, 'User%d')", testTableName, i, i); } // Create checkpoint at version 5 DeltaLog.forTable(spark, new Path(testTablePath)).checkpoint(); // Delete 0.json to simulate log cleanup Path logPath = new Path(testTablePath, "_delta_log"); Path logFile0 = new Path(logPath, "00000000000000000000.json"); File file0 = new File(logFile0.toUri().getPath()); if (file0.exists()) { file0.delete(); } // Now test with startingVersion=1 Configuration hadoopConf = spark.sessionState().newHadoopConf(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); // Get file changes from version 1 onwards try (CloseableIterator kernelChanges = stream.getFileChanges( /* fromVersion= */ 1L, /* fromIndex= */ DeltaSourceOffset.BASE_INDEX(), /* isInitialSnapshot= */ false, /* endOffset= */ Optional.empty())) { List kernelFilesList = new ArrayList<>(); while (kernelChanges.hasNext()) { kernelFilesList.add(kernelChanges.next()); } // Filter to get only actual data files (addFile != null) long actualFileCount = kernelFilesList.stream().filter(f -> f.getAddFile() != null).count(); // Should be able to read 5 data files from versions 1-5 assertEquals( 5, actualFileCount, "Should read 5 data files from versions 1-5 even though version 0 log is deleted"); } } // ================================================================================================ // Tests for latestOffset parity between DSv1 and DSv2 // ================================================================================================ /** * Parameterized test that verifies parity between DSv1 DeltaSource.latestOffset and DSv2 * SparkMicroBatchStream.latestOffset. */ @ParameterizedTest @MethodSource("latestOffsetParameters") public void testLatestOffset_notInitialSnapshot( Long startVersion, Long startIndex, ReadLimitConfig limitConfig, String testDescription, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_latest_offset_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); insertVersions( testTableName, /* numVersions= */ 5, /* rowsPerVersion= */ 10, /* includeEmptyVersion= */ true); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); String tableId = deltaLog.tableId(); Offset startOffset = new DeltaSourceOffset(tableId, startVersion, startIndex, /* isInitialSnapshot= */ false); ReadLimit readLimit = limitConfig.toReadLimit(); // dsv1 DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath); Offset v1EndOffset = deltaSource.latestOffset(startOffset, readLimit); // dsv2 Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); Offset v2EndOffset = stream.latestOffset(startOffset, readLimit); compareOffsets(v1EndOffset, v2EndOffset, testDescription); } /** Provides test parameters for the parameterized latestOffset test. */ private static Stream latestOffsetParameters() { long BASE_INDEX = DeltaSourceOffset.BASE_INDEX(); long END_INDEX = DeltaSourceOffset.END_INDEX(); // TODO(#5318): Add tests for initial offset & latestOffset(null, ReadLimit) return Stream.of( // No limits Arguments.of( /* startVersion= */ 1L, /* startIndex= */ BASE_INDEX, ReadLimitConfig.noLimit(), "NoLimits1"), Arguments.of( /* startVersion= */ 2L, /* startIndex= */ 5L, ReadLimitConfig.noLimit(), "NoLimits2"), Arguments.of( /* startVersion= */ 3L, /* startIndex= */ END_INDEX, ReadLimitConfig.noLimit(), "NoLimits3"), Arguments.of( /* startVersion= */ 5L, /* startIndex= */ END_INDEX, ReadLimitConfig.noLimit(), "NoLimits4"), // Max files Arguments.of( /* startVersion= */ 3L, /* startIndex= */ 5L, ReadLimitConfig.maxFiles(10), "MaxFiles1"), Arguments.of( /* startVersion= */ 4L, /* startIndex= */ BASE_INDEX, ReadLimitConfig.maxFiles(5), "MaxFiles2"), Arguments.of( /* startVersion= */ 1L, /* startIndex= */ END_INDEX, ReadLimitConfig.maxFiles(1), "MaxFiles3"), Arguments.of( /* startVersion= */ 5L, /* startIndex= */ END_INDEX, ReadLimitConfig.maxFiles(1), "MaxFiles4"), // Max bytes Arguments.of( /* startVersion= */ 3L, /* startIndex= */ BASE_INDEX, ReadLimitConfig.maxBytes(1000), "MaxBytes1"), Arguments.of( /* startVersion= */ 3L, /* startIndex= */ 5L, ReadLimitConfig.maxBytes(1000), "MaxBytes2"), Arguments.of( /* startVersion= */ 1L, /* startIndex= */ END_INDEX, ReadLimitConfig.maxBytes(1000), "MaxBytes3"), Arguments.of( /* startVersion= */ 5L, /* startIndex= */ END_INDEX, ReadLimitConfig.maxBytes(1000), "MaxBytes4")); } /** * Parameterized test that verifies sequential batch advancement produces identical offset * sequences for DSv1 and DSv2. This simulates real streaming where latestOffset is called * multiple times, each using the previous offset as the starting point. */ @ParameterizedTest @MethodSource("sequentialBatchAdvancementParameters") public void testLatestOffset_sequentialBatchAdvancement( long startVersion, long startIndex, ReadLimitConfig limitConfig, int numIterations, String testDescription, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_sequential_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); insertVersions( testTableName, /* numVersions= */ 5, /* rowsPerVersion= */ 10, /* includeEmptyVersion= */ true); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); String tableId = deltaLog.tableId(); DeltaSourceOffset startOffset = new DeltaSourceOffset(tableId, startVersion, startIndex, /* isInitialSnapshot= */ false); // dsv1 ReadLimit readLimit = limitConfig.toReadLimit(); DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath); List dsv1Offsets = advanceOffsetSequenceDsv1(deltaSource, startOffset, numIterations, readLimit); // dsv2 Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); List dsv2Offsets = advanceOffsetSequenceDsv2(stream, startOffset, numIterations, readLimit); compareOffsetSequence(dsv1Offsets, dsv2Offsets, testDescription); } /** Provides test parameters for sequential batch advancement test. */ private static Stream sequentialBatchAdvancementParameters() { long BASE_INDEX = DeltaSourceOffset.BASE_INDEX(); long END_INDEX = DeltaSourceOffset.END_INDEX(); return Stream.of( // No limits Arguments.of( /* startVersion= */ 0L, /* startIndex= */ BASE_INDEX, ReadLimitConfig.noLimit(), /* numIterations= */ 3, "NoLimits1"), Arguments.of( /* startVersion= */ 1L, /* startIndex= */ BASE_INDEX, ReadLimitConfig.noLimit(), /* numIterations= */ 3, "NoLimits2"), Arguments.of( /* startVersion= */ 4L, /* startIndex= */ END_INDEX, ReadLimitConfig.noLimit(), /* numIterations= */ 3, "NoLimits3"), Arguments.of( /* startVersion= */ 4L, /* startIndex= */ END_INDEX, ReadLimitConfig.noLimit(), /* numIterations= */ 3, "NoLimits4"), // Max files Arguments.of( /* startVersion= */ 0L, /* startIndex= */ BASE_INDEX, ReadLimitConfig.maxFiles(5), /* numIterations= */ 5, "MaxFiles1"), Arguments.of( /* startVersion= */ 1L, /* startIndex= */ BASE_INDEX, ReadLimitConfig.maxFiles(5), /* numIterations= */ 3, "MaxFiles2"), Arguments.of( /* startVersion= */ 4L, /* startIndex= */ END_INDEX, ReadLimitConfig.maxFiles(1), /* numIterations= */ 10, "MaxFiles3"), // Max bytes Arguments.of( /* startVersion= */ 1L, /* startIndex= */ BASE_INDEX, ReadLimitConfig.maxBytes(1000), /* numIterations= */ 3, "MaxBytes1"), Arguments.of( /* startVersion= */ 1L, /* startIndex= */ 5L, ReadLimitConfig.maxBytes(1000), /* numIterations= */ 3, "MaxBytes2"), Arguments.of( /* startVersion= */ 4L, /* startIndex= */ END_INDEX, ReadLimitConfig.maxBytes(1000), /* numIterations= */ 3, "MaxBytes3")); } /** * Parameterized test that verifies behavior when calling latestOffset but no new data is * available (we're already at the latest version). */ @ParameterizedTest @MethodSource("noNewDataAtLatestVersionParameters") public void testLatestOffset_noNewDataAtLatestVersion( long startIndex, Long expectedVersionOffset, Long expectedIndex, String testDescription, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_no_new_data_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); insertVersions( testTableName, /* numVersions= */ 5, /* rowsPerVersion= */ 1, /* includeEmptyVersion= */ false); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); String tableId = deltaLog.tableId(); long latestVersion = deltaLog .update( /* isForce= */ false, /* timestamp= */ scala.Option.empty(), /* version= */ scala.Option.empty()) .version(); DeltaSourceOffset startOffset = new DeltaSourceOffset(tableId, latestVersion, startIndex, /* isInitialSnapshot= */ false); ReadLimit readLimit = ReadLimit.allAvailable(); // dsv1 DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath); org.apache.spark.sql.connector.read.streaming.Offset dsv1Offset = deltaSource.latestOffset(startOffset, readLimit); // dsv2 Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); Offset dsv2Offset = stream.latestOffset(startOffset, readLimit); compareOffsets(dsv1Offset, dsv2Offset, testDescription); // Verify expected offset if (expectedVersionOffset == null) { assertNull( dsv1Offset, String.format( "Test: %s | Expected null offset but got: %s", testDescription, dsv1Offset)); } else { assertNotNull( dsv1Offset, String.format("Test: %s | Expected non-null offset but got null", testDescription)); DeltaSourceOffset dsv1DeltaOffset = (DeltaSourceOffset) dsv1Offset; long expectedVersion = latestVersion + expectedVersionOffset; assertEquals( expectedVersion, dsv1DeltaOffset.reservoirVersion(), String.format( "Test: %s | Expected version: %d, Actual version: %d", testDescription, expectedVersion, dsv1DeltaOffset.reservoirVersion())); assertEquals( expectedIndex, dsv1DeltaOffset.index(), String.format( "Test: %s | Expected index: %d, Actual index: %d", testDescription, expectedIndex, dsv1DeltaOffset.index())); } } /** Provides test parameters for no new data at latest version test. */ private static Stream noNewDataAtLatestVersionParameters() { long BASE_INDEX = DeltaSourceOffset.BASE_INDEX(); long END_INDEX = DeltaSourceOffset.END_INDEX(); // Arguments: (startIndex, expectedVersionOffset, expectedIndex, testDescription) // expectedVersionOffset is relative to latestVersion (null means expect null offset) return Stream.of( Arguments.of(BASE_INDEX, 1L, BASE_INDEX, "Latest version BASE_INDEX, no new data"), Arguments.of(END_INDEX, 0L, END_INDEX, "Latest version END_INDEX, no new data"), Arguments.of(0L, 1L, BASE_INDEX, "Latest version index=0, no new data")); } // ================================================================================================ // Tests for availableNow parity between DSv1 and DSv2 // ================================================================================================ @ParameterizedTest @MethodSource("availableNowParameters") public void testAvailableNow_SequentialBatchAdvancement( Long startVersion, Long startIndex, ReadLimitConfig limitConfig, int numIterations, String testDescription, @TempDir File tempDir) { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_availableNow_sequential" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); insertVersions( testTableName, /* numVersions= */ 5, /* rowsPerVersion= */ 10, /* includeEmptyVersion= */ true); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); String tableId = deltaLog.tableId(); DeltaSourceOffset startOffset = new DeltaSourceOffset(tableId, startVersion, startIndex, /* isInitialSnapshot= */ false); ReadLimit readLimit = limitConfig.toReadLimit(); // dsv1 source DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath); // Enable availableNow deltaSource.prepareForTriggerAvailableNow(); // Advance through multiple batches using dsv1, collecting offset after each batch List dsv1Offsets = advanceOffsetSequenceDsv1(deltaSource, startOffset, numIterations, readLimit); // dsv2 source Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); // Enable availableNow stream.prepareForTriggerAvailableNow(); // Advance through multiple batches using dsv2, collecting offset after each batch List dsv2Offsets = advanceOffsetSequenceDsv2(stream, startOffset, numIterations, readLimit); // Ensure dsv1 and dsv2 produce identical offset sequences compareOffsetSequence(dsv1Offsets, dsv2Offsets, testDescription); } private static Stream availableNowParameters() { long BASE_INDEX = DeltaSourceOffset.BASE_INDEX(); long END_INDEX = DeltaSourceOffset.END_INDEX(); return Stream.of( // No limits respects availableNow Arguments.of( /* startVersion= */ 0L, /* startIndex= */ BASE_INDEX, ReadLimitConfig.noLimit(), /* numIterations= */ 3, "NoLimits1"), Arguments.of( /* startVersion= */ 1L, /* startIndex= */ BASE_INDEX, ReadLimitConfig.noLimit(), /* numIterations= */ 3, "NoLimits2"), Arguments.of( /* startVersion= */ 4L, /* startIndex= */ END_INDEX, ReadLimitConfig.noLimit(), /* numIterations= */ 3, "NoLimits3"), // Max files respects availableNow Arguments.of( /* startVersion= */ 0L, /* startIndex= */ BASE_INDEX, ReadLimitConfig.maxFiles(1), /* numIterations= */ 10, "MaxFiles1"), Arguments.of( /* startVersion= */ 0L, /* startIndex= */ BASE_INDEX, ReadLimitConfig.maxFiles(1000), /* numIterations= */ 3, "MaxFiles2"), Arguments.of( /* startVersion= */ 1L, /* startIndex= */ BASE_INDEX, ReadLimitConfig.maxFiles(2), /* numIterations= */ 10, "MaxFiles3"), Arguments.of( /* startVersion= */ 0L, /* startIndex= */ BASE_INDEX, ReadLimitConfig.maxFiles(0), /* numIterations= */ 3, "MaxFiles4"), // Max bytes respects availableNow Arguments.of( /* startVersion= */ 0L, /* startIndex= */ BASE_INDEX, ReadLimitConfig.maxBytes(1), /* numIterations= */ 100, "MaxBytes1"), Arguments.of( /* startVersion= */ 0L, /* startIndex= */ BASE_INDEX, ReadLimitConfig.maxBytes(1000000), // ensure larger than total file size /* numIterations= */ 3, "MaxBytes2"), Arguments.of( /* startVersion= */ 1L, /* startIndex= */ BASE_INDEX, ReadLimitConfig.maxBytes(1000), /* numIterations= */ 100, "MaxBytes3"), Arguments.of( /* startVersion= */ 0L, /* startIndex= */ BASE_INDEX, ReadLimitConfig.maxBytes(0), /* numIterations= */ 3, "MaxBytes4")); } // ================================================================================================ // Tests for planInputPartitions // ================================================================================================ @ParameterizedTest @MethodSource("planInputPartitionsParameters") public void testPlanInputPartitions_dataParity( long fromVersion, long toVersion, Optional maxFiles, Optional maxBytes, String testDescription, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_plan_partitions_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); insertVersions( testTableName, /* numVersions= */ 5, /* rowsPerVersion= */ 10, /* includeEmptyVersion= */ true); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); DeltaSourceOffset startOffset = new DeltaSourceOffset( deltaLog.tableId(), fromVersion, DeltaSourceOffset.BASE_INDEX(), /* isInitialSnapshot= */ false); DeltaSourceOffset planPartitionsEndOffset = new DeltaSourceOffset( deltaLog.tableId(), toVersion, DeltaSourceOffset.END_INDEX(), /* isInitialSnapshot= */ false); // Ground truth: Read directly from Delta table List expectedRows = new ArrayList<>(); Dataset toVersionData = spark.read().format("delta").option("versionAsOf", toVersion).load(testTablePath); if (fromVersion > 0) { Dataset beforeFromVersionData = spark.read().format("delta").option("versionAsOf", fromVersion - 1).load(testTablePath); toVersionData = toVersionData.except(beforeFromVersionData); } expectedRows.addAll(toVersionData.collectAsList()); // DSv2: planInputPartitions + createReaderFactory PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); org.apache.spark.sql.delta.Snapshot deltaSnapshot = deltaLog.unsafeVolatileSnapshot(); StructType dataSchema = deltaSnapshot.metadata().schema(); StructType partitionSchema = deltaSnapshot.metadata().partitionSchema(); SparkMicroBatchStream stream = new SparkMicroBatchStream( snapshotManager, snapshotManager.loadLatestSnapshot(), spark.sessionState().newHadoopConf(), spark, emptyDeltaOptions(), testTablePath, dataSchema, partitionSchema, dataSchema, new org.apache.spark.sql.sources.Filter[0], Map$.MODULE$.empty()); InputPartition[] partitions = stream.planInputPartitions(startOffset, planPartitionsEndOffset); PartitionReaderFactory readerFactory = stream.createReaderFactory(); // Simulates how Spark calls the reader factory and reads the data List dsv2Rows = new ArrayList<>(); for (InputPartition partition : partitions) { if (readerFactory.supportColumnarReads(partition)) { PartitionReader reader = readerFactory.createColumnarReader(partition); while (reader.next()) { org.apache.spark.sql.vectorized.ColumnarBatch batch = reader.get(); // Convert ColumnarBatch to Rows org.apache.spark.sql.catalyst.expressions.UnsafeProjection projection = org.apache.spark.sql.catalyst.expressions.UnsafeProjection.create(dataSchema); for (int rowId = 0; rowId < batch.numRows(); rowId++) { InternalRow internalRow = batch.getRow(rowId); Row row = convertInternalRowToRow(internalRow, dataSchema); dsv2Rows.add(row); } } reader.close(); } else { PartitionReader reader = readerFactory.createReader(partition); while (reader.next()) { InternalRow internalRow = reader.get(); // Convert InternalRow to Row for comparison using the dataSchema we already have Row row = convertInternalRowToRow(internalRow, dataSchema); dsv2Rows.add(row); } reader.close(); } } // Compare results compareDataResults(expectedRows, dsv2Rows, testDescription); } /** Provides test parameters for the planInputPartitions data parity test. */ private static Stream planInputPartitionsParameters() { Optional noMaxFiles = Optional.empty(); Optional noMaxBytes = Optional.empty(); return Stream.of( // Basic version range tests Arguments.of( /* fromVersion= */ 1L, /* toVersion= */ 2L, noMaxFiles, noMaxBytes, "Single version (1 to 2)"), Arguments.of( /* fromVersion= */ 1L, /* toVersion= */ 3L, noMaxFiles, noMaxBytes, "Multiple versions (1 to 3)"), Arguments.of( /* fromVersion= */ 0L, /* toVersion= */ 5L, noMaxFiles, noMaxBytes, "From version 0 to 5"), Arguments.of( /* fromVersion= */ 2L, /* toVersion= */ 4L, noMaxFiles, noMaxBytes, "Mid-range versions (2 to 4)"), // Rate limiting tests Arguments.of( /* fromVersion= */ 1L, /* toVersion= */ 5L, Optional.of(5), noMaxFiles, "With maxFiles limit"), Arguments.of( /* fromVersion= */ 1L, /* toVersion= */ 5L, noMaxFiles, Optional.of(5000L), "With maxBytes limit"), Arguments.of( /* fromVersion= */ 1L, /* toVersion= */ 5L, Optional.of(10), Optional.of(10000L), "With both limits"), // Edge cases Arguments.of( /* fromVersion= */ 3L, /* toVersion= */ 3L, noMaxFiles, noMaxBytes, "Same version (3 to 3)")); } // ================================================================================================ // Tests for planInputPartitions with excludeRegex // ================================================================================================ /** * Parameterized test that verifies planInputPartitions correctly applies the excludeRegex read * option. * *

Uses a partitioned table so file paths contain partition directory names (e.g. * "category=alpha/..."), making it straightforward to craft regex patterns that target specific * subsets of files. * *

Verifies correctness by extracting the {@code id} column (always at ordinal 0) from the * reader output and comparing the resulting set of IDs against the expected set. */ @ParameterizedTest @MethodSource("excludeRegexPlanInputPartitionsParameters") public void testPlanInputPartitions_excludeRegex( String excludeRegexPattern, long fromVersion, long toVersion, int[] expectedIds, boolean isInitialSnapshot, String testDescription, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_exclude_regex_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); // Create table partitioned by category with 4 versions of data. // Version 0: CREATE TABLE (empty) // Version 1: ids {1,2} in category=alpha // Version 2: ids {3,4} in category=beta // Version 3: ids {5,6} in category=alpha // Version 4: ids {7,8} in category=gamma sql( "CREATE TABLE %s (id INT, name STRING, category STRING) " + "USING delta LOCATION '%s' PARTITIONED BY (category)", testTableName, testTablePath); sql("INSERT INTO %s VALUES (1, 'Alice', 'alpha'), (2, 'Bob', 'alpha')", testTableName); sql("INSERT INTO %s VALUES (3, 'Charlie', 'beta'), (4, 'David', 'beta')", testTableName); sql("INSERT INTO %s VALUES (5, 'Eve', 'alpha'), (6, 'Frank', 'alpha')", testTableName); sql("INSERT INTO %s VALUES (7, 'Grace', 'gamma'), (8, 'Harry', 'gamma')", testTableName); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); DeltaSourceOffset startOffset = new DeltaSourceOffset( deltaLog.tableId(), fromVersion, DeltaSourceOffset.BASE_INDEX(), isInitialSnapshot); DeltaSourceOffset endOffset = new DeltaSourceOffset( deltaLog.tableId(), toVersion, DeltaSourceOffset.END_INDEX(), isInitialSnapshot); DeltaOptions options = createDeltaOptions("excludeRegex", excludeRegexPattern); // DSv2: planInputPartitions + createReaderFactory PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); org.apache.spark.sql.delta.Snapshot deltaSnapshot = deltaLog.unsafeVolatileSnapshot(); StructType fullSchema = deltaSnapshot.metadata().schema(); StructType partitionSchema = deltaSnapshot.metadata().partitionSchema(); Set partitionColNames = Arrays.stream(partitionSchema.fieldNames()).collect(Collectors.toSet()); StructType dataSchema = new StructType( Arrays.stream(fullSchema.fields()) .filter(f -> !partitionColNames.contains(f.name())) .toArray(StructField[]::new)); SparkMicroBatchStream stream = new SparkMicroBatchStream( snapshotManager, snapshotManager.loadLatestSnapshot(), spark.sessionState().newHadoopConf(), spark, options, testTablePath, dataSchema, partitionSchema, fullSchema, new org.apache.spark.sql.sources.Filter[0], Map$.MODULE$.empty()); InputPartition[] partitions = stream.planInputPartitions(startOffset, endOffset); PartitionReaderFactory readerFactory = stream.createReaderFactory(); // Extract IDs (column ordinal 0) from reader output. // We read only the id to avoid schema complications with partitioned tables where // the vectorized reader may not include partition columns in the InternalRow. List dsv2Ids = new ArrayList<>(); for (InputPartition partition : partitions) { if (readerFactory.supportColumnarReads(partition)) { PartitionReader reader = readerFactory.createColumnarReader(partition); while (reader.next()) { org.apache.spark.sql.vectorized.ColumnarBatch batch = reader.get(); for (int rowId = 0; rowId < batch.numRows(); rowId++) { dsv2Ids.add(batch.getRow(rowId).getInt(0)); } } reader.close(); } else { PartitionReader reader = readerFactory.createReader(partition); while (reader.next()) { dsv2Ids.add(reader.get().getInt(0)); } reader.close(); } } List expected = Arrays.stream(expectedIds).sorted().boxed().collect(Collectors.toList()); Collections.sort(dsv2Ids); assertEquals( expected, dsv2Ids, String.format("[%s] ID mismatch: expected=%s, got=%s", testDescription, expected, dsv2Ids)); } /** Provides test parameters for the excludeRegex planInputPartitions test. */ private static Stream excludeRegexPlanInputPartitionsParameters() { return Stream.of( Arguments.of( /* excludeRegexPattern= */ (String) null, /* fromVersion= */ 1L, /* toVersion= */ 4L, /* expectedIds= */ new int[] {1, 2, 3, 4, 5, 6, 7, 8}, /* isInitialSnapshot= */ false, "No excludeRegex - all versions"), Arguments.of( /* excludeRegexPattern= */ "category=alpha", /* fromVersion= */ 1L, /* toVersion= */ 4L, /* expectedIds= */ new int[] {3, 4, 7, 8}, /* isInitialSnapshot= */ false, "Exclude alpha category"), Arguments.of( /* excludeRegexPattern= */ "category=beta", /* fromVersion= */ 1L, /* toVersion= */ 4L, /* expectedIds= */ new int[] {1, 2, 5, 6, 7, 8}, /* isInitialSnapshot= */ false, "Exclude beta category"), Arguments.of( /* excludeRegexPattern= */ "category=gamma", /* fromVersion= */ 1L, /* toVersion= */ 4L, /* expectedIds= */ new int[] {1, 2, 3, 4, 5, 6}, /* isInitialSnapshot= */ false, "Exclude gamma category"), Arguments.of( /* excludeRegexPattern= */ "category=(alpha|beta)", /* fromVersion= */ 1L, /* toVersion= */ 4L, /* expectedIds= */ new int[] {7, 8}, /* isInitialSnapshot= */ false, "Exclude alpha and beta categories"), Arguments.of( /* excludeRegexPattern= */ "nonexistent_xyz_pattern", /* fromVersion= */ 1L, /* toVersion= */ 4L, /* expectedIds= */ new int[] {1, 2, 3, 4, 5, 6, 7, 8}, /* isInitialSnapshot= */ false, "Regex matching no files"), Arguments.of( /* excludeRegexPattern= */ "category=", /* fromVersion= */ 1L, /* toVersion= */ 4L, /* expectedIds= */ new int[] {}, /* isInitialSnapshot= */ false, "Regex matching all files"), Arguments.of( /* excludeRegexPattern= */ "category=alpha", /* fromVersion= */ 2L, /* toVersion= */ 4L, /* expectedIds= */ new int[] {3, 4, 7, 8}, /* isInitialSnapshot= */ false, "Exclude alpha versions 2 to 4"), Arguments.of( /* excludeRegexPattern= */ "category=beta", /* fromVersion= */ 2L, /* toVersion= */ 2L, /* expectedIds= */ new int[] {}, /* isInitialSnapshot= */ false, "Exclude beta version 2 only"), Arguments.of( /* excludeRegexPattern= */ (String) null, /* fromVersion= */ 0L, /* toVersion= */ 4L, /* expectedIds= */ new int[] {1, 2, 3, 4, 5, 6, 7, 8}, /* isInitialSnapshot= */ true, "Initial snapshot - no excludeRegex"), Arguments.of( /* excludeRegexPattern= */ "category=alpha", /* fromVersion= */ 0L, /* toVersion= */ 4L, /* expectedIds= */ new int[] {3, 4, 7, 8}, /* isInitialSnapshot= */ true, "Initial snapshot - exclude alpha")); } /** * Helper method to convert InternalRow to Row for comparison. * * @param internalRow The InternalRow to convert * @param schema The schema of the row * @return A Row object */ private Row convertInternalRowToRow(InternalRow internalRow, StructType schema) { // Use Spark's built-in conversion from InternalRow to Row scala.collection.Seq seq = internalRow.toSeq(schema); Object[] values = scala.collection.JavaConverters.seqAsJavaList(seq).toArray(); return new org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(values, schema); } /** * Helper method to compare data results between expected (ground truth) and DSv2. * * @param expectedRows Rows from ground truth (batch read from Delta table) * @param dsv2Rows Rows from DSv2's planInputPartitions() * @param testDescription Description of the test case for error messages */ private void compareDataResults( List expectedRows, List dsv2Rows, String testDescription) { assertEquals( expectedRows.size(), dsv2Rows.size(), String.format( "[%s] Number of rows should match: Expected=%d, DSv2=%d", testDescription, expectedRows.size(), dsv2Rows.size())); // Sort both lists for consistent comparison (order may differ due to partitioning) Comparator rowComparator = (r1, r2) -> { // Compare by id field (first column) int id1 = r1.getInt(0); int id2 = r2.getInt(0); return Integer.compare(id1, id2); }; List sortedExpected = expectedRows.stream().sorted(rowComparator).collect(Collectors.toList()); List sortedDsv2 = dsv2Rows.stream().sorted(rowComparator).collect(Collectors.toList()); // Compare each row for (int i = 0; i < sortedExpected.size(); i++) { Row expectedRow = sortedExpected.get(i); Row dsv2Row = sortedDsv2.get(i); assertEquals( expectedRow.length(), dsv2Row.length(), String.format( "[%s] Row %d length mismatch: Expected=%d, DSv2=%d", testDescription, i, expectedRow.length(), dsv2Row.length())); // Compare each field for (int fieldIdx = 0; fieldIdx < expectedRow.length(); fieldIdx++) { Object expectedValue = expectedRow.get(fieldIdx); Object dsv2Value = dsv2Row.get(fieldIdx); // Convert both values to strings for comparison to handle UTF8String vs String String expectedStr = expectedValue == null ? null : expectedValue.toString(); String dsv2Str = dsv2Value == null ? null : dsv2Value.toString(); assertEquals( expectedStr, dsv2Str, String.format( "[%s] Row %d, field %d mismatch: Expected=%s, DSv2=%s", testDescription, i, fieldIdx, expectedStr, dsv2Str)); } } } // ================================================================================================ // Tests for getStartingVersion parity between DSv1 and DSv2 // ================================================================================================ /** * Parameterized test that verifies parity between DSv1 DeltaSource.getStartingVersion and DSv2 * SparkMicroBatchStream.getStartingVersion. */ @ParameterizedTest @MethodSource("getStartingVersionParameters") public void testGetStartingVersion( String startingVersion, Optional expectedVersion, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_starting_version_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); // Create 5 versions (version 0 = CREATE TABLE, versions 1-5 = INSERTs) insertVersions( testTableName, /* numVersions= */ 5, /* rowsPerVersion= */ 1, /* includeEmptyVersion= */ false); testAndCompareStartingVersion( testTablePath, startingVersion, expectedVersion, "startingVersion=" + startingVersion); } /** Provides test parameters for the parameterized getStartingVersion test. */ private static Stream getStartingVersionParameters() { return Stream.of( Arguments.of(/* startingVersion= */ "0", /* expectedVersion= */ Optional.of(0L)), Arguments.of(/* startingVersion= */ "1", /* expectedVersion= */ Optional.of(1L)), Arguments.of(/* startingVersion= */ "3", /* expectedVersion= */ Optional.of(3L)), Arguments.of(/* startingVersion= */ "5", /* expectedVersion= */ Optional.of(5L)), Arguments.of(/* startingVersion= */ "latest", /* expectedVersion= */ Optional.of(6L)), Arguments.of(/* startingVersion= */ null, /* expectedVersion= */ Optional.empty())); } /** * Test that verifies both DSv1 and DSv2 handle the case where no DeltaOptions are provided. DSv1 * receives an empty DeltaOptions (no parameters), while DSv2 receives Optional.empty(). This * tests the equivalence between these two approaches. */ @Test public void testGetStartingVersion_noOptions(@TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_no_options_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); // Create 5 versions (version 0 = CREATE TABLE, versions 1-5 = INSERTs) insertVersions( testTableName, /* numVersions= */ 5, /* rowsPerVersion= */ 1, /* includeEmptyVersion= */ false); // dsv1 DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); DeltaOptions emptyOptions = emptyDeltaOptions(); DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath, emptyOptions); scala.Option dsv1Result = deltaSource.getStartingVersion(); // dsv2 PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, new Configuration()); SparkMicroBatchStream dsv2Stream = createTestStreamWithDefaults(snapshotManager, new Configuration(), emptyDeltaOptions()); Optional dsv2Result = dsv2Stream.getStartingVersion(); compareStartingVersionResults(dsv1Result, dsv2Result, Optional.empty(), "No options provided"); } /** Test that verifies both DSv1 and DSv2 handle negative startingVersion values identically. */ @Test public void testGetStartingVersion_negativeVersion_throwsError(@TempDir File tempDir) throws Exception { // Negative values are rejected during DeltaOptions parsing, before getStartingVersion is // called. assertThrows(IllegalArgumentException.class, () -> createDeltaOptions("startingVersion", "-1")); } /** * Parameterized test that verifies both DSv1 and DSv2 handle the protocol validation behavior * identically with the validation flag on/off. * *

When protocol validation is enabled, validateProtocolAt is called and must succeed. When * disabled, the code immediately falls back to checkVersionExists without protocol validation. */ @ParameterizedTest @MethodSource("protocolValidationParameters") public void testGetStartingVersion_protocolValidationFlag( boolean enableProtocolValidation, String startingVersion, String testDescription, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_protocol_fallback_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); // Create 5 versions (version 0 = CREATE TABLE, versions 1-5 = INSERTs) insertVersions( testTableName, /* numVersions= */ 5, /* rowsPerVersion= */ 1, /* includeEmptyVersion= */ false); // Test with protocol validation enabled/disabled String configKey = DeltaSQLConf.FAST_DROP_FEATURE_STREAMING_ALWAYS_VALIDATE_PROTOCOL().key(); try { spark.conf().set(configKey, String.valueOf(enableProtocolValidation)); testAndCompareStartingVersion( testTablePath, startingVersion, Optional.of(Long.parseLong(startingVersion)), testDescription); } finally { spark.conf().unset(configKey); } } /** Provides test parameters for protocol validation scenarios. */ private static Stream protocolValidationParameters() { return Stream.of( Arguments.of( /* enableProtocolValidation= */ true, /* startingVersion= */ "2", "Protocol validation enabled"), Arguments.of( /* enableProtocolValidation= */ false, /* startingVersion= */ "3", "Protocol validation disabled")); } // TODO(#5320): Add test for unsupported table feature // Test case where protocol validation encounters an unsupported table feature and throws // (does NOT fall back to checkVersionExists). This is difficult to test reliably as it // requires creating a table with features that Kernel doesn't support, which Spark SQL // validates upfront. This scenario is tested through integration tests. /** * Test case where protocol validation fails with a non-feature exception (snapshot cannot be * recreated), but checkVersionExists succeeds (commit logically exists). * *

Scenario: After creating a checkpoint at version 10, old log files 0-5 are deleted * (simulating log cleanup by timestamp). This makes version 7 non-recreatable (it exists between * the deleted logs and the checkpoint). Protocol validation fails when trying to build snapshot * at version 7, but checkVersionExists succeeds because the commit still logically exists. */ @Test public void testGetStartingVersion_protocolValidationNonFeatureExceptionFallback( @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_non_recreatable_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); // Create 10 versions (version 0 = CREATE TABLE, versions 1-10 = INSERTs) insertVersions( testTableName, /* numVersions= */ 10, /* rowsPerVersion= */ 1, /* includeEmptyVersion= */ false); // Create checkpoint at version 10 DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); Snapshot snapshotV10 = deltaLog.getSnapshotAt( 10, Option.empty(), Option.empty(), false); deltaLog.checkpoint(snapshotV10, Option.empty()); // Simulate log cleanup by timestamp: delete logs 0-5 // This makes version 7 non-recreatable while allowing DeltaLog to load the latest snapshot Path logPath = new Path(testTablePath, "_delta_log"); for (long version = 0; version <= 5; version++) { Path logFile = new Path(logPath, String.format("%020d.json", version)); File file = new File(logFile.toUri().getPath()); if (file.exists()) { file.delete(); } } // Test with startingVersion=7 (a version that's no longer recreatable but logically exists) String startingVersion = "7"; // dsv1 DeltaLog freshDeltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); DeltaSource deltaSource = createDeltaSource( freshDeltaLog, testTablePath, createDeltaOptions("startingVersion", startingVersion)); scala.Option dsv1Result = deltaSource.getStartingVersion(); // dsv2 PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, new Configuration()); SparkMicroBatchStream dsv2Stream = createTestStreamWithDefaults( snapshotManager, new Configuration(), createDeltaOptions("startingVersion", startingVersion)); Optional dsv2Result = dsv2Stream.getStartingVersion(); compareStartingVersionResults( dsv1Result, dsv2Result, Optional.of(Long.parseLong(startingVersion)), "Protocol validation fallback with non-recreatable version"); } /** * Test that verifies parity between DSv1 DeltaSource.getStartingVersion and DSv2 * SparkMicroBatchStream.getStartingVersion when using startingTimestamp option. * *

Uses ICT (In-Commit Timestamps) so we can read exact commit timestamps from the delta log * and test the boundary case where startingTimestamp exactly equals a commit time. */ @Test public void testGetStartingVersionFromTimestamp(@TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_starting_timestamp_" + System.nanoTime(); // Enable ICT so commit timestamps are deterministic and stored in the delta log sql( "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'" + " TBLPROPERTIES ('delta.enableInCommitTimestamps' = 'true')", testTableName, testTablePath); // Version 0 String beforeV1TS = new Timestamp(System.currentTimeMillis()).toString(); // Version 1 sql("INSERT INTO %s VALUES (1, 'User1')", testTableName); Thread.sleep(10); String betweenV1V2TS = new Timestamp(System.currentTimeMillis()).toString(); // Version 2 sql("INSERT INTO %s VALUES (2, 'User2')", testTableName); Thread.sleep(10); String afterV2TS = new Timestamp(System.currentTimeMillis()).toString(); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); // Read exact commit timestamps from the delta log for boundary testing scala.collection.Seq history = deltaLog.history().getHistory(0L, scala.Option.apply(2L), scala.Option.empty()); java.util.Map versionTimestamps = new java.util.HashMap<>(); for (int i = 0; i < history.size(); i++) { DeltaHistory entry = history.apply(i); long version = (long) entry.version().get(); versionTimestamps.put(version, entry.timestamp()); } String v1ExactTS = versionTimestamps.get(1L).toString(); String v2ExactTS = versionTimestamps.get(2L).toString(); class TimestampTestCase { final String timestamp; final long expectedVersion; final String message; TimestampTestCase(String timestamp, long expectedVersion, String message) { this.timestamp = timestamp; this.expectedVersion = expectedVersion; this.message = message; } } TimestampTestCase[] testCases = { new TimestampTestCase(beforeV1TS, 1L, "timestamp between v0 and v1 should return version 1"), new TimestampTestCase( v1ExactTS, 1L, "timestamp exactly at v1 commit time should return version 1"), new TimestampTestCase( betweenV1V2TS, 2L, "timestamp between v1 and v2 should return version 2"), new TimestampTestCase( v2ExactTS, 2L, "timestamp exactly at v2 commit time should return version 2") }; for (TimestampTestCase testCase : testCases) { String timestamp = testCase.timestamp; long expectedVersion = testCase.expectedVersion; String message = testCase.message; // dsv1 DeltaSource deltaSource = createDeltaSource( deltaLog, testTablePath, createDeltaOptions("startingTimestamp", timestamp)); scala.Option dsv1Result = deltaSource.getStartingVersion(); // dsv2 PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, new Configuration()); SparkMicroBatchStream dsv2Stream = createTestStreamWithDefaults( snapshotManager, new Configuration(), createDeltaOptions("startingTimestamp", timestamp)); Optional dsv2Result = dsv2Stream.getStartingVersion(); compareStartingVersionResults(dsv1Result, dsv2Result, Optional.of(expectedVersion), message); } // dsv1 DeltaSource deltaSource = createDeltaSource( deltaLog, testTablePath, createDeltaOptions("startingTimestamp", afterV2TS)); DeltaAnalysisException dsv1Exception = assertThrows( DeltaAnalysisException.class, deltaSource::getStartingVersion, String.format( "DSv1 should throw when no commit after timestamp and not allow out of range")); // dsv2 PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, new Configuration()); SparkMicroBatchStream dsv2Stream = createTestStreamWithDefaults( snapshotManager, new Configuration(), createDeltaOptions("startingTimestamp", afterV2TS)); DeltaAnalysisException dsv2Exception = assertThrows( DeltaAnalysisException.class, dsv2Stream::getStartingVersion, String.format( "DSv2 should throw when no commit after timestamp and not allow out of range")); assertEquals( dsv1Exception.getErrorClass(), dsv2Exception.getErrorClass(), "v1 connector and v2 connector should throw the same error class when no commit after timestamp and not allow out of range"); assertEquals( dsv1Exception.getMessageParameters(), dsv2Exception.getMessageParameters(), "v1 connector and v2 connector should throw the same error messages when no commit after timestamp and not allow out of range"); } // ================================================================================================ // Tests for checkReadIncompatibleSchemaChanges parity between v1 connector vs v2 connector // ================================================================================================ // TODO(#5319): Tests on RESTORE on delta table after applying an additive schema change /** * Parameterized test that verifies both DSv1 and DSv2 throw DeltaIllegalStateException when * encountering forward-fill additive schema change actions. */ @ParameterizedTest @MethodSource("additiveSchemaEvolutionScenarios") public void testSchemaEvolution_onForwardAdditiveChanges_throwsError( ScenarioSetup scenarioSetup, Map sparkConf, String testDescription, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_forward_additive_changes" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); createSchemaEvolutionTestTable(testTablePath, testTableName); // Try to read from version 0, which should include commits with METADATA actions long fromVersion = 0L; long fromIndex = DeltaSourceOffset.BASE_INDEX(); boolean isInitialSnapshot = false; Option endOffset = Option.empty(); try { // setup specific spark config sparkConf.forEach((key, value) -> spark.conf().set(key, value)); // Create DSv1 DeltaSource DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath); // Create DSv2 SparkMicroBatchStream Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); // Execute schema change after source initialization to ensure forward change scenarioSetup.setup(testTableName, tempDir); DeltaIllegalStateException dsv1Exception = assertThrows( DeltaIllegalStateException.class, () -> { ClosableIterator deltaChanges = deltaSource.getFileChanges( fromVersion, fromIndex, isInitialSnapshot, endOffset, /* verifyMetadataAction= */ true); // Consume the iterator to trigger validation while (deltaChanges.hasNext()) { // Exception is thrown by .next() when it encounters a REMOVE deltaChanges.next(); } deltaChanges.close(); }, String.format("DSv1 should throw on METADATA for scenario: %s", testDescription)); DeltaIllegalStateException dsv2Exception = assertThrows( DeltaIllegalStateException.class, () -> { CloseableIterator kernelChanges = stream.getFileChanges( fromVersion, fromIndex, isInitialSnapshot, ScalaUtils.toJavaOptional(endOffset)); try { // Consume the iterator to trigger validation (if not already triggered) while (kernelChanges.hasNext()) { kernelChanges.next(); } kernelChanges.close(); } finally { // Make sure to close the iterator even if exception occurs if (kernelChanges != null) { try { kernelChanges.close(); } catch (Exception ignored) { } } } }, String.format("DSv2 should throw on METADATA for scenario: %s", testDescription)); assertEquals( dsv1Exception.getErrorClass(), dsv2Exception.getErrorClass(), "v1 connector and v2 connector should throw the same error class on forward-fill additive schema changes"); assertEquals( dsv1Exception.getMessageParameters(), dsv2Exception.getMessageParameters(), "v1 connector and v2 connector should throw the same error messages on forward-fill additive schema changes"); } finally { // recover spark config to original state sparkConf.forEach((key, value) -> spark.conf().unset(key)); } } /** * Parameterized test that verifies both DSv1 and DSv2 return the same file changes when * encountering backfill additive schema change actions. */ @ParameterizedTest @MethodSource("additiveSchemaEvolutionScenarios") public void testSchemaEvolution_onBackfillAdditiveChanges( ScenarioSetup scenarioSetup, Map sparkConf, String testDescription, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_backfill_additive_changes" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); createSchemaEvolutionTestTable(testTablePath, testTableName); // Execute schema change before source initialization to ensure backfill change scenarioSetup.setup(testTableName, tempDir); // Try to read from version 0, which should include commits with METADATA actions long fromVersion = 0L; long fromIndex = DeltaSourceOffset.BASE_INDEX(); boolean isInitialSnapshot = false; Option endOffset = Option.empty(); try { // setup specific spark config sparkConf.forEach((key, value) -> spark.conf().set(key, value)); // Test DSv1 DeltaSource DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath); ClosableIterator deltaChanges = deltaSource.getFileChanges( fromVersion, fromIndex, isInitialSnapshot, endOffset, /* verifyMetadataAction= */ true); List deltaFilesList = new ArrayList<>(); while (deltaChanges.hasNext()) { deltaFilesList.add(deltaChanges.next()); } deltaChanges.close(); // Test DSv2 SparkMicroBatchStream Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); try (CloseableIterator kernelChanges = stream.getFileChanges( fromVersion, fromIndex, isInitialSnapshot, ScalaUtils.toJavaOptional(endOffset))) { List kernelFilesList = new ArrayList<>(); while (kernelChanges.hasNext()) { kernelFilesList.add(kernelChanges.next()); } compareFileChanges(deltaFilesList, kernelFilesList); } } finally { // recover spark config to original state sparkConf.forEach((key, value) -> spark.conf().unset(key)); } } /** * Parameterized test that verifies both DSv1 and DSv2 throw Exception when encountering * forward-fill non-additive schema change actions. */ @ParameterizedTest @MethodSource("nonAdditiveSchemaEvolutionScenarios") public void testSchemaEvolution_onForwardNonAdditiveChanges_throwsError( ScenarioSetup scenarioSetup, Map sparkConf, String testDescription, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_forward_non_additive_changes" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); createSchemaEvolutionTestTable(testTablePath, testTableName); // Try to read from version 0, which should include commits with METADATA actions long fromVersion = 0L; long fromIndex = DeltaSourceOffset.BASE_INDEX(); boolean isInitialSnapshot = false; Option endOffset = Option.empty(); try { // setup specific spark config sparkConf.forEach((key, value) -> spark.conf().set(key, value)); // Create DSv1 DeltaSource DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath); // Create DSv2 SparkMicroBatchStream Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); // Execute schema change after source initialization to ensure forward change scenarioSetup.setup(testTableName, tempDir); DeltaUnsupportedOperationException dsv1Exception = assertThrows( DeltaUnsupportedOperationException.class, () -> { ClosableIterator deltaChanges = deltaSource.getFileChanges( fromVersion, fromIndex, isInitialSnapshot, endOffset, /* verifyMetadataAction= */ true); // Consume the iterator to trigger validation while (deltaChanges.hasNext()) { // Exception is thrown by .next() when it encounters a REMOVE deltaChanges.next(); } deltaChanges.close(); }, String.format("DSv1 should throw on METADATA for scenario: %s", testDescription)); DeltaUnsupportedOperationException dsv2Exception = assertThrows( DeltaUnsupportedOperationException.class, () -> { CloseableIterator kernelChanges = stream.getFileChanges( fromVersion, fromIndex, isInitialSnapshot, ScalaUtils.toJavaOptional(endOffset)); try { // Consume the iterator to trigger validation (if not already triggered) while (kernelChanges.hasNext()) { kernelChanges.next(); } kernelChanges.close(); } finally { // Make sure to close the iterator even if exception occurs if (kernelChanges != null) { try { kernelChanges.close(); } catch (Exception ignored) { } } } }, String.format("DSv2 should throw on METADATA for scenario: %s", testDescription)); // TODO(#5319): assertEqual after schema tracking log is supported String expectedPrefix = "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE"; assertTrue( dsv1Exception.getErrorClass().startsWith(expectedPrefix), String.format( "v1 connector error class should start with %s, but got: %s", expectedPrefix, dsv1Exception.getErrorClass())); assertTrue( dsv2Exception.getErrorClass().startsWith(expectedPrefix), String.format( "v2 connector error class should start with %s, but got: %s", expectedPrefix, dsv2Exception.getErrorClass())); assertEquals( dsv1Exception.getMessageParameters(), dsv2Exception.getMessageParameters(), "v1 connector and v2 connector should throw the same error messages on forward-fill non-additive schema changes"); } finally { // recover spark config to original state sparkConf.forEach((key, value) -> spark.conf().unset(key)); } } /** * Parameterized test that verifies both DSv1 and DSv2 throw Exception when encountering backfill * non-additive schema change actions. */ @ParameterizedTest @MethodSource("nonAdditiveSchemaEvolutionScenarios") public void testSchemaEvolution_onBackfillNonAdditiveChanges_throwsError( ScenarioSetup scenarioSetup, Map sparkConf, String testDescription, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_backfill_non_additive_changes" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); createSchemaEvolutionTestTable(testTablePath, testTableName); // Try to read from version 0, which should include commits with METADATA actions long fromVersion = 0L; long fromIndex = DeltaSourceOffset.BASE_INDEX(); boolean isInitialSnapshot = false; Option endOffset = Option.empty(); try { // setup specific spark config sparkConf.forEach((key, value) -> spark.conf().set(key, value)); // Execute schema change before source initialization to ensure backfill change scenarioSetup.setup(testTableName, tempDir); // Create DSv1 DeltaSource DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath); // Create DSv2 SparkMicroBatchStream Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); DeltaUnsupportedOperationException dsv1Exception = assertThrows( DeltaUnsupportedOperationException.class, () -> { ClosableIterator deltaChanges = deltaSource.getFileChanges( fromVersion, fromIndex, isInitialSnapshot, endOffset, /* verifyMetadataAction= */ true); // Consume the iterator to trigger validation while (deltaChanges.hasNext()) { // Exception is thrown by .next() when it encounters a REMOVE deltaChanges.next(); } deltaChanges.close(); }, String.format("DSv1 should throw on METADATA for scenario: %s", testDescription)); DeltaUnsupportedOperationException dsv2Exception = assertThrows( DeltaUnsupportedOperationException.class, () -> { CloseableIterator kernelChanges = stream.getFileChanges( fromVersion, fromIndex, isInitialSnapshot, ScalaUtils.toJavaOptional(endOffset)); try { // Consume the iterator to trigger validation (if not already triggered) while (kernelChanges.hasNext()) { kernelChanges.next(); } kernelChanges.close(); } finally { // Make sure to close the iterator even if exception occurs if (kernelChanges != null) { try { kernelChanges.close(); } catch (Exception ignored) { } } } }, String.format("DSv2 should throw on METADATA for scenario: %s", testDescription)); // TODO(#5319): assertEqual after schema tracking log is supported String expectedPrefix = "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE"; assertTrue( dsv1Exception.getErrorClass().startsWith(expectedPrefix), String.format( "v1 connector error class should start with %s, but got: %s", expectedPrefix, dsv1Exception.getErrorClass())); assertTrue( dsv2Exception.getErrorClass().startsWith(expectedPrefix), String.format( "v2 connector error class should start with %s, but got: %s", expectedPrefix, dsv2Exception.getErrorClass())); assertEquals( dsv1Exception.getMessageParameters(), dsv2Exception.getMessageParameters(), "v1 connector and v2 connector should throw the same error messages on backfill non-additive schema changes"); } finally { // recover spark config to original state sparkConf.forEach((key, value) -> spark.conf().unset(key)); } } /** * Test that verifies DSv1 and DSv2 throw errors when the starting snapshot has an incompatible * schema change that gets reverted before the latest version. * *

Edge case: checkReadIncompatibleSchemaChange only checks metadata actions, so it misses the * incompatible intermediate state (id → userId → id). The * checkReadIncompatibleSchemaChangeOnStreamStartOnce method catches this by validating each * snapshot in the range. */ @Test public void testSchemaEvolution_onStreamStartOnce(@TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testDescription = "testSchemaEvolution_onStreamStartOnce"; String testTableName = "test_schema_changes_on_stream_start_once" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); createSchemaEvolutionTestTable(testTablePath, testTableName); // Execute schema change before source initialization to ensure backfill change spark.sql(String.format("ALTER table %s RENAME COLUMN id TO userId", testTableName)); spark.sql( String.format( "INSERT INTO %s VALUES (3, 'Cathy', 5, named_struct('col1', 18, 'col2', 'SF'))", testTableName)); // Record the version prior to reverting schema change long incompatibleSchemaVersion = DeltaLog.forTable(spark, new Path(testTablePath)) .update(false, Option.empty(), Option.empty()) .version(); // Revert the schema change spark.sql(String.format("ALTER table %s RENAME COLUMN userId TO id", testTableName)); spark.sql( String.format( "INSERT INTO %s VALUES (4, 'David', 8, named_struct('col1', 47, 'col2', 'DC'))", testTableName)); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); String tableId = deltaLog.tableId(); // Try to read from version 0 without readLimit to check all commits DeltaSourceOffset startOffset = new DeltaSourceOffset( tableId, incompatibleSchemaVersion, DeltaSourceOffset.BASE_INDEX(), /* isInitialSnapshot= */ false); ReadLimit readLimit = ReadLimitConfig.noLimit().toReadLimit(); // Test DSv1 DeltaSource DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath); DeltaUnsupportedOperationException dsv1Exception = assertThrows( DeltaUnsupportedOperationException.class, () -> deltaSource.latestOffset(startOffset, readLimit), String.format( "DSv1 should throw error on stream start for scenario: %s", testDescription)); assertThat(dsv1Exception.getStackTrace()) .as("Error should be thrown by 'checkReadIncompatibleSchemaChangeOnStreamStartOnce'") .anyMatch( element -> element.toString().contains("checkReadIncompatibleSchemaChangeOnStreamStartOnce")); // Test DSv2 SparkMicroBatchStream Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); DeltaUnsupportedOperationException dsv2Exception = assertThrows( DeltaUnsupportedOperationException.class, () -> stream.latestOffset(startOffset, readLimit), String.format( "DSv2 should throw error on stream start for scenario: %s", testDescription)); assertThat(dsv2Exception.getStackTrace()) .as("Error should be thrown by 'checkReadIncompatibleSchemaChangeOnStreamStartOnce'") .anyMatch( element -> element.toString().contains("checkReadIncompatibleSchemaChangeOnStreamStartOnce")); // TODO(#5319): assertEqual after schema tracking log is supported String expectedPrefix = "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE"; assertTrue( dsv1Exception.getErrorClass().startsWith(expectedPrefix), String.format( "v1 connector error class should start with %s, but got: %s", expectedPrefix, dsv1Exception.getErrorClass())); assertTrue( dsv2Exception.getErrorClass().startsWith(expectedPrefix), String.format( "v2 connector error class should start with %s, but got: %s", expectedPrefix, dsv2Exception.getErrorClass())); assertEquals( dsv1Exception.getMessageParameters(), dsv2Exception.getMessageParameters(), "v1 connector and v2 connector should throw the same error messages on stream start schema changes"); } /** Provides test scenarios that generate additive schema changes actions. */ private static Stream additiveSchemaEvolutionScenarios() { return Stream.of( // Add nullable INT column Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("ALTER TABLE %s ADD COLUMN age INT", tableName); }, /* sparkConf */ Map.of(), "Add nullable INT column"), // Add nullable STRING column Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("ALTER TABLE %s ADD COLUMN address STRING", tableName); }, /* sparkConf */ Map.of(), "Add nullable STRING column"), // Add nullable STRUCT column Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql( "ALTER TABLE %s ADD COLUMN (address STRUCT)", tableName); }, /* sparkConf */ Map.of(), "Add nullable STRUCT column"), // Add multiple nullable columns Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql( "ALTER TABLE %s ADD COLUMN (address STRING, zip INT, time TIMESTAMP)", tableName); }, /* sparkConf */ Map.of(), "Add multiple nullable columns"), // Make non-nullable column nullable Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("ALTER TABLE %s ALTER COLUMN id DROP NOT NULL", tableName); }, /* sparkConf */ Map.of(), "Make non-nullable column nullable"), // Add nullable column and then make non-nullable column nullable Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("ALTER TABLE %s ALTER COLUMN id DROP NOT NULL", tableName); sql("ALTER TABLE %s ADD COLUMN age INT", tableName); }, /* sparkConf */ Map.of(), "Add nullable column and then make non-nullable column nullable"), // Make non-nullable column nullable and then add nullable column Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("ALTER TABLE %s ADD COLUMN age INT", tableName); sql("ALTER TABLE %s ALTER COLUMN id DROP NOT NULL", tableName); }, /* sparkConf */ Map.of(), "Make non-nullable column nullable and then add nullable column"), // Widen INT column to BIGINT Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("ALTER TABLE %s ALTER COLUMN id TYPE BIGINT", tableName); }, // Set enableSchemaTrackingForTypeWidening to be false to treat widening type changes as // additive /* sparkConf */ Map.of( DeltaSQLConf.DELTA_TYPE_WIDENING_ENABLE_STREAMING_SCHEMA_TRACKING().key(), "false"), "Widen INT column to BIGINT")); } /** Provides test scenarios that generate non-additive schema changes actions. */ private static Stream nonAdditiveSchemaEvolutionScenarios() { return Stream.of( // Rename column Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("ALTER TABLE %s RENAME COLUMN id TO userId", tableName); }, /* sparkConf */ Map.of(), "Rename column"), // Drop nullable, non-nullable and struct columns Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("ALTER TABLE %s DROP COLUMNS (id, value, info)", tableName); }, /* sparkConf */ Map.of( DeltaSQLConf .DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_COLUMN_MAPPING_SCHEMA_CHANGES() .key(), "false"), "Drop nullable, non-nullable and struct columns"), // Drop column in nested struct Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("ALTER TABLE %s DROP COLUMNS info.col1", tableName); }, /* sparkConf */ Map.of( DeltaSQLConf .DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_COLUMN_MAPPING_SCHEMA_CHANGES() .key(), "false"), "Drop column in nested struct"), // Widen INT column to BIGINT Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { sql("ALTER TABLE %s ALTER COLUMN id TYPE BIGINT", tableName); }, // Set enableSchemaTrackingForTypeWidening to be true to treat widening type changes as // non-additive /* sparkConf */ Map.of( DeltaSQLConf.DELTA_TYPE_WIDENING_ENABLE_STREAMING_SCHEMA_TRACKING().key(), "true"), "Widen INT column to BIGINT")); } // ================================================================================================ // Helper methods // ================================================================================================ /** Functional interface for setting up test scenarios. */ @FunctionalInterface interface ScenarioSetup { /** * Set up the test scenario by executing SQL statements. * * @param tableName The name of the test table * @param tempDir The temporary directory for this test */ void setup(String tableName, File tempDir) throws Exception; } static class ReadLimitConfig { private final Optional maxFiles; private final Optional maxBytes; private ReadLimitConfig(Optional maxFiles, Optional maxBytes) { this.maxFiles = maxFiles; this.maxBytes = maxBytes; } static ReadLimitConfig noLimit() { return new ReadLimitConfig(Optional.empty(), Optional.empty()); } static ReadLimitConfig maxFiles(int files) { return new ReadLimitConfig(Optional.of(files), Optional.empty()); } static ReadLimitConfig maxBytes(long bytes) { return new ReadLimitConfig(Optional.empty(), Optional.of(bytes)); } ReadLimit toReadLimit() { if (maxFiles.isPresent()) { return ReadLimit.maxFiles(maxFiles.get()); } else if (maxBytes.isPresent()) { return new ReadMaxBytes(maxBytes.get()); } else { return ReadLimit.allAvailable(); } } } private void compareOffsets(Offset dsv1Offset, Offset dsv2Offset, String testDescription) { if (dsv1Offset == null && dsv2Offset == null) { return; // Both null is valid (no data case) } // Both should be non-null or both should be null if (dsv1Offset == null || dsv2Offset == null) { throw new AssertionError( String.format( "Offset mismatch for test '%s': DSv1=%s, DSv2=%s", testDescription, dsv1Offset, dsv2Offset)); } DeltaSourceOffset dsv1DeltaOffset = (DeltaSourceOffset) dsv1Offset; DeltaSourceOffset dsv2DeltaOffset = (DeltaSourceOffset) dsv2Offset; assertEquals( dsv1DeltaOffset.reservoirVersion(), dsv2DeltaOffset.reservoirVersion(), String.format( "Version mismatch for test '%s': DSv1=%d, DSv2=%d", testDescription, dsv1DeltaOffset.reservoirVersion(), dsv2DeltaOffset.reservoirVersion())); assertEquals( dsv1DeltaOffset.index(), dsv2DeltaOffset.index(), String.format( "Index mismatch for test '%s': DSv1=%d, DSv2=%d", testDescription, dsv1DeltaOffset.index(), dsv2DeltaOffset.index())); assertEquals( dsv1DeltaOffset.isInitialSnapshot(), dsv2DeltaOffset.isInitialSnapshot(), String.format( "isInitialSnapshot mismatch for test '%s': DSv1=%b, DSv2=%b", testDescription, dsv1DeltaOffset.isInitialSnapshot(), dsv2DeltaOffset.isInitialSnapshot())); } /** Helper method to execute SQL with String.format. */ private static void sql(String query, Object... args) { DeltaV2TestBase.spark.sql(String.format(query, args)); } /** * Helper method to insert multiple versions of data into a test table. * * @param tableName The name of the table to insert into * @param numVersions The number of versions (commits) to create * @param rowsPerVersion The number of rows to insert per version * @param includeEmptyVersion Whether to include an empty version (metadata-only change) at * version 1 */ private void insertVersions( String tableName, int numVersions, int rowsPerVersion, boolean includeEmptyVersion) { for (int i = 0; i < numVersions; i++) { if (i == 1 && includeEmptyVersion) { sql("ALTER TABLE %s SET TBLPROPERTIES ('test.property' = 'value')", tableName); } else { StringBuilder values = new StringBuilder(); for (int j = 0; j < rowsPerVersion; j++) { if (j > 0) values.append(", "); int id = i * rowsPerVersion + j; values.append(String.format("(%d, 'User%d')", id, id)); } sql("INSERT INTO %s VALUES %s", tableName, values.toString()); } } } private Optional createAdmissionLimits( DeltaSource deltaSource, Optional maxFiles, Optional maxBytes) { Option scalaMaxFiles = ScalaUtils.toScalaOption(maxFiles.map(i -> (Object) i)); Option scalaMaxBytes = ScalaUtils.toScalaOption(maxBytes.map(l -> (Object) l)); if (scalaMaxFiles.isEmpty() && scalaMaxBytes.isEmpty()) { return Optional.empty(); } DeltaOptions options = emptyDeltaOptions(); return Optional.of(new DeltaSource.AdmissionLimits(options, scalaMaxFiles, scalaMaxBytes)); } /** Helper method to format a DSv1 IndexedFile for debugging. */ private String formatIndexedFile(org.apache.spark.sql.delta.sources.IndexedFile file) { return String.format( "IndexedFile(version=%d, index=%d, hasAdd=%b)", file.version(), file.index(), file.add() != null); } /** Helper method to format a DSv2 IndexedFile for debugging. */ private String formatKernelIndexedFile(IndexedFile file) { return String.format( "IndexedFile(version=%d, index=%d, hasAdd=%b)", file.getVersion(), file.getIndex(), file.getAddFile() != null); } private List advanceOffsetSequenceDsv1( DeltaSource deltaSource, Offset startOffset, int numIterations, ReadLimit limit) { List offsets = new ArrayList<>(); offsets.add(startOffset); Offset currentOffset = startOffset; for (int i = 0; i < numIterations; i++) { Offset nextOffset = deltaSource.latestOffset(currentOffset, limit); offsets.add(nextOffset); currentOffset = nextOffset; } return offsets; } private List advanceOffsetSequenceDsv2( SparkMicroBatchStream stream, Offset startOffset, int numIterations, ReadLimit limit) { List offsets = new ArrayList<>(); offsets.add(startOffset); Offset currentOffset = startOffset; for (int i = 0; i < numIterations; i++) { Offset nextOffset = stream.latestOffset(currentOffset, limit); offsets.add(nextOffset); currentOffset = nextOffset; } return offsets; } private void compareOffsetSequence( List dsv1Offsets, List dsv2Offsets, String testDescription) { assertEquals( dsv1Offsets.size(), dsv2Offsets.size(), String.format( "Offset sequence length mismatch for test '%s': DSv1=%d, DSv2=%d", testDescription, dsv1Offsets.size(), dsv2Offsets.size())); for (int i = 0; i < dsv1Offsets.size(); i++) { compareOffsets( dsv1Offsets.get(i), dsv2Offsets.get(i), String.format("%s (iteration %d)", testDescription, i)); } } private DeltaSource createDeltaSource(DeltaLog deltaLog, String tablePath) { DeltaOptions options = emptyDeltaOptions(); return createDeltaSource(deltaLog, tablePath, options); } private DeltaSource createDeltaSource(DeltaLog deltaLog, String tablePath, DeltaOptions options) { Seq emptySeq = JavaConverters.asScalaBuffer(new ArrayList()).toList(); Snapshot snapshot = deltaLog.update(false, Option.empty(), Option.empty()); return new DeltaSource( spark, deltaLog, /* catalogTableOpt= */ Option.empty(), options, /* snapshotAtSourceInit= */ snapshot, /* metadataPath= */ tablePath + "/_checkpoint", /* metadataTrackingLog= */ Option.empty(), /* filters= */ emptySeq); } /** Helper method to create a SparkMicroBatchStream with default values for testing. */ private SparkMicroBatchStream createTestStreamWithDefaults( PathBasedSnapshotManager snapshotManager, Configuration hadoopConf, DeltaOptions options) { io.delta.kernel.Snapshot snapshot = snapshotManager.loadLatestSnapshot(); StructType tableSchema = io.delta.spark.internal.v2.utils.SchemaUtils.convertKernelSchemaToSparkSchema( snapshot.getSchema()); return new SparkMicroBatchStream( snapshotManager, snapshot, hadoopConf, spark, options, /* tablePath= */ "", /* dataSchema= */ tableSchema, /* partitionSchema= */ new StructType(), /* readDataSchema= */ new StructType(), /* dataFilters= */ new org.apache.spark.sql.sources.Filter[0], /* scalaOptions= */ scala.collection.immutable.Map$.MODULE$.empty()); } /** Helper method to create DeltaOptions with read option for testing. */ private DeltaOptions createDeltaOptions(String optionName, String optionValue) { if (optionName == null || optionValue == null) { // Empty options return emptyDeltaOptions(); } else { // Create Scala Map with read option scala.collection.immutable.Map scalaMap = Map$.MODULE$.empty().updated(optionName, optionValue); return new DeltaOptions(scalaMap, spark.sessionState().conf()); } } /** Helper method to test and compare getStartingVersion results from DSv1 and DSv2. */ private void testAndCompareStartingVersion( String testTablePath, String startingVersion, Optional expectedVersion, String testDescription) throws Exception { // DSv1: Create DeltaSource and get starting version DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); DeltaSource deltaSource = createDeltaSource( deltaLog, testTablePath, createDeltaOptions("startingVersion", startingVersion)); scala.Option dsv1Result = deltaSource.getStartingVersion(); // DSv2: Create SparkMicroBatchStream and get starting version PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, new Configuration()); SparkMicroBatchStream dsv2Stream = createTestStreamWithDefaults( snapshotManager, new Configuration(), createDeltaOptions("startingVersion", startingVersion)); Optional dsv2Result = dsv2Stream.getStartingVersion(); compareStartingVersionResults(dsv1Result, dsv2Result, expectedVersion, testDescription); } /** Helper method to compare getStartingVersion results from DSv1 and DSv2. */ private void compareStartingVersionResults( scala.Option dsv1Result, Optional dsv2Result, Optional expectedVersion, String testDescription) { Optional dsv1Optional; if (dsv1Result.isEmpty()) { dsv1Optional = Optional.empty(); } else { dsv1Optional = Optional.of((Long) dsv1Result.get()); } assertEquals( dsv1Optional, dsv2Result, String.format("DSv1 and DSv2 getStartingVersion should match for %s", testDescription)); assertEquals( expectedVersion, dsv2Result, String.format("DSv2 getStartingVersion should match for %s", testDescription)); } @Test public void testMemoryProtection_initialSnapshotTooLarge(@TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_memory_protection_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); // At version 5, there will be at least 25 files. insertVersions( testTableName, /* numVersions= */ 10, /* rowsPerVersion= */ 5, /* includeEmptyVersion= */ false); String configKey = DeltaSQLConf.DELTA_STREAMING_INITIAL_SNAPSHOT_MAX_FILES().key(); spark.conf().set(configKey, "5"); try { Configuration hadoopConf = new Configuration(); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); long version = 5L; long fromIndex = DeltaSourceOffset.BASE_INDEX(); boolean isInitialSnapshot = true; RuntimeException exception = assertThrows( RuntimeException.class, () -> { try (CloseableIterator iter = stream.getFileChanges( version, fromIndex, isInitialSnapshot, Optional.empty())) { while (iter.hasNext()) { iter.next(); } } }); String errorMessage = exception.getMessage(); assertTrue(errorMessage.contains("DELTA_STREAMING_INITIAL_SNAPSHOT_TOO_LARGE")); assertTrue( errorMessage.contains("initial snapshot") || errorMessage.contains("Initial snapshot")); } finally { spark.conf().unset(configKey); } } /** * Simulates the Kernel integration path where {@code DefaultJsonHandler.hasNext()} wraps a {@link * ClosedByInterruptException} inside a {@link io.delta.kernel.exceptions.KernelEngineException}. * Verifies that {@code findClosedByInterruptCause} extracts the interrupt cause so {@code * latestOffset()} can re-throw it as {@link java.io.UncheckedIOException} for Spark's {@code * isInterruptedByStop}. */ @Test public void testFindClosedByInterruptCause() { // KernelEngineException wrapping ClosedByInterruptException -> present ClosedByInterruptException cbie = new ClosedByInterruptException(); assertThat( SparkMicroBatchStream.findClosedByInterruptCause( new io.delta.kernel.exceptions.KernelEngineException("readJsonFile", cbie))) .isPresent() .contains(cbie); // Plain RuntimeException -> empty assertThat(SparkMicroBatchStream.findClosedByInterruptCause(new RuntimeException("unrelated"))) .isEmpty(); // KernelEngineException wrapping a different IOException -> empty assertThat( SparkMicroBatchStream.findClosedByInterruptCause( new io.delta.kernel.exceptions.KernelEngineException( "readJsonFile", new java.io.FileNotFoundException("missing")))) .isEmpty(); } /** Regression test: closing the wrapped iterator must also close CommitActions. */ @Test public void testWrapIteratorWithCommitClose_closesCommitOnIteratorClose() throws Exception { AtomicBoolean commitClosed = new AtomicBoolean(false); AtomicBoolean innerClosed = new AtomicBoolean(false); CloseableIterator inner = new CloseableIterator() { private boolean consumed = false; @Override public boolean hasNext() { return !consumed; } @Override public IndexedFile next() { consumed = true; return new IndexedFile(/* version= */ 1L, /* index= */ 0L, /* addFile= */ null); } @Override public void close() { innerClosed.set(true); } }; try (CloseableIterator wrapped = SparkMicroBatchStream.wrapIteratorWithCommitClose( inner, newTrackingCommitActions(commitClosed))) { while (wrapped.hasNext()) { wrapped.next(); } } assertTrue(innerClosed.get(), "Inner iterator should be closed"); assertTrue(commitClosed.get(), "CommitActions should be closed"); } /** Closing the wrapped iterator before full consumption must still close CommitActions. */ @Test public void testWrapIteratorWithCommitClose_closesCommitOnEarlyClose() throws Exception { AtomicBoolean commitClosed = new AtomicBoolean(false); CloseableIterator inner = new CloseableIterator() { private int remaining = 10; @Override public boolean hasNext() { return remaining > 0; } @Override public IndexedFile next() { remaining--; return new IndexedFile(/* version= */ 1L, /* index= */ remaining, /* addFile= */ null); } @Override public void close() {} }; try (CloseableIterator wrapped = SparkMicroBatchStream.wrapIteratorWithCommitClose( inner, newTrackingCommitActions(commitClosed))) { assertTrue(wrapped.hasNext()); wrapped.next(); // Intentionally don't consume the rest } assertTrue(commitClosed.get(), "CommitActions should be closed on early termination"); } /** If inner.close() throws, CommitActions must still be closed. */ @Test public void testWrapIteratorWithCommitClose_closesCommitEvenWhenInnerCloseThrows() throws Exception { AtomicBoolean commitClosed = new AtomicBoolean(false); CloseableIterator inner = new CloseableIterator() { @Override public boolean hasNext() { return false; } @Override public IndexedFile next() { throw new java.util.NoSuchElementException(); } @Override public void close() { throw new RuntimeException("inner close failed"); } }; CloseableIterator wrapped = SparkMicroBatchStream.wrapIteratorWithCommitClose( inner, newTrackingCommitActions(commitClosed)); try { wrapped.close(); } catch (RuntimeException e) { // Expected — inner.close() threw } assertTrue(commitClosed.get(), "CommitActions should be closed even when inner.close() throws"); } /** Creates a CommitActions stub that sets {@code closedFlag} to true on close. */ private static CommitActions newTrackingCommitActions(AtomicBoolean closedFlag) { return new CommitActions() { @Override public long getVersion() { return 1L; } @Override public long getTimestamp() { return 0L; } @Override public CloseableIterator getActions() { throw new UnsupportedOperationException("not needed for this test"); } @Override public void close() { closedFlag.set(true); } }; } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanBuilderTest.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.read; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import io.delta.kernel.Snapshot; import io.delta.kernel.expressions.Column; import io.delta.kernel.expressions.Literal; import io.delta.kernel.expressions.Predicate; import io.delta.spark.internal.v2.DeltaV2TestBase; import io.delta.spark.internal.v2.snapshot.PathBasedSnapshotManager; import java.io.File; import java.lang.reflect.Field; import java.math.BigDecimal; import java.util.Arrays; import java.util.HashSet; import java.util.Optional; import java.util.Set; import org.apache.spark.sql.connector.read.Scan; import org.apache.spark.sql.connector.read.streaming.MicroBatchStream; import org.apache.spark.sql.sources.*; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; public class SparkScanBuilderTest extends DeltaV2TestBase { @Test public void testBuild_returnsScanWithExpectedSchema(@TempDir File tempDir) { String path = tempDir.getAbsolutePath(); String tableName = "scan_builder_test"; spark.sql( String.format( "CREATE TABLE %s (id INT, name STRING, dep_id INT) USING delta PARTITIONED BY (dep_id) LOCATION '%s'", tableName, path)); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); Snapshot snapshot = snapshotManager.loadLatestSnapshot(); StructType dataSchema = DataTypes.createStructType( new StructField[] { DataTypes.createStructField("id", DataTypes.IntegerType, true), DataTypes.createStructField("name", DataTypes.StringType, true) }); StructType partitionSchema = DataTypes.createStructType( new StructField[] {DataTypes.createStructField("dep_id", DataTypes.IntegerType, true)}); StructType tableSchema = DataTypes.createStructType( new StructField[] { DataTypes.createStructField("id", DataTypes.IntegerType, true), DataTypes.createStructField("name", DataTypes.StringType, true), DataTypes.createStructField("dep_id", DataTypes.IntegerType, true) }); SparkScanBuilder builder = new SparkScanBuilder( tableName, snapshot, snapshotManager, dataSchema, partitionSchema, tableSchema, Optional.empty(), CaseInsensitiveStringMap.empty()); StructType expectedSparkSchema = DataTypes.createStructType( new StructField[] { DataTypes.createStructField("id", DataTypes.IntegerType, true /*nullable*/), DataTypes.createStructField("dep_id", DataTypes.IntegerType, true) }); builder.pruneColumns(expectedSparkSchema); Scan scan = builder.build(); assertTrue(scan instanceof SparkScan); assertEquals(expectedSparkSchema, scan.readSchema()); } @Test public void testToMicroBatchStream_returnsSparkMicroBatchStream(@TempDir File tempDir) { String path = tempDir.getAbsolutePath(); String tableName = "microbatch_test"; spark.sql( String.format( "CREATE TABLE %s (id INT, name STRING, dep_id INT) USING delta PARTITIONED BY (dep_id) LOCATION '%s'", tableName, path)); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); Snapshot snapshot = snapshotManager.loadLatestSnapshot(); StructType dataSchema = DataTypes.createStructType( new StructField[] { DataTypes.createStructField("id", DataTypes.IntegerType, true), DataTypes.createStructField("name", DataTypes.StringType, true) }); StructType partitionSchema = DataTypes.createStructType( new StructField[] {DataTypes.createStructField("dep_id", DataTypes.IntegerType, true)}); StructType tableSchema = DataTypes.createStructType( new StructField[] { DataTypes.createStructField("id", DataTypes.IntegerType, true), DataTypes.createStructField("name", DataTypes.StringType, true), DataTypes.createStructField("dep_id", DataTypes.IntegerType, true) }); SparkScanBuilder builder = new SparkScanBuilder( tableName, snapshot, snapshotManager, dataSchema, partitionSchema, tableSchema, Optional.empty(), CaseInsensitiveStringMap.empty()); Scan scan = builder.build(); String checkpointLocation = "/tmp/checkpoint"; MicroBatchStream microBatchStream = scan.toMicroBatchStream(checkpointLocation); assertNotNull(microBatchStream, "MicroBatchStream should not be null"); assertTrue( microBatchStream instanceof SparkMicroBatchStream, "MicroBatchStream should be an instance of SparkMicroBatchStream"); } @Test public void testPushFilters_singleSupportedDataFilter(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); checkSupportsPushDownFilters( builder, // input filters new Filter[] {new EqualTo("id", 100)}, // expected post-scan filters new Filter[] {new EqualTo("id", 100)}, // expected pushed filters new Filter[] {new EqualTo("id", 100)}, // expected pushed kernel predicates new Predicate[] {new Predicate("=", new Column("id"), Literal.ofInt(100))}, // expected data filters new Filter[] {new EqualTo("id", 100)}, // expected kernelScanBuilder.predicate Optional.of(new Predicate("=", new Column("id"), Literal.ofInt(100)))); } @Test public void testPushFilters_singleUnsupportedDataFilter(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); checkSupportsPushDownFilters( builder, new Filter[] {new StringEndsWith("name", "test")}, // input filters new Filter[] { new StringEndsWith("name", "test") }, // expected post-scan filters (unsupported, stays for row-level eval) new Filter[] {}, // expected pushed filters (nothing pushed) new Predicate[] {}, // expected pushed kernel predicates new Filter[] {new StringEndsWith("name", "test")}, // expected data filters Optional.empty() // expected kernelScanBuilder.predicate ); } @Test public void testPushFilters_singleSupportedDataFilter_StringStartsWith(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); checkSupportsPushDownFilters( builder, new Filter[] {new StringStartsWith("name", "test")}, // input filters new Filter[] { new StringStartsWith("name", "test") }, // expected post-scan filters (data filter still needs row-level eval) new Filter[] {new StringStartsWith("name", "test")}, // expected pushed filters new Predicate[] { new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("test")) }, // expected pushed kernel predicates new Filter[] {new StringStartsWith("name", "test")}, // expected data filters Optional.of( new Predicate( "STARTS_WITH", new Column("name"), Literal.ofString("test"))) // expected kernelScanBuilder.predicate ); } @Test public void testPushFilters_multiSupportedDataFilters(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); checkSupportsPushDownFilters( builder, // input filters new Filter[] {new EqualTo("id", 100), new GreaterThan("id", 50)}, // expected post-scan filters new Filter[] {new EqualTo("id", 100), new GreaterThan("id", 50)}, // expected pushed filters new Filter[] {new EqualTo("id", 100), new GreaterThan("id", 50)}, // expected pushed kernel predicates new Predicate[] { new Predicate("=", new Column("id"), Literal.ofInt(100)), new Predicate(">", new Column("id"), Literal.ofInt(50)) }, // expected data filters new Filter[] {new EqualTo("id", 100), new GreaterThan("id", 50)}, // expected kernelScanBuilder.predicate Optional.of( new Predicate( "AND", Arrays.asList( new Predicate("=", new Column("id"), Literal.ofInt(100)), new Predicate(">", new Column("id"), Literal.ofInt(50)))))); } @Test public void testPushFilters_mixedSupportedAndUnsupportedDataFilters(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); checkSupportsPushDownFilters( builder, // input filters new Filter[] { new EqualTo("id", 100), // supported new StringEndsWith("name", "test") // unsupported }, // expected post-scan filters new Filter[] {new EqualTo("id", 100), new StringEndsWith("name", "test")}, // expected pushed filters (only the supported EqualTo is pushed) new Filter[] {new EqualTo("id", 100)}, // expected pushed kernel predicates new Predicate[] {new Predicate("=", new Column("id"), Literal.ofInt(100))}, // expected data filters new Filter[] {new EqualTo("id", 100), new StringEndsWith("name", "test")}, // expected kernelScanBuilder.predicate (only EqualTo was pushed to kernel) Optional.of(new Predicate("=", new Column("id"), Literal.ofInt(100)))); } @Test public void testPushFilters_singleSupportedPartitionFilter(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); checkSupportsPushDownFilters( builder, // input filters new Filter[] {new EqualTo("dep_id", 1)}, // expected post-scan filters new Filter[] {}, // expected pushed filters new Filter[] {new EqualTo("dep_id", 1)}, // expected pushed kernel predicates new Predicate[] {new Predicate("=", new Column("dep_id"), Literal.ofInt(1))}, // expected data filters new Filter[] {}, // expected kernelScanBuilder.predicate Optional.of(new Predicate("=", new Column("dep_id"), Literal.ofInt(1)))); } @Test public void testPushFilters_singleUnsupportedPartitionFilter(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); checkSupportsPushDownFilters( builder, new Filter[] {new StringStartsWith("dep_id", "1")}, // input filters new Filter[] {}, // expected post-scan filters (partition filter, fully pushed) new Filter[] {new StringStartsWith("dep_id", "1")}, // expected pushed filters new Predicate[] { new Predicate("STARTS_WITH", new Column("dep_id"), Literal.ofString("1")) }, // expected pushed kernel predicates new Filter[] {}, // expected data filters Optional.of( new Predicate( "STARTS_WITH", new Column("dep_id"), Literal.ofString("1"))) // expected kernelScanBuilder.predicate ); } @Test public void testPushFilters_multiSupportedPartitionFilters(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); checkSupportsPushDownFilters( builder, // input filters new Filter[] {new EqualTo("dep_id", 2), new GreaterThan("dep_id", 1)}, // expected post-scan filters new Filter[] {}, // expected pushed filters new Filter[] {new EqualTo("dep_id", 2), new GreaterThan("dep_id", 1)}, // expected pushed kernel predicates new Predicate[] { new Predicate("=", new Column("dep_id"), Literal.ofInt(2)), new Predicate(">", new Column("dep_id"), Literal.ofInt(1)) }, // expected data filters new Filter[] {}, // expected kernelScanBuilder.predicate Optional.of( new Predicate( "AND", Arrays.asList( new Predicate("=", new Column("dep_id"), Literal.ofInt(2)), new Predicate(">", new Column("dep_id"), Literal.ofInt(1)))))); } @Test public void testPushFilters_mixedSupportedAndUnsupportedPartitionFilters(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); checkSupportsPushDownFilters( builder, // input filters new Filter[] { new EqualTo("dep_id", 1), // supported new StringStartsWith("dep_id", "1") // unsupported }, // expected post-scan filters new Filter[] {}, // expected pushed filters new Filter[] {new EqualTo("dep_id", 1), new StringStartsWith("dep_id", "1")}, // expected pushed kernel predicates new Predicate[] { new Predicate("=", new Column("dep_id"), Literal.ofInt(1)), new Predicate("STARTS_WITH", new Column("dep_id"), Literal.ofString("1")) }, // expected data filters new Filter[] {}, // expected kernelScanBuilder.predicate Optional.of( new Predicate( "AND", new Predicate("=", new Column("dep_id"), Literal.ofInt(1)), new Predicate("STARTS_WITH", new Column("dep_id"), Literal.ofString("1"))))); } @Test public void testPushFilters_mixedFilters(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); checkSupportsPushDownFilters( builder, // input filters new Filter[] { new EqualTo("id", 100), // data filter, supported new StringStartsWith("name", "foo"), // data filter, unsupported new GreaterThan("dep_id", 1), // partition filter, supported new StringEndsWith("dep_id", "1") // partition filter, unsupported }, // expected post-scan filters new Filter[] { new EqualTo("id", 100), // data filter, supported new StringStartsWith("name", "foo"), // data filter, unsupported new StringEndsWith("dep_id", "1") // partition filter, unsupported }, // expected pushed filters new Filter[] { new EqualTo("id", 100), // data filter, supported new StringStartsWith("name", "foo"), // data filter, supported new GreaterThan("dep_id", 1) // partition filter, supported }, // expected pushed kernel predicates new Predicate[] { new Predicate("=", new Column("id"), Literal.ofInt(100)), new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("foo")), new Predicate(">", new Column("dep_id"), Literal.ofInt(1)) }, // expected data filters new Filter[] { new EqualTo("id", 100), // data filter, supported new StringStartsWith("name", "foo") // data filter, supported }, // expected kernelScanBuilder.predicate Optional.of( new Predicate( "AND", new Predicate( "AND", new Predicate("=", new Column("id"), Literal.ofInt(100)), new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("foo"))), new Predicate(">", new Column("dep_id"), Literal.ofInt(1))))); } @Test public void testPushFilters_ORFilters(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); checkSupportsPushDownFilters( builder, // input filters new Filter[] {new Or(new EqualTo("id", 100), new GreaterThan("id", 50))}, // expected post-scan filters new Filter[] {new Or(new EqualTo("id", 100), new GreaterThan("id", 50))}, // expected pushed filters new Filter[] {new Or(new EqualTo("id", 100), new GreaterThan("id", 50))}, // expected pushed kernel predicates new Predicate[] { new Predicate( "OR", Arrays.asList( new Predicate("=", new Column("id"), Literal.ofInt(100)), new Predicate(">", new Column("id"), Literal.ofInt(50)))) }, // expected data filters new Filter[] {new Or(new EqualTo("id", 100), new GreaterThan("id", 50))}, // expected kernelScanBuilder.predicate Optional.of( new Predicate( "OR", Arrays.asList( new Predicate("=", new Column("id"), Literal.ofInt(100)), new Predicate(">", new Column("id"), Literal.ofInt(50)))))); } @Test public void testPushFilters_ORSupportedAndUnsupportedDataFilters(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); // OR(supported, unsupported) cannot be partially pushed: if one branch is unsupported, // the whole OR must remain for post-scan evaluation and nothing is pushed to the kernel. checkSupportsPushDownFilters( builder, // input filters new Filter[] {new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo"))}, // expected post-scan filters (whole OR stays, since one branch is unsupported) new Filter[] {new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo"))}, // expected pushed filters (nothing pushed) new Filter[] {}, // expected pushed kernel predicates new Predicate[] {}, // expected data filters new Filter[] {new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo"))}, // expected kernelScanBuilder.predicate Optional.empty()); } @Test public void testPushFilters_ORSupportedDataAndPartitionFilters(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); checkSupportsPushDownFilters( builder, // input filters new Filter[] { new Or(new EqualTo("id", 100), new GreaterThan("dep_id", 1)), }, // expected post-scan filters new Filter[] {new Or(new EqualTo("id", 100), new GreaterThan("dep_id", 1))}, // expected pushed filters new Filter[] {new Or(new EqualTo("id", 100), new GreaterThan("dep_id", 1))}, // expected pushed kernel predicates new Predicate[] { new Predicate( "OR", Arrays.asList( new Predicate("=", new Column("id"), Literal.ofInt(100)), new Predicate(">", new Column("dep_id"), Literal.ofInt(1)))) }, // expected data filters new Filter[] {new Or(new EqualTo("id", 100), new GreaterThan("dep_id", 1))}, // expected kernelScanBuilder.predicate Optional.of( new Predicate( "OR", Arrays.asList( new Predicate("=", new Column("id"), Literal.ofInt(100)), new Predicate(">", new Column("dep_id"), Literal.ofInt(1)))))); } /* * (partitionFilterA AND partitionFilterB) OR partitionFilterC * where A = EqualTo("dep_id", 1), B = StringStartsWith("dep_id", "1"), C = GreaterThan("dep_id", 2) * All three are fully supported partition filters, so the whole expression is pushed down. * * Expected post-scan filters: none (all partition filters, fully pushed) * Expected pushed filters: (A AND B) OR C * Expected pushed kernel predicates: (predicateA AND predicateB) OR predicateC * Expected data filters: none * Expected kernelScanBuilder.predicate: (predicateA AND predicateB) OR predicateC */ @Test public void testPushFilters_mixedORandAND(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); checkSupportsPushDownFilters( builder, // input filters new Filter[] { new Or( new And(new EqualTo("dep_id", 1), new StringStartsWith("dep_id", "1")), new GreaterThan("dep_id", 2)) }, // expected post-scan filters new Filter[] {}, // expected pushed filters new Filter[] { new Or( new And(new EqualTo("dep_id", 1), new StringStartsWith("dep_id", "1")), new GreaterThan("dep_id", 2)) }, // expected pushed kernel predicates new Predicate[] { new Predicate( "OR", Arrays.asList( new Predicate( "AND", Arrays.asList( new Predicate("=", new Column("dep_id"), Literal.ofInt(1)), new Predicate( "STARTS_WITH", new Column("dep_id"), Literal.ofString("1")))), new Predicate(">", new Column("dep_id"), Literal.ofInt(2)))) }, // expected data filters new Filter[] {}, // expected kernelScanBuilder.predicate Optional.of( new Predicate( "OR", Arrays.asList( new Predicate( "AND", Arrays.asList( new Predicate("=", new Column("dep_id"), Literal.ofInt(1)), new Predicate( "STARTS_WITH", new Column("dep_id"), Literal.ofString("1")))), new Predicate(">", new Column("dep_id"), Literal.ofInt(2)))))); } @Test public void testPushFilters_NOTFilters(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); checkSupportsPushDownFilters( builder, // input filters new Filter[] {new Not(new EqualTo("id", 100))}, // expected post-scan filters new Filter[] {new Not(new EqualTo("id", 100))}, // expected pushed filters new Filter[] {new Not(new EqualTo("id", 100))}, // expected pushed kernel predicates new Predicate[] { new Predicate("NOT", new Predicate("=", new Column("id"), Literal.ofInt(100))) }, // expected data filters new Filter[] {new Not(new EqualTo("id", 100))}, // expected kernelScanBuilder.predicate Optional.of( new Predicate("NOT", new Predicate("=", new Column("id"), Literal.ofInt(100))))); } @Test public void testPushFilters_NOTSupportedDataANDSupportedPartitionFilters(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); checkSupportsPushDownFilters( builder, // input filters new Filter[] { new Not(new And(new EqualTo("id", 100), new GreaterThan("dep_id", 1))), }, // expected post-scan filters new Filter[] { new Not(new And(new EqualTo("id", 100), new GreaterThan("dep_id", 1))), }, // expected pushed filters new Filter[] { new Not(new And(new EqualTo("id", 100), new GreaterThan("dep_id", 1))), }, // expected pushed kernel predicates new Predicate[] { new Predicate( "NOT", new Predicate( "AND", Arrays.asList( new Predicate("=", new Column("id"), Literal.ofInt(100)), new Predicate(">", new Column("dep_id"), Literal.ofInt(1))))) }, // expected data filters new Filter[] { new Not(new And(new EqualTo("id", 100), new GreaterThan("dep_id", 1))), }, // expected kernelScanBuilder.predicate Optional.of( new Predicate( "NOT", new Predicate( "AND", Arrays.asList( new Predicate("=", new Column("id"), Literal.ofInt(100)), new Predicate(">", new Column("dep_id"), Literal.ofInt(1))))))); } @Test public void testPushFilters_NOTSupportedDataANDUnsupportedDataFilters(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); checkSupportsPushDownFilters( builder, // input filters new Filter[] {new Not(new And(new EqualTo("id", 100), new StringEndsWith("name", "bar")))}, // expected post-scan filters new Filter[] {new Not(new And(new EqualTo("id", 100), new StringEndsWith("name", "bar")))}, // expected pushed filters new Filter[] {}, // expected pushed kernel predicates new Predicate[] {}, // expected data filters new Filter[] {new Not(new And(new EqualTo("id", 100), new StringEndsWith("name", "bar")))}, // expected kernelScanBuilder.predicate Optional.empty()); } @Test public void testPushFilters_NOTSupportedDataORSupportedPartitionFilters(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); checkSupportsPushDownFilters( builder, // input filters new Filter[] { new Not(new Or(new EqualTo("id", 100), new GreaterThan("dep_id", 1))), }, // expected post-scan filters new Filter[] {new Not(new Or(new EqualTo("id", 100), new GreaterThan("dep_id", 1)))}, // expected pushed filters new Filter[] {new Not(new Or(new EqualTo("id", 100), new GreaterThan("dep_id", 1)))}, // expected pushed kernel predicates new Predicate[] { new Predicate( "NOT", new Predicate( "OR", Arrays.asList( new Predicate("=", new Column("id"), Literal.ofInt(100)), new Predicate(">", new Column("dep_id"), Literal.ofInt(1))))) }, // expected data filters new Filter[] {new Not(new Or(new EqualTo("id", 100), new GreaterThan("dep_id", 1)))}, // expected kernelScanBuilder.predicate Optional.of( new Predicate( "NOT", new Predicate( "OR", Arrays.asList( new Predicate("=", new Column("id"), Literal.ofInt(100)), new Predicate(">", new Column("dep_id"), Literal.ofInt(1))))))); } @Test public void testPushFilters_NOTSupportedDataORUnsupportedDataFilters(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); // NOT(OR(supported, unsupported)): the OR branch contains an unsupported filter // (StringEndsWith), // so the whole NOT(OR(...)) cannot be pushed to the kernel. checkSupportsPushDownFilters( builder, // input filters new Filter[] { new Not(new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo"))), }, // expected post-scan filters (whole NOT(OR) stays, unsupported branch blocks pushdown) new Filter[] {new Not(new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo")))}, // expected pushed filters (nothing pushed) new Filter[] {}, // expected pushed kernel predicates new Predicate[] {}, // expected data filters new Filter[] {new Not(new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo")))}, // expected kernelScanBuilder.predicate Optional.empty()); } private void checkSupportsPushDownFilters( SparkScanBuilder builder, Filter[] inputFilters, Filter[] expectedPostScanFilters, Filter[] expectedPushedFilters, Predicate[] expectedPushedKernelPredicates, Filter[] expectedDataFilters, Optional expectedKernelScanBuilderPredicate) throws Exception { Filter[] postScanFilters = builder.pushFilters(inputFilters); assertEquals( new HashSet<>(Arrays.asList(expectedPostScanFilters)), new HashSet<>(Arrays.asList(postScanFilters))); assertEquals( new HashSet<>(Arrays.asList(expectedPushedFilters)), new HashSet<>(Arrays.asList(builder.pushedFilters()))); Predicate[] pushedPredicates = getPushedKernelPredicates(builder); assertEquals( new HashSet<>(Arrays.asList(expectedPushedKernelPredicates)), new HashSet<>(Arrays.asList(pushedPredicates))); Filter[] dataFilters = getDataFilters(builder); assertEquals( new HashSet<>(Arrays.asList(expectedDataFilters)), new HashSet<>(Arrays.asList(dataFilters))); Optional predicateOpt = getKernelScanBuilderPredicate(builder); assertEquals(expectedKernelScanBuilderPredicate, predicateOpt); } private SparkScanBuilder createTestScanBuilder(File tempDir) { StructType dataSchema = DataTypes.createStructType( new StructField[] { DataTypes.createStructField("id", DataTypes.IntegerType, true), DataTypes.createStructField("name", DataTypes.StringType, true) }); StructType partitionSchema = DataTypes.createStructType( new StructField[] {DataTypes.createStructField("dep_id", DataTypes.IntegerType, true)}); StructType tableSchema = DataTypes.createStructType( new StructField[] { DataTypes.createStructField("id", DataTypes.IntegerType, true), DataTypes.createStructField("name", DataTypes.StringType, true), DataTypes.createStructField("dep_id", DataTypes.IntegerType, true) }); return createScanBuilder(tempDir, dataSchema, partitionSchema, tableSchema); } /** * Integration test: decimal widening end-to-end through pushFilters() → classifyFilter() → * convertComparisonLiteral(). Verifies that a decimal literal Decimal(5,2) is widened to match * column type Decimal(7,2) when pushed through the full filter pushdown path. */ @Test public void testPushFilters_decimalWideningEndToEnd(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createDecimalTestScanBuilder(tempDir); // price = 100.00 where literal is Decimal(5,2) and column is Decimal(7,2) Filter[] sparkFilter = new Filter[] {new EqualTo("price", new BigDecimal("100.00"))}; Predicate kernelPredicate = new Predicate("=", new Column("price"), Literal.ofDecimal(new BigDecimal("100.00"), 7, 2)); checkSupportsPushDownFilters( builder, sparkFilter, // input filters sparkFilter, // expected post-scan filters (data filter, stays for row-level eval) sparkFilter, // expected pushed filters new Predicate[] {kernelPredicate}, // expected pushed kernel predicates (widened) sparkFilter, // expected data filters Optional.of(kernelPredicate)); // expected kernelScanBuilder.predicate } /** * Integration test: decimal literal with scale exceeding column scale is rejected during * pushFilters(). The filter AND(price > 99.999, price < 200.00) should partially push only the * right side because 99.999 has scale=3 exceeding column's Decimal(7,2) scale=2. */ @Test public void testPushFilters_decimalRejectionPartialPushDown(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createDecimalTestScanBuilder(tempDir); // AND(price > 99.999, price < 200.00): left side has scale=3 exceeding column's scale=2 Filter[] sparkFilter = new Filter[] { new And( new GreaterThan("price", new BigDecimal("99.999")), new LessThan("price", new BigDecimal("200.00"))) }; // Only the right side (price < 200.00) is pushed as a kernel predicate Predicate kernelPredicate = new Predicate("<", new Column("price"), Literal.ofDecimal(new BigDecimal("200.00"), 7, 2)); checkSupportsPushDownFilters( builder, sparkFilter, // input filters sparkFilter, // expected post-scan filters (partial conversion, stays for row-level eval) new Filter[] {}, // expected pushed filters (partial: Spark filter not added) new Predicate[] {kernelPredicate}, // expected pushed kernel predicates sparkFilter, // expected data filters Optional.of(kernelPredicate)); // expected kernelScanBuilder.predicate } private SparkScanBuilder createDecimalTestScanBuilder(File tempDir) { StructType dataSchema = DataTypes.createStructType( new StructField[] { DataTypes.createStructField("price", DataTypes.createDecimalType(7, 2), true), DataTypes.createStructField("quantity", DataTypes.IntegerType, true) }); StructType partitionSchema = DataTypes.createStructType( new StructField[] {DataTypes.createStructField("dep_id", DataTypes.IntegerType, true)}); StructType tableSchema = DataTypes.createStructType( new StructField[] { DataTypes.createStructField("price", DataTypes.createDecimalType(7, 2), true), DataTypes.createStructField("quantity", DataTypes.IntegerType, true), DataTypes.createStructField("dep_id", DataTypes.IntegerType, true) }); return createScanBuilder(tempDir, dataSchema, partitionSchema, tableSchema); } /** * Shared helper for creating a SparkScanBuilder with the given schemas. Both * createTestScanBuilder and createDecimalTestScanBuilder delegate to this method to avoid * duplicating snapshot loading and builder instantiation logic. */ private SparkScanBuilder createScanBuilder( File tempDir, StructType dataSchema, StructType partitionSchema, StructType tableSchema) { String path = tempDir.getAbsolutePath(); String tableName = String.format("test_%d", System.currentTimeMillis()); // Build CREATE TABLE SQL from the tableSchema and partitionSchema StringBuilder columns = new StringBuilder(); Set partitionCols = new HashSet<>(); for (StructField f : partitionSchema.fields()) { partitionCols.add(f.name()); } for (StructField f : tableSchema.fields()) { if (columns.length() > 0) columns.append(", "); columns.append(f.name()).append(" ").append(f.dataType().sql()); } String partitionColNames = String.join(", ", partitionCols); spark.sql( String.format( "CREATE OR REPLACE TABLE %s (%s) USING delta PARTITIONED BY (%s) LOCATION '%s'", tableName, columns, partitionColNames, path)); PathBasedSnapshotManager snapshotManager = new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); Snapshot snapshot = snapshotManager.loadLatestSnapshot(); return new SparkScanBuilder( tableName, snapshot, snapshotManager, dataSchema, partitionSchema, tableSchema, Optional.empty(), CaseInsensitiveStringMap.empty()); } private Predicate[] getPushedKernelPredicates(SparkScanBuilder builder) throws Exception { // TODO: replace reflection with other testing manners, possibly Mockito ArgumentCaptor Field field = SparkScanBuilder.class.getDeclaredField("pushedKernelPredicates"); field.setAccessible(true); return (Predicate[]) field.get(builder); } private Filter[] getDataFilters(SparkScanBuilder builder) throws Exception { // TODO: replace reflection with other testing manners, possibly Mockito ArgumentCaptor Field field = SparkScanBuilder.class.getDeclaredField("dataFilters"); field.setAccessible(true); return (Filter[]) field.get(builder); } private Optional getKernelScanBuilderPredicate(SparkScanBuilder builder) throws Exception { // TODO: replace reflection with other testing manners, possibly Mockito ArgumentCaptor Field field = SparkScanBuilder.class.getDeclaredField("kernelScanBuilder"); field.setAccessible(true); Object kernelScanBuilder = field.get(builder); Field predicateField = kernelScanBuilder.getClass().getDeclaredField("predicate"); predicateField.setAccessible(true); Object raw = predicateField.get(kernelScanBuilder); if (raw == null) { return Optional.empty(); } Optional opt = (Optional) raw; return opt.map(Predicate.class::cast); } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java ================================================ package io.delta.spark.internal.v2.read; import static org.junit.jupiter.api.Assertions.*; import io.delta.spark.internal.v2.DeltaV2TestBase; import io.delta.spark.internal.v2.catalog.SparkTable; import io.delta.spark.internal.v2.utils.ScalaUtils; import java.io.File; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.TableIdentifier; import org.apache.spark.sql.catalyst.catalog.CatalogColumnStat; import org.apache.spark.sql.catalyst.catalog.CatalogStatistics; import org.apache.spark.sql.catalyst.catalog.CatalogTable; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.expressions.Expression; import org.apache.spark.sql.connector.expressions.FieldReference; import org.apache.spark.sql.connector.expressions.LiteralValue; import org.apache.spark.sql.connector.expressions.NamedReference; import org.apache.spark.sql.connector.expressions.filter.Predicate; import org.apache.spark.sql.connector.read.Batch; import org.apache.spark.sql.connector.read.HasPartitionKey; import org.apache.spark.sql.connector.read.InputPartition; import org.apache.spark.sql.connector.read.Scan; import org.apache.spark.sql.connector.read.ScanBuilder; import org.apache.spark.sql.connector.read.Statistics; import org.apache.spark.sql.connector.read.colstats.ColumnStatistics; import org.apache.spark.sql.connector.read.partitioning.KeyGroupedPartitioning; import org.apache.spark.sql.connector.read.partitioning.Partitioning; import org.apache.spark.sql.connector.read.partitioning.UnknownPartitioning; import org.apache.spark.sql.delta.DeltaOptions; import org.apache.spark.sql.execution.datasources.FilePartition; import org.apache.spark.sql.execution.datasources.PartitionedFile; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; public class SparkScanTest extends DeltaV2TestBase { private static String tablePath; private static final String tableName = "deltatbl_partitioned"; @BeforeAll public static void setupPartitionedTable(@TempDir File tempDir) { createPartitionedTable(tableName, tempDir.getAbsolutePath()); tablePath = tempDir.getAbsolutePath(); } private final CaseInsensitiveStringMap options = new CaseInsensitiveStringMap(new java.util.HashMap<>()); private final SparkTable table = new SparkTable( Identifier.of(new String[] {"spark_catalog", "default"}, tableName), tablePath, options); protected static final Predicate cityPredicate = new Predicate( "=", new Expression[] { FieldReference.apply("city"), LiteralValue.apply("hz", DataTypes.StringType) }); protected static final Predicate datePredicate = new Predicate( "=", new Expression[] { FieldReference.apply("date"), LiteralValue.apply("20180520", DataTypes.StringType) }); protected static final Predicate partPredicate = new Predicate( ">", new Expression[] { FieldReference.apply("part"), LiteralValue.apply(1, DataTypes.IntegerType) }); protected static final Predicate dataPredicate = new Predicate( ">", new Expression[] { FieldReference.apply("cnt"), LiteralValue.apply(10, DataTypes.IntegerType) }); protected static final Predicate negativeCityPredicate = new Predicate( "=", new Expression[] { FieldReference.apply("city"), LiteralValue.apply("zz", DataTypes.StringType) }); protected static final Predicate interColPredicate = new Predicate( "!=", new Expression[] {FieldReference.apply("city"), FieldReference.apply("date")}); protected static final Predicate negativeInterColPredicate = new Predicate( "=", new Expression[] {FieldReference.apply("city"), FieldReference.apply("date")}); // a full set of cities in the golden table, repsents all partitions protected static final List allCities = Arrays.asList("city=hz", "city=sh", "city=bj", "city=sz"); // =============================================================================================== // Tests for columnarSupportMode // =============================================================================================== @Test public void testColumnarSupportModeReturnsSupported() { // Table schema uses simple types (INT, STRING) which are batch-read-compatible SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan = (SparkScan) builder.build(); assertEquals( Scan.ColumnarSupportMode.SUPPORTED, scan.columnarSupportMode(), "columnarSupportMode should return SUPPORTED for batch-compatible schema"); } @Test public void testColumnarSupportModeDoesNotTriggerPlanning() throws Exception { // Calling columnarSupportMode() must NOT trigger file planning (the whole point of the // override is to avoid the early planInputPartitions() call that PARTITION_DEFINED causes). SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan = (SparkScan) builder.build(); // Call columnarSupportMode before any planning scan.columnarSupportMode(); // Verify the scan has not been planned yet Field plannedField = SparkScan.class.getDeclaredField("planned"); plannedField.setAccessible(true); assertFalse( (boolean) plannedField.get(scan), "columnarSupportMode() should not trigger file planning"); } @Test public void testColumnarSupportModeWithUnsupportedSchema(@TempDir File tempDir) throws Exception { // Create a table with a MAP column and disable nested column vectorized reading // to ensure the schema is not batch-read-compatible String path = tempDir.getAbsolutePath(); String mapTableName = "columnar_map_table"; withTable( new String[] {mapTableName}, () -> { spark.sql( String.format( "CREATE TABLE %s (id INT, tags MAP) USING delta LOCATION '%s'", mapTableName, path)); spark.sql(String.format("INSERT INTO %s VALUES (1, map('k', 'v'))", mapTableName)); withSQLConf( "spark.sql.parquet.enableNestedColumnVectorizedReader", "false", () -> { SparkTable mapTable = new SparkTable( Identifier.of(new String[] {"spark_catalog", "default"}, mapTableName), path, options); SparkScanBuilder builder = (SparkScanBuilder) mapTable.newScanBuilder(options); SparkScan scan = (SparkScan) builder.build(); assertEquals( Scan.ColumnarSupportMode.UNSUPPORTED, scan.columnarSupportMode(), "columnarSupportMode should return UNSUPPORTED for schema with MAP type" + " when nested column vectorized reader is disabled"); }); }); } @Test public void testColumnarSupportModeWithVectorizedReaderDisabled() throws Exception { // When spark.sql.parquet.enableVectorizedReader is false, columnarSupportMode should // return UNSUPPORTED even for batch-compatible schemas. withSQLConf( "spark.sql.parquet.enableVectorizedReader", "false", () -> { SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan = (SparkScan) builder.build(); assertEquals( Scan.ColumnarSupportMode.UNSUPPORTED, scan.columnarSupportMode(), "columnarSupportMode should return UNSUPPORTED when vectorized reader is disabled"); }); } @Test public void testColumnarSupportModeWithDeletionVectors(@TempDir File tempDir) throws Exception { // For a DV-enabled table with a batch-compatible schema, columnarSupportMode should still // return SUPPORTED because the DV internal column (__delta_internal_is_row_deleted, ByteType) // is also batch-compatible. This verifies consistency with PartitionUtils reader factory. String dvPath = tempDir.getAbsolutePath(); String dvTableName = "columnar_dv_table"; withTable( new String[] {dvTableName}, () -> { spark.sql( String.format( "CREATE TABLE %s (id INT, value STRING) USING delta " + "TBLPROPERTIES ('delta.enableDeletionVectors' = 'true') " + "LOCATION '%s'", dvTableName, dvPath)); spark.sql(String.format("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", dvTableName)); SparkTable dvTable = new SparkTable( Identifier.of(new String[] {"spark_catalog", "default"}, dvTableName), dvPath, options); SparkScanBuilder builder = (SparkScanBuilder) dvTable.newScanBuilder(options); SparkScan scan = (SparkScan) builder.build(); // DV-enabled table with simple types should still return SUPPORTED because the // DV column (ByteType) is batch-compatible assertEquals( Scan.ColumnarSupportMode.SUPPORTED, scan.columnarSupportMode(), "columnarSupportMode should return SUPPORTED for DV-enabled table with" + " batch-compatible schema"); }); } // =============================================================================================== // Tests for getDataSchema, getPartitionSchema, getReadDataSchema, getOptions, getConfiguration // =============================================================================================== @Test public void testGetDataSchemaPartitionSchemaReadDataSchemaOptionsConfiguration() { SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan = (SparkScan) builder.build(); // Table schema: (part INT, date STRING, city STRING, name STRING, cnt INT) // Partition columns: (date STRING, city STRING, part INT) // Data columns: (name STRING, cnt INT) StructType dataSchema = scan.getDataSchema(); StructType partitionSchema = scan.getPartitionSchema(); StructType readDataSchema = scan.getReadDataSchema(); CaseInsensitiveStringMap scanOptions = scan.getOptions(); Configuration configuration = scan.getConfiguration(); assertEquals(2, dataSchema.fields().length, "dataSchema should have 2 fields (name, cnt)"); assertNotNull(dataSchema.fieldNames()); assertTrue( Arrays.asList(dataSchema.fieldNames()).containsAll(Arrays.asList("name", "cnt")), "dataSchema should contain name and cnt"); assertEquals( 3, partitionSchema.fields().length, "partitionSchema should have 3 fields (date, city, part)"); assertTrue( Arrays.asList(partitionSchema.fieldNames()) .containsAll(Arrays.asList("date", "city", "part")), "partitionSchema should contain date, city, part"); assertEquals( dataSchema, readDataSchema, "readDataSchema should equal dataSchema without column pruning"); assertNotNull(scanOptions, "options should not be null"); assertEquals(options, scanOptions, "options should match the scan options"); assertNotNull(configuration, "configuration should not be null"); // Verify configuration matches expected: built from same options via Spark session Configuration expectedConf = spark.sessionState().newHadoopConfWithOptions(ScalaUtils.toScalaMap(options)); assertEquals( expectedConf.get("fs.defaultFS"), configuration.get("fs.defaultFS"), "fs.defaultFS should match expected"); assertEquals( expectedConf.get("fs.default.name"), configuration.get("fs.default.name"), "fs.default.name should match expected"); } @Test public void testGetTablePathReturnsTablePath() { SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan = (SparkScan) builder.build(); String retrievedPath = scan.getTablePath(); assertNotNull(retrievedPath, "getTablePath should not return null"); // getTablePath returns file URI with trailing slash; tablePath is from tempDir String expectedUri = new File(tablePath).toURI().toString(); String expectedPath = expectedUri.endsWith("/") ? expectedUri : expectedUri + "/"; assertEquals( expectedPath, retrievedPath, "getTablePath should return path matching table location (with trailing slash)"); } @Test public void testGetConfigurationWithHadoopOptions() { // Pass Hadoop options and verify they appear in the returned Configuration Map optionsWithHadoop = new HashMap<>(); optionsWithHadoop.put("fs.file.impl.disable.cache", "true"); optionsWithHadoop.put("dfs.replication", "2"); CaseInsensitiveStringMap optionsWithHadoopMap = new CaseInsensitiveStringMap(optionsWithHadoop); SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(optionsWithHadoopMap); SparkScan scan = (SparkScan) builder.build(); Configuration configuration = scan.getConfiguration(); assertEquals( "true", configuration.get("fs.file.impl.disable.cache"), "Hadoop option fs.file.impl.disable.cache should flow through to Configuration"); assertEquals( "2", configuration.get("dfs.replication"), "Hadoop option dfs.replication should flow through to Configuration"); } @Test public void testGetReadDataSchemaWithColumnPruning() { SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); StructType prunedSchema = new StructType() .add("name", DataTypes.StringType) .add("date", DataTypes.StringType) .add("city", DataTypes.StringType) .add("part", DataTypes.IntegerType); builder.pruneColumns(prunedSchema); SparkScan scan = (SparkScan) builder.build(); StructType dataSchema = scan.getDataSchema(); StructType readDataSchema = scan.getReadDataSchema(); assertEquals(2, dataSchema.fields().length, "dataSchema should still have 2 fields"); assertEquals( 1, readDataSchema.fields().length, "readDataSchema should have 1 field (name) after pruning cnt"); assertEquals("name", readDataSchema.fields()[0].name()); } @Test public void testDPP_singleFilter() throws Exception { checkSupportsRuntimeFilters( table, options, new Predicate[] {cityPredicate}, Arrays.asList("city=hz")); checkSupportsRuntimeFilters( table, options, new Predicate[] {datePredicate}, Arrays.asList("date=20180520")); checkSupportsRuntimeFilters( table, options, new Predicate[] {partPredicate}, Arrays.asList("part=2")); } @Test public void testDPP_multiFilters() throws Exception { checkSupportsRuntimeFilters( table, options, new Predicate[] {cityPredicate, datePredicate}, Arrays.asList("date=20180520/city=hz")); } @Test public void testDPP_ANDFilters() throws Exception { Predicate andPredicate = new Predicate("AND", new Expression[] {cityPredicate, datePredicate}); checkSupportsRuntimeFilters( table, options, new Predicate[] {andPredicate}, Arrays.asList("date=20180520/city=hz")); } @Test public void testDPP_ORFilters() throws Exception { Predicate orPredicate = new Predicate("OR", new Expression[] {cityPredicate, datePredicate}); checkSupportsRuntimeFilters( table, options, new Predicate[] {orPredicate}, Arrays.asList("city=hz", "date=20180520")); } @Test public void testDPP_NOTFilter() throws Exception { Predicate notPredicate = new Predicate("NOT", new Expression[] {cityPredicate}); checkSupportsRuntimeFilters( table, options, new Predicate[] {notPredicate}, Arrays.asList("city=sh", "city=bj", "city=sz")); } @Test public void testDPP_INFilter() throws Exception { Predicate inPredicate = new Predicate( "IN", new Expression[] { FieldReference.apply("city"), LiteralValue.apply("hz", DataTypes.StringType), LiteralValue.apply("sh", DataTypes.StringType) }); checkSupportsRuntimeFilters( table, options, new Predicate[] {inPredicate}, Arrays.asList("city=hz", "city=sh")); } @Test public void testDPP_negativeFilter() throws Exception { checkSupportsRuntimeFilters( table, options, new Predicate[] {negativeCityPredicate}, Arrays.asList()); } @Test public void testDPP_ANDNegativeFilter() throws Exception { Predicate andPredicate = new Predicate("AND", new Expression[] {cityPredicate, negativeCityPredicate}); checkSupportsRuntimeFilters(table, options, new Predicate[] {andPredicate}, Arrays.asList()); } @Test public void testDPP_ORNegativeFilter() throws Exception { Predicate orPredicate = new Predicate("OR", new Expression[] {cityPredicate, negativeCityPredicate}); checkSupportsRuntimeFilters( table, options, new Predicate[] {orPredicate}, Arrays.asList("city=hz")); } @Test public void testDPP_nonPartitionColumnFilter() throws Exception { checkSupportsRuntimeFilters( table, options, new Predicate[] {cityPredicate, dataPredicate}, Arrays.asList("city=hz")); } @Test public void testDPP_nonPartitionColumnFilterOnly() throws Exception { checkSupportsRuntimeFilters(table, options, new Predicate[] {dataPredicate}, allCities); } @Test public void testDPP_ANDDataPredicate() throws Exception { Predicate andPredicate = new Predicate("AND", new Expression[] {cityPredicate, dataPredicate}); checkSupportsRuntimeFilters(table, options, new Predicate[] {andPredicate}, allCities); } @Test public void testDPP_ORDataPredicate() throws Exception { Predicate orPredicate = new Predicate("OR", new Expression[] {cityPredicate, dataPredicate}); checkSupportsRuntimeFilters(table, options, new Predicate[] {orPredicate}, allCities); } @Test public void testDPP_interColumnFilter() throws Exception { checkSupportsRuntimeFilters(table, options, new Predicate[] {interColPredicate}, allCities); } @Test public void testDPP_negativeInterColumnFilter() throws Exception { checkSupportsRuntimeFilters( table, options, new Predicate[] {negativeInterColPredicate}, Arrays.asList()); } @Test public void testDPP_integerFilter() throws Exception { checkSupportsRuntimeFilters( table, options, new Predicate[] {partPredicate}, Arrays.asList("part=2")); } protected static void checkSupportsRuntimeFilters( SparkTable table, CaseInsensitiveStringMap scanOptions, org.apache.spark.sql.connector.expressions.filter.Predicate[] runtimeFilters, List remainingPartitionValueAfterDpp) throws Exception { ScanBuilder newBuilder = table.newScanBuilder(scanOptions); SparkScanBuilder builder = (SparkScanBuilder) newBuilder; Scan scan = builder.build(); SparkScan sparkScan = (SparkScan) scan; List beforeDppFiles = getPartitionedFiles(sparkScan); // make a copy for comparison after DPP beforeDppFiles = new ArrayList<>(beforeDppFiles); long beforeDppTotalBytes = getTotalBytes(sparkScan); long beforeDppEstimatedSize = getEstimatedSizeInBytes(sparkScan); assert (beforeDppFiles.size() == 5); // Without column pruning, estimatedSizeInBytes should equal totalBytes assertEquals(beforeDppTotalBytes, beforeDppEstimatedSize); sparkScan.filter(runtimeFilters); List afterDppFiles = getPartitionedFiles(sparkScan); long afterDppTotalBytes = getTotalBytes(sparkScan); long afterDppEstimatedSize = getEstimatedSizeInBytes(sparkScan); assert (beforeDppFiles.containsAll(afterDppFiles)); assert (beforeDppTotalBytes >= afterDppTotalBytes); List expectedPartitionFilesAfterDpp = new ArrayList<>(); long expectedTotalBytesAfterDpp = 0; for (PartitionedFile pf : beforeDppFiles) { for (String partitionValue : remainingPartitionValueAfterDpp) { if (pf.filePath().toString().contains(partitionValue)) { expectedPartitionFilesAfterDpp.add(pf); expectedTotalBytesAfterDpp += pf.fileSize(); break; } } } assertEquals(expectedPartitionFilesAfterDpp.size(), afterDppFiles.size()); assertEquals(new HashSet<>(expectedPartitionFilesAfterDpp), new HashSet<>(afterDppFiles)); assertEquals(expectedTotalBytesAfterDpp, afterDppTotalBytes); // Without column pruning, estimatedSizeInBytes should equal totalBytes after filtering too assertEquals(afterDppTotalBytes, afterDppEstimatedSize); } private static List getPartitionedFiles(SparkScan scan) throws Exception { scan.estimateStatistics(); // ensurePlanned Field field = SparkScan.class.getDeclaredField("partitionedFiles"); field.setAccessible(true); return (List) field.get(scan); } private static long getTotalBytes(SparkScan scan) throws Exception { scan.estimateStatistics(); // ensurePlanned Field field = SparkScan.class.getDeclaredField("totalBytes"); field.setAccessible(true); return (long) field.get(scan); } private static long getEstimatedSizeInBytes(SparkScan scan) throws Exception { scan.estimateStatistics(); // ensurePlanned Field field = SparkScan.class.getDeclaredField("estimatedSizeInBytes"); field.setAccessible(true); return (long) field.get(scan); } // ================================================================================================ // Tests for streaming options validation // ================================================================================================ @Test public void testValidateStreamingOptions_SupportedOptions() { // Test with supported options (case insensitive) and custom user options Map javaOptions = new HashMap<>(); javaOptions.put("startingVersion", "0"); javaOptions.put("MaxFilesPerTrigger", "100"); javaOptions.put("MAXBYTESPERTRIGGER", "1g"); javaOptions.put("myCustomOption", "value"); scala.collection.immutable.Map supportedOptions = ScalaUtils.toScalaMap(javaOptions); DeltaOptions deltaOptions = new DeltaOptions(supportedOptions, spark.sessionState().conf()); // Verify DeltaOptions can recognize the options (case insensitive) assertEquals(true, deltaOptions.maxFilesPerTrigger().isDefined()); assertEquals(100, deltaOptions.maxFilesPerTrigger().get()); assertEquals(true, deltaOptions.maxBytesPerTrigger().isDefined()); // Should not throw - supported and custom options are allowed SparkScan.validateStreamingOptions(deltaOptions); } @Test public void testValidateStreamingOptions_UnsupportedOptions() { // Test with blocked DeltaOptions, supported options, and custom user options Map javaOptions = new HashMap<>(); javaOptions.put("startingVersion", "0"); javaOptions.put("readChangeFeed", "true"); javaOptions.put("myCustomOption", "value"); scala.collection.immutable.Map mixedOptions = ScalaUtils.toScalaMap(javaOptions); DeltaOptions deltaOptions = new DeltaOptions(mixedOptions, spark.sessionState().conf()); UnsupportedOperationException exception = assertThrows( UnsupportedOperationException.class, () -> SparkScan.validateStreamingOptions(deltaOptions)); // Verify exact error message - only the blocked option should appear // Note: DeltaOptions uses CaseInsensitiveMap which lowercases keys during iteration assertEquals( "The following streaming options are not supported: [readchangefeed]. " + "Supported options are: [startingVersion, startingTimestamp, maxFilesPerTrigger, " + "maxBytesPerTrigger, ignoreDeletes, skipChangeCommits, excludeRegex].", exception.getMessage()); } // ================================================================================================ // Tests for equals and hashCode // ================================================================================================ @Test public void testEqualsAndHashCode() { // Create two scans from the same table with same options SparkScanBuilder builder1 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan1 = (SparkScan) builder1.build(); SparkScanBuilder builder2 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan2 = (SparkScan) builder2.build(); // Same table, same options should be equal assertEquals(scan1, scan2); assertEquals(scan1.hashCode(), scan2.hashCode()); } @Test public void testEqualsWithDifferentOptions() { SparkScanBuilder builder1 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan1 = (SparkScan) builder1.build(); Map differentOptions = new HashMap<>(); differentOptions.put("customOption", "value"); CaseInsensitiveStringMap optionsMap = new CaseInsensitiveStringMap(differentOptions); SparkScanBuilder builder2 = (SparkScanBuilder) table.newScanBuilder(optionsMap); SparkScan scan2 = (SparkScan) builder2.build(); // Different options should not be equal and hashCodes should differ assertNotEquals(scan1, scan2); assertNotEquals(scan1.hashCode(), scan2.hashCode()); } @Test public void testEqualsWithSameFilters() { // Both scans with equivalent filters created separately (not same instance) SparkScanBuilder builder1 = (SparkScanBuilder) table.newScanBuilder(options); builder1.pushFilters( new org.apache.spark.sql.sources.Filter[] { new org.apache.spark.sql.sources.EqualTo("city", "hz") }); SparkScan scan1 = (SparkScan) builder1.build(); SparkScanBuilder builder2 = (SparkScanBuilder) table.newScanBuilder(options); builder2.pushFilters( new org.apache.spark.sql.sources.Filter[] { new org.apache.spark.sql.sources.EqualTo("city", "hz") }); SparkScan scan2 = (SparkScan) builder2.build(); // Same options and equivalent filters should be equal assertEquals(scan1, scan2); assertEquals(scan1.hashCode(), scan2.hashCode()); } @Test public void testEqualsWithDifferentFilters() { // Scan without filters SparkScanBuilder builder1 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan1 = (SparkScan) builder1.build(); // Scan with filters pushed SparkScanBuilder builder2 = (SparkScanBuilder) table.newScanBuilder(options); builder2.pushFilters( new org.apache.spark.sql.sources.Filter[] { new org.apache.spark.sql.sources.EqualTo("city", "hz") }); SparkScan scan2 = (SparkScan) builder2.build(); // Same options but different filters should not be equal and hashCodes should differ assertNotEquals(scan1, scan2); assertNotEquals(scan1.hashCode(), scan2.hashCode()); } // ================================================================================================ // Tests for estimated size with column projection // ================================================================================================ @Test public void testEstimatedSizeMatchesStatistics() throws Exception { // Test that estimateStatistics().sizeInBytes() returns the estimatedSizeInBytes field SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan = (SparkScan) builder.build(); long estimatedSizeFromStats = scan.estimateStatistics().sizeInBytes().getAsLong(); long estimatedSizeFromField = getEstimatedSizeInBytes(scan); assertEquals(estimatedSizeFromField, estimatedSizeFromStats); } @Test public void testEstimatedSizeWithColumnPruning() throws Exception { // Test that with column pruning, estimatedSizeInBytes is computed correctly // Table schema: (part INT, date STRING, city STRING, name STRING, cnt INT) // Partition columns: (date STRING, city STRING, part INT) // Data columns: (name STRING, cnt INT) // // Formula: estimatedBytes = (totalBytes * outputRowSize) / fullSchemaRowSize // Where: // ROW_OVERHEAD = 8 // dataSchema.defaultSize() = 20 (STRING) + 4 (INT) = 24 // partitionSchema.defaultSize() = 20 + 20 + 4 = 44 // fullSchemaRowSize = 8 + 24 + 44 = 76 // // With pruning to only 'name' column: // readDataSchema.defaultSize() = 20 (STRING only) // readSchema().defaultSize() = 20 + 44 = 64 // outputRowSize = 8 + 64 = 72 // estimatedBytes = (totalBytes * 72) / 76 SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); // Prune columns to only include 'name' (a data column) and partition columns // This simulates: SELECT name, date, city, part FROM table StructType prunedSchema = new StructType() .add("name", DataTypes.StringType) // only one data column .add("date", DataTypes.StringType) // partition columns are always included .add("city", DataTypes.StringType) .add("part", DataTypes.IntegerType); builder.pruneColumns(prunedSchema); SparkScan scan = (SparkScan) builder.build(); long totalBytes = getTotalBytes(scan); long estimatedSize = getEstimatedSizeInBytes(scan); // Calculate expected estimated size using the formula // outputRowSize = 8 + 64 = 72, fullSchemaRowSize = 8 + 24 + 44 = 76 // Note: We don't use Math.max(1, ...) here because totalBytes is guaranteed to be large enough // (parquet files with actual data) that the division result won't be zero. long expectedEstimatedSize = (totalBytes * 72) / 76; assertTrue(totalBytes > 0, "totalBytes should be positive"); assertEquals( expectedEstimatedSize, estimatedSize, String.format( "estimatedSize should be (totalBytes * 72) / 76 = (%d * 72) / 76 = %d", totalBytes, expectedEstimatedSize)); } @Test public void testEstimatedSizeWithColumnPruningAndFiltering() throws Exception { // Test that column pruning and runtime filtering work together correctly // Using same formula as testEstimatedSizeWithColumnPruning: // estimatedBytes = (totalBytes * 72) / 76 SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); // Prune columns to only include 'name' column StructType prunedSchema = new StructType() .add("name", DataTypes.StringType) .add("date", DataTypes.StringType) .add("city", DataTypes.StringType) .add("part", DataTypes.IntegerType); builder.pruneColumns(prunedSchema); SparkScan scan = (SparkScan) builder.build(); // Get initial stats with column pruning long initialTotalBytes = getTotalBytes(scan); long initialEstimatedSize = getEstimatedSizeInBytes(scan); // Verify initial estimated size matches formula // Note: No Math.max(1, ...) needed - totalBytes from parquet files is large enough long expectedInitialEstimated = (initialTotalBytes * 72) / 76; assertEquals( expectedInitialEstimated, initialEstimatedSize, "Initial estimatedSize should match formula"); // Apply a runtime filter scan.filter(new Predicate[] {cityPredicate}); // city=hz // After filtering, verify both values are updated correctly long afterFilterTotalBytes = getTotalBytes(scan); long afterFilterEstimatedSize = getEstimatedSizeInBytes(scan); // Verify estimated size matches formula with new totalBytes long expectedAfterFilterEstimated = (afterFilterTotalBytes * 72) / 76; assertEquals( expectedAfterFilterEstimated, afterFilterEstimatedSize, "After filter, estimatedSize should match formula with new totalBytes"); // Verify both values were reduced assertTrue(afterFilterTotalBytes < initialTotalBytes, "totalBytes should be reduced"); assertTrue(afterFilterEstimatedSize < initialEstimatedSize, "estimatedSize should be reduced"); } @Test public void testEstimatedSizeZeroAfterFilteringOutAllFiles() throws Exception { // Test that filtering out all files results in zero for both sizes SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan = (SparkScan) builder.build(); // Apply filter that matches nothing scan.filter(new Predicate[] {negativeCityPredicate}); // city=zz doesn't exist long afterFilterTotalBytes = getTotalBytes(scan); long afterFilterEstimatedSize = getEstimatedSizeInBytes(scan); assertEquals(0, afterFilterTotalBytes, "totalBytes should be 0 after filtering out all files"); assertEquals( 0, afterFilterEstimatedSize, "estimatedSize should be 0 after filtering out all files"); assertEquals( 0, scan.estimateStatistics().sizeInBytes().getAsLong(), "Statistics sizeInBytes should be 0 after filtering out all files"); } // ================================================================================================ // Tests for equals and hashCode with runtime filters // ================================================================================================ @Test public void testEqualsAndHashCodeWithSameRuntimeFilter() { // Same filter applied to both scans (same instance) SparkScanBuilder builder1 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan1 = (SparkScan) builder1.build(); SparkScanBuilder builder2 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan2 = (SparkScan) builder2.build(); scan1.filter(new Predicate[] {cityPredicate}); scan2.filter(new Predicate[] {cityPredicate}); assertEquals(scan1, scan2); assertEquals(scan1.hashCode(), scan2.hashCode()); } @Test public void testEqualsAndHashCodeWithEquivalentRuntimeFilters() { // Equivalent filters (different instances) SparkScanBuilder builder1 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan1 = (SparkScan) builder1.build(); SparkScanBuilder builder2 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan2 = (SparkScan) builder2.build(); scan1.filter(new Predicate[] {cityPredicate}); Predicate cityPredicateCopy = new Predicate( "=", new Expression[] { FieldReference.apply("city"), LiteralValue.apply("hz", DataTypes.StringType) }); scan2.filter(new Predicate[] {cityPredicateCopy}); assertEquals(scan1, scan2); assertEquals(scan1.hashCode(), scan2.hashCode()); } @Test public void testEqualsAndHashCodeWithMultipleRuntimeFiltersInSameOrder() { // Multiple filters in same order SparkScanBuilder builder1 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan1 = (SparkScan) builder1.build(); SparkScanBuilder builder2 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan2 = (SparkScan) builder2.build(); scan1.filter(new Predicate[] {cityPredicate, datePredicate}); scan2.filter(new Predicate[] {cityPredicate, datePredicate}); assertEquals(scan1, scan2); assertEquals(scan1.hashCode(), scan2.hashCode()); } @Test public void testEqualsAndHashCodeWithIdempotentRuntimeFilters() { // Filter idempotency - applying same filter once vs twice SparkScanBuilder builder1 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan1 = (SparkScan) builder1.build(); SparkScanBuilder builder2 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan2 = (SparkScan) builder2.build(); scan1.filter(new Predicate[] {cityPredicate}); scan2.filter(new Predicate[] {cityPredicate}); scan2.filter(new Predicate[] {cityPredicate}); // Apply same filter twice assertEquals(scan1, scan2); assertEquals(scan1.hashCode(), scan2.hashCode()); } @Test public void testEqualsAndHashCodeWithSeparateRuntimeFilterCalls() { // Multiple separate filter() calls vs single call with multiple filters SparkScanBuilder builder1 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan1 = (SparkScan) builder1.build(); SparkScanBuilder builder2 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan2 = (SparkScan) builder2.build(); scan1.filter(new Predicate[] {cityPredicate}); scan1.filter(new Predicate[] {datePredicate}); scan2.filter(new Predicate[] {cityPredicate, datePredicate}); assertEquals(scan1, scan2); assertEquals(scan1.hashCode(), scan2.hashCode()); } @Test public void testEqualsAndHashCodeWithRuntimeFiltersInDifferentOrder() { // Same filters in different order (order-independent) SparkScanBuilder builder1 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan1 = (SparkScan) builder1.build(); SparkScanBuilder builder2 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan2 = (SparkScan) builder2.build(); scan1.filter(new Predicate[] {cityPredicate, datePredicate}); scan2.filter(new Predicate[] {datePredicate, cityPredicate}); assertEquals(scan1, scan2); assertEquals(scan1.hashCode(), scan2.hashCode()); } @Test public void testEqualsAndHashCodeWithNonPartitionColumnRuntimeFilters() { // Non-partition column predicates should not affect equality // Only partition column predicates should be tracked SparkScanBuilder builder1 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan1 = (SparkScan) builder1.build(); SparkScanBuilder builder2 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan2 = (SparkScan) builder2.build(); // cityPredicate is on partition column, dataPredicate is on non-partition column (cnt) scan1.filter(new Predicate[] {cityPredicate}); scan2.filter(new Predicate[] {cityPredicate, dataPredicate}); // They should be equal because dataPredicate doesn't produce an evaluator assertEquals(scan1, scan2); assertEquals(scan1.hashCode(), scan2.hashCode()); } @Test public void testNotEqualsWithDifferentRuntimeFilters() { // Different filters SparkScanBuilder builder1 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan1 = (SparkScan) builder1.build(); SparkScanBuilder builder2 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan2 = (SparkScan) builder2.build(); scan1.filter(new Predicate[] {cityPredicate}); scan2.filter(new Predicate[] {datePredicate}); assertNotEquals(scan1, scan2); assertNotEquals(scan1.hashCode(), scan2.hashCode()); } @Test public void testNotEqualsWithAndWithoutRuntimeFilter() { // One with filter, one without SparkScanBuilder builder1 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan1 = (SparkScan) builder1.build(); SparkScanBuilder builder2 = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan2 = (SparkScan) builder2.build(); scan2.filter(new Predicate[] {cityPredicate}); assertNotEquals(scan1, scan2); assertNotEquals(scan1.hashCode(), scan2.hashCode()); } // ================================================================================================ // Tests for output partitioning (SupportsReportPartitioning) // ================================================================================================ @Test public void testOutputPartitioningForPartitionedTable() { // Partitioned table should return KeyGroupedPartitioning SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan = (SparkScan) builder.build(); Partitioning partitioning = scan.outputPartitioning(); assertTrue( partitioning instanceof KeyGroupedPartitioning, "Partitioned table should return KeyGroupedPartitioning"); KeyGroupedPartitioning kgp = (KeyGroupedPartitioning) partitioning; // The partitioned table has 3 partition columns: date, city, part Expression[] keys = kgp.keys(); assertEquals(3, keys.length, "Should have 3 partition key expressions"); // Verify partition key names match partition schema Set keyNames = new HashSet<>(); for (Expression key : keys) { assertTrue(key instanceof FieldReference, "Key should be a FieldReference"); keyNames.add(((FieldReference) key).fieldNames()[0]); } assertTrue(keyNames.containsAll(Arrays.asList("date", "city", "part"))); // numPartitions returns partitionedFiles.size() (file count, not unique partition count). // In this test data, each partition has one file, so file count equals partition count. assertEquals(5, kgp.numPartitions(), "Should have 5 files (one per partition)"); } /** Creates a non-partitioned table with sample data and returns a SparkScan for it. */ private SparkScan createNonPartitionedScan(File tempDir, String tableName) { spark.sql( String.format( "CREATE TABLE `%s` (id INT, name STRING) USING delta LOCATION '%s'", tableName, tempDir.getAbsolutePath())); spark.sql(String.format("INSERT INTO %s VALUES (1, 'Alice'), (2, 'Bob')", tableName)); SparkTable nonPartTable = new SparkTable( Identifier.of(new String[] {"spark_catalog", "default"}, tableName), tempDir.getAbsolutePath(), options); SparkScanBuilder builder = (SparkScanBuilder) nonPartTable.newScanBuilder(options); return (SparkScan) builder.build(); } @Test public void testOutputPartitioningForNonPartitionedTable(@TempDir File tempDir) { // Non-partitioned table should return UnknownPartitioning SparkScan scan = createNonPartitionedScan(tempDir, "deltatbl_nonpartitioned"); Partitioning partitioning = scan.outputPartitioning(); assertTrue( partitioning instanceof UnknownPartitioning, "Non-partitioned table should return UnknownPartitioning"); } @Test public void testOutputPartitioningAfterRuntimeFilter() { // Output partitioning should reflect filtered partition count SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan = (SparkScan) builder.build(); // Apply filter to only keep city=hz (2 rows: part=1/date=20180520 and part=1/date=20180718) scan.filter(new Predicate[] {cityPredicate}); Partitioning partitioning = scan.outputPartitioning(); assertTrue(partitioning instanceof KeyGroupedPartitioning); KeyGroupedPartitioning kgp = (KeyGroupedPartitioning) partitioning; // numPartitions returns partitionedFiles.size() (file count); here each partition has one file. assertEquals( 2, kgp.numPartitions(), "After filtering to city=hz, should have 2 files (one per partition)"); } // ================================================================================================ // Tests for DeltaInputPartition in planInputPartitions // ================================================================================================ @Test public void testPlanInputPartitionsReturnsHasPartitionKeyForPartitionedTable() { SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); SparkScan scan = (SparkScan) builder.build(); Batch batch = scan.toBatch(); InputPartition[] partitions = batch.planInputPartitions(); assertTrue(partitions.length > 0, "Should have at least one partition"); for (InputPartition partition : partitions) { assertTrue( partition instanceof DeltaInputPartition, "Partitioned table should return DeltaInputPartition instances"); assertTrue( partition instanceof HasPartitionKey, "DeltaInputPartition should implement HasPartitionKey"); DeltaInputPartition deltaPartition = (DeltaInputPartition) partition; assertNotNull(deltaPartition.partitionKey(), "Partition key should not be null"); assertNotNull(deltaPartition.getFilePartition(), "FilePartition should not be null"); } } @Test public void testPlanInputPartitionsGroupsFilesByPartition(@TempDir File tempDir) throws Exception { // Create a table with multiple files per partition to actually exercise the grouping logic // in planPartitionedInputPartitions (the default test table has 1 file per partition, // which would pass even without grouping). String multiFileTableName = "deltatbl_multifile_partitioned"; spark.sql( String.format( "CREATE TABLE `%s` (id INT, data STRING, part INT) USING delta " + "LOCATION '%s' PARTITIONED BY (part)", multiFileTableName, tempDir.getAbsolutePath())); // Insert in separate statements to create multiple files per partition spark.sql( String.format("INSERT INTO `%s` VALUES (1, 'a', 1), (2, 'b', 2)", multiFileTableName)); spark.sql( String.format("INSERT INTO `%s` VALUES (3, 'c', 1), (4, 'd', 2)", multiFileTableName)); spark.sql(String.format("INSERT INTO `%s` VALUES (5, 'e', 1)", multiFileTableName)); // Now part=1 has 3 files, part=2 has 2 files SparkTable multiFileTable = new SparkTable( Identifier.of(new String[] {"spark_catalog", "default"}, multiFileTableName), tempDir.getAbsolutePath(), options); // Force maxPartitionBytes=1 so each file gets its own FilePartition, making the // totalPartitions > 2 assertion deterministic regardless of default parallelism. withSQLConf( "spark.sql.files.maxPartitionBytes", "1", () -> { SparkScanBuilder builder = (SparkScanBuilder) multiFileTable.newScanBuilder(options); SparkScan scan = (SparkScan) builder.build(); Batch batch = scan.toBatch(); InputPartition[] partitions = batch.planInputPartitions(); // Verify all partitions are DeltaInputPartition with partition keys Map> partitionsByKey = new HashMap<>(); for (InputPartition p : partitions) { assertTrue(p instanceof DeltaInputPartition); DeltaInputPartition dp = (DeltaInputPartition) p; partitionsByKey.computeIfAbsent(dp.partitionKey(), k -> new ArrayList<>()).add(dp); } // Should have exactly 2 unique partition keys (part=1 and part=2) assertEquals(2, partitionsByKey.size(), "Should have 2 unique partition keys"); // Verify that the grouping actually produced multiple DeltaInputPartitions for a // single partition key (since multiple files exist per partition and each gets its // own FilePartition when maxPartitionBytes=1) int totalPartitions = partitions.length; assertTrue( totalPartitions > 2, "With 5 files across 2 partitions, should have more than 2 input partitions, " + "got " + totalPartitions); // Verify all DeltaInputPartitions with the same key share the same partition key for (Map.Entry> entry : partitionsByKey.entrySet()) { List group = entry.getValue(); InternalRow expectedKey = group.get(0).partitionKey(); for (DeltaInputPartition dp : group) { assertEquals( expectedKey, dp.partitionKey(), "All partitions in the same group should have equal partition keys"); } } }); } @Test public void testPlanInputPartitionsReturnsFilePartitionForNonPartitionedTable( @TempDir File tempDir) { SparkScan scan = createNonPartitionedScan(tempDir, "deltatbl_nonpartitioned_batch"); Batch batch = scan.toBatch(); InputPartition[] partitions = batch.planInputPartitions(); assertTrue(partitions.length > 0, "Should have at least one partition"); for (InputPartition partition : partitions) { assertTrue( partition instanceof FilePartition, "Non-partitioned table should return FilePartition instances, not DeltaInputPartition"); assertFalse( partition instanceof DeltaInputPartition, "Non-partitioned table should NOT return DeltaInputPartition"); } } @Test public void testOutputPartitioningForEmptyPartitionedTable(@TempDir File tempDir) { // Empty partitioned table should return KeyGroupedPartitioning with 0 partitions String emptyTableName = "deltatbl_empty_partitioned"; spark.sql( String.format( "CREATE TABLE `%s` (id INT, name STRING, part INT) USING delta " + "LOCATION '%s' PARTITIONED BY (part)", emptyTableName, tempDir.getAbsolutePath())); SparkTable emptyTable = new SparkTable( Identifier.of(new String[] {"spark_catalog", "default"}, emptyTableName), tempDir.getAbsolutePath(), options); SparkScanBuilder builder = (SparkScanBuilder) emptyTable.newScanBuilder(options); SparkScan scan = (SparkScan) builder.build(); Partitioning partitioning = scan.outputPartitioning(); assertTrue( partitioning instanceof KeyGroupedPartitioning, "Empty partitioned table should still return KeyGroupedPartitioning"); KeyGroupedPartitioning kgp = (KeyGroupedPartitioning) partitioning; assertEquals(0, kgp.numPartitions(), "Empty table should have 0 partitions"); Batch batch = scan.toBatch(); InputPartition[] partitions = batch.planInputPartitions(); assertEquals(0, partitions.length, "Empty table should return 0 input partitions"); } // ================================================================================================ // Tests for catalog statistics propagation // ================================================================================================ /** * Helper to inject CatalogStatistics into a catalog table via alterTableStats. This is needed * because ANALYZE TABLE is not supported for V2 tables in the test environment. */ private CatalogTable injectCatalogStats(String tblName, CatalogStatistics stats) throws Exception { TableIdentifier tableId = new TableIdentifier(tblName); spark.sessionState().catalog().alterTableStats(tableId, scala.Option.apply(stats)); return spark.sessionState().catalog().getTableMetadata(tableId); } @Test public void testEstimateStatisticsWithCatalogStats_cboEnabled(@TempDir File tempDir) throws Exception { String path = tempDir.getAbsolutePath(); String tblName = "stats_cbo_enabled"; spark.sql( String.format( "CREATE TABLE %s (id INT, name STRING, value DOUBLE) USING delta LOCATION '%s'", tblName, path)); spark.sql(String.format("INSERT INTO %s VALUES (1, 'a', 1.0), (2, 'b', 2.0)", tblName)); // Inject catalog stats with column stats for "id" CatalogColumnStat idColStat = new CatalogColumnStat( scala.Option.apply(scala.math.BigInt.apply(2L)), // distinctCount scala.Option.apply("1"), // min scala.Option.apply("2"), // max scala.Option.apply(scala.math.BigInt.apply(0L)), // nullCount scala.Option.apply((Object) 4L), // avgLen scala.Option.apply((Object) 4L), // maxLen scala.Option.empty(), // histogram CatalogColumnStat.VERSION()); CatalogStatistics catalogStats = new CatalogStatistics( scala.math.BigInt.apply(1024L), scala.Option.apply(scala.math.BigInt.apply(2L)), buildColStatsMap(new String[] {"id"}, new CatalogColumnStat[] {idColStat})); CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); withSQLConf( "spark.sql.cbo.enabled", "true", () -> { Identifier id = Identifier.of(new String[] {"default"}, tblName); SparkTable sparkTable = new SparkTable(id, catalogTable, Collections.emptyMap()); SparkScanBuilder builder = (SparkScanBuilder) sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); SparkScan scan = (SparkScan) builder.build(); Statistics stats = scan.estimateStatistics(); // Should have numRows from catalog stats assertTrue(stats.numRows().isPresent(), "numRows should be present with CBO enabled"); assertEquals(2L, stats.numRows().getAsLong(), "numRows should be 2"); // sizeInBytes should still come from planned files (more accurate) assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); // Should have column stats Map colStats = stats.columnStats(); assertNotNull(colStats, "columnStats should not be null"); assertFalse(colStats.isEmpty(), "columnStats should not be empty"); // Check that column stats contain expected columns ColumnStatistics idStats = colStats.get(FieldReference.apply("id")); assertNotNull(idStats, "id column stats should be present"); assertTrue(idStats.nullCount().isPresent(), "id nullCount should be present"); assertTrue(idStats.distinctCount().isPresent(), "id distinctCount should be present"); assertTrue(idStats.min().isPresent(), "id min should be present"); assertTrue(idStats.max().isPresent(), "id max should be present"); assertEquals(1, idStats.min().get(), "id min should be 1"); assertEquals(2, idStats.max().get(), "id max should be 2"); }); } @Test public void testEstimateStatisticsWithCatalogStats_cboDisabled(@TempDir File tempDir) throws Exception { String path = tempDir.getAbsolutePath(); String tblName = "stats_cbo_disabled"; spark.sql( String.format( "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tblName, path)); spark.sql(String.format("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tblName)); // Inject catalog stats CatalogStatistics catalogStats = new CatalogStatistics( scala.math.BigInt.apply(512L), scala.Option.apply(scala.math.BigInt.apply(2L)), scala.collection.immutable.Map$.MODULE$.empty()); CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); withSQLConf( "spark.sql.cbo.enabled", "false", () -> { Identifier id = Identifier.of(new String[] {"default"}, tblName); SparkTable sparkTable = new SparkTable(id, catalogTable, Collections.emptyMap()); SparkScanBuilder builder = (SparkScanBuilder) sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); SparkScan scan = (SparkScan) builder.build(); Statistics stats = scan.estimateStatistics(); // With CBO disabled, numRows should be empty (matching V1 behavior) assertFalse(stats.numRows().isPresent(), "numRows should be empty with CBO disabled"); // sizeInBytes should still come from planned files assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); }); } @Test public void testEstimateStatisticsWithCatalogStats_planStatsEnabled(@TempDir File tempDir) throws Exception { String path = tempDir.getAbsolutePath(); String tblName = "stats_plan_stats_enabled"; spark.sql( String.format( "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tblName, path)); spark.sql(String.format("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tblName)); // Inject catalog stats with numRows CatalogStatistics catalogStats = new CatalogStatistics( scala.math.BigInt.apply(512L), scala.Option.apply(scala.math.BigInt.apply(2L)), scala.collection.immutable.Map$.MODULE$.empty()); CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); // CBO disabled but planStatsEnabled=true should still surface catalog stats withSQLConf( "spark.sql.cbo.enabled", "false", () -> { withSQLConf( "spark.sql.cbo.planStats.enabled", "true", () -> { Identifier id = Identifier.of(new String[] {"default"}, tblName); SparkTable sparkTable = new SparkTable(id, catalogTable, Collections.emptyMap()); SparkScanBuilder builder = (SparkScanBuilder) sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); SparkScan scan = (SparkScan) builder.build(); Statistics stats = scan.estimateStatistics(); // With planStatsEnabled, numRows should come from catalog stats assertTrue( stats.numRows().isPresent(), "numRows should be present with planStatsEnabled"); assertEquals(2L, stats.numRows().getAsLong(), "numRows should be 2"); // sizeInBytes should still come from planned files assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); }); }); } @Test public void testEstimateStatisticsWithoutCatalogStats(@TempDir File tempDir) throws Exception { // Path-based table has no catalog stats String path = tempDir.getAbsolutePath(); String tblName = "stats_no_catalog"; spark.sql( String.format( "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tblName, path)); spark.sql(String.format("INSERT INTO %s VALUES (1, 'a')", tblName)); withSQLConf( "spark.sql.cbo.enabled", "true", () -> { // Path-based table — no catalog table, no stats Identifier id = Identifier.of(new String[] {"default"}, tblName); SparkTable sparkTable = new SparkTable(id, path); SparkScanBuilder builder = (SparkScanBuilder) sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); SparkScan scan = (SparkScan) builder.build(); Statistics stats = scan.estimateStatistics(); // Without catalog stats, numRows should be empty assertFalse(stats.numRows().isPresent(), "numRows should be empty for path-based table"); assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); }); } @Test public void testEstimateStatisticsWithPartitionedTableAndCatalogStats(@TempDir File tempDir) throws Exception { String path = tempDir.getAbsolutePath(); String tblName = "stats_partitioned"; spark.sql( String.format( "CREATE TABLE %s (id INT, name STRING, part INT) USING delta " + "PARTITIONED BY (part) LOCATION '%s'", tblName, path)); spark.sql( String.format("INSERT INTO %s VALUES (1, 'a', 1), (2, 'b', 1), (3, 'c', 2)", tblName)); // Inject catalog stats with column stats for both data and partition columns CatalogColumnStat idColStat = new CatalogColumnStat( scala.Option.apply(scala.math.BigInt.apply(3L)), scala.Option.apply("1"), scala.Option.apply("3"), scala.Option.apply(scala.math.BigInt.apply(0L)), scala.Option.empty(), scala.Option.empty(), scala.Option.empty(), CatalogColumnStat.VERSION()); CatalogColumnStat partColStat = new CatalogColumnStat( scala.Option.apply(scala.math.BigInt.apply(2L)), scala.Option.apply("1"), scala.Option.apply("2"), scala.Option.apply(scala.math.BigInt.apply(0L)), scala.Option.empty(), scala.Option.empty(), scala.Option.empty(), CatalogColumnStat.VERSION()); CatalogStatistics catalogStats = new CatalogStatistics( scala.math.BigInt.apply(2048L), scala.Option.apply(scala.math.BigInt.apply(3L)), buildColStatsMap( new String[] {"id", "part"}, new CatalogColumnStat[] {idColStat, partColStat})); CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); withSQLConf( "spark.sql.cbo.enabled", "true", () -> { Identifier id = Identifier.of(new String[] {"default"}, tblName); SparkTable sparkTable = new SparkTable(id, catalogTable, Collections.emptyMap()); SparkScanBuilder builder = (SparkScanBuilder) sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); SparkScan scan = (SparkScan) builder.build(); Statistics stats = scan.estimateStatistics(); assertTrue(stats.numRows().isPresent(), "numRows should be present"); assertEquals(3L, stats.numRows().getAsLong(), "numRows should be 3"); // Verify column stats include both data and partition columns Map colStats = stats.columnStats(); assertNotNull(colStats.get(FieldReference.apply("id")), "id stats should be present"); assertNotNull(colStats.get(FieldReference.apply("part")), "part stats should be present"); // Check partition column stats ColumnStatistics partStats = colStats.get(FieldReference.apply("part")); assertTrue(partStats.min().isPresent(), "part min should be present"); assertTrue(partStats.max().isPresent(), "part max should be present"); assertEquals(1, partStats.min().get(), "part min should be 1"); assertEquals(2, partStats.max().get(), "part max should be 2"); }); } @Test public void testEstimatedSizeUsesAvgLenFromCatalogStats(@TempDir File tempDir) throws Exception { // Verify that computeEstimatedSizeWithColumnProjection uses avgLen from catalog stats // instead of defaultSize(), mirroring EstimationUtils.getSizePerRow() (#5952). String path = tempDir.getAbsolutePath(); String tblName = "stats_avglen"; spark.sql( String.format( "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tblName, path)); spark.sql(String.format("INSERT INTO %s VALUES (1, 'a'), (2, 'bb'), (3, 'ccc')", tblName)); // Create column stats with avgLen=5 for 'name' (STRING defaultSize is 20) CatalogColumnStat nameColStat = new CatalogColumnStat( scala.Option.empty(), scala.Option.empty(), scala.Option.empty(), scala.Option.empty(), scala.Option.apply((Object) 5L), // avgLen = 5 (vs STRING defaultSize 20) scala.Option.empty(), scala.Option.empty(), CatalogColumnStat.VERSION()); CatalogStatistics catalogStats = new CatalogStatistics( scala.math.BigInt.apply(1024L), scala.Option.apply(scala.math.BigInt.apply(3L)), buildColStatsMap(new String[] {"name"}, new CatalogColumnStat[] {nameColStat})); CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); // avgLen is used for sizeInBytes estimation regardless of CBO/planStats settings withSQLConf( "spark.sql.cbo.enabled", "false", () -> { withSQLConf( "spark.sql.cbo.planStats.enabled", "false", () -> { Identifier id = Identifier.of(new String[] {"default"}, tblName); SparkTable sparkTable = new SparkTable(id, catalogTable, Collections.emptyMap()); SparkScanBuilder builder = (SparkScanBuilder) sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); // Prune to only 'name' column to trigger column projection estimation StructType prunedSchema = new StructType().add("name", DataTypes.StringType); builder.pruneColumns(prunedSchema); SparkScan scan = (SparkScan) builder.build(); long totalBytes = getTotalBytes(scan); long estimatedSize = getEstimatedSizeInBytes(scan); assertTrue(totalBytes > 0, "totalBytes should be positive"); // Schema: dataSchema = (id INT, name STRING), partitionSchema = empty // readSchema = (name STRING) after pruning // // With avgLen=5 for name (STRING: avgLen + 12 = 17, vs defaultSize 20): // fullSchemaRowSize = 8 + (4 [INT default] + 17 [STRING avgLen]) = 29 // outputRowSize = 8 + 17 = 25 // estimated = (totalBytes * 25) / 29 // // Without avgLen (defaultSize only): // fullSchemaRowSize = 8 + (4 + 20) = 32 // outputRowSize = 8 + 20 = 28 // estimated = (totalBytes * 28) / 32 long expectedWithAvgLen = (totalBytes * 25) / 29; long expectedWithoutAvgLen = (totalBytes * 28) / 32; assertEquals( expectedWithAvgLen, estimatedSize, "estimatedSize should use avgLen from catalog stats"); assertNotEquals( expectedWithoutAvgLen, estimatedSize, "estimatedSize should differ from defaultSize-only calculation"); }); }); } @Test public void testEstimateStatisticsWithoutAnalyze(@TempDir File tempDir) throws Exception { // Table exists in catalog but no stats were injected String path = tempDir.getAbsolutePath(); String tblName = "stats_no_analyze"; spark.sql( String.format( "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tblName, path)); spark.sql(String.format("INSERT INTO %s VALUES (1, 'a')", tblName)); withSQLConf( "spark.sql.cbo.enabled", "true", () -> { CatalogTable catalogTable = spark.sessionState().catalog().getTableMetadata(new TableIdentifier(tblName)); Identifier id = Identifier.of(new String[] {"default"}, tblName); SparkTable sparkTable = new SparkTable(id, catalogTable, Collections.emptyMap()); SparkScanBuilder builder = (SparkScanBuilder) sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); SparkScan scan = (SparkScan) builder.build(); Statistics stats = scan.estimateStatistics(); // Without catalog stats, we fall back to file-only stats assertFalse(stats.numRows().isPresent(), "numRows should be empty without catalog stats"); assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); }); } @SuppressWarnings({"rawtypes", "unchecked"}) private static scala.collection.immutable.Map buildColStatsMap( String[] keys, CatalogColumnStat[] values) { scala.collection.mutable.Builder b = scala.collection.immutable.Map$.MODULE$.newBuilder(); for (int i = 0; i < keys.length; i++) { b.$plus$eq(new scala.Tuple2<>(keys[i], values[i])); } return (scala.collection.immutable.Map) b.result(); } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/read/deletionvector/ColumnVectorWithFilterTest.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.read.deletionvector; import static org.junit.jupiter.api.Assertions.*; import java.util.function.IntFunction; import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector; import org.apache.spark.sql.execution.vectorized.WritableColumnVector; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.vectorized.ColumnVector; import org.junit.jupiter.api.Test; public class ColumnVectorWithFilterTest { @Test void testIntegerColumnVector() { try (WritableColumnVector delegate = new OnHeapColumnVector(5, DataTypes.IntegerType)) { for (int i = 0; i < 5; i++) { delegate.putInt(i, (i + 1) * 10); // [10, 20, 30, 40, 50] } ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {1, 3}); assertMappedValues(mappedVector, mappedVector::getInt, new Integer[] {20, 40}); } } @Test void testLongColumnVector() { try (WritableColumnVector delegate = new OnHeapColumnVector(4, DataTypes.LongType)) { for (int i = 0; i < 4; i++) { delegate.putLong(i, (i + 1) * 100L); } ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {0, 2}); assertMappedValues(mappedVector, mappedVector::getLong, new Long[] {100L, 300L}); } } @Test void testDoubleColumnVector() { try (WritableColumnVector delegate = new OnHeapColumnVector(3, DataTypes.DoubleType)) { delegate.putDouble(0, 1.1); delegate.putDouble(1, 2.2); delegate.putDouble(2, 3.3); ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {0, 2}); assertMappedValues(mappedVector, mappedVector::getDouble, new Double[] {1.1, 3.3}); } } @Test void testBooleanColumnVector() { try (WritableColumnVector delegate = new OnHeapColumnVector(4, DataTypes.BooleanType)) { delegate.putBoolean(0, true); delegate.putBoolean(1, false); delegate.putBoolean(2, true); delegate.putBoolean(3, false); ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {1, 2}); assertMappedValues(mappedVector, mappedVector::getBoolean, new Boolean[] {false, true}); } } @Test void testStringColumnVector() { try (WritableColumnVector delegate = new OnHeapColumnVector(3, DataTypes.StringType)) { delegate.putByteArray(0, "alice".getBytes()); delegate.putByteArray(1, "bob".getBytes()); delegate.putByteArray(2, "charlie".getBytes()); ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {2}); assertMappedValues( mappedVector, i -> mappedVector.getUTF8String(i).toString(), new String[] {"charlie"}); } } @Test void testStructColumnVector() { StructType structType = new StructType().add("id", DataTypes.IntegerType).add("name", DataTypes.StringType); try (WritableColumnVector delegate = new OnHeapColumnVector(3, structType)) { WritableColumnVector idChild = (WritableColumnVector) delegate.getChild(0); for (int i = 0; i < 3; i++) { idChild.putInt(i, i + 1); // [1, 2, 3] } WritableColumnVector nameChild = (WritableColumnVector) delegate.getChild(1); nameChild.putByteArray(0, "a".getBytes()); nameChild.putByteArray(1, "b".getBytes()); nameChild.putByteArray(2, "c".getBytes()); ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {0, 2}); ColumnVector mappedIdColumn = mappedVector.getChild(0); ColumnVector mappedNameColumn = mappedVector.getChild(1); assertMappedValues(mappedIdColumn, mappedIdColumn::getInt, new Integer[] {1, 3}); assertMappedValues( mappedNameColumn, i -> mappedNameColumn.getUTF8String(i).toString(), new String[] {"a", "c"}); } } @Test void testNullColumnVector() { try (WritableColumnVector delegate = new OnHeapColumnVector(4, DataTypes.IntegerType)) { // original rows: [10, null, 30, null] delegate.putInt(0, 10); delegate.putNull(1); delegate.putInt(2, 30); delegate.putNull(3); // selected rows (in order): [3, 0, 1] => [null, 10, null] ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {3, 0, 1}); assertMappedValues(mappedVector, mappedVector::getInt, new Integer[] {null, 10, null}); } } @Test void testAllNullColumnVector() { try (WritableColumnVector delegate = new OnHeapColumnVector(3, DataTypes.IntegerType)) { delegate.putNull(0); delegate.putNull(1); delegate.putNull(2); ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {2, 0}); assertMappedValues(mappedVector, mappedVector::getInt, new Integer[] {null, null}); } } @Test void testNoneSelectColumnVector() { try (WritableColumnVector delegate = new OnHeapColumnVector(3, DataTypes.IntegerType)) { delegate.putInt(0, 10); delegate.putInt(1, 20); delegate.putInt(2, 30); ColumnVectorWithFilter noRowsSelected = new ColumnVectorWithFilter(delegate, new int[] {}); assertEquals(DataTypes.IntegerType, noRowsSelected.dataType()); } } @Test void testAllSelectColumnVector() { try (WritableColumnVector delegate = new OnHeapColumnVector(3, DataTypes.IntegerType)) { delegate.putInt(0, 10); delegate.putInt(1, 20); delegate.putInt(2, 30); // all select ColumnVectorWithFilter allRowsSelected = new ColumnVectorWithFilter(delegate, new int[] {0, 1, 2}); assertMappedValues(allRowsSelected, allRowsSelected::getInt, new Integer[] {10, 20, 30}); } } private static void assertMappedValues( ColumnVector vector, IntFunction getter, T[] expected) { for (int i = 0; i < expected.length; i++) { if (expected[i] == null) { assertTrue(vector.isNullAt(i), "Expected null at mapped row " + i); } else { assertFalse(vector.isNullAt(i), "Expected non-null at mapped row " + i); assertEquals(expected[i], getter.apply(i), "Mismatch at mapped row " + i); } } } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/read/deletionvector/DeletionVectorReadFunctionTest.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.read.deletionvector; import static io.delta.spark.internal.v2.InternalRowTestUtils.*; import static org.junit.jupiter.api.Assertions.*; import java.nio.charset.StandardCharsets; import java.util.List; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector; import org.apache.spark.sql.execution.vectorized.WritableColumnVector; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.vectorized.ColumnVector; import org.apache.spark.sql.vectorized.ColumnarBatch; import org.junit.jupiter.api.Test; public class DeletionVectorReadFunctionTest { private static final StructType DATA_SCHEMA = new StructType().add("id", DataTypes.IntegerType).add("name", DataTypes.StringType); private static final StructType PARTITION_SCHEMA = new StructType(); // ===== Row-based tests ===== @Test public void testFilterDeletedRowsAndProjectRemovesDvColumn() { // Input: 3 rows, middle one is deleted. List inputRows = List.of( row(1, "alice", (byte) 0), // Not deleted. row(2, "bob", (byte) 1), // Deleted. row(3, "charlie", (byte) 0)); // Not deleted. DeletionVectorSchemaContext context = new DeletionVectorSchemaContext(DATA_SCHEMA, PARTITION_SCHEMA); DeletionVectorReadFunction readFunc = DeletionVectorReadFunction.wrap(mockReader(inputRows), context, false); List result = collectRows(readFunc.apply(/* file= */ null)); // Verify filtered and projected output (DV column removed, deleted row filtered). assertRowsEquals(result, List.of(row(1, "alice"), row(3, "charlie"))); } @Test public void testAllRowsDeleted() { List inputRows = List.of(row(1, "alice", (byte) 1), row(2, "bob", (byte) 1)); // All deleted. DeletionVectorSchemaContext context = new DeletionVectorSchemaContext(DATA_SCHEMA, PARTITION_SCHEMA); DeletionVectorReadFunction readFunc = DeletionVectorReadFunction.wrap(mockReader(inputRows), context, false); List result = collectRows(readFunc.apply(/* file= */ null)); assertRowsEquals(result, List.of()); } @Test public void testNoRowsDeleted() { List inputRows = List.of(row(1, "alice", (byte) 0), row(2, "bob", (byte) 0), row(3, "charlie", (byte) 0)); DeletionVectorSchemaContext context = new DeletionVectorSchemaContext(DATA_SCHEMA, PARTITION_SCHEMA); DeletionVectorReadFunction readFunc = DeletionVectorReadFunction.wrap(mockReader(inputRows), context, false); List result = collectRows(readFunc.apply(/* file= */ null)); assertRowsEquals(result, List.of(row(1, "alice"), row(2, "bob"), row(3, "charlie"))); } // ===== ColumnarBatch (vectorized) tests ===== @Test public void testBatchFilterDeletedRowsAndProjectRemovesDvColumn() { // 3 rows: row 1 deleted, rows 0 and 2 kept. ColumnarBatch inputBatch = createBatch(new int[] {1, 2, 3}, new byte[] {0, 1, 0}); List result = runBatchRead(inputBatch); assertEquals(1, result.size()); // Filtered: row 0 -> original 0 (id=1), row 1 -> original 2 (id=3) assertBatchRows(result.get(0), new int[] {1, 3}, new String[] {"name_0", "name_2"}); } @Test public void testBatchAllRowsDeleted() { ColumnarBatch inputBatch = createBatch(new int[] {1, 2}, new byte[] {1, 1}); List result = runBatchRead(inputBatch); assertEquals(1, result.size()); assertBatchRows(result.get(0), new int[] {}, new String[] {}); } @Test public void testBatchNoRowsDeleted() { ColumnarBatch inputBatch = createBatch(new int[] {1, 2, 3}, new byte[] {0, 0, 0}); List result = runBatchRead(inputBatch); assertEquals(1, result.size()); assertBatchRows( result.get(0), new int[] {1, 2, 3}, new String[] {"name_0", "name_1", "name_2"}); } @Test public void testBatchMultipleBatchesWithDifferentBatch() { ColumnarBatch allDeleted = createBatch(new int[] {1, 2, 3}, new byte[] {1, 1, 1}); ColumnarBatch mixed = createBatch(new int[] {4, 5, 6}, new byte[] {0, 1, 0}); ColumnarBatch allLive = createBatch(new int[] {7}, new byte[] {0}); // Ordering 1: allDeleted, mixed, allLive List result1 = runBatchRead(allDeleted, mixed, allLive); assertEquals(3, result1.size()); assertBatchRows(result1.get(0), new int[] {}, new String[] {}); assertBatchRows(result1.get(1), new int[] {4, 6}, new String[] {"name_0", "name_2"}); assertBatchRows(result1.get(2), new int[] {7}, new String[] {"name_0"}); // Ordering 2: allLive, allDeleted, mixed List result2 = runBatchRead(allLive, allDeleted, mixed); assertEquals(3, result2.size()); assertBatchRows(result2.get(0), new int[] {7}, new String[] {"name_0"}); assertBatchRows(result2.get(1), new int[] {}, new String[] {}); assertBatchRows(result2.get(2), new int[] {4, 6}, new String[] {"name_0", "name_2"}); } private List runBatchRead(ColumnarBatch... inputBatches) { DeletionVectorSchemaContext context = new DeletionVectorSchemaContext(DATA_SCHEMA, PARTITION_SCHEMA); DeletionVectorReadFunction readFunc = DeletionVectorReadFunction.wrap(mockBatchReader(List.of(inputBatches)), context, true); return collectBatches(readFunc.apply(/* file= */ null)); } private static void assertBatchRows( ColumnarBatch batch, int[] expectedIds, String[] expectedNames) { assertEquals(expectedIds.length, expectedNames.length, "Expected id/name lengths must match"); assertEquals(expectedIds.length, batch.numRows(), "Unexpected number of filtered rows"); for (int i = 0; i < expectedIds.length; i++) { assertEquals(expectedIds[i], batch.column(0).getInt(i)); assertEquals(expectedNames[i], batch.column(1).getUTF8String(i).toString()); } } /** * Creates a ColumnarBatch with columns [id (int), name (string), is_row_deleted (byte)]. * *

Name values are auto-generated as "name_0", "name_1", etc. */ private static ColumnarBatch createBatch(int[] ids, byte[] deletionVector) { int numRows = ids.length; WritableColumnVector idCol = new OnHeapColumnVector(numRows, DataTypes.IntegerType); WritableColumnVector nameCol = new OnHeapColumnVector(numRows, DataTypes.StringType); WritableColumnVector dvCol = new OnHeapColumnVector(numRows, DataTypes.ByteType); for (int i = 0; i < numRows; i++) { idCol.putInt(i, ids[i]); nameCol.putByteArray(i, ("name_" + i).getBytes(StandardCharsets.UTF_8)); dvCol.putByte(i, deletionVector[i]); } return new ColumnarBatch(new ColumnVector[] {idCol, nameCol, dvCol}, numRows); } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/read/deletionvector/DeletionVectorSchemaContextTest.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.read.deletionvector; import static org.junit.jupiter.api.Assertions.*; import org.apache.spark.sql.delta.DeltaParquetFileFormat; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; import org.junit.jupiter.api.Test; public class DeletionVectorSchemaContextTest { // Common test schemas. private static final StructType DATA_SCHEMA = new StructType().add("id", DataTypes.IntegerType).add("name", DataTypes.StringType); private static final StructType PARTITION_SCHEMA = new StructType().add("date", DataTypes.StringType); @Test void testWithFullSchemas() { DeletionVectorSchemaContext context = new DeletionVectorSchemaContext(DATA_SCHEMA, PARTITION_SCHEMA); StructType expectedSchemaWithDv = DATA_SCHEMA.add(DeltaParquetFileFormat.IS_ROW_DELETED_STRUCT_FIELD()); assertEquals(expectedSchemaWithDv, context.getSchemaWithDvColumn()); assertEquals(2, context.getDvColumnIndex()); // Input: 2 data + 1 DV + 1 partition = 4. assertEquals(4, context.getInputColumnCount()); StructType expectedOutputSchema = DATA_SCHEMA.merge(PARTITION_SCHEMA, /* handleDuplicateColumns= */ false); assertEquals(expectedOutputSchema, context.getOutputSchema()); } @Test void testEmptyPartitionSchema() { StructType emptyPartition = new StructType(); DeletionVectorSchemaContext context = new DeletionVectorSchemaContext(DATA_SCHEMA, emptyPartition); StructType expectedSchemaWithDv = DATA_SCHEMA.add(DeltaParquetFileFormat.IS_ROW_DELETED_STRUCT_FIELD()); assertEquals(expectedSchemaWithDv, context.getSchemaWithDvColumn()); assertEquals(2, context.getDvColumnIndex()); // Input: 2 data + 1 DV = 3. assertEquals(3, context.getInputColumnCount()); assertEquals(DATA_SCHEMA, context.getOutputSchema()); } @Test void testEmptyDataSchema() { StructType emptyData = new StructType(); DeletionVectorSchemaContext context = new DeletionVectorSchemaContext(emptyData, PARTITION_SCHEMA); StructType expectedSchemaWithDv = emptyData.add(DeltaParquetFileFormat.IS_ROW_DELETED_STRUCT_FIELD()); assertEquals(expectedSchemaWithDv, context.getSchemaWithDvColumn()); assertEquals(0, context.getDvColumnIndex()); // Input: 1 DV + 1 partition = 2. assertEquals(2, context.getInputColumnCount()); assertEquals(PARTITION_SCHEMA, context.getOutputSchema()); } @Test void testDuplicateDvColumnThrowsException() { // Schema that already contains the DV column. StructType schemaWithDv = new StructType() .add("id", DataTypes.IntegerType) .add(DeltaParquetFileFormat.IS_ROW_DELETED_COLUMN_NAME(), DataTypes.ByteType); IllegalArgumentException exception = assertThrows( IllegalArgumentException.class, () -> new DeletionVectorSchemaContext(schemaWithDv, new StructType())); assertTrue( exception.getMessage().contains(DeltaParquetFileFormat.IS_ROW_DELETED_COLUMN_NAME())); } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/snapshot/PathBasedSnapshotManagerTest.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.snapshot; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import io.delta.kernel.Snapshot; import io.delta.kernel.internal.DeltaHistoryManager; import io.delta.spark.internal.v2.DeltaV2TestBase; import io.delta.spark.internal.v2.exception.VersionNotFoundException; import java.io.File; import java.sql.Timestamp; import java.util.stream.Stream; import org.apache.hadoop.fs.Path; import org.apache.spark.sql.delta.DeltaLog; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import scala.Option; public class PathBasedSnapshotManagerTest extends DeltaV2TestBase { private PathBasedSnapshotManager snapshotManager; @Test public void testUnsafeVolatileSnapshot(@TempDir File tempDir) { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_volatile_snapshot"; createEmptyTestTable(testTablePath, testTableName); snapshotManager = new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); org.apache.spark.sql.delta.Snapshot deltaSnapshot = deltaLog.unsafeVolatileSnapshot(); Snapshot kernelSnapshot = snapshotManager.loadLatestSnapshot(); spark.sql(String.format("INSERT INTO %s VALUES (4, 'David')", testTableName)); assertEquals(0L, deltaSnapshot.version()); assertEquals(deltaSnapshot.version(), kernelSnapshot.getVersion()); } @Test public void testLoadLatestSnapshot(@TempDir File tempDir) { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_update"; createEmptyTestTable(testTablePath, testTableName); snapshotManager = new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); Snapshot initialSnapshot = snapshotManager.loadLatestSnapshot(); assertEquals(0L, initialSnapshot.getVersion()); spark.sql(String.format("INSERT INTO %s VALUES (4, 'David')", testTableName)); org.apache.spark.sql.delta.Snapshot deltaSnapshot = deltaLog.update(false, Option.empty(), Option.empty()); Snapshot updatedSnapshot = snapshotManager.loadLatestSnapshot(); org.apache.spark.sql.delta.Snapshot cachedSnapshot = deltaLog.unsafeVolatileSnapshot(); Snapshot kernelcachedSnapshot = snapshotManager.loadLatestSnapshot(); assertEquals(1L, updatedSnapshot.getVersion()); assertEquals(deltaSnapshot.version(), updatedSnapshot.getVersion()); assertEquals(1L, kernelcachedSnapshot.getVersion()); assertEquals(cachedSnapshot.version(), kernelcachedSnapshot.getVersion()); } @Test public void testMultipleLoadLatestSnapshot(@TempDir File tempDir) { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_multiple_updates"; createEmptyTestTable(testTablePath, testTableName); snapshotManager = new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); assertEquals(0L, snapshotManager.loadLatestSnapshot().getVersion()); for (int i = 0; i < 3; i++) { spark.sql( String.format("INSERT INTO %s VALUES (%d, 'User%d')", testTableName, 20 + i, 20 + i)); org.apache.spark.sql.delta.Snapshot deltaSnapshot = deltaLog.update(false, Option.empty(), Option.empty()); Snapshot kernelSnapshot = snapshotManager.loadLatestSnapshot(); long expectedVersion = i + 1; assertEquals(expectedVersion, deltaSnapshot.version()); assertEquals(expectedVersion, kernelSnapshot.getVersion()); } } @Test public void testLoadSnapshotAt(@TempDir File tempDir) { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_load_at_version"; createEmptyTestTable(testTablePath, testTableName); snapshotManager = new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); // Create multiple versions for (int i = 0; i < 3; i++) { spark.sql( String.format("INSERT INTO %s VALUES (%d, 'User%d')", testTableName, 10 + i, 10 + i)); } // Load specific versions Snapshot snapshot0 = snapshotManager.loadSnapshotAt(0L); assertEquals(0L, snapshot0.getVersion()); Snapshot snapshot1 = snapshotManager.loadSnapshotAt(1L); assertEquals(1L, snapshot1.getVersion()); Snapshot snapshot2 = snapshotManager.loadSnapshotAt(2L); assertEquals(2L, snapshot2.getVersion()); Snapshot snapshot3 = snapshotManager.loadSnapshotAt(3L); assertEquals(3L, snapshot3.getVersion()); // Note: loadSnapshotAt does not update the cached snapshot } private void setupTableWithDeletedVersions(String testTablePath, String testTableName) { createEmptyTestTable(testTablePath, testTableName); for (int i = 0; i < 10; i++) { spark.sql( String.format("INSERT INTO %s VALUES (%d, 'User%d')", testTableName, 100 + i, 100 + i)); } File deltaLogDir = new File(testTablePath, "_delta_log"); File version0File = new File(deltaLogDir, "00000000000000000000.json"); File version1File = new File(deltaLogDir, "00000000000000000001.json"); assertTrue(version0File.exists()); assertTrue(version1File.exists()); version0File.delete(); version1File.delete(); assertFalse(version0File.exists()); assertFalse(version1File.exists()); } @Test public void testGetActiveCommitAtTime_pastTimestamp(@TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_commit_past"; setupTableWithDeletedVersions(testTablePath, testTableName); snapshotManager = new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); Thread.sleep(100); Timestamp timestamp = new Timestamp(System.currentTimeMillis()); spark.sql(String.format("INSERT INTO %s VALUES (200, 'NewUser')", testTableName)); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); org.apache.spark.sql.delta.DeltaHistoryManager.Commit deltaCommit = deltaLog .history() .getActiveCommitAtTime( timestamp, Option.empty() /* catalogTable */, false /* canReturnLastCommit */, true /* mustBeRecreatable */, false /* canReturnEarliestCommit */); DeltaHistoryManager.Commit kernelCommit = snapshotManager.getActiveCommitAtTime( timestamp.getTime(), false /* canReturnLastCommit */, true /* mustBeRecreatable */, false /* canReturnEarliestCommit */); assertEquals(deltaCommit.version(), kernelCommit.getVersion()); assertEquals(deltaCommit.timestamp(), kernelCommit.getTimestamp()); } @Test public void testGetActiveCommitAtTime_futureTimestamp_canReturnLast(@TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_commit_future_last"; setupTableWithDeletedVersions(testTablePath, testTableName); snapshotManager = new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); Timestamp futureTimestamp = new Timestamp(System.currentTimeMillis() + 10000); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); org.apache.spark.sql.delta.DeltaHistoryManager.Commit deltaCommit = deltaLog .history() .getActiveCommitAtTime( futureTimestamp, Option.empty() /* catalogTable */, true /* canReturnLastCommit */, true /* mustBeRecreatable */, false /* canReturnEarliestCommit */); DeltaHistoryManager.Commit kernelCommit = snapshotManager.getActiveCommitAtTime( futureTimestamp.getTime(), true /* canReturnLastCommit */, true /* mustBeRecreatable */, false /* canReturnEarliestCommit */); assertEquals(deltaCommit.version(), kernelCommit.getVersion()); assertEquals(deltaCommit.timestamp(), kernelCommit.getTimestamp()); } @Test public void testGetActiveCommitAtTime_futureTimestamp_notMustBeRecreatable(@TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_commit_future_not_recreatable"; setupTableWithDeletedVersions(testTablePath, testTableName); snapshotManager = new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); Timestamp futureTimestamp = new Timestamp(System.currentTimeMillis() + 10000); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); org.apache.spark.sql.delta.DeltaHistoryManager.Commit deltaCommit = deltaLog .history() .getActiveCommitAtTime( futureTimestamp, Option.empty() /* catalogTable */, true /* canReturnLastCommit */, false /* mustBeRecreatable */, false /* canReturnEarliestCommit */); DeltaHistoryManager.Commit kernelCommit = snapshotManager.getActiveCommitAtTime( futureTimestamp.getTime(), true /* canReturnLastCommit */, false /* mustBeRecreatable */, false /* canReturnEarliestCommit */); assertEquals(deltaCommit.version(), kernelCommit.getVersion()); assertEquals(deltaCommit.timestamp(), kernelCommit.getTimestamp()); } @Test public void testGetActiveCommitAtTime_earlyTimestamp_canReturnEarliest(@TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_commit_early"; setupTableWithDeletedVersions(testTablePath, testTableName); snapshotManager = new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); Timestamp earlyTimestamp = new Timestamp(0); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); org.apache.spark.sql.delta.DeltaHistoryManager.Commit deltaCommit = deltaLog .history() .getActiveCommitAtTime( earlyTimestamp, Option.empty() /* catalogTable */, false /* canReturnLastCommit */, true /* mustBeRecreatable */, true /* canReturnEarliestCommit */); DeltaHistoryManager.Commit kernelCommit = snapshotManager.getActiveCommitAtTime( earlyTimestamp.getTime(), false /* canReturnLastCommit */, true /* mustBeRecreatable */, true /* canReturnEarliestCommit */); assertEquals(deltaCommit.version(), kernelCommit.getVersion()); assertEquals(deltaCommit.timestamp(), kernelCommit.getTimestamp()); } @Test public void testGetActiveCommitAtTime_earlyTimestamp_notMustBeRecreatable_canReturnEarliest( @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_commit_early_not_recreatable"; setupTableWithDeletedVersions(testTablePath, testTableName); snapshotManager = new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); Timestamp earlyTimestamp = new Timestamp(0); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); org.apache.spark.sql.delta.DeltaHistoryManager.Commit deltaCommit = deltaLog .history() .getActiveCommitAtTime( earlyTimestamp, Option.empty() /* catalogTable */, false /* canReturnLastCommit */, false /* mustBeRecreatable */, true /* canReturnEarliestCommit */); DeltaHistoryManager.Commit kernelCommit = snapshotManager.getActiveCommitAtTime( earlyTimestamp.getTime(), false /* canReturnLastCommit */, false /* mustBeRecreatable */, true /* canReturnEarliestCommit */); assertEquals(deltaCommit.version(), kernelCommit.getVersion()); assertEquals(deltaCommit.timestamp(), kernelCommit.getTimestamp()); } private static Stream checkVersionExistsTestCases() { return Stream.of( Arguments.of( "current", 10L /* versionToCheck */, true /* mustBeRecreatable */, false /* allowOutOfRange */, false /* shouldThrow */), Arguments.of( "notAllowOutOfRange", 21L /* versionToCheck */, true /* mustBeRecreatable */, false /* allowOutOfRange */, true /* shouldThrow */), Arguments.of( "allowOutOfRange", 21L /* versionToCheck */, true /* mustBeRecreatable */, true /* allowOutOfRange */, false /* shouldThrow */), Arguments.of( "belowEarliest", 1L /* versionToCheck */, true /* mustBeRecreatable */, false /* allowOutOfRange */, true /* shouldThrow */), Arguments.of( "mustBeRecreatable_false", 2L /* versionToCheck */, false /* mustBeRecreatable */, false /* allowOutOfRange */, false /* shouldThrow */), Arguments.of( "mustBeRecreatable_true", 2L /* versionToCheck */, true /* mustBeRecreatable */, false /* allowOutOfRange */, true /* shouldThrow */)); } @ParameterizedTest(name = "{0}") @MethodSource("checkVersionExistsTestCases") public void testCheckVersionExists( String testName, long versionToCheck, boolean mustBeRecreatable, boolean allowOutOfRange, boolean shouldThrow, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_version_" + testName; setupTableWithDeletedVersions(testTablePath, testTableName); snapshotManager = new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); if (shouldThrow) { assertThrows( VersionNotFoundException.class, () -> snapshotManager.checkVersionExists( versionToCheck, mustBeRecreatable, allowOutOfRange)); assertThrows( org.apache.spark.sql.delta.VersionNotFoundException.class, () -> deltaLog .history() .checkVersionExists( versionToCheck, Option.empty(), mustBeRecreatable, allowOutOfRange)); } else { snapshotManager.checkVersionExists(versionToCheck, mustBeRecreatable, allowOutOfRange); deltaLog .history() .checkVersionExists(versionToCheck, Option.empty(), mustBeRecreatable, allowOutOfRange); } } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/snapshot/unitycatalog/UCTableInfoTest.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.snapshot.unitycatalog; import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.HashMap; import java.util.Map; import org.junit.jupiter.api.Test; /** Tests for {@link UCTableInfo}. */ class UCTableInfoTest { @Test void testConstructor() { // Use distinctive values that would fail if implementation had hardcoded defaults String tableId = "uc_tbl_7f3a9b2c-e8d1-4f6a"; String tablePath = "abfss://container@acct.dfs.core.windows.net/delta/v2"; String ucUri = "https://uc-server.example.net/api/2.1/uc"; String ucToken = "dapi_Kx9mN$2pQr#7vWz"; Map authConfig = new HashMap<>(); authConfig.put("type", "static"); authConfig.put("token", ucToken); UCTableInfo info = new UCTableInfo(tableId, tablePath, ucUri, authConfig); assertEquals(tableId, info.getTableId(), "Table ID should be stored correctly"); assertEquals(tablePath, info.getTablePath(), "Table path should be stored correctly"); assertEquals(ucUri, info.getUcUri(), "UC URI should be stored correctly"); Map ret = info.getAuthConfig(); assertEquals("static", ret.get("type"), "Type should be static"); assertEquals(ucToken, ret.get("token"), "UC token should be stored correctly in configMap"); } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/utils/CloseableIteratorTest.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.utils; import static org.junit.jupiter.api.Assertions.*; import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import org.junit.jupiter.api.Test; import scala.collection.Iterator; import scala.collection.JavaConverters; public class CloseableIteratorTest { @Test public void testFilterMapAndClose() throws IOException { AtomicBoolean closed = new AtomicBoolean(false); Iterator base = new CloseableTestIterator<>(Arrays.asList(1, 2, 3, 4, 5, 6), closed); CloseableIterator iter = CloseableIterator.wrap(base).filterCloseable(x -> x % 2 == 0).mapCloseable(x -> "v" + x); List result = new ArrayList<>(); while (iter.hasNext()) { result.add(iter.next()); } assertEquals(List.of("v2", "v4", "v6"), result); assertFalse(closed.get()); iter.close(); assertTrue(closed.get()); } private static class CloseableTestIterator implements Iterator, Closeable { private final Iterator delegate; private final AtomicBoolean closed; CloseableTestIterator(List elements, AtomicBoolean closed) { this.delegate = JavaConverters.asScalaIterator(elements.iterator()); this.closed = closed; } @Override public boolean hasNext() { return delegate.hasNext(); } @Override public T next() { return delegate.next(); } @Override public void close() { closed.set(true); } } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.utils; import static org.junit.jupiter.api.Assertions.*; import io.delta.kernel.expressions.Literal; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.types.*; import java.math.BigDecimal; import java.util.HashSet; import java.util.Optional; import java.util.Set; import java.util.function.Supplier; import java.util.stream.Stream; import org.apache.spark.sql.connector.expressions.FieldReference; import org.apache.spark.sql.connector.expressions.LiteralValue; import org.apache.spark.sql.connector.expressions.NamedReference; import org.apache.spark.sql.sources.*; import org.apache.spark.unsafe.types.UTF8String; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; /** Tests for {@link ExpressionUtils}. */ public class ExpressionUtilsTest { // Test data provider for comparison filters static Stream comparisonFiltersProvider() { return Stream.of( Arguments.of("EqualTo", (Supplier) () -> new EqualTo("id", 42), "="), Arguments.of("GreaterThan", (Supplier) () -> new GreaterThan("age", 18), ">"), Arguments.of( "GreaterThanOrEqual", (Supplier) () -> new GreaterThanOrEqual("age", 18), ">="), Arguments.of("LessThan", (Supplier) () -> new LessThan("age", 65), "<"), Arguments.of( "LessThanOrEqual", (Supplier) () -> new LessThanOrEqual("age", 65), "<=")); } @ParameterizedTest(name = "{0} filter should be converted to {2}") @MethodSource("comparisonFiltersProvider") public void testComparisonFilters( String filterName, Supplier filterSupplier, String expectedOperator) { Filter filter = filterSupplier.get(); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(filter); assertTrue(result.isPresent(), filterName + " filter should be converted"); assertFalse(result.isPartial(), filterName + " filter should be fully converted"); assertEquals(expectedOperator, result.get().getName()); assertEquals(2, result.get().getChildren().size()); } // Test data provider for null filters static Stream nullFiltersProvider() { return Stream.of( Arguments.of("IsNull", (Supplier) () -> new IsNull("name"), "IS_NULL"), Arguments.of("IsNotNull", (Supplier) () -> new IsNotNull("name"), "IS_NOT_NULL")); } @ParameterizedTest(name = "{0} filter should be converted to {2}") @MethodSource("nullFiltersProvider") public void testNullFilters( String filterName, Supplier filterSupplier, String expectedOperator) { Filter filter = filterSupplier.get(); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(filter); assertTrue(result.isPresent(), filterName + " filter should be converted"); assertFalse(result.isPartial(), filterName + " filter should be fully converted"); assertEquals(expectedOperator, result.get().getName()); assertEquals(1, result.get().getChildren().size()); } @Test public void testEqualNullSafeFilter() { // Test EqualNullSafe with null value - converted to IS_NULL // Cannot use IS NOT DISTINCT FROM because kernel requires typed null literals EqualNullSafe nullFilter = new EqualNullSafe("name", null); ExpressionUtils.ConvertedPredicate nullResult = ExpressionUtils.convertSparkFilterToKernelPredicate(nullFilter); assertTrue(nullResult.isPresent(), "EqualNullSafe with null should be converted"); assertFalse(nullResult.isPartial(), "EqualNullSafe with null should be fully converted"); assertEquals("IS_NULL", nullResult.get().getName()); assertEquals(1, nullResult.get().getChildren().size()); // Test EqualNullSafe with non-null value - uses "=" operator EqualNullSafe nonNullFilter = new EqualNullSafe("id", 42); ExpressionUtils.ConvertedPredicate nonNullResult = ExpressionUtils.convertSparkFilterToKernelPredicate(nonNullFilter); assertTrue(nonNullResult.isPresent(), "EqualNullSafe with value should be converted"); assertFalse(nonNullResult.isPartial(), "EqualNullSafe with value should be fully converted"); assertEquals("=", nonNullResult.get().getName()); assertEquals(2, nonNullResult.get().getChildren().size()); } static Stream stringStartsWithProvider() { return Stream.of(Arguments.of("non-empty prefix", "Al"), Arguments.of("empty prefix", "")); } @ParameterizedTest(name = "StringStartsWith with {0} should be converted") @MethodSource("stringStartsWithProvider") public void testStringStartsWithFilter(String desc, String prefix) { StringStartsWith filter = new StringStartsWith("name", prefix); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(filter); assertTrue(result.isPresent(), "StringStartsWith filter should be converted"); assertFalse(result.isPartial(), "StringStartsWith filter should be fully converted"); assertEquals("STARTS_WITH", result.get().getName()); // Children: column + string literal assertEquals(2, result.get().getChildren().size()); } @Test public void testStringStartsWithFilter_NullValue() { // A null prefix cannot be converted — treated as unsupported, falls back to post-scan StringStartsWith filter = new StringStartsWith("name", null); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(filter); assertFalse(result.isPresent(), "StringStartsWith with null value should not be converted"); } @Test public void testInFilter_BasicConversion() { In filter = new In("city", new Object[] {"hz", "sh", "bj"}); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(filter); assertTrue(result.isPresent(), "In filter should be converted"); assertFalse(result.isPartial(), "In filter should be fully converted"); assertEquals("IN", result.get().getName()); // Children: column + 3 literals assertEquals(4, result.get().getChildren().size()); } @Test public void testInFilter_SingleValue() { In filter = new In("id", new Object[] {42}); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(filter); assertTrue(result.isPresent(), "In filter with single value should be converted"); assertFalse(result.isPartial(), "In filter should be fully converted"); assertEquals("IN", result.get().getName()); // Children: column + 1 literal assertEquals(2, result.get().getChildren().size()); } @Test public void testInFilter_EmptyValues() { In filter = new In("city", new Object[] {}); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(filter); // Empty IN list always evaluates to FALSE; push ALWAYS_FALSE so the kernel skips all files. assertTrue(result.isPresent(), "In filter with empty values should push ALWAYS_FALSE"); assertEquals("ALWAYS_FALSE", result.get().getName()); } @Test public void testInFilter_WithNullValue() { // null in the values array makes the IN expression unsafe to push down (SQL null semantics) In filter = new In("city", new Object[] {"hz", null, "bj"}); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(filter); assertFalse(result.isPresent(), "In filter with null value should not be pushed down"); } @Test public void testInFilter_WithUnsupportedType() { In filter = new In("col", new Object[] {42, new Object()}); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(filter); assertFalse(result.isPresent(), "In filter with unconvertible value should not be pushed down"); } @Test public void testInFilter_InAndFilter() { // AND(In(...), EqualTo(...)) — both convertible, should be fully pushed down In inFilter = new In("city", new Object[] {"hz", "sh"}); EqualTo eqFilter = new EqualTo("part", 1); org.apache.spark.sql.sources.And andFilter = new org.apache.spark.sql.sources.And(inFilter, eqFilter); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(andFilter); assertTrue(result.isPresent(), "AND(In, EqualTo) should be converted"); assertFalse(result.isPartial(), "AND(In, EqualTo) should be fully converted"); assertTrue( result.get() instanceof io.delta.kernel.expressions.And, "Result should be an AND predicate"); } @Test public void testInFilter_NotInWithNullValue() { // NOT(IN(col, 1, null)): null in the IN list causes IN to bail → NOT also bails In inFilter = new In("city", new Object[] {"hz", null}); Not notFilter = new Not(inFilter); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(notFilter); assertFalse( result.isPresent(), "NOT(IN(..., null)) should not be pushed down due to null in IN list"); } @Test public void testInFilter_ORWithUnsupportedFilter() { // OR(In(...), StringEndsWith(...)) — one branch unsupported, whole OR cannot be pushed In inFilter = new In("city", new Object[] {"hz", "sh"}); StringEndsWith endsWithFilter = new StringEndsWith("name", "foo"); Or orFilter = new Or(inFilter, endsWithFilter); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(orFilter); assertFalse( result.isPresent(), "OR(In, unsupported) should not be pushed down when one branch is unsupported"); } // Test data provider for parameterized literal conversion tests static Stream valueTypesProvider() { return Stream.of( // Primitive types Arguments.of("Boolean", true, BooleanType.BOOLEAN), Arguments.of("Byte", (byte) 42, ByteType.BYTE), Arguments.of("Short", (short) 1000, ShortType.SHORT), Arguments.of("Integer", 12345, IntegerType.INTEGER), Arguments.of("Long", 123456789L, LongType.LONG), Arguments.of("Float", 3.14f, FloatType.FLOAT), Arguments.of("Double", 2.718281828, DoubleType.DOUBLE), // BigDecimal - precision=6, scale=3 for "123.456" Arguments.of("BigDecimal", new BigDecimal("123.456"), new DecimalType(6, 3)), // String type Arguments.of("String", "hello world", StringType.STRING), Arguments.of("UTF8String", UTF8String.fromString("hello world"), StringType.STRING), // Binary data Arguments.of("byte[]", new byte[] {1, 2, 3, 4, 5}, BinaryType.BINARY), // Date/time types (java.sql types for V1 Filters) Arguments.of("java.sql.Date", java.sql.Date.valueOf("2023-01-15"), DateType.DATE), Arguments.of( "java.sql.Timestamp", java.sql.Timestamp.valueOf("2023-01-15 10:30:00"), TimestampType.TIMESTAMP)); } @Test public void testUnsupportedFilter() { // Create an unsupported filter (StringContains is not implemented in our conversion method) Filter unsupportedFilter = new StringContains("col1", "test"); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(unsupportedFilter); assertFalse(result.isPresent(), "Unsupported filters should return empty Optional"); assertFalse(result.isPartial(), "Unsupported filters should not be marked as partial"); } @Test public void testAndFilter() { EqualTo leftFilter = new EqualTo("id", 1); GreaterThan rightFilter = new GreaterThan("age", 18); org.apache.spark.sql.sources.And andFilter = new org.apache.spark.sql.sources.And(leftFilter, rightFilter); ExpressionUtils.ConvertedPredicate andResult = ExpressionUtils.convertSparkFilterToKernelPredicate(andFilter); assertTrue(andResult.isPresent(), "And filter should be converted"); assertFalse(andResult.isPartial(), "And filter should be fully converted"); assertTrue( andResult.get() instanceof io.delta.kernel.expressions.And, "Result should be And predicate"); assertEquals(2, andResult.get().getChildren().size()); } @Test public void testAndFilter_PartialPushDownWithLeftConvertible() { // Create an AND filter where left can be converted but right cannot EqualTo leftFilter = new EqualTo("id", 1); Filter unsupportedRightFilter = new StringContains("unsupported_col", "test"); org.apache.spark.sql.sources.And andFilter = new org.apache.spark.sql.sources.And(leftFilter, unsupportedRightFilter); // Without partial pushdown - should return empty ExpressionUtils.ConvertedPredicate resultWithoutPartial = ExpressionUtils.convertSparkFilterToKernelPredicate(andFilter, false); assertFalse( resultWithoutPartial.isPresent(), "AND filter with unconvertible operand should return empty without partial pushdown"); assertFalse(resultWithoutPartial.isPartial(), "Empty result should not be marked as partial"); // With partial pushdown - should return the convertible part ExpressionUtils.ConvertedPredicate resultWithPartial = ExpressionUtils.convertSparkFilterToKernelPredicate(andFilter, true); assertTrue( resultWithPartial.isPresent(), "AND filter with partial pushdown should return the convertible operand"); assertTrue( resultWithPartial.isPartial(), "AND filter with partial pushdown should be marked as partial"); assertEquals("=", resultWithPartial.get().getName()); assertEquals(2, resultWithPartial.get().getChildren().size()); } @Test public void testAndFilter_PartialPushDownWithRightConvertible() { // Create an AND filter where right can be converted but left cannot Filter unsupportedLeftFilter = new StringContains("unsupported_col", "test"); GreaterThan rightFilter = new GreaterThan("age", 18); org.apache.spark.sql.sources.And andFilter = new org.apache.spark.sql.sources.And(unsupportedLeftFilter, rightFilter); // With partial pushdown - should return the convertible part (right side) ExpressionUtils.ConvertedPredicate resultWithPartial = ExpressionUtils.convertSparkFilterToKernelPredicate(andFilter, true); assertTrue(resultWithPartial.isPresent(), "AND filter should return the convertible operand"); assertTrue(resultWithPartial.isPartial(), "AND filter should be marked as partial"); assertEquals(">", resultWithPartial.get().getName()); assertEquals(2, resultWithPartial.get().getChildren().size()); // Without partial pushdown - should return empty ExpressionUtils.ConvertedPredicate resultWithoutPartial = ExpressionUtils.convertSparkFilterToKernelPredicate(andFilter, false); assertFalse( resultWithoutPartial.isPresent(), "AND filter should return empty without partial pushdown"); assertFalse(resultWithoutPartial.isPartial(), "Empty result should not be marked as partial"); } @Test public void testAndFilter_PartialPushDown_BothUnconvertible() { // Create an AND filter where neither side can be converted Filter unsupportedLeftFilter = new StringContains("unsupported_col1", "test"); Filter unsupportedRightFilter = new StringContains("unsupported_col2", "test"); org.apache.spark.sql.sources.And andFilter = new org.apache.spark.sql.sources.And(unsupportedLeftFilter, unsupportedRightFilter); ExpressionUtils.ConvertedPredicate resultWithPartial = ExpressionUtils.convertSparkFilterToKernelPredicate(andFilter); assertFalse( resultWithPartial.isPresent(), "AND filter should return empty if both operands are unconvertible"); assertFalse(resultWithPartial.isPartial(), "Empty result should not be marked as partial"); } @Test public void testOrFilter_RequiresBothConvertible() { // Create an OR filter where left can be converted but right cannot EqualTo leftFilter = new EqualTo("id", 1); Filter unsupportedRightFilter = new StringContains("unsupported_col", "test"); org.apache.spark.sql.sources.Or orFilter = new org.apache.spark.sql.sources.Or(leftFilter, unsupportedRightFilter); ExpressionUtils.ConvertedPredicate resultWithPartial = ExpressionUtils.convertSparkFilterToKernelPredicate(orFilter); assertFalse( resultWithPartial.isPresent(), "OR filter with unconvertible operand should return empty even with partial pushdown"); assertFalse(resultWithPartial.isPartial(), "Empty result should not be marked as partial"); } @Test public void testNotFilter() { EqualTo leftFilter = new EqualTo("id", 1); Not notFilter = new Not(leftFilter); ExpressionUtils.ConvertedPredicate notResult = ExpressionUtils.convertSparkFilterToKernelPredicate(notFilter); assertTrue(notResult.isPresent(), "Not filter should be converted"); assertFalse(notResult.isPartial(), "Not filter should be fully converted"); assertEquals("NOT", notResult.get().getName()); assertEquals(1, notResult.get().getChildren().size()); } @Test public void testNotFilter_RequiresChildConvertible() { // StringContains is not yet supported Filter unsupportedFilter = new StringContains("unsupported_col", "test"); Not notFilter = new Not(unsupportedFilter); // NOT requires child to be convertible. ExpressionUtils.ConvertedPredicate resultWithoutPartial = ExpressionUtils.convertSparkFilterToKernelPredicate(notFilter); assertFalse( resultWithoutPartial.isPresent(), "NOT filter with unconvertible child should return empty"); assertFalse(resultWithoutPartial.isPartial(), "Empty result should not be marked as partial"); // Create NOT(A AND B) where A is convertible but B is not // This tests that NOT disables partial pushdown for semantic correctness EqualTo convertibleFilter = new EqualTo("id", 1); org.apache.spark.sql.sources.And andFilter = new org.apache.spark.sql.sources.And(convertibleFilter, unsupportedFilter); // Now verify that NOT(AND) returns empty because NOT disables partial pushdown Not notAndFilter = new Not(andFilter); ExpressionUtils.ConvertedPredicate notResult = ExpressionUtils.convertSparkFilterToKernelPredicate(notAndFilter); assertFalse( notResult.isPresent(), "NOT(A AND B) should return empty when B is unconvertible, even with partial pushdown enabled" + " - this preserves semantic correctness as NOT(A AND B) != NOT(A)"); assertFalse(notResult.isPartial(), "Empty result should not be marked as partial"); } @ParameterizedTest(name = "convertValueToKernelLiteral should support {0}") @MethodSource("valueTypesProvider") public void testConvertValueToKernelLiteral( String typeName, Object value, DataType expectedDataType) { Optional result = ExpressionUtils.convertValueToKernelLiteral(value); assertTrue(result.isPresent(), "Value of type " + typeName + " should be convertible"); Literal literal = result.get(); assertNotNull(literal, "Literal should not be null"); assertNotNull(literal.getDataType(), "DataType should not be null"); assertEquals( expectedDataType, literal.getDataType(), "DataType should match expected type for " + typeName); } @Test public void testConvertValueToKernelLiteral_NullValue() { Optional result = ExpressionUtils.convertValueToKernelLiteral(null); assertFalse(result.isPresent(), "null values should return empty Optional"); } @Test public void testConvertValueToKernelLiteral_UnsupportedType() { // Test with an unsupported type like a custom object Object unsupportedValue = new Object(); Optional result = ExpressionUtils.convertValueToKernelLiteral(unsupportedValue); assertFalse(result.isPresent(), "Unsupported types should return empty Optional"); } @Test public void testNestedFieldParsing() { EqualTo nestedFieldFilter = new EqualTo("user.profile.name", "John"); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(nestedFieldFilter); assertTrue(result.isPresent(), "Nested field filter should be convertible"); assertFalse(result.isPartial(), "Nested field filter should be fully convertible"); Predicate predicate = result.get(); io.delta.kernel.expressions.Column column = (io.delta.kernel.expressions.Column) predicate.getChildren().get(0); assertArrayEquals( new String[] {"user", "profile", "name"}, column.getNames(), "Nested field names should be parsed correctly"); } @Test public void testSingleColumnNameWithDots() { EqualTo singleColumnFilter = new EqualTo("`user.profile.name`", "value"); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(singleColumnFilter); assertTrue(result.isPresent(), "Single column filter should be convertible"); assertFalse(result.isPartial(), "Single column filter should be fully convertible"); Predicate predicate = result.get(); io.delta.kernel.expressions.Column column = (io.delta.kernel.expressions.Column) predicate.getChildren().get(0); assertArrayEquals( new String[] {"user.profile.name"}, column.getNames(), "Single column name with dots should be preserved as-is"); } // Tests for dsv2PredicateToCatalystExpression private final org.apache.spark.sql.types.StructType testSchema = new org.apache.spark.sql.types.StructType() .add("id", org.apache.spark.sql.types.DataTypes.IntegerType, false) .add("name", org.apache.spark.sql.types.DataTypes.StringType, true) .add("age", org.apache.spark.sql.types.DataTypes.IntegerType, true); @Test public void testDsv2PredicateToCatalystExpression_IsNull() { NamedReference nameRef = FieldReference.apply("name"); org.apache.spark.sql.connector.expressions.filter.Predicate isNullPredicate = new org.apache.spark.sql.connector.expressions.filter.Predicate( "IS_NULL", new org.apache.spark.sql.connector.expressions.Expression[] {nameRef}); Optional result = ExpressionUtils.dsv2PredicateToCatalystExpression(isNullPredicate, testSchema); assertTrue(result.isPresent(), "Result should be present"); assertTrue( result.get() instanceof org.apache.spark.sql.catalyst.expressions.IsNull, "Result should be IsNull expression"); } @Test public void testDsv2PredicateToCatalystExpression_IsNotNull() { NamedReference nameRef = FieldReference.apply("name"); org.apache.spark.sql.connector.expressions.filter.Predicate isNotNullPredicate = new org.apache.spark.sql.connector.expressions.filter.Predicate( "IS_NOT_NULL", new org.apache.spark.sql.connector.expressions.Expression[] {nameRef}); Optional result = ExpressionUtils.dsv2PredicateToCatalystExpression(isNotNullPredicate, testSchema); assertTrue(result.isPresent(), "Result should be present"); assertTrue( result.get() instanceof org.apache.spark.sql.catalyst.expressions.IsNotNull, "Result should be IsNotNull expression"); } @Test public void testDsv2PredicateToCatalystExpression_EqualTo() { NamedReference idRef = FieldReference.apply("id"); LiteralValue value = LiteralValue.apply(42, org.apache.spark.sql.types.DataTypes.IntegerType); org.apache.spark.sql.connector.expressions.filter.Predicate equalToPredicate = new org.apache.spark.sql.connector.expressions.filter.Predicate( "=", new org.apache.spark.sql.connector.expressions.Expression[] {idRef, value}); Optional result = ExpressionUtils.dsv2PredicateToCatalystExpression(equalToPredicate, testSchema); assertTrue(result.isPresent(), "Result should be present"); assertTrue( result.get() instanceof org.apache.spark.sql.catalyst.expressions.EqualTo, "Result should be EqualTo expression"); } @Test public void testDsv2PredicateToCatalystExpression_LessThan() { NamedReference ageRef = FieldReference.apply("age"); LiteralValue value = LiteralValue.apply(30, org.apache.spark.sql.types.DataTypes.IntegerType); org.apache.spark.sql.connector.expressions.filter.Predicate lessThanPredicate = new org.apache.spark.sql.connector.expressions.filter.Predicate( "<", new org.apache.spark.sql.connector.expressions.Expression[] {ageRef, value}); Optional result = ExpressionUtils.dsv2PredicateToCatalystExpression(lessThanPredicate, testSchema); assertTrue(result.isPresent(), "Result should be present"); assertTrue( result.get() instanceof org.apache.spark.sql.catalyst.expressions.LessThan, "Result should be LessThan expression"); } @Test public void testDsv2PredicateToCatalystExpression_GreaterThanOrEqual() { NamedReference ageRef = FieldReference.apply("age"); LiteralValue value = LiteralValue.apply(18, org.apache.spark.sql.types.DataTypes.IntegerType); org.apache.spark.sql.connector.expressions.filter.Predicate greaterThanOrEqualPredicate = new org.apache.spark.sql.connector.expressions.filter.Predicate( ">=", new org.apache.spark.sql.connector.expressions.Expression[] {ageRef, value}); Optional result = ExpressionUtils.dsv2PredicateToCatalystExpression(greaterThanOrEqualPredicate, testSchema); assertTrue(result.isPresent(), "Result should be present"); assertTrue( result.get() instanceof org.apache.spark.sql.catalyst.expressions.GreaterThanOrEqual, "Result should be GreaterThanOrEqual expression"); } @Test public void testDsv2PredicateToCatalystExpression_In() { NamedReference idRef = FieldReference.apply("id"); LiteralValue val1 = LiteralValue.apply(1, org.apache.spark.sql.types.DataTypes.IntegerType); LiteralValue val2 = LiteralValue.apply(2, org.apache.spark.sql.types.DataTypes.IntegerType); LiteralValue val3 = LiteralValue.apply(3, org.apache.spark.sql.types.DataTypes.IntegerType); org.apache.spark.sql.connector.expressions.filter.Predicate inPredicate = new org.apache.spark.sql.connector.expressions.filter.Predicate( "IN", new org.apache.spark.sql.connector.expressions.Expression[] {idRef, val1, val2, val3}); Optional result = ExpressionUtils.dsv2PredicateToCatalystExpression(inPredicate, testSchema); assertTrue(result.isPresent(), "Result should be present"); assertTrue( result.get() instanceof org.apache.spark.sql.catalyst.expressions.In, "Result should be In expression"); org.apache.spark.sql.catalyst.expressions.In inExpr = (org.apache.spark.sql.catalyst.expressions.In) result.get(); assertEquals(3, inExpr.list().size(), "IN expression should have 3 values"); } @Test public void testDsv2PredicateToCatalystExpression_And() { NamedReference ageRef = FieldReference.apply("age"); LiteralValue value1 = LiteralValue.apply(18, org.apache.spark.sql.types.DataTypes.IntegerType); LiteralValue value2 = LiteralValue.apply(65, org.apache.spark.sql.types.DataTypes.IntegerType); org.apache.spark.sql.connector.expressions.filter.Predicate leftPredicate = new org.apache.spark.sql.connector.expressions.filter.Predicate( ">", new org.apache.spark.sql.connector.expressions.Expression[] {ageRef, value1}); org.apache.spark.sql.connector.expressions.filter.Predicate rightPredicate = new org.apache.spark.sql.connector.expressions.filter.Predicate( "<", new org.apache.spark.sql.connector.expressions.Expression[] {ageRef, value2}); org.apache.spark.sql.connector.expressions.filter.Predicate andPredicate = new org.apache.spark.sql.connector.expressions.filter.Predicate( "AND", new org.apache.spark.sql.connector.expressions.Expression[] { leftPredicate, rightPredicate }); Optional result = ExpressionUtils.dsv2PredicateToCatalystExpression(andPredicate, testSchema); assertTrue(result.isPresent(), "Result should be present"); assertTrue( result.get() instanceof org.apache.spark.sql.catalyst.expressions.And, "Result should be And expression"); } @Test public void testDsv2PredicateToCatalystExpression_Or() { NamedReference ageRef = FieldReference.apply("age"); LiteralValue value1 = LiteralValue.apply(18, org.apache.spark.sql.types.DataTypes.IntegerType); LiteralValue value2 = LiteralValue.apply(65, org.apache.spark.sql.types.DataTypes.IntegerType); org.apache.spark.sql.connector.expressions.filter.Predicate leftPredicate = new org.apache.spark.sql.connector.expressions.filter.Predicate( "<", new org.apache.spark.sql.connector.expressions.Expression[] {ageRef, value1}); org.apache.spark.sql.connector.expressions.filter.Predicate rightPredicate = new org.apache.spark.sql.connector.expressions.filter.Predicate( ">", new org.apache.spark.sql.connector.expressions.Expression[] {ageRef, value2}); org.apache.spark.sql.connector.expressions.filter.Predicate orPredicate = new org.apache.spark.sql.connector.expressions.filter.Predicate( "OR", new org.apache.spark.sql.connector.expressions.Expression[] { leftPredicate, rightPredicate }); Optional result = ExpressionUtils.dsv2PredicateToCatalystExpression(orPredicate, testSchema); assertTrue(result.isPresent(), "Result should be present"); assertTrue( result.get() instanceof org.apache.spark.sql.catalyst.expressions.Or, "Result should be Or expression"); } @Test public void testDsv2PredicateToCatalystExpression_Not() { NamedReference ageRef = FieldReference.apply("age"); LiteralValue value = LiteralValue.apply(18, org.apache.spark.sql.types.DataTypes.IntegerType); org.apache.spark.sql.connector.expressions.filter.Predicate childPredicate = new org.apache.spark.sql.connector.expressions.filter.Predicate( "<", new org.apache.spark.sql.connector.expressions.Expression[] {ageRef, value}); org.apache.spark.sql.connector.expressions.filter.Predicate notPredicate = new org.apache.spark.sql.connector.expressions.filter.Predicate( "NOT", new org.apache.spark.sql.connector.expressions.Expression[] {childPredicate}); Optional result = ExpressionUtils.dsv2PredicateToCatalystExpression(notPredicate, testSchema); assertTrue(result.isPresent(), "Result should be present"); assertTrue( result.get() instanceof org.apache.spark.sql.catalyst.expressions.Not, "Result should be Not expression"); } @Test public void testDsv2PredicateToCatalystExpression_AlwaysTrue() { org.apache.spark.sql.connector.expressions.filter.Predicate alwaysTruePredicate = new org.apache.spark.sql.connector.expressions.filter.Predicate( "ALWAYS_TRUE", new org.apache.spark.sql.connector.expressions.Expression[] {}); Optional result = ExpressionUtils.dsv2PredicateToCatalystExpression(alwaysTruePredicate, testSchema); assertTrue(result.isPresent(), "Result should be present"); assertTrue( result.get() instanceof org.apache.spark.sql.catalyst.expressions.Literal, "Result should be Literal expression"); org.apache.spark.sql.catalyst.expressions.Literal literal = (org.apache.spark.sql.catalyst.expressions.Literal) result.get(); assertEquals(true, literal.value(), "ALWAYS_TRUE should return literal true"); } @Test public void testDsv2PredicateToCatalystExpression_UnsupportedPredicate() { org.apache.spark.sql.connector.expressions.filter.Predicate unsupportedPredicate = new org.apache.spark.sql.connector.expressions.filter.Predicate( "UNSUPPORTED_OPERATOR", new org.apache.spark.sql.connector.expressions.Expression[] {}); Optional result = ExpressionUtils.dsv2PredicateToCatalystExpression(unsupportedPredicate, testSchema); assertFalse(result.isPresent(), "Unsupported predicates should return empty Optional"); } // ===== Tests for decimal type alignment with table schema ===== private final org.apache.spark.sql.types.StructType decimalSchema = new org.apache.spark.sql.types.StructType() .add("price", new org.apache.spark.sql.types.DecimalType(7, 2), true) .add("quantity", org.apache.spark.sql.types.DataTypes.IntegerType, true); @Test public void testDecimalLiteralWidenedToColumnType() { // Literal 100.00 has Decimal(5,2), column is Decimal(7,2) → should widen to Decimal(7,2) EqualTo filter = new EqualTo("price", new BigDecimal("100.00")); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); assertTrue(result.isPresent(), "Decimal filter should be converted with widened type"); assertFalse(result.isPartial()); Predicate pred = result.get(); assertEquals("=", pred.getName()); Literal literal = (Literal) pred.getChildren().get(1); assertEquals(new DecimalType(7, 2), literal.getDataType()); } @Test public void testDecimalLiteralScaleWidened() { // Literal 100 has Decimal(3,0), column is Decimal(7,2) → should widen to Decimal(7,2) EqualTo filter = new EqualTo("price", new BigDecimal("100")); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); assertTrue(result.isPresent(), "Decimal filter with lower scale should be widened"); Literal literal = (Literal) result.get().getChildren().get(1); assertEquals(new DecimalType(7, 2), literal.getDataType()); assertEquals(new BigDecimal("100.00"), literal.getValue()); } @Test public void testDecimalLiteralHigherScaleThanColumn() { // Literal 99.999 has scale=3, column is Decimal(7,2) → scale exceeds column, skip pushdown GreaterThan filter = new GreaterThan("price", new BigDecimal("99.999")); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); assertFalse( result.isPresent(), "Decimal literal with higher scale than column should not be pushed down"); } @Test public void testDecimalLiteralExceedsColumnPrecision() { // Literal 123456.00 has 6 integral digits + 2 scale = precision 8, // column is Decimal(7,2) which holds max 99999.99 → skip pushdown LessThan filter = new LessThan("price", new BigDecimal("123456.00")); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); assertFalse( result.isPresent(), "Decimal literal exceeding column precision should not be pushed down"); } @Test public void testDecimalLiteralMatchingType() { // Literal already matches column type Decimal(7,2) → no widening needed EqualTo filter = new EqualTo("price", new BigDecimal("12345.67")); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); assertTrue(result.isPresent(), "Decimal filter with matching type should be converted"); Literal literal = (Literal) result.get().getChildren().get(1); assertEquals(new DecimalType(7, 2), literal.getDataType()); } @Test public void testDecimalLiteralWithoutSchema() { // Without schema, decimal literal retains its intrinsic type EqualTo filter = new EqualTo("price", new BigDecimal("100.00")); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(filter); assertTrue(result.isPresent(), "Decimal filter without schema should still be converted"); Literal literal = (Literal) result.get().getChildren().get(1); // 100.00 has precision=5, scale=2 assertEquals(new DecimalType(5, 2), literal.getDataType()); } @Test public void testDecimalLiteralNonDecimalColumn() { // Column "quantity" is IntegerType, not DecimalType → use default conversion EqualTo filter = new EqualTo("quantity", new BigDecimal("42")); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); assertTrue(result.isPresent(), "BigDecimal for non-decimal column should use default type"); Literal literal = (Literal) result.get().getChildren().get(1); // Default BigDecimal conversion: precision=2, scale=0 assertEquals(new DecimalType(2, 0), literal.getDataType()); } @Test public void testDecimalLiteralCaseInsensitiveColumnLookup() { // Filter uses "PRICE" but schema has "price" → should still widen EqualTo filter = new EqualTo("PRICE", new BigDecimal("100.00")); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); assertTrue(result.isPresent(), "Case-insensitive column lookup should match"); Literal literal = (Literal) result.get().getChildren().get(1); assertEquals(new DecimalType(7, 2), literal.getDataType()); } @Test public void testDecimalLiteralColumnNotInSchema() { // Column "unknown" not in schema → use default conversion EqualTo filter = new EqualTo("unknown", new BigDecimal("100.00")); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); assertTrue(result.isPresent(), "Unknown column should fall back to default conversion"); Literal literal = (Literal) result.get().getChildren().get(1); assertEquals(new DecimalType(5, 2), literal.getDataType()); } @Test public void testDecimalPartialPushDownInAndFilter() { // AND(price > 99.999, price < 200.00) where left has scale=3 exceeding column's scale=2. // Only the right side should be pushed down (partial pushdown). GreaterThan left = new GreaterThan("price", new BigDecimal("99.999")); LessThan right = new LessThan("price", new BigDecimal("200.00")); org.apache.spark.sql.sources.And andFilter = new org.apache.spark.sql.sources.And(left, right); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(andFilter, decimalSchema); assertTrue(result.isPresent(), "AND filter should partially push down the valid side"); assertTrue(result.isPartial(), "Result should be marked as partial conversion"); // Only the right side (price < 200.00) should be pushed, not a compound AND Predicate pred = result.get(); assertNotEquals( "AND", pred.getName(), "Left side (price > 99.999) should be dropped, not pushed as compound AND"); assertEquals("<", pred.getName()); Literal literal = (Literal) pred.getChildren().get(1); assertEquals(new DecimalType(7, 2), literal.getDataType()); assertEquals(new BigDecimal("200.00"), literal.getValue()); } @Test public void testDecimalLiteralInCompoundFilter() { // AND(price >= 100.00, price <= 200.00) with Decimal(7,2) column GreaterThanOrEqual left = new GreaterThanOrEqual("price", new BigDecimal("100.00")); LessThanOrEqual right = new LessThanOrEqual("price", new BigDecimal("200.00")); org.apache.spark.sql.sources.And andFilter = new org.apache.spark.sql.sources.And(left, right); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(andFilter, decimalSchema); assertTrue(result.isPresent(), "AND filter with decimal operands should be converted"); assertFalse(result.isPartial()); // Verify both literals are widened to Decimal(7,2) io.delta.kernel.expressions.And andPred = (io.delta.kernel.expressions.And) result.get(); Predicate leftPred = (Predicate) andPred.getChildren().get(0); Predicate rightPred = (Predicate) andPred.getChildren().get(1); assertEquals(new DecimalType(7, 2), ((Literal) leftPred.getChildren().get(1)).getDataType()); assertEquals(new DecimalType(7, 2), ((Literal) rightPred.getChildren().get(1)).getDataType()); } @Test public void testDecimalLiteralInOrFilter() { // OR(price >= 100.00, price >= 200.00) with Decimal(7,2) column GreaterThanOrEqual left = new GreaterThanOrEqual("price", new BigDecimal("100.00")); GreaterThanOrEqual right = new GreaterThanOrEqual("price", new BigDecimal("200.00")); org.apache.spark.sql.sources.Or orFilter = new org.apache.spark.sql.sources.Or(left, right); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(orFilter, decimalSchema); assertTrue(result.isPresent(), "OR filter with decimal operands should be converted"); } @Test public void testDecimalLiteralInNotFilter() { // NOT(price = 100.00) with Decimal(7,2) column EqualTo eq = new EqualTo("price", new BigDecimal("100.00")); Not notFilter = new Not(eq); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(notFilter, decimalSchema); assertTrue(result.isPresent(), "NOT filter with decimal operand should be converted"); Predicate notPred = result.get(); assertEquals("NOT", notPred.getName()); Predicate innerPred = (Predicate) notPred.getChildren().get(0); Literal literal = (Literal) innerPred.getChildren().get(1); assertEquals(new DecimalType(7, 2), literal.getDataType()); } @Test public void testDecimalLiteralWithEqualNullSafe() { // EqualNullSafe(price, 100.00) with Decimal(7,2) column EqualNullSafe filter = new EqualNullSafe("price", new BigDecimal("100.00")); ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); assertTrue(result.isPresent(), "EqualNullSafe with decimal should be converted"); Literal literal = (Literal) result.get().getChildren().get(1); assertEquals(new DecimalType(7, 2), literal.getDataType()); } @Test public void testDecimalLiteralAllComparisonOperators() { // Test all comparison operators widen decimals correctly BigDecimal value = new BigDecimal("50.00"); // Decimal(4,2) → should widen to Decimal(7,2) Filter[] filters = new Filter[] { new EqualTo("price", value), new GreaterThan("price", value), new GreaterThanOrEqual("price", value), new LessThan("price", value), new LessThanOrEqual("price", value), }; String[] expectedOps = new String[] {"=", ">", ">=", "<", "<="}; for (int i = 0; i < filters.length; i++) { ExpressionUtils.ConvertedPredicate result = ExpressionUtils.convertSparkFilterToKernelPredicate(filters[i], decimalSchema); assertTrue(result.isPresent(), expectedOps[i] + " filter should be converted"); Literal literal = (Literal) result.get().getChildren().get(1); assertEquals( new DecimalType(7, 2), literal.getDataType(), expectedOps[i] + " should widen decimal to column type"); } } @Test public void testClassifyFilterWithNullSchemaMatchesTwoArgOverload() { // Directly validates that classifyFilter(filter, partitionColumns, null) produces // the same result as the 2-arg classifyFilter(filter, partitionColumns). Set partitionColumns = new HashSet<>(); partitionColumns.add("dep_id"); EqualTo filter = new EqualTo("price", new BigDecimal("100.00")); ExpressionUtils.FilterClassificationResult resultTwoArg = ExpressionUtils.classifyFilter(filter, partitionColumns); ExpressionUtils.FilterClassificationResult resultThreeArg = ExpressionUtils.classifyFilter(filter, partitionColumns, null); assertEquals(resultTwoArg.isKernelSupported, resultThreeArg.isKernelSupported); assertEquals(resultTwoArg.isPartialConversion, resultThreeArg.isPartialConversion); assertEquals(resultTwoArg.isDataFilter, resultThreeArg.isDataFilter); assertEquals(resultTwoArg.kernelPredicate, resultThreeArg.kernelPredicate); } @Test public void testDsv2PredicateToCatalystExpression_ColumnNotFound() { NamedReference invalidRef = FieldReference.apply("nonexistent_column"); LiteralValue value = LiteralValue.apply(42, org.apache.spark.sql.types.DataTypes.IntegerType); org.apache.spark.sql.connector.expressions.filter.Predicate predicate = new org.apache.spark.sql.connector.expressions.filter.Predicate( "=", new org.apache.spark.sql.connector.expressions.Expression[] {invalidRef, value}); Optional result = ExpressionUtils.dsv2PredicateToCatalystExpression(predicate, testSchema); assertFalse( result.isPresent(), "Should return empty Optional when column is not found in schema"); } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/utils/PartitionUtilsTest.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.utils; import static io.delta.kernel.internal.util.VectorUtils.stringStringMapValue; import static org.junit.jupiter.api.Assertions.*; import io.delta.kernel.Scan; import io.delta.kernel.Snapshot; import io.delta.kernel.Table; import io.delta.kernel.data.FilteredColumnarBatch; import io.delta.kernel.data.MapValue; import io.delta.kernel.data.Row; import io.delta.kernel.internal.actions.AddFile; import io.delta.kernel.utils.CloseableIterator; import io.delta.spark.internal.v2.DeltaV2TestBase; import java.time.ZoneId; import java.util.HashMap; import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.connector.read.PartitionReaderFactory; import org.apache.spark.sql.execution.datasources.PartitionedFile; import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.sources.Filter; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.junit.jupiter.api.Test; import scala.collection.immutable.Map$; public class PartitionUtilsTest extends DeltaV2TestBase { private static final long MB = 1024 * 1024; @Test public void testGetPartitionRow_FieldOrdering() { // Schema defines order: year, month, day StructType partitionSchema = new StructType( new StructField[] { DataTypes.createStructField("year", DataTypes.IntegerType, true), DataTypes.createStructField("month", DataTypes.IntegerType, true), DataTypes.createStructField("day", DataTypes.IntegerType, true) }); // map value has different order: day, year, month Map partitionValues = new HashMap<>(); partitionValues.put("day", "25"); partitionValues.put("year", "2024"); partitionValues.put("month", "11"); MapValue mapValue = stringStringMapValue(partitionValues); InternalRow row = PartitionUtils.getPartitionRow(mapValue, partitionSchema, ZoneId.of("UTC")); // verify order is schema order: year, month, day assertEquals(2024, row.getInt(0)); assertEquals(11, row.getInt(1)); assertEquals(25, row.getInt(2)); } @Test public void testGetPartitionRow_SizeMismatchExtraKeys() { StructType partitionSchema = new StructType( new StructField[] { DataTypes.createStructField("year", DataTypes.IntegerType, true), DataTypes.createStructField("month", DataTypes.IntegerType, true) }); Map partitionValues = new HashMap<>(); partitionValues.put("year", "2024"); partitionValues.put("month", "11"); partitionValues.put("day", "25"); partitionValues.put("hour", "10"); MapValue mapValue = stringStringMapValue(partitionValues); assertThrows( AssertionError.class, () -> PartitionUtils.getPartitionRow(mapValue, partitionSchema, ZoneId.of("UTC"))); } @Test public void testGetPartitionRow_SizeMismatchMissingKeys() { StructType partitionSchema = new StructType( new StructField[] { DataTypes.createStructField("year", DataTypes.IntegerType, true), DataTypes.createStructField("month", DataTypes.IntegerType, true), DataTypes.createStructField("day", DataTypes.IntegerType, true) }); Map partitionValues = new HashMap<>(); partitionValues.put("year", "2024"); partitionValues.put("month", "11"); MapValue mapValue = stringStringMapValue(partitionValues); assertThrows( AssertionError.class, () -> PartitionUtils.getPartitionRow(mapValue, partitionSchema, ZoneId.of("UTC"))); } @Test public void testCreateDeltaParquetReaderFactory_Basic() { String tablePath = createTestTable("test_delta_reader_factory_" + System.nanoTime(), true); Table table = Table.forPath(defaultEngine, tablePath); Snapshot snapshot = table.getLatestSnapshot(defaultEngine); StructType dataSchema = new StructType( new StructField[] { DataTypes.createStructField("id", DataTypes.LongType, true), }); StructType partitionSchema = new StructType( new StructField[] {DataTypes.createStructField("part", DataTypes.StringType, true)}); StructType readDataSchema = dataSchema; Filter[] filters = new Filter[0]; scala.collection.immutable.Map options = Map$.MODULE$.empty(); Configuration hadoopConf = new Configuration(); SQLConf sqlConf = SQLConf.get(); PartitionReaderFactory factory = PartitionUtils.createDeltaParquetReaderFactory( snapshot, dataSchema, partitionSchema, readDataSchema, filters, options, hadoopConf, sqlConf); assertNotNull(factory, "PartitionReaderFactory should not be null"); } @Test public void testCalculateMaxSplitBytes_Basic() { SQLConf sqlConf = SQLConf.get(); long minPartitionNum = 4; sqlConf.setConfString("spark.sql.files.minPartitionNum", String.valueOf(minPartitionNum)); long totalBytes = 100 * MB; int fileCount = 10; long result = PartitionUtils.calculateMaxSplitBytes(spark, totalBytes, fileCount, sqlConf); long openCostInBytes = sqlConf.filesOpenCostInBytes(); long maxPartitionBytes = sqlConf.filesMaxPartitionBytes(); long calculatedTotalBytes = totalBytes + (long) fileCount * openCostInBytes; assertEquals(calculatedTotalBytes / minPartitionNum, result); } @Test public void testCalculateMaxSplitBytes_BoundaryConditions() { SQLConf sqlConf = SQLConf.get(); // Set minPartitionNum=1 for predictable calculations sqlConf.setConfString("spark.sql.files.minPartitionNum", "1"); long openCostInBytes = sqlConf.filesOpenCostInBytes(); long maxPartitionBytes = sqlConf.filesMaxPartitionBytes(); // Zero files and bytes long result1 = PartitionUtils.calculateMaxSplitBytes(spark, 0L, 0, sqlConf); assertEquals(openCostInBytes, result1); // Single large file (exceeds maxPartitionBytes) long result2 = PartitionUtils.calculateMaxSplitBytes(spark, 1000 * MB, 1, sqlConf); assertEquals(maxPartitionBytes, result2); // Very small totalBytes long result3 = PartitionUtils.calculateMaxSplitBytes(spark, 1024L, 1, sqlConf); long expected3 = 1024L + openCostInBytes; assertEquals(expected3, result3); // Many small files long result4 = PartitionUtils.calculateMaxSplitBytes(spark, 1 * MB, 1000, sqlConf); assertEquals(maxPartitionBytes, result4); } @Test public void testCalculateMaxSplitBytes_UndefinedMinPartitionNum() { SQLConf sqlConf = SQLConf.get(); // Ensure filesMinPartitionNum is undefined if (sqlConf.filesMinPartitionNum().isDefined()) { sqlConf.unsetConf("spark.sql.files.minPartitionNum"); } long totalBytes = 200 * MB; int fileCount = 10; long result = PartitionUtils.calculateMaxSplitBytes(spark, totalBytes, fileCount, sqlConf); // Verify the result is still valid assertTrue(result > 0); assertTrue(result >= sqlConf.filesOpenCostInBytes()); assertTrue(result <= sqlConf.filesMaxPartitionBytes()); long calculatedTotalBytes = totalBytes + (long) fileCount * sqlConf.filesOpenCostInBytes(); assertTrue(result <= calculatedTotalBytes); } @Test public void testBuildPartitionedFile() throws Exception { String tablePath = createTestTable("test_build_partitioned_file_" + System.nanoTime(), true); // Get an AddFile from the table Table table = Table.forPath(defaultEngine, tablePath); Scan scan = table.getLatestSnapshot(defaultEngine).getScanBuilder().build(); FilteredColumnarBatch batch = scan.getScanFiles(defaultEngine).next(); CloseableIterator rows = batch.getRows(); AddFile addFile = new AddFile(rows.next().getStruct(0)); rows.close(); // Build PartitionedFile StructType partitionSchema = new StructType( new StructField[] {DataTypes.createStructField("part", DataTypes.StringType, true)}); String normalizedTablePath = tablePath.endsWith("/") ? tablePath : tablePath + "/"; PartitionedFile partitionedFile = PartitionUtils.buildPartitionedFile( addFile, partitionSchema, normalizedTablePath, ZoneId.of("UTC")); assertNotNull(partitionedFile); assertEquals(addFile.getSize(), partitionedFile.fileSize()); assertEquals(1, partitionedFile.partitionValues().numFields()); } /** Helper to create a test Delta table. */ private String createTestTable(String tableName, boolean partitioned) { String tablePath = getTempTablePath(tableName); if (partitioned) { spark .range(10) .selectExpr("id", "cast(id % 3 as string) as part") .write() .format("delta") .partitionBy("part") .save(tablePath); } else { spark.range(10).write().format("delta").save(tablePath); } return tablePath; } private String getTempTablePath(String tableName) { return java.nio.file.Paths.get(System.getProperty("java.io.tmpdir"), "delta-test-" + tableName) .toString(); } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/utils/RowTrackingUtilsTest.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.utils; import static org.junit.jupiter.api.Assertions.*; import io.delta.kernel.data.ArrayValue; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.defaults.internal.json.JsonUtils; import io.delta.kernel.internal.TableConfig; import io.delta.kernel.internal.actions.Format; import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import io.delta.kernel.internal.actions.SingleAction; import io.delta.kernel.types.IntegerType; import io.delta.kernel.types.StructType; import java.util.*; import java.util.stream.Stream; import org.apache.spark.sql.catalyst.expressions.FileSourceConstantMetadataStructField; import org.apache.spark.sql.delta.DeltaIllegalStateException; import org.apache.spark.sql.delta.RowTracking; import org.apache.spark.sql.delta.actions.Action; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.MetadataBuilder; import org.apache.spark.sql.types.StructField; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import scala.collection.JavaConverters; public class RowTrackingUtilsTest { @Test public void testIsEnabled_NotEnabledInMetadata_ReturnsFalse() { Protocol protocol = createProtocol(3, 7, Set.of("rowTracking"), Set.of("rowTracking")); Metadata metadata = createMetadata(Collections.emptyMap()); assertFalse(io.delta.kernel.internal.rowtracking.RowTracking.isEnabled(protocol, metadata)); } @Test public void testIsEnabled_SupportedAndEnabled_ReturnsTrue() { Protocol protocol = createProtocol(3, 7, Set.of("rowTracking"), Set.of("rowTracking")); Map config = new HashMap<>(); config.put("delta.enableRowTracking", "true"); Metadata metadata = createMetadata(config); assertTrue(io.delta.kernel.internal.rowtracking.RowTracking.isEnabled(protocol, metadata)); } @Test public void testIsEnabled_EnabledButNotSupported_ThrowsError() { Protocol protocol = createProtocol(1, 1, Collections.emptySet(), Collections.emptySet()); Map config = new HashMap<>(); config.put("delta.enableRowTracking", "true"); Metadata metadata = createMetadata(config); IllegalStateException exception = assertThrows( IllegalStateException.class, () -> io.delta.kernel.internal.rowtracking.RowTracking.isEnabled(protocol, metadata)); assertTrue( exception .getMessage() .contains("doesn't support table feature 'delta.feature.rowTracking'")); } @Test public void testCreateMetadataStructFields_NotEnabled_ReturnsEmptyList() { Protocol protocol = createProtocol(3, 7, Set.of("rowTracking"), Set.of("rowTracking")); Metadata metadata = createMetadata(Collections.emptyMap()); List fields = RowTrackingUtils.createMetadataStructFields(protocol, metadata, false, false); assertTrue(fields.isEmpty()); } @Test public void testCreateMetadataStructFields_MissingMaterializedRowId_ThrowsException() { Protocol protocol = createProtocol(3, 7, Set.of("rowTracking"), Set.of("rowTracking")); Map config = new HashMap<>(); config.put("delta.enableRowTracking", "true"); // Missing MATERIALIZED_ROW_ID_COLUMN_NAME config.put(TableConfig.MATERIALIZED_ROW_COMMIT_VERSION_COLUMN_NAME.getKey(), "__row_version"); Metadata metadata = createMetadata(config); DeltaIllegalStateException exception = assertThrows( DeltaIllegalStateException.class, () -> RowTrackingUtils.createMetadataStructFields(protocol, metadata, false, false)); assertTrue(exception.getMessage().contains("Row ID")); } @Test public void testCreateMetadataStructFields_MissingMaterializedRowCommitVersion_ThrowsException() { Protocol protocol = createProtocol(3, 7, Set.of("rowTracking"), Set.of("rowTracking")); Map config = new HashMap<>(); config.put("delta.enableRowTracking", "true"); config.put(TableConfig.MATERIALIZED_ROW_ID_COLUMN_NAME.getKey(), "__row_id"); // Missing MATERIALIZED_ROW_COMMIT_VERSION_COLUMN_NAME Metadata metadata = createMetadata(config); DeltaIllegalStateException exception = assertThrows( DeltaIllegalStateException.class, () -> RowTrackingUtils.createMetadataStructFields(protocol, metadata, false, false)); assertTrue(exception.getMessage().contains("Row Commit Version")); } private static Stream createMetadataStructFieldsTestProvider() { // Note: withMaterializedColumns must be true when row tracking is enabled // because materialized column names are required return Stream.of( // nullableConstant, nullableGenerated Arguments.of(false, false), Arguments.of(false, true), Arguments.of(true, false), Arguments.of(true, true)); } @ParameterizedTest @MethodSource("createMetadataStructFieldsTestProvider") public void testCreateMetadataStructFields(boolean nullableConstant, boolean nullableGenerated) { // Create Kernel Protocol and Metadata // Note: materialized columns are always configured when row tracking is enabled Protocol kernelProtocol = createProtocol(3, 7, Set.of("rowTracking"), Set.of("rowTracking")); Map config = new HashMap<>(); config.put("delta.enableRowTracking", "true"); config.put(TableConfig.MATERIALIZED_ROW_ID_COLUMN_NAME.getKey(), "__row_id"); config.put(TableConfig.MATERIALIZED_ROW_COMMIT_VERSION_COLUMN_NAME.getKey(), "__row_version"); Metadata kernelMetadata = createMetadata(config); // Get actual result from Kernel-Spark API List actualFields = RowTrackingUtils.createMetadataStructFields( kernelProtocol, kernelMetadata, nullableConstant, nullableGenerated); // Build expected fields List expectedFields = new ArrayList<>(); // row_id (generated field) expectedFields.add( new StructField( "row_id", DataTypes.LongType, nullableGenerated, new MetadataBuilder() .withMetadata( org.apache.spark.sql.catalyst.expressions.FileSourceGeneratedMetadataStructField .metadata("row_id", "__row_id")) .putBoolean("__row_id_metadata_col", true) .build())); // base_row_id (constant field) expectedFields.add( new StructField( "base_row_id", DataTypes.LongType, nullableConstant, new MetadataBuilder() .withMetadata(FileSourceConstantMetadataStructField.metadata("base_row_id")) .putBoolean("__base_row_id_metadata_col", true) .build())); // default_row_commit_version (constant field) expectedFields.add( new StructField( "default_row_commit_version", DataTypes.LongType, nullableConstant, new MetadataBuilder() .withMetadata( FileSourceConstantMetadataStructField.metadata("default_row_commit_version")) .putBoolean("__default_row_version_metadata_col", true) .build())); // row_commit_version (generated field) expectedFields.add( new StructField( "row_commit_version", DataTypes.LongType, nullableGenerated, new MetadataBuilder() .withMetadata( org.apache.spark.sql.catalyst.expressions.FileSourceGeneratedMetadataStructField .metadata("row_commit_version", "__row_version")) .putBoolean("__row_commit_version_metadata_col", true) .build())); String protocolJson = JsonUtils.rowToJson(SingleAction.createProtocolSingleAction(kernelProtocol.toRow())); org.apache.spark.sql.delta.actions.Protocol sparkV1Protocol = Action.fromJson(protocolJson).wrap().protocol(); String metadataJson = JsonUtils.rowToJson(SingleAction.createMetadataSingleAction(kernelMetadata.toRow())); org.apache.spark.sql.delta.actions.Metadata sparkV1Metadata = Action.fromJson(metadataJson).wrap().metaData(); scala.collection.Iterable sparkV1FieldsIterable = RowTracking.createMetadataStructFields( sparkV1Protocol, sparkV1Metadata, nullableConstant, nullableGenerated); List v1Fields = new ArrayList<>(JavaConverters.asJavaCollection(sparkV1FieldsIterable)); assertEquals(expectedFields, actualFields); // Ensure both delta implementation return same result. assertEquals(v1Fields, actualFields); } private Protocol createProtocol( int minReaderVersion, int minWriterVersion, Set readerFeatures, Set writerFeatures) { return new Protocol(minReaderVersion, minWriterVersion, readerFeatures, writerFeatures); } private Metadata createMetadata(Map configuration) { StructType schema = new StructType().add("id", IntegerType.INTEGER); ArrayValue emptyPartitionColumns = new ArrayValue() { @Override public int getSize() { return 0; } @Override public ColumnVector getElements() { return null; } }; return new Metadata( "id", Optional.empty() /* name */, Optional.empty() /* description */, new Format(), schema.toJson(), schema, emptyPartitionColumns, Optional.empty() /* createdTime */, io.delta.kernel.internal.util.VectorUtils.stringStringMapValue(configuration)); } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ScalaUtilsTest.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.utils; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Collections; import java.util.Map; import org.junit.jupiter.api.Test; class ScalaUtilsTest { @Test void testToJavaMap_NullInput_ReturnsNull() { assertNull(ScalaUtils.toJavaMap(null), "Null scala maps should return null"); } @Test void testToJavaMap_EmptyInput_ReturnsEmptyMap() { scala.collection.immutable.Map emptyScalaMap = ScalaUtils.toScalaMap(Collections.emptyMap()); Map javaMap = ScalaUtils.toJavaMap(emptyScalaMap); assertTrue(javaMap.isEmpty(), "Empty scala maps should convert to empty java maps"); } @Test void testToJavaMap_PopulatedInput_PreservesEntries() { scala.collection.immutable.Map scalaMap = ScalaUtils.toScalaMap(Map.of("foo", "bar")); Map javaMap = ScalaUtils.toJavaMap(scalaMap); assertEquals(Map.of("foo", "bar"), javaMap, "Scala map entries should be preserved"); } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/utils/SchemaUtilsTest.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.utils; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import io.delta.kernel.types.ArrayType; import io.delta.kernel.types.BinaryType; import io.delta.kernel.types.BooleanType; import io.delta.kernel.types.ByteType; import io.delta.kernel.types.DataType; import io.delta.kernel.types.DateType; import io.delta.kernel.types.DecimalType; import io.delta.kernel.types.DoubleType; import io.delta.kernel.types.FieldMetadata; import io.delta.kernel.types.FloatType; import io.delta.kernel.types.IntegerType; import io.delta.kernel.types.LongType; import io.delta.kernel.types.MapType; import io.delta.kernel.types.ShortType; import io.delta.kernel.types.StringType; import io.delta.kernel.types.StructType; import io.delta.kernel.types.TimestampNTZType; import io.delta.kernel.types.TimestampType; import io.delta.kernel.types.VariantType; import java.util.stream.Stream; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.Metadata; import org.apache.spark.sql.types.MetadataBuilder; import org.apache.spark.sql.types.StructField; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; /** Tests for {@link SchemaUtils}. */ public class SchemaUtilsTest { @Test public void testPrimitiveTypes() { checkConversion(DataTypes.StringType, StringType.STRING); checkConversion(DataTypes.BooleanType, BooleanType.BOOLEAN); checkConversion(DataTypes.IntegerType, IntegerType.INTEGER); checkConversion(DataTypes.LongType, LongType.LONG); checkConversion(DataTypes.BinaryType, BinaryType.BINARY); checkConversion(DataTypes.ByteType, ByteType.BYTE); checkConversion(DataTypes.DateType, DateType.DATE); checkConversion(DataTypes.createDecimalType(10, 2), new DecimalType(10, 2)); checkConversion(DataTypes.DoubleType, DoubleType.DOUBLE); checkConversion(DataTypes.FloatType, FloatType.FLOAT); checkConversion(DataTypes.ShortType, ShortType.SHORT); checkConversion(DataTypes.TimestampType, TimestampType.TIMESTAMP); checkConversion(DataTypes.TimestampNTZType, TimestampNTZType.TIMESTAMP_NTZ); checkConversion(DataTypes.VariantType, VariantType.VARIANT); } @Test public void testArrayType() { checkConversion( DataTypes.createArrayType(DataTypes.IntegerType, true /* containsNull */), new ArrayType(IntegerType.INTEGER, true /* containsNull */)); checkConversion( DataTypes.createArrayType(DataTypes.StringType, false /* containsNull */), new ArrayType(StringType.STRING, false /* containsNull */)); } @Test public void testMapType() { checkConversion( DataTypes.createMapType( DataTypes.StringType, DataTypes.IntegerType, true /* valueContainsNull */), new MapType(StringType.STRING, IntegerType.INTEGER, true /* valueContainsNull */)); checkConversion( DataTypes.createMapType( DataTypes.LongType, DataTypes.BooleanType, false /* valueContainsNull */), new MapType(LongType.LONG, BooleanType.BOOLEAN, false /* valueContainsNull */)); } @Test public void testStructType() { org.apache.spark.sql.types.StructType sparkStruct = DataTypes.createStructType( new StructField[] { DataTypes.createStructField("a", DataTypes.IntegerType, true /* nullable */), DataTypes.createStructField("b", DataTypes.StringType, false /* nullable */) }); StructType kernelStruct = new StructType() .add("a", IntegerType.INTEGER, true /* nullable */) .add("b", StringType.STRING, false /* nullable */); checkConversion(sparkStruct, kernelStruct); } @Test public void testNestedTypes() { org.apache.spark.sql.types.StructType sparkStruct = DataTypes.createStructType( new StructField[] { DataTypes.createStructField( "a", DataTypes.createArrayType(DataTypes.IntegerType, true /* containsNull */), true /* nullable */), DataTypes.createStructField( "b", DataTypes.createMapType( DataTypes.StringType, DataTypes.BooleanType, false /* valueContainsNull */), false /* nullable */) }); StructType kernelStruct = new StructType() .add( "a", new ArrayType(IntegerType.INTEGER, true /* containsNull */), true /* nullable */) .add( "b", new MapType(StringType.STRING, BooleanType.BOOLEAN, false /* valueContainsNull */), false /* nullable */); checkConversion(sparkStruct, kernelStruct); } @Test public void testConvertSparkSchemaToKernelSchema() { org.apache.spark.sql.types.StructType sparkSchema = DataTypes.createStructType( new org.apache.spark.sql.types.StructField[] { DataTypes.createStructField("a", DataTypes.IntegerType, true), DataTypes.createStructField("b", DataTypes.StringType, false) }); StructType expectedKernelSchema = new StructType().add("a", IntegerType.INTEGER, true).add("b", StringType.STRING, false); StructType actualKernelSchema = SchemaUtils.convertSparkSchemaToKernelSchema(sparkSchema); assertEquals(expectedKernelSchema, actualKernelSchema); } @Test public void testConvertKernelSchemaToSparkSchema() { StructType kernelSchema = new StructType().add("a", IntegerType.INTEGER, true).add("b", StringType.STRING, false); org.apache.spark.sql.types.StructType expectedSparkSchema = DataTypes.createStructType( new org.apache.spark.sql.types.StructField[] { DataTypes.createStructField("a", DataTypes.IntegerType, true), DataTypes.createStructField("b", DataTypes.StringType, false) }); org.apache.spark.sql.types.StructType actualSparkSchema = SchemaUtils.convertKernelSchemaToSparkSchema(kernelSchema); assertEquals(expectedSparkSchema, actualSparkSchema); } static Stream nullInPrimitiveArraysProvider() { return Stream.of( Arguments.of( "Long", FieldMetadata.builder().putLongArray("ids", new Long[] {1L, null, 3L}).build(), 1), Arguments.of( "Double", FieldMetadata.builder().putDoubleArray("scores", new Double[] {1.1, 2.2, null}).build(), 2), Arguments.of( "Boolean", FieldMetadata.builder() .putBooleanArray("flags", new Boolean[] {true, null, false}) .build(), 1)); } @ParameterizedTest(name = "{0} array with null at index {2}") @MethodSource("nullInPrimitiveArraysProvider") public void testNullInPrimitiveArrays( String typeName, FieldMetadata kernelMetadata, int expectedNullIndex) { Exception ex = assertThrows( Exception.class, () -> SchemaUtils.convertKernelFieldMetadataToSparkMetadata(kernelMetadata)); assertTrue(ex.getMessage().contains("Null element at index " + expectedNullIndex)); } @Test public void testSchemaRoundTripWithFieldMetadata() { // Schema with field metadata (e.g., column mapping) org.apache.spark.sql.types.StructType sparkSchema = new org.apache.spark.sql.types.StructType() .add( "user_id", DataTypes.IntegerType, true, new MetadataBuilder() .putLong("delta.columnMapping.id", 123L) .putString("delta.columnMapping.physicalName", "col-abc-123") .build()); StructType expectedKernelSchema = new StructType() .add( "user_id", IntegerType.INTEGER, true, FieldMetadata.builder() .putLong("delta.columnMapping.id", 123L) .putString("delta.columnMapping.physicalName", "col-abc-123") .build()); // Verify Spark → Kernel conversion StructType actualKernelSchema = SchemaUtils.convertSparkSchemaToKernelSchema(sparkSchema); assertEquals(expectedKernelSchema, actualKernelSchema); // Verify Kernel → Spark conversion org.apache.spark.sql.types.StructType sparkSchema2 = SchemaUtils.convertKernelSchemaToSparkSchema(actualKernelSchema); assertEquals(sparkSchema, sparkSchema2); } private void checkConversion( org.apache.spark.sql.types.DataType sparkDataType, DataType kernelDataType) { DataType toKernel = SchemaUtils.convertSparkDataTypeToKernelDataType(sparkDataType); assertEquals(kernelDataType, toKernel); org.apache.spark.sql.types.DataType toSpark = SchemaUtils.convertKernelDataTypeToSparkDataType(kernelDataType); assertEquals(sparkDataType, toSpark); } //////////////////////////////// // Field Metadata Tests // //////////////////////////////// static Stream metadataTypesProvider() { return Stream.of( // Empty Arguments.of("Empty", Metadata.empty(), FieldMetadata.empty()), // Primitives Arguments.of( "Long", new MetadataBuilder().putLong("id", 123L).build(), FieldMetadata.builder().putLong("id", 123L).build()), Arguments.of( "Double", new MetadataBuilder().putDouble("score", 3.14).build(), FieldMetadata.builder().putDouble("score", 3.14).build()), Arguments.of( "Boolean", new MetadataBuilder().putBoolean("flag", true).build(), FieldMetadata.builder().putBoolean("flag", true).build()), Arguments.of( "String", new MetadataBuilder().putString("name", "test").build(), FieldMetadata.builder().putString("name", "test").build()), Arguments.of( "Null", new MetadataBuilder().putNull("empty").build(), FieldMetadata.builder().putNull("empty").build()), // Arrays Arguments.of( "LongArray", new MetadataBuilder().putLongArray("ids", new long[] {1L, 2L, 3L}).build(), FieldMetadata.builder().putLongArray("ids", new Long[] {1L, 2L, 3L}).build()), Arguments.of( "DoubleArray", new MetadataBuilder().putDoubleArray("scores", new double[] {1.1, 2.2, 3.3}).build(), FieldMetadata.builder().putDoubleArray("scores", new Double[] {1.1, 2.2, 3.3}).build()), Arguments.of( "BooleanArray", new MetadataBuilder() .putBooleanArray("flags", new boolean[] {true, false, true}) .build(), FieldMetadata.builder() .putBooleanArray("flags", new Boolean[] {true, false, true}) .build()), Arguments.of( "StringArray", new MetadataBuilder().putStringArray("names", new String[] {"a", "b", "c"}).build(), FieldMetadata.builder().putStringArray("names", new String[] {"a", "b", "c"}).build()), // Nested Arguments.of( "NestedMetadata", new MetadataBuilder() .putMetadata("inner", new MetadataBuilder().putString("nested", "value").build()) .build(), FieldMetadata.builder() .putFieldMetadata( "inner", FieldMetadata.builder().putString("nested", "value").build()) .build()), // Metadata Array Arguments.of( "MetadataArray", new MetadataBuilder() .putMetadataArray( "items", new Metadata[] { new MetadataBuilder().putString("name", "first").build(), new MetadataBuilder().putString("name", "second").build() }) .build(), FieldMetadata.builder() .putFieldMetadataArray( "items", new FieldMetadata[] { FieldMetadata.builder().putString("name", "first").build(), FieldMetadata.builder().putString("name", "second").build() }) .build()), // Complex (multiple types mixed) Arguments.of( "ComplexMetadata", new MetadataBuilder() .putLong("id", 123L) .putDouble("score", 3.14) .putBoolean("active", true) .putString("name", "test") .putLongArray("versions", new long[] {1L, 2L}) .putDoubleArray("scores", new double[] {1.1, 2.2}) .putBooleanArray("flags", new boolean[] {true, false}) .putStringArray("tags", new String[] {"a", "b"}) .putMetadata( "nested", new MetadataBuilder() .putString("type", "nested") .putLongArray("ids", new long[] {1L, 2L, 3L}) .build()) .putNull("empty") .build(), FieldMetadata.builder() .putLong("id", 123L) .putDouble("score", 3.14) .putBoolean("active", true) .putString("name", "test") .putLongArray("versions", new Long[] {1L, 2L}) .putDoubleArray("scores", new Double[] {1.1, 2.2}) .putBooleanArray("flags", new Boolean[] {true, false}) .putStringArray("tags", new String[] {"a", "b"}) .putFieldMetadata( "nested", FieldMetadata.builder() .putString("type", "nested") .putLongArray("ids", new Long[] {1L, 2L, 3L}) .build()) .putNull("empty") .build())); } @ParameterizedTest(name = "Metadata type: {0}") @MethodSource("metadataTypesProvider") public void testMetadataConversion( String typeName, Metadata sparkMetadata, FieldMetadata kernelMetadata) { assertEquals( kernelMetadata, SchemaUtils.convertSparkMetadataToKernelFieldMetadata(sparkMetadata)); assertEquals( sparkMetadata, SchemaUtils.convertKernelFieldMetadataToSparkMetadata(kernelMetadata)); } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/utils/StatsUtilsTest.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.utils; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Map; import org.apache.spark.sql.catalyst.catalog.CatalogColumnStat; import org.apache.spark.sql.catalyst.catalog.CatalogStatistics; import org.apache.spark.sql.connector.expressions.FieldReference; import org.apache.spark.sql.connector.expressions.NamedReference; import org.apache.spark.sql.connector.read.Statistics; import org.apache.spark.sql.connector.read.colstats.ColumnStatistics; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; import org.junit.jupiter.api.Test; import scala.Option; class StatsUtilsTest { @Test void testToV2Statistics_sizeAndRowCount() { StructType dataSchema = new StructType().add("id", DataTypes.IntegerType).add("name", DataTypes.StringType); StructType partitionSchema = new StructType(); CatalogStatistics catalogStats = new CatalogStatistics( BigInt(1024L), Option.apply(BigInt(100L)), scala.collection.immutable.Map$.MODULE$.empty()); Statistics v2Stats = StatsUtils.toV2Statistics(catalogStats, dataSchema, partitionSchema); assertEquals(1024L, v2Stats.sizeInBytes().getAsLong(), "sizeInBytes should match"); assertTrue(v2Stats.numRows().isPresent(), "numRows should be present"); assertEquals(100L, v2Stats.numRows().getAsLong(), "numRows should match"); assertTrue(v2Stats.columnStats().isEmpty(), "columnStats should be empty"); } @Test void testToV2Statistics_sizeOnlyNoRowCount() { StructType dataSchema = new StructType().add("id", DataTypes.IntegerType); StructType partitionSchema = new StructType(); CatalogStatistics catalogStats = new CatalogStatistics( BigInt(512L), Option.empty(), scala.collection.immutable.Map$.MODULE$.empty()); Statistics v2Stats = StatsUtils.toV2Statistics(catalogStats, dataSchema, partitionSchema); assertEquals(512L, v2Stats.sizeInBytes().getAsLong(), "sizeInBytes should match"); assertFalse(v2Stats.numRows().isPresent(), "numRows should be empty"); } @Test void testToV2Statistics_withColumnStats() { StructType dataSchema = new StructType().add("id", DataTypes.IntegerType).add("name", DataTypes.StringType); StructType partitionSchema = new StructType().add("part", DataTypes.IntegerType); // Create column stats for "id" column CatalogColumnStat idColStat = new CatalogColumnStat( Option.apply(BigInt(10L)), // distinctCount Option.apply("1"), // min Option.apply("100"), // max Option.apply(BigInt(0L)), // nullCount Option.apply((Object) 4L), // avgLen Option.apply((Object) 4L), // maxLen Option.empty(), // histogram CatalogColumnStat.VERSION()); scala.collection.immutable.Map colStatsMap = buildScalaMap(new String[] {"id"}, new CatalogColumnStat[] {idColStat}); CatalogStatistics catalogStats = new CatalogStatistics(BigInt(2048L), Option.apply(BigInt(50L)), colStatsMap); Statistics v2Stats = StatsUtils.toV2Statistics(catalogStats, dataSchema, partitionSchema); assertEquals(2048L, v2Stats.sizeInBytes().getAsLong()); assertEquals(50L, v2Stats.numRows().getAsLong()); Map colStats = v2Stats.columnStats(); assertEquals(1, colStats.size(), "Should have 1 column stat"); ColumnStatistics idStats = colStats.get(FieldReference.apply("id")); assertNotNull(idStats, "id column stats should be present"); assertEquals(10L, idStats.distinctCount().getAsLong(), "distinctCount should be 10"); assertEquals(0L, idStats.nullCount().getAsLong(), "nullCount should be 0"); assertEquals(4L, idStats.avgLen().getAsLong(), "avgLen should be 4"); assertEquals(4L, idStats.maxLen().getAsLong(), "maxLen should be 4"); assertTrue(idStats.min().isPresent(), "min should be present"); assertTrue(idStats.max().isPresent(), "max should be present"); assertEquals(1, idStats.min().get(), "min should be 1"); assertEquals(100, idStats.max().get(), "max should be 100"); } @Test void testToV2Statistics_skipsColumnsNotInSchema() { // Only "id" is in schema, "unknown" should be skipped StructType dataSchema = new StructType().add("id", DataTypes.IntegerType); StructType partitionSchema = new StructType(); CatalogColumnStat colStat = new CatalogColumnStat( Option.apply(BigInt(5L)), Option.empty(), Option.empty(), Option.apply(BigInt(1L)), Option.empty(), Option.empty(), Option.empty(), CatalogColumnStat.VERSION()); scala.collection.immutable.Map colStatsMap = buildScalaMap(new String[] {"id", "unknown"}, new CatalogColumnStat[] {colStat, colStat}); CatalogStatistics catalogStats = new CatalogStatistics(BigInt(100L), Option.empty(), colStatsMap); Statistics v2Stats = StatsUtils.toV2Statistics(catalogStats, dataSchema, partitionSchema); Map result = v2Stats.columnStats(); assertEquals(1, result.size(), "Should only have 1 column stat (unknown skipped)"); assertNotNull(result.get(FieldReference.apply("id")), "id stats should be present"); } private static scala.math.BigInt BigInt(long value) { return scala.math.BigInt.apply(value); } @SuppressWarnings({"rawtypes", "unchecked"}) private static scala.collection.immutable.Map buildScalaMap( String[] keys, CatalogColumnStat[] values) { scala.collection.mutable.Builder< scala.Tuple2, scala.collection.immutable.Map> b = (scala.collection.mutable.Builder) scala.collection.immutable.Map$.MODULE$.newBuilder(); for (int i = 0; i < keys.length; i++) { b.$plus$eq(new scala.Tuple2<>(keys[i], values[i])); } return b.result(); } } ================================================ FILE: spark/v2/src/test/java/io/delta/spark/internal/v2/utils/StreamingHelperTest.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.utils; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import io.delta.kernel.Snapshot; import io.delta.kernel.internal.DeltaHistoryManager; import io.delta.spark.internal.v2.DeltaV2TestBase; import io.delta.spark.internal.v2.exception.VersionNotFoundException; import io.delta.spark.internal.v2.snapshot.PathBasedSnapshotManager; import java.io.File; import java.sql.Timestamp; import java.util.stream.Stream; import org.apache.hadoop.fs.Path; import org.apache.spark.sql.delta.DeltaLog; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import scala.Option; public class StreamingHelperTest extends DeltaV2TestBase { private PathBasedSnapshotManager snapshotManager; @Test public void testUnsafeVolatileSnapshot(@TempDir File tempDir) { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_volatile_snapshot"; createEmptyTestTable(testTablePath, testTableName); snapshotManager = new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); org.apache.spark.sql.delta.Snapshot deltaSnapshot = deltaLog.unsafeVolatileSnapshot(); Snapshot kernelSnapshot = snapshotManager.loadLatestSnapshot(); spark.sql(String.format("INSERT INTO %s VALUES (4, 'David')", testTableName)); assertEquals(0L, deltaSnapshot.version()); assertEquals(deltaSnapshot.version(), kernelSnapshot.getVersion()); } @Test public void testLoadLatestSnapshot(@TempDir File tempDir) { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_update"; createEmptyTestTable(testTablePath, testTableName); snapshotManager = new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); Snapshot initialSnapshot = snapshotManager.loadLatestSnapshot(); assertEquals(0L, initialSnapshot.getVersion()); spark.sql(String.format("INSERT INTO %s VALUES (4, 'David')", testTableName)); org.apache.spark.sql.delta.Snapshot deltaSnapshot = deltaLog.update(false, Option.empty(), Option.empty()); Snapshot updatedSnapshot = snapshotManager.loadLatestSnapshot(); org.apache.spark.sql.delta.Snapshot cachedSnapshot = deltaLog.unsafeVolatileSnapshot(); Snapshot kernelcachedSnapshot = snapshotManager.loadLatestSnapshot(); assertEquals(1L, updatedSnapshot.getVersion()); assertEquals(deltaSnapshot.version(), updatedSnapshot.getVersion()); assertEquals(1L, kernelcachedSnapshot.getVersion()); assertEquals(cachedSnapshot.version(), kernelcachedSnapshot.getVersion()); } @Test public void testMultipleLoadLatestSnapshot(@TempDir File tempDir) { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_multiple_updates"; createEmptyTestTable(testTablePath, testTableName); snapshotManager = new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); assertEquals(0L, snapshotManager.loadLatestSnapshot().getVersion()); for (int i = 0; i < 3; i++) { spark.sql( String.format("INSERT INTO %s VALUES (%d, 'User%d')", testTableName, 20 + i, 20 + i)); org.apache.spark.sql.delta.Snapshot deltaSnapshot = deltaLog.update(false, Option.empty(), Option.empty()); Snapshot kernelSnapshot = snapshotManager.loadLatestSnapshot(); long expectedVersion = i + 1; assertEquals(expectedVersion, deltaSnapshot.version()); assertEquals(expectedVersion, kernelSnapshot.getVersion()); } } private void setupTableWithDeletedVersions(String testTablePath, String testTableName) { createEmptyTestTable(testTablePath, testTableName); for (int i = 0; i < 10; i++) { spark.sql( String.format("INSERT INTO %s VALUES (%d, 'User%d')", testTableName, 100 + i, 100 + i)); } File deltaLogDir = new File(testTablePath, "_delta_log"); File version0File = new File(deltaLogDir, "00000000000000000000.json"); File version1File = new File(deltaLogDir, "00000000000000000001.json"); assertTrue(version0File.exists()); assertTrue(version1File.exists()); version0File.delete(); version1File.delete(); assertFalse(version0File.exists()); assertFalse(version1File.exists()); } @Test public void testGetActiveCommitAtTime_pastTimestamp(@TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_commit_past"; setupTableWithDeletedVersions(testTablePath, testTableName); snapshotManager = new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); Thread.sleep(100); Timestamp timestamp = new Timestamp(System.currentTimeMillis()); spark.sql(String.format("INSERT INTO %s VALUES (200, 'NewUser')", testTableName)); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); org.apache.spark.sql.delta.DeltaHistoryManager.Commit deltaCommit = deltaLog .history() .getActiveCommitAtTime( timestamp, Option.empty() /* catalogTable */, false /* canReturnLastCommit */, true /* mustBeRecreatable */, false /* canReturnEarliestCommit */); DeltaHistoryManager.Commit kernelCommit = snapshotManager.getActiveCommitAtTime( timestamp.getTime(), false /* canReturnLastCommit */, true /* mustBeRecreatable */, false /* canReturnEarliestCommit */); assertEquals(deltaCommit.version(), kernelCommit.getVersion()); assertEquals(deltaCommit.timestamp(), kernelCommit.getTimestamp()); } @Test public void testGetActiveCommitAtTime_futureTimestamp_canReturnLast(@TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_commit_future_last"; setupTableWithDeletedVersions(testTablePath, testTableName); snapshotManager = new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); Timestamp futureTimestamp = new Timestamp(System.currentTimeMillis() + 10000); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); org.apache.spark.sql.delta.DeltaHistoryManager.Commit deltaCommit = deltaLog .history() .getActiveCommitAtTime( futureTimestamp, Option.empty() /* catalogTable */, true /* canReturnLastCommit */, true /* mustBeRecreatable */, false /* canReturnEarliestCommit */); DeltaHistoryManager.Commit kernelCommit = snapshotManager.getActiveCommitAtTime( futureTimestamp.getTime(), true /* canReturnLastCommit */, true /* mustBeRecreatable */, false /* canReturnEarliestCommit */); assertEquals(deltaCommit.version(), kernelCommit.getVersion()); assertEquals(deltaCommit.timestamp(), kernelCommit.getTimestamp()); } @Test public void testGetActiveCommitAtTime_futureTimestamp_notMustBeRecreatable(@TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_commit_future_not_recreatable"; setupTableWithDeletedVersions(testTablePath, testTableName); snapshotManager = new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); Timestamp futureTimestamp = new Timestamp(System.currentTimeMillis() + 10000); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); org.apache.spark.sql.delta.DeltaHistoryManager.Commit deltaCommit = deltaLog .history() .getActiveCommitAtTime( futureTimestamp, Option.empty() /* catalogTable */, true /* canReturnLastCommit */, false /* mustBeRecreatable */, false /* canReturnEarliestCommit */); DeltaHistoryManager.Commit kernelCommit = snapshotManager.getActiveCommitAtTime( futureTimestamp.getTime(), true /* canReturnLastCommit */, false /* mustBeRecreatable */, false /* canReturnEarliestCommit */); assertEquals(deltaCommit.version(), kernelCommit.getVersion()); assertEquals(deltaCommit.timestamp(), kernelCommit.getTimestamp()); } @Test public void testGetActiveCommitAtTime_earlyTimestamp_canReturnEarliest(@TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_commit_early"; setupTableWithDeletedVersions(testTablePath, testTableName); snapshotManager = new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); Timestamp earlyTimestamp = new Timestamp(0); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); org.apache.spark.sql.delta.DeltaHistoryManager.Commit deltaCommit = deltaLog .history() .getActiveCommitAtTime( earlyTimestamp, Option.empty() /* catalogTable */, false /* canReturnLastCommit */, true /* mustBeRecreatable */, true /* canReturnEarliestCommit */); DeltaHistoryManager.Commit kernelCommit = snapshotManager.getActiveCommitAtTime( earlyTimestamp.getTime(), false /* canReturnLastCommit */, true /* mustBeRecreatable */, true /* canReturnEarliestCommit */); assertEquals(deltaCommit.version(), kernelCommit.getVersion()); assertEquals(deltaCommit.timestamp(), kernelCommit.getTimestamp()); } @Test public void testGetActiveCommitAtTime_earlyTimestamp_notMustBeRecreatable_canReturnEarliest( @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_commit_early_not_recreatable"; setupTableWithDeletedVersions(testTablePath, testTableName); snapshotManager = new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); Timestamp earlyTimestamp = new Timestamp(0); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); org.apache.spark.sql.delta.DeltaHistoryManager.Commit deltaCommit = deltaLog .history() .getActiveCommitAtTime( earlyTimestamp, Option.empty() /* catalogTable */, false /* canReturnLastCommit */, false /* mustBeRecreatable */, true /* canReturnEarliestCommit */); DeltaHistoryManager.Commit kernelCommit = snapshotManager.getActiveCommitAtTime( earlyTimestamp.getTime(), false /* canReturnLastCommit */, false /* mustBeRecreatable */, true /* canReturnEarliestCommit */); assertEquals(deltaCommit.version(), kernelCommit.getVersion()); assertEquals(deltaCommit.timestamp(), kernelCommit.getTimestamp()); } private static Stream checkVersionExistsTestCases() { return Stream.of( Arguments.of( "current", 10L /* versionToCheck */, true /* mustBeRecreatable */, false /* allowOutOfRange */, false /* shouldThrow */), Arguments.of( "notAllowOutOfRange", 21L /* versionToCheck */, true /* mustBeRecreatable */, false /* allowOutOfRange */, true /* shouldThrow */), Arguments.of( "allowOutOfRange", 21L /* versionToCheck */, true /* mustBeRecreatable */, true /* allowOutOfRange */, false /* shouldThrow */), Arguments.of( "belowEarliest", 1L /* versionToCheck */, true /* mustBeRecreatable */, false /* allowOutOfRange */, true /* shouldThrow */), Arguments.of( "mustBeRecreatable_false", 2L /* versionToCheck */, false /* mustBeRecreatable */, false /* allowOutOfRange */, false /* shouldThrow */), Arguments.of( "mustBeRecreatable_true", 2L /* versionToCheck */, true /* mustBeRecreatable */, false /* allowOutOfRange */, true /* shouldThrow */)); } @ParameterizedTest(name = "{0}") @MethodSource("checkVersionExistsTestCases") public void testCheckVersionExists( String testName, long versionToCheck, boolean mustBeRecreatable, boolean allowOutOfRange, boolean shouldThrow, @TempDir File tempDir) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = "test_version_" + testName; setupTableWithDeletedVersions(testTablePath, testTableName); snapshotManager = new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); if (shouldThrow) { assertThrows( VersionNotFoundException.class, () -> snapshotManager.checkVersionExists( versionToCheck, mustBeRecreatable, allowOutOfRange)); assertThrows( org.apache.spark.sql.delta.VersionNotFoundException.class, () -> deltaLog .history() .checkVersionExists( versionToCheck, Option.empty(), mustBeRecreatable, allowOutOfRange)); } else { snapshotManager.checkVersionExists(versionToCheck, mustBeRecreatable, allowOutOfRange); deltaLog .history() .checkVersionExists(versionToCheck, Option.empty(), mustBeRecreatable, allowOutOfRange); } } } ================================================ FILE: spark/v2/src/test/scala/io/delta/spark/internal/v2/snapshot/unitycatalog/UCManagedTableSnapshotManagerSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.snapshot.unitycatalog import java.util.Optional import scala.jdk.CollectionConverters._ import io.delta.kernel.exceptions.KernelException import io.delta.kernel.unitycatalog.{InMemoryUCClient, UCCatalogManagedClient, UCCatalogManagedTestUtils} import io.delta.spark.internal.v2.exception.VersionNotFoundException import io.delta.storage.commit.uccommitcoordinator.InvalidTargetTableException import org.scalatest.funsuite.AnyFunSuite /** Integration tests for [[UCManagedTableSnapshotManager]]. */ class UCManagedTableSnapshotManagerSuite extends AnyFunSuite with UCCatalogManagedTestUtils { private val testUcTableId = "testUcTableId" private val testUcUri = "https://test-uc.example.com" private val testUcToken = "test-token" private val testUcAuthConfig = Map("token" -> testUcToken).asJava private def createManager( ucClient: InMemoryUCClient, tablePath: String) = { val client = new UCCatalogManagedClient(ucClient) val tableInfo = new UCTableInfo(testUcTableId, tablePath, testUcUri, testUcAuthConfig) new UCManagedTableSnapshotManager(client, tableInfo, defaultEngine) } // ==================== Constructor ==================== test("constructor rejects null arguments") { val ucClient = new InMemoryUCClient("testMetastore") val client = new UCCatalogManagedClient(ucClient) val tableInfo = new UCTableInfo(testUcTableId, "/test/path", testUcUri, testUcAuthConfig) val ex1 = intercept[NullPointerException] { new UCManagedTableSnapshotManager(null, tableInfo, defaultEngine) } assert(ex1.getMessage == "ucCatalogManagedClient is null") val ex2 = intercept[NullPointerException] { new UCManagedTableSnapshotManager(client, null, defaultEngine) } assert(ex2.getMessage == "tableInfo is null") val ex3 = intercept[NullPointerException] { new UCManagedTableSnapshotManager(client, tableInfo, null) } assert(ex3.getMessage == "engine is null") } // ==================== loadLatestSnapshot ==================== test("loadLatestSnapshot: returns snapshot at max ratified version") { withUCClientAndTestTable { (ucClient, tablePath, maxRatifiedVersion) => val manager = createManager(ucClient, tablePath) val snapshot = manager.loadLatestSnapshot() assert(snapshot.getVersion == maxRatifiedVersion) } } test("loadLatestSnapshot: throws when table does not exist in catalog") { val ucClient = new InMemoryUCClient("ucMetastoreId") val tableInfo = new UCTableInfo("nonExistentTableId", "/fake/path", testUcUri, testUcAuthConfig) val client = new UCCatalogManagedClient(ucClient) val manager = new UCManagedTableSnapshotManager(client, tableInfo, defaultEngine) val ex = intercept[RuntimeException] { manager.loadLatestSnapshot() } assert(ex.getCause.isInstanceOf[InvalidTargetTableException]) } // ==================== loadSnapshotAt ==================== test("loadSnapshotAt: valid versions including v0 succeed, invalid versions throw") { withUCClientAndTestTable { (ucClient, tablePath, maxRatifiedVersion) => val manager = createManager(ucClient, tablePath) assert(manager.loadSnapshotAt(0L).getVersion == 0L) assert(manager.loadSnapshotAt(1L).getVersion == 1L) intercept[IllegalArgumentException] { manager.loadSnapshotAt(-1L) } intercept[IllegalArgumentException] { manager.loadSnapshotAt(maxRatifiedVersion + 10) } } } // ==================== checkVersionExists ==================== test("checkVersionExists: validates version bounds and allowOutOfRange flag") { withUCClientAndTestTable { (ucClient, tablePath, maxRatifiedVersion) => val manager = createManager(ucClient, tablePath) // Valid versions including v0 do not throw manager.checkVersionExists(0L, true /* mustBeRecreatable */, false /* allowOutOfRange */ ) manager.checkVersionExists( maxRatifiedVersion, true /* mustBeRecreatable */, false /* allowOutOfRange */ ) manager.checkVersionExists( maxRatifiedVersion - 1, true /* mustBeRecreatable */, false /* allowOutOfRange */ ) manager.checkVersionExists(1L, true /* mustBeRecreatable */, false /* allowOutOfRange */ ) manager.checkVersionExists(1L, false /* mustBeRecreatable */, false /* allowOutOfRange */ ) // Out-of-bounds versions throw val belowLowerBound = intercept[VersionNotFoundException] { manager.checkVersionExists(-1L, true /* mustBeRecreatable */, false /* allowOutOfRange */ ) } assert(belowLowerBound.getUserVersion == -1L) assert(belowLowerBound.getEarliest == 0L) assert(belowLowerBound.getLatest == maxRatifiedVersion) val aboveUpperBound = intercept[VersionNotFoundException] { manager.checkVersionExists( maxRatifiedVersion + 10, true /* mustBeRecreatable */, false /* allowOutOfRange */ ) } assert(aboveUpperBound.getUserVersion == maxRatifiedVersion + 10) assert(aboveUpperBound.getEarliest == 0L) assert(aboveUpperBound.getLatest == maxRatifiedVersion) // allowOutOfRange=true bypasses upper bound check manager.checkVersionExists( maxRatifiedVersion + 10, true /* mustBeRecreatable */, true /* allowOutOfRange */ ) } } // ==================== getActiveCommitAtTime ==================== test("getActiveCommitAtTime: resolves timestamps across all boundaries") { withUCClientAndTestTable { (ucClient, tablePath, _) => val manager = createManager(ucClient, tablePath) // Before first commit (v0) - throws without canReturnEarliestCommit intercept[KernelException] { manager.getActiveCommitAtTime( v0Ts - 1, false /* canReturnLastCommit */, true /* mustBeRecreatable */, false /* canReturnEarliestCommit */ ) } intercept[KernelException] { manager.getActiveCommitAtTime( -100L, false /* canReturnLastCommit */, true /* mustBeRecreatable */, false /* canReturnEarliestCommit */ ) } // With canReturnEarliestCommit, returns v0 val earliestCommit = manager.getActiveCommitAtTime( v0Ts - 1, false /* canReturnLastCommit */, true /* mustBeRecreatable */, true /* canReturnEarliestCommit */ ) assert(earliestCommit.getVersion == 0L) // Exact and between-commit timestamps def activeVersion(ts: Long): Long = manager .getActiveCommitAtTime( ts, false /* canReturnLastCommit */, true /* mustBeRecreatable */, false /* canReturnEarliestCommit */ ) .getVersion assert(activeVersion(v0Ts) == 0L) assert(activeVersion(v0Ts + 1) == 0L) assert(activeVersion(v1Ts) == 1L) assert(activeVersion(v1Ts + 1) == 1L) assert(activeVersion(v2Ts) == 2L) // After last commit (v2) - throws without canReturnLastCommit intercept[KernelException] { manager.getActiveCommitAtTime( v2Ts + 1, false /* canReturnLastCommit */, true /* mustBeRecreatable */, false /* canReturnEarliestCommit */ ) } intercept[KernelException] { manager.getActiveCommitAtTime( Long.MaxValue, false /* canReturnLastCommit */, true /* mustBeRecreatable */, false /* canReturnEarliestCommit */ ) } // With canReturnLastCommit, returns v2 val lastCommit = manager.getActiveCommitAtTime( v2Ts + 1, true /* canReturnLastCommit */, true /* mustBeRecreatable */, false /* canReturnEarliestCommit */ ) assert(lastCommit.getVersion == 2L) } } test("getActiveCommitAtTime: non-recreatable path returns earliest delta file") { withUCClientAndTestTable { (ucClient, tablePath, _) => val manager = createManager(ucClient, tablePath) val active = manager.getActiveCommitAtTime( v0Ts - 1, false /* canReturnLastCommit */, false /* mustBeRecreatable */, true /* canReturnEarliestCommit */ ) assert(active.getVersion == 0L) } } // ==================== getTableChanges ==================== test("getTableChanges: returns valid ranges and rejects invalid arguments") { withUCClientAndTestTable { (ucClient, tablePath, maxRatifiedVersion) => val manager = createManager(ucClient, tablePath) // Valid ranges including v0 and latest boundaries val fullRange = manager.getTableChanges(defaultEngine, 0L, Optional.of(maxRatifiedVersion)) assert(fullRange.getStartVersion == 0L) assert(fullRange.getEndVersion == maxRatifiedVersion) val toLatest = manager.getTableChanges(defaultEngine, 1L, Optional.empty()) assert(toLatest.getStartVersion == 1L) assert(toLatest.getEndVersion == maxRatifiedVersion) val single = manager.getTableChanges(defaultEngine, 1L, Optional.of(1L)) assert(single.getStartVersion == 1L) assert(single.getEndVersion == 1L) val first = manager.getTableChanges(defaultEngine, 0L, Optional.of(0L)) assert(first.getStartVersion == 0L) assert(first.getEndVersion == 0L) val last = manager.getTableChanges( defaultEngine, maxRatifiedVersion, Optional.of(maxRatifiedVersion)) assert(last.getStartVersion == maxRatifiedVersion) assert(last.getEndVersion == maxRatifiedVersion) // Invalid ranges throw intercept[IllegalArgumentException] { manager.getTableChanges( defaultEngine, maxRatifiedVersion, Optional.of(maxRatifiedVersion - 1)) } intercept[IllegalArgumentException] { manager.getTableChanges(defaultEngine, maxRatifiedVersion + 5, Optional.empty()) } } } // ==================== Exception Propagation ==================== test("operations propagate InvalidTargetTableException from client") { val ucClient = new InMemoryUCClient("ucMetastoreId") val tableInfo = new UCTableInfo("nonExistentTableId", "/fake/path", testUcUri, testUcAuthConfig) val client = new UCCatalogManagedClient(ucClient) val manager = new UCManagedTableSnapshotManager(client, tableInfo, defaultEngine) val ex1 = intercept[RuntimeException] { manager.loadLatestSnapshot() } assert(ex1.getCause.isInstanceOf[InvalidTargetTableException]) val ex2 = intercept[RuntimeException] { manager.loadSnapshotAt(0L) } assert(ex2.getCause.isInstanceOf[InvalidTargetTableException]) val ex3 = intercept[RuntimeException] { manager.checkVersionExists(0L, true, false) } assert(ex3.getCause.isInstanceOf[InvalidTargetTableException]) val ex4 = intercept[RuntimeException] { manager.getTableChanges(defaultEngine, 0L, Optional.empty()) } assert(ex4.getCause.isInstanceOf[InvalidTargetTableException]) } } ================================================ FILE: spark/v2/src/test/scala/io/delta/spark/internal/v2/snapshot/unitycatalog/UCUtilsSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.snapshot.unitycatalog import java.net.URI import java.util.{HashMap => JHashMap} import io.delta.kernel.internal.tablefeatures.TableFeatures import io.delta.spark.internal.v2.utils.CatalogTableTestUtils import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.test.SharedSparkSession /** * Unit tests for [[UCUtils]]. * * Tests use distinctive, high-entropy values that would fail if the implementation * had hardcoded defaults instead of actually extracting values from the inputs. */ class UCUtilsSuite extends SparkFunSuite with SharedSparkSession { // Use the same constants as CatalogTableUtils to ensure consistency private val FEATURE_CATALOG_MANAGED = TableFeatures.SET_TABLE_FEATURE_SUPPORTED_PREFIX + TableFeatures.CATALOG_MANAGED_RW_FEATURE.featureName() private val FEATURE_SUPPORTED = TableFeatures.SET_TABLE_FEATURE_SUPPORTED_VALUE private val UC_TABLE_ID_KEY = UCCommitCoordinatorClient.UC_TABLE_ID_KEY private val UC_CATALOG_CONNECTOR = "io.unitycatalog.spark.UCSingleCatalog" // Distinctive values that would fail if hardcoded private val TABLE_ID_ALPHA = "uc_8f2b3c9a-d1e7-4a6f-b8c2" private val TABLE_PATH_ALPHA = "abfss://delta-store@prod.dfs.core.windows.net/warehouse/tbl_v3" private val UC_URI_ALPHA = "https://uc-server-westus2.example.net/api/2.1/unity-catalog" private val UC_TOKEN_ALPHA = "dapi_Xk7mP$9qRs#2vWz_prod" private val CATALOG_ALPHA = "uc_catalog_westus2_prod" // ==================== Helper Methods ==================== private def makeNonUCTable(): CatalogTable = { CatalogTableTestUtils.createCatalogTable(locationUri = Some(new URI(TABLE_PATH_ALPHA))) } private def makeUCTable( tableId: String = TABLE_ID_ALPHA, tablePath: String = TABLE_PATH_ALPHA, catalogName: Option[String] = None): CatalogTable = { val storageProps = new JHashMap[String, String]() storageProps.put(FEATURE_CATALOG_MANAGED, FEATURE_SUPPORTED) storageProps.put(UC_TABLE_ID_KEY, tableId) CatalogTableTestUtils.createCatalogTable( catalogName = catalogName, storageProperties = storageProps, locationUri = Some(new URI(tablePath))) } private def withUCCatalogConfig( catalogName: String, uri: String, token: String)(testCode: => Unit): Unit = { val configs = Seq( s"spark.sql.catalog.$catalogName" -> UC_CATALOG_CONNECTOR, s"spark.sql.catalog.$catalogName.uri" -> uri, s"spark.sql.catalog.$catalogName.token" -> token) val originalValues = configs.map { case (key, _) => key -> spark.conf.getOption(key) }.toMap try { configs.foreach { case (key, value) => spark.conf.set(key, value) } testCode } finally { configs.foreach { case (key, _) => originalValues.get(key).flatten match { case Some(v) => spark.conf.set(key, v) case None => spark.conf.unset(key) } } } } // ==================== Tests ==================== test("returns empty for non-UC table") { val table = makeNonUCTable() val result = UCUtils.extractTableInfo(table, spark) assert(result.isEmpty, "Non-UC table should return empty Optional") } test("returns empty when UC table ID present but feature flag missing") { val storageProps = new JHashMap[String, String]() storageProps.put(UC_TABLE_ID_KEY, "orphan_id_9x7y5z") // No FEATURE_CATALOG_MANAGED - simulates corrupted/partial metadata val table = CatalogTableTestUtils.createCatalogTable( storageProperties = storageProps, locationUri = Some(new URI("gs://other-bucket/path"))) val result = UCUtils.extractTableInfo(table, spark) assert(result.isEmpty, "Missing feature flag should return empty") } test("throws IllegalArgumentException for UC table with empty table ID") { val storageProps = new JHashMap[String, String]() storageProps.put(FEATURE_CATALOG_MANAGED, FEATURE_SUPPORTED) storageProps.put(UC_TABLE_ID_KEY, "") val table = CatalogTableTestUtils.createCatalogTable( storageProperties = storageProps, locationUri = Some(new URI("s3://empty-id-bucket/path"))) val exception = intercept[IllegalArgumentException] { UCUtils.extractTableInfo(table, spark) } assert(exception.getMessage.contains("Cannot extract ucTableId")) } test("throws exception for UC table without location") { val storageProps = new JHashMap[String, String]() storageProps.put(FEATURE_CATALOG_MANAGED, FEATURE_SUPPORTED) storageProps.put(UC_TABLE_ID_KEY, "no_location_tbl_id_3k9m") val table = CatalogTableTestUtils.createCatalogTable(storageProperties = storageProps) // Spark throws AnalysisException when location is missing val exception = intercept[Exception] { UCUtils.extractTableInfo(table, spark) } assert(exception.getMessage.contains("locationUri") || exception.getMessage.contains("location")) } test("throws IllegalArgumentException when no matching catalog configuration") { val table = makeUCTable(catalogName = Some("nonexistent_catalog_xyz")) val exception = intercept[IllegalArgumentException] { UCUtils.extractTableInfo(table, spark) } assert(exception.getMessage.contains("Unity Catalog configuration not found") || exception.getMessage.contains("Cannot create UC client")) } test("extracts table info when UC catalog is properly configured") { val table = makeUCTable(catalogName = Some(CATALOG_ALPHA)) withUCCatalogConfig(CATALOG_ALPHA, UC_URI_ALPHA, UC_TOKEN_ALPHA) { val result = UCUtils.extractTableInfo(table, spark) assert(result.isPresent, "Should return table info") val info = result.get() // Each assertion uses the specific expected value - would fail if hardcoded assert(info.getTableId == TABLE_ID_ALPHA, s"Table ID mismatch: got ${info.getTableId}") assert( info.getTablePath == TABLE_PATH_ALPHA, s"Table path mismatch: got ${info.getTablePath}") assert(info.getUcUri == UC_URI_ALPHA, s"UC URI mismatch: got ${info.getUcUri}") val configMap = info.getAuthConfig assert( configMap.get("type") == "static", s"Type should be static: got ${configMap.get("type")}") assert( configMap.get("token") == UC_TOKEN_ALPHA, s"UC token mismatch: got ${configMap.get("token")}") } } test("selects correct catalog when multiple catalogs configured") { // Use completely different values for each catalog to prove selection works val catalogBeta = "uc_catalog_eastus_staging" val ucUriBeta = "https://uc-server-eastus.example.net/api/2.1/uc" val ucTokenBeta = "dapi_Yz3nQ$8wRt#1vXa_staging" val tableIdBeta = "uc_tbl_staging_4d7e2f1a" val tablePathBeta = "s3://staging-bucket-us-east/delta/tables/v2" val catalogGamma = "uc_catalog_euwest_dev" val ucUriGamma = "https://uc-server-euwest.example.net/api/2.1/uc" val ucTokenGamma = "dapi_Jk5pL$3mNq#9vBc_dev" // Table is in catalogBeta val table = makeUCTable( tableId = tableIdBeta, tablePath = tablePathBeta, catalogName = Some(catalogBeta)) val configs = Seq( // catalogGamma config (should NOT be used) s"spark.sql.catalog.$catalogGamma" -> UC_CATALOG_CONNECTOR, s"spark.sql.catalog.$catalogGamma.uri" -> ucUriGamma, s"spark.sql.catalog.$catalogGamma.token" -> ucTokenBeta, // catalogBeta config (should be used) s"spark.sql.catalog.$catalogBeta" -> UC_CATALOG_CONNECTOR, s"spark.sql.catalog.$catalogBeta.uri" -> ucUriBeta, s"spark.sql.catalog.$catalogBeta.token" -> ucTokenBeta) val originalValues = configs.map { case (key, _) => key -> spark.conf.getOption(key) }.toMap try { configs.foreach { case (key, value) => spark.conf.set(key, value) } val result = UCUtils.extractTableInfo(table, spark) assert(result.isPresent, "Should return table info") val info = result.get() // Verify it selected catalogBeta's config, not catalogGamma's assert( info.getUcUri == ucUriBeta, s"Should use catalogBeta's URI, got: ${info.getUcUri}") val configMap = info.getAuthConfig assert(configMap.get("type") == "static", s"Type should be static") assert( configMap.get("token") == ucTokenBeta, s"Should use catalogBeta's token, got: ${configMap.get("token")}") assert(info.getTableId == tableIdBeta, s"Should extract tableIdBeta, got: ${info.getTableId}") assert( info.getTablePath == tablePathBeta, s"Should extract tablePathBeta, got: ${info.getTablePath}") } finally { configs.foreach { case (key, _) => originalValues.get(key).flatten match { case Some(v) => spark.conf.set(key, v) case None => spark.conf.unset(key) } } } } } ================================================ FILE: spark/v2/src/test/scala/io/delta/spark/internal/v2/utils/CatalogTableTestUtils.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.spark.internal.v2.utils import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} import org.apache.spark.sql.types.StructType /** * Helpers for constructing [[CatalogTable]] instances inside Java tests. * * Spark's [[CatalogTable]] is defined in Scala and its constructor signature shifts between Spark * releases. Centralising the construction in Scala keeps the kernel tests insulated from those * binary changes and saves Java tests from manually wiring the many optional parameters. */ object CatalogTableTestUtils { /** * Creates a [[CatalogTable]] with configurable options. * * @param tableName table name (default: "tbl") * @param catalogName optional catalog name for the identifier * @param properties table properties (default: empty) * @param storageProperties storage properties (default: empty) * @param locationUri optional storage location URI * @param nullStorage if true, sets storage to null (for edge case testing) * @param nullStorageProperties if true, sets storage properties to null */ def createCatalogTable( tableName: String = "tbl", catalogName: Option[String] = None, properties: java.util.Map[String, String] = new java.util.HashMap[String, String](), storageProperties: java.util.Map[String, String] = new java.util.HashMap[String, String](), locationUri: Option[java.net.URI] = None, nullStorage: Boolean = false, nullStorageProperties: Boolean = false): CatalogTable = { val scalaProps = ScalaUtils.toScalaMap(properties) val scalaStorageProps = if (nullStorageProperties) null else ScalaUtils.toScalaMap(storageProperties) val identifier = catalogName match { case Some(catalog) => TableIdentifier(tableName, Some("default"), Some(catalog)) case None => TableIdentifier(tableName) } val storage = if (nullStorage) { null } else { CatalogStorageFormat( locationUri = locationUri, inputFormat = None, outputFormat = None, serde = None, compressed = false, properties = scalaStorageProps) } CatalogTable( identifier = identifier, tableType = CatalogTableType.MANAGED, storage = storage, schema = new StructType(), provider = None, partitionColumnNames = Seq.empty, bucketSpec = None, properties = scalaProps) } } ================================================ FILE: spark-connect/client/src/main/scala/io/delta/connect/tables/DeltaColumnBuilder.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables import org.apache.spark.annotation.Evolving import org.apache.spark.sql.catalyst.parser.DataTypeParser import org.apache.spark.sql.types.{DataType, LongType, MetadataBuilder, StructField} /** * :: Evolving :: * * Builder to specify a table column. * * See [[DeltaTableBuilder]] for examples. * * @since 4.0.0 */ @Evolving class DeltaColumnBuilder private[tables](private val colName: String) { private var dataType: DataType = _ private var nullable: Boolean = true private var generationExpr: Option[String] = None private var comment: Option[String] = None private var identityStart: Option[Long] = None private var identityStep: Option[Long] = None private var identityAllowExplicitInsert: Option[Boolean] = None /** * :: Evolving :: * * Specify the column data type. * * @param dataType string column data type * @since 4.0.0 */ @Evolving def dataType(dataType: String): DeltaColumnBuilder = { this.dataType = DataTypeParser.parseDataType(dataType) this } /** * :: Evolving :: * * Specify the column data type. * * @param dataType DataType column data type * @since 4.0.0 */ @Evolving def dataType(dataType: DataType): DeltaColumnBuilder = { this.dataType = dataType this } /** * :: Evolving :: * * Specify whether the column can be null. * * @param nullable boolean whether the column can be null or not. * @since 4.0.0 */ @Evolving def nullable(nullable: Boolean): DeltaColumnBuilder = { this.nullable = nullable this } /** * :: Evolving :: * * Specify a expression if the column is always generated as a function of other columns. * * @param expr string the the generation expression * @since 4.0.0 */ @Evolving def generatedAlwaysAs(expr: String): DeltaColumnBuilder = { this.generationExpr = Option(expr) this } /** * :: Evolving :: * * Specify a column as an identity column with default values that is always generated * by the system (i.e. does not allow user-specified values). * * @since 4.0.0 */ @Evolving def generatedAlwaysAsIdentity(): DeltaColumnBuilder = { generatedAlwaysAsIdentity(start = 1, step = 1) } /** * :: Evolving :: * * Specify a column as an identity column that is always generated by the system (i.e. does not * allow user-specified values). * * @param start the start value of the identity column * @param step the increment step of the identity column * @since 4.0.0 */ @Evolving def generatedAlwaysAsIdentity(start: Long, step: Long): DeltaColumnBuilder = { this.identityStart = Some(start) this.identityStep = Some(step) this.identityAllowExplicitInsert = Some(false) this } /** * :: Evolving :: * * Specify a column as an identity column that allows user-specified values such that the * generated values use default start and step values. * * @since 4.0.0 */ @Evolving def generatedByDefaultAsIdentity(): DeltaColumnBuilder = { generatedByDefaultAsIdentity(start = 1, step = 1) } /** * :: Evolving :: * * Specify a column as an identity column that allows user-specified values. * * @param start the start value of the identity column * @param step the increment step of the identity column * @since 4.0.0 */ @Evolving def generatedByDefaultAsIdentity(start: Long, step: Long): DeltaColumnBuilder = { this.identityStart = Some(start) this.identityStep = Some(step) this.identityAllowExplicitInsert = Some(true) this } /** * :: Evolving :: * * Specify a column comment. * * @param comment string column description * @since 4.0.0 */ @Evolving def comment(comment: String): DeltaColumnBuilder = { this.comment = Option(comment) this } /** * :: Evolving :: * * Build the column as a structField. * * @since 4.0.0 */ @Evolving def build(): StructField = { val metadataBuilder = new MetadataBuilder() if (generationExpr.nonEmpty) { metadataBuilder.putString("delta.generationExpression", generationExpr.get) } identityAllowExplicitInsert.foreach { allowExplicitInsert => if (generationExpr.nonEmpty) { throw DeltaTable.createAnalysisException( "IDENTITY column cannot be specified with a generated column expression.") } if (dataType != null && dataType != LongType) { throw DeltaTable.createAnalysisException( s"DataType ${dataType.typeName} is not supported for IDENTITY columns.") } metadataBuilder.putBoolean("delta.identity.allowExplicitInsert", allowExplicitInsert) metadataBuilder.putLong("delta.identity.start", identityStart.get) if (identityStep.get == 0L) { throw DeltaTable.createAnalysisException("IDENTITY column step cannot be 0.") } metadataBuilder.putLong("delta.identity.step", identityStep.get) } if (comment.nonEmpty) { metadataBuilder.putString("comment", comment.get) } val fieldMetadata = metadataBuilder.build() if (dataType == null) { throw DeltaTable.createAnalysisException( s"The data type of the column $colName is not provided") } StructField( colName, dataType, nullable = nullable, metadata = fieldMetadata) } } ================================================ FILE: spark-connect/client/src/main/scala/io/delta/connect/tables/DeltaMergeBuilder.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables import java.util.Arrays import scala.collection.JavaConverters._ import scala.collection.Map import io.delta.connect.proto import io.delta.connect.spark.{proto => spark_proto} import org.apache.spark.annotation.Unstable import org.apache.spark.sql.{functions, Column, DataFrame} import org.apache.spark.sql.connect.ColumnNodeToProtoConverter.toExpr import org.apache.spark.sql.connect.ConnectConversions._ import org.apache.spark.sql.connect.delta.ImplicitProtoConversions._ import org.apache.spark.sql.functions.expr /** * Builder to specify how to merge data from source DataFrame into the target Delta table. * You can specify any number of `whenMatched` and `whenNotMatched` clauses. * Here are the constraints on these clauses. * * - `whenMatched` clauses: * * - The condition in a `whenMatched` clause is optional. However, if there are multiple * `whenMatched` clauses, then only the last one may omit the condition. * * - When there are more than one `whenMatched` clauses and there are conditions (or the lack * of) such that a row satisfies multiple clauses, then the action for the first clause * satisfied is executed. In other words, the order of the `whenMatched` clauses matters. * * - If none of the `whenMatched` clauses match a source-target row pair that satisfy * the merge condition, then the target rows will not be updated or deleted. * * - If you want to update all the columns of the target Delta table with the * corresponding column of the source DataFrame, then you can use the * `whenMatched(...).updateAll()`. This is equivalent to * 

 *         whenMatched(...).updateExpr(Map(
 *           ("col1", "source.col1"),
 *           ("col2", "source.col2"),
 *           ...))
 *
* * - `whenNotMatched` clauses: * * - The condition in a `whenNotMatched` clause is optional. However, if there are * multiple `whenNotMatched` clauses, then only the last one may omit the condition. * * - When there are more than one `whenNotMatched` clauses and there are conditions (or the * lack of) such that a row satisfies multiple clauses, then the action for the first clause * satisfied is executed. In other words, the order of the `whenNotMatched` clauses matters. * * - If no `whenNotMatched` clause is present or if it is present but the non-matching source * row does not satisfy the condition, then the source row is not inserted. * * - If you want to insert all the columns of the target Delta table with the * corresponding column of the source DataFrame, then you can use * `whenNotMatched(...).insertAll()`. This is equivalent to * 
 *         whenNotMatched(...).insertExpr(Map(
 *           ("col1", "source.col1"),
 *           ("col2", "source.col2"),
 *           ...))
 *
* * - `whenNotMatchedBySource` clauses: * * - The condition in a `whenNotMatchedBySource` clause is optional. However, if there are * multiple `whenNotMatchedBySource` clauses, then only the last one may omit the condition. * * - When there are more than one `whenNotMatchedBySource` clauses and there are conditions (or * the lack of) such that a row satisfies multiple clauses, then the action for the first * clause satisfied is executed. In other words, the order of the `whenNotMatchedBySource` * clauses matters. * * - If no `whenNotMatchedBySource` clause is present or if it is present but the * non-matching target row does not satisfy any of the `whenNotMatchedBySource` clause * condition, then the target row will not be updated or deleted. * * * Scala example to update a key-value Delta table with new key-values from a source DataFrame: * {{{ * deltaTable * .as("target") * .merge( * source.as("source"), * "target.key = source.key") * .withSchemaEvolution() * .whenMatched() * .updateExpr(Map( * "value" -> "source.value")) * .whenNotMatched() * .insertExpr(Map( * "key" -> "source.key", * "value" -> "source.value")) * .whenNotMatchedBySource() * .updateExpr(Map( * "value" -> "target.value + 1")) * .execute() * }}} * * Java example to update a key-value Delta table with new key-values from a source DataFrame: * {{{ * deltaTable * .as("target") * .merge( * source.as("source"), * "target.key = source.key") * .withSchemaEvolution() * .whenMatched() * .updateExpr( * new HashMap() {{ * put("value", "source.value"); * }}) * .whenNotMatched() * .insertExpr( * new HashMap() {{ * put("key", "source.key"); * put("value", "source.value"); * }}) * .whenNotMatchedBySource() * .updateExpr( * new HashMap() {{ * put("value", "target.value + 1"); * }}) * .execute(); * }}} * * @since 4.0.0 */ class DeltaMergeBuilder private( private val targetTable: DeltaTable, private val source: DataFrame, private val onCondition: Column, private val whenMatchedClauses: Seq[proto.MergeIntoTable.Action], private val whenNotMatchedClauses: Seq[proto.MergeIntoTable.Action], private val whenNotMatchedBySourceClauses: Seq[proto.MergeIntoTable.Action], private val schemaEvolutionEnabled: Boolean) { // Schema Evolution is off by default in Merge. def this( targetTable: DeltaTable, source: DataFrame, onCondition: Column, whenMatchedClauses: Seq[proto.MergeIntoTable.Action], whenNotMatchedClauses: Seq[proto.MergeIntoTable.Action], whenNotMatchedBySourceClauses: Seq[proto.MergeIntoTable.Action]) = this(targetTable, source, onCondition, whenMatchedClauses, whenNotMatchedClauses, whenNotMatchedBySourceClauses, schemaEvolutionEnabled = false) /** * Build the actions to perform when the merge condition was matched. This returns * [[DeltaMergeMatchedActionBuilder]] object which can be used to specify how * to update or delete the matched target table row with the source row. * * @since 4.0.0 */ def whenMatched(): DeltaMergeMatchedActionBuilder = { DeltaMergeMatchedActionBuilder(this, None) } /** * Build the actions to perform when the merge condition was matched and * the given `condition` is true. This returns [[DeltaMergeMatchedActionBuilder]] object * which can be used to specify how to update or delete the matched target table row with the * source row. * * @param condition boolean expression as a SQL formatted string * @since 4.0.0 */ def whenMatched(condition: String): DeltaMergeMatchedActionBuilder = { whenMatched(expr(condition)) } /** * Build the actions to perform when the merge condition was matched and * the given `condition` is true. This returns a [[DeltaMergeMatchedActionBuilder]] object * which can be used to specify how to update or delete the matched target table row with the * source row. * * @param condition boolean expression as a Column object * @since 4.0.0 */ def whenMatched(condition: Column): DeltaMergeMatchedActionBuilder = { DeltaMergeMatchedActionBuilder(this, Some(condition)) } /** * Build the action to perform when the merge condition was not matched. This returns * [[DeltaMergeNotMatchedActionBuilder]] object which can be used to specify how * to insert the new sourced row into the target table. * @since 4.0.0 */ def whenNotMatched(): DeltaMergeNotMatchedActionBuilder = { DeltaMergeNotMatchedActionBuilder(this, None) } /** * Build the actions to perform when the merge condition was not matched and * the given `condition` is true. This returns [[DeltaMergeNotMatchedActionBuilder]] object * which can be used to specify how to insert the new sourced row into the target table. * * @param condition boolean expression as a SQL formatted string * @since 4.0.0 */ def whenNotMatched(condition: String): DeltaMergeNotMatchedActionBuilder = { whenNotMatched(expr(condition)) } /** * Build the actions to perform when the merge condition was not matched and * the given `condition` is true. This returns [[DeltaMergeNotMatchedActionBuilder]] object * which can be used to specify how to insert the new sourced row into the target table. * * @param condition boolean expression as a Column object * @since 4.0.0 */ def whenNotMatched(condition: Column): DeltaMergeNotMatchedActionBuilder = { DeltaMergeNotMatchedActionBuilder(this, Some(condition)) } /** * Build the actions to perform when the merge condition was not matched by the source. This * returns [[DeltaMergeNotMatchedBySourceActionBuilder]] object which can be used to specify how * to update or delete the target table row. * @since 4.0.0 */ def whenNotMatchedBySource(): DeltaMergeNotMatchedBySourceActionBuilder = { DeltaMergeNotMatchedBySourceActionBuilder(this, None) } /** * Build the actions to perform when the merge condition was not matched by the source and the * given `condition` is true. This returns [[DeltaMergeNotMatchedBySourceActionBuilder]] object * which can be used to specify how to update or delete the target table row. * * @param condition boolean expression as a SQL formatted string * @since 4.0.0 */ def whenNotMatchedBySource(condition: String): DeltaMergeNotMatchedBySourceActionBuilder = { whenNotMatchedBySource(expr(condition)) } /** * Build the actions to perform when the merge condition was not matched by the source and the * given `condition` is true. This returns [[DeltaMergeNotMatchedBySourceActionBuilder]] object * which can be used to specify how to update or delete the target table row . * * @param condition boolean expression as a Column object * @since 4.0.0 */ def whenNotMatchedBySource(condition: Column): DeltaMergeNotMatchedBySourceActionBuilder = { DeltaMergeNotMatchedBySourceActionBuilder(this, Some(condition)) } /** * Enable schema evolution for the merge operation. This allows the schema of the target * table/columns to be automatically updated based on the schema of the source table/columns. * * @since 4.0.0 */ def withSchemaEvolution(): DeltaMergeBuilder = { new DeltaMergeBuilder( this.targetTable, this.source, this.onCondition, this.whenMatchedClauses, this.whenNotMatchedClauses, this.whenNotMatchedBySourceClauses, schemaEvolutionEnabled = true) } /** * Execute the merge operation based on the built matched and not matched actions. * * @since 4.0.0 */ def execute(): DataFrame = { val sparkSession = targetTable.toDF.sparkSession val merge = proto.MergeIntoTable .newBuilder() .setTarget(targetTable.toDF.plan.getRoot) .setSource(source.plan.getRoot) .setCondition(toExpr(onCondition)) .addAllMatchedActions(whenMatchedClauses.asJava) .addAllNotMatchedActions(whenNotMatchedClauses.asJava) .addAllNotMatchedBySourceActions(whenNotMatchedBySourceClauses.asJava) .setWithSchemaEvolution(schemaEvolutionEnabled) val relation = proto.DeltaRelation.newBuilder().setMergeIntoTable(merge).build() val extension = com.google.protobuf.Any.pack(relation) val sparkRelation = spark_proto.Relation.newBuilder().setExtension(extension).build() val resultDf = sparkSession.newDataFrame(_.mergeFrom(sparkRelation)) val resultSchema = resultDf.schema // Ensure this is actually executed instead of just passing the DataFrame directly back to // the caller, in case they just drop it. The return type used to be Unit so dropping is // likely common. val result = resultDf.collect() sparkSession.createDataFrame(Arrays.asList(result: _*), resultSchema) } /** * :: Unstable :: * * Private method for internal usage only. Do not call this directly. */ @Unstable private[delta] def withWhenMatchedClause( clause: proto.MergeIntoTable.Action): DeltaMergeBuilder = { new DeltaMergeBuilder( this.targetTable, this.source, this.onCondition, this.whenMatchedClauses :+ clause, this.whenNotMatchedClauses, this.whenNotMatchedBySourceClauses, this.schemaEvolutionEnabled) } /** * :: Unstable :: * * Private method for internal usage only. Do not call this directly. */ @Unstable private[delta] def withWhenNotMatchedClause( clause: proto.MergeIntoTable.Action): DeltaMergeBuilder = { new DeltaMergeBuilder( this.targetTable, this.source, this.onCondition, this.whenMatchedClauses, this.whenNotMatchedClauses :+ clause, this.whenNotMatchedBySourceClauses, this.schemaEvolutionEnabled) } /** * :: Unstable :: * * Private method for internal usage only. Do not call this directly. */ @Unstable private[delta] def withWhenNotMatchedBySourceClause( clause: proto.MergeIntoTable.Action): DeltaMergeBuilder = { new DeltaMergeBuilder( this.targetTable, this.source, this.onCondition, this.whenMatchedClauses, this.whenNotMatchedClauses, this.whenNotMatchedBySourceClauses :+ clause, this.schemaEvolutionEnabled) } } object DeltaMergeBuilder { /** * :: Unstable :: * * Private method for internal usage only. Do not call this directly. */ @Unstable private[delta] def apply( targetTable: DeltaTable, source: DataFrame, onCondition: Column): DeltaMergeBuilder = { new DeltaMergeBuilder(targetTable, source, onCondition, Nil, Nil, Nil) } } /** * Builder class to specify the actions to perform when a target table row has matched a * source row based on the given merge condition and optional match condition. * * See [[DeltaMergeBuilder]] for more information. * * @since 4.0.0 */ class DeltaMergeMatchedActionBuilder private( private val mergeBuilder: DeltaMergeBuilder, private val matchCondition: Option[Column]) { /** * Update the matched table rows based on the rules defined by `set`. * * @param set rules to update a row as a Scala map between target column names and * corresponding update expressions as Column objects. * @since 4.0.0 */ def update(set: Map[String, Column]): DeltaMergeBuilder = { addUpdateClause(set) } /** * Update the matched table rows based on the rules defined by `set`. * * @param set rules to update a row as a Scala map between target column names and * corresponding update expressions as SQL formatted strings. * @since 4.0.0 */ def updateExpr(set: Map[String, String]): DeltaMergeBuilder = { addUpdateClause(toStrColumnMap(set)) } /** * Update a matched table row based on the rules defined by `set`. * * @param set rules to update a row as a Java map between target column names and * corresponding expressions as Column objects. * @since 4.0.0 */ def update(set: java.util.Map[String, Column]): DeltaMergeBuilder = { addUpdateClause(set.asScala.toMap) } /** * Update a matched table row based on the rules defined by `set`. * * @param set rules to update a row as a Java map between target column names and * corresponding expressions as SQL formatted strings. * @since 4.0.0 */ def updateExpr(set: java.util.Map[String, String]): DeltaMergeBuilder = { addUpdateClause(toStrColumnMap(set.asScala.toMap)) } /** * Update all the columns of the matched table row with the values of the * corresponding columns in the source row. * * @since 4.0.0 */ def updateAll(): DeltaMergeBuilder = { val clause = proto.MergeIntoTable.Action .newBuilder() .setUpdateStarAction(proto.MergeIntoTable.Action.UpdateStarAction.newBuilder()) matchCondition.foreach(c => clause.setCondition(toExpr(c))) mergeBuilder.withWhenMatchedClause(clause.build()) } /** * Delete a matched row from the table. * * @since 4.0.0 */ def delete(): DeltaMergeBuilder = { val clause = proto.MergeIntoTable.Action .newBuilder() .setDeleteAction(proto.MergeIntoTable.Action.DeleteAction.newBuilder()) matchCondition.foreach(c => clause.setCondition(toExpr(c))) mergeBuilder.withWhenMatchedClause(clause.build()) } private def addUpdateClause(set: Map[String, Column]): DeltaMergeBuilder = { if (set.isEmpty && matchCondition.isEmpty) { // This is a catch all clause that doesn't update anything: we can ignore it. mergeBuilder } else { val assignments = set.map { case (field, value) => proto.Assignment.newBuilder().setField(toExpr(expr(field))).setValue(toExpr(value)).build() } val action = proto.MergeIntoTable.Action.UpdateAction .newBuilder() .addAllAssignments(assignments.asJava) val clause = proto.MergeIntoTable.Action .newBuilder() .setUpdateAction(action) matchCondition.foreach(c => clause.setCondition(toExpr(c))) mergeBuilder.withWhenMatchedClause(clause.build()) } } private def toStrColumnMap(map: Map[String, String]): Map[String, Column] = map.mapValues(functions.expr).toMap } object DeltaMergeMatchedActionBuilder { /** * :: Unstable :: * * Private method for internal usage only. Do not call this directly. */ @Unstable private[delta] def apply( mergeBuilder: DeltaMergeBuilder, matchCondition: Option[Column]): DeltaMergeMatchedActionBuilder = { new DeltaMergeMatchedActionBuilder(mergeBuilder, matchCondition) } } /** * Builder class to specify the actions to perform when a source row has not matched any target * Delta table row based on the merge condition, but has matched the additional condition * if specified. * * See [[DeltaMergeBuilder]] for more information. * * @since 4.0.0 */ class DeltaMergeNotMatchedActionBuilder private( private val mergeBuilder: DeltaMergeBuilder, private val notMatchCondition: Option[Column]) { /** * Insert a new row to the target table based on the rules defined by `values`. * * @param values rules to insert a row as a Scala map between target column names and * corresponding expressions as Column objects. * @since 4.0.0 */ def insert(values: Map[String, Column]): DeltaMergeBuilder = { addInsertClause(values) } /** * Insert a new row to the target table based on the rules defined by `values`. * * @param values rules to insert a row as a Scala map between target column names and * corresponding expressions as SQL formatted strings. * @since 4.0.0 */ def insertExpr(values: Map[String, String]): DeltaMergeBuilder = { addInsertClause(toStrColumnMap(values)) } /** * Insert a new row to the target table based on the rules defined by `values`. * * @param values rules to insert a row as a Java map between target column names and * corresponding expressions as Column objects. * @since 4.0.0 */ def insert(values: java.util.Map[String, Column]): DeltaMergeBuilder = { addInsertClause(values.asScala) } /** * Insert a new row to the target table based on the rules defined by `values`. * * @param values rules to insert a row as a Java map between target column names and * corresponding expressions as SQL formatted strings. * * @since 4.0.0 */ def insertExpr(values: java.util.Map[String, String]): DeltaMergeBuilder = { addInsertClause(toStrColumnMap(values.asScala)) } /** * Insert a new target Delta table row by assigning the target columns to the values of the * corresponding columns in the source row. * @since 4.0.0 */ def insertAll(): DeltaMergeBuilder = { val clause = proto.MergeIntoTable.Action .newBuilder() .setInsertStarAction(proto.MergeIntoTable.Action.InsertStarAction.newBuilder()) notMatchCondition.foreach(c => clause.setCondition(toExpr(c))) mergeBuilder.withWhenNotMatchedClause(clause.build()) } private def addInsertClause(setValues: Map[String, Column]): DeltaMergeBuilder = { val assignments = setValues.map { case (field, value) => proto.Assignment.newBuilder().setField(toExpr(expr(field))).setValue(toExpr(value)).build() } val action = proto.MergeIntoTable.Action.InsertAction .newBuilder() .addAllAssignments(assignments.asJava) val clause = proto.MergeIntoTable.Action .newBuilder() .setInsertAction(action) notMatchCondition.foreach(c => clause.setCondition(toExpr(c))) mergeBuilder.withWhenNotMatchedClause(clause.build()) } private def toStrColumnMap(map: Map[String, String]): Map[String, Column] = map.mapValues(functions.expr).toMap } object DeltaMergeNotMatchedActionBuilder { /** * :: Unstable :: * * Private method for internal usage only. Do not call this directly. */ @Unstable private[delta] def apply( mergeBuilder: DeltaMergeBuilder, notMatchCondition: Option[Column]): DeltaMergeNotMatchedActionBuilder = { new DeltaMergeNotMatchedActionBuilder(mergeBuilder, notMatchCondition) } } /** * Builder class to specify the actions to perform when a target table row has no match in the * source table based on the given merge condition and optional match condition. * * See [[DeltaMergeBuilder]] for more information. * * @since 4.0.0 */ class DeltaMergeNotMatchedBySourceActionBuilder private( private val mergeBuilder: DeltaMergeBuilder, private val notMatchBySourceCondition: Option[Column]) { /** * Update an unmatched target table row based on the rules defined by `set`. * * @param set rules to update a row as a Scala map between target column names and * corresponding update expressions as Column objects. * @since 4.0.0 */ def update(set: Map[String, Column]): DeltaMergeBuilder = { addUpdateClause(set) } /** * Update an unmatched target table row based on the rules defined by `set`. * * @param set rules to update a row as a Scala map between target column names and * corresponding update expressions as SQL formatted strings. * @since 4.0.0 */ def updateExpr(set: Map[String, String]): DeltaMergeBuilder = { addUpdateClause(toStrColumnMap(set)) } /** * Update an unmatched target table row based on the rules defined by `set`. * * @param set rules to update a row as a Java map between target column names and * corresponding expressions as Column objects. * @since 4.0.0 */ def update(set: java.util.Map[String, Column]): DeltaMergeBuilder = { addUpdateClause(set.asScala) } /** * Update an unmatched target table row based on the rules defined by `set`. * * @param set rules to update a row as a Java map between target column names and * corresponding expressions as SQL formatted strings. * @since 4.0.0 */ def updateExpr(set: java.util.Map[String, String]): DeltaMergeBuilder = { addUpdateClause(toStrColumnMap(set.asScala)) } /** * Delete an unmatched row from the target table. * @since 4.0.0 */ def delete(): DeltaMergeBuilder = { val clause = proto.MergeIntoTable.Action .newBuilder() .setDeleteAction(proto.MergeIntoTable.Action.DeleteAction.newBuilder()) notMatchBySourceCondition.foreach(c => clause.setCondition(toExpr(c))) mergeBuilder.withWhenNotMatchedBySourceClause(clause.build()) } private def addUpdateClause(set: Map[String, Column]): DeltaMergeBuilder = { if (set.isEmpty && notMatchBySourceCondition.isEmpty) { // This is a catch all clause that doesn't update anything: we can ignore it. mergeBuilder } else { val assignments = set.map { case (field, value) => proto.Assignment.newBuilder().setField(toExpr(expr(field))).setValue(toExpr(value)).build() } val action = proto.MergeIntoTable.Action.UpdateAction .newBuilder() .addAllAssignments(assignments.asJava) val clause = proto.MergeIntoTable.Action .newBuilder() .setUpdateAction(action) notMatchBySourceCondition.foreach(c => clause.setCondition(toExpr(c))) mergeBuilder.withWhenNotMatchedBySourceClause(clause.build()) } } private def toStrColumnMap(map: Map[String, String]): Map[String, Column] = map.mapValues(functions.expr).toMap } object DeltaMergeNotMatchedBySourceActionBuilder { /** * :: Unstable :: * * Private method for internal usage only. Do not call this directly. */ @Unstable private[delta] def apply( mergeBuilder: DeltaMergeBuilder, notMatchBySourceCondition: Option[Column]): DeltaMergeNotMatchedBySourceActionBuilder = { new DeltaMergeNotMatchedBySourceActionBuilder(mergeBuilder, notMatchBySourceCondition) } } ================================================ FILE: spark-connect/client/src/main/scala/io/delta/connect/tables/DeltaOptimizeBuilder.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables import scala.collection.JavaConverters._ import io.delta.connect.proto import io.delta.connect.spark.{proto => spark_proto} import org.apache.spark.annotation.Unstable import org.apache.spark.sql.{DataFrame, SparkSession} import org.apache.spark.sql.connect.ConnectConversions._ import org.apache.spark.sql.connect.delta.ImplicitProtoConversions._ /** * Builder class for constructing OPTIMIZE command and executing. * * @param sparkSession SparkSession to use for execution * @param tableIdentifier Id of the table on which to * execute the optimize * @param options Hadoop file system options for read and write. * @since 4.0.0 */ class DeltaOptimizeBuilder private( private val sparkSession: SparkSession, private val table: proto.DeltaTable) { private var partitionFilters: Seq[String] = Seq.empty /** * Apply partition filter on this optimize command builder to limit * the operation on selected partitions. * * @param partitionFilter The partition filter to apply * @return [[DeltaOptimizeBuilder]] with partition filter applied * @since 4.0.0 */ def where(partitionFilter: String): DeltaOptimizeBuilder = { this.partitionFilters = this.partitionFilters :+ partitionFilter this } /** * Compact the small files in selected partitions. * * @return DataFrame containing the OPTIMIZE execution metrics * @since 4.0.0 */ def executeCompaction(): DataFrame = { execute(Seq.empty) } /** * Z-Order the data in selected partitions using the given columns. * * @param columns Zero or more columns to order the data * using Z-Order curves * @return DataFrame containing the OPTIMIZE execution metrics * @since 4.0.0 */ @scala.annotation.varargs def executeZOrderBy(columns: String*): DataFrame = { execute(columns) } private def execute(zOrderBy: Seq[String]): DataFrame = { val optimize = proto.OptimizeTable .newBuilder() .setTable(table) .addAllPartitionFilters(partitionFilters.asJava) .addAllZorderColumns(zOrderBy.asJava) val relation = proto.DeltaRelation.newBuilder().setOptimizeTable(optimize).build() val extension = com.google.protobuf.Any.pack(relation) val sparkRelation = spark_proto.Relation.newBuilder().setExtension(extension).build() val result = sparkSession.newDataFrame(_.mergeFrom(sparkRelation)).collectResult() val data = try { result.toArray.toSeq.asJava } finally { result.close() } sparkSession.createDataFrame(data, result.schema) } } private[delta] object DeltaOptimizeBuilder { /** * :: Unstable :: * * Private method for internal usage only. Do not call this directly. */ @Unstable private[delta] def apply( sparkSession: SparkSession, table: proto.DeltaTable): DeltaOptimizeBuilder = { new DeltaOptimizeBuilder(sparkSession, table) } } ================================================ FILE: spark-connect/client/src/main/scala/io/delta/connect/tables/DeltaTable.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables import scala.collection.JavaConverters._ import io.delta.connect.proto import io.delta.connect.spark.{proto => spark_proto} import io.delta.tables.execution.{CreateTableOptions, ReplaceTableOptions} import org.apache.spark.annotation.Evolving import org.apache.spark.sql.{functions, AnalysisException, Column, DataFrame, Dataset, Row, SparkSession} import org.apache.spark.sql.catalyst.encoders.AgnosticEncoders.PrimitiveBooleanEncoder import org.apache.spark.sql.connect.ColumnNodeToProtoConverter.toExpr import org.apache.spark.sql.connect.ConnectConversions._ import org.apache.spark.sql.connect.delta.ImplicitProtoConversions._ /** * Main class for programmatically interacting with Delta tables. * You can create DeltaTable instances using the static methods. * {{{ * DeltaTable.forPath(sparkSession, pathToTheDeltaTable) * }}} * * @since 4.0.0 */ class DeltaTable private[tables]( private val df: Dataset[Row], private val table: proto.DeltaTable) extends Serializable { private def sparkSession: SparkSession = df.sparkSession /** * Apply an alias to the DeltaTable. This is similar to `Dataset.as(alias)` or * SQL `tableName AS alias`. * * @since 4.0.0 */ def as(alias: String): DeltaTable = new DeltaTable(df.as(alias), table) /** * Apply an alias to the DeltaTable. This is similar to `Dataset.as(alias)` or * SQL `tableName AS alias`. * * @since 4.0.0 */ def alias(alias: String): DeltaTable = as(alias) /** * Get a DataFrame (that is, Dataset[Row]) representation of this Delta table. * * @since 4.0.0 */ def toDF: Dataset[Row] = df /** * Helper method for the vacuum APIs. * * @param retentionHours The retention threshold in hours. Files required by the table for * reading versions earlier than this will be preserved and the * rest of them will be deleted. * * @since 4.0.0 */ private def executeVacuum(retentionHours: Option[Double]): DataFrame = { val vacuum = proto.VacuumTable .newBuilder() .setTable(table) retentionHours.foreach(vacuum.setRetentionHours) val command = proto.DeltaCommand .newBuilder() .setVacuumTable(vacuum) .build() execute(command) sparkSession.emptyDataFrame } /** * Recursively delete files and directories in the table that are not needed by the table for * maintaining older versions up to the given retention threshold. This method will return an * empty DataFrame on successful completion. * * @param retentionHours The retention threshold in hours. Files required by the table for * reading versions earlier than this will be preserved and the * rest of them will be deleted. * @since 4.0.0 */ def vacuum(retentionHours: Double): DataFrame = { executeVacuum(Some(retentionHours)) } /** * Recursively delete files and directories in the table that are not needed by the table for * maintaining older versions up to the given retention threshold. This method will return an * empty DataFrame on successful completion. * * note: This will use the default retention period of 7 days. * * @since 4.0.0 */ def vacuum(): DataFrame = { executeVacuum(None) } /** * Helper method for the history APIs. * * @param limit The number of previous commands to get history for. * * @since 4.0.0 */ private def executeHistory(limit: Option[Int]): DataFrame = { val describeHistory = proto.DescribeHistory .newBuilder() .setTable(table) val relation = proto.DeltaRelation.newBuilder().setDescribeHistory(describeHistory).build() val extension = com.google.protobuf.Any.pack(relation) val sparkRelation = spark_proto.Relation.newBuilder().setExtension(extension).build() val df = sparkSession.newDataFrame(_.mergeFrom(sparkRelation)) limit match { case Some(limit) => df.limit(limit) case None => df } } /** * Get the information of the latest `limit` commits on this table as a Spark DataFrame. * The information is in reverse chronological order. * * @param limit The number of previous commands to get history for. * * @since 4.0.0 */ def history(limit: Int): DataFrame = { executeHistory(Some(limit)) } /** * Get the information available commits on this table as a Spark DataFrame. * The information is in reverse chronological order. * * @since 4.0.0 */ def history(): DataFrame = { executeHistory(limit = None) } /** * :: Evolving :: * * Get the details of a Delta table such as the format, name, and size. * * @since 4.0.0 */ @Evolving def detail(): DataFrame = { val describeDetail = proto.DescribeDetail .newBuilder() .setTable(table) val relation = proto.DeltaRelation.newBuilder().setDescribeDetail(describeDetail).build() val extension = com.google.protobuf.Any.pack(relation) val sparkRelation = spark_proto.Relation.newBuilder().setExtension(extension).build() sparkSession.newDataFrame(_.mergeFrom(sparkRelation)) } /** * Helper method for the delete APIs. * * @param condition Boolean SQL expression. * * @since 4.0.0 */ private def executeDelete(condition: Option[Column]): Unit = { val delete = proto.DeleteFromTable .newBuilder() .setTarget(df.plan.getRoot) condition.foreach(c => delete.setCondition(toExpr(c))) val relation = proto.DeltaRelation.newBuilder().setDeleteFromTable(delete).build() val extension = com.google.protobuf.Any.pack(relation) val sparkRelation = spark_proto.Relation.newBuilder().setExtension(extension).build() sparkSession.newDataFrame(_.mergeFrom(sparkRelation)).collect() } /** * Delete data from the table that match the given `condition`. * * @param condition Boolean SQL expression. * * @since 4.0.0 */ def delete(condition: String): Unit = { delete(functions.expr(condition)) } /** * Delete data from the table that match the given `condition`. * * @param condition Boolean SQL expression. * * @since 4.0.0 */ def delete(condition: Column): Unit = { executeDelete(condition = Some(condition)) } /** * Delete data from the table. * * @since 4.0.0 */ def delete(): Unit = { executeDelete(condition = None) } /** * Optimize the data layout of the table. This returns * a [[DeltaOptimizeBuilder]] object that can be used to specify * the partition filter to limit the scope of optimize and * also execute different optimization techniques such as file * compaction or order data using Z-Order curves. * * See the [[DeltaOptimizeBuilder]] for a full description * of this operation. * * Scala example to run file compaction on a subset of * partitions in the table: * {{{ * deltaTable * .optimize() * .where("date='2021-11-18'") * .executeCompaction(); * }}} * * @since 4.0.0 */ def optimize(): DeltaOptimizeBuilder = { DeltaOptimizeBuilder(sparkSession, table) } /** * Helper method for the update APIs. * * @param condition boolean expression as Column object specifying which rows to update. * @param set rules to update a row as a Scala map between target column names and * corresponding update expressions as Column objects. * * @since 4.0.0 */ private def executeUpdate(condition: Option[Column], set: Map[String, Column]): Unit = { val assignments = set.toSeq.map { case (field, value) => proto.Assignment .newBuilder() .setField(toExpr(functions.expr(field))) .setValue(toExpr(value)) .build() } val update = proto.UpdateTable .newBuilder() .setTarget(df.plan.getRoot) .addAllAssignments(assignments.asJava) condition.foreach(c => update.setCondition(toExpr(c))) val relation = proto.DeltaRelation.newBuilder().setUpdateTable(update).build() val extension = com.google.protobuf.Any.pack(relation) val sparkRelation = spark_proto.Relation.newBuilder().setExtension(extension).build() sparkSession.newDataFrame(_.mergeFrom(sparkRelation)).collect() } /** * Update rows in the table based on the rules defined by `set`. * * Scala example to increment the column `data`. * {{{ * import org.apache.spark.sql.functions._ * * deltaTable.update(Map("data" -> col("data") + 1)) * }}} * * @param set rules to update a row as a Scala map between target column names and * corresponding update expressions as Column objects. * * @since 4.0.0 */ def update(set: Map[String, Column]): Unit = { executeUpdate(condition = None, set) } /** * Update rows in the table based on the rules defined by `set`. * * Java example to increment the column `data`. * {{{ * import org.apache.spark.sql.Column; * import org.apache.spark.sql.functions; * * deltaTable.update( * new HashMap() {{ * put("data", functions.col("data").plus(1)); * }} * ); * }}} * * @param set rules to update a row as a Java map between target column names and * corresponding update expressions as Column objects. * * @since 4.0.0 */ def update(set: java.util.Map[String, Column]): Unit = { update(set.asScala.asInstanceOf[Map[String, Column]]) } /** * Update data from the table on the rows that match the given `condition` * based on the rules defined by `set`. * * Scala example to increment the column `data`. * {{{ * import org.apache.spark.sql.functions._ * * deltaTable.update( * col("date") > "2018-01-01", * Map("data" -> col("data") + 1)) * }}} * * @param condition boolean expression as Column object specifying which rows to update. * @param set rules to update a row as a Scala map between target column names and * corresponding update expressions as Column objects. * * @since 4.0.0 */ def update(condition: Column, set: Map[String, Column]): Unit = { executeUpdate(Some(condition), set) } /** * Update data from the table on the rows that match the given `condition` * based on the rules defined by `set`. * * Java example to increment the column `data`. * {{{ * import org.apache.spark.sql.Column; * import org.apache.spark.sql.functions; * * deltaTable.update( * functions.col("date").gt("2018-01-01"), * new HashMap() {{ * put("data", functions.col("data").plus(1)); * }} * ); * }}} * * @param condition boolean expression as Column object specifying which rows to update. * @param set rules to update a row as a Java map between target column names and * corresponding update expressions as Column objects. * * @since 4.0.0 */ def update(condition: Column, set: java.util.Map[String, Column]): Unit = { executeUpdate(Some(condition), set.asScala.toMap) } /** * Update rows in the table based on the rules defined by `set`. * * Scala example to increment the column `data`. * {{{ * deltaTable.updateExpr(Map("data" -> "data + 1"))) * }}} * * @param set rules to update a row as a Scala map between target column names and * corresponding update expressions as SQL formatted strings. * * @since 4.0.0 */ def updateExpr(set: Map[String, String]): Unit = { update(toStrColumnMap(set)) } /** * Update rows in the table based on the rules defined by `set`. * * Java example to increment the column `data`. * {{{ * deltaTable.updateExpr( * new HashMap() {{ * put("data", "data + 1"); * }} * ); * }}} * * @param set rules to update a row as a Java map between target column names and * corresponding update expressions as SQL formatted strings. * * @since 4.0.0 */ def updateExpr(set: java.util.Map[String, String]): Unit = { update(toStrColumnMap(set.asScala.toMap)) } /** * Update data from the table on the rows that match the given `condition`, * which performs the rules defined by `set`. * * Scala example to increment the column `data`. * {{{ * deltaTable.update( * "date > '2018-01-01'", * Map("data" -> "data + 1")) * }}} * * @param condition boolean expression as SQL formatted string object specifying * which rows to update. * @param set rules to update a row as a Scala map between target column names and * corresponding update expressions as SQL formatted strings. * * @since 4.0.0 */ def updateExpr(condition: String, set: Map[String, String]): Unit = { executeUpdate(Some(functions.expr(condition)), toStrColumnMap(set)) } /** * Update data from the table on the rows that match the given `condition`, * which performs the rules defined by `set`. * * Java example to increment the column `data`. * {{{ * deltaTable.update( * "date > '2018-01-01'", * new HashMap() {{ * put("data", "data + 1"); * }} * ); * }}} * * @param condition boolean expression as SQL formatted string object specifying * which rows to update. * @param set rules to update a row as a Java map between target column names and * corresponding update expressions as SQL formatted strings. * * @since 4.0.0 */ def updateExpr(condition: String, set: java.util.Map[String, String]): Unit = { executeUpdate(Some(functions.expr(condition)), toStrColumnMap(set.asScala.toMap)) } /** * Merge data from the `source` DataFrame based on the given merge `condition`. This returns * a [[DeltaMergeBuilder]] object that can be used to specify the update, delete, or insert * actions to be performed on rows based on whether the rows matched the condition or not. * * See the [[DeltaMergeBuilder]] for a full description of this operation and what combinations of * update, delete and insert operations are allowed. * * Scala example to update a key-value Delta table with new key-values from a source DataFrame: * {{{ * deltaTable * .as("target") * .merge( * source.as("source"), * "target.key = source.key") * .whenMatched * .updateExpr(Map( * "value" -> "source.value")) * .whenNotMatched * .insertExpr(Map( * "key" -> "source.key", * "value" -> "source.value")) * .execute() * }}} * * Java example to update a key-value Delta table with new key-values from a source DataFrame: * {{{ * deltaTable * .as("target") * .merge( * source.as("source"), * "target.key = source.key") * .whenMatched * .updateExpr( * new HashMap() {{ * put("value" -> "source.value"); * }}) * .whenNotMatched * .insertExpr( * new HashMap() {{ * put("key", "source.key"); * put("value", "source.value"); * }}) * .execute(); * }}} * * @param source source Dataframe to be merged. * @param condition boolean expression as SQL formatted string * @since 4.0.0 */ def merge(source: DataFrame, condition: String): DeltaMergeBuilder = { merge(source, functions.expr(condition)) } /** * Merge data from the `source` DataFrame based on the given merge `condition`. This returns * a [[DeltaMergeBuilder]] object that can be used to specify the update, delete, or insert * actions to be performed on rows based on whether the rows matched the condition or not. * * See the [[DeltaMergeBuilder]] for a full description of this operation and what combinations of * update, delete and insert operations are allowed. * * Scala example to update a key-value Delta table with new key-values from a source DataFrame: * {{{ * deltaTable * .as("target") * .merge( * source.as("source"), * "target.key = source.key") * .whenMatched * .updateExpr(Map( * "value" -> "source.value")) * .whenNotMatched * .insertExpr(Map( * "key" -> "source.key", * "value" -> "source.value")) * .execute() * }}} * * Java example to update a key-value Delta table with new key-values from a source DataFrame: * {{{ * deltaTable * .as("target") * .merge( * source.as("source"), * "target.key = source.key") * .whenMatched * .updateExpr( * new HashMap() {{ * put("value" -> "source.value") * }}) * .whenNotMatched * .insertExpr( * new HashMap() {{ * put("key", "source.key"); * put("value", "source.value"); * }}) * .execute() * }}} * * @param source source Dataframe to be merged. * @param condition boolean expression as a Column object * @since 4.0.0 */ def merge(source: DataFrame, condition: Column): DeltaMergeBuilder = { DeltaMergeBuilder(this, source, condition) } private def executeClone( target: String, isShallow: Boolean, replace: Boolean, properties: Map[String, String], versionAsOf: Option[Int] = None, timestampAsOf: Option[String] = None): DeltaTable = { val clone = proto.CloneTable .newBuilder() .setTable(table) .setTarget(target) .setIsShallow(isShallow) .setReplace(replace) .putAllProperties(properties.asJava) versionAsOf.foreach(clone.setVersion) timestampAsOf.foreach(clone.setTimestamp) val command = proto.DeltaCommand.newBuilder().setCloneTable(clone).build() execute(command) DeltaTable.forPath(sparkSession, target) } /** * Clone a DeltaTable to a given destination to mirror the existing table's data and metadata. * * Specifying properties here means that the target will override any properties with the same key * in the source table with the user-defined properties. * * An example would be * {{{ * io.delta.tables.DeltaTable.clone( * "/some/path/to/table", * true, * true, * Map("foo" -> "bar")) * }}} * * @param target The path or table name to create the clone. * @param isShallow Whether to create a shallow clone or a deep clone. * @param replace Whether to replace the destination with the clone command. * @param properties The table properties to override in the clone. * * @since 4.0.0 */ def clone( target: String, isShallow: Boolean, replace: Boolean, properties: Map[String, String]): DeltaTable = { executeClone(target, isShallow, replace, properties, versionAsOf = None, timestampAsOf = None) } /** * Clone a DeltaTable to a given destination to mirror the existing table's data and metadata. * * An example would be * {{{ * io.delta.tables.DeltaTable.clone( * "/some/path/to/table", * true, * true) * }}} * * @param target The path or table name to create the clone. * @param isShallow Whether to create a shallow clone or a deep clone. * @param replace Whether to replace the destination with the clone command. * * @since 4.0.0 */ def clone(target: String, isShallow: Boolean, replace: Boolean): DeltaTable = { clone(target, isShallow, replace, properties = Map.empty[String, String]) } /** * Clone a DeltaTable to a given destination to mirror the existing table's data and metadata. * * An example would be * {{{ * io.delta.tables.DeltaTable.clone( * "/some/path/to/table", * true) * }}} * * @param target The path or table name to create the clone. * @param isShallow Whether to create a shallow clone or a deep clone. * * @since 4.0.0 */ def clone(target: String, isShallow: Boolean): DeltaTable = { clone(target, isShallow, replace = false) } /** * Clone a DeltaTable at a specific version to a given destination to mirror the existing * table's data and metadata at that version. * * Specifying properties here means that the target will override any properties with the same key * in the source table with the user-defined properties. * * An example would be * {{{ * io.delta.tables.DeltaTable.cloneAtVersion( * 5, * "/some/path/to/table", * true, * true, * Map("foo" -> "bar")) * }}} * * @param version The version of this table to clone from. * @param target The path or table name to create the clone. * @param isShallow Whether to create a shallow clone or a deep clone. * @param replace Whether to replace the destination with the clone command. * @param properties The table properties to override in the clone. * * @since 4.0.0 */ def cloneAtVersion( version: Int, target: String, isShallow: Boolean, replace: Boolean, properties: Map[String, String]): DeltaTable = { executeClone( target, isShallow, replace, properties, versionAsOf = Some(version), timestampAsOf = None) } /** * Clone a DeltaTable at a specific version to a given destination to mirror the existing * table's data and metadata at that version. * * An example would be * {{{ * io.delta.tables.DeltaTable.cloneAtVersion( * 5, * "/some/path/to/table", * true, * true) * }}} * * @param version The version of this table to clone from. * @param target The path or table name to create the clone. * @param isShallow Whether to create a shallow clone or a deep clone. * @param replace Whether to replace the destination with the clone command. * * @since 4.0.0 */ def cloneAtVersion( version: Int, target: String, isShallow: Boolean, replace: Boolean): DeltaTable = { cloneAtVersion(version, target, isShallow, replace, properties = Map.empty[String, String]) } /** * Clone a DeltaTable at a specific version to a given destination to mirror the existing * table's data and metadata at that version. * * An example would be * {{{ * io.delta.tables.DeltaTable.cloneAtVersion( * 5, * "/some/path/to/table", * true) * }}} * * @param version The version of this table to clone from. * @param target The path or table name to create the clone. * @param isShallow Whether to create a shallow clone or a deep clone. * * @since 4.0.0 */ def cloneAtVersion(version: Int, target: String, isShallow: Boolean): DeltaTable = { cloneAtVersion(version, target, isShallow, replace = false) } /** * Clone a DeltaTable at a specific timestamp to a given destination to mirror the existing * table's data and metadata at that timestamp. * * Timestamp can be of the format yyyy-MM-dd or yyyy-MM-dd HH:mm:ss. * * Specifying properties here means that the target will override any properties with the same key * in the source table with the user-defined properties. * * An example would be * {{{ * io.delta.tables.DeltaTable.cloneAtTimestamp( * "2019-01-01", * "/some/path/to/table", * true, * true, * Map("foo" -> "bar")) * }}} * * @param timestamp The timestamp of this table to clone from. * @param target The path or table name to create the clone. * @param isShallow Whether to create a shallow clone or a deep clone. * @param replace Whether to replace the destination with the clone command. * @param properties The table properties to override in the clone. * * @since 4.0.0 */ def cloneAtTimestamp( timestamp: String, target: String, isShallow: Boolean, replace: Boolean, properties: Map[String, String]): DeltaTable = { executeClone( target, isShallow, replace, properties, versionAsOf = None, timestampAsOf = Some(timestamp)) } /** * Clone a DeltaTable at a specific timestamp to a given destination to mirror the existing * table's data and metadata at that timestamp. * * Timestamp can be of the format yyyy-MM-dd or yyyy-MM-dd HH:mm:ss. * * An example would be * {{{ * io.delta.tables.DeltaTable.cloneAtTimestamp( * "2019-01-01", * "/some/path/to/table", * true, * true) * }}} * * @param timestamp The timestamp of this table to clone from. * @param target The path or table name to create the clone. * @param isShallow Whether to create a shallow clone or a deep clone. * @param replace Whether to replace the destination with the clone command. * * @since 4.0.0 */ def cloneAtTimestamp( timestamp: String, target: String, isShallow: Boolean, replace: Boolean): DeltaTable = { cloneAtTimestamp(timestamp, target, isShallow, replace, properties = Map.empty[String, String]) } /** * Clone a DeltaTable at a specific timestamp to a given destination to mirror the existing * table's data and metadata at that timestamp. * * Timestamp can be of the format yyyy-MM-dd or yyyy-MM-dd HH:mm:ss. * * An example would be * {{{ * io.delta.tables.DeltaTable.cloneAtTimestamp( * "2019-01-01", * "/some/path/to/table", * true) * }}} * * @param timestamp The timestamp of this table to clone from. * @param target The path or table name to create the clone. * @param isShallow Whether to create a shallow clone or a deep clone. * * @since 4.0.0 */ def cloneAtTimestamp(timestamp: String, target: String, isShallow: Boolean): DeltaTable = { cloneAtTimestamp(timestamp, target, isShallow, replace = false) } /** * Helper method for the restoreToVersion and restoreToTimestamp APIs. * * @param version The version number of the older version of the table to restore to. * @param timestamp The timestamp of the older version of the table to restore to. * * @since 4.0.0 */ private def executeRestore(version: Option[Long], timestamp: Option[String]): DataFrame = { val restore = proto.RestoreTable .newBuilder() .setTable(table) version.foreach(restore.setVersion) timestamp.foreach(restore.setTimestamp) val relation = proto.DeltaRelation.newBuilder().setRestoreTable(restore).build() val extension = com.google.protobuf.Any.pack(relation) val sparkRelation = spark_proto.Relation.newBuilder().setExtension(extension).build() val result = sparkSession.newDataFrame(_.mergeFrom(sparkRelation)).collectResult() val data = try { result.toArray.toSeq.asJava } finally { result.close() } sparkSession.createDataFrame(data, result.schema) } /** * Restore the DeltaTable to an older version of the table specified by version number. * * An example would be * {{{ io.delta.tables.DeltaTable.restoreToVersion(7) }}} * * @since 4.0.0 */ def restoreToVersion(version: Long): DataFrame = { executeRestore(version = Some(version), timestamp = None) } /** * Restore the DeltaTable to an older version of the table specified by a timestamp. * * Timestamp can be of the format yyyy-MM-dd or yyyy-MM-dd HH:mm:ss * * An example would be * {{{ io.delta.tables.DeltaTable.restoreToTimestamp("2019-01-01") }}} * * @since 4.0.0 */ def restoreToTimestamp(timestamp: String): DataFrame = { executeRestore(version = None, timestamp = Some(timestamp)) } /** * Converts a map of strings to expressions as SQL formatted string * into a map of strings to Column objects. * * @param map A map where the value is an expression as SQL formatted string. * @return A map where the value is a Column object created from the expression. */ private def toStrColumnMap(map: Map[String, String]): Map[String, Column] = { map.toSeq.map { case (k, v) => k -> functions.expr(v) }.toMap } /** * Generate a manifest for the given Delta Table * * @param mode Specifies the mode for the generation of the manifest. * The valid modes are as follows (not case sensitive): * - "symlink_format_manifest" : This will generate manifests in symlink format * for Presto and Athena read support. * See the online documentation for more information. * @since 4.0.0 */ def generate(mode: String): Unit = { val generate = proto.Generate .newBuilder() .setTable(table) .setMode(mode) val command = proto.DeltaCommand.newBuilder().setGenerate(generate).build() execute(command) } /** * Updates the protocol version of the table to leverage new features. Upgrading the reader * version will prevent all clients that have an older version of Delta Lake from accessing this * table. Upgrading the writer version will prevent older versions of Delta Lake to write to this * table. The reader or writer version cannot be downgraded. * * See online documentation and Delta's protocol specification at PROTOCOL.md for more details. * * @since 4.0.0 */ def upgradeTableProtocol(readerVersion: Int, writerVersion: Int): Unit = { val upgrade = proto.UpgradeTableProtocol .newBuilder() .setTable(table) .setReaderVersion(readerVersion) .setWriterVersion(writerVersion) val command = proto.DeltaCommand.newBuilder().setUpgradeTableProtocol(upgrade).build() execute(command) } /** * Modify the protocol to add a supported feature, and if the table does not support table * features, upgrade the protocol automatically. In such a case when the provided feature is * writer-only, the table's writer version will be upgraded to `7`, and when the provided * feature is reader-writer, both reader and writer versions will be upgraded, to `(3, 7)`. * * See online documentation and Delta's protocol specification at PROTOCOL.md for more details. * * @since 4.0.0 */ def addFeatureSupport(featureName: String): Unit = { val addFeatureSupport = proto.AddFeatureSupport .newBuilder() .setTable(table) .setFeatureName(featureName) val command = proto.DeltaCommand.newBuilder().setAddFeatureSupport(addFeatureSupport).build() execute(command) } private def executeDropFeature(featureName: String, truncateHistory: Option[Boolean]): Unit = { val dropFeatureSupport = proto.DropFeatureSupport .newBuilder() .setTable(table) .setFeatureName(featureName) truncateHistory.foreach(dropFeatureSupport.setTruncateHistory) val command = proto.DeltaCommand.newBuilder().setDropFeatureSupport(dropFeatureSupport).build() execute(command) } /** * Modify the protocol to drop a supported feature. The operation always normalizes the * resulting protocol. Protocol normalization is the process of converting a table features * protocol to the weakest possible form. This primarily refers to converting a table features * protocol to a legacy protocol. A table features protocol can be represented with the legacy * representation only when the feature set of the former exactly matches a legacy protocol. * Normalization can also decrease the reader version of a table features protocol when it is * higher than necessary. For example: * * (1, 7, None, {AppendOnly, Invariants, CheckConstraints}) -> (1, 3) * (3, 7, None, {RowTracking}) -> (1, 7, RowTracking) * * The dropFeatureSupport method can be used as follows: * {{{ * io.delta.tables.DeltaTable.dropFeatureSupport("rowTracking") * }}} * * See online documentation for more details. * * @param featureName The name of the feature to drop. * @param truncateHistory Whether to truncate history before downgrading the protocol. * @return None. * @since 4.0.0 */ def dropFeatureSupport(featureName: String, truncateHistory: Boolean): Unit = { executeDropFeature(featureName, Some(truncateHistory)) } /** * Modify the protocol to drop a supported feature. The operation always normalizes the * resulting protocol. Protocol normalization is the process of converting a table features * protocol to the weakest possible form. This primarily refers to converting a table features * protocol to a legacy protocol. A table features protocol can be represented with the legacy * representation only when the feature set of the former exactly matches a legacy protocol. * Normalization can also decrease the reader version of a table features protocol when it is * higher than necessary. For example: * * (1, 7, None, {AppendOnly, Invariants, CheckConstraints}) -> (1, 3) * (3, 7, None, {RowTracking}) -> (1, 7, RowTracking) * * The dropFeatureSupport method can be used as follows: * {{{ * io.delta.tables.DeltaTable.dropFeatureSupport("rowTracking") * }}} * * Note, this command will not truncate history. * * See online documentation for more details. * * * @param featureName The name of the feature to drop. * @return None. * @since 4.0.0 */ def dropFeatureSupport(featureName: String): Unit = { executeDropFeature(featureName, None) } private def execute(command: proto.DeltaCommand): Unit = { val extension = com.google.protobuf.Any.pack(command) val sparkCommand = spark_proto.Command .newBuilder() .setExtension(extension) .build() sparkSession.execute(sparkCommand) } } /** * Companion object to create DeltaTable instances. * * {{{ * DeltaTable.forPath(sparkSession, pathToTheDeltaTable) * }}} * * @since 4.0.0 */ object DeltaTable { /** * Helper method to get the active SparkSession. * * @return The active SparkSession if one exists. * @throws IllegalArgumentException if no active SparkSession is found. */ private def getActiveSparkSession(): SparkSession = { SparkSession.getActiveSession.getOrElse { throw new IllegalArgumentException("Could not find active SparkSession") } } /** * Instantiate a [[DeltaTable]] object representing the data at the given path, If the given * path is invalid (i.e. either no table exists or an existing table is not a Delta table), * it throws a `not a Delta table` error. * * Note: This uses the active SparkSession in the current thread to read the table data. Hence, * this throws error if active SparkSession has not been set, that is, * `SparkSession.getActiveSession()` is empty. * * @since 4.0.0 */ def forPath(path: String): DeltaTable = { forPath(getActiveSparkSession(), path) } /** * Instantiate a [[DeltaTable]] object representing the data at the given path, If the given * path is invalid (i.e. either no table exists or an existing table is not a Delta table), * it throws a `not a Delta table` error. * * @since 4.0.0 */ def forPath(sparkSession: SparkSession, path: String): DeltaTable = { forPath(sparkSession, path, Map.empty[String, String]) } /** * Instantiate a [[DeltaTable]] object representing the data at the given path, If the given * path is invalid (i.e. either no table exists or an existing table is not a Delta table), * it throws a `not a Delta table` error. * * @param hadoopConf Hadoop configuration starting with "fs." or "dfs." will be picked up * by `DeltaTable` to access the file system when executing queries. * Other configurations will not be allowed. * * {{{ * val hadoopConf = Map( * "fs.s3a.access.key" -> "", * "fs.s3a.secret.key" -> "" * ) * DeltaTable.forPath(spark, "/path/to/table", hadoopConf) * }}} * * @since 4.0.0 */ def forPath( sparkSession: SparkSession, path: String, hadoopConf: scala.collection.Map[String, String]): DeltaTable = { val table = proto.DeltaTable .newBuilder() .setPath( proto.DeltaTable.Path .newBuilder().setPath(path) .putAllHadoopConf(hadoopConf.asJava)) .build() forTable(sparkSession, table) } /** * Java friendly API to instantiate a [[DeltaTable]] object representing the data at the given * path, If the given path is invalid (i.e. either no table exists or an existing table is not a * Delta table), it throws a `not a Delta table` error. * * @param hadoopConf Hadoop configuration starting with "fs." or "dfs." will be picked up * by `DeltaTable` to access the file system when executing queries. * Other configurations will be ignored. * * {{{ * val hadoopConf = Map( * "fs.s3a.access.key" -> "", * "fs.s3a.secret.key", "" * ) * DeltaTable.forPath(spark, "/path/to/table", hadoopConf) * }}} * * @since 4.0.0 */ def forPath( sparkSession: SparkSession, path: String, hadoopConf: java.util.Map[String, String]): DeltaTable = { val fsOptions = hadoopConf.asScala.toMap forPath(sparkSession, path, fsOptions) } /** * Instantiate a [[DeltaTable]] object using the given table name. If the given * tableOrViewName is invalid (i.e. either no table exists or an existing table is not a * Delta table), it throws a `not a Delta table` error. Note: Passing a view name will also * result in this error as views are not supported. * * The given tableOrViewName can also be the absolute path of a delta datasource (i.e. * delta.`path`), If so, instantiate a [[DeltaTable]] object representing the data at * the given path (consistent with the [[forPath]]). * * Note: This uses the active SparkSession in the current thread to read the table data. Hence, * this throws error if active SparkSession has not been set, that is, * `SparkSession.getActiveSession()` is empty. * * @since 4.0.0 */ def forName(tableOrViewName: String): DeltaTable = { forName(getActiveSparkSession(), tableOrViewName) } /** * Instantiate a [[DeltaTable]] object using the given table name using the given * SparkSession. If the given tableName is invalid (i.e. either no table exists or an * existing table is not a Delta table), it throws a `not a Delta table` error. Note: * Passing a view name will also result in this error as views are not supported. * * The given tableName can also be the absolute path of a delta datasource (i.e. * delta.`path`), If so, instantiate a [[DeltaTable]] object representing the data at * the given path (consistent with the [[forPath]]). * * @since 4.0.0 */ def forName(sparkSession: SparkSession, tableName: String): DeltaTable = { val table = proto.DeltaTable .newBuilder() .setTableOrViewName(tableName) .build() forTable(sparkSession, table) } private def forTable(sparkSession: SparkSession, table: proto.DeltaTable): DeltaTable = { val relation = proto.DeltaRelation .newBuilder() .setScan(proto.Scan.newBuilder().setTable(table)) .build() val extension = com.google.protobuf.Any.pack(relation) val sparkRelation = spark_proto.Relation.newBuilder().setExtension(extension).build() val df = sparkSession.newDataFrame(_.mergeFrom(sparkRelation)) new DeltaTable(df, table) } /** * Check if the provided `identifier` string, in this case a file path, * is the root of a Delta table using the given SparkSession. * * An example would be * {{{ * DeltaTable.isDeltaTable(spark, "path/to/table") * }}} * * @since 4.0.0 */ def isDeltaTable(sparkSession: SparkSession, identifier: String): Boolean = { val relation = proto.DeltaRelation .newBuilder() .setIsDeltaTable(proto.IsDeltaTable.newBuilder().setPath(identifier)) .build() val extension = com.google.protobuf.Any.pack(relation) val sparkRelation = spark_proto.Relation.newBuilder().setExtension(extension).build() sparkSession.newDataset(PrimitiveBooleanEncoder)(_.mergeFrom(sparkRelation)).head() } /** * Check if the provided `identifier` string, in this case a file path, * is the root of a Delta table. * * Note: This uses the active SparkSession in the current thread to search for the table. Hence, * this throws error if active SparkSession has not been set, that is, * `SparkSession.getActiveSession()` is empty. * * An example would be * {{{ * DeltaTable.isDeltaTable(spark, "/path/to/table") * }}} * * @since 4.0.0 */ def isDeltaTable(identifier: String): Boolean = { isDeltaTable(getActiveSparkSession(), identifier) } /** * :: Evolving :: * * Return an instance of [[DeltaTableBuilder]] to create a Delta table, * error if the table exists (the same as SQL `CREATE TABLE`). * Refer to [[DeltaTableBuilder]] for more details. * * Note: This uses the active SparkSession in the current thread to read the table data. Hence, * this throws error if active SparkSession has not been set, that is, * `SparkSession.getActiveSession()` is empty. * * @since 4.0.0 */ @Evolving def create(): DeltaTableBuilder = { create(getActiveSparkSession()) } /** * :: Evolving :: * * Return an instance of [[DeltaTableBuilder]] to create a Delta table, * error if the table exists (the same as SQL `CREATE TABLE`). * Refer to [[DeltaTableBuilder]] for more details. * * @param spark sparkSession sparkSession passed by the user * @since 4.0.0 */ @Evolving def create(spark: SparkSession): DeltaTableBuilder = { new DeltaTableBuilder(spark, CreateTableOptions(ifNotExists = false)) } /** * :: Evolving :: * * Return an instance of [[DeltaTableBuilder]] to create a Delta table, * if it does not exists (the same as SQL `CREATE TABLE IF NOT EXISTS`). * Refer to [[DeltaTableBuilder]] for more details. * * Note: This uses the active SparkSession in the current thread to read the table data. Hence, * this throws error if active SparkSession has not been set, that is, * `SparkSession.getActiveSession()` is empty. * * @since 4.0.0 */ @Evolving def createIfNotExists(): DeltaTableBuilder = { createIfNotExists(getActiveSparkSession()) } /** * :: Evolving :: * * Return an instance of [[DeltaTableBuilder]] to create a Delta table, * if it does not exists (the same as SQL `CREATE TABLE IF NOT EXISTS`). * Refer to [[DeltaTableBuilder]] for more details. * * @param spark sparkSession sparkSession passed by the user * @since 4.0.0 */ @Evolving def createIfNotExists(spark: SparkSession): DeltaTableBuilder = { new DeltaTableBuilder(spark, CreateTableOptions(ifNotExists = true)) } /** * :: Evolving :: * * Return an instance of [[DeltaTableBuilder]] to replace a Delta table, * error if the table doesn't exist (the same as SQL `REPLACE TABLE`) * Refer to [[DeltaTableBuilder]] for more details. * * Note: This uses the active SparkSession in the current thread to read the table data. Hence, * this throws error if active SparkSession has not been set, that is, * `SparkSession.getActiveSession()` is empty. * * @since 4.0.0 */ @Evolving def replace(): DeltaTableBuilder = { replace(getActiveSparkSession()) } /** * :: Evolving :: * * Return an instance of [[DeltaTableBuilder]] to replace a Delta table, * error if the table doesn't exist (the same as SQL `REPLACE TABLE`) * Refer to [[DeltaTableBuilder]] for more details. * * @param spark sparkSession sparkSession passed by the user * @since 4.0.0 */ @Evolving def replace(spark: SparkSession): DeltaTableBuilder = { new DeltaTableBuilder(spark, ReplaceTableOptions(orCreate = false)) } /** * :: Evolving :: * * Return an instance of [[DeltaTableBuilder]] to replace a Delta table * or create table if not exists (the same as SQL `CREATE OR REPLACE TABLE`) * Refer to [[DeltaTableBuilder]] for more details. * * Note: This uses the active SparkSession in the current thread to read the table data. Hence, * this throws error if active SparkSession has not been set, that is, * `SparkSession.getActiveSession()` is empty. * * @since 4.0.0 */ @Evolving def createOrReplace(): DeltaTableBuilder = { createOrReplace(getActiveSparkSession()) } /** * :: Evolving :: * * Return an instance of [[DeltaTableBuilder]] to replace a Delta table, * or create table if not exists (the same as SQL `CREATE OR REPLACE TABLE`) * Refer to [[DeltaTableBuilder]] for more details. * * @param spark sparkSession sparkSession passed by the user. * @since 4.0.0 */ @Evolving def createOrReplace(spark: SparkSession): DeltaTableBuilder = { new DeltaTableBuilder(spark, ReplaceTableOptions(orCreate = true)) } /** * :: Evolving :: * * Return an instance of [[DeltaColumnBuilder]] to specify a column. * Refer to [[DeltaTableBuilder]] for examples and [[DeltaColumnBuilder]] detailed APIs. * * Note: This uses the active SparkSession in the current thread to read the table data. Hence, * this throws error if active SparkSession has not been set, that is, * `SparkSession.getActiveSession()` is empty. * * @param colName string the column name * @since 4.0.0 */ @Evolving def columnBuilder(colName: String): DeltaColumnBuilder = { new DeltaColumnBuilder(colName) } /** * :: Evolving :: * * Return an instance of [[DeltaColumnBuilder]] to specify a column. * Refer to [[DeltaTableBuilder]] for examples and [[DeltaColumnBuilder]] detailed APIs. * * @param spark sparkSession sparkSession passed by the user * @param colName string the column name * @since 4.0.0 */ @Evolving def columnBuilder(spark: SparkSession, colName: String): DeltaColumnBuilder = { new DeltaColumnBuilder(colName) } private[tables] def createAnalysisException(message: String): AnalysisException = { // TODO: We should refactor this to use DeltaErrors. Until then, we need to use a dummy Spark // error class to initialize the AnalysisException, which we then remove in the copy method. new AnalysisException( errorClass = "ALL_PARTITION_COLUMNS_NOT_ALLOWED", messageParameters = Map("message" -> message)).copy( message = message, errorClass = None, messageParameters = Map.empty) } } ================================================ FILE: spark-connect/client/src/main/scala/io/delta/connect/tables/DeltaTableBuilder.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables import scala.collection.JavaConverters._ import scala.collection.mutable import io.delta.connect.proto import io.delta.connect.spark.{proto => spark_proto} import io.delta.tables.execution.{CreateTableOptions, DeltaTableBuilderOptions, ReplaceTableOptions} import org.apache.spark.annotation.Evolving import org.apache.spark.sql.SparkSession import org.apache.spark.sql.connect.ConnectConversions._ import org.apache.spark.sql.connect.common.DataTypeProtoConverter import org.apache.spark.sql.connect.delta.ImplicitProtoConversions._ import org.apache.spark.sql.types.{DataType, StructField, StructType} /** * :: Evolving :: * * Builder to specify how to create / replace a Delta table. * You must specify the table name or the path before executing the builder. * You can specify the table columns, the partitioning columns, the location of the data, * the table comment and the property, and how you want to create / replace the Delta table. * * After executing the builder, an instance of [[DeltaTable]] is returned. * * Scala example to create a Delta table with generated columns, using the table name: * {{{ * val table: DeltaTable = DeltaTable.create() * .tableName("testTable") * .addColumn("c1", dataType = "INT", nullable = false) * .addColumn( * DeltaTable.columnBuilder("c2") * .dataType("INT") * .generatedAlwaysAs("c1 + 10") * .build() * ) * .addColumn( * DeltaTable.columnBuilder("c3") * .dataType("INT") * .comment("comment") * .nullable(true) * .build() * ) * .partitionedBy("c1", "c2") * .execute() * }}} * * Scala example to create a delta table using the location: * {{{ * val table: DeltaTable = DeltaTable.createIfNotExists(spark) * .location("/foo/`bar`") * .addColumn("c1", dataType = "INT", nullable = false) * .addColumn( * DeltaTable.columnBuilder(spark, "c2") * .dataType("INT") * .generatedAlwaysAs("c1 + 10") * .build() * ) * .addColumn( * DeltaTable.columnBuilder(spark, "c3") * .dataType("INT") * .comment("comment") * .nullable(true) * .build() * ) * .partitionedBy("c1", "c2") * .execute() * }}} * * Java Example to replace a table: * {{{ * DeltaTable table = DeltaTable.replace() * .tableName("db.table") * .addColumn("c1", "INT", false) * .addColumn( * DeltaTable.columnBuilder("c2") * .dataType("INT") * .generatedAlwaysBy("c1 + 10") * .build() * ) * .execute(); * }}} * * @since 4.0.0 */ @Evolving class DeltaTableBuilder private[tables]( spark: SparkSession, builderOption: DeltaTableBuilderOptions) { private var identifier: Option[String] = None private var partitioningColumns: Seq[String] = Nil private var clusteringColumns: Seq[String] = Nil private var columns: mutable.Seq[StructField] = mutable.Seq.empty private var location: Option[String] = None private var tblComment: Option[String] = None private var properties = Map.empty[String, String] /** * :: Evolving :: * * Specify the table name, optionally qualified with a database name [database_name.] table_name * * @param identifier string the table name * @since 4.0.0 */ @Evolving def tableName(identifier: String): DeltaTableBuilder = { this.identifier = Some(identifier) this } /** * :: Evolving :: * * Specify the table comment to describe the table. * * @param comment string table comment * @since 4.0.0 */ @Evolving def comment(comment: String): DeltaTableBuilder = { tblComment = Option(comment) this } /** * :: Evolving :: * * Specify the path to the directory where table data is stored, * which could be a path on distributed storage. * * @param location string the data location * @since 4.0.0 */ @Evolving def location(location: String): DeltaTableBuilder = { this.location = Option(location) this } /** * :: Evolving :: * * Specify a column. * * @param colName string the column name * @param dataType string the DDL data type * @since 4.0.0 */ @Evolving def addColumn(colName: String, dataType: String): DeltaTableBuilder = { addColumn( DeltaTable.columnBuilder(spark, colName).dataType(dataType).build() ) this } /** * :: Evolving :: * * Specify a column. * * @param colName string the column name * @param dataType dataType the DDL data type * @since 4.0.0 */ @Evolving def addColumn(colName: String, dataType: DataType): DeltaTableBuilder = { addColumn( DeltaTable.columnBuilder(spark, colName).dataType(dataType).build() ) this } /** * :: Evolving :: * * Specify a column. * * @param colName string the column name * @param dataType string the DDL data type * @param nullable boolean whether the column is nullable * @since 4.0.0 */ @Evolving def addColumn(colName: String, dataType: String, nullable: Boolean): DeltaTableBuilder = { addColumn( DeltaTable.columnBuilder(spark, colName).dataType(dataType).nullable(nullable).build() ) this } /** * :: Evolving :: * * Specify a column. * * @param colName string the column name * @param dataType dataType the DDL data type * @param nullable boolean whether the column is nullable * @since 4.0.0 */ @Evolving def addColumn(colName: String, dataType: DataType, nullable: Boolean): DeltaTableBuilder = { addColumn( DeltaTable.columnBuilder(spark, colName).dataType(dataType).nullable(nullable).build() ) this } /** * :: Evolving :: * * Specify a column. * * @param col structField the column struct * @since 4.0.0 */ @Evolving def addColumn(col: StructField): DeltaTableBuilder = { columns = columns :+ col this } /** * :: Evolving :: * * Specify columns with an existing schema. * * @param cols structType the existing schema for columns * @since 4.0.0 */ @Evolving def addColumns(cols: StructType): DeltaTableBuilder = { columns = columns ++ cols this } /** * :: Evolving :: * * Specify the columns to partition the output on the file system. * * Note: This should only include table columns already defined in schema. * * @param colNames string* column names for partitioning * @since 4.0.0 */ @Evolving @scala.annotation.varargs def partitionedBy(colNames: String*): DeltaTableBuilder = { partitioningColumns = colNames this } /** * :: Evolving :: * * Specify the columns to use as clustering keys for liquid clustering. * * Note: This should only include table columns already defined in schema. * * @param colNames string* column names for liquid clustering * @since 4.0.0 */ @Evolving @scala.annotation.varargs def clusterBy(colNames: String*): DeltaTableBuilder = { clusteringColumns = colNames this } /** * :: Evolving :: * * Specify a key-value pair to tag the table definition. * * @param key string the table property key * @param value string the table property value * @since 4.0.0 */ @Evolving def property(key: String, value: String): DeltaTableBuilder = { this.properties = this.properties + (key -> value) this } /** * :: Evolving :: * * Execute the command to create / replace a Delta table and returns a instance of [[DeltaTable]]. * * @since 4.0.0 */ @Evolving def execute(): DeltaTable = { if (identifier.isEmpty && location.isEmpty) { val exMessage = "Table name or location has to be specified" throw DeltaTable.createAnalysisException(exMessage) } val mode = builderOption match { case CreateTableOptions(ifNotExists) => if (ifNotExists) proto.CreateDeltaTable.Mode.MODE_CREATE_IF_NOT_EXISTS else proto.CreateDeltaTable.Mode.MODE_CREATE case ReplaceTableOptions(orCreate) => if (orCreate) proto.CreateDeltaTable.Mode.MODE_CREATE_OR_REPLACE else proto.CreateDeltaTable.Mode.MODE_REPLACE } val createDeltaTable = proto.CreateDeltaTable .newBuilder() .setMode(mode) .addAllPartitioningColumns(partitioningColumns.asJava) .addAllClusteringColumns(clusteringColumns.asJava) .putAllProperties(properties.asJava) identifier.foreach(createDeltaTable.setTableName) location.foreach(createDeltaTable.setLocation) tblComment.foreach(createDeltaTable.setComment) val protoColumns = columns.map { f => val builder = proto.CreateDeltaTable.Column .newBuilder() .setName(f.name) .setDataType(DataTypeProtoConverter.toConnectProtoType(f.dataType)) .setNullable(f.nullable) if (f.metadata.contains("delta.generationExpression")) { builder.setGeneratedAlwaysAs(f.metadata.getString("delta.generationExpression")) } if (f.metadata.contains("delta.identity.allowExplicitInsert")) { builder.setIdentityInfo( proto.CreateDeltaTable.Column.IdentityInfo .newBuilder() .setStart(f.metadata.getLong("delta.identity.start")) .setStep(f.metadata.getLong("delta.identity.step")) .setAllowExplicitInsert(f.metadata.getBoolean("delta.identity.allowExplicitInsert")) .build() ) } if (f.metadata.contains("comment")) { builder.setComment(f.metadata.getString("comment")) } builder.build() } createDeltaTable.addAllColumns(protoColumns.asJava) val command = proto.DeltaCommand.newBuilder().setCreateDeltaTable(createDeltaTable).build() val extension = com.google.protobuf.Any.pack(command) val sparkCommand = spark_proto.Command.newBuilder().setExtension(extension).build() spark.execute(sparkCommand) if (location.isDefined) { DeltaTable.forPath(spark, location.get) } else { DeltaTable.forName(spark, identifier.get) } } } ================================================ FILE: spark-connect/client/src/main/scala/io/delta/connect/tables/execution/DeltaTableBuilderOptions.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables.execution /** * DeltaTableBuilder option to indicate whether it's to create / replace the table. */ sealed trait DeltaTableBuilderOptions /** * Specify that the builder is to create a Delta table. * * @param ifNotExists boolean whether to ignore if the table already exists. */ case class CreateTableOptions(ifNotExists: Boolean) extends DeltaTableBuilderOptions /** * Specify that the builder is to replace a Delta table. * * @param orCreate boolean whether to create the table if the table doesn't exist. */ case class ReplaceTableOptions(orCreate: Boolean) extends DeltaTableBuilderOptions ================================================ FILE: spark-connect/client/src/main/scala-shims/spark-4.0/SparkStringUtilsShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.util import org.apache.spark.sql.catalyst.util.SparkStringUtils.sideBySide /** * Shim for SparkStringUtils to handle package relocation between Spark versions. * In Spark 4.0, SparkStringUtils was moved from org.apache.spark.sql.catalyst.util * to org.apache.spark.util. */ object SparkStringUtilsShims { def sideBySide(left: Seq[String], right: Seq[String]): Seq[String] = { SparkStringUtils.sideBySide(left, right) } } ================================================ FILE: spark-connect/client/src/main/scala-shims/spark-4.1/SparkStringUtilsShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.util import org.apache.spark.util.SparkStringUtils /** * Shim for SparkStringUtils to handle package relocation between Spark versions. * In Spark 4.1+, SparkStringUtils was moved from org.apache.spark.sql.catalyst.util * to org.apache.spark.util. */ object SparkStringUtilsShims { def sideBySide(left: Seq[String], right: Seq[String]): Seq[String] = { SparkStringUtils.sideBySide(left, right) } } ================================================ FILE: spark-connect/client/src/main/scala-shims/spark-4.2/SparkStringUtilsShims.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.catalyst.util import org.apache.spark.util.SparkStringUtils /** * Shim for SparkStringUtils to handle package relocation between Spark versions. * In Spark 4.2, SparkStringUtils is in org.apache.spark.util (same as Spark 4.1+). */ object SparkStringUtilsShims { def sideBySide(left: Seq[String], right: Seq[String]): Seq[String] = { SparkStringUtils.sideBySide(left, right) } } ================================================ FILE: spark-connect/client/src/test/scala/io/delta/connect/tables/DeltaMergeBuilderSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables import org.apache.spark.sql.Row import org.apache.spark.sql.connect.ConnectConversions._ import org.apache.spark.sql.functions.{col, expr} import org.apache.spark.sql.test.DeltaQueryTest class DeltaMergeBuilderSuite extends DeltaQueryTest with RemoteSparkSession { private def writeTargetTable(path: String): Unit = { val session = spark import session.implicits._ Seq(("a", 1), ("b", 2), ("c", 3), ("d", 4)).toDF("key", "value") .write.mode("overwrite").format("delta").save(path) } private def testSource = { val session = spark import session.implicits._ Seq(("a", -1), ("b", 0), ("e", -5), ("f", -6)).toDF("k", "v") } test("string expressions in merge conditions and assignments") { withTempPath { dir => val path = dir.getAbsolutePath writeTargetTable(path) val deltaTable = DeltaTable.forPath(spark, path) val mergeOutput = deltaTable .merge(testSource, "key = k") .whenMatched().updateExpr(Map("value" -> "value + v")) .whenNotMatched().insertExpr(Map("key" -> "k", "value" -> "v")) .whenNotMatchedBySource().updateExpr(Map("value" -> "value - 1")) .execute() checkAnswer( mergeOutput, Seq(Row( 6, // affected rows 4, // updated rows (a and b in WHEN MATCHED and c and d in WHEN NOT MATCHED BY SOURCE) 0, // deleted rows 2 // inserted rows (e and f) ))) checkAnswer( deltaTable.toDF, Seq(Row("a", 0), Row("b", 2), Row("c", 2), Row("d", 3), Row("e", -5), Row("f", -6))) } } test("column expressions in merge conditions and assignments") { withTempPath { dir => val path = dir.getAbsolutePath writeTargetTable(path) val deltaTable = DeltaTable.forPath(spark, path) deltaTable .merge(testSource, col("key") === col("k")) .whenMatched().update(Map("value" -> (col("value") + col("v")))) .whenNotMatched().insert(Map("key" -> col("k"), "value" -> col("v"))) .whenNotMatchedBySource().update(Map("value" -> (col("value") - 1))) .execute() checkAnswer( deltaTable.toDF, Seq(Row("a", 0), Row("b", 2), Row("c", 2), Row("d", 3), Row("e", -5), Row("f", -6))) } } test("multiple when matched then update clauses") { withTempPath { dir => val path = dir.getAbsolutePath writeTargetTable(path) val deltaTable = DeltaTable.forPath(spark, path) deltaTable .merge(testSource, expr("key = k")) .whenMatched("key = 'a'").updateExpr(Map("value" -> "5")) .whenMatched().updateExpr(Map("value" -> "0")) .execute() checkAnswer( deltaTable.toDF, Seq(Row("a", 5), Row("b", 0), Row("c", 3), Row("d", 4))) } } test("multiple when matched then delete clauses") { withTempPath { dir => val path = dir.getAbsolutePath writeTargetTable(path) val deltaTable = DeltaTable.forPath(spark, path) deltaTable .merge(testSource, "key = k") .whenMatched("key = 'a'").delete() .whenMatched().delete() .execute() checkAnswer( deltaTable.toDF, Seq(Row("c", 3), Row("d", 4))) } } test("redundant when matched then update and delete clauses") { withTempPath { dir => val path = dir.getAbsolutePath writeTargetTable(path) val deltaTable = DeltaTable.forPath(spark, path) deltaTable .merge(testSource, col("key") === col("k")) .whenMatched("key = 'a'").updateExpr(Map("value" -> "5")) .whenMatched("key = 'a'").updateExpr(Map("value" -> "0")) .whenMatched("key = 'b'").updateExpr(Map("value" -> "6")) .whenMatched("key = 'b'").delete() .execute() checkAnswer( deltaTable.toDF, Seq(Row("a", 5), Row("b", 6), Row("c", 3), Row("d", 4))) } } test("interleaved when matched then update and delete clauses") { withTempPath { dir => val path = dir.getAbsolutePath writeTargetTable(path) val deltaTable = DeltaTable.forPath(spark, path) deltaTable.as("t") .merge(testSource, col("t.key") === col("k")) .whenMatched("t.key = 'a'").delete() .whenMatched("t.key = 'a'").updateExpr(Map("value" -> "5")) .whenMatched("t.key = 'b'").delete() .whenMatched().updateExpr(Map("value" -> "6")) .execute() checkAnswer( deltaTable.toDF, Seq(Row("c", 3), Row("d", 4))) } } test("multiple when not matched then insert clauses") { withTempPath { dir => val path = dir.getAbsolutePath writeTargetTable(path) val deltaTable = DeltaTable.forPath(spark, path) deltaTable.as("t") .merge(testSource.toDF("key", "value").as("s"), col("t.key") === col("s.key")) .whenNotMatched("s.key = 'e'").insertExpr(Map("t.key" -> "s.key", "t.value" -> "5")) .whenNotMatched().insertAll() .execute() checkAnswer( deltaTable.toDF, Seq(Row("a", 1), Row("b", 2), Row("c", 3), Row("d", 4), Row("e", 5), Row("f", -6))) } } test("redundant when not matched then insert clauses") { withTempPath { dir => val path = dir.getAbsolutePath writeTargetTable(path) val deltaTable = DeltaTable.forPath(spark, path) deltaTable .merge(testSource, expr("key = k")) .whenNotMatched("k = 'e'").insertExpr(Map("key" -> "k", "value" -> "5")) .whenNotMatched("k = 'e'").insertExpr(Map("key" -> "k", "value" -> "6")) .whenNotMatched("k = 'f'").insertExpr(Map("key" -> "k", "value" -> "7")) .whenNotMatched("k = 'f'").insertExpr(Map("key" -> "k", "value" -> "8")) .execute() checkAnswer( deltaTable.toDF, Seq(Row("a", 1), Row("b", 2), Row("c", 3), Row("d", 4), Row("e", 5), Row("f", 7))) } } test("multiple when not matched by source then update clauses") { withTempPath { dir => val path = dir.getAbsolutePath writeTargetTable(path) val deltaTable = DeltaTable.forPath(spark, path) deltaTable.merge(testSource, expr("key = k")) .whenNotMatchedBySource("key = 'c'").updateExpr(Map("value" -> "5")) .whenNotMatchedBySource().updateExpr(Map("value" -> "0")) .execute() checkAnswer( deltaTable.toDF, Seq(Row("a", 1), Row("b", 2), Row("c", 5), Row("d", 0))) } } test("multiple when not matched by source then delete clauses") { withTempPath { dir => val path = dir.getAbsolutePath writeTargetTable(path) val deltaTable = DeltaTable.forPath(spark, path) deltaTable.merge(testSource, expr("key = k")) .whenNotMatchedBySource("key = 'c'").delete() .whenNotMatchedBySource().delete() .execute() checkAnswer( deltaTable.toDF, Seq(Row("a", 1), Row("b", 2))) } } test("redundant when not matched by source then update and delete clauses") { withTempPath { dir => val path = dir.getAbsolutePath writeTargetTable(path) val deltaTable = DeltaTable.forPath(spark, path) deltaTable.merge(testSource, expr("key = k")) .whenNotMatchedBySource("key = 'c'").updateExpr(Map("value" -> "5")) .whenNotMatchedBySource("key = 'c'").updateExpr(Map("value" -> "0")) .whenNotMatchedBySource("key = 'd'").updateExpr(Map("value" -> "6")) .whenNotMatchedBySource("key = 'd'").delete() .whenNotMatchedBySource().delete() .execute() checkAnswer( deltaTable.toDF, Seq(Row("a", 1), Row("b", 2), Row("c", 5), Row("d", 6))) } } test("interleaved when not matched by source then update and delete clauses") { withTempPath { dir => val path = dir.getAbsolutePath writeTargetTable(path) val deltaTable = DeltaTable.forPath(spark, path) deltaTable.merge(testSource, expr("key = k")) .whenNotMatchedBySource("key = 'c'").delete() .whenNotMatchedBySource("key = 'c'").updateExpr(Map("value" -> "5")) .whenNotMatchedBySource("key = 'd'").delete() .whenNotMatchedBySource().updateExpr(Map("value" -> "6")) .execute() checkAnswer( deltaTable.toDF, Seq(Row("a", 1), Row("b", 2))) } } test("string expressions in all conditions and assignments") { withTempPath { dir => val path = dir.getAbsolutePath writeTargetTable(path) val deltaTable = DeltaTable.forPath(spark, path) deltaTable .merge(testSource, "key = k") .whenMatched("k = 'a'").updateExpr(Map("value" -> "v + 0")) .whenMatched("k = 'b'").delete() .whenNotMatched("k = 'e'").insertExpr(Map("key" -> "k", "value" -> "v + 0")) .whenNotMatchedBySource("key = 'c'").updateExpr(Map("value" -> "value + 0")) .whenNotMatchedBySource("key = 'd'").delete() .execute() checkAnswer( deltaTable.toDF, Seq(Row("a", -1), Row("c", 3), Row("e", -5))) } } test("column expressions in all conditions and assignments") { withTempPath { dir => val path = dir.getAbsolutePath writeTargetTable(path) val deltaTable = DeltaTable.forPath(spark, path) deltaTable .merge(testSource, expr("key = k")) .whenMatched(expr("k = 'a'")).update(Map("value" -> (col("v") + 0))) .whenMatched(expr("k = 'b'")).delete() .whenNotMatched(expr("k = 'e'")).insert(Map("key" -> col("k"), "value" -> (col("v") + 0))) .whenNotMatchedBySource(expr("key = 'c'")).update(Map("value" -> (col("value") + 0))) .whenNotMatchedBySource(expr("key = 'd'")).delete() .execute() checkAnswer( deltaTable.toDF, Seq(Row("a", -1), Row("c", 3), Row("e", -5))) } } test("no clause conditions and insertAll/updateAll + aliases") { withTempPath { dir => val path = dir.getAbsolutePath writeTargetTable(path) val deltaTable = DeltaTable.forPath(spark, path) deltaTable.as("t") .merge(testSource.toDF("key", "value").as("s"), expr("t.key = s.key")) .whenMatched().updateAll() .whenNotMatched().insertAll() .execute() checkAnswer( deltaTable.toDF, Seq(Row("a", -1), Row("b", 0), Row("c", 3), Row("d", 4), Row("e", -5), Row("f", -6))) } } test("string expressions in all clause conditions and insertAll/updateAll + aliases") { withTempPath { dir => val path = dir.getAbsolutePath writeTargetTable(path) val deltaTable = DeltaTable.forPath(spark, path) deltaTable.as("t") .merge(testSource.toDF("key", "value").as("s"), "t.key = s.key") .whenMatched("s.key = 'a'").updateAll() .whenNotMatched("s.key = 'e'").insertAll() .execute() checkAnswer( deltaTable.toDF, Seq(Row("a", -1), Row("b", 2), Row("c", 3), Row("d", 4), Row("e", -5))) } } test("column expressions in all clause conditions and insertAll/updateAll + aliases") { withTempPath { dir => val path = dir.getAbsolutePath writeTargetTable(path) val deltaTable = DeltaTable.forPath(spark, path) deltaTable.as("t") .merge(testSource.toDF("key", "value").as("s"), expr("t.key = s.key")) .whenMatched(expr("s.key = 'a'")).updateAll() .whenNotMatched(expr("s.key = 'e'")).insertAll() .execute() checkAnswer( deltaTable.toDF, Seq(Row("a", -1), Row("b", 2), Row("c", 3), Row("d", 4), Row("e", -5))) } } test("automatic schema evolution") { val session = spark import session.implicits._ withTempPath { dir => val path = dir.getAbsolutePath Seq("a", "b", "c", "d").toDF("key") .write.mode("overwrite").format("delta").save(path) val deltaTable = DeltaTable.forPath(spark, path) withSQLConf("spark.databricks.delta.schema.autoMerge.enabled" -> "true") { deltaTable.as("t") .merge(testSource.toDF("key", "value").as("s"), expr("t.key = s.key")) .whenMatched().updateAll() .whenNotMatched().insertAll() .execute() } checkAnswer( deltaTable.toDF, Seq( Row("a", -1), Row("b", 0), Row("c", null), Row("d", null), Row("e", -5), Row("f", -6))) } } test("merge with the withSchemaEvolution API") { val session = spark import session.implicits._ withTempPath { dir => val path = dir.getAbsolutePath Seq("a", "b", "c", "d").toDF("key") .write.mode("overwrite").format("delta").save(path) val deltaTable = DeltaTable.forPath(spark, path) deltaTable.as("t") .merge(testSource.toDF("key", "value").as("s"), expr("t.key = s.key")) .withSchemaEvolution() .whenMatched().updateAll() .whenNotMatched().insertAll() .execute() checkAnswer( deltaTable.toDF, Seq( Row("a", -1), Row("b", 0), Row("c", null), Row("d", null), Row("e", -5), Row("f", -6))) } } test("merge with no withSchemaEvolution while the source's schema " + "is different than the target's schema") { val session = spark import session.implicits._ withTempPath { dir => val path = dir.getAbsolutePath Seq("a", "b", "c", "d").toDF("key") .write.mode("overwrite").format("delta").save(path) val deltaTable = DeltaTable.forPath(spark, path) deltaTable.as("t") .merge(testSource.toDF("key", "value").as("s"), expr("t.key = s.key")) .whenMatched().updateAll() .whenNotMatched().insertAll() .execute() checkAnswer( deltaTable.toDF, Seq(Row("a"), Row("b"), Row("c"), Row("d"), Row("e"), Row("f"))) } } test("merge dataframe with many columns") { withTempPath { dir => val path = dir.getAbsolutePath val id = col("id") val numColumns = 100 val cols1 = id +: Seq.tabulate(numColumns)(i => id.as(s"col$i")) val df1 = spark.range(1).select(cols1: _*) df1.write.mode("overwrite").format("delta").save(path) val deltaTable = io.delta.tables.DeltaTable.forPath(spark, path) val cols2 = id +: Seq.tabulate(numColumns)(i => (id + 1).as(s"col$i")) val df2 = spark.range(1).select(cols2: _*) deltaTable .as("t") .merge(df2.as("s"), "s.id = t.id") .whenMatched().updateAll() .execute() checkAnswer( deltaTable.toDF, Seq(df2.collectAsList().get(0))) } } } ================================================ FILE: spark-connect/client/src/test/scala/io/delta/connect/tables/DeltaQueryTest.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.test import java.util.TimeZone import org.scalatest.Assertions import org.scalatest.funsuite.AnyFunSuite import org.apache.spark.sql.{DataFrame, Dataset, Row} import org.apache.spark.sql.catalyst.util.SparkStringUtilsShims.sideBySide import org.apache.spark.sql.connect.ConnectConversions._ import org.apache.spark.sql.connect.SparkSession import org.apache.spark.sql.connect.test.SQLHelper import org.apache.spark.util.ArrayImplicits._ // TODO: Copied from Spark until SPARK-48341 is resolved. abstract class DeltaQueryTest extends AnyFunSuite with SQLHelper { def spark: SparkSession /** * Runs the plan and makes sure the answer matches the expected result. * * @param df * the [[DataFrame]] to be executed * @param expectedAnswer * the expected result in a [[Seq]] of [[Row]]s. */ protected def checkAnswer(df: => DataFrame, expectedAnswer: Seq[Row]): Unit = { DeltaQueryTest.checkAnswer(df, expectedAnswer) } protected def checkAnswer(df: => DataFrame, expectedAnswer: Row): Unit = { checkAnswer(df, Seq(expectedAnswer)) } protected def checkAnswer(df: => DataFrame, expectedAnswer: DataFrame): Unit = { checkAnswer(df, expectedAnswer.collect().toImmutableArraySeq) } protected def checkAnswer(df: => DataFrame, expectedAnswer: Array[Row]): Unit = { checkAnswer(df, expectedAnswer.toImmutableArraySeq) } /** * Evaluates a dataset to make sure that the result of calling collect matches the given * expected answer. */ protected def checkDataset[T](ds: => Dataset[T], expectedAnswer: T*): Unit = { val result = ds.collect() if (!DeltaQueryTest.compare(result.toSeq, expectedAnswer)) { fail(s""" |Decoded objects do not match expected objects: |expected: $expectedAnswer |actual: ${result.toSeq} """.stripMargin) } } /** * Evaluates a dataset to make sure that the result of calling collect matches the given * expected answer, after sort. */ protected def checkDatasetUnorderly[T: Ordering]( ds: => Dataset[T], expectedAnswer: T*): Unit = { val result = ds.collect() if (!DeltaQueryTest.compare(result.toSeq.sorted, expectedAnswer.sorted)) { fail(s""" |Decoded objects do not match expected objects: |expected: $expectedAnswer |actual: ${result.toSeq} """.stripMargin) } } } object DeltaQueryTest extends Assertions { /** * Runs the plan and makes sure the answer matches the expected result. * * @param df * the DataFrame to be executed * @param expectedAnswer * the expected result in a Seq of Rows. */ def checkAnswer(df: DataFrame, expectedAnswer: Seq[Row], isSorted: Boolean = false): Unit = { getErrorMessageInCheckAnswer(df, expectedAnswer, isSorted) match { case Some(errorMessage) => fail(errorMessage) case None => } } /** * Runs the plan and makes sure the answer matches the expected result. If there was exception * during the execution or the contents of the DataFrame does not match the expected result, an * error message will be returned. Otherwise, a None will be returned. * * @param df * the DataFrame to be executed * @param expectedAnswer * the expected result in a Seq of Rows. */ def getErrorMessageInCheckAnswer( df: DataFrame, expectedAnswer: Seq[Row], isSorted: Boolean = false): Option[String] = { val sparkAnswer = try df.collect().toSeq catch { case e: Exception => val errorMessage = s""" |Exception thrown while executing query: |${df.analyze} |== Exception == |$e |${org.apache.spark.util.SparkErrorUtils.stackTraceToString(e)} """.stripMargin return Some(errorMessage) } sameRows(expectedAnswer, sparkAnswer, isSorted).map { results => s""" |Results do not match for query: |Timezone: ${TimeZone.getDefault} |Timezone Env: ${sys.env.getOrElse("TZ", "")} | |${df.analyze} |== Results == |$results """.stripMargin } } def prepareAnswer(answer: Seq[Row], isSorted: Boolean): Seq[Row] = { // Converts data to types that we can do equality comparison using Scala collections. // For BigDecimal type, the Scala type has a better definition of equality test (similar to // Java's java.math.BigDecimal.compareTo). // For binary arrays, we convert it to Seq to avoid of calling java.util.Arrays.equals for // equality test. val converted: Seq[Row] = answer.map(prepareRow) if (!isSorted) converted.sortBy(_.toString()) else converted } // We need to call prepareRow recursively to handle schemas with struct types. def prepareRow(row: Row): Row = { Row.fromSeq(row.toSeq.map { case null => null case bd: java.math.BigDecimal => BigDecimal(bd) // Equality of WrappedArray differs for AnyVal and AnyRef in Scala 2.12.2+ case seq: Seq[_] => seq.map { case b: java.lang.Byte => b.byteValue case s: java.lang.Short => s.shortValue case i: java.lang.Integer => i.intValue case l: java.lang.Long => l.longValue case f: java.lang.Float => f.floatValue case d: java.lang.Double => d.doubleValue case x => x } // Convert array to Seq for easy equality check. case b: Array[_] => b.toSeq case r: Row => prepareRow(r) case o => o }) } private def genError( expectedAnswer: Seq[Row], sparkAnswer: Seq[Row], isSorted: Boolean = false): String = { val getRowType: Option[Row] => String = row => row .map(row => if (row.schema == null) { "struct<>" } else { s"${row.schema.catalogString}" }) .getOrElse("struct<>") s""" |== Results == |${sideBySide( s"== Correct Answer - ${expectedAnswer.size} ==" +: getRowType(expectedAnswer.headOption) +: prepareAnswer(expectedAnswer, isSorted).map(_.toString()), s"== Spark Answer - ${sparkAnswer.size} ==" +: getRowType(sparkAnswer.headOption) +: prepareAnswer(sparkAnswer, isSorted).map(_.toString())).mkString("\n")} """.stripMargin } def includesRows(expectedRows: Seq[Row], sparkAnswer: Seq[Row]): Option[String] = { if (!prepareAnswer(expectedRows, true).toSet.subsetOf( prepareAnswer(sparkAnswer, true).toSet)) { return Some(genError(expectedRows, sparkAnswer, true)) } None } def compare(obj1: Any, obj2: Any): Boolean = (obj1, obj2) match { case (null, null) => true case (null, _) => false case (_, null) => false case (a: Array[_], b: Array[_]) => a.length == b.length && a.zip(b).forall { case (l, r) => compare(l, r) } case (a: Map[_, _], b: Map[_, _]) => a.size == b.size && a.keys.forall { aKey => b.keys.find(bKey => compare(aKey, bKey)).exists(bKey => compare(a(aKey), b(bKey))) } case (a: Iterable[_], b: Iterable[_]) => a.size == b.size && a.zip(b).forall { case (l, r) => compare(l, r) } case (a: Product, b: Product) => compare(a.productIterator.toSeq, b.productIterator.toSeq) case (a: Row, b: Row) => compare(a.toSeq, b.toSeq) // 0.0 == -0.0, turn float/double to bits before comparison, to distinguish 0.0 and -0.0. case (a: Double, b: Double) => java.lang.Double.doubleToRawLongBits(a) == java.lang.Double.doubleToRawLongBits(b) case (a: Float, b: Float) => java.lang.Float.floatToRawIntBits(a) == java.lang.Float.floatToRawIntBits(b) case (a, b) => a == b } def sameRows( expectedAnswer: Seq[Row], sparkAnswer: Seq[Row], isSorted: Boolean = false): Option[String] = { if (!compare(prepareAnswer(expectedAnswer, isSorted), prepareAnswer(sparkAnswer, isSorted))) { return Some(genError(expectedAnswer, sparkAnswer, isSorted)) } None } } ================================================ FILE: spark-connect/client/src/test/scala/io/delta/connect/tables/DeltaTableBuilderSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.test.DeltaQueryTest import org.apache.spark.sql.types.{IntegerType, LongType, MetadataBuilder, StringType, StructType} class DeltaTableBuilderSuite extends DeltaQueryTest with RemoteSparkSession { // Define the information for a default test table used by many tests. private val defaultTestTableSchema = "c1 int, c2 int, c3 string" private val defaultTestTableGeneratedColumns = Map("c2" -> "c1 + 10") private val defaultTestTableIdentityColumns = Map.empty[String, String] private val defaultTestTablePartitionColumns = Seq("c1") private val defaultTestTableColumnComments = Map("c1" -> "foo", "c3" -> "bar") private val defaultTestTableComment = "tbl comment" private val defaultTestTableNullableCols = Set("c1", "c3") private val defaultTestTableProperty = Map("foo" -> "bar") private val defaultTestTableClusteringColumns = Seq("c1", "c3") /** * Verify if the table metadata matches the test table. We use this to verify DDLs * write correct table metadata into the transaction logs. * If clusteringCols are not empty, it verifies the clustering columns for liquid. */ protected def verifyTestTableMetadata( table: String, schemaString: String, generatedColumns: Map[String, String] = Map.empty, identityColumns: Map[String, String] = Map.empty, colComments: Map[String, String] = Map.empty, colNullables: Set[String] = Set.empty, tableComment: Option[String] = None, expectedPartitionCols: Seq[String] = Seq.empty, expectedTableProperties: Map[String, String] = Map.empty, expectedClusteringCols: Seq[String] = Seq.empty): Unit = { val session = spark import session.implicits._ val expectedSchema = StructType(StructType.fromDDL(schemaString).map { field => val newMetadata = new MetadataBuilder().withMetadata(field.metadata) if (colComments.contains(field.name)) { newMetadata.putString("comment", colComments(field.name)) } field.copy( nullable = colNullables.contains(field.name), metadata = newMetadata.build()) }) val deltaTable = DeltaTable.forName(spark, table) assert(deltaTable.toDF.schema == expectedSchema) val (description, partitionColumns, properties, clusteringColumns) = deltaTable.detail() .select("description", "partitionColumns", "properties", "clusteringColumns") .as[(String, Seq[String], Map[String, String], Seq[String])] .head() assert(description == tableComment.orNull) assert(partitionColumns == expectedPartitionCols) // It may contain other properties other than the ones we added. expectedTableProperties.foreach { prop => properties.contains(prop._1) && properties.get(prop._1).get == prop._2 } assert(clusteringColumns == expectedClusteringCols) val schemaFields = deltaTable.toDF.schema.fields // Verify generated columns generatedColumns.foreach { case (col, expr) => val fieldOpt = schemaFields.find(_.name == col) assert(fieldOpt.isDefined, s"Generated column $col not found in table schema") val field = fieldOpt.get // Check if the metadata contains the generation expression key if (field.metadata.contains("delta.generationExpression")) { val generationExpr = field.metadata.getString("delta.generationExpression") assert(generationExpr == expr, s"Generated column $col has expression '$generationExpr' but expected '$expr'") } } // Verify identity columns without detailed metadata validation identityColumns.foreach { case (col, _) => val fieldOpt = schemaFields.find(_.name == col) assert(fieldOpt.isDefined, s"Identity column $col not found in table schema") } } protected def testCreateTable(testName: String)(createFunc: String => Unit): Unit = { test(testName) { withTable(testName) { createFunc(testName) verifyTestTableMetadata( testName, defaultTestTableSchema, defaultTestTableGeneratedColumns, defaultTestTableIdentityColumns, defaultTestTableColumnComments, defaultTestTableNullableCols, Some(defaultTestTableComment), defaultTestTablePartitionColumns, defaultTestTableProperty ) } } } protected def testCreateTableWithNameAndLocation( testName: String)(createFunc: (String, String) => Unit): Unit = { test(testName + ": external - with location and name") { withTempPath { path => withTable(testName) { createFunc(testName, path.getCanonicalPath) verifyTestTableMetadata( testName, defaultTestTableSchema, defaultTestTableGeneratedColumns, defaultTestTableIdentityColumns, defaultTestTableColumnComments, defaultTestTableNullableCols, Some(defaultTestTableComment), defaultTestTablePartitionColumns, defaultTestTableProperty ) } } } } protected def testCreateTableWithLocationOnly( testName: String)(createFunc: String => Unit): Unit = { test(testName + ": external - location only") { withTempPath { path => withTable(testName) { createFunc(path.getCanonicalPath) verifyTestTableMetadata( s"delta.`${path.getCanonicalPath}`", defaultTestTableSchema, defaultTestTableGeneratedColumns, defaultTestTableIdentityColumns, defaultTestTableColumnComments, defaultTestTableNullableCols, Some(defaultTestTableComment), defaultTestTablePartitionColumns, defaultTestTableProperty ) } } } } def defaultCreateTableBuilder( ifNotExists: Boolean, tableName: Option[String] = None, location: Option[String] = None): DeltaTableBuilder = { val tableBuilder = if (ifNotExists) { io.delta.tables.DeltaTable.createIfNotExists(spark) } else { io.delta.tables.DeltaTable.create(spark) } defaultTableBuilder(tableBuilder, tableName, location) } def defaultReplaceTableBuilder( orCreate: Boolean, tableName: Option[String] = None, location: Option[String] = None): DeltaTableBuilder = { val tableBuilder = if (orCreate) { io.delta.tables.DeltaTable.createOrReplace(spark) } else { io.delta.tables.DeltaTable.replace(spark) } defaultTableBuilder(tableBuilder, tableName, location) } private def defaultTableBuilder( builder: DeltaTableBuilder, tableName: Option[String], location: Option[String], clusterBy: Boolean = false): DeltaTableBuilder = { var tableBuilder = builder if (tableName.nonEmpty) { tableBuilder = tableBuilder.tableName(tableName.get) } if (location.nonEmpty) { tableBuilder = tableBuilder.location(location.get) } tableBuilder.addColumn( io.delta.tables.DeltaTable.columnBuilder(spark, "c1").dataType("int") .nullable(true).comment("foo").build() ) tableBuilder.addColumn( io.delta.tables.DeltaTable.columnBuilder(spark, "c2").dataType("int") .nullable(false).generatedAlwaysAs("c1 + 10").build() ) tableBuilder.addColumn( io.delta.tables.DeltaTable.columnBuilder(spark, "c3").dataType("string") .comment("bar").build() ) if (clusterBy) { tableBuilder.clusterBy("c1", "c3") } else { tableBuilder.partitionedBy("c1") } tableBuilder.property("foo", "bar") tableBuilder.comment("tbl comment") tableBuilder } test("create table with existing schema and extra column") { withTable("table") { withTempPath { dir => spark.range(10).toDF("key").write.format("parquet").saveAsTable("table") val existingSchema = spark.read.format("parquet").table("table").schema io.delta.tables.DeltaTable.create(spark) .location(dir.getAbsolutePath) .addColumns(existingSchema) .addColumn("value", "string", false) .execute() verifyTestTableMetadata(s"delta.`${dir.getAbsolutePath}`", "key bigint, value string", colNullables = Set("key")) } } } test("create table with variation of addColumns - with spark session") { withTable("test") { io.delta.tables.DeltaTable.create(spark) .tableName("test") .addColumn("c1", "int") .addColumn("c2", IntegerType) .addColumn("c3", "string", false) .addColumn("c4", StringType, true) .addColumn( io.delta.tables.DeltaTable.columnBuilder(spark, "c5") .dataType("bigint") .comment("foo") .nullable(false) .build ) .addColumn( io.delta.tables.DeltaTable.columnBuilder(spark, "c6") .dataType(LongType) .generatedAlwaysAs("c5 + 10") .build ).execute() verifyTestTableMetadata( "test", "c1 int, c2 int, c3 string, c4 string, c5 bigint, c6 bigint", generatedColumns = Map("c6" -> "c5 + 10"), colComments = Map("c5" -> "foo"), colNullables = Set("c1", "c2", "c4", "c6") ) } } test("test addColumn using columnBuilder, without dataType") { val e = intercept[AnalysisException] { DeltaTable.columnBuilder(spark, "value") .generatedAlwaysAs("true") .nullable(true) .build() } assert(e.getMessage == "The data type of the column value is not provided") } testCreateTable("create_table") { table => defaultCreateTableBuilder(ifNotExists = false, Some(table)).execute() } testCreateTableWithNameAndLocation("create_table") { (name, path) => defaultCreateTableBuilder(ifNotExists = false, Some(name), Some(path)).execute() } testCreateTableWithLocationOnly("create_table") { path => defaultCreateTableBuilder(ifNotExists = false, location = Some(path)).execute() } test("create table - errors if already exists") { withTable("testTable") { spark.sql(s"CREATE TABLE testTable (c1 int) USING DELTA") intercept[AnalysisException] { defaultCreateTableBuilder(ifNotExists = false, Some("testTable")).execute() } } } test("create table - ignore if already exists") { withTable("testTable") { spark.sql(s"CREATE TABLE testTable (c1 int) USING DELTA") defaultCreateTableBuilder(ifNotExists = true, Some("testTable")).execute() verifyTestTableMetadata("testTable", "c1 int", colNullables = Set("c1")) } } testCreateTable("create_table_if_not_exists") { table => defaultCreateTableBuilder(ifNotExists = true, Some(table)).execute() } testCreateTableWithNameAndLocation("create_table_if_not_exists") { (name, path) => defaultCreateTableBuilder(ifNotExists = true, Some(name), Some(path)).execute() } testCreateTableWithLocationOnly("create_table_if_not_exists") { path => defaultCreateTableBuilder(ifNotExists = true, location = Some(path)).execute() } test("replace table - errors if not exists") { intercept[AnalysisException] { defaultReplaceTableBuilder(orCreate = false, Some("testTable")).execute() } } testCreateTable("replace_table") { table => spark.sql(s"CREATE TABLE replace_table(c1 int) USING DELTA") defaultReplaceTableBuilder(orCreate = false, Some(table)).execute() } testCreateTableWithNameAndLocation("replace_table") { (name, path) => spark.sql(s"CREATE TABLE $name (c1 int) USING DELTA LOCATION '$path'") defaultReplaceTableBuilder(orCreate = false, Some(name), Some(path)).execute() } testCreateTableWithLocationOnly("replace_table") { path => spark.sql(s"CREATE TABLE delta.`$path` (c1 int) USING DELTA") defaultReplaceTableBuilder(orCreate = false, location = Some(path)).execute() } testCreateTable("replace_or_create_table") { table => defaultReplaceTableBuilder(orCreate = true, Some(table)).execute() } testCreateTableWithNameAndLocation("replace_or_create_table") { (name, path) => defaultReplaceTableBuilder(orCreate = true, Some(name), Some(path)).execute() } testCreateTableWithLocationOnly("replace_or_create_table") { path => defaultReplaceTableBuilder(orCreate = true, location = Some(path)).execute() } test("test no identifier and no location") { val e = intercept[AnalysisException] { io.delta.tables.DeltaTable.create(spark).addColumn("c1", "int").execute() } assert(e.getMessage.equals("Table name or location has to be specified")) } test("partitionedBy only should contain columns in the schema") { val e = intercept[AnalysisException] { io.delta.tables.DeltaTable.create(spark).tableName("testTable") .addColumn("c1", "int") .partitionedBy("c2") .execute() } assert(e.getMessage.contains("Couldn't find column c2")) } test("errors if table name and location are different paths") { withTempPath { dir => val path = dir.getAbsolutePath // TODO: This should be an AnalysisException, but it ends up as a Spark Exception // that arises from the Connect Client failing to enrich the exception with the Delta // error class that we expect. val e = intercept[Exception] { io.delta.tables.DeltaTable.create(spark).tableName(s"delta.`$path`") .addColumn("c1", "int") .location("src/test/resources/delta/non_generated_columns") .execute() } val deltaErrorClass = "DELTA_CREATE_TABLE_IDENTIFIER_LOCATION_MISMATCH" assert(e.getMessage.contains(deltaErrorClass)) } } test("table name and location are the same") { withTempPath { dir => val path = dir.getAbsolutePath io.delta.tables.DeltaTable.create(spark).tableName(s"delta.`$path`") .addColumn("c1", "int") .location(path) .execute() } } test("errors if use parquet path as identifier") { withTempPath { dir => val path = dir.getAbsolutePath val e = intercept[AnalysisException] { io.delta.tables.DeltaTable.create(spark).tableName(s"parquet.`$path`") .addColumn("c1", "int") .location(path) .execute() } assert(e.getMessage == "Database 'main.parquet' not found" || e.getMessage == "Database 'parquet' not found" || e.getMessage.contains("is not a valid name") || e.getMessage.contains("schema `parquet` cannot be found") ) } } private def testCreateTableWithClusterBy(testName: String)(createFunc: String => Unit): Unit = { test(testName) { withTable(testName) { createFunc(testName) verifyTestTableMetadata( testName, defaultTestTableSchema, defaultTestTableGeneratedColumns, defaultTestTableIdentityColumns, defaultTestTableColumnComments, defaultTestTableNullableCols, Some(defaultTestTableComment), expectedTableProperties = defaultTestTableProperty, expectedClusteringCols = defaultTestTableClusteringColumns ) } } } testCreateTableWithClusterBy("create_table_with_clusterBy") { table => val builder = DeltaTable.create(spark) defaultTableBuilder(builder, Some(table), None, true).execute() } testCreateTableWithClusterBy("replace_table_with_clusterBy") { table => spark.sql(s"CREATE TABLE $table(c1 int) USING DELTA") val builder = DeltaTable.replace(spark) defaultTableBuilder(builder, Some(table), None, true).execute() } testCreateTableWithClusterBy("create_or_replace_table_with_clusterBy") { table => spark.sql(s"CREATE TABLE $table(c1 int) USING DELTA") val builder = DeltaTable.createOrReplace(spark) defaultTableBuilder(builder, Some(table), None, true).execute() } test("clusterBy only should contain columns in the schema") { val e = intercept[AnalysisException] { io.delta.tables.DeltaTable.create(spark).tableName("testTable") .addColumn("c1", "int") .clusterBy("c2") .execute() } assert(e.getMessage.contains("`c2` is missing")) } test("partitionedBy and clusterBy cannot be used together") { // TODO: This should be an AnalysisException, but it ends up as a Spark Exception // that arises from the Connect Client failing to enrich the exception with the Delta // error class that we expect. val e = intercept[Exception] { io.delta.tables.DeltaTable.create(spark).tableName("testTable") .addColumn("c1", "int") .addColumn("c2", "int") .partitionedBy("c2") .clusterBy("c1") .execute() } val deltaErrorClass = "DELTA_CLUSTER_BY_WITH_PARTITIONED_BY" assert(e.getMessage.contains(deltaErrorClass)) } test("create table with identity columns") { withTempPath { dir => val path = dir.getAbsolutePath DeltaTable.create(spark) .tableName("testTable") .addColumn( DeltaTable.columnBuilder(spark, "id1") .dataType("long") .nullable(false) .generatedAlwaysAsIdentity() .build() ) .addColumn( DeltaTable.columnBuilder(spark, "id2") .dataType("long") .nullable(false) .generatedByDefaultAsIdentity() .build() ) .addColumn( DeltaTable.columnBuilder(spark, "id3") .dataType("long") .nullable(false) .generatedAlwaysAsIdentity(start = 100, step = 10) .build() ) .addColumn( DeltaTable.columnBuilder(spark, "id4") .dataType("long") .nullable(false) .generatedByDefaultAsIdentity(start = 100, step = 10) .build() ) .location(path) .execute() // Verify the columns exist in the schema val table = DeltaTable.forPath(spark, path) val schema = table.toDF.schema // Verify basic structure assert(schema.fieldNames.toSeq === Seq("id1", "id2", "id3", "id4")) schema.fields.foreach { field => assert(field.dataType.typeName === "long") assert(!field.nullable) } // Test identity column functionality: // Insert data without providing values for GENERATED ALWAYS identity columns // and verify correct identity values are generated spark.sql(s"INSERT INTO delta.`$path` (id2, id4) VALUES (20, 200)") // First row should have id1=1, id3=100 (start values) generated automatically // id2=20, id4=200 should be the explicitly provided values val result = spark.sql(s"SELECT * FROM delta.`$path`").collect() assert(result.length === 1) assert(result(0).getLong(0) === 1) // id1 = 1 (default start) assert(result(0).getLong(1) === 20) // id2 = 20 (explicitly provided) assert(result(0).getLong(2) === 100) // id3 = 100 (custom start) assert(result(0).getLong(3) === 200) // id4 = 200 (explicitly provided) // Verify identity column insertion restrictions // Attempting to explicitly insert a value for a GENERATED ALWAYS identity column should fail val e = intercept[Exception] { spark.sql(s"INSERT INTO delta.`$path` (id1, id2, id3, id4) VALUES (10, 20, 30, 40)") } // TODO: This should be an AnalysisException, but it ends up as a Spark Exception // that arises from the Connect Client failing to enrich the exception with the Delta // error class that we expect. val deltaErrorClass = "DELTA_IDENTITY_COLUMNS_EXPLICIT_INSERT_NOT_SUPPORTED" assert( e.getMessage.contains(deltaErrorClass), "Explicit inserts are never possible for a GENERATED ALWAYS identity column.") } } } ================================================ FILE: spark-connect/client/src/test/scala/io/delta/connect/tables/DeltaTableSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables import java.io.File import java.nio.charset.StandardCharsets import java.nio.file.Files import java.text.SimpleDateFormat import org.apache.spark.sql.Row import org.apache.spark.sql.functions.{col, lit} import org.apache.spark.sql.DataFrame import org.apache.spark.sql.test.DeltaQueryTest class DeltaTableSuite extends DeltaQueryTest with RemoteSparkSession { private lazy val testData = spark.range(100).toDF("value") test("forPath") { withTempPath { dir => testData.write.format("delta").save(dir.getAbsolutePath) checkAnswer( DeltaTable.forPath(spark, dir.getAbsolutePath).toDF, testData.collect().toSeq ) } } test("forName") { withTable("deltaTable") { testData.write.format("delta").saveAsTable("deltaTable") checkAnswer( DeltaTable.forName(spark, "deltaTable").toDF, testData.collect().toSeq ) } } test("as") { withTempPath { dir => testData.write.format("delta").save(dir.getAbsolutePath) checkAnswer( DeltaTable.forPath(spark, dir.getAbsolutePath).as("tbl").toDF.select("tbl.value"), testData.select("value").collect().toSeq ) } } test("vacuum") { withTempPath { dir => testData.write.format("delta").save(dir.getAbsolutePath) val table = io.delta.tables.DeltaTable.forPath(spark, dir.getAbsolutePath) // create a uncommitted file. val notCommittedFile = "notCommittedFile.json" val file = new File(dir, notCommittedFile) Files.write(file.toPath, "gibberish".getBytes(StandardCharsets.UTF_8)) // set to ancient time so that the file is eligible to be vacuumed. file.setLastModified(0) assert(file.exists()) table.vacuum() val file2 = new File(dir, notCommittedFile) assert(!file2.exists()) } } test("history") { val session = spark import session.implicits._ withTempPath { dir => Seq(1, 2, 3).toDF().write.format("delta").save(dir.getAbsolutePath) Seq(4, 5).toDF().write.format("delta").mode("append").save(dir.getAbsolutePath) val table = DeltaTable.forPath(spark, dir.getAbsolutePath) checkAnswer( table.history().select("version"), Seq(Row(0L), Row(1L)) ) } } test("detail") { val session = spark import session.implicits._ withTempPath { dir => Seq(1, 2, 3).toDF().write.format("delta").save(dir.getAbsolutePath) val deltaTable = DeltaTable.forPath(spark, dir.getAbsolutePath) checkAnswer( deltaTable.detail().select("format"), Seq(Row("delta")) ) } } test("isDeltaTable - path - with _delta_log dir") { withTempPath { dir => testData.write.format("delta").save(dir.getAbsolutePath) assert(DeltaTable.isDeltaTable(spark, dir.getAbsolutePath)) } } test("isDeltaTable - path - with empty _delta_log dir") { withTempPath { dir => new File(dir, "_delta_log").mkdirs() assert(!DeltaTable.isDeltaTable(spark, dir.getAbsolutePath)) } } test("isDeltaTable - path - with no _delta_log dir") { withTempPath { dir => assert(!DeltaTable.isDeltaTable(spark, dir.getAbsolutePath)) } } test("isDeltaTable - path - with non-existent dir") { withTempPath { dir => assert(!DeltaTable.isDeltaTable(spark, dir.getAbsolutePath)) } } test("isDeltaTable - with non-Delta table path") { withTempPath { dir => testData.write.format("parquet").mode("overwrite").save(dir.getAbsolutePath) assert(!DeltaTable.isDeltaTable(spark, dir.getAbsolutePath)) } } test("generate") { withTempPath { dir => val path = dir.getAbsolutePath testData.toDF().write.format("delta").save(path) val table = DeltaTable.forPath(spark, path) val manifestDir = new File(dir, "_symlink_format_manifest") assert(!manifestDir.exists()) table.generate("symlink_format_manifest") assert(manifestDir.exists()) } } test("delete") { val session = spark import session.implicits._ withTempPath { dir => val path = dir.getAbsolutePath Seq(("a", 1), ("b", 2), ("c", 3), ("d", 4)).toDF("key", "value") .write.format("delta").save(path) val deltaTable = DeltaTable.forPath(spark, path) deltaTable.delete("key = 'a'") checkAnswer(deltaTable.toDF, Seq(Row("b", 2), Row("c", 3), Row("d", 4))) deltaTable.delete(col("key") === lit("b")) checkAnswer(deltaTable.toDF, Seq(Row("c", 3), Row("d", 4))) deltaTable.delete() checkAnswer(deltaTable.toDF, Nil) } } test("update") { val session = spark import session.implicits._ withTempPath { dir => val path = dir.getAbsolutePath Seq(("a", 1), ("b", 2), ("c", 3), ("d", 4)).toDF("key", "value") .write.format("delta").save(path) val deltaTable = DeltaTable.forPath(spark, path) deltaTable.updateExpr("key = 'a' or key = 'b'", Map("value" -> "1")) checkAnswer(deltaTable.toDF, Seq(Row("a", 1), Row("b", 1), Row("c", 3), Row("d", 4))) deltaTable.update(col("key") === lit("a") || col("key") === lit("b"), Map("value" -> lit(0))) checkAnswer(deltaTable.toDF, Seq(Row("a", 0), Row("b", 0), Row("c", 3), Row("d", 4))) deltaTable.updateExpr(Map("value" -> "-1")) checkAnswer(deltaTable.toDF, Seq(Row("a", -1), Row("b", -1), Row("c", -1), Row("d", -1))) deltaTable.update(Map("value" -> lit(37))) checkAnswer(deltaTable.toDF, Seq(Row("a", 37), Row("b", 37), Row("c", 37), Row("d", 37))) } } private def getTimestampForVersion(path: String, version: Long): String = { val logPath = new File(path, "_delta_log") val file = new File(logPath, f"$version%020d.json") val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS") sdf.format(file.lastModified()) } test("clone") { withTempPath { dir => val baseDir = dir.getAbsolutePath val srcDir = new File(baseDir, "source").getCanonicalPath val dstDir = new File(baseDir, "destination").getCanonicalPath spark.range(10).write.format("delta").save(srcDir) val srcTable = io.delta.tables.DeltaTable.forPath(spark, srcDir) srcTable.clone(dstDir, true) checkAnswer( spark.read.format("delta").load(dstDir), spark.read.format("delta").load(srcDir)) } } private def writeOptimizeTestData(path: String): Unit = { testData .withColumn("col1", col("value") % 7) .withColumn("col2", col("value") % 27) .withColumn("p", col("value") % 10) .repartition(numPartitions = 4) .write.partitionBy("p").format("delta").save(path) } private def checkOptimizeMetrics( result: DataFrame, numFilesAdded: Long, numFilesRemoved: Long): Unit = { val metrics = result.select("metrics.*").head() assert(metrics.getLong(0) == numFilesAdded) assert(metrics.getLong(1) == numFilesRemoved) } private def checkOptimizeHistory( table: DeltaTable, expectedPredicates: Seq[String], expectedZorderCols: Seq[String]): Unit = { val session = table.toDF.sparkSession import session.implicits._ val (operation, operationParameters) = table.history() .select("operation", "operationParameters") .as[(String, Map[String, String])] .head() assert(operation == "OPTIMIZE") assert(operationParameters("predicate") == expectedPredicates.map(p => s"""\"($p)\"""").mkString(start = "[", sep = ",", end = "]")) assert(operationParameters("zOrderBy") == expectedZorderCols.map(c => s"""\"$c\"""").mkString(start = "[", sep = ",", end = "]")) } test("optimize - compaction") { withTempPath { dir => val path = dir.getAbsolutePath writeOptimizeTestData(path) val numDataFilesPreCompaction = spark.read.format("delta").load(path) .select("_metadata.file_path") .distinct().count() val table = io.delta.tables.DeltaTable.forPath(spark, path) val result = table.optimize().executeCompaction() checkOptimizeMetrics(result, numFilesAdded = 10, numFilesRemoved = numDataFilesPreCompaction) checkOptimizeHistory(table, expectedPredicates = Nil, expectedZorderCols = Nil) } } test("optimize - compaction - with partition filter") { withTempPath { dir => val path = dir.getAbsolutePath writeOptimizeTestData(path) val numDataFilesPreCompaction = spark.read.format("delta").load(path) .where("p = 2") .select("_metadata.file_path") .distinct().count() val table = io.delta.tables.DeltaTable.forPath(spark, path) val result = table.optimize().where("p = 2").executeCompaction() checkOptimizeMetrics(result, numFilesAdded = 1, numFilesRemoved = numDataFilesPreCompaction) checkOptimizeHistory(table, expectedPredicates = Seq("'p = 2"), expectedZorderCols = Nil) } } test("optimize - zorder") { withTempPath { dir => val path = dir.getAbsolutePath writeOptimizeTestData(path) val numDataFilesPreZOrder = spark.read.format("delta").load(path) .select("_metadata.file_path") .distinct().count() val table = io.delta.tables.DeltaTable.forPath(spark, path) val result = table.optimize().executeZOrderBy("col1", "col2") checkOptimizeMetrics(result, numFilesAdded = 10, numFilesRemoved = numDataFilesPreZOrder) checkOptimizeHistory( table, expectedPredicates = Nil, expectedZorderCols = Seq("col1", "col2")) } } test("optimize - zorder - with partition filter") { withTempPath { dir => val path = dir.getAbsolutePath writeOptimizeTestData(path) val numDataFilesPreZOrder = spark.read.format("delta").load(path) .where("p = 2") .select("_metadata.file_path") .distinct().count() val table = io.delta.tables.DeltaTable.forPath(spark, path) val result = table.optimize().where("p = 2").executeZOrderBy("col1", "col2") checkOptimizeMetrics(result, numFilesAdded = 1, numFilesRemoved = numDataFilesPreZOrder) checkOptimizeHistory( table, expectedPredicates = Seq("'p = 2"), expectedZorderCols = Seq("col1", "col2")) } } test("cloneAtVersion/timestamp - with filesystem options") { val session = spark import session.implicits._ withTempPath { dir => val baseDir = dir.getAbsolutePath val srcDir = new File(baseDir, "source").getCanonicalPath val dstDir1 = new File(baseDir, "destination1").getCanonicalPath val dstDir2 = new File(baseDir, "destination2").getCanonicalPath val df1 = Seq(1, 2, 3).toDF("id") val df2 = Seq(4, 5).toDF("id") val df3 = Seq(6, 7).toDF("id") // version 0. df1.write.format("delta").save(srcDir) // version 1. df2.write.format("delta").mode("append").save(srcDir) // version 2. df3.write.format("delta").mode("append").save(srcDir) val srcTable = io.delta.tables.DeltaTable.forPath(spark, srcDir) srcTable.cloneAtVersion(1, dstDir1, true) checkAnswer( spark.read.format("delta").load(dstDir1), df1.union(df2)) val timestamp = getTimestampForVersion(srcDir, version = 0) srcTable.cloneAtTimestamp(timestamp, dstDir2, isShallow = true, replace = true) checkAnswer( spark.read.format("delta").load(dstDir2), df1) } } test("restore") { val session = spark import session.implicits._ withTempPath { dir => val path = dir.getPath val df1 = Seq(1, 2, 3).toDF("id") val df2 = Seq(4, 5).toDF("id") val df3 = Seq(6, 7).toDF("id") // version 0. df1.write.format("delta").save(path) // version 1. df2.write.format("delta").mode("append").save(path) // version 2. df3.write.format("delta").mode("append").save(path) checkAnswer( spark.read.format("delta").load(path), df1.union(df2).union(df3)) val deltaTable = io.delta.tables.DeltaTable.forPath(spark, path) deltaTable.restoreToVersion(1) checkAnswer( spark.read.format("delta").load(path), df1.union(df2)) val deltaTable2 = io.delta.tables.DeltaTable.forPath(spark, path) val timestamp = getTimestampForVersion(path, version = 0) deltaTable2.restoreToTimestamp(timestamp) checkAnswer( spark.read.format("delta").load(path), df1) } } test("upgradeTableProtocol") { withTempPath { dir => val path = dir.getAbsolutePath testData.write.format("delta").save(path) val table = DeltaTable.forPath(spark, path) table.upgradeTableProtocol(1, 2) checkAnswer( table.history().select("version", "operation"), Seq(Row(0L, "WRITE"), Row(1L, "SET TBLPROPERTIES")) ) } } test("addFeatureSupport") { withTempPath { dir => val path = dir.getAbsolutePath testData.write.format("delta").save(path) val table = DeltaTable.forPath(spark, path) checkAnswer( table.history().select("version", "operation"), Seq(Row(0L, "WRITE")) ) table.addFeatureSupport("testReaderWriter") checkAnswer( table.history().select("version", "operation"), Seq(Row(0L, "WRITE"), Row(1L, "SET TBLPROPERTIES")) ) } } test("dropFeatureSupport") { withTempPath { dir => val path = dir.getAbsolutePath testData.write.format("delta").save(path) val table = DeltaTable.forPath(spark, path) checkAnswer( table.history().select("version", "operation"), Seq(Row(0L, "WRITE")) ) table.addFeatureSupport("testRemovableWriter") checkAnswer( table.history().select("version", "operation"), Seq(Row(0L, "WRITE"), Row(1L, "SET TBLPROPERTIES")) ) // Attempt truncating the history when dropping a feature that is not required. // This verifies the truncateHistory option was correctly passed. assert(intercept[Exception] { table.dropFeatureSupport("testRemovableWriter", truncateHistory = true) }.getMessage.contains("The particular feature does not require history truncation.")) table.dropFeatureSupport("testRemovableWriter") checkAnswer( table.history().select("version", "operation"), Seq(Row(0L, "WRITE"), Row(1L, "SET TBLPROPERTIES"), Row(2L, "DROP FEATURE")) ) } } test("DataFrameWriter V1 overwrite preserves partitioning information") { withTable("foo") { val data = Seq( (1, "Alice", 29), (2, "Bob", 35), (3, "Charlie", 23) ) val df = spark.createDataFrame(data).toDF("id", "name", "age") df.write.partitionBy("age").format("delta").saveAsTable("foo") val overwriteData = Seq( (4, "Flip", 11), (5, "Flap", 11), (6, "Flop", 13), (6, "Flep", 13) ) val df1 = spark.createDataFrame(overwriteData).toDF("id", "name", "age") df1.write .format("delta") .mode("overwrite") .saveAsTable("foo") // Verify partitioning is preserved assert( DeltaTable .forName(spark, "foo") .detail() .select("partitionColumns") .head() .getSeq[String](0) == Seq("age")) // Verify row count after overwrite assert(DeltaTable.forName(spark, "foo").toDF.count() == 4) } } test("DataFrameWriter V1 replaceWhere preserves non-overwritten partitions") { withTable("foo") { val data = Seq( (1, "Alice", 29), (2, "Bob", 35), (3, "Charlie", 23) ) val df = spark.createDataFrame(data).toDF("id", "name", "age") df.write.partitionBy("age").format("delta").saveAsTable("foo") val overwriteData = Seq((4, "Daniel", 29), (5, "Eve", 29)) val df1 = spark.createDataFrame(overwriteData).toDF("id", "name", "age") df1.write .format("delta") .option("replaceWhere", "age = 29") .mode("overwrite") .saveAsTable("foo") // Verify partitioning is preserved assert( DeltaTable .forName(spark, "foo") .detail() .select("partitionColumns") .head() .getSeq[String](0) == Seq("age")) // Verify row count - should have 4 rows (2 replaced + 2 preserved) assert(DeltaTable.forName(spark, "foo").toDF.count() == 4) } } } ================================================ FILE: spark-connect/client/src/test/scala/io/delta/connect/tables/RemoteSparkSession.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables import java.io.{BufferedReader, File, InputStreamReader} import java.util.concurrent.TimeUnit import org.scalatest.{BeforeAndAfterAll, Suite} import org.apache.spark.sql.connect.SparkSession /** * An util class to start a local Delta Connect server in a different process for local E2E tests. * Pre-running the tests, the Delta Connect artifact needs to be built using e.g. `build/sbt * assembly`. It is designed to start the server once but shared by all tests. It is equivalent to * use the following command to start the connect server via command line: * * {{{ * bin/spark-shell \ * --jars `ls spark-connect/server/target/**/delta-connect-server-assembly*SNAPSHOT.jar | paste -sd ',' -` \ * --conf spark.plugins=org.apache.spark.sql.connect.SparkConnectPlugin * --conf spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension * --conf spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog * }}} * * Set system property `delta.test.home` or env variable `DELTA_HOME` if the test is not executed * from the Delta project top folder. */ trait RemoteSparkSession extends BeforeAndAfterAll { self: Suite => // TODO: Instead of hard-coding the server port, assign port number the same way as in Spark Connect. private val serverPort = 15003 var spark: SparkSession = _ private val javaHome = System.getProperty("java.home") private val sparkHome = System.getProperty("delta.spark.home") private lazy val server = { // We start SparkSubmit directly. This saves us from downloading an entire Spark distribution // for a single test. The parameters used here are the ones that would have been used to start // spark-submit. val command = Seq.newBuilder[String] command += s"$javaHome/bin/java" // Uncomment for debugging the server process. // command += "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005" command += "-cp" += sparkHome + "/jars/*" command += "-Xmx1g" command += "-XX:+IgnoreUnrecognizedVMOptions" command += "--add-modules=jdk.incubator.vector" command += "--add-opens=java.base/java.lang=ALL-UNNAMED" command += "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED" command += "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED" command += "--add-opens=java.base/java.io=ALL-UNNAMED" command += "--add-opens=java.base/java.nio=ALL-UNNAMED" command += "--add-opens=java.base/java.net=ALL-UNNAMED" command += "--add-opens=java.base/java.util=ALL-UNNAMED" command += "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED" command += "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED" command += "--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED" command += "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED" command += "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED" command += "--add-opens=java.base/sun.security.action=ALL-UNNAMED" command += "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED" command += "--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED" command += "-Djdk.reflect.useDirectMethodHandle=false" command += "-Dio.netty.tryReflectionSetAccessible=true" command += "-Dderby.connection.requireAuthentication=false" command += "-Dlog4j.configurationFile=conf/log4j2.properties" command += "org.apache.spark.deploy.SparkSubmit" command += "--class" += "io.delta.tables.SimpleDeltaConnectService" command += "--conf" += s"spark.connect.grpc.binding.port=$serverPort" command += "--conf" += "spark.connect.extensions.relation.classes=" + "org.apache.spark.sql.connect.delta.DeltaRelationPlugin" command += "--conf" += "spark.connect.extensions.command.classes=" + "org.apache.spark.sql.connect.delta.DeltaCommandPlugin" // Spark submit requires a jar. We pick one we know exists. command += s"$sparkHome/jars/unused-1.0.0.jar" val builder = new ProcessBuilder(command.result(): _*) builder.environment().put("SPARK_HOME", sparkHome) builder.environment().put("SPARK_LOCAL_IP", "127.0.0.1") builder.directory(new File(sparkHome)) builder.redirectError(ProcessBuilder.Redirect.INHERIT) builder.start() } private val SERVER_READY_TIMEOUT_SECONDS = 120 /** * Wait for the server process to print "Ready for client connections." to stdout, * indicating the gRPC port is bound and accepting connections. */ private def waitForServerReady(process: Process): Unit = { val reader = new BufferedReader(new InputStreamReader(process.getInputStream)) val deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(SERVER_READY_TIMEOUT_SECONDS) var ready = false var line: String = null while (!ready && System.nanoTime() < deadline) { line = reader.readLine() if (line == null) { throw new RuntimeException( "Spark Connect server process exited before becoming ready. " + s"Exit code: ${process.exitValue()}") } // Mirror to stdout so server logs are still visible in test output. // scalastyle:off println println(line) // scalastyle:on println if (line.contains("Ready for client connections.")) { ready = true } } if (!ready) { process.destroy() throw new RuntimeException( s"Spark Connect server did not become ready within $SERVER_READY_TIMEOUT_SECONDS seconds") } // Continue forwarding server stdout in background so the process doesn't block on a full // output buffer. val forwarder = new Thread(() => { var l: String = null while ({ l = reader.readLine(); l != null }) { // scalastyle:off println println(l) // scalastyle:on println } }) forwarder.setDaemon(true) forwarder.start() } override def beforeAll(): Unit = { super.beforeAll() val process = server waitForServerReady(process) spark = SparkSession.builder().remote(s"sc://localhost:$serverPort").create() } override def afterAll(): Unit = { server.destroy() super.afterAll() } } ================================================ FILE: spark-connect/common/src/main/buf.gen.yaml ================================================ # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # version: v1 plugins: - plugin: buf.build/protocolbuffers/python:v21.7 out: gen/proto/python - name: mypy out: gen/proto/python ================================================ FILE: spark-connect/common/src/main/buf.work.yaml ================================================ # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # version: v1 directories: - protobuf ================================================ FILE: spark-connect/common/src/main/protobuf/buf.yaml ================================================ # # Copyright (2024) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # version: v1 breaking: use: - FILE lint: use: - DEFAULT ================================================ FILE: spark-connect/common/src/main/protobuf/delta/connect/base.proto ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ syntax = 'proto3'; package delta.connect; option java_multiple_files = true; option java_package = "io.delta.connect.proto"; // Information required to access a Delta table either by name or by path. message DeltaTable { // (Required) oneof access_type { Path path = 1; string table_or_view_name = 2; } message Path { // (Required) Path to the Delta table. string path = 1; // (Optional) Hadoop configuration used to access the file system. map hadoop_conf = 2; } } ================================================ FILE: spark-connect/common/src/main/protobuf/delta/connect/commands.proto ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ syntax = 'proto3'; package delta.connect; import "delta/connect/base.proto"; import "spark/connect/types.proto"; option java_multiple_files = true; option java_package = "io.delta.connect.proto"; // Message to hold all command extensions in Delta Connect. message DeltaCommand { oneof command_type { CloneTable clone_table = 1; VacuumTable vacuum_table = 2; UpgradeTableProtocol upgrade_table_protocol = 3; Generate generate = 4; CreateDeltaTable create_delta_table = 5; AddFeatureSupport add_feature_support = 6; DropFeatureSupport drop_feature_support = 7; } } // Command that creates a copy of a DeltaTable in the specified target location. message CloneTable { // (Required) The source Delta table to clone. DeltaTable table = 1; // (Required) Path to the location where the cloned table should be stored. string target = 2; // (Optional) Optional parameter to clone a previous state of the source table. The current // state of the table is cloned when it is not specified. oneof version_or_timestamp { // Clones the source table as of the provided version. int32 version = 3; // Clones the source table as of the provided timestamp. string timestamp = 4; } // (Required) Performs a clone when true, this field should always be set to true. bool is_shallow = 5; // (Required) Overwrites the target location when true. bool replace = 6; // (Required) User-defined table properties that override properties with the same key in the // source table. map properties = 7; } // Command that deletes files and directories in the table that are not needed by the table for // maintaining older versions up to the given retention threshold. message VacuumTable { // (Required) The Delta table to vacuum. DeltaTable table = 1; // (Optional) Number of hours retain history for. If not specified, then the default retention // period will be used. optional double retention_hours = 2; } // Command to updates the protocol version of the table so that new features can be used. message UpgradeTableProtocol { // (Required) The Delta table to upgrade the protocol of. DeltaTable table = 1; // (Required) The minimum required reader protocol version. int32 reader_version = 2; // (Required) The minimum required writer protocol version. int32 writer_version = 3; } // Command that generates manifest files for a given Delta table. message Generate { // (Required) The Delta table to generate the manifest files for. DeltaTable table = 1; // (Required) The type of manifest file to be generated. string mode = 2; } // Command that creates or replace a Delta table (depending on the mode). message CreateDeltaTable { enum Mode { MODE_UNSPECIFIED = 0; // Create the table if it does not exist, and throw an error otherwise. MODE_CREATE = 1; // Create the table if it does not exist, and do nothing otherwise. MODE_CREATE_IF_NOT_EXISTS = 2; // Replace the table if it already exists, and throw an error otherwise. MODE_REPLACE = 3; // Create the table if it does not exist, and replace it otherwise. MODE_CREATE_OR_REPLACE = 4; } // Column in the schema of the table. message Column { message IdentityInfo { // (Required) The start value of the identity column. int64 start = 1; // (Required) The increment value of the identity column. int64 step = 2; // (Required) Whether the identity column is BY DEFAULT (true) or ALWAYS (false). bool allow_explicit_insert = 3; } // (Required) Name of the column. string name = 1; // (Required) Data type of the column. spark.connect.DataType data_type = 2; // (Required) Whether the column is nullable. bool nullable = 3; // (Optional) SQL Expression that is used to generate the values in the column. optional string generated_always_as = 4; // (Optional) Comment to describe the column. optional string comment = 5; // (Optional) Identity information for the column. optional IdentityInfo identity_info = 6; } // (Required) Mode that determines what to do when a table with the given name or location // already exists. Mode mode = 1; // (Optional) Qualified name of the table. optional string table_name = 2; // (Optional) Path to the directory where the table date is stored. optional string location = 3; // (Optional) Comment describing the table. optional string comment = 4; // (Optional) Columns in the schema of the table. repeated Column columns = 5; // (Optional) Columns used for partitioning the table. repeated string partitioning_columns = 6; // (Optional) Properties of the table. map properties = 7; // (Optional) Columns used for clustering the table. repeated string clustering_columns = 8; } // Command to add a supported feature to the table by modifying the protocol. message AddFeatureSupport { // (Required) The Delta table to add the supported feature to. DeltaTable table = 1; // (Required) The name of the supported feature to add. string feature_name = 2; } // Command to drop a supported feature from the table by modifying the protocol. message DropFeatureSupport { // (Required) The Delta table to drop the supported feature from. DeltaTable table = 1; // (Required) The name of the supported feature to drop. string feature_name = 2; // (optional) Whether to truncate history. When not specified, history is not truncated. optional bool truncate_history = 3; } ================================================ FILE: spark-connect/common/src/main/protobuf/delta/connect/relations.proto ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ syntax = 'proto3'; package delta.connect; import "delta/connect/base.proto"; import "spark/connect/expressions.proto"; import "spark/connect/relations.proto"; import "spark/connect/types.proto"; option java_multiple_files = true; option java_package = "io.delta.connect.proto"; // Message to hold all relation extensions in Delta Connect. message DeltaRelation { oneof relation_type { Scan scan = 1; DescribeHistory describe_history = 2; DescribeDetail describe_detail = 3; ConvertToDelta convert_to_delta = 4; RestoreTable restore_table = 5; IsDeltaTable is_delta_table = 6; DeleteFromTable delete_from_table = 7; UpdateTable update_table = 8; MergeIntoTable merge_into_table = 9; OptimizeTable optimize_table = 10; } } // Relation that reads from a Delta table. message Scan { // (Required) The Delta table to scan. DeltaTable table = 1; } // Relation containing information of the latest commits on a Delta table. // The information is in reverse chronological order. message DescribeHistory { // (Required) The Delta table to read the history of. DeltaTable table = 1; } // Relation containing the details of a Delta table such as the format, name, and size. message DescribeDetail { // (Required) The Delta table to describe the details of. DeltaTable table = 1; } // Command that turns a Parquet table into a Delta table. // // This needs to be a Relation as it returns the identifier of the resulting table. // We cannot simply reuse the input identifier, as it could be a path-based identifier, // and in that case we need to replace "parquet.`...`" with "delta.`...`". message ConvertToDelta { // (Required) Parquet table identifier formatted as "parquet.`path`" string identifier = 1; // (Optional) Partitioning schema of the input table oneof partition_schema { // Hive DDL formatted string string partition_schema_string = 2; // Struct with names and types of partitioning columns spark.connect.DataType partition_schema_struct = 3; } } // Command that restores the DeltaTable to an older version of the table specified by either a // version number or a timestamp. // // Needs to be a Relation, as it returns a row containing the execution metrics. message RestoreTable { // (Required) The Delta table to restore to an earlier version. DeltaTable table = 1; // (Required) Version to restore to. oneof version_or_timestamp { // The version number to restore to. int64 version = 2; // The timestamp to restore to. string timestamp = 3; } } // Relation containing a single row containing a single boolean that indicates whether the provided // path contains a Delta table. message IsDeltaTable { // (Required) The path to check. string path = 1; } // Command that deletes data from the target table that matches the given condition. // // Needs to be a Relation, as it returns a row containing the execution metrics. message DeleteFromTable { // (Required) Target table to delete data from. Must either be a DeltaRelation containing a Scan // or a SubqueryAlias with a DeltaRelation containing a Scan as its input. spark.connect.Relation target = 1; // (Optional) Expression returning a boolean. spark.connect.Expression condition = 2; } // Command that updates data in the target table using the given assignments for rows that matches // the given condition. // // Needs to be a Relation, as it returns a row containing the execution metrics. message UpdateTable { // (Required) Target table to delete data from. Must either be a DeltaRelation containing a Scan // or a SubqueryAlias with a DeltaRelation containing a Scan as its input. spark.connect.Relation target = 1; // (Optional) Condition that determines which rows must be updated. // Must be an expression returning a boolean. spark.connect.Expression condition = 2; // (Optional) Set of assignments to apply to the rows matching the condition. repeated Assignment assignments = 3; } // Command that merges a source query/table into a Delta table, // // Needs to be a Relation, as it returns a row containing the execution metrics. message MergeIntoTable { // (Required) Target table to merge into. spark.connect.Relation target = 1; // (Required) Source data to merge from. spark.connect.Relation source = 2; // (Required) Condition for a source row to match with a target row. spark.connect.Expression condition = 3; // (Optional) Actions to apply when a source row matches a target row. repeated Action matched_actions = 4; // (Optional) Actions to apply when a source row does not match a target row. repeated Action not_matched_actions = 5; // (Optional) Actions to apply when a target row does not match a source row. repeated Action not_matched_by_source_actions = 6; // (Optional) Whether Schema Evolution is enabled for this command. optional bool with_schema_evolution = 7; // Rule that specifies how the target table should be modified. message Action { // (Optional) Condition for the action to be applied. spark.connect.Expression condition = 1; // (Required) oneof action_type { DeleteAction delete_action = 2; UpdateAction update_action = 3; UpdateStarAction update_star_action = 4; InsertAction insert_action = 5; InsertStarAction insert_star_action = 6; } // Action that deletes the target row. message DeleteAction {} // Action that updates the target row using a set of assignments. message UpdateAction { // (Optional) Set of assignments to apply. repeated Assignment assignments = 1; } // Action that updates the target row by overwriting all columns. message UpdateStarAction {} // Action that inserts the source row into the target using a set of assignments. message InsertAction { // (Optional) Set of assignments to apply. repeated Assignment assignments = 1; } // Action that inserts the source row into the target by setting all columns. message InsertStarAction {} } } // Represents an assignment of a value to a field. message Assignment { // (Required) Expression identifying the (struct) field that is assigned a new value. spark.connect.Expression field = 1; // (Required) Expression that produces the value to assign to the field. spark.connect.Expression value = 2; } // Command that optimizes the layout of a Delta table by either compacting small files or // by ordering the data. Allows specifying partition filters to limit the scope of the data // reorganization. // // Needs to be a Relation, as it returns a row containing the execution metrics. message OptimizeTable { // (Required) The Delta table to optimize. DeltaTable table = 1; // (Optional) Partition filters that limit the operation to the files in the matched partitions. repeated string partition_filters = 2; // (Optional) Columns to z-order by. Compaction is performed when no z-order columns are provided. repeated string zorder_columns = 3; } ================================================ FILE: spark-connect/common/src/main/protobuf/spark/connect/base.proto ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ syntax = 'proto3'; package spark.connect; import "google/protobuf/any.proto"; import "spark/connect/commands.proto"; import "spark/connect/common.proto"; import "spark/connect/expressions.proto"; import "spark/connect/relations.proto"; import "spark/connect/types.proto"; import "spark/connect/ml.proto"; import "spark/connect/pipelines.proto"; option java_multiple_files = true; option java_package = "io.delta.connect.spark.proto"; option go_package = "internal/generated"; // A [[Plan]] is the structure that carries the runtime information for the execution from the // client to the server. A [[Plan]] can either be of the type [[Relation]] which is a reference // to the underlying logical plan or it can be of the [[Command]] type that is used to execute // commands on the server. message Plan { oneof op_type { Relation root = 1; Command command = 2; } } // User Context is used to refer to one particular user session that is executing // queries in the backend. message UserContext { string user_id = 1; string user_name = 2; // To extend the existing user context message that is used to identify incoming requests, // Spark Connect leverages the Any protobuf type that can be used to inject arbitrary other // messages into this message. Extensions are stored as a `repeated` type to be able to // handle multiple active extensions. repeated google.protobuf.Any extensions = 999; } // Request to perform plan analyze, optionally to explain the plan. message AnalyzePlanRequest { // (Required) // // The session_id specifies a spark session for a user id (which is specified // by user_context.user_id). The session_id is set by the client to be able to // collate streaming responses from different queries within the dedicated session. // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` string session_id = 1; // (Optional) // // Server-side generated idempotency key from the previous responses (if any). Server // can use this to validate that the server side session has not changed. optional string client_observed_server_side_session_id = 17; // (Required) User context UserContext user_context = 2; // Provides optional information about the client sending the request. This field // can be used for language or version specific information and is only intended for // logging purposes and will not be interpreted by the server. optional string client_type = 3; oneof analyze { Schema schema = 4; Explain explain = 5; TreeString tree_string = 6; IsLocal is_local = 7; IsStreaming is_streaming = 8; InputFiles input_files = 9; SparkVersion spark_version = 10; DDLParse ddl_parse = 11; SameSemantics same_semantics = 12; SemanticHash semantic_hash = 13; Persist persist = 14; Unpersist unpersist = 15; GetStorageLevel get_storage_level = 16; JsonToDDL json_to_ddl = 18; } message Schema { // (Required) The logical plan to be analyzed. Plan plan = 1; } // Explains the input plan based on a configurable mode. message Explain { // (Required) The logical plan to be analyzed. Plan plan = 1; // (Required) For analyzePlan rpc calls, configure the mode to explain plan in strings. ExplainMode explain_mode = 2; // Plan explanation mode. enum ExplainMode { EXPLAIN_MODE_UNSPECIFIED = 0; // Generates only physical plan. EXPLAIN_MODE_SIMPLE = 1; // Generates parsed logical plan, analyzed logical plan, optimized logical plan and physical plan. // Parsed Logical plan is a unresolved plan that extracted from the query. Analyzed logical plans // transforms which translates unresolvedAttribute and unresolvedRelation into fully typed objects. // The optimized logical plan transforms through a set of optimization rules, resulting in the // physical plan. EXPLAIN_MODE_EXTENDED = 2; // Generates code for the statement, if any and a physical plan. EXPLAIN_MODE_CODEGEN = 3; // If plan node statistics are available, generates a logical plan and also the statistics. EXPLAIN_MODE_COST = 4; // Generates a physical plan outline and also node details. EXPLAIN_MODE_FORMATTED = 5; } } message TreeString { // (Required) The logical plan to be analyzed. Plan plan = 1; // (Optional) Max level of the schema. optional int32 level = 2; } message IsLocal { // (Required) The logical plan to be analyzed. Plan plan = 1; } message IsStreaming { // (Required) The logical plan to be analyzed. Plan plan = 1; } message InputFiles { // (Required) The logical plan to be analyzed. Plan plan = 1; } message SparkVersion { } message DDLParse { // (Required) The DDL formatted string to be parsed. string ddl_string = 1; } // Returns `true` when the logical query plans are equal and therefore return same results. message SameSemantics { // (Required) The plan to be compared. Plan target_plan = 1; // (Required) The other plan to be compared. Plan other_plan = 2; } message SemanticHash { // (Required) The logical plan to get a hashCode. Plan plan = 1; } message Persist { // (Required) The logical plan to persist. Relation relation = 1; // (Optional) The storage level. optional StorageLevel storage_level = 2; } message Unpersist { // (Required) The logical plan to unpersist. Relation relation = 1; // (Optional) Whether to block until all blocks are deleted. optional bool blocking = 2; } message GetStorageLevel { // (Required) The logical plan to get the storage level. Relation relation = 1; } message JsonToDDL { // (Required) The JSON formatted string to be converted to DDL. string json_string = 1; } } // Response to performing analysis of the query. Contains relevant metadata to be able to // reason about the performance. // Next ID: 16 message AnalyzePlanResponse { string session_id = 1; // Server-side generated idempotency key that the client can use to assert that the server side // session has not changed. string server_side_session_id = 15; oneof result { Schema schema = 2; Explain explain = 3; TreeString tree_string = 4; IsLocal is_local = 5; IsStreaming is_streaming = 6; InputFiles input_files = 7; SparkVersion spark_version = 8; DDLParse ddl_parse = 9; SameSemantics same_semantics = 10; SemanticHash semantic_hash = 11; Persist persist = 12; Unpersist unpersist = 13; GetStorageLevel get_storage_level = 14; JsonToDDL json_to_ddl = 16; } message Schema { DataType schema = 1; } message Explain { string explain_string = 1; } message TreeString { string tree_string = 1; } message IsLocal { bool is_local = 1; } message IsStreaming { bool is_streaming = 1; } message InputFiles { // A best-effort snapshot of the files that compose this Dataset repeated string files = 1; } message SparkVersion { string version = 1; } message DDLParse { DataType parsed = 1; } message SameSemantics { bool result = 1; } message SemanticHash { int32 result = 1; } message Persist { } message Unpersist { } message GetStorageLevel { // (Required) The StorageLevel as a result of get_storage_level request. StorageLevel storage_level = 1; } message JsonToDDL { string ddl_string = 1; } } // A request to be executed by the service. message ExecutePlanRequest { // (Required) // // The session_id specifies a spark session for a user id (which is specified // by user_context.user_id). The session_id is set by the client to be able to // collate streaming responses from different queries within the dedicated session. // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` string session_id = 1; // (Optional) // // Server-side generated idempotency key from the previous responses (if any). Server // can use this to validate that the server side session has not changed. optional string client_observed_server_side_session_id = 8; // (Required) User context // // user_context.user_id and session+id both identify a unique remote spark session on the // server side. UserContext user_context = 2; // (Optional) // Provide an id for this request. If not provided, it will be generated by the server. // It is returned in every ExecutePlanResponse.operation_id of the ExecutePlan response stream. // The id must be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` optional string operation_id = 6; // (Required) The logical plan to be executed / analyzed. Plan plan = 3; // Provides optional information about the client sending the request. This field // can be used for language or version specific information and is only intended for // logging purposes and will not be interpreted by the server. optional string client_type = 4; // Repeated element for options that can be passed to the request. This element is currently // unused but allows to pass in an extension value used for arbitrary options. repeated RequestOption request_options = 5; message RequestOption { oneof request_option { ReattachOptions reattach_options = 1; // Extension type for request options google.protobuf.Any extension = 999; } } // Tags to tag the given execution with. // Tags cannot contain ',' character and cannot be empty strings. // Used by Interrupt with interrupt.tag. repeated string tags = 7; } // The response of a query, can be one or more for each request. Responses belonging to the // same input query, carry the same `session_id`. // Next ID: 17 message ExecutePlanResponse { string session_id = 1; // Server-side generated idempotency key that the client can use to assert that the server side // session has not changed. string server_side_session_id = 15; // Identifies the ExecutePlan execution. // If set by the client in ExecutePlanRequest.operationId, that value is returned. // Otherwise generated by the server. // It is an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` string operation_id = 12; // Identified the response in the stream. // The id is an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` string response_id = 13; // Union type for the different response messages. oneof response_type { ArrowBatch arrow_batch = 2; // Special case for executing SQL commands. SqlCommandResult sql_command_result = 5; // Response for a streaming query. WriteStreamOperationStartResult write_stream_operation_start_result = 8; // Response for commands on a streaming query. StreamingQueryCommandResult streaming_query_command_result = 9; // Response for 'SparkContext.resources'. GetResourcesCommandResult get_resources_command_result = 10; // Response for commands on the streaming query manager. StreamingQueryManagerCommandResult streaming_query_manager_command_result = 11; // Response for commands on the client side streaming query listener. StreamingQueryListenerEventsResult streaming_query_listener_events_result = 16; // Response type informing if the stream is complete in reattachable execution. ResultComplete result_complete = 14; // Response for command that creates ResourceProfile. CreateResourceProfileCommandResult create_resource_profile_command_result = 17; // (Optional) Intermediate query progress reports. ExecutionProgress execution_progress = 18; // Response for command that checkpoints a DataFrame. CheckpointCommandResult checkpoint_command_result = 19; // ML command response MlCommandResult ml_command_result = 20; // Response containing pipeline event that is streamed back to the client during a pipeline run PipelineEventResult pipeline_event_result = 21; // Pipeline command response PipelineCommandResult pipeline_command_result = 22; // Support arbitrary result objects. google.protobuf.Any extension = 999; } // Metrics for the query execution. Typically, this field is only present in the last // batch of results and then represent the overall state of the query execution. Metrics metrics = 4; // The metrics observed during the execution of the query plan. repeated ObservedMetrics observed_metrics = 6; // (Optional) The Spark schema. This field is available when `collect` is called. DataType schema = 7; // A SQL command returns an opaque Relation that can be directly used as input for the next // call. message SqlCommandResult { Relation relation = 1; } // Batch results of metrics. message ArrowBatch { // Count rows in `data`. Must match the number of rows inside `data`. int64 row_count = 1; // Serialized Arrow data. bytes data = 2; // If set, row offset of the start of this ArrowBatch in execution results. optional int64 start_offset = 3; } message Metrics { repeated MetricObject metrics = 1; message MetricObject { string name = 1; int64 plan_id = 2; int64 parent = 3; map execution_metrics = 4; } message MetricValue { string name = 1; int64 value = 2; string metric_type = 3; } } message ObservedMetrics { string name = 1; repeated Expression.Literal values = 2; repeated string keys = 3; int64 plan_id = 4; } message ResultComplete { // If present, in a reattachable execution this means that after server sends onComplete, // the execution is complete. If the server sends onComplete without sending a ResultComplete, // it means that there is more, and the client should use ReattachExecute RPC to continue. } // This message is used to communicate progress about the query progress during the execution. message ExecutionProgress { // Captures the progress of each individual stage. repeated StageInfo stages = 1; // Captures the currently in progress tasks. int64 num_inflight_tasks = 2; message StageInfo { int64 stage_id = 1; int64 num_tasks = 2; int64 num_completed_tasks = 3; int64 input_bytes_read = 4; bool done = 5; } } } // The key-value pair for the config request and response. message KeyValue { // (Required) The key. string key = 1; // (Optional) The value. optional string value = 2; } // Request to update or fetch the configurations. message ConfigRequest { // (Required) // // The session_id specifies a spark session for a user id (which is specified // by user_context.user_id). The session_id is set by the client to be able to // collate streaming responses from different queries within the dedicated session. // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` string session_id = 1; // (Optional) // // Server-side generated idempotency key from the previous responses (if any). Server // can use this to validate that the server side session has not changed. optional string client_observed_server_side_session_id = 8; // (Required) User context UserContext user_context = 2; // (Required) The operation for the config. Operation operation = 3; // Provides optional information about the client sending the request. This field // can be used for language or version specific information and is only intended for // logging purposes and will not be interpreted by the server. optional string client_type = 4; message Operation { oneof op_type { Set set = 1; Get get = 2; GetWithDefault get_with_default = 3; GetOption get_option = 4; GetAll get_all = 5; Unset unset = 6; IsModifiable is_modifiable = 7; } } message Set { // (Required) The config key-value pairs to set. repeated KeyValue pairs = 1; // (Optional) Whether to ignore failures. optional bool silent = 2; } message Get { // (Required) The config keys to get. repeated string keys = 1; } message GetWithDefault { // (Required) The config key-value pairs to get. The value will be used as the default value. repeated KeyValue pairs = 1; } message GetOption { // (Required) The config keys to get optionally. repeated string keys = 1; } message GetAll { // (Optional) The prefix of the config key to get. optional string prefix = 1; } message Unset { // (Required) The config keys to unset. repeated string keys = 1; } message IsModifiable { // (Required) The config keys to check the config is modifiable. repeated string keys = 1; } } // Response to the config request. // Next ID: 5 message ConfigResponse { string session_id = 1; // Server-side generated idempotency key that the client can use to assert that the server side // session has not changed. string server_side_session_id = 4; // (Optional) The result key-value pairs. // // Available when the operation is 'Get', 'GetWithDefault', 'GetOption', 'GetAll'. // Also available for the operation 'IsModifiable' with boolean string "true" and "false". repeated KeyValue pairs = 2; // (Optional) // // Warning messages for deprecated or unsupported configurations. repeated string warnings = 3; } // Request to transfer client-local artifacts. message AddArtifactsRequest { // (Required) // // The session_id specifies a spark session for a user id (which is specified // by user_context.user_id). The session_id is set by the client to be able to // collate streaming responses from different queries within the dedicated session. // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` string session_id = 1; // User context UserContext user_context = 2; // (Optional) // // Server-side generated idempotency key from the previous responses (if any). Server // can use this to validate that the server side session has not changed. optional string client_observed_server_side_session_id = 7; // Provides optional information about the client sending the request. This field // can be used for language or version specific information and is only intended for // logging purposes and will not be interpreted by the server. optional string client_type = 6; // A chunk of an Artifact. message ArtifactChunk { // Data chunk. bytes data = 1; // CRC to allow server to verify integrity of the chunk. int64 crc = 2; } // An artifact that is contained in a single `ArtifactChunk`. // Generally, this message represents tiny artifacts such as REPL-generated class files. message SingleChunkArtifact { // The name of the artifact is expected in the form of a "Relative Path" that is made up of a // sequence of directories and the final file element. // Examples of "Relative Path"s: "jars/test.jar", "classes/xyz.class", "abc.xyz", "a/b/X.jar". // The server is expected to maintain the hierarchy of files as defined by their name. (i.e // The relative path of the file on the server's filesystem will be the same as the name of // the provided artifact) string name = 1; // A single data chunk. ArtifactChunk data = 2; } // A number of `SingleChunkArtifact` batched into a single RPC. message Batch { repeated SingleChunkArtifact artifacts = 1; } // Signals the beginning/start of a chunked artifact. // A large artifact is transferred through a payload of `BeginChunkedArtifact` followed by a // sequence of `ArtifactChunk`s. message BeginChunkedArtifact { // Name of the artifact undergoing chunking. Follows the same conventions as the `name` in // the `Artifact` message. string name = 1; // Total size of the artifact in bytes. int64 total_bytes = 2; // Number of chunks the artifact is split into. // This includes the `initial_chunk`. int64 num_chunks = 3; // The first/initial chunk. ArtifactChunk initial_chunk = 4; } // The payload is either a batch of artifacts or a partial chunk of a large artifact. oneof payload { Batch batch = 3; // The metadata and the initial chunk of a large artifact chunked into multiple requests. // The server side is notified about the total size of the large artifact as well as the // number of chunks to expect. BeginChunkedArtifact begin_chunk = 4; // A chunk of an artifact excluding metadata. This can be any chunk of a large artifact // excluding the first chunk (which is included in `BeginChunkedArtifact`). ArtifactChunk chunk = 5; } } // Response to adding an artifact. Contains relevant metadata to verify successful transfer of // artifact(s). // Next ID: 4 message AddArtifactsResponse { // Session id in which the AddArtifact was running. string session_id = 2; // Server-side generated idempotency key that the client can use to assert that the server side // session has not changed. string server_side_session_id = 3; // The list of artifact(s) seen by the server. repeated ArtifactSummary artifacts = 1; // Metadata of an artifact. message ArtifactSummary { string name = 1; // Whether the CRC (Cyclic Redundancy Check) is successful on server verification. // The server discards any artifact that fails the CRC. // If false, the client may choose to resend the artifact specified by `name`. bool is_crc_successful = 2; } } // Request to get current statuses of artifacts at the server side. message ArtifactStatusesRequest { // (Required) // // The session_id specifies a spark session for a user id (which is specified // by user_context.user_id). The session_id is set by the client to be able to // collate streaming responses from different queries within the dedicated session. // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` string session_id = 1; // (Optional) // // Server-side generated idempotency key from the previous responses (if any). Server // can use this to validate that the server side session has not changed. optional string client_observed_server_side_session_id = 5; // User context UserContext user_context = 2; // Provides optional information about the client sending the request. This field // can be used for language or version specific information and is only intended for // logging purposes and will not be interpreted by the server. optional string client_type = 3; // The name of the artifact is expected in the form of a "Relative Path" that is made up of a // sequence of directories and the final file element. // Examples of "Relative Path"s: "jars/test.jar", "classes/xyz.class", "abc.xyz", "a/b/X.jar". // The server is expected to maintain the hierarchy of files as defined by their name. (i.e // The relative path of the file on the server's filesystem will be the same as the name of // the provided artifact) repeated string names = 4; } // Response to checking artifact statuses. // Next ID: 4 message ArtifactStatusesResponse { // Session id in which the ArtifactStatus was running. string session_id = 2; // Server-side generated idempotency key that the client can use to assert that the server side // session has not changed. string server_side_session_id = 3; // A map of artifact names to their statuses. map statuses = 1; message ArtifactStatus { // Exists or not particular artifact at the server. bool exists = 1; } } message InterruptRequest { // (Required) // // The session_id specifies a spark session for a user id (which is specified // by user_context.user_id). The session_id is set by the client to be able to // collate streaming responses from different queries within the dedicated session. // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` string session_id = 1; // (Optional) // // Server-side generated idempotency key from the previous responses (if any). Server // can use this to validate that the server side session has not changed. optional string client_observed_server_side_session_id = 7; // (Required) User context UserContext user_context = 2; // Provides optional information about the client sending the request. This field // can be used for language or version specific information and is only intended for // logging purposes and will not be interpreted by the server. optional string client_type = 3; // (Required) The type of interrupt to execute. InterruptType interrupt_type = 4; enum InterruptType { INTERRUPT_TYPE_UNSPECIFIED = 0; // Interrupt all running executions within the session with the provided session_id. INTERRUPT_TYPE_ALL = 1; // Interrupt all running executions within the session with the provided operation_tag. INTERRUPT_TYPE_TAG = 2; // Interrupt the running execution within the session with the provided operation_id. INTERRUPT_TYPE_OPERATION_ID = 3; } oneof interrupt { // if interrupt_tag == INTERRUPT_TYPE_TAG, interrupt operation with this tag. string operation_tag = 5; // if interrupt_tag == INTERRUPT_TYPE_OPERATION_ID, interrupt operation with this operation_id. string operation_id = 6; } } // Next ID: 4 message InterruptResponse { // Session id in which the interrupt was running. string session_id = 1; // Server-side generated idempotency key that the client can use to assert that the server side // session has not changed. string server_side_session_id = 3; // Operation ids of the executions which were interrupted. repeated string interrupted_ids = 2; } message ReattachOptions { // If true, the request can be reattached to using ReattachExecute. // ReattachExecute can be used either if the stream broke with a GRPC network error, // or if the server closed the stream without sending a response with StreamStatus.complete=true. // The server will keep a buffer of responses in case a response is lost, and // ReattachExecute needs to back-track. // // If false, the execution response stream will will not be reattachable, and all responses are // immediately released by the server after being sent. bool reattachable = 1; } message ReattachExecuteRequest { // (Required) // // The session_id of the request to reattach to. // This must be an id of existing session. string session_id = 1; // (Optional) // // Server-side generated idempotency key from the previous responses (if any). Server // can use this to validate that the server side session has not changed. optional string client_observed_server_side_session_id = 6; // (Required) User context // // user_context.user_id and session+id both identify a unique remote spark session on the // server side. UserContext user_context = 2; // (Required) // Provide an id of the request to reattach to. // This must be an id of existing operation. string operation_id = 3; // Provides optional information about the client sending the request. This field // can be used for language or version specific information and is only intended for // logging purposes and will not be interpreted by the server. optional string client_type = 4; // (Optional) // Last already processed response id from the response stream. // After reattach, server will resume the response stream after that response. // If not specified, server will restart the stream from the start. // // Note: server controls the amount of responses that it buffers and it may drop responses, // that are far behind the latest returned response, so this can't be used to arbitrarily // scroll back the cursor. If the response is no longer available, this will result in an error. optional string last_response_id = 5; } message ReleaseExecuteRequest { // (Required) // // The session_id of the request to reattach to. // This must be an id of existing session. string session_id = 1; // (Optional) // // Server-side generated idempotency key from the previous responses (if any). Server // can use this to validate that the server side session has not changed. optional string client_observed_server_side_session_id = 7; // (Required) User context // // user_context.user_id and session+id both identify a unique remote spark session on the // server side. UserContext user_context = 2; // (Required) // Provide an id of the request to reattach to. // This must be an id of existing operation. string operation_id = 3; // Provides optional information about the client sending the request. This field // can be used for language or version specific information and is only intended for // logging purposes and will not be interpreted by the server. optional string client_type = 4; // Release and close operation completely. // This will also interrupt the query if it is running execution, and wait for it to be torn down. message ReleaseAll {} // Release all responses from the operation response stream up to and including // the response with the given by response_id. // While server determines by itself how much of a buffer of responses to keep, client providing // explicit release calls will help reduce resource consumption. // Noop if response_id not found in cached responses. message ReleaseUntil { string response_id = 1; } oneof release { ReleaseAll release_all = 5; ReleaseUntil release_until = 6; } } // Next ID: 4 message ReleaseExecuteResponse { // Session id in which the release was running. string session_id = 1; // Server-side generated idempotency key that the client can use to assert that the server side // session has not changed. string server_side_session_id = 3; // Operation id of the operation on which the release executed. // If the operation couldn't be found (because e.g. it was concurrently released), will be unset. // Otherwise, it will be equal to the operation_id from request. optional string operation_id = 2; } message ReleaseSessionRequest { // (Required) // // The session_id of the request to reattach to. // This must be an id of existing session. string session_id = 1; // (Required) User context // // user_context.user_id and session+id both identify a unique remote spark session on the // server side. UserContext user_context = 2; // Provides optional information about the client sending the request. This field // can be used for language or version specific information and is only intended for // logging purposes and will not be interpreted by the server. optional string client_type = 3; // Signals the server to allow the client to reconnect to the session after it is released. // // By default, the server tombstones the session upon release, preventing reconnections and // fully cleaning the session state. // // If this flag is set to true, the server may permit the client to reconnect to the session // post-release, even if the session state has been cleaned. This can result in missing state, // such as Temporary Views, Temporary UDFs, or the Current Catalog, in the reconnected session. // // Use this option sparingly and only when the client fully understands the implications of // reconnecting to a released session. The client must ensure that any queries executed do not // rely on the session state prior to its release. bool allow_reconnect = 4; } // Next ID: 3 message ReleaseSessionResponse { // Session id of the session on which the release executed. string session_id = 1; // Server-side generated idempotency key that the client can use to assert that the server side // session has not changed. string server_side_session_id = 2; } message FetchErrorDetailsRequest { // (Required) // The session_id specifies a Spark session for a user identified by user_context.user_id. // The id should be a UUID string of the format `00112233-4455-6677-8899-aabbccddeeff`. string session_id = 1; // (Optional) // // Server-side generated idempotency key from the previous responses (if any). Server // can use this to validate that the server side session has not changed. optional string client_observed_server_side_session_id = 5; // User context UserContext user_context = 2; // (Required) // The id of the error. string error_id = 3; // Provides optional information about the client sending the request. This field // can be used for language or version specific information and is only intended for // logging purposes and will not be interpreted by the server. optional string client_type = 4; } // Next ID: 5 message FetchErrorDetailsResponse { // Server-side generated idempotency key that the client can use to assert that the server side // session has not changed. string server_side_session_id = 3; string session_id = 4; // The index of the root error in errors. The field will not be set if the error is not found. optional int32 root_error_idx = 1; // A list of errors. repeated Error errors = 2; message StackTraceElement { // The fully qualified name of the class containing the execution point. string declaring_class = 1; // The name of the method containing the execution point. string method_name = 2; // The name of the file containing the execution point. optional string file_name = 3; // The line number of the source line containing the execution point. int32 line_number = 4; } // QueryContext defines the schema for the query context of a SparkThrowable. // It helps users understand where the error occurs while executing queries. message QueryContext { // The type of this query context. enum ContextType { SQL = 0; DATAFRAME = 1; } ContextType context_type = 10; // The object type of the query which throws the exception. // If the exception is directly from the main query, it should be an empty string. // Otherwise, it should be the exact object type in upper case. For example, a "VIEW". string object_type = 1; // The object name of the query which throws the exception. // If the exception is directly from the main query, it should be an empty string. // Otherwise, it should be the object name. For example, a view name "V1". string object_name = 2; // The starting index in the query text which throws the exception. The index starts from 0. int32 start_index = 3; // The stopping index in the query which throws the exception. The index starts from 0. int32 stop_index = 4; // The corresponding fragment of the query which throws the exception. string fragment = 5; // The user code (call site of the API) that caused throwing the exception. string call_site = 6; // Summary of the exception cause. string summary = 7; } // SparkThrowable defines the schema for SparkThrowable exceptions. message SparkThrowable { // Succinct, human-readable, unique, and consistent representation of the error category. optional string error_class = 1; // The message parameters for the error framework. map message_parameters = 2; // The query context of a SparkThrowable. repeated QueryContext query_contexts = 3; // Portable error identifier across SQL engines // If null, error class or SQLSTATE is not set. optional string sql_state = 4; } // Error defines the schema for the representing exception. message Error { // The fully qualified names of the exception class and its parent classes. repeated string error_type_hierarchy = 1; // The detailed message of the exception. string message = 2; // The stackTrace of the exception. It will be set // if the SQLConf spark.sql.connect.serverStacktrace.enabled is true. repeated StackTraceElement stack_trace = 3; // The index of the cause error in errors. optional int32 cause_idx = 4; // The structured data of a SparkThrowable exception. optional SparkThrowable spark_throwable = 5; } } message CheckpointCommandResult { // (Required) The logical plan checkpointed. CachedRemoteRelation relation = 1; } // Main interface for the SparkConnect service. service SparkConnectService { // Executes a request that contains the query and returns a stream of [[Response]]. // // It is guaranteed that there is at least one ARROW batch returned even if the result set is empty. rpc ExecutePlan(ExecutePlanRequest) returns (stream ExecutePlanResponse) {} // Analyzes a query and returns a [[AnalyzeResponse]] containing metadata about the query. rpc AnalyzePlan(AnalyzePlanRequest) returns (AnalyzePlanResponse) {} // Update or fetch the configurations and returns a [[ConfigResponse]] containing the result. rpc Config(ConfigRequest) returns (ConfigResponse) {} // Add artifacts to the session and returns a [[AddArtifactsResponse]] containing metadata about // the added artifacts. rpc AddArtifacts(stream AddArtifactsRequest) returns (AddArtifactsResponse) {} // Check statuses of artifacts in the session and returns them in a [[ArtifactStatusesResponse]] rpc ArtifactStatus(ArtifactStatusesRequest) returns (ArtifactStatusesResponse) {} // Interrupts running executions rpc Interrupt(InterruptRequest) returns (InterruptResponse) {} // Reattach to an existing reattachable execution. // The ExecutePlan must have been started with ReattachOptions.reattachable=true. // If the ExecutePlanResponse stream ends without a ResultComplete message, there is more to // continue. If there is a ResultComplete, the client should use ReleaseExecute with rpc ReattachExecute(ReattachExecuteRequest) returns (stream ExecutePlanResponse) {} // Release an reattachable execution, or parts thereof. // The ExecutePlan must have been started with ReattachOptions.reattachable=true. // Non reattachable executions are released automatically and immediately after the ExecutePlan // RPC and ReleaseExecute may not be used. rpc ReleaseExecute(ReleaseExecuteRequest) returns (ReleaseExecuteResponse) {} // Release a session. // All the executions in the session will be released. Any further requests for the session with // that session_id for the given user_id will fail. If the session didn't exist or was already // released, this is a noop. rpc ReleaseSession(ReleaseSessionRequest) returns (ReleaseSessionResponse) {} // FetchErrorDetails retrieves the matched exception with details based on a provided error id. rpc FetchErrorDetails(FetchErrorDetailsRequest) returns (FetchErrorDetailsResponse) {} } ================================================ FILE: spark-connect/common/src/main/protobuf/spark/connect/catalog.proto ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ syntax = 'proto3'; package spark.connect; import "spark/connect/common.proto"; import "spark/connect/types.proto"; option java_multiple_files = true; option java_package = "io.delta.connect.spark.proto"; option go_package = "internal/generated"; // Catalog messages are marked as unstable. message Catalog { oneof cat_type { CurrentDatabase current_database = 1; SetCurrentDatabase set_current_database = 2; ListDatabases list_databases = 3; ListTables list_tables = 4; ListFunctions list_functions = 5; ListColumns list_columns = 6; GetDatabase get_database = 7; GetTable get_table = 8; GetFunction get_function = 9; DatabaseExists database_exists = 10; TableExists table_exists = 11; FunctionExists function_exists = 12; CreateExternalTable create_external_table = 13; CreateTable create_table = 14; DropTempView drop_temp_view = 15; DropGlobalTempView drop_global_temp_view = 16; RecoverPartitions recover_partitions = 17; IsCached is_cached = 18; CacheTable cache_table = 19; UncacheTable uncache_table = 20; ClearCache clear_cache = 21; RefreshTable refresh_table = 22; RefreshByPath refresh_by_path = 23; CurrentCatalog current_catalog = 24; SetCurrentCatalog set_current_catalog = 25; ListCatalogs list_catalogs = 26; } } // See `spark.catalog.currentDatabase` message CurrentDatabase { } // See `spark.catalog.setCurrentDatabase` message SetCurrentDatabase { // (Required) string db_name = 1; } // See `spark.catalog.listDatabases` message ListDatabases { // (Optional) The pattern that the database name needs to match optional string pattern = 1; } // See `spark.catalog.listTables` message ListTables { // (Optional) optional string db_name = 1; // (Optional) The pattern that the table name needs to match optional string pattern = 2; } // See `spark.catalog.listFunctions` message ListFunctions { // (Optional) optional string db_name = 1; // (Optional) The pattern that the function name needs to match optional string pattern = 2; } // See `spark.catalog.listColumns` message ListColumns { // (Required) string table_name = 1; // (Optional) optional string db_name = 2; } // See `spark.catalog.getDatabase` message GetDatabase { // (Required) string db_name = 1; } // See `spark.catalog.getTable` message GetTable { // (Required) string table_name = 1; // (Optional) optional string db_name = 2; } // See `spark.catalog.getFunction` message GetFunction { // (Required) string function_name = 1; // (Optional) optional string db_name = 2; } // See `spark.catalog.databaseExists` message DatabaseExists { // (Required) string db_name = 1; } // See `spark.catalog.tableExists` message TableExists { // (Required) string table_name = 1; // (Optional) optional string db_name = 2; } // See `spark.catalog.functionExists` message FunctionExists { // (Required) string function_name = 1; // (Optional) optional string db_name = 2; } // See `spark.catalog.createExternalTable` message CreateExternalTable { // (Required) string table_name = 1; // (Optional) optional string path = 2; // (Optional) optional string source = 3; // (Optional) optional DataType schema = 4; // Options could be empty for valid data source format. // The map key is case insensitive. map options = 5; } // See `spark.catalog.createTable` message CreateTable { // (Required) string table_name = 1; // (Optional) optional string path = 2; // (Optional) optional string source = 3; // (Optional) optional string description = 4; // (Optional) optional DataType schema = 5; // Options could be empty for valid data source format. // The map key is case insensitive. map options = 6; } // See `spark.catalog.dropTempView` message DropTempView { // (Required) string view_name = 1; } // See `spark.catalog.dropGlobalTempView` message DropGlobalTempView { // (Required) string view_name = 1; } // See `spark.catalog.recoverPartitions` message RecoverPartitions { // (Required) string table_name = 1; } // See `spark.catalog.isCached` message IsCached { // (Required) string table_name = 1; } // See `spark.catalog.cacheTable` message CacheTable { // (Required) string table_name = 1; // (Optional) optional StorageLevel storage_level = 2; } // See `spark.catalog.uncacheTable` message UncacheTable { // (Required) string table_name = 1; } // See `spark.catalog.clearCache` message ClearCache { } // See `spark.catalog.refreshTable` message RefreshTable { // (Required) string table_name = 1; } // See `spark.catalog.refreshByPath` message RefreshByPath { // (Required) string path = 1; } // See `spark.catalog.currentCatalog` message CurrentCatalog { } // See `spark.catalog.setCurrentCatalog` message SetCurrentCatalog { // (Required) string catalog_name = 1; } // See `spark.catalog.listCatalogs` message ListCatalogs { // (Optional) The pattern that the catalog name needs to match optional string pattern = 1; } ================================================ FILE: spark-connect/common/src/main/protobuf/spark/connect/commands.proto ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ syntax = 'proto3'; import "google/protobuf/any.proto"; import "spark/connect/common.proto"; import "spark/connect/expressions.proto"; import "spark/connect/relations.proto"; import "spark/connect/ml.proto"; import "spark/connect/pipelines.proto"; package spark.connect; option java_multiple_files = true; option java_package = "io.delta.connect.spark.proto"; option go_package = "internal/generated"; // A [[Command]] is an operation that is executed by the server that does not directly consume or // produce a relational result. message Command { oneof command_type { CommonInlineUserDefinedFunction register_function = 1; WriteOperation write_operation = 2; CreateDataFrameViewCommand create_dataframe_view = 3; WriteOperationV2 write_operation_v2 = 4; SqlCommand sql_command = 5; WriteStreamOperationStart write_stream_operation_start = 6; StreamingQueryCommand streaming_query_command = 7; GetResourcesCommand get_resources_command = 8; StreamingQueryManagerCommand streaming_query_manager_command = 9; CommonInlineUserDefinedTableFunction register_table_function = 10; StreamingQueryListenerBusCommand streaming_query_listener_bus_command = 11; CommonInlineUserDefinedDataSource register_data_source = 12; CreateResourceProfileCommand create_resource_profile_command = 13; CheckpointCommand checkpoint_command = 14; RemoveCachedRemoteRelationCommand remove_cached_remote_relation_command = 15; MergeIntoTableCommand merge_into_table_command = 16; MlCommand ml_command = 17; ExecuteExternalCommand execute_external_command = 18; PipelineCommand pipeline_command = 19; // This field is used to mark extensions to the protocol. When plugins generate arbitrary // Commands they can add them here. During the planning the correct resolution is done. google.protobuf.Any extension = 999; } } // A SQL Command is used to trigger the eager evaluation of SQL commands in Spark. // // When the SQL provide as part of the message is a command it will be immediately evaluated // and the result will be collected and returned as part of a LocalRelation. If the result is // not a command, the operation will simply return a SQL Relation. This allows the client to be // almost oblivious to the server-side behavior. message SqlCommand { // (Required) SQL Query. string sql = 1 [deprecated=true]; // (Optional) A map of parameter names to literal expressions. map args = 2 [deprecated=true]; // (Optional) A sequence of literal expressions for positional parameters in the SQL query text. repeated Expression.Literal pos_args = 3 [deprecated=true]; // (Optional) A map of parameter names to expressions. // It cannot coexist with `pos_arguments`. map named_arguments = 4 [deprecated=true]; // (Optional) A sequence of expressions for positional parameters in the SQL query text. // It cannot coexist with `named_arguments`. repeated Expression pos_arguments = 5 [deprecated=true]; // (Optional) The relation that this SQL command will be built on. Relation input = 6; } // A command that can create DataFrame global temp view or local temp view. message CreateDataFrameViewCommand { // (Required) The relation that this view will be built on. Relation input = 1; // (Required) View name. string name = 2; // (Required) Whether this is global temp view or local temp view. bool is_global = 3; // (Required) // // If true, and if the view already exists, updates it; if false, and if the view // already exists, throws exception. bool replace = 4; } // As writes are not directly handled during analysis and planning, they are modeled as commands. message WriteOperation { // (Required) The output of the `input` relation will be persisted according to the options. Relation input = 1; // (Optional) Format value according to the Spark documentation. Examples are: text, parquet, delta. optional string source = 2; // (Optional) // // The destination of the write operation can be either a path or a table. // If the destination is neither a path nor a table, such as jdbc and noop, // the `save_type` should not be set. oneof save_type { string path = 3; SaveTable table = 4; } // (Required) the save mode. SaveMode mode = 5; // (Optional) List of columns to sort the output by. repeated string sort_column_names = 6; // (Optional) List of columns for partitioning. repeated string partitioning_columns = 7; // (Optional) Bucketing specification. Bucketing must set the number of buckets and the columns // to bucket by. BucketBy bucket_by = 8; // (Optional) A list of configuration options. map options = 9; // (Optional) Columns used for clustering the table. repeated string clustering_columns = 10; message SaveTable { // (Required) The table name. string table_name = 1; // (Required) The method to be called to write to the table. TableSaveMethod save_method = 2; enum TableSaveMethod { TABLE_SAVE_METHOD_UNSPECIFIED = 0; TABLE_SAVE_METHOD_SAVE_AS_TABLE = 1; TABLE_SAVE_METHOD_INSERT_INTO = 2; } } message BucketBy { repeated string bucket_column_names = 1; int32 num_buckets = 2; } enum SaveMode { SAVE_MODE_UNSPECIFIED = 0; SAVE_MODE_APPEND = 1; SAVE_MODE_OVERWRITE = 2; SAVE_MODE_ERROR_IF_EXISTS = 3; SAVE_MODE_IGNORE = 4; } } // As writes are not directly handled during analysis and planning, they are modeled as commands. message WriteOperationV2 { // (Required) The output of the `input` relation will be persisted according to the options. Relation input = 1; // (Required) The destination of the write operation must be either a path or a table. string table_name = 2; // (Optional) A provider for the underlying output data source. Spark's default catalog supports // "parquet", "json", etc. optional string provider = 3; // (Optional) List of columns for partitioning for output table created by `create`, // `createOrReplace`, or `replace` repeated Expression partitioning_columns = 4; // (Optional) A list of configuration options. map options = 5; // (Optional) A list of table properties. map table_properties = 6; // (Required) Write mode. Mode mode = 7; enum Mode { MODE_UNSPECIFIED = 0; MODE_CREATE = 1; MODE_OVERWRITE = 2; MODE_OVERWRITE_PARTITIONS = 3; MODE_APPEND = 4; MODE_REPLACE = 5; MODE_CREATE_OR_REPLACE = 6; } // (Optional) A condition for overwrite saving mode Expression overwrite_condition = 8; // (Optional) Columns used for clustering the table. repeated string clustering_columns = 9; } // Starts write stream operation as streaming query. Query ID and Run ID of the streaming // query are returned. message WriteStreamOperationStart { // (Required) The output of the `input` streaming relation will be written. Relation input = 1; // The following fields directly map to API for DataStreamWriter(). // Consult API documentation unless explicitly documented here. string format = 2; map options = 3; repeated string partitioning_column_names = 4; oneof trigger { string processing_time_interval = 5; bool available_now = 6; bool once = 7; string continuous_checkpoint_interval = 8; } string output_mode = 9; string query_name = 10; // The destination is optional. When set, it can be a path or a table name. oneof sink_destination { string path = 11; string table_name = 12; } StreamingForeachFunction foreach_writer = 13; StreamingForeachFunction foreach_batch = 14; // (Optional) Columns used for clustering the table. repeated string clustering_column_names = 15; } message StreamingForeachFunction { oneof function { PythonUDF python_function = 1; ScalarScalaUDF scala_function = 2; } } message WriteStreamOperationStartResult { // (Required) Query instance. See `StreamingQueryInstanceId`. StreamingQueryInstanceId query_id = 1; // An optional query name. string name = 2; // Optional query started event if there is any listener registered on the client side. optional string query_started_event_json = 3; // TODO: How do we indicate errors? // TODO: Consider adding status, last progress etc here. } // A tuple that uniquely identifies an instance of streaming query run. It consists of `id` that // persists across the streaming runs and `run_id` that changes between each run of the // streaming query that resumes from the checkpoint. message StreamingQueryInstanceId { // (Required) The unique id of this query that persists across restarts from checkpoint data. // That is, this id is generated when a query is started for the first time, and // will be the same every time it is restarted from checkpoint data. string id = 1; // (Required) The unique id of this run of the query. That is, every start/restart of a query // will generate a unique run_id. Therefore, every time a query is restarted from // checkpoint, it will have the same `id` but different `run_id`s. string run_id = 2; } // Commands for a streaming query. message StreamingQueryCommand { // (Required) Query instance. See `StreamingQueryInstanceId`. StreamingQueryInstanceId query_id = 1; // See documentation for the corresponding API method in StreamingQuery. oneof command { // status() API. bool status = 2; // lastProgress() API. bool last_progress = 3; // recentProgress() API. bool recent_progress = 4; // stop() API. Stops the query. bool stop = 5; // processAllAvailable() API. Waits till all the available data is processed bool process_all_available = 6; // explain() API. Returns logical and physical plans. ExplainCommand explain = 7; // exception() API. Returns the exception in the query if any. bool exception = 8; // awaitTermination() API. Waits for the termination of the query. AwaitTerminationCommand await_termination = 9; } message ExplainCommand { // TODO: Consider reusing Explain from AnalyzePlanRequest message. // We can not do this right now since it base.proto imports this file. bool extended = 1; } message AwaitTerminationCommand { optional int64 timeout_ms = 2; } } // Response for commands on a streaming query. message StreamingQueryCommandResult { // (Required) Query instance id. See `StreamingQueryInstanceId`. StreamingQueryInstanceId query_id = 1; oneof result_type { StatusResult status = 2; RecentProgressResult recent_progress = 3; ExplainResult explain = 4; ExceptionResult exception = 5; AwaitTerminationResult await_termination = 6; } message StatusResult { // See documentation for these Scala 'StreamingQueryStatus' struct string status_message = 1; bool is_data_available = 2; bool is_trigger_active = 3; bool is_active = 4; } message RecentProgressResult { // Progress reports as an array of json strings. repeated string recent_progress_json = 5; } message ExplainResult { // Logical and physical plans as string string result = 1; } message ExceptionResult { // (Optional) Exception message as string, maps to the return value of original // StreamingQueryException's toString method optional string exception_message = 1; // (Optional) Exception error class as string optional string error_class = 2; // (Optional) Exception stack trace as string optional string stack_trace = 3; } message AwaitTerminationResult { bool terminated = 1; } } // Commands for the streaming query manager. message StreamingQueryManagerCommand { // See documentation for the corresponding API method in StreamingQueryManager. oneof command { // active() API, returns a list of active queries. bool active = 1; // get() API, returns the StreamingQuery identified by id. string get_query = 2; // awaitAnyTermination() API, wait until any query terminates or timeout. AwaitAnyTerminationCommand await_any_termination = 3; // resetTerminated() API. bool reset_terminated = 4; // addListener API. StreamingQueryListenerCommand add_listener = 5; // removeListener API. StreamingQueryListenerCommand remove_listener = 6; // listListeners() API, returns a list of streaming query listeners. bool list_listeners = 7; } message AwaitAnyTerminationCommand { // (Optional) The waiting time in milliseconds to wait for any query to terminate. optional int64 timeout_ms = 1; } message StreamingQueryListenerCommand { bytes listener_payload = 1; optional PythonUDF python_listener_payload = 2; string id = 3; } } // Response for commands on the streaming query manager. message StreamingQueryManagerCommandResult { oneof result_type { ActiveResult active = 1; StreamingQueryInstance query = 2; AwaitAnyTerminationResult await_any_termination = 3; bool reset_terminated = 4; bool add_listener = 5; bool remove_listener = 6; ListStreamingQueryListenerResult list_listeners = 7; } message ActiveResult { repeated StreamingQueryInstance active_queries = 1; } message StreamingQueryInstance { // (Required) The id and runId of this query. StreamingQueryInstanceId id = 1; // (Optional) The name of this query. optional string name = 2; } message AwaitAnyTerminationResult { bool terminated = 1; } message StreamingQueryListenerInstance { bytes listener_payload = 1; } message ListStreamingQueryListenerResult { // (Required) Reference IDs of listener instances. repeated string listener_ids = 1; } } // The protocol for client-side StreamingQueryListener. // This command will only be set when either the first listener is added to the client, or the last // listener is removed from the client. // The add_listener_bus_listener command will only be set true in the first case. // The remove_listener_bus_listener command will only be set true in the second case. message StreamingQueryListenerBusCommand { oneof command { bool add_listener_bus_listener = 1; bool remove_listener_bus_listener = 2; } } // The enum used for client side streaming query listener event // There is no QueryStartedEvent defined here, // it is added as a field in WriteStreamOperationStartResult enum StreamingQueryEventType { QUERY_PROGRESS_UNSPECIFIED = 0; QUERY_PROGRESS_EVENT = 1; QUERY_TERMINATED_EVENT = 2; QUERY_IDLE_EVENT = 3; } // The protocol for the returned events in the long-running response channel. message StreamingQueryListenerEvent { // (Required) The json serialized event, all StreamingQueryListener events have a json method string event_json = 1; // (Required) Query event type used by client to decide how to deserialize the event_json StreamingQueryEventType event_type = 2; } message StreamingQueryListenerEventsResult { repeated StreamingQueryListenerEvent events = 1; optional bool listener_bus_listener_added = 2; } // Command to get the output of 'SparkContext.resources' message GetResourcesCommand { } // Response for command 'GetResourcesCommand'. message GetResourcesCommandResult { map resources = 1; } // Command to create ResourceProfile message CreateResourceProfileCommand { // (Required) The ResourceProfile to be built on the server-side. ResourceProfile profile = 1; } // Response for command 'CreateResourceProfileCommand'. message CreateResourceProfileCommandResult { // (Required) Server-side generated resource profile id. int32 profile_id = 1; } // Command to remove `CashedRemoteRelation` message RemoveCachedRemoteRelationCommand { // (Required) The remote to be related CachedRemoteRelation relation = 1; } message CheckpointCommand { // (Required) The logical plan to checkpoint. Relation relation = 1; // (Required) Locally checkpoint using a local temporary // directory in Spark Connect server (Spark Driver) bool local = 2; // (Required) Whether to checkpoint this dataframe immediately. bool eager = 3; // (Optional) For local checkpoint, the storage level to use. optional StorageLevel storage_level = 4; } message MergeIntoTableCommand { // (Required) The name of the target table. string target_table_name = 1; // (Required) The relation of the source table. Relation source_table_plan = 2; // (Required) The condition to match the source and target. Expression merge_condition = 3; // (Optional) The actions to be taken when the condition is matched. repeated Expression match_actions = 4; // (Optional) The actions to be taken when the condition is not matched. repeated Expression not_matched_actions = 5; // (Optional) The actions to be taken when the condition is not matched by source. repeated Expression not_matched_by_source_actions = 6; // (Required) Whether to enable schema evolution. bool with_schema_evolution = 7; } // Execute an arbitrary string command inside an external execution engine message ExecuteExternalCommand { // (Required) The class name of the runner that implements `ExternalCommandRunner` string runner = 1; // (Required) The target command to be executed. string command = 2; // (Optional) The options for the runner. map options = 3; } ================================================ FILE: spark-connect/common/src/main/protobuf/spark/connect/common.proto ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ syntax = 'proto3'; package spark.connect; option java_multiple_files = true; option java_package = "io.delta.connect.spark.proto"; option go_package = "internal/generated"; // StorageLevel for persisting Datasets/Tables. message StorageLevel { // (Required) Whether the cache should use disk or not. bool use_disk = 1; // (Required) Whether the cache should use memory or not. bool use_memory = 2; // (Required) Whether the cache should use off-heap or not. bool use_off_heap = 3; // (Required) Whether the cached data is deserialized or not. bool deserialized = 4; // (Required) The number of replicas. int32 replication = 5; } // ResourceInformation to hold information about a type of Resource. // The corresponding class is 'org.apache.spark.resource.ResourceInformation' message ResourceInformation { // (Required) The name of the resource string name = 1; // (Required) An array of strings describing the addresses of the resource. repeated string addresses = 2; } // An executor resource request. message ExecutorResourceRequest { // (Required) resource name. string resource_name = 1; // (Required) resource amount requesting. int64 amount = 2; // Optional script used to discover the resources. optional string discovery_script = 3; // Optional vendor, required for some cluster managers. optional string vendor = 4; } // A task resource request. message TaskResourceRequest { // (Required) resource name. string resource_name = 1; // (Required) resource amount requesting as a double to support fractional // resource requests. double amount = 2; } message ResourceProfile { // (Optional) Resource requests for executors. Mapped from the resource name // (e.g., cores, memory, CPU) to its specific request. map executor_resources = 1; // (Optional) Resource requests for tasks. Mapped from the resource name // (e.g., cores, memory, CPU) to its specific request. map task_resources = 2; } message Origin { // (Required) Indicate the origin type. oneof function { PythonOrigin python_origin = 1; JvmOrigin jvm_origin = 2; } } message PythonOrigin { // (Required) Name of the origin, for example, the name of the function string fragment = 1; // (Required) Callsite to show to end users, for example, stacktrace. string call_site = 2; } message JvmOrigin { // (Optional) Line number in the source file. optional int32 line = 1; // (Optional) Start position in the source file. optional int32 start_position = 2; // (Optional) Start index in the source file. optional int32 start_index = 3; // (Optional) Stop index in the source file. optional int32 stop_index = 4; // (Optional) SQL text. optional string sql_text = 5; // (Optional) Object type. optional string object_type = 6; // (Optional) Object name. optional string object_name = 7; // (Optional) Stack trace. repeated StackTraceElement stack_trace = 8; } // A message to hold a [[java.lang.StackTraceElement]]. message StackTraceElement { // (Optional) Class loader name optional string class_loader_name = 1; // (Optional) Module name optional string module_name = 2; // (Optional) Module version optional string module_version = 3; // (Required) Declaring class string declaring_class = 4; // (Required) Method name string method_name = 5; // (Optional) File name optional string file_name = 6; // (Required) Line number int32 line_number = 7; } message Bools { repeated bool values = 1; } message Ints { repeated int32 values = 1; } message Longs { repeated int64 values = 1; } message Floats { repeated float values = 1; } message Doubles { repeated double values = 1; } message Strings { repeated string values = 1; } ================================================ FILE: spark-connect/common/src/main/protobuf/spark/connect/example_plugins.proto ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ syntax = 'proto3'; import "spark/connect/relations.proto"; import "spark/connect/expressions.proto"; option go_package = "internal/generated"; package spark.connect; option java_multiple_files = true; option java_package = "io.delta.connect.spark.proto"; message ExamplePluginRelation { Relation input = 1; string custom_field = 2; } message ExamplePluginExpression { Expression child = 1; string custom_field = 2; } message ExamplePluginCommand { string custom_field = 1; } ================================================ FILE: spark-connect/common/src/main/protobuf/spark/connect/expressions.proto ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ syntax = 'proto3'; import "google/protobuf/any.proto"; import "spark/connect/types.proto"; import "spark/connect/common.proto"; package spark.connect; option java_multiple_files = true; option java_package = "io.delta.connect.spark.proto"; option go_package = "internal/generated"; // Expression used to refer to fields, functions and similar. This can be used everywhere // expressions in SQL appear. message Expression { ExpressionCommon common = 18; oneof expr_type { Literal literal = 1; UnresolvedAttribute unresolved_attribute = 2; UnresolvedFunction unresolved_function = 3; ExpressionString expression_string = 4; UnresolvedStar unresolved_star = 5; Alias alias = 6; Cast cast = 7; UnresolvedRegex unresolved_regex = 8; SortOrder sort_order = 9; LambdaFunction lambda_function = 10; Window window = 11; UnresolvedExtractValue unresolved_extract_value = 12; UpdateFields update_fields = 13; UnresolvedNamedLambdaVariable unresolved_named_lambda_variable = 14; CommonInlineUserDefinedFunction common_inline_user_defined_function = 15; CallFunction call_function = 16; NamedArgumentExpression named_argument_expression = 17; MergeAction merge_action = 19; TypedAggregateExpression typed_aggregate_expression = 20; SubqueryExpression subquery_expression = 21; // This field is used to mark extensions to the protocol. When plugins generate arbitrary // relations they can add them here. During the planning the correct resolution is done. google.protobuf.Any extension = 999; } // Expression for the OVER clause or WINDOW clause. message Window { // (Required) The window function. Expression window_function = 1; // (Optional) The way that input rows are partitioned. repeated Expression partition_spec = 2; // (Optional) Ordering of rows in a partition. repeated SortOrder order_spec = 3; // (Optional) Window frame in a partition. // // If not set, it will be treated as 'UnspecifiedFrame'. WindowFrame frame_spec = 4; // The window frame message WindowFrame { // (Required) The type of the frame. FrameType frame_type = 1; // (Required) The lower bound of the frame. FrameBoundary lower = 2; // (Required) The upper bound of the frame. FrameBoundary upper = 3; enum FrameType { FRAME_TYPE_UNDEFINED = 0; // RowFrame treats rows in a partition individually. FRAME_TYPE_ROW = 1; // RangeFrame treats rows in a partition as groups of peers. // All rows having the same 'ORDER BY' ordering are considered as peers. FRAME_TYPE_RANGE = 2; } message FrameBoundary { oneof boundary { // CURRENT ROW boundary bool current_row = 1; // UNBOUNDED boundary. // For lower bound, it will be converted to 'UnboundedPreceding'. // for upper bound, it will be converted to 'UnboundedFollowing'. bool unbounded = 2; // This is an expression for future proofing. We are expecting literals on the server side. Expression value = 3; } } } } // SortOrder is used to specify the data ordering, it is normally used in Sort and Window. // It is an unevaluable expression and cannot be evaluated, so can not be used in Projection. message SortOrder { // (Required) The expression to be sorted. Expression child = 1; // (Required) The sort direction, should be ASCENDING or DESCENDING. SortDirection direction = 2; // (Required) How to deal with NULLs, should be NULLS_FIRST or NULLS_LAST. NullOrdering null_ordering = 3; enum SortDirection { SORT_DIRECTION_UNSPECIFIED = 0; SORT_DIRECTION_ASCENDING = 1; SORT_DIRECTION_DESCENDING = 2; } enum NullOrdering { SORT_NULLS_UNSPECIFIED = 0; SORT_NULLS_FIRST = 1; SORT_NULLS_LAST = 2; } } message Cast { // (Required) the expression to be casted. Expression expr = 1; // (Required) the data type that the expr to be casted to. oneof cast_to_type { DataType type = 2; // If this is set, Server will use Catalyst parser to parse this string to DataType. string type_str = 3; } // (Optional) The expression evaluation mode. EvalMode eval_mode = 4; enum EvalMode { EVAL_MODE_UNSPECIFIED = 0; EVAL_MODE_LEGACY = 1; EVAL_MODE_ANSI = 2; EVAL_MODE_TRY = 3; } } message Literal { oneof literal_type { DataType null = 1; bytes binary = 2; bool boolean = 3; int32 byte = 4; int32 short = 5; int32 integer = 6; int64 long = 7; float float = 10; double double = 11; Decimal decimal = 12; string string = 13; // Date in units of days since the UNIX epoch. int32 date = 16; // Timestamp in units of microseconds since the UNIX epoch. int64 timestamp = 17; // Timestamp in units of microseconds since the UNIX epoch (without timezone information). int64 timestamp_ntz = 18; CalendarInterval calendar_interval = 19; int32 year_month_interval = 20; int64 day_time_interval = 21; Array array = 22; Map map = 23; Struct struct = 24; SpecializedArray specialized_array = 25; } message Decimal { // the string representation. string value = 1; // The maximum number of digits allowed in the value. // the maximum precision is 38. optional int32 precision = 2; // declared scale of decimal literal optional int32 scale = 3; } message CalendarInterval { int32 months = 1; int32 days = 2; int64 microseconds = 3; } message Array { DataType element_type = 1; repeated Literal elements = 2; } message Map { DataType key_type = 1; DataType value_type = 2; repeated Literal keys = 3; repeated Literal values = 4; } message Struct { DataType struct_type = 1; repeated Literal elements = 2; } message SpecializedArray { oneof value_type { Bools bools = 1; Ints ints = 2; Longs longs = 3; Floats floats = 4; Doubles doubles = 5; Strings strings = 6; } } } // An unresolved attribute that is not explicitly bound to a specific column, but the column // is resolved during analysis by name. message UnresolvedAttribute { // (Required) An identifier that will be parsed by Catalyst parser. This should follow the // Spark SQL identifier syntax. string unparsed_identifier = 1; // (Optional) The id of corresponding connect plan. optional int64 plan_id = 2; // (Optional) The requested column is a metadata column. optional bool is_metadata_column = 3; } // An unresolved function is not explicitly bound to one explicit function, but the function // is resolved during analysis following Sparks name resolution rules. message UnresolvedFunction { // (Required) name (or unparsed name for user defined function) for the unresolved function. string function_name = 1; // (Optional) Function arguments. Empty arguments are allowed. repeated Expression arguments = 2; // (Required) Indicate if this function should be applied on distinct values. bool is_distinct = 3; // (Required) Indicate if this is a user defined function. // // When it is not a user defined function, Connect will use the function name directly. // When it is a user defined function, Connect will parse the function name first. bool is_user_defined_function = 4; // (Optional) Indicate if this function is defined in the internal function registry. // If not set, the server will try to look up the function in the internal function registry // and decide appropriately. optional bool is_internal = 5; } // Expression as string. message ExpressionString { // (Required) A SQL expression that will be parsed by Catalyst parser. string expression = 1; } // UnresolvedStar is used to expand all the fields of a relation or struct. message UnresolvedStar { // (Optional) The target of the expansion. // // If set, it should end with '.*' and will be parsed by 'parseAttributeName' // in the server side. optional string unparsed_target = 1; // (Optional) The id of corresponding connect plan. optional int64 plan_id = 2; } // Represents all of the input attributes to a given relational operator, for example in // "SELECT `(id)?+.+` FROM ...". message UnresolvedRegex { // (Required) The column name used to extract column with regex. string col_name = 1; // (Optional) The id of corresponding connect plan. optional int64 plan_id = 2; } // Extracts a value or values from an Expression message UnresolvedExtractValue { // (Required) The expression to extract value from, can be // Map, Array, Struct or array of Structs. Expression child = 1; // (Required) The expression to describe the extraction, can be // key of Map, index of Array, field name of Struct. Expression extraction = 2; } // Add, replace or drop a field of `StructType` expression by name. message UpdateFields { // (Required) The struct expression. Expression struct_expression = 1; // (Required) The field name. string field_name = 2; // (Optional) The expression to add or replace. // // When not set, it means this field will be dropped. Expression value_expression = 3; } message Alias { // (Required) The expression that alias will be added on. Expression expr = 1; // (Required) a list of name parts for the alias. // // Scalar columns only has one name that presents. repeated string name = 2; // (Optional) Alias metadata expressed as a JSON map. optional string metadata = 3; } message LambdaFunction { // (Required) The lambda function. // // The function body should use 'UnresolvedAttribute' as arguments, the sever side will // replace 'UnresolvedAttribute' with 'UnresolvedNamedLambdaVariable'. Expression function = 1; // (Required) Function variables. Must contains 1 ~ 3 variables. repeated Expression.UnresolvedNamedLambdaVariable arguments = 2; } message UnresolvedNamedLambdaVariable { // (Required) a list of name parts for the variable. Must not be empty. repeated string name_parts = 1; } } message ExpressionCommon { // (Required) Keep the information of the origin for this expression such as stacktrace. Origin origin = 1; } message CommonInlineUserDefinedFunction { // (Required) Name of the user-defined function. string function_name = 1; // (Optional) Indicate if the user-defined function is deterministic. bool deterministic = 2; // (Optional) Function arguments. Empty arguments are allowed. repeated Expression arguments = 3; // (Required) Indicate the function type of the user-defined function. oneof function { PythonUDF python_udf = 4; ScalarScalaUDF scalar_scala_udf = 5; JavaUDF java_udf = 6; } // (Required) Indicate if this function should be applied on distinct values. bool is_distinct = 7; } message PythonUDF { // (Required) Output type of the Python UDF DataType output_type = 1; // (Required) EvalType of the Python UDF int32 eval_type = 2; // (Required) The encoded commands of the Python UDF bytes command = 3; // (Required) Python version being used in the client. string python_ver = 4; // (Optional) Additional includes for the Python UDF. repeated string additional_includes = 5; } message ScalarScalaUDF { // (Required) Serialized JVM object containing UDF definition, input encoders and output encoder bytes payload = 1; // (Optional) Input type(s) of the UDF repeated DataType inputTypes = 2; // (Required) Output type of the UDF DataType outputType = 3; // (Required) True if the UDF can return null value bool nullable = 4; // (Required) Indicate if the UDF is an aggregate function bool aggregate = 5; } message JavaUDF { // (Required) Fully qualified name of Java class string class_name = 1; // (Optional) Output type of the Java UDF optional DataType output_type = 2; // (Required) Indicate if the Java user-defined function is an aggregate function bool aggregate = 3; } message TypedAggregateExpression { // (Required) The aggregate function object packed into bytes. ScalarScalaUDF scalar_scala_udf = 1; } message CallFunction { // (Required) Unparsed name of the SQL function. string function_name = 1; // (Optional) Function arguments. Empty arguments are allowed. repeated Expression arguments = 2; } message NamedArgumentExpression { // (Required) The key of the named argument. string key = 1; // (Required) The value expression of the named argument. Expression value = 2; } message MergeAction { // (Required) The action type of the merge action. ActionType action_type = 1; // (Optional) The condition expression of the merge action. optional Expression condition = 2; // (Optional) The assignments of the merge action. Required for ActionTypes INSERT and UPDATE. repeated Assignment assignments = 3; enum ActionType { ACTION_TYPE_INVALID = 0; ACTION_TYPE_DELETE = 1; ACTION_TYPE_INSERT = 2; ACTION_TYPE_INSERT_STAR = 3; ACTION_TYPE_UPDATE = 4; ACTION_TYPE_UPDATE_STAR = 5; } message Assignment { // (Required) The key of the assignment. Expression key = 1; // (Required) The value of the assignment. Expression value = 2; } } message SubqueryExpression { // (Required) The ID of the corresponding connect plan. int64 plan_id = 1; // (Required) The type of the subquery. SubqueryType subquery_type = 2; // (Optional) Options specific to table arguments. optional TableArgOptions table_arg_options = 3; // (Optional) IN subquery values. repeated Expression in_subquery_values = 4; enum SubqueryType { SUBQUERY_TYPE_UNKNOWN = 0; SUBQUERY_TYPE_SCALAR = 1; SUBQUERY_TYPE_EXISTS = 2; SUBQUERY_TYPE_TABLE_ARG = 3; SUBQUERY_TYPE_IN = 4; } // Nested message for table argument options. message TableArgOptions { // (Optional) The way that input rows are partitioned. repeated Expression partition_spec = 1; // (Optional) Ordering of rows in a partition. repeated Expression.SortOrder order_spec = 2; // (Optional) Whether this is a single partition. optional bool with_single_partition = 3; } } ================================================ FILE: spark-connect/common/src/main/protobuf/spark/connect/ml.proto ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ syntax = 'proto3'; package spark.connect; import "spark/connect/relations.proto"; import "spark/connect/expressions.proto"; import "spark/connect/ml_common.proto"; option java_multiple_files = true; option java_package = "io.delta.connect.spark.proto"; option go_package = "internal/generated"; // Command for ML message MlCommand { oneof command { Fit fit = 1; Fetch fetch = 2; Delete delete = 3; Write write = 4; Read read = 5; Evaluate evaluate = 6; CleanCache clean_cache = 7; GetCacheInfo get_cache_info = 8; } // Command for estimator.fit(dataset) message Fit { // (Required) Estimator information (its type should be OPERATOR_TYPE_ESTIMATOR) MlOperator estimator = 1; // (Optional) parameters of the Estimator optional MlParams params = 2; // (Required) the training dataset Relation dataset = 3; } // Command to delete the cached objects which could be a model // or summary evaluated by a model message Delete { repeated ObjectRef obj_refs = 1; } // Force to clean up all the ML cached objects message CleanCache { } // Get the information of all the ML cached objects message GetCacheInfo { } // Command to write ML operator message Write { // It could be an estimator/evaluator or the cached model oneof type { // Estimator or evaluator MlOperator operator = 1; // The cached model ObjectRef obj_ref = 2; } // (Optional) The parameters of operator which could be estimator/evaluator or a cached model optional MlParams params = 3; // (Required) Save the ML instance to the path string path = 4; // (Optional) Overwrites if the output path already exists. optional bool should_overwrite = 5; // (Optional) The options of the writer map options = 6; } // Command to load ML operator. message Read { // (Required) ML operator information MlOperator operator = 1; // (Required) Load the ML instance from the input path string path = 2; } // Command for evaluator.evaluate(dataset) message Evaluate { // (Required) Evaluator information (its type should be OPERATOR_TYPE_EVALUATOR) MlOperator evaluator = 1; // (Optional) parameters of the Evaluator optional MlParams params = 2; // (Required) the evaluating dataset Relation dataset = 3; } } // The result of MlCommand message MlCommandResult { oneof result_type { // The result of the attribute Expression.Literal param = 1; // Evaluate a Dataset in a model and return the cached ID of summary string summary = 2; // Operator information MlOperatorInfo operator_info = 3; } // Represents an operator info message MlOperatorInfo { oneof type { // The cached object which could be a model or summary evaluated by a model ObjectRef obj_ref = 1; // Operator name string name = 2; } // (Optional) the 'uid' of a ML object // Note it is different from the 'id' of a cached object. optional string uid = 3; // (Optional) parameters optional MlParams params = 4; // (Optional) warning message generated during the ML command execution optional string warning_message = 5; } } ================================================ FILE: spark-connect/common/src/main/protobuf/spark/connect/ml_common.proto ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ syntax = 'proto3'; package spark.connect; import "spark/connect/expressions.proto"; option java_multiple_files = true; option java_package = "io.delta.connect.spark.proto"; option go_package = "internal/generated"; // MlParams stores param settings for ML Estimator / Transformer / Evaluator message MlParams { // User-supplied params map params = 1; } // MLOperator represents the ML operators like (Estimator, Transformer or Evaluator) message MlOperator { // (Required) The qualified name of the ML operator. string name = 1; // (Required) Unique id of the ML operator string uid = 2; // (Required) Represents what the ML operator is OperatorType type = 3; enum OperatorType { OPERATOR_TYPE_UNSPECIFIED = 0; // ML estimator OPERATOR_TYPE_ESTIMATOR = 1; // ML transformer (non-model) OPERATOR_TYPE_TRANSFORMER = 2; // ML evaluator OPERATOR_TYPE_EVALUATOR = 3; // ML model OPERATOR_TYPE_MODEL = 4; } } // Represents a reference to the cached object which could be a model // or summary evaluated by a model message ObjectRef { // (Required) The ID is used to lookup the object on the server side. // Note it is different from the 'uid' of a ML object. string id = 1; } ================================================ FILE: spark-connect/common/src/main/protobuf/spark/connect/pipelines.proto ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ syntax = 'proto3'; package spark.connect; import "spark/connect/relations.proto"; import "spark/connect/types.proto"; option java_multiple_files = true; option java_package = "io.delta.connect.spark.proto"; // Dispatch object for pipelines commands. See each individual command for documentation. message PipelineCommand { oneof command_type { CreateDataflowGraph create_dataflow_graph = 1; DefineDataset define_dataset = 2; DefineFlow define_flow = 3; DropDataflowGraph drop_dataflow_graph = 4; StartRun start_run = 5; DefineSqlGraphElements define_sql_graph_elements = 6; } // Request to create a new dataflow graph. message CreateDataflowGraph { // The default catalog. optional string default_catalog = 1; // The default database. optional string default_database = 2; // SQL configurations for all flows in this graph. map sql_conf = 5; message Response { // The ID of the created graph. optional string dataflow_graph_id = 1; } } // Drops the graph and stops any running attached flows. message DropDataflowGraph { // The graph to drop. optional string dataflow_graph_id = 1; } // Request to define a dataset: a table, a materialized view, or a temporary view. message DefineDataset { // The graph to attach this dataset to. optional string dataflow_graph_id = 1; // Name of the dataset. Can be partially or fully qualified. optional string dataset_name = 2; // The type of the dataset. optional DatasetType dataset_type = 3; // Optional comment for the dataset. optional string comment = 4; // Optional table properties. Only applies to dataset_type == TABLE and dataset_type == MATERIALIZED_VIEW. map table_properties = 5; // Optional partition columns for the dataset. Only applies to dataset_type == TABLE and // dataset_type == MATERIALIZED_VIEW. repeated string partition_cols = 6; // Schema for the dataset. If unset, this will be inferred from incoming flows. optional spark.connect.DataType schema = 7; // The output table format of the dataset. Only applies to dataset_type == TABLE and // dataset_type == MATERIALIZED_VIEW. optional string format = 8; } // Request to define a flow targeting a dataset. message DefineFlow { // The graph to attach this flow to. optional string dataflow_graph_id = 1; // Name of the flow. For standalone flows, this must be a single-part name. optional string flow_name = 2; // Name of the dataset this flow writes to. Can be partially or fully qualified. optional string target_dataset_name = 3; // An unresolved relation that defines the dataset's flow. optional spark.connect.Relation plan = 4; // SQL configurations set when running this flow. map sql_conf = 5; // If true, this flow will only be run once per full refresh. optional bool once = 6; } // Resolves all datasets and flows and start a pipeline update. Should be called after all // graph elements are registered. message StartRun { // The graph to start. optional string dataflow_graph_id = 1; } } // Parses the SQL file and registers all datasets and flows. message DefineSqlGraphElements { // The graph to attach this dataset to. optional string dataflow_graph_id = 1; // The full path to the SQL file. Can be relative or absolute. optional string sql_file_path = 2; // The contents of the SQL file. optional string sql_text = 3; } // Dispatch object for pipelines command results. message PipelineCommandResult { oneof result_type { CreateDataflowGraphResult create_dataflow_graph_result = 1; } message CreateDataflowGraphResult { // The ID of the created graph. optional string dataflow_graph_id = 1; } } // The type of dataset. enum DatasetType { // Safe default value. Should not be used. DATASET_TYPE_UNSPECIFIED = 0; // A materialized view dataset which is published to the catalog MATERIALIZED_VIEW = 1; // A table which is published to the catalog TABLE = 2; // A view which is not published to the catalog TEMPORARY_VIEW = 3; } // A response containing an event emitted during the run of a pipeline. message PipelineEventResult { PipelineEvent event = 1; } // An event emitted during the run of a graph. message PipelineEvent { // The time of the event. optional string timestamp = 1; // The message that should be displayed to users. optional string message = 2; } ================================================ FILE: spark-connect/common/src/main/protobuf/spark/connect/relations.proto ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ syntax = 'proto3'; package spark.connect; import "google/protobuf/any.proto"; import "spark/connect/expressions.proto"; import "spark/connect/types.proto"; import "spark/connect/catalog.proto"; import "spark/connect/common.proto"; import "spark/connect/ml_common.proto"; option java_multiple_files = true; option java_package = "io.delta.connect.spark.proto"; option go_package = "internal/generated"; // The main [[Relation]] type. Fundamentally, a relation is a typed container // that has exactly one explicit relation type set. // // When adding new relation types, they have to be registered here. message Relation { RelationCommon common = 1; oneof rel_type { Read read = 2; Project project = 3; Filter filter = 4; Join join = 5; SetOperation set_op = 6; Sort sort = 7; Limit limit = 8; Aggregate aggregate = 9; SQL sql = 10; LocalRelation local_relation = 11; Sample sample = 12; Offset offset = 13; Deduplicate deduplicate = 14; Range range = 15; SubqueryAlias subquery_alias = 16; Repartition repartition = 17; ToDF to_df = 18; WithColumnsRenamed with_columns_renamed = 19; ShowString show_string = 20; Drop drop = 21; Tail tail = 22; WithColumns with_columns = 23; Hint hint = 24; Unpivot unpivot = 25; ToSchema to_schema = 26; RepartitionByExpression repartition_by_expression = 27; MapPartitions map_partitions = 28; CollectMetrics collect_metrics = 29; Parse parse = 30; GroupMap group_map = 31; CoGroupMap co_group_map = 32; WithWatermark with_watermark = 33; ApplyInPandasWithState apply_in_pandas_with_state = 34; HtmlString html_string = 35; CachedLocalRelation cached_local_relation = 36; CachedRemoteRelation cached_remote_relation = 37; CommonInlineUserDefinedTableFunction common_inline_user_defined_table_function = 38; AsOfJoin as_of_join = 39; CommonInlineUserDefinedDataSource common_inline_user_defined_data_source = 40; WithRelations with_relations = 41; Transpose transpose = 42; UnresolvedTableValuedFunction unresolved_table_valued_function = 43; LateralJoin lateral_join = 44; // NA functions NAFill fill_na = 90; NADrop drop_na = 91; NAReplace replace = 92; // stat functions StatSummary summary = 100; StatCrosstab crosstab = 101; StatDescribe describe = 102; StatCov cov = 103; StatCorr corr = 104; StatApproxQuantile approx_quantile = 105; StatFreqItems freq_items = 106; StatSampleBy sample_by = 107; // Catalog API (experimental / unstable) Catalog catalog = 200; // ML relation MlRelation ml_relation = 300; // This field is used to mark extensions to the protocol. When plugins generate arbitrary // relations they can add them here. During the planning the correct resolution is done. google.protobuf.Any extension = 998; Unknown unknown = 999; } } // Relation to represent ML world message MlRelation { oneof ml_type { Transform transform = 1; Fetch fetch = 2; } // Relation to represent transform(input) of the operator // which could be a cached model or a new transformer message Transform { oneof operator { // Object reference ObjectRef obj_ref = 1; // Could be an ML transformer like VectorAssembler MlOperator transformer = 2; } // the input dataframe Relation input = 3; // the operator specific parameters MlParams params = 4; } } // Message for fetching attribute from object on the server side. // Fetch can be represented as a Relation or a ML command // Command: model.coefficients, model.summary.weightedPrecision which // returns the final literal result // Relation: model.summary.roc which returns a DataFrame (Relation) message Fetch { // (Required) reference to the object on the server side ObjectRef obj_ref = 1; // (Required) the calling method chains repeated Method methods = 2; // Represents a method with inclusion of method name and its arguments message Method { // (Required) the method name string method = 1; // (Optional) the arguments of the method repeated Args args = 2; message Args { oneof args_type { Expression.Literal param = 1; Relation input = 2; } } } } // Used for testing purposes only. message Unknown {} // Common metadata of all relations. message RelationCommon { // (Required) Shared relation metadata. string source_info = 1 [deprecated=true]; // (Optional) A per-client globally unique id for a given connect plan. optional int64 plan_id = 2; // (Optional) Keep the information of the origin for this expression such as stacktrace. Origin origin = 3; } // Relation that uses a SQL query to generate the output. message SQL { // (Required) The SQL query. string query = 1; // (Optional) A map of parameter names to literal expressions. map args = 2 [deprecated=true]; // (Optional) A sequence of literal expressions for positional parameters in the SQL query text. repeated Expression.Literal pos_args = 3 [deprecated=true]; // (Optional) A map of parameter names to expressions. // It cannot coexist with `pos_arguments`. map named_arguments = 4; // (Optional) A sequence of expressions for positional parameters in the SQL query text. // It cannot coexist with `named_arguments`. repeated Expression pos_arguments = 5; } // Relation of type [[WithRelations]]. // // This relation contains a root plan, and one or more references that are used by the root plan. // There are two ways of referencing a relation, by name (through a subquery alias), or by plan_id // (using RelationCommon.plan_id). // // This relation can be used to implement CTEs, describe DAGs, or to reduce tree depth. message WithRelations { // (Required) Plan at the root of the query tree. This plan is expected to contain one or more // references. Those references get expanded later on by the engine. Relation root = 1; // (Required) Plans referenced by the root plan. Relations in this list are also allowed to // contain references to other relations in this list, as long they do not form cycles. repeated Relation references = 2; } // Relation that reads from a file / table or other data source. Does not have additional // inputs. message Read { oneof read_type { NamedTable named_table = 1; DataSource data_source = 2; } // (Optional) Indicates if this is a streaming read. bool is_streaming = 3; message NamedTable { // (Required) Unparsed identifier for the table. string unparsed_identifier = 1; // Options for the named table. The map key is case insensitive. map options = 2; } message DataSource { // (Optional) Supported formats include: parquet, orc, text, json, parquet, csv, avro. // // If not set, the value from SQL conf 'spark.sql.sources.default' will be used. optional string format = 1; // (Optional) If not set, Spark will infer the schema. // // This schema string should be either DDL-formatted or JSON-formatted. optional string schema = 2; // Options for the data source. The context of this map varies based on the // data source format. This options could be empty for valid data source format. // The map key is case insensitive. map options = 3; // (Optional) A list of path for file-system backed data sources. repeated string paths = 4; // (Optional) Condition in the where clause for each partition. // // This is only supported by the JDBC data source. repeated string predicates = 5; } } // Projection of a bag of expressions for a given input relation. // // The input relation must be specified. // The projected expression can be an arbitrary expression. message Project { // (Optional) Input relation is optional for Project. // // For example, `SELECT ABS(-1)` is valid plan without an input plan. Relation input = 1; // (Required) A Project requires at least one expression. repeated Expression expressions = 3; } // Relation that applies a boolean expression `condition` on each row of `input` to produce // the output result. message Filter { // (Required) Input relation for a Filter. Relation input = 1; // (Required) A Filter must have a condition expression. Expression condition = 2; } // Relation of type [[Join]]. // // `left` and `right` must be present. message Join { // (Required) Left input relation for a Join. Relation left = 1; // (Required) Right input relation for a Join. Relation right = 2; // (Optional) The join condition. Could be unset when `using_columns` is utilized. // // This field does not co-exist with using_columns. Expression join_condition = 3; // (Required) The join type. JoinType join_type = 4; // Optional. using_columns provides a list of columns that should present on both sides of // the join inputs that this Join will join on. For example A JOIN B USING col_name is // equivalent to A JOIN B on A.col_name = B.col_name. // // This field does not co-exist with join_condition. repeated string using_columns = 5; enum JoinType { JOIN_TYPE_UNSPECIFIED = 0; JOIN_TYPE_INNER = 1; JOIN_TYPE_FULL_OUTER = 2; JOIN_TYPE_LEFT_OUTER = 3; JOIN_TYPE_RIGHT_OUTER = 4; JOIN_TYPE_LEFT_ANTI = 5; JOIN_TYPE_LEFT_SEMI = 6; JOIN_TYPE_CROSS = 7; } // (Optional) Only used by joinWith. Set the left and right join data types. optional JoinDataType join_data_type = 6; message JoinDataType { // If the left data type is a struct. bool is_left_struct = 1; // If the right data type is a struct. bool is_right_struct = 2; } } // Relation of type [[SetOperation]] message SetOperation { // (Required) Left input relation for a Set operation. Relation left_input = 1; // (Required) Right input relation for a Set operation. Relation right_input = 2; // (Required) The Set operation type. SetOpType set_op_type = 3; // (Optional) If to remove duplicate rows. // // True to preserve all results. // False to remove duplicate rows. optional bool is_all = 4; // (Optional) If to perform the Set operation based on name resolution. // // Only UNION supports this option. optional bool by_name = 5; // (Optional) If to perform the Set operation and allow missing columns. // // Only UNION supports this option. optional bool allow_missing_columns = 6; enum SetOpType { SET_OP_TYPE_UNSPECIFIED = 0; SET_OP_TYPE_INTERSECT = 1; SET_OP_TYPE_UNION = 2; SET_OP_TYPE_EXCEPT = 3; } } // Relation of type [[Limit]] that is used to `limit` rows from the input relation. message Limit { // (Required) Input relation for a Limit. Relation input = 1; // (Required) the limit. int32 limit = 2; } // Relation of type [[Offset]] that is used to read rows staring from the `offset` on // the input relation. message Offset { // (Required) Input relation for an Offset. Relation input = 1; // (Required) the limit. int32 offset = 2; } // Relation of type [[Tail]] that is used to fetch `limit` rows from the last of the input relation. message Tail { // (Required) Input relation for an Tail. Relation input = 1; // (Required) the limit. int32 limit = 2; } // Relation of type [[Aggregate]]. message Aggregate { // (Required) Input relation for a RelationalGroupedDataset. Relation input = 1; // (Required) How the RelationalGroupedDataset was built. GroupType group_type = 2; // (Required) Expressions for grouping keys repeated Expression grouping_expressions = 3; // (Required) List of values that will be translated to columns in the output DataFrame. repeated Expression aggregate_expressions = 4; // (Optional) Pivots a column of the current `DataFrame` and performs the specified aggregation. Pivot pivot = 5; // (Optional) List of values that will be translated to columns in the output DataFrame. repeated GroupingSets grouping_sets = 6; enum GroupType { GROUP_TYPE_UNSPECIFIED = 0; GROUP_TYPE_GROUPBY = 1; GROUP_TYPE_ROLLUP = 2; GROUP_TYPE_CUBE = 3; GROUP_TYPE_PIVOT = 4; GROUP_TYPE_GROUPING_SETS = 5; } message Pivot { // (Required) The column to pivot Expression col = 1; // (Optional) List of values that will be translated to columns in the output DataFrame. // // Note that if it is empty, the server side will immediately trigger a job to collect // the distinct values of the column. repeated Expression.Literal values = 2; } message GroupingSets { // (Required) Individual grouping set repeated Expression grouping_set = 1; } } // Relation of type [[Sort]]. message Sort { // (Required) Input relation for a Sort. Relation input = 1; // (Required) The ordering expressions repeated Expression.SortOrder order = 2; // (Optional) if this is a global sort. optional bool is_global = 3; } // Drop specified columns. message Drop { // (Required) The input relation. Relation input = 1; // (Optional) columns to drop. repeated Expression columns = 2; // (Optional) names of columns to drop. repeated string column_names = 3; } // Relation of type [[Deduplicate]] which have duplicate rows removed, could consider either only // the subset of columns or all the columns. message Deduplicate { // (Required) Input relation for a Deduplicate. Relation input = 1; // (Optional) Deduplicate based on a list of column names. // // This field does not co-use with `all_columns_as_keys`. repeated string column_names = 2; // (Optional) Deduplicate based on all the columns of the input relation. // // This field does not co-use with `column_names`. optional bool all_columns_as_keys = 3; // (Optional) Deduplicate within the time range of watermark. optional bool within_watermark = 4; } // A relation that does not need to be qualified by name. message LocalRelation { // (Optional) Local collection data serialized into Arrow IPC streaming format which contains // the schema of the data. optional bytes data = 1; // (Optional) The schema of local data. // It should be either a DDL-formatted type string or a JSON string. // // The server side will update the column names and data types according to this schema. // If the 'data' is not provided, then this schema will be required. optional string schema = 2; } // A local relation that has been cached already. message CachedLocalRelation { // `userId` and `sessionId` fields are deleted since the server must always use the active // session/user rather than arbitrary values provided by the client. It is never valid to access // a local relation from a different session/user. reserved 1, 2; reserved "userId", "sessionId"; // (Required) A sha-256 hash of the serialized local relation in proto, see LocalRelation. string hash = 3; } // Represents a remote relation that has been cached on server. message CachedRemoteRelation { // (Required) ID of the remote related (assigned by the service). string relation_id = 1; } // Relation of type [[Sample]] that samples a fraction of the dataset. message Sample { // (Required) Input relation for a Sample. Relation input = 1; // (Required) lower bound. double lower_bound = 2; // (Required) upper bound. double upper_bound = 3; // (Optional) Whether to sample with replacement. optional bool with_replacement = 4; // (Required) The random seed. // This field is required to avoid generating mutable dataframes (see SPARK-48184 for details), // however, still keep it 'optional' here for backward compatibility. optional int64 seed = 5; // (Required) Explicitly sort the underlying plan to make the ordering deterministic or cache it. // This flag is true when invoking `dataframe.randomSplit` to randomly splits DataFrame with the // provided weights. Otherwise, it is false. bool deterministic_order = 6; } // Relation of type [[Range]] that generates a sequence of integers. message Range { // (Optional) Default value = 0 optional int64 start = 1; // (Required) int64 end = 2; // (Required) int64 step = 3; // Optional. Default value is assigned by 1) SQL conf "spark.sql.leafNodeDefaultParallelism" if // it is set, or 2) spark default parallelism. optional int32 num_partitions = 4; } // Relation alias. message SubqueryAlias { // (Required) The input relation of SubqueryAlias. Relation input = 1; // (Required) The alias. string alias = 2; // (Optional) Qualifier of the alias. repeated string qualifier = 3; } // Relation repartition. message Repartition { // (Required) The input relation of Repartition. Relation input = 1; // (Required) Must be positive. int32 num_partitions = 2; // (Optional) Default value is false. optional bool shuffle = 3; } // Compose the string representing rows for output. // It will invoke 'Dataset.showString' to compute the results. message ShowString { // (Required) The input relation. Relation input = 1; // (Required) Number of rows to show. int32 num_rows = 2; // (Required) If set to more than 0, truncates strings to // `truncate` characters and all cells will be aligned right. int32 truncate = 3; // (Required) If set to true, prints output rows vertically (one line per column value). bool vertical = 4; } // Compose the string representing rows for output. // It will invoke 'Dataset.htmlString' to compute the results. message HtmlString { // (Required) The input relation. Relation input = 1; // (Required) Number of rows to show. int32 num_rows = 2; // (Required) If set to more than 0, truncates strings to // `truncate` characters and all cells will be aligned right. int32 truncate = 3; } // Computes specified statistics for numeric and string columns. // It will invoke 'Dataset.summary' (same as 'StatFunctions.summary') // to compute the results. message StatSummary { // (Required) The input relation. Relation input = 1; // (Optional) Statistics from to be computed. // // Available statistics are: // count // mean // stddev // min // max // arbitrary approximate percentiles specified as a percentage (e.g. 75%) // count_distinct // approx_count_distinct // // If no statistics are given, this function computes 'count', 'mean', 'stddev', 'min', // 'approximate quartiles' (percentiles at 25%, 50%, and 75%), and 'max'. repeated string statistics = 2; } // Computes basic statistics for numeric and string columns, including count, mean, stddev, min, // and max. If no columns are given, this function computes statistics for all numerical or // string columns. message StatDescribe { // (Required) The input relation. Relation input = 1; // (Optional) Columns to compute statistics on. repeated string cols = 2; } // Computes a pair-wise frequency table of the given columns. Also known as a contingency table. // It will invoke 'Dataset.stat.crosstab' (same as 'StatFunctions.crossTabulate') // to compute the results. message StatCrosstab { // (Required) The input relation. Relation input = 1; // (Required) The name of the first column. // // Distinct items will make the first item of each row. string col1 = 2; // (Required) The name of the second column. // // Distinct items will make the column names of the DataFrame. string col2 = 3; } // Calculate the sample covariance of two numerical columns of a DataFrame. // It will invoke 'Dataset.stat.cov' (same as 'StatFunctions.calculateCov') to compute the results. message StatCov { // (Required) The input relation. Relation input = 1; // (Required) The name of the first column. string col1 = 2; // (Required) The name of the second column. string col2 = 3; } // Calculates the correlation of two columns of a DataFrame. Currently only supports the Pearson // Correlation Coefficient. It will invoke 'Dataset.stat.corr' (same as // 'StatFunctions.pearsonCorrelation') to compute the results. message StatCorr { // (Required) The input relation. Relation input = 1; // (Required) The name of the first column. string col1 = 2; // (Required) The name of the second column. string col2 = 3; // (Optional) Default value is 'pearson'. // // Currently only supports the Pearson Correlation Coefficient. optional string method = 4; } // Calculates the approximate quantiles of numerical columns of a DataFrame. // It will invoke 'Dataset.stat.approxQuantile' (same as 'StatFunctions.approxQuantile') // to compute the results. message StatApproxQuantile { // (Required) The input relation. Relation input = 1; // (Required) The names of the numerical columns. repeated string cols = 2; // (Required) A list of quantile probabilities. // // Each number must belong to [0, 1]. // For example 0 is the minimum, 0.5 is the median, 1 is the maximum. repeated double probabilities = 3; // (Required) The relative target precision to achieve (greater than or equal to 0). // // If set to zero, the exact quantiles are computed, which could be very expensive. // Note that values greater than 1 are accepted but give the same result as 1. double relative_error = 4; } // Finding frequent items for columns, possibly with false positives. // It will invoke 'Dataset.stat.freqItems' (same as 'StatFunctions.freqItems') // to compute the results. message StatFreqItems { // (Required) The input relation. Relation input = 1; // (Required) The names of the columns to search frequent items in. repeated string cols = 2; // (Optional) The minimum frequency for an item to be considered `frequent`. // Should be greater than 1e-4. optional double support = 3; } // Returns a stratified sample without replacement based on the fraction // given on each stratum. // It will invoke 'Dataset.stat.freqItems' (same as 'StatFunctions.freqItems') // to compute the results. message StatSampleBy { // (Required) The input relation. Relation input = 1; // (Required) The column that defines strata. Expression col = 2; // (Required) Sampling fraction for each stratum. // // If a stratum is not specified, we treat its fraction as zero. repeated Fraction fractions = 3; // (Required) The random seed. // This field is required to avoid generating mutable dataframes (see SPARK-48184 for details), // however, still keep it 'optional' here for backward compatibility. optional int64 seed = 5; message Fraction { // (Required) The stratum. Expression.Literal stratum = 1; // (Required) The fraction value. Must be in [0, 1]. double fraction = 2; } } // Replaces null values. // It will invoke 'Dataset.na.fill' (same as 'DataFrameNaFunctions.fill') to compute the results. // Following 3 parameter combinations are supported: // 1, 'values' only contains 1 item, 'cols' is empty: // replaces null values in all type-compatible columns. // 2, 'values' only contains 1 item, 'cols' is not empty: // replaces null values in specified columns. // 3, 'values' contains more than 1 items, then 'cols' is required to have the same length: // replaces each specified column with corresponding value. message NAFill { // (Required) The input relation. Relation input = 1; // (Optional) Optional list of column names to consider. repeated string cols = 2; // (Required) Values to replace null values with. // // Should contain at least 1 item. // Only 4 data types are supported now: bool, long, double, string repeated Expression.Literal values = 3; } // Drop rows containing null values. // It will invoke 'Dataset.na.drop' (same as 'DataFrameNaFunctions.drop') to compute the results. message NADrop { // (Required) The input relation. Relation input = 1; // (Optional) Optional list of column names to consider. // // When it is empty, all the columns in the input relation will be considered. repeated string cols = 2; // (Optional) The minimum number of non-null and non-NaN values required to keep. // // When not set, it is equivalent to the number of considered columns, which means // a row will be kept only if all columns are non-null. // // 'how' options ('all', 'any') can be easily converted to this field: // - 'all' -> set 'min_non_nulls' 1; // - 'any' -> keep 'min_non_nulls' unset; optional int32 min_non_nulls = 3; } // Replaces old values with the corresponding values. // It will invoke 'Dataset.na.replace' (same as 'DataFrameNaFunctions.replace') // to compute the results. message NAReplace { // (Required) The input relation. Relation input = 1; // (Optional) List of column names to consider. // // When it is empty, all the type-compatible columns in the input relation will be considered. repeated string cols = 2; // (Optional) The value replacement mapping. repeated Replacement replacements = 3; message Replacement { // (Required) The old value. // // Only 4 data types are supported now: null, bool, double, string. Expression.Literal old_value = 1; // (Required) The new value. // // Should be of the same data type with the old value. Expression.Literal new_value = 2; } } // Rename columns on the input relation by the same length of names. message ToDF { // (Required) The input relation of RenameColumnsBySameLengthNames. Relation input = 1; // (Required) // // The number of columns of the input relation must be equal to the length // of this field. If this is not true, an exception will be returned. repeated string column_names = 2; } // Rename columns on the input relation by a map with name to name mapping. message WithColumnsRenamed { // (Required) The input relation. Relation input = 1; // (Optional) // // Renaming column names of input relation from A to B where A is the map key // and B is the map value. This is a no-op if schema doesn't contain any A. It // does not require that all input relation column names to present as keys. // duplicated B are not allowed. map rename_columns_map = 2 [deprecated=true]; repeated Rename renames = 3; message Rename { // (Required) The existing column name. string col_name = 1; // (Required) The new column name. string new_col_name = 2; } } // Adding columns or replacing the existing columns that have the same names. message WithColumns { // (Required) The input relation. Relation input = 1; // (Required) // // Given a column name, apply the corresponding expression on the column. If column // name exists in the input relation, then replace the column. If the column name // does not exist in the input relation, then adds it as a new column. // // Only one name part is expected from each Expression.Alias. // // An exception is thrown when duplicated names are present in the mapping. repeated Expression.Alias aliases = 2; } message WithWatermark { // (Required) The input relation Relation input = 1; // (Required) Name of the column containing event time. string event_time = 2; // (Required) string delay_threshold = 3; } // Specify a hint over a relation. Hint should have a name and optional parameters. message Hint { // (Required) The input relation. Relation input = 1; // (Required) Hint name. // // Supported Join hints include BROADCAST, MERGE, SHUFFLE_HASH, SHUFFLE_REPLICATE_NL. // // Supported partitioning hints include COALESCE, REPARTITION, REPARTITION_BY_RANGE. string name = 2; // (Optional) Hint parameters. repeated Expression parameters = 3; } // Unpivot a DataFrame from wide format to long format, optionally leaving identifier columns set. message Unpivot { // (Required) The input relation. Relation input = 1; // (Required) Id columns. repeated Expression ids = 2; // (Optional) Value columns to unpivot. optional Values values = 3; // (Required) Name of the variable column. string variable_column_name = 4; // (Required) Name of the value column. string value_column_name = 5; message Values { repeated Expression values = 1; } } // Transpose a DataFrame, switching rows to columns. // Transforms the DataFrame such that the values in the specified index column // become the new columns of the DataFrame. message Transpose { // (Required) The input relation. Relation input = 1; // (Optional) A list of columns that will be treated as the indices. // Only single column is supported now. repeated Expression index_columns = 2; } message UnresolvedTableValuedFunction { // (Required) name (or unparsed name for user defined function) for the unresolved function. string function_name = 1; // (Optional) Function arguments. Empty arguments are allowed. repeated Expression arguments = 2; } message ToSchema { // (Required) The input relation. Relation input = 1; // (Required) The user provided schema. // // The Sever side will update the dataframe with this schema. DataType schema = 2; } message RepartitionByExpression { // (Required) The input relation. Relation input = 1; // (Required) The partitioning expressions. repeated Expression partition_exprs = 2; // (Optional) number of partitions, must be positive. optional int32 num_partitions = 3; } message MapPartitions { // (Required) Input relation for a mapPartitions-equivalent API: mapInPandas, mapInArrow. Relation input = 1; // (Required) Input user-defined function. CommonInlineUserDefinedFunction func = 2; // (Optional) Whether to use barrier mode execution or not. optional bool is_barrier = 3; // (Optional) ResourceProfile id used for the stage level scheduling. optional int32 profile_id = 4; } message GroupMap { // (Required) Input relation for Group Map API: apply, applyInPandas. Relation input = 1; // (Required) Expressions for grouping keys. repeated Expression grouping_expressions = 2; // (Required) Input user-defined function. CommonInlineUserDefinedFunction func = 3; // (Optional) Expressions for sorting. Only used by Scala Sorted Group Map API. repeated Expression sorting_expressions = 4; // Below fields are only used by (Flat)MapGroupsWithState // (Optional) Input relation for initial State. Relation initial_input = 5; // (Optional) Expressions for grouping keys of the initial state input relation. repeated Expression initial_grouping_expressions = 6; // (Optional) True if MapGroupsWithState, false if FlatMapGroupsWithState. optional bool is_map_groups_with_state = 7; // (Optional) The output mode of the function. optional string output_mode = 8; // (Optional) Timeout configuration for groups that do not receive data for a while. optional string timeout_conf = 9; // (Optional) The schema for the grouped state. optional DataType state_schema = 10; // Below fields are used by TransformWithState and TransformWithStateInPandas // (Optional) TransformWithState related parameters. optional TransformWithStateInfo transform_with_state_info = 11; } // Additional input parameters used for TransformWithState operator. message TransformWithStateInfo { // (Required) Time mode string for transformWithState. string time_mode = 1; // (Optional) Event time column name. optional string event_time_column_name = 2; // (Optional) Schema for the output DataFrame. // Only required used for TransformWithStateInPandas. optional DataType output_schema = 3; } message CoGroupMap { // (Required) One input relation for CoGroup Map API - applyInPandas. Relation input = 1; // Expressions for grouping keys of the first input relation. repeated Expression input_grouping_expressions = 2; // (Required) The other input relation. Relation other = 3; // Expressions for grouping keys of the other input relation. repeated Expression other_grouping_expressions = 4; // (Required) Input user-defined function. CommonInlineUserDefinedFunction func = 5; // (Optional) Expressions for sorting. Only used by Scala Sorted CoGroup Map API. repeated Expression input_sorting_expressions = 6; // (Optional) Expressions for sorting. Only used by Scala Sorted CoGroup Map API. repeated Expression other_sorting_expressions = 7; } message ApplyInPandasWithState { // (Required) Input relation for applyInPandasWithState. Relation input = 1; // (Required) Expressions for grouping keys. repeated Expression grouping_expressions = 2; // (Required) Input user-defined function. CommonInlineUserDefinedFunction func = 3; // (Required) Schema for the output DataFrame. string output_schema = 4; // (Required) Schema for the state. string state_schema = 5; // (Required) The output mode of the function. string output_mode = 6; // (Required) Timeout configuration for groups that do not receive data for a while. string timeout_conf = 7; } message CommonInlineUserDefinedTableFunction { // (Required) Name of the user-defined table function. string function_name = 1; // (Optional) Whether the user-defined table function is deterministic. bool deterministic = 2; // (Optional) Function input arguments. Empty arguments are allowed. repeated Expression arguments = 3; // (Required) Type of the user-defined table function. oneof function { PythonUDTF python_udtf = 4; } } message PythonUDTF { // (Optional) Return type of the Python UDTF. optional DataType return_type = 1; // (Required) EvalType of the Python UDTF. int32 eval_type = 2; // (Required) The encoded commands of the Python UDTF. bytes command = 3; // (Required) Python version being used in the client. string python_ver = 4; } message CommonInlineUserDefinedDataSource { // (Required) Name of the data source. string name = 1; // (Required) The data source type. oneof data_source { PythonDataSource python_data_source = 2; } } message PythonDataSource { // (Required) The encoded commands of the Python data source. bytes command = 1; // (Required) Python version being used in the client. string python_ver = 2; } // Collect arbitrary (named) metrics from a dataset. message CollectMetrics { // (Required) The input relation. Relation input = 1; // (Required) Name of the metrics. string name = 2; // (Required) The metric sequence. repeated Expression metrics = 3; } message Parse { // (Required) Input relation to Parse. The input is expected to have single text column. Relation input = 1; // (Required) The expected format of the text. ParseFormat format = 2; // (Optional) DataType representing the schema. If not set, Spark will infer the schema. optional DataType schema = 3; // Options for the csv/json parser. The map key is case insensitive. map options = 4; enum ParseFormat { PARSE_FORMAT_UNSPECIFIED = 0; PARSE_FORMAT_CSV = 1; PARSE_FORMAT_JSON = 2; } } // Relation of type [[AsOfJoin]]. // // `left` and `right` must be present. message AsOfJoin { // (Required) Left input relation for a Join. Relation left = 1; // (Required) Right input relation for a Join. Relation right = 2; // (Required) Field to join on in left DataFrame Expression left_as_of = 3; // (Required) Field to join on in right DataFrame Expression right_as_of = 4; // (Optional) The join condition. Could be unset when `using_columns` is utilized. // // This field does not co-exist with using_columns. Expression join_expr = 5; // Optional. using_columns provides a list of columns that should present on both sides of // the join inputs that this Join will join on. For example A JOIN B USING col_name is // equivalent to A JOIN B on A.col_name = B.col_name. // // This field does not co-exist with join_condition. repeated string using_columns = 6; // (Required) The join type. string join_type = 7; // (Optional) The asof tolerance within this range. Expression tolerance = 8; // (Required) Whether allow matching with the same value or not. bool allow_exact_matches = 9; // (Required) Whether to search for prior, subsequent, or closest matches. string direction = 10; } // Relation of type [[LateralJoin]]. // // `left` and `right` must be present. message LateralJoin { // (Required) Left input relation for a Join. Relation left = 1; // (Required) Right input relation for a Join. Relation right = 2; // (Optional) The join condition. Expression join_condition = 3; // (Required) The join type. Join.JoinType join_type = 4; } ================================================ FILE: spark-connect/common/src/main/protobuf/spark/connect/types.proto ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ syntax = 'proto3'; package spark.connect; option java_multiple_files = true; option java_package = "io.delta.connect.spark.proto"; option go_package = "internal/generated"; // This message describes the logical [[DataType]] of something. It does not carry the value // itself but only describes it. message DataType { oneof kind { NULL null = 1; Binary binary = 2; Boolean boolean = 3; // Numeric types Byte byte = 4; Short short = 5; Integer integer = 6; Long long = 7; Float float = 8; Double double = 9; Decimal decimal = 10; // String types String string = 11; Char char = 12; VarChar var_char = 13; // Datatime types Date date = 14; Timestamp timestamp = 15; TimestampNTZ timestamp_ntz = 16; // Interval types CalendarInterval calendar_interval = 17; YearMonthInterval year_month_interval = 18; DayTimeInterval day_time_interval = 19; // Complex types Array array = 20; Struct struct = 21; Map map = 22; Variant variant = 25; // UserDefinedType UDT udt = 23; // UnparsedDataType Unparsed unparsed = 24; } message Boolean { uint32 type_variation_reference = 1; } message Byte { uint32 type_variation_reference = 1; } message Short { uint32 type_variation_reference = 1; } message Integer { uint32 type_variation_reference = 1; } message Long { uint32 type_variation_reference = 1; } message Float { uint32 type_variation_reference = 1; } message Double { uint32 type_variation_reference = 1; } message String { uint32 type_variation_reference = 1; string collation = 2; } message Binary { uint32 type_variation_reference = 1; } message NULL { uint32 type_variation_reference = 1; } message Timestamp { uint32 type_variation_reference = 1; } message Date { uint32 type_variation_reference = 1; } message TimestampNTZ { uint32 type_variation_reference = 1; } message CalendarInterval { uint32 type_variation_reference = 1; } message YearMonthInterval { optional int32 start_field = 1; optional int32 end_field = 2; uint32 type_variation_reference = 3; } message DayTimeInterval { optional int32 start_field = 1; optional int32 end_field = 2; uint32 type_variation_reference = 3; } // Start compound types. message Char { int32 length = 1; uint32 type_variation_reference = 2; } message VarChar { int32 length = 1; uint32 type_variation_reference = 2; } message Decimal { optional int32 scale = 1; optional int32 precision = 2; uint32 type_variation_reference = 3; } message StructField { string name = 1; DataType data_type = 2; bool nullable = 3; optional string metadata = 4; } message Struct { repeated StructField fields = 1; uint32 type_variation_reference = 2; } message Array { DataType element_type = 1; bool contains_null = 2; uint32 type_variation_reference = 3; } message Map { DataType key_type = 1; DataType value_type = 2; bool value_contains_null = 3; uint32 type_variation_reference = 4; } message Variant { uint32 type_variation_reference = 1; } message UDT { string type = 1; // Required for Scala/Java UDT optional string jvm_class = 2; // Required for Python UDT optional string python_class = 3; // Required for Python UDT optional string serialized_python_class = 4; // Required for Python UDT optional DataType sql_type = 5; } message Unparsed { // (Required) The unparsed data type string string data_type_string = 1; } } ================================================ FILE: spark-connect/common/src/main/scala/org/apache/spark/sql/connect/delta/ImplicitProtoConversions.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.connect.delta import io.delta.connect.spark.{proto => delta_spark_proto} import org.apache.spark.connect.{proto => spark_proto} import org.apache.spark.sql.connect.ConnectProtoUtils object ImplicitProtoConversions { implicit def convertRelationToSpark( relation: delta_spark_proto.Relation): spark_proto.Relation = { ConnectProtoUtils.parseRelationWithRecursionLimit(relation.toByteArray, recursionLimit = 1024) } implicit def convertRelationToDelta( relation: spark_proto.Relation): delta_spark_proto.Relation = { // TODO: Recursion limits delta_spark_proto.Relation.parseFrom(relation.toByteArray) } implicit def convertCommandToSpark( command: delta_spark_proto.Command): spark_proto.Command = { ConnectProtoUtils.parseCommandWithRecursionLimit(command.toByteArray, recursionLimit = 1024) } implicit def convertCommandToDelta( command: spark_proto.Command): delta_spark_proto.Command = { // TODO: Recursion limits delta_spark_proto.Command.parseFrom(command.toByteArray) } implicit def convertExpressionToSpark( expr: delta_spark_proto.Expression): spark_proto.Expression = { ConnectProtoUtils.parseExpressionWithRecursionLimit(expr.toByteArray, recursionLimit = 1024) } implicit def convertExpressionToDelta( expr: spark_proto.Expression): delta_spark_proto.Expression = { // TODO: Recursion limits delta_spark_proto.Expression.parseFrom(expr.toByteArray) } implicit def convertDataTypeToSpark( dataType: delta_spark_proto.DataType): spark_proto.DataType = { ConnectProtoUtils.parseDataTypeWithRecursionLimit(dataType.toByteArray, recursionLimit = 1024) } implicit def convertDataTypeToDelta( dataType: spark_proto.DataType): delta_spark_proto.DataType = { // TODO: Recursion limits delta_spark_proto.DataType.parseFrom(dataType.toByteArray) } } ================================================ FILE: spark-connect/server/src/main/scala/io/delta/connect/DeltaCommandPlugin.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.connect.delta import scala.collection.JavaConverters._ import com.google.protobuf import io.delta.connect.proto import io.delta.connect.spark.{proto => spark_proto} import io.delta.tables.DeltaTable import org.apache.spark.sql.connect.common.{DataTypeProtoConverter, InvalidPlanInput} import org.apache.spark.sql.connect.delta.ImplicitProtoConversions._ import org.apache.spark.sql.connect.planner.SparkConnectPlanner import org.apache.spark.sql.connect.plugin.CommandPlugin /** * Planner plugin for command extensions using [[proto.DeltaCommand]]. */ class DeltaCommandPlugin extends CommandPlugin with DeltaPlannerBase { override def process(raw: Array[Byte], planner: SparkConnectPlanner): Boolean = { val command = protobuf.Any.parseFrom(raw) if (command.is(classOf[proto.DeltaCommand])) { process(command.unpack(classOf[proto.DeltaCommand]), planner) true } else { false } } private def process(command: proto.DeltaCommand, planner: SparkConnectPlanner): Unit = { command.getCommandTypeCase match { case proto.DeltaCommand.CommandTypeCase.CLONE_TABLE => processCloneTable(planner, command.getCloneTable) case proto.DeltaCommand.CommandTypeCase.VACUUM_TABLE => processVacuumTable(planner, command.getVacuumTable) case proto.DeltaCommand.CommandTypeCase.UPGRADE_TABLE_PROTOCOL => processUpgradeTableProtocol(planner, command.getUpgradeTableProtocol) case proto.DeltaCommand.CommandTypeCase.GENERATE => processGenerate(planner, command.getGenerate) case proto.DeltaCommand.CommandTypeCase.CREATE_DELTA_TABLE => processCreateDeltaTable(planner, command.getCreateDeltaTable) case proto.DeltaCommand.CommandTypeCase.ADD_FEATURE_SUPPORT => processAddFeatureSupport(planner, command.getAddFeatureSupport) case proto.DeltaCommand.CommandTypeCase.DROP_FEATURE_SUPPORT => processDropFeatureSupport(planner, command.getDropFeatureSupport) case _ => throw InvalidPlanInput(s"${command.getCommandTypeCase}") } } private def processCloneTable( planner: SparkConnectPlanner, cloneTable: proto.CloneTable): Unit = { val deltaTable = transformDeltaTable(planner, cloneTable.getTable) if (cloneTable.hasVersion) { deltaTable.cloneAtVersion( cloneTable.getVersion, cloneTable.getTarget, cloneTable.getIsShallow, cloneTable.getReplace, cloneTable.getPropertiesMap.asScala.toMap ) } else if (cloneTable.hasTimestamp) { deltaTable.cloneAtTimestamp( cloneTable.getTimestamp, cloneTable.getTarget, cloneTable.getIsShallow, cloneTable.getReplace, cloneTable.getPropertiesMap.asScala.toMap ) } else { deltaTable.clone( cloneTable.getTarget, cloneTable.getIsShallow, cloneTable.getReplace, cloneTable.getPropertiesMap.asScala.toMap ) } } private def processVacuumTable(planner: SparkConnectPlanner, vacuum: proto.VacuumTable): Unit = { val deltaTable = transformDeltaTable(planner, vacuum.getTable) if (vacuum.hasRetentionHours) { deltaTable.vacuum(vacuum.getRetentionHours) } else { deltaTable.vacuum() } } private def processUpgradeTableProtocol( planner: SparkConnectPlanner, upgradeTableProtocol: proto.UpgradeTableProtocol): Unit = { val deltaTable = transformDeltaTable(planner, upgradeTableProtocol.getTable) deltaTable.upgradeTableProtocol( upgradeTableProtocol.getReaderVersion, upgradeTableProtocol.getWriterVersion) } private def processGenerate(planner: SparkConnectPlanner, generate: proto.Generate): Unit = { val deltaTable = transformDeltaTable(planner, generate.getTable) deltaTable.generate(generate.getMode) } private def processCreateDeltaTable( planner: SparkConnectPlanner, createDeltaTable: proto.CreateDeltaTable): Unit = { val spark = planner.session val tableBuilder = createDeltaTable.getMode match { case proto.CreateDeltaTable.Mode.MODE_CREATE => DeltaTable.create(spark) case proto.CreateDeltaTable.Mode.MODE_CREATE_IF_NOT_EXISTS => DeltaTable.createIfNotExists(spark) case proto.CreateDeltaTable.Mode.MODE_REPLACE => DeltaTable.replace(spark) case proto.CreateDeltaTable.Mode.MODE_CREATE_OR_REPLACE => DeltaTable.createOrReplace(spark) case _ => throw new Exception("Unsupported table creation mode") } if (createDeltaTable.hasTableName) { tableBuilder.tableName(createDeltaTable.getTableName) } if (createDeltaTable.hasLocation) { tableBuilder.location(createDeltaTable.getLocation) } if (createDeltaTable.hasComment) { tableBuilder.comment(createDeltaTable.getComment) } for (column <- createDeltaTable.getColumnsList.asScala) { val colBuilder = DeltaTable .columnBuilder(spark, column.getName) .nullable(column.getNullable) if (column.getDataType.getKindCase == spark_proto.DataType.KindCase.UNPARSED) { colBuilder.dataType(column.getDataType.getUnparsed.getDataTypeString) } else { colBuilder.dataType(DataTypeProtoConverter.toCatalystType(column.getDataType)) } if (column.hasGeneratedAlwaysAs) { colBuilder.generatedAlwaysAs(column.getGeneratedAlwaysAs) } if (column.hasIdentityInfo) { val identityInfo = column.getIdentityInfo if (identityInfo.getAllowExplicitInsert) { colBuilder.generatedByDefaultAsIdentity(identityInfo.getStart, identityInfo.getStep) } else { colBuilder.generatedAlwaysAsIdentity(identityInfo.getStart, identityInfo.getStep) } } if (column.hasComment) { colBuilder.comment(column.getComment) } tableBuilder.addColumn(colBuilder.build()) } val partitioningColumns = createDeltaTable.getPartitioningColumnsList.asScala.toSeq if (!partitioningColumns.isEmpty) { tableBuilder.partitionedBy(partitioningColumns: _*) } val clusteringColumns = createDeltaTable.getClusteringColumnsList.asScala.toSeq if (!clusteringColumns.isEmpty) { tableBuilder.clusterBy(clusteringColumns: _*) } for ((key, value) <- createDeltaTable.getPropertiesMap.asScala) { tableBuilder.property(key, value) } tableBuilder.execute() } private def processAddFeatureSupport( planner: SparkConnectPlanner, addFeatureSupport: proto.AddFeatureSupport): Unit = { val deltaTable = transformDeltaTable(planner, addFeatureSupport.getTable) deltaTable.addFeatureSupport(addFeatureSupport.getFeatureName) } private def processDropFeatureSupport( planner: SparkConnectPlanner, dropFeatureSupport: proto.DropFeatureSupport): Unit = { val deltaTable = transformDeltaTable(planner, dropFeatureSupport.getTable) if (dropFeatureSupport.hasTruncateHistory) { deltaTable.dropFeatureSupport( dropFeatureSupport.getFeatureName, dropFeatureSupport.getTruncateHistory) } else { deltaTable.dropFeatureSupport(dropFeatureSupport.getFeatureName) } } } ================================================ FILE: spark-connect/server/src/main/scala/io/delta/connect/DeltaPlannerBase.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.connect.delta import io.delta.connect.proto import io.delta.tables.DeltaTable import org.apache.spark.sql.connect.planner.SparkConnectPlanner /** * Base trait for the planner plugins of Delta Connect. */ trait DeltaPlannerBase { protected def transformDeltaTable( planner: SparkConnectPlanner, deltaTable: proto.DeltaTable): DeltaTable = { deltaTable.getAccessTypeCase match { case proto.DeltaTable.AccessTypeCase.PATH => DeltaTable.forPath( planner.session, deltaTable.getPath.getPath, deltaTable.getPath.getHadoopConfMap) case proto.DeltaTable.AccessTypeCase.TABLE_OR_VIEW_NAME => DeltaTable.forName(planner.session, deltaTable.getTableOrViewName) } } } ================================================ FILE: spark-connect/server/src/main/scala/io/delta/connect/DeltaRelationPlugin.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.connect.delta import java.util.Optional import scala.collection.JavaConverters._ import com.google.protobuf import com.google.protobuf.{ByteString, InvalidProtocolBufferException} import io.delta.connect.proto import io.delta.tables.DeltaTable import org.apache.spark.SparkEnv import org.apache.spark.sql.{Dataset, Encoders, SparkSession} import org.apache.spark.sql.catalyst.analysis.UnresolvedStar import org.apache.spark.sql.catalyst.expressions.Literal import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.classic.Dataset import org.apache.spark.sql.connect.common.{DataTypeProtoConverter, InvalidPlanInput} import org.apache.spark.sql.connect.config.Connect import org.apache.spark.sql.connect.delta.DeltaRelationPlugin.{parseAnyFrom, parseRelationFrom} import org.apache.spark.sql.connect.delta.ImplicitProtoConversions._ import org.apache.spark.sql.connect.planner.SparkConnectPlanner import org.apache.spark.sql.connect.plugin.RelationPlugin import org.apache.spark.sql.delta.DataFrameUtils import org.apache.spark.sql.delta.commands.ConvertToDeltaCommand import org.apache.spark.sql.types.StructType /** * Planner plugin for relation extensions using [[proto.DeltaRelation]]. */ class DeltaRelationPlugin extends RelationPlugin with DeltaPlannerBase { override def transform(raw: Array[Byte], planner: SparkConnectPlanner): Optional[LogicalPlan] = { val relation = parseAnyFrom(raw, SparkEnv.get.conf.get(Connect.CONNECT_GRPC_MARSHALLER_RECURSION_LIMIT)) if (relation.is(classOf[proto.DeltaRelation])) { Optional.of( transform( parseRelationFrom(relation.getValue, SparkEnv.get.conf.get(Connect.CONNECT_GRPC_MARSHALLER_RECURSION_LIMIT)), planner )) } else { Optional.empty() } } private def transform( relation: proto.DeltaRelation, planner: SparkConnectPlanner): LogicalPlan = { relation.getRelationTypeCase match { case proto.DeltaRelation.RelationTypeCase.SCAN => transformScan(planner, relation.getScan) case proto.DeltaRelation.RelationTypeCase.DESCRIBE_HISTORY => transformDescribeHistory(planner, relation.getDescribeHistory) case proto.DeltaRelation.RelationTypeCase.DESCRIBE_DETAIL => transformDescribeDetail(planner, relation.getDescribeDetail) case proto.DeltaRelation.RelationTypeCase.CONVERT_TO_DELTA => transformConvertToDelta(planner, relation.getConvertToDelta) case proto.DeltaRelation.RelationTypeCase.RESTORE_TABLE => transformRestoreTable(planner, relation.getRestoreTable) case proto.DeltaRelation.RelationTypeCase.IS_DELTA_TABLE => transformIsDeltaTable(planner.session, relation.getIsDeltaTable) case proto.DeltaRelation.RelationTypeCase.DELETE_FROM_TABLE => transformDeleteFromTable(planner, relation.getDeleteFromTable) case proto.DeltaRelation.RelationTypeCase.UPDATE_TABLE => transformUpdateTable(planner, relation.getUpdateTable) case proto.DeltaRelation.RelationTypeCase.MERGE_INTO_TABLE => transformMergeIntoTable(planner, relation.getMergeIntoTable) case proto.DeltaRelation.RelationTypeCase.OPTIMIZE_TABLE => transformOptimizeTable(planner, relation.getOptimizeTable) case _ => throw InvalidPlanInput(s"Unknown DeltaRelation ${relation.getRelationTypeCase}") } } private def transformScan(planner: SparkConnectPlanner, scan: proto.Scan): LogicalPlan = { val deltaTable = transformDeltaTable(planner, scan.getTable) deltaTable.toDF.queryExecution.analyzed } private def transformDescribeHistory( planner: SparkConnectPlanner, describeHistory: proto.DescribeHistory): LogicalPlan = { val deltaTable = transformDeltaTable(planner, describeHistory.getTable) deltaTable.history().queryExecution.analyzed } private def transformDescribeDetail( planner: SparkConnectPlanner, describeDetail: proto.DescribeDetail): LogicalPlan = { val deltaTable = transformDeltaTable(planner, describeDetail.getTable) deltaTable.detail().queryExecution.analyzed } private def transformConvertToDelta( planner: SparkConnectPlanner, convertToDelta: proto.ConvertToDelta): LogicalPlan = { val spark = planner.session val tableIdentifier = spark.sessionState.sqlParser.parseTableIdentifier(convertToDelta.getIdentifier) val partitionSchema = if (convertToDelta.hasPartitionSchemaStruct) { Some(DataTypeProtoConverter.toCatalystType(convertToDelta.getPartitionSchemaStruct) .asInstanceOf[StructType]) } else if (convertToDelta.hasPartitionSchemaString) { Some(StructType.fromDDL(convertToDelta.getPartitionSchemaString)) } else { None } val cvt = ConvertToDeltaCommand( tableIdentifier, partitionSchema, collectStats = true, deltaPath = None) cvt.run(spark) val result = if (cvt.isCatalogTable(spark.sessionState.analyzer, tableIdentifier)) { convertToDelta.getIdentifier } else { s"delta.`${tableIdentifier.table}`" } spark.createDataset(result :: Nil)(Encoders.STRING).queryExecution.analyzed } private def transformRestoreTable( planner: SparkConnectPlanner, restoreTable: proto.RestoreTable): LogicalPlan = { val deltaTable = transformDeltaTable(planner, restoreTable.getTable) val df = if (restoreTable.hasVersion) { deltaTable.restoreToVersion(restoreTable.getVersion) } else if (restoreTable.hasTimestamp) { deltaTable.restoreToTimestamp(restoreTable.getTimestamp) } else { throw new RuntimeException() } df.queryExecution.commandExecuted } private def transformIsDeltaTable( spark: SparkSession, isDeltaTable: proto.IsDeltaTable): LogicalPlan = { val result = DeltaTable.isDeltaTable(spark, isDeltaTable.getPath) spark.createDataset(result :: Nil)(Encoders.scalaBoolean).queryExecution.analyzed } private def transformDeleteFromTable( planner: SparkConnectPlanner, deleteFromTable: proto.DeleteFromTable): LogicalPlan = { val target = planner.transformRelation(deleteFromTable.getTarget) val condition = if (deleteFromTable.hasCondition) { Some(planner.transformExpression(deleteFromTable.getCondition)) } else { None } DataFrameUtils.ofRows( planner.session, DeleteFromTable(target, condition.getOrElse(Literal.TrueLiteral))) .queryExecution.commandExecuted } private def transformUpdateTable( planner: SparkConnectPlanner, updateTable: proto.UpdateTable): LogicalPlan = { val target = planner.transformRelation(updateTable.getTarget) val condition = if (updateTable.hasCondition) { Some(planner.transformExpression(updateTable.getCondition)) } else { None } val assignments = updateTable.getAssignmentsList.asScala.map(transformAssignment(planner, _)) DataFrameUtils.ofRows(planner.session, UpdateTable(target, assignments.toSeq, condition)) .queryExecution.commandExecuted } private def transformMergeIntoTable( planner: SparkConnectPlanner, protoMerge: proto.MergeIntoTable): LogicalPlan = { val target = planner.transformRelation(protoMerge.getTarget) val source = planner.transformRelation(protoMerge.getSource) val condition = planner.transformExpression(protoMerge.getCondition) val matchedActions = protoMerge.getMatchedActionsList.asScala .map(transformMergeWhenMatchedAction(planner, _)) val notMatchedActions = protoMerge.getNotMatchedActionsList.asScala .map(transformMergeWhenNotMatchedAction(planner, _)) val notMatchedBySourceActions = protoMerge.getNotMatchedBySourceActionsList.asScala .map(transformMergeWhenNotMatchedBySourceAction(planner, _)) val withSchemaEvolution = protoMerge.getWithSchemaEvolution val merge = DeltaMergeInto( target, source, condition, matchedActions.toSeq ++ notMatchedActions.toSeq ++ notMatchedBySourceActions.toSeq, withSchemaEvolution ) Dataset.ofRows(planner.session, merge).queryExecution.commandExecuted } private def transformMergeActionCondition( planner: SparkConnectPlanner, protoAction: proto.MergeIntoTable.Action): Option[Expression] = { if (protoAction.hasCondition) { Some(planner.transformExpression(protoAction.getCondition)) } else { None } } private def transformMergeWhenMatchedAction( planner: SparkConnectPlanner, protoAction: proto.MergeIntoTable.Action): DeltaMergeIntoMatchedClause = { val condition = transformMergeActionCondition(planner, protoAction) protoAction.getActionTypeCase match { case proto.MergeIntoTable.Action.ActionTypeCase.DELETE_ACTION => DeltaMergeIntoMatchedDeleteClause(condition) case proto.MergeIntoTable.Action.ActionTypeCase.UPDATE_ACTION => val actions = transformMergeAssignments( planner, protoAction.getUpdateAction.getAssignmentsList.asScala.toSeq) DeltaMergeIntoMatchedUpdateClause(condition, actions) case proto.MergeIntoTable.Action.ActionTypeCase.UPDATE_STAR_ACTION => DeltaMergeIntoMatchedUpdateClause(condition, Seq(UnresolvedStar(None))) } } private def transformMergeWhenNotMatchedAction( planner: SparkConnectPlanner, protoAction: proto.MergeIntoTable.Action): DeltaMergeIntoNotMatchedClause = { val condition = transformMergeActionCondition(planner, protoAction) protoAction.getActionTypeCase match { case proto.MergeIntoTable.Action.ActionTypeCase.INSERT_ACTION => val actions = transformMergeAssignments( planner, protoAction.getInsertAction.getAssignmentsList.asScala.toSeq) DeltaMergeIntoNotMatchedInsertClause(condition, actions) case proto.MergeIntoTable.Action.ActionTypeCase.INSERT_STAR_ACTION => DeltaMergeIntoNotMatchedInsertClause(condition, Seq(UnresolvedStar(None))) } } private def transformMergeWhenNotMatchedBySourceAction( planner: SparkConnectPlanner, protoAction: proto.MergeIntoTable.Action): DeltaMergeIntoNotMatchedBySourceClause = { val condition = transformMergeActionCondition(planner, protoAction) protoAction.getActionTypeCase match { case proto.MergeIntoTable.Action.ActionTypeCase.DELETE_ACTION => DeltaMergeIntoNotMatchedBySourceDeleteClause(condition) case proto.MergeIntoTable.Action.ActionTypeCase.UPDATE_ACTION => val actions = transformMergeAssignments( planner, protoAction.getUpdateAction.getAssignmentsList.asScala.toSeq) DeltaMergeIntoNotMatchedBySourceUpdateClause(condition, actions) } } private def transformMergeAssignments( planner: SparkConnectPlanner, protoAssignments: Seq[proto.Assignment]): Seq[Expression] = { if (protoAssignments.isEmpty) { Seq.empty } else { DeltaMergeIntoClause.toActions(protoAssignments.map(transformAssignment(planner, _))) } } private def transformAssignment( planner: SparkConnectPlanner, assignment: proto.Assignment): Assignment = { Assignment( key = planner.transformExpression(assignment.getField), value = planner.transformExpression(assignment.getValue)) } private def transformOptimizeTable( planner: SparkConnectPlanner, optimizeTable: proto.OptimizeTable): LogicalPlan = { val deltaTable = transformDeltaTable(planner, optimizeTable.getTable) var optimizeBuilder = deltaTable.optimize() for (partitionFilter <- optimizeTable.getPartitionFiltersList.asScala) { optimizeBuilder = optimizeBuilder.where(partitionFilter) } val df = if (optimizeTable.getZorderColumnsList.isEmpty) { optimizeBuilder.executeCompaction() } else { optimizeBuilder.executeZOrderBy(optimizeTable.getZorderColumnsList.asScala.toSeq: _*) } df.queryExecution.commandExecuted } } object DeltaRelationPlugin { private def parseAnyFrom(ba: Array[Byte], recursionLimit: Int): protobuf.Any = { val bs = ByteString.copyFrom(ba) val cis = bs.newCodedInput() cis.setSizeLimit(Integer.MAX_VALUE) cis.setRecursionLimit(recursionLimit) val plan = protobuf.Any.parseFrom(cis) try { // If the last tag is 0, it means the message is correctly parsed. // If the last tag is not 0, it means the message is not correctly // parsed, and we should throw an exception. cis.checkLastTagWas(0) plan } catch { case e: InvalidProtocolBufferException => e.setUnfinishedMessage(plan) throw e } } private def parseRelationFrom(bs: ByteString, recursionLimit: Int): proto.DeltaRelation = { val cis = bs.newCodedInput() cis.setSizeLimit(Integer.MAX_VALUE) cis.setRecursionLimit(recursionLimit) val plan = proto.DeltaRelation.parseFrom(cis) try { // If the last tag is 0, it means the message is correctly parsed. // If the last tag is not 0, it means the message is not correctly // parsed, and we should throw an exception. cis.checkLastTagWas(0) plan } catch { case e: InvalidProtocolBufferException => e.setUnfinishedMessage(plan) throw e } } } ================================================ FILE: spark-connect/server/src/main/scala/io/delta/connect/SimpleDeltaConnectService.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains code from the Apache Spark project (original license above). * It contains modifications, which are licensed as follows: */ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.tables import java.util.concurrent.TimeUnit import scala.io.StdIn import scala.sys.exit import org.apache.spark.SparkConf import org.apache.spark.sql.SparkSession import org.apache.spark.sql.connect.service.SparkConnectService /** * A simple main class method to start the delta connect server as a service for client tests * using spark-submit: * {{{ * bin/spark-submit --class io.delta.tables.SimpleDeltaConnectService * }}} * The service can be stopped by receiving a stop command or until the service get killed. */ object SimpleDeltaConnectService { private val stopCommand = "q" def main(args: Array[String]): Unit = { val conf = new SparkConf() .set("spark.plugins", "org.apache.spark.sql.connect.SparkConnectPlugin") .set("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .set("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") val sparkSession = SparkSession.builder().config(conf).getOrCreate() // scalastyle:off println println("Ready for client connections.") // scalastyle:on println while (true) { val code = StdIn.readLine() if (code == stopCommand) { // scalastyle:off println println("No more client connections.") // scalastyle:on println // Wait for 1 min for the server to stop SparkConnectService.stop(Some(1), Some(TimeUnit.MINUTES)) sparkSession.close() exit(0) } } } } ================================================ FILE: spark-connect/server/src/test/scala/io/delta/connect/DeltaConnectPlannerSuite.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.connect.delta import java.io.File import java.text.SimpleDateFormat import java.util.UUID import scala.collection.JavaConverters._ import com.google.protobuf import io.delta.connect.proto import io.delta.connect.spark.{proto => spark_proto} import io.delta.tables.DeltaTable import org.apache.spark.SparkConf import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.{AnalysisException, Dataset, QueryTest, Row, SparkSession} import org.apache.spark.sql.catalyst.analysis.ResolvedTable import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.catalyst.plans.logical.LocalRelation import org.apache.spark.sql.classic.{Dataset, DataFrame} import org.apache.spark.sql.connect.config.Connect import org.apache.spark.sql.connect.delta.ImplicitProtoConversions._ import org.apache.spark.sql.connect.planner.{SparkConnectPlanTest, SparkConnectPlanner} import org.apache.spark.sql.connect.service.{SessionHolder, SparkConnectService} import org.apache.spark.sql.delta.{DataFrameUtils, DeltaConfigs, DeltaHistory, DeltaLog, DeltaTableFeatureException, TestReaderWriterFeature, TestRemovableWriterFeature} import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions.Protocol import org.apache.spark.sql.delta.commands.{DescribeDeltaDetailCommand, DescribeDeltaHistory} import org.apache.spark.sql.delta.commands.optimize.OptimizeMetrics import org.apache.spark.sql.delta.hooks.GenerateSymlinkManifest import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf} import org.apache.spark.sql.delta.sources.DeltaSourceUtils.GENERATION_EXPRESSION_METADATA_KEY import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.test.DeltaTestImplicits._ import org.apache.spark.sql.delta.util.{DeltaFileOperations, FileNames} import org.apache.spark.sql.types.{IntegerType, LongType, MetadataBuilder, StructField, StructType} import org.apache.spark.sql.functions._ class DeltaConnectPlannerSuite extends QueryTest with DeltaSQLCommandTest with SparkConnectPlanTest { import DeltaConnectPlannerSuite._ protected override def sparkConf: SparkConf = { super.sparkConf .set(Connect.CONNECT_EXTENSIONS_RELATION_CLASSES.key, classOf[DeltaRelationPlugin].getName) .set(Connect.CONNECT_EXTENSIONS_COMMAND_CLASSES.key, classOf[DeltaCommandPlugin].getName) } def createDummySessionHolder(session: SparkSession): SessionHolder = { val ret = SessionHolder( userId = "testUser", sessionId = UUID.randomUUID().toString, session = session ) SparkConnectService.sessionManager.putSessionForTesting(ret) ret } test("scan table by name") { withTable("table") { DeltaTable.create(spark).tableName(identifier = "table").execute() val input = createSparkRelation( proto.DeltaRelation .newBuilder() .setScan( proto.Scan .newBuilder() .setTable(proto.DeltaTable.newBuilder().setTableOrViewName("table")) ) ) val result = transform(input) val expected = DeltaTable.forName(spark, "table").toDF.queryExecution.analyzed comparePlans(result, expected) } } test("scan table by path") { withTempDir { dir => DeltaTable.create(spark).location(dir.getAbsolutePath).execute() val input = createSparkRelation( proto.DeltaRelation .newBuilder() .setScan( proto.Scan .newBuilder() .setTable( proto.DeltaTable .newBuilder() .setPath( proto.DeltaTable.Path.newBuilder().setPath(dir.getAbsolutePath) ) ) ) ) val result = transform(input) val expected = DeltaTable.forPath(spark, dir.getAbsolutePath).toDF.queryExecution.analyzed comparePlans(result, expected) } } test("convert to delta") { withTempDir { dir => spark.range(100).write.format("parquet").mode("overwrite").save(dir.getAbsolutePath) val input = createSparkRelation( proto.DeltaRelation .newBuilder() .setConvertToDelta( proto.ConvertToDelta .newBuilder() .setIdentifier(s"parquet.`${dir.getAbsolutePath}`") ) ) val plan = transform(input) val result = DataFrameUtils.ofRows(spark, plan).collect() assert(result.length === 1) val deltaTable = DeltaTable.forName(spark, result.head.getString(0)) assert(!deltaTable.toDF.isEmpty) } } test("convert to delta with partitioning schema string") { withTempDir { dir => spark.range(100) .select(col("id") % 10 as "part", col("id") as "value") .write .partitionBy("part") .format("parquet") .mode("overwrite") .save(dir.getAbsolutePath) val input = createSparkRelation( proto.DeltaRelation .newBuilder() .setConvertToDelta( proto.ConvertToDelta .newBuilder() .setIdentifier(s"parquet.`${dir.getAbsolutePath}`") .setPartitionSchemaString("part LONG") ) ) val plan = transform(input) val result = DataFrameUtils.ofRows(spark, plan).collect() assert(result.length === 1) val deltaTable = DeltaTable.forName(spark, result.head.getString(0)) assert(!deltaTable.toDF.isEmpty) } } test("convert to delta with partitioning schema struct") { withTempDir { dir => spark.range(100) .select(col("id") % 10 as "part", col("id") as "value") .write .partitionBy("part") .format("parquet") .mode("overwrite") .save(dir.getAbsolutePath) val input = createSparkRelation( proto.DeltaRelation .newBuilder() .setConvertToDelta( proto.ConvertToDelta .newBuilder() .setIdentifier(s"parquet.`${dir.getAbsolutePath}`") .setPartitionSchemaStruct( spark_proto.DataType .newBuilder() .setStruct( spark_proto.DataType.Struct .newBuilder() .addFields( spark_proto.DataType.StructField .newBuilder() .setName("part") .setNullable(false) .setDataType( spark_proto.DataType .newBuilder() .setLong(spark_proto.DataType.Long.newBuilder()) ) ) ) ) ) ) val plan = transform(input) val result = DataFrameUtils.ofRows(spark, plan).collect() assert(result.length === 1) val deltaTable = DeltaTable.forName(spark, result.head.getString(0)) assert(!deltaTable.toDF.isEmpty) } } test("history") { withTable("table") { DeltaTable.create(spark).tableName(identifier = "table").execute() val input = createSparkRelation( proto.DeltaRelation .newBuilder() .setDescribeHistory( proto.DescribeHistory.newBuilder() .setTable(proto.DeltaTable.newBuilder().setTableOrViewName("table")) ) ) val result = transform(input) result match { case lr: LocalRelation if lr.schema == ExpressionEncoder[DeltaHistory]().schema => case other => fail(s"Unexpected plan: $other") } } } test("detail") { withTable("table") { DeltaTable.create(spark).tableName(identifier = "table").execute() val input = createSparkRelation( proto.DeltaRelation .newBuilder() .setDescribeDetail( proto.DescribeDetail.newBuilder() .setTable(proto.DeltaTable.newBuilder().setTableOrViewName("table")) ) ) val result = transform(input) val expected = DeltaTable.forName(spark, "table").detail().queryExecution.analyzed assert(result.isInstanceOf[DescribeDeltaDetailCommand]) val childResult = result.asInstanceOf[DescribeDeltaDetailCommand].child val childExpected = expected.asInstanceOf[DescribeDeltaDetailCommand].child assert(childResult.asInstanceOf[ResolvedTable].identifier.name === childExpected.asInstanceOf[ResolvedTable].identifier.name) } } private val expectedRestoreOutputColumns = Seq( "table_size_after_restore", "num_of_files_after_restore", "num_removed_files", "num_restored_files", "removed_files_size", "restored_files_size" ) test("restore to version number") { withTable("table") { spark.range(start = 0, end = 1000, step = 1, numPartitions = 1) .write.format("delta").saveAsTable("table") spark.range(start = 0, end = 2000, step = 1, numPartitions = 2) .write.format("delta").mode("append").saveAsTable("table") val input = createSparkRelation( proto.DeltaRelation .newBuilder() .setRestoreTable( proto.RestoreTable.newBuilder() .setTable(proto.DeltaTable.newBuilder().setTableOrViewName("table")) .setVersion(0L) ) ) val plan = transform(input) assert(plan.columns.toSeq == expectedRestoreOutputColumns) val result = DataFrameUtils.ofRows(spark, plan).collect() assert(result.length === 1) assert(result.head.getLong(2) === 2) // Two files should have been removed. assert(spark.read.table("table").count() === 1000) } } test("restore to timestamp") { withTempDir { dir => spark.range(start = 0, end = 1000, step = 1, numPartitions = 1) .write.format("delta").save(dir.getAbsolutePath) spark.range(start = 0, end = 2000, step = 1, numPartitions = 2) .write.format("delta").mode("append").save(dir.getAbsolutePath) val deltaLog = DeltaLog.forTable(spark, dir) val input = createSparkRelation( proto.DeltaRelation .newBuilder() .setRestoreTable( proto.RestoreTable.newBuilder() .setTable( proto.DeltaTable.newBuilder().setPath( proto.DeltaTable.Path.newBuilder().setPath(dir.getAbsolutePath) ) ) .setTimestamp(getTimestampForVersion(deltaLog, version = 0)) ) ) val plan = transform(input) assert(plan.columns.toSeq === expectedRestoreOutputColumns) val result = DataFrameUtils.ofRows(spark, plan).collect() assert(result.length === 1) assert(result.head.getLong(2) === 2) // Two files should have been removed. assert(spark.read.format("delta").load(dir.getAbsolutePath).count() === 1000) } } test("isDeltaTable - delta") { withTempDir { dir => val path = dir.getAbsolutePath spark.range(end = 1000).write.format("delta").mode("overwrite").save(path) val input = createSparkRelation( proto.DeltaRelation.newBuilder() .setIsDeltaTable(proto.IsDeltaTable.newBuilder().setPath(path)) ) val plan = transform(input) assert(plan.schema.length === 1) val result = DataFrameUtils.ofRows(spark, plan).collect() assert(result.length === 1) assert(result.head.getBoolean(0)) } } test("isDeltaTable - parquet") { withTempDir { dir => val path = dir.getAbsolutePath spark.range(end = 1000).write.format("parquet").mode("overwrite").save(path) val input = createSparkRelation( proto.DeltaRelation.newBuilder() .setIsDeltaTable(proto.IsDeltaTable.newBuilder().setPath(path)) ) val plan = transform(input) assert(plan.schema.length === 1) val result = DataFrameUtils.ofRows(spark, plan).collect() assert(result.length === 1) assert(!result.head.getBoolean(0)) } } test("delete without condition") { val tableName = "table" withTable(tableName) { spark.range(end = 1000).write.format("delta").saveAsTable(tableName) val input = createSparkRelation( proto.DeltaRelation.newBuilder() .setDeleteFromTable( proto.DeleteFromTable.newBuilder() .setTarget(createScan(tableName)) ) ) val plan = transform(input) val result = DataFrameUtils.ofRows(spark, plan).collect() assert(result.length === 1) assert(result.head.getLong(0) === 1000) assert(spark.read.table(tableName).isEmpty) } } test("delete with condition") { val tableName = "table" withTable(tableName) { spark.range(end = 1000).write.format("delta").saveAsTable(tableName) val input = createSparkRelation( proto.DeltaRelation.newBuilder() .setDeleteFromTable( proto.DeleteFromTable.newBuilder() .setTarget(createScan(tableName)) .setCondition(createExpression("id % 2 = 0")) ) ) val plan = transform(input) val result = DataFrameUtils.ofRows(spark, plan).collect() assert(result.length === 1) assert(result.head.getLong(0) === 500) assert(spark.read.table(tableName).count() === 500) } } test("update without condition") { val tableName = "target" withTable(tableName) { spark.range(end = 1000).select(col("id") as "key", col("id") as "value") .write.format("delta").saveAsTable(tableName) val input = createSparkRelation( proto.DeltaRelation.newBuilder() .setUpdateTable( proto.UpdateTable.newBuilder() .setTarget(createScan(tableName)) .addAssignments(createAssignment(field = "value", value = "value + 1")) ) ) val plan = transform(input) val result = DataFrameUtils.ofRows(spark, plan).collect() assert(result.length === 1) assert(result.head.getLong(0) === 1000) checkAnswer( spark.read.table(tableName), Seq.tabulate(1000)(i => Row(i, i + 1)) ) } } test("update with condition") { val tableName = "target" withTable(tableName) { spark.range(end = 1000).select(col("id") as "key", col("id") as "value") .write.format("delta").saveAsTable(tableName) val input = createSparkRelation( proto.DeltaRelation.newBuilder() .setUpdateTable( proto.UpdateTable.newBuilder() .setTarget(createScan(tableName)) .setCondition(createExpression("key % 2 = 0")) .addAssignments(createAssignment(field = "value", value = "value + 1")) ) ) val plan = transform(input) val result = DataFrameUtils.ofRows(spark, plan).collect() assert(result.length === 1) assert(result.head.getLong(0) === 500) checkAnswer( spark.read.table(tableName), Seq.tabulate(1000)(i => Row(i, if (i % 2 == 0) i + 1 else i)) ) } } test("merge - insert only") { val targetTableName = "target" val sourceTableName = "source" withTable(targetTableName, sourceTableName) { spark.range(end = 100).select(col("id") as "key", col("id") as "value") .write.format("delta").saveAsTable(targetTableName) spark.range(end = 100) .select(col("id") + 50 as "id") .select(col("id") as "key", col("id") + 1000 as "value") .write.format("delta").saveAsTable(sourceTableName) val input = createSparkRelation( proto.DeltaRelation.newBuilder() .setMergeIntoTable( proto.MergeIntoTable.newBuilder() .setTarget(createSubqueryAlias(createScan(targetTableName), alias = "t")) .setSource(createSubqueryAlias(createScan(sourceTableName), alias = "s")) .setCondition(createExpression("t.key = s.key")) .addNotMatchedActions( proto.MergeIntoTable.Action.newBuilder() .setInsertAction( proto.MergeIntoTable.Action.InsertAction.newBuilder() .addAssignments(createAssignment(field = "t.key", value = "s.key")) .addAssignments(createAssignment(field = "t.value", value = "s.value")) ) ) ) ) val plan = transform(input) val result = Dataset.ofRows(spark, plan).collect() assert(result.length == 1) assert(result.head.getLong(0) == 50) // num_affected_rows assert(result.head.getLong(1) == 0) // num_updated_rows assert(result.head.getLong(2) == 0) // num_deleted_rows assert(result.head.getLong(3) == 50) // num_inserted_rows checkAnswer( spark.read.table(targetTableName), Seq.tabulate(100)(i => Row(i, i)) ++ Seq.tabulate(50)(i => Row(i + 100, i + 1100)) ) } } test("merge - update only") { val targetTableName = "target" val sourceTableName = "source" withTable(targetTableName, sourceTableName) { spark.range(end = 100).select(col("id") as "key", col("id") as "value") .write.format("delta").saveAsTable(targetTableName) spark.range(end = 100) .select(col("id") + 50 as "id") .select(col("id") as "key", col("id") + 1000 as "value") .write.format("delta").saveAsTable(sourceTableName) val input = createSparkRelation( proto.DeltaRelation.newBuilder() .setMergeIntoTable( proto.MergeIntoTable.newBuilder() .setTarget(createSubqueryAlias(createScan(targetTableName), alias = "t")) .setSource(createSubqueryAlias(createScan(sourceTableName), alias = "s")) .setCondition(createExpression("t.key = s.key")) .addMatchedActions( proto.MergeIntoTable.Action.newBuilder() .setUpdateAction( proto.MergeIntoTable.Action.UpdateAction.newBuilder() .addAssignments(createAssignment(field = "t.key", value = "s.key")) .addAssignments(createAssignment(field = "t.value", value = "s.value")) ) ) ) ) val plan = transform(input) val result = Dataset.ofRows(spark, plan).collect() assert(result.length == 1) assert(result.head.getLong(0) == 50) // num_affected_rows assert(result.head.getLong(1) == 50) // num_updated_rows assert(result.head.getLong(2) == 0) // num_deleted_rows assert(result.head.getLong(3) == 0) // num_inserted_rows checkAnswer( spark.read.table(targetTableName), Seq.tabulate(50)(i => Row(i, i)) ++ Seq.tabulate(50)(i => Row(i + 50, i + 1050)) ) } } test("merge - mixed") { val targetTableName = "target" val sourceTableName = "source" withTable(targetTableName, sourceTableName) { spark.range(end = 100).select(col("id") as "key", col("id") as "value") .write.format("delta").saveAsTable(targetTableName) spark.range(end = 100) .select(col("id") + 50 as "id") .select(col("id") as "key", col("id") + 1000 as "value") .write.format("delta").saveAsTable(sourceTableName) val input = createSparkRelation( proto.DeltaRelation.newBuilder() .setMergeIntoTable( proto.MergeIntoTable.newBuilder() .setTarget(createSubqueryAlias(createScan(targetTableName), alias = "t")) .setSource(createSubqueryAlias(createScan(sourceTableName), alias = "s")) .setCondition(createExpression("t.key = s.key")) .addMatchedActions( proto.MergeIntoTable.Action.newBuilder() .setUpdateStarAction(proto.MergeIntoTable.Action.UpdateStarAction.newBuilder()) ) .addNotMatchedActions( proto.MergeIntoTable.Action.newBuilder() .setInsertStarAction(proto.MergeIntoTable.Action.InsertStarAction.newBuilder()) ) .addNotMatchedBySourceActions( proto.MergeIntoTable.Action.newBuilder() .setCondition(createExpression("t.value < 25")) .setDeleteAction(proto.MergeIntoTable.Action.DeleteAction.newBuilder()) ) ) ) val plan = transform(input) val result = Dataset.ofRows(spark, plan).collect() assert(result.length == 1) assert(result.head.getLong(0) == 125) // num_affected_rows assert(result.head.getLong(1) == 50) // num_updated_rows assert(result.head.getLong(2) == 25) // num_deleted_rows assert(result.head.getLong(3) == 50) // num_inserted_rows checkAnswer( spark.read.table(targetTableName), Seq.tabulate(25)(i => Row(25 + i, 25 + i)) ++ Seq.tabulate(50)(i => Row(i + 50, i + 1050)) ++ Seq.tabulate(50)(i => Row(i + 100, i + 1100)) ) } } test("merge - withSchemaEvolution") { val targetTableName = "target" val sourceTableName = "source" withTable(targetTableName, sourceTableName) { spark.range(end = 100).select(col("id") as "key", col("id") as "value") .write.format("delta").saveAsTable(targetTableName) spark.range(end = 100) .select(col("id") + 50 as "id") .select(col("id") as "key", col("id") + 1000 as "value", col("id") + 1 as "extracol") .write.format("delta").saveAsTable(sourceTableName) val input = createSparkRelation( proto.DeltaRelation.newBuilder() .setMergeIntoTable( proto.MergeIntoTable.newBuilder() .setTarget(createSubqueryAlias(createScan(targetTableName), alias = "t")) .setSource(createSubqueryAlias(createScan(sourceTableName), alias = "s")) .setCondition(createExpression("t.key = s.key")) .addMatchedActions( proto.MergeIntoTable.Action.newBuilder() .setUpdateStarAction(proto.MergeIntoTable.Action.UpdateStarAction.newBuilder()) ) .addNotMatchedActions( proto.MergeIntoTable.Action.newBuilder() .setInsertStarAction(proto.MergeIntoTable.Action.InsertStarAction.newBuilder()) ) .addNotMatchedBySourceActions( proto.MergeIntoTable.Action.newBuilder() .setCondition(createExpression("t.value < 25")) .setDeleteAction(proto.MergeIntoTable.Action.DeleteAction.newBuilder()) ) .setWithSchemaEvolution(true) ) ) val plan = transform(input) val result = Dataset.ofRows(spark, plan).collect() assert(result.length === 1) assert(result.head.getLong(0) === 125) // num_affected_rows assert(result.head.getLong(1) === 50) // num_updated_rows assert(result.head.getLong(2) === 25) // num_deleted_rows assert(result.head.getLong(3) === 50) // num_inserted_rows checkAnswer( spark.read.table(targetTableName), Seq.tabulate(25)(i => Row(25 + i, 25 + i, null)) ++ Seq.tabulate(50)(i => Row(i + 50, i + 1050, i + 51)) ++ Seq.tabulate(50)(i => Row(i + 100, i + 1100, i + 101)) ) } } test("merge with no withSchemaEvolution while the source's schema " + "is different than the target's schema") { val targetTableName = "target" val sourceTableName = "source" withTable(targetTableName, sourceTableName) { spark.range(end = 100).select(col("id") as "key", col("id") as "value") .write.format("delta").saveAsTable(targetTableName) spark.range(end = 100) .select(col("id") + 50 as "id") .select(col("id") as "key", col("id") + 1000 as "value", col("id") + 1 as "extracol") .write.format("delta").saveAsTable(sourceTableName) val input = createSparkRelation( proto.DeltaRelation.newBuilder() .setMergeIntoTable( proto.MergeIntoTable.newBuilder() .setTarget(createSubqueryAlias(createScan(targetTableName), alias = "t")) .setSource(createSubqueryAlias(createScan(sourceTableName), alias = "s")) .setCondition(createExpression("t.key = s.key")) .addMatchedActions( proto.MergeIntoTable.Action.newBuilder() .setUpdateStarAction(proto.MergeIntoTable.Action.UpdateStarAction.newBuilder()) ) .addNotMatchedActions( proto.MergeIntoTable.Action.newBuilder() .setInsertStarAction(proto.MergeIntoTable.Action.InsertStarAction.newBuilder()) ) .addNotMatchedBySourceActions( proto.MergeIntoTable.Action.newBuilder() .setCondition(createExpression("t.value < 25")) .setDeleteAction(proto.MergeIntoTable.Action.DeleteAction.newBuilder()) ) ) ) val plan = transform(input) val result = Dataset.ofRows(spark, plan).collect() assert(result.length === 1) assert(result.head.getLong(0) === 125) // num_affected_rows assert(result.head.getLong(1) === 50) // num_updated_rows assert(result.head.getLong(2) === 25) // num_deleted_rows assert(result.head.getLong(3) === 50) // num_inserted_rows checkAnswer( spark.read.table(targetTableName), Seq.tabulate(25)(i => Row(25 + i, 25 + i)) ++ Seq.tabulate(50)(i => Row(i + 50, i + 1050)) ++ Seq.tabulate(50)(i => Row(i + 100, i + 1100)) ) } } test("clone - shallow") { withTempDir { targetDir => val sourceTableName = "source_table" withTable(sourceTableName) { spark.range(end = 1000).write.format("delta").saveAsTable(sourceTableName) transform(createSparkCommand( proto.DeltaCommand.newBuilder() .setCloneTable( proto.CloneTable.newBuilder() .setTable(proto.DeltaTable.newBuilder().setTableOrViewName(sourceTableName)) .setTarget(targetDir.getAbsolutePath) .setIsShallow(true) .setReplace(true) ) )) // Check that we have successfully cloned the table. checkAnswer( spark.read.format("delta").load(targetDir.getAbsolutePath), spark.read.table(sourceTableName)) // Check that a shallow clone was performed. val deltaLog = DeltaLog.forTable(spark, targetDir) deltaLog.update().allFiles.collect().foreach { f => assert(f.pathAsUri.isAbsolute) } } } } test("optimize - compaction") { val tableName = "test_table" withTable(tableName) { spark.range(1000).select(col("id") % 3 as "key", col("id") as "val") .write.format("delta").saveAsTable(tableName) val plan = transform(createSparkRelation( proto.DeltaRelation.newBuilder() .setOptimizeTable( proto.OptimizeTable.newBuilder() .setTable(proto.DeltaTable.newBuilder().setTableOrViewName(tableName)) ) )) assert(plan.columns.toSeq == Seq("path", "metrics")) val df = Dataset.ofRows(spark, plan) checkOptimizeMetrics(df) checkOptimizeUsingHistory(tableName, expectedPredicates = Nil, expectedZorderCols = Nil) } } test("optimize - compaction with partition filters") { val tableName = "test_table" withTable(tableName) { spark.range(1000).select(col("id") % 3 as "key", col("id") as "val") .write.partitionBy("key").format("delta").saveAsTable(tableName) val plan = transform(createSparkRelation( proto.DeltaRelation.newBuilder() .setOptimizeTable( proto.OptimizeTable.newBuilder() .setTable(proto.DeltaTable.newBuilder().setTableOrViewName(tableName)) .addPartitionFilters("key = 1") ) )) assert(plan.columns.toSeq == Seq("path", "metrics")) val df = Dataset.ofRows(spark, plan) checkOptimizeMetrics(df) checkOptimizeUsingHistory( tableName, expectedPredicates = Seq("'key = 1"), expectedZorderCols = Nil) } } test("optimize - z-order") { val tableName = "test_table" withTable(tableName) { spark.range(1000).select(col("id") % 3 as "key", col("id") as "val") .write.format("delta").saveAsTable(tableName) val plan = transform(createSparkRelation( proto.DeltaRelation.newBuilder() .setOptimizeTable( proto.OptimizeTable.newBuilder() .setTable(proto.DeltaTable.newBuilder().setTableOrViewName(tableName)) .addZorderColumns("val") ) )) assert(plan.columns.toSeq == Seq("path", "metrics")) val df = Dataset.ofRows(spark, plan) checkOptimizeMetrics(df) checkOptimizeUsingHistory( tableName, expectedPredicates = Nil, expectedZorderCols = Seq("val")) } } test("optimize - z-order with partition filters") { val tableName = "test_table" withTable(tableName) { spark.range(1000).select(col("id") % 3 as "key", col("id") as "val") .write.partitionBy("key").format("delta").saveAsTable(tableName) val plan = transform(createSparkRelation( proto.DeltaRelation.newBuilder() .setOptimizeTable( proto.OptimizeTable.newBuilder() .setTable(proto.DeltaTable.newBuilder().setTableOrViewName(tableName)) .addPartitionFilters("key = 1") .addZorderColumns("val") ) )) assert(plan.columns.toSeq == Seq("path", "metrics")) val df = Dataset.ofRows(spark, plan) checkOptimizeMetrics(df) checkOptimizeUsingHistory( tableName, expectedPredicates = Seq("'key = 1"), expectedZorderCols = Seq("val")) } } private def checkOptimizeMetrics(df: DataFrame): Unit = { import testImplicits._ val result = df.as[(String, OptimizeMetrics)].collect() assert(result.length == 1) val (_, metrics) = result.head assert(metrics.numFilesRemoved > metrics.numFilesAdded) } private def checkOptimizeUsingHistory( tableName: String, expectedPredicates: Seq[String], expectedZorderCols: Seq[String]): Unit = { import testImplicits._ val (operation, operationParameters) = DeltaTable.forName(spark, tableName).history() .select("operation", "operationParameters").as[(String, Map[String, String])].head() assert(operation == "OPTIMIZE") assert(operationParameters("predicate") == expectedPredicates.map(p => s"""\"($p)\"""").mkString(start = "[", sep = ",", end = "]")) assert(operationParameters("zOrderBy") == expectedZorderCols.map(c => s"""\"$c\"""").mkString(start = "[", sep = ",", end = "]")) } test("vacuum - without retention hours argument") { val tableName = "test_table" def runVacuum(): Unit = { transform(createSparkCommand( proto.DeltaCommand.newBuilder() .setVacuumTable( proto.VacuumTable.newBuilder() .setTable(proto.DeltaTable.newBuilder().setTableOrViewName(tableName)) ) )) } def setRetentionInterval(retentionInterval: String): Unit = { spark.sql( s"""ALTER TABLE $tableName |SET TBLPROPERTIES ( | '${DeltaConfigs.TOMBSTONE_RETENTION.key}' = '$retentionInterval', | '${DeltaConfigs.LOG_RETENTION.key}' = '$retentionInterval' |)""".stripMargin ) } withTable(tableName) { // Set up a Delta table with an untracked file. spark.range(1000).write.format("delta").saveAsTable(tableName) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val tempFile = new File(DeltaFileOperations.absolutePath(deltaLog.dataPath.toString, "abc.parquet").toUri) tempFile.createNewFile() // Run a vacuum with the untracked file still within the retention period. setRetentionInterval(retentionInterval = "1000 hours") runVacuum() assert(tempFile.exists()) // Run a vacuum with the untracked file outside of the retention period. setRetentionInterval(retentionInterval = "0 hours") runVacuum() assert(!tempFile.exists()) } } test("vacuum - with retention hours argument") { val tableName = "test_table" def runVacuum(retentionHours: Float): Unit = { transform(createSparkCommand( proto.DeltaCommand.newBuilder() .setVacuumTable( proto.VacuumTable.newBuilder() .setTable(proto.DeltaTable.newBuilder().setTableOrViewName(tableName)) .setRetentionHours(retentionHours) ) )) } withTable(tableName) { // Set up a Delta table with an untracked file. spark.range(1000).write.format("delta").saveAsTable(tableName) val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val tempFile = new File(DeltaFileOperations.absolutePath(deltaLog.dataPath.toString, "abc.parquet").toUri) tempFile.createNewFile() // Run a vacuum with the untracked file still within the retention period. runVacuum(retentionHours = 1000.0f) assert(tempFile.exists()) // Run a vacuum with the untracked file outside of the retention period. withSQLConf(DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED.key -> "false") { runVacuum(retentionHours = 0.0f) } assert(!tempFile.exists()) } } test("upgrade table protocol") { val tableName = "test_table" withTable(tableName) { // Create a Delta table with protocol version (1, 1). val oldReaderVersion = 1 val oldWriterVersion = 1 spark.range(1000) .write .format("delta") .option(DeltaConfigs.MIN_READER_VERSION.key, oldReaderVersion) .option(DeltaConfigs.MIN_WRITER_VERSION.key, oldWriterVersion) .saveAsTable(tableName) // Check that protocol version is as expected, before we upgrade it. val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val oldProtocol = deltaLog.update().protocol assert(oldProtocol.minReaderVersion === oldReaderVersion) assert(oldProtocol.minWriterVersion === oldWriterVersion) // Use Delta Connect to upgrade the protocol of the table. val newReaderVersion = 2 val newWriterVersion = 5 transform(createSparkCommand( proto.DeltaCommand.newBuilder() .setUpgradeTableProtocol( proto.UpgradeTableProtocol.newBuilder() .setTable(proto.DeltaTable.newBuilder().setTableOrViewName(tableName)) .setReaderVersion(newReaderVersion) .setWriterVersion(newWriterVersion) ) )) // Check that protocol version is as expected, after we have upgraded it. val newProtocol = deltaLog.update().protocol assert(newProtocol.minReaderVersion === newReaderVersion) assert(newProtocol.minWriterVersion === newWriterVersion) } } test("add table feature support") { val tableName = "test_table" withTable(tableName) { // Create a Delta table with protocol version (1, 1). val oldReaderVersion = 1 val oldWriterVersion = 1 spark.range(1000) .write .format("delta") .option(DeltaConfigs.MIN_READER_VERSION.key, oldReaderVersion) .option(DeltaConfigs.MIN_WRITER_VERSION.key, oldWriterVersion) .saveAsTable(tableName) // Check that protocol version is as expected, before we add the TestReaderWriterFeature. val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) val oldProtocol = deltaLog.update().protocol assert(oldProtocol.minReaderVersion === oldReaderVersion) assert(oldProtocol.minWriterVersion === oldWriterVersion) // Use Delta Connect to add table feature. transform(createSparkCommand( proto.DeltaCommand.newBuilder() .setAddFeatureSupport( proto.AddFeatureSupport.newBuilder() .setTable(proto.DeltaTable.newBuilder().setTableOrViewName(tableName)) .setFeatureName(TestReaderWriterFeature.name) ) )) // Check that protocol version is as expected, after we added the // TestReaderWriterFeature. val newProtocol = deltaLog.update().protocol assert(newProtocol.minReaderVersion === TestReaderWriterFeature.minReaderVersion) assert(newProtocol.minWriterVersion === TestReaderWriterFeature.minWriterVersion) } } test("drop table feature support") { val tableName = "test_table" withTable(tableName) { spark.range(1000) .write .format("delta") .option(DeltaConfigs.MIN_READER_VERSION.key, 1) .option(DeltaConfigs.MIN_WRITER_VERSION.key, 1) .saveAsTable(tableName) sql( s"""ALTER TABLE $tableName SET TBLPROPERTIES ( |delta.feature.${TestRemovableWriterFeature.name} = 'supported' |)""".stripMargin) // Check that protocol version is as expected. val deltaLog = DeltaLog.forTable(spark, TableIdentifier(tableName)) assert(deltaLog.update().protocol === Protocol( minReaderVersion = 1, minWriterVersion = 7, readerFeatures = None, writerFeatures = Some(Set(TestRemovableWriterFeature.name)))) // Attempt truncating the history when dropping a feature that is not required. // This verifies the truncateHistory option was correctly passed. assert(intercept[DeltaTableFeatureException] { transform(createSparkCommand( proto.DeltaCommand.newBuilder() .setDropFeatureSupport( proto.DropFeatureSupport.newBuilder() .setTable(proto.DeltaTable.newBuilder().setTableOrViewName(tableName)) .setFeatureName(TestRemovableWriterFeature.name) .setTruncateHistory(true) ) )) }.getErrorClass === "DELTA_FEATURE_DROP_HISTORY_TRUNCATION_NOT_ALLOWED") // Use Delta Connect to drop table feature. transform(createSparkCommand( proto.DeltaCommand.newBuilder() .setDropFeatureSupport( proto.DropFeatureSupport.newBuilder() .setTable(proto.DeltaTable.newBuilder().setTableOrViewName(tableName)) .setFeatureName(TestRemovableWriterFeature.name) ) )) assert(deltaLog.update().protocol === Protocol(1, 1)) } } test("generate manifest") { withTempDir { dir => spark.range(1000).write.format("delta").mode("overwrite").save(dir.getAbsolutePath) val manifestFile = new File(dir, GenerateSymlinkManifest.MANIFEST_LOCATION) assert(!manifestFile.exists()) transform(createSparkCommand( proto.DeltaCommand.newBuilder() .setGenerate( proto.Generate.newBuilder() .setTable( proto.DeltaTable.newBuilder() .setPath(proto.DeltaTable.Path.newBuilder().setPath(dir.getAbsolutePath))) .setMode("symlink_format_manifest")))) assert(manifestFile.exists()) } } test("create delta table - basic") { withTempDir { dir => val tableName = "test_table" withTable(tableName) { val tableComment = "table comment" val nameColA = "colA" val generatedAlwaysAsColA = "1" val nameColB = "colB" val commentColB = "colB comment" val nameColC = "colC" val tableProperties = Map("k1" -> "v1", "k2" -> "v2") transform(createSparkCommand( proto.DeltaCommand.newBuilder() .setCreateDeltaTable( proto.CreateDeltaTable.newBuilder() .setMode(proto.CreateDeltaTable.Mode.MODE_CREATE) .setTableName(tableName) .setLocation(dir.getAbsolutePath) .setComment(tableComment) .addColumns( proto.CreateDeltaTable.Column.newBuilder() .setName(nameColA) .setDataType(UNPARSED_INT_DATA_TYPE) .setNullable(true) .setGeneratedAlwaysAs(generatedAlwaysAsColA)) .addColumns( proto.CreateDeltaTable.Column.newBuilder() .setName(nameColB) .setDataType(LONG_DATA_TYPE) .setNullable(false) .setComment(commentColB)) .addColumns( proto.CreateDeltaTable.Column.newBuilder() .setName(nameColC) .setDataType(LONG_DATA_TYPE) .setNullable(false) .setIdentityInfo( proto.CreateDeltaTable.Column.IdentityInfo.newBuilder() .setStart(100) .setStep(10) .setAllowExplicitInsert(true) ) ) .putAllProperties(tableProperties.asJava)))) val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName)) val metadata = DeltaLog.forTable(spark, TableIdentifier(tableName)).snapshot.metadata assert(table.comment.contains(tableComment)) assert(table.location.getPath == dir.getAbsolutePath) assert(metadata.configuration == tableProperties) val colAMetadata = new MetadataBuilder() .putString(GENERATION_EXPRESSION_METADATA_KEY, generatedAlwaysAsColA) .build() val colBMetadata = new MetadataBuilder().putString("comment", commentColB).build() val colCMetadata = new MetadataBuilder() .putLong(DeltaSourceUtils.IDENTITY_INFO_START, 100) .putLong(DeltaSourceUtils.IDENTITY_INFO_STEP, 10) .putBoolean(DeltaSourceUtils.IDENTITY_INFO_ALLOW_EXPLICIT_INSERT, true) .build() val expectedSchema = StructType(Seq( StructField(nameColA, IntegerType, nullable = true, colAMetadata), StructField(nameColB, LongType, nullable = false, colBMetadata), StructField(nameColC, LongType, nullable = false, colCMetadata))) assert(metadata.schema == expectedSchema) } } } test("create delta table - modes") { val tableName = "test_table" def createTable(mode: proto.CreateDeltaTable.Mode, colName: String): Unit = { transform(createSparkCommand( proto.DeltaCommand.newBuilder() .setCreateDeltaTable( proto.CreateDeltaTable.newBuilder() .setMode(mode) .setTableName(tableName) .addColumns( proto.CreateDeltaTable.Column.newBuilder() .setName(colName) .setDataType(LONG_DATA_TYPE) .setNullable(true))))) } def getColumnName(): String = { DeltaLog.forTable(spark, TableIdentifier(tableName)).snapshot.metadata.schema.head.name } val expectedQualifiedTableName = s"`default`.`$tableName`" withTable(tableName) { val replaceError = intercept[AnalysisException] { createTable(proto.CreateDeltaTable.Mode.MODE_REPLACE, colName = "column1") } checkError( replaceError, condition = "TABLE_OR_VIEW_NOT_FOUND", parameters = Map("relationName" -> expectedQualifiedTableName)) createTable(proto.CreateDeltaTable.Mode.MODE_CREATE, colName = "column2") assert(getColumnName() == "column2") val createError = intercept[AnalysisException] { createTable(proto.CreateDeltaTable.Mode.MODE_CREATE, colName = "column3") } checkError( createError, condition = "TABLE_OR_VIEW_ALREADY_EXISTS", parameters = Map("relationName" -> s"`default`.`$tableName`")) createTable(proto.CreateDeltaTable.Mode.MODE_REPLACE, colName = "column4") assert(getColumnName() == "column4") createTable(proto.CreateDeltaTable.Mode.MODE_CREATE_IF_NOT_EXISTS, colName = "column5") assert(getColumnName() == "column4") spark.sql(s"DROP TABLE $tableName") createTable(proto.CreateDeltaTable.Mode.MODE_CREATE_IF_NOT_EXISTS, colName = "column6") assert(getColumnName() == "column6") createTable(proto.CreateDeltaTable.Mode.MODE_CREATE_OR_REPLACE, colName = "column7") assert(getColumnName() == "column7") spark.sql(s"DROP TABLE $tableName") createTable(proto.CreateDeltaTable.Mode.MODE_CREATE_OR_REPLACE, colName = "column8") assert(getColumnName() == "column8") } } private def getTimestampForVersion(log: DeltaLog, version: Long): String = { val file = new File(FileNames.unsafeDeltaFile(log.logPath, version).toUri) val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS") sdf.format(file.lastModified()) } } object DeltaConnectPlannerSuite { val UNPARSED_INT_DATA_TYPE: spark_proto.DataType = spark_proto.DataType .newBuilder() .setUnparsed(spark_proto.DataType.Unparsed.newBuilder().setDataTypeString("int")) .build() val LONG_DATA_TYPE: spark_proto.DataType = spark_proto.DataType .newBuilder() .setLong(spark_proto.DataType.Long.newBuilder()) .build() private def createSparkCommand(command: proto.DeltaCommand.Builder): spark_proto.Command = { spark_proto.Command.newBuilder().setExtension(protobuf.Any.pack(command.build())).build() } private def createSparkRelation(relation: proto.DeltaRelation.Builder): spark_proto.Relation = { spark_proto.Relation.newBuilder().setExtension(protobuf.Any.pack(relation.build())).build() } private def createScan(tableName: String): spark_proto.Relation = { createSparkRelation( proto.DeltaRelation.newBuilder() .setScan( proto.Scan.newBuilder() .setTable(proto.DeltaTable.newBuilder().setTableOrViewName(tableName)) ) ) } def createSubqueryAlias( input: spark_proto.Relation, alias: String): spark_proto.Relation = { spark_proto.Relation.newBuilder() .setSubqueryAlias( spark_proto.SubqueryAlias.newBuilder() .setAlias(alias) .setInput(input) ) .build() } private def createExpression(expr: String): spark_proto.Expression = { spark_proto.Expression.newBuilder() .setExpressionString( spark_proto.Expression.ExpressionString.newBuilder() .setExpression(expr) ) .build() } private def createAssignment(field: String, value: String): proto.Assignment = { proto.Assignment.newBuilder() .setField(createExpression(field)) .setValue(createExpression(value)) .build() } } ================================================ FILE: spark-unified/README.md ================================================ # Delta Spark Unified Module This module contains the final, published `delta-spark` JAR that unifies both: - **V1 (hybrid DSv1 and DSv2)**: The traditional Delta Lake connector using `DeltaLog` for Delta support - **V2 (Pure DSv2)**: The new Kernel-backed connector. ## Architecture The unified module provides single entry points for both V1 and V2: - `DeltaCatalog`: Extends `AbstractDeltaCatalog` from the `spark` module - `DeltaSparkSessionExtension`: Extends `AbstractDeltaSparkSessionExtension` from the `spark` module ## Module Structure ``` spark-unified/ (This module - final published artifact) ├── src/main/java/ │ └── org.apache.spark.sql.delta.catalog.DeltaCatalog.java └── src/main/scala/ └── io.delta.sql.DeltaSparkSessionExtension.scala spark/ (sparkV1 - V1 implementation) ├── Core Delta Lake classes with DeltaLog ├── AbstractDeltaCatalog, AbstractDeltaSparkSessionExtension └── v2/ (sparkV2 - V2 implementation) └── Kernel-backed DSv2 connector ``` ## How It Works 1. **sparkV1** (`spark/`): Contains production code for the V1 connector using DeltaLog 2. **sparkV1Filtered** (`spark-v1-shaded/`): Filtered version of V1 excluding DeltaLog, Snapshot, OptimisticTransaction, and actions.scala 3. **sparkV2** (`spark/v2/`): Kernel-backed V2 connector that depends on sparkV1Filtered 4. **spark** (this module): Final JAR that merges V1 + V2 classes The final JAR includes: - All classes from sparkV1, sparkV2, and storage - Python files - No internal module dependencies in the published POM ## Internal vs Published Modules **Internal modules** (not published to Maven): - `delta-spark-v1` - `delta-spark-v1-filtered` - `delta-spark-v2` **Published module**: - `delta-spark` (this module) - contains merged classes from all internal modules ## Build The module automatically: 1. Merges classes from V1, V2, and storage modules 2. Detects duplicate classes (fails build if found) 3. Filters internal modules from POM dependencies 4. Exports as JAR ## Testing Tests are located in `spark/src/test/` and run against the combined JAR to ensure V1+V2 integration works correctly. ================================================ FILE: spark-unified/src/main/java/org/apache/spark/sql/delta/catalog/DeltaCatalog.java ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.catalog; import io.delta.spark.internal.v2.catalog.SparkTable; import org.apache.spark.sql.delta.DeltaV2Mode; import java.util.HashMap; import java.util.function.Supplier; import org.apache.hadoop.fs.Path; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.catalyst.catalog.CatalogTable; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.Table; /** * A Spark catalog plugin for Delta Lake tables that implements the Spark DataSource V2 Catalog API. * * To use this catalog, configure it in your Spark session: * 
{@code
 * // Scala example
 * val spark = SparkSession
 *   .builder()
 *   .appName("...")
 *   .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")
 *   .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
 *   .getOrCreate()
 *
 *
 * // Python example
 * spark = SparkSession \
 *   .builder \
 *   .appName("...") \
 *   .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \
 *   .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \
 *   .getOrCreate()
 * }
* *

Architecture and Delegation Logic

* * This class sits in the delta-spark (unified) module and provides a single entry point for * Delta Lake catalog operations. * *

The unified module can access both implementations:

*
    *
  • V1 connector: {@link DeltaTableV2} - Legacy connector using DeltaLog, full read/write support
    • *
  • V2 connector: {@link SparkTable} - sparkV2 connector, read-only support
    • *
* *

See {@link DeltaV2Mode} for V1 vs V2 connector definitions and enable mode configuration.

*/ public class DeltaCatalog extends AbstractDeltaCatalog { /** * Loads a Delta table that is registered in the catalog. * *

Routing logic based on {@link DeltaV2Mode}: *

    *
  • STRICT: Returns sparkV2 {@link SparkTable} (V2 connector)
    • *
  • NONE (default): Returns {@link DeltaTableV2} (V1 connector)
    • *
* * @param ident The identifier of the table in the catalog. * @param catalogTable The catalog table metadata containing table properties and location. * @return Table instance (SparkTable for V2, DeltaTableV2 for V1). */ @Override public Table loadCatalogTable(Identifier ident, CatalogTable catalogTable) { return loadTableInternal( () -> new SparkTable(ident, catalogTable, new HashMap<>()), () -> super.loadCatalogTable(ident, catalogTable)); } /** * Loads a Delta table directly from a path. * This is used for path-based table access where the identifier name is the table path. * *

Routing logic based on {@link DeltaV2Mode}: *

    *
  • STRICT: Returns sparkV2 {@link SparkTable} (V2 connector)
    • *
  • NONE (default): Returns {@link DeltaTableV2} (V1 connector)
    • *
* * @param ident The identifier whose name contains the path to the Delta table. * @return Table instance (SparkTable for V2, DeltaTableV2 for V1). */ @Override public Table loadPathTable(Identifier ident) { return loadTableInternal( // delta.`/path/to/table`, where ident.name() is `/path/to/table` () -> new SparkTable(ident, ident.name()), () -> super.loadPathTable(ident)); } /** * Loads a table based on the {@link DeltaV2Mode} SQL configuration. * *

This method checks the configuration and delegates to the appropriate supplier: *

    *
  • STRICT mode: Uses V2 connector (sparkV2 SparkTable) - for testing V2 capabilities
    • *
  • NONE mode (default): Uses V1 connector (DeltaTableV2) - production default with full features
    • *
* *

See {@link DeltaV2Mode} for detailed V1 vs V2 connector definitions. * * @param v2ConnectorSupplier Supplier for V2 connector (sparkV2 SparkTable) - used in STRICT mode * @param v1ConnectorSupplier Supplier for V1 connector (DeltaTableV2) - used in NONE mode (default) * @return Table instance from the selected supplier */ private Table loadTableInternal( Supplier

v2ConnectorSupplier, Supplier
v1ConnectorSupplier) { DeltaV2Mode connectorMode = new DeltaV2Mode(spark().sessionState().conf()); if (connectorMode.shouldCatalogReturnV2Tables()) { return v2ConnectorSupplier.get(); } else { return v1ConnectorSupplier.get(); } } } ================================================ FILE: spark-unified/src/main/scala/io/delta/internal/ApplyV2Streaming.scala ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.internal import scala.jdk.CollectionConverters._ import scala.jdk.OptionConverters._ import io.delta.spark.internal.v2.catalog.SparkTable import io.delta.spark.internal.v2.utils.ScalaUtils import org.apache.spark.sql.delta.DeltaV2Mode import org.apache.spark.sql.delta.sources.DeltaSourceUtils import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.streaming.StreamingRelationV2 import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.delta.Relocated.StreamingRelation import org.apache.spark.sql.util.CaseInsensitiveStringMap /** * Rule for applying the V2 streaming path by rewriting V1 StreamingRelation * with Delta DataSource to StreamingRelationV2 with SparkTable. * * This rule handles the case where Spark's FindDataSourceTable rule has converted * a StreamingRelationV2 (with DeltaTableV2) back to a StreamingRelation because * DeltaTableV2 doesn't advertise STREAMING_READ capability. We convert it back to * StreamingRelationV2 with SparkTable (from sparkV2) which does support streaming. * * See [[DeltaV2Mode]] for configuration behavior. * * @param session The Spark session for configuration access */ class ApplyV2Streaming( @transient private val session: SparkSession) extends Rule[LogicalPlan] { private def isDeltaStreamingRelation(s: StreamingRelation): Boolean = { // Check if this is a Delta streaming relation by examining: // 1. The source name (e.g., "delta" from .format("delta")) // 2. The catalog table's provider (e.g., "DELTA" from Unity Catalog) s.dataSource.catalogTable match { case Some(catalogTable) => DeltaSourceUtils.isDeltaDataSourceName(s.sourceName) || catalogTable.provider.exists(DeltaSourceUtils.isDeltaDataSourceName) case None => false } } private def shouldApplyV2Streaming(s: StreamingRelation): Boolean = { if (!isDeltaStreamingRelation(s)) { return false } val deltaV2Mode = new DeltaV2Mode(session.sessionState.conf) deltaV2Mode.isStreamingReadsEnabled(s.dataSource.catalogTable.toJava) } override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperators { case s: StreamingRelation if shouldApplyV2Streaming(s) => // catalogTable is guaranteed to be defined because shouldApplyV2Streaming checks it // via isDeltaStreamingRelation. val catalogTable = s.dataSource.catalogTable.get val ident = Identifier.of(catalogTable.identifier.database.toArray, catalogTable.identifier.table) val table = new SparkTable( ident, catalogTable, // Use user-specified streaming options to override catalog storage properties. // SparkTable handles merging catalogTable storage props internally. ScalaUtils.toJavaMap(s.dataSource.options)) val catalog = catalogTable.identifier.catalog.map( session.sessionState.catalogManager.catalog) StreamingRelationV2( // We rebuild this from the resolved V2 table, so there is no V1 source to carry through. // This is only non-None when StreamingRelationV2 is created by wrapping a V1 streaming // data source; in that case Spark keeps the underlying V1 DataSource instance here. source = None, sourceName = DeltaSourceUtils.NAME, table = table, extraOptions = new CaseInsensitiveStringMap(s.dataSource.options.asJava), output = toAttributes(table.schema), catalog = catalog, identifier = Some(ident), // Keep this None to force the V2 path; we don't want to fall back to V1 here. v1Relation = None) } } ================================================ FILE: spark-unified/src/main/scala/io/delta/sql/DeltaSparkSessionExtension.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.sql import io.delta.internal.ApplyV2Streaming import org.apache.spark.sql.SparkSessionExtensions import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.rules.Rule /** * An extension for Spark SQL to activate Delta SQL parser to support Delta SQL grammar. * * Scala example to create a `SparkSession` with the Delta SQL parser: * {{{ * import org.apache.spark.sql.SparkSession * * val spark = SparkSession * .builder() * .appName("...") * .master("...") * .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") * .getOrCreate() * }}} * * Java example to create a `SparkSession` with the Delta SQL parser: * {{{ * import org.apache.spark.sql.SparkSession; * * SparkSession spark = SparkSession * .builder() * .appName("...") * .master("...") * .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") * .getOrCreate(); * }}} * * Python example to create a `SparkSession` with the Delta SQL parser (PySpark doesn't pick up the * SQL conf "spark.sql.extensions" in Apache Spark 2.4.x, hence we need to activate it manually in * 2.4.x. However, because `SparkSession` has been created and everything has been materialized, we * need to clone a new session to trigger the initialization. See SPARK-25003): * {{{ * from pyspark.sql import SparkSession * * spark = SparkSession \ * .builder \ * .appName("...") \ * .master("...") \ * .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ * .getOrCreate() * if spark.sparkContext().version < "3.": * spark.sparkContext()._jvm.io.delta.sql.DeltaSparkSessionExtension() \ * .apply(spark._jsparkSession.extensions()) * spark = SparkSession(spark.sparkContext(), spark._jsparkSession.cloneSession()) * }}} * * @since 0.4.0 */ class DeltaSparkSessionExtension extends AbstractDeltaSparkSessionExtension { override def apply(extensions: SparkSessionExtensions): Unit = { // First register all the base Delta rules from the V1 implementation. super.apply(extensions) // Register a post-hoc resolution rule that rewrites V1 StreamingRelation plans that // read catalog owned Delta tables into V2 StreamingRelationV2 plans backed by SparkTable. // // NOTE: This rule is functional (not a placeholder). Binary compatibility concerns are // handled separately via the nested NoOpRule class below (kept for MiMa). extensions.injectResolutionRule { session => new ApplyV2Streaming(session) } } /** * NoOpRule for binary compatibility with Delta 3.3.0 * This class must remain here to satisfy MiMa checks */ class NoOpRule extends Rule[LogicalPlan] { override def apply(plan: LogicalPlan): LogicalPlan = plan } } ================================================ FILE: spark-unified/src/test/scala/io/delta/internal/ApplyV2StreamingSuite.scala ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.internal import java.net.URI import java.util.{HashMap => JHashMap} import scala.jdk.CollectionConverters._ import io.delta.spark.internal.v2.catalog.SparkTable import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTableType} import org.apache.spark.sql.catalyst.streaming.StreamingRelationV2 import org.apache.spark.sql.delta.Relocated.StreamingRelation import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation import org.apache.spark.sql.execution.datasources.DataSource import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.types.StructType import org.apache.spark.sql.util.CaseInsensitiveStringMap class ApplyV2StreamingSuite extends DeltaSQLCommandTest { private def applyRule(plan: LogicalPlan): LogicalPlan = { new ApplyV2Streaming(spark).apply(plan) } private def assertV2(result: LogicalPlan): Unit = { result match { case StreamingRelationV2(_, _, _: SparkTable, _, _, _, _, v1Relation) => assert(v1Relation.isEmpty) case other => fail(s"Expected StreamingRelationV2, got $other") } } private def assertV1(result: LogicalPlan): Unit = { assert(result.isInstanceOf[StreamingRelation]) assert(!result.isInstanceOf[StreamingRelationV2]) } private def createDeltaTable(path: String): Unit = { spark.range(1).selectExpr("id").write.format("delta").save(path) } private def createCatalogTable(locationUri: URI, ucManaged: Boolean): CatalogTable = { val storageProps = new JHashMap[String, String]() if (ucManaged) { storageProps.put(UCCommitCoordinatorClient.UC_TABLE_ID_KEY, "uc-table-id") storageProps.put("delta.feature.catalogManaged", "supported") } val identifier = TableIdentifier("tbl", Some("default"), Some("spark_catalog")) val storage = CatalogStorageFormat( locationUri = Some(locationUri), inputFormat = None, outputFormat = None, serde = None, compressed = false, properties = storageProps.asScala.toMap) CatalogTable( identifier = identifier, tableType = CatalogTableType.MANAGED, storage = storage, schema = new StructType(), provider = None, partitionColumnNames = Seq.empty, bucketSpec = None, properties = Map.empty) } private val relationBuilders: Seq[(String, (CatalogTable, String) => LogicalPlan)] = Seq( ("streaming", (catalogTable, path) => { val dataSource = DataSource( sparkSession = spark, userSpecifiedSchema = None, className = "delta", options = Map("path" -> path), catalogTable = Some(catalogTable)) StreamingRelation(dataSource) }), ("non-streaming", (catalogTable, _) => { val ident = Identifier.of( catalogTable.identifier.database.toArray, catalogTable.identifier.table) val table = new SparkTable(ident, catalogTable, new JHashMap[String, String]()) DataSourceV2Relation.create( table, None, None, CaseInsensitiveStringMap.empty) }) ) private val modes: Seq[(String, Boolean)] = Seq( ("STRICT", true), ("AUTO", false), ("NONE", false) ) modes.foreach { case (mode, expectV2) => relationBuilders.foreach { case (relationType, buildPlan) => test(s"ApplyV2Streaming respects $mode mode for $relationType relation") { withTempDir { dir => val path = dir.getCanonicalPath createDeltaTable(path) val catalogTable = createCatalogTable(dir.toURI, ucManaged = false) val plan = buildPlan(catalogTable, path) withSQLConf(DeltaSQLConf.V2_ENABLE_MODE.key -> mode) { val result = applyRule(plan) if (relationType == "streaming") { if (expectV2) { assertV2(result) } else { assertV1(result) } } else { assert(result == plan) } } } } } } } ================================================ FILE: spark-unified/src/test/scala/org/apache/spark/sql/delta/DataFrameWriterV2WithV2ConnectorSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.test.V2ForceTest /** * Test suite that runs OpenSourceDataFrameWriterV2Tests with Delta V2 connector * mode forced to STRICT. */ class DataFrameWriterV2WithV2ConnectorSuite extends OpenSourceDataFrameWriterV2Tests with V2ForceTest { /** * Tests that we expect to fail because they require write operations after initial * table creation. * * Kernel's SparkTable (V2 connector) only implements SupportsRead, not SupportsWrite. * Tests that perform append/replace operations after table creation are expected to fail. */ override protected def shouldFail(testName: String): Boolean = { val shouldFailTests = Set( // Append operations - require SupportsWrite "Append: basic append", "Append: by name not position", // Overwrite operations - require SupportsWrite "Overwrite: overwrite by expression: true", "Overwrite: overwrite by expression: id = 3", "Overwrite: by name not position", // OverwritePartitions operations - require SupportsWrite "OverwritePartitions: overwrite conflicting partitions", "OverwritePartitions: overwrite all rows if not partitioned", "OverwritePartitions: by name not position", // Create operations - TODO: fix SparkTable's name() to match DeltaTableV2 // SparkTable.name() returns simple table name, but tests expect catalog.schema.table format "Create: basic behavior", "Create: with using", "Create: with property", "Create: identity partitioned table", "Create: fail if table already exists", // Replace operations - require SupportsWrite "Replace: basic behavior", "Replace: partitioned table", // CreateOrReplace operations - require SupportsWrite "CreateOrReplace: table does not exist", "CreateOrReplace: table exists" ) shouldFailTests.contains(testName) } } ================================================ FILE: spark-unified/src/test/scala/org/apache/spark/sql/delta/ProtocolMetadataAdapterV2Suite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta import io.delta.kernel.internal.actions.{Format, Metadata, Protocol} import io.delta.kernel.internal.util.VectorUtils import io.delta.spark.internal.v2.read.ProtocolMetadataAdapterV2 import io.delta.spark.internal.v2.utils.SchemaUtils import io.delta.kernel.types.{ArrayType, StringType => KernelStringType} import org.apache.spark.sql.types.{IntegerType, StructType} import org.scalactic.source.Position import org.scalatest.Tag import java.util.Optional import scala.jdk.CollectionConverters._ /** * Unit tests for ProtocolMetadataAdapterV2. * * This suite tests the V2 wrapper implementation that adapts kernel's Protocol and Metadata * to the ProtocolMetadataAdapter interface. */ class ProtocolMetadataAdapterV2Suite extends ProtocolMetadataAdapterSuiteBase { /** * Tests that are not applicable to V2 (kernel-based) implementation. * These tests can be ignored because V2 has different behavior or limitations. */ protected def ignoredTests: Set[String] = Set( // TODO(delta-io/delta#5649): add type widening validation "assertTableReadable with table with unsupported type widening", // V1 IcebergCompat is not supported in Kernel (only V2/V3) "isIcebergCompatAnyEnabled when v1 enabled", "isIcebergCompatGeqEnabled when v1 enabled" ) override protected def test( testName: String, testTags: Tag*)(testFun: => Any)(implicit pos: Position): Unit = { if (ignoredTests.contains(testName)) { super.ignore(s"$testName - not applicable to V2 implementation")(testFun) } else { super.test(testName, testTags: _*)(testFun) } } override protected def createWrapper( minReaderVersion: Int = 1, minWriterVersion: Int = 2, readerFeatures: Option[Set[String]] = None, writerFeatures: Option[Set[String]] = None, schema: StructType = new StructType().add("id", IntegerType), configuration: Map[String, String] = Map.empty): ProtocolMetadataAdapter = { // Create kernel Protocol val protocol = new Protocol( minReaderVersion, minWriterVersion, readerFeatures.map(_.asJava).getOrElse(java.util.Collections.emptySet()), writerFeatures.map(_.asJava).getOrElse(java.util.Collections.emptySet()) ) // Convert Spark schema to Kernel schema val kernelSchema = SchemaUtils.convertSparkSchemaToKernelSchema(schema) val schemaString = kernelSchema.toJson // Create kernel Metadata val metadata = new Metadata( "test-id", Optional.of("test-table"), Optional.of("test description"), new Format("parquet", java.util.Collections.emptyMap()), schemaString, kernelSchema, VectorUtils.buildArrayValue( java.util.Collections.emptyList(), new ArrayType(KernelStringType.STRING, true)), Optional.of(System.currentTimeMillis()), VectorUtils.stringStringMapValue(configuration.asJava) ) // Create and return the V2 adapter new ProtocolMetadataAdapterV2(protocol, metadata) } } ================================================ FILE: spark-unified/src/test/scala/org/apache/spark/sql/delta/catalog/DeltaCatalogSuite.scala ================================================ /* * Copyright (2025) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.catalog import io.delta.spark.internal.v2.catalog.SparkTable import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import java.io.File import java.util.Locale /** * Unit tests for DeltaCatalog's V2 connector routing logic. * * Verifies that DeltaCatalog correctly routes table loading based on * DeltaSQLConf.V2_ENABLE_MODE: * - STRICT mode: Kernel's SparkTable (V2 connector) * - NONE mode (default): DeltaTableV2 (V1 connector) */ class DeltaCatalogSuite extends DeltaSQLCommandTest { private val modeTestCases = Seq( ("STRICT", classOf[SparkTable], "Kernel SparkTable"), ("NONE", classOf[DeltaTableV2], "DeltaTableV2") ) modeTestCases.foreach { case (mode, expectedClass, description) => test(s"catalog-based table with mode=$mode returns $description") { withTempDir { tempDir => val tableName = s"test_catalog_${mode.toLowerCase(Locale.ROOT)}" val location = new File(tempDir, tableName).getAbsolutePath withSQLConf(DeltaSQLConf.V2_ENABLE_MODE.key -> mode) { sql(s"CREATE TABLE $tableName (id INT, name STRING) USING delta LOCATION '$location'") val catalog = spark.sessionState.catalogManager.v2SessionCatalog .asInstanceOf[DeltaCatalog] val ident = org.apache.spark.sql.connector.catalog.Identifier .of(Array("default"), tableName) val table = catalog.loadTable(ident) assert(table.getClass == expectedClass, s"Mode $mode should return ${expectedClass.getSimpleName}") } } } } modeTestCases.foreach { case (mode, expectedClass, description) => test(s"path-based table with mode=$mode returns $description") { withTempDir { tempDir => val path = tempDir.getAbsolutePath withSQLConf(DeltaSQLConf.V2_ENABLE_MODE.key -> mode) { sql(s"CREATE TABLE delta.`$path` (id INT, name STRING) USING delta") val catalog = spark.sessionState.catalogManager.v2SessionCatalog .asInstanceOf[DeltaCatalog] val ident = org.apache.spark.sql.connector.catalog.Identifier .of(Array("delta"), path) val table = catalog.loadTable(ident) assert(table.getClass == expectedClass, s"Mode $mode should return ${expectedClass.getSimpleName} for path-based table") } } } } } ================================================ FILE: spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceDeletionVectorsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test import org.apache.spark.sql.delta.DeltaSourceDeletionVectorsSuite import org.apache.spark.sql.delta.sources.DeltaSQLConf /** * Test suite that runs DeltaSourceDeletionVectorsSuite using the V2 connector * (V2_ENABLE_MODE=STRICT). */ class DeltaV2SourceDeletionVectorsSuite extends DeltaSourceDeletionVectorsSuite with V2ForceTest { override protected def useDsv2: Boolean = true /** * Override executeDml to temporarily use V1 connector for DML operations. * SparkTable (V2) is read-only and does not support writes, so DML must * go through the V1 path. Only streaming reads use the V2 connector. */ override protected def executeDml(sqlText: String): Unit = { withSQLConf(DeltaSQLConf.V2_ENABLE_MODE.key -> "NONE") { sql(sqlText) } } private lazy val shouldPassTests = Set( "allow to delete files before starting a streaming query", "allow to delete files before staring a streaming query without checkpoint", "multiple deletion vectors per file with initial snapshot", "deleting files fails query if ignoreDeletes = false", "allow to delete files after staring a streaming query when ignoreDeletes is true", "updating the source table causes failure when ignoreChanges = false - using DELETE", "updating source table when ignoreDeletes = true fails the query - using DELETE", "subsequent DML commands are processed correctly in a batch - DELETE->DELETE - List()", "subsequent DML commands are processed correctly in a batch - DELETE->DELETE" + " - List((ignoreDeletes,true))", "subsequent DML commands are processed correctly in a batch - DELETE->DELETE" + " - List((skipChangeCommits,true))", "subsequent DML commands are processed correctly in a batch - INSERT->DELETE - List()", "subsequent DML commands are processed correctly in a batch - INSERT->DELETE" + " - List((ignoreDeletes,true))", "subsequent DML commands are processed correctly in a batch - INSERT->DELETE" + " - List((skipChangeCommits,true))" ) private lazy val shouldFailTests = Set( // TODO(#5319): enable these tests after ignoreChanges/ignoreFileDeletion read options // are supported by the V2 connector. "allow to delete files after staring a streaming query when ignoreFileDeletion is true", "allow to update the source table when ignoreChanges = true - using DELETE", "deleting files when ignoreChanges = true doesn't fail the query", "subsequent DML commands are processed correctly in a batch - DELETE->DELETE" + " - List((ignoreChanges,true))", "subsequent DML commands are processed correctly in a batch - INSERT->DELETE" + " - List((ignoreChanges,true))", "multiple deletion vectors per file - List((ignoreFileDeletion,true))", "multiple deletion vectors per file - List((ignoreChanges,true))" ) override protected def shouldFail(testName: String): Boolean = { val inPassList = shouldPassTests.contains(testName) val inFailList = shouldFailTests.contains(testName) assert(inPassList || inFailList, s"Test '$testName' not in shouldPassTests or shouldFailTests") assert(!(inPassList && inFailList), s"Test '$testName' in both shouldPassTests and shouldFailTests") inFailList } } ================================================ FILE: spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.delta.DeltaConfigs import org.apache.spark.sql.delta.DeltaOperations import org.apache.spark.sql.delta.DeltaSourceSuite /** * Test suite that runs DeltaSourceSuite using the V2 connector (V2_ENABLE_MODE=STRICT). */ class DeltaV2SourceSuite extends DeltaSourceSuite with V2ForceTest { override protected def useDsv2: Boolean = true /** * Override disableLogCleanup to use DeltaLog API instead of SQL ALTER TABLE. * Path-based ALTER TABLE doesn't work properly with V2_ENABLE_MODE=STRICT. * TODO(#5731): pending kernel v2 connector support. */ override protected def disableLogCleanup(tablePath: String): Unit = { val deltaLog = DeltaLog.forTable(spark, tablePath) val metadata = deltaLog.snapshot.metadata val newConfiguration = metadata.configuration ++ Map( DeltaConfigs.ENABLE_EXPIRED_LOG_CLEANUP.key -> "false" ) deltaLog.startTransaction().commit( metadata.copy(configuration = newConfiguration) :: Nil, DeltaOperations.SetTableProperties( Map(DeltaConfigs.ENABLE_EXPIRED_LOG_CLEANUP.key -> "false")) ) } private lazy val shouldPassTests = Set( // ========== Core streaming tests ========== "basic", "initial snapshot ends at base index of next version", "new commits arrive after stream initialization - with explicit startingVersion", "SC-11561: can consume new data without update", "Delta sources don't write offsets with null json", // === Schema Evolution === "add column: restarting with new DataFrame should recover", "add column: restarting with stale DataFrame should fail", "relax nullability: restarting with new DataFrame should recover", "type widening: restarting with new DataFrame should recover", "disallow to change schema after starting a streaming query", "allow to change schema before starting a streaming query", "drop column: should fail with non-additive schema change error", "drop column: should succeed with unsafe column mapping schema change flag enabled", "rename column: should fail with non-additive schema change error", "rename column: should throw schema change error with unsafe flag enabled", "type widening: should fail with non-additive schema change error when enable schema tracking", // === Read options === "excludeRegex works and doesn't mess up offsets across restarts - parquet version", "streaming with ignoreDeletes = true skips delete-only commits", "streaming with ignoreDeletes = true still fails on change commits", "streaming with skipChangeCommits = true skips both delete and change commits", // ========== startingVersion option tests ========== "startingVersion", "startingVersion latest", "startingVersion latest defined before started", "startingVersion latest works on defined but empty table", "startingVersion specific version: new commits arrive after stream initialization", "startingVersion: user defined start works with mergeSchema", "startingVersion latest calls update when starting", "startingVersion should be ignored when restarting from a checkpoint, withRowTracking = true", "startingVersion should be ignored when restarting from a checkpoint, withRowTracking = false", "startingVersion and startingTimestamp are both set", "startingTimestamp", // ========== Rate limiting tests ========== "maxFilesPerTrigger", "maxBytesPerTrigger: process at least one file", "maxFilesPerTrigger: change and restart", "maxFilesPerTrigger: invalid parameter", "maxFilesPerTrigger: ignored when using Trigger.Once", "maxFilesPerTrigger: Trigger.AvailableNow respects read limits", "maxBytesPerTrigger: change and restart", "maxBytesPerTrigger: invalid parameter", "maxBytesPerTrigger: Trigger.AvailableNow respects read limits", "maxBytesPerTrigger: max bytes and max files together", "Trigger.AvailableNow with an empty table", "Rate limited Delta source advances with non-data inserts", "ES-445863: delta source should not hang or reprocess data when using AvailableNow", "startingVersion should work with rate time", "maxFilesPerTrigger: metadata checkpoint", "maxBytesPerTrigger: metadata checkpoint", // ========== Error handling tests ========== "streaming query should fail when table is deleted and recreated with new id", "deltaSourceIgnoreDeleteError contains removeFile, version, tablePath", "deltaSourceIgnoreChangesError contains removeFile, version, tablePath", "excludeRegex throws good error on bad regex pattern", // ========== Misc tests ========== "a fast writer should not starve a Delta source", "should not attempt to read a non exist version", "can delete old files of a snapshot without update", "Delta source advances with non-data inserts and generates empty dataframe for " + "non-data operations" ) private lazy val shouldFailTests = Set( // === Null Type Column Handling === "streaming delta source should not drop null columns", "streaming delta source should drop null columns without feature flag", // === Schema Evolution === // TODO(#6232): enable the two tests after spark streaming engine supports leaf node projection // for datasource v2 such that we can adopt the two schema changes without refreshing the // dataframe "relax nullability: restarting with stale DataFrame should recover", "type widening: restarting with stale DataFrame should recover", // === Data Loss Detection === "fail on data loss - starting from missing files", "fail on data loss - gaps of files", "fail on data loss - starting from missing files with option off", "fail on data loss - gaps of files with option off", // === Misc === // TODO(#5900): fix exception mismatch "no schema should throw an exception", // TODO(#5900): fix exception mismatch "Delta sources should verify the protocol reader version", // TODO(#5895): gracefully handle corrupt checkpoint "start from corrupt checkpoint", // === Tests that bypass V2 by not using loadStreamWithOptions === "disallow user specified schema", // Uses .schema() directly "make sure that the delta sources works fine", // Uses .delta() directly "self union a Delta table should pass the catalog table assert", // Uses .table() directly "handling nullability schema changes", // Uses .table() directly "allow user specified schema if consistent: v1 source", // Uses DataSource directly // Calls deltaSource.createSource() directly "createSource should create source with empty or matching table schema provided" ) override protected def shouldFail(testName: String): Boolean = { val inPassList = shouldPassTests.contains(testName) val inFailList = shouldFailTests.contains(testName) assert(inPassList || inFailList, s"Test '$testName' not in shouldPassTests or shouldFailTests") assert(!(inPassList && inFailList), s"Test '$testName' in both shouldPassTests and shouldFailTests") inFailList } } ================================================ FILE: spark-unified/src/test/scala/org/apache/spark/sql/delta/test/V2ForceTest.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.sql.delta.test import org.apache.spark.SparkConf import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.scalatest.Tag import org.scalactic.source.Position import scala.collection.mutable /** * Trait that forces Delta V2 connector mode to STRICT, ensuring all operations * use the Kernel-based SparkTable implementation (V2 connector) instead of * DeltaTableV2 (V1 connector). * * See [[DeltaSQLConf.V2_ENABLE_MODE]] for V1 vs V2 connector definitions. * * Usage: * {{{ * class MyKernelTest extends MyOriginalSuite with V2ForceTest { * override protected def shouldSkipTest(testName: String): Boolean = { * testName.contains("unsupported feature") * } * } * }}} */ trait V2ForceTest extends DeltaSQLCommandTest { private val testsRun: mutable.Set[String] = mutable.Set.empty /** * Override `test` to apply the `shouldFail` logic. * Tests that are expected to fail are converted to ignored tests. */ abstract override protected def test( testName: String, testTags: Tag*)(testFun: => Any)(implicit pos: Position): Unit = { if (shouldFail(testName)) { // TODO(#5754): Assert on test failure instead of ignoring super.ignore( s"$testName - expected to fail with Kernel-based V2 connector (not yet supported)")(testFun) } else { super.test(testName, testTags: _*) { testsRun.add(testName) testFun } } } /** * Determine if a test is expected to fail based on the test name. * Subclasses should override this method to define which tests are expected to fail. * By default, no tests are expected to fail. * * @param testName The name of the test * @return true if the test is expected to fail, false otherwise */ protected def shouldFail(testName: String): Boolean = false /** * Override `sparkConf` to set V2_ENABLE_MODE to "STRICT". * This ensures all catalog operations use Kernel SparkTable (V2 connector). */ abstract override protected def sparkConf: SparkConf = { super.sparkConf .set(DeltaSQLConf.V2_ENABLE_MODE.key, "STRICT") } override def afterAll(): Unit = { super.afterAll() } } ================================================ FILE: storage/src/main/java/io/delta/storage/AzureLogStore.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage; import java.io.IOException; import java.util.Iterator; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; /** * LogStore implementation for Azure. *

* We assume the following from Azure's [[FileSystem]] implementations: *

    *
  • Rename without overwrite is atomic.
    • *
  • List-after-write is consistent.
    • *
*

* Regarding file creation, this implementation: *

    *
  • Uses atomic rename when overwrite is false; if the destination file exists or the rename * fails, throws an exception.
    • *
  • Uses create-with-overwrite when overwrite is true. This does not make the file atomically * visible and therefore the caller must handle partial files.
    • *
*/ public class AzureLogStore extends HadoopFileSystemLogStore { public AzureLogStore(Configuration hadoopConf) { super(hadoopConf); } @Override public void write( Path path, Iterator actions, Boolean overwrite, Configuration hadoopConf) throws IOException { writeWithRename(path, actions, overwrite, hadoopConf); } @Override public Boolean isPartialWriteVisible(Path path, Configuration hadoopConf) { return true; } } ================================================ FILE: storage/src/main/java/io/delta/storage/CloseableIterator.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage; import java.io.Closeable; import java.util.Iterator; /** * :: DeveloperApi :: * * An iterator that may contain resources which should be released after use. Users of * CloseableIterator are responsible for closing the iterator if they are done with it. * * @since 1.0.0 */ public interface CloseableIterator extends Iterator, Closeable {} ================================================ FILE: storage/src/main/java/io/delta/storage/GCSLogStore.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage; import com.google.common.base.Throwables; import io.delta.storage.internal.ThreadUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; import java.io.IOException; import java.io.InterruptedIOException; import java.nio.charset.StandardCharsets; import java.nio.file.FileAlreadyExistsException; import java.util.Iterator; import java.util.concurrent.Callable; /** * The {@link LogStore} implementation for GCS, which uses gcs-connector to * provide the necessary atomic and durability guarantees: * *
    *
  1. Atomic Visibility: Read/read-after-metadata-update/delete are strongly * consistent for GCS.
    2. * *
  2. Consistent Listing: GCS guarantees strong consistency for both object and * bucket listing operations. * https://cloud.google.com/storage/docs/consistency
    3. * *
  3. Mutual Exclusion: Preconditions are used to handle race conditions.
    4. *
* * Regarding file creation, this implementation: *
    *
  • Opens a stream to write to GCS otherwise.
    • *
  • Throws [[FileAlreadyExistsException]] if file exists and overwrite is false.
    • *
  • Assumes file writing to be all-or-nothing, irrespective of overwrite option.
    • *
*

* This class is not meant for direct access but for configuration based on storage system. * See https://docs.delta.io/latest/delta-storage.html for details. */ public class GCSLogStore extends HadoopFileSystemLogStore { final String preconditionFailedExceptionMessage = "412 Precondition Failed"; public GCSLogStore(Configuration hadoopConf) { super(hadoopConf); } @Override public void write( Path path, Iterator actions, Boolean overwrite, Configuration hadoopConf) throws IOException { final FileSystem fs = path.getFileSystem(hadoopConf); // This is needed for the tests to throw error with local file system. if (fs instanceof LocalFileSystem && !overwrite && fs.exists(path)) { throw new FileAlreadyExistsException(path.toString()); } // GCS may upload an incomplete file when the current thread is interrupted, hence we move // the write to a new thread so that the write cannot be interrupted. // TODO Remove this hack when the GCS Hadoop connector fixes the issue. // If overwrite=false and path already exists, gcs-connector will throw // org.apache.hadoop.fs.FileAlreadyExistsException after fs.create is invoked. // This should be mapped to java.nio.file.FileAlreadyExistsException. Callable body = () -> { FSDataOutputStream stream = fs.create(path, overwrite); while (actions.hasNext()) { stream.write((actions.next() + "\n").getBytes(StandardCharsets.UTF_8)); } stream.close(); return ""; }; try { ThreadUtils.runInNewThread("delta-gcs-logstore-write", true, body); } catch (org.apache.hadoop.fs.FileAlreadyExistsException e) { throw new FileAlreadyExistsException(path.toString()); } catch (IOException e) { // GCS uses preconditions to handle race conditions for multiple writers. // If path gets created between fs.create and stream.close by an external // agent or race conditions. Then this block will execute. // Reference: https://cloud.google.com/storage/docs/generations-preconditions if (isPreconditionFailure(e)) { if (!overwrite) { throw new FileAlreadyExistsException(path.toString()); } } else { throw e; } } catch (InterruptedException e) { InterruptedIOException iio = new InterruptedIOException(e.getMessage()); iio.initCause(e); throw iio; } catch (Error | RuntimeException t) { throw t; } catch (Throwable t) { // Throw RuntimeException to avoid the calling interfaces from throwing Throwable throw new RuntimeException(t.getMessage(), t); } } private boolean isPreconditionFailure(Throwable x) { return Throwables.getCausalChain(x) .stream() .filter(p -> p != null) .filter(p -> p.getMessage() != null) .anyMatch(p -> p.getMessage().contains(preconditionFailedExceptionMessage)); } @Override public Boolean isPartialWriteVisible(Path path, Configuration hadoopConf) throws IOException { return false; } } ================================================ FILE: storage/src/main/java/io/delta/storage/HDFSLogStore.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage; import java.io.IOException; import java.io.InterruptedIOException; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.nio.file.FileAlreadyExistsException; import java.util.EnumSet; import java.util.Iterator; import io.delta.storage.internal.LogStoreErrors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.CreateFlag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * The {@link LogStore} implementation for HDFS, which uses Hadoop {@link FileContext} API's to * provide the necessary atomic and durability guarantees: *

    *
  1. Atomic visibility of files: `FileContext.rename` is used write files which is atomic for * HDFS.
    2. *
  2. Consistent file listing: HDFS file listing is consistent.
    3. *
*/ public class HDFSLogStore extends HadoopFileSystemLogStore { private static final Logger LOG = LoggerFactory.getLogger(HDFSLogStore.class); public static final String NO_ABSTRACT_FILE_SYSTEM_EXCEPTION_MESSAGE = "No AbstractFileSystem"; public HDFSLogStore(Configuration hadoopConf) { super(hadoopConf); } @Override public void write( Path path, Iterator actions, Boolean overwrite, Configuration hadoopConf) throws IOException { final boolean isLocalFs = path.getFileSystem(hadoopConf) instanceof RawLocalFileSystem; if (isLocalFs) { // We need to add `synchronized` for RawLocalFileSystem as its rename will not throw an // exception when the target file exists. Hence we must make sure `exists + rename` in // `writeInternal` for RawLocalFileSystem is atomic in our tests. synchronized(this) { writeInternal(path, actions, overwrite, hadoopConf); } } else { // rename is atomic and also will fail when the target file exists. Not need to add the // extra `synchronized`. writeInternal(path, actions, overwrite, hadoopConf); } } @Override public Boolean isPartialWriteVisible(Path path, Configuration hadoopConf) { return true; } /** * @throws IOException if this HDFSLogStore is used to write into a Delta table on a non-HDFS * storage system. * @throws FileAlreadyExistsException if {@code overwrite} is false and the file at {@code path} * already exists. */ private void writeInternal( Path path, Iterator actions, Boolean overwrite, Configuration hadoopConf) throws IOException { final FileContext fc; try { fc = FileContext.getFileContext(path.toUri(), hadoopConf); } catch (IOException e) { if (e.getMessage().contains(NO_ABSTRACT_FILE_SYSTEM_EXCEPTION_MESSAGE)) { final IOException newException = LogStoreErrors.incorrectLogStoreImplementationException(e); LOG.error(newException.getMessage(), newException.getCause()); throw newException; } else { throw e; } } if (!overwrite && fc.util().exists(path)) { // This is needed for the tests to throw error with local file system throw new FileAlreadyExistsException(path.toString()); } final Path tempPath = createTempPath(path); boolean streamClosed = false; // This flag is to avoid double close boolean renameDone = false; // This flag is to save the delete operation in most cases. final FSDataOutputStream stream = fc.create( tempPath, EnumSet.of(CreateFlag.CREATE), Options.CreateOpts.checksumParam(Options.ChecksumOpt.createDisabled()) ); try { while (actions.hasNext()) { stream.write((actions.next() + "\n").getBytes(StandardCharsets.UTF_8)); } stream.close(); streamClosed = true; try { final Options.Rename renameOpt = overwrite ? Options.Rename.OVERWRITE : Options.Rename.NONE; fc.rename(tempPath, path, renameOpt); renameDone = true; // TODO: this is a workaround of HADOOP-16255 - remove this when HADOOP-16255 is // resolved tryRemoveCrcFile(fc, tempPath); } catch (org.apache.hadoop.fs.FileAlreadyExistsException e) { throw new FileAlreadyExistsException(path.toString()); } } finally { if (!streamClosed) { stream.close(); } if (!renameDone) { fc.delete(tempPath, false); // recursive=false } } msyncIfSupported(path, hadoopConf); } /** * Normally when using HDFS with an Observer NameNode setup, there would be read after write * consistency within a single process, so the write would be guaranteed to be visible on the * next read. However, since we are using the FileContext API for writing (for atomic rename), * and the FileSystem API for reading (for more compatibility with various file systems), we * are essentially using two separate clients that are not guaranteed to be kept in sync. * Therefore we "msync" the FileSystem instance, which is cached across all uses of the same * protocol/host combination, to make sure the next read through the HDFSLogStore can see this * write. * Any underlying FileSystem that is not the DistributedFileSystem will simply throw an * UnsupportedOperationException, which can be ignored. Additionally, if an older version of * Hadoop is being used that does not include msync, a NoSuchMethodError will be thrown while * looking up the method, which can also be safely ignored. */ private void msyncIfSupported(Path path, Configuration hadoopConf) throws IOException { try { FileSystem fs = path.getFileSystem(hadoopConf); Method msync = fs.getClass().getMethod("msync"); msync.invoke(fs); } catch (InterruptedIOException e) { throw e; } catch (Throwable e) { if (e instanceof InterruptedException) { // Propagate the interrupt status Thread.currentThread().interrupt(); } // We ignore non fatal errors as calling msync is best effort. } } /** * @throws IOException if a fatal exception occurs. Will try to ignore most exceptions. */ private void tryRemoveCrcFile(FileContext fc, Path path) throws IOException { try { final Path checksumFile = new Path(path.getParent(), String.format(".%s.crc", path.getName())); if (fc.util().exists(checksumFile)) { // checksum file exists, deleting it fc.delete(checksumFile, true); // recursive=true } } catch (Throwable e) { if (!LogStoreErrors.isNonFatal(e)) { throw e; } // else, ignore - we are removing crc file as "best-effort" } } } ================================================ FILE: storage/src/main/java/io/delta/storage/HadoopFileSystemLogStore.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage; import java.io.*; import java.nio.charset.StandardCharsets; import java.nio.file.FileAlreadyExistsException; import java.util.Arrays; import java.util.Comparator; import java.util.Iterator; import java.util.UUID; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; /** * Default implementation of {@link LogStore} for Hadoop {@link FileSystem} implementations. */ public abstract class HadoopFileSystemLogStore extends LogStore { public HadoopFileSystemLogStore(Configuration hadoopConf) { super(hadoopConf); } @Override public CloseableIterator read(Path path, Configuration hadoopConf) throws IOException { FileSystem fs = path.getFileSystem(hadoopConf); FSDataInputStream stream = fs.open(path); Reader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)); return new LineCloseableIterator(reader); } @Override public Iterator listFrom(Path path, Configuration hadoopConf) throws IOException { FileSystem fs = path.getFileSystem(hadoopConf); if (!fs.exists(path.getParent())) { throw new FileNotFoundException( String.format("No such file or directory: %s", path.getParent()) ); } FileStatus[] files = fs.listStatus(path.getParent()); return Arrays.stream(files) .filter(f -> f.getPath().getName().compareTo(path.getName()) >= 0) .sorted(Comparator.comparing(o -> o.getPath().getName())) .iterator(); } @Override public Path resolvePathOnPhysicalStorage( Path path, Configuration hadoopConf) throws IOException { return path.getFileSystem(hadoopConf).makeQualified(path); } /** * An internal write implementation that uses FileSystem.rename(). *

* This implementation should only be used for the underlying file systems that support atomic * renames, e.g., Azure is OK but HDFS is not. */ protected void writeWithRename( Path path, Iterator actions, Boolean overwrite, Configuration hadoopConf) throws IOException { FileSystem fs = path.getFileSystem(hadoopConf); if (!fs.exists(path.getParent())) { throw new FileNotFoundException( String.format("No such file or directory: %s", path.getParent()) ); } if (overwrite) { final FSDataOutputStream stream = fs.create(path, true); try { while (actions.hasNext()) { stream.write((actions.next() + "\n").getBytes(StandardCharsets.UTF_8)); } } finally { stream.close(); } } else { if (fs.exists(path)) { throw new FileAlreadyExistsException(path.toString()); } Path tempPath = createTempPath(path); boolean streamClosed = false; // This flag is to avoid double close boolean renameDone = false; // This flag is to save the delete operation in most cases final FSDataOutputStream stream = fs.create(tempPath); try { while (actions.hasNext()) { stream.write((actions.next() + "\n").getBytes(StandardCharsets.UTF_8)); } stream.close(); streamClosed = true; try { if (fs.rename(tempPath, path)) { renameDone = true; } else { if (fs.exists(path)) { throw new FileAlreadyExistsException(path.toString()); } else { throw new IllegalStateException( String.format("Cannot rename %s to %s", tempPath, path) ); } } } catch (org.apache.hadoop.fs.FileAlreadyExistsException e) { throw new FileAlreadyExistsException(path.toString()); } } finally { if (!streamClosed) { stream.close(); } if (!renameDone) { fs.delete(tempPath, false); } } } } /** * Create a temporary path (to be used as a copy) for the input {@code path} */ protected Path createTempPath(Path path) { return new Path( path.getParent(), String.format(".%s.%s.tmp", path.getName(), UUID.randomUUID()) ); } } ================================================ FILE: storage/src/main/java/io/delta/storage/LineCloseableIterator.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage; import java.io.BufferedReader; import java.io.IOException; import java.io.Reader; import java.io.UncheckedIOException; import java.util.NoSuchElementException; /** * Turn a {@link Reader} to {@link CloseableIterator} which can be read on demand. Each element is * a trimmed line. */ public class LineCloseableIterator implements CloseableIterator { private final BufferedReader reader; // Whether `nextValue` is valid. If it's invalid, we should try to read the next line. private boolean gotNext = false; // The next value to return when `next` is called. This is valid only if `getNext` is true. private String nextValue = null; // Whether the reader is closed. private boolean closed = false; // Whether we have consumed all data in the reader. private boolean finished = false; public LineCloseableIterator(Reader reader) { this.reader = reader instanceof BufferedReader ? (BufferedReader) reader : new BufferedReader(reader); } @Override public boolean hasNext() { try { if (!finished) { // Check whether we have closed the reader before reading. Even if `nextValue` is // valid, we still don't return `nextValue` after a reader is closed. Otherwise, it // would be confusing. if (closed) { throw new IllegalStateException("Iterator is closed"); } if (!gotNext) { String nextLine = reader.readLine(); if (nextLine == null) { finished = true; close(); } else { nextValue = nextLine.trim(); } gotNext = true; } } return !finished; } catch (IOException e) { throw new UncheckedIOException(e); } } @Override public String next() { if (!hasNext()) { throw new NoSuchElementException("End of stream"); } gotNext = false; return nextValue; } @Override public void close() throws IOException { if (!closed) { closed = true; reader.close(); } } } ================================================ FILE: storage/src/main/java/io/delta/storage/LocalLogStore.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RawLocalFileSystem; import java.io.IOException; import java.util.Iterator; /** * Default {@link LogStore} implementation (should be used for testing only!). * * Production users should specify the appropriate {@link LogStore} implementation in Spark properties.

* * We assume the following from {@link FileSystem} implementations: *

    *
  • Rename without overwrite is atomic.
    • *
  • List-after-write is consistent.
    • *
* Regarding file creation, this implementation: *
    *
  • Uses atomic rename when overwrite is false; if the destination file exists or the rename * fails, throws an exception.
    • *
  • Uses create-with-overwrite when overwrite is true. This does not make the file atomically * visible and therefore the caller must handle partial files.
    • *
*/ public class LocalLogStore extends HadoopFileSystemLogStore{ public LocalLogStore(Configuration hadoopConf) { super(hadoopConf); } /** * This write implementation needs to wrap `writeWithRename` with `synchronized` as rename() * for {@link RawLocalFileSystem} doesn't throw an exception when the target file * exists. Hence, we must make sure `exists + rename` in `writeWithRename` is atomic in our tests. */ @Override public void write( Path path, Iterator actions, Boolean overwrite, Configuration hadoopConf) throws IOException { synchronized(this) { writeWithRename(path, actions, overwrite, hadoopConf); } } @Override public Boolean isPartialWriteVisible(Path path, Configuration hadoopConf) throws IOException { return true; } } ================================================ FILE: storage/src/main/java/io/delta/storage/LogStore.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import java.io.IOException; import java.nio.file.FileAlreadyExistsException; import java.util.Iterator; /** * :: DeveloperApi :: * *

* General interface for all critical file system operations required to read and write the * Delta logs. The correctness is predicated on the atomicity and durability guarantees of * the implementation of this interface. Specifically, *

*
    *
  1. Atomic visibility of files: If isPartialWriteVisible is false, any file written through * this store must be made visible atomically. In other words, this should not generate * partial files.
    2. * *
  2. Mutual exclusion: Only one writer must be able to create (or rename) a file at the final * destination.
    3. * *
  3. Consistent listing: Once a file has been written in a directory, all future listings for * that directory must return that file.
    4. *
*

* All subclasses of this interface is required to have a constructor that takes Configuration * as a single parameter. This constructor is used to dynamically create the LogStore. *

*

* LogStore and its implementations are not meant for direct access but for configuration based * on storage system. See [[https://docs.delta.io/latest/delta-storage.html]] for details. *

* * @since 1.0.0 */ public abstract class LogStore { private Configuration initHadoopConf; public LogStore(Configuration initHadoopConf) { this.initHadoopConf = initHadoopConf; } /** * :: DeveloperApi :: * * Hadoop configuration that should only be used during initialization of LogStore. Each method * should use their `hadoopConf` parameter rather than this (potentially outdated) hadoop * configuration. */ public Configuration initHadoopConf() { return initHadoopConf; } /** * :: DeveloperApi :: * * Load the given file and return an `Iterator` of lines, with line breaks removed from each line. * Callers of this function are responsible to close the iterator if they are done with it. * * @throws IOException if there's an issue resolving the FileSystem * @since 1.0.0 */ public abstract CloseableIterator read( Path path, Configuration hadoopConf) throws IOException; /** * :: DeveloperApi :: * * Write the given `actions` to the given `path` with or without overwrite as indicated. * Implementation must throw {@link java.nio.file.FileAlreadyExistsException} exception if the * file already exists and overwrite = false. Furthermore, if isPartialWriteVisible returns false, * implementation must ensure that the entire file is made visible atomically, that is, * it should not generate partial files. * * @throws IOException if there's an issue resolving the FileSystem * @throws FileAlreadyExistsException if the file already exists and overwrite is false * @since 1.0.0 */ public abstract void write( Path path, Iterator actions, Boolean overwrite, Configuration hadoopConf) throws IOException; /** * :: DeveloperApi :: * * List the paths in the same directory that are lexicographically greater or equal to * (UTF-8 sorting) the given `path`. The result should also be sorted by the file name. * * @throws IOException if there's an issue resolving the FileSystem * @throws FileAlreadyExistsException if {@code path} directory can't be found * @since 1.0.0 */ public abstract Iterator listFrom( Path path, Configuration hadoopConf) throws IOException; /** * :: DeveloperApi :: * * Resolve the fully qualified path for the given `path`. * * @throws IOException if there's an issue resolving the FileSystem * @since 1.0.0 */ public abstract Path resolvePathOnPhysicalStorage( Path path, Configuration hadoopConf) throws IOException; /** * :: DeveloperApi :: * * Whether a partial write is visible for the underlying file system of `path`. * * @throws IOException if there's an issue resolving the FileSystem * @since 1.0.0 */ public abstract Boolean isPartialWriteVisible( Path path, Configuration hadoopConf) throws IOException; } ================================================ FILE: storage/src/main/java/io/delta/storage/S3SingleDriverLogStore.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InterruptedIOException; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.*; import com.google.common.io.CountingOutputStream; import io.delta.storage.internal.PathLock; import io.delta.storage.internal.S3LogStoreUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RawLocalFileSystem; /** * Single JVM LogStore implementation for S3. *

* We assume the following from S3's {@link FileSystem} implementations: *

    *
  • File writing on S3 is all-or-nothing, whether overwrite or not.
    • *
  • List-after-write is strongly consistent.
    • *
*

* Regarding file creation, this implementation: *

    *
  • Opens a stream to write to S3 (regardless of the overwrite option).
    • *
  • Failures during stream write may leak resources, but may never result in partial * writes.
    • *
*/ public class S3SingleDriverLogStore extends HadoopFileSystemLogStore { /** * Enables a faster implementation of listFrom by setting the startAfter parameter in S3 list * requests. The feature is enabled by setting the property delta.enableFastS3AListFrom in the * Hadoop configuration. * * This feature requires the Hadoop file system used for S3 paths to be castable to * org.apache.hadoop.fs.s3a.S3AFileSystem. */ private final boolean enableFastListFrom = initHadoopConf().getBoolean("delta.enableFastS3AListFrom", false); /////////////////////////// // Static Helper Methods // /////////////////////////// /** * A global path lock to ensure that no concurrent writers writing to the same path in the same * JVM. */ private static final PathLock pathLock = new PathLock(); ///////////////////////////////////////////// // Constructor and Instance Helper Methods // ///////////////////////////////////////////// public S3SingleDriverLogStore(Configuration hadoopConf) { super(hadoopConf); } private Path resolvePath(FileSystem fs, Path path) { return stripUserInfo(fs.makeQualified(path)); } private Path stripUserInfo(Path path) { final URI uri = path.toUri(); try { final URI newUri = new URI( uri.getScheme(), null, // userInfo uri.getHost(), uri.getPort(), uri.getPath(), uri.getQuery(), uri.getFragment() ); return new Path(newUri); } catch (URISyntaxException e) { // Propagating this URISyntaxException to callers would mean we would have to either // include it in the public LogStore.java interface or wrap it in an // IllegalArgumentException somewhere else. Instead, catch and wrap it here. throw new IllegalArgumentException(e); } } /** * List files starting from `resolvedPath` (inclusive) in the same directory, which merges * the file system list and the cache list when `useCache` is on, otherwise * use file system list only. */ private Iterator listFromInternal( FileSystem fs, Path resolvedPath) throws IOException { final Path parentPath = resolvedPath.getParent(); if (!fs.exists(parentPath)) { throw new FileNotFoundException( String.format("No such file or directory: %s", parentPath) ); } FileStatus[] statuses; if ( // LocalFileSystem and RawLocalFileSystem checks are needed for tests to pass fs instanceof LocalFileSystem || fs instanceof RawLocalFileSystem || !enableFastListFrom ) { statuses = fs.listStatus(parentPath); } else { statuses = S3LogStoreUtil.s3ListFromArray(fs, resolvedPath, parentPath); } return Arrays .stream(statuses) .filter(s -> s.getPath().getName().compareTo(resolvedPath.getName()) >= 0) .sorted(Comparator.comparing(a -> a.getPath().getName())) .iterator(); } //////////////////////// // Public API Methods // //////////////////////// @Override public void write( Path path, Iterator actions, Boolean overwrite, Configuration hadoopConf) throws IOException { final FileSystem fs = path.getFileSystem(hadoopConf); final Path resolvedPath = resolvePath(fs, path); try { pathLock.acquire(resolvedPath); try { if (fs.exists(resolvedPath) && !overwrite) { throw new java.nio.file.FileAlreadyExistsException( resolvedPath.toUri().toString() ); } final CountingOutputStream stream = new CountingOutputStream(fs.create(resolvedPath, overwrite)); while (actions.hasNext()) { stream.write((actions.next() + "\n").getBytes(StandardCharsets.UTF_8)); } stream.close(); } catch (org.apache.hadoop.fs.FileAlreadyExistsException e) { // Convert Hadoop's FileAlreadyExistsException to Java's FileAlreadyExistsException throw new java.nio.file.FileAlreadyExistsException(e.getMessage()); } } catch (java.lang.InterruptedException e) { throw new InterruptedIOException(e.getMessage()); } finally { pathLock.release(resolvedPath); } } @Override public Iterator listFrom(Path path, Configuration hadoopConf) throws IOException { final FileSystem fs = path.getFileSystem(hadoopConf); final Path resolvedPath = resolvePath(fs, path); return listFromInternal(fs, resolvedPath); } @Override public Boolean isPartialWriteVisible(Path path, Configuration hadoopConf) { return false; } } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/Commit.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit; import org.apache.hadoop.fs.FileStatus; import java.util.Objects; /** * Representation of a commit file */ public class Commit { private long version; private FileStatus fileStatus; private long commitTimestamp; public Commit(long version, FileStatus fileStatus, long commitTimestamp) { this.version = version; this.fileStatus = fileStatus; this.commitTimestamp = commitTimestamp; } public long getVersion() { return version; } public FileStatus getFileStatus() { return fileStatus; } public long getCommitTimestamp() { return commitTimestamp; } public Commit withFileStatus(FileStatus fileStatus) { return new Commit(version, fileStatus, commitTimestamp); } public Commit withCommitTimestamp(long commitTimestamp) { return new Commit(version, fileStatus, commitTimestamp); } @Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null || getClass() != obj.getClass()) return false; Commit commit = (Commit) obj; if (version != commit.version) return false; if (commitTimestamp != commit.commitTimestamp) return false; return Objects.equals(fileStatus, commit.fileStatus); } @Override public int hashCode() { return Objects.hash(version, fileStatus, commitTimestamp); } } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/CommitCoordinatorClient.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit; import java.io.IOException; import java.util.Iterator; import java.util.Map; import java.util.Optional; import io.delta.storage.commit.actions.AbstractMetadata; import io.delta.storage.commit.actions.AbstractProtocol; import io.delta.storage.LogStore; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; /** * The CommitCoordinatorClient is responsible for communicating with the commit coordinator * and backfilling commits. It has four main APIs that need to be implemented * *
    *
  • Commit a new version of the table. See {@link #commit}.
    • *
  • Ensure that commits are backfilled if/when needed. See {@link #backfillToVersion}
    • *
  • Tracks and returns unbackfilled commits. See {@link #getCommits}.
    • *
  • Determine the table config during commit coordinator registration. * See {@link #registerTable}
    • *
*/ public interface CommitCoordinatorClient { /** * API to register the table represented by the given `logPath` at the provided * currentTableVersion with the commit coordinator this commit coordinator client represents. * * This API is called when the table is being converted from a file system table to a * coordinated-commit table. * * When a new coordinated-commit table is being created, the currentTableVersion will be -1 and * the upgrade commit needs to be a file system commit which will write the backfilled file * directly. * * @param logPath The path to the delta log of the table that should be converted * @param tableIdentifier The optional tableIdentifier for the table. Some commit coordinators may * choose to make this compulsory and error out if this is not provided. * @param currentVersion The currentTableVersion is the version of the table just before * conversion. currentTableVersion + 1 represents the commit that * will do the conversion. This must be backfilled atomically. * currentTableVersion + 2 represents the first commit after conversion. * This will go through the CommitCoordinatorClient and the client is * free to choose when it wants to backfill this commit. * @param currentMetadata The metadata of the table at currentTableVersion * @param currentProtocol The protocol of the table at currentTableVersion * @return A map of key-value pairs which is issued by the commit coordinator to identify the * table. This should be stored in the table's metadata. This information needs to be * passed to the {@link #commit}, {@link #getCommits}, and {@link #backfillToVersion} * APIs to identify the table. */ Map registerTable( Path logPath, Optional tableIdentifier, long currentVersion, AbstractMetadata currentMetadata, AbstractProtocol currentProtocol); /** * API to commit the given set of actions to the table represented by logPath at the * given commitVersion. * * @param logStore The log store to use for writing the commit file. * @param hadoopConf The Hadoop configuration required to access the file system. * @param tableDescriptor The descriptor for the table. * @param commitVersion The version of the commit that is being committed. * @param actions The actions that need to be committed. * @param updatedActions The commit info and any metadata or protocol changes that are made * as part of this commit. * @return CommitResponse which contains the file status of the committed commit file. If the * commit is already backfilled, then the file status could be omitted from the response * and the client could retrieve the information by itself. * @throws CommitFailedException if the commit operation fails. */ CommitResponse commit( LogStore logStore, Configuration hadoopConf, TableDescriptor tableDescriptor, long commitVersion, Iterator actions, UpdatedActions updatedActions) throws CommitFailedException; /** * API to get the unbackfilled commits for the table represented by the given logPath. * Commits older than startVersion or newer than endVersion (if given) are ignored. The * returned commits are contiguous and in ascending version order. * * Note that the first version returned by this API may not be equal to startVersion. This * happens when some versions starting from startVersion have already been backfilled and so * the commit coordinator may have stopped tracking them. * * The returned latestTableVersion is the maximum commit version ratified by the commit * coordinator. Note that returning latestTableVersion as -1 is acceptable only if the commit * coordinator never ratified any version, i.e. it never accepted any unbackfilled commit. * * @param tableDescriptor The descriptor for the table. * @param startVersion The minimum version of the commit that should be returned. Can be null. * @param endVersion The maximum version of the commit that should be returned. Can be null. * @return GetCommitsResponse which has a list of {@link Commit}s and the latestTableVersion which * is tracked by {@link CommitCoordinatorClient}. */ GetCommitsResponse getCommits( TableDescriptor tableDescriptor, Long startVersion, Long endVersion); /** * API to ask the commit coordinator client to backfill all commits up to {@code version} * and notify the commit coordinator. * * If this API returns successfully, that means the backfill must have been completed, although * the commit coordinator may not be aware of it yet. * * @param logStore The log store to use for writing the backfilled commits. * @param hadoopConf The Hadoop configuration required to access the file system. * @param tableDescriptor The descriptor for the table. * @param version The version till which the commit coordinator client should * backfill. * @param lastKnownBackfilledVersion The last known version that was backfilled before this API * was called. If it is None or invalid, then the commit * coordinator client should backfill from the beginning of * the table. Can be null. * @throws IOException if there is an IO error while backfilling the commits. */ void backfillToVersion( LogStore logStore, Configuration hadoopConf, TableDescriptor tableDescriptor, long version, Long lastKnownBackfilledVersion) throws IOException; /** * Determines whether this CommitCoordinatorClient is semantically equal to another * CommitCoordinatorClient. * * Semantic equality is determined by each CommitCoordinatorClient implementation based on * whether the two instances can be used interchangeably when invoking any of the * CommitCoordinatorClient APIs, such as {@link #commit}, {@link #getCommits}, etc. For example, * both instances might be pointing to the same underlying endpoint. */ boolean semanticEquals(CommitCoordinatorClient other); } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/CommitFailedException.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit; import java.util.Iterator; import java.util.Map; import io.delta.storage.LogStore; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; /** * Exception raised by * {@link CommitCoordinatorClient#commit(LogStore, Configuration, TableDescriptor, long, Iterator, UpdatedActions)} * * 
 *  | retryable | conflict  | meaning                                                         |
 *  |   no      |   no      | something bad happened (e.g. auth failure)                      |
 *  |   no      |   yes     | permanent transaction conflict (e.g. multi-table commit failed) |
 *  |   yes     |   no      | transient error (e.g. network hiccup)                           |
 *  |   yes     |   yes     | physical conflict (allowed to rebase and retry)                 |
 *
*/ public class CommitFailedException extends Exception { private boolean retryable; private boolean conflict; public CommitFailedException(boolean retryable, boolean conflict, String message) { super(message); this.retryable = retryable; this.conflict = conflict; } public CommitFailedException(boolean retryable, boolean conflict, String message, Throwable cause) { super(message, cause); this.retryable = retryable; this.conflict = conflict; } public boolean getRetryable() { return retryable; } public boolean getConflict() { return conflict; } } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/CommitResponse.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit; import java.util.Iterator; import java.util.Map; import io.delta.storage.LogStore; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; /** * Response container for * {@link CommitCoordinatorClient#commit(LogStore, Configuration, TableDescriptor, long, Iterator, UpdatedActions)}. */ public class CommitResponse { private Commit commit; public CommitResponse(Commit commit) { this.commit = commit; } public Commit getCommit() { return commit; } } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/CoordinatedCommitsUtils.java ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import io.delta.storage.LogStore; import io.delta.storage.commit.actions.AbstractMetadata; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import java.io.IOException; import java.util.Iterator; import java.util.Map; import java.util.Optional; import java.util.UUID; public class CoordinatedCommitsUtils { private CoordinatedCommitsUtils() {} /** The subdirectory in which to store the delta log. */ private static final String LOG_DIR_NAME = "_delta_log"; /** The subdirectory in which to store the unbackfilled commit files. */ private static final String COMMIT_SUBDIR = "_staged_commits"; /** The configuration key for the coordinated commits owner name. */ private static final String COORDINATED_COMMITS_COORDINATOR_NAME_KEY = "delta.coordinatedCommits.commitCoordinator-preview"; /** The configuration key for the coordinated commits owner conf. */ private static final String COORDINATED_COMMITS_COORDINATOR_CONF_KEY = "delta.coordinatedCommits.commitCoordinatorConf-preview"; /** The configuration key for the coordinated commits table conf. */ private static final String COORDINATED_COMMITS_TABLE_CONF_KEY = "delta.coordinatedCommits.tableConf-preview"; /** * Creates a new unbackfilled delta file path for the given commit version. * The path is of the form: * `tablePath/_delta_log/_staged_commits/00000000000000000001.uuid.json`. */ public static Path generateUnbackfilledDeltaFilePath( Path logPath, long version) { String uuid = UUID.randomUUID().toString(); Path basePath = new Path(logPath, COMMIT_SUBDIR); return new Path(basePath, String.format("%020d.%s.json", version, uuid)); } /** * Returns the path to the backfilled delta file for the given commit version. * The path is of the form `tablePath/_delta_log/00000000000000000001.json`. */ public static Path getBackfilledDeltaFilePath( Path logPath, Long version) { return new Path(logPath, String.format("%020d.json", version)); } /** * Returns true if the commit is a coordinated commits to filesystem conversion. */ public static boolean isCoordinatedCommitsToFSConversion( Long commitVersion, UpdatedActions updatedActions) { boolean oldMetadataHasCoordinatedCommits = getCoordinatorName(updatedActions.getOldMetadata()).isPresent(); boolean newMetadataHasCoordinatedCommits = getCoordinatorName(updatedActions.getNewMetadata()).isPresent(); return oldMetadataHasCoordinatedCommits && !newMetadataHasCoordinatedCommits && commitVersion > 0; } /** * Get the table path from the provided log path. */ public static Path getTablePath(Path logPath) { return logPath.getParent(); } /** * Returns the un-backfilled uuid formatted delta (json format) path for a given version. * * @param logPath The root path of the delta log. * @param version The version of the delta file. * @return The path to the un-backfilled delta file: logPath/_staged_commits/version.uuid.json */ public static Path getUnbackfilledDeltaFile( Path logPath, long version, Optional uuidString) { Path basePath = commitDirPath(logPath); String uuid = uuidString.orElse(UUID.randomUUID().toString()); return new Path(basePath, String.format("%020d.%s.json", version, uuid)); } /** * Write a UUID-based commit file for the specified version to the table at logPath. */ public static FileStatus writeUnbackfilledCommitFile( LogStore logStore, Configuration hadoopConf, String logPath, long commitVersion, Iterator actions, String uuid) throws IOException { Path commitPath = new Path(getUnbackfilledDeltaFile( new Path(logPath), commitVersion, Optional.of(uuid)).toString()); // Do not use Put-If-Absent for Unbackfilled Commits files since we assume that UUID-based // commit files are globally unique, and so we will never have concurrent writers attempting // to write the same commit file. logStore.write(commitPath, actions, true /* overwrite */, hadoopConf); return commitPath.getFileSystem(hadoopConf).getFileStatus(commitPath); } /** Returns path to the directory which holds the delta log */ public static Path logDirPath(Path tablePath) { return new Path(tablePath, LOG_DIR_NAME); } /** Returns path to the directory which holds the unbackfilled commits */ public static Path commitDirPath(Path logPath) { return new Path(logPath, COMMIT_SUBDIR); } /** * Retrieves the coordinator name from the provided abstract metadata. * If no coordinator is set, an empty optional is returned. * * @param metadata The abstract metadata from which to retrieve the coordinator name. * @return The coordinator name if set, otherwise an empty optional. */ public static Optional getCoordinatorName(AbstractMetadata metadata) { String coordinator = metadata .getConfiguration() .get(COORDINATED_COMMITS_COORDINATOR_NAME_KEY); return Optional.ofNullable(coordinator); } private static Map parseConfFromMetadata( AbstractMetadata abstractMetadata, String confKey) { String conf = abstractMetadata .getConfiguration() .getOrDefault(confKey, "{}"); try { return new ObjectMapper().readValue( conf, new TypeReference>() {}); } catch (JsonProcessingException e) { throw new RuntimeException("Failed to parse conf: ", e); } } /** * Get the coordinated commits owner configuration from the provided abstract metadata. */ public static Map getCoordinatorConf(AbstractMetadata abstractMetadata) { return parseConfFromMetadata(abstractMetadata, COORDINATED_COMMITS_COORDINATOR_CONF_KEY); } /** * Get the coordinated commits table configuration from the provided abstract metadata. */ public static Map getTableConf(AbstractMetadata abstractMetadata) { return parseConfFromMetadata(abstractMetadata, COORDINATED_COMMITS_TABLE_CONF_KEY); } } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/GetCommitsResponse.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit; import java.util.List; import java.util.Map; import org.apache.hadoop.fs.Path; /** * Response container for * {@link CommitCoordinatorClient#getCommits(TableDescriptor, Long, Long)}. */ public class GetCommitsResponse { private List commits; private long latestTableVersion; public GetCommitsResponse(List commits, long latestTableVersion) { this.commits = commits; this.latestTableVersion = latestTableVersion; } public List getCommits() { return commits; } public long getLatestTableVersion() { return latestTableVersion; } } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/TableDescriptor.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit; import java.util.Arrays; import java.util.Map; import java.util.Optional; import org.apache.hadoop.fs.Path; /** * Container for all the info to uniquely identify the table */ public class TableDescriptor { private Path logPath; private Optional tableIdentifier; private Map tableConf; public TableDescriptor(Path logPath, Optional tableIdentifier, Map tableConf) { this.logPath = logPath; this.tableIdentifier = tableIdentifier; this.tableConf = tableConf; } public Optional getTableIdentifier() { return tableIdentifier; } public Path getLogPath() { return logPath; } public Map getTableConf() { return tableConf; } } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/TableIdentifier.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit; /** * Identifier for a table. */ public class TableIdentifier { // The name of the table. private String name; // The namespace of the table. e.g. / private String[] namespace; public TableIdentifier(String[] namespace, String name) { this.namespace = namespace; this.name = name; } public TableIdentifier(String firstName, String... rest) { String[] namespace = new String[rest.length]; String name; if (rest.length > 0) { name = rest[rest.length-1]; namespace[0] = firstName; System.arraycopy(rest, 0, namespace, 1, rest.length-1); } else { name = firstName; } this.namespace = namespace; this.name = name; } /** * Returns the namespace of the table. */ public String[] getNamespace() { return namespace; } /** * Returns the name of the table. */ public String getName() { return name; } } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/UpdatedActions.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit; import io.delta.storage.commit.actions.AbstractCommitInfo; import io.delta.storage.commit.actions.AbstractMetadata; import io.delta.storage.commit.actions.AbstractProtocol; /** * A container class to inform the CommitCoordinatorClient about any changes in Protocol/Metadata */ public class UpdatedActions { private AbstractCommitInfo commitInfo; private AbstractMetadata newMetadata; private AbstractProtocol newProtocol; private AbstractMetadata oldMetadata; private AbstractProtocol oldProtocol; public UpdatedActions( AbstractCommitInfo commitInfo, AbstractMetadata newMetadata, AbstractProtocol newProtocol, AbstractMetadata oldMetadata, AbstractProtocol oldProtocol) { this.commitInfo = commitInfo; this.newMetadata = newMetadata; this.newProtocol = newProtocol; this.oldMetadata = oldMetadata; this.oldProtocol = oldProtocol; } public AbstractCommitInfo getCommitInfo() { return commitInfo; } public AbstractMetadata getNewMetadata() { return newMetadata; } public AbstractProtocol getNewProtocol() { return newProtocol; } public AbstractMetadata getOldMetadata() { return oldMetadata; } public AbstractProtocol getOldProtocol() { return oldProtocol; } } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/actions/AbstractCommitInfo.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit.actions; /** * Interface for objects that represents the base information for a commit. * Commits need to provide an in-commit timestamp. This timestamp is used * to specify the exact time the commit happened and determines the target * version for time-based time travel queries. */ public interface AbstractCommitInfo { /** * Get the timestamp of the commit as millis after the epoch. */ long getCommitTimestamp(); } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/actions/AbstractMetadata.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit.actions; import java.util.Map; import java.util.List; /** * Interface for metadata actions in Delta. The metadata defines the metadata * of the table. */ public interface AbstractMetadata { /** * A unique table identifier. */ String getId(); /** * User-specified table identifier. */ String getName(); /** * User-specified table description. */ String getDescription(); /** The table provider format. */ String getProvider(); /** The format options */ Map getFormatOptions(); /** * The table schema in string representation. */ String getSchemaString(); /** * List of partition columns. */ List getPartitionColumns(); /** * The table properties defined on the table. */ Map getConfiguration(); /** * Timestamp for the creation of this metadata. */ Long getCreatedTime(); } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/actions/AbstractProtocol.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit.actions; import java.util.Set; /** * Interface for protocol actions in Delta. The protocol defines the requirements * that readers and writers of the table need to meet. */ public interface AbstractProtocol { /** * The minimum reader version required to read the table. */ int getMinReaderVersion(); /** * The minimum writer version required to read the table. */ int getMinWriterVersion(); /** * The reader features that need to be supported to read the table. */ Set getReaderFeatures(); /** * The writer features that need to be supported to write the table. */ Set getWriterFeatures(); } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/CommitLimitReachedException.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit.uccommitcoordinator; /** * This exception is thrown by the UC client in case UC has reached the maximum * number of commits that it is allowed to track (50 by default). Upon receiving * this exception, the client should run a backfill. */ public class CommitLimitReachedException extends UCCommitCoordinatorException { public CommitLimitReachedException(String message) { super(message); } } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/InvalidTargetTableException.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit.uccommitcoordinator; /** * This exception is thrown by the UC client in case a commit attempt tried to add * a UUID-based commit to the wrong table. The table is wrong if the path prefixes * of the table and the UUID commit do not match. * For example, adding /path/to/table1/_staged_commits/01-uuid.json to the table at * /path/to/table2 is not allowed. */ public class InvalidTargetTableException extends UCCommitCoordinatorException { public InvalidTargetTableException(String message) { super(message); } } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCClient.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit.uccommitcoordinator; import io.delta.storage.commit.Commit; import io.delta.storage.commit.CommitFailedException; import io.delta.storage.commit.GetCommitsResponse; import io.delta.storage.commit.actions.AbstractMetadata; import io.delta.storage.commit.actions.AbstractProtocol; import io.delta.storage.commit.uniform.UniformMetadata; import java.io.IOException; import java.net.URI; import java.util.Optional; /** * Interface for interacting with the Unity Catalog. * * This interface defines the contract for operations related to the Unity Catalog, * including retrieving the metastore ID, and adding new commits to Delta tables where UC * acts as the Commit Coordinator and similarly retrieving unbackfilled commits. * * Implementations of this interface should handle the specifics of connecting to and * communicating with the Unity Catalog, including any necessary authentication and * request handling. */ public interface UCClient extends AutoCloseable { /** * Retrieves the metastore ID associated with this Unity Catalog instance. * * @return A String representing the unique identifier of the metastore * @throws IOException if there's an error in retrieving the metastore ID */ String getMetastoreId() throws IOException; /** * Commits new changes to a Delta table using the Unity Catalog as the Commit Coordinator. * * This method is responsible for committing changes to a Delta table, including new data, * metadata updates, and protocol changes. It interacts with the Unity Catalog to ensure * proper coordination and consistency of the commit process. * Note: At least one of commit or lastKnownBackfilledVersion must be present. * * @param tableId The unique identifier of the Delta table. * @param tableUri The URI of the storage location of the table. e.g. s3://bucket/path/to/table * (and not s3://bucket/path/to/table/_delta_log). * If the tableId exists but the tableUri is different from the one previously * registered (e.g., if the table as moved), the request will fail. * @param commit An Optional containing the Commit object with the changes to be committed. * If empty, it indicates that no new data is being added in this commit. * @param lastKnownBackfilledVersion An Optional containing the last known backfilled version * of the table. This value serves as a hint to the UC about the * most recent version that has been successfully backfilled. * UC can use this information to optimize its internal state * management by cleaning up tracking information for backfilled * commits up to this version. * If not provided (Optional.empty()), UC will rely on its * current state without any additional cleanup hints. * @param disown A boolean flag indicating whether to disown the table after commit. * If true, the coordinator will release ownership of the table after the commit. * @param newMetadata An Optional containing new metadata to be applied to the table. * If present, the table's metadata will be updated atomically with the commit. * @param newProtocol An Optional containing a new protocol version to be applied to the table. * If present, the table's protocol will be updated atomically with the commit. * @param uniform An Optional containing UniForm metadata for Delta Universal Format support. * If present, this metadata will be used by UC to manage format conversions * (e.g., Iceberg, Hudi). * @throws IOException if there's an error during the commit process, such as network issues. * @throws CommitFailedException if the commit fails due to conflicts or other logical errors. * @throws UCCommitCoordinatorException if there's an error specific to the Unity Catalog * commit coordination process. */ void commit( String tableId, URI tableUri, Optional commit, Optional lastKnownBackfilledVersion, boolean disown, Optional newMetadata, Optional newProtocol, Optional uniform ) throws IOException, CommitFailedException, UCCommitCoordinatorException; /** * Retrieves the unbackfilled commits for a Delta table within a specified version range. * * @param tableId The unique identifier of the Delta table. * @param tableUri The URI of the storage location of the table. e.g. s3://bucket/path/to/table * (and not s3://bucket/path/to/table/_delta_log). * If the tableId exists but the tableUri is different from the one previously * registered (e.g., if the table as moved), the request will fail. * @param startVersion An Optional containing the start version of the range of commits to * retrieve. * @param endVersion An Optional containing the end version of the range of commits to retrieve. * @return A GetCommitsResponse object containing the unbackfilled commits within the specified * version range. If all commits are backfilled, the response will contain an empty list. * The response also contains the last known backfilled version of the table. If no * commits are ratified via UC, the lastKnownBackfilledVersion will be -1. * @throws IOException if there's an error during the commit process, such as network issues. * @throws UCCommitCoordinatorException if there's an error specific to the Unity Catalog such as * the table not being found. */ GetCommitsResponse getCommits( String tableId, URI tableUri, Optional startVersion, Optional endVersion) throws IOException, UCCommitCoordinatorException; /** * Closes any resources used by this client. * This method should be called to properly release resources such as network * connections (e.g., HTTPClient) when the client is no longer needed. * Once this method is called, the client should not be used to perform any operations. * * @throws IOException if there's an error while closing resources */ @Override void close() throws IOException; } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCCommitCoordinatorClient.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit.uccommitcoordinator; import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; import java.nio.file.FileAlreadyExistsException; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import java.util.stream.Collectors; import javax.annotation.Nonnull; import io.delta.storage.CloseableIterator; import io.delta.storage.LogStore; import io.delta.storage.commit.*; import io.delta.storage.commit.actions.AbstractMetadata; import io.delta.storage.commit.actions.AbstractProtocol; import io.delta.storage.internal.FileNameUtils; import io.delta.storage.internal.LogStoreErrors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * A commit coordinator client that uses unity-catalog as the commit coordinator. */ public class UCCommitCoordinatorClient implements CommitCoordinatorClient { public UCCommitCoordinatorClient(Map conf, UCClient ucClient) { this.conf = conf; this.ucClient = ucClient; } /** * Logger for UCCommitCoordinatorClient class operations and diagnostics. */ private static final Logger LOG = LoggerFactory.getLogger(UCCommitCoordinatorClient.class); // UC Protocol Version Control Constants /** Supported version for read operations in the Unity Catalog protocol. */ private static final int SUPPORTED_READ_VERSION = 0; /** Supported version for write operations in the Unity Catalog protocol. */ private static final int SUPPORTED_WRITE_VERSION = 0; /** Key used to identify the read version in protocol communications with the UC server. */ private static final String READ_VERSION_KEY = "readVersion"; /** Key used to identify the write version in protocol communications with the UC server. */ private static final String WRITE_VERSION_KEY = "writeVersion"; /** * Temporary kill switch for sending metadata updates through UC from the Spark path. * TODO(issue #6296): remove once metadata updates are supported end-to-end. */ private static final boolean SHOULD_PASS_METADATA_TO_UC = false; // Unity Catalog Identifiers /** * Key for identifying Unity Catalog table ID in `delta.coordinatedCommits.tableConf{-preview}`. */ final static public String UC_TABLE_ID_KEY = "io.unitycatalog.tableId"; // Previously this key was ucTableId. It was later renamed. final static public String UC_TABLE_ID_KEY_OLD = "ucTableId"; /** * Key for identifying Unity Catalog metastore ID in * `delta.coordinatedCommits.commitCoordinatorConf{-preview}`. */ final static public String UC_METASTORE_ID_KEY = "ucMetastoreId"; // Backfill and Retry Configuration /** * Offset from current commit version for backfill listing optimization. * Used to prevent expensive listings from version 0. */ public static int BACKFILL_LISTING_OFFSET = 100; /** Maximum number of retry attempts for transient errors. */ protected static final int MAX_RETRIES_ON_TRANSIENT_ERROR = 15; /** Initial wait time in milliseconds before retrying after a transient error. */ protected static final long TRANSIENT_ERROR_RETRY_INITIAL_WAIT_MS = 100; /** Maximum wait time in milliseconds between retries for transient errors. */ protected static final long TRANSIENT_ERROR_RETRY_MAX_WAIT_MS = 1000 * 60; // 1 minute // Thread Pool Configuration /** Size of the thread pool for handling asynchronous operations. */ static protected int THREAD_POOL_SIZE = 20; /** * Thread pool executor for handling asynchronous tasks like backfilling. * Configured with daemon threads and custom naming pattern. */ private static final ThreadPoolExecutor asyncExecutor; // Static Initializer Block static { asyncExecutor = new ThreadPoolExecutor( THREAD_POOL_SIZE, THREAD_POOL_SIZE, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(Integer.MAX_VALUE), new ThreadFactory() { private final ThreadFactory defaultFactory = Executors.defaultThreadFactory(); @Override public Thread newThread(@Nonnull Runnable r) { Thread t = defaultFactory.newThread(r); // Set the thread name to uc-commit-coordinator-pool-1-thread-1 t.setName("uc-commit-coordinator-" + t.getName()); t.setDaemon(true); return t; } }); asyncExecutor.allowCoreThreadTimeOut(true); } // Instance Variables /** Unity Catalog client instance for interacting with UC services. */ public final UCClient ucClient; /** Configuration map containing settings for the coordinator client. */ public final Map conf; /** * Runs a task asynchronously using the backfillThreadPool. * * @param task The task to be executed asynchronously * @return A Future representing pending completion of the task */ protected Future executeAsync(Callable task) { return asyncExecutor.submit(task); } protected String extractUCTableId(TableDescriptor tableDesc) { Map tableConf = tableDesc.getTableConf(); if (!tableConf.containsKey(UC_TABLE_ID_KEY)) { throw new IllegalStateException("UC Table ID not found in " + tableConf); } return tableConf.get(UC_TABLE_ID_KEY); } /** * For UC, table registration is a no-op because we already contacted UC during table * creation and that already obtained the necessary table config and added * it to the metadata (this is for performance reasons and ease of use). As a result, * this method only verifies that the metadata has been added correct and is present. * Otherwise, it throws an exception. */ @Override public Map registerTable( Path logPath, Optional tableIdentifier, long currentVersion, AbstractMetadata currentMetadata, AbstractProtocol currentProtocol) { Map tableConf = CoordinatedCommitsUtils.getTableConf(currentMetadata); checkVersionSupported(tableConf, false /* compareRead */); // The coordinatedCommitsTableConf must have been instantiated prior to this call // with the UC table ID. if (!tableConf.containsKey(UC_TABLE_ID_KEY)) { throw new IllegalStateException("Could not verify if the table is registered with the " + "UC commit coordinator because the table ID is missing from the table metadata."); } // The coordinatedCommitsCoordinatorConf must have been instantiated prior to this call // with the metastore ID of the metastore, which stores the table. if (!CoordinatedCommitsUtils.getCoordinatorConf(currentMetadata).containsKey( UC_METASTORE_ID_KEY)) { throw new IllegalStateException("Could not verify if the table is registered with the UC " + "commit coordinator because the metastore ID is missing from the table metadata."); } return tableConf; } /** * Find the last known backfilled version by doing a listing of the last * {@link #BACKFILL_LISTING_OFFSET} commits. If no backfilled commits are found * among those, a UC call is made to get the oldest tracked commit in UC. */ public long getLastKnownBackfilledVersion( long commitVersion, Configuration hadoopConf, LogStore logStore, TableDescriptor tableDesc ) { Path logPath = tableDesc.getLogPath(); long listFromVersion = Math.max(0, commitVersion - BACKFILL_LISTING_OFFSET); Optional lastKnownBackfilledVersion = listAndGetLastKnownBackfilledVersion(listFromVersion, logStore, hadoopConf, logPath); if (!lastKnownBackfilledVersion.isPresent()) { // In case we don't find anything in the last 100 commits (should not happen) // we go to UC to find the earliest commit it is tracking as the commit prior // to that must have been backfilled. recordDeltaEvent( UCCoordinatedCommitsUsageLogs.UC_LAST_KNOWN_BACKFILLED_VERSION_NOT_FOUND, new HashMap() {{ put("commitVersion", commitVersion); put("conf", conf); put("listFromVersion", listFromVersion); put("tableConf", tableDesc.getTableConf()); }}, logPath.getParent() ); long minVersion = getCommits(tableDesc, null, null) .getCommits() .stream() .min(Comparator.comparingLong(Commit::getVersion)) .map(Commit::getVersion) .orElseThrow(() -> new IllegalStateException("Couldn't find any unbackfilled commit " + "for table at " + logPath + " at version " + commitVersion)); lastKnownBackfilledVersion = listAndGetLastKnownBackfilledVersion( minVersion - 1, logStore, hadoopConf, logPath); if (!lastKnownBackfilledVersion.isPresent()) { throw new IllegalStateException("Couldn't find any backfilled commit for table at " + logPath + " at version " + commitVersion); } } return lastKnownBackfilledVersion.get(); } protected Iterator listFrom( LogStore logStore, long listFromVersion, Configuration hadoopConf, Path logPath) { Path listingPath = CoordinatedCommitsUtils.getBackfilledDeltaFilePath(logPath, listFromVersion); try { return logStore.listFrom(listingPath, hadoopConf); } catch (IOException e) { LOG.error("Failed to list files from {} due to: {}", listingPath, exceptionString(e)); throw new IllegalStateException(e); } } protected Optional listAndGetLastKnownBackfilledVersion( long listFromVersion, LogStore logStore, Configuration hadoopConf, Path logPath) { Optional lastKnownBackfilledVersion = Optional.empty(); Iterator deltaLogFileIt = listFrom(logStore, listFromVersion, hadoopConf, logPath); while (deltaLogFileIt.hasNext()) { FileStatus fileStatus = deltaLogFileIt.next(); if (FileNameUtils.isDeltaFile(fileStatus.getPath())) { lastKnownBackfilledVersion = Optional.of(FileNameUtils.deltaVersion(fileStatus.getPath())); } } return lastKnownBackfilledVersion; } @Override public CommitResponse commit( LogStore logStore, Configuration hadoopConf, TableDescriptor tableDesc, long commitVersion, Iterator actions, UpdatedActions updatedActions) throws CommitFailedException { return commitImpl( logStore, hadoopConf, tableDesc, commitVersion, actions, updatedActions); } /** * Commits the provided actions as the specified version. The steps are as follows. * * 1. Write the actions to a UUID-based commit file * 2. In parallel to 1. determine the last known backfilled version. * If a backfill hint is provided, we verify that it exists via a single HEAD call. Otherwise, * the last known backfilled version is determined via a listing. * 3. Send commit request to UC to commit the version and register backfills up to the * found last known backfilled version. * 4. Backfill all unbackfilled commits (including the latest one made in this call) * asynchronously. * A getCommits call is made to UC to retrieve all currently unbackfilled commits. */ protected CommitResponse commitImpl( LogStore logStore, Configuration hadoopConf, TableDescriptor tableDesc, long commitVersion, Iterator actions, UpdatedActions updatedActions) throws CommitFailedException { Path logPath = tableDesc.getLogPath(); Map coordinatedCommitsTableConf = tableDesc.getTableConf(); checkVersionSupported(coordinatedCommitsTableConf, false /* compareRead */); // Writes may also have to perform reads to determine the last known backfilled // version/the commits to backfill in case we don't have a backfill hint. To // prevent to write to succeed but then fail the read, we do the read protocol // version check here. checkVersionSupported(coordinatedCommitsTableConf, true /* compareRead */); if (commitVersion == 0) { throw new CommitFailedException( false /* retryable */, false /* conflict */, "Commit version 0 must go via filesystem."); } long startTimeMs = System.currentTimeMillis(); Map eventData = new HashMap<>(); eventData.put("commitVersion", commitVersion); eventData.put("coordinatedCommitsTableConf", coordinatedCommitsTableConf); eventData.put("updatedActions", updatedActions); BiConsumer, String> recordUsageLog = (exception, opType) -> { exception.ifPresent(throwable -> { eventData.put("exceptionClass", throwable.getClass().getName()); eventData.put("exceptionString", exceptionString(throwable)); }); eventData.put("totalTimeTakenMs", System.currentTimeMillis() - startTimeMs); recordDeltaEvent(opType, eventData, logPath.getParent()); }; // After commit 0, the table ID must exist in UC String tableId = extractUCTableId(tableDesc); LOG.info("Attempting to commit version " + commitVersion + " to table " + tableId); // Asynchronously verify/retrieve the last known backfilled version // Using AtomicLong instead of Long because we need to update the value in the lambda // and "Variable used in lambda expression should be final or effectively final". AtomicLong timeSpentInGettingLastKnownBackfilledVersion = new AtomicLong(System.currentTimeMillis()); Future lastKnownBackfilledVersionFuture; try { lastKnownBackfilledVersionFuture = executeAsync(() -> { long foundVersion = getLastKnownBackfilledVersion( commitVersion, hadoopConf, logStore, tableDesc); timeSpentInGettingLastKnownBackfilledVersion.getAndUpdate(start -> System.currentTimeMillis() - start); return foundVersion; }); } catch (Exception e) { // Synchronously verify/retrieve last known backfilled version. LOG.warn("Error while submitting task to verify/retrieve last known backfilled version " + "due to: " + exceptionString(e) + ". Verifying/retrieving synchronously"); recordUsageLog.accept( Optional.of(e), UCCoordinatedCommitsUsageLogs.UC_BACKFILL_VALIDATION_FALLBACK_TO_SYNC); long foundVersion = getLastKnownBackfilledVersion( commitVersion, hadoopConf, logStore, tableDesc); timeSpentInGettingLastKnownBackfilledVersion.getAndUpdate(start -> System.currentTimeMillis() - start);; lastKnownBackfilledVersionFuture = CompletableFuture.completedFuture(foundVersion); } // In parallel to verifying/getting the last known backfilled version, write the commit file. long writeStartTimeMs = System.currentTimeMillis(); FileStatus commitFile; try { commitFile = CoordinatedCommitsUtils.writeUnbackfilledCommitFile( logStore, hadoopConf, logPath.toString(), commitVersion, actions, UUID.randomUUID().toString() ); } catch (IOException e) { throw new CommitFailedException( true /* retryable */, false /* conflict */, "Failed to write commit file due to: " + e.getMessage(), e); } eventData.put("writeCommitFileTimeTakenMs", System.currentTimeMillis() - writeStartTimeMs); // Using AtomicLong instead of Long because we need to access the value in the lambda // and "Variable used in lambda expression should be final or effectively final". AtomicLong lastKnownBackfilledVersion = new AtomicLong(); try { lastKnownBackfilledVersion.set(lastKnownBackfilledVersionFuture.get()); } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); } long commitTimestamp = updatedActions.getCommitInfo().getCommitTimestamp(); boolean disown = isDisownCommit( updatedActions.getOldMetadata(), updatedActions.getNewMetadata()); eventData.put("tableId", tableId); eventData.put("lastKnownBackfilledVersion", lastKnownBackfilledVersion.get()); eventData.put("commitTimestamp", commitTimestamp); eventData.put("disown", disown); eventData.put( "timeSpentInGettingLastKnownBackfilledVersion", timeSpentInGettingLastKnownBackfilledVersion); int transientErrorRetryCount = 0; while (transientErrorRetryCount <= MAX_RETRIES_ON_TRANSIENT_ERROR) { try { commitToUC( tableDesc, logPath, Optional.of(commitFile), Optional.of(commitVersion), Optional.of(commitTimestamp), Optional.of(lastKnownBackfilledVersion.get()), disown, updatedActions.getNewMetadata() == updatedActions.getOldMetadata() || !SHOULD_PASS_METADATA_TO_UC ? Optional.empty() : Optional.of(updatedActions.getNewMetadata()), updatedActions.getNewProtocol() == updatedActions.getOldProtocol() ? Optional.empty() : Optional.of(updatedActions.getNewProtocol()) ); break; } catch (CommitFailedException cfe) { if (transientErrorRetryCount > 0 && cfe.getConflict() && cfe.getRetryable() && hasSameContent( logStore, hadoopConf, logPath, CoordinatedCommitsUtils.getBackfilledDeltaFilePath(logPath, commitVersion), commitFile.getPath())) { // The commit was persisted in UC, but we did not get a response. Continue // because the commit was successful eventData.put("alreadyBackfilledCommitCausedConflict", true); break; } else { // Rethrow the exception here as is because the caller needs to handle it. recordUsageLog.accept(Optional.of(cfe), UCCoordinatedCommitsUsageLogs.UC_COMMIT_STATS); throw cfe; } } catch (IOException ioe) { if (transientErrorRetryCount == MAX_RETRIES_ON_TRANSIENT_ERROR) { // Rethrow exception in case we've reached the retry limit. recordUsageLog.accept(Optional.of(ioe), UCCoordinatedCommitsUsageLogs.UC_COMMIT_STATS); throw new CommitFailedException( true /* retryable */, false /* conflict */, ioe.getMessage(), ioe); } // Exponentially back off. The initial wait time is set to 100ms and the max retry count // is 15. The max wait time is 1 min so overall, we'll be waiting for a max of ~8 min. long sleepTime = Math.min( TRANSIENT_ERROR_RETRY_INITIAL_WAIT_MS << transientErrorRetryCount, TRANSIENT_ERROR_RETRY_MAX_WAIT_MS ); LOG.info("Sleeping for " + sleepTime + "ms before retrying commit after transient error " + ioe.getMessage()); try { Thread.sleep(sleepTime); } catch (InterruptedException e) { throw new RuntimeException(e); } transientErrorRetryCount++; eventData.put("transientErrorRetryCount", transientErrorRetryCount); } catch (UpgradeNotAllowedException unae) { // This is translated to a non-retryable, non-conflicting commit failure. recordUsageLog.accept(Optional.of(unae), UCCoordinatedCommitsUsageLogs.UC_COMMIT_STATS); throw new CommitFailedException( false /* retryable */, false /* conflict */, unae.getMessage(), unae); } catch (InvalidTargetTableException itte) { // Just rethrow, this will propagate to the user. recordUsageLog.accept(Optional.of(itte), UCCoordinatedCommitsUsageLogs.UC_COMMIT_STATS); throw new CommitFailedException( false /* retryable */, false /* conflict */, itte.getMessage(), itte); } catch (CommitLimitReachedException clre) { // We attempt a full backfill and then retry the commit. try { AtomicReference caughtException = new AtomicReference<>(null); lastKnownBackfilledVersion.getAndUpdate(lastKnownBackfilledVersionVal -> { try { return attemptFullBackfill( logStore, hadoopConf, tableDesc, commitVersion, tableId, lastKnownBackfilledVersionVal, eventData ); } catch (Exception e) { caughtException.set(e); return lastKnownBackfilledVersionVal; // Return unchanged value on exception } }); if (caughtException.get() != null) { throw caughtException.get(); } } catch (Throwable e) { recordUsageLog.accept( Optional.of(e), UCCoordinatedCommitsUsageLogs.UC_FULL_BACKFILL_ATTEMPT_FAILED); String message = String.format( "Commit limit reached (%s) for table %s. A full backfill attempt failed due to: %s", exceptionString(clre), tableId, exceptionString(e)); throw new CommitFailedException( true /* retryable */, false /* conflict */, message, clre); } eventData.put("lastKnownBackfilledVersion", lastKnownBackfilledVersion.get()); eventData.put("encounteredCommitLimitReachedException", true); // Retry the commit as there should be space in UC now. We set isCommitLimitReachedRetry // to true so that in case the full backfill attempt was unsuccessful in freeing up space // in UC, we don't indefinitely retry but rather throw the CommitLimitReachedException. // Don't increase transientErrorRetryCount as this is not a transient error. } catch (UCCommitCoordinatorException ucce) { // Just rethrow, this will propagate to the user. recordUsageLog.accept(Optional.of(ucce), UCCoordinatedCommitsUsageLogs.UC_COMMIT_STATS); throw new CommitFailedException( false /* retryable */, false /* conflict */, ucce.getMessage(), ucce); } } LOG.info("Successfully wrote " + commitFile.getPath() + " as commit " + commitVersion + " to table " + tableId); // Asynchronously backfill everything up to the latest commit. Callable doBackfill = () -> { backfillToVersion( logStore, hadoopConf, tableDesc, commitVersion, lastKnownBackfilledVersion.get() ); return null; }; try { executeAsync(doBackfill); } catch (Throwable e) { if (LogStoreErrors.isFatal(e)) { throw e; } // attempt a synchronous backfill LOG.warn("Error while submitting backfill task: " + exceptionString(e) + ". Performing synchronous backfill now."); recordUsageLog.accept( Optional.of(e), UCCoordinatedCommitsUsageLogs.UC_BACKFILL_FALLBACK_TO_SYNC); try { doBackfill.call(); } catch (Throwable t) { if (LogStoreErrors.isFatal(t)) { throw new RuntimeException(t); } } } recordUsageLog.accept(Optional.empty(), UCCoordinatedCommitsUsageLogs.UC_COMMIT_STATS); return new CommitResponse(new Commit(commitVersion, commitFile, commitTimestamp)); } /** * Attempts a full backfill of all currently unbackfilled versions in order to free * up space in UC. After the attempt, will do a listing to find the new last known * backfilled version and returns it. */ protected long attemptFullBackfill( LogStore logStore, Configuration hadoopConf, TableDescriptor tableDesc, long commitVersion, String tableId, long lastKnownBackfilledVersion, Map eventData) throws IOException, UCCommitCoordinatorException, CommitFailedException { Path logPath = tableDesc.getLogPath(); LOG.info("Too many unbackfilled commits in UC at version {} for table at {} " + "and ID {}. Last known backfill version is {}. Attempting a full backfill.", commitVersion, logPath, tableId, lastKnownBackfilledVersion); long backfillStartTime = System.currentTimeMillis(); backfillToVersion( logStore, hadoopConf, tableDesc, commitVersion, lastKnownBackfilledVersion ); long backfillDuration = System.currentTimeMillis() - backfillStartTime; long updatedLastKnownBackfilledVersion = getLastKnownBackfilledVersion( commitVersion, hadoopConf, logStore, tableDesc); long commitStartTime = System.currentTimeMillis(); commitToUC( tableDesc, logPath, Optional.empty() /* commitFile */, Optional.empty() /* commitVersion */, Optional.empty() /* commitTimestamp */, Optional.of(updatedLastKnownBackfilledVersion), true /* disown */, Optional.empty() /* newMetadata */, Optional.empty() /* newProtocol */ ); long commitDuration = System.currentTimeMillis() - commitStartTime; recordDeltaEvent( UCCoordinatedCommitsUsageLogs.UC_ATTEMPT_FULL_BACKFILL, new HashMap(eventData) {{ put("commitVersion", commitVersion); put("coordinatedCommitsTableConf", tableDesc.getTableConf()); put("lastKnownBackfilledVersion", lastKnownBackfilledVersion); put("updatedLastKnownBackfilledVersion", updatedLastKnownBackfilledVersion); put("tableId", tableId); put("backfillTime", backfillDuration); put("ucCommitTime", commitDuration); }}, logPath.getParent() ); return updatedLastKnownBackfilledVersion; } protected void commitToUC( TableDescriptor tableDesc, Path logPath, Optional commitFile, Optional commitVersion, Optional commitTimestamp, Optional lastKnownBackfilledVersion, boolean disown, Optional newMetadata, Optional newProtocol ) throws IOException, CommitFailedException, UCCommitCoordinatorException { Optional commit = commitFile.map(f -> new Commit( commitVersion.orElseThrow(() -> new IllegalArgumentException( "Commit version should be specified when commitFile is present")), f, commitTimestamp.orElseThrow(() -> new IllegalArgumentException( "Commit timestamp should be specified when commitFile is present")) )); ucClient.commit( extractUCTableId(tableDesc), CoordinatedCommitsUtils.getTablePath(logPath).toUri(), commit, lastKnownBackfilledVersion, disown, newMetadata, newProtocol, Optional.empty() /* uniform */ ); } /** * Detects whether the current commit is a downgrade (disown) commit by checking * that the UC commit coordinator name is present in the old metadata but removed from * the new metadata. */ protected boolean isDisownCommit(AbstractMetadata oldMetadata, AbstractMetadata newMetadata) { return CoordinatedCommitsUtils .getCoordinatorName(oldMetadata) .filter("unity-catalog"::equals).isPresent() && !CoordinatedCommitsUtils.getCoordinatorName(newMetadata).isPresent(); } /** * This method provides idempotency under network failures by verifying whether the currently * attempted commit already exists as a backfilled commit. This prevents duplicate data from * being written when UC returns a retryable conflict for a commit that was actually successful * but the client didn't receive the success response. * * Failure sequence requiring this check: * 1. Client attempts to make commit v. * 2. UC persists the commit in its database but the connection to the client breaks. * 3. The client receives a transient error (retryable=true, conflict=false). * 4. Before retrying, a concurrent client commits v + 1 and backfills v. * 5. Another subsequent commit registers the backfill of v with UC, leading UC to * delete the commit for v from its database. * 6. Now this client retries commit v (without conflict resolution since conflict=false * in step 3). * 7. UC rejects the commit because v {@literal <=} latest_table_version and returns a retryable * conflict (retryable=true, conflict=true). * * Without this check, Delta's default response to retryable=true, conflict=true would be to * rebase the commit on top of the latest table version and retry, effectively trying to * commit the contents of v as v+2. This would result in duplicate data being written. * * This method prevents that by checking if the backfilled commit (v.json) has the same * content as our retry attempt (v.{@literal }.json). If yes, we know our original commit * succeeded and can safely ignore the conflict and exit early without rebasing. * * Below is a concrete example of the failure and retry sequence: * - Attempt 1: Try to commit v. UC responds with retryable=true, conflict=false under * network failure. * - Attempt 2: Try to commit v without conflict resolution since conflict=false in attempt-1. * UC responds with retryable=true, conflict=true in the above scenario. * (i.e. v is backfilled and latest version is v+1). * - Fix: Compare v.{@literal }.json and v.json and *early exit* here. * - Attempt 3: [Without fix] Rebase, conflict-resolution + Try to commit v+2 * {@literal =>} double-commit for contents of v {@literal =>} bug. */ protected boolean hasSameContent( LogStore logStore, Configuration hadoopConf, Path logPath, Path backfilledCommit, Path unbackfilledCommit) { try { FileSystem fs = logPath.getFileSystem(hadoopConf); if (fs.getFileStatus(backfilledCommit).getLen() != fs.getFileStatus(unbackfilledCommit).getLen()) { return false; } } catch (FileNotFoundException e) { // If we get a FileNotFoundException, it should be for the backfilled // commit because we are only calling this method from commit() at the moment, // which means we just wrote the unbackfilled commit. return false; } catch (IOException e) { throw new RuntimeException(e); } // Compare content. try (CloseableIterator contentBackfilled = logStore.read(backfilledCommit, hadoopConf); CloseableIterator contentUnbackfilled = logStore.read(unbackfilledCommit, hadoopConf)) { while (contentUnbackfilled.hasNext() && contentBackfilled.hasNext()) { if (!contentUnbackfilled.next().equals(contentBackfilled.next())) { return false; } } return !contentBackfilled.hasNext() && !contentUnbackfilled.hasNext(); } catch (IOException e) { throw new RuntimeException(e); } } @Override public GetCommitsResponse getCommits( TableDescriptor tableDesc, Long startVersion, Long endVersion) { checkVersionSupported(tableDesc.getTableConf(), true /* compareRead */); GetCommitsResponse resp = getCommitsFromUCImpl( tableDesc, Optional.ofNullable(startVersion), Optional.ofNullable(endVersion)); // Sort by version just in case commits in the response from UC aren't sorted. List sortedCommits = resp .getCommits() .stream() .sorted(Comparator.comparingLong(Commit::getVersion)) .collect(Collectors.toList()); return new GetCommitsResponse(sortedCommits, resp.getLatestTableVersion()); } protected GetCommitsResponse getCommitsFromUCImpl( TableDescriptor tableDesc, Optional startVersion, Optional endVersion) { try { return ucClient.getCommits( extractUCTableId(tableDesc), CoordinatedCommitsUtils.getTablePath(tableDesc.getLogPath()).toUri(), startVersion, endVersion); } catch (IOException | UCCommitCoordinatorException e) { throw new RuntimeException(e); } } @Override public void backfillToVersion( LogStore logStore, Configuration hadoopConf, TableDescriptor tableDesc, long version, Long lastKnownBackfilledVersion) throws IOException { // backfillToVersion currently does not depend on write. However, it is // technically a write operation, so we also add a write version check here // in case we ever introduce a write dependency. checkVersionSupported(tableDesc.getTableConf(), true /* compareRead */); checkVersionSupported(tableDesc.getTableConf(), false /* compareRead */); Path logPath = tableDesc.getLogPath(); String tableId = extractUCTableId(tableDesc); long startVersion = (lastKnownBackfilledVersion == null) ? 0L : lastKnownBackfilledVersion; long startTimeMs = System.currentTimeMillis(); LOG.info("Backfilling {}: startVersion {} to endVersion {}", tableId, startVersion, version); // Check that the last known backfilled version actually exists if it // has been specified. If it doesn't exist, we fail the backfill. If it // hasn't been specified backfill everything that hasn't been backfilled yet. if (lastKnownBackfilledVersion != null) { FileSystem fs = logPath.getFileSystem(hadoopConf); // Check that the last known backfilled version actually exists. if (!fs.exists(CoordinatedCommitsUtils .getBackfilledDeltaFilePath(logPath, lastKnownBackfilledVersion))) { LOG.error("Specified last known backfilled version {} does not exist for table {}", lastKnownBackfilledVersion, tableId); recordDeltaEvent( UCCoordinatedCommitsUsageLogs.UC_BACKFILL_DOES_NOT_EXIST, new HashMap() {{ put("lastKnownBackfilledVersion", lastKnownBackfilledVersion); put("version", version); put("tableConf", tableDesc.getTableConf()); }}, logPath.getParent() ); throw new IllegalStateException("Last known backfilled version " + lastKnownBackfilledVersion + " doesn't exist for table at " + logPath); } } GetCommitsResponse commitsResponse = getCommits(tableDesc, lastKnownBackfilledVersion, version); for (Commit commit : commitsResponse.getCommits()) { boolean backfillResult = backfillSingleCommit( logStore, hadoopConf, logPath, commit.getVersion(), commit.getFileStatus(), false /* failOnException */); if (!backfillResult) { break; } } recordDeltaEvent( UCCoordinatedCommitsUsageLogs.UC_BACKFILL_TO_VERSION, new HashMap() {{ put("coordinatedCommitsTableConf", tableDesc.getTableConf()); put("totalTimeTakenMs", System.currentTimeMillis() - startTimeMs); put("lastKnownBackfilledVersion", lastKnownBackfilledVersion); put("tableId", tableId); put("version", version); }}, logPath.getParent() ); } /** * Backfill the specified commit as the target version. Returns true if the * backfill was successful (or the backfilled file already existed) and false * in case the backfill failed. */ protected boolean backfillSingleCommit( LogStore logStore, Configuration hadoopConf, Path logPath, long version, FileStatus fileStatus, Boolean failOnException) { Path targetFile = CoordinatedCommitsUtils.getBackfilledDeltaFilePath(logPath, version); try (CloseableIterator commitContentIterator = logStore.read(fileStatus.getPath(), hadoopConf)) { // Use put-if-absent for backfills so that files are not overwritten and the // modification time does not change for already backfilled files. logStore.write(targetFile, commitContentIterator, false /* overwrite */, hadoopConf); } catch (FileAlreadyExistsException e) { LOG.info("The backfilled file {} already exists.", targetFile); } catch (Exception e) { if (LogStoreErrors.isFatal(e) || failOnException) { throw new RuntimeException(e); } LOG.warn("Backfill for table at {} failed for version {} due to: {}", logPath, version, exceptionString(e)); recordDeltaEvent( UCCoordinatedCommitsUsageLogs.UC_BACKFILL_FAILED, new HashMap() {{ put("version", version); put("exceptionClass", e.getClass().getName()); put("exceptionString", exceptionString(e)); }}, logPath.getParent() ); return false; } return true; } @Override public boolean semanticEquals(CommitCoordinatorClient other) { if (!(other instanceof UCCommitCoordinatorClient)) { return false; } UCCommitCoordinatorClient otherStore = (UCCommitCoordinatorClient) other; return this.conf == otherStore.conf; } protected void recordDeltaEvent(String opType, Object data, Path path) { LOG.info("Delta event recorded with opType={}, data={}, and path={}", opType, data, path); } protected String exceptionString(Throwable e) { if (e == null) { return ""; } else { StringWriter stringWriter = new StringWriter(); e.printStackTrace(new PrintWriter(stringWriter)); return stringWriter.toString(); } } protected void checkVersionSupported(Map tableConf, boolean compareRead) { int readVersion = Integer.parseInt(tableConf.getOrDefault(READ_VERSION_KEY, "0")); int writeVersion = Integer.parseInt(tableConf.getOrDefault(WRITE_VERSION_KEY, "0")); int targetVersion = compareRead ? readVersion : writeVersion; int supportedVersion = compareRead ? SUPPORTED_READ_VERSION : SUPPORTED_WRITE_VERSION; String op = compareRead ? "read" : "write"; if (supportedVersion != targetVersion) { throw new UnsupportedOperationException("The version of the UC commit coordinator protocol" + " is not supported by this version of the UC commit coordinator client. Please upgrade" + " the commit coordinator client to " + op + " this table."); } } } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCCommitCoordinatorException.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit.uccommitcoordinator; /** * Base class for all exceptions thrown by the UC client from coordinated commits-related APIs. */ public abstract class UCCommitCoordinatorException extends Exception { public UCCommitCoordinatorException(String message) { super(message); } } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCCoordinatedCommitsUsageLogs.java ================================================ package io.delta.storage.commit.uccommitcoordinator; /** Class containing usage logs emitted by Coordinated Commits. */ public class UCCoordinatedCommitsUsageLogs { // Common prefix for all coordinated-commits usage logs. private static final String PREFIX = "delta.coordinatedCommits"; // Usage log emitted after backfilling to a version. public static final String UC_BACKFILL_TO_VERSION = PREFIX + ".uc.backfillToVersion"; // Usage log emitted if the specified last known backfilled version does not exist. public static final String UC_BACKFILL_DOES_NOT_EXIST = PREFIX + ".uc.backfillDoesNotExist"; // Usage log emitted if a backfill attempt for a single file failed. public static final String UC_BACKFILL_FAILED = PREFIX + ".uc.backfillFailed"; // Usage log emitted when the last known backfilled version cannot be determined from the last // `BACKFILL_LISTING_OFFSET` commits. public static final String UC_LAST_KNOWN_BACKFILLED_VERSION_NOT_FOUND = PREFIX + ".uc.lastKnownBackfilledVersionNotFound"; // Usage log emitted if UC commit coordinator client falls back to synchronous backfill. public static final String UC_BACKFILL_VALIDATION_FALLBACK_TO_SYNC = PREFIX + ".uc.backfillValidation.fallbackToSync"; // Usage log emitted when commit limit is reached, and we attempt a full backfill. public static final String UC_ATTEMPT_FULL_BACKFILL = PREFIX + ".uc.attemptFullBackfill"; // Usage log emitted if UC commit coordinator client falls back to synchronous backfill. public static final String UC_BACKFILL_FALLBACK_TO_SYNC = PREFIX + ".uc.backfill.fallbackToSync"; // Usage log emitted as part of [[UCCommitCoordinatorClient.commit]] call. public static final String UC_COMMIT_STATS = PREFIX + ".uc.commitStats"; // Usage log emitted when a full backfill attempt has failed public static final String UC_FULL_BACKFILL_ATTEMPT_FAILED = PREFIX + ".uc.fullBackfillAttemptFailed"; } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCRestClientPayload.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit.uccommitcoordinator; import com.fasterxml.jackson.annotation.JsonInclude; import io.delta.storage.commit.Commit; import io.delta.storage.commit.actions.AbstractMetadata; import io.delta.storage.commit.actions.AbstractProtocol; import io.delta.storage.commit.uniform.UniformMetadata; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import java.util.ArrayList; import java.util.List; import java.util.Map; /** * Container for internal REST classes used by UCTokenBasedRestClient. * Encapsulates all necessary classes for JSON serialization/deserialization. */ class UCRestClientPayload { // ============================== // CommitInfo Class // ============================== static class CommitInfo { Long version; Long timestamp; String fileName; Long fileSize; Long fileModificationTimestamp; Boolean isDisownCommit; static CommitInfo fromCommit(Commit externalCommit, boolean isDisownCommit) { if (externalCommit == null) { throw new IllegalArgumentException("externalCommit cannot be null"); } if (externalCommit.getFileStatus() == null) { throw new IllegalArgumentException("externalCommit.getFileStatus() cannot be null"); } CommitInfo commitInfo = new CommitInfo(); commitInfo.version = externalCommit.getVersion(); commitInfo.timestamp = externalCommit.getCommitTimestamp(); commitInfo.fileName = externalCommit.getFileStatus().getPath().getName(); commitInfo.fileSize = externalCommit.getFileStatus().getLen(); commitInfo.fileModificationTimestamp = externalCommit.getFileStatus().getModificationTime(); commitInfo.isDisownCommit = isDisownCommit; return commitInfo; } static Commit toCommit(CommitInfo commitInfo, Path basePath) { FileStatus fileStatus = new FileStatus( commitInfo.fileSize, false /* isdir */, 0 /* block_replication */, 0 /* blocksize */, commitInfo.fileModificationTimestamp, new Path(basePath, commitInfo.fileName)); return new Commit(commitInfo.version, fileStatus, commitInfo.timestamp); } } // ============================== // Protocol Class // ============================== static class Protocol { Integer minReaderVersion; Integer minWriterVersion; @JsonInclude(JsonInclude.Include.NON_EMPTY) List readerFeatures; @JsonInclude(JsonInclude.Include.NON_EMPTY) List writerFeatures; static Protocol fromAbstractProtocol(AbstractProtocol externalProtocol) { if (externalProtocol == null) { throw new IllegalArgumentException("externalProtocol cannot be null"); } Protocol protocol = new Protocol(); protocol.minReaderVersion = externalProtocol.getMinReaderVersion(); protocol.minWriterVersion = externalProtocol.getMinWriterVersion(); protocol.readerFeatures = new ArrayList<>(externalProtocol.getReaderFeatures()); protocol.writerFeatures = new ArrayList<>(externalProtocol.getWriterFeatures()); return protocol; } } // ============================== // Metadata Class // ============================== static class Metadata { String deltaTableId; String name; String description; String provider; OptionsKVPairs formatOptions; ColumnInfos schema; List partitionColumns; PropertiesKVPairs properties; String createdTime; static Metadata fromAbstractMetadata(AbstractMetadata externalMetadata) { if (externalMetadata == null) { throw new IllegalArgumentException("externalMetadata cannot be null"); } Metadata metadata = new Metadata(); metadata.deltaTableId = externalMetadata.getId(); metadata.name = externalMetadata.getName(); metadata.description = externalMetadata.getDescription(); metadata.provider = externalMetadata.getProvider(); metadata.formatOptions = OptionsKVPairs.fromFormatOptions( externalMetadata.getFormatOptions()); metadata.schema = ColumnInfos.fromSchemaString(externalMetadata.getSchemaString()); metadata.partitionColumns = externalMetadata.getPartitionColumns(); metadata.properties = PropertiesKVPairs.fromProperties(externalMetadata.getConfiguration()); metadata.createdTime = externalMetadata.getCreatedTime().toString(); // Assuming ISO format return metadata; } } // ============================== // OptionsKVPairs Class // ============================== static class OptionsKVPairs { Map options; static OptionsKVPairs fromFormatOptions(Map externalOptions) { if (externalOptions == null) { throw new IllegalArgumentException("externalOptions cannot be null"); } OptionsKVPairs kvPairs = new OptionsKVPairs(); kvPairs.options = externalOptions; return kvPairs; } } // ============================== // PropertiesKVPairs Class // ============================== static class PropertiesKVPairs { Map properties; static PropertiesKVPairs fromProperties(Map externalProperties) { if (externalProperties == null) { throw new IllegalArgumentException("externalProperties cannot be null"); } PropertiesKVPairs kvPairs = new PropertiesKVPairs(); kvPairs.properties = externalProperties; return kvPairs; } } // ============================== // ColumnInfos Class // ============================== static class ColumnInfos { List columns; static ColumnInfos fromSchemaString(String schemaString) { // TODO: Implement actual schema parsing logic based on schema format return null; } static class ColumnInfo { String name; String type; Boolean nullable; static ColumnInfo fromColumnDetails(String name, String type, Boolean nullable) { if (name == null || type == null || nullable == null) { throw new IllegalArgumentException("Column details cannot be null"); } ColumnInfo columnInfo = new ColumnInfo(); columnInfo.name = name; columnInfo.type = type; columnInfo.nullable = nullable; return columnInfo; } } } // ============================== // IcebergMetadata Class // ============================== static class IcebergMetadata { String metadataLocation; Long convertedDeltaVersion; String convertedDeltaTimestamp; static IcebergMetadata fromIcebergMetadata(io.delta.storage.commit.uniform.IcebergMetadata icebergMetadata) { if (icebergMetadata == null) { throw new IllegalArgumentException("icebergMetadata cannot be null"); } IcebergMetadata iceberg = new IcebergMetadata(); iceberg.metadataLocation = icebergMetadata.getMetadataLocation(); iceberg.convertedDeltaVersion = icebergMetadata.getConvertedDeltaVersion(); iceberg.convertedDeltaTimestamp = icebergMetadata.getConvertedDeltaTimestamp(); return iceberg; } } // ============================== // Uniform Class // ============================== static class Uniform { @JsonInclude(JsonInclude.Include.NON_EMPTY) IcebergMetadata iceberg; static Uniform fromUniformMetadata(UniformMetadata uniformMetadata) { Uniform uniform = new Uniform(); uniformMetadata.getIcebergMetadata().ifPresent( icebergMeta -> uniform.iceberg = IcebergMetadata.fromIcebergMetadata(icebergMeta)); return uniform; } } // ============================== // CommitRequest Class // ============================== static class CommitRequest { String tableId; String tableUri; CommitInfo commitInfo; Long latestBackfilledVersion; Metadata metadata; Protocol protocol; @JsonInclude(JsonInclude.Include.NON_NULL) Uniform uniform; } // ============================== // GetCommitsRequest Class // ============================== static class GetCommitsRequest { String tableId; String tableUri; Long startVersion; Long endVersion; } // ============================== // RestGetCommitsResponse Class // ============================== static class RestGetCommitsResponse { public List commits; public Long latestTableVersion; } // ============================== // GetMetastoreSummaryResponse Class // ============================== static class GetMetastoreSummaryResponse { String metastoreId; } } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCTokenBasedRestClient.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit.uccommitcoordinator; import io.delta.storage.commit.Commit; import io.delta.storage.commit.CommitFailedException; import io.delta.storage.commit.CoordinatedCommitsUtils; import io.delta.storage.commit.GetCommitsResponse; import io.delta.storage.commit.actions.AbstractMetadata; import io.delta.storage.commit.actions.AbstractProtocol; import io.delta.storage.commit.uniform.IcebergMetadata; import io.delta.storage.commit.uniform.UniformMetadata; import io.unitycatalog.client.ApiClient; import io.unitycatalog.client.ApiClientBuilder; import io.unitycatalog.client.ApiException; import io.unitycatalog.client.api.DeltaCommitsApi; import io.unitycatalog.client.api.MetastoresApi; import io.unitycatalog.client.auth.TokenProvider; import io.unitycatalog.client.model.DeltaCommit; import io.unitycatalog.client.model.DeltaCommitInfo; import io.unitycatalog.client.model.DeltaCommitMetadataProperties; import io.unitycatalog.client.model.DeltaGetCommits; import io.unitycatalog.client.model.DeltaGetCommitsResponse; import io.unitycatalog.client.model.DeltaMetadata; import io.unitycatalog.client.model.DeltaUniform; import io.unitycatalog.client.model.DeltaUniformIceberg; import io.unitycatalog.client.model.GetMetastoreSummaryResponse; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import java.io.IOException; import java.net.URI; import java.util.*; /** * A REST client implementation of {@link UCClient} for interacting with Unity Catalog's commit * coordination service. This client uses the Unity Catalog SDK with TokenProvider-based * authentication for managing Delta table commits and metadata. * *

The client handles the following primary operations: *

    *
  • Retrieving metastore information
    • *
  • Committing changes to Delta tables
    • *
  • Fetching unbackfilled commit histories
    • *
* *

All requests are authenticated using a TokenProvider that generates Bearer tokens dynamically. * The client uses the Unity Catalog SDK's {@link DeltaCommitsApi} and {@link MetastoresApi} for * API interactions. * *

Usage example: * 

{@code
 * TokenProvider tokenProvider = ... // Create or configure TokenProvider
 * try (UCTokenBasedRestClient client = new UCTokenBasedRestClient(baseUri, tokenProvider, Map.of())) {
 *     String metastoreId = client.getMetastoreId();
 *     // Perform operations with the client...
 * }
 * }
* * @see UCClient * @see Commit * @see GetCommitsResponse * @see TokenProvider */ public class UCTokenBasedRestClient implements UCClient { private DeltaCommitsApi deltaCommitsApi; private MetastoresApi metastoresApi; // HTTP status codes for error handling private static final int HTTP_BAD_REQUEST = 400; private static final int HTTP_NOT_FOUND = 404; private static final int HTTP_CONFLICT = 409; private static final int HTTP_TOO_MANY_REQUESTS = 429; /** * Constructs a new UCTokenBasedRestClient with the specified base URI, TokenProvider, * and application version information for telemetry. * * @param baseUri The base URI of the Unity Catalog server * @param tokenProvider The TokenProvider to use for authentication * @param appVersions A map of application name to version string * (e.g. {@code "Delta" -> "4.0.0"}). Each entry is * registered for User-Agent telemetry. May be empty. */ public UCTokenBasedRestClient( String baseUri, TokenProvider tokenProvider, Map appVersions) { Objects.requireNonNull(baseUri, "baseUri must not be null"); Objects.requireNonNull(tokenProvider, "tokenProvider must not be null"); Objects.requireNonNull(appVersions, "appVersions must not be null"); ApiClientBuilder builder = ApiClientBuilder.create() .uri(baseUri) .tokenProvider(tokenProvider); appVersions.forEach((name, version) -> { if (version != null) { builder.addAppVersion(name, version); } }); ApiClient apiClient = builder.build(); this.deltaCommitsApi = new DeltaCommitsApi(apiClient); this.metastoresApi = new MetastoresApi(apiClient); } /** * Ensures the client has not been closed. Must be called before any API operation. */ private void ensureOpen() { if (deltaCommitsApi == null || metastoresApi == null) { throw new IllegalStateException("UCTokenBasedRestClient has been closed."); } } @Override public String getMetastoreId() throws IOException { ensureOpen(); try { GetMetastoreSummaryResponse response = metastoresApi.summary(); return response.getMetastoreId(); } catch (ApiException e) { throw new IOException( String.format("Failed to get metastore ID (HTTP %s): ", e.getCode()), e); } } @Override public void commit( String tableId, URI tableUri, Optional commit, Optional lastKnownBackfilledVersion, boolean disown, Optional newMetadata, Optional newProtocol, Optional uniform ) throws IOException, CommitFailedException, UCCommitCoordinatorException { ensureOpen(); Objects.requireNonNull(tableId, "tableId must not be null."); Objects.requireNonNull(tableUri, "tableUri must not be null."); // Build the DeltaCommit request using SDK models DeltaCommit deltaCommit = new DeltaCommit() .tableId(tableId) .tableUri(tableUri.toString()); // Add commit info if present commit.ifPresent(c -> deltaCommit.commitInfo(toDeltaCommitInfo(c))); // Add latest backfilled version if present lastKnownBackfilledVersion.ifPresent(deltaCommit::latestBackfilledVersion); // Add metadata if present newMetadata.ifPresent(m -> deltaCommit.metadata(toDeltaMetadata(m))); // Add uniform metadata if present uniform.flatMap(u -> u.getIcebergMetadata().map(this::toDeltaUniformIceberg)) .ifPresent(iceberg -> deltaCommit.uniform(new DeltaUniform().iceberg(iceberg))); // Note: protocol and disown are not part of the DeltaCommit schema in the Unity Catalog // OpenAPI spec. They are intentionally not sent. try { deltaCommitsApi.commit(deltaCommit); } catch (ApiException e) { handleCommitException(e); } } @Override public GetCommitsResponse getCommits( String tableId, URI tableUri, Optional startVersion, Optional endVersion) throws IOException, UCCommitCoordinatorException { ensureOpen(); Objects.requireNonNull(tableId, "tableId must not be null."); Objects.requireNonNull(tableUri, "tableUri must not be null."); // Build the DeltaGetCommits request using SDK models DeltaGetCommits request = new DeltaGetCommits() .tableId(tableId) .tableUri(tableUri.toString()) .startVersion(startVersion.orElse(0L)); endVersion.ifPresent(request::endVersion); try { DeltaGetCommitsResponse response = deltaCommitsApi.getCommits(request); return toGetCommitsResponse(response, tableUri); } catch (ApiException e) { int statusCode = e.getCode(); String responseBody = e.getResponseBody(); if (statusCode == HTTP_NOT_FOUND) { throw new InvalidTargetTableException( String.format("Invalid Target Table (HTTP %s) due to: %s", statusCode, responseBody)); } else { throw new IOException( String.format("Unexpected getCommits failure (HTTP %s): due to: %s", statusCode, responseBody), e); } } } @Override public void close() throws IOException { // Nulling out the API instances makes them eligible for GC. Once garbage collected, // the underlying connection pool is freed and destroyed. this.deltaCommitsApi = null; this.metastoresApi = null; } /** * Converts a Delta {@link Commit} to a Unity Catalog SDK {@link DeltaCommitInfo}. * * @param commit The Delta commit to convert * @return The converted DeltaCommitInfo */ private DeltaCommitInfo toDeltaCommitInfo(Commit commit) { if (commit == null) { throw new IllegalArgumentException("commit cannot be null"); } if (commit.getFileStatus() == null) { throw new IllegalArgumentException("commit.getFileStatus() cannot be null"); } return new DeltaCommitInfo() .version(commit.getVersion()) .timestamp(commit.getCommitTimestamp()) .fileName(commit.getFileStatus().getPath().getName()) .fileSize(commit.getFileStatus().getLen()) .fileModificationTimestamp(commit.getFileStatus().getModificationTime()); } /** * Converts a Delta {@link IcebergMetadata} to a Unity Catalog SDK * {@link DeltaUniformIceberg}. * *

Field mapping (Delta internal -> OpenAPI snake_case): *

    *
  • metadataLocation -> metadata_location
    • *
  • convertedDeltaVersion -> converted_delta_version
    • *
  • convertedDeltaTimestamp -> converted_delta_timestamp
    • *
*/ private DeltaUniformIceberg toDeltaUniformIceberg(IcebergMetadata iceberg) { return new DeltaUniformIceberg() .metadataLocation(URI.create(iceberg.getMetadataLocation())) .convertedDeltaVersion(iceberg.getConvertedDeltaVersion()) .convertedDeltaTimestamp(iceberg.getConvertedDeltaTimestamp()); } /** * Converts an {@link AbstractMetadata} to a Unity Catalog SDK {@link DeltaMetadata}. * * @param metadata The abstract metadata to convert * @return The converted DeltaMetadata */ private DeltaMetadata toDeltaMetadata(AbstractMetadata metadata) { if (metadata == null) { throw new IllegalArgumentException("metadata cannot be null"); } DeltaMetadata deltaMetadata = new DeltaMetadata() .description(metadata.getDescription()); // Set properties if available if (metadata.getConfiguration() != null && !metadata.getConfiguration().isEmpty()) { DeltaCommitMetadataProperties properties = new DeltaCommitMetadataProperties() .properties(metadata.getConfiguration()); deltaMetadata.properties(properties); } // Schema conversion is not directly supported as the SDK expects ColumnInfos // which requires parsing the schema string. For now, we skip schema conversion. // If needed, implement schema string parsing to ColumnInfos. return deltaMetadata; } /** * Converts a Unity Catalog SDK {@link DeltaGetCommitsResponse} to a Delta * {@link GetCommitsResponse}. * * @param response The SDK response to convert * @param tableUri The table URI for constructing file paths * @return The converted GetCommitsResponse */ private GetCommitsResponse toGetCommitsResponse(DeltaGetCommitsResponse response, URI tableUri) { Path basePath = CoordinatedCommitsUtils.commitDirPath( CoordinatedCommitsUtils.logDirPath(new Path(tableUri))); List commits = new ArrayList<>(); for (DeltaCommitInfo commitInfo : response.getCommits()) { commits.add(fromDeltaCommitInfo(commitInfo, basePath)); } return new GetCommitsResponse(commits, response.getLatestTableVersion()); } /** * Converts a Unity Catalog SDK {@link DeltaCommitInfo} to a Delta {@link Commit}. * * @param commitInfo The SDK commit info to convert * @param basePath The base path for constructing file paths * @return The converted Commit */ private Commit fromDeltaCommitInfo(DeltaCommitInfo commitInfo, Path basePath) { FileStatus fileStatus = new FileStatus( commitInfo.getFileSize(), false /* isdir */, 0 /* block_replication */, 0 /* blocksize */, commitInfo.getFileModificationTimestamp(), new Path(basePath, commitInfo.getFileName())); return new Commit(commitInfo.getVersion(), fileStatus, commitInfo.getTimestamp()); } // =========================== // Exception Handling Methods // =========================== /** * Handles {@link ApiException} from commit operations by converting to appropriate Delta * exceptions. * * @param e The API exception to handle * @throws CommitFailedException If the commit failed due to various reasons * @throws UCCommitCoordinatorException If there's a UC-specific error * @throws IOException If there's an unexpected error */ private void handleCommitException(ApiException e) throws CommitFailedException, UCCommitCoordinatorException { int statusCode = e.getCode(); String responseBody = e.getResponseBody(); switch (statusCode) { case HTTP_BAD_REQUEST: throw new CommitFailedException( false /* retryable */, false /* conflict */, "Invalid commit parameters: " + responseBody, e); case HTTP_NOT_FOUND: throw new InvalidTargetTableException("Invalid Target Table: " + responseBody); case HTTP_CONFLICT: throw new CommitFailedException( true /* retryable */, true /* conflict */, "Commit conflict: " + responseBody, e); case HTTP_TOO_MANY_REQUESTS: throw new CommitLimitReachedException("Backfilled commits limit reached: " + responseBody); default: throw new CommitFailedException( true /* retryable */, false /* conflict */, "Unexpected commit failure (HTTP " + statusCode + "): " + responseBody, e); } } } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UpgradeNotAllowedException.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit.uccommitcoordinator; /** * This exception is thrown by the UC client in case the client attempted an upgrade * of a table to a UC managed table but the upgrade is not allowed because the previous * commit was a downgrade. */ public class UpgradeNotAllowedException extends UCCommitCoordinatorException { public UpgradeNotAllowedException(String message) { super(message); } } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/uniform/IcebergMetadata.java ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit.uniform; import java.util.Objects; /** * Metadata for Delta Uniform Iceberg conversion. * *

This class contains information about the latest Iceberg conversion for a Delta table, * which is sent to Unity Catalog to track the Iceberg metadata state. */ public class IcebergMetadata { private final String metadataLocation; private final long convertedDeltaVersion; private final String convertedDeltaTimestamp; /** * Constructs IcebergMetadata with the specified conversion details. * * @param metadataLocation The Iceberg metadata file location (e.g., "s3://bucket/metadata/v1.json") * @param convertedDeltaVersion The Delta version that was converted (e.g., 1044) * @param convertedDeltaTimestamp The timestamp of the conversion (e.g., "2025-01-04T03:13:11.423") */ public IcebergMetadata( String metadataLocation, long convertedDeltaVersion, String convertedDeltaTimestamp) { this.metadataLocation = Objects.requireNonNull(metadataLocation, "metadataLocation is null"); this.convertedDeltaVersion = convertedDeltaVersion; this.convertedDeltaTimestamp = Objects.requireNonNull(convertedDeltaTimestamp, "convertedDeltaTimestamp is null"); } /** Returns the Iceberg metadata file location. */ public String getMetadataLocation() { return metadataLocation; } /** Returns the Delta version that was converted to Iceberg. */ public long getConvertedDeltaVersion() { return convertedDeltaVersion; } /** Returns the timestamp when the conversion occurred. */ public String getConvertedDeltaTimestamp() { return convertedDeltaTimestamp; } } ================================================ FILE: storage/src/main/java/io/delta/storage/commit/uniform/UniformMetadata.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit.uniform; import java.util.Optional; /** * Metadata for Delta Universal Format (UniForm) conversions. * *

UniForm allows Delta tables to be read by other table formats. This class contains * conversion metadata for supported formats that is sent to Unity Catalog. */ public class UniformMetadata { private final IcebergMetadata icebergMetadata; // Future: private final HudiMetadata hudiMetadata; /** * Constructs UniformMetadata with Iceberg conversion metadata. * * @param icebergMetadata The Iceberg conversion metadata (can be null if not enabled) */ public UniformMetadata(IcebergMetadata icebergMetadata) { this.icebergMetadata = icebergMetadata; } /** * Returns the Iceberg metadata if Iceberg conversion is enabled. * * @return Optional containing Iceberg metadata, or empty if not enabled */ public Optional getIcebergMetadata() { return Optional.ofNullable(icebergMetadata); } // Future: public Optional getHudiMetadata() { ... } } ================================================ FILE: storage/src/main/java/io/delta/storage/internal/FileNameUtils.java ================================================ package io.delta.storage.internal; import java.util.regex.Pattern; import org.apache.hadoop.fs.Path; /** * Helper for misc functions relating to file names for delta commits. */ public final class FileNameUtils { static Pattern DELTA_FILE_PATTERN = Pattern.compile("\\d+\\.json"); /** * Returns the delta (json format) path for a given delta file. */ public static Path deltaFile(Path path, long version) { return new Path(path, String.format("%020d.json", version)); } /** * Returns the version for the given delta path. */ public static long deltaVersion(Path path) { return Long.parseLong(path.getName().split("\\.")[0]); } /** * Returns true if the given path is a delta file, else false. */ public static boolean isDeltaFile(Path path) { return DELTA_FILE_PATTERN.matcher(path.getName()).matches(); } } ================================================ FILE: storage/src/main/java/io/delta/storage/internal/LogStoreErrors.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.internal; import java.io.IOException; public class LogStoreErrors { /** * Returns true if the provided Throwable is to be considered non-fatal, or false if it is to be * considered fatal */ public static boolean isNonFatal(Throwable t) { // VirtualMachineError includes OutOfMemoryError and other fatal errors if (t instanceof VirtualMachineError || t instanceof ThreadDeath || t instanceof InterruptedException || t instanceof LinkageError) { return false; } return true; } /** * Returns true if the provided Throwable is to be considered fatal, or false if it is to be * considered non-fatal */ public static boolean isFatal(Throwable t) { return !isNonFatal(t); } public static IOException incorrectLogStoreImplementationException(Throwable cause) { return new IOException( String.join("\n", "The error typically occurs when the default LogStore implementation, that", "is, HDFSLogStore, is used to write into a Delta table on a non-HDFS storage system.", "In order to get the transactional ACID guarantees on table updates, you have to use the", "correct implementation of LogStore that is appropriate for your storage system.", "See https://docs.delta.io/latest/delta-storage.html for details." ), cause ); } } ================================================ FILE: storage/src/main/java/io/delta/storage/internal/PathLock.java ================================================ package io.delta.storage.internal; import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.fs.Path; /** * A lock that provides per-file-path `acquire` and `release` semantics. Can be used to ensure that * no two writers are creating the same external (e.g. S3) file at the same time. *

* Note: For all APIs, the caller should resolve the path to make sure we are locking the correct * absolute path. */ public class PathLock { private final ConcurrentHashMap pathLock; public PathLock() { this.pathLock = new ConcurrentHashMap<>(); } /** Release the lock for the path after writing. */ public void release(Path resolvedPath) { final Object lock = pathLock.remove(resolvedPath); synchronized(lock) { lock.notifyAll(); } } /** Acquire a lock for the path before writing. */ public void acquire(Path resolvedPath) throws InterruptedException { while (true) { final Object lock = pathLock.putIfAbsent(resolvedPath, new Object()); if (lock == null) { return; } synchronized (lock) { while (pathLock.get(resolvedPath) == lock) { lock.wait(); } } } } } ================================================ FILE: storage/src/main/java/io/delta/storage/internal/S3LogStoreUtil.java ================================================ /* * Copyright (2022) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.internal; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.s3a.*; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.HashSet; import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_MAX_PAGING_KEYS; import static org.apache.hadoop.fs.s3a.Constants.MAX_PAGING_KEYS; import static org.apache.hadoop.fs.s3a.S3AUtils.iteratorToStatuses; /** * Static utility methods for the S3SingleDriverLogStore. * * Used to trick the class loader so we can use methods of org.apache.hadoop:hadoop-aws without needing to load this as * a dependency for tests in core. */ public final class S3LogStoreUtil { private S3LogStoreUtil() {} private static PathFilter ACCEPT_ALL = new PathFilter() { @Override public boolean accept(Path file) { return true; } @Override public String toString() { return "ACCEPT_ALL"; } }; /** * Uses the S3ListRequest.v2 interface with the startAfter parameter to only list files * which are lexicographically greater than resolvedPath. */ private static RemoteIterator s3ListFrom( S3AFileSystem s3afs, Path resolvedPath, Path parentPath) throws IOException { int maxKeys = S3AUtils.intOption(s3afs.getConf(), MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS, 1); Listing listing = s3afs.getListing(); // List files lexicographically after resolvedPath inclusive within the same directory return listing.createFileStatusListingIterator(resolvedPath, S3ListRequest.v2( ListObjectsV2Request.builder() .bucket(s3afs.getBucket()) .maxKeys(maxKeys) .prefix(s3afs.pathToKey(parentPath)) .startAfter(keyBefore(s3afs.pathToKey(resolvedPath))) .build() ), ACCEPT_ALL, new Listing.AcceptAllButSelfAndS3nDirs(parentPath), s3afs.getActiveAuditSpan()); } /** * Uses the S3ListRequest.v2 interface with the startAfter parameter to only list files * which are lexicographically greater than resolvedPath. * * Wraps s3ListFrom in an array. Contained in this class to avoid contaminating other * classes with dependencies on recent Hadoop versions. * * TODO: Remove this method when iterators are used everywhere. */ public static FileStatus[] s3ListFromArray( FileSystem fs, Path resolvedPath, Path parentPath) throws IOException { S3AFileSystem s3afs; try { s3afs = (S3AFileSystem) fs; } catch (ClassCastException e) { throw new UnsupportedOperationException( "The Hadoop file system used for the S3LogStore must be castable to " + "org.apache.hadoop.fs.s3a.S3AFileSystem.", e); } return iteratorToStatuses(S3LogStoreUtil.s3ListFrom(s3afs, resolvedPath, parentPath)); } /** * Get the key which is lexicographically right before key. * If the key is empty return null. * If the key ends in a null byte, remove the last byte. * Otherwise, subtract one from the last byte. */ static String keyBefore(String key) { byte[] bytes = key.getBytes(StandardCharsets.UTF_8); if(bytes.length == 0) return null; if(bytes[bytes.length - 1] > 0) { bytes[bytes.length - 1] -= 1; return new String(bytes, StandardCharsets.UTF_8); } else { return new String(bytes, 0, bytes.length - 1, StandardCharsets.UTF_8); } } } ================================================ FILE: storage/src/main/java/io/delta/storage/internal/ThreadUtils.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.internal; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.Callable; public final class ThreadUtils { /** * Based on Apache Spark's ThreadUtils.runInNewThread * Run a piece of code in a new thread and return the result. */ public static T runInNewThread( String threadName, boolean isDaemon, Callable body) throws Throwable { // Using a single-element list to hold the throwable and result, // since values used in static method must be final List exceptionHolder = new ArrayList<>(1); List resultHolder = new ArrayList<>(1); Thread thread = new Thread(threadName) { @Override public void run() { try { resultHolder.add(body.call()); } catch (Throwable t) { exceptionHolder.add(t); } } }; thread.setDaemon(isDaemon); thread.start(); thread.join(); if (!exceptionHolder.isEmpty()) { Throwable realException = exceptionHolder.get(0); // Remove the part of the stack that shows method calls into this helper method // This means drop everything from the top until the stack element // ThreadUtils.runInNewThread(), and then drop that as well (hence the `drop(1)`). List baseStackTrace = new ArrayList<>(); boolean shouldDrop = true; for (StackTraceElement st : Thread.currentThread().getStackTrace()) { if (!shouldDrop) { baseStackTrace.add(st); } else if (st.getClassName().contains(ThreadUtils.class.getSimpleName())){ shouldDrop = false; } } // Remove the part of the new thread stack that shows methods call from this helper // method. This means take everything from the top until the stack element List extraStackTrace = new ArrayList<>(); for (StackTraceElement st : realException.getStackTrace()) { if (!st.getClassName().contains(ThreadUtils.class.getSimpleName())) { extraStackTrace.add(st); } else { break; } } // Combine the two stack traces, with a placeholder just specifying that there // was a helper method used, without any further details of the helper StackTraceElement placeHolderStackElem = new StackTraceElement( String.format( // Providing the helper class info. "... run in separate thread using %s static method runInNewThread", ThreadUtils.class.getSimpleName() ), " ", // method name containing the execution point, not required here. "", // filename containing the execution point, not required here. -1); // source line number also not required. -1 indicates unavailable. List finalStackTrace = new ArrayList<>(); finalStackTrace.addAll(extraStackTrace); finalStackTrace.add(placeHolderStackElem); finalStackTrace.addAll(baseStackTrace); // Update the stack trace and rethrow the exception in the caller thread realException.setStackTrace( finalStackTrace.toArray(new StackTraceElement[0]) ); throw realException; } else { return resultHolder.get(0); } } } ================================================ FILE: storage/src/test/scala/io/delta/storage/ThreadUtilsSuite.scala ================================================ package io.delta.storage import java.io.IOException import scala.util.Random import org.scalatest.funsuite.AnyFunSuite class ThreadUtilsSuite extends AnyFunSuite { test("runInNewThread") { import io.delta.storage.internal.ThreadUtils.runInNewThread assert(runInNewThread("thread-name", true, () => { Thread.currentThread().getName }) === "thread-name" ) assert(runInNewThread("thread-name", true, () => { Thread.currentThread().isDaemon }) ) assert(runInNewThread("thread-name", false, () => { Thread.currentThread().isDaemon } === false) ) val ioExceptionMessage = "test" + Random.nextInt() val ioException = intercept[IOException] { runInNewThread("thread-name", true, () => { throw new IOException(ioExceptionMessage) }) } assert(ioException.getMessage === ioExceptionMessage) assert(ioException.getStackTrace.mkString("\n") .contains("... run in separate thread using ThreadUtils")) assert(!ioException.getStackTrace.mkString("\n").contains("ThreadUtils.java")) } } ================================================ FILE: storage/src/test/scala/io/delta/storage/commit/InMemoryCommitCoordinator.scala ================================================ /* * Copyright (2024) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit import java.lang.{Long => JLong} import java.nio.file.FileAlreadyExistsException import java.util.{ArrayList, Collections, Iterator => JIterator, Map => JMap, Optional, TreeMap, UUID} import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.locks.ReentrantReadWriteLock import io.delta.storage.LogStore import io.delta.storage.commit.CoordinatedCommitsUtils import io.delta.storage.commit.actions.AbstractMetadata import io.delta.storage.commit.actions.AbstractProtocol import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.FileStatus import org.apache.hadoop.fs.Path import org.slf4j.Logger import org.slf4j.LoggerFactory class InMemoryCommitCoordinator(val batchSize: Long) extends CommitCoordinatorClient { protected val logger: Logger = LoggerFactory.getLogger(classOf[InMemoryCommitCoordinator]) /** * @param maxCommitVersion represents the max commit version known for the table. This is * initialized at the time of pre-registration and updated whenever a * commit is successfully added to the commit-coordinator. * @param active represents whether this commit-coordinator has ratified any commit or not. * |----------------------------|------------------|---------------------------| * | State | maxCommitVersion | active | * |----------------------------|------------------|---------------------------| * | Table is pre-registered | currentVersion+1 | false | * |----------------------------|------------------|---------------------------| * | Table is pre-registered | X | true | * | and more commits are done | | | * |----------------------------|------------------|---------------------------| */ private[commit] class PerTableData( var maxCommitVersion: Long = -1, var active: Boolean = false ) { def updateLastRatifiedCommit(commitVersion: Long): Unit = { this.active = true this.maxCommitVersion = commitVersion } /** * Returns the last ratified commit version for the table. If no commits have been done from * commit-coordinator yet, returns -1. */ def lastRatifiedCommitVersion: Long = if (!active) -1 else maxCommitVersion // Map from version to Commit data val commitsMap: TreeMap[Long, Commit] = new TreeMap[Long, Commit] // We maintain maxCommitVersion explicitly since commitsMap might be empty // if all commits for a table have been backfilled. val lock: ReentrantReadWriteLock = new ReentrantReadWriteLock() } private[commit] val perTableMap = new ConcurrentHashMap[String, PerTableData]() override def registerTable( logPath: Path, tableIdentifier: Optional[TableIdentifier], currentVersion: Long, currentMetadata: AbstractMetadata, currentProtocol: AbstractProtocol): JMap[String, String] = { val newPerTableData = new PerTableData(currentVersion + 1) perTableMap.compute(logPath.toString, (_, existingData) => { if (existingData != null) { if (existingData.lastRatifiedCommitVersion != -1) { throw new IllegalStateException( s"Table $logPath already exists in the commit-coordinator.") } // If lastRatifiedCommitVersion is -1 i.e. the commit-coordinator has never // attempted any commit for this table => this table was just pre-registered. If // there is another pre-registration request for an older version, we reject it and // table can't go backward. if (currentVersion < existingData.maxCommitVersion) { throw new IllegalStateException( s"Table $logPath already registered with commit-coordinator") } } newPerTableData }) Collections.emptyMap[String, String]() } override def commit( logStore: LogStore, hadoopConf: Configuration, tableDesc: TableDescriptor, commitVersion: Long, actions: JIterator[String], updatedActions: UpdatedActions): CommitResponse = { val logPath = tableDesc.getLogPath val tablePath = CoordinatedCommitsUtils.getTablePath(logPath) if (commitVersion == 0) { throw new CommitFailedException(false, false, "Commit version 0 must go via filesystem.") } logger.info("Attempting to commit version {} on table {}", commitVersion, tablePath) val fs = logPath.getFileSystem(hadoopConf) if (batchSize <= 1) { // Backfill until `commitVersion - 1` logger.info( "Making sure commits are backfilled until {}" + " version for table {}", commitVersion - 1, tablePath) backfillToVersion(logStore, hadoopConf, tableDesc, commitVersion - 1, null) } // Write new commit file in `_staged_commits` directory val fileStatus = CoordinatedCommitsUtils.writeUnbackfilledCommitFile( logStore, hadoopConf, logPath.toString, commitVersion, actions, generateUUID()) // Do the actual commit val commitTimestamp = updatedActions.getCommitInfo.getCommitTimestamp val commitResponse = addToMap(logPath, commitVersion, fileStatus, commitTimestamp) val mcToFsConversion = CoordinatedCommitsUtils.isCoordinatedCommitsToFSConversion( commitVersion, updatedActions) // Backfill if needed if (batchSize <= 1) { // Always backfill when batch size is configured as 1 backfill(logStore, hadoopConf, logPath, commitVersion, fileStatus) val targetFile = CoordinatedCommitsUtils.getBackfilledDeltaFilePath(logPath, commitVersion) val targetFileStatus = fs.getFileStatus(targetFile) val newCommit = commitResponse.getCommit.withFileStatus(targetFileStatus) return new CommitResponse(newCommit) } else if (commitVersion % batchSize == 0 || mcToFsConversion) { logger.info( "Making sure commits are backfilled till {} version for table {}", commitVersion, tablePath) backfillToVersion(logStore, hadoopConf, tableDesc, commitVersion, null) } logger.info("Commit {} done successfully on table {}", commitVersion, tablePath) commitResponse } override def getCommits( tableDesc: TableDescriptor, startVersion: JLong, endVersion: JLong) : GetCommitsResponse = withReadLock[GetCommitsResponse](tableDesc.getLogPath) { val tableData = perTableMap.get(tableDesc.getLogPath.toString) val startVersionOpt = Optional.ofNullable(startVersion) val endVersionOpt = Optional.ofNullable(endVersion) val effectiveStartVersion = startVersionOpt.orElse(0L) // Calculate the end version for the range, or use the last key if endVersion is not // provided val effectiveEndVersion = endVersionOpt.orElseGet( () => if (tableData.commitsMap.isEmpty) effectiveStartVersion else tableData.commitsMap.lastKey) val commitsInRange = tableData.commitsMap.subMap(effectiveStartVersion, effectiveEndVersion + 1) new GetCommitsResponse( new ArrayList[Commit](commitsInRange.values), tableData.lastRatifiedCommitVersion) } override def backfillToVersion( logStore: LogStore, hadoopConf: Configuration, tableDesc: TableDescriptor, version: Long, lastKnownBackfilledVersion: JLong): Unit = { val logPath = tableDesc.getLogPath // Confirm the last backfilled version by checking the backfilled delta file's existence. var validLastKnownBackfilledVersion = lastKnownBackfilledVersion if (lastKnownBackfilledVersion != null) { val fs = logPath.getFileSystem(hadoopConf) if (!fs.exists(CoordinatedCommitsUtils.getBackfilledDeltaFilePath(logPath, version))) { validLastKnownBackfilledVersion = null } } var startVersion: JLong = null if (validLastKnownBackfilledVersion != null) startVersion = validLastKnownBackfilledVersion + 1 val commitsResponse = getCommits(tableDesc, startVersion, version) commitsResponse.getCommits.forEach((commit: Commit) => { backfill(logStore, hadoopConf, logPath, commit.getVersion, commit.getFileStatus) }) } override def semanticEquals(other: CommitCoordinatorClient): Boolean = this == other /** Backfills a given `fileStatus` to `version`.json */ protected def backfill( logStore: LogStore, hadoopConf: Configuration, logPath: Path, version: Long, fileStatus: FileStatus): Unit = { val targetFile = CoordinatedCommitsUtils.getBackfilledDeltaFilePath(logPath, version) logger.info("Backfilling commit " + fileStatus.getPath + " to " + targetFile) val commitContentIterator = logStore.read(fileStatus.getPath, hadoopConf) try { logStore.write(targetFile, commitContentIterator, false, hadoopConf) registerBackfill(logPath, version) } catch { case _: FileAlreadyExistsException => logger.info("The backfilled file " + targetFile + " already exists.") } finally commitContentIterator.close() } protected def generateUUID(): String = UUID.randomUUID().toString private def addToMap( logPath: Path, commitVersion: Long, commitFile: FileStatus, commitTimestamp: Long): CommitResponse = withWriteLock[CommitResponse](logPath) { val tableData = perTableMap.get(logPath.toString) val expectedVersion = tableData.maxCommitVersion + 1 if (commitVersion != expectedVersion) { throw new CommitFailedException( commitVersion < expectedVersion, commitVersion < expectedVersion, s"Commit version $commitVersion is not valid. Expected version: $expectedVersion.") } val commit = new Commit(commitVersion, commitFile, commitTimestamp) tableData.commitsMap.put(commitVersion, commit) tableData.updateLastRatifiedCommit(commitVersion) logger.info("Added commit file " + commitFile.getPath + " to commit-coordinator.") new CommitResponse(commit) } /** * Callback to tell the CommitCoordinator that all commits <= `backfilledVersion` are * backfilled. */ protected[delta] def registerBackfill(logPath: Path, backfilledVersion: Long): Unit = { withWriteLock(logPath) { val tableData = perTableMap.get(logPath.toString) if (backfilledVersion > tableData.lastRatifiedCommitVersion) { throw new IllegalArgumentException( "Unexpected backfill version: " + backfilledVersion + ". " + "Max backfill version: " + tableData.maxCommitVersion) } // Remove keys with versions less than or equal to 'untilVersion' val iterator = tableData.commitsMap.keySet.iterator while (iterator.hasNext) { val version = iterator.next if (version <= backfilledVersion) { iterator.remove() } else { return } } } } private[commit] def withReadLock[T](logPath: Path)(operation: => T): T = { val tableData = perTableMap.get(logPath.toString) if (tableData == null) { throw new IllegalArgumentException(s"Unknown table $logPath.") } val lock = tableData.lock.readLock() lock.lock() try { operation } finally { lock.unlock() } } private[commit] def withWriteLock[T](logPath: Path)(operation: => T): T = { val tableData = Option(perTableMap.get(logPath.toString)).getOrElse { throw new IllegalArgumentException(s"Unknown table $logPath.") } val lock = tableData.lock.writeLock() lock.lock() try { operation } finally { lock.unlock() } } } ================================================ FILE: storage/src/test/scala/io/delta/storage/commit/uccommitcoordinator/UCTokenBasedRestClientSuite.scala ================================================ /* * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.commit.uccommitcoordinator import java.net.{InetSocketAddress, URI} import java.nio.charset.StandardCharsets import java.util.{Collections, Optional} import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} import com.sun.net.httpserver.{HttpExchange, HttpServer} import io.delta.storage.commit.{Commit, CommitFailedException} import io.delta.storage.commit.actions.AbstractMetadata import io.delta.storage.commit.uniform.{IcebergMetadata, UniformMetadata} import io.unitycatalog.client.auth.TokenProvider import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.http.HttpStatus import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} import org.scalatest.funsuite.AnyFunSuite class UCTokenBasedRestClientSuite extends AnyFunSuite with BeforeAndAfterAll with BeforeAndAfterEach { private val testTableId = "test-table-id" private val testTableUri = new URI("s3://bucket/path/to/table") private val testMetastoreId = "test-metastore-123" private var server: HttpServer = _ private var serverUri: String = _ private var metastoreHandler: HttpExchange => Unit = _ private var commitsHandler: HttpExchange => Unit = _ private val objectMapper = new ObjectMapper() override def beforeAll(): Unit = { server = HttpServer.create(new InetSocketAddress("localhost", 0), 0) server.createContext("/api/2.1/unity-catalog/metastore_summary", exchange => { if (metastoreHandler != null) metastoreHandler(exchange) else sendJson(exchange, HttpStatus.SC_OK, s"""{"metastore_id":"$testMetastoreId"}""") exchange.close() }) server.createContext("/api/2.1/unity-catalog/delta/preview/commits", exchange => { if (commitsHandler != null) commitsHandler(exchange) else { val body = if (exchange.getRequestMethod == "POST") "{}" else """{"commits":[],"latest_table_version":-1}""" sendJson(exchange, HttpStatus.SC_OK, body) } exchange.close() }) server.start() serverUri = s"http://localhost:${server.getAddress.getPort}" } override def afterAll(): Unit = if (server != null) server.stop(0) override def beforeEach(): Unit = { metastoreHandler = null commitsHandler = null } private def readRequestBody(exchange: HttpExchange): String = { val is = exchange.getRequestBody try new String(is.readAllBytes(), StandardCharsets.UTF_8) finally is.close() } private def sendJson(exchange: HttpExchange, status: Int, body: String): Unit = { val bytes = body.getBytes(StandardCharsets.UTF_8) exchange.getResponseHeaders.add("Content-Type", "application/json") exchange.sendResponseHeaders(status, bytes.length) exchange.getResponseBody.write(bytes) exchange.getResponseBody.close() } private def createTokenProvider(): TokenProvider = new TokenProvider { override def accessToken(): String = "mock-token" override def initialize(configs: java.util.Map[String, String]): Unit = {} override def configs(): java.util.Map[String, String] = Collections.emptyMap() } private def createClient(): UCTokenBasedRestClient = new UCTokenBasedRestClient(serverUri, createTokenProvider(), Collections.emptyMap()) private def withClient(fn: UCTokenBasedRestClient => Unit): Unit = { val client = createClient() try fn(client) finally client.close() } private def createCommit(version: Long): Commit = { val fs = new FileStatus(1024L, false, 1, 4096L, System.currentTimeMillis(), new Path(s"/path/_delta_log/_staged_commits/$version.uuid.json")) new Commit(version, fs, System.currentTimeMillis()) } private def createMetadata(): AbstractMetadata = new AbstractMetadata { override def getId: String = "id" override def getName: String = "name" override def getDescription: String = "desc" override def getProvider: String = "delta" override def getFormatOptions: java.util.Map[String, String] = Collections.emptyMap() override def getSchemaString: String = """{"type":"struct","fields":[]}""" override def getPartitionColumns: java.util.List[String] = Collections.emptyList() override def getConfiguration: java.util.Map[String, String] = Collections.emptyMap() override def getCreatedTime: java.lang.Long = 0L } private def createUniformMetadata(): UniformMetadata = new UniformMetadata( new IcebergMetadata("s3://bucket/metadata/v1.json", 42L, "2025-01-04T03:13:11.423Z")) // Constructor tests test("constructor validates required parameters") { intercept[NullPointerException] { new UCTokenBasedRestClient(null, createTokenProvider(), Collections.emptyMap()) } intercept[NullPointerException] { new UCTokenBasedRestClient(serverUri, null, Collections.emptyMap()) } intercept[NullPointerException] { new UCTokenBasedRestClient(serverUri, createTokenProvider(), null) } } // getMetastoreId tests test("getMetastoreId returns ID on success") { withClient { client => assert(client.getMetastoreId() === testMetastoreId) } } test("getMetastoreId throws IOException on error") { metastoreHandler = exchange => sendJson(exchange, HttpStatus.SC_INTERNAL_SERVER_ERROR, "{}") withClient { client => intercept[java.io.IOException] { client.getMetastoreId() } } } // commit tests test("commit succeeds with valid parameters") { withClient { client => client.commit(testTableId, testTableUri, Optional.of(createCommit(1L)), Optional.empty(), false, Optional.empty(), Optional.empty(), Optional.empty()) } } test("commit succeeds with metadata") { withClient { client => client.commit( testTableId, testTableUri, Optional.of(createCommit(1L)), Optional.of(java.lang.Long.valueOf(0L)), true, Optional.of(createMetadata()), Optional.empty(), Optional.empty()) } } test("commit validates required parameters") { withClient { client => intercept[NullPointerException] { client.commit(null, testTableUri, Optional.empty(), Optional.empty(), false, Optional.empty(), Optional.empty(), Optional.empty()) } intercept[NullPointerException] { client.commit(testTableId, null, Optional.empty(), Optional.empty(), false, Optional.empty(), Optional.empty(), Optional.empty()) } } } test("commit throws appropriate exceptions for HTTP errors") { def commitWith(status: Int): Unit = { commitsHandler = exchange => sendJson(exchange, status, s"""{"error":"$status"}""") withClient { client => client.commit(testTableId, testTableUri, Optional.of(createCommit(1L)), Optional.empty(), false, Optional.empty(), Optional.empty(), Optional.empty()) } } // 400 -> CommitFailedException (non-retryable, non-conflict) val e400 = intercept[CommitFailedException] { commitWith(400) } assert(!e400.getRetryable && !e400.getConflict) // 404 -> InvalidTargetTableException intercept[InvalidTargetTableException] { commitWith(404) } // 409 -> CommitFailedException (retryable, conflict) val e409 = intercept[CommitFailedException] { commitWith(409) } assert(e409.getRetryable && e409.getConflict) // Note: 429 (CommitLimitReachedException) cannot be tested here because the SDK's // RetryingHttpClient intercepts 429 before it reaches handleCommitException. // The 429 path is exercised via UCCommitCoordinatorClientSuite integration tests. // 500 (default branch) -> CommitFailedException (retryable, non-conflict) val e500 = intercept[CommitFailedException] { commitWith(500) } assert(e500.getRetryable && !e500.getConflict) } // getCommits tests test("getCommits returns commits correctly") { val responseJson = """{"commits":[{"version":1,"file_name":"1.json","file_size":100,""" + """"timestamp":1000,"file_modification_timestamp":1001}],"latest_table_version":1}""" commitsHandler = exchange => sendJson(exchange, HttpStatus.SC_OK, responseJson) withClient { client => val response = client.getCommits( testTableId, testTableUri, Optional.empty(), Optional.empty()) assert(response.getCommits.size() === 1) assert(response.getCommits.get(0).getVersion === 1L) assert(response.getLatestTableVersion === 1L) } } test("getCommits validates required parameters") { withClient { client => intercept[NullPointerException] { client.getCommits(null, testTableUri, Optional.empty(), Optional.empty()) } intercept[NullPointerException] { client.getCommits(testTableId, null, Optional.empty(), Optional.empty()) } } } test("getCommits throws InvalidTargetTableException on 404") { commitsHandler = exchange => sendJson(exchange, HttpStatus.SC_NOT_FOUND, "{}") withClient { client => intercept[InvalidTargetTableException] { client.getCommits(testTableId, testTableUri, Optional.empty(), Optional.empty()) } } } // uniform tests test("commit with uniform.iceberg sends correct snake_case JSON per all.yaml") { var capturedBody: String = null commitsHandler = exchange => { capturedBody = readRequestBody(exchange) sendJson(exchange, HttpStatus.SC_OK, "{}") } withClient { client => client.commit(testTableId, testTableUri, Optional.of(createCommit(1L)), Optional.empty(), false, Optional.empty(), Optional.empty(), Optional.of(createUniformMetadata())) } val json: JsonNode = objectMapper.readTree(capturedBody) assert(json.get("table_id").asText() === testTableId) val iceberg = json.get("uniform").get("iceberg") assert(iceberg.get("metadata_location").asText() === "s3://bucket/metadata/v1.json") assert(iceberg.get("converted_delta_version").asLong() === 42L) assert(iceberg.get("converted_delta_timestamp").asText() === "2025-01-04T03:13:11.423Z") assert(!json.has("protocol"), "protocol is not in the OpenAPI spec and must not be sent") } test("commit without uniform does not include uniform field in JSON") { var capturedBody: String = null commitsHandler = exchange => { capturedBody = readRequestBody(exchange) sendJson(exchange, HttpStatus.SC_OK, "{}") } withClient { client => client.commit(testTableId, testTableUri, Optional.of(createCommit(1L)), Optional.empty(), false, Optional.empty(), Optional.empty(), Optional.empty()) } val json = objectMapper.readTree(capturedBody) assert(!json.has("uniform") || json.get("uniform").isNull) } test("commit with uniform but no iceberg metadata does not include uniform field") { var capturedBody: String = null commitsHandler = exchange => { capturedBody = readRequestBody(exchange) sendJson(exchange, HttpStatus.SC_OK, "{}") } withClient { client => client.commit(testTableId, testTableUri, Optional.of(createCommit(1L)), Optional.empty(), false, Optional.empty(), Optional.empty(), Optional.of(new UniformMetadata(null))) } val json = objectMapper.readTree(capturedBody) assert(!json.has("uniform") || json.get("uniform").isNull) } } ================================================ FILE: storage/src/test/scala/io/delta/storage/integration/S3LogStoreUtilIntegrationTest.scala ================================================ package io.delta.storage.integration import io.delta.storage.internal.{FileNameUtils, S3LogStoreUtil} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.hadoop.fs.s3a.S3AFileSystem import org.scalatest.Tag import org.scalatest.funsuite.AnyFunSuite import java.net.URI import scala.math.max import scala.math.ceil /** * These integration tests are executed by setting the * environment variables * S3_LOG_STORE_UTIL_TEST_BUCKET=some-s3-bucket-name * S3_LOG_STORE_UTIL_TEST_RUN_UID=some-uuid-for-test-run * and running * python run-integration-tests.py --s3-log-store-util-only * * Alternatively you can set the environment variables * S3_LOG_STORE_UTIL_TEST_ENABLED=true * S3_LOG_STORE_UTIL_TEST_BUCKET=some-s3-bucket-name * S3_LOG_STORE_UTIL_TEST_RUN_UID=some-uuid-for-test-run * and run the tests in this suite using your preferred * test execution mechanism (e.g., the IDE or sbt) * * S3_LOG_STORE_UTIL_TEST_BUCKET is the name of the S3 bucket used for the test. * S3_LOG_STORE_UTIL_TEST_RUN_UID is a prefix for all keys used in the test. * This is useful for isolating multiple test runs. */ class S3LogStoreUtilIntegrationTest extends AnyFunSuite { private val runIntegrationTests: Boolean = Option(System.getenv("S3_LOG_STORE_UTIL_TEST_ENABLED")).exists(_.toBoolean) private val bucket = System.getenv("S3_LOG_STORE_UTIL_TEST_BUCKET") private val testRunUID = System.getenv("S3_LOG_STORE_UTIL_TEST_RUN_UID") // Prefix for all S3 keys in the current run private lazy val fs: S3AFileSystem = { val fs = new S3AFileSystem() fs.initialize(new URI(s"s3a://$bucket"), configuration) fs } private val maxKeys = 2 private val configuration = new Configuration() configuration.set("fs.s3a.paging.maximum", maxKeys.toString) private def touch(key: String) { fs.create(new Path(s"s3a://$bucket/$key")).close() } private def key(table: String, version: Int): String = s"$testRunUID/$table/_delta_log/%020d.json".format(version) private def path(table: String, version: Int): Path = new Path(s"s3a://$bucket/${key(table, version)}") private def version(path: Path): Long = FileNameUtils.deltaVersion(path) private val integrationTestTag = Tag("IntegrationTest") def integrationTest(name: String)(testFun: => Any): Unit = if (runIntegrationTests) test(name, integrationTestTag)(testFun) def testCase(testName: String, numKeys: Int): Unit = integrationTest(testName) { // Setup delta log (1 to numKeys).foreach(v => touch(s"$testRunUID/$testName/_delta_log/%020d.json".format(v))) // Check number of S3 requests and correct listing (1 to numKeys + 2).foreach(v => { val startCount = fs.getIOStatistics.counters().get("object_list_request") + fs.getIOStatistics.counters().get("object_continue_list_request") val resolvedPath = path(testName, v) val response = S3LogStoreUtil.s3ListFromArray(fs, resolvedPath, resolvedPath.getParent) val endCount = fs.getIOStatistics.counters().get("object_list_request") + fs.getIOStatistics.counters().get("object_continue_list_request") // Check that we don't do more S3 list requests than necessary val numberOfKeysToList = numKeys - (v - 1) val optimalNumberOfListRequests = max(ceil(numberOfKeysToList / maxKeys.toDouble).toInt, 1) val actualNumberOfListRequests = endCount - startCount assert(optimalNumberOfListRequests == actualNumberOfListRequests) // Check that we get consecutive versions from v to the max version. The smallest version is 1 assert((max(1, v) to numKeys) == response.map(r => version(r.getPath)).toSeq) }) } integrationTest("setup empty delta log") { touch(s"$testRunUID/empty/some.json") } testCase("empty", 0) testCase("small", 1) testCase("medium", maxKeys) testCase("large", 10 * maxKeys) } ================================================ FILE: storage/src/test/scala/io/delta/storage/internal/S3LogStoreUtilTest.scala ================================================ package io.delta.storage.internal import org.scalatest.funsuite.AnyFunSuite class S3LogStoreUtilTest extends AnyFunSuite { test("keyBefore") { assert("a" == S3LogStoreUtil.keyBefore("b")) assert("aa/aa" == S3LogStoreUtil.keyBefore("aa/ab")) assert(Seq(1.toByte, 1.toByte) == S3LogStoreUtil.keyBefore(new String(Seq(1.toByte, 2.toByte).toArray)).getBytes.toList) } test("keyBefore with emojis") { assert("♥a" == S3LogStoreUtil.keyBefore("♥b")) } test("keyBefore with zero bytes") { assert("abc" == S3LogStoreUtil.keyBefore("abc\u0000")) } test("keyBefore with empty key") { assert(null == S3LogStoreUtil.keyBefore("")) } } ================================================ FILE: storage-s3-dynamodb/integration_tests/dynamodb_logstore.py ================================================ # # Copyright (2021) The Delta Lake Project Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os import sys import threading from pyspark.sql import SparkSession from multiprocessing.pool import ThreadPool import time """ Create required dynamodb table with: $ aws dynamodb create-table \ --region \ --table-name \ --attribute-definitions AttributeName=tablePath,AttributeType=S \ AttributeName=fileName,AttributeType=S \ --key-schema AttributeName=tablePath,KeyType=HASH \ AttributeName=fileName,KeyType=RANGE \ --provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5 Enable TTL with: $ aws dynamodb update-time-to-live \ --region \ --table-name \ --time-to-live-specification "Enabled=true, AttributeName=expireTime" Run this script in root dir of repository: # ===== Mandatory input from user ===== export RUN_ID=run001 export S3_BUCKET=delta-lake-dynamodb-test-00 # ===== Optional input from user ===== export DELTA_CONCURRENT_WRITERS=20 export DELTA_CONCURRENT_READERS=2 export DELTA_STORAGE=io.delta.storage.S3DynamoDBLogStore export DELTA_NUM_ROWS=200 export DELTA_DYNAMO_REGION=us-west-2 export DELTA_DYNAMO_ERROR_RATES=0.00 # ===== Optional input from user (we calculate defaults using S3_BUCKET and RUN_ID) ===== export RELATIVE_DELTA_TABLE_PATH=___ export DELTA_DYNAMO_TABLE_NAME=___ ./run-integration-tests.py --use-local \ --run-storage-s3-dynamodb-integration-tests \ --packages org.apache.hadoop:hadoop-aws:3.3.1 \ --dbb-conf io.delta.storage.credentials.provider=com.amazonaws.auth.profile.ProfileCredentialsProvider \ spark.hadoop.fs.s3a.aws.credentials.provider=com.amazonaws.auth.profile.ProfileCredentialsProvider """ # ===== Mandatory input from user ===== run_id = os.environ.get("RUN_ID") s3_bucket = os.environ.get("S3_BUCKET") # ===== Optional input from user ===== concurrent_writers = int(os.environ.get("DELTA_CONCURRENT_WRITERS", 2)) concurrent_readers = int(os.environ.get("DELTA_CONCURRENT_READERS", 2)) # className to instantiate. io.delta.storage.S3DynamoDBLogStore or .FailingS3DynamoDBLogStore delta_storage = os.environ.get("DELTA_STORAGE", "io.delta.storage.S3DynamoDBLogStore") num_rows = int(os.environ.get("DELTA_NUM_ROWS", 16)) dynamo_region = os.environ.get("DELTA_DYNAMO_REGION", "us-west-2") # used only by FailingS3DynamoDBLogStore dynamo_error_rates = os.environ.get("DELTA_DYNAMO_ERROR_RATES", "") # ===== Optional input from user (we calculate defaults using RUN_ID) ===== relative_delta_table_path = os.environ.get("RELATIVE_DELTA_TABLE_PATH", "tables/table_" + run_id)\ .rstrip("/") dynamo_table_name = os.environ.get("DELTA_DYNAMO_TABLE_NAME", "ddb_table_" + run_id) delta_table_path = "s3a://" + s3_bucket + "/" + relative_delta_table_path relative_delta_log_path = relative_delta_table_path + "/_delta_log/" if delta_table_path is None: print(f"\nSkipping Python test {os.path.basename(__file__)} due to the missing env variable " f"`DELTA_TABLE_PATH`\n=====================") sys.exit(0) test_log = f""" ========================================== run id: {run_id} delta table path: {delta_table_path} dynamo table name: {dynamo_table_name} concurrent writers: {concurrent_writers} concurrent readers: {concurrent_readers} number of rows: {num_rows} delta storage: {delta_storage} dynamo_error_rates: {dynamo_error_rates} relative_delta_table_path: {relative_delta_table_path} relative_delta_log_path: {relative_delta_log_path} ========================================== """ print(test_log) spark = SparkSession \ .builder \ .appName("utilities") \ .master("local[*]") \ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \ .config("spark.delta.logStore.s3.impl", delta_storage) \ .config("spark.delta.logStore.s3a.impl", delta_storage) \ .config("spark.delta.logStore.s3n.impl", delta_storage) \ .config("spark.io.delta.storage.S3DynamoDBLogStore.ddb.tableName", dynamo_table_name) \ .config("spark.io.delta.storage.S3DynamoDBLogStore.ddb.region", dynamo_region) \ .config("spark.io.delta.storage.S3DynamoDBLogStore.errorRates", dynamo_error_rates) \ .config("spark.io.delta.storage.S3DynamoDBLogStore.provisionedThroughput.rcu", 12) \ .config("spark.io.delta.storage.S3DynamoDBLogStore.provisionedThroughput.wcu", 13) \ .getOrCreate() # spark.sparkContext.setLogLevel("INFO") data = spark.createDataFrame([], "id: int, a: int") print("writing:", data.collect()) data.write.format("delta").mode("overwrite").partitionBy("id").save(delta_table_path) def write_tx(n): data = spark.createDataFrame([[n, n]], "id: int, a: int") print("writing:", data.collect()) data.write.format("delta").mode("append").partitionBy("id").save(delta_table_path) stop_reading = threading.Event() def read_data(): while not stop_reading.is_set(): print("Reading {:d} rows ...".format( spark.read.format("delta").load(delta_table_path).distinct().count()) ) time.sleep(1) def start_read_thread(): thread = threading.Thread(target=read_data) thread.start() return thread print("===================== Starting reads and writes =====================") read_threads = [start_read_thread() for i in range(concurrent_readers)] pool = ThreadPool(concurrent_writers) start_t = time.time() pool.map(write_tx, range(num_rows)) stop_reading.set() for thread in read_threads: thread.join() print("===================== Evaluating number of written rows =====================") actual = spark.read.format("delta").load(delta_table_path).distinct().count() print("Actual number of written rows:", actual) print("Expected number of written rows:", num_rows) assert actual == num_rows t = time.time() - start_t print(f"{num_rows / t:.02f} tx / sec") print("===================== Evaluating DDB writes =====================") import boto3 from botocore.config import Config my_config = Config( region_name=dynamo_region, ) dynamodb = boto3.resource('dynamodb', config=my_config) table = dynamodb.Table(dynamo_table_name) # this ensures we actually used/created the input table response = table.scan() items = response['Items'] items = sorted(items, key=lambda x: x['fileName']) print("========== All DDB items ==========") for item in items: print(item) print("===================== Evaluating _delta_log commits =====================") s3_client = boto3.client("s3") print(f"querying {s3_bucket}/{relative_delta_log_path}") response = s3_client.list_objects_v2(Bucket=s3_bucket, Prefix=relative_delta_log_path) items = response['Contents'] print("========== Raw _delta_log contents ========== ") for item in items: print(item) delta_log_commits = filter(lambda x: ".json" in x['Key'] and ".tmp" not in x['Key'], items) delta_log_commits = sorted(delta_log_commits, key=lambda x: x['Key']) print("========== _delta_log commits in version order ==========") for commit in delta_log_commits: print(commit) print("========== _delta_log commits in timestamp order ==========") delta_log_commits_sorted_timestamp = sorted(delta_log_commits, key=lambda x: x['LastModified']) for commit in delta_log_commits_sorted_timestamp: print(commit) print("========== ASSERT that these orders (version vs timestamp) are the same ==========") assert(delta_log_commits == delta_log_commits_sorted_timestamp) ================================================ FILE: storage-s3-dynamodb/src/main/java/io/delta/storage/BaseExternalLogStore.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage; import java.io.IOException; import java.io.InterruptedIOException; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Iterator; import java.util.Optional; import java.util.concurrent.TimeUnit; import com.google.common.annotations.VisibleForTesting; import io.delta.storage.internal.FileNameUtils; import io.delta.storage.internal.PathLock; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * A base {@link LogStore} implementation for cloud stores (e.g. Amazon S3) that do not provide * mutual exclusion. *

* This implementation depends on child methods, particularly `putExternalEntry`, to provide * the mutual exclusion that the cloud store is lacking. * * Notation: * - N: the target commit version we are writing. e.g. 10 for 0000010.json * - N.json: the actual target commit we want to write. * - T(N): the temp file path for commit N used during the prepare-commit-acknowledge `write` * algorithm below. We will eventually copy T(N) into N.json * - E(N, T(N), complete=true/false): the entry we will atomically commit into the external * cache. */ public abstract class BaseExternalLogStore extends HadoopFileSystemLogStore { private static final Logger LOG = LoggerFactory.getLogger(BaseExternalLogStore.class); /** * A global path lock to ensure that no two writers/readers are copying a given T(N) into N.json * at the same time within the same JVM. This can occur * - while a writer is performing a normal write operation AND a reader happens to see an * external entry E(complete=false) and so starts a recovery operation * - while two readers see E(complete=false) and so both start a recovery operation */ private static final PathLock pathLock = new PathLock(); /** * The delay, in seconds, after an external entry has been committed to the delta log at which * point it is safe to be deleted from the external store. * * We want a delay long enough such that, after the external entry has been deleted, another * write attempt for the SAME delta log commit can FAIL using ONLY the FileSystem's existence * check (e.g. `fs.exists(path)`). Recall we assume that the FileSystem does not provide mutual * exclusion. * * We use a value of 1 day. * * If we choose too small of a value, like 0 seconds, then the following scenario is possible: * - t0: Writers W1 and W2 start writing data files * - t1: W1 begins to try and write into the _delta_log. * - t2: W1 checks if N.json exists in FileSystem. It doesn't. * - t3: W1 writes actions into temp file T1(N) * - t4: W1 writes to external store entry E1(N, complete=false) * - t5: W1 copies (with overwrite=false) T1(N) into N.json. * - t6: W1 overwrites entry in external store E1(N, complete=true, expireTime=now+0) * - t7: E1 is safe to be deleted, and some external store TTL mechanism deletes E1 * - t8: W2 begins to try and write into the _delta_log. * - t9: W1 checks if N.json exists in FileSystem, but too little time has transpired between * t5 and t9 that the FileSystem check (fs.exists(path)) returns FALSE. * Note: This isn't possible on S3 (which provides strong consistency) but could be * possible on eventually-consistent systems. * - t10: W2 writes actions into temp file T2(N) * - t11: W2 writes to external store entry E2(N, complete=false) * - t12: W2 successfully copies (with overwrite=false) T2(N) into N.json. FileSystem didn't * provide the necessary mutual exclusion, so the copy succeeded. Thus, DATA LOSS HAS * OCCURRED. * * By using an expiration delay of 1 day, we ensure one of the steps at t9 or t12 will fail. */ protected static final long DEFAULT_EXTERNAL_ENTRY_EXPIRATION_DELAY_SECONDS = TimeUnit.DAYS.toSeconds(1); /** * Completed external commit entries will be created with a value of * NOW_EPOCH_SECONDS + getExpirationDelaySeconds(). */ protected long getExpirationDelaySeconds() { return DEFAULT_EXTERNAL_ENTRY_EXPIRATION_DELAY_SECONDS; } //////////////////////// // Public API Methods // //////////////////////// public BaseExternalLogStore(Configuration hadoopConf) { super(hadoopConf); } /** * First checks if there is any incomplete entry in the external store. If so, tries to perform * a recovery/fix. * * Then, performs a normal listFrom user the `super` implementation. */ @Override public Iterator listFrom(Path path, Configuration hadoopConf) throws IOException { final FileSystem fs = path.getFileSystem(hadoopConf); final Path resolvedPath = stripUserInfo(fs.makeQualified(path)); // VACUUM operations may use this LogStore::listFrom API. We don't need to attempt to // perform a fix/recovery during such operations that are not listing the _delta_log. if (isDeltaLogPath(resolvedPath)) { final Path tablePath = getTablePath(resolvedPath); final Optional entry = getLatestExternalEntry(tablePath); if (entry.isPresent() && !entry.get().complete) { // Note: `fixDeltaLog` will apply per-JVM mutual exclusion via a lock to help reduce // the chance of many reader threads in a single JVM doing duplicate copies of // T(N) -> N.json. fixDeltaLog(fs, entry.get()); } } // This is predicated on the storage system providing consistent listing // If there was a recovery performed in the `fixDeltaLog` call, then some temp file // was just copied into some N.json in the delta log. Because of consistent listing, // the `super.listFrom` is guaranteed to see N.json. return super.listFrom(path, hadoopConf); } /** * If overwrite=true, then write normally without any interaction with external store. * Else, to commit for delta version N: * - Step 0: Fail if N.json already exists in FileSystem. * - Step 1: Ensure that N-1.json exists. If not, perform a recovery. * - Step 2: PREPARE the commit. * - Write `actions` into temp file T(N) * - Write with mutual exclusion to external store and entry E(N, T(N), complete=false) * - Step 3: COMMIT the commit to the delta log. * - Copy T(N) into N.json * - Step 4: ACKNOWLEDGE the commit. * - Overwrite entry E in external store and set complete=true */ @Override public void write( Path path, Iterator actions, Boolean overwrite, Configuration hadoopConf) throws IOException { final FileSystem fs = path.getFileSystem(hadoopConf); final Path resolvedPath = stripUserInfo(fs.makeQualified(path)); try { // Prevent concurrent writers in this JVM from either // a) concurrently overwriting N.json if overwrite=true // b) both checking if N-1.json exists and performing a "recovery" where they both // copy T(N-1) into N-1.json // // Note that the mutual exclusion on writing into N.json with overwrite=false from // different JVMs (which is the entire point of BaseExternalLogStore) is provided by the // external cache, not by this lock, of course. // // Also note that this lock path (resolvedPath) is for N.json, while the lock path used // below in the recovery `fixDeltaLog` path is for N-1.json. Thus, no deadlock. pathLock.acquire(resolvedPath); if (overwrite) { writeActions(fs, path, actions); return; } else if (fs.exists(path)) { // Step 0: Fail if N.json already exists in FileSystem and overwrite=false. throw new java.nio.file.FileAlreadyExistsException(path.toString()); } // Step 1: Ensure that N-1.json exists final Path tablePath = getTablePath(resolvedPath); if (FileNameUtils.isDeltaFile(path)) { final long version = FileNameUtils.deltaVersion(path); if (version > 0) { final long prevVersion = version - 1; final Path deltaLogPath = new Path(tablePath, "_delta_log"); final Path prevPath = FileNameUtils.deltaFile(deltaLogPath, prevVersion); final String prevFileName = prevPath.getName(); final Optional prevEntry = getExternalEntry( tablePath.toString(), prevFileName ); if (prevEntry.isPresent() && !prevEntry.get().complete) { fixDeltaLog(fs, prevEntry.get()); } else { if (!fs.exists(prevPath)) { throw new java.nio.file.FileSystemException( String.format("previous commit %s doesn't exist on the file system but does in the external log store", prevPath) ); } } } else { final String fileName = path.getName(); final Optional entry = getExternalEntry( tablePath.toString(), fileName ); if (entry.isPresent()) { if (entry.get().complete && !fs.exists(path)) { throw new java.nio.file.FileSystemException( String.format( "Old entries for table %s still exist in the external log store", tablePath ) ); } } } } // Step 2: PREPARE the commit final String tempPath = createTemporaryPath(resolvedPath); final ExternalCommitEntry entry = new ExternalCommitEntry( tablePath, resolvedPath.getName(), tempPath, false, // not complete null // no expireTime ); // Step 2.1: Create temp file T(N) writeActions(fs, entry.absoluteTempPath(), actions); // Step 2.2: Create externals store entry E(N, T(N), complete=false) putExternalEntry(entry, false); // overwrite=false try { // Step 3: COMMIT the commit to the delta log. // Copy T(N) -> N.json with overwrite=false writeCopyTempFile(fs, entry.absoluteTempPath(), resolvedPath); // Step 4: ACKNOWLEDGE the commit writePutCompleteDbEntry(entry); } catch (Throwable e) { LOG.info( "{}: ignoring recoverable error", e.getClass().getSimpleName(), e ); } } catch (java.lang.InterruptedException e) { throw new InterruptedIOException(e.getMessage()); } finally { pathLock.release(resolvedPath); } } @Override public Boolean isPartialWriteVisible(Path path, Configuration hadoopConf) { return false; } ///////////////////////////////////////////////////////////// // Protected Members (for interaction with external store) // ///////////////////////////////////////////////////////////// /** * Write file with actions under a specific path. */ protected void writeActions( FileSystem fs, Path path, Iterator actions ) throws IOException { LOG.debug("writeActions to: {}", path); FSDataOutputStream stream = fs.create(path, true); while (actions.hasNext()) { byte[] line = String.format("%s\n", actions.next()).getBytes(StandardCharsets.UTF_8); stream.write(line); } stream.close(); } /** * Generate temporary path for TransactionLog. */ protected String createTemporaryPath(Path path) { String uuid = java.util.UUID.randomUUID().toString(); return String.format(".tmp/%s.%s", path.getName(), uuid); } /** * Returns the base table path for a given Delta log entry located in * e.g. input path of $tablePath/_delta_log/00000N.json would return $tablePath */ protected Path getTablePath(Path path) { return path.getParent().getParent(); } /** * Write to external store in exclusive way. * * @throws java.nio.file.FileAlreadyExistsException if path exists in cache and `overwrite` is * false */ abstract protected void putExternalEntry( ExternalCommitEntry entry, boolean overwrite) throws IOException; /** * Return external store entry corresponding to delta log file with given `tablePath` and * `fileName`, or `Optional.empty()` if it doesn't exist. */ abstract protected Optional getExternalEntry( String tablePath, String fileName) throws IOException; /** * Return the latest external store entry corresponding to the delta log for given `tablePath`, * or `Optional.empty()` if it doesn't exist. */ abstract protected Optional getLatestExternalEntry( Path tablePath) throws IOException; ////////////////////////////////////////////////////////// // Protected Members (for error injection during tests) // ////////////////////////////////////////////////////////// /** * Wrapper for `copyFile`, called by the `write` method. */ @VisibleForTesting protected void writeCopyTempFile(FileSystem fs, Path src, Path dst) throws IOException { copyFile(fs, src, dst); } /** * Wrapper for `putExternalEntry`, called by the `write` method. */ @VisibleForTesting protected void writePutCompleteDbEntry(ExternalCommitEntry entry) throws IOException { putExternalEntry(entry.asComplete(getExpirationDelaySeconds()), true); // overwrite=true } /** * Wrapper for `copyFile`, called by the `fixDeltaLog` method. */ @VisibleForTesting protected void fixDeltaLogCopyTempFile(FileSystem fs, Path src, Path dst) throws IOException { copyFile(fs, src, dst); } /** * Wrapper for `putExternalEntry`, called by the `fixDeltaLog` method. */ @VisibleForTesting protected void fixDeltaLogPutCompleteDbEntry(ExternalCommitEntry entry) throws IOException { putExternalEntry(entry.asComplete(getExpirationDelaySeconds()), true); // overwrite=true } //////////////////// // Helper Methods // //////////////////// /** * Method for assuring consistency on filesystem according to the external cache. * Method tries to rewrite TransactionLog entry from temporary path if it does not exist. * * Should never throw a FileAlreadyExistsException. * - If we see one when copying the temp file, we can assume the target file N.json already * exists and a concurrent writer has already copied the contents of T(N). * - We will never see one when writing to the external cache since overwrite=true. */ private void fixDeltaLog(FileSystem fs, ExternalCommitEntry entry) throws IOException { if (entry.complete) { return; } final Path targetPath = entry.absoluteFilePath(); try { pathLock.acquire(targetPath); int retry = 0; boolean copied = false; while (true) { LOG.info("trying to fix: {}", entry.fileName); try { if (!copied && !fs.exists(targetPath)) { fixDeltaLogCopyTempFile(fs, entry.absoluteTempPath(), targetPath); copied = true; } fixDeltaLogPutCompleteDbEntry(entry); LOG.info("fixed file {}", entry.fileName); return; } catch (java.nio.file.FileAlreadyExistsException e) { LOG.info("file {} already copied: {}:", entry.fileName, e.getClass().getSimpleName(), e); copied = true; // Don't return since we still need to mark the DB entry as complete. This will // happen when we execute the main try block on the next while loop iteration } catch (Throwable e) { LOG.info("{}:", e.getClass().getSimpleName(), e); if (retry >= 3) { throw e; } } retry += 1; } } catch (java.lang.InterruptedException e) { throw new InterruptedIOException(e.getMessage()); } finally { pathLock.release(targetPath); } } /** * Copies file within filesystem. * * @param fs reference to [[FileSystem]] * @param src path to source file * @param dst path to destination file */ private void copyFile(FileSystem fs, Path src, Path dst) throws IOException { LOG.info("copy file: {} -> {}", src, dst); final FSDataInputStream inputStream = fs.open(src); try { final FSDataOutputStream outputStream = fs.create(dst, false); // overwrite=false IOUtils.copy(inputStream, outputStream); // We don't close `outputStream` if an exception happens because it may create a partial // file. outputStream.close(); } catch (org.apache.hadoop.fs.FileAlreadyExistsException e) { throw new java.nio.file.FileAlreadyExistsException(dst.toString()); } finally { inputStream.close(); } } /** * Returns path stripped user info. */ private Path stripUserInfo(Path path) { final URI uri = path.toUri(); try { final URI newUri = new URI( uri.getScheme(), null, // userInfo uri.getHost(), uri.getPort(), uri.getPath(), uri.getQuery(), uri.getFragment() ); return new Path(newUri); } catch (URISyntaxException e) { // Propagating this URISyntaxException to callers would mean we would have to either // include it in the public LogStore.java interface or wrap it in an // IllegalArgumentException somewhere else. Instead, catch and wrap it here. throw new IllegalArgumentException(e); } } /** Returns true if this path is contained within a _delta_log folder. */ @VisibleForTesting protected boolean isDeltaLogPath(Path normalizedPath) { return Arrays.stream(normalizedPath .toUri() .toString() .split(Path.SEPARATOR) ).anyMatch("_delta_log"::equals); } } ================================================ FILE: storage-s3-dynamodb/src/main/java/io/delta/storage/ExternalCommitEntry.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage; import org.apache.hadoop.fs.Path; /** * Wrapper class representing an entry in an external store for a given commit into the Delta log. * * Contains relevant fields and helper methods. */ public final class ExternalCommitEntry { /** * Absolute path to this delta table */ public final Path tablePath; /** * File name of this commit, e.g. "000000N.json" */ public final String fileName; /** * Path to temp file for this commit, relative to the `_delta_log */ public final String tempPath; /** * true if delta json file is successfully copied to its destination location, else false */ public final boolean complete; /** * If complete = true, epoch seconds at which this external commit entry is safe to be deleted. * Else, null. */ public final Long expireTime; public ExternalCommitEntry( Path tablePath, String fileName, String tempPath, boolean complete, Long expireTime) { this.tablePath = tablePath; this.fileName = fileName; this.tempPath = tempPath; this.complete = complete; this.expireTime = expireTime; } /** * @return this entry with `complete=true` and a valid `expireTime` */ public ExternalCommitEntry asComplete(long expirationDelaySeconds) { return new ExternalCommitEntry( this.tablePath, this.fileName, this.tempPath, true, System.currentTimeMillis() / 1000L + expirationDelaySeconds ); } /** * @return the absolute path to the file for this entry. * e.g. $tablePath/_delta_log/0000000N.json */ public Path absoluteFilePath() { return new Path(new Path(tablePath, "_delta_log"), fileName); } /** * @return the absolute path to the temp file for this entry */ public Path absoluteTempPath() { return new Path(new Path(tablePath, "_delta_log"), tempPath); } } ================================================ FILE: storage-s3-dynamodb/src/main/java/io/delta/storage/RetryableCloseableIterator.java ================================================ package io.delta.storage; import java.io.IOException; import java.io.UncheckedIOException; import java.util.NoSuchElementException; import java.util.Objects; import java.util.function.Supplier; import io.delta.storage.utils.ThrowingSupplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * This class presents an iterator view over the iterator supplier in the constructor. * * This class assumes that the iterator supplied by the supplier can throw, and that subsequent * supplier.get() calls will return an iterator over the same data. * * If there are any RemoteFileChangedException during `next` and `hasNext` calls, will retry * at most `MAX_RETRIES` times. If there are similar exceptions during the retry, those are handled * and count towards the MAX_RETRIES. * * Internally, keeps track of the last-successfully-returned index. Upon retry, will iterate back * to that same position. */ public class RetryableCloseableIterator implements CloseableIterator { private static final Logger LOG = LoggerFactory.getLogger(RetryableCloseableIterator.class); public static final int DEFAULT_MAX_RETRIES = 3; private final ThrowingSupplier, IOException> iterSupplier; private final int maxRetries; /** * Index of the last element successfully returned without an exception. A value of -1 means * that no element has ever been returned yet. */ private int lastSuccessfullIndex; private int numRetries = 0; private CloseableIterator currentIter; public RetryableCloseableIterator( ThrowingSupplier, IOException> iterSupplier, int maxRetries) throws IOException { if (maxRetries < 0) throw new IllegalArgumentException("maxRetries can't be negative"); this.iterSupplier = Objects.requireNonNull(iterSupplier); this.maxRetries = maxRetries; this.lastSuccessfullIndex = -1; this.currentIter = this.iterSupplier.get(); } public RetryableCloseableIterator( ThrowingSupplier, IOException> iterSupplier) throws IOException { this(iterSupplier, DEFAULT_MAX_RETRIES); } ///////////////// // Public APIs // ///////////////// @Override public void close() throws IOException { currentIter.close(); } /** * `hasNext` must be idempotent. It does not change the `lastSuccessfulIndex` variable. */ @Override public boolean hasNext() { try { return hasNextInternal(); } catch (IOException ex) { if (isRemoteFileChangedException(ex)) { try { replayIterToLastSuccessfulIndex(ex); } catch (IOException ex2) { throw new UncheckedIOException(ex2); } return hasNext(); } else { throw new UncheckedIOException(ex); } } } @Override public String next() { if (!hasNext()) throw new NoSuchElementException(); try { final String ret = nextInternal(); lastSuccessfullIndex++; return ret; } catch (IOException ex) { if (isRemoteFileChangedException(ex)) { try { replayIterToLastSuccessfulIndex(ex); } catch (IOException ex2) { throw new UncheckedIOException(ex2); } if (!hasNext()) { throw new IllegalStateException( String.format( "A retried iterator doesn't have enough data " + "(hasNext=false, lastSuccessfullIndex=%s)", lastSuccessfullIndex ) ); } return next(); } else { throw new UncheckedIOException(ex); } } } ////////////////////////////////////// // Package-private APIs for testing // ////////////////////////////////////// /** Visible for testing. */ int getLastSuccessfullIndex() { return lastSuccessfullIndex; } /** Visible for testing. */ int getNumRetries() { return numRetries; } //////////////////// // Helper Methods // //////////////////// /** Throw a checked exception so we can catch this in the caller. */ private boolean hasNextInternal() throws IOException { return currentIter.hasNext(); } /** Throw a checked exception so we can catch this in the caller. */ private String nextInternal() throws IOException { return currentIter.next(); } /** * Called after a RemoteFileChangedException was thrown. Tries to replay the underlying * iter implementation (supplied by the `implSupplier`) to the last successful index, so that * the previous error open (hasNext, or next) can be retried. If a RemoteFileChangedException * is thrown while replaying the iter, we just increment the `numRetries` counter and try again. */ private void replayIterToLastSuccessfulIndex(IOException topLevelEx) throws IOException { LOG.warn( "Caught a RemoteFileChangedException. NumRetries is {} / {}.\n{}", numRetries + 1, maxRetries, topLevelEx ); currentIter.close(); while (numRetries < maxRetries) { numRetries++; LOG.info( "Replaying until (inclusive) index {}. NumRetries is {} / {}.", lastSuccessfullIndex, numRetries + 1, maxRetries ); currentIter = iterSupplier.get(); // Last successful index replayed. Starts at -1, and not 0, because 0 means we've // already replayed the 1st element! int replayIndex = -1; try { while (replayIndex < lastSuccessfullIndex) { if (currentIter.hasNext()) { currentIter.next(); // Disregard data that has been read replayIndex++; } else { throw new IllegalStateException( String.format( "A retried iterator doesn't have enough data " + "(replayIndex=%s, lastSuccessfullIndex=%s)", replayIndex, lastSuccessfullIndex ) ); } } // Just like how in RetryableCloseableIterator::next we have to handle // RemoteFileChangedException, we must also hadnle that here during the replay. // `currentIter.next()` isn't declared to throw a RemoteFileChangedException, so we // trick the compiler into thinking this block can throw RemoteFileChangedException // via `fakeIOException`. That way, we can catch it, and retry replaying the iter. fakeIOException(); LOG.info("Successfully replayed until (inclusive) index {}", lastSuccessfullIndex); return; } catch (IOException ex) { if (isRemoteFileChangedException(ex)) { // Ignore and try replaying the iter again at the top of the while loop LOG.warn("Caught a RemoteFileChangedException while replaying the iterator"); } else { throw ex; } } } throw topLevelEx; } private boolean isRemoteFileChangedException(IOException ex) { // `endsWith` should still work if the class is shaded. final String exClassName = ex.getClass().getName(); return exClassName.endsWith("org.apache.hadoop.fs.s3a.RemoteFileChangedException"); } private void fakeIOException() throws IOException { if (false) { throw new IOException(); } } } ================================================ FILE: storage-s3-dynamodb/src/main/java/io/delta/storage/S3DynamoDBLogStore.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage; import io.delta.storage.utils.ReflectionUtils; import org.apache.hadoop.fs.Path; import java.io.InterruptedIOException; import java.io.UncheckedIOException; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; import com.amazonaws.services.dynamodbv2.model.AttributeDefinition; import com.amazonaws.services.dynamodbv2.model.AttributeValue; import com.amazonaws.services.dynamodbv2.model.ComparisonOperator; import com.amazonaws.services.dynamodbv2.model.Condition; import com.amazonaws.services.dynamodbv2.model.ConditionalCheckFailedException; import com.amazonaws.services.dynamodbv2.model.DescribeTableResult; import com.amazonaws.services.dynamodbv2.model.TableDescription; import com.amazonaws.services.dynamodbv2.model.ExpectedAttributeValue; import com.amazonaws.services.dynamodbv2.model.GetItemRequest; import com.amazonaws.services.dynamodbv2.model.KeySchemaElement; import com.amazonaws.services.dynamodbv2.model.KeyType; import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput; import com.amazonaws.services.dynamodbv2.model.PutItemRequest; import com.amazonaws.services.dynamodbv2.model.QueryRequest; import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException; import com.amazonaws.services.dynamodbv2.model.ResourceInUseException; import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType; import com.amazonaws.regions.Region; import com.amazonaws.regions.Regions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * A concrete implementation of {@link BaseExternalLogStore} that uses an external DynamoDB table * to provide the mutual exclusion during calls to `putExternalEntry`. * * DynamoDB entries are of form * - key * -- tablePath (HASH, STRING) * -- filename (RANGE, STRING) * * - attributes * -- tempPath (STRING, relative to _delta_log) * -- complete (STRING, representing boolean, "true" or "false") * -- commitTime (NUMBER, epoch seconds) */ public class S3DynamoDBLogStore extends BaseExternalLogStore { private static final Logger LOG = LoggerFactory.getLogger(S3DynamoDBLogStore.class); /** * Configuration keys for the DynamoDB client. * * Keys are either of the form $SPARK_CONF_PREFIX.$CONF or $BASE_CONF_PREFIX.$CONF, * e.g. spark.io.delta.storage.S3DynamoDBLogStore.ddb.tableName * or io.delta.storage.S3DynamoDBLogStore.ddb.tableName */ public static final String SPARK_CONF_PREFIX = "spark.io.delta.storage.S3DynamoDBLogStore"; public static final String BASE_CONF_PREFIX = "io.delta.storage.S3DynamoDBLogStore"; public static final String READ_RETRIES = "read.retries"; public static final String DDB_CLIENT_TABLE = "ddb.tableName"; public static final String DDB_CLIENT_REGION = "ddb.region"; public static final String DDB_CLIENT_CREDENTIALS_PROVIDER = "credentials.provider"; public static final String DDB_CREATE_TABLE_RCU = "provisionedThroughput.rcu"; public static final String DDB_CREATE_TABLE_WCU = "provisionedThroughput.wcu"; // WARNING: setting this value too low can cause data loss. Defaults to a duration of 1 day. public static final String TTL_SECONDS = "ddb.ttl"; /** * DynamoDB table attribute keys */ private static final String ATTR_TABLE_PATH = "tablePath"; private static final String ATTR_FILE_NAME = "fileName"; private static final String ATTR_TEMP_PATH = "tempPath"; private static final String ATTR_COMPLETE = "complete"; private static final String ATTR_EXPIRE_TIME = "expireTime"; /** * Member fields */ private final AmazonDynamoDBClient client; private final String tableName; private final String credentialsProviderName; private final String regionName; private final long expirationDelaySeconds; public S3DynamoDBLogStore(Configuration hadoopConf) throws IOException { super(hadoopConf); tableName = getParam(hadoopConf, DDB_CLIENT_TABLE, "delta_log"); credentialsProviderName = getParam( hadoopConf, DDB_CLIENT_CREDENTIALS_PROVIDER, "com.amazonaws.auth.DefaultAWSCredentialsProviderChain" ); regionName = getParam(hadoopConf, DDB_CLIENT_REGION, "us-east-1"); final String ttl = getParam(hadoopConf, TTL_SECONDS, null); expirationDelaySeconds = ttl == null ? BaseExternalLogStore.DEFAULT_EXTERNAL_ENTRY_EXPIRATION_DELAY_SECONDS : Long.parseLong(ttl); if (expirationDelaySeconds < 0) { throw new IllegalArgumentException( String.format( "Can't use negative `%s` value of %s", TTL_SECONDS, expirationDelaySeconds)); } LOG.info("using tableName {}", tableName); LOG.info("using credentialsProviderName {}", credentialsProviderName); LOG.info("using regionName {}", regionName); LOG.info("using ttl (seconds) {}", expirationDelaySeconds); client = getClient(); tryEnsureTableExists(hadoopConf); } @Override public CloseableIterator read(Path path, Configuration hadoopConf) throws IOException { // With many concurrent readers/writers, there's a chance that concurrent 'recovery' // operations occur on the same file, i.e. the same temp file T(N) is copied into the target // N.json file more than once. Though data loss will *NOT* occur, readers of N.json may // receive a RemoteFileChangedException from S3 as the ETag of N.json was changed. This is // safe to retry, so we do so here. final int maxRetries = Integer.parseInt( getParam( hadoopConf, READ_RETRIES, Integer.toString(RetryableCloseableIterator.DEFAULT_MAX_RETRIES) ) ); return new RetryableCloseableIterator(() -> super.read(path, hadoopConf), maxRetries); } @Override protected long getExpirationDelaySeconds() { return expirationDelaySeconds; } @Override protected void putExternalEntry( ExternalCommitEntry entry, boolean overwrite) throws IOException { try { LOG.debug(String.format("putItem %s, overwrite: %s", entry, overwrite)); client.putItem(createPutItemRequest(entry, overwrite)); } catch (ConditionalCheckFailedException e) { LOG.debug(e.toString()); throw new java.nio.file.FileAlreadyExistsException( entry.absoluteFilePath().toString() ); } } @Override protected Optional getExternalEntry( String tablePath, String fileName) { final Map attributes = new ConcurrentHashMap<>(); attributes.put(ATTR_TABLE_PATH, new AttributeValue(tablePath)); attributes.put(ATTR_FILE_NAME, new AttributeValue(fileName)); Map item = client.getItem( new GetItemRequest(tableName, attributes).withConsistentRead(true) ).getItem(); return item != null ? Optional.of(dbResultToCommitEntry(item)) : Optional.empty(); } @Override protected Optional getLatestExternalEntry(Path tablePath) { final Map conditions = new ConcurrentHashMap<>(); conditions.put( ATTR_TABLE_PATH, new Condition() .withComparisonOperator(ComparisonOperator.EQ) .withAttributeValueList(new AttributeValue(tablePath.toString())) ); final List> items = client.query( new QueryRequest(tableName) .withConsistentRead(true) .withScanIndexForward(false) .withLimit(1) .withKeyConditions(conditions) ).getItems(); if (items.isEmpty()) { return Optional.empty(); } else { return Optional.of(dbResultToCommitEntry(items.get(0))); } } /** * Map a DBB query result item to an {@link ExternalCommitEntry}. */ private ExternalCommitEntry dbResultToCommitEntry(Map item) { final AttributeValue expireTimeAttr = item.get(ATTR_EXPIRE_TIME); return new ExternalCommitEntry( new Path(item.get(ATTR_TABLE_PATH).getS()), item.get(ATTR_FILE_NAME).getS(), item.get(ATTR_TEMP_PATH).getS(), item.get(ATTR_COMPLETE).getS().equals("true"), expireTimeAttr != null ? Long.parseLong(expireTimeAttr.getN()) : null ); } private PutItemRequest createPutItemRequest(ExternalCommitEntry entry, boolean overwrite) { final Map attributes = new ConcurrentHashMap<>(); attributes.put(ATTR_TABLE_PATH, new AttributeValue(entry.tablePath.toString())); attributes.put(ATTR_FILE_NAME, new AttributeValue(entry.fileName)); attributes.put(ATTR_TEMP_PATH, new AttributeValue(entry.tempPath)); attributes.put( ATTR_COMPLETE, new AttributeValue().withS(Boolean.toString(entry.complete)) ); if (entry.expireTime != null) { attributes.put( ATTR_EXPIRE_TIME, new AttributeValue().withN(entry.expireTime.toString()) ); } final PutItemRequest pr = new PutItemRequest(tableName, attributes); if (!overwrite) { Map expected = new ConcurrentHashMap<>(); expected.put(ATTR_FILE_NAME, new ExpectedAttributeValue(false)); pr.withExpected(expected); } return pr; } private void tryEnsureTableExists(Configuration hadoopConf) throws IOException { int retries = 0; boolean created = false; while(retries < 20) { String status = "CREATING"; try { // https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/dynamodbv2/model/TableDescription.html#getTableStatus-- DescribeTableResult result = client.describeTable(tableName); TableDescription descr = result.getTable(); status = descr.getTableStatus(); } catch (ResourceNotFoundException e) { final long rcu = Long.parseLong(getParam(hadoopConf, DDB_CREATE_TABLE_RCU, "5")); final long wcu = Long.parseLong(getParam(hadoopConf, DDB_CREATE_TABLE_WCU, "5")); LOG.info( "DynamoDB table `{}` in region `{}` does not exist. " + "Creating it now with provisioned throughput of {} RCUs and {} WCUs.", tableName, regionName, rcu, wcu); try { client.createTable( // attributeDefinitions java.util.Arrays.asList( new AttributeDefinition(ATTR_TABLE_PATH, ScalarAttributeType.S), new AttributeDefinition(ATTR_FILE_NAME, ScalarAttributeType.S) ), tableName, // keySchema Arrays.asList( new KeySchemaElement(ATTR_TABLE_PATH, KeyType.HASH), new KeySchemaElement(ATTR_FILE_NAME, KeyType.RANGE) ), new ProvisionedThroughput(rcu, wcu) ); created = true; } catch (ResourceInUseException e3) { // race condition - table just created by concurrent process } } if (status.equals("ACTIVE")) { if (created) { LOG.info("Successfully created DynamoDB table `{}`", tableName); } else { LOG.info("Table `{}` already exists", tableName); } break; } else if (status.equals("CREATING")) { retries += 1; LOG.info("Waiting for `{}` table creation", tableName); try { Thread.sleep(1000); } catch(InterruptedException e) { throw new InterruptedIOException(e.getMessage()); } } else { LOG.error("table `{}` status: {}", tableName, status); break; // TODO - raise exception? } }; } private AmazonDynamoDBClient getClient() throws java.io.IOException { try { final AWSCredentialsProvider awsCredentialsProvider = ReflectionUtils.createAwsCredentialsProvider(credentialsProviderName, initHadoopConf()); final AmazonDynamoDBClient client = new AmazonDynamoDBClient(awsCredentialsProvider); client.setRegion(Region.getRegion(Regions.fromName(regionName))); return client; } catch (ReflectiveOperationException e) { throw new java.io.IOException(e); } } /** * Get the hadoopConf param $name that is prefixed either with $SPARK_CONF_PREFIX or * $BASE_CONF_PREFIX. * * If two parameters exist, one for each prefix, then an IllegalArgumentException is thrown. * * If no parameters exist, then the $defaultValue is returned. */ protected static String getParam(Configuration hadoopConf, String name, String defaultValue) { final String sparkPrefixKey = String.format("%s.%s", SPARK_CONF_PREFIX, name); final String basePrefixKey = String.format("%s.%s", BASE_CONF_PREFIX, name); final String sparkPrefixVal = hadoopConf.get(sparkPrefixKey); final String basePrefixVal = hadoopConf.get(basePrefixKey); if (sparkPrefixVal != null && basePrefixVal != null && !sparkPrefixVal.equals(basePrefixVal)) { throw new IllegalArgumentException( String.format( "Configuration properties `%s=%s` and `%s=%s` have different values. " + "Please set only one.", sparkPrefixKey, sparkPrefixVal, basePrefixKey, basePrefixVal ) ); } if (sparkPrefixVal != null) return sparkPrefixVal; if (basePrefixVal != null) return basePrefixVal; return defaultValue; } } ================================================ FILE: storage-s3-dynamodb/src/main/java/io/delta/storage/utils/ReflectionUtils.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.utils; import com.amazonaws.auth.AWSCredentialsProvider; import org.apache.hadoop.conf.Configuration; import java.util.Arrays; public class ReflectionUtils { private static boolean readsCredsFromHadoopConf(Class awsCredentialsProviderClass) { return Arrays.stream(awsCredentialsProviderClass.getConstructors()) .anyMatch(constructor -> constructor.getParameterCount() == 1 && Arrays.equals(constructor.getParameterTypes(), new Class[]{Configuration.class})); } /** * Create AWS credentials provider from given provider classname and {@link Configuration}. * * It first check if AWS Credentials Provider class has constructor Hadoop configuration as parameter. * If yes - create instance of class using this constructor. * If no - create instance with empty parameters constructor. * * @param credentialsProviderClassName Fully qualified name of the desired credentials provider class. * @param hadoopConf Hadoop configuration, used to create instance of AWS credentials * provider, if supported. * @return {@link AWSCredentialsProvider} object, instantiated from the class @see {credentialsProviderClassName} * @throws ReflectiveOperationException When AWS credentials provider constrictor do not matched. * Means class has neither an constructor with no args as input * nor constructor with only Hadoop configuration as argument. */ public static AWSCredentialsProvider createAwsCredentialsProvider( String credentialsProviderClassName, Configuration hadoopConf) throws ReflectiveOperationException { Class awsCredentialsProviderClass = Class.forName(credentialsProviderClassName); if (readsCredsFromHadoopConf(awsCredentialsProviderClass)) return (AWSCredentialsProvider) awsCredentialsProviderClass .getConstructor(Configuration.class) .newInstance(hadoopConf); else return (AWSCredentialsProvider) awsCredentialsProviderClass.getConstructor().newInstance(); } } ================================================ FILE: storage-s3-dynamodb/src/main/java/io/delta/storage/utils/ThrowingSupplier.java ================================================ package io.delta.storage.utils; @FunctionalInterface public interface ThrowingSupplier { T get() throws E; } ================================================ FILE: storage-s3-dynamodb/src/test/java/io/delta/storage/FailingS3DynamoDBLogStore.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage; import java.io.IOException; import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; /** * An ExternalLogStore implementation that allows for easy, probability-based error injection during * runtime. * * This is used to test the error-handling capabilities of S3DynamoDBLogStore during integration * tests. */ public class FailingS3DynamoDBLogStore extends S3DynamoDBLogStore { private static java.util.Random rng = new java.util.Random(); private final ConcurrentHashMap errorRates; public FailingS3DynamoDBLogStore(Configuration hadoopConf) throws IOException { super(hadoopConf); errorRates = new ConcurrentHashMap<>(); // for each optional key in set { write_copy_temp_file, write_put_db_entry, // fix_delta_log_copy_temp_file, fix_delta_log_put_db_entry }, `errorRates` string is // expected to be of form key1=value1,key2=value2 etc where each value is a fraction // indicating how often that method should fail (e.g. 0.10 ==> 10% failure rate). String errorRatesDef = getParam(hadoopConf, "errorRates", ""); for (String s: errorRatesDef.split(",")) { if (!s.contains("=")) continue; String[] parts = s.split("=", 2); if (parts.length == 2) { errorRates.put(parts[0], Float.parseFloat(parts[1])); } } } @Override protected void writeCopyTempFile(FileSystem fs, Path src, Path dst) throws IOException { injectError("write_copy_temp_file"); super.writeCopyTempFile(fs, src, dst); } @Override protected void writePutCompleteDbEntry(ExternalCommitEntry entry) throws IOException { injectError("write_put_db_entry"); super.writePutCompleteDbEntry(entry); } @Override protected void fixDeltaLogCopyTempFile(FileSystem fs, Path src, Path dst) throws IOException { injectError("fix_delta_log_copy_temp_file"); super.fixDeltaLogCopyTempFile(fs, src, dst); } @Override protected void fixDeltaLogPutCompleteDbEntry(ExternalCommitEntry entry) throws IOException { injectError("fix_delta_log_put_db_entry"); super.fixDeltaLogPutCompleteDbEntry(entry); } private void injectError(String name) throws IOException { float rate = errorRates.getOrDefault(name, 0.1f); if (rng.nextFloat() < rate) { throw new IOException(String.format("injected failure: %s", name)); } } } ================================================ FILE: storage-s3-dynamodb/src/test/java/io/delta/storage/MemoryLogStore.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import java.io.IOException; import java.util.Comparator; import java.util.concurrent.ConcurrentHashMap; import java.util.Optional; /** * Simple ExternalLogStore implementation using an in-memory hashmap (as opposed to an actual * database) */ public class MemoryLogStore extends BaseExternalLogStore { public static String IS_DELTA_LOG_PATH_OVERRIDE_KEY = "spark.hadoop.io.delta.storage.MemoryLogStore.isDeltaLogPath.alwaysTrue"; public static int numGetLatestExternalEntryCalls = 0; public MemoryLogStore(Configuration hadoopConf) { super(hadoopConf); } /////////////////// // API Overrides // /////////////////// @Override protected void putExternalEntry( ExternalCommitEntry entry, boolean overwrite) throws IOException { final String key = createKey(entry.tablePath.toString(), entry.fileName); final ExternalCommitEntry correctedEntry = new ExternalCommitEntry( // some tests use "failing:" scheme to inject errors, but we want to store normal paths new Path(fixPathSchema(entry.tablePath.toString())), entry.fileName, entry.tempPath, entry.complete, entry.expireTime ); if (overwrite) { hashMap.put(key, correctedEntry); } else if (hashMap.containsKey(key)) { // and overwrite=false throw new java.nio.file.FileAlreadyExistsException("already exists"); } else { hashMap.put(key, correctedEntry); } } @Override protected Optional getExternalEntry( String tablePath, String fileName) { final String key = createKey(tablePath, fileName); if (hashMap.containsKey(key)) { return Optional.of(hashMap.get(key)); } return Optional.empty(); } @Override protected Optional getLatestExternalEntry(Path tablePath) { numGetLatestExternalEntryCalls++; final Path fixedTablePath = new Path(fixPathSchema(tablePath.toString())); return hashMap .values() .stream() .filter(item -> item.tablePath.equals(fixedTablePath)) .max(Comparator.comparing(ExternalCommitEntry::absoluteFilePath)); } @Override protected boolean isDeltaLogPath(Path normalizedPath) { if (initHadoopConf().getBoolean(IS_DELTA_LOG_PATH_OVERRIDE_KEY, false)) { return true; // hardcoded to return true } else { return super.isDeltaLogPath(normalizedPath); // only return true if in _delta_log folder } } //////////////////// // Static Helpers // //////////////////// /** * ExternalLogStoreSuite sometimes uses "failing:" scheme prefix to inject errors during tests * However, we want lookups for the same $tablePath to return the same result, regardless of * scheme. */ static String fixPathSchema(String tablePath) { return tablePath.replace("failing:", "file:"); } static String createKey(String tablePath, String fileName) { return String.format("%s-%s", fixPathSchema(tablePath), fileName); } static ExternalCommitEntry get(Path path) { final String tablePath = path.getParent().getParent().toString(); final String fileName = path.getName(); final String key = createKey(tablePath, fileName); return hashMap.get(key); } static boolean containsKey(Path path) { final String tablePath = path.getParent().getParent().toString(); final String fileName = path.getName(); final String key = createKey(tablePath, fileName); return hashMap.containsKey(key); } static ConcurrentHashMap hashMap = new ConcurrentHashMap<>(); } ================================================ FILE: storage-s3-dynamodb/src/test/java/io/delta/storage/utils/ReflectionsUtilsSuiteHelper.java ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.utils; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; import org.apache.hadoop.conf.Configuration; public class ReflectionsUtilsSuiteHelper { // this class only purpose to test DynamoDBLogStore logic to create AWS credentials provider with reflection. public static class TestOnlyAWSCredentialsProviderWithHadoopConf implements AWSCredentialsProvider { public TestOnlyAWSCredentialsProviderWithHadoopConf(Configuration hadoopConf) {} @Override public AWSCredentials getCredentials() { return null; } @Override public void refresh() { } } // this class only purpose to test DynamoDBLogStore logic to create AWS credentials provider with reflection. public static class TestOnlyAWSCredentialsProviderWithUnexpectedConstructor implements AWSCredentialsProvider { public TestOnlyAWSCredentialsProviderWithUnexpectedConstructor(String hadoopConf) {} @Override public AWSCredentials getCredentials() { return null; } @Override public void refresh() { } } } ================================================ FILE: storage-s3-dynamodb/src/test/scala/io/delta/storage/ExternalLogStoreSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage import java.io.File import java.net.URI import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs._ import org.scalatest.funsuite.AnyFunSuite import org.apache.spark.sql.delta.FakeFileSystem import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.storage.LogStoreAdaptor import org.apache.spark.sql.delta.util.FileNames import org.apache.spark.sql.functions.col ///////////////////// // Base Test Suite // ///////////////////// class ExternalLogStoreSuite extends org.apache.spark.sql.delta.PublicLogStoreSuite { protected def shouldUseRenameToWriteCheckpoint: Boolean = false override protected val publicLogStoreClassName: String = classOf[MemoryLogStore].getName protected override def beforeEach(): Unit = { super.beforeEach() MemoryLogStore.numGetLatestExternalEntryCalls = 0 } testHadoopConf( expectedErrMsg = "No FileSystem for scheme \"fake\"", "fs.fake.impl" -> classOf[FakeFileSystem].getName, "fs.fake.impl.disable.cache" -> "true" ) def getDeltaVersionPath(logDir: File, version: Int): Path = { FileNames.unsafeDeltaFile(new Path(logDir.toURI), version) } def getFailingDeltaVersionPath(logDir: File, version: Int): Path = { FileNames.unsafeDeltaFile(new Path(s"failing:${logDir.getCanonicalPath}"), version) } test("#3423: listFrom only checks latest external store entry if listing a delta log file") { withTempDir { tempDir => val store = createLogStore(spark) .asInstanceOf[LogStoreAdaptor].logStoreImpl .asInstanceOf[MemoryLogStore] val logDir = new File(tempDir.getCanonicalPath, "_delta_log") logDir.mkdir() val deltaFilePath = getDeltaVersionPath(logDir, 0) val dataFilePath = new Path(tempDir.getCanonicalPath, ".part-00000-da82aeb5-snappy.parquet") val fs = deltaFilePath.getFileSystem(sessionHadoopConf) fs.create(deltaFilePath).close() fs.create(dataFilePath).close() assert(MemoryLogStore.numGetLatestExternalEntryCalls == 0) store.listFrom(deltaFilePath, sessionHadoopConf) assert(MemoryLogStore.numGetLatestExternalEntryCalls == 1) // contacted external store store.listFrom(dataFilePath, sessionHadoopConf) assert(MemoryLogStore.numGetLatestExternalEntryCalls == 1) // did not contact external store } } test("#3423: VACUUM does not check external store for latest entry") { // previous behaviour: always check external store for latest entry when listing // current behaviour: only check external store for latest entry when listing a delta log file def doVacuumTestGetNumVacuumExternalStoreCalls(usePreviousListBehavior: Boolean): Int = { var ret = -1 withTempDir { tempDir => withSQLConf(DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED.key -> "false") { spark.conf.set( MemoryLogStore.IS_DELTA_LOG_PATH_OVERRIDE_KEY, usePreviousListBehavior) val path = tempDir.getCanonicalPath spark.range(100) .withColumn("part", col("id") % 10) .write .format("delta") .partitionBy("part") .save(path) spark.sql(s"DELETE FROM delta.`$path`") val numExternalCallsBeforeVacuum = MemoryLogStore.numGetLatestExternalEntryCalls spark.sql(s"VACUUM delta.`$path` RETAIN 0 HOURS") val numExternalCallsAfterVacuum = MemoryLogStore.numGetLatestExternalEntryCalls ret = numExternalCallsAfterVacuum - numExternalCallsBeforeVacuum } } ret } assert( doVacuumTestGetNumVacuumExternalStoreCalls(true) > doVacuumTestGetNumVacuumExternalStoreCalls(false) ) } test("#3423: BaseExternalLogStore::isDeltaLogPath") { val store = createLogStore(spark) .asInstanceOf[LogStoreAdaptor].logStoreImpl .asInstanceOf[MemoryLogStore] // json file assert(store.isDeltaLogPath(new Path("s3://bucket/_delta_log/0000.json"))) // checkpoint file assert(store.isDeltaLogPath(new Path("s3://bucket/_delta_log/0010.checkpoint.parquet"))) // file listing prefix assert(store.isDeltaLogPath(new Path("s3://bucket/_delta_log/0000."))) // delta_log folder (with / prefix) assert(store.isDeltaLogPath(new Path("s3://bucket/_delta_log/"))) // delta_log folder (without / prefix) assert(store.isDeltaLogPath(new Path("s3://bucket/_delta_log"))) // obvious negative cases assert(!store.isDeltaLogPath(new Path("s3://bucket/part-000-UUID.parquet"))) // edge cases of `_delta_log` in a folder assert(!store.isDeltaLogPath(new Path("s3://bucket/__delta_log/"))) assert(!store.isDeltaLogPath(new Path("s3://bucket/_delta_log_"))) } test("single write") { withTempLogDir { tempLogDir => val store = createLogStore(spark) val path = getDeltaVersionPath(tempLogDir, 0) store.write(path, Iterator("foo", "bar"), overwrite = false, sessionHadoopConf) val entry = MemoryLogStore.get(path); assert(entry != null) assert(entry.complete); } } test("double write") { withTempLogDir { tempLogDir => val store = createLogStore(spark) val path = getDeltaVersionPath(tempLogDir, 0) store.write(path, Iterator("foo", "bar"), overwrite = false, sessionHadoopConf) assert(MemoryLogStore.containsKey(path)) assertThrows[java.nio.file.FileSystemException] { store.write(path, Iterator("foo", "bar"), overwrite = false, sessionHadoopConf) } } } test("overwrite") { withTempLogDir { tempLogDir => val store = createLogStore(spark) val path = getDeltaVersionPath(tempLogDir, 0) store.write(path, Iterator("foo", "bar"), overwrite = false, sessionHadoopConf) assert(MemoryLogStore.containsKey(path)) store.write(path, Iterator("foo", "bar"), overwrite = true, sessionHadoopConf) assert(MemoryLogStore.containsKey(path)) } } test("write N fails if overwrite=false and N already exists in FileSystem " + "and N does not exist in external store") { withTempLogDir { tempLogDir => val delta0 = getDeltaVersionPath(tempLogDir, 0) val delta1_a = getDeltaVersionPath(tempLogDir, 1) val delta1_b = getDeltaVersionPath(tempLogDir, 1) val store = createLogStore(spark) store.write(delta0, Iterator("zero"), overwrite = false, sessionHadoopConf) store.write(delta1_a, Iterator("one_a"), overwrite = false, sessionHadoopConf) // Pretend that BaseExternalLogStore.getExpirationDelaySeconds() seconds have // transpired and that the external store has run TTL cleanup. MemoryLogStore.hashMap.clear(); val e = intercept[java.nio.file.FileAlreadyExistsException] { store.write(delta1_b, Iterator("one_b"), overwrite = false, sessionHadoopConf) } assert(e.getMessage.contains(delta1_b.toString)) } } test("write N fails and does not write to external store if overwrite=false and N " + "already exists in FileSystem and N already exists in external store") { withTempLogDir { tempLogDir => val delta0 = getDeltaVersionPath(tempLogDir, 0) val delta1_a = getDeltaVersionPath(tempLogDir, 1) val delta1_b = getDeltaVersionPath(tempLogDir, 1) val store = createLogStore(spark) store.write(delta0, Iterator("zero"), overwrite = false, sessionHadoopConf) store.write(delta1_a, Iterator("one_a"), overwrite = false, sessionHadoopConf) assert(MemoryLogStore.hashMap.size() == 2) val e = intercept[java.nio.file.FileAlreadyExistsException] { store.write(delta1_b, Iterator("one_b"), overwrite = false, sessionHadoopConf) } assert(e.getMessage.contains(delta1_b.toString)) assert(MemoryLogStore.hashMap.size() == 2) } } test("write N+1 fails if N doesn't exist in external store or FileSystem") { withTempLogDir { tempLogDir => val store = createLogStore(spark) val delta0 = getDeltaVersionPath(tempLogDir, 0) val delta1 = getDeltaVersionPath(tempLogDir, 1) val e = intercept[java.nio.file.FileSystemException] { store.write(delta1, Iterator("one"), overwrite = false, sessionHadoopConf) } assert(e.getMessage == s"previous commit $delta0 doesn't exist on the file system but does in the external log store") } } // scalastyle:off line.size.limit test("write N+1 fails if N is marked as complete in external store but doesn't exist in FileSystem") { // scalastyle:on line.size.limit withTempLogDir { tempLogDir => val store = createLogStore(spark) val delta0 = getDeltaVersionPath(tempLogDir, 0) val delta1 = getDeltaVersionPath(tempLogDir, 1) store.write(delta0, Iterator("one"), overwrite = false, sessionHadoopConf) delta0.getFileSystem(sessionHadoopConf).delete(delta0, true) val e = intercept[java.nio.file.FileSystemException] { store.write(delta1, Iterator("one"), overwrite = false, sessionHadoopConf) } assert(e.getMessage == s"previous commit $delta0 doesn't exist on the file system but does in the external log store") } } test("write N+1 succeeds and recovers version N if N is incomplete in external store") { withSQLConf( "fs.failing.impl" -> classOf[FailingFileSystem].getName, "fs.failing.impl.disable.cache" -> "true" ) { withTempLogDir { tempLogDir => val store = createLogStore(spark) val delta0_normal = getDeltaVersionPath(tempLogDir, 0) val delta0_fail = getFailingDeltaVersionPath(tempLogDir, 0) val delta1 = getDeltaVersionPath(tempLogDir, 1) // Create N (incomplete) in external store, with no N in FileSystem FailingFileSystem.failOnSuffix = Some(delta0_fail.getName) store.write(delta0_fail, Iterator("zero"), overwrite = false, sessionHadoopConf) assert(!delta0_fail.getFileSystem(sessionHadoopConf).exists(delta0_fail)) assert(!MemoryLogStore.get(delta0_fail).complete) // Write N + 1 and check that recovery was performed store.write(delta1, Iterator("one"), overwrite = false, sessionHadoopConf) assert(delta0_fail.getFileSystem(sessionHadoopConf).exists(delta0_fail)) assert(MemoryLogStore.get(delta0_fail).complete) assert(MemoryLogStore.get(delta1).complete) } } } test("listFrom performs recovery") { withSQLConf( "fs.failing.impl" -> classOf[FailingFileSystem].getName, "fs.failing.impl.disable.cache" -> "true" ) { withTempLogDir { tempLogDir => val store = createLogStore(spark) val delta0_normal = getDeltaVersionPath(tempLogDir, 0) val delta0_fail = getFailingDeltaVersionPath(tempLogDir, 0) // fail to write to FileSystem when we try to commit 0000.json FailingFileSystem.failOnSuffix = Some(delta0_fail.getName) // try and commit 0000.json store.write(delta0_fail, Iterator("foo", "bar"), overwrite = false, sessionHadoopConf) // check that entry was written to external store and that it doesn't exist in FileSystem val entry = MemoryLogStore.get(delta0_fail) assert(!entry.complete) assert(!delta0_fail.getFileSystem(sessionHadoopConf).exists(delta0_fail)) // Now perform a `listFrom` read, which should fix the transaction log val contents = store.read(entry.absoluteTempPath(), sessionHadoopConf).toList FailingFileSystem.failOnSuffix = None store.listFrom(delta0_normal, sessionHadoopConf) val entry2 = MemoryLogStore.get(delta0_normal) assert(entry2.complete) assert(store.read(entry2.absoluteFilePath(), sessionHadoopConf).toList == contents) } } } test("write to new Delta table but a DynamoDB entry for it already exists") { withTempLogDir { tempLogDir => val store = createLogStore(spark) // write 0000.json val path = getDeltaVersionPath(tempLogDir, 0) store.write(path, Iterator("foo"), overwrite = false, sessionHadoopConf) // delete 0000.json from FileSystem val fs = path.getFileSystem(sessionHadoopConf) fs.delete(path, false) // try and write a new 0000.json, while the external store entry still exists val e = intercept[java.nio.file.FileSystemException] { store.write(path, Iterator("bar"), overwrite = false, sessionHadoopConf) }.getMessage val tablePath = path.getParent.getParent assert(e == s"Old entries for table $tablePath still exist in the external log store") } } test("listFrom exceptions") { val store = createLogStore(spark) assertThrows[java.io.FileNotFoundException] { store.listFrom("/non-existing-path/with-parent") } } test("MemoryLogStore ignores failing scheme") { withSQLConf( "fs.failing.impl" -> classOf[FailingFileSystem].getName, "fs.failing.impl.disable.cache" -> "true" ) { withTempLogDir { tempLogDir => val store = createLogStore(spark) val delta0_normal = getDeltaVersionPath(tempLogDir, 0) val delta0_fail = getFailingDeltaVersionPath(tempLogDir, 0) store.write(delta0_fail, Iterator("zero"), overwrite = false, sessionHadoopConf) assert(MemoryLogStore.get(delta0_fail) eq MemoryLogStore.get(delta0_normal)) } } } } /////////////////////////////////// // S3DynamoDBLogStore Test Suite // /////////////////////////////////// class S3DynamoDBLogStoreSuite extends AnyFunSuite { test("getParam") { import S3DynamoDBLogStore._ val sparkPrefixKey = "spark.io.delta.storage.S3DynamoDBLogStore.ddb.tableName" val basePrefixKey = "io.delta.storage.S3DynamoDBLogStore.ddb.tableName" // Sanity check require(sparkPrefixKey == SPARK_CONF_PREFIX + "." + DDB_CLIENT_TABLE) require(basePrefixKey == BASE_CONF_PREFIX + "." + DDB_CLIENT_TABLE) // Case 1: no parameters exist, should use default assert(getParam(new Configuration(), DDB_CLIENT_TABLE, "default_table") == "default_table") // Case 2: spark-prefix param only { val hadoopConf = new Configuration() hadoopConf.set(sparkPrefixKey, "some_other_table_2") assert(getParam(hadoopConf, DDB_CLIENT_TABLE, "default_table") == "some_other_table_2") } // Case 3: base-prefix param only { val hadoopConf = new Configuration() hadoopConf.set(basePrefixKey, "some_other_table_3") assert(getParam(hadoopConf, DDB_CLIENT_TABLE, "default_table") == "some_other_table_3") } // Case 4: both params set, same value { val hadoopConf = new Configuration() hadoopConf.set(sparkPrefixKey, "some_other_table_4") hadoopConf.set(basePrefixKey, "some_other_table_4") assert(getParam(hadoopConf, DDB_CLIENT_TABLE, "default_table") == "some_other_table_4") } // Case 5: both param set, different value { val hadoopConf = new Configuration() hadoopConf.set(sparkPrefixKey, "some_other_table_5a") hadoopConf.set(basePrefixKey, "some_other_table_5b") val e = intercept[IllegalArgumentException] { getParam(hadoopConf, DDB_CLIENT_TABLE, "default_table") }.getMessage assert(e == (s"Configuration properties `$sparkPrefixKey=some_other_table_5a` and " + s"`$basePrefixKey=some_other_table_5b` have different values. Please set only one.")) } } } //////////////////////////////// // File System Helper Classes // //////////////////////////////// /** * This utility enables failure simulation on file system. * Providing a matching suffix results in an exception being * thrown that allows to test file system failure scenarios. */ class FailingFileSystem extends RawLocalFileSystem { override def getScheme: String = FailingFileSystem.scheme override def getUri: URI = FailingFileSystem.uri override def create(path: Path, overwrite: Boolean): FSDataOutputStream = { FailingFileSystem.failOnSuffix match { case Some(suffix) => if (path.toString.endsWith(suffix)) { throw new java.nio.file.FileSystemException("fail") } case None => ; } super.create(path, overwrite) } } object FailingFileSystem { private val scheme = "failing" private val uri: URI = URI.create(s"$scheme:///") var failOnSuffix: Option[String] = None } ================================================ FILE: storage-s3-dynamodb/src/test/scala/io/delta/storage/RetryableCloseableIteratorSuite.scala ================================================ package io.delta.storage import java.io.{FileNotFoundException, IOException} import scala.collection.JavaConverters._ import io.delta.storage.utils.ThrowingSupplier import org.apache.hadoop.fs.s3a.RemoteFileChangedException import org.scalatest.funsuite.AnyFunSuite class RetryableCloseableIteratorSuite extends AnyFunSuite { private def getIter( range: Range, throwAtIndex: Option[Int] = None): CloseableIterator[String] = new CloseableIterator[String] { var index = 0 val impl = range.iterator.asJava override def close(): Unit = { } override def hasNext: Boolean = { impl.hasNext } override def next(): String = { if (throwAtIndex.contains(index)) { throw new RemoteFileChangedException(s"path -> index $index", "operation", "msg"); } index = index + 1 impl.next().toString } } /** * Fails at indices 25, 50, 75, 110. * * Provide a suitable input range to get the # of failures you want. e.g. range 0 to 100 will fail * 3 times. */ def getFailingIterSupplier( range: Range, failIndices: Seq[Int] = Seq.empty): ThrowingSupplier[CloseableIterator[String], IOException] = new ThrowingSupplier[CloseableIterator[String], IOException] { var numGetCalls = 0 override def get(): CloseableIterator[String] = { if (numGetCalls < failIndices.length) { val result = getIter(range, Some(failIndices(numGetCalls))) numGetCalls = numGetCalls + 1 result } else { getIter(range) } } } test("simple case - internally keeps track of the correct index") { val testIter = new RetryableCloseableIterator(() => getIter(0 to 100)) assert(testIter.getLastSuccessfullIndex == -1) for (i <- 0 to 100) { val elem = testIter.next() assert(elem.toInt == i) assert(testIter.getLastSuccessfullIndex == i) } assert(!testIter.hasNext) // this would be index 101 } test("complex case - replays underlying iter back to correct index after error") { // Here, we just do the simplest verification val testIter1 = new RetryableCloseableIterator( getFailingIterSupplier(0 to 100, Seq(25, 50, 75))) // this asserts the size, order, and elements of the testIter1 assert(testIter1.asScala.toList.map(_.toInt) == (0 to 100).toList) // Here, we do more complex verification val testIter2 = new RetryableCloseableIterator( getFailingIterSupplier(0 to 100, Seq(25, 50, 75))) for (_ <- 0 to 24) { testIter2.next() } assert(testIter2.getLastSuccessfullIndex == 24) assert(testIter2.getNumRetries == 0) assert(testIter2.next().toInt == 25) // this will fail once, and then re-scan assert(testIter2.getLastSuccessfullIndex == 25) assert(testIter2.getNumRetries == 1) for (_ <- 26 to 49) { testIter2.next() } assert(testIter2.getLastSuccessfullIndex == 49) assert(testIter2.getNumRetries == 1) assert(testIter2.next().toInt == 50) // this will fail once, and then re-scan assert(testIter2.getLastSuccessfullIndex == 50) assert(testIter2.getNumRetries == 2) for (_ <- 51 to 74) { testIter2.next() } assert(testIter2.getLastSuccessfullIndex == 74) assert(testIter2.getNumRetries == 2) assert(testIter2.next().toInt == 75) // this will fail once, and then re-scan assert(testIter2.getLastSuccessfullIndex == 75) assert(testIter2.getNumRetries == 3) for (_ <- 76 to 100) { testIter2.next() } assert(testIter2.getLastSuccessfullIndex == 100) assert(!testIter2.hasNext) } test("handles exceptions while retrying") { // Iterates normally until index 50 (return [0, 49] successfully). Then fails. // Tries to replay, but fails at 30 // Tries to replay again, but fails at 20 // Successfully replays to 49, starts returning results from index 50 (inclusive) again val testIter1 = new RetryableCloseableIterator(getFailingIterSupplier(0 to 100, Seq(50, 30, 20))) assert(testIter1.asScala.toList.map(_.toInt) == (0 to 100).toList) // Iterates normally until index 50 (return [0, 49] successfully). Then fails. // Successfully replayed to 49, starts returning results from index 50 (inclusive) // Fails at index 50 (returned [50, 69]). Tries to replay, but fails at 5 // Successfully replays until 69, then normally returns results from 70 val testIter2 = new RetryableCloseableIterator(getFailingIterSupplier(0 to 100, Seq(50, 70, 5))) assert(testIter2.asScala.toList.map(_.toInt) == (0 to 100).toList) } test("throws after maxRetries exceptions") { val testIter = new RetryableCloseableIterator(getFailingIterSupplier(0 to 100, Seq(20, 49, 60, 80))) for (i <- 0 to 79) { assert(testIter.next().toInt == i) } assert(testIter.getNumRetries == 3) val ex = intercept[RuntimeException] { testIter.next() } assert(ex.getCause.isInstanceOf[RemoteFileChangedException]) } test("can specify maxRetries") { val testIter1 = new RetryableCloseableIterator( getFailingIterSupplier(0 to 100, Seq(5, 10, 15, 20, 25, 30, 35, 40, 45, 50)), 10 // maxRetries ) assert(testIter1.asScala.toList.map(_.toInt) == (0 to 100).toList) val testIter2 = new RetryableCloseableIterator( getFailingIterSupplier(0 to 100, Seq(5, 10, 15, 20, 25, 30)), 5 // maxRetries ) for (i <- 0 to 29) { assert(testIter2.next().toInt == i) } assert(testIter2.getNumRetries == 5) val ex = intercept[RuntimeException] { testIter2.next() } assert(ex.getCause.isInstanceOf[RemoteFileChangedException]) } test("retried iterator doesn't have enough data (underlying data changed!)") { val testIter = new RetryableCloseableIterator( new ThrowingSupplier[CloseableIterator[String], IOException] { var getCount = 0 override def get(): CloseableIterator[String] = getCount match { case 0 => getCount = getCount + 1 getIter(0 to 100, Some(50)) // try to iterate 0->100, fail at 50 case 1 => getCount = getCount + 1 getIter(0 to 30) // try to replay 0 to 50, but no elements after 30! } } ) for (_ <- 0 to 49) { testIter.next() } val e = intercept[IllegalStateException] { testIter.next() } assert(e.getMessage.contains("A retried iterator doesn't have enough data")) } test("after replaying the iter, hasNext is false") { val testIter = new RetryableCloseableIterator( new ThrowingSupplier[CloseableIterator[String], IOException] { var getCount = 0 override def get(): CloseableIterator[String] = getCount match { case 0 => getCount = getCount + 1 getIter(0 to 100, Some(50)) // try to iterate 0->100, fail at 50 case 1 => getCount = getCount + 1 // when we failed at index 50 above, the lastSuccessfulIndex was 49. here, we can // replay back to index 49, but the `hasNext` call will be false! getIter(0 to 49) } } ) for (_ <- 0 to 49) { testIter.next() } assert(testIter.getLastSuccessfullIndex == 49) val e = intercept[IllegalStateException] { testIter.next() } assert(e.getMessage.contains("A retried iterator doesn't have enough data (hasNext=false, " + "lastSuccessfullIndex=49)")) } test("throws FileNotFoundException (i.e. not UncheckedIOException) if file not found") { intercept[FileNotFoundException] { new RetryableCloseableIterator(() => { throw new FileNotFoundException() }) } } } ================================================ FILE: storage-s3-dynamodb/src/test/scala/io/delta/storage/utils/ReflectionsUtilsSuite.scala ================================================ /* * Copyright (2021) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.delta.storage.utils import com.amazonaws.auth.EnvironmentVariableCredentialsProvider import io.delta.storage.utils.ReflectionsUtilsSuiteHelper.TestOnlyAWSCredentialsProviderWithHadoopConf import org.apache.hadoop.conf.Configuration import org.scalatest.funsuite.AnyFunSuite class ReflectionsUtilsSuite extends AnyFunSuite { private val emptyHadoopConf = new Configuration() test("support AWS credentials provider with hadoop Configuration as constructor parameter") { val awsProvider = ReflectionUtils.createAwsCredentialsProvider( "io.delta.storage.utils.ReflectionsUtilsSuiteHelper" + "$TestOnlyAWSCredentialsProviderWithHadoopConf", emptyHadoopConf ) assert( awsProvider.isInstanceOf[TestOnlyAWSCredentialsProviderWithHadoopConf] ) } test("support AWS credentials provider with empty constructor(default from aws lib)") { val awsProvider = ReflectionUtils.createAwsCredentialsProvider( classOf[EnvironmentVariableCredentialsProvider].getCanonicalName, emptyHadoopConf ) assert(awsProvider.isInstanceOf[EnvironmentVariableCredentialsProvider]) } test("do not support AWS credentials provider with unexpected constructors parameters") { assertThrows[NoSuchMethodException] { ReflectionUtils.createAwsCredentialsProvider( "io.delta.storage.utils.ReflectionsUtilsSuiteHelper" + "$TestOnlyAWSCredentialsProviderWithUnexpectedConstructor", emptyHadoopConf ) } } } ================================================ FILE: testDeltaIcebergJar/src/test/scala/JarSuite.scala ================================================ /* * Copyright (2023-present) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import java.io.File import java.net.JarURLConnection import java.util.jar.JarFile import scala.collection.JavaConverters._ import org.scalatest.funsuite.AnyFunSuite class JarSuite extends AnyFunSuite { val allowedClassPrefixes = Seq( // e.g. shadedForDelta/org/apache/iceberg/BaseTable.class "shadedForDelta/", // e.g. scala/collection/compat/immutable/ArraySeq.class // e.g. scala/jdk/CollectionConverters.class "scala/", // e.g. org/apache/spark/sql/delta/icebergShaded/IcebergTransactionUtils.class "org/apache/spark/sql/delta/icebergShaded/", // Server-side planning support "org/apache/spark/sql/delta/serverSidePlanning/", // We explicitly include all the /delta/commands/convert classes we want, to ensure we don't // accidentally pull in some from delta-spark package. "org/apache/spark/sql/delta/commands/convert/IcebergFileManifest", "org/apache/spark/sql/delta/commands/convert/IcebergSchemaUtils", "org/apache/spark/sql/delta/commands/convert/IcebergTable", // e.g. org/apache/iceberg/transforms/IcebergPartitionUtil.class "com/github/benmanes/caffeine/", "org/apache/avro/" ) test("audit files in assembly jar") { // Step 1: load the jar (and make sure it exists) // scalastyle:off classforname val classUrl = Class.forName("org.apache.spark.sql.delta.icebergShaded.IcebergConverter").getResource("IcebergConverter.class") // scalastyle:on classforname assert(classUrl != null, "Could not find delta-iceberg jar") val connection = classUrl.openConnection().asInstanceOf[JarURLConnection] val url = connection.getJarFileURL val jarFile = new JarFile(new File(url.toURI)) // Step 2: Verify the JAR has the classes we want it to have try { val jarClasses = jarFile .entries() .asScala .filter(!_.isDirectory) .map(_.toString) .filter(_.endsWith(".class")) // let's ignore any .properties or META-INF files for now .toSet // 2.1: Verify there are no prohibited classes (e.g. io/delta/storage/...) // // You can test this code path by commenting out the "io/delta" match case of the // assemblyMergeStrategy config in build.sbt. val prohibitedJarClasses = jarClasses .filter { clazz => !allowedClassPrefixes.exists(prefix => clazz.startsWith(prefix)) } if (prohibitedJarClasses.nonEmpty) { throw new Exception( s"Prohibited jar class(es) found:\n- ${prohibitedJarClasses.mkString("\n- ")}" ) } // 2.2: Verify that, for each allowed class prefix, we actually loaded a class for it (instead // of, say, loading an empty jar). // // You can test this code path by adding the following code snippet to the delta-iceberg // assemblyMergeStrategy config in build.sbt: // case PathList("shadedForDelta", xs @ _*) => MergeStrategy.discard // Map of prefix -> # classes with that prefix val allowedClassesCounts = scala.collection.mutable.Map( allowedClassPrefixes.map(prefix => (prefix, 0)) : _* ) jarClasses.foreach { clazz => allowedClassPrefixes.foreach { prefix => if (clazz.startsWith(prefix)) { allowedClassesCounts(prefix) += 1 } } } val missingClasses = allowedClassesCounts.filter(_._2 == 0).keys if (missingClasses.nonEmpty) { throw new Exception( s"No classes found for the following prefix(es):\n- ${missingClasses.mkString("\n- ")}" ) } } finally { jarFile.close() } } } ================================================ FILE: version.sbt ================================================ ThisBuild / version := "4.1.0-SNAPSHOT"